From 3a67c0979cd45011038f54503f6a4f70fe6a998a Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Thu, 5 Mar 2026 19:39:14 +0100 Subject: [PATCH] feat: update workspace paths and enhance gitignore - Updated stablediffusion crate path from "../stable-diffusion-burn" to "./crates/stable-diffusion-burn" for proper workspace resolution - Enhanced .gitignore to include generated model files (.mpk, .pt, .bin, .safetensors, .ckpt) and user_data directory - Added Cargo.lock to gitignore with appropriate comment - Reorganized IDE files section in gitignore for better clarity - Added newline at end of file for proper formatting --- .gitignore | 14 +- Cargo.toml | 2 +- crates/stable-diffusion-burn/Cargo.toml | 28 + crates/stable-diffusion-burn/LICENSE | 21 + crates/stable-diffusion-burn/README.md | 75 + .../bpe_simple_vocab_16e6.txt | 262145 +++++++++++++++ .../burn-crates/burn-autodiff/Cargo.toml | 45 + .../burn-crates/burn-autodiff/LICENSE-APACHE | 1 + .../burn-crates/burn-autodiff/LICENSE-MIT | 1 + .../burn-crates/burn-autodiff/README.md | 8 + .../burn-crates/burn-autodiff/src/backend.rs | 138 + .../burn-autodiff/src/checkpoint/base.rs | 82 + .../burn-autodiff/src/checkpoint/builder.rs | 304 + .../burn-autodiff/src/checkpoint/mod.rs | 9 + .../src/checkpoint/retro_forward.rs | 116 + .../burn-autodiff/src/checkpoint/state.rs | 144 + .../burn-autodiff/src/checkpoint/strategy.rs | 102 + .../burn-crates/burn-autodiff/src/grads.rs | 85 + .../burn-autodiff/src/graph/base.rs | 17 + .../burn-autodiff/src/graph/mod.rs | 9 + .../burn-autodiff/src/graph/node.rs | 87 + .../burn-autodiff/src/graph/requirement.rs | 38 + .../burn-autodiff/src/graph/traversal.rs | 74 + .../burn-crates/burn-autodiff/src/lib.rs | 43 + .../burn-autodiff/src/ops/activation.rs | 167 + .../burn-autodiff/src/ops/backward.rs | 88 + .../burn-crates/burn-autodiff/src/ops/base.rs | 317 + .../burn-autodiff/src/ops/bool_tensor.rs | 161 + .../burn-autodiff/src/ops/int_tensor.rs | 406 + .../burn-autodiff/src/ops/maxmin.rs | 27 + .../burn-crates/burn-autodiff/src/ops/mod.rs | 15 + .../burn-autodiff/src/ops/module.rs | 1828 + .../burn-autodiff/src/ops/qtensor.rs | 106 + .../burn-crates/burn-autodiff/src/ops/sort.rs | 27 + .../burn-autodiff/src/ops/tensor.rs | 3480 + .../burn-autodiff/src/ops/transaction.rs | 24 + .../burn-autodiff/src/runtime/client.rs | 18 + .../burn-autodiff/src/runtime/graph.rs | 335 + .../src/runtime/memory_management.rs | 294 + .../burn-autodiff/src/runtime/mod.rs | 6 + .../burn-autodiff/src/runtime/server.rs | 143 + .../burn-crates/burn-autodiff/src/tensor.rs | 189 + .../burn-crates/burn-autodiff/src/utils.rs | 25 + .../burn-backend-tests/.cargo/config.toml | 10 + .../burn-crates/burn-backend-tests/Cargo.toml | 120 + .../burn-crates/burn-backend-tests/README.md | 111 + .../burn-crates/burn-backend-tests/src/lib.rs | 22 + .../burn-backend-tests/tests/autodiff.rs | 20 + .../burn-backend-tests/tests/autodiff/abs.rs | 58 + .../tests/autodiff/adaptive_avgpool1d.rs | 50 + .../tests/autodiff/adaptive_avgpool2d.rs | 96 + .../burn-backend-tests/tests/autodiff/add.rs | 74 + .../tests/autodiff/aggregation.rs | 138 + .../tests/autodiff/avgpool1d.rs | 102 + .../tests/autodiff/avgpool2d.rs | 129 + .../tests/autodiff/backward.rs | 24 + .../tests/autodiff/bridge.rs | 27 + .../tests/autodiff/broadcast.rs | 56 + .../burn-backend-tests/tests/autodiff/cast.rs | 28 + .../burn-backend-tests/tests/autodiff/cat.rs | 110 + .../burn-backend-tests/tests/autodiff/ceil.rs | 21 + .../tests/autodiff/checkpoint.rs | 215 + .../tests/autodiff/complex.rs | 81 + .../tests/autodiff/conv1d.rs | 277 + .../tests/autodiff/conv2d.rs | 962 + .../tests/autodiff/conv3d.rs | 690 + .../tests/autodiff/conv_transpose1d.rs | 292 + .../tests/autodiff/conv_transpose2d.rs | 706 + .../tests/autodiff/conv_transpose3d.rs | 711 + .../tests/autodiff/cross.rs | 103 + .../tests/autodiff/cross_entropy.rs | 33 + .../tests/autodiff/cummax.rs | 117 + .../tests/autodiff/cummin.rs | 117 + .../tests/autodiff/cumprod.rs | 132 + .../tests/autodiff/cumsum.rs | 89 + .../tests/autodiff/deform_conv2d.rs | 1804 + .../burn-backend-tests/tests/autodiff/div.rs | 105 + .../burn-backend-tests/tests/autodiff/erf.rs | 29 + .../burn-backend-tests/tests/autodiff/exp.rs | 29 + .../tests/autodiff/expand.rs | 39 + .../burn-backend-tests/tests/autodiff/flip.rs | 29 + .../tests/autodiff/floor.rs | 21 + .../tests/autodiff/gather_scatter.rs | 99 + .../burn-backend-tests/tests/autodiff/gelu.rs | 29 + .../tests/autodiff/gradients.rs | 24 + .../burn-backend-tests/tests/autodiff/log.rs | 30 + .../tests/autodiff/log1p.rs | 28 + .../tests/autodiff/log_sigmoid.rs | 19 + .../burn-backend-tests/tests/autodiff/mask.rs | 65 + .../tests/autodiff/matmul.rs | 83 + .../tests/autodiff/maxmin.rs | 82 + .../tests/autodiff/maxpool1d.rs | 134 + .../tests/autodiff/maxpool2d.rs | 271 + .../tests/autodiff/memory_management.rs | 290 + .../burn-backend-tests/tests/autodiff/mod.rs | 74 + .../burn-backend-tests/tests/autodiff/mul.rs | 68 + .../tests/autodiff/multithread.rs | 88 + .../tests/autodiff/nearest_interpolate.rs | 97 + .../burn-backend-tests/tests/autodiff/neg.rs | 26 + .../tests/autodiff/nonzero.rs | 41 + .../tests/autodiff/permute.rs | 29 + .../burn-backend-tests/tests/autodiff/pow.rs | 93 + .../tests/autodiff/recip.rs | 22 + .../burn-backend-tests/tests/autodiff/relu.rs | 27 + .../tests/autodiff/remainder.rs | 41 + .../tests/autodiff/repeat_dim.rs | 44 + .../tests/autodiff/reshape.rs | 26 + .../tests/autodiff/round.rs | 20 + .../tests/autodiff/select.rs | 89 + .../tests/autodiff/sigmoid.rs | 35 + .../burn-backend-tests/tests/autodiff/sign.rs | 42 + .../tests/autodiff/slice.rs | 67 + .../tests/autodiff/slice_assign.rs | 163 + .../tests/autodiff/softmax.rs | 90 + .../burn-backend-tests/tests/autodiff/sort.rs | 82 + .../burn-backend-tests/tests/autodiff/sqrt.rs | 31 + .../burn-backend-tests/tests/autodiff/sub.rs | 73 + .../tests/autodiff/transpose.rs | 60 + .../burn-backend-tests/tests/autodiff/trig.rs | 371 + .../tests/autodiff/unfold.rs | 18 + .../tests/common/autodiff.rs | 35 + .../tests/common/backend.rs | 47 + .../burn-backend-tests/tests/common/tensor.rs | 39 + .../burn-backend-tests/tests/cubecl.rs | 17 + .../tests/cubecl/avg_pool2d.rs | 96 + .../tests/cubecl/bernoulli.rs | 48 + .../burn-backend-tests/tests/cubecl/cast.rs | 45 + .../burn-backend-tests/tests/cubecl/cat.rs | 42 + .../burn-backend-tests/tests/cubecl/clamp.rs | 20 + .../tests/cubecl/contiguous.rs | 40 + .../burn-backend-tests/tests/cubecl/conv2d.rs | 78 + .../burn-backend-tests/tests/cubecl/conv3d.rs | 27 + .../tests/cubecl/conv_transpose2d.rs | 48 + .../tests/cubecl/conv_transpose3d.rs | 51 + .../burn-backend-tests/tests/cubecl/cross.rs | 159 + .../burn-backend-tests/tests/cubecl/gather.rs | 44 + .../tests/cubecl/mask_fill.rs | 64 + .../tests/cubecl/mask_where.rs | 80 + .../tests/cubecl/max_pool2d.rs | 52 + .../tests/cubecl/max_pool2d_backward.rs | 67 + .../burn-backend-tests/tests/cubecl/mod.rs | 30 + .../burn-backend-tests/tests/cubecl/normal.rs | 36 + .../tests/cubecl/quantization.rs | 240 + .../burn-backend-tests/tests/cubecl/reduce.rs | 135 + .../tests/cubecl/repeat_dim.rs | 71 + .../tests/cubecl/scatter.rs | 66 + .../burn-backend-tests/tests/cubecl/select.rs | 21 + .../tests/cubecl/select_assign.rs | 54 + .../burn-backend-tests/tests/cubecl/slice.rs | 19 + .../tests/cubecl/slice_assign.rs | 21 + .../burn-backend-tests/tests/cubecl/unary.rs | 24 + .../tests/cubecl/uniform.rs | 113 + .../burn-backend-tests/tests/fused_ops/mod.rs | 1 + .../tests/fused_ops/reduce_broadcasted.rs | 158 + .../burn-backend-tests/tests/fusion.rs | 41 + .../burn-backend-tests/tests/tensor.rs | 15 + .../tests/tensor/bool/mod.rs | 3 + .../tests/tensor/bool/ops/all.rs | 34 + .../tests/tensor/bool/ops/any.rs | 23 + .../tests/tensor/bool/ops/argwhere_nonzero.rs | 107 + .../tests/tensor/bool/ops/cat.rs | 29 + .../tests/tensor/bool/ops/comparison.rs | 71 + .../tests/tensor/bool/ops/create_like.rs | 34 + .../tests/tensor/bool/ops/expand.rs | 16 + .../tests/tensor/bool/ops/flip.rs | 33 + .../tests/tensor/bool/ops/full.rs | 16 + .../tests/tensor/bool/ops/gather_scatter.rs | 29 + .../tests/tensor/bool/ops/init.rs | 31 + .../tests/tensor/bool/ops/logical.rs | 49 + .../tests/tensor/bool/ops/mask.rs | 30 + .../tests/tensor/bool/ops/mod.rs | 26 + .../tests/tensor/bool/ops/movedim.rs | 56 + .../tests/tensor/bool/ops/permute.rs | 31 + .../tests/tensor/bool/ops/repeat.rs | 64 + .../tests/tensor/bool/ops/repeat_dim.rs | 34 + .../tests/tensor/bool/ops/reshape.rs | 13 + .../tests/tensor/bool/ops/select.rs | 241 + .../tests/tensor/bool/ops/stack.rs | 15 + .../tests/tensor/bool/ops/take.rs | 41 + .../tests/tensor/bool/ops/transpose.rs | 41 + .../tests/tensor/bool/ops/tri_mask.rs | 94 + .../tests/tensor/bool/ops/unfold.rs | 36 + .../tests/tensor/clone_invariance.rs | 761 + .../tests/tensor/float/activation/celu.rs | 34 + .../tests/tensor/float/activation/elu.rs | 32 + .../tests/tensor/float/activation/gelu.rs | 20 + .../tests/tensor/float/activation/glu.rs | 28 + .../tensor/float/activation/hard_sigmoid.rs | 27 + .../tensor/float/activation/leaky_relu.rs | 16 + .../tensor/float/activation/log_sigmoid.rs | 37 + .../tests/tensor/float/activation/mish.rs | 21 + .../tests/tensor/float/activation/mod.rs | 22 + .../tests/tensor/float/activation/prelu.rs | 101 + .../tensor/float/activation/quiet_softmax.rs | 15 + .../tests/tensor/float/activation/relu.rs | 13 + .../tests/tensor/float/activation/selu.rs | 37 + .../tests/tensor/float/activation/sigmoid.rs | 27 + .../tests/tensor/float/activation/silu.rs | 15 + .../tests/tensor/float/activation/softmax.rs | 15 + .../tests/tensor/float/activation/softmin.rs | 15 + .../tests/tensor/float/activation/softplus.rs | 28 + .../tests/tensor/float/activation/softsign.rs | 27 + .../float/activation/tanh_activation.rs | 15 + .../float/activation/thresholded_relu.rs | 26 + .../tests/tensor/float/grid/affine_grid.rs | 82 + .../tests/tensor/float/grid/meshgrid.rs | 153 + .../tests/tensor/float/grid/mod.rs | 4 + .../tensor/float/linalg/cosine_similarity.rs | 100 + .../tests/tensor/float/linalg/diag.rs | 259 + .../tensor/float/linalg/lu_decomposition.rs | 113 + .../tests/tensor/float/linalg/matvec.rs | 102 + .../tests/tensor/float/linalg/mod.rs | 9 + .../tests/tensor/float/linalg/outer.rs | 262 + .../tests/tensor/float/linalg/trace.rs | 114 + .../tests/tensor/float/linalg/vector_norm.rs | 241 + .../tests/tensor/float/mod.rs | 13 + .../tensor/float/module/adaptive_avgpool1d.rs | 70 + .../tensor/float/module/adaptive_avgpool2d.rs | 101 + .../tests/tensor/float/module/attention.rs | 372 + .../tests/tensor/float/module/avgpool1d.rs | 168 + .../tests/tensor/float/module/avgpool2d.rs | 220 + .../float/module/bicubic_interpolate.rs | 196 + .../float/module/bilinear_interpolate.rs | 270 + .../tests/tensor/float/module/conv1d.rs | 138 + .../tests/tensor/float/module/conv2d.rs | 652 + .../tests/tensor/float/module/conv3d.rs | 299 + .../tensor/float/module/conv_transpose1d.rs | 145 + .../tensor/float/module/conv_transpose2d.rs | 361 + .../tensor/float/module/conv_transpose3d.rs | 749 + .../tensor/float/module/deform_conv2d.rs | 438 + .../tests/tensor/float/module/forward.rs | 18 + .../tests/tensor/float/module/linear.rs | 59 + .../tests/tensor/float/module/maxpool1d.rs | 155 + .../tests/tensor/float/module/maxpool2d.rs | 523 + .../tests/tensor/float/module/mod.rs | 22 + .../float/module/nearest_interpolate.rs | 127 + .../tests/tensor/float/module/unfold4d.rs | 132 + .../tests/tensor/float/ops/abs.rs | 13 + .../tests/tensor/float/ops/add.rs | 118 + .../tests/tensor/float/ops/aggregation.rs | 460 + .../tests/tensor/float/ops/all.rs | 18 + .../tests/tensor/float/ops/any.rs | 58 + .../tests/tensor/float/ops/arg.rs | 46 + .../tests/tensor/float/ops/cast.rs | 51 + .../tests/tensor/float/ops/cat.rs | 148 + .../tests/tensor/float/ops/ceil.rs | 16 + .../tests/tensor/float/ops/chunk.rs | 85 + .../tests/tensor/float/ops/clamp.rs | 81 + .../tests/tensor/float/ops/close.rs | 40 + .../tests/tensor/float/ops/comparison.rs | 303 + .../tests/tensor/float/ops/create_like.rs | 57 + .../tests/tensor/float/ops/cross.rs | 101 + .../tests/tensor/float/ops/cumulative.rs | 152 + .../tests/tensor/float/ops/div.rs | 49 + .../tests/tensor/float/ops/dot.rs | 35 + .../tests/tensor/float/ops/erf.rs | 34 + .../tests/tensor/float/ops/exp.rs | 16 + .../tests/tensor/float/ops/expand.rs | 80 + .../tests/tensor/float/ops/finite.rs | 23 + .../tests/tensor/float/ops/flatten.rs | 64 + .../tests/tensor/float/ops/flip.rs | 48 + .../tests/tensor/float/ops/floor.rs | 16 + .../tests/tensor/float/ops/fmod.rs | 295 + .../tests/tensor/float/ops/full.rs | 28 + .../tests/tensor/float/ops/gather_scatter.rs | 173 + .../tests/tensor/float/ops/grid_sample.rs | 126 + .../tests/tensor/float/ops/inf.rs | 21 + .../tests/tensor/float/ops/init.rs | 62 + .../tests/tensor/float/ops/iter_dim.rs | 246 + .../tests/tensor/float/ops/log.rs | 20 + .../tests/tensor/float/ops/log1p.rs | 20 + .../tests/tensor/float/ops/mask.rs | 165 + .../tests/tensor/float/ops/matmul.rs | 275 + .../tests/tensor/float/ops/maxmin.rs | 269 + .../tests/tensor/float/ops/mod.rs | 76 + .../tests/tensor/float/ops/movedim.rs | 123 + .../tests/tensor/float/ops/mul.rs | 54 + .../tests/tensor/float/ops/nan.rs | 24 + .../tests/tensor/float/ops/narrow.rs | 98 + .../tests/tensor/float/ops/neg.rs | 21 + .../tests/tensor/float/ops/one_hot.rs | 71 + .../tests/tensor/float/ops/padding.rs | 470 + .../tests/tensor/float/ops/permute.rs | 58 + .../tests/tensor/float/ops/powf.rs | 105 + .../tests/tensor/float/ops/powf_scalar.rs | 55 + .../tests/tensor/float/ops/prod.rs | 48 + .../tests/tensor/float/ops/random.rs | 53 + .../tests/tensor/float/ops/recip.rs | 16 + .../tests/tensor/float/ops/remainder.rs | 241 + .../tests/tensor/float/ops/repeat.rs | 108 + .../tests/tensor/float/ops/repeat_dim.rs | 166 + .../tests/tensor/float/ops/reshape.rs | 90 + .../tests/tensor/float/ops/round.rs | 29 + .../tests/tensor/float/ops/select.rs | 235 + .../tests/tensor/float/ops/sign.rs | 12 + .../tests/tensor/float/ops/slice.rs | 591 + .../tests/tensor/float/ops/slice_assign.rs | 366 + .../tests/tensor/float/ops/sort_argsort.rs | 216 + .../tests/tensor/float/ops/split.rs | 206 + .../tests/tensor/float/ops/sqrt.rs | 18 + .../tests/tensor/float/ops/square.rs | 17 + .../tests/tensor/float/ops/squeeze.rs | 207 + .../tests/tensor/float/ops/stack.rs | 69 + .../tests/tensor/float/ops/sub.rs | 42 + .../tests/tensor/float/ops/take.rs | 206 + .../tests/tensor/float/ops/topk.rs | 21 + .../tests/tensor/float/ops/transaction.rs | 31 + .../tests/tensor/float/ops/transpose.rs | 116 + .../tests/tensor/float/ops/tri.rs | 21 + .../tests/tensor/float/ops/trig.rs | 242 + .../tests/tensor/float/ops/trunc.rs | 67 + .../tests/tensor/float/ops/unfold.rs | 35 + .../tests/tensor/float/primitive.rs | 16 + .../tensor/float/quantization/calibration.rs | 51 + .../tests/tensor/float/quantization/data.rs | 39 + .../tests/tensor/float/quantization/mod.rs | 48 + .../float/quantization/ops/extended/abs.rs | 19 + .../float/quantization/ops/extended/add.rs | 106 + .../quantization/ops/extended/aggregation.rs | 166 + .../float/quantization/ops/extended/all.rs | 24 + .../float/quantization/ops/extended/any.rs | 25 + .../float/quantization/ops/extended/arg.rs | 47 + .../float/quantization/ops/extended/cat.rs | 65 + .../float/quantization/ops/extended/ceil.rs | 17 + .../float/quantization/ops/extended/chunk.rs | 98 + .../float/quantization/ops/extended/clamp.rs | 49 + .../float/quantization/ops/extended/cos.rs | 17 + .../float/quantization/ops/extended/cosh.rs | 17 + .../float/quantization/ops/extended/div.rs | 50 + .../float/quantization/ops/extended/erf.rs | 33 + .../float/quantization/ops/extended/exp.rs | 17 + .../float/quantization/ops/extended/expand.rs | 112 + .../float/quantization/ops/extended/flip.rs | 39 + .../float/quantization/ops/extended/floor.rs | 17 + .../ops/extended/gather_scatter.rs | 198 + .../float/quantization/ops/extended/log.rs | 20 + .../float/quantization/ops/extended/log1p.rs | 20 + .../ops/extended/map_comparison.rs | 157 + .../float/quantization/ops/extended/mask.rs | 39 + .../float/quantization/ops/extended/maxmin.rs | 149 + .../float/quantization/ops/extended/mod.rs | 50 + .../float/quantization/ops/extended/mul.rs | 60 + .../float/quantization/ops/extended/narrow.rs | 58 + .../float/quantization/ops/extended/neg.rs | 19 + .../quantization/ops/extended/permute.rs | 66 + .../float/quantization/ops/extended/powf.rs | 60 + .../quantization/ops/extended/powf_scalar.rs | 56 + .../float/quantization/ops/extended/recip.rs | 17 + .../quantization/ops/extended/remainder.rs | 208 + .../quantization/ops/extended/repeat_dim.rs | 42 + .../quantization/ops/extended/reshape.rs | 79 + .../float/quantization/ops/extended/round.rs | 32 + .../float/quantization/ops/extended/select.rs | 120 + .../float/quantization/ops/extended/sin.rs | 17 + .../float/quantization/ops/extended/sinh.rs | 17 + .../float/quantization/ops/extended/slice.rs | 229 + .../quantization/ops/extended/sort_argsort.rs | 225 + .../float/quantization/ops/extended/split.rs | 151 + .../float/quantization/ops/extended/sqrt.rs | 18 + .../float/quantization/ops/extended/stack.rs | 69 + .../float/quantization/ops/extended/sub.rs | 46 + .../float/quantization/ops/extended/tan.rs | 17 + .../float/quantization/ops/extended/tanh.rs | 17 + .../float/quantization/ops/extended/topk.rs | 72 + .../quantization/ops/extended/transpose.rs | 43 + .../tensor/float/quantization/ops/matmul.rs | 219 + .../tensor/float/quantization/ops/mod.rs | 8 + .../tensor/float/quantization/ops/quantize.rs | 209 + .../tests/tensor/float/quantization/scheme.rs | 71 + .../tests/tensor/float/stats/cov.rs | 66 + .../tests/tensor/float/stats/display.rs | 338 + .../tests/tensor/float/stats/eye.rs | 17 + .../tests/tensor/float/stats/median.rs | 92 + .../tests/tensor/float/stats/mod.rs | 7 + .../tests/tensor/float/stats/var.rs | 69 + .../tests/tensor/int/mod.rs | 4 + .../tests/tensor/int/ops/abs.rs | 24 + .../tests/tensor/int/ops/add.rs | 66 + .../tests/tensor/int/ops/aggregation.rs | 81 + .../tests/tensor/int/ops/all.rs | 23 + .../tests/tensor/int/ops/any.rs | 23 + .../tests/tensor/int/ops/arange.rs | 32 + .../tests/tensor/int/ops/arange_step.rs | 45 + .../tests/tensor/int/ops/arg.rs | 24 + .../tests/tensor/int/ops/bitwise.rs | 173 + .../tests/tensor/int/ops/cartesian_grid.rs | 21 + .../tests/tensor/int/ops/cast.rs | 30 + .../tests/tensor/int/ops/cat.rs | 28 + .../tests/tensor/int/ops/chunk.rs | 16 + .../tests/tensor/int/ops/comparison.rs | 267 + .../tests/tensor/int/ops/create_like.rs | 22 + .../tests/tensor/int/ops/cumulative.rs | 90 + .../tests/tensor/int/ops/div.rs | 45 + .../tests/tensor/int/ops/expand.rs | 41 + .../tests/tensor/int/ops/flip.rs | 22 + .../tests/tensor/int/ops/full.rs | 11 + .../tests/tensor/int/ops/gather_scatter.rs | 69 + .../tests/tensor/int/ops/init.rs | 31 + .../tests/tensor/int/ops/mask.rs | 56 + .../tests/tensor/int/ops/matmul.rs | 203 + .../tests/tensor/int/ops/mod.rs | 47 + .../tests/tensor/int/ops/movedim.rs | 52 + .../tests/tensor/int/ops/mul.rs | 42 + .../tests/tensor/int/ops/one_hot.rs | 59 + .../tests/tensor/int/ops/permute.rs | 49 + .../tests/tensor/int/ops/random.rs | 18 + .../tests/tensor/int/ops/remainder.rs | 27 + .../tests/tensor/int/ops/repeat.rs | 100 + .../tests/tensor/int/ops/repeat_dim.rs | 46 + .../tests/tensor/int/ops/reshape.rs | 179 + .../tests/tensor/int/ops/roll.rs | 108 + .../tests/tensor/int/ops/select.rs | 38 + .../tests/tensor/int/ops/sign.rs | 12 + .../tests/tensor/int/ops/slice.rs | 29 + .../tests/tensor/int/ops/slice_assign.rs | 55 + .../tests/tensor/int/ops/sort_argsort.rs | 153 + .../tests/tensor/int/ops/stack.rs | 33 + .../tests/tensor/int/ops/sub.rs | 42 + .../tests/tensor/int/ops/take.rs | 31 + .../tests/tensor/int/ops/topk.rs | 59 + .../tests/tensor/int/ops/transpose.rs | 28 + .../tests/tensor/int/ops/tri.rs | 85 + .../tests/tensor/int/ops/unfold.rs | 39 + .../tests/tensor/int/primitive.rs | 16 + .../burn-backend-tests/tests/tensor/mod.rs | 10 + .../tests/tensor/multi_threads.rs | 171 + .../burn-crates/burn-backend/Cargo.toml | 46 + .../burn-crates/burn-backend/README.md | 4 + .../burn-backend/src/backend/base.rs | 391 + .../burn-backend/src/backend/device.rs | 17 + .../burn-backend/src/backend/mod.rs | 10 + .../src/backend/ops/activation.rs | 279 + .../burn-backend/src/backend/ops/argwhere.rs | 56 + .../src/backend/ops/bool_tensor.rs | 563 + .../burn-backend/src/backend/ops/cat.rs | 40 + .../src/backend/ops/int_tensor.rs | 1372 + .../burn-backend/src/backend/ops/mod.rs | 20 + .../src/backend/ops/modules/attention.rs | 107 + .../src/backend/ops/modules/base.rs | 1132 + .../src/backend/ops/modules/conv.rs | 1408 + .../src/backend/ops/modules/grid_sample.rs | 312 + .../src/backend/ops/modules/mod.rs | 18 + .../src/backend/ops/modules/pool.rs | 176 + .../src/backend/ops/modules/unfold.rs | 146 + .../burn-backend/src/backend/ops/qtensor.rs | 1373 + .../src/backend/ops/repeat_dim.rs | 39 + .../burn-backend/src/backend/ops/sort.rs | 377 + .../burn-backend/src/backend/ops/tensor.rs | 1650 + .../src/backend/ops/transaction.rs | 139 + .../burn-backend/src/backend/primitive.rs | 77 + .../burn-backend/src/data/compare.rs | 427 + .../burn-crates/burn-backend/src/data/mod.rs | 5 + .../burn-backend/src/data/tensor.rs | 815 + .../burn-backend/src/distribution.rs | 125 + .../burn-backend/src/element/base.rs | 295 + .../burn-backend/src/element/cast.rs | 706 + .../burn-backend/src/element/mod.rs | 10 + .../burn-backend/src/element/scalar.rs | 105 + .../burn-crates/burn-backend/src/lib.rs | 122 + .../burn-backend/src/tensor/alias.rs | 23 + .../burn-backend/src/tensor/container.rs | 92 + .../burn-backend/src/tensor/kind.rs | 44 + .../burn-backend/src/tensor/mod.rs | 12 + .../burn-backend/src/tensor/ops/autodiff.rs | 49 + .../burn-backend/src/tensor/ops/base.rs | 807 + .../burn-backend/src/tensor/ops/bool.rs | 218 + .../burn-backend/src/tensor/ops/float.rs | 700 + .../burn-backend/src/tensor/ops/int.rs | 426 + .../burn-backend/src/tensor/ops/mod.rs | 21 + .../burn-backend/src/tensor/ops/numeric.rs | 556 + .../burn-backend/src/tensor/ops/ordered.rs | 650 + .../src/tensor/quantization/calibration.rs | 5 + .../src/tensor/quantization/mod.rs | 7 + .../src/tensor/quantization/parameters.rs | 15 + .../src/tensor/quantization/scheme.rs | 70 + .../burn-crates/burn-candle/Cargo.toml | 44 + .../burn-crates/burn-candle/LICENSE-APACHE | 1 + .../burn-crates/burn-candle/LICENSE-MIT | 1 + .../burn-crates/burn-candle/README.md | 14 + .../burn-crates/burn-candle/src/backend.rs | 300 + .../burn-crates/burn-candle/src/element.rs | 32 + .../burn-crates/burn-candle/src/lib.rs | 27 + .../burn-candle/src/ops/activation.rs | 17 + .../burn-crates/burn-candle/src/ops/base.rs | 572 + .../burn-candle/src/ops/bool_tensor.rs | 212 + .../burn-candle/src/ops/candle_utils.rs | 46 + .../burn-candle/src/ops/int_tensor.rs | 521 + .../burn-crates/burn-candle/src/ops/mod.rs | 10 + .../burn-crates/burn-candle/src/ops/module.rs | 327 + .../burn-candle/src/ops/qtensor.rs | 88 + .../burn-crates/burn-candle/src/ops/tensor.rs | 612 + .../burn-candle/src/ops/transaction.rs | 11 + .../burn-crates/burn-candle/src/ops/utils.rs | 29 + .../burn-crates/burn-candle/src/tensor.rs | 114 + .../burn-crates/burn-collective/Cargo.toml | 73 + .../burn-crates/burn-collective/README.md | 139 + .../multinode-tests/Cargo.toml | 34 + .../burn-collective/multinode-tests/README.md | 32 + .../multinode-tests/src/bin/global.rs | 19 + .../multinode-tests/src/bin/node.rs | 157 + .../multinode-tests/src/bin/test_launcher.rs | 354 + .../multinode-tests/src/lib.rs | 1 + .../multinode-tests/src/shared.rs | 43 + .../burn-crates/burn-collective/src/api.rs | 121 + .../burn-crates/burn-collective/src/config.rs | 337 + .../burn-collective/src/global/base.rs | 23 + .../burn-collective/src/global/mod.rs | 10 + .../burn-collective/src/global/node/base.rs | 203 + .../src/global/node/centralized.rs | 96 + .../burn-collective/src/global/node/mod.rs | 6 + .../burn-collective/src/global/node/ring.rs | 216 + .../burn-collective/src/global/node/sync.rs | 100 + .../burn-collective/src/global/node/tree.rs | 198 + .../burn-collective/src/global/node/worker.rs | 297 + .../src/global/orchestrator/base.rs | 138 + .../src/global/orchestrator/mod.rs | 4 + .../src/global/orchestrator/state.rs | 219 + .../burn-collective/src/global/shared.rs | 132 + .../burn-crates/burn-collective/src/lib.rs | 21 + .../src/local/all_reduce/base.rs | 118 + .../src/local/all_reduce/centralized.rs | 26 + .../src/local/all_reduce/mod.rs | 11 + .../src/local/all_reduce/op.rs | 141 + .../src/local/all_reduce/ring.rs | 194 + .../src/local/all_reduce/tree.rs | 89 + .../src/local/broadcast/centralized.rs | 29 + .../src/local/broadcast/mod.rs | 7 + .../burn-collective/src/local/broadcast/op.rs | 172 + .../src/local/broadcast/tree.rs | 98 + .../burn-collective/src/local/client.rs | 271 + .../burn-collective/src/local/mod.rs | 12 + .../src/local/reduce/centralized.rs | 30 + .../burn-collective/src/local/reduce/mod.rs | 7 + .../burn-collective/src/local/reduce/op.rs | 163 + .../burn-collective/src/local/reduce/tree.rs | 77 + .../burn-collective/src/local/server.rs | 495 + .../burn-collective/src/local/tensor_map.rs | 33 + .../burn-collective/src/tests/all_reduce.rs | 174 + .../burn-collective/src/tests/broadcast.rs | 126 + .../burn-collective/src/tests/mod.rs | 3 + .../burn-collective/src/tests/reduce.rs | 162 + .../burn-crates/burn-communication/Cargo.toml | 44 + .../burn-crates/burn-communication/README.md | 15 + .../burn-communication/src/base.rs | 104 + .../burn-communication/src/data_service.rs | 258 + .../burn-crates/burn-communication/src/lib.rs | 13 + .../burn-communication/src/util.rs | 46 + .../burn-communication/src/websocket/base.rs | 35 + .../src/websocket/client.rs | 109 + .../burn-communication/src/websocket/mod.rs | 7 + .../src/websocket/server.rs | 141 + .../burn-crates/burn-core/Cargo.toml | 151 + .../burn-crates/burn-core/LICENSE-APACHE | 1 + .../burn-crates/burn-core/LICENSE-MIT | 1 + .../burn-crates/burn-core/README.md | 15 + .../burn-crates/burn-core/src/config.rs | 98 + .../burn-core/src/data/dataloader/base.rs | 49 + .../burn-core/src/data/dataloader/batch.rs | 259 + .../burn-core/src/data/dataloader/batcher.rs | 31 + .../burn-core/src/data/dataloader/builder.rs | 260 + .../burn-core/src/data/dataloader/mod.rs | 16 + .../src/data/dataloader/multithread.rs | 441 + .../burn-core/src/data/dataloader/split.rs | 134 + .../burn-core/src/data/dataloader/strategy.rs | 87 + .../burn-crates/burn-core/src/data/mod.rs | 15 + .../burn-crates/burn-core/src/lib.rs | 118 + .../burn-crates/burn-core/src/module/base.rs | 470 + .../burn-core/src/module/display.rs | 543 + .../burn-core/src/module/initializer.rs | 627 + .../burn-crates/burn-core/src/module/mod.rs | 16 + .../burn-core/src/module/param/base.rs | 424 + .../burn-core/src/module/param/constant.rs | 408 + .../burn-core/src/module/param/id.rs | 116 + .../burn-core/src/module/param/mod.rs | 13 + .../burn-core/src/module/param/primitive.rs | 426 + .../burn-core/src/module/param/running.rs | 258 + .../burn-core/src/module/param/tensor.rs | 571 + .../burn-core/src/module/param/visitor.rs | 38 + .../burn-core/src/module/quantize.rs | 65 + .../burn-core/src/module/reinit.rs | 203 + .../burn-crates/burn-core/src/record/base.rs | 17 + .../burn-crates/burn-core/src/record/file.rs | 421 + .../burn-core/src/record/memory.rs | 141 + .../burn-crates/burn-core/src/record/mod.rs | 22 + .../burn-core/src/record/primitive.rs | 336 + .../burn-core/src/record/recorder.rs | 329 + .../burn-core/src/record/serde/adapter.rs | 83 + .../burn-core/src/record/serde/data.rs | 399 + .../burn-core/src/record/serde/de.rs | 1006 + .../burn-core/src/record/serde/error.rs | 40 + .../burn-core/src/record/serde/mod.rs | 17 + .../burn-core/src/record/serde/ser.rs | 387 + .../burn-core/src/record/settings.rs | 40 + .../burn-core/src/record/tensor.rs | 159 + .../burn-crates/burn-core/src/tensor.rs | 1 + .../burn-crates/burn-core/src/vision.rs | 1 + .../burn-core/tests/test_derive_config.rs | 113 + .../burn-core/tests/test_derive_module.rs | 323 + .../burn-core/tests/test_derive_record.rs | 17 + .../burn-core/tests/test_record_resilience.rs | 344 + .../burn-crates/burn-cpu/Cargo.toml | 42 + .../burn-crates/burn-cpu/README.md | 12 + .../burn-crates/burn-cpu/src/lib.rs | 47 + .../burn-crates/burn-cubecl-fusion/Cargo.toml | 52 + .../burn-crates/burn-cubecl-fusion/README.md | 3 + .../burn-cubecl-fusion/src/base.rs | 165 + .../src/engine/codegen/base.rs | 6 + .../src/engine/codegen/io.rs | 792 + .../src/engine/codegen/ir.rs | 917 + .../src/engine/codegen/kernel.rs | 927 + .../src/engine/codegen/mod.rs | 8 + .../src/engine/codegen/tensor.rs | 90 + .../src/engine/codegen/view.rs | 358 + .../burn-cubecl-fusion/src/engine/fuser.rs | 769 + .../src/engine/launch/base.rs | 99 + .../src/engine/launch/executor.rs | 292 + .../src/engine/launch/input.rs | 247 + .../src/engine/launch/mod.rs | 11 + .../src/engine/launch/output.rs | 696 + .../src/engine/launch/plan.rs | 273 + .../src/engine/launch/runner.rs | 96 + .../src/engine/launch/vectorization/base.rs | 439 + .../src/engine/launch/vectorization/mod.rs | 5 + .../engine/launch/vectorization/planner.rs | 438 + .../burn-cubecl-fusion/src/engine/mod.rs | 6 + .../burn-cubecl-fusion/src/engine/settings.rs | 59 + .../src/engine/trace/base.rs | 377 + .../src/engine/trace/block.rs | 555 + .../src/engine/trace/fuser.rs | 330 + .../src/engine/trace/mod.rs | 7 + .../burn-crates/burn-cubecl-fusion/src/lib.rs | 11 + .../burn-cubecl-fusion/src/optim/base.rs | 63 + .../src/optim/elemwise/fuser.rs | 87 + .../src/optim/elemwise/mod.rs | 5 + .../src/optim/elemwise/optimization.rs | 140 + .../src/optim/matmul/args.rs | 548 + .../src/optim/matmul/fuser.rs | 160 + .../src/optim/matmul/mod.rs | 8 + .../src/optim/matmul/optimization.rs | 649 + .../src/optim/matmul/tune.rs | 269 + .../burn-cubecl-fusion/src/optim/mod.rs | 8 + .../src/optim/reduce/args.rs | 208 + .../src/optim/reduce/fuser.rs | 328 + .../src/optim/reduce/mod.rs | 8 + .../src/optim/reduce/optimization.rs | 492 + .../src/optim/reduce/tune.rs | 196 + .../optim/reduce_broadcasted/fuser/base.rs | 375 + .../optim/reduce_broadcasted/fuser/block.rs | 214 + .../optim/reduce_broadcasted/fuser/full.rs | 163 + .../reduce_broadcasted/fuser/full_analyzer.rs | 159 + .../src/optim/reduce_broadcasted/fuser/mod.rs | 6 + .../src/optim/reduce_broadcasted/launch.rs | 139 + .../src/optim/reduce_broadcasted/mod.rs | 9 + .../optim/reduce_broadcasted/optimization.rs | 222 + .../src/optim/reduce_broadcasted/tune.rs | 180 + .../src/optim/reduce_broadcasted/unit.rs | 202 + .../burn-cubecl-fusion/src/tune.rs | 107 + .../burn-crates/burn-cubecl/Cargo.toml | 88 + .../burn-crates/burn-cubecl/LICENSE-APACHE | 1 + .../burn-crates/burn-cubecl/LICENSE-MIT | 1 + .../burn-crates/burn-cubecl/README.md | 3 + .../burn-crates/burn-cubecl/src/backend.rs | 196 + .../burn-crates/burn-cubecl/src/element.rs | 94 + .../burn-crates/burn-cubecl/src/fusion.rs | 205 + .../burn-cubecl/src/kernel/attention/base.rs | 150 + .../burn-cubecl/src/kernel/attention/mod.rs | 5 + .../burn-cubecl/src/kernel/attention/tune.rs | 166 + .../burn-cubecl/src/kernel/binary.rs | 302 + .../burn-cubecl/src/kernel/binary_float.rs | 119 + .../burn-cubecl/src/kernel/binary_int.rs | 229 + .../burn-cubecl/src/kernel/cast/base.rs | 70 + .../burn-cubecl/src/kernel/cast/bool_cast.rs | 55 + .../burn-cubecl/src/kernel/cast/mod.rs | 5 + .../burn-cubecl/src/kernel/clamp.rs | 42 + .../burn-cubecl/src/kernel/comparison.rs | 432 + .../burn-cubecl/src/kernel/contiguous.rs | 124 + .../src/kernel/conv/backward_data/fallback.rs | 142 + .../backward_data/implicit_gemm/launch.rs | 132 + .../conv/backward_data/implicit_gemm/mod.rs | 2 + .../src/kernel/conv/backward_data/mod.rs | 8 + .../src/kernel/conv/backward_data/tune.rs | 172 + .../kernel/conv/backward_weight/fallback.rs | 112 + .../backward_weight/implicit_gemm/launch.rs | 132 + .../conv/backward_weight/implicit_gemm/mod.rs | 2 + .../src/kernel/conv/backward_weight/mod.rs | 8 + .../src/kernel/conv/backward_weight/tune.rs | 175 + .../burn-cubecl/src/kernel/conv/base.rs | 189 + .../src/kernel/conv/conv_transpose2d/base.rs | 54 + .../kernel/conv/conv_transpose2d/col2im.rs | 302 + .../src/kernel/conv/conv_transpose2d/mod.rs | 15 + .../conv/conv_transpose2d/transpose_direct.rs | 185 + .../src/kernel/conv/conv_transpose2d/tune.rs | 91 + .../src/kernel/conv/conv_transpose3d.rs | 222 + .../src/kernel/conv/deform_conv2d.rs | 314 + .../kernel/conv/deform_conv_transpose2d.rs | 720 + .../burn-cubecl/src/kernel/conv/direct.rs | 320 + .../conv/forward/implicit_gemm/launch.rs | 167 + .../kernel/conv/forward/implicit_gemm/mod.rs | 2 + .../src/kernel/conv/forward/mod.rs | 7 + .../src/kernel/conv/forward/tune.rs | 174 + .../burn-cubecl/src/kernel/conv/im2col.rs | 187 + .../burn-cubecl/src/kernel/conv/mod.rs | 25 + .../burn-cubecl/src/kernel/conv/tune_key.rs | 50 + .../burn-cubecl/src/kernel/cross.rs | 101 + .../src/kernel/grid_sample/base.rs | 163 + .../src/kernel/grid_sample/bilinear.rs | 177 + .../burn-cubecl/src/kernel/grid_sample/mod.rs | 4 + .../burn-cubecl/src/kernel/index/flip.rs | 99 + .../burn-cubecl/src/kernel/index/gather.rs | 76 + .../burn-cubecl/src/kernel/index/mod.rs | 18 + .../src/kernel/index/repeat_dim.rs | 93 + .../burn-cubecl/src/kernel/index/scatter.rs | 109 + .../burn-cubecl/src/kernel/index/select.rs | 82 + .../src/kernel/index/select_assign.rs | 98 + .../burn-cubecl/src/kernel/index/slice.rs | 247 + .../src/kernel/index/slice_assign.rs | 258 + .../src/kernel/interpolate/base.rs | 79 + .../src/kernel/interpolate/bicubic.rs | 194 + .../src/kernel/interpolate/bilinear.rs | 149 + .../burn-cubecl/src/kernel/interpolate/mod.rs | 7 + .../src/kernel/interpolate/nearest.rs | 80 + .../kernel/interpolate/nearest_backward.rs | 103 + .../burn-cubecl/src/kernel/mask/base.rs | 39 + .../burn-cubecl/src/kernel/mask/mask_fill.rs | 88 + .../burn-cubecl/src/kernel/mask/mask_where.rs | 91 + .../burn-cubecl/src/kernel/mask/mod.rs | 8 + .../burn-cubecl/src/kernel/matmul/base.rs | 155 + .../burn-cubecl/src/kernel/matmul/mod.rs | 10 + .../src/kernel/matmul/tune/base.rs | 409 + .../burn-cubecl/src/kernel/matmul/tune/mod.rs | 5 + .../burn-cubecl/src/kernel/matmul/utils.rs | 16 + .../burn-crates/burn-cubecl/src/kernel/mod.rs | 51 + .../src/kernel/pool/adaptive_avg_pool2d.rs | 117 + .../pool/adaptive_avg_pool2d_backward.rs | 119 + .../burn-cubecl/src/kernel/pool/avg_pool2d.rs | 166 + .../src/kernel/pool/avg_pool2d_backward.rs | 183 + .../burn-cubecl/src/kernel/pool/max_pool2d.rs | 255 + .../src/kernel/pool/max_pool2d_backward.rs | 156 + .../burn-cubecl/src/kernel/pool/mod.rs | 15 + .../burn-cubecl/src/kernel/pool/pool2d.rs | 155 + .../burn-cubecl/src/kernel/prng/bernoulli.rs | 18 + .../burn-cubecl/src/kernel/prng/mod.rs | 7 + .../burn-cubecl/src/kernel/prng/normal.rs | 20 + .../burn-cubecl/src/kernel/prng/uniform.rs | 43 + .../src/kernel/quantization/dequantize.rs | 34 + .../src/kernel/quantization/mod.rs | 5 + .../src/kernel/quantization/quantize.rs | 29 + .../burn-cubecl/src/kernel/reduce/base.rs | 242 + .../burn-cubecl/src/kernel/reduce/mod.rs | 7 + .../burn-cubecl/src/kernel/reduce/tune.rs | 286 + .../burn-cubecl/src/kernel/unary_float.rs | 191 + .../burn-cubecl/src/kernel/unary_int.rs | 142 + .../burn-cubecl/src/kernel/unary_numeric.rs | 90 + .../burn-cubecl/src/kernel/utils.rs | 198 + .../burn-crates/burn-cubecl/src/lib.rs | 50 + .../burn-cubecl/src/ops/activation.rs | 11 + .../burn-crates/burn-cubecl/src/ops/base.rs | 432 + .../burn-cubecl/src/ops/bool_tensor.rs | 200 + .../burn-cubecl/src/ops/int_tensor.rs | 546 + .../burn-crates/burn-cubecl/src/ops/mod.rs | 14 + .../burn-crates/burn-cubecl/src/ops/module.rs | 344 + .../burn-cubecl/src/ops/numeric.rs | 442 + .../burn-cubecl/src/ops/qtensor.rs | 313 + .../burn-crates/burn-cubecl/src/ops/tensor.rs | 620 + .../burn-cubecl/src/ops/transaction.rs | 143 + .../burn-cubecl/src/template/base.rs | 103 + .../burn-cubecl/src/template/mod.rs | 5 + .../burn-cubecl/src/template/source.rs | 69 + .../burn-cubecl/src/tensor/base.rs | 379 + .../burn-crates/burn-cubecl/src/tensor/mod.rs | 5 + .../burn-cubecl/src/tensor/quantization.rs | 122 + .../burn-crates/burn-cubecl/src/tune_key.rs | 30 + .../burn-crates/burn-cuda/Cargo.toml | 41 + .../burn-crates/burn-cuda/README.md | 30 + .../burn-crates/burn-cuda/src/lib.rs | 47 + .../burn-crates/burn-dataset/Cargo.toml | 84 + .../burn-crates/burn-dataset/LICENSE-APACHE | 1 + .../burn-crates/burn-dataset/LICENSE-MIT | 1 + .../burn-crates/burn-dataset/README.md | 17 + .../burn-dataset/examples/hf_dataset.rs | 22 + .../burn-dataset/examples/speech_commands.rs | 23 + .../burn-crates/burn-dataset/src/audio/mod.rs | 3 + .../burn-dataset/src/audio/speech_commands.rs | 208 + .../burn-dataset/src/dataset/base.rs | 71 + .../burn-dataset/src/dataset/dataframe.rs | 465 + .../burn-dataset/src/dataset/fake.rs | 38 + .../burn-dataset/src/dataset/in_memory.rs | 192 + .../burn-dataset/src/dataset/iterator.rs | 31 + .../burn-dataset/src/dataset/mod.rs | 25 + .../burn-dataset/src/dataset/sqlite.rs | 851 + .../burn-crates/burn-dataset/src/lib.rs | 52 + .../burn-dataset/src/nlp/ag_news.rs | 211 + .../burn-crates/burn-dataset/src/nlp/mod.rs | 7 + .../burn-dataset/src/nlp/text_folder.rs | 421 + .../src/source/huggingface/downloader.rs | 367 + .../src/source/huggingface/importer.py | 207 + .../src/source/huggingface/mod.rs | 3 + .../burn-dataset/src/source/mod.rs | 3 + .../burn-dataset/src/transform/composed.rs | 56 + .../burn-dataset/src/transform/mapper.rs | 60 + .../burn-dataset/src/transform/mod.rs | 30 + .../burn-dataset/src/transform/options.rs | 199 + .../burn-dataset/src/transform/partial.rs | 206 + .../burn-dataset/src/transform/sampler.rs | 438 + .../burn-dataset/src/transform/selection.rs | 374 + .../burn-dataset/src/transform/shuffle.rs | 109 + .../burn-dataset/src/transform/window.rs | 290 + .../burn-dataset/src/vision/cifar.rs | 241 + .../burn-dataset/src/vision/image_folder.rs | 1124 + .../burn-dataset/src/vision/mnist.rs | 221 + .../burn-dataset/src/vision/mod.rs | 9 + .../burn-dataset/tests/data/dataset-fmt.csv | 2 + .../burn-dataset/tests/data/dataset.csv | 3 + .../burn-dataset/tests/data/dataset.json | 2 + .../burn-dataset/tests/data/dataset_coco.json | 132 + .../tests/data/image_folder/orange/dot.jpg | Bin 0 -> 727 bytes .../tests/data/image_folder/red/dot.jpg | Bin 0 -> 634 bytes .../tests/data/image_folder/red/dot.png | Bin 0 -> 120 bytes .../data/image_folder_coco/dot_triangle.jpg | Bin 0 -> 1566 bytes .../tests/data/image_folder_coco/one_dot.jpg | Bin 0 -> 1434 bytes .../two_dots_and_triangle.jpg | Bin 0 -> 1706 bytes .../annotations/mask_checkerboard.png | Bin 0 -> 117 bytes .../annotations/mask_checkerboard.txt | 8 + .../annotations/mask_random_2colors.png | Bin 0 -> 123 bytes .../annotations/mask_random_2colors.txt | 8 + .../annotations/mask_random_3colors.png | Bin 0 -> 137 bytes .../annotations/mask_random_3colors.txt | 8 + .../images/image_checkerboard.png | Bin 0 -> 165 bytes .../images/image_random_2colors.png | Bin 0 -> 133 bytes .../images/image_random_3colors.png | Bin 0 -> 204 bytes .../burn-dataset/tests/data/sqlite-dataset.db | Bin 0 -> 12288 bytes .../data/text_folder/negative/sample1.txt | 1 + .../data/text_folder/negative/sample2.txt | 1 + .../data/text_folder/positive/sample1.txt | 1 + .../data/text_folder/positive/sample2.txt | 1 + .../burn-crates/burn-derive/Cargo.toml | 23 + .../burn-crates/burn-derive/LICENSE-APACHE | 1 + .../burn-crates/burn-derive/LICENSE-MIT | 1 + .../burn-crates/burn-derive/README.md | 6 + .../burn-derive/src/config/analyzer.rs | 87 + .../burn-derive/src/config/analyzer_enum.rs | 141 + .../burn-derive/src/config/analyzer_struct.rs | 380 + .../burn-derive/src/config/base.rs | 24 + .../burn-crates/burn-derive/src/config/mod.rs | 9 + .../burn-crates/burn-derive/src/lib.rs | 34 + .../burn-derive/src/module/base.rs | 39 + .../burn-derive/src/module/codegen.rs | 319 + .../burn-derive/src/module/codegen_enum.rs | 236 + .../burn-derive/src/module/codegen_struct.rs | 267 + .../burn-derive/src/module/display.rs | 94 + .../burn-crates/burn-derive/src/module/mod.rs | 11 + .../burn-derive/src/module/record.rs | 8 + .../burn-derive/src/module/record_enum.rs | 41 + .../burn-derive/src/module/record_struct.rs | 40 + .../burn-derive/src/record/base.rs | 13 + .../burn-derive/src/record/codegen.rs | 145 + .../burn-derive/src/record/item/codegen.rs | 21 + .../src/record/item/codegen_enum.rs | 137 + .../src/record/item/codegen_struct.rs | 136 + .../burn-derive/src/record/item/mod.rs | 3 + .../burn-crates/burn-derive/src/record/mod.rs | 5 + .../burn-derive/src/shared/attribute.rs | 49 + .../burn-derive/src/shared/enum_variant.rs | 103 + .../burn-derive/src/shared/field.rs | 99 + .../burn-derive/src/shared/generics.rs | 63 + .../burn-crates/burn-derive/src/shared/mod.rs | 4 + .../burn-crates/burn-dispatch/Cargo.toml | 84 + .../burn-crates/burn-dispatch/README.md | 3 + .../burn-crates/burn-dispatch/build.rs | 35 + .../burn-crates/burn-dispatch/src/backend.rs | 392 + .../burn-crates/burn-dispatch/src/device.rs | 415 + .../burn-crates/burn-dispatch/src/lib.rs | 90 + .../burn-crates/burn-dispatch/src/macros.rs | 1197 + .../burn-dispatch/src/ops/activation.rs | 50 + .../burn-dispatch/src/ops/bool_tensor.rs | 222 + .../burn-dispatch/src/ops/int_tensor.rs | 503 + .../burn-crates/burn-dispatch/src/ops/mod.rs | 7 + .../burn-dispatch/src/ops/module.rs | 628 + .../burn-dispatch/src/ops/qtensor.rs | 212 + .../burn-dispatch/src/ops/tensor.rs | 594 + .../burn-dispatch/src/ops/transaction.rs | 26 + .../burn-crates/burn-dispatch/src/tensor.rs | 274 + .../burn-crates/burn-fusion/Cargo.toml | 42 + .../burn-crates/burn-fusion/LICENSE-APACHE | 1 + .../burn-crates/burn-fusion/LICENSE-MIT | 1 + .../burn-crates/burn-fusion/README.md | 3 + .../burn-crates/burn-fusion/src/backend.rs | 240 + .../burn-crates/burn-fusion/src/client.rs | 307 + .../burn-crates/burn-fusion/src/lib.rs | 29 + .../burn-fusion/src/ops/activation.rs | 4 + .../burn-crates/burn-fusion/src/ops/base.rs | 15 + .../burn-crates/burn-fusion/src/ops/binary.rs | 117 + .../burn-fusion/src/ops/bool_tensor.rs | 856 + .../burn-fusion/src/ops/int_tensor.rs | 1980 + .../burn-crates/burn-fusion/src/ops/mod.rs | 12 + .../burn-crates/burn-fusion/src/ops/module.rs | 1563 + .../burn-fusion/src/ops/qtensor.rs | 501 + .../burn-crates/burn-fusion/src/ops/tensor.rs | 2403 + .../burn-fusion/src/ops/transaction.rs | 36 + .../burn-crates/burn-fusion/src/ops/unary.rs | 319 + .../burn-fusion/src/search/block.rs | 271 + .../burn-fusion/src/search/merging.rs | 441 + .../burn-crates/burn-fusion/src/search/mod.rs | 7 + .../src/search/optimization/blocks.rs | 232 + .../src/search/optimization/mod.rs | 4 + .../src/search/optimization/stream.rs | 277 + .../burn-crates/burn-fusion/src/server.rs | 215 + .../burn-fusion/src/stream/base.rs | 1 + .../burn-fusion/src/stream/context.rs | 1274 + .../burn-fusion/src/stream/execution/base.rs | 16 + .../src/stream/execution/explorer.rs | 91 + .../burn-fusion/src/stream/execution/mod.rs | 17 + .../src/stream/execution/ordering.rs | 71 + .../src/stream/execution/policy.rs | 572 + .../src/stream/execution/processor.rs | 184 + .../burn-fusion/src/stream/execution/tests.rs | 671 + .../src/stream/execution/validator.rs | 136 + .../burn-fusion/src/stream/memory_checks.rs | 249 + .../burn-crates/burn-fusion/src/stream/mod.rs | 33 + .../burn-fusion/src/stream/multi.rs | 472 + .../burn-fusion/src/stream/queue/base.rs | 95 + .../burn-fusion/src/stream/queue/execution.rs | 153 + .../burn-fusion/src/stream/queue/mod.rs | 4 + .../burn-fusion/src/stream/shared_tensors.rs | 306 + .../burn-fusion/src/stream/store/base.rs | 94 + .../burn-fusion/src/stream/store/index.rs | 293 + .../burn-fusion/src/stream/store/mod.rs | 5 + .../burn-crates/burn-fusion/src/tensor.rs | 232 + .../burn-crates/burn-ir/Cargo.toml | 33 + .../burn-crates/burn-ir/README.md | 7 + .../burn-crates/burn-ir/src/backend.rs | 63 + .../burn-crates/burn-ir/src/builder.rs | 1113 + .../burn-crates/burn-ir/src/handle.rs | 216 + .../burn-crates/burn-ir/src/lib.rs | 21 + .../burn-crates/burn-ir/src/operation.rs | 3011 + .../burn-crates/burn-ir/src/scalar.rs | 77 + .../burn-crates/burn-ir/src/tensor.rs | 67 + .../burn-crates/burn-ndarray/Cargo.toml | 97 + .../burn-crates/burn-ndarray/LICENSE-APACHE | 1 + .../burn-crates/burn-ndarray/LICENSE-MIT | 1 + .../burn-crates/burn-ndarray/README.md | 30 + .../burn-crates/burn-ndarray/build.rs | 6 + .../burn-crates/burn-ndarray/src/backend.rs | 221 + .../burn-crates/burn-ndarray/src/element.rs | 207 + .../burn-crates/burn-ndarray/src/lib.rs | 29 + .../burn-ndarray/src/ops/activation.rs | 18 + .../burn-ndarray/src/ops/adaptive_avgpool.rs | 103 + .../burn-ndarray/src/ops/avgpool.rs | 172 + .../burn-crates/burn-ndarray/src/ops/base.rs | 1448 + .../burn-ndarray/src/ops/bool_tensor.rs | 220 + .../burn-crates/burn-ndarray/src/ops/conv.rs | 574 + .../burn-ndarray/src/ops/deform_conv.rs | 662 + .../burn-ndarray/src/ops/grid_sample.rs | 214 + .../burn-ndarray/src/ops/int_tensor.rs | 497 + .../burn-ndarray/src/ops/interpolate.rs | 302 + .../burn-ndarray/src/ops/macros.rs | 107 + .../burn-ndarray/src/ops/matmul.rs | 362 + .../burn-ndarray/src/ops/maxpool.rs | 247 + .../burn-crates/burn-ndarray/src/ops/mod.rs | 24 + .../burn-ndarray/src/ops/module.rs | 367 + .../burn-ndarray/src/ops/padding.rs | 72 + .../burn-ndarray/src/ops/qtensor.rs | 346 + .../burn-ndarray/src/ops/quantization.rs | 218 + .../burn-ndarray/src/ops/simd/avgpool.rs | 443 + .../burn-ndarray/src/ops/simd/base.rs | 115 + .../burn-ndarray/src/ops/simd/binary.rs | 299 + .../src/ops/simd/binary_elemwise.rs | 419 + .../burn-ndarray/src/ops/simd/cmp.rs | 374 + .../burn-ndarray/src/ops/simd/conv.rs | 494 + .../burn-ndarray/src/ops/simd/maxpool.rs | 394 + .../burn-ndarray/src/ops/simd/mod.rs | 10 + .../burn-ndarray/src/ops/simd/unary.rs | 234 + .../burn-ndarray/src/ops/tensor.rs | 688 + .../burn-ndarray/src/ops/transaction.rs | 13 + .../burn-crates/burn-ndarray/src/parallel.rs | 76 + .../burn-crates/burn-ndarray/src/rand.rs | 36 + .../burn-crates/burn-ndarray/src/sharing.rs | 19 + .../burn-crates/burn-ndarray/src/storage.rs | 514 + .../burn-crates/burn-ndarray/src/tensor.rs | 864 + .../burn-crates/burn-nn/Cargo.toml | 87 + .../burn-crates/burn-nn/README.md | 3 + .../src/activation/activation_wrapper.rs | 598 + .../burn-nn/src/activation/celu.rs | 104 + .../burn-crates/burn-nn/src/activation/elu.rs | 85 + .../burn-nn/src/activation/gelu.rs | 82 + .../burn-crates/burn-nn/src/activation/glu.rs | 51 + .../burn-nn/src/activation/hard_shrink.rs | 98 + .../burn-nn/src/activation/hard_sigmoid.rs | 97 + .../burn-nn/src/activation/hard_swish.rs | 58 + .../burn-nn/src/activation/leaky_relu.rs | 124 + .../burn-crates/burn-nn/src/activation/mod.rs | 68 + .../burn-nn/src/activation/prelu.rs | 87 + .../burn-nn/src/activation/relu.rs | 39 + .../burn-nn/src/activation/selu.rs | 38 + .../burn-nn/src/activation/shrink.rs | 114 + .../burn-nn/src/activation/sigmoid.rs | 38 + .../burn-nn/src/activation/soft_shrink.rs | 98 + .../burn-nn/src/activation/softplus.rs | 105 + .../burn-nn/src/activation/softsign.rs | 38 + .../burn-nn/src/activation/swiglu.rs | 153 + .../burn-nn/src/activation/tanh.rs | 38 + .../src/activation/thresholded_relu.rs | 82 + .../burn-crates/burn-nn/src/lib.rs | 63 + .../burn-nn/src/loss/binary_cross_entropy.rs | 432 + .../burn-nn/src/loss/cosine_embedding.rs | 317 + .../burn-nn/src/loss/cross_entropy.rs | 466 + .../burn-crates/burn-nn/src/loss/ctc.rs | 1730 + .../burn-crates/burn-nn/src/loss/huber.rs | 215 + .../burn-crates/burn-nn/src/loss/kldiv.rs | 200 + .../burn-crates/burn-nn/src/loss/lp_loss.rs | 672 + .../burn-crates/burn-nn/src/loss/mod.rs | 23 + .../burn-crates/burn-nn/src/loss/mse.rs | 93 + .../burn-crates/burn-nn/src/loss/poisson.rs | 417 + .../burn-crates/burn-nn/src/loss/reduction.rs | 19 + .../burn-crates/burn-nn/src/loss/smooth_l1.rs | 520 + .../src/modules/attention/cross_attention.rs | 621 + .../burn-nn/src/modules/attention/mask.rs | 161 + .../burn-nn/src/modules/attention/mha.rs | 531 + .../burn-nn/src/modules/attention/mod.rs | 7 + .../src/modules/cache/autoregressive.rs | 52 + .../burn-nn/src/modules/cache/base.rs | 27 + .../burn-nn/src/modules/cache/mod.rs | 4 + .../burn-nn/src/modules/conv/checks.rs | 22 + .../burn-nn/src/modules/conv/conv1d.rs | 283 + .../burn-nn/src/modules/conv/conv2d.rs | 349 + .../burn-nn/src/modules/conv/conv3d.rs | 276 + .../src/modules/conv/conv_transpose1d.rs | 216 + .../src/modules/conv/conv_transpose2d.rs | 216 + .../src/modules/conv/conv_transpose3d.rs | 221 + .../burn-nn/src/modules/conv/deform_conv2d.rs | 295 + .../burn-nn/src/modules/conv/mod.rs | 17 + .../burn-nn/src/modules/dropout.rs | 124 + .../burn-nn/src/modules/embedding.rs | 111 + .../src/modules/interpolate/interpolate1d.rs | 258 + .../src/modules/interpolate/interpolate2d.rs | 261 + .../burn-nn/src/modules/interpolate/mod.rs | 49 + .../burn-crates/burn-nn/src/modules/linear.rs | 340 + .../burn-crates/burn-nn/src/modules/mod.rs | 38 + .../burn-crates/burn-nn/src/modules/noise.rs | 123 + .../burn-nn/src/modules/norm/batch.rs | 484 + .../burn-nn/src/modules/norm/group.rs | 336 + .../burn-nn/src/modules/norm/instance.rs | 228 + .../burn-nn/src/modules/norm/layer.rs | 232 + .../burn-nn/src/modules/norm/mod.rs | 28 + .../src/modules/norm/normalization_wrapper.rs | 368 + .../burn-nn/src/modules/norm/rms.rs | 134 + .../src/modules/pool/adaptive_avg_pool1d.rs | 77 + .../src/modules/pool/adaptive_avg_pool2d.rs | 79 + .../burn-nn/src/modules/pool/avg_pool1d.rs | 220 + .../burn-nn/src/modules/pool/avg_pool2d.rs | 223 + .../burn-nn/src/modules/pool/max_pool1d.rs | 214 + .../burn-nn/src/modules/pool/max_pool2d.rs | 219 + .../burn-nn/src/modules/pool/mod.rs | 13 + .../burn-nn/src/modules/pos_encoding.rs | 291 + .../burn-nn/src/modules/rnn/basic.rs | 742 + .../src/modules/rnn/gate_controller.rs | 101 + .../burn-nn/src/modules/rnn/gru.rs | 1074 + .../burn-nn/src/modules/rnn/lstm.rs | 922 + .../burn-nn/src/modules/rnn/mod.rs | 15 + .../burn-nn/src/modules/rope_encoding.rs | 581 + .../src/modules/transformer/decoder.rs | 573 + .../src/modules/transformer/encoder.rs | 489 + .../burn-nn/src/modules/transformer/mod.rs | 7 + .../burn-nn/src/modules/transformer/pwff.rs | 117 + .../burn-crates/burn-nn/src/modules/unfold.rs | 104 + .../burn-crates/burn-nn/src/padding.rs | 247 + .../burn-crates/burn-nn/tests/quantize.rs | 167 + .../burn-crates/burn-no-std-tests/Cargo.toml | 32 + .../burn-crates/burn-no-std-tests/README.md | 29 + .../burn-no-std-tests/src/burnpack.rs | 158 + .../burn-crates/burn-no-std-tests/src/conv.rs | 49 + .../burn-crates/burn-no-std-tests/src/lib.rs | 9 + .../burn-crates/burn-no-std-tests/src/mlp.rs | 67 + .../burn-no-std-tests/src/model.rs | 66 + .../burn-no-std-tests/src/safetensors.rs | 111 + .../burn-no-std-tests/tests/burnpack_tests.rs | 12 + .../tests/safetensors_tests.rs | 12 + .../tests/test_integration.rs | 31 + .../burn-crates/burn-optim/Cargo.toml | 102 + .../burn-crates/burn-optim/README.md | 3 + .../burn-optim/src/grad_clipping/base.rs | 144 + .../burn-optim/src/grad_clipping/mod.rs | 2 + .../burn-crates/burn-optim/src/lib.rs | 63 + .../burn-optim/src/lr_scheduler/base.rs | 85 + .../burn-optim/src/lr_scheduler/composed.rs | 195 + .../burn-optim/src/lr_scheduler/constant.rs | 50 + .../burn-optim/src/lr_scheduler/cosine.rs | 191 + .../src/lr_scheduler/exponential.rs | 139 + .../burn-optim/src/lr_scheduler/linear.rs | 173 + .../burn-optim/src/lr_scheduler/mod.rs | 24 + .../burn-optim/src/lr_scheduler/noam.rs | 136 + .../burn-optim/src/lr_scheduler/step.rs | 218 + .../burn-optim/src/optim/adagrad.rs | 306 + .../burn-crates/burn-optim/src/optim/adam.rs | 521 + .../burn-crates/burn-optim/src/optim/adamw.rs | 598 + .../burn-crates/burn-optim/src/optim/base.rs | 88 + .../burn-crates/burn-optim/src/optim/decay.rs | 68 + .../burn-optim/src/optim/grad_accum.rs | 121 + .../burn-crates/burn-optim/src/optim/grads.rs | 192 + .../burn-crates/burn-optim/src/optim/lbfgs.rs | 978 + .../burn-crates/burn-optim/src/optim/mod.rs | 30 + .../burn-optim/src/optim/momentum.rs | 94 + .../burn-crates/burn-optim/src/optim/muon.rs | 775 + .../burn-optim/src/optim/rmsprop.rs | 566 + .../burn-crates/burn-optim/src/optim/sgd.rs | 181 + .../burn-optim/src/optim/simple/adaptor.rs | 210 + .../burn-optim/src/optim/simple/base.rs | 36 + .../burn-optim/src/optim/simple/mod.rs | 8 + .../src/optim/simple/record/base.rs | 93 + .../burn-optim/src/optim/simple/record/mod.rs | 5 + .../burn-optim/src/optim/simple/record/v1.rs | 201 + .../burn-optim/src/optim/visitor.rs | 60 + .../burn-crates/burn-remote/Cargo.toml | 78 + .../burn-crates/burn-remote/README.md | 0 .../burn-remote/src/client/base.rs | 123 + .../burn-remote/src/client/channel.rs | 82 + .../burn-crates/burn-remote/src/client/mod.rs | 8 + .../burn-remote/src/client/runner.rs | 294 + .../burn-remote/src/client/worker.rs | 129 + .../burn-crates/burn-remote/src/lib.rs | 86 + .../burn-remote/src/server/base.rs | 182 + .../burn-crates/burn-remote/src/server/mod.rs | 7 + .../burn-remote/src/server/processor.rs | 132 + .../burn-remote/src/server/session.rs | 170 + .../burn-remote/src/server/stream.rs | 134 + .../burn-crates/burn-remote/src/shared/mod.rs | 7 + .../burn-remote/src/shared/task.rs | 87 + .../burn-crates/burn-rl/Cargo.toml | 31 + .../burn-crates/burn-rl/LICENSE-APACHE | 1 + .../burn-crates/burn-rl/LICENSE-MIT | 1 + .../burn-crates/burn-rl/README.md | 6 + .../burn-rl/src/environment/base.rs | 46 + .../burn-rl/src/environment/mod.rs | 3 + .../burn-crates/burn-rl/src/lib.rs | 166 + .../burn-rl/src/policy/async_policy.rs | 485 + .../burn-crates/burn-rl/src/policy/base.rs | 108 + .../burn-crates/burn-rl/src/policy/mod.rs | 5 + .../burn-rl/src/transition_buffer/base.rs | 244 + .../burn-rl/src/transition_buffer/mod.rs | 5 + .../src/transition_buffer/slice_access.rs | 36 + .../burn-crates/burn-rocm/Cargo.toml | 42 + .../burn-crates/burn-rocm/README.md | 7 + .../burn-crates/burn-rocm/src/lib.rs | 14 + .../burn-crates/burn-router/Cargo.toml | 48 + .../burn-crates/burn-router/README.md | 3 + .../burn-crates/burn-router/src/backend.rs | 75 + .../burn-router/src/bridge/base.rs | 32 + .../burn-router/src/bridge/byte.rs | 6 + .../burn-crates/burn-router/src/bridge/mod.rs | 5 + .../burn-router/src/channel/base.rs | 66 + .../burn-router/src/channel/direct.rs | 16 + .../burn-router/src/channel/mod.rs | 5 + .../burn-router/src/client/base.rs | 128 + .../burn-crates/burn-router/src/client/mod.rs | 3 + .../burn-crates/burn-router/src/lib.rs | 49 + .../burn-router/src/ops/activation.rs | 4 + .../burn-crates/burn-router/src/ops/binary.rs | 69 + .../burn-router/src/ops/bool_tensor.rs | 333 + .../burn-router/src/ops/int_tensor.rs | 1037 + .../burn-crates/burn-router/src/ops/mod.rs | 9 + .../burn-crates/burn-router/src/ops/module.rs | 796 + .../burn-router/src/ops/qtensor.rs | 92 + .../burn-crates/burn-router/src/ops/tensor.rs | 1248 + .../burn-router/src/ops/transaction.rs | 5 + .../burn-crates/burn-router/src/ops/unary.rs | 155 + .../burn-crates/burn-router/src/runner.rs | 1575 + .../burn-crates/burn-router/src/tensor.rs | 142 + .../burn-crates/burn-router/src/types.rs | 386 + .../burn-crates/burn-std/Cargo.toml | 57 + .../burn-crates/burn-std/LICENSE-APACHE | 1 + .../burn-crates/burn-std/LICENSE-MIT | 1 + .../burn-crates/burn-std/README.md | 7 + .../burn-crates/burn-std/src/id.rs | 69 + .../burn-crates/burn-std/src/lib.rs | 97 + .../burn-crates/burn-std/src/network.rs | 57 + .../burn-crates/burn-std/src/tensor/dtype.rs | 224 + .../burn-crates/burn-std/src/tensor/mod.rs | 221 + .../burn-std/src/tensor/quantization.rs | 393 + .../burn-crates/burn-std/src/tensor/shape.rs | 271 + .../burn-crates/burn-std/src/tensor/slice.rs | 937 + .../burn-crates/burn-store/Cargo.toml | 106 + .../burn-crates/burn-store/MIGRATION.md | 325 + .../burn-crates/burn-store/README.md | 77 + .../burn-store/benches/download_resnet18.py | 82 + .../benches/generate_unified_models.py | 175 + .../burn-store/benches/resnet18_loading.rs | 213 + .../burn-store/benches/unified_loading.rs | 332 + .../burn-store/benches/unified_saving.rs | 183 + .../burn-store/benches/zero_copy_loading.rs | 596 + .../burn-store/examples/burnpack_inspect.rs | 148 + .../burn-store/pytorch-tests/Cargo.toml | 13 + .../burn-store/pytorch-tests/src/lib.rs | 1 + .../burn-store/pytorch-tests/tests/backend.rs | 1 + .../tests/batch_norm/export_weights.py | 41 + .../pytorch-tests/tests/batch_norm/mod.rs | 62 + .../tests/boolean/export_weights.py | 38 + .../pytorch-tests/tests/boolean/mod.rs | 58 + .../tests/buffer/export_weights.py | 38 + .../pytorch-tests/tests/buffer/mod.rs | 53 + .../tests/complex_nested/export_weights.py | 69 + .../pytorch-tests/tests/complex_nested/mod.rs | 240 + .../tests/config/export_weights.py | 60 + .../pytorch-tests/tests/config/mod.rs | 53 + .../tests/conv1d/export_weights.py | 39 + .../pytorch-tests/tests/conv1d/mod.rs | 97 + .../tests/conv2d/export_weights.py | 39 + .../pytorch-tests/tests/conv2d/mod.rs | 134 + .../tests/conv_transpose1d/export_weights.py | 39 + .../tests/conv_transpose1d/mod.rs | 87 + .../tests/conv_transpose2d/export_weights.py | 39 + .../tests/conv_transpose2d/mod.rs | 99 + .../tests/embedding/export_weights.py | 37 + .../pytorch-tests/tests/embedding/mod.rs | 86 + .../tests/enum_module/export_weights.py | 60 + .../pytorch-tests/tests/enum_module/mod.rs | 197 + .../tests/group_norm/export_weights.py | 37 + .../pytorch-tests/tests/group_norm/mod.rs | 90 + .../tests/integer/export_weights.py | 38 + .../pytorch-tests/tests/integer/mod.rs | 72 + .../tests/key_remap/export_weights.py | 48 + .../pytorch-tests/tests/key_remap/mod.rs | 118 + .../tests/key_remap_chained/export_weights.py | 57 + .../tests/key_remap_chained/mod.rs | 179 + .../tests/layer_norm/export_weights.py | 37 + .../pytorch-tests/tests/layer_norm/mod.rs | 82 + .../tests/linear/export_weights.py | 60 + .../pytorch-tests/tests/linear/mod.rs | 154 + .../missing_module_field/export_weights.py | 24 + .../tests/missing_module_field/mod.rs | 37 + .../non_contiguous_indexes/export_weights.py | 42 + .../tests/non_contiguous_indexes/mod.rs | 110 + .../pytorch-tests/tests/test_mod.rs | 22 + .../tests/top_level_key/export_weights.py | 24 + .../pytorch-tests/tests/top_level_key/mod.rs | 48 + .../burn-store/safetensors-tests/Cargo.toml | 13 + .../burn-store/safetensors-tests/src/lib.rs | 1 + .../safetensors-tests/tests/backend.rs | 1 + .../tests/multi_layer/mod.rs | 92 + .../tests/multi_layer/multi_layer.py | 49 + .../safetensors-tests/tests/test_mod.rs | 3 + .../burn-crates/burn-store/src/adapter.rs | 663 + .../burn-crates/burn-store/src/applier.rs | 608 + .../burn-store/src/apply_result.rs | 300 + .../burn-store/src/burnpack/base.rs | 231 + .../burn-store/src/burnpack/mod.rs | 62 + .../burn-store/src/burnpack/reader.rs | 761 + .../burn-store/src/burnpack/store.rs | 507 + .../src/burnpack/tests/alignment.rs | 434 + .../src/burnpack/tests/edge_cases.rs | 365 + .../burn-store/src/burnpack/tests/header.rs | 61 + .../burn-store/src/burnpack/tests/helpers.rs | 19 + .../burn-store/src/burnpack/tests/mod.rs | 11 + .../burn-store/src/burnpack/tests/reader.rs | 775 + .../src/burnpack/tests/round_trip.rs | 606 + .../burn-store/src/burnpack/tests/store.rs | 1175 + .../burn-store/src/burnpack/tests/writer.rs | 744 + .../src/burnpack/tests/zero_copy.rs | 211 + .../burn-store/src/burnpack/writer.rs | 331 + .../burn-crates/burn-store/src/collector.rs | 1137 + .../burn-crates/burn-store/src/filter.rs | 625 + .../burn-crates/burn-store/src/keyremapper.rs | 674 + .../burn-crates/burn-store/src/lib.rs | 118 + .../burn-store/src/pytorch/lazy_data.rs | 567 + .../burn-crates/burn-store/src/pytorch/mod.rs | 48 + .../burn-store/src/pytorch/pickle_reader.rs | 1528 + .../burn-store/src/pytorch/reader.rs | 1133 + .../burn-store/src/pytorch/store.rs | 442 + .../burn-store/src/pytorch/tests/mod.rs | 2 + .../reader/create_legacy_with_offsets.py | 76 + .../pytorch/tests/reader/create_tar_format.py | 361 + .../src/pytorch/tests/reader/mod.rs | 1223 + .../src/pytorch/tests/reader/simple_legacy.py | 25 + .../src/pytorch/tests/reader/test_data.py | 227 + .../tests/reader/test_data/tar_2d_tensor.tar | Bin 0 -> 10240 bytes .../tests/reader/test_data/tar_float32.tar | Bin 0 -> 10240 bytes .../tests/reader/test_data/tar_float64.tar | Bin 0 -> 10240 bytes .../tests/reader/test_data/tar_int64.tar | Bin 0 -> 10240 bytes .../reader/test_data/tar_multi_dtype.tar | Bin 0 -> 10240 bytes .../reader/test_data/tar_weight_bias.tar | Bin 0 -> 10240 bytes .../burn-store/src/pytorch/tests/store/mod.rs | 1200 + .../store/test_data/generate_enum_test.py | 81 + .../burn-store/src/safetensors/mod.rs | 322 + .../burn-store/src/safetensors/store.rs | 1091 + .../src/safetensors/tests/adapter.rs | 193 + .../src/safetensors/tests/direct_access.rs | 341 + .../src/safetensors/tests/error_handling.rs | 51 + .../src/safetensors/tests/file_io.rs | 288 + .../src/safetensors/tests/filtering.rs | 169 + .../src/safetensors/tests/integration.rs | 172 + .../src/safetensors/tests/metadata.rs | 115 + .../src/safetensors/tests/mixed_datatypes.rs | 442 + .../burn-store/src/safetensors/tests/mod.rs | 12 + .../safetensors/tests/multi_layer_verify.rs | 114 + .../src/safetensors/tests/pytorch_import.rs | 204 + .../src/safetensors/tests/round_trip.rs | 106 + .../burn-store/src/tensor_snapshot.rs | 618 + .../burn-crates/burn-store/src/traits.rs | 288 + .../burn-crates/burn-tch/Cargo.toml | 38 + .../burn-crates/burn-tch/LICENSE-APACHE | 1 + .../burn-crates/burn-tch/LICENSE-MIT | 1 + .../burn-crates/burn-tch/README.md | 246 + .../burn-crates/burn-tch/build.rs | 243 + .../burn-crates/burn-tch/src/backend.rs | 175 + .../burn-crates/burn-tch/src/bin/cpu.rs | 14 + .../burn-crates/burn-tch/src/bin/cuda.rs | 19 + .../burn-crates/burn-tch/src/bin/mps.rs | 16 + .../src/cuda_hack/dummy_cuda_dependency.cpp | 28 + .../src/cuda_hack/fake_cuda_dependency.cpp | 5 + .../burn-crates/burn-tch/src/element.rs | 51 + .../burn-crates/burn-tch/src/lib.rs | 14 + .../burn-tch/src/ops/activation.rs | 37 + .../burn-crates/burn-tch/src/ops/base.rs | 737 + .../burn-tch/src/ops/bool_tensor.rs | 219 + .../burn-tch/src/ops/int_tensor.rs | 501 + .../burn-crates/burn-tch/src/ops/mod.rs | 10 + .../burn-crates/burn-tch/src/ops/module.rs | 473 + .../burn-crates/burn-tch/src/ops/qtensor.rs | 140 + .../burn-crates/burn-tch/src/ops/tensor.rs | 539 + .../burn-tch/src/ops/transaction.rs | 5 + .../burn-crates/burn-tch/src/tensor.rs | 507 + .../burn-tensor-testgen/Cargo.toml | 20 + .../burn-tensor-testgen/LICENSE-APACHE | 1 + .../burn-tensor-testgen/LICENSE-MIT | 1 + .../burn-crates/burn-tensor-testgen/README.md | 6 + .../burn-tensor-testgen/src/lib.rs | 130 + .../burn-crates/burn-tensor/Cargo.toml | 62 + .../burn-crates/burn-tensor/LICENSE-APACHE | 1 + .../burn-crates/burn-tensor/LICENSE-MIT | 1 + .../burn-crates/burn-tensor/README.md | 12 + .../burn-crates/burn-tensor/katex-header.html | 1 + .../burn-crates/burn-tensor/src/device.rs | 465 + .../burn-crates/burn-tensor/src/lib.rs | 23 + .../burn-tensor/src/tensor/activation/base.rs | 647 + .../burn-tensor/src/tensor/activation/mod.rs | 3 + .../burn-tensor/src/tensor/api/autodiff.rs | 75 + .../burn-tensor/src/tensor/api/base.rs | 3319 + .../burn-tensor/src/tensor/api/bool.rs | 429 + .../src/tensor/api/cartesian_grid.rs | 56 + .../burn-tensor/src/tensor/api/check.rs | 1551 + .../burn-tensor/src/tensor/api/float.rs | 1011 + .../burn-tensor/src/tensor/api/fmod.rs | 111 + .../burn-tensor/src/tensor/api/int.rs | 182 + .../burn-tensor/src/tensor/api/mod.rs | 27 + .../burn-tensor/src/tensor/api/numeric.rs | 1255 + .../burn-tensor/src/tensor/api/options.rs | 116 + .../burn-tensor/src/tensor/api/orderable.rs | 1155 + .../burn-tensor/src/tensor/api/pad.rs | 363 + .../burn-tensor/src/tensor/api/take.rs | 98 + .../burn-tensor/src/tensor/api/transaction.rs | 57 + .../burn-tensor/src/tensor/api/trunc.rs | 42 + .../src/tensor/grid/affine_grid.rs | 60 + .../burn-tensor/src/tensor/grid/meshgrid.rs | 107 + .../burn-tensor/src/tensor/grid/mod.rs | 68 + .../src/tensor/linalg/cosine_similarity.rs | 48 + .../burn-tensor/src/tensor/linalg/diag.rs | 44 + .../src/tensor/linalg/lu_decomposition.rs | 79 + .../burn-tensor/src/tensor/linalg/matvec.rs | 59 + .../burn-tensor/src/tensor/linalg/mod.rs | 42 + .../burn-tensor/src/tensor/linalg/outer.rs | 77 + .../burn-tensor/src/tensor/linalg/trace.rs | 24 + .../src/tensor/linalg/vector_norm.rs | 291 + .../burn-tensor/src/tensor/loss/mod.rs | 23 + .../burn-crates/burn-tensor/src/tensor/mod.rs | 67 + .../burn-tensor/src/tensor/module.rs | 555 + .../burn-tensor/src/tensor/named/base.rs | 85 + .../burn-tensor/src/tensor/named/dims.rs | 95 + .../burn-tensor/src/tensor/named/matmul.rs | 59 + .../burn-tensor/src/tensor/named/mod.rs | 7 + .../burn-tensor/src/tensor/named/swap_dims.rs | 62 + .../burn-tensor/src/tensor/quantization.rs | 52 + .../burn-tensor/src/tensor/report.rs | 106 + .../burn-tensor/src/tensor/stats/mod.rs | 74 + .../burn-crates/burn-train/Cargo.toml | 80 + .../burn-crates/burn-train/LICENSE-APACHE | 1 + .../burn-crates/burn-train/LICENSE-MIT | 1 + .../burn-crates/burn-train/README.md | 6 + .../src/checkpoint/async_checkpoint.rs | 171 + .../burn-train/src/checkpoint/base.rs | 51 + .../burn-train/src/checkpoint/file.rs | 86 + .../burn-train/src/checkpoint/mod.rs | 9 + .../src/checkpoint/strategy/base.rs | 34 + .../src/checkpoint/strategy/composed.rs | 146 + .../src/checkpoint/strategy/lastn.rs | 56 + .../src/checkpoint/strategy/metric.rs | 136 + .../burn-train/src/checkpoint/strategy/mod.rs | 9 + .../burn-crates/burn-train/src/components.rs | 66 + .../burn-train/src/evaluator/base.rs | 72 + .../burn-train/src/evaluator/builder.rs | 212 + .../burn-train/src/evaluator/components.rs | 25 + .../burn-train/src/evaluator/mod.rs | 7 + .../src/learner/application_logger.rs | 69 + .../burn-train/src/learner/base.rs | 255 + .../burn-train/src/learner/classification.rs | 159 + .../burn-train/src/learner/early_stopping.rs | 298 + .../burn-crates/burn-train/src/learner/mod.rs | 24 + .../burn-train/src/learner/regression.rs | 46 + .../burn-train/src/learner/rl/checkpointer.rs | 75 + .../burn-train/src/learner/rl/components.rs | 115 + .../src/learner/rl/env_runner/async_runner.rs | 703 + .../src/learner/rl/env_runner/base.rs | 343 + .../src/learner/rl/env_runner/mod.rs | 302 + .../burn-train/src/learner/rl/mod.rs | 15 + .../burn-train/src/learner/rl/off_policy.rs | 189 + .../burn-train/src/learner/rl/output.rs | 32 + .../burn-train/src/learner/rl/paradigm.rs | 525 + .../burn-train/src/learner/rl/strategy.rs | 99 + .../burn-train/src/learner/sequence.rs | 131 + .../burn-train/src/learner/summary.rs | 475 + .../burn-train/src/learner/supervised/mod.rs | 7 + .../src/learner/supervised/paradigm.rs | 488 + .../src/learner/supervised/step/mod.rs | 2 + .../src/learner/supervised/step/train.rs | 151 + .../src/learner/supervised/strategies/base.rs | 154 + .../supervised/strategies/ddp/README.md | 17 + .../supervised/strategies/ddp/epoch.rs | 234 + .../learner/supervised/strategies/ddp/mod.rs | 5 + .../supervised/strategies/ddp/strategy.rs | 140 + .../supervised/strategies/ddp/worker.rs | 135 + .../src/learner/supervised/strategies/mod.rs | 8 + .../supervised/strategies/multi/epoch.rs | 219 + .../supervised/strategies/multi/mod.rs | 4 + .../supervised/strategies/multi/strategy.rs | 100 + .../supervised/strategies/single/epoch.rs | 136 + .../supervised/strategies/single/mod.rs | 4 + .../supervised/strategies/single/strategy.rs | 108 + .../burn-train/src/learner/train_val.rs | 129 + .../burn-crates/burn-train/src/lib.rs | 118 + .../burn-train/src/logger/async_logger.rs | 91 + .../burn-crates/burn-train/src/logger/base.rs | 9 + .../burn-crates/burn-train/src/logger/file.rs | 46 + .../burn-train/src/logger/in_memory.rs | 16 + .../burn-train/src/logger/metric.rs | 375 + .../burn-crates/burn-train/src/logger/mod.rs | 11 + .../burn-crates/burn-train/src/metric/acc.rs | 164 + .../burn-train/src/metric/auroc.rs | 231 + .../burn-crates/burn-train/src/metric/base.rs | 278 + .../burn-crates/burn-train/src/metric/cer.rs | 239 + .../burn-train/src/metric/classification.rs | 33 + .../burn-train/src/metric/confusion_stats.rs | 351 + .../burn-train/src/metric/cpu_temp.rs | 76 + .../burn-train/src/metric/cpu_use.rs | 103 + .../burn-crates/burn-train/src/metric/cuda.rs | 108 + .../burn-train/src/metric/fbetascore.rs | 259 + .../burn-train/src/metric/hamming.rs | 198 + .../burn-train/src/metric/iteration.rs | 89 + .../burn-train/src/metric/learning_rate.rs | 67 + .../burn-crates/burn-train/src/metric/loss.rs | 89 + .../burn-train/src/metric/memory_use.rs | 111 + .../burn-crates/burn-train/src/metric/mod.rs | 71 + .../burn-train/src/metric/perplexity.rs | 438 + .../burn-train/src/metric/precision.rs | 231 + .../src/metric/processor/async_wrapper.rs | 137 + .../burn-train/src/metric/processor/base.rs | 126 + .../burn-train/src/metric/processor/full.rs | 257 + .../src/metric/processor/metrics.rs | 341 + .../src/metric/processor/minimal.rs | 76 + .../burn-train/src/metric/processor/mod.rs | 77 + .../src/metric/processor/rl_metrics.rs | 268 + .../src/metric/processor/rl_processor.rs | 177 + .../burn-train/src/metric/recall.rs | 226 + .../burn-train/src/metric/rl/cum_reward.rs | 78 + .../burn-train/src/metric/rl/ep_len.rs | 71 + .../src/metric/rl/exploration_rate.rs | 78 + .../burn-train/src/metric/rl/mod.rs | 7 + .../burn-train/src/metric/state.rs | 144 + .../burn-train/src/metric/store/aggregate.rs | 251 + .../burn-train/src/metric/store/base.rs | 105 + .../burn-train/src/metric/store/client.rs | 171 + .../burn-train/src/metric/store/log.rs | 72 + .../burn-train/src/metric/store/mod.rs | 9 + .../burn-train/src/metric/top_k_acc.rs | 185 + .../burn-train/src/metric/vision/dice.rs | 345 + .../src/metric/vision/dists/l2pool.rs | 164 + .../src/metric/vision/dists/metric.rs | 498 + .../burn-train/src/metric/vision/dists/mod.rs | 14 + .../src/metric/vision/dists/vgg16_l2pool.rs | 169 + .../src/metric/vision/dists/weights.rs | 128 + .../src/metric/vision/lpips/alexnet.rs | 100 + .../src/metric/vision/lpips/metric.rs | 802 + .../burn-train/src/metric/vision/lpips/mod.rs | 21 + .../src/metric/vision/lpips/squeezenet.rs | 157 + .../burn-train/src/metric/vision/lpips/vgg.rs | 116 + .../src/metric/vision/lpips/weights.rs | 228 + .../burn-train/src/metric/vision/mod.rs | 11 + .../burn-train/src/metric/vision/psnr.rs | 616 + .../burn-train/src/metric/vision/ssim.rs | 875 + .../burn-crates/burn-train/src/metric/wer.rs | 225 + .../burn-train/src/renderer/base.rs | 198 + .../burn-train/src/renderer/cli.rs | 45 + .../burn-train/src/renderer/mod.rs | 34 + .../burn-train/src/renderer/tui/base.rs | 106 + .../burn-train/src/renderer/tui/controls.rs | 46 + .../src/renderer/tui/full_history.rs | 295 + .../src/renderer/tui/metric_numeric.rs | 326 + .../src/renderer/tui/metric_text.rs | 124 + .../burn-train/src/renderer/tui/mod.rs | 23 + .../burn-train/src/renderer/tui/plot_utils.rs | 37 + .../burn-train/src/renderer/tui/popup.rs | 144 + .../burn-train/src/renderer/tui/progress.rs | 328 + .../src/renderer/tui/recent_history.rs | 249 + .../burn-train/src/renderer/tui/renderer.rs | 533 + .../burn-train/src/renderer/tui/status.rs | 111 + .../burn-crates/burn-vision/Cargo.toml | 71 + .../burn-vision/src/backends/cpu/base.rs | 42 + .../src/backends/cpu/connected_components.rs | 279 + .../Spaghetti_center_line_forest_code.rs | 1954 + .../Spaghetti_first_line_forest_code.rs | 223 + .../spaghetti/Spaghetti_forest_labels.rs | 191 + .../Spaghetti_last_line_forest_code.rs | 787 + .../Spaghetti_single_line_forest_code.rs | 91 + .../cpu/connected_components/spaghetti/mod.rs | 273 + .../Spaghetti4C_center_line_forest_code.rs | 42 + .../Spaghetti4C_first_line_forest_code.rs | 31 + .../spaghetti_4c/Spaghetti4C_forest_labels.rs | 21 + .../connected_components/spaghetti_4c/mod.rs | 102 + .../burn-vision/src/backends/cpu/mod.rs | 10 + .../src/backends/cpu/morphology/filter.rs | 720 + .../backends/cpu/morphology/filter_engine.rs | 415 + .../src/backends/cpu/morphology/mod.rs | 300 + .../burn-vision/src/backends/cpu/nms.rs | 212 + .../burn-vision/src/backends/cpu/ops.rs | 54 + .../hardware_accelerated.rs | 624 + .../backends/cube/connected_components/mod.rs | 63 + .../cube/connected_components/prefix_sum.rs | 262 + .../burn-vision/src/backends/cube/mod.rs | 2 + .../burn-vision/src/backends/cube/ops.rs | 211 + .../burn-vision/src/backends/mod.rs | 5 + .../burn-crates/burn-vision/src/base.rs | 19 + .../burn-crates/burn-vision/src/lib.rs | 31 + .../burn-crates/burn-vision/src/ops/base.rs | 340 + .../burn-crates/burn-vision/src/ops/mod.rs | 3 + .../burn-crates/burn-vision/src/tensor.rs | 186 + .../burn-crates/burn-vision/src/tests/mod.rs | 27 + .../burn-vision/src/transform/mod.rs | 3 + .../burn-vision/src/transform/transform2d.rs | 229 + .../burn-crates/burn-vision/src/utils/mod.rs | 3 + .../burn-crates/burn-vision/src/utils/save.rs | 191 + .../burn-vision/tests/common/mod.rs | 82 + .../burn-vision/tests/connected_components.rs | 144 + .../tests/images/morphology/Base_1.png | Bin 0 -> 422 bytes .../tests/images/morphology/Base_2.png | Bin 0 -> 1636 bytes .../images/morphology/Dilate_1_3x5_Cross.png | Bin 0 -> 1267 bytes .../images/morphology/Dilate_1_3x5_Rect.png | Bin 0 -> 1262 bytes .../images/morphology/Dilate_1_5x5_Cross.png | Bin 0 -> 1262 bytes .../morphology/Dilate_1_5x5_Ellipse.png | Bin 0 -> 1257 bytes .../images/morphology/Dilate_1_5x5_Rect.png | Bin 0 -> 396 bytes .../images/morphology/Dilate_2_3x5_Cross.png | Bin 0 -> 2927 bytes .../images/morphology/Dilate_2_3x5_Rect.png | Bin 0 -> 2871 bytes ...te_2_5x7_Cross_ANCHOR_BORDER_REPLICATE.png | Bin 0 -> 3148 bytes .../morphology/Dilate_2_5x7_Rect_ANCHOR.png | Bin 0 -> 3076 bytes .../Dilate_2_7x7_Cross_BORDER_REFLECT.png | Bin 0 -> 3162 bytes .../Dilate_2_7x7_Cross_BORDER_REFLECT101.png | Bin 0 -> 3162 bytes .../Dilate_2_7x7_Cross_BORDER_REPLICATE.png | Bin 0 -> 3162 bytes .../Dilate_2_7x7_Rect_BORDER_REFLECT.png | Bin 0 -> 3094 bytes .../Dilate_2_7x7_Rect_BORDER_REFLECT101.png | Bin 0 -> 3094 bytes .../Dilate_2_7x7_Rect_BORDER_REPLICATE.png | Bin 0 -> 3094 bytes .../images/morphology/Erode_1_5x5_Cross.png | Bin 0 -> 1247 bytes .../images/morphology/Erode_1_5x5_Ellipse.png | Bin 0 -> 1243 bytes .../images/morphology/Erode_1_5x5_Rect.png | Bin 0 -> 370 bytes .../burn-vision/tests/morphology.rs | 638 + .../burn-crates/burn-vision/tests/nms.rs | 92 + .../burn-crates/burn-wgpu/Cargo.toml | 56 + .../burn-crates/burn-wgpu/LICENSE-APACHE | 1 + .../burn-crates/burn-wgpu/LICENSE-MIT | 1 + .../burn-crates/burn-wgpu/README.md | 59 + .../burn-crates/burn-wgpu/src/lib.rs | 189 + .../burn-crates/burn/Cargo.toml | 207 + .../burn-crates/burn/LICENSE-APACHE | 1 + .../burn-crates/burn/LICENSE-MIT | 1 + .../burn-crates/burn/README.md | 1 + .../burn-crates/burn/src/backend.rs | 72 + .../burn-crates/burn/src/collective.rs | 1 + .../burn-crates/burn/src/lib.rs | 187 + crates/stable-diffusion-burn/img0.png | Bin 0 -> 693444 bytes .../python/autoencoder.py | 92 + .../python/bpe_simple_vocab_16e6.txt.gz | Bin 0 -> 1356917 bytes crates/stable-diffusion-burn/python/clip.py | 40 + crates/stable-diffusion-burn/python/dump.py | 652 + .../python/requirements.txt | 1 + crates/stable-diffusion-burn/python/save.py | 100 + .../python/stablediffusion.py | 14 + crates/stable-diffusion-burn/python/test.py | 54 + .../stable-diffusion-burn/python/test_tiny.py | 41 + .../stable-diffusion-burn/python/tokenizer.py | 145 + crates/stable-diffusion-burn/python/unet.py | 153 + crates/stable-diffusion-burn/src/backend.rs | 139 + .../src/bin/convert/main.rs | 58 + .../src/bin/sample/main.rs | 142 + crates/stable-diffusion-burn/src/lib.rs | 3 + .../src/model/attention.rs | 56 + .../src/model/autoencoder/load.rs | 195 + .../src/model/autoencoder/mod.rs | 603 + .../src/model/clip/load.rs | 88 + .../src/model/clip/mod.rs | 226 + .../src/model/groupnorm/load.rs | 35 + .../src/model/groupnorm/mod.rs | 82 + .../stable-diffusion-burn/src/model/load.rs | 177 + crates/stable-diffusion-burn/src/model/mod.rs | 11 + .../stable-diffusion-burn/src/model/silu.rs | 17 + .../src/model/stablediffusion/load.rs | 30 + .../src/model/stablediffusion/mod.rs | 237 + .../src/model/unet/load.rs | 300 + .../src/model/unet/mod.rs | 740 + crates/stable-diffusion-burn/src/token.rs | 54 + crates/stable-diffusion-burn/src/tokenizer.rs | 222 + 1605 files changed, 537032 insertions(+), 2 deletions(-) create mode 100644 crates/stable-diffusion-burn/Cargo.toml create mode 100644 crates/stable-diffusion-burn/LICENSE create mode 100644 crates/stable-diffusion-burn/README.md create mode 100644 crates/stable-diffusion-burn/bpe_simple_vocab_16e6.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/builder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/retro_forward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/state.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/strategy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/grads.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/node.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/requirement.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/traversal.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/maxmin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/sort.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/client.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/graph.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/memory_management.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/server.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/utils.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/.cargo/config.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/abs.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/add.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/aggregation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/bridge.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/broadcast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/ceil.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/checkpoint.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/complex.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross_entropy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummax.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumprod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumsum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/deform_conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/div.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/erf.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/exp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/expand.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/flip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/floor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gather_scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gelu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gradients.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log1p.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log_sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/matmul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxmin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/memory_management.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/multithread.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nearest_interpolate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/neg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nonzero.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/permute.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/pow.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/recip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/remainder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/reshape.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/round.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/softmax.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sort.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sqrt.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sub.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/transpose.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/trig.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/unfold.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/autodiff.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/avg_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/bernoulli.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/clamp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/contiguous.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cross.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/gather.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_fill.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_where.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d_backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/normal.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/quantization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/reduce.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/unary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/uniform.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/reduce_broadcasted.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fusion.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/all.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/any.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/argwhere_nonzero.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/comparison.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/create_like.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/expand.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/flip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/full.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/gather_scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/init.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/logical.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/movedim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/permute.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/reshape.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/stack.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/take.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/transpose.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/tri_mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/unfold.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/clone_invariance.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/celu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/elu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/gelu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/glu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/hard_sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/leaky_relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/log_sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mish.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/prelu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/quiet_softmax.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/selu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/silu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmax.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softplus.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softsign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/tanh_activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/thresholded_relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/affine_grid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/meshgrid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/cosine_similarity.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/diag.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/lu_decomposition.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/matvec.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/outer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/trace.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/vector_norm.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/attention.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bicubic_interpolate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bilinear_interpolate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/deform_conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/forward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/linear.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/nearest_interpolate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/unfold4d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/abs.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/add.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/aggregation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/all.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/any.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/arg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/ceil.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/chunk.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/clamp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/close.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/comparison.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/create_like.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cross.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cumulative.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/div.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/dot.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/erf.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/exp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/expand.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/finite.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flatten.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/floor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/fmod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/full.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/gather_scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/grid_sample.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/inf.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/init.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/iter_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log1p.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/matmul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/maxmin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/movedim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/nan.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/narrow.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/neg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/one_hot.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/padding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/permute.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf_scalar.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/prod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/random.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/recip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/remainder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/reshape.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/round.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sort_argsort.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/split.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sqrt.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/square.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/squeeze.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/stack.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sub.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/take.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/topk.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transpose.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/tri.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trig.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trunc.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/unfold.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/primitive.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/calibration.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/data.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/abs.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/add.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/aggregation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/all.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/any.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/arg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/ceil.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/chunk.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/clamp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cos.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cosh.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/div.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/erf.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/exp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/expand.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/flip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/floor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/gather_scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log1p.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/map_comparison.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/maxmin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/narrow.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/neg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/permute.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf_scalar.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/recip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/remainder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/reshape.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/round.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sin.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sinh.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sort_argsort.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/split.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sqrt.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/stack.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sub.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tan.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tanh.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/topk.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/transpose.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/matmul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/quantize.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/scheme.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/cov.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/display.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/eye.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/median.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/var.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/abs.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/add.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/aggregation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/all.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/any.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange_step.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/bitwise.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cartesian_grid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/chunk.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/comparison.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/create_like.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cumulative.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/div.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/expand.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/flip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/full.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/gather_scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/init.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/matmul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/movedim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/one_hot.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/permute.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/random.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/remainder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/reshape.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/roll.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sort_argsort.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/stack.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sub.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/take.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/topk.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/transpose.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/tri.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/unfold.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/primitive.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/multi_threads.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/device.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/argwhere.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/cat.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/attention.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/conv.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/grid_sample.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/pool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/unfold.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/sort.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/primitive.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/compare.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/distribution.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/cast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/scalar.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/alias.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/container.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/kind.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/autodiff.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/bool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/float.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/int.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/numeric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/ordered.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/calibration.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/parameters.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/scheme.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/element.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/candle_utils.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/utils.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-candle/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/global.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/node.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/test_launcher.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/shared.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/api.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/config.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/centralized.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/ring.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/sync.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/tree.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/worker.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/state.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/shared.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/centralized.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/op.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/ring.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/tree.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/centralized.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/op.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/tree.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/client.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/centralized.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/op.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/tree.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/server.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/tensor_map.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/all_reduce.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/broadcast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/reduce.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/data_service.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/util.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/client.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/server.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/config.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batcher.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/builder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/multithread.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/split.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/strategy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/data/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/display.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/initializer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/constant.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/id.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/primitive.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/running.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/visitor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/quantize.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/module/reinit.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/file.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/memory.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/primitive.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/recorder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/adapter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/data.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/de.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/error.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/ser.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/settings.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/record/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/src/vision.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_config.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_record.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_record_resilience.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cpu/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cpu/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cpu/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/io.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/ir.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/kernel.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/view.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/fuser.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/executor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/input.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/output.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/plan.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/runner.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/planner.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/settings.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/block.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/fuser.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/fuser.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/optimization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/args.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/fuser.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/optimization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/args.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/fuser.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/optimization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/block.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full_analyzer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/launch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/optimization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/unit.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/element.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/fusion.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_float.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_int.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/bool_cast.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/clamp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/comparison.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/contiguous.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/fallback.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/launch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/fallback.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/launch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/col2im.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/transpose_direct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv_transpose2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/direct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/launch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/im2col.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/tune_key.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cross.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/bilinear.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/flip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/gather.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/repeat_dim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/scatter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice_assign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bicubic.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bilinear.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest_backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_fill.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_where.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/utils.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d_backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d_backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d_backward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/bernoulli.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/normal.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/uniform.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/dequantize.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/quantize.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/tune.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_float.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_int.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_numeric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/utils.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/numeric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/source.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/quantization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tune_key.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cuda/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cuda/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-cuda/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/hf_dataset.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/speech_commands.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/speech_commands.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/dataframe.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/fake.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/in_memory.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/iterator.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/sqlite.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/ag_news.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/text_folder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/downloader.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/importer.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/composed.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mapper.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/options.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/partial.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/sampler.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/selection.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/shuffle.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/window.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/cifar.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/image_folder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mnist.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset-fmt.csv create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.csv create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.json create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset_coco.json create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/orange/dot.jpg create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/red/dot.jpg create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/red/dot.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder_coco/dot_triangle.jpg create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder_coco/one_dot.jpg create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder_coco/two_dots_and_triangle.jpg create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_checkerboard.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_random_2colors.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_random_3colors.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/sqlite-dataset.db create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample1.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample2.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample1.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample2.txt create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_enum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_struct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_enum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_struct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/display.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_enum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_struct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/codegen.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_enum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_struct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/attribute.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/enum_variant.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/field.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/generics.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/build.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/device.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/macros.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/client.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/binary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/unary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/block.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/merging.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/blocks.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/stream.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/server.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/context.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/explorer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/ordering.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/policy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/processor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/tests.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/validator.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/memory_checks.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/multi.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/execution.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/shared_tensors.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/index.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-fusion/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/builder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/handle.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/operation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/scalar.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ir/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/build.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/element.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/adaptive_avgpool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/avgpool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/conv.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/deform_conv.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/grid_sample.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/interpolate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/macros.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/matmul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/maxpool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/padding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/quantization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/avgpool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary_elemwise.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/cmp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/conv.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/maxpool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/unary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/parallel.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/rand.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/sharing.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/storage.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/activation_wrapper.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/celu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/elu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/gelu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/glu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_shrink.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_swish.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/leaky_relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/prelu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/selu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/shrink.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/sigmoid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/soft_shrink.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softplus.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softsign.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/swiglu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/tanh.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/thresholded_relu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/binary_cross_entropy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cosine_embedding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cross_entropy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/ctc.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/huber.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/kldiv.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/lp_loss.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mse.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/poisson.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/reduction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/smooth_l1.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/cross_attention.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mask.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mha.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/autoregressive.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/checks.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose3d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/deform_conv2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/dropout.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/embedding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/linear.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/noise.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/batch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/group.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/instance.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/layer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/normalization_wrapper.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/rms.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool1d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pos_encoding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/basic.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gate_controller.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gru.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/lstm.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rope_encoding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/decoder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/encoder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/pwff.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/unfold.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/src/padding.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-nn/tests/quantize.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/burnpack.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/conv.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/mlp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/model.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/safetensors.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/burnpack_tests.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/safetensors_tests.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/test_integration.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/composed.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/constant.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/cosine.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/exponential.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/linear.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/noam.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/step.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adagrad.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adam.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adamw.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/decay.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grad_accum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grads.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/lbfgs.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/momentum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/muon.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/rmsprop.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/sgd.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/adaptor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/v1.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/visitor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/channel.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/runner.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/worker.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/processor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/session.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/stream.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/task.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/async_policy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/slice_access.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rocm/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rocm/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-rocm/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/byte.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/direct.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/client/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/client/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/binary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/unary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/runner.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-router/src/types.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/id.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/network.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/dtype.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/quantization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/shape.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/slice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/MIGRATION.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/benches/download_resnet18.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/benches/generate_unified_models.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/benches/resnet18_loading.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_loading.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_saving.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/benches/zero_copy_loading.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/examples/burnpack_inspect.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/backend.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/test_mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/export_weights.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/mod.rs create mode 100755 crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/multi_layer.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/test_mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/adapter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/applier.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/apply_result.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/reader.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/store.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/alignment.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/edge_cases.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/header.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/helpers.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/reader.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/round_trip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/store.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/writer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/zero_copy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/writer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/collector.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/filter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/keyremapper.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/lazy_data.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/pickle_reader.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/reader.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/store.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_legacy_with_offsets.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_tar_format.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/simple_legacy.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_2d_tensor.tar create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_float32.tar create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_float64.tar create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_int64.tar create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_multi_dtype.tar create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_weight_bias.tar create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/test_data/generate_enum_test.py create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/store.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/adapter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/direct_access.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/error_handling.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/file_io.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/filtering.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/integration.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/metadata.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mixed_datatypes.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/multi_layer_verify.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/pytorch_import.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/round_trip.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/tensor_snapshot.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-store/src/traits.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/build.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cpu.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cuda.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/mps.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/dummy_cuda_dependency.cpp create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/fake_cuda_dependency.cpp create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/element.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/activation.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/bool_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/int_tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/qtensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tch/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/README.md create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-tensor/katex-header.html create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/device.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/autodiff.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/bool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/cartesian_grid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/check.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/float.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/fmod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/int.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/numeric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/options.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/orderable.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/pad.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/take.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/transaction.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/trunc.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/affine_grid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/meshgrid.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/cosine_similarity.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/diag.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/lu_decomposition.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/matvec.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/outer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/trace.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/vector_norm.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/loss/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/module.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/dims.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/matmul.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/swap_dims.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/quantization.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/report.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/stats/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/async_checkpoint.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/file.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/composed.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/lastn.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/metric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/components.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/builder.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/components.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/application_logger.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/classification.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/early_stopping.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/regression.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/checkpointer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/components.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/async_runner.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/off_policy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/output.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/paradigm.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/strategy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/sequence.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/summary.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/paradigm.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/train.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/epoch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/strategy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/worker.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/epoch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/strategy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/epoch.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/strategy.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/train_val.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/async_logger.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/file.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/in_memory.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/metric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/acc.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/auroc.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/classification.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/confusion_stats.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_temp.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_use.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cuda.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/fbetascore.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/hamming.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/iteration.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/learning_rate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/loss.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/memory_use.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/perplexity.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/precision.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/async_wrapper.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/full.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/metrics.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/minimal.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_metrics.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_processor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/recall.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/cum_reward.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/ep_len.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/exploration_rate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/state.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/aggregate.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/client.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/log.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/top_k_acc.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dice.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/l2pool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/metric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/vgg16_l2pool.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/weights.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/alexnet.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/metric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/squeezenet.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/vgg.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/weights.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/psnr.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/ssim.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/wer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/cli.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/controls.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/full_history.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_numeric.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_text.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/plot_utils.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/popup.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/progress.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/recent_history.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/renderer.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/status.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/Cargo.toml create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_center_line_forest_code.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_first_line_forest_code.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_forest_labels.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_last_line_forest_code.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_single_line_forest_code.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti_4c/Spaghetti4C_center_line_forest_code.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti_4c/Spaghetti4C_first_line_forest_code.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti_4c/Spaghetti4C_forest_labels.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti_4c/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/morphology/filter.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/morphology/filter_engine.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/morphology/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/nms.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/ops.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cube/connected_components/hardware_accelerated.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cube/connected_components/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cube/connected_components/prefix_sum.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cube/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cube/ops.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/ops/base.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/ops/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/tensor.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/tests/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/transform/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/transform/transform2d.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/utils/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/src/utils/save.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/common/mod.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/connected_components.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Base_1.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Base_2.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_1_3x5_Cross.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_1_3x5_Rect.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_1_5x5_Cross.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_1_5x5_Ellipse.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_1_5x5_Rect.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_3x5_Cross.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_3x5_Rect.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_5x7_Cross_ANCHOR_BORDER_REPLICATE.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_5x7_Rect_ANCHOR.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_7x7_Cross_BORDER_REFLECT.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_7x7_Cross_BORDER_REFLECT101.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_7x7_Cross_BORDER_REPLICATE.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_7x7_Rect_BORDER_REFLECT.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_7x7_Rect_BORDER_REFLECT101.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Dilate_2_7x7_Rect_BORDER_REPLICATE.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Erode_1_5x5_Cross.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Erode_1_5x5_Ellipse.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/images/morphology/Erode_1_5x5_Rect.png create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/morphology.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-vision/tests/nms.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-wgpu/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-wgpu/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn-wgpu/LICENSE-MIT create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-wgpu/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn-wgpu/src/lib.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn/Cargo.toml create mode 120000 crates/stable-diffusion-burn/burn-crates/burn/LICENSE-APACHE create mode 120000 crates/stable-diffusion-burn/burn-crates/burn/LICENSE-MIT create mode 120000 crates/stable-diffusion-burn/burn-crates/burn/README.md create mode 100644 crates/stable-diffusion-burn/burn-crates/burn/src/backend.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn/src/collective.rs create mode 100644 crates/stable-diffusion-burn/burn-crates/burn/src/lib.rs create mode 100644 crates/stable-diffusion-burn/img0.png create mode 100644 crates/stable-diffusion-burn/python/autoencoder.py create mode 100644 crates/stable-diffusion-burn/python/bpe_simple_vocab_16e6.txt.gz create mode 100644 crates/stable-diffusion-burn/python/clip.py create mode 100644 crates/stable-diffusion-burn/python/dump.py create mode 100644 crates/stable-diffusion-burn/python/requirements.txt create mode 100644 crates/stable-diffusion-burn/python/save.py create mode 100644 crates/stable-diffusion-burn/python/stablediffusion.py create mode 100644 crates/stable-diffusion-burn/python/test.py create mode 100644 crates/stable-diffusion-burn/python/test_tiny.py create mode 100644 crates/stable-diffusion-burn/python/tokenizer.py create mode 100644 crates/stable-diffusion-burn/python/unet.py create mode 100644 crates/stable-diffusion-burn/src/backend.rs create mode 100644 crates/stable-diffusion-burn/src/bin/convert/main.rs create mode 100644 crates/stable-diffusion-burn/src/bin/sample/main.rs create mode 100644 crates/stable-diffusion-burn/src/lib.rs create mode 100644 crates/stable-diffusion-burn/src/model/attention.rs create mode 100644 crates/stable-diffusion-burn/src/model/autoencoder/load.rs create mode 100644 crates/stable-diffusion-burn/src/model/autoencoder/mod.rs create mode 100644 crates/stable-diffusion-burn/src/model/clip/load.rs create mode 100644 crates/stable-diffusion-burn/src/model/clip/mod.rs create mode 100644 crates/stable-diffusion-burn/src/model/groupnorm/load.rs create mode 100644 crates/stable-diffusion-burn/src/model/groupnorm/mod.rs create mode 100644 crates/stable-diffusion-burn/src/model/load.rs create mode 100644 crates/stable-diffusion-burn/src/model/mod.rs create mode 100644 crates/stable-diffusion-burn/src/model/silu.rs create mode 100644 crates/stable-diffusion-burn/src/model/stablediffusion/load.rs create mode 100644 crates/stable-diffusion-burn/src/model/stablediffusion/mod.rs create mode 100644 crates/stable-diffusion-burn/src/model/unet/load.rs create mode 100644 crates/stable-diffusion-burn/src/model/unet/mod.rs create mode 100644 crates/stable-diffusion-burn/src/token.rs create mode 100644 crates/stable-diffusion-burn/src/tokenizer.rs diff --git a/.gitignore b/.gitignore index 7ea2bcb..a08ab7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ # Rust build artifacts target/ + +# Cargo lock file Cargo.lock -# Rust IDE files +# IDE files .vscode/ .idea/ @@ -131,3 +133,13 @@ demo/ # Additional Rust specific *.lock + +# Generated model files +*.mpk +*.pt +*.bin +*.safetensors +*.ckpt + +# User-specific data +user_data/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index a6559f9..71b32a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ clap = { version = "4.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" anyhow = "1.0" -stablediffusion = { path = "../stable-diffusion-burn" } +stablediffusion = { path = "./crates/stable-diffusion-burn" } burn = "0.14.0" burn-autodiff = "0.14.0" burn-ndarray = "0.14.0" diff --git a/crates/stable-diffusion-burn/Cargo.toml b/crates/stable-diffusion-burn/Cargo.toml new file mode 100644 index 0000000..00c23aa --- /dev/null +++ b/crates/stable-diffusion-burn/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "stablediffusion" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +wgpu-backend = ["burn-wgpu"] + +[dependencies.burn-wgpu] +package = "burn-wgpu" +git = "https://github.com/burn-rs/burn.git" +optional = true + +[dependencies] +burn = "0.14.0" +burn-ndarray = "0.14.0" +burn-tch = "0.14.0" +burn-autodiff = "0.14.0" +tch = "0.15.0" +serde = {version = "1.0.171", features = ["std", "derive"]} +npy = "0.4.0" +num-traits = "0.2.15" +rust_tokenizers = "8.1.0" +regex = "1.9.1" +image = "0.24.6" +cfg-if = "0.1" diff --git a/crates/stable-diffusion-burn/LICENSE b/crates/stable-diffusion-burn/LICENSE new file mode 100644 index 0000000..ed2a339 --- /dev/null +++ b/crates/stable-diffusion-burn/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Gadersd + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/stable-diffusion-burn/README.md b/crates/stable-diffusion-burn/README.md new file mode 100644 index 0000000..e92c52c --- /dev/null +++ b/crates/stable-diffusion-burn/README.md @@ -0,0 +1,75 @@ +# Stable-Diffusion-Burn + +Stable-Diffusion-Burn is a Rust-based project which ports the V1 stable diffusion model into the deep learning framework, Burn. This repository is licensed under the MIT Licence. + +## How To Use + +### Step 0: Install libtorch v2.4.1 + +### Step 1: Download the Model and Set Environment Variables + +Start by downloading the SDv1-4 model provided on HuggingFace. + +```bash +wget https://huggingface.co/Gadersd/Stable-Diffusion-Burn/resolve/main/SDv1-4.mpk +``` + +### Step 2: Run the Sample Binary + +Invoke the sample binary provided in the rust code. By default, torch is used. The WGPU backend is unstable for SD but may work well in the future as burn-wpu is optimized. + +```bash +# torch (at least 6 GB VRAM, possibly less) +# Arguments: [cuda, mps, cpu] + +# Cuda +cargo run --release --bin sample burn SDv1-4 7.5 20 "An ancient mossy stone." img cuda + +# Mps(Mac) +cargo run --release --bin sample burn SDv1-4 7.5 20 "An ancient mossy stone." img mps + +# wgpu (UNSTABLE) +# Arguments: +cargo run --release --features wgpu-backend --bin sample burn SDv1-4 7.5 20 "An ancient mossy stone." img +``` + +This command will generate an image according to the provided prompt, which will be saved as 'img0.png'. + +![An image of an ancient mossy stone](img0.png) + +### Optional: Extract and Convert a Fine-Tuned Model + +If users are interested in using a fine-tuned version of stable diffusion, the Python scripts provided in this project can be used to transform a weight dump into a Burn model file. This does not work on Windows. + +```bash +# Step into the Python directory +cd python + +# Download the model, this is just the base v1.4 model as an example +wget https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt + +# Install tinygrad +pip install -r requirements.txt + +# Extract the weights +CPU=1 python3 dump.py sd-v1-4.ckpt + +# Move the extracted weight folder out +mv params .. + +# Step out of the Python directory +cd .. + +# Convert the weights into a usable form +cargo run --release --bin convert params SDv1-4 +``` + +The binaries 'convert' and 'sample' are contained in Rust. Convert works on CPU whereas sample needs CUDA. + +Remember, `convert` should be used if you're planning on using the fine-tuned version of the stable diffusion. + +## License + +This project is licensed under MIT license. + +We wish you a productive time using this project. Enjoy! diff --git a/crates/stable-diffusion-burn/bpe_simple_vocab_16e6.txt b/crates/stable-diffusion-burn/bpe_simple_vocab_16e6.txt new file mode 100644 index 0000000..aff3ec8 --- /dev/null +++ b/crates/stable-diffusion-burn/bpe_simple_vocab_16e6.txt @@ -0,0 +1,262145 @@ +"bpe_simple_vocab_16e6.txt#version: 0.2 +i n +t h +a n +r e +a r +e r +th e +in g +o u +o n +s t +o r +e n +o n +a l +a t +e r +i t +i n +t o +r o +i s +l e +i c +a t +an d +e d +o f +c h +o r +e s +i l +e l +s t +a c +o m +a m +l o +a n +a y +s h +r i +l i +t i +f or +n e +ð Ł +r a +h a +d e +o l +v e +s i +u r +a l +s e +' s +u n +d i +b e +l a +w h +o o +d ay +e n +m a +n o +l e +t o +ou r +i r +g h +w it +i t +y o +a s +s p +th is +t s +at i +yo u +wit h +a d +i s +a b +l y +w e +th e +t e +a s +a g +v i +p p +s u +h o +m y +. . +b u +c om +s e +er s +m e +m e +al l +c on +m o +k e +g e +ou t +en t +c o +f e +v er +a r +f ro +a u +p o +c e +gh t +ar e +s s +fro m +c h +t r +ou n +on e +b y +d o +t h +w or +er e +k e +p ro +f or +d s +b o +t a +w e +g o +h e +t er +in g +d e +b e +ati on +m or +a y +e x +il l +p e +k s +s c +l u +f u +q u +v er +ðŁ ĺ +j u +m u +at e +an d +v e +k ing +m ar +o p +h i +.. . +p re +a d +r u +th at +j o +o f +c e +ne w +a m +a p +g re +s s +d u +no w +y e +t ing +y our +it y +n i +c i +p ar +g u +f i +a f +p er +t er +u p +s o +g i +on s +g r +g e +b r +p l +' t +m i +in e +we e +b i +u s +sh o +ha ve +to day +a v +m an +en t +ac k +ur e +ou r +â Ģ +c u +l d +lo o +i m +ic e +s om +f in +re d +re n +oo d +w as +ti on +p i +i r +th er +t y +p h +ar d +e c +! ! +m on +mor e +w ill +t ra +c an +c ol +p u +t e +w n +m b +s o +it i +ju st +n ing +h ere +t u +p a +p r +bu t +wh at +al ly +f ir +m in +c a +an t +s a +t ed +e v +m ent +f a +ge t +am e +ab out +g ra +no t +ha pp +ay s +m an +h is +ti me +li ke +g h +ha s +th an +lo ve +ar t +st e +d ing +h e +c re +w s +w at +d er +it e +s er +ac e +ag e +en d +st r +a w +st or +r e +c ar +el l +al l +p s +f ri +p ho +p or +d o +a k +w i +f re +wh o +sh i +b oo +s on +el l +wh en +il l +ho w +gre at +w in +e l +b l +s si +al i +som e +ðŁ Ĵ +t on +d er +le s +p la +ï ¸ +e d +s ch +h u +on g +d on +k i +s h +an n +c or +. . +oun d +a z +in e +ar y +fu l +st u +ou ld +st i +g o +se e +ab le +ar s +l l +m is +b er +c k +w a +en ts +n o +si g +f e +fir st +e t +sp e +ac k +i f +ou s +' m +st er +a pp +an g +an ce +an s +g ood +b re +e ver +the y +t ic +com e +of f +b ack +as e +ing s +ol d +i ght +f o +h er +happ y +p ic +it s +v ing +u s +m at +h om +d y +e m +s k +y ing +the ir +le d +r y +u l +h ar +c k +t on +on al +h el +r ic +b ir +vi e +w ay +t ri +d a +p le +b ro +st o +oo l +ni ght +tr u +b a +re ad +re s +ye ar +f r +t or +al s +c oun +c la +t ure +v el +at ed +le c +en d +th ing +v o +ic i +be st +c an +wor k +la st +af ter +en ce +p ri +p e +e s +i l +âĢ ¦ +d re +y s +o ver +i es +ðŁ ij +com m +t w +in k +s un +c l +li fe +t t +a ch +l and +s y +t re +t al +p ol +s m +du c +s al +f t +' re +ch e +w ar +t ur +ati ons +ac h +m s +il e +p m +ou gh +at e +st ar +wee k +! !! +c lu +th ere +n er +t om +s el +ï¸ ı +wor ld +v es +c am +go t +in ter +of f +u m +ton ight +o ther +h ou +loo k +j e +i d +si on +be au +at t +el i +or t +re c +f f +st er +su pp +g en +be en +il y +te am +m m +i c +pe op +it t +at s +on ly +mb er +en g +b ri +m p +k now +b ur +b ar +in s +lo w +sh e +ro w +â Ŀ +t ro +peop le +vi a +lo w +ag a +be t +x t +f ac +ch ar +e ar +w al +s en +f am +b le +n ati +is h +n or +g ame +li ve +s co +le y +d on +ic k +b all +ver y +the se +p an +i a +at ing +c r +a re +g ir +ma ke +st re +sho w +. " +f l +u p +d r +than ks +il li +w om +st s +i g +s ur +ever y +c ur +vie w +le t +in to +mo st +n a +in di +g ar +ha d +s ou +v ed +an t +iti on +ma de +f ol +un i +it ed +ðŁ ı +ic al +th r +read y +ch ec +d ra +k es +boo k +e p +si c +mor ning +ne ws +c au +c t +w ell +an c +pho to +th an +or s +bir th +g g +ou t +ne xt +som e +en ing +stor y +ch ri +do wn +hom e +f fe +fre e +d a +b or +f il +ci al +than k +si de +le ar +qu e +l ine +t en +at es +ye ars +m y +pho to +beau ti +ri ght +n u +for m +shi p +b an +th er +d ays +g am +as on +g y +ðŁ İ +birth day +se t +ic k +e t +st ill +com ing +ta ke +ðŁ ĩ +b b +s ol +s on +d en +e p +mu sic +the m +de n +wh y +f oo +c ra +am az +w n +h ol +t ting +w r +u e +ma g +c ro +l an +c lo +b ra +a k +s ing +c al +re ad +' ve +jo h +b ab +d ri +b lo +bi g +er ic +in t +t or +tr y +l a +le g +hou se +m ic +v al +beauti ful +l itt +chec k +ne w +ver s +s w +ar i +pla y +h er +âĢ ĵ +w in +m a +con gr +sch ool +f un +. @ +he al +ic h +d el +wh ere +l on +ke t +tw o +mu ch +wat ch +v en +d ed +a st +k ed +b as +go ing +m p +e ver +w ays +ro o +de sig +l y +s ed +to p +l in +ch an +to o +it ing +d ent +gh ts +t y +sp o +ne ed +b lu +in st +be ing +âĿ ¤ +w el +l s +hi m +m ay +st ing +n a +el y +litt le +g a +n at +tom or +m c +h on +w ant +a ir +pi c +am eric +p er +le ss +wee k +ve l +a h +c ap +ch am +g er +ti m +tomor row +ne ss +st ate +h al +ser v +z e +o s +p at +v is +ex c +s in +f f +c ity +c en +an y +b el +su mm +t in +w ould +loo king +k o +ce le +fam ily +m er +po w +hel p +bu s +c o +c le +sel f +en s +ic s +th o +an i +ch o +le ad +b s +t wee +th ink +for e +ch il +vi de +di d +al e +ch i +v il +en ds +w ing +p as +' ll +v ol +s a +g s +man y +j ec +be fore +gra ph +n y +ur ing +w il +d d +bu il +f av +st ed +tr an +l ing +ou d +d ge +fi el +nati onal +st a +c er +w ere +in a +se ason +c ou +n ed +amaz ing +ti ons +cele br +n s +a th +he ad +s day +d ar +lo c +v in +an other +g oo +s at +n y +jo in +pre s +s es +s ing +an a +in ing +.. .. +c our +ï¸ ı +ac t +cau se +li ght +am s +t a +b al +f c +hi gh +off ici +t t +chri st +d ic +d ay +ra l +h or +: ) +vi si +n am +o b +ma s +gh t +re ally +t un +fin d +thr ough +por t +u t +ti ve +st y +n e +or e +ðŁĺ Ĥ +supp ort +ne ver +ev en +ðŁ Ķ +h a +y a +l d +u k +r an +j am +wi th +me di +d es +ne y +ch ing +al e +h y +k in +! ! +d y +pl ace +al so +b le +wh ich +bl ack +b li +s ay +par k +pl ay +ir e +vide o +week end +a il +ke y +p t +w ard +fri day +d in +ine ss +g ro +b en +al ways +t ball +ag o +m il +c y +pro duc +di sc +un der +ple ase +sp or +fu ll +e y +ðŁ Ļ +is e +iti es +c at +k no +u se +fo re +k er +ar t +hi gh +op en +s an +e f +our s +sh ed +st ri +d ro +aga in +i m +ðŁ ĵ +en jo +fu n +ge tting +p en +g er +c li +an y +ever y +e u +wom en +â ľ +e st +c ould +r y +" @ +th ou +sh a +comm un +b er +d ents +di s +wh ile +aw ay +di o +h am +g la +d ate +k a +mis s +un ch +w on +in f +roo m +g a +re al +ex per +di rec +sh ould +sp r +g ol +l ong +bet ter +or i +e y +i ence +il s +z z +h an +f ound +v s +â Ļ +po st +ti c +par t +m en +ren ce +ce ss +v ic +s il +sho p +ðŁĺ Ĥ +f ood +v al +sti c +y ou +s ays +e lec +st ar +o c +l and +i d +c tion +fiel d +s of +st art +wat er +fri ends +on es +ðŁ Į +f la +f ar +wh ite +par ty +in st +gr ou +t v +every one +m ent +j a +ch a +pr in +an ts +d uring +l at +l ar +we st +th en +k a +y oun +in sp +in te +we en +visi t +aga inst +re le +he ad +c es +to wn +loo ks +th re +re gi +ren t +pro jec +gir l +se ar +w o +m om +c ar +h un +pu bli +d i +p le +c all +c ri +u m +for d +per fe +fri end +h ard +ssi on +te st +pla ying +ar ound +be cause +ke ts +me et +sat ur +ar ti +wor k +j un +v en +r un +me mber +por t +su per +t wit +s am +el s +t ly +ad v +ati ve +at h +s ure +av ail +la r +s qu +ar ds +ev ent +m en +l l +o ver +lo gy +it al +tim es +m al +b ack +c oo +ma king +st ru +â ģ +it u +sh ar +g an +c as +s n +summ er +pic ture +f an +h in +christ mas +c y +pr oud +cham pi +desig n +pp ing +ho pe +c a +avail able +ma y +we d +photo graph +spe cial +sal e +sto p +er y +a we +al ity +hi story +am a +pre si +b ru +wor king +d one +d r +k en +fe at +w ood +ate st +sun day +mo vi +vel y +s le +f ace +sp ec +stu dents +b y +ha m +sp on +bus iness +d at +i e +i p +so ci +g lo +h and +re cor +r s +me e +ke ep +p ur +heal th +sh e +com ple +go d +da vi +col lec +li st +r a +clu b +t ers +in clu +th ings +pl an +â ĺ +joh n +sh ing +at ul +so on +blu e +g or +satur day +w on +congr atul +se e +âĿ¤ ï¸ı +tho se +ðŁĺ į +fin al +d ou +it h +o wn +ro ad +t our +a st +indi a +ti l +n d +f er +fav or +su l +lear n +fir e +ju st +grou p +a h +r ac +bo dy +u r +c are +à ¸ +p lo +o h +po s +gi ve +te ch +su b +c ent +er ing +y m +il ity +f ic +lon don +v ir +gu ys +b a +ðŁ ¤ +bab y +sc re +ðŁĺ į +tru mp +un der +chan ge +i an +col le +ss es +l er +ss ed +n ice +ann oun +pow er +s ar +a king +min i +s li +s wee +k ar +fu l +c ru +ac tion +a ther +) . +st and +de vel +a a +g an +le ft +lo l +re l +tran s +m ents +in t +e f +man ag +di g +gen er +do wn +p au +ti v +k u +th ur +k en +st on +f ans +tal k +twee t +t oo +sty le +pro te +se con +fr on +awe some +g l +p al +ne t +s or +la u +g on +sin ce +t ty +ser ies +me mor +b eli +fil m +di d +di es +o t +congratul ations +p ra +e ve +w oo +offici al +su c +in cre +b on +par t +pp ed +cla ss +si ve +bo y +cu l +perfe ct +t ou +d am +wel come +foo tball +h i +p ap +wa it +ad a +congr ats +youn g +exc ited +re ce +j an +v a +re d +st ra +medi a +' d +do es +le t +mu l +ill s +gre en +m el +to ge +fu ture +ye ster +vers ity +for m +ta in +i de +ch es +ki ds +qu i +ha ha +de ta +bi g +favor ite +gir ls +con tin +do m +sear ch +u al +a ir +d ers +mon th +c er +yester day +commun ity +ad e +do g +vil le +ic es +d eli +sy ste +ru n +is m +he art +c up +en ti +fe w +presi dent +e ds +un til +fe sti +o k +f lo +sa id +ol e +me d +tra vel + £ +ph one +toge ther +fa st +lo t +gam es +sh ir +bet ween +y es +th ers +do ing +m ac +at or +b and +fol low +projec t +devel op +di ffe +con fe +spe ci +ca st +y s +bo ard +r d +i al +sh oo +r am +ha ving +sh are +fol low +on e +n ame +m r +pu t +disc u +or y +c ame +ou s +s ite +twit ter +t b +t it +fin ally +z ed +su per +com pan +us ing +all s +li st +r is +sho t +g al +t ar +de l +joh n +âĢ Ķ +some thing +ra m +inte re +wh e +b it +ðŁ į +stre et +oun d +a i +tic kets +movi e +re al +k y +ta king +o pp +c c +l am +m oun +in ve +bl ack +us ed +on line +y or +loc al +gu e +c ks +o w +ge st +bo ys +illi on +con t +re ci +in ed +eu ro +no w +se en +p h +te ach +de f +sou th +su ch +aw ard +mu st +is su +ca re +fe el +p lu +l atest +spor ts +we b +te x +e ment +s k +fi c +w an +te ch +o t +bo x +n er +fre e +t al +a sh +c ase +ho t +won der +mee ting +er a +ch all +ðŁ IJ +jo b +il i +c ool +j our +th s +m o +f el +di e +mic ha +e le +te am +serv ice +st and +ma kes +p ing +ear ly +com es +e k +ho li +v ers +ag ue +s au +thre e +mon day +fa shi +some one +th ro +se a +b ad +supp or +tur n +ur y +m ing +photograph y +n ic +mar k +pre tty +ss ing +wat ching +me mb +ar ri +coun ty +be ach +fr an +cen ter +pol ice +b at +publi c +t an +pre ss +s af +s y +ge ts +ro y +n ers +y our +bu y +st ers +sho w +as ed +chil dre +af ric +in es +sp ace +sc ri +h all +pa in +ar ing +hom e +m ur +heal th +ch ed +s and +rece i +gu y +e a +americ an +re si +childre n +- - +i ri +ing ton +coun try +ro ss +le n +ann a +boo ks +b c +e ce +d om +lo vely +k h +pe t +g y +g ri +st age +off ice +ro ck +m on +b ay +t able +su n +m ed +th in +l or +f low +( @ +uni versity +stor e +fron t +goo d +z a +vo te +nor th +he y +an im +or der +mi d +with out +a de +re member +mar ket +? ? +mu s +tra ining +e duc +bu t +co ver +st an +sc en +b la +bre ak +l ou +s ame +g old +a in +o s +bo th +l it +ver n +a i +al bu +p a +enjo y +be g +ell ing +thur sday +inf o +s an +americ a +ha ir +te l +mar ch +con cer +colle ge +confe rence +ap p +h our +ch ang +â ļ +s our +ol s +we ather +w ar +p hi +festi val +secon d +cu te +pr ac +en er +str y +le a +pol it +s av +se n +o w +m i +ne ar +ou ght +z e +co ffe +w illi +d an +se y +davi d +e se +f an +de ci +the at +no v +ati on +tr ac +sc i +re view +c el +e m +u n +ju ly +or ig +ti on +d ru +form er +st ay +af ter +in v +too k +dat a +b al +tu es +d an +ev ening +ðŁĺĤ ðŁĺĤ +d ol +u res +pro vi +t s +e st +sig n +j ac +u k +s ong +ye t +bo w +in du +j ap +h oo +po int +any one +z y +i st +h ur +it al +buil ding +wom an +ch ur +j er +per for +co ach +le ague +ce ss +ne t +i mag +nati on +br it +qu e +aw ards +ag es +wor ks +c ed +man ce +l ate +ig n +mon ey +tru e +i i +t ell +pl ac +p ac +as y +wor ld +be hin +im port +read ing +gra m +gi ving +me t +h it +for ward +st om +pres ent +jun e +so cial +no on +mar t +hal f +s we +go vern +k er +deta ils +li sh +_ _ +ac y +si a +ber t +f all +! !!! +) , +th i +d iti +sp ort +k ing +f it +st af +c at +mu se +cen tr +y er +con tro +b loo +wal k +ac tu +did n +li m +lear ning +re search +wed ne +au th +h ours +k y +f ar +h en +.. .. +it ch +ri l +str ong +sk y +que sti +jam es +r on +d g +f ur +c in +do es +app ro +mar ke +tu res +ful ly +ch at +behin d +te m +fin i +mis sion +b att +fe el +he av +every thing +b ar +w ish +pre mi +i ma +exper ience +e ach +re port +swee t +tic s +spr ing +re spon +syste m +vic tor +l in +sa w +al ready +gh ter +f le +ã ĥ +br ing +albu m +- - +ell s +st an +to m +inter national +w ent +an ni +mat ch +pp er +st one +sm all +ra in +fashi on +are a +v an +ag ram +k o +thou ght +wor th +v an +m er +coffe e +it es +g n +arti st +c on +ar ch +c ir +se cre +gr ound +is o +h and +co m +bri dge +h s +x i +l ink +pu l +sp l +r ace +f li +ri ver +g as +di sco +d al +play er +f it +photo s +it y +o k +j or +tr a +ap ril +ad s +a di +sol u +beau ty +do or +me ss +up date +ali a +sch o +en ed +mom ent +sco t +sc ience +i or +ti es +ac ross +ous ly +sh es +does n +p age +wat er +m illion +cla ssi +l ic +ca st +form ation +micha el +ell o +s mo +in ts +vi sion +op ening +ld n +au str +tues day +win ner +po ssi +r ound +shir t +di t +b o +u es +il led +al ong +tri p +star ting +im pro +k an +per son +no t +re co +ne eds +c le +li e +re st +r ing +win ter +si mp +mo m +be er +fac e +tor s +us a +collec tion +ge or +se ssion +tr ying +la s +la ke +j en +orig in +stu dent +se cur +v in +pic s +ex pe +com p +gon na +e qu +b ad +le y +a u +memb ers +bre ak +w all +gi c +din ner +bu l +insp ir +r i +min d +ic a +win ning +tal king +t ren +s is +t en +wonder ful +s now +he ar +th om +no thing +gu i +st in +blo g +fe st +b un +le e +war ds +ch ance +dre ss +re n +pau l +p es +tech no +ru ssi +c ard +e ast +mar i +w ine +t i +la w +str ic +k i +ap e +au gu +pro fe +as h +cour se +ma il +ren tly +d un +m un +lo ve +is land +dri ve +s l +end ed +ma in +lo st +nat ure +âĿ¤ ï¸ı +ch ic +re por +p in +pr o +st ation +ce p +ta kes +compan y +go es +on d +ma ch +ra dio +d ad +ro ck +j a +p ay +champi on +e e +in de +tt a +ati c +t ab +beli eve +ener gy +z i +t at +wor d +on ce +re sul +y l +and re +an o +inst agram +clo se +t am +cu stom +w a +con om +sho ws +li fe +k in +ro b +t age +n ation +al most +list en +sa ve +re li +ac e +mar y +tre e +for get +j ack +wa iting +direc tor +h ill +bor n +te mp +f l +st e +on a +sing le +wedne sday +un ited +in o +@ _ +ne l +celebr ate +en ding +de al +j i +can ada +hu ge +tr ack +âĢ ¢ +f y +fan ta +an g +yor k +rele ase +p un +ep iso +wor ds +t our +p ack +i gh +classi c +perfor mance +ke t +after noon +recor d +win s +pro ble +âĿ ¤ +f our +b ed +ban k +d ance +s la +cal led +mi ght +a p +pa st +ðŁ ļ +diffe rent +it e +gi ft +ssi ve +chur ch +c us +pro gram +ho tel +ic e +ma d +secur ity +en ge +d c +en ough +st a +e ty +de ad +g un +he ar +m ir +hu man +gre ss +oun ds +pi ece +bre aking +gar den +fi ght +vie ws +f ish +star ted +run ning +gre en +ser i +s m +as k +d or +de ath +e conom +er i +ir d +s er +l unch +âģ ¦ +bo x +nat u +ba se +b an +f al +glo bal +wil d +wo w +out side +mo ve +le ad +an al +muse um +on g +ha w +pow er +than k +b ac +char ac +cam pa +dig ital +r o +op er +de v +w ol +p ati +f a +m ale +pap er +ill ing +c s +â ĥ +educ ation +ta ken +e ffe +m ou +s ad +" . +bas ed +staf f +inclu ding +li ving +a c +ch ina +mo b +stor m +lu ck +ph il +o o +y n +tra vel +k el +ti al +pr ice +boo k +import ant +bi o +p ool +ny c +f ab +lo ad +? ! +chall enge +cr y +ser ve +we ar +bu s +ta in +nu mber +ro r +k at +i z +th ough +ho sp +m m +fa ir +ut es +ho t +po p +fi ed +cam p +develop ment +li br +c ali +em s +âģ¦ @ +b ol +is ed +stand ing +mo del +it a +g le +bro wn +ima ge +ve red +for ce +o il +par tic +sh u +da ily +la w +se c +cla ss +cam p +holi day +cl in +k ers +pres ent +gam e +incre di +er ship +inter view +b ill +du e +and y +ab o +in nov +ke y +ac ade +p il +mo der +st ars +br and +f er +wee ks +con si +pr e +sa fe +wr it +di um +la unch +marke ting +ann ual +as si +cour t +la dy +c ted +and a +in side +chil d +opp or +sm ith +centr e +gu e +âģ © +f ren +st y +for t +ent ly +is n +ke ep +to ber +on y +bo y +al d +col la +de mo +le vel +com pet +ad o +b our +fanta stic +m ate +s u +sou th +oppor tun +vers ary +lat er +bu d +face book +la un +ster n +p it +! " +ma j +gr am +tb t +fi re +happ y +a ks +wh ole +actu ally +ill er +ell a +lo ts +al ex +an ge +lan ds +ðŁĺ Ń +en ter +r ou +episo de +p ed +in ten +sh ire +wh o +pl an +h o +ca ke +we st +mag az +fre sh +c c +n ar +ch ris +wr iting +w er +n om +l o +mi dd +dre am +o l +ti onal +de b +> > +be come +s i +gr and +all ing +hi stor +ri de +i red +saf e +que en +ci l +in tro +vi l +d ani +.. . +ar tic +st at +sh ort +or ing +sel fi +mis si +do c +b it +g all +b om +i re +se lec +d ition +ðŁĶ ¥ +fri end +be at +gh ting +ðŁĺ Ĭ +pe ace +ex hi +ant a +ab ility +il lu +j on +qu ality +tri bu +m es +play ers +fa ir +cu t +c ab +suc cess +b i +su s +pro mo +sch e +an ge +ic o +comm it +cat ch +ill a +kin d +feel ing +qu o +s ay +anni versary +spo t +mo ther +an e +p end +your self +op s +app le +min utes +p o +gr and +ri es +ha ha +care er +ed ition +de c +ric k +am i +concer t +iti ve +ge ous +d ly +t te +adv ent +i g +li ghts +ak er +sk y +âĥ £ +r ay +fini shed +w ay +s d +ac coun +ðŁĴ ķ +ck y +ch el +lit er +pain ting +lo s +st un +techno logy +n as +ma r +b il +afric a +ki e +ey es +gol f +plu s +ni a +it ec +serv ices +wed ding +kno wn +te le +.. ... +star ts +pa ren +w ants +ati onal +mon ths +win do +fav our +er t +magaz ine +ex clu +re ve +b c +origin al +e ss +n al +an ti +st ro +t ice +stu dy +à ¤ +v ac +nation al +fi ve +ra in +ve ment +u te +ver se +em er +ar my +possi ble +gue ss +val ley +ther n +cro w +m r +col or +on to +pic k +cle ar +dar k +t ac +wan ted +it ting +can cer +govern ment +di e +ri se +z ing +col d +f oun +stu dio +str ation +bro ther +a head +sh el +mic ro +ic ally +d au +sig ned +vi ol +a x +as se +i o +w re +spl ay +ch ick +augu st +pl at +ti ps +sp i +hu man +e asy +lo gi +mi ke +gro w +ag re +w w +sh ad +mo tiv +wi de +tur ns +om g +v ar +de fin +su g +j im +ðŁĶ ¥ +t d +campa ign +nam ed +re tweet +co p +t v +le av +k is +dou ble +s mar +issu e +vil la +in formation +li es +sto ck +n t +di stric +sh or +mi x +er o +se p +me x +see ing +li ve +re min +co de +g ur +s c +wil d +l un +h ood +spo t +fa ther +fore ver +up d +tra f +f ly +ne ed +gra du +tra in +ma ke +s ab +be y +si ze +lead er +tal ks +e u +lo g +fo x +gor geous +le ss +le ts +sur pri +my self +no te +li ves +f ru +lo ved +se ver +de m +j i +so c +h old +do gs +n i +â ŀ +lea ve +air port +ben ef +ex pl +shi ps +comple te +ach i +gre at +vin tage +j ack +ro c +woo d +pri v +off er +ey e +ver sion +te a +co ach +off ic +w ell +g en +s at +h h +you th +o x +? " +m t +mi x +g g +d le +natu ral +buil d +break fast +thin king +theat re +mo on +ber g +go als +geor ge +en e +exc ell +il ing +tun e +y ed +g ate +m it +net work +jo e +h ello +f b +tu be +we aring +ath le +stru c +har d +gla ss +g ers +thro w +g es +b t +indu stry +manag ement +ali st +go al +stre am +y el +a vi +ici ous +o thers +s ki +chri sti +bir d +e sc +m in +tr o +l t +j an +im p +ri ghts +sh a +or gan +cent ral +ar a +ro ll +favour ite +che ster +el se +p ay +car s +m ine +ste p +prac tice +maj or +h ang +ðŁĺ ĺ +n on +v ari +eng ine +vol un +di a +i led +arch itec +p ink +d s +th y +wa sh +web site +ba g +contro l +el li +f ra +an sw +d ence +y u +r on +ol a +g in +dr in +li c +cou ple +sp ar +g on +cre ate +c t +celebr ating +de ep +e at +te e +vo ice +dro p +vis it +at ors +sta dium +f t +w is +ro l +gra de +fam il +po ints +re pre +w as +traf fic +jap an +or g +hon or +tex as +man u +âĻ ¥ +safe ty +re r +b ag +em plo +rele ased +re gu +ak a +n av +ro le +sen ior +spec t +cro ss +lin es +be st +p ack +s in +ti e +mis sing +sun set +li ber +is ing +j ay +sk i +champion ship +ac tiv +la dies +play ed +y y +pu bl +al o +pri de +s r +pa ki +lu x +sur vi +ck ed +e ts +cho col +austr alia +par is +mi les +h at +ment al +al a +me an +mob ile +en a +in si +f ound +chi ef +t ag +incredi ble +re turn +à © +goo gle +fren ch +cre w +hal lo +ali an +j az +ch er +sil ver +nor th +eng lish +base ball +c af +lim ited +follow ing +app reci +ear th +k ir +ve mber +w ed +p tion +g ed +oc tober +fl ori +c r +en cy +ga ve +lor d +stu ff +ber ry +po st +sm ile +bro ad +st ate +gg er +me ans +ic y +gu n +y o +ma ster +bur g +han ds +ni e +/ / +uni on +brit ish +big gest +distric t +am ing +h il +o ce +per son +pas s +en vir +scho ols +arri ved +anc es +insp ired +ex pla +be n +libr ary +bo tt +am p +ste ph +cont act +b ang +m s +cali for +t old +batt le +b b +chic ago +âľ ¨ +str ate +sh i +de ce +- ) +ad d +la b +j ones +leg end +cast le +ing er +st ance +be l +ur a +re fu +lead ers +po t +se x +h ic +artic le +ki d +fr ance +x x +ex e +gui de +volun te +pr int +al i +ce o +twee ts +w x +scen e +vol u +ant i +h an +as soci +shar ing +ro se +mini ster +sh er +in ste +cle an +demo cr +po ster +sk in +p sy +pro per +cra zy +i am +o re +in i +any thing +po d +mo ving +cl ick +ex plo +com b +cra ft +f i +bloo d +is ra +publ ic +d ent +ol ym +eng land +a si +ch er +fac t +envir on +har ry +g one +me dic +enjo ying +just ice +j r +indi an +wi fe +s ound +t es +dra wing +p al +ide a +cr it +ju li +il er +war m +cl ar +thou ghts +def en +coun cil +intro duc +di ed +jan u +an i +s end +li er +m l +intere sting +tra de +win d +b ay +s ac +anc y +sour ce +b es +org ani +ar ly +lar ge +ff ici +ta g +u t +de sp +o es +tit le +sy m +pic tures +op en +wom en +sho wing +ri a +le ast +lead ership +cur rent +elec tr +val ent +list ening +c key +gener al +de ser +du ce +; ) +c ent +ðŁĺį ðŁĺį +sco tt +po or +selfi e +ev ents +i on +wr ong +de v +h ill +sep te +cul ture +l ine +sor ry +s ent +si ster +ce pt +k ri +no vember +ar i +announ ce +z ation +br an +g ent +d u +l en +per s +f m +mart in +o p +e mb +om e +midd le +suc cess +pe ter +janu ary +f lu +rac ing +d av +bi ke +ðŁı » +pe t +shoo t +profe ssi +feat uring +septe mber +now playing +sta ur +z a +on ic +qu ick +bas ke +spe aking +mil it +z er +chick en +b ell +s ad +co ast +lo ving +y ers +d j +pan el +ver age +s wit +ic ks +b ou +califor nia +s am +paren ts +er o +k illed +ph ys +jo bs +mi gr +an th +e mo +hallo ween +and er +c m +compet ition +e ag +s ket +sp ir +may be +exclu sive +app e +jour ney +scre en +for d +i o +h ate +u g +sou l +her o +soci ety +sy n +gu it +n h +d j +as es +im pre +ti me +sal es +d d +f ts +summ it +stun ning +om s +tur ned +cle an +sof t +be at +re staur +de red +en ces +ma gic +di o +sh ine +gu est +health y +exhi b +stor ies +po pu +n is +el a +bel ow +fun ny +resul ts +s ne +cur rently +ar d +down load +f light +m al +f ine +p ad +ch u +ent ed +h at +ðŁij ı +ste ve +j o +mar k +r at +b all +p c +p on +b by +o li +ar ts +as ure +bow l +att ack +mi c +de ar +ran ge +en ter +chocol ate +br illi +ac cess +, " +? ?? +ch ap +con st +t n +mat ter +blu e +gall ery +em p +work shop +lead ing +y ours +baske tball +w anna +th u +_ _ +mar ri +sle ep +bi a +ch e +ma d +imp act +o wn +si r +chan nel +euro pe +e sp +k itch +hosp ital +w ra +roy al +f s +ne u +qu ar +ne y +ac ks +ch ase +pp y +st al +at ely +ti m +dece mber +r are +per form +cre am +we ight +ch oo +ni ght +ha ven +fr anc +kh an +buil t +hel ping +tru st +ty pe +gol den +ta x +s now +s wi +di sa +questi ons +ve y +li ght +c n +cl oud +thom as +ag ed +sh ou +te ams +gr an +re ason +a a +you tube +v p +pi zz +manag er +bur y +cre dit +tre at +ma x +i k +ma in +g ing +de ad +pro bab +ye ah +ã Ĥ +br and +so li +pl ant +ta yl +gir l +ðŁĺ Ń +nam ent +au to +mess age +ko re +n ur +ter r +ag u +ma p +sen ting +lo ves +gi ves +g ab +z en +ro bert +con fir +w ars +o m +sta in +cam era +and er +won der +a b +ca p +s old +su it +wal king +contin ue +effe c +dau ghter +d anc +cha in +mul ti +ki d +y an +champi on +v o +ta ins +ho st +min i +mis sed +re sc +ly n +fin ish +del icious +s as +tayl or +i b +pro mis +produc ts +moun tain +flori da +regi ster +tre at +rec ent +fe male +boo th +mat t +ve hic +s op +mo tor +suppor ting +phi c +ex tre +dr ink +lan e +th ird +p s +con stru +ce re +far m +ðŁİ ī +tu red +ðŁij ī +c ats +a j +gi e +shoo ting +as ked +paki stan +am e +m b +g il +leg al +squ are +in vol +dra w +oo oo +!! !! +opportun ity +p y +e i +b ts +teach er +charac ter +john son +br on +ly wood +ch ine +c ing +c ine +d ge +gam ing +russi a +ci a +quo te +ric h +go v +flow ers +sp iri +st in +grow th +ðŁı ¼ +comm er +j uni +mu m +r an +s na +a ren +c b +ac tor +col or +si t +pa ir +ch i +bo w +acade my +hel d +r ang +me tal +y l +ac tive +probab ly +t ch +need ed +spe e +cho ice +ital y +ry an +ðŁĩ º +flow er +v it +m n +found ation +b ak +si ons +ne igh +f loo +he ard +re mo +fre sh +ing ing +re f +to wn +cl ou +je sus +spiri t +cou ldn +z es +ðŁĴ Ļ +willi ams +pro ce +moder n +pro cess +sho es +cre ated +tri c +issu es +ann e +att en +de but +h r +n it +sti g +a po +e ps +z u +ã Ģ +si x +car ds +lan gu +fam ous +tour nament +se l +e bay +y n +st on +k ick +announ ced +k am +vo c +brilli ant +hou se +che ese +war ri +mus ic +ho ckey +ðŁĺĤ ðŁĺĤ +sk ills +au tom +smar t +med ical +mon y +e x +gu ar +gi ve +pers onal +ven tion +al li +pre ss +flo or +m c +victor y +hi m +simp le +th or +ðŁĩº ðŁĩ +ta il +lu cky +ale x +qu ite +bo t +ssi ons +chall eng +c ann +amaz on +h ell +b ought +) : +ed y +secre t +produc tion +inde pend +de fe +ad ded +p r +p ag +be d +gre atest +with in +j ay +ðŁ ¥ +ire land +re ly +s d +te xt +dri ving +pro gram +spe ed +col um +str on +à © +fore st +â ĸ +mach ine +co in +sc ar +oun t +bi e +¡ ï¸ı +por tra +comm on +wre st +recei ved +kno w +inve st +pl ans +ac cor +ad op +ter y +re ali +p p +k al +art work +me an +go d +inste ad +an ci +motiv ation +as ing +inspir ation +up coming +polit ical +euro pe +m ers +heav y +ðŁij į +fe bru +scot land +ou gh +b t +bo ss +sche du +spe ak +n ick +u red +in o +e k +ri sk +tor y +pres ents +b on +ru g +st ates +exhib ition +il o +m ill +br ought +: -) +tou ri +com e +offici ally +champi ons +do ors +re p +po se +ex tra +k ings +soc cer +squ ad +app lic +at a +some times +t ari +excell ent +ðŁĺ ĺ +stra ight +car ol +ri p +âĢ į +gra phic +m ol +elec tion +febru ary +as ons +l i +di r +m t +n ick +u su +m rs +com ics +inst itu +cor por +v i +ðŁĻ ı +tu ral +di se +ac ci +we are +am ong +sho pping +t ill +wh at +cha ir +sp an +chine se +innov ation +jo y +k it +cent ury +ob ama +ph ili +f c +re ach +c iti +ul ous +n on +d ang +happ ening +bur n +p el +or ange +d v +k ick +cla im +ing ham +ph y +no v +pod cast +wh i +ni ghts +ear lier +be ar +la h +exc iting +or a +gi ven +s lo +memor ies +contin ues +produc t +gh o +c d +kno ws +ðŁİ ī +publi shed +discu ss +y ard +i phone +tri es +w all +fe b +are n +tru th +win ners +tu re +diti onal +milit ary +proble m +m and +do g +lo ss +c ric +can adi +ve ter +villa ge +" , +y r +un g +don ald +ag ing +bir ds +sci enti +le s +th is +regi on +tic al +itt en +il a +ðŁĺ İ +d ad +di am +abo ve +st ren +li t +p ir +la b +fo cus +bus y +d ur +app ly +s ma +auth or +ac i +exe cu +dom in +re la +jack son +at o +wash ington +ðŁĻ Į +k ill +popu lar +ce ment +ro ad +e ating +loc ation +v ent +ar re +n an +cu sto +advent ure +or din +spor t +ul t +lo ck +questi on +dri ver +land sc +on i +k ins +p d +jor dan +te red +k k +a f +chil d +s p +just in +en i +s elling +z o +wh it +bo ston +partic ip +sig ning +happ ened +he at +m am +dre ams +lo ws +gra ph +the day +head ing +br o +ble ssed +vi c +ve gas +h d +in ning +ro man +and ro +den ti +u se +c it +pro gress +writ er +bo b +ff s +gro wing +b ly +aw are +ex am +sp ent +be t +sc ore +bey ond +do cu +ad el +s f +cou ra +colla bor +in c +priv ate +bo at +* * +z one +p ha +b ill +to tal +plan ning +to wards +plac es +pre view +cre ative +dam n +ide as +se ems +po ten +say ing +di splay +s w +a qu +lou is +by e +li l +e mail +we stern +ger many +ell er +re s +f ant +ment ary +de als +ric hard +jer sey +stren g +ra d +pizz a +mon d +w are +l ac +g i +ar chi +c d +yel low +rec ently +re ach +à ¹ +kitch en +desig ned +tr y +g al +restaur ant +at ure +w w +j as +l ma +ðŁij Į +pa in +av o +min ute +sch ol +ther ap +tic ket +d ry +jap an +diti ons +ter ri +sel ves +happ en +t up +ma g +cop y +sh er +free dom +f ile +speci ally +tor onto +lo ad +g ary +re y +answ er +lo y +cau ght +pri ze +u ne +fic ation +ni ger +sy d +tou ch +feat ure +jaz z +recor ds +him self +di sh +ro ber +spot ted +ma ster +wa ve +fin als +bu ll +for um +al d +re comm +ch a +a e +d oo +inst ru +tru ly +l g +in k +bro thers +de st +j im +m it +clo sed +is on +tri ed +s anta +af fe +w an +hor se +g row +camp us +rel ation +nati ve +jour n +go v +o ct +k it +b ound +part ner +re ma +crow d +! ) +c alls +ra il +qu ali +solu tion +con test +con vers +sn ap +b ase +in iti +ta x +y e +ent repre +it or +constru ction +foo d +present ed +n ings +cli mate +k m +mo del +b j +blo ck +present ation +dre am +fi x +c alling +bus ine +con gress +under stand +we b +val ue +ï¸ı âĥ£ +mex ico +it ely +ki m +char ity +ref lec +bl an +fl ying +anal y +famil ies +b and +reci pe +celebr ation +ac cep +ar y +to t +g b +intere sted +cap tain +âĻ ¥ +ti p +ab sol +bra z +inve stig +o logy +de c +tru ck +ver ing +c lear +don t +go tta +ad vis +beg ins +ma ss +de scri +blo ck +k im +davi d +son gs +memor ial +feat ures +su stain +' . +gra b +jo se +v a +con serv +se ts +man chester +fi ghting +de gre +ag a +in d +sle ep +pos ition +ha ir +sig ns +pol icy +it o +al ert +st am +sp end +w y +absol ut +d m +anim al +my ster +success ful +proble ms +ro bo +k ay +gar den +p d +may or +d ale +t ol +off ers +vis iting +friend ly +tre es +offic er +accoun t +ke vin +ðŁij į +gi ant +contin u +con su +tr act +n fl +ðŁĺ Ĭ +h q +b ility +a ar +dis ney +te en +on ed +wh ite +tra iler +de dic +al one +absolut ely +dig ital +willi am +in ation +s wa +e e +enti re +ger man +ro ll +h its +co st +st ay +th a +ali ve +accor ding +co t +liter ally +her it +re ti +haha ha +exper i +li kes +g t +ste el +__ __ +ch air +christi an +to wer +diffe rence +m d +tre ss +mi d +prin ce +afric an +fe der +foo t +car ri +ser ved +r ice +sh all +feat ured +ck er +rec ru +po e +sen se +ni fic +com edy +cont ent +f at +po sted +con tribu +tim ate +li ver +mb le +inter net +ag e +europe an +cl ing +gla d +ff ic +sc o +ak es +el le +ter min +ton y +p ale +col our +seri ous +pat ri +movi es +b m +professi onal +ad o +al u +br inging +f alls +isra el +ter m +langu age +bro ok +man n +commun ic +can not +ac ti +p he +y an +entrepre ne +tur key +log ical +lon g +ar m +ur s +work ers +ing ly +gg s +ri c +tu al +recei ve +op ens +ge ar +soci al +fe et +c king +ad ver +fin an +fe els +sp la +h r +ea ster +bra in +ã ģ +fi g +le dge +ne arly +prote ct +ma ssive +e th +aw a +ðŁĺ ģ +y rs +aware ness +defin itely +k n +imag ine +k u +syste ms +ðŁij ı +f as +li k +provi de +am o +disco ver +inf lu +ma ker +g az +fit ness +stre et +er s +te d +w c +ys is +pos itive +hel ped +que st +andre w +bra d +b in +hang ing +l ing +bri ght +se ction +ma ss +ðŁĻ Į +follow ers +ho sting +tem por +fla g +a ve +let ter +k ur +re qui +of ten +cry p +su ff +âļ ½ +russi an +treat ment +al le +ha y +l an +keep ing +hol y +power ful +pre dic +fun d +e specially +windo w +je wel +il y +ðŁĴ ľ +gener ation +app a +seri ously +o d +ðŁĺĤðŁĺĤ ðŁĺĤ +cer ti +iri sh +ðŁij Į +mi ami +be th +v ity +se cu +che f +cri me +graph y +ma x +arti sts +re volu +gu ard +spee ch +u c +upd ates +fac es +st ant +chang ed +repor ts +low er +pe ar +n c +k il +loo ked +spe aker +s f +re spect +ok ay +oce an +s itting +architec ture +tra il +se at +i ra +le g +japan ese +d am +u lar +sw im +polit ics +finan cial +ol d +mou th +at temp +de stin +fi shing +atten tion +me m +chang es +deci ded +reli gi +g in +c av +z z +ad am +ma c +wr ite +beg in +sc ul +al ter +is s +ath on +imag es +m oo +jo ined +ðŁĺ ī +âŀ ¡ï¸ı +pas sed +mu sli +h ir +lar gest +cam er +com ic +gh ted +rug by +bur gh +gg ing +te sting +pre par +lau gh +al ed +impro ve +beli ev +adv ice +sha res +he art +tur ning +s b +t el +caf e +n es +dani el +pat ter +t z +se tt +par k +c and +st ick +happ ens +bri an +ne west +e pic +ad or +ki es +war ning +anim als +custo m +ar c +di an +gol d +cor e +t f +c ity +pan ts +re ality +con fi +in ju +fo x +gu il +k new +âĺ º +cor rec +itu de +d den +. # +re duc +pas s +f on +y a +ow ner +re turns +n c +e ast +ap ol +in sur +th o +si m +juni or +be e +ang el +att le +elec tric +hor ror +cra sh +e ye +pat h +sou thern +emplo ye +ge o +t an +ha z +r ally +ðŁı » +proper ty +was n +enjo yed +gre y +g as +bre w +nor thern +hol ding +g p +ta ke +ch art +ly n +dr ama +z o +pa id +throw back +cu p +discu ssion +down town +w ill +le w +b is +t ary +bre ad +up on +r ate +teach ers +it ation +anc ed +cy cle +choo se +d c +ir an +co w +da ve +ra ise +prin cess +fa ith +- > +indu stri +sp ain +guit ar +fac ts +m n +sp en +cour te +go tt +projec ts +au di +o sc +pe ter +s and +intere st +happ iness +ven ue +sol di +surpri se +poten tial +per io +custom er +i i +g ni +manu fac +e co +bro ken +sing er +vel s +wal es +hu s +in j +f our +tal ent +d ying +mat the +fil m +jo ining +s ell +j ar +lma o +sur ger +bb c +sour ces +au stin +ni k +char les +f am +prin ci +ange l +cas h +lo t +o red +pla ys +pl ate +don e +memor y +br ings +n ba +solu tions +teach ing +gr ace +cir cu +hel ps +foun der +mar y +expl ore +de cor +par ts +ch o +inte gr +ha u +is es +pu tting +in er +r it +v y +mic hel +blu es +every day +for ms +bi o +ye ar +p in +t ter +spr ing +) ) +po t +al ing +perform ing +sh an +plan et +mus ical +head s +it alian +stru gg +âĢį âĻ +w ings +pu mp +h h +tr ou +a id +pri me +ear th +pa int +mon t +am y +bb c +fab ulous +fru it +andro id +bour ne +cere mony +enti al +? ? +deb ate +on ing +dra ft +sol ar +t x +j am +cor n +!! !!! +bro o +mil k +po sed +o hi +mo vement +b ren +part ner +p g +et te +ar ies +sh out +n g +leav ing +t ells +sen s +ta ste +kel ly +wor l +gy m +ric h +e gy +pi d +ma s +â Ĥ +courte sy +fran k +incre ase +wr itten +pp ers +re l +ha i +s as +s ound +tt i +w ich +ri ver +.. ." +a g +fel low +ro me +sm all +gen cy +ic an +lux ury +pro of +me t +wild life +mom ents +ra ther +cor ner +com pe +canadi an +lik ely +therap y +li am +econom ic +indi e +rou te +fi ght +ho pe +se tting +ant ly +cro ss +fant asy +de e +sket ch +comp li +ym i +ru les +engine ering +fig ure +ro w +. , +f w +syd ney +w ou +t ation +dre w +us es +the re +sp read +struc ture +pat rick +appa rently +ro s +h ills +w we +ann y +com mission +di v +f ying +con sul +anal ysis +ex i +ten nis +vehic le +ðŁĺŃ ðŁĺŃ +as s +high ly +op ened +b ann +ðŁĴ Ļ +mp h +wi shing +v or +fi f +give away +r r +ra y +je ss +g at +ic ymi +x it +high est +yor k +pi e +invol ved +high er +ri e +mal ay +int elli +desp ite +che e +sar ah +be an +reco gni +ar sen +tal ented +pas sion +ic h +ab c +lead s +dise ase +v is +se c +pre senting +m illi +hol e +sho ts +de part +surger y +gov t +b in +du al +e vi +lon ger +ev ol +scre en +portra it +et c +lo se +ch at +p en +p i +om a +s ick +er c +compan ies +en try +plan e +gr y +ven e +liver pool +premi ere +sha red +a red +fil ms +ir a +holi days +cric ket +ici an +v ing +. ) +ul timate +di vision +con duc +se pt +for ces +mon t +s mart +disa pp +sun shine +in d +b less +ma de +col ors +fran k +ir on +bott le +s go +m ood +j ason +er ic +bir th +te en +respon se +tar get +state ment +fe ar +th el +al um +ar ab +bl in +direc tion +ste ps +er ial +wor ked +at l +ðŁĴ ķ +fel t +pol i +scen es +hom es +b ell +e at +ate ful +t in +l ace +fol ks +p se +an n +wis dom +fa v +but ter +s r +are as +sm oo +bi z +dg es +app o +mo re +the m +effe ct +windo ws +sun ny +cap ital +tot ally +c ities +gr ant +mb ers +s low +au tu +il ities +w ro +ri sing +st ics +viol ence +i gh +qu ot +h it +t c +herit age +bu ff +ne s +z ar +den tial +ex ac +ed ge +de ep +aren a +be came +benef its +mar ks +mb er +a z +am es +pre ci +dra gon +re g +d ings +do s +ðŁĴ ª +n el +s ity +me al +di st +leg end +pur chase +pic al +st ick +f at +du ba +profe ss +car to +pro f +coun tries +respon si +se qu +fa b +tribu te +hon ored +prac tic +pur ple +an ton +pa red +t ough +summ er +environ ment +s ons +ðŁĻ ı +m ps +gi es +her oes +t elling +hen ry +f en +know ledge +Ģ ï¸ı +f r +ne g +u re +ac king +hear ts +s oo +hol lywood +ju mp +sau ce +schedu le +tur n +yo ga +cre ating +c ket +cre ek +â Ń +custom ers +ma dri +gu l +asse mb +moun t +c ell +to p +st al +dav is +t wi +sig n +premi er +iti ons +he aring +un k +pati ents +app ear +heav en +al ty +doc tor +a e +plat form +je ff +ðŁĵ · +regi onal +bi d +box ing +ex ten +or ity +a w +w ise +il le +sever al +bi e +s itu +sy ria +âľ ħ +remin der +enter tain +li on +part ners +in n +ph ar +f au +pl s +expe cted +sug ar +deci sion +s b +ch ron +associ ation +leav es +vis ited +sh ap +ðŁĴ ĸ +fur ther +h ann +w i +run s +l er +fun ding +fil led +.. .... +tin y +han g +or g +co ol +se min +ðŁı Ĩ +spon s +nav y +sa int +dru g +d al +r oun +co vered +tra ditional +invest ment +de te +al ism +f low +n is +sun rise +fe at +f ted +we ird +je re +ve gan +medic ine +an o +ac cu +deli very +temp le +chang ing +wil son +phili pp +re fe +n d +is er +g ay +r and +ati ves +t ely +p and +intelli g +g are +am bas +de mon +commit tee +strate gy +refu ge +bud get +prote c +pi er +ex press +nom in +econom y +al low +ic on +gal ax +o h +indi vi +dem and +vir gin +lu ke +ali sts +man i +s mi +ju dge +ent y +mic hi +resul t +am ed +spe aks +' , +hou ston +sh in +b ing +fl y +ch em +au to +v as +ge t +ar m +thank s +d in +gan g +x x +si on +loc ated +p l +jo sh +in fo +jo ins +adver ti +ot d +el d +si e +re asons +v ent +ðŁĩºðŁĩ ¸ +â ł +convers ation +stu di +ðŁĶ¥ ðŁĶ¥ +go s +s ounds +un it +mu sc +ge l +ack ed +pac i +co s +de re +u u +a o +la m +inspir ing +ar ms +tw are +mat ters +ad dic +du de +ex t +cri sis +b ath +me et +sing h +expe ct +del hi +resc ue +wor st +au g +shi pping +ser ving +st o +dar k +ac es +histor ic +landsc ape +desig ner +b illion +gr ateful +wa ke +e ve +m iller +hou sing +dy nam +is co +be ha +sh op +pr ou +e as +a sia +e ding +k on +depart ment +aw ar +mar ine +in ci +photograph er +ta pe +lo go +r ings +d it +-- -- +vin yl +w c +vo ting +se ven +ambas sad +dal las +t u +com ment +k ra +b les +w ag +u d +au dio +stri ke +offici al +o ts +me tho +to ols +ra di +al an +hun t +wat ched +a ke +fa ke +drin king +mer ry +m l +b day +ri o +ni ke +c ant +re pe +co stu +mur der +ak ers +ch ers +ou ts +beg inning +so s +ad es +n in +not es +wro te +sol o +c i +li ghting +ur ban +bre xit +att end +shir ts +pla yo +ac tress +pl ic +stand ard +quot es +par ade +anci ent + © +tur ing +re e +pri mary +fla sh +citi z +mat es +ste in +z i +clin ton +sk in +gen e +hu m +g ar +t le +y i +fo cu +de an +pl ants +cy ber +b u +om e +ho p +ad dress +ti x +gi fts +relation ship +sub scri +fe ed +exac tly +haw ks +ex o +stre ss +s n +arre sted +an e +sof tware +z ero +the me +mu mb +im migr +mi a +make up +ple asure +uni vers +har b +eng ine +ap er +r in +br a +institu te +le ather +al th +sing ing +co s +gh ty +me as +st ic +si de +insur ance +co t +pit ch +moun tains +cri min +su pre +valent ine +at er +wou ldn +sc ale +rel ated +re gar +star tup +pack ed +mi ke +week ly +p ts +coun t +ha r +gott en +min d +ber lin +con ditions +swit ch +cor n +sa ve +g li +emer gency +tun ed +sto ck +discu ssing +every body +s day +whe ther +wrest ling +ec es +gen der +ch en +ðŁij Ģ +madri d +mar athon +e gg +i er +th x +as king +kore a +wol f +ay a +g m +g au +at ory +v r +gra ss +k illing +b ble +ur o +un i +e th +sh ore +th en +re ale +bot tom +ex erc +k ar +or ies +ad ri +san ds +se x +. ' +volunte ers +per form +par liam +inclu de +deli ghted +execu tive +fu el +kis s +ã ħ +char ge +h u +ca kes +ve t +g lu +agre e +pr ices +n au +h l +g ru +ra j +streng th +b ic +sp ending +al es +av en +b last +: ( +yo f +nor mal +si x +qu ick +se a +d aw +mee ts +lo vers +upd ated +po tat +comple ted +coo k +opportun ities +p ure +organ ic +tem per +c am +avo id +par king +duba i +and o +di stri +to y +comple tely +don ald +tri al +bas s +b oun +back ground +v as +mar vel +lu m +ru s +t ool +com missi +throw back +fin ding +is lam +! ? +st op +e vil +or al +resi dents +i denti +o ak +ðŁİ ¶ +l il +span ish +chap ter +sto pped +direc t +ho sted +pic ked +lab our +lew is +defen se +à ® +health care +wh is +mat h +pe ak +ra ised +fi x +bu ll +th ir +chel sea +fol k +tr e +can di +pau l +ei ther +ad am +poe try +jewel ry +ðŁ ¦ +pr ay +Ø § +g c +o z +wi shes +fore ign +sun g +lear ned +en e +n ing +micha el +illu stration +legend ary +w av +b au +ðŁļ ¨ +cal end +stre ets +â Ĩ +mon ster +bu ck +g r +scho ol +ba th +wa ste +ne ck +ha wa +be ach +re plac +jec t +on er +fac tory +coun t +ðŁĵ ¸ +mor gan +der ing +se an +steph en +de p +no vel +vide os +ic al +press ure +arsen al +ex pre +ir s +tren ding +ss a +fla sh +re sear +thr ough +profess or +scul p +to s +gg ed +mm a +be e +a pe +hun ter +am i +he i +pla stic +bu cks +uni verse +le gen +niger ia +ple ased +ri s +thin ks +autu mn +i ds +d is +anth ony +ðŁı ½ +ak ed +gla sses +fin ance +z er +k as +con tract +nu mbers +sh aw +partner ship +t il +laun ched +s al +victor ia +theat er +usu al +nam es +perio d +eli za +i th +bar cel +ro cks +bag s +mat e +distri bu +j on +di ffic +ali zed +cur ren +sco red +b ha +du blin +ro se +in ted +soli d +beha vi +wal ker +simp ly +garden s +head ed +in i +ohi o +we ap +f o +gl en +e state +ran dom +th under +thr u +k ill +jac ket +it i +entertain ment +thanks giving +ent al +en coura +el o +a ther +tan k +high lights +f ting +ru le +model s +bor der +bj p +hus band +in done +ken ya +be ars +al o +n inten +pi x +str o +or ders +sal ad +ro ads +n or +l ation +sop hi +ðŁı ¼ +pi eces +b one +min s +inclu des +nu tr +phi l +s ent +fun dra +ga in +bor ough +n ad +mon day +activ ity +it ems +be coming +ken ne +de tro +car di +gue sts +u x +world wide +sever e +new s +thank ful +fic tion +ve ge +m all +si an +er al +inj ury +le e +men u +danc ing +scot ti +exam ple +( # +na i +studi os +ba i +ðŁĴ Ľ +j av +diam ond +vin ce +ric k +prote ction +lin col +cham ps +appro ach +d ar +m ile +clou ds +je ff +in fin +l ers +p les +pe ace +go p +âĻ ¡ +tech n +str a +a verage +ef fort +introduc ing +di versity +austr alian +am p +boo st +s ke +pati ent +appreci ate +ici ans +pu r +f ell +woo ds +illu str +ðŁ ĸ +ag ency +ac tions +brit ain +under way +se attle +el and +ag o +f ill +stre aming +pro test +challeng es +ky o +et sy +coo king +exper t +ru ss +rain bow +commer cial +sp in +be ats +c ry +val u +el i +th row +gr ams +le vels +michi gan +c ad +ador able +const itu +w s +pu b +mid night +th at +net fli +braz il +die go +regu lar +jo y +âĤ ¬ +li qu +ea stern +k ni +fl at +n p +bro wn +w er +se y +tt ers +ac ting +v anc +cy cling +program me +ra w +comple x +tat too +throwback thursday +se ssions +ro oms +si ght +speci es +bom b +lau gh +ke eps +mo on +offic ers +con ver +t r +ha sh +t ack +ri ous +ad ap +a j +reco gn +ex po +sug ge +confir med +rol ling +dre ssing +ic t +fri day +ph ones +ri dge +con cept +ro y +ke ys +ef for +c ate +k ne +ev en +l ay +commun ities +mo d +n az +every where +al ab +bit coin +ban ks +out door +feder al +sto res +h p +c al +m ely +sig nific +be ar +re public +clo ser +al lah +pic k +x d +pal ace +ch ill +b am +er ous +un a +al len +out standing +olym pic +supp ly +fi gu +v au +l p +char lie +un es +> >> +legen ds +ici al +co ast +benef it +mul ti +f its +far mers +am ount +si sters +har ve +hon ey +que en +b ers +pl ann +âŃ IJ +m u +barcel ona +al ber +stat us +re main +ex tra +c andy +vi ous +âľ Į +o v +warri ors +-- > +ju mp +am ar +x mas +stu dies +i ors +k or +don ate +pre p +fi sh +im a +pain ted +ad mini +co splay +spor ts +dro ps +fi ghter +evi dence +ðŁĴ ª +la ke +ro b +cine ma +pro file +à ± +stan ds +leg acy +sh ape +ro of +ci vil +i ans +sy l +sh am +vo ted +re tail +ph illi +li sted +du ty +n b +th es +f are +au ction +ffici al +stor ms +d p +l oun +sh ops +al y +ani me +multi ple +ðŁĺį ðŁĺį +psy cho +je an +ap art +candi date +gg y +con f +jose ph +w ick +me at +fr ame +c l +for got +ph y +f ing +li ed +re p +se ed +f all +u fc +nu t +lin d +mo de +fiel ds +en ce +s ley +ðŁ¤ Ķ +ch ill +follow ed +announ ces +cor ru +tro phy +them selves +ac le +al du +k ong +l on +s v +bro ke +ander son +ta i +stor y +tempor ary +activ ities +k ati +ari z +cry stal +spo ke +extre mely +tra ding +ðŁĴ ļ +à ¼ +in ch +ed in +out fit +equ ip +ma di +form ed +be ef +po p +ti ger +this day +ti red +neigh b +re tro +is a +un t +t as +kan sas +de st +secon ds +ta y +hur ric +o u +galax y +dad dy +bro w +bur ger +en ced +de sk +ac cur +secre tary +el ite +k ab +ch in +touri sm +bud dy +ici de +dre ssed +u d +vac ation +che ers +com for +charac ters +j et +bu ying +l ins +n ap +reale state +li e +af c +i ii +f ame +n r +b at +ag ent +ma kers +âĢ ¼ +sec tor +op ti +le on +di et +pra yer +hi p +mi r +le x +br y +an a +pas sing +w en +reco very +ak i +po pul +res ort +mar ia +stu ck +read s +ti er +perfe c +netfli x +p oo +cham p +o c +re duce +we red +comm ents +cla im +acci dent +s ag +h ack +sal t +kin da +k iller +i os +z y +ex change +lec ture +eng er +ic king +t au +reve als +pri son +z om +gh an +u l +jour nal +i ot +tr in +jon a +govern or +cap e +quar ter +spec tive +impre ssive +bab ies +t x +m ill +o y +har ri +jo int +su e +collabor ation +tren d +revolu tion +re new +alum ni +ge tt +sh ell +sun day +ent u +ni c +donald trump +block chain +paci fic +expla ins +sp y +ad voc +par adi +to f +star ring +p av +fe ed +br ac +smo ke +ham p +y am +to kyo +si mon +d h +e ffici +phys ical +n j +ell i +s low +gradu ate +americ ans +ti fy +f red +ap ore +fin ds +rob in +we t +not ice +se mi +un ve +k om +pil ot +scre ening +da ily +ðŁĴ Ĺ +roy al +sp a +vo tes +n ag +wh ate +att ending +exper im +ad dition +k ate +sto l +m ali +foo t +chri st +ch an +de e +lic en +glo bal +mo ore +ti a +bri gh +myster y +y ay +âĿ¤ï¸ı âĿ¤ï¸ı +cre ati +me chan +clo ck +di c +âĢ Ķ +pp er +al ph +through out +al low +re sources +selec tion +ham il +bb q +aa aa +virgin ia +dis ney +en g +so red +drin ks +f ancy +consi der +end a +jan e +hand made +du l +on tari +i us +s ville +color ado +whate ver +whe el +promis e +ne ver +desig ns +ab ly +sex ual +vanc ou +at i +con vention +cul tural +sing apore +pro mo +load ed +gla sgo +pp l +n oo +ke e +ste m +men tion +i do +cru ise +ri ding +be comes +be y +âļ½ ï¸ı +tw in +dedic ated +na sh +de si +work out +jen ni +i v +grou ps +rela x +pho eni +li ft +mix ed +m ck +p c +mu st +me tro +ci es +y ar +a im +ang er +i e +rec y +marri ed +dro pped +eng ag +le st +ambassad or +op h +de s +w ick +assi stant +nat ur +fa il +l td +shor t +k ap +sha w +bi gger +rema ins +crit ical +sur vey +co verage +er son +win d +n b +bil ly +let es +ac ts +jim my +at lan +al and +t c +import ance +dam age +f g +stor age +tw t +bon d +bal ance +cr ying +pu ppy +vo te +pu sh +ðŁĴ ľ +pol y +me l +lon don +terr ori +effec tive +corpor ate +atl anta +jac o +nas a +gre ek +sen ate +i sh +ev a +intellig ence +effor ts +al co +k un +h all +di ag +claim s +fir st +h b +ba e +v ul +pu ll + ° +se par +spe ed +vic ti +on thisday +audi ence +r ates +te ach +fil ming +bu sh +son g +y um +br un +ra ine +aw a +par ks +ð Ŀ +ra bb +ra ch +ra id +reach ed +ra il +mo ves +selec ted +fr i +ra ising +om y +st ones +su k +franc isco +cas es +cap it +con fu +w tf +po ke +equip ment +gre g +ess ential +off ering +ne x +pi es +be c +cre ation +chair man +cro wn +w al +john ny +shi ft +ne ck +ban g +bir d +ðŁĺ ı +du ck +re serve +de pu +ma sters +over all +no tic +ju ice +sne ak +che er +cla sses +eag les +n ca +car pet +ci vil +coach es +har ris +u ps +b alls +dec or +mar tin +ro s +v ice +announ cement +who se +ti gers +ste red +c ts +dr am +ste el +youn g +inst all +supp o +recor ding +de ck +se ats +l der +ang le +bo t +sty les +elec tions +for tun +n ab +but ter +ari an +ka sh +in ner +ou red +be ast +we i +ic onic +exper ts +ne cess +b eng +jam es +li a +gre ece +ðŁĵ · +ðŁĺ ģ +good bye +m itch +tw ice +mumb ai +ste am +ru sh +med al +ne tt +fashi on +t ar +r s +sav ing +ric ul +l m +sleep ing +brook lyn +mis s +sen ding +disco vered +sp here +of theday +k icks +missi ons +w right +er n +ght ly +i ous +mel bourne +star tu +mo ved +car ry +d ak +ag ues +bel gi +e ma +way ne +do t +er ie +pe l +it unes +matthe w +no body +est ab +cal m +win ds +lu c +prep are +tren ds +exerc ise +adv ant +ðŁĴ ¯ +athle tics +app s +c tions +adv ance +laun ches +litt le +real donaldtrump +eliza beth +carol ina +hu b +hi dden +n w +us er +pol l +great er +mo st +f ed +p at +life style +s ati +sco res +marri age +l r +aven ue +de serve +ri f +ðŁ Ĺ +wat ch +champion ships +gr ay +en ni +cot ton +g om +whe re +pack age +su m +ab solu +new ly +foo ds +ty ler +assemb ly +musli m +ban k +re memb +op tions +produc er +land o +fun ds +u pper +shad ow +pro gre +co p +ing e +leg s +detro it +hill ary +jo se +gi ants +sou p +sustain able +t us +clo thes +roc king +n z +min ne +mat eri +bru ce +ear t +ca sting +independ ent +thou sands +ta h +de cl +veter ans +li ons +wra p +âĢ ¦ +de ss +bl ing +st ine +e ggs +o on +clo sing +z ay +at t +bac on +fa il +ariz ona +de pre +gho st +new sp +w ers +vi p +li ked +id ent +volunte er +ad ult +pu pp +cir cle +mat erial +degre e +gro wn +boo m +calend ar +su r +vie wing +ath letes +ch and +re ll +asi an +en tr +vol ley +victi ms +bo dy +m ama +trans fer +ge ek +in dic +sav ed +ma i +g ent +it s +loun ge +k ol +the ory +situ ation +is lands +ar th +z oo +floo d +vi ously +show ed +parliam ent +ch ev +el ine +at trac +ab ad +ta il +h rs +lu s +por tu +gor y +provi des +to ys +de ath +in fe +an ce +g le +li am +lo ver +hu d +dv d +reve aled +g w +re ment +ca the +l ying +ra dio +der by +stor s +che mi +hosp it +âľ ¨ +' : +ilo ve +le mon +re public +s ni +ne ss +do or +re action +pre gn +fla v +schol ar +spo tify +is ation +vis ual +aw are +spon sored +jo ke +less ons +leg is +lo ck +si mil +ðŁĺ ĭ +kin d +la y +ma h +ho ping +vancou ver +as er +clean ing +gal a +thre at +la p +ach e +ro mance +ex pen +re post +z am +e pi +mir ror +o ak +ad ul +bat man +s lu +l c +vie wed +re views +d ates +indone sia +acti vi +off en +lea f +i si +ag ricul +costu me +s ites +spir itu +appear ance +ir y +st air +applic ation +spec tac +ic ity +ski es +hand le +pun k +paradi se +t n +de al +provi ding +do c +recei ving +bre w +micro soft +à ¶ +fer r +me tro +th ail +y um +car ter +à ¡ +gent le +bre aks +coo per +show case +cu tting +egy pt +bab y +semin ar +gl ori +ss on +fa ve +re hear +lo tte +la dy +al as +pre p +deli vered +nu clear +ir o +engag ement +at ta +con ven +z an +gl ory +hol ds +busine sses +str ange +sch e +it self +gra d +mar kets +f alling +st ats +ge on +bu dd +li s +she et +thi si +co lo +deser t +regi stration +ig n +expla in +inter ior +la ws +writ ers +spr ings +k r +fri ed +blo om +inf ra +a o +cre d +pa st +line up +bo o +bre a +boo ts +celebr ity +att acks +bro ok +ev es +ex cu +cher ry +oo p +fas cin +boy friend +se as +n ine +effec ts +po wered +k ha +ðŁĺ Ģ +sh out +con dition +i j +her o +enter pri +win ter +applic ations +sho e +g el +batt le +pro grams +w art +ðŁĴ ¥ +ra p +ho l +dang erous +di a +coun ter +ric s +i or +k night +co at +emo tional +at ures +d as +whe el +fore cast +tran sport +glasgo w +king dom +prepar ing +im medi +ff in +awar ded +prin ting +ro man +fight ers +any more +bel t +p ine +win e +x i +employe es +logi es +al led +de mo +birth day +ange les +lo g +dri vers +neck lace +k ath +s it +athle te +ef s +s burg +pur pose +resi stance +rele ases +t is +vari ous +deli ver +ch al +s anc +opp o +cra w +neu ro +dr a +suppor ters +sna p +diffic ult +swe ar +logi st +pa th +attemp t +à ¥ +swim ming +ste ve +hur t +inclu ded +b ap +wa re +ðŁĴ ĭ +end ers +ja ke +le eds +cli mb +l b +im ple +li sa +clo thing +ðŁĺ İ +d t +com pla +sw ing +stra w +v als +k le +us ers +stor m +cu ts +ontari o +p an +hand some +i ow +ar gu +chec king +scotti sh +Ķ ï¸ı +si er +em ma +po d +patter n +de sh +en h +ed ward +t ing +k h +hal f +lincol n +mo ther +al leg +r c +volley ball +d n +g ay +all y +le ton +gro ve +l oud +adv anced +re spec +cli ent +supre me +thail and +ho w +gi g +to i +do t +dol lar +ðŁij ĩ +p it +r b +h n +produc ed +gg ers +âĨ Ĵ +ml b +can vas +fin eart +us d +in the +p son +actu al +s l +t b +ip ad +en sure +u mb +w d +sk a +mar s +k end +f eli +th ing +count down +absolu te +r out +dra l +p y +inju red +min t +hun ting +mm er +s age +li gh +ac ity +ex pan +mur ray +ar o +sec ure +four th +eag le +reli ef +st akes +industri al +clar k +under standing +see m +pl enty +sil ver +cla u +thre at +sa il +pro duce +ab str +is is +b r +eng ers +wor ry +bie ber +s j +just in +reali ze +ky le +esp n +fil ter +s ch +ty pes +game dev +d ing +twit ter +soldi ers +p om +car bon +y ards +child hood +ri ed +ke l +ele ph +t ons +key note +qui et +wi re +po sting +is sa +repre senting +bac ks +alex ander +celebr ates +ta ining +| | +ch or +esc ape +pe ek +ti ves +fiel d +ssi e +im pac +spons or +r c +we dd +cann ab +si des +trac ks +com par +con trac +techn ical +bi ble +expl oring +sh are +tra v +n ate +ill o +sc ru +m ingham +gun s +of the +sh ame +se es +ca tho +ac cess +ce l +repor ted + » +mari o +p ad +hope fully +ou se +y on +disapp o +ol o +p itt +pa c +ga p +cru sh +s g +k le +ge m +emp ire +dir ty +a is +avi ation +ze aland +fac ing +high way +d anny +spi der +ot ta +ðŁĺ Ħ +w y +col ours +in fl +co sts +olym pics +au s +h m +ho ward +pas ses +lau ren +mu sh +op in +r ho +disc ount +oper ation +em ily +mm m +cham ber +d il +to yo +shi p +sam u +pic tured +un ic +po l +keep er +carto on +st en +ig nor +n ations +n l +ta sting +deta il +offici als +mo tor +franc is +ed itor +ðŁij ĩ +pe ts +rang ers +t g +r n +w ri +nic hol +i se +spo ts +ani e +chec k +tri ple +ku mar +spe akers +ic ing +pre pared +ab use +friend ship +mon th +swi m +air e +sc ent +hamil ton +indi an +j es +yum my +te ars +da wn +i zed +worl ds +ðŁ ķ +b illi +st one +n hs +ba sic +p or +st le +ir on +ol der +cle vel +e ing +ðŁĺįðŁĺį ðŁĺį +prin ts +fir m +air craft +fin est +devel op +aar on +t z +gra ham +own ers +fo li +less on +qu es +bab e +cra ft +ph en +ju n +bir mingham +v ine +ll er +i an +fineart america +evol u +st ab +im per +war d +com ic +wi z +inv ited +du ke +mat ch +por ts +ro ger +diag no +ke pt +te st +vis u +r hy +so c +to x +b aker +sur face +co vers +man s +b its +x box +ff le +n an +gar d +h art +wat ers +v illa +re tro +light ning +catho lic +democr acy +neigh bor +pen n +cr an +jona than +la ura +vi bes +su b +coach ing +clear ly +uk raine +bra ve +commit ment +t all +mar t +ra p +mo di +sco tt +bro s +show er +ðŁı ¾ +âĺº ï¸ı +cou sin +appro ach +br e +com pos +hil ari +phil ly +g ad +quick ly +ri an +t m +vir tual +hou ses +k t +phoeni x +w ire +ff y +b unch +anc ing +tal e +snap chat +star ter +h t +k icking +ap art +th y +) ! +blo gger +it z +com fort +ang els +w ash +" : +ar gent +re quest +hon est +mi ghty +bo bby +k g +ro l +thou se +ex po +h c +tab les +mag ical +po sts +de m +n w +or lando +ab er +* ** +ðŁĺ ľ +environ mental +trans formation +mi le +w ic +hir ing +ma ine +bo ar +r ying +ti s +nit ure +twee ted +anton io +opin ion +fin ale +di y +f is +th in +trou ble +le go +fi les +qu art +sp a +curren cy +cli mate +fan art +rail way +sp ace +ban ds +dani el +mo tion +l eng +hol der +oc cu +mar ie +cathe dral +bu zz +bi es +nas car +bm w +bat tery +char lotte +doc tor +zz le +se ven +in san +d dy +st en +lab or +thr illed +se ren +docu mentary +wav es +cer tain +can did +allow ed +ninten do +star wars +ta p +home made +d les +ther ing +bre e +emp ty +pi ano +pos iti +coun try +por k +pu ts +per ry +m atic +spot light +ti st +or ities +we alth +c p +bar bar +commit ted +as sau +pro fit +e ight +hu l +fini shing +run ner +ss o +insp ec +char ged +christ op +lo sing +co al +ho o +ele v +de le +mo ham +don ation +c able +clin ic +j in +manag ed +ter ing +â ¬ +ur ban +depu ty +bb er +bur n +acade mic +o tt +sta ke +it er +sto wn +ack er +advent ures +ad ams +gre g +pro m +vo l +ac qu +con gre +pa int +citiz ens +c all +af ford +v c +as ks +the tic +independ ence +â Ľ +h itting +bl on +fu ture +â ı +in no +gen e +bo ards +di stance +se t +re mem +th al +pre vent +l ang +ob jec +su sp +mat t +in duc +bor o +pi one +re di +vir tu +prin ted +sco pe +shar k +suc ce +a stron +il legal +j ag +c ting +ine e +at o +rob in +nutr ition +b f +du tch +b n +fur niture +for gotten +at ar +ru p +hy per +bran ch +communic ation +degre es +on ia +un cle +promo te +or che +wi i +j s +but ton +ma jor +c bs +bri stol +premi um +ordin ary +e dit +m g +we ed +st even +: ' +gu s +te s +cap tured +dru gs +do w +wr ites +bi shop +whe els +ali zation +disco very +w r +rach el +ne il +hy dr +cu test +entreprene ur +kore an +ore gon +ul ty +perfec tly +suppor ted +histor ical +t wins +ell y +we l +de vil +in come +scienti sts +de leg +h en +on i +ic ed +gi o +cur ry +reve al +e g +buff alo +n ol +op era +camer on +haha haha +j ab +gradu ation +cra ig +r al +i f +organi zation +le ge +g ang +su d +edin burgh +l ack +fli es +g ate +thr ones +q b +the real +e leg +pp in +c les +jam ie +tn am +cryp to +ou l +p ages +a se +roo ts +stu pid +a did +boo t +prote in +s ap +si um +su s +end or +fun ction +don t +en na +ch y +squ e +wor ker +m tv +e a +k an +ðŁĴ ļ +mu s +professi on +t to +oper ations +al lo +c tor +inv ite +sc and +ou th +z im +lin ks +cli ents +sam sung +discu sses +n ell +ul tra +some where +ste wart +ine t +de z +b out +fac tor +ti an +tr ans +jere my +d b +ðŁĩ ¬ +or n +develop ing +spo l +coo per +ma u +rememb ering +tre k +famil y +sen iors +fo ster +att ended +w ing +trans form +ele mentary +hor iz +li sting +malay sia +it ch +warri or +philipp ines +russ ell +m end +initi ative +cre ep +to ps +br iti +a ur +shar p +adverti sing +ug ly +achi ev +materi als +bu g +dev ice +bon us +fac ility +col e +nh l +y as +plann ed +pol e +excell ence +tr ick +con fl +r p +achi eve +lo an +swa g +jess ica +ho we +p our +sc u +z oo +r ated +dre sses +re bel +mex ican +co ordin +me ss +atlan tic +t l +osc ar +wal ks +phar mac +investig ation +... # +cc i +eas ily +monday motivation +y ment +au ti +for ced +ar med +colle agues +pap ers +pro per +sha ke +bu c +le an +exhi bit +e vement +co tt +bi z +sp er +k ent +sw an +/ @ +girl friend +haw k +âĺ Ģï¸ı +mon o +ðŁĴ Ľ +stat ue +ðŁĺ ³ +ra s +te eth +preci ous +t ile +p am +swi ft +v ali +no se +dr unk +experi ences +come back +gen ius +wor se +sh ef +ra d +ed it +hon our +au spol +lar ry +h ire +gor don +achi evement +.... .... +su icide +alter native +su p +sur roun +sha ke +ke ith +pe pper +tur k +crimin al +be ck +su m +w alls +cn n +an tic +of fe +col li +win es +high light +hawa ii +emb ar +l fc +ðŁĩ ® +m v +> > +at mo +wor d +car l +shout out +bre wing +ì Ŀ +do f +s ic +hot test +col on +hh h +shu t +low ing +volu me +apart ment +agre ement +de stro +we e +religi ous +iow a +ro d +land ing +re present +ðŁĵ· : +la s +usu ally +h l +c ac +sal v +al ong +laugh ing +be ans +remin ds +pha se +some body +ma sk +ran ked +dest roy +sc i +âĢ¼ ï¸ı +gab ri +le o +ro a +fa iled +si l +refuge es +re vi +r ing +ber ries +coo kies +y y +conserv ation +sh ab +human s +de termin +a in +ni all +as su +mb a +fro m +extre me +vic es +commer ce +ght ful +or dered +suppor ts +re cap +v or +dro pping +correc t +pay ing +mean ing +n j +qui z +" # +busine ss +ðŁĩ® ðŁĩ +indi gen +du st +box es +bl ind +x xx +zz y +ðŁĩ¬ ðŁĩ +ss els +s ant +dd le +hilari ous +desig n +wonder ing +vehic les +k re +ju d +rece ption +par ker +Ã Ń +pri vi +hy dro +sof tball +pol lu +lo cked +ba h +e ar +scri pt +di vi +br ace +geor ge +the ast +bel o +j al +tion ary +dent al +roc ket +pur ch +sh ak +manufac turing +e z +it is +con cep +tb all +ch s +direc ted +pra yers +oo k +phil os +vari ety +che ss +ser ver +g and +bal ti +ðŁĵ ¸ +sel y +cru z +spectac ular +bur ning +re present +i z +t one +mer ce +h ell +bed room +estab li +bo l +com mon +ãĥ » +ab or +kit ty +hei ghts +re pair +willi am +qu ake +alab ama +popul ation +re v +re tt +i sts +n ite +le m +a ha +clevel and +r m +po ver +ob se +mon tre +man ia + ® +con ne +car ni +sh ah +f y +u a +sc or +strugg le +bo b +' ' +appro pri +deci de +ff ed +ca ster +s ort +hun gry +dra g +ا Ù +gr ounds +d w +sli ghtly +car din +dead line +bron ze +web in +bar ry +sil ence +e uro +op tion +ear n +ðŁĴ ĸ +howe ver +na ren +na ils +bath room +v ine +ph d +min ing +gar age +( ) +shou lder +defe at +di r +o v +liber ty +ple as +x on +com pre +a v +j in +ab les +sil ent +fam ili +vis its +di pl +ha bit +milli ons +regar ding +innov ative +sen ator +r ts +v on +k l +wh il +requi red +âĿ Ħ +lu v +presi dential +po cket +hun dre +sho wn +fro zen +to ward +fa st +confi dence +r ough +indivi dual +qu et +ðŁı ½ +dom e +fi fa +engine er +z en +re mix +ðŁĺ ĥ +pl ant +min or +robin son +as y +pul led +cer tain +potat o +( : +pre s +oc ca +w it +it em +si e +d ating +thom pson +own ed +an u +vi e +te dly +good night +ex cept +ðŁĮ Ł +ira q +ki e +ren ces +li p +simil ar +sau di +vi g +arth ur +pic ks +mil an +hon da +ma xi +o g +ste st +ar ch +analy tics +ba sti +pear l +ter ry +hor se +ast ro +ac ce +laun ching +inter national +s no +ta sty +den ver +ir l +pe te +tor n +advant age +var sity +" " +sol e +g c +lan g +demon str +ol ds +un ity +ne ts +insp ire +cre te +nash ville +nel son +e ter +wal k +hy un +m ack +tre as +see king +ra ge +bru sh +ab and +whil st +co con +h ong +shel ter +i p +possi bly +so o +it ed +â Ħ +rac es +war ming +qu in +tele vision +mat ches +ra pi +ment al +pal m +jenni fer +rol ls +indi ana +b ars +cat ching +resc u +candid ates +fa re +âł Ģ +se o +vie tnam +alph a +michel le +visi ble +re gre +wn ed +app le +li p +f fe +li z +york shire +ha il +se asons +be gan +m d +k c +la p +fascin ating +hel p +ur y +u ms +nu ts +se m +along side +bri dge +ori al +o ve +world cup +briti sh +comfor table +i ve +hot els +fair s +hor ri +so x +d ining +stre am +bar ri +ss y +w im +ter ms +v u +pe re +l ens +wal ked +r or +l ars +shi eld +dou bt +pro to +cro ssing +me ant +medi um +ad ding +e b +che ap +fun c +pap er +bran ds +ry an +feed back +col lins +un known +tro pical +sand wich +fal len +for mu +selec t +lo ads +answ ers +or i +mag a +d or +du o +ali e +dru m +ur i +de er +sou l +sh ut +âĺ º +sto len +don ated +bu zz +patri ots +ha l +na sty +nomin ated +mon te +ki a +th ri +ing u +te sts +pe tro +ðŁij ij +ho sts +ne st +to pic +pat ch +m my +hu gh +ab ilities +ma the +s miles +g b +ag enda +insi ghts +chi p +ph an +fail ure +dg ers +ha i +signific ant +sho ck +ru ral +gl am +figu res +pot us +o ta +mini stry +appe ars +fe ar +r h +americ an +h att +son y +fi res +e di +n ou +e qui +wh en +univers al +mad ness +i x +sculp ture +b ach +t to +swe den +et a +en to +develop ed +month ly +ma ps +ra h +le d +del ta +sa ints +is lam +ben ch +fif th +v ard +so cks +wel coming +j e +tur ner +v b +ad i +nor way +ad y +hurric ane +por sche +tra dition +ex am +newsp aper +lu ci +a ver +ide al +d na +madi son +ðŁ § +wit ness +ac ou +insi ght +si mon +robo t +sna ke +n bc +ac o +ro ss +sh ment +religi on +ch ann +in su +camp bell +inst alled +we ather +hor ses +ol i +rober t +k az +ðŁı Ģ +veter an +th read +quar ter +ea sier +cap ture +hi pho +law rence +roman tic +pas sion +cl ay +ox ford +th ai +stu dying +fi a +elec ted +most ly +c b +tu mb +âĢįâĻ Ĥ +x l +sh an +fa ster +ev ans +sli de +sh ri +see k +mi es +chemi stry +pump kin +tu m +, , +ro om +fi red +li ps +pres ence +af f +brew ery +arri ve +sw ag +photo graph +pen gu +chi ps +at tor +val ues +accur ate +con temporary +princi pal +cannab is +ari o +any where +gi a +democr ats +buil dings +li ved +ap s +neg ative +m are +bal lo +li on +diam on +loo k +re form +tom my +il la +tre ats +hundre ds +port land +wor thy +ex cep +ar ia +ido l +be er +cd n +y u +aw k +ðŁĩ ¨ +c ells +à ³ +ident ity +dra wn +de vil +f inger +th am +ðŁij Ĭ +ear ned +fin tech +dol ph +twee ting +evolu tion +ðŁĵ į +est im +m vp +n one +ðŁĩºðŁĩ ¸ +toyo ta +au x +mar in +b old +l bs +ste ak +mur phy +it able +lou is +sol ve +pi a +sk ir +ill ino +webin ar +ban ana +lo v +th on +vo ters +afford able +defe ated +lm fa +air lines +super b +any way +deb t +bo red +ver si +me tal +responsi ble +m k +s se +f ay +cau sed +f p +recomm end +pla za +spor ting +alli ance +au stri +n n +t ours +surpri sed +arti f +th under +sur ve +wor e +bri ef +necess ary +z ie +ash ley +dra ke +r t +kni fe +im mun +char ges +a the +bri de +rep ly +g av +broad cast +pu er +brace let +cap acity +harve st +id k +perfor man +d ding +il ers +par a +jam a +pro vince +ch in +id ers +har i +te aser +ch en +re stor +r at +fl at +col om +ðŁĴ ŀ +ðŁĩ¨ ðŁĩ +smoo th +r t +p itch +stay ing +isra eli +t cot +per spective +do ck +open er +lo vel +x o +class room +l ington +go al +kenne dy +sh am +sp aces +mitch ell +home coming +uk i +claim ed +recru it +ing o +mu fc +mon it +g roo +resi dent +per cent +per man +otta wa +int ment +an xi +stand ards +wor ship +sche me +f x +pot ter +bi an +athle tic +af gh +s se +sat ell +par ties +âĿ¤ âĿ¤ +infra structure +rela x +mo du +wor n +smo king +y ach +practic es +wc w +am b +dome stic +tay lor +k entu +provi ded +mo di +ve g +" ... +ob serv +ðŁĺ © +be ard +m our +an gry +ðŁĺ ± +startu ps +woo den +di ve +na il +anti que +ro ses +torn ado +m at +^ ^ +su spect +far m +de vices +me ga +tu l +scholar ship +ge e +disa ster +arri val +po in +mar c +kati e +bb ed +fal se +deser ves +ric hard +ju ana +fre y +tion ed +hy bri +r w +sar ah +ach i +c ure +o le +mor ris +ch ic +broad way +la bel +pa k +pover ty +gol f +e red +f u +er ies +be es +alo gue +st el +wire less +je wish +ti de +blo cked +life time +b har +sp lit +am ster +th i +jo shu +br unch +ha ps +s for +oo ps +ka poor +hi king +suppo sed +ro of +re as +tra in +ti ght +tru mp +bas ically +r r +ea red +see ds +entr ance +c p +wi e +son ic +vic tim +he re +e h +ear rings +sal mon +arc tic +an ne +dou gla +corru ption +hann ah +ha sn +vo ices +con ce +att a +fle et +clin ical +democr atic +ton y +st ood +le f +twit ch +a il +honest ly +incre ased +dro me +don na +accep ted +visit ors +ap ar +ad or +p ar +jer ry +ra i +brand on +ab u +!! !!!! +me me +in gh +glori ous +b hu +pu mp +j ol +li ke +fi sher +ma z +ag an +destin ation +play list +le tters +gen u +br ace +celebr ated +bann er +r he +dra gon +ðŁĺ ħ +sig nature +gre y +âľ Ķï¸ı +al ice +be red +ph er +ber n +ca th +ga thering +sc oring +influ ence +sm iling +de pt +lo cal +a x +ac u +reti rement +hon or +her self +chem ical +asse ss +y all +fre qu +appreci ation +ac a +cho ir +cu z +so il +c il +repor ting +u h +enterpri se +gr at +jaco b +ru m +fe e +j ak +sp in +bi kes +phi a +ste re +p is +bloo d +t att +ra ft +war ren +sh eri +back stage +mar sh +hash tag +ther ine +re in +game day +guar an +reci pes +min ds +stron ger +issu ed +bic y +n ak +ment ed +sc ary +u x +pre vious +tt le +th ats +ac tors +u ma +tin a +bun ny +promo tion +u ss +oli ver +montre al +what s +appreci ated +la kes +excu se +kno wing +pri zes +musc le +shad es +sco t +ing redi +electr onic +ju an +comb at +s ri +e h +turk ish +l om +stri kes +pri son +re e +po pe +vi d +ol dest +dol l +sw iss +certi fied +cli p +re turning +lat or +le igh +tt es +wat son +heal ing +el im +per haps +ha ss +k au +d der +mou se +new castle +indigen ous +wel comes +co le +tau ght +no ise +appe ar +jo e +can on +wedne sday +u tah +c tive +dri ven +i v +c ell +stri p +ac c +focu sed +ar rest +sto cks +wo o +â Ĺ +notic ed +shad o +di spla +ter ror +bor ne +secon d +que ens +wo ke +ja il +no tt +cam bridge +har t +se af +fa x +ac cept +âĺ ħ +goo ds +k at +t win +h s +thou sand +s ins +su ite +amp ton +ar n +rele v +ric har +hoo ps +n bc +class ic +p ab +soldi er +de plo +le ans +install ation +cla sh +le ban +ee e +ti re +belo ved +fu sion +travel ing +ne i +coo kie +glo be +phys ics +s q +co l +wol ves +d l +ex it +" - +foo tball +le af +ster ling +hi de +minne so +fresh man +natu re +indi e +supp lies +bri s +iri sh +ink tober +doo dle +ic op +mess ages +adul ts +recor ded +fix ed +ar do +offe red +under ground +dr one +p ine +ma inten +and re +ham mer +s x +r ound +hi ke +bra d +ro me +fu ll +on ey +ro ws +colum bia +archi ves +appro ved +bat ch +illino is +recogn ition +shou ldn +fo g +nca a +ke vin +human ity +al though +pow ers +p ou +s ar +pe st +alco hol +con sci +phil adel +en o +t m +ok la +cate gory +particip ate +accu sed +bri ef +po em +clu bs +consul t +ja b +big data +amster dam +ac ing +certi fic +n u +d at +impro ved +and y +campa ig +pale stin +p ace +mo bi +feel ings +wol f +bra in +pro pos +inter active +prin ce +inde x +c is +cha e +peace ful +co vering +ac o +cour ses +mon key +re place +b l +bloo dy +tal es +brigh ton +neighbor hood +g ates +spiritu al +af raid +bre ast +b ones +ðŁij ī +vide o +w au +tou ch +inju ries +car l +ri x +une x +âĢ ¢ +fre d +consi dered +thu si +an ch +on y +u sa +graph ics +ac re +ðŁĺ © +com memor +com mod +go ti +guar dian +star bucks +pre vention +haha haha +admini stration +portu gal +fac ulty +bet a +ul a +al bert +bre ath +er i +le tting +tr ic +ment ation +incredi bly +ten nes +v d +ðŁĻ Ī +ed die +br ick +gr ill +bt w +wat ches +resear chers +t ney +ni e +p as +a ster +vi br +poke mon +ch rome +go at +pitt s +il ly +festi ve +y d +can al +ðŁ Ĩ +fi es +car los +re que +partic i +tra ins +sam ple +temper ature +sym ph +pic king +in door +z ers +playo ffs +____ ____ +ap es +ly rics +islam ic +performan ces +d ick +spar k +se as +hom a +gr ound +disc i +employe e +com mu +alas ka +al an +fe ast +dg ing +ban king +manu el +slow ly +tru cks +mc car +oo o +sc rat +orche stra +indivi du +m x +bre ath +stair s +equ ality +bla ke +loc ations +cocon ut +balti more +aa a +l c +ðŁı Ĩ +har vey +resi st +immigr ation +adid as +fil i +re f +lg bt +mo s +pp i +ken ny +terr or +ban e +apol is +s g +social media +ka i +hon est +as sas +bol lywood +âĢįâĻ Ģï¸ı +ferr ari +hor n +cryp to +bo om +mainten ance +i di +s man +w l +ext ended +in sul +ve s +go sp +tr i +pi g +tar ge +cel er +st ati +sm h +ri dic +appe al +? ) +con clu +cos me +she ep +christop her +en thusi +po lish +me ts +oun ded +sustain ability +creati vity +con crete +ra i +ali en +ble ss +te es +clu b +ro t +bo s +ex ist +perfe ction +lu ck +rock y +expen sive +mean while +happy birthday +pre t +thr iller +ca ve +playo ff +som er +l u +le x +def ence +am writing +home less +pro phe +ch et +past or +ðŁ¤ £ +land er +ww w +Ģ ï¸ı +tic a +! # +o tic +rad ar +po sters +pow der +po li +ha un +tra p +bl in +assau lt +shor ts +re y +sh y +squ ir +rac ist +gar lic +fu r +remo te +sm ell +impre ssed +fing ers +âł Ģ +din o +le ment +s nu +promo ting +str ing +produc tive +b age +ma son +ra z +direc tly +j k +ev al +ðŁij Ĭ +doc tors +co w +ri der +st v +re move +w u +na than +ro d +n r += > +affe cted +inve st +mp tion +g inger +o d +agricul ture +s que +mu g +coun ting +ke e +mag nific +coo k +ani stan +roo t +plac ed +sym po +gh ana +un d +che er +thro wing +secre ts +f illing +opti mi +butter fly +bu bb +ðŁĺ ī +terri ble +d g +sil k +obse ssed +lo u +ai de +sal ute +mon u +philadel phia +scienti fic +i st +u ae +dess ert +bott les +can yon +ðŁĺ Ī +car ib +o ther +w ich +re source +guil ty +un d +le on +e ss +kan e +el e +tra iner +he im +an te +man age +roo kie +tre ated +po ses +rs vp +cau ses +aw ak +je well +le tt +on ics +tit les +cardi ff +g aga +bu mp +use ful +? ! +loo se +bb ing +: : +argent ina +de bu +cy cl +wh el +dis gu +j el +k ills +bio logy +ex ter +tra sh +bo dies +tr am +circu it +expe ct +la ds +w ells +sho t +ge e +naren dr +fa stest +b ent +b ills +mar shall +h ats +intro duce +citi zen +im possible +gi b +az z +net working +r ant +thin k +in dy +st ops +f theday +bri an +* * +amo di +dom e +coura ge +pac king +af fairs +g n +si zed +ent ary +pol and +swit zer +afgh anistan +w u +ten der +subscri be +mo sco +att end +republic an +hon ey +âĢ ĭ +si mul +we ster +foo die +or o +midd le +ab t +co pies +ma je +narendr amodi +ty pical +inspir ational +vit am +wis con +cu bs +tiv ity +h ali +e ars +k ay +d are +mari juana +cu rious +an ia +tom ato +re mind +ðŁĩ · +sc ared +cou p +po et +land ed +ri d +wra pped +mor ri +climb ing +e ws +fe eding +con tra +tho logy +gri d +ti vely +read er +la ser +di ving +di g +lat in +ti ed +shake spe +o ci +ad m +show ers +chu ck +mar cus +oo s +kne e +o live +ow l +dy lan +an no +g ym +deci sions +well ness +arri ves +sati s +chri s +thur s +ðŁ¤ £ +inter views +thank you +switzer land +over night +journ alist +ser ves +vol can +.... ... +plo t +nic ol +car rying +mag ne +tre asure +ex p +be ver +ðŁĺ ¢ +mar ty +mo le +don ations +recogni zed +b h +du s +sh ann +al do +success fully +ent e +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +cab inet +cu is +tit led +d as +so l +strate gies +deli vering +ad ds +ani an +ne ther +ðŁĴ ĥ +con tain +su its +pa irs +to dd +rel la +ro pe +ci o +cro p +paint ings +su z +re jec +bu st +d h +fra ud +m h +contro l +je al +destroy ed +al lows +wo ol +minneso ta +om en +j u +sympo sium +d af +lim it +accoun ts +load ing +inter n +re solution +hol land +qu al +meet ings +gra ve +cam ping +v am +re nov +liber al +am ber +gre e +hu mb +fe ver +el ing +broo ks +à ² +be th +ad ed +al t +ro e +perform ed +jo sh +frank lin +nic ole +de ss +bb s +m g +net works +min im +al t +weap ons +gu y +jas on +g ha +harb our +at on +pra ise +kentu cky +bel fast +st icks +blo ss +ho pes +an thro +famili ar +wa it +ch ile +depre ssion +la x +je ts +le ice +recei ves +si er +an k +de x +inde ed +fle xi +fab ric +lam b +hel icop +am anda +âĢĶ âĢĶ +compe te +sn ack +techno logies +sy rian +mom s +mu ham +cho sen +an at +dev on +shar ks +re t +fundra iser +selfi es +st ations +communic ations +tennes see +tu tor +ro t +valu able +dynam ic +nur se +i ed +earth quake +deser ved +a ve +sar a +stre tch +dougla s +ne pal +à § +ob viously +d ame +ra pe +any body +k w +pat rol +hol ders +h anna +info graphic +ec o +be ating +stan ley +bo ats +ri bb +e z +wit ch +inv a +ac id +boar ding +- @ +gi l +da ve +care ers +opp os +l loy +in ter +do pe +re su +j agu +sh ade +in dy +on ist +rel ations +ag en +ab le +inci dent +me ter +shar ma +id r +pro ve +immedi ately +tro ops +am an +g low +gaz a +blo cks +person al +chron ic +all er +si d +sh r +whats app +lu cy +ar chae +ho u +journ alism +our selves +go t +the med +shap ed +we ak +cas ual +leng th +sla m +ab bey +e v +coun ter +est a +reci pi +cha pel +expan sion +sel f +suff ering +sp ice +n z +sp art +desp er +boo king +quart ers +y on +ðŁĴ Ĺ +p k +continu ed +- # +man hatt +tal ked +sh en +com bo +hybri d +je ans +liqu id +se al +re tweets +ac celer +collec tive +t as +: )) +profession als +ra w +o tt +su san +ir ing +okla homa +re ven +survi val +cre ator +tran sit +st ac +sur f +i k +ed iting +ch illing +bai ley +ste al +ra ble +pa rent +hun ger +sn app +collec t +philos oph +dedic ation +c f +c m +le ep +repe at +re ha +un fortun +a er +a ero +abstr act +mon itor +ag ents +bu l +sci ence +harb or +drag ons +floo ding +ac compli +d ash +juli a +the red +tues day +cy ber +b low +ta ined +le m +refe rence +pp o +ne goti +char le +con nor +au lt +access ories +commissi oner +rain y +re ar +advis ory +luc as +ma id +co al +k av +pol o +ðŁı ¾ +tran sport +mar gare +straw berry +bur ns +gre ens +ne v +partici pants +col in +belgi um +col our +in form +d ell +br on +cal y +kick off +strate gic +re union +hon ors +li b +egy p +âŃIJ ï¸ı +hy po +si zes +regi stered +bet es +relax ing +bloo m +inten se +valent ines +insan e +w wii +p x +tri o +bla de +wiscon sin +con e +plat in +ali ze +ra ven +incre asing +indi ans +il ian +bl u +rabb it +exten sion +je f +au di +fer ry +s ell +a day +us b +swe at +cham pag +metho d +mem ph +assi st +s by +ca pe +remo ved +mag n +v t +r ams +f bi +tack le +phe w +h on +motor cycle +su spec +eleph ant +sub ject +let te +da iry +whe at +awk ward +ac t +tro l +mit ted +zay n +sheri ff +ene my +con s +ke tt +bul ls +ev alu +bt c +satell ite +ho lo +por ter +dia betes +bet ter +rele asing +sur f +: - +se basti +collec ting +en cing +e thi +go ds +al ley +health y +m ills +sma sh +co pper +cr ack +read ers +sp ac +licen se +bas ket +bang la +en tic +om i +m ere +si vely +anim ation +lan es +dent ally +chill in +fi e +k aren +dep th +li pse +n g +ri p +mel o +sand y +ðŁijı ðŁijı +vin cent +nu t +hu g +who le +cre ates +? ??? +âĿ¤ï¸ı âĿ¤ï¸ı +bak ed +up grade +rober ts +har a +carib bean +auth entic +mb s +mosco w +attor ney +wi ki +ch lo +hu ll +cor k +" ! +sty lish +ðŁĵ¸ : +di ary +impro ving +ex pand +bri ght +pollu tion +k nights +person ality +chec ked +fac ilities +z el +bow ling +gu er +ðŁİ Ĥ +on going +un its +hoo k +be ck +confl ict +to dd +far ming +educ ational +k ak +cla y +stro ke +bel ly +explo re +mill enni +th m +loo p +sm s +consi st +cir ca +br yan +d ab +youn ger +soli dar +pp a +experi enced +b ella +bo ard +shef field +steph en +consu mer +sub mit +spon sor +t ang +ag gre +comb ined +trac king +sand ers +b az +survi ve +fer red +equ al +se p +re ed +str ong +priv acy +st ap +un g +ac ry +pa sta +pir ates +ag er +fair y +du p +introduc ed +wi p +let s +spr ay +ðŁĵ º +gre w +a sts +pitts burgh +new york +jo ey +lau ren +tra de +ch op +pi pe +cla ire +behavi or +v ap +cre ws +lap top +ðŁ¤ Ĺ +che ster +disci pl +d f +out doors +k s +go ver +super star +cas ino +far mer +; -) +re turned +ðŁı Ī +ma il +roa sted +co sta +v ill +pe z +gard ening +distribu tion +sh ining +inve stors +ra sp +dec ades +reali zed +bar n +p ti +st able +ut d +pan thers +m ens +b n +ca de +bu cket +yn n +when ever +wa ke +da is +ber nie +lo dge +ju lie +atmo sphere +ðŁĺĺ ðŁĺĺ +major ity +par ti +exc it +cu t +me h +musli ms +be gun +fli ghts +vene ss +ce me +po sing +so le +g ou +dark ness +pe ach +cel tic +auth ority +grand ma +ful ness +smi th +speci fic +gar cia +co ins +good ness +aldu b +recru iting +den nis +gar y +sle eve +weap on +pl z +disco ver +harri son +recruit ment +ja i +ch im +com pared +tom s +mo thers +am y +archi ve +t ask +ben jam +se g +law yer +al um +inve sting +mi e +che z +j p +a ke +fl am +wall paper +âĻ¥ ï¸ı +t ton +che st +favor ites +we igh +coo lest +r ating +relev ant +lo gan +ma ple +run ners +pri or +peop le +ma ur +terrori st +te sted +carni val +su spen +me asure +m v +cyber security +app ren +terror ism +o z +v ital +ni es +gon z +fun ded +twi st +assess ment +die sel +en for +colum n +ad dressing +ca sts +pay ment +x ton +fi er +, ' +la st +ne e +un less +clo se +sk ill +cuis ine +fun eral +ti les +a un +k ru +relation ships +ðŁĴ ¯ +ev ent +âĢįâĻĤ ï¸ı +kind ness +pro posed +acou stic +a es +defen der +dan ce +h tt +w at +vo y +ðŁ¤ ĺ +au s +cli ff +sear ching +beauti fully +in qu +at l +speci alist +ðŁIJ ¶ +da i +tra ils +class ics +inst ant +v ous +re venue +mar ch +kir k +fr inge +fire works +tri via +âĺ ħ +tr action +wal ter +mo to +l ily +att itude +cli mb +sc an +sav ings +c w +fa ith +cred its +ab led +gra ff +auto graph +he he +ran ch +ha d +ro gers +ðŁĮ ¹ +f in +re qu +fol k +ad ditional +lyn n +u ber +dol lars +lo gic +wor th +so m +the sis +p ound +bi c +st ur +cer am +spen cer +en tered +v amp +organi zed +âľ Ī +pp s +tr on +merce des +no ti +compet itive +do w +ous ness +vic tor +gr illed +na i +pu tin +ab ra +bl ame +alex and +anim al +dec ent +p ent +inter ior +:' ) +but ler +bal let +ðŁĴ Ķ +albu ms +down s +la d +si r +pla in +p ers +blon de +dis c +paki stan +se ment +ga a +w age +ch as +man i +co ps +terr it +lo l +lau ghter +ri vers +magnific ent +lam p +w b +new sle +char ts +ble ssing +p unch +lon gest +fl oral +cu tie +fare well +sto pping +mb b +bu d +chee se +de cla +si m +mc donald +de ter +you th +t ch +fre der +kin dle +fer n +at or +as leep +p ond +spr int +p ounds +la zy +gh e +fundra ising +dead ly +gran de +dou g +he y +lin da +consi dering +i um +gol den +vi k +auth ors +di ss +u ally +appropri ate +mor ning +y le +hon oring +foli o +be c +re bec +fin land +formu la +corn wall +sh ay +cau sing +bl end +sig nal +t ent +kash mir +nation als +har mony +sc out +acce ssi +he ight +medi eval +impro vement +ke es +prac tical +car d +de par +hu n +om ing +cal gary +ste l +bu bble +gur u +ma h +unex pe +n h +ed a +me at +i ge +si o +god dess +in ches +tun es +br itt +sti on +ra j +âĻ « +mer cy +ðŁĴ ĺ +sen ds +i est +pol ici +val e +reduc ed +as ap +vi jay +defen sive +celebr ations +ri ders +med itation +har mon +g ing + ¡ +program ming +in au +sud den +m h +replac ement +sk u +j ar +gra des +ta st +k itt +brand ing +k aw +boo t +f ought +p ays +g f +iz ation +ho p +k k +activi st +v end +coast al +cha os +ðŁĶ ´ +se me +bill board +li fting +cu mb +sc al +ðŁĸ ¤ +stru ck +l v +indie dev +beat en +jun gle +al right +destin y +m ing +k c +ch ances +om an +q atar +cra f +tra ined +pri x +char m +o tive +s mu +e c +and ers +hand ed +al ban +certain ly +arri ving +i ze +sa i +tr ack +pain ter +hu mble +appo intment +head line +manag ing +mo d +as pe +andre a +à ¤ +ethi op +un ited +exi st +bal i +k ad +n t +d red +re x +recogni ze +tam pa +be ers +ati a +he els +no te +transport ation +tur tle +re de +hipho p +sp icy +sp urs +⬠ĩ +cor p +ther n +to ast +hur ry +proper ties +ma ge +mar co +ele ments +bou ti +syn drome +ms g +develop er +gra ders +he im +re sil +off ices +del ay +di men +vin tag +barbar a +ðŁĺ ± +vene zu +cu lar +fac ed +bar n +ðŁĺ Ĩ +survi vor +wor m +confu sed +passion ate +Ø ± +identi fy +electr icity +sou ls +brad ley +repor tedly +lun ch +shel f +eli a +swee t +smoo th +emplo yment +am el +manhatt an +ste am +oun ts +ye p +li ving +un e +descri be +ca res +man ila +sha wn +ac ted +bas h +st even +re st +pet ition +div ine +wel sh +rac e +platin um +ðŁĮ ¸ +p b +extra ordinary +solidar ity +m all +on ion +schedu led +game of +fer gu +de ms +nor m +p k +tri als +polici es +publi shing +st ole +fron t +charac ter +van ia +ex ce +sti e +sc a +resi dential +sa iling +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ +spons ors +th ick +champag ne +she pher +continu ing +ven ice +per th +na p +a ster +y ak +un limited +cho ices +ne o +hi v +repor ter +bru ssels +f old +dy s +se mi +la wn +it alia +wi fi +as k +em ed +fr ame +monit oring +ste ad +i da +gr in +is a +fli p +re stric +offen sive +atta ched +di sh +wh y +philli ps +gre et +p als +mix tape +v ou +fiel der +spar k +alber ta +g len +ca sh +s ri +u ri +ro dri +entreprene urs +climate change +p sy +d le +em ents +lin ked +nether lands +acci dentally +oppos ition +vel vet +ra ys +c w +om o +m f +lmfa o +newsle tter +: ) +toi let +liter ature +di sp +phili p +uni form +sudden ly +head er +cool er +-- - +prou d +bri g +nis san +scienti st +j ah +con centr +pac ks +appo inted +so ap +eng age +cho se +âĻ ¡ +se tup +jeal ous +har ry +g ation +tun nel +te mp +osc ars +dec ade +recomm ended +child ren +ab a +anxi ety +ve ments +sal on +pho too +organi z +mach ines +ab s +vil le +hy pe +ti ff +emer ging +av geek +[ # +contribu tion +bra dy +re sto +g mail +fit z +photo shoot +hel met +h t +eleg ant +ug anda +nur sing +or leans +pen n +na h +foo tage +em a +w o +w ad +concer ns +ve re +re mark +who ever +str ang +p t +qu it +sh ang +histor y +s ick +perman ent +ill ness +col d +visi on +he m +ar row +con vic +pin k +oc cup +bal d +ex hau +u of +am o +on t +ãĥ » +adop t +la id +smo ked +inter pre +ess enti +associ ated +b d +bb y +fi er +inst all +dipl om +con diti +c f +w ak +any a +gr aci +fi sher +s ss +ap r +il it +mus ician +symph ony +cor d +h ack +le gi +l v +bless ings +hum or +sc ra +e ti +min ster +trav elling +bu sh +jewell ery +li me +!! ! +pregn ant +pe e +lo b +cap ital +ip a +pen cil +la bor +duc ks +prou dly +wedd ing +dere k +m w +pe g +valent ine +an gu +re treat +pro spect +dang er +vul ner +up set +, # +sr k +x im +thur sday +n fl +kis ses +re ds +cr ack +re ward +c u +ko k +me te +aband oned +it t +me als +sp ell +stan bul +del ays +ru m +le op +gu m +no va +super man +ch ick +m is +dram atic +inno cent +r ounds +re c +auti sm +bangla desh +mor al +mo vie +sp oo +k la +âĥ £ +ou ting +mess i +ab road +loo kin +a im +q i +st ack +colla ge +à ¯ +hud son +sc an +ho e +ch au +oc cur +comm ander +ho les +ðŁİ Ħ +bi as +v on +stick er +ma k +responsi bility +colum bus +sa int +ed mon +rac ism +far ms +w en +gul f +may o +!!!! !!!! +corpor ation +ba chel +el a +inter nal +je ep +fol lows +di alogue +de rer +smart phone +he len +rich mond +equ ity +s land +b g +ne ar +av i +memph is +we ir +discu ssed +bad ge +p up +mi stake +phen omen +un ite +ðŁ Ľ +de pic +ri des +in augu +n at +sof twitter +comb ination +gosp el +âļ ¾ +ad mission +retro gaming +ðŁIJ ¾ +sch u +mb o +jun ction +al arm +à ¦ +gr ac +kh ali +k ul +m ale +cap tion +wi sh +te re +cor ps +ru bber +play station +er in +effici ent +l or +jo kes +in ary +nor man +lu is +inaugu ral +ch ed +âļ½ ï¸ı +di p +to e +str at +aa c +am u +pi er +co tt +comm and +tt en +sn oo +cu be +clo ses +class ical +s word +expre ssion +reach ing +n app +co st +affe ct +ric o +gi f +brea the +tri be +or tho +h ay +l g +fri es +n m +hi ding +richar ds +en de +mic ro +capit ol +cop y +ro m +regi me +mary land +tax i +di al +embar ra +un believ +ch t +v s +elim in +o dd +pen ny +sound track +l ings +trans ition +rema ining +a is +mali k +? !? +rand om +def end +ul tra +tru m +danc er +st ol +dri ve +a ver +ro ast +defin ition +se an +excit ement +partic ul +su rely +sh av +ber y +di shes +com m +is ol +i am +ob li +gho st +hugh es +chi efs +b as +conserv ative +speci al +fe min +sh ri +n ancy +inte l +tu ne +ðŁĩ ª +jo el +gg le +mo to +ðŁĺ Ķ +bu ck +d ag +antic ip +mont ana +gu id +fro g +ec raft +op e +dri ves +nu mer +x y +color ful +wednesday wisdom +illu min +bey on +inau gur +deep ly +pre fer +for tune +coo ked +ti ble +âĺ ķ +swe ater +it ter +tt y +u i +gi e +com plic +~ ~ +tax es +cu ps +di verse +sam anth +âłĢ âłĢ +ba king +sy mp +wa i +be half +mer cur +travel s +ðŁİī ðŁİ +or ia +eng aged +jump ing +reti red +n aked +p uni +speed way +sci ences +rehear sal +on ym +dy ou +pl ates +r ati +kri sh +jaz z +car ol +ra f +pen alty +tim eline +ru by +engine ers +ra f +bel le +do se +che on +esc ap +me g +ran k +or d +me gan +mer ch +ec lipse +âĺº ï¸ı +ple dge +kir k +per si +leice ster +sa k +w k +saf ely +yy y +je t +promis ed +j c +en ne +no ah +re no +re a +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +tra il +ðŁij Ģ +f d +soo o +ri min +w k +ภ² +i al +x ox +bis cu +d ale +fan dom +particip ating +fla g +privi lege +pe ach +mach ine +bo ston +gro ss +o g +mir acle +adop tion +u ss +mon sters +be ij +clar ke +pu shing +pra ying +ar o +d n +ell is +apol lo +od ds +refuge e +to w +b p +ðŁĩ¬ðŁĩ § +h end +app eared +memb ership +pe an +du m +viol ent +v y +potat oes +aw w +greet ings +t ts +ac on +sh ane +photograph ed +cra b +temper atures +cu ba +c fc +wel com +he l +in nings +m k +co de +kno ck +gra ss +swe dish +p ta +ick y +v at +lin ing +s q +sa p +ar c +announ cing +sk ins +cit yof +br ing +co x +gam er +it arian +i da +h d +ros se +sad ly +ge o +âļ ¡ï¸ı +tag s +fa ther +chan ge +l ance +whis key +adel aide +te c +stick ers +marke t +class y +bad ass +flo rence +lin er +fro st +k ate +ac on +scand al +es sex +ðŁĺ ı +vi vi +dr ill +blo ggers +recomm end +d ha +ac res +ro ma +bu y +gro cer +er ia +ma har +ff er +patter ns +ver i +com pu +st ev +ang a +ment or +do o +it ali +cdn poli +on ly +conduc t +elec tro +de f +wh ale +prepar ation +bicy cle +vi ral +turn out +bra ss +qu ad +hospit ality +pack aging +den cy +ceme tery +abo ard +dre aming +pic ture +t all +inv ent +ad mi +o e +tem ps +qu an +fun dam +pro mp +resi dence +mu d +sour i +âĦ ¢ +graff iti +gi f +d nd +com p +s war +pe eps +pale stine +devil s +san g +assi stance +bi ke +missi ssi +inter viewed +ne phew +dru ms +v and +gentle men +n sw +inst a +leban on +ee ee +oli via +ver y +rou gh +industri es +m ation +ðŁĺ Ĵ +bar rel +n ay +po ps +moder n +ill y +are st +on ents +protec ting +v ans +e o +vi kings +restaur ants +re ck +jac kie +andre w +w illing +he ath +citiz en +disc rimin +๠Ī +stu art +m ys +hi p +tran sp +" ? +te x +su shi +ke d +cro ssed +dist ur +pe dia +f ate +some how +mo th +proce ssing +is s +r in +u ts +yy c +ver t +lg bt +re id +on to +arab ia +habit at += = +stre ak +simp son +addic tion +wim ble +deli vers +challeng ing +ðŁİ ¶ +fran ch +e du +s me +ai ds +hur st +th am +tari an +remem bered +palestin ian +fe es +tru m +sket ch +ur u +fit ting +jes se +ðŁĶ¥ ðŁĶ¥ +---- ---- +ba ch +ici a +colo red +da h +associ ate +int el +s eller +p u +stu ffed +ac s +b s +sh in +cooper ation +certific ate +ab u +ingredi ents +re v +in ge +el der +christi an +bun dle +th ic +dir t +beij ing +comm it +ted dy +ed u +to day +s field +w yn +confir ms +lo o +j v +ene ss +al pha +vir us +ari um +gr ind +bri dges +introduc tion +pol ls +bac ter +z ach +termin al +ra iders +fla vor +zom bie +vo d +sp reading +gameof thrones +effici ency +lat ely +ale m +twee t +cri mes +cl er +de y +dg ed +hy un +pay ments +cir cus +ðŁĺŃ ðŁĺŃ +mis souri +lu b +episo des +c age +po s +mat ching +tumb lr +lin ed +ge st +am bi +nar r +ing ton +regu l +blo wn +is le +co co +on don +joshu a +tour ing +sm a +sau sage +best friend +bo eing +desi re +sav age +ra pper +de vo +te ar +take over +cow boys +po ker +par ag +pp e +h int +we ars +se th +ro les +l anc +man ga +form at +fl yer +c ay +mo or +ba ke +spla sh +v ad +ker ala +proce eds +sil ly +reflec tion +di str +wi d +su it +ci vic +yan kees +by n +migr ation +di stin +or ch +fe mini +quali fying +tu ri +o be +hun dred +cra p +wan g +mathe mat +bu re +expo sure +fergu son +seme ster +re serv +pl ym +a hu +fac ial +wa x +wor ried +ca b +vi o +as a +co d +to pics +p cs +hal o +rescu ed +horiz on +ar k +âļ ª +hol ly +el f +ul ti +pu p +quali fied +attend ance +ati vely +destro y +y c +for th +photoo ftheday +c ents +ic eland +meas ures +de sk +port folio +artic les +direc tors +dat ab +e w +creep y +oun ding +hon oured +mi st +j it +men tioned +port able +iti c +d ann +friday feeling +am id +ti ger +scri p +helicop ter +hard ware +expl or +work place +austri a +beat les +ber nar +spi der +disc o +cul t +lim its +shor tly +fin al +nin ja +lu ke +le bron +wal mart +o il +van illa +shi re +ye g +ak y +c s +bl er +collec ted +t g +rol led +speci als +b ff +pier re +sh im +vi er +flash back +restor ation +individu als +pro d +fre aking +tu rer +o a +re fre +mor oc +gre et +re yn +care ful +our ing +u sh +is d +g ill +vie w +thunder storm +b led +pic nic +guar di +pi g +ar k +syl vania +bann ed +u cl +vi jay +ori um +av engers +believ es +eu r +monu ment +concer ned +la bs +ber g +a ap +vi sh +sing les +can cel +z el +ar ab +ru th +too th +ar ta +sh af +chair s +r ack +dise ases +crow d +cl y +fle x +christ ma +artif icial +tom at +fin e +dra ws +advoc ate +fran ce +Ù Ĭ +ðŁĺ ³ +heav y +s our +compre hen +no ble +aa p +hin du +cor al +g ars +ow en +n l +st all +yel low +mar ina +in ver +suppor t +tou gh +promis es +pi e +master piece +sco re +for ce +mor tg +crypto currency +o x +r ors +rock in +pro vin +ho g +no stal +oak land +pat rick +inclu sion +tra ffic +ah med +a ha +lux ury +con secu +de mon +âĸ º +b lowing +st ag +: " +encoura ge +ben e +sku ll +do dge +bu ster +kin son +wit ne +er ror +lo west +fel low +à ° +sh re +bl ur +vir gin +compos er +sli p +mor nings +ga ins +tab le +gra in +ari st +braz ilian +w we +tu es +ribb on +an ag +di st +sac rif +em brace +entreprene ur +af fili +de o +t ali +touri st +fat al +ì Ĭ +autom atic +ðŁĩ µ +we ak +wel fare +confir m +benjam in +fi ghts +alleg ed +me ad +strugg ling +pro secu +che f +à ¨ +propos al +er n +ðŁĺ Ħ +dy k +on gs +hon g +m ack +mel on +on ent +ru sh +d ap +tol er +pro pag +c ze +trans lation +wal let +cott age +sa il +constitu tion +ðŁĴ Ģ +mun ici +fav or +storm hour +i h +ðŁĺ Į +approach ing +pin ned +j ed +niger ian +n ach +sh at +particul arly +mc don +camer as +anni e +admini str +he at +electr ical +char ming +gib son +bouti que +ex posed +ac tor +pil low +beach es +genu ine +margare t +ben nett +lou isi +pos itions +el y +shin y +ten tion +architec t +ren tal +ac qui +goo gle +sub way +mom ent +ðŁļ ¨ +ri m +metho ds +cy cli +nor folk +Ù Ī +over whel +ra pid +we ar +happy birthday +progre ssive +ðŁĴ ¥ +co gn +pap a +f ool +philosoph y +pol ar +jim my +wi g +ðŁĴ ĭ +oper ating +reduc tion +ph i +fla gs +to the +o di +a res +k oo +k ang +ar kansas +ash ton +wimble don +sci fi +attrac tive +mississi ppi +logi sts +ral ph +la bel +gradu ates +ma ha +home town +âľĮ ï¸ı +foun ded +on the +li z +trans l +mini mum +pre sti +ta m +gener ations +re bel +journ alists +par am +mc m +acry lic +death s +tes la +w t +bry ant +jer us +i stanbul +muham mad +ri ley +k ris +work shops +is o +coun ts +stre t +prote cted +trin ity +man ual +r hin +r il +pleas ant +le mon +ner d +har der +dar ren +bur y +ra h +bas is +mi gu +occa sion +li sts +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı +e b +de cre +hamp ton +ìĿ ´ +tra vis +trans form +puer to +nh l +av oc +tri ps +unexpe cted +ve t +di dyou +bar ber +st ages +m son +re presented +for t +l al +pp le +nic ely +ignor e +qu il +qu inn +h k +carri er +remin ded +am ong +pass enger +el len +gue z +sc ape +mu ral +youn gest +ma sh +d ill +rout ine +stain less +jack son +gand hi +th al +on ers +edit orial +convers ations +sd ale +autom ation +i ke +า ภ+ðŁĩ ª +hau l +la ying +men tions +am en +abor tion +i bi +coun ties +ca therine +man ds +jam e +roll er +au t +n am +o logical +cep tion +ran king +tox ic +sn acks +victor ian +bang kok +psycho logy +re g +ang ela +respon d +sty le +sophi e +dak ota +achiev ed +mar ked +imper ial +in as +glo ves +sli m +confi dent +att acked +gg er +lon ely +valentine sday +re b +craft beer +orig in +zim bab +ce iling +te ens +other wise +w b +f ers +day sof +advis or +y ah +âĻ ª +en der +republic ans +av a +skir t +pi pel +chi e +jan e +ja x +ðŁĺ ĭ +âľ Ĭ +j ays +bre tt +bal o +cru cial +d har +as is +de au +lloy d +chat ting +âĿĦ ï¸ı +rel ay +remark able +n s +we t +bris bane +ðŁĶ ´ +tion ally +f k +la yer +house hold +consecu tive +es is +pend ant +st ir +crit ic +su gar +photo shop +pa res +arti stic +do dgers +c un +cra fted +am end +bo at +âŃIJ ï¸ı +egyp tian +sa w +tra ge +small er +ox y +pa ired +nex t +i res +tac o +o y +u c +st i +a erial +: // +dr o +dot com +gg ins +r pg +ay e +le an +stri ker +lo bby +prote sts +pri ority +congre ss +am ate +inv it +r ington +mom my +th us +allow ing +pione er +enfor cement +g ori +tal k +dra g +du mb +bul let +san ge +er y +tar gets +ðŁĩ ¦ +he ather +consi der +seaf ood +ve st +ris ks +% . +p g +sac red +he ating +kick ed +tto t +. - +chan di +co ven +po ol +pul se +i a +ro ster +shakespe are +es a +car go +pean ut +tro op +ac tion +tab let +home work +cast le +stru ction +mus icians +free zing +bu tt +justin bieber +j j +bah rain +an them +au dit +didyou know +na vig +guid ance +âĸ ¶ +tur f +n un +fic ations +ye men +char ging +x c +bron cos +su bur +p ale +bor ing +among st +for the +em per +om fg +p j +expe cting +ðŁĴ « +st l +ad min +expect ations +sw an +shoo t +oooo o +min ent +ãĢ IJ +wall ace +stan g +satur day +adop ted +dou bles +hom ie +ome z +d han +vent ure +surroun ding +fi le +mob ility +de es +w ski +broo ke +emb ro +re members +kar a +test im +bo tan +m tv +sacrif ice +jerus alem +d l + ´ +proper ly +ili on +as i +leg it +co pe +m cla +recy cling +lar ger +ðŁĴ ĵ +pat ric +gener ous +ja red +p f +mol ly +thom as +ju dges +h b +sor ts +bl vd +o ven +enter ing +plan es +be et +integr ation +boo ked +fre ed +ver n +ash es +to pped +de pot +welcom ed +ren a +m ick +d and +see ks +gam er +ran kings +ren e +mu t +whis ky +fire fighters +gu es +ga ther +tour ney +de men +y ang +new ton +autom otive +back yard +deta iled +mi st +to bac +fi ber +un usual +grat itude +sp are +ne ys +: * +per i +flo ating +fin alist +don ating +dre ss +bro ad +be the +econom ics +tai wan +ed wards +plu g +pra iri +val en +bab a +f ad +an as +har per +dis order +app lied +p att +bi kin +li ver +cu ri +carol ine +ann er +juli an +wal king +mal col +screen shot +co ding +skin care +activi sts +myster ious +ex act +blo cking +mercur y +bat ter +du mp +âľ Į +en se +li sh +ridic ulous +prote sters +ðŁĻ Ī +lu st +swe at +as s +ali ke +co dy +re ments +win ds +as pir +vi enna +pra y +.. .@ +bo i +cand le +assi sts +te e +der son +p ony +f ence +con spir +âĺħ âĺħ +oo th +e pic +ba rely +a unt +b am +diamon ds +end less +scre ens +can cer +gr o +p st +pro spec +mo sque +help ful +ou ri +bro ther +gu jar +cri sti +ine z +to wers +ad dresses +gra y +bur ton +re tweeted +ðŁ¤ Ķ +n ity +du ck +super vis +jo an +kin der +sanc tu +pi ed +âı ° +ł ï¸ı +m ati +reven ge +ce ster +eli fe +desig ners +back ed +bo li +wei ght +cou ch +su res +s its +shri mp +la gos +auth orities +os ity +hol ly +compu ting +fac tors +ab e +pan els +ram ad +sent ence +missi on +hol m +r b +d ads +shang hai +mon ey +she ets +sk ate +thre w +cup cakes +infin ite +l is +practic ing +ess ay +ka i +as ci +mo b +u gh +hol mes +re gg +ik h +mo ck +collec tions +pe p +o va +sal t +nan dez +co y +thre ats +tex ts +cin nam +pregn ancy +pen ding +stam p +flow er +g is +agre ed +pay ne +ro ver +ph ra +sof t +f fin +fa thers +pass engers +aw ays +al a +h es +li van +in s +samu el +ingu i +h of +j j +chen nai +cat al +om ic +he ath +ni ece +pump ed +integr ated +are l +no m +produc tivity +wan ting +vis a +di ana +tw il +it v +cam ps +ro wing +d ley +black and +gu ards +b ells +re verse +vi be +ric ky +mo ss +ny t +âĺ Ģï¸ı +el le +tro y +cu dd +ev an +women s +fo to +mi stakes +wick ed +mi l +c led +me mes +co smo +schol ar +ren o +ðŁĺ Ģ +v ents +# âĢ¦ +terrori sts +ca sey +cardin als +ðŁĺĬ ðŁĺĬ +venezu ela +bol a +liter acy +t w +en o +con tains +au stin +fin anci +ev an +har vard +origin ally +chev ro +her ald +nott ingham +manag ers +âŀ ¡ +accep ting +wal sh +tutor ial +entrepreneur ship +yach t +requi rements +glen n +pe de +unfortun ately +ach ing +dais y +gi an +night mare +âĿ Ĺ +r ina +b art +ema ils +oppo site +who m +sa ke +pu zzle +da shi +par ty +blan ket +bus es +lo re +beau ty +reas on +pun jab +winds or +func tional +exi sting +hel lo +gli mp +con vin +la k +scre aming +rebec ca +bli ss +north west +infin ity +cosme tics +pul ling +coffe e +pl ing +op ho +colom bia +interior design +( + +emo tions +sa c +sun glasses +sav es +d f +six th +al y +ðŁĺ » +de en +dev ast +polit icians +lac rosse +g u +pe i +jav a +comb ine +coal ition +er ts +survi v +ch ad +stri an +n n +de vi +coun c +concer n +contro ller +bre ast +j ury +tu m +introduc es +la di +mobi le +al z +ste ady +nur ses +h acking +on line +oce an +ðŁİ Ħ +a am +ju ven +ic c +louisi ana +ar te +street art +is on +wn s +fr m +p anda +no ir +main tain +del ay +symp toms +thor n +ge ome +ter n +carri ed +p ru +pan or +as sy +per u +clou d +sp ra +pe di +e ste +tag ged +ðŁĺ Ŀ +shado ws +naz i +ا٠Ħ +cor ri +âĻ¥ âĻ¥ +j ad +ðŁĩ « +form al +spo ken +ðŁĮ ŀ +enjo y +lo pez +out look +in ho +w ander +Ù ħ +ma ya +pe e +d ine +ãĢ ij +brief ing +suppor ter +ar ily +ght ers +natur ally +doctor who +j en +v ar +new year +re se +si mm +re x +con sequ +tomat oes +bur st +bra vo +bur gers +cr acking +nor theast +bi om +mush room +mar que +dou ble +ni er +v ag +tw enty +key board +win ni +jama ica +par ish +: - +mental health +ali zing +ren der +wa king +ðŁİ Ĥ +g ly +na than +wa shing +mel issa +jun g +loy al +chil i +song writer +guit arist +bo wie +neighb ors +onym ous +as set +ta i +head quarters +ðŁĮ Ī +i hear +ci gare +sur g +) " +re pl +dar ling +ðŁĻ Ħ +z ak +sa re +ãħ ĭ +mic key +ware house +mass age +ine es +did nt +i w +hur ts +eng aging +mag ic +women in +k itten +mor s +c art +tit ans +colle ague +compe ting +er an +k hal +mar ble +dem and +del ight +et ary +bli zz +lou ise +m ls +fini shes +experim ent +conduc ted +electr onics +itt ers +car ing +wh ats +sym bol +jun g +e cu +pi x +con text +char ger +ðŁĺ ĩ +re ig +fra g +ë ĭ +ch ad +tru e +ker ry +def ending +a int +au ton +check out +bar nes +less ly +d t +m me +clou dy +second ary +are z +_ : +app a +const ant +" ) +ve ts +jo b +i ent +ðŁĺŃðŁĺŃ ðŁĺŃ +m j +fren ch +di ver +davi es +hh hh +e book +๠ī +mar iti +bree ze +susp ended +mat o +vi et +ra hu +se i +bol t +en ary +le is +kar l +fr amed +expla ining +ab c +de aling +nat o +ja ke +exp and +leon ard +establi shed +du b +ar men +el led +voc al +nichol as +ori ent +k yo +illustr ated +ah h +danc ers +milli on +ge ta +po pp +as u +mur dered +gi ble +sto ked +gri ffin +maxi mum +adri an +en counter +ther o +david son +ðŁį » +holi day +ev o +asse ts +car son +memor able +âļ ½ +ob am +represent ative +cb d +tr icks +vo gue +vo ice +mm mm +sebasti an +cli f +ath y +par alle +ðŁ¤ · +pa k +ev acu +e ats +ا Ø +tou ched +organ ised +spir its +can ad +gui ded +frame work +ðŁĮ Ł +pe d +natur al +ag ar +replac ed +anch or +ti t +sha h +organ is +super ior +r n +ch ro +eric a +st ill +cor on +chu ck +loc ks +or gan +ro sen +sc am +ben ed +/ # +ke en +tre vor +vamp ire +sor ted +! ' +af ford +in tro +gr ace +ðŁĺ ľ +sau r +kick starter +influ en +v u +y up +po c +ðŁİ ¥ +a ar +s ang +tre k +et sy +tb h +scre am +chevro let +pix el +shepher d +an or +gabri el +tw ood +sd cc +me ters +develop ers +clo sure +v w +twit ch +ì Ĺ +se oul +pr ice +ho g +n ish +hill ary +scrat ch +in cen +wag on +dis ability +pan ther +ch ats +g d +wit z +sus sex +l ate +den mark +ger ald +cancel led +net te +i x +nav al +bap tist +te t +y ad +ma th +ho y +r andy +po int +intel lec +fru its +w ool +gu in +pr on +the ft +con dem +mar ry +n ola +architec ts +cin cin +roc kets +gentle man +ex plan +t ate +do e +ra ises +wild life +w l +insi der +blan c +w p +for sale +ny c +po well +unbeliev able +pen s +goo dies +mu stang +p ens +st ays +squ ash +xox o +near by +ever ton +co co +le agu +k han +stu d +south west +con struc +s worth +cro atia +le a +su ms +aim s +e an +van ess +iti ous +pa thy +arc ade +b end +sugge sts +sac ram +roy als +ri er +em ir +in cl +an k +clar k +ri ght +vac c +ठ¾ +tan e +li b +u sc +sal es +hu h +s ally +ver a +p ga +gro ws +dru m +tre e +eth ics +sug gest +is ab +se aled +pre viously +anim ated +ab du +ri ses +glo b +pre dat +scar f +del ic +om ar +ll i +sx sw +py thon +ne bra +fun k +reflec t +pav ilion +tic ally +ch asing +bak ery +inva sion +ko h +believ ed +co hen +con qu +cra fts +nat i +cle ver +govern ance +sam ples +fa ils +â Ķ +ti mo +r itu +stri king +inclu sive +sho cking +can t +requi res +dra wings +à¸ Ń +purch ased +du m +z ach +war ner +con sole +man sion +foun tain +circu m +e sh +is land +mil k +pro fits +hali fax +ri val +âľĪ ï¸ı +jen ny +sand ra +ny e +k elly +y al +qu ad +no s +inste in +fin alists +mid fielder +cu e +excep tional +a an +sa pp +gett in +sa a +f ati +sl ice +vol k +s wal +la sting +sum mary +it as +sm o +s z +âĺ Ĩ +ip l +fl ames +ene ws +ha v +hoo die +pitch er +win dy +re vol +centr al +ton ite +ðŁİī ðŁİī +sol ved +mil wau +organiz ations +wee ts +re fin +s th +ãĥ ¼ +el in +ton a +cinnam on +ðŁİ ¨ +ðŁİ ģ +ron aldo +pen insu +ome ga +el ds +desig ning +e igh +blu et +ben z +nu g +ash a +robo ts +su dan +choo sing +en do +ser ge +clo sely +hand y +fing er +be ing +ar te +survi ved +fl ame +mile stone +gu t +d war +fu tures +é e +el o +fri dge +eli c +ou ch +u b +p v +tit an +col lar +st ation +nev ada +aur ora +r d +dun can +âģ ł +bri en +mar sh +Ð ¾ +to tal +ch ry +s ers +su ffe +ra chel +colle ge +to days +cour ts +ch it +re united +gym na +gen esis +be side +re presentation +ch ant +collec tor +ra k +ath ens +ni gh +mun ich +langu ages +fl u +particip ation +__ _ +c v +spec trum +so da +co ver +refe ren +ab bo +ap a +public ation +ed m +mon ica +ar my +ðŁļ Ģ +div or +dr y +stre ams +robo tics +ci der +bull ying +appro val +sto ke +plat forms +sier ra +ex tin +i b +ha yes +succe ed +suff er +at ically +da i +lyn ch +h ound +del ines +ack now +d ated +exclu sively +he res +fac ilit +dam aged +char ter +la kers +fal con +unve iled +wel ove +e ase +pati ence +l one +gent le +gene tic +produc ing +g our +shann on +bil ities +zimbab we +p int +dau ghters +liter ary +bel le +cl am +surroun ded +k any +ne il +pir ate +rang er +hb d +nat alie +bel ong +olym pi +emb assy +sc ol +en er +ak in +lo ren +b h +: / +di va +den im +hi pp +ðŁĩµ ðŁĩ +arn old +? ' +we ren +em power +dis abled +man or +rasp berry +b af +aw ful +dru mmer +kar dashi +n ash +machine learning +ch u +rebel s +tim ing +mon roe +ton gue +ran ge +pup ils +re ss +amaz on +b z +har ley +pal mer +ballo on +s ings +ic ec +j b +c ers +g ps +whi st +ri se +l t +oo oo +c attle +shoo ter +vod ka +uc l +mt g +le sli +jon as +di spo +at ric +ste in +vintag e +fir ms +flo yd +cow boy +soo oo +is aac +war craft +disney land +beauti ful +be am +franch ise +bu n +k ag +an on +tur bo +swee p +made in +kar achi +dete ctive +penn sylvania +contro versi +vitam in +a side +chron ic +descri bes +remo val +ha h +ap er +ten ed +u to +bad ly +mir ac +f ry +ye a +in jec +ther mal +comp act +th or +te ed +ur gent +l ite +g illi +sop hom +ic o +che m +p m +for k +fre ak +ch ak +recipi ent +i y +ni k +model ing +c ans +ðŁı Ģ +del ux +se am +surviv ors +rad ical +investig ating +reli able +f m +tur t +ligh thouse +to ol +go wn +) ) +bo ts +auto graph +a id +bu ffe +h mm +horri ble +ssi onal +ann i +๠Ģ +k its +sch i +eter nal +hu ss +sens itive +r u +tast es +chec ks +im o +por tion +sk ate +e den +half time +fri ed +ri hanna +ti se +fl ick +ca in +s gt +âľ Ķ +sh au +sta ined +ra ffle +dro ve +sal man +princi ples +sh o +ar u +je ss +gu ine +gar bage +my an +jel ly +dis ru +z ia +q ld +ent ries +la v +fle w +ad mit +objec ts +comp are +ny times +cann es +p n +suff ol +ro c +d ana +e gg +hi st +coun sel +' ! +phy si +imag ination +ad just +explo sion +plym outh +hor ror +elli ott +bour ne +de x +bre ed +au dio +lob ster +disappo inted +nation wide +( ( +incre ases +austr ali +ce dar +star ing +rac ial +e is +g mt +visi ons +stay ed +discu ssions +de an +cur tis +mai den +stel lar +happ iest +h wy +pre season +car av +mon days +hospit als +glimp se +schol ars +ja i +ter race +ann a +goo se +gra ded +lot us +hun g +grocer y +stam ps +emper or +sc oop +in ser +c as +exist ence +he al +fal cons +mar vel +reduc ing +terri fic +magne tic +perfor ms +bar re +p us +tre ating +ic on +w h +decla red +tra uma +do d +come dian +nik on +bu gs +as m +mont gom +ibi za +comprehen sive +ha s +san ti +fellow ship +da sh +p sal +louis ville +sp y +fau lt +d the +fi led +vi sta +de sc +fe ars +you tu +sp s +es p +ri g +cri me +ber ger +wonder land +k ent +in formed +stev ens +my th +ast on +ir i +visit or +at ri +produc ers +al la +person ally +separ ate +agen cies +af ri +il an +spo ke +n ina +squ ad +di ves +de pend +li v +fier ce +enter taining +cha in +sc at +bor ders +pal ette +sp ro +os is +der by +tobac co +zi o +willi e +ju vent +zoo m +hol y +enti rely +af e +mart inez +be ds +pe a +bull dogs +ðŁĩª ðŁĩ +ib m +ne on +ethiop ia +team mates +plan ting +tw er +any time +for bes +ó n +run way +ner vous +ro ger +p ile +ch anc +apo caly +u w +o i +dr ought +territ ory +br ick +cre atures +go in +w aff +gre n +sou theast +je an +am bul +ed ited +stra p +c v +aar on +ãĥ» ãĥ» +t su +descri ption +kin dly +clu tch +im mer +en or +women sday +or ange +ra g +ob vious +hy der +chann els +man go +me yer +ra ining +ge tty +pil gri +coordin ator +up load +ninten do +don uts +san chez +app arel +j r +zz i +, @ +jeff erson +accessi ble +great ly +e id +initi al +budd ha +par is +ma scot +â¬ĩ ï¸ı +sch war +si ri +sp inning +mortg age +e cho +end ange +ge dly +chlo e +enh ance +kar nat +k ry +explo res +ðŁĴ ģ +af fair +ic als +all a +dar t +dolph ins +diffe rences +squir rel +au gh +dr ones +ell en +re store +pa w +un for +pi ke +hil ton +colla b +consu mers +co inci +out comes +pp p +a q +coup on +li est +si ms +k ho +av es +spo on +pu dding +cor byn +hat ers +ex ams +sla ve +. ! +p sa +app les +tam il +se d +co ke +zz o +lo sange +car bon +cla ir +... ) +k hu +cra ig +explor ation +sanctu ary +su e +al way +demen tia +won ders +super hero +pakistan i +brown s +bluet ooth +lo cker +mar c +ev entu +delux e +rodri guez +âĿ¤ âĿ¤ +ro bb +ðŁĴ ¦ +lin ux +ten s +intellig ent +se ed +vo ter +s ler +pe aks +inter n +teen age +peninsu la +hand ling +ti e +cou sins +wen dy +me e +à¹Ģ ภ+din o +ðŁĴ ° +ðŁĺ ĥ +ze e +s bury +trage dy +b k +bo re +z in +war ns +idi ot +tou ching +contin ental +tac os +saf ari +wa shed +po dium +morri son +fore sts +c bc +al on +partic ular +be ads +inv ented +lo ch +li ghter +where ver +i de +docu ments +a we +k r +no where +min er +st it +ro x +contribu te +har dy +cl an +ob ject +ca it +ðŁĴķ ðŁĴķ +happ ier +vege tables +t art +g ag +nom inee +heav ily +pan ic +j d +there sa +at m +u ph +s fc +su ri +drin k +n al +re vel +k l +avoc ado +nom ination +ma donna +shar on +malcol m +control led +sh ers +revi val +legis lation +shoo ts +n in +comm entary +pro s +human rights +str anger +mit ch +pipel ine +leg ally +th u +gil bert +tol l +gran ted +gh s +ir anian +refre shing +du k +ab i +pri me +jose ph +mo sa +stati stics +produc tions +mer ry +pat el +sa x +human itarian +struc tures +e missions +town s +fre el +ster ing +rat ings +alle gedly +cab in +st l +w ade +fl yers +tri m +promis ing +z u +bal lot +compar ison +free ze +ou ter +great ness +as sign +snow y +r ale +tor ies +med iter +kno ck +consult ant +cincin nati +analy st +sc oo +je ws +appro xim +pu re +portra its +cy rus +ation al +lo ans +acqu is +el u +accep table +uni on +water color +ru st +batt les +per fu +seas onal +ser ial +mind set +ri ot +fel d +enni al +clo set +pri est +tan ks +int l +scre w +bu m +ab dul +ou x +expla ined +ric a +imag ing +law yers +bu ried +ãĥ»ãĥ» ãĥ» +ear l +âĢ ķ +l ton +resto red +stri pes +fo ss +de mands +ste aling +alex is +mun d +ak er +ur us +war dro +hu gs +gen re +e go +Ù Ħ +particip ated +bab es +ban quet +ti ous +he mi +ds b +lo st +milwau kee +jen ner +ge m +ou tra +lo ses +id i +re ps +ðŁİ § +regu lation +fla w +f ang +vibr ant +ram p +ra ins +well being +so viet +vie wers +de po +libr aries +bi go +ser y +g ill +de struction +co z +c x +bri dal +al ds +plan ted +amate ur +lu d +che ering +show cas +pro file +i u +ver tical +pack ers +wiz ard +ski p +s light +be au +air ways +mu ch +re ra +ðŁĮ Ĭ +ab sor +pati o +pack ages +s ells +ment ally +ðŁĺ ¢ +reyn olds +k are +tri bun +wal t +kn it +ta ste +sur rey +boun ce +cre ature +b are +bet ting +su re +mi ley +laugh s +al ore +cy n +t l +arti st +ann ah +war mer +dynam ics +lunch time +mariti me +vulner able +ðŁĴ ĥ +wol ver +dur ham +const antly +am in +si bl +: @ +bul let +k ach +angel o +wil der +doo m +desk top +law suit +k ca +hen derson +inv iting +bet ty +ta wards +ra fa +le aked +and i +ge ms +af l +vel o +mediter ran +pro be +to tten +steph anie +sn ation +com be +q s +over come +assas sin +ra v +fil ip +winni peg +sh il +determin ed +k as +ou tre +regre t +gui des +aa a +ðŁĺ Ī +wi ves +mani fe +er ly +sm y +sh ima +x ing +pix el +jac ob +ac commod +to y +on o +po o +ti er +an swe +ðŁĴ ģ +ro sa +le ase +bel ongs +th ar +eventu ally +nei ther +go a +ski ing +at ra +ag h +broad casting +f ury +py ram +d ice +volk swag +wom ens +provi der +bom bs +miss ile +whi p +d ick +nor we +back up +el der +mat ure +concer ts +gi ous +sque e +good morning +bra ves +^ _ +au ssie +lun a +mal es +he ck +for tn +rome o +steel ers +p n +pe er +re presents + « +kat y +migu el +requ ire +cha ins +l ur +immedi ate +ti mber +âĸ¶ ï¸ı +advoc acy +ex port +an z +tiff any +auth or +ðŁİ Ī +du des +chil ly +hi d +har m +bu g +mon ster +terri er +tu c +story telling +ta k +in ti +immigr ants +b is +reach es +com passion +john ny +contribu tions +ðŁIJ ¶ +mechan ical +impre ssion +ran ks +ko be +men ting +bloss om +pab lo +buil der +bom bing +tw el +sul livan +om o +pe te +de mi +ku dos +w bb +t gif +mass ach +neighb or +che fs +eng ines +pun e +ga ined +phan tom +s days +ext end +gr an +cent ers +jac qu +dat asci +sleep y +el vis +answe red +s lot +con y +flexi ble +ti ally +le tics +% , +andre ws +si ble +mom ma +vin o +do x +invit ational +twil ight +j ade +ill ery +joh ns +f ou +p v +-- -> +break down +billi on +prin ter +mon d +c bc +mag gie +legi on +du b +kur t +po or +paren ting +regi ons +bikin i +be ware +si onal +au burn +kid ding +amp les +sp an +con tempor +c ic +ha bits +ak o +pre fe +bud dies +it z +em ily +person nel +moun tain +ver sus +ðŁĺ ¬ +ear ning +s ink +dar i +u u +s win +i ster +bru tal +n ac +kat a +clo th +am and +ðŁĶ Ĺ +ne o +alu min +week ends +nebra ska +co des +delay ed +brun o +pro ven +in c +i ght +fl an +or o +lam bert +regu lat +w f +massach use +kardashi an +bern ard +fi esta +volcan o +grand pa +anc a +d re +st itu +mean ing +fo am +au ck +at ed +r l +hot el +pers ons +dy nasty +ell or +ma i +am ne +sty ling +avi er +e g +vege tarian +, âĢ¦ +foun ders +sta in +g d +cy cles +sky line +trac tor +exi sts +tra l +kid ney +mar il +inst ag +se tte +addic t +tri angle +flash back +controversi al +z on +p ins +i as +tr ay +town ship +deleg ates +sp am +h ms +cr ane +peop les +o lo +fac tion +but es +on ica +deleg ation +new profile +eli er +mc a +w and +g ely +losange les +ber ke +ti ve +dis rup +zz a +cas a +jor dan +ford shire +ga thered +ic hi +atten dees +à¸Ń ภ+pe ppers +co in +bour bon +ern ity +ro tary +behavi our +jere my +team work +compli ance +tre mend +ðŁĩ § +bu hari +cam bo +bu yers +ha gen +bu ds +bay ern +mon te +sm ells +an za +ath lon +descri bed +work force +gi ving +ap i +invest ments +da il +sel ena +datab ase +th um +mor tal +stu dent +bu yer +do ver +gar ten +att le +loy alty +gen oci +holo cau +theat ers +ru ling +ven us +pat ent +ch un +ab by +awa ke +mass acre +bang alore +break ing +simm ons +ju sti +hal e +ed chat +gg les +haw k +mar king +head lines +stro m +co ve +breath taking +med als +hair cut +christ ine +tele graph +gujar at +ju ra +can e +sho re +propag anda +mu eller +.... .... +sa vi +stom ach +thro ws +ta b +war m +j ong +reno wned +hi r +ra is +mush rooms +guaran teed +bo a +m j +revolu tionary +certi fication +bru ins +jo in +w es +pas sport +c g +sex u +cap able +w v +ton es +jac kets +ac compan +spin ach +fore ver +bla ir +wat ts +g l +cou ples +prairi e +newprofile pic +logi stics +massachuse tts +jagu ar +o id +we al +under water +mo z +y i +ma ths +myan mar +pre ps +suffe red +tr ace +wal i +ah hh +bor g +st itch +cu lin +real ise +infe ction +discrimin ation +sh ame +an kle +hu mid +y t +brac ket +tru ck +tri u +ea ster +commun ity +post card +invol ving +ty ler +car amel +over view +ex amples +integr ity +base ment +instru ments +ani um +at us +gh er +laun dry +achi eve +gen eva +pr icing +hyder abad +beli ef +me ta +j aw +accoun ting +lead er +cristi ano +cou ture +cy p +vis ed +, ,, +k nu +h ick +break er +br am +ra b +mo or +ham as +gradu ating +pupp ies +ak h +ta h +ach es +ri e +op ini +g ta +re ign +tra gic +re ver +p ill +pine apple +tou ches +da re +le ys +il o +inter iors +sc outs +bar t +en zie +don o +bro ck +christi ans +ense mble + · +cine mas +new port +air line +win ston +le igh +cont ents +pre scri +ur ge +tr out +fic ally +il ia +sub si +are r +âļ¾ ï¸ı +w ounded +ðŁĻ Ĥ +pe pper +ðŁĴ ŀ +fit ted +af f +re sur +thursday thoughts +z ero +archae ology +di v +je e +i on +awa iting +co zy +beauti es +bal d +dat a +gri zz +stal k +kin ds +cle ared +jess ic +regu lar +ali ens +plac e +bo s +bi zar +thisi s +ðŁĴ Ģ +totten ham +ma fia +s lam +ari ana +car roll +back pack +care y +uni v +r g +pe p +dig it +tatt oos +ag on +volunte ering +diffe ren +consu mption +ka thr +head phones +t shirt +o b +ele ment +re tail +sh ru +al gori +contain er +consci ous +fi l +com ing +ra sh +u rope +def ine +gi or +femini st +flow ing +rout es +gl aci +fer t +somer set +ant es +twee ps +$ $ +h our +endange red +year sof +ro h +po pped +bac king +ba sil +bra ke +mon aco +lgbt q +pra gue +ut ility +cas si +gate way +haun ted +sch ul +ðŁİ µ +shou ld +walking dead +comple ting +dann y +montgom ery +pengu in +ss i +mer chandi +ðŁij ij +chur ch +h ates +cap tain +brea thing +ce t +fair ly +approach es +compan ion +surpri sing +kany e +pe y +hin di +targe ted +lor ds +de ut +di gging +ger man +ru t +ener gy +close st +y un +apo logi +ภ± +s ack +ru p +dd y +port al +d ough +b ats +ðŁĵ ° +at ur +graph er +pi res +mo tors +ðŁĮ ¹ +j c +dan g +tu k +clu e +us c +pag e +d less +bro ws +ju s +ad ing +re marks +oo m +car dio +ste fan +arm strong +âĢ¢ âĢ¢ +ni est +belgi an +bi op +so y +lo f +í ĥ +q t +flashback friday +ce e +ģ ภ+wre ck +mar ines +amend ment +wardro be +vo y +bur ned +guit ars +ra inf +li fel +ssi l +oun ce +exter nal +c key +me sh +she ikh +inv itation +sugge sti +pop corn +phenomen al +an onymous +tun a +chic ago +o val +del y +loc als +( & +pro f +no vel +fin der +spar ks +la ven +in fu +nic ks +qu ant +ra e +exe c +dist ingui +st ances +mu tual +sh al +unve ils +edmon ton +zan ia +a dio +vie wer +brad ford +audit orium +qu is +re act +htt p +l ero +chee ky +impac ts +ta k +ed t +desper ate +t ay +ì Ħ +sett le +bar gain +resu me +un ite +thro wn +ke st +se ys +mar ching +am it +decl ine +sch ar +me tr +stan ford +lin ke +ber ra +dol ls +rug by +jam i +b or +road trip +dino saur +mi k +sun der +re m +b k +over seas +nau ghty +imple mentation +iam srk +lun cheon +fir ing +mi ami +pere z +the e +z on +gi fted +con version +ceram ic +¡ ï¸ı +pe dro +ì Ĩ +v ick +! @ +he ed +si d +b w +docu ment +pl un +gr ants +fant asy +predic tions +vali d +car ved +gradu ated +ðŁijį ðŁı» +nation ally +ch y +af l +re sso +blan k +ri vals +j ig +e ties +om ics +une mp +b ound +sk o +inspec tion +par al +high s +cri sp +b ans +ob a +[ @ +co spla +costu mes +rec all +mou th +ni gel +b ts +ter a +ko v +do cs +west minster +dic t +gra vity +kar i +ro gue +t ted +war k +ida ho +w end +aw i +queen sland +proce sses +cli ffe +m ick +com pens +op ol +the y +cl ari +wiki pedia +salman khan +haz ard +pre ston +swee test +pd f +che es +tr ilo +south africa +bur nt +( $ +con tain +t p +sub mitted +sound cloud +at u +re z +word press +corru pt +n f +ma ker +í ķ +par as +adv ent +ri al +ca fe +fo ssil +!!!! !!! +co ws +c j +sp ur +institu tions +land mark +ent it +re ut +h is +alz heim +we mb +regg ae +mo squ +st at +identi fied +deal er +re am +re land +ten sion +ðŁĩ © +wra pping +deep er +fr at +red dit +ar is +moroc co +.. " +b low +ma pping +pri orities +ing a +swa p +re wards +conspir acy +creati ve +c j +congre ssional +vau lt +ple x +sophom ore +shad ow +ele ss +ðŁĺ ħ +dar ts +aldu b +anno ying +pro ps +n as +alumin um +h bo +offen se +j ill +oni ons +la ur +ta e +har dest +sh ro +ga ining +meas ure +ed tech +cyp rus +tar a +ang eli +car lo +go on +all i +im plic +ju pit +resil ience +ha il +bal anced +) ... +joy ce +gr a +th eli +defin ed +shi pped +main ly +min a +l m +sac ri +o ber +p im +claim ing +ent ers +co rey +bo k +cri ed +cool ing +dani elle +pharmac y +thor ough +ca ke +k lo +outre ach +z ens +digital marketing +val ent +sn p +her b +mr w +caf é +cap tures +no tre +triu mph +pan cakes +cu mber +spi ke +d ation +bi gg +sp er +crit ical +am al +too th +foun ding +a stro +' # +quan tum +th ames +un c +pri de +air bus +kno cked +un defeated +mediterran ean +cal cu +clo wn +sens or +ham mer +for give +cu shi +ber ry +maje stic +elec t +polit an +g ta +k ari +bur ke +sea hawks +volkswag en +re i +landsc apes +cas u +grand father +list ened +/ / +star trek +rainf all +fur ry +vi er +star k +rif le +ff a +leg es +hillary clinton +min us +correc tly +architec tural +pre ce +up side +box er +ðŁĻĮ ðŁı¼ +is ai +de t +pro vo +tis sue +spoo ky +ve led +re con +prospec ts +que bec +âļ « +ig no +anat omy +shap es +w p +p interest +hor e +an es +pick up +ti p +pra desh +hu gh +co e +po k +gram my +well ington +sti gate +ri gh +lea p +king ston +scen ic +go sh +v ani +au g +s ary +zi er +bure au +lin son +con te +fra gr +all an +g aw +lan a +colli sion +surve ill +ren ais +ar range +s ali +do in +br ance +bren dan +our se +in coming +suspen sion +à ´ +l la +educ ators +in tri +da e +bio graphy +bul gar +villa in +go thic +rw anda +e w +may or +meet up +democr at +mor gan +su dden +te sco +car rot +bom ber +mck in +re ne +fun day +agricul tural +haha h +show time +form ing +col a +scor pi +quo te +po ppy +s life +d az +tu b +ne n +mo t +ðŁĺ » +s ore +elder ly +o ve +skin ny +um i +anc o +man ship +we re +g v +k ah +fol ding +ne at +samanth a +dan ish +uk rain +humid ity +nu tri +jak arta +cand les +oooo oooo +at ile +streng th +i bra +bap ti +charle ston +fr ames +girl s +clear ing +glu ten +# # +super natural +ju bi +ph one +he in +dr un +le ak +invest or +y er +dom ain +ball room +mi sh +app li +off shore +bla ze +dor o +âĺķ ï¸ı +win ery +shar if +ad ore +n ir +saf er +si gh +as cri +strong ly +trac y +ck er +ol l +faith ful +ey ed +deli ghtful +vis m +karnat aka +tit an +wh ar +jer seys +re fur +heav en +gri p +pan ama +pre li +glu ten +o dd +cont ent +pon ti +tion ing +e commerce +feder ation +flaw less +ge ar +ti res +by r +pol ice +cu ban +tri butes +tic ul +chur ches +nur sery +di aries +muse ums +snapp ed +i van +wi ght +touri sts +ramad an +t rent +prophe t +won dered +focu sing +hi d +ic ons +i q +ambul ance +pi st +fun niest +time less +sr ilan +bu ys +ki ds +colour ful +a shi +ch ir +mu m +ðŁĵ ļ +let ter +x en +reut ers +pre serve +in ting +ste p +fu ji +uni ver +i u +show down +po ems +surveill ance +suspec ted +ta e +sol ving +tom b +mother sday +car pen +recru it +pil ots +bro c +mix ing +fri days +ty r +represent atives +tra pped +abdu l +free style +clu ster +âļ łï¸ı +k d +sk ill +pit t +ex o +commer ci +muse um +loc ally +g ina +no bel +immun e +fr ac +cap su +main ed +attemp ts +bull dog +be spoke +sing ers +sp elling +seg ment +nat ures +tic k +lip stick +clean er +gett able +preci sion +âĢ¼ ï¸ı +th ood +re ef +no pe +bill y +di gi +mu si +ri val +figu red +tal ity +sun ny +ber k +aw ww +awa its +un real +co pen +asy lum +ex otic +bu en +mo ck +en able +arch y +fr a +pla stic +al mond +amp li +displa ys +abbo tt +s me +x p +ðŁĻ ĥ +graph ic +i ved +mar a +cau tion +lea ks +en berg +ul u +unic orn +cann on +appren tic +ðŁĺĺ ðŁĺĺ +b ball +wil low +at ics +am as +manufac turer +campaig ns +port ers +flo ors +l su +ty pe +ke j +honor ary +it im +to le +min ecraft +d x +ma sh +ri o +consequ ences +ron ald +go ssi +suffol k +mu se +r bi +live music +i van +ðŁİ ¤ +le u +patri ot +man it +lan ca +home decor +de ar +sig ma +ti de +str ings +v ita +sequ el +try na +inve stigate +bor is +ve gan +barri er +mind fulness +web b +hu stle +in da +tan zania +str ay +tex as +c ag +diagno sis +wom an +g w +ob session +l ative +nu fc +fl ynn +moment um +sof a +wal d +vege table +tu cker +supp er +se ab +ar ro +se ag +ven ting +counc ill +sp lat +cal cul +.. # +com fy +odi sha +sto pp +war fare +ca es +à ¨ +co y +price less +in sec +ðŁĺ Ľ +contro ls +empower ment +datasci ence +per pe +gen ic +e res +tru deau +man o +sla very +expand ing +ma he +fa iling +s aga +photograph s +cre st +re on +surf ing +hi e +ðŁį Ģ +ja e +fel lows +south ampton +sol om +ce ster +tab ility +hor n +se ct +he e +cole man +at las +explo rer +consul tation +copy right +organi zing +den ied +mon keys +noo dles +br is +fl or +dou gh +bon ds +sho cked +eco system +care fully +w m +apart ments +cur ve +san diego +must ard +comm en +cere mon +e ch +ru th +ðŁĻĮ ðŁı» +hawa i +fil med +te ar +as ingly +ca ir +wat t +instru ment +ou tta +ye ol +river side +ë ° +. : +nor wich +alo g +migr ants +new man +ri de +spr ink +targe ting +beli eve +tor ch +reflec ts +per mission +ff man +ene mies +bas ics +se ized +sun days +le i +hass an +en do +h c +st ad +le ments +kk kk +nan o +shar k +man a +on ic +treat ments +ear ly +collabor ative +shu ttle +bran ches +mis ses +mained cm +ap ers +ky le +carri e +leis ure +sh et +bir ding +adv ances +ðŁĵ Ŀ +popu lar +di ane +a be +re war +neigh bour +k pop +remem brance +play ground +ru b +krish na +e bola +inqu iry +ep a +lu min +organ isation +abra ham +norm ally +pre ten +jan et +w t +ðŁĴ İ +encoura ging +a stic +bu mp +syd ney +s z +ss ss +gar rett +ðŁĵ » +consul ting +roman ia +spo tting +chanc ellor +ar ma +presti gious +ðĿ IJ +t ad +cry st +compe tit +rati o +cat aly +bro w +j ur +vi king +commu te +y day +la yers +du mb +esc al +genoci de +f ill +gu pta +ste pping +se i +fo to +wild cats +col i +projec t +ear nings +st r +ge ons +comple tion +b m +decor ated +craw ford +af ghan +sc are +visi bility +hi b +direc tion +stro ll +christ ina +alter nate +cl are +sty list +be hold +s ance +leop ard +acqui red +narr ative +ash i +the a +?? ?? +pe as +at ch +sli des +le en +renew able +eng lish +qu ir +co aster +r x +fo ols +match day +mis m +amaz ing +z ig +ke ting +won t +to wel +di ab +sta ke +n m +mel t +e than +gra pe +polit ician +sm en +í ĺ +re o +wedd ings +cat cher +or acle +me mo +ðŁĮ ´ +ec k +rob bie +norwe gian +oper ator +am or +se wing +ju l +x ie +u v +fif ty +me ga +tatt oo +liber als +u pri +traffic king +richard son +su v +ki p +mess y +tremend ous +gl ou +cour tney +la d +stere o +my ers +i dio +^_ ^ +man ning +dy e +w d +thr one +jun k +as u +provin cial +k ook +wr c +fine art +hamp shire +renais sance +b red +fall out +s j +sn l +al am +tor ture +fy i +sh ines +pa w +ch ar +hen ry +c row +aci ous +di an +pa ige +ba re +stock holm +scen ery +ðŁĩ · +jef frey +pu sh +decor ation +ne d +cu te +brig ade +laven der +inv ites +e sports +vo ir +dri ed +tran spl +sur geon +no vels +pul ls +son y +lun ar +man e +i vy +fru str +dor set +sa i +tor res +ssi on +shut down +suggesti ons +writ ing +e o +battle field +u ga +ðŁIJ ¾ +vac u +spl ac +g it +u g +high land +% ) +mer maid +sacram ento +ta ils +p w +ka h +t ell +enh anced +ì ķ +auck land +cru el +ðŁ¤ © +au dre +sail or +gram mar +g love +de on +infl am +fresh ly +k ell +zi p +christi e +mil d +di xon +instru ctor +g ence +ãħ ł +sub jec +constitu tional +crow ds +in visible +ru ins +da k +si p +pla que +p ouring +comple x +z ine +ste ad +f let +trans mission +lo way +ar un +incre asingly +au d +transp aren +cro wned +sc oun +blizz ard +lux u +fi ers +achieve ments +hun ters +rock ed +bas in +vio let +pro ves +achiev ing +pro sper +se ga +flo at +vi an +xi v +pol ic +tur a +approxim ately +wander lust +keep ers +geta way +co d +pol is +br yan +col ts +tal ents +yo gur +gluten free +wri st +gr y +cze ch +ðŁİ Ī +ev ille +ðŁı Ī +to x +dani els +am er +bi ds +weare one +me tab +g t +boy z +pd x +pos session +pu shed +shr ine +reali stic +tri gger +na vi +ru mors +n af +jen kins +tr un +comm uni +Ã Ĺ +gam ers +arm or +moham med +bal cony +y ah +stron gest +rhy thm +unfor gettable +k p +ho bb +custo dy +greg or +r ita +aes thetic +il ation +sponsor ing +n ay +kid napp +sh s +ra jas +me g +signific antly +butt ons +la c +ver sions +essenti als +opini ons +k ro +d printing +wi dely +d k +ur an +y al +reque sted +c n +cur ric +plu m +gr un +v m +dev on +m yo +rel ation +juvent us +rou ge +min ority +min es +jupit er +n ine +oxy gen +fran kie +une sco +fab ric +disgu sting +sal man +dete ction +lan ka +d ac +ðŁĩ« ðŁĩ· +argu ment +shel ves +cel tics +rober to +pi gs +he dge +fau l +pow ering +butter flies +fi r +re make +att i +com o +emp ha +kend all +poke mon +se ating +d ans +bald win +ðŁij » +lesli e +one direction +ti mber +im an +fon t +e der +di on +ste ph +for mat +gre gory +pro p +he x +ru in +sor y +inf er +n aw +bar ak +sd gs +kar ao +lu sh +v ander +end ent +g is +a fro +soc cer +ay an +t uni +lun g +da yof +alex a +mar ath +addic ted +ag ile +hy gi +light weight +ì § +mand ela +jo ey +anc y +hu m +bi r +memor ial +jim in +ging er +v ak +jav ascri +cro ps +orig ins +d ari +pi per +im port +aggre ssive +predic tion +re pairs +cr acker +voy age +ni ke +mu mmy +linke din +country side +bor der +gla ss +per t +s als +sho e +autograph ed +wal nut +colle gi +sal ary +pa iring +ðŁĮ ¸ +cath ol +swee the +defe ats +streng then +roof top +impro vements +barri ers +ur u +t ally +ru led +ðŁĨ ļ +nai ja +emo ji +per cent +gi o +pro bs +on ce +adm its +pa ths +li ar +day tona +pe ters +cal i +cal li +mu g +o sa +ap h +ab y +hy de +eth nic +pla ins +ol f +haha hahaha +holi c +?! ?! +su bli +bl acks +mo t +gh ton +lo vin +b rent +bar u +l ati +de w +ate au +q a +pain ful +bu sters +st atic +ðŁĩ¨ðŁĩ ¦ +note book +out fits +si es +r f +floo ds +Ñ Ģ +thro at +su ici +ro vers +beng al +pre pares +blo g +mini ature +Ø ¨ +am phi +com b +r sp +in timate +green e +Ì ĩ +al tar +surg ical +ves sel +... ? +gav in +g ator +threat ened +z ar +rob bery +di er +promo ted +y g +x s +su bs +inter viewing +threat ening +do zen +me ado +water fall +nintendo switch +cal um +mini sters +dro p +univers ities +war ned +tac tics +ðŁĩ ² +refu se +ad ju +v ast +ðŁĺ ´ +mc fc +lib ya +no filter +distribu ted +re ser +ron nie +de co +javascri pt +mon k +intere sts +fle x +mar tha +sti es +oo d +ðŁ¤£ ðŁ¤£ +e un +b ali +g omez +sti mul +moder ate +d ity +ir is +stra w +consist ent +direc tions +adop t +sal sa +cro o +reco vered +black friday +lan caster +accep t +weareone exo +buil ds +free man +air plane +diti on +bel ong +jam ie +pit ching +li f +om in +cri spy +pre pping +ve g +chan g +accompli shed +graci as +dolph in +elec tor +culin ary +super bowl +wal a +pur suit +black berry +be an +cardin al +pro ved +immigr ant +stric tly +holocau st +pass age +ha us +cou p +pur se +har ass +< < +le ed +ado be +st ad +legis lat +par ked +pri yan +sil va +kri st +s the +fun ky +ig a +sett lement +ph s +t mrw +stre ssed +hun t +ho ckey +treas ures +cham bers +ol u +hu t +mar ley +tex ture +wilder ness +mm ing +poten tially +om aha +ju dy +to es +spo iler +distingui shed +feli x +ah u +recommend ations +zom bies +hit ler +tri ple +colla pse +motiv ated +ulti mat +gg ling +so y +ci gar +fo ren +vine yard +gl itter +fin dings +colon ial +hun ter +eri k +den s +beet le +lot te +sub tle +s matter +tru sted +experim ental +nam ents +ðŁĺ Ĩ +regi on +acquis ition +bre eding +quarter back +am reading +oo td +ru de +initi atives +st out +hy ung +out come +al fred +mic s +exper tise +bacter ia +pengu ins +jump er +valen cia +bar k +ing day +sell ers +contrac ts +hou ston +commissi oned +adap tation +swan sea +santi ago +common wealth +ju dging +sub mission +sco rer +tom my +ñ o +ex quis +fil ing +explan ation +alli son +wemb ley +ri dge +chev y +san tos +own ership +cogn itive +favour ites +sh ed +phil anthro +dele ted +go dd +s nor +gui delines +ff ing +je ep +cli ps +sw amp +an or +guil d +bol ton +spring field +munici pal +goal keeper +ye on +ðŁĺįðŁĺį ðŁĺįðŁĺį +ãħĭ ãħĭ +water front +gra ve +contempor ary +ar ity +ÃŃ a +sle eps +sy rup +al am +pi re +co yo +moto gp +ty son +kej ri +cir cul +sing ly +cr unch +complic ated +nostal gia +k op +mo ve +k ale +mac ro +mid west +h ans +tri bal +nu de +௠į +bey once +congratul ate +cat er +leagu e +ðŁĻ Ĭ +la dder +cra shed +tech nic +karao ke +harass ment +ro ts +experi encing +kri sten +ðŁĩ ³ +ðŁ¤ Ĺ +reflec tions +guin ness +illustr ator +ðŁĻı ðŁı» +cen ter +nar row +comm ons +regul ations +Ù Ĩ +har m +cro ft +cu ssion +hong kong +st ical +intern ship +zo e +cho p +hoo ds +estim ated +batter ies +berke ley +smooth ie +shau n +cro s +~ ~ +cam pe +hu mp +b g +proto type +cl ick +shaw n +re viewed +tem pl +p f +jed i +blo gs +ray mond +as th +ba h +av ail +scot ch +leaf s +nik ki +to k +hol low +ur ges +of t +un like +lat in +u e +cat ering +mil i +alter nati +ma ver +Ð ¸ +ag le +pre order +lu x +cu cu +ðŁijı ðŁijı +t art +âĿ¤âĿ¤ âĿ¤ +arab ic +rapi dly +ar rang +all en +travel tuesday +pa ws +flo ws +st ability +flu id +ca pp +can berra +uu uu +sp ani +demon stration +m la +plac ement +m w +presi dents +awe som +bever ly +ani st +ne al +father sday +referen dum +la hore +o aks +deb bie +half way +gho sts +de bor +matthe ws +fi at +t fw +pre sen +rob i +de d +bro ck +laugh ed +am ounts +bam boo +kinder garten +eat en +mtv hottest +break out +u sic +fra ser +legis lative +p ang +modu le +sam my +go ver +ear ns +expe dition +gar h +concep ts +char lie +la va +bachel or +veg gies +deter mine +el lie +un locked +fru it +dal la +cou pe +wash ington +depo sit +iv ory +pau la +chic ag +gu cci +ðŁİ ĥ +cul tiv +pier ce +li fted +stu mb +re cover +musc les +conduc ting +cb s +mcla ren +sophi a +cel lu +oce ans +up loaded +game play +mal dives +kim ber +avo i +rac er +ca ine +cav s +h ana +li ga +ra ven +inter vention +inaugur ation +oo h +at traction +merchandi se +tune in +li king +juni ors +int ended +att acking +aqu arium +i wd +comp onents +sur ing +cent u +yogur t +ðŁı ĥ +show room +op tical +ty our +ju dge +yi eld +an to +pl c +transparen cy +recy cled +chi ef +ar om +ambassad ors +plan et +âĿĦ ï¸ı +om ed +vaness a +cour t +mar gar +hal ey +v r +reg ina +pd ates +hi span +live stream +âģ £ +ya hoo +gal la +secu red +w ir +bene ath +off l +n il +am b +ye g +out let +u te +pe ep +lind say +bent ley +... ! +he el +trilo gy +vo s +ty re +there fore +tor onto +ab i +simp li +ja e +exten sive +eleph ants +s or +orient ation +im peach +re play +constru cted +peter son +pa is +por ted +custom s +colla p +ad u +high lands +sal em +shel by +ko vic +stra in +ro sie +sen ators +snap s +bo bb +suz uki +bla des +k p +lo lo +gener ate +si ght +ma e +struc tural +predic t +jump ed +ah mad +sun g +just ice +gla m +vol vo +jubi lee +de tention +lo sses +pu ri +every time +Ð ° +ra o +ed ge +li mer +rese mb +har old +re tri +sacri fic +surpri ses +am c +srilan ka +bar bie +men s +fin n +ag s +ukrain ian +em brac +î IJ +flav ors +hom er +lau re +ou th +pr iced +ver de +fir m +ah s +cu b +tre y +par anor +pro fit +in dv +who a +har sh +al ot +crit ics +hu bby +fi gur +gi ra +ca stro +chan el +in put +origin als +ten ant +yy yy +ture rs +lincol n +co on +lear n +ch ou +ac are +o les +din er +hy p +bizar re +mc r +let sgo +decor ating +ðŁĮ İ +al ison +ar vin +f d +reha b +mccar thy +lot tery +da h +minne apolis +eli gible +diagno sed +emer ald +destin ations +s ans +or y +bla zers +n v +ba il +digital art +no c +mal ta +sol ar +pi pes +alleg ations +no ck +po pe +bri d +premi er +n x +present ations +ef a +bo ws +val ve +opp onent +Į ë +visu al +ing le +cate gor +e ter +po is +dan i +at tract +neu tral +th ene +cra shes +fred die +ut ili +c st +awak ening +slo ven +quali fy +pro of +fair y +le v +fre ight +enjo ys +cup cake +flav our +â ķ +protec tive +ðŁijı ðŁı» +is u +ad mir +h mmm +continu ous +ai res +rap tors +showcas ing +y uk +pa ste +follow er +instru ctions +sp ru +@ __ +the o +debu ts +ve tte +sto w +es of +ach ed +sul tan +sand wich +som alia +franc o +car ne +flu ffy +al pine +jas mine +he ated +viol in +ple ss +divor ce +per former +phi es +port sm +dar a +kir by +lo p +chill i +for th +sky pe +ðŁĩ®ðŁĩ ¹ +celebr ities +ed y +ve e +po ison +ey el +gra bs +ssi c +un o +wester n +rail road +am er +numer ous +s v +fo w +fi st +âĢ ĭ +reque sts +mar tial +em my +accept ance +lau ra +ภ´ +er up +hyun dai +out lander +u tt +wrest le +esp resso +demand ing +g dp +geo graphy +sas kat +tro ll +confe der +su es +se m +be ts +t ful +to sh +teach es +col oured +gal way +mac y +dis orders +bb cra +at em +fen der +lit ter +e sh +provi ders +renov ation +nomin ate +ps g +nomin ations +jen na +shar p +some day +z ur +bra ins +che shire +pre y +hu go + ¿ +to ken +r v +car r +tac tical +zel da +kay la +fern ando +photograph ers +j our +umb rella +woo dy +congress man +du mp +le vy +ju an +d azz +sign als +la in +an u +mic hel +por ch +al den +sibl ings +y ale +pe el +sw ick +gg in +ll c +k ale +s con +il d +pat reon +re el +qu in +wit t +mar ty +moo dy +ton i +der y +g ators +speci fically +dd in +ly on +tr ick +meado ws +p j +bor gh +vi k +tu r +bron x +pu ff +lan tern +ðŁ¤ ¦ +g ently +be stie +fac t +refu sed +fas ci +mp y +ðŁĶ µ +cross over +mead ow +indian apolis +duc ation +sle y +loo m +mix er +new music +film maker +prosper ity +li m +week end +cre amy +neu tr +lu ther +h v +nor thern +tw o +h ra +cat ches +appear ances +ha bit +kitt ens +n v +illa c +inf an +regar dless +liz ard +dun k +cur tain +ac om +in tu +ve z +e min +fl ats +calend ars +em power +ru ined +hun gary +vi d +we x +u lum +aber deen +o sa +k t +ma ssi +se emed +s den +' ? +tele phone +de fi +insp ires +me ow +z ones +bl ind +pl y +tuc son +advent ure +ge d +oy ster +ðŁijıðŁijı ðŁijı +out put +tt t +metal lic +sma sh +ucl a +sco ts +perfe ct +lu cy +regular ly +sp ic +rel ative +ath ers +mis e +batt ling +deci des +mat a +occu pied +random ly +cat softwitter +gi an +ball y +al ties +al lies +im men +sy rac +ðŁĴľ ðŁĴľ +l lan +au r +k ut +lam ar +affe cts +n ra +star war +ðŁ¤ ĺ +sc ram +en chan +pro cess +luxu rious +ar ray +sher lock +comp ati +dor f +stre ss +m su +s with +sal a +sof instagram +fo il +under stood +qu ay +r p +c ade +ja w +en ab +en coun +ðŁİī : +do ck +satur n +mu ll +lay out +ra rely +happ ily +fix ture +or ph +over looking +her bs +m itt +pil lar +nol an +pe tty +str y +u i +mu k +o res +o vers +á µ +re creation +we sley +ri t +kejri wal +sto cking +g v +subscri bers +moo se +ma e +ber t +opp re +assign ment +u ro +high lighting +cal vin +we igh +cambo dia +av on +ke m +dis abilities +read y +char gers +p ads +iz ing +illi an +tru ste +col leges +associ ates +alban y +mil ton +cr on +bu r +har dly +si ghts +anti ques +e cho +surpri singly +ha iti +cap t +ph p +op io +ine quality +equ al +ken y +sch mid +autograph s +ren t +qu er +cit rus +challeng ed +te c +epi de +fe st +z hou +li me +citizen ship +cry stal +convin ced +mess enger +copen hagen +âĿĹ ï¸ı +war ran +develop ments +ï¸ı âĥ£ +fore x +hi ro +sne akers +xi de +vi va +stere o +bat ting +ss el +ho st +beng al +critic ism +q c +cr un +attemp ted +ry e +determin ation +cre ations +d read +label s +pos se +anc er +joh an +si ster +partner ships +les bian +k st +guaran tee +bar o +fix ing +ma son +m ous +chem icals +t less +bio diversity +par o +bhar at +ac ol +refu ge +en te +t iti +dys sey +respon ds +lef to +in er +se vel +rahu l +ol ine +frank fur +cho reo +enjoy able +c to +strugg les +wood land +heavy weight +gen s +rece p +ac cred +ðŁĺ ¡ +trans formed +list en +at op +n k +sur ge +be re +gover nor +prison ers +clau de +t ill +mu lator +emo tion +water loo +star t +ðŁĩ º +clean ed +grand mother +fear less +afric an +astron omy +ðŁı ģ +ภĻ +the world +su itable +anth ony +k and +tt en +meaning ful +disc lo +jaco bs +à ¸ +tom linson +ghe tti +ty pho +sub stan +as co +te k +nag ar +mu d +am on +vacc ine +f ty +fle sh +no el +infl ation +portu gue +glam our +tra m +v re +te qu +roun dup +w yn +rejec ted +mosa ic +si ghting +cal f +o ta +com position +go pro +gonz ale +e ed +b ard +tu e +effec tively +we en +al to +ri bs +rel ate +thir sty +fu rious +di m +ch ard +perfu me +s ny +chur chill +k of +master class +wa ve +ðŁĶ µ +er in +own s +to be +sk illed +te m +go f +en i +tor i +cra zy +l ick +resi stant +ici al +ag ar +! : +g ali +del aware +bl itz +koh li +pu ck +avail ability +hi malay +influ ential +cro chet +victor i +read ing +ho bby +vie t +j as +en gra +sk ul +ðŁĩ² ðŁĩ +educ ate +tech no +distric ts +blu es +se tt +seven th +lear ns +ee ee +apocaly pse +hang out +cru el +mu tu +bru h +hel en +she er +c tion +kle in +tex ans +ce real +sh ine +ne red +gra s +am bro +f ella +hin du +matthe w +li ma +mir anda +je wel +so ho +euro vision +neighb ours +chand ler +be sides +ðŁ¥ ° +ast ros +thu mbs +ren ault +ra ve +hi red +ðŁĸ ¤ +it ary +z or +bla zer +k ine +ea u +kat y +dc comics +pe c +ro dgers +water proof +kill ers +super int +pre serv +as so +brew ers +promo tional +sc am +villa ges +sket ches +ju icy +for life +au dit +so lo +fundam ental +len e +philipp ine +t end +conserv atives +sponsor ship +dd le +a ine +h tc +os i +hul k +w af +ภĻ +evalu ation +ant ine +sle e +robert son +roo sevel +ag i +sophi stic +emplo yers +bubb les +ko wski +inter action +sh u +bou le +ic an +j are +han k +leg itim +k nicks +kar ma +recei ver +per ks +u h +sta ir +sun i +labor atory +gra ves +voc als +oo t +c ture +thri ve +tic o +ãĥ ³ +b w +carto ons +mcdon alds +dra w +y ung +pl er +li d +eth ical +groo ve +ent a +international womensday +pat ron +wor ries +ðŁİ ħ +ðŁij ĭ +ka therine +di az +tor i +bach chan +tru st +min eral +ic om +buil ders +bor n +col oring +lat te +ca se +revolu tion +tra der +ox id +chi pot +inst antly +sou thern +se hun +pro b +her nandez +lis bon +hu awe +p ong +me a +ro oney +wheel chair +ke en +be tt +cor in +regulat ory +di splac +ka ren +sch em +sun sets +wh ales +remin is +he p +hi de +mar cel +pand ora +do yle +th fc +ot to +no kia +trans gender +ko v +hawai ian +sha ve +so vere +exc er +nick i +pu g +st or +ro th +wee t +leg al +dig nity +po w +hom age +ðŁĩ³ ðŁĩ +s re +can on +la x +wo ah +quart z +ñ a +gree ting +flick r +nai robi +advoc ates +an c +vi i +eu gene +th ra +c re +el an +pen sion +th letics +ton i +re agan +x v +sto re +ben ch +har lem +todd ler +sent enced +âĻ¥ ï¸ı +glob ally +che aper +u f +ma m +nic o +ik u +tho u +ni st +dam i +th ala +rho des +sal e +bow ls +â Ī +las vegas +sanc tions +adm ire +mat ched +un able +travel er +ele ven +straw berries +âĢĶâĢĶ âĢĶâĢĶ +stu dio +jac ques +im s +valu ed +s no +cheese cake +n xt +e os +s x +f x +ton ic +hat ch +chic ks +gra ds +hand ic +r ory +as p +ri pped +denti st +n en +lu fc +âľ Ĭ +di ge +hop kins +sher man +f da +for all +ash ley +str and +h y +liqu or +buffe t +ess ence +phar ma +suri ya +ðŁĴĻ ðŁĴĻ +festi vals +z an +re fresh +pur ple +uni forms +kenne th += ) +as an +hel sin +transform ers +k ali +person alized +chal k +bo bby +â Į +the mes +depar ture +prin t +illustr ations +qui et +agre es +gri ff +Ø ³ +m iti +toge ther +conven ience +ab ar +car lo +turt les +info sec +some what +ar lington +scholar ships +emir ates +mu ms +st ella +auton om +fe ather +g ore +nom inees +fragr ance +Ñ Ĥ +w ong +thea stern +gr e +z illa +is i +bump er +go o +do zens +ab duc +âļª ï¸ı +o ils +don ors +sil icon +i pod +fortn ite +ðŁĴ ¨ +tor o +spark ling +consci ousness +pal a +nu m +moun ted +ffin s +thi eves +team mate +pra b +om er +ta pes +bo d +mit su +ste w +e re +p bs +tu sc +lo we +ra de +parliam entary +h m +ed gar +ðŁijĩ ðŁijĩ +to a +a gh +hon i +s late +ge ek +ap t +hard t +ta p +horiz on +grow th +make over +hi l +paper back +id an +reha bil +gi u +possi bilities +let tu +fran co +bo ss +ach er +does nt +mo e +ta ker +huss ain +ml k +di l +th ia +ham a +real ised +raven s +curric ulum +m ith +k night +ted x +r v +isai ah +cumb ria +birth days +f ing +pre z +mu barak +exquis ite +clear ance +y en +par i +ev o +à º +modi fied +app lying +imple ment +disco vering +chap man +indie game +dis k +crowd funding +mach in +li vel +sty led +âĿ Į +ma king +rehear sals +nutr iti +subscri ption +and ro +cre ators +car ries +ky lie +cam den +appren tice +tax pay +c ca +tuesday thoughts +pis sed +er man +dete c +freed om +mer i +.. ! +psal m +sun light +per spec +be ings +book store +rock star +fun ctions +p ence +fav es +z n +obam acare +sp ill +coven try +pi geon +pi vo +ba it +kol kata +av al +don or +wa h +privi leg +tra ditions +rajas than +ten ess +portugue se +yn es +tack les +de fic +tor n +pol ling +thor ne +in a +bened ict +bar ry +cal ories +ver dict +save the +nor ton +off ice +main stream +impro ves +fr on +respon ding +real tor +scotti sh +de clar +r l +shi v +supp lier +re sting +swee ts +qu i +. âĢ¦ +whit ney +startu p +thank you +teach er +h alls +ha ve +hand made +pro ving +quar tet +ro chester +li an +virtu al +mend es +of icial +mid lands +x box +meas uring +o vo +accommod ation +bri des +collegi ate +intellec tual +in car +ni ag +ðŁį · +sf w +coco a +co ats +civil ians +presi dency +mat rix +sweethe art +tri athlon +wag ner +ra dic +plann er +the o +execu tion +k um +the walkingdead +sc ar +ro tation +blo gging +bom b +re son +bb les +st are +assi sted +e do +brand ed +war nings +thor pe +acknow le +satis fied +sho res +ri d +dor a +phys ically +bi gh +appro ves +ha h +ric al +vers atile +pret end +lu m +ab hi +ye e +sp it +ãĢ Į +dj s +ash tra +j t +ven ues +gram mys +cy clo +tr acker +over watch +repl ica +el yn +nr l +lind sey +hom o +ballo ons +kitch en +si s +am os +ende av +ðŁĴ » +a rec +thu g +hoo ked +hr c +new york +bur gh +americ as +patric ia +ug u +ap athy +ha st +psy chi +cor k +petro l +ðŁİ ¬ +ak u +po pping +psycho logical +au x +g ma +cad illac +wa ste +auth ent +bri stol +nam e +que er +to ber +jer ry +com in +ch ant +privileg ed +op ar +lo ser +tex t +mar ker +stri es +equ ally +ak i +christ mas +gare th +ble w +em ma +imag in +se als +che at +conditi oning +j ana +ren s +dar ies +o asis +disc ounts +coun cil +i ka +shir ley +vou cher +al ps +w x +q r +dri ft +attemp ting +ut c +Ø ª +gonzale z +m f +jo ker +paralle l +pa re +aspe cts +proce du +n p +am a +rale igh +bright en +gu ire +radi ation +cre scent +ho b +il le +str and +v ore +n ard +che st +di wali +av atar +al der +d ling +pa thetic +ðŁĴ ĺ +spir it +jor ge +film making +ðŁĻı ðŁĻı +challeng er +b j +down town +ht ml +ade qu +twi sted +in ely +( ' +wra ps +oper ational +y ne +n us +mag net +market place +health ier +snap shot +dam on +inter ven +fe derer +ow ls +biscu its +j p +ro deo +blue berry +lec tion +fron tier +summ ers +re yes +pede strian +go l +caf fe +refur bi +bou lder +me ghan +speci alty +la ss +e i +suspec ts +appro x +rr r +ra th +st im +cru shed +he d +wh un +lo af +cr ore +river a +gene tics +so ck +wa sted +ny pd +answ ering +do ve +bel la +ol in +du n +fi ji +pre tty +spar kle +y un +j d +euro pa +li fts +am ber +mu r +te k +boy d +roy alty +in do +ri b +go tham +ti est +inst alling +ke mp +the photo +cos mic +) )) +whole sale +loy ment +eas y +su ing +sett led +af p +pro ver +suppor tive +re es +ne ath +deli ber +c é +wel come +pic oftheday +new born +pat ty +sun s +si est +fl int +diffe rently +spo ilers +troop er +g ins +cor y +look out +equi pped +ta pe +to by +resear cher +u sh +ke yes +al ma +induc tion +k w +k har +sl ick +bri de +e ur +cra ving +book ings +ch es +tr unk +vern on +sp her +cryst als +rel atively +pom pe +uni ons +val ley +par a +w ant +ok c +de af +ser gio +len non +sh ay +cr a +v at +he e +t we +liqu id +pol y +ðŁİ ģ +b ent +be aring +motor sport +bar be +te sti +han i +fin ancing +astron aut +water colour +ri sh +comic con +gar t +wr ong +ber n +it an +ste pped +fil ters +c low +me x +dem ons +all o +expand ed +comm and +et ers +go ats +si ri +y r +pot tery +mari on +i le +el an +san to +person a +du ke +hom eless +li ghted +wheel er +chang er +cab bage +sur real +ham burg +sma shed +str an +k not +i art +ob i +be dro +di al +th ick +b ingo +fu s +vacu um +con ve +ati ve +accur acy +accoun t +re fer +ri z +spider man +ban a +r ite +u b +ab s +medic al +lin k +si em +> >>> +be tra +g lowing +re actions +pupp et +spa ghetti +ang s +re medi +pray for +roy ce +char lotte +£ ï¸ı +gh et +affe cting +ro de +soci alist +mo ses +az i +o it +re porters +cd t +ap ing +s nat +minim al +wa ist +sie ge +>> >> +ri g +schmid t +h are +ec a +thor n +he mp +es the +cly de +th a +don ut +moham ed +ling erie +le gg +carpen ter +perform ers +de a +imag ined +cur se +la sh +ct r +agu a +ro ar +gr i +ro le +j fk +resur rec +roosevel t +maril yn +sm alle +will is +wa ited +char ities +the res +li k +origin al +car i +c ough +cru ci +la gun +contra st +k ou +arm our +re moving +t ent +maz da +bri ghter +thi ef +cor ner +tequ ila +buzz ing +al bi +p am +az ure +disc oun +pixel art +possi bility +ham ont +tra des +bu da +hi ve +vers y +fin ch +tran spa +em i +terri fying +in qui +g ba +sub stitu +collec ti +plac ing +cin dy +k ann +pa tho +diamon d +mour inho +guine a +anthro po +air s +pu mps +ì ļ +pas o +cur ling +an ita +resi dency +ne wh +jo on +cigare tte +que ue +ex trac +gam es +spl en +ex press +public ly +bon nie +tribun e +ba ek +reason able +c or +timo thy +she eran +Ä ± +f dn +su tton +concentr ation +carav an +x avier +al ger +cy lin +freder ick +ner ve +pe ak +lettu ce +j ail +pre game +kav an +up graded +eco logy +squad ron +gra pes +goo g +pa stry +ðŁĹ £ +ãĥ¼ ãĥ +mil ano +awa z +presen ter +ðŁĮ ¿ +her d +king s +tem plate +fl our +h v +k ley +i ya +spe c +at er +frankfur t +co ch +tex ting +del i +communi st +regi ment +ele anor +anticip ated +ðŁijĮ ðŁı» +thephoto hour +ran o +survi ving +simul ation +daw son +ar in +aqu a +m or +âĢ¦ . +cin o +ira qi +sh az +dun dee +we s +dra u +hann ah +s news +occup ation +ste en +x m +ang les +sett ings +gur u +kno x +or ca +shap ing +w ent +dr illing +zz ie +br i +kis sing +fin d +ma ine +âŃIJï¸ı âŃIJï¸ı +ðŁĮ į +lar ry +bu sted +ta vern +acti vely +- " +replac ing +no d +un lock +. " +âŀ ¤ +affili ate +to w +l n +happy newyear +di f +j m +green wich +contro versy +daw g +con dol +sav annah +compens ation +touch down +te o +amb itious +embro i +convic ted +iart g +bar ack +tr ance +testim ony +au dition +thum b +my ths +be x +que z +orch id +den y +entit led +hoo d +gr ant +in box +blue jays +r illa +smalle st +bur den +in famous +divi ded +boun daries +t ter +el t +wy oming +be verage +me sm +one ws +budd hist +y ana +as sad +is ms +bar rett +predic ted +back to +tw it +e there +cap tains +escap ed +ay o +lam borgh +gard ner +la ps +k al +adverti sement +insec ts +na po +am en +ac y +r and +g k +te h +k athle +tri dge +pan cake +at ro +pyram id +bu la +paral ym +gau ge +en cies +tom y +biscu it +but cher +quali fier +coun ty +ke i +po ols +dar ker +should ers +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +sp re +( " +writ ers +g m +ðŁİ ĵ +k nit +hu ff +mt b +philli es +o st +den is +g art +licen sed +inter face +ex cel +d well +from the +co fficial +az zi +appear ing +fore st +n ana +ke ith +manufac turers +beck ham +) ? +e se +col ony +delic ate +ut ter +mc in +transpl ant +pre ferred +par d +ari e +hu b +po ds +perspec tives +pic t +del u +app er +be than +p mo +crimin als +femin ism +sh ack +circum stances +fel las +prote sting +wa x +sugge sted +t ator +dre w +om ni +fa ke +kath y +re b +del ine +ber ni +mi sty +ðŁij © +er able +break through +men swear +millenni als +chan yeol +la z +inser t +rep lies +phra se +n x +ihear tawards +audre y +gran ite +rac ec +ori e +ter ra +innov ations +britt any +at eral +pe ar +bio logical +sh ments +institu tion +m sn +frequ ency +d man +neg lec +t f +ste fan +fox news +ty po +comm s +sequ ence +car men +wh ites +econom ist +exe ter +se um +re sorts +cas ually +bun de +divi de +Ø ¹ +ga g +cre ed +reti re +cau cus +rapi ds +wrestle mania +tul sa +sunder land +fundam ent +o di +yam aha +v ary +intri gu +el se +be acon +an gie +tra ded +tran sm +g ents +kn itting +gal ac +ðĿ Ĺ +u to +sea side +hol t +re rs +far go +train ers +mon soon +b ale +sou ght +mad die +h w +co li +fr an +fav s +ðŁĴ Ķ +int ent +r ally +s bs +lemon ade +barack obama +bre ad +stick y +explo sive +chel ten +t j +as soc +ram en +hom ies +v log +mi ster +lor d +âĢįâĻ Ģï¸ı +aly ssa +sketch book +ru mble +cat ch +migr ant +discipl ine +un likely +chronic les +fl ora +sl ams +am id +s boro +coo p +ju mps +tran qu +mel is +sof ia +en ri +gab e +sy ri +nicol as +cha i +w v +be cky +foo ty +ta o +suppo se +ðŁĺįðŁĺį ðŁĺįðŁĺį +plu sh +ri sh +ðŁ¤ ĵ +k ha +satur days +ac cent +he c +lim it +carl ton +wi red +taylor swift +ðŁĺ ij +sq l +har ro +recipi ents +g at +go p +th of +amaz ed +gh an +ðŁıĨ ðŁıĨ +por to +cla re +di stant +na c +ohi o +ðŁĻı ðŁı¼ +mt n +anti bio +dino sa +me sa +par tial +b v +lear nt +lov ato +questi on +ex tract +gossi p +gi bb +niag ara +ðŁij ¨ +displa yed +so oner +ste vie +nug gets +ml n +bro m +tur b +give aways +stu pi +bl ink +c ili +conven ient +mo h +vi ve +f ric +cau se +cham ber +cu les +ne arest +is se +small biz +t j +canadi ans +smar ter +bra sil +ra re +que tte +w ha +cand le +at omic +ðŁijį ðŁijį +warri or +relax ed +stri ps +ne ur +k ka +r fc +jen sen +reco vering +respon ses +sal am +ortho dox +acti ve +ell ers +n it +âŃ IJ +metro politan +centu ries +vi da +gra ding +transpa rent +sim ple +do ts +superint endent +elev ator +autom ated +red skins +ima m +summer time +jona than +ge aring +michel le +confl ic +m ice +to te +publi sh +pa x +) - +na iled +á ´ +tele scope +ser bia +ba b +ape u +st ically +sen ti +r ats +isol ated +grou p +hat red +paranor mal +stan ley +ali on +safe ty +l s +ठ° +nex us +alexand ra +mas ks ++ + +tr on +au k +brother hood +brow se +mix es +sim one +mu sk +appro ve +lo la +ex p +per th +fu turi +un seen +d m +chel se +sc outing +o we +portsm outh +k ram +mi ze +di spen +su p +d lc +adver t +tere sa +is le +cy cle +met all +shi elds +marin ers +ra z +ing en +fun d +an go +jon es +o ka +mad den +broc coli +domin ic +situ ations +mer o +cric ke +puni shment +d b +sha king +ðŁĺ ļ +m q +ari ans +le h +cla w +we ds +d ure +ni el +j elly +gour met +tra ders +le vi +w ages +kne es +wi se +heaven ly +avi d +melo dy +z ack +ban anas +apprentic e +pro p +fun ny +o de +respec ted +me gan +fe wer +dra fted +med it +gra pe +us army +cru sad +vo cali +prepar ations +non sense +us age +th r +ro th +wiz ards +insi de +promo tions +mon a +red sox +si g +eleg ance +ch ia +univer sal +ãĢ į +ra ja +un ga +pol lin +filip ino +ak a +t sun +ik on +bi king +decor ations +z ac +cade ts +hum our +ag m +re ppin +vac cin +elo ve +u w +dia be +galla gher +az er +do l +a while +pro minent +wel sh +t ann +' ) +bi en +wa g +in al +c wc +wic ket +ur st +q anon +x e +out door +dun n +star r +co logy +ric ky +u efa +reb ounds +s music +inf ant +ðŁĻ ĭ +so p +u mber +hand ing +beg in +sor ting +ha sh +sp ati +re k +buda pest +black hawks +dele te +ro m +can did +auth ori +de bris +spe cul +inter section +marri ott +im ran +ðŁĺģ ðŁĺģ +cru ises +ram sey +rafa el +aware ness +vas cular +beyon cé +ru g +ðŁĺ Į +festi v +ar am +s able +bas il +p ill +flo oring +un beaten +implic ations +u f +w ound +for ge +poin ting +po ts +popular ity +ðŁijı ðŁı» +mani pul +s lots +deb ates +abs ence +ver mont +never forget +wri st +gl oria +ren ce +hu sk +mel ting +ðŁİ Ł +br aces +tim ely +transform ing +am ps +ma k +po e +ah an +gener ally +nd p +ale ppo +unic ef +pro fs +nor d +ma sk +jackson ville +v v +sh ells +bloom ing +oper ators +char coal +ne ville +ma gi +chi p +sam a +ir an +re forms +accu mul +ru e +æ ľ +web sites +ga on +devast ating +sto s +glaci er +ra pp +chipot le +pr a +or ous +rom ney +seas on +decor ative +c isco +dit ch +compla in +ll o +assu me +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +n els +cent ric +ft w +car rots +tat a +can ter +per ience +li ers +demo s +bl unt +oper ate +reserv ations +le ah +sub stance +di son +an te +elec tion +v ue +squ are +non profit +ca a +f su +y am +ãĤ ¤ +v ladi +comple tes +mar i +philli p +ne ill +er as +ka it +men do +mahar ashtra +g p +dan e +provi dence +ther apeu +juven ile +me mo +in corpor +aa aa +seven teen +teen ager +à £ +or ns +wi de +cu teness +tw d +ff les +bar a +com edy +over time +y az +bar on +unemp loyment +ðŁij ĭ +exter ior +den se +cent res +match up +history month +artif icial +qu it +e sk +war n +cr itic +j af +ðŁĵ ² +inform ative +fu els +recy cle +nam ing +stri pe +sol ic +mole cular +dee pi +con vo +s sel +na e +de scent +ti z +accoun tability +ter ry +r ito +sl ay +em o +dem ol +sens ation +co v +tor e +round table +y ol +excu ses +ॠį +tur quo +hh hh +pod casts +cele b +me ssi +li o +man n +contribu ted +u z +gener ator +ele ts +veg gie +indu l +en suring +detro it +pun jab +tran spor +instru ction +ad d +por cel +pan eli +cir cles +persi st +clay ton +sp n +dog softwitter +is nt +sp r +retail ers +p w +hun gar +el ena +mon aster +gu atem +je ssie +an z +ra shi +fle e +car ving +fau x +l al +hen ri +d jo +du ll +s ana +lar a +glo be +cri mson +com pass +pau se +na b +lion el +ba ths +u fo +invent ory +sin gh +sat an +ðŁĩ ¸ +ce ments +in form +gener ated +bi den +av g +tas ks +de er +sa u +ja iled +pa stel +sc c +na il +steel e +per is +lamborgh ini +pur sue +mar gin +u ch +bo sch +dra in +cl ara +bo m +lat ino +web ster +rose mary +r ha +s oun +billion aire +not ch +percent age +con or +' " +hom es +earth day +h ort +big gest +di sin +wal ton +edit ors +im ma +om ar +equi valent +pharmac eu +ah med +cam eo +han ni +under rated +ge ment +micro bi +v oo +honor able +obe sity +âļ ¡ï¸ı +limer ick +invol vement +st agram +boule vard +bur g +blackand white +liber ation +fi ve +inter im +sm m +rival ry +cap abilities +stat ements +thu mb +ve d +sw ans +bar ber +e que +seren a +hel m +noo dle +sam pling +n awaz +sing le +thunder storms +sh on +in ev +ë ¯ +to pp +orch ard +bi an +ðŁĺ Ķ +door step +salv ation +marke ting +r ons +cle mson +ra vi +in take +stand with +sin a +ha iku +ple y +elector al +ph illy +la ys +electr ic +cap turing +u pp +er gy +believ ing +cul tures +es day +inva sive +ed ed +spee ch +end ur +viet nam +boy cott +pe de +deli ver +ðŁĴĸ ðŁĴĸ +mer chant +st ir +den ies +poc kets +o ti +cu ddle +ro land +mm ed +den ed +lear ners +hoo p +sour cing +h acked +di m +environ ments +ben son +jud icial +wor cester +pear ls +govern ments +arri vals +cor ners +tun ing +la bour +y m +or dering +le wi +i fe +hygi ene +thou ghtful +indone sian +campaig ning +princi ple +assau l +ru bb +at v +wil ly +en tre +il i +ph on +du ties +âĻ¥ âĻ¥ +sn akes +lo op +am ar +conver tible +bon ding +ment oring +max well +ethere um +destro ying +ax is +ca iro +fin nish +sho ck +ðŁĺ IJ +cal eb +com a +pe dal +co re +contin ent +el son +temp o +helsin ki +ac p +tack ling +st ated +bl a +dou b +sma shing +a ja +camer on +disru ption +warm th +being salmankhan +bullet in +o de +syrac use +ar an +mc gregor +bul k +an ton +confir mation +sp ine +im ran +instru c +jac ks +chi o +pal m +str e +embarra ssing +un t +elimin ate +to ss +c ise +a ws +oni sts +sh inee +jo s +ho se +li vely +opp onents +mo vements +recogni zing +sandwich es +sh akes +exerc ises +se at +profe ssion +merry christmas +lu gg +adopt dont +mar vin +byr ne +un le +he t +ku wait +rah man +aspe ct +humb led +gen es +f and +long time +) ; +cam pu +an gus +ðŁijį ðŁı¼ +q uran +sle eves +s lic +¸ ë +twel ve +your e +i ke +go gh +b st +dic tionary +reflec ting +to on +yar n +em bed +ðŁı ´ +re serves +floo ded +ver iz +du sk +estab lish +pro li +au d +ritu al +or bit +declar ation +recor dings +cam o +cas sette +good luck +cu tter +bo p +b ho +che ating +paci fic +ma res +tim er +col t +tr ous +tomor row +han sen +ci e +w ang +ban i +circu lar +ac ute +far mer +co ys +p se +ir ving +w j +haw kins +b ison +ur day +cru ising +o te +k ath +whi stle +your selves +ant is +sla sh +thorough ly +ke sh +ser ie +ex em +en ig +guil d +sh red +ho gan +ap o +ä ¸ +pu zz +ne tball +au ssi +panor ama +ws j +av is +ar ming +hum ph +brow ser +cri es +fo ggy +mat te +ðŁĮ » +it er +tal lest +by ron +cap tiv +je su +any ways +flag ship +p ton +we y +fay ette +financi al +f oul +solom on +jenni fer +cucu mber +ar gue +tex tile +wrest ler +john ston +pa stor +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +cac tus +edi ble +re served +ric hie +met res +ingredi ent +h ella +un to +ch ol +cele bs +po ets +gra ham +hay den +coinci dence +b aw +communic ate +flet cher +/ - +tole do +ecu ador +coun sel +s laughter +line ar +at p +os u +jo el +ev ed +conqu er +ru stic +plic ity +recogn ise +room mate +cr acked +jas per +ph er +ðŁĮ º +wo ven +mo ist +ff c +ste ering +ni sh +stand ings +frequ ent +ar di +haz el +as msg +bau m +d art +si dd +nat h +ch ero +card board +c ss +n sfw +pa ir +ðŁĺį ðŁĺĺ +occur red +homeless ness +mal one +ph e +xi a +pad dy +decl are +theat re +b f +per sian +ta d +ax e +susp icious +lam b +mu cho +sen ior +st as +k ite +st ing +gra d +k af +wat ering +Ø ¯ +spi ral +th ms +educ ator +jer ome +of c +clo ck +su l +pe mb +.... ..... +park way +de aux +restric tions +m ons +need le +e j +le agues +water melon +am an +pl enary +max im +w ab +coming soon +bry ce +vi gil +super market +fortun ate +turquo ise +presi dent +li v +inter ns +feel in +fix tures +stun t +st aged +premi eres +lo k +prac titi +shor tage +log ne +ve c +con cor +roc ke +li g +com posed +syn thetic +di p +cam ila +ch is +j ou +su san +eye brows +supp lement +satis faction +moham mad +ti bet +house of +pu n +as sam +shado whun +psy ched +se duc +mand atory +her bert +sc allo +stream ers +proto col +block buster +produc es +sch nei +lau rel +tri be +time hop +pl a +mod elling +tv time +mtv stars +wi dow +me tric +ch am +con do +flow ering +ale c +d ms +inten sity + ¨ +mccar tney +islam abad +k b +f fi +ph al +anal og +f ond +h acks +positi vity +treat y +sub marine +conne ct +sel en +categor ies +cu b +organi ze +si k +quote oftheday +remin ding +am or +loc king +ðŁijı ðŁı¼ +comp ound +et te +b out +rec ur +fe rence +mi zz +tren d +hip ster +for tress +forth coming +preli min +o dyssey +ang p +del ici +even ings +ðŁĶ ¹ +i q +d w +da ir +kathr yn +christian ity +moon light +ha b +wh oo +f bf +se th +genu inely +pa x +char ity +deplo yed +b nb +bu cs +ju dg +con ge +plant ation +im press +car a +sc lub +sco py +land ers +compla ints +b ama +re build +x y +real ism +sh our +le in +brac elets +mer a +assas sin +an chor +ðŁijĮ ðŁı¼ +lin en +con fron +chronic le +comm ent +cat alog +il les +gor ge +me try +jung kook +love my +sent in +se em +fit ness +alli ed +ts man +digital transformation +pr an +lo ft +min ton +alden richards +en vel +cher ish +certain ty +zz z +rhin o +per kins +en rich +cape town +ome ter +sec tions +ske leton +def enders +ðŁĺ Ŀ +pen c +bri t +ja h +capital ism +ðŁ¥ ĩ +baz aar +re me +ex t +kk k +conver t +stor my +b ye +kar an +chry sler +ad os +pre ssed +syn c +ation day +dang er +bad ges +refu ses +em powering +ly m +ex ports +adoptdont shop +ðŁĩ ¯ +th c +awa ited +focu ses +fin ed +o at +haha hah +âģ © +n family +fi ona +luck ily +thr illing +ty ping +out break +di es +he u +craw l +ne sses +o ath +scri pts +gee ks +ðŁIJ Ŀ +p b +mathemat ics +al is +________ ________ +gymna stics +acti vism +recommend ation +gre n +wa in +cour ty +n apol +cau li +hor nets +g als +jo ckey +dir ty +at ar +enor mous +pe st +greg ation +an os +ii ii +def ends +black historymonth +at x +mb c +lugg age +wit ch +co b +la sts +cu m +gg g +ba thing +n ar +ce bu +ðŁį ĥ +navig ation +min e +re jo +ðŁİ Ģ +gif tide +re ta +use less +pu ll +defic it +al lu +ati me +it v +tr illion +pu e +ac ies +proce dure +l ori +jen ny +c ad +ul ously +dr ac +promo tes +ing the +can u +woo hoo +na omi +zar dari +ts u +be ir +sd g +le ver +we ber +ab ud +lun d +crow ded +deplo yment +ter rain +ken ny +ho f +witne ssed +lo ch +j k +bul ly +w ren +poe try +do ff +ww i +mo red +din i +cul ture +promp t + ¥ +maur ice +to pps +r m +cor respon +ab out +jewel s +gi br +eag le +ðŁĺĺ ðŁĺĺðŁĺĺ +l ending +sou ven +ç Ķ +contemporary art +establi shment +j ong +âĢ¦ " +gat or +patri otic +mc coy +v ape +human e +feli z +coach ella +re posting +ste als +fu ller +n ering +at ra +( - +bla ke +he ather +wor ms +discipl inary +rede mption +y ard +am in +" @_ +d nc +t ds +k appa +ne wark +comm its +spe ars +j ams +t and +msn bc +inter medi +aim ed +at ic +teen th +observ ation +kash mir +kavan augh +ou l +san francisco +re u +bel ated +cho w +pass word +st ills +deta ined +sar i +day ton +dar ren +itali an +ar th +amu sic +ar bit +w m +v m +he m +dou g +my r +a sho +pre v +vin d +bra h +sta g +ภµ +pre views +gu k +con taining +leon ardo +sad dle +ru shing +st av +lon gh +gam bling +ve gas +reserv ation +end ale +bal a +fl a +vari ant +he dge +bulgar ia +nat ali +we aver +sol st +encoura ged +ap c +as parag +ne st +cycli sts +fe l +ìĬ ¤ +overwhel ming +pey ton +j it +a post +mb le +ble eding +neighbour hood +a very +expre ssions +mac donald +gi gs +mon ds +illu sion +n ct +cam ero +over head +my th +ol y +vi o +et v +lau rie +unve iling +pri or +con n +iron man +di ff +day in +crit ici +con go +re vision +wal e +direc tor +p ines +black pink +gar ner +cur ated +manit oba +h ac +common ly +bar ton +.... # +mor tality +live smatter +philos op +shor ter +con vince +fre ak +vend ors +insi ghtful +el ly +sens ors +e led +s berg +weight loss +u kip +sp ur +priv ate +qu a +ss c +, ... +supervis or +advis er +amaz ingly +less er +at es +mah on +oooo oo +sar as +pmo india +waff le +un ders +toler ance +sculp tures +her sh +kno cking +smo ke +cathol ic +gri m +tra veled +fli p +ge off +dinosa urs +sle pt +scar let +ok i +compla int +ob sc +nam i +la g +cross fit +u fc +mc cain +refe ree +sad ness +pen ny +li eu +mo de +ki er +vol s +w is +el on +she a +ba o +son ia +cla ire +em manuel +moist ure +di gest +vi ii +t eller +ch on +access ory +night club +foss il +aw an +hu sky +ab original +brand on +ffici ent +cou gars +ste d +ad mitted +igno red +content marketing +ag as +v ase +execu ted +negoti ations +she ad +n and +tab lets +go th +ts al +d fw +on ep +protec tor +sp ho +gaz ette +andre as +ss er +comp ilation +ha v +contain ers +bro ker +soc al +porcel ain +hy uk +air ing +ðŁĴ ° +publi sher +scen ario +spart ans +re viewing +itu des +ed el +pear son +ba sh +mau i +a ad +ðŁĮ Ĭ +li u +ul ate +program mes +fav our +web design +real ty +motiv ational +cro sses +' ... +bus ch +adjust able +ar jun +mist ak +dimen sion +pi stol +weigh s +en y +unve il +indy car +gor don +f ade +fran ken +qual ities +bet t +loc ate +ker r +sp c +confu sion +ne e +luck y +bas es +dep ends +fire fighter +ol a +re t +mar oon +ðŁĶ Ĭ +w am +defin ing +whe at +bi l +é s +b hai +psy ch +ta u +ic ans +thi k +ob ile +inspec tor +ìĨ Įë +ill on +go s +ev angel +fa i +si st +voc ation +bur ge +chi stan +renew ed +enthusi asm +en ting +ag ri +ike a +m sc +aero space +sens iti +memo ir +hosp ice +co caine +der ry +mechan ics +Ħ ภ+tin o +reduc es +collec tors +in justice +supp re +v ana +ab un +nap a +su sa +os lo +e ff +en core +lic ence +ched dar +z al +moun t +ðŁĴ IJ +threat ens +!! " +archi e +fu tsal +scu ba +jo s +gn on +se xi +s official +compar ing +domin ant +tof theday +fa it +propos als +gi ft +y as +cn c +l r +ha b +reser voir +beli efs +gener al +mar ti +t d +est e +ì ł +wi l +ðŁij ¯ +ðŁĶ « +sp x +et work +excer pt +e instein +hir o +sil hou +team ed +per ception +corri dor +mental health +hin ts +ben ny +induc ted +sw x +wi desp +spe ak +cher yl +dru g +ðŁĺ ķ +h f +asparag us +myster ies +fitz gerald +off er +therap ist +care er +dam aging +ts d +per u +wei bo +y ay +phoeni x +disc re +mac book +bar ker +stig ma +sp read +roc kies +kang ar +bri dg +pa i +bi shop +ta iled +capsu le +ðŁĴ ĵ +ge of +roy ale +short listed +o ste +ash amed +ch app +key e +cl a +screen shot +austri an +nati ve +en ight +juli et +michel e +ðŁĮ ´ +travel ers +pi l +football er +win chester +ðŁĻ Ħ +azer bai +gold eng +organis ations +interpre tation +predat or +ofthe week +lo gan +pok é +mari e +cal la +t nt +cin de +ge tic +fit fam +gra v +ow ens +ðŁĮ ± +shoot out +sal is +commissi ons +co he +p tic +ni xon +hi a +amb ition +mar ine +cruel ty +t k +cru de +sal ty +jim a +mon go +ir ony +on wards +arre sts +strang ers +ig er +cycli st +ra g +exten ds +tra dio +bour g +mo i +el la +e able +lex us +au l +der a +histor ian +mor ton +ti ff +man ner +ko t +d k +po inted +mar qu +a an +en ey +du blin +on poli +em ili +secre t +fl o +âļ ¡ +ba j +ste ep +accompan ied +rum ours +dev i +purch asing +fi g +pu b +sch oo +autonom ous +go alie +x ia +autom atically +re vers +ter o +fu ku +titan ic +shoo k +sand als +see kers +exc av +nor dic +bigo live +ba ke +r att +z ak +ne p +ðŁĺ ¤ +cand y +billi ons +book worm +pp et +à ³ +sur faces +sc ars +phil ip +do gg +ci gars +co te +transl ated +cur ator +sin dh +han gover +bre wer +on es +el ton +ðŁĴª ðŁı¼ +mar cu +elli ot +righ te +di oce +ru ss +rail ways +grand son +as cen +apo logy +awa it +mob ili +re spir +parti san +oli vi +stri ke +yo o +white house +expre ssed +pu ps +bed ford +cul tur +fro gs +fly ing +cav ali +c ds +fri ger +street photography +re solve +tali ban +kan g +cru shing +ju m +ðŁĺ Ĵ +william son +tan g +cur ly +t man +veter an +fa ire +artificial intelligence +un anim +pre n +back drop +fr ances +oc cer +doro thy +work ing +ar thr +conver ted +day light +serv ant +pad dle +compla ining +thir ty +nad al +ak u +ibra him +ad dressed +p iss +green house +batt alion +si mulator +out lets +embroi dery +ðŁĵ ± +fis cal +ger ard +sas sy +ðŁİī ðŁİīðŁİī +vent ures +mer it +public ity +ðŁij Ī +sophistic ated +c tu +conven tional +condol ences +isra el +tra dition +ar an +te ss +gla d +ðŁĺĬ ðŁĺĬ +correc tion +ge on +am d +or ship +be ast +ch ment +ì ŀ +nic o +wk nd +wel s +cushi on +beli e +vo c +idio ts +under neath +pu ma +corn ell +en ation +lu l +swa ch +ab ig +u rer +mi e +form erly +ca f +er nal +chor us +juli us +sen ator +âľ į +wh ir +salv ador +ph d +uni fied +boo ster +graph ical +w rec +son ny +mi z +dere rs +s all +ven s +tusc any +wi d +y ong +kur ds +w az +trol ls +mac ro +cat urday +pre ssing +sa sha +cent ennial +gu sts +em c +be fore +den ise +cu st +ðŁĵ ¢ +lo oo +base l +eng land +y olo +ar du +manife sto +do ha +ì ľ +kni ves +bourne mouth +bi bl +bar b +al icia +Ø © +com er +cycl one +g it +ane ws +character i +vent ura +in tra +sf giants +hu t +be a +dar win +ell er +al v +re ese +bl y +kar an +conclu sion +man ny +fla kes +unite blue +nad u +co pp +ed ges +lanca shire +i als +o tta +philipp e +l ent +che e +ment ors +festi val +an ism +compli mentary +r j +pu g +d ine +we i +cli ffs +sar my +ti veness +treas ury +il and +after math +rabb i +ou n +bou quet +herit age +zi on +sur render +shen an +in ks +kar l +gh ty +pol icing +exam ination +ce y +per su +measure ment +hydro gen +lu han +âłĢâłĢ âłĢâłĢ +war i +о Ð +j y +fow ler +mis h +al fre +âĺ ij +bb naija +cat alogue +recogn ised +sa ver +hu skies +col in +mun do +si va +p ng +discoun ted +man utd +fre sno +de vin +prelimin ary +tro phies +pla stics +du g +pro cu +indi go +g ard +dy lan +pit ches +ground breaking +in son +bl ac +an thology +f h +expl ic +r ard +admi ral +so chi +la shes +splen did +en vy +ad v +sex y +festiv ities +stic king +bi b +thr ill +op p +ari el +botan ical +endur ance +fe males +br icks +vat ican +black pool +ber mu +br ough +roll er +bi d +sue de +sloven ia +mm ing +ml b +med alist +di ans +rehabil itation +ne on +s go +li thu +ram os +z ed +pi anist +inten sive +broad band +stu dy +peter sburg +lu ca +ah hhh +phys ician +dill on +tele com +gri ef +mu n +ac ro +si ded +s ly +blo ws +classic cars +tri um +ar gy +? : +h ri +marsh mal +âĢ ĵ +to pping +war saw +tran sc +preserv ation +b av +re friger +experim ents +ä º +gl it +sli ga +g age +fac tor +flav ours +br ony +sp o +cook book +carri age +aw ay +ny fw +on ian +w g +simp sons +ro lex +ðŁı ¿ +cro sby +ãħ ¤ +cre di +syn dic +pu bs +ali fe +poor ly +mac ed +ðŁĺ ŀ +behin dthe +w enger +n ats +ðŁİ Ł +rubb ish +procedu res +typho on +opho bia +er do +fu el +vi era +bu mps +millenni um +new zealand +lec tures +it on +mil ky +respon ded +ê ° +landsc ape +.. @ +bo ther +âĸ ¶ +z hang +huawe i +tu ition +s worn +in u +y or +pa olo +au ditions +ab il +malay sian +ho ps +fe athers +mp le +au ts +ã o +boun ty +ic he +ì ĺ +sh q +pin ot +ge ars +disapp ear +video games +t na +alzheim er +ðŁĮ ŀ +a ji +under wear +swit ching +sign age +o scar +ec on +dro w +cl int +pl ated +gun dy +emb lem +ho es +ici st +nel ly +juni or +road show +miner als +at le +alexand ria +ac claimed +v ell +shi va +ad he +en ne +amne sty +h ounds +councill or +ðŁĴ ¦ +aes the +part nering +influ enced +mag no +fl are +extin ction +civil ian +maje sty +va il +law makers +rac ks +mc c +ori an +sp ices +er rors +may er +co ca +pa i +s ooooo +reti ring +ba thro +ðŁĻĮ ðŁĻĮ +âĸ ª +su f +endor sement +buil ding +broo ch +pal la +arvin d +ag ent +kar ate +r hi +c tv +ta ine +um m +ba x +reig ns +uni of +enterpri ses +adel e +fla ke +at tire +bru ce +ba hamas +gra vy +sa in +che ek +tri vi +lo v +e en +bb lo +lady gaga +itt a +. "- +du stin +observ atory +eigh th +bloom berg +kh s +f cc +gi st +commemor ate +ve er +sexu ality +ed c +nic ole +vac ancy +u ser +son a +:' ( +dipl oma +t end +up grades +Å Ł +jura ssic +cardi ac +dr s +widesp read +à ł +dail ies +vend or +sim plicity +wi der +len ses +supp lements +de pos +ob served +vin es +parti ally +renew al +collabor ate +ali g +fin ity +ph u +zz y +pe tit +ðŁĵ ħ +z in +i gu +sm ack +fall on +ðŁĵ £ +back wards +comp onent +o so +compati ble +bin ding +zur ich +thom e +w ounds +ly ric +fresh men +sne aky +fi bro +di et +emplo yer +in sect +h ated +sch er +raz or +n sw +boo ker +califor ni +av fc + ° +preten ding +pep si +al is +un titled +k art +grand parents +e the +o ck +lux emb +visu als +small business +abdul lah +min ho +su baru +h ra +reve aling +heart breaking +clar ity +am g +sl r +** ** +âŀ ĸ +recor d +ici ary +min ded +ye h +exce ssive +knu ck +icec ream +tru th +ev ic +ta stic +ant arc +ren dering +, , +mit t +loren zo +st patrick +bound ary +zi g +vo cab +osa ka +fur n +tu n +gu l +s ounding +blo gger +utter ly +g af +adv ancing +l cd +mar gin +lifel ong +solst ice +sh ra +wa its +ple ar +bre ach +en ligh +ad er +itt le +c ation +ho on +stu died +?? ??? +k ash +ev angeli +ps l +wei ghts +met als +ty res +tur no +wi e +car b +g ale +se al +sun ite +am ic +patter son +á n +eu ph +up stairs +quali fiers +khali fa +apple music +ìĨĮë ħ +vau ghan +al ter +cru iser +mu a +t ana +kat rina +id ols +spo iled +secre tly +fi bre +part nered +um es +gi ov +com et +screenshot saturday +k eller +fil tr +fe t +con way +pe u +bad minton +gi d +m ound +don key +bu ff +lea ther +lar gely +bro ch +int ments +am use +r k +sto ve +impac ted +con t +cr acks +prison er +bar i +contrac tor +ori oles +domin ate +pol ar +am elia +dr c +ðŁijĮ ðŁijĮ +vi st +su arez +injec tion +blo oms +ðŁļ¨ ðŁļ¨ +sti ff +pay pal +sno wing +thur sdays +goo se +we dge +educ ated +weak ness +de cker +abud ha +bree zy +Û Į +hope ful +o bi +rai der +gh am +de u +se ve +par tly +fu t +infu sed +mer ri +than e +some time +hu e +me in +cre dit +sli ding +ran de +cher ry +dead pool +sh ol +ar am +under wood +sky e +distur bing +m nt +poli shed +guardi ans +ha dn +pic asso +ari us +ak shay +ir ri +j h +happ en +la kh +dal ton +at the +s well +mar sha +re h +cour s +j kt +top us +serv ice +r ink +hack ers +dono van +hor o +tc m +may hem +cha se +dev ops +ken sing +sc up +sh ere +quali fication +c live +ton g +n ancy +mar is +der dale +ber man +cinde rella +jol ly +ci c +loo t +collecti bles +hom icide +g ge +epide mic +su ites +mu ddy +gi mme +e rec +- * +tal la +lis le +embro ide +ðŁĩ© ðŁĩª +veriz on +ve ctor +be anie +arti san +ga in +flo res +vi gil +u so +ðŁĻı ðŁı½ +grin ding +gh er +air ports +respon sive +shaf t +can cel +ceremon ies +e me +at ari +bru shes +eag er +bo hemi +children s +yan kee +ma a +suspen se +mor an +mac ar +sun flower +cre w +vo id +ke ar +fashi oned +jen nings +sunday funday +sub missions +me ad +her man +wa i +crit ically +le um +baek hyun +for cing +co bra +ãģ ® +acqu ire +al k +ge ology +pri mar +import antly +ire z +bunde sliga +curi osity +sen a +stric t +con soli +win ters +ven om +chelten ham +ðŁį º +cen a +t at +ba in +glo ver +under cover +as ses +car n +memorial day +am eli +i rene +ch on +syn thesis +spe edy +mitsu bi +sla yer +compos ite +under stands +pe w +inter rup +hen ri +mor row +an om +thof july +g lee +thre e +ðŁĺ ® +and hi +ch att +renew ables +ye s +trans fers +!!!! !!!! +bab u +du ter +lo ops +pe ers +o ilers +pau lo +ic ation +h mu +war a +mer cer +hom eland +fu ji +ale y +year book +re m +re en +ab sur +bo is +] : +caes ar +shot gun +kur dish +o ren +ra e +anci es +ty pic +f h +def ault +re plic +lu k +trans actions +r ys +infan try +ðŁį ¾ +cho w +chick ens +ba gh +wy att +ay e +gg i +bre ws +ed itions +mi ra +commen cement +pre su +peris cope +ic hi +guatem ala +zam bia +pain ts +wit ches +wan i +un dere +cro y +vo ws +us mc +hear ted +theat res +shu ffle +le vel +mul tic +squee ze +fer n +app et +post al +mal t +on board +ld nt +co o +s sc +k ac +ðŁĺ ĩ +sc rap +mar cos +deal ers +ann u +mill er +co ve +ul ary +vladi mir +be ef +th ur +pick led +se same +bengal uru +mo tt +kathle en +hi st +no tor +dr ank +du chess +snow fall +e ff +tin y +j n +sy our +speci alists +scot us +bay lor +eve rest +mali bu +pre m +harm ful +l ali +b ates +g ye +differen ti +and ra +geome try +el over +black out +== == +ko ta +inter act +asi an +la yo +samu rai +fi del +exhau sted +gla di +pd t +spher ic +anti qu +guit ar +stu ri +ho pper +ang le +f ills +sla p +mi th +rod ney +ong i +in som +pre venting +cassi dy +ap ho +ore gon +lo in +ham mond +contribu ting +f n +gar ri +ori on +comp elling +escap ing +aim ing +plu mb +bi stro +be asts +concer ning +bo e +do pp +shop local +stumb led +âĤ ¹ +naz is +âĢįâĻĤ ï¸ı +gest ure +war ts +us open +hi ggins +char li +hang s +bom bers +° : +fe eds +c ch +st il +nic ola +ðŁĵ º +clam ation +tro pic +af ro +ou k +expen ses +der rick +al ine +fa w +reg ard +im er +sat in +thi um +ry der +pear l +te ss +mm mmm +sen ses +ðŁĩ ¹ +positi ve +exhau st +occu r +nor ris +lil ly +is les +direc ting +yo fficial +count less +sam ar +on stage +flo ck +mir rors +arch er +mo i +k d +vi v +in os +si kh +le i +sen sory +br its +kno x +chest nut +op y +coli seum +z af +di vin +adap ter +:) )) +tem ple +ku n +hel mets +t df +gu ide +m old +o ids +lu ther +he is +monaster y +sp ree +k lu +brit ney +jagu ars +gre ats +c cc +ky rie +machin ery +cric ket +re ro +ab o +aspir ing +semi finals +ale ss +sig natures +var d +me th +her bal +hol den +king dom +ap or +reg gie +ore o +palestin ians +em mys +sec tional +ro i +ney mar +qu el +cu ll +l ka +haz el +estim ate +ul ties +go w +be a +purch ases +bel ts +protec ts +m é +gue ssing +bb o +clau dia +fr acking +jon ny +el k +cel tic +al mighty +ra je +courty ard +ig i +can es +ðŁĴª ðŁı» +bank rup +le thal +âľĮ ï¸ı +graphic design +vad er +penc ils +rough ly +dan te +m fg +const ell +cam el +j b +bloss oms +en to +balo chistan +cine mato +ill ard +jer sey +con sent +dent ed +con templ +sch er +hol i +lou gh +st our +a yo +begin ners +cur b +v hs +a jax +du ff +av eng +dom est +commit ting +ai red +cha p +hedge hog +disappo inting +freel ance +in land +char ms +ðŁĺį âĿ¤ï¸ı +ai sh +m x +buck le +ti dal +per mit +bo ating +ra cha +kend rick +b ello +b hi +ple a +estim ates +l b +apo logies +jay a +bb l +ast oni +inter state +main taining +el bow +mu p +ep it +ðŁĺ ¡ +viol ations +def end +be h +sl c +am ir +pur i +ti um +fi fa +blur ry +scri m +ðŁĻı ðŁı¾ +ma ple +rel atives +âĺ Ŀ +cho c +con nor +⾨ ⾨ +whi sp +list ings +ma ze +than king +ri dd +grass roots +shi fting +desper ately +gor illa +den i +ju les +stra th +g ley +ja in +bu ick +t anner +ðŁĴ Ŀ +ga e +pri m +it ors +n ano +separ ation +armen ia +bor deaux +ðŁ ħ +pj net +bu rial +e bon +glo ss +re new +gri er +spe eds +comic books +sym boli +pur poses +ãħł ãħł +spati al +no table +ci on +n ps +ho ffman +nor man +rt g +du sty +situ ated +tr an +k fc +em en +nic kel +hast ings +sett ling +gr it +l ena +w aw +art s +gu m +ca regi +le wis +sapp hire +rememb er +embed ded +t lc +bl at +serge ant +el sa +boot camp +bow man +photo graphic +pill ars +direction ers +classi fied +no is +ve er +barre ls +wh oop +ðŁĺ± ðŁĺ± +fe male +petro leum +medi a +e fc +poké mon +ठķ +enthusi astic +var un +pro files +pedi atric +acci dents +con rad +jan g +jo jo +ac or +ob server +l f +live stock +for gi +fo s +el m +an and +go e +c ere +avoi ding +gri t +om an +thank fully +scat tered +nick y +cylin der +chees y +di ver +mahe sh +cav es +ear liest +qu inte +subjec ts +b end +gul f +vocali st +glu e +pat ches +un stopp +sny der +demonstr ating +pi o +hor ns +wic kets +and the +r ama +yo on +stra ight +bed time +or ang +bul lets +sa urus +min ers +inci dents +! ... +ðŁİ ¸ +ag ers +hand les +stat es +in ity +d ons +incredi ble +emin em +avi v +ru dy +moz art +folk lore +appli ances +mt l +fre y +di as +hu a +page ant +stri ve +im prison +bul lish +r ana +al erts +bb mas +hy per +derby shire +re cre +re dd +debor ah +cosmo s +law son +mel anie +psy cho +ho or +doo dles +sni per +shad y +man tle +canadi an +new year +inter actions +separ ated +cor ds +spiritu ality +ap u +it o +p ct +pel osi +rebel lion +se iz +wor cester +sec tors +ul i +san ta +Ð µ +ðŁĩªðŁĩ ¸ +bi ased +class ical +gam ma +dee plear +emer ge +back er +sur ance +hand crafted +ðŁİ ¥ +franc is +mill an +ic i +cro wn +wo w +stri ped +un fair +relax ation +³ ï¸ı +embrac ing +she alth +pale o +martin i +dist illery +wr ink +or k +na th +hay ley +cour thouse +si ber +sa di +quiet ly +mel t +m sm +me h +smart phones +rel ent +pp ing +war wick +co logne +gli a +cot ton +pro g +lon e +ip sw +star ters +expan ds +u mp +su ed +ski pper +infe ctions +ing le +à ¡ +cler k +demonstr ate +ac ar +ðŁĺĤðŁĺĤ ðŁĺĤ +ti bet +bun s +alo m +demol ition +ssi a +g st +[ ] +so ar +âĺ Ģ +ðŁĺ ª +ðŁĵ Ĭ +dee pest +beyon d +are t +att ends +activ ated +di mit +âļª ï¸ı +high lighted +magaz ines +rum or +az za +steph ens +dol ph +sho ckey +mat s +we av +mel an +serv ers +tra um +ku sh +æ Ĺ +bab ys +pa z +a al +la use +break ers +canter bury +ul ture +mi ri +euro s +tane ous +impre ssions +du tch +il d +gh i +pur due +adequ ate +l p +sy ner +ang ler +du rable +gal ore +ro wn +mg mt +ðŁĵ Į +lu cia +âĺij ï¸ı +zay n +bor row +. ( +north umber +cru sh +eng a +su sh +extra vag +t out +ma hal +ali stic +ther mo +gall eries +es se +chi bi +attrac tions +lex ington +legislat ure +docu mented +resi den +brow nies +w f +st ool +plan ets +sho ppers +conduc tor +ms p +tr icky +fru ity +end ra +feel the +whi pped +hair style +re fer +oo k +oc topus +audi ences +ku mar +after no +op tim +c fl +ni p +gen i +alpha bet +ann ab +lam in +accep ts +l ng +ðŁĺ « +t ine +ac om +cheer leaders +t k +gr on +v g +k ung +ja x +dha bi +r ss +mack enzie +beir ut +clean up +gy psy +st ell +bur ger +hurric anes +educ ation +st ina +âĻ¡ âĻ¡ +unfortun ate +jere mi +bad ger +at ers +: âĢ¦ +ter ra +subli me +stu d +y mca +mr u +duter te +bren nan +bul b +mel o +yl on +hack er +c red +gu d +as an +pad illa +embroide red +vietnam ese +pione ers +projec tion +re boot +id c +an ey +pri mer +suff ers +win ding +p on +sto day +mor n +u ch +all in +adid as +eliza beth +tu ck +o graphy +ðŁļ Ģ +be g +os borne +ghet to +r h +cn n +ir ma +ma kin +cab les +mur ders +oc ks +inst a +al as +si k +cu ff +la re +foo dies +o vic +at om +geome tric +em pathy +ภµ +cent enary +newsp apers +administr ative +ðŁİ Ĭ +sti ve +contrac tors +le tt +tas mania +awesom eness +den sity +ve en +prince ton +frequ ently +re ject +gh i +modu lar +ceram ics +sh ag +ki wi +can vas +sweat shirt +an j +ti mm +napol i +il er +appe als +hamil ton +ma yo +we ave +arrang ed +whar f +occu py +b vb +as aki +ot ter +nor m +vi es +de tox +tion al +dere k +id ad +ad missions +constitu ency +u pper +woo t +allo y +se ve +lu b +un comfortable +ed win +ab re +d wight +ar che +virtu ally +sp ol +pri e +ai i +er r +swit ch +bar ack +se ok +cou l +wn t +pou l +o live +caffe ine +cardi ff +notor ious +de mp +ex cess +bar r +t ford +a jay +bump ed +my thology +shel ley +fal con +shakespe are +must angs +no ted +bon e +civil ization +sy d +par sons +un official +hy ped +sp ends +oppo sed +v ings +space x +noti fication +deci ding +bio tech +out si +sal ah +! . +fe d +ss y +c ms +bad gers +cr o +ela ine +n ba +dy our +n ant +honey moon +climb ed +conom y +ath a +m ell +ne bula +nature photography +juli e +bm x +inve sted +mon o +lieu tenant +wat kins +techn ician +o se +ka e +ì Ľ +mc queen +pre ach +trav eller +flexi bility +ze bra +reta iler +p ant +ben der +brand t +squ id +war rant +veri fied +cas s +pier cing +hon ours +t ying +mor ris +kis sed +op rah +panor amic +me i +splat oon +wich ita +ari as +gal li +indy ref +good times +athe ist +confe ssion +ow ski +re pping +ad ditions +mechan ism +z im +j ans +su f +cho pped +beg innings +vitam ins +ãħ¤ ãħ¤ +or th +po les +ru b +antarc tica +indie film +web cam +ket ch +bre tt +cle ment +her on +defe ating +hydr o +buc ket +wand ering +sid ney +future of +b inge +on ies +knock out +administr ator +syn the +l ent +jan i +bar ley +premier league +ner ds +cr m +bra s +bot any +evol ved +rot ter +ro wed +tum or +weal thy +Â Ń +mon arch +li shed +da hl +ðŁİ ĥ +bu ch +ken yan +Ø § +red ness +assemb led +se mit +hud der +shro p +ran i +lear ning +mor y +iti a +geo graphic +worl dof +f b +pho sp +boo gie +am ped +? ... +che w +dwar f +ar us +s sen +ru sty +recru its +h k +gar de +app lause +vol umes +invol ves +ta c +hand bag +trans late +ffe l +se ym +aqu atic +trans fer +zo di +and r +acade mia +cr ater +te z +ar se +adap t +col oni +snow man +mal i +hang in +di schar +oy sters +pho e +colon el +w ba +hispan ic +thri ving +sh y +ag les +sales force +cre me +so les +la fayette +â ī +ter ia +ach a +sp erson +go go +car ly +the ore +am ore +vo x +af t +ãĤ ¹ +stap le +mu ffin +di agram +ino x +su stained +av ent +me ta +arbit r +dec ay +ado le +Ð ½ +ec ol +ph o +n k +o cu +gr anny +ç a +luxemb our +stad t +alber to +le vit +am as +d x +or phan +co bb +as c +lo gy +immen se +chan ts +off line +p ent +bre x +w inger +plan e +i el +nichol s +ca thy +nar uto +low ed +/ // +ignor ance +cat astro +you ts +sch en +buil d +haz i +s ine +critical role +du g +dete ct +lo gs +en amel +stpatrick sday +ed die +co pa +cigare ttes +ho ff +kay a +la goon +ra pha +air borne +choo se +puer tor +ke v +gui ding +fro sty +bor ough +mir a +ðŁİ Ĭ +cade t +anu sh +yo gi +e ger +fl ing +slo pe +nin th +we ston +foot wear +f n +may weather +a am +pla in +stair case +witne sses +work outs +ro bust +dex ter +co hort +ðŁļ Ĺ +sp ell +ha ze +o om +organ ising +wild fire +cont acts +av on +min o +upd ating +ðŁį » +li thium +ing ual +k is +au ga +lo com +de duc +u da +th ak +boy le +mp er +hot tie +eri k +re vised +is la +travel photography +oo za +en qui +confe rences +clo ver +g room +cur ves +live on +per f +displac ed +bo log +xx xx +ðŁĺ© ðŁĺ© +te al +ve ssels +rain forest +cal ci +pan ther +gira ffe +ta sted +imag ery +pad res +day time +bas s +ri pe +opio id +nu e +vin yl +invent or +sen s +process or +mu t +gad gets +bibl ical +shann on +jacqu eline +car y +the resistance +ali en +n vi +co sy +bi har +fo ley +ren d +mu gs +fa ken +cl one +ni allo +gra bbed +chi hu +power house +n tt +chero kee +spon ge +imple menting +rh ine +le one +ðŁį Ģ +pret tiest +infra red +impro v +swit ched +tu bes +con tr +bl k +projec ted +be aver +yo t +bbcra dio +thi gh +per secu +apologi ze +w ack +po ster +oli ver +az a +lou d +( ?) +f the +women shi +spar row +blu sh +us able +sc ales +it ative +peu ge +ne eding +legg ings +glam orous +mat ur +c z +wat t +da b +tam ar +et sym +bau er +heart felt +h n +else where +bir ch +alu mini +hu ck +e me +j l +traf ford +d z +por tions +ana sta +arthr itis +esp n +ber gen +viol ation +yo shi +c z +northumber land +clo sures +ðŁĩ¯ ðŁĩ +smi ley +r w +tel ugu +inten si +gre gg +ve ga +dun geon +south bound +ba il +domin ican +semi final +chap ters +h itch +van ity +trans iti +recomm ends +sati sf +bar ca +queen s +( ( +de struc +stra it +ra vi +dess erts +in tru +har am +k os +fo e +fat ty +pais ley +magn itude +dri dge +com ey +schem es +vision ary +our t +down loaded +ðŁĻĮ ðŁı½ +gd pr +lan i +p wc +gu ad +nic est +stake holders +re ferred +george town +arvind kejriwal +schnei der +in doors +all star +strand ed +gen der +ze pp +ma sses +ðŁIJ ± +pati ently +bl dg +z ab +we arab +vi vid +he ck +d ella +sy mb +je opar +la ger +à ª +comb ines +ne c +br ay +flo p +tx wx +jo ys +pon t +pro found +sur round +mad hu +ma ble +ay r +te as +n sa +open ly +er nest +ãĥ © +to po +g na +anti oxid +ti an +e tr +c ello +ma thi +gener osity +b iting +man ic +kel sey +chee ks +ten der +w th +pron oun +ultimat ely +gu sta +ari anag +ger ry +ble ed +red dy +mic h +mitsubi shi +oper ated +sex ually +ma u +cl lr +vi ds +co c +mel ted +ðŁĮ Ī +q ld +ite ch +instru mental +end game +ðŁĵ ĸ +ener gi +brow nie +tam il +at in +domin ated +pra ises +fire place +sens ational +men a +k arti +un prece +ru pt +ori ental +mc cor +tour naments +scen ter +re eves +prescri ption +sam e +fra u +tru ffle +em bo +roman s +bla sts +techno logical +pr at +b sb +y ar +tren dy +ac l +al ad +ðŁį ģ +o hh +bankrup t +tho ven +regar ds +is er +war wick +vine yards +real m +niallo fficial +do ta +ge mini +to do +v able +¨ ¨ +la u +wre ath +ju ve +nat asha +le ver +lor i +hor ser +cc tv +air bnb +es anders +sin clair +ema biggest +high school +con test +optimi stic +t te +ðŁĴķ ðŁĴķ +ss d +ye e +hel ena +con sen +ric ks +jes se +an ic +ðŁİ ¯ +re acts +ro be +independ ence +vol tage +m ington +s ant +à¸Ļ ภ+-------- -------- +sentin el +ke tt +rehear sing +aaaa aaaa +sof the +stir ling +sear ch +wi gan +stand out +sna il +pent agon +Ä ģ +ch lor +cru st +net any +chemi st +disapp eared +ric ardo +sp iders +bo se +war ren +me ssing +bann ers +gu el +par ach +ma id +coun ted +epi le +bon fire +speech less +se tter +meas ured +rejec ts +nik ki +le ster +foren sic +fab rics +alo ha +pre served +wat ford +deta iling +dar th +bo u +car ly +... ' +tail gate +noti fications +å ¤ +pas sive +trous ers +balo ch +ro ther +typic ally +à ¥ +sp it +wi z +sic ily +technic ally +ex pose +st age +hu bb +cre am +cap s +po ke +sle ek +ju ne +tempor arily +de z +awak ens +l ame +_ - +ji ha +tues days +advis ed +advis ors +exi sted +dis agree +news room +lo sers +world tour +dr ying +al di +har ness +foot print +hobb it +p mln +i ro +que red +asse ss +gaz e +sa b +th ian +í Ĭ +ti f +ob serve +ev il +dra wer +swee p +cor y +co dy +kyo to +cal lum +n inj +lau rent +be i +sket ching +custom ized +du r +regre ts +knox ville +ìķ Ħ +mess aging +grac ie +abun dance +bi dding +bre wed +fl ouri +therapeu tic +alt itude +ho gs +bur ner +elec tro +wonder fully +he ater +post pon +li very +r all +ad as +a ac +sau l +brook lyn +play house +âĻ¥âĻ¥ âĻ¥ +char itable +in y +z ah +compet itions +be av +plu gged +o is +do om +astron om +speci alized +max i +ta ps +cellu lar +depre ssed +folklore thursday +cri b +e mul +ë° © +fi gh +ru z +car lisle +spe ar +side walk +de i +depend ent +lac es +nh s +ðŁĮ Ļ +reali zing +net work +ric he +re gin +re fresh +st ral +pa thology +pla id +psyched elic +hin d +u ka +algori thm +lin king +progre ssi +fe y +d ade +hydr ated +b ant +fam ed +cot sw +bo ise +as c +rac ing +ja vier +ww en +mar lins +poo p +swe pt +toni ghts +we f +ani me +slo vak +âŀĸ âŀĸ +cla us +lem me +cli ppers +re ls +arianag rande +r te +ko t +thal apathy +hungar ian +zu ma +y von +is u +jour neys +clin ics +be be +ww f +n ws +super heroes +er it +sle ague +identi fication +mo tto +ba i +sour ced +ill er +ap i +pri se +unprece dented +dam as +tuni sia +dra in +undere stim +e ther +quarter ly +rewar ding +al ham +wolver ine +cab ine +hyp no +nad ine +hav ana +da e +ðŁĵ Ī +dr on +read ings +b ati +pic o +mer ci +iti an +wal kers +el ope +mi key +god zilla +bur lington +abu ja +social ism +at ility +sh ell +harry potter +g no +ab ur +re leg +fel ici +ro gen +neuro science +inst in +ath am +vou chers +j arre +fu se +def ici +monte rey +de port +mid day +pp ard +fre ed +ame ter +wil t +n ingham +pr att +liber ty +slo gan +o to +pr i +co ated +c pd +ne tt +il las +mal awi +evol ve +accessi bility +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +or nament +b p +el is +son line +chi ro +fl ick +ib m +ar ak +en ables +gar land +san e +cu ties +tri p +rotter dam +n ys +lam ps +lu cas +bo g +ra ils +travel led +hic ks +en u +sab ha +scru b +hi er +hart ford +fo o +fer nandez +tre vor +mat tress +appo intments +ale j +fe i +o logist +saf ar +oc ta +sr c +sha un +ambi ent +dri c +bi ker +she e +must ache +h ta +bo one +her ty +car dio +bra kes +rec ital +consi sts +overwhel med +cau l +robb ins +im it +al th +ur l +bi bli +on ne +black livesmatter +diffic ulties +tel ang +tall er +ðŁĵ Ĩ +deb ating +bur rito +mo vember +strength ening +bo e +te stam +mirac les +base ball +re nee +ðŁijī ðŁı» +al fa +âĺ ĺ +unstopp able +ec s +g mo +giftide as +path way +fen cing +ðŁİ ¤ +b ham +ra s +sk o +d led +thel ast +magn um +bin ary +wil de +wil der +wh ati +barbe cue +h ism +can oe +kur di +eli ve +advant ages +mad ame +bi er +mis sing +enter tain +air force +y ama +c is +hash tags +j is +ve il +dream y +ten se +may ward +ch ateau +hunt ington +âļ ĵ +v all +up on +bl ouse +dun es +ðŁĺ ´ +fert ility +m ole +curren cies +st u +ber lin +toa sted +div as +wal t +lar k +por a +hit ter +um er +chil led +bal ancing +fa is +y in +or tiz +east enders +h ate +ur al +ap ril +tim el +à ± +per o +sto cked +respec ts +th t +best friends +giving tuesday +be ad +inv ent +im i +nap les +comb ining +tok ens +thir st +ma sc +par rot +sp u +dent on +* -* +t res +subur ban +wid th +si ve +con tender +siri us +lo k +troop ers +outra ge +tur bo +frag ile +me ssed +do h +disc ord +netany ahu +re sign +forgi veness +mo han +mun ch +cam ou +identi fying +enab ling +hot ter +thorn ton +jai pur +ar ya +ðŁı» âĢįâĻĢï¸ı +mu staf +maj ors +o ke +du ffy +roh ing +til t +ðŁĩ®ðŁĩ ³ +rock star +she ep +hend rix +ra v +in vention +do u +lagun a +gru mpy +sw is +im pe +) ' +you ths +bun ker +st ache +oppo se +indi es +acceler ate +ml p +ed en +w ann +k ail +akshay kumar +su pt +pol ym +midd leton +extra ordin +wil son +australi an +alumini um +way ne +alum nus +mat ics +gri m +er nie +opp a +competit ors +rand all +h ence +decla res +pre aching +sha he +can e +sustain able +stap les +le dge +ad ena +doctor al +bur gundy +decor ate +ren dered +ri sen +pr ank +di or +bee thoven +flo or +ac com +to t +ho dg +touri sm +say in +objec tive +mar kers +premi ership +en abled +camou fla +gi ant +Ñ ģ +smo key +ric ket +pan g +de pending +s ation +evol ving +inter cep +cen sus +tof the +re en +mendo za +trum pet +marke ters +an it +ðŁĻ Ĭ +north western +v la +foto gra +blackand white +che wan +wi g +tro om +ginger bread +k n +ro mero +n fc +or chi +fun ko +sour ce +f s +ra ped +o st +tar ot +ann ually +ðŁĺ ¬ +r ill +del av +.. !! +se s +can n +medic are +ph el +ape x +guardi an +rema ined +r pm +a ñ +story month +instag ood +neighb our +p ing +sem ite +my stic +as cot +mat er +hand ful +dang ers +ti d +ana heim +opol y +sh allow +nami bia +tor ia +procu rement +big bang +announ cements +prosecu tor +beng als +sal le +en roll +ga stro +sugge stion +ba k +ha ul +budd hism +berni esanders +flu te +fati gue +cyn thia +cho i +ir win +gu a +str ous +h p +ba p +satisf ying +play a +ðŁİ ¼ +inst ap +al ice +t p +irri gation +ðŁĩ¬ðŁĩ § +in tric +clu es +ple x +sa x +he pat +dump ed +signific ance +by u +medic ation +pro v +tough est +corn ish +âŀ ľ +kel ley +u v +si zz +si bling +me st +di stor +diplom atic +aun tie +b hat +son ic +bren da +pump kins +ro ch +black burn +ur ged +shi a +arrange ments +floo d +sa unders +lec turer +nou ri +popul ations +diplom acy +consist ently +ðŁ¤ Ļ +t mund +cauli flower +l ily +vocab ulary +vari eties +coo ker +up town +qu ent +mo sa +re inde +velo city +spru ce +social medi +i ber +volun tary +proce ssed +bal tic +y ang +leban ese +d p +dol ly +arrange ment +y uri +cran berry +kal yan +elev ation +cli ff +pu shes +ìĬ ¤ +sil ic +co wx +eter nity +sla ves +vine gar +glou cester +con tained +breaking news +aga inst +renov ated +norm andy +hero in +ys m +mo ds +gre ek +un di +tren ch +v h +encoura ges +head ache +gr ange +: ' +ever green +Ù Ĭ +reck on +ab used +th ru +cho ice +ti dy +col der +scho ice +ha in +bru m +li ars +bre it +yor ker +sh ack +he idi +micha els +sco pic +fasci st +play ful +ca c +yas ss +sh ad +.. ? +qu en +ram irez +clif ton +pr s +best fan +âģ ł +gener ating +head set +disappo intment +abstr act +bo iled +paren thood +azerbai jan +exhib iting +bom bay +oli vier +ko so +un lea +mat ernity +iz er +si ves +r hu +col l +saskat chewan +fre akin +de k +na g +stab ili +ðŁį ķ +organi zer +bo sses +ar u +u va +at able +ta un +after wards +fert ili +ver ge +az i +mor ph +๠ģภ+jer k +cosme tic +ko w +stru st +ap ache +post cards +for mul +ì ĭ +spin al +jack pot +elec tri +Ã Ń +lo y +gra der +diab lo +ar di +he sit +f w +arch ery +pa sh +the ories +repe al +re live +per cy +âĺ Ĩ +im in +syn chron +sham poo +coup ons +o to +la i +thou ght +luxembour g +mo v +ðŁĺ ¥ +ge mma +se ated +m ga +strat ford +un certainty +shi fts +est o +fo ol +fire arms +cor rie +ki ki +appa rent +p ills +olym pia +fi d +elev ated +de cks +ignor ing +av alan +ro v +whist le +p tsd +milit ants +robo tic +pac ers +quil t +bankrupt cy +lic h +per cussion +celebr ity +al s +( ; +su t +pokemon go +h g +off s +gibr altar +scre ams +billi e +gen ome +mar in +be ams +arch bishop +em in +bedro oms +g ated +ol ly +warran ty +at own +cudd les +gun na +k ic +vi ve +cy mru +nar row +pro b +le o +refe rences +manufac tured +cho pper +brun swick +sem is +don ia +r ye +man o +hur ting +? # +hol li +investig ations +c els +ðŁĵ ŀ +le ster +temp les +sto rey +mc mahon +toi lets +wo of +ï¸ İ +le verage +at om +night mares +victor ious +haun ting +custom er +ag i +yo ongi +mon ty +ver onica +w ur +inti mid +blan kets +volu tion +j m +âĺ İ +am on +jud ith +ðŁĺİ ðŁĺİ +distr acted +dri p +hurric ane +and es +revel ation +tro op +ab leg +col lin +tibet an +wor rying +inter nationally +eat er +camero on +brad or +y uk +ðŁĴĹ ðŁĴĹ +tra k +slo pes +ci er +ne a +ol er +ta ka +albi on +volcan ic +am n +a fi +ob stac +face time +ger ing +n pr +metall ica +organ ic +ðŁĴ ¡ +ki dd +d ances +pemb ro +wash er +m its +om er +emo tionally +tan go +ip o +do cks +scan ning +spec s +tho m +the ology +emer gen +om i +g pa +selec tions +un necessary +ima ge +ter s +induc ed +gi gan +rent als +supp lied +m fa +shan kar +lat er +pa jam +cla ve +Ù ģ +ma hin +carl son +avi an +ano va +kati e +aj ith +design ated +chocol ates +investig ators +gla zed +prin cess +er ry +ra gn +ou rable +hr u +sun dance +peuge ot +steam punk +gh lin +gre ase +hi res +z ap +per ce +j ill +tom e +he hehe +joy ful +mae stro +ni shed +gene alo +v ich +p its +fox es +good man +emer son +lo bes +con verse +o ats +thom son +ra him +mal ware +ah i +man kind +re sin +im g +sw ood +kin der +sc roll +ar a +sak ura +ro bbed +xi on +ny a +c ism +ce dar +be in +mour ning +tor to +heath row +done gal +bar b +hydr ation +k or +elim ination +su pdates +hill s +appe ti +star red +ko m +gw en +dd d +cra y +sc anner +personal ised +seren ity +re design +meta ph +box ed +judg ment +no se +ë ¹ +er ad +ac ne +supp liers +ener getic +v om +as ap +ðŁĶ ¸ +ir vine +hat ch +la ss +ad ren +waff les +accur ately +ici o +itt le +se un +occup y +web cam +thene w +ent es +ga i +j w +accoun table +vis or +ir rit +licen sing +hudder sfield +gen ie +ðŁİ ¾ +atmo spheric +ten sions +spart an +clif ford +ol an +north bound +ame en +cen sor +u el +ster y +$ $ +far rell +hy ster +cl t +se dan +rep lied +descri bing +micro wave +sla b +pro sp +assi sting +ru bio +e than +hh hhh +gu ay +z man +ra ise +roll ing +o e +n ile +ambro se +scar borough +hero ic +coo ks +mor t +chop ra +ðŁĮ · +to b +shav ing +stac ey +dor m +motor sports +wi ki +fol ds +sp iced +stress ful +liter al +fu dge +pe ggy +wa ite +tre sses +se sh +pr ic +ðŁİ ħ +fri ght +r va +mumb ai +po m +tt v +cel lar +tom e +andro id +dor is +tsun ami +tin der +o ec +m wc +dor tmund +no thin +l iti +so u +believe in +at u +kno cks +mag ni +ss sss +ro hit +ine ws +ang i +m andy +ke ttle +intermedi ate +av ant +cur l +endor sed +ori o +ur t +consider ation +wi res +shel ters +b ino +vik ram +imple mented +ly dia +bu k +paro dy +c news +under graduate +canu cks +sam i +polit ically +ro tten +gh z +tex tiles +over load +moder ni +recre ational +fli r +bat on +typo graphy +ov ation +intrigu ing +pilgri mage +al ge +ad ays +tcm party +sp elled +cur ls +boo ze +ste m +ann es +ir ls +spon ge +sho pper +sig nation +bra ss +mi stress +le ah +beg inner +lau derdale +augu st +pre school +ta ping +tai pei +execu tives +b d +rhe tor +esc or +immun o +deeplear ning +stat ues +it us +manu script +ly ric +cor vette +mol ly +la ge +de p +cn bc +le st +je ssi +fi fe +griff ith +oppo sing +ran g +dr ills +respec tful +p ity +d ell +har ding +play boy +blo ke +shut out +k ili +o sp +se attle +bc poli +mis es +journ als +team ing +es ther +fre ddy +Ķ ï¸ı +metr ics +no tre +gar ry +for ty +navi gate +perio ds +bened ic +j id +da w +ance stors +restor ing +con g +aller gy +tit anium +c ence +lean ing +ab bas +v ast +uc f +roof ing +e man +seve rely +vo gue +ve au +in bound +d z +tane ously +stret ching +man chester +dr yer +dav is +kan th +the game +it ted +re tain +el les +conge stion +frat ernity +ol lie +lo ki +fre ely +cho o +pon y +sc ep +tab ly +bal t +rock n +di me +lo gging +ðŁį · +ad u +ha voc +water ford +char is +swee tie +run ning +ner d +erdo gan +z ara +weigh ing +fif ty +pre cise +low ell +kurdi stan +r yo +or th +syn th +lin ers +phenomen on +art illery +il legally +constru ct +nostal gic +gar th +al ta +shel ton +a sean +w ander +dur ban +di versi +bon o +cl on +le man +sh un +obstac les +appet ite +fe eder +respir atory +di xie +formu la +an to +so ber +extin ct +au c +ing les +legitim ate +; ; +min nie +ipsw ich +dram atically +ðŁijı ðŁı¼ +ingh am +milit ary +mon et +us navy +for k +dun no +play er +q otd +st oo +ex or +ethiop ian +film fest +pe red +c ate +sau di +in ner +sin cere +tion ality +ale e +de eds +cooper ative +ir onic +cro cod +br ary +post season +cam per +can ary +e in +exten sions +nb d +sher wood +spo kane +hu mp +jit su +ê ¹ +dar yl +p si +stab bed +offer ings +expe cts +cav al +body building +fr aming +f ca +ye arly +bom bed +sk il +resear ching +jud iciary +gree ted +tu dor +mil o +innov ate +ðŁĺ Ľ +r hs +ru by +contribu tor +fam er +soci ally +m lin +fi ery +ut ter +beau t +it os +de voted +rain bow +bar ney +pe ren +ar jun +r na +gab by +ut i +hann ity +pick le +ser v +qu akes +pp e +fe m +wh itec +j n +victor ies +ðŁ§ ¡ +gol fer +congratul ates +resul ting +mechan ic +ur ve +cen tered +kie v +an s +in cub +< < +c mo +bestfan army +dap h +en ham +on cology +ku sh +t xt +ori ented +fashion able +c sr +sa hara +r ack +pd p +han son +ภĩ +ti ers +ra r +pan am +in sky +sa hi +testam ent +asth ma +in her +fisher ies +or der +ho we +gall on +ep is +suz anne +drow ning +paneli sts +ðŁĺ ² +ë ¦ +al ach +commemor ative +at tribu +ðŁij » +mo o +visi onal +week sary +gu st +ak in +poin te +ee e +di spar +ni pp +dent al +st all +pi an +bor e +ul ster +tic k +ir r +tae hyung +micro phone +bermu da +ga ard +el er +plumb ing +hu gely +âļ« ï¸ı +race way +cam bridge +mar cel +burn ley +to ast +holly wood +fa sting +me red +hib ition +ca pped +benef icial +ow ning +cont amin +arab ian +to on +cap ac +hul u +sm ir +nutri ents +se in +graph s +con ditional +ðŁij ħ +or ac +play in +nor the +tor nad +mar ian +ju mbo +lex i +incredible india +road to +uk one +confu sing +sp h +shan k +pi ed +mq m +positi vely +sher ry +path ways +consi ders +tof u +argu ments +resil ient +che tt +with dra +ter o +ated ly +sw ana +he b +fli ght +har ley +decre ase +kind le +book shop +³ ï¸ı +marty rs +sm ur +mc cl +concer to +sti me +rejo ice +app lau +cle ment +mer kel +jai me +im mortal +isle of +mar co +youtu ber +stal king +me too +st ack +sp ouse +u st +lu v +âļ¾ ï¸ı +eque strian +ev ing +fl in +nick name +the big +as ar +st acks +wal ker +bor a +kidnapp ed +hur ling +humb old +rec alls +co pper +ann is +se o +mer ger +mu ir +ad dy +ðŁĴª ðŁĴª +be x +cr acy +con an +congratul ation +mid st +âĻ ¬ +for bi +op tic +cr ate +crocod ile +mad agas +secur ing +ast on +o gue +savi or +salis bury +love it +fuji film +cast les +as st +ar rows +sp acious +tr s +poly vore +progre ssion +m ri +nel son +bi m +indic ator +o da +pe pe +re signation +gu t +sne aker +log ically +az y +are lla +te aring +jo shi +ssion ism +q pr +mari ah +p x +ble ed +mi an +med ley +we iss +ker ry +gat ory +at al +madi son +av enger +nab y +pl and +gi les +fresh water +d ington +ta j +demonstr ates +n tv +bul bs +sunday morning +pe ake +souven ir +wa h +ton nes +m kt +complex ity +con den +ross i +b ing +y ds +su k +n go +mid land +ol y +life is +ri pple +mo reno +dd ers +tu s +á ĥ +bou l +x a +hol dings +wn y +shadowhun ters +ke i +asp ire +m ous +ow en +so ak +skir ts +moun taine +stor ming +ch rome +ri ots +sar ato +amaz e +less ness +nav ar +crit eria +ra fa +indul ge +ay er +por to +nam o +........ ........ +yi elds +val le +j h +mac ron +sa ins +dur ant +tra ilers +wo t +confeder ate +sh rin +id ol +form ally +ten e +motor cycles +than g +no de +bang er +dal y +p ats +enroll ment +au ctions +at al +ar bor +lo gos +de arest +trans action +dom ingo +fle a +ser mon +de ck +sin cere +questi oning +juli o +was p +pre tz +armen ian +k ham +inflam mation +picture sque +acci dental +film makers +ðŁĺ ļ +ðŁĴ į +ca sey +so b +yee zy +good will +parag ra +ss ly +fe ather +dy ed +assassin ation +na de +b cs +app lies +femin ine +fe u +ext ent +depu ties +l ack +psy chic +go i +kill ings +pse u +ðŁ¤ ª +un c +mar l +tan e +mck enna +sur fer +influ ences +free way +hack ney +mal aria +el and +te au +rema stered +Ø ± +raz or +gg y +cor ro +lak sh +fla ir +honest y +hoor ay +de pp +am c +wedne sdays +q a +ed its +- $ +se villa +dou bled +human ities +c cot +som os +r ine +af a +si oux +re construction +wel ding +th reads +am ish +encoura gement +po der +bo ck +bal m +p tions +stand up +accompli shments +guar ding +convic tion +ac ion +napo leon +depic ting +att ack +su i +wear able +âĸª ï¸ı +pot ter +esc ort +vis e +to ts +bo on +event profs +angu lar +womenshi storymonth +bar row +sch i +ac comp +ti k +l end +kensing ton +wol fe +st acked +cra shing +exhi bit +wing ed +sab rina +ma sa +k ms +alway s +et t +pla sma +counsel ing +pick les +nfl draft +mr s +inev itable +coura geous +staf ford +writers life +ho s +e j +gh yun +trade mark +adri an +influen cer +coron ation +ra ging +explo red +usa f +excep tion +eu x +tan ker +sw ami +pac ket +ðŁij¨ âĢį +f en +she en +a ero +j l +re gal +nw t +au ster +meh ta +char ge +a ste +b ate +inf eld +racec ourse +collap sed +fle ece +z il +al lie +alternati ves +geor ges +ðŁĵ į +quir ky +fc b +nat geo +philanthro py +bra i +every day +ðŁIJ ° +ach ers +ja an +fin es +q i +fisher man +distin ct +gri mes +nation alist +comm ence +ro wn +âĢ ³ +z ing +f ter +hr w +baro que +bl ender +kitt y +hoo ks +c ited +w anda +consen sus +reinde er +an and +supp ly +me ds +v n +ol ph +rat chet +shel don +secur ities +ë°© íĥ +cro m +mosqu ito +j eric +im mac +dimen sions +â ¤ +di ssi +sponge bob +dami en +steven son +jo anne +del ish +yi kes +than x +surve ys +postpon ed +alco holic +al ised +ðŁĻı ðŁı» +do ch +sen tim +mered ith +com pares +b ago +happy days +mo ss +ãħ ĭ +ne c +gn ment +frustr ated +comb in +ri v +ec lec +col lo +compli ment +actor slife +ct to +nic ar +op hon +apar the +man t +ja de +trol ley +optimi zation +eye on +eco logical +qui st +ep he +ॠĩ +cin co +appo ints +old school +c pr +behavi oral +min aj +:- ( +tag ging +ev al +jo aqu +ðŁĺ « +ha k +de me +jama ican +so s +hy att +hand book +libr arian +hanni bal +pump ing +ch om +f man +ga i +hu ll +respon ders +green ville +n us +vau gh +ðŁİī ðŁİī +ta xi +gold berg +man tra +te ase +forbi dden +metho dist +ati vity +* *** +ec t +mc gr +Ħ ëĭ +se b +amid st +disapp ear +thy ro +phili ps +er ina +v icious +stream er +million aire +ma p +str ick +hack athon +gh a +ed ic +mi ka +pe ck +ill i +anto ine +ar ca +op tic +ma ure +ðŁĩ¦ ðŁĩº +cla shes +man ly +âĺ ģ +al var +and res +me i +el m +ww ww +al tered +l te +ê¹ Ģ +mo jo +for rest +thal ai +non t +spee ches +acknow ledge +ign ite +x factor +ðŁ¥ Ĥ +mead ow +disru pt +debu ted +scrim mage +pharmaceu tical +fi dd +found ations +philosop her +et al +publi shers +bo ys +c ke +ru gged +opti mism +re be +phil harmon +nar cis +ral lies +lu is +go blue +fol ded +un acceptable +optim al +li sa +pol aro ++ . +en za +âĿ £ï¸ı +mon opoly +grace ful +dair y +du a +diffic ulty +judge ment +o si +mer sey +flu x +new found +ter ns +dimen sional +in vic +al ba +am it +abudha bi +alger ia +autom obile +the ad +lo tion +acceler ator +vac ant +iti on +lu f +al ic +pl l +bla zing +ba z +sen e +ðŁij ¼ +villa ins +direc tory +eis en +to ck +broch ure +ri pp +hb d +zayn malik +nic he +lo lol +certific ates +mor se +fac up +x ham +un wanted +im ports +carne gie +fan sign +mo u +r alph +destroy er +sw ing +trek king +cili ation +pit bull +g aps +ho well +defin itive +mc le +f ps +et z +bol ly +lyn n +gan o +at ure +fur suit +co il +na v +but ts +tro jans +eu re +en ko +sch umer +horri fic +install ment +br b +subur bs +a bel +vi r +de sh +cun ningham +ðŁIJ » +span n +sch we +ke mp +tr u +ste alth +qu es +le w +deli ghts +ko ch +hu mili +cr iti +il t +sp ells +mi ley +car ic +ðŁį ´ +lc fc +substitu te +oun g +? !! +af fir +predic table +class of +er r +cy press +chand ra +age ing +__ __ +ther land +don caster +el in +yo shi +sail ors +har ris +jo anna +niger ians +h ers +pla gue +pro cra +k no +can ton +busine s +un h +pra kash +c in +bow en +co ating +m als +be gging +smith son +ponti ac +sp ies +dam ian +pl ine +und ant +al ta +one ss +shame less +da q +bb m +wal es +stam pede +ser um +Ù Ĩ +cataly st +x n +ab sc +free zer +ch un +ari os +mc cre +fore head +he ars +damas cus +tac oma +ardu ino +encoun ters +stan ton +lg b +ab as +" .. +ke te +drac ula +ele m +g ne +zepp elin +la brador +pul p +op tional +or n +russi ans +san itation +hil ary +etsym ntt +pen alties +au st +ig ans +olympi an +medic aid +vers ace +va pe +re stra +pe ep +sexi est +st alls +di le +the a +punjab i +pupp y +tuesday motivation +ðŁĵ ļ +the flash +roc ket +mo dest +chihu ahu +on na +k sa +hur dles +ca ve +fail ures +sp lit +bo ho +gur l +disappo int +ho ward +nug get +fran z +stal ert +kaz akh +for getting +sch ri +ag ate +am at +eve rett +du et +veter inary +juli an +ch ills +bra ve +ghost busters +lan do +gre ets +profit able +d é +ti r +ze e +om en +pd x +gray son +har i +fix es +stab bing +swim mer +symb ols +compli ments +po se +func tioning +th nx +gi r +corpor ations +bar low +lo e +off season +distin ctive +marvel ous +nik on +enri que +ky u +ja ws +amo to +lom bar +travel blogger +fa h +ouri sm +tri stan +so e +ce ase +ðŁı ħ +z ac +mck enzie +taxpay ers +swim suit +bl o +les ley +kan sas +w ks +ki el +provo king +my les +str ing +kangar oo +galac tic +fif th +s ke +we ir +ll is +mat ory +ðŁĩ ¿ +un ci +re productive +roo ting +ti des +gad get +.... ...... +alex ander +bow ler +scre w +apo log +eri ka +wal ters +shet ty +lan e +ban ter +as ant +me so +v ain +" "" +us i +fer din +accomp lish +man sfield +bom bar +collabor ating +cla p +it ure +s da +smo ky +na k +im person +car la +com ra +bur gl +lo co +ti es +in hi +trac ey +se is +diss er +rr rr +dra y +prote ct +cor ona +hun ger +ck en +c eli +trou bled +predat ors +fic tional +shav ed +riche st +metab oli +ful ham +gro oming +mono chrome +wa sting +as co +ast e +ti sta +remedi es +ung soo +south end +perman ently +bu mble +procra stin +ident ical +practic ally +ma scul +su ke +assu red +val erie +devi ant +grizz lies +thi er +pur a +ne pal +not ts +bil ateral +spo il +car mel +cine matic +ph l +ni fty +ma o +hypo cri +la ser +pan try +mathemat ical +el isa +coordin ation +bel mont +a it +radi ant +bo iler +man g +f ag +cr c +h ams +br in +â¬ĩ ï¸ı +famil ia +âĿ £ +sab er +ru pert +gg an +rit z +mic h +sal ford +le vi +gra l +ðŁĴ ¤ +n ino +ce d +business man +ul tr +sim ply +compre ssion +pa ins +hal t +ë°©íĥ Ħ +landsc aping +n f +croo ked +er d +itt in +ddle ston +sur passed +ino a +da g +bl en +exten ding +at ing +al gae +ball er +u mar +snoo ker +col lu +flo wn +thu b +ridic ulously +ki sh +op le +di re +as ser +ari sto +sc iss +h ating +trou ble +syl via +suc cul +plo ts +sincere ly +al er +laure ate +br ack +att n +rif les +me to +collec tible +cu omo +conte stant +consist ency +ant z +rang es +abig ail +de b +mini ster +grow ers +an oo +hoo ver +dream er +nu cle +resear ch +mi y +sha hid +ma v +d honi +cin i +do j +hin dus +part ying +dal i +alon so +inform al +clark son +it ton +ki an +cit yo +mor i +la sted +as pen +libr ary +susp ici +qu at +den ial +fol der +ch ori +swee ping +eni x +ðŁį Ĥ +Ø Ń +nas car +handmade hour +mou l +heat wave +em er +exam ine +ib n +gr ind +po v +tion ist +m bo +she ila +integr ate +om es +take away +cer v +con nie +tic ket +ce led +bi en +visu ally +madagas car +sor ry +gu i +park run +tra its +la be +pois oning +ॠĢ +vi able +bohemi an +denti stry +bad os +spr outs +mask ed +te ddy +ðŁĺ · +sa f +sa as +ji ang +ti ght +spe aker +withdra wal +bc n +as signed +class rooms +fle ming +ðŁĴ « +super girl +tot als +table top +e books +horizon tal +cra z +flu sh +j ard +c dc +er son +ãħ ł +green wood +ni h +co x +ad a +lit re +go ing +v icky +cur ved +lou ie +gra ins +hy e +lon ge +reme dy +tra inee +san jay +super stars +ma ser +man u +s age +wh l +ðŁĺĤ ðŁĺŃ +ðŁijį ðŁı» +m sd +en z +rab hu +j oo +gh u +ac er +e po +resurrec tion +justice for +bl ended +mo da +avalan che +france sco +re spective +g s +ye ast +wel ch +devo tion +ge tin +athe ism +am ic +carol yn +lo c +ld nont +ave c +us da +le gged +bra very +b lower +cow boy +he h +sti ble +buff al +chann el +run chat +âĺķ ï¸ı +ide ology +best seller +y oo +pe anu +bon ne +fel ic +edi son +fr actu +naren dra +pp ets +seym our +ri viera +he ctor +necess arily +bi anca +soci eties +the best +w g +sent ences +win k +vacc ines +pal ooza +jam ming +as f +mp us +agre ements +ec k +ba c +hon ore +com pul +wild cat +im posed +yo ga +hud son +can celed +l ich +fu zzy +es que +ch uk +w vu +se k +fli pping +r hon +wi shed +wh a +cap ability +len ovo +ìĨĮëħ Ħëĭ +vi vo +tv d +nor a +sil k +pas adena +yo semite +valu ation +clo cks +u ber +mr c +dar kest +au bre +ss o +bell y +wrest lers +kill in +lou der +buck ley +ge el +ad on +un s +appe aling +ðŁij ¯ +semit ism +list ens +fit z +ãĥ³ ãĥ +ny lon +ar ty +seem ingly +hal a +su ited +et y +she ds +mu ffins +ap ric +um ents +u ta +jam mu +chelse afc +star z +yo ko +roo t +clean sing +di ar +pione ering +ihear tradio +dig iti +fin dyour +can o +ðŁĴ İ +z ol +spac ecraft +six ers +moi sturi +b ile +ti sts +hor ton +rang ing +colum bi +mete oro +senti ment +ep l +foo th +text book +drain age +r ly +sc ue +imran khan +ðŁĴ ¸ +margar ita +ed dy +predic ts +gamer gate +advis e +growth hacking +love you +ug and +v f +beng hazi +s later +ne wor +ch el +independence day +p np +cul len +hoo dies +num bered +brit t +t sa +kl tu +s ages +mom o +onep lus +col l +gu ts +w ta +mesm eri +enh ancing +chiro prac +j is +teen agers +m one +constell ation +sweep stakes +e ze +slovak ia +la ye +pear ce +wa ver +po gba +k ron +sur geons +mar x +ti d +gg a +desc end +p ours +upri sing +wal la +sab bath +bachel ore +mack in +k am +peter borough +hor a +ðŁĮŁ ðŁĮŁ +think big +r j +hy drau +sp al +univers it +ðŁı ī +mail online +league of +ten ants +w ally +lan ce +heav ens +dd r +bol ts +am ir +i phone +ci gar +en du +re i +el abor +r inging +john son +characteri stics +sal oon +algori thms +tal kin +m tn +di ve +region als +ff ice +hat i +deviant art +so tto +shir o +l ama +k we +f aded +por ting +tu mmy +est ates +buen os +ðŁ¦ ģ +beli ever +pen etr +dar n +sp ite +can opy +fashi oni +t illa +pet als +eli jah +bra wl +marty r +ë°©íĥĦ ìĨĮëħĦëĭ +mid town +eric h +d apper +sm town +me gam +ww w +le le +on s +cat fish +fir th +fossil friday +ball park +th aw +pot ent +illi e +cre ep +car p +so ap +gun dam +infe c +yy yyy +ठ¨ +z ag +rit t +calcu lator +bo ca +ok o +to ad +threat en +refin ed +olym pic +accompli shment +bacter ial +a ji +tat um +feli z +she ed +j at +th ic +jam al +ðĿ ĺ +lin a +ðŁIJ ¯ +jo king +yot po +pin ch +ak ron +her b +motiv ation +li a +ho stage +cre ek +gam ble +russ ell +patt i +fo tos +c pc +bro ken +back the +cla ys +u mm +stock ton +mat ernal +ü r +la kel +cent ury +be k +infe cted +ภ¡ +smack down +man ned +ta hoe +sm es +bas a +su la +augu sta +. * +rohing ya +gre ed +counsel or +silhou ette +gra vit +cla use +' - +bo bc +occa sions +now adays +dic tat +be ard +n ally +brigh test +kab ul +inc india +dhan ush +archae ological +che ape +mizz ou +d hi +ov ski +bax ter +asse mble +à ¢ +gi gi +ac am +wis ely +haz ard +north ampton +âľĪ ï¸ı +me th +bla sting +re unite +mu lus +ali zes +t read +mil a +ed ward +ko va +pe sto +ðŁij ¶ +vit z +hydrau lic +refurbi shed +mo tel +isab ella +hom me +sever ance +uph ol +mis erable +f ari +lat ter +ef er +crack ers +es l +ac io +yy j +in an +ec b +z ind +pan as +tru cking +re ed +sh aker +burge ss +em pire +ag nes +n ington +art works +fr s +ti le +bi ome +eu n +ch ong +americ ana +god father +go blin +i shi +! ). +temp ted +gen omics +mand ate +ck y +ðŁĴĻ ðŁĴĽ +som ali +br andy +in ven +spoke sperson +pc b +yu an +h g +fa z +starwar s +ro wan +blue grass +don g +d day +trin idad +er ton +ban ning +re tention +cu red +tober fest +re set +we is +deta ched +behindthe scenes +immun ity +ph a +bra y +ðŁij ½ +ran cho +ram say +est onia +nd tv +] . +cab aret +tar o +d v +show cases +plu m +ðŁij ¸ +son oma +pre pa +memor ab +e stu +drive way +u les +magn us +x r +nn n +much as +en ge +stre amed +fore stry +audio book +tro y +reck less +kil om +ru ler +ra k +proce ssion +i ons +po ole +noc tur +wh s +farm house +per a +par me +hypocri sy +s ics +v ant +cas k +holi stic +au st +Ð ¿ +in do +ðŁij© âĢį +di so +disp atch +ol sen +make it +en nis +cent re +ar range +ðŁĮ ¼ +sal ted +ea siest +f ate +reg atta +mo zz +ac an +sin i +g ically +ch ops +chick en +work in +ha gg +invol ve +wee ds +book day +wake up +ky r +michel in +fu ss +re juven +vac ancies +incar cer +m st +sc ents +sovere ign +kick er +à § +bo d +âĢĶ > +sa h +mob il +shrop shire +oph one +dress er +mis suni +hep burn +i mo +foli age +diagno stic +as san +cycl ing +guil t +c sa +puertor ico +win elover +wake field +do ggy +k he +pa pp +co g +al lot +cu ck +poe tic +mi o +re vit +mag ician +ç ¥ +ant enna +west wood +mber g +lux e +oat meal +Ø ¬ +te at +ffe e +sear ches +l ly +plu to +el on +let tering +inno cence +fa i +ann on +telang ana +ma it +neu ral +can ni +ar oma +a stor +fe x +co cac +mon etary +f ent +un sure +' @ +indi rec +teh ran +isol ation +li bs +make up +merce des +ff y +he tero +de o +sco m +cur sed +veteran sday +franken stein +shre ws +de co +ge ese +lefto ver +ha did +vari able +acade mics +carol in +under going +vari ation +na h +ssi er +gamer sunite +pur suing +emer ged +ll ers +control ling +ro aring +mete or +vol t +daw gs +be aver +is life +bathro oms +aci onal +pre vent +lake district +in als +y ani +gra bbing +sac ks +le z +sw ay +k ool +time s +klo pp +la de +con cord +resul ted +revi ve +recon ciliation +ol and +az z +gir o +mand arin +de en +nutriti onal +is coming +van i +aw www +der ived +love your +stop the +shou ting +nov ak +ðŁĻĮ ðŁı¾ +lo af +displa ying +sunday with +ma guire +ch eri +ðŁı Ł +re match +qu ic +Ú © +y in +ðŁĺ ¹ +ili ve +z ip +our ke +down loads +sw at +missi ss +care rs +t ment +proper ty +hahahaha haha +gi bbs +sur rey +ar ise +tic ism +sti a +ir ling +fro g +co se +bas sist +fore ig +lea u +pil lows +hol la +eli e +disclo sure +peanu ts +inte ch +ww c +plun ge +trium ph +cor i +sli ppers +ðŁĻı ðŁĻı +neutr ality +ma re +hair y +gang ster +hu mming +cust ard +mer lin +ale a +s by +dam p +mo han +ver bal +j st +gu tted +b jor +un finished +ðŁĩ¯ðŁĩ µ +un happy +âļ« ï¸ı +by pass +at su +fis cher +sa v +afric ans +re use +mid way +demo lished +ger rard +her cules +Ä Ł +medic ines +cl icking +sur round +jo ong +wav ing +tri bes +wet lands +offici el +argu ing +l le +do va +su zy +club house +ne gro +ob tain +ga o +gl ance +assi st +ch os +ãĤ ¢ +âĺ ķ +adri d +occur s +st ans +par don +livel i +emplo yed +re visit +ff xiv +bb le +ne aring +min er +ðŁĺ ¹ +giov anni +up to +mar vell +mar se +to wels +cb n +engine ered +y elling +spart an +si ans +ðŁĻĮ ðŁı¼ +se v +coyo te +sta di +t cm +app en +shenan igans +open access +so aked +ma squ +le vine +stro kes +l k +aparthe id +hipho p +char don +may may +ha asan +stri pped +fr o +scri ption +f ton +h f +pri sons +marsh al +ķ ãĤ +an cho +com promise +classi fication +buzz feed +bblo ggers +deser ving +) / +s way +ob o +camp ers +poder nfamily +p oured +bri e +squir rels +se ize +: # +le k +ti mb +st acy +nas daq +repe atedly +br at +mi ghty +competit or +mah one +de si +o ke +bm w +shi e +f cb +cheape st +minim alist +par amount +n ate +har as +insan ity +lat eral +ment ality +mo zam +ta pped +yad av +u sp +b way +the od +bil t +ra ids +em press +adap ted +pat ron +nut shell +ag ra +be aded +sundaywith marsha +vi king +proce ed +main tained +thinkbig sundaywithmarsha +sn es +mus ica +to wer +ch ab +bo k +sm t +insul t +harve sting +windo w +ru ther +be ige +dec al +indic ate +ma iling +ri ft +po le +ander son +ch oral +sp ride +l ili +ev elyn +imrankhan pti +.... " +ke red +un dp +water falls +se ars +le mans +world series +ri el +ani e +app ar +score rs +lam p +a than +phys icians +qu inoa +refu sing +vu itton +unle ash +s la +pat i +shou ts +inten tions +fo amed +europe an +neighbor hoods +me er +man son +du h +br at +con es +bow l +kazakh stan +ठ¿ +in appropriate +del hi +ketch up +ful ton +s ys +consul t +gar field +to go +f ml +f led +b ds +facilit ate +ree bok +selfi e +elev ate +activ ate +bi ble +ca wx +b ys +cam ille +sy ou +sk ool +her t +w bc +ple dges +recor der +po sh +ac re +so aking +mat il +v sco +shoot ings +pla r +e con +ðŁĻĮ ðŁı» +rashi d +u bi +ðŁ¤ ¤ +sw inging +wi pe +rap tor +m su +music video +dur ham +at tic +apar ty +fe tus +activ ation +aa z +motiv ate +ðŁĴķ ðŁĴķðŁĴķ +j al +ठ® +ag on +sche er +stal ker +fo ster +az zo +tele gram +vi gor +s laugh +screen shots +entrepre neu +kri stin +inten tion +ch illi +fr action +don a +ge a +tc u +s ite +la k +em il +d nt +bor o +wil kinson +re cu +ato day +t anya +bl anco +cd n +brilli antly +g cc +ac c +evacu ated +ther ine +den ny +cait lin +she pard +pou ch +hand held +sou theastern +ha a +à ´ +re solutions +led ger +sr in +r ar +shat tered +chim ney +im with +mete or +hand led +ra ke +town send +en han +shi py +duc t +tw x +inflam matory +war hammer +theat rical +gro s +sk ar +sco tty +ni el +tit o +tin i +conne ction +_ . +goldeng lobes +sha q +ðŁı ³ï¸ı +hall way +fron ts +effec tiveness +gla ston +d hs +ex pi +to h +c pl +sc s +re o +ha g +resemb lance +hor an +abu sive +qu er +virtu e +cho lester +a q +shan e +m ce +carri ers +di stress +re wind + ¡ +voo doo +int act +ann o +ðŁĺ ¤ +pi led +adi a +ãĥ ³ +en ow +di gs +light ly +goo fy +turb ine +governor s +con te +re open +pa h +i ve +cra fting +swee ps +jo di +an de +zu cker +kaw aii +o ko +v ai +out line +kri sti +ts n +insp o +qu int +fil thy +lyn ne +listen ers +depar ting +or d +t weed +, & +ale k +sel fish +nor ther +recogni zes +i ps +be s +a ed +w ills +pe at +surround ings +mon uments +ais le +be cker +la v +quant ity +v ah +helicop ters +tu cked +alv arez +sha pe +o bey +ad diti +road side +m ite +bl ers +ep age +j au +ignor ant +b ins +lu lu +x o +c fo +ee eee +apprentice ship +shef fiel +to i +ho k +faken ews +deplo y +aid an +husk ers +ãĢ İ +west brook +mi ster +confi gur +car r +fic a +proceed ings +ha w +ste ak +mur derer +pay day +a jo +p vc +don ates +bi af +nom nom +be it +k ali +x rp +ahmed abad +se mic +che y +x tra +an twer +head lining +squ ares +roun ded +flu ore +bol d +disa sters +am oo +gener ic +cran es +brief ly +gi g +auster ity +anticip ation +for ti +treas urer +cann y +ce cil +dete cted +check list +ภ§ +pam ela +bar bados +an field +hear ty +tx lege +peren ni +arro g +ing ram +âĹ ı +ty ne +spo on +r ation +am ba +m be +cam el +h hs +york shire +reflec tive +fre aks +to k +ju do +partic les +du bs +ban jo +accred itation +prover bs +over dose +inte gral +gu ang +mc s +super car +af b +al vin +ail s +x tre +st aging +tw ent +rabb its +mar o +inste m +dol l +cr ay +sant ana +ble ach +mini ons +che ap +man t +di vers +catal onia +lo is +mat ri +cou gar +kay ak +e gre +p so +a ia +å ® +char lton +tr acked +sc ari +pe tt +f wd +x in +gra vel +br ic +bigg boss +ar den +hu gging +pal ms +st v +li mb +the movie +handic ap +ri me +z ai +stu b +indi a +lithu ania +rhy th +p ita +maced onia +high ered +brid get +schwar z +ske let +hi kes +ant arctic +c ps +mash up +Ð ° +n ell +chand ra +he ir +an us +sher idan +mi mi +muse u +bec ca +an ir +bar rie +dioce se +compar able +ðŁı³ï¸ı âĢį +yuk on +me p +hor mon +mer ic +al f +con quered +christ church +ðŁĴĻ ðŁĴĻ +hazard ous +poo h +cont ing +retro spective +par ame +na ir +con sor +ho tra +astoni shing +cater pillar +u man +ti sm +t vs +serv ic +croy don +mor ales +c g +cu m +te ur +scan ada +s all +magno lia +el ise +th our +à® ¿ +ag omez +phel ps +ë°©íĥĦìĨĮëħĦëĭ ¨ +wh os +weav ing +si sd +pro poses +cro ws +pre sale +econom ies +bernar do +sha hid +air show +mc cann +hor ticul +nr l +du el +mongo lia +tou lou +requi rement +struc tured +ed i +o lives +he a +cu ter +Ð º +enthusi ast +harri et +domin ion +sub mer +ðŁį ĥ +sa ab +nes burg +mo ff +def ended +bur t +rewar ded +gold man +op tics +khali d +house holds +buc kets +ce cil +che ss +substan tial +ef l +oper ation +evalu ate +st n +rece ssion +l ll +tom as +tru ths +ak bar +s words +p act +embarra ss +ha o +ay urve +scrip ture +ny cc +op t +di ameter +sc ented +organi zers +re lat +ha e +dream ers +de se +ðŁĮ » +restric ted +n ale +r hp +dol an +mun ster +ha ired +consult ants +jo ints +hu mil +d ill +relent less +t é +af il +ut ilities +japan ese +condem n +pet ite +colli de +q f +peach es +cou rier +l ore +âĺİ ï¸ı +reli ability +ch uk +ðŁĻ ĥ +stu res +ge ther +ho stel +bi er +- _- +â ĩ +e ze +ta ilo +di ent +blu ff +chu ffed +pil ip +mon arch +e em +bu chan +b ick +op au +ku ps +ภ¢ +pist ons +sp ins +m and +ce st +bur ne +v ile +cher ries +bec kett +need les +pan ch +ë Ĥ +haha h +trou bles +insi sts +do you +g mc +mor tar +deleg ate +in n +g anda +sin atra +ठ¤ +spee ding +pu pil +pre mises +ali gnment +pi kach +as us +j alan +Ø µ +lime stone +fol kl +parme san +ce il +mo y +shawn mendes +ac up +hu st +ot es +med ina +ma di +gta v +censor ship +ar g +swe eney +sy kes +col o +foot steps +cann ed +adv ance +gta online +healthy living +ðŁį ¾ +a ig +p ality +oc s +he brew +im minent +berk shire +jeremi ah +out going +bak er +entr ata +ma ids +gro ves +bo c +a del +m fw +con science +arm ys +nut ella +conte stalert +novel ist +la h +ban ker +marque z +ðŁı ¡ +to ff +out age +gr p +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +musc le +du dley +nvi dia +mi di +m uni +ess ays +dat ac +car ter +ภ£ +t ans +i ves +public ations +al er +ok wx +il u +cu tt +har p +out law +luther an +br ill +bo lic +do well +green land +be sties +path i +pay ton +gue st +har den +ðŁ¤ © +ann ed +evacu ation +po ised +mc der +b han +o i +envel ope +ci d +ca vi +ta pas +book review +grey hound +âĻ ª +fe ud +lun gs +for te +rai der +ff er +oni x +dep end +yn wa +rel ating +de vs +ðŁĴ IJ +acqui res +d ha +j yo +priv ati +can ine +k b +cra b +sar din +imag ining +k j +em por +down hill +ne z +ta eyeon +nick imin +gb p +à µ +w ap +sec co +ma shed +ðŁĴ¥ ðŁĴ¥ +augu stine +diss ol +dic tator +â ĵ +vi per +ed fringe +vau x +hard work +book let +no x +chi ff +ðŁĴ ¨ +observ ations +xbox one +u sher +ke er +lu p +dal las +cal gary +ma dra +di ous +k bs +wood ward +hero ine +lu mber +sea world +o ws +mc ke +maver ick +gu la +cross roads +fan g +s ade +nik ol +chee tah +me c +pp g +er ick +ðŁİ µ +tox ic +bj j +viol a +sp ire +ch ino +tra vis +institu tional +ha as +low ry +w ac +ea e +hu mid +mp ton +ru ck +je w +c ine +zim mer +se f +bhar at +fre es +aam ir +ðŁĴ ħ +z inc +wan e +multi player +royal wedding +e el +preci pit +qu ery +kimber ly +isa bel +ful fill +ig an +vau l +pan e +sc y +dig it +gun n +u tah +dog day +fi on +xia omi +da c +el ast +cha vez +ro blo +g ine +ten th +ab h +ke to +hur dle +na dia +memorab ilia +ha bs +qu an +h w +hv ac +pix ar +ec cle +kram er +accu ses +ðŁĴļ ðŁĴļ +per se +mean time +wa hl +atle tico +âĢ¢âĢ¢ âĢ¢âĢ¢ +ott oman +no vo +k us +conne cted +tru sts +d mv +spen cer +rahu lg +do ve +sto kes +bolog na +enthusi asts +à ª +rockstar games +ted cruz +du ras +s acked +late x +immer sive +cer t +lu cin +princi pals +fa res +sa ils +far n +am ent +saf fron +quent in +check point +fer ris +ex cur +ðŁijī ðŁı¼ +bai ley +se h +ter re +mad am +s band +wan derers +cumber batch +yy c +digit ally +blackandwhite photography +roll in +moroc can +ðŁĮ ħ +din ner +d well +to om +m ye +ez ra +cp fc +war hol +me er +jon ah +no aa +s gate +so on +secu lar +g ating +ti o +dri ver +si ssy +assan ge +ta th +ed mund +bobc ats +ra ji +po stage +stu ds +m gm +kat o +edin burgh +meet the +shir t +fa a +mens fashion +sp reads +wi m +car ts +phoe be +j ars +bot swana +Ù Ĥ +ed war +sk ar +ri ve +gu sty +c tv +ferdin and +su therland +nickimin aj +k v +si us +bee ch +re z +desi res +on ial +camp o +quar ry +lor raine +gil more +ig gy +µ ï¸ı +ho pping +avi z +ðŁĮ º +uni sex +dedic ate +att itudes +ste er +jun kie +rail way +y b +whi sper +key an +k us +ju g +di x +a ins +sum mon +ov ich +sy ed +her ald +ma ison +me ded +wild flower +main land +ri sky +ru kh +over looked +ki c +destro ys +nam an +ki p +z ano +champion sleague +ban dit +quin cy +smi le +cal vin +open ings +ta pp +ol ulu +spec tro +accred ited +ap k +pra ised +bar nett +pol len +premi ered +selen agomez +tou red +screen ings +uu u +mis o +en se +adam lambert +guel ph +har yana +hu tto +le ar +l tc +po ached +brex it +æ Ŀ +tt c +pa vement +mon gers +ro e +ad ers +ling ton +particip ant +ca red +ga il +y ates +lan tic +dash board +jo o +feli pe +ssi onist +bu m +s end +a eri +thu gs +luci fer +a he +dete ctor +fil ly +gas oline +ham per +hump day +the ta +the band +fore casts +o hhh +lo bb +hol l +cp u +az u +ad ar +hai ley +bu b +car t +quo ted +an archy +pan cre +twit art +al den +st ash +the less +or ni +belie bers +mor mon +partic le +avi ation +⬠Ĩ +webcam toy +sad dened +cru is +ham let +n ct +roll ins +marque e +saw yer +reli ance +a ura +di ec +soo thing +sig nings +ak is +à ³ +at kins +aer op +ðŁĮ ¿ +y ab +sh ari +con nol +du bbed +manufac ture +convin cing +feelthe bern +ra u +pu lit +on ec +gem stone +ur ging +bag u +ga h +aci ds +fi anc +zodi ac +sn oop +her rera +initi ated +ven ge +profess ors +pro di +stron ger +e mission +bb a +hal le +ta pp +haw an +wh im +compe ted +myr tle +ir port +cold play +ach e +ske p +m son +ss ic +calli graphy +swim mers +me y +pp c +thri ft +po c +re places +commu ter +âģ¦ âģ¦@ +go ers +lo gue +para dig +bas kets +sensiti vity +joh an +atl antis +& & +suit case +anxi ous +l h +str i +gal loway +stre ad +war den +gr ounded +ffici ency +li feat +reli c +disgu ise +island ers +f cofficial +classical music +b mc +en field +bi que +oak ley +bat man +sla ying +ner ves +mul tit +calci um +projec tor +scott sdale +ant ino +gri ps +kim mel +des mond +prote stors +hi atus +metaboli sm +conclu ded +press er +ti pping +sli de +e to +hun ting +aus open +ri k +pp ery +innov ators +pitch ers +ag ger +fun gi +z ad +proli fic +rockn roll +bl ames +ct ar +stam ford +q ad +mozz arella +insan ely +den ver +ph ouse +nom ad +ï ¿ +s ris +pro du +hen ley +pag an +am trak +ru bi +in cl +tu tor +sco tia +wo es +sing apo +fun nel +turn bull +know ledge +gri mm +real madrid +we are +missi les +con sol +emo jis +sne ak +smi ths +ru iz +br ou +i el +ha ver +ðŁĮ ļ +kin gof +basil ica +circul ation +prin ters +ta pping +ri dley +dra gged +ha j +writ er +fundament als +personal ities +me tre +stereo types +bur le +best of +n ffc +ha th +mini stries +a ali +trac ing +pav ed +ł ï¸ı +g ic +insp ire +tu g +ha re +repe ated +ex pon +lol li +rho de +pre cin +install ations +instag ram +az ar +i es +sole ly +du kes +mission ary +van guard +fursuit friday +on d +pol ari +ma st +har an +jos é +jack ed +ec oun +al ities +ne ph +ra vel +moder ated +sco w +s fb +uru guay +as o +ni g +au du +p ints +lat ina +ben z +m itting +char ted +mat ology +cit ro +biop ic +ðŁij Ń +djo kovic +fox y +agu il +so to +an ada +sin king +sc rap +hair s +bethan y +fact friday +ðŁIJ IJ +unlea shed +) ( +contra dic +ram on +coast line +y ong +sn sd +li gan +p ome +mit age +ge tt +wat i +ri sk +so aring +bru sh +f pl +av an +å Ĩ +lar son +sh ear +mul til +blu r +multi media +chun ky +par i +n ani +weir d +cholester ol +char les +dream ed +tan ning +puzz les +fr am +hand ball +ch ag +beli ze +al u +bang s +Ñ Ħ +detec tives +mc g +ish q +bo thered +saf c +mp ing +ten eri +g ays +sail or +an gi +mul ticul +gue ssed +ros é +high ways +bro om +chatt anoo +- ' +see ker +on ed +at f +lu c +> < +bar i +per cep +jewel ry +as ph +sor row +sl ing +mam moth +jac kie +ë § +wilt shire +sa o +can cell +im paired +tor ial +bre ed +guy en +jud ice +tit le +pro spective +applic ants +ðŁį Ĭ +epis cop +e id +b yo +stock ings +ðŁĴĥ ðŁĴĥ +ll p +sna g +keep it +l ough +ol son +matur ity +!! !" +cop ter +i sha +bl i +wil mington +tr youts +th ai +ðŁ¥ ³ +pe bble +kra ft +f p + º +ssi vely +li vin +contest ants +tex tures +jo an +h dr +film festival +prov ence +wi do +op end +c si +sto wn +cro ati +ad just +host ile +analy sts +il an +cu ppa +bru m +newfound land +good win +me tt +mall orca +plu gs +bu k +bb hutto +wrest le +sa ire +sho pped +for za +le head +vi vo +ba st +ro xy +reg is +hard working +hon olulu +desp air +young sters +ni g +impro mp +roll tide +de emed +tre ason +ru shed +for ged +ff f +pikach u +bri ggs +do it +ac cent +la us +gla ze +compet ent +a ho +photo g +mid field +le go +har vard +min orities +re illy +slic ed +once upon +initi ally +financi ally +landscape photography +har dro +qu o +mm ers +par kinson +smu gg +read iness +bru tally +glou cester +mp ed +bbhutto zardari +mur der +ye d +dat aviz +sr t +dow ning +bi ans +m ü +fle ck +fli pped +s ly +brilli ance +ri m +k um +bubb a +ko i +knit ted +sor g +ma is +ðŁĮ ² +ti ss +su stain +sen su +ak han +zi est +exam ines +chardon nay +user name +short list +re bs +on o +dar ing +hard wood +che que +righte ous +light ening +dir k +shra dd +du ra +down stairs +sh al +ami gos +ru ff +s law +ri es +red nation +man us +ðŁĩ§ ðŁĩ· +distin ction +u bun +dur an +mi gra +thi ans +la ver +domest ic +k x +jaz zy +justi fy +belong ing +insul ation +color stv +drun ken +chann eling +qu and +xi ii +enligh ten +kan o +fati ma +teen choice +terri fied +p ba +as ley +met museum +dun e +pack er +ki o +ðŁĴľ ðŁĴľ +bo iler +fas cism +ar mored +back grounds +in mates +embarra ssed +defin es +th d +we go +silic one +lo on +el ding +bor rowed +he mp +ak sh +kaw asaki +br y +de af +kill er +dispo sal +ðŁĩ ° +glaston bury +un covered +o xide +po ff +d ant +k j +ku ro +dri zzle +peop les +fe e +pro pri +dd lovato +pi ggy +ot is +aller gies +u bis +pengu in +ser a +vi z +prosp erous +ici des +tornad oes +sene gal +web cast +sto red +enchan ted +bb cone +bay area +entrepreneu rial +rednation rising +experim enting +ang an +lot to +they re +por e +er p +seren e +east wood +bro kers +bar ge +stal lion +timber lake +tailo red +dy stop +b ate +lat ors +di xit +bran son +dynam o +ky lie +shame ful +bt wn +spring time +mix ture +s ounded +lu ton +dad es +mal a +op ra +en ic +rahulg andhi +se wer +~~ ~~ +ky u +nor theastern +ca er +bc u +nir vana +kitch ens +ous y +al m +river dale +hid den +fl int +sp d +pat rons +katy perry +au gh +exhib itions +sm c +shu ts +at ore +da in +some thing +ber th +bo g +por ter +gen to +con cussion +ang lic +ro we +gr illing +scar lett +master ing +mor nin +comm ented +si me +si zing +christ y +ce os +st m +at ry +tari ffs +vac ation +pre judice +p su +paren tal +far age +can a +cap com +koso vo +you re +men stru +stal in +grape fruit +br an +che sa +dav en +exc el +!! ) +๠Į +distribu tor +ce a +bride sma +millenni al +wa in +ob serving +mis ery +plan etary +expo sing +bra ised +comp ton +don gha +q l +spring steen +th ul +syl ve +cab o +pal ad +niel sen +gaz ing +ba ja +r oud +orchi ds +johan nesburg +se man +d ji +oper ative +affe ction +eclec tic +at c +mut ant +aw x +nic e +mel bourne +indu lg +tu lip +dias pora +wel p +big gie +mississ auga +retri ever +or an +tam my +c ta +hipp o +seas oned +ger mans +eng v +marvell ous +im f +rela ys +mon tan +maur iti +me ister +as surance +reig ning +su fficient +han e +no thing +pos se +nav y +in love +brigh ton +en qu +ch ung +sweat y +es c +cal ed +man s +nicar agua +sl ices +mo cha +washington post +bb n +dam ned +grow ing +en burg +lo an +me s +wh oops +believ ers +spi el +vo daf +l at +s led +cricke ter +brown e +golf ers +bar ra +wat chers +lu igi +sw amy +mom s +pit ched +san tor +cr s +si re +sc amp +bo de +ste war +jon ny +ent ity +pac qui +mind ful +min india +bear ded +temp t +scorpi on +eat on +authori zed +ar to +s vp +op athy +cch ini +house music +disney world +âĢĶ @ +pro pose +di y +expen se +ten g +pupp ets +sm el +d aca +per ry +fin n +boo sting +lefto vers +cou gs +satell ites +man y +az e +g ong +fi e +metho do +fer ries +ðŁ¤Ķ ðŁ¤Ķ +explore rs +load er +attrac ted +il ton +godd amn +pi azza +doc tr +sav ing +paragra ph +visu alization +may ors +work flow +ack les +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ठ¸ +twer k +clu t +lo ver +te ases +si an +o te +deter ior +accor d +l fw +swar ovski +nat al +tra ps +k ina +analy ze +laye red +bever ages +un it +ran som +pe shaw +dest ined +astro logy +si pping +miley cyrus +cam ino +marshmal low +bli ss +out back +fa q +int oler +humil ity +po ppin +hallo ween +mon tene +op hy +nu n +tattoo ed +a as +ðŁĮ ³ +dale y +qual ity +du sa +fisher men +swi f +ter rac +st au +le in +trol ling +ship ment +garden er +march madness +head band +gr t +bur nett +w and +!!!! !!!!! +gh e +du x +hu d +war ner +ðŁĩ ¦ +ex ile +rescu e +rat a +d han +duc ati +dro wn +bl ends +spi e +alli gator +simul taneously +broo ke +u ke +k har +comm union +ri ka +ford fc +chin atown +you rown +me y +can al +syste matic +de pri +ox ford +an il +w ut +equ ation +be z +fle ur +the good +lang ley +ad ity +ed ith +al fie +о ÑĤ +en cry +br ill +ex emp +ce sar +mb ling +ab ri +sc icom +j ing +school ing +mi ka +mechan isms +impromp tu +rhe a +moo re +crime a +be sto +wri ght +el ders +ro ds +kam al +folkl ore +be et +mini on +reli eve +thr o +team usa +pas cal +made with +boli via +itt i +free bies +desi red +best selling +l iness +la den +ke ane +mi sts +hipp ie +atta chment +@ / +se w +flan agan +âĿĹ ï¸ı +supre mac +stl cards +si as +q u +rh ys +ste ep +val leys +v w +pav ing +disp at +al ison +por te +id u +new sc +soc ket +mo s +co star +re vo +prote ins +stanley cup +m cal +ear ring +se cs +mc lean +cap ric +nick elo +ad en +v c +shou se +adap tive +maxi mize +entertain er +pro se +gri ffi +six teen +lam ar +mi rage +saudi arabia +awe ather +ru st +in filtr +fashion week +ðŁĺĬðŁĺĬ ðŁĺĬ +selec tive +bubb le +a den +fen nel +deci sive +m ta +mock ing +mb les +st amp +mu le +bernar do +gr in +po tt +j ingle +vet tel +colom bian +cam o +motivation monday +ba han +p ly +dh ary +k ami +x men +sleep er +gar a +my sti +confi dential +conflic ts +p neu +ce s +insur tech +clean se +me rely +va is +tu x +the great +shar on +ma j +hol a +eco systems +aj ay +aa j +hu sh +har mon +backto school +wiki leaks +reflec ted +ðŁĺ ĵ +commemor ating +ac et +buck ingham +messi ah +tu ous +hor net +to be +d q +he ine +mi g +pl ate +nichol son +sp ie +cumber land +nor mal +pho bia +happy halloween +city fc +mc el +gilli an +ke to +lu de +de mise +su ga +str ate +mcgr ath +visit scotland +foo led +cb r +gc se +col ori +po td +missuni verse +fin ances +ma poli +for ks +Ø ´ +cann on +medic inal +ðŁĹ ĵ +kh o +wre ck +pan to +bag el +gu ll +syndic ate +ic y +pr c +ki en +zi ka +ti sh +pe ta +c co +li za +ch ut +ex traction +el g +gl i +fu eled +pos it +respec tively +leice ster +br ink +vulner ability +im ported +e sha +ðŁ¦ ħ +r ural +re ll +gam ing +atlan tic +aband on +no ah +re solved +pro state +aller gic +ps d +âĺ ¹ +dun geon +fang irl +illumin ated +m hs +white sox +d ently +ck o +endor se +over ly +dazz ling +prior iti +night life +ut il +be have +flam en +east bound +ðŁĴ Ł +ilove you +gov uk +mozam bique +alle gi +dr i +testim onial +ath s +ì§ Ģ +mm y +shab by +pro secco +friend ships +cal am +dam ages +off set +jura ssic +jun o +arre ll +ðŁĴ © +interven tions +dare devil +car ver +run away +ran e +truste es +ha ute +dep ths +ðŁİ Ń +me in +sacrific es +con cier +ne sting +i zzy +me tam +ilove my +ur ine +du lu +mal hotra +ve ins +night ly +co at +an di +he witt +lon el +ci ble +wr ite +jen nie +sant ac +ĸ ï¸ı +str ato +singapo re +sop rano +kri sten +cheer ful +flee twood +fa iri +m eli +wa st +tur nt +sfor sale +sc rolling +angel ina +ren dition +jeric ho +nick y +or b +fla vo +patri ot +ash eville +sick ness +re fund +aggre ssion +b pl +ãĥ ĥ +elu sive +thi story +hang er +bu ffs +vil las +at kinson +sp h +ja it +decl ined +wo k +supre macy +oo tball +ey ang +ðŁİ ĵ +s ford +ath i +consu me +road ster +e so +u pro +reci pe +au f +uc i +ar on +oo oh +cs go +re ich +mc d +min ute +ladi es +pun k +rut gers +mee k +ariz on +ta j +land lord +de gra +autu mn +lyn x +us f +b hi +fairy tale +dongha e +bet sy +explo ded +chen nai +op a +pro tag +br ant +ðŁĵ °: +g f +pal li +ðŁı¼ âĢįâĻĢï¸ı +su t +ill ini +colum nist +shir tless +de centr +sear ched +ec or +bu ggy +s ack +ðŁĺĤ ðŁĺŃ +de t +ther i +or naments +bring back +to v +quarter finals +ic he +con stra +gi er +buchan an +vi x +kay aking +mu stread +swal low +mel b +sc af +op al +may oral +har at +ðŁ¦ ĭ +schedu les +id f +ha gue +ro z +a ah +d mc +du plic +ca che +orph an +frac ture +rec on +ch av +bun nies +al ain +mustaf a +ðŁİ Ļ +vac ations +dynam ite +tex ted +broad caster +ðŁĴ £ +ste amed +rock er +di etary +luxury travel +inaugur ated +sa wards +vaugh n +lincoln shire +click ed +kra ja +f anc +remo ves +layo ffs +mc far +bre eds +win nie +jon ghyun +incen tive +vari ations +pat ton +atur day +persist ent +pr un +pi ers +dal es +æ ĸ +breast feeding +r ance +ta wa +Ĥ âĸ +mur doch +cap tive +thi stle +nic a +commod ity +cou ldnt +board walk +graci ous +practiti oners +n gc +scru m +ner o +camoufla ge +col on +he i +phys icist +saturday morning +ten er +si won +colum ns +bru ne +y vr +ba ir +reti res +hal am +cab er +shaz am +min u +cas cade +milk shake +gri d +d ren +vin cent +so dium +plat ter +cheer leader +chen ko +y ak +elimin ated +ty po +y man +re think +âĿ Ĺ +ts ville +bernardo kath +ex tr +ðŁĺģ ðŁĺģðŁĺģ +ta o +re per +mo ths +em powered +c iting +transpor ted +mon ks +san at +cle ars +bachelore tte +camp bell +racha el +har le +hand ler +climb s +inter ference +rele ase +sh and +r bs +hr h +ãģ ª +val le +r é +sli me +w akes +chu bby +slo an +el ves +ath en +attor neys +micro scope +ston er +sc aling +o be +c out +se man +mid week +bal sam +ðŁĺį âĿ¤ +ti ful +v ish +lo tta +ri pping +re mn +ti re +le ap +ha vent +la by +hi mach +whisp ers +we in +ðŁİ ¸ +wild flowers +se le +u cc +li ability +az ine +sw ings +k ya +ta ir +re main +e do +flo ps +poc ket +grand ad +exam iner +gr is +ffe ct +ðŁijĬ ðŁı» +stud ded +heart beat +de acon +firm ly +infec tious +ste f +out lines +le asing +cla ws +sen se +tab s +hoo t +mo sul +spa wn +co a +hog warts +ve in +alban ia +manu el +b ino +vaux hall +scot land +go bucks +mat ty +phy sio +tor ino +const able +investig ated +s lower +mistak en +bay er +wild fires +vo ic +x on +time to +chas sis +bar ric +pi on +bald head +woo k +regi str +dra fts +b hs +li gue +l ick +staf fordshire +baf ta +dar ry +je anne +ven ding +cor p +⼠³ï¸ı +kid dos +fen way +ca o +west bound +ðŁĺ Ļ +dv r +quick er +bla h +goo die +ðŁĴĭ ðŁĴĭ +vo x +esp er +fac ade +cor relation +red bull +rou p +decl ining +chi ve +mc gee +tur o +in der +f eller +fu g +il ysm +mar di +peshaw ar +ki eran +ine ma +meat balls +pe ck +depre ssing +sen sing +gi z +dd ington +spring watch +ro aming +yellow stone +horse shoe +am man +week day +ol or +ðŁ¥ ° +boo sts +spr int +scar ves +je e +bee tro +cl an +all the +ìĦ ¸ë +enlighten ment +ado be +re generation +? @ +cont ag +yach ts +to u +mor a +en voy +r ani +go li +dhanush kraja +wood working +streng ths +se di +disc s +ar ina +sc on +lit e +ano ther +ðŁ¥ Ĭ +ye men +gu ern +sav vy +lo yed +biom ed +heart break +comra des +milli e +pat ch +un f +jar vis +bl aming +commemor ation +ge y +å ¥ +cardio vascular +alig ned +docu ment +. ? +aesthe tics +em u +the irs +le h +ps ic +si f +pl ateau +ex pend +domin ating +rob es +mauriti us +excep tionally +hom er +discover ies +bra un +ten nant +insul in +ðŁİ ® +car bs +te as +? !" +zi e +franco is +brow sing +th ol +cla rence +hel per +ob tained +cas sie +le es +! , +pome gran +hu bs +presti ge +] [ +mach er +bott led +pun ch +pi pe +o ch +gall ons +deliver ies +u ra +un day +mon de +depic ts +re gency +outra geous +khal ed +car o +he arti +za g +develop mental +over coming +stati stical +flavo red +for ds +cre atives +lau rence +di as +sun screen +in ked +pre acher +n ul +impac ting +auti stic +âļ Ķï¸ı +o ss +pel icans +cele ste +v b +ru mp +mc gra +fair fax +hu mor +bbc news +row ling +cal der +seam less +ag ne +p ti +mix ed +t shirts +mer ci +b tob +women instem +genealo gy +pre ven +l our +cra dle +gi use +Ð ¾ +chron o +fair ness +chocol ate +tor y +as da +pre scott +stret ched +al man +u il +re charge +in tre +ob st +hosp ital +hay ward +teneri fe +fried man +vap ing +confe ssions +ye ah +bal li +luck now +cor pse +sculp tor +amp ton +t pp +indic ates +sur plus +tru man +ðĿ Ļ +sin ha +in vo +sovere ign +ke v +establi shing +engra ved +assu ming +ðŁı ģ +sou za +fab i +ton ed +oun ge +del oit +dow ney +no ble +om or +car tridge +ðŁı IJ +u hur +hol loway +succe sses +r sa +âĦ ¢ +ma zz +tw d +disc ourse +. < +y at +satis fy +com pri +ठ¹ +graph ite +disser tation +ar ter +í Ķ +b ally +zom bi +ly ons +a ic +u bc +pra da +e il +da x +cla i +grand daughter +extravag anza +chall enge +ðŁ¤ ŀ +po ver +primar ily +dad dy +man a +bi kers +inqui ries +da un +fel ine +gener ative +he f +benef iting +lind sey +pol ka +demonstr ated +al le +rand y +o su +low key +weir dest +red bull +our y +n ous +wood stock +cre denti +nic er +g ado +aly ss +ap h +prepa redness +station ary +incorpor ated +dy er +sarato ga +cele sti +: " +antibio tics +or gs +inde fin +ap ron +и Ð +fif teen +no f +ðŁĶ Ŀ +ph x +te ga +m z +organiz ational +on air +band ung +pleas ures +mor i +secre tari +rac coon +ca shi +pil ates +k on +geof frey +la o +kam p +depart ments +back packing +an am +à « +crack down +aun ty +on do +li zzie +ph ers +cu n +ðŁĩ ± +k pop +pu t +inten tional +connol ly +bar clays +hs fb +swin don +u ku +s ally +a int +âľ ħ +pen ang +up lifting +epile psy +inter ro +bun gal +go ku +blue berries +ठ¦ +u ssia +sil ky +mou red +i stic +bri efs +me ats +go b +ch aser +state wide +pra sad +gl itch +ar in +ban ff +memb er +ðŁĺŃ âĿ¤ï¸ı +lo ving +hall a +ภ¡ +smo kers +yak u +scicom m +physi o +sw ol +lem ons +gel ato +ch ool +capit als +ki stan +ti ghts +spi kes +trav ellers +ik lan +commissi oning +ar ine +emabiggest fans +empha sis +front line +pad dock +destruc tive +ba ha +l inger +je wish +shet land +mc gin +mon key +ko z +s one +raj ini +te h +y en +c vs +masqu er +gir ly +we sle +was nt +bro dy +termin ator +gil le +mag gi +bir die +jeopar dy +cu bic +vm ware +intric ate +an up +to pia +east on +sab res +investig ates +bu sting +bil ingual +valent ino +in format +fer re +advent ur +hydr ate +for sy +az iz +san to +e de +whist ler +continu ously +d ham +un used +ji had +addic tive +vi dy +do b +i do +fi ed +ni versary +n one +fu er +ðŁĺį ðŁĺĺ +coven ant +prin table +immac ulate +o em +cl t +serv ants +consu med +un released +sc um +pack aged +me re +ìĦ¸ë ¸ +to by +ta f +spo ons +me al +f ball +fair field +jan et +silver stone +dart mouth +follow me +voy ager +kom bat +anni ver +ene w +mag dal +ho ve +sa th +grizz ly +car di +gart ner +sand y +kan ye +post ure +po ign +im pulse +radio logy +horiz ons +si am +aish war += => +no che +tr is +el yn +com me +du i +ce c +councill ors +cudd ling +creep ing +loc ke +manag es +trans ferred +ne cks +di er +dan o +v ick +lun ches +d he +en sures +cri ss +ul ster +bann on +cont enders +sp am +sweet ness +med al +hon duras +arc tic +ultra sound +in fr +disco vers +ei ffel +ca sters +ru ben +du st +awe ed +atri um +lest we +se ared +ðŁĵº : +ty ne +ex changes +little mix +l le +astron auts +hersh ey +work day +kno b +so v +re signs +today show +der man +an th +af c +ta ster +sw oo +sa eed +per ing +narrow ly +rn li +best buy +panas onic +obst acle +farmer s +ðŁİ Ļ +pa wan +ki est +ang ers +absur d +oh my +sin o +pist achi +sp ice +giu li +prime time +ko w +k ens +ex agger +! ?! +u ba +midd les +ju dd +e jec +slam med +pen sions +of a +re create +b hp +xx l +liver pool +thre sh +pur ity +ni eu +hol ics +wr ath +ra do +gli o +am ma +dile mma +cr u +lets go +.... @ +âĿ ĵ +sugge sting +tru mps +hor us +f v +ic om +refer ring +predic tive +tar ts +ge tte +so ck +glo ssy +pin ky +al ec +thy me +ou ra +thero ad +pe tr +cr am +p fi +dv n +me ier +incen tives +tun nels +mobi l +rec ap +extra s +upri ght +rev amp +per severance +, - +ot p +mir ror +ar wx +ger ry +ma her +g or +hom epage +am is +ag ra +made le +best friend +sirius xm +bun dles +admir ing +t dsb +ðŁį ģ +ch as +slow ing +ro h +wall papers +âĢ¦ / +tek ken +gang s +tal a +lind say +shou l +line backer +tool kit +ur anium +caly p +ab rams +mat thi +ðŁı ¿ +hon ourable +da yo +ver sail +tan k +st c +fr itz +spl end +pat ag +anno yed +on day +devast ated +chattanoo ga +national ism +mas sey +jen n +tail or +dev gn +org ans +zu cchini +on fox +sat ire +wex ford +dis grace +no to +vol ta +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +à ¶ +home owners +poin ter +m cr +au sten +day sto +mo ons +pal ma +gra zing +e so +influen cers +shahid kapoor +compli ant +measure ments +develop s +y d +par l +p vt +rand olph +tor tured +ger ald +eli as +deepi kap +war mup +hick ory +g ap +co ffin +am our +re neg +moun ting +seven s +ig le +hi er +dec ad +tri ght +esc apes +wer ner +t fl +ful filled +ni ger +sour dough +re aper +choo ses +spin ner +week nd +fil tered +sh uk +kat i +old ham +open source +kh anna +at elier +conne c +opho bic +gla s +complic ations +ar son +counc ils +sm ol +as sy +lur king +ling ui +han ks +e in +Ù ħ +ru gs +n guyen +nou veau +men ace +le v +alad din +ru ining +round about +k m +con or +shoo ps +may day +traum atic +prab has +ka iser +k ita +rou ter +pe dro +re tar +stun ner +spani sh +distur bed +acade my +e learning +wit ty +sen g +fer al +av y +sta b +ke aton +ur du +ko to +hu i +coo ke +ari an +the personal +u ma +se ap +a sting +rhetor ic +hand writing +munici pality +consor tium +ðŁIJ Ł +glasgo w +ra ya +eli za +polym er +bro th +prac ti +correspon dent +addic ts +gay le +ail ing +o fe +p li +hear tw +st itch +sight ings +prie sts +sam o +slo th +good wood +roc co +sab c +summ it +l ace +pres ley +itt en +cin cy +thepersonal network +s week +pe gas +af con +regi stry +ci m +le th +dic ap +cand ice +flu ent +sm ack +pede stri +al oud +car ac +priyan kach +p gh +ir ons +dol ce +lat via +dece ased +thero ck +cla p +cen e +fo am +morris sey +gre t +essenti ally +com cast +be agle +argu es +ing ed +- âĢ¦ +sa g +ha san +ðŁĻ Ĩ +ðŁį ° +nh ra +kann ada +indic ators +on er +bri xton +at as +screen play +sor ority +sha heed +he em +class mates +tain ment +es i +breast cancer +zucker berg +aur or +en cia +ref ers +kae per +vor tex +com part +lym ph +photograph ing +ste ff +rest ling +par sley +mom ento +th man +lac king +du tt +ocu lus +fin o +fren zy +ra sc +der n +dis missed +noo k +met gala +sh ill +rapha el +maver icks +exhib its +eag erly +c pa +amen ities +. âłĢ +exo dus +ern st +lit a +deal t +womens march +i ain +score board +campe ones +c en +ti ki +garri son +fidel ity +bra g +road map +psy chop +lo e +ble u +ðŁijĬ ðŁı¼ +sau vi +spr inger +temp tation +ru dolph +ac ura +wic z +parach ute +stro l +len ny +zi k +dom s +nb af +al pac +vivi an +ro ve +pre et +perpe tu +sna ke +air soft +infl atable +prin ces +ati e +ffe y +pati ent +m ire +chel le +sl ack +groo vy +# : +up loading +!!!!!!!! !!!!!!!! +siem ens +provi sion +v fx +need y +f ats +to poli +bhu tto +sa thletics +alu ms +t winning +south western +adop ting +last night +man ne +la ga +tw ell +ac ia +-- -- +eye wear +hur ley +fle e +sa ch +pe cker +cost ly +is k +cr ates +polic y +ero sion +in go +wer k +ðŁIJ į +torto ise +therap ies +inter net +chihuahu a +ri ps +fre i +ed or +tai ji +t fc +do d +demp sey +christ in +chen g +hi ps +gra eme +com passionate +cavali ers +histor ic +soul ful +crimin al +ja c +vin ci +expi red +sur at +turi smo +k ona +se aweed +ber ts +le ica +expre ssing +a al +wor t +break fast +her ring +am used +rhu barb +mar tian +cospla yer +y ash +stri al +ra ul +refer ral +dw ts +j w +ad ler +cur tains +gu r +val ence +tyr one +sw fc +coach ed +re born +diabe tic +cho ke +nor folk +investig ative +ðŁĴ¯ ðŁĴ¯ +z id +v mas +phi e +objec tives +âľ ĭ +over due +di vers +mat su +ðŁİŁ ï¸ı +casu alties +ภ§ +al k +stand ardi +re alist +arti facts +pand or +ke x +in vin +( !) +ine y +par aly +mr t +fay e +the voice +on ga +de ed +skin ner +az wx +speci men +priyankach opra +nu evo +bar kley +toulou se +resu mes +football ers +cit i +fe tch +è re +lestwe forget +ðŁĻ ĭ +ch unk +dri fting +manipul ation +equ als +pu tt +ky ungsoo +âĿ¤ï¸ı # +ela stic +par ano +fo y +do ping +cin cy +ss ler +interrup ted +al ay +ado res +ame thy +con voy +ãĢ ı +Ĭ ãģ +black list +gener als +sa chin +bru shed +oun ces +non stop +illi ams +bt sarmy +u av +ru ff +bur ma +bi k +defen ce +schul tz +bo asts +lonel iness +go re +trans forms +alum na +@ @ +ra ppers +ne hru +car o +himalay an +wearab les +ge h +pepper mint +re development +flam ingo +cos by +big baldhead +ag ri +bare foot +sco pes +re gram +gh ana +ðŁİ « +i heart +sa die +carri e +microbi al +ku ala +sk ater +quer que +âĻ © +gen res +reas oning +ch ased +as o +sli pped +en can +vam os +ker s +ad verse +mo il +commod ities +with you +sil ent +hy pe +an de +am ination +whi spe +lit z +âļ½ï¸ı âļ½ï¸ı +ri ff +pp y +lam bs +gan esh +ab sent +regu lator +marse ille +en roll +par cel +wa p +by rd +ðŁĩ Ń +tu ber +country music +par l +contro llers +responsi bilities +we y +ch ate +montene gro +chic o +mil an +l ms +tra inees +appropri ately +un certain +popp ies +ed sheeran +nutr itious +gar o +deut sch +awe some +ãĥ ¼ +comfor tably +land marks +et i +re usable +daniel le +ro sal +co les +just ic +c cs +f anny +ni m +mc u +clin ch +at ene +mer ge +im db +ang lo +uc cino +pan ini +an not +bur berry +feat ure +predic ting +fashioni sta +s ask +imag inary +mm o +south sudan +spe ar +hu bble +jo inthe +coyo tes +sli go +ko dak +sit com +polaro id +roo ted +corru p +ðŁĻĮ ðŁĻĮ +bris ban +at z +ah l +re my +tal ent +aval on +ra da +pau line +locom otive +go ons +ne mo +maser ati +ic u +stu tt +histor ically +sm b +pres by +avo id +so oners +rhine stone +w ad +ri sing +tro t +mo des +reg ent +optimi ze +re ece +sm u +ver ti +newyork city +cor tez +ra c +in case +sin c +fiel ding +e tta +tiff any +al monds +sad dle +k rat +mat ter +g low +star ving +gl o +cra ppy +sl ur +st d +monit ors +recei pt +maymay entrata +mc il +un is +rain bows +cal dwell +pacqui ao +j op +a fe +hoo k +es sen +wiz ard +medi an +fla ws +com s +âĿ Ħ +ing h +ha ynes +anton io +tem plates +ou ter +na w +cardi gan +bel grade +ðŁĴ ī +hom o +a ise +ro pes +no ve +what you +tri gge +concep tion +ad ukone +na di +fri ars +sw er +adju sted +hot line +san ity +kau r +down loading +c gi +ten or +eth nic +app alach +ภ¸ +pa g +gol ds +on set +investig ator +car tel +peace fully +jarre tt +cat alan +poli o +n um +fru stration +dhar ma +my life +âľĮ ðŁı» +aber deen +mu sa +bin der +spark ly +fle eing +instin ct +co ping +domin ance +ill ers +er a +u conn +lo oms +living ston +gal i +he s +c ma +bel a +se ley +mon k +la ch +mar x + ´ +m erica +woman in +es sex +ra ina +jim i +nep tune +z ack +chine se +mart ins +chand elier +her n +with us +ear l +asph alt +modu les +st p +ul la +psychi atric +mile age +captiv ating +si der +men to +mor t +tran ce +tal bot +ab by +ì ĥ +âľĮ ðŁı¼ +j ak +daw n +turn up +scre wed +fe ds +blue print +ðŁĴĸ ðŁĴĸ +har sh +er os +insom nia +ban kers +ta emin +mis conduct +hu mber +gi di +edu ardo +con a +musc ular +consu ming +ra sh +don nie +di pped +col lie +samu el +melt down +ðŁĺįðŁĺį ðŁĺį +me z +exam ining +schwar tz +pri stine +ðŁIJ Ŀ +ve it +ful filling +an esthe +gue sses +dra ft +som me +soli d +pati onal +ho ped +evolu tionary +all er +enter tained +sli ps +lud wig +conclu des +sen sible +bon net +cra ze +tra s +haz ards +const antine +ed ics +star trek +to c +occu pational +in cheon +deepikap adukone +pizz as +new comer +de part +oppre ssion +ebon y +foss ils +tro jan +el en +ste aks +k hou +positi oning +ug by +red cross +ak h +dol ce +us mnt +pp en +dil ig +ma vs +call er +cost ello +⼠Ħ +dy n +thing s +rhin os +a xi +sar kar +con vocation +att ers +ss ss +fun gus +eu gen +russ o +squ at +w sb +eli on +william sburg +s off +defici ency +be arer +o kin +key stone +t wain +cal ming +break able +wa res +horser acing +com bs +bun ting +u it +t land +ðŁĴĻðŁĴĻ ðŁĴĻ +ga stron +sab ot +ick ers +commissi oners +sen ate +ii ot +ath ena +nit rogen +an tony +ero tic +di alo +mis sou +hypo cr +âľ Ī +kaeper nick +can v +d roo +clevel and +o sh +mon sta +stefan o +^ ) +sh ul +po ison +ha e +commerci als +ma ul +nit ro +co worker +alo e +vap or +t ents +russi an +qu id +question able +mid get +po ker +girl friends +sin the +erit rea +ten ure +depos its +buc keyes +spot ter +theod ore +trin ity +joaqu in +u cci +follow the +caf c +mp a +ðŁIJ » +plo tting +dom ino +ta ek +sion ally +dicap rio +pa p +car mel +ig er +bt cc +beth le +www bigbaldhead +foo die +bagh dad +mason ry +off ended +à · +ภģ +sc ro +vers es +ori ent +ar ches +pi yu +know your +gre e +ta kers +gu ard +dish on +bucket list +bha fc +war dly +ðŁİīðŁİ Ĭ +leigh ton +pe w +stra y +assaul ted +in hal +ly fe +amar keting +l x +kat z +ubun tu +me o +carto onist +turno ver +mi z +dis like +mul len +mo f +bl and +hi des +emer ges +chori zo +truste e +ma hog +lan sing +paralym pic +fa int +fa una +ch al +sn ar +cat h +bent on +cast illo +sli ppery +apric ot +oec d +bar o +l z +he ming +clow ns +co workers +peru vian +commu ters +y ell +ðŁļ ´ +under ing +v j +tt p +fli pk +w ana +soc ent +Ĥâĸ Ĥâĸ +ठĤ +oo sa +jag ger +di sm +e less +d ham +cali f +a official +ec lip +harro gate +gra pp +com rade +n tr +concentr ate +thi ghs +bit coin +bel arus +ë ĵ +end uring +now watching +industri al +pi p +ar on +ar at + ® +whit by +oooo ooo +sa ree +tic als +mis leading +yo on +year s +sle igh +roman ian +sciss ors +vam pires +ac up +ab ba +th weeksary +cent ri +fl ye +u o +c bi +bu ena +sin d +mar ino +bur r +re building +ठ² +anniver saire +ac ca +ðŁĴĢ ðŁĴĢ +gett ing +tu lips +wolf pack +âľį ï¸ı +more than +ta kin +ðŁ¤ĺ ðŁı» +u be +mon ic +dou bts +mo wer +co balt +don ne +specul ation +argu ably +kak u +htt ps +prosecu tion +din ah +stam atic +disclo sed +bever ly +fl wx +cra bs +extraordin aire +war mest +imper i +o logists +trac es +par c +lake side +am r +ter i +hour ly +domin ation +ar row +shrews bury +ance stry +wr angler +trigge red +pen sac +roo ster +survi ves +a on +bo ko +val or +love is +la g +pe y +fo cal +out laws +bl anc +artic ho +wit s +marsh all +die go +support small +u ca +sa h +je et +syn ago +gover ning +ðŁĴ ¬ +sal ads +cre ate +miri am +cen sored +ami de +no u +z eta +allegi ance +* ) +bl m +ric an +pa stors +oly mpus +blo c +whir l +star ry +pr one +y k +p ne +congratul ating +be v +so ber +love island +sa ir +an ing +tutor ials +q e +lun d +in ist +cle ver +taxpay er +ali z +wren ch +dd ling +cap ri +h pa +ðŁı» âĢįâĻĤï¸ı +na j +o j +futuri stic +jelly fish +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +cel ery +plan k +fil a +ne me +un healthy +lec tions +ðŁ§ ¡ +rit chie +n ws +mi kha +wonder woman +âĢ İ +hip stamatic +ka g +ðŁĴľðŁĴľ ðŁĴľ +poul try +mo w +wor ds +lo ff +ðŁ¤£ ðŁ¤£ +relat able +re mixes +keny atta +ke m +re signed +fo d +stra igh +j lo +hu tch +box ers +colle en +mag s +instruc tional +ko l +attrac ts +pra g +account ant +go ggles +br u +th ole +mar row +leu ke +oc to +pon ds +bubb ly +he ist +ìĹ ij +im p +a har +ha unt +hall mark +psy ch +kkkk kkkk +col umb +jump suit +cost co +si delines +ag gies +over turned +ni b +key chain +fu k +f af +mi am +assist ants +cy cled +ri der +dam mit +red wings +mag es +kin s +ì Ĥ +ho d +son t +carol ine +" ' +cu le +bra id +fel ony +ar ities +ruther ford +depic tion +isab elle +ro ach +k day +fifth harmony +em y +li gam +bari sta +albu querque +gro ss +ðŁį º +oo ks +ðŁij ¼ +dun can +try in +jag s +g ould +li tho +âģ £ +а Ð +sam my +tun g +cas ser +apo lo +aaaa a +man g +as ics +sh en +p ye +tur bul +ss p +saint sfc +on lin +n anny +he ster +do z +ภĶ +th read +ren ts +kh and +ðŁĴª ðŁı½ +un conditional +rob son +car re +ph on +sacrific ed + £ +auto s +par ker +oc a +log in +kee gan +hard cover +dough nuts +ðŁĮ İ +spit fire +refresh ments +saskat oon +commod ore +j f +rub ber +halam adrid +child care +stra da +io m +ri k +dak ar +ther mom +cro pped +gar u +ali k +ven i +i ft +si ka +ritu als +z ul +e ch + © +su dan +l land +i me +do cker +ì ¤ +fe ared +fa o +wal ter +no g +mutu als +l h +ali gn +mon ia +concep tart +ðŁĻı ðŁı¼ +sco e +compet ence +sw ine +ly me +laun ch +green er +abstract art +inqu is +gran ada +ga elic +flu ff +d backs +grave yard +ba be +acade mic +adventur ous +joh ann +~ ! +bi bi +| # +pl ings +gett y +as b +âĿ¤ï¸ı @ +staf f +religi ons +bang or +world bookday +me gh +de vin +ash ore +meri dian +gi thub +qui z +all stars +be stest +ir resi +ack er +do te +war rington +pol ly +newor leans +cr ou +wi gs +che y +smithson ian +la sag +de tour +bor is +stra ps +mari ah +inten tionally +ko h +ðŁį ¸ +ssi an +mar issa +cor al +episcop al +casu alty +tom o +supply chain +sam p +on go +ro o +cavi ar +p fw +clau dio +buff alo +s ations +mat ty +snap back +l ds +al arms +mat te +âĺ Ķï¸ı +conditi oner +d ors +he x +fi zz +a stri +sus sex +secur ity +qa eda +all star +cocac ola +as one +cl icks +sc ans +mu te +he avier +ðŁİ § +âĺ ŀ +lv l +book boost +youtu be +fla shes +f jor +c su +explo de +do dge +cair n +gonz ales +th ill +pel le +hart ley +renew able +re tin +e stre +costar ica +shipy ard +nc fc +pri ya +a ghan +an ath +plu gin +co rey +re bound +or u +kat rin +hor mone +gi m +mahin dra +s sus +park land +har per +fanta stic +infer no +ep ilo +wrest ling +fe ct +c it +ac oun +to ssed +monu mental +char tered +bu st +pe tra +âĮ ļ +wildflower hour +sweat ers +* . +bl er +ate ch +go wan +demo graphic +bra l +suici de +renov ations +vu el +sin ister +ar mani +miso gy +ph arrell +nap s +un iting +crusad ers +cor gi +insu red +than i +no or +g q +d ada +bicy cles +snu ggle +sch an +ten berg +ss al +fe mme +bo il +½ ï¸ı +re ap +occur ring +hus sein +divi d +sto ke +sh alom +na ia +o lic +frustr ating +Ù ĩ +ig s +gro ver +scen arios +n ds +bru tality +med alli +bu on +sas s +skate boarding +ony x +lor ry +ny u +gau tam +mm ings +gu g +end i +lo thian +comm ando +chal k +ph ora +asse ssing +ti gh +crun chy +ad ay +is l +ci ara +pilgri ms +kam al +p to +brit anni +t ani +sm c +l ure +app store +ab y +golf ing +cl c +fa u +an as +shu tting +regul ated +carn age +scow boys +all enge +c ma +humbold t +rel le +ku mb +her i +refin ery +sound check +d wayne +bos nia +i sp +the alth +anni v +relev ance +my a +bag gage +dre ad +s bc +th ed +bu h +hi jab +lo id +ke w +c te +respec t +lovel ies +cu bes +celebr ate +dir t +sav ers +_ , +gar ment +pulit zer +mas jid +beat port +al arts +encry ption +s ner +ple ads +found ry +sym metry +ru mi +birth place +scallo ps +supp le +pivo tal +t ati +no de +so d +pro xim +tr ics +col dest +bren t +mand u +cla ir +e ach +and alu +hi ddleston +ðŁIJ º +mel ts +v ance +pin n +se ments +scre ened +sa chs +o bl +ic ha +âĺĺ ï¸ı +school ers +heal ed +lo gged +ðŁ¤ĺ ðŁı¼ +ic us +bore dom +b ish +b ffs +tal king +sure sh +hoo kem +de on +de fl +ei leen +ðŁį ķ +women intech +ri sotto +rang er +adverti se +ภģภ+tel ly +la go +dart moor +d ong +sk ates +lo go +un ner +mail box +ma sala +lo oooo +amethy st +che wing +c bb +australi ans +rc mp +game art +# ... +kor n +extre mism +fruit ful +anci ent +pu bg +pol ite +wh it +mur als +m gr +line man +dav ao +ste ms +ten nis +av age +tu pac +gigan tic +hs bc +auto biography +up the +ี à¹Ī +re gal +fig uring +ku l +mis sy +hoo p +gra s +for ums +back lash +abduc ted +p nw +min ic +bu tt +bott oms +at on +ven g +ðŁĮ ı +del aney +prab hu +fan club +over haul +health ye +sy no +aa f +ren amed +kim i +un cle +man city +se u +qu anti +este em +um in +en zo +mel vin +under go +j har +far ah +coast ers +humph rey +mh z +children s +^ . +d hi +disrup tive +integr ating +r nb +over sized +a ide +ne au +docu mentation +ðŁijĢ ðŁijĢ +pal o +hear th +ri yad +pun ctu +abc news +secu res +boy band +bir ch +ju co +tra ff +legislat ors +bay a +ãĤ ¯ +no ises +collec ts +s warm +k ner +bi shops +stur geon +snapp ing +mo l +fre aky +chair person +tro p +lyn ch +car cin +art sy +e sto +cha i +fl ur +inv ali +sau sages +im el +j or +fun fact +wit ter +puni shed +ac ons +h ya +re versi +em c +dif fu +z x +sp aw +cla d +d mit +hol land +fre sco +pay roll +ab undant +stu ffing +mor o +c ny +boy cott +wend y +ele ven +pro voc +pil ot +tr x +be ad +climate action +ri on +assi e +ì ĸ +o sm +islam ic +ho ar +good reads +al ici +afterno ons +spoke sman +jo lie +it as +masc ara +âĻ© âĻ« +pre vail +beetro ot +lu jah +k li +dod ger + » +ru le +l n +scre am +ho bart +col bert +r tc +er m +pat ro +quo ting +s live +que st +non fiction +semin ary +prosecu tors +ve st +express way +g ge +nau tical +et f +ðŁİīðŁİ Ĭ +dur ation +cha ired +the film +fab io +she h +can o +ðŁĴª ðŁı» +with draw +! :) +cor pus +phen om +yel p +la wn +ent om +snapp er +but te +pin ball +pro xy +libr e +alle vi +n ada +gabri el +fo wl +eure ka +daph ne +tu nes +pun ched +wh ore +jo g +ren tial +man ners +o pe +wh ufc +gu th +revol t +sne aker +philharmon ic +ho ste +sovereign ty +ðŁĻıðŁĻı ðŁĻı +fish ing +sci art +fe ta +i pp +dump ing +kel own +gir i +dig its +sal u +san jay +twee ters +sp as +col chester +sc ab +ma dd +๠Ħภ+Ä ĩ +ged don +march for +do p +maure en +un plugged +di do +fashion blogger +up a +mex ic +tar y +pol ye +jame son +v t +grin der +mad dy +consult ancy +¬ ë +leagueof legends +ac cents +um ni +jane iro +tu ss +h ens +ampli fier +to shi +pret tier +pre vents +new town +red wood +vant age +ball ard +ar tof +a she +a sion +lac ey +ap at +gro ve +ภĦ +rw and +real tors +tra itor +bed ding +ö r +zi on +fla shing +cam pan +boom er +secretari at +ab ol +liti gation +cont amination +se dly +shred ded +in for +do herty +bench mark +ro che +skate board +sho vel +i zz +to pper +o ster +laby rin +autu m +k ong +hum mus +vi z +tech news +kla us +am using +socialmedi amarketing +i des +cast ell +ste e +underestim ate +cal ab +pa ign +b illing +unanim ously +g mb +fly fishing +hath away +commerci al +colour ing +skul ls +pivo t +te p +tb c +motor way +x press +construc tive +pu k +under lying +kir sten +mani ac +cha o +se ma +chiff on +ðŁijĮ ðŁı» +ver ona +kom o +stan doff +wi ped +c ated +bla ir +wor kin +m sc +bethle hem +swi pe +unexpe c +pe es +pe tri +orig ami +ðŁij ħ +mex ico +flav or +ru dd +cannab is +mar u +ri ddle +wor shi +sil on +sch at +ap se +tang er +bi ous +e er +questi oned +o zar +dan k +angle sey +char an +bak u +compe ten +re pri +bat ter +sa xon +cal ves +leng ths +$ $$ +âŀ ¡ï¸ı +immer sion +ga unt +car ry +cy to +b anda +shu tt +experi ence +el gin +mous se +ta z +ê µ +in correct +en z +b ham +mor on +so ver +ar un +ti pped +la ble +de arly +bau tista +í Ļ +mor tal +woo p +dt la +sho cks +dav os +ðŁĵ Ŀ +swim wear +her man +ðŁijĩ ðŁijĩ +z ir +neglec ted +grac ed +campu ses +av s +ar ora +swach hb +live pd +ac cra +enqui ries +shoo ters +kur t +vancou ver +brad ley +gar da +g ü +ol la +attrac ting +up ton +ne win +lu mia +furn ace +ev ers +e on +sw a +roo kies +a oc +v ss +bris ket +tor ch +yo da +heart land +tac o +ph ony +food bank +ab bey +bab ylon +u y +gre ate +expre sses +d andy +sc apes +survi vor +ron d +e ci +ha vin +ab el +chil dish +tor que +wav y +ur self +kanye west +year of +ale stine +o brien +al fon +sk ag +kore an +anchor age +val eri +de w +ðŁİ ¨ +land slide +car ole +christ en +go phers +af i +priyan ka +q q +power of +it te +pc so +tw ol +pr y +intellec tu +guer rero +pi les +wish list +w ren +time table +ë ı +prodi gy +gibb ons +. / +ne ur +anz ac +mur ray +vie st +pla ster +la ir +art gallery +inter continental +g br +bell ator +nam joon +mam mals +am el +y aw +saras ota +cam ar +bud ding +sum mari +aco sta +la sh +ey ou +post graduate +instruc tors +ti g +const ant +were wolf +ic os +cla s +glen n +bud ge +ðŁĻ Ĥ +er ta +sta ins +persecu tion +cumb ri +o ch +syner gy +hu ang +scand in +mid terms +comment ator +regar ded +perpe tual +bo iling +al p +lan ge +sch le +fac eli +twee ta +ri dden +ok toberfest +charlotte sville +ik lan +jo u +ch atham +b sc +ðŁį ¦ +stra uss +mel low +xx xx +happy hour +re actor +ww er +distr action +at orial +ðŁĴª ðŁı¼ +twin peaks +fay ette +a or +ko k +bro om +sy fy +ou se +am ag +Ø · +ubis oft +lu lu +hall mark +stu art +it ya +si deline +venge ance +re lu +sex ism +boun cing +un ites +gu stav +te ssa +stu mp +pro clamation +ima x +divid end +col by +ðŁį İ +play wright +un safe +co smo +ðŁĩ²ðŁĩ ½ +cup board +constitu ents +ang lia +ram page +ðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį +than ked +take aways +shro ff +de bat +kh ur +conduc ts +format s +à © +port age +graph ers +u ten +pre m +mo ines +condem ns +s ous +l ps +f cs +deal ership +leuke mia +bure au +ski d +guardi ola +ca ster +thir d +avoi ded +en cyclo +c sr +vi xx +analy zing +she ar +dulu th +shap iro +chan ting +stre sses +as be +mil itia +ãĥ ª +col lin +arsen e +sure sh +teach ings +yi xing +sh ill +nu des +sv u +clear water +war ped +pro life +artist son +it u +versail les +galax y +ax el +spring st +cal a +hu hu +sc u +commit ments +exe ter +poign ant +mo tion +conserv atory +row dy +rec alled +mu sk +emb elli +so the +âĺ Ģ +sto pper +sch ild +to pe +el mo +zi el +j om +barn sley +snow den +on tour +jour ney +hills borough +par ole +w ts +mo ving +ag ility +tiv o +ff ers +kindle unlimited +g wen +ann an +ah mad +tex tured +hepat itis +dra m +insi ders +tis sues +ãĥ Ħ +fc barcelona +cr atic +na acp +pe can +f gm +custom ize +concer t +g sm +pe g +p one +justin trudeau +super cars +happy holidays +bu lar +ado x +lap tops +digital health +destin ation +gradu ally +áĥ ¦ +popp y +ss l +inhi bit +star light +of fro +glo omy +x per +hal der +im plants +le to +hass el +a as +un told +en ci +liber ia +or an +con tests +il ah +sma g +sc out +mari anne +cr yo +schedu ling +lo s +kan e +stutt gart +ne se +law rence +da in +pho tom +car ou +ภ£ +g wy +national dogday +roa sting +band camp +kentu cky +stret ches +ke rel +ca she +ãĤ ¸ +sta x +tran si +dog gie +at ric +hal le +ci vic +brow ning +lein ster +cat day +high land +joy ous +in cumb +or lando +ro mo +col ton +del ta +car ab +ro tc +aster oid +goose bumps +mo logy +yo ko +an ds +tomor rows +red carpet +sm p +ca sio +ðŁ¤£ðŁ¤£ ðŁ¤£ +se au +rejec tion +rot ating +bi partisan +th un +mat i +bon i +ol l +ener gye +do it +l j +mother hood +lou ise +neck laces +el ite +ni x +l cs +en v +gl u +le sh +cran k +su sie +m clau +so tu +crow ley +rat ri +use d +bre ton +alfre do +ye o +travel pics +ti pp +elli son +sax ophone +me red +heu ghan +ta ine +f es +vi ro +suppo sedly +i as +dige stive +y le +li zzy +wildlife photography +bri anna +west field +ra ined +am her +ðŁĺĦ ðŁĺĦ +distribu te +bott om +pre serving +oil and +craf ty +de scen +col ling +shakespeare sunday +r wc +ang led +ci an +t ations +mon tage +me yers +france sca +ðŁĮ · +wi ggins +san ford +volunte er +car ra +bar k +vari ed +pl in +am u +kap il +rock ers +qu ind +br ane +in mate +ent al +impro vis +michi gan +re tweeting +progre ssing +mercedes benz +smo ker +physi ology +dor ado +watt pad +h wa +sr bachchan +w ga +vol atility +hi re +ac ap +wn ba +hein z +stit ches +kidnapp ing +bur ys +lim b +f itters +thumb nail +ton e +mir and +desi rable +ad dison +tar an +tamil nadu +spec tator +soci ology +amit shah +remo tely +âĻ ¦ +ham id +r ds +g lee +smooth ly +sch ro +er c +lali ga +he als +us f +ni shi +d hu +un il +h le +tro mb +bhu tan +pilip inas +se ung +whit man +te y +min ce +snow boarding +re au +k ker +av o +zach ary +ran veer +ti k +gover n +qu al +beck y +anthropo logy +att en +grocer ies +de bit +war p +sil icon +hawa ii +ðŁĴ ħ +pomegran ate +pe er +orang es +people schoice +end ure +ðŁĴĽ ðŁĴĽ +ãĤ¹ ãĥ +ac ial +a haha +stu k +imper ial +bl ond +pow der +kno ts +vin ce +wood lands +den a +watch in +mat cha +ma hat +galax ies +middles brough +k ö +stre e +resc ues +wal do +lero y +desp ic +real ities +tm nt +ha q +un o +pe c +bolly wood +blin ds +design thinking +he ms +and hra +ab sen +fan s +ste ch +shire hour +bla ine +shak ti +pu rely +ðŁı ı +tra fal +ke ynes +gr ate +to bias +spon taneous +satur ated +caval ry +pri sc +ðŁĺ ij +wh t +pas si +~~ ~ +vir at +patt inson +la o +weir do +sym pathy +ju da +occa sionally +cred ited +stat u +es co +hil ly +esc ape +dischar ge +se er +may nard +sud bury +z lat +or al +we er +encoun tered +sm elling +over sight +ê ¸ +that cher +mack ay +you can +fre ep +freed oms +prophe cy +ho e +ishq ba +dra ke +qu its +pel led +tur k +o vi +wesle yan +new music +leg g +ch eng +h illi +ay y +pan ties +ad versity +ad jac +vaccin ation +ju ke +ga c +exce ed +time sof +sta ining +ep cot +v ital +up ward +bethe sda +apar k +ma hi +camp fire +enchan ting +rha pso +h z +na ver +fa x +vali dation +ac ad +ny r +as ym +coordin ated +depar ted +all ery +var ies +spr ite +chap lin +ss occer +s wat +bre t +relu ct +tunes app +super star +reminis cing +o co +home grown +dough nut +un canny +la pd +thyro id +! âĿ¤ï¸ı +botan ic +bre s +sp ade +i ste +echo es +du lil +bur sting +qui ero +ðŁij İ +loy ola +amuse ment +ha ils +sleep y +burgl ary +âľ ı +ro gue +cot land +mo ors +low er +wic ked +ðŁĶ Ĭ +compet iti +argent ine +yvon ne +karti keyan +ili ary +gat sby +precin ct +six ty +na ji +cam s +practiti oner +ðŁĺ³ ðŁĺ³ +pu ne +neg li +juli en +inv aded +cali br +cla m +duba i +mu k +lan tic +produc t +fe dex +ï¸ı : +eu ra +dari us +s ling +virtual reality +home stead +ðŁı³ï¸ıâĢį ðŁĮĪ +pac ed +in ha +pul mon +la zy +premi ering +ma stered +in he +con gregation +ba jo +sport ing +new jersey +hor ny +lma oo +leng thy +du t +yo gh +swe aring +philosoph ical +pap ua +in ski +know les +dy ke +âĢ ² +to ken +mc guire +ri ot +probab ility +mc con +gro s +su mat +c ite +da a +on da +mad dow +che w +board games +spar ked +re claimed +ad hd +ny se +imwith her +equ inox +boo ths +balsam ic +ha zy +dor chester +ag os +se aw +moder ator +seri ea +ander sen +pilgri m +âŃIJ âŃIJ +itch en +hal li +x ton +nathan iel +mun ition +celesti al +ga f +zo om +mark le +pen thouse +cal e +s fa +bar king +tu cket +em ery +cal orie +li que +ad ar +mc nam +tor tilla +wood pecker +mo town +bad ger +ayr shire +scram ble +dd ay +cra ziest +per rie +cho co +cast e +i ot +wre cked +selec ting +uss r +gra ft +pun t +lab ou +ir st +ba ek +Û Į +su ki +que u +ach at +te ster +aug mented +wc vb +sin ks +ðŁĵ » +ra ke +inter ne +be cause +belle vue +une arth +light en +ðŁĺ £ +turn around +labe led +unemp loyed +twitter kurds +le ia +h ye +great er +ðŁIJ İ +tim ed +i red +e tt +limit ations +cab e +s out +bee ch +anni hil +re trac +yo ona +ang er +den nis +supp lying +di z +" ( +sc ur +gun man +su ho +sauvi gnon +ภ¥ +wi ley +land on +choreo graphy +pre historic +ðŁı ĥ +var gas +assess ments +pinn acle +di i +chamber lain +ì Ī +v p +present ers +deut sche +sun shine +sal utes +r one +bu siest +- .- +motor ists +hemi sphere +al wx +ps p +ow a +den ying +cho c +gu tier +han uk +mus kete +jait ley +se wage +t ame +thin kers +shi m +se quo +pap ar +middle east +k wa +ke g +patag onia +no y +bar ça +take off +he a +à ¬ +n sc +g dc +ðŁij Ī +mou stache +mel ania +thr a +â¬Ĩ ï¸ı +pier ced +ze us +fon ts +ber a +it iner +q atar +contr ary +ire land +i fy +ou los +commun al +fin s +un paid +pa a +ðŁijĩ ðŁı» +ri os +ou p +f iller +cafe teria +à¸ Ń +kas i +cali ber +z ulu +v sco +ts ford +dragon fly +smo kin +pi st +psycho logist +diplom at +we bs +buc cane +à® ¾ +motiv ational +du ne +ba e +c fs +with out +er on +i ac +ate e +pen sion +fra zier +en sis +sk is +par ting +ger y +territ ories +nach os +eni ght +ever lasting +msd honi +tel e +sp un +po di +sab ah +environ mentally +ce ase +beau mont +mar ta +kel vin +ho ff +sun il +n da +co b +sh ale +ree dus +un boxing +u bio +re opened +n all +capsu les +mar r +himalay as +swee ter +ja z +f mr +twee ter +dha ka +na u +de mi +d fs +ta urus +fad ing +it utes +ci p +over flow +jef frey +don ny +car tunesapp +ðŁį ij +prefe cture +danc ed +c pt +ple asing +ital k +earth quakes +ul ation +hi o +ãĢ ĭ +ant an +nutri ent +de ere +selec ts +enrich ment +r iti +tram pol +bl amed +j ia +contribu tors +chesa peake +pi geons +tribun al +mad uro +w su +ilo ve +effici ently +dar cy +war ms +ar ra +ec u +ho wer +strugg led +rajini kanth +ðŁĺ¢ ðŁĺ¢ +hou sing +str at +eli x +disp ro +raf fic +thi erry +na sty +c fb +staf fing +al ma +back ers +hen son +sky walker +reale state +roo s +ness y +chan ce +cair ns +c ci +pe dal +ly ft +cross word +wait er +only in +kru ger +k ir +alej andro +car tier +car rera +re paired +ou at +un clear +un breakable +today in +qu eries +jo dy +gen ital +win ner +to l +kelown a +fascin ated +ãĥ ¬ +sris ri +squ ared +spr ung +negoti ate +priv ately +av en +>> >>> +g ical +gav in +chester field +zu mba +or r +nat alia +impeach ment +mn l +car at +criti que +credi ble +trac y +tan i +musi k +jig saw +gam bia +tol kien +fe u +as per +sav ory +fo xx +f itt +mar lon +l rt +v ell +p br +imprison ed +i om +chu l +wind shield +kay e +ba a +chor d +s art +al gon +minister ial +nat geo +la zio +nor ms +ðŁijį ðŁijį +lic king +fut bol +un sung +dalla scowboys +sh red +distur b +dev ine +be ards +ch f +b day +ro sso +ig or +ay i +si ren +k air +sti les +ro f +mag nets +un cover +mou se +bang ing +si ghted +spe ople +impac t +row land +kir a +environ ment +love the +p sis +mish ra +gl endale +ca jun +o che +de ception +sex ist +stra ws +s ga +buff er +apost le +sp l +pop up +ðŁļ Ĺ +r g +up er +ball in +i dy +occa sional +national park +ðŁı Ĭ +u an +innov ation +ภ« +te aparty +re tte +counter fe +b ha +rec s +ig en +ðŁĮ IJ +humming bird +cu r +ha ven +la zar +pue blo +: : +zi onist +op ath +inver ness +promo ter +carto on +cabine ts +mahog any +surve ying +r ational +feel ing +testi fy +so w +oc on +ภ¢ +ne el +mar is +sol itary +che mo +rad cliffe +sim ons +ros ary +new er +jo die +re tali +pra wn +pad dy +hen ge +k ala +im plant +at y +bren twood +par adox +ene z +re designed +p our +wy d +al de +௠ģ +sol d +biomed ical +๠Ĥ +tt tt +mat teo +ys er +new ton +de bun +ner dy +loo l +wo on +elisa beth +ec c +wh i +ach o +salv age +sal aries +qu ity +navig ating +oph thal +con soles +re built +o pec +ast ers +sho red +set list +kathr yn +rhy mes +re visiting +ash ish +li ft +re post +sole il +âı ± +weal th +sa at +we c +king james +flipk art +field work +se gu +mo dal +bu b +are rs +ðŁį Ĵ +clo oney +pad dington +necess ity +guth rie +pen te +li mo +jo sie +ar tin +en c +l hs +betra yal +info graphics +i er +mo a +hear ings +bon jour +sym bolic +ag ro +wed ges +krist ina +wild flower +athle tic +photograph y +pe sh +ca hill +chi lean +gou l +fi oren +ðŁij ¶ +z il +sk im +bad oo +deli a +tre ble +n cc +ðŁĩ¦ ðŁĩ +a house +bul lock +sol itude +ا٠Ĩ +can cers +futureof work +hu tch +water shed +war mongers +sp illed +colom bo +mo th +associ ations +weigh ed +global goals +not just +christ i +tor g +swe ating +man eu +clu sters +âĢ¼ï¸ı âĢ¼ï¸ı +ta ped +ul y +tru sting +yu suf +te in +ra b +, ,,, +sin ai +audi ble +explic it +cro wns +sch iz +at least +ðŁĹ £ +de bra +je suit +ene gger +z hen +one sie +i it +ss f +gur gaon +chak ra +bear cats +k ran +k awa +reque sting +han over +g end +sor os +mer cy +lovel y +do omed +tim my +ku z +ul l +ab ram +sa ison +ãĥ « +clean ers +re mo +circu its +bar red +o th +mo ist +madele ine +gall o +u j +per mits +hea viest +car ols +az te +gior gio +flo ats +decl aring +us rc +min at +craf ts +pri ma +conven i +nickelo deon +danc ing +ceremon ial +blo gg +tw p +anglic an +she k +k nick +( (( +hubb ard +harve y +hit man +fen g +we some +for za +s word +op us +bro m +gi bility +z al +m unch +dance hall +gre edy +hd mi +re birth +ðŁĺĭ ðŁĺĭ +s world +figur ine +com post +k f +engra ving +gior no +st ana +k man +ham ster +compos ers +aj e +func tionality +pol k +is ons +air planes +te se +hor rors +musc at +gi ven +sp ence +ðŁĩ¸ ðŁĩ +eli ot +ach illes +fre ck +crypto currencies +sou ther +hal o +bor neo +polit ic +hahahaha h +up state +si ena +obsc ure +hau sen +lloy d +happy friday +motor bike +bon a +americ as +hol s +- ( +spor ty +un aware +reven ues +christop her +bank sy +av an +ev apor +com press +eyel iner +to dos +buff y +renewable energy +ly rical +ar chan +rapi st +fair trade +lma ooo +beat z +pro active +la pse +ir ical +revers al +po de +mcin tyre +mac au +ãĥ ķãĤ +nash grier +f sa +g all +çĶ Ł +perpe tr +il ya +configur ation +% ; +str ange +rac i +ภĩ +pic kups +kov sky +mam mal +w ps +g able +compar ative +z h +save our +da vey +on etsy +mu ssels +mis er +cri stina +electr on +cra ve +lo ren +precipit ation +m z +ðŁį « +vin cen +snow board +no ida +ah n +marin ated +g tr +town hall +min is +bethe l +adv an +su ra +shi el +fur ry +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +lyn d +so il +sc ence +sen eca +shar jah +dick ens +credenti als +av ar +per k +requ iring +pre fer +j ian +de ca +r ach +ing for +del e +be ep +ðŁĴ » +cis ely +hu ddle +green sboro +haw king +ho ax +hang ar +ç ľ +mis o +lo vin +gre ta +ab ad +logi e +at an +snow flake +mahe sh +fear the +al kal +bobb lehead +ba hn +ju dged +fu tu +feli x +ðŁį ĵ +pi ke +der iv +notic es +au er +dis super +or da +wi pes +am ino +stri kers +foo tb +dram as +pun ching +score less +heming way +bi h +bal lad +chat ter +am mo +kle in +fabric ation +kari m +z end +hi sto +vol ta +rock y +marke ter +xtre me +sequ encing +paradig m +cle ats +boom ing +âģł âģł +block ade +promp ts +yogh urt +pur pose +nu r +regu late +nois y +ing rid +bird watching +bar tender +Ù ĥ +wor dof +cha otic +shor ty +el dest +z app +onceupon atime +fl yo +rit os +mike quind +ðŁIJ ´ +regi stering +. ] +ad ol +gg gg +pur ge +kid lit +ar bor +val ves +synago gue +o th +unanim ous +veri fication +dar rell +ãģ Ħ +vander bilt +tape stry +pro sper +did dy +dra fting +de cep +marqu is +st int +michael jackson +pee led +men us +bb b +sc are +ema il +wri gley +it is +f ell +some thin +bar ra +ed gar +di pping +pu ddle +sla de +lear ner +jal en +ðŁ§ IJ +the daily +mikequind azzi +ju x +iq bal +mckin ney +ra iser +ef an +dr one +cat o +pic ket +cro we +l att +uk o +giuse ppe +hin i +synthe si +ponti fex +song writing +to d +swit ches +din ners +h q +gabri elle +pensac ola +cir cle +expo ses +ev s +riyad h +pro men +o ck +sa j +cit ation +brew co +jo si +ep aper +dri f +point less +tang led +cri pp +line ups +fairi es +daz e +mour n +bla dder +sal z +bur undi +book mark +the people +sub sequ +princi pal +sk er +court ney +a oki +rac ers +ad m +mom a +critical role +hou n +shed ding +sa ka +ace ous +mck ay +hus bands + ½ +me da +accu sations +ro sel +nc is +witne ssing +or ama +go ds +hil ton +el man +ÃŃ n +meg ap +cra ven +announ cer +crit eri +sheffiel dissuper +milit ant +consu l +hoo ded +aby ss +b x +ma dam +lo cu +mary am +manic ure +grat is +ac tresses +ros ario +this dayin +king ly +gn ome +cel ine +r ous +he el +lil ac +vish al +ab h +thor ns +s ls +ne al +construc ting +be ren +s lang +ma ins +far ra +sar ko +pai ge +gu iller +l ala +ice berg +nou n +plann ers +u mmm +ou ses +ill ary +ma an +box ing +zi pper +srin agar +migu el +o str +mp o +responsi bly +lan terns +appli ance +x b +gren ade +neglec t +dy sle +ham mock +ne ctar +wit cher +r gv +di ence +ser bian +seed ed +cru z +bi sh +sp he +e q +sky rim +alge bra +phil ately +bungal ow +ge off +y ves +demand ed +consider ations +the vamp +pawan kalyan +co ded +grit ty +erup tion +se infeld +uni denti +ëĭ Ī +wor m +ac us +se ung +dun g +ro land +su d +di visions +ab lanc +shor test +j f +p oun +plant based +be to +tough er +mc o +don et +mark us +v fl +ðŁı ł +open ing +co ward +caber net +o xi +burle sque +sand ra +su mo +consi st +tho t +cay man +motor ola +gutier rez +d slr +y w +no bel +nov ice +moms demand +grun ge +sp or +d cc +pre sses +sli st +allot ment +voc ational +ft c +pu ja +lo ven +utt arak +tan dem +sh ep +come dians +anat om +cant wait +healthye ating +west side +mar gins +chi ang +asbe stos +stupi dity +proble matic +fit bit +: $ +ceil ings +shu a +protec tions +bio tic +beng ali +re sts +bien nale +tim o +cul min +e minent +affe ction +unbeliev ably +individu ally +canvas sing +wh itt +nov asco +chin son +h pe +go w +gloucester shire +pa o +thresh old +chev ron +s ine +we ther +pp ie +aqu ino +antwer p +âĸ ¬ +po on +inst af +equ ine +cinemato graphy +nbaf inals +vali ant +kil kenny +te rence +syste mic +sr l +p ound +made ira +pl ough +tre cht +mat ed +mp d +ransom ware +ph in +li qui +bb ce +boom er +i standwith +con ju +r te +nar a +foo lish +da shing +vier nes +br ite +da u +juni per +ai da +you now +ra zer +de i +repe ating +comfor ting +adjac ent +e to +ca sted +chat ur +mu er +syn th +san itary +mac le +independ ent +law ful +e erie +h or +ðŁĴ Ń +am rit +vel o +station ery +mu f +may may +contempl ating +elabor ate +gre gor +dri es +ac col +ภļ +schwarz enegger +ill nesses +day break +follow back +collu sion +electr onic +jo vi +hiro shima +ta w +hom ec +mic ah +qu itting +fro sting +ben fica +hel i +s ical +pic cad +corpor ate +ment orship +you are +sing er +shi va +ru ne +ing er +ri um +play able +doo p +wil low +ter re +ni p +at d +war bler +profession ally +er ase +proce ed +pedestri ans +mis chief +ben ding +alas kan +c kett +mo p +dd les +shut ter +ge ared +atene o +ma deline +g ations +o sha +der ick +sw ild +an gry +pat ents +hun k +decre ased +fr y +ðŁĴĸðŁĴĸ ðŁĴĸ +sal on +quant ities +d ario +ni gel +ku ma +jen n +happ ye +xx x +rex perience +pro s +au sch +rele ssly +ham burger +fuku shima +er ne +stat ec +ren d +may field +j one +lef ty +bern stein +sm il +gener ates +fore station +band its +ta yo +r ca +ac ci +rodri go +kn app +elo vers +vege tation +u ral +le ft +ħ ï¸ı +worl dre +sur i +embar k +w son +ba you +mu ller +mo vers +ðŁķ º +presby ter +l f +cre e +bat b +sal am +demonstr ations +an ec +n pc +it ics +to graphy +re inst +thur st +tal e +off ences +smart city +bro tha +ofthe year +in valuable +ear n +ðŁijı ðŁı½ +kre mlin +gra dy +town fc +guern sey +ma ha +contag ious +dre x +be en +( £ +nati vity +k tm +somer halder +comp ounds +íķ ĺ +" âĢ¦ +af g +ott news +h ound +fire fly +cil an +donet sk +volunte ered +ak ira +è ª +sing ul +st h +dro wned +mand o +he ir +ðŁİīðŁİ Ī +tax is +y uki +vel d +k ans +el k +ran ts +hash tag +t eng +ro g +a at +gru b +e ber +in india +colo ssus +sig ni +so ever +mile stones +der o +differen tial +phu ket +master mind +an gh +mel ani +bro ker +actor vijay +stun ned +continu ity +af fl +vo cal +perenni al +fianc é +in complete +hun ts +re issue +domin ates +tur meric +ro am +ri on +bag ged +nas sau +fu t +x ox +national trust +jo ye +san o +hearth stone +dis respect +le es +h se +siber ian +offe e +re stock +wolf gang +re gan +plan o +un wind +re par +mil le +] , +skul l +fat ally +concep tual +ðŁĮ ² +f é +ber to +b ms +u a +mag na +notre dame +le te +la undering +heartw arming +buffe tt +go at +pe abo +wind mill +v ac +continu ally +az alea +mem brane +can cels +make yourown +athe red +p to +tor pe +ðŁĺ ł +ðŁĴ § +sc ares +le aking +z et +pix els +ac i +kh il +marath i +ðŁĻı ðŁı½ +u la +tam u +chandi garh +z agre +aa b +pronoun ced +aubre y +sand er +pun ta +har low +ic elan +celebr atory +so t +unci ation +stru ly +mc dowell +deepi ka +remin ders +my stical +ct c +chat ted +s ica +bar gains +ch hat +ru bin +m net +oiland gas +pel ican +o at +mor ality +k our +i h +nu clear +gc u +ric her +vene zia +m ma +le ith +ac company +rich mond +sports net +ba ahu +smu ggling +mm i +ðŁĩ®ðŁĩ ª +twi sts +sahi b +.... . +amb itions +il lo +histor ical +fo rec +show biz +pon ies +chas ers +remo del +will ing +prince sses +am ple +cushi ons +ac les +lot r +da ch +an the +in corporate +new bury +ki ri +fried rich +ab v +ball ers +alber t +ðŁij Ń +let i +nan op +ci de +anal o +n sf +)) )) +griffi ths +valen ci +ro ano +fun run +babys itting +ca day +ent re +u ck +slu g +tic al +the sims +ro ar +car ney +g am +sto we +fi d +bun ny +sham rock +pe cu +mol ina +go cougs +con tributes +transform ation +mo y +v aj +sever y +antioxid ants +thir teen +sight seeing +l j +reversi ble +odd ly +hoo kah +nou vel +hal al +fe i +stab les +mul t +ho pped +bra ids +inter change +ghana ian +ww ww +eth no +con junction +ago v +ye ti +earth and +ts p +con serve +heir loom +metaph or +woo f +tor io +self less +n wa +em ilia +yl ene +y xe +gi ar +moder ating +pro bz +b fi +ne er +du mmy +hanuk kah +we bber +k v +eye brow +dag ger +su mp +ra ges +ork ney +tb o +hal sey +assign ments +tr onic +scri b +co on +an war +# âĢİ +jal ape +flori da +qu aid +haw keyes +âĻ¡ âĻ¡ +street car +ro g +dat lantic +gran ola +un changed +expect ation +Ù ĩ +mar lin +gu mmy +ðŁĻı ðŁı¾ +awareness month +oil painting +mu th +per ch +jun to +villa gers +mor g +che ated +web comic +the future +d ps +la kings +men tioning +vo or +ident ities +accor d +mc gu +l pga +rum our +massi vely +m pls +heal y +d ate +sp oli +re visited +on t +al and +scru tiny +lakel and +bl ending +< / +an kara +jami edor +metab olic +f ences +ann y +å ħ +semic on +oo tt +space ship +wack y +le ta +ap ac +she e +in herit +do res +ðŁĩ¨ðŁĩ ¦ +gent e +tw ick +ri ms +gal ve +de ville +king fisher +scorpi o +ow l +al ar +vari an +ðŁĹ ĵ +vene tian +star dust +then orth +q ing +har rington +consul ate +spectac le +ho bbs +tur ks +gre er +mat ing +ðŁİ Ģ +ðŁĮ Ģ +direc ts +í ĭ +pompe o +vo iced +la os +tz u +pro me +pri sm +mer c +fortun ately +bc fc +mcdon nell +not sorry +smi led +t ba +for war +mid term +dar by +we instein +up grading +wol ff +bron co +cab ello +ðŁ¥ ĩ +fi able +shar pe +bat tered +sat o +myth ical +instap ic +pre pped +eni um +e spo +di aper +explan ations +who pping +ragn ar +pe el +antibio tic +l acks +harri son +li sm +au l +qu ail +martin a +sent encing +sc ams +di di +tr onics +ãħł ãħł +go ff +za in +param ore +cha ined +clin ton +li ff +cott ages +em on +reve rend +consu mer +ce an +t any +lum pur +e bay +sto ol +ðŁĺ» ðŁĺ» +ta pro +h ath +modern art +just ine +prover b +app y +tra x +mani fest +am bu +nai k +pe pp +r sd +mer chants +kitch ener +shi fted +li zz +âĺħâĺħ âĺħâĺħ +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +uto pia +tom o +ou ted +com ers +chiroprac tic +book club +cin dy +pro hibition +se uss +ë¯ ¼ +thin kin +rr rr +go fund +t ack +om b +catastro phic +ling u +guild ford +bo td +ॠĭ +plan ter +^ ^ +win k +kath mandu +sto ppers +smooth ies +re efs +hin d +bell amy +Ħ ë +waste water +vo or +nat l +! ] +re el +y ap +scoo by +work space +corin thians +bl un +obli gation +g bbo +dy son +cra vings +ell ington +dap l +wre xham +earthand clouds +uk runchat +positi oned +kal b +four square +jo ck +im pending +even ing +ath y +pro claimed +c ites +ann apolis +san i +mar th +ir l +accom mo +ka a +fin a +y aa +di sper +ec ar +bha k +will y +ðŁĺĢ ðŁĺĢ +mcder mott +mo j +gener ational +u said +train ing +lon ely +lo res +impe cc +âĢ IJ +beav ers +ma ki +he b +aap l +å ı +wolver hampton +leader board +me u +c fa +easter n +hu r +civil war +ou rage +hor ned +le high +awar ds +evi dent +gi gab +r ous +ma del +ro byn +ur gently +k ors +en as +heis man +bam bam +fab ian +f om +evalu ating +assemb ly +out sourcing +hun tsville +ðŁĶ ª +justi fied +cashi er +sp aper +buc keye +analy tical +illumin ati +au tho +o j +sha de +geel ong +wh ey +he aton +terri bly +ele k +un charted +sd live +moto cross +her mes +dar shan +dar lington +cash mere +gri pping +cilan tro +pun ish +... : +ðŁĴ Ħ +inst ance +der i +lo bal +muk her +sp ar +thin ker +fre mont +com piled +color ado +vig ne +sm d +whe ad +villa ge +le ek +formula e +ta res +persist ence +?? ???? +ped ago +he z +alzheim ers +vul ture +off ence +is great +suff ra +kick in +h mmmm +broad way +ï¸ı @ +art i +alli son +endor ses +ry u +lolli pop +soy bean +kend all +cer a +inv ade +( ðŁĵ·: +conver ter +car pets +ho bo +fr it +pe ac +es qu +ern an +ou f +an il +di ffer +ch ing +bre cht +sp g +daven port +stra va +sever n +n gos +stor ians +fe te +parame dic +j hb +al amo +sne aking +gold coast +roof s +isi l +depic ted +projec tions +nu mb +o ss +ep i +glu cose +zid ane +infin iti +íĺ Ħ +ran som +ton ics +fal k +g ler +ou tw +re ss +week ly +the on +n ole +ðŁĩªðŁĩ º +vol ley +sum mar +neg ativity +sam son +ye w +aus votes +ju l +ju dy +f art +pra yed +pal ate +multicul tural +double header +cycl ones +pier re +ãģ ¨ +âĺ łï¸ı +rt w +conver ting +wir ral +l ari +ir relevant +austin mahone +an che +ya an +sd f +$ . +explo ding +ulti mate +prof ici +gofund me +cell ence +ep stein +bul lied +sep tic +à® ¤ +lu mber +cu ff +vsco cam +pl or +ภ¥ +se ok +ro to +venezu elan +sor ta +spir ited +daniel padilla +team sisd +radio active +icelan dic +ðŁĴ ¤ +ver e +accommo date +shi pp +ot ter +ol ina +e go +su la +san antonio +de as +simil arities +âļ ¾ +y om +bro ward +å ° +can cun +veri fy +on te +candle light +ìł ķ +inf ants +az am +ðŁĺ ° +le ven +un stable +bloom ington +x ford +con tour +y p +innov ator +histor ies +po y +lolo lol +ex pires +cat alo +bill boards +an ab +el ic +novasco tia +fa ire +ìĿ ´ +rock well +gr ille +az tec +joh or +ur struly +fi ren +dun lop +id le +port man +jo es +tx hsfb +hol m +cham ele +under world +lo ss +ti em +therap ists +past ure +pa ste +ing now +vul can +ra gon +lar kin +o shi +ho co +child hood +umb rel +success or +kath y +iz en +° ï¸ı +share holders +ol ga +ai b +he ap +fl aming +ro u +air tel +rat t +z ane +vo w +thor ough +sn ag +par th +un conscious +ve y +new release +gh ee +croati an +facilit ating +swan son +astor ia +to logy +master y +ðŁ¤ ij +bil bao +trou pe +the ori +chey enne +ro tt +shore line +gra sso +master chef ++ ) +vi x +ellen show +as g +an ak +ku ya +safar ilive +debu ting +blu m +list ener +v ins +book shelf +smart cities +makeyourown lane +; ; +ðŁIJ ¯ +ri zz +on ward +bull dog +bear ish +vir uses +fri gh +lin den +we iser +sn t +gon a +dre sden +fl anders +cu k +wheel ing +ba u +atu esday +surf ers +swi ft +mc call +arbitr ation +aw d +mon c +b ine +at x +re fr +mi ro +po sey +n are +rit ter +âģ ¦ +play book +blow out +sports manship +s oooooo +malay alam +gri ms +bur bank +infin ity +sar gent +oit nb +joseph ine +ski pping +par kin +excur sion +semin ars +jo har +par tridge +post game +ll ll +blan che +temp ting +m na +lu ka +is ers +to ffee +bar ron +he mmings +sa e +go hawks +cu pid +li mbs +con se +un common +z ada +head shot +so ils +pione er +mam ma +sem itic +pan dey +jamiedor nan +spl its +vel a +son i +ra ff +t mobile +âŀ ĸ +pra wns +lit er +enjo yment +egg plant +tu b +cultur al +us ic +suspici on +sy cam +summ ed +ma du +ho ck +up wards +eye ing +ri ve +assas sins +âĤ ¬ +out fy +chi ves +t ner +la is +por ridge +sad dest +w cc +vick i +sna ils +biz italk +mill an +ðŁĮ į +sam oa +j ing +mi key +gu j +chel ms +eli gibility +arma da +thro p +surger ies +ãĤ ¿ +mo hawk +ex its +me m +is lington +c me +land fill +kait lyn +ðŁİ ¼ +combin ations +tomorrow land +ver b +cor a +pre cisely +na om +ðŁĨ ķ +shr ink +sof tly +merce de +mand el +poo dle +ball erina +sop h +jux ta +y at +ary an +hesit ate +lo wered +gu lar +dungeon sand +ron an +my ri +sp f +men opau +gra sp +pa thi +fe asi +fla w +shi story +ste ward +gg le +fay re +cli que +credi bility +yo g +sec tion +mu sko +se ville +no tt +cal m +mate o +indic ted +fi ba +by l +lin o +u kin +!! # +enig ma +siri us +bu sc +ðŁį Ĭ +mac kerel +psal ms +a at +tomorrow spaper +ðŁĺ ĸ +p fc +........ ... +shre k +mul let +o sh +danger ously +immen sely +am ur +ðŁį Ĥ +pro por +sy a +london marathon +abo ve +obli gatory +pro v +ra cha +alex is +pri mary +sh h +ether net +d stv +cou gar +un lucky +ni l +steak house +mel a +fc bayern +cause way +ca therine +fluore scent +nx t +to kyo +au sp +releg ation +qui zz +shored itch +proud tobe +promo s +inter acting +home brew +da esh +w pg +stead ily +provin ces +bal lots +i ah +al to +< << +you u +ri ley +prefe rence +tra verse +incen se +am munition +ho dges +# @ +hail state +tart an +witch craft +vent ilation +liber tarian +! âĢ¦ +ow es +% ! +ong chang +bru shing +le ic +fi ber +under attack +down load +ex pir +hy o +pompe y +mc bride +y ag +stre e +com bat +ten ding +ai ra +gug gen +ab ra +in na +fli ps +aw al +m ach +dol lar +inspir ations +z um +o du +it ty +video game +aqu aman +har u +bel fast +je b +but ch +us gs +calcu lus +go yal +mor gen +x finity +stand up +contrac ep +sab re +na be +in secure +gener ously +epit ome +l w +t ca +narr atives +don nell +pand as +ber gh +tu t +ker al +fel icity +br ampton +quinte t +nom ore +ðŁĶ ij +lo i +alham dulil +ðŁĶ¥ ðŁĶĹ +ston er +shaw l +clin ical +bren dan +gon e +fla wed +tri ppy +j g +al location +po aching +ve vo +mo cks +lef tist +bon uses +condem ned +abil ity +st ating +microbi ome +bio logist +for you +wahl berg +ss or +ift ar +w ul +ÑĦ оÑĤ +pom er +me me +ver te +tre ll +tra it +in let +hormon es +deliber ately +vill ar +battle ship +p bl +tw enti +ho kies +dal ail +say a +may fair +han s +die ts +⾨ ⾨ +od in +hot spur +pap i +k ana +k amp +fin na +flo tus +ti ans +unic orns +tribe ca +chang ers +fore ground +out a +inv aders +gett ys +tomorrowspaper stoday +mac millan +hand written +w fp +u de +state of +base d +âĺģ ï¸ı +cas m +psy ched +histor ians +fol d +d da +ag grav +p ans +green way +au sv +ðŁĺ ¶ +shradd ha +inde x +be sti +zim mer +t ness +eye shadow +ot te +go ts +distribu ting +pro min +yo l +ace a +tram rahim +hoo per +supre me +jam min +intu itive +quali fications +sli m +sid di +jay ne +tri pping +g tx +pun s +e manuel +om g +mid summer +in to +succul ent +ri en +new mexico +o or +hoo king +in f +ðŁ¤ Ŀ +flir ting +na hi +g friend +t ps +hel ix +z s +on ie +ct f +kri s +irresi stible +fla p +ðŁijıðŁı» ðŁijıðŁı» +us wnt +ru d +ram ps +pin oy +ot w +lol z +low ering +favor ite +t mc +phra ses +her mi +aver aging +em br +ben o +estu ary +sle eve +ribb ons +ta sh +ภ¹ +x f +aw gs +sun ited +brew eries +anir ud +pun ches +ol die +ip ads +wi fey +land lords +d ji +gun ner +íķ ´ +tex an +ex op +cas sandra +s off +ðŁļ « +igh ton +bak ers +awareness week +v all +ear p +bts bbmas +apologi zes +âļĵ ï¸ı +was ps +states man +snat ch +watch dog +ra fi +after party +spi ke +j er +peri ph +r nc +mu ll +le en +shi es +li eu +urstruly mahesh +mer ton +de sai +shi f +ðŁĮ ± +pe dic +gos ling +arrang ing +ww g +gen y +you uu +netfli x +e ttes +k wi +bernar dino +am iga +Ø ¨ +kashmir i +t ings +emer itus +de cat +ab domin +dc i +pha ses +d jan +be am +op ry +i shed +the ellenshow +the st +habit ats +to ons +mclau ghlin +ri pper +micro biology +tal aga +clu eless +ss u +cro che +bro mance +longe vity +zagre b +prev ented +tra ve +spo ilt +darry l +migra ine +al cat +dd dd +vi v +ser pent +mat tel +jam a +con quest +î Ħ +sam sung +presbyter ian +ket ch +fire fox +mo tif +le c +cho pping +cher no +j ann +ðŁIJ ° +pro lon +wake up +conver gence +mersey side +heart broken +lo oming +hal lucin +mai ze +commun ism +mo h +twitter storians +serge y +res eller +favor able +ed gy +re iter +mal aga +live me +ka hn +pul sion +big g +kim kardashian +ati o +tyr anny +ru ption +q ant +pro ven +by z +pu shaw +kri stin +e er +tar dis +ri z +awak en +mi ko +un documented +path finder +indirec t +resemb les +h ler +conce aled +scand al +re im +d nb +cr itters +attend ant +apprentice ships +aa u +scre amed +l su +fa h +har bour +ed d +bat sman +li ss +mi sha +spani el +it f +advan cement +fa c +close up +cecil ia +medi c +narcis si +lav ish +gi ac +ma ys +le it +wine wednesday +pushaw ard +let to +curren ts +bug atti +out ine +w j +un do +ler osis +devo tional +ðŁij « +on na +fais al +sa una +himach al +am ii +à® ® +di zzy +screen writing +ph x +sp n +ick i +ag irl +fi shes +wb z +pi m +bo ar +ac id +! .. +rocke feller +n ga +dra stically +simpli fy +dru mming +autum nal +gur mee +lor de +jo ann +give up +b our +am ura +der land +sim pler +wat son +tri dent +concor dia +bel lum +bre k +dum plings +vi on +dungeonsand dragons +sp ri +ascen sion +wil datlantic +u st +rob ins +legi on +insi st +jar o +gue ss +so b +bigh it +pool side +negoti ating +mc gill +bil d +techn icians +miti gation +ajay devgn +b to +ant en +cosmo politan +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ +patri oti +temp er +promen ade +nav ajo +nam m +wrink les +dc fc +le ach +bru nette +r f +cout inho +al ti +tradition ally +op tome +na z +accord ingly +rec ard +de ets +sw ell +po sure +whit ening +strang er +illi on +here ford +u wu +ro bber +cotsw olds +cl en +gor ge +nam aste +re lish +gri ff +adren aline +bla sio +val e +ê ² +toler ate +rail minindia +jen sen +ho ven +el lu +ob sole +eisen hower +unidenti fied +than niversary +body guard +Ø ¯ +i dge +sch al +stock port +sn i +re taining +po po +pix ie +oli thic +ki er +ha jj +sa z +cor bin +!!!! !!!!!! +v it +me gat +de h +circu it +af fleck +theore tical +hope less +u ab +slu mp +b ice +jam med +let stalk +can i +side ways +labyrin th +re fs +ha hn +jare d +ðŁį ¹ +jam bo +ph yl +enhan cement +c tr +ful lest +se ye +do ba +cho ic +yo s +cb j +andr é +re watch +pri ma +doctr ine +for gets +u hm +ar ound +u le +art lovers +shi raz +har th +ex tor +Å ¡ +unexpec tedly +eli us +y x +em my +se ac +ðŁijĩðŁijĩ ðŁijĩ +correc ted +com bu +wom anc +cou gh +what son +publi shes +divers ity +back bone +lock down +mesmeri zing +nor te +ma b +desig ner +í ģ +ra gh +mole cules +get outside +the beatles +semicon duc +nach o +lun es +ham mers +sul tan +o on +fe ren +att ach +ar qu +uttarak hand +s ash +; - +tre ad +i ko +ar thur +scandin avian +r ation +ga el +charge able +fish y +v ma +hand bags +char a +ay ne +de fam +sett lers +qad ri +pal ais +in wx +apocaly ptic +poo ja +a es +at ories +proof ing +n lp +ts la +v ina +li do +dee phouse +informat ics +v v +pp ings +di ss +à ¯ +uhur u +st ony +betra yed +b aff +my ra +as pen +allow ance +tam ara +ci f +cor bett +ser ge +di go +ambi gu +pain ters +p cr +p ca +nom s +lo ft +ve e +opend ata +ðŁIJ ± +alex andre +identi fies +fantasy football +re production +brom ley +ware agle +mm er +p ss +cu es +ay at +hut chinson +sar ac +jack man +ira h +ap ink +col s +aussi es +ex ecs +day ton +ðŁĻ Ĩ +im v +har am +chuck le +authent icity +ar do +incub ator +ภª +photo shopped +embrac ed +fight for +gor man +zz zz +schol astic +cri sps +te apo +mid night +ga ine +col lier +s ate +de tte +å Ń +imag ine +i ff +tw ili +i fication +teat ro +nor ma +es ur +emergen cies +rise up +r inger +hass le +cait lyn +tranqu il +vers a +se b +over look +gin i +bo go +se re +may ne +henri k +contamin ated +rhapso dy +pro portion +wildatlantic way +âģ© . +organis ers +tran e +stand ard +sper m +laun cher +ric ci +her ts +paper work +showcas ed +mer yl +pen a +p imp +disa strous +^. ^ +phar a +x is +fron tal +sw irl +sp ills +swag ger +smart watch +sizz ling +savi our +cat ar +bb cr +refurbi shment +dr is +citro en +absor b +patrioti sm +il leg +chro mo +fresh ers +ru s +lim iting +ef ish +down ed +man dir +hazel nut +p all +mac on +disappear ing +quali fies +bo on +bar racks +am ine +gen dere +ðŁļ ĺ +j es +ãĥ Ń +qu ito +middle weight +sch au +quad ru +aci ones +limit less +ðŁijĮ ðŁı½ +ch man +ar av +regulat ors +it up +batter sea +mil ford +g z +tic king +gh ou +cru shes +tu tu +dread ful +fam ine +for change +dalail ama +ðŁĴ į +whit aker +hash mi +h us +vo d +bet te +aa ah +iso o +ðŁ¥ Ī +ha ar +la ine +b v +all day +spr out +indie games +free bie +gree ks +but ler +ill in +ha al +ware ness +si ma +public health +gam a +wa a +oun g +goo oo +okin awa +off enders +im pose +ho c +young ster +story teller +sc ap +figh ter ++ , +whit es +music monday +re za +go ducks +bri a +mi um +cas per +cru mbs +a ad +marti alarts +ch p +ri gged +tn g +harve sted +sa k +do jo +mill wall +b nw +oc d +histor yof +t mr +si rens +fan ci +caregi vers +vir a +son i +recur ring +acknowle dged +ðŁı Ł +oph ile +bu cky +stre ssing +roo k +di gger +vi val +san do +fle et +si ers +sel caday +refre shed +anti fa +a que +po lo +disappear ance +de mb +âĮļ ï¸ı +ren ted +ber ger +g mb +cu la +ss al +goo dy +u hh +marcel o +w anna +soft ware +shop small +turt le +tom as +fri sco +ðŁĺį ðŁĴķ +jim enez +c su +day z +an do +wyn ne +choreo grapher +cerv ical +trail blazers +ed g +zend aya +travel blog +el s +whole some +co g +lab out +ar ney +del le +su isse +ma si +ine se +om be +fi ddle +re claim +pa u +wat cher +sla in +ber ty +opti mum +el ites +min is +tur key +patro ls +ger ard +au reli +wild ly +wal tz +br gy +w ob +cre st ++ ++ +ve z +fro sted +davi do +the x +param edics +p into +han k +du pont +ur g +fo stering +micro poetry +spec tre +---- > +ne uro +fri da +music al +galve ston +e ffic +sc ape +pal azzo +th all +pro visional +p js +au re +ðŁĶ ľ +mam amoo +kit ties +cre e +wa k +lo ool +lu pus +cn blue +à º +ðŁİ ¬ +rac ed +tro se +om as +stri de +co ors +⤠µï¸ı +in comparable +cy ril +broad er +arec lipse +ðŁį Ķ +inter val +ti ru +co working +w aco +a ham +a bee +flouri sh +the times +ol ini +kick boxing +lu cer +at la +as un +casser ole +mi aw +lobb ying +jan ice +cir que +re flex +le ary +sanat omy +tem pest +se mb +mur dering +us av +ro bo +on et +p cc +nati ves +life of +sa ha +ruth less +rel ates +appeti zer +pye ongchang +nor d +er u +a thing +ug ly +pl ying +bran ce +organ ise +kend ra +dat o +chees es +par ma +burn out +a stra +pre toria +adjust ment +uk u +sl o +li ken +fav ors +cli ve +be ets +snow donia +go tv +sy n +open house +pan i +portra yed +sl ated +me cca +ren al +supportsmall streamers +staf fs +da o +bi ker +vik tor +tit us +admi red +ðŁĵ ± +hurric an +he ats +gl ory +photo genic +mer i +de por +burn ham +or angu +dj ing +impre ssionism +ign ition +ca i +w ynn +de pe +cove ted +colla gen +sau s +or nam +administr ators +ss on +nh politics +hahahaha hahahaha +aspir ations +r gb +swol len +so we +sc r +diver gent +hou ghton +han oi +d ory +ni ki +land ry +b cci +ðŁijĮ ðŁijĮ +is mail +tri pod +her d +bhat t +dress age +tab by +ingu ish +hur on +à³ į +à ł +to das +evangel ical +chor ds +st john +slo ppy +marty r +face book +ali ght +sen sei +kath niel +r ites +zi one +u o +revel ations +weight lifting +pan o +nc wx +ac ton +à® ķ +Ø ² +som a +à¸ Ĺ +respec ting +mar che +fore man +be tty +ki k +shi bu +po on +argy le +k swx +et z +mar bella +brac kets +stand by +fire side +defi ance +v ex +britanni a +in habit +appo int +piyu sh +le ash +sci ento +fla sk +sen na +> : +at roc +sand erson +id lib +dhan ush +ðŁĺ Ļ +en thr +hit ch +de dly +al ley +dor k +mon do +cudd ly +mis sin +ye sss +night ing +j pn +w ary +ump ire +ma z +ê ³ +bab s +ĭ ãģ +stan ford +posse ssed +exce eded +ðŁĶ ¶ +wall art +tra p +j il +hi bis +sp ying +scri be +khali l +trans lator +lu mb +di zed +ch c +super vision +shut ter +ja g +_ * +yester days +ms f +hi hi +gonz aga +gille spie +vive k +ec static +this morning +ch us +ed es +ston ed +be es +ðŁĩ¹ ðŁĩ +tur in +ho ver +at rics +ster n +sam heughan +auti sm +mi ya +eye witness +writ ings +travel tips +chut ney +px rtg +keny ans +my stic +k rit +/ $ +red head +world ly +am us +op la +le ve +gab bana +se en +o clock +gang a +keen an +sc ent +ol dies +go green +corner stone +comp ly +con cours +ðŁİ¶ ðŁİ¶ +ha an +con fis +aw son +cle op +î Ģ +su zu +sau té +al gar +subscri ber +este emed +ãĤ¤ ãĥ +worth while +mel rose +flo ck +bri ghtly +viol inist +p ere +sli pping +and co +si gh +ha van +cu lo +m sa +fibro sis +matil da +ra fting +aw ard +ë ª +mm mm +ge aux +ste iner +sin n +help ers +beet les +ai mee +tai wan +pistachi o +mac beth +m zan +descend ants +on sale +in r +il m +grou se +sa ig +mo w +bi gre +adjust ments +tu la +mathe w +transl ates +mu h +bol lah +ðŁĴĽ ðŁĴĻ +amo res +ab outs +bomb shell +bla ster +x avi +s ns +k roger +ga ther +erad ic +daf t +chem o +ben ches +ðŁĩ© ðŁĩ +ut v +our a +n ko +gator ade +biaf ra +ok state +im danielpadilla +dom ains +open ingday +kid do +do i +ric e +day care +mac millan +ba thurst +cheer leading +ðŁ¦ ģ +cash back +k won +hob bies +exem pl +ries ling +âļ ª +ag les +ny s +every thing +nav is +ad di +magne sium +faceli ft +ark ham +grand es +extre mist +don at +vit ality +pump kin +be tta +sl td +arti san +li by +pe aked +ah hhhh +mary am +assi m +un sc +ment e +al aya +low ers +ar as +gri ev +le ip +gr ati +cri ses +spr ints +exe cute +w to +ms d +mag ical +re viewer +spark les +juke box +ðŁĺĤ âĿ¤ï¸ı +pay back +licen ses +dun kin +bel t +lake wood +h ateful +bud gets +rev amped +ph erson +ky iv +went worth +ro sen +cru ise +gi ggle +def star +assassin scre +ym outh +win kle +w fc +band wagon +b kk +w iring +kear ney +south side +pe tit +! ðŁĺį +nor dic +mir za +mu gabe +v l +scon es +k tv +sand al +du c +m alls +ðŁĴŀ ðŁĴŀ +it c +al ay +im pair +un rest +flo ss +c é +ab ou +var ying +muse o +ser ver +di ya +hibis cus +ero y +mer ritt +fin dom +f pp +un usually +go tt +conting ent +ali aa +ball on +jo l +hi ked +zy me +ay r +ag n +ga z +perio dic +spar ty +practi sing +lin ton +tal is +cy pri +womanin biz +radio disney +ðŁĮ ¼ +jump ers +endo cr +ðŁļ¨ ðŁļ¨ +and on +shar apo +mi er +ma sonic +fac tories +vi en +bb ers +ìĽ IJ +hol d +ke bab +be ak +approach ed +ac milan +mun ro +ko sher +excell ency +negoti ation +walt disneyworld +cr ouch +te asing +suppre ssion +en ya +b ce +transformation tuesday +cal lie +vis was +p gat +ic ted +end ings +esc u +recru ited +it fc +collabor ations +g ino +snu ck +ausch witz +i fc +x ii +ke sha +ger vais +clo ak +x l +sa ad +prob ation +pre cau +mac in +anasta si +le k +e azy +daysof code +mariah carey +yo g +stit ched +boy friends +sh ar +ph ile +ag u +twin kle +phi shing +week ender +ic ton +gurmee tramrahim +al ton +l eness +all an +pen ultimate +kry stal +go u +lan de +dis mant +ab using +nor se +pat erson +ed mun +ap an +xi umin +sk el +cat walk +re act +wal led +t angle +br yn +ve to +super moon +cas ablanc +appreci ates +ski d +bo th +catal ina +ele ague +cyber monday +cau tious +ðŁ¤ ĵ +nov o +hamp ton +ha ye +jose f +var an +lo bos +roano ke +orph ans +tt in +squ ads +ishqba aaz +black panther +e tu +k sh +cru mble +cess na +reli eved +scul ly +pollin ators +explore canada +ki es +kam loops +kir an +pri mal +sett lements +hot spot +brain storming +ce dric +bi ennial +sh ant +âĻ¡âĻ¡ âĻ¡ +do on +hear n +walk way +fe m +ve al +deport ation +tox ins +elimin ating +descen ding +by the +bla sphe +ha sta +comple ment +as cent +ri ga +provo st +âĸ ª +wee ping +anti semitism +employe e +unearth ed +pin o +natali e +bla d +ang ola +lock heed +in ian +ag r +ni ster +im pala +m ke +fan atic +âĺħ âĺħ +ðŁij ¸ +lu ch +simpli fied +gall ery +econom ic +cy borg +con i +sel ma +in ception +ko ala +dv ds +cre sted +m mor +visi ble +n sd +ðŁĻĮ ðŁı½ +w under +refriger ator +re opening +e era +carou sel +as p +balli stic +victor y +mo tive +tre y +sharapo va +si i +mon ter +int end +west chester +sp e +cy mb +vi dal +ll ama +uni v +fin er +crafts manship +jazz fest +b ch +ag gio +n cc +lamb da +tranqu ility +cis co +ba den +so bbing +of i +go ta +ru mored +war med +ore an +ac ton +mar ci +gh ani +âľ ĵ +as sorted +pembro ke +pen elope +da f +at ty +aim o +pretz el +carni val +than os +ko chi +mer sal +ham radio +ar twit +cas c +guer rilla +kush ner +k app +al ise +todd lers +steward ship +o tti +ter ri +tem pe +rest less +vit o +zay ed +rsp b +pi on +hi ppo +haw thorne +in as +am ily +nut cracker +lo p +d ali +tro pic +ðŁ¤ ł +ul o +jare dle +py rene +pale o +usa ir +m ould +it ated +gene tically +biom ass +ðŁĩ³ðŁĩ ± +do dd +practic ed +monarch s +un manned +m buhari +am al +photo gra +ko ol +bren don +ju ices +cu re +world bank +poin ters +ðŁĴ Ŀ +tur f +le ds +bor ussia +bapti sm +warwick shire +moun ts +gay o +be gg +co pied +asi ans +k g +moder nist +gi d +front man +concentr ated +y t +sc avenger +iron ically +adi c +ps n +ðŁ¥ ī +cultur ally +yu v +mac arthur +fertili zer +be withyou +ri gor +min ors +z oning +âĸ ł +ri r +adole scent +vin ny +ren g +sand stone +gu et +we sth +ple dged +lac ed +sp ide +v ai +ty coon +seiz ure +du p +appalach ian +ro k +cathol ics +sey chel +posse ss +la ger +jo di +cham p +stra s +d ina +cent uri +cal der +blur ay +ðŁĩ¨ðŁĩ ³ +mo do +an nette +youtu bers +chap s +ang ling +label ing +a qui +pk wy +ly le +bi sexual +lit ur +dug out +li bby +grey sanatomy +sub stances +august us +rall ying +fi del +ing ue +äº º +hallmark channel +tooth brush +m á +adi rond +ag gi +ðŁĵį : +cru sade +tax ation +k z +i ver +dou bling +room ie +wa b +en rolled +az on +a ju +grand children +as df +ðŁ¥ º +mat ic +ough ton +utili ze +ðŁĴ £ +pon der +rais in +dys function +co bain +butter nut +e man +su red +dri an +and friends +with the +on omy +heine ken +bri dal +leader ship +pyram ids +deutsch land +jo cel +bo wel +y qr +horse power +be acon +ing eni +gra dient +fer mented +mo om +thing y +pot assi +wrist band +bor d +bo died +ðŁĺŃ ðŁĺį +ma pp +ka u +cyber punk +ph ish +loo king +co ates +ap ur +am ie +uk labour +at in +g la +adop table +shel by +v illi +ri ya +m ingly +cli mber +bumble bee +ðŁĺ ¸ +c sd +âĿ ¥ +hospit alized +c ki +hat er +ch r +re tina +it a +fan base +beat rice +gwy ne +go ss +fo s +favor ited +swachhb harat +mal ade +mon mouth +" [ +si van +sh hh +command ing +sains burys +wee d +g man +ss w +rep tile +iv y +tro pics +roll ers +over cast +ex position +masquer ade +man crush +wa ist +spr inter +sle et +le vin +j pg +_ ( +o pel +explo it +ap a +po we +wrec king +jong in +or b +er ick +bo sco +pra ising +ber tr +to wing +in security +ku t +resto cked +rr p +prescri bed +trafal gar +per t +g ases +app rais +g har +music als +âĸ¬ âĸ¬ +mc fad +ag ony +conditi on +equi p +shi k +atra vel +ðŁĩ¿ ðŁĩ¦ +ke h +abduc tion +pe oria +wil kins +g ms +as d +ev i +ðŁĴĹ ðŁĴĹðŁĴĹ +u z +mo c +halle lujah +guad alu +lou vre +dra wing +go ve +ph ant +fri e +web dev +program mer +z able +games com +clari fy +li th +kin ky +âĿ £ +labour doorstep +son ata +ju ris +mai den +vi adu +buch arest +conditi oned +capit alist +u de +ps b +sp ca +lul la +footh ills +kay o +bon d +wom b +roun der +ce sar +bur sts +ap ra +sw oon +sab rin +fra grant +cle arer +ku brick +cli max +jour no +ag le +ðŁı½ âĢįâĻĢï¸ı +poo ch +hal e +sol it +sal mon +organis ms +bron son +art en +hodg son +alo ve +vent ure +bb i +ae a +ðŁIJ ¢ +ld n +d nr +o zone +el las +man ny +azz ur +un beat +tru ffles +th ong +ma ñ +las ers +ley e +gettys burg +back packs +or is +ma ison +craw ling +la bra +cl ing +dra gging +ste al +dou bt +de van +ck ers +agent sof +photo bomb +elon musk +abo y +dist ances +story line +sp i +nor than +europe ans +wh ale +ser pent +ðŁļ ² +fi or +tr it +ox o +awar ding +class mate +su fc +smar test +rich es +pr k +big foot +ar mb +bi polar +dw elling +om ars +k wan +gri me +m eng +freder ick +navar ro +sorry notsorry +jaredle to +pa ve +sl ack +barn sley +att ar +evic tion +accumul ation +o ir +cat chy +wel ter +vik as +has see +nik ita +mo yes +mathe ws +shi v +gat wick +pro filing +compan ions +mar rake +an tics +ðŁĻĮðŁĻĮ ðŁĻĮ +se se +bo i +bart lett +poison ous +ab uses +ym m +kam pala +guggen heim +imv kohli +dol om +bre e +thro ttle +gare th +fitz patrick +un ya +par ad +mar got +j nr +we a +potassi um +p nc +disgu ised +cra sh +ren ergy +ill ic +coup led +ni els +ci ones +æĹ ¥ +im ent +despic able +d ye +what cha +conne ctions +paralym pics +gaunt let +wait rose +suici dal +star ship +vap or +st ou +law maker +coo led +si mo +then o +offro ad +ja den +bas que +vick y +lu kaku +centr o +tri sh +strate gist +medic ations +hor st +b fc +gra il +sharp ly +ad itya +tom b +kau fman +tri pad +sam ba +pastor al +brit ney +sag an +hill side +mas ons +sar a +z one +x u +to tes +rob bie +app en +mon tag +der o +short film +charis matic +tat ors +ki ba +and ri +al arming +split ting +ic ar +th ug +scari est +sylve ster +an an +u trecht +a difference +me ade +bu ster +air strikes +cu ffs +account ants +ðŁĺ¡ ðŁĺ¡ +new t +bo tt +issu ing +cl ancy +wwen etwork +kyu hyun +rese mble +pajam as +sin k +kin ney +sul ph +or k +li es +la gh +or ton +ra hul +d sc +we will +re am +collo qui +shar ia +hec tic +sar casm +land er +tm z +endor f +ro z +ham mered +fri s +w adi +pope francis +he it +flash light +un born +op es +hol iness +ðŁIJ ¦ +nach t +im sa +gr acing +bj p +ver ts +c sc +home owner +a que +bigo try +anni e +bag h +âĿ¤ï¸ı ðŁĺį +car i +thom p +dispo sable +cardio logy +pat ented +hh hhhh +ld r +stephen son +cro res +fan ning +cli mat +ðŁijį ðŁijįðŁijį +ðŁijį ðŁı¼ +aer on +piccad illy +bank rupt +sil via +emplo y +don ny +commen ting +screen writer +io ta +ce an +anc ers +tu an +street wear +ठ¯ +sk ine +esp a +asi f +os ce +she ppard +more cam +bott le +der s +orac le +google play +aver aged +edmon ton +steph an +sister hood +cru sted +stag gering +methodo logy +congress woman +c abo +tri ggers +mil ky +gli de +tooth paste +room mates +nu ff +gu am +sprink les +alternati ve +wat fordfc +uof t +hal ey +cont acted +bun dy +pro stitu +gh ar +pre ston +on site +hil ar +g ts +c att +hamp stead +? ?! +ðŁĩ§ ðŁĩ +bbc qt +aless andro +resi st +ma idan +t ko +shad ing +pin up +gal lo +sin u +at ec +fun k +ac lu +stri des +rhy me +wet land +bbc springwatch +t ins +wild card +st our +flamen co +pau la +onto logy +gang sta +am ade +ãĤ « +t bs +skelet al +run ner +jard in +harri er +hun ted +z hen +believein film +de mean +au diti +re start +chon dri +âĿ¤ï¸ı ðŁĴĻ +mcla ren +ga b +sh um +au sa +lewi sham +y pg +k jv +fur nished +dor o +bon ded +mor ty +lat itude +_ ) +lo va +water ways +vin ai +shor th +drun k +c ay +ay ana +kap lan +capp uccino +spr o +life boat +has bro +spol ice +tor on +do ing +dam n +sh ree +foun tains +ent ation +mar u +boar der +to pless +j ada +chan ning +ul ls +en closure +gib son +fractu red +brit ton +à ¶ +t ous +por th +dra f +tra iling +mar gate +eli fe +down ward +lin n +gla des +girl power +ak rish +u ki +ron da +ts c +appreci ationday +vis ing +lo om +ðŁį ³ +mex ican +ar gos +y ya +jad ine +south port +d end +si sta +rede em +men g +bra xton +antioxid ant +s key +mp g +fin ding +vibr ation +ce u +kh art +di mini +cl ine +shel ly +hin es +ī ï¸ı +to pical +no ver +ma xx +prim itive +illustr ate +b ounds +tren ton +join tly +breed ers +u chi +wakeup america +b ada +ðŁĹ £ï¸ı +gu acam +sp heres +pere gr +youth ful +lo lo +bir min +t ly +jeremy corbyn +defe cts +co sm +a rent +v aa +bag els +medi ac +cori ander +ic ago +g haz +ab bas +re model +struc turing +pu m +out law +ad ani +r bc +gul ls +n li +confu se +ðŁijĩ ðŁı¼ +vil a +mcnam ara +correc tions +mug hal +ser i +re gain +ss b +lea ve +haha hah +gran de +di stressed +re chargeable +ho a +hou sed +sti l +attribu ted +opath ic +di ps +pri t +head phone +conclu de +pil o +he t +ut sa +nit in +je m +sni ppet +tutor ing +op er +sun k +en sla +cha u +ac orn +quinte ss +ran kin +affili ated +our lives +cl int +se ater +isa ac +ba shing +sme ar +nur se +doo dling +" ; +sa ku +atroc ities +im am +g fs +viol ating +comm end +brad shaw +er ville +b illed +b be +thul hu +i phones +moo se +di os +re w +me thane +strang ely +whis ky +ti ghtly +spiel berg +radi us +notic ing +wi f +ig nati +i fa +ap is +w ali +ha itian +bu shes +y z +v l +ex ited +asse l +tru ec +dom en +ash er +in king +newyear seve +hend ricks +bat i +ìĿ´ ì +rich ter +mon santo +con line +agre at +ðŁ¤ ¯ +master pieces +ar n +rough s +cle ve +se v +fashi ons +to ya +sh ail +cop eland +aqu ari +dec als +are you +y aya +a str +fon t +ml m +ar ca +pp or +pol lock +xper ia +conserv ation +chain saw +ag gie +?! ?!? +si le +sh on +ìĹ IJ +note books +marque tte +de us +bb led +spic er +mc cabe +nor wich +modi fication +boo sted +stru m +sales man +bang le +nis san +hez bollah +brea sts +a af +anth us +sk er +ow ed +her os +gi fs +fo sters +eat ers +du es +_ / +lymph oma +sf am +me gal +afri di +ag ic +p amp +jeal ousy +ðŁijĮ ðŁı¼ +calcul ate +napp ing +g ale +ðŁ¦ Ħ +lub bock +assu med +ren ting +íĥ ľ +subur b +ãĤ · +tech nic +u cla +in front +gar net +ster oids +stri ving +ho war +mo ver +le ton +bull do +is in +ci ao +sn z +fore front +d ams +mid wife +ma wards +cla pton +we in +subsi dies +spr oud +rother ham +phan tom +ar ach +spi el +rac ket +sel amat +no on +l bc +enti ally +ðŁĴ ¸ +sil ve +m oud +kine tic +y asi +ðŁİ © +o ol +mi ku +i za +fer a +flo ren +barber shop +groo t +z est +ne ars +stan is +z and +police man +juris dic +form ations +appar atus +sp d +arti fact +to sc +motiv ating +womanc rush +re dro +diagno stics +ra za +out fitters +el xn +dod gy +ry n +sh d +ortho don +ol de +jay anti +bal ances +quic kest +can ton +friday reads +! * +na a +a ak +ðŁĶ · +behavi ors +rasp berries +ä » +polit ical +cam il +å ľ +di k +ast ounding +lie be +novel ty +tur moil +sul ly +spring break +hon ouring +cc g +ðŁı Ĵ +my little +ky c +pro ms +ðŁķ Ĭ +à ¨ +bi ge +av ril +ðŁĩµðŁĩ ° +mari on +as ants +sur ya +oc tag +luf than +ac ron +fayette ville +ti que +love s +en ca +de kalb +ta ver +de vote +aux iliary +joh annes +tread mill +ay an +qu r +donald son +cher yl +" .... +s ven +kir sty +gun ners +ra dish +o ahu +v sky +i ble +con course +b ps +elo qu +ash ford +te bow +roblo x +ma da +dri ving +th day +spro ject +m ms +band ed +. !! +libr arians +flan nel +intoler ance +her al +ç µ +neme sis +list a +tar ak +cry pt +star plus +vish nu +sc ale +cr is +% ), +j illian +regg ae +pegas us +ol in +ip ment +man ic +l fc +godd ard +ite am +parl our +anch ors +lee minho +talla hassee +ant it +d ho +kid ney +y ash +batt led +az ad +gar is +faul kner +sni ff +papar azzi +ed m +phy llis +con tested +aa ay +se ca +k ton +vel ve +rain ier +for um +tam pab +ho sp +trac tors +ox fordshire +no tion +guang zhou +ðŁĺ ¯ +ref ill +wednesday motivation +sli der +mukher jee +pr att +fon taine +alph on +af ar +ts i +pest icides +fi ends +mo cking +bra w +tran sat +do ses +co res +hom ophobia +docu menting +zlat an +con doms +s é +sun set +kun st +ton ga +ภª +v ation +sp ray +chow der +ra ps +palla dium +nor wood +music history +hoo ker +si si +osp rey +ph ys +conce ded +bob cat +ar mad +ze it +Ù Ħ +ðŁĺģ ðŁĺģ +mer idi +ðŁĩ· ðŁĩº +corn wall +! ), +touch downs +ze it +chal et +mm m +al che +gor illa +fo ss +ati ku +lumin ous +ivan ka +be ek +sta res +sw iss +âĿ¤âĿ¤ âĿ¤âĿ¤ +scru bs +me ath +gusta v +jo gging +confe tti +as os +ers fc +breit bart +applic able +autho red +ya ho +h in +displac ement +j v +ðŁĮ¹ ðŁĮ¹ +ot c +non profits +diec ast +gu sto +inte stin +c ages +me en +lu kas +moon ey +ðŁĺ · +very day +tor ah +is sion +wa c +lever aging +ish able +cu se +le wood +may an +turn table +ju ice +tru sty +tu p +eti quette +supervis ors +stu n +gu zman +confe ren +ric o +fe ast +back ward +pol aris +mic he +jo g +h ing +field house +vel ing +sho cker +esc ence +ठ¾ +vi be +anasta sia +mar ched +kill ing +Ķ ë +fe tt +exop lan +... ( +snow day +lo h +ir ani +la khs +del a +po caly +boom ers +dictat orship +ac er +tur keys +quarter final +muskete ers +ðŁĴĽ ðŁĴļ +sf x +museum week +sc ala +ri sis +( ðŁĵ· +ãĢ Ĥ +z ies +bo eh +hu es +lu sci +dol a +impeach trump +roo d +don caster +tor re +hero es +fo yer +tar i +blur red +ke w +frank ly +dro id +ap al +Ð ¼ +y af +bre t +par agu +cac ao +ðŁĻĮ ðŁı¾ +ru e +head aches +shaw ty +char ley +pal er +go wns +correc tional +ðŁĺ© ðŁĺ© +breaking bad +ol ing +da p +endeav our +cit adel +tra d +incumb ent +medit ate +foo ted +ðŁĴ µ +shab bat +dayof the +wil lem +gal way +to red +marri age +f illion +sleeve less +aud itor +jin young +invin cible +kad una +a and +volcan oes +mon eti +indie gogo +buccane ers +ðŁijī ðŁı½ +ãĢ Ĥ +lay ton +cuck oo +hu mber +buzz er +Ï ī +to re +stra ins +sto m +pa ine +s we +du ff +z ou +si mi +li pp +ur n +se agu +ðŁĶ ® +sun dae +hi c +ðŁĺ ¨ +bull pen +u per +flyo ver +al dridge +glo bes +ali es +ken zie +ge es +y cle +sp lin +mag enta +j ha +bal u +gh orn +ti pper +wick er +taste of +con clave +ch ale +inv asi +cat er +dio xide +me gab +win n +at p +transform ative +nest led +hi g +bri dging +lil ies +chee red +bad dest +sc rolls +real is +dipl o +ðŁĶ « +conce ssion +prefe rences +explo des +er gon +introduc tory +ine au +ch af +som es +land rover +spir ation +sex y +sco recard +illustr ates +soul mate +wi en +inter disciplinary +fore casting +ent ities +glu ed +en lar +cur t +percep tions +boot leg +mi re +asho k +v az +hor ne +cal le +ac ulture +ther oy +night time +oc al +character design +ar mist +ðŁĺı ðŁĺı +yah oo +ac eae +to se +even to +sou t +nay anth +wh om +v are +ri gging +gen us +hi ve +com mands +sti e +day a +ethan ol +en f +hi fi +flu ence +cle mson +re invent +thermom eter +humor ous +emer ging +aci ón +ðŁĺĺ ðŁĺį +s ity +haw ke +accompan ying +t ility +ðŁĺ ª +re cess +protag onist +l ery +dun dal +int l +britt any +q bs +off the +marri ages +how to +viol ated +adel aide +wit t +lanc er +pak v +hu me +st ade +bra gging +ou tright +ad c +super st +real time +cu res +garden ers +ero ck +dale jr +ver o +bar tol +mo ti +mc fly +v pn +st ink +over rated +guer ra +e tis +ath ome +twd family +th ab +tn x +rafa el +family travel +x ley +sat anic +equ ations +ru dy +wal dorf +stan i +tu be +meas les +zimmer man +obli gations +i ously +bow ser +trans former +sho ppe +shak en +gh ouse +to d +ke tball +share holder +mar ca +kp mg +ak an +given chy +coast al +au th +roller coaster +mar ches +coordin ate +cine ma +apprentic es +par lor +mit o +men on +consider able +bar re +glo ss +enh ances +jaz eera +fal mouth +thra sh +stat en +k zn +eng el +samanth ap +flo ppy +sal om +ðŁıĨ ðŁıĨ +w ack +deliber ate +osc ill +herit ag +du sted +orni thology +pad dle +fer ns +bar un +cl ans +anticip ate +a ay +mat ically +é ĩ +tu mble +post man +unic ef +tro tter +op d +leaf let +ge ist +cease fire +scre ws +cre ation +wal nuts +longh orns +under statement +ab b +proxim ity +na x +un ity +turn pike +orda ined +dub step +chak ra +me ch +love her +look alike +donne in +vir on +Ù Ī +bang ers +vari ants +out dated +in ta +cri sto +sp elt +food and +f on +stefan i +margin al +hu tton +ti ara +tel ford +qu en +fair grounds +que tta +mikha il +heal er +v ball +ty re +under grad +gl end +hom ers +scri bed +main tains +po che +mis sal +mar ko +u as +á n +sh p +con vey +pad re +sab a +pu glia +madhu ri +pa xton +chap lain +n ago +ca si +... !!! +fli rt +sal eh +k are +di re +stam ped +extre me +ðŁĺĥ ðŁĺĥ +ho ppy +guadalu pe +advant aged +eu char +p low +un n +mac qu +port land +cla sh +pe s +lou bout +y p +keep ing +arca dia +fran kie +fi u +de th +encyclo pedia +si ze +inve sts +ðŁį © +geo logical +fran ç +con front +ðŁĺ ¥ +d ys +af m +tex an +graph ene +repost app +ac f +ur sula +gaz a +dd led +fu m +wsb tv +m be +fron tiers +chrono graph +ke s +inter faith +tab oo +spar ta +won do +flori st +em braces +ca w +no el +arch ers +ðŁIJ · +roman o +ban an +sh akers +melo dies +geo thermal +se phora +ìļ ° +оР´ +pro c +hand shake +pan de +popul ated +slow down +hor tons +registr ations +un deni +lan ts +pas sover +thak ur +li ef +adhe sive +pe tal +micro scopy +memph is +confir ming +air drop +mesm er +perce ived +ming le +lifel ine +gh j +worcester shire +pas sions +ach er +el lar +ah o +firen ze +bar ang +letter man +hat field +lu cha +je ter +e shop +william s +horo scope +pre de +east bourne +dur ga +di version +al trin +seis mic +premi osm +nar co +ti r +ori g +or m +land fall +ci ous +lin do +max ine +x ico +tra y +os wald +c ba +ric otta +n cr +mar au +ภ² +gladi ator +ch ery +lun g +u me +po psic +lon ging +can als +ta ya +decentr alized +sho pp +pres sures +mahar aj +eti had +wal greens +succe ssion +sign aling +li g +staf fer +north korea +def ying +as ma +de g +peri meter +oak ville +m sk +balti more +rece ip +de ple +ðŁĺŃ ðŁĺĤ +jambo ree +> .< +rsp b +puni sher +consider ably +in tothe +pari sian +acceler ated +polye ster +low es +fr ying +sauté ed +mou ths +seychel les +ra x +go dis +dak ota +house wives +the me +mat inee +black bird +ye sung +pre fers +pelle gr +in ated +trun ks +stronger together +re pet +re pairing +ped als +toler ant +her r +dun ne +indic ation +decat ur +b tv +exhibit ors +ik on +friday motivation +bra gg +live tweet +al ves +womens art +foreig ners +wal lets +min dy +lan ey +bb in +tv miaw +lif ter +tar get +tam e +dr ou +astro photography +mp c +g pu +nord strom +fric tion +run off +lov able +sp nfamily +ext ingui +bloo dy +sch el +arti stry +sw ish +scar ce +ph ils +max im +pos sum +com promised +sty li +sc fc +is sa +birmin gham +sket ched +angel ica +ordin ance +je ts +conqu er +ðŁĺ IJ +online shopping +s ori +reason ably +nue stro +ar turo +ch l +benef ici +spho to +wel t +ni kk +ðŁ¤ ŀ +dan ao +for mid +as se +af irst +âľ Ĥ +gil lette +as sor +an onym +sel ca +fe mi +bear able +y and +ar mory +cre pe +celtic fc +bra vo +in expensive +de lec +ge cko +new market +snow flakes +kab ir +con tra +can ning +mor pho +gar wal +ðŁĴĥ ðŁı» +fight ing +mu tation +woo dy +ju gg +gr aces +premiosm tvmiaw +kenne dy +gu p +sa e +op ha +off spring +fini sher +bet ts +span ning +mar j +h one +sh ing +contin ents +samanthap rabhu +un related +l acy +explo sions +benjam in +sophi e +no ting +micro soft +as sen +a hoy +i ker +ho fer +mo e +ah madi +yan n +an ak +ma hi +be u +aha h +creep er +baahu bali +am at +pri ory +haw keye +deloit te +sko da +print making +assemb ling +mirac ulous +no ch +sw o +leg a +oper ates +border lands +eli e +stron gh +rep tiles +pir ate +un fold + ¯ +qual comm +un predictable +ot r +rose wood +direc tional +counsel ors +corn ell +liber ated +j ad +ir regular +bulgar ian +high ness +vodaf one +sw ild +mini mize +gra zie +๠ĩ +r stats +stre ep +ome tric +humb le +lu mp +l ille +b ü +home depot +tripad visor +ki wan +a via +er z +ex ico +du f +blu men +mi zing +ar ma +in im +con stan +sor a +ju al +au n +tw ell +tren ches +her a +r k +po plar +recipe oftheday +ll an +bhu ban +short ages +ing don +bridge water +ðŁIJ ĺ +fortn ite +cam den +un cture +pro w +colon ies +t ks +n go +b hm +live pd +spl ace +sli ke +happye aster +ter rence +revol ver +j ed +yy yy +office of +m ts +exist ential +r ourke +explore bc +sse d +pri est +vix en +si ding +k pa +a har +ju ic +ob struc +foren sics +uk mfg +cancell ation +we ary +ab q +ele c +pri zed +deb ts +me zz +salv atore +m dc +gre tte +c gc +th on +snow storm +ts ch +cook ery +å ¹ +wa xing +n acional +mur s +ra ve +cap es +ger main +dri pping +sub mitting +ome lette +iter ation +aj es +shim mer +fu eling +ðŁĩ§ ðŁĩª +li po +bo bble +un follow +islam ist +hi ber +cat s +agentsof shield +sen si +____ _ +ster ia +inst al +ausp icious +har row +over land +femini sts +inst ant +char iot +blind ness +sp ed +sc arec +nu it +mini atures +ho seok +glo ck +fifa worldcup +e te +dis m +we iner +ex foli +ear ts +ภĶ +my art +man il +iss ant +form a +in cu +buffal ob +in tim +mc cul +anj ali +po po +un doub +hil a +fun gal +thank ful +fu tur +en dish +ren ds +th ar +she ff +ring o +nichol ls +io wa +po tom +cl ams +ãģ Ħ +acon f +stadi ums +di mp +di k +residen ces +do v +caric ature +seagu ll +kl m +confe ss +sla pped +cele b +turb ines +pp v +nur ture +el ab +.... .# +tu ff +de press +al far +amii bo +di spon +e wing +que er +friend s +for re +âĺ ¼ +sw t +aqu arius +head liner +cur d +fi gs +o tters +love fl +kare em +go vegan +fri yay +consol ation +at ri +ì§ Ħ +âĺĿ ï¸ı +poly ne +gu ed +o ya +la us +intestin al +cam illa +scal p +pi r +leed s +horri fying +bore tum +dand elion +fer rer +ell ic +as x +so ren +re loaded +ale ague +navig ator +ine tte +add ams +al chemist +ak shay +dystop ian +awe c +n aya +al isa +ai led +ag or +avi ator +ali zer +smo bile +findyour park +cop ying +to ddy +sh ti +mon ger +cal houn +nap kin +break up +y atra +se thu +ric hi +eras mus +fer ry +am ore +prac tise +bo bo +power point +oo se +li ffe +chin a +sh ka +fad navis +du ane +war on +fal se +ðŁļ Ĥ +wa shes +disc ip +==== ==== +g k +ab b +stub born +medi eval +p ci +ðŁį ª +maril yn +h yo +man di +cr i +prede cess +continu ation +om usic +s lat +wh al +mall ory +bon n +shen zhen +ca i +âĺ ĥ +sa fest +for wards +dra wers +bla sted +sle e +mor phe +mb ta +dumb ass +ÑĦоÑĤ о +alhamdulil lah +ec lub +al beit +heal ey +ayurve da +adverti sed +cro cs +itt les +bry son +be i +nj pw +honore e +fu sed +ðŁĶ ĺ +mul tin +n aga +de parts +ko p +kin o +jhar khand +ed na +ax le +mil ton +supremac ist +marrake ch +domin ic +tran script +] [# +: ). +wo c +sur rounds +o gil +leaf lets +co well +whe w +tru de +proli fer +succe s +sports man +con dom +po che +k up +imprison ment +{ } +scram bled +å Ľ +ka ine +cell phone +metam or +con i +remn ants +ee z +down pour +afterno on +exerc ising +ber ser +architec ture +wick low +m ns +is p +bo c +n iss +mn wild +stu mble +r si +lu ffy +sil en +dd ad +bul lies +haw ker +bb cc +scu ba +e pp +que ts +for aging +pal let +ha di +cinemato grapher +cat chers +to aster +k hi +lite coin +kid lit +amher st +maur icio +ip ad +mar malade +fe y +don nelly +g to +est as +cere bral +ant grasso +zz led +vir gil +swa pped +ðŁĺħ ðŁĺħ +no dapl +greate st +nhl bruins +fra ser +b mo +ane w +. âĿ¤ï¸ı +se gregation +remark ably +mccor mick +lo gger +er as +contrac ting +âłĢ âłĢ +yor ks +uku lele +touch screen +de cked +ben n +south wark +ra vin +nu mis +ðŁ¤ Ļ +ru t +gre co +eth ic +red neck +ar r +t cs +ih ri +ðŁĩ« ðŁĩ· +l k +inher ited +zy k +viadu ct +marty red +hi gu +ss n +be in +street style +fer gie +bank of +æĹ ¥ +stake holder +exempl ary +cre ss +ess a +ero tica +intre pid +gom es +bra un +bethan y +bang tan +pulmon ary +m illing +doctor ate +trump russia +ठ° +s ani +bl att +pla u +depri ved +t le +ful ly +bour n +st ak +lufthan sa +kio sk +far oo +def y +bad an +ðŁĺĺ âĿ¤ï¸ı +rit z +tri sha +ran ds +middle sex +arab s +pro j +sport scenter +repe ats +iv f +bleed blue +as sure +o bs +territ orial +ele n +bever ley +ann ah +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +z l +for good +science fiction +gla u +son ya +pri th +st weets +mix ers +mari o +ant elope +writing community +went z +den ham +be di +sf o +harley davidson +look book +immuno therapy +or phe +es ville +ed ged +tas k +sb ball +corro sion +kilom eters +co sting +play back +ke ke +di visi +u ter +re location +yel led +pen g +up beat +ser ve +âļ ł +hal en +stir ring +reh man +en v +schu macher +frag ment +alkal ine +sb k +resil i +share point +rol lover +tra sh +counter part +âĻ « +ob itu +à ½ +ãĤ ¹ +mul berry +ðŁİ Ĩ +auton omy +spra ying +nat l +love you +fran ki +nu k +esc ar +can teen +ali baba +de plor +mole cule +pu d +fort night +blon die +sp hin +portra yal +ta che +bu te +consi sting +freep alestine +c sp +im mort +d ns +ðŁĴ¥ ðŁĴ¥ +tour de +coo king +archi val +ga thers +bit t +b anc +pre mature +snow ball +poetry day +lou dly +fug itive +ed ay +em ra +ðŁĩ¸ ðŁĩª +sci en +node js +jur gen +je ong +band ana +un is +fox sports +v andy +pro visions +wee p +tu k +i ko +h oun +zig gy +z r +fil let +bat a +tin k +con e +we want +k ilo +hor ace +sl t +sc t +stay tuned +victor ia +umb ria +att acker +ingham shire +fright ening +no ir +fr at +con tempt +lia ison +ho i +br ink +tr ill +ni agar +kick ass +dun das +not my +rho de +bu mble +no xi +fa g +spec tators +mancrush monday +jin ping +distr act +dais y +wal den +portra it +ar thistory +vol tron +ev el +is c +ac m +r ite +na o +de ported +swe ats +ru fus +lo bo +labor day +gam o +ihri thik +bl it +abdomin al +ãħ¤ãħ¤ ãħ¤ãħ¤ +i it +e q +bu sy +allu arjun +un disclosed +de ton +pro create +ki l +ðŁİĤ ðŁİĤ +mitch ell +ki i +inherit ance +al p +jo burg +pat rolling +compul sory +un signed +ni am +l ga +eshop suk +tr illi +ma w +appreci ating +rock ab +mañ ana +an tal +mal vern +roy o +grand prix +sut ton +go ftheday +dig i +ãħĭãħĭ ãħĭãħĭ +t les +varan asi +erec ted +discip les +cont act +ðŁĺ µ +li d +⬠ĩ +scen tre +radi ator +ing tips +trans itions +thursday motivation +chem ical +separ ati +sal is +mi m +geo graphical +book fest +/ . +âľ ĭ +v ae +cur rie +ag garwal +acceler ation +the ses +lg m +u mass +pro portions +nat a +ani ans +ku ch +be acons +ap r +@ # +ðŁĴª ðŁı¾ +nu ke +sher aton +ki o +ma kati +polit ico +mor ale +ì Ļ +econom ically +gg ly +ss en +pa stries +intern ships +vic ente +fanta ken +aveng ers +accu se +slee pover +indic ated +the dream +ster one +ren ders +fro st +ou i +gre gg +d ore +⾨ ⾨⾨ +pu gs +sat y +nu mb +hems worth +tam i +la ssic +schi ff +igle sias +ag awa +] " +re shi +game stop +divor ced +theat er +clau di +un conventional +prophe ts +ac in +twel f +tow ering +t ml +sc lerosis +k wan +ge ts +distur b +na ira +ener g +pir acy +pru itt +noti fied +hen na +bra m +ground water +bl s +opti mis +$ ) +luci e +biz hour +fang irling +gr ills +or l +ver se +c ina +law less +artistson twitter +tele vised +marshmal lows +radio head +bar r +m fc +bre vi +mmor pg +g aya +âĸ « +sub titles +j t +disney land +to bago +nh m +groo ve +fi awec +" / +ba o +scra bble +om ni +ff l +um c +si mba +ali er +ter rell +plu me +mi di +dig nit +co c +bru t +ad ata +alche my +d sm +ðŁĺĨ ðŁĺĨ +win try +spa res +cu er +conclu sions +to ys +od or +fl ann +gar vey +scrip tions +inspec tions +cat ap +ang lo +st louis +heim er +at ay +tr ich +en yc +chil ds +vent il +mont p +guiller mo +circu lare +z ell +mode led +craf tsman +al ina +stimul ation +cashe w +ju das +best of +to ire +susp ends +scol lege +real ising +by tes +bloo ds +as si +ðŁĴ ¿ +o hs +ðŁį ĭ +scallo p +ठµ +gi fting +camo gie +wil kes +o zzy +ðŁ¤ ¤ +ver onic +sav oy +deme tri +baby girl +ðŁĺį ðŁĺŃ +so x +cly de +induc tee +count down +self care +ठľ +vi ka +tor re +phd chat +pe ars +aw h +suff rage +le sn +admir ation +mp p +shark week +schul z +santor ini +clo ver +( * +stras bourg +ex iting +so yu +finger print +che a +ãĢ ľ +vin dic +song writers +so a +prou der +nam a += )) +simple st +delici ously +gil les +u q +mn wx +ep p +sh un +ken nel +fall on +ðŁIJ £ +sin d +tra gically +out es +modern ism +co ke +gy n +spi on +âĺ¹ ï¸ı +le am +compress or +apolog ise +twent yon +fan atics +âĻ » +sco tsman +sa wa +ko u +as er +ภļ +welter weight +phen om +twick enham +stri a +p out +ka z +gi am +cd p +ho y +emplo y +red mond +ภĦภ+sm ere +trance family +proto cols +pie ce +lu iz +iter acy +carl s +united states +har med +phd life +ch aw +foot prints +l é +cho ker +z ana +sli pper +eric sson +insul ting +articho ke +advis ing +acquis itions +op or +mut ations +re ar +ॠģ +pod cast +wi ther +kun g +íĺ ¸ +win slow +di apers +ðŁĵ¸ @ +ec ker +col lar +hu ey +gi ro +mono gram +kas ich +si veness +malay si +arom atic +gre s +gali leo +u ji +rob b +dr m +none theless +as a +: > +lo a +l np +at work +ag t +laksh mi +pipel ines +id al +stre l +re all +chain z +stone wall +san sk +ðŁı ´ +pied mont +hoste ss +ci u +t é +analy ses +wil helm +scott y +rw by +mosqu it +use mb +qu ins +ðŁij İ +tu cker +s conf +speci fications +psychi atry +broo kes +s ils +ol af +de to +co di +cli p +fil th +womancrush wednesday +go to +ang erous +be ale +w tc +paneli st +ne x +lar sen +emili o +tab leau +h itters +conce ived +americ ani +or tega +mar di +Ñ ĥ +pain tball +thir sty +new yorker +etis ation +go ss +we aker +u gh +tro ll +har ga +du al +ght ning +at ine +ðŁĺİ ðŁĺİðŁĺİ +cook out +pyrene es +po ss +authent ication +sports wear +yun ho +kir o +archi pel +shen ko +ren der +nov ation +divin ity +ðŁij £ +su fi +humb ling +ge opol +devote es +wait ress +tr ough +py ro +i ba +bl ing +gra f +epilo ts +bt r +of tball +bas king +domin os +so om +r ath +sher yl +qu el +astronom ical +wel d +track list +sig nee +slee pless +com man +ch ron +summ on +pure michigan +cri spr +sli p +la gi +ra q +um u +thal ap +char med +scru mp +quad copter +ski p +peter sen +mun i +ðŁĮ ¾ +mon aghan +tra ys +ick ed +canad aday +te gr +ï¿ ½ +hot ness +heavy metal +ab ar +gop debate +az ul +spider man +sun flowers +ľ ë +web comics +bar d +Ð ² +nichol as +slu sh +ram an +mark ham +ffici al +ff ler +íĬ ¸ +ple ss +anush ka +to to +sk aters +pro wrestling +compet es +ay ala +myster y +thr ills +mp g +independ ently +y ul +imper ative +formid able +tire less +st acking +ton gues +mal tese +pot ts +mat ti +char ting +chill out +super nova +ome o +sky sports +nu tty +ðŁĹĵ ï¸ı +ro han +insp ired +concier ge +ser ra +ma kk +gal at +chi pp +ye v +ì £ +reim bur +op ul +kimber ley +i eee +bre men +ch itec +or in +nak u +bon kers +foo ty +emer gence +ðŁĨ ĺ +sti p +serge i +zo ey +ai me +wou ld +dy es +destin y +vinai grette +dri er +circulare conomy +an archi +ss r +sch el +cin er +gro om +determin ing +gar min +cal ais +incarcer ation +bu kit +no i +chelms ford +mckin ley +chi pped +belong ed +tu mors +str oud +mi i +influen za +wwen xt +tun dra +tele communications +cat sofinstagram +t ages +beat ty +o du +ml kday +oo per +dang le +ak ley +cru mb +anti gua +ti mbers +rou hani +ðŁĴª ðŁĴªðŁĴª +ha fi +... !! +w cs +coo p +sn c +lit res +ãĢ Ĭ +ha z +co z +k ant +green field +cur ti +y ale +flye agles +what soever +wor thing +rou lette +flyeagles fly +un da +a inted +stand ing +lusci ous +h pc +effic acy +ash land +me ghan +ky wx +n pr +bath tub +ac os +h ani +mar cor +man tis +da isi +bo ba +ab bie +mu til +vi al +spy der +po z +g ti +el fie +nigh tw +metro id +anton i +mad die +dh ry +dar lings +ten ds +taek wondo +atlan ta +me ow +chlo e +ãĥ İ +ym es +siber ia +k con +gu es +mar iner +fac il +azz le +[ ... +han nover +bav aria +vir go +te uk +u sps +) # +wall a +sam pson +need less +ver bally +hay ley +bow led +pi us +lam pard +ham string +vol vo +road safety +cho king +sor bet +a hem +healthy food +brai ded +horticul ture +cr ative +che ek +ad do +the force +ko ko +schiz oph +j ie +w ada +twentyon epilots +h bcu +pro ton +pau ls +lou isa +lat am +kyr gy +com pac +sd k +sap i +?? ? +liber alism +ep silon +ai den +w usa +spra yed +baske tball +kim ono +blue wave +ali as +ë§ Ī +mug shot +ce c +do gre +ad ora +ðŁĵ· @ +kra kow +intrigu ed +exhau sting +astron omer +ven ison +lady bug +ci v +bra e +us m +bri be +acup uncture +pembro ke +ke ating +chi e +y ad +t si +sm i +see ding +gate shead +lis boa +gy p +canv ass +ðŁĶ´ âļªï¸ı +op i +ni r +soci etal +ly te +ati es +c sm +ar tery +al in +aka poor +abstr acts +âĢ¦ âĢ¦ +teen wolf +ne we +travel gram +sentim ental +per ched +han del +ho ek +f ay +coordin ating +anim ate +man ian +effor t +jer ky +f ck +adri enne +ma bly +tra ding +my el +spi ro +sol a +stor ing +over drive +monday morning +dream team +pul se +bon di +ber nie +pgat our +tri poli +son am +plat t +âļ ¡ +ag roup +îIJ Ĵ +inv ading +v cu +k ell +ñ os +un dead +pod casting +mercede sam +mana fort +cor tex +que so +impecc able +pal mer +wil doz +sport sc +guacam ole +dispen ser +cate gori +stun ts +per il +invit ations +dune din +xi e +achi eves +saf er +pre ds +ph an +knuck les +k ak +igno res +lovemy job +aru ba +ound ation +datac enter +co vert +gr ing +cou ple +ا ر +vol i +mc cle +arti sans +lu do +kal am +arom a +under taker +hu la +wiz kid +gu mb +god frey +bakers field +ker n +engine er +car ve +pal in +guaran tees +pe bbles +b ays +zi eg +fin k +â¬ĩï¸ı â¬ĩï¸ı +down pours +ro chelle +rasp berry +ðŁĺ ® +gra phies +stom p +caf es +ari zed +utt ar +cal vary +dri e +crusad er +bus an +tux edo +si u +seam us +cul tured +blan chard +town house +ge red +butter milk +flu ctu +roger federer +hel i +ðŁ¦ ĥ +u ous +ram esh +mu ppets +email marketing +ye ss +br ice +ri zio +pel o +donnein arte +u rable +inve stin +bump ing +raji v +sav a +thro wer +fore x +o hhhh +th rust +pull man +r fid +sep sis +le ed +fri ght +roun ding +ne b +ph ins +ai sha +utili zing +squ ats +gold smith +j ic +bo ks +vau s +i po +exclu sion +tari ff +po kes +min al +land s +en force +washington dc +or char +g x +mar ys +ey our +aussi e +bak ers +un popular +latin os +lar ge +pu tnam +bol o +wa de +pel o +di zz +ob struction +fla ppy +weare the +depend ence +pajam a +e te +y ann +e wan +disc la +a ay +kar ina +e ic +an trim +w soc +neg atively +kai do +fotogra fia +dh ru +colo ssal +mcle od +k wang +mani pu +ex hilar +us atoday +summer slam +co les +tapro om +unbeat able +de ma +tic ks +k ling +fil s +campaig ners +ภķ +brew ster +audu bon +qu ay +ch s +ki gali +d ler +strength ens +som al +sign ingday +gol ds +pig ment +orche stral +g q +lin kin +ðŁı ĩ +ta w +algar ve +ho v +ear le +gold fish +am ig +ex er +ben in +dru id +ðŁIJ ¸ +she m +quat tro +mer cen +men te +incorpor ating +bon anza +state fair +en de +concep tions +e es +âĻ¥ï¸ı âĻ¥ï¸ı +d son +fire arm +orb ital +we h +multi p +fo b +requi em +p light +thou se +sa id +oc re +remem brance +n old +chi pping +be v +er t +ca thy +sy m +ri ggs +m ley +dialo gues +sl ender +how l +gau teng +wd w +to bi +smo kes +im plo +b pm +ad n +mom basa +cap sul +bloom field +artic ul +cle o +goog led +flu ffy +l ard +en zyme +ve sti +ibra hi +fl ame +e mea +out ages +dispro por +ble ak +an sel +ick er +st louis +stock market +good friday +sau lt +stal led +pro m +ep som +b é +the se +sau ces +me w +lit fest +pre d +re u +kar ak +si enna +ell in +bio technology +ï¸ıâĥ£ - +tac tic +sa in +por k +mon za +ka j +lu sh +compart ment +chang ing +shraddha kapoor +fo al +ar tem +cu ando +can ola +ori ente +me sse +d ited +br c +box er +bbc two +s st +ment day +em ing +de wey +kof i +âŀĸâŀĸ âŀĸâŀĸ +reali zation +smo l +tw ood +san je +flag staff +ber wick +cor set +can ary +whistle blower +et ched +com posing +squee zed +bow er +auto desk +ne h +mathi eu +ba ja +Å Ĥ +hy dra +da im +am eri +insi sted +mer lot +gar ros +heart news +gaine sville +cut ler +bo de +ðŁĺī ðŁĺī +lew es +scoun try +g sa +us u +cc m +god awgs +phara oh +cra e +mor ley +hyp noti +f ades +neur ons +fu zz +ing co +high landers +star k +vig ne +pac kets +amar illo +reu ben +insul ts +bas ic +vec tor +n me +ac ruz +tro s +transm itter +ðŁĺ ŀ +interpre t +ðŁĺ ² +pre quel +mc gowan +dis semin +ðŁĴĺ ðŁĴĺ +mascul inity +indie gamedev +ali ve +te t +pe tal +ema iled +ar med +ko o +he er +ba ird +super junior +metro polis +delav in +decl ines +stit utes +Û ģ +p tbo +g lan +cho res +e aling +chri ssy +ste mc +vi an +assassin ated +pron ounce +illeg als +discover y +cav ill +fri fotos +f al +so i +sabot age +t int +p dc +ðŁİīðŁİ Ī +ãĤ Ĭãģ +ji o +endeav or +in sig +commit tees +she arer +me tz +mar rying +h dd +g by +fre t +tri sh +pu l +scrip ted +sa ki +l w +ke ye +shim i +nan aimo +ca h +à « +tem pered +ici an +du gg +dish washer +air field +s rugby +gr inch +y st +r ms +mahat ma +lan kan +disc ar +dige stion +no des +l ls +om ic +gu tter +tis garh +feder ico +election day +bo he +master card +fire ball +âľ Ķï¸ı +oy ster +p ong +do k +en route +m vc +beat the +ali stair +shu b +sh aming +cherno byl +ghi bli +the s +pin ion +d bs +sal ts +ic tion +epi ph +nc pol +in convenience +whit ley +inspec ting +wood ley +wi ener +skil let +no les +m ca +h ina +a sha +willing ness +well ness +tam ed +show time +dis advantaged +ber nat +us n +mission aries +coun selling +arrog ant +quant itative +leg alization +ho dge +energye fficiency +cameron dallas +pos sessions +p bb +harris burg +v g +hindu ism +happy thanksgiving +fi b +re acting +tweeta picture +pol iti +mu ppet +hur rah +pac e +coast guard +guar ded +as am +par ry +fore very +x q +oom f +ke anu +j ind +ri st +customer service +sac red +ðŁĺ º +ton er +occur rence +mat u +val dez +red d +is ak +power rangers +pe asant +raj ini +abra ham +e mil +car do +tr il +hair styles +obsole te +sam pler +direc tive +delavin kisses +ver ton +glo s +sp ay +paler mo +com ets +man ziel +chicag of +ski pped +pic torial +h ant +b mi +a ol +re opens +pad dling +devo s +fra ud +bas eline +que ues +sp ired +sn are +eu ve +descri ptions +daisi es +ca ching +gall eria +tri mmed +stin o +recy cla +ic ular +bir ken +raw lings +fli x +chic as +b gt +lik eli +argy ll +thel ove +ga ston +bl anca +ha k +f one +sailor moon +h aci +ima c +fl yn +de can +bel les +ap ic +zo g +taun ton +con stance +lasag na +ker nel +in ka +har bor +collec tively +calcul ated +av ille +shil pa +pur du +gi mm +fun er +a est +pembroke shire +nighting ale +n unes +hyper tension +hu bert +sli ders +infer tility +comm ended +transat lantic +metr ical +!! @ +Å Ł +ss g +bac ca +inver ted +fun factfriday +it ans +albu m +acqu ainted +ri er +whel an +sar ab +mu e +snoo ze +pi ff +agre eing +sp itting +jer maine +n ye +âľı ï¸ı +am bush +ze ph +con greg +univers ity +s app +wann abe +pat rice +ib d +do glo +fri dges +sun d +king ston +ar gon +kam en +hardro ck +ds ley +do lores +ì ° +ota ku +pi ping +be having +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ı +blue bird +an sari +teapo t +fire work +cro p +log ans +ty ped +thick ness +ig ers +c fp +dys functional +contra sting +et ty +aston martin +tx st +dra grace +at tributes +marath on +manu scripts +john stone +ðŁĺ± ðŁĺ± +bo er +ay u +aru gula +poo rest +con du +assu mption +anag h +no h +delav in +sit ter +g ö +mor ow +kick start +com i +gl acial +ghe ad +ba in +ker shaw +en dof +fre ud +om at +i af +hu g +sign up +each other +defin ite +tu bing +shak ira +ðŁijı ðŁı½ +uu uu +sw in +sham bles +ol as +sk ell +brit ain +kn w +clu tter +om y +j ens +hang ed +city scape +scra ps +un locking +dead liest +er no +breast cancer +a it +inspec t +fu ri +ðŁĴ Į +ku d +ju le +or ah +mi ds +m dt +bur gring +r attle +pu sa +stal k +cle ans +iss ance +z ek +worth it +nam eis +musko ka +council man +urban art +bar rac +un solved +tu l +g ita +white board +soy beans +em ent +cont i +saturday motivation +conveni ently +doc king +t ado +âı © +sp ino +puppy love +po f +fabric ated +robb ers +adop ts +ti fied +kk r +indulg ence +notic eable +macqu arie +chap el +sensu al +ki ko +melan oma +lore tta +li ance +ab en +sp lus +ga al +ac ele +lib dems +compar isons +ðŁĮ µ +rhy thms +mer y +en capsul +nap ier +ðŁijĮ ðŁijĮðŁijĮ +ðŁij IJ +plat z +fre sno +re formed +ran bir +el it +the best +bhu shan +vin nie +impro vised +s ittin +re created +e ba +ec ker +ac rob +pon te +cor d +gi ddy +eur usd +fe ver +intu ition +gar i +dum mies +bud weiser +amend ments +te tra +sch nit +ay as +mar ys +ci st +k ani +ker mit +ðŁĺ±ðŁĺ± ðŁĺ± +tin ker +strol ling +di visional +niger i +omin ous +menstru al +kar ab +k hy +bw fc +pan handle +l illi +well er +stra pped +son the +transfer ring +ethe real +sne aks +ru dol +gab les +jac king +cin code +for tune +canadi ens +con for +ab normal +frank lin +tit a +mu la +persi st +cu ties +ki el +ðŁĩ± ðŁĩ +her mann +aw k +fi asco +ko to +we ta +hi ker +budd y +preven tive +mcgra w +game boy +forsy th +top shop +si ob +sad h +in tram +follow art +so aps +dragon ball +ou x +morri son +๠ĥ +lu bric +adul thood +morri sons +âļ łï¸ı +her mo +ta ka +stall one +mis use +team gb +ra gha +con fined +at y +hom ophobic +nw o +sky news +ho ya +ac rosse +wi iu +pur ée +jed dah +ðŁ¤ § +advis ers +ph ine +an is +scrump tious +ë° ķ +c ke +vin y +ter m +s dc +o do +home school +vas c +leop ards +debor ah +illic it +cur ran +as roma +nau ght +mar ig +brand i +em p +ðŁĺį ðŁijĮ +î Į +su spend +lu z +initi ation +sch aft +jensen ackles +craw ler +post doc +des ks +trail blazer +den omin +tri x +no ise +po et +± ï¸ı +s mug +vol atile +proof s +pharmac ist +sardin ia +mash able +kim chi +co ed +schal ke +doo dled +c sw +sh ur +ro x +do k +chris brown +mathemat ician +ab ound +ang elic +rock ford +d ole +yor kers +ms n +g man +xavi er +bor rowing +mark ings +longh orn +k ja +diver ted +mm it +euph oria +ay yy +te a +pa h +ck i +un cut +li ven +ky ung +fan art +mer ing +red ding +amo vie +gri di +c thulhu +schol arly +ju dah +th bewithyou +eu calyp +ðŁIJ ķ +hert fordshire +cour troom +by u +auc tioned +ple ase +mar cia +ê° ĵ +succe eded +el as +arvin d +t lot +saig on +re tt +ra kesh +fd ny +as en +se bring +gladi ators +you know +v lad +gol a +par ap +ÑĢ и +sab cnews +one team +oh l +sun e +ri j +cd c +star gate +run down +plat o +ph c +chat ter +ra viol +mn f +mand ala +li et +ภķ +mari a +hun gover +consoli dation +fer rell +tradition al +ilove art +gal ap +ðŁı Į +que zon +espa ña +ðŁĩ¨ðŁĩ Ń +ho bby +steam boat +mali gn +guil lau +pro hi +its me +íĥ Ģ +in scription +al z +mari an +k ade +mm on +adju sting +ne sts +intern ally +ci r +vik ram +mal ala +k ph +fel icia +the real +cap tivity +at is +marcor ubio +kale ido +che v +mano j +le more +gent ri +vi ps +tro pe +" âĢĶ +pair ings +mal nutrition +fr ay +desig nation +brun omars +az e +tor rential +pan zer +ga il +under the +the ological +schizoph re +dazz le +freder ic +mo par +ad illa +so ggy +ra un +medi ocre +colo rec +i fe +p inst +blu ef + ² +world water +gir oud +clar inet +ad olf +tar antino +receip ts +assu mp +ðŁij Ł +coffe es +âľĬ ðŁı¾ +du plex +s of +r x +lin o +timber wolves +pan dit +mo tm +e ga +ay ama +ach s +outsi der +ll en +co er +til ly +cheese burger +ma ds +ple dis +emp ty +national parks +az iz +p mi +jun kies +f ener +sq n +è s +gener ation +cleop atra +bhuban es +mosqu es +ty free +popp ins +tw c +or well +n age +ka whi +hol low +dal ai +¨¨ ¨¨ +ou ro +m health +gi on +az o +vis as +reneg ade +re ic +w sop +ðŁĴļ ðŁĴĽ +e chel +tox icity +mü n +bun k +stimul ating +asth our +\ ' +ep h +ende mic +cn bc +shrin king +peabo dy +michel angelo +can yon +wal e +su mi +si ders +inu it +? . +profession alism +dr acing +plat oon +p ons +out bound +maple leafs +de sol +cen cy +a than +ver ma +ru bbing +ok an +ðŁij ł +mull ins +authent ic +Å į +alman ac +ga ia +bb q +on imo +ke h +ty a +tou ts +y av +re posit +, . +wi ght +se eyou +cal lof +done sia +bar gaining +gr anth +sd su +amphi theater +p su +re watching +wine tasting +peak district +dete cting +thur man +phe e +èª ķ +u mich +re r +sculp ted +go le +name sake +ðŁĶ ģ +serv icing +bau gh +pu gh +pen cil +dar th +munch kin +at orium +ten ers +sun y +rolling stones +mag ing +star rer +i dris +fe instein +ag ron +âĺºï¸ı âĺºï¸ı +supervis ed +chamele on +aggre gate +succe ssive +mo gul +inst yle +pol dark +custom e +ohio state +ha ya +ci des +broker age +angel ou +fifa wwc +de forestation +al ton +pam ph +hu gged +ho bo +change able +ku ber +bur roughs +demon etisation +cape cod +vers atility +or ice +le ila +womenin science +tu a +he dges +embarrass ment +ali fe +so ars +ni ghter +hy mn +gi pp +chas u +tech s +ni all +k illa +hi ka +cam els +valu e + ¢ +sc oops +mah moud +clu sive +adri ana +pac o +oz il +un as +transl ations +whispe rer +s bi +bu xton +bio tics +indi ffe +ken ney +k lar +et ching +barra best +inst ability +se ine +vo tel +blo gged +whis key +my space +t ant +lan dia +give back +illu s +aw ak +ac ab +f bloggers +cloud computing +blat ant +syri ans +band ra +sty n +an em +ke ted +kar thik +barun sob +pin ot +gu bernat +gay e +arti ste +i fied +conven tions +hu an +geni uses +eeee ee +fol ly +somer ville +pride month +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +chemo therapy +paul s +bak ar +ìĦ¸ë¸ IJ +taiwan ese +fol lo +c ss +re ign +nn nn +fla un +catastro phe +iti es +frag ments +extre mists +ym oun +car men +eze kiel +conne cting +se h +man ta +remodel ing +we ymouth +at oms +ce m +ne well +lu mi +the open +mo c +mili band +g land +z shq +mag gie +mani acs +m sp +ad y +cre ams +le anne +e sta +py g +af finity +pray er +dun bar +ligh troom +ac adi +wyn onna +roman tic +state dept +sick le +wh os +lam o +et our +fin ity +shru b +shar pen +pun dit +ed on +af ore +mar s +jeff ery +ter ps +medal list +kath arine +accu sing +ta z +roy d +from home +confron tation +alle gh +ðŁijī ðŁijī +refresh er +ran veer +never land +jo jo +lu crative +en am +ca ver +pa edi +man jaro +flu ids +the ssal +oppre ssed +mu ss +joh anna +Ø ® +cn g +buil dthe +sett les +s ith +fu ego +cl amp +ar ag +pay er +ted x +mand y +inter stellar +fr c +ch and +b cc +mo lo +len til +johan sson +grims by +nature lovers +ðŁļ¨ ðŁļ¨ðŁļ¨ +shin de +x in +international dayof +transiti onal +sat a +cad dy +wo d +if u +ha ys +holl yo +j ang +ir c +co im +grad able +" " +ðŁį ´ +ঠ¾ +a el +n yo +west lake +time out +sof i +phenom ena +cultiv ation +ag no +un armed +so t +con j +gen o +royal navy +nutriti on +fair mont +ti relessly +sn g +re ty +mic a +lu cent +slo ane +droo l +riz al +od ell +critici zed +. '" +la ze +deser ted +co der +pra s +l illian +itiner ary +dav y +an ap +whi pping +hobo ken +kare ena +çľ Ł +vi us +ter n +nan tucket +mis understood +bu laga +st ant +chin ook +z am +reli es +d ss +ed mond +sket chy +m ell +fe x +rec tor +dist ill +day dream +wine maker +ri pley +billion aires +hel ene +ati f +cul prit +bertr and +wou ldnt +ma pped +v ak +gla dly +parliam ent +kidlit art +ware ness +goli ath +âĨ ĵ +view point +tat ted +fu ls +dor sey +ang lers +li ds +ki ya +bow les +be h +b ite +compati bility +ance stral +pro x +beha ved +gubernat orial +ch field +sab an +z h +teen y +shibu ya +holli day +pan cy +âĿĦï¸ı âĿĦï¸ı +seun gri +? , +ðŁĩ¦ ðŁĩ· +im itation +impac tful +any i +gene vie +añ os +bate man +gli der +af ar +ra sheed +effor tless +sh war +dach sh +er un +at os +kin i +ch d +kha ki +k lin +felici dades +bel o +as l +to ppers +fin ley +stac ey +rigor ous +kar ting +le ppard +car michael +be ret +c se +ak hi +mer ingue +ab an +ha ke +ger i +er jee +re sto +comm anders +pr it +fl or +ad ven +ex termin +remain der +å IJ +es g +martin o +lulla by +| @ +mi gn +in store +big bang +cor di +cau ley +ante bellum +dg ate +cro ck +span dex +scaf folding +ore os +ê°ĵ ìĦ¸ë¸IJ +pom ona +ma uro +uni versi +re mi +af ootball +t ant +sm alls +ne h +worl do +tropic al +mor ph +jav elin +gla r +arqu itec +reminis cent +tu bs +spide y +make u +syl la +progressi ves +blo t +shor ten +keep in +ch ak +ang st +super food +decad ent +ston y +neuro logical +ar boretum +ann ak +fe ma +per cu +dis respectful +small biz +lo x +co om +c sc +bs bi +pre valence +him ss +esp an +mo ga +fr ampton +sky map +mas se +levi athan +( ). +noctur nal +car ameli +ang or +amne sia +outsi ders +she alth +rhin o +ant ag +ag io +ðŁĴ° ðŁĴ° +take me +kab addi +c si +m sh +coch rane +thessal oni +sil a +ha us +du sting +obe se +mack lemore +mani sh +len in +m dc +gro wn +shef field +s rs +ke le +car son +ch um +dah lia +can tore +opp o +how ling +cyber crime +sur realism +sc ran +fa iz +thre n +rac ists +r out +pk not +se mana +sin i +mc cull +ma chi +alfon so +y b +sar dar +kend rick +den g +reci pro +on f +doom sday +bri bery +custom iz +art is +c pi +ðŁĻĪ ðŁĻĪ +sla va +let te +en s +âĿ¤ï¸ı ðŁĺĺ +cra yon +ad an +tr c +migr ate +simp son +row ers +king sley +farmers market +shee han +ne phe +bor non +car ton +mic key +all ure +u lu +sli pknot +heb do +gui do +dog celebration +online marketing +acceler ating +) .. +origin ated +macar oni +ed tech +out field +mit z +disc us +adverti ser +man or +ha shi +descri p +cap ita +ful bright +recep tor +con n +con ey +spion age +r attle +pre st +u li +blog post +acker ay +) âĢ¦ +red velvet +mat th +inspir ing +b sd +ker ri +po con +mil lar +re pur +accent ure +ä ¹ +ram bo +ragnar ok +dele ting +british museum +pat ory +leip zig +flori an +sci fi +in ers +br ate +yo y +melis sa +ab er +ma sa +po te +mosquit oes +transpl ant +r pa +; )) +bast ille +yl an +joye ux +melo dic +cap tions +atri st +roch dale +gott i +pew die +cuties aturday +who is +aqu aculture +tiv a +sp el +he ss +ha ji +fred die +co per +brand o +v k +photo book +* , +my dayin +micha ela +brune i +sr ini +in te +Ä ± +de ol +d fc +separ ately +bun d +ve sts +to c +me ck +rein forced +constra ints +car roll +sq ft +re ver +cam per +bird man +in action +gener ators +triumph ant +pe sts +o vo +gy pt +al amo +sc aled +suresh pp +sd n +is mo +gi os +) @ +justic eleague +restaur ant +gab i +den gue +next gen +exemp li +ap ex +inspir ational +down side +kid z +u pl +et na +alvar o +fel dman +bar net +m ha +es ch +bloo ded +>>>> >>>> +kan i +ho fficial +casablanc a +bir ds +ty ga +sw amp +o day +new castle +nb ap +ci sion +cho ols +af lo +ne p +mon ton +ak b +super model +down time +th os +sc wx +snoo py +ag greg +yo ke +nor cal +we tt +prolon ged +me tast +beat er +f ta +t lap +disgu sted +y h +voice over +itch y +ip c +ðŁİ ¾ +phe asant +stra its +ram pant +j g +fer til +assu res +fortun es +sal inas +liz ards +kett le +i bs +cyn thi +he g +mc cr +soccer oos +happen ings +cor den +ðŁĺĤ ðŁijĮ +t ches +egre t +wolver ines +congratul ated +ho gg +bott ling +wr i +fer ri +bo sch +af ire +og den +s jo +j dm +sv t +con tex +tol lywood +min k +me se +super sonic +op oulos +å ¸ +âĶ ģ +knuck le +gu ise +gam i +chu cky +z inger +radi al +compla ined +bo da +fe tal +discipl ines +cor ro +ðŁĩ®ðŁĩ ¹ +op ted +filtr ation +ad nan +em cee +mi stre +insom ni +fer gus +tra jec +on don +med tech +tanger ine +madra s +gru e +cab s +z hu +sureshpp rabhu +insul ated +day swild +pp m +band ai +v day +s ff +squ id +lo thing +not dead +expre ssive +cu ll +ala stair +x u +up front +fish ers +en es +um d +dis missal +sti er +sel s +lu st +re active +prote ster +eyel ashes +al im +goo de +gre eng +da ir +com pen +anush ka +proto typing +ma pu +bear ings +ðŁIJ Ł +for me +bsbi botany +timo thy +out skirts +am bed +are tha +wend ell +stre aks +ni m +k pk +sne e +fit ter +quo ta +p ate +win ning +ðŁį Ń +sho pping +ma inst +cul ver +ste vie +mcfad den +counter parts +gren fell +fol som +dor set +tech crunch +⬠ħï¸ı +tip tuesday +us l +tre x +geor gie +ranveer official +lic ks +se wn +k f +' âĢ¦ +jap s +p ate +orth op +fe sta +stra s +mon tal +hammer smith +fore most +wido ws +mad re +ite z +mito chondri +lig ans +z ona +cari bou +m ss +andre i +weather channel +gh c +: ... +ta ft +awe ather +al isation +bru tal +bliss ful +nik ola +mal icious +q m +mpg vip +bro die +bl itz +applau d +dri bb +v ague +dog go +transl ating +interpre ted +hat ched +ge tyour +benefici aries +spar ring +caes ars +aw illiams +la hat +bro ke +ti mp +virtu es +rel ying +pie tro +k tn +ici sts +pab lo +lou i +a ag +pn pp +cha st +pul ses +fini sh +usair force +type writer +thomp son +dog s +ut to +ãģ į +sand al +new ly +do ge +z w +wan kers +ne gr +mu cha +determin es +black fish +sk unk +mu ps +instru ment +phy to +daysto go +skin ned +hai der +con ten +ðŁIJ¾ ðŁIJ¾ +we iler +undoub tedly +chair ing +wall is +sh ard +zind abad +adul t +absor ption +pre sto +deplo ying +drum mond +battle front +seag ulls +how dy +juda ism +des de +part ition +âľ Ŀ +no logy +national bestfriend +lesn ar +film fare +co asts +christen sen +ac an +mb u +co pped +ru bble +sw c +fun nier +far ther +where as +nano technology +with stand +pil low +bow ers +to pe +it ly +con fit +ma kar +comfor ts +bo sh +cli pper +bal la +sti k +mil b +safe guard +musi que +eas port +ya z +pad ded +bad er +fore ign +chop in +archi ve +o ka +tran sporting +tml talk +aj it +consequ ence +sc roo +ff o +collabor ated +pug chat +ye mi +jav ed +au burn +o of +ma w +sau cer +miti gate +i les +evangeli st +ter ie +re cl +indic tment +cat a +bright ness +may the +whim sical +un lv +key word +cu min +med way +west world +tra w +im posing +form ity +coul ter +ab z +ny pd +grass i +kel sey +qld pol +clock work +f dr +di anne +âĺ ij +ad h +p ann +bra vely +ae ge +un lawful +ver di +pocaly pse +phar o +kar la +reson ance +ma stiff +la dak +bu u +ma iled +hi i +craw ley +tor rent +mach ado +liby an +effort lessly +fal sely +q vist +ke ef +craf thour +cheri shed +val kyrie +s ari +kal amaz +be he +ðŁĮ Ļ +th im +ro ddy +col trane +but chers +ach im +wk end +awk ward +cab rera +:) ))) +fran c +decl an +con dos +a ja +pandor amusic +char ter +ph ill +mon trose +hatch back +handic app +gre aves +eucalyp tus +ut most +t son +bur ton +mid wives +in cur +ðŁĺį # +moo d +compre ssed +tom a +must ang +mo g +as ana +te stic +sho tel +in sol +cor sair +nh q +ben ny +sm ma +kap ur +in con +jon as +ener gies +don al +as ad +se z +n pa +archi ved +stimul ate +do p +hy d +gri eving +ãĥ Ī +ron a +why te +tree house +ss ell +sand ro +ko bo +ther most +se clu +hi ya +ge ez +mam as +prisc illa +flav oured +fas s +w old +maker space +cospla y +p tv +happy valentinesday +sequo ia +love craft +gu an +d tm +ci i +yoko hama +pos thum +re q +ðŁĶµ âļªï¸ı +galat asar +dol by +hamp tons +disturb ance +stone henge +ok c +disrup ting +month sary +jun gle +head lights +du stin +micro sof +happy mothersday +ko ko +gra zi +te sto +na idu +mal ay +ari al +ru mb +ab oo +har man +tra pe +spo ils +je ho +go dly +lock screen +z un +pi ous +ma gento +l enders +prob able +corpor al +m our +aw al +su a +call me +ton ne +go vin +devast ation +x j +gear box +war lock +per me +it ate +gaza underattack +du val +paras ite +clement e +le th +i va +fro zen +tho les +to bin +cair n +s ill +luc kiest +conver ts +st ale +pan cra +euro pale +wis dom +sch ur +ì ¶ +verti go +bi j +u bc +nu re +righte ousness +mt c +factor y +ver st +revers ed +hur i +hee chul +fab er +ar r +ul ous +ven om +ph at +green ery +bra dy +à ¦ +: (( +never giveup +di sha +mo ta +health care +dun ham +dex po +den zel +bb ins +f ics +wh am +mc g +eli an +wat a +str alia +tel lu +pe sky +spin off +ar moured +re acted +do fficial +te du +sag ar +mor ally +paralle led +fi os +dow ner +dau gh +re do +world cup +tari q +bar ne +glaci ers +oc cult +barbar ian +her mosa +!! !) +y ur +inter nation +p ss +sit u +p int +american air +sw am +dopp ler +ðŁĴĻ ðŁĴľ +cincode mayo +le van +hell enic +mc ne +ju di +yu h +st x +qu are +ðŁĺĤ . +sti g +g els +mot ley +hard work +euro zone +e ad +ç¥ Ń +seab ir +ci us +la id +alpac a +presu mably +pewdie pie +boo ted +am ari +tam ine +sol ace +bar row +acade mies +x ian +om ination +dun geons +b ma +de ity +ai k +stab il +hir a +affection ate +ving ne +new port +ãħĭ ãħĭ +thir ds +re tains +aroma therapy +ski er +ni ma +do pe +cr inge +con domin +to or +anim ator +sar aj +seas cape +minim alism +lake shore +calla way +berg man +à¤ Ĺ +whisp ering +stupi d +ri ghtful +requ is +ir n +se va +ut pol +tuber culo +squ ish +de but +govern mental +christ ine +all man +weap on +s ito +bur i +lo lita +leaf y +fu ch +tin ted +mck en +a hahaha +ðŁĩµðŁĩ ¹ +repe al +ne gan +ðŁķ Ĭ +tail gating +game insight +ðŁıŁ ï¸ı +yaku za +z t +ti ring +pro posing +bow lers +tra itors +ak shi +cler gy +cit o +up sets +tu scal +symph onic +sil ently +shu ff +black well +ðŁĺĤ ) +ko be +rober to +ri dg +dc u +mer ino +ft p +east side +. ~ +nb l +mn leg +ts for +frau dul +ca pping +in my +gymna st +ston es +ss in +twe aks +shag gy +oak land +dem sin +sang ria +mm va +hen nessy +down ton +ri ghtly +in it +aga ve +ob last +northe ast +friend ship +dal a +tro phy +ðŁij ½ +mag in +margar itas +ê · +ww fc +fa sh +di ke +cu d +char t +ðŁij ® +refuge es +jop lin +n cs +imp y +firm ware +pas cu +flam in +health tech +bell letstalk +w aka +ol ls +la go +co wan +bombar dier +sh ome +ðŁĻ ħ +mc master +na ve +well s +u ta +tell ers +mis fits +kap il +face off +af firm +a pro +whit epaper +super yacht +speci mens +al located +... , +- __ +ka w +dachsh und +djo ker +s work +qui ere +or um +ðŁIJ ł +som m +c mt +ingh our +skin ny +lgb ti +gi ggles +break away +resear ched +par ity +my al +ms l +re tained +si vity +make inindia +sol ves +defam ation +wal tham +sri racha +road way +concep tu +al in +iw ant +å Ī +del ft +tender loin +ga ins +faul ts +sw ire +st ellen +pol lo +dy ne +bornon thisday +asdf ghj +sq l +sali m +advis es +vo ip +ìĹij ìĨ +un touched +she il +ontari o +uph ill +so bre +de shi +nov ella +du tton +craw fish +ا٠Ĩ +ma a +tw ine +kal in +ðŁĩµðŁĩ Ń +ye ss +brook s +hoo siers +ton ka +umbrel las +ay ers +ate am +acqu iring +su ction +ä n +wi es +tari ans +soci o +mat tb +shepher ds +o so +charity tuesday +s logans +ninj as +al bat +by te +bash ir +trampol ine +mydayin la +i ja +bas el +ror y +gol die +fi rec +un noticed +pecu liar +sch a +ker son +mour ns +liquid ity +qu ipment +hi bs +ar s +aeron au +slide show +sla bs +delici ousness +sk itchen +hta fc +full erton +cre ighton +aer ob +procrastin ation +az ores +white hall +uss occer +medi ation +djoker nole +and me +um en +noxi ous +jo ss +ili fe +anni vers +sudan ese +et res +under mine +whole foods +diso be +kor i +ade le +eli z +can ti +al on +gymna sium +sarko die +meteoro logist +yl de +ste en +stamp collecting +nas al +lo tt +fran ks +ex ol +ack i +good year +animal rights +y les +vio lets +mm es +s thel +ra pping +tu scan +wai ver +tur ner +eat local +northe asthour +anim ations +tom morow +t sh +ff ame +bra e +pe tron +glam our +br yn +d cs +bal es +ðŁĶ ¶ +bro v +bre v +b ons +physi que +car ne +x e +elix ir +vol ved +l oma +ìľ ł +æ ĺ +van u +ri gs +bal ance +va res +bon ita +sprink le +perfec to +di on +le ak +calcu tta +o ba +d ma +c mon +tun er +pneu monia +bo gus +apolo ge +cl ough +bor ne +)) )) +revi ved +o varian +ner f +c legg +fan fest +cho u +reali zes +mc n +li gu +leg alize +just saying +for ster +bo sni +k hi +in dom +hei del +en cryp +si ss +ed di +mar bles +brisban e +y ing +pre paid +wal sall +cooper ate +orche str +mar isa +ho wie +che wy +bren ner +andro meda +e gan +sto cki +cav endish +ag an +ban o +de ir +go g +bl k +re thinking +ch ig +rhe u +sni p +p eng +semin ole +m swx +an nex +lyn da +lewisham ilton +cu mul +tb l +dolph in +agu ero +........ .... +pre lude +at our +gr anger +too ting +ro tun +dis ar +home items +da res +**** **** +ðŁij Ĩ +compre h +jin x +as well +iri e +circul ating +ðŁIJ ¥ +over board +cultiv ate +rhe tt +oriente ering +ca k +bal kans +s itt +jas min +britney spears +ro tor +se aling +g bc +oc ci +f as +eman cip +com er +war time +tic kle +son ny +pac es +log g +at rix +sr p +g win +do bbs +uz be +the wanted +dru sh +ex tru +m icky +honore es +dar win +re dux +mm j +ram i +jalape ño +io c +do ver +ju ju +whit ney +s eng +en ly +au ch +archipel ago +vigil ant +man gal +wil dest +parano id +hal i +bb ly +sanc tioned +real ms +con co +u ddin +c sk +play time +libr a +sav ag +oc tane +rec tan +re turn +par rish +mor rha +cc p +c mu +sa iled +se vent +ro sie +pil ing +he w +boar ded +seg ments +neph ro +( . +cr ats +bak es +ðŁį ¸ +back tothe +sibl ing +kirk land +ke o +gu wa +bre ads +ðŁĺľ ðŁĺľ +t q +haras sed +ga u +wil bur +j isoo +ep er +li sam +tri ppin +sh ino +ru kh +beast mode +cho a +inst aweather +rich land +gar i +fe z +cowboy snation +fur suit +k run +a en +sycam ore +se gun +ent ennial +di h +o ax +demsin philly +ðŁĻ Ģ +sn hl +pen nies +pass words +ma kin +ty e +d eng +kni gh +jeep life +hel pline +a for +zz zz +ste amy +pic ker +iter ate +happen ingnow +ki b +bloom berg +martyr dom +bul ly +assor tment +a hora +zo e +no i +illu stri +agar wal +p sc +electr onica +recruit er +gar diner +rad ha +naf ta +dot net +pi ero +geor g +bel s +ðŁĺĤ ðŁĺį +tuberculo sis +run nin +mor is +haul ing +ev oc +bre thren +sha ir +frame works +a stu +ri gid +ku ma +kre me +jin nah +insu rers +ny u +f ere +nol lywood +good vibes +- ... +toi le +sk ril +instaweather pro +cze ch +pa vel +one piece +nike plus +fi let +cav ity +ðŁı½ âĢįâĻĤï¸ı +ðŁİ £ +dra stic +dail ys +siam ese +re bu +oste o +lar k +f re +sh elling +p é +glad ys +ðŁıĢ ðŁıĢ +gusta ve +submer ged +grand stand +att u +won t +f pv +b ley +jon i +ang ames +weigh ted +al ou +ठ¶ +les bians +f j +anni es +am l +dor ia +dav in +be ta +can c +madewith unity +ha j +bad lands +mu l +blu ec +pa wn +cov ington +neuro logy +htt weets +dysle xia +thel ove +ne at +fork lift +autom ate +une ven +monte ss +he in +ha g +rel ics +competiti veness +can elo +mar tens +bullet proof +sk ittles +g ya +pri mo +americ afirst +woo o +abor tions +?? !! +ma che +ld ers +rl ly +preli ms +direc t +cour se +swa in +super cell +ec centric +sting ray +ple ts +wil cox +west in +okan agan +kir an +car bo +bomb ings +ra rest +bo h +gaw d +di gg +mo ana +enti rety +en closed +dodge ball +par ton +milky way +at r +thorough bred +re ally +qant as +epiph any +ine e +aero smith +spi eth +ar thro +ell ini +du bu +bra ving +âļ½ âļ½ +re structuring +illumin ate +equ ili +mp i +ash ton +pony tail +ma scots +flat tering +cru m +ast a +à® ° +stranger things +bar nab +ر ÙĬ +make shift +got cha +will am +cho irs +kilom etres +gho sh +eu than +dol ly +un ning +the ar +cre we +w sw +j ace +dis miss +ke an +ho ta +kh at +~ > +thir u +ren dez +hart man +tee ssi +cas ca +z ah +hydr ange +fo d +aw p +mzan si +thick er +nago ya +ne va +sti que +cast el +dam ian +there by +ji ang +ale k +music islife +ra q +calla han +gou ache +somal iland +sean hannity +ra heem +lo se +elo ve +whar ton +rectan gular +illustr ating +har ne +auti sma +scra pped +ell and +decre e +nag pur +ki pp +so re +n md +ma as +gun a +gart ner +bel li +then ight +je on +gendere quality +gi ver +a el +gar ments +ne u +mardi gras +mar sden +ro wer +pollu ted +camer aman +vin od +be asley +cro c +ji u +hollyo aks +anesthe sia +al les +ste ward +lati mes +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +tic ian +gor ia +come dic +ðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ +nai ve +sli ons +ł Ī +bur glar +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ +york shi +se ñ +fan boy +lau rel +inci dence +potom ac +rober ta +presi den +pr yor +os bourne +w ku +te me +pal ae +ðŁ¥ º +re boun +itu de +red dish +k hand +coloni alism +north carolina +ðĿ Ĵ +manne quin +lady bird +ta sty +knowledge able +g shore +ðŁĮ Į +à® © +qu aker +salz burg +med alists +chy na +bridesma id +ma ori +ro p +outra ged +in adequate +truck ers +al ana +ìĿ ¼ +ri x +oooo oooo +command ments +lam beth +aa j +eco friendly +bla z +morecam be +boun cy +rou x +rai ded +mi zed +sh c +gaw x +labor atories +ru bs +rest room +consult ations +ca jun +virgin i +so ir +rev ue +ple in +wag er +ç ¹ +we do +growing up +! ðŁĺĬ +face ted +sin ners +ho vering +ti ene +seas oning +an ja +leg go +il is +fla x +dev o +ash ram +mati sse +ker i +go wer +bo tox +mar shes +unh cr +ts m +opti mus +dun i +stu ffs +so k +order ly +n bad +islam ophobia +raviol i +fab er +cre ds +won ka +in fusion +over weight +daily news +assi mil +acol lege +medalli on +kili manjaro +sti ff +tham es +sun ken +th ard +my dubai +hilari ously +han nel +plu mber +fair view +separ ating +rasc al +qui en +necess ities +confeder ation +ll ll +: ] +weak nesses +bron co +ra ffles +el ot +ãĤ¸ ãĥ +advent calendar +ðŁİ ¹ +stra vel +tun ic +k su +im peach +e spionage +! - +di ment +cur rant +bio de +commu ting +by ron +ðŁĴĵ ðŁĴĵ +shad ed +tr uro +cray ons +ar ne +h sc +fre aked +dram ati +fle ek +u cd +marl borough +^ - +cross ings +mal o +black ops +bin ance +cho ked +chen ey +pl o +ge stures +val edic +ryan air +rem ington +v cs +mc kee +ec z +be gs +nail art +mayor of +happy fathersday +war t +pet itions +n ingly +clean energy +bro x +sl alom +exist ent +ab ay +ug liest +tom p +stom a +sel by +goal scorer +ben ji +overwhel mingly +lan s +semiconduc tor +south korea +re scheduled +sk yl +en listed +dow ski +si del +rosen berg +nas ser +white head +pri us +har are +en n +ry der +í Ĥ +mon g +clas ico +transpor ter +po tty +is me +** *** +vic e +sk it +ode ssa +l mp +her n +raci ally +pin oy +paragu ay +obitu ary +go es +bu cha +side walks +angu lar +un constitutional +transiti oning +i bu +gu ys +un packing +oooo oo +black girl +ber gs + ¯ +wordof theday +trump train +thunder bolt +m si +fasci sts +ठ¬ +t sk +collap ses +raje sh +loveis love +migr ating +set back +ðŁĺĬ âĿ¤ï¸ı +t els +safety first +nar rated +jae joong +un answered +lique ur +en nes +dal go +bill ings +salt water +mer maids +lon gs +clap ham +we arec +pic collage +n ach +h ace +pois oned +lo th +ag na +adel rey +guar dia +poli shing +peace keeping +d all +p isa +la pland +process ors +de andre +so bs +p once +dra ins +c be +ðŁİ¥ : +spla sh +meat ball +fon tana +worcester shirehour +ne v +bri sk +b int +ac r +po x +cay enne +skril lex +j fc +hahahaha hahaha +gla s +en gul +tempor al +oni zed +con cre +com pose +vibr ations +plant ers +fer t +criticalrole fanart +t bli +sch allenge +huck abee +munici pal +iam bic +radi os +ne vis +dura bility +mc cla +horse back +inst itutes +ful fill +atta ch +ate ur +ak an +resi sting +illumin ation +hand le +hair care +om ent +macle od +ka iser +g no +bear down +ly f +gl omer +distor tion +z m +san k +roo sters +is now +as ports +ag en +wo ken +st george +ro mper +my le +econom ists +ru to +t will +health and +d ito +ws l +tair p +pra kash +mic heal +h ts +w rights +kat su +fioren tina +defen seman +d itch +var sity +texan scheer +ba ham +sc anned +we il +seduc tive +ðŁijį ðŁı½ +fu e +er win +dav ison +ter ran +moo ds +wool f +re source +@ . +cu sh +ðŁį ° +regre ssion +cur led +la zer +jo anne +ab bott +mo z +down ers +mm mmmm +valent ina +k hair +dream t +cro ok +che k +ste aming +nephe ws +cl eric +as ober +indefin itely +w ye +us news +joy ce +flu shing +wynonna earp +ron do +kis s +hot dog +bar ns +sax ophon +far ley +gas p +decre asing +al way +pe x +l sd +shi ft +p outine +ra zz +rescu ing +ni ko +ho ch +cc l +u aap +n ts +m car +il wx +conqu ering +ket tering +stur dy +delay ing +sto k +vani shed +cath ar +bin gham +in v +ic hiro +he mo +budge ting +[... ] +be ss +sebasti an +slow ed +ðĿ ij +musli m +stun s +acton climate +ve a +se ton +rose tta +oun t +hard in +flu id +ca w +ðŁ¥ Ĥ +yach t +un l +sp hy +provoc ative +or ic +is back +__ _ +nicol as +gy an +loo se +fl in +reb ate +: :: +! "@ +com icon +she ff +down stream +chic hester +beach life +mom life +diabe te +ar ra +van e +ok u +ye o +man go +try out +app ell +he irs +arjun a +dd u +na veen +movi c +soci alists +s back +criteri on +soyu z +k her +da z +yol anda +wine oclock +re ina +one w +leon ard +en dez +u bs +support local +facilit ated +carameli zed +b pa +vuel ta +my tho +m ami +spe are +nbap layoffs +fe vre +nick jonas +im print +c so +craig slist +la salle +gi deon +ha doop +dis regard +w ud +tu c +ma gee +acou stics +ta a +qui e +pol a +cr t +dw yer +dis sec +capit ol +men tion +kn oll +he igh +fin ders +plac ements +l se +indi ra +gur i +madhuri dixit +kingdom s +iambic pent +geor gina +je ky +conflic ting +bay an +aga tha +uph old +dr on +vic ar +ex pat +periph eral +pe ssi +fa f +ance stor +? .. +wid get +pun c +comm enced +beav s +air waves +ad dis +po a +de sses +co den +vu e +ru pee +kar in +spo ck +m sy +ภ° +pr ick +fill more +ti fication +thing sto +sar de +em ile +pere ira +n ad +bright ening +arre sting +wo king +usc g +sp ill +raspberry pi +hu go +ite c +is ma +cuff links +optimi zed +oc c +mi wx +en ka +el ited +afford able +sa kh +coron ado +ho h +at ul +ai oli +jim cantore +accoun ted +vin ay +her mit +groo ves +ran ch +r illa +we tter +ou tof +veter in +ni kov +ki an +fair banks +ram apho +n iti +k ko +ru sty +ne stle +tv xq +shahe er +âĿ¤âĿ¤ âĿ¤âĿ¤ +penn ant +gem stones +dem debate +ðŁIJ Ĭ +auton ews +support indiefilm +mach o +ve x +new sat +ne ti +conce ssions +can died +yof the +mac au +den ds +cricke ters +san iti +mari ano +gh at +ar toftheday +¡ ľ +e gos +gen oa +chat bots +bri er +al labout +mon ty +spi ed +r tr +comfor t +sni ppets +real time +gra in +exam ined +en lightening +tt u +god bless +release the +sing ular +ki ans +ha ka +sor ren +defe ct +mar g +equ ities +d orian +su ka +per l +aishwar ya +pul lover +preci sion +fair way +ne ve +rive ting +vill anova +en com +ak o +passion ately +europale ague +siem pre +x vi +enligh tened +c fr +âĺħâĺħ âĺħâĺħ +wast eland +is f +new comers +emergen cy +amphi theatre +- . +text books +figur ative +tre mb +pe sc +ab hin +ab bot +ac acia +har ds +por sche +kau ai +el isa +car rick +abo u +elli er +be ch +neu tron +galap agos +ru ben +in nis +how to +nun s +sab ine +i ac +clin ched +no tori +fi ves +cairn gor +per i +gr c +ðŁĴ¯ ðŁĴ¯ +mal m +twelf th +di ff +rout ines +marty n +lin den +synthesi zer +nu mber +game cube +fal kirk +byz antine +queu ing +gr ill +scal able +char red +rou ting +her bali +gri zz +ðŁĺŃðŁĺŃ ðŁĺŃ +tol l +termin als +l pc +ab d +war mups +remo vable +¯ \ +vi go +pap aya +ne ve +lov ingly +jo kers +ib les +sse tt +poten ti +pel e +gi gi +sadi q +leg acy +son o +ru pees +retar ded +ele e +par r +fi ance +ey re +say ers +pend ants +mak nae +al bans +adap ting +p ff +pu berty +ji u +ing rad +hypocr ite +diplom ats +phys ical +rob by +bon sai +ãģ · +f att +catal unya +âľ ĸï¸ı +ro ma +more land +so e +conver sions +stl blues +shol m +gra ssy +pra do +on u +assaul ting +> _ +sett es +dis graceful +aph ra +âļ½ï¸ı âļ½ï¸ı +ठª +kil n +goal tender +s ru +philanthro pist +b als +th n +stu den +sando val +dogre scue +eli ons +asse ssed +lar go +hec tares +sh rm +sa if +cle avage +no ches +n ene +fat alities +cur ing +clean ser +al es +p vp +south bank +pizz eria +marsh als +kni fe +an dover +tbli ghtning +sr sly +ou te +digi mon +timesof india +prome the +le bo +f su +wit z +rever e +man as +mam ba +ch ica +gu an +exhibit or +csr racing +d ere +xx xxx +gu sta +story time +ston ey +organ ics +and u +se am +min ogue +anushka sharma +ab a +ðŁİĻ ï¸ı +ugand an +chro matic +as sn +document aries +sh t +ru paul +loy d +k ats +e us +ite ch +me dusa +pan ty +kel logg +et to +talla de +sha a +do st +p ms +mari ana +je ster +croo ks +ðŁĶ ¬ +min danao +ind hoven +ðŁ¤ ª +le xi +tv n +jan is +co te +ãģ Ĩ +ser rano +iw m +ðŁIJ ¬ +k ke +distribu tors +cap u +counterfe it +camp site +ag gie +ðŁĺ ¼ +chhat tisgarh +~ @ +state u +san di +prevent able +cl s +can ne +mm c +i ver +sa haran +pal is +night out +do s +ap ia +absc bn +manag erial +aro se +mo wx +aro sa +ðŁĮ ³ +under dog +remo ver +astronom ers +lent ils +su scep +smoo ther +pend leton +fau cet +e mory +dal mati +af cb +tic us +exem pt +en rol +d heim +ðŁIJ º +restric tion +star fish +sto w +snor kel +thunder birds +she ad +homo sexual +dy n +as li +andre tti +dou che +dom o +tar mac +slu mber +pr onto +first dayof +mini ature +mari achi +argu s +recomm ending +mobi les +in ce +illustri ous +or c +adver ts +gr its +wea sel +pag oda +over pass +gre ys +maxi mus +arma gh +wood land +sun ni +ðŁĴ ī +ë Ŀ +ti one +soci o +ho s +ðŁ¤Ĺ ðŁ¤Ĺ +wind sor +subsequ ent +munch ies +id h +exclu ding +e mi +cu th +z ai +week days +law suits +barn ard +Ø ª +pe tting +net es +mul ligan +pharmac ists +ra quel +e ton +cran ston +gil ded +cle ary +ce ph +ra a +pam per +lombar di +as in +sher ry +pro d +for te +ari anism +buffalob ills +æľ ¬ +ðŁĶ¥ # +uu u +just ices +car ina +nat in +mas low +dro oling +cog nac +cam ber +el ong +r dr +in en +convic tions +am use +tro ck +harm less +visit ation +gen omic +bl and +beno it +chim p +tuscal oosa +gre asy +x po +gil t +se q +per mitted +christma seve +book s +mu e +old school +human right +be ati +ðŁĶ Ŀ +sh at +sculp ting +h wan +fern andes +sci utto +fu entes +endeav ors +maid stone +un paralleled +shou ted +queen of +mer c +band ic +ve da +sel angor +pi le +ja han +intimid ating +disapp ears +cl ich +za ha +w urst +hi v +fod ils +cor dless +aaaa aa +hy dra +bel inda +e els +bu f +su staining +rugby league +no c +brig itte +( ðŁĵ¸: +tromb one +soo the +smo g +ad p +stab le +ing ley +diagno se +ms g +we ss +tic keting +one e +nsw pol +e up +auto psy +adity anath +sun down +river front +si ya +p is +hier archy +dur ango +di jk +ren shaw +he aps +epide mi +david bowie +interne tof +dd i +nation ality +mb ar +air y +win der +w alia +elli ott +c x +bav arian +pl att +an tw +wi wx +sof ter +ne ha +h eller +th and +dani ela +bo ast +degra dation +ðŁĴ¦ ðŁĴ¦ +transform ing +man e +av ut +ðŁĺĪ ðŁĺĪ +vo ter +the e +t ate +pu ff +in door +sop roud +boy ce +boris johnson +wait in +immun ology +ðŁıĨðŁıĨ ðŁıĨ +âĿ Į +street food +liz asober +cavali er +c elia +need le +motor ing +g ato +, ) +ra de +harve st +t ms +jar pad +on ey +air men +v re +impair ment +abhi shek +snoo p +l ant +fam ously +bl ou +s ze +g ander +un touch +tu f +dee jay +col lateral +b ind +ðŁļ © +pin ning +ic n +' ; +the economist +ul tram +worldwater day +ti poff +the i +feed ers +campa ign +sc umb +day weekend +yo m +pe dic +h ough +ps v +pl in +on de +boston marathon +az zy +* _* +con ley +thi ago +hoo o +gal erie +luci d +je tt +gl itz +final fantasy +achiev ers +y ung +peregr ine +op hi +dam es +biom ar +âĺĢï¸ı âĺĢï¸ı +sk c +l ics +fl ank +ar rahman +ho of +uphol stery +t ats +wo z + ¿ +snor ing +ra er +l ju +ap d +pl ating +kan u +im ation +fragr ances +m ra +mor ay +mo tt +im muni +hearti es +bho pal +tim ers +g ata +color way +car nation +win get +si ghs +s ville +optimi st +chate au +olympi ans +ci o +singer songwriter +ny o +fi bers +bur ch +ag ro +mil ne +ig bo +cr amer +ation als +dan ube +pad ma +nor mani +en forced +bre ck +boeh ner +ar den +sur rendered +pros thetic +om a +ha iled +calcul ations +w fa +bi b +fcb live +fon da +west coast +que sts +friend ly +to wie +fit ch +bal ot +star dom +scrat ching +ho sa +thi ka +o ven +stro ke +out post +pharmaceu ticals +hi kari +mu y +af d +fallon tonight +squ at +or u +dra ined +chocol at +ë¯ ¼ +wor ths +ri b +mu j +that s +residen te +it el +boo st +mi gos +mul led +la a +etsy shop +don keys +me k +p tc +flin ders +e hs +ro hit +mu ir +g ad +compos itions +åĨ Ļ +combu stion +i kh +yemen i +wav ed +gar ci +ak os +oo ds +fu sion +se que +s lan +pl ur +kic chasu +shenan do +s ams +worl den +horo witz +with me +mic robes +k ki +ðŁĴĶ ðŁĴĶ +w su +patch work +fre er +y aki +the art +symboli sm +mil er +bt n +ma bu +side kick +motiv ates +sag itt +natur als +serv iced +ps ori +pa ola +qu ig +i badan +gi ggs +ë ³ +sciento logy +si oux +salam at +d res +cad bury +d hawan +ci ón +_ ' +swa pping +maris ka +james bond +explo sives +ay les +af er +s agu +cen sor +tom a +jeff erson +ring ed +par tist +ir responsible +aguil ar +vac ay +equ itable +altrin cham +ac ur +man ish +ger min +schoo led +pu tter +ed ad +nav al +toast y +sol areclipse +dish u +coy ne +ac co +mu ck +mar an +el os +len der +cro ix +worth less +ha ber +gun men +ðŁį ĵ +zen ith +t enders +hur st +hol tz +itali ans +car low +u cd +characteri stic +bun g +av l +u th +sa sia +rs l +red man +neighbor ing +green peace +sti ps +follow party +y gk +en os +omni bus +na issance +chri ssy +secu re +call back +ji hoon +memor y +block er +l anta +daf fodils +bil t +ffer ty +fau st +ie c +nipp les +so g +m nd +jagu ar +bol dly +ab poli +pro position +gun sense +evan sville +cu tters +we go +dou n +do x +stal lions +ka j +shi ppers +j awa +vol o +le ven +pap rika +kov ich +jor di +induc tees +app alling +dial ysis +allevi ate +âĢĶ âĢĶ +pie ter +mid wi +q tr +juli ette +inter mission +haw ks +act ment +one ill +k lin +vam ps +fam ous +cou ld +autom obi +da an +west end +elli p +nh c +mel anch +web series +ton gue +snat ched +smy th +tan gible +sl i +e asing +bar stool +over lay +afford ability +ting ed +ter as +ay ush +wanna one +rh ine +dan a +sh ana +kend al +fer tile +w ir +repl eni +lar vae +is ro +con vos +ab brevi +u cc +hun gry +bur rows +ag er +nav i +mat in +du per +cer n +ma don +ķ ï¸ı +é ģ +tu ps +hy att +sh ep +friday night +wis er +hei di +hat ton +p gh +foun tain +wrist bands +ahmadi yya +aeri al +subscri bed +so los +m ace +sla yed +for fe +dul ce +christ mass +arun jaitley +viol ate +ob stru +ni eces +w vu +idy l +fa ze +pre serves +infr inge +premi ers +inter vals +agen cy +( © +stand alone +di mes +bo er +param eters +ge tit +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ +tu lane +for given +scol l +mb ps +smash bros +rob bi +prima vera +ali st +ghost ly +ay at +ye ats +impre ssionist +ear phones +caul field +wai kiki +sal ute +sc ou +mu ay +louis vuitton +bak hta +ado g +inven tions +hur d +forec lo +stream line +thalai var +ch snews +will ard +t sn +euro parl +cru sher +my sore +gro wer +ra ping +pat ti +g den +sm w +muf ti +kid man +ab r +soun ders +skep tical +ðŁĶ İ +sun dar +i me +fer g +feather weight +ar lington +pas qu +ag azine +wearab le +nati c +mccl ure +inter mitt +hor de +six ties +car te +bha v +ze al +experi ential +ador ned +som mer +eno te +hypo thesis +stin ky +pro to +dead lines +vo gel +mus ings +monc ton +gu ter +f le +aci on +voice of +ta sha +inhabit ants +type face +s ba +bts x +ðŁĶ Ĵ +wor x +u hc +jo ko +cell ars +gor o +continu um +... & +weather cee +ha p +sr k +ris ers +lonely planet +un named +co eur +ðŁį Į +the world +ili ke +fa sten +ami go +ri ba +ramapho sa +staf fers +had ley +? ?" +fi ore +sal ut +hu ff +bez os +Ñ ĭ +ra der +kam ala +in line +fill ers +um atic +all in +shat ter +re in +o ku +ch ases +fla gged +baby metal +water stones +ts b +cut out +op hel +aam a +rockab illy +sto lic +jet blue +ich ick +down ton +uzbe kistan +pat na +la q +gr ange +) _/ +subsi di +sc p +newsc ast +it sa +twee tyour +e mor +archae ologists +uni fication +por ta +q x +protec tors +pro hib +charis ma +car tag +ren fre +scul pt +guwa hati +de ma +boo p +unf pa +dex ter +lay la +alleg es +sou ps +never again +l ys +cal c +bar oness +visu alize +ger ber +absor bed +i ers +a han +fon tein +detec tors +verst appen +sv c +formul ated +ac dc +li x +in competent +bh k +lour des +water house +snow ed +appreci ative +sig ma +lizasober ano +pen ned +pay check +tall inn +fanc afe +par isi +av alley +vi g +ru fc +hard ship +so cute +po ise +ì ¹ +roth schild +k ly +???? ???? +l hp +il ay +f hs +am ad +ide als +brad bury +bal boa +nic ot +kid nap +wol ve +tas manian +op t +matthi as +ãĥ³ ãĤ +super markets +mylittle pony +me lee +li ster +gr oun +fe dora +kind ness +en en +bra hms +¯\ _( +ros well +mar lene +ic u +re formation +or ail +he brides +dispar ities +terrac otta +swal lows +re id +influ encing +flu or +den e +tum our +blon des +thunder bird +sh eva +moga dishu +ka b +cre eps +i ving +ene ed +anno y +âĶ Ģ +intri gue +enqu iry +ar aj +tur al +kuber netes +end lessly +divi dends +tor a +ti sh +commemor ates +un ra +tri b +pon ty +ne m +diss ent +brew ingco +ðŁĺ ½ +nor mali +bi of +( ... +chil len +ì£ ¼ +mell on +av is +mccor mack +ing ra +enrich ed +custome rexperience +testo sterone +snu g +sett i +ger onimo +inqui rer +bre aches +very thing +bloom ing +mu ra +dispo s +bi de +de va +shade sof +in trin +sh ev +s ven +nayanth ara +gan esha +c ws +ber ta +label led +use um +nick named +ma han +car uso +ap ur +ðŁij Ĩ +w q +orphan age +discar ded +mag nu +lu e +je on +bridge port +pac ing +mercur y +( ðŁĵ¸ +marx ist +amphi bious +transplant ation +stit ching +then burg +gradu al +ãĤ Į +ro ft +ma ils +ine c +guy ana +dopp elg +ver o +re write +head less +harb augh +gate way +car sforsale +sw i +st is +mach t +un de +sura baya +stap leton +nur turing +mil ner +ya o +lma oooo +ko sh +arsen al +k ame +er ry +ar royo +dis misses +ru bbed +rc b +lew d +dil u +and or +vi de +ur in +inter sec +ha ar +al b +year swith +app leton +é al +ul livan +suc cu +monter rey +d mx +artem is +ron nie +farm land +s football +gro tto +anth i +ãĢ ģ +à® Ł +vid ya +jimmy fallon +ൠį +t zer +gravit ational +w thr +u hhh +e hr +tin ker +ti juana +scran ton +ram charan +bar clay +re van +m si +ka p +wr s +we thenorth +tor al +sat u +gro m +fac ep +erick son +z yn +se dge +oo dle +spur sofficial +ds p +sic ilian +soli hull +recei vers +ladak h +hend rick +ther i +presi ding +mc guinness +litt ers +gun nar +gh oul +wi b +n tv +kar o +fro ck +b lau +ampli fy +all is +ul lah +memo irs +kh loe +intercep tions +pet day +lo oney +con fin +ch ay +piyush goyal +frequ encies +ut z +event ual +warm ly +obli vion +an ka +ta it +âĿ¤ï¸ı . +director ial +ru lers +prince s +mu ck +stur ridge +deu ce +abri dged +bagu ette +un cles +pen du +min ding +forre ster +av ila +wall er +wall street +ment or +hin o +high way +crom well +fanart friday +mb i +co yle +a hi +tro ve +spie gel +pay tm +mcin tosh +jan sen +nit i +nash ville +len o +leicester shire +le gos +dic t +ðŁĵ ½ +sp ad +beverly hills +sy rah +separ ates +z ain +un fit +dra gs +tan ia +over flowing +hri thik +haw thorn +z ani +mac far +fi de +to tem +pe ds +fundament ally +cal ico +sin ner +j ä +hil de +ds d +ten ay +ta hit +mil f +lie b +inform ing +up lift +ra el +mortg ages +lec t +ii ii +guillau me +compos ites +old smobile +l end +gar th +com mish +bapti zed +scorpi ons +ru cker +bringback our +alli ance +thalap athy +tal i +sp ans +eri dge +wither spoon +lin da +sky lar +kor n +hom s +Ä į +sil enced +caf fe +ar ty +dist inguish +to wed +pun g +jessic a +ear nest +beau fort +t ama +study abroad +si khs +new bie +nav ratri +mar ble +loun ging +lit ter +dal it +so sa +iz es +gra de +com promising +tr iton +de tta +v j +chau ffe +spec tral +powe red +montess ori +artic ulate +hal ton +al co +ye y +mn twins +acoun ty +ðŁijı ðŁı¾ +âī Ī +mad men +kal a +gru m +chi k +ati s +su me +akh tar +job search +high lighter +bo ath +âĦ ¹ +tar zan +lam bo +âĽĦ ï¸ı +ox fam +dump ster +pretz els +mac os +incl ined +fac tual +adverti sers +shu i +pu ree +ml pfi +anti dote +cap o +pa str +merc ado +but ton +ar min +ag g +lol la +horri bly +er rands +christop he +time snow +monday motiv +li ss +scand als +mc i +dispropor tion +âĺ İ +sur pass +samar itan +so tho +pu rest +fl att +trivi atuesday +delec table +leop old +hermi one +chou dhary +en rich +¡ ¡ +subsi diary +ine qualities +bachel or +auto immune +la kota +i hop +ad jec +the simpsons +sh es +se k +gret chen +up stream +hin akhan +coper nic +x tina +lu g +tough ness +e ad +cli pped +bi us +sl v +fah ren +dee pak +ca u +x an +im mature +dig ni +bo bs +shred ding +but tery +accommod ations +de ven +chun ks +super league +sky bet +kil dare +je et +ë į +ce k +wrec ks +pro pane +oh l +tb d +quo i +trum pp +mi mo +reluct ant +ver ne +o ic +ma gh +ar nau +se ver +li dge +stair way +kicchasu deep +ðŁĶ º +mach ining +aama admi +ot i +c da +al it +pan y +inst alls +ac ct +e shop +di em +hard well +fulfill ment +sc afe +qu ack +extrac ts +swee tened +fi ghton +f di +d inger +wal tham +us ur +refe rees +seok jin +gran n +af rin +th n +sch af +par cels +bet is +amar ine +nom an +kh tar +mor itz +cou pling +bar ons +ðŁIJ ¸ +à ¸ +sl p +sad ler +x ander +tri ad +mc millan +kh z +divi ding +ìĹijìĨ Į +dar yl +zed d +le ys +pla ques +flu ori +tipper ary +on nell +di dier +lang ford +im c +the sun +bir dies +ar cha +ye ssss +t di +dar ia +cand ace +al tam +pal aces +ch it +sant am +event ful +book of +ad b +mon stax +cre ole +co el +âĸ ½ +we aren +sten nis +she ath +ati sm +gron ingen +mlpfi m +le pre +wrong ly +rsp ca +rendez vous +acknowle dging +pel vic +solic itor +sla ys +nue stra +lo d +is lander +fer oci +fashion show +ra ss +dge on +adole scents +sma shes +negli gence +grate ful +ved ere +sw oop +ing l +apol ice +vand alism +gan n +jo ao +di supdates +zimbab we +under age +radi ance +w of +bour geo +pla s +cr ani +gh ue +wrec kem +warran ts +re form +jim mie +at wood +ys l +neil himself +l bj +i man +tan to +nois se +ver bs +equip o +al together +mam ent +l ice +dou glass +tier ney +pri med +j hal +furn itu +braz ili +v ill +past els +n ison +u ff +paral ysis +jay e +im po +ðŁij ģ +strate gically +pakistan is +was sup +super bike +thank u +tru elove +sha ikh +israel is +vi p +to g +li en +la ker +grey hounds +cul ars +bian chi +balot elli +ar ran +loo s +str ates +he bron +ar vo +sunder land +the al +tomb stone +sand man +c pac +thanks giving +love him +lat ino +an in +aka if +ĭ ãĤ +tor quay +di est +alli anz +ðŁĺ ķ +golf club +cl lr +wal cott +sch nau +promp ted +nomin ating +len nox +val et +mon ro +may ward +e ph +ðŁĶ Ķ +inter oper +r da +re flex +arm chair +ê° ķ +stri pper +por ti +ph arm +ham za +ni reland +ne ue +h pv +port foli +sun burn +fris bee +be al +bapti ste +x h +ty m +pr ati +o vers +haz rat +deser t +der ry +us ky +em mett +ach arya +)_/ ¯ +shu d +may a +ham ill +ra im +nr c +fitt ings +cur vy +ðŁı ĩ +ster ling +ॠĢ +wal kin +short cuts +mil ly +ast ur +alpha be +pl i +pe z +miss you +rad ford +ml g +ta eyang +notjust lakes +du mps +seren dip +le ur +ra ving +e ster +de priv +absc bn +ðŁijĩ ðŁı» +scar city +o cr +mean ings +cap t +da hl +fer mentation +bri oche +to win +out lander +massi mo +en cro +ðŁ¥ ³ +buil t +po tam +kir i +tm w +monit ored +k ites +peoples vote +gray son +íģ ¬ +afri ka +a dies +i vote +gy ne +g annon +di x +c mc +ou ral +fox andfriends +bel i +ig ne +gl an +katrin akaif +co politics +qual itative +p si +lu cci +disc oura +âĺ ® +kel li +gau tam +carac as +reale st +pu la +in us +hill top +make aw +atten borough +tw y +r arity +peck ham +ma hon +corn elius +clin icians +ton line +tb i +paradi se +ka si +inev it +fresh ness +colling wood +lun atic +defen se +cop d +in fra +wain wright +sains bury +alab am +te ma +lac o +chec ker +releg ated +tren t +stal ks +huff post +bhubanes war +ast ral +share your +prim rose +hi me +cat an +end ment +en dow +cle mens +mal oney +hil ary +game time +den ise +collabor ators +b wo +radic als +gue tta +ici on +au a +snap matic +sat chel +excav ation +base man +s ão +gn ation +fel d +surve y +shah zad +ma st +anirud hofficial +tru cker +ot ago +geo graph +ethe l +âļ¡ï¸ı âļ¡ï¸ı +s ver +mu tt +internetof things +ancho red +wh ouse +bang la +bal main +ç¹ ĭãģ +break fa +á Ģ +twi ster +te tris +ca v +stag s +g z +au b +stor med +hel ens +yar mouth +st asy +gustav o +co sc +vin son +up p +sc ricket +assump tions +app e +nu h +u er +pre mise +n aga +e amon +coron ary +na f +north side +el mer +ro tar +out lining +el f +re surg +kat elyn +in can +hyster ia +ce e +am bani +pro lly +Į ãĤĬãģ +ax es +san jose +rem brandt +mag pie +even ly +scor sese +qu aint +f g +b buk +indian football +weare all +spd wy +pis ces +ec g +âĺħâĺħâĺħâĺħ âĺħ +pre orders +: | +ni pple +sal azar +ju me +jail break +min n +bas sett +ze tta +jef free +ad jun +tic on +san diego +drink local +chol era +solic itors +o bo +com post +ni an +wr a +tre ach +ic ic +profession al +del ve +leg ate +histor ia +cro issant +con noisse +nam o +palli ative +chem trails +i ority +global warming +comic art +behavi oural +re sted +li as +cli mates +Ł ãģĦ +rut land +nou rish +menopau se +hot ties +demen ti +ve spa +mel ville +anal ogue +tz man +str ung +im perfect +gl are +cir cling +ros berg +rec o +oc ity +lo ire +em be +do ssier +ne el +nan do +me a +gal vani +fin esse +ag p +berke ley +asi m +âĺº âĺº +quil ted +ish ere +un matched +po tion +for z +at re +selfi es +juli ana +ðŁļ ¶ +âĸ º +mel ton +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +spin rilla +pur cell +ed p +at leti +tony awards +ra ja +pro gno +mol ten +stu ff +p ally +nobel prize +âĻ» ï¸ı +spiritu al +spe ake +sa sha +bri um +tru ss +critici ze +assassinscre ed +yor uba +u lo +fire man +workin progress +ef cc +fla res +ro bot +hi kers +cl l +shado wing +pat sy +leh man +c ns +å ± +guad al +à± į +ra pe +r honda +paralle ls +son ja +langu age +land ings +z ola +cr amps +bur ning +apprais al +jol la +ham m +kas a +gul ly +f go +uly sses +ri be +ðŁĴ Ħ +ib u +eti enne +bri ar +fin ely +comb ating +y ql +go tham +we chat +to paz +primar ies +l se +iz z +hel e +dispon ible +cy stic +bel ichick +th rush +kansas city +ge om +soli di +red bubble +by stand +cambridge shire +par fait +ast le +ow o +ind ore +stom ping +sm elly +ðŁ¤ ĸ +locom o +adm itting +hol me +clock wise +min sk +mc co +for get +ev p +cam ra +ab ella +yo tes +universit yof +mé xico +silver ado +ric ket +crom bie +pu j +eradic ate +deli ght +y go +glam ping +vic a +du ggan +coun ters +cf d +sc our +react js +pu ram +paras ites +in ki +vill en +stel la +li mbo +ang as +k cr +ðŁĴļðŁĴļ ðŁĴļ +vap ori +mum ford +oli gar +à ¼ +al oo +boo ties +ad r +k elli +dru mmers +av ici +nature uk +ron al +in trac +un splash +le che +g oma +el ine +envir o +bi onic +bu eno +mi k +av in +star ling +em powers +cake day +boy cot +ðŁĴļ ðŁĴļ +ðŁĮ¸ ðŁĮ¸ +v ach +m ci +fractu res +ger i +sk ing +exclu ded +lu ce +ja ve +ig gy +evi den +aki stan +a wn +mor als +luci fer +ha ban +tumb ling +sunday motivation +mo sley +captain america +sch icago +the one +mo td +d ts +ðŁIJ ¼ +rep ell +ii i +locu st +geo spatial +mer sey +immer se +desc end +ber nade +j s +boat sales +win der +cran k +sing leton +candid acy +ben a +ðŁı» âĢį +high lander +ol t +k prs +healthy lifestyle +four teen +end the +ith aca +circul ated +r ans +pre valent +ha vas +splend or +roo ster +kalamaz oo +jewell ers +enne dy +rou sey +es y +cann ons +ornam ental +// // +ren don +win ne +mol ding +eid mubarak +coun tess +simon a +ha wa +fo es +du ster +sb u +por tray +mar ries +goo dday +cho co +achi ever +ðŁĺ¹ ðŁĺ¹ +pre neur +tr amp +tom i +n bat +garden chat +farra khan +ever glades +ab ru +sou sa +se ce +homes wee +terre strial +bar it +sri devi +ol u +mel inda +f rick +can dies +ðŁĺŃ ðŁĴķ +qu reshi +family fun +exor cist +cardin al +ny t +dies el +cu mulus +capric orn +si ology +lor na +dou gie +an die +super sport +c fl +п ÑĢи +say ang +pe ek +ภĬ +lo be +j em +ing lis +gg led +c sn +amne sty +chu ps +ba es +sau er +ðŁı IJ +mongo lian +en et +back street +dr illed +acce ssing +ce o +b se +ai ken +pur r +wor sen +whe res +war k +testi fying +bu ri +bla st +aw g +ðŁĵ ĭ +re defining +hear ing +u ci +c mp +bon i +tail oring +ta ji +noc chi +em t +stephen king +ne et +compla ins +campaig ner +luci ano +twili ght +ti esto +pas sports +flo yd +cathe dr +na ked +caregi ver +b coz +ade cides +ku ri +ly k +br aries +dren ched +disc lose +ðŁĴª ðŁı½ +le blanc +je tty +gar ty +chip mun +b su +rhyth mic +ic z +fri d +anne x +ame x +solo ist +lanc ers +arro whead +speci fication +simul ated +na is +inver te +bo wing +wor ship +f z +abo ss +sha q +ì¶ ķ +challeng ers +an arch +aamaadmi party +ãħĭãħĭ ãħĭ +suffol k +so corro +sn ell +cla dding +absor bing +shaw a +particip ates +ðŁį Ķ +book stores +bak u +seap ort +ko jima +gab y +pack ard +electr ician +let it +mo wing +fa wad +young jae +hot mail +men ing +u rie +intim acy +con ti +: ") +lifeis good +in ciner +i dri +craz iness +jour nos +fran chi +bott len +al da +ff es +k x +south we +air a +clay ton +sco ti +f j +bri ga +ðŁ¤ĺ ðŁı» +demonstr ators +y z +stor k +na q +casc ades +travel chat +plat a +pad ma +fran ci +at tain +bat girl +lom bard +hoo s +d dos +neon atal +discla imer +r ss +r ant +di sen +tex aste +so cal +frac tal +cam ry +stri fe +sn acking +mu h +sant ander +mor ons +gra f +par ades +hu ston +dru pal +mi ento +kir stel +hy de +vom it +forti fied +sphin x +da v +bir yani +win nings +s baseball +mer ged +lovel ondon +ling ering +dream big +car leton +liveli hood +djan go +astri d +gri ds +down e +bru ised +s ne +scarec row +hel ium +f nc +bi ggs +an ter +restor ative +em pires +ab del +life style +kiwan is +colloqui um +me en +pr ick +anti que +ze b +mi mic +edmon ds +ðŁijĬ ðŁijĬ +q ing +pp el +mc gill +interpre ting +âŀ ķ +rash ad +do ka +narr ator +electro magnetic +ash by +sau ra +iran deal +âģ īï¸ı +krish nan +in di +ff en +bre a +os man +multin ational +chi ppe +recruit ers +aus biz +p ounding +re gen +cur sor +refu sal +mac s +in ak +ax ial +wa ifu +up cycled +hindu stan +cas sini +carly le +scrat ches +re ef +man atee +eat ery +ðŁĵ ¢ +un condition +sen pai +on ther +comic book +pro sciutto +de mar +mi se +ma ge +fre ec +aye sha +al der +android games +ley ton +ho ck +door way +chicagof ire +aali yah +sw elling +bi x +. ðŁĺĤ +evan kirstel +torpe do +kon stant +genevie ve +ma ia +ha user +do torg +hide ous +fi k +sp raw +e ek +z appa +wan dered +' ' +ra jan +bam bi +( $) +wid ening +tool box +sa ir +illumin ating +pra ys +out patient +i w +day o +lo b +sw fl +sha des +gu ms +coo kin +ko di +gri ffin +traum ati +ste a +slaugh tered +god bless +air time +pseu do +b sa +hau led +ar if +à¸Ńภĩ +le l +wc po +mil iti +char ters +worl da +ru k +k gs +digital india +is able +idyl lic +esp ino +marie tta +e bo +team canada +ab our +wil ton +rock stars +fav ored +phys ic +wrink le +tb r +d print +ball arat +ad al +z ey +ðŁĺį ðŁĶ¥ +tom lin +mt r +pal sy +fener bah +tight en +phil ia +ir oning +ry u +b ant +enqu ire +ca ir +abur ger +tru n +green berg +chau han +ir ina +sh ani +trend setter +pre tt +zaf ar +alo ve +v ici +pan ic +no o +lu stre +disrup ted +bal lis +son sof +mon si +inst ac +ake st +ëĭ ¤ +kw ame +horror movies +distric t +sau cy +mb an +ar mies +with drawn +med ics +loft us +er oom +be kind +ar ns +all on +un ison +davi ds +cr at +nicot ine +so or +sm x +on co +cospla ying +zombi es +har ms +e ger +ro sy +moon shine +fe in +ce tt +du brov +reg ents +ben itez +ðŁijıðŁı¼ ðŁijıðŁı¼ +ste c +m alia +prioriti ze +ic eland +ft se +v amo +lam ont +homo sexuality +bre es +regu i +cb p +te j +sky sports +deter gent +sha sta +de rel +conserv ancy +colori zed +accol ades +vis o +show your +nan ow +bice ps +us ability +bi m +dailys ketch +pearl jam +stran gest +mega deth +broad casts +bar ren +ar ton +chri ss +confi gu +lu res +is the +e ul +railway ana +global health +gi anni +u aap +s lum +consci ously +ab re +n up +bud get +v ada +e sch +real ness +er ased +th unt +be z +armist ice +ðŁij ¹ +sh run +o led +driver less +ðŁ¤· ðŁı»âĢįâĻĢï¸ı +won dr +sk an +sal aam +mother land +h wang +gen o +gang nam +tw right +endor sing +en ic +ador ation +pau sed +patric ks +do cked +plat te +ff xv +ethnic ity +auto show +side show +after life +re located +orphan ed +food network +dare to +and ra +sla ps +v live +swim s +re imagined +mist le +re vise +real ity +bhar ti +ðŁĴĻ ðŁĴĽ +late st +prou dest +gra sses +lan yard +fresh est +carcin oma +anom aly +zieg ler +sum ner +ly rix +gor g +is d +av el +swild life +me squ +john cena +euro league +sab er +master ful +yar ra +cogn ition +jacob son +abo lic +sir loin +shuk la +moj ito +su pere +st weet +me z +e sa +rudol f +gur a +where you +tt m +win s +trust worthy +ny k +bra den +table top +good food +es on +be k +lingui stic +gra ys +ch ath +h cs +mon i +de ans +cu ssions +ch ell +slo ws +he mi +d app +shar pie +boo sters +a os +str ack +se dona +mu eller +hard wick +or nate +thor a +sal ud +o twol +ch um +mi ho +for age +thel ittle +tear ful +ones elf +min dy +sm g +gmb h +emer ald +ðŁĶ´ âļªï¸ı +tu tti +recep tions +re vising +i brox +tope ka +sal ami +expan se +i books +dob son +cli o +at s +ðŁļ Į +mo ha +is ance +shu tters +moo t +jan ine +marvel comics +jor dani +pos er +kenne th +hy ung +de ja +ase ball +speci ality +eu ston +classic car +had ith +ðŁIJ ī +chas ing +iz o +gros ven +ag lia +thisdayin history +t row +om ile +hu ar +by n +sal ine +div ine +demon ic +ty ran +han dover +revit alization +pa ella +cryp tic +se dg +m end +dun kirk +bre d +wal d +sport scar +a ard +whe aton +da ener +k lan +br t +bakhta war +spi res +schu bert +ro ti +poli sh +o se +ag ame +wonder con +prote stant +bo sa +ðŁĺ Ł +d ü +joy ride +ger trude +âĿ Ŀ +gil a +v h +tw a +tra v +swal lowed +star ve +la in +ent ren +rei ki +su kh +cra ic +az u +web page +kee fe +hypo the +hir sch +hel le +camp ground +w amy +tra vi +sha hi +san deep +ru i +han uman +dw p +reposit ory +no or +no ff +un real +p ell +black history +har vick +ma scar +pay ee +pa sha +gastron omy +d ÃŃ +ai g +rosen thal +open day +embelli shed +t tip +sun bathing +go pack +end ome +ï¸ı # +invali d +final four +st fu +squish y +ra sta +mo sch +jam esc +die trich +sel a +mel b +el vi +t dp +sun i +sli t +j ha +bi za +spi ked +l li +l illard +vam pi +syno psis +az har +kendrick lamar +ĮãĤĬãģ ŁãģĦ +heart less +country file +air play +arrog ance +pre e +virtu oso +ãħłãħł ãħłãħł +raj u +le bu +for ward +tu g +dro s +mondaymotiv aton +concep cion +thel o +pad i +looo ol +ÑĢ од +it ss +eth ical +end uro +__ : +expend iture +mon ste +mas king +terri ers +ib is +e mber +cu mple +punctu ation +pi per +ir vin +ade e +yy yyyy +flash backs +cel sius +don nie +bo gota +ben evol +the script +shil pa +pro se +fin dia +ze ke +ne ko +do ves +blues lyrix +fro sh +sowe to +mp lo +al ai +sab i +raq qa +wf tv +stro ller +ian somerhalder +ðŁĶ ª +an on +mo seley +! ?!? +sta king +mol y +car tri +c sg +ast or +transc end +ma er +de ux +cow girl +sas k +pun ter +ma ken +o ates +love tt +grow ler +sag in +v n +ssi ble +officeof rg +y mc +sab ar +faul ty +ap ha +ak on +ðŁij « +snow don +ae w +raise the +ðĿ ĵ +grue some +clement ine +sp ing +lat a +worlden viron +mi mic +can aria +bakhtawar bz +ao a +fal a +ãĤ Ń +avi va +you uuu +thi gh +la dders +gu mbo +tz ky +fu zz +plastic pollution +est ate +strength ened +k ant +dr in +cal vert +transform ational +frigh tened +mac lean +elited angerous +ear thy +t son +to da +j nu +.. , +mic hal +i ban +je ong +is real +sim coe +exclu sives +blue bells +ben e +te u +pil sner +pens ke +athe ists +m pu +cartag ena +ðŁĴĹ ðŁĴĹ +million aires +kk kk +it ar +subscri ptions +remo te +ma fi +hin ton +w cc +ho k +ds b +ab leton +sevent y +pun ks +e indhoven +sh one +mcfar lane +lim popo +empha si +à ¼ +sin fo +pe tre +man grove +ch ino +ber tie +play lists +push awards +p af +deb bie +c do +r ino +ðŁı¾ âĢįâĻĤï¸ı +fol ke +bon nar +th ine +sl an +hal ter +evi e +aw some +vul tures +spar ky +seiz ures +âľ Ķ +ram one +ine ffe +al n +pro ctor +ast ra +the voice +gro te +sci on +dead line +am aya +tain ted +patter ned +exce eding +cross fit +kay lee +drop box +ru shes +tack led +mo by +retro gamer +n cbd +benef itting +shay kh +guild hall +gen try +dream cast +dread ed +bun dled +th aw +revol ving +n pt +kylie jenner +imagin ative +ron i +over came +family time +ds burg +car naval +relation ship +recogni zable +cor oner +ho le +fan fic +emir ates +bur ritos +analy se +thin ner +ne es +galli poli +bl r +cat woman +-- >> +au lt +ada ily +nau ghty +ili o +solit aire +mtv br +jocel yn +arun ach +rep ent +south gate +hy acin +essenti al +fent on +and um +it or +go pal +sl inger +po sei +aw il +wi elding +ra ila +eli as +a sto +à ¤ +tend ency +str ata +ker t +< - +im acele +da es +sti mulus +han ley +fit nes +ec stasy +lim ous +ha iling +ðŁ¤ Ń +chis wick +tar ies +sla v +pul i +moderni zation +black mail +b ingham +h fx ++ + +ðŁĩ®ðŁĩ ³ +ni v +we a +profess or +k off +bol ster +su ave +sequ ences +pepper oni +not te +dre n +ãģ¨ ç¹ĭãģ +hs v +o ga +ap tly +z ad +excel si +rin ka +mol dova +min n +ma bel +conferen cing +bas ing +of er +ob si +hamill himself +care less +brief ed +inhe rent +par ish +dub nation +town sville +sar awak +gee ky +doncaster isgreat +was abi +gu p +phen o +dra inthe +carrie underwood +ble eds +bbc world +ane w +alta f +dul wich +ani ston +w ti +sumat ra +gra fton +bl n +me ster +bode ga +re go +es q +an jo +sump tuous +mai sie +ï¿ ½ +wil t +jak ob +el vis +se pul +mu ster +air pollution +president e +happy monday +exten sively +fl ondon +t ls +play ing +pe ed +din ho +var dy +pi ka +n iro +au cus +ðŁį ¦ +nu ll +el ondon +juvent us +imag ines +dis ab +lit o +d ura +work places +promo te +mc caf +wood work +waw x +à® ª +tt ino +shar i +sem per +better together +ðŁijĬ ðŁı» +ze bra +pon dering +en chil +ho m +cosm ic +tan z +mo cked +ec cc +ath ed +abo lish +prop eller +paris agreement +assemb lies +indu stry +fraudul ent +pe sa +chang min +ax x +ðŁĴ µ +irr ational +cu sa +ramad han +octa via +on elove +jac ki +bar ak +taxi der +seri ous +nathan fillion +mc en +ch k +po part +grav ity +copp ola +reading fc +illu sions +j ig +ww x +re sh +ex porting +buzz ard +âĻ ¤ +p cm +lan apar +ko s +arom as +antal ya +ww dc +ven a +phil a +ball in +ðŁij Ħ +quin ta +ma o +f ery +eigh ty +sentim ents +safe guarding +r wa +pu ffs +luc ille +de cath +sl u +nu gent +de ter +braz il +ze iss +super bowl +subsi dy +alter n +hi dalgo +enz ymes +ä ½ +tag ne +hair dresser +adri en +walk out +oppo ses +can tina +bed side +af an +ðŁĶ Ĺ +prophe tic +dan es +un successful +super charged +pk k +exem ption +hart le +secu lar +cli pping +br s +united way +c net +pat chy +ha gan +e en +âļ ľ +var a +sym pathi +never trump +affir mation +om f +ny cfc +ma ja +sur ro +keer th +up scale +sandal wood +mon archy +kno bs +å ĭ +po tholes +hunger games +ter races +na sir +coun sell +welcome to +wa q +se aman +m ita +stun ningly +on theroad +in ability +) !! +bon go +ant v +sp ut +worldenviron mentday +resu sc +y td +fi m +eun hyuk +sa chin +rose anne +cler mont +ape c +am ina +v ening +n antes +al most +sin us +ex as +ty l +ti en +ple ad +lanc s +bur naby +re k +jo om +observ ers +disco graphy +cl g +âĻ ¦ +sn ack +r ti +o ily +crystal li +bru te +web development +topp ings +la f +an is +ad der +reli ving +car lin +battle of +we g +syri an +pon t +n dc +lagh ate +yu ma +sp p +p iti +ro bbing +mart ing +rey kja +raj put +nc ds +kie wicz +âĢ¢ âĢ¢ +vam pire +substan tially +opio ids +nepal i +k line +ar oo +under stand +lit t +u it +thro mbo +sar ies +qu ot +b alling +t tr +s gh +philip p +br ant +ac l +m ello +whit taker +. ; +defi ant +b gc +repl ying +mir ren +metamor pho +sch wab +bul ge +utili zed +pick ering +par don +d sa +ภĪ +doo ley +cumul ative +Ð » +ur gency +e mir ++ /- +¦ Ī +ot as +âı ³ +station ed +grape vine +ar ac +karan johar +f ancy +sau l +coo gs +lgbt q +ا٠ħ +jav i +u mmer +pl l +den is +dai pur +pu ffin +lewi sham +fand om +co pe +ves matter +s ve +hel pless +deo dor +ostr ich +kaz an +friday the +con dor +v x +sophom ores +rob les +cu tt +cli mbers +ë¦ ¬ +sle g +sn f +mac ys +hydr ating +grou pe +po yn +mou lin +hg tv +lmfa ooo +sulph ur +asdfghj kl +annab elle +hump back +bra ved +viswas am +multi purpose +hu midi +escor ted +barb ican +f ad +cor sa +ðŁ¤ « +pi ppa +here to +can y +ser gi +or cas +o vie +ed ou +s any +glob alization +man cini +food truck +f is +defi brill +sch re +sma fia +love wins +la ut +k aka +hol lande +game on +resurg ence +out side +olympi ad +int an +abstr action +rapi d +pal om +cal le +jas min +attack ers +swag g +mit ra +ky lo +à® ² +her mitage +gor do +e ira +so sfam +roll out +exc ite +sy nod +mer rill +c als +as sa +liveli hoods +ju ve +the black +gopack go +ant lers +alban ian +wool ly +qu iche +puri fication +are th +smar thome +ne k +all blacks +mex icans +is m +ger ms +comple xion +mar ck +u shi +ðŁIJ IJ +char l +ca stic +till erson +giuli ani +biode gradable +mal bec +bo is +ju bil +im es +r ame +gene tic +esp nu +ch ley +so ho +go pher +g sc +buu ren +cu be +bridesma ids +webin ars +to e +mani pur +viol ently +notic ias +ex changing +chi ev +replac eable +muay thai +bu ss +sp il +instal ment +div ya +cait lin +o lim +fil tering +whirl wind +sta red +prior it +pr am +pompe ii +mono logue +k ite +bu ka +âĢ¦ .. +vac cine +bre ro +woz ni +sol ent +re ferr +my rt +gridi ron +galatasar ay +fro ze +clare mont +ðŁ¥ ĥ +victori as +ssel dorf +pa stures +net neutrality +ch or +ðŁij ģ +ಠ¿ +we ho +symp tom +jo sel +in ous +dragon con +power ball +p te +four thofjuly +ec la +ear buds +where abouts +salt life +depriv ation +ch ter +wi ggle +syste m +ps st +ch az +d any +ri mo +oax aca +lanapar rilla +barcel on +melanch oly +way back +ho tro +n si +l illy +kur o +ja han +intellec t +board game +ðŁı Ĭ +sneak peek +k prc +jail s +cand el +zan zi +mor timer +star ch +ra gs +p fa +long live +k art +gir ona +cro cker +christop h +precau tions +war ship +per m +paren t +van gogh +gif ford +allegh eny +ra yn +ut m +sten cil +rec alling +pen ney +z azzle +ìĥ Ŀ +hin ds +aren as +nu ev +law ler +gu in +do this +ðŁij ķ +ì¶ķ íķĺ +we g +ti b +ri din +complex es +turbul ent +pe sos +de marcus +vall arta +sam sun +kis ses +hein rich +deport es +wil ms +ur d +then ext +inki gayo +ho wi +fir sts +carri age +clean liness +mas war +is ch +ax el +si zzle +road house +fr ans +ent ourage +co bble +boo th +benedic t +tal on +fc u +year ofthe +ray on +raider nation +fo yle +ko val +pi anos +l pg +bur mese +man ure +geo caching +cosc ino +b np +fer ra +stro phy +mar ais +ce es +legen dof +kat niss +eno ch +av ed +you know +d prk +ðŁĺ¢ ðŁĺ¢ +sp un +pro st +sor rows +cent red +ke a +gal icia +? ðŁ¤Ķ +ÑĢод а +bou chard +ðŁĴĻ ðŁĴľ +yu i +seed lings +jon ah +reco vers +ny rd +board room +su ma +my japs +tun g +sha i +ir gc +eli o +wag ons +ka shi +polic emen +john nie +ale coscino +shop ify +dot ted +de tri +va w +to fficial +in your +chal mers +trac ed +no vi +by es +ari el +nipp on +la pel +gri ez +b gs +fool ing +d ita +vijay sethu +nm wx +as ot +kr anti +hel m +ve di +sic kest +mo chi +k abo +shru bs +he red +b sp +sq m +ham r +dul kar +anth a +nr f +avoid ance +at en +publi x +be arers +nas i +ha p +h ells +ðŁĸ ¥ +ภ· +thelast jedi +oh wx +ðŁį « +wa hoo +there se +rec aps +ss nhq +bird photography +v ay +pet ti +pau lo +bel vedere +( * +gr l +du vet +c pec +sa it +por sch +meas urable +avi ators +fre mantle +bre en +on om +me and +life saving +eu ref +en don +embar as +aira sia +el is +dun kin +star magic +s ill +porto bello +ki efer +ex e +mu ted +ãģ ¦ +we thepeople +logi a +liber al +theforce awakens +min ed +haun ts +freck les +care taker +s india +âķ IJ +dev lin +list on +direction er +oh n +fi garo +em manuel +du bois +cl ones +bru ise +ðŁİĪ ðŁİī +disin fe +der matology +as r +s watch +dis comfort +tam anna +pi day +mack en +k atic +delu sional +shaw nee +gu d +al bino +p ali +din gh +cucu mbers +coffe y +anticip ating +treas ured +web summit +shel tered +sav or +pedago gy +m gs +sh ma +s bu +den ali +cam pos +bubble gum +o ir +le aps +y ler +r one +sansk rit +min t +meat less +futuri st +du de +a vel +prote sted +squ ire +z aki +sz n +har court +cycl one +bour dain +gather ings +d ant +advent urer +parag on +alt man +dd ing +ban erjee +snorkel ing +mother well +mis sy +en der +glo ws +ki wis +chick pea +por o +e fron +app t +u y +speci fied +gab by +e strada +com bos +bour bon +vin i +var un +steph ani +key words +car vings +amit abh +wr ought +tw al +re els +clu bbing +ubi quit +cri t +ambed kar +æ Ļ +prun ing +vaccin ated +boe ing +s ks +lo ona +hypno sis +edel man +pho l +he w +colo sse +mckin sey +u on +to te +sacrific ing +ox i +n ang +e mu +пÑĢи ÑĢода +m th +kers wednesday +argu ed +timel apse +ris king +regul ating +ni gh +likeli hood +cu bic +au ction +rein for +pi stor +no ses +ye l +snu ggles +pe i +jean ette +ta ku +ri th +guy z +ภŀ +y te +ver ted +pay soff +jau regui +hoo ligans +procedu ral +mi b +har dy +el eng +chec kers +all ine +the met +prou dof +keerth yofficial +collabor ator +ni u +infl icted +adv ani +re twee +memor iam +f icial +ti ghter +sal em +re viewers +br ics +ben digo +am ell +tur kish +sush maswar +paul son +pal awan +mol lie +stitch er +s burgh +ir u +hay dn +en ers +aro a +u zzi +saraj evo +hel a +apol lo +nine ty +vac a +sp on +vent u +jel ena +hei fer +avo ids +sp ine +pri ze +mar ist +re creating +me de +woo den +find lay +ro fl +n di +compreh end +yu go +y ü +to work +u fos +son ar +pi ston +recor ding +tent ative +art forsale +pel lets +fre do +ÙĪ ر +mu ses +custom ization +pro found +is ner +ide ally +si am +plan kton +cm dr +man ger +fran ken +customiz able +ठ® +walk away +swi vel +vast ly +no ton +lex a +ex moor +z as +tan te +reduc tions +lol ly +hip sters +benef ited +ë ² +ww www +mascul ine +fi ji +dre y +ph ill +ane ous +nic ol +men dez +disapp ro +ch ner +through s +shen mue +east man +ðŁIJ İ +yu ck +under tale +re ys +go beavs +eng en +c na +mer r +bir k +ãģ¨ç¹ĭãģ ĮãĤĬãģŁãģĦ +âĥ£ @ +yn na +ste ed +offen der +at um +vani shing +presi denti +love them +g nocchi +fri ggin +per il +mad hya +ag ne +dee jay +mar nock +m tb +fold able +@ ___ +stand re +bron x +bow ski +fin ite +cro ckett +b sf +ge tit +seren awilliams +mir o +ignati us +sla y +rin se +fon due +sel dom +s more +gan i +dy ce +dmit ry +cru mb +late post +pri mark +oh ana +flor als +do a +remembrance day +d ds +azi one +toon ami +air port +æĿ ± +th ad +fi st +dine sh +dr who +ad words +admi rer +pro je +kyrgy z +à « +manife station +le wan +j ic +thi bau +le ased +van ity +nouri shed +never theless +aug mente +fu elled +che ad +wil shere +ru di +p z +my co +mor ro +herbali fe +hardro ck +de man +dre ality +sp ades +ce vic +bha i +bar on +ultimat efan +hou news +to bi +stru t +ke el +affili ation +the masters +sm al +hu e +este ban +con v +om nic +datab ases +co v +ter ti +st g +snoop dogg +metab ol +leth bridge +ðŁı» âĢįâĻĢï¸ı +year ling +residente vil +nws l +iy aki +griez mann +c ous +ðŁĵĿ : +tor ian +sam i +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ +g are +alli ances +whit field +we ther +refin ing +coy i +kra ken +ðŁĺĺ âĿ¤ +singul arity +lil i +h ns +bol dand +waw rinka +misogy ny +lo vers +c q +b dg +ad ona +gar ter +women of +sc d +recogn ising +mun a +str ou +sign alling +lare do +hell boy +alek sand +un available +pedi atric +as in +mer ia +ri shi +futuri sm +w ye +polari zed +e we +pro pel +in forms +cre ase +~ " +arti ston +like for +heidel berg +er ra +life in +len ny +inter rupt +cohe rent +ca z +vick ers +le veled +f bs +cab ins +bu mmed +apost les +we h +ten don +souven irs +infu ri +pier ce +asse t +m las +go th +di ggin +ann as +yl or +th waite +sw el +pan era +mur derers +croo ked +bs go +ac u +a on +re an +one of +ko hl +bloo dh +pest icide +lost dog +fle xing +ëĤ ĺ +su pra +eter nally +ðŁļ Ļ +pa olo +ol an +mom o +is elle +captain marvel +s lou +mistak enly +akhi lesh +mer t +il inan +bu on +bal kan +mir ro +mill en +der ail +dam on +tit i +bi os +re don +pic ard +par te +ðŁ¤ Ł +Ø º +son ics +fir sth +dd c +veg ans +tur ban +ni gan +lot tie +lyn don +star buck +pink floyd +life styles +am ara +a she +r sc +val a +sm er +cw gc +cli ent +buen as +jag an +coo ps +ðŁijij ðŁijij +speci alizes +snag ged +g lar +ben net +wildlife wednesday +bow den +pi k +art in +empor ium +ar l +re ba +pas ser +disappo ints +additi ve +âľĬ ðŁı½ +bay er +missou la +ha skell +comm ences +ni x +ne man +explo ited +plastic surgery +cc d +aso cial +vo t +sie gel +fro ome +kap am +far a +e ha +pro bes +mw f +meet ing +p bb +ak ins +mistle toe +kingdom hearts +for kids +ec r +bal e +escor ts +adidas originals +k wa +k ts +hallo ffame +ðŁĺį . +wag s +pot ted +o wing +honey comb +he fty +uro logy +mer le +b pd +stri pping +re ich +k state +gu ay +yon ge +shak ti +g loom +bat t +son om +n ery +el ba +blan ks +hel le +triple ts +bom bay +ak arta +ab ia +transm itted +rol f +ja is +angular js +fi erc +m ss +trac e +ॠĩ +tom bs +old man +kom bucha +fo l +e health +cere als +are lli +in ari +ðŁĴ © +wo l +liber ties +fa wn +af firm +nun avut +hyster ical +k drama +art es +âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢ +valent in +man slaughter +gal es +eo in +energi zed +del s +with draws +st les +sar castic +ram esh +incredi bles +lock hart +ya wn +ultimatefan live +oooooooo oooooooo +mu en +guru dev +te er +pe eling +new snow +lingui stics +direc tv +ag end +uni lever +ru ger +han dedly +ero se +li mel +the c +royal ties +fini shers +nr g +m gt +fid get +com ps +bac on +aggre ssively +ab it +ch â +tar de +slu gger +q anda +gre ening +d ats +ensla ved +spec tor +o ye +fre ef +b hand +stop brexit +mis conceptions +cav a +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį +multit asking +hou sel +ferre ira +cen time +ank les +jo dh +hel ly +fro me +out tuesday +nar nia +bal aji +l bloggers +jyo ti +ðŁį ĩ +lan cia +cap ri +y ap +nat ash +down fall +." âĢĶ +à ® +ligam ent +coat ings +ai ded +hi ko +fall ing +encryp ted +yeg food +infringe ment +cu di +ce p +ðŁĺį ðŁĺĤ +tra d +super rugby +ed win +wh iche +vi meo +lay ne +in vigor +he he +dubrov nik +bie ber +u tr +sham an +op ers +ham ill +en ig +di f +ar um +scrap book +min h +diver gence +mckin non +life time +guter res +wil le +ple as +patt y +mic ron +k z +dom aine +ru sher +m ds +ches ney +screw driver +âģ© , +sle dge +hau er +chan a +stam ina +sprink ler +pl n +he ff +bol ton +om on +car rington +accor dion +jor ge +inter ception +in puts +gu ll +tran scription +vanu atu +it ical +eth os +tic h +spac ey +pee king +u mi +ha ger +psycho tic +illi an +illi a +bonnar oo +an ese +pu c +laghate parth +en hall +econom ical +dre dge +% - +u we +tu bular +scoun cil +pe asants +fl er +tumb ler +he p +ford ham +row ley +initi als +ev asion +er nation +plu gins +coch ran +c attle +acid ity +ðŁİĬ ðŁİī +re grann +jump man +ef ace +x ma +patri archy +esco bar +cristi an +tip ton +nu eva +hack ney +back seat +kill arney +aid an +sta dion +simul taneous +ida ho +a je +u th +figu re +clo s +bur k +volun tar +rec ite +macfar lane +cur few +bou do +w gn +sti x +sla p +scrat ched +philli p +jour ne +ex pelled +wa z +u ke +tati ana +ou e +ho pp +dimit ri +ðŁĵ £ +mato logist +electri fying +blu ffs +bill smafia +az cardinals +y aa +x mas +shar a +r ith +g ills +dre s +bar ton +authori zation +imperi alism +home of +to do +foot path +band width +visit spain +moh sin +erup ted +mi ki +insig nia +mike l +ss h +ger a +bank holiday +aw an +t weak +star craft +e al +construc tion +skelet ons +le ep +ine m +bar clay +ship wreck +monsi eur +yo h +ron t +form ative +ser o +le p +horse man +hoo sier +haz mat +cylin ders +cen ti +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ +re em +na ire +mus ically +gras shopper +est onian +termin ology +ro main +blogger rt +tox in +stan ce +cultiv ated +an ast +ðŁIJ į +shi mano +go pher +ene i +recycla ble +gam ification +fight for +c q +avoc ados +ke ys +eli ke +gly cer +shak ur +mobili zation +gal ley +expla in +ex changed +pe th +obe dience +illa ge +en nis +ãĥ ŀ +wi v +walla bies +ma ar +ig ers +fin tech +fin alized +wo j +meaning less +in field +onna ise +e et +bron te +pass ages +ðŁij § +strick land +northern lights +lom ond +h tc +wr ay +shi fter +di alog +ðŁį į +>> >>>> +te atime +ste ch +sic huan +qu ill +fran ca +comple mentary +bar rington +marcu s +mal am +goo oo +for sa +elec tra +af s +âĹ Ĩ +tri fe +sn azzy +fo lia +and olan +after dark +wood son +stra de +litt lest +o gun +con wy +co wards +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +íĬ ¸ +se ul +mur phy +dun ks +kapil shar +jo achim +wom ack +equal ity +aver ages +a ine +ðŁ¦ Ī +tac ular +dis ability +u ked +mid century +bar thol +teas ers +tab ern +nj caa +sp out +op i +ku bball +bl om +so ar +popu lism +meth yl +ðŁijĬ ðŁı¼ +o spre +alo ils +ðŁĵ ĸ +ðŁĮ ļ +x er +sp illing +publ ica +car dam +adi sh +sa cha +p kg +bu da +lyric ist +i bc +gru mp +ho ver +hal ep +anti body +anem one +âĻ¥âĻ¥ âĻ¥âĻ¥ +m cl +litho graph +cc u +s fest +path ic +calli ster +otta wa +gun sn +rut ger +hali but +en vision +differenti ate +ðŁļĢ ðŁļĢ +pir an +lat el +uc n +trou bad +ra ine +fierc ely +learn english +lea se +wex mondays +em it +dray ton +bur rell +scuba diving +hol ler +dr u +clo cked +w ral +ap ro +trans lucent +w bo +patri arch +mo ja +lan nister +fish ery +ne derland +mil dly +mi rai +ma ko +ja p +ðŁĺ©ðŁĺ© ðŁĺ© +pro statec +p anna +ar ama +under taking +tomp kins +ne op +soli ds +sav oury +e ames +cut lery +wood bridge +steam er +ri zzo +wild cat +rat na +lamin ated +kin eni +jal ap +ai des +acknowle dges +?! ?!?! +! ðŁİī +w afc +mag gio +ha ves +dar je +of i +gr il +v asi +bru x +mo hd +fake speare +arn old +r mb +for be +wal leye +ro di +therapeu tics +strate gi +ob ste +mu dder +download able +dd ings +d ca +asi angames +campe on +appropri ation +th century +ram atta +dra ped +bul lion +mu c +one x +se greg +ophel ia +bod ily +âĿ¤ ðŁĺį +wi zar +te ased +ade my +to id +sur a +lazar us +sn ickers +ma se +lo h +bow ed +bibli o +x change +har lan +gho shal +flavor ful +bha gat +alle z +whiche ver +ten stein +disc er +organ iser +mt g +dream liner +t se +hok kaido +mo k +indulg ent +hick man +blin ded +al yn +aaa ah +sp ool +lough borough +inter pret +et v +aristo tle +optimi zing +avici i +madu rai +ju li +naw az +mat chups +ab ide +paint ing +w elling +vel i +octag on +in scribed +po king +plac er +life cycle +kili g +g sp +eli ves +cle ments +na sheed +me sut +incarcer ated +dist illed +wal ang +delic acy +del gado +che z +ch ita +ad ero +tu x +pati l +o do +abh cosmetics +tv c +p bc +in accurate +hardwork paysoff +ball er +quot ation +merchandi sing +ga stri +defen ses +dro gba +bex hill +ban kno +win ona +si eg +p gs +hahah ha +agu chi +su bram +mirac le +de sch +li bre +ba cher +ent ine +bbcra di +lou dest +r ps +pi erc +fr yer +storm trooper +rafael nadal +pas co +exhau stion +epic onetsy +rc tid +kel lie +ga ines +d bz +sm riti +s bridge +lim ited +cla w +technic al +bio graphical +ado red +ภ° +exclu de +ac adia +key boards +fur man +so ca +sur u +ni ps +sw aps +server less +run e +pu ffy +north ampton +nish ings +hen der +cartri dges +gun shot +ðŁĵ ¹ +fil ament +respon dents +pey ton +mountaine er +mer ging +life span +intimid ation +p afc +nl wx +expan sive +pur r +f ck +ca e +at ti +tele thon +so hn +mend el +lo pes +dor i +un broken +te red +tast ings +in active +disin tegr +t assel +share the +pi ano +is lay +air space +z awa +ricci ardo +ming ton +fresh er +cur ry +re vs +pharo ah +h mv +exhilar ating +wh oo +lin kin +kri spy +competen cy +ste wards +ne bu +kat su +ad mins +baz ar +as ar +giving back +s summit +song z +lin us +raj kumar +farm ington +fanta sia +ðŁĺ´ ðŁĺ´ +so bri +lis se +barry more +pri sm +blo b +sen ew +mono xide +exp ire +eigh teen +di pper +xi ao +kil t +hin ch +bbc sport +bam boo +p ter +ex al +ðŁ¦ ĭ +ham lin +expe ditions +star gazing +food security +wy lie +ul f +st ingly +on storm +lo eb +bro ome +bn ha +pancre atic +eli ve +!!!!!!!! !!! +ther apper +ortho pedic +avengers endgame +antit rust +ìļ ° +go te +om d +off side +gy llen +win eries +white water +ad l +lu pita +exce eds +consi sted +chew bacca +ash leigh +nhl jets +is san +sh ld +hay at +cran berries +ðŁ¤ĺ ðŁı½ +rock the +spring training +fall out +dairy free +wa j +un decided +so wn +rc n +north wales +htt r +fu mble +d its +comp elled +popu list +min ted +blan chett +. '' +pro pulsion +m illa +au berg +her tz +h ta +u daipur +serendip ity +azte cs +als ace +ðŁIJ ij +lu n +sho es +char li +gar za +ðŁĴ Ł +pro biotics +fox tv +ol is +mi ff +loc alized +diffu ser +si gue +fun ko +rend ous +ðŁĴ ij +jeky ll +ha bib +fre ya +fjor d +ex porter +to sa +store day +maj id +ba the +cham paign +ðŁĵ Ĭ +der ma +h ittin +gor illas +emo te +ac ic +mad ly +lland ud +kru eger +eleven th +ash raf +umm it +at as +per sie +mo tives +i ona +finger tips +ss m +pon te +bri g +rb is +tu sk +ps vita +jor dyn +ci el +bas ket +are d +arbitr ary +go ed +chron o +sand box +performan ce +na ke +ant our +vas quez +quad rant +mat tis +ìĪ ĺ +sa har +numis matics +ma this +tr ams +pot w +as quez +? !!! +thro b +of life +_ ! +pan tone +mcil roy +er u +ma sto +endu red +co vent +ab hi +physio therapy +civil ized +ant asy +snap dragon +on screen +micro bio +l cc +di mple +sl ough +ma ven +col m +villar real +or p +fr ye +bar u +v tg +perio dic +concor de +childrens books +ym ru +re mark +je w +u tica +seclu ded +rogue one +ag li +why we +ro bu +nur sing +lu ster +automobi les +ic um +cl i +sagin aw +pean ut +ec ra +transp ho +bl ins +aw wwww +âĻ¦ ï¸ı +jere z +inst ances +sy on +s de +wp xi +rob ben +man x +journ al +erne sto +belle ville +as ur +wal rus +h j +cab le +blizz con +bean ies +vic inity +te igen +ta ire +pa v +navi dad +extr ater +bun gie +bbc papers +algon quin +zanzi bar +out fielder +mer ced +m q +kon ami +sho ton +hor rendous +ad vo +spoo k +nbc sn +tu tors +en tos +sur name +pay ers +mul der +be be +radic ally +bu eno +re brand +it ching +fer o +zo u +la i +gon g +weather network +rick son +recon naissance +f sc +differenti ation +be stin +y q +st as +lon gre +pro fan +mar ac +opol is +ba its +ab se +sy r +ph us +u don +schizophre nia +reg gi +jen a +deto xi +com plac +z b +pr t +ibrahi movic +bm j +seduc tion +oooo h +gonz alo +w ham +sha p +deser ts +callof duty +ðŁķ º +photo booth +bri m +si en +scri pt +cas par +line age +x ero +may bell +co ta +carls bad +ðŁĴĥ ðŁĴĥ +im ba +the car +law x +il k +g ana +sli d +ma halo +g ant +enri que +te i +jo ck +bla de +h la +i hs +west on +trans it +au bam +lone some +kobe bryant +fun ky +v andy +sh aka +an an +person alization +rede emed +hat ter +day s +par ac +living stone +for man +de mar +ðŁijıðŁijı ðŁijıðŁijı +per sia +pe der +ðŁĩµðŁĩ ± +reli ever +ith appen +dc p +den burg +Û Ĵ +k assi +un pleasant +ij u +far o +car mar +ke ren +ha u +scot tie +s bury +r sc +pistor ius +mp ire +mo at +mar uti +lion s +back country +rash ford +haras sing +ze etv +t la +tor so +sau d +ent ang +ef abz +toshi ba +resi des +âĿ ŀ +r ct +mohan lal +memor andum +hor ner +bon neville +g sd +exoplan et +blasphe my +am et +ðŁĴľ ðŁĴĽ +spo iling +ma as +ka sey +coim bat +ðŁį Ĵ +tu ske +su zan +still water +mit z +keep the +gosp el +dum best +distr actions +ch lori +ãĥ ī +sophistic ation +mm u +lithu anian +bell ingham +ðŁijĢ ðŁijĢðŁijĢ +strongh old +mon aco +k ad +dog sofinstagram +ðŁij Ļ +west ward +sedi ment +pal met +ko de +ki do +nom ads +ff ff +augmente dreality +ðŁĺĺ ðŁĴķ +upro ar +ty rant +sty lus +sli e +deli rium +occu pancy +hat t +hair stylist +ear tist +spal ding +never mind +read able +p met +fac ts +ot to +she im +sch am +go thenburg +ex it +ty n +tam worth +roof tops +mutu ally +j mu +fis k +cun ning +re news +me tic +an tho +mcel roy +con tre +ab ank +mi les +deser veit +dear born +ab ir +cruci ble +character ized +tahit i +mur cia +che tte +uni vision +pres se +love e +im pun +ast ana +a au +o vs +loo sely +ell ing +echel on +connor s +n th +ty ch +jim bo +cor don +app reh +. ðŁĺį +jiu jitsu +acol lins +sushmaswar aj +strike outs +proto types +ascen ding +argent inian +ren ner +# ' +j y +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ +nanop articles +iz ers +! ðŁĺĤ +por cup +edwar dian +dx b +.. !!! +mil king +f ours +the d +ðŁ¦ ħ +writing tips +sim ms +ele mental +whis kers +rain er +ou che +influ x +å¥ ½ +snap chats +pi eter +g awa +c nt +ley n +slaugh ter +k lay +ger m +bon ne +ðŁı¼ âĢįâĻĤï¸ı +wic ke +i at +border line +* .* +ent on +ou ss +yas min +tow son +roll s +ho ho +bant am +skill z +cl o +sf u +conden sed +school boy +london ers +ãĢ ij +vand als +sat oshi +ðŁĵ» : +sin cer +ni etz +i awx +grey son +graph ed +gabri ela +je p +ha di +fron tier +ellu lar +con fluence +ðŁĮ ł +white out +mer it +shi ra +sculp tural +incar nation +life guard +mi de +bar rio +attribu tion +app re +re eve +mag ically +din al +broad casters +tend encies +su bb +reykja vik +min ts +goe the +shi i +aubam eyang +:- / +ี à¹ī +eat ing +du mbo +oc key +ber tha +am ata +aa g +evacu ate +hu tt +dr f +what aburger +tb acks +li vin +ap an +vo a +vi kas +grand mas +inter fere +dor itos +bon ner +f gc +pi ñ +per mitting +limel ight +de anna +le ela +ha st +fahren heit +ale ssi +ðŁĻ ĩ +lie b +dee zer +cul tura +vo ss +pa si +ma ud +is it +bent on +din ers +theroy al +refu eling +ent ro +sky f +mar ital +ke ene +super power +rebec ca +inform ational +hi deo +co wardly +ãģ· ãĤĮ +u sha +t ere +summ ons +ar da +or land +freel ancer +bbce arth +v agu +in sh +blo or +pot luck +poche ttino +che ats +wondr ous +euchar ist +canc elling +st es +esc ent +en den +ssi es +sand usky +bi anco +oppor tuni +liqui ds +kyrgyz stan +ai ah +gn i +mo vin +ina via +coo kie +âĢĵ âĢĵ +ol icity +ðŁį ½ +un filtered +dre ary +bbc breakfast +amar ia +rais ins +ðŁİĤ ðŁİī +sand ler +gan j +fe in +music awards +ne ta +flur ry +er re +bri ana +posei don +mul an +execu tive +dhar thm +ch ins +thirsty thursday +jam as +bar th +tn f +tac ob +k hor +mi ma +fil ms +ington post +epit om +ec w +cor ral +weak ened +ak ov +shi pper +m line +la sal +bra iner +aw m +ðŁĴĻ âĿ¤ï¸ı +twi g +this girl +man of +re count +lan zar +for ci +dischar ged +world news +mon strous +in toxic +fo ie +demean or +af firms +hal ves +che shire +se of +lanca ster +g enders +star r +chick fila +new england +jay den +ðŁĺĤ @ +sha allah +ase efabz +flamin gos +confron ted +chi anti +a om +cab ot +af loo +pi kes +leav ers +h cc +chap o +young stown +re solu +okla hom +o ons +lamin ate +cash less +ðŁĺ³ ðŁĺ³ +z aw +sa ires +rebel li +in adver +ben i +tra c +hun tsman +ðŁİĦ ðŁİħ +mer may +gi b +die u +ce ases +ðŁĺĤ # +mind less +i der +a tho +wheat ley +profit ability +un attended +in ec +han sika +backthe blue +st f +drau ght +anto inette +v ah +se ash +b ina +cl r +ari zation +ben to +ภĪ +ze man +inspec ted +ar agon +ðŁijĮ ðŁı¾ +tack y +rich ly +race track +anthe ms +abbo tsford +sheri ffs +i ah +en ough +e strang +road ways +bun k +sh anti +jurisdic tion +gur us +far r +ad on +in cogn +home improvement +dal am +col lars +co hn +be da +ai re +wester ly +avo te +spin ners +sp res +occup ying +sch rei +reinfor cement +es er +sun rise +mc manus +gold stein +gg gg +ann on +yo s +re patri +hud gens +data analytics +ag us +ðŁį ¿ +pol l +just e +gi annis +star struck +dundal k +z ap +sk ol +un miss +u man +t cr +plat y +pac man +na an +pleas antly +ob vs +corrup ted +am ari +sho ve +nau til +shi van +sh reve +go bears +we akest +bren tford +st us +pre v +basing stoke +reper toire +am ala +ç § +ch ong +c aged +bil al +! ~ +yo w +wan derer +l ila +en clave +ae c +æ µ +don ne +ðŁĴĥ ðŁı» +tru ce +he il +scor ching +iri descent +ob taining +fon dly +centuri on +buff on +seren ade +break the +sap s +ny gov +la zi +\ ( +puer to +neu tered +ta sia +racec ar +hic key +gan gu +ðŁĴ ĩ +ran cher +cla se +ðŁĶ´ ðŁĶµ +ho b +bi zz +ding le +tw restling +go go +freder icton +block chain +tu ary +perce ive +jo int +es u +emabiggest fans +bis a +win ton +re counts +re launch +m ths +ar ises +ad kins +mo tions +la wns +eno cide +reminis ce +ra pun +w kr +fass bender +e manu +sexu al +hi ppy +wine house +f dc +care r +al ai +profound ly +our o +mon toya +mee e +is cher +imp lies +fifty shades +ym on +together we +isleof wight +cru e +am zn +âļ « +me ps +haun ted +got vintage +ter son +pet smart +sell out +ne cked +entom ology +eng ar +deal er +alo re +ðŁĩ¹ ðŁĩ· +par tum +limer ick +f ates +dwell ers +diag rams +ðŁİĪ ðŁİĪ +pl ore +in ca +divisi ve +blow ers +wel les +predecess or +infin ite +theore m +hot dogs +americani dol +dam e +cap ers +reco ver +lolla palooza +in correctly +colle en +brac ing +observ ance +o ise +mr n +gran ville +estrang ed +íĭ ´ +replac ements +je sus +d st +wild wood +ta f +sar ri +horser adish +am ax +cor by +con d +cit rix +t ze +sa ic +i os +mon gering +ðŁijı ðŁı¾ +jeffree star +bar ometer +avo y +yu le +var na +v ÃŃa +paraly zed +under went +ge tter +dau phin +stra r +aberdeen shire +organ ism +ing an +fei sty +ri da +worldof warcraft +tic ker +sho u +ri ff +craft beer +thur ston +s abo +meatless monday +migr atory +ma jo +gro sse +ag chat +net te +essenti aloils +chau dhary +teddy bears +archan gel +rotun da +re us +ham ad +con vent +britt le +mar che +lo han +inti mi +eu cli +b ole +s ra +ho d +m fs +di sts +cha stain +z or +she k +canne slions +l ends +cal um +bru in +alam eda +ach ri +privi leges +indie music +fel ton +po ty +cor so +ri shi +ha bi +a see +weir dly +r ho +myster iously +low down +fur s +fe t +e die +ro sh +inqu ire +vulner abilities +sil o +nation alists +ad iti +tra pp +ti i +scrat ch +ag ora +psy che +davi de +book marks +ðŁĴĽ ðŁĴĽ +re former +lu tz +ðŁĺ» ðŁĺ» +long island +awar dee +po stu +d printed +succul ents +poo rer +l da +r cc +ivote btsbbmas +cath letics +ti psy +quin ce +pupp yday +âĸ« ï¸ı +tz el +sel fridges +i onic +wab ash +turbul ence +leam ington +tt ttt +obsi dian +o hara +legitim ately +spa in +mor al +equal iser +ap g +watch ful +w ls +h ng +ro shan +mart es +falk lands +d hl +tri angles +sta un +south bank +ren ame +quo ti +god desses +col gate +z ant +trail running +summ its +dd ick +ac ad +sc g +medi ated +ko hl +here wego +discrimin ate +sat irical +publ ici +g tc +dre dd +auto sport +si ps +correspon dence +ash win +dragon ball +ðŁ§ Ģ +ship ments +gly co +fe a +pur ses +le er +gie g +ba bel +ni on +n ca +ko a +go on +rec a +female bloggerrt +elector ate +da x +ic ulture +elli a +tun i +tor til +le tour +coimbat ore +activ ating +news night +had dock +free shipping +cano eing +ay n +ocean side +nick el +jame stown +fri gate +depend ency +cho wk +cataly tic +backstreet boys +Ð ´ +ele ment +^- ^ +zen it +ro a +fortun a +fi zz +ac lub +ÙĬ Ø© +in tra +hy ena +do dging +archi bald +mari us +ing enu +steph anie +scand inavia +ma ier +joy ner +christ ening +b td +sug ary +men e +immer sed +dom ain +ðŁı ī +pap al +ic ann +ta hir +me jor +it ys +inter fer +im pul +allo ys +" ). +z ance +an ar +tam ar +coy big +au ghter +manhatt an +ko di +wedd inghour +gla zing +bh f +depor tivo +any c +nouri shing +no tify +j py +de dition +big brother +work station +r allied +ob u +impun ity +gyllen haal +you rown +sm ite +n du +s le +o am +home opathy +gro ssing +pa e +le mb +was ser +audre y +ðŁĩ· ðŁĩ +sho pee +par que +ophthal mology +ðŁ¤ĺ ðŁı¼ +thou ses +t itu +st illness +nygov cuomo +no ta +disa ster +car den +b sl +ðŁı ħ +re po +r ate +hil da +ck en +g pi +crit ter +u hd +deadline day +tom hiddleston +sem pre +mull in +make americ +ar id +am t +n se +n ch +moz illa +food waste +co or +sagitt arius +po el +e try +c fc +kil o +av ant +pist ols +mis sive +bah rain +fa e +drin ker +war mers +sv eng +po co +its the +inter ce +pra dhan +id sday +tain able +sub marines +magn us +bo ye +am are +pen it +g fx +aren e +ãĥ ĩ +su rah +jay son +in ch +bo yer +o sun +str ati +scrip tures +master che +ster ili +program med +kn its +inj uring +sea of +reli ant +p ina +mix tapes +man tri +jind al +hac kett +bio shock +v ash +sp m +light saber +w icks +rune scape +vari ables +dimp les +ol yn +hol lis +getty images +galax ys +ed l +trajec tory +thr illers +positi ves +kit esur +del le +feel good +shan kar +ma is +is lip +ricky gervais +ingeni ous +rr bc +si p +acro polis +p buh +mesmer ising +bernar d +too t +restric t +murder ous +fo i +dul les +belie ber +sha an +ph ant +hamp den +ha ye +ch ro +ðŁ¤· âĢįâĻĤï¸ı +vi endo +mag pies +habit at +fl icks +stan za +pu tney +new smel +nd n +m ity +contrac ted +uked chat +sp ouses +plu ms +l icious +quan tum +j hope +mm r +cu sd +usa in +section als +bar bers +re vered +d ite +aw ine +mc daniel +pur dy +pelo ton +out lined +ben ito +pedic ure +moisturi zer +clif ton +prece dent +ital y +bi x +tro ye +tren ding +shan ks +le tic +wilms low +ta ir +kry p +en u +kar thi +hoar ding +surve yor +inst aweather +ri ffs +evic ted +town e +ordin ation +lux or +tampab ay +guine as +fu mes +no ck +ki ara +en visi +no e +geor gi +cruel tyfree +whe eled +te mb +mi aa +bu oy +abbas i +mc col +jas per +it als +author itarian +ma ura +tang y +mu ssel +hi gg +chlor ine +al vin +whi ps +re side +hra ya +ed ging +utt ar +ide l +du d +wo p +summon ed +ìĻ Ģ +å į +si kh +en viro +tan kers +nbc news +le bone +gw r +con ia +colosse um +rod ney +par atro +nau ghton +fe athered +chand ler +au se +! âĿ¤ +ni ko +miami heat +collap sing +ib f +gaf fer +father hood +camp ed +ro gan +hi jacked +coordin ates +am il +ðŁĺĴ ðŁĺĴ +e ther +water gate +leg er +d wy +c tly +acry lic +whole sal +ven kate +shadow ed +hor sham +bangla deshi +to ed +inst atravel +opt outside +aar p +far ce +ag in +!! !# +rap ture +lou th +mon ti +jiha di +initi ate +gro hl +u do +tech nicol +ou pe +but ti +ðŁIJ ´ +nar ayan +car la +ma kh +indi visible +ground hog +yn c +sin bajo +ban tam +wc f +sug g +pin di +them atic +rit i +kk h +val i +ty ou +lebu hraya +ko witz +sla sher +kit kat +cy pher +val u +us man +rock ville +kar ni +do re +í Ľ +fer ret +ðŁĺĬ ðŁijį +wood ford +statu tory +love and +tar p +referr als +discipl ined +yach ting +ktv u +dec king +au m +ph or +key west +a ina +ped tour +ge ti +sla shed +cric kets +gr ated +steph an +lewan dowski +intru der +al c +ðŁĺĦ ðŁĺĦðŁĺĦ +merci ful +lok sab +con sign +ab m +o shawa +fi eds +di jon +y ass +wre aths +well come +tath lon +mitt al +age of +rein force +dra ining +coy b +ac ec +inten sely +hagg is +fle mish +wool worths +partici patory +lan y +convic t +stereo type +ðŁ¦ ĩ +re sale +len i +hol ster +âĺĨ âĺĨ +âĺ ¹ +renew ing +par ted +batt ers +weak en +erup ts +sun il +nouvel le +lemon grass +tour e +h x +ç ¾ +schi phol +mess ina +han bin +daener ys +butter cream +gu o +con roy +bla k +ad ic +ach en +Ë ĺ +tran sylvania +radi of +te ren +dr fc +b ber +ay ing +alcat raz +w ld +mill ard +ìĿ ¸ +super fan +ph am +gh wa +fre ight +µ ï¸ı +infer ior +libr o +goo o +cam bria +six es +quintess ential +mat ern +j ours +hy mns +gen a +wil de +white chapel +shav en +q q +slu dge +eat clean +mariti me +ka if +bjor n +pire lli +ja sh +i gi +whis kerswednesday +the originals +sch illing +ph in +jo ke +jal sa +gen ial +rod ite +for ge +ad er +ðŁijĩ ðŁı½ +deb ated +ðŁĴĻ ðŁĴļ +woo ded +mun oz +dism al +condem ning +ant ara +saturday night +re consider +ðŁĵ ² +ol amide +hit achi +harro ds +ta way +ja a +ing uk +au c +az ette +as bury +ultra s +ri que +de ca +al oft +om ba +mi gh +me sh +fa ze +sp ital +v ado +r z +mori arty +tu ck +tou m +mon stro +sain te +ru skin +re discovered +par ais +mocking bird +cf b +tu sk +model led +black berries +spo wer +j ale +hot spots +bri m +" ," +yor ke +ap ri +mi eux +carlo s +welove you +firsth and +es thetic +rox as +j me +ho i +sch mitt +u chi +orangu tan +lead ing +def o +weekend vibes +refriger ation +inter viewer +faroo q +... :) +wy combe +rejec ting +red knapp +pi que +twee tab +middle town +palad in +balti stan +ðŁĩ³ðŁĩ ¬ +mc phee +bl medieval +ide o +e special +cc fc +ath ai +am pa +su ss +story tellers +min hyuk +tier ra +ðŁIJ § +span king +silver man +read ily +dep t +ambi ance +ðŁĴĭ ðŁĴĭ +xi x +sug ars +meteoro logical +hat chet +foreig ner +vive kan +tag ore +res ent +breath es +tele coms +pancra s +du l +ya ar +ar is +r mc +my er +jo bs +with draw +back story +u mich +sebasti en +nu est +standardi zed +sli ve +si ac +sc alli +lu be +lmfa oo +mel ons +be than +å¤ § +muer tos +hon k +din os +ãĤ ³ +team india +pet co +mo ren +fe aring +bb can +me le +kne el +gunsn roses +bau haus +ygo fficial +ygofficial blink +music fest +de marco +aro d +acce ssed +obse ssive +o con +nel lie +kil da +je well +power lifting +on en +á s +bal ism +dan ke +wol fen +pro logue +nar rows +hol o +geor die +confron ting +cab ana +loubout in +s anti +image comics +foo fighters +wester nu +g fuel +disci ple +ðŁĺī ) +su h +sc illy +next gen +eg u +aflo at +xi an +pang ilinan +di en +b ca +co ons +spo d +s dg +fall en +dol la +ðŁĶ´ âļ«ï¸ı +ä ¼ +tor rance +nc isla +ta wny +jen ni +fitness motivation +bl ount +fascin ation +p da +ip f +aege an +van o +se vered +pol s +physi ological +ju ggling +gu ev +calcul ation +sli mming +fe mmes +em pan +daw g +sto v +poly technic +municipal ities +gre tzky +defin itions +correc ting +s family +rock and +on my +homeswee thome +wt cc +sc at +mo co +lar sson +kag ame +corn bread +lc bo +head shots +fire house +d news +uc as +tem pe +son ne +di ggs +bo ilers +anti bodies +sibling sday +hobb es +fly knit +li se +ze sty +substitu tion +sic em +revolution ize +mu rad +besto wed +mill ers +liveon k +interpre ter +sant abar +queen stown +event brite +d by +chur chill +sp at +pal oma +eura sian +bu at +beau x +vor ous +naz areth +daz ed +al me +rit a +con ch +col t +hamp ers +g su +ad j +professi ons +b wi +ac b +â ĭ +univers ally +trou bling +conve y +ck ley +we asley +tra der +so td +scra ppy +nelson mandela +rup tly +pe ele +every body +conse cr +short bread +sh rou +o sama +ch ach +bino culars +pl en +nam i +k la +ce tte +wine wankers +ste f +oxy gen +ha ag +yu zu +wh olly +tri gg +me cha +subjec ted +inhibit ors +repre ssion +manipu late +cur ly +black man +u red +convers ation +bag ging +at el +vote for +eli brary +vis age +ta per +st ani +prote in +pe mber +niger ian +gle ason +behin d +trick ed +haw ley +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +psychi atrist +consoli dated +bru gge +ge twell +es opha +chine senew +ach t +s fu +fe mal +turn bull +mirro red +bobb i +ben id +ado ss +vit ch +man hunt +log itech +fa king +cul t +wor st +dy na +desc ended +pu ig +fre dri +chrome book +af fe +vam os +moo c +m le +lach lan +all for +ë¯ ¸ +à® µ +ye ee +paul mccartney +as au +a sive +the great +son fire +pre k +photo journalism +meh ra +le tta +h ss +dh ury +persecu ted +ha f +demo graphics +beet le +sk ath +shah rukh +k lim +esp añ +sleep ing +opp s +mun dial +extrac ted +ðŁ¥ ģ +ta ur +jeep mafia +inser ts +igu ana +fthe week +do tes +secre tary +rin poche +favor it +corri dors +eli ers +birth s +la ban +drop out +cav an +o zz +mar adona +lec turing +fan fiction +ele anor +desper ation +character ization +bu sier +or die +holo gram +son ville +av geeks +eat bulaga +" ~ +rox anne +t asked +sp k +sam ir +respec table +ha ku +di ane +bey e +fanta sies +win news +uten sils +spy ro +red mi +mer son +i be +cro ok +co pa +wa vering +ðŁĮĬ ðŁĮĬ +zz ard +selfi sh +scroo ge +p ml +bu ms +art basel +syrac use +sarac ens +n itt +har rowing +ah c +worlda idsday +strat ton +sav ages +fur nishings +billi ards +o ia +m ola +inten ds +coy g +var ma +f sb +the queen +teessi de +re locate +no one +interoper ability +fam u +planet arium +nit ro +d lr +cor an +ìĿ´ ìĬ¤ +shoul da +man an +car melo +gh o +ðŁİĦ ðŁİģ +stee ple +her zog +dev our +chan te +arun del +rio ja +box office +bo v +tri b +sn r +re for +ne wn +blake shelton +sul li +eng ages +treas ure +o yo +bi za +. _. +ãģ ĵ +oo w +ch able +brand y +ich t +âĮ ļ +z ines +shar per +plym outh +mam mo +hydr ates +el lo +do e +centri fu +ob j +laus anne +eli st +con genital +under armour +ent ree +critici zing +vogue magazine +cast ell +aga in +a ab +ld f +co ined +well done +un planned +swee ty +q p +loy al +iz ations +ir ror +ch is +sw ann +me w +custom ised +cream ery +cevic he +wrong ful +stellen bosch +n ella +house mates +e hr +c sn +tor m +pseu do +moo dy +un folding +tel aviv +small business +montp ellier +manu ally +best sellers +gin ny +leop ard +ed in +un heard +hi ero +thero ad +gr l +apho to +americ ano +nap kins +gall ant +embo ssed +avi sta +sar ts +prosecu ted +food safety +tan aka +f v +cav alli +swe den +sc ourt +bar naby +tart are +hear st +butti gieg +af icion +abo de +mtvbr kpop +flouri shing +pol ly +or son +blue sky +sound tracks +mountaine ers +ym ount +ro jo +davi e +. ðŁĺĬ +sa de +op ed +mah ler +re gs +ram ones +lanzar ote +indu s +black rock +vo cab +the hill +ni us +go ya +ru l +tin es +mu ne +cl ic +dynam ic +aggrav ated +on or +mur ph +par ka +indigen ous +ready for +boldand beautiful +au t +somer se +so good +road torio +bb t +sau k +air strike +âĥ£ - +speaker ryan +fli er +. @_ +ven detta +fre en +chap er +san ay +p fei +nu dity +mr x +h ha +ro ku +re dar +fuch sia +war ships +d fb +chau dhry +ra wal +granth am +an gio +tab loid +stran ds +portfoli os +an ning +work load +ho to +head light +general hospital +chri se +later gram +ga v +h out +bi dder +show man +sha we +servic emen +bra vest +ach y +te de +pran ks +juli anne +ema iling +car do +testim oni +supreme court +calder on +st le +wh in +tro jan +ma honey +co u +! < +gen con +bh atia +am us +vo ting +far ah +be van +å · +lin c +ka shi +gif tsfor +fas o +du tta +institu t +code x +tongue outtuesday +olo gy +nat ty +ju gger +de cency +ch ul +aw o +mont clair +gol o +g lyn +ðŁĺĭ ðŁĺĭðŁĺĭ +qu antic +n ics +h bt +cal eb +tra vers +thisi sus +shi sha +deodor ant +cr d +ac ao +ðŁĴĽ ðŁĴļ +y il +endow ment +z ur +for ts +mar tech +fifty shades +ci v +aqu atics +accumul ated +---- - +un published +poro shenko +iz u +gn all +le mur +ilo ilo +glad stone +esqu ire +sk aya +revi ving +nigh thaw +:- )) +national puppyday +mi amid +kamal haasan +guest list +gentri fication +dale k +water way +t gt +sle dding +math chat +hu da +elan ds +cap aldi +bm g +pong al +fac tions +ðŁı Ħ +p ham +el ton +, .. +op ium +lake view +excelsi or +an ic +fin serv +ent i +true story +pa id +special olympics +me tte +ta pper +ship building +z brush +tosc ana +t ants +straight forward +k sh +ca hon +bra him +simul ations +gu mp +car til +distr acting +pa is +mu rak +gre t +ma hama +eque stri +emra an +at k +la galaxy +ho ku +can to +bo gart +inher it +colli ded +carol inas +adon is +years ago +roo ts +girl sin +title ist +itch ell +fat ality +clo ths +center piece +tis land +mi ker +u bu +sh k +in tran +cob bler +bor ns +z em +sub po +expon ential +car p +uri g +panty hose +pa wan +mac cle +brigh tens +aliz ations +the weeknd +t down +t ash +ferr ara +âľĤ ï¸ı +mee k +gro omed +bar am +pl ough +letter press +edit ori +imo gen +gregor y +g mos +bree der +reduc ation +lic hen +he er +distor ted +beat tie +yum m +spla y +paras itic +brook field +ver pool +thri ves +sto ves +ru ssel +cor r +ar min +profici ency +jack y +debat enight +wh iting +nure mberg +denti sts +baja j +ari ka +vivi and +pne fc +sr h +sick ening +cu lar +å ¼ +mil let +gar age +mc murray +infin itely +aw as +anti virus +par fum +gorilla z +qui x +it sal +hair line +bo ta +ë ¸ +yan ne +ven kat +ro ta +kel a +kath niel +èªķ ç¥Ń +sch ne +deriv atives +dakota johnson +ip v +bus a +ìĦ¸ë¸ IJ +un intended +in dra +pro pelled +ne olithic +hil o +hi ves +gwin nett +co tta +can aver +b ne +magi strate +es ri +zam an +weir dos +short cut +mocking jay +ðŁİĦ ðŁİĦ +so h +wh ip +spec tra +rober ts +rob ber +promin ently +ecz ema +bu stle +b cli +sk ol +jordani an +ev ich +æĸ ° +ro jas +mizz ou +sa shimi +install er +gu chi +pon cho +hi man +democr ati +al be +pp ies +chlori de +bly th +âı °: +yo yo +ss ard +sp at +mad dox +salam ander +boun ced +asu mmit +al mer +scru tin +am editing +transform ations +tag line +neur al +mu tton +br d +ayurve dic +re vel +humili ation +ik aze +benz ema +natur alist +mac cosmetics +fi a +ram on +pre domin +li son +goo de +ce res +materi al +herald sun +cannon ball +bob dylan +bo thering +s gb +know ingly +che ung +cha z +hand gun +chinesenew year +un treated +rabb it +place bo +ble ssed +ay am +ire ann +grosven or +b light +nu ne +co stal +c span +sa an +sol l +ra jam +k q +gary en +ben nington +on tag +muse veni +black jack +o casio +mol asses +inter cept +gh ent +fu rever +bla y +aqu i +tele cast +s ats +nat gas +ho v +neighbour ing +mag ell +escal ated +newmusic friday +per ish +bru tus +lav rov +jo dy +gira ffes +bha gav +stig mati +pais ajes +it ra +bi ases +un control +hil fi +ðŁĴģ ðŁı¼ +stom ars +© ï¸ı +ur inary +:" > +s cone +grapp ling +af ran +ace way +nau t +level ing +bre ather +aud acity +loo ting +drex el +game changer +stam pe +p mo +marchfor ourlives +ger t +cre amer +ron son +gu z +ðŁį İ +k ast +hd tv +accompan iment +trade show +sacram ento +prolifer ation +wh n +facilit ator +om bre +en y +ìķ Ħ +ve h +gi ri +bal let +ðŁĹ ½ +ðŁĨ ĺ +take the +promo ters +mu ff +iti vely +crun chy +prose cute +gu antan +! ⾨ +lex ie +kar un +joshu ad +fit spo +bagu io +........ ..... +voluntar ily +sti gers +pron unciation +loo ted +ju ke +er adio +dum pling +barit one +neu er +mac cab +in ations +at te +๠ij +nbad raft +df w +chil i +bc ps +amrit sar +ta vi +ro tor +ra bi +gh os +de smo +ca g +fan meeting +ram ona +au tam +waver ley +tu sa +t se +say noto +pra ise +es mith +time piece +o jo +k ü +cu ffed +z um +juli et +vege ta +pen tax +is inha +ni ño +mall ard +gran ting +ðĿ Ł +tag gart +south land +pas se +maryam n +grum man +boot strap +amo tor +soci edad +nath alie +x es +tr out +mo ji +mar go +g ld +ma hal +bal enci +ten n +pedi gree +na omi +las vegas +ke ssel +gun fire +ak kineni +ten e +grand master +un ru +k sen +rebe kah +mon e +kol lywood +reci eved +fire fighting +cha o +de led +memor i +fr nds +b gm +shaf ts +saxophon ist +ry n +st fc +mis chiev +annu al +y vette +var i +tah rir +perth news +o bey +naji b +isab el +z ler +van de +somm elier +sole mn +interven e +con cise +. âĻ¥ +ref rain +kri sta +conver ge +trife cta +trac er +predat ory +pu li +i ow +brass erie +sco ts +pti official +bon ni +bl t +sung min +regi strar +re define +pa stime +gy u +canad as +blue bell +ye revan +south australia +sli ves +en id +vi ole +e el +death metal +avent ura +:) :) +ple dging +i ucn +daf i +bet ween +to workday +sa ur +be le +super store +hair cuts +fil ter +an ore +sp resso +shiel d +digni fied +b fa +so jour +br in +sham eless +harri ers +er ab +au ld +tight ening +prevent ative +si star +narco tics +yst wy +s nyc +ir u +for real +b ends +fala fel +si vak +free port +ce ch +say ings +don ut +dial ec +x ml +wom ani +ott city +ke re +book lovers +london is +augu ste +wonder ful +win elovers +๠Ĭ +pe da +miner va +ar de +" !! +or biting +nationalbestfriend day +flur ries +ang kor +z d +strick en +photo volta +nan dos +hou dini +fu ente +chad wick +ce rer +wh ack +terti ary +ny pl +low carb +hai ley +d ness +bla c +thar aman +re treats +pic chu +mari am +l ale +decre ases +b he +ðŁijĮ # +ou sa +o ye +nhl draft +ly on +car u +expect ancy +back log +audio books +sur ges +provin ci +pa ol +gr ate +ðŁĺİ ðŁĺİ +r nd +parais o +kee pp +hu l +ap ed +ðŁij ĵ +po tters +eeee eeee +we work +tom i +quil ting +in dra +haw t +anarchi st +pit falls +co stab +zom bie +re flexi +mar low +hen rie +gra ff +dribb ble +am joy +v ases +unex plo +tri mmer +bl ic +the or +le sley +kh urst +fix er +fili ppo +cli que +av tweeps +sc alia +festival of +mc govern +ku hn +hol z +cor ning +ym pics +villi ers +solu ble +hook up +black ed +elimin ates +t va +f endi +dent e +alger ian +re uniting +sel le +pe au +news feed +southwe stair +pendu lum +air man +ut en +in humane +gol an +av a +inci pal +d fid +bla ze +cd nag +mor bi +gal lup +wyn dham +open stack +h isd +on shore +analo gy +v ado +jo t +l la +go of +dum fries +macmillan cancer +em ur +wra pper +par mi +log ical +indi ana +lo bby +kit ts +ki z +ðŁİŁ : +vid con +phy sed +jacqu i +follow friday +cancer day +er g +____ __ +ðŁĽ Ĵ +un lock +suf fo +must go +ay lor +so va +shi gh +scienti fically +sar k +pitts burgh +barb our +arri ve +aren ergy +hon da +ãĤ· ãĥ +mother board +li ps +g ac +fore ster +ffe cts +e si +de stin +r ini +mu les +daun ting +it zer +de sal +to ad +main z +lin burg +group on +< -- +bu en +gipp sland +inv ader +hatt ers +fel la +eat in +del orean +bau m +ma un +je ez +indefin ite +ro gu +bru t +z ay +hamilton musical +emb le +sla x +he es +full moon +ant an +li one +ðŁijĬ ðŁı½ +mac kie +tk ts +loksab ha +aw i +smur f +man che +british gp +sha hi +lon sdale +hom bre +wav eleng +scoun ty +in ja +in de +darje eling +ðŁ¤ ¡ +nietz sche +nb n +win frey +pre ached +cap er +t pa +replic ate +di ii +Ì ¶ +su u +speci alizing +rep ent +jp morgan +hul me +clow n +min ster +bo ise +ðŁĻĦ ðŁĻĦ +m soc +the fancy +m re +president sday +pau ly +new delhi +jan elle +heritage month +car pool +car pe +nab i +mau rizio +es ki +bern hard +th tr +oun ced +kirk wood +in comes +ido li +coo ley +art deco +ðŁ¤ ij +waltham stow +mut ants +mal ema +son aksh +pan theon +lucer ne +intro vert +out take +dean ambrose +child birth +megap ix +gh outa +ap m +pan o +illi es +ba ez +red nose +le ston +x ero +sfor life +mid land +ir re +er th +bad al +ren ault +re spite +am ani +come on +fuku oka +b q +chai kovsky +ðŁ¤ ¨ +tab lec +an sel +war frame +sul try +sobri ety +bridge stone +arm and +ðŁĩ©ðŁĩ ° +ste u +s ny +gun ned +ji b +fo u +exac er +aper ture +ye on +k sat +gir lies +â ij +fo h +feroci ous +pep si +hay den +bry ce +ðŁĺ £ +shahe en +n mapp +mu shar +clo vis +bri bes +po sh +music festival +injec ted +ìĦ ± +li pa +sla via +ar l +an et +ðŁĮŁðŁĮŁ ðŁĮŁ +z we +meer kat +expe dition +oni k +df wwx +bat ches +kisses delavin +inter faces +ino v +cast or +âĶģ âĶģ +south park +new sday +go bble +anton i +al us +wick ham +st ly +guantan amo +fan cies +str on +moo g +ira q +i yer +cl p +vis cer +vo ten +k lon +atmo s +zach ary +michelle obama +ph ine +inven tive +bal four +s ita +remo deled +he ed +breath able +ju ju +weak ening +ol den +hel lu +g ast +extre mes +school er +perfe cted +hil al +dl su +cau ca +go to +bal fe +ab id +selec tor +yo t +surve yed +llandud no +sc ann +bour ke +us d +tow nof +j at +drin kers +ðŁĴ¯ðŁĴ¯ ðŁĴ¯ +rr rrr +maccle sfield +cor als +bra king +tr icia +collec tive +bet sy +w un +sonaksh isinha +coin base +chelten ham +usemb assy +myri ad +mc pherson +con ni +c mos +sj sharks +perth shire +kno wn +bump y +è me +supp ress +thel ma +fey eno +ðŁĴĢ ðŁĴĢ +ss erie +makk ah +bru ssel +ðŁĮ ® +mil os +sv b +emb ank +ta yy +u le +top gear +j ira +ch affe +bra dio +an ac +lu la +za a +evalu ated +ar ic +Ħ Ī +ा _ +ru ck +buy local +sag awards +k sleg +def aul +chant al +butter fly +ha vens +car ats +can lab +br k +dou x +bee hive +new bury +jodh pur +free hold +ferr ari +y ells +uncondition ally +play through +nanow rimo +dic tate +ar mor +swi fts +sc e +huss le +say ed +ro cha +at en +abil ene +ar mi +d tv +action able +tri pp +sn k +od inga +w kyc +time out +roo ks +myal gia +insul ted +an am +ts ar +o leg +me tt +j ble +escal ation +qui eter +house wife +experim entation +u ary +to ssing +re mixed +la ird +it arianism +extrater re +z are +k tor +pay load +ber ge +restra int +bethe change +be w +çĶŁ èªķç¥Ń +f ells +r ta +persu ade +line art +b do +adop tive +ðŁĩ¬ðŁĩ · +ìľ ¤ +ke ssler += = +gran ds +v aish +sa fi +emil ie +car twright +and ale +ye st +w or +po ts +pam el +boomer ang +lju bl +ham ish +el g +christ y +ðŁĶ Ł +spectro scopy +po fficial +m yeon +Ê » +sto ols +nab bed +insh allah +gi da +c sl +li dar +exper tly +deterior ating +bru ges +sati va +testi fies +py th +hero ines +chi me +facep alm +street fighter +ph oo +may onnaise +canni bal +ðŁļ Ĥ +wat ered +ðŁĺ § +cor rea +lunch box +hybri ds +s fs +is an +cul tu +zoo logy +ric ci +pi pers +be spoke +asc end +ðŁĺĬ # +stopp age +ana esthe +prostitu tion +w mc +regu lars +oce ano +comm a +shenando ah +regin ald +nas a +cohe sion +bli mp +z as +tag li +sm al +ra ga +min or +gabri ella +moja ve +m crae +earth ly +sail boat +gad kari +worth ington +lin cs +itch ard +cit ra +sor cer +racha el +pag i +ne ta +good news +ed ly +wee t +ab storm +realtime chem +over heard +g ish +barang ay +ritz carlton +miche le +hir st +gosp ur +bu sts +par rots +ke ira +hal la +bot ched +ai o +æ ¸ +su pri +ot b +hass an +sl ick +sb p +ni o +shru ti +ba it +: * +ng l +hall o +di age +qu arri +qq q +lud low +hel mut +ge al +establi shments +ax a +melan in +di ri +da red +aless andra +met cal +car val +bru ises +li u +lat ch +lap is +jurassic world +chalk board +bo sworth +batman v +awareness day +ðŁĸ ¥ +sm tl +second stomars +hen ne +pra s +fidd ler +ec ast +ve sp +kh ill +fai ths +acqu a +sold out +francis can +dat enight +h st +te acup +muham mad +manu als +ar cs +iel ts +hr t +m ro +ii fa +flu ke +ar lene +yeo vil +nut meg +lo dging +scre e +oli vier +jac que +international catday +innov ate +doglo vers +comprehen sion +bea stie +stu bbs +sol is +inter pol +hal ted +bly the +andre y +âģ£ âģ£ +schan nel +chance therapper +pott iteam +norther nireland +chee tos +belong ings +mer ida +jan o +oce ania +fear less +le ung +la pping +iver son +huff ingtonpost +hu ts +de akin +d ili +prick ly +kids deserveit +id p +est es +co sa +wi c +ne wal +confron ts +bar bi +appreci ation +Ð ± +rati os +r pi +monste renergy +apple watch +yu l +sas uke +pe gs +bow tie +ute p +salu ting +po res +home boy +char cu +ca it +ಠ¾ +mon tr +li ams +gy ms +ad in +ha slam +easy jet +col le +beyon dthe +stu co +my n +gospur sgo +ge ophy +sk a +rock land +ence phal +dispo se +ภ± +tol ls +pew ter +nom ore +div yan +californi an +undeni able +tra ver +par ri +infl ated +eu v +downton abbey +com au +n su +minis eries +tor t +prepar atory +maryamn sharif +ga els +ðŁĺ ł +pic kers +nan jing +ex u +bun ches +ðŁı ĭ +raf ale +ko sci +d of +pale ttes +on is +new sl +micro services +bar code +à¥Ģ _ +rat che +jun ta +j and +drainthe swamp +anno y +sc ards +pc gaming +aveng er +pax east +hur ray +condomin ium +sheri ff +li ra +hard back +far ts +demo lish +assaul ts +w dy +vo ort +tion ism +philanthro pic +j ci +inim itable +ft b +swar aj +ri so +qu ah +pi ps +pe so +cor olla +rolling stone +peach tree +carl ton +be b +austr al +tacob ell +ro ver +murak ami +del mar +sun dar +jeho vah +hilfi ger +emraan hashmi +emabiggestfans justinbieber +dis qualified +vi val +fren chie +brian may +bingham ton +ttr pg +refur b +il let +da ver +bath ed +bar rel +s ra +i vo +am ak +wearable tech +shahrukh khan +ne ander +le il +gren ada +ðŁĺį âĿ¤ï¸ı +swif tly +sho wr +re posted +ad il +î ģ +fir sta +easport s +aaa ay +& @ +wolf sburg +s sports +li dl +ab an +sports biz +s na +pr ank +po i +em bodies +sky papers +re ek +mc neil +el ow +dolom ites +lec ar +lau ri +grass land +at ica +hypocr ites +so ya +ro scoe +pow dered +nom nomnom +mixed media +ic p +grand kids +tray von +seaf ront +mach ina +bueno saires +multi ply +wr x +ro chester +on et +kar u +k awar +an ed +aber crombie +shak y +emp irical +bal or +anti microbial +pula ski +n ance +mi a +heart breaker +gal lop +rock away +er is +joy train +ĤâĸĤâĸ ĤâĸĤâĸ +cl un +gi z +sal ve +pan eer +out now +boni fac +wr y +sel fle +rattle snake +pi al +gh g +gastri c +walk through +nc l +ju arez +ja un +seam lessly +bur j +shim mering +outw ard +m chale +ðŁĺĤ ðŁ¤£ +stead fast +hu y +tra pping +to a +thre es +j ello +innov ating +friend lies +cor re +tic le +thi est +ot tery +mis information +hill crest +gam bino +what son +bel las +the cable +penn ington +op sis +mon ash +water fowl +storm water +ne tting +body builder +aber ystwy +ka therine +hartle pool +execu tions +vi m +sha ve +lich field +insi ght +jim mie +emb raer +cody simpson +giftide a +fu s +ci g +mand i +schwar z +ro tt +dad i +bent ley +ang ed +zar ago +worl dr +train or +pushaward skath +he iser +withdraw als +pri mera +mi gnon +diar rhea +vm world +o dom +ky ra +u mass +sp ud +ou li +c gi +ro de +quizz es +moon rise +il ty +hedge hogs +gil bert +ar ising +pi ers +p ada +fellow ships +cardam om +anni ka +un humanrights +sunday thoughts +kid neys +gand hi +mar guer +ari sts +un ny +ti ka +law d +kind red +fra pp +no sed +real madri +our i +i fi +car am +char m +sp ared +la do +ke pler +bree der +earn hardt +winder mere +viviand sena +progressi ve +mus th +jag ann +amp bell +affili ates +rick shaw +ha in +compri sed +happ ine +cambo dian +ro tting +bru nel +ê¸ ° +shreve port +ri gg +phan toms +far rah +a die +todayin history +of er +e ssi +tre ss +st am +sn d +la tham +for giving +bi ff +winter iscoming +wa hi +w ut +tor rey +silver ware +jai me +flu tter +co ders +be p +k hel +br ates +one ment +b bling +âĻª âĻª +right to +net de +e ster +ver ano +stay cation +motor home +ag ood +ì¶ķíķĺ íķ´ +ภĽ +xen ob +ven ice +sw ap +ol lins +mon i +li ka +imran khan +der m +saf aris +mon tic +better than +pa edo +may flower +hypno tic +communi sts +clari on +band o +âĶ Ģ +i j +schro eder +pre achers +ity day +b ini +oo lie +m wa +k ula +alber ta +phys icists +mi asan +do gg +whit tier +down under +dono ghue +se vere +margin alized +gb v +che eri +wat an +o an +too t +stric tly +in verse +chau n +b hic +x plo +un ner +tun ia +ro be +persu asion +dog ma +swal low +infe sted +dor an +asu ka +tortil las +mi ya +ago sto +eri als +ag ric +âĢĵ âĢ¦ +twer king +sales force +d mk +cre pes +u me +stac y +smar ts +repet itive +dul quer +ke sel +aur ang +way ans +s wind +world s +cho y +rede emer +uc sf +starwar sday +lager feld +expre ssionism +cir rus +t sum +vote blue +kaleido scope +hydrange a +chick peas +íĤ ¤ +å ī +zion ism +ty son +new som +fal k +toa sting +schrei ber +recu per +fiel ders +Î ± +{ } +thor acic +ic d +ar se +adverti sements +sink hole +liber ate +bis marck +ag ed +sylla bus +del a +cl ary +shadow y +mom my +lim ite +gr s +âŃIJï¸ı âŃIJï¸ı +uk ah +good all +f ong +envel opes +de paul +ufc fight +pe to +cur b +critic ised +wa key +steven age +hen ny +err ari +der p +canaver al +mu sta +incl ine +e bb +b ks +ter ic +re defined +ðŁĵ § +tw alk +sor cerer +me c +cali bre +ðŁĺ ĵ +se co +k andi +jun i +egyp tians +depar tures +ãĥ¼ ãĤ +ti mess +ilo ven +come true +art museum +apo the +ap l +on ation +kangar oos +x avi +sh om +pu i +na am +bom ber +tc g +ply wood +no re +h ame +electri fication +blu stery +un rival +un authorized +plu ral +lu t +gilli es +ecu ad +co quit +av rilla +yol k +pou los +her nia +est one +pe aking +id w +hispan ic +ah l +wor shipping +pe arly +ðŁĺĢ ðŁĺĢðŁĺĢ +ap s +turnbull malcolm +se av +mc l +ma koto +leu ven +b sr +zu c +te u +sc le +ph onic +shat ner +san z +caul dron +war ra +po k +pierc ings +i aaf +grande ur +fran ck +dis section +affir ming +py le +ham el +equ alizer +bernade tte +syour bestfriend +stumb ling +prith vi +polym ers +mcal len +jun ky +hu gger +da vide +ver itas +mp f +mc neill +ley land +jo zi +candy monday +bas u +mon g +list o +hair spray +sun dance +film photography +far row +u sta +oci ety +me mento +fe o +ab ruptly +c ska +ti vi +on en +calcul ating +shi rec +sequ in +mal ang +gen sen +é n +up take +r tx +free the +wi per +sym bi +co qu +ภ´ +z oned +te ak +id ps +alon zo +qu ish +oc chio +artiston twitter +summ ery +su is +sil as +j alli +forci bly +del ano +cu d +blogg ing +air craft +thri ft +pal myra +ber liner +under gone +fu j +v ray +on b +mand olin +kra ut +butter cup +br ats +termin ation +penn state +moff at +mo dem +ashi on +ðŁijį # +thin king +re publi +ou ta +beh ance +ra ha +loc ket +parag li +l hr +crow der +ma gu +light ning +jo c +fire bird +any l +an vil +sus anna +r ö +feder al +cas ket +bo or +ðŁIJ ¢ +tion ists +pushawardskath niels +jog ja +ha kim +ðŁĩ®ðŁĩ ± +kendall jenner +fri es +boo l +boath ouse +ðŁIJ ĺ +ðŁį Ĺ +s lin +kash yap +i que +care free +ðŁĴ¨ ðŁĴ¨ +hu ber +do b +bre con +an acon +pho bic +deta inees +ðŁĩ³ðŁĩ ´ +un ter +sea horse +man c +la ila +cy tes +author ised +wi pe +stor ia +mi yaz +k ling +isol ate +he brews +gu cci +australian open +tex plain +dis continued +crow ding +mer its +all ar +ðŁĸ ķ +tiem po +spo ti +balenci aga +l cm +kay la +der mot +cor dill +ou ille +oh m +le thal +free zes +ut b +she pp +rou te +dar y +bha van +breath less +ash anti +aci fic +ne followers +kristi an +on c +mary lebone +ber ks +x posure +unmiss able +tul ly +tor bay +raven s +kar thi +ad vers +ðŁĴª ðŁı¾ +us z +sc is +mil ion +at ura +peach y +cra m +ev ils +pel ham +paradi so +meteor ite +kra vitz +yo te +confis cated +bru ck +pla sh +mur ano +maro ons +ðŁĴ ¡ +yn x +pick le +lovin it +k ra +r ns +ann apur +u ct +le ander +lan adelrey +gab on +numer ical +mens style +ma pp +ju g +gli ding +steve aoki +fric kin +food fest +ch int +y pres +sidd harth +butter field +i ff +ad jour +w gr +tam my +me kong +it forward +g td +cryst alline +sur faced +super cross +dilig ence +v z +sd al +s fm +in version +sni ffing +pun to +le vis +ka jol +ini esta +the future +squ all +end alz +di me +con tention +ce sc +shin y +band age +nom adic +dif fusion +aver y +stir red +rig by +os mo +hard ships +wh or +s sp +dis ks +ðŁį © +ìĦ¸ë¸IJ íĭ´ +wil ding +yy j +ovie do +n pp +excep tions +sever ity +made by +harve ster +del inqu +pedi atrics +human trafficking +appre hen +w lax +thermost at +wi gnall +d pr +woo oo +tra u +gor such +east ward +conclu ding +team jesus +fla k +cc r +sa sh +man te +hi k +vag ab +pur sued +legis lator +di ri +ray mond +nu dge +mun dane +s ars +mccon augh +ck in +âľĮ ðŁı½ +pho p +me yer +haci enda +feasi bility +sapp hire +mu gh +ly di +lo west +ers ville +god speed +gabri elle +d agen +beer fest +bang alore +ra ff +n pl +lu kas +ach an +sno ws +ml c +hu mming +ten ter +resi dual +mou ssa +le andro +ke strel +d reads +resu med +hr m +com ix +agreat again +un loading +lovel ife +jack ass +cu yaho +wh ining +power shell +n gs +front page +barbar ic +uni q +ol phins +intensi fies +ea c +dy sp +seabir ds +tann ed +sti el +ho ws +aaj ith +mc avoy +á ¹ +windo ws +wh u +muham med +ide ological +mi de +j ingle +bbcra dio +ultra violet +next door +lei den +con un +an thro +air way +wa irport +tr p +race day +l ml +g ough +in stig +den berg +es ther +meat y +da vie +co founder +tour mal +shir ley +ob noxious +loo sing +ðŁįĢ ðŁįĢ +⾨ # +spiritu ally +sc rob +go for +coffee day +ðŁıĪ ðŁıĪ +i em +extra dition +sett ers +demb ele +tur nip +mathi as +liken ess +roo st +i en +ero ck +dro ppin +mu ay +feyeno ord +bon ang +sv g +ous ell +mar vin +cas ing +mor ata +edi bles +co a +av n +ta ken +ice man +t cc +sand berg +id gaf +consider ate +ps f +ay y +scho en +hake em +excer pts +no elle +inevit ably +blumen thal +wh yi +under taken +sp ub +oni um +daw kins +pro tip +âĺ Ħ +troye sivan +t ye +stati stic +sm l +ðŁĮ § +ger anium +ver watch +yo ak +world wide +volta ire +ns x +na iling +mo ira +band ar +lay ering +kin dest +ef fi +cham plain +apo stolic +ta vares +lero ck +appeti zers +ac centu +;- ; +w awa +or ning +on der +easports fifa +ar p +ðŁĺĬ ðŁĴķ +up setting +str inger +sho ggi +lu pin +l ny +su bor +pr itz +mor o +hil i +tro ye +scor p +her story +ent ral +ch ine +mar ques +hop kin +mo g +h cafc +g j +y aaaa +ru moured +iti ans +cotton wood +basti on +nine teen +mish acollins +men i +handicapp ed +alt coin +min der +at socialmedia +allen town +ak on +ðŁĺĿ ðŁĺĿ +gw u +ay ah +cannab ino +anth on +air stream +i wc +cbc nl +ðŁĴĥ ðŁı¼ +w soccer +se ong +aad haar +l anger +ì ¦Ī +the bachelorette +t chaikovsky +pep tide +p sl +agri business +oun i +scat ter +nul li +inhibit or +vie ira +ra iling +law ley +ðŁİī ðŁİĤ +ì ² +su tter +mi u +husk er +har rys +con cac +c ates +as ak +ãĥ Ł +serpent ine +santa fe +pat taya +modi fy +jay hawks +h ors +brain storm +be sik +wat h +qu on +creep y +u ic +sc aring +peel schools +ðŁį ª +sh yam +rou se +gov ts +go pal +par th +maxim ise +ken il +hhhh hhh +health iest +so or +r acker +bb on +vintag ec +the w +marl boro +d any +aven ues +ag it +ro sh +sc ania +pr itchard +p mb +glass ware +your bestfriend +whist ling +la e +indigen ou +brad ford +co q +bloom sbury +spirit of +op eng +flick er +cre ed +confi dently +aw fully +um n +hermo so +tom y +sn ape +kar ma +wa isi +nw dogrescue +mon mouth +de fun +bu ren +west gate +s show +goog ling +gibb on +deci der +q vc +pat ra +m chen +bra ille +wh opp +de bac +one al +willy levy +white side +the red +im patient +saat chi +depic t +war na +pick ens +gh um +fi bon +opla sty +director ate +wh ittle +kim my +gru dge +al tu +simil arity +eng ro +cham onix +alic ante +secre cy +re master +pyg my +low ski +gujar ati +figur ines +at uri +agar cia +ultra sonic +out breaks +inno cents +under goes +acou stic +nhl blackhawks +dan ville +ðŁ¥ Ģ +holo graphic +f able +cum ple +ev ens +acqu aint +she ba +the drum +equili brium +sincer ity +premi ums +jelly belly +buildthe wall +and rade +staur ant +savethe date +re election +prescri bing +kno tt +some ones +cook ware +sal ford +popsic le +dr ury +c age +ag gi +portra ying +pande mic +pan tom +v d +ero es +backtothe future +ë ħ +trans gre +suici depre +stay safe +o bas +ju ma +heigh tened +endome tri +a jo +v yn +nd t +lif es +tu ll +dic tion +chilli es +calla ghan +take out +su bbed +stephen curry +sol i +promp tly +aw ang +a theatre +ni th +d ney +aji th +abas ketball +sk it +mol ded +duc tion +an ker +ìķĦ ìĿ´ì +world sbk +syn onymous +rr r +ro dent +ash win +Ñĭ Ð +ge ton +real talk +mul ch +j ani +dray mond +ast in +harmon ic +h ms +dwar fs +ambi ence +ab laze +th grade +ra kh +mc david +bar bic +pre t +mun ster +kis sim +indic a +cy r +ac nl +ðŁĩªðŁĩ ¸ +turno vers +rae es +plu to +m hr +lec tric +kon en +ca stan +mitz vah +bo wie +une asy +pode sta +phy lo +im moral +hour sof +decath lon +c br +kham enei +ja in +ex tric +cu shing +z hao +y id +plo ver +nor ge +yak ima +women shealth +to ff +gil mour +ch ay +าภ£ +visit wales +art fair +al en +willam ette +lu zon +elli e +blin ders +the john +co lette +o zzie +drun ken +bur kina +adirond ack +rescu ers +pay out +mar ge +ju ly +car parts +su shi +goo dison +ag wa +cor doba +box set +ad un +.. ) +le sotho +layo ver +ke an +al b +ठľ +son net +mus ke +mach i +i sto +bran de +syn ony +li oness +ak ia +texaste ch +stun g +hang ers +commerci ally +du mas +uni magin +spar king +ri f +z ic +tabern acle +g aff +creati vely +coura ge +arma geddon +ðŁIJ · +s st +gerald ine +ss chool +son am +ne ha +man c +j query +eleph ant +ejec ted +cam i +yy z +cle m +x games +wi ft +sw we +ra bi +back in +man j +hol t +ho ist +fire stone +ðŁĵ ¦ +ur anus +la ing +ðŁĩ » +nfl network +insp ace +god win +clari fication +tre spas +multi plic +hack er +ðŁį ¹ +pol enta +heat ers +mk tg +mercedesam gf +ãĥ Ĺ +wwer oman +gu ing +gfuel energy +ภ· +u ste +di ony +cu sack +cor ned +( - +thex files +v news +sind hu +le high +fun times +fo g +exp ats +a beach +dun fer +deduc tion +beauti full +ol us +modi fications +mul la +not ation +wweroman reigns +thal aajith +kar im +harmon ica +salv ador +oc co +plan tain +faith fulness +prefer ably +car th +!! ? +womenin film +so br +enterpri se +che at +ab del +sar coma +mca fee +chu a +museu mof +black stone +ar af +un dies +smugg lers +yo se +ten dulkar +preci p +fc v +trac ey +in voice +am bo +theo logi +li ye +chronic pain +bash ar +war burton +the more +sol dering +ho sse +gine bra +g ly +flash y +é ĥ +schu ster +livepd nation +ind ler +bon jovi +black ened +silhou ettes +gar go +ni les +mu zik +gau rav +chant illy +recl ining +mc cur +lou doun +har old +ad ha +f ata +ali l +tb f +v am +twenti eth +thisi sy +the bachelor +lan ark +sni der +bar an +fi sts +cra i +al go +pl ice +or ang +gen ds +cor nish +ste dt +di shing +ci on +rel li +in bound +cent en +va z +sc ia +une th +mock up +lac s +dr an +design museum +cos mon +c dr +bull seye +s ds +pamph let +fi zzy +silicon valley +barthol ome +' .. +tra e +pett is +osh kosh +o ast +mal ice +body suit +all uring +pu tra +no ki +ar news +wil lows +urban a +radi sson +podesta emails +ne apol +j timberlake +ti q +om ents +cc c +what wedo +mat is +ign acio +ho ss +hill song +gra be +fran kel +e us +cre epi +benedict cumberbatch +âľĮ ðŁı» +ra bies +mc m +batmanv superman +sym path +s ry +roland garros +ku ch +gross man +du als +coco on +bri scoe +rebelli ous +pride of +mi mosa +k ola +hear th +gil more +caled onia +c md +py jamas +am end +ðŁĻ ħ +hau te +ev r +ðŁį ij +ðŁĩ« ðŁĩ +vo res +marj orie +in explic +dat piff +spr in +rub ens +lam ent +apo d +re stores +ra hm +madein italy +cas ed +ca pre +bang les +ag ile +refresh ment +parkin sons +gri eve +expon entially +gr yl +drin kin +ठ¸ +sch la +snap shots +mis on +sf v +nov i +cun y +the snp +kin ks +josi ah +é r +megam an +m dm +blu eli +x ena +par ab +maker s +cle f +ðŁĺ ¸ +t cr +pa io +cron in +the boss +scar y +ran os +ko e +daim ler +wy man +te es +s beer +ise ach +in is +and an +ðŁĴª ðŁĴª +ë¹ Ħ +stal wart +ni shing +jun k +gu s +perfec ting +new x +ir us +co preps +supp er +suc cumb +mosch ino +hi ggs +ãĥ ĸ +shan ahan +mark t +lor a +hou thi +ex c +or dan +ko d +gro in +just doit +bell ar +rho a +psori asis +ma arten +insu fficient +impac twrestling +g aff +du stry +summer of +news week +mur a +is la +do yle +ma ic +luke bryan +fibro myalgia +ر ÙĪ +m la +kar am +ju d +evoc ative +ठļ +tro tters +tri pped +ple aded +fall in +et fs +venom ous +mcconaugh ey +flam boy +chang i +good morning +fri gid +th aman +re claim +bo leyn +ãĤ ¦ +recon c +ke sh +el sie +bed fordshire +be ss +sub continent +kat erina +bo z +thessaloni ki +termin ated +rag ons +intro s +dr r +cre ss +brief case +blin ks +ran bir +perfu mes +exc ited +ever ton +cou k +c pp +yr kkh +sk u +ri va +kit sch +di pa +do do +bo ho +ticket master +ling en +lau er +dat sun +ðŁĶĹ : +m ro +gon dola +ci elo +chapp ell +fit r +ski ps +nc ga +mur dock +multi disciplinary +ki wi +cer os +cac ti +vene er +on u +k ars +evangeli on +Ñ ı +titan fall +secu rely +eyel ash +îIJ Ĵ +s watches +heal ing +ton ya +n q +mi stry +high e +cab rio +m ö +kinder gar +in nate +vi pers +nucle us +mac key +al pine +ox y +mor tem +fol ders +a fest +á ŀ +repur posed +green belt +de port +west port +pu sb +news brisbane +arquitec tura +set life +mag ick +macar ons +dark horse +vau x +mu zaf +ðŁij ° +ì§ Ħ +pro wl +gon i +edmun ds +vie jo +lau rier +enqui rer +embank ment +ðŁĮ ĥ +ro mel +ma ury +line a +k lee +bis ons +b able +we athers +o deon +de volution +cordi ally +bu ch +sti an +o varies +lov ell +cru iser +c th +v ay +un nie +tro w +t ler +ben az +- £ +nas asocial +meto ffice +gu en +clu msy +? ¿ +or ps +jac ket +in nes +regi men +mah mood +kam ala +fi end +da al +co as +å ½ +twitter less +tao iseach +buk hari +panther pride +delight fully +book case +pan tera +ms ley +mesqu ite +here by +he morrha +gun control +du ma +colla red +av l +ador n +vaul ts +teme cula +sky diving +play maker +mur ug +lat vian +here fordshire +god mother +till man +shoo ting +mar it +mal function +fr inge +tu bed +nab show +ed dy +do ge +diag onal +as mith +好 ãģį +sti est +spectac ul +pinst ri +pi pp +d sw +ðŁĮ Ŀ +nam in +mb ur +propri etary +gener ale +dic ed +ba hia +ðŁĺĬ âĿ¤ +urban ism +pe ps +dri scoll +u tt +cay ne +tul ku +national siblingsday +ya an +v adi +together ness +o en +juli en +cam pion +ðŁį ī +ye ahh +wo e +t alia +lepre chaun +p ice +fin i +de ver +carri ages +we aves +scho les +ra deon +lil o +j cc +icec ream +hagg ard +el ks +cir cled +yl le +tu cci +ic loud +dr an +analy zed +ðŁĴĽðŁĴĽ ðŁĴĽ +âĢĭ , +win x +sonam akapoor +s fl +ni ka +lock out +injec tions +erad ication +bio chemistry +rot ate +rang ersfc +playo verwatch +kr iti +hand lers +win ks +mis ss +k su +best fiends +ðŁijī ðŁı¾ +âĶĢ âĶĢ +super iority +kri sti +flan ked +alt coins +mimo sas +hallo ws +yo i +tro ller +re pay +ny g +ie a +fol lic +ðŁij ¾ +tele caster +pro claim +fear ful +whi z +mosa ics +improvis ation +bic entennial +we sley +pad ukone +every ones +ain en +lat i +lac ma +gram mer +fore arm +de ir +colum bian +tyne side +sh ingles +rou sing +rand le +cru mbling +tu pelo +glo s +cor mor +bosni an +rac ine +k ington +ha ines +children sday +at un +analy zer +at ch +meat loaf +amaz es +isa acs +corner back +os wego +multi ple +electro cu +admi rable +sho als +red mayne +lo sa +mcdon ough +ker ber +te ddington +rh one +plu mp +ne stor +kw h +hat ching +girl z +bel uga +.... ? +ðŁijĭ ðŁı» +y ms +ble achers +ang es +tor tu +refugees welcome +pu th +vul can +nu i +mad hy +doubt ful +dami en +yu u +si ppin +ky la +ospre ys +mache te +lad bro +sh era +scoo ped +jy p +z co +u bi +smugg led +dre d +cl ondon +der berg +e hl +du mont +de de +è ne +s bb +pru dential +life saver +key notes +bal t +un settling +pu ente +out fit +leg acies +exam inations +red hawks +manipul ated +gaz ebo +tou hou +medical marijuana +ing week +gi bb +zero hunger +rac king +tu ba +sun a +seaw ol +w pc +oz an +cav ite +broo d +wool wich +vol de +un fur +shadowhun ter +jo bless +har var +food blogger +ca wards +ta hs +st b +no wh +jo es +h j +cahon tas +oper ahouse +mi ght +flag ler +b ch +sp ire +bun gee +b x +ri fle +cu rie +ba ines +ru pauls +) ." +vi vac +health it +wel lesley +throw down +sa ver +ri vier +x ray +nap erville +induc ing +charcu terie +berser k +ba at +spart acus +noo b +dead ass +bel e +vi ri +niam h +mountain ous +si xx +qu a +te sters +prince ton +in q +ber gam +democr acy +bre am +aun ts +re is +pet r +par ramatta +nic ht +arte facts +un just +jet pack +in venting +filip inos +farn ham +do il +chu cks +ac ross +sp ass +r anc +hundre d +euro sport +slam ming +house mate +gam bit +d ÃŃa +azzur ri +stam ping +nar ra +campe on +suici des +colon ization +be zel +xin jiang +stand still +hiero gly +gou da +cam bs +thr ill +star vation +night shift +adi l +spher ical +loc alization +clean tech +zarago za +wor ka +spec k +sou the +lip sticks +cb t +ak im +ag es +st ica +un k +pion ship +shell fish +kyr gios +far is +sty lish +squ aw +kel p +id w +cap stone +w gi +trol led +pe ppa +gam mon +anti och +; ( +z ations +un realistic +ss cot +slu g +ke ats +en th +ad iti +uni onist +ol or +ke ita +exagger ated +briti shar +Ø £ +suzan ne +so z +n gr +campu s +bri o +pet iti +eh ler +ci k +ab io +ubiquit ous +Å į +wi gan +plac id +bank holiday +my sql +mc nally +fire wall +bay lor +bar stool +az ur +âĿ¤ ðŁĺĺ +mid as +ãĥ ¡ +sun downs +sto gether +sequ ins +m va +c ph +ðŁĩ¦ðŁĩ ¹ +trail er +so wing +am ary +mol lu +mackin tosh +al di +wil fred +vaccine swork +lo ls +dial ect +cas inos +militi as +go thic +fort worth +calibr ation +br ine +ble ached +ke k +n ss +har u +acry lics +mar ou +ger sh +bor ac +sam ar +rome o +mr p +light ing +ab p +spra sad +main line +flav our +bo sp +alber to +the show +santa anit +plu s +n fu +morning joe +m chu +gi mb +water loo +ut z +motor ized +mar icop +inst adaily +rever sing +mm ons +cen ta +salv ation +jaco by +inquis ition +he id +bantam weight +sun d +stri p +sime on +fu tile +carpen try +al ondon +ðŁĵ ¡ +p ma +the hobbit +mo ab +keep sake +for mmva +water mark +free iran +folke stone +drif twood +sen sor +maybell ine +for s +fer ous +ane mia +glen coe +atl ant +at lee +incre ibles +cor t +refuge e +elli ot +Î ± +tim or +tann er +take down +m nu +ha bad +proud to +nu tt +hann on +castle vania +timm er +restric tive +l tv +delu sion +ay la +a ann +ze al +j ant +im bi +bat smen +um o +ther on +smir k +per ishable +d wind +aa ja +pla giar +lu dic +kesel owski +clin ically +reck oning +mountaine ering +conj uring +yo gi +west land +toma hawk +montr éal +jaf fa +b de +ra fts +n lc +avrilla vigne +ux design +sun roof +ram s +gw yn +die ter +awak ened +ab l +sur realist +der mat +ban get +the cat +latin x +gar nett +ay or +wel der +state house +love joy +gir lie +coquit lam +refu el +po u +man candymonday +ma q +bus by +tt f +picture oftheday +ak ade +yi pp +y ere +wi p +tre search +li ya +wh ol +dig ic +bel lied +abar th +will ough +vil nius +tellu ride +kar at +anth rax +t work +as say +ach am +wil shire +rain drops +l are +gigab it +do san +ab p +ðŁį ¬ +tr ynna +orthop ae +h inter +go irish +gian carlo +gas ol +chat bot +where is +si h +holli ster +cli c +abc network +dress ers +fe asting +elev ate +constitu ent +adventure time +sr iti +el ou +de soto +dav i +contain ment +lo di +ko da +gl in +wr itten +wind chill +out spoken +make adifference +j annah +" -- +tro t +summer fest +sil os +joom la +game design +ar go +ru pp +perio d +new quay +mitt ens +ici ally +emplo ys +du bious +bail out ++ @ +ðŁĮ § +âĺ Ľ +special ties +pan ini +mb ridge +gar nier +du els +anton ia +u j +ph u +aw at +robo cop +mac abre +dom en +band ing +anat oli +ad n +nam co +laco ste +buy out +fav ourable +esc o +sexu als +kait lin +en try +ad ly +yang on +win ston +wau gh +pati sserie +ozar k +kristi an +kha shoggi +g mm +embar king +mo aning +mal kin +j el +di ggers +bee keeping +whirl pool +hor gan +bb cin +ðŁ¦ Ĭ +ðŁ¤· ðŁı¼âĢįâĻĢï¸ı +suffra gette +mer u +dro ck +cru fts +woo dru +pi ero +om bud +esp ana +advis ories +aby ss +us ar +ren ato +jean ine +endocr ine +. âĿ¤ +ðŁĺį @ +ìĥĿ ìĿ¼ +wand sworth +slo vak +reli ance +in competence +ey oung +ap as +a sen +s lander +ljubl jana +iti ve +ar gent +tion day +reson ate +more house +en chant +b sg +ri vers +n ils +m da +indul ging +gal le +sav annah +no k +mn h +lu h +hi berni +tor turing +le b +girls who +dro gh +adri atic +shar pen +swa sti +se urope +i fs +gi mpo +eri e +amade us +ipf conline +ðŁĺ© ðŁĺĤ +tr l +as syrian +ðŁĻ Ģ +vi ene +data protection +dream catcher +thro win +red undant +pen rith +n ne +amal gam +sense less +par v +national guard +kne eling +guine ap +fa qs +cy an +ãĥ IJ +whi le +loun ge +sik kim +makeu partist +instin cts +ha ji +cot to +vil i +mb l +com mo +mi ga +lu s +ar mp +ŀ ï¸ı +æ Ī +platin um +na am +lukebryan online +gulf stream +ad der +tot ally +pal i +wel i +alter ing +ts x +par ake +mon ol +air lift +sym pathetic +su pa +recep tors +pat a +orchar ds +op m +lo dged +ky i +bru n +villen euve +ko e +electro ly +dead mau +a ed +sharp ton +re branding +nu est +hub spot +hemp stead +gw end +bourgeo is +wn w +living thedream +friday night +orthopae dic +kx ly +is and +f co +f ada +bla s +all l +: + +r cb +mi kel +live streaming +din ing +de ford +she esh +lon nie +ho ard +zar if +thevamp sband +spiro smar +spirosmar garis +n hi +ft k +biome tric +bas f +auberg ine +acti vision +vari ability +pi ans +med an +l nk +ira h +t pc +r tv +ofi eld +dr aco +bri c +x perience +we stin +santabar bara +quadru ple +connec tivity +bru ssel +marriage equality +dat am +concac af +ë ¬ +w acom +truth ful +sw irling +sher lock +archae ologist +aque duct +york town +ton k +ten n +sti letto +jo on +ab ril +f ft +boiler up +? ðŁĺĤ +shi sh +deci mal +unle ash +pl at +ec risis +nar c +suff ice +jellybelly friday +it an +inv ades +ctr l +santaanit apark +le aping +invic tus +ful fil +x ic +under stated +l é +higu ain +ct is +bo realis +annihil ation +z hu +ul rich +shar ing +pul w +eth andolan +vard han +timber land +corin ne +spac ef +resili ency +pu k +inspec tors +cer ve +beli us +avent ure +par ris +pag ing +hy land +debac le +first look +bast ille +Ľ ï¸ı +ðŁĵ ° +ðŁĮŁ ðŁĮŁ +rel i +raje ev +fand oms +val verde +med ford +vo wed +v amp +sweat pants +dee z +par nell +glen wood +bur ners +road works +no ire +lek ki +ðŁĺ³ ðŁĺĤ +sus que +sp as +s dr +launch pad +de tto +sa q +cam po +ðŁĺŃ ðŁĴĶ +vi va +ne g +jol ly +di anna +waf fle +trick le +th w +scumb ag +henrie tta +foo lish +expo s +caf er +bil awal +âĢ¢âĢ¢ âĢ¢ +stri be +se ward +n de +lou th +cyn ical +bat o +m ily +inclu sive +hai yan +aj ar +ðŁĺĬ . +me redi +d pt +can tab +ven n +gan e +di was +bird club +tr ina +o gs +mon ic +he en +de mented +she ik +noman ss +itu nes +gly pho +ðŁİ ¯ +y ous +wi fe +vom iting +om gro +tax onomy +ri eu +berlin ale +ad ag +tur ret +maul ana +mag icians +ag ul +xx i +the age +shel ter +gru ber +cri mson +bal di +ab sin +h inge +me ij +loc a +ge iger +dri guez +atten tive +dit ched +cat nip +íĬ ¸ë +loaf monday +joko wi +ce bu +chur n +breeder scup +stap le +lef tists +train ings +fu ku +e bb +colon els +â Ĭ +whist les +shon en +mc ge +vel asquez +tes lamo +lom o +car rey +in ton +kent on +isleof man +aaj tak +ven ous +tuske gee +original funko +jewelry onetsy +ðĿ ķ +per ak +eye balls +dom ingu +ath al +ðŁı İ +tg dn +ta v +spam ming +ren ters +not ably +kav anagh +pp ert +m db +fox sport +ex ec +besik tas +auth am +ka iju +ðŁĮ Ħ +utili zation +spo of +indic es +hin der +gir ard +deep en +anag ar +ðŁĶ ¹ +termin us +s wore +rita ora +li ven +bra sil +alham bra +ðŁijıðŁı» ðŁijıðŁı» +ton ews +ore gano +boat eng +joh ann +bu mmer +ba ston +à® ķ +then ation +spac ec +cru m +bu sch +sarah g +lo we +aw arri +âĪ ļ +zel o +wayback wednesday +tent acles +l hh +jo ec +eras mu +de witt +rick man +dill ard +curi os +poin ty +po thole +mc nair +he mat +dr m +de fies +w sb +plant ations +ha im +pal trow +up i +ter ies +shor tened +al ac +pon der +la ker +best fandom +ambul ances +safe way +pas ser +melo dy +ima r +spo tty +in der +hear tedly +ge ss +bi ga +ðŁij Ĺ +fl ack +bott as +y ara +si b +disci ple +ti dal +sol ve +lon a +âľĬ ðŁı¼ +strato caster +k rs +eng age +du chen +buon giorno +ঠ° +pi geon +lets dothis +fe et +ci roc +ðŁIJ Ī +som ers +ko ch +i ain +al m +veg am +re pu +promethe us +pede stal +ke swick +i ol +ori z +cotsw old +a er +Į Ģ +æ ° +head sets +al ona +volde mort +gor d +fu se +dont care +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +spl at +port als +lets get +inform ation +en ugu +attend ants +th ackeray +progre sses +ke igh +alpha bets +ðŁķ Ĵ +sand ton +derel ict +ìĬ¤ íĥĢ +un familiar +super human +ri an +insur gency +cor rug +trage dies +si sland +del ilah +and aman +fu chs +br ati +adam son +par as +live by +even ting +ç ķ +ra go +le z +il ook +bou lders +bo j +tom brai +ring ton +ma ul +fi que +complic it +wel beck +gryl ls +discrimin atory +une p +scra ping +pan a +ocean ic +mat tered +ಠ° +tropic ana +house wares +bell i +agu irre +un censored +mult itude +mon god +met life +kapilshar mak +gal len +confin ement +tru est +new bies +chil ton +cal cio +ballis life +________________ ________________ +âĺĥ ï¸ı +santam onica +open ness +faw kes +co leg +bo yn +ðŁĶ į +r caf +pr in +pic colo +dev oured +av at +adequ ately +ìĬ ¹ +thi e +mc ity +madi ba +le mmy +in ject +farm ing +it el +beauti fu +ৠĩ +Ù ¾ +miz uno +en rolling +du mber +aqu inas +wo ws +sque aky +l ons +impro per +esk om +emancip ation +bar ba +a hahah +âĺĺ ï¸ı +mc mur +eye sight +dissi p +cairngor ms +baf ana +s movie +li ang +ger d +andalu cia +am mon +yl de +t mi +s group +poly mer +newin dia +li i +te w +le ge +go ha +for ay +dissol ve +th ks +so ire +lan dis +go blins +glau coma +jble fevre +d cu +th ony +p tx +margare t +mal in +íĶ ¼ +li shing +cough ing +conce aler +und p +sw ir +g te +sil ica +ro asters +po go +ou sted +in play +bird sof +hist med +dep tof +bon g +ric key +mad man +fundra isers +e al +portsm outh +mu th +predic tor +iz one +compens ate +sh inju +po achers +nbc dfw +ci ano +ðŁı ° +uof a +po cus +open ers +insi dious +a the +yi el +sup date +pel let +n sc +f fr +cha e +½ ï¸ı +lo m +l fa +kissim mee +hafi z +å ¿ +tr ich +elec tive +brant ley +mi g +mee ee +lun ar +laver ne +cor related +carto graphy +ar au +z az +yi p +viol ates +negoti ated +law ton +âĢĭ âĢĭ +to ads +reno ir +follow your +arma an +w apo +th yr +n gu +mark sand +rein force +pension ers +pa f +mu kesh +fer ro +çĶ ° +ven u +re run +man zan +fin earts +bray den +x m +wag yu +un bearable +ri deau +ec m +c pm +b itt +ðŁĻĥ ðŁĻĥ +ye ahhh +temp ura +re view +noo o +moder ates +li ef +lat ory +deplor able +co yr +re gas +gov ina +dv or +angh ami +seri es +pal lets +lin d +sha olin +colli sions +than a +l lu +jume irah +honey well +compan y +ve dic +twenti es +t was +snu ggling +la f +gossi p +bow yer +ba si +vigil ance +sni pe +senti ent +represent ations +formul ation +adven tist +âĨ Ĵ +t sa +ss ors +isu zu +bon ham +vi vegam +liver more +join us +ðŁĮ ¶ +stage coach +con tre +clique art +ðŁĵ Ī +ðŁĴ ² +par son +ful ham +ª ï¸ı +omgro bots +bridg end +wink ler +waver ly +ton to +slu gs +glit tering +ni d +dog sof +ah hhhhh +thisis queensland +pro wess +pale y +n ga +gangu ly +dor mant +agchat oz +vi acom +song bird +ron ny +after school +un insured +ther a +bc afc +. "@ +ja o +ip cc +hef ner +gen dered +cou ch +be there +v ann +retali ation +moder ation +j pl +mac adam +dan sk +y us +mu ri +car amel +bro mpton +ar mando +agu stin +.... ! +ski y +kitty loafmonday +ido t +en son +ha vill +extravag ant +ðŁĴŀ ðŁĴŀ +the op +un done +ephe sians +nott inghamshire +nec tar +ch q +bai x +te z +stream lined +fl oun +all timel +republic day +mer curi +cc w +ak ash +ðŁijĭ ðŁı¼ +twi gs +tul le +ti ram +red ford +ne ttle +el ms +bu gger +fitz roy +! ( +ver ve +bottom less +blu shing +valedic torian +tin iest +recy cle +ju di +ather ton +time for +ti mi +kis umu +fron ted +e ola +digiti zation +cu ster +baz aar +tri angular +st ann +paedi atric +mer can +ma ren +gv prakash +wind screen +un pack +la do +finan ce +saf rica +cron ulla +bit ty +bel ter +be bop +âĢ¼ï¸ı âĢ¼ï¸ı +my x +ker man +dd ell +bringbackour girls +sau ce +rac al +pap a +nu f +fa red +cartil age +c renshaw +vas a +rele ss +book ish +w mata +ite x +dor al +astur geon +tremend ously +info sys +fan fare +die ting +ðŁĺ ° +suscep tible +sex po +ry erson +mo fo +yel len +var nish +ðŁĸ¤ ðŁĸ¤ +ðŁIJ ® +mo sh +lif toff +kamala harris +crow ning +# . +âĩ Ĵ +tu f +pac er +shaf fer +en lighten +swe ars +apolo getic +yi elding +un opened +n vr +ken ner +jump start +en sured +à° ¾ +t mt +pack ham +cd mx +swer ve +sprink led +day dreaming +boudo ir +nicol asturgeon +be im +motor speedway +ane ur +acron ym +mer cer +facil itation +d ass +as il +,, ,, +tb ol +ba er +auto correct +won ky +the garden +remn ant +mv ps +mun ity +ling o +kar am +is ma +dignit aries +boy hood +st inger +marath ons +lo fficial +jo ero +flat bread +er aser +court side +y ville +n ila +li mo +im ho +tho se +pre viewing +missou ri +explo its +cry in +( ~ +y ooo +sal ma +po cahontas +ex uber +an ad +united we +pun cture +explo iting +deci sion +cauli ffe +be curious +⼠³ï¸ı +z av +newh ome +carbon ate +bust ling +bts fanart +az ur +ade bay +ac cli +kit t +c love +bur lon +ภĤ +new town +im perfec +hit z +depe che +carne gie +twitter blades +qu art +nu isance +ih sa +t series +knu tsford +doug all +at ourism +and beyond +bli ster +am es +prob ate +ex ported +ca icos +toast masters +noo oo +fa kes +pe at +maa stric +ha rem +bha g +aus vind +preli m +chippe wa +b ni +po g +pa wx +t day +e ep +benedic tine +trigg ering +e chi +v k +pretty little +har k +mam at +el more +cu ad +ar nab +j hs +c mp +v ra +stor mers +lan ier +jar rod +ice hockey +wren ching +wreck age +om ia +na shik +ar co +sveng oolie +en tran +bake off +thisi smy +sw r +grateful dead +mus sen +m ff +fal co +dor se +win n +prin cer +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺį +fir mino +yu z +sk op +mum mies +mor si +l ca +art pop +ame er +qu ant +confe ction +cag li +mccla in +se ye +s int +ro bi +ra aj +á ¶ +way ward +mi mi +back side +ãģ Ĭ +regi sters +gru po +dis graced +i ghts +analy sing +advance ments +trum peter +tr ice +stlouis blues +sapp oro +ofthe month +j son +j cc +c of +bo real +anz ac +ro ch +pan tal +ny rangers +n nam +ic arus +dre au +ë Ķ +mo an +ðŁĴķ # +yann ick +pope in +ma sha +house keeping +gy o +giz mo +book stagram +samu i +ex xon +cri sto +chi sholm +sas qu +ric cardo +rel ativity +lu i +d ori +we can +super cup +se aton +func tional +chil is +sf r +p wd +le eu +l ha +ide as +and a +fli gh +ash u +tou rof +starwars rebels +per alta +on an +descri be +daf fo +se ma +monaco gp +k ink +himalay a +gi za +chim pan +law school +j z +im mobili +dick erson +chan ey +chain smokers +and hra +vir al +over take +madein usa +et ano +ca ther +quin ton +ik onic +co pley +anc entre +amal fi +ðŁĺį ðŁĻĮ +super bly +q z +hu is +sier rale +my name +em ph +yi sts +snap seed +self love +r cs +shout gamers +newsl ine +gn r +ec co +ca vern +ha pp +environ mental +dream in +ag il +! ðŁĺĺ +winter fest +sh hhh +s ye +sch on +mlb draft +bla ise +dunfer mline +be aming +a star +ðŁĺ ¿ +tu fts +har inge +f ü +aq sa +abu bakar +!!! @ +wad ing +fla panthers +dun dee +bo hol +rejuven ation +ers week +cor se +wag ga +tor o +tat er +sa ira +o tra +mck night +for thelove +t tawa +baff led +lex us +davis cup +sw ick +penetr ation +b dn +whe res +twitch tv +pr ing +heal the +f ms +tm j +pir lo +me zzo +man ley +lovel ive +hu ffman +ðŁĮ ¶ +wa w +to toro +cur tiss +chi u +bil is +ti kka +r td +im poster +edic ine +olive ira +neat ly +é Ľ +wells fargo +s mbb +f ick +alltimel ow +shim la +sat l +rein venting +pen h +san te +nu kes +im pal +bohe mia +ðŁijıðŁı»ðŁijıðŁı» ðŁijıðŁı» +stin son +bt l +ëı Ļ +ul tron +sand ing +n th +mir amar +bre l +bre cken +re draw +evi dently +e zy +re unites +miami beach +indian army +crunchy roll +Ø§Ø ª +l of +gg en +fl ay +do ol +wil ds +th ir +rent ine +ren nie +napp y +lesli e +ag ec +wood side +vi zag +ro ker +over loaded +esta dio +ภĬ +un u +kah lo +hi king +ðŁIJ ³ +ta king +ol m +ingenu ity +el p +common wealth +baha dur +wiz khalifa +stray kids +southbank centre +show er +humph ries +de vol +cor ny +micha ele +fon do +anc er +è ī +ron in +ar ou +proud ly +pp d +donnie wahlberg +copy writing +cap a +bro d +un imel +ome let +le berry +eccle si +cla ret +ter ro +si fy +lon dres +legend sof +doub ted +ö z +sony tv +rebe cc +vote yes +tv show +sun burst +de y +benef it +z ico +t cd +rejuven ate +don ato +to is +im porting +fa kh +e fe +and ela +zind agi +love y +high school +gordon ramsay +fur ries +f cim +chi vas +ax ed +p mc +jay am +brew dog +gam y +captiv ated +shout outs +sab bat +ru fai +lat enight +descrip tive +s racing +pr p +nad in +mushar raf +grump y +gn arly +font sunday +fon dant +classi fy +ðŁĴĥ ðŁı½ +ryder cup +pne umatic +i phon +ra glan +mam bo +gilli an +enig matic +cor dova +spoti fy +har ish +emo tes +ar gh +m bi +love to +cur ve +ad ore +po sa +pa inte +be gum +> @ +ro che +mag i +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +x lr +stoo ges +newsline weather +wc l +linkin park +bush wick +hei ght +cla pping +capp ella +bad i +loo t +def con +super hero +shore ham +mc c +k lam +ale ducation +é Ł +the democrats +sher ri +dioce san +d mb +sen sex +lovel iest +ai ko +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ıâŃIJï¸ıâŃIJï¸ı +gra z +cla sp +chec o +ar nie +stra d +dar ou +britt ney +bra h +festi ve +er ley +the blacklist +tb ay +pau lin +basti an +affir med +stre isand +gan esh +stat ute +re load +lu l +id is +youcan texplain +nu tt +migr ated +zi ps +pro dig +ma geddon +for ging +ðŁĺ ¨ +st mas +plu gging +dur o +correc tive +t elly +sj p +pi et +anu i +adap tations +v ant +myel oma +cap one +sier ra +black water +zeph yr +yon kers +thr ac +screen cap +pa seo +mi kes +lock wood +h rd +er rol +colum bus +ab al +pp t +indv aus +char lo +par aphra +daniel e +r joseph +hir sch +carol yn +thro ated +sli mming +adi os +v logs +mun ching +j akes +fi k +bar rage +shan gri +pin occhio +pa kh +min as +icha el +diversi fied +caes ar +ome try +ham ble +cuyaho ga +bai leys +seat belt +jeopar di +brown sville +scandal ous +oni ans +ble aching +found ation +the le +rye o +kaf ka +ja ja +feder ic +fat al +best price +bandic oot +ðŁĺĤ ðŁĻĪ +kor o +fac to +dispen sary +br ation +ur ray +makeameric agreatagain +wit ness +toyo ta +pat ties +black board +ad is +te rer +ss chat +sh alt +record storeday +la da +gi ann +íĽ Ī +un holy +kh ana +godis good +palis ades +he for +ci ve +hered itary +hay wood +cor ker +spr ingh +sand i +re du +natu ro +many vids +jessi e +âĵ Ĵ +schnit zel +o vie +gren ades +gat es +ab ed +ms ft +medic ally +led ore +l ousy +mentalhealth awareness +glee son +col ly +cabrio let +wee e +sp end +snow mobile +hi j +Ï ĥ +tal kies +rich ness +jor dy +giu lia +acti v +do pam +alleg ation +your life +sk elton +v ny +mu riel +lat t +inaugur ates +foreclo sure +tain er +harne ssing +aguil era +x rp +coo lidge +car ta +ser gio +news radio +k tr +sol arenergy +r sprasad +home design +ho stages +hat a +al ali +thal er +a sturi +tri pura +hydro power +free bie +escal ating +m ha +getin volved +protec tour +od is +mus ician +mu le +u wa +ter iyaki +rip city +race horse +loaf ers +kha o +fi vb +bal con +an ou +ðŁĽ « +vis ayas +sh all +fire flies +ठķ +re morse +pho tonics +let splay +imp lied +hes itation +gur ney +ol om +une ar +pi d +mo gg +itt al +âĿ¤ï¸ı ðŁİī +ma ku +ar man +mo ke +han ts +cor fu +ä¸ ĸ +digit alization +ti ana +su bo +schu yl +e redi +ven cy +v room +ro ars +growth mindset +cosme tic +chir p +stra u +seh wag +ric ha +pin ellas +elo tti +dur and +deta chment +qu ando +mau sole +ma su +black wood +aspir ation +bell ator +shi ka +mar oc +ki ra +pi k +gta photographers +gand alf +sto y +spee do +mand alay +fan o +un ice +sol ange +po pper +den ch +ne warri +cel ta +d lers +ce tta +cc f +black smith +bhan gra +w anders +hide away +employ ability +z te +under take +tw tr +team building +ta pa +virtu alization +pro vo +eigh ties +che ery +ay u +we ber +per ro +inspirational quotes +d hoo +aj ara +ðŁIJ ł +sub du +bill clinton +am oun +stro oms +soldi er +mouth watering +malay a +legitim acy +gr ats +syl vi +sleep ers +boar ders +ðŁĺĤ ðŁĺĺ +up loads +sports news +ske wers +referen cing +fo dder +ea a +remo s +ra ss +n ann +cor azon +alas ka +shak h +pig mentation +incogn ito +as ca +miamid olphins +le land +ig t +gn es +boo s +cla ps +major ing +di als +---------------- ---------------- +regi mes +pe an +emul ate +mer ga +med hat +head liners +go h +con di +wi gg +ser af +ric kie +bor ty +âľ § +re is +cel eri +âĿ£ ï¸ı +ye ez +ni ki +draft kings +diet itian +at weets +ampli fied +nee son +mac ross +dystop ia +bor ges +blat ter +b ade +direc to +bha skar +sch ae +kar my +scot spirit +moment ous +an ation +lm k +kne e +in da +ig g +bo tham +barber a +toi lette +r tl +metcal fe +lin x +clo thed +vo ila +se well +region al +ple dge +intere stingly +f nl +ru f +mun di +bur sary +bout i +âĺ Ķï¸ı +âĸ¶ ï¸İ +sto liveby +star let +pic stitch +car wash +aw ar +round house +margar it +manag eable +bon ito +us ab +on n +flow ed +cher che +s ju +kensing ton +jo yed +cal e +???? ??? +zu mel +wir th +re pul +gary barlow +coel ho +âı ±ï¸ı +work er +sp t +siob han +segu in +s gp +glypho sate +clin ching +charlie hebdo +bati k +uph eld +just o +her ne +euro star +mccull ough +fent anyl +diamond backs +âĺ Ķ +indian express +jis ung +hen e +conve yor +appe tit +yn g +over kill +kim mel +in it +y ff +st ine +le urs +her ring +ear ths +or ne +leis u +go pro +sky way +scri bble +by nature +water colors +tin tin +inter ruption +br rr +wide screen +shake down +knit wear +karab akh +id as +cor der +tt our +sac king +good ell +thisisy sr +goo ey +german town +fa w +cat amar +ðŁĴĥðŁĴĥ ðŁĴĥ +vijay a +ti ber +sa ad +ha are +seul gi +mischiev ous +isu per +hellu va +confe ssed +litur gy +in man +ce tti +tuni sian +tic ut +sur ging +sau er +ry erson +popu p +mb k +br itten +v f +gray ling +abstr ac +us p +un fa +so rely +ri ona +r dc +of sted +ju icy +horri fied +grac eland +fa k +justi fication +jam esp +bat u +. âģ¦@ +tu i +te jas +sul fur +indirec tly +a ford +under privileged +northan ts +m countdown +ji ve +ha des +lac quer +humbold t +gi ggling +jun ho +digital painting +aph rodite +ab di +tel us +pric ey +hahahaha hah +fibon acci +dis mantle +ar ne +an ine +willough by +motivational quotes +mid west +inter lude +ge re +be come +s illy +felic it +tap tap +st ings +ilovemy job +cospla yers +bra u +votel abour +sto ver +ru ddy +meh boo +hon e +gift for +phosp hate +it ano +do sa +babys itter +kri sty +dele on +dd ard +confu cius +stewar t +s worth +com ed +arn hem +a ño +ym er +smo re +pur suits +flee ting +dat ing +sav anna +delic acies +comic book +co arse +cir ca +calam ity +bro ads +pre natal +jo c +cyclo ps +c lec +yam amoto +un b +pu sd +plu mmer +per ils +gu mi +ath ar +val o +timel ines +ense mbles +b sc +maha jan +wa sim +techn ics +sorcer y +jo bo +havill and +themore youknow +ta ki +rest ful +mer thyr +cro ck +wee ps +reneg ades +lear nings +g lands +ti dying +ec tomy +dol la +ya g +rev it +kar ts +f natic +apologi zed +win dy +w bc +si esta +pe ony +gor an +autisma wareness +af fi +wh aling +v ps +train spotting +ra kul +mel anie +mal let +ky our +? "@ +reti rees +ip kk +in hale +electronic music +cur ators +pocon o +l sc +an son +u si +lu r +hide out +twi sting +samp led +jun su +ing s +dead ly +auditi oning +happy bday +emplo ying +ti voli +nic u +fu sil +ak am +palmet to +ofthe wild +my kon +mahar aja +deut sch +selec t +nat ura +me ddling +land is +i have +comm encing +.... .@ +ðŁį Ł +mer gers +m dp +ben i +ne led +lino cut +kn ack +j pm +battle ground +at ter +rat on +pente cost +organis ational +decan ter +cn r +boo zy +bap u +al ve +fast pitch +ðŁ¤· âĢįâĻĢï¸ı +z hang +token sale +hear tof +ha den +rapun zel +lar o +fro yo +bikin is +siddi qui +scri pps +pi ec +lin de +story board +red lands +op o +mb r +grace fully +c de +th enews +cas pian +and ali +ä¸ Ģ +ro ald +optome try +medic i +ken g +der ma +ðŁİĥ ðŁij» +mar chi +bi al +al ab +âĢĭ . +red wine +k gb +gun violence +gi vin +fan page +adri ve +ëª ¬ +tel stra +fl t +biome chan +ðŁİīðŁİ ģ +dram a +اÙĦ Ùħ +ti rol +de ferred +be sts +spr ouse +o hh +chead le +im balance +gy n +cruis in +ci m +un h +tra pp +ob scene +ir fan +impre gn +deut sche +cymb als +ob an +au er +atal anta +roo k +men zies +g went +entran ts +co pac +or tho +marksand spencer +lee ks +lac tose +spac ed +sh ak +pale ontology +ine mas +e an +bi bi +alban y +ìĭ ľ +ko in +be ssie +ar dent +latt ice +be sie +ade h +# _ +surro gate +sand hu +rain water +k hun +cau sal +be wit +at las +agu e +water polo +l ts +jam ess +ae on +sch ofield +motor cyclist +ge force +dre y +wai vers +sh us +no excuses +di ade +sweethe arts +macin tosh +l indi +ar junk +am writing +æ Ľ +luci an +ink jet +citi es +survi val +gam ep +g j +ðŁijĩ ðŁı¼ +zz ah +objec tions +dit to +ìĤ ¬ë +trump care +lof ty +tool ing +intrin sic +fenerbah ce +cle men +will power +su tra +sim pl +save shadowhunters +san aa +pri mate +or bit +kel sen +asho k +artisan al +ê¸ ° +tri stan +shre ya +midwi fery +lit ers +la dd +fla x +bry ant +nau sea +la vo +ul m +sid hu +she ena +gott lieb +cr n +we id +âĸ Ī +motor ist +ma vic +laut ner +endo wed +spar rows +ka de +ip p +o vens +ti led +stin ks +keen eland +kath a +c te +mass ages +interro gation +ðŁı ĸ +sen sanders +fish in +dro se +ðŁĴģ ðŁı» +sustain ably +sh ant +propor tional +mis cell +kh l +chemi sts +m ra +her pes +f lux +disgu st +bon nie +artin fo +~ $ +sau dis +pollu tants +op ia +mo fficial +dark side +cultiv ating +civil isation +champion ing +bl f +armed force +ðŁĺ³ðŁĺ³ ðŁĺ³ +tian jin +lar avel +fe men +baf tas +absen tee +ra onic +provo ked +pla gued +cool ers +uc davis +sand er +s books +di orama +un ab +sub division +pritz ker +pa sty +j ks +account ancy +tri bul +re tta +er ty +! ðŁĴķ +ðŁıĨðŁıĨ ðŁıĨðŁıĨ +ri beye +theli al +nin ja +g ls +cer ro +usa id +pu ma +pascu al +chev y +brati slava +bra ga +bi gs +ap nea +åĨĻ 羣 +sd p +marguer ite +khu sh +vec chio +glit ter +el issa +dumb ledore +car gill +ann am +trium phs +templ erun +ru min +lny hbt +cla sse +êµ Ń +ri ri +gun ning +boy e +am ento +limite dedition +gra w +gan ache +ðŁĮ ½ +resemb ling +on tv +moti fs +i mani +hel ms +epo xy +clear ances +ba ha +this day +re eling +gur o +fi eri +faw cett +chec kered +ti v +narcissi stic +i tha +guil le +go e +dart ford +comment ators +cl out +ch illa +ky li +hun d +ro maine +jum bo +fil ip +de au +tyler rjoseph +the un +orphan black +om ans +in manila +tho reau +sa ar +ra bin +en heim +tn t +state parks +kour tney +we th +kair a +ec r +gas par + ¸ +olu tion +me ars +home town +execu ting +defic its +car bide +blan ey +stri dge +sh r +ho tty +grow yourown +fal cao +îIJĴ îIJĴ +âĺģ ï¸ı +un wavering +more tz +hoo dy +h ine +dain ty +bak ing +ภŀ +salom on +disin formation +pu sha +la the +ad di +abi ding +zig gler +sic ilia +mening itis +hol ling +aus gp +ri mes +barrac uda +al do +af tra +pe di +lith gow +analy tic +vanc ity +se f +pc as +c ya +afric a +w ella +ra ys +n cat +fe ez +don i +din amo +breast stroke +truec rime +tofthe week +south ampton +el ina +zain ab +sw az +ph elan +kri stine +k lit +er ation +bud d +wrist watch +the week +simil arly +qui k +over throw +naku ul +itali ano +bigg bos +se ashore +arnau d +le p +fan site +ding o +cler ks +cas illas +jo die +de generation +ãģª ãģĨ +her ze +adjun ct +ac ard +ðŁĴĻ ðŁĴļ +à¤ Ń +wa al +rad hika +chim es +ti pp +o or +ki ye +he c +ba hu +ab at +sam us +inver ter +indi spen +he ge +ภģ +l ff +bi ele +mu ja +indone si +f wa +beat rix +bbc football +sa ks +q c +cont acting +att enti +vivi en +taf fairs +so crates +sen e +se mper +co bb +ðŁĴĭ ðŁĴĭðŁĴĭ +ladbro kes +á » +will ingly +p are +p able +occi dental +ich es +can tor +rox bury +fre ddy +ed ger +de capit +wgr z +ma im +ku sen +ulti ma +solar power +schan el +dishon est +beck ons +yad av +over ton +ne ale +me sses +darou sey +auc tione +ap aaja +ðŁ¤¦ âĢįâĻĤï¸ı +u tri +.... .. +taco tuesday +prote as +introduc tions +pi ds +gat linburg +pam pering +marig old +f isa +con tor +clean est +short stop +ne em +pin kie +let our +kathle en +imp ly +ðŁĴ · +work out +ro que +estim ation +countdown to +conom ics +al tar +q e +jesu schri +grayson dolan +st ines +smith field +shi gh +ir replaceable +spr itz +mon op +lo real +lab elling +fli ppin +ðŁIJ Ĵ +sher pa +jan el +embr yo +bir dy +ball game +vul gar +un f +spra ined +cr f +come back +col man +att y +at a +ar lo +u alberta +technicol or +rush more +respon der +ou re +obli ged +mt ve +mac c +wood house +vent uring +sen john +light ers +her bi +wall ace +exo tic +ag em +virgin ity +the ma +mart ine +mar ang +lee teuk +ho list +watch men +s ile +le ck +kan ji +hoo ters +st ile +n ala +e am +clari fies +am é +voc ations +succe eding +jesu is +conqu ers +uc berkeley +nz v +cas o +bar ist +bar bed +patri arch +p cos +ben ign +re read +mn ang +fly weight +fc live +cre t +bar ks +lione sses +benid orm +bc z +an je +ti wari +ol li +nak amura +ha san +fun imation +co ss +bir der +anna bel +tim er +rn cin +real ale +ra vine +no s +ðŁĩ° ðŁĩ· +te ke +sa hab +bal tic +âļ Ķï¸ı +sur g +ka hit +tru ss +stein berg +replic ation +elev ators +ðŁļ ´ +ma ki +har ming +h si +del ena +t ke +sime one +pat ina +shutter stock +she ars +aller ton +ai ka +temple ton +raf ters +perio don +note worthy +mongod b +man fred +ko wal +stu b +join ery +an gie +u kenyatta +shel ving +kenil worth +instru cted +ta ey +retri eve +interpret ations +first world +d si +biop sy +benef iciary +saf a +philosoph ers +pand ya +ner i +bow ery +wel ly +the cw +spr ites +ner v +mon orail +jac uzzi +de mysti +con sort +program mers +news desk +l sa +hong kong +home and +ed ale +dor p +dar in +vali date +tear drop +syn ap +repeal the +premi um +nik hil +blan k +ai ding +squee zing +rncin cle +ret ard +park sand +bru ck +wr u +su zi +specul ative +hew lett +cru st +as m +app end +¢ ħ +tele mundo +st aley +sn rtg +samu els +jae ger +farn borough +womenin business +ron darousey +min es +au de +shi ba +m ne +estre lla +swim ming +ë¹ Ħ +tam an +it é +cuth bert +casu al +ag ing +offici ating +li gan +fo iled +ver i +ms r +dr ay +usf ws +ta heri +se thi +ç ī +uneth ical +kick ers +hi jab +ak ash +pur po +pellegr ini +neu mann +man del +de ver +ap u +ìĭ ł +z ha +indic ations +imag en +clean india +car pe +so fe +mart ine +stra pping +wit ter +no tin +fic ent +bbcc ricket +tur ally +cou rier +tri xie +sw am +i ab +alfar omeo +stal ked +so h +oooo ooooo +miasan mia +con f +thisgirl can +tar rant +re reading +present ly +pow ys +nj devils +mart i +fe b +cerv antes +tam bo +retro games +lang ston +kell er +ar nol +ठµ +shinju ku +sasqu atch +dan ica +akrish na +so ko +cat ter +car park +inde cent +er gy +bur ley +brother ly +xi v +post doctoral +polyne sian +suspec ting +mass a +make ithappen +fab ri +cu ti +cor sica +bor den +un impressed +sli gh +he don +gon zo +fic o +eloqu ent +dic key +podcast addict +le ona +jewel ers +wic ca +uniof oxford +u den +gene res +ban shee +u ya +she khar +doll house +blu eno +af alls +wra ith +ta fe +moun ds +j ct +in clement +fun g +fluori de +! âĻ¥ +raji v +b me +waz e +squad goals +pre ak +hand painted +c sgo +sat h +leg oland +in la +d pi +c actu +aband on +tali b +janet jackson +ãģ Ĺ +khal sa +gl c +c fm +ab ang +ali sha +we m +sur passes +ou st +nai as +max ima +lind bergh +lic o +it syour +h ä +gul li +anacon da +woodru ff +pr m +h é +anonym ously +sun nah +scat tering +sc int +sal mond +pe king +j cb +ed ine +diversi fication +ari on +all state +t ley +gam bler +b hatt +ra ku +pit ts +j enga +ri di +pun dits +papp u +now spinning +ha drian +az ure +autom o +aran eta +a stray +il m +yong guk +wel ded +parano ia +explic itly +co f +ðŁİīðŁİ ģ +som uch +post partum +ler c +gu ava +enhan cements +ber gen +con glomer +âļ½ï¸ıâļ½ï¸ı âļ½ï¸ı +milli gan +% ). +âľĮ ðŁı¼ +sh yam +ry man +megat ron +koh ler +de schanel +chel sea +zoo topia +wr t +valle jo +tri pp +positive vibes +irrit ating +book fair +aac r +san it +di sk +depeche mode +classi fieds +ðŁij ¦ +ven erable +ra ves +fav re +ek iti +quarter backs +he ma +h sd +g ach +con template +l ant +kh atta +inter sections +harle quin +ncc u +m dr +pp ro +il legitimate +gre be +ler man +ite ach +cspan wj +voy ages +sat x +rednose day +oo king +dan ic +char lene +a hor +ty sm +gri ffins +cheng du +boo ka +âĢ ¼ +ye h +ur ra +tari o +pou ches +dd ddd +staf fed +revo ked +ran chers +ou z +oni ka +share d +n bs +li mp +etsy seller +cl one +visi e +ksat news +good life +cow l +chic o +н ÑĭÐ +vigil ante +skate park +re it +mar av +man ja +le et +co der +ðŁį ĭ +w gn +ld c +duck lings +don de +avoid able +í Ī +over lord +mp r +anc elotti +intermitt ent +colli der +ste ins +sque ak +dispar ity +colorec tal +clark sville +ado u +Û ģ +women swear +di bu +bar ts +sa ws +recogn ises +re pa +m cauliffe +hear tache +good music +gg ish +Å ¾ +tech house +aci dic +re pro +mal low +d printer +cro tch +kir sten +kan pur +il kley +fan i +ev ry +dit ches +cher ie +bat angas +lost girl +liam payne +la be +rid dim +den ni +? !) +vo o +tw c +s za +fo als +fic tion +lim ate +dab s +mc f +er na +smriti irani +espo sito +duc ts +blackgirl magic +weare one +sen ec +sch ie +nath singh +fo ia +en or +don bas +ç İ +wi jk +over lap +gron kowski +full er +e ssie +ðŁĺ¤ ðŁĺ¤ +ye ye +loksabha elections +guev ara +bi es +âĢ¦ âĢ¦ +z he +pic a +homer un +con stip +apo stro +refr actor +sa an +o gun +man sions +re sh +nam ely +high er +evan ston +up town +ring side +naz arene +ir regular +invent ors +pope ye +mnang agwa +fern and +sp litter +mor ten +asc end +whi ff +cr ink +be ste +at to +shad y +el n +bjor k +ðŁĴ Ĭ +no tting +england rugby +be tawards +we all +dan is +Í ¡ +mo she +miff lin +lu walia +al un +li pid +ha van +cmp unk +bur row +underestim ated +ste yn +pul pit +pir i +p nr +jer ks +fin ney +tra volta +roman ce +buck wheat +black metal +st oops +intra day +huff post +ale state +so ver +pat o +obstru cted +jen na +fi o +bo den +bill on +my fox +la hi +kar na +bartholome w +vin o +lovel and +ecu men +whitec aps +the king +qu il +human oid +alab ad +g ome +fri ar +c bre +broch ures +we si +mal anga +dr brianmay +assi si +sav our +prefe ct +mt ns +inter changeable +hex ag +gar nish +c ce +amar o +ðŁį ħ +gu sting +dismant ling +bay one +tv showtime +sun ion +rad ha +ozar ks +nautil us +la youts +did sbury +dance floor +suicide squad +ok preps +j ura +alum nae +ron ni +ma f +george tte +coordin ators +bad u +ðŁı Ŀ +ted ness +re prise +pain fully +jo ie +hill billy +thisi sd +ss er +oo ka +mm g +meredi th +klim t +kat an +blood borne +bab u +trivi al +th rif +o cla +mo yo +milk weed +âľĬ ðŁı¾ +sul u +occu pants +farm life +cru sty +sing in +r tr +penn state +met adata +fab ulously +wee zer +schnau zer +rash tra +ni moy +h cp +freebie friday +do le +sti me +jef fre +gro ban +broad ly +bil oxi +woo hyun +u zi +ny jets +fi ver +vado dara +it ake +check up +supremac ists +ro mp +the sunday +contin ence +liz quen +ha ut +ellip tical +sho o +pan ty +pa as +immigr ation +park service +os age +i sta +homes ick +cyber attack +colom bi +boy ne +ðŁIJ Ĭ +ty lero +iron maiden +hand a +ch off +al ya +ry o +red acted +mo ts +intern ment +die hard +>_ < +ðŁĺ© ðŁĺį +m sle +li bel +f ant +embar go +soc kets +ski ers +photo journalist +mchen ry +bou cher +ric ard +jayam ravi +dock lands +annot ated +ag ata +ðŁĶ » +prev ails +new ington +las sen +hydr ant +te o +rough riders +murug adoss +at su +afil m +admini stered +v fw +calam ari +bay less +sw ung +sag er +ple ated +mo dric +hi my +golf clubs +citizen science +rehear se +pic kett +le aky +polit ely +gra zia +sk or +sar an +cl inging +âĢ ł +ther oo +squir t +on your +bag gy +at oll +th ys +coul son +vivi enne +s anya +recogn isable +gr u +scoti abank +milk shakes +fundra ise +d mu +bu tters +ra wr +lin dy +al ed +sk am +ryeo wook +referen ced +quad r +cri sp +bio informatics +âľ © +che wed +sm acked +commend ation +classic rock +ðŁĵ ½ï¸ı +star ved +pu ddles +do sing +bru e +zah ra +wo king +sun rises +stro p +sal ons +lif ters +cor coran +ala ina +kom onews +ken yon +gre tsch +ge zi +floor ball +fi q +tw om +re clamation +ineffe ctive +cop tic +cit ic +ch ute +unle ashes +ran s +mo sc +joe biden +sw il +osi ris +ni h +embo diment +cruci fied +ãĥķãĤ ¡ +schu mann +horri d +pri mer +northampton shire +je b +graph ing +back street +ìĺ ģ +wood row +tar garyen +t bl +sky ler +ru ffle +joc keys +info s +deir dre +bosp horus +âĻ¡ âĻ¥ +p led +chu an +bi got +ren nes +ra va +parmi gi +chi ar +vide om +stau b +exc ali +ex clamation +city council +barnab as +se du +ker ri +is che +fr actions +fly by +bau er +where in +ra ge +ou lton +mu ah +co stac +co lec +char mer +capit an +secular ism +mumb a +hu k +hen e +blon de +so dia +r tb +de coding +cad ence +art an +ðŁĺ ĸ +too cute +tn c +chen na +brux elles +à® ¤ +t ft +ss ssss +sp ital +poun der +p ch +mega star +in junction +al ent +æľ Ī +x k +tro pez +tombrai der +m mi +amp i +tac tile +sel ina +ma sai +good bye +dod ger +al fred +vote maine +qu ads +ad die +sep ta +s brewery +ric oh +monte ith +humb ly +histo ire +me agan +low ery +del o +ab il +out numbered +er os +craz ies +r bg +pollin ator +il c +gin sburg +cy gn +ab vp +wi dest +rescue dog +ho re +y g +sof theday +rac y +la ban +i bb +ci aran +robin son +ali kes +fre n +ban bury +ball point +atten dee +ati m +ìĹ ĺ +pi p +ergon omic +ad re +rem itt +pir ates +over see +it sen +hali fax +dedic ates +cycl on +as wamy +ãĤ ¯ +mo ser +ge op +d mg +chatur thi +chag all +bu gle +ðŁĶ · +samp doria +low n +davi dg +newsmel b +fix ed +cordill era +ar ri +ðŁIJ Ļ +y ra +to pi +n aka +g é +d life +beau ts +p date +min ty +hy uk +er am +e bert +at chee +theno tori +ni kit +muham ma +fan zine +negoti ator +gad ot +bli stering +bg su +yoga day +un sw +john deere +zhen g +struc ture +porcup ine +cl ack +boun ces +qu ali +mat ador +otta w +mo es +gil git +esp lan +dy strophy +b dc +ठ¬ +sh illing +one world +on k +formu las +tru ff +teach ing +robin sons +rij ks +qu ays +her ry +fle dged +fish eye +guardian softhe +du bl +z ane +sty list +qu ack +leah y +j air +hawa i +diversi fy +darth vader +und ated +su pe +< > +p oms +memorial dayweekend +lu ms +kc mo +de hydrated +taptap fish +re com +pode sta +j ir +far med +cait rion +bor no +bc m +sk as +se dition +rub icon +noo sa +le page +haw kes +additi ves +a ol +th q +ru ta +jo burg +bas k +yan ks +mont blanc +lu a +jam mer +cho ol +rou baix +re clin +mat ured +dimit rov +ãĤ £ +m ites +âĺ Ŀ +ment alist +mam ata +g na +fla thead +canad a +caled onian +bo we +yor kie +re print +g bo +en thal +al ka +x com +u ka +tr alee +spread the +ni i +joh nathan +human kind +ha sina +du vall +craft buzz +ton ic +re lli +ra ged +jo f +free masons +____ ___ +un ter +sc aff +lee ch +extor tion +ari e +ra ghu +pin eda +att an +vehic ular +ra oul +lu pe +eng er +divi des +cr b +cap ture +bo ii +maricop a +hearti est +fair child +alter ations +no ta +mar d +s kids +no k +gui deline +deplo ys +carni vore +g sw +au ren +at ac +ani el +of c +ko vac +web site +spectac les +sn an +sher yl +rever b +nik ond +n ce +big sky +ra pes +mat o +night ly +mp n +ke sbury +jep sen +gul p +french man +bar ley +andor ra +king pin +in ns +dra shti +catar act +cand o +p co +last ly +herze govina +gr ounding +íĬ¸ ìĻĢ +testi fied +talk show +f day +even song +eat on +tor ment +love ee +cooper stown +al gi +ðŁļ Ĵ +rare disease +meth yl +lu isa +inbound marketing +ec re +t sem +ro sam +nove m +more ton +monster mmorpg +home buyers +fru gal +escal ator +de sa +boul der +bel mont +âĺ łï¸ı +twitch con +tur ners +sal ut +per vert +le ye +hiber nation +gg ggg +food service +ex porters +d assault +after market +we athered +un ravel +si di +plu ssi +mp t +ky m +bo spoli +sand piper +gole se +col itis +br al +adventure rs +ìĪ ĺ +ve do +preten ds +b da +sac re +ju m +hin ch +acci o +hy nes +back stroke +ly ce +h ve +v bs +ou m +brand enburg +âĿ¤ï¸ı ðŁĴĽ +t itude +spe cu +rapp ler +raje sh +dream land +av in +ma rek +ke ss +ho oman +pu tin +par then +hair do +aaaa a +è ģ +votemaine fpp +sha stri +remedi ation +it m +hai fa +ex xon +empha sizes +u conn +sim monds +mo ire +e scher +cru tches +vi vi +ar mitage +air ani +shi pley +prou ddad +mar v +inter nationals +bb k +prostatec ancer +mon ash +task force +god son +levan te +barric ade +tr lt +rapi sts +p vr +kimber ly +ge p +eri de +acknowle dge +will rock +u pland +aa i +willi e +une dited +tri ples +or ch +hai ku +gene see +en gel +bay e +plex us +pen zance +ko bane +f ash +cac c +xi p +scam mers +sal i +notic e +modern ity +chi en +bis cayne +nba allstar +mar ston +june au +ðŁij ¤ +van life +ro sters +r st +labra dor +brek kie +ro ssi +my x +lmfa oooo +je tta +ãĥ « +zo ya +w pg +mer chant +goul ding +food for +dosan jh +connec ticut +it to +gho on +can aan +bo le +retrac table +mar zo +lau ght +discoura ged +dan forth +a holic +th d +marri ott +in lay +duplic ate +c zar +woo fer +scre ek +ni pi +intimid ated +god ard +elu ci +ab omin +sco ping +program mable +mexico city +me tat +h mrc +h man +ashi sh +sierrale one +in sha +hero ism +gu staf +rein hart +lumber jack +gu sa +fella ini +eu f +belgi an +bab ys +an ski +yu gi +spot lights +rockand roll +k has +dark ly +but cher +bal lads +auto cherish +water man +peps ico +lo ggers +infinity war +as ach +ðŁ¦ Ħ +spring boks +ðŁĻ ģ +sp acing +la val +ffici ently +endor sements +ed am +sharp ening +sor rel +prev ailing +ove rex +nbc snl +native american +ex oner +rin gof +ml ins +disobe dience +pre car +cic ero +âĿ¤ # +per taining +mani fold +su gi +of africa +mu sh +morgan town +gel ding +co zumel +ny x +ab ili +ðŁĺĤðŁĺŃ ðŁĺĤ +world cancerday +pal acio +go k +ecoun try +âľĿ ï¸ı +ro tary +íĬ¸ìĻĢ ìĿ´ìĬ¤ +spin dle +neck line +maneu ver +mary land +i player +bal led +am ended +ne b +en rico +el dorado +dubu que +blood moon +as lam +comple te +a of +wil ma +the band +kn g +revi sta +mal ak +goha bsgo +car y +ve en +tung sten +snu b +shab aab +kil marnock +le pi +re eds +ne m +can op +ble d +vali dity +ju icing +hun a +chauffe ur +water town +wait ers +lo d +fel der +dow nie +don ate +san down +rh schel +or f +lulu lemon +kare en +galac tica +bre vard +strat us +rose ville +fall acy +-- - +w eneed +san kranti +local gov +billi e +ac cr +k illian +comm ing +worldo cean +watan abe +sothe bys +sin ful +rol i +lynch burg +integr ates +hefor she +an p +al aw +ðŁİ¶ ðŁİ¶ +maxim ilian +g ma +bre tagne +ail le +world food +turi smo +ra u +mini van +dis belief +bay ashi +us atf +skate boards +uy gh +plo ws +don bass +sugar cane +cham omile +yel lo +in quest +end ron +ani sts +te eing +cos worth +tune chi +sothe by +sle w +sher o +le f +ho wit +gly ce +joshuad un +enfor cing +baby lon +reck ons +apu ram +toler ated +resur facing +pla inti +o tr +kylie minogue +dis able +pod u +mul tiv +inter cultural +gy ro +goal keepers +cac cia +ag am +íķ ij +vic o +tri mester +ka e +coffee shop +tsem tulku +sta sia +go coogs +ds worth +wast es +boun ce +u vic +stri der +m Äģ +laur yn +kevin smith +jan ssen +ðŁį Ŀ +the one +same er +cret aceous +! ): +re gent +project management +person alize +mah di +zah n +kra f +vivi d +ten news +re re +ogil vy +tiram isu +photo bombed +n ke +cau stic +ra hi +nca adi +lauren jauregui +ki ku +i aa +dine sh +dal ton +ch ata +ba ht +x as +wi ping +ripp les +de generative +ðŁijĩðŁijĩ ðŁijĩðŁijĩ +vodaf one +thou gh +sla pping +naz ion +mer rick +intern acional +do y +bucket challenge +y au +vi stas +present a +mal t +expir ation +e ich +pu ttin +ve su +si oned +ct fu +circum stance +blu er +. ), +pra ia +mcfar land +ju eves +hat tie +du bey +myrt le +asturi as +ðŁİ ĩ +reason s +gri z +co ils +at lu +mode sto +lo gger +iber ia +af tr +v ang +transp lants +prairi e +maur it +haha a +t ink +sci fest +sc lero +random ized +glam organ +feasi ble +splin ter +blu eline +subb an +nakuul mehta +m ch +johnny depp +car cas +vin sky +hex agon +* " +women empowerment +s ooooooo +invic ta +gla de +ðŁĴĢðŁĴĢ ðŁĴĢ +we bb +desig nate +con figure +ay l +steven universe +spo il +recep tionist +plo t +na is +!!!!!!!! !!!! +ra gh +ne war +fr anta +sar aki +pi aa +obse ssing +m anda +f cc +em d +ba ins +rev ital +ki shan +fe c +ep son +def lec +cbs news +camper van +c ty +be so +at tribute +ari ya +ì¤ Ģ +tun is +out flow +not the +mykon os +clare ts +ci pri +brain wars +sign al +min ing +tr amp +ten go +jimmie johnson +christma stree +am x +a ung +v z +tran spo +spr incipal +reve rence +nam ma +key ring +ne gro +ate show +ingle wood +inadver t +impre sses +awak en +avi es +sale em +ess ence +es sel +doo dle +win ch +son e +nett les +mi me +alzheim er +vall is +insur ance +h pm +as sam +tun ity +post p +monte carlo +luxury lifestyle +ju dd +in sensitive +wax ed +was a +fand uel +chor ley +bon y +back packer +te urs +ho ag +surf board +stati st +sober ing +f anta +bob s +bar dot +ban i +ad da +rec tangle +pe tro +pa ign +n ang +influen cer +in u +hard line +gwen stefani +wood cut +s ors +for nia +cater pillars +ãĤ µ +tari anism +shel lac +nd su +illino is +el gar +cel list +ato day +ðŁĺĶ ðŁĺĶ +silver stone +log ic +fac om +weal thiest +sun tv +gy ang +com posting +candle stick +can more +z ard +x t +sc athing +ps r +north land +collec table +wa si +rip on +qu is +ge v +se men +ough lin +ob x +intoxic ated +bay side +é Ń +van poli +tim hortons +taji kistan +po i +peep ing +dri zz +contempl ation +ballo on +un folds +tre aties +aishwar yar +l hc +foodie chats +excav ator +cocon uts +apple bees +ambigu ous +repar ations +man goes +kh mer +fr itters +do wd +antonio guterres +cu a +impro vise +g fc +ec cles +carcin o +ðŁİ © +theori st +sj c +mu je +ir acing +camel ot +um mah +bachel ors +lumb ar +c sk +bi ya +re fine +queen willrock +ci os +bicy cle +bali k +à° ¿ +wil co +ow icz +loy ol +wa is +mer al +har ma +ba ili +ste e +jo ker +hein ous +bu oy +the sun +nou gat +ic ant +an anda +dji bouti +s night +relationship goals +polit ici +mb appe +google maps +ba shi +ta ap +now listening +flaun ts +ðŁĺĨ ðŁĺĨðŁĺĨ +vene to +car pent +bre do +ot l +di vert +br oughton +flat tered +bra denton +as ada +alo pe +ar b +an akin +ten cent +nad y +constant in +sb l +reinforce ments +ol b +jol li +il han +yu ri +the other +punk rock +lanark shire +bi st +ti ma +th oo +sun beam +r all +kar achi +iy ama +dhar am +k state +bat on +ay yyy +tre v +br unt +ar ashi +à® ļ +zu ka +can es +ar cade +uuuu uuuu +ma han +to pes +sen try +myster io +mis sa +deliver ance +ëĿ ¼ +un wrapped +ol li +holi sm +h pe +free view +calab ria +twent y +po g +pe cans +gru den +^ __ +ðŁĶ ¨ +os c +leng th +jef fries +ðŁĴ ı +sh el +mesmeri zed +ke gs +er at +cor r +caver ns +british library +⼠³ +pap y +al or +w age +jac lyn +co ols +bio logists +tur d +plau sible +ðŁĺĥ ðŁĺĥðŁĺĥ +yu mm +tic ho +te atro +ash lee +ve iled +umb a +obli gated +moisturi zing +magaz ine +long board +dang ling +c elli +west bury +v ä +ucl final +hou ser +chap elle +admir als +par o +mutu al +clean eating +vacu um +kh our +hab itu +foo din +fil mis +es con +sm it +awar dees +௠Ī +wan ds +thug life +mc cut +calyp so +Ù ĥ +yas a +radi om +hen ley +fron tage +critici zes +access ori +sardin es +id ge +gold finch +front end +d wi +beach front +# ï¸ıâĥ£ +âļľ ï¸ı +sh l +sh eng +rai shi +mari ota +fen der +con do +un covers +de van +ðŁĺĤ ðŁĴķ +renov ate +compri sing +tamanna ah +mash allah +humph reys +en acted +ar tu +wo e +sine ad +om ore +del phi +bre n +bo z +wellness wednesday +v sc +t iller +progre ssed +on ze +love for +gor a +don the +dear th +de ve +cas us +b ili +alla habad +ag ni +t zu +produc tion +com bed +cha un +s ce +mem bran +don ner +bla der +darkhorse comics +c zyk +ayat ol +sacram ent +pere z +pe in +n ite +ken ya +intensi fy +eleg antly +auto pilot +rou ted +iu bb +celi ac +ste infeld +moc cas +ka ther +ex hale +dad dys +sta unch +spring board +pe gg +kirk man +cin nab +aor tic +ruth wignall +moder ately +lau ds +au ral +vagu ely +spread sheet +merri mack +da thletics +we stie +pesh merga +coo pers +tan trum +t cu +ga be +criminal minds +com ix +q ur +fi q +bor g +pue bla +nis mo +cy ano +bak h +ti g +tho tel +il ford +guadal ajara +inclu sivity +atom ic +ven tric +u el +mono gram +man kato +fu q +ðŁĺĤ ðŁ¤£ +stra d +plu mage +op ath +kom en +im al +che tta +bill ing +s vr +hu mmer +d lt +water cress +pil ar +human ist +her r +ha ig +cor ino +swan ky +spo tters +sn h +medi aday +under ground +ton bridge +roblo x +re take +men in +inno cent +ton g +lar amie +be gged +arti sta +uste des +san tho +lac ro +is ations +fox y +bio sphere +temp tations +stan lee +semin al +ini go +ãĥ ĭ + º +spod cast +o shi +lloy ds +gree ce +fore word +david tennant +brain tree +tam i +sam mie +predomin antly +coffee time +clut ches +acap ul +val halla +pro ck +mytho logical +meh met +idio tic +d su +ðŁİ Ĩ +puri fier +oc ca +ma f +hoo t +over joyed +ha ke +behavi ours +ðŁĴķ ⾨ +le ena +gau l +carne gi +u ws +repell ent +maxi me +kar lie +j iro +gla ser +fr ites +earth hour +de k +se ams +l ly +gilli am +davin ci +we sh +swe de +pe tra +p tm +my b +min ig +hon dac +al f +staf ford +plu shie +pal in +metast atic +go reng +flan ery +ro java +ani x +ðŁĶ Į +ap c +activ ates +vic i +tu ally +delici ous +aw ith +air base +stabili zation +solic ited +pic ky +im pin +fo p +af x +the man +the green +pee wee +mis smar +kitchen ware +gil man +scoun dre +kra use +j nj +hend ry +eng r +dar lene +mal don +fam icom +mi er +di su +breakfa sts +اÙĦ ع +ver gne +sunshine coast +h ed +di vest +cri mean +v fc +su le +sam ay +sloven ian +is bn +borac ay +tu bby +sp are +reas suring +mu tt +intram ural +at b +aaa and +âĻ Ľ +o ge +fi u +caer philly +string ent +steel er +n pl +macadam ia +indi go +grow l +gimm ick +españ ol +z ü +th alia +soul mates +more au +cri stin +anxi ously +vod acom +pam pered +lo z +content ment +pharmac o +n sp +mon ika +mika el +mac d +da o +sw ill +sheep dog +yo st +seiz es +off shore +nor walk +lo scab +bo sque +r acked +parri kar +na st +lo tti +bi j +Ð · +tb thursday +question naire +mercen aries +evo ke +di sal +al tru +what ever +str ö +san jose +man tel +in convenient +uaap season +tow no +spec k +al bright +the girl +ha shi +em oun +dow ney +desp ise +amuse um +sm elled +rugby union +over turn +ak ron +son ora +pau ly +mont ague +min ar +mill i +suzu ka +seo hyun +portra ys +ou mi +mo hit +lor dof +ayles bury +appropri ations +en act +conclu sive +chim ic +rasc als +j rotc +ind aba +ma iler +sun dry +subjec tive +stu mps +secre tive +red blacks +qu asi +oni ze +cany ons +nav a +lor ne +for de +cocon ut +us at +shear ing +sai yan +prat ap +modu lation +flann ery +gol ding +di da +ag ong +patho gens +k ye +dow ling +chi ara +sadi q +ol ander +journ alistic +croche ted +assu mes +ai ba +s vet +re visions +ph at +elis sakh +sur passing +person ified +mon ta +in ar +wil fried +âĹ ı +x ler +ta kam +out done +hall ways +gow da +fam ou +chop sticks +antiqu ity +% )... +nak uru +forsa ken +role play +kan ban +fly in +dock ers +con jun +ìĹ ° +swee tener +sin es +re kha +imagin ed +h su +der ailed +mano j +ji mi +acab ello +âĿĹï¸ı âĿĹï¸ı +stre atham +parad ox +ma stop +like aboss +leisu rely +iti ger +faire r +er cy +ðŁĮ ¾ +ma sch +lun e +la sd +historical fiction +bol t +, ( +ta pi +jor d +d na +cant be +alder man +to sses +or deal +nur burgring +kin gh +du os +de ze +clin ches +nap avalley +indispen sable +heath ro +fro ma +dece it +ba ira +ws ers +jack al +ch ula +bal ay +sur bhic +sal at +lolo lolol +had don +fear twd +scar lets +fanta stically +eredi visie +conting ency +seven oaks +me tra +futur ama +ce phal +vi kes +termin ate +sequ els +chipmun k +val ky +stu la +se vents +lo dges +jess i +j bl +gar rett +es es +con ferred +certific ations +anter ior +te ague +ha da +âĸ ¸ +win less +ji b +grit te +ga el +z d +un covering +si mr +roy ce +gwyne th +goo ooo +conquer or +ald rich +vin ny +re generative +propag ation +cri stal +correc tness +wh m +un lv +so w +net ted +k haz +an ise +plough ing +million th +em ac +ec ki +yar ns +impo sible +good byes +ff h +fel ice +ero a +du omo +du ll +barri ster +v ity +st irs +shu gden +se per +re discover +mas sey +al fal +over ture +mole skine +ma ka +li ppers +jay len +gron k +fel dt +puni shing +gary vee +dim ple +briga dier +happy sunday +freel ancers +cajun s +bloss oming +scar let +s ities +ph en +one ida +ken nels +inadvert ently +hu sky +fi fi +bj d +?! ?! +ur chin +to s +ste pp +au tor +ãĥ ¢ +~~~~ ~~~~ +wid nes +she re +just icia +fant as +el aw +Ì Ħ +wi steria +u q +shar kn +dragon flies +al dean +go b +di bs +bo rer +petron as +doub ting +chap ar +shil oh +gabri ele +cor pu +taka shi +selfie day +pe tru +harmon ies +u ck +sty x +k ine +high gate +gang sters +ga pol +desi ree +over comes +mc cas +hersh ey +he ston +duck worth +das gupta +zam ora +har rell +am ble +visu alizing +uc u +su zie +mil dred +lu t +d wi +civil izations +preten tious +obl iter +mil len +girl boss +% : +win field +rever ber +mat ta +in ery +cro ke +ch lan +bep anna +ðŁį Į +ðŁĩºðŁĩ¸ # +squ a +se id +la ve +gh is +be headed +emer son +awk wardly +ar h +à Ł +w mu +tw i +shiva ji +on of +bun kers +tar aji +pal ace +i ang +de crimin +whit lock +wal eed +ax well +viny ls +kan al +exc ise +don ington +carl sberg +under served +solu tion +portra iture +ke urig +incorpor ates +민 íĺ¸ +velo drome +pe tta +len eck +for gott +fe fe +eu c +art book +through put +share pict +pe eing +high bury +dog strust +ðŁij IJ +w wi +m lax +dd in +con de +affordable housing +ðŁij ¿ +t fl +sho aib +ne ues +n de +mor in +flu tter +fi ver +separ able +kenne saw +irish times +ink tober +fantasy art +ed sa +shruti haasan +sc ous +cro pping +abu sh +y c +tem plar +motor cycle +gab ba +ar ic +aldub nation +af r +whispe red +va x +pollu ting +head ers +great awakening +eer ily +bal my +ida e +bon es +a fu +shi res +f ww +uni fy +press sec +consign ment +ma ken +terr ance +swe ar +prag matic +night fall +jor dans +ic han +ta hir +ma dal +g eller +alber thall +social selling +recur rent +pho ton +organi zes +fight back +evacu ations +du ci +chicag opd +bird life +ðŁĺ« ðŁĺ« +tu mul +cam pi +ðŁį Ħ +sin ski +dissol ved +an ch +yon der +rs na +river view +pul sar +pro pon +multil ingual +kab ali +be agle +air canada +sor ghum +fron trun +la val +im mortal +epi phone +de preci +cor dial +bet fair +ìĹij ìĬ¤ +ti us +ombud sman +kar p +gy psy +autom ata +you sse +tam pering +red birds +kar in +trail head +legend ary +fi shed +es l +steph ane +hill sboro +her cu +camil acabello +cam ellia +al ts +adju dic +se kar +scaff old +mel aka +ill iterate +ha sle +fa thom +campeon ato +v logger +spi zza +mccur dy +ine ed +dev ito +team em +sun nies +morning side +elec ting +ãħ ¤ +bad die +aksh mi +sport sp +program matic +ge eta +fla unt +? ). +zu ela +sun ku +spider verse +shreya ghoshal +sh ams +benevol ent +spo int +jee zy +equ ine +dunkin donuts +comp troller +arto fli +tre v +pap e +ohmy god +bou ts +bik elife +mo te +g cs +acce s +y art +ut austin +trans lators +tab or +sho wered +andre as +ak l +women who +ãģªãģĨ ãģ·ãĤĮ +ram zan +lake front +du cky +civil rights +ë l +team sters +str is +shat tering +char lot +bag an +alo a +aaaa aa +mo se +mar ce +gun powder +ge isha +ðŁĺ µ +per mac +elo pe +chri sto +april fools +wolfen stein +there of +tar n +taka hashi +poly gon +lu mix +independ ents +hier arch +fc fans +wc q +law ay +av ro +at one +tru ex +s ro +r ur +mt l +i ft +defi antly +ch itt +r vp +o bel +k mart +ela ine +bed ford +se men +sab out +bathro om +ðŁĴķ @ +khy ber +i will +business woman +tar as +param ilit +mer sal +chor lton +ak ram +w dc +universi dad +ja vier +j ara +gil as +contracep tion +cat aw +c nd +bu co +au ri +ton ight +ma vote +i fi +albat ross +vegan ism +tw ich +lu d +le ases +de ben +worsen ing +qu ia +power less +end ra +ðŁı¾ âĢįâĻĢï¸ı +ا٠ħ +ventu recap +su do +nom ad +indiffe rence +ge tup +explo ren +bay ley +u gg +pad ra +orange county +co bra +tic ked +ti ss +shrun k +k sb +ic m +gett es +aberystwy th +de ars +vas an +tab asco +life mag +fin lay +tam iz +poon am +plat former +mu sco +joy ful +it b +gol ang +ga stro +enrich ing +eli ke +p wc +kelly anne +jama ic +hal lam +bri ar +well ing +wat s +vo icing +t tering +ra vel +pa wards +mu zz +h tm +alig ning +wedding wednesday +dri fts +rolls royce +multic olor +luci o +han son +f gcu +sound garden +pan cetta +oc tober +free masonry +boun tiful +bigbang theory +behe moth +º ï¸ı +sne eze +saat chi +raw at +mobili ze +mo he +fur ther +dy bala +boli var +tan o +ghu lam +femini st +bo f +ben dy +ant in +we is +t sar +fav ours +fab regas +sh ang +pro biotic +ad mit +sol der +mo vable +dra dio +cyclo cross +australi aday +âļ Ĵ +z at +chi m +si reland +s mb +air fare +Ú ¾ +me politics +mal com +kop f +k all +chi pper +ban que +ðŁĻ ī +sol ver +in sinu +avent ador +zero waste +hell cat +universal orl +pu so +pal au +gav a +e intra +d je +arunach al +vi u +the ad +sep tic +santac ruz +mor gen +guil ford +ber tol +b dutt +world poetryday +sc m +ple thora +london traffic +cre ma +t iness +si em +el ated +before hand +b q +aw ada +nag arjuna +ev y +ðŁĮ ı +lu cha +call in +se aco +retri eval +kun al +car on +reserv a +negoti able +lang a +do fe +cab al +ani k +ðŁļ § +ðŁĺĬ ðŁĺĺ +sub station +no x +long ford +gad dafi +dic a +zi oni +mitochondri al +it on +in sp +be tray +va in +tr ts +pen di +oppre ssive +long wood +limous ine +iz umi +green light +go ggle +pl t +to kio +long beach +gene ss +visual art +s ball +pacific a +kir by +itu dinal +i den +foo ting +en sues +dit ching +sunku writer +sig nor +m news +l tr +b sn +whi de +ver des +taylor nation +sen n +n spoli +match box +mac gy +chro mato +íķij íģ¬ +q in +mo ca +la gging +b ander +sm r +red woods +de gas +ti ered +ma ko +kno tts +ble sses +bella gio +barunsob ti +ad t +sc rape +p mp +no y +mu har +whit stable +man sell +dri bble +n di +mun cie +ho yt +fiber glass +alo u +thing si +th au +sanay airani +s forever +rale igh +mccol lum +but tered +bang in +shab azz +cho ir +pension er +n ando +mu sa +kra u +intoler ant +h mcs +ka jol +ch ale +block ers +bi el +v su +sh ona +gh ol +decor ator +chi ppy +~ # +wel lies +sub hash +ou ld +oc r +o ann +ken wood +c bus +ton ing +ruck us +ro ca +pi b +op aque +for bid +britishar my +sens ation +lo bla +inde b +conne cts +ðŁİ ® +next level +jaz zy +e fe +wh l +im mortality +gym life +dopam ine +arter ies +ts g +steeple chase +resurrec ted +ka izer +hotty toddy +dg b +dal ry +attach ments +wind mills +ul cer +share ef +oy ce +n tc +loan ed +ani ka +ag r +west coast +sha ver +bolog nese +su ez +photo card +mi splac +limb augh +barbic ancentre +accor dance +ze bras +thalapathy vijay +men style +final fantasy +app ing +angel ic +raf fa +hitch in +he ct +bre da +blind spot +pre eti +mentalhealth day +dae hyun +bore rs +ãģ Ł +talk radio +Ú º +itiger shroff +end on +ra pp +ph onics +tar a +rand paul +face ts +art ful +ar anda +âĺ ĺ +stor mb +sau cony +member ships +al ur +treach erous +tis the +stan sted +re work +pre emp +hol la +gi anna +beauty andthe +âļ«ï¸ı âļªï¸ı +surbhic hand +health forall +anatom ical +ve c +trans national +or la +en n +ak ali +to tti +car mine +sub mits +projec tors +gu ten +cruz crew +sc ele +ken yan +conce de +! ðŁĶ¥ +saturday thoughts +ru k +ec er +win ing +pic mix +li mon +lau gha +iti e +fa un +bru cele +ठŁ +vocal oid +t news +miser ably +em brace +don kiss +de position +clever ly +thic ke +sky dive +play bill +her t +door i +de letes +bo asting +analy tica +sh su +pro sthe +f bc +du arte +c wa +bad as +in visible +geo logist +ec t +cas settes +ba ir +revolu tions +off end +nutriti onist +line men +dele vingne +si da +man chu +ji an +bli mey +ar gan +wildlife mag +snow drops +pre cursor +o co +neapol itan +it su +birth stone +amate urs +pin ks +ley ton +gram my +giac omo +sei u +second hand +out cast +lon er +b ites +ðŁĺŃ âĿ¤ +è Ĭ +un cg +slimming world +si o +shin ji +ou ch +kan te +ðŁİī âĿ¤ï¸ı +pe k +hu zzah +de kar +belt way +ab und +ðŁ¤Ļ ðŁı¼ +cri m +caregi ving +ara jan +q pr +fon seca +daw son +bu li +alter cation +we the +rawal pindi +messi er +je mima +den ounce +debu ssy +chiroprac tor +antiqu ities +wa hl +un packed +tri athlete +cl iche +super card +re tour +re petition +re actors +lead ers +hol lie +fa z +bad o +ta os +raj avi +multi verse +aj al +ìĹ ¬ +uof l +o ha +ka c +pd l +baro ssa +ari k +ðŁĺ¢ðŁĺ¢ ðŁĺ¢ +met calf +ciu dad +chi yaan +ash lyn +am ity +way nes +pa wn +ox nard +ìĬ¤ íĬ¸ë +es b +dod son +âĺºï¸ı âĿ¤ï¸ı +is che +ev ry +bre m +to en +pronoun s +graff iti +flat bush +che p +pig let +it ye +empha size +c bee +á Ī +tele graph +sing am +or dn +mg m +kno tted +ale igh +ranbir kapoor +mar ko +lic orice +ax ia +arti e +ðŁij ¥ +pu m +plun ging +ore illy +gastro enter +cis o +bombar d +bas qui +speci alize +nr m +nak hon +fre sco +am ity +aero bic +sin ow +re produc +pri mor +straight en +bound less +bone less +stampe ders +re tweet +quint ana +luci en +kri eger +jam z +dou che +audi ophile +abol ition +to ho +tab a +se gal +sch ö +s gov +m td +len ox +cru tch +ðŁij ł +platy pus +ing old +re think +kh attar +dri ft +apaaja iklan +ancho vy +tri este +lu mp +quen ch +on it +gill ingham +gal van +com miss +mi sty +col ton +bob cat +white horse +ric ha +pu ke +perpetr ators +no b +kt la +ira s +g wa +d pd +co tti +teas er +olym pu +franç ois +sh amed +oul ding +offici ald +inaugur ate +der went +con formity +sil vers +is ka +gat ineau +ce ce +any thing +splend our +pin tura +on am +dur bin +d gs +vali dated +se go +par ra +last fm +ka in +h cm +ak ar +ab ram +learn to +ar ke +_ ; +som mer +she amus +kam ara +flir ty +ðŁIJ ¬ +on ice +hu ma +el bows +conce al +col leg +acqu itted +tax ed +mc cau +health tips +defend ant +chee tahs +business men +re h +mat y +andro s +the musical +ob ar +ge mm +dalmati an +wob ble +spho tography +prairi es +ma dge +kail ash +fun der +datab reach +tender ness +sper ry +mc cu +debu gging +ko pp +jame is +diss atis +n cl +logan o +h mas +e ren +ac os +tumb le +natl parkservice +cross fire +bat mobile +ag iri +war ne +vi my +supervis ing +ren ko +pool ing +ma wr +dock yard +nar o +multi plex +exc l +conten tious +ab bi +好ãģį ãģª +twitter less +obli vious +kore ans +fre ne +tul si +hi ma +pasqu ale +oc clu +o key +go dof +dan ilo +ar de +ðŁ¤ § +nn nn +fur y +fi ka +ber sama +wise man +tru c +sh em +q atari +ox fam +as lan +soci ety +ri dges +mur ky +frederick sburg +f k +char ly +buck nell +atta ined +than am +mez cal +hef fron +w jc +sty lists +daffo dil +wend ys +r tz +famili es +crack le +tun nel +nit z +mur ph +ad avis +esp rit +win nin +ki pling +jer o +young blood +ru z +paranor mal +pantom ime +mat lock +inst am +gall ardo +f wy +ca en +vel le +kut i +fur thest +com ings +av b +shu dder +north am +: ^) +ðŁĴ § +swe dish +prettylittle liars +braz os +r ations +pizz a +go gators +fire wood +esto y +ad hi +lec lerc +ice bucketchallenge +hippo drome +bloodh ound +ad gpi +suc cin +sub committee +moris sette +the view +show ering +j awa +a ic +tro m +surbhichand na +ro din +restra ined +ram bl +mo bs +cubic le +then y +park our +he ute +ep c +bit coins +auc tioning +mal a +gi gem +concentr ating +er nie +bun e +tran sports +ki shore +har aju +ðŁĵ © +sweat shirts +pol yu +need ham +nc te +kh loe +fire safety +er ian +dri fter +deta chable +woof wednesday +tric ol +shu ps +dise ase +u min +story book +start ling +sg f +ma kan +ðŁIJ Ķ +agend as +hi it +dispat ched +synchron ized +shu man +radic al +pu tt +pre cure +go fficial +de code +vi ans +vi ability +v sp +tam ales +pra bhu +snor kel +eas th +bl under +: (( +wai ved +ide al +e ury +americ a +t dy +shock ingly +fran z +eric s +ce f +bal ear +ren zo +ko enig +c bbc +biome trics +suffe rers +su ch +smo kies +mur u +k hawa +i im +gha zi +íĶ Ħë +white tail +un chained +thenotori ou +sh ino +ken obi +cour gette +clint ons +al ala +sexi er +never stop +ne gros +ne ca +x cx +song book +ren ding +cal ms +amar u +ag tech +ãģ Ń +qui dd +new years +mar gher +eye ball +ati er +vivekan anda +somer set +rin ce +mu kh +ho h +col er +bu kas +⼠µï¸ı +tu cking +pi ggy +iban ez +ho skins +decep tive +click bait +bu le +world view +woo ster +wo td +stin king +dam i +pau lina +fel on +cross body +wb tv +sub han +lon zo +flat iron +burn side +win throp +tallade ga +sp angled +sf p +ro wan +real ises +wash burn +ÙĬ ÙĨ +ston ec +emp tied +ci ren +cha ise +am bu +. !!! +se tbacks +sad dam +con naught +av enge +af fluent +u ob +that ch +swin ton +sat ya +es z +equ ity +re invented +kas per +c fa +aesthe tically +mi ku +mar cy +fin anced +look up +ecoun ty +ðŁĺ º +t dih +ro pe +me ch +dy ing +book seller +aa sh +vel oci +o vi +im m +feat ured +nu g +fun g +ell sworth +sett ler +say o +mu zzle +su omi +ragn ar +mentalhealth awarenessweek +maastric ht +il in +gl und +ak ar +intellectu als +flor al +brack en +ti ps +sub ver +se duce +scu deri +nev ad +je eps +jaw an +scar red +med school +ec p +catap ult +additi ve +sm itten +q d +lock ers +like agirl +keral afloo +bub bling +ari ze +ðŁİīðŁİ Ĥ +ìŀ ¬ +ted talks +rhschel sea +pu y +ok ra +logo design +hen g +hammer head +dri bbles +gan ja +for ds +cou scous +ar gen +ë³ ´ +van ish +ne tapp +my love +anch oring +ç Ļ + ¨ +por ous +over seeing +musi k +ma gen +dar nell +r ha +per o +land a +jurassic park +fre o +bron ze +ãĥĥ ãĥ +ðŁĶ ¸ +ðŁĴIJ ðŁĴIJ +wa ch +tri gger +eng inak +d bl +ðŁĺĩ ðŁĺĩ +speake asy +solid works +sheh baz +pu sher +p ty +fat loss +discre te +di onne +ch iller +applau ds +u mp +ra staf +neg atives +macar on +islamic state +cap tion +anti aging +pember ton +long ong +issu s +res ounding +offen ses +new balance +n ley +mont auk +mc ga +dispos ition +purpo sely +ir anians +ðŁİ » +fu lani +corrug ated +ðŁĩ³ðŁĩ ¿ +ston ia +par snip +jam ison +ge is +ðŁĶ Ħ +re claiming +pleas ant +on boarding +edou ard +a ah +swee per +nu nez +mu dd +holo lens +chee z +brigh tened +âĿ ĵ +wo wed +sch ko +nis d +co ffe +ba hama +auck land +super mari +oun cing +op ting +mc clu +âĢ¦ # +sant as +sli brary +revit alize +qu ai +men acing +kkkkkkkk kkkkkkkk +der ulo +scre ech +ko enig +crowd fire +bravo tv +ay ee +ar kar +si mi +me era +jiha dists +je we +bu ss +ari ane +- , +ਠ¾ +n su +ðŁĶµ ðŁĶ´ +fi bre +ar ched +à ¥ +t so +re my +light foot +far han +embarra ss +bro derick +breath ofthewild +as ino +superst ition +new ry +mer ck +kip lier +ab ag +van g +pic to +li f +bag pipes +at ru +royal alberthall +movie review +lil ley +ju t +bang bang +. ). +grass lands +flower report +chat sworth +aam ir +syn tax +pro bing +nomanss ky +lau der +we tting +so ta +rappler dotcom +photovolta ic +pharmac ology +luc ca +le gging +gumb all +full back +dece ive +sop ranos +s bar +ru pert +com bi +clar ks +billi es +alle gro +m ce +dam an +chicago bears +cas as +vap elife +mal in +byo b +âĹ ¾ +rac er +mv p +memor ize +jiha dist +ing life +com ber +ar tex +applic ant +je a +in former +ho xton +hen ning +h ls +ðŁĩ© ðŁĩª +p q +in london +ilay athal +âľ į +sciss or +sch amp +li able +stra ining +pu ra +mon kees +let ch +kom pany +by design +mo dul +har dened +brecken ridge +wol longong +tri er +man ate +lyn ching +con cur +c sf +wood ard +ol ab +l st +jamie oliver +insur gent +wre cker +or mond +kim ball +sn ic +s ere +mal ar +gar ages +fel a +fa de +pastr ami +ic rc +hor atio +cle aver +ab be +wwer ollins +privati sation +nature guide +hol me +h eng +esk imo +may noo +lever kusen +ax l +sk sk +n our +fi do +famil le +dis i +br inger +age less +x g +pi an +path ak +ab domen +tri mble +in ns +idw publishing +focu ssed +ei ght +mandel aday +fa ve +da ire +bul ance +u mmmm +pe res +tomo da +pp ed +may all +ler ner +elder flower +bar nar +transp lan +mausole um +fier ce +alleg ing +neti zens +ky ler +il de +gam b +e as +lit es +go er +bur u +alice in +ðŁ¤· ðŁı»âĢįâĻĤï¸ı +ti pple +rupauls dragrace +pee ks +inter twin +æ ķ +y j +shan ia +techno logy +nba on +mul tan +motor head +lu kes +ken zo +mccre ery +er te +dra s +blo kes +ber nal +apple by +south carolina +new borns +tw ir +spartan burg +o cala +l ü +dwy ane +bra va +ace h +à¸ Ĺ +sv s +sula wesi +stoke city +shar ks +fo a +anti depress +ðŁĩ¬ðŁĩ Ń +sidel ined +shu l +seren geti +ll lll +kab ir +bout a +bi zzle +bam ba +que ttes +nb cla +mj hl +mg mavote +mac chi +cag r +ale ah +ðŁĺ Ĺ +whi plash +tim i +pal mas +domingu ez + · +il on +hil arity +ru a +organi st +mit ts +gas ket +ðŁıĢ ðŁıĢ +ten sor +steep le +smy rna +rand o +r ma +pl enti +fo go +aph one +ðŁijĩ ðŁı¾ +ðŁį ı +th ad +re sor +petri fied +di xi +ti gan +jalap eno +deser ve +sav age +mad sen +gre mlin +for women +depend able +conve ction +bo sc +yel yah +watch mixer +ìĺ ¤ +z ell +tur ku +soul ja +she et +sel i +ma kas +k league +an go +ak al +........ ...... +te tsu +i heart +her bst +cyber attacks +sum ter +rijks museum +raj ya +ar ba +rock ingham +mus sel +micro scopic +ke babs +cand ice +get fit +adam ant +we ing +sa shab +h gv +emp tiness +cur ation +brit natureguide +um n +ra fi +an er +viscer al +up trend +um pires +ts now +out last +le sh +ih saa +shal lots +sco pe +dan za +c vc +bac ardi +air brush +ae gis +ðŁ¥ Ĭ +tru sh +scha efer +resist bot +fior ina +bran ch +whit efish +ru l +con spic +ar ig +twe aking +tub man +ste tson +robber ies +iso l +em m +condem nation +cast ing +aud its +vish al +vand alized +oc i +gi m +work place +van buuren +nig ella +mis guided +cas cad +after ward +:- * +sub tly +car rick +ple ading +original art +omnic hannel +nancy pelosi +great lakes +glimp ses +ent ino +down right +arter ial +ðŁIJ ķ +maxi mizing +er acing +cy te +chur ros +stur gis +microbio ta +mass ac +kun al +ku b +h ny +blin ding +articul ated +an es +piers morgan +ker alab +ho cken +coles law +gas sed +d md +ðŁıģ ðŁıģ +nor bert +mi i +long ines +go de +del ray +carval ho +bou quets +bat ty +bake well +st oop +sare es +pug life +kau ffman +g ds +free speech +cul pr +basqui at +pan dian +g ws +do glover +den ces +beach y +wan ing +press freedom +home made +con stric +bhu mi +bb all +t iling +popular ly +accol ade +tar ra +sta ve +kardashi ans +jac lyn +ic ed +endange red +art finder +ðŁİ ¹ +vanity fair +tr ill +psychop ath +multi functional +lou p +jag ged +gr ama +sanctu ary +her schel +et ch +capac ities +z ora +slu ms +jad ine +bag ga +ani m +sp ress +hand some +cape town +by ers +w sc +sh reds +miyaz aki +iti st +coll ard +è ² +sal ina +pac ino +nune aton +jan ella +grass ley +ৠį +p eli +don line +comfort food +c ÃŃa +so ba +fl ys +spay ed +med ve +hol man +ãĤ Į +stef anie +nais mith +home girl +g bb +bul k +au ce +afternoon tea +ac ular +su iting +spot less +sky lar +shop kins +j ona +clo cking +car cass +bo gey +be ig +bac chus +ari es +ad k +ðŁĺ¬ ðŁĺ¬ +upro o +tri a +hom ing +han n +el vira +cdc gov +br ind +al en +wn c +with ers +ro ast +os mond +ke ele +e bs +marke ted +imperfec tions +en cin +e sses +c zar +worth iness +watch man +me ena +man ali +kat ow +historyof painting +edit or +° . +rosen stein +itsen rique +dal hou +begg ars +sym metrical +surg ical +star a +st ent +so excited +sa ac +robert pattinson +pe dd +ker o +ç Ľ +thu mp +re tribu +powered by +ober oi +hybri d +grid lock +cd m +av al +ãģ § +thenotoriou smma +sub conscious +he id +go bruins +cy ani +bo den +uc sd +sar o +el c +de ley +bol den +spie gel +re made +n anda +jodi arias +h any +too o +salis bury +op ing +cab all +an gr +... * +to ggle +shorth air +sas sy +san am +fl on +apologe tics +ac cel +shirt day +ac me +ic t +hal es +danc in +co exist +ðŁļ ¢ +è µ +yelyah williams +straw berry +fc n +ce t +sanc tion +r fa +po tt +iz om +id t +; ;; +vol a +mess engers +cher i +sfor all +rhine stones +r ann +us k +sole mn +sen tai +retro fit +mait land +ac tus +ig loo +desc ends +u day +ic c +hu ck +endometri osis +elec ts +crab tree +rakul preet +jur ors +holy rood +feder ally +di az +de mil +wash ers +ver tically +untouch able +mal o +curtiss mith +cali bur +book keeping +ym o +x men +per mian +national theatre +n assi +lein ster +c ld +aw az +apolog ises +al ysis +son u +mi dge +inte ch +gy na +come th +band stand +viol ence +survey ors +n ms +es d +dr ing +clar ke +beat on +u bud +ons laught +ne vers +m sk +ee vee +con qui +bump in +bel ted +ac ris +c our +blo em +bien ven +v mi +da de +as ic +su vs +speci fy +gaz za +fe dex +voor hees +shr ines +# âĥ£ +ito hs +del co +cruci fi +bo h +ath os +âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ âłĢ +tri pathi +think tank +t ta +satur ation +mmm sie +bear cat +rick and +pa z +kand ar +jo ss +al ang +sab ina +ree ce +m kh +jin der +est elle +ell on +pellegr ino +o skar +im man +n ona +k go +engag ements +dest abili +bb g +y x +shiva ay +see saw +mo tu +ger hard +al armed +houston rockets +fin ally +dra vid +corpor ates +c ô +c mo +blood shed +ti h +strong man +sol an +sn ick +r ara +pau li +n ge +horse men +ch ism +? .... +.. ' +li via +issan ts +bc g +bar one +wat ters +val eria +optome tri +jess y +j hl +com be +be toor +ðŁ¤ ¢ +ut in +iggy azalea +grou ping +commun e +columb ine +af for +sa is +panch ayat +h ro +floyd mayweather +esplan ade +z vere +sony music +no a +i vey +d onal +cher son +c ack +betoor ourke +susque hanna +kak kar +don es +derail ment +compul sive +cardi b +ca e +tim ings +ap l +after hours +ac ting +un fore +transforming india +suppre ssed +sidd har +r sm +mah rez +in capable +green wich +misplac ed +love this +insi der +biomar kers +panini america +multiplic ation +ice breaker +discre et +chec kin +人 ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +vel cro +pre scriptions +hetero gene +dru dge +ìĬ Ī +Ø§Ø ¯ +van swar +tu pper +spar ade +m callister +e ko +ve ep +mar gi +ker k +kar a +dic ho +cos grove +val des +pu mas +off ending +k andy +hhhh hhhh +h pd +complex ities +car te +buf fo +k hun +ta char +in ky +bat es +at ms +syd ne +hri thi +der mal +ðŁĴ» : +ea sel +diss oci +bikin i +n sui +mon tes +mol loy +mo par +h di +dom a +ari ous +alphon se +âļĵ ï¸ı +wer der +uni x +seg mentation +micha l +lam beau +what the +thread ed +sa am +pfei ffer +fu sa +fr ack +aur us +te dious +nag el +ken an +island life +ge sh +cate red +bilt more +kam i +bul le +teamem mmmsie +t tu +sl b +newal bum +ma zing +gra phi +en vy +con g +we sson +sch il +gur ru +be de +aqu amarine +kand insky +emor y +den iz +ri sa +pul p +o cho +neuro surgery +le sions +h ons +big cat +sak ti +psycho sis +nsi tharaman +sw ard +le gu +fi ennes +se att +marketing tips +man groves +loop er +dh ya +quar tered +pri este +pres scon +ll amas +com elec +ri sd +r ine +pp r +dete cts +vival di +valle tta +fle sh +alfre sco +testim onies +quil ts +lat ency +k els +grun t +crimin ally +h tg +apo or +p ga +or m +ol ly +modi ji +hin ckley +na ve +n ong +heffron drive +gulf stream +gar rick +enti a +man mohan +iphone ography +flo ated +co en +c ally +armb and +te ton +tar te +ns wr +max ed +in ward +hydra ul +armin vanbuuren +hob son +creep in +re ins +kentucky derby +dream s +blou in +armaan malik +ab ana +.... ." +ten acity +ðŁı¼ âĢįâĻĢï¸ı +tam ing +oper atives +lec turers +cam us +áµ Ĵ +therap y +se dge +qu é +l ene +judi th +ac claim +on as +l ill +ben nett +sh atta +go dre +fle ury +e ath +posthum ously +pla ined +n ace +mor bid +mas ood +bac olo +mic ron +intercep tor +g acha +talk sport +requis ite +intru sion +dom es +brea the +affili ate +nyc marathon +house warming +blur b +si ren +ss an +mill on +gra inger +col by +campaig ned +kir sty +illu sion +f omo +c illa +armad illo +a better +t mc +soo young +sec tarian +rede mp +om p +chapp elle +and ar +ðŁIJ¶ ðŁIJ¶ +por tia +la pd +imit ating +do ers +cam b +bas alt +w sp +w los +tal es +lov atics +fat ah +sle eved +rand i +de foe +Ñ Ī +vo k +spraw ling +smo thered +kin ab +isu ppor +i wo +diff ering +al ine +scour ge +restra ining +kh j +joero gan +ed ina +chiyaan vikram +web ster +ty rell +take i +marting arri +j edward +e ke +dil wale +sur face +pu get +le sc +green tea +di xon +mi si +huar ache +cher y +aqu il +altern ating +my thic +lans downe +fil an +z ey +s att +o as +kerri gan +ty r +startrek discovery +ds man +bre ached +banc roft +ìĦ ¸ +lobb yists +lil ian +c ve +bull ard +bring the +st of +plane spotting +mit o +wak anda +mo wers +le la +had field +bouti que +ðŁĴ ¼ +s bc +lone star +disciple ship +qu akers +ecclesi ast +dead lift +c po +botan icals +ac al +stein way +je u +h cl +cru x +ë · +sh ani +palm beach +men non +du da +cho t +wn yc +cou pon +ca an +am f +ab users +ðŁĽ ij +ðŁĹ ³ +yu catan +w ird +tele medicine +ster oid +mayorof london +hy ay +sni pers +n andi +mo x +ga uri +xen ophobia +the arts +slo p +s val +llan elli +in consistent +do ki +demo ing +char lat +carl sen +bel lies +ìĸ ´ +ve ts +sen atorial +krit is +grun dy +golden knights +di vo +arch er +resi sted +connoisse ur +celebr ating +yousse f +par inee +de blasio +darren criss +ronal dinho +mt p +match play +entit lement +ðŁİ Ń +ภĭ +ple asee +men di +ev as +y una +that kevinsmith +red shirt +lin c +kung fu +epidemi ology +du z +sto ker +say er +mad huri +if ttt +gye om +fau lt +chan ukah +used cars +unimel b +la ha +eco logist +conserv atism +bar ro +art station +star citizen +spr outed +sho ved +shark tank +pro filed +jam i +hu xley +grote sque +be cc +ìł ľ +ภ¹ +ww t +work fromhome +ud ine +mar lowe +her bal +fur long +deb by +bou tta +ve dder +pri miti +mb t +e ia +dill on +akrish nan +wi ener +tun bridge +thy self +pav illion +om ggg +kevin hart +aw ry +tv news +si one +qu ds +n ita +loop hole +te chie +sab les +ber ing +worldocean sday +se g +pat tie +ne pale +indi o +bi anc +be ingh +air line +su ne +sj w +m wah +h ca +gre noble +gn c +council or +call a +weird ness +spo ken +sh ined +rotar act +om in +city life +vanswar pedtour +t ine +sp b +sof test +ram med +mentalhealth matters +gar ry +ex iled +adap table +smir noff +sedg wick +glori ously +bit strips +af an +tam er +q adi +origin ality +john kerry +es se +soire e +jo ggers +c gn +boo sie +se thro +le cht +in al +de generes +bog or +algori th +abo lished +scram bling +ici ones +hed ger +har ing +gen omes +bol locks +ram ble +bepanna ah +ðŁ¤Ķ ðŁ¤Ķ +the sp +t so +hof stra +stor ied +ol lege +jan os +gold wyn +donny pangilinan +ëĭ Ī +âĻ¡ âĻ¥ +yo w +sab ado +defen ces +ap ts +inter personal +el ynn +b ff +." ~ +un discovered +red deer +py ro +muhamma dali +lam on +kevin harvick +itu res +mol ds +just sarahg +irr itation +fre u +fort itude +du ality +archa ic +æ ¥ +sc loud +narcis sist +mu tiny +malign ant +du cho +culpr its +cross walk +berger on +back lit +ye sssss +tro l +sil ks +ran cher +nil sson +store front +sco ffee +pur o +fla herty +fa j +compen dium +car ds +si mu +mo sk +joe jonas +hand ker +y h +screen saver +ravi shing +hu mm +del mar +cro mer +cape cod +í Į +transi ent +taey ong +segreg ated +man ji +ki dd +jam il +cze cho +au ds +ãĥ ¯ +ma si +athle te +tu fted +tobac co +the l +bird land +transm it +thra sher +suit ably +seawol ves +ma so +lo vic +ing ford +communic ator +be gon +pr s +co ker +at ticus +tel co +stu bble +mp act +je anne +home schooling +est rogen +dt by +de hydration +com et +aper iti +work wear +tc p +pan t +men endez +air pods +tick led +me ws +may bach +li ar +inc iting +hal cy +fo m +fjor ds +wd su +saf ridi +produc er +out there +im ala +er b +butter scotch +ble tch +anc up +ãĤ ª +tan doori +shi d +p ds +ny x +insp ort +i fb +hydro gen +battle grounds +work s +meij er +mary ville +kal yan +cas sava +bo zeman +mat us +in human +ec ur +ðŁĮ µ +schoo lof +hispan ics +ga j +es qui +bt g +ac ing +pr amo +maer sk +ga iman +biza v +bir ders +whoo ping +vit ro +s ö +re telling +pal o +mar kiplier +hipp ies +cre ator +brom wich +ste ely +oo o +louth chat +nin ers +mil der +simon e +pl m +ho tt +devon shire +bon ny +victorias secret +the city +sch wei +pra bha +lil tunechi +inter galactic +cit ations +car thy +bi ow +vil lec +ut d +t st +shay ne +shakh tar +reson ates +per col +kat ana +asi ap +ak ki +shel ley +ke ston +jade ja +hutch ison +disp ers +bro mo +rai ding +o dy +n news +martingarri x +lu g +g lish +ver so +tan tal +om ag +o tak +free ing +yam in +un ser +multi family +haha ha +h sm +fi go +f ma +em bre +ab normal +nu ig +mall ya +d pa +bu i +ar no +amp shire +af fin +ab ook +pel ican +mee ks +heathro wairport +bhai jaan +ภĽ +st are +sar o +mathe son +mar ts +eucli d +w sc +seven ties +se cy +s not +motivational monday +mar que +karl sson +imit ate +if bb +houseof cards +ba sta +ðŁĩ²ðŁĩ ¾ +oc cul +na vel +manag h +ic her +ent icing +tw ente +trac ts +room ies +little big +el dor +humidi fier +depe che +su pp +si b +se ong +safridi official +nebra sk +make your +hiro shi +el khart +edi ble +du t +barrow man +balo ch +ude my +rwand an +me ts +footb alls +conun drum +ti u +p low +news stands +constell ations +ch n +lu han +khu shi +hope fuls +confe sses +ati ya +w ms +v ite +syn dro +shameless ly +khloe kardashian +hi sp +haban ero +descend ant +con scienti +black caps +ban dof +wad sworth +museu mo +ban king +anu rag +va ill +tele health +\ \ +w gc +v aqu +up cycling +k sl +aw ol +up cycle +nick names +diver se +centi pede +br indu +bur ying +bi gger +bedro ck +re solving +rang a +or icon +nikol ai +god in +excali bur +cur tin +chir anje +ab sa +wh irl +monday blogs +ll ang +bj ö +trip led +re imagining +lo ko +govern ment +craft smen +oste opor +lo bo +la vigne +grand view +v rin +v anna +s net +nomin al +ju ri +es m +cra dio +pr ingle +key chains +imagined ragons +ig ned +hill man +e ases +catch ment +ðŁĮ ª +transc end +qu ita +no sql +hav re +ðŁIJ £ +âľ ¿ +rani eri +por ta +yun nan +y ac +tam ale +ir t +gar gan +dis agreement +cy st +busine ssc +sten c +sm f +shino da +qu adri +off site +liter ate +chap ter +boun cer +asym metric +wi den +sch n +j han +ak wa +rheu mato +le de +in patient +he ide +chec ker +inf light +im pover +ha res +ayush man +ðŁı « +uter us +fly catcher +du ques +ka st +jahan gir +con vo +skin da +san sa +qu im +presu med +p ils +nbat v +mainst age +bri xham +s game +rho dod +qu ake +per ci +never hillary +love birds +loo kie +la vi +wes tham +pomer anian +ner o +montic ello +const itutes +warner bro +synth wave +nr w +fand ango +con d +grin dr +dé cor +cu h +come dies +bir kin +bap uji +smu dge +scru ffy +pan cakeday +ove se +ni d +li eve +laz iness +imple ments +ad ri +ðŁį ŀ +vi sts +ve u +risk ed +pro football +pless is +meso potam +ma ret +lu pa +koto ko +k ura +clin ic +am ends +state fb +goo ood +<< << +âĢ¢ Ì +th icc +mc do +hd fc +configu red +ck in +back ups +the mo +pol ska +insi sting +et su +sis coming +kin ect +conce iv +ar ry +go heels +vac ances +to sca +te sco +symboli zes +pnpp ro +palla vi +os born +ori ole +k sen +cro issants ++ $ +the man +li gn +jump in +hoo ligan +dictat ors +anal og +wai kato +ha vi +gis elle +fin ches +c di +ar at +tra shed +the academy +steel book +ove rest +home ward +gen ev +david son +ti bur +loo ker +brindu sab +tra shy +sl v +illustr ation +bread th +ba f +ri del +expre ssionist +co pic +clu s +ag chat +wiscon sin +sn ick +sh s +ricket ts +mlb network +han sel +dari en +chi val +wh u +sal as +phi pps +cor responding +chicago bulls +blat antly +bil a +bay watch +" :" +ìĿ ĺ +su mb +rous seau +p we +ed d +dam ning +benaz ir +bb mastop +unlea shing +hour glass +bur nie +buck les +ticho pra +tee thing +per ri +pen der +inf atu +he il +alum ni +ॠĪ +wh im +ver ge +newly weds +an ach +wo h +sj su +mi an +lom bok +j adi +ail ments +ft m +cro quet +blu ff +fa iz +chromo some +qu t +iti onist +ma dera +breastcancer awareness +b so +tra pper +tole do +o ys +fe ats +bt p +beli ve +a sey +ser t +bor i +æ Ń +tr ition +nun n +nbc thevoice +form ers +cav all +ðŁį µ +l ingham +hang zhou +we stand +inju res +gr rr +fer managh +cygn us +amster dam +t ns +spar row +ro logy +ray ner +pe onies +lu ton +huff ington +ha si +pri es +ev ol +ds l +. âģ£ +wins let +parinee tichopra +nur series +es ri +de mor +con texts +con rad +ðŁı» âĢįâĻĤï¸ı +sp rays +pres suri +don or +... ðŁĺĤ +gru b +der asach +ðŁĻ ĩ +zvere v +thi el +slo e +om w +kha di +ic hel +pun ters +f gs +commemor ated +brick ell +box eo +school house +on enote +lu men +l ye +ar ah +alex ei +ab ingdon +schol ast +magdal ene +for a +foot bridge +embo died +ble e +sm w +ren ton +mad havan +estim ating +son of +inthe world +ce ta +asau da +ঠ¿ +vue js +shar ad +sh unt +o val +local ity +first ly +de jav +whe elie +no zzle +no bu +han es +cu ban +aj ram +s radio +reen actment +play grounds +ordn ance +mu ggy +hor i +col ouri +b aka +vi ber +sle dge +ro si +off aly +im u +ende aring +concentr ations +ari th +ver me +south sea +sha ws +second life +re ac +mob i +la ff +exxon mobil +domestic violence +condol ence +cd g +bi i +ab cd +venturecap ital +thra shing +fox sports +ferra gamo +dang al +acapul co +ser rat +uphol stered +u gu +ro bs +play station +forwar ding +beautiful pakistan +x vg +tit us +su se +in sure +havas u +flam mable +ðŁĴĽðŁĴļ ðŁĴĻðŁĴľ +wh ine +tuc son +tame side +sc f +is so +afl cio +cal tech +theat lantic +taylor made +q ot +pp i +hy alur +hect are +de mir +su kho +scrap booking +sc ic +s sport +harmon izers +fol lies +che tti +med ellin +ken osha +hal ts +fuji film +b hd +epic enter +civil ity +te ac +rajam ouli +ho zier +summon ing +music news +laugh lin +friday thoughts +derasach asauda +cauca sian +z ha +total ing +sa rena +ratt lers +go se +by ul +b mc +ti st +seri ousness +kid dies +gre mlins +con testing +ë łĪ +z g +snapp y +pud sey +hor ton +ho ses +der ozan +sar ge +plastic ity +intercep ted +ðŁij ¬ +tre c +more lli +her ron +dj t +ðŁĴķðŁĴķ ðŁĴķðŁĴķ +year ning +j hu +hyacin th +che stra +ya w +sequ ential +ol ite +moo red +t assie +sop h +is brill +insec tic +fou ls +ab ook +sli ver +cripp led +transl ational +shock ers +she er +seman tic +mumbai police +accu ser +? - +the official +sam ara +jac into +fal ken +expo sures +car repair +amand a +ðŁļ Ķ +twee tup +til ted +ro phy +ske et +pamp anga +it take +eto bic +dess in +aa shi +us ga +paris attacks +ate ch +am ici +scrob bler +nintendo america +mol son +mag ne +haw es +ex pres +âļ ĸï¸ı +we got +scram bler +pra m +fic tional +elli eg +ðŁ§ ł +sw tor +quir k +karti k +s rock +ni er +land on +he dron +ber yl +^__ ^ +pin back +dar ling +c mon +and sons +al ca +severy thing +ram an +ra dy +permac ulture +be vin +see australia +man ga +kau shal +half term +fet ching +divyan ka +bureau cracy +al ena +stin i +sho vel +rho bh +raz ak +co schools +peril ofafrica +o choa +gi mp +facilit ators +blueli vesmatter +ah ly +adul ter +the art +revol ves +photogra phie +be happy +ahu e +s are +fc l +counsell or +bio gas +avi base +wh ys +v ad +santor um +les sen +don k +cover girl +bacolo d +ach en +-__ - +zir conia +roo p +brack nell +à± ģ +mis spelled +imperson ation +hand soff +( @_ +rou en +cl er +stabili ze +st t +jun aid +defibrill ator +she skinda +rox y +ra jar +pr ingles +over alls +jin ks +mchu gh +fra u +abig ail +ab adi +ro sco +re ims +ho shi +quig ley +pu rim +police uk +cu pping +aro v +a state +xero x +nz l +noctur ne +mortal kombat +clou dexpo +ain tree +hur lers +e ffing +bi athlon +al os +kin ky +hut cherson +bol l +wood bury +tart ar +sav o +q o +cou ghlin +civ ics +blogger stribe +ther oux +royal rumble +ni bbles +k ro +gar fun +west jet +track suit +syl van +sof ten +reg tech +goo oooo +bio graphies +barnsley isbrill +adam levine +ic f +guit arists +gal ing +cour tois +black hawk +ta gh +sa kes +religi ous +o er +an j +table ware +ru de +my first +mun itions +ah m +ðŁĩ«ðŁĩ ® +sli ppin +sharkn ado +gab y +early biz +ðŁı ¡ +sw ad +sorren to +koh ls +kend ra +hahahaha hahahaha +d mr +` ) +é ĸ +mel e +anten nas +work ings +i wa +ha fen +di ah +the k +prophe t +mc callum +m re +cripp ling +ate ment +ab omination +! (: +âĪ ŀ +world heritage +un reliable +t into +sho gun +que sta +ho tep +b po +al r +supple mental +mm f +it en +dor n +con current +arsen ic +martin is +cu sp +ðŁį ľ +za hid +is fun +as ahi +ðŁĨ ļ +wal kie +spo d +natural hair +blader unner +an se +it ory +infe station +gover ned +dic e +custo dian +sulli van +r ong +n dam +hi z +d ba +teen choice +sid harth +sh ami +magdal ena +john lennon +f nb +en rol +con form +unh inged +sp ay +flat ts +dar shan +to ver +si ang +one er +mo ga +lead ed +ef ur +din burgh +mezz anine +angeli que +e fl +ba ar +you ra +nbc washington +et u +disco vern +dimini shed +ten acious +precar ious +lo tu +kel e +j illo +gag reader +bre s +bal ding +u is +right now +richi e +euro maidan +dwar a +cur v +chann elling +ben zo +unreal engine +u shu +n mr +let ts +is r +fergu son +elev ations +dream works +tape red +ruff alo +pen ne +ful ton +down trend +depre ssive +actu al +vijaysethu pathi +th monthsary +fla p +de human +bol she +a sta +uchi ha +sha b +scen ic +pla gi +lan sbury +몬 ìĬ¤íĥĢ +v ri +un interrupted +sw ami +concre te +world mentalhealthday +work hard +tru ms +ser if +py on +os x +oh t +le dit +la gs +graci e +ðŁĻ ī +summer camp +karan patel +av p +ãĢ į +weather nation +the division +miser ables +liverpool fc +king sc +ju ba +holocau st +co eli +ade y +âľĮ âľĮ +un marked +swag gy +finger prints +yel lows +vo m +sm th +ri ser +on ge +no tions +vac y +tn wx +sh ala +nc state +leav eno +ec ke +dutch man +cor o +bang ed +te ver +rout inely +new schannel +hec tor +g mp +fo z +cor tina +w ce +su zy +motor spdwy +ma ye +zimbabwe an +sa ip +head ingley +glit tery +establi shes +es cotland +ander lecht +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +wood man +ri az +kritis anon +ko dak +ham lets +ha c +flee twood +antic or +z it +yar n +tu t +tin ashe +mand ed +dam m +d fl +comfor ter +bicy cli +u il +succe eds +pat ernity +no ds +cu sco +aband oning +white ley +weather man +shi h +marau ders +hilli ard +di rek +chor ale +ali c +ðŁİī ðŁİīðŁİīðŁİī +makeaw ish +maha thir +loch te +dro plets +cob ham +cle matis +besie ged +pan kaj +illustr ators +co burn +affl iction +travel o +ruff led +nag aland +doc trin +bul acan +aqu ap +me de +h sr +du bbing +cat z +ready to +fin alizing +e pping +defun ct +bonifac io +agu as +zo ic +taxider my +pra dhan +har ass +grad school +counter tops +clt motorspdwy +-- @ +secre taries +å ĩ +u pped +ser ap +pel t +id an +humili ated +ðŁİĦ ðŁİħ +ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ +psycho logists +mil le +extraterre strial +emirates facup +chatter jee +bre con +t gs +pan el +men ag +ig le +cb se +ron ic +guide book +bystand er +valu ing +nko tb +men sah +go cards +exten der +er bil +draw string +demo day +pho t +fe red +chic har +beth une +be m +mol y +loc ke +excur sions +data set +bre ds +ab aba +ðŁij £ +stab s +scrip ting +par m +paper less +mu zi +kra sno +cle ft +accu sation +mon oc +gre wal +e dia +cb x +cal ender +besti val +v ying +uph olds +sa oir +me tic +châ teau +alle gri +yo gy +tro is +mar ley +g fuel +english heritage +emb ed +counter top +ba chi +:- ) +Ø ¶ +r sac +mill eni +ko komo +deduc tible +am ia +yam aha +total ed +th war +p br +mor den +mo yne +k attu +cali ente +tra ve +th ayer +scoo t +mc crack +gu bati +gas s +fel ices +bac a +un fairly +trust the +sb m +sadiq khan +pri mates +gen i +/ " +wou lda +un well +rac quet +pa than +me wx +hol comb +hero academia +fro do +eng vind +bet fred +ãģ £ +west end +sch ke +rat at +les worth +kashmir is +juxta position +je annie +green day +g bs +f nd +temper ate +philadel phi +fé in +eye hinakhan +ate k +ðŁĺį ðŁĴĻ +v awx +j col +er d +aj u +adirond acks +y na +tra shers +super powers +re funds +ox on +mu rah +llan o +aw ana +sly ther +kh on +cast ello +blo ch +าภĻ +up t +illa warra +gram mat +eintra cht +as aram +sprink lers +las se +haul s +chry san +cas ks +ठª +woo jin +ther mo +oppos ites +le ttes +heath en +goal less +gi ga +esper anza +anzac day +ac ul + « +te sol +obas anjo +fac es +sw w +stat en +az io +shor n +radi on +maiden head +inspec ts +ri ordan +jen son +gb h +ek u +albi on +ta ffy +slu r +rit er +nouri shment +mot ley +life guards +frei burg +cent ro +bir u +bb ci +aj english +swee tt +per shing +new haven +ake up +abru zzo +the b +sig ur +remember ing +ram pal +qu eri +conven ed +braz ak +alco holism +ॠĤ +firec racker +dat emy +dar o +ðŁį Ń +junior bachchan +dis likes +af oot +t ff +i one +deer field +cory booker +pull back +mail bag +j emma +daily photo +cur ating +tourmal ine +pupp ete +pots dam +nis ar +madi gan +bas set +un intentionally +til apia +smo s +naji brazak +xy z +squ amish +produc thunt +ni z +ellieg oulding +ch r +è ¶ +un followed +heu er +wo ww +un prepared +pe els +moul ding +mar le +bellator mma +ar rays +z uk +social justice +ri ma +ent angled +absin the +⼠° +thro ats +theat r +sligh test +shame on +notmy president +humanright sday +. :) +ðŁĺ»ðŁĺ» ðŁĺ» +your world +testimoni als +f br +cumbri aweather +| | +spr outing +san gre +roa sts +respect fully +ma el +l x +kan ata +k ells +dam nit +spur geon +pitt man +k we +geopol itics +dec can +chri sc +venkate sh +ba ad +plun kett +led zeppelin +lang don +gilli gan +fur sday +bu gg +blogging gals +u lit +sab r +ivan kat +gl ene +cd f +am m +am ble +âľĬ ðŁı» +vote snp +star gaz +cma awards +alder shot +vv v +lent en +ax is +tt r +thi bo +sunny side +pulw ama +jan us +ðŁİ · +tru er +shel fie +pas ar +lowest oft +gu ac +go diva +extraordin arily +country man +view ership +mag ma +gen g +qu into +m fl +gh d +cr p +class ico +sho veling +se same +re at +ed ining +boyn ton +bea sties +west life +trous er +ter i +myo gi +min ced +inde struc +domin ican +ê ´ +t sai +son n +sm in +jeff gordon +cypri ot +cast ro +boy with +americ orps +ac al +ðŁĴ ¢ +pa e +i shi +gau rav +evangel ism +e ic +cur ate +ti sdale +synth pop +spaw ning +role playing +national ities +hormon al +re constructed +g ne +ed way +dom i +doc u +* : +shre w +seth our +out day +mor ia +ma ther +latt es +ka it +k ri +jolli bee +di or +defe ctive +al bin +adri ano +: ((( +u cha +serv ings +il los +green brier +dye ing +congr atz +moon stone +exi les +bell ini +additive manufacturing +tin ent +ster dam +sar kar +re lin +ov c +o del +hat ties +feel s +colla ges +as sign +nik kei +my r +minim ally +bel ting +Ø ³ +stat ely +mac pherson +lob ster +hus band +ca po +bigh orn +ç ão +ver tical +sten berg +locomo tives +stocki st +norman die +mo yer +homec are +hassel blad +dad dies +cal laway +ai x +tar te +s gs +pr une +ner os +mush y +mar go +bel ton +bag ley +ai e +youn is +tom cruise +swaz iland +samar itans +nat ur +mi ata +la paro +anti bacterial +u we +ori on +loscab os +loc ates +ig p +gi ff +plun ges +maj ldr +ha ba +fac ul +cran ky +air drie +ag ron +r fk +p vd +hey ward +dat eline +bo ko +t ma +sin ce +objec tion +harmon ious +gren ache +clash of +)) ))) +ym ents +si mb +ridd ell +rb ny +mur da +may hew +ton in +swal lowing +rand wick +ob c +ig non +f ann +begg ar +ab q +suppor ting +se go +plate let +l chf +Ø§Ø ³ +wi pers +web toon +out cry +bio l +urban decay +taran tula +na uru +megh alaya +medit ating +me ren +lett ings +hol born +ðŁĴĻ # +trish trashers +ry lan +n ne +mand ated +full ness +field trip +chi sel +buil dup +ty ra +made with +ha ile +forgott en +dan gote +women smar +ti mid +ski m +si kor +rig or +reig ate +pu tty +illu m +fat ale +bra sile +bass fishing +af a +âļ ĵ +su prise +n endor +hair dressing +cd l +be cks +bart ley +wit tle +tang a +l acked +fox business +ducho vny +day time +audu bon +think able +se marang +roman ces +north umb +nl cs +io e +bt sport +ste dd +pa b +shr oud +red line +pla ge +p ell +lip ton +achiev able +take over +ru ci +o vr +mide ast +jun tos +amo ah +ve tting +v eng +ti my +new shour +le ste +indu ce +hard waj +de se +ba idu +my cleanindia +leg alized +am monia +web by +un tuk +stone ware +ap id +sol sk +satis factory +head master +fulham fc +chi dam +bere tta +ðŁĹ » +kil len +early bird +away days +ni ve +narr ation +is b +eter nal +tylero akley +tri g +scoun tdown +ol en +myogi adityanath +indi atoday +f news +engul fed +th aa +subsequ ently +music app +constantin ople +sta hl +recu er +em m +u om +stone bwo +south wales +mi zu +joy stick +hydro electric +hat trick +vivo ree +ayr ton +ðŁĺħ ðŁĺħðŁĺħ +u sch +k ham +d proud +ðŁĩ®ðŁĩ ª +ton io +lal u +kil os +hel las +gle aming +face of +east coast +the truth +ston ers +r gv +jo liet +e spar +al cs +@ âĢ¦ +sh ingle +enchil adas +cast ile +bio fuels +am il +al pin +r ile +mu da +chri so +aw ad +to b +stor mont +mat tresses +hel o +hee led +dul lah +chom p +chic os +bis que +lovely z +gali lee +co va +vir k +subli minal +phosp horus +l mu +footb alling +drogh eda +cro cus +madhy apra +graci ously +gen ova +ex pos +cruiser weight +bi ken +af amily +accr ington +tt w +ted dies +spon taneously +som o +sla sh +ben et +afri que +vand al +un till +tor ius +stadi um +nnam di +migr ant +man na +ll b +kar oo +chi les +cave man +ðŁı³ï¸ıâĢį ðŁĮĪ +separati st +ron pa +pa cha +oper a +macau lay +frank fort +fr ills +ev ade +aud iting +theli on +par take +mck ellen +man is +ka yo +dee pak +cas sp +zam be +sunday brunch +ra sa +qui p +adhe rence +s wed +le mieux +stu mp +litt les +evalu ations +amu let +ðŁĺĬ ðŁĺį +n ch +ðŁĴ¤ ðŁĴ¤ +âĻ¥ï¸ı âĻ¥ï¸ı +were wolves +ste ers +scar face +par tied +de su +creepi est +controversi es +adri ft +su mer +sou p +ri go +let stalk +irrit ated +grou pp +carni vorous +autonom ous +au e +al pes +t fa +m gb +incan descent +glo ve +cant ando +tas man +sab re +liveon komo +kapam ilya +fang s +di lem +deb bi +bah ra +moha bb +g mg +g da +ke xp +bal an +ux bridge +t of +some things +keigh ley +embarrass yourbestfriend +cho ke +nab s +am mar +adjec tive +ðŁĴĺ ðŁĴĺðŁĴĺ +vol l +pin to +nhs england +krit i +it age +collec tor +black twitter +b more +ab and +sher i +north west +mtve ma +kel so +iz ard +bur gos +ãĤ ° +wet test +ma sti +i stan +tri al +th enight +purpose ful +off ical +bbmastop social +ar g +vent ured +vas co +male ficent +har k +barre tt +re adies +quantic o +jen ks +centr alized +ye m +un tapped +un m +n bas +ivankat rump +ingl ory +haare tz +ul cers +sky nyrd +ru ms +pre cast +md w +horticul tural +geel ong +egg nog +cataly sts +y all +woo ooo +to bo +shru gs +ev in +ser mons +nau tica +it in +emb a +coloni al +bow er +blin king +bbcc in +thin ning +stu mped +sh awar +psycho therapy +o ssa +dolce gabbana +bra zen +: . +stur m +ribe iro +nbc days +zz zzz +wozni acki +with love +mag ick +id l +func tion +car li +ai ya +sp its +sn fl +os m +mo ya +hi jack +great britain +a vey +âĸ¬âĸ¬ âĸ¬âĸ¬ +u ea +stom y +quidd itch +pine apples +spoon ie +sch rader +ram blers +knuck le +gra ze +durand uran +d har +âĻ¥âĻ¥ âĻ¥âĻ¥ +patron age +nieu ws +mee ster +ij n +i is +construc ts +ðŁį ¯ +taap see +death ly +back door +aero sol +wh c +t ss +of honor +bring it +athe dral +ate c +ðŁĮ ķ +v us +tokio hotel +speck led +scon i +sa under +ra be +fairy tales +e is +av ers +ab rupt +ðŁĶ ŀ +umb c +su ren +pfi zer +love yourself +in uk +ger son +en ish +the archers +te pe +solom on +sign ite +s new +rav aged +ra ul +hon ky +ci b +chester ton +tv d +neu tro +n lt +musth ave +lu vs +han lon +coinci dentally +æ ² +projec ting +h sa +digiti zed +di min +chilli wack +kick sonfire +id ad +haraju ku +du eling +discre tion +ten ny +progno sis +pitch fork +le vee +d hy +co ven +co pic +san disk +ilook like +be sar +ar ind +try on +nor way +levit t +eun ice +w pa +scan me +quin n +met z +land au +in wood +er to +cruis ers +craw led +chap in +car nit +angel is +fl an +chel t +bri l +na in +integr ative +here sy +d app +bn pp +ut k +stam os +sco de +pen ta +name less +ka is +in elli +ill ating +sa ina +renov ating +nut anix +grand child +bo keh +bat ch +b ure +approxim ate +몬ìĬ¤íĥĢ ìĹijìĬ¤ +zam bian +fallout boy +atl traffic +un mistak +o ink +je k +ik amal +emin ence +wor ding +unimagin able +mock ery +hy man +hand er +go onies +franch ises +collabor ates +she ik +immuni zation +fre es +ayatol lah +as on +un abridged +rec iting +jen winget +du ly +& âĢ¦ +stra pless +han ey +chev alier +ber th +ansel m +acet ate +water park +vio let +s mann +s illi +of t +movi enight +do reen +collabor atively +ìŀ IJ +un confirmed +rubi k +ru di +ny knicks +longe xposure +k ur +vitam in +tra x +megapix el +lat robe +in deli +hoo oo +dream hack +dive st +deng an +cover up +comb ing +colum bu +wil kerson +lo la +flu shed +fi gue +dou in +contin ental +capit alize +baj wa +wind power +sha e +se asi +plan ks +pi i +n cbn +extin ction +ÄŁ an +tot p +rex po +oc tu +mo k +clo t +pick ford +osteopor osis +m alian +intelli gent +dimen sion +beetle juice +abre u +yo jana +touri sme +scat ter +ro per +pue de +mar tell +he sse +z ags +ta ch +sen schumer +montre al +cou ghs +ab usa +willi an +sur in +stain ed +north wood +lil ith +gun ner +ab ay +sen der +corp ses +u go +house gop +stro m +li ddell +ki ki +dir k +( {} +rela y +ma ire +cray fish +se da +id h +boy co +ðŁĻĪ ðŁĺĤ +sam son +post pone +n ra +es n +de wan +ber nabe +an thrac +ìķĦ ìĿ´ +under mining +sm v +gior dano +cor ne +ca stig +bal moral +peder sen +pap s +du e +ad here +vanc ity +ta za +t ada +le if +incre mental +house full +secre ts +eth am +ex es +r itic +keto genic +kerry washington +kean ure +du go +dra b +college gameday +co gni +ac ap +uc sb +nab il +corri gan +al ain +sh ale +s ws +im ti +bre ve +ar ai +pc gs +kaw i +har ford +gerry mand +casu als +an ish +th ap +lo aves +go alies +cle e +pash tun +ven mo +vaul ted +shi var +re gur +plum me +fun ders +t sch +rapp or +r ten +ple t +deb ilit +chil ders +black ness +black heath +az im +anthro pom +alco hol +wednesday thoughts +wan ker +lon goria +ne spresso +holland aise +artist es +ðŁij ¦ +singapore an +miam is +ent or +d lp +be ero +ak ka +united kingdom +unic orn +stan k +shi k +pres sured +person of +impre ssing +grat uit +grac ia +gang es +detroit redwings +century link +inter collegiate +boo ed +shi ki +opti ma +onthe blog +margher ita +ling us +en bc +don i +yi fan +r ba +fit test +dor ff +dep tford +dd g +woodland trust +j cu +er skine +dab o +re tr +pe eta +interpre tive +comman dos +son o +ru ffles +bi bs +mercuri al +lo pe +grim shaw +fairy tail +d ood +con nacht +bot anist +yam ato +wal ton +tri ke +sh ards +motor rad +mach u +fa had +demon eti +de h +cy ril +ch roma +bla zer +wau kee +the fan +sj s +si ro +sch iller +play wrights +geopol itical +cb l +c mb +brick yard +ëĤ ¨ +sul ts +policy makers +marx ism +el paso +dil ly +at tainment +watch ing +inser ted +bl ick +as pi +of course +la ois +a sti +ju illet +har ness +enrol ment +ðŁĻı ðŁı¿ +ðŁijĢ ðŁijĢ +hon ne +evo kes +curi ous +clo thes +tu lum +mo x +lo fc +ka os +gun point +carav an +boo boo +tran scrip +pollin ation +gas m +den ison +cam e +ãĥ ģ +obsc ur +liter ary +g ati +disneyland paris +ag ames +mn p +mitt romney +maha dev +hang a +ðŁ¤ ¬ +pre ordered +mj fam +ku al +in day +duck ling +div yas +bo v +af tere +" ), +wo bbly +transi stor +thom son +sc l +l ach +gur ley +fu tur +door bell +cau casus +ile ana +george town +be ste +ðŁļ ģ +ðŁĺĦ ðŁĺĦ +st ence +s ü +or ti +male c +islam ists +heart throb +crucifi xion +ali ster +wiz ki +cole en +app alled +sk am +sh indi +nightw ing +fix ation +tri vand +stir ling +sing ham +sh able +fro wn +cu ses +ano inted +tar yn +presu me +nu anced +meck len +ku bo +hl pf +funer als +flo at +wh edon +trans fusion +fc ps +af u +subor din +she khar +seaof thieves +plenti ful +pente costal +pa sig +beat le +squ ires +conge sted +som brero +ring ling +rein hardt +is love +bal last +annapur na +al ban +/ : +vi ent +tit ties +gro oms +du xford +dan vers +bab ar +ack erman +x factor +v ms +uniq lo +sporting kc +pen al +over run +ne arer +nad er +life hack +ko ku +cr pf +vehic le +un ners +serv o +n ta +i wan +h md +emp tying +de kker +chu bb +back yard +news flash +n st +ley ball +lam bing +jamie son +folk sy +cram med +polyu re +mpu malanga +karnat ak +ef er +w has +v age +till is +street art +nit rate +nas s +gues thouse +blan ken +save butterflies +photo bombing +pe bble +nbc sports +ke mb +jessi ej +human ism +ge ki +ern yo +dancing abc +all ard +al ford +ab r +shin hye +repent ance +lym pho +don c +di ol +no l +ठ¨ +work book +vincen zo +spra yer +mental illness +by te +ðŁĶ ° +sel var +puri fy +min zy +ce ci +cbc news +âĺ ł +win tery +toronto star +gar ret +cassp ernyo +atl é +al can +one more +hist fic +hat ches +ha se +gy ro +gamb hir +erik sen +afore ver +yl o +valu ations +sel tzer +nus ra +ðŁı ¹ +plagiar ism +per la +kun st +jon athon +inqui rer +black face +tri e +pas a +joh no +chicag oland +chi al +ag al +trin ket +fran tic +din on +cancell ations +un be +sch me +promin ence +o stro +com ical +e ads +weav ers +antwer pen +tri an +ec ole +bil bo +b su +cospla ys +conven e +cm te +barric ades +amazing phil +) ] +tat i +sh app +scis sor +north ridge +nazion ale +gro cer +eat more +ea ves +de sley +bbc weather +b vi +ðŁijıðŁı¼ ðŁijıðŁı¼ðŁijıðŁı¼ +youth day +thur rock +tensor flow +man z +katow ice +high life +deci pher +pig ments +mu mma +bu f +amar in +trouble shooting +snap deal +ol ar +jeffgordon web +dog wood +kat ya +itsenrique gil +bigo ts +ðŁļ ² +ker now +jay alali +in separable +x files +war at +mu z +mo ped +break throughs +bran ching +bouti ques +word sof +wi st +tren ded +ren aming +r hom +maced onian +keanure eves +approach able +y bridge +ve il +ty l +tamannaah speaks +sti f +photo friday +e ir +cav ities +proce eding +pix ies +key hole +eeee eee +ultimat um +stu ffer +mar sala +groo vy +dal ston +ðŁıĮ ï¸ı +vin ay +lat inas +ga is +fo les +be yer +app al +th ales +soun dof +moderni ze +ligu ria +jav a +carib bean +aa yog +wiki media +socio economic +k cr +im raina +hygi enic +the kid +stret cher +scot ch +pan cho +oo g +nat west +nam ur +ðŁĴ ĩ +re shuffle +o a +go m +es f +dill inger +bu sses +bac cal +sa al +person ali +n ought +lovers day +kew gardens +ge mini +du x +bud den +blood line +bi les +air quality +ìĤ¬ë ŀ +âĸ ² +razor back +londonis lovinit +konstant in +k vue +ima h +: ,) +spu ds +skyl ine +lux uri +loy alist +horn by +deb t +charle ston +more head +health day +ess endon +ef m +cow es +timm y +oxid ation +invest ment +inthe city +geo g +ale gre +ðŁħ °ï¸ı +waf er +ri bu +m tsu +fab ulous +zyn ski +va inglory +under whel +ri bble +men sa +kim ber +insol vency +gen ous +ck d +person as +na e +iv ory +dagen ham +ra o +mouth piece +mor ne +le mmon +gl ace +etsy social +chiranje evi +tv series +the u +sait ama +ging rich +flag day +b snl +au ra +ao i +hol brook +green ish +consult ative +win drush +water side +n ff +lovel iness +live in +for heroes +ðŁĶ ± +vo i +p ne +nol i +l all +horse hour +bre whouse +be mid +pd p +fron ten +fri eze +ar acing +æ ł +sub tle +sm ac +ah san +ts v +restric ting +li ano +is mail +fianc ée +ad oo +yn olds +pret ended +om yo +n aca +convic ts +battle ofthe +ðŁĴĥ ðŁı½ +re vo +kil lah +jad hav +gree ley +fc cc +ev in +y oooo +te al +shiv raj +rival ries +rel ational +pos ite +nct smtown +fi at +anam bra +aerop lane +# / +ðŁĩ¹ðŁĩ Ń +rein forcing +just sayin +incub ation +de u +( ...) +vern on +new swire +lan ge +hypo critical +ac ity +abu zz +star news +rhino ceros +rais ing +pm qs +pin as +ne cn +mtv lak +harry potter +att is +sof as +patho logists +oc to +mont mar +hah ha +far aday +ar murugadoss +appell ate +saku ra +imperson ator +er go +dog sare +bour go +talis man +pon dic +il legal +work flows +thn ks +sm itty +skin care +poin set +pic spam +man soor +exac to +ech lin +as at +alleg ory +y asha +u mc +re kind +rat an +pu ck +ip ur +humble isd +christ o +bel tran +az a +ab bi +vi sto +shin hwa +playo ff +pa ve +hun an +bush nell +) !!! +ðŁĺļ ðŁĺļ +st win +place tobe +non violent +lon go +kal ing +geo engineering +audit ors +è ¡ +uof l +tal ker +s borough +patho logical +or as +elm wood +bur l +bear den +b hat +relent lessly +men om +j alil +e bene +augu in +men tos +im d +fur sona +ras mussen +ran ting +kas ab +k lang +ide k +dy nasty +cbs thismorning +mt bos +ðŁĺ ½ +re worked +mali bu +lo ban +la zar +host els +do in +def ra +breit ling +bis on +an r +sa want +quin nipi +mcar thur +ally son +aler ted +y lang +tr ul +ron ald +pro ds +master son +hel io +get the +fire emblem +cup final +bre st +ðŁij Ł +y aaa +van quish +track ers +rosal ind +persu asive +new found +g sk +el ke +dev op +ci ar +buck le +aly tics +yah ya +ty me +the dailysketch +th aan +personof interest +e bel +atlu td +Ä « +tson ga +scari er +rise and +pass able +pa than +lib crib +im g +execu tion +yal it +re port +op ie +dun geness +dream home +ne ssa +monu ment +mill enium +dani sh +bert son +é Ļ +w impy +spanish gp +slic ing +n oun +la borers +ji hyo +f st +dad dario +bang or +' ." +pra ha +mau de +jacqu ard +hi ra +cook books +th wart +sor riso +me din +infe rence +gr inning +cor du +ano inting +íĺ Ħ +val do +ss oc +screen print +s ree +privati zation +national poetryday +healthand safety +er ner +the five +technic a +run es +per in +don ahue +bra c +ber nab +wizki dayo +ra bat +pyon gyang +lion el +fi da +cla us +bay are +aldub the +ðŁĴİ ðŁĴİ +suz uk +retro grade +moun ta +ma der +her ding +ðŁĶ ® +soun der +s forum +gre tel +ಠ¨ +pa the +edg baston +do h +bob bie +ðŁĴĶ ðŁĴĶ +se alife +s ree +mu gg +monte rey +no am +much os +lu red +t dc +superstar rajini +spal ace +show us +i go +faw ad +wa j +smash bro +jacob sen +dvor ak +regre tted +ral f +no b +lobby ist +isai ah +etobic oke +brant ford +bon ey +believ able +agre en +ðŁĩµ ðŁĩ· +sky fall +shilpa shinde +re spl +rail road +pau k +fun inthe +fi est +co cc +cho ck +beli ke +alli e +qu at +public schools +mar o +h ing +gloss ary +flo tilla +figu eroa +f illies +birth right +bar olo +am ag +é Ģ +tab itha +ra shi +pu tra +com or +ky un +il u +cad re +belle w +ab ort +sp fl +nick carter +naw ab +jol t +christma sin +carr illo +affirm ations + ª +yipp ee +as sail +à° ° +ske leton +river walk +per l +nin ado +mis understanding +hunting ton +holly woo +bel lows +¨ ï¸ı +unru ly +the weather +sw ar +ru stic +reggae ton +my ungsoo +muske gon +fili gree +czech republic +ch ch +un thinkable +vaccin ations +swasti ka +sol vent +ipkk nd +hel ve +aldu beb +raun er +pho en +jo ya +twi st +trade marks +spor tive +scor cher +razor backs +ra ik +infiltr ation +biow are +archi vist +ak ita +ç¥ ŀ +meek mill +kn ap +cag ayan +wh id +tu ll +sri devi +mis fit +ma v +imacele b +fo ils +cc b +bren don +bic ep +al ittle +thr ice +reg alia +ra bo +pain less +overest im +marin ara +klit schko +ig f +hr inger +gu st +captain swan +ar ay +ðŁİ º +á il +u day +co bras +caitrion am +u ig +hard top +eci g +bach mann +k wara +eric h +de bs +contra sts +turbo charged +rich man +provo ke +long mire +dilem mas +the blue +me di +ley park +fam s +e sport +bi ko +bar ium +aveng ed +allar dyce +aar hus +better call +king sbury +gn ant +friendship day +substan ti +sch ip +pep tides +mate en +اÙĦ س +tur alism +st ang +ra aj +peace keepers +li ana +exc ites +vaz quez +us gp +travel ing +pill ar +gu h +competen cies +ar tur +vo lo +jer ome +di adel +den ny +av fcofficial +u dd +mo dy +mini str +ge min +cryp tonews +chitec ture +z infan +super fast +st ace +saj id +kra zy +ðŁĵ Ģ +philipp ians +nis a +book sellers +Ä ģ +victor ian +the body +su pt +salmon ella +rat ty +jo gger +fu biz +cree ks +bled soe +ad ell +zinfan del +trape ze +si z +sho eing +le pro +ja vid +custom ed +sa ath +quar antine +mis sk +detri mental +champag ne +bi k +; _; +wa f +tiger woods +star burst +rach man +ok ada +new day +ly ca +der rick +anec dotes +stemc ells +pas cal +hu sain +clai borne +bol son +apar te +ai pac +wi k +w ch +stimul ates +morpho logy +logi stic +indom itable +gal oo +comm end +chaw la +' ( +tru jillo +lown des +log ics +liber ating +as am +arrive alive +aro ons +а н +shepher d +p bc +li po +er l +citic bs +cc sd +caitrionam balfe +br fc +se ki +it out +ish q +dil do +ati k +amar inder +tal kie +state hood +ca be +bos well +ðŁļ ij +wer th +va al +sky ping +ear phone +dilig ently +co chin +ap hi +am ente +timesof israel +sel assie +road runner +ok ay +ny der +ni ven +la ir +ce ased +categori zed +ðŁĴ Ĩ +u fo +tele scopes +om ania +cam ino +b illa +aw ning +Ĵ ï¸ı +ðŁIJ ħ +ðŁįķ ðŁįķ +wom ans +re iner +peace building +neu ter +dj ia +cyber bullying +cs x +constitu te +b the +zam bo +on ta +cal loway +steel head +one team +ini ans +i zzo +abor ted +se to +maldon ado +good day +fil mo +bre ck +hang outs +gibr an +z sa +whit more +stru p +short story +jen i +energi zing +con vening +check mate +batt en +amazon in +alfal fa +star ks +q v +ma eve +le fish +ide vad +earth capture +bn buzz +bau lt +amate ur +us l +twitch kittens +tri ms +mb bs +kodi ak +din ky +choreo graphed +ben son +ar aw +ÑĢ Ñ +real tor +fun facts +f nf +d mp +ben ue +baye sian +the old +subscri bing +ra king +official monstax +g ak +drink able +detec tive +trilli um +snow men +shah rukh +eli ds +dismant led +mo dest +lud acris +can trell +ðŁĶ Ļ +âĿ¤ï¸ı ⾨ +Ú ¯ +yw ca +tb adly +sa ha +por tof +lu cre +la ken +ha skins +vinyl records +p ima +mol o +ign ited +gau ges +f sd +ðŁĽ į +mat su +g ant +hen nes +h bo +bu sta +se tups +scor ner +reli eving +neur on +irish man +fo gle +d bn +summ a +pi ppin +micro finance +fanci ed +chair woman +brah ma +fal low +anti social +wi a +t ments +ram i +ra iney +mind blown +ly man +afgh an +billi ard +author itative +ye hun +sput nik +bombard ment +nl traffic +mar ic +ma vis +in nov +central park +bli ge +ry de +plun ged +patho logist +host ility +groove musicapp +enti st +em be +chi ba +chast ity +boul dering +bc l +accumul ating +ðŁĴļ ðŁĴĻ +smo king +sm town +pre ssie +k lik +je une +ikamal haasan +highe reducation +e music +ðŁĺı ðŁĺıðŁĺı +war ing +up c +stra chan +sp itz +rober son +nick laus +mue ang +interpre ters +cy c +casspernyo vest +cam acho +sl png +pent icton +min hyun +ki ah +i vo +energi ze +dou gal +alo ha +winter wonderland +ir win +i ar +handle bar +gal lows +est ro +en vi +trivand rum +sty rene +medi ums +gains borough +dr ina +dis agrees +d la +aud acious +wizard world +us ac +subdu ed +slaughter house +n wc +macchi ato +ham er +gat os +debun ked +contact less +c ing +z f +ver meer +trick or +opul ent +is ure +gaz i +filmis notdead +canon uk +bam ford +ske chers +shi ver +ko gi +h mi +gh ats +cor leone +su g +som uch +lo athing +l int +foot work +en list +du rian +canonuk andie +ab ot +x dd +jar gon +ban di +:) " +the only +sm n +n ha +looo ool +idevad hikari +h bl +fol l +traffic alert +kau f +dd p +ad in +so d +rom ford +re strooms +bol linger +sc cc +guardiansofthe galaxy +ash er +api ece +ÙĦ س +rod man +ren z +proce eded +hul kho +equi pe +whit worth +visual isation +under pass +thor p +tae hyun +power fully +pag ani +memor ials +mar vels +intan gible +win o +pe o +o der +ilo vel +gil christ +deep ening +chrise vans +chi ka +br ü +persi an +jer i +ful lof +em manu +cu pp +awesom ely +alvar ado +woof woofwednesday +me ticul +info wars +friend sof +fair mont +cov ina +cma fest +bul ky +agno stic +far ne +anton ov +ta pi +ofe x +men tored +e pps +a had +ãģ ķãĤ +treas on +ro dents +riz vi +pari shes +mal am +ka sey +appreh ended +absolu t +tech ed +pitt sburg +o real +mar itim +li us +laun ce +correspon dents +wat ery +s jr +ron ey +neta ji +glori fy +ar son +âĿ ķ +Ø§Ø ¦ +wood block +rt p +m th +iam jericho +hu ge +gh at +car go +a edt +wig more +shutt les +retribu tion +pinot noir +back room +abhi yan +ðŁĩ§ ðŁĩ· +sir l +se gura +latin america +ex id +be aker +arch ite +wo z +un load +student life +motiv ator +ku ta +green ock +go zo +con st +respl endent +e mulator +atten u +the zon +reser vo +mc gon +in dah +ga it +domen ico +do sage +ant ler +oh ne +inter ning +cor mier +ci ence +å ij +ss ur +red hat +ou ach +high score +exclu des +ear ls +cal u +simul ate +mur frees +kru g +gat to +christ a +an h +start shere +sin n +n wo +lo ween +g lynn +flo rentine +dra go +spi kers +shar m +north wich +liquid ation +are llo +walk about +ting ling +public art +on earth +mu ker +interrup ting +il va +de brief +cancer ous +big sean +week night +t cc +gene si +el ka +ci pher +cali ph +ti eth +re produce +koo kab +kel lo +aldub x +shoe maker +imagin able +าภ¢ +w bu +th ay +strato sphere +red stone +pla s +pimp in +mi p +lu te +hatties burg +hal lowed +fen wick +tweetapicture youcantexplain +pro gro +not ley +jaw line +dev ouring +resi due +redon do +mm t +mccaf frey +human it +gr m +dou ghty +ë¦ ¬ +wit te +til bury +men sch +intellectu ally +col ada +ar ad +ðŁĮ ¤ +understand able +gu lati +ghi story +ase m +ram ping +pr is +mt p +ic ul +gerrymand ering +fan nie +dealer ships +coc cus +carav aggio +ameli e +ra ger +twi sted +succumb ed +spino sa +ku mari +iu pui +horn sby +cro sse +c fis +t ingly +ss ohn +sab ers +red cross +over priced +ni sha +kat t +j peg +internationaldayof happiness +fau x +ym c +ug i +tn z +sw irls +strike out +st k +shred der +ninado brev +hulkho gan +gh ia +discer ning +bru yne +! .... +tac loban +r ind +major ca +le uk +grand mothers +g bu +buck inghamshire +ðŁijĮ ðŁı½ +ìĽ Įë +âļĵ ï¸ı@ +photo synthesis +jugger naut +dan te +stick y +soft bank +im mer +ha ber +! "- +y ue +ru in +id m +hing es +cricket worldcup +wisdom wednesday +tra xx +old skool +now reading +mar la +kas per +gersh win +dugg ar +ber mond +yid dish +tayl ors +mus graves +ft l +dun yan +chrome cast +ðŁ¤© ðŁ¤© +nc p +mick elson +mart en +mahar shi +contracep tive +à » +vit ae +ten ni +stal kers +mirror less +excav ations +cnbc tv +christmass y +cav ed +ste pan +sep p +oo dles +âĹ ĭ +vil les +ro ving +pan am +nen shi +l ö +bun n +âļ¡ï¸ı âļ¡ï¸ıâļ¡ï¸ı +w wa +kae mia +pre print +magi strates +kam ikaze +ka jal +inge x +equ ator +box ingday +aw ara +ser ye +not ched +membran es +mar an +humili ating +some one +sno qual +sn j +r pt +pries thood +man hole +bur ke +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ +ug c +pim ple +n aco +made inthe +al em +zi onists +wau kesha +nip sey +march and +// /// +⼠Ħ +to or +new ham +metamorpho sis +mag num +don ning +ci vit +bn k +v oot +the v +sar nia +sabbat ical +pa ppy +on fire +leon id +go ff +gi joe +gel atin +garfun kel +camoufla ged +air ship +paras it +nca ad +dor man +chann ing +ce bit +at ina +ðŁ¥ ħ +weare alive +rall ye +nightly news +kiba athai +fate h +codi les +amp he +yz f +su man +dm z +colli ding +assembly man +à® ¯ +year in +to ga +po tions +patric i +matter horn +ma so +kath thi +kale b +hashtag roundup +appo inting +ac g +stro be +re twt +o don +ming yu +la z +ci p +alex ey +shradd ha +schoo lo +ma do +gh ey +curren cy +mis sus +ingh ome +glob alist +di vo +cbs la +be amer +pier son +nb supdates +beau coup +bb it +anom alies +rally cross +man gan +ha zing +bred dit +sony alpha +out source +haram be +esp ys +bu x +bol and +bench marks +as ka +time travel +protect the +moor head +kar ate +jag er +gri ffey +ex port +craf ty +aw ild +arche ology +and mail +pad u +ham i +draf thouse +de ane +yugo slavia +wall paper +tyran no +quar k +lic ences +admit tedly +y re +lau rie +fore sight +al yn +ðŁIJ ¥ +yogy akarta +war gaming +tyre se +romel u +revolution izing +lil le +ing in +ah q +xi ao +ti ffin +teacher sday +sau stralia +mid western +gel e +fin chley +fantastic beasts +cla s +âĢ £ +mountain bike +l ates +ðŁĶ IJ +scu der +re direct +her bie +ge sh +frat ernal +ran chi +modu s +mar bled +g ans +f ss +compli mented +clean water +um rah +rock in +mal achi +lasag ne +gee zer +f ated +du be +de jan +as ante +anti semitic +shan ed +rou sh +resent ment +pokemon go +mi ths +catal ysis +sor cere +rati fied +jewe led +go swami +cho ppy +bra r +ar ay +a wh +zo omed +su breddit +stric ter +short cake +onlin ecraft +nes bitt +nepale se +knick ers +hotro d +d pm +bc b +ba ren +agne z +âľ ¦ +u co +s best +k dvr +ðŁį į +param ount +mar lon +machine gun +it ers +do berman +attack on +ac orns +ro stov +re gained +py r +out board +am ol +re sa +ig lobal +hill on +f kn +crowd sourcing +rc p +lo rena +e ow +d mu +ðŁijĮ ðŁĺį +tony abbott +sw b +hu blot +hom mes +gal vin +vat os +un biased +terrac ed +oc ta +mel hor +ilayathal apathy +f lead +burgl ars +electr on +cam brian +aure ate +ali b +under valued +t mr +our ce +ja er +ous al +len oir +ðŁĮ Ģ +than et +r aring +quix ote +loc ator +la porte +endocr ino +change makers +bo dh +so hail +kam il +fu sions +compri ses +ranger over +pau lie +mush room +go shen +fr nd +erc ise +bur t +strath clyde +north umbria +keepp ounding +k cal +htg awm +german y +far thest +eng lewood +block buster +wor shipped +geor g +condu it +weir der +under water +spe y +shi pp +sam aj +fon ia +ðŁĶµ âļªï¸ı +çµ µ +yehun dinon +well ington +s ood +dog gies +wa ites +ss ac +se ep +reas surance +ram sgate +di us +con fer +at too +ìĺ ģ +vaness a +us as +observ ational +na st +mis carriage +io i +ec up +af oundation +live at +gram ps +gigi hadid +end am +bu z +aspe edway +ren é +pin hole +my day +mendel ssohn +k bc +downey jr +ti gger +spe x +radio show +ft r +Ø ¹ +the series +shivan gi +senate majldr +oak wood +i mi +chuk wu +asi a +witz ki +see ley +ro deo +pin point +mod ded +home m +gor i +gb pusd +un timely +sh atta +severy where +nic hole +den ce +ðŁijıðŁijı ðŁijıðŁijı +whit t +reali dad +kin en +in or +fad er +dri fted +def leppard +ä¸ĸ çķ +sling shot +ka iz +cer o +blac kex +ap na +aaaa aaa +ðŁ¤¦ ðŁı»âĢįâĻĢï¸ı +ver acruz +she ph +pi awards +à´ ¿ +will i +visit norway +the voic +swee ties +royal airforce +pic nic +lin z +har wood +cla rendon +super foods +stri ves +ridic ul +incar nate +absa prem +toronto police +ond that +loo sen +de ws +je st +ir cle +chu ms +bri stow +zan te +mulla hs +mi as +ha bbo +à Ń +z aza +war lord +shab ab +sero tonin +ren berg +on coming +ex cre +ev ada +b cos +un dying +special ised +sirius xm +ques adilla +open science +kar olina +dil la +u fa +mir chi +juventus fc +j sa +dio de +command ed +cbs baltimore +be ys +as kar +art collector +am ira +who dat +t wn +po ppers +bl ame +u mber +sword fish +lam ini +for america +fal cone +envisi oned +der anged +wh r +t fs +seag rass +se fton +on di +kemp er +ju do +do pest +disar mament +antag onist +ali m +ak p +sm er +risk management +oo c +fan ni +eclip se +curric ular +ca stel +bal le +ate a +ar by +te quil +k ander +goal keeping +cra igh +bb h +un itarian +maas ai +just ine +be gu +arc gis +sr b +south all +mus ée +juli anna +exce ed +auditi oned +aneur ysm +ac customed +thi erry +syl vie +itu r +humming birds +fortnite game +ent ley +cro hns +cl oning +chak ras +c ely +== = +whit er +rose tte +phi r +ko bayashi +kir kle +gri st +disproportion ately +corin th +bu c +av aya +we iz +vo i +s iti +min ha +meticul ously +k si +herring bone +every one +work man +north shore +mor dor +ir ving +hammer ing +scien cen +nine ties +mar shawn +l illie +for tu +f elling +cro codiles +wi den +tiger pride +rou ss +pl r +os ab +o ski +gu zz +fen ced +end fgm +din es +tr in +sto ic +my er +k ass +k ama +eye glasses +bol an +bob marley +ation ary +air tel +pro ye +mad den +inhe rently +bush fire +ble acher +beautyandthe beast +re ak +op c +lex is +ho bi +eu genie +bar ter +bar bra +ĥ âĸ +à § +wa e +po ppa +long standing +indestruc tible +embar ked +co ppa +bel los +teslamo tors +influen ster +immac ul +hav n +chu g +apple bee +mal lo +je di +er rr +ö zil +morri stown +mo bbed +gun ter +foo tie +e ce +bun d +u ae +so ka +ra bid +lu pu +destiny thegame +class men +bo thers +ar ah +ze phy +mini stering +he ire +fo caccia +distric ting +cros stown +ðŁij© ðŁı»âĢį +sop ro +snu ggled +sho al +ch kin +ari anna +thu ggin +thor pe +mother ship +f end +cam ille +arch i +tor ches +si smo +dome ter +cur tis +tar ragon +oxi di +m ellor +i rena +head y +entr ances +bre mont +bn ppo +tro tting +sang am +no d +daugh ter +cold well +buff y +bbc proms +ann once +tt tt +em ce +culin ary +bre scia +bor uto +big p +bettercall saul +ven er +tor neo +rodri gues +de pra +stam il +ðŁı Ħ +salv aged +madein america +fu gee +comic relief +anu eva +âĺºï¸ı âĺºï¸ı +you ve +rooster teeth +mur thy +lan za +ja xx +invo ke +ign ite +fi ori +boston globe +ðŁı Ĵ +jesuschri st +cgi ar +ðŁĹ ŀ +w fd +s gc +kow loon +deloit te +the cat +sval bard +rickand morty +nun o +jun cker +implic it +har sha +ghou ls +end rick +bow es +te y +te ign +rin ks +o ce +metaph or +ish ly +intern al +in clin +day out +sil age +par ang +ny n +murfrees boro +k org +cin der +ba ji +tell in +mic rone +kang ana +id x +ðŁĴľðŁĴľ ðŁĴľðŁĴľ +te ca +sy co +madhyapra desh +expe dia +clo g +av ol +u mar +the chive +se izing +pri v +pav ili +p els +lin de +jam al +de walt +alle ys +ì§ Ģ +summer reading +orphe um +gla sto +europe antour +ci gs +beat riz +tonyabbott mhr +par le +kil au +hit ched +bere a +be holder +bas sne +arro z +í Ħ +syn ths +nfl combine +new stv +; ;) +y ams +whom ever +tor un +they re +south west +shru g +scot landis +pione ered +haul er +h ss +dav it +be fri +tro on +se pang +lo z +ea p +ay len +wri t +toile tries +lac ey +engineer ing +ch vr +wan ted +w am +silverstone uk +si al +sha sh +pe le +pan to +gor dy +g st +âŀ ¡ +upper deck +nat chez +mach e +collec tables +b cuz +ðŁIJ ij +tin ubu +snow shoe +moham ed +mal divi +mal den +thankful for +royal ascot +private equity +nine teenth +hus sey +bo ggs +zin ski +ter pen +son os +radio therapy +quic kie +pro vement +north ward +inns bruck +flash mob +expe di +boyco tting +rd guk +o ole +mar chers +mand u +griev ances +diss on +call ers +ble mi +bill gates +suni versity +sen warren +ru d +ros common +palad ins +monday thoughts +beer day +al c +n tr +kag ura +jiang su +flow ery +conce ding +che red +an ay +ĵ ãģ +âĢĵ âĢĵ +resto re +ny ard +muzaf far +ine za +esp anto +cannabis community +smithson ian +s london +regul ates +making adifference +at v +à¹ĢภĽ +ross ini +re settlement +ini k +in the +fo t +color ways +ak en +ur qu +under pants +n ns +medic ine +l lo +ik an +g acy +em body +ef ter +ban j +ar f +âĿĦï¸ıâĿĦï¸ı âĿĦï¸ı +ts r +mr ss +fi es +cor az +ago go +then orth +resolu te +nas l +magaz in +gr rr +et weets +business insider +bench marking +montmar tre +home stuck +he it +e chin +ac ai +-- " +u chicago +ph ra +m clen +cumple años +clo sing +bald win +par kes +orni tho +mi on +art sed +outer wear +farm to +endu res +dor king +ai i +per severe +o ar +knight ley +ho hoho +democr at +bernabe u +ap la +yam an +veterin arian +tin the +sonsof anarchy +shad er +hawaii an +ge tz +am bb +ðŁĮ ħ +u mu +that smy +one day +door n +cr aters +cn ni +ast ate +council member +back pain +ad r +âķIJ âķIJ +stra ined +sas su +globe andmail +geta fe +fri vol +fanta stical +allen de +thezon ecast +shipp uden +saras wati +rou ge +pu pper +mo dena +gerard way +gaz elle +du sse +dog friendly +ðŁ¤ĺ ðŁı¾ +t fi +sam mi +row dy +kinab alu +jagann ath +u ff +so da +sg d +mum taz +glo bo +faf sa +b ced +visi bly +miner al +di ra +bees wax +shail ene +presti ge +dissec ting +bm wi +b ich +an tic +wil hel +tax scam +son tour +or am +north field +k tv +her ds +fu jitsu +es v +et ch +bro ms +borough s +anchor man +smash ing +j mc +fra ppe +ct l +ðŁĵ ī +un ger +screen er +nbc nightlynews +lay zhang +hugh jackman +devo tee +defici ent +charli ze +ðŁĶ¥ @ +water crisis +vish nu +t fm +sof war +mau ve +ken ji +datemy family +cer u +bear cat +su zi +o ad +leit rim +iso de +cre scen +astro physics +wc co +w pt +tou lon +to pple +pu c +pe ay +ninj ago +manufac tures +dissi dent +col vin +u ist +tra i +t ola +swan k +splat ter +robbie williams +nu ps +mcn ulty +lacro ix +bati m +ðŁij § +ðŁİµ ðŁİ¶ +th elife +gent ina +cl at +( ), +you gov +yehundinon kibaathai +us g +re branded +mar mite +ludic rous +kc co +j ft +hur acan +huck leberry +at ingly +ðŁĩ¨ðŁĩ ´ +the storm +rec tomy +monstro sity +memor ies +kitt en +en a +dough erty +bt b +are g +ðŁĺį ðŁĺĬ +sla v +moon lit +mad d +dis advantage +comm enter +bram ble +bor rowers +bel ow +staun ton +sali va +kwi k +ken na +fal si +edge water +de ur +d souza +d me +contradic tion +stan ning +sch loss +sc tv +i aff +golden eye +archi e +ali khan +al brecht +aa as +slo west +in scriptions +girl guiding +chu ca +ðŁIJĿ ðŁIJĿ +âĢĶ # +styli zed +sam ford +ph nom +long view +jy j +bu ford +as n +wall y +li en +decad ence +dan e +bar uch +ry land +halli well +g tb +g out +fou l +f ye +cur lew +con golese +bhat tachar +ðŁĺĤ ðŁijı +sla ve +gli mmer +d alla +cy l +un fccc +tram way +ti gre +sin dhi +ob ac +ben et +beef y +strat os +pregn ancies +plan te +el ux +chennai ipl +ภ« +wa hab +out kast +manit ou +lu au +io u +hock ney +contre ras +baby sit +uh cougar +mobile app +maria sharapova +expect ant +consoli date +ton al +steep ed +kaz i +ih ate +fl acco +e show +catholic ism +ar adio +sack ville +mackin ac +i was +english man +cantbe broken +bu ble +united fc +ron de +pack er +bristol city +ãĢ ° +sol s +schur ch +sak shi +meto pera +me ang +i vor +ech l +ba an +ðŁĴŀ ðŁĴŀðŁĴŀ +vin ho +sm ttt +prince sa +por trush +aa o += ] +profici ent +lick ed +ig nis +head lined +fast ball +do ss +Ð ¶ +squir rel +pro c +plan ned +izom bie +itali angp +hu may +h town +co bh +bar un +run g +res ford +mo jo +adop tions +t la +shar d +pri vacy +ko in +don na +ceph alo +cent aur +bau d +ar cel +apol i +ak en +ç ´ +wit chy +tic ats +spo x +real food +mac e +fi fe +be ch +ðŁij Ļ +ÙĪ د +seac rest +renfre w +pi xi +neu f +entertain ers +co oney +cll rs +bo ing +well and +the office +nad ine +inspired by +dilig ent +devi ant +r pf +lin ens +k news +instap lace +em itting +dur and +uuuu u +strö m +sp hilly +o dia +nd tv +jab ba +ing t +d ceu +co et +cl v +pon ders +parliament arians +kei sha +k illi +jae hyun +c ph +an un +sho el +ly tham +em bi +break out +anim a +faith fully +cu pe +ceram ic +am way +wasser man +scotlandis now +me tics +instaplace app +g mc +ðŁ¦ ī +water proofing +resi sts +par quet +go c +brave heart +qu arium +pom ade +pit stop +ng k +for g +bankholiday weekend +resi dent +kol o +hel ic +go lov +found ational +e ine +de tain +cont end +bb f +ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ +ãĥ Ĭ +her ts +ðŁIJ ¦ +visi oning +sk storm +la vi +ran ia +go h +fas cia +dunyan ews +chidam baram +bra zz +bain bridge +sp ook +sno d +silver stein +sav ile +plat onic +msk ajal +geta ways +fri sky +explor atory +u ll +ton ia +sy r +stru ts +spl an +rec tion +oll yofficial +n elly +malm ö +confe rence +clar o +bi bby +topp scards +ther ight +mat ricul +kajol atun +gl enda +church yard +bt m +æ ´ +utah jazz +p sat +high line +disu sed +al mo +af t +âĻ ¬ +troubad our +mull an +lie ge +k oma +he wson +with a +vent a +t ched +show me +leg ali +gift card +boy band +al las +vir at +spacef light +ou ture +lie berman +li kel +i sha +head way +ear hart +bhar ata +å £ +th as +rec ce +pa yo +gab or +foun der +cold war +car pet +ag ha +park nyc +mis a +li mass +len g +infin it +indeb ted +deep state +bloom sbury +ðŁĸ ĸ +yay yy +toy story +sb g +n wa +lar issa +gn ar +resi ding +penetr ate +fr itt +ex tro +clo ak +sens ations +rang ers +pur ring +mskajal aggarwal +hi ram +en vious +de stru +and heri +agron omy +ä¸ Ń +to va +lock heed +face timing +bon ny +ab user +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ro derick +out perform +ne pa +ir in +explain afil +buffalo es +ðŁ¥ ģ +uno dc +trespas sing +to h +rock music +dolce amore +on eness +lob sters +in tr +hind sight +enlar ged +brit ons +ago d +va id +tt l +there after +so ir +nat ale +ecor ps +chipot le +yo k +ga h +al va +pic s +ow sky +cin ta +anec dote +ðŁĺĽ ðŁĺĽ +rein carnation +ple aser +ni q +mackin non +can ber +wa he +ur bex +ud inese +selfies unday +sa hel +extrac ting +champion ship +buzz feed +/ £ +yasi r +smi thing +rela pse +libr i +inclu sion +fab ian +!! ... +ðŁĻıðŁı» ðŁĻıðŁı» +ne der +gu sh +blen heim +âĦ ĸ +yugi oh +seh gal +po dol +je thro +jama al +g oun +co zy +bri gid +v lc +roc keting +pin ch +maggi ore +ken a +k ony +wi ke +whistle podu +stupi dly +liv able +lion heart +lak me +jong dae +ge en +electr ics +cyani de +arc ades +to kas +mono poli +mar key +dimit ri +dale ks +convers ational +aj c +tes bury +ste pper +ri as +repl enish +ker ou +imel da +ig eria +ðŁĶ ½ +une qual +sad dles +reci fe +ms f +mal practice +io dine +dar ko +stu esday +ho dge +dc f +wal kin +spe op +pe ering +okc thunder +jac o +ha snt +ge ico +bree zes +berea vement +ðŁķ · +lu saka +irish rugby +el aw +tyr rell +thunder up +inter acted +goooo od +far oe +ver mouth +tren ch +l rc +east leigh +copernic us +ber dy +special ty +prostitu te +po ftheday +li ster +gry ffin +ed f +bra ith +boo kies +bid den +ber nier +ac ause +was s +to ku +rit u +internet marketing +electr ons +akan shag +tv r +sni ps +pac s +mu ss +ho da +fire power +drum sticks +childhood cancer +scri bbles +off i +vo gel +t gi +star s +spo st +se atur +koo tenay +gy al +ban ff +ap ush +ad mk +sp op +ri stor +nit ish +mel lo +kat erin +goo g +aw ol +ai ley +ãĤ ļ +ภĦ +wal kies +te j +jar red +g ley +" !!! +ø r +sh andy +parake et +me tering +man ju +lets movie +j ls +j aga +en shr +of s +honey bee +fire truck +ad f +an try +tom cat +p russia +li sal +ew es +du d +boy d +analy zes +real life +head teacher +a vie +yu le +wal lo +sha g +m signite +deli rious +cool in +apo s +w illa +villa inous +hac er +dece ived +cv n +wind ham +ta q +mplo tbadly +mee han +man ner +macgy ver +de pau +beer co +arch it +over looks +k ard +g litters +embe zz +dan k +atro city +ë¸ Ķë +è ĭ +tam u +sul timate +south florida +nendor oid +ge yser +di z +al ston +sul fate +ri alto +revol ve +o is +ing ress +goo od +ask mrn +waste ful +ver mic +un drafted +so as +ici c +god dess +gill an +carpent ers +stan hope +scul ture +r sr +cr ème +bri en +ðŁĶ ĵ +ob v +hi karu +flo rence +flat ter +contin ual +art vs +ðŁĩŃ ðŁĩ +z ale +ww d +ut ely +tre lief +tip ster +sto tt +st pete +muss olini +mc b +jun o +an aco +pra deep +ect ady +ãĥ ı +ঠ¨ +z uk +twe aked +hu huhu +chatur bate +bo j +a jax +subhan allah +snar ky +rw b +rever b +hu mes +gilmore girls +cel ine +âĸ ¼ +rak sha +m to +intelli gen +fla pol +co pia +can isi +bible verse +tre vino +trac i +thom e +pharmac ies +gram mar +en sign +w miwx +swach h +ra gged +par ian +p ff +gro sso +adi eu +âĢĭ ' +zor ro +swin ney +osc eola +nigeri adecides +n ma +mm d +insomni ac +in design +harley quinn +full time +er z +cer sei +male k +mac aw +cling y +beero clock +band han +ðŁĻĪ ðŁĻĪðŁĻĪ +gulfstream park +fic he +entro py +ci led +amaz e +me us +ie v +emb lem +em ate +donal dj +ban yan +akanshag autam +save america +ney mar +missi ble +leav en +capac itor +bod hi +slyther in +explainafil mplotbadly +em re +e tru +cohe rence +schen ectady +positi onal +nai arind +lao gha +jay z +bro ward +bat tista +at te +apple event +al g +te ary +replic as +polari zation +mor ton +inste iger +cotsw old +brook line +amphi bian +who res +parti san +monogram med +low land +lewi ston +ger o +cam in +boy gan +wear in +unex plained +m ke +her tford +hat cher +matu rely +i be +ac tin +wol ves +d fa +chromato graphy +arri va +sham rock +role model +legendsof tomorrow +prat chett +iom tt +franc a +makas sar +eliza bethan +cav an +ir re +ill ini +gre sham +g é +ero yale +ant m +ðŁ¤· ðŁı½âĢįâĻĢï¸ı +u vm +network rail +lo tz +la ston +inter com +conduc tors +ai p +ðŁĺį ðŁĺĭ +thin ly +mu te +jun myeon +epo ch +black cat +bir thing +tar pon +stil ton +robert downeyjr +qu int +philosop hi +kar gil +fu elling +corp gov +aw scloud +---- -> +park run +double tree +co ted +ari vera +the beat +sapphire now +pro tools +par chment +ondthat cantbebroken +ni mble +ho ts +han di +thir teenth +pow ders +musli mban +lor is +hi paa +domin ica +caramel ised +pan de +miami herald +medic are +jam est +invasi ve +half marathon +configur ations +box ingh +aug mentation +ag c +ðŁijĬ ðŁı¾ +vil leg +un natural +shoul der +roo kie +ration ale +norther ly +motor cade +mi shap +maker faire +ki ernan +ken more +jo em +instru mentation +ge stion +di ggy +cell phones +ör de +var ad +toot sie +r mt +parkin son +pakistan zindabad +ne ct +horn sup +gurru choudhary +fiel ds +cri m +blo ated +vec tors +t pu +s that +r pg +pu d +kl out +kau litz +pa sto +mar ous +man ge +kor ra +ko d +gor ka +d tc +bed lam +b anco +v cr +shakespeare an +sch atz +der ic +baby shower +mer cat +factor ing +£ £ +gi sh +dri zzled +chvr ches +cha im +! ". +tex tile +mo ssy +as v +vi enne +un suspecting +qu b +gt fc +cor tana +bal inese +v rc +un locks +prox ies +dal er +al issa +ìĿ ¸ +spon dy +s vegas +my stics +mis c +may wards +marci ano +escon dido +accumul ate +w acker +til da +sp ines +na ina +mit b +h cl +coach ing +bra v +tran mere +suicidepre vention +story lines +spin y +reci eve +kri sty +don key +cur a +comp ag +blo m +articho kes +wee b +wal luk +ssi mo +radi shes +mechan ic +aj it +ag olf +ðŁĺĢ ðŁĺĢ +wyn onna +pax ex +kon o +âĻ¦ ï¸ı +trip tych +southampton fc +old ham +god z +gibson guitar +e ka +diol ch +cu ddy +che ws +æ ı +watch list +u so +pred snhl +pad ua +of an +never forgotten +kab hi +italian food +har mac +cra g +clo gged +b ong +pay ed +norman reedus +cre ature +wc u +valu ables +run aways +cooper ating +co pious +si der +senjohn mccain +da il +uof t +under dogs +speed ster +sk ye +royal academy +prieste ss +j illian +h sn +am ey +that ched +kau l +il ver +dwell ings +dur ante +ap ache +wo burn +tanz anian +soci able +saatchi art +s wor +jo ann +ji ji +can ister +* __ +wing man +vit ale +osmo sis +organ ically +nom e +hand cuffs +ffe st +eup he +creep ers +* ! +won woo +spla shing +sear th +roll o +lud hi +head ley +fleetwood mac +differenti ated +de thr +brex ite +al drin +zi pp +sno b +sf n +hum mel +bu ona +af ox +wh ok +pilo ting +ml i +j cp +gru mps +danc er +c mg +addic ting +ad dition +hiberni an +g ham +b ame +ðŁį ¼ +sta tham +smugg ler +n tu +morning show +j ö +in love +zap ata +souther ly +shir i +obam as +letch worth +he yy +g bi +et or +ba iting +abund antly +rock port +ph ics +metall ur +dor nan +t lt +o den +lets football +kay leigh +ir ate +goog lec +fa ked +dwell er +du tt +ber k +robb o +lot ine +i yc +hey man +âĪ Ģ +me cum +jam mu +defence man +carth age +moham ad +ill an +d hol +brown lee +ven us +rapp ort +mck el +head wear +got ta +âĺħâĺħ âĺħ +we es +ven kai +n gr +m twx +jai hind +berk ley +ax eem +ye ager +w pial +matern al +gre ys +for love +daily mail +cincin nati +chau tau +awarri or +zü rich +ww os +stream er +peri shed +pearl harbor +pe ggy +online shop +me cfs +vish was +sre es +po re +ichi go +ar gento +wa ive +proble m +pat ched +nowh iring +k pk +fc porto +do on +whopp er +tri bal +sier ran +por te +bassne ctar +af ol +a sted +wit ney +inst alike +del ine +cil ic +bud light +better ment +ab lu +pe corino +and ria +ìĽĮë ĦĪ +ver t +uni er +tic elli +re max +main street +kick ball +coinci de +book lover +aver t +south america +sav ille +oriz ons +kun le +kul karni +ger ty +for better +eil ge +dimin ish +mcen roe +mb poli +kn x +im possi +grat u +doppelg än +cre atine +be safe +ðŁĴķ ðŁĺĺ +sweet water +reg ine +py aar +meh di +explan atory +dr ill +deco ded +danger ous +ci ab +uk business +oo b +mit re +metr on +kop i +gros jean +franch ising +cow ley +van qui +theologi an +re ho +publ icist +pistachi os +local news +ge ck +dor ks +ch h +ay res +river side +in continence +fear some +aster oids +men is +etsy retwt +buck ner +tr ing +tom ar +si ste +aspir in +âĽĦ ï¸ı +reclin er +product design +chim neys +alu it +å º +ti ously +r wd +pe m +nickel back +mclaren f +ly ra +inv ents +cu rio +ciren cester +and gold +� � +ãĢ IJ +ठ¹ +time sup +seag ate +purdu e +pul ley +pepper dine +ali ya +à³ ģ +z hi +ting le +po ked +hunt ley +go dogs +fa kel +" """ +take your +re writing +celeri ac +appro vals +ve rer +toom uch +mis demeanor +ly mp +gar ts +fer ia +culmin ation +bayone tta +y f +moo res +je ju +ef i +conserv ative +base camp +ab aseball +tom ar +sensi bility +r lc +pap as +jcol enc +ware houses +v ashi +transp o +spho to +poo ped +glit ches +ren i +pre et +lma ooooo +hu f +st inging +sne aked +re plays +r hi +inter racial +bu cking +tax ing +sig nature +party time +kid die +inverte brates +impro bable +gastro intestinal +to ki +tang les +om ap +loo d +gold blum +gle be +ec enter +ear ners +si belius +po los +over taken +daysof christmas +cross rail +cob webs +bir la +bil let +z x +vis i +un folded +ste ele +schwe insteiger +rak hi +mar lo +fe tty +cn u +war nock +spoke s +na shua +h ci +way lon +ph t +jan es +im al +ðŁĮŀ ðŁĮŀ +over grown +monday blues +l yo +ho yas +deter rent +crimin ology +anc on +work hard +wel don +nag y +mac omb +lin ux +dap at +ca shed +be is +th b +premi ere +ny sc +jan son +institu tion +ban ish +b hawan +al pradesh +ve lez +spo oner +gret chen +fin edining +dwar ves +drive safe +dishon ored +as so +ver anda +ty rion +stri ppers +nxt takeover +electri fied +sd b +pal moil +n sync +k ith +hart nett +gil berto +fl ate +car in +under grads +tun ga +tot w +s be +mei ji +leg ere +l lyn +head board +fru c +birken head +algon quin +wee ding +she ikh +posse sses +op ent +love able +eso teric +bigo ted +ana heim +wel le +vigor ous +tab la +sp el +par nas +land line +g po +brush less +ble au +au ger +launce ston +hr man +fav a +employee engagement +sus ana +of erta +dari us +crown e +ðŁĵ ½ +w top +te thered +man et +gu te +grim mie +forza juve +cho ppers +bere aved +andhra pradesh +al bor +weather ing +ran gi +out chea +mi zation +fro me +ed vard +bat aan +vic ar +trump jr +sea shepherd +lo cos +dun king +crypto graphy +mat ron +bo ggling +beck enham +wi x +pat ernal +hun tress +herd smen +bi sping +av ati +ac d +un married +turt leneck +tar un +ste ez +se in +lockheed martin +it takes +insur gents +botan ical +bis mil +un speak +o gre +bolson aro +bethe best +sa heb +cathedr als +captain cy +âĨ ij +t cd +se villa +ðŁĺĪ ðŁĺĪðŁĺĪ +w pi +scra pping +prote st +pi dou +mel ba +houston texans +bur dens +app ly +ìĸ ´ +typho on +r ink +j bj +flip grid +com te +carriage way +radi op +pic nics +on as +ham burgers +go red +drin king +bloom ed +sho stak +mul tim +gh en +b ss +afil m +âŃ ķï¸ı +z hi +the projec +rel ati +pr at +dj py +carl ile +sy mp +sad d +oral health +mus i +ka vi +hetero sexual +abo x +ðŁļ « +vis on +tele communication +r nr +pu la +na ir +mccrack en +hazel nuts +gre ig +flamboy ant +fiver r +aussi eed +swa c +sel va +oth ello +or ville +is ch +ing es +family history +course work +chal ice +cat ed +bli sters +xy lo +the p +sk ennedy +school girl +ki v +k nick +fu me +bri gg +bk lyn +uw madison +stumb les +stu pend +stag nant +lax is +ing en +cam corder +tzu yu +sat chat +prohi bit +heis enberg +den iz +aad mi +theli on +sty lin +reinst ated +he ur +ffici el +confi denti +ca z +âĻ¥ï¸ıâĻ¥ï¸ı âĻ¥ï¸ı +te cum +sus anne +n anna +la guardia +king sman +ing in +gil ber +f eng +ephe mera +enf ants +chom sky +chi k +bre anna +uk r +sar an +r ting +pa checo +del and +opti mise +kee gan +caliph ate +ari z +z it +west side +ul la +to it +shif ters +sh river +pan i +must fall +eth ically +br no +in cess +hu sh +happine ss +far qu +fai rest +f ka +eli m +ecumen ical +ec ity +dre dging +dc fcfans +am ichi +å Į +sp ak +sati sh +pu cks +match making +go enka +ea sement +ci der +c mu +b z +aqu as +ðŁĶ Ĩ +{ # +smoke house +separati sts +saf fron +ma at +l loren +iro quo +cor ns +cassi dy +ah met +z hou +ck man +yor i +social ite +pro mise +plo tted +metic ulous +fluore scence +se wing +sa it +hunting don +gi gat +cascad ing +atl hawks +ue fa +nor i +is son +iac aucus +gir a +der ail +bad boy +way to +too wo +se u +rath ore +pra shant +ol ay +oce ana +le vin +in dent +heavy weights +guer illa +cor o +alay sia +x ing +uc ous +storm zy +sky light +sen dai +sch s +pa il +mac miller +leo dicaprio +hell fire +g dansk +fa han +aa w +south well +ram bler +persi sts +brut alist +an men +tari k +se oul +popsic les +man sour +daes ung +carra sco +sas a +mat tie +maro on +lever aged +kom odo +h sc +fac toftheday +cur ses +aff in +ðŁĮ ĭ +à ² +yvon ne +w mur +meadow lands +gra vel +fa hey +cze chia +tupper ware +mar awi +lac azette +hero ics +far i +del aware +pe cial +pancre atic +out and +la zz +e vel +toom ey +tam pere +squid ward +sop a +shar man +er nest +black pool +ao ife +air i +y eri +sig mar +ni han +liz ard +hom ed +dry den +chu b +blac ke +aldu blo +re iss +olympi acos +love story +lent icular +lan gue +jcc aylen +i hc +ban ked +trum pets +sx m +ob inson +ma homes +k nes +dissemin ation +pe derson +or bs +ner dist +lar ies +fon te +expedition ary +ex a +dam sel +chi ang +ab on +âľĬ ðŁı½ +ta v +sil o +plan ing +ny wx +m ú +data sets +c rick +ye ay +vol e +slin ky +m srp +c pp +bex ley +ve dra +open rp +mysti que +micro phones +dra ghi +atri al +?? ) +yu ki +xenob lade +words worth +u sted +influencer marketing +ds g +as ingh +roz ay +rail ings +m ks +kan an +intimi date +bat ted +barunsob ti +asho ka +anne curtissmith +so ares +mercen ary +lg f +eyel low +embar ks +ssi ans +lung cancer +in fini +ec c +brun son +bloom er +ar can +walla by +tr ici +showr unner +r dg +net to +mi zes +erup t +dur ban +baby y +tric ycle +sam smith +pre z +pe pe +gal low +g day +dist illing +ðŁijĮ ðŁijį +ðŁIJ ļ +tw ic +see e +job seekers +why ilove +poster ior +li rr +freed man +e ge +do xy +cur y +ang al +suzu ki +mc qu +ct w +antic hri +stu f +spring er +pow ell +om aha +marri ed +go home +g vt +fox trot +uc p +ow er +fit ch +con nie +wander lust +sym bio +sunny day +mic ah +haye k +cor mac +boj angles +é es +right fully +on a +nor ah +k cc +hillary for +h mp +bru in +ath lone +action news +ì¤ Ģ +se hen +pu lau +priest ley +posthum ous +pe do +nai doc +ling ard +le ben +jum per +... - +pet tic +ob serves +nuest ros +now smoking +night cap +mc p +lan downers +k ta +eng ad +ele x +alle z +w z +un ai +md f +jard ine +al ax +wh aley +tec no +son dheim +junky ard +insu rer +e hl +de vote +car ra +ca id +ca diz +ar thi +us ch +tyour self +per n +natgeo travel +ic ism +amo g +scuderi af +que m +pow wow +kur tz +head bands +gla zer +getwell soon +gam enight +euthan asia +catap ul +asy mp +sol ano +red hill +pu ffed +ap ura +allstar game +ye sp +sal war +nag asaki +medieval twitter +loo sen +inst ax +i sti +go si +contempl ative +chanc ell +appo inte +tal ley +corn well +war dens +v ate +sori ano +rad ley +pu a +m cro +inktober day +ero th +constip ation +ðŁ¤Ļ ðŁı» +wil mer +vani er +val lado +tic ons +reconc ile +mort als +li bby +h dp +yu gyeom +ym ac +un bound +sch ach +sb sb +plastic free +napo le +mum mi +ligh test +lam ent +de vised +un intentional +sheskinda hot +saint s +place making +paste uri +ke ir +kab oom +in san +dool ittle +cassi us +te va +shutt le +row dies +dra k +bren na +blu m +ki ii +ker ton +deriv ative +bu ts +bar zani +ador bs +ad ami +zu cker +time isnow +su lli +sat i +sabar imala +min na +- >: +ðŁ¤ ķ +stu r +princer oyce +marvel studios +itali ana +hahahah ha +gg ard +flu tes +xx xxxx +pin ski +n pd +must see +immort als +gar b +fi estas +dor a +bi bles +angu ish +sco trail +pa se +lazi z +ivan ovic +chronic illness +ar me +zu lu +tech stars +ro cher +ric ar +plo y +glar ing +des antis +cancel ing +ab ha +; * +. "# +ðŁĺĬ @ +y ch +vas i +skybet champ +sha una +ke ty +goggle box +balcon ies +ag as +un tamed +t ce +ne i +montgom ery +min ot +lombar dia +brain washed +bo tha +block cha +âĺºï¸ı ðŁĴķ +te te +star tin +ric hs +new sma +fab les +fa rer +de feat +cy an +compan ion +bio diesel +ben g +ðŁĴ¦ ðŁĴ¦ðŁĴ¦ +mail man +lin cs +fc bayer +bla sters +baccal aureate +attle boro +vic eg +limass ol +isak hi +himy m +franca is +da real +bush craft +ann ou +z aky +vie ja +kirk by +ca ste +a en +zd net +fac et +ali x +è ° +raj nathsingh +n ini +mc cloud +il ano +du ous +dil jit +dal lah +cycl ical +cur acao +âĢĭ ! +wr k +west boro +rat o +r z +ne gra +land mark +john stown +interfer ing +hann an +duc ated +zin ke +tamar ind +sub title +ro eth +le at +kar ls +ka hoot +insta art +hi aleah +embi id +drum line +assess or +anore xia +~~ ~~ +wemb ley +ra jap +ouni versity +jal al +copy cat +ak as +r jd +u hhhh +re flux +mor mon +mb f +lucin da +j drf +fon taine +defend the +crack ling +ë Ł +uk housing +tb m +shi vers +pri m +per in +in for +h ba +furry art +cav ani +ale se +acce ssion +zin ta +wr ongs +worldre fugee +tam bour +reb ates +ple ases +paramilit ary +ool ong +l mc +facilit ates +espan yol +car ling +k of +hu is +brew er +a hahahaha +will son +sh ure +sain z +play hard +lu p +lau b +gour d +dun ked +vi vino +sw restling +pi en +o ks +l leg +euv res +do zier +canadi en +blood stock +bac all +ठħ +vol ver +e sti +do sto +car b +u pping +say fie +ro she +m ru +emo tional +dit alia +book shelves +ab oo +s lane +on tem +mon drian +laogha ire +ds f +conflic ted +charlotte town +ðŁĵĬ : +whites nake +uni fying +tsi pras +smar ter +fron ti +f dic +emb ers +dece iving +bel air +ac kered +ur su +taste ful +sti let +pas ok +k sc +inv ari +in gof +dar tist +clin ician +chef life +bla b +avi ary +ðŁĸ IJ +stream ing +out ings +natural beauty +gil roy +du ets +animal welfare +ad air +sounder sfc +sashab ank +ro lando +qu aker +me ses +ka jal +jesuis charlie +brucele e +u albany +re ts +install ers +ilo cos +fin ca +ar kan +ðŁĺ¡ ðŁĺ¡ +shru ti +sha qu +leak age +cy cla +bre aded +ander sson +th reading +polaro ids +korean updates +hard ball +flock bn +cw g +sauer kraut +os we +ine w +du ped +à¸ŃภĻ +un reasonable +tour billon +tat su +pab st +optimis ation +in significant +il ah +cn tr +!!!! " +sl f +ra pap +o dor +masse ffect +lac lippers +ur sing +reco very +ne ff +mon do +gymna sts +fin negan +ct m +bro wsers +w ert +pag et +ou d +ong kir +nas sar +mind ful +k si +bak i +] ) +ðŁĺľ ðŁĺľðŁĺľ +non chal +men e +ly cra +bay view +barn um +âĢ¼ï¸ı âĢ¼ï¸ıâĢ¼ï¸ı +virtu ous +tre es +pal ak +mid century +kel le +jof frey +ho ppers +tor os +se pia +pay itforward +minim ise +fresh en +el u +d pp +ce us +å° ij +yu ta +ton ian +the king +te jas +rand all +me up +gut sy +dor ms +ch ore +½ ĺ +tsu i +thu mping +s va +girl scouts +bla h +ðŁĺĤ ðŁĴĢ +stor yof +ske g +san am +kumb h +ic ts +go lem +coffee house +be more +tweet fleet +tol stoy +paul sen +film noir +deduc tions +bear ded +be len +be ke +ÙĬ ÙĪ +win win +opau lo +da is +amo eba +sun y +london derry +gil ly +davi dj +craf ter +water colours +triple h +super tuesday +papy rus +foo di +demil ov +christma stime +ber tie +v ities +to il +sky lark +mi pim +foodand wine +fl c +usa in +te ers +rom com +morph ed +ho cke +hem lock +fire storm +comp te +bo dle +vis xx +scor ched +re ams +jones boro +choo ps +cheeri os +ðŁĨ Ļ +â ħ +n ity +miri am +me trou +man de +indie comics +bree ze +boom in +bol sho +android dev +aero dynamic +sm en +kri spy +in nu +im ps +ham id +fas a +emper ors +eli sha +ëĶ Ķ +in q +i zzy +bre sson +ãĥ³ãĥ Ģ +syndic ated +sashabank swwe +ou a +kah ne +june teenth +go tigers +diljit dosanjh +can tal +ball a +asc ended +v ga +t ass +made it +hennes sey +distri butes +top man +ti ago +the game +nav as +mur at +mtvlak pop +hol ley +bau bles +am rita +/ ? +ìĤ ¬ +ëĭ ¤ +âĿ Ģ +k ä +cel lo +bio fuel +ashu tosh +adop ter +] ] +ðŁĺİ ðŁijį +sw y +pete y +home ownership +go van +ecol lege +bres lin +bbc strictly +aga i +yum i +wil lett +w isn +tr visxx +solsk jaer +so vie +sle uth +ot ley +m ame +honey bees +bal dy +ðŁĺİ # +un na +ti more +the ti +progre ss +k was +jama at +h fx +crime fiction +c ado +bad d +anu bis +ê te +nom a +h ps +vintage jewelry +to fino +stra vinsky +go ffin +gh q +ef fe +ch acha +cam eos +n ci +lim m +invic ta +fle urs +en cu +brilli ant +ban sal +alan is +wan ee +spr inting +long mont +klar oline +feed ing +ðŁ¥° ðŁ¥° +un sustainable +tw of +luci us +ld sconf +hubb a +fri o +de hra +ur in +tre me +n mc +com es +bun bury +b cu +ad ina +£ ) +za b +ske ptic +shop rite +sand o +re shaping +magne to +kel ley +ilove you +cri pple +contain er +wham my +wfa aweather +techno logist +hl man +dynam os +cru ze +ble s +ati d +ork shire +tt an +poyn ter +pac a +mar mo +hyun dai +wi eld +sabrin aann +quick books +petro chemical +nol an +les den +mc lisse +lit chfield +gur ls +done gal +debut ant +com is +am ul +vigne sh +shi ge +re portage +pru dence +poten cy +pan hellenic +me ms +kling on +impre za +eri es +cash flow +winne bago +uter ine +ss k +do bby +canv ases +vau deville +pradhan bjp +myth os +medi ocr +mattb omer +g ns +final ised +election night +d pradhanbjp +coch ran +calm ly +amag azine +ç§ ģ +wareness month +tar ga +ta hun +sche tta +mu tter +liber t +hal low +de ities +chrysan the +c ms +am ents +sabrinaann lynn +ri ghted +r wot +ko i +ha feez +g dt +ul ls +u vu +ol adi +metro parks +it ness +extingui shed +simpli stic +pl ath +menag erie +kcr w +cam bi +ðŁĺį ðŁĴĸ +worldrefugee day +kel vin +i message +emo s +ah ca +sk illing +share this +ov sky +mcin nes +inter city +innov ates +gor die +g ons +fusil iers +duques ne +artofli ving +advance d +ph ane +oli va +met is +mal loy +jer rys +fu ming +follow for +f bla +eyel id +ak enya +ac ara +yu va +x code +wyn wood +tor te +te gan +superint end +sh inde +ly ne +hay market +england cricket +at z +ðŁĴľ ðŁĴĻ +speaker pelosi +pon zi +ole miss +men ter +keo gh +do td +bun t +bdn mb +are y +mathemat icians +fi ance +ce cili +spot the +pilo ted +escor ting +av ali +ale e +ðŁ¤Ĺ ðŁ¤ĹðŁ¤Ĺ +weapon ry +trapp ist +tho l +sur rog +reci ationday +plain field +phi les +os as +mi kayla +ma ham +favour ite +bl ane +ba sto +auer bach +vit esse +one republic +di ma +caer nar +we trust +sa jid +peek aboo +mam moth +hatch ery +h re +grand dad +dail ym +correc ts +californi ans +ë ĮĢ +ster ile +land sat +in fielder +imperson ating +hypothe tical +ठĨ +ul ta +the xfactor +pas sat +nw r +he ss +atlanta falcons +ah madi +âĹ Ħ +tu lip +ti ra +this week +spar tak +ot f +l enews +at ical +newsp erth +ha ge +car box +Å « +t pb +succes strain +sel man +sal umni +mor te +mor ro +lu k +elo ise +cust serv +ðŁĸ ¼ +ri bo +f tisland +do ble +cu ma +clinical trials +bla ding +bergam ot +ti zed +si v +day ever +by o +avat ars +alle giant +ðŁĺį ðŁĺŃ +yar ra +welcome tothe +um no +tur ing +tinker bell +ter ton +swo oning +regre tting +pin nac +pat rol +iv ar +im manuel +ig lia +bar ds +vit torio +rac in +park side +ktr trs +gu el +ðŁ¤ ¯ +whitt ington +sanje ev +s festival +o rel +miami open +jan es +intensi fied +ig ar +derek theweather +barbe que +ðŁ¤ IJ +âĢ İ +yun us +ny post +abudha bigp +semin oles +ground work +fu ss +eamon n +du ol +col ympics +chi an +bo oms +ani ac +~ * +y omi +thankful thursday +tech tuesday +la ur +iphone x +ha aa +fla va +e ku +d all +cor tes +care w +blun ts +ðŁļ ¶ +îĦ Ĩ +val tter +un written +top sy +som al +re generate +man chu +hun nam +hoo ts +e com +ðŁIJ Ń +Ú Ĩ +gn or +gl ens +bartol o +avi d +antibio tic +anc i +star key +ru he +practic al +penny wise +o cular +jim marous +calvin harris +ane mon +ãģ Ĥ +tat ts +suff er +sp ics +on kyo +o jai +greg orio +big elow +ath i +an ta +ðŁĩ²ðŁĩ ½ +zo olander +sha adi +kho v +di zzy +ann um +wo m +th orium +summari zes +scam paign +kc tv +ju mb +bit z +si al +r mit +promp ting +f cm +by me +bron ies +wj z +ti ya +ri az +re sign +provinci als +go vs +brock ton +ãĤ ¬ +z orn +ola the +leh mann +juan ita +g dr +dis bur +tab ling +pi azz +per plex +milit ar +gin seng +fred di +fire birds +f ags +dig g +aug sburg +ac as +. & +ðŁ¤ Ł +stabili zer +sa thy +re gt +ni ker +mode sty +fo garty +cap p +bal oo +bal ac +ar oy +al corn +agu sta +un zi +stoo pid +pue blo +micro chip +k affe +cut lass +clin tock +tu il +startup life +speci ale +pp pp +mi ent +illumin ates +e spe +d der +pla stered +pas sive +o jo +nay lor +go te +sp and +rush ers +me b +lyn ne +kyle busch +gic lee +ee g +destroy ers +cap rice +as mr +ab f +preak ness +orphe us +ni bs +ko ji +be sse +ac anada +year sfor +thi em +san cho +philli ps +ne f +local food +ki en +inver clyde +den ergy +st paul +plu r +other worldly +li er +l q +depos ited +alope cia +ëĭĪ ìĹĺ +p bs +magne tism +long weekend +l ssc +fav oured +fa ç +ce y +ric hey +moom in +mole sted +life less +in hibition +fa k +cat lovers +blu el +le ot +beau lieu +tou ted +the fts +i frs +home stead +gg yu +cy g +written river +pe ña +ov ar +o bu +neb preps +kerou ac +forevery one +bol ling +ï £ +wu han +un til +semi finalist +persu aded +new berry +mutil ation +medi um +kr t +con da +xox oxo +ti po +lo th +j il +go eagles +dab ang +yay y +we hr +ther ings +ja eh +dang an +we pt +ol der +mcla chlan +magnus sen +dispers al +cap gemini +ðŁİģ ðŁİī +stark ville +or lan +abor tion +t outes +legal tech +lead lap +hanni gan +as ers +ver lander +sw bb +s com +little ton +fe h +empower ing +dead wood +al go +ñ as +val eyellow +patti son +d abad +buck land +ro han +pu du +ma ari +ine fficient +cra ppie +ch swx +br ca +z anne +shostak ovich +hy una +de acons +canadien smtl +byr nes +abandon ment +âļ Ķ +presi ded +predic tive +per ma +mnu chin +maccab i +ha shim +die te +antiqu es +mt m +mm da +johan sen +ex change +clut tered +ti vism +love less +id ler +bb va +am arie +all ll +air liner +yen naiarind +vern acular +tec tonic +pur gatory +photo shoots +or man +mel ina +khawa ja +ken tish +eb m +chrissy teigen +ðŁIJ ĩ +un resolved +ultram an +the heraldsun +ren ch +cab ling +bix by +beck brom +´ ï¸ı +the ism +th un +simp les +popul ous +pad ding +mar on +crun ch +catamar an +be est +zoo s +sush ma +rel les +mccr ory +ge f +ev ra +rev lon +oo ak +mit ro +ki os +ðŁĺĤ âĿ¤ +ðŁĺ¹ ðŁĺ¹ +w z +stra di +grou ped +ge x +family law +eu an +ear my +confi g +abdul la +ðŁĵ ŀ +ðŁĴĵ ðŁĴĵðŁĴĵ +she ith +quote stoliveby +of fic +alli anz +zal mi +tenny son +stor age +pol led +mac ao +lips comb +im er +dj khaled +cancer research +bbcin tro +up f +mu es +ma gritte +hyper loop +flu ency +edmonton oilers +co vey +bel low +bar ba +pau ley +etu de +e it +broo ding +at ori +ðŁĩŃ ðŁĩ· +tooth less +ha worth +ge b +bur p +bi bb +zin da +tro ts +ca shing +be ep +ðŁĩ¯ðŁĩ ² +¡ ľ +war r +shri mps +pay able +dimini shing +b tr +å ¯ +r cm +ouro cean +no ts +mil i +j deep +duc ati +bak shi +traff ickers +ta hini +pro am +ho yer +tom ic +n ce +mar ron +ki st +kemp ton +cal abas +c vd +^ )/ +ðŁı Ķ +wor sen +side board +sad o +rock on +ij in +h mc +ðŁĩ¿ ðŁĩ¦ +water islife +te ix +sty ler +sarde gna +oper atic +nl poli +hoff mann +com anche +boat ers +ðŁIJ ¼ +ï¸ıâĥ£ . +not ches +g ash +excav ated +dom me +dese cr +char ing +art and +!! ] +sur fer +mow bray +mat lab +kati ec +inside out +a shar +ðŁij Ĥ +ëı Ħ +von n +un apologetic +seven fold +ni ak +mis syou +kab uki +har tn +care ll +cal ic +bat ley +ap it +" // +tr ical +stra han +se tia +main frame +love parkrun +kait lyn +bo vine +alej andro +z us +tw oo +mal ts +dr p +cath ay +ðŁijį ðŁijĮ +zan u +video grapher +vez da +thou sand +tar an +rein vigor +inspir ation +grou per +dd dd +col ds +chur ning +be seen +automo tive +* # +ver son +numer o +michael kors +ka ala +fotogra fie +bc f +ur f +tan trums +ro sales +min ate +ki va +il in +de fied +athle ticism +tu cks +throw backs +roth ko +ro gues +per idot +new seum +national petday +erdo ÄŁan +erasmu splus +cad die +be heading +spectacul arly +sp rain +seg way +post card +mano har +ing p +he aney +schuyl kill +s anger +migra ines +m st +gor mley +ebene zer +battle born +ðŁĺı ðŁĺĤ +umber to +sm k +saturday night +palm springs +eye son +dre cords +clari fied +âĦ¢ ï¸ı +terr ors +stati stically +par tofthe +naw azu +disc ourage +bou gain +yan kee +wish ful +sla shing +oni ous +iri dium +ff el +elev ating +crew sc +craft shout +br anco +ac ri +abstract painting +bro oms +ðŁĺij ðŁĺij +un masked +super ficial +pine y +par king +our ney +lauren s +hydro pon +hand y +d ells +cel ina +au de +at ico +ðŁ§ Ļ +re di +profootball hof +nb s +fa ints +ar aja +win kle +un tol +seaf ood +scot ts +kee per +i feel +go wan +g auguin +fam ers +bü sum +brown low +am ul +ìĺ ¤ +wal liams +tsu ki +señ or +sch indler +mur phys +laugha ble +gor d +escal ate +e oc +ye swe +un solicited +spra wl +le bowski +he mel +grow lers +gar uda +ap rons +thel ine +nor throp +nab j +kin sey +hor as +dallas stars +chit ty +bu si +bar do +ul is +straight ening +sd l +ra yo +mirac ulous +ko c +har die +do y +dani ella +costu me +t ss +st iller +plu mb +on demand +cot ter +w dw +uh f +today s +s sh +s illa +roblox dev +re districting +lo de +kh or +cover ings +ba w +ali express +peace day +men o +marin ade +kear ns +how ler +har pers +au ge +alla hu +sa z +ro well +revi ves +pul is +pre ppy +illion aire +ic el +chev elle +ç Ł +vix ens +redbull racing +plac es +pal os +os ke +mid season +mendo cino +k ron +geni us +gay nor +vic tim +sn itch +hyper ion +good food +sking dom +mediocr ity +live t +ku wa +i work +ha gia +fromthe archives +chen o +bann er +ah d ++ # +relax es +mal foy +fo sse +fire places +dar pa +corin thian +ðŁı ¢ +warran ted +um d +soci et +of love +gun ther +de main +vol ts +ti zi +klo of +ith waite +har is +h ky +firsta id +bee b +av ic +mul lah +lim es +j rs +hipp os +felici ano +whe e +un plug +r ng +pren tice +mor inga +mer ah +ma sque +k mbc +j hon +fare well +bor ic +.... ..# +ðŁĺģ ðŁijį +ìĥĿìĿ¼ ì¶ķíķĺíķ´ +âĢ ¿ +mas se +hurricane harvey +de vere +cy nic +yaz idi +ro ld +pon toon +mirac ulously +h cv +gar da +g mu +der ton +d ink +coy h +av ing +âĤ ¦ +th win +mo wed +martin sville +mal lik +life sciences +kiba at +ke ym +em r +dur k +coun tering +cor vallis +bro t +baro da +w any +vijay awada +sy ty +r ill +oy ama +ole miss +mor aine +loom is +kd trey +is c +indi as +hau lage +eng le +cre s +c ct +be you +stand ar +numer acy +l pl +bru schetta +wi k +q sl +pha sed +mix cloud +fi facom +comp ile +Ì Ħ +pent agram +monday mood +financi als +do th +debilit ating +ard t +ut u +tru gby +stre tch +sp al +san tosh +me st +lo real +gen ting +cre o +air cadets +ðŁķ · +wi ff +tri os +tou rer +rhi annon +o hhhhh +kier an +james maslow +flock a +e ww +ang ar +ab di +v hf +sound system +mi ura +manipul ating +ino cul +govin da +den o +birthday girl +bad man +ak ura +ab n +нÑĭÐ µ +zand t +par dew +mo ja +misogy ni +lind ley +empire state +ejec tion +atac ama +whi ppet +tu cc +timm ons +ps x +novem bre +na it +minic amp +exc ruci +caffe inated +smoo ve +revo ke +mccar ron +inter sect +duol ingo +ch aching +brun ch +ìĹIJ ìĿ´ +е Ð +w alling +take shi +showus your +ni da +jimmy kimmel +h ri +di ed +clou ded +sap ar +ra don +practic e +pim ms +kan a +head space +gat i +frustr ations +anno ys +____ __ +ro te +per tinent +orthodon tics +be kasi +aeronau tical +wei wei +vic ki +rel son +ra he +mar ry +do ak +chan ted +bbc countryfile +uk gif +qu iri +qu ag +much ach +loren z +k roos +for your +far away +dark souls +so k +s land +k for +future ready +car din +advent uring +zambo anga +yon ghwa +u mma +kan an +hand yman +fi duci +edge wood +domestic ated +consu lar +wn d +super hit +pa cho +mono chromatic +im bu +gla ad +dar ken +cor ti +byo d +bar at +({} ) +wein berg +the chase +or ge +miy agi +j ali +in ac +en atics +ban ished +b fg +wheel ers +neo liberal +mi mics +enfor cer +we aning +py p +i pm +her ni +cla flin +chitec ts +carbo hydrates +ae i +work week +ver ity +sleepy hollow +sani bel +per due +global bc +bin ds +sty ro +show stopper +par able +i dism +hin ata +fore casted +du ffel +de cent +bott omed +bbci player +at elle +anthropo logist +see able +re creates +raven na +pu ffins +mand al +fla ps +cou sin +cool ness +che tan +cbs miami +ao te +ai leen +u twx +u di +ram bling +o ggi +ne en +meteor ology +ir respective +illi ers +domin ick +cann oli +adjour ned +âĿ¤âĿ¤ âĿ¤âĿ¤âĿ¤ +wy p +ste au +o cial +k hay +h mv +ene x +dj life +u ko +tro pes +swar tz +pu yo +play house +patient safety +labou rers +ite a +e wa +deal oftheday +bog dan +wild star +wed g +the gap +pa ki +is bell +devo id +aw w +as b +was pi +te fl +sver ige +qui eres +mo en +asi o +afri ka +vi agra +twitter verse +syour friendship +pry ce +mon on +elm hurst +bring iton +videom tv +tam er +jalli kattu +foodie friday +federic amog +extrac ur +cal len +anton y +âĺĢï¸ıâĺĢï¸ı âĺĢï¸ı +ta ichi +rishi kesh +pand ana +n ung +la ver +gaul tier +fair ways +d ello +alcat el +z le +re frac +prote omics +pent at +or bits +mor gue +maver ick +kand ahar +issu ance +intertwin ed +i ren +git mo +faz al +err atic +dor mit +beauti fy +( (: +you n +mx gp +it zy +dam en +colorec tal +co sm +chri sk +c ita +apologi zing +îĦ Ĩ +te tra +saoir se +penit enti +man deville +lo dge +ja xon +j la +aph rodi +alter bridge +ç ¦ +te dd +re fit +injec ting +di stro +brig ham +year challenge +ra he +hit men +ha bi +grammat ical +george michael +bro x +bo ck +am la +tour life +stry ker +p all +marqu ise +gher ty +el z +cla pper +cataw ba +tisthe season +scrip tw +ma doka +int ently +gee king +galac tic +do the +bl c +ap tor +anci ente +al icec +ÙĬ ÙĨ +wool y +ralph lauren +jane austen +hun ky +dry wall +chen in +at n +anticor ruption +three some +the t +metal core +li ga +lex icon +eura sia +dor sethour +daily mail +vel vet +mou lton +colle tte +bra ai +ben ning +asi acup +wor ded +social work +shu man +s ich +ment a +kin sale +i hansika +du cey +dif fuse +cur sing +cordu roy +å ² +velve ty +ur inal +tucker carlson +temper ance +fro ggy +af cv +shan ty +l ner +good rich +ge j +eli x +ef ood +b of +artvs artist +vo wel +sut cliffe +sta v +se ous +ra ines +masto don +booka day +tag alog +ridge wood +progressi vely +lock smith +kan er +dic kie +cel los +break downs +bo ssy +ba al +aveng ing +sky landers +fa jar +ci u +be aks +b ere +ðŁĴķ ðŁĺį +su fficiently +may weather +de scar +bil i +tat o +revolution aries +kwa wesome +em g +cad ill +c bre +ðŁij Ķ +wheel chair +ten a +r amaz +jo kingly +har lem +go knights +enu mer +dili p +con serving +beckbrom fl +bat avia +ag gregation +ta ip +stat ure +selfish ness +scuderiaf errari +ke u +in sati +fix it +eric garner +entr ant +âĢĶ " +Ñ ĩ +wi ley +ty nd +pa ke +infl ict +bu law +black more +sacramento kings +ol ta +ker r +gra dio +bro snan +se on +po ws +patho gen +fra zer +en large +athe on +ware housing +ur se +fil ings +dis content +che t +wri gh +vacation ing +ta rek +str angle +nic ki +ident ity +dhan i +d ary +construc tions +cagli ari +age o +a fia +wedding dress +sch ed +per rin +me ur +in art +wor c +un settled +mini bus +mat ric +christ ened +bell inger +mo xie +kar bala +e hn +at tainable +ta va +sw ells +ro da +pi stol +p z +lis burn +k roll +fcbayer nen +bel lion +ro byn +grou pie +an ant +ĥâĸ ĥâĸ +ri at +ly me +ju te +hall oumi +glu t +cor te +vas sar +then and +terro ir +ring tone +musta ine +homer oom +fu tbol +fr in +bo ba +basil don +to ews +summer fun +it sa +in ia +im plying +hark ness +gn u +deplo yments +bir dc +bar my +ì ¢ħ +sten son +roman a +note pad +me mon +cellu lite +by gone +ator y +wood lawn +thevamp s +la sses +embaras syourbestfriend +affection ately +ws j +vision aries +ren te +po iti +hann es +ha ger +ge me +distribu tions +bas sad +waveleng th +van n +or me +neighbour hoods +jersey city +fu te +car adele +bru iser +am ed +sub a +bas so +ðŁĻĮðŁı» ðŁĻĮðŁı» +ë· Ķ +â¬ĩï¸ıâ¬ĩï¸ı â¬ĩï¸ı +ss unday +sli fe +skar sg +li an +gallo ping +boc ce +tou che +ro my +ou la +n ll +mo ir +le mony +is ine +hoo ver +evacu ees +dd national +addic tions +ðŁĵ İ +ri ker +nca a +ka ia +h anya +dayofthe girl +crust ace +acrob at +white field +vill anueva +vallado lid +s mor +s ico +ro ping +open mic +gen ia +fast lane +eci gs +dod ds +board man +zin edine +u che +q at +n abo +m wa +kon rad +knock outs +i of +co lic +weight loss +w pb +shri ke +re vert +library congress +gate fold +åĨ Ĩ +se ad +sam u +piran ha +om ena +g aven +dayo ff +cray ola +y ai +under sea +shap ers +perl man +hy rule +di manche +âĻ łï¸ı +your take +sung jae +ple c +o ik +neil tyson +jam on +id le +i go +i bra +cast illa +brem ner +bot w +ðŁĺį âĿ¤ +pre fab +men orca +maine mendoza +glori fied +divyas pandana +determin ants +black sburg +ìĽ IJ +make red +ly in +ic w +filip ina +est i +del ves +dat uk +absolu te +whal ers +tt al +spaw ned +newin dian +mix er +lead off +kash etty +ha ddad +gau ahar +f ct +eis ner +ate in +w sm +th eni +school children +rel ls +orchestr ated +octa ve +ob sessions +meg adrive +jab i +ideo logies +har tt +fe sto +boo ting +band h +bacter ium +won g +tw ing +lepro sy +k vit +bye lection +bat chel +alter nat +reli ed +ke el +fresh ener +ec lan +cor i +chi ko +aw ed +anil kapoor +whis k +var o +shi o +san ia +laure ates +j rpg +guineap ig +grizz ly +doppelg anger +bar rows +a hon +ðŁijĩðŁı» ðŁijĩðŁı» +Î ´ +ye v +woo tton +sn ps +re collection +ma al +ket an +fe west +far az +cd w +bi f +shah baz +qui er +hust ling +hern don +deu ter +cl u +adul tery +ru it +gre na +gn omes +free hand +ref ills +po tting +diagno sing +ban nister +al ms +ag et +sam rat +s vi +pe tri +o virus +mid lothian +ëį ° +scal ability +me tac +just us +individu ality +usa c +the karanpatel +s bar +re shape +pa war +jf k +is sac +custom ary +bol der +Î º +r mx +pla i +ber nice +ar cana +ntv news +melan cho +h ö +bro aden +andali olo +ðŁķ ĸ +ze h +ww t +us ma +substan ce +ray man +kr ati +de ze +bron er +ðŁijį @ +ze ec +x wx +sil very +kac ey +ic ardi +et works +ba st +aap a +ðŁĶ Ľ +vap our +smu le +hu ang +end ar +dis located +cari be +) .... +ðŁĶ¥ . +vo to +tu tan +tele metry +pa karmy +opini on +o tome +nz pol +is en +e amon +co bbled +cit rine +audit ory +ween ie +uk snow +long itudinal +har pers +gni er +fasten ers +em met +cross bow +cau tionary +bre aching +yam una +wash i +swit zer +swee tly +spar is +sp ilt +nig am +indian food +fin alize +bach ata +wi w +spe wing +ra do +dh c +an ow +kis se +go frogs +alpac as +âĻ ĺ +red grave +new era +kid der +hel ios +del on +c anna +ðŁİ ª +xi on +stone man +polyure thane +ni do +mr ng +mac a +iso topes +co fc +twit pic +round trip +numer ology +n ooooo +marc marquez +just ly +ga ir +french bulldog +fish and +felix stowe +er k +bag e +ag itation +twit s +moment arily +wi st +saira jdeep +n tu +mur o +mc gi +man hood +man ford +love eee +l ta +ha dri +fer ro +doro thea +beach body +arri ba +angu illa +vape fam +spres ley +sli mmer +sday time +ophthal mo +lac ing +j ür +grin del +din ah +ce x +c sun +breath s +bra bham +war f +sp ang +ku bota +hay ne +h mo +gv su +go sa +fun nies +cre a +zak ir +stru mmer +kur tis +h kt +ch aka +bach man +b jer +adventure travel +y ves +tor r +nitt any +hi mes +cen ota +bay ern +иР² +ur ses +sn ags +saura bh +nico sia +nick ed +in shaallah +friend liest +ever note +austri angp +ar mou +anthropo logie +skag it +shrou ded +little mix +hello kitty +eli z +br é +apothe cary +amphi bians +tro p +tr t +suc ces +rup ture +metrou k +ky t +gla sto +g int +congratul atory +volunte ersweek +video clip +swoo sh +neu x +man power +format ting +fl r +fer nando +deal ings +thequeen mzansi +shawar ma +shand ong +hurricane irma +con vul +yo han +tr us +re forming +r ter +lax mi +ho hen +fu turo +down grade +dehra dun +boo ts +b ct +aaaaaaaa aaaaaaaa +ðŁĺĺ @ +shill ings +s ge +ou le +e gon +du pree +dri bbling +contradic tory +canton ese +avar ro +ze enews +y adi +was atch +th alas +rv smtown +o ap +ma dinah +ber ton +éŃ Ķ +ter yx +ss ant +sc av +realmadri den +park s +ome tre +hl f +re signing +ki ana +k cs +gal ine +el dredge +co han +anthropo cene +ðŁļ Ļ +ãĤ » +things that +ome ga +om bo +ny an +le gia +instrument als +firstdayof spring +ecuad orian +dic es +chautau qua +chas m +ðŁijį ðŁı¾ +wit ten +wang i +no bles +chan el +castle ford +bloss om +whole heartedly +v ab +un aids +pal tan +off c +meta physical +cor net +car bine +acknowledge ment +radio city +mal ach +w whl +total ity +r sp +power up +mar tel +ice day +go ings +g me +family day +es k +cb sdaytime +yam ada +wn y +spe th +os walt +man heim +make comics +in securities +ici us +ha ge +âĸº âĸº +won ho +m action +lo zano +k uk +jar ry +indi visible +in nit +go er +ff i +dut chess +cle mons +cla ssed +cham i +back end +wat u +war games +van illa +ru bin +neop rene +lo x +gly phs +develop ment +char ger +cesar o +x c +van guard +poe hler +pau ses +p sc +mis bah +mad ura +eli very +de coy +d ouro +coast path +biop hy +ìķĦìĿ´ì ½ĺ +with drawing +schwei z +sarde sairajdeep +san ji +proven ance +pic ker +nade em +he hehehe +form by +en ed +elvi spresley +ku du +ke at +kam eez +curios ities +cr amped +childre ss +wra ppers +wolf man +st ell +passion fruit +no sh +ni eve +fang irls +avon dale +z ace +sar ang +preserv atives +lo co +ig l +hand set +hai lee +ge i +g be +distin ctly +bring in +f enix +enf ant +elast ic +don o +commer ce +budd ha +wh ang +sz cz +roa dies +retin al +mc ghee +halli day +cu tie +slu m +cosmon aut +yoshi da +t ney +t ge +sm riti +d ls +at orio +ali e +ìĤ¬ëŀ ij +tink ering +ske le +rath bone +pr g +phon ec +mc w +lifetime tv +lead up +dy r +spho tos +pu ffer +prospec ting +osa urus +nv m +mor phs +maur ice +m show +le grand +iran protests +cartoon network +bet i +acrylic painting +ab id +ģภģภ+ðŁĩºðŁĩ ¦ +è res +wait ingfor +min has +leh enga +bag ans +a or +multil ateral +lig ne +hot shots +de classified +wish ers +tiss ot +mess aged +lo on +kul tur +kil ometer +ital o +fer rero +co pier +bar net +shal lot +sea view +dri ven +com press +chic ano +bou vier +âĺ ® +time flies +sal ty +rother ham +rex ha +ni al +i story +h town +chi v +afro beat +yellow knife +vil s +va sive +sin fonia +ponty pri +hou zz +di ble +âĹ ¼ +wine making +w ca +van re +scho oner +os r +na se +mi zu +klo bu +journ aling +fa ker +emmanuel macron +an jun +win t +j ari +impin v +earth athon +di ffers +c gm +supp lic +stay in +sieg fried +ni val +j ith +ho cking +u hr +shab u +hot test +g any +bigre d +ðŁ¦ Ģ +ï¸ İ +swe des +pupp etry +prin se +mc donald +fran cia +at ino +ar yn +ultr alight +the j +ra dar +pre caution +ly a +kasper sky +jeff eries +in fir +gaz zetta +face less +diver ting +chrome books +agh a +ab normally +Ù ģ +sho win +shab a +psy chic +ja unt +de formed +awan inews +a ily +unfore seen +picture ireland +n gt +down y +dalhou sie +council woman +cor nyn +bet sey +wing span +par id +ming ling +loc us +in no +husk er +fl ys +carroll ton +tr icity +scra ped +safar icom +occup ations +nawaz sharif +hoo ves +cathar ines +ag ger +à¸Ńภ£ +wad dle +syl vain +st johns +so yl +ol ds +g ack +fire men +fin o +en tex +de constructed +bc p +asser t +ب ص +wh ow +vari an +ne sta +max well +kru se +dr b +cool ant +aw kins +al et +ab rown +ðŁijĮ @ +smashbro sultimate +ir rig +cobble stone +cas anova +buzz ed +tele kom +should nt +pt p +memor ia +cham isa +alme ida +wi reless +re visits +no ize +ne go +la garde +is th +is ac +extingui sher +es an +w cd +se ful +dead lock +ðŁĺħ ðŁĺĤ +wen atchee +sla g +sen za +o in +ne hill +ko vind +kan ter +jo be +ci a +cat ers +agh i +suni ons +sop er +sli z +pac cio +mo sh +ma ddy +lo rence +herb icide +grati fication +cu it +bar bell +? ". +l pd +kil mer +car no +ball entine +sh iner +ne tta +loo kat +il ocks +iam the +ch ola +ul an +tr fc +termin ally +ori st +o gle +light bulb +zo or +web store +wait ing +render ings +poetry month +parach u +miniature monday +metro link +m ple +kre w +empha sized +car rot +íĺ ķ +vari us +roman ticism +mahesh babu +lake show +jol y +cormor ant +break in +ag ni +v av +shack leton +po of +mass ager +man ay +m br +kag awa +brew ery +att ila +ade d +pav lova +lan ning +king khan +i ata +fl our +dun ning +az awa +are th +yee haw +shel tering +se bi +ru pts +pin i +nar rates +far rar +cho kes +bo ssa +snoo ki +sep tum +p ounce +my team +my n +metaph ors +mag ine +leaven worth +lay man +lat ch +hi jacking +hard away +gu gu +godbless america +dil ip +cla r +brun o +ا٠Ħ +wer ks +vision zero +t whi +sei ko +ibm watson +emul sion +bho ys +ws vn +voice mail +v cu +robo tic +ro k +o woc +mari ecla +cric h +av c +su bi +shivangi joshi +hai da +s keeper +mid ler +kn ackered +kirk patrick +kil leen +i fc +y ala +vege tarians +sub terran +shat ter +mdc ps +max ine +mat ta +amic hele +pro claims +pri sma +h dl +for de +fo ams +end less +bill eric +an si +ti rana +smoo thing +roun ders +man ics +koo kie +invin ci +ari ad +adop ters +timess quare +ta ec +sco li +s wash +ou tt +o sho +gas co +fo c +dru pal +coy w +bus king +bat ti +ĸ ï¸ı +ðŁĺ© ðŁĺŃ +twal ker +the vote +o ha +man on +kri t +jose p +e inste +contex tual +caradele vingne +wee zy +som er +ro an +pro ton +oke anos +halo gen +german gp +choc taw +ta q +syste matically +sha shi +om ens +mad dison +focu s +ess ay +air bag +tsh wane +scho ice +pr é +hed wig +deze en +ab dic +ðŁĴ°ðŁĴ° ðŁĴ° +vaid ya +holist ic +down ing +desal ination +de are +adole scence +ou trun +nu di +má s +indie author +f agan +dof theday +conce ive +chest nuts +arch diocese +ac are +world war +gun g +g ada +cel ts +as una +à® ¨ +un cu +sun ggyu +seaf arers +red field +pis ses +odd ity +blow in +( ': +à° Ĥ +r q +nan ak +iri er +ha velo +cri key +chase elliott +c mr +bar atheon +sat the +newindian xpress +imple ment +div ul +delu ge +bla en +be el +ìĽĮëĦĪ ìĽIJ +ul timo +stained glass +ro lex +pla it +narcissi sm +mi gno +mal abar +leu kaemia +ing up +hot ch +d cr +chath am +blanc pa +ti pper +glas shouse +drag ster +dil apid +anim ators +w fla +toi let +pi o +paratro opers +mi stic +hir sh +guru gram +Ħ Ī +vie ux +sub sea +quin lan +nie der +nes n +li day +lau t +ampli fiers +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +w ami +over crowded +fir s +d nd +carto onists +barre tto +wh eni +uproo ted +stun ting +spital fields +smur fs +perfor ated +n fa +kios ks +him chan +fli pper +daily deal +brand new +ðŁ¤ ¸ +íĪ ¬ +re mb +mm mmmmm +iber ian +freak y +falk land +art news +ak ha +ðŁĺ ¶ +the cure +strath cona +sel fe +omar keting +om ani +for tw +brad dock +ðŁĺĮ ðŁĺĮ +ðŁĶ´ ðŁĶ´ +san remo +hu ma +guil lotine +foot bal +dun lap +dre a +cast away +but ch +sl ant +rout ledge +on sen +litur gical +grunt led +discovern i +bou che +and am +ðŁı ¥ +tuss auds +think pad +super group +summer solstice +que sto +notice ably +fi bres +ende d +colly more +buzz in +ai k +w ate +vivi an +stav anger +re produ +pancre as +gar cetti +ceme teries +bird song +arith metic +ten is +soo thes +post modern +mul holland +cn j +bi agio +ar tapp +antichri st +yol and +so be +run time +puri fied +prou st +jo m +godd am +far id +cru yff +ðŁij ¨ +un ig +ta chi +syn chro +pa sir +ob la +lind t +le de +dist iller +cry o +ca h +atro cious +ãĥ © +x en +wi dow +veg gie +scre wing +roman reigns +ker nels +cream fields +ak ala +wri sts +work sheet +mar su +mar nie +mac o +io d +con volu +ar les +. ðŁĴķ +mo te +j ds +ers for +du ty +div ina +animal alphabets +accu weather +west minster +quin cy +pou ting +n live +lat our +ketch um +gi le +Å Ľ +wood bine +paragra phs +nad da +ac tic +white sides +ung u +som ber +min ot +lati fah +horror news +hero isl +gem ma +sky train +ni em +ne urs +mon line +jay hawk +fe cha +fast company +ce m +armedforce sday +! " +supre mely +st exas +premi o +pal mi +nie to +n ge +abe g +âĺ ģ +x r +reno wn +mor ten +ga sh +ap j +ë¹ ħ +whats the +rain y +conceiv able +af db +t live +shi itake +r mw +om alley +ke ving +stun tin +espino sa +de br +constant in +art show +ze wski +z ander +summer school +mo rena +ferr ar +d wight +boko haram +slo ths +shill ong +ky e +kry st +equ atorial +capital weather +bi onic +bc i +bb mf +arche ological +aly son +acquaint ance +!!!!!!!! !!!!! +z ina +ye ong +th ali +red car +iti zen +he cho +gri gio +du sky +de grassi +bermond sey +b nwt +aram co +ab ut +wine makers +tu al +rhetor ical +hesit ant +ay aan +at op +ap ha +sel kirk +sd v +neck tie +jo inted +jo fficial +hi bern +fle xi +dow ry +chap stick +x anth +la ren +fla shed +eg x +bin ay +agnez mo +zu mab +ra at +mat suri +ly wood +jose f +har ald +bal sam +ðŁıĥ âĢįâĻĢï¸ı +shannon leto +sa aho +s anne +mo ans +gott alent +dam us +co e +bam ber +swallow tail +snat ching +sharpen er +ligam ents +ka in +evan escence +appalach ia +à° ¨ +the ir +skag gs +sau st +partic k +lin ks +john legend +i bo +gn an +twit t +n fp +em b +doub ters +bi ak +ad ria +âı ° +segun do +sapi ens +cm h +yadav akhilesh +win i +t pt +ron d +mau rer +margi ela +man olo +jec ts +ha wn +green point +ev on +atlé tico +scam med +n nw +key less +i he +hol den +attackon titan +voo doo +thi an +tau pe +nal ang +me ath +je i +i ann +hr tech +dar lin +blackex cellence +best fans +b wa +ðŁĺī # +vo z +rup tured +mam ac +m bu +lu gar +indeli ble +he bert +al aa +seag les +ruck sack +dav y +copy writer +am ok +ab sa +ror o +q amar +new wave +multip lier +mc adams +ly chee +latel ateshow +hi ke +gen er +dra ken +cul lo +as cap +where are +radi ate +power star +ms w +hon do +gas light +bre y +az oo +at erials +ãĥ £ +she boygan +regi ster +quinnipi ac +pedro sa +mu ffs +habit able +buck head +taun ting +princi pe +na ar +hi ba +duck tales +british columbia +sug i +road block +pic kin +op tera +le os +il ford +hand picked +da shed +bal los +acceler ates +un orthodox +trend line +sy cho +hex ham +ev ita +malar key +dor mer +bri x +alici a +adel phi +ro ssa +plu mbers +newe gg +nai res +jo dha +impover ished +h mmmmm +gal en +em v +defend ants +ðŁİ ³ +way farer +to ca +ste vien +sli go +perci val +jk corden +g pl +aer of +ac es +yester year +sc as +salam anca +rey na +pv fc +p tr +harry styles +dan n +ãģ ¾ +belle za +alo y +ab alone +xian limm +hur r +hot topic +home ware +eas ports +clashof clans +ber ti +an ad +v ca +st ach +square pants +shin zo +cor ks +ðŁĨ ĵ +what syour +sh ey +ra af +pri mus +narc issus +massi ve +klobu char +jor nada +ben elux +a ou +âĿ¤ï¸ı ðŁĩºðŁĩ¸ +tre acle +tion alism +st oun +sab o +jalape ños +dar kk +ci ma +bu ku +bridge t +beha ves +wim mer +national gallery +mis conception +epi ste +b na +ani vers +us ka +u soc +ne ocon +ly e +kang in +cry baby +cler ken +car m +re ga +par ameter +over taking +nu man +mor nin +job fairy +ha f +fil o +exceed ingly +end point +b kapoor +an x +amaz in +sau teed +mal ick +lu gano +front row +di en +Ø§Ø ¨ +ys se +sti pul +sr m +sc roll +rever se +no tal +key boar +immort alized +com d +arch way +aficion ado +up heav +ta ker +m ä +hou rof +dog show +mo oring +meat less +king scol +he ter +hand maid +cani ff +bo ssing +amaz ons +x lm +sav it +ice cube +don te +woo oo +ven trilo +sy ring +sen or +pondic herry +plan ck +par ov +p ening +mcdon agh +dwind ling +dar fur +credit ors +cra zed +cr j +an ong +mar coni +devi led +carmar then +bik ram +ðŁij ª +vig or +v fb +tro ss +to th +pe u +in paradise +dev out +que tz +mi dr +hag ya +fu sing +displa y +ben teke +amir ite +ty rol +tom atic +tic ke +ro bredo +kum kumb +hurricane matthew +grand canyon +chapar ral +cat ania +car ousell +seri al +seme sters +reper cussions +ouach ita +moon shot +ic les +how doyou +d sen +comix ology +children in +richard branson +read er +p so +g dragon +far ro +ski pton +shoe gaze +ni dhi +kö ln +green wald +smu ggle +shel led +sh of +hern ando +edu ard +am is +vau lt +more llo +m ll +inter generational +i ab +don agh +bur kin +ä¸ĸçķ Į +âľĮ ðŁı¾ +venezu el +v ato +sto pover +som bra +sal ad +pav ers +i bi +beaver ton +aerial photography +aber g +åŃ IJ +wy ck +progro ck +ni vel +mc do +land rover +esc a +bis d +ðŁĵ· :@ +s gr +re stin +nar uto +longre ads +deliber ation +a ight +ðŁĺ ¦ +ssi ma +ri bbed +intro verts +end re +ah r +ðŁİ¶ ðŁİµ +á rez +squ i +park life +mo se +dal its +calm ness +bc t +angeli que +un surprisingly +un necessarily +tor ched +sw u +sof i +reimbur sement +qu inox +may e +cy stic +clt traffic +ac ed +xi ang +waz iri +supper club +se ti +pa oli +ol on +kr g +ing at +u stad +u gl +twhi ddleston +phine as +ing rosso +digital nomad +ar to +ver milion +val po +sch om +penetr ating +ky at +hand woven +fle mington +( =) +w tae +tent acion +ste em +shri e +mp l +ic am +i pan +ation ally +ðŁį ³ +th k +reti re +re mission +re done +phar ma +ove chkin +it sm +donaldj trumpjr +crack er +barist as +ari ah +app liqu +aote aroa +ab scon +west wick +veriz on +sydney fc +enthr alling +chad ha +bn n +bi stro +ðŁį ĩ +trans missions +straigh tened +mol in +letsgo bucs +jordan knight +gro ff +freel ancing +fin gered +car show +ac in +nt fc +klam ath +hitch ens +gee bung +el vin +cre amed +bourgo gne +pie monte +j su +ha bana +gran turismo +aqu at +** **** +!! . +zhe jiang +twol ves +q wer +mb urg +im partial +hor d +har ps +gr r +g illum +dar by +b ens +ap b +air lifted +pale onto +no things +gr unt +c sb +at ree +afgh ans +ðŁĩºðŁĩ¸ @ +support local +sub stitutes +gu la +ba ju +ate gate +amig as +ab ell +ve m +tw ing +o don +long hair +is ley +gu tters +gre ase +g fa +fu mi +wul f +se ase +post code +e gal +champion scup +c sis +ali yah +y rf +w saz +sr fc +me gyn +mag net +kno wns +i hs +drug store +biomechan ics +aver a +wimble don +slu ggish +si mmer +science museum +qué bec +nok xl +man do +k lub +gran bluef +dü sseldorf +col ab +ch ars +boo ger +tin nit +ra fferty +ne k +mo v +hand out +ei u +cat skills +business intelligence +boywith luv +raik konen +rachel le +pro g +mt pol +mccre ary +com pote +child marriage +aa at +âľ ¾ +zak zaky +womens rights +tre port +tramp led +no tb +m ri +lucas film +lo stin +law son +jun cke +juncke reu +ho tty +syr inge +su ds +st ooth +ka ar +ity uk +inter play +hon dar +ho gan +fu ssy +exal ted +en crusted +c bo +absor bs +ãĥ ij +ter tainment +styro foam +reali stically +n pg +men orah +mcgin n +lan dish +i ki +hr p +c chs +yo self +shi vika +petro l +morphe brushes +men os +mck agan +k uni +gob let +davi do +beau t +bart enders +ðŁį» ðŁį» +west moreland +war planes +py ne +princi pled +pen sive +par s +need to +mar salis +local e +harper collin +gi v +ap riv +al tos +zace fron +z at +takeme back +sridevi bkapoor +py ar +pla w +expend itures +de bug +ðŁĺ´ ðŁĺ´ðŁĺ´ +z ok +s itec +ne fer +n na +ki ely +co ty +anim ation +an war +ye shua +royal operahouse +nf v +cur t +beat le +........ ....... +ä ¾ +âĢ į +sy phil +sy an +op ts +lu ang +hol yoke +en tel +do terra +bl und +anag ement +alum pur +si ra +reiter ates +parad is +kpk updates +e ased +command ant +ande ren +Ļ ï¸ı +too ts +nott ingham +ley fc +ec i +ec d +comp iling +bm supdates +berdy ch +ar ron +val der +stri kingly +snoo zing +si ento +nikki haley +mar lies +ic illin +femin inity +fat boy +cal dera +bon ey +boat show +affiliate marketing +ðŁ¦ Ĩ +win nie +win dies +une ducated +mac aroons +iiii iiii +critic ise +coron el +beng a +twitter ati +p cl +n mb +les bian +jacqu eline +hom bres +encan to +dog slife +suppor tour +ral ston +cine plex +ðŁ¤ĺ ðŁ¤ĺ +work horse +tour nage +sa at +new sasia +k ish +indic ative +chat ty +cali pari +blin dly +street photo +slu mped +reservo irs +lac tic +ble ts +w tt +ta jinder +sobr ang +ro the +la uri +idi oms +hor ts +cran brook +cb f +bulaw ayo +au ro +ze a +southe ast +par ale +ing alls +drawing august +co existence +ðŁĺī . +tin sel +syn chro +stedd fod +sh et +rp gs +poppy legion +out loud +in dr +eli jah +electric vehicles +co wh +chit ra +as ahe +yu mmm +vene ws +swach h +pc p +over ride +mu z +k ada +el bert +du sty +con cussions +brazili ans +ar ame +sna il +out burst +ni hr +mun do +jean nette +har greaves +fin sbury +fa yo +dylan obrien +se ssion +sd m +sc run +procrastin ating +gol dy +brid lington +________________ ________ +tr uly +mon ies +jour no +halcy on +fer b +ex mouth +all day +soft ness +its all +hard style +bo yl +az adi +uni formed +six nations +sekar pandian +nikon usa +nc i +master of +ice bergs +hair pin +demilov ato +deben hams +crowd fund +ash croft +ang ering +: )! +stu ffers +pushawards maywards +p yo +m tu +hand ley +é » +u on +tobi as +tal aq +sig ner +ru sted +no zom +magni fying +divi der +al de +:) )))) +!! .. +ภ¶ +ठ¡ +po ons +oil field +do tty +air bags +sl urs +rapap ort +ms me +klon dike +. > +why not +tw omen +reboun ding +mi ken +ho dl +fru ition +do er +cin que +certain ties +âĢĶ - +tiss erie +theprojec ttv +se ducation +jewell ery +in between +impre ssively +hair y +floo red +flo wered +de carbon +bari atric +adar shan +ãģ ı +water ford +tre stle +tann ins +mo in +gi sts +g be +brande is +boo b +behavi or +b pi +acade m +yam aguchi +penitenti ary +mosc ato +dusse hra +democr acies +bla key +bad dies +azte ca +ar cy +tru ely +squ ab +ghazi abad +bu gging +bal vin +am nh +âŀ¡ï¸ı âŀ¡ï¸ı +าภĻ +ri aa +mennon ite +ice ps +hey wood +we fly +sig nat +shatta wale +shab irah +moor ish +men tee +hudson valley +bas mati +? ), +tigh trope +sor i +raj sekarpandian +ne manja +lu zer +fre t +en list +el dridge +e hh +bett ingtips +apple sauce +ðŁĻı âĿ¤ï¸ı +âĹ ķ +trumpp ence +sol berg +po inte +ero ar +energi zer +death penalty +ch iro +ww l +fla u +evol ves +à ¦ +ther ain +slo g +sk ova +rc mp +kumkumb hagya +go dolphin +camber well +be ading +ax ing +z ki +war blers +une qui +toowo omba +salt lake +panam ap +ny p +mc cord +light year +je fe +itu tion +hydropon ics +car paccio +sho spital +mai da +indi erock +cu enca +bati sta +all access +____ ____ +ì µ +sab e +my life +e dex +ber ne +av ings +ani ello +stor onto +pre aches +head piece +hair dressers +f sc +ex patri +dana white +ts f +tal eb +stein beck +pin der +mol l +lu ge +lil kim +jin woo +camp ing +broc ade +al locate +ï¸ıâĥ£ : +áµ ĥ +out takes +monte video +lom b +fun ke +flet ch +ê³ ł +z nation +vi Äĩ +ve sted +shabirah luwalia +pur su +o ath +nas r +mer cato +dave y += " +wi red +uni do +t ili +re ston +fren te +di aled +cf pb +/ âĢ¦ +vel our +sle u +o ren +mad ina +ken worth +kell yanne +enchant ment +b ce +av ana +ana than +! ðŁijį +ðŁijĬ ðŁijĬðŁijĬ +ðŁĮ ® +ur b +sar ap +reli a +knes set +cy p +chan neled +caball ero +bcli ons +v ella +pri sing +pine wood +n ane +insi des +gorge ously +flet cher +al jazeera +pre cep +per vasive +pen arth +mam my +kins ella +connor franta +colla bs +ahmad shahzad +w tm +mercan tile +loop ing +loo ky +i got +fa jr +s dotcom +pat naik +do brev +bor os +ad erie +stell ar +liv re +impre ssi +da hil +bas ing +wedding photography +stu pa +or z +mar ky +mag da +id len +grat in +dies els +cas ino +appe aled +machinegun kelly +m ct +beck man +at water +ëĭ¤ ëĭĪìĹĺ +tur i +st david +sre bren +smo k +pu yal +mor pinoy +inter st +you uuuu +yo der +roo i +rith vik +re payment +rat cliffe +law ren +flatt ened +cu so +ar tic +tal en +sig nees +hart mann +ev ac +dri vin +clo ves +ab lation +yy yyyyy +thro tt +th é +sw f +squ ig +jhal ak +ig nit +calabas as +al one +ðŁĺģ ðŁĺĤ +ÄŁ lu +throw ers +sway ze +srees anth +sex iness +gen ji +algi ers +z oro +roa die +posse ssing +paras ol +over watch +o dm +mal mo +ec khart +desi st +call me +) & +! ðŁĴĻ +ðŁĺĥ ðŁĺĥ +tas a +nor vina +kom o +i kaw +brutal ism +bar aka +tablec loth +out pouring +lovel ace +guar da +ga vi +circa dian +ba q +umb o +tri gon +the f +tc dsb +ta ku +sni pes +protag onists +par kin +das adarshan +cur ried +c ne +st ico +ro ja +or p +noton fire +dragonball super +dac ia +blue monday +b fs +are e +any how +adopt adog +ë ± +åŃ IJ +y ur +syl vani +rip ken +ore a +milton keynes +la it +je z +gay lord +g ase +edam ame +ba iled +v ry +si ds +rain storm +emer alds +cent ra +becky lynch +à® ³ + § +viceg and +then or +tem bre +o tw +jad ines +ain sley +petal uma +nz wine +ha emo +dor ky +ãħĭãħĭ ãħĭãħĭ +ãĥ¼ãĥ Ī +utili zes +shaned awson +ri ze +har ts +ha gar +effici encies +deu ces +def tones +centr ally +wildlife trusts +n fr +gt fo +cuis ines +boeing airplanes +ãĤ ¤ +v su +treas u +tam pon +sth lm +staf fie +simr an +sh ey +home wood +dougla s +tn tweeters +spoo ked +in ag +i pl +guang dong +culmin ating +botan ics +bha v +yl ation +very where +vel y +ten ner +ru bies +nar ita +muje res +kar ol +fa o +custo dial +uof g +ra heel +plac ard +lawn mower +ja ar +ation ist +âľ ¿ +un accompanied +sleep in +side car +qatar airways +fright fest +blu me +batt lec +tampab ay +syn gent +pend le +i bom +hu er +head gear +cosmo polit +wal ther +transpho bia +san gi +or da +hexag onal +hb cu +gryffin dor +disrup tions +ber lu +ark ham +app el +ðŁı ı +wash room +po y +pk r +new sies +mon ahan +f ene +e mas +dispo sed +the moment +shir a +kuma si +hypno therapy +dhan an +ang ler +wh et +vo u +newh ampshire +manchester united +mam as +if you +hor sey +h ma +gin sberg +de po +tran scri +tajinder bagga +oun i +lees burg +k imp +happy weekend +en coding +bru ton +broo ker +broo ches +bor k +ang lais +îĢ ¢ +st eves +sk t +negr oni +hir i +e ber +dic tion +amal fic +tho tels +som i +shap er +q asim +invigor ating +gan try +fle er +cc m +blue water +atro phy +ìĨĮëħ Ģ +tourde france +fet ched +che aters +centr icity +armp it +yu cca +tax reform +snu g +ma up +li go +hr mann +fu ses +under represented +strath more +seab ird +gulf port +dam sel +colli er +az er +a online +worldfood day +sil vio +nz d +nach a +gr illo +fair fax +book blogger +zam o +work bench +we do +traditional art +thel ight +rain forests +or phic +l ma +ko z +indiffe rent +gu apo +cw m +conspir acies +brum hour +be el +vari eg +pay et +is ang +go sport +empan adas +conver ged +am ping +wom bat +wa u +the way +merci er +mccar ty +itt y +is beautiful +hu w +was ser +s first +oni stic +mtvbrkpop bts +galvani zed +ei ghts +ðŁ¤ ł +ma ac +kel ving +grindel wald +co sas +calab ar +ar aw +# # +ðŁIJ ² +tag sfor +pur rs +nai ledit +msh sl +k ore +ham mett +ec ret +dra goon +d cm +clo i +v ics +trail blazing +loc ation +lati f +islam i +geh ry +ff xiv +dai quiri +chipotle tweets +bha gw +ab end +ðŁļ ļ +tre x +shre ya +re gen +qu illo +noon an +can ciones +âĺĢï¸ı âĺĢï¸ı +wa heed +u ggs +ni et +go da +fra il +dis gruntled +app u +anti a +ak ha +un sg +super charger +quoti ent +q l +non na +ne ely +m cauley +g fx +ford ham +far ns +⼠ħï¸ı +to ke +team moth +sr x +ordin ary +mini mizing +borough market +beckylynch wwe +az an +appro ving +yiel ded +we remember +metro polit +here ford +for rest +er ne +dar la +sp rocket +sl en +outsi der +kas kade +iam cardib +hon our +fom c +fia formulae +ev is +! ðŁĺģ +van loon +fif ties +sun gai +sil encing +pop corn +p sm +ou sh +nigh tri +naam kar +el ing +cup cake +bo te +am ac +ack le +scar lett +saf ar +pl f +n pg +msi sodia +men lo +mc ps +lu thor +h hi +b sn +ature uk +voice less +uttar pradesh +qu raishi +pover ty +l fi +kis singer +bon aparte +at eli +sur bhi +re designing +ma dan +ha id +fi stula +dra pe +car ded +asi mov +pear se +p tl +infu se +enor th +clu j +chri scol +cat riona +tr d +thingsto do +tat u +sil vi +schaf er +q at +naz ar +man ts +jab ari +fi ddle +baby boy +al politics +turi st +sur ly +re purpose +pare ce +men dy +ku ching +iso m +anime expo +ag ung +a achen +ðŁİħ ðŁı» +âľ ı +Ð ¸ +pesh awar +pe plum +n fu +liqu orice +inte stine +ingh ouse +footh ill +áµ ī +vegan uary +skep ticism +oo p +gor on +ak at +ak ai +ðŁijī ðŁijīðŁijī +the t +sport ster +ph ire +n fs +cere digi +artif icially +v rs +l bor +eri ver +cant stop +bead le +bao bab +ðŁĶ ĭ +ðŁ¥ Ī +ner dy +medi ab +fly rts +f ty +craf ters +ar dern +wl f +sr hr +s ft +mac ros +id it +hard man +ham eed +co da +boo kie +arri eta +sketch notes +pr u +o tor +granbluef antasy +co by +universal hub +there samay +spor tif +ri h +pper ton +mal le +ike ja +deut ch +audio visual +ati ans +sar ai +mik ko +fal z +dest ine +cow bell +carav ans +ðŁIJ¶ âĿ¤ï¸ı +âĤ ± +sad c +pari shi +no won +me ads +de vious +ta ÅŁ +sal one +q h +oo fficial +friday fact +easth ampton +aq a +v sk +sch ap +ras mus +ot us +osteo arthritis +orangu tans +concier to +cit ym +ah s +un loaded +sidd aram +le as +ha gger +gam bar +ðŁĴĥ ðŁı¼ +un spoken +tuesday tip +native plants +gran blue +fic ci +cart els +ðŁİħ ðŁİĦ +à ¬ +wi ggles +sheph ard +sar andon +saku rai +lumi ere +human e +dapp er +cal med +th abo +taylor made +po si +mi ston +hoo ch +freedom of +ational park +ai lee +sophi abush +sc mp +quick silver +han teo +ðŁĮ» ðŁĮ» +sab ah +remedi al +knick er +exc els +dau gherty +alex ia +sque e +matri mon +mad di +kun war +hell raiser +har uka +gi es +evolu tion +coo ke +bell at +ari elle +ak hil +active wear +tak sim +mari ab +kun dal +gar cÃŃa +con esto +click er +thir ty +sub strate +ra ye +pro league +p gc +gc se +gain with +ct n +consu mes +vi ks +stupid dope +smi a +sfor th +lifel ong +kha bib +ga ea +den o +brink ley +army selcaday +ðŁķĬ ï¸ı +ðŁİ ² +ãģ ĭãĤ +ww f +wheel er +surrog acy +squ int +marc ello +lolli pops +ic ole +chel t +travel with +read ying +fur ness +ey elids +evening standard +d ll +whe e +p ks +om gggg +logi stical +hun gama +er ve +cor ked +brig ades +book loversday +ðŁijįðŁı» ðŁijįðŁı» +wh ockey +tu ttle +son ko +ros anna +non i +in atureuk +tr f +sk ated +scri pp +mad verse +jo ked +i bc +er ri +daph ne +collec tion +aw ood +abdu laziz +ãĤ º +vas und +vapor wave +mo res +li ger +is ing +intru sive +ak mal +ä ¿ +ãĥ Ŀ +weather ly +w la +schen ker +ruth ie +eye care +eco tourism +di ap +cross stitch +benton ville +barstool bigcat +ati que +af re +ad ama +é Ŀ +ni ghty +lon i +kk u +funko pop +ev oo +ec at +che alth +az quez +polyne sia +nat ge +micro fiber +mi o +manag ement +king sof +its ch +enlar gement +emer gent +e od +barri er +acor p +teas poon +tb sp +stat t +squat ting +r fp +pas cale +p flu +ma el +jawa har +da id +con ey +vo s +sa un +goo ding +g andy +cogn itive +y dd +vis ser +tri m +su pe +so ared +six th +rizz oli +mi kas +kat arina +gulli ble +as pin +alexand ri +tri fle +tomi ho +sha in +nn nnn +mand ar +j ink +gu tenberg +discover ireland +c kie +weg mans +wedding day +v ail +so tom +music thaman +kil i +ka ke +ci el +bt cusd +be wil +âĿ ģ +under side +q b +inquis itive +har relson +gut feld +forevery child +duc ci +can ap +ag un +Į Ģ +wee per +wc ws +spe w +ri ra +pimp les +mother nature +min seok +leav y +it aryan +ir k +day um +cristo bal +cat acom +alti ma +ty pos +off beat +nc su +in tox +hur ri +gow dy +go an +edu c +d mn +ber ly +low country +in set +hom ey +help forheroes +gr out +fl ung +f enty +elimin ator +bro ly +bal th + ± +wag ner +sm ell +iv ana +in ds +hi ga +ha vas +fed cup +fe sts +f mcg +eigh teenth +daw es +can arias +âĿ¤ @ +swa hili +surrey bc +redd ick +camar aderie +animal cruelty +vali ant +shou jo +dun lop +[ [ +twitter bestfandom +sho spice +ma al +ke eler +ju les +food photography +f locks +dangan ronpa +responsi ble +oh no +octu bre +mo leg +can el +bri dle +ad ream +talla ght +qu ens +link age +la de +clam ps +al maty +ðŁıĨ # +ver dad +su i +ringof honor +mix tape +mai dens +lem ire +cen se +ber nd +aw ww +tom hanks +home opathic +dis ick +bethany mota +bahra ingp +ait ken +ðŁİ¶ ðŁİ¶ðŁİ¶ +zi pline +twi ggy +stead man +ss aint +sd d +sch ka +preven tion +mike brown +l land +!! ?? +sle m +senate gop +os an +heire ss +gi ii +fo w +bur ney +as wan +s ja +mour a +hump ty +cutt ings +cra w +an ky +sp ed +running man +pdx traffic +digital isation +deple ted +church of +staf a +ss j +soom piawards +se der +pete buttigieg +per f +le ym +burg laries +avi va +ar thouse +ðŁĴ ¿ +y lona +to cks +ss mith +sam thaan +rec en +pro bowl +overs old +euro pa +vaj payee +or say +of m +myle skennedy +methodo logies +ko jo +history teacher +gu j +dre m +cap ella +bun yan +s apol +parti do +ju gs +hun za +dan se +bobb le +yar is +skir k +laugh ing +j ell +hoursof lemans +fram ingham +da eh +ch anda +u ab +tam pons +re pair +ne ko +kw o +good time +ag in +we have +renfre w +qu att +mul cair +jeff ers +cater ham +view points +sf su +kindergar teners +gartner sym +el ong +c wl +br rrr +ðŁ§ IJ +å° ı +ro then +pil bara +olo red +my heart +mand ates +ma ith +barbe cu +adag gubati +ad oring +( £) +ðŁĩ¬ðŁĩ§ ðŁĩ¬ðŁĩ§ +tra dar +my ung +move ment +br r +blogg ers +are z +aller genic +âĿ¤ï¸ı ðŁĺĬ +womens fashion +ton kin +rakh ine +rajas than +lich tenstein +i ad +g sx +exc elled +eli se +s blog +l bi +kine sis +is ometric +enthr alled +e il +duc ing +dri zzy +clar issa +to pic +summ itt +ridd led +mag nate +fi anna +eu er +book my +ali enation +---- -- +yuv raj +von ne +tn r +ten ey +shin ing +rhe in +po to +pen ed +new book +kel len +jack sons +flat bed +el ah +cre do +cor nered +zu g +wad ers +sub hash +smol lett +p sa +mm c +mar rakesh +gir is +elast icity +disney channel +carbon ara +bi ar +anc ourt +sunny leone +mv g +mun roe +meat free +mac y +he matology +ev enti +x cel +us agi +stock bridge +star board +r pd +mad hu +le ma +boise state +african union +ê°ķ ëĭ¤ëĭĪìĹĺ +好ãģįãģª 人ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +yu mmy +win ans +ran jan +no du +n gay +mk x +massac red +koo k +aidan turner +adi um +ðŁİ¨ : +ìĭ ľë +à¥ĩ _ +sunny vale +ra jab +pr d +kat un +ign ites +harvard biz +es y +deep a +at own +ðŁĩ¨ðŁĩ ± +toronto fc +sc v +re ni +ot g +neymar jr +mar mot +kal on +io p +equ in +echo ing +c du +bis i +beau jol +barric aded +amar athon +x ps +ts wim +super car +magical kenya +l pa +kri eg +be sser +waziri stan +save slives +pro kabaddi +or t +mü ller +mi ui +ha zza +em es +animal sin +âŃIJâŃIJ âŃIJ +united nations +tc f +se gg +nsp cc +ka o +inter modal +gill is +fri ar +danis notonfire +ba hru +amen ity +like wise +jard ins +ill at +idlen omore +gwyne dd +go ol +cess ation +am ay +nat su +ga vel +fran gi +dun n +ati va +and el +tur pin +sh ind +mo hr +ma ggi +king man +heart burn +h fc +glu co +f ll +b nw +am ae +affirm ative +,, ,,, +video graphy +sal esp +n º +jo er +jap on +f ylde +bu a +anush kashetty +win chester +scon to +no tyour +m é +kual alumpur +juli anne +ju r +four seasons +dev itt +cur sive +chiang mai +asp ca +am ico +ad joining +sta c +kee ley +jo i +hal low +go y +em f +dill i +diag on +cb sd +cal o +war ring +survivor series +stol l +stay strong +qu y +moo kie +m ally +hospit able +girl problems +exquis itely +drive in +down turn +d modeling +co pping +cad y +br ough +b ould +$ , +visit portugal +subver sive +run ny +oti v +musc ulo +k illie +in habit +hand stand +fil le +ro coco +l ge +facebook live +eu vre +black friday +thrombo sis +standre ws +r gs +mie expert +lu sa +fra sier +epi genetics +bant u +artistson instagram +ðŁĴĸ ⾨ +o sos +ipo h +cardio logist +é ¦ +white wash +ri que +peter bilt +pencil drawing +numb ering +mon tag +g ell +dr t +cra shers +ani mo +íĶĦë ¡ľ +travel skills +ro ped +fore sted +ed is +br l +bo ol +sn out +sd c +reli eves +ram in +ha usa +amp us +mar itz +initi ates +dero gatory +caru ana +bol i +tian anmen +shiv sena +opho tos +ol ice +li feli +den iro +ann unciation +zar o +xxx tentacion +sto cha +spac er +sco field +rai pur +no ctis +jam mers +gra fting +dent on +baira vaa +- / +ðŁħ ± +z onal +z ille +vagab ond +shoe box +regre ssive +kel d +jo elle +home stand +can you +anup amp +play mobil +p á +kra uss +jame el +e uk +bra s +su med +sto ys +sting rays +stat ue +remember them +refe re +produ kt +o asi +mary jane +hong ki +el mira +conduc tive +char io +bu kowski +ðŁĮ¹ ðŁĮ¹ +tibet ans +tcr wp +small ville +por tal +love whereyou +ie bc +gor ham +communic ators +cl acton +cat ford +at an +ðŁĩ° ðŁĩ +stor ies +shattawale gh +re sorted +pad stow +p wr +grey cup +fou cau +for tis +curren t +co sted +chong qing +ta vish +ta illi +ste arns +le murs +iri t +gad sden +ep r +be stow +appen dix +amalfic oast +truc kee +ther un +th nk +t fios +mon tero +min ous +har yan +ep fl +dy kes +de fund +tim kaine +rose mont +li vi +la usd +jo vic +ghan ai +boston strong +am ama +z ent +ush ka +swan queen +mis match +millen nia +jaw ans +el va +an ee +admi rers +ro lo +optic gaming +much love +i ha +dot ca +dig by +ag al +adul ting +ç§ģ ãģ® +v la +tun bridge +sur bit +no ho +la fc +jerry brown +gil insky +fb pe +eli ad +do good +dark matter +ban de +yu my +w thr +temper ament +p wa +ou ija +no sey +napp a +houn slow +h cp +goo den +american art +% + +south lake +ron an +reg gie +marth as +kir in +de a +cu toff +\ \ +ðŁĹ º +ðŁĩ¦ ðŁĩª +te vez +shel don +ragh u +push kin +passi oned +nar ayan +illi ons +hol din +c sos +anupamp kher +ë ¶ +z et +vo stok +vo gt +support the +sr ar +mour ners +laun dro +k cam +co hen +class ically +pss st +hassel hoff +fa erie +black mon +ac tin +tribu te +fil me +expir y +d mc +atur n +voy age +old field +mkh itaryan +lean er +jerrybrown gov +is enough +e ft +c suf +bu ca +batter y +b gc +ðŁĮ Ľ +sae ed +ocean front +michael j +k ross +fla ky +compost able +au den +u tion +t ite +royal baby +rod ger +och re +disproportion ate +devon hour +dand en +da ar +bas ins +b tec +pe mbs +khal e +gra b +dun ia +clo gs +be ath +ter man +shut down +re pose +raj endra +quar te +national catday +mir i +ma q +lok sabha +hyper rts +ðŁij ¹ +super woman +sris ri +oro sa +mother sday +hill sdale +eni us +disc red +bel aru +bar ring +av as +amble side +ãĤ ¨ +rot ational +pseu don +podol ski +om l +no body +nap alm +high tower +cran king +brew pub +ðŁĶ ij +Ù ī +zer os +ur ham +slo pe +sl ings +om phe +mel i +flyn n +erit rean +bri and +bel lo +aly cia +ag ap +tari an +sc ant +plu cked +p lowed +olu tions +o kee +le sm +inter vie +gall er +ent ice +ax x +wo b +ugand ans +nit ty +m kii +lin field +ine pt +bo sco +autom ating +as ketball +ane gi +smar ty +lingu ine +bb cle +ðŁijī @ +tinnit us +m vs +ko ons +k lia +ic as +gg mu +de ren +camel back +wk bw +sta ad +so res +is ola +en ps +cy n +ced ric +ber gha +te f +slur pee +me thinks +le ann +hor des +energy storage +alternati vely +ai ims +ðŁĻĮðŁı¼ ðŁĻĮðŁı¼ +war an +wai sted +man ip +madri gal +de ye +bon hams +stupi dest +ri ghty +org and +just for +j na +ipo b +gra af +fab ol +diagno ses +bio dynamic +[ + +ÅŁ k +ro bby +jewell er +exacer b +spo ty +skop je +sid ney +shadowhunter stv +paw tucket +om nia +n ong +lyce um +ingu ishable +free form +concep t +alco tt +sh weta +s sex +morpinoy biga +marilyn monroe +ic ol +di allo +alli gators +sop hila +pa hang +mascar pone +frapp uccino +clar ita +zil low +up bringing +so ch +sin nott +n q +man os +j tbc +i will +han an +flamen go +far go +doctor strange +cull man +ash ima +ad den +tutan kham +seren o +ry zen +ru mba +om agh +monol ith +cu ous +const ab +un read +demon ium +bismil lah +ap as +ab aby +te g +li que +jo son +jim in +jal isco +int i +gran by +g fw +ari at +stil t +sor tie +sof ie +re th +gw ynn +flex in +fa ireland +dispen sing +cra in +ðŁİīðŁİī ðŁİīðŁİī +ãģ į +tre mors +terr ine +presidenti al +moder nis +kkkk k +gyne co +gran ul +afro jack +unic ode +soci als +sh ec +scal ise +re entry +new stalk +m wan +li abilities +le athers +gur ung +gis ele +cont ouring +ak l +a ene +tt g +ther mal +t wor +nishi kori +cor tland +co sheriff +us vi +tsn hockey +sj u +p lowing +notori ously +menopau sal +mal ted +ma thers +kettle bell +is best +constitu encies +che ts +with standing +sport sm +op al +nh k +hu mps +edmon dson +call i +arte m +ðŁ¦ ĥ +scho tt +re ticul +ke ele +hu f +hi ma +ar ton +ar os +tim m +sc su +ru mp +oc sb +healthy life +ct u +basic income +andro ids +whitec ap +reck oned +om ir +louis ville +ju mia +ó r +sanje ev +ran adaggubati +ra ki +pil la +deci duous +carab ao +bon aventure +bel ight +the fall +o ineza +keyboar dist +impeach ed +fais alabad +clow ning +brooklyn nets +app lec +petri e +n ma +i isuper +foot loose +chakra borty +cal or +ar thu +Ùģ ÙĦس +pag an +n pm +n co +lind berg +jenni e +g outes +du ma +cul ts +! ?? +tic es +scot gov +pad mav +mam a +helsin ki +fnf jb +el am +co bbles +volu sia +per oxide +om is +new lands +hound stooth +ga eilge +so happy +re agan +o en +forthe kids +ciab atta +bal ham +af un +vicegand ako +v alli +sen i +mete o +living room +ko dy +hust le +harle quins +be ka +ag new +íĶ¼ ëĭĪ +Ê ° +wr d +usain bolt +under card +theori sts +th century +nu ance +n lex +mccul loch +light ness +fridaynight lights +e stor +! ðŁĺī +ìłķ êµŃ +Ñģ ÑĤ +yn t +we ald +ster man +pro vocation +nik ko +mi gli +max on +hol lins +fli x +environment alist +en camp +cho reo +ï ¹ +wb g +sh eroes +ridd ance +paragli ding +obsc ura +james arthur +deep ens +deb au +char geon +boot suk +book shops +ar px +ami sh +am ay +vit ri +sc ad +lac key +f ateful +do your +cabo chon +blo x +ann el +am ato +we sty +sprink ling +rain coat +ong c +le vy +ii iii +dr gn +ìĿ Ģ +wet suit +we imar +mag ica +la ff +kaz akh +flo of +dul wich +dinner ware +differen t +dan and +cellu lose +ðŁĺĭ ðŁĺĭ +tri dent +the grand +scro ssed +p sac +me hr +mark us +marau der +k up +in col +emir ate +dam o +com mi +bu shy +ìĿ¸ íĶ¼ëĭĪ +wa iling +theli fe +so der +provin ce +pol ter +pe ake +opening ceremony +nca atf +kei thur +bi v +av ast +andaliolo isa +wol le +shi kari +pau ld +mon son +medit ative +iq aluit +im pur +cri st +copac abana +un true +uc b +over heating +kale y +iisuper womani +ho si +d wn +cu tee +cler ical +bro gan +bemid ji +ðŁ¥ µ +spiderman ps +sax ony +real james +mu see +ka po +j ure +iam ahmadshahzad +i y +esc uch +vic kie +razor back +nar rowed +mat ts +mangan ese +kil ometre +iisuperwomani i +da ws +beat down +bb d +ÙģÙĦس Ø· +vil as +tu dors +p ka +mis ch +len berg +ep onymous +ðŁĴ ł +slo o +k gw +freight liner +discre dit +chi er +bur sa +ben ched +uni bet +singapore gp +noo se +jingle ball +gra ppa +black cat +anu rag +west cdnag +up sc +ru ti +ros setti +r mh +philipp a +oun ty +gau di +effe cted +co ward +ðŁı į +⾨ ðŁĴķ +zer matt +startup india +shindi g +re test +prev ailed +mill ones +light box +brim stone +and ry +al ene +á º +wre tched +the g +sky pe +hungar i +depic tions +dan bury +cra zier +camer oun +as kin +academic ally +watch out +theal thy +mac ca +honey moon +f wm +esp y +dar is +ci as +canadi angp +wil ko +wa hoo +kam ran +good fellas +fluctu ations +chi o +ch ka +ìµ ľ +wc pss +siddaram aiah +gi allo +defl ategate +bla zin +a ima +san de +pit as +kraf twerk +hel ion +fi p +ar bro +ðŁĺ¡ðŁĺ¡ ðŁĺ¡ +ðŁıĢðŁıĢ ðŁıĢ +ra it +ra gaz +om on +fri sky +bur rata +ðŁıĥ âĢįâĻĤï¸ı +wa v +ukrain ians +tor ta +sn ook +simpli fying +side bar +reflex ology +rb w +qu ers +phe asants +nor r +no ws +ko tak +ko on +fu tura +epi x +ba chao +ع ÙĦ +scy cling +sa ini +resu ming +ph ons +japanese gp +gur dwara +farns worth +bitcoin cash +å ł +tori es +ti ga +lgbtq ia +la chey +ko si +im ro +gol drush +fi ley +fet sy +fer ran +air force +ðŁĺī ðŁĺīðŁĺī +tri vedi +tor tell +sch mid +panamap apers +ley te +guil ty +golov kin +cla uses +canary islands +ber mann +as qu +a he +tun de +miscell aneous +may es +chi x +ca den +av alon +ash anegi +ðŁIJ § +world war +tid bits +se w +padu cah +goo den +girls generation +gir ar +exter n +exemp tions +ch alo +bur lap +bi ja +b mt +trich y +she art +sb r +r ls +po it +muhammad u +mer cies +low poly +kame ham +jür gen +chab lis +bil ingu +bi ot +bal y +say ye +patt en +ma this +lam pp +jag r +frag ility +erec t +di maggio +creep ed +ch re +ati ous +ðŁĴĻðŁĴĻ ðŁĴĻðŁĴĻ +vi per +ss in +r ü +par do +light stick +kidder minster +con don +ab uri +ðŁİĤ ðŁİĤ +ver million +ti wa +laz uli +la sc +irregular ities +gin za +cle ve +all ace +after glow +z k +snee zing +nfl x +moun table +infiltr ated +fra ilty +fle ets +com pa +cec ile +car n +íķ ´ +us mle +sla vic +se ema +marys ville +it weet +hal ton +eng ler +dd t +book bloggers +ب ÙĬ +v na +ut sav +t ville +ha sty +gra di +fl ore +bab i +b bie +ar na +ðŁĩ®ðŁĩ © +wof ford +ta vis +summari zed +phil a +mccle llan +m xm +lit tering +emir ati +can tona +bet te +an tares +ag n +pepper corn +motor bikes +im poses +enfor ce +cal de +vin tages +kin ge +ja f +go stanford +fac ials +chand eliers +reflec tor +ptx official +paleo art +national signingday +g gio +di am +cooper atives +ple in +kun is +fc cla +deterior ation +c pe +bel cher +ann ac +an dean +thel ance +sor o +sh asa +recur rence +play set +pee bles +n ars +entertain ment +citizen s +alej andra +reli ably +reason sto +kut cher +geograph y +fran kin +fet tucc +edwar ds +delu ca +d ger +å± ± +yo e +wl wt +sierran evada +sha ha +sear le +se pp +san gh +reinst ate +nic hi +gratuit ous +gra eme +ephe meral +chep stow +c mc +ar able +>>>> >>> +y als +wi el +trum bull +shri mp +rhe tt +pen tathlon +lle well +in organic +in accessible +head dress +haz ara +e gi +contrac tual +compu ticket +swim suits +sledge hammer +pu di +marc in +li smo +la ins +cor nu +chick as +snap backs +sal vo +raz ors +ra pped +he ms +gab bi +fr ith +equestri an +ek won +desol ation +cw m +val paraiso +te lec +tam ron +super fly +situ ational +sh k +mou sep +cu da +âĺĦ ï¸ı +ske pta +qi ang +ol sson +np fl +mau i +integr ity +ig ns +design week +cor mack +busc ando +ba ile +ðŁIJ¶ ðŁIJ¾ +ãĥ ¥ +ul ina +tast ing +koo ten +ah p +unil ateral +spring bok +purple pride +ox tail +n rs +fr itter +bur ry +arch e +ak am +uk sopro +spec tion +sa kho +ratt an +po ste +new sa +kil bride +hbl psl +handmade jewelry +con nach +bill deblasio +toxic ology +ortho doxy +new car +lar ke +godof war +f of +dar m +admir alty +s bay +parthen on +paedo phile +or ly +mo et +jackson wang +j se +iroquo is +gor da +best dayever +! ðŁĺİ +zig zag +that describe +star fleet +mac allan +interven tional +es in +snu bbed +precau tionary +orlan doc +no one +gill ard +gar van +fl icking +fa ver +cal dy +ar shad +vo ix +stru del +nu ances +ne in +kro shan +ken seth +ciu dad +bel fort +aver ted +âĿĦï¸ı âĿĦï¸ı +var ney +th il +swi sher +spec ter +lom l +kin ok +itys c +frag mented +de in +bag ong +tz bach +reb be +re issued +mar gie +lu ff +fr ack +fil les +ex oske +blue birds +ðŁijıðŁı½ ðŁijıðŁı½ +thri ve +offici ale +m schar +freak show +bour n +al fie +acre age +ú n +wag en +stevien icks +span dek +si bs +scu f +s not +olive oil +liber o +kitesur fing +ers ack +car lin +capre se +é ĺ +rhy ming +red dead +pe aky +pal ay +museum modernart +mc coy +la very +la gged +ker b +griff on +eis enberg +condu cive +aa and +ãĤ ī +puc cini +north star +lg b +leadup chat +jo h +intran et +hu sk +fur nishing +with holding +new ze +mo tte +met policeuk +dec aying +baseball hall +ar utz +alien ware +à¸Ńภĩ +uit m +sur renders +super charge +ra fe +me than +ger rit +cow en +tor ri +tal lied +sriti anne +sho ah +mc master +lun ching +li sab +is my +hay abusa +fol ger +tremb lay +ni se +moss ad +kry pton +as amoah +ann enberg +t ween +ni alls +mul ls +may the +loves you +leon ardo +i ol +har kin +circum cision +æı ı +oper as +moose heads +heart beats +du athlon +assassin ate +armen iang +rott weiler +north york +micro sd +im ic +h ant +gru ff +gaff ney +cre dential +bnpp ari +beech craft +?? ?" +ti mon +lose weight +ist ically +house plants +e ef +dun k +cosmo logy +cam ara +apart ner +âĿ¤ï¸ı ðŁĴĻ +wester os +theri ver +t wee +ke yser +inser tion +f pga +expla iner +di dd +. ðŁijį +stair well +pu ckett +por ky +pat nam +fle es +der ay +tro tt +ten by +nutt all +daf bama +comm ends +car les +c pg +ðŁĩ¦ ðŁĩº +z eller +sj c +palmer ston +no bu +ni le +flu e +cr é +am jad +ìļ Ķ +up ad +q ay +pop star +ov sk +match maker +lef ties +jeb bush +in ked +ge un +f ite +eco boost +brit on +blancpa ingt +asym metrical +ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ +w yer +to shi +s fire +q nh +main enatics +hou this +her ons +fem me +c tb +ī ´ +winnin gest +ro is +pon d +oned ay +mun na +ld l +ge ant +focu ssing +ev sky +do gan +ash y +weare r +th app +santho sh +sal for +ram allah +r bc +pin scher +noun s +mün chen +keithur ban +head stone +foodfor thought +brent fordfc +andalu sia +thalai vaa +suspici ously +sc tweets +oo re +kon dap +kin ase +disney parks +cru de +com mie +raje ev +mid life +me ag +mari posa +listen ing +ka ha +jet lovers +cer vic +yor ton +walk the +tell tale +son nen +sit ka +scru ff +pro vol +ki x +di ese +camar illo +au sunions +Ù IJ +t ards +resource ful +nb k +moo ch +mar ÃŃa +manof steel +il volo +ger ardo +ate x +ag allery +* ~ +yo p +tten berg +thunder ing +rab bi +play boy +pe ddling +morph ing +menstru ation +he ster +fr ito +eu se +davi dd +beauty blogger +soul talk +ny ong +mar ple +jer vis +gho se +cul ver +atle ti +un successfully +refuge ecrisis +r sf +j aff +gram bling +enter tains +c sharp +barbar ians +at well +v é +un ending +t pd +sanc tum +reverb nation +per c +only at +mee eee +lot sa +kel man +dam ask +anim o +thumb nails +prote afire +me ir +bull itt +bi ase +ann ad +tele port +sthel ens +sa way +raw ls +o cho +morg ana +h inged +gra hame +ch ha +aj as +un sc +land slides +cur ds +blizz ard +ar ching +ãĥ İ +vit t +un ison +ske e +pin ter +nu ma +melis sam +intox ication +gamep ad +fluffy fursday +emp ties +dalla sma +compens ated +wj xt +te ther +rhy me +real saltlife +ony c +meso theli +ke et +fin dyou +boun ded +yuv strong +sco il +pr y +pp f +nys na +her on +gran canaria +fra iche +d uni +aureli us +al un +al anna +tal lu +ro am +naamkar ann +mer ges +lit trell +ko b +hi dd +grave stone +ge c +ed an +duplic ation +cast ings +bio energy +bi p +thr ones +thail and +s don +pom pano +pig gies +moo res +it wednesday +is al +fl x +faz eclan +ash tray +af y +acham pionship +zz ini +true blood +sa dio +ra rer +new england +ne who +mi h +li fer +hemorrha ge +free standing +za idi +yü rek +r lfc +mo sby +mar gol +bra ined +ber gamo +ðŁį· ðŁį· +oxy moron +looo ove +instru cting +g ago +curti ss +arutz sheva +ali ga +al ake +whit eness +state police +o po +mark s +geh rig +embed ding +debun king +ca ren +ai red +âĢ º +villa ger +sweet dreams +sub tropical +produkt fang +pick up +partisan ship +mus grave +mouth ful +mob ley +empha sizing +avenger sin +ar ron +aci di +wad dell +stor tford +ra ym +po irier +keep going +kari shma +iam andalioloisa +hammer stein +crossfit games +whistle blowers +toyo tar +it w +he on +gly ph +a era +ze c +volkswag en +vivi dly +t act +stru tting +hu erta +gro ssed +giro ditalia +ep t +bb r +amazon uk +alto ona +ðŁIJ Ħ +prime video +n sm +kings way +shay mitch +rocke ted +po lok +nhl flyers +la bon +k arel +don gh +cheese steak +b tt +atay lor +ah on +vocali sts +stocki sts +soir ée +sat ay +n bac +model o +melo dious +gha stly +an en +a ita +âĿ¤ï¸ı ðŁĴĭ +sk its +mar ky +ic ke +euro cup +dge e +whal en +w cnc +the jeepmafia +tear th +murray field +lakel and +go cats +fur t +congr at +v air +ther mia +snoqual mie +sk ri +j q +geo de +f locking +av ak +art z +you sef +vi gn +tre n +t ür +rejuven ating +maje stic +lay er +im passioned +fel ted +fac up +cruci fix +rac on +prop ylene +prag ya +mer ion +maj lis +lumin osity +li fer +ko eman +gold cup +families belong +et ly +chri ssie +where i +pre sa +mues li +kit sune +i wan +hat sune +en cer +chang a +soren sen +ha ste +brook side +ade pt +ãĥ ķ +water bury +w man +new some +ana x +ðŁĮĪ ðŁĮĪ +tunbridge wells +thu le +manas sas +lur ks +le ting +k mov +ho ss +hel ton +ab ell +ðŁį Ĩ +âĺ¹ ï¸ı +sch ir +lan dof +hu ay +ga ol +ðŁİ ŀ +yl er +tre ati +th ay +tex pre +spla shes +shan ah +scar amu +rrrr rr +ody sse +kuz net +ir reversible +cu fc +con tri +w ys +stam pa +perfect shot +panam era +lom as +diab ol +chief skingdom +act sof +ðŁĺ± ðŁĺį +ti gres +sing along +silk road +mal vern +kis sonline +gujar at +desh mukh +anne cy +tele phony +re vis +pa ak +mi kk +m hc +hy nd +hol stein +ho garth +gry phon +cute animals +classic fm +ble ms +sk ind +scre amer +music producer +endo don +em en +av anti +rhode island +medicare forall +md x +la belle +ame dia +albe marle +ðŁij Ĺ +than h +si f +pre viewed +o sinbajo +mm k +is an +ge de +e del +common sense +clown fish +bas al +ana is +ìŀ ¥ +y vr +vie ws +thel and +sy l +pre schoolers +j omo +hoo sier +fotogra f +elong ated +áµ ī +pa inst +na han +cenota ph +wh p +ti vist +s words +releasethe memo +mika ela +le duc +eng li +der n +brown field +af n +a version +âĿ¤ï¸ı ðŁĴļ +wen de +mosa ic +margin ally +ma zi +ku ll +harb inger +cocac ol +ar apa +apl enty +us an +to zer +tai me +sick lec +life lessons +kirk uk +kat elyn +ga an +ag lio +å¤ © +âĸ Ķ +util ising +s fan +provol one +per n +mon astic +mi go +lu mens +hoo die +es me +dare devils +ais les +sph one +rou th +p nb +over lapping +or ds +norwe gian +ir fan +hus q +goo fball +d zi +cro ce +cap i +bu sk +us djpy +sy man +london er +ky un +hospital ised +felic itated +wildlife day +reven ant +re tic +po cky +pi zar +os d +ne ment +mi sta +manu ka +lin gh +g ned +evangel icals +cy dia +ðŁİĥ ðŁİĥ +ðŁ¦ Į +topo graphy +ss cotland +por tico +more tti +mor de +k q +colori st +assi es +ae o +a hern +seap lane +red bridge +ra ft +omo vie +l viv +kl f +jewel er +gg wp +falcon er +di plo +all american +ðŁĺ ¯ +ðŁĴ Ń +to ba +srar auf +mo ar +ma una +lic ht +kla srarauf +dora emon +bar is +ðŁĩ¨ðŁĩ ¿ +sy ch +pun tland +par ade +obsc ured +jiha dis +e tti +cap tioned +bab oon +tic on +stor i +respec tyourself +pu tri +p ce +npg london +mar cell +clari fying +chapp y +ca as +broad casted +nikk or +e bc +cli ppings +yan kovic +sh ali +sanjose sharks +salt water +re tty +prescri be +peril ous +la fd +jon o +cam ber +bra ke +// // +xin hua +tre mor +the david +scann ers +san gh +lubric ant +it za +daun tless +solo ists +nighthaw k +golf channel +go rey +gin kgo +diet itians +ann as +ðŁij¼ ðŁı¼ +âĨ IJ +wall ingford +visit england +tom ford +thrif ty +sher aton +rite ish +ridel ondon +persi sted +pan er +ma ir +ic ao +hi jack +g so +blo em +bant ams +way side +tor toi +t dot +stupend ous +park shinhye +mer man +mar zi +jab s +glasto fest +co fe +brah man +boon dock +apostro phe +ðŁijıðŁijı ðŁijı +zak aria +yar row +wor den +ph l +om my +li bt +hall am +ha yes +gun slinger +gir oux +frei ghter +cannabino ids +torn ado +ra fre +por th +or ban +newmusic alert +itsyour wales +celebr ation +ÑĢ а +tho t +tar leton +t cl +ri fe +mann heim +ludhi ana +kero sene +jud son +e qt +disrup tors +dg p +cr ac +beati ful +barunsobti says +au p +andre am +vet te +swar a +short ening +mob b +f hm +bhak ti +az usa +yu ba +ys u +wedg wood +smi reland +sm alling +sh m +n ino +hourof code +food blog +ebon y +dag gers +v da +tri pper +shi elding +our future +man ama +d bc +che x +candi ds +ìĦ ľ +аР» +su spending +stro ll +conglomer ate +xbox share +w had +uphol ding +k sp +ic sc +how lin +ha inan +giftsfor her +does nt +bellat wins +anim at +my nameis +lau ded +land t +la sik +jen morrison +instaf ood +dud ley +delic ately +biggbos stamil +a island +y ani +wro claw +sel enium +ru ise +ri poff +ra ab +prospec tus +nu bian +mis fortune +in focus +ich en +hh n +fl w +cru ces +bu hay +wol ds +too oo +row ing +pen nine +par son +magnific ence +m elia +lo or +kam oto +hi mm +good woo +buck ling +boy friend +volunteer ism +twitch tv +regi mental +pet te +new son +ne po +nar i +hen ney +gag s +ext inguish +ds w +balance forbetter +ay tay +abo realis +ye ahhhh +wash able +lang ham +info comm +fiance e +et n +chem ically +annivers aries +å ĵ +to ve +spon ges +si mbu +sh ir +sand storm +pull in +high rise +ave ga +addition ally +vi har +ne mato +micro max +fe ira +enab ler +conduc tivity +chri sg +ìĬ¤ íĬ¸ +pos ner +pha ge +ha dh +colli ery +ch els +bur lington +ant man +al v +un fao +storm troopers +smy the +sc ough +g lyn +fan expo +el kins +apat rick +ðŁĩªðŁĩ ¬ +pro fu +an them +tou ssaint +ti go +swi g +om iya +of champions +kir chen +gargo yle +f nr +eat eries +bra xton +bleed green +ano brien +! ?" +wali d +ti xs +sho wn +rr c +lo de +hungari angp +gott fried +forsy the +boo king +ab ooks +wal pole +ve the +te ds +str angel +pad dles +om gg +mun n +gri sw +gri mm +gl er +de bon +coom bs +chief tain +ur ls +take that +still life +re ch +luf tw +kang daniel +flood lights +enough isenough +bra i +a ek +ðŁĴ° ðŁĴ° +worldr x +we sthe +thr i +scree ching +obel isk +my elo +iz i +h sp +fute bol +em bry +ðŁı½ âĢįâĻĢï¸ı +ristor ante +pa isa +ny y +mobil ity +infor mant +christmas gifts +wy ch +jenmorrison live +gru eling +blu ee +viswas am +th ng +taste buds +t gi +rack ers +okla ed +mu sed +mr f +mol ine +giftfor her +fettucc ine +distingui shing +comple ments +c ce +wi x +wal trip +virgin ia +umb il +stra sse +s vit +re mi +le sp +feb vre +cu se +chri sl +.. ( +tw u +specul ate +ru tter +obe dient +mÄģ ori +mo dal +lan de +kla xon +g ere +clau stro +auto biographical +ac s +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ +ãĤ ĭ +star lings +lur k +black magic +bart news +ath abas +press release +phil o +pat el +ou lt +magell an +len ation +for progress +fin nair +cow girls +sat sang +penn ines +lo tro +kvit ova +gro ssly +fer ment +du chess +do is +b hu +sal ter +presu mp +lock er +krun gy +ing ar +er vin +effi gy +dur st +araf at +w ma +revolution ized +q r +po pl +lur cher +k ora +goo fing +atur ner +trans link +t ard +su baru +sn ba +ridge way +pet es +ing ram +escal ade +cup head +bul an +anchor down +ðŁĴģ ðŁı½ +ðŁijĩ ðŁı½ +ze an +wish bone +jimi hendrix +j post +injec tor +hom in +h ine +dr acon +beni off +baby doll +' ). +woo o +ur sa +thesp ian +mik kelsen +low o +ki yo +getin spired +for bidden +can aries +ble us +bibli ophile +spla shed +seis mo +s view +prohi bits +nar cos +nan twich +mil led +gian luca +cat stagram +bo ge +ar it +mid winter +josh groban +cat lover +ca sement +advoc ated +act now +zu shi +ratat ouille +mom oa +enti als +cou ches +college football +cis d +ced ars +bra gg +ìľ Ħë +wen g +nuig alway +no ona +mer rell +lu o +ju in +heg de +gun pla +di ma +dah lone +br z +bj ym +wal sh +teach ers +pap illon +mo gu +kru ger +k elling +impur ities +don ga +casca dia +bot l +ale y +.. :) +® , +sandwich ed +sag aftra +mor mons +ire ne +ir un +dru cker +alexand ra +aap i +ðŁij¨ ðŁı»âĢį +๠Ĩ +ze ko +langue doc +ju mma +infir mary +f ice +e ers +der matologist +cu cina +contradic tions +col ino +co ley +ay ya +<< << +w tsp +the who +sl inging +rappor teur +or gy +g cats +david bowie +bag gins +ab hor +soo hyun +revers es +r ya +ou de +maha bharata +igle sia +ford performance +ec mwf +dispat ches +cheno weth +ìĿ¸íĶ¼ëĭĪ íĬ¸ +ver tex +shab ana +reho both +rand yorton +pi ste +om ber +ol ondon +ob scen +ha chi +visit britain +thel ost +re affirms +mor g +m sm +let the +jan itor +im pedi +fuer te +dhoo m +cbc mb +ben and +armeniang enocide +te vin +ste aler +q aland +lec rae +lat ent +disappear ances +be to +as kew +alab use +shar ky +sc r +sat ter +sa ward +resurrec t +reper tory +ra ver +multicul turalism +liber ator +kri sh +hor acio +hell yeah +hal lie +euro basket +cin der +amar ket +tu dor +sou k +omor ph +mar tha +lo pe +hur t +bm f +advoc ate +un willing +un licensed +rabb is +puyal lup +par rot +hail wv +gri ss +en cased +confir m +chee ch +am bi +six teenth +rescue dogs +remo vals +reg gio +pl under +painte d +la ve +k pl +iti ves +exempli fies +cron ies +ë¸Ķë ŀ +vogue team +tv g +opening night +mcgin ley +god less +er asure +de bian +bo whun +ãĢ ½ï¸ı +sy bil +ri me +po tro +geaux tigers +el sin +dahlone ga +comedy central +clu be +buy back +bl end +!!! ... +w yan +turk meni +pu ro +prodig al +pit bulls +mat ric +k appa +il yn +he heh +fil er +f enders +wk d +qu ora +fau t +cou ps +co stanza +ann avy +âĿ ¯ +min x +implic ated +handker chief +ep d +e ker +bel fas +ac umen +sc al +pi de +new man +inspir on +fox tel +clean air +chihuahu as +c zak +ðŁİĦ ðŁİģ +word press +thisisd sp +signi fies +manne quins +gra ven +dingh y +bro lin +s kie +puzz led +merci less +gro mit +fier y +expand able +equi val +but toned +ðŁı¼ âĢį +yel fie +y tes +lamar r +gro m +for peace +cp t +ca str +beat box +bak elite +wrong doing +resusc itation +han ford +go bi +depart mental +cycli c +an um +á ī +ri shab +mce wan +kang an +encoun tering +bat ra +sh rey +li at +lean startup +cliff hanger +chir ping +c va +apple podcasts +ad trip +absur dity +:: :: +w ku +so das +sa ku +prioriti zing +netflix uk +mar zio +ken n +dermat itis +ðŁĴļ ðŁĴĽ +whim sy +vapori zer +tre ks +tele scopic +scam mer +pre sci +me tag +iri on +feature tte +be f +a ath +⤠µ +work ers +sur ry +ri us +myth busters +lu ll +coo gan +cerve za +beauti fication +aty pical +an az +zen fone +x xiii +ti gre +lor n +l to +eccle stone +der t +bo fficial +vir gins +uten sil +to efl +sub woofer +pur nima +nak ba +mo hawk +guatem alan +experim ent +encom passes +botan ic +bad shah +servic enow +pass age +oxid ative +me f +ma us +ho ga +gimb al +gent ile +ers ons +doro thy +burle son +boo zer +à° ² +wipe out +vari ance +sc c +o boe +gun z +gran di +country men +cohe sive +answ er +á Ĭ +way nero +ti mur +shuff ling +ple c +hunter hayes +basti en +bankno tes +vast ava +u hs +pun chy +pent a +marketing strategy +hein z +frau ds +foam posite +ee a +chry salis +bo as +ðŁĶĬ ðŁĶĬ +touri smireland +sarko zy +print maker +kian lawley +hu ddled +du sseldorf +chron ological +cat box +y ali +sch lu +sc ast +mun go +lon er +jel lies +g agar +dat or +bo ils +ber i +andro gy +wood burn +winter olympics +vene w +tre monti +supp le +south wold +plu shies +over shadowed +temb lor +st w +spra gue +re discovering +ol lie +ma thur +lall ana +k tb +ero tic +burger king +biz party +éģ ĵ +up ta +thisi se +initi ating +imag ed +go army +crooked hillary +cro well +arab i +amher st +ðŁijį ðŁĺĬ +zi ppy +x ctf +we ah +trickor treat +teix eira +reluct antly +mamat aofficial +freder ik +for gives +fly er +clair voy +acy cling +ðŁij ° +si ppy +schu yler +in voluntary +h ite +gl ories +fac ial +as you +ark ali +um bra +sta dio +o iled +nad ler +charli es +abol itionist +war io +t ct +squa shed +no way +megan e +kon ta +in doctrin +howar th +az uma +ale house +acti ves +ach an +stand outs +sear ing +haringe y +gr ana +exoplan ets +cab ar +ap ach +stre p +square enix +pin ky +pet its +ne gre +morph ine +mer on +e on +dro pper +d pw +ar aj +adi o +yu eng +sur facing +sc ampi +sa if +rut ledge +men cap +im practical +charlie puth +ze tti +saw grass +rox ie +mul ling +hu w +eng chat +dr j +club man +bo tics +av ell +u cm +tecum seh +re runs +op on +ome trics +man x +jes y +in laid +grazi ano +ch ella +cal derdale +all red +ภ¸ +sea water +mo les +lauren t +kra us +ground hog +feder alist +da es +x peri +st elle +pati os +o tra +nt ds +n bl +mitch el +kingof the +ki ley +hom en +ge tre +dise mbo +den sely +columb o +carbon ated +af ra +wa ilers +ro pa +on top +ne f +mou ss +m hs +ind veng +fan boys +bas kin +" ..... +ju manji +gr w +governor ate +gen flynn +fl i +ep au +cont ador +ca di +aa ja +zu zu +ta kingly +special ise +sc lass +s fe +ra ic +luftw affe +hack en +gar bo +and field +am li +alcan tara +tb il +tamiz ha +om m +ne ma +natural gas +li man +hand s +fil min +boom box +vol l +un apologe +swif ties +ste els +sar ro +nar rowing +j ll +foo te +ei leen +basel world +æ ī +wil kie +wes tham +turkmeni stan +th wick +t asking +sil vers +repe l +re arranged +out puts +miy amoto +miti gating +ilo va +fe ss +fast back +cul lin +che ch +cb d +! ðŁļ¨ +visu alized +tru man +se ager +sd wx +co bo +wor ry +strate gic +pok al +mi p +fro y +ex cision +ang ell +ðŁij¨âĢį ðŁij©âĢį +zz or +subterran ean +pom p +mesotheli oma +li ao +ir at +i ap +god addy +et en +dragon age +catch up +# ( +tum ours +the star +rho ads +pi gg +ori anthi +jeep ers +infra structures +har shad +hal os +ha ver +day al +b si +b se +ac yday +sl m +mu jer +mor atorium +mon go +lo is +bw f +spani ard +musculo skeletal +mis ra +i ks +!! :) +wi eder +s á +n ts +moo g +ci bo +yr sof +young and +ye son +w myb +svet lana +mam o +jy pe +dor sal +af zal +re cherche +ra gu +hy dari +hin os +acrob atic +up tick +tel ugu +so so +rad hi +mu guru +infe ct +gosp els +down es +ded ness +ðŁį¾ ðŁį¾ +xen ophobic +ra kow +pe h +medi as +ju s +dv c +ðŁIJ ŀ +track side +tess ell +rhy l +poli shes +mal nourished +lar va +icon ic +hol der +for ç +far ron +chev al +cas sis +c ich +ball ant +wester ns +tu tu +tat ters +rich t +neander thal +jo akim +insati able +hoar der +fran ky +fla v +womensmar ch +traff icked +the gathering +te eth +sho ving +per cy +on so +nic hk +nancy ajram +kir stie +jab bar +gr ingo +fair trade +demo iselle +sch a +madam e +brah ma +ag am +æĿ± äº +yoshi ki +southern most +ra po +pre eti +ou g +mc shane +mani k +kad er +car ole +bol ger +ðŁIJ ı +tattoo ing +sketch note +poker stars +ip tv +cb u +ank ita +ı @ +sat uk +regur git +ph oning +o akes +neu ville +jor don +bre uer +as sini +ap en +tran g +te w +mn g +m te +m so +ky ri +gossip girl +b ck +ab ro +wy er +venture beat +mesopotam ia +in crimin +hel mand +ha iti +ha it +gro sse +dar den +cli matic +ç§ģãģ® ä¸ĸçķĮ +wh e +sm ill +rising star +percent ages +mu ddy +modu lar +kit ted +je une +is in +incur red +fir stre +communic ated +bt sin +ðŁ¤¦ âĢįâĻĢï¸ı +zapp os +win es +tortoi se +swan sea +shoot around +ri ding +hi ga +di van +att led +stiff ness +stereo typical +satur n +philipp ine +mati as +khar toum +j dt +hend rik +consul ted +ame c +x finity +voic es +un grateful +lizz ie +lar p +eur onews +ac n +ru mah +pan icked +men u +la grange +go lions +gam eday +dk ny +winter ing +will never +te us +t vac +su ma +sanctu aries +in justices +fi ore +dance music +án dez +zeal ous +robin hood +river bank +man zano +gy i +gati ss +fritt ata +enthusi ast +clu se +ba res +to po +spartan race +sha key +sh ue +queens ferry +mur ri +i biz +hurrican ef +hi mi +flow chart +dilu ted +ac sports +ï · +p sp +oc y +inc ite +corn elia +co very +sim ile +scuder ia +rot ator +op to +har ish +easter sunday +dish one +ch back +ðŁı İ +west point +st eller +search able +pad ang +mon goose +malcol m +hanni bal +gr n +far ting +emili ano +may berry +gal braith +ex pulsion +dow ne +congreg ational +bar bie +yb nl +oladi po +nus rat +mor nington +maurit ania +kag ut +k th +ju ror +hib bert +d va +bri k +schnei der +motley crue +live your +hospital ity +gul ation +for ty +faç ade +fair port +ely sium +de ion +come backs +cli part +ad af +w pp +un restricted +top anga +tony stewart +rare disease +ou trage +men ow +gand ol +as al +ambigu ity +tt alk +sti m +re used +mar lin +il han +fle dg +eun ji +chi le +by nr ++ ! +zom ato +wed ge +newly wed +mis management +mi f +m alling +lo fts +ho or +field hockey +bally mena +bal las +íĻ Ķ +yo ff +world healthday +tin der +ship man +pa ok +nhl flames +ma kayla +bay t +sto ve +ru dder +raj at +ra iler +pen e +mercedesam g +im on +i ee +brow der +adren al +x ed +skeg ness +sill iness +sc lassic +pe pa +fit t +diss olution +copa america +book lets +ate urs +asahe b +aln wick +the hobby +taran aki +shan k +rep john +our ses +mobili zing +iso tope +gan z +ðŁķ ¯ +vis count +val les +v se +sk enya +schau b +play mate +park scanada +ma ren +ju wa +bu mb +be rea +am orph +waist coat +piti ful +na ji +l fl +ing news +t mobile +nami bian +jan sson +hand son +ev n +catbox sunday +bhak ts +spe y +sky rocket +ever t +y ab +wen n +ve k +travel photo +ti ere +ski ppy +sa ab +qui er +payo ff +miller lite +j ps +emb attled +el ma +depos itory +complac ency +co h +ansel mo +show ings +shi ro +se ger +p fl +mat ti +knight sbridge +dumb arton +con son +bee ston +asci i +worldre cord +tim mins +men ssoccer +kilau ea +jo van +deni ers +beach side +tran scripts +punjab i +pre ssion +on ov +music hall +jam ar +err one +ep en +democr atic +contra band +& ... +su mp +sk ov +ray ne +mur ali +m eric +ed ays +cu test +con roe +bra unf +ab in +ðŁĺĤ ðŁijį +syl van +procrastin ate +la var +fing al +ak ind +admi res +à¤ Ĺ +treat ers +tre mbling +ral phie +p bis +hand outs +sam o +li one +griev ance +co tes +californi a +bis cotti +stock ton +sitec ore +podi atry +in ky +ic ici +fil lets +fail te +e pub +domin ik +day trading +braunf els +um b +tro phic +tric olor +ther ules +spl c +se con +paint brush +or ry +ni zam +nat weets +metal head +mam moo +esopha geal +de celer +allthe way +ac at +yn g +sh elly +poi rot +mon cri +fra z +ðŁij ® +å¹ ´ +yul in +util ise +tak is +stilet tos +ri v +rein carn +re imagine +pl p +ner t +ki shi +kaiser tone +ga iner +dier ksb +deptof defense +cut throat +chuk ka +black friars +alam bert +ak vari +ad om +wr ld +wkr n +w pl +tale za +ta unt +spey side +punch line +original character +maz el +help ing +har ries +bl aring +bag ger +accommod ating +predecess ors +peris cope +le ish +d illa +confe ssional +th yo +sho pper +royal visit +meridi an +me is +lovethe darts +kitchen aid +iti onal +it son +hin de +ha ss +gigat own +fri ghts +feast day +ðŁIJ ĵ +si ima +ray burn +mercedes benz +ford nation +cont ented +app i +re butt +qui k +qu and +muguru za +l ss +kan colle +ip r +fo wey +direc tives +car is +can nock +at ment +air flow +west pac +wdy t +trave sty +the artof +s fi +re ale +pa ws +new song +napol itano +mess er +lo fo +ke dar +david guetta +astu te +ðŁij¸ ðŁı¼ +ta an +ferri ss +bu chan +amuse veni +sho vels +newsle tters +ho tly +hen sley +ge dd +g ingham +esc am +drum roll +car dle +âĻ ł +spir ito +speci fic +skill s +leg alizing +daw ood +bb cn +{ " +zo oming +val erian +tand on +sk un +park view +paper craft +mr c +con traction +beautiful destinations +tas er +shaw ks +salt lake +ple y +pessi mistic +jae ger +ide e +gor ges +gar am +clo thing +cl ink +cent enni +ali ab +ta sters +sit ges +mo shi +ji iva +har borough +firstworld problems +atri sts +# $ +ðŁį ¿ +un professional +tre ach +team work +s ills +gaz ette +finger scrossed +b pc +undeni ably +th g +mo hand +kagut amuseveni +haw ken +ers world +daily art +balay age +watch us +th ame +sustainable development +re iter +ol ica +lucy hale +ku ba +foucau lt +crime stoppers +cre pt +choo sel +as w +ambassad or +. ðŁĻı +u omo +ki me +check points +wing ate +sau ber +nor ton +mei ster +lec oul +kar yn +duc a +cor te +ak aya +a sic +short stories +gol ly +elli sts +bour se +strol ls +niagar afalls +newyear s +n ines +lor ain +le win +geor die +ath lon +un knowingly +han go +bo dice +bay onet +tu mi +str ick +r ity +mis susa +le el +garth brooks +f mc +as ss +s ate +ron ics +guer re +ger u +exfoli ating +a ak +woo ten +subur bia +se wer +mecklen burg +ken shin +dj o +de wi +col ston +blue star +blanc pain +transc ends +te ma +scrib able +schi ele +mo ff +is sey +indi ab +cu bed +cand i +alp has +alle ge +ðŁ ĥ +r he +p fp +new west +lack aw +h ree +cru mbles +al ap +wthr com +to kio +state side +sit is +se vern +ro mb +ico se +gri sham +fla gging +com posure +cathe ter +can ines +ðŁį Ĺ +z la +v ander +mom oland +hil ux +gar nished +coven try +bi gi +stu cco +oo ty +kac ey +guess the +goose berry +for it +death match +ali bre +af ari +ab cs +val our +sush ant +ra hal +publ ics +lati mer +k oop +h iss +go oner +g pc +f tv +con front +c ada +archi ving +apo logist +å Ĵ +Ø§Ø ¨ +stl wx +small holder +reta iling +recording studio +of sky +le le +desol ate +algorith mic +íķ ľ +wy k +vers ed +tre spass +take it +pr aline +nue stras +mat thar +li psy +ky lian +fli pp +fa wards +clar ine +all lll +sta ar +scaramu cci +reas sure +kir ch +j cp +commend able +bank roll +baf fling +angel a +ðŁĸ Į +tre mont +spook tacular +raj kot +kent a +home stay +ho even +fontaine bleau +decapit ated +ar abe +april ia +thorn hill +tat t +si bir +no limits +newze aland +naz ir +morph in +la ken +hinch cliffe +gor se +gaz prom +fit n +defici encies +d ool +bohemi an +ar ad +z ax +tambour ine +sp elman +multi modal +milleni als +melt zer +henry cavill +han ia +w zz +sever us +planned parenthood +ni b +multip lied +cal lum +be inspired +ðŁĺĤ ðŁĺ© +yq g +uk weather +laundro mat +kir stin +ip i +fair ground +di vision +d ando +be als +based god +âģ£ âģ£ +whis kies +weak ens +to watch +te pp +seash ell +pa inter +o ast +inde scribable +g ani +el rufai +devil ish +bo capsule +bha ji +yeez us +work sop +ques ad +phosp hor +mo ffe +lan z +indi scri +id d +giz modo +el pas +co als +chim era +carbo hydrate +am oment +sta at +sof tener +shrin ks +plate lets +ok la +di b +deplor ables +car ling +cal gar +breath takingly +ann n +ðŁijĮ ðŁĺĤ +ж ив +ze m +white haven +we isse +virat kohli +sc ap +fir ma +co rea +c mi +ðŁķ ° +ðŁı ij +pn p +mess er +gue sting +gran tee +gi st +che ater +bur na +ak im +uni birmingham +kan di +her tha +feli pe +b bery +super dome +os f +mid town +letter box +la far +juni o +food trucks +fish man +âĺĿ ï¸ı +west bengal +u up +spla yer +patri k +man gan +kram pus +hyalur onic +fra un +curi ou +charl ton +bike share +bah ay +studen tath +n ant +d hillon +cre ssi +ar ta +twitch streamer +snake skin +saura bh +pre maturely +frankin cense +conden ser +capp ado +tweetab ondthatcantbebroken +ti ms +man cave +jal en +hand i +cafer acer +bar ger +as ena +" > +wic can +ver de +standing rock +puri fying +paste ur +gal t +fc king +dierksb entley +car away +batt lero +asse m +ad week +ðŁIJ Ľ +us am +thor pes +supervis ory +sc lub +pas saic +mil la +form al +° ) +travel theworld +ti sha +pic t +per oni +lore to +ku y +ff m +watch this +u lam +medit ations +emb assy +bir o +wheel chairs +su pers +si me +run corn +ne to +ke ke +hun ts +donut day +ci ders +brief ings +bren ton +ãĥķãĤ¡ ãĤ¤ +we den +tumul tuous +tr ine +shaqu ille +ran goon +pal pable +geri atric +ea stere +cfb playoff +brun ner +apro pos +ðŁĩµðŁĩ ° +w fm +tee ter +od f +nov artis +ni jme +n tw +matsu moto +intersec tionality +ham ed +contex tu +avengersin finity +sd ale +nat o +mac gregor +gar ber +ele m +c ps +bay elsa +back fired +anal ge +ni u +mini aturi +li fers +ke dah +ai mee +ad dy +ðŁĺĤ ðŁĺħ +wp tv +trouble some +li ani +deep water +ðŁı ł +wor sley +w un +si sley +s fight +mai mane +long itude +ec lare +ck a +cabine try +brook lands +anastasi a +vonne gut +swat h +science week +mutu a +can oes +brun n +aishwaryar ai +vesu vius +travel bloggers +traumati zed +te din +shaf tesbury +prow ler +ni bble +mi ko +es mer +crock pot +waynero oney +un harmed +spell bound +s ram +play suit +man che +fraud sters +fore shore +du gan +ask the +vol go +sav ant +par si +ol le +look s +fu mi +fais al +exor cism +candi da +wl w +vin yasa +vent v +urban ization +tam imi +sports betting +shar ma +rejo icing +gla sse +dar aa +d fat +bb j +bankno te +anonym ity +whi zz +shiv ratri +ri vas +popo vich +mil dew +jimmy kimmel +gon er +frag mentation +e aves +affi davit +nott m +fa ires +dr l +deeplear n +de scu +care lli +bra bant +-__ _- +ðŁĵ Ħ +the hungergames +schem ing +ro tisserie +ri pa +present e +over crowding +fear lessly +cer rone +vic theatre +ukbusiness rt +substitu ted +shut outs +pau lette +pal ing +ola unch +henne pin +bow man +a was +yaw ning +with am +vs fashionshow +ver ture +tra b +th ath +st peter +ross endale +may an +heritage day +f mi +ca ith +bel gra +tavi stock +sur ged +str am +ra tha +prem rugby +ny cacc +mor ay +fiftyshades darker +fayo se +en actment +conden sation +carra gher +british vogue +bom ba +apric ots +alessi o +war tho +sex es +pra veen +lis berger +ki bum +frac tional +ew tn +conco ction +cater ina +am aker +symbi osis +supre mo +sesame street +polok wane +new burgh +khal i +k agan +di pp +broad bent +boni face +auror aborealis +ภĸ +sub way +screen printing +h ati +dc universe +vicar age +u ah +tiger zinda +stol en +space man +sam ple +pil kington +medi ator +lu mps +joyful leaders +e ason +be agles +paren ting +padra ig +obli que +ma es +inst ar +har git +gen ie +de em +cbs sports +back stop +vern ay +t sur +rock hall +night stand +musc lecar +journ alis +eff erve +do zer +darken ed +cu per +col ne +brook ings +world premiere +vel ma +south dakota +sh inning +s ically +le er +elo ck +di pika +winni peg +wa an +vaccin ate +si ms +sho x +q t +oli o +net ball +mc as +magni fique +ma ples +i kar +how ells +v ence +richar dd +pur ina +mend ra +je z +im is +h tt +forthelove of +fight night +exhi b +earth bound +e sol +butter worth +blo c +bi ol +before you +ãĤ¹ãĥ Ī +¡ ¡¡ +wc th +tom mie +th art +stra ys +speop le +sh we +sea hawk +pp et +ol ler +n si +me tv +ma kar +kur ta +k xip +k nysna +friend zone +de scan +bint ang +as pire +aquari ums +p mr +one perfectshot +non linear +nom ura +nichk hun +he yyy +faf bulldog +du ane +all u +un sig +power bi +mill brook +lion sgate +but lins +be o +al ok +suspici ons +r ins +mid nite +mam an +el way +e bi +ast i +al ah +wi ther +senn heiser +plym uni +pan icking +night photography +lecoul tre +harri et +clerken well +ci da +chick adee +car tney +cap tained +be te +am ee +ðŁ¤Ļ ðŁı½ +what chu +scream queens +k mt +en heimer +dont be +der ive +davi dar +cr and +ëĵ ľë +âļ ķï¸ı +wi thr +pe cos +mar kie +hat teras +garden a +arti stic +stony brook +pra xis +one w +gar nered +e gor +crew neck +bn f +acon ference +zoo logical +u ic +swe b +men ard +mayo clinic +lin css +hu ggins +dl f +award winning +wrest led +tranqu ill +re voir +re charging +pro fo +pro claiming +p tole +jimmykimmel live +gman ews +douche bag +cle m +ce ylon +accent ed +aaaa and +u ks +symboli ze +swan age +safety week +mo si +law fully +ira d +idit arod +hen rico +fir a +âĻ¥ . +wak o +vo ye +susp enders +probab ilities +ox ley +hye ong +gru yere +auto cad +accumul ations +(- : +ðŁĸ¥ ï¸ı +ver os +tough mudder +thwar ted +shor ten +koo zie +kameham eha +jung les +ide ation +hill ar +el tham +chem in +assini bo +ar cane +an ai +ab bot +ðŁĮ ĩ +ye un +subpo ena +selvar ag +red box +fe eney +f wc +ab is +: ( +ðŁı½ âĢį +ðŁĩ¿ ðŁĩ +| âĢ¦ +to read +mill inery +le ssi +feu dal +fajar do +cyn di +chronic le +boyl ston +beautiful day +wj sn +w md +va g +tech n +sumat ran +pre sales +mind body +likefor like +chro mium +cha it +ab aker +~ ) +pre ma +nadin elu +missmar is +men del +man sion +kle enex +ji yong +in of +entry way +bump ers +au k +ê ± +stal bans +rs v +paddle boarding +le as +evo que +enginak yürek +em al +dang elo +berg dahl +az ad +amphe tamine +ç « +z rh +x ddd +puzz ling +mont serrat +man ns +jesu ss +hatt a +canon ical +x z +lun dy +leav ed +jo dha +epic fail +el wood +du ali +conver ters +a et +!! ' +z is +wal z +v vs +slo ping +no str +mel li +graphic novel +dol o +cork screw +ul uru +the way +sto k +spell binding +ru be +ro den +re ay +gu shing +bra wn +av at +su mac +pis sarro +mano ir +ly nette +comprehen sible +absor bent +winter solstice +the q +share r +mur ali +mm urd +md l +light skin +gg g +el ny +consoli dating +command ment +bur dened +bin ders +asi atic +Î » +um bro +suicide girls +rail uk +n ale +missmaris racal +master chef +de generate +boo sh +aze alia +united way +technical analysis +t ended +spo kes +sheep skin +ram ayana +queen ie +je f +i ana +h indi +gra pple +el itist +el ap +d pc +d mv +bet cha +b ape +ðŁijĩ ðŁı¾ +ton gs +shoton iphone +reli shing +re pra +powder puff +os man +bu tty +ba ie +aco ke +the w +re making +pl atters +per jury +ni zam +movie poster +fredri k +fa sten +ener ge +el don +bird day +wy att +wh atta +uygh ur +tx motorspeedway +strau ght +sta de +pe kka +ki k +cri spin +cat t +ayushman nk +ðŁĮ ľ +weather ford +vern al +ta stiest +suspen sions +s ada +perfec tionist +jo go +del ving +con chit +. ⾨ +top friends +summar ies +ste wie +scrat cher +pre sets +mar ana +her mano +g dn +edm family +bug sy +us ical +ste tho +qu ities +lin ings +king swood +dhar ma +wil k +ou lu +ori ous +or om +optometri st +one se +er rand +end y +doo bie +coo ki +ber tram +akvari stan +torto ises +tatters alls +ric ular +p ough +ok tober +nyx cosmetics +lo oooooooo +hoi sted +ho des +dw t +dist illers +day light +coeli ac +bo bro +arra igned +tag uig +sf pd +pres sure +flaw lessly +ev geny +conscienti ous +buc ci +we can +them all +plu ck +pless ness +me v +lab ly +it te +ar v +ab lack +wt k +up tempo +stil ts +sma c +jaf fe +hur ri +hta fc +head quartered +gul ch +g ca +with drew +selfle ss +meh mood +la bo +sunderland afc +st ens +potat o +pic card +o go +mer vyn +ma se +koenig segg +ilu str +hom opho +hari bo +di ario +calvin klein +le ec +c sharp +apat ow +al bury +yellow ish +y tv +ut ley +ro san +ram snfl +national coffeeday +kuro sawa +judg ments +it si +idio m +ho led +cl ank +citiz ent +candi dat +ae g +wom p +the opening +the bhf +shar da +north van +nas scom +mm p +inqu iring +glu t +dar te +sin pics +sas cha +re pp +do go +bag gies +u ottawa +tu ber +stor my +st lv +re it +re fil +palay eroyale +omo juwa +my suru +lo li +bio science +ang ello +ace o +ãħ İ +victor ians +tyr ann +tit o +sand hill +ous se +moneti zation +mo ka +iz mir +gr ins +gentle man +disson ance +deep avali +danic apatrick +president trump +par mar +pain killers +pag asa +origin ates +nex us +aspir ants +whatever ittakes +stock well +ste alth +s de +l bf +ign an +her z +gon da +fu sc +fe dor +dra x +d arian +ca thr +ama al +yu t +spl ice +s attar +re sses +mt f +inter acts +infiltr ate +happ end +den ounces +car row +vir gil +v vip +ti bles +oce arch +cour ant +z adar +wille ms +u ze +sympath ies +revi val +pe ase +ou fc +gren fell +globe trotters +g lin +fur thering +fla pper +war ds +raven a +mit su +eu g +cated ral +bex ar +be douin +zi oso +y aaay +sg t +refin ement +mol ine +lam y +d lamini +climate strike +bythe sea +brat ton +av r +ah ill +ad an +wolf son +m ne +ci ak +char d +bright side +â¬ĩï¸ı â¬ĩï¸ı +à¸ļ าภ+sch l +saniti zer +master plan +la vish +kar ant +hull city +gur kha +gat lin +com cast +bi ar +b ww +ak bar +x eno +wo wo +spin al +per ts +ic ent +famil ial +ally brooke +à ² +z oro +ver te +se mp +sab ato +rela p +puerto vallarta +pe dre +pat ria +moo dle +make me +im porter +fish tank +f yo +co pi +bicy cling +awil son +above andbeyond +wa ar +tat a +smallbusiness saturday +rhi an +ko ya +gr ation +dict ates +d tn +be it +ðŁ¤ĺ ðŁı½ +x eon +son akshi +sch en +ratt led +pro long +g pp +fast track +dr anath +de central +copp ell +breath ed +ðŁ¥ İ +who dun +submer sible +scallo ped +r itten +mal don +l hd +just another +joseph s +hope well +fa stand +dhar na +clar ice +walk off +unspeak able +sp ac +soap box +ross r +jay son +ce ps +af faire +ad minister +x clusive +tar f +special ising +kir ill +hand som +daytime emmys +congress men +ceredigi on +ca ix +apc nigeria +al al +âĺº âĺºâĺº +ru ps +pop music +mr and +gold man +gi vens +de ffo +art sand +alu a +viv atech +tarra gona +shak in +sa irport +recap ture +pat r +mano har +law rie +hi ver +ash am +ÃŃ s +ther m +sim mon +religi ously +opul ence +nawazu ddin +mc ca +la sso +bir a +y ami +y af +tele photo +su sten +sego via +rio de +go han +f bu +ey bl +clic quot +bu x +ber ley +âŀ Ķ +wal mart +sar war +r fs +p ylon +mign ola +go pokes +cu oco +car li +appreciation week +anti um +ali yev +Ĭãģ Ĺ +wordsof wisdom +wi ggly +wa di +u do +strand ing +sto bart +shadesof grey +port noy +port illo +pa sties +mi spr +mam elo +lor ax +la ire +janos kians +ham dan +disc ern +country life +ai les +t cher +sail fish +saf i +pro fil +nothing ness +n ri +har iri +grou cho +far outa +ev m +enthusiast ically +en da +du sk +dread nought +cru mp +coul da +certi fied +bot ticelli +ba x +au me +ske tt +sc b +rose hill +mb c +isra eli +h ne +great day +folk fest +faire y +er ink +en ry +craw ford +broms grove +bo drum +travel channel +sar dar +rec tify +newsa del +micro be +inci dentally +in still +fe cal +eu gene +dru gged +b man +whoo pi +un a +twi z +street view +our day +nicar agu +mr k +mouth ed +intu it +ingra ham +groo ving +cute ee +chil tern +che ol +boomer sooner +arbro ath +to ko +te ab +smo ak +ser aph +sal ert +re w +pol k +pim ps +ma ho +ik ay +he sper +cit ru +black sabbath +short fall +mar a +ib as +easter ly +ca stiel +ìĨĮëħĢ ìĭľë +ë Ĭ +rein vention +la vin +jo ong +con cur +clu stering +bra ver +ba aa +alge bra +al ita +aberdeen fc +wholesal er +vo et +vin od +st alling +daun ted +âĺ ¯ +walk ways +sadi stic +ridd les +o ar +ne ves +match y +lex y +kine tics +gil da +ðŁĺĺ ðŁİī +sant ino +predic tability +fo kker +ana ero +vesp ers +sy ne +stock ing +self help +r bl +mak ita +ju ego +in fidelity +hei de +devi ation +cur zon +com mis +ci bc +bbc wthr +ba hai +aaaa ah +ðŁĮ ª +wer ise +tom mo +seren ading +m itten +loose women +ite e +ic arly +ha va +gop ats +ufc w +the chainsmokers +t chat +seab ass +san ju +pepp ered +or illia +ministr yof +inf ant +fortune magazine +augu sto +ais ling +ðŁ¤· ðŁı½âĢįâĻĤï¸ı +scru bbing +rac coons +mon et +mcke an +jay y +experim ented +cost as +cam my +base ment +al te +worshi ppers +wal eg +t co +sier rac +santi ago +s ø +s ily +s aga +k sd +inj ury +fi jian +exeter chiefs +d ja +com erica +bee cher +u du +ti ernan +sol eno +show jumping +purr fect +mer tens +fr p +feder alism +constab ulary +ba shed +air max +syner gies +shi da +pi ña +mis lead +ma ud +eye z +air and +z of +wizar ding +w cha +tab u +spo ssible +sol vers +red zone +nhl stats +ne iman +mile high +mc vey +lew y +laur amar +incen tivi +i stria +goti ger +en amel +bb on +alco holics +ðŁĻĦ ðŁĻĦðŁĻĦ +we as +time pieces +swee ten +st ah +rehear sed +n wc +frontrun ner +fi vb +d our +cataly zed +bron ch +blo k +ðŁİħ ðŁı¼ +ven do +ra vers +obi spo +k alli +iner tia +g ny +d ni +bi hari +anaheim ducks +altu ve +air bus +ac a +we sts +voc ally +rati fication +nj it +lar son +izz ard +i ec +gb m +city wide +call an +bob sled +bbcwthr watchers +ìľĦë ĦĪ +sun risers +pediatric ian +pan ning +nar asi +liber ian +end ic +base balls +v anian +um g +tai ko +ri sd +magno lias +le em +ken ai +fric ken +dom ed +d atta +col fax +cephal us +adopt me +what a +pre mon +mass age +go buffs +enor m +dolla sign +dal es +bon aire +bertie schip +applau ded +ann n +wind swept +ss football +recover ies +raj at +pro tru +hoo kers +bio security +ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ãħ¤ãħ¤ +ton o +selvarag havan +pitt i +n ro +l pr +je vic +goog ly +char tre +ðŁĮ´ ðŁĮ´ +âłĢâłĢ âłĢ +u bere +sb d +ri vi +po conor +pan ellists +matt ingly +ken y +ibe w +foolish ness +farouta khtar +dream work +whit erab +west field +ten ors +mu sume +mo rey +md traffic +i af +easy branches +ch aff +carden as +ab vote +å ¾ +s ours +mul grew +me su +kd ka +food truck +der mal +child abuse +time share +se ti +pha se +oka for +lough lin +jan ine +around theworld +ॠĭ +rein forces +jane the +hel io +he man +dra kes +c sports +ye ee +vis iti +st john +percu ssionist +non violence +f ase +di ac +break y +" * +sn b +saf ran +pat ching +nickelo deon +intru ders +enlist ment +el les +cost ner +coo s +be sson +base less +appe ase +super se +su mit +sab ian +gene simmons +g don +frat ern +emph atic +d np +constra ined +clee thorpes +catal ans +an ae +yu en +sori bada +sky bet +saw dust +s film +nag ano +n ari +le ong +la is +in eligible +idi bia +go dav +disper se +bur man +an jel +re za +pough keep +ph oned +me du +ka ori +ive co +com uni +chinese gp +chim ps +twin kies +o ise +natge ophotos +na irn +mitochondri a +ju hi +cy lind +churchill downs +christma siscoming +atta ching +ar ras +. "" +timb aland +the hedgehog +sustainable fashion +summ ing +more los +me tta +man tan +kut ch +evan s +dazz led +stu ssy +royal family +roeth lisberger +prism atic +jam shed +ge s +brou ssard +blue angels +b mo +ann af +alis son +al gal +ë ī´ +wal ang +scar ab +m ingo +fruc tose +force fully +eu w +cri er +bai k +ar ter +alphabe tical +al lot +waz ir +to ffe +opio id +non existent +nephro logy +mc at +ing it +har ts +dad life +tx h +twit ters +tross achs +ss oa +so koto +rein ce +real bread +ray theon +ragha v +periodic ally +mayo gaa +gio vin +ed on +down graded +de pay +costac offee +colli ers +canu ck +vo tre +onthe move +margarit aville +kw az +gour met +foo dre +exo tics +de grom +daeh wi +ðŁĮ¹ðŁĮ¹ ðŁĮ¹ +te dros +ss rajamouli +ru ble +p news +ot one +ny i +fu ge +dam an +dal ert +as bury +allow ances +tel la +t dr +spir ulina +rugby united +rel ly +pass ers +oooo oh +medic ated +evangel ine +enti al +conditi oners +âĺ Ĥ +scoli osis +h ro +gift guide +g ally +dv f +cru mlin +moy nihan +mo disar +master classes +mac ular +be cau +bair stow +aun e +us gbc +thelion king +overwhel m +foo ter +and ler +she ard +ridge field +na as +n so +m sia +leg on +c sp +bo zo +autism speaks +as ch +ðŁĩ¯ ðŁĩµ +âĿ¤ . +» » +zo ella +syphil is +shim ura +sen tosa +new er +m clou +kri spies +im fc +gar h +g hazi +charle se +by d +ush ers +spread sheets +sel in +projec tile +p gm +over turns +must aches +mun son +muchach os +mol on +itss sr +ino is +fanc am +d cc +bu dge +pe gged +ing dom +cymb al +tul are +kryp tonite +ino va +feed the +f eni +ci ster +na eun +individu alized +fi h +fer al +ef fie +d so +???? ???? +syman tec +ss f +sma ug +si bal +okee cho +md pi +ku di +ho wer +gar gano +a pren +âĭ Ĩ +y is +w tv +thorn ton +subsi dized +speed wagon +pas so +mat ted +hargit ay +grave send +gi dd +friday fun +detec table +wild lands +w soc +tw is +san ji +sam bora +sal via +fakh ri +bella thorne +ak var +scint illating +ne er +n usa +m pl +leg iti +ku a +guer re +grou ch +en baum +ej f +col la +wind hoek +ut dfc +trey songz +stra damus +ro sar +mol ler +lordof therings +ill ar +drex el +dot tie +di straught +chaper one +bring your +bay shore +am ur +um ph +stock port +sitt ing +radi sson +ok al +jol lof +hor net +havelo ck +de j +cab bie +a arti +° , +van de +sch wan +let cher +lero ck +j mu +dw ells +dis qualification +bru s +amaze balls +ðŁ¤ ® +sc ac +radi ates +grow ling +ge th +et ter +dis fru +colo ssians +cd w +an arkali +alde burgh +ag ot +s west +or ro +on l +max x +imman composer +fro mmy +dam nation +d int +beer week +tribu to +til ak +t da +savethe children +pim lico +mississi pp +mar gau +ak ana +ag ami +âī § +wool ley +reven ge +over size +k res +ir ce +h news +et x +con yers +bill shorten +ban v +at el +v sphere +sule iman +stack able +petro v +pale y +pal atine +pa arl +le ch +kil patrick +k shs +ju v +hit am +ash down +abomin able +var k +uni an +u wi +thel u +shoot film +sand lot +pau sing +l lega +hor nb +íķ ľ +ठ¤ +Ùħ ر +y ha +wzz m +way back +t suk +stom achs +star i +pizz ahu +pa sted +nameis nani +kan to +car ley +be ur +ðŁĴ¸ ðŁĴ¸ +yn j +us army +sen eg +roa ster +mo rel +inthe park +ff acup +cre an +billshorten mp +ann arbor +abo y +rock wood +pill sbury +lu go +explor ations +broom field +az mi +atul a +akvar yum +show en +mc nab +d ws +wa see +nijme gen +john kasich +f pc +cr at +êµ ¬ +ื à¹Ī +velo ve +rose bud +orche stras +mortg age +flate arth +dailym irror +charle stown +bra ff +bo ku +bel kin +ãģ « +ร าภ+ti is +sacrif icial +lo esch +vide omarketing +un dul +supe rel +sh as +musi q +ki era +kam en +jam ey +encan ta +den u +ar cus +æ Ĵ +sor kin +son ali +ros alie +pushaward sliz +no ord +iam specialized +cap tioning +ðŁļĢðŁļĢ ðŁļĢ +sange et +rashtra pati +rain yday +paralym pian +must n +kun e +gen z +en viable +ef b +ami ens +à® ± +t de +re painted +ma zer +lay up +keh lani +jor gensen +der g +con chita +bloem fontein +all yn +synony ms +sten house +sli my +shal ini +den ier +assi stive +aquari en +am bar +subram anian +rebu ke +mam mam +ing ers +h itt +dog fish +cr l +am are +te uil +soci alize +shi z +rar ities +e ire +cincy tennis +benet ton +aven atti +ëĵ Ģ +un geneva +saan ich +r sa +poconor aceway +p liers +inter rupts +dark room +bau man +affe ctive +tou ro +tag aytay +sw ole +sc n +o ston +min ah +lam pung +coni ston +biken yc +bali ye +win i +spec trum +h ick +ely se +pet ter +i sel +emb assies +dj iglobal +dec ca +chal amet +an ony +ta ar +stemc ell +po sium +muen chen +bblo grt +app dev +anirud h +ad ah +toler able +sula iman +sec network +rhon j +prece ded +ob vi +kp mg +exclu sive +cou steau +une arth +space walk +pen der +il k +fari ous +excited ly +common place +bin ge +alec ki +a ert +w mma +transc ei +sw amin +sch ec +s anga +lec tive +ki pp +gl itch +f any +elli s +eal ing +di man +ãĤ¹ ãĤ¿ +ÙĨ ÙĪ +ville a +ver ily +putra jaya +head land +h elly +ë ŀ +un announced +techno logically +pushawardsliz quens +phra im +mar z +ma scot +kindness matters +hu ski +her ren +amary llis +a isa +sten osis +shi ite +mv fc +ml p +mirand alambert +me jia +lo ger +like able +ge vents +cold field +bu de +appli que +^ * +windows ill +ste mming +sql server +sh ur +mschar lotte +mscharlotte wwe +katerin burg +i spr +hinter land +fre i +er asing +concentr ates +blood bath +bk lyn +ari ka +st mary +prime minister +parap hern +pa ket +om ie +mun d +medic a +law yer +ka poor +gotiger sgo +enorm ously +dop ening +cur l +ang irl +ðŁĩŃ ðŁĩº +vo tered +oooooooo oo +om bré +neer aj +n vey +marcel lus +mar illion +el fon +dro z +ane a +abre ak +wont stop +sof love +sher idan +sch utz +ry ne +old town +kr p +jype twice +int end +ghanai ans +flying tr +doppelgän ger +bro lly +agn olo +ðŁ¥ ´ +ìĦ Ŀ +yn drome +y ate +mic keym +life coach +en ke +cap that +b ne +stere ophon +pal mo +la et +franc ine +bm x +âī ¦ +whit eri +til ting +post production +knicker bo +em boli +umbrel la +ri i +refu elling +rally together +ne th +matri arch +lg r +fore shadowing +eye witness +ðŁĺį ⾨ +u can +ty rants +pav es +omic ron +mir r +medit ated +gal atians +dro m +cabine t +buy now +skill ful +sha v +pit bull +meand ering +indic tments +gu tt +f ens +br ity +bar f +ìĦ ± +su st +sn ort +sky ward +reincarn ated +posit ano +neuro pathy +mag and +lit tered +line backers +jule p +car tons +ben shapiro +ax l +ðŁIJ ĭ +rejec ted +o ssi +gai ther +en sue +b gg +uncontrol lably +sur bhi +so de +sha an +re join +pre e +higg in +cav s +yu b +w hal +use rexperience +spoon ful +sli ght +sar in +sachin ita +rhodod endron +rep til +rel enting +refere eing +paral lax +mün de +lea shed +il ms +col onia +chow dhury +cer i +ap are +and son +ðŁİ ¢ +ìĬ¤ íĦ +åľ Ł +work loads +up ers +tenter den +snapp ers +sm acking +she v +redd itch +ilo v +dinosa ur +bi jou +bankof america +wag tail +vi se +ud hay +pic turing +festiv us +expe c +ep o +encro ach +co ding +ba ad +ಠ¦ +wye th +sc raw +ove re +n ena +l z +j anie +gar g +e de +arti fic +window sphone +ver dun +under standings +to g +silver ton +shack les +ho ppin +fa zio +election results +cbsd fw +ca pel +bio ethics +wrong fully +vel i +singul ar +pe sh +o chs +kat er +kar li +hango vers +flo pped +financial inclusion +fin ns +ff en +eart g +e sche +dy na +consecr ated +ce u +sam bo +s zy +reyn old +mat uring +lol ly +lau d +gel man +gear sofwar +g sl +fledg ling +epilo gue +cal led +bo ssier +zo id +yas in +whos next +stabili zed +spo res +spi ky +rol lie +ra vic +prinse sachinita +ph ds +mun g +mamelo di +maker bot +fur by +fin der +ct fc +brun ello +avengersinfinity war +ac cru +ab us +ðŁı Ŀ +ìļ © +âľĪï¸ı âľĪï¸ı +sp u +se pe +se aboard +power puff +impre ssion +gold end +ft f +e gy +drink water +b int +affl icted +Ñ ı +sch on +respect the +ram ming +piñ ata +park lands +math ur +la vuelta +far ia +disney cruise +deci dedly +simul cast +que bec +p ge +mit te +lc pl +ill ing +har oon +eu pol +enh ancer +der gaard +ard more +accli mati +á ĭ +wat e +tat oo +sh g +od b +la gan +equi pping +dhru v +cystic fibrosis +al aac +ðŁĺĴ ðŁĺĴðŁĺĴ +âĪ Ĵ +win theday +total itarian +it sm +elle smere +de kho +daugh try +childrenin need +by s +bak it +tallade gas +supple mentary +stu ck +pav lo +obla stoma +n jo +mix x +lan ez +krat os +kay aks +gar ret +favor it +civil ised +am pl +ac ra +¨¨¨¨ ¨¨¨¨ +wor ley +tri omphe +st ak +porto fino +pin ec +percent ile +om ari +kus ama +inverte brate +guild wars +gu id +ei b +bo gs +analy sed +san thanam +rang ed +le j +gains bourg +feel goodfriday +den hall +cros scountry +confeder acy +cen trum +blak ely +belgi angp +ðŁIJ¾ ðŁIJ¾ +ðŁĮ Ń +y aaa +up time +sound wave +renfrew shire +pati ala +mi m +k adi +hum bug +hey day +fox woods +fab rizio +ely sian +deterior ated +cover version +afrika ans +Ì ² +sn it +slo t +samsmith world +r dj +py aar +black hole +bar man +abstrac texpre +xox ox +where by +m raz +green est +fly be +dro wns +cu mu +bla m +al af +ain sworth +trump shutdown +sk at +set to +sc outed +mal ton +law lor +fini shed +emo tive +dynam ite +ar shi +ano e +жив оÑĤ +sing let +sar torial +ni shes +hel big +hart ford +boy le +ðŁį £ +z c +tuss le +sti ves +skir mish +red to +phen ology +matil das +jen son +integr a +heart ily +dolly parton +breit bartnews +b mp +ðŁĶ¥ ðŁĺį +ðŁĮ¸ðŁĮ¸ ðŁĮ¸ +way v +si stine +poughkeep sie +oro ssi +loc kett +hindu tva +dead man +aqu it +ðŁį ¬ +âŀĸâŀĸâŀĸâŀĸ âŀĸâŀĸâŀĸâŀĸ +Ñ Ģ +uni onists +ther oe +sm elt +r natweets +kal u +family guy +exagger ation +des ic +chate aux +birdc age +bic ol +anc tuary +ad nan +" @__ +went worth +u ros +se ss +se ss +power ment +mi sia +mar ku +gen itals +flo g +distill ation +bun dt +bor tles +w ile +scalli ons +sat t +imperial college +gu v +aerob ics +çµµ æıı +pope yes +pi sta +neglec ting +ik ki +house boat +ge ary +don er +spear head +sol aris +ob ili +eur on +dun stable +ë¸Ķëŀ Ļ +un claimed +spoo ky +persi mmon +it smy +fight in +ar ley +z eni +th yl +shav es +predic tably +me ach +may day +ma sti +hq trivia +bien venue +be bo +âĿ¤ï¸ı ðŁĺŃ +ô me +ve tch +val lec +v dc +spru it +pat ent +o she +guru ji +do ch +cor tical +cashe ws +bu eller +bau chi +super ior +sand r +r cr +ir in +hrithi kroshan +embr yos +dom ens +do per +cha peau +ðŁij» ðŁİĥ +yl ine +y us +un am +su kk +stoner fam +recep tive +os p +in ke +hil ia +green energy +gor od +cap er +c co +b wc +redro ck +ra ekwon +g yo +eu bank +complac ent +bedro om +ðŁijī ðŁijĪ +⼠Ī +живоÑĤ нÑĭе +water melons +total divas +spring dale +sp edes +slu shy +re ve +nur ser +men ez +bil lab +ad l +ç IJ +term ites +r fu +lo ll +ip u +cr acing +chas se +zi va +trilli ons +red fish +pat on +long champ +li sd +fol lo +fin ex +do goftheday +ce do +adap tor +wil lem +transiti oned +swee teners +ps vr +na agin +la was +kar no +guad ag +gal ena +exclu si +conspir ing +ber d +any ang +andr ze +tur an +stra yed +spl urge +personal finance +nat bynature +legendof zelda +food travelchat +delu ded +conce al +besto fetsy +ac companies +ab al +numer als +mb laq +dar rows +anach ron +ame thi +af ca +water color +under mines +sh ish +paraphern alia +ke gan +index es +hydraul ics +cl onal +campan ia +c bb +ber gh +======== ======== +................ ................ +the par +taste fully +scoo ping +s fc +om atic +mi q +lv g +itunes music +eng ar +du la +dra ch +dn cin +bloomberg tv +bever ley +bak r +and ha +âľħ âľħ +o bel +mah endra +la j +kun o +khatta k +k rug +hu iz +fen n +dn ce +colino donoghue +blaz blue +éĩ İ +vas eline +un cw +ts w +snow shoeing +refin eries +pho s +muer te +jumbo tron +in ners +im mu +e br +bri d +bram ley +bab son +at lus +a om +sim ha +rip tide +oh saa +dam pen +d te +bahrain i +vibr ating +st marys +redar my +gui dores +g di +fu k +bo bber +aler ting +( ^ +ver ton +retar dant +let tered +in vis +ha dd +gr instead +e wok +before and +âĺºï¸ı âĺºï¸ıâĺºï¸ı +yu me +thatdescribe syourfriendship +super lative +sovie ts +oro ck +lar cen +hy gge +hon duran +hilli er +hat in +h pm +est an +decentr alization +at ology +andre a +wi pro +typho id +stub born +scalli on +levit ation +esc u +dis sect +car done +bro dy +ay ew +alab a +ab ras +íĤ¤ ì¦Ī +sil i +rock band +rin con +mo cs +kick back +ju ssie +ar ayan +alai kum +ðŁĺ ¼ +ãģ¦ ãĤ +str ans +ship sinpics +ree ze +mat z +ko th +gun metal +ds n +di ved +cur ley +contamin ants +catch ing +tyne mouth +my k +mor neau +bud gie +apolog ised +adam s +ðŁĻĭ âĢįâĻĢï¸ı +ãħ ¡ +work life +mult nom +la fferty +dove cameron +a em +í ļ +æ ¨ +why dont +sur fs +st ü +repor ter +rec al +photograph yday +p isco +ko y +gram ma +dong woo +cor t +astro logical +ðŁĩª ðŁĩº +you were +u zu +ti dings +red bul +pre set +lamp shade +inthe air +icic les +hol zer +gi psy +gc p +cli x +bible study +w sr +the dog +tas sels +movi star +kur ti +im ed +icon ocla +fire dept +dg in +ant illes +a awards +sugar loaf +ric ken +motiv ations +ili st +hep worth +fan meet +do an +davi ds +chron ology +bol in +at g +[ !] +weh be +tortell ini +team dairy +new cast +manate es +mag alu +fre itas +forwar ded +college of +buffal osab +spor trelief +sotom ayor +nbaon tnt +matthew mercer +governor ship +al ger +wol fe +tit ch +stephen athome +ru pa +p onic +origin ating +nbc universal +info tech +eu logy +car ters +bum garner +ance y +yeg dt +wind surfing +st ons +poz nan +not ary +music is +men shealth +l pt +ha pur +el or +crun ching +terr arium +royal society +par ke +ner a +muru gan +mem grizz +joshu agarcia +hin ted +harmon y +ga ur +flu me +el rey +doc ket +be ga +twitter nature +s water +pu gli +ordin ator +one sies +mu kun +cru mp +bur leigh +ar chil +aftere ffects +stro mberg +pim ento +meh ndi +lo bal +kin near +intech nology +holiday season +con summ +cli ffe +cer f +buffalosab res +? âĢ¦ +topo logy +su ga +sne ver +skep tics +shinde shil +ru h +mar at +ll or +hear thealth +ha vil +bhar ati +ar ang +weare united +w kyt +o tro +minne tonka +mal ag +g sc +Ĺ ï¸ı +un rwa +twitternature community +seym our +se ar +r nr +q ab +linkin bio +ku an +ha ku +ch aco +butt ler +bc wine +sket chers +shake up +ram m +pol on +photo aday +mosqu itos +fotograf ÃŃa +fli ers +encephal itis +el as +du page +terra pin +sath ish +har at +g ell +fe dor +disc ard +co ole +am ph +adop ta +ye z +ty dollasign +the win +sub trac +roy ston +once abc +od p +i im +fa kis +diplom as +bru ising +vene ers +tu i +thesunday times +shop e +moneti ze +mo ol +mann kibaat +khil adi +ipsw ich +electrocu ted +el do +cyber space +car naby +ãĤ ¢ +tech week +swing in +stocha stic +mall ory +li r +land fills +kala hari +fa of +à° ķ +this is +rap sheet +radi ating +ra pha +p me +niti aayog +ne gara +mand al +kra bi +iam k +hin ting +erup tions +dmit ri +ab ington +up mc +tc b +raj nath +multi function +lec ted +grin ds +dj ian +cad bury +burge ss +bron z +ang la +ac mawards +yah weh +pu ss +lei bo +lanc elot +bang kok +back field +b sm +as ce +whit mer +tou n +pre ju +max preps +j crew +ed camp +deport ations +cho cs +beat sby +ash worth +za heer +val ery +tr ini +sy sad +sun dial +sti p +sange les +san gu +roman esque +le al +lam ents +hit is +equi fax +clu tch +chi apas +af sc +zig lar +un qualified +tend in +stanis laus +rock chalk +ri vet +rhon y +ra ppa +man tras +fromthe east +dy ck +boy f +bi ome +ba strop +à´ ¾ +tw ise +perenni als +multiple sclerosis +mccar thy +disper sed +dau phine +ber ner +aubre y +xen on +ss outh +sar ahah +par in +muker ji +lu ci +hyo yeon +evangeli sta +ce asing +an dis +tim on +lu sk +f ha +esof instagram +duke u +tex tual +steff en +sagu aro +ridic ule +re unification +leap day +kra ine +idol ssa +hot shot +financial services +envy us +con templates +al ters +ðŁĺ· ðŁĺ· +ðŁĴ¨ ðŁĴ¨ðŁĴ¨ +ãĤ Ĭ +tu gs +sl er +pro wrestling +po ck +patri zi +nadi ya +hahahaha h +be as +wan ska +sle azy +ri ku +rad nor +r sv +nadinelu stre +man galore +kil gore +inno va +green leaf +ad mon +å¥ ³ +u ously +sung woon +sho d +sal erno +roller derby +r tm +pitt a +pau line +ni mitz +moores ville +lan ark +jav its +indv pak +hi the +here after +gri pped +encin itas +edtech chat +do pen +demo lishing +beck ford +ban h +ðŁĹ ŀï¸ı +ud ice +taste less +promp ter +nat ter +mi el +ii hf +han over +guj rat +dis dain +b news +aw c +ab g +ãĤ ½ +âĿ ® +y fm +transm itters +tigh tens +stel ter +sc ouse +sal liance +ir v +ick a +fa inted +dethr oned +bo tte +sa hil +rhon a +proof ed +juven iles +isuppor t +gh ton +fli r +champion ed +c span +alde hyde +zam alek +waf ers +sul tans +sn apple +re capping +n daa +gov t +followfor follow +discrimin ated +dg c +brid led +âĸĪ âĸĪ +for mance +fac ades +du pe +de mir +bl fc +biomar ker +sin st +ry ka +ple i +ny m +nur tured +moi stu +mal aika +gh ill +eli os +court ship +cal mer +an ey +ag ye +yose ob +ved anta +uss ell +um l +trick ster +th ali +pen and +pe et +ob er +loo kers +ia as +gam ba +ethno graphy +bor dering +bal er +an en +walk man +then ation +ri dding +pen rose +la ssie +hydro ponic +east coast +wwe universe +tom boy +to ir +ro dan +p th +on ef +care ss +bee z +the comedy +son goftheday +sab or +rten ews +ro hr +peak y +pare des +in come +gre l +en is +chocol atier +cas sa +aon b +an f +ampli fication +accom plice +wel by +stre wn +sand well +o for +kim on +kim my +k dp +ik al +hoo pla +gan as +ei steddfod +drum stick +demonstr ator +centrifu gal +bl chat +ìĦ Ŀ +vit er +ssy dney +nan om +deter red +anim ating +aeronau tics +ab ull +tick ling +testic les +soo t +sax ena +qu ine +pet us +mousep ad +jo ols +german shepherd +b th +alabam af +ðŁļ ¬ +ðŁĩ¸ðŁĩ ¬ +uof glasgow +tra bajo +th ics +rap tor +pro stitutes +orlandoc itysc +heart disease +first nations +bo ces +ãĥ¼ãĥ Ī +âĩ ¨ +yueng ling +talladegas upers +tab ula +ske l +re affirm +pan es +ir k +d oun +chan tel +bron t +wether by +spec savers +sch ema +precin cts +pan acea +inf eri +gint ama +fir stal +fin sup +e studi +de in +c á +yu van +the bear +paley fest +page ants +krist off +har dik +ha shanah +cr g +bu do +amli ventv +a jan +ðŁķ ¸ +ठĸ +susten ance +onlin ed +nostr ils +mol ar +f sl +ente bbe +de ed +chival ry +bib chat +aj mal +adju sts +[ !!] +ðŁĺŃ ðŁĴĸ +w mn +qu ang +pil lai +misogyni stic +mar bs +its me +holy spirit +h se +critic ising +co ff +cm w +chel seaf +ch abad +ad ry +uru gu +tom bo +pl u +mass acres +jack o +it l +id capthat +hl f +go red +chri ssi +av ani +anthrac ite +am ous +t ity +su ggs +se maine +safar icom +po z +mey dan +medi al +kan en +je taime +il ver +gu adel +gre nier +duchen ne +ale ssia +abra sive +wind fall +t itious +ra yy +mind blowing +le b +kati a +in charge +fu d +chit ra +alvin foo +re dress +me gha +ha grid +du champ +cudd led +buc ke +woman hood +vey ron +pat ton +ou is +lar ch +j x +fla via +bran ched +bas ses +agron om +reach er +ram ses +ra han +prohib iting +pl er +pe eve +oo zing +luke warm +kru sty +hai lee +el d +ardu ous +' .... +watchthis space +vi ot +road runners +q mjhl +pel le +ned bank +mos cone +mam et +lit is +kosci elny +j uri +j ra +in am +han zo +hahah haha +gamer girl +consumer ism +chipp enham +centuri ons +as ya +ancho vies +ste ver +sk r +roo ker +que be +organ za +nar ry +l itu +kl cc +accompli shing +Î ´ +u she +sw d +official helly +montre ux +len ingrad +ic ola +her kim +fuer te +e wn +dilapid ated +dau s +colli son +cold war +boo g +à³ Ĩ +to dor +ter mite +shine down +on ye +mer ck +law of +garden design +fighter z +de grading +bra u +ange red +al labou +wra h +to logist +smallbiz satuk +s wati +mon gol +mari age +man uk +gold finger +em mal +cit rix +ar rhyth +quadr atic +pat chou +mcil roy +iteach math +art v +Ø ¢ +valdo sta +to ks +ste ppin +sal gado +moo k +maz ar +irish times +comment ating +brown ish +ac ism +ãĤ § +play list +ol f +lucha underground +kol b +gc f +ðŁijij ðŁijij +show rooms +rafre darrows +on nbc +mew two +kondap aar +jud as +j illa +goal scorers +g autham +dump trump +de bra +cov fefe +chur ro +t ando +ly medi +ergon omics +capit alists +capecod times +ðŁįģ ðŁįĤ +ws of +squ ish +om c +meghan markle +lha sa +jan ney +hust ings +photo set +kis an +gard ner +ben zo +bat am +z ito +sub ju +sar k +pun itive +maure en +kaw ai +groupp alestine +fi j +en lists +ch ini +bang a +w abi +vit ali +valder rama +sou thea +p ku +om x +flori an +cn d +bt u +ast ley +am ai +ach amp +heath ens +go lobos +dan ia +cn rs +authori ze +ar oo +. [ +wonder full +w pl +taun ts +sonom achat +pi otr +pan ache +mc n +exper t +do than +alex i +ðŁį ī +íĶĦë¡ľ ëĵĢ +u calgary +tigerzinda hai +spin nin +shar inge +migr ations +mac don +ma ssie +key pad +karls ruhe +ili g +har issa +ha vok +figur ation +d ld +cle arest +broad cast +brit pop +biom ed +att t +arto is +zh eng +slu tty +ser c +ro fficial +plex es +pe du +moul ds +le ek +dak ot +dais uke +chry so +bon fires +tick les +stun t +sikor sky +gr d +def rau +chimpan zee +bha sin +worshi ping +w ylde +w ole +thejohn abraham +s re +ra ig +pinst ripe +orient birdclub +mc morris +lumin aries +lou ch +la shing +gro omer +elo we +clut ching +cal ving +accessori ze +ðŁİī @ +the todayshow +t ld +spectro metry +pa ka +minot aur +man gi +karant acker +hay stack +fr d +ef en +diabe tics +bul i +av s +andr és +al ty +x k +uni e +sof itel +shi do +riteish d +mystic ism +kundal ini +ho te +ho sen +hin kle +good luck +go gi +fried rich +con gle +chap lains +bur net +ang lian +é « +ston ey +rede eming +random ness +pr sa +ober on +newh ouse +gonz á +den im +del ph +con ic +an kit +wolf ram +wine bar +unmistak able +power play +nag ging +lincss kies +gh h +desk tops +bore anaz +as port +ad wala +íĺ ¸ +theyre theone +sal dana +nes se +ci an +chemi stry +can is +b hc +zoo t +x an +sylve ster +ici dal +hmo india +gav i +gam ma +g itt +critic isms +bi do +be bold +aashi qui +tu ff +street life +ro mp +monk fish +mal evol +looo ve +k cl +gad get +d bu +ben carson +ail a +ì ¡ +re playing +noc turn +labe ouf +j hb +game on +ast aire +% ? +ðŁĺī ðŁĺĤ +ri yad +nyc parks +nm su +ly mph +kwan zaa +in sg +hack saw +gh nessy +dand ruff +basti an +au ber +atla ssian +al icious +wel ker +ris sur +pra h +pit ino +mt w +la thtr +jong suk +in subcontinent +ev elyn +dav ina +cri bs +cre u +cit ys +chin chilla +canter bury +adhe sives +tower of +su ite +rapp ler +op h +new sin +don ot +co ts +bair n +ãĥ© ãĤ¤ãĥ +w aging +sl acker +siem en +sand bags +of e +ig ars +hygi en +hcl dr +fuerte ventura +fore see +f td +f sm +ev ict +bun g +at tica +whitecap sfc +ugl iness +ko hn +in animate +gaf fi +fe yn +empire fox +dv ent +co inde +chuck d +aber gaven +ðŁĻıðŁĻı ðŁĻıðŁĻı +verse oftheday +titan ic +microsof tedu +l atives +eri ka +au f +adjec tives +ðŁĶ´ âļª +z ari +xi jinping +vir ul +the ville +tar ot +su va +s magazine +ri ggins +py e +isi er +der ick +barn staple +thu man +sprin ters +r mu +mexic ana +loo ters +lan i +jaeh wan +hi me +fr u +east end +cr amp +char izard +out ons +ni ppy +f xx +d agu +sky cam +ner kondapaar +chu gging +argent ino +alab i +âĿ¤ ðŁĴĻ +u waterloo +redi aries +mi da +jar os +in ching +hon i +gold ilocks +dra pes +d td +bi ed +anemon es +aku mari +ak hil +yam an +vel ife +surin ame +ru ud +r hd +kill zone +i my +hur a +es inc +cric keting +cor busier +bridg ford +ble sse +as sur +; " +wal lah +up r +thor in +sc bwi +re mus +ol oured +news stand +new sonline +mal li +mahar ash +li tho +jun ga +il ies +first friday +cu evas +clo sets +bur j +bac c +b hs +ae sop +a alto +wembley stadium +wal len +under graduates +stag g +pla stering +le l +ity fc +it ur +im gur +homec ooking +hear se +g se +eski mos +dr ys +dailymail uk +bi ot +arav ind +ðŁĶ § +âĿ¤ï¸ı ðŁIJ¶ +âĿ İ +tapi oca +syn cing +sw p +mcgin ty +i wata +hon ing +de graded +boy kin +aurang abad +aun ties +vienne se +unexplo red +pal u +look alikes +ham sters +for taleza +ed am +diction aries +care y +ty ree +tom tom +stra vel +re aring +periph ery +mcle llan +ju hu +i je +gd x +dent ures +au gie +architec tures +am ador +ac at +yu g +ve he +sh is +sall ye +kut v +impossi ble +chat t +billeric ay +war birds +turn in +tol y +the mb +sc lothing +nbc bayarea +lun areclipse +li be +kin ross +et es +dar ke +advant age +wing ers +stri ve +ru se +modi fying +mcilroy rory +hi ght +hair loss +critic ises +bob bi +autonomous vehicles +ar go +̲ Ì +sub bar +spar kle +sar dine +ran aut +nu c +na sional +lo kom +impeach kavanaugh +folk lor +defen sively +bigg in +ave da +غ ر +ubere ats +sy mon +mimic king +ini um +eatmore fish +ca zor +bo ds +a fore +< --- +ðŁı Ĥ +winni pe +tooth ed +seren aded +har ic +drow sy +domin oes +dog finder +costab rava +bob sleigh +bich on +all iteracy +ðŁĻĭ âĢįâĻĤï¸ı +ಠ² +out lier +n ites +lanca shire +idi ocy +guz mand +far ris +caernar fon +bar ney +az eroth +au dra +amazon prime +x haka +valent in +tumb led +t ph +retro spect +rajap ak +ni kes +nad i +lu br +giov anna +elek tra +de ku +cl b +cash man +art lover +anap hy +! ðŁĴľ +⾨ @ +west virginia +nur ses +mac on +hul k +heath ers +ach ak +ðŁ¤· ðŁı¾âĢįâĻĤï¸ı +sp ore +ling ers +kid ar +har poon +gran dopening +chel an +anaero bic +à ° +toyotar acing +tar on +rays baseball +pilot life +ori vera +kur u +c wu +alan te +ab ate +wil ber +tou can +the fosters +shar key +r illo +lo per +life goals +jam ba +gall atin +coin collecting +bhatt i +è¶ ĬãģĹ +utri ents +s rd +po h +o ds +fun ding +fili pe +digit ale +cycling life +c vt +aband ons +tem pah +tar sands +stat a +sher bet +prosthe tics +pi ppen +ne sted +le va +ferr in +da ho +af ina +sports radio +sam edi +li ffey +lex is +gen eve +cal lu +bri st +bar ty +bar ic +you suf +it up +woking ham +wizard ry +we ster +si di +pan sy +me des +ke ya +hilary duff +de barge +crani al +win esof +symph onies +she hu +re sp +mis ano +lin der +infer nal +engro ssed +dallasma vs +cron kite +ðŁ§ ļ +ãĥ Ĩ +se vent +sd avis +pru d +olu min +hog manay +gin n +et ted +cul kin +corro bor +x ti +we ck +ud der +sta ines +reig ned +particul ate +nu mmer +gro sser +gro g +gon awaz +f bc +encan ta +ce i +(( (( +ventric ular +tr k +ta al +o ong +no vena +n cr +lob bies +ini showen +in oue +i up +hallo we +fore seeable +con done +vegan food +pr ally +moun tb +mi ki +jake tapper +gra iny +gil i +gh s +gaw ker +forever more +experi en +ex asper +ep lus +chuck les +cervic al +anom tv +ah old +ðŁİŁ ï¸ı: +sphy nx +shon da +ra khan +pel vis +kil burn +ic or +as at +york ville +travel diaries +th ack +shan th +sear cy +n dr +looooo oo +lip gloss +it achi +hartn ell +gar dent +chriscol fer +ch ies +bor d +bla ken +nep tunia +my switzerland +mu mmy +d de +cl twx +ac ek +: < +sho ba +rico chet +mark up +fy re +fire rescue +christ en +al eta +zo oms +youre welcome +wi gw +unis outh +twil dlife +sun ning +sin tra +seed ling +ru gg +public safety +pitch ero +mm ff +mid fielders +kn k +hyuk jae +fif teenth +emb al +bra zier +ðŁĶ¥ ðŁĴ¯ +sp itta +pa chel +jour dan +gold mine +flip board +eric o +az adi +ë¹ Ī +à® µ +visit london +reco il +que t +oc up +ni vea +new combe +k ome +foss ili +duck dynasty +dev ents +csharp corner +cheek bones +aishwaryarai bachchan +ðŁ¤ ¡ +æ ĥ +âĹ ¡ +yar aj +tre llis +stra f +myrtle beach +ligh thouses +cr unk +ðŁļĢ ðŁļĢ +ê° Ģ +unsc athed +tt ur +team sky +real y +pin na +orthodon tic +nike sb +let me +lean ed +gro en +dono hue +bra sh +traw ler +taxi ing +ros sum +photo art +pakh tun +origin ate +nu ovo +more over +man ti +machi av +long fellow +inj ure +hen y +ces are +am v +ðŁļ ĺ +t lv +ru grats +reg als +pad alecki +lun ga +kh wa +jan ette +fc i +de tours +cle ese +ðŁĺĻ ðŁĺĻ +ãĢ ı +top gun +peak challenge +le thar +institu te +hemat ite +fri sk +( ´ +ðŁįº ðŁįº +vote fifthharmony +un checked +th rash +sassu olo +ra kyat +proof reading +new deal +ma ree +lo ins +letour yorkshire +godd aughter +elsin ore +companion ship +bon fire +big time +beast fromtheeast +ðŁij ¬ +el salvador +asse sses +amo ore +ahar ashtra +adul tswim +swan sofficial +star c +se wa +sa xo +old man +ga on +centime ters +bluef in +bet way +ast wood +art sakh +are al +ag ee +ag ape +ðŁijį ðŁijı +vul cano +unrival led +tues news +se khar +sac char +oni an +kau n +im position +goul burn +fru m +free man +fou led +fin all +eger ton +dri e +x uan +victoria beckham +ver min +trun k +tam aram +super mario +need for +mess in +me ar +io g +fe ces +ce tera +cab os +tren tino +re paint +on etv +off screen +niger ia +mccon nell +kin ship +fore igno +christma scountdown +bag well +çİ ĭ +yo kai +yar os +wad dington +ur band +real hughjackman +r wy +ou ette +mo res +llang ol +fly thew +dl r +bis choff +al ak +اÙĦ ج +vent ur +tab bed +st ls +seam aster +ratt ler +pro cure +nott s +con forming +ðŁİ · +william stown +var ou +tranquill ity +th rissur +sn ark +sevilla fc +pe asy +paper backs +law an +day uk +app iah +uri ah +som mes +showyour hits +sc ancer +mal inga +lauren ce +hurricanef lorence +bride tobe +bri and +blind folded +beg g +azzur ro +ðŁ¤ ¼ +tu stin +scy the +ma din +luxury homes +ker atin +gw yn +ff d +dam o +bt ts +be cer +Î ² +wid gets +var ner +tbil isi +shock wave +sa hl +rock wall +qu eria +kel le +invasive species +flam ing +ve tri +surf boards +sukho i +ox one +mm l +fr act +c sul +ಠ¤ +w ss +sar u +ro by +ra bin +myan c +erup ting +des ro +ci aa +ac ro +thyro idism +schla fly +parksand rec +mut ated +lifeis strange +gh y +ford mustang +dor ney +cat o +body guards +ani els +è¶ĬãģĹ ãģ® +sl g +s iting +resear ches +lofo ten +i and +cop ha +assemb lage +; - +wo t +tcd dublin +sten ch +no sy +net worked +ma eda +gher kin +cuper tino +com o +wre ak +shel f +padmav ati +mon ti +lol lies +ho tb +entren ched +tron dheim +srini vas +shor ty +shiv n +projec trun +low ly +lin wood +kier on +eth el +es ce +ðŁĺī ðŁĺĺ + ³ +wil ts +unc tad +smar ties +pat t +ne jm +mad hav +jayalali thaa +g tv +the city +o gle +mu sing +mcke own +matri x +f sf +Ñ ĭ +poinset tia +magne tic +fle as +ed hi +ed bookfest +bow o +ba har +x lt +working together +wo a +with refugees +ss chools +score line +run for +regre tt +ha der +e it +case study +ad ot +ab ha +ðŁĺĨ ðŁĺĨ +trac tor +sub culture +special ises +san u +pl tw +mis led +mari kina +maneu vers +hoo ps +gri me +fort lauderdale +dy spla +cel o +aw am +at our +Ë ĺ +william sport +sk int +å¹ ´ +t anna +shou ses +rheumato id +pla sty +pa wa +oscar pistorius +nott ingh +m we +lor raine +kar tel +i dont +har te +ghost adventures +g listening +ep som +a acc +ãĥ ł +âĶ Ĭ +watch dogs +time x +spec t +sp aul +salute to +rin se +qant as +plur alism +neil son +mo ine +maha bharat +mad don +electroly tes +du ches +adap ters +ا٠Ĭ +valen zuela +r ca +pit man +o ars +micro plastics +ho tt +ho ti +dou ma +dimple verse +der nier +commo dores +b boy +wor ri +seung yoon +or is +no ban +men shealth +i dy +hi g +greg orian +f sprint +conj ure +cazor la +but chery +ad versary +x amarin +thorn berry +t ü +sw ann +sta al +santac lar +repe aled +quin tu +qu é +per tur +mé tis +man ning +ic es +go ji +agne tic +ðŁijĭ ðŁijĭ +zo d +wal dron +tree hill +spo p +ig or +hal ley +cotton candy +ar kansas +acceler ators +vis alia +tr iceps +qing dao +od ac +li key +lat enight +itv corrie +empor ia +electron ically +cer ritos +b ns +are cords +ad du +ðŁIJ Ĥ +ve dalam +spar se +on tap +monoc le +la il +gn t +car dia +cap sic +bou w +bear dsley +bas i +plan ds +pi et +personal trainer +ow er +ol as +janu ary +jack and +environment alists +dr qadri +dog fish +vuel ve +th waites +steff i +schul man +les ville +food tech +stephen fry +pos ers +cur so +cor bin +br uni +ðŁ¤ Ń +tweet like +sme tics +rene e +post malone +pat ter +p sni +or no +ll am +i du +en tro +bl ica +b nd +the park +son atas +prime day +paw some +official bsb +dro wn +danger field +beach clean +ðŁĺį ðŁĴľ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +wefly asone +wa seem +rac kete +pri des +op u +grand slam +dolphin project +cun ard +zi ppo +wi les +sh b +san toro +muse o +me mos +inde xing +dri est +bronch itis +arte yart +world peace +theat ric +ke ely +inve stor +far ru +dam el +criti qu +coron et +channing tatum +body work +as ser +éĩ ij +road ing +quin te +nation ale +gu di +great ful +fu gees +adri enne +sal ish +quot as +qu elle +pro team +neo liberalism +n elle +khal ee +jaw bone +impair ments +go ggle +dul la +di ari +black adder +ven ge +spro gram +she w +science magazine +lind or +h pi +forthe people +fac esof +fab rice +ef ra +d ne +cate rer +canisi us +bu sca +bran den +bibli ote +bee keeper +ation matters +arri a +ðŁĴĸðŁĴĸ ðŁĴĸðŁĴĸ +y au +sub ways +state ofthe +sher ri +rho ea +patchou li +lu mpy +la vor +hal wa +creek side +coler aine +car bure +bloom in +albu s +al bers +ta ha +ren ata +polter geist +net gear +kaz e +dazz les +cre mation +chan baek +cal oo +mour ne +kor o +ha ight +gas se +fi m +eg linton +desi der +chri sm +bat tering +ak onda +ðŁķ ¶ +zip an +sen tedcruz +rin go +re me +or cs +mist ook +marthas vineyard +lu th +li vid +iti l +er tz +tag h +step dad +staten island +rol la +riode janeiro +province town +lu lar +ken e +expe l +boom town +bh vn +Î µ +sh as +se is +quatt ro +p fe +over use +moder ne +hype beast +folk music +fish tail +ca jon +ang ora +ðŁĴĶðŁĴĶ ðŁĴĶ +ðŁij§ âĢį +no filter +mc gann +lam or +hist stm +el los +cre we +art nouveau +am atsu +ac cs +a em +ðŁĺ Ĺ +ðŁĸ Į +yuk o +turk ana +torch wood +spi ffy +si ii +sel fridge +roc ca +ro chel +mat er +life with +len i +kil le +ij s +hard ness +ben net +t ml +son es +sic ili +road to +pric hard +p va +midd ay +chihu ly +back fires +ak il +ade v +& / +âľ ª +wf my +supere agles +rang as +py ri +pix ar +pan khurst +la hore +ho stel +expend ables +el nino +circu lar +bizar ro +be bold +ais d +tre ce +sr f +orland om +o ed +ny time +munster rugby +invent ories +gate house +gar m +camer a +be then +asser tive +ìĨ ¡ +âĿ¤ï¸ı ðŁ§¡ +we p +w td +t mp +coven ey +ceil idh +born to +aw f +autom akers +asi l +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį +wol l +thedaily show +t dm +sher man +scru ggs +samo an +rear view +over takes +mad max +geo logy +condi ments +by num + ¤ +wood fc +tan gi +san rio +oo ster +le u +k wu +hiber nate +cay man +bewit ched +ali bi +you ll +y fc +win ed +warat ahs +spit fires +sne ha +love dublin +impul sive +ibra xton +hop kins +har o +blue jacket +bee be +ar paio +ðŁij Ħ +Ø§Ø ¹ +ve on +tham mer +sh ta +pudu cherry +pitch perfect +me ine +me gs +li mp +j sc +fu dd +can et +bel k +acro bat +ó w +west meath +sa opaulo +pro jet +lam ba +frit illary +er by +dg al +deliver oo +ye eeee +vul garis +start led +repe ater +ray ban +ra val +por que +o han +ni eves +mur ica +kenne tt +haar lem +gro he +constitu ted +best boyband +èģ ´ +y aga +turn key +sx m +su raj +sk poli +s di +psycho social +nar cole +n and +level up +leis ure +kis d +jam ia +house work +cra dle +compost ela +comp iler +anne marie +aleksand r +su bic +season ally +king sland +jam b +jal and +f blogger +drey fus +din ed +cron enberg +conspic uous +co ton +ca pps +bo hra +bo gum +bal aya +americ anc +u mic +pau s +o kie +mul roney +mer maid +melo drama +lis more +it ations +im mol +ful mer +br ining +bol ero +bin h +ast y +we standwith +thunder ous +stub hub +ro by +r kc +path um +o ac +nb r +mun ir +legi ons +jeon ghan +habit ation +ge ht +cappado cia +( !!!) +èĭ ± +yn om +the grammys +tab lo +rec er +pu ller +ny ack +new beginnings +maynoo th +inf low +en stein +de ano +cr in +confection ery +berlu sconi +ash raf +aby te +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ıâŃIJï¸ı +wing sup +syri za +presci ent +new sad +nare sh +lis zt +gre ath +extra pol +divest ment +dis orderly +cu st +body weight +ave don +walk able +red fern +push kar +pro kof +mind charity +marin elife +dul u +as son +win kel +to ky +the p +t pr +refu ges +phoe be +ec fc +comic con +bro od +br m +asam sakti +adulter ated +qu illa +pol anco +po vers +no shame +montan o +kaz u +ham mocks +gu ana +el v +der ange +delle mc +bic on +bi or +bean stalk +ve tt +saur on +or bust +ol ic +li zzy +ik at +hand cuffed +fa p +ani an +ac ell +à¤ Ń +whatyou eat +syty cd +star cinema +s net +rat cha +om b +john green +jit as +h si +fun time +e ac +dad lani +clean air +bay e +zo a +wo on +wh smith +vo wels +un secured +steph on +st é +per v +i aea +ger maine +dis respected +birthday boy +ba on +as gard +âľĬ âľĬ +us atoday +tri age +tho ts +kipp ur +fam y +equal pay +dncin phl +del ph +dd w +al qaeda +" + +௠ĩ +str zok +sh ome +on ward +kas auti +hy ping +excell ence +caric atures +bblo gger +ay n +winter time +sac co +or no +musical theatre +la cher +juni per +happy place +ero de +dt p +color ad +rak uten +pla ss +no ite +mccul lum +hosi ery +ether idge +enrique iglesias +dru id +dra gan +com unic +( = +ðŁıĪ ðŁıĪðŁıĪ +uncu lus +twee ty +squaw ka +p russian +oro ville +m pe +ker i +colin morgan +ay ush +_ âģ© +. âĢĶ +à ¹ +shed d +she en +rose dale +pa chy +mix mag +incen diary +gil git +cat ac +bold ness +ambu shed +alco a +ü n +so con +rati fy +plu ri +on air +fl p +eng els +eb ner +bron zer +bro s +alig ns +ķ ï¸ı +ì Ķ +prosecu ting +profo sinbajo +obscur ity +n tl +lu ger +gonzá lez +epile psy +bo fa +ali fornia +' !! +ãĤ Ĵ +ठ² +wwe supercard +wt f +win ch +wee vil +twit song +tv guide +supp os +spir o +i see +ate ez +vy as +soom pi +ny dailynews +hand loom +co bble +bolsho i +az ing +aw olf +an kit +ðŁ¤¦ ðŁı»âĢįâĻĤï¸ı +way land +track andfield +tear oom +scoundre ls +po b +over fishing +en ia +bar bosa +alicec ooper +) * +we day +in activity +hel lish +dor dog +axi om +ë¸ĶëŀĻ íķijíģ¬ +thra shed +su tter +stra iner +so ren +ram o +ope ia +nikki sixx +ky derby +flori dian +callaway golf +c ml +bran ford +bird house +baby face +the cho +simon cowell +move able +meatfree monday +lo red +laun chers +ko alas +kingscol leg +ja v +gor gonz +femin a +car mona +an sky +z ep +verte brae +time y +skill susa +shir in +ser gi +re gan +pha il +north gate +mo eller +keralafloo ds +ke swick +iti e +har psic +fin cher +dc l +carmar then +amit abh +alzheimer ssoc +ab jp +pace maker +ore m +lyca productions +en sued +ee c +donald glover +bot tega +wy z +run d +pour ri +o dys +my ron +le ti +la dd +jc ps +heal ers +greys abc +fair mount +bru v +anton in +ajay i +ê² ½ +spr inge +press forprogress +p ase +lo ons +kellyanne polls +ic are +fre da +fox conn +de france +ag all +ac ne +[ âĢ¦ +ðŁĮ ij +v gc +show ground +pound land +olympi que +manife sted +kar as +jack fruit +instru ct +in our +il ab +hel sing +al meria +ì§Ģ 민 +Ð ´ +ver ock +tl m +oc elot +gas pari +data security +cher ub +c vb +birth ed +bel voir +bar rack +bak lava +ad min +ðŁĺģ . +un er +tech ni +su ena +rot ated +penny dreadful +pel tier +mic ally +f naf +cipri ani +auto car +any day +ÃŃ o +vignesh shivn +sr u +re ttes +mb p +marsh alls +legi ble +labrador ite +e az +day ne +con val +ci se +chimic hurri +black currant +bar y +ba ale +ash burn +ðŁļĹ ðŁĴ¨ +y adi +su bang +save money +on in +nhl allstar +k fc +grin ders +gates foundation +âģ© ! +ti dy +sky lines +mor land +full house +ex l +every man +to ft +pp l +perpetr ated +nand ini +mines weeper +love of +ingra ham +en elson +da as +cam pari +ann ul +a art +ðŁļ ¦ +stun tman +spr inging +nou vel +million dollar +in hib +her der +entang lement +di spl +com batting +battle star +whel p +tru ssell +srebren ica +rt é +o tica +mumb les +er st +coc teau +uc f +summer ville +suggesti ve +g pus +escar p +ed son +dg asm +cap ta +ab ir +zak har +woo kie +victi mized +thu pp +the book +stra it +sports man +scher zer +raj kum +own it +mc cour +le ib +hor ia +holy field +excel ente +est ilo +el am +e ben +coyo te +amazon ian +rocket league +ritten house +public lands +mat ador +manife sting +kar n +afri end +w yl +w lc +t mb +qui que +patriarch al +p sac +j ago +gi bby +de colon +contra ption +brid ger +astoni shed +å ³ +water sports +timeout london +ten o +quanti fy +nap time +moh fw +know sley +hei fers +gasco igne +free k +ei ffel +collar bone +brazili an +vi dad +uc davis +r itt +open street +moun ties +min ton +kryst led +k uni +hol ton +flash point +duali pa +will be +v alls +ry t +re issues +na die +luhan sk +l pool +guil le +di strust +des man +apo o +ðŁIJ ¤ +sin tl +rever ie +ma kon +le ve +jak ob +hor ni +dd b +cam ryn +ë ¡ľ +ymoun tains +wedding photographer +vit oria +tome try +tal ons +sche in +ran jit +pau lin +past el +or atory +neve rending +mon fils +library life +li si +indi atv +bin ks +bi da +ai kido +victor s +tur day +sport smen +shru gged +sal ves +re gn +peer less +pate k +jj ba +guern sey +exuber ant +black berry +wel fare +stu deb +quay side +nar ay +lou don +f wx +dise ño +cel e +bohe me +awe b +antic o +anthony f +ador ably +aamir khan +stock pile +pe tty +pas se +pa stu +moderni zed +man z +le vers +jun tas +gras shoppers +a stre +w ta +thegreat awakening +pas coe +ng k +jen o +d se +cla vic +& ' +we faq +we care +sun spot +rai den +optic al +once acard +jav dekar +house made +first born +erec tion +de angelo +bal ah +alp ina +ðŁĴķðŁĴķ ðŁĴķðŁĴķ +sla shes +ra ucous +pal ac +hey man +gh ir +fo yles +crou ching +changing lives +au la +as ghar +apol lon +ab stin +san kal +monro via +l alo +kangan aran +car yl +birth time +am ano +ðŁĴĽ ðŁĴľ +unapologe tically +som o +sh of +sey i +prop tech +ong ed +ni hon +nas ri +mat tia +man ik +lo gar +jur ong +it ti +hay dock +don russ +dis respecting +carnegie hall +ano dized +© ï¸ı +tex po +ss an +robin williams +pun che +mon ero +mo hair +manit ou +interst iti +home red +fsprint monday +tan u +su bbing +shiel ded +ratt ling +rak itic +quest love +man orama +look north +jun hyung +isma el +grumpy cat +fro lic +escal ators +dÃŃ as +de bar +colleg elife +cle v +brick work +bom er +all ama +y urt +vul a +spreadthe word +ped ics +lom ax +l uring +kr u +human resources +he dging +har wich +goo ber +crutch low +cly ne +y v +thessaloni ans +spir als +plant ar +hang man +hai fa +gyp sum +gl p +gameof thrones +advo care +wa yof +star times +or a +occu pies +misogy nist +ku d +kello gg +g under +foli um +emily bett +big blue +aw oman +sp aring +sop p +par ound +mind body +mail boxes +la zer +j so +great british +gi lead +f ba +ch aves +ce vents +c so +aw oke +al aw +ak ong +young thug +ru ll +poly styrene +pe ñ +oko ye +lip stick +ke fir +hi x +flu oro +di aling +am ana +traxx as +su j +stra m +sas soon +mm snippets +han if +fiduci ary +co stan +blu shes +av ale +af p +/ ' +è ĩ +wedding venue +univer se +shrin ers +reli ef +ob in +mike the +mat os +jo ckey +jam in +intric ately +il da +gli ders +ex xx +yu lia +wg me +w pli +sc lo +ra ker +patter n +ob it +master works +landsc aped +l sp +l ers +kav ita +ih g +fly past +extracur ricular +end anger +cape breton +bra x +bor row +action figures +w bbl +tal kers +sau cers +re adi +mam mal +m sport +i ee +g hal +back light +b ww +ak ane +sti ve +my peakchallenge +il ana +sand ed +pi ety +ke es +hur l +harry shum +eag s +dro ids +do v +city news +brai ding +barclay scenter +band era +ภŁ +y ada +wedd le +varou fakis +swel tering +special needs +sjo fficial +sax ons +riaz theboss +rashtrapati bhvn +mi ms +kro q +har nesses +g sma +freder ik +dy an +colo res +centri st +brain wash +be ow +ay ton +ax o +aureli a +ace vedo +ç¾ İ +ا٠ģ +yas mine +stel vio +scoo by +mul van +i got +endo scopy +dil bert +ðŁĴħ ðŁı» +x drive +power train +h de +foster care +eloqu ently +carbon dale +wh all +un ing +ti fying +superintend ents +sm fh +of tware +mu tts +krystled souza +far thing +transm itting +sig net +portrait photography +o varian +kit sap +kar ya +d alian +b sb +b ho +ari zing +ãĥ³ãĥĢ ãĥ¼ +à¹ģภ¥ +sam bal +new in +music ed +monaster ies +marke tresearch +lovel o +di op +deta inee +whe ate +sol er +sa wyer +red ales +lan es +dan zig +bac chan +b iss +austr alis +ab acus +what vegan +upp sala +tull amore +soci ological +s van +ru ffin +nepo tism +ms gs +ke mi +ka hu +ex pun +ec ks +ðŁĺģ ðŁĺģðŁĺģðŁĺģ +whit church +w elive +un block +u cr +tow path +sen er +rede ye +r ch +pear land +o afc +lamb ton +imagin ations +fashion wk +daily doodle +ay man +apart ment +í Ĺ +tahir ul +sei do +ok on +o jos +mu dra +mor tuary +is thenew +fore fathers +fern dale +del phine +carre four +bor gs +ðŁ¥ ¶ +tie fling +th and +sel hurst +re ya +nei ge +mi ha +medic board +jann at +i movie +hol dsworth +gu en +gat ari +garden ia +cho bani +ca sta +ben nie +yule tide +r se +proud coach +lu te +josh ane +gu era +gl ac +far mb +exu des +eng le +battle fields +ap akistan +ÙĨ ا +ss gt +shar pest +power shot +mar fa +laur am +harryshum jr +go ole +espan ol +dis d +cas sel +cam ise +argon ne +yos bourne +uk bloggers +ta kay +m strong +lubric ants +kine siology +kas am +game sh +eu ri +disc golf +dev ans +cha os +auto desk +âĿ ¯ +wim ming +wa id +valiant comics +simul ating +po pes +on dor +mari anna +lop sided +isab ela +game maker +flead h +easter weekend +bhu mi +bar ges +ani shin +íĺ ģ +sand ara +may i +lo es +kin son +godav ari +b fast +avi onics +ab elle +. ðŁĺī +loveyour petday +kru sh +import ers +fro mage +east side +e ey +c mom +bo die +bl x +ul cer +tm h +sa ito +reti ree +ps al +pret ties +maritim es +magalu f +m fm +jenson button +in am +car hartt +bun da +avi gnon +need a +ip f +ic le +dews bury +bar ker +andre ww +ðŁĺĿ ðŁĺĿðŁĺĿ +ther mos +sonic thehedgehog +m nc +la vine +glo u +car club +ag gs +ac tional +ac cou +ab cf +y atta +vis co +ver bena +syner gi +pri mes +phar ao +p ella +ner uda +mo tos +guel ph +cor re +bang erz +aw l +auth ackeray +all saints +ae v +un circulated +ste ading +precision medicine +o stomy +must see +mediac ell +kwe si +ju al +im c +ghet to +fla vi +em pathetic +dip brow +criti ques +cri si +ador n +ðŁ¤ ¬ +stap h +rol le +re tur +bab yyy +ye et +wild horses +wel wyn +stop her +see ger +reiter ated +nl ds +lo ge +head ings +gorgonz ola +en caustic +di u +di sta +dam me +ch iron +bike to +ðŁĮ¸ ðŁĮ¸ +war is +usa hockey +the placetobe +snow boarder +sheskindahot vma +sal ome +owen sboro +k ler +im perfection +if ta +house keeper +gu v +game changers +est amos +bu kol +bom ba +tol ling +steal thy +sta x +sketch up +sc lu +pol and +mis cha +jin ja +gre go +da star +as al +ar arat +ãĤ¤ ãĥ© +ಠ® +tw irl +t ws +pu cci +par ading +kal am +is fahan +himach alpradesh +et l +copy righted +co heed +ar kin +ah n +ag ad +ack o +ac op +ðŁļ ľ +ðŁij Ĥ +âĺºï¸ı ⾨ +yan ew +un divided +ull man +t q +st ds +pa sion +minim alistic +menis cus +jo st +ich thyo +gol e +wsb k +spo kane +leav ing +kan n +iter ative +cel ica +bl arney +ðŁĴ Ĩ +zz les +womenshi story +vick sburg +un p +swa b +sof tball +ro or +pamp ers +pa ch +ni ya +neutr ino +it f +haver ford +groo vin +fa thom +f mx +art space +ab ounds +âľĬ ðŁı¼ +t ams +n ham +ju ggle +jol ene +brandy wine +augu stin +ðŁĴ£ ðŁĴ£ +ëī´ ìĿ´ +z is +sm d +pa del +ni ec +man fro +ke iser +grown up +blo opers +bar tow +ad hi +run ways +rang i +portu gal +papp as +men der +mal aw +ex ert +amwriting fantasy +çĶ » +åĪ Ĩ +trek ker +tele matics +sf d +pap an +ou tro +optic ians +niker unning +lmfa ooooo +lal it +iso bel +fair play +expen sed +canary wharf +call for +be ster +ah li +zambe zi +ut ara +stru mp +sal to +pen y +om id +obstruc ting +ne re +kre bs +glyce mic +ext ant +dominican republic +cour ting +ar re +x eno +ren ta +new video +make the +horn bill +gu ero +fut sal +fertili zers +d di +constan tia +ó ¾ +trustthe process +tid bit +te ese +st ler +seri als +pr ate +lan ai +ge ta +feu er +bun dling +tent acle +silen cer +short comings +safe guards +pal atable +pag ano +missi t +epile ptic +ed h +de santiago +bur k +alab ang +wsoc tv +worka holics +we iss +uto pian +ster t +om ma +loo o +lol la +ho on +gre ggs +beat y +we br +up d +un committed +trivi um +t ce +pine hurst +maple wood +gor gon +ek ta +anthonyf joshua +son ar +oste opathic +gru elling +director siva +d th +boye ga +boli vian +tan gel +strate gy +st paul +shel burne +sch mitz +pla c +pal me +niti sh +mom mies +human a +fern anda +faver sham +di ana +chu ckie +âŃ ķ +vig nette +sail ing +pre show +li gt +kar loff +hou sen +h ft +em pres +be vel +be sh +âĺĥ ï¸ı +ta iling +silk screen +pri mal +off erman +mil dura +king sport +ferr ous +dg en +chair manship +ðŁĴľ # +sp outs +sil ove +schme ichel +s lau +ri ken +mc clintock +lu strous +k lau +jumper day +go atee +global isation +ari ef +after shock +zi ki +we aver +smo m +sa si +recor ders +ra is +pear son +ip as +i pe +humber side +f ce +buck y +bo ars +wis s +re ine +prob st +ph ong +intellectu al +handic rafts +fd fs +enterpri sing +cocc al +cic lismo +carr ara +b vs +ak c +ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ +yy ah +web shop +sym biotic +stu bby +me phi +mb ang +e sea +but chered +u vm +revol ting +mac ca +hhhh hhhh +gun tur +el be +dragonball z +catch phrase +at tic +an ee +vo e +vio lette +unra veling +tu ms +subur ban +struc turally +stre ls +se ch +re si +puj ols +pras anna +om arosa +nar ro +lumber jacks +ja an +free book +boss man +black ish +av ali +ãĥķãĤ¡ãĤ¤ ãĥ³ãĥĢãĥ¼ +vo ce +search light +rejuven ated +pr n +mar th +goal scoring +gi vers +ga w +fat ter +vac ated +ts ons +ta pia +shri ya +oswe stry +op ter +now all +mas ss +lang kawi +janethe virgin +carlo sp +budd hi +brit ains +be eps +ak han +w ff +prep on +navar re +kas ar +gran dest +elev ation +ele on +bra ithwaite +beaujol ais +and ante +าภģ +yard age +stal emate +ol lywood +ne ssie +mam matus +inf alli +de ering +crude oil +angu s +am ex +!!!!!!!! !!!!!! +olm sted +lin n +iri ses +in xs +impac ton +faul k +curren tly +ba az +wan ews +ske wed +shind ong +re po +p itu +oo ops +mar zipan +mar te +make sme +h bs +gedd es +g bm +first time +es ks +asi ago +sar copha +nal oxone +kab out +ing rid +globe trotter +c sir +back fire +ba reng +y w +sa adi +q asi +opportun ity +ni khil +ms v +mau ri +iron mantri +iron man +ingraham angle +indie authors +grey son +íĭ ° +x mr +studio green +se mma +ridicul ousness +rec tory +kon do +inf ancy +el clasico +deli very +an caster +ach in +Ì ¶ +sv pol +sun o +sp anner +solidar ity +ro hn +eag les +d ti +clau dette +â¬Ĩ ï¸ı +xi amen +word play +wiki artapp +v ath +sur fri +pat anjali +ortho pedics +ingu ish +gle am +eu elections +epi genetic +cold water +ay a +ant agon +aer om +ade a +ab as +Í Ļ +ze eland +sig graph +phon o +of com +mar ni +inve stec +im not +gener alized +fromthe past +ad ob +âĿ¯ âĿ¯ +what doyou +theo dor +stitu te +sac s +k app +di mas +cos ford +carry on +book ends +ai mim +Í ľ +red sox +k mc +jun cture +inhal er +harro gate +afro beats +vin cere +subli mation +so ton +pe per +mid week +mi zer +medve dev +lombar do +fineart photography +col eridge +co i +chu b +cardi omyo +bro phy +balear ic +aster isk +ar und +alabamaf tbl +wrest lec +un loved +time ter +sav vy +ro sas +recy cles +r cc +mi speedway +mat aram +lund qvist +lo vie +fare awards +classi que +boo know +tr n +pu ddings +post o +magdal en +dav entry +carnival cruise +bureau crats +beforeand after +b dp +we tzel +spring field +mosco w +hepat itis +gre cia +game development +dro it +diversi fying +class room +ut an +up dat +s itters +port ada +ou bli +novo tel +nag ar +kw ang +krat om +croy don +ax on +à¸Ļ à¸Ļ +ye vents +wan aka +tion less +t sm +shab bos +refriger ated +ra ku +om f +mari en +lead right +kla as +k ering +jen kinson +inte x +gro te +galaxy note +delu sions +chu mp +toys rus +st pi +spell man +som atic +red und +mci ver +k pi +inqui res +icon ography +dro ck +astro loger +abc tv +u cir +su mer +retri eved +per vez +nik las +kar olin +infl at +hol l +hand guns +good beer +food festival +divin ation +dash cam +bbc doctorwho +b caa +łĪ ìĿ´ +ì¤ ij +tri pel +sizz ler +ro op +q na +m gh +lindsey graham +limo ges +j anna +goo oooo +ghe ads +curi ously +cl á +cantal oupe +brook haven +blin der +barri os +yas sss +ws room +winter fell +v cf +su spiri +st end +ro omy +r aro +marchfor science +har dracing +fc p +fat wa +end zone +dol lop +ru dra +rio ting +poul ter +poche tte +on ds +o ge +lu igi +impro v +g bf +del as +can tik +all you +wasee mb +sno hom +poster ity +pash mina +nb avote +mg k +de shaun +clark gregg +cav ing +ั à¹ī +ਠ° +ug m +mil os +live sport +ho vers +gam blers +four th +form en +fire up +far ing +execu tes +dumb bell +su cht +sny der +sle d +scorpi on +rie gel +fe asts +f dp +di b +conne ctor +cb doil +ar gon +âĢ¦ ) +u ft +ss ou +re trace +ms b +lone star +kin shasa +jam balaya +fan z +cyber ne +seneg alese +ne ther +mid ori +law enforcement +jawahar lal +harper collins +burning man +at itis +adol phe +æ ĭ +wheel barrow +tan ah +si ff +saw mill +rose bowl +own er +nt l +nathan sykes +morten sen +kan sai +kajal aggarwal +he user +es auce +cet ace +༠ĭ +welcome back +tin c +superel mo +repri eve +prokof iev +pis mo +go vic +en j +corri ere +bel ushi +ali za +ur ya +tb g +se va +nd l +jaf ri +ghost face +fino alla +de war +colli des +au sten +ãĢ ľ +ton ey +though toftheday +ta kara +sher ingham +shakti arora +pal ak +mut ant +mk r +lo ony +kno tt +in cest +gul u +cri ssc +central parknyc +c ca +bar bs +x ander +supp s +sky y +samsung mobile +nail polish +mak toum +le da +lar der +four nier +dicho tomy +bigre d +as un +ale sso +t dim +suz hou +song stress +pla sm +mind sets +keer thy +ju k +i in +che sts +anivers ario +after shave +teacher life +star fire +shi vani +peck ers +pancreatic cancer +kana wha +inst inc +htc vive +bulldo zer +bliss fully +angel o +ÙĪ ÙĦ +Ë ļ +ze es +time frame +she etz +ser o +sent ral +pt g +nc is +nak o +lo pez +hive works +hany u +f pa +enab lement +electr ically +cam ilo +caled on +ade yemi +team love +revolution ise +mach ined +fili pp +fate go +d hu +chri sp +bon afi +b sw +tus ks +refresh ingly +muhar ram +high roller +fre eland +dat as +cru ella +twee dy +see b +per k +merit orious +lu do +l wc +in dc +ic hoo +hare wood +bur rowing +bur ris +back waters +al don +sun devil +sne st +ph es +part i +ka ha +eng l +con cu +bed azz +ðŁĴħ ðŁı¼ +tin ts +t mg +shan tae +nighthaw ks +ni es +miraculous ladybug +ming us +ma kings +lhh atl +joy news +i ums +bystand ers +!!! ] +âļ½ âļ½ +wo g +vive k +tr attor +to bler +simpli fies +sc er +new z +lam my +jay ne +ham my +hair less +gra u +gat uck +fri ghtful +e au +delinqu ent +chech nya +ari sen +ali ons +! âĿ¤ï¸ı +ઠ¾ +ਠ¿ +ve get +town sh +te entit +tc as +soo k +sl acking +roman tics +rock steady +orange ville +neural networks +motor show +maya wati +ma hia +lu sit +isi ah +er ken +ch allah +a one +Ð ³ +ö ster +u aw +the matte +si go +ro bust +mo hali +mi staking +maje ed +le sion +jc penney +fung icide +dy k +comic bookday +boba fett +row d +potenti als +post punk +jig s +inf lows +inf ar +en vis +ds r +der py +big ten +vast u +signi fy +puer ta +poo le +lindi s +lim itation +i sps +dor is +co satu +chromo somes +boo the +al arm +ðŁĮ Į +weare uk +vare la +sun glass +sec def +savethe bees +s love +per missions +mi zor +macro photography +girl friend +emmanu elle +des don +cl m +chesa peake +cal is +bo ps +ðŁ¤ ľ +v st +no vices +me son +love our +itten den +it r +ir oned +clu ster +char i +cap sized +ave tt +asy lum +arrang es +ab angan +zi er +yo d +u sos +te el +sne ek +ru der +ori el +mcne ese +kill the +kid ston +jam my +inexplic able +ho th +griffi th +galax ie +death stroke +but i +ðŁĺį ðŁĺİ +wick er +thi essen +san gha +puni shable +pre ma +me u +interpre ts +ida h +harsh vardhan +gen naro +ff p +exhau stive +e ke +cha hal +catacom bs +amaal mallik +Ù Ĥ +ut r +ul in +tab lo +south paw +sor or +road blocks +ren zi +pre term +lead generation +he dy +gun shots +feyn man +e phraim +de grade +d ama +b cc +am und +af in +ðŁĴª # +ðŁİħ ðŁı» +yand r +tari q +stre it +store ys +sky blue +o connor +naz ism +moun i +macar oon +i bex +gen ces +gee tha +free press +dayo ff +cle at +bad awi +an ko +ðŁļ Ĵ +âĢ ı@ +tornad os +stra sburg +post ale +om ers +nim bus +murri eta +il yn +hou ser +equ ate +eclip sed +dis liked +aleksand ar +al oo +aku fo +âĦ ĥ +ty ron +t with +sport sday +ml w +is l +in experienced +hom ily +ex xx +depri ve +deon tay +can ter +bin di +arab iya +adap tion +tu belight +lo ong +le ith +ku wtk +ke ta +ka izen +fro ch +el ish +el if +dun ya +diec ast +communic ates +black outs +armen ians +åĭ Ŀ +á ķ +ri ke +park land +ou tri +new bridge +national donutday +hender sonville +hailee steinfeld +d antes +co ffman +z ano +t lp +summari ze +sud hir +sole dad +rami fications +pee te +otak on +ng ss +loop holes +le der +insomni ac +h pu +gag ne +dhe er +const ables +boom studios +block age +bas sey +as ad +al ittle +ac le +رÙĪ ب +whi pp +tweet master +tg it +re manded +moff itt +ky an +kno wh +kidnapp ers +ki di +ji won +in abudhabi +drive time +cic ada +chitt agong +challenge cup +soul less +pos i +morning star +manny pacquiao +fre eride +cap tor +bro se +bak ker +alli um +yy y +xen ia +we iland +vad achen +se dent +luxury realestate +lac er +kanganaran aut +ire x +hager stown +format ted +fashioni stas +ec f +dema go +clone wars +cha day +book reviews +ay eee +at trition +asu per +am d +actor leeminho +abp newstv +ãĢ ĭ +wr p +su al +stead y +shim on +re gener +photo sof +n tsc +martin sville +k vb +haw key +fish bowl +fil thy +din ge +dar on +cher on +barn acle +archan a +ut ilit +us abasketball +shack le +ol itics +mel endez +lat h +graph ically +geor gin +dv b +dig itali +cô te +cover dale +che quer +assimil ation +under appreciated +todd ler +qu arti +pig lets +p ounded +mom entary +int eli +ent um +dal glish +chop ard +aqu at +al lam +ra scal +ole mis +mat era +late show +heff ernan +ex ols +en emy +com stock +ze gna +wel la +viol ins +t ti +studeb aker +scotti a +pash tun +on wisconsin +nca as +n lin +mis anthro +lul z +deco der +con lon +cau sa +ay s +anaesthe sia +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı +vine yard +on it +ko ep +ho echlin +heart land +haw thorn +free mason +blood lines +bbc f +a ot +! âłĢ +ye bo +ple ader +ne ssi +matt j +ke fal +door s +cylind rical +crumb led +conve ctive +bruck ner +waseemb adami +vici ously +tu bbs +treat able +slide share +shop lifting +port landia +nor cross +mix ology +leap frog +inte stines +hw dsb +harness racing +ha igh +glo bin +ge au +flori dians +eli quid +dread locks +collap sible +west co +sang at +rooi bos +mobile apps +her vey +feature friday +et us +cor relates +biz kit +ank let +ta ff +stick er +sof la +r tf +proc ter +photom ode +ou ge +me sa +mal acca +lar ks +kot lin +ki hyun +inaugur ating +godre j +car le +cal o +black listed +? ", +âĶ Ī +trouble maker +tol le +spot lighting +s gr +rep sol +rangas thal +pan demonium +fr é +forti fication +custo dians +capu chin +alab aster +w ound +un sere +sal tire +now ruz +n tsb +move on +mahi rakhan +head start +distin ctions +annex ation +ðŁĴĿ ðŁĴĿ +yas ser +westham utd +th ong +sab road +ro ld +po bl +oc p +loun ges +l fp +go ffs +franco phone +devo tions +d ni +alex i +tur ous +taj mahal +se ad +ram ach +neo classical +l anny +hal p +ðŁļ £ +superst itious +show y +relay forlife +pi sto +part ite +no dding +it n +gra smere +for zam +fire emblem +ethno graphic +die hl +der i +dat i +cam bma +br anca +apo li +structure fire +spark ling +n sta +kosci us +joo st +horizon zero +gun nar +der ives +dc ps +cre mat +choc ta +cay uga +ble mish +bi olumin +bar co +al ky +a health +u aa +sub mariner +quot able +ponder osa +nan omat +meen akshi +jojo ba +im als +gu ia +dig is +der ich +corti sol +coach j +bra gs +benef itted +ðŁĩµðŁĩ ¸ +ðŁ¥ ij +wol fie +tn es +quick en +lov u +jam m +it sin +fe es +because of +ðŁİī # +sa ket +oli a +ke ds +happy diwali +ful fills +ent omo +cle xa +anc ity +alon ga +! ;) +william shatner +sunday supper +sun ita +sh anna +rep in +mar l +madd i +kis sy +ke mal +k fan +jak ub +hoff enheim +ha ko +fron tera +danden ong +cooper tire +cau tiously +bon gs +b him +angr ily +aldu bi +z p +snu ggly +sas kia +preci ous +prakash javdekar +infin iti +in roads +cur bs +cat i +bu stin +black women +ben j +ballo oning +bali k +ðŁļ ĵ +thin blueline +ther yan +the justice +shipy ards +li bros +j ase +gre tna +fi ba +che khov +avl news +une sco +tr g +ro dol +ri ppin +pit to +pad lock +no tan +nic he +ink ling +haver hill +cro hn +chicago tribune +back flip +ty p +thursday thought +social good +re ise +pw g +nor r +nepal earthquake +min y +metho d +living wage +jon gup +duke of +cub ism +b ough +âľĬ ðŁı¿ +âĥ£ . +youn gs +yaz oo +whi sper +tre cords +su erte +me tax +long list +kub ica +indv nz +ey y +cla ren +bread crumbs +zig gy +yu suke +win king +tt rell +pa pel +m sps +gb ps +wag on +ut p +thel ondon +tang ent +standard news +south beach +sece ssion +fri c +felici dad +ev alley +en bridge +cour sera +chro m +canni balism +burn ham +bill i +beau champ +accent ure +ðŁIJ Ī +u mesh +to vey +smile day +pass the +lar i +jas mine +hodg kin +gaf fe +forest service +f mc +enth used +dis advantages +cur ren +bl ended +ë © +¿ ? +univers es +tweetab ond +tau po +s las +pan ahon +occu piers +il len +ec y +dro ppings +boo yah +bar as +ba cha +af r +ภ³ +wi u +prece dence +o gier +nca aw +manohar parrikar +mac aque +io a +gh man +france sc +burj khalifa +bog dan +av ala +trans for +sto go +pon to +n sn +ch achi +catholic twitter +yy t +un environment +sof ar +pen ance +mole station +massi f +line ker +kcam exico +buil der +bak ken +apic ture +si res +sav oy +relin qui +mu ñ +mag con +k her +i aa +fo ther +fam a +editori als +chel sey +âĿĹ âĿĹ +z inn +w ud +sel fre +schau mburg +lo carno +dar cy +co star +austr o +âĿ ® +Å ¡ +wa ha +to end +sus annah +stag nation +shi vering +sau mya +ra gin +paul wesley +maldivi an +combin ator +aven ue +) ~ +âĺķï¸ı âĺķï¸ı +whitec ol +well come +tra iled +ste pup +smar kets +savo ie +ren ard +rabb itohs +prefe rential +panas onic +new forest +ka den +fu en +fix e +far ber +ba ig +an jan +zu kic +uni watch +under cut +sul tana +retri evers +ramad an +pi xiv +ob b +jun ctions +hall oran +endor ser +decl an +cu c +cardi ac +be ath +ba al +assi dy +as pires +adver sely +wee kes +un l +training camp +thene w +ter med +rec itation +pu cker +ome gle +ki drau +ho cus +gru ff +electr ical +doctr ination +cra bby +cosme tology +ce spedes +carni vores +br yo +blit zer +be hr +b nc +am bo +actu ator +ther aces +state bound +star ts +dil ated +burn in +bir b +you saf +social impact +re gaining +n ku +holy week +h mc +crack in +clich é +ato ire +wine and +ufcfight pass +sal va +pen den +maryam rajavi +jay e +hide ki +ar nett +ì¹ ´ +âĿ® âĿ® +xxxx xxxx +tomor ow +to yo +sun mi +pur sues +mini figure +hu ez +clu bb +beingh uman +aqu ila +ab ah +" -@ +y tfc +triple t +sun belt +stan dee +shaw na +rece ding +p sh +master minds +llewell yn +im post +geograph ically +youn ger +wer k +perplex ed +ju z +ima d +hep tathlon +hay man +eye sore +du hamel +brock lesnar +ॠ¤ +o scopy +mor in +lour d +hil lel +flo re +ero berts +doge coin +board game +am ond +ðŁĺį ðŁĺ© +scru m +ly ke +ju illi +ji yeon +griss om +fou ling +fo addo +fidel is +ero ded +en so +e wr +down sizing +aie sec +twi x +the final +pla sti +nit o +national selfieday +malay sians +li re +lepi dop +ess o +dab bing +complic ity +bre merton +ÙĪر Ø© +zay ns +wau sau +stim es +slou ch +pis gah +pf ft +n ans +medalli sts +kar ol +en sburg +da aa +arch daily +alpha retta +upheav al +pp hoto +pan ig +fla ir +eras er +emper or +bla dder +bir t +! ", +âĹ¾ ï¸ı +un met +reu ter +ran cid +pert wee +lin ds +kansas speedway +jodha akbar +jersey shore +heral ded +deple tion +cu taway +complic ation +cal trans +ant ar +un see +u tic +tw ar +sp en +ram say +nav an +j bs +hu mm +herman os +forever home +com prise +. ðŁijĮ +ãģ ¤ +win tour +ren tal +real paige +pin ching +pharao hs +or den +onthe go +newyears resolution +mend ler +k da +is ler +inde pendi +im iss +i eee +han ky +da emon +& # +ðŁĻĪ ðŁĻĪ +u gg +primor dial +nam mshow +mamac ita +ku i +k gf +emer ick +d ach +cm shehbaz +ðŁįĬ ðŁįĬ +vis ite +tun atuesday +shindeshil pas +one direction +his ar +hazel wood +haun tingly +h ley +gif fords +fo gg +ed le +bram hall +blac kest +anxi eties +yer ba +tem pran +sound ly +imperi alist +go les +ghaz al +disp el +coffe elover +vy bz +small pox +predictive analytics +ou tofthe +no kid +mr robot +gi udice +euph oric +ele gy +da sher +czecho slovakia +censor ing +burn snight +vic eroy +ukgif thour +ts am +snow mass +sa ar +robb in +rho d +mce voy +mc cauley +joy ann +i bar +for two +f cn +charlat ans +c ga +at onement +af oo +ad ine +ðŁĸ ¼ +ðŁĮ ł +video production +v kenya +v fl +ue ber +neutr als +nay ak +lili ana +la se +herb aceous +he en +chau vin +bit sy +ann ap +w aged +ven kat +tail ors +t shirt +spor ted +royal caribbean +pe tunia +myr na +in visi +af o +ab dsc +world wildlifeday +ti gress +th unders +h na +gon or +class y +aval dez +pois ons +par abolic +nba a +n ado +mcnab b +loc sin +hallucin o +gri gor +enter a +din ero +d icky +vis itt +ul ver +sk al +short wave +rae us +pleas anton +ou za +kre m +ith i +ipad pro +hi sham +fill ings +wh yy +vio gnier +u led +thir l +scu tt +m ck +kab a +k one +i shaq +home front +eclip ses +deau ville +caith ness +ber son +aviation photography +ðŁĸ¤ðŁĸ¤ ðŁĸ¤ +varad kar +shan kly +rev ell +re kindle +li sac +lazar o +g si +david duchovny +climate change +circul ate +can twell +can field +bir ra +Ùħ ÙĨ +john legere +j att +henri que +fro mb +concer ted +cic i +aller gens +za id +w ena +virgin trains +sp ina +oro zco +oak dale +lec tor +ir r +inquis itor +ine ff +how th +hof mann +gum mies +gilli ana +gel dof +final ising +christ church +Ù ¹ +y id +wal ken +tro ck +tr ong +tar d +sasha alexander +me che +latest news +gay atri +f mf +bi ffle +su ha +shore birds +ol do +lackaw anna +l grw +i ic +gw adar +cra ve +con ga +b pl +ar ao +æľ ¨ +vend ra +silve stre +row land +m power +ja vi +ec lair +compart ments +aaron carter +wh s +tol o +sy denham +sti pe +skarsg ard +ric kie +ric ans +pay al +outrage ously +inciner ator +iam joshuagarcia +hin dered +herr mann +gi ga +daw ns +cf da +bla zed +bene detto +ba sha +red heads +ko da +i ano +gh hh +fre ep +cu tty +acknowle dg +abre w +ðŁ¤¤ ðŁ¤¤ +âļłï¸ı âļłï¸ı +p ä +oc cit +l gg +l bd +kuwa iti +gb v +ap roud +******** ******** +ug lier +to know +swin burne +surg ically +myo ttawa +ke tamine +c mn +an amor +aci vil +ìĨĮëħ Ģ +trump ed +testic ular +spen der +singh a +ror schach +pad am +men i +le vinson +kron os +ko ta +hi stam +harb ha +ear ner +do dd +capital one +ap ne +ab es +wc s +ven ation +the dark +sp iller +richar dar +plac enta +me tv +lov ren +itt ing +in du +down syndrome +des mond +bukol asar +agn st +. âĢ¢ +uk ti +spe t +se ton +requ ited +olive t +hobb it +car is +bas ra +ban yana +ak ing +án chez +ya el +y eni +un faithful +tahirul qadri +stetho scope +santa fe +rot ations +nar whal +n berg +muse u +mc caw +ja official +infuri ating +hispanic heritagemonth +gir dle +daw ning +con anobrien +bri sto +black coffee +ðŁĽ į +ten sile +ss un +she edy +ili stic +gi on +c mb +bhattachar ya +b æ +am boy +ton do +tom oz +sel ba +per gola +okeecho bee +hou wx +he ya +ground sman +griev ous +co tab +arma an +app g +stop rush +se ach +russi agate +me gyn +in doctrination +he ee +handi work +gas sing +fred rick +enab lers +diabol ical +chicago history +bukolasar aki +uro pa +tun ney +steel er +on der +ly sm +ger wen +en tails +as says +ಠķ +Ùħ ÛĮ +vampi rediaries +se ig +ran ches +pa vements +over sees +mu ffler +medi ev +human sof +go yal +ear nest +decep tively +back packers +at lan +ww p +turn ing +st elling +sel atan +s you +pur y +masquer ading +luxu ries +har ington +gil d +findyou re +eve sham +esc row +ery kah +endeav ours +comfor ted +chant elle +ari ver +am ey +wit ted +stal ingrad +spill way +sli mbridge +shel ved +real isation +meer ut +green lantern +grat z +flat tery +ef ly +cy pru +count ing +bookmy show +at ini +al bi +âļ¾ï¸ı âļ¾ï¸ı +vi render +vari ety +sun da +quot ations +public speaking +office space +mi ffy +mad house +eun hae +el td +dro ss +buck ler +⼠µ +vi bes +up grade +u ga +rag time +par ro +knu st +ke ven +insul ating +ici on +gar ba +el ph +del t +ax s +vis conti +r tx +pos al +my heroacademia +mm in +groundhog day +cuer vo +childhood memories +cag ney +bear grylls +b awa +whatvegan seat +un wto +un officially +thibau t +the struggle +raj on +pho spho +jesuss aves +gur ud +fur tado +fligh tradar +es mol +el eni +bletch ley +an otte +ad ol +ãĥķãĤ¡ãĤ¤ãĥ³ãĥĢãĥ¼ è¶ĬãģĹãģ® +usp to +ten ured +por ch +oo ze +ne go +nbc newyork +mal ty +logger head +av il +âĸ ij +tw g +theod or +quesad illas +pit ted +mountain top +ml s +megap lane +ma sc +just listed +jo ana +geo science +fos goodwood +del o +conve yed +b ö +as ado +vibr ancy +sey fried +scott s +pam pl +nation wide +ma shaallah +log i +ko st +go z +gb k +foot note +dau m +bam bino +arcel or +adel ine +y j +u ppers +transc endent +thi stle +stran s +ri mmer +moon lighting +lie ch +inexplic ably +in motion +head quarter +guadel oupe +el um +dordog ne +diamond back +c td +ðŁijĬ ðŁı½ +with friends +spot light +splat form +so beys +shin obi +scu ffle +re sorting +peri pher +kak ar +is sy +hel ga +cmom aharashtra +bo logy +app legate +w os +sway ing +ma uk +hun ny +grand rapids +gee ky +fore caster +e ate +ys r +to si +she af +sec ts +repatri ation +real es +piyu sh +pi as +mexico gp +incon gru +dee side +custom ise +chi kan +ÙĤ رÙĪب +wil frid +wi sher +topi ary +south yorkshire +s ge +real tree +nd is +marilyn manson +m of +kas parov +geof frey +for ges +aw anda +west midlands +ti ppy +sit ar +sho al +shi zu +room ing +puzz le +mur illo +jes sup +hur tful +by un +bront ë +b wa +work manship +w nep +um ami +the apprentice +smoo ch +se hun +se ff +reve rent +pleas ance +mm k +her mine +al art +thevoice uk +psychop aths +or gul +mor peth +inger soll +hydro xy +hiphop tamizha +ge dy +cha pe +at rol +ย ย +wine day +wa o +u news +tiger shroff +som nath +ramm stein +pre neurs +peter pan +pan jang +national championship +man gal +for gery +dru ids +ðŁįĤ ðŁįģ +rl wc +pin ner +nan ce +guar dra +bridg water +bor de +black star +atta ining +ag ates +ðŁį ¨ +yau pdates +tru dy +tortoise shell +sch ell +meg aphone +joey logano +jin ki +induc es +go oners +gigab yte +gack t +escal ates +classic movies +wet land +tumb les +istan bul +i ic +fran tically +fore runner +esh war +childre nof +ba aghi +al ente +âĢ¦ : +shab bat +ou ise +ic j +gi i +fav ela +cran ked +co iled +bul king +ap am +af ood +sir f +se to +pop culture +perpetu ally +li dia +la pp +kai ros +deaf ening +cr ate +cool more +conve ys +ðŁijĮ ðŁı¾ +vic hy +sm acks +letsgo pens +hu mong +her m +emma us +e and +cali per +ah hhhhhh +\( ^ +~ !! +yar ch +univer so +ste me +sp ader +pe pit +pe ct +night sky +gla dwell +florida keys +curb side +con joined +c ville +bb z +visit devon +sky net +sas so +sand ford +plu mes +impost or +homes forsale +hair dontcare +disru pts +cin elli +suppre ssing +speak up +seren di +phil bin +pdp nig +non leagu +ne ville +mtp spride +mar ci +len non +ld p +kam an +front line +ever ly +do cker +co vet +clo ser +ìĬ¤íĬ¸ë łĪìĿ´ +æ Ħ +win it +welove history +webcam s +w mp +ur bang +sw ood +stro wman +port ad +ot ts +obse ss +mic ra +ir repar +helen clar +campaign for +beck in +bar ros +al barn +ad vi +var vatos +ske eter +sit c +is z +hygien ist +en on +cro ke +crich ton +car nou +ag round +âĪ ĩ +wh izz +welove you +stour bridge +sentin els +regal o +official pdpnig +myn tra +gonawaz go +bu sker +bio economy +angle sey +.. & ++ / +z vezda +union strong +te man +sav o +polit as +pe tron +orig i +op ro +mit ty +hen nig +g aku +foren sic +far c +and an +tb n +speci ale +sm ite +siri ano +press ley +ox lade +kingscolleg elon +kau shik +holi est +fu sel +fre di +dalmati ans +bc b +bat woman +- ____ +vegas born +un supervised +ste le +m ft +k md +ho shi +go vols +four nette +chaun cey +annivers ary +srilan kan +mil ad +lac ombe +jake miller +gall a +ff r +ec to +cu bby +be cher +ri gg +kra ft +hin dr +ha tha +fundament alist +erat ops +dun yaupdates +cor niche +congre s +bumble bees +block party +absur dly +wh et +ve ta +un bridled +ulls water +tri star +str inging +mickeym ouse +ic ar +hope lessly +gu zzi +fire proof +dul ce +d co +common core +cis se +bol ted +a bet +zebra fish +t ze +t pm +swi ped +sean flanery +san ford +pas ay +network marketing +mitch el +mike shinoda +mediacell ppp +mach en +krispy kreme +key ston +k hon +humm els +grow thegame +camino desantiago +za z +peace maker +pay pig +le yofficial +iu cn +gri l +diony sus +dinner time +de ming +dc fcofficial +bi ba +amity ville +ðŁĺ¤ ðŁĺ¤ +ë° Ķ +watch tower +va v +tel eno +sto the +shi pping +rhe in +p ft +ma aa +m sci +itsal labout +h bc +fri sco +east lake +dark siders +croati a +char n +brah min +bol ing +sie h +sh allo +s ram +ney land +nature lover +mh saa +martinsville swy +m pi +ling ua +li za +inst one +ic up +hu x +emerging markets +bur d +amo tors +! ðŁijĮ +ðŁį ² +âļ½ï¸ı ðŁıĨ +âĺº âĻ¥ +tc s +taek ook +sh ina +q n +ple te +ni mb +ml k +mil by +li ma +kel ce +j ingles +groovy bruce +for aged +ct g +conspir ators +wind breaker +war lords +tol uca +lan o +joyann reid +il uv +hoku sai +getty museum +extermin ation +cro f +ball ina +ðŁķ ij +à¸Ńภ¢ +ofc trendsetter +mu star +illi st +christi esinc +cancer awareness +éŁ ³ +ภĤ +yuk on +w fu +vintage style +tel kom +pho tol +mi stresses +ll cool +ku an +i fly +espino za +duff erin +dono van +chil terns +cal train +bio grapher +ber nese +belie ber +aw ami +ann als +alber tans +zing is +wv wx +rat ch +mu ms +ku lit +kav a +inter view +i hl +emra an +d li +c ll +broad church +bl ks +aw wwwww +annot ations +vent i +ru bles +pl ondon +ol lo +mor avian +inter cess +intensi fying +gra vely +game week +flint stone +dis missing +dash boards +cr outons +belt line +ðŁĴªðŁı» ðŁĴªðŁı» +ðŁ§ Ģ +will ys +twitch affiliate +st nt +re mun +out weigh +noo ks +min dia +i led +home screen +furnitu re +be om +azerbai jani +alley way +? ] +ðŁĵ ķ +ta chy +shuttle worth +quit os +peri winkle +miti gated +lu mix +itv news +i ep +gaz pacho +gay i +en trusted +em me +cc n +caf f +bombar ded +bo chum +tul sa +p gr +mic ror +kur z +ku da +jacob i +insur rection +el ko +east london +ak ure +women artists +spin ning +park dale +n gi +lud wig +amar k +w pri +rashi da +pod caster +o ster +ku chen +it es +gh sa +follow us +defe cted +deci mated +comple anno +c qc +beach wear +aw ash +po ff +mitch am +mc clo +in cit +hol ic +ev ander +dy no +dur an +do co +co efficient +calgary stampede +big south +animal kingdom +æ ± +wy ch +te les +sud hir +stu ar +gw ang +g lynne +eu in +dav ila +bir i +agu as +ðŁİĪ ðŁİĪ +z adeh +to st +techno logists +sy th +suzi day +sk l +rein hard +mis soni +mat ely +hell er +fa jitas +eu ch +er go +cm j +bo hs +vape on +trace ability +tom holland +stac ie +sapp hires +sam phire +re ge +prin temps +nic hol +multiv it +mari el +lo tions +li ddle +it stime +indi as +gi rish +foreign policy +far ina +em bry +disc ord +aj style +tw ye +set te +physical therapy +je ze +fu ele +fat bike +ex hor +cap ra +b chs +re dox +picture books +me gaz +me gar +jo sep +gre ent +cre ating +ce v +b fi +adebay or +âĢ » +uw m +te g +t ni +ro es +res tha +re fra +pc w +oc n +mole cular +mi it +kirkle es +int x +gyp sies +gu da +gr anger +ger on +fle ur +fin kel +fe ely +evi an +end ran +cat lady +biar ritz +ban ky +youtu ber +yanew ade +tic ia +supre mes +ross lyn +pi pit +passi v +nc state +mr chuckd +mayor ship +mati sm +euchar istic +dw yanewade +d sc +cur mu +boy z +am ours +ac ms +yu rion +tricol our +sa al +ren aud +ramesh laus +photoo ftheweek +ol vi +occlu sion +moul ded +kour tney +illumin ations +ali garh +alas dair +ye yes +turn tables +sil ence +sharpen ed +re sp +patric io +pal omar +kre we +internation ale +emb ert +desmo ines +ar ap +amotor sport +abergaven ny +ðŁİ ¡ +townsh end +over doses +mile split +middle bury +less on +i pa +hat ter +george galloway +fla vin +disappo intments +conchit aw +cantab ria +blur ring +be ec +ast m +work able +ty sons +swing ers +in humans +dormit ory +dc ms +daryl dixon +da shi +burn um +ym iller +vill alo +ri bery +play room +m ander +jo a +digiti zing +dar ley +cloi sters +brad paisley +up enn +stay strong +re edy +macken zie +k stp +j inn +it g +instrument alist +buff ed +bl acc +abra hams +zu l +thisi scle +rear don +photo cards +per verted +ou ble +kh ati +k ly +is kcon +gha da +fr itz +fa ites +en etwork +east bay +dev con +d bt +country wide +columb a +clo the +alex ab +ิ à¹ī +wy then +vinay ak +sch ick +ru fu +making amur +go pi +ec kel +door mat +ca vie +aw ines +wan ath +vege tarian +van halen +sa wareness +rocket man +pan di +mom mas +lloren te +kavan augh +k ı +ic ro +de pts +conti gu +claudi us +ven u +su ki +scru pul +sap a +min ce +man to +gau lle +ba esy +ati mes +ap alach +an ana +ug x +save alife +nu ri +kre uk +honey dew +ha de +fre q +farra gut +famili arity +doo t +de cou +ba ils +ba ier +az har +and ri +uni form +th ylene +spokes woman +sp ate +shi on +presi dio +on key +ncaadi i +min woo +it yof +gal ette +fur nish +fe de +cedar point +brin dle +bigh it +accu mulator +........ ........ +âĿ¤ï¸ı ðŁĴķ +ver na +tri vikram +sel y +nik os +moff att +make sthe +j x +i spower +hen ce +elis se +da unt +cric info +back less +aj k +ab ner +ðŁĸ ĸ +us w +sei ya +pry dz +p mest +on us +mic honne +kap uso +jets go +jaf ar +hen don +goo domens +em its +egre gious +conchitaw urst +bird guides +ber to +aur i +ajstyle sorg +vo tto +spr out +seri e +re finance +protectour care +omo hoy +ne os +national poetrymonth +min strel +min eo +lighting design +kin caid +foo dis +far b +div as +davidbowie real +d wc +confu ses +ala unch +ðŁĮŁðŁĮŁ ðŁĮŁðŁĮŁðŁĮŁ +ëĿ ¼ +å Ŀ +w aker +ri xton +philli ppe +o au +kan ka +do cher +them out +sm it +seaco ast +scott morrison +ro magna +mc part +life savers +l bw +hu me +er v +ding dong +con us +cha at +ad al +aco yotes +ðŁĺĬ ðŁİī +áµ Ĺ +tour ny +sp ic +secon dly +samu rai +po te +per ovsk +parkrun uk +paralym pic +idri selba +ha pi +gra v +gr rl +fawad khan +dj snake +chil ife +capt cha +vol voo +t ci +sab ri +pul ver +onlined ating +newarri vals +melis sab +j ala +domest ically +devon life +aper ol +amare tto +ad olph +ðŁİ « +ìĿ ¼ +wake boarding +multi plying +moffe tt +mais y +jo z +in hab +germin ation +di dit +cre pu +ad ella +ðŁıĥ ðŁıĥ +wen sley +twel ve +spur red +spoo ks +spear mint +snow mageddon +selec tors +sc ms +par si +new listing +large mouth +kak ashi +cab elas +bry ony +ba as +ash ken +apriv acy +ðŁĺģ # +ðŁij» ðŁij» +war si +ru te +mapp le +foot paths +elan tra +cc cc +bluejacket snhl +ãĤ ± +treati se +stret chy +re pro +o din +it sk +ino y +hal ei +gy e +fronten ac +captiv ate +ax o +amy jackson +a pre +ðŁĺĢ # +zoo ey +w faa +then et +squ awk +schu le +s warming +plan tains +official livepd +kam u +hom et +hel pin +har de +g sb +chas in +tu ts +philosophi es +osc ale +orchi d +oceano graphy +mod ding +l sh +firstre spon +educ ate +demo ted +comp ad +co gs +âĨ ĵ +wolf pack +wen del +w gs +tooth brushes +th ame +ra hat +pet ter +pad ova +mel ly +mar aming +ma sen +ky lee +invest iture +how es +hol st +blockcha intechnology +z app +star wood +san born +sam osa +muh lenberg +mc ve +loop y +ir ty +hyper x +fri el +cr ony +bo bi +best ofthe +ber o +bee feat +ano che +wil w +ta pering +self defense +le vis +ivo ire +iti ba +edinburgh uni +camber ley +boo ster +as soc +alta f +adaf ruit +ðŁĮ ½ +ye ssi +ver ma +san dia +refrac tory +re organization +ph ul +pe ti +me izu +hat ec +gc ses +gam ous +ai za +tw ick +ti ki +star boy +sign or +sh pk +practic ality +om me +mail chimp +ma bry +ju ssi +ju a +ho ke +extru sion +dirt bag +chin os +ìĬ¤íĬ¸ëłĪìĿ´ íĤ¤ì¦Ī +su fficiency +slat tery +saving lives +ri ss +pro bin +new bery +iz ind +it p +hb r +daz n +bio sciences +ber line +ahu ja +w yo +um f +tx ed +otta waf +or land +napole onic +moly neux +main event +lax man +kut ty +ic hung +husq varna +huawe i +ferrari friday +ak syon +. ): +tru sses +scottmorrison mp +repe aling +r dy +pizz agate +phil at +malach ite +l q +ju ss +den se +car path +ðŁĩ ¾ +wr f +who scored +volgo grad +visit greece +vi el +v cc +tagsfor likes +shi fty +rotor ua +opportun e +naq vi +na eem +mo stre +le mu +hill ingdon +cup boards +bon ed +absen tia +ห à¸Ļ +tri pe +te th +schoice awards +ranveer singh +ilford photo +hl mann +deep sea +ber us +b ta +aku mar +âĺ Ħ +t pl +studio life +stimul i +sec pompeo +repur posing +re strain +organ ises +op l +midcentury modern +forza horizon +co cos +bik elife +beckin sale +ar aki +Ñ Į +wany ama +w chs +vi ens +tribul ations +rip ening +pro grama +photo challenge +l era +embry onic +de yes +t th +spread love +snu ff +sand bar +sac a +s ánchez +ri mini +ph ants +juic er +for sberg +eh ren +chocolate day +c to +b anna +aust ell +ar mand +ðŁIJ ī +ze i +ur als +ob f +barnsley fc +a hou +wind ward +su sie +sketch books +regi o +pha l +par an +ign iting +gri a +cb m +bc w +al thea +âĩ ¢ +safe co +rand stad +pra dio +makeup by +howit zer +foo sball +fe eh +design showcase +cla rence +avi an +ani dhi +welcome home +ush l +recomm s +parv ati +n pf +lindseygraham sc +glen non +gain z +domen ica +dis agreements +dar row +c ff +au chi +type script +the alth +ste rer +ss ens +sine ma +re treating +plin th +ic l +hen do +den euve +colour pop +ch t +burger day +bor rows +ano v +ad age +the white +str ano +realjames woods +onlyin mn +modernis ation +mast ectomy +late ch +khu malo +horizonzero dawn +g ak +fu ja +flo cked +fier cest +ex pat +etsy chaching +coon hound +cal endu +ban ting +ðŁİ £ +treason ous +ten ancy +statist ician +occup ational +nun ca +lau da +gram ercy +flu ttering +ble vins +acidi fication +ðŁĺĤðŁĺĤ ðŁĺŃ +west mid +w inging +uk parliament +sc tr +mach akos +go jetsgo +early years +eagle pride +da fuq +cot te +cospla yed +carol la +cap y +bo leh +aa rena +wi dens +thor ragnarok +pa jh +n ellis +g tf +ìŬ ìŀIJ +ro ddick +lev ant +ky rie +emb er +c gs +amae chi +wh cd +soo ty +sni ffer +sl c +seem ing +madri d +it ating +is lav +girls with +electr icians +do ings +dal en +d gy +cu ss +con fer +busc emi +brat wurst +an sar +abstractexpre ssionism +a india +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ul li +tele vision +sol on +pri ming +persi b +pap ryka +ol ac +o bie +n ade +multic olored +im mune +hu x +holy head +hen k +fel led +diam ante +cla udine +char l +but z +( âĢ¦ +yurion ice +vel as +uk r +the beast +sy ah +st ourism +so co +snohom ish +sin o +siddhar tha +rho dium +ren n +red shift +na ise +mutil ated +khale esi +ka abil +j aded +hen chman +ga st +discern ment +court ne +big green +bi plane +be ano +anti po +tu lisa +sion ary +re written +oy u +ll oris +kra sinski +jan ice +franc s +familiesbelong together +cricket aus +co si +bu sey +ade m +tal o +show girls +sen ess +pad hyay +op el +moss berg +man zo +interior decor +f les +day y +cu e +ber ia +ठ§ +tou chy +tennes sean +mac lachlan +fa inting +den omination +ci encia +ca poe +all caps +al ade +è Ī +ठ· +the tford +so bel +radio thon +postp ones +ine urope +harve sts +edin burg +dexter ity +dar rin +bho j +til ley +th ore +roman cing +realpaige wwe +r atti +proble ms +nor se +kre wella +ja ket +ir cus +half penny +chit o +bun ton +bu u +boun ty +y rs +ste u +ro ald +resto cking +qld votes +ou i +nay arivera +labou rer +is ak +i bo +coc ina +aer on +ad show +sur rendering +len nox +hindu stani +gop taxscam +car itas +byr ne +bo gle +always learning +æ¥ ½ +titan up +starmagic ball +sna res +multnom ah +lap u +interce ssion +her barium +greg son +furi ously +fly wheel +eck ler +del tag +d aco +break outs +alaska air +îIJ ĺ +âļ¡ âļ¡ +zam boni +suit cases +stri pe +roto world +no bility +inf ill +dynam ically +ch é +band aid +angel i +a org +sheh zad +s office +pu pa +ph or +pajh wok +pag er +natu red +mis used +iam ejf +hast ing +e chop +dever akonda +daniel ricciardo +cr one +coet zee +ar ms +ar as +ðŁĺİ ðŁijĮ +ë¦ ° +æĸ ¹ +ั à¹Ī +wonder fu +why not +ta re +su mit +slo an +sa shes +perin atal +man ek +guer lain +gar man +egyp tology +do ghouse +appar ition +ac cre +uten a +umm er +skim ming +ri re +nf pa +ne tapp +mi sta +mang aniello +don nell +cit adel +axl rose +ad obo +acrobat ics +aci os +ðŁĶ ¬ +uk ong +tro yer +thi one +t ase +s fashion +par ikh +mono graph +ja ish +hor ford +h qs +foot sc +col ini +ðŁ§ ĺ +ure a +stol tenberg +ma an +index ed +hydro ly +hin kley +adel rio +ab solution +Ø « +til burg +sa ppy +replic ated +ree bok +pat erno +nand am +i pe +home base +coun tr +cbee bies +boot le +uof sc +seduc ed +scan lon +priorit ise +pc n +or ks +occupy hk +nan ay +mil o +mao ist +genoci dal +ex tractor +con lan +bro k +av ir +app an +war ra +stri x +sedi mentary +real ty +ps w +paris jetaime +m can +li um +li aquat +jo shi +inquirer dotnet +ha gen +este fan +coinci dental +bett is +ðŁħ ° +winter ishere +re unions +nu ss +mix tures +mir amichi +la pi +ko pe +kir chner +kel i +emur phy +de tti +d q +confe ssion +ç ģ +âĿ¤ï¸ı ðŁĻı +w de +u stream +swamp y +sub missive +sleep wear +out back +on ond +in corporation +fan friday +desch utes +cush man +cour tenay +blon don +ÙģÙĦسط ÙĬÙĨ +st george +sp ag +sic le +sh ins +resul t +ka ik +j ms +get cha +fo lie +depo k +cri p +bit stamp +azz am +an bar +ãĢ ° +u pm +ph thal +over tones +game show +es r +duf fie +cas sad +si ssi +scru bbed +ron n +ren n +path os +mur mur +kul deep +gril led +gal o +ez ra +ext end +dav an +cl f +beha r +avant garde +am far +ach ill +volvo car +visit florida +sea ford +prote stor +pl dt +gir ling +fc d +bi zzy +ash benzo +aquap onics +wh alley +vet ted +te ment +tal ks +super drug +sun coast +sav elives +sam y +pro duk +lec tronics +je suit +ii hf +gan sett +dat aprivacy +circu mc +c ch +be o +bay ley +ak ot +twi sty +ton ians +skir ting +s golf +phil pott +k mh +in world +go or +du s +devon port +d ago +community engagement +ac ta +w wed +scot parl +sc ut +s bt +richardar mitage +o den +nol l +jan mash +hy an +green houses +goo p +excruci ating +dur i +das ilva +d ich +c pr +algebra ic +war am +th ins +sho ddy +sad der +plum lee +pay wall +od our +mp ton +kic au +kan sen +jubi lee +exce ssively +euro fighter +dri ps +diage o +daf t +cat alan +cam girl +ðĿ ļ +ra kim +ot ay +mob ster +leg lass +kp is +en el +crush wednesday +chakra var +ar ranger +xtre me +sali ent +kar lo +her manus +chlor ophy +bang sie +un affected +plein air +pink ish +p yl +n cle +head lamp +hay worth +fun a +fore urope +floun der +cork gaa +compress ors +an anya +yoo chun +u shistory +strike force +ste ine +social security +re makes +pim ping +nau ght +moh sin +mem bered +le yes +ihs net +d ner +clu mp +âĸ ½ +z sl +si red +krp kab +capital reports +bu ries +angry birds +twin ned +roy er +pu reb +pand ering +ne gi +men tees +joh ny +gru bby +b ine +vie wings +vel asco +tec no +sk is +pun ny +primary care +po key +par lay +llangol len +ju ghead +graphic designer +fashion friday +endocrino logy +b way +aldubx dtby +thought fully +soft ening +my ard +judge me +h unch +g ss +g sm +carmarthen shire +vol k +sp atula +sin dh +shan mu +kn are +ip d +har penden +h up +fab ul +encom passing +ann amari +with held +vin t +to ty +si den +ren se +regi me +pv sindhu +middle school +mel ine +mau er +lu thier +kk ar +justi fying +int an +ill iteracy +her ty +gil len +geek and +electro de +dry ness +can ons +bar ca +arab ella +up for +tre mble +thematte spinosa +social enterprise +scoo p +re to +ran es +pu tts +projectrun way +pp h +lar ue +jar re +homo gene +geo logic +faj ita +don do +den nison +demetri us +beth nal +a hahahah +t fclive +s iti +roe buck +pen ni +pal tz +ol af +kat zen +home office +expan sions +eat aly +can te +boyband ph +ap titude +ambu latory +wa pp +t mom +saddle back +re focus +ngo zi +mb ach +mah mud +loaf er +iam dj +ga etz +d wn +clash royale +be resford +an keny +agi ikay +à¸Ħภ£ +wack en +ther ide +sher win +rule of +ren u +pre pare +naf isa +mix on +ke sari +girls rock +classi fying +bri e +boo ker +blo x +star gate +sor rell +sop e +sam hain +rocke f +nozom i +likel y +inter locking +grace ffa +gq magazine +du bbo +die ppe +coinde sk +broad stairs +ano a +anaco stia +al au +temper a +sque al +selfle ssly +rivier amaya +rhe um +molon labe +lin i +ko st +incu rable +fe ige +ca water +ber is +barre ra +wol l +ta reen +ome thing +humong ous +ha ut +g wa +eu k +e pa +confidenti ality +be az +ðŁĺį ðŁIJ¶ +ðŁIJ ® +x lp +sm dh +ju pp +hierarch ical +f sr +ev a +eng g +disinfe ct +dis arm +bot ton +bab ble +ðŁİ º +word less +virtual photography +twin kling +tennes se +sch wal +sam p +rece ssed +ou b +lec tion +j id +ha up +co ho +carlosp ena +swachhb harat +stip end +salesp erson +rb g +prow se +occup ant +medi ate +fix ers +cu ore +ar fa +ðŁĻı ðŁĻĮ +à ª +tori kelly +si fication +nu e +neuro scientist +live wire +lin dale +lan vin +jeremi h +herkim er +cutt ing +cho k +cab ral +bil ic +big timer +âĻ¥ .âĻ¥ +volvoo cean +sten o +neha kakkar +hoo da +for o +disney springs +dam per +chap lain +cal zone +brad man +alyss avaldez +whir ling +wh ack +wall flower +wake board +un turned +stab ler +sp dc +par am +pad man +odd ities +l cl +imti az +g mr +eugen io +di feren +c ella +aeri als +young life +see in +pran ab +laksh mi +kent uc +john mayer +free download +fod map +top golf +tomoda chilife +to ole +siste ma +n ny +mcgin nis +kru mah +ishi i +hyung won +geri atrics +f fa +cl erics +aqu a +worklife balance +w bir +the saurus +sibl ingday +ru mbles +ridd ler +pro let +ow t +mc q +ma dar +lo isa +gue te +ds max +dir tiest +di stru +coni fer +cen tering +prote ge +nat alya +kur di +ho ka +gan pati +feeh ily +cad ero +ag ric +⬠Ĩ +wor sens +war rick +the k +south end +s yed +plo rer +p ili +n é +im ura +hen sel +fili buster +city walk +brexit shambles +ba ked +amic us +wild land +wag en +thad deus +sh atters +lu po +kar u +ig al +hutch ins +har ie +bra wl +bay liss +baske tb +ðŁĺ¥ ðŁĺ¥ +westin ghouse +un knowns +timo th +pi sts +man gos +mal di +kasauti izind +ite ja +interior designer +hor ry +dro mo +ch all +cavie zel +[âĢ¦ ] +viro logy +spice jet +software development +se uss +school girls +mac an +l bp +koval ev +hitch hiker +fi brill +face app +dra sh +circum ference +annot ation +wow za +super annu +snow mob +si sco +pel opon +lor ries +gar neau +chicag omed +c mx +brasil ia +an ok +al fa +( ^ +thro p +shr ink +man si +l las +kitchen design +is lington +install ments +gab a +dele tion +twee ts +ran jit +post gre +ny j +monte go +lipo suction +kut z +ha que +gir lon +ba illy +arre tt +agre eable +ðŁIJ ½ +wrist let +vadi m +tat ar +sibir sk +l boro +hoo ah +ho wrah +ey o +bi ere +b do +al ev +vo ight +mete ors +lymph atic +live well +li shes +kr ill +i barra +greet ing +en ric +eldo ret +bren den +angel ico +afore mentioned +èģ´ ãģĦ +we ald +un isa +sta dia +o cha +n iner +mait re +i ki +cur bing +chri scor +bo gnor +bird ing +baton rouge +allo a +t ka +shoo p +reser ve +par tie +opportuni stic +ma scher +ka di +ep il +conce des +b hn +wo ks +t lr +t dd +shul tz +ne vin +lou ie +hyde park +hero d +az am +al q +â¬ĩ â¬ĩ +web developer +uc sf +te dge +suppor tyour +rastaf ari +lin ic +is ai +ic ri +hege mony +beow ulf +artic ulation +ap ir +see k +sap lings +muja hi +mid del +ku do +inter sex +high tech +af am +# - +yearin review +qui ver +post grad +pho today +ni vers +my baby +mi ed +meche len +ky ush +inclu siveness +gn ss +dv t +dastar dly +dal key +clo th +chi evo +awe e +tre llo +ti reland +pupp et +metro s +j hun +horizon te +gom be +gastro pub +gari baldi +fon dness +e ley +do tch +dart moor +cott rell +axi os +asi e +Ø§Ø ± +youtube gaming +wash wizards +t andy +sumer ian +ser b +secrets anta +pedal board +mizor am +li days +draken sberg +dic kies +courte ous +car mack +bor on +af k +êµ ¬ +wilde beest +victor inox +shan nen +seun ghoon +py rex +proje kt +pri x +moccas in +kuz ma +floren tino +bachelore tte +wan k +ve sh +ug ar +s network +my new +mari u +manu ela +k nut +jo ão +indv saus +il ham +hal ford +goode ats +dg r +bun ker +blues kies +amas sa +ðŁı į +wh urst +smar tly +sh rap +seaw all +school boy +or ator +not d +ma ac +live music +in yo +howar du +dor sett +audio logy +ðŁĶ« ðŁĶ« +vote kathryn +vol ga +sta shed +serendi pit +se led +p cd +lam y +ko dal +glo b +cm f +chrysanthe mum +chocol at +black box +spo iler +sof itel +smo ck +pla st +pa this +obstruc tive +krist of +histo logy +h ird +flu i +feather stone +ch aya +box x +toys for +robe son +postcard sto +n gin +merri man +kh oury +exist enti +bo les +be b +bb qs +ìľ ł +twitter support +sw asan +sh ura +raven claw +jp nadda +findyoure pic +d out +cutt lefish +ye shiva +r giii +parliament arian +o gg +modi in +marath a +hoo ping +do gra +d ard +char dy +cal on +ap ati +ak ennedy +we f +soc keye +shane filan +sedent ary +san aya +ri gi +re produced +r ought +orthodon tist +ner dv +en raged +do young +calu met +bor rego +boo gie +aa as +zom ba +world tour +under belly +un ob +tom m +mal tings +hr d +ex ple +cu le +cis d +chi bok +chan son +business news +brune ttes +bluff ton +aqui fer +range finder +makingamur derer +ma go +gran ular +don n +cressi da +cr ans +capac itors +c dr +arizon acoyotes +ve es +ta ko +su pa +slope style +seat belts +pron oun +men thol +knigh thood +key ed +ji ffy +issa quah +ide o +ende ar +chop e +bo red +an at +z any +uss sa +se mple +rider ship +mariecla ire +kra v +drys dale +deb i +congre so +c cu +ðŁ¥ Ĺ +ç¥ ĸ +worm hole +teen agec +standardi zation +perse phone +perfect game +ough s +l pool +hahahaha hahahahaha +gw ali +do dged +cru tch +co ped +clo gging +be ver +band mates +ven to +ti fo +rd x +pavili ons +nip sey +is las +il frac +han sol +grisw old +emanu ele +devo e +bull ring +at ala +ãĥ¼ãĥ ³ +win ecountry +stal warts +not ations +macin tyre +job fair +je sper +in ne +holm firth +hoi sting +geh lot +gagar in +fla red +bou logne +aw o +agas si +afran klin +xox ox +wn l +waw ine +wash u +top model +tain ers +su cha +si aa +ough ta +kil ig +guana ju +go do +gar n +fly air +ff ff +dec i +bri dle +bran ning +blu estone +bl r +ìĺ Ī +synchron icity +spl unk +mel chi +mc tavish +loot crate +icic le +hot wheels +go y +die z +dar vish +ble ep +arab ica +Í Ł +z ka +schi av +my cology +kumar aswamy +edwin a +ati v +^_ ^ +ðŁij ĸ +y ac +wey bridge +ti mor +si ke +roman i +ro ther +quint an +ham pi +fla c +co vent +cham in +c sm +bour get +ðŁĺİ . +wind ell +waf a +stan ak +seab rook +san chez +russi angp +re arrange +pen tium +paki stani +nfl playoffs +mo hit +mari am +mar ne +four four +conesto ga +co ff +bus quets +ar jen +ðŁĴľðŁĴľ ðŁĴľðŁĴľ +ðŁ¤ ³ +virender sehwag +valeri e +semi finalists +lower case +khu sh +in vocation +hc sm +dunlop btcc +bla u +barb ary +auctione er +ac cu +x lix +water spout +w pd +vand enberg +swe en +in soles +del os +cutt ack +car ru +byr ds +black widow +ath in +a ç +yol ks +tan go +sto cke +no is +mk t +miya ke +mc dougall +manish malhotra +fon d +fare ham +by l +and is +an gui +ad as +ðŁĮ ¬ +wil ley +swim dive +shoo ter +se wers +sch efter +os b +mm b +jane oineza +jami es +colli sion +chron ically +bo jan +aro ss +ts l +tories out +sens ical +ol ins +official r +life quotes +karnat aka +hir u +cir cas +amo vie +sports book +sangi ovese +ravin dra +prof iting +pro gen +pois son +ji day +bm wm +this week +synchron ised +sou ff +people of +o campo +norwich cityfc +mt k +mor phic +lor o +k home +identi fiable +ic ula +flint stones +bibli ography +à´ ¤ +vent e +unite the +ter ill +pamph lets +nas aj +md g +l ı +ker rang +k bc +fer ran +cu bans +biz awards +un winding +swe g +ru mmy +resur faced +ocon ee +nat as +jo iner +i oc +gra ys +chop on +carol ing +be p +terrori zing +slack hq +sch mal +ra du +ponte fract +pi aget +p yn +o gp +o edi +el ven +digital signage +an ight +a arts +$ ... +world rugby +wb ko +ti verton +th ati +tar tu +sk ink +sharinge conomy +sal lie +recipro cal +propon ent +poe tics +o q +novo sibirsk +nb stv +mini stry +me unier +lyn ched +je tair +in fighting +hello bangsie +book list +as apo +ðŁĴĺ ðŁĴĺ +ìł Ħ +wy den +wareness day +ta wat +t ph +sky hawks +personali zed +me za +int ouch +fü r +franca ise +dejav u +de spo +bur ks +astro dome +v mc +uncontrol lable +th ie +spike lee +park city +matri mony +ho pen +h x +brook ing +bre aux +ap ace +alicein wonderland +aj am +ac pa +; ) +âĤ¬ . +ve sta +tri d +offici ate +natu recomms +mient ras +lan gh +im measurable +gif tof +fash ola +candle lit +bal der +baj rangi +agre en +y aar +tee pee +re structure +rc si +mis cre +lu rie +libert adores +li ssa +geordie shore +gent leness +flo gging +bre win +bahu bali +and ere +ad ana +ãģ £ +vil lar +ucl an +tyr one +tro dden +sub zero +scol lection +nu bia +na in +men de +jubil ant +gre gh +freedom day +fin ery +deuter onomy +byu football +brain erd +bour sin +ben ven +belgra via +ar una +app state +trattor ia +polye thylene +nikon photography +marc ella +fn f +film twitter +far si +back hand +è ı +vel t +twin k +sau sage +sar ban +reproduc ción +mo dis +jo ta +cad do +ad don +vis akha +thu mper +sy x +p dr +oil sands +mar oo +m soccer +hen a +glen fidd +ethical fashion +emo ticon +cam embert +be il +ar ro +ab x +vian ney +sweat er +su bar +sh w +raredisease day +meang irls +le win +im planted +hun han +ha kk +fit life +ei gen +ear a +bur de +bloss omed +Ù İ +u con +ty o +sta p +pru dent +p fs +jar man +fire stone +blund ell +ðŁijıðŁı¾ ðŁijıðŁı¾ +nam ed +kei ko +ju b +ide x +hemo philia +everton fc +ede ma +d be +cor gis +ðŁİ » +to que +rambl ings +por zingis +or chy +mi rad +land mines +kom ets +hi ddle +go team +fyo dor +escarp ment +du k +dn f +bro deur +as ki +an ks +a ee +un framed +rich land +ra di +ma qu +leinster rugby +kali mantan +hit ching +economic times +dump ty +craw ls +asado waisi +as oci +as and +to bey +poetry community +official bhafc +mon alisa +jag er +ha x +h ff +flat ware +duc king +di vi +bio chemical +ðŁĴ ij +í Ŀ +su o +sl k +predic ament +nerdv ana +m live +le von +gaither sburg +com ox +by water +ðŁıĨ @ +vaul ting +to ta +thel onious +pre cari +ios dev +hon king +her nan +h ice +enchil ada +en reproducción +da ed +bi ki +bau ble +band it +we c +venge ful +tobler one +tay ler +schar ity +revit alizing +r vs +r rs +love craf +k age +ei bar +dysle xic +cro lla +chit ral +ðŁijijðŁijij ðŁijij +x vii +wil la +tang lewood +ta iga +su football +squ ier +sas sen +per rier +n ld +ko lo +conservation ist +c fe +block busters +an ah +ü ber +sun ba +sty our +smil in +pillow talk +le pas +kru pp +hosp ices +hel ipad +fil i +dro sophila +bo som +yennaiarind haal +uk in +standup comedy +sni ping +sand castle +qu avo +nom bre +n la +man tar +gu bler +gr ano +elo y +d bh +cy r +car pal +bor i +air france +aali zardari +ðŁĩ° ðŁĩª +yak o +un women +sundance fest +small mouth +seash ells +o waisi +mul doon +cuis inart +bo gie +bas soon +an jan +rock o +po ste +pim entel +pe avey +nos fer +kir che +inter pol +haji me +en l +ar ak +ðŁĺ¹ðŁĺ¹ ðŁĺ¹ +еР¹ +Ï Ĩ +woo fers +vo tive +ver dant +u leta +trum pe +ship wrecks +shim my +sc ats +salut ations +s anna +pat ani +nag s +indi gn +gaffi gan +eag an +cr v +bad r +ant and +annu ity +the afric +terrori st +sol ana +rape seed +poo ping +m chs +fast food +emul ation +elev ates +de sean +wel yn +w yo +th birthday +speed boat +pinstri pe +oneof akind +maritz burg +k hai +j nj +gil ani +chri sw +ay our +ap il +a ini +ðŁİ Ĺ +v ln +ther sc +sw en +restor ations +reiter ate +photo call +ob p +ny p +m hp +fil mb +d aps +ðŁIJ Į +z ec +uniof nottingham +tra shing +stra ub +sequ al +ry back +ro thes +mummi fied +millenni um +marsh field +j cs +is art +hugh es +gau cho +defen sible +ce mented +bor land +bon nets +ðŁİĤðŁİĤ ðŁİĤ +wonder wall +wim ps +vivo ipl +tallu lah +taec yeon +sport sawards +sher brooke +q sa +pin ck +ph r +oun ty +nu ala +kung fu +hel sing +dalry mple +ate acher +animal crossing +afc wimbledon +] - +seven teenth +saip an +ku o +ka an +in ta +huss ain +epi thelial +den iso +as kan +wam bach +su ko +son oran +sn ola +pr ong +plu g +nb cs +mt u +logar ith +local es +kelle her +kat ch +flu ff +cr yer +cont ours +con jec +ce real +calendu la +a icc +åij ¨ +tent atively +tempran illo +succu mb +south ward +raj jat +r fl +par ham +ny our +my p +mur ry +ligh thear +in time +gag gle +f lim +city hall +ceme x +brexite ers +bi glo +at ly +ห ล +women insport +un invited +town es +the botanics +sensu ality +sc el +pre occupied +onlin ec +men ai +long term +le ich +land y +ig ong +conservation ists +black light +az aren +architec tural +ðŁijĪ ðŁı» +u et +tu red +stal ybridge +r za +perfectgame usa +pap adop +motion less +mil t +di el +cre at +black birds +bal eno +att itude +about lastnight +ãģ ¯ +respir ation +re defines +pic c +pale stine +now tv +m cuban +lo ka +gman etwork +chi z +angu age +alli ed +alicia keys +w ning +us se +the people +sd t +reson ant +nyc mayor +n bt +hoo pers +don ned +do sed +d illi +centre piece +blog spot +tu so +t mo +md na +land rieu +kann ur +ka rena +in slee +giu lio +alle lu +ak un +thejustice dept +simm ering +ro ly +o ki +nh at +metal work +hou ten +contag ion +aka worldwide +å İ +ãĥķãĤ¡ãĤ¤ãĥ³ãĥĢãĥ¼è¶ĬãģĹãģ® ç§ģãģ®ä¸ĸçķĮ +under tones +sun daes +pi os +on de +o intment +mo bo +kev lar +ket te +ing lori +ic ano +i ag +hay festival +doctor ow +chir ps +bill board +! ðŁijı +âĢ¼ ï¸İ +year n +ven er +ul te +treat yoself +ton ys +som os +s man +oreilly factor +laparo scopic +hah haha +free se +domin ator +chau cer +ch lamy +birdsof prey +armed forces +aero dynamics +ad ors +vol com +vancouver island +the killers +ob fusc +mú sica +lil bthe +lilbthe basedgod +gor akh +fool proof +etsy gifts +cho d +bu e +ac p +ðŁĺ© âĿ¤ï¸ı +war r +vapor max +sr tc +sen ayan +ri man +onond aga +on ference +metro plex +mcgill u +kath ie +kak o +je tting +get out +fuja irah +fertil iser +ex propri +eli ght +don tt +car jacking +bi ri +bal de +y ella +wil ton +wheat grass +vani shes +thel on +sedi ments +pu yol +postcardsto voters +mu to +miss america +ley la +len ovo +justi fies +in co +ear plugs +bur o +blue prints +b school +ver and +ou k +ny giants +jo vo +deter rence +dc cc +con diment +an l +wor cs +v di +tt d +moor land +lun acy +inti mately +idio syn +bod min +belli ge +. ðŁĺİ +work sheets +wil led +ulster rugby +th july +teen age +super janella +sty lings +sh ingly +p spk +ost asis +om sk +n acc +mi ren +ko bi +im ola +fe f +bil le +bb mp +ae ther +! ðŁĴ¥ +tear gas +tapp an +sig ourney +sam ira +pap hos +kat weets +hocken heim +gen ghis +gate keeper +acap ella +âľĮï¸ı âľĮï¸ı +unrival ed +sla ven +russell crowe +py rr +poo ja +ni z +mike tyson +lero i +lan sman +fran sch +end violence +don y +dian ade +bour que +b tv +anci ents +ab dallah +ðŁĵį @ +ðŁĴµ ðŁĴµ +z os +wozni ak +wer ri +sin jar +in visibility +cor si +cen to +ar ine +adebay o +⼠Ī +pur p +n bab +mari ee +ma sta +ly les +l chs +i ak +de gan +creu set +co ppin +blu eri +bag us +ai on +wh ut +urban fantasy +stephen amell +snod grass +sand hurst +pool party +plat form +plan king +p ona +no sleep +mr sa +luci ana +live show +jais almer +it smore +finish line +film maker +fc f +e bol +destru ct +an sele +suppre ssor +spit zer +real berta +pl iny +nr t +nfl pa +lal aland +eric hards +bil tong +and ai +ak ro +war hawks +redund ancy +q ian +pu shups +grou pies +gel lar +com inghome +clan destine +chait anya +an ys +ab aya +tx dot +su ble +r du +migh tiest +mccre e +jurisdic tions +hu dd +hans ard +flo rent +d ce +colla bro +ch oma +bar sand +adi se +v ago +tic keted +th p +st lucia +snow pack +sher borne +ration ing +promote horror +mobil ise +luxury hotel +kand la +k awak +ho se +he dley +dr yers +cre scent +ab or +w sa +su le +sister ly +re bar +ramaz an +long mire +inhal ation +dissol ves +commerci alization +cha ine +carri on +cam erica +boo galoo +big deal +b let +aspir ant +ur gh +tiru pati +sl acrosse +sb ach +poor people +oo gie +ki ambu +jab lon +how ls +b hardwaj +b ds +ant u +a aw +ðŁĶ Ķ +é ľ +van v +plussi ze +link later +lin lithgow +kla ss +finoalla fine +envir ons +bren nan +appeti zing +$$ $$ +$ ! +wa pol +tu fc +ther o +sivak arthi +si va +plastic free +my hero +la gh +fau sto +ev c +cross overs +bn sf +bern thal +au li +ìĿ ĺ +tin sley +ti sch +straigh tener +scotty mccreery +rece p +pun ky +no to +in pics +happy day +criteri um +bikelife dotch +worldcup final +to let +shin kansen +popu late +orche stration +naku foaddo +lindis farne +lat akia +integr ations +ig c +ib g +hopp us +food lover +eng ard +du ds +df b +depau w +bel af +bc n +bar c +ba sie +as sad +af resh +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺðŁĺĺ +ya ay +sar sour +ric ardo +prophe cies +power boat +om r +newh omes +magic thegathering +m dr +lokom otiv +li ii +jä ger +ju egos +iter ations +inclin ation +ig ne +gro gan +fu sco +cran ks +be sos +âģ© ' +tric eratops +spic y +spac ers +scri bbled +reach able +over ground +microsoft teams +m hm +g mt +future is +fell ini +fel ines +fab s +cri ssy +ca il +book worms +bo do +ar rington +ðŁĺı ðŁĺı +y ameen +sa kamoto +re ared +nu ys +market cap +mail lot +inhi bit +filmo graphy +falcon ry +engag em +de faced +car at +buc keye +bay front +bangalore ashram +atp worldtour +am un +ad om +y ate +mediac ityuk +j fl +gun ung +fre s +che on +bagh dadi +bab at +aug ment +ari sta +alk official +ê· ¸ë +wie sel +trin idad +sof summer +orp ington +nose bleed +jay me +foot locker +em pathi +bo bi +anti bes +ansele lg +aerob atic +ðŁİ ĩ +ãĥ¼ãĥ « +âĺĨ âĺĨ +water works +water logged +no bar +n cd +ka huna +is ar +fli rts +d mb +cp us +coo kers +cas co +c fi +band ain +ayo dhya +aj man +surf in +o carina +gu tter +fries land +cast rol +bon plan +be so +à¹Ħภ¡ +ven ter +spr oul +sport back +sp j +parti zan +oc ket +mathur a +m fl +ha poel +gre i +g mf +dru p +cover art +contra dict +ay ub +anselelg ort +abse il +war bird +tro ma +ph ro +nerv ously +kw ch +kun j +j illy +id b +hundred th +hal alan +dece it +ca wl +bon t +tash kent +ph lebo +march forlife +mar red +l cr +krish namur +he bei +fra g +bill ballentine +bha gya +august ana +anastasia beverlyhills +amc cartney +íĻ Ķ +th all +ta thletics +stu es +st anc +re als +ol ino +n tn +jet lag +hi ii +aller gy +wn bl +suit ors +sin bad +scotland team +sal combe +roll back +rey no +point less +pl ou +octo pu +n xp +hy po +happy bday +bou dreau +alla ah +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +world star +the wire +t gc +sun trust +sore ness +sk ap +sci o +re started +low lands +lewis ville +gastron omic +expir ing +defl ation +de tox +cu s +blu est +ti rade +schol l +prof en +pol itic +po sen +par cs +liber tad +la ye +jan ic +her ing +hae morrha +en cino +cal ori +andre j +anatoli a +ðŁĮŁ # +unfor gi +tab i +shad i +opar ks +my world +ho dor +azaren ka +ðŁĵ ľ +vic firth +tre vi +tran shuman +squir tle +speci alization +read e +ra kes +one treehill +mu khtar +mar ymount +kaz u +k wai +ic er +gla ssy +forfe it +felic ity +f els +dual shock +de fra +cer i +as phy +ang ri +ঠ¬ +worldof outlaws +ta or +slic er +na si +mis se +lock the +jen kin +friday sfor +fasten er +dream force +bloom sday +bl ck +biop harma +apple jack +: /// +. ðŁĴĻ +val ero +uri be +un doing +tw w +to tnes +tic les +tech fest +sun nier +stream lining +s gi +offen sively +lund gren +lc sm +inhal ing +enci as +cr on +comment aries +code ine +cly ro +br n +bo wel +tutankham un +spru ce +sky walk +pang olin +mod ine +men ta +lan cet +horizon tally +gu rira +grant gust +gas par +freder icks +curv ature +cam bio +ask is +éŃĶ éģĵ +u cs +tcm ff +sen or +pu dge +mal dini +kirk d +ken berg +kam pung +iri a +cotab ato +bay onne +anesthe si +and ron +ðŁij © +zen o +stock man +sk an +r pc +on canvas +mtvbrkpop exo +ci re +band ages +aurang zeb +a star +ðŁĻĮðŁĻĮ ðŁĻĮðŁĻĮ +ðŁĻĮðŁı¾ ðŁĻĮðŁı¾ +à· Ĭ +yeg wx +w mt +volvoocean race +vari etal +simpli fication +rock climbing +mis matched +h nd +geome trical +dh sgov +danger uss +center ville +ble acher +ar quette +afc bournemouth +âļ Ĵ +ty bee +tre d +thur good +oast al +nax os +mainst ay +kar rue +joaqu in +iff i +ic re +hover board +evic tions +dun more +d ado +both ell +twee zers +se ep +ry lee +power ade +mur phys +lang chat +execution er +camp out +bice ster +æ · +à¸ŃภĻ +zo ey +le ach +guide dogs +fi bro +ef x +bi el +be fitting +ðŁį ı +workhard playhard +sojour n +sav am +rss org +ni i +nay eon +li han +h bu +gab bar +eng lund +e space +citizent vkenya +aw ah +ab bit +-- # +thor ny +squ aring +ra wal +lou der +li sar +lex ington +intro spective +guin ness +daily telegraph +co pland +cen o +vi zio +transplan ted +shi z +ros ari +ml khattar +indie dev +huck nall +fe ct +embr yo +dy l +di go +dg n +am dg +al locations +ðŁ¦ ĩ +ãĥ Ĵ +sym one +season ably +no stra +mee ke +loud speaker +jam bo +bo got +beck ley +scep tical +sa hm +parame tric +oc tor +nn pc +logan henderson +limb urg +lak as +gor ton +drone photography +deb ate +char me +cele stron +can tu +avi on +ab in +éŃĶéģĵ ç¥ĸ +éŃĶéģĵç¥ĸ å¸ +x terra +we x +wal worth +tra di +team coopertire +sin aloa +r anda +pu tih +physio therapist +over falls +or omo +ny islanders +mmi w +la ki +go av +gee ked +fan e +enu ff +dr at +al evel +yu vi +vol pe +stren uous +shaq iri +shaf i +oun a +mu fasa +ju i +happy ness +em pe +eg l +degra sse +comple tions +chili peppers +bi dders +ðŁĸ¤ ðŁĸ¤ +ys gol +y ano +ur at +un attractive +scienti fic +sam pai +re vels +ou ma +nic ation +ly don +in voices +fer v +expe dite +dangeruss wilson +cumb rian +cram ming +caption this +bas spro +an au +ak it +air crafts +vis ita +shi ps +out grown +north cote +kair i +k nut +ini sts +dio go +dc united +cur ler +cellu loid +baller inas +arapa hoe +yn h +up ma +true blue +t pg +ste via +sever ino +rajjat tokas +r ational +pol kad +os in +nasaj pl +mann schaft +manipul ative +kayo de +k ray +hur ry +euph or +educ ates +cu cam +cor don +black hat +b side +ane o +ai admk +adri anna +tie breaker +south side +param singh +koko bop +kaffe e +in croy +icon f +health news +distin c +bigtimer ush +wi aa +tore tto +to wel +thelittle things +tele fon +smor gas +smo ked +positive thinking +on theroad +music production +mar ab +kasautiizind agiikay +hahahaha hahah +fr antz +empty the +e ffin +cu eto +cross ing +ace of +ðŁijħ ðŁĴ¦ +âķ ± +sorcere ss +o ad +ni aller +nh ms +mu gged +metro pole +kau st +hit man +croche ting +crimin ality +⼠½ï¸ı +sme ars +mir ko +leep ace +kirkd borne +kaz oo +kan ina +ham sa +ennis killen +e scri +di shed +cr na +bo iz +best nine +âĿ¤ï¸ı ðŁĺĤ +wise st +vou cher +vac uu +tac ks +south land +ridic uled +rasp utin +port ability +pat ine +mus ket +la ya +gh ese +free form +flu ently +ferre tti +chand on +ch oux +bli p +betra ying +bet tie +arte ta +all night +un ica +toom any +te mer +switch foot +sme ared +s vs +quir ks +prin sloo +northern soul +excit ing +dwar f +do tte +de ven +corro sive +ci b +certi fy +bam burgh +ak op +si bly +sherlock holmes +ro to +re turno +re launched +outlaw ed +nat archives +mon so +lo tt +lo ire +detroit pistons +del ly +coward ice +at ur +vi ki +tion ate +te v +speci fics +se ale +parth samthaan +mist ral +mea india +lent z +lasal le +is g +gu llo +cull en +che tte +billab ong +at at +aspir ational +air jordan +web gl +wa stage +underwhel ming +ufcfight night +tre yarch +su cess +ru islip +op tus +no regrets +mar r +loo e +it sy +handic raft +graph ql +fen cer +ener o +dun hill +dre search +be thel +ari ed +tim peake +scandal abc +re visi +pu ffin +ph um +memor ized +ka han +hale storm +ep al +dublin airport +w ca +vic er +thaw ks +so to +shu ck +pureb red +pere ra +mug ello +l fcc +ki vu +fre hley +e brd +clack amas +che vel +ðŁijį . +woo young +sti hl +show times +jane way +e be +cir c +blast fromthepast +big ben +bel grave +v force +skim med +san ofi +r cn +out number +mont fort +major ly +f bi +cob blers +cave at +asse ur +adel sol +wel o +ur ns +tor ii +mor rho +man nered +lymedi sease +dg t +âľĿ ï¸ı +woo ps +ur ch +tane ja +tal al +snor kelling +sas se +ric one +nu thin +n art +me ck +lan tern +ker ridge +i ram +her re +followyour nola +flexi ble +ep at +e day +dun de +de id +dal v +cul lum +coinci des +bern hardt +be friend +and han +tik tok +ranch life +por phy +ol ito +new southwales +nak a +move mber +len z +kim bo +kids books +ken nington +k ase +fan shawe +ce ballos +capac itive +bro ek +bom a +battle for +ba ap +ðŁĹ£ ðŁĹ£ +ãĤ¸ãĥ ¥ +war field +vas ily +t sc +se mo +prince of +le tic +cack ling +bett ina +b ms +è Ĺ +äº º +{ @ +scor ch +sau v +s disease +rev amping +piyush goyal +pe chak +nac ion +mn statefair +mag al +la fleur +isol ating +i au +gal gadot +cd ne +bill maher +beg ley +ba io +west australia +vul gar +tv b +tar lac +stenc ils +sch lei +normali ze +me anie +jo elo +hereto stay +dy ne +cre at +chartre use +aj mer +storm team +river o +re spe +paner ai +pale mbang +lu ty +lo athe +judg mental +huff le +hr c +hh c +eu il +c illian +br p +all natural +aar p +yo gis +xan adu +uw f +topp led +se ers +ophi les +mo sque +m ame +ju er +jan asen +guanaju ato +employ ment +bryan cranston +berne se +bbcintro ducing +ad eni +Ø ¸ +yan k +wr angling +wind farm +webr tc +tr one +timber line +the cube +team chevy +tac tical +pur p +o steen +ing rown +gilgit baltistan +gau thier +fee ble +der went +bra chy +ami ento +ðŁIJ ³ +vy ch +the boy +sky blues +po acher +nico lette +na az +dit ka +Î ¹ +youn gin +was ps +tu ks +stat ic +makeaw ish +houseof commons +her sh +ecr chat +de ji +ac ru +xi u +vs sea +u vic +tt weets +sthel en +pr ana +oc ado +ob jet +negli gent +ko tor +kar yak +flax seed +daf fy +conve x +aristo crat +whist ler +vas cular +theone show +standre w +south field +screen writers +kan hai +athe i +to you +sam y +sag rada +ring ers +opp enheimer +mono gatari +m wave +j angan +gil gamesh +dai ley +d ancy +boo by +bbc looknorth +sw asth +sound design +sci am +sadh na +pret enders +out en +mis sm +ma guk +ike da +gil lette +el fman +defl ated +col u +buddhi sts +av u +with pride +wc bs +t mb +t ld +sydney swans +swan ted +st acker +pratt pratt +nan oscale +lil acs +ju ul +high street +fren d +fer ru +de ve +con klin +un relenting +trans actional +stro mb +s ach +religious freedom +n tm +n ela +lu i +h iller +flo tation +en cy +disrup tor +ci er +cas per +bul la +bet ti +w tn +ursu line +up silon +thur mond +splat fest +sal o +p gc +mm h +makesthe dreamwork +lean in +ka ji +gro ping +g cc +d ening +col ter +ar al +anni gan +a week +ðŁĻĮ ðŁĺį +x abi +u wc +true social +timb ersfc +rich mon +prattpratt pratt +ny am +lo thian +leot ard +j ma +itu te +ek ay +echin acea +dur acell +ìĹ ° +tro feo +she tra +shang hai +sab i +qu inter +nhl canes +me rer +ly nyrd +lin del +lawof attraction +lam ela +kho sla +has set +finger nails +end angering +dro plet +dies er +cont ac +center pieces +a sharma +ðŁijĩðŁijĩ ðŁijĩðŁijĩ +vero truesocial +segun da +plum met +pan ch +mal le +li sav +hi bit +h ru +g ct +bon amassa +blu th +backto work +aphi ds +ti bility +sc ount +ra pt +place holder +lane way +fo stered +fo red +fi el +emplo i +eme ka +ca k +ante ater +!!!! ! +the ist +tech o +tec mo +sw best +su da +sky hawk +se itz +s academy +pra j +plac ards +pivo ts +mountain biking +jum mah +jj f +ig ata +eu co +de constructing +d ft +al mond +weis z +vi j +to li +south wark +slo tted +ra gin +pro actively +obste trics +north woods +nh of +jeune sse +ae um +tv p +thero ck +sym metric +so afx +seag lass +non league +night crawler +m de +ky uu +kl ance +kab balah +cri sis +chemical physics +anarch ism +å¤ ľ +tr m +smo res +sa xton +re construct +pettic oat +out scored +mini mum +luci atas +luciatas san +loy alists +ligh thouse +lake ville +industri e +ic aew +i ie +ho gging +fro mm +ephe sus +dur rell +blood shot +beech wood +american cancer +ach allenge +v cg +tom ellis +tempor ada +sel la +morri gan +lom ography +li der +googlec loud +ger ie +fe ild +ev os +cine world +bha bhi +amy schumer +afsc me +vic toire +vi a +sub i +na sir +multi ples +lu stig +lat timore +k cb +i din +guy ss +di stressing +ðŁijį ðŁı½ +wil f +tom bola +tigh tened +sl peeps +sig ye +sham rocks +sat z +qu ec +no gales +new ss +natur ale +k ss +k cap +et fo +epic ure +bbc four +barrier reef +ab on +ãĥ Ģ +tw os +ro id +re eve +natu rema +mal ac +m sh +i jo +extermin ate +chou han +cas i +yn or +tele visions +storm doris +spor adic +soli hull +soci alizing +sh amp +pnpp atrol +out fest +it orial +idh lig +how land +ch ur +belgi que +and ran +w mf +tan nehill +ta ye +s thu +ro que +rik ki +ra dium +pat er +pac sun +p cusa +obli ge +ob vi +n sf +mi es +mc busted +lingu ist +li ppy +di ms +ce g +canni bals +candid ly +barre tto +scholast ic +q fs +propri etor +paci fier +offici alu +nott m +mexic ano +mar yann +la hm +grand parent +forz amotorsport +formula oneworld +burn leyofficial +bax ter +apal mer +ab loh +ðŁĸ Ĭ +wh ittle +throwback thursdays +sla yers +ma key +lauramar ano +athan asi +ap el +vo is +vi ves +tur nips +snor e +pl ines +or do +mac rame +ir b +hl n +global dev +fuss ball +evol ve +epit aph +dang les +dalrymple will +carn elian +as cd +ana esthetic +Ê ĸ +un du +shabbat shalom +ridd ick +ol ney +li da +lal un +im possibly +her at +groom smen +gay le +co ffs +capoe ira +can ta +bak eries +vik ki +tu ra +t ne +pl zz +per ky +peace and +ord way +n anc +la vin +doo d +digi byte +com promises +co bbs +at am +vik tor +ser aph +re arranging +pil sen +marque tte +mari ob +fic us +do pey +d ng +cur ries +ce ec +caf cl +wee ee +urugu ayan +ru ffi +pre ppers +h ü +gob ind +ga stown +baham ut +attrac tiveness +ad ra +zar ia +wis p +sang ster +ribb le +mo ises +martin luther +leagu er +le one +kat v +inlove with +encamp ment +d ct +ba di +âĥ£ : +senior night +rosel and +rand al +pampl ona +link ages +inspec tor +ha bibi +equ is +dam ing +cat chin +been ie +ba haha +al cu +ac ar +èªķ çĶŁ +war ri +tom morrow +ti oga +te sla +sh rooms +orb ital +mulvan ey +mu gging +ku i +distingui shes +abnormal ities +a verse +wb w +vit us +trac ie +the end +t week +speed master +sag ging +re tainer +panch oli +n po +ing ame +in sk +har apan +dif fraction +custom izing +buckle up +are search +tweet whatyoueat +shi pla +pon ting +or us +north america +lucer o +lam i +kit z +green y +de composition +dabang g +belo ve +asper ger +ap ai +antidepress ants +ac tory +) ". +yor ku +yo h +te res +si ft +red bird +movie awards +li mon +dispat cher +compet ition +à´ ¨ +tin dall +skele tor +qv cuk +pnppatrol plan +licen sure +letter kenny +leaf leting +grate fully +gorge ousness +er ste +b fd +ave tt +aloy sius +ow d +ol ine +nom akeup +n tas +man ch +jer oen +had don +gri ggs +golden retriever +fact check +digit ised +dd h +bella donna +ðŁĺģ ðŁĺį +w sd +the z +prith vi +ou en +or ford +mush taq +ma b +ger a +frank ston +fab led +f rick +deleg ations +æ © +xti andela +per fil +ong we +mp v +jammu and +il op +geekand sundry +fi dge +feder ated +da we +cli f +black veil +tu scar +span ky +or ob +moline ux +mar ano +ma pa +hol tz +fret board +ec ac +dup atta +biscu it +bij oux +am illo ++ : +volunteer week +vac ate +v fd +self portrait +north dakota +mull ingar +make overs +he ke +h ct +ever a +deliber ations +chickas aw +bo bbing +big daddy +aro ck +ৠģ +tb ar +sanc tity +ny cha +mgm grand +jones y +jan go +fri st +di fun +chouhan shivraj +ad agio +âĺĢï¸ı # +y bor +upl b +ti fa +s fans +ri ven +pol yam +ol am +gn am +fre dd +dog toworkday +cr an +cin que +bee keepers +be ÅŁ +at au +ar la +an ah +y ura +te rence +te ck +su ge +re insurance +play store +l ile +ker ns +hy the +h tx +gn ani +centenni al +bu ter +ash ville +agre at +ach u +a see +ಠ¸ +sus ang +super dry +sp rime +sc ity +re aping +out sourced +obstru ct +green room +fe heroes +fa in +cla pped +cab in +be inn +av ai +ðŁijī # +vector stock +teamwork makesthedreamwork +sha urya +le ch +kristen stewart +in between +gin ny +fy c +fre er +fr inged +em itted +di ba +cross bones +cow ichan +conve ying +bolshe vik +band i +alexab liss +ador o +ðŁĺį ðŁİī +wan amaker +ve ena +sr v +nit rous +mor aes +loving life +kay ak +iq rtg +hil d +competiti vely +cleveland clinic +cit ron +ar aya +ëĤ ĺ +uc sc +recor deli +puli sic +pis cat +mow ry +ma est +ke p +is ko +fal lujah +dun a +cor byn +zeit geist +wedding planner +spor ttv +schem atic +ram ya +ra ji +napo leon +muen ster +m se +le bron +judas priest +imper ium +did nt +brecon beacons +bobb in +bex ley +bel k +à® ª +ÙĪ ÙĬ +z yl +y con +west africa +spac es +ory x +oran je +of w +odys sey +n ür +japanese food +il lest +grind core +gla dy +fre ude +exerc ised +diar mid +da th +curren sy +awe struck +andrew lincoln +ðŁĴĽ ðŁĸ¤ +ãĥķãĤ © +yof theday +vinyl collection +vancity reynolds +un compromising +su de +spi ele +sing karaoke +rout ers +rei sen +red bulls +priv at +ma day +live strong +k mc +har land +goo ch +fi an +bit moji +aj or +ach ar +ðŁĮ· ðŁĮ· +work force +soci opath +pro fun +mer kley +love team +le itch +kin z +inflat ables +ge y +e esc +chat ta +al dini +æ ¼ +w aven +reich stag +off erson +nat west +moo s +mid nigh +gubler nation +grind in +goal tending +du jour +com an +charlo tten +bm th +blooming dales +appal achi +:- ). +ðŁĺĺðŁĺĺ ðŁĺĺ +will smith +unexplo ded +thegood life +tar ver +sy es +sush mit +stop adani +sh or +ol and +mon di +meet sworld +ka isoo +indv sl +fra ses +car in +bo ve +ðŁķ Ľ +ti pi +sustain ab +strang ling +provi den +ol den +ner ium +merr ily +janmash tami +in famy +docher ty +cassi eclare +carnit as +car ing +all thing +al at +y onex +worsen ed +w aff +tr onix +ste y +name is +mole sting +ma gg +low rider +lo it +jab er +id ling +i et +cra bb +beau regard +au tor +yousaf zai +un structured +syl lable +perman ente +o gu +nu b +kyrie irving +kele chi +he ther +f sh +csr classics +chees in +chandra babu +bar am +ðŁİīðŁİĬ ðŁİĪ +tew kesbury +super coach +prison break +onco logist +ocu lu +itz man +in house +go dot +ge aux +fah lo +disneyland today +bre gman +tor mented +nou ve +margau x +mar kov +loo kit +kimp ton +ish mael +goss elin +denti al +cultural heritage +cin ch +capsic um +ðŁ¦ ij +æ ģ +yumy um +studi ous +social media +seong wu +sat ory +q an +par rott +mac ey +funer al +fiery verse +e bit +congle ton +com as +chario ts +but thole +ap te +an et +ador ning +x hosa +un women +un dy +talk like +dhar mendra +da ele +con fines +cad dy +bury fc +av oce +altru ism += ))) +/ ... +t assi +scu f +ri a +renew als +rec ited +que re +psyched el +ow ar +geck os +egyp tian +ye p +seri ou +rosel le +public relations +oak man +me theny +make money +ll ins +k alo +ho sea +hau ghton +ha gel +gram matically +at ro +armist iceday +worldof tanks +vindic ated +triumph ed +ti eri +oni us +mon cler +mo ps +mis ed +mat ures +i gem +hilton hotels +geo logists +dishone sty +din ning +but te +alger ie +ðŁĴĻ @ +sym pathis +repleni shment +md k +mau mee +margin alised +manil ow +kar ta +im passe +hy vee +green away +d st +ba hl +ap ic +aerof lot +visakha patnam +the wall +style blogger +smoke free +sig mund +omo to +leg room +jig gy +ja unes +gai ety +free code +express o +ek man +drou ghts +cu i +chall a +ber nan +am pang +way sto +vol ante +ti redness +sen gupta +scoun cil +ro amed +mt k +linden wood +l alo +k lose +jac que +ilhan mn +hoot suite +ci pd +ampli fy +ðŁĴ¯ ðŁĶ¥ +ðŁĴ ³ +tor rington +ne farious +muj he +l ó +krug man +ki mani +jammuand kashmir +in h +im en +g bt +fla vio +dee mable +bo sh +blues fest +bi on +as awa +ðŁĩ²ðŁĩ ¦ +var icose +ton ous +surbit on +ssi veness +pre form +post docs +n sr +n le +jun a +iz akaya +gul liver +futu rec +fa ster +e of +bastille dan +apo cry +ðŁĺİ ðŁĶ¥ +ðŁĺĤ " +up start +ro ff +ran cho +paw paw +llll ll +kor an +humid or +her c +haw tin +goo gl +chic ory +car ro +ax les +announ cers +what sfor +per use +p ml +drag me +dis array +belo it +bar neys +ìķ ¼ +ì° ¬ +western australia +tweet perth +petru cci +oo ga +mc enter +mb d +lawrence ville +lak sa +kre is +ke own +kar at +fro licking +fac ulties +ed ra +dism ay +de kh +davi doff +cur atorial +brue gel +acro ft +- : +ðŁĹ ºï¸ı +y ny +sp reader +share thelove +lu ca +lic a +flower power +d ka +clo tted +aton in +am ori +ðŁIJ ¨ +wood peckers +titu lar +sudo ku +sd proud +pol ynom +mu sso +mi mo +figur atively +cor nea +ak iss +Ñģ к +za j +te eming +social es +n sp +mo pping +le bo +id ina +i want +harm reduction +har ian +darm stadt +arre st +âļªï¸ı âļ«ï¸ı +wedding planning +und ate +social care +sale speople +re ck +ra che +megyn kelly +me ille +ger r +enor th +cani sters +c mof +bi u +ðŁIJ ľ +uf fi +realestate agent +ny times +mor ial +mi shima +ken do +je suits +insp ain +hyun jin +gastroenter ology +eiffel tower +cheltenham races +à® ħ +we k +v logging +shoo m +rom mel +repre ssed +newho pe +nar rating +n cd +metal gear +gloss op +ger aint +fa is +ed ition +e book +coron as +car tman +accor ds +youn gg +un certainties +suspiri a +sal vini +preeti katweets +peru gia +ke p +in shore +guin nes +gi ger +family business +bin aural +au try +acron yms +---- --- +ãĥ ĵ +x viii +val di +urban o +star ry +pla ster +fli rt +zir con +un defined +tre st +the gold +su árez +sle ds +sk elly +moderni zing +mer lin +li ere +lam u +j hel +gol lum +cr ysis +chu la +cali pers +ca ille +bri x +bou lton +big finish +bc r +bar tending +world class +welove our +te emu +sed ation +sabot aging +q lik +pos ada +mother ing +jer ker +hello love +cinnab on +can poli +autom aker +ðŁĻıðŁı¼ ðŁĻıðŁı¼ +wm ns +vand alised +ul trac +mon soon +miz uki +legis l +ju ried +je anie +intro spection +ho ggard +cor rine +br ynn +bell erin +astro physicist +a bed +à¹ĢภĶ +won ton +whok new +un scheduled +the que +sig a +ricky rozay +pp p +llcool j +keer thy +kat sina +k cc +hop scotch +defin ately +d att +ðŁ¤Ķ ðŁĺĤ +æĿ İ +we ill +shirec c +qu orum +nd x +kha imah +ist g +ge et +fle ischer +fidd ling +exclu sions +electro lyte +dispar ate +boric ua +ar mas +tu delft +te ous +li de +leg ality +jil lette +f hp +boy scouts +ar jan +al ami +௠į +tat ler +steve harvey +shrap nel +sf g +sensiti zation +miss world +le me +industri alization +ic ting +har man +fol ate +den haag +clo sings +ch are +aru sha +ado c +us j +to ying +sof life +sna pe +pé rez +poorpeople scampaign +no le +looooo ol +jit singh +il b +gi ans +dot son +do sh +bra il +battlero yale +ðŁĩ·ðŁĩ ´ +ë Ħ +Å Ħ +this s +sn v +reddead redemption +pal me +pa wel +no witzki +nar ayana +mobile photography +m sin +live at +le ones +jaclyn hill +euph rates +engv pak +dc ad +co ffins +book launch +ðŁĥ ı +strat for +speed paint +s zi +ram in +perpetr ator +paint job +ol m +mr m +hal ved +flint shire +eri o +blaken ey +bin ky +aqui les +y out +watch able +w sf +the carlospena +roy le +ri ers +py d +piyushgoyal offc +magni fication +iso c +hurrican es +diet z +c wu +br ich +border collie +bis son +æ ŀ +val ance +u kh +truck in +ter y +rick ards +pest control +natu res +mo fos +m vb +gruff alo +ful tron +e ben +doo s +dar bar +car melo +busines stips +bou din +transpho bic +schae ffer +pre cords +mee tups +isaac son +e talk +dr g +barsand melody +aye sha +au dley +ash tanga +amar anth +ðŁĺ¬ ðŁĺ¬ðŁĺ¬ +ðŁIJ ¹ +shap s +r dp +mol lywood +kun dra +ki ba +dig vijaya +cycla des +co il +back gammon +b more +wensley dale +u ar +the house +tb b +sha o +nor ri +mer alco +l ée +is our +her ak +go x +consecr ation +chrisg packham +chester field +animo sity +! ðŁĺĦ +ìĥ ¤ +ya ad +v x +ta ren +syn dergaard +road kill +nat chito +mountain view +min ec +lighthear ted +leg is +illi er +grand daughters +ay ed +aqu il +ðŁĮĬðŁĮĬ ðŁĮĬ +w gbh +typo graphic +the be +ta cha +suc re +spr att +rom toronto +ol leyball +my st +lack luster +kal ash +ilfrac ombe +il ley +hon ed +heyman hustle +gu ill +go tha +crystal lo +bho omi +âĿ¤ï¸ı ðŁĩºðŁĩ¸ +ঠ² +z oni +ucir vine +t ga +ro vani +nipsey hussle +lun atics +les vos +kidrau hl +jovo vich +comic s +beck yg +arbor day +ad tech +ðŁĶ´ âļª +umbil ical +tan que +swag gin +stor ch +show off +sallye aves +picture book +my rr +jo ele +hor chata +el dr +dil iman +cmof karnataka +choose day +al ish +ver itable +tre jo +ran gel +rail roads +ny sut +morphe us +masterche fau +mani ac +kowal ski +jaz mine +ic ahn +credit unions +cra d +ann ation +yn ski +wilhel mina +sare an +nosfer atu +gri ffs +dias por +d jash +d iller +ct p +contigu ous +bottlen ose +baha sa +âĸ¶ ï¸ı +stal bert +profan ity +pharmac y +oc chi +ju co +ishi da +fe mur +di minu +comple mented +clo ts +bal akrishna +asv px +art net +ah ed +ag b +stanak atic +show girl +resc o +res ell +re group +pra vin +mt news +mb m +li ais +kell erman +kaz uki +gr ater +dis gaea +dere rs +def lect +concer tos +bha dra +beig nets +anak ar +ê° Ģ +stall ings +photo gs +music fans +mon gol +min now +mam ie +ib jp +e ta +cd ma +cath al +c mt +arun ning +aquit aine +win ery +to res +super latives +recep tac +par ched +loun ger +ja ap +i ia +hill billies +grey stone +ge tover +fashion ably +ad eno +yay yyy +west bourne +su stains +star buck +so so +sh ner +rave ena +oned rive +k town +in ar +gw g +gir ardi +cec ily +c ations +advers aries +иР´ +yeo vil +v allo +spas ms +so ton +ra bble +r ch +q gis +n bt +lake s +lady smith +is y +iol ani +iam j +drif ters +compar atively +cli pper +business owner +birth date +battle field +ym ur +winter classic +vic ari +sub species +spe er +sor ia +sion er +si mcity +o glu +mar cell +jeremi ah +ho pi +gar vin +further more +flo ssing +dogfish beer +discoun t +denomin ator +block chains +b fp +ah at +ðŁķ IJ +trow bridge +stool presidente +sky rocketing +sho tt +shan gril +ro pp +par ine +news line +m cly +le sia +kun duz +kon o +k fm +ic er +har twell +eng in +char ot +bel per +as yn +alter ation +a ish +æ ³ +transcend ental +sugar free +semiconduc tors +sau vage +red devils +mun dy +msle amichele +mo her +milwau kee +mclen nan +ll ws +j lin +gur meet +g tm +farm ville +f bb +burge oning +belly dance +ba sti +athabas ca +aran sas +a historyof +thisi sm +tek no +stif tung +south asia +prom posal +orient ated +needle work +local business +le iter +if as +ho cane +gran ary +domin ion +bo go +bar fi +abdul lahi +zane tti +woo len +si fting +natur ally +lu ongo +jaland har +interrup tions +ge u +game plan +fro cks +foun ders +facup final +dem convention +d ici +coup é +circu ses +bar gain +à® £ +up an +tram mell +tab led +seag ames +rest itution +q igong +pull out +opar ty +no p +ko dan +juli a +hal stead +ga the +dani il +bat su +b ng +ab ca +âĢ¦ ? +vali dating +transcei ver +re touching +mindy kaling +la gu +ke mba +hi ght +fibrill ation +dei ros +cor man +con spired +arcelor mittal +âĢ ¹ +z ata +yorkshire hour +ventil ated +ueber tangel +to ile +ter us +rho da +prank ster +m ston +lumin ary +kk rv +ker rang +gru bb +bu ki +bo one +aque ous +âģł # +young people +wi ig +wh ich +wax aha +synony m +summer lin +struc tural +saddle worth +rush die +pher om +p mr +oli go +og den +ne hemi +michel in +israel ites +hip ster +go duke +fu gue +evacu ating +de fer +cb schicago +wi v +spart ner +simon son +selec ta +rat liff +ra zz +plainti ffs +lu coz +kar st +iw news +hone ys +f sen +dinah jane +cec elia +ðŁį Ł +vote leave +tom daley +tibur on +srini vasan +roth well +mon dial +man chin +lovecraf tian +l mc +ha ving +gun i +den man +de ga +chu y +bru k +blue devils +ageo fultron +a ie +( !!) +wir ral +tm f +skybet league +ra ds +pk d +neil young +lad ys +is ys +ion ian +inhal ed +hoodie allen +ellic ott +car sten +al bay +adi da +acci dent +Ï Ħ +visual ise +tre viso +tra che +speed run +ra joy +prospec t +orlandom agic +nokid hungry +margare tat +kri ss +ik onics +grrr l +go hoos +g sf +do ty +applau ding +ac tu +ëĵ ľ +suffra gettes +star gat +jonas brothers +it alien +g luck +deton ated +can andai +bo st +begon ia +beef cake +bann at +anderson cooper +affor ded +travel guide +stell amccartney +re spawn +panig ale +one il +ny ongo +nike football +mow gli +montan amoment +mid size +kel antan +jamm wal +ha se +golds mith +fo den +da ren +child hoo +ald ine +adri en +ðŁĶ¶ ðŁĶ· +ðŁ¦ į +ss eries +spear headed +se xt +sad hana +ram bam +pe ta +oligar chs +mc court +loc s +ðŁĺį ðŁĴķ +и Ñı +~ âĻ¡ +yee zy +wil ks +tcc andler +que tball +posse ssive +moff ice +medi at +materi alism +jon ath +hat su +flu ous +craf turday +car re +b hala +am hq +veloci raptor +teen vogue +table tennis +se away +pre amp +pn pd +mc clean +labon te +invic tus +ic r +help desk +exclu sivity +etsy uk +episo dic +dat sy +bu teo +ðŁĮ Ĩ +ye a +sky box +sing let +pi f +or te +om ara +man alo +mac tic +li sd +feder ica +fati h +ener gia +el ines +coden ame +cho ckey +birth da +w ssu +ver bier +ush ering +talk to +t me +ro swell +neuro surgeon +ne pen +national siblingday +mess y +mascher ano +k vy +iy i +hong bin +flutter shy +chi i +ay go +y amaz +whit ford +un welcome +si yak +scri bes +sad lers +re imer +r cr +paw sox +parale gal +my picmix +moo ts +kirk caldy +k rum +ische mic +int z +gui da +gh es +gb w +fransch hoek +finn balor +east on +blu ish +atthe disco +âľ īï¸ı +ye huda +wi jn +wag ging +terri er +swar th +state champs +star fighter +schec ter +sas soc +pod casters +omor phic +ma dy +ine bri +go pack +de tv +d xy +cra ss +chag rin +bur den +ay m +app soc +al haji +z wolle +theore tically +tel ford +ri bera +problems night +po lis +mel ind +ish an +indi anc +ga ana +food allergy +equine hour +dream z +bi mbo +alou ettes +wal dor +tri angle +ste k +ra imi +qu ell +nieu we +nickelodeon tv +mohabb atein +lot l +liech tenstein +ir p +gu stin +decor ators +cl ack +bha ira +y cles +we music +train wreck +stam kos +sar tre +ru h +remin i +pizar ro +mu scul +liven ation +jazz festival +il ence +i ffy +constitu tionally +b ld +ìĤ ¬ +åī £ +stra ppy +sever ing +priv y +oo zes +nightw ish +hom ely +grin nell +fantastic four +du vernay +ce ts +ay den +ar pur +apar na +andrew smp +wyn n +vet med +town homes +tips for +tat oo +ste t +sa iy +rock hampton +pro choice +pnpd pcr +organd onation +n ago +meg ali +k po +jan ef +i mex +het field +gen et +free diving +fis ker +fe tu +ep n +democr atically +chap book +cas sper +carto oning +betra ys +ðŁİ ± +west bank +vis es +som ali +sivakarthi keyan +sc athedral +reflec tivity +postgre sql +o fus +no da +mu kh +mitch um +m fab +hyster ically +gi ano +force ful +debun k +cru ised +cic ely +brain washing +ak aran +ab ul +rash tra +pneu mo +oun tain +manit owoc +lo ic +it all +ik or +id n +hu ppert +gg gggg +z ite +thir st +te an +strang led +peanut butter +pc gamer +lo ta +kurt busch +ko stas +kib ben +jer main +gab bott +yas u +t pe +ry and +platt sburgh +nicole scher +nb nnews +mr james +kauf mann +it san +get outdoors +gam on +eugen ia +car man +bon heur +anti polo +ðŁ¤¦ ðŁı¼âĢįâĻĢï¸ı +âłĢâłĢ âłĢâłĢ +ÙĦ ÙĬ +ut as +super man +pickle ball +optimis ed +les ford +ko tt +journey man +gra bber +co inte +bra eden +bj s +atur k +ad ler +ðŁĴĻ âĿ¤ +won ga +wi er +wartho g +tribul ation +tan ker +stan for +shev chenko +regar der +r int +pun ya +nab y +mill ican +ha er +ev alon +dwar ka +cyclon enation +boo gi +blu ed +tra vail +so aker +plainti ff +mar kh +loreal paris +kovac s +fis ch +di ab +brew master +az ole +rugby worldcup +ny lon +nat t +jas si +igu anas +flap jack +energ ised +ed x +buccane er +baz ooka +ati l +ar dee +ðŁĮ ¬ +wil mot +the stage +super massive +seva sto +sc rit +river trust +podi ums +part iti +montag ne +mer chan +meetthe team +loubout inworld +kindness day +heb den +dur kin +cynic ism +cape x +ag ulation +abid jan +ðŁİī ðŁĴķ +yo sef +un avoidable +sting y +soyl ent +shar am +re using +offic er +mend enhall +je eves +hi day +day soff +bay swater +ban ned +ative art +april fool +apple wood +app easement +allelu ia +tri o +trax source +ss mb +re publica +raz r +p ingu +ouri er +mcgra th +magn ac +k mph +irrit able +ing roup +harvard med +hak una +gre nad +ero se +ed clv +door steps +counter terrorism +andis les +à¹ĦภĶ +whiterab bit +wh ill +vad ra +tooth pick +te mber +suspen seful +shar pens +natchito ches +minute men +mick y +mer ge +libr arian +laha ina +jugg ler +james on +in ker +gen x +fin de +engra ver +chi yaan +amon day +aband oned +a ami +twitter clarets +ter baru +spen ce +shav ings +sf moma +par ke +id ly +grena dier +bu ko +ðŁĺĥ ðŁijį +ðŁķ ¯ +tuesday trivia +ro el +mul la +min ami +luf kin +heart s +gine tta +g ff +dise ased +cute emergency +cor dell +christma sday +cer ts +authent ically +ap ta +am stel +wilber force +was sily +var am +se daris +naz ar +mori ah +kis ser +k ba +high heels +hh s +give blood +ging ers +eti salat +ener gie +dopp el +dex perience +cole gio +chester fc +bha iya +ag l +we w +stu y +ss ang +sal ento +psy trance +pan ko +paign ton +im pt +hoo se +goooo ood +erink rakow +design boom +clon tarf +b per +afc cup +abhi shek +wether spoons +ventil ator +tweet deck +stap ler +pow r +plo vers +nur i +northan t +mc garry +ma ur +lang ley +kla ine +justi fiable +habitu al +g soc +fin est +extre mer +exc elling +en coder +eil ish +duc kie +bon ucci +bct gb +si bley +red hat +philharmon ie +pe cs +mete o +m ound +liter acy +io ka +i hr +hyper bolic +happy holi +ess er +con temp +cau cuses +bm th +ym urray +when in +tw irling +sex ting +scar ring +ru den +ru bi +rom ney +ridge back +ok ka +off ends +ny mag +kla ge +fix ings +excav ating +digit isation +am alia +zam fara +w kc +unc aged +tele tub +purpose fully +mex po +mass governor +kha di +cor neal +bin son +allot ments +abur ro +âĿĹï¸ı âĿĹï¸ıâĿĹï¸ı +wicked ness +vaj al +tw im +tt weet +tru es +tan jung +sin ned +rain dance +priest ly +pra e +p fi +non sensical +meta irie +le omin +ha ase +g nac +eth ic +dou gi +bucci gross +bre y +a etv +/ = +zu bair +zephy r +vo id +un ed +sc ani +sav oir +recom end +mic ho +mer ch +lo cum +jun os +instagram mer +gago sian +eri ous +cau tions +best photo +an abolic +ag ame +âĿ¤ï¸ı ðŁIJ¾ +vol ks +up vc +terra zzo +spl icing +rte one +mc cray +g pm +emoun tains +east lothian +du bz +dmit ri +den ning +c sic +blood matters +baku gou +arame tta +al pa +âĻ £ +travel chat +tayy ip +su et +rebutt al +prote a +pontypri dd +pi ac +per d +lu ker +hypo allergenic +haha haa +fun friday +elisse joson +at rump +tom maso +slo ver +on omics +metz ger +lor ca +lek ker +ja ipur +inf ood +gl ent +full metal +cucam onga +cu taneous +cp as +coron ation +cal abre +bul ging +b ko +ap sa +* -- +yo ta +wo ke +util ised +tin cture +rhon dda +pc f +ngay on +mic hi +margaretat wood +ld i +hi ther +guil ds +cleve don +bank side +af ans +- >> +vers i +v ld +under classmen +tri an +te v +stone bridge +smi ley +rinse fm +real me +re affirmed +pla st +oo dyssey +nei stat +medalli ons +mc kibben +mbe ki +hashi moto +ha zzard +ha ther +ele y +ak ko +af ashion +western sahara +villeg as +su man +nor e +monte cito +mari bor +m ch +em watson +bu la +bas sy +bar ratt +yorkshi redales +ware ham +v pd +selfless ness +refil lable +om aker +mb l +fear nley +ea b +de marc +che quered +br ze +ame ga +." -- +yar mou +x series +ri gan +pig mented +patrizi arametta +pa ppa +of ah +mu cus +lets gor +leader boards +eff ingham +drive ways +dragon sden +cl n +cit ron +chi esa +bron wyn +brack en +bl v +are id +ami stad +ae oli +ae com +а к +wax wing +sz abo +openg olf +o berlin +mac ul +inf omer +ho de +ge ert +encapsul ates +cro mpton +con centric +bil le +bc jr +as gar +aired ale +usa a +tra gus +to pher +reed timmer +rare books +per verse +mo star +lom an +ll m +je p +ir ang +fi br +f mg +e ir +child line +book challenge +bon o +augu stin +at night +anup am +ðŁĺ² ðŁĺ² +what sup +u avs +t ittle +sw amps +st petersburg +so shi +mar ni +je je +inde mn +humili ate +do ped +cate chism +bur bs +awkward ness +ðŁĻĬ ðŁĻĬ +ðŁIJ¸ âĺķï¸ı +âľ ĸ +world league +vi di +theatre day +tal is +t be +sterili zation +shaf er +scal er +plan ar +nhl ducks +mapple thorpe +get covered +esopha gus +em el +cir o +braw ler +bottlen eck +ðŁĺį ðŁijį +ðŁı¾ âĢįâĻĤï¸ı +ಠĤ +Ø ² +vin eland +thr iller +side burns +se cours +pit ting +nu tz +nj pw +mogu ls +mee ch +ine a +houston dynamo +grav ure +gor ba +glyn de +fri en +daw are +commemor ations +bay max +ðŁ¤ « +xx v +tran quil +th um +spad ina +sol ly +mo ti +metast asis +mapu to +len se +im on +hilde brand +h sj +fur man +etsy finds +esmer alda +e goti +d fo +cham a +bri el +bor dered +ðŁĴ« ðŁĴ« +wido wed +thro bbing +themo in +ta it +synchro tron +stand er +skate boarder +samu ell +pa ire +free town +csi ro +ber ners +bar buda +squ ash +so well +raf ter +rad ine +oregon ian +northern most +mo hic +master fully +jar on +inter sectional +hass am +fla grant +emce eing +captiv a +buck led +ze ki +ye oman +welsh rugbyunion +tur ney +tam aki +stro llers +nn r +merri am +lien hardracing +hi pp +ev ander +ers burg +erik sson +cn b +bas ker +aphra gm +the year +stereo typing +sensor y +rovani emi +lo gues +kre mer +four teenth +bri ann +bow ling +bio logically +bang z +b har +arch uleta +a et +ðŁĺ ¿ +ðŁĶ´ âļ«ï¸ı +swit cher +se gre +ne da +mountb atten +la dle +catar acts +b cl +varieg ated +sou d +she is +rad ars +mistre ated +mc cal +gam el +g pab +conte ssa +chri sj +che ques +ch saa +bun nings +ambi ente +~ < +ye ol +under mined +trans lat +or to +ol oughlin +off load +neuro logist +mu ba +men ong +m cre +letic ia +iz u +hence forth +gai ther +e ws +cer berus +car ia +boy george +ac entre +zen o +w ür +vanessa hudgens +sushi l +pla z +ma za +kar dash +di va +di aphragm +cloud appsoc +acci dently +ðŁĴ Ī +Ø§Ø Ń +sw illiams +stie boys +sterling silver +si xx +s bee +re td +northyork moors +min olta +migr ation +ma shing +ma sam +lo ach +indiedev hour +ga is +ep al +ec l +bye bye +bic i +at elli +asen sio +anti o +ala stro +à° ¤ +un ir +to asts +specific ity +sma sher +shop keeper +ram ada +oni e +n ph +meet s +lular oe +li sto +kaf tan +j mi +fon tan +cardiff uni +bur ro +! ðŁĻĮ +vigor ously +themoin montrose +thel asto +t sang +slur p +sed ans +regre so +mun k +lar ds +ha sil +fra p +flin ching +dragon s +disappro val +del aire +chu cking +be coz +anarchi sts +ðŁĵ¸ :@ +wic ke +v axx +tex oma +tall a +summ ers +su si +ro wy +provoc ateur +pg achampionship +oko toks +o tv +magick ingdom +khome ini +hs sports +here tic +happ py +h ito +gbm fg +de paul +come di +coffee morning +cannon dale +bra ini +au robin +till am +plann ing +ph ir +panic atthedisco +mc pe +kanka kee +ful tz +fm radio +dissatis fied +con gru +bat ors +ambro sio +adol fo +acom be +æĴ ® +ãĤ Ī +y ona +tri as +to yn +thefuture is +pen icillin +os lo +mt gs +menong autham +med tronic +inf om +her ve +gau l +essence fest +blackveil brides +amas sed +aficion ados +aesthe tic +wo de +wal lop +ts d +thermo dynamics +school games +ram dev +pal patine +hom an +go vind +g va +fe il +el v +bjor n +av u +aaa at +ðŁĻĪ ðŁĻī +twin cities +tri alling +si ue +relax in +rapi de +kal o +gover ment +gl ick +fun fair +fick le +e ed +dre vival +che o +bull terrier +berk lee +ðŁĩºðŁĩ ¬ +çµ µ +tr yan +substan tive +sol heim +sh into +scotland hour +s oooooooo +ro he +ril ke +pro mi +nam az +mini figures +fraud ster +engad get +bb b +aperiti f +... "@ +$ - +ðŁĴ¯ % +» . +west cott +smo sh +odd ball +mee ker +la wards +hacken sack +fr act +fer menting +fac s +er rant +call the +buen os +broad ening +bar bo +afl w +ac sa +⾨ ðŁĴ« +woo din +ton awanda +sin ise +re ka +mu rad +kl is +ingl és +ij f +ham els +gre gabbott +f mp +egyp tair +egor aptor +csgo giveaway +contrac t +bar nes +together stronger +su ze +slo tt +rani al +lamar cus +hick ory +exploren l +beach club +yy ccc +sw all +suc on +storm chasers +sound scape +objec tively +nov ich +ni kel +neur onal +me aghan +manny mua +iber ico +fer ty +fa thead +dol lywood +dareto dream +d jen +cr pd +courier mail +baon pdx +vi vre +thomas rhett +seal ant +sa arc +qu asi +pac o +macken zi +k ole +john lewis +head rest +gn ini +generale lection +ben affleck +zul fiq +tac om +spel un +run dle +pr ana +la pped +kni ghted +gold fields +can oe +bellar ke +ba hr +amo led +acro ix +willi ston +wen ch +vig ny +ti the +se cul +sad r +pick ler +ne pean +may r +karrue che +is sf +han solo +fri zz +flood waters +fari dabad +dar ya +coden ew +cell ar +batchel or +ar co +ak t +* ... +ðŁijıðŁijı ðŁijıðŁijıðŁijı +æł ¼ +we will +un ch +sni ped +sat yan +ren fro +re ena +rd ma +ra am +iti ative +hear to +h mong +ght me +cine max +bon obo +atta ches +af tab +) âĢĶ +ðŁĴļ âĿ¤ï¸ı +ðŁIJ µ +âŀ ° +ç e +we gian +vin i +trans continental +tear down +tar as +tam agot +semb lance +pet care +notre ally +money maker +lu can +jazz club +her tz +great barrierreef +dec can +bogot á +a az +ï· º +twili o +tug boat +top brew +se ren +santac laus +roman empire +pr ite +pay outs +n sg +n att +gun d +bon nies +back woods +ante tok +an kh +ali f +able me +ver dic +van camp +tynd all +se vier +scele bration +ro darte +pe scat +par affin +kir wan +isi dro +io sa +hun chback +gas quet +fl it +el rod +cott ag +camero onian +buck s +at wain +ðŁijĮ ðŁijı +ðŁį ħ +sunrise on +shen hua +r vd +pr yn +on side +nom i +mour n +kno t +kha l +iri es +independi ente +guer ri +ffe t +cal lo +ðŁĵ Į +âĢ ³ +sj m +s inter +recipro city +pir at +pi do +nuclear ban +nag le +ingh e +golf club +goldman sachs +geography teacher +g mw +g inge +fu g +france sco +cor bis +cli theroe +bas co +alta ir +al of +ag over +tu do +tam per +ster il +say s +ri ss +pr unes +l ck +in decisive +guide d +gou lash +gold schmidt +geaux cajuns +fo is +dr ona +ct x +anup am +all things +achri st +ðŁĮ´ âĺĢï¸ı +ãģķãĤ ĵ +ve taffairs +sar is +qwer ty +ore illy +mcgu iness +je c +ir lam +h vac +for an +follow up +elix ir +clau sen +bram all +bighit ent +baum gartner +y mm +van ce +ta pur +s fa +pre ity +mach el +got g +dess ert +client ele +br una +bo ylan +al td +spy der +spirit week +semper fi +re developed +r ko +pre face +mc adoo +mal kovich +m mu +kanan askis +iw obi +ell yn +dream ville +dick y +coo lio +char maine +canal rivertrust +brown back +brac ed +a ena +tal kin +sw ot +si raj +say n +ryan gosling +ole um +mil denhall +ka dir +gram m +eng ined +dont try +death bed +cor sets +close the +aro or +amaz ement +al akshmi +é u +upp olice +tem be +stev o +scan lan +reco de +ma pper +lux e +ke yn +hr v +horror story +flaun ting +em s +dor je +dignit as +dar ul +chor ley +chav o +b hoy +ar us +ac ram +ðŁĹ ½ +uof cincy +universit yo +te aday +sal k +pin kerton +mc all +man oa +ma kat +ke wl +it x +ili us +ibu profen +go el +gi glio +f and +bau mann +bastille day +un balanced +ter rence +shot els +row ena +ra she +pein ture +moham med +mis sc +gau che +daniel son +cud litz +counter act +ca vern +ah soka +after show +wh ot +unner ving +to ko +sho pe +rise of +red friday +pobl ano +noble sville +naturema pr +mam malian +ma goo +know le +jam shed +go k +fo wl +dh ana +dand elions +cu ador +colleen b +co ba +bug ti +au guri +ap ad +am be +и н +vin ton +to vote +sentim ent +red chilli +rar itan +ra quel +min ter +kore atown +ha bl +final ise +fa ison +engra vings +ab at +éŃĶéģĵç¥ĸå¸ Ī +yo gan +x anax +we er +wahl burgers +town ships +stra gg +ste er +ste de +sel ive +my st +lu des +liv onia +kite boarding +kemp inski +joy fully +j hu +ig ner +go har +centr ic +bri bed +bla zes +ag rit +ver mon +u cle +sc ard +por g +plex ig +no plac +man nion +j abo +i aff +crest wood +co org +co horts +cla dd +can ard +bi kel +bannat yne +ban n +ðŁijĩðŁı¼ ðŁijĩðŁı¼ +zi ers +yesp lease +su fi +spell ings +quar ant +pa di +ki ff +end gunviolence +e ggers +con signed +ce au +brum bies +brit o +aldi uk +ad sor +abo lish +win itwednesday +thre elions +tech ies +snat ches +sei ze +pu is +ol mos +min chin +mce wen +mar ner +malam ute +made myday +labour day +da ar +cott age +ban u +ak land +ðŁĻĮ ðŁĻı +Å § +the wine +shuff le +s map +road work +re defin +mud slide +leon ie +head waters +hay don +clyde bank +cas in +cardiff cityfc +ber li +bar bour +au ston +ab us +ç Į +yi k +wa pping +sun der +scen ter +par snips +no bby +jen i +icom be +hpm kt +gla sne +ga han +fre ier +co is +bru baker +vis ite +te sta +te flon +roman tically +my c +kir tan +get some +carry on +asian et +_ âĢ¦ +wat terson +waste management +room ba +red ick +re mou +r team +prince harry +pr ca +post ings +new mexico +net galley +mp loyment +mil ano +cry ing +cd b +á´ ĩ +z ang +weather proof +tang ling +strat ford +sleep out +shown u +nir mala +n krumah +mon iz +lan et +del onge +box ster +bij lani +ag upta +a quar +yoon min +win the +un afraid +ug ent +supervis e +sn u +shak ib +sal taire +ru sk +pre fabricated +pentat onix +pe ston +na stur +l pin +go dal +faith full +enshr ined +crusad es +aldu bb +al ok +whole meal +riz in +re dri +or ta +nutr i +kelly file +gen k +farm shop +erken ci +du ffle +dev endra +ci gn +bc ity +av ram +ale u +ye ung +unic ycle +sp rang +si ones +ri parian +plu ton +pierce the +pha sing +od dest +non o +natur alized +n ls +my favorite +k ran +ic bm +hom i +gro cers +gov christie +four some +fl keys +d ally +ayour b +yor g +when you +tw ang +ti als +r tel +nationalbestfriend sday +mcgu igan +kath i +invo king +ev ading +dor tiz +col borne +bur qa +balu chistan +and proud +am ba +adidas uk +âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢ +uk is +tra ore +then fl +quarri e +origin ator +om nis +m gh +knare sborough +it d +ho yle +donat ella +cho ses +capit alization +are pa +ar ua +un cann +twitter afterdark +over the +ley va +le ggy +john wick +her pe +ev ora +der mato +a wat +ðŁķ Ĺ +âĿ¤ï¸ı ðŁĸ¤ +ภ¿ +wil ted +the tonyawards +sig s +sha hr +sel leck +re van +pan eling +jun ket +id g +gol dent +gaz ian +don gle +car fax +at w +| || +wheate ar +whe ein +tabletop games +speed way +solic iting +shk reli +ser ia +s ann +pol anski +on ara +iw f +indi st +iam nagarjuna +gre ville +fan zone +ee ek +do vey +bhar atiya +astro turf +antetok oun +amazon music +all saint +al amy +v ora +tv t +sy fy +rob ison +ra zzle +pli skova +offshore wind +no id +nain ital +ma dog +inter reg +il bert +hot elier +gu gli +chri shem +chas ka +ath om +and om +vo st +ter p +sun tory +summari zing +stor mer +steve jobs +st x +sc dsb +po tre +news boys +mc crae +luc ite +it at +excu sable +daz s +colon na +b andy +war game +w ans +valenci ano +sa chet +phx traffic +phila union +mumb aim +mar gret +fon dation +explo realberta +defe c +david caruso +a egy +ðŁĶ » +미 ìĬ¤íĦ +vir gen +ren dra +n ack +mon deo +live the +l sch +j lt +di ka +con dors +berry man +anc illary +acor n +о ÑĢÑ +wythen shawe +tobe apartner +tai z +street light +star lin +si u +pro sser +ph is +on track +m wh +humanit arian +travel ogue +trans duc +theop ap +seman tics +sat work +sahi h +pas schen +nik ka +narra gansett +na thy +man ado +m po +l sc +kcap inoy +kcapinoy star +i dd +ge ass +g onal +fair field +d alia +clean up +chor ong +ay ang +yn n +x fm +wil a +ren dez +ra vish +qui ff +puppete er +nat asha +inst on +fi f +e star +do shi +cu zzo +corre ia +cole man +annoy ance +.. ðŁĺĤ +á l +wi ps +try st +top chef +spoken word +sel ah +madel yn +lg fa +give me +e wood +don ington +ci gna +chry stal +calic ut +zimmer mann +tre et +spon tane +sour is +sag et +palli ative +mo jit +htafc dotcom +dre yer +dev ore +cycl o +cor by +bey hadh +banque ting +aber ry +ãĥ ł +wo ong +tess er +ss sssss +shop boys +screen caps +sa dies +obliter ated +ni as +mel ty +knock down +ka ji +ep f +din i +british council +ðŁij¯ ðŁij¯ +ðŁĩ®ðŁĩ ± +ðŁ¥ ĥ +wu v +un opposed +sw enson +stu ffy +spee der +raw ford +r gc +prayfor gaza +pg achamp +p nc +oni sta +mtv la +military monday +k he +fiel ded +engagem entr +en amor +cas sell +cad res +arund hati +.. ?? +⼠¹ +war ks +ver ny +theopap hitis +subtle ty +stat us +spro blems +spin n +simu lators +sail boats +rais man +oc are +mw angi +london symphony +freddie gray +con way +class act +bebe rexha +air bender +u yo +the music +re did +queu eing +leav in +kitchen rocknroll +hau d +glo ck +fe ile +be vy +bass master +barretto julia +band on +abar ça +a ep +¨ ë² +z ko +we support +trol leys +transcend ence +tal esof +silver lake +sharp shooter +schwe itzer +real gdt +oh yeah +life blood +king sme +heart attack +glori etta +extre mity +cro y +com motion +collu ded +col fer +checker board +cath ay +buen dia +am uses +aa ahhh +. ðŁĺĺ +ðŁĶ ľ +wo hoo +twitter vforce +rot ates +qu els +pizzahu t +pan tai +or me +man gesh +happy saturday +h kg +ge station +communic able +coast lines +âĺĨâĺĨ âĺĨ +y ooooo +thiru van +steve austin +ni azi +gg anu +em w +d itt +buff ering +am ma +ðŁĨ ķ +wha aaat +vs gb +spe ight +re sis +m se +j ho +ib aka +fro ot +evalon goria +din klage +bio hazard +beli a +ac as +ðŁij ķ +Ð º +tw r +sysad min +sun burn +rrrr rrrr +pr ater +kyush u +go by +consequ ential +come together +beÅŁ ik +bab b +annak endrick +ðŁ¤ ĸ +x rd +too good +seal er +re ira +ra ut +pet tit +own tv +ol ler +mountain dog +mis sp +goodbeer tweet +european union +efur niture +dra dio +disc ern +call ous +âī¦ ) +ut mb +spur rier +soli der +or bison +od g +mic a +ktn kenya +koep ka +ic ca +gau lt +g x +g dn +for ts +fil mawards +eu tical +ea g +dier ks +cannabino id +bul bas +;; ;; +ðŁĸ IJ +vit toria +up lift +under writing +sne ad +sn ell +re distribution +p do +no akes +narayan an +j vc +gram ophone +franç ais +ell ery +convey ancing +bi ked +aw we +ab ulous +wan te +sh wara +pay son +lu mumba +lifeat att +le ics +iron fist +gr int +figh to +copper head +aqu are +ÙģÙĦسطÙĬÙĨ ÙĬ +we make +t ys +qu t +play as +off a +ne revs +must apha +meta physics +mammoo tty +legali zeit +jun oon +jan n +flatt ening +du ral +cam a +bub ba +antand dec +actu allyn +aar ons +ðŁį § +á IJ +wi zz +twin peak +sle wis +parishi oners +oak ham +mai du +jessica jones +bay town +az s +ates sen +anc ing +ðŁĻĮ ðŁı¿ +ðŁĺĢ ðŁijį +ঠ® +ske w +fi af +da sha +cladd agh +bino cular +bal le +az ria +v ented +ts laq +sn m +pen chant +mod ality +gand hin +frivol ous +del am +cc na +ang an +am os +alente jo +across america +y ore +twee ter +the clash +ny lons +needfor speed +mag got +lion king +har id +h sieh +fabi en +ul hassan +ui design +ste vi +sl ats +retwee et +radio graphy +por poise +man cuso +lap wing +ki bble +gram pian +fai ers +ec nl +dun phy +disney pixar +de eney +ca pote +ðŁ¦ Ī +Ì · +v int +tyranno saurus +tu gal +sw amped +su strans +small town +seag al +salvation army +ready stock +kri ders +hen an +groom ers +earth lings +ce da +bom i +actuallyn ph +vand al +sch rö +polic eng +nbc blacklist +mul ca +jack johnson +eeee eeee +bri elle +brazil gp +b ages +woo gie +wat tle +ve ley +tede schi +tape stries +stain less +sb s +pri yad +parish ilton +nam pa +mor rell +melo dic +kam o +impro ve +hill climb +eur or +dev ant +dal umni +chi ellini +al chem +ak ashi +vote trump +steel heads +six pence +po wn +offici ated +new yor +magnum photos +lin dy +la yed +int ar +immortal ised +hall fame +f hd +cor dy +ba a +ar ru +ðŁĵį # +âĮ Ľï¸ı +tt b +ra pper +pier cer +pe m +nomin ates +marathon training +le vert +kodal ine +el ford +e gl +doyle stown +ay re +as suring +yo tu +vel lum +up sers +tg f +supple mentation +phy sorg +never stops +mean est +maple story +kid dy +incu bus +goav sgo +fic h +cot illard +carmelo anthony +c ny +c me +az pi +âľ ° +suf jan +sneaker head +sher if +sa har +rum mage +rub instein +remitt ance +rail a +phant asm +onyour side +mccut chen +main streaming +it ag +hoss ain +end or +de briefing +cou ros +boo tie +bharat anen +baesy stem +aud ited +am un +ðŁĨĺ ðŁĨĺ +ঠķ +v apes +superannu ation +ry anc +rec ourse +re working +pom pidou +pok hara +nma ahc +equip ments +do ha +cham bray +ba ste +year lings +vap ors +tom kins +tom hardy +san s +quo tes +pixel ated +mur tagh +md ma +mau led +erec tile +dd j +brah man +blood stream +alway sin +ai kman +whad dup +un authorised +topbrew stues +sea horses +remitt ances +ra id +play ers +lee son +joh nam +ipan ema +dust bin +devan te +ab hay +! ðŁĺĢ +un ni +tar heel +o jib +mal lett +machin ist +got chu +gb l +e ish +discrimin ating +bc d +az tec +avi c +ðŁĴ¥ # +âĢ¼ï¸ı # +wool len +timm c +sun se +st oughton +sethro gen +ro tten +ro sey +over lords +night shade +mou ld +min c +mi ele +line of +lifeli ke +glut tony +fla galine +fan made +e art +destin o +desc artes +bun dy +artist as +we bradio +ty agi +there in +su si +sp rit +side by +ro isin +pt bo +pro bed +passchen daele +n ich +man as +jor dy +gwend olyn +far rington +ef ury +eamon n +cu v +buzz y +ut tered +t ally +surbhi jyoti +stu m +shar an +q v +pre tender +ji kook +hol ger +gh is +co axial +che wie +blue moon +ash bourne +up cycle +tes acker +sy monds +silent film +service design +pre go +pa wns +one ttes +nc ss +monmouth shire +lum ley +level led +fun nels +flint watercrisis +flick ering +edel weiss +croke park +cloud flare +cis neros +b appa +un protected +sp anned +som in +score sheet +look outs +libr ar +jen der +jas am +g land +french open +disclo sures +az ura +ðŁĺĬ ðŁijĮ +ðŁijı ðŁijį +wel a +vit ra +spine less +my way +le anne +lat ics +kri ssy +k va +inge sted +hu bris +h me +furnitu redesign +f md +discre tionary +d mm +comple to +bc sm +balo gun +womanin bizhour +som mers +pd m +ol um +o sho +ne en +mobili zed +me gas +incess ant +gu aj +ga th +fa ste +ed un +col lies +arche type +ad us +ç Ħ +yo yo +ul lo +re wilding +mac ron +m peg +kk un +ji ju +for senate +er ud +edi son +com ey +ðŁĵ± : +æ Ģ +un worthy +talk ative +sc rolled +s ment +rainbow six +pin up +p tv +nc w +hager ty +di xie +cor delia +coles windell +ch ito +c pim +ali ef +ðŁļ ª +âľı ï¸ı +âĢĶâĢĶ âĢĶ +x ss +ww ltv +tv l +sel van +ra gini +ph ore +par ry +o show +mar ref +mam aya +high field +fis ch +e amad +dg allery +dail ye +ck ont +ce ce +buon anotte +be ary +ðŁĺĤðŁĺŃ ðŁĺĤðŁĺŃ +sw ope +snow drop +sin dhu +pet worth +mur row +mou st +manife stations +gra dation +gorba chev +gh ul +fc dallas +euro trip +dw b +dom ic +datasci entist +ali sta +ac ps +ðŁij µ +ve mos +tur nar +the first +su var +nord ics +dizz iness +dit ko +complic ate +come on +cogni zant +citro en +am ory +ðŁĩ®ðŁĩ ¸ +z ela +y are +super fans +r ry +meas ura +mari ok +je ux +green newdeal +gi um +d zeko +bicycli st +approxim ation +appli ed +actu ally +ðŁIJ Ķ +wwi i +under lined +so ty +slur ry +sho ta +scol ded +o ona +no ord +naturale za +loveyour self +kim ura +hack man +go sh +dru mand +de jec +chri sco +cel le +apr ile +ad ot +åĨĻ羣 好ãģįãģªäººãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +un provoked +tt ps +step father +sen tra +ro hini +rabb a +personal isation +mirr oring +mc mullen +lun ges +lat itudes +koon tz +kevin jonas +jimmy johns +forzam ilan +car bons +ach enko +ye sh +worl dd +war sz +use fulness +su pra +sol as +rapp el +mo sth +ki is +im bec +efan dom +drou ght +co ax +bur saries +black bear +best oftheday +ar up +ðŁĴĸ ðŁĴķ +woo commerce +waist line +tr ini +super liga +recur ve +ra ho +nj ca +nas r +mesmer ised +mer tesacker +lu ce +j illi +im mobile +de commissioning +bo ta +] ... +vo j +tibet an +sponsor ships +sp ad +roger scup +re filled +pune eth +olivier awards +nether land +n whl +kil kenny +kedar nath +kap a +ha shem +follow train +eth yl +dar my +cr ps +bay ard +wre tch +w mag +super girl +su an +prece ding +ni uk +multi faceted +mali ka +insp i +fr b +emble matic +cap uto +bur ren +xim en +ul ta +smo key +si zable +remain ers +mesu to +men zel +mc daniels +is kandar +fuel cell +fron ds +bu xton +ari ba +americas cup +am iz +ðŁĴ ² +x is +ur chins +sur fl +sn p +see it +or ra +nf örde +lat ex +kre m +ir v +hel der +fore t +ecker nförde +drum heads +car nal +ðŁİŁ ï¸ı +whati f +vas ili +succu bus +s wales +ret ford +mon di +ma ina +lun ge +la shed +indie wire +gla sper +fresh eyes +forec ourt +fan k +don n +disturb ances +denomin ations +boy ish +arav ind +( âĤ¬ +⼠Ķï¸ı +whe elie +u plands +scru ises +pet s +me chat +mac am +like mike +lgbt qi +jo li +ido sis +iat se +he di +er oo +eamad den +ê¹Ģ ìŀ¬ +ste ered +rh s +pop sugar +n ape +mun nar +ling field +leban on +lan te +kend ricks +jelly bean +igh ton +ho dder +gor ky +give sback +dayin wa +cor tic +c caa +buzz ards +ar awa +aaron rodgers +ãģĵ ãĤĮ +yearsof onedirection +wood wind +true to +sal inity +re sin +pl ural +nor cal +liz quen +kay ne +gu rion +gi org +gallo ps +conti go +chil de +car issa +ye oh +win ky +w nu +son parade +show case +sho walter +ru ston +nicolas maduro +newarri val +monster mile +kumar an +kad ri +jim cramer +gu lab +gravity falls +g chq +esper ance +cur lers +chamin ade +brad field +travelchat sa +tor rens +rh swis +ree se +mal vi +lof ton +law firm +kp cc +it ab +fer i +el lum +diversity and +counter point +chrishem sworth +chaplain cy +biz journal +bi sp +bi elsa +at cha +assur ances +ak ay +aer lingus +ya yoi +sode xo +reme ber +ord nance +or ation +lin donesia +jo sey +hast ily +go pin +fan atic +el oun +depend encies +comp ounding +az aki +al wefaq +ðŁĺī ðŁijį +ðŁį Ĩ +venkai ah +stimul ated +pp act +pmr outine +papp ar +mel oni +mc gur +j itters +it sc +harsh ly +ham ish +el ca +dece mber +de wy +copper field +bha kt +be more +apple seed +all yn +aby smal +ðŁĺħ ðŁĺħ +ys sen +tu q +to ei +thor ax +se din +sar cast +po way +or se +jan asena +cityo flondon +cat lin +car lie +bie bs +bc fc +ap y +[ !!!] +:- ))) +trav elling +raun chy +pim ped +kat ja +ju tland +h pl +first day +crew life +colla bo +che ong +che chen +bl ink +bernab éu +ban c +win x +ur gent +tul u +sof c +repri ses +pe pin +optimis ing +gau chos +com bo +chang wat +bo ca +b dm +audi sport +ðŁįĢ ðŁįĢ +é« ĺ +white house +sav in +r so +p bo +k de +illi brand +g sr +conver ging +conduc tion +adequ acy +ab ane +wood all +tha ic +tant alizing +soren to +satis fies +rush theband +rhyth m +ner c +ma ilers +jin hwan +exem plar +en acting +dar r +d ars +ball o +agr itech +ðŁĺı ðŁijĮ +wide body +u ow +tur ley +sab u +red waver +perse us +out do +nam c +mm el +las z +kne cht +interne tradio +haw kn +ey fs +dur bar +aegy o +. -. +w awa +venkaiah naidu +sure fire +stone walluk +ru slan +royal enfield +pollu te +natur alization +mo oning +li otta +iow ac +he yer +eli ver +don th +cal ma +bri anne +am ission +action news +vish war +treach ery +talk back +sav chenko +ri pon +pur vis +no e +mne monic +kol kat +k oni +johnny cash +jam el +gall i +fer nie +extr alife +eeee eats +dom ani +dann er +cy b +bel fry +ðŁİ ¿ +zil djian +yam aham +tur lock +to play +si sa +rho c +passiv haus +paratro oper +ju ara +insectic ide +fat boy +brigh ouse +be cket +ao e +wel lian +tim tebow +thegirl gang +su c +sto watch +sp iti +octa vi +jen g +jac aranda +improvis ing +hoo doo +gry phons +fri t +be ane +ðŁ¤ Ĵ +yo ka +wo gan +witha view +un controlled +tw oman +ti z +thereal taraji +rams bottom +ra bles +pen ce +pe per +mi hal +man ti +mal to +ja u +ig ar +ice service +hosse in +gen italia +g age +fascin ator +baz os +abyss rium +we bex +viole tta +une lected +un ashamed +sor row +ram akrishna +pe f +pay a +na ev +mor gon +l th +j iri +f sp +ethnic ities +elle magazine +co leg +ali bab +ëª ¨ +⾨ ðŁİĤ +up fronts +stoner nation +stack house +retali ate +ram apo +preity zinta +osc illo +n pc +instam ood +in ck +hun ks +hi b +fluor ite +disc losing +br g +appropri ated +amé rica +y pe +way anad +vi ñ +v le +trin kets +to to +syn bio +stru th +se wed +r ce +pain killer +night mare +loan ee +implic ation +guer in +fi i +deb out +dal le +clut tering +astra zen +as saf +afric ana +# ## +ðŁĩ¨ðŁĩ ³ +⤠µï¸ı +tex turing +steel workers +star man +son n +scho on +roo de +nit in +mi ah +inten ding +happen in +hali m +gun fight +ge ffen +de pot +che tt +am sa +ðŁ¤£ ðŁĺĤ +yess ssss +sha ina +scen e +sb spop +rol lin +penand ink +our n +ok ami +mer cure +me thu +mari ya +en closures +dmn takeover +athle ta +aggreg ator +wash out +sunday sunsets +re watched +nr cs +ma shi +lynd sey +k adam +ik ka +i sen +gc n +fl un +ent wi +discipl in +antic a +. _ +ðŁĸ¤ ðŁĴĽ +vit is +ur laub +trans at +tra inee +tom petty +the powerof +next generation +mo is +mac er +liam gallagher +lev elling +k aga +int ell +gh ard +dol man +cu ten +cla ves +cam ill +bur well +ag ia +accu sers +à´ ķ +zak k +yan cey +wi jaya +w rest +ven ables +te sonline +sha z +se gal +ri r +pin us +phone tic +nor s +natgeo wild +le asure +hi an +ham mar +goo gl +ga den +el che +cab ot +bu lova +bah n +an agram +agency life +ðŁĺ« ðŁĺ« +u ña +tro wel +tam im +se me +pap u +mfab oston +marin as +ha de +evapor ation +com miser +bor sch +bor ja +yo del +toho ku +ssi ve +new marke +mine head +mar wan +mal ari +m mb +kor fball +im part +hedger ow +he uri +gab bar +elpas o +e wu +cour chevel +col qu +char ol +buzz word +ab vp +visual novel +tac s +san ghi +ph all +per kin +op hia +mexican food +math ilde +li do +har grove +gor abbit +fun house +envir on +e der +de sen +confi dant +b ya +ay k +ant ina +an anth +ภ® + ¦ +yang tze +tagli atelle +sr w +sor ley +sk ellington +sever in +s oooo +mo ku +mar ri +iphon ex +invo kes +guil len +get to +enti rl +en cel +e bro +digg ity +cr itch +ci morelli +ðŁĴľ ðŁĴĽ +yo ho +su deep +so cool +sine k +see ker +roy soc +ro ps +re mington +re lo +paul walker +ode tte +martine z +lec ture +laban pilipinas +ken z +hibern ating +flag pole +fight club +fer nan +ab ack +tam iya +stone hill +stin k +sell ing +re treated +pig tails +pe eler +par ten +n ku +loaf ing +ko vo +i sie +ferr aris +cdne con +c ti +bi le +ber cow +bar ing +augh n +ace res +ter se +sten de +rizzoli andisles +ri son +rav iteja +ph q +lo ews +jaw ad +gim me +fridaysfor future +cal cite +by line +z aya +west mont +v ce +tt ac +t ø +super show +stel ena +scape goat +mesuto zil +mer s +livel ove +g end +g ann +fun kad +evan cho +conver sing +ak uma +ðŁĴ¤ ðŁĴ¤ +wh itten +ti gnes +skysports news +sex press +rum maging +ov ary +mu v +ma homies +ha chette +gi gging +gi gg +fel ting +con vivi +blo or +acoun cil +à² Ĺ +Ú© ا +z ner +sc w +rose mary +rhswis ley +rabin dranath +polari zing +mel atonin +len nie +leed sr +ke zia +infan try +he k +gen nady +ey oung +change theworld +bu te +bay bay +assemb les +ðŁ¤ ¢ +wise words +we ws +was aga +sw v +run k +pul s +mon iker +mer o +hur ried +garden ersworld +frisky friday +ev b +cn c +c mv +c ati +actionnews jax +w st +shot guns +scottish labour +sau stin +new single +merr ill +ma jer +kitesur f +impecc ably +grand fathers +go bi +glu ta +fe moral +fa thered +e sports +cre spo +bhagw at +au coin +aram irez +ang u +after care +w aca +trac eable +sav oring +purple reign +pas qual +may ans +maidu guri +li ens +im t +ful bright +f ram +domin atrix +be my +ai ww +wal sall +w res +ti ri +ti ed +sch engen +reasons why +luxury life +le pore +kn itters +he k +bibliote ca +bene factor +bedazz led +bbc three +ad g +ðŁĴĻðŁĴĻ ðŁĴĻðŁĴĻ +si stas +sch alk +roch mn +r pu +pic ton +paper weight +over se +mat zo +masi h +gwali or +gau r +football manager +flin toff +fitz ro +dal and +crescen do +bow ery +ateli er +ark ana +antetokoun mpo +ws fa +wi zz +st angs +ro v +poo kie +parid hi +my lo +itu ne +hu ed +gorabbit ohs +fred do +ed ical +dj mag +beacon sfield +( > +z ep +wab bit +u om +stu bb +stap o +singular ity +p gp +nehemi ah +music education +ho key +gun nison +fri zzy +feed ly +chap man +ch alo +bien nal +belaru sian +aga ins +> " +ì¹ ľ +ton gan +th ais +stor me +seque stration +s fra +psycho active +ol ph +mi dat +marc jacobs +mar ini +m ellen +layo ff +kan chi +hi hihi +gul zar +equ us +can va +bellar mine +bad minton +anag rams +ðŁķ ĺ +ç © +valenci acf +tanger ang +ss ociety +shaw shank +sche rer +sc ity +red v +ra whide +petr us +od as +nsc aa +man am +lock yer +lar ams +kiri shima +im petus +gu lag +french gp +cu bano +bil lo +aw in +asser tion +tre f +the expanse +raisethe wage +o smo +melancho lic +luci an +koo pa +cor relate +colourpop co +c zer +bis ky +beck with +all ga +al ang +ðŁij¶ ðŁı¼ +whydont wemusic +unfor giving +str ath +sell in +ron paul +ri sm +qu ino +music day +mat ata +legion of +heat nation +gro ats +fawad chaudhry +ebit da +chriscor nell +adam awa +าภĩ +z n +waq ar +vel e +treat yourself +so cratic +pie tro +net suite +leon ards +lam bert +kyrgy z +k cb +ike ji +he f +gfx coach +fat tah +fashion blog +chi story +b sk +ðŁĺľ ðŁĺľ +v lm +shaw cross +plo p +pixel s +indy star +in compatible +home brewing +fri eda +dun gare +consumm ate +cha eyoung +brow ski +are llano +ar sh +anni es +- _ +ðŁı½ âĢįâĻĤï¸ı +teddy bear +humane society +geo graphers +for sake +de cap +com plying +col onists +car ay +bu is +ðŁij¶ ðŁı» +veer am +tra jan +to ch +shyam alan +ri ki +pre neur +pin wheel +per v +o sei +its not +iiii ii +hydro logy +haram bee +gossi ping +fix in +ec mo +be art +ar x +agra wal +ãĥ § +univers itas +tremb lant +to saurus +shin ki +sci oto +om itted +my asu +lou ghton +hypo thermia +ee as +cre mated +az ale +as as +!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!! +wi po +suffer ings +ro dr +nouri shes +noban nowall +me irion +kle ine +kan aan +ho ek +form entera +eng vaus +ck c +celine dion +ber sih +ae us +a hotels +z ke +sch im +poo dles +po sible +on ted +meri jaan +luke evans +lor re +li ber +ili pp +her nan +haw kesbury +eugen ics +bates motel +assam ese +win denergy +ver tic +th aus +stan field +peru ana +new ick +george takei +g mp +fer me +dis su +day mond +cha el +cer ia +ai shu +ĤâĸĤâĸĤâĸĤâĸ ĤâĸĤâĸĤâĸĤâĸ +ðŁĴĥ ðŁı¾ +un specified +tor ry +sp ousal +sapol iceservice +re funded +p andy +master sof +madel ine +m rd +julian assange +gon u +dep ended +delmar va +de commissioned +de activated +com usic +bir fday +ance day +alber tsons +al man +a etna +yas min +whoo pie +vidy ut +tv land +too led +sutton nick +pro mon +nudi branch +lm g +kh ans +jac inda +heterogene ous +e zio +com pass +calibr ated +bio logics +ben j +back channel +v aya +tra yvon +suffe rer +snow white +scre amin +pledge music +palad ins +mo va +m mo +lan k +kr rish +cd ns +brown s +bah nhof +babylon ian +b á +ath ia +arm oury +ad ou +abo lishing +== => +th us +quen ching +proud mom +pre phoo +pheno type +par atha +monte iro +ex hal +evalu ates +drop the +ashi on +all my +ðŁĴ Ĥ +you m +writers community +tye ight +tren to +stu bs +southea stasia +some place +per using +pepsi ipl +no ggin +mc curry +gustaf son +fon si +chri sy +ar peg +abram ovich +... ðŁĺį +âŃIJ âŃIJ +uni for +u af +tap ings +tank ard +sn cf +small town +sid mouth +se ta +rosen baum +rc f +pay sage +oo th +mer v +kit o +ka ito +jewish press +go bolts +fen way +fc bb +clu stered +cap lan +bo ater +beur re +bar nyard +anti viral +z uri +thorough breds +super boy +pha ges +par ibas +our team +ol denburg +mccas kill +ky ga +ku la +kk city +jo jos +girlswho code +far cry +da ren +clon mel +ar ci +alo y +> .> +ðŁĺį ðŁĴĺ +viron ment +ul s +u ssa +run ny +riv ington +pp u +over worked +loc i +is si +gradi ents +gol i +go eags +gla iz +csul b +cn l +ash field +am k +ab ject +ðŁIJ ª +you thin +table lands +ta de +sewer age +sau sal +ro la +py rite +palm dale +mur doch +love capetown +ka unas +hur l +gib ber +ge stal +fu mbles +eu banks +down tempo +dae bak +cra ves +cal stampeders +caber net +be yourself +bay field +val ken +tow ner +terri ble +syrian refugees +sun lit +splen did +saf b +rapha elite +pseudon ym +p mp +night light +lo rele +kiran ks +in off +horror movie +gr und +gilli vray +col son +cas well +bu mba +ØŃ Ùħ +ver batim +she h +scu le +regime change +r cd +pla id +os sett +mobil isation +mel d +marcel a +leg day +ick ness +fo lic +farm bureau +f hc +dee pp +cli ft +cla dies +bu gged +adel phia +shel p +ok av +nav ami +mclou ghlin +femal eartist +ed r +e juice +dissec ted +din er +clean ing +x tra +we tt +wa ii +w vb +vi ana +ve das +tal le +stat er +sale en +sa iler +s do +onas sis +o der +nitish kumar +new garden +lea ke +ho kie +h ds +ge w +fu qua +dor g +chlorophy ll +brain y +ai bo +âľį ï¸ı +âĸĶ âĸĶ +๠ĭ +venezuel ans +teren gganu +syl lab +sling ers +shar pe +san ce +re es +mor k +mill man +mick le +lo ser +jor dyn +horse shoes +gr ath +dre cht +dar ley +bow en +ar beit +aero drome +aditi rao +ðŁĻĮðŁı½ ðŁĻĮðŁı½ +® . +w dr +voll mer +vis sa +vermic elli +v ÃŃ +un de +son nen +po da +md wx +ky an +kilo grams +j ko +gran tees +gin ter +for acause +en coded +ela er +el sen +christma sparty +alk maar +оР² +wed ged +war crimes +wa ja +virgin atlantic +su bro +stabili zing +sc ab +ru f +olivi a +medi o +mc iner +je ka +im onday +ho va +hier ro +grey hound +great day +gibb ous +gay travel +footsc ray +far had +alon te +your say +tri state +the box +sed don +sb f +re sol +op als +nay sayers +mcco wn +m atia +jab ber +gg c +ft ar +fr anti +f ndn +ess endon +elyn n +at it +the on +som m +share mondays +polish girl +pic ka +pi ped +p mu +nave en +mus ch +lal it +hollywood bowl +fow ler +d ga +cor on +car leton +brow ne +b blf +as ante +wr n +vol ker +syste ms +state park +seduc ing +san de +row lands +riz wan +pakhtun khwa +kri st +ka sia +hudder s +frac turing +f yn +esmol lett +dc b +brisbane broncos +becer ra +ak ir +ðŁĩ ³ +í ĸ +Ú© ر +vil les +responsi veness +rain maker +pig skin +marti ans +mariu pol +h wc +ge h +gar ra +fre ire +flight less +di biase +ðŁİĦðŁİĦ ðŁİĦ +wwww wwww +ti pu +ti amat +succes ful +shi seido +nas akennedy +mu x +mu swell +methyl ation +live from +li est +lavin ia +jes u +glynde bourne +g atti +fro thy +coinci dences +bou lud +bizar rely +ber nad +avel ino +ast enders +Ì µ +v sa +un finished +soci ologist +seaf arer +poly glo +pa chi +ki as +ke th +karthi k +jac qui +ha ik +g cl +ferr aro +cornu copia +brock port +arte fact +aj ah +trun dle +steu ben +p gi +opportuni st +mussel burgh +mat ar +innu endo +hawk man +h under +figu arts +ey ama +exter nally +dun gy +debu tante +con sol +chand ni +bin d +au den +ak ari +af ood +ãĤ Ĥ +yu mmmm +yellow fin +volley ball +to gs +sidi be +nür burgring +newor der +len ient +lar imer +justanother dayinwa +ill aries +hamid mir +fine gael +bri enne +blog tour +be ter +bar to +ard elli +yotu bazos +tah itian +spit als +sin sider +see ing +parisi en +over hauled +op lan +mic kie +long shot +la pierre +hoag ie +heel ziggler +gi les +ge to +fossili zed +eu g +di ot +bhar ath +ðŁĺ±ðŁĺ± ðŁĺ±ðŁĺ± +ðŁ¥ ķ +ä¸ ī +still man +redchilli es +pan sies +newh orizons +mouth wash +mi shaps +mad dux +lincoln center +ju mble +here for +dr illers +congr at +chi bi +charlese sten +break water +big dog +aro se +viv ant +rac quetball +pu ffing +plei ades +par di +nar gis +michael phelps +lec ce +has sen +escuch ando +dun garvan +dou ce +de ff +cy sts +a vide +⬠ħ +tr anger +the west +suri gao +sn sw +re touch +re smi +r mnp +peaky blinders +mu bar +mi gs +mi glia +merc ado +koo ks +inu yasha +firec rackers +debau chery +cress well +cat suit +cast a +aim i +y ne +v q +unfa thom +sp ind +si sta +shay la +s ago +one minute +nu ka +n aki +leomin ster +ju iced +institu to +hi ja +das a +co so +chan ia +cav anagh +be amish +atay ulu +applic ator +y v +var ane +uni dos +tarte cosmetics +swin dle +sli eve +run disney +ren da +per ly +p ft +or gullo +on lookers +notal one +jet team +h spa +gu ic +fox boro +exoske leton +earth worm +das co +cu ppy +cro quettes +brook dale +bo lo +b ht +av ac +z sbp +y alla +vou ch +smo or +rak shab +push back +pardon ed +news watch +ic ma +god se +eu stace +er aces +caesar spalace +by city +bun ga +bu oys +al iso +ðŁĺĪ ðŁĺĪ +ðŁ¥ĩ ðŁ¥ĩ +un assuming +studi os +p md +mcclu skey +kq ed +hol len +flash cards +final y +fam erica +f ds +cre dence +commonwealth games +built by +bri xton +bar bac +ago v +å Ł +à¸Ńภ¡ +yo gini +wing stop +virtu alastro +stonebwo yb +statu ette +sc ad +san juan +plo sone +pe ga +op ah +o id +dg m +cir rho +charm ander +bur rard +anti gen +aless andro +ðŁĴĥ ðŁı¾ +west papua +w fm +u mentary +u fm +thereal stanlee +thereal pcb +tham endment +t fp +steep er +raff ling +nicolescher zy +maxim ising +list ers +kn c +ke tu +h itech +gymna stic +goo oooooooo +dra gic +de caf +cor dero +aster ix +af at +visu alizations +v uk +sadh guru +rais ers +par thi +on ah +oc at +nav rat +n pcs +minu tos +kin tyre +fun dy +extinction rebellion +broad side +bro iled +brid port +aard man +ze id +sun sport +sle igh +sg v +se on +ran ching +our e +ol p +mi stress +mi ff +me hr +lym ington +knowledge ispower +ine ke +g dl +cro ker +coom bes +centa uri +ber ber +ann alise +aditirao hydari +to pp +sk r +si do +sch ef +ross dale +redwaver ising +q as +princes scruises +pap ier +ol v +mohe gan +levit icus +dun stan +ci e +cav uto +ðŁĺį ðŁĻĪ +w mu +st pat +sd lp +samsun gg +pu tters +paper cut +ous er +ma sin +kla us +john sen +fictional death +explore archives +esc on +edel stein +dig beth +chair men +ch ert +can elo +cali entes +bath rugby +asvpx rocky +ash mi +as sed +ys d +y aqu +un seat +un detected +twee dle +style z +solidi fy +skull candy +s loot +ren ss +my y +give away +freu dian +fluid ity +fix ie +di ren +de ers +dani ele +d ta +bry de +bla kes +ben ji +un boxed +thir sk +si phon +ra val +park uk +moham mad +mish and +major a +indist inguishable +inbetween ers +immigr ated +i watch +fro d +fav reau +digital media +cra bb +convin ces +bb age +aerop ress +wor dy +thel im +pil af +pand a +mcla in +lieb herr +her me +hart land +grou puk +e wc +compli menting +chur ra +bl unt +bhand ari +andre wr +ãģ ķ +war paint +tu tto +tom ball +spur t +rescu er +rein hold +pump rules +muscle cars +jo chen +har py +gowan us +g hai +en uf +eli an +counsell ors +carcas sonne +af ters +// / +âĿ¤ï¸ı ! +âĢĵâĢĵ âĢĵâĢĵ +shoo tin +qui pe +n inger +my girl +kof a +h ach +fer ris +dy l +choic efandom +us open +ugh h +snet terton +si as +s mat +racon teur +r iting +par roto +one u +of saa +k ich +k agu +i voted +gai den +dog training +dis sement +concor d +color ing +career advice +at ori +aro tti +woo w +votekathryn fpp +un requited +uc sd +thir tyeight +tan ka +sti pp +sear chers +schar les +sandr ingham +sac o +men sbball +jun ko +j ho +fang oria +djash ba +chip set +cap tives +biom aterials +back ing +ambro sia +ald ous +ãģ Į +âĿĦ âĿĦ +wim borne +tr icking +tl ds +q azi +melbourne cup +lord ship +k loss +inti fada +gate au +f ss +edmun d +debat able +civil war +cast leton +bb els +è° · +ãĤ ® +twi zz +tere rs +sle aford +shar mar +ru es +ran gra +pro state +porti shead +pe ga +oz una +mad hav +ino id +happy anniversary +e tten +demoneti zation +cryo therapy +corru pts +bre mbo +ban us +app are +aphrodi siac +al pe +ade t +visi o +tail wind +steeler snation +som ers +rtr naps +ra si +pilip ino +o poty +montag u +merri on +lv mh +lec ter +kan chan +fabol ous +da ad +cb g +bulbas aur +# , +ðŁİĦ ðŁİĦ +âĿ¤ï¸ı ðŁĶ¥ +âĻ¥ _ +tit os +the spi +re schedule +pin o +parroto td +my k +me athe +jou sting +hockey hallfame +hcp ss +guys ss +gri gor +dat adri +dan o +dam son +aren dt +aero postale +a jan +ðŁĶ´ # +t ars +sh ola +se vier +piero gi +pe ma +in undated +heck ler +gu ignon +escap ade +em mitt +debun ks +ann nn +y ve +williams racing +shack led +rn as +reyn ard +per ros +par fu +or op +nurser y +nol te +mac as +j inj +d áil +citi bank +chocolate y +cbs sportsnet +bott i +ðŁĩµðŁĩ Ń +trot sky +tre view +the big +shoe less +s brewing +quar rel +p ellic +longh orn +jou bert +jo yof +ill is +george harrison +g win +comp il +camp agne +beth page +b gr +í İ +zzzz zz +ty pic +sarcopha gus +pre nd +mol inari +lynn wood +luci c +house party +harbha jan +hall yday +gram pa +gos ford +gator nation +endangered species +di ke +cs v +comp action +clemen cy +ca iro +c tures +ðŁĴķ ðŁİī +v ra +us ag +se gw +nh v +negoti ators +mer yl +long island +lgb thm +irrig ated +intellig ently +humay un +har row +har dik +gul bis +gera ghty +fusel age +classi est +charlotte gshore +bar tram +ban ts +ap lin +antiqu arian +all ank +ab harat +!! âĿ¤ï¸ı +âĨĴ @ +sky new +serv itude +ri mb +ra pa +port is +on ya +need ling +magno li +kath arina +eco was +bru lee +bro o +any on +anti microbi +aller gen +wham mer +western bulldogs +star key +spar ty +rheu matology +ren dell +ph un +p out +my o +lo ol +ki yoko +icy cles +hi sham +gener ale +gag non +fitness model +dev ries +con descending +christian sen +cassi opeia +bi gart +af remo +ðŁĺĤðŁĺĤ @ +take back +stimul ant +siri sh +silic ate +rh cp +prisc illa +port ation +pic kings +ph ering +mu ppet +mo tu +lost boy +liveli fe +in ordin +grind house +col bert +ch onews +!! : +ãĥ į +âĿĦï¸ı âĽĦï¸ı +zu mb +ww u +vi bram +tra verse +squ atters +sandy hook +saf f +oper able +iraq is +instru cts +hotb ed +finger less +en ame +cul ling +cla wed +cam is +be que +back splash +apocaly p +Ŀ ¼ +sand burg +resi a +repul sive +queen su +perse polis +om ag +n elli +minor ity +me sen +li sp +la ku +hor seri +ha im +extre m +d mt +am am +ðŁ¤Ĺ ðŁ¤Ĺ +zachary levi +wis bech +ut f +rule book +mel on +ko on +kh oo +k ame +jj watt +imit ates +he ine +ha vering +elk horn +co sproject +aldub big + ¬ +wat auga +queen of +photoo f +paraphra se +mol oney +mcve igh +lap sed +kim soohyun +ker o +jennifer winget +jason derulo +go goi +fish net +fest us +e tam +den i +be eld +ðŁĶ Ń +ðŁıĨðŁıĨ ðŁıĨðŁıĨ +t mm +shar ps +richardd awkins +rev d +rag doll +north port +i was +gw ent +dun away +duff mckagan +br f +as pi +acon gress +war head +w mc +v sb +tec tonics +ta ki +ste pin +slo b +re at +ram m +race forlife +perma frost +ni kova +new age +nb cc +k hair +cy pres +college bound +bungal ows +brain health +bad rin +à¸Ńภ¢ +tu h +street scape +snick er +shoe string +seacoast online +scar l +red neck +pu ddin +post war +normal cy +mobi us +man airport +l hs +krati ka +in el +hom mage +har uki +g wr +fas d +end poverty +em path +ctv news +cho wski +agu stus +ac aci +âľį ðŁı» +tad poles +sw ane +st man +sher rod +scot ties +py m +oster ia +ned bank +ma ar +leon idas +la ssi +jeze bel +je h +inform ations +feliz lunes +commu tes +ci stern +bo car +black er +akin dele +ah oops +ðŁĴĻ ⾨ +Î ¼ +way finding +w oun +tend ons +swi ping +smi thy +side kicks +red start +ra ith +pt w +pre requisite +n ti +mitt el +kw k +hand maid +fren s +boo hoo +bal ti +arte sian +ammon ite +ðŁĴIJ ðŁĴIJ +z ena +warr nam +val do +tu pper +shot show +ru mbo +poe sia +n ha +mp loyed +lion pride +l tg +kaiz er +gru mble +fin lay +end lich +egre ts +econ dev +chlo eg +alo vel +afi b +ü e +zoo keeper +we believe +vers al +ra ked +politi k +om u +n ff +mu sky +kath ak +jack kirby +j ell +iron bridge +in ab +il se +il f +en suite +de ira +change the +blah nik +bin ny +author itarianism +add ario +ab do +wildlife crime +un productive +the shelf +sou mya +soleno id +re surface +pro geny +out fitted +ne mann +lam o +innov ative +g do +forest of +fam e +am ars +admir al +ðŁĩ·ðŁĩ ¸ +wear orange +utr gv +t mann +stur t +sm ita +sit coms +sit ara +shani atwain +rangasthal am +pe dition +lo ggins +life hacks +lan sky +it sli +info tainment +hol lander +go wer +g mat +fore casters +d ack +abre ast +a if +âľ § +wester ville +theat reco +ste yer +sp ite +sam ad +ra sk +ple bis +p vam +lar ne +koo t +kill joys +ig ital +ent z +ðŁĺŃ ðŁĴĻ +world juniors +stre aking +s worthy +s les +s du +read venture +prabhu pada +pan elling +nat ick +li anne +gre cian +condomin iums +cam as +bur dock +be m +ðŁİ Ĵ +ìĨ Į +âĻ łï¸ı +wri gley +van adium +the dead +sti val +steve z +sh ink +saint john +ren ae +pres su +p anned +mat tw +ju ssi +hill song +harrison burg +exagger ating +crum pets +ash leigh +apha sia +ach il +___ ^ +wb pictures +valentine sday +un godly +ru mble +ric he +pun x +pren der +pet shopboys +mp ong +liqu ors +lion fish +ka hani +jan esville +hom icides +gar yline +fla pping +five thirtyeight +empor io +ecker t +bo hm +tab ul +t storm +sw l +starmagic phils +sound city +sof tail +so i +sheffiel duni +re joins +perform ing +oh my +mari anne +lan yards +jan oski +ab original +⾨⾨ ⾨⾨ +us ask +te tons +spani ards +sk elli +shop aholic +post box +poly propylene +or mond +lau der +last man +kr k +f art +eli k +do ff +cli m +cat life +cas sy +af ta +whol eness +wer un +tiffany young +thai food +riot fest +re starting +pill ay +lor rie +le do +inf antino +bi fur +ali gn +ac el +( +) +ðŁĶª ðŁĶª +we hr +ste ppe +stat on +si ed +sher wood +pic ar +palom ino +mp w +me her +mack y +lati sm +home wares +fre und +fin ner +false hood +ero ses +er ster +encapsul ated +en tra +ec am +brown stone +brain tu +bed and +band b +bal ven +ðŁĺª ðŁĺª +veri fying +sto sur +sp leen +scoun ty +ready tor +pe aty +pan tages +pal it +museum selfie +milit arized +ly le +ll sif +gr annies +garyline ker +ed g +ber ne +w engen +toy in +tin en +sky view +r mc +n oooooo +lib spill +leyton stone +jama is +imper man +im min +hall yu +gal es +f si +de ye +bra id +ber ths +bar z +bake house +b ttf +av illa +ðŁ¦ Ĭ +wizar do +thegreat khalid +south ie +pur ging +p ago +mu mble +mo co +mand zukic +kat v +jay araj +gav inde +fore hand +el aide +distill eries +ani el +ali enable +al cal +ak kar +advis able +unil ag +sial kot +schro der +sc or +pe ws +nh p +mon is +md anderson +les bos +kasab ian +ink l +heart strings +freder ic +eh y +drop ship +bian ca +adhe sion +vor one +tumb lers +t reading +poly carbonate +papadop oulos +on this +mer cia +ludo gore +koo ky +klu ber +he mato +gar on +depo ts +dan son +bo seman +ac q +ðŁĺį ðŁĴŀ +å Ĭ +women leaders +wi est +okav ango +mathemat ically +mar isol +jack al +gum by +el az +du is +brown university +biaf rans +ban go +wn cn +w ily +us m +um h +thra wn +sath yaraj +ricken backer +prox ima +por ches +over seer +meri den +ma jum +lt fc +leg ge +kir ke +king z +har low +cor nette +birthday y +answ all +time zone +smart contracts +si do +ro day +mendi ola +hou ma +gu ang +gran dio +dil aik +contradic ts +cardi al +cad rought +breakfast club +* ( +âľ Ŀ +âĺ ĥ +trin it +tom ato +six ty +refu tes +phant asy +perpetu ate +ol c +ny cosmos +needle point +milan ese +goog leglass +gold stone +fle tt +ev ar +de kh +cas ings +bic ic +bi ddle +at ay +ar z +ar rl +ä¾ ¡ +virgin ian +team followback +span thers +siyak eram +shu g +prince sse +po em +mu ka +metro logy +major crimes +la res +la byu +ki ffin +kar o +kah lil +gay pride +g app +fire base +every town +e su +cust exp +af faires +ðŁĴĶ ðŁĺŃ +sec toral +prod mgmt +omni um +lindsey stirling +ki pper +gar rix +freel ance +explo res +climate emergency +bu rak +b com +av eni +air fix +x jr +wan ton +un sw +tur kiye +teacher appreciationweek +sar ay +pay ment +param us +neuro degenerative +kumar vishwas +inter nets +fri gi +dy nia +did cot +de formation +asset store +antibiotic resistance +wa ver +vel ocity +ted by +tat tered +suz ette +stom per +sm outh +rs g +plant ings +ott olen +mel low +life and +lad bach +kat es +infl ate +head in +he ung +fr inges +food banks +depreci ation +chipmun ks +bro ski +ale ister +ac ito +ëĿ¼ ìĿ´ +tin ction +taf rica +sau ro +rio ters +raila odinga +queu ed +out stretched +one time +ni y +leg olas +jun ky +fo il +du as +dah mer +cell ent +bull er +bt posse +as ket +un cc +snow bird +rhin oplasty +oro ad +mala hide +lu ma +la four +king wood +kentucky weather +jun hoe +inter planetary +har ada +fla ppy +ek g +di fc +cool pix +char ade +bl ant +vene zi +sw m +sen ko +samsungg alax +run yon +party poker +parmigi ano +moder ators +me ac +lu sso +live chonews +ken nard +ig in +h mo +fren chart +exxx otica +do err +; ))) +" < +ðŁĺĺ ðŁĴĭ +ó s +worm wood +with hold +vell ore +stan cia +r ma +phil ae +mocking jay +mag en +luke cage +kur d +hearing loss +gau ri +e spor +den ounced +clean sed +cay ce +cash el +boo ing +athen aeum +art station +ais dproud +a qi +ðŁĴľ @ +v liet +tx s +tamaram c +spin ks +small wood +si th +severe weather +ny sph +morning motivation +for lorn +car ino +bul len +b bott +ðŁĴ Ĵ +ste go +smith son +res se +par ise +levit ating +haw ick +go bert +fl s +cor ding +bu ell +bbce astenders +arctic monkeys +angel us +ðŁĶ´âļªï¸ı ðŁĶµ +win stead +vor acious +up coming +tn hs +sof london +on me +o rec +munch kins +li x +kookab urra +hyper car +he sh +gow rie +gen es +film works +dev illiers +daily deals +co pilot +bad gley +alex andro +agr arian +worshi per +vor tex +up loader +tribe chat +tex ash +su che +r dm +o possum +hal ve +fer mi +e bt +der on +chee ky +andre ss +zin da +yard ley +whit acre +u os +twith aca +trophy hunting +sy a +spa ghet +pixels website +ox for +newbury port +mier coles +get ready +flor in +ev ely +city con +argent in +åĭĿ è² +tsar naev +tati an +symb ol +spar c +sou ffle +skid more +sh restha +ru pan +rehabil itate +ratche ts +pp age +pla za +pb x +op ark +ma ille +lilli es +le be +lari at +kapam ily +aaaa aaaa +)) )))) +ì¡ ° +wee i +vibr ator +ta kan +scifi art +les mis +kb tribechat +jant ar +etru scan +esc alante +edu chat +dy che +di shoom +bal once +ak ye +! ðŁĴĸ +writers fest +timi glia +time less +thi op +syn apse +sop ho +sh ula +repu di +pu rab +ori ver +of ford +monster jam +m ph +le mak +incumb ents +har dee +ety mology +eey ore +du du +do van +cou leurs +con served +codenew bie +co ton +cau sal +audubon society +alle gra +al eph +ðŁİ¤ ðŁİ¶ +worl demo +west country +touri st +tost ada +ran khan +plat te +ou el +nz herald +nu a +nouvel le +no th +new by +mo salah +kidnapp er +he morrho +har lingen +gat o +ent ric +dot com +color less +chir u +an kar +ê Ĵ +vijay antony +tt ura +senate dems +mh sm +ja une +hoo tie +han e +friend liness +fre shair +formul ations +fit ri +dr do +dee pa +c int +c illo +bathro be +. | +ಠł +y ello +white man +rust ling +per ley +mush kil +lead ville +just ina +j rm +ira ins +gro an +fu ton +de tt +damn day +d mac +ca hu +black foot +apar ks +af lock +adun ham +ðŁĴĽ âĿ¤ï¸ı +zi pped +sn avy +qu ater +mus c +mal ai +ing am +gi let +ge et +el ich +crow borough +chan woo +bobi wine +a ali +ðŁĺį ðŁĴĹ +with it +win ec +weing arten +website design +vo v +ut tox +sequ encer +sc su +save children +sat yam +sa eng +s ace +ri be +pos sums +non ton +instag rams +h da +dw drums +commercial realestate +auth oring +apple tv +ðŁ¤ŀ ðŁı» +zwe ig +wyan dotte +u bu +the dj +silve stri +sil t +pre pper +pay less +pax west +log itech +lec tured +l tw +kari sh +j dk +h tf +cheng ladbach +anup ama +a field +ðŁĴĸðŁĴĸ ðŁĴĸðŁĴĸ +v illian +tw ars +pro pped +min den +lay am +lady antebellum +ktb ffh +ha iga +e www +delic atessen +chill ico +broad moor +anthropom orphic +ale ks +sef ton +ron nies +plant sci +per ini +pay ton +orchar d +m pm +k ym +jan ovic +golden boy +f ana +dynamite comics +dorje shugden +cr inge +co ombe +ation of +activ a +ç · +you are +ren zo +man spla +kelving rove +kat ara +ham den +furn aces +favor ably +f wp +coch on +Í¡ ° +zak kw +wealth management +touch stone +resi stance +ras ool +private jet +officialu om +lc f +kit ch +keto sis +it alie +hy enas +grati fying +futur ity +b di +ðŁIJ ĩ +ëłĪ ëĵľë +vol ine +tiger s +tamaramc cleary +potter more +on elove +mc fly +m ffl +libr aries +j zarif +irish bizparty +hy thm +hon a +hail state +gal le +fail ings +brand is +br ons +w ry +ve sper +uch el +ta is +spro te +ser ato +save water +re des +psyched elic +miss guided +lin na +la vie +gui dic +green span +gor ing +gold water +fi bau +fan mily +dro go +cr c +chee to +bo son +yu ka +wnu krt +web master +state farm +sp ero +ony mus +mel aye +lou i +i fm +engro ssing +empirestate bldg +dissol ving +boo tle +blue bonnet +b mr +thr ace +ps ch +pic sart +per fom +nuf field +nar rate +may u +l tu +elder berry +dibu jo +desc ence +buoy ancy +bay lee +bald ness +ar ish +ani ght +aj hl +activ ator +z hong +vorone zh +sing a +road trip +property management +ple tt +pin inf +multic hannel +mill sap +male model +ma shaba +lubr ication +loc ate +kin nick +kid z +fe h +defund pp +dd icombe +cu tch +chun gha +bard stown +> $ +year sand +veu ve +unit edin +p ire +oxi dized +or ada +mug ler +mom sen +matur ation +kol be +gl engar +father sday +farm workers +family planning +fact sheet +dilla shaw +digic el +c whl +c sur +c ite +and read +* ^* +ðŁĵ ħ +ã Ħ +âŃIJâŃIJ âŃIJâŃIJâŃIJ +photo contest +n ü +lat te +kwaz ulu +ici ou +hy lton +ghost town +funkad elic +for theday +fen nec +eviden ced +do ce +dige sted +cor sten +back drops +un said +supermari oodyssey +retro spec +oligar ch +mispr on +la as +il logical +galli ano +flash fiction +clou dera +cab oose +brian mb +bon bon +ac ep +ðŁ¥ ī +âĻ¡âĻ¡ âĻ¡âĻ¡ +thunder cat +smoo thest +sing e +si eve +sa ko +par co +muse elou +miy uki +mag gots +led better +it ous +integr ator +grow with +gir t +fal a +day an +arden nes +ag ence +trias sic +samuell jackson +painter ly +nar d +mcm comiccon +mari anas +mahon ing +i est +i ba +hills borough +greg gutfeld +genealo gical +g ys +em pi +de spi +bla ire +anciente gypt +alo vers +ac lu +ä» Ĭ +оР³ +tt x +sy ke +q ai +pad ron +ma her +it so +fa wr +de ee +carmel ite +# " +wo i +urban isation +theli br +s daily +pri ve +my ers +k vm +ic hand +fol au +ext inguish +edger ton +daf oe +aro va +âĿ¤ ðŁĴĭ +º c +yy carts +seab ed +pal acios +me ik +mck ay +mb ti +ma stin +lu pe +love u +le ero +lc ps +ine quity +fastand furious +bund aberg +bu do +ap ort +ðŁĺį ðŁijı +willi enelson +ut el +tri force +teas dale +spell ing +shi h +san down +mer sin +kh at +incit ement +ho smer +ha ier +follow me +do ggy +deer hunting +day sun +d no +christian kane +candel abra +*__ * +te gan +slu mber +scrib bling +ran di +plat ini +n sl +mo relia +mel amine +le ee +kash ima +h nl +box y +audi olaunch +ad ro += ( +yahoo finance +sho ai +out smar +men ding +ma ks +ki ing +jami ro +gn l +gio vani +gain ers +ent rap +duplic ated +dublin town +dar lo +c sh +breastcancer awarenessmonth +baff les +åĭĿè² ł +un justly +ste mmed +smol dering +pd fs +men do +mcdon nell +lat on +lar yn +ing field +hell eni +head z +fron ting +e wen +dur rant +cas s +af oto +âľĮï¸ı # +w cm +verton ghen +ve ils +ste wed +span k +sen ews +selec tively +san usi +retin ol +re build +prou do +pene tra +oo pam +nar c +millenni al +m clo +kat z +inter ned +hot docs +for honor +fitt ingly +fan gio +fal lo +egyp t +constitu tion +chri stos +$ âĢ¦ +Î µ +watch making +wat an +verte brate +sand a +nu mba +mikethe miz +mans bestfriend +leedsr hinos +le sm +kal an +in ah +b han +admini stering +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +ঠ¤ +غر د +z ot +sup streamers +shab by +pachel bel +mer k +ki da +inte ger +hear ne +goodwoo drevival +cro oner +cl ings +ðŁ¤· ðŁı¼âĢįâĻĤï¸ı +t kach +soo ke +sony pictures +ros acea +psycho logically +pl srt +ligh twood +lib ations +j ony +home style +geophy sics +disco verer +des se +bu ie +bbc tennis +bar ri +astoni shingly +arre ars +and drive +ad vices +x fl +weightloss journey +tyne castle +tran k +sweep stake +sw wap +russ west +la ure +icant breathe +hol len +ha ptic +go ty +fri zz +bay o +b ich +ar au +âĶģâĶģ âĶģâĶģ +vol tron +vis cose +un scripted +twe ens +tor res +tak umi +swansea uni +stav ros +rold an +rev ents +re da +rc gp +ng ong +meteoro logists +marl ene +kil ns +janel lemon +induc t +hur ler +hok age +hend rickson +happy customer +gow yo +dag h +d med +bron ner +trans figuration +topo graphic +the doors +shang ha +sam mam +sa wn +rom ita +re pub +pho logy +pe que +open gov +mul i +laz are +hon cho +ga bel +dun leavy +disembo died +as pa +aqu es +amitabh bachchan +ad dle +ðŁĵ ¹ +âĿ¤ï¸ı " +vik toria +van zant +sre sorts +spoon bill +sp az +sho sp +shau ghnessy +mdanderson news +making comics +kal amata +gri sly +gr g +go g +encroach ment +en ate +dw t +di sch +cu ra +buffo on +bi polar +b inging +ago stino +yo ssi +valent e +tram lines +tal lies +sydney is +sun dress +sci atica +may on +mali k +itt t +har ri +fr anny +drumand bass +crypto currency +community policing +com po +chan i +buch holz +ba hu +apex legends +and l +an aya +ðŁĮĬ ðŁĮĬ +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +your shot +swe e +summon er +sul king +spy ware +si stance +ny eri +mon om +mar lena +m nufc +ky ler +hol z +gre alish +far han +em outh +dil ma +chenna irains +cam ren +axx ess +amy loid +wa aaay +us man +sparrow hawk +porth cawl +pad man +nysph saa +new por +n mnh +n dr +march on +magh rib +mac ie +indv spak +gar son +g and +f anny +british gt +bit o +bam berg +asiap acific +al w +al omar +è ¦ +zu ber +sausal ito +ro ath +po table +monit or +mk ts +mis searth +me ity +har mer +gin or +ei ji +charle magne +bere ts +bel tel +apo gee +an two +am cham +________ __ +stead fast +rot ana +iw gp +im b +disco s +contrac tions +cal listo +be great +an w +a eng +௠Ĩ +walking dead +th ener +rem and +mom entos +m clu +lou lou +kacey musgraves +hol ler +fh saa +det tori +de es +cu boulder +arra ign +ach icago += $ +ðŁIJ Ĺ +ðŁı¾ âĢį +zun iga +widow maker +tz at +sportsp lex +sla w +shi pin +re booted +pr news +kha kis +ka se +gam al +fro ck +da inik +bird seye +tat yana +sw ac +ste emit +mc x +ho tri +fu ries +car nes +bassad or +at ka +an mol +ak g +価 æł¼ +zy deco +z ouk +yessi r +y ck +ty cho +tot ame +tor u +too tough +syl vanian +rat i +nswr l +n gad +martin dale +m wd +legu mes +kha ya +ho je +gan net +eco x +aj s +ðŁķ ¸ +wanderlust wednesday +w ym +tech nik +sri kanth +soul calibur +sof india +ring wood +read more +psychedel ics +oak ridge +na dir +mlb tv +kareen ak +ishi kawa +climate justice +at test +abstin ence +wal low +t pi +sel e +scissor hands +s do +po der +paddy power +next level +kid der +ker alam +he ber +for this +ban ton +b him +am ali +al gore +trafal gar +thir ties +swan se +rol fe +rang oli +ke z +ja wn +go etz +glo bular +face timed +fabi ano +fab io +e cht +cu ticle +chloeg moretz +car io +blo s +bap s +art by +ale em +zi ed +whatyou do +wel ders +water ville +vt poli +tourof britain +tight ness +ten able +some on +sar g +pre sh +me gg +iow ans +french tech +ei hl +death core +da ichi +co com +chan ak +b ga +at eness +a op +( ^_ +ÙĪ ر +yp sil +tat su +taor mina +stradi varius +son ali +pk p +pil lage +or it +he wett +dug dale +chi pper +busk ers +blanken ship +tt ner +sub tweet +sh eu +season ings +ni thi +mn m +l sk +kay den +favor ita +cu touts +coloni ze +boat sforsale +ðŁĵ ¦ +ðŁ¦ ĸ +y ll +y hm +wit tenberg +t cea +steve ston +sci rocco +sal ud +portad own +n pas +lo dhi +ke hoe +k anda +jo en +ivan ov +han h +ge f +anc s +ðŁĴļ # +ye garts +vege mite +ri ofer +ri go +mb fw +mar ki +magnific ently +li steria +kar zai +dar ragh +d prin +chu ka +car as +c ys +be w +ast o +æľ ¬ +tan zan +search engine +ross on +prab hat +pan tene +noti fying +mo dig +micro scopes +lak elife +j hene +fre w +dom o +christmas gift +cc su +cas sie +anc ini +² ¨ë² +yo ffice +tr iti +the street +st aw +set zer +re doing +pun ts +on slow +nv me +night gown +narco tic +n its +hb cus +gi el +farm stead +ding ton +ct vo +cinnam on +chatta hoo +beren ice +audu sd +æľ¬ åĭĿè²ł +worldemo jiday +vie ques +truth fully +tru bisky +to ss +t auto +st event +shu ll +s ff +pol arity +off ing +must do +mc cand +magni fied +jor am +gab or +f ld +danny wood +c mm +aurobin do +wood head +women ssoccer +the state +hi roy +en suing +ac lan +w aga +u wp +u rian +tar u +swer ving +stou ff +shi mizu +pun ite +psychop ath +pap aro +n sca +moisturi ser +khand el +fu gel +fro th +cocon uto +ca in +bil al +x v +tootough totame +succin ct +scot twalker +ray mon +pom o +pluton ium +phen yl +nd f +music inc +marble head +leg it +l si +kan wal +jour n +jesu sis +gall antry +freeform tv +fo aming +fish burne +crink le +bur a +blu em +bla q +ak wyz +adap ts +a ip +trevor moran +ten i +sl ate +scuf gaming +re printed +quad ro +polar vortex +ot ol +marou ane +m fr +is this +har ker +fra zetta +fon daz +ed c +dc traffic +cor ky +cal las +bulgar i +x pro +write up +wat che +w ds +veterin arians +tuil eries +toy photography +th apa +staf fy +sha i +seng illibrand +pand an +oo ts +grocer y +goog les +fitch burg +eye on +dre es +di zation +di eng +de vise +d sl +book keeper +bean bag +aer ts +² ï¸ı +take da +stre as +sal lah +royal mail +memor izing +mar lee +m sport +inner peace +i wc +ge en +fighto wen +dra iders +de spon +chou dhury +can tar +al ola +ðŁİħ ðŁı¼ +west indies +wan stead +univ of +smi thers +sel ene +rose bank +red necks +na hyan +l pu +invasi ons +independence day +har bin +gri ddle +eri ot +ell wood +dul ous +cover reveal +bel mond +beach ed +architecture photography +al stom +¦ Ī +un rated +tu u +tri met +simon pegg +remou lade +prover bial +over berg +normal ity +mor ticia +kal ra +fuer za +fightowen sfight +diver sions +ðŁĮ¹ @ +ú l +vo igt +tal on +scu ri +saw tooth +pen ning +n sm +medi alab +m ty +elisa betta +eer ste +cute sy +ceru lean +adap tability +âľ ´ï¸ı +tumble weed +thi en +tex arkana +tar ch +t fb +st aked +seduc tive +rogu elike +oste opathy +os good +eye patch +ami x +> # +ðŁĺĤ ðŁĺģ +ðŁ§ Ł +tan ager +sun way +poly com +jar musch +j anta +iber o +gian luigi +er win +dit ty +compre ssing +brae head +bo cca +ap hor +ac ey +yeah that +wa il +vap ers +tyrann ical +st augustine +saf ari +raj ni +min to +medi acom +me ander +human ly +ga illard +fro om +dapp led +cum bia +cro co +bol le +bi ght +an itta +a stru +us ana +twit ty +tun ers +pe h +no es +ni mr +ll f +jay cee +it en +ik is +hur led +et r +e pics +dro id +dis land +car play +briel arson +ar w +anti septic +ai do +Ï Ģ +thelance t +thel ong +th ongs +tart an +sper fect +sli ghts +ser ling +prie to +mu sso +khal ed +kareenak apoor +in uk +fe res +fan atical +boy y +ban jar +( âī§ +ãħ İ +wim p +ti gris +ta ye +propag ate +pl it +pin ar +ph vote +parab ens +on ces +moo i +mark down +m fi +li fen +ko smos +ka v +jhene aiko +hello bc +ha qq +girar deau +fe ster +el gin +de q +de ena +ar agon +amazing race +ëłĪëĵľë ²¨ë² +wu thering +whid bey +tam pico +sh ami +ri day +out landish +itiner aries +gu ha +e katerina +cul vert +chronic ling +car dona +because itsthe +reci di +re evaluate +pal u +m fp +lun enburg +kr one +hyper bole +hydro carbon +fr n +fir kin +def er +bowhun ting +bird gang +ðŁij¸ ðŁı» +ठ¼ +write tip +tv shows +sag arika +ray sup +l so +hic cup +ha pe +gu len +din wid +but lers +brock ville +bang er +b lec +ad h +ðŁĺ Ł +ðŁijĮðŁı» ðŁijĮðŁı» +wf sb +we ft +twer p +stri ping +scrupul ous +sab in +ros lyn +rev ving +rack ham +ra ph +mp as +m hi +kor ver +kar k +jewel led +heath cote +dv m +duma guete +door ways +detroit become +cru soe +connoisse urs +c we +brit tain +blood pressure +blemi shes +repent ant +pi ana +otta wac +open follow +ndam endment +la homa +kicau promo +ju ste +invis align +inthe house +indi anews +ibg drgn +gri gg +gene seo +force awakens +bode gas +ayk royd +âĦ¢ . +ye eee +ulta beauty +u ol +tomas z +ter berg +tam ina +stepp en +sam m +q g +practic um +oro x +kri stie +inter governmental +in ce +g anna +dom ino +di yar +dar ter +cr amp +cad aver +bru beck +bo tu +yin ka +spaul ding +shun ter +shir di +sam ia +recidi vism +r to +pi quet +kis i +karli ek +jordan spieth +ip sos +dr x +datsy uk +cute eee +critic schoice +bernan ke +as inan +ak ery +soren son +nucle o +ne ice +mis read +max is +le am +ka ise +gur r +fre sca +cat oftheday +birk dale +wi h +w ct +vi aje +vere in +spice girls +soccer am +sar am +reflex es +kam insky +gu tta +gli ano +dick head +depu ty +as r +air con +yu val +sec gen +q af +ob in +nv wx +mellen camp +kenne dys +k pm +j ini +i mas +ent wined +end en +dyspla sia +dl m +book love +bon appetit +wahe guru +vacuum ing +tt ler +tra ut +sun moon +ro sado +new stalk +ko bani +janellemon ae +ine os +houston chron +historic england +geo technical +gag ging +g tg +do ren +cla ses +as ai +ap ride +super valu +stro ker +si mm +scar le +quo it +morbi dity +lam ma +jo anie +finger board +ethnic ally +eson oma +el ab +dou bly +attor ney +ðŁij ´ +viol ent +multim illion +l pr +k norr +fac toid +dou la +de valuation +de ker +aw allace +ðŁĩµ ðŁĩª +year sold +vol up +ver lag +tar avi +stre eter +sh bo +sclero derma +rabb in +pro x +mary kay +lic ht +l pn +ka as +j ame +haz al +gru ppe +coun table +bri gg +ah in +way a +walk for +vi dic +the dan +tar nished +stand for +semic olon +phy sic +naz im +n cb +mun ds +kre uz +i league +hy m +gli des +fees mustfall +dr ano +dis av +daw lish +classic car +ch olo +w tg +tax day +spec ks +sen feinstein +salom on +pe ppy +par kh +new ssydney +mul k +liber alis +le apt +ke di +heav ing +grand pas +gno sis +fiftyshades freed +edg ware +digiti ze +cel t +carnou stie +aa v +âĻ¥ @ +world photographyday +tack le +swane poel +sound scapes +sole watch +rise ofthe +pe dan +oregon standoff +newyear sday +mir th +kratika only +kil la +je ann +gir ly +gate ways +gall eries +et now +eli ana +dark seid +cur bed +chim amanda +bit finex +bell atrix +and company +an engineer +a beer +y fn +vit or +vag rant +su u +ryerson u +retrac ement +pal az +mtvlakpop bts +im l +gour mand +dim sum +dah lias +as li +ari sti +areth afranklin +am alie +ac r +wini fred +ven ky +uni q +un known +over done +nofilter needed +mu mm +mark sman +lin ares +land care +kel lie +ferran te +eng i +cigar chairman +y k +show grounds +sho ts +seren ades +sal inger +roman ov +resi dues +refre shes +prop els +news mornings +n mp +monte bello +mat sui +make money +lady bugs +inter scope +house plant +fugel sang +fu hrman +du que +dil ute +co ss +c atie +bar stow +abu ela +a beats +wut ang +warrior nation +v tc +uk an +trans vest +super stock +su won +su ll +sto cker +stgeorge sday +r pl +pin head +kul u +kareenakapoor khan +je wala +i ffe +heck en +glamour mag +gan ay +g ous +et an +el ric +disper sion +coch lear +bb p +analy tic +amal gam +af cf +_ @ +å· Ŀ +ud all +opioid crisis +neel am +mo j +methan ol +lang s +ka oh +iv re +ho z +he bri +ash etty +as port +arson ist +.. : +zi ja +trac ted +solemn ity +samo yed +rapha el +premon ition +pa ver +mis demean +karliek loss +jor g +hann i +ga unt +fork lifts +ee en +death wish +cw p +celebrity news +car ty +brook wood +ae gon +we bbed +polit o +ph ari +miami oh +mel le +lind gren +kath arine +ja qu +it down +i em +gradu ate +gluta thione +gardent ags +fundament alism +eu l +ck less +chi ri +br w +bla k +bayley wwe +a pres +ðŁİĤ ðŁİģ +ìĸ ij +un limited +super fund +sun id +rough ness +pam pa +on em +nong mo +ick son +hospit alization +geophy sical +fra ge +folk life +fo ley +fi z +fa wales +ek h +din ar +cc ps +ben ard +ar up +zee bo +var da +tou ting +space suit +so aks +sm rt +si gil +sam aj +potam us +mi f +ludogore ts +lu z +kit ti +jacqu ie +io logy +har tz +ex hu +estor il +duchess of +cake decorating +animal rescue +adobe summit +yo ver +work house +wil lo +stand point +rockef eller +renss elaer +pres stitutes +n ene +maison valentino +ken to +hail storm +fre era +disapp rove +co bourg +ar q +and ru +ê³ µ +Ï Ĥ +sel wyn +ocean view +i book +donat elife +dani ell +da hiya +cas sin +be em +wwed aniel +wwedaniel bryan +ta pen +stan den +rainbowsix siege +pan ip +noir alley +new brunswick +ja ir +ill ation +d ous +brad bery +bar kha +ane jo +ag ir +winter hawks +thyro id +out did +ni vin +museelou vre +lalun asan +inhi bits +anti gon +amazon kindle +un tested +thunder struck +stor ks +sp age +run the +refur bish +qu ed +oh y +law and +l me +its bayleywwe +ibc show +gr ü +dr ams +denomin ational +b fm +ale o +ðŁĴ Ĭ +ðŁİĦ # +ãĤ¹ãĥ Ī +wn e +who dey +un credited +tu va +syngent a +su ther +sax on +resu mption +po gi +pen iel +on wheels +magi karp +i ñ +harid war +funinthe philippines +frame less +fer vent +eros now +eradic ated +doo dy +di av +ce ci +br il +ber ge +are i +ard or +ar sal +ani ma +alton brown +ðŁ¤ĺ ðŁı¾ +wil ke +vi aggio +vas il +un sold +studi op +speed ometer +ment ly +lli ott +jan ata +go bucs +gar ay +g tp +fract als +et p +en bach +ci do +be bes +arc teryx +ðŁĴ ı +wha aat +ventrilo quist +vel azquez +un likely +umich football +sober ano +sideby side +sick kids +r bb +na hh +mustsee iran +mam ta +luc ina +ll in +li pped +j da +innis free +hb p +franch ised +fen nell +ever deen +deci bel +chom ping +cau s +bon illa +al pur +âĻ¥âĻ¥ âĻ¥âĻ¥âĻ¥ +voucher codes +ree ks +rand l +pot ash +north yorkshire +n sb +limon cello +lah ti +la stest +ke o +hat rick +freera if +feliz martes +extremer ules +contracep tives +cam illa +bru ch +bri bing +........ ......... +⾨ âĿ¤ï¸ı +whe c +spectro meter +rter adio +pe stle +ma am +k war +je mison +jai mie +j mm +iko yi +hav ant +gu tting +dis banded +compe l +but tocks +at birth +am ann +su su +sk en +re pos +petal ing +pechak ucha +ordnance survey +mis spelling +kit ak +ig uri +hal ting +gou ging +evis cer +es and +dept vetaffairs +con man +city v +boer ne +blooming dale +b md +astron au +ðŁijħ ðŁijħ +wwe hof +win ski +vg punite +ur b +un usable +u pan +lar gs +hf player +gri p +god ley +gad ing +co whide +weigh ting +ste gen +st ling +somerse thouse +sh um +sa kai +s é +pic h +mul o +lt gov +lan gui +kh q +juli o +j aga +he ep +fig ma +epide mics +daman sara +cold well +chro matin +chic hester +arbor ist +amo vement +agent carter +ìļ ´ +âĹ Ģ +Ï Ģ +tzat ziki +tas ker +in appropriately +host ilities +gar r +epic a +disc world +cra ps +christi ane +caf és +tipp ett +team b +strou dsburg +rose water +pu ig +n psl +mick jagger +lik able +ig inal +ic ata +huawe ip +guid ing +fil my +fer c +fearthe deer +embelli shments +cho e +ch il +blue book +big band +am sey +alli ster +\ _ +the sam +steveaustin bsr +spiderman homecoming +so bie +rich field +pr ato +parac ord +o skar +non profit +lar val +im at +daniel andrewsmp +cly des +blen heim +ban ne +avin ash +alad ha +ðŁĵ¢ ðŁĵ¢ +âĦ¢ , +zombies quad +wor f +vas sil +sc ada +profan e +over t +ny man +man zoor +li vers +ko taku +kin go +glit zy +fre igh +equi valence +env agency +en sen +en liven +den ouncing +crest view +cl ere +city beer +chanc ery +atle tic +ag ang +wra paround +v italy +to remember +stun ted +slap stick +se phar +op w +newyear new +may ak +li ev +ker sh +fou st +ef g +bur ge +au vergne +ðŁı ¹ +ëłĪëĵľë²¨ë² ³ +xen omorph +uk iss +to ic +rifle man +phal lic +per turb +leg ac +isu ke +hyper baric +hall er +elg ort +don an +con fox +car ron +cab lecar +baby daddy +aby tes +wh nt +war zone +treasu ries +swal e +scal zi +pes ach +p ous +norman ikor +montal cino +mid gets +kourtney kardash +jo dre +hy person +hi di +her by +har ra +guj ran +gu ppies +family first +f ach +con ical +ch ace +ar mie +ид е +soli dly +shay na +segw it +rough necks +may im +macro economic +ler oux +len adunham +k tg +in z +hend rick +gui mar +furry fandom +dest itute +z aire +sun fish +ro za +rac esonoma +r po +not withstanding +is chool +hydrange as +ham ra +generale lectric +fon te +fe w +fairy house +comp tia +bet as +bed head +b drm +ag ini +aaron paul +ðŁļ © +ðŁĶ¥ âĿ¤ï¸ı +ðŁĶ º +ॠIJ +w golf +te ater +ren ter +re so +qu are +parti ality +lac quered +krasno dar +go cu +dyna sties +dublin city +cle ef +ban try +advent u +abhi gya +a hai +) / +ìł ķ +yarmou k +wie der +wal est +u zo +tetsu ya +success factors +schwar tzman +ne bo +me ttle +mc man +hol kham +gandhin agar +esp nc +e tro +bo st +bel u +av ari +ðŁij ľ +zombiesquad hq +wal dman +wa cha +v tv +sh hhhh +russi a +ra itt +prest wich +normanikor dei +mar zano +lu as +liqu or +lig and +k ck +k anna +insta video +gul shan +ge oc +fright en +dr tedros +die mannschaft +cheese board +biom imic +an ze +ag oo +vi b +tu le +tele pathy +sd j +raaj jain +humber to +go ku +fourfour two +den berry +co weta +bur go +albin ism +ac án +vasund har +to bin +sol ange +sof tens +shadow run +pro mis +pre fix +paralle l +p fd +more ira +larcen y +kann ad +j cw +hl m +her ne +grand in +gi deon +g lynd +fertili zation +fc w +dun woody +deserve dly +by ung +bar king +ar har +... $ +wu shu +vital i +too ooo +s theatre +ren ditions +pto sis +par ley +o ja +na x +li vi +jetair ways +indian railways +hand washing +guidic elli +el ottery +che sters +bern ini +b cr +art fully +al mere +ab leness +zimmer n +westand together +new vision +new belgium +mit zi +lux ur +luc cio +ku mble +is aw +he sa +gom vfc +gh today +french montana +cro sley +ce b +caro winds +bal do +b ps +adder all +w gm +ve era +vai o +taur anga +ta shi +spi king +sch winn +or ica +of o +jay am +im patiently +final sweek +fe thi +fa wsl +cu pe +cla shing +brett eldredge +bookof mormon +ap al +ac tress +ac amp +>>>>>>>> >>>>>>>> +âĺĢï¸ı ðŁĮ´ +ื à¹ī +vest ment +swi pes +step mom +ste dman +sr n +sa warenessmonth +redchillies ent +pri zm +pal ais +nd football +lea ver +in paris +i pos +give blood +else vier +el lus +de ann +clu ck +ber wyn +!!!!!!!! !!!!!!! +view fromthe +un founded +to da +tin ge +ta itt +od ors +mu di +money ball +max ie +herpe tology +gad i +fortw ayne +drey fuss +dissatis faction +care rs +adel ante +âľĤ ï¸ı +è ge +ten ement +ssss ssss +snor lax +sc ymru +personal development +ma dero +kor d +ki shi +it re +implo sion +fu x +eque stria +ei vissa +chic ha +å ¨ +Ä ĵ +tu bers +tin a +the ads +tamagot chi +sy k +sun n +sd ar +sammam ish +rene ga +prun us +ponti f +pier rot +oak mont +kur i +joo heon +howard stern +he o +ge stapo +gas kets +gab bard +conven er +brazz ers +bol ly +bajrangi bhaijaan +" :" +ðŁij · +wi dened +watch maker +vit ru +un ico +rough ing +poly phen +m the +led by +in tv +h vy +end is +cumber land +cn traveler +choreo graphers +chan te +bar nier +bar di +ë n +to ting +pa stry +o hi +mis eric +m venkaiahnaidu +kum amoto +kis sme +kin es +iq rar +h gh +envisi oning +c acci +bla ss +beauty tips +ather osc +x cel +the standard +sou q +ram er +pregn ancy +periscope co +n scc +mariok art +lass ics +ko shi +k itu +flo yds +er ma +endor phins +e aux +de compression +ðŁİī ðŁĺĺ +ðŁħ ±ï¸ı +unfa zed +te ak +set ti +má laga +mur ch +let go +is n +is awesome +inf lection +he cht +g bc +fra pp +combu stible +cali ph +black mirror +at last +ðŁĻĦ ðŁĺĤ +view finder +us af +un ta +tommy hilfiger +so wer +rug ged +nav in +lu ks +loyal ty +hs football +harry and +gal eria +fic titious +fair hope +croche t +al ver +ðŁĻĥ ðŁĻĥ +wf mu +tel enor +tai wo +sb w +sam pauk +real saltlake +reac tionary +progre ssions +makeit count +is om +inst an +ib h +holo grams +ham pered +game grumps +dole zal +di wali +cutie pie +ch ung +cal low +be coming +ag irl +ae gon +welsh man +wel ove +weap onized +slaugh tering +repre ssive +q m +optic ian +milk man +kazoo ie +indemn ity +imit ated +form aldehyde +chlamy dia +car radine +bur gun +am ah +è į +اÙĦ ÙĦ +ú s +vent oux +thel aw +the bridge +swed u +sir pat +s medley +nov ate +nectar ine +muir head +mam adou +ii um +glend al +fon ten +eas ley +cre sts +col wyn +chelse ac +cart wheel +bead work +a jac +z official +un seasonably +t singh +sul fide +steam ers +ra uf +predic tors +phe v +lg c +imo gen +ic chio +hallucin ations +dev ry +crow bar +bri sco +bonafi de +ball ondor +b wv +b hr +att as +war horse +v ss +un due +teletub bies +state less +pom o +ohio state +offici ally +no ddy +n rel +mu layam +mix radio +indigenou speople +hu q +golds miths +fin ales +f ti +dra ven +corn eli +architecture mw +å ĥ +âļ ł +wi j +vietnam war +th acker +sub set +sk g +pun x +pre made +ox bow +op cw +mo ba +man an +mal lee +kra kat +inf on +hu tto +en car +ann eli +aer c +wick i +un sweetened +tar dy +shil ton +q ts +per ri +on ite +official corkgaa +nf hs +lit mus +lan ie +kul ture +kend al +ja emin +is not +innovate uk +gran ite +gavinde graw +for far +diss enting +couple goals +br ano +bill ingham +bi ffy +whitney museum +wh d +vadachen nai +tin ned +si bility +sea ver +re cast +pru sa +orthop a +multi dimensional +mick foley +k ory +it am +haifa wehbe +diar rhoea +chau d +brun el +ðŁı Ľ +ðŁ§ ľ +wallo ps +un helpful +thecar ousell +tex press +terpen es +stereophon ics +sh ic +quanti fying +play ball +nh art +lo so +keep calm +ine s +in coherent +heat map +hayes grier +grizz led +glycer in +gin u +ex changer +auto bahn +am me +am fam +ag lobal +ðŁĻıðŁı¾ ðŁĻıðŁı¾ +work ington +under scores +su ju +sti mu +n ft +mus lin +mani sha +gran ad +forza inter +end cancer +eli est +der bies +by law +bran agh +ben assi +at man +asapo fficial +v festival +tt tttt +to ps +su mo +si ak +rin ne +play test +p ba +om ission +na ke +mu ji +mu i +major lazer +holocaust memorialday +h any +guay aquil +goodday atlanta +forever alone +disco gs +cro x +con cave +calde cott +ar ap +wind surf +teenagec ancer +reali gnment +rav elling +ram adi +rall ye +qu id +prest wick +milli meter +jud icial +intric acies +fear n +dur ch +bur ly +bigart boost +as part +ãĥĥ ãĥĪ +Ê» i +tr ous +slovak ian +red land +nev ans +mi ggy +m pp +je y +haw kers +edi th +eccle ston +despic ableme +deir dre +ad sense +... ðŁ¤Ķ +. ðŁĺģ +ภĵ +à® ´ +vi ation +the usa +sirpat stew +sad at +re ylo +psycho analysis +po demos +pap aw +ncaadi ii +mc cool +may be +her z +give syou +dun blane +cnn brk +b xl +ao d +american express +aliab hatt +ìķ ¼ +Ú © +wheel base +var und +untouch ables +thu d +squ awk +riofer dy +re assured +r ted +ph p +pan do +o dot +late ch +la hiri +l sm +ken zi +eh lers +e ko +dro ves +copernic use +be tten +be ti +ame e +) _ +vibr ate +to pa +tel o +son nets +seam er +re aves +poly gamy +pan zer +milit arization +mandal orian +h ingham +elfon theshelf +dom ore +contamin ate +con s +ash ar +akam ai +áµĴ áµ +à® ĩ +zach arias +wy re +undul ating +thank fulness +star sand +r mb +need n +n our +minig olf +march of +m ings +je u +fr yday +fau l +eamadden nfl +door man +com passion +biker ide +ay yy +ðŁıģ ðŁıģ +ठ¦ +wad dup +ug lies +ti aras +ther ia +soul cycle +s bli +ran cic +psychi atrists +pis o +p ve +p tx +nu it +mar as +jama ican +illi am +holling sworth +heir looms +hallmark movie +emir ate +chimpan zees +sweett ooth +self made +ru mpus +pl on +land es +laid law +ken ham +insp ira +hh m +cook stown +con ran +cm ll +clyde sdale +c mas +berli oz +bel mar +apal ooza +antwo ord +ale wis +ت ص +ó n +westh our +v tec +un wrap +snoo ping +she es +proof read +pride and +om eters +mani p +king sville +impacton pop +hi biki +fal un +evo ice +ash ura +aer ation +a ana +wheelie wednesday +warrior s +w enders +v fa +up side +tech summit +sli ms +phil hecken +l mu +kun s +kon da +kol hapur +g pd +embroi der +demetri a +car dy +blue jay +bay es +% ... +ঠ¯ +ze alo +wend i +v ats +unforgi vable +ms ft +min haj +m hockey +kab an +ju kes +dracon ian +damsel fly +bhagav ad +bernese mountaindog +tosc ano +ter rap +te vents +te an +sl n +share able +restaur ateur +ok cupid +modern family +mei sel +mcal pine +keerthy suresh +haiku challenge +fire dup +fer i +ail sa +adop tion +wies enthal +what sin +ugh hh +te jano +tar ry +snat cher +premi os +peter man +p mot +nch saa +me ter +kit out +gre inke +fa had +dispen se +dd l +bru dda +ah lul +z uko +wh er +way s +tid ore +staf fs +ron de +mon olithic +lolo lolo +leigh ton +kaoh si +hersh el +foo s +cha sh +centri sm +cen sors +boo z +bloom ers +black top +athen ian +you saf +sun dried +stouff ville +sj ws +renais sance +re don +ra gan +plun ger +ma bles +josel ine +infu sions +hig ham +heart soup +ee b +e bike +de wine +comer cial +b va +an ons +!! "@ +ðŁĵ¸ | +ãħłãħł ãħłãħł +ö l +y aaaay +world smileday +usc is +th as +tem po +taravi kapoor +stereo scopic +shus wap +sco ff +q aw +over stock +oppre ssors +mil d +hi aw +h ous +eviden ce +ek taravikapoor +dream scape +dar an +cle burne +cla shed +ben ig +bau sch +amer on +aly son +afri day +adel son +ac og +abi des +ðŁıĥ âĢįâĻĢï¸ı +âĢ¢ ) +waron drugs +visual studio +vis ors +theart bond +th usi +swe st +sp hd +soy inka +sor lando +sadh guru +ra isa +pe ds +ozz yosbourne +occupy gezi +lo ke +ke ast +h wasa +fashion tech +em ichael +ec rash +differenti ating +collec t +chi st +bart shealth +b dd +ase an +ag ü +z ang +wales coastpath +w ten +thresh olds +tatoo ine +tan king +sch ner +rock fish +pro bono +photo gram +ne ye +nar gis +marie curi +gau d +co fe +christi ana +berk ham +bebold forchange +be ery +b chl +archi ves +ar bys +ãģĭãĤ ī +power bank +pas ko +nefer titi +nasr allah +maur ici +mary se +mah jong +j he +it aylor +free ze +fatal moves +dispat chers +buck horn +ate day +asi r +ar bour +á´ Ģ +turf grass +tu gger +track nation +topo fthe +toen ails +sof ar +pic hu +ong ar +na seem +klin smann +ip w +impro perly +im parting +good will +gat tuso +exer cise +escu ela +de jong +bridge tte +bloo p +abull dog +âĹ ĩ +௠ĭ +w bs +v elling +tu pp +spun ky +sl fl +sin f +rad he +p tero +martin truex +mad u +le mar +ka ws +je ta +it na +fu jian +cruis enor +boot sy +at la +astrazen eca +al ani +triple j +tri z +resc in +proye cto +omme gang +make rere +ko en +har iv +excell ently +cab rio +be han +ba shes +ðŁijī ðŁı¿ +the smiths +the scotsman +tak oma +sath ya +r ri +pu bg +pi anists +patri c +p bm +nov us +nj wx +mcke on +mang led +m dot +kir ti +kar yo +informat ica +his lop +gu apos +fren chy +franco ise +for business +extra dited +deci phering +cremat orium +color ation +ble ecker +bell is +ðŁĽ ł +zi o +un saturated +sarato v +ridg eline +rare bir +r sherman +orti gas +nu h +metaphor ical +m gc +jab ra +hom os +ganesh chaturthi +foam rad +de ac +d br +bbcle eds +ar ran +ðŁĴ¦ ðŁĴ¦ +z ama +vol beat +ur ple +thick ening +reduc er +pp able +nw f +ib g +geo de +form ica +dis ra +amir mateen +adar sh +' '. +ðŁĩ¨ðŁĩ Ń +whis ker +v ars +sunny deol +sar fraz +run nings +resi sters +pi relli +mustaf i +mus ici +mc combs +liken ed +i ep +hu u +hover craft +home steading +halla day +green beauty +f adi +calis then +box e +af aso +á´ ı +wunder bar +whatyou love +var nished +v aw +toom uch +st marys +pro sen +pere tti +mu so +mo bo +mistre atment +lo dd +kno tweed +im magine +hydr ants +he se +girl scout +german ic +bu cci +australian labor +addi son +v ru +v pc +s fox +repa ira +po wai +paranormal romance +o tero +no so +ni pper +men ko +i op +em poli +cory monteith +col ts +best place +at kin +ç Ī +woo s +west man +w bay +ve ti +un regulated +the blaze +tan ja +schri stmas +power plant +pocon os +pleas ure +pa sts +official csa +nabe el +me iner +lo ath +li pids +kom atsu +iy engar +hit list +em cees +car lit +arron dissement +anky lo +al bac +afric aday +ठĵ +waxaha chie +transfer able +titi an +tar des +sus ann +shel bourne +sf mta +pizz aday +per ris +paren ts +or ting +op tion +nyc council +more to +mill ington +hur witz +eve online +dayofthe year +daes ang +da bears +car nations +ar ae +ðŁĺĺ # +ðŁĺ©ðŁĺ© ðŁĺ©ðŁĺ© +ãĤ·ãĥ § +wra p +vari o +the mighty +tas mani +so go +sierrac lub +neu ve +may ed +ma demoiselle +lu ne +k lux +hoe down +glene agles +di ga +dee dee +d cd +co vers +black thorn +bass line +bad ass +alife time +æĿ± æĸ¹ +âĢ¦ ' +wild man +u pre +t are +stroll call +soor aj +smart watches +ro tors +radi ocity +q army +mi res +mi ac +me deiros +mabu hay +krau thammer +jaz zed +in ny +hai der +f dot +eli ka +chal ky +celebr ant +з аР+vas cul +sli braries +sas ol +pyl ons +pom me +pal pit +ol factory +missing person +kell in +inf j +fantasy land +er j +beÅŁik taÅŁ +b st +aw aaz +arraign ment +wood ville +su priya +seri o +sapar li +re va +palm sunday +keen um +gri eg +eye health +east sussex +dol or +derers fc +dand ridge +cor gan +contr arian +comm ited +blan ton +athei strollcall +am mon +ac te +ðŁļ¨ # +ðŁIJ Ģ +ãĥ ľ +à £ +the dream +scon ce +sc orn +ow c +n agu +mort ars +me sti +massi mo +lat or +johnlewis retail +j world +islam ophobic +ho ey +glu tes +ed avis +duf field +boiler makers +ba illie +ac ampbell +ðŁĮº ðŁĮº +wonderfu lindonesia +u iowa +tor bay +shi geru +sare coming +pu edo +im printed +homeof cricket +gh alib +drou in +di di +aprilfool sday +ðŁijĮ ðŁĴ¯ +wool lim +to lex +synchron ous +rag gedy +quarti er +penetra ble +new post +k sdk +h cr +char ds +call out +ca ip +agu in +zo ë +z un +you ve +urqu hart +thunder clap +single malt +sequ ined +mornin ge +mor tis +lu ga +lib or +li si +ken net +kell ys +jail house +iqrar ulhassan +ham mad +grey joy +gorge ous +gi el +delle mc +cun ha +?? ?) +ðŁĮ Ŀ +val lab +v é +sub surface +sl its +person ification +perfu me +olympi a +mik lan +kel pies +jo ko +h kane +doo h +dialec ts +cruisenor wegian +connach trugby +buc key +an cho +ðŁĵ£ ðŁĵ£ +âĤ © +zak at +wur z +westco aste +vac aciones +tel fer +sch ou +roll tribe +pinstri pes +on ine +musli mah +leather back +la ziest +i drc +fox la +fis cally +ever last +euro vision +ela ar +dream league +corn ing +carre ras +bu ech +bru den +bridg north +brek ky +bat ang +acu ity +wu z +thur les +thor ne +shaw k +shak o +se gel +res ins +rajapak sa +quint in +purp lish +mcgra dy +k sl +jam meh +ilo u +escap ism +e well +e mar +defra govuk +configu rable +bu dur +blue bloods +becauseitsthe cup +aw en +ashraf ghani +ðŁĴľ ⾨ +vehe mently +v ries +un se +tu tte +stan wawrinka +ro lo +re home +rashi d +pre side +one pride +ole k +n sh +mug shots +mu ggle +ma sia +li ot +ju lexis +helleni stic +ham let +gr ating +forfe iture +fat one +divor cing +de porte +british flowers +bor go +bb ca +an sa +ÃŃ guez +wor tham +tr w +tele phones +sur plu +sque aling +sab c +ken e +hu gg +g ath +face mask +elastic search +daysun til +cr t +cheese burgers +cast elli +busch gardens +bu cker +bhu tan +amber alert +all llll +accom od +ðŁĺİ ðŁĺį +ðŁıĭ ï¸ı +womenin biz +with her +wa el +tech talk +orang ery +ne ots +im plant +il ani +ha gi +fool ery +donat ello +do sis +de frost +co host +christmas jumperday +cham pa +break age +al so +academy awards +ac iam +war hawk +se ys +rodol fo +pizza express +pec toral +mon str +mag ome +itz hak +iam stevent +geop ark +fal do +f fier +embelli shment +electric vehicle +deliber ating +de mille +cat ello +ar yo +% " +ðŁĺ § +ðŁĴ Ĥ +world bank +willnever getover +uro vision +tribu ne +thu b +sky lights +re et +ol use +mccoy dl +mc bride +ly la +john shopkins +iceland air +i vi +ge tac +gam ut +fu fu +div ing +descar ga +ctv morning +corrup tion +clint bowyer +char on +bull i +boney ard +blind fold +be ca +ax e +artist ically +af ra +ðŁĮ IJ +um kc +ui uc +ten or +tam sin +ski ff +si ar +plo v +photo shopping +peter head +pe des +online business +no bis +mum ford +je bel +ilove sydney +gy ros +ema baras +di mmer +da hyun +co xon +as ds +af ton +word smith +whit by +up with +tuf nell +pi en +nr j +kak is +hr va +ghos n +get me +gar of +ft x +deta il +demp ire +bomb shells +bnppari bas +ben chers +abbrevi ation +åľ ° +woe fully +way ward +shizu oka +sche ese +qpr fc +prosecu tions +pc g +ori enting +nanop ore +mon ta +limous in +la ther +kry stal +hor ology +h ks +geo sciences +delhi police +alo gy +waz a +upperdeck hockey +shar mila +se pat +rock fest +ran ji +pd illon +musa shi +mumbai rains +mi zz +mccand less +mahesh wari +ma vin +fol lett +de young +con figuring +cbc ns +ðŁĮ ķ +ye i +what sup +vivac ious +tru tv +tre sp +tagh euer +t veit +pat oran +nswr fs +mu mps +mi reland +leed sunited +gau ze +efferve scent +de mont +cham plin +canyon lands +boss lady +bo az +at noon +⼠ªï¸ı +vel u +u wa +ter a +sportsc aster +sme g +s loop +pininf arina +pe ori +o ire +mur r +gla de +flood plain +esri uc +eco logically +cier ra +car bo +bu stos +b dr +un wrapping +ta stier +shop online +sch lo +sau dia +run r +pod squad +nm pol +moccas ins +mix ta +mi shaw +mam oru +jin der +gin obili +f kin +ep as +bu z +bel tr +bar ran +av ance +age orge +zx spectrum +thor o +ox en +noble man +mobil ised +min ato +manche go +il lop +hel ix +g local +etch ings +energy transition +dj b +dead beat +cul p +cor ps +cheap ly +carru thers +big boss +b ny +ar ray +and u +ack man +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ +tsu basa +strong women +river bend +reale stat +nvi diage +n abc +me dea +ko el +kefal onia +kar una +invo icing +igu o +i ola +ham strings +hah n +fresh man +flat mate +ferre ts +di ppin +dez bryant +de regulation +crab apple +bo va +ari ki +ðŁĵ Ģ +ä¸Ģ æľ¬åĭĿè²ł +à¸Ļ à¹Ģภ+vijay fan +t du +sli dell +si won +serv ing +sam os +sa ker +riti ka +naw ab +mer le +jar rod +james gunn +i anc +hiro shige +hil t +green sburg +gilli brand +gibber ish +fire and +fin ch +bing ley +bi di +bati ste +» , +widen er +wash ere +wait list +traver sing +tooth ache +sr r +spir aling +space port +ra hane +prithvi raj +power line +plumme ted +pien aar +mo ven +mac i +lat oya +kil ly +gu thealth +ex olu +emp ir +ec ake +direction er +ðŁĺįðŁĺį ðŁĺĺ +z ta +y eng +ted der +surfri der +stone street +micro organisms +mar ra +man ly +lovel i +lee jongsuk +labra doodle +hy at +heen an +har v +gwyn ne +galvan ised +alam os +a ami +ë¬ ´ +toby mac +theno ise +stronger in +selfie with +se edy +sc alex +saurabh raajjain +q ash +new girl +mccut cheon +madewith paper +lo le +he ure +fet tered +el it +dur in +de cs +coffe elovers +chil tern +cal aver +ble ach +ad block +ac wri +worri some +wi than +vic enza +ushu aia +under lines +tax able +tanzan ite +t elle +sa ith +l wa +kr ant +j anc +harne ssed +hadri answall +h isa +gw p +fam itsu +ex tran +dux bury +d aci +cow boys +air tattoo +ag io +acqu ittal +south african +ski jumping +shan gr +postp onement +plo s +paro dies +nr dc +mar z +lynd hurst +le ón +ki ele +jo res +hb k +hb g +coconuto il +car bone +- [ +yaz idis +vijay an +urban sketchers +sub traction +na je +lin dam +gran it +courage ously +chillico the +carpen ter +barn stable +agu y +wo y +ur r +t sun +star ting +ro ssy +prin cely +pin c +mak si +luty ens +ker rie +ke tones +it was +isab elle +ilove dn +hubb ell +euthan ized +eid aladha +ci h +chapp ie +ce ans +anthro con +! ðŁĺĥ +zar ry +wb ff +wag amama +travel er +thi st +tbl st +sleepy head +shau kat +scar ry +qu orn +porter house +nu ck +mu gen +let splay +l mt +jo eye +hereford hour +d row +bunny men +bra ham +blow fish +biop la +bee zy +ba char +am na +à© Ģ +yemen is +y ola +tr b +sur tees +sc af +sa quon +sa j +quintess entially +pertur bed +p mg +ottolen ghi +no fthe +new ness +n elli +mar ish +k illu +g tas +dari o +can ales +adi ge +a thology +wich it +vos ges +sweet corn +shear water +schul te +sch uh +ruleof law +re played +land ing +lampp ost +hyder abad +fo co +en listing +electro pop +d fm +buoy ant +bewit ching +bethen ny +aryn go +ac offee +a world +un cool +tl chat +ther u +swin don +pang asinan +ox ing +ne stl +mp s +mic o +mb it +let them +ho ge +fore sters +fl aring +divin ely +de ano +bron n +bl m +and az +ai ai +ty us +tn bc +st j +snu bs +shun ned +see ya +rodri gue +pi ppi +mish ka +horo scopes +harb ors +gge tt +game over +fl gov +emp r +eldr itch +clemson fb +ch ri +cate rers +car ine +ban zai +authentic ated +ano ka +ali d +alex oloughlin +adam schefter +! ?!?! +were ld +wa il +rock it +pa stic +le by +kel pie +j ln +fe tch +d afc +ðŁĴ · +ðŁİĤ ðŁİĪ +z ild +wolf hound +wa st +the talk +sar geant +no reen +mom mie +jor is +hen ch +doctor s +com ida +colo red +aggreg ates +⼠ħ +âĶ ĥ +w ll +to ka +tipper ary +super villain +shou se +sach sen +practi sed +pi ons +pen sa +nk la +iq baale +infan til +huffle puff +homo gen +her ders +haz lewood +ha q +flyair nz +elg ato +du chy +demp ster +day na +d aga +cp f +biele ma +ald ana +mass y +leather head +kur z +i ar +h cc +gra sping +gan nett +explores ask +ðŁĻıðŁı½ ðŁĻıðŁı½ +Ù © +vox el +vig go +tw ay +thunder cats +thru way +star ke +shepp ey +plat ts +no val +k pr +john cornyn +h sf +gu ppy +fitzro via +fe stuk +enfor cers +de con +chall ange +bodh is +bet ch +beach comber +ðŁĶ ¨ +un corked +terr aria +stone house +slee povers +she th +puls ating +pan os +p of +julie plec +godal ming +ever more +cli pse +cal les +bio blitz +bi olo +bi ggins +bang tan +aw nings +ali stair +îģ ĸ +ypsil anti +wtn h +we ver +varun tej +si willnevergetover +san fran +sab athia +pers and +ogil vy +ne opla +nand it +man no +lac a +kar r +im plantation +i big +elin or +du bb +ar dro +af ca +y ric +west cork +vic ks +valeri o +trin am +scep tre +rusty rockets +osc uro +no daysoff +kvy at +khu x +ic ha +ho ge +haz are +han eda +gugu dan +fr mr +fireup chips +fer mil +fay sal +brit ann +bran stad +ben del +art inez +am bar +ak wx +w smv +tow ski +ti am +tam many +step mother +re dre +pas chal +n fd +motor happy +marion ette +litt lec +keeping it +gri eved +gl r +fit a +f of +coloni als +chevro let +canop ies +can ale +ad z +ðŁIJ Ĩ +ðĿIJ ¨ +ye aren +ultra book +truck load +to tt +t gr +sx c +sick o +scru bber +sant ander +republi k +post mates +pic stv +perovsk ite +most wanted +ml g +me k +labor dayweekend +kir t +ka f +heart breakers +gym khana +fl ange +es rc +d pd +bush els +ban field +ðŁij¨âĢįðŁij©âĢį ðŁij§âĢį +ula ganay +then ational +sp ly +shari ah +sh fly +radi ated +ordin ances +offe l +mar bling +link up +like me +ke effe +k tn +jack daniels +her m +henri ksen +hen shaw +han sa +guaran teeing +gi ster +fin minindia +dest abil +dental care +dad ar +d tg +bc ndp +archite k +à® ° +vincen nes +spino za +sold ado +sale sian +mal uma +lyn sey +hit s +famil ie +c ingly +> :) +âĺĿ ðŁı¼ +tam ils +photo shop +pe art +palliative care +mar tie +lagun itas +knuck lehead +j pop +inf er +ep am +cor ded +che mise +cal atra +blo ons +ðŁĺįðŁĺį ðŁĺĺðŁĺĺ +z ima +ww r +tour n +t sub +sinu log +r na +p ith +marj ory +listen in +l gus +kil im +home ostasis +hal lett +guil foyle +gri g +go har +ging i +fre ts +ca ked +bul b +bike packing +band anas +ally son +ag gro +wag oner +v te +urban i +tu tus +too faced +the musketeers +ten et +spoo fing +se es +san fl +qu ash +prism acol +per ton +pe ddle +mk z +mer cs +kon ia +inci dental +gimm icks +fur r +erkenci ku +dra wn +devil s +cont in +burkin afaso +belaf onte +ðŁ¥º ðŁ¥º +wit ting +stef ani +sat com +ole h +kru l +joshu atree +hy omin +gest ational +black bear +bird watching +bai lee +are al +ak as +ðŁĩ¾ ðŁĩª +ãĢ° ãĢ° +yaku bu +ty n +thin nest +sand bach +p msl +oo da +me so +master node +mahin dr +la ferrari +ice hogs +g pp +for ney +brant ley +bl ick +ari za +al il +ðŁĺŃ . +ðŁĺIJ ðŁĺIJ +ðŁİĥ ðŁij» +wicke dly +w fan +thecomedy store +ta pos +sensu ous +se ul +rol ly +re sent +pon ta +pleas urable +on el +military history +jhan si +je wski +infl amed +god like +eic hel +dres sup +dist al +cru d +bun gle +bol sa +boiler maker +as ch +abo ys +à© ģ +zam ani +ur man +tic ino +thegood wife +thecat reviewer +taylor caniff +sri vastava +river boat +random actsof +pen e +pc sos +or zo +ny ias +ny ers +montp elier +migh tier +la autoshow +ku li +kal ani +ju ta +in comprehensible +i believe +hunger ford +gam boa +fore sth +dee gan +day break +cr ouse +carp fishing +can i +boy zone +blo ating +bil as +bharatanen enu +an ac +amo red +ðŁĸ ķ +y kj +w mas +ti red +se apor +ro i +pe tt +op kins +multi sport +ineff able +in capac +i ji +fat ass +es lone +document ary +clip board +ani mas +ang sty +am ou +ag irls +---------------- -------- + ¼ +ucl an +the witcher +that awkward +shut tered +sen ergy +pres stv +por scher +mis aki +ke zi +inser ting +ha veli +e ap +dispo sals +di keh +da egu +cur rants +cro pper +charol ais +c una +as f +yorkshire day +villa iny +ve ga +ruben stein +rack space +pr onged +pon i +ow ings +nikol ay +ne sh +mu mu +mck end +ma ka +ly s +kis sing +k offee +jon jo +indi ec +incon spic +fl atten +ex h +essential oil +ec are +dock ery +dis illu +d wan +coer cion +^ ___^ +super gt +sne ij +real mike +par taking +michael b +man cha +lak me +jab al +green s +doyou know +dis ation +cur tain +cat skill +believe tour +amalgam ation +ag itated +un registered +sl at +sha ho +ou thouse +mel in +ku stom +jc poa +grenfell tower +es ar +er yn +eff lu +cau very +c sa +âĸ Ī +ur is +traver tine +thevoic eau +teen top +pry de +ple isto +pan tries +pack the +or r +ntv kenya +mahin dra +kar los +juli us +ih ana +harle quin +gi fu +gen stein +der ails +coted azur +control now +bi kin +bc ferries +bb ca +ðŁĺĶ ðŁĺĶðŁĺĶ +ðŁķ µï¸ı +ðŁĮ§ ï¸ı +zulfiq ar +zero es +wester nu +vit iculture +ver don +teen mom +sun oco +snat chers +shorth and +po real +net z +me do +mark ruffalo +lo kesh +led ley +knit ter +insta story +fishand chips +fat i +di mon +bk b +artstation hq +vis cous +rehabil itated +pin kett +not meus +ne scafe +j ka +gareth bale +co ch +austral asia +ale ssi +adi pose +a ou +ðŁĻı âĿ¤ +à¸ļà¸²à¸ Ĺ +viz sla +tw n +to pi +schwar tz +scep tic +red is +po sto +pieter sen +n sa +lu mi +favour it +de throne +cy d +youn gh +tan ked +strick er +strategi zing +sho pee +raj ah +nic om +march ant +kam ra +joong ki +is ur +ipad games +i fu +green space +gen cy +e sport +din c +cru k +bis sell +bec ca +ay t +aq ours +andy bvb +ðŁİ Į +Å « +win ked +welling borough +twitch sharer +seg all +re cht +p tt +or dia +open text +matthew daddario +last day +kat sucon +just ins +invo ked +dk pol +cac ophon +bre yer +beck en +az ar +ars ène +ari o +ageof sigmar +villalo bos +torch light +saltlake city +pon ty +piece of +nw l +ne gev +l ente +l ci +k wok +e gh +dise mbar +cor sage +consi sten +charli ec +bow ness +blo at +bel tand +antic oup +ðŁĴĵ ðŁĴĵ +shul man +sho ver +sha shi +retire ment +remo tes +re wa +r ci +protec tionism +ob d +mr l +mountain side +mit el +let stal +jami emc +hil de +hey its +german wings +flex es +bush el +blu dge +and hi +age ism +ab owl +whi ley +un reported +ter prise +tam pines +som bre +selfie for +sd mf +sciencen ews +radi ators +periodon tal +ny la +nascar hall +mer z +mechan ical +made of +i wish +e akins +capi strano +angel ine +ai ja +?? !!! +ty ra +th aroor +seat ac +red men +red fored +pl b +modal ities +li ms +incroy able +hol lows +gu ing +flann els +cu miklan +chel yab +bjö rn +bb its +ðŁ¥ ŀ +will ingham +v tech +trek kers +tr al +tel cos +st wee +represent in +red hot +nepal quake +nanomat erials +minn elli +lau ter +kon ta +kail ua +jo cks +hi eld +fanta stique +fal ta +echi dna +cler gy +cl oned +calatra va +be prepared +bat ts +bak o +anch al +à® ® +yu u +tennes see +ta chi +snap matic +seam stress +sab lon +rie del +ri kers +rakshab andhan +qu asar +nach baliye +morecam be +ko koro +epi k +di marco +day sout +blum house +a vision +ìĹ ´ +ãĥĥ ãĤ¯ +vish alkofficial +tho th +s girl +ro zier +painting warhammer +observ ant +nie wski +l ous +jan ey +itch at +ge bouw +gam elan +fin nan +day star +dance moms +crou ch +city police +ch up +archi medes +am aka +alien ated +world vision +usa af +tuni sie +re constructing +pag li +over seen +n lr +min of +il ish +glu m +f pd +et ze +e bu +de ft +conqui st +bc storm +bag ue +al te +ah our +ãĥ Ħ +zi en +work life +va ing +ur inals +tren to +sou ness +shopper shour +provi dent +new ood +le sean +le hill +iron wood +ic ab +ib t +ia af +ha pless +gar ag +fibro ids +dishon or +day challenge +curve ball +crow ther +cn f +cataly st +bal to +à Ł +vel aik +su ey +si ed +shav in +sant as +pro prie +keen ly +j ima +extre madura +differen ce +cham an +á ħ +zor an +y ari +x w +wh acked +un diagnosed +trade marked +shrun ken +sap ling +revel stoke +mu la +marqu and +man irat +ir reverent +international yogaday +hai kyuu +gl or +gh ari +done channel +cau k +awaken ings +ventil ating +t list +su wan +sam ir +rain drop +quat ro +pro fi +polar bear +normali zation +mcle more +mb as +math ru +mar ino +man ya +maic hard +littlebig planet +intermedi aries +i hm +her rick +hel les +emer ge +consequ ently +ay az +allga ier +vote themout +visco sity +tu mp +ti ber +tar ps +take uchi +t illery +special offer +shop now +parmigi ana +parisi enne +o sso +minneso tans +micro chipped +merci lessly +kaw an +kar li +indi gestion +in vert +han uman +hali k +guar dsman +good tobe +ger ani +ero dgers +emer cury +desp ised +cu pof +bir n +bbc wm +b town +ðŁIJ Ĵ +ws g +wi thered +who vian +vel d +thal le +t ach +sub standard +stock dale +sal ar +oba femi +g ine +fal ter +disal lowed +book blog +bl v +awesom en +âĢĶâĢĶ âĢĶâĢĶ +yellow jackets +woo ow +water brew +un tapp +speed test +sin action +sch elling +sa ic +ru pi +re fried +line sman +k se +fi v +euphe mism +do bie +bur go +anupama here +åĪĨ ä¸Ģæľ¬åĭĿè²ł +vol te +truste d +states men +pre release +pag ina +osp rey +op reps +oec d +north view +int ol +how ser +haw i +evapor ated +cros by +cor rer +cine mark +bur ling +biom es +bhag wan +bedand breakfast +b union +aure us +am z +⼠· +wre kin +winter green +walang pasok +traf ford +tn ick +sun burnt +sf jazz +remembrance sunday +r tn +pi rie +nor n +new ts +la ika +knock in +ju mi +fertili zed +f ns +el ang +change over +canandai gua +argon auts +ur s +so cin +ske wer +sioux sie +since rest +saif alikhan +jav anese +fe sto +e ren +dog gos +descan so +body building +aid s +yu to +tab ata +stry der +se red +scre ed +phon ie +phen ia +o er +mach a +jeong yeon +jen nette +in saf +gloucester rugby +da igle +bollywood news +biop rin +bay nes +autumn watch +í į +æ° ´ +ãĥ ½ +usc ap +tol u +the score +the brave +over used +myo cardial +ku biak +graph y +fast net +eth ylene +enjoy life +da ikon +chir p +be inte +ur vashi +tro cks +shi ki +se id +pri ya +pasteuri zed +ma en +lu gan +le ann +ho ddle +g we +fran zen +dyk stra +car ding +bum rah +berk shires +bed spread +ax ton +afremo v +к а +we ire +se my +ro sin +ra es +jor din +flo pping +affin itweet +abse iling +; ). +& , +ðŁĻĭ ðŁı» +ys f +wake man +teign mouth +syring es +sy p +se kar +sav ind +po co +panther a +orient al +myo pia +mind less +med twitter +man se +log on +lis ch +ju wan +inter active +integr ated +ha pha +gigan te +en compass +d ce +cr ane +col ate +chil is +chelyab insk +bridg it +ashwin i +alt press +ðŁĻı ðŁı¿ +yoak am +woo zi +vets get +vetsget scanning +spin nin +sa ida +reptil ian +pinot age +ok ayama +man hunter +kle pto +jarry d +ip sum +ii it +hi rez +ger rit +fr ill +euro millions +c mm +adic hie +ðŁĻı # +ðŁĺ ¾ +ë ł +thur t +theli ght +stra bane +sko kie +sk al +rid ley +re introduce +pro ge +paul ding +open studios +li se +la velle +go stars +ev ille +ename led +corpu schri +clau de +cir que +cin é +child like +bobs burgers +bi ersack +al jaz +wen lock +then ame +tele portation +taste the +something new +s je +ricky pdillon +py l +ox in +om kar +nau ld +mile sdavis +law al +kha bar +ke mang +jor die +homo sexuals +euro stat +d do +cor rhi +ala qsa +ðŁĨ Ĵ +transiti on +ni ge +mx n +mam iya +m ki +kings ford +he yyyy +formul ate +dar dan +d reading +cb sphilly +cashi ers +bra ver +ater alism +abbo ts +] ' +á ¥ +trick ery +ter me +spi sil +space time +simple mente +sc ac +ru sa +ra za +ore tti +mon ico +max imo +la ia +holt by +ham ann +er is +ener gie +duf ner +cha eyeon +canap és +ab t +vo re +thread less +storm watch +shre w +re loading +ph leg +lewin sky +iu fb +gel ly +cross ley +am iller +al pert +ว ว +zand voort +worship ful +woman sday +wg tn +ultr amarathon +ty la +tat ty +supportour troops +numer ic +nom en +newsp aper +net book +meridi en +magome dov +leav itt +islam orada +flouri shes +cook off +convin cingly +choco bo +camero ons +bo ggy +awa ke +allfor one +ai fe +à¸Ļà¸ Ĺ +wel comeback +trit ons +schoolboy q +pew pew +mor onic +mail day +laure us +k sr +jer maine +j rn +gun nery +ew b +be is +ap ap +ðŁļ £ +ushe red +swild cats +sp here +secon f +ra jan +pun y +pi po +ma ffe +lesp aul +lar naca +il orin +hu guen +hate m +for mosa +chri sd +c jp +bla zing +barak at +ah t +aga th +ac cts +` ` +ìĶ ¨ +wester ners +villa in +un responsive +tu scu +sof ia +sli ther +sh mu +sen doff +quarri es +ninj atur +jaff na +jacqu es +intensi fication +il ang +gu b +glad bach +ero ding +env agency +elec t +ci pes +chat el +ca ñ +btsx amas +am h +aber nathy +ðŁĹ ¡ +ta ze +sthlm tech +stein meier +sing led +sal ta +pleisto cene +pix abay +mel k +ma theny +m hu +intere sting +ever green +eng l +em sc +cz w +amir kingkhan +" @/ +whats next +the empire +swith out +ss as +say onara +save d +s women +rite sh +repleni shed +po il +pecu li +lu vin +in or +ic ac +e os +cryp tom +cra p +ch evening +bristol uni +boe heim +bewil dered +zano tti +ye wear +tre acy +tc w +scu pp +sant amaria +rc car +h gc +faith less +do como +ch ug +cal oric +bureau cratic +anth apuram +yeg traffic +wellcome trust +we ve +vac aville +ump iring +son unig +road america +qu itter +pic story +pag en +oak en +miro slav +masterche fuk +lou se +lon eliest +har ney +gal eri +ga shi +fi fam +eti ha +d tr +bab o +abi ola +. $ +wit ching +wi erd +warner bros +tou rette +seung kwan +refriger ators +post al +pics depot +par an +mega force +m golf +le imer +l wb +khe dira +je y +ha asf +gra hn +gerald ton +en actus +eat drink +cat en +brat z +bian con +b ge +wha dd +w so +the food +subbar aj +sty lo +secre tion +r do +michi el +ko b +hay ashi +haw keye +full screen +dinwid die +de fund +cullin an +cloud less +bl g +ãħłãħł ãħł +wr on +weight lifter +team dcs +te ve +student s +sr ry +san key +pinec rest +mu sta +kas ama +jan ath +fren z +forthe win +ev ga +bath time +auto zone +all out +after work +ðŁıĪ ðŁıĪ +z epp +tempe h +siob han +sho hei +rez nor +rarebir daler +peter sfield +non binary +na hl +mer gency +kar sh +gu er +etsy handmade +din the +crock ery +cri ss +broad sheet +black town +balac lava +athin ai +" @. +âĻ © +wor rell +wal le +stpi india +ro mi +rel ink +q ty +pent land +ngin x +mo xley +mar ten +mar cos +m sla +jar row +in ton +huff po +he ave +flow ers +fc cincinnati +dr kumarvishwas +brain injury +blue planet +bed ded +ann u +anag i +âĢİ # +zen desk +win dle +ven ue +the color +tg v +t bex +st pauls +san dow +parish ad +of cal +north coast +mujahi deen +mu mm +miit omo +lin ce +ksd knews +kat ona +jud waa +jou le +im all +heart ening +head canon +gou rock +golf course +fan photo +er of +dro cks +dime bag +david lynch +cur itiba +az ov +aft union +" ] +âŀ¡ï¸ı : +âĺ Ķ +wil lesden +wg tn +trek kie +sx swedu +sta vern +sh f +ru mbling +re counting +pot ence +mo stra +luch alibre +lower show +kab ila +hir ani +gavin newsom +dur r +chicag om +bi ma +ber ni +bab oy +arri e +ðŁĻĥðŁĻĥ ðŁĻĥ +اÙĦ ÙĤ +y pj +vis itu +v pl +phil omena +per pend +ot sego +on zo +n bi +metabol omics +mac ri +ll n +kap i +ju ries +indiscri minate +hosse ini +hal vor +goldeng irls +gargo yles +f bu +equ ates +du moulin +chi ma +char is +al derson +âļ½âļ½ âļ½ +âļªï¸ı ðŁĶ´ +ww wwww +vick ery +u lus +u eno +spol itics +sol aire +so hio +sar ang +saf ter +protec tion +peabo dy +mol lie +l fg +kail ash +k kriders +itu ate +ingle se +hand sets +drag way +car pedi +car ib +az el +zag ato +street z +ps g +pl atten +pal las +mapu tra +hoo ky +flame thrower +elly wood +dir tb +dig ans +cli o +chal ce +cap ris +book oftheday +am ra +am ite +wor thing +wis c +who soever +v bc +tom eter +stac i +souff lé +shoul dered +ship wrecked +sethu pathi +sch loss +o dern +jess a +ga fe +g cu +dar gah +ab dou +ðŁĴĽ ðŁĴĻðŁĴľ +women and +vern is +tam asha +re pr +rarebirdaler tuk +poiti er +live install +le ly +kla ssi +kha i +d bd +cl in +cann ery +birch wood +beck er +at ara +anamor phic +american horrorstory +tur bot +thunder dome +ter remo +size able +r fd +mus ics +mask ell +l bl +kap adia +gam bian +car vajal +bl unders +aker man +wi the +tembe akenya +si mar +ri jiju +q w +press o +pr ins +pad dlers +morgan stanley +mali ki +lin ne +in j +history museum +dat af +cou lis +cor onal +compla in +be gum +amo e +ai me +a hier +åĮ Ĺ +wc i +ve ster +uni vers +uil state +ser kis +r dg +ohh ill +ni gg +ne sco +mccla ren +li zar +k het +indi annavy +his ense +fi ably +do ze +digiti zer +cardiff city +aw ssummit +as ami +ãģ ¿ +âŀ ¨ +xk cd +w ounding +ste ff +ni shi +mun ger +mil ks +manife sts +man ju +front lines +fro id +fri ended +fall colors +du san +dream cometrue +da iries +d nc +п ей +zer man +tomor o +the aviators +sing ed +shelby ville +sch wab +port lao +plov div +pi pa +over coat +muk ka +marke table +ma zza +k lum +is as +ginu wine +fortun er +dur g +dri vel +claus ura +cardiomyo pathy +brisban eroar +bo des +bak ery +b dl +wo donga +wi shi +utr ition +ther ain +seman gat +sali h +po catello +p â +olympu suk +oc kets +inci sive +fas cial +dr dre +cushi oned +concert photography +cathar tic +carol ina +c ze +bunde stag +bas se +ban on +b fr +b afc +altrin cham +westcoaste agles +victor ias +tar aba +rock smith +rizz oli +pe cking +my lan +f pi +dazz ler +dav es +conjun c +cal f +associ ating +world book +safe ties +s fo +r jal +novel a +monash uni +mi do +meth amphetamine +luci d +le mond +k bo +innis fil +glen core +du sh +buck ingham +bo ssed +bhu shan +bb cdp +atlantic city +ac kie +ðŁ¤© ðŁ¤©ðŁ¤© +çľ Ł +wasps rugby +us fs +ur umi +o vid +monster hunter +ma aa +kant ar +jer om +i photo +ho si +beefeat er +a ef +ðŁĩ¨ðŁĩ º +ðŁ¤ Ŀ +water stone +vit ili +tail gat +squ es +pro phy +mendel sohn +lazy sunday +kaizer chiefs +joh o +illi ps +hau gh +ha kan +fortn um +dro bot +do wager +do ily +cfis d +canon photography +bry son +bri erley +an sible +am persand +tall is +summer holidays +sof icial +sa wing +river bed +q mul +po zo +onen ation +nah j +mon davi +kop p +h db +columbi a +co burg +bar rell +bankholiday monday +as sn +ang ara +andy warhol +amend i +affili ations +absen ces +ðŁĴªðŁı¼ ðŁĴªðŁı¼ +âĶ Ĥ +win sor +victor ville +uk awa +tor sion +tau t +sport sin +ski ed +shab ba +sab in +popul ace +nir up +mari elle +li mping +kaohsi ung +high ly +h fa +forç abarça +flam in +don is +de iro +character ize +cay o +b he +ano tha +ach o +ðŁĺı ðŁĺį +wind proof +uw g +up lifted +un tv +shum pert +os oph +mr kt +mi amic +me dev +maser ati +levit ate +jack s +gr at +fru tti +dee ks +cor oman +apar k +íĶ Ħ +âľĮ ðŁı½ +yor ku +tra fic +sur charge +stro gan +ster o +ste acher +shad ers +ru tte +roadto omaha +po dge +ou trigger +o wo +narrow boat +mu ggles +mo hun +ket chikan +kawar tha +j cm +ine scap +iguo dala +hang gang +gra do +colle n +cn h +cla sse +cir co +car afe +boon docks +ðŁ¤Ķ # +wheel house +out crop +nith ya +ni hil +m per +gad v +d rick +cowh erd +clear view +catalo gues +bit torrent +af el +yo be +wrong ed +u maru +tom s +stat ler +sle eds +sal uki +ropp ongi +pi ston +not so +na ilers +heterogene ity +gra sse +eddie hearn +cle e +bau delaire +yellow ston +vintage fashion +ur mston +trevor noah +ti veros +tar yn +sul ley +qot sa +pay ne +no wak +nevers ay +n zo +medical cannabis +magell anic +keepit inthe +ing rained +in l +im penetrable +i fr +for u +flouri shed +em mam +e steel +ba ikal +asse tt +app development +al con +aj ag +zo on +v anya +she kar +schizophre nic +pp pp +new books +mon real +might ily +man bij +lau trec +ke eling +ho omans +god smack +d vi +bo gut +bene detti +au ma +apprehen sive +ðŁĴģ ðŁı¼ +woo bin +sof ig +shepp arton +sa ira +pro t +petit es +ms gr +maddie ziegler +fu zzy +fu kin +f te +check lists +c fe +au der +animal planet +yas achi +wan n +ti poftheday +subsi stence +ss a +scrat chy +mor oni +kle z +estre llas +eg f +creed ence +consi dine +candy man +bull frog +am ash +ðŁĮ ¤ +Ø§Ø ª +y assi +spri ze +sick ly +sab s +ro we +ro sco +plumme ting +pi ven +oooooooo ooo +ni rav +na bel +moo cs +man ors +loit ering +lib dem +gi acom +erik son +den bigh +cur cumin +bi har +ar ob +ac cc +¿ ï¸ı +wall st +tri but +suscep tibility +sgo fficial +re solves +pig gott +p end +mb aa +john nie +job sin +iri global +inqu inn +gav askar +fl amed +es ss +ecc mid +di acon +detroitbecome human +c cio +bronco scountry +berg kamp +" ðŁĺĤ +yach ty +survivor ship +st itt +s open +pt sa +mo er +lec o +kgw news +hel la +grand national +ett one +esthe tics +dent ure +corn meal +bt f +beat plasticpollution +ani sh +amand ak +ðŁIJ¶ ðŁIJ¶ +ta pa +so iled +shou trts +nol a +no sh +n wan +love me +li zz +gene sis +gas ser +gag ner +fre donia +fa q +euph onium +dissi dents +dev t +col angelo +cirrho sis +cas sia +br ack +ap ink +adverti ses +пей заР+yo lo +u is +tay tay +sub titled +par se +ny ce +no ve +mor zine +mar imba +mac eo +lanc er +iam beckyg +i know +h ml +go lic +exoner ated +eco chic +dis agreed +cy non +bruden ell +ðŁijī ðŁı» +win sor +tv o +tul la +tranqu ili +town home +sport media +sel ig +sebasti en +sa jj +resili ent +re joint +r hum +oneminute briefs +na oki +ke shav +kam chat +kah f +jeff lemire +co rel +bo gen +... ?? +ðŁ¥ ļ +wish in +wild ness +we bbing +wales online +vent ana +trash can +spor tage +na it +ló pez +irr fan +i ago +hl hockey +had dish +fug itives +free stuff +f bm +ey ck +bi mini +anakar ylle +ail ment +acadi ana +aal borg +tae il +stylish ly +saran ac +republic an +linke din +l na +kirk ham +gor ams +google edu +getty sport +ener gi +ding ell +com eau +co pes +climate march +cal ver +bro d +as mara +ab j +ðŁĵ ¬ +าภ¡ +пейзаР¶ +w dsd +vaing lor +stap hy +soo m +rangra siya +ra bb +prie bus +pre f +pessi mism +mega store +ku ku +joey mcintyre +in fact +harsh est +ha x +dont drink +dock side +differenti ates +dalry mple +constra int +buck thorn +ami ya +ä t +yeahthat greenville +world animal +tri ppers +sav ills +quer cus +psal ter +pow dery +parine eti +over took +oscill ator +nor throp +ni igata +ms x +mine field +liber a +k att +h gs +gyne cology +glori fying +for tin +elliott wave +ech r +denver channel +b mr +as cott +ab ul +vu du +visit ca +te ve +tara strong +stone haven +sm its +show a +rep tour +rayy an +puni shes +pol dark +mu laney +mu cky +life s +lever ages +land mine +kick stand +ka aba +ig u +gu shes +green wall +gr ans +go sforth +fair ing +dies es +chape co +chaffe e +britt ney +blo bs +bat wing +av entu +angelique kerber +ì ¿ +اÙĦ ØŃ +yellow tail +stu ous +sorren tino +sc ular +pura vida +piggy back +pav an +past i +on gress +o hr +more e +laet itia +je tz +id week +fal kirk +emphasi ses +arig ato +abhin av +ìĨĮëħĢìĭľë ĮĢ +va isakhi +trinam ool +take ak +stre tton +slim line +ski ppers +ro on +ri ends +prat ap +o log +m wr +louisi an +lok pal +infalli ble +fan fave +en dish +embrace the +distor t +de wa +cher t +bull head +bry don +auto parts +åĽ ½ +wor li +tex el +reign ited +quir k +qu ila +procu red +pick ling +pan athinai +o he +nigh trts +mall ards +lob bied +lady like +kid sin +hh of +h mh +cro ft +close out +cl k +band z +agra ha +ad ela +ðŁ¤¦ ðŁı½âĢįâĻĢï¸ı +é ķ +woo die +up l +thorn bury +thiruvan anthapuram +ther mals +sh ames +se un +ren u +que ers +post natal +panch ay +mcg lynn +keralab o +jump ing +jon i +its janeoineza +glu tin +flo rez +everyday sexism +ee ting +d cre +cher o +centr alia +centr ale +bon k +beauti fying +am usic +af ern +ac ure +ðŁĺĴ ðŁĺĴ +un gu +thrash metal +pelican snba +mo ley +mis am +make over +loud speakers +lifelong learning +joy sms +ima gem +ic ho +hu lu +hover fly +foo duk +financial times +conven es +bo gan +ajith kumar +ac cola +un capped +r ws +pav illon +new job +nay yar +mccle lland +lan ter +kar n +go iu +game spot +ey yy +ero oms +dun ford +di reland +de ire +ca stron +ay es +av ar +amb iti +ale gria +ãĥ ¬ +âĶĪ âĶĪ +summer land +mor ing +march ing +ke ying +jeni fer +hun dley +hick en +have fun +fo y +bett ing +ba che +ล าภ+Ø ¥ +Ã¥ rd +werri bee +v ba +staat soper +song kran +scand i +re zz +p mf +n flu +lo onie +l rs +ku ech +krist aps +kar ang +hey it +gur l +freel ander +flo ater +dy o +be ady +. ,, +Å ĵ +wai ving +ts in +tru mbo +toen ail +star dust +se ki +richmond bc +punc tured +protest ants +propon ents +ow usu +orange burg +official wrc +o ju +nade shot +le tran +kir ti +kex po +in gest +idi ol +gh el +fictionaldeath siwillnevergetover +f pj +escape e +dianade p +v lad +titos vodka +te so +ser na +scar bto +robo tech +return march +r mr +modisar kar +man gas +ma hot +lan ky +l gs +kel sea +ing live +ho ffa +global ism +gb bo +g itex +eco logists +dev ening +darkk night +cour tau +catalo gs +bha ya +vir unga +v ta +un punished +spectac led +si ghtly +semin arians +role models +ro ble +paw nee +palmi eri +n ter +m sa +in stock +gan is +f hs +ene e +castell ano +bun cha +bl enders +र त +wca x +uuuu uu +us dt +strat um +ser gior +ri ad +prender gast +muni z +magni fy +kah n +in considerate +her ma +har nett +gye ong +ginor mous +gar ra +contempl ated +chit own +aun ay +alla h +ulster uni +sting er +sho to +sh urst +profess orship +prit am +pi af +pe dometer +momo fuku +missy elliott +mas oo +loh ri +infu sing +gustaf sson +dg ar +co ord +clock tower +cl orox +bro kaw +boxingh eads +ba hahaha +ab ba +witt ingly +uo it +steuben ville +sie te +sco l +re tract +re ce +pre scrip +north devon +lam bic +jack rabbit +dough boy +do remi +d ff +calabre se +bre i +bbc breaking +ar ce +aper tura +a jr +ൠģ +ww week +tour neys +soft shell +sc news +ori ans +one goal +nyc feelings +li ban +lash ley +k eli +interven ed +i idx +fut ility +du sit +disillu sioned +china sea +che if +ath u +am my +a ichi +" $ +ĺ ï¸ı +what com +w ani +te ahouse +tat amotors +ste ws +so led +sin us +scram bles +retri eving +pe as +p nw +owen jones +metal working +la hey +l man +l ct +kk b +k hel +go ven +for sk +fl ation +clon ak +can tt +can lit +bur han +aur is +warner archive +torpe does +te agas +stat em +sm iler +sha rec +san die +rc sd +profu sely +po spisil +opp of +ml r +li bra +ki xi +ine uk +edge of +crow ne +cri bbage +castle bar +book cover +an heuser +ì¹ľ 구 +âļ¡ï¸ı âļ¡ï¸ı +tter dam +ts itsi +tri balism +st je +snag ging +ri den +re press +r we +pre clinical +pat ap +om and +mu su +del tas +cun ard +cre wed +bath y +ar mes +am ino +íĥľ íĺķ +zaf ar +yan ke +ti was +rous seff +ro tarians +pol ina +hic cups +hall andale +g add +electro lux +eas a +duchen e +ding y +d xc +city centre +che sham +caloo can +ant am +tiem bre +rela is +refin ancing +pre stat +otter bein +mul hern +lin er +ha irex +fel ons +de ducation +surviv alist +ri u +r ra +mil ken +louis farrakhan +lam bie +kom i +hassan rouhani +harpers bazaar +gre iner +foo se +di leep +bay ne +baham ian +arin en +ðŁijĭ ðŁı½ +wild about +well man +under score +un democratic +u tta +tr h +ta eny +stri ding +sil lu +sal li +sa avn +radic alisation +rachman inov +ot en +medi al +m nc +lot sof +ku hl +kov ac +kor ner +instaf ashion +infatu ated +har un +gujran wala +fan ia +dor n +do work +confor mist +be ena +antidepress ant +alfre do +a ang +ðŁį ¤ +war ring +wal ru +vo sa +thru xton +the dragon +senior bowl +ree der +raven swood +pro té +pag ans +ome tti +o des +ng v +na hu +multit ask +mr g +market screener +great returnmarch +festi vity +chang in +cat ty +cad u +c ta +annab el +an tti +allot ted +ðŁIJ ħ +íĶĦë¡ľëĵĢ ìĬ¤ +un install +steam ship +sou le +san go +opp olo +ol ap +nicaragu an +mixta pez +las se +kran z +khair ykj +kar sten +j rc +infiltr ating +glan ville +freecode camp +fox new +el ation +edtech team +clow ney +by nes +bis bee +ad av +a unch +ðŁij ¾ +ye w +way yy +wa ir +tyler g +thursday aesthetic +stra il +st room +shor ting +read out +pri mero +pic hai +pg ce +par rilla +par dons +nat us +manirat nam +lo kal +lef thand +king fish +jun hui +isa ak +indi scre +gorakh pur +going to +far ted +east gate +demysti fying +de ek +d wan +compil ations +cel a +cath arine +book pro +bl p +! ðŁĻı +ðŁıİ ï¸ı +ãģĬ çµµæıı +âĢĵ @ +yun hyeong +yi wa +wd bj +tourism day +terri fy +specul ating +ser u +pal umbo +pal grave +or wellian +nithi in +lu pine +liais ons +h wang +gb g +fi sho +ed chat +christmasin july +antic ancer +ys k +tor c +tom es +ta pas +su st +stag gered +shanmu gh +season of +s mond +qu ally +o sun +nou ghts +mm w +leg gett +kabo b +hu mi +ha fez +h dc +fin ne +every damnday +cw l +curren ces +bige ast +ash more +ar and +ad pi +absor ber +wil ders +paparo ach +me ma +mac rae +law y +kar is +jä ger +inter nazionale +heart month +freck le +financi er +en gi +coast mag +character istically +bur ford +bour guignon +b ph +ar bonne +ap in +sty les +sl icker +s mas +ro cio +pyth ons +pur beck +pil a +patter ning +over saw +ky ung +ke gaard +im balances +hin dering +head stones +har to +ga ster +fab ia +euro p +done ws +de du +consci ous +cas bah +c mn +bever idge +ðĿIJ¨ ðĿIJ +ู à¹ī +w sh +tv line +tri ste +tany abur +solidi fied +snow balls +smir ks +size more +shu ayi +sam an +ris me +pot m +porsch es +p ining +ni sts +nar mada +n dc +ma wson +khandel wal +jeremy scott +infatu ation +for congress +fon z +anom al +ãģ Ļ +v anda +rec at +pan cies +numb ing +no doubt +newmexico true +mis mo +lat ers +lac tate +ko ck +gustav us +gat ecra +fujifilm x +force d +f bo +don ni +conven tion +ber on +âĿ¤âĿ¤âĿ¤âĿ¤ âĿ¤âĿ¤âĿ¤âĿ¤ +ठ¥ +zo v +yo semit +wolf gang +shu ster +rudy ard +pres se +ic en +fun nel +cryp tid +back roads +at aturk +ar ga +yamaz aki +trento antonio +trave münde +tele serye +shrop shire +sco ast +sb st +resc ence +pi es +op tioned +o ssie +nick las +low y +joey badass +i hi +g its +cuv ée +ct xwx +cat arina +calisthen ics +c bu +buzz horn +bil ia +âĿ¤ ðŁĺĬ +а ÑĢ +win wood +under paid +tran sa +sum lin +shan kill +separ ator +pat an +no fear +nau m +metr orail +lam poon +ke shi +ian mckellen +he garty +flan ker +encry pt +dilu tion +chattahoo chee +am ellywood +z ino +ym ca +wildlife refuge +thre es +teach able +sydneyis skyblue +swami ji +row th +need ful +nano tech +microne sia +go greek +gandhi ji +en right +edit able +cin ia +carbox y +ble u +ach ine +wall decor +vitam ind +vibr ational +rodr ÃŃguez +redd warf +re gains +mac laren +leh rer +la uper +khar kiv +j if +in usa +her mana +gur das +embezz lement +domin us +dh b +car ro +bloo died +ari sto +aravind ha +af ree +? :) +ðŁĻı . +ðŁįĢðŁįĢ ðŁįĢ +zol tan +wine time +w ach +terrap ins +tat ton +sycam ores +ssi onists +plau ction +pir an +pick ard +per sson +olu fsen +net ty +ne etu +mean s +lu ft +joo x +gu is +ful la +ev ant +bt k +boxing news +as mile +anim ity +an up +yan uko +wareness week +vel de +val ais +upl ink +sar faraz +rin gette +public library +mill icent +mcder mid +let ang +lee filters +il p +flam me +e gs +decor ates +br indi +bea stieboys +bark sdale +albor an +! / +âľ ´ +è ve +women sequal +ut ama +u igh +shoel aces +se kai +re trial +r kelly +peri yar +past as +na o +monte pul +michael gove +gode acs +eno ise +dizz ying +cho ta +che ong +begu iling +ale gacy +... âĿ¤ï¸ı +ðŁĺĤðŁĺĤ . +ðŁİĪðŁİĪ ðŁİĪ +âĺ Ĥ +workout wednesday +w ft +u zi +u ve +tn q +substanti ated +su x +st c +shankar shanmugh +plo s +play group +philharmon ia +p cie +n vc +me on +jer zy +is ds +if y +hut son +fe ig +fare ed +entrap ment +eh f +d sson +cut lets +cur ragh +chlor o +bizar readventure +ðŁİ Ĺ +wy ld +wies baden +unis wag +tur rets +tre m +sor aya +pg l +nu kem +lucy lawless +haute couture +google doodle +can nes +by ram +bo ch +az eem +albi hariv +ah sfx +!! ðŁĺį +yu ge +whit ley +west sussex +us ola +tn m +sen edd +resemb led +read abook +re develop +pri a +paraly sed +n intend +my photo +mass on +lori ent +jan ak +im prison +heart failure +go wen +gar rard +dumb bells +discre pancy +am pe +Ñ Ĩ +wr ds +we itzman +viol adavis +verma elen +tweet chat +subsidi sed +shi g +sf h +samson ite +ri sc +re dit +q doba +nv da +e let +continu a +co kes +ba jan +am iri +:(( (( +ðŁĩºðŁĩ ¾ +ver dasco +that matter +schnei derman +rebec cam +re mor +quie test +pulit zer +princi pality +pre rog +pr ssa +monta igne +lo tus +kar te +k assel +hy phen +ho ang +g aga +fruit cake +fan te +ew f +el fs +eate mu +defense less +caw ley +air d +ðŁĻĪðŁĻī ðŁĻĬ +âĻª âĻ« +th ich +t sk +su is +stat in +rr t +rand hawa +psychopath ic +petra eus +now showing +lu u +little hampton +here is +ha zi +green bay +gli ac +em pren +dun drum +di ac +demo graphy +coul thard +australian coins +apply now +a ham +.... !!! +vidyut jammwal +un leaded +un friendly +un favorable +ug by +th doctor +rac c +quiz up +q ol +peac ecorps +pas scode +oedi pus +notic ia +mari ka +mal la +ki official +khur ana +ke on +kaw ai +ka sim +hi f +ger alt +ay ut +ar ara +ap x +you go +wer the +t ila +stol ler +stam per +sf dn +sch ick +sat suma +raf bbmf +r ter +police week +pen field +p de +nit to +mono po +live united +lib dem +in sole +i official +her sey +doubt fire +dis mayed +dic hro +char don +cal v +ad ra +zi ppers +tri ste +sm alley +scal y +ram y +it ens +i one +eeee ee +du ps +de cries +criticalrole art +bug gies +boc cia +bigh ero +argent ine +alyss a +z eu +west view +vo ta +un sealed +teenchoice fox +tam ir +sympo sia +ssang yong +solve ig +ofe urope +ni pped +mol dy +kom en +ken ley +k ws +har pist +gon na +dh all +desh pande +cruci ate +coroman del +bo ssi +... < +ðŁĺį ðŁĺģ +ðŁĺĩ ðŁĺĩðŁĺĩ +unfathom able +ul ter +trit on +torto rella +thumb sup +ten ets +ten ergy +tan quer +soo kie +scuri osity +sar don +sar ath +ri mmed +polkad ot +om ak +le se +ke te +k una +jun aid +gr iner +golden rod +gig antes +ful crum +ell an +di ani +currently reading +broad view +ber al +am bode +< ) +ðŁij© ðŁı¼âĢį +âĮ £ +wr angler +week nights +under foot +twit ching +turn berry +su dir +ss is +shangr ila +ser rated +sea forth +rubber maid +rive ted +read ers +re marked +p bloggers +out flows +non verbal +ni v +nabo kov +human ity +fin den +er f +das soc +d vs +auction update +as la +angel ina +.... ..@ +( ? +the ak +ste ffy +lu blin +k win +g ago +full metal +for bes +ell ars +dd is +car mina +by d +boardgame geek +ber wick +auto bot +as ura +ap hex +ander pump +air brushed +your way +wil more +tur ki +si a +s later +ro ared +pur itan +om ori +nbc philadelphia +mor cha +mi me +ku char +ken newick +kb ps +indie book +hen do +ft th +flo of +bru sco +ben elli +asser ting +aqu atic +ठ£ +sur real +st kil +shet land +sed bergh +scots magazine +ri za +playlist live +pa il +mi us +mh k +lor as +leicester tigers +lc g +hul la +hu ms +help me +gin and +ea f +dungare es +don tw +deterior ate +dar cey +dal al +d wr +conspir acy +ch ere +bander as +all iteration +* âĢ¢ +âŀ Ł +xen overse +wheel in +u al +tur bos +sfor kids +saturday kitchen +s ja +rand olph +pr b +ou w +o in +new look +nd wx +lew ski +incur sion +gr rrr +car shal +buc ca +ban an +asset management +arau jo +apo logists +af ly +ðŁijģ ï¸ı +you got +wax es +toile try +stow away +ste adi +standup to +si ski +sel tine +school s +pho t +miamis up +lore tto +lam ba +kr ita +ib nlive +hin dley +frank ford +exfoli ation +diabe tes +de cer +cra gs +bin dings +bat cave +aj j +wo www +waveleng ths +w abe +toysfor tots +rash mika +pizz o +pha ser +ore l +musso orie +la pp +hit t +happy independenceday +ga ius +colon oscopy +ang ing +adidas football +yon o +whit eli +ton ks +stumb le +solemn ly +ru r +ragh un +qi ao +pay gap +mal an +lisav anderpump +ku be +k son +ib l +homemade tools +gw h +favor ing +ec lamp +deven ter +deceit ful +dag en +cur ling +can ey +big brother +bau s +ah ri +winter sun +willi es +wi bw +twer k +table spoon +t eli +sze chuan +super tramp +reminis ces +rally finland +quizz ing +papa johns +naco gdo +mccre a +i zzie +fitz william +fal le +de j +dar relle +canti lever +business women +bush fires +yas sin +vander pump +th als +t dn +ri ko +proce dur +opti k +omo vies +nai doo +minute man +kasey kahne +fo olin +ex chequer +corn rows +black mailed +bl at +bil derberg +ar twalk +ðŁļ Į +ðŁIJ Ļ +ëĭ ¬ +å § +ye eee +thaw ing +shak ir +ra zed +pitto drie +p atta +mccre ady +jack i +inside the +fla m +can am +camp sites +ban nock +at si +ar bo +ao g +anch ine +vand ana +sw ade +show and +rishab h +pi raeus +phar md +mat son +m gp +lau g +kal inga +injec tors +hypnoti zed +hil bert +fric king +e if +du sd +do herty +de bi +cab a +brah maputra +ber rys +ban offee +af fer +] ." +ðŁĻıðŁı» ðŁĻıðŁı»ðŁĻıðŁı» +âľĬ ðŁı» +z ele +thom es +te ide +super marine +stress ors +sing ing +si bi +self care +scalex tric +pres que +podi atrist +p mb +naval ny +mother ly +ko ku +ingh istory +do pen +cj reform +chor al +as amy +ampli fying +ali i +water ton +vw fc +unex plain +strangel ove +she sh +qu bool +pre via +person i +offic in +gn on +g ps +fi est +farah khan +engul fing +energ ys +.. ðŁĺį +ت ع +y anni +warnerbro stv +tor rid +summer sale +or bis +motor cycling +mojit os +le jeune +hippo campus +gil pin +flgov scott +fin dthe +edwards ville +dsw d +d mi +cur r +bon neau +blue eyes +b bo +am ills +win sford +weid man +upcycle d +sung yeol +shi shi +re setting +ravic hand +r ty +nt australia +dham aal +da res +d cd +cb colympics +bapti sed +bab yyyy +adi an +ðŁĽ ģ +wil ford +wak aflock +sp ink +sie ge +sad am +qash qai +phra sing +ling a +kin ka +indigenouspeople sday +il ie +gin ho +giac ometti +extru der +cand or +callthe midwife +bow erman +bague ttes +at oz +arche types +anten nae +without you +to cin +th ts +th aya +sh int +s guild +musk rat +mc gre +man aus +mag nit +lun di +lumin escent +lap ses +kin dof +je han +if ad +gu iness +greg ori +gi jon +gg is +foo dgasm +floor plan +f sg +ess ss +di marzio +dd ata +clu mps +al at +ãĤ ´ +ze en +y cling +w eli +trou p +tote bag +shre ws +scur ll +repjohn lewis +or te +ma ho +kaz i +jor dana +irrit ate +ha vi +ge c +f ici +avi e +ari jit +am rit +am gen +wre g +wit a +tor ide +ti died +shu bh +se mua +ride share +r vc +outfit oftheday +nypd news +novel ties +kid cudi +khali stan +k ren +ide c +gru p +gon nam +conne ct +confe ssing +cere bral +bal am +ash u +won k +twom bly +river ton +repaira ble +re constructive +ra west +ple at +play writing +paul kagame +nurse sday +lo dz +ghou lish +dra xler +dolom iti +de te +cru do +billy joel +atom izer +as ol +al car +y cfc +we sl +under floor +tre maine +te tr +sween ey +sket chaday +se ba +s sec +rjal ok +rin der +rally gb +pr ine +port as +jam mies +horseri ding +gra phie +gi menez +gar oppolo +gar dai +ey enews +clun y +cavan augh +cal lie +cal ey +brou gham +berline tta +ben tham +arou sed +wh aler +vo les +ty ner +twee thearts +tor na +si ap +shu ja +sar avan +sand awana +s fr +quik silver +pter odac +pm harper +ob tains +neo geo +mog wai +mid year +mi kasa +eh ne +droit wich +conservative vw +cho l +bigten network +arach no +æ¸ ĭ +ur bo +u sta +sub prime +sle aze +s ber +ru sia +neb biolo +man al +lun t +it ori +is good +ho ard +hel dens +go ve +fla gg +et at +emma watson +cas so +as aba +aro ha +am ica +alfar o +wer den +tri glycer +to ho +re ema +punx su +om nomnom +ol de +mack in +li vor +kw gt +kh ris +john c +harpsic hord +gal ent +francis co +dr g +come to +cater pillar +calcu lators +bbc world +augu stal +ad sl +tran spon +tra eger +string ed +sr hv +sowe to +sle ad +se ur +santac ruz +run happy +nhs bartshealth +ken cen +it all +hot sauce +good fellow +gian franco +ec ap +b ening +aha b +take flight +symbio te +sou da +solar panels +si gue +ru bric +ri voli +rema x +ome gam +n kandla +mores by +mik ado +migno let +may bank +man gum +makar sankranti +kam eron +i ero +hart pury +gab ay +ft nhs +ever son +come dia +color blind +be aune +bank stown +amend ola +---- ---- +ðŁĴĹ ðŁĴĹðŁĴĹðŁĴĹ +ðŁĴª ðŁijĬ +ðŁĩµðŁĩ ¹ +à º +ÑĤ е +ve sting +up keep +traw ling +team breezy +star scream +ss av +sper son +slu mps +shin ya +re package +po were +po ort +pla b +pic hand +ok kad +o brien +nu ff +n ani +illi am +harold sinnott +green party +glen elg +ge er +dreamleague soccer +diso wned +constan ce +cas sandra +al gui +ty per +tore ros +tan us +swar mapp +sin dy +shee ting +sham si +sep tiembre +sar ita +palae o +indv swi +fiel duni +david walliams +cool down +color ador +camise ta +ap ul +ad ac +wet shaving +van hansen +tw ard +tou areg +syn ced +str ang +sp w +sh acks +sati ri +ron it +reali sts +pramo d +ori ol +fann in +en nale +embro iled +cul vers +chat room +buff ing +ban e +ac m +ðŁı ĸï¸ı +writing life +vasundhar abjp +un desirable +tho ckey +ram anu +pa store +nin ian +ly tton +knu dsen +gg v +fi z +emble ms +emb arc +dispen sers +ca sid +asu tra +app y +ðŁĴŁ ðŁĴŁ +y ut +wb ball +vin icius +pre dation +pa sting +noplac elike +ne wark +mammo gram +ma ji +luch ador +ilove jax +i plauction +espar za +el ley +contempor aries +clo aked +cele stia +car ola +bt x +brave hearts +bi ghead +benand jerrys +ar len +ap it +ap at +anjun abeats +am erika +ãĤ¹ãĥ Ĺ +ا٠Ī +y aba +wau kegan +tw p +tter man +stra ddle +statec ap +rash tri +quarte ts +plat num +pax aus +morg ans +li baly +leopard stown +kro hn +it security +hun ty +here dia +gra ined +express oshow +d he +ak f +* : +á Ĺ +yanuko vych +ty ger +sun limited +shealth care +sc itech +oppre ssor +m tt +he ssen +gon gs +funny picsdepot +flip side +fam iglia +du o +cathedr al +bal anchine +af pphoto +.... ( +ðŁij Ĵ +tho o +seaf loor +san kara +rac ial +open air +ner ve +mat ryo +kilo gram +khal il +it weets +he is +embo ssing +egg man +bachar ach +att va +ðŁĺĭ ðŁĺį +ðŁĩ³ðŁĩ ± +vox dotcom +un learn +super cross +ros ita +re paid +pan ettone +nor fol +mii verse +mari ai +loud ness +ley den +j dc +fm news +fasci itis +eye glass +eloqu ence +daw ned +chron ometer +chri swe +cho i +carling ford +bhar gava +bbc mtd +bal tics +uof m +ty d +swasth abharat +stor noway +shu ffles +sen o +reson ated +re ag +no via +monster cat +mb ank +lo te +kir ito +hoo ligan +her up +h ite +fox news +early modern +derby shire +day trip +cudd le +consecu tively +bli c +black out +be mis +ar ash +âĻ¥_ âĻ¥ +vishwar oopam +vash on +trajec tories +sine ad +sat ri +pu fc +new lyn +natu rel +min tz +d pan +cru k +bor u +ta ko +se and +s america +pri yam +navar ra +monte cristo +mill is +ingh ope +hep atic +hall in +fc ity +electro chemical +dr martens +cj ad +as rc +weather ill +varund hawan +teh reek +stocke xchange +sko ol +shi be +rubi dilaik +n pe +mo ko +ma ic +indi ak +in takes +impedi ment +il ent +go tye +getin to +fing ering +clau son +c ni +bal o +ann andale +an ju +an ers +am g +al goma +womensequal ityday +tew ks +sugar land +prospec tor +mil ian +man made +li iga +laz ada +hum per +hb v +green bush +ep k +con tro +biomimic ry +ठĤ +uk tour +the happy +scro ft +punxsu taw +on the +newmarke trace +me ca +lie tta +itsmore funinthephilippines +is born +haringe y +fri sch +eye candy +electro des +con ant +co done +w br +sch y +rad wanska +newn an +nei man +nb poli +megam i +ma da +lunar newyear +lei va +inthe sky +i vs +glend ora +foreigno ffice +fashion photography +eu ticals +d kr +c st +c ftc +bri stles +bic ent +az family +ai ff +ðŁĴ¥ @ +ðŁİ ĸ +wa aay +up u +tho d +sle dging +sig ne +oire ach +nor ad +noel gallagher +new comb +ma suk +kra b +ken ner +jet star +in ert +hon ore +global ed +bur pees +bs v +bett man +at sushi +arjun bijlani +airand space +ab bin +ó r +sonunig am +se mat +ro vin +nat galleries +natgalleries sco +nar co +miz rahi +lero y +kno pe +hi ker +hang ing +comple teness +cha vez +cab ell +bil der +av m +ak y +a are +pretty much +po ta +over arching +or nl +kovac ic +ken n +kam ui +hel f +har r +ga stonia +fo h +fidd lers +fac to +aren al +âĿ ĩ +zol ciak +toyo tag +tan gier +spot ligh +spo ols +san bernardino +s burg +ra pati +p dd +n age +mu cking +j io +is cool +i mus +hassel beck +har shad +gn g +forex trading +du es +borgh ese +bi kaner +am uk +al wys +waist band +w league +tot alling +summer house +srin ath +pun gent +pe dr +pab st +mulca hy +infr inged +fir daus +bur ka +brian cox +bi ola +bc bg +ðŁĺľ ðŁĺĤ +x el +sul kin +sal ve +rafi ki +pan ky +pag lia +na aa +malibu comics +lear jet +lac una +keen eland +kau ff +her acles +hair color +fur st +cor rin +cas al +ale h +ب ÙĨ +y aaaa +vi ra +te sy +styli stic +strong bow +sport pesa +savi o +pyram id +may er +mante gna +light sout +lease hold +lan n +kit tel +kick solution +jet set +ic at +f gr +donttry this +de jesus +cw lps +character ised +buzz words +bail on +awesom esauce +asi ana +articul ating +abra sion +ðŁ¥° ðŁ¥°ðŁ¥° +ãħ¤ ãħ¤ +Ñ Ĥ +z hong +worth the +talking picstv +sen goku +r é +pic o +nit ric +mr ti +mal u +la ster +jac ke +is simo +gra fia +game audio +down for +do something +di donato +d fir +chal ked +adi t +accor hotels +ab rac +âĺĿ ðŁı» +wt kr +wl tx +vote redto +ve sper +spur lock +sc limate +s mid +recen sione +paper clip +oom ph +national birdday +ke shav +k df +ichoo se +gmt games +di mm +dan ews +clo seness +c se +al tidore +after thought +zimbabwe ans +za id +wizardo foz +un flattering +thar p +tal ong +sump ter +stein brenner +sn w +sb n +sat o +pl d +mar com +mal ina +luxury cars +kho bar +j ss +ice house +hicken looper +dead by +d loesch +cas sino +budo kan +bi zz +amar one +tic e +sou vla +sin uses +seam us +samu elson +pre poster +news rooms +mel wood +maest ros +ma gus +lyn x +kav i +ir f +hal eso +get out +ent in +dog walk +cu al +ðŁIJIJ ðŁIJIJ +âĺĢ âĺĢ +what evs +wex ler +vi stara +swag s +soc biz +sneij der +sm on +si se +pr ancing +le ff +khadi ja +j sm +hill toppers +emer il +dar nold +comp o +chan tic +can aan +bl inn +ðŁĴ© ðŁĴ© +york ton +yl i +world building +w syx +u hi +stre l +stop kavanaugh +space ships +ski i +sel as +rac oon +pri mula +platnum z +paren tal +pal ah +nim rod +min doro +mc mullin +lo in +il en +em merson +cricket merijaan +ze o +w afl +thel oud +specialty coffee +soap y +say no +sab adell +rosam und +ravi dubey +pray ersfor +patrick dempsey +ower ri +oc u +mari as +lifeis beautiful +go tto +d wee +circu latory +child less +bag ay +awol nation +analo gies +aldublo ve +ðŁĻĪ ðŁĴķ +troubad ours +tou te +timb ur +so dy +see the +rachman inoff +n tt +mol ars +mo tta +love ukweather +k ates +il keston +hol gate +hair styling +fel onies +chen ille +camp grounds +am asa +å¤ © +© @ +st ape +sl ung +righ ton +plan es +p oul +mic ha +methu en +kore y +ke ener +ke ck +jarre ll +in fidel +il ona +herb alist +ff re +dog meat +cur sed +cron k +centr a +cam rose +bright man +as ce +ac cade +abas ket +ys ers +wy se +warsz awa +vik ander +ver onika +unfinished business +su ter +steven age +startup grind +roth stein +rio olympics +name plate +myrr h +mer cu +me aux +low nz +lin seed +ir un +i aw +gi ani +fij inews +ef an +early ed +detoxi fication +deta ins +cor rado +burn sville +bri thday +bri stle +bodle ian +bj j +bau t +aude mars +as ys +ðŁĺİ @ +yan ong +trayvon martin +suf ism +stern show +stainless steel +sp all +sant ini +ripp on +panathinai kos +musko gee +loo ts +local elections +la yan +kit teh +khur shid +kel son +iron side +illi c +hick son +ha aa +gooden ough +brand en +ann ast +we ger +va o +uk news +talking dead +spi ers +sculp ture +ridg way +re sets +ra ved +nex gen +nat aka +ligh tened +lie ber +inter i +goe bbels +gal lau +free play +bu kan +at ani +a of +ðŁijĢ ðŁĶ¥ +ï¸ıâĥ£ , +sy ard +squ alls +ran deep +r nb +qui el +proudtobe abulldog +pom eroy +o brig +moe bius +kar ine +juni e +jou st +joe ys +jo k +ir y +ha is +gin o +ester o +del ands +coo t +bbcradio wales +assimil ate +and ouille +ðŁijįðŁı¼ ðŁijįðŁı¼ +wine fest +wai heke +ve sic +star tribune +sid well +scale up +sc cm +pru ett +perfec tionism +night marish +nca aw +nc f +in bkk +hirsh horn +he tero +griff en +green e +fat test +faceof mlb +el r +chuck grassley +christ oph +chip tune +c itt +brick ed +bo ga +blasphe mous +ber m +are dux +thel and +sk op +shak er +o ems +mciner ney +k ween +i ppo +gas ps +col mar +cla xton +castan eda +? ðŁĺį +ðŁ§Ļ âĢįâĻĤï¸ı +ìŀ Ī +wed ded +ve te +uka id +tribut ary +syracuse u +san pedro +on location +ngr president +mon oli +modig liani +luxemb urg +leg anes +iam will +ecclesiast ical +du plass +ded ham +comp els +blan ch +bill nye +âĿ ¦ +âĻ« âĻª +weight watchers +wax man +tede sco +te zuka +sneak peak +rec ir +ran dee +radio times +py re +oom pa +messi anic +hawks bill +ha ga +glen livet +gil mer +fabric ate +edin son +eco smetics +colorado springs +co tte +bag a +b ä +b anta +antarc tic +ambro sius +a sea +ðŁĺij ðŁĺij +th il +te avana +tam era +shann ara +sch aff +s ence +rhe e +re ta +pe al +mari ach +kri dge +ic co +fratern ities +endic ott +dere cho +dam er +cad mium +brick town +ì º +v pa +tau s +ta rek +sun downer +rose burg +pel agic +pa es +ou nos +nicol ai +lindel of +libt ards +leadership development +laure ls +hot star +goldend oodle +gi untol +dand c +cros sh +ch ym +cannab idiol +bure ss +bmw motorrad +blin ky +bel asco +apol itics +am bler +ale sha +ðŁĺ® ðŁĺ® +white boards +wa hoos +us y +stro de +sar as +pro visioning +oni giri +may ank +mal inois +low ell +ke chara +hyperson ic +herbi vore +hard castle +blue star +bio diversity +av os +and white +ware house +viol ators +v asha +tul loch +tor fa +th ony +sh iller +pun tac +proce ssions +piec ed +p ca +mayo clinic +ma shups +la goons +in suff +illustr ative +golfclub sforsale +frie sen +drizz ly +do ane +deare vanhansen +cross bar +bri on +au rea +aro berts +aqu al +ðŁĻĤ ðŁĻĤ +weis man +uz bek +traumati sed +switch board +st app +smo vement +sa arinen +re growth +ra wing +nu ke +melissam c +hun na +glasgow warriors +dict ated +bv l +balonce sto +amar al +ag dq +velo ce +the hague +tet ley +tattoo ed +son us +sc india +sar un +preemp tive +pr oro +pi dgeon +mon tel +magi k +ke ylor +ine x +h pt +f cbd +cyril ramaphosa +co ppers +chri sho +bur r +actor jiiva +trans verse +str one +stin kin +pil atus +occupy central +nephro pathy +looo ong +le ight +language learning +l rb +hy annis +di ppy +col ville +cho ate +central coast +car illion +camp y +bol dest +b hay +all ston +xplo rer +wy wx +w ur +ur so +taver na +summer nights +rock dale +re supply +qot dapp +pan etta +pal azz +oh well +monon oke +loe we +listen to +l eri +kun dp +if p +i onia +fro mm +cé sar +cu enta +col ley +be gotten +ang rier +ad are +abhor rent +! âĻ¡ +ðŁİģ ðŁİģ +za al +v sp +there min +su br +s doh +qaland ars +presi des +nup tials +mis behaving +im ams +hc mc +happy tuesday +fru iting +di abo +datam ining +augustal sina +an zi +!!! . +ļ ðĿIJ +ðŁĴķ âĿ¤ï¸ı +ðŁĩ±ðŁĩ § +ëĵ ľ +wise au +we artv +war ne +te pper +strategi sts +stargaz er +sp ann +siss oko +sal a +physical activity +newn ham +na im +n cel +me aden +mar cin +kay aker +j iz +hagger ty +gun ge +gu yan +ernie ball +di splace +de el +code pend +cay etano +ay yyyy +ar irang +adren alin +achan nel +ston eroses +sto ga +sport scars +solom ons +q hu +ph nom +palla dio +lun gu +lo i +j ari +hob goblin +gathe rer +de volved +come and +celebr at +bra inde +ba atar +avie more +as ky +: \ +ãģĬçµµæıı ãģį +uro logical +un declared +u ob +su ess +sc ura +sc ast +sam en +roo l +ri pen +raise your +ra ju +pra bang +pinar ayi +paign ton +os int +lake wood +kon an +je ffs +jacob whitesides +incu bators +ichi ban +hb l +fr illy +fo gerty +conju res +ain slie +. ðŁĴľ +wor te +wol ters +wo wow +tra gic +teleno vela +smar athon +shaw ols +sex ta +salvation army +quan tu +pinnac les +on itunes +nestl é +myelo id +mu y +mit er +meg ac +mc kee +jo van +heart break +gas ped +func tioned +freak out +endthe stigma +disab ling +carne vale +cam uto +bernar di +ðŁĺ¢ ðŁĴĶ +âľ ³ï¸ı +ÃŃ as +un ni +ter p +sin op +pre co +opi ate +men in +mandu rah +lon gu +integrity tt +hr tech +great north +fr nd +eli k +dad dys +construc tor +conce ited +can em +ðŁĺį " +su ll +oper andi +on ster +mm x +lost cat +leg less +karim loo +ju ga +j sp +hand rail +gri pen +glori ous +di mming +bury sted +bt c +be eck +am ai +algui en +youn es +ti sham +stil t +soul ful +sid cup +seg awa +p ex +open shift +mechan ically +hd x +har tigan +dhanan jay +cu atro +canal side +bon gino +berger ac +as cle +an ju +ag low +ag et +.... !! +âĺº âĿ¤ +tom er +the us +teac ups +sa urs +real mickfoley +perman ent +pa chuca +matric es +loud phillips +kof i +k ago +g cr +flu stered +de anie +bloo diest +bis u +am ni +selen ators +sc ens +rine hart +objec tivity +moving forward +masa hiro +marath oner +lom i +logitech g +koin ange +is wa +ing ues +hyun gs +hi ther +end anger +ele v +consu mables +caval cade +cap ilano +black beard +arte misia +arian ators +actor madhavan +yo c +un win +u am +shahe er +sci der +s mbs +p ish +my mixtapez +j oma +he yn +gui do +federal reserve +fair mon +dist t +direc tories +cab rini +ber ri +beau voir +be the +a head +y sle +warrnam bool +up market +tv personality +tuesday morning +schri stie +sar gon +re bus +r bu +presi den +pow ells +nfl draft +nacogdo ches +music group +kis lands +insomniac games +il or +exter iors +end res +der ot +de composing +das ani +camp agnolo +but ted +br ann +anti gone +ahistoryof paint +ठ¯ +thim ble +the stor +sul ly +starwar sthe +sc avenging +red wood +palah niuk +nove mber +mat eria +longmire posse +kerrang magazine +ing els +industri alist +i dai +ghe alth +dont miss +del any +cook man +brain child +book nerd +bland ford +backto back ++ ] +ðŁļ ¿ +ye z +v ash +un stuck +summar ises +pen manship +mumb o +minimum wage +maz ur +mar cas +k ray +id wx +gold in +follo back +ear pers +É ª +well being +var g +ubis oft +tom brady +some where +qu ire +pax south +od ar +london bridge +justin formmva +it ar +ha at +gup tas +gom or +glaci er +ge b +gan ic +cam ron +c pap +brianmb endis +brantley gilbert +bow doin +boo z +ale jo +ag at +âķ ° +th f +ta zar +sex tet +sam osas +pl is +pel tz +pedestri an +oo t +newh am +mc williams +koinange jeff +k tr +ji be +gas lamp +gar ou +fit ment +ened ict +en tail +duck face +coin age +co ale +car very +atho l +aj lee +afca jax +ðŁĵ ĺ +tra it +tor ms +stri bune +snow boards +shadowhun ters +sere mban +prefer able +pesc ara +nz mustdo +nar ine +multic oloured +les by +g att +ey al +en fin +day made +congre s +ade vi +accoun ting +! ðŁĩºðŁĩ¸ +ðŁĺ¤ðŁĺ¤ ðŁĺ¤ +Ë ļ +z illion +yel awolf +un question +thalai va +shay e +savag ery +poly cystic +nh ra +nc b +mathis on +made a +jay as +indul ged +ho well +f mt +erud ite +drash tid +d anna +cire bon +ch ander +ca ity +bay ou +ant en +alban ese +æµ · +âļ ĺ +zom at +v si +tay la +sultan ate +sagarika ghose +rt l +re eses +re charged +pla zas +pi eters +passi one +p mt +merry xmas +men of +marti al +ly can +ku antan +jojos bizarreadventure +is ac +cullo den +chef s +cam omile +bean z +an nette +a itor +ãĢ ½ï¸ı +à· Ĵ +whom ade +whi ppin +sun corp +ru lings +obl ong +marsh mello +ly re +live mixtapes +lafar ge +je anni +hot chocolate +ge ty +fu rio +for all +fall a +ez ral +eun kwang +cumber nauld +c gr +bleacher report +apo pka +al italia +agil ent +ðŁĺĢ . +ðŁĩ» ðŁĩª +wed d +tro ika +torch relay +terrori ze +t inge +t anger +stat ics +ron y +re assures +ra ze +pre so +pl am +orph ism +matthi eu +fun chal +f sn +est ation +en el +e pos +dist o +den ys +dam ore +da hi +car natic +bur un +airtel india +your self +wonder woman +wi eners +tv m +swords man +so ha +seductive sunday +pet kovic +oil oncanvas +jugg alo +hu dak +home automation +gu mmi +go ch +girlsin stem +fli m +electri fy +dig nity +commissi on +canon usa +av ro +american u +ag f +a adi +up and +under arm +team adidas +sta westland +smd streams +single track +r sn +quanti fied +pocket camp +pan kaj +oxy tocin +outlaw queen +or rin +of time +nigh tof +ke ter +k sg +jim lee +jeopardi ze +jax a +janath agar +j harden +ho isin +h kd +giuntol i +fra yed +for trump +doo zy +deli ghting +del ray +dair ways +chir ico +car crisis +c dj +arin ers +thre sher +strictly comed +sp akistan +seas capes +scal pel +ro mulus +pro po +prin z +pre clu +pat in +kis sin +kareen akapoor +gl eng +flam borough +dece mb +d andy +cli k +⼠± +âĹ » +wo u +v oom +th it +seven teen +serv ant +sar ovar +sam er +quinter o +qadi r +puj ara +publi sher +pla sia +per domo +pc bs +nau seous +n gn +lom poc +jig gly +ir refu +hero escon +he sp +ge er +f wiw +exempli fied +exemp ted +do is +d xd +bang sam +ban jo +av n +wed gie +thom ason +sway ed +sv cs +spen ny +slam min +re starts +orm skirk +meadow lark +mar scuriosity +man sa +maith ri +ma sato +li saf +ko ehler +kab e +ja key +gar lic +flori o +du pont +dd ler +ation week +arsenal fc +ye k +y ss +true detective +thel aw +sun beams +school memories +pra bowo +oi af +life times +lank ford +k ci +in sead +howdoyou pixels +fthe year +down grades +dam mam +cor champs +colle gian +aul x +ðŁijĮ . +wo h +sc um +ra ham +play hearthstone +pagen aud +nik ola +mcl ane +lucoz ade +lake tahoe +lac tation +kno p +kei fer +janu zaj +home bound +hol lowed +heat on +gor gon +fur baby +fox glove +fau cets +colbert lateshow +barnet fc +agar h +! ðŁĴª +thab uri +statue of +snee zed +singapore ans +perth glory +patho genic +orthopa edics +odysse us +mq m +k tg +far ouk +cw f +creative bizhour +bo ice +beo grad +water parks +vitam inc +re broadcast +ph enix +perfec ta +palm beach +ni mh +my croft +mu tv +liber tarians +lang dale +l tm +jad u +ine k +har macy +gun day +fair lane +entwi stle +csur ams +canni b +bu tane +aveni da +afe ty +.... ) +ë§Ī íģ¬ +wax wings +video game +un daunted +ther yman +staple scenter +plu gge +nis d +nab e +mari el +lor il +leather face +key ne +kam o +hu la +herb icides +gw apo +grego ire +flav on +fion n +fatt y +do ke +disab led +dam ion +col as +che quers +ðŁį ŀ +ðŁħ ° +ë ´ +zon da +yuz uru +whe aties +under took +u bl +tu sc +sonn tag +raz on +pu kka +pe ssoa +nab lus +mus kie +misam ental +lo pen +lar ch +lan downer +jame scharles +gra cht +gly col +gi onee +g cm +er ob +cé line +cay e +c no +air liners +ag era +abdu r +ìĺ ¨ +æ· ± +wing tip +violence against +throwback tuesday +sound s +servic es +rin aldi +pun ting +porscher aces +p wr +luzer n +ind re +for humanity +fair ford +ent rada +dan mark +ati st +ati me +and blue +à ± +wwe shop +vitili go +ur bandic +under hill +thisi sour +tex a +slan ted +remote work +radio shack +molo kai +mid somer +mb ap +jar od +ih sa +har rah +fir mer +fa ure +cla ires +car oti +c ang +b gp +assi on +app u +af fo +ðŁĺī ðŁĺī +web md +swarth more +ste g +re efer +rab ino +promo code +play land +o wain +mill ersville +le anna +kuech ly +hypo thyroidism +green build +forthe many +fair ley +er ice +di sing +cv g +busines stravel +brun e +è © +un itas +small youtuber +nal cs +move the +morde cai +micro bit +jack gilinsky +ir vine +graphi x +gra ha +g live +fri pp +disgu ising +chuck todd +amal ai +zan esville +worshi ped +wart burg +u cu +star ter +sol way +sag na +ro den +por tra +mer cad +li ane +it sd +illumin a +hu shed +fc p +experim ental +e ol +du val +chri e +belmont stakes +beis bol +ant ander +al fi +ðŁİ¸ ðŁİ¶ +ðŁĮ² ðŁĮ² +whi pple +water aid +vin b +un wittingly +str ana +sd wan +reson able +notori ous +nang arhar +li sas +lfc family +le ic +hump day +h mr +go the +evo king +ei der +detoxi fy +con cili +cin tiq +bla is +bar ris +au bin +andri y +alder weire +ðŁĽ ¡ +ðŁij» : +sand hya +quar ry +pol ley +oc currences +nvidiage force +neverstop exploring +mo onee +man ed +helenclar kundp +gag ged +descri pt +da shes +cl ang +car dano +can geo +av ond +as sa +wwe sheamus +wut ang +wewill rememberthem +we know +vibes only +van canucks +u tiful +tür kiye +th l +talk talk +summer side +str itch +roo tedin +re ous +quay le +obe ying +grand sons +cnn money +chat sworth +char tres +br att +au dia +ae ter +âĿ ķ +warrior pride +virtual assistant +va sia +tre de +talk za +sal ou +ran ce +r fi +pir zada +pd b +pa rel +os ler +oh p +need lessly +met all +meado whall +mcel wain +mccull ers +eldor aspeedway +dele phant +del tar +budge tary +alternat or +addic tion +ys jagan +wood carving +u ffici +turkish airlines +triu mp +stephen ville +silhou etted +shant anu +scottish fa +ro aches +pe dra +p mc +nu de +looooo ve +li velo +kis er +kar on +k ma +ho by +com pas +cau x +bre ch +author ship +ar mer +ðŁıĢ : +woe ful +wode house +un categorized +tiwas avage +stru ck +ros se +r ps +prithvi official +no bs +kor ma +ken zi +jone stown +jav y +il it +ga ad +fe ei +esp a +end childmarriage +do en +cooper ates +ci bility +bigg ar +alex morgan +al x +aa ahh +ìłľ ìĿ´ +u ria +t ve +so you +share my +rother hithe +pierre pinna +nts live +not en +ni ks +mark gatiss +lifeat purdue +law lessness +lati me +kru k +kn b +hyun day +gre end +din n +del aney +d tl +combat ants +bon gos +athe ist +all that +a bet +ðŁĸ Ĭ +ðŁĩ¬ðŁĩ Ń +we al +understand ably +thel ake +te kno +tamanna ah +street lights +squ an +shul er +sh inn +seth macfarlane +ro stock +ren an +plu cking +n vw +mariai rene +kor da +kad y +itti had +hov ski +hobb its +gr ates +fern and +digital artist +ball fanfave +bab bar +alessi acara +travel news +sw g +sau gus +rou n +re booting +pre is +pe ps +ota ku +nom o +mce ach +k official +k anti +in sa +glaiz aredux +fuller house +f q +cw bb +back line +actu arial +í ı +å ¦ +w pro +ver anomtv +un happiness +un developed +travi ata +synap tic +singlet ary +sharp ness +se gun +sam berg +ryan j +ro ca +pin al +pi olo +per ro +par khurst +nc sc +kri stol +kat rin +gra dy +gn ats +glyce mia +fall back +cyber news +cor netto +catching fire +bom berman +ar ris +aless and +accor di +ðĿIJ ŀ +wat kin +ty co +tri gg +ta int +sv m +street cars +store house +sh esthe +se aly +rou ges +r co +quere taro +py c +pre zi +o oni +nyo vest +mar rao +mall on +gio ia +gau dy +ecoun cil +dan ang +confe ssor +clo ister +bio engineering +bac carat +az central +ta hi +sport stalk +ri pper +phoenix raceway +mon bebe +min ds +mal ad +kyle larson +kri shan +hul ls +hi att +h fh +ge tters +follic les +duis burg +diss apo +dele te +cu bist +corn field +con ec +cat an +ag ta +ðŁĺģ âĿ¤ï¸ı +ðŁİĬ ðŁİĬ +u op +tyl desley +streaming party +st pi +smoke less +si gep +shut up +scot tho +rose gold +reinst atement +pre sti +pil i +out performed +laser jet +lale ge +kine tic +je evan +jam m +humb u +grand erson +dress ings +cr üe +cal dic +c cac +bhar ath +amy g +amoun ted +âĿĮ âĿĮ +tol entino +terrori zed +som s +sah in +real bencarson +re introduced +ovarian cancer +nam ah +monte z +mis fit +kamp ong +ice age +gum tree +gou sa +gli oblastoma +gat royd +figue res +er ror +entertain ments +ec an +dream boat +dis membered +di mble +cro m +cor win +conspir ator +colle tt +bon ney +apir ateday +ðŁĴ Į +v ts +tiff ani +sof tened +re collections +pom pad +mis adventures +li gab +kal an +intermedi ary +hammer heads +gal ata +frat ton +ea rena +din h +big bad +be hringer +bab ad +alder ley +Ä ij +swi zzle +sri ram +sp hl +so wn +rit su +r ga +pur u +pointless blog +pide mic +opinion ated +mo stafa +mel ange +heaven ly +fort nightly +first class +essay ist +e ons +crump led +cri ver +c ór +book signing +bicycli sts +bb ys +ball ston +ap arelli +amc theatres +ðŁ¤¦ ðŁı½âĢįâĻĤï¸ı +èĩ ª +ym posium +war heads +wan de +usc cb +un suitable +thomson reuters +syndro me +ste tson +smart grid +rut ger +r nc +pro gs +pre to +pre text +pla gues +pink socks +pathe tically +musk ing +memories of +list e +kylelarson racin +ja hang +hos anna +follow vintage +du ong +de arie +bol lards +bey blade +archil overs +antho logies +ãħ ľ +yo shin +yeez ys +vv vv +the mike +stone work +shuff led +par c +osp ina +mu mmers +mouse trap +maz dar +h sathletics +future leaders +f dl +don nar +disney sea +can seco +ab use +? "- +âĸª ï¸ı +าภģภ+yofthe week +uk books +to sh +to live +semen ya +seep ing +sa die +non toxic +n ss +ma dre +kin k +kim xi +ki ef +j angp +fré dé +fo low +etiha dairways +cro s +car le +bou gie +ak leg +Ø§Ø ¡ +w sf +vali antly +u pe +titus ville +th ill +sto wers +red wave +pan et +o ea +ne mt +marketing automation +m cafe +la borer +k tt +iron pigs +hero clix +gart ners +fran ke +fate h +carly rae +cab ela +c jr +brill ant +bj b +back court +babe sonparade +adri anne +åī į +z umi +uk smallbiz +t zen +rancher o +pho ton +p tc +nav ara +mea gher +maced on +juli ab +intro verted +gar issa +ga rena +f nd +eco s +do tie +diffu sers +c tober +bt vs +bi deford +al roker +ab stain +>>>> >>>> +ðŁIJ ĸ +w lv +the perfect +sun ami +retweet tuesday +ragn i +or ally +newsma kers +ne ster +lon im +in ra +ido psis +ham es +h sg +gra u +far a +fac simile +dar rel +d nb +craf ter +cla as +羣 åī£ +âĶĢâĶĢ âĶĢâĶĢ +vol tag +ts field +tech nom +t and +stal ban +st witter +peace makers +noo dle +newvision wire +monty python +iloven y +hein lein +hard son +ge sch +fri mpong +door ne +doctor who +derma bra +ban que +adder ley +ãħĭãħĭãħĭãħĭ ãħĭ +vol t +vine et +ok in +no pinion +ker n +kal goor +hender son +grey ish +dharam sala +curmu dgeon +crab be +cade t +boden see +ax i +arab ian +ðŁĴľ ðŁĴļ +world teachersday +wil m +thi eving +strol led +stre pto +sting ers +st aves +soil health +sharmar ashmi +prince strust +pontif ical +pi der +nu trac +meteor ites +mag z +ma zes +lu dd +grl probs +fresh eners +ev asive +endanger ment +diab los +coney island +can tata +bra bus +bou lt +boo oo +at rol +amazon as +al bom +ag oura +ad dress +à ¬ +worl de +whati s +ser rat +re pack +pinto fotografia +per dana +noo t +neuro muscular +intol erable +ib sen +grandio se +fe tz +e sher +drunken ly +d back +cadill ac +bou l +bex mader +ak al +âĶ » +z ir +win star +vent as +teapo ts +team hendrick +stick man +raw food +ol vera +octag onal +ms dyn +mo et +mand alu +ly ne +le im +kim brel +gill ani +ent in +eno ist +dog fight +by passing +brisban elions +bir git +au snavy +ake em +ac v +âĻ¡âĻ¥ âĻ¡ +âĻ £ +x tc +wer n +tu uk +tapen ade +smo ther +shar o +roy ale +reach higher +prin tables +paci fist +on or +o ken +mi ps +leg en +j bc +im h +hell man +gri z +cin c +carmel a +at un +accentu ate +ti pal +ti ghe +ss ds +si ba +sas aki +robb enedict +resign ations +quo c +pag anism +oshe aga +om ur +naw al +mariairene ali +mack lin +mach in +kriti ka +gran bury +glo zell +endo scopic +da im +d able +cu si +cn p +cl ann +cken ya +brown band +bo jack +ze hra +vote green +then yra +superhero day +se phi +ro sa +or issa +metro bus +magand ang +ke ster +good man +fox star +fer min +educ ational +eat well +colling woodfc +cap ed +bww ings +burn the +ban y +ag enocide +ad x +top team +sycho logy +spaw ns +she ela +ru ffin +ri mary +puni shments +pan ting +pal es +nun u +mal lika +lip man +let itia +jody highroller +id li +hot pot +franco is +fer rig +fa sted +end es +domain names +dissemin ate +disp leasure +dem ire +council ors +citi group +bougain villea +bau x +band mate +bal list +ðŁķº ðŁı» +we sto +vi als +uffi zi +sp ud +souther ners +solar impulse +shy ness +roo o +re ma +pin ker +pin ay +pal ati +pacific o +northrop grumman +matricul ation +master nodes +ln sm +laugh in +gw v +gui ana +fis cal +den ovic +clifton ville +cham ps +âĸ ł +ver hoeven +thr illist +ta vi +synthe size +stre u +shih tzu +roth fuss +rajkum marrao +per it +os orio +oak ridge +nov oro +musi al +mc comb +ik oro +i ap +gh mc +esp ero +der ri +be tul +abbey road +ur m +tony goldwyn +taker u +s onal +pet food +pare kh +or in +mo dao +men ino +love va +lince cum +le is +form is +dou lton +corn flower +ci h +chair lift +blog share +auto trader +amin u +air power +we ig +ta han +s from +re apers +peto skey +out i +meson et +lo ar +ici est +heal dsburg +he mant +gom ariners +go ong +for texas +for hire +for go +fa zer +cor nel +cancer awarenessmonth +can cion +bo shi +ab ena +म ह +umb ai +ta ffe +stas era +shar ps +sar torius +s vo +s mex +ro co +neutro gena +neg at +mac arena +light bulbs +kam ar +k aki +jor di +hc ps +fre elo +edi fice +eat er +dream job +disgu ises +cre di +ðŁĹ ŀ +w psd +talklike apirateday +rosh ni +rock lin +remun eration +re forestation +pp ls +philipp e +pan as +ni ere +ne geri +n sic +long beach +kenny chesney +joe budden +jap onica +hair ston +go ths +funer ary +fig ment +f ps +emabaras syourbestfriend +durg apu +dul ci +craw l +blo oper +blizz heroes +battle star +bamboo z +bail ando +aust intx +ë ¥ +Î ¿ +tide water +tech radar +t mn +stro op +queen ofthe +plebis cite +om ggggg +ngad c +nay an +mi il +md pi +loy alties +ili ad +il au +high point +gal ang +gail simone +fro u +epi der +dynam ism +crand all +cou ture +bilt more +adam o +ðŁijĬ ðŁĴ¥ +x prize +well er +way fair +sym ons +sky warn +men ards +ladies night +kaz oo +hin denburg +geor gi +fun da +d sk +bren ham +au ght +annu cci +ë t +team sheet +sho k +sa org +p silo +ow al +oure ux +ortho tics +ofex eter +ni ers +mil am +mcnam ee +ma def +laid back +l mbo +kiran bedi +jav on +ha vel +ew g +dol ite +dar na +chi eng +book binding +ban jos +ab gt +âľ ¯ +yoo jung +wee den +thick ens +the secret +t wa +swi zz +sor ter +sec under +resi dential +per y +palmo live +oculu sri +nerkondapaar vai +mu ddled +lif ton +knickerbo cker +ke b +gri dge +form er +fo gnini +en im +dream like +caille botte +bourne mou +bar res +abbrevi ated +[ $ +Ùĩ Ùĩ +ws dot +well fleet +we ss +uof g +pic oult +orange army +oph on +op aths +ohi ou +mar ton +l ory +keep sakes +jhb traffic +hutch ings +gendere quity +entrepreneur life +e pee +dising enu +dex trous +ðŁ¤ŀ ðŁı¼ +ãĥ© ãĥĸ +we an +t cnj +sunny side +south chinasea +solange knowles +serap him +saq ib +sa thome +re mit +r df +pan cit +our revolution +op r +my corrhi +mi ike +mamo vies +liberalis misamental +jam ila +hen dy +hatch lings +fy fe +fi aerc +er am +ecclesiast es +de forest +crystalli zed +beatsby dre +) + +é £ +âĸ ¬ +Å ¼ +west ph +un os +tu olum +stra hovski +stop watch +road tothe +pl unk +non stop +mohand as +ma saya +lik ens +leon ora +ide alism +half pipe +hak flak +en ji +desi igner +co si +bro gue +br ith +bil la +yam mer +xi u +wait angi +vell yn +temp us +scot tw +sal ukis +ren ne +rec ou +r ft +qe ii +pun an +por chetta +ot an +malcol mx +leg azpi +lady birds +ket ts +head line +grey friars +eu council +eclamp sia +bri ghts +balik papan +archie comics +a ok +Ø ´ +vs det +swit ched +sw it +stre aker +st ela +sojour ner +sam a +re ham +rak shi +prit chett +modao zushi +leaveno one +kai ley +jo sie +har sher +ham esha +hal ston +genu ine +gant t +for rent +f me +exfoli ate +exc o +dru sh +di um +chau d +carri gan +av anti +ðĿIJ ļðĿIJ +woo oooo +twit cam +twil son +schul er +pump er +pro ve +pd k +moti v +mary j +l mb +key blade +jam un +invicta fc +helen zille +gome z +ge co +fi ero +effec ting +disra eli +diction ary +core tta +compul sion +colouri st +at ella +an ant +ah at +ðŁıĥ ðŁı» +âķ Ń +wing o +turtle day +sw k +sv k +sun less +stay woke +starwar scelebration +ss k +sam bal +por gy +pollu ters +pedal ing +mo een +ming gu +mb led +lar ose +idi opathic +holy well +franco phonie +felici aday +ear piece +citro ën +car ies +business growth +bu bs +bree zy +big thing +bend tner +bank head +au ssi +arab idopsis +afa ith +upd ate +side winder +ser p +red hot +red bone +rash mi +radio x +pom pey +news agents +ne sh +kui per +ko td +karl lagerfeld +hun e +har po +frau ght +fe brero +contempor ain +ben q +ban nock +b db +aus law +annihil ated +acquaint ances +x n +wood shop +stdavid sday +row ski +re le +par acet +miro la +man college +ki sii +ken ora +inspire them +hor ati +hodg man +hal ong +fm ri +eto wah +dol led +asap ferg +ab ac +âĨ ³ +tre au +tram ps +tn b +time in +thereal luke +srk universe +skr tel +shu sh +peep les +ni yo +lom ond +lew ick +il over +hy d +hand lettering +good night +givesyou wings +giuli ano +galvani ze +forget ful +fill on +en q +echo ed +dou sed +card holders +bel ve +ar leg +af o +ðŁĽ ¬ +ì ½ +water craft +tw omen +tu bu +tren dy +ton ibraxton +thermom ix +straf fic +si su +rac lette +phal lus +n sic +m hl +ke zi +irish water +ido u +he igl +gc w +eman ating +ema scul +elias sports +con fers +bay ev +atch ison +are public +ÛĮ Úº +weare bc +van te +un lawfully +typic al +thro ck +sumed h +pro mul +pre d +phan atic +pen ge +new born +moor park +lu ang +lock up +lind as +kir sch +is wonderful +ger st +g pr +fast post +diabete suk +dc p +cy st +con nell +bo bo +big gio +witt genstein +victori abc +vi ajes +the bes +tar r +re ignite +pon yup +out performing +my be +kr ingle +keepitinthe ground +interro gated +ghos thun +gant eng +dio des +dhar mamovies +devil man +carnegi em +beu lah +ant or +ðŁĻıðŁĻı ðŁĻıðŁĻı +voten o +u icide +the social +stag n +st outs +soul music +ratt les +qu be +pru e +n online +moon byul +magni fier +mac neil +lil ia +kon stan +ily as +ik shank +hen stridge +gu cc +faze up +cor relations +cham bered +caf a +braun schwe +bb cy +b indi +am mi +ðŁĴħ ðŁı½ +vi vor +vare se +ta pir +spe ach +schiav one +sam buru +rose marie +q ms +philli pe +over cooked +on point +mtv teenwolf +mo cca +le febvre +jess y +i gen +ho to +head scarf +hazal kaya +gar cons +dru mmer +cur ricul +ap hid +am eri +ah ps +о Ñģ +whee zing +tam pered +soul j +shaz am +pap ill +nadi adwala +mor ts +mil ag +joburg za +jen nette +in at +hoe ffer +ha ylor +gol dust +fri uli +fertil ity +con spire +co ty +cityof joburgza +ch ac +bu bu +ðŁĩ¹ ðŁĩ· +will and +white washed +suri yaf +sun burned +summer vibes +sr g +si re +se gue +pur ve +po what +ou ster +opp ress +o donnell +mur i +market watch +ka ir +jazz master +j lc +hu ss +ger ais +elo g +e ject +cho sen +boston terrier +baahu bali +ઠ° +xxxx xxx +ton en +the family +sub strates +savi ors +sand banks +quel wine +pel vic +one onta +nur ul +news live +n pb +mat ar +le eroy +jmi quelwine +char ley +chante relles +brack ley +art as +ap f +aene as +aap tards +ç ¬ +tro po +til ton +tan u +ster ne +stefan ia +scar sdale +ra bia +post mortem +pen ob +nk jv +mu sha +mor rill +mal functioning +jag dish +individu alism +im presi +im at +husband ry +hiking adventures +heritag elottery +free assange +diyar bak +cro que +bear dy +west ley +truck suk +tran scribed +ti vo +shi khar +she f +pour quoi +pinck ney +nam joo +lexis nex +ladi esof +kun d +keep on +inspiredby pets +he dral +ge ss +fry denberg +dominic ana +def y +cour gettes +bnpparibas open +all size +ad lington +absc ess +ve toed +v log +us opengolf +tin ley +tech i +strictlycomed ancing +so kol +sil ences +pic u +p nd +or un +or l +no th +meet and +jennifer lawrence +huay ra +hu evos +fant abulous +fa ery +but land +bul lah +balth azar +ba ster +ar pels +v hp +un sightly +t ny +sag amore +ri jeka +resc ent +pokemon sword +mar rone +mag alona +kra bs +indic t +her mit +hem phill +erkenciku ÅŁ +e dem +den zel +d jer +bre mer +blac kie +bir nie +big boy +be si +arabe sque +aap ke +a erie +à° ¸ +ॠĪ +ö ping +week long +ur ination +un bothered +tropic a +tb k +super saturday +si rac +scri m +ru an +qu als +pe avy +ow w +nu ova +new menu +little things +lepidop tera +kil ts +ire v +frat er +footh old +du tyfree +corrup ting +cor aline +conven or +consul ts +cl amp +carrie fisher +bra himi +br annon +bab by +ap ics +an thea +à´ ® +ب ÙĬ +wou ter +west allen +vi bra +tsun dere +tin isto +thic ket +ter io +su z +sarah k +ph lox +nick els +nat sume +ma sandawana +joh na +i fl +extinguish ers +de spe +cunard line +cor ley +class man +chang eling +bio logy +ap supdate +???? ????? +: ** +ðŁĻĦ ðŁĻĦ +w fa +to we +the dress +the day +ten emos +so viet +sam ine +queens way +pho tonic +n annies +i ae +ho xton +hel met +ha be +exam en +ethere um +e ks +de ion +day project +ball mer +as me +aber dare +~ ^^ +wy clef +u wh +the magic +stan dish +st ich +ss np +rc navy +pile up +pedago gical +pav el +p ings +n mb +keyston exl +gin ni +gam ers +fer rier +ex if +du plo +dillon francis +dash berlin +chi vers +carm ella +bre chin +bloom sburg +ar nt +aji thfc +ðŁijıðŁı¼ ðŁijıðŁı¼ +ðŁı Ļ +ìĿ´ 민íĺ¸ +the alex +t ance +soho theatre +smart city +skyline skaters +ro atan +nu vo +nouvel les +ms j +jave c +is let +inspirethem retweettuesday +gov au +ge ver +g ling +dermabra sion +corn flakes +coo gee +ck nw +chul z +candy land +ðŁij© ðŁı½âĢį +wel k +tr onica +tex om +supercross live +shin ola +san at +pav arotti +pan cake +lunch break +ic g +hi bachi +head room +han gul +gir aldo +gen isys +elo pez +bed time +bar won +akin ator +ahlul bayt +í Ĩ +zac brownband +w cu +stou demire +states ville +ser p +se dia +scru b +schle singer +rick ard +refe c +propri o +pro cycling +ou ya +ma ston +health tip +h mm +gradu ations +for tify +fitz simmons +far oe +fang ir +collu ding +bit umen +aram is +ðŁļ´ âĢįâĻĤï¸ı +zig er +y ell +ve ttes +tree house +theor ville +telefon ica +sub version +sizz les +shi an +secre to +scot national +sco ps +sal ley +pflu ger +peace ful +ning bo +mor kel +mo ise +kra k +kn itting +hul man +gwyne th +ge z +fe c +f ête +c aceres +bjö rk +at twood +as cor +armp its +animalsin churches +& - +̵ Ì +wales rallygb +tink off +tb t +shin ers +rock man +ro skil +ram ped +ra bil +om agazine +nu st +ntl museum +may war +le wa +huer tas +ha gley +frigi daire +flori das +bu dg +brock ley +bly th +am ath +åı ¤ +zin nia +wyn ton +work men +wag oneer +ru stom +resi stor +patt ys +new suk +nan di +mock ups +laid back +jer rod +j oun +inglori ous +helve tica +go transit +chint skap +abdul rahman +ðŁijĬ ðŁı¾ +ðŁ¥ ĵ +z adi +wonder kid +wad er +ro vell +rafi que +pi el +or gre +o da +newyork redbulls +negro es +mother well +li gat +ki a +institu tional +im pulses +epi per +cu ba +commiss ary +big news +az is +arse guidores +wo hl +west mount +voy ag +vit i +valtter i +tinisto essel +tan tra +sto ddard +spres ents +pre quels +pran ked +ne gril +love eeee +la kh +jetz t +hel vellyn +har grave +dun barton +buck scounty +bauer hockey +aw b +asi fa +as v +art print +ar al += # +ãĢ °ï¸ı +ô ne +verme il +unfur l +tc disrupt +sat ch +sapp hi +ou da +oh c +na sties +man liness +ky w +jo sue +inter op +import ation +her alds +f nm +education forall +dimitri vegas +de ana +day yy +datam anagement +cy no +ct ags +confi dence +cho ge +chang sha +chab ot +bi gh +beck i +bae kho +b enda +ðŁĶ¥ ! +z ong +ward ell +u dc +ticket master +the ss +symboli zing +sch muck +salt illo +rive ter +pre history +pe ven +pain relief +over powered +mis understandings +mark tremonti +l ats +kemp t +ili ve +h sk +gir th +eve rest +dol enz +doglo stuk +cost adelsol +che ep +bc age +banan arama +anti etam +ì§Ģ íĽĪ +thi every +sp unk +skel ter +secunder abad +sav arkar +re upload +pt k +pelle tier +peep ers +our self +oku ta +never know +mitsubi shi +le dges +john stamos +hindr ance +harmon ize +fau n +er ases +duncan ville +classi fications +califor nication +barstool sports +ðŁĩ¸ ðŁĩ¦ +¦Ī ìĽIJ +vs ne +vil ma +v ater +think big +str al +sta thletics +speed y +selfish ly +sad dens +pride inlondon +pressuri zed +pre ece +nicol l +na ï +mm pr +mj f +mega watt +l zzy +l ks +hou d +fi zzle +cole optera +ch aca +carcas ses +yo kai +ym phony +y one +ww j +wat cha +vir g +scep ticism +rc ti +ra fin +pic ts +patron us +ni fty +mess ner +merry gold +ma hila +lor dy +hou sley +hom i +guadag nino +glo sses +gim na +fil my +di ssing +daniel j +ci f +bad ri +adju vant +trek bikes +too funny +than g +ten ney +stri b +sin ab +ru sev +rhe umatic +residen cies +ren jun +pathan kot +mil ena +lin dos +libr is +le mma +k age +hy poten +hur dler +fanci er +expo west +aug gie +ar ted +an w +accur acies +a ek +à¹Ģภ£ +ye ahs +win some +un adulterated +s nel +publici zed +pren sa +pos y +patri o +o cre +ne cker +hann a +go lightly +glenn beck +explore r +el ita +di sta +delephant day +debon air +dawg pound +cone jo +cc v +brick man +bc it +b pt +alli a +ab dn +ãģ Ĩ +âļ¡ï¸ı # +wet suits +vad os +thelasto fus +sun room +sin do +ser re +rob zombie +region ally +ra uch +prs guitars +on elast +no ct +nay apakistan +mu q +mo vin +ma ite +leavenoone behind +lake head +l vs +jau me +gre itens +gr anda +firstrespon ders +down beat +di mash +cy nd +ct c +crimin alization +chriso donnell +ch b +c ades +us al +ting gi +shon dar +s rising +russell ville +rp crd +pu es +ntlmuseum sscot +nick erson +mika el +mand an +mainten ant +listen live +leader shi +ic hin +hugh ie +hod son +gun j +first time +essendon fc +d apps +crad dock +by un +bu ehler +as signing +antiqu ated +ad dio +ab acha +/ âĤ¬ +ðŁį ½ï¸ı +è Į +zh ny +race horses +ob r +nor co +my cen +mur u +len c +ka en +j mc +j dr +iran talks +icha bod +encel adus +emmy lou +doper ahouse +dige sting +day sfor +cher u +cast elo +black book +al mirola +Í¡ °) +ze iro +xi o +un altered +tra desman +tiber ius +span ked +sha ho +sal len +rabo bank +ma der +ke ren +irresi sti +gan on +g by +far fromhome +ever e +darren atics +chennai yin +cedar ville +bo stic +bla det +why te +wa ig +vi dar +urbandic tionary +tal c +stret ford +som bor +skir mi +scam ming +rec ali +pic tou +p lies +nil erodgers +locomo tion +kar don +kag iso +iz h +hon iton +ho wie +gas ly +g ci +ent endre +b ami +yam ig +v st +tran scanada +toend abortion +spondy litis +sky rocketed +player one +oscar del +offici all +nu ms +mi umi +lo vi +land locked +ky les +juli eta +jac ke +hard ing +fine baum +ess ar +emptythe tanks +dun gannon +desp acito +cul ly +cow les +clover field +clean ses +casca is +bus k +be same +arl berg +al bie +ag onist +wolver ine +valtteri bottas +tomoda chi +the wild +the play +termin ating +tan may +tall ships +ta queria +stonebwo y +sta hp +sou rav +sh allows +ra ison +pan day +nam ath +mumb aic +mor ricone +medi ators +lon dra +h ould +e mus +demi god +dee per +deduc ted +cru ikshank +correc tor +chichar ito +cer cle +backthe brits +asper gers +as aurus +acci on +w ylde +un dress +tro ms +tee hee +raw son +pollu tes +pi ri +oro so +ni mmo +me taco +kill y +juilli ard +iihf worlds +hockey canada +gusta f +ge ddy +faul kerson +f sog +elizabeth town +crowd sourced +cnn philippines +ba aaa +ash ington +ap ni +aha h +ì ħ +udhay stalin +tra eger +te rest +spi ros +so com +pr or +petre l +or ita +not ables +nai jap +monc ada +mol t +mo sel +mediac rooks +kari joys +k ente +ig man +heal thre +goo o +fro sin +do ji +dan is +cur i +creep show +creati vity +cliff side +chil islove +cham ba +cele ste +be tro +aven ir +are se +ðŁĺĤ ðŁĻĮ +ìķĦìĿ´ì ¦ĪìĽIJ +u da +swimming pool +st impy +se ol +sar di +redu ce +passion passport +pappar delle +nit v +new collection +mil burn +make my +kam au +go friars +en core +ellen berger +den on +dal by +cri pps +cooke ville +be u +barbac oa +ari ane +ðŁĺĦ # +zi le +world music +wo wing +w br +w ades +strogan off +sax e +s mee +myri am +moon light +mo ko +mn dassoc +mirad or +lobla w +lam beth +jeff sessions +har una +h yeon +glu ing +game jam +fascin ates +donagh y +compens ating +chennaiyin fc +cas se +bi um +au die +anag an +ag d +* - +the pink +sto y +sli ppy +scham pion +sag as +ra sia +ra az +non ame +nb g +mer gan +marin ating +kr c +know sbest +is enberg +fa king +er land +day swith +coloni zed +at orian +amer sham +:) :):) +ðŁĻĪ ðŁĻĬ +ðŁİĦ âĿ¤ï¸ı +yas por +van s +under class +story boards +so true +si bos +roy ally +pi dgin +not as +mand rake +mal as +ling ual +kari us +k ry +it sabout +hugh laurie +hist sci +datac entre +d hon +bro iler +bis was +basti a +as pr +am tv +ac ry +ê²½ ìĪĺ +wol fs +ve sta +tor ship +t po +t ela +sg dq +san deep +sa ia +ru sse +randee phoo +py jama +pre owned +pax man +knowh ow +knop fler +institu ted +home building +hol sters +end polio +dun ker +ds world +do sti +dixi eland +berkham sted +bat son +bal ert +amand apalmer +all time +al ys +wi relessly +thank ateacher +superst itions +sec u +ri vets +pole star +photography isart +penguin random +olim pico +o cal +nor ovirus +much hal +mlp season +michigan stateu +matryo shka +lu ms +lu ann +kuro ko +hu th +far hank +et as +do gged +di dy +dent ity +dark wave +cruel ly +cr inging +capit alizing +bizz are +beach boys +american gods +al aves +zach braff +un important +u vb +the wrap +repleni shing +pin ang +pi guet +pha blet +per spex +pelopon nese +pe tere +paracet amol +marty n +lat ingram +ir ala +gi ada +gemin itv +gal ahad +ery thro +el stree +const antino +ch ali +car den +bro wer +yu chun +world wetland +vibr ant +uri el +tim ms +tanyabur r +tablo ids +pau lus +para ben +oz plants +manitou lin +intrin sically +i ros +hol by +gaz es +food bank +flu ted +farhank virk +co ster +brian stelter +bil o +ben atar +arch duke +well deserved +ven lo +v á +un informed +tre m +trading cards +sp lu +shondar himes +san ci +re vision +punctu ated +pott stown +ot rends +nishi da +in za +ig es +g music +firstdayof school +espo o +cath cart +c aya +ast ounded +app el +am ik +action bronson +ðŁĮ ¨ +war craft +w mmr +super models +st ich +spor k +sc ituate +pal er +leil ani +lant ana +la fit +kar th +horn church +gat land +fir ming +fal low +den za +de vising +day er +cher ubs +cab i +black comb +athle tico +any an +anten atal +å ĺ +Ì ħ +wi ese +vig no +tattoo ist +s media +s mbc +run way +resur gent +re plete +rd v +ra bly +r mp +pit ty +pis co +pan erab +nj e +lets ride +l ally +k lay +imbi be +here tics +haw kes +go bills +extra judicial +car port +bird sup +bf v +b sy +appointe es +è ¨ +ãģ ¡ +Ë ¢ +z iti +w ence +uuuu uu +stev an +skag en +sech skies +rin ds +pu ggle +oni on +ob tu +mer ito +kis sel +king maker +goo f +fab i +ex alt +drex ler +co del +can io +c sir +brook land +bre c +as king +as ado +animat ronic +andre am +alleg any +acces sto +yas u +y cc +stin the +sch aaf +pati entex +nathan son +mc vie +matt goss +lorele i +kom bi +innocent ly +illu si +hosp ice +gr dc +cw t +coronation street +c end +bi ddy +apprehen sion +anime art +ancient aliens +. âĢĭ +âĺº âĺº +you version +voteredto saveamerica +voet bal +ur c +u gs +su mn +self publishing +ro el +ref illing +re generating +peninsu lar +parker sburg +pan sexual +music uk +hus kie +glad ness +endo thelial +earth en +dram atur +bar negat +aq ha +ani tha +and al +al ag +ye quipment +un failing +tu dyk +ti mbs +th old +stra p +st ly +single handedly +sid har +red hawk +power ful +pou lton +phant as +maur ya +mal ai +load shedding +i acp +hamp shire +h tl +girl crush +fy ne +found dog +do wer +ander ton +z ink +yez idi +world photoday +whin cup +uu tah +tri ggs +sig nups +reen act +rafa h +n phc +min dedness +mc entire +kru pa +kni ghton +in town +grate fulness +gi one +en dive +c ji +b ws +are wa +allank lownz +. ") +ðŁĴ¤ðŁĴ¤ ðŁĴ¤ +⼠· +zombies quad +v fx +turn stiles +ti guan +si kka +shat ru +sel o +salv ad +red shank +r football +one ys +obam agate +kal in +k con +gree ter +extend able +beg ining +aver age +ari ann +ak om +ðŁĴģ ðŁı»âĢįâĻĢï¸ı +ðŁĩ®ðŁĩ · +ur p +uni fied +u mah +the ia +schre iner +schaf fer +san e +rejo ices +portlao ise +ntv uganda +min ke +massi mili +mari juan +lma oooooo +leis real +jo dor +immigration reform +illusion ist +i wa +h tv +fren chri +fe ction +di ure +dg ingly +d banj +criminal isation +cr w +bu p +bo ban +black women +av as +alpine stars +å ´ +you ss +y gg +tat ay +stop light +sick ened +sen de +sag ar +oculusri ft +oak ley +nor den +mash pee +liv uni +kam akura +heat stroke +gre ggy +fo ward +fc px +em ura +den n +dec ry +cap ello +buc s +bu ono +bal khan +zeit ung +younger tv +wee tab +si sy +se el +rv life +o ho +neutr alize +merri ment +m vr +long boat +lay in +kinder gartners +homeand away +historical romance +gen eliad +eric ho +asser ts +abhi man +æĺ İ +âĮ Ĵ +wordpress dotcom +winter watch +w gal +vi da +valky ria +universi ade +tt inger +tillam ook +tamar aws +stra us +she ehy +reit man +re de +pan gea +nhl playoffs +male e +ma ite +joseph morgan +ix els +ici er +fe ist +fair haven +epis co +dat av +dar ken +dal matia +cur zon +cityof pg +chec s +char lo +can tante +bas c +androgy nous +ac risis +ab iz +ãĤ¤ãĥ© ãĤ¹ãĥĪ +wh er +tube less +ten ant +tell tale +sun dog +so red +sigur dsson +sig rid +samsung mobile +rat ner +randeephoo da +quote softheday +pitts field +mu tombo +la wa +l sl +j vm +j ii +ing mar +hard woods +happy girl +grace helbig +glasne vin +footy show +fa wl +choo ks +c and +bo res +berser ker +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ðŁĴ¥ +y im +vo ici +vai bhav +tt n +that cher +supri sed +su plex +si op +sen bob +sa wed +rr l +ri gat +q so +pro tour +pat oo +nat ed +mis d +mar ke +ma thon +ker at +hypno tist +go huskies +g atta +es group +em pt +dol ittle +del ancey +cour bet +confer ring +carlyrae jepsen +canon usa +beat ings +a holics +ðŁĺŀ ðŁĺŀ +verte bral +str ac +stat eline +regin eval +regineval casid +real blackcoffee +on myo +offici alo +mar dan +lar osa +k ny +in conveni +ilove it +hi ms +hand ily +fan n +fan acc +f to +ech ever +de activate +cn w +camis ole +ay outh +ar wen +all and +ab ot +ðŁĶ ĭ +wr gb +wh iny +toy fair +sun glasse +ste x +skull girls +rab bids +of hope +mumford andsons +mou ton +mel vyn +mc diarmid +le mont +ki ir +ja wor +ef c +dyr dek +broad com +basspro shops +ar bon +all kpop +ðŁĺİ ðŁĺĤ +yrs ago +un dressed +ste pson +short ness +se vic +respon dent +re decorating +pessi mist +ob en +ni f +lland aff +ke f +hurricane maria +h pp +grenad ines +ful fil +exam iners +equal payday +daysof ourlives +chec kitout +bell flower +befri ended +beaver creek +azerbai jangp +all sopp +aim an +whis ked +ur du +sho tz +seque ster +sau ter +pro tracted +oy w +oun cil +on ight +nit ride +nca as +kim brough +kh in +home forsale +gra ber +gol die +flui dic +erst while +der vish +con temp +child hoods +captain americ +cade au +c ft +bron c +bri ones +ale vel +agar den +adri atico +' ), +ðŁį ĸ +vivid sydney +up stat +re elected +re cluse +quad ra +prime knit +play fully +par ik +ny r +mill iner +mate o +kil ian +jin shi +ine quities +idin amen +flim sy +d wayne +bi dge +bare foot +bannock burn +amu st +ag ut +ade kunle +ðŁĺį ðŁĴĭ +wic ket +tur rell +tr all +stu ttering +smo thers +slu gging +sl benfica +sam ut +saj jan +re turner +ran unculus +ow asso +litho graphy +le son +jef free +ha das +gurud wara +gaspar illa +ffff ff +fah my +es ny +dha ba +de bru +content strategy +canonusa imaging +can tin +besto ws +benz ene +ame er +al mir +Ñ ħ +w uk +te ena +spand au +sl acks +shra van +se er +ru x +re can +popp unk +om arion +ob gyn +li ppi +i robot +gun ship +gu dang +good to +for gettable +el isse +dis lav +cc ma +bud da +brod sky +britt a +bon avista +bet we +arth i +ar vada +acor ta +ä¸ Ń +tweet storm +sal u +ro mu +perpend icular +partofthe pride +o dometer +moncri ef +mad lib +lur ch +kon go +jam il +injec table +hu y +gal lego +g afe +freen az +dunbarton shire +disney infinity +da han +bar ingo +ballant yne +ba je +al ors +ab devilliers +ðŁĴķ . +ðŁĩºðŁĩ ² +yar brough +whit erock +vee am +tw ales +sk ai +septe mber +ring git +red sea +ra fred +quig g +pollin i +ofthe world +madein britain +kz n +kalin ingrad +j sw +hawk nation +h la +glen rothes +em mac +ear nit +doug ducey +condo lee +as sis +ane es +acci es +worl dradio +veronic amars +tele prompter +tee public +sailor moon +rat ed +mon ast +mark it +makon nen +mad ness +leh ner +k ca +info en +gi ms +ge sso +fr amer +fi era +f sb +down ham +darshan raval +daddys girl +ab hay +vicari ously +twee p +tom aso +tik har +season ality +prime val +onec lub +nargis fakhri +me te +mag fest +fre ida +fat ma +donington parkuk +corpuschri sti +confe d +chuck ling +bridge town +b halla +anticip ates +! ðŁĺĬ +zon do +worl delephantday +wis ley +win c +unsig ne +su cess +ra gg +q ar +olim pi +linde mann +kali l +irrepar able +gab bie +free books +em lyn +e brahim +dam busters +cu pola +cham berlin +bro co +av atar +at albihariv +amar nath +af ish +.... .." +" ?? +wb ko +vel ly +tho b +streas ury +stop km +sec tor +ride sharing +plum mets +mill ville +mary beth +mar bury +mal ini +is chia +im pu +haver ford +happy womensday +gh ero +fo e +expe dited +charle voix +cc p +ca o +backthe birds +ab bs +ðŁĺĽ ðŁĺĽðŁĺĽ +Ùħ ÙĬ +walk athon +ver on +tra ktor +ton igh +tes ers +ss ons +sam and +repra p +o bra +nir mal +niq ab +national park +mat adors +mal evich +g sn +dun lo +dh fc +de tal +citi field +ce ded +cait lyn +ausv pak +art fest +appropri ating +all women +z ella +web hosting +to pra +sy s +spiel man +snapmatic art +scent ral +refr active +re frame +pat ern +magic rock +khush sundar +hemp field +gab i +g war +fort inet +dark ening +chick lit +cer velo +bag gio +ap t +ðŁĺĺ ðŁĺĬ +ب ر +´ ´ +wel ford +uck field +td garden +spi vey +septic eye +roll wave +reboun ded +raf ale +pu rohit +promon tory +plu cky +museumo flondon +mu fc +moon walk +le sham +kol lam +jessic am +head winds +fre mont +fla ked +fit ton +eto ile +brain less +be tel +ar be +ðŁİģ ðŁİĦ +âĢĭ âĢĭ +wei maran +wat ts +wagen ingen +w mo +tual atin +tro d +til de +strategi ze +stewar dess +stan sfield +propor tioned +per ot +official aldub +mun da +mo ong +mid lands +marine tte +k roc +ham idi +gri pper +gob bler +go ins +euphor bia +dreams cometrue +di adora +def lection +cyan obac +collin sville +claustro phobic +ce dia +cal lus +buri ram +as jad +à° ® +uc ine +tun s +tory canvass +sun tan +ste deli +sensi bilities +seed less +sag al +ruby rose +preten se +n fb +mon tes +lo sal +lar oche +kar isma +jen s +gru dges +fore al +excav ators +enni o +emboli sm +el dora +di able +cou se +cor ic +carr boro +aa e +a oyama +zo zeebo +zar co +val eri +uni ofexeter +tram onto +tra sk +t dk +subli me +ro ys +resurrec ting +pro vision +mari sha +mare mma +looking good +lex po +kutz town +kop itar +jo ed +jay ryan +inferi ority +hil le +gol da +fashion police +fairy land +ex im +euro pol +clif bar +cir illo +brit to +atul lah +agor as +accu radio +. » +wood turning +un disturbed +uk h +sp liff +sher rill +sh elli +sale stips +sa chi +s ld +radio logist +o sten +nan ette +miami dade +lat ic +kil roy +ki zer +kh en +ke shar +j ci +green building +g md +femen ino +empan ada +candle sticks +bye bye +bul loch +blo tter +around the +alli ving +wal u +um or +ther ton +tal war +ss mann +sha ile +run t +ro ze +p ander +ny lander +no zzles +naga i +maz das +martin i +ly ca +loving it +ki owa +eras ers +cas save +bis co +am ini +íį ¼ +ti gard +th ig +stateof mind +slu ice +sitting bourne +sham bhala +red list +quiet ness +o iq +nbas ummer +metax as +mat ts +mar ling +ma ad +li ed +j ina +inter laken +inte xas +hand shakes +gall bladder +g br +far relly +boston college +asyn chron +ar le +antiques roadshow +and ed +an ahi +ador ns +xi ang +world vision +wood hall +rutger su +ro main +pronoun cing +piec ing +nai vasha +mishaw aka +lamp work +jay ce +ivan hoe +indivisible team +idol master +gab s +final level +fe tc +f jb +di sses +decision making +cro stini +cour sing +car ves +an tof +wine spectator +ver it +un kind +spinn aker +sle p +seper ate +pre loved +ous mane +min econ +mal zahn +love day +lav azza +kl inger +kac i +for us +f ú +f se +et tore +deer hunter +cand ela +bobble heads +bel tre +ban do +bab i +b illu +acu te +z sl +wat ling +tele com +t gm +surpri se +super valu +sevier ville +sch o +sa hi +ren dang +regi a +perpetu ating +par veen +mood board +mer lo +me go +kom al +ki efer +in extric +i dar +hu ish +gon do +foot notes +cham bord +blizz ards +bbc africa +b fc +aq aba +ais d +ðŁĽ ¸ +wal nut +un selfish +uf ti +timoth ée +tewks bury +summ ation +stephen asmith +so dom +selec cion +ro ya +repa ire +prosp ero +pha i +ou ston +o zy +mel vin +love thi +lamp shades +kh t +k eng +ir ua +in cur +iam steveharvey +howe y +hom icidal +he cker +feed backs +du pon +de be +blood thirsty +ar ni +and uil +Ä « +y eng +we izen +springh ill +sp rig +sch ler +np bot +min aret +maha shivratri +littlemix offic +le van +lab ours +jj ong +iko shi +hy olyn +hat o +ha sten +d mn +cycl amen +chicag op +black heart +bl yn +barne veld +ambi valent +ðŁ¥ Ľ +w bal +tu ft +sun downers +subsi diaries +set tembre +rel td +plan ed +mar mara +mad town +liv uni +jar dim +jan is +harry hausen +eu a +est reno +do able +dissi dia +dis ordered +ca at +annoy ingly +al ax +Ä į +ww y +wr ing +ur ner +twee d +tw ire +thought fulness +sho ji +sar co +pho gat +ohio ans +ny rr +nov a +north westernu +nac ac +mour ned +mam mukka +mal tesers +lan sing +edin boro +dr ones +depra vity +conor maynard +cherry blossom +ch oli +biophy sics +asse en +( / +vi ento +sri man +sf chronicle +schol z +row lett +ribb on +ren ga +rec tal +rascal flatts +mi v +materi alize +mag say +koo p +invinci bles +imacele brity +hello ween +gor ica +gi ge +fire starter +fe p +enqui res +be jeweled +ang ana +albu mo +si sulu +sand paper +re designs +raff i +quad ril +over paid +n gw +megam all +mac ie +he avies +ha aaa +h itec +f dd +by catch +bla in +ax stv +ar ocks +ðŁ¦ģ ðŁ¦ģ +wor ke +ve stas +shin di +percep tive +p wm +ncss bethebest +navig ators +lu men +ligh tup +kak amega +jake owen +in conceivable +ha gee +green hills +got land +garda ÃŃ +docu sign +dalla spd +com mas +bra gged +biz arre +bat ov +ag nes +aam u +Ä Ł +ulaganay agan +s ited +river ina +palo alto +o shie +never more +n land +mc coys +maxim al +ho bie +h cg +frome arth +exor bit +exe ge +copy rights +clear field +clai mants +cau sation +bu stam +boo zy +bon hoeffer +bam m +aw ur +?! ?? +wholesal ers +super sunday +richar do +re packaged +pr iti +penguin ukbooks +pas aden +ot m +nigh y +mi ao +maz ari +ka oru +ju sth +incre ment +green man +glenfidd ich +for st +f ourier +este e +e speci +dallas news +cuad rado +c pl +bu chi +brace bridge +ben guet +bella ire +b heem +aro oms +abi ke +Ñģ п +toyo tac +thir u +team envyus +star sky +sol ent +smar ty +shine y +ric ki +penn sylvani +montepul ciano +me sports +kail a +j one +ine u +gun controlnow +go slings +foot fall +far rier +el ucas +el nella +de composed +ch andy +black ford +beat rix +alma gro +adden dum +ad ress +abduc t +vidy alaya +vent us +trol ley +tin tag +speci alt +roo sting +pur ported +pa sta +openstreet map +mu ang +maxim ili +led bury +kel seab +kat u +k weli +is ra +hoard ings +gc b +fu ze +friendship goals +cyr illic +creepy pasta +ce zanne +bon zo +bo thy +blur ry +aziz ansari +ami right +ys weden +woj ci +va shi +thevamps james +stee pest +shahi di +puneeth rajkumar +pack aging +over valued +mu tha +motor ised +mend i +la an +k old +jas pers +idinamen zel +i vers +gas ping +elec tors +dur rani +col li +chi est +ch utes +bmw motorsport +blo bby +wend t +week ende +us weekly +type faces +tor ts +spr i +prank sters +pancre atitis +pak ka +im pro +heart day +hab sburg +fresco es +expedi achat +car pooling +be jealous +a iga +ðŁĺĤ ðŁĺľ +ðŁĴķ ðŁĴĸ +ys c +w annab +tra ger +tor us +the bar +sy nes +swi the +subordin ate +sin clar +si ab +sel ing +scienti st +s rule +re told +r inge +profe ss +pra chi +nat al +ma soud +ma ble +lou pe +load ers +j wt +ice vic +hebri dean +fountain pen +fet ches +de ems +child labour +bo ren +adu ba +vi f +torpe do +sla inte +sar ada +ono van +maxine waters +mach u +intra venous +housel dn +gwang ju +geo graphies +gal eries +fein berg +e my +cross breed +cre ston +consisten cies +col ou +be mo +b hel +au tre +au ch +astro biology +air strip +ag andhi +advantage ous +! ðŁĴĹ +x ts +uzu maki +tin foil +teenchoice awards +tad ashi +sonymusic south +soci ale +se urat +san tee +re th +ppor tunity +newsad elaide +mol en +metallur gy +jamiro quai +ir anga +hydro therapy +g les +fran che +fra se +eri sts +dam as +biele feld +aller ini +ðŁį Ŀ +y ax +trans media +sur y +summer tour +su iza +si ra +sh ada +reminis cence +pro tists +o soy +nf ld +mar mont +magic johnson +lan c +jessic aal +hur ley +had leigh +ha dron +gui seley +fo td +b bau +au berge +acti vel +ye m +vac caro +under study +un fulfilled +un ca +su chet +seaco ast +ready playerone +ram ey +plussi ze +pai va +newer acap +min oz +m pe +li ske +legion ella +kom men +kate y +iv lp +in m +hr vat +finger ling +ea thealthy +e jer +disinfect ant +dark horse +cro que +cow bridge +ast an +ðŁĶ Ĵ +ðŁĩ» ðŁĩ³ +ðŁ¥ ¤ +ÙĦ ÙĪ +un clean +tuesday treat +transcri bing +tain an +sing hal +sher rie +shako pee +sarab are +s ward +ro ams +r ct +plane spotter +ol x +off ame +n als +muñ oz +me chs +maz inger +m hd +len ow +ku bert +know the +hann o +flat iron +er ys +en chant +conquer ors +cd x +bu shido +bonfire night +auto bots +audrey hepburn +as signs +ak ara +tit ania +sub han +stat oil +som alis +pun cher +pe sci +pal as +noir vember +mathru bhumi +li mber +fo iling +ffxiv snaps +ecoun ty +dou cet +deli c +ble tt +bar ham +aard vark +. ðŁĶ¥ +un affordable +um al +ty ke +the war +she eps +sc old +retin opathy +pol ski +l illi +k you +jan ina +indom ie +hor wood +ho gue +glob alists +era iders +embarc adero +co ddington +canvas sers +bird seye +bein sports +art an +amaz onia +am studios +allevi ation +alas kan +al vi +ðŁIJ¾ âĿ¤ï¸ı +ಠµ +à° µ +yen press +ud f +the golden +t kd +sequo yah +sap teched +ray na +ra ad +py ard +ph m +p yo +oli phant +morning news +mar den +mandalu yong +lu mina +irrefu table +i wi +e oy +di dier +desch amps +cornwall hour +brooking sinst +bor romeo +allthe time +adr ille +work spaces +train er +su th +stand swith +sc ola +ru mm +quag mire +pad er +ob or +nu er +motor ways +mohe gan +mi en +me mp +marke dly +ku chi +koth ari +kelseab allerini +gi ana +geom agnetic +fu m +fri se +en ick +di vide +cyber sec +clá sico +bro c +be fully +au stral +atu ral +yoko ono +university leeds +sti glitz +shre wd +restaur ante +oo ja +oh tani +monte zuma +mit i +marsh mell +lo zi +kkkk kk +gov mike +el ane +e pr +cra ved +cr anium +cc as +boy ce +bo gged +bill erica +ar sen +amp stead +ðŁĺĤ ðŁĺı +ðŁĮŀ ðŁĮŀ +z j +wo ve +win a +walla sey +w swan +tin ie +thr anduil +tal mud +stom ach +squ ished +small youtuber +seri en +salam anders +s ness +one big +lloyd minster +kim ble +kas sandra +joey bats +hamp son +gli zzy +gle d +gg j +es cott +erick a +e um +de gale +da che +confis cate +bul gogi +arthr itis +ali x +af er +à®ķ ள +war mb +vander meer +u in +so co +oiq fc +lu gs +ll bean +ke ma +k rush +j mp +hi x +flori stry +convolu ted +cle a +chil ies +ar vin +tin dustry +th une +syri ac +survi ve +spark lers +shaho ffice +sem ites +sag er +ry le +re kt +ra ita +quad ric +psilo cy +path ophy +oak well +ni antic +n acion +mis using +lpr tg +ler i +k music +jet ti +god wit +gn ition +fer vor +fel ter +fe mail +dream world +disc ou +destination wedding +de clutter +curly hair +ch hs +c gc +bournemou thecho +bil ge +ac ac ++ - +ðŁĺī @ +women shealth +wack y +van wa +twee tuk +te wari +te che +swal edale +summar ised +psych ics +par os +o euvres +mill ward +long march +ke k +ka sem +hower ton +g su +fon ds +de posed +crack head +bad en +arri er +ann en +ìŬìŀIJ ì¹ľêµ¬ +âľį ðŁı¼ +zax bys +z df +terremo to +tann in +se ph +rebec cas +prioriti zed +octa vio +i funny +haqq ani +eu m +ef o +dan one +d lo +cordon ed +che p +bel itt +anat oly +al h +ste iger +s friday +present able +mar ama +man on +ji th +jaf frey +ha sa +glu tamine +fre shies +foo ts +el den +dese ret +d drive +clear the +campaignfor leo +bangsam oro +angla ise +amand at +åĨĻ 羣 +wi spy +v fr +urban ist +touch line +toffe es +the ben +stri l +qubool hai +preci o +ox en +ov sk +nov ello +no yes +mar gre +lou ghe +jess ical +gid dens +gen ome +challeng er +caroti d +bly the +bl am +bi v +bam ma +bally castle +ac am +âĢ ij +zab aleta +wip wednesday +twitter india +tunnel ing +trans world +t ween +stilt skin +stab enow +sarabare illes +san desh +quizz ed +penob scot +pal ouse +pa an +of fi +mer rier +m we +k way +ia wn +em un +egg shell +cou turi +coo ker +class less +chi os +cag atayulu +bay reu +ap ie +an son +am stel +agronom ist +è ± +y gent +weare rs +vla anderen +very one +sp s +pl ers +nivers ary +neiman marcus +ma ut +la gers +kalgoor lie +gl t +ge ena +dictat orial +cwm bran +be ee +ठ« +w bal +vit ally +ulver ston +te tanus +tab oos +sthe band +sta an +sque als +seab reeze +savag ely +r mu +p be +n ke +jo ven +j mo +hypo theses +hen n +health ily +guil lo +feliz jueves +dn cle +de de +crossh airs +clow es +british airways +ami ka +alcar az +" : +ye aaah +wol ff +un reached +twiz tid +turn tab +sal im +read ership +quin ones +quanti fication +over lays +national cheese +low brow +lang ton +la fayette +horror art +gr ls +gib ney +bow tie +ble phar +bit co +band leader +anarch o +acker mann +๠Ĩ +wall ington +tab c +t md +sm ilers +ri pened +ra ging +li ri +lg v +kn oll +jak u +im be +elo him +dono stia +d hr +cyber aware +chit wood +ðŁijįðŁijį ðŁijįðŁijį +à© ĩ +trill anes +thought works +te ared +san gel +out shine +nr b +ni bbling +mueller report +mehboo ba +m jol +kali spell +inv ade +inf ante +iggy pop +high lighters +dd dddd +contra ils +coer ced +chil dri +caterpillar inc +cad dies +beef ed +bar ajas +aco in +a joy +ðŁĮ ° +wq ad +wn a +twis cone +suz an +sm kt +sche id +scham pionship +sav aus +sa hy +p sin +nj transit +nanop article +mine strone +marshmell omusic +lanc a +kings go +gas kell +friday feei +fel tham +draw something +cri s +casablanc as +ver ges +schwar ber +rr m +rise vans +revel ry +requis ites +prestat yn +ping pong +no fx +nine veh +napp ies +le up +in decision +i gre +ho ka +hass ell +hard case +gau dreau +flex ed +fat to +eber le +dissi mil +defin itively +cra ven +canu ck +best life +be better +am bridge +ach risevans +¸ .âĢ¢ +y aj +vi as +t sh +su ji +sar my +rose hip +radi ok +on gan +ner oli +mi ja +long sleeve +lis beth +er ocks +ef lower +doc ent +ching ford +cb k +byz antium +am r +! ** +ðŁĶ ĥ +ãĥ Ń +à µ +y ma +whit estone +ur k +theri dge +sandown park +p bp +nw p +no well +mr david +mill s +ma gia +little john +ku ra +ko ski +hur ston +g night +cor ina +com el +be fit +aro y +ab ney +. âĿŀ +wiki data +war minster +tro yes +todor oki +stat ins +re touched +pen ting +os metics +nin h +nick jr +min it +memory lane +man cy +l ce +kip choge +kal k +il hoon +ig ami +han rahan +fridayfeei ing +fly away +coldwell banker +co ady +cha el +bo gge +ar xiv +amar ok +af ir +acadi an +ู à¹Ī +urban outfitters +un spoiled +tab riz +sun deep +stom pin +ru ido +rep ton +re activity +rav ana +pre debut +na ito +mr in +mill wall +lind strom +ki bera +jo ve +intelli gible +inst as +indiana jones +hedge hog +fre itag +el ana +dau sa +cham ois +bil lowing +anti freeze +alice springs +ðŁį Ħ +yu su +wa xy +wa aaaa +vir tus +tin gh +soor aj +sh kh +sal aried +pray toendabortion +nor di +motor cyclists +malevol ent +ig lio +homo e +here we +ger aldo +fron d +floo daware +ep src +e prix +e bel +du pri +cu nei +americ ann +ðŁĻĮðŁı» ðŁĻĮðŁı»ðŁĻĮðŁı» +v sc +the si +ten bach +tel kom +span x +sp eller +ni am +nathan thewanted +man nar +m cla +l alla +ko at +kar pov +kar la +journey to +hu esca +ho ffer +guang xi +gone but +ek ur +egg leston +ca ire +bo hen +barr haven +avoc a +army strong +ano di +??? !!! +ðŁĮ¹ # +åİ Ł +âļ ľ +zi am +wa th +tun n +te p +scumb ags +sco ffed +rol land +right move +raim ondi +que e +pushawards jadines +notori ety +ninjatur tles +ke dar +g sg +fro wned +de bo +d da +court land +chi seled +and ad +af ri +$$ $$ +" & +Ø ° +zo i +w ska +tur lington +the young +ser ai +sec tarianism +re aper +on ico +om yel +nam ur +ink master +har vin +gle b +fatt ening +ehl ers +dwar fed +com it +cly burn +bas sa +ant one +altern ates +ðŁij ĵ +ðŁIJ ı +yn ch +tv week +tu ta +tomat illo +som mar +scho en +pv t +prop ellers +prior at +na stics +ma aran +lu lac +kin sler +ke mono +ke keke +grub hub +gridi ron +gir lin +fe hr +covent garden +boom boom +bianc adelrio +bas sin +abcf pll +ðŁĶ ¦ +yoland aph +wel ton +thel ord +ten zin +sav i +ren ée +r ne +phys ician +nu ig +nd win +michelle visage +merck x +measura bly +manfro tto +magne to +jae bum +inst at +in azuma +hurrican emichael +hali burton +g bt +disco vere +di po +cas c +blue bird +blu efish +at ali +art scouncil +andrze j +anaphy laxis +american made +albac ore +ðŁ¤ij ðŁ¤ij +zav ala +vacuu ms +shopee my +sch nee +rez oning +play makers +pin ups +part out +narcole psy +nai ro +miil kkk +man owar +kis met +hau ght +fish el +f anime +er ici +ed sel +dutt sanjay +dun ce +de music +cer novich +bor at +b days +ang li +w tp +souvla ki +rec ti +nah in +lovewhereyou live +li gon +jo hal +im movable +hil son +hesper ia +gn at +f tt +ess el +en amored +elton official +ed a +dee speak +d wa +d don +cumu lonim +be avis +an ji +af lgf +ðŁĴ« ⾨ +ðŁijĮðŁijĮ ðŁijĮðŁijĮ +ðŁ¦ ī +оР» +wm police +wan ita +v sd +uro pe +up ton +sure ty +stef on +ru sten +recir cul +press uring +p ela +mc alister +lin na +l nr +kri swu +kim jon +ish ment +industrial design +hr g +hi mesh +fer ri +del aunay +carbure tor +blu en +a home +ðŁĺī ðŁĺį +wish art +up o +tu it +tri ennial +sema show +ram iro +pur posed +private eyenews +plough ed +onthe beach +minne haha +man ne +inj al +gwend oline +geor gel +fle mington +ed b +di ouf +creation ism +cran ford +bin du +ìĹ Ĩ +è ³ +yn z +uk g +trans iting +thr itis +smu dged +si en +shin in +sher rod +rus set +roman ceno +repri sing +plan er +photo bombs +oc f +mo dic +kejri wal +k umi +hemi spheres +goo devening +financial planning +dy fi +distr ito +cor ian +cel i +bur nished +aw el +art ph +ag ging +ud l +thedaily beast +tax ic +ta kuya +stair cases +stag ger +show me +pre ble +pitu itary +pad gett +no bun +maj o +lumber ton +lime house +leagu ers +l sat +jam an +it isation +hedger ows +go pichand +g eld +doub lec +de bby +daily qotdapp +cu neta +chri schristie +chain mail +cake p +bir ks +amy klobuchar +ðŁĶ ŀ +âĢ¹ " +za ar +town hall +topo logical +timmc graw +sel on +sbu x +quick sand +pin nock +o strava +mp ath +le lo +kar ang +kag i +judge ments +ju tsu +inf antic +go kingsgo +folk art +fli pit +ever grande +dav el +cut ts +custom isable +con c +commit tee +blueno se +belfast giants +barn acles +bar nhart +b tech +ar mani +an adol +agh an +ag ie +ê² Į +é ¢ +yorkshi rec +vote uk +tur no +ther mic +stu di +sre eni +soci ete +sil ken +si rs +sami yusuf +qu acks +pren tiss +national nightout +mp k +mono logues +mo hawks +ma vi +ish tar +ing our +han kins +g force +embarrass ingly +ek ay +dil i +de boer +chester tweets +ca pper +ash mole +app or +al yankovic +after taste +(* ´ +ãĥķãĤ § +wave form +wa hid +un recognizable +sos fam +scien cer +re la +po thead +nu buck +ni st +nai ja +mot ör +mo sses +mc quarrie +mak ro +m provement +luton town +ih ra +hay y +first post +et ting +dance day +cough lan +car ti +ber cy +barca stuff +bal ms +axel rod +ar trave +amit shahoffice +âľ ĸï¸ı +Ø§Ø ² +ty c +speci es +senator collins +re wire +pepper corns +mo sman +mer ly +lo ssi +kany akumari +health ful +he pp +g wc +debr ame +coor slight +centrifu ge +budd has +bed sheets +bate son +b ingen +anurag kashyap +ãĥ³ãĥ ī +âļ Ļï¸ı +yo gesh +y alls +wh q +wait ress +tortu gas +stir rups +still born +rcb tweets +pft commenter +pc u +ow y +neer aj +mar yanne +mar ga +let us +le chon +kin t +joh ny +ja hn +ing apore +hou lt +ho dak +high ball +hh h +e fi +dosto evsky +de th +custom isation +csk v +clu bbers +anto ine +aci ously + ¢ +~ âĻ¥ +yo gap +w era +vishal dadlani +st ena +quan to +poyn ton +open university +music city +maz atlan +mag pul +lavor o +lam as +kar ak +ho wick +her me +fore told +daw ah +chak o +bron zed +bron cs +bro king +beard foundation +ba sho +an museum +a hino +Ñ Į +wood worker +wood s +woo dro +winkle voss +ve toes +tb buccaneers +t lc +spen ser +s mike +prof briancox +pomegran ates +o chi +night ers +mete ora +liber tines +kamchat ka +hel ter +grass fed +god liness +germin ate +gab o +du pes +dead heads +croatia fullof +coach es +cas sand +bram bles +biz ness +bath ory +aw ks +at ma +ðŁķ ¹ +visit canberra +unear thing +rott nest +ross iter +r tt +pau lg +moul trie +loan ing +great ormond +gil ding +ger tru +gal era +discred ited +d fe +cand ler +ani ah +ah sa +ab orig +yamig autam +y ie +the original +sun times +sh n +sal ahu +robin hood +re introduction +kap o +jan el +it each +intri gues +fas s +enter shikari +en dow +doyour job +can ova +au tres +anglo phone +ab n +ðŁ¤ Ľ +~~ > +v ally +stromb oli +star fox +smir king +s su +ring tones +ragha van +po sta +news x +mc cam +matty bra +jag ex +itali c +i see +goldeng ate +girl probs +gipp snews +fin borough +dun c +de formity +clam ations +chand an +bu ra +bree ches +ash ford +anti pasto +ಠ¡ +za hir +we rent +ty len +th inspo +ta kas +t sen +suwan nee +sor vino +sold by +sch amber +per ty +pas orob +only fans +mic hell +mc quaid +ja und +garri do +franchi sees +foo ds +entit lements +elector al +cy rano +convo ys +christma ses +bapti sms +ðŁĶ ½ +ðŁĮŀðŁĮŀ ðŁĮŀ +ÑĤ а +zipp ered +tu li +speaker boehner +slam mers +shake el +ser bs +potter head +poe tr +pen test +p noy +ophthal mic +ng u +lock herup +lance bass +l tz +in numer +granger smith +facul ty +du four +der ham +decou page +cull is +cri ps +cen tos +blackcat appreciationday +bal lester +and juliet +weare in +v ax +v ata +und son +tem er +ta ichung +sun bathe +sni ffles +re painting +nore aster +nel spruit +master s +ineffe c +har as +gn ar +ff g +end ing +em ple +ei shq +din as +deaf ness +cor in +ch g +bly ton +ann coulter +ac utely +ðŁij ¿ +walk to +wal o +shire en +restra ints +poo ches +pdd ancing +palati al +north westhour +motiv ators +may ra +j ury +in me +field park +exuber ance +cre ased +cond é +c gr +bor ing +antic li +am av +ðŁĺĤ ðŁĺİ +ãĥķãĤ £ +à± ĩ +wol sey +tu gg +so ws +pick wick +panther nation +nell ore +mul sanne +lime ade +lee ann +hul lar +here foryou +he as +gi v +fun with +fi real +fascin ate +dream weaver +daniel howell +cushi oning +cou leur +birdwatching mag +bar at +b ation +ail y +acknowledg ment +âŀ ¼ +س ÙĬ +z ine +ws bt +ur thy +u ce +trouble shoot +tin os +super natural +states boro +she ree +seaf oods +ori flame +neu man +nau d +n la +n ky +model er +mi av +le ck +intu it +hyper market +his sing +harbour front +gon ski +gam ay +dok ken +de construction +cute cats +cran field +confeder ations +co ex +cd h +car lito +c moffice +bar ga +af fa +yy am +whi shaw +trigon ometry +tal ento +rothe say +pet m +pa via +lug nuts +lu kash +lash ings +kali ko +fe men +e disto +bike shop +ape l +anc ou +zin hle +veu ve +tu ohy +to td +sue ño +star ck +smo del +rigat oni +prostate uk +ple bs +nike basketball +narasi mha +mu sty +mehboo bam +mano euvres +lief eld +invictus games +infe cting +i ber +hor sley +ho om +gau tier +fat tuesday +f pm +ezral evant +ex x +ec ity +derby day +cali gula +boc elli +besse mer +bed bugs +beat cancer +at m +arom agna +an ica +ðŁĺį âĺºï¸ı +âı ²ï¸ı +ा à¤Ĥ +w bur +ul ere +sk ap +ration ality +preci ou +pay ee +ny it +mor tified +man us +lon gue +lets gov +kerr ville +hitch hiking +good stuff +fy ingly +flood light +feu ds +ero ad +end as +donny brook +declar ations +blant yre +balloon fiesta +aki ha +ver ia +su so +sportsm ed +snoo ker +science day +reboun der +panerab read +lon ged +klez mer +inec nigeria +hol ker +grand addy +for no +fast ening +e migration +dri de +dis location +davidar chie +dar uss +che viot +bogge ss +barn stormers +bar tel +art life +angel fish +womenin music +wi union +travel alberta +ti zen +st pete +sp amal +sexy saturday +screen awards +sch rute +ru mple +om ele +nase eru +nar rati +n una +n mu +mo slem +mc minn +madeinthe usa +lu jan +kro enke +he pa +haru hi +gri pe +ear then +diverse books +dan go +ber rien +b mb +atar decer +ðŁĺļ ðŁĺļðŁĺļ +ñ ez +yo b +trump er +soci alist +sig an +scher zinger +sch au +refurbi shing +ra gga +qu ero +ncle x +massimili ano +mand alas +jaund ice +is right +ir acle +hrd ministry +grand er +gra ble +f bn +desp atch +bul bul +brasile iro +bor age +bend is +bal zac +baad shaho +aku lam +a ahh +ठ¿ +zack ryder +wr dsb +wai mea +up to +tech review +tar k +sp ick +scaf ell +sa chets +rod denberry +r ø +pl cs +pac ey +mono type +lot to +lich ens +le pto +le of +just the +juli ag +j rs +int c +in deci +ic dc +he ze +di anna +dhru va +dab ble +cumulonim bus +clairvoy ant +cat on +bu mi +bl on +ar ai +a ich +. âĻ¡ +ðŁĺģ ðŁijĮ +ðŁij µ +yar nold +umh langa +tra itor +the beer +sun aga +scar am +regar de +not to +mil ani +m me +le man +ko by +int u +hu li +energie wende +dn v +cor tona +car ted +calaver as +c scs +bro il +break dance +birthday party +wardro bes +w lu +v au +tw t +tigh test +thcentury fox +startup week +sports india +se hir +sch mu +orient ations +nv leg +midland shour +ly mm +k ps +ish am +gish wh +geode sic +est ado +emer yville +du lehill +dg ates +den ne +cou cou +bun sen +bo id +bal k +ado gs +주 ëħ +èĬ ± +zombi ea +ze ch +wre aking +synthe sized +swir led +sto o +ske in +ren ounce +photo grid +no pain +nic obar +network rail +metron ome +m di +j ski +hd v +hal gh +h war +gar l +e gp +dic o +di ggle +con ker +cat at +c myk +book makers +bo ding +ang panahon +________ _ +>> > +(( (( +we ill +val era +truck ing +tro polis +tam mi +so fu +scho ir +sch aller +readi ed +pou ty +o clock +nemt sov +mo rec +mal te +judge jeanine +gro th +f fie +brooklyn museum +ðŁİ ŀ +wake forest +tro pa +thi stime +sle ek +rival ry +q bal +pinstripe pride +op ti +me stre +kings bridge +eso ter +danand shay +cuten ess +be amed +ani ya +af owl +zhou mi +voc acy +vel and +vander bil +stan wyck +snowmob iling +sheu gs +se us +sc itech +sand hills +rit o +re serving +quintan illa +pollin ator +ph s +per p +mu ti +mehboobam ufti +matthi js +maj ic +ly tle +ki is +k oun +ili ana +go ggins +gi verny +gi anni +geo grapher +fu gazi +fir stalert +em ic +don at +cro c +cn x +city and +ch acos +canadian forces +bon nard +bleed ing +asym metry +amy peruana +> < +ðŁĴĭ âĿ¤ï¸ı +walt disney +udu pi +u sher +tread well +rit mo +rev ved +rash mi +pre ssies +pompad our +patric ia +lg d +ko sta +ko balt +kidi ki +in cis +himan shu +fi baw +fan service +dist ancing +chav an +cassave tes +aqu ab +ant ana +adventure sof +ad tr +ab ut +[ - +ðŁĴª ðŁı¿ +ðŁİĤ ðŁİĪ +weather tech +vm ware +viz media +vic votes +ut v +the mentalist +ten fold +stun na +skill fully +pl ent +other side +men sday +medical devices +li sad +kush al +kas umi +k era +juri spru +inno cuous +in conclusive +iamk sgofficial +hit z +gri ft +go pies +gam os +def ame +dd lj +copernicuse u +car low +befully informed +arach nid +ap n +amp at +air crew +âĹ ¾ +zan elowe +wh ooo +wet ter +wat c +vs den +vas u +u du +syllab les +surf side +sur ly +sg u +revital ise +palpit ations +padma avat +maup in +mano euv +len s +le beau +kne ad +insuff erable +hun s +home coming +guitar center +eu geni +equ it +e discovery +bro ma +bot net +ber ita +beinte haa +and r +ale conomics +ðĿĹ ² +âĿ¤ï¸ı ðŁİ¶ +á´ Ĺ +te tsu +t anda +symboli ses +spontane ity +sou per +shan ley +san skar +sab it +r ils +r dd +pul len +ple xing +pla guing +ntv tonight +north park +max field +madhu bala +inst illed +hea dies +hal perin +earthen ware +discou raging +crustace ans +black mailing +auror a +ar der +agro forestry +ðŁļ ķ +âŃ ķï¸ı +yorkshi repost +val lee +th ut +tar di +sp hero +skin cancer +se ms +sc ant +sach sen +s combe +ru hr +or vis +night line +nes bit +m sl +love food +kni evel +itt ance +im patience +i vr +fis alpine +ferrig no +dedic ations +collar ds +chipp endale +c ren +bbc scot +al ten +ak shar +y sa +wal ford +v so +ucl draw +time bomb +tam pa +t oun +sear cher +ran za +pedu to +p ch +nov ato +mb storm +love sick +lov sky +long worth +line han +l va +he ures +freddi emercury +er im +em conf +eli g +decent ly +brain power +astar isborn +zhu hai +z uni +wi the +un in +tortu ga +stream ys +specul ations +sol vang +smil ing +seed orf +sajid javid +nab a +mil ford +mb assy +jim carrey +jay ant +hippo potamus +har kins +gray scale +daily caller +dai go +carpedi em +calgary expo +by rn +brek ko +bre thart +br rrrr +bon is +an ther +actu alliving +a ameen +whar fe +vigil antes +u ee +top sail +the res +soul food +so cs +se op +r bd +preju dices +postpon ing +neander thals +joh ne +i pods +hal es +ed mnangagwa +cham beau +calibr ate +cal vo +bul ma +bobby bones +bo sse +bl urs +bei ber +arn auto +ðŁĺĺ ðŁĺĤ +tylerg posey +t als +sur ulere +stur gess +sat nam +robb in +ra ster +obste tric +nc dot +ms dyn +mobile marketing +mel rose +maj in +li kud +kas bah +infl ating +ether ington +dic i +at axia +ðŁıĥ âĢįâĻĤï¸ı +ðŁĩŃðŁĩ ° +âĺĢï¸ı ðŁĮĬ +wunderbar films +ta vern +sr ila +square space +sign post +riff trax +pe qu +nave ed +na stiest +local history +life skills +j br +it now +ipp o +in bev +gon salves +gargan tuan +g dm +drop kick +dr harshvardhan +d yo +conver gent +ci hr +blueno te +black girlsrock +befri ending +b nl +anadol u +alca sid +abhi she +a asa +visit novascotia +un st +tri un +tod morden +super center +stay ing +rocke ttes +ric flair +pe ac +p janic +p dac +non league +mediterran ean +lounge wear +hal al +geo chemistry +fi ra +feel good +fag ans +eff southafrica +e urs +du an +circuit ry +childrens book +caro tene +broc colini +black day +bar ret +ball antine +annu als +yyyy yyyy +tm x +testic le +nu man +men newsdesk +letsgov cu +kids fashion +kak adu +h ink +ger tie +fir me +fe v +don gho +diete tics +depri ving +coolmore stud +clu e +che etham +cat trall +c ja +bio chem +bache let +b hil +teentit ans +sw tor +strugg leisreal +stone brewingco +sto xx +rock st +nil sen +muk hi +mo thra +metro pcs +mael strom +ma zar +lo oney +land i +kay y +in aba +ikoro du +g ade +e migrated +e iger +count ach +che sil +bus i +breast feed +better with +beatthe heat +be stro +íĥ Ģ +wright sville +wom end +us ar +tees dale +t fi +scra pes +s weather +run de +repe at +pend le +pav lov +ni ang +line wed +kaz uo +grand final +gi mli +cal ton +bro kered +bo stick +bo sley +arrhyth mia +wak and +vaill ant +ul alaunch +tw op +th ac +str ated +str ack +stone chat +stad t +sh ingo +scooby doo +oci c +mp u +mira da +l np +ic ey +hh t +handle bars +gup till +e he +duplic ates +consol ing +arti slife +al x +acar son +ðŁĨļ : +âĺºï¸ı ðŁĺĺ +à ¯ +work shop +thegreen party +th xs +swoo ping +skar du +siz we +sas si +rebe kah +po es +pap p +panor amas +mou sa +mel an +matt son +lee ward +keu ken +kar un +joeye ssex +hobby craft +hal cruises +go ps +giu lietta +dog town +dat ang +bu pa +bow ties +advers arial +ðŁĺł ðŁĺł +ðŁį© ðŁį© +ðĿ ĸ +ุ à¹Ī +zakkw ylde +yo wl +un r +un jab +the am +ta shan +raven ous +rain n +pe ppery +micro grid +long line +kak ao +intellectual property +ice ster +houston isd +hon oka +gravit as +forthe planet +flu or +fin kel +en r +en cant +disc ourses +dem ers +comis key +but thead +bring ing +afl ame +war dle +tre bek +stre aky +some body +sci fit +roch dale +ro fl +restaurant week +prophe sy +over street +mill field +matti as +len to +kla asen +ke ough +jo ji +ii ac +ga there +fe ws +excep tionalism +dragon born +daw a +d ants +ch elli +canton ment +black sails +bhu tto +ban shees +au teur +ðŁĴ¯ . +ðŁ¤Ļ ðŁı¾ +zakkwylde bls +y rago +wom bats +wing ard +tb oll +plo ve +philly police +pat snation +pam ban +meren gue +ma hira +long leat +light sabers +la ine +ju pil +i believe +hour ly +fli ppers +e ffy +devere ux +deire zzor +bu shing +br ined +bor u +bi dity +bi a +adju ster +unexplain able +the block +software testing +smu ts +rim mel +pro audio +per verts +nsc lc +nab isco +manchu rian +j cr +ic ant +house democrats +ear worm +disc olor +cv m +coal ition +chanak ya +boe hm +bla stoma +aldu m +af on +? '" +... ?" +* "@ +æĹ¥ æľ¬ +whatsfor dinner +weetab ix +un masking +turn up +trade war +the sly +tako yaki +ta pps +t vo +soulj aboy +sf ed +sco tu +rick ville +pend ragon +peep er +o it +norm alized +no ël +lake michigan +hy der +haci endo +gru bs +gazian tep +fork sup +every man +disp leased +darley stallions +crime watch +ck enzie +chron i +baji rao +auror as +@ $ +wy ck +ver hof +sti le +sna red +pin arello +nick o +n we +mod ell +min ess +lyric ally +li ason +lar ra +la ges +kimber ley +kas sam +jennette mccurdy +gret sch +gl ances +feu ille +endo za +dam ir +cre er +black panther +bas er +av t +wy the +visual ising +tr oughton +see ee +raw lins +pu dong +pu ddle +pl k +per ignon +ow ill +misss aig +mc duffie +kay akers +ire l +ing ol +gol son +gay er +deep dale +crowd funded +az o +awal ker +asi f +" ?! +âĺ® ï¸ı +zion sville +tvd family +sha sha +sh ko +s act +out smart +nu z +ni ac +ner c +ma iz +la pin +kou fax +ji ani +in ing +ii b +freenaz anin +des well +cra ke +ay din +al monte +âģ¦ # +wrest les +un blocked +tre al +ten sei +ski pp +sf aye +serv in +rw f +rodeo houston +r vi +nastur tium +mike and +hon es +he gel +haz arde +get to +future stars +emo ticons +di pi +de gli +cul peper +cat oc +bu gab +bo cuse +âĻ¦ï¸ı âĻ¦ï¸ı +y se +worldwetland sday +volu metric +vey ors +un comfortably +stumble upon +sto vall +star dew +si ro +shout factory +sc ca +ro wett +metalgear solid +mal me +lam pas +khati b +imperson ate +home team +happ i +hal fords +gri maldi +fri ez +en vo +dani e +cow per +conce aling +channel newsasia +cen k +brussel s +at ak +angelina jolie +vend i +un obstructed +thermo plastic +t ds +shkh rasheed +reti ro +ps supt +phyl lum +ma us +ma doff +lyn ton +ly so +kth opkins +just giving +jou les +eze quiel +euse bio +ct ms +conce ssion +by er +book sand +a od +tu ft +thespi ans +st thomas +sower by +ran tham +news network +micro cosm +maya angelou +m ka +ko gan +inclu sions +htown takeover +hil ty +har ge +happ ys +h fm +gra zer +gd ns +digital clou +digitalclou dgal +dero ga +car vers +back hoe +art studio +ðĿĹ ¼ +x er +usu al +the kid +tal us +stu tter +sh uri +mcdo wall +match room +marcel ine +man nan +kel sie +k ler +it ol +gi onal +faysal quraishi +farqu har +cooper ated +abraham lincoln +ðŁĩ¦ ðŁĩ· +wswan derersfc +vrin davan +vivi r +vau ghan +sop ra +scal ping +quad ro +pit lane +per ip +omni potent +n tn +mobile games +lc v +kej ri +intercep ts +fil aments +ed f +bo eck +arab ic +aff le +ãĥ ª +Ú Ī +wrink led +worship ers +vibe z +vari et +to sin +sp ica +shel vey +schi aparelli +riot games +pfluger ville +perme able +online store +me igs +ly ss +jen carlo +ig r +hr r +hom elo +hi jo +hen ch +drashtid hami +courte eners +cor vus +cine mathe +ar aman +ad ur +vijay rupan +tran scen +the jazz +spill ane +son tag +smite game +smash words +sav eluci +reu ben +pay e +ol entang +nc sa +manj re +knock off +inv ite +ge sund +flash lights +fau quier +engar dens +en r +el pha +eh san +combat ant +co it +clydes dales +cir ce +chu cked +cent com +bi pasha +barbar ism +baha wal +ay ear +ar slan +an bu +ãģ® æĹ¥ +à® ľ +Ùģ ÙĬ +аР² +vigne ttes +tur namc +sab ra +ram part +ra iz +pollin ating +peuge ot +perfom ance +pe dag +out performs +new season +murch ison +ministryof sound +market places +kul tur +k lat +ham blin +fu bar +ci mm +caro ten +canon australia +bo euf +bibi mbap +bapti smal +your story +we my +vijayrupan ibjp +tad pole +sc ud +sau ter +ph ol +park race +mono gamy +mom en +mat alan +kwi buka +hol bein +hoff man +hart son +go vote +gira ud +gar cinia +fu i +critiqu ing +cotton tail +clip trends +cabe za +bethe difference +ar ancini +ðŁļ Ľ +z war +white washing +weight less +vide otrends +valtter i +ser vi +nomen cl +morris ville +milk day +male h +legal ise +le ke +kar as +incrimin ating +hydro carbons +ha ftar +gra uman +g of +diminu tive +congreg ations +cit ric +chan sung +brum mie +broke back +à ´ +vo ile +under wire +tru deau +tro i +steve angello +ste aua +sr bija +reliance jio +perthglory fc +ml scup +mez ze +lo stand +in ck +he j +haw thorns +flo ating +en cum +empathi ze +dra ping +deli o +dar wish +curricul a +cold play +co dec +bf bs +bant en +as sun +art design +anup ama +al films +] ! +ðŁİµ ðŁİµ +tz in +thisi sla +sti fle +serge ants +rose parade +restin peace +reasons why +r mac +p wn +or wx +nar rower +mystic messenger +manip al +luv urself +la gann +he mming +he brew +er furt +draw backs +coim bra +breakout artist +al ar +ag ay +actor life +.... ' +yasi el +v ff +u fs +thr ong +spider web +russian art +re fraction +paddy sday +oy ang +ne do +nai ve +lo of +lat o +kar m +interro gate +gur up +cc pa +amar avati +ðŁĴģ ðŁı» +éĥ ¨ +z r +y alo +whet stone +thri ved +tc n +pre workout +onthe hill +novi embre +navig ational +mp tourism +mol ton +l tl +ko ster +ka el +jyo thika +in un +fur stenberg +food share +fi les +famili arize +exempli fy +detoxi fying +che se +char lies +cag le +bir r +biot in +ar ounds +af am +ðŁ¤¼ âĢįâĻĤï¸ı +wh iz +wee dy +volu minous +us dcad +ud get +there is +th andi +super draft +ss os +solom id +snow plow +ru ge +rock solid +re section +raj dhani +rain ham +psy locke +pro line +passion for +pa ster +nipp ert +medi ap +ma sts +lock port +ko il +hor vath +hair brush +gi lets +g ants +far fetch +f xc +dissi pate +debrame ssing +co sco +by product +braz ile +apho bia +ah gases +/ /@ +ðŁij¯ âĢįâĻĢï¸ı +the bell +sun spots +scrim mages +ra úl +poly math +kum ite +khal ili +jon ty +j ku +hyper allergic +hugh ton +histam ine +gul la +e ib +d la +civil society +chicago ans +bu di +be ile +aviva prem +are ers +ann s +zo or +yan tra +vand ross +val k +the truth +sy oung +spu blic +se thu +rel la +refec tory +pi z +pen elope +ku shi +kangar oo +jan vier +h lc +fan sites +decep ticon +clif ford +chec kat +bl ouses +ah db +ado g +xi sts +var ia +thibau lt +the power +sh oring +scu omo +resul ts +pot belly +ne ary +low ther +indi eartist +h als +goo s +gl f +fel e +disingenu ous +dese mber +del acroix +co ya +bou le +b sm +appet ites +ye ux +walt zing +vac om +ump qua +truck er +siti o +sham u +pvam u +phar m +on ne +jap a +it ek +it ad +in bend +hugh ley +hu mi +gen too +free style +fa in +f q +di zon +agu stawestland +ach ter +ðŁĶ ģ +ðŁijĬ ðŁijĬ +ðŁ¤¦ ðŁı¾âĢįâĻĤï¸ı +⼠Ķ +âĸ · +y team +wis d +v mt +um w +tw h +tt an +ren wick +re mix +po polo +pad am +la er +kno b +jag ran +hol bycity +f dom +eng arden +bocar aton +alex x +a ÄŁ +è ij +uni ofe +u wcl +swin son +sco p +pe gida +patter dale +p fm +o if +mc ity +materi ality +m gi +la di +kick boxer +jor n +fat man +eo ghan +einste in +dc n +ch ala +bullet ins +bla sey +bird life +am ol +akade mi +ah all +acor tes +ðŁĴİðŁĴİ ðŁĴİ +ìĹIJìĿ´ íķijíģ¬ +ëī´ìĿ´ ìĬ¤íĬ¸ +س ÙĪ +wi ak +wen dover +van ovic +val an +twin kie +team youtube +so ppy +scru mmy +sa ath +rick les +ra dy +pun kin +pro j +pim m +pete wentz +pal abras +news worthy +nascar onfox +nan of +mit osis +lawn dale +la die +la bi +jay anthi +jason isbell +iti me +hol lands +h sd +gam ora +ferr and +f anned +em meline +dol by +dal oo +burling ame +ben cher +ballo u +an ational +victori apolice +v ds +tah le +sc olds +ru be +ragaz zi +paro chial +no stril +newor lean +live for +jor ts +ip b +im plore +frigh ten +em aci +e ering +don nab +dat u +ch avis +benefit beauty +ba ited +aun t +ê· ľ +wor rall +wijn aldum +wag ering +vel oc +ucd dublin +tooth picks +su raj +st ec +sriman thu +spirit day +sleeping beauty +sethro llins +rec ites +philipp s +per ú +mcle ish +mau rier +ma he +loan er +kne b +ic bc +how den +hail stones +doc fest +ck es +china open +cd baby +bper rion +azale as +alo ka +waffle house +util isation +ti b +si ge +qu aking +pre zzo +pol ling +nu its +mobile game +mb om +life size +l fb +ki ee +ke hinde +in semin +how l +hin sdale +hes key +fon dest +fe style +evil regals +den bigh +cri bbs +bhu van +be on +aga dir +after all +ye dd +un fettered +theli ving +tad alal +suche tadalal +musician ship +l ali +ky l +kath ua +kan tai +k ds +it sn +ini b +hyun joong +her ps +head wind +head lamps +he pc +gay est +fit out +es in +eric garcetti +ef it +down syndrome +con sig +bperrion ni +be see +b sd +asi k +al ki +ðŁĽ © +ঠ¦ +Ø§Ø ¯ +to ki +te ja +sy arief +sq u +rho ades +result sday +prerog ative +pik min +no z +mile post +mc nab +mass illon +is sing +ima de +gr ito +globe master +f dj +do dges +corin ne +anthropo logical ++ " +zam ir +ty nan +thenorth face +tam ang +super imposed +son d +som i +ret ards +pre ying +person able +paradig ms +micro s +mer we +kit son +kel by +hul t +hand print +fun icular +fu me +form in +fifty shadesofgrey +fari da +escap eroom +eil at +dep an +daw ar +by blos +billi kens +bed minster +bad hopper +awo lowo +anand ani +çĶ Ł +âĹ ĭ +ÅĤ aw +tib co +ta seer +stal ag +spro mo +snow shoes +sick er +sal om +s sey +roof top +pari ah +ow asp +no gain +night wear +nak amoto +mc ourt +ing lot +ha es +go tops +go dukes +g staad +fe it +ear marked +chinese food +cats rule +black lightning +bir dof +bet tere +bbc south +as sss +art for +amo ja +aler t +wh igs +ur kel +spray berry +sol vents +so v +roth man +rec ool +proc ter +pew research +p ci +nic hes +nflu k +ly all +jail avak +hybrid cloud +hahahah haha +green peace +g za +freedomof speech +eat your +e ep +den ne +commu n +cheri shing +cc gs +cbc mtl +bar as +wash ten +topo graphical +toler ating +sc ool +no ssa +nab il +n ool +muzz le +mis sed +mg lit +loop ed +jo taro +jam tarts +i up +h ys +fre re +f dm +e mile +e bp +defense men +cro fton +chis ora +cha ren +brun ton +bra vado +artex hibition +am ro +am ac +adn ams +ðŁ§ ¢ +wy vern +spam mers +shu g +schoo logy +rec to +pink berry +pak vaus +p di +or kut +nat ch +mr r +m vt +l nc +john fugelsang +hur u +hum bucker +horn sey +high more +h cd +gods end +dun nes +comm enters +ale b +administr ations +a itch +? !!!! +ðŁĺ¢ ðŁĺŃ +yu ji +worldanimal day +war by +tribun als +tow ne +snow pocalypse +s wine +run away +rob ed +rad ley +ph ang +ol lers +official wolves +n li +m andre +ke it +ke er +kasi h +kal pana +ici ously +had ley +depra ved +dd as +cough ed +co ates +cher bourg +capit ulation +al tai +ì° ½ +z ow +wool worth +wigw am +trum pre +propos itions +pork chop +over flows +ori ana +megal odon +meat packing +mcder mitt +le graph +irv ington +inde cency +ge yer +fre tless +difun dir +case in +ben oni +wat ford +un ravelling +uhcougar fb +thibo deau +sme aring +revisi onist +pi de +phylo genetic +lu an +loren zo +long meadow +livor no +kit sil +joe gibb +it syn +her ac +great war +free bsd +fermil ab +di still +cu sh +clear water +can ela +brun s +bish kek +big dat +ðŁĴ ¬ +ภĭ +war th +ver ticals +top cybernews +stat utes +sob ti +sni ffed +sky lab +shin ty +ro ane +mentor ing +mag andhi +liveon k +it k +g mo +endear ment +east africa +dom us +ci w +carlo tta +bos combe +bath gate +ë§Ī ë§Ī +æĿ ¾ +z ny +vit ara +v enda +twizz lers +tor d +spen cers +sel den +se bo +mcar dle +lasz lo +l ra +karthik subbaraj +h no +g fr +fri gates +franki ero +corn huskers +bing bing +al ite +ภ® +son us +ren derer +point illi +phil ic +novo sel +let down +l cp +har v +fuele dby +fra w +er ving +duke energy +dan ko +circumc ised +chandra yaan +carli sle +can onized +c vr +bor ine +bor ge +bl dgs +a opa +بص ÙĪرة +week of +turbo prop +tol bert +tam ika +t lan +seuss ical +scu ll +sb h +pp es +pet ar +my la +m busa +le and +jo gia +ha gi +golden hour +finger tip +er rat +dog life +cy nic +chi ral +b ty +av irus +ani shi +ag day +ðŁĻĮ ðŁijı +ðŁIJ¶ ðŁĴķ +thru ster +th ya +t wat +solic it +so tl +silver tone +signor e +shu ckers +shi bir +sha in +sepul chre +por irua +paulin ho +listo wel +la ba +ing with +ho sh +har ari +first net +dischar ging +de tr +chi quita +bullet club +bachelor inparadise +audi om +ade eb +⼠ĵ +Ê ³ +xx ii +woodin ville +wheni was +tra shes +thu man +te soro +supp ers +special ities +sean ad +sch at +ra if +professional development +mi est +mat osis +life science +kal mar +juni ata +jump a +ir ctc +hoor ah +guess who +gra fted +f da +essi en +dt lv +carpath ian +ca hoots +bi ps +alex jones +al n +wu pper +w gr +use i +ston ight +st martin +rev ellers +photogram metry +parson age +na oto +muff led +micha ud +metag en +lee ann +le ford +kow ska +kookab ur +invari ably +grand central +gar nacha +future ofeurope +frank turner +eye let +ed modo +dro d +dre sse +den nys +chil aquiles +buc co +botan ica +be ppo +bar ron +b ti +ar sed +ak ova +: } +yo ak +x al +wv tm +te er +ta pit +mo vado +mexic ali +may pac +leav ened +joegibb sracing +is sar +i ag +go away +ge ve +fedex cup +emphasi se +dis continue +den ials +costu mer +by night +busc ema +bi ju +bay eux +bar codes +alco ve +al ast +ac enter +ðŁĺľ # +whodun nit +var ga +us w +tre s +ti ko +the fallen +som an +social ising +she mar +sevasto pol +se cc +s movies +rye grass +reser vist +re joined +re factoring +py roman +ox ted +nw u +magi ster +lu cena +lind ner +ide a +hill house +gam el +franken muth +fla sher +emul si +election swith +demo lishes +creati vel +av ita +!! , +xrp community +tru sive +sneak y +sal an +og awa +j aren +hin son +en grave +e itc +datab ase +ch he +brind ley +blue jay +baux ite +amig urumi +alfre ton +actu ators +abra r +abduc tions +ðŁijĢ ðŁĺĤ +âĿ¤ï¸ı ' +⾨ ðŁĴĸ +âī ¥ +wright son +u op +tober mory +supervalu irl +sport fishing +sl ates +sin h +schedu ler +sch nabel +re cumb +rab bani +pur nell +power tothe +persu ading +natural england +mou lt +me use +l ans +haz ell +go oner +gal leon +footh ill +fav ed +exhib ition +em path +elo die +dvent ure +den arius +dementi afri +defe cation +ci x +chor ro +bar dugo +are qu +arbit rage +a defenders +yah t +to sk +tele conference +taylormade golf +tan z +shaw ls +red fm +post secondary +kri ss +kaik oura +juventus fc +ity ours +i verse +hi jinks +gri f +col gan +blueri bbon +bips luvurself +altaf hussain +alam ance +ðŁį ½ +w ux +un ic +tr ically +tam bor +success ful +spo wer +si rena +sc roun +sab ras +re itz +physic ality +park sville +par aded +oo per +ne va +mu gi +lv motorspeedway +log ica +lec at +kram er +k hama +infantic ide +food lovers +civil engineering +c wd +c boe +brown sburg +aviation lovers +ani dol +alo ck +ðŁ¦ İ +âĪ Ĩ +veri fiable +tri k +ti ang +she go +self catering +reimbur se +prefer ring +por thar +mk to +ka am +fun dac +filip ino +dre lief +chart mill +caf frey +assinibo ine +afro punk +ab lo +wutang clan +ur ry +uni formity +t ferriss +sub pop +stom pers +sand r +sam bar +sad i +robbi es +rec oup +r itch +national volunteerweek +n td +midd lew +lau f +kun ta +ke ppel +immun o +hor wath +hereis gina +goo des +faber castell +ef r +dor ic +do da +de paris +conju red +carol inians +cali stoga +ben ching +bat aclan +af cofficial +ห ม +ร ะ +wild in +ud acity +tra dio +theo do +the is +t bol +sque aks +smorgas bord +shawn michaels +pon go +no stradamus +nike store +mor sels +j ó +in exor +igne ous +ig nific +hodak otb +heb den +hay ride +ham on +ge v +gal an +chapeco ense +bun combe +bu o +bod acious +bb ard +authori zing +auctione ers +atta che +anatom ically +ak id +af oe +af fiche +ðŁĵ ¡ +ðŁIJ ļ +yu uri +ye c +u ja +trans fixed +ti bia +te cho +sol ilo +school work +roof ers +re zz +pur veyors +pu z +pokemon sunmoon +ok oro +mahat magandhi +lu e +leban ese +laura prepon +kam enri +je han +intersec ting +happy humpday +golds boro +ga im +fro lics +f ads +encar nacion +ce vic +cat sof +burysted munds +birthday yy +at ai +ap ital +alter yx +ad ua +you andme +t bb +supplic ation +so gno +rah ma +puntac ana +pat cho +pa ar +om it +naz anin +mar kovic +ma ssed +legis late +ko irala +k re +imbec ile +hot seat +he eded +gran ules +ge yman +fran chi +e pm +ds all +d tes +crepu scular +cer ise +bermu dez +ben anti +ay umi +ater ally +ap us +answ erable +an pur +ac omohoy +ãģ ij +y omi +wing field +tw all +this was +thereal sambora +smooth ness +roh tak +produ ce +pro pen +private er +pa jero +mre azi +mou la +marku ss +maneu vering +lo ki +k sw +ju ma +joshu a +jon freier +in ion +cozy mystery +cor ot +chicag ol +carl son +b ink +ampl itude +ìĭ ľ +worldradio day +wb ca +th x +stop bullying +sat mar +sa aa +rock ymoun +qu ade +press man +nabe el +ma sdar +lic eo +im passable +frod sham +f du +ex chang +eric bolling +dick en +del on +cou lee +construc tors +bri er +ber ardi +bag pipe +ba j +b pp +alder ney += - +ðŁļ´ âĢįâĻĢï¸ı +ðŁĩ¿ðŁĩ ¼ +zer rie +wer d +study in +sh is +sch wimmer +rhi z +paul smith +oscillo scope +or ca +nav as +nag isa +n np +len ore +ir repre +ima b +im alt +hand craft +gravit ate +gr anny +gam u +g bag +dis agreeing +diar mu +cor tado +bring ers +am ies +alber ts +ad han +abram son +ðŁĺĴ ðŁĺĤ +ìĦĿ ì§Ħ +young adult +y si +wx w +ther cn +roc kett +rb news +milwau ke +laun dry +jung shin +hon olu +gym time +dv rs +all livesmatter +y nez +xenob lade +u mat +sign sof +profe sional +o stric +letter box +l lap +ke in +jar ring +hon d +g tl +fraun hofer +fo ort +du brow +crown theempire +cincy wx +c ley +baw al +ann al +adon ai +ðŁĩ ¶ +yu rappa +wor l +wei gel +trin oma +syndic ation +side walk +shaan xi +sci ver +saveluci fer +sar aha +r sm +prophy laxis +pine apple +ol bermann +mo fa +lori keet +lec am +iow aspeedway +exac ting +ent or +engv nz +dream house +dove tail +door stop +d stv +cy borgs +bou gue +bal on +awo ken +æ Ĭ +âĻ¥ âĻ¡ +Ãł idhlig +uc ca +thenand alfilms +stop and +sk b +sen ryu +rovin j +pare sh +n out +lu gosi +kro eger +kon k +ki awah +iy anya +hawthorn fc +haber dash +freef all +fla iling +expon ent +en doc +drag queen +cove red +com mies +char ice +ch ima +ceme tary +at ria +are k +an jou +al ented +ac igars +) -- +vse vil +verhof stadt +var sha +val ov +un sworth +uk on +show manship +seal ofhonor +ph alan +mel y +me tered +match day +k inston +hs baseball +he tty +fulfil ment +ful ster +earl ham +dev ita +d my +contradic ting +braini ac +bon et +bau l +bar ong +astr al +app er +an tim +.... .? +Ê ¸ +world toilet +un um +tit ling +sol vable +sant an +sa bel +rall ysweden +r boy +ocla ure +numis matic +natural skincare +nano tubes +mass dot +marcel oclaure +kitt le +greatormond st +e gm +dire k +bos sho +bj c +b tw +b enders +adel le +âĿ¤ï¸ı ðŁĮ¹ +ye vsky +w aki +spon se +simil ars +scol ding +pou dre +penn dot +offe e +mol i +mc sorley +ma char +green hill +gon oles +freer ange +engagementr ing +dundal kfc +defibrill ators +che ick +cam bo +atx wx +ar mid +am uro +aber ration +ľ ï¸ı +ðŁĩ§ ðŁĩª +ðŁ¥ Ģ +wreck ers +we stover +vad is +un box +ste ver +spic tures +smo kin +shah nafisa +never winter +nad ar +mcpart lin +mayo ress +mart ÃŃ +lu cker +love cornwall +lam brusco +kol n +ismail i +if tikhar +h mh +good ale +ger ar +gau ssian +eag er +du lux +do die +bvl gari +als icebucketchallenge +ðŁĺĬ ðŁĴĻ +ਠ¨ +zack snyder +yi ppie +world turtleday +wind storm +vin oth +vene tian +ve olia +ti f +than gs +steppen wolf +schi ffer +say lor +po pp +philanthro pists +park bogum +ni va +ne ct +me hn +love ya +lo zada +hidden figures +her men +fin nish +feng shui +ex ude +com al +cabar rus +biof ilm +bam bu +all endale +ł ãģĭãĤī +ðŁĺĬ ðŁĴľ +ìĤ¬ëŀij íķ´ +tylen ol +tro it +transpor ters +to pinion +spi key +sand or +rec tangles +q ian +preposter ous +p supt +mb or +gor dhan +gl anced +figur a +ell chat +di mm +dat ors +d bo +com ically +cli pe +bige ast +bienven ido +battlestar galactica +b hal +albert son +ص ر +twitter world +tren ton +town houses +surger y +sun bird +stan niversary +sicklec ell +shrink age +sh and +semat ary +sassen ach +s food +pu lido +one way +nose bleeds +mccr ary +mar din +log book +kn z +injec ts +infl icting +hu da +hems ley +health ier +great lake +free styles +ear ley +cunei form +clark sburg +ap bio +an h +accompli shes +ðŁĺĬ ðŁĻı +âĿ¤ ï¸İ +ti ra +theroo ts +tex aco +teach out +sop with +sack ler +route master +quil ter +pyth ag +per dido +panel list +opho to +k pd +ilipp ines +how lett +hau g +g fl +faber ge +ek ka +e iz +dream actnow +corru pt +chaffe tz +bu ggers +austral asian +as us +alon dra +twim bos +to ku +so hard +sla yin +sky watch +san som +official gaa +now ay +ni mo +mori moto +local isation +jack y +for our +emb ol +ed j +dra vi +col ai +ci ene +bar rick +bal dock +bab oons +auto pha +ar nd +âľ Ĵ +x lv +wil mington +vo ids +ven ise +tw ingo +saint srugby +pep far +our ay +op n +oligar chy +nat an +n do +moun tie +madein chelsea +ma ze +humboldt strong +hipp oly +hello ooo +good by +frat elli +ev geni +es mo +ensla vement +da chau +char train +am entor +' âĢĶ +ðŁĻĭ ðŁĻĭ +ت ر +yl la +willow brook +video conferencing +then ci +tat ter +sw aff +sp illage +si pa +sev yn +sa head +ry che +pu sheen +poly phonic +oc tane +no irs +national gri +n wi +lap dog +l vc +hun i +holocau stre +h wa +guinnes spro +flash sale +dv la +dav uto +d sm +d mf +causal ity +car illon +be cuz +wa hi +tintag el +the deanambrose +tat in +shan n +searchengine optimization +pl se +petiti oning +pand o +o gi +nucle i +missi onal +magnific ent +k ks +isc out +imp d +fa kir +evil dead +emul ating +dish washing +des jardins +clothes line +caver sham +ba ikon +anno u +ani er +al mu +ah rar +a sexual +! ðŁĴļ +y aka +wish ful +vi gan +unisouth ampton +the buzz +tan amon +taffe ta +stopp ard +sin ker +sha araw +schu man +reck lessly +pro pping +maran ello +ma sal +lol ll +hoo ooo +ho ban +gas monkey +er dem +det ach +darius rucker +clean water +black heads +biop harmac +belve dere +bart lett +ask ren +ðŁĺŃ ðŁĺ© +ðŁĩ· ðŁĩº +trav eller +tor na +theop hil +succin ctly +stan bic +smith ville +sikh ism +se pa +rayn or +plas mic +over heated +optimi sts +mo si +meetthe press +mc so +lamon tagne +kirk us +in ne +har vie +hallucin ation +green ham +gre xit +gas karth +errone ous +ef arm +cook son +con over +con don +care n +burgh ley +belfas thour +be du +bash ful +ariad ne +anim alabuse +acrob ats +ab ap +wann acry +un incorporated +te b +spor tnz +sa ari +ro vio +rad ler +pra c +pi voting +ph ono +pearl man +mun day +mon ch +modern slavery +mi yu +md zs +life way +k sm +jas oos +hor ta +galac tus +fossil fuels +ex us +end it +c leg +bron fman +beef steak +ar but +akiha bara +ðŁķ ĵ +womens month +torri don +t je +spring bank +spin elli +shab bir +rock your +poc keting +parliam ents +meal prep +mathemat ica +mar q +luxury living +loo ong +lar kana +ki zomba +ig cse +him mel +high st +head lands +gl m +da ines +corn elis +bett ys +beck mann +bb ons +b sp +ar ks +am iss +. ðŁĺĤðŁĺĤ +wol cott +un accounted +sub consciously +splin ts +sa onb +ru per +pader born +n bag +mid south +march esa +lu sty +lu cario +ky go +juli ette +inter feres +hypo xia +gand u +free gle +fra ggle +far ren +fac i +cryp tos +change therules +cap iz +bru ford +b ge +at re +activ ations +ðŁĹ ¿ +wood craft +us ama +tun stall +so ch +sm cr +sin an +salfor duni +punc tual +proud moment +pr icks +poe tic +pine cone +oscill ation +nag i +my fav +mer aki +man gold +makemoney online +letter head +k hoo +interstiti al +hyper ten +hick on +gul den +grey houn +galla gher +fit tipal +ente ast +end u +ecol lection +dr v +dis band +del f +decor um +com ikaze +cob web +chatter box +c fos +bo zz +bionic le +ar ke +voc ate +vo g +vai z +v ry +twi stle +ti ba +thro es +the wave +tender ly +shaz ams +sc avengers +re organized +propag an +port meirion +n gi +my chal +manipu lator +lam enting +kr w +kilo watt +jubil ation +iron works +hon y +hiaw atha +hel plessness +he mb +gil ad +gen ovese +en actu +dor fman +csir onews +corri entes +bore ham +ben ni +bath house +ath ur +arcade fire +amon te +al tus +? ( +yi u +wk tv +wer u +vsp hi +ve stal +synthesi zers +super sporttv +stra de +sag en +ravichand ran +rai ya +o doi +medi kal +live able +le vity +koch har +jessicaal ba +heral dry +harryand meghan +glendal ough +gil ley +ed n +drive club +devi ous +denu clear +cy o +cryo genic +cho gm +bu ssel +brou ck +ar moire +aj payee +ab ta +a aya +wool sey +unearth ly +ultra fast +spinnin records +scot te +res ellers +read acrossamerica +n cea +mcqu ade +martha stewart +loosen ing +j harris +girl talkza +g bo +fin nigan +elias son +bri ley +bow land +boo bie +blue field +actu ary +ðŁIJŁ ðŁIJŁ +ë¹ħ ìĬ¤ +y pc +y ager +un skilled +u gle +ty pi +tric ity +tin ie +thomas ville +stran raer +ssf crabb +ssfcrabb itohs +sk ic +reic hen +ram e +raj deep +pu shy +pic ad +p wi +oo f +naturo pathic +n ps +mccl endon +keshar ose +jeremy clarkson +je ster +in bred +h pp +f cr +close ted +c ton +ash tabula +an cona +alla board +ìŀ IJ +wad dy +voyage urs +tanamon geau +pr r +pon ti +pgc ps +our g +metro id +lauren laverne +kri ya +kohl rabi +key bank +kar ag +kal ab +is adora +grow nups +der osa +datadri ven +dan ks +ast one +ames bury +alife style +ภį +vijaysethu offl +t shi +ron don +pu s +planet jedward +pen alized +observ able +no sso +nca c +mon santo +ke se +ka ur +in en +i fl +greyson chance +golds worthy +gocu bsgo +foolish ly +fat cat +esqui re +dise mb +bon di +body con +birk beck +battle tech +av ent +an the +z cz +w ä +w drb +une ven +un peacekeeping +u er +ti gru +the vampire +sorrow ful +ru stle +ru hr +print works +pe kin +omni present +musking um +mool ah +mid point +mi hai +mar cie +jit su +ireland sanci +ham pur +gh ome +fro gg +fix ated +fer oz +dead stock +city center +campe che +ca sia +br ampton +blitz krieg +at first +ar tur +ushu aiai +ushuaiai biza +urve di +tph online +sympathi zer +side tracked +sf gate +pal infoen +man sur +ly d +li mos +jacque es +gun sup +gli se +ge thin +fri e +fen u +dy spra +cla ym +chap a +c gl +bar rens +an isa +ī ï¸ı +world tourismday +w oc +w ladi +van itas +sudir man +shab ir +ros icky +pre eminent +potter ies +pi awur +ne eti +my fitnes +men or +mark j +love dogs +k hou +ir anian +insi stence +flexi on +exorbit ant +du ele +desk ill +dd am +cy tometry +box wood +ben avide +assimil ated +ade e +ÙĦ ÙĦ +~ ^ +y ps +w iller +vali dates +u kes +tone itup +tom ars +she ared +rush ton +plu gger +pir ro +nar sissi +lynd say +lyn c +liver ies +k jr +inner most +ho tham +herman miller +h var +fanta stico +ec anada +dd m +daily motion +bed fellows +arbitr arily +am cham +ye son +weekend reads +vast ness +tre molo +the greatest +sil oam +sar pong +sandal sresorts +r grosjean +public transport +power ing +neuro tic +nan as +moo lool +mol itor +mi am +m sr +lou w +kra v +gab f +fier ro +f mb +don lemon +del ong +boro budur +ar nav +agro ecology +ac to +wür z +wil helm +v ingly +travelo dge +thre ep +the irish +staf fel +sta ats +snic ket +p gt +ol ab +o sts +numer al +neuro diversity +mat ters +la ga +jay ant +jaw breaker +in lays +home builder +gray don +gis borne +gas pard +fethi ye +fear thewalkingdead +ek ad +crum pet +cr h +cc sd +boar dof +backin theday +ðŁĺĦ ðŁijį +ãĢ Į +vetri maaran +tri ad +tough ened +ta hu +sp litters +sher riff +polar is +pe or +or ab +one year +nam carey +mu ito +make ityours +m no +l ch +juxta posed +jussi esmollett +it works +isthenew black +irelandsanci enteast +har z +e zz +dimble by +de wayne +de mic +co ves +cav y +cancell ara +bridge gate +bougue reau +bore ham +balu strade +al righty +ðĿĹ ® +âĻª " +wj hl +wi ffle +war rington +tony the +thi opia +spir acy +sp ry +social ise +shaaraw y +se dum +roman esco +ri ssa +red dog +rare diseases +rain i +plural sight +pau per +o dense +navig ated +made ley +len ape +k ys +k afe +home buyer +eul cs +dip tych +cube sat +ch isle +car nar +be ko +baf fert +av ai +ع ÙĪد +zombiea pocalypse +wit ton +unequi vocally +tri angu +thero ar +soccer grlprobs +roch as +revi e +pic ballot +meer kats +kr z +kq ed +kon kan +ker stin +innumer able +gu is +gu ber +ely ria +bo gu +aly zer +alphabe tically +alban ians +ade cco +ðŁIJį ðŁIJį +ÃŃ k +w under +te ow +shi ga +rick man +n ph +micro brewery +mi ffed +mazdas peed +marchi sio +loo b +lea v +laugh ton +kear ny +ip aded +i think +hod der +glen more +gle aner +exper ian +co bs +cau tioned +cab bage +border less +athle isure +ale do +a ard +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +u tu +tw al +too ting +spr a +sky tower +sal ado +rex press +pub med +om usic +no fficial +ni hon +i ams +h mt +goo sen +giuli ana +elpha ba +dream girls +concor ds +clover dale +citru sy +bra w +boun ties +barang ay +ase prite +antw on +an ja +aller gan +ðŁĩ²ðŁĩ ¨ +zal giris +wesl aco +um sl +tree top +tarry town +tac tically +slo pp +said haram +sa wan +sa kur +pan african +not good +nick j +must be +mul lets +miss ary +mand aue +lab ors +kam er +he met +gar rity +cre ases +ch oline +bro ch +blemi shed +zapp ed +ulter ior +turf club +sur fact +su leman +sk n +pre rna +on campus +nu dging +n up +matrimon ial +lore tt +ine sday +in mar +hydro phobic +hawkn poke +eury th +em ts +dun oon +cott en +constru ed +be red +ba sher +ðŁĺĺ . +ðŁį ¼ +z ze +ying luck +y de +vit als +ve day +tropic o +tra ynor +ticon deroga +the farahkhan +st asi +ss v +sk lar +sal icy +ro sne +rackete ering +pennstate fball +p bi +o ji +o doo +new mark +my fox +mor ies +marke l +mar oney +mac ra +ke izer +kal ak +human ists +hazarde den +fis sion +fe der +dy mium +buy art +at co +ash an +al sa +al kali +ë° ° +à¹Ģภ¥ +ver bo +th air +sas ol +s vel +nu f +mil lais +mc cook +mackin aw +light speed +lethal weapon +impal ed +hoo ten +hat ers +fil o +fantasy baseball +del isle +bu cked +blo t +ax mi +ash vsevil +aristo cratic +acro stic +ac rime +< = +!!!!!!!! !!!!!!!! +ìĿ Ģ +w cyb +vau se +uw tsd +suffo cate +special report +semi o +sat anism +ric or +ra res +pm modi +livel ife +le lla +ir inha +i Äĩ +hor o +her bed +ham mam +glen view +extrac tive +down loader +christian grey +chae ology +bur nette +bow more +bor ini +boo geyman +big sby +tu ffy +th unk +son air +siddi q +sch nell +rook ery +read athon +ong ate +men shoops +melis sar +meal time +kh ough +ju sta +ger ton +fal mouth +es as +ed and +der ian +da unt +d pl +conserv ator +concert gebouw +chi raq +char leroi +bull fighting +bon efish +ban stead +ðŁı Į +æ·± å¤ľ +âľ ī +tun der +to ffs +theli st +swar ms +snap chat +sk ra +shot gun +sch ee +s be +pros thesis +pink shirtday +or adi +one health +mp ha +molin aro +moisturi ze +lac i +l á +k ult +ide m +ho bi +gam eface +fittipal di +film struck +ed ens +cow town +com media +carte ret +blan cos +barbi eri +atra vel +amal u +alway ss +al tona +ye katerinburg +y israel +wal msley +w wee +under funded +u cb +tim ento +thomas fire +sho spitals +shail ene +sev ak +ru stin +romantic suspense +provo kes +pic ku +pe rel +pe en +pap ad +paleonto logist +minneso ta +kie hls +inde pth +e ero +concer ting +co valent +ce st +cardiff devils +bro da +bra k +ash bury +apprais als +appointe e +y ir +xx vi +workin ghard +wing suit +us borne +ra shes +patientex perience +ocho cinco +ny l +new kirk +myfitnes spal +made on +l atta +hu ys +ga jah +finger nail +eart ening +drew brees +do vico +diso wn +dan ai +colori zation +catter ick +call ing +bulk head +bo kuno +b ice +aul lah +ab ai +ðŁĻĭ ðŁı¼ +ı n +wil ber +upcycled hour +swor dar +sum é +steve smith +sinn fein +si dra +rochel le +ran n +raim ondo +polit ico +ph rine +outdoor sman +ne smith +nc pa +nal edi +mo len +ma show +m fb +ke bbi +im mo +ilo ilop +ha itians +gill iland +for christmas +escar got +download fest +dev gan +colla b +clonak ilty +click able +cali endo +c wx +blackandwhite photo +bangla desh +av y +ang ala +aj in +ze b +usar ugby +twitter sisters +turk sand +tee ism +sou let +se squ +sculp tors +refu te +reas oned +q iu +pie day +oyster catcher +oo st +mo da +ma ach +kah lua +g fp +fr ys +flood lit +dul hania +dan ne +cro ton +col icchio +cle x +ch boursin +cb cs +bern at +bat um +ì ¼ +ëı Ħ +z ori +wi thering +wang ar +teas poons +ske em +s gm +recuper ating +rattle snakes +r sh +po vich +mil en +m out +ki edis +her mos +her mon +garri son +gam ist +ga etano +cu bana +cru zeiro +chig well +ato se +ar bu +annihil ate +agu ila +ðŁĺĤðŁ¤£ ðŁĺĤ +y zerman +y po +wwer aw +tw iler +tal os +sw p +sw ab +shaz ier +ry ans +r pi +pur ples +phyto plankton +pha r +pa ise +motor homes +mccour ty +lun as +lauren tian +kol i +jim caviezel +gen io +flex friday +fair lady +ema ar +down the +devan ey +cre tin +ca shire +blu ey +blo is +black jacks +barnar dos +axo lotl +ashvsevil dead +zyn ga +tomo fromearth +sk ine +se wol +reu ters +promp to +por cello +per ron +pash to +paedi atric +oo z +oo kie +lich ter +leder hosen +leather man +jets fc +ippo iloilop +im pede +howard donald +gent es +ev ms +el co +dan tonio +coch ise +chil dish +á ¸ +vish u +un berg +ti enne +t fr +sir car +ribb it +r be +pla gne +pike tty +pi rit +personal injury +my lar +lzzy hale +ku ria +frau en +foot stool +farmb ill +fac inelli +escam bia +e we +cw o +chelseac linton +char ades +and ile +ac kee +yo soy +wol d +usp oli +us na +u val +tusc ola +toy ama +the sia +tac it +super ga +strang lers +steph mcmahon +solo travel +sh ampton +sa enz +robu sta +re framing +po ch +life insurance +la day +kar apar +jur gens +guild ford +ghir ardelli +ebb s +diav olo +crowd sale +chester zoo +char r +bigg boss +big cats +bhan sali +bad boy +at sea +as sent +Ï ģ +trek ked +touch pad +tim buk +rob portman +raku ten +pre trial +party nextdoor +os g +o tom +no yer +madd ening +jon son +inlovewith switzerland +gro ped +diam andis +big little +b me +aw t +ak ingly +wat sons +wat p +vote onedirection +u hn +tweetapicture thatdescribesyourfriendship +tr anny +son gof +sim racing +she par +sebasti án +scarec rows +sal cedo +o samu +nurses week +ner ship +narsissi st +n saina +mole ster +machiav elli +li ek +lc ms +ip fw +infon doro +infondoro tweet +gre tath +gerani ums +ent v +earth pix +du bh +dimit ris +corner brook +cor respond +cic er +chir py +cheltenham festival +bom bus +b fl +ai den +adar na +aber ne +a hir +unt angle +tu shar +stein bach +señ ora +schlei fe +offen bach +nobel peace +margo tro +hi ac +ge ma +eze kiele +ed ers +cule bra +comor os +win stone +way point +war pedtour +trick ling +su dar +spring cleaning +shaf en +pin oe +pg w +ola ju +leit ner +le se +ippoiloilop np +induc tive +hhhh hhhhh +gretath unberg +go tto +exo tica +erec ting +ea res +di ary +der os +de meter +co wie +che ema +books beseen +bo to +bb do +av aro +as of +am mer +a joe +........ .......... +violenceagainst women +under weight +ter r +t sing +sty l +stu bb +spi er +slack ers +shal f +public an +plo tter +os ment +nuss baum +moor fields +money penny +mal thouse +lu ka +lament ations +jai brooks +hel t +hahahahahahahaha hahahahahahahaha +er lang +emo tion +dy son +dro it +dorse twildlife +donkey kong +di anne +craf ted +cn m +bad ham +ay c +ari ate +ðŁIJ © +âĶ Ĭ +ü rk +ymc mb +wat ley +w toc +virgin media +super sonics +sh it +sch ilis +sad da +recycla bles +pu shin +pro xi +per reg +new shq +mother f +mo red +mal ak +love struck +kre ss +imam ali +hel msley +gri sel +flemington vrc +far scape +el ge +chi rag +alphon so +. ". +ðŁĺį ðŁĶ¥ +âļ½ï¸ıâļ½ï¸ı âļ½ï¸ıâļ½ï¸ı +âī ł +à¥ĩ à¤Ĥ +y ayo +way o +vi jender +tamar braxton +tamarbraxton her +student success +spi ece +sker ries +sha shank +sche de +pun dit +pt cl +perse vered +omg its +nko tb +ms deskill +msdeskill india +mr na +motor sport +mon ics +mcgon agall +kitsil ano +k vo +jo enbc +jas s +it em +ilo g +ie h +fe est +favour ited +far in +debar linea +comp ton +cheryl cole +bree ch +brac kett +barrett jackson +ðŁ¤ ¨ +âĺİ ï¸ı +yaros lav +us ace +ti do +sw as +sn hs +plu sh +pale olithic +out set +net eller +natur alists +naje eb +mi paltan +mer ce +lock lear +ley man +ip h +ini x +in ed +fre dette +entomo logist +devo tion +can g +boat man +b bow +ak ou +;; ;; +! âĺºï¸ı +ðŁĩ©ðŁĩ ¿ +羣åī£ ãģĬçµµæııãģį +wist ful +ver afar +the time +shon en +saidharam tej +po kies +paris i +no stro +monso ons +mix up +mith un +min oru +mb ball +man to +magnac arta +kalash nikov +hy phen +fist ful +expo se +day after +co gic +co bia +cin os +ber oi +ber at +an ki +' / +tre foil +tb p +tan en +steu ben +sportsc ards +skynew saust +pri stin +pay phone +onto logical +nikel ab +milli ken +mendo ta +k lasse +indi ain +imp ounded +iam nehakakkar +hal sted +gat ling +for dre +far ne +dump er +di dion +decre es +d ph +book suk +barber ing +ace c +ðŁĻĪ ðŁĺį +ðŁij ± +åĨ į +winnie the +tomb stones +thin ned +sty a +si pho +pseu dom +propagan dist +pe ct +over tly +ound ingly +o suna +nu er +nk peace +ni gg +livel ong +li ott +l sg +gom is +farming dale +ec ko +e iri +dsw d +deton ation +de voting +clu n +burn s +bring backthe +bo pper +ber l +ale i +al ba +zo iep +yo yp +tl eroy +te res +suit ability +sk ales +sa inte +reprodu cibility +puri jagan +pre ys +pr l +phy tes +mo there +miy avi +mc vay +lo ya +ko ke +green back +goo oooooo +giant s +fl studio +feather y +extraordin ary +dow son +defaul ts +dar wen +d ms +cur tin +clark sdale +ci les +chan elle +cassin is +ain u +á Ł +ya ari +wall ach +w tol +usf ws +twing lish +turn stile +sunil grover +sensi bly +schul ich +pro claim +prana v +peter borough +perreg aux +per nod +ne use +m lt +m ko +lyn den +ky n +ho ole +hali de +ha ys +friend lier +fer ment +f sw +er ing +enti sts +dis ch +cth agod +bor iso +bo wels +bam bam +au dis +angel s +al arabiya +ðŁĴ ½ +âĿ¤ï¸ı ðŁĴĽðŁĴļðŁĴĻðŁĴľ +woo lies +wdw today +to pper +te ap +super computing +sp angler +raise high +queens ryche +pri sms +opor to +mt mte +machi da +jan kovic +in excusable +gu ile +finlay son +dram edy +doo san +dayofthe dead +col gate +caddy shack +bt toronto +ammon ium +ami ami +ðŁİ§ : +ðŁĩ¦ðŁĩ ¿ +zoiep almer +want age +w ma +voiceof obrien +ver di +ve o +towerof london +sté phane +stur ges +sti ger +soci alized +sie g +rhy s +pro bert +over thinking +ole ic +no a +mikas ingh +ki per +iv d +ing rich +hyper active +gear best +followthe whiterabbit +fic tions +ec ou +darshanraval dz +commercial isation +catalo ging +ben evento +amund sen +aim lessly +- &- +ìĥ¤ ìĿ´ +ü yü +vin er +sutter ink +sp g +rotherham iswonderful +ri au +re deemable +pl lc +paw sup +or pol +le vent +lan come +kk an +ish ra +ha stag +gue ira +fil bert +eth i +d su +buis ness +ben ik +arre sted +al lover +agin court +wwe games +unra vels +twin sies +time management +tean eck +scar fs +r cl +pur ley +power houses +po stures +pedro ia +painst akingly +ow let +ope th +motör head +mol de +hugo boss +g oud +fri a +flo tsam +el dn +dan ai +ast side +ap apa +ans ys +° ° +vikram prabhu +up endra +sli ving +sky uk +si hh +show stoppers +see king +sc imit +reading challenge +ra du +pod bean +piawur tzbach +pel ley +p wn +os n +open banking +old timer +ol anus +mel ano +kensington royal +fc x +cp im +coss ack +comic strip +comb ative +co sida +ce v +cb w +cassinis aturn +ari jitsingh +app o +amee sha +adobe max +( $ +ðŁij¸ ðŁı½ +éĩ ij +voc ates +un habitat +tur vy +tro be +today schilis +tipp ec +sir te +saf es +ru dra +red white +ph ala +nag iri +multivit amin +me wes +lu ttrell +j ole +intrac oastal +in ici +im fnews +hand books +ha ft +go lo +crypto trading +char grilled +centra al +british bakeoff +bas set +aust ere +ali wal +ad ini +yw am +wor cs +tran scribe +to winit +sou py +sie w +show piece +shamrock rovers +sc ry +rou lade +re finished +ra vin +par ter +ou tit +ole ander +oh ms +neer ja +mag all +machin ima +job seeker +fun der +distur bs +bo dine +bm ps +as oiaf +ab atement +ìĬĪ íį¼ +worldtoilet day +victor iam +su ave +state university +smi l +sentin el +punxsutaw ney +patton oswalt +pad awan +modest ly +minim al +lass iter +ku b +ingh urst +hu lt +ho ol +hierogly phs +hay ter +hag ler +flan igan +fish town +fire brand +fel dy +et b +em at +el rey +doctor sday +dc family +daily pics +bro mide +bal aton +baff in +app raiser +app omat +aad har +ÑĢа ÑĦ +your future +war burg +summer hill +ro sses +patrol man +omyel itis +may tag +madein france +ka the +insur mountable +insig ne +happy christmas +gru mp +fo yt +draw this +disser vice +cur va +brigh ter +brad lee +ar cos +ħ ï¸ı +ì ± +wand le +w sw +university challenge +tag team +stel ugu +squ otes +so dex +script chat +ril is +rap monster +r ind +pre acher +ph rey +ly love +lur gan +lark spur +fair tax +dro opy +comple t +christi es +carri bean +box office +an eta +aggi es +ðŁ¤ª ðŁ¤ª +⼠ºï¸ı +us dot +tr r +roa stery +reson ator +phenomen ally +matt jackson +ley town +j elli +ic ana +ibi za +hy n +guess work +goti t +episo de +el veli +dr kent +boor man +ber kowitz +an hui +ah me +! ðŁıĨ +ðŁļ ¤ +ðŁĴĥðŁĴĥ ðŁĴĥðŁĴĥ +âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +water for +val d +under secretary +stren ds +sten house +soci alization +snow globe +sil van +romeo andjuliet +rhode sia +phnom penh +ou est +nu thatch +month of +mm ys +hierogly phics +fanci ful +er ac +e studio +com i +cab anas +bioprin ting +be ggin +bak ula +ausv nz +am ado +)))) )))) +wh our +weimaran er +triump hal +so ju +serv a +serp ents +scu detto +pol qc +oli v +o die +ma hou +lover boy +la sa +jeff bezos +irish examiner +indie booksbeseen +ill amas +hel ical +fear lessness +ezekiele lliott +e ys +de gu +dat are +cool er +cointe legraph +chu gg +character isation +cb insights +ca ppy +bel lucci +alt ars +? * +stevenage fc +steel series +shu b +shabby chic +se bago +scream ing +real i +plu meria +o wat +markuss chulz +jar ra +il ot +il lah +el ight +edgar wright +don cic +domestic abuse +disinfe ction +den by +bull snation +bel in +bech tel +beau te +bar bi +اÙĦ ÙĬ +wonder land +vote march +ther anos +ston ing +shipp ensburg +sch mel +sa chat +s farm +rafin ha +po inci +ode on +meryl streep +masam une +mary anne +mammo graphy +m oooo +links ys +ke ez +ka sie +gel o +deid re +cher well +cher lloyd +cardio twitter +cal z +boo zing +bar mouth +as ma +aquas cape +amus ica +ðŁij¼ ðŁı½ +á¹ £ +zak yn +xl viii +wgn news +ti as +taken ote +syner gy +stabil isation +sacri lege +ro ark +re action +radic alization +pla sters +ph ala +om ingo +new sprint +mu ffet +mr at +lique fied +le gui +iz od +ht p +frenchri viera +ca hn +arquitec tos +a ari +ðŁķ ¶ +ðŁĴ¯ # +åĽ ½ +âĿ¤ ðŁĴĽ +ا ÛĮ +vide ocon +verafar miga +v min +upper cut +uo fu +spire ites +sor bonne +sol t +smooth ed +shiv am +sand point +reli evers +lang u +i ze +how son +hil ic +glam is +g antz +fle gends +dispen saries +discoun ting +chri sb +ber tu +âĻ« âĻ« +z ky +wen zel +wbc boxing +top shelf +spike tv +spic er +sp engler +sh ate +sev co +pal mers +om undson +ner vo +marsu pial +lec kie +ld h +kili fi +kiel basa +jerry lawler +hy eri +help fully +finger printing +e son +depend ents +cyto sis +chi on +bom p +bai lee +astro gaming +assassinscre ed +ashmole anmuseum +an ouk +alejandro sanz +al ys +ak kuma +a sec +ðŁĴģ ðŁĴģ +y annis +u bon +troop ing +spamal ot +sn ood +sa kin +ruth lessly +remote sensing +r ines +pp k +pedd ler +meet southafrica +mcminn ville +mas ri +ma ggs +keyn sham +jaun ty +israel news +hell skitchen +have yoursay +haar p +diab y +delmar racing +ch ury +carav anning +can in +bur pee +ballester os +ar rambam +ar ge +ani am +and ys +an nett +ach io +y f +west mids +ultr amarine +tip tree +te mu +st z +spl int +shan ker +pil ate +pel i +panam anian +new land +mu bb +mis informed +mer sea +me ate +mc stu +may fair +lu pton +kud row +ke dge +it ae +indv wi +iam chris +en sor +dien st +d po +cyr ille +barro so +ari de +alternati verock +ak ak +ze ta +we iser +thero of +tani sha +super fruit +shin suke +sex smith +needto breathe +mr mark +mol inos +mar ussia +llan beris +kin ch +jupit er +ho thouse +glyco lic +ge auga +fu la +fish n +fa fighting +dar ned +bin ib +an airlines +주 ëĭĪ +ve in +used gov +timeto talk +tail or +suffo cation +san lucas +re ek +queen su +prece des +pen day +pad illa +oc ad +nopain nogain +man ton +lor ing +li vigno +keto diet +k anta +juli puli +heral dic +giving day +gabor one +g pt +ft fc +f ack +el agh +dc g +colli erville +brit ches +af w +ðŁijĨ ðŁijĨ +water hole +vladi mir +tec ate +switch blade +str t +sr kians +shailene woodley +re fle +rat ch +r fm +pre vost +pae dia +nas arawa +mug am +mc gi +land lady +lalunasan gre +ki ama +keen an +kan eo +j rock +im so +fl inch +fix er +fe mi +fc bayern +electric car +e chs +destin ies +cav ell +c vi +borsch t +bore lli +' [ +ðŁ¥ ĭ +Ñ ģ +Ð · +the is +thalai v +stom s +sco w +san kran +salon edel +redd it +pu paid +pas wan +mar lies +magnific at +ma ther +lisi eux +jamshed pur +its showtime +ick man +ical ondon +heck le +gang land +ep dm +cur vy +cre mona +colon el +co aster +cab rillo +cab incre +big ten +as z +alger non +Ħภ² +une vent +twin ing +ton sils +sso cial +south ridge +shop ian +pied ra +oo i +naka jima +music industry +mcafe eshow +mar sters +liber ace +green ways +g couros +for cible +durham bulls +cru mmy +cp k +clam ping +bri dged +biri yani +ac me +ðŁijĪ ðŁı¼ +é» Ħ +wash rooms +u eda +swithe xo +so ds +sel im +rau sch +port stewart +phi v +pacific rim +or on +ond ins +masu da +lan ey +kurt cobain +i ku +he ik +gir ondins +fre o +etsy specialt +do val +bur ra +ìĿ Ħ +ìĭ Ŀ +way zata +wall er +w mb +ty an +sinf oni +rud dock +rejec t +rav ish +ra skin +pre mam +poly graph +player pro +pen tel +pa ix +on dp +oh rid +nur magomedov +meadow brook +l ill +kr all +jac inda +iso u +ie o +go tit +fri dge +fer d +ec ream +disintegr ation +cultiv ar +cp b +city tv +bishop s +bel um +bb camerica +bat b +ban ega +ba isakhi +assembly woman +yoshi kiofficial +ym nastics +xmen apocalypse +wy cliffe +water world +val or +street fashion +show boat +river keeper +realestate investing +pha sma +nen agh +mur ciel +mc andrew +match less +mack en +lu ckey +less than +ic ici +holiday inn +hasi dic +gla w +g aping +di os +dha k +countdownto christmas +c dd +boy o +berk us +b kc +are sort +ap ra +an tos +>_ > +ðŁĶĬ ðŁĶĬðŁĶĬ +zet terberg +vex ed +tril ateral +trans america +the sm +str act +sti an +stern berg +spl enda +sh iso +rat ing +photograph e +ou te +maran atha +len nar +j oli +hyun sik +hoek stra +halli burton +for days +fo k +ely see +diversityand inclusion +butch ering +bro gues +bo hr +ared itor +ar ul +app ts +add ington +vintage toys +ti ppin +reson ating +mer ton +kad ar +kab ab +happy stpatricksday +gishwh es +game z +fra gran +ee baftas +drunk ard +chaper ones +asur geon +arnauto vic +ar ow +ãĥ Ļ +velo so +tur co +t ello +sth lm +spati al +rak is +raisethe bar +pr ying +or ris +ol ong +never quit +ly u +lewis burg +karun anidhi +gru po +flu tist +fabi ola +end hunger +develop mentally +dean ery +da st +cu bao +cou pland +bv m +br attleboro +bore hole +bar ked +av ino +are ma +animal lover +am ora +al vor +un tied +umb i +tory burch +thr all +sv w +snow birds +sheik ha +ri dder +re ars +ontari ans +olentang y +mott led +ma thi +leich hardt +kim f +keuken hof +janathagar age +ir say +hisham mu +he ssian +fest of +fa hd +ensen ada +enforce able +con cub +co k +ca world +bran di +boi ster +bar ram +ar pita +al dgate +wearen ato +uch ak +toa stie +te jada +spin el +show biz +seun ggi +rajas thani +r mg +mon signor +m radio +ke pa +judd apatow +isab ell +gir on +gi p +gam bier +fal vi +ero y +eli ving +dis concerting +cu ellar +cap illary +bin ski +angel os +amo sa +ðŁķ ķ +tuesday selfie +te ste +suni verse +sud ha +sap d +over population +numb ness +nicol let +min o +mid way +mag an +le bon +koz hi +kn ell +kli psch +her rin +hd pe +car crash +atlantic council +and soul +and son +am ple +ac w +x ham +vo td +vo ssen +trade smen +sper th +spear heading +sky dome +shin ichi +registr ants +prime iro +p tom +mou che +ko oning +kind ling +gad ge +ful i +fos dem +dis qualify +dan ab +dae jeon +d ja +com patriot +chlor op +blood stained +baikon ur +az in +... ðŁİ¶ +! ðŁĮŁ +ðŁĮ¿ ðŁĮ¿ +å Ģ +âĢ¦ @ +wol seley +wie be +up surge +tony robbins +swan ston +sal vi +s ä +rouss illon +raj ag +od ile +michael son +me ted +lombar dy +li san +kait lin +h sp +get z +ger ia +gathere rs +f elly +emc world +ed reform +dor mir +dish eartening +crit ter +cre ta +ash le +ar ini +an sley +actualliving scientist +âĹ ķ +winn able +uni fi +te ka +ta stic +supermari omaker +ry lands +raven el +r le +py ke +prun ed +nam rata +n qt +min ator +melissab enoist +mante ca +inform a +head hunters +hard ening +fent y +est ac +el kin +east land +dal trey +country file +comb ate +colle tti +arup group +ar ka +anno poulos +an kle +ãĥ¼ãĥ © +ãĤ ĵãģ +wein berger +vir o +thames link +skim mer +rush den +roo y +ram sey +pas sport +or atorio +nct m +mu gi +ma adi +litt lerock +la sor +junky ard +fis sure +fay dee +fare wells +draw down +dom hn +distric tn +cout ts +cori olanus +clam ped +bumble foot +ay ne +an it +amit os +vo hra +vladi vostok +tun ities +syndro mes +symp tomatic +sun pictures +sim provement +reb elling +quer rey +photosof dublin +no ther +min nows +matern ity +ko ti +jo vial +inza ghi +imp hal +idio cracy +ha bis +cele stine +call sign +c cr +?? ?! +transhuman ism +the cus +ta ek +studentath lete +sho rey +sa sharo +rv n +rish ta +pac er +new ydd +mol to +mel ting +medi atek +man jima +litt en +li bres +kuznet sov +kir yu +karak oram +kar gil +ill matic +hafi z +gan o +d bi +d acosta +chee ked +cast es +bab in +b wb +ay ia +and country +wh ig +the mba +tat ra +stro man +smash ville +score cards +rav ello +ran z +pi pi +ner ton +national day +mg sv +mali gned +le mmings +j ev +intram uros +inf a +ij en +hopkins medicine +gal ant +far ring +deta ining +boun cers +arequ ipa +ðŁ¥ § +ze etv +zal ando +vo ces +vil let +vent uri +su do +sm r +sil marillion +shap ing +see you +sam sara +rod well +resi ze +radic chio +qui ps +predat or +par ky +manu life +mahot sav +m ge +lat itude +krem lin +khe er +jay lon +i den +ha bla +gru en +fluctu ate +flappy bird +fab le +denny hamlin +centen ario +bun ce +broom sticks +an ut +ðŁļ ij +ìĥ ģ +wid ths +waf b +the word +te efury +spy ro +shi el +sch lumber +sa ppers +re launching +pal ladi +mon a +li q +kum o +jan ae +is si +im ate +g ile +e missary +coinci ding +chalce dony +cal do +apo el +an ole +aggre ssor +ðŁĹ Ŀ +à¹Ģภ¥ +w day +under pinning +troms ø +te faf +super mom +stri als +story tell +sco in +s need +roo d +plain ly +pac ade +ne brown +mtk global +margotro bbie +kir ch +khq localnews +jim mies +holo cene +he em +fu ta +fetu ses +dysp hagia +dayoff all +cu tbacks +cb ssunday +cas soulet +bul wark +ban al +an del +am sat +yas meen +will ful +west isbest +u ds +tw ice +thel ab +stove top +ste es +patho genesis +orel se +ogil vie +of the +of ten +of ici +neuro blastoma +more ll +martÃŃ nez +loss less +k lip +har land +handicapp ing +hal ima +gau ghan +gar ia +dd ance +cu man +cru sts +chiroprac tors +ben ham +baw ling +arthu rian +allevi ating +ac ole +---- --> +ðŁĺį ! +âĿ¤ï¸ı âļ½ï¸ı +ঠ¸ +yo jin +ye u +y pa +vand en +v gl +us ada +tsingh ua +thousand th +telltale games +sho jo +sco ville +sat ori +repe atable +quent in +mari me +malac añ +ma kai +lat en +kin kade +kil lam +hit maker +h th +gram app +ge um +fri z +fe ek +daysof activism +chur ned +chico pee +bu j +bet w +berk man +ah f +worl dar +women crushwednesday +vi kings +vanqui shed +trit onal +super series +sn afu +rimb aud +rally for +quie ting +music city +mattybra ps +mac ias +lovelo velove +lees ville +k chenoweth +jaz min +inter vening +ilooklike anengineer +ho pp +ho ice +hex a +gri ma +goo glen +gener a +g wan +der z +birken stock +bend re +bel en +be ira +ath ar +aerob atics +ðŁļ µ +ðŁĺŃ ðŁĴľ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +walk away +vallec ano +v end +swtor family +stro s +splat tered +robo to +red wing +or gin +oak lawn +nat u +mclaren auto +m ti +it sy +immobili er +ido o +her she +gir der +gamel oft +faroe islands +east link +do bbins +be sok +atri ots +artic a +altam ont +ðŁĵ Ĩ +ëį Ķ +zar ate +world champs +wil tern +tele mark +tang le +ste ppers +sp ha +smith field +smel ter +sleep iness +shor thorn +se hat +scast aldi +ricflair nat +ricflairnat rboy +ra un +p cy +noon time +mercy hurst +lo ling +little field +it su +ich allenge +homopho be +harmon ix +gar wood +fu a +ei p +dam ani +cla es +chal dean +brass ica +barcelon a +ar tweet +al om +aj kt +ðŁĶ´ âļ« +z ita +z ari +ur der +studio ghibli +sk ys +shareyour weather +seag rant +ry k +ou at +no guchi +meeting profs +man av +ine ar +gul ly +gul lah +guan lin +grow up +footb al +explore tnq +del acruz +creation ist +asphy xi +as ings +appreh end +alton towers +_/ \_ +year books +whoo pi +thel ady +temp tress +tatian amas +tatianamas lany +tab er +sur fed +sheep shead +ru c +ro tax +ra yan +post mark +op elika +olim pia +no bigdeal +na sti +n fib +my city +moon walker +metro linx +medev ac +mal ware +li gh +l fr +jah lil +icecream day +hon ky +h ô +gas ses +from italy +fc sp +eu erasmusplus +dol i +di mond +cor bridge +chic ane +bully ing +ball z +aggie up +world champion +ven eta +var ley +v aren +swin ford +story boarding +sor um +short films +self driving +sci reports +salv age +quat re +prime time +pat mcafeeshow +pand its +news max +ma zi +lo ko +kan mani +it ri +ink drawing +il ha +horo witz +holistic health +heath y +govmike huckabee +gluco samine +fredri k +fran corchamps +fenerbah çe +er mann +en ame +dor m +doit better +ch sler +cdn hist +bat shuayi +atmo spheres +ash urst +ant s +acu pressure +? ..... +. -- +whit lam +unab ated +u ky +tu bab +tear drops +sor orities +rekind led +qu ails +pat re +pad me +north allerton +n se +my nd +me gha +k fest +ima i +hy ang +fore du +ff un +favor ito +fashion designer +esz ka +disney sm +cheese cakes +cd k +bq quinn +:-) :-) +wic kes +un supported +squ inting +sam iz +pra gati +pk ats +nab o +like this +le scott +ku fball +kar imi +fl ink +eul er +dv d +da ke +bon in +bha va +bennet ts +ay cliffe +avali able +archi vists +al bar +çī Ī +à³ ĭ +wu f +wis in +trade govuk +sha fiq +rey k +phi fer +ow ry +n chs +mi strust +la chen +kal ki +je er +industri ous +iiii ii +for ked +dot news +desp at +dd ry +cycl onic +chief tains +ce des +bra yer +boo ba +blan keted +anti gu +alach ua +ðŁĴķ ðŁĺĬ +utilit arian +uss i +tantal ising +su hr +stri per +sal thill +re wing +ophon ic +o vember +ni ble +mc griff +ma hara +ju mla +ink well +i pt +honolu lu +green grass +fer net +ear muffs +d under +coted azur +bra ves +boy sen +asun cion +al ynn +ðŁĩºðŁĩ ¦ +ภł +yo day +vi bing +up do +uc ci +so do +si rah +seattle times +rp dr +reluct ance +pol ak +ot c +mcco ist +malay an +infe rences +in nis +ha shoah +h ga +giov anni +er ith +davin cis +compag nie +char less +blue bell +bern al +art therapy +al son +zan si +ye su +yan is +transcen ding +tommy flanagan +swal well +stick tennis +sp v +ro ms +ranjit rophy +progre so +pro bosc +nord schleife +mo jang +mill work +mase kela +laura bailey +kaz im +jaeger lecoultre +in is +hol ness +hit omi +gno l +gillette stadium +gett v +front end +di aper +d tt +country living +clo jure +clark county +bbcscot weather +ba thers +ac mi +abe okuta +ðŁijĮ ðŁĴķ +⼠ª +wanna one +un complicated +tyra banks +twitter land +tro u +tre a +tb aker +re drawing +no bodies +naom ic +md gs +mar wa +mar ly +made for +m de +la sters +jen elle +hairex tensions +green ville +ev h +ec ce +eatemu pkats +du ka +discolor ation +der mott +dc publicschools +clen ched +cart mel +brow ner +brainde ad +benevol ence +bel vo +bed worth +bab ly +auto dromo +at ak +armad ale +.. "@ +æľ Ģ +æ ¯ +to ver +tn p +there tte +teagas c +spar adise +solst ice +nott oo +n kc +my oung +mass aro +man av +hin do +h fd +fur yk +estudi antes +en ay +didd ley +cul led +comp as +cham pers +britann ica +bluer idge +bill ington +be funky +b ort +aw sm +ang gun +v ab +tol ley +the don +ted to +stro king +sm acc +shivar atri +sepul veda +ry del +poly morph +phy no +personal ise +o ing +nor ml +nerc science +mk dons +methodo logical +meas ly +landsc ap +kamp f +hel fer +gui zhou +gu ins +glengar ry +fo ghorn +est ella +embo dying +day bed +categor ically +bo yes +bench tm +beltand road +bead les +be stre +aric ci +afar ber +achri stie +ðŁĺĬ ) +ðŁįķðŁįķ ðŁįķ +x bone +worshi pper +tran som +sn elling +savit ri +sar miento +sab aton +re vocation +pd sa +mol ino +min ni +men y +mar at +macas kill +lj mu +li zzo +le aker +l ti +kimf coates +jit endra +jak ks +it zel +immer sing +hend ra +gold rick +giovin co +gal vez +food blogger +comedi enne +circuit americas +chamber sburg +bech del +à° ª +z ug +yak itori +walk ley +vocab ul +serendipit ous +sa dia +rug gi +roo ke +rit zy +resur faces +ren veen +re padam +rand burg +o vitch +nko si +motherwell fc +mo tul +mis interpreted +madi rakshi +le maire +jo you +instinc tively +hench men +haddon field +flavour ful +ex a +en al +el um +dow dy +do bb +cumber some +cervical cancer +buc ssport +bird watch +wr anglers +vi ña +vesti bular +sar ma +op g +nsm q +mag ny +kitch n +khan i +in x +hu ggy +he uss +he din +gn p +gau teng +cum be +conjec ture +carshal ton +bumble bee +black cats +ber yl +be mused +bal kan +at ron +ðŁĩµðŁĩ ± +าภ¢ +visit dublin +valent i +trampol ines +thenand now +ta hi +suss man +staff suni +slo ss +sinnfein ireland +sigh tedness +scar pa +ro ga +reen acting +po hl +plai sir +paralympic sgb +now t +mill stone +midr ange +le bat +kun ming +khur ram +kat amar +kale e +i shaan +hy gro +ho jo +go z +gar ni +gamer rter +fle ming +dv s +dru ze +done gan +deb at +carri er +ben ne +ar thurs +aly syarief +al hilal +af flu +ac una +ìĺ ¹ +ãĤ ĵ +yeah hhhh +xi ong +x man +wood haven +wan athan +ut ler +u manitoba +tweet the +top soil +tam angpanahon +starwars story +ru ba +pomo doro +per le +optimi zer +multil ateralism +mor phe +man ay +luxuri ously +kon ya +jess amine +iowac aucus +in ay +guardra il +gu tt +f pl +ern akulam +dimini shes +di sa +cort landt +coo lest +cas ita +cal vados +bo ying +blen z +bio similars +billu ps +billing sley +be res +be brave +assail ants +asak usa +an coats +ac te +æ·±å¤ľ ãģ® +ãĥ¼ãĥ « +your best +womens fiction +vibr ates +ve tri +tsitsi pas +t plf +stone ham +sq a +sk inst +ru sting +polic escotland +pet zl +more ton +k rock +jeffre ys +jad on +fu fa +fe v +donnar umma +cu jo +bun ty +brighton fringe +boling brook +ar oos +anant nag +am ando +am adou +ad dle +ðŁĽ ° +ðŁĮ ī +yn ne +wire tap +war hols +vivi er +virtu a +tv dsb +travel in +sun flower +retro viral +pre schooler +mu zzy +mdc ps +mary hill +mand alay +mah n +lil durk +lan den +la paz +ko te +jac inta +gun ny +elap sed +el son +del ap +de schu +cu eva +ci ampa +ca dia +ba iling +ay ay +ati ps +amorph ous +Ò ī +we tn +vi ver +tb in +super bugs +spac ed +sh lomo +se gas +sa is +roskil de +rag out +pie tr +o euvre +north brook +mc clar +ma su +m he +litt ler +li una +kno tty +holy land +h ku +gate keepers +fcc ca +far ma +de ka +ctv atlantic +cro zier +comedy club +co se +bryan fuller +aw wwww +ðĿIJ ¢ +ì¹´ ìĿ´ +zealo ts +vikas khanna +thel ook +terra pin +south ington +sj ö +segal ink +ring wald +pur suit +porter robinson +n jor +n able +music week +maneu ver +loc as +lat vala +jo esp +jill stein +je pson +jar omir +gre ener +gh lan +den na +cre vas +ca ja +but kus +barbecu ed +ban ia +ba shers +and als +ðŁļ į +à ¾ +Ùħ ع +za har +wa af +v uni +un hurt +sasharo iz +sar gas +rag weed +pri mers +persu asi +old town +na sim +mo sey +megab us +luci ous +lle ida +ku ka +ke sler +katy cats +juventusfc en +jale el +gri mac +form es +echo smith +dw f +deser tification +chat uchak +bol land +bec kett +bad boys +australi angp +amaz igh +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį +ðŁİĵ ðŁİĵ +ìķĦ ìĬ¤íĬ¸ë +woo ing +wire frame +warwick uni +vere en +tag a +s dorf +ruther glen +ri gger +pu renew +ound le +ophthalmo logist +nov as +mess age +lang en +ki ke +jj j +instru ctive +im cr +hud l +hahahaha hha +h nc +grow ls +ffe d +extrac ellular +egal itarian +dere khough +casc ada +cand lewick +bra chi +bi shes +aspen ideas +aro ck +am gen +aguin aldo +wit wer +vi sta +taran tu +stor ie +stenhouse jr +skin ner +sj hl +sel fi +se ck +se basto +salt ash +re turners +po ach +paras ailing +pali o +ot ti +nav ya +man gala +lil uzi +l fs +kier kegaard +kei ki +hop sin +g linda +fun e +flying treasures +first dayoffall +feed feed +extrem ities +chow kidar +bright future +bra que +bodle ian +abb vie +wait resses +vo se +streu sel +squee zes +so ham +si skin +scri vener +pied ras +pan ts +oscar noms +os weiler +multin ationals +melis sag +leibo vitz +kit na +ki h +jan os +int led +industri alized +henri k +glo om +gilets jaunes +gar ratt +door darshan +cu vee +bu hr +bbcworld service +b ita +ausv sa +ard show +zero hedge +twilight zone +treas ury +swind ell +shaha dat +se ance +s lough +quer ying +plain view +pas co +p na +me tu +maxi mum +kur uk +kare v +ic f +hur tado +hiro ki +hac kett +gi spr +chit wan +chan tal +cann ae +band ar +an pr +ad ley +acivil war +ðŁĴ ¶ +water sheds +tol an +see thing +sain toftheday +reham khan +redsox nation +pol man +nicky byrne +new build +mn r +legion naires +kw ame +in lombardia +hur ghada +ge omor +fra c +fr fr +el z +e warren +counter culture +col ney +boondock saints +ble v +be proud +bar dem +alam w +ye tic +univers alist +tiffany andco +si ar +ser pong +propag ating +preneur ship +org ana +on da +neu en +ly coming +ku ga +indv sa +glee fully +fore boding +energ ise +dr congo +diamon dre +de sar +d vp +cuck oos +cop en +coo puk +camp bel +cal pe +ca shes +ber batov +band ito +bab ushka +as sos +artist s +an cora +akat suki +yearsand years +w tb +twee ting +tre l +transgre ssions +stru mming +someon eyou +ski athos +san on +saddle dome +runny mede +ru ms +poo ts +pardon my +new son +national boyfriend +melani el +mc carter +laurabailey vo +kn br +jop hie +ic se +hu ddle +gra zed +go blets +ff ner +cham el +cedar wood +car per +bex po +bakh ti +adverti sing +ðŁıĢ ðŁıĨ +ठ¶ +vicer ylle +ust fccca +sten gel +spear fishing +sheep le +p ke +nol and +mee eeee +mb ol +leon i +la than +kru m +kan agawa +gyneco logist +ga ap +flor is +eu logi +ef p +ec n +dee j +cyber men +cross an +coin treau +cody rhodes +chori sters +car ls +belle vue +asylum seekers +ðŁijį ðŁĺİ +íĻ ĺ +women day +trun ning +to god +tali esin +spo oning +ro ve +proxim al +mr bobbybones +maul din +kel is +kaf fir +iw m +il ai +go ssett +g Ãłidhlig +fly tying +f pu +est evan +dysp horia +dosto yevsky +decap itation +dai wa +cl anton +ci ut +ce menting +ag ara +ðŁIJ ľ +ðŁ¦ ij +ys z +wel led +u sso +tu li +tok us +the futurist +t illie +st itt +sofi acarson +scrib ner +rudi mental +robert kirkman +resi stors +re gon +ram ana +r nib +po du +no tes +mount sin +inhabit able +hu i +grom met +forthe city +fore stry +ding ers +dend ritic +d ins +brown stein +ay ahu +... âĢ¦ +team coco +sunday lunch +sub tract +si ssy +sen ran +sapi en +ro go +ro f +reggi ano +re integration +re imagines +pl ices +ol on +ogle thorpe +objec ted +mr b +motor cars +mathe w +ma fi +lil ah +l pm +j balvin +inge stion +gu lies +grun ts +gior gio +gi ma +fre elibrary +fmp sd +fe is +far k +fal staff +expos é +donate aphoto +deci ma +cu c +cri eff +co sti +choo k +bridge water +bab bling +aureli o +architec turally +alt itudes +alfar ome +alderweire ld +yaht zee +ver raz +upper case +tro glo +te il +st legion +sli me +sin hala +sh uk +se ps +schen n +nc ss +mart ello +mar ka +khal e +intelligent sia +ic b +happ yn +epic games +edward barber +dy nat +colour pop +cal var +bur wood +bo dom +al tright +ãĤ ĭ +young min +trab zon +tra xx +sl ington +sim mered +ren fro +pu ke +phala en +nax al +mx r +mun iland +mac i +ior i +her p +fx networks +ec to +deb ina +chrisl hayes +cbs newyork +campbell town +bat esville +av it +ash wednesday +aco b +ðŁĺĤ ðŁĴ¯ +zi ad +wo tc +tu sh +the return +tem kin +suspen der +st j +st bat +spe kker +se án +quad rup +or os +moh sen +me jores +mcbride melissa +iri s +insul ate +holl ering +hear ns +gau ging +frosin one +fe bre +entr anced +elik kol +durgapu ja +disney xd +d max +cormor ants +cis c +chief keef +c wru +butter mere +bre tta +boycot ts +be al +à± ĭ +xanth in +wannab es +w me +tr at +too thy +tom ography +tin elli +th ack +sw ati +soda stream +pul sed +preserv ative +n ced +mon ad +mo bbing +mica ela +mar it +lor awan +khu da +jam o +jackson yb +ill ine +hor wich +hen sible +gol deyes +go or +fitness friday +f na +ett ner +echo cardio +dr phil +dema tha +cas cade +can yaman +box car +bar den +b nt +, : +ðŁļ ½ +ðŁIJ Ń +Ë ¡ +york s +un hrc +un funny +trump ers +tra k +today i +thesly stallone +tab ak +shi l +seg mented +se idel +ri ple +re man +nep tun +lie v +kw ad +he swall +goe sto +gad or +g ones +g had +fer gal +cu rable +cb cott +bu tta +brick house +beat ers +an m +# âĿ¤ï¸ı +venice beach +ved monton +team o +stedeli jk +purve yor +ponto toc +pit loch +orange theworld +o isin +nw bb +ni hr +naijap als +mark r +ko tori +heit kamp +hal pern +ha use +cor s +change maker +ai ke +a hahahahaha +ච± +wil kin +wak in +v gma +un inspired +superbowl sunday +sport snation +son wheels +sha ik +sel ife +sat yar +restric ts +most beautiful +jor ia +jodor owsky +ic ho +homogene ous +hen kel +han kook +goh mert +godz illa +g we +g ative +ech os +diso bey +car milla +be head +ðŁĹ ³ï¸ı +ðŁĴģ âĢįâĻĢï¸ı +ìķĦìĬ¤íĬ¸ë ¡ľ +⼠µï¸ı +wer ner +steve martin +sn us +shr ine +shira zi +schnau zer +n ée +mat ting +ko bach +kevin spacey +keral atourism +kam iya +in nyc +green up +gi golo +gar lands +gam ec +floor boards +d gamer +coinci ded +buli mia +a starwarsstory +í ľ +âĺĢï¸ı ðŁĺİ +ಠ¯ +ç on +way yyy +takethe crown +stri b +serv al +sa vic +rown tree +mn ths +long ley +lagun abeach +kin folk +kap iti +kan tha +ev ah +disk on +dar lo +crystallo graphy +cou n +chocta ws +chis nall +cast leg +carab iner +c pa +brock well +ban bridge +. ãĢĤ +ðŁĩºðŁĩ¸ âĿ¤ï¸ı +wu ff +van den +un loads +theti ser +shop clues +ri si +procter gamble +pom pous +photo books +ph ic +mon clo +krav maga +keep fighting +k als +i ht +exacer bate +dh l +de sil +cat chand +bo te +all and +:) @ +... ??? +ðŁ¤· ðŁı»âĢįâĻĢï¸ı +wear it +underestim ating +u en +tw m +ta fa +scen eso +ru dely +real dj +per re +pe gula +par at +or ji +oluse gun +o tha +o af +loving blogs +late ef +lamin ating +kingof pop +james bourne +j gh +in ya +har ty +h py +gore tti +go team +follic le +explor atorium +e checs +dt k +climat ecrisis +can wait +brun er +beck oning +atherosc lerosis +all road +ðŁĶ Ī +wee hawken +ty rod +ti enes +tail ings +sur fing +stam ford +silli est +self driving +sco splay +sav ini +sa kit +rock of +re snick +que an +pic ci +om ac +melani ec +mccar ran +marion spekker +mag ill +mad cap +lie w +ko bold +kee ch +its great +har ring +gw ash +fire wire +fenu greek +exagger ate +down size +davi dro +comple menting +br inson +book marking +bo chy +aln wick +alek sandra +acon vention +ãĤ ² +yi dol +water line +v co +thom sen +scor ned +s martin +re union +re finishing +pu king +photom on +nr g +n lg +megapix els +learnto code +kuri haran +kis sward +inter sper +hi sten +hae chan +go won +gn ash +func tionally +e by +colla ge +chris delia +ang in +anc at +worl duk +wee ms +un ranked +traff icker +thru sters +ta wan +stü n +stephan opoulos +sp reps +sc ouse +sa hib +rec itals +promo tional +pan nier +mic he +me scal +mar rickville +mal aise +her rington +hallucin ating +h mg +grapp les +gl int +ge thard +g dg +ful via +fow les +esp er +en mity +dream scape +don ville +dab bling +d sh +cu ck +constitu tionday +consci ou +by way +barbar ap +an elli +ad vice +ðŁijı ðŁİī +âĤ Ĥ +б оР+yuk ata +yogan anda +way forward +uk ku +twi sters +tr illo +tire some +tho don +the jim +tat as +quin ce +quattro porte +pre ening +perfect wedding +pad dies +p ws +nick kyrgios +mumbai kar +mai read +l ations +jacinda ardern +his d +h mb +gi sele +gener alist +gel in +gas lighting +gainwith xtiandela +fre unde +flat ley +fabric io +expre ssion +duni ya +del ores +brou wer +bor ro +bmth official +ðŁ¤ ´ +ãĤ ĵ +under growth +tuesday tips +top news +til lage +sceneso fulster +prince george +pitloch ry +ol me +ok sana +na ha +meg acon +mat ica +kh q +kab b +fashion style +enrich es +craft brew +chair man +canes football +c inv +be sted +ba ia +ba che +at oes +acci dental +absolut ely +á ij +un spoilt +twee tin +ric heli +re trou +rat u +raj ni +ps ys +po sta +pes ce +michael buble +mc naughton +match stick +mach ar +mac dougall +lur k +loren zen +lore m +lit any +len k +kw ong +ju gular +jhel um +inescap able +ha zen +farmto table +estim ator +do seof +cri mcourt +campan ile +bri les +b hoo +ar gin +aq ib +am ending +al bin +agon izing +ðŁİ¶ ðŁİµ +twitch share +the ed +speci esday +r th +ply m +patriot snation +overwatch league +ori ya +ny quist +nick jacksonyb +mo thman +me mentos +love whatyoudo +loo ser +jan itorial +hemo globin +hass elt +galla cher +electro static +ed ch +annot ate +andl rai +aan andlrai +a inge +ðŁĹ£ ï¸ı +ðŁijĬ ðŁĴª +ðŁİ¸ ðŁİ¸ +è ¥ +years of +wu yifan +w up +theo dora +ter u +sil i +r wn +pont chartrain +on tana +mon ae +men ara +leve sque +kell ys +kay cee +k cac +jun ichi +joe manganiello +in america +hir su +graffiti art +gra fx +geor gios +g eric +focu son +esqu imalt +eas tham +drun ks +do ku +diag on +de fuse +con dit +chery lofficial +cast aways +c jc +bring onthe +back track +îIJĴ îIJĴ +ç ao +z ic +waven ey +tol les +to sser +silver back +sav an +rebel lion +random weather +our ne +neen ah +monte verdi +hill is +giz aka +ge ms +emerging tech +east end +dev yn +de humidifier +dau d +com ission +campi one +calam ities +as it +ar ney +ðŁĴķ ðŁĴĭ +ê ® +âĦ¢ : +zel da +yu suf +win dia +urin ating +the st +ste w +spring y +so ga +sardin ian +ro mer +p tar +o el +lle welyn +kyo cera +key shia +hopkin sville +green eville +exi ge +eric trump +e aly +at ale +/ \ +yric fm +wyn dham +umb rell +th s +t ented +sur names +shilpa shetty +rtel yricfm +recumb ent +re combin +pit ter +om ari +no ster +no gizaka +nig th +master work +love song +kozhi kode +jerry seinfeld +itv thismorning +i ai +give it +fu so +fre dy +dor k +dit is +cent e +ca ches +balven ie +an jaan +aj ide +z hiv +um ali +tre vi +track pad +t illy +sw o +south port +shopp ixels +sac kett +rantham bore +q as +pa olini +n lc +mu tch +mcke sson +mc ing +manu al +ma sher +illamas qua +i ww +hoste sses +grim ace +golf digest +gen dering +from within +eis ley +dur um +cr tc +con toured +book birthday +ald is +ab ú +ðŁ¥ © +zul ily +wac coe +util ity +swar med +sky tree +rm hc +pot pourri +plant life +mor rie +jock owill +j eng +im en +fly trap +eleven ses +dog gett +do po +ding wall +cup ca +col n +bic he +beat ncds +bad ge +aun tie +amer rell +âľĶï¸ı âľĶï¸ı +yves ant +web kinz +warri er +w of +tu ras +tin kle +terri fyingly +ter t +stu yvesant +sk us +river dance +revolution ising +par ing +pag al +mull ally +mck ellar +mc nish +le et +kur la +ken zie +journe ying +jen kin +hux table +hsi en +gra bbers +go leta +flav oring +fenway park +escar dio +ed da +clu ms +ci mb +bo dn +birthday cake +arc light +ðŁĺİ âĺĢï¸ı +ÅŁ e +z ef +whit elist +w azz +vi mal +ti har +specu lators +sin di +sheh la +road master +rat er +petti grew +palaz zo +over power +omar athon +no ora +mu kul +moon child +mon biot +logo type +lash kar +kau shal +ju jitsu +jor ginho +jockowill ink +ing out +haz leton +harri man +group love +dre gs +dil ation +de mary +dar lington +counter productive +castell anos +cas satt +carly fiorina +ca ines +burn ell +ar nica +. ðŁĺĢ +wo che +whitecol lar +tu tored +thal asse +star crossed +school days +ridge crest +re shma +public health +post master +pegas us +one spn +nicky romero +net zero +national geographic +mineral monday +medi as +like a +kwa k +ke pp +jordan peele +honor é +home stretch +hock en +ha en +gla ive +gany mede +franchi see +fab rica +elo ad +du rex +dou ala +bluec laws +best dressed +an andi +aic pa +ðŁijı ðŁijĮ +âĢ¦ ! +zakyn thos +uru gan +uj jain +twitch yteam +tit mouse +thereal roseanne +tel lem +t anti +suppre ssant +suffra gist +su pran +racing post +paint work +of mind +nyc dailypics +nin er +nie meyer +n va +mur ti +maris ka +m gi +lit zer +klo k +kent state +jad akiss +han dof +han de +ger shon +ge kko +ga ja +fu hr +en zi +e pe +del auren +cur ragh +colon nade +be govic +aquare lle +aor ta +william sville +what not +vote btr +tor k +ti sa +tam mie +skysports newshq +si ge +shock proof +s research +reh man +q ade +pumpkin day +per on +mp b +megh na +materi alistic +lead enhall +lav ell +lac c +intl crimcourt +immu table +hat o +dw ts +cent r +bo di +bhoj puri +any ou +ahan nock +xox o +top notch +sal lam +sal afi +red rum +pon o +pis cina +on r +old days +mus well +merry christma +matthe wk +locker bie +ki i +ic pc +hendo polis +gal rani +finger style +dr seuss +collin s +aron ofsky +annn nd +ach other +è ¾ +ãĤ¸ãĥ £ +âĹ¼ ï¸ı +Ê · +win o +what ley +vint ners +t reads +stevemartin togo +sg x +roth bury +raj kumar +quad rang +pro publica +metaphor ically +mat ey +life after +ke mer +i etour +gou ge +g fi +frees at +fis d +elisa beth +eh le +ech ino +drive train +cór doba +cla ud +cl wyd +chaf finch +bla den +bad asses +aren o +am ita +ðŁĺĺ ðŁĴľ +ðŁijı ðŁĻĮ +wor s +weare wayne +w mn +victoria justice +tsa ww +tram ore +tecno logia +tap out +ste inem +social change +shu tu +ong allery +obsole scence +new tg +napp er +missing dog +mil ked +machu picchu +lv n +lolo lololol +li day +le moyne +kann apolis +jr h +inter mountain +ibm cloud +hang eng +fruit less +emphasi sing +divi ders +dh t +daft punk +bil as +anatoli an +ðŁĴĽ ⾨ +zeec inema +y ae +under fire +tuf te +stu mpy +rochester ny +prou k +preci pice +patri ka +mon de +lincoln city +iti onists +in ism +im m +ice fishing +duba imall +den so +dead pool +datac enters +darren rovell +d hol +co field +celebrate monday +cattle men +broo dy +broad hurst +bid vest +bal sa +ath el +an ke +an acortes +ad har +action aid +... ?! +ðŁĺĤ ðŁĺĬ +åĽ ŀ +w underground +view able +seclu sion +sali da +river dogs +re cline +quick ness +ol one +nx ne +l wf +ko ba +king stown +kevin richardson +kal ani +ju u +i us +gafe summit +estate agents +e mad +deme tri +born day +bent leys +! ðŁĺį +yedd yurappa +van gi +tristate wx +se molina +ro tarian +pou ssin +phosp ital +or de +okal oosa +newtg ingrich +me ar +lo pa +laidback luke +l fe +jefferson ville +h sb +gan assi +frédé ric +fli t +fla b +expect us +escap ades +discre pancies +clo ison +car sharing +cado gan +bird news +ba shment +ag rande +/ ) +ðŁĶ´ ðŁĶ´ +wit tering +waynes boro +tobo ggan +tn l +tho rens +tele pathic +sport sperson +sho bha +se ef +rit is +raghu ram +popl ar +pol onia +po tu +park ville +nomencl ature +nas cent +mtv news +mi dem +la sco +ku h +jay ate +ip g +ill or +hor rocks +ho co +gar forth +fr itos +fish monger +ebbs fleet +dru ck +del is +color guard +ble dis +av ni +as ker +arti sth +ac ast +, $ +ðŁĻĮ . +ì§Ħ ìĺģ +zo brist +za e +w pu +v rou +ut tra +torpedo ed +sur o +ste tter +seaf oam +sam sun +ring leader +resusc itate +rare bit +r gm +pointilli sm +pen fold +o aa +mc suk +ma ug +liveon wlos +kiwan uka +john tory +hyp no +ga hanna +fr ing +end al +en ation +dere ham +decor ative +customer success +ciene ga +ci marron +char o +card game +bra am +bom bo +bal in +aircraft forsale +ab ela +... !!!! +ðŁį» ðŁį»ðŁį» +âĺºï¸ı ðŁĺį +ا٠ĩ +wai lea +unitethe union +tippec anoe +sympathi ze +suppre sses +sinus itis +shale en +sac c +pro syndicate +pra vda +ph oops +off grid +novel ists +kx ip +kol ar +k ort +intern at +h plc +gu ss +gan nets +feels good +fan on +encu entro +devil ishly +cul pepper +cro quette +cho on +ban aras +bad in +ba ji +ari falvi +ai se +ðŁijıðŁı½ðŁijıðŁı½ ðŁijıðŁı½ +ðŁİ¶ " +wi ggy +tw u +tri partite +swa in +si van +re ham +per plexing +neve ren +morro wind +mono clonal +manu kau +ic ad +i at +hy wel +ger bera +g go +fra zzled +esc ul +el lef +dj fresh +confe ction +c gp +ast anglia +aldub nation +ad vic +ðŁijĮ âĿ¤ï¸ı +yu e +wy ler +wh ome +water less +uto pi +unquestion ably +salut ation +not ching +mans bridge +man tel +mali gn +lil ongwe +jon ge +iti zation +hunger strike +hol ter +hit the +f pv +dy brand +dun st +crumb ly +cr r +city scapes +cheap trick +catalo guing +born today +ban gui +ðŁĻĮ âĿ¤ï¸ı +win net +vi xx +un detectable +to co +team canon +seatur tles +santo sh +ro timi +rake sh +p illi +mon otone +m fa +m dd +luc man +kun z +kag ami +hometown hockey +giorg i +fl ings +eu lar +down wards +brae mar +ben ihana +be kaa +b ors +b nib +ar ko +." -@ +ðŁĴĥðŁı» ðŁĴĥðŁı» +îĢ ¢ +ìŀ ¬ +wladi mir +wash ing +u tero +tom an +super soul +su iko +sk ou +s museum +pi h +pen folds +parnas se +osoy oos +omni vore +ml u +mc minn +koda chrome +kno cker +harpersbazaar us +fit for +ep ine +e bu +dublincity uni +de bre +dav a +c gt +boy zi +adap tive +sto dd +shenando ah +s ella +run a +rickman sworth +re agents +prece de +pre meditated +poster deal +ou chi +one for +noti fies +n awa +mo by +middle sex +ket one +illumin ator +ghe art +fac tored +ess aou +en visions +davi dortiz +ca iman +brain wave +ba ine +ba illi +an ari +ak id +ab ry +ðŁĵ ĵ +ê³ ¤ +âļ½ï¸ı âĿ¤ï¸ı +vesta via +un subscribe +ste med +sp hero +sound stage +sol ari +scrun chie +regen sburg +pet friendly +n aff +mell itus +m wp +lump kin +liluzi vert +lets getit +lavish ly +laver ne +l ba +jiggly puff +jd mahama +jac opo +hermos illo +grou chy +gr nd +emb u +deb ase +cyg net +couturi er +can tile +buckey enation +bel monte +ar buckle +air tight +ðŁĴĸ # +⼠° +yeg cc +ye chury +that cham +tele marketing +tat chell +tai mur +sy doperahouse +re targeting +per missible +notre dam +mani kar +kel si +k pc +jurispru dence +jab alpur +il way +holly wills +hem poil +harvard health +h cf +fro sts +eaves dropping +cu zz +con ews +cle tus +beach wood +alicein chains +af ana +abbott abad +äº ķ +à® Ļ +zombi eland +un playable +tu o +super highway +stir rup +sri sh +se tx +rock bridge +re tails +quo d +purch asers +pen shoppe +pashtun s +pan ola +ou ettes +nai robi +menin blazers +long bottom +lo sc +kü stün +keralab lasters +kaf fe +jackal ope +is sn +in consistency +hug gies +hock ley +hedon ism +hack ing +good son +go jira +gil lett +ge ther +gad fly +fanta si +expan sion +enri quez +ef es +d li +carnit ine +bom bard +bod ys +be sigye +bas ara +aggrav ating +ad hering +ab rera +; @ +ë§Īë§Ī 무 +veti ver +tri athletes +su itor +shi kha +pun tos +p aged +one and +n wilson +n atty +maternal health +lat z +l nd +kal ai +ka sim +k ka +jean ette +in accuracies +ic x +hu elva +ho ch +gur un +gou ld +expatri ate +en jin +ed show +du cked +diffic ile +culture night +contain eri +cho pper +can vey +bas splayer +ai ello +acre w +a ahhh +ðŁĺĤðŁĺĤðŁĺĤ . +ðŁį ļ +ðŁĩ±ðŁĩ ° +ï£ « +w ens +transp ired +top side +skysports f +six flags +sh art +seasi ders +sas saf +pun cak +po b +pbs kids +passer by +parfu ms +p sd +ol low +ne ill +mo sse +mid d +mag u +li thi +ke aton +k won +ho x +eleg ans +ein horn +cu base +conce aled +bourne mouth +ab ata +travel life +thank god +relev ancy +of god +nu ttin +mic hy +martin elli +london fire +lin sey +kx an +in ada +hy mes +gar min +fun do +fox borough +fal setto +eliza be +e ston +cu buffs +co dw +cast lerock +car z +cal mac +boycot ted +bor th +bal ing +apocry pha +anz stadium +ade mola +ze sco +zam or +wel t +v alla +u ve +twitch creative +ta virus +strad broke +steme ducation +soder bergh +smo ck +slight ly +shoel ace +shab alala +sabras radio +s art +rockst areditor +red bud +real c +li oni +implement ations +hiroy uki +har well +gu any +fish friday +fi af +elic it +dli ssing +dis organized +buzz city +bo panna +bap homet +bah ria +avi k +ap ids +alamo dome +al cala +the east +the bi +speak out +so rel +sel ly +sch ofe +sa thi +rice university +re joining +pon tus +out liers +on able +md politics +lun n +logi stic +liske ard +letsgo ducks +l ha +kar ach +jet son +hur n +hern ández +gautam gambhir +e hhh +dun gey +dem is +defe ction +dagu er +chol ula +cav alli +capri ati +bol les +we aned +til ts +ti bbs +thor son +th yssen +quadrup led +pho tons +ny quil +irrepre ssible +inst ab +ii p +holiday ing +hee hee +gym pie +frene mies +fero city +exhib ited +environment alism +el lish +dol ant +cw b +confor to +b ny +az n +) ", +ðŁijı # +ðŁįģ ðŁįĤ +yan dex +vent nor +thin section +tao ist +takeak nee +si sto +sebasto pol +seas ick +sat yr +ro settes +nc cc +mont clair +micro focus +ma quette +kil main +je mi +gur jar +gold frapp +g ce +dont buy +di ssed +d model +cru shers +carpe ting +buffalo trace +art work +ar men +an sett +an ha +al ah +é º +è che +wes sel +val ve +tu v +tran salt +sol ym +singapore air +sch ap +revit alized +reg is +priorit ising +min ny +mi dri +mcmaster u +loo g +kon i +know le +kin ders +internation alization +i he +fab lab +ed filmfest +dad on +credit union +ci ent +che ta +be kah +batt aglia +ag ny +ðŁĴĹ @ +âĿ¤âĿ¤ âĿ¤ +x le +son top +so ko +powhat an +pe try +our ts +mo chrie +lau drup +l co +k cl +hi ye +handmaid stale +f end +diade m +d hara +case miro +bud dah +ano vic +ad sb +ðŁ¥³ ðŁ¥³ +� � +æ Ł +zam bales +wheat en +v scot +the legend +pri mos +pri madonna +pri ddy +pas i +official marko +must watch +lou yanong +la batt +jag ga +iti m +i ha +h pr +gal d +founders brewing +ep ers +e spark +clou gh +c tic +bha gy +abo at +ðŁijĩ ðŁı¿ +ç ļ +е н +zoom in +z oil +xbox onex +wa aa +tis rael +sw art +subpo enas +stu ckey +sor did +sic ario +sant é +rog elio +radic alism +play bold +p elli +op ta +nu l +norm and +ni shan +metro losangeles +medal ofhonor +ma iam +ley den +koko da +impregn ated +im ents +gw t +gar ri +edge hill +e cet +d cy +cu esta +col lie +cali sta +boxer dog +bishop sgate +avent urine +att weet +ak kara +ðŁĴī ðŁĴī +work mates +wal lof +town beer +south shore +roof er +rib fest +ri et +reimbur sed +ra zi +prayer ful +po pa +paignton zoo +out fitter +ni hal +m fd +killu a +kc ca +iron de +h ita +ei lean +e gr +din sider +di stance +de shawn +dart mouth +cinnab ar +boo ze +bi det +! ^^ +vach eron +uttox eter +ur fa +uno cha +tri stram +ten ough +stam endment +siski you +ser ing +scottho ying +rag brai +pay day +o gl +mcstu ffins +lbor ouniversity +kill inge +imiss you +graham rahal +flip grid +electric cars +city line +br annan +bad la +av ola +annie mac +ad one +.... .! +ðŁĺ«ðŁĺ« ðŁĺ« +Ñĥ ÑĪ +wild food +wic om +war ping +walu igi +vi agem +tr oughs +speci fying +say aka +piercethe veil +orig en +nh d +ma wx +la han +k Äģ +is better +inhib iting +i wu +holy wood +h fd +goosen eck +ga ine +g our +fm sa +f also +ey news +elope ment +dew point +che tty +ca org +bas qu +am ra +after shocks +tyler perry +tread mills +then y +terri fies +stor r +statu ary +sl tchat +shuffle board +serv ation +sc our +sam cro +rif kin +rebe ca +re tweeters +peanut butter +ni ah +national beerday +nas ci +mous er +me ji +lea rena +kristen sen +ino e +he mmer +grac evander +goo ch +go socceroos +go j +e tal +domest ication +do zing +dev net +b mar +amb az +air bu +acol lier +ðŁĵ ĥ +ðŁĴ ® +èªķçĶŁ ç¥Ń +west chester +un friended +uk raine +toll way +the americans +sty dia +stein metz +sar ong +sab ir +nieu we +mor mont +marry me +ma plin +kni k +kni fed +kash thefuturist +ih f +g wil +fun n +dianna agron +bry anc +blues y +anton ella +an ar +ðŁĺŃ ðŁĴĹ +ðŁij ³ +ðŁİĵ ðŁİī +ਠ¸ +ye hu +wil bon +to tten +ti zation +ssi s +sh anti +sare gam +samaj wadi +rot ted +realtor life +qu alls +perez hilton +om gb +never settle +nationalboyfriend day +matthar dybrand +mart z +lovel ier +leg omovie +lat rell +je sh +iam diddy +hypo chondri +ho ppe +great night +firestone walker +engv sa +dr k +di of +counterfe iting +counter attack +c wc +be shear +ar chang +al mas +ag oo +ðŁĻĮðŁı¼ ðŁĻĮðŁı¼ðŁĻĮðŁı¼ +ðŁĩ±ðŁĩ » +z oll +ton da +te shwar +su ara +ste mon +se moga +sau kee +sage brush +running day +ru eda +re install +rancher os +quean bey +our home +obse ssively +mike e +ke x +k ila +in wards +ill man +he ske +hasle mere +free birds +dist anced +clich y +cen sure +cb gb +b ateau +aspart ame +alti eri +ad mu +ðŁĴķðŁĴķ ðŁĴķðŁĴķðŁĴķ +ãģĵãĤĮ èģ´ãģĦ +whoopi goldberg +the chew +thak ur +tele text +su tro +spo tt +sa iz +rak ash +prismacol or +penn jillette +one gin +molo tov +mat sunaga +ma ir +kristen bell +kil by +ki z +ine ss +in cul +immun isation +hur tigru +hur rell +holocaustre mem +he me +girl sbball +ger al +fo or +earth works +dic tation +di one +dead head +de ts +bru le +ave dra +at rust +amp as +> :( +t vin +signi fying +pon ce +n gh +mon grel +mo sko +mil ind +jor d +j df +ide olo +hari haran +han bok +g ander +et c +dere ck +ce ca +cau sally +car rol +bbc cornwall +ap co +antigon ish +allo f +aldubeb tamangpanahon +warby parker +wam u +trader joes +ter an +stock yards +soon er +season al +pra gue +most ly +medi am +kul fi +kor tri +j mb +i yan +hershe ys +her be +entr at +dur kan +digital trends +cot terill +chante relle +cen ote +cantstop wontstop +bro mpton +british cycling +bo of +bo cas +bayreu th +b oud +ayo tzin +aw ing +al ans +agra phics +ago e +ê¹Ģìŀ¬ ì¤ij +ye vans +wy mond +wu xi +vil sack +un processed +se quality +s foundation +s bts +pho g +pang s +oc b +makeover monday +loner gan +live streamed +lister ine +ku be +joe perry +iron clad +hur ls +gro ening +ff x +e set +dian amary +cyber tron +clem ence +c we +ber gin +belli ssima +a op +welove bath +up v +un consciously +the avett +tashan eishq +tari fa +t á +sport car +song sinfo +snar l +sho ves +sco tref +reli shes +prob ationary +pre ity +park head +mazer unner +lin ed +kan ak +ich u +i et +hy pere +heide gger +g ms +esteel auder +ec ook +do ws +cra yon +classic ism +ch ail +cello phane +cathar sis +bri stle +bra thwaite +boo o +ba az +ar vi +ald red +ag ama +you can +visit bath +v fs +trust pilot +stree twise +random ised +r sw +pur pu +perman ence +otol aryngo +me gumi +may pole +lo dg +lan tau +kuro da +kill switch +its jeremyscott +infar ction +gul lit +go shawk +geno type +g ó +fal tering +du ms +de be +corner stone +chert sey +bul u +bu tting +bol dt +bo ser +bay sox +al ph +ab lett +ãĥķãĤ© ãĥŃ +ÙĪ ÙĨ +wire tapping +tom eu +shapp ening +rev war +qhu beka +pla ine +or adio +maynoo th +lo td +lets gom +lee ches +larry fitzgerald +ju le +jrs bbq +humb leness +harve sters +groo te +go saints +dru zy +de ws +dag ga +con tu +bbcn wt +arts fest +anci c +yan go +woo dridge +triple crown +the ft +t ton +summer iscoming +sub text +so ori +sarah m +refriger ant +real preityzinta +re sourcing +pol amalu +out land +nihil ism +naw al +marig olds +m ite +gu ti +gro sses +geni ality +fear ne +eu referendum +ehlers dan +dig ne +cre em +co design +blon die +abbe ville +ðŁĺĺ ðŁĴĸ +âĶ ³ +w fl +un flinching +trump is +ton ton +thinsection thursday +t tam +sun land +sr j +se ast +re pped +r studio +quarant ined +pbc sd +outer hebrides +outdoor living +off re +neg ation +mm ering +martin freeman +man cup +mac ewan +lobla ws +lin ke +kav ya +kam la +k lo +histen vscot +han gry +green wood +gar in +flori sts +flash dance +fe scue +dic ey +deci mals +chris froome +cdn health +brit a +bono bos +bl ay +bil lets +ber na +barley wine +apit ol +abb formulae +⼠ı +ਠ¤ +zieg feld +winter bottom +vo cs +tou jours +team er +te em +rohing yas +ra ba +pasorob les +n cube +mel in +mega watts +medical device +kuan lin +kinder morgan +impac to +gom er +fu taba +fu shimi +emb o +elu ded +ed man +discre etly +dan afarber +cypri ots +cot illion +boister ous +be ee +appomat tox +am strad +am ers +al tro +abhin andan +wwe tlc +work site +womenin leadership +williams ruto +v rt +trede gar +tay y +sf old +se te +ril akkuma +palm trees +na day +mil in +mas ada +mad res +live and +ktg tool +fir man +etsym ktgtool +dren the +daz ai +ctv montreal +cre asey +ci en +bor as +beem ji +and ries +ach amber +ðŁĸ ĭ +Ã¥ land +wink les +wallpaper wednesday +walk ability +w amp +tre tweets +transfer news +thevampire slayer +srimanthu du +shu bh +shrews bury +rei dy +que sting +pun a +par cours +pan jabi +ori ente +opp y +ne em +magsay say +m tech +m ended +london live +log ÃŃa +leaf sforever +kry stle +krist ine +issu arez +insan e +employment law +comp ounded +clin cher +carr é +bethe one +beach volleyball +ðŁĺĤðŁĺĤ # +x press +victor i +tm gt +sulph ate +si mages +ri ffing +ni guel +ne ke +mouth feel +main net +mac tan +ke met +john shop +inter dependence +humor ist +hawk girl +gu yer +fi des +fab b +dele o +dairy queen +bartol i +anor ak + ¾ +y anga +wood men +wh is +wade bridge +vindic ation +v so +uw bb +u in +th z +sun life +su as +steam whistle +spoke mon +spaghet ti +snicker doodle +pan ik +on music +north we +meach am +lin nea +kun ingan +kann an +ith ia +id night +humboldt broncos +hick ok +hershey park +ge ol +freder ica +flu oro +ex uma +dro ols +dispar aging +con air +cler k +bu ick +bi du +as ync +argu ment +an stey +wait for +vz la +ver a +vand a +u ot +tur ke +the creator +te arooms +tam as +t shirtday +sou bry +sand castles +re mixing +pleasant ville +phi le +park wood +o gc +n cap +moris ot +mo hs +mary dale +mam avote +kap alua +justin bieber +hal sall +greate scape +far myard +empowering women +bra him +black burn +biaf ran +bar be +aff able +ó nica +zombies run +wi der +wh ence +vig our +tu bal +stock piling +stan ton +shaf ted +phra sal +o ÄŁlu +mo wat +mascar as +mark levin +lion head +j nk +j ati +he mming +gr ice +ga it +es wara +el ora +ehr lich +drawthis inyour +deu ce +dan ge +coming outday +b ise +am ad +aldu bang +al insky +aggre ssions +abbrevi ations +wol dt +vh sl +trueto atlanta +theloud house +sub ha +stat s +stanis las +stag ger +self build +saha bat +s bo +recap it +rajni kanth +puri st +paul stanley +nau lt +msg networks +mon ts +mo selle +michael strahan +le may +ky speedway +kiri bati +infin eon +ilau per +hg se +henri kh +gc as +fi bs +fain tly +encapsul ation +dham i +de mer +cynd ilauper +brisbane tennis +boss anova +arom atics +ðŁı´ âĢį +à´ Ĥ +yo d +x fre +un concerned +tu am +to gram +tar kovsky +syl het +sy mph +ste g +school boys +scho ck +sch ill +sas sa +sas i +ree led +re hydration +psy chon +pro ffe +pon zu +play testing +official asroma +n anny +mon ie +meg ach +lo wes +kot tay +kai den +k gal +inter fered +hydro cephalus +ho fe +green well +ge urope +fried lander +flick ed +fishing life +dyna stic +digital humanities +di vino +dail yui +coral reef +cit rus +chow dhry +capac ity +bro yles +ber isha +austin dillon +:) # +ç » +ste mi +shoto kan +sh xt +sem rush +poly ps +pap ag +mo dot +marketing digital +mal lows +lad ner +inver ters +im pair +frost burg +fin is +enchan tress +em ol +d kk +clean room +cit inews +c ck +beat mania +b ican +ali ef +al bee +âĿ¤ï¸ıðŁ§¡ ðŁĴĽðŁĴļðŁĴĻðŁĴľ +üyü küstün +ze v +ygg dra +y upp +web designer +ve tter +tune up +tri ffi +teng ah +spi ker +silen cio +sat ara +roll call +red cliffe +re routed +ran k +ps news +per so +n radio +n cri +mun ros +mitch grassi +mis er +mil hist +mar otta +mac edo +lo am +kim a +kenne bunk +kati punan +in wed +highland park +far ms +engul fs +chym al +catho de +back board +attach é +ane wh +akhil akkineni +è ¿ +w lan +tend on +suvar nab +strou d +signat ories +reedtimmer tvn +my t +mohegan sun +ml f +marmo set +letterbox d +lane gan +is caring +inst illing +ine jad +higgin botham +gai ag +four thbewithyou +forti fications +dj p +dige stible +dic h +bath letics +ayahu asca +z cash +yan kton +wendy williams +twom ey +ti x +tar nish +super lig +sle ft +refugee week +ra st +pan ova +nectar ines +mao ists +make it +made jski +jah re +itas ca +iam su +honour ary +ho ve +hin ojo +from is +fl atul +down wind +dom ini +dark art +cp w +ci dade +che gg +cas am +ca reless +buffy thevampireslayer +boreham wood +bc tf +ðŁĺĶ ðŁĴĶ +world star +ur ine +sunburn festival +strang eness +snyder cut +ser ums +screw ball +sauvi gnon +rust lers +rag land +q ed +po va +past imes +over stated +os l +ne cc +me ed +marydale entrat +lv mpd +lin q +ja anu +ick y +i ghter +how toge +guy fieri +gro ans +ell oni +ele k +dax ter +cor sic +bread sticks +andreww k +ai st +ãĥ ¦ +âĿ¤ï¸ı ðŁĴĽ +าภ£ +wa shoe +ver is +spoo py +spe ws +spark man +snee zes +sn hu +silver dale +seth meyers +sensiti zed +sen yor +san ton +s fe +rosal yn +play adel +pe ale +ovi ya +newcastle jetsfc +mun dine +meetand greet +m bia +looks like +jo vi +jimmy choo +j ls +isu mmit +heritage week +e chuca +dro ga +dream warrior +ce ren +captainameric acivilwar +capit alized +broad water +blu est +be ppe +bar dsley +zild jian +ye mi +weigh ty +vas ant +tu tto +tex tural +te b +tag ma +streng thin +step ney +sta u +sli pper +sel loff +radio logists +mole sey +laver ty +lan er +ky humane +katamar ayu +hay lie +god ha +exc itable +e gov +dis arm +chak rav +business week +bil son +ðŁļ Ķ +ðŁĴļ âĿ¤ +ìĦ± ìļ° +wu stl +wr angle +ve ers +val polic +true man +timbuk tu +sch ris +sam s +sam in +salt burn +re affirming +pu ffer +pp b +po yet +par ia +p life +ow ww +mono tony +lon avala +l cc +kir st +ju sty +jan ec +is ay +ino u +inci sion +ha segawa +gust avo +fn ce +fag en +eric fisher +eng rish +dino vember +dharam shala +deutsche bank +best jo +-____ - +ðŁĶ ¼ +ðŁį· ðŁį· +ðĿIJŀ ðĿIJ +ula an +typho ons +tree tops +spra ggan +sb x +retur nees +r ila +pro ff +play pen +performing arts +newar tist +nd lovu +nascar playoffs +mur rell +maren go +ma kenzie +k ays +jun ction +jhal ak +ired ell +i der +glo bu +for res +fon tina +feel slike +es n +epitom izes +e tive +disp elling +db x +cro marty +co que +clip studiop +cher as +ca os +c pd +bomba stic +bed sheet +av ca +ac im +aay yub +za o +vo cm +va sec +tic ians +thom ond +sta at +sm un +sep si +re ton +pun g +peri ences +pel ting +pay checks +ome dia +o ban +o ake +nh wx +multil ayer +morpho logical +media set +fran cona +flat mates +eu gen +elrey network +digital skills +cross walks +col ten +bv b +aw wa +andy black +anand mahindra +è » +à¹Ħà¸ Ĺ +ú a +z enger +yn w +yel chin +x plore +woun ded +t fully +sv n +sky bound +shu bert +sg l +se de +san ews +ri ds +pu chong +ost ent +live tv +keegan allen +inver sely +gle aned +f hi +edu tech +dy stonia +del r +dar rell +cor am +bou les +be vo +ar nott +anci en +alo of +water stones +u buffalo +sweat shop +sto icism +spot ted +show box +read ability +rav age +raj e +r ze +py r +mip com +mi edo +mcdonnell mp +john mcdonnellmp +head first +glu ck +for india +fon terra +fill more +du ero +cam il +anthropo genic +and sara +amandat apping +al lee +èĹ ¤ +y sp +y hu +winter thur +trabzon spor +sumat era +sta i +six pack +rum ney +re style +ple gic +pal lotta +music therapy +metal heads +maythe fourthbewithyou +matth dgamer +magi d +le pen +l sr +kim s +kala ign +jun co +jamie foxx +it ates +imagine ering +il son +go snell +glen ville +gam ble +fibre glass +el dar +doing it +classic tv +boo lean +ba yof +b ster +art scape +aj p +zind agi +visit cz +verraz ano +un scented +uhur a +thevamps brad +then ame +sh ama +serious ly +sas m +rose of +red foo +ra chi +r bf +q al +orlando pirates +mor ra +mission impossible +me hl +j elle +hilde gard +ge eth +g nd +fro sty +family friendly +chris stapleton +cb cradio +cacci atore +bur ne +bak tan +aussie grit +astro world +archit onic +an col +alban o +wil kin +vindic tive +troglo dy +ti sci +standup for +sm tv +relais chateaux +propen sity +mmmm mmmm +li ppo +gett in +get better +fil ia +em ilion +e sen +dol in +cityof vancouver +cal oun +cairngor m +bay hawks +ate er +amer it +âļªï¸ı ðŁĶµ +vi st +tit ic +sur rep +sty linson +stu sa +stego saurus +sonam kapoor +sau cep +rat aj +pv l +pike ville +pando cruises +ny phil +news space +nand ita +n du +megam ix +m musi +lmfa oooooo +jeff flake +j anne +ir n +implo de +ger da +fi ras +cra gg +con fig +cle ave +ch anna +ca stration +bor gata +bellige rent +as cs +ab kiba +âļłï¸ı âļłï¸ı +wom w +ti ous +te sh +syl vain +summar ising +soccer aid +sk or +ra om +oe il +north norfolk +mo ët +mjol nir +litt lerock +le var +le pt +kelly rowland +k hul +jo kic +io an +hedge funds +ging a +gi otto +do ink +dianamary sharpton +cow den +com d +chon dro +cal eg +c sh +ber ns +americ andream +wa ster +v ons +t fg +sud ha +stoo ge +spl ace +so chi +seal ants +photom ag +pedag oo +mc muffin +mammo grams +lat ure +i in +gram pians +gemm ell +for ager +ev illage +de toured +day fiance +cor ry +con notations +ch ink +biker un +ar ima +ado on +ìļ © +ãĥ« ãĥ +иР¸ +Ä Ļ +whywe do +villar real +twy ford +thelo cal +tc cc +squee ze +si sson +scoundre l +sam mi +planet newsspace +peek skill +p cr +one india +near pod +ms ba +malay alam +jail broken +jag gi +grou pe +go bulls +gi ds +g fe +fra ga +fox tail +fitness goals +fero ze +evolu tions +en trust +ecu rie +den pasar +culmin ated +confis cation +ci oc +chem nitz +bron cho +agu ero +.... ...# +æ £ +ãĤ¢ ãĤ¤ +ãģ¦ãĤ ĭãĤ +valentine s +turksand caicos +tubab üyüküstün +tour ing +ta von +stay positive +star rs +sa af +ro isin +pollin ate +plan ecrash +phoen ician +o des +mt z +mau ro +manag e +lafit te +l sf +ki ka +house guest +hispan ici +fin tan +dead liest +ctv windsor +cop p +coon awar +come froma +chel ly +charn wood +ch up +brown ed +ar amark +aerop lanes +z elle +willand grace +watson ottawa +v elli +un recognisable +to ti +ti bi +talk live +so st +show tim +ser co +re tort +pd ate +moren cy +mor ita +mil pitas +men udo +manag ed +m wm +kh saa +intern acion +incar nations +helle bore +gas a +as ra +alternative facts +alca zar +ðŁĴ¸ ðŁĴ¸ðŁĴ¸ +á¹ ĩ +ve sey +underthe stars +tre bu +thisi sa +sp azio +sau dades +ruf ous +represent ationmatters +re wrote +ram esses +profe ssed +pay ette +pa quette +ontari oparks +nam ish +mill in +men dip +man es +jic hang +ja ide +ice a +hos ny +glu tton +es ke +es fc +cabo sanlucas +bla ine +big fm +ben salem +bc m +ame obi +: [ +Ê Ģ +wi es +wak ulla +thelibr arians +the bo +strong sville +sportsman like +spe aring +skri pal +sc aggs +repub blica +q asr +pix ect +op har +ontari op +national lottery +ko ss +k lassen +k ere +jungle book +jav an +intervie wee +home maker +ham ara +ha ssett +gar ages +frank sinatra +fin ning +england hockey +df n +dan ske +d pg +craz ily +com patriots +but an +bor thwick +bel anger +anu sha +an b +al mighty +âĻ ¨ï¸ı +va stav +tis ans +te yana +tay to +stay lor +schen k +sanc erre +recy cling +priyan kac +pau lma +mu see +monte fiore +mmusi maimane +load out +kis an +k los +jim watsonottawa +ilo vers +i ate +hun ga +holiday shopping +h pu +glo ssier +form work +fir dous +far well +falcon pride +en our +craftbeer hour +co ren +cat on +car news +c isa +bam f +ad k +a hari +ðŁĻĢ ðŁĻĢ +wall flowers +visual arts +turk men +tanquer ay +sol des +sla dy +sch nee +sar ina +rick grimes +revolution ised +pu gets +pra ga +post ables +mt scores +moder na +min are +mar ula +man tic +long listed +llu via +lawren son +lament able +kil dare +fr and +dri dge +devon days +de gea +dar c +dang an +dal is +coral reefs +con gee +col bie +bo ji +be sity +barn well +ak zon +ĥâĸĥâĸ ĥâĸĥâĸ +Ø§Ø ² +wolf ville +um news +tran ada +tin sel +tar ah +son gh +skill set +see red +sad hu +ros anne +rober tir +po zz +po we +nb p +lasor da +l sb +kun dan +ke b +ja ques +ind superleague +in oc +haku sho +gri eves +gov ph +gi kwang +f mp +eni or +ditch book +cut scene +bergha us +bander snatch +ant ena +an adar +agap anthus +íĤ ¤ +thur ber +thoro good +spe zia +sh eckler +sea ham +sc aa +re learn +punctu ality +prop eller +prayfor paris +pen se +p caa +os er +nam ib +mic hu +mcclo skey +mac laine +legu me +la ba +kc star +ir cuit +iam rashmika +he ol +ground skeeper +flower pot +f mb +e online +du bose +di atri +corn ering +chi ari +cad ena +cab an +bussel ton +brock le +big ly +au kerman +y ellin +wal der +ut leg +tahle quah +shat tuck +se min +rosen feld +republic of +ren aul +re star +ok ine +n naji +mumbaim irror +mar joram +ki elder +jump y +john sons +iran regimechange +in bloom +hom iny +ho ssa +hal as +ey c +eff ler +dynam ix +di vot +den sities +conceal ment +cha v +buri als +bs official +bloomsbury books +armid ale +ador no +$ / +ðŁ¤ · +x wb +uc sandiego +team liquid +taze well +stefan o +ramsay z +people power +north sea +neo dymium +multit alented +m els +losange les +kh u +jo yl +idy ll +houseof lords +en gram +diver ged +dheer aj +dash on +d abo +cra bbing +cr ony +cor rs +christmass ale +am supdates +allyou nee +ðŁĵ Ħ +ti bi +ryan lochte +roysoc chem +ro ddy +por tre +ol av +oh m +o dac +newh art +nbc chicago +n no +mont parnasse +mid ges +melbourne fc +marin ecorps +magicrock brewco +lu da +kay lie +i pac +hed nes +hand els +gameover greggy +frau en +fe as +erici dle +end ura +e hi +diarmu id +cp n +blue bombers +anat ole +ab bie +. (: +åħ ¥ +ãĥĥ ãĤ¯ +ü h +yam agata +vin as +vi di +thro bs +the journey +reli efs +quiz let +preco cious +pre fall +por ation +pe irce +ox x +no gi +mallo famerica +mack ay +kin card +ke g +kam m +is af +gol maal +giveblood nhs +gha stly +gh illie +fo bs +earth worms +deo dato +craw dads +bobb ins +beile in +az ul +alab s +ach em +... ðŁĺĬ +ðŁijī ðŁı¼ +tor iyama +tnt drama +sherrod brown +scru tine +r gt +poo ps +obtu se +ny kaa +mul ching +makin ghistory +latte art +laken heath +jet stream +ir by +guine vere +fion a +eni ghts +de ped +cat girl +carnegiem ellon +vill ers +uchi da +ski v +si fu +sc a +sbspop asia +s ape +plane tearth +pic os +pet z +parsi pp +metac ritic +lu sting +le tty +in humanity +ima an +haw ke +graven hurst +grand i +glu tam +en gie +devin nunes +dev fest +cur ving +bud ha +ave o +atalbihariv ajpayee +ao ka +agu st +adri el +aco splay +* ¨ +ìķ Ī +ãĢIJ # +za die +wat ney +washington ian +unfor given +u oc +tax on +sul k +squat ter +sp asm +side bottom +shut downs +sand bag +run with +re producing +r br +pal ast +op io +novoro ssi +nin aturner +me shu +ky d +ju rist +jen nab +jami at +in chem +hy bris +gracevander waal +dav on +dak ah +care ers +carbon ite +car ley +bun a +bal four +amazon video +al cor +ac is +ðŁĺĤ ðŁĴĻ +ðŁį· # +ðŁ¤ŀ ðŁı½ +ìĥ¤ìĿ´ ëĭĪ +åIJ į +⼠½ï¸ı +wee ded +v tol +temp ers +sc ute +saf dar +reha bbing +purenew zealand +pul liam +publi b +pri sma +poun cey +polit eness +mccl at +maggi el +kil mac +juic eplus +it sha +i fa +i ava +hir t +hak one +fac tually +explore more +ell c +discover tasmania +day lights +cosmon auts +cer rito +burgess dave +bron chos +ap ter +ani mus +ale g +ab ag +ðŁĺħ ) +wo ky +wag ers +wa as +van go +un refined +u streasury +trum prally +too ele +t acked +soldby auctions +sk ank +sirac usa +pu mm +prote us +parach utes +or nothing +olympi atheatre +mus kies +making memories +li ard +ge mb +g ch +fr its +eu stis +esc congress +emotional intelligence +di mmed +culo ttes +chall is +cer ner +books thatmatter +bo das +aperiti vo +ad ice +ðŁĺķ ðŁĺķ +ãģ Ĭãģ +zip p +your mind +whe ad +v ap +u co +trav chat +thuman ity +te abreak +szcz esny +synchron ization +su stran +ssi o +ske cher +sati sh +reconc iling +rec tified +ra wi +petro bras +nu dges +nom us +mor ag +mag de +hr l +hel mer +h lp +fear on +fa strack +euro gamer +drunken ness +div ar +den nard +d cli +conserv atoire +club app +bu ty +bloom ber +bart els +bam ako +angel as +اÙĦ Ø´ +Ì Ĭ +u za +thupp akki +shu te +sandra bland +s assi +re play +pasi fika +p vi +monstro sities +malay sian +macca bees +ki ps +imper ator +gam bo +g pac +eloun or +bha g +ิ à¹Ī +whit sundays +voy agers +river head +non partisan +muri el +matsu yama +maje stically +macro phages +insu fficiency +inhab iting +ilooklike asurgeon +high smith +guitar ra +goti ges +flu ential +fai led +fac man +de ten +clement ines +chu u +carolin elucas +bre be +bigg le +ajay maken +affo gato +ac abo +ðŁĺį ðŁĺı +yu ga +will mott +wheel ies +wee l +wb ina +squ ids +skeem saam +scar cely +proud mama +poc keted +plac entia +penn zoil +paulstanley live +nak ano +mir zas +mcke ever +mc michael +mc flurry +m dn +lion sofficial +lily ach +le louch +indi gnation +horse head +frie sian +food drink +eval ley +eric church +drum set +co rel +brze zinski +bit sch +banned book +alar sson +abscbn news +yebo ah +wehr macht +w out +uk nikon +tar heels +sw y +southern charm +sof ty +smi there +sk impy +sk ane +seatur tle +se ely +s fund +s da +pois oni +ok an +mn n +k any +im possibility +ico tin +ho cks +hed berg +head stock +he aped +flo ss +eng lander +delinqu ency +debut ants +crystal palace +conven tionally +cich lid +car mody +boyzi imen +both vote +atat ürk +ann ag +ank sha +air o +abra xton +ðŁĺ³ ðŁĺį +ðŁij ŀ +âĺ ł +youn tville +y air +v cd +tv f +te gel +stor rs +si han +shi fa +shaz ia +q az +picture cumbria +pass more +pa ined +pa bebe +mar rs +lu y +ju x +jam at +happy thursday +guing amp +grand daddy +f aya +daruss alam +bol zano +bol din +bar is +ax n +ali as +under pin +tu to +tall i +styli sed +simp kins +santam onic +ry lance +preserv ation +ppy grewal +pix er +om t +oireach tas +obste tr +namish taneja +na res +miro tic +mini me +m ta +kaise ki +ities news +hay ling +government shutdown +go wildcats +ey er +dra win +dol f +discover on +di bble +del la +d jones +crun ches +clo p +ban yu +bali stic +ba at +as alonga +an tuk +alber ti +? ðŁĺı +ðŁij Ķ +ðŁ¥ ĺ +ìĪĺ íĺĦ +wa ir +unevent ful +unbe arab +sin os +scottish borders +q o +puri fiers +por i +pl ena +no pd +nj siaa +neu schwan +neck er +mun tin +mill z +ma ith +lost dogs +ld jam +laugh er +k ml +jal sauk +irrit ates +il us +hr h +hope for +ham line +ha be +google analytics +god spell +ge x +edu n +dreamwarrior pic +dave mustaine +che yne +cel so +cal umni +besti ary +bankof england +bab ie +awil de +ad achi +unir dg +tu chel +then est +swar aj +sedge field +resor t +qur anic +qual ms +ok on +nak ai +na oko +macau ley +locu sts +leve son +le sa +le oni +juven il +j ere +iv c +in ane +h acc +guyan ese +gor tat +goal ball +go kul +gho strider +eddie redmayne +ed int +cor tney +claw sup +ba ar +at ap +ar mament +anim es +wat u +v ski +tre ve +tr x +timp son +th wa +tcs nycmarathon +song joongki +sha po +sapi enza +ram pride +pdx tst +patcho gue +p ki +ori be +on uk +movie challenge +maren morris +kore as +i en +hell omy +hallo we +em mi +dar ke +co stel +chat urvedi +cassa dee +c sun +broken hearted +bel ag +an dia +اÙĦ ب +vacation rental +ultra boost +trun cated +to pline +revolution ary +ram pling +proudof you +peace keeper +pe mra +pare to +mote gi +mass aging +je de +hell fest +f illi +en cores +ell amy +deb tors +dayo faction +cs ra +bro z +book mobile +bm l +bar field +apel doorn +air conditioning +ach ts +wash times +ut ela +u ia +through the +te il +sri dhar +sal af +ruido so +ro tat +ro ka +pom pei +mc d +mar nier +mar ga +lumin eers +leagueof performance +ken z +gl ings +gau tama +forthe throne +for ger +flam es +fin landia +faul con +fall fashion +ent revista +den ard +dc moments +cold stream +chak kar +cal exico +bun i +bi annual +bar kin +ar ola +aj stream +âľĪï¸ı âľĪï¸ıâľĪï¸ı +stu cky +sn er +shol ay +pre load +modern ise +kal ki +jo gi +j pt +in def +hen rie +havan ese +hau schka +ham an +g df +em acs +devi ations +dean er +chisle hurst +campbell sville +bu emi +atal ks +ðŁĴļðŁĴļ ðŁĴļðŁĴļ +ðŁij½ ðŁij½ +you uuuuu +womenin medicine +then e +tat v +sv u +sla c +single payer +sanjeev kapoor +pale stra +ork ney +or ono +nor well +mtv india +melo die +la gov +kri ek +kashi f +jon z +jeep ney +iu bloomington +hop kinson +high wire +go st +gho sting +fren zied +even ingen +eg ham +dhan teras +cla wson +camp bells +cal poly +ashi i +alek sander +[ .] +y ero +vu eling +union ism +ucl g +u vam +u pei +tuolum ne +them is +summer field +su ss +ssi d +sni per +se jeong +sand rine +reser vists +pe ville +palme iras +of d +ni ans +ne uk +mo el +m hi +kim so +har borne +dim mable +dali o +clever ley +catchand release +brac ey +be leagu +bar bra +art prize +ar ana +weill cornell +vi dence +tax onom +swash buckling +sre mmurd +soe karno +siddi que +sergio kun +scu ff +screw drivers +sa avedra +sa ag +pen ns +overe ating +noo bs +mar vell +lingh urst +lau rier +kel ton +jo ists +horowitz la +hann aford +fro mp +flu tie +fe tty +eric sson +duc kett +ash ree +aguero sergiokun +adam horowitzla +ðŁij¨âĢį ðŁį³ +à¹Ģภŀ +x ell +vic t +tvac tress +sub vert +spam mer +sch ade +oore doo +nicol son +mu tag +man iz +mal avi +lilyach ty +ky n +kra borty +k ady +j ons +hy u +ho he +her name +h su +du ally +dru k +dai kin +car digans +bread winner +bick i +ab lack +!!! "@ +æĸ ĩ +âĨ ªï¸ı +zee music +yiel d +x ilin +to pol +t fd +stu mping +sk f +sd hc +sau té +sab le +po so +pink man +ph ere +north olt +monday night +malign aggi +lo to +le si +jon sson +indigenous x +hin n +gour ds +glo vers +gir ard +essaou ira +ci ales +broadway world +be lem +ar shad +ðŁijıðŁı½ ðŁijıðŁı½ +yugo slav +us fw +ug adi +testimoni al +star mer +sleu ths +sla ving +shrey as +she in +re drew +pan ag +pah lavi +pa kenham +neu tering +nar al +jor i +jan h +ig we +i sherwood +harge isa +gu al +gandhi jayanti +copic marker +che atham +candel aria +ba red +apat ite +ani maniacs +am phora +adel anto +abr live +yh wh +whoo p +vernis sage +trini da +sugar man +sil ber +shir ting +sande ul +ques ada +problem solving +pear le +nu disco +ni zar +nando suk +mit ri +life e +l br +ie ga +ide alistic +home away +ho ley +free mium +eli pe +ed ler +d ld +cinque terre +centre ville +cat rina +c gy +c bus +beau tician +atx traffic +ani o +american history +ag v +. ðŁĺŃ +é ħ +âľĮ âľĮâľĮ +âĸ½ ` +y arm +wkc dogshow +velve teen +travel leisure +the ists +robert carlyle +rir insider +rachel notley +ra ashi +qual it +pre conceived +par lo +out live +national hot +magall anes +ma io +lend l +le witt +kra zy +kir tland +ir am +im r +ho bos +hess le +friend shipin +far c +ev is +confe ctions +com atose +ch aga +bi hu +bas si +back strom +alexand ani +acti o +yu pp +yak ult +we work +w cr +vi enti +uk m +ti ge +the hunt +the edge +sop hy +si sig +sher bert +pur ged +pu sat +pon ca +po blac +ob ed +n any +missing persons +ling aa +lat us +kay tranada +ir m +hunter sville +gl itchy +fa ia +eve do +ea w +doc ile +detroit gp +de cryp +carrick fergus +c vo +ble h +bd ch +back links +ade st +wood chuck +w ch +u maga +tol kien +theologi ans +the bachelor +stro mbo +sk ara +si z +say re +ra ppin +past es +of it +nutri bullet +nhs ft +napo les +mu gan +mc men +mari bel +mar ker +m haw +le elan +lauren cohan +kidney disease +kell an +ke efer +kach in +j ts +j ins +ili ke +go bain +ex tram +dru pa +diver ging +cove ting +clark ston +clann ad +cel yn +carolin ian +canad are +calibr ating +bis ley +b pg +b marketing +un cooked +tumb lr +st ong +spi zz +slow food +sapp ho +rich mix +ran aayyub +por tic +pic ante +pand i +nat ely +li sette +lais sez +kingsc ross +joon dal +j tf +it weet +incub ate +ill u +hor vat +heen im +giant spride +fr action +del in +clay pool +bul ous +bo ka +bir acial +benedic tion +ame et +ë ĦĪ +yo del +wangar atta +stefan ovic +sp gh +sno p +sm oul +sk ream +re iser +re considering +pr f +out looks +msle asalonga +mi j +materi alise +lib ation +k ise +jar din +ho yas +for b +flu shes +decentral ised +car rer +barre tte +bail ona +ard ley +apol lon +antic i +ðŁ¤ĺ # +æĹ¥ æľ¬ +ÙĬ ا +uni studios +unfur led +tat to +roller blading +recu se +ponty pool +omi dy +oc cas +nav es +music ph +move over +mas roor +ick off +hunter ian +dum fries +dermal og +cy tic +boy er +ask me +an ina +aggreg ated +> . +ðŁĮ¼ ðŁĮ¼ +z omg +yorkshirec cc +uri sm +together for +th av +sub unit +su ture +schn apps +rosal ina +re investment +planet shakers +ok t +npr music +moor lands +le ff +jr nl +jaf fer +hel looo +he ere +gre endale +gho sted +fo zzy +ex moor +esthe tician +ed om +dc w +crazy richa +cham blee +catholic church +by laws +as ou +arm rest +am ada +alessand rac +ðŁķ ¹ +âģ ¿ +zer ian +une ce +u bin +stop ing +stad ler +smi k +re balance +raw story +prabha kar +path finder +pas h +mimic ry +marsh acollier +low life +ini sh +ha pa +gal legos +elimin ations +coqu elin +cli max +chi aki +boot co +ath iy +alle tti +allabou tit +active snp +,, ,,,, +ÙĨ د +zo ids +xx y +war angal +roo h +qu b +pc po +par x +nv q +mt x +mc sally +mahar ishi +ke on +islam ia +i fic +g attis +fabric ator +f pr +es am +ear tha +draw bridge +don ie +dean sgate +dak h +cul pa +cran leigh +cor fe +clon eclub +c panel +bo p +belfast cc +barre led +ìĬ ¨ +ê ¶ +âĨ ª +viole ta +toic itiesnews +stat uses +sol dered +red bird +r mf +ov ulation +no ordinary +niz ami +mi gas +lucas oil +ley enda +lear ned +laser disc +jose on +j lr +j har +id fc +harro ld +gestal t +ger m +d bm +cou g +cooking with +commun it +cardin ale +bu gg +book cases +bol stered +blended learning +bir do +bha dra +atra k +andre ea +anast acia +ÑĦоÑĤ ог +wu b +wrist watches +stake over +spir a +sexy list +sa pere +rhi wbina +rady o +quantum computing +pin son +person alizing +path finders +ny y +nag be +mat zah +margar ine +knock hill +infin ito +ic lei +ic ate +en slave +dream coat +death note +ct b +cre ar +city winery +cic illine +christi e +cat man +cas kett +bre guet +blue hens +apa thetic +ani as +ald ine +ðŁĴĭ @ +æĿ±äº ¬ +wen o +wel ter +van ek +u hhhhh +to cando +swi fty +suriyaf an +stu tz +sch eveningen +per lis +paulo coelho +over hang +lin ley +hul kenberg +hot z +he man +google foredu +funinthe sun +fl travelchat +dynam ical +dutch men +die ter +deton ate +co gan +boule h +benavide z +an ek +x js +wor sted +win mau +win ded +we wan +ta iz +step daughter +sten cia +so wed +si si +salut atorian +ryo bi +philat elic +oned ream +nx tuk +namo in +mar can +mak kal +lille hammer +ii da +guil dhall +g illy +euro group +ere bus +dy isi +disturb ingly +could be +com ex +cl and +chat field +caf cc +biancon eri +bet z +bar da +az os +aeoli an +ac tof +<< <<< +zhiv ago +virgin ians +vi vam +uni brow +ti ques +sten holme +stel e +soundof music +revi v +resc ind +poblac ion +oscill ating +oo ol +om as +nex on +new statesman +ne sham +mu ffin +mi ere +mar rero +ma este +li ans +leopard stown +lakme fashionweek +kin o +kav u +history vikings +hair salon +h cb +gaz er +f out +ex ander +esp ress +end times +dra wer +docu mental +con geniality +chi ddy +charlie hunnam +carlton fc +butter finger +beach soccer +atp finals +>> << +å¥ Ī +zam an +the hockey +sad da +roy alist +rough ed +ross man +ram parts +punc tures +por sha +peripher als +outdoor learning +night watch +nct c +mac lin +ma gher +loud oun +littlebig town +ligh tens +le quipe +lake superior +kawar thal +if sc +hit ya +her iot +gold key +gaw ain +eh fcl +ee oc +dig weed +de tractors +dat um +dami ano +cho be +auto complete +app lenews +air mail +ac char +wow zers +whit ton +to rey +stan ly +sju bb +sir leaf +rusten burg +ron en +richmix london +pun it +people first +pauly d +pan chami +no emi +ni pping +mc devitt +mayor bowser +madison ville +mac dill +levis stadium +je sper +hyun da +hydro thermal +hed lund +hadas sah +goo dread +gla res +ge ysers +frene tic +firm ness +filip inof +feder ations +exer tion +e glin +d pf +creative cloud +cre me +con an +bro thas +bill cosby +b ings +ar mer +ap on +aer os +ðŁı¼ âĢįâĻĤï¸ı +íģ ¬ +âĻ¡âĻ¡ âĻ¡âĻ¡ +war plane +te ton +te star +starwarsthe forceawakens +signat ory +sho bha +shi rer +rh ône +repre hensible +ra es +per ma +ob stin +nap ster +mo ses +marime kko +iq ra +i thome +hun tin +hun stanton +haleso wen +gal as +g ica +dis repair +bra vos +awas see +apol o +alb ace +ac ls +ðŁļ¨ : +ya q +whit ef +w bf +stewar tha +soap stone +slo th +ru el +re mender +pe che +ng v +mu ggin +me es +maken na +khoo b +it jobs +iphone photography +hu mus +honeymoon ers +go st +ge stu +fran conia +fill the +en cen +eli verpool +disp rove +din ck +cy ru +chef jose +canad ago +bom be +aloha friday +ê´ Ģ +wa chu +up swing +un occupied +tuc kered +topo f +to go +sh ills +sch itt +sch eck +royce da +ros lin +pu bl +post office +or ga +openg l +om adrid +nal u +mini mizes +meteor ic +maryj blige +mam y +jump suits +he ft +hahahaha hahahahahaha +great devondays +gen ki +fla il +epicure an +dan sby +coffee break +char tist +bun des +ape hu +ap tn +ap aches +ag ios +a aye +ðŁį ¯ +åĺ ī +trent bridge +tor rie +thi stime +ric ker +ri bena +po sses +ple be +ph iri +ni vin +mike bloomberg +meh reen +martin sburg +lu cho +kapil sibal +kant or +joey graceffa +isol de +is ks +im vu +ho be +gis ela +gene therapy +f sx +earnest ly +do by +display port +depos iting +de mba +bart lesville +bald acci +ay az +at mel +ar ang +al shon +al fc +aau w +:" "" +Ľ ï¸ı +ðŁĩ®ðŁĩ³ ðŁĩ®ðŁĩ³ +É Ļ +zo g +where fore +web isode +travis barker +te mi +synthe tic +stinger sup +spra ins +specul ated +sob scura +sleep walking +sd u +program m +ne igh +mur ata +mi is +merri weather +mar mel +lul worth +jack septiceye +i fo +he mo +guest book +fú tbol +dais o +co sponsor +charity miles +cat son +bou ton +belgi ans +avail ing +at ou +at ennis +ਠ¹ +tip toe +the biancadelrio +st d +s wr +ram pal +priyan kag +prett iness +pom mes +out grow +ny fa +nov ak +nam ik +man is +lo fi +livepd fans +liveon fox +le ol +ji ao +is les +ida hot +haverford west +esk o +elton john +eamonn holmes +dau k +constric tor +choose chicago +bu mbling +bau me +band aged +aw amba +ar it +al ongs +af finity +us ns +tor rence +the kiranbedi +teessi de +sh antan +scra pyard +rade be +r hc +outer space +nf ca +nbc chicagofire +mon zo +me da +mary poppins +k db +jug ando +indent ured +hoo ting +hard shell +ghaz ali +gal it +foo dies +em mie +ee et +ech of +dru mpf +dontdrink anddrive +dol drums +d ury +calli ope +caff è +br illo +arte mis +ao sta +and rus +alessandrac icc +! ðŁİ¶ +ðĿ Ķ +z ile +yu sef +vivi ane +vi os +v lt +v angeli +un scrupulous +trom pe +to ph +thorn bridge +the gro +stra ding +soul child +sav el +richeli eu +red ruth +pr illy +por ing +our world +on ca +nerv ousness +nap h +mc bryde +lam e +juicy j +j fc +ine fficiency +igh i +femin is +farring don +dublin ers +dj e +cli psal +cassadee pope +bodhis attva +bar bies +back page +as ab +anci o +ance stry +all rounder +afro futurism +win et +wa ah +tor um +ta vr +superlig aph +nau man +mu stered +ly sis +kra i +k las +jac kier +j hon +ima go +horn sea +hed da +ger bil +dontmiss out +conserv ators +conden sing +cad well +bru der +bra he +af in +? " +âı ª +ti fo +th ara +steam roller +shane west +sa a +rye dale +rou ts +recover able +punche stown +p bn +our perfectwedding +opio ide +on on +obl iterate +no kom +nc n +nam ara +na seer +mart ingu +mar xists +lasal lian +kar ky +int aglio +hi u +gou let +gabbar singh +fur fest +florida state +editori al +cnn news +cal tex +bush mills +blan chard +bel it +bab as +ðŁĺĪ ðŁĶ¥ +̶ Ì²Ì +vaul ter +tokus atsu +ti ddy +stanis lav +sports woman +spor tiva +sor t +so cha +q pac +prime ira +overwhel ms +out lying +ott omans +nm leg +nk f +nelson chamisa +ne gri +mother jones +mir na +love u +li gier +ku yt +in ou +groo t +great again +ghaz ni +gh unt +fal ken +er om +colon isation +cb h +c md +bra vas +bougain ville +beach day +av chenko +ar ashi +ap ac +anton elli +z its +tre lle +t sing +stom ped +sky racing +should be +shan ice +san ur +rain nwilson +outsider art +ore ver +mer kle +lon d +la ith +kiwi fruit +killinge ve +ir shad +inthe morning +international artist +goul art +gla u +fin efood +e ki +dejec ted +dead lifts +coer cive +coder re +coal itions +cli ss +class ica +cab aye +c dd +bu hler +bin dra +basto gne +as sey +white wine +water quality +the dj +solar city +sir tis +sin ning +scar ia +q rs +py thon +portugue se +pick guard +pi pi +path to +noti fs +nl m +mo sca +min sky +mat ers +hot tub +hoo f +ha ws +g agan +fo amy +fan expo +e wan +deci sively +colouri sed +cash in +care r +callof duty +blue mix +bino che +bel tane +bel ding +be are +anim ated +ðŁ¤ ® +âķ ij +Å Ĥ +xylo phone +we tin +w tw +time stamp +sunny brook +su bre +stal ker +she el +season s +rootedin oakland +privati ze +o hhhhhh +marin as +la zen +insp ite +good year +god z +family vacation +diagon ally +del hs +cru ick +becken bauer +at so +ðŁĺ· ðŁĺ· +welo vel +usarmy reserve +un surpassed +u ty +tor tie +su mi +springe quinox +sp roles +riv onia +one ill +oly nyk +not be +mar g +kurz weil +itu al +hand sworth +ham mond +haemorrha ge +gan gre +for zan +fla ke +financi ers +fashion illustration +fal i +cli mes +cin q +champion s +cecil thelion +az ion +ash burton +ðŁĽ °ï¸ı +yo ffs +wi ston +velo ster +unite here +un surprising +u fos +ton ko +the punisher +sudhir chaudhary +sh mup +rou sh +pal lett +omak ase +nod ded +ne ste +milli e +loui stom +lam ine +i believein +dun kel +der r +cap taining +bowman ville +billi ee +afric as +adon na +ãħİ ãħİ +¡ ¡ +zar alarsson +uk manufacturing +the zone +sun ity +suicidepre ven +stan n +st johnam +slo cum +re caro +pil ger +par fitt +maur itian +marac as +leon or +ki drobot +juer gen +job centre +inter tidal +hen shall +gom usic +fire blade +ers music +duck duck +di zzle +dain tree +cour teney +conden sate +com poses +cc j +cbc sports +aki ko +absen teeism +zo on +win nick +the division +talen ted +song birds +sam bit +sam adhi +rs x +rob ic +pu ka +pro tons +patron ising +omele ttes +ne hra +multil ingu +lovel an +ko le +kn au +kam at +j ica +induc tions +hiphop music +heide cker +equal ities +coat bridge +bre nebrown +bi gro +apolog ising +âĺ » +о ÑĢ +zat anna +your voice +w co +ub ens +suzuk icup +shif frin +roch ford +rob gronkowski +queen sugar +q aida +pre scot +po plin +ph ool +penetr ated +olemiss fb +ny ff +mu ggs +monro eville +min oan +magical realism +lovin dublin +lo ths +len exa +ky loren +kof app +iam amyjackson +hy ou +hednes ford +green castle +gi rish +gi gaf +fa as +du miny +dev astate +dee ley +cav allo +casey neistat +bey hive +bas sman +babat unde +bab er +ann ina +am oo +zu lia +un warranted +spen ny +re homing +ny university +neon icotin +mini mized +ly ing +lit toral +lear nin +ke van +i sto +hyper trophy +honey cutt +gre ve +gal van +ecra ft +dol phy +dar ron +dal last +calcul ates +by rd +ar jo +alu shta +abi y +ठ§ +yand ere +woo hooo +win oo +uu ut +tri plex +toad stool +the struggleisreal +sou le +se ger +sam buca +re aver +ra gi +pag ar +ozar k +orchestr ating +o pere +new forest +mo have +ma dan +lu bin +lo ha +lac ie +kr at +ka elin +isth mus +house guests +go derich +fu shi +ema w +defec tors +d ö +colour way +blues man +bac i +amers foort +aly st +ach tung +ðŁIJ « +ãĤ¯ ãĥŃ +ty ger +town post +sunday vibes +sunday business +su goi +quick enden +poinci ana +play fulness +pin ar +par p +nom o +neuro biology +mul t +mu re +metro trains +maug ham +marque ss +k maq +jin xed +james martin +il ink +edge worth +delicious food +de eps +bal lets +bail ar +tall ying +suvarnab humi +star sportsindia +shan klin +se caucus +sc alab +san che +robo calls +re organizing +pwe de +pim s +ol ate +nas pa +nam aste +n te +log is +kr antz +heck ling +hate breed +haj duk +fcv afc +em iller +ear nyour +e hc +diamondre sorts +cri mp +ci ac +car no +brun swick +bir ches +aman ecer +ad s +âĻ¥ "@ +à´ Ł +vi on +these us +the hashtag +slo van +sk d +sab yasachi +real m +rai ola +pam yu +p chs +out boards +nieu w +moor house +mid stream +ly onne +leopardstown rc +leather jacket +kha yel +j st +im pres +illini football +hyper ledger +hair streak +f agi +es x +dor je +do bro +copic markers +che son +blanc s +bit trex +ben oit +barran quilla +b dubs +av ilion +é ļ +Äģ h +wido wer +un called +tab bouleh +t tered +ste ps +sk inning +se bo +sar um +ru ka +ross er +ri ves +real joey +po pov +ped alling +mc call +man ni +ma ile +inge sting +heather ton +han ami +ger mania +fla bber +este pona +der ren +de construct +buy backs +book end +book aday +black y +bengal ur +bar bz +ay anna +an tra +ak hen +ah ra +ad disab +academic twitter +... ðŁĺ³ +å Ħ +ÙĬÙĪ Ùħ +zeal anders +wv prep +w bb +se jal +rossi gnol +pvt ltd +print makers +pict spam +peter loo +pc v +park zoo +o go +mitro vic +mis i +love d +leaders debate +ki f +ker ato +ju e +hawk pride +du it +con currently +chocol atec +calmac ferries +bu escher +bon spiel +biggle swade +belo ve +al ama +! ðŁĺĭ +yo gali +viol ator +valpolic ella +th ave +tet bury +t fo +sway am +sati rist +richar dg +raj yas +quadro phenia +pho resis +pert ama +mon roe +macro economics +lymp he +le der +jam bi +healthe quity +hatec rime +gre as +gil do +fre m +france sa +far kas +drug discovery +deepp urple +deco rex +bride groom +bodleian libs +ben es +bapti ze +anomal ous +alle mand +a design +ä¸ Ĭ +vm fa +tre von +topp ling +tay ga +steff an +ssk roughriders +sal maan +rc w +rc m +police brutality +picker ington +or ad +maxim mag +m go +lat os +lam pe +khalee j +ka an +in safi +ick le +ge in +fian akis +ff ootball +exhu med +emily deschanel +emb ellish +e br +cro fts +bis sau +beaver townbeer +be ggs +altar piece +al sop +ak kad +ab be +aak ash +@ : +ðŁĮ Ń +ðŁĩºðŁĩ¸ . +zer i +yeh rish +uefa euro +star nes +softhe week +sig i +siem reap +rou ille +rocke teer +ric ko +perse id +pac io +ol tl +monopo lies +mo ak +mill on +micro controller +lu anda +look oftheday +l nb +k adam +jan ko +idol m +ich or +hul ton +hon eye +flori dag +flor issant +ex terminator +du puis +din fo +de sco +cran bourne +con cho +ch m +cal kins +ber tel +aw u +al una +aim er +ðŁĺĬ ðŁĴĸ +ðŁĮ¹ðŁĮ¹ ðŁĮ¹ðŁĮ¹ +ðŁ¦ Ĩ +ãģ¦ãĤĭãĤ ĵãģ +vi ken +twitter carclub +twit pic +trainee ship +tis one +tal en +sh oma +sar s +remo percussion +one ok +on ville +ne whi +muntin lupa +khe de +jack a +ja se +is nt +igu al +hrvat ska +gut tering +fre et +foun dry +fied ler +fang irl +du pdates +dish patani +co za +chu seok +braunschwe ig +bo ole +ban os +are zzo +ap so +ali p +æĿ±äº ¬ +un satisfied +tra wl +tom oko +tam blyn +sto we +puff ball +n ays +marsh alling +marqu ardt +leti zia +la chie +l vt +kid naps +ke em +fur la +f uring +eli ghts +dan aper +bear s +bay ani +ball state +azadi march +aldubeb forlove +ðŁ¤ĺ ðŁ¤ĺðŁ¤ĺ +wau wat +ulaan baatar +to eing +thirl wall +then ick +the week +the queen +spe k +sham anic +res life +nuf field +mag lia +ku jo +kof fie +kat amari +jan o +ja j +is ches +hu fc +hai b +gu ice +ge man +fe tte +edch atie +dulci mer +condi viso +con dor +buck fast +blo o +bi sexuality +alar con +ðŁĺį ðŁĻı +ðŁįĵ ðŁįĵ +ìĺ¹ ìĦ±ìļ° +ë¹Ħ íĪ¬ +ver mont +un an +to kai +te uk +sports medicine +schul ze +sa hir +roy alo +que ta +pit er +pir a +pd g +ound table +nor rie +mal oof +m tw +li zer +ki yom +ji p +its just +has well +gy le +gu ar +ent rees +dd f +carto grapher +bor ger +bin ns +apple baum +ali ste +aer in +ab ile +ðŁĺ±ðŁĺ± ðŁĺ±ðŁĺ± +ym all +wol k +von ne +vivi ana +thero se +team spirit +sto at +skeleton clique +pn co +pi gging +on trent +o za +mar chi +manc unian +jum mah +i gel +hier onymus +fer rie +el ston +e per +do ig +day dreams +comi endo +allu sirish +ade p +¨ ¨ +yan del +vi shak +togetherwe can +tel ing +tann ery +seaf air +scho enberg +re appear +r hen +out bursts +or han +motor city +mans field +lilly pulitzer +lg u +le derer +fun es +fle mming +dis assembled +da stur +car ranza +cam isa +bush land +bolly woo +bick er +anae mia +an jum +al war +ðŁĺĢ @ +ìĬĪíį¼ 주ëĭĪ +ãĥ¼ãĤ º +à ¹ +wi ven +wfa achannel +w pr +vol ition +vi en +tw ani +tf n +supran ational +stre ak +star oftheday +sr c +sheikh hasina +roger s +red angel +queanbey an +qu as +penn ants +peace fulness +over passes +mg d +ku wait +ko hin +hu ber +head strong +gr b +gift shop +floo dgates +dai hatsu +cryo gen +compli ed +ame h +ðŁĴļ ðŁĴľ +welsh pool +vegas baby +v tr +tri sk +tall grass +sl soccer +sho veled +se date +school yard +sac p +sa chem +re ville +rath lin +public is +p sk +mis understand +mechan ized +later ra +khatta b +kemp ton +ke rem +karan ka +jur is +jk live +hin cap +ha ze +guitar player +gran ita +gab bert +g sx +esper ando +ero b +dom ina +di q +danai gurira +capital ise +book plate +bi ka +aus veng +arun vijay +anec do +*__ _* +ðŁĺľ ) +ðŁĴį ðŁĴį +ðŁĮŁ @ +ï£ ¿ +å® ® +âĿ¤ï¸ı ðŁĴĽðŁĴļ +âĿ¤ ðŁĴķ +wren ches +w out +ulla pool +tiger day +stan more +shop keepers +sacramento proud +nam or +maras chino +mammal watching +ma wa +ma af +lar isa +kab i +jennifer beals +irish research +idoli ze +htt yd +high mark +ga ve +frequ ented +ec j +dogg one +dic ke +de compress +dab ba +dab a +comedy fest +co production +ch igi +cent relink +br û +artsand crafts +Î ½ +valedic tory +ten do +severy one +prodi gious +pri stina +pathophy siology +pa ho +neh wal +ma estro +london life +lad die +l mr +ky aw +inter milan +hel ier +good job +fu mo +fenty beauty +ed s +don ghyun +ce f +bli ghted +as sal +wc g +waron women +von d +twee ks +toyn bee +thi ep +steacher sa +saw yers +sau dio +roche fort +quand ary +pu be +penetr ates +ouro boros +o stia +ma hersh +jim iny +in star +head lock +he v +goal posts +geor gie +fle d +faun tleroy +et oo +danaper ino +castle field +buil dyour +bar gained +ay eeee +asiap rince +arunvijay no +api ary +am peg +ðŁļ ® +x un +wh ig +ve vo +square ly +squ et +spar ingly +pas cag +olympi akos +ob and +middlew ich +kottay am +inar ow +illu st +hi rise +head hunter +hall marks +gla x +girlswho lift +fla ppers +el vish +cryptocurrency news +ci bul +car wyn +autom ation +ato dd +and still +aga sta +à¤Ĥ _ +what areyou +voyag eur +v vd +v awa +ufos facts +stra ddling +snap chatting +si pper +sch ut +ra wa +power lines +plit vice +phar r +one family +michelle malkin +me ze +me dec +mc daid +man tova +m ws +les miserables +lav anya +k assie +inter lock +ic ef +hi fi +fuj ita +fo on +e gen +dram atics +craf ton +black pink +berg dorf +beer men +audi ence +app ic +ant al +ãģĵãĤĮèģ´ãģĦ ãģ¦ãĤĭãĤĵãģ +ãģĵãĤĮèģ´ãģĦãģ¦ãĤĭãĤĵãģ łãģĭãĤī +women swrestling +wag g +wa ite +vienti ane +tu bri +tru ong +throw away +swift lang +sab ino +re acquainted +or ff +op us +ok azaki +niam h +mu ga +mo yet +mlb theshow +mag no +mad max +hay fever +gali fianakis +for president +foot ings +faiz al +esp ada +er ase +encro aching +eleon ora +dun o +dre ws +date just +com rie +cause way +caf tan +bbc wales +band t +ba day +ant el +ani on +am co +... ðŁĺĤðŁĺĤðŁĺĤ +ðŁĺĤ ðŁĺī +ðŁijĮ âĿ¤ +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +ye t +well spring +wander ing +visit nepal +under line +spe terson +so cean +service man +ro es +ra yo +prud hoe +pi aggio +p wp +out played +ou h +ny dd +nhl pa +n zi +more ls +mol le +masa ki +maj id +librar yof +legui zamo +larry hogan +ka hi +indu bai +hun tly +ho ki +harbour side +ha shing +grain ne +for ties +ell ar +doctors strike +cle o +cap ito +asser ted +ðŁĴĭ ðŁĴķ +ãĤ·ãĥ £ +yo suke +u houston +sue ños +spo kan +si moes +ren contre +re installed +re authorization +press conference +po logy +pl zzz +pitts ford +phono graph +over reach +nun head +nil joshi +moon beam +may ur +lugan sk +le hi +jag dish +ilo ves +h mn +griff o +gold link +fresh water +e woks +chand ran +casi mir +cannes filmfestival +bush man +bir ther +ball inger +b df +ac ke +ðŁıĬ âĢįâĻĢï¸ı +yan is +who are +ver dugo +thatawkward moment +solic itation +shine on +sg n +segu ir +science daily +sau dade +ry ley +re traction +r vr +om u +o sam +neo pets +marlies live +malt by +ly si +lisam ur +le ats +hay nes +hat chie +ham ming +gold star +flabber ga +finkel stein +e mobility +devil may +dag ens +cen te +ðŁĺİ ðŁĺİðŁĺİðŁĺİ +yyc food +ven use +tv nz +strati fication +ste pan +samar as +rec t +re te +re conditioned +pres sclub +plant based +p has +on ata +oko cha +nfu tweets +natural stone +mil ka +metabol ites +mega star +mahi ma +lums den +len sing +kool aid +insp irit +im morality +hor nady +heck led +frees avchenko +fle uri +fi annaf +fab re +dre her +de acon +dam ar +crewe alex +cosmetic surgery +com en +cham bre +card board +b sh +ap layer +ak awa +ðŁĴĽ ðŁĴĻ +âŀ Ļ +à¸Ń à¸Ń +à¸ķ à¹Ĥ +zac atec +water jet +un itary +tri athlon +ther oom +ta ze +su dah +ston ington +sta sh +sp ano +shi atsu +shak u +sen den +pa izo +neti zen +nar ok +mull en +le manoir +lali que +l rp +in fest +h day +gaiag auden +evapor ative +en ver +du tty +domest ics +c mpd +book trust +banne ker +bab ak +ar j +al anya +.. * +wig gum +warran ties +wan o +tv one +tin type +sy ah +stjohnam bulance +shi ko +on live +moon landing +legg i +latenight seth +la vas +juvent ud +hart ley +gra sped +ge ren +gales burg +fa sts +ero ica +davincis demons +d fg +coffe eco +can do +buy ing +bar ang +ac app +[ : +ðŁij º +vale ant +upri ght +thel p +subi aco +som ni +shir k +sdc india +sal vos +sag u +sad dlers +recur sive +pu w +petro s +per shore +ou dna +oro chi +opioide pidemic +nbat witter +nathan iel +mun chin +mounta inde +miscre ants +mis communication +mbar ara +lo rela +liberalismisamental disorder +le andra +kentucky mbb +je tt +jav el +iri descence +indi av +im potent +hend ri +geth se +desper ado +cra dio +cr x +col nago +cla vier +cadbury uk +c sb +ame in +âĿ¤ï¸ı , +yoga everydamnday +win with +west wood +w eri +w ce +un sound +u wo +timor leste +tat ar +street sof +strato spheric +ss man +spre sent +spit ze +soli man +si pped +sau li +pterodac tyl +nc u +nash villec +mariecuri euk +mani acal +m ni +love bug +length ening +kw aku +kud low +kpop starz +ken yon +ji ve +he gan +greyhoun dracing +go broncos +formul ating +foot balla +foodand drink +flip the +e stevez +di she +de meaning +capy bara +blu emountains +ble ek +billiee ilish +bi partisanship +az ania +army bowl +a hoe +âı ± +wild west +whit eface +ver ve +sung jong +sunday roast +sixx am +shar lene +shame fully +se dated +ra pinoe +r dra +quo ted +post paid +ous ins +obscen ity +moust aches +midat lantic +mal achy +lee man +la gan +kot ler +jalap enos +hyun bin +hing is +gi gg +gam s +gaiagauden zi +fair weather +excep ted +duck hunting +do vi +den e +de my +cork cityfc +chem bur +che ons +bokuno heroacademia +biom ime +back es +asu g +armou ries +ðŁİ¥ ðŁİ¬ +ðŁİ ŀï¸ı +ëıĻ ë°© +ãħĭãħĭãħĭãħĭ ãħĭãħĭãħĭãħĭ +व र +ym ed +wol laston +weare family +unsigne dartist +travel tip +task master +succu mbs +stimu lants +st luke +si vas +shu u +s ve +on sea +o orah +lo lli +lex mark +ke ham +kar man +jr f +jol ts +iu lia +hai le +gar land +flow y +fant agraphics +fan sof +exolu xion +ex el +espnc fb +dr iller +dogmeat trade +consen sual +codw wii +clam shell +bou illon +bosh off +be for +ar jona +ampli fies +agric ola +ab ora +ðŁIJ ¿ +with purpose +tom oe +to bar +tend rils +slam mer +richmond hill +pur ser +po el +nuer burgring +messer schmitt +mant le +m vd +kirk stall +key shawn +ke tel +inthe uk +ho ppen +god parents +gn ano +g itta +g dg +fle amarket +fi field +down state +down sides +de contamination +dae woo +ch romeo +busine sse +british tennis +bat anes +avoce t +alarm ingly +al ann +ðŁĴĭ âĿ¤ +wight man +who ami +un appreciated +tou bia +tf si +terr an +ta ven +stol tz +shemar moore +sharing iscaring +ring ling +re gi +pune et +phi fe +par ables +pandi raj +mun de +mo ke +metat ron +inver sions +ic ap +ha plo +fu uu +f ounds +el gato +desi re +d hal +coraz ones +col y +bush ra +bron ycon +black sheep +beam iller +badas steachersa +aw ang +arch digest +ad at +Ì¶Ì²Ì ¥ +whati m +was i +under performing +to tt +th ile +st anger +rod stewart +pulp fiction +polar bear +pear ld +pan ics +op s +nord see +noo bde +ni mes +ne sa +nas sau +min ette +mar maris +levit town +leng then +kaz iranga +k hid +juda ica +ic hat +go canadago +gen sler +funny bones +dyisi sit +dress making +dj ur +devi ate +cu id +crustace an +crank shaft +co bie +bar one +b hl +aven kat +ass ate +ac ab +à¹ģภļ +Ã¥ s +y ath +waynes ville +valley wx +val ens +touri sty +suk hum +splendid ly +si oning +shiv anna +ser ine +sar ahe +samar inda +sab ar +ry ano +ring master +ridg eland +rat cliff +po gues +oi shi +ne gga +nam en +mur ree +mo omin +mil ia +lin um +kit tie +ki x +i dent +g ä +ffun ded +est ars +elvi shistory +eco sport +dress maker +dc tv +costu med +con ing +chi vas +bin aries +baj payee +! ðŁĺľ +yel love +wil s +whitt ingham +sky rim +real kevin +read allaboutit +r cl +pun o +par li +na ghan +mun e +matchroom boxing +la sd +kal os +k ring +ind ye +hero esof +ham ber +gg t +fac ey +diab lo +ct vedmonton +bre de +bla vat +be ssy +attenti vely +as ot +aristo crats +ane ws +ðŁĺįðŁĺĺ âĿ¤ +á´ Ľ +yo ps +wom ad +virgil abloh +vi vel +vat raffic +va art +toly mpus +the ip +tas so +sn ak +skill india +sie ben +rod ham +pr ata +po ors +pic kn +need s +mx px +ll er +le ers +latic sofficial +la pointe +kago shima +k mb +ju anes +it our +he da +ha kka +gu gu +growing up +gold standard +fen ce +den r +cur tesy +cor ban +beh rens +am isom +air drops +- . +z ville +youknow youre +vi dor +tsaww assen +thiswas cle +suni elv +sunielv shetty +summer stage +spark ler +sise puede +sag esse +p mk +nj c +mephi sto +lam our +kg bt +kaw as +jets ons +is berg +hor muz +gif ted +fit spiration +evapor ate +el ain +dou se +chic hen +captiv ates +beleagu ered +as ms +acup unc +a ec +@ ____ +ut tam +un wise +tri xie +tag g +style awards +sati e +sap na +san gram +sam pras +ray donovan +ra zi +pt fe +pir ata +new day +n rd +mu schamp +ma user +lamb skin +ker by +iphone only +ilustr acion +ham o +glo ster +gi ddings +ful ford +films video +fe en +dri p +cred iting +clipstudiop aint +charles worth +block b +bir ge +bac o +az ia +ay ian +arrow filmsvideo +am w +ac adie +with my +us bc +to z +thre l +rock hold +rob ing +reha b +pu yat +pillow case +perry sburg +nur tures +normali zing +nee ley +ne eta +mon ona +mal m +m by +llan de +li pinski +lai ki +kris jenner +kon ga +kar ren +italian wine +gu us +ex ci +ema zing +d ya +buo yed +bick ering +bette midler +bed lington +ban ister +ban a +ation tv +agny aath +- & +ðŁĻĪ âĿ¤ï¸ı +ãĥĿ ãĥ¼ãĥĪ +ãĥ Ķ +tip ton +tal ert +switched atbirth +su tt +sp ren +ra sp +q an +pin eville +piccad illy +pf hof +over bought +nor ma +nic helle +navig ates +morbi dly +maha devan +ll r +hop wood +en field +em l +dr yeye +dee wana +car pio +biom ole +bhar ara +art sc +arre dondo +ac costed +@ ) +z ini +wee eee +union ville +ted die +st asis +spirit u +radiof ree +petro glyphs +oliver i +naj jar +mi jn +mammam ia +maid ana +jama icans +ig ang +ife anyi +ic hert +fore play +fergu sson +etsy aaa +de test +de ke +cor tisone +bon bons +bo tero +bajpayee manoj +b te +ade es +ðŁĴ¥ ðŁĶ¥ +ðŁıħ ðŁıħ +wilder ness +tabletop gaming +t bbt +submer ge +so dal +si ah +sexi est +sen n +rober tw +rit on +pro jo +pra ther +ovi ya +oj eda +oc ke +nar u +more no +mor nay +marshall town +kil meade +ja hr +independ encia +indef ati +in sel +imagin ary +halloween costume +georgi ou +edu ardo +east view +defen sor +de joria +clai rec +by erly +at c +anore xic +annex ed +ai aa +ï¸ıâĥ£ ! +ï s +ww at +wupper tal +wheel set +tru ecol +tor tas +tion news +thre sh +te vent +swamin arayan +rep elling +real ron +re prints +re directed +quare sma +pen ne +pat en +mur k +metho dman +malaysi a +lov sk +lo ir +kill ington +ke u +kat en +jo ven +janu ari +in law +hol ts +gun dam +gled hill +garof alo +free thinker +father land +fashion history +fall acies +ee sh +dom sherwood +desi ring +cm ha +brew fest +break ups +big game +batman day +an sah +alpha bet +, < +ðŁĺĤðŁĺĤðŁĺĤ @ +âľĮï¸ı @ +whit est +whang anui +us is +tsuki ji +thom ash +the toughest +summer in +sing tel +simon coveney +sidd hi +si yah +sen ile +que remos +presu mption +pair c +na at +mc cann +maro oned +mah alia +lon ga +ja sta +j ata +ill hu +hack aday +gi zz +ga em +fin ny +fault less +far rah +ali abbas +ðŁį ľ +ðŁĮ¹ âĿ¤ï¸ı +Ú© ÛĮ +س ÙĦ +zeph aniah +w dm +villa real +sydney roosters +pap illion +ne had +myster i +mult itudes +mil s +mate us +loughe ed +le var +kenny wood +house cat +ham mons +gw f +gr ic +gl ancing +frighten ingly +free bird +fetty wap +father less +fai ro +espn fantasy +dou gie +co sho +chan cel +cardo so +brooks brothers +anadar ko +âĦ ĥ +Ð ± +zu id +verand ah +upan ish +up north +tr ona +sm sports +skag way +sigh ed +shaf fir +sab ie +ry ang +re clusive +pyth agoras +pete gui +nonchal ant +mud slides +mor ant +mclu han +man school +ku gel +kirk by +ka ali +jaw ed +is f +helen sburgh +h series +fu dd +fan army +ex claimed +enter gy +dyisisit manila +di at +cru se +car m +break neck +bilingu alism +always be +aki ba +ad abra +ðŁĺ£ ðŁĺ£ +ðŁıĢ ðŁĶ¥ +ðĿĹ ¶ +âĸª ï¸İ +ye ducation +watche spn +trampol ining +tay side +so wa +sh allow +sema phore +q ew +proud teacher +perry man +onom ato +nether realm +mun y +metamor phic +man tua +legg era +le web +le ssing +le per +ke well +jw st +je el +go beach +fro gger +forever orange +edu topia +chippe was +c mh +brexit party +biz et +beat king +aw n +asap h +anal yser +ade sina +?! ?!?!? +ðŁ¦ Ģ +ëıĻë°© ìĭł +Ùĥ ÙĦ +york dale +wey burn +wex ford +ul loa +u sip +tre ed +sx onfox +sham it +sephi roth +sch in +proper t +mo ats +jain ism +illhu emin +hirsch feld +emaci ated +eddie izzard +demysti fy +deck ard +bush ey +buff ers +append ic +ant artica +adi k +yu kari +west shore +wat sky +w fs +vac ature +super moto +ste pp +roller ball +roc nation +ran elagh +r md +professor green +produc tively +person als +pa jar +nov onor +nn n +ni emi +new salert +mal ani +ma sco +lo petegui +jersey ci +inj awarrior +i aapa +gg p +dra c +comm is +coffee with +chi hiro +cc ss +bull finch +blay ney +aul kner +ar ber +ab dl +zip car +yu shin +win kel +vo wing +une n +the ory +t pc +t adi +sho witz +sherry rehman +ser ta +reci eving +r de +q rt +pollin ated +pear lv +pe th +off ood +north western +ni der +man dem +kennebunk port +keep britain +jal alabad +howi ed +folk tales +es ra +dil worth +chess ington +calab ash +br k +bal as +ato z +amaz one +adam saleh +' !!! +ðŁĩ¬ðŁĩ · +ëĵ ¤ +yl ing +wyn ter +will cox +vi vere +valentine day +transpon der +titch marsh +theavett bros +su ce +ski doo +sk oo +ros ine +rese da +perfu med +out lived +our ces +one world +nu dist +mcr museum +lake side +kaneo he +justice orelse +instant aneous +in co +hungar oring +gri moire +great reads +ghastly gastronomy +ger n +fun fun +fu c +foodblogger ai +dur ning +datav isu +cri mbo +clinical trial +cho tels +car berry +bou dre +bil ston +biblio the +bab ie +ay aka +as core +adele ke +ðŁĶĿ ðŁĶĿ +ðŁı¾ âĢįâĻĢï¸ı +we ge +washten aw +wal kon +w ads +vic enews +v hs +un shak +thisismy crew +ter adata +tan door +sw kly +stol len +sno ke +sni ffs +shali mar +seri us +sare back +sanc tified +res ch +pseudo science +philly now +matthe ws +manag ua +laun dered +hou lihan +hou l +hou ghton +hot eliers +hoo ley +go back +ero gers +elan eri +e chan +dur ance +dj sbu +dish washers +dial er +clever bot +ben ner +bas sen +ache be +? ðŁĺī +ãģŃ ãģ£ +ãģĵãĤĮèģ´ãģĦãģ¦ãĤĭãĤĵãģłãģĭãĤī ãģŃãģ£ +د ÙĬ +w bez +upperdeck sports +uk storm +trun king +three uk +tho ward +solo astarwarsstory +se gam +schul ler +sayye shaa +reposit ories +ram bling +r aro +prayfor us +poli sher +p mk +own the +mel drum +kimon os +intern als +ine pt +human ely +gar ters +g nev +fug ly +for mo +dispo sing +charlat an +cerebral palsy +bou ton +bhutan ese +assau lt +aran ch +am hara +ake a +ac cor +;- )) +ðŁĴİ ðŁĴİ +ze alot +yr insider +xrp thestandard +wend ell +tram ple +tour o +taver nier +ta fel +solic itations +sh yne +sf b +scrib blen +s brew +ren ch +ra dek +pla its +online learning +one less +one ers +ni xa +neo sho +mor d +mo bbin +md h +mb h +man tic +ma dara +kof ta +ig naz +hooten anny +gl eni +ge tin +fo ck +evin rude +en code +emanu elaneri +chon buri +blind side +bill yidol +ank ur +aaaaa and +ðŁĴIJðŁĴIJ ðŁĴIJ +ðŁİ ° +ม าภ+à° ¦ +x xiv +wri ggle +wp moy +vi ani +ve to +theme forest +super fluous +selfe steem +se lem +ru ffed +ra kul +ptole my +pro fil +old london +newh all +me isner +madam secretary +lu me +led ger +ke ir +histo gram +hear say +hay stacks +governor perry +gor ry +ghe tt +gate wood +fandom memories +easter bunny +double day +degre es +decemb erists +chal font +bus way +base ments +b ick +ah k +ìĦ¸ íĽĪ +åĩ º +zakhar ova +wh ist +wen urses +vicky kaushal +twor g +transi stors +thak kar +t vet +t no +south wick +rach id +r la +propag andi +poli sci +pl tworg +oh are +nov gorod +neuschwan stein +nano science +man nn +hyun seung +gandol fini +g adventures +descend ents +de ta +borde aux +bo se +beck on +ali dad +alar ic +å°ij 女 +ym atic +yav apai +whit bread +water keeper +tuuk ka +ti da +the greatest +tamar ack +sto well +sky sport +ring gold +ra bacher +r pe +ponti us +pc dd +oni st +mince meat +mattb ellamy +mar gera +mar cio +maite oficial +maach oops +lifestyle blogger +lake erie +kent ridge +homestead miami +gra phia +golf news +fur row +en tech +e stos +e gu +ch imp +cat rion +bl ings +big fish +b ava +armen ian +amazon ite +al tra +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ +à¹Ģภķ +à¸Ĥ à¸Ńà¸ĩ +z aka +warring ah +u mo +tough en +sta si +sque aking +south coast +sil at +sar dis +san te +richar diii +r mcf +punk rock +nas sr +n once +mould ings +megali thic +l ann +kamenri der +getin my +fl aco +fab e +ema e +dwi gh +deduc t +cor vo +ch ato +arche ologists +ar ys +appropri ateness +anu j +alo is +adrian peterson +ãĥĥãĥ Ī +vandy ke +toho shinki +tai res +sundar am +sal ton +ri hann +pro cor +norm ative +ly kke +lisamur kowski +ligu ori +leg oland +kwan kwas +kar lovic +kal man +insu lator +horse manship +harri ott +et d +ery thr +er ae +dg son +deple ting +den son +chen le +cand ic +ca thra +bal u +art institu +ame deo +' ?" +ðŁĺį ðŁĺĺðŁĺĺ +ðŁĶ Ľ +ðŁIJ Ħ +yog endra +ye ap +tu lia +trap music +thu ram +thom a +sling back +recuer dos +ram pur +punch bowl +prag matism +phalaen opsis +per pi +ok k +na iry +mah langu +m ht +k lip +humber college +hu gues +hal pin +hal fling +gri my +governor va +gal go +fol ger +fla sks +firstal ert +fab ius +ei sen +chi ppa +cam elli +bu sia +bag g +ba star +amb ard +aliabbas zafar +zim ba +you like +wych wood +u ac +the summit +sv d +shi vak +re publics +re assess +pro strate +pray forthe +pal it +oo td +onmyo ji +nam ma +mond ello +mo xy +m ld +lov ich +lom achenko +lia oning +le te +kas ingh +ju py +ingol stadt +hipp ie +grim lock +go wolves +gallau det +fernand ina +fac ile +ed as +cre sta +control lable +block head +bel size +bel ind +b ck +appendic itis +al locates +yu uki +vandy boys +vag anza +thom a +sy f +sy ahu +ste pup +sp ens +sch moo +scep ter +rho s +pyn chon +psycho therapist +philli e +o ii +nehad hu +n sdcindia +lu h +lo li +ke c +inve gas +hen ness +gun k +gin ous +fit fluential +en p +ek ko +dr john +dd p +cas ade +calder wood +bur kett +buck aroo +bri gan +bon ez +accade mia +ðŁĴģ ðŁı¼âĢįâĻĢï¸ı +wha aaaa +wh of +volup tuous +up ra +tu lo +trol lope +tri ano +temp el +syn tagma +sw ays +stone wall +star kid +serie atim +say an +s xt +ridge view +reflexi on +pul sing +poon ch +pearlv puri +on du +om m +official randl +nay arit +mur taza +min it +l ns +kn elt +home pod +foster thepeople +e bf +der bys +cyber war +chal king +bli gh +bambooz led +ayo ko +akzon obel +اÙĦس عÙĪد +wh ims +volk sk +titic aca +tim perley +spre ss +smu g +rhi annon +pizz azz +pile ated +percep tual +mu shi +mix uk +lori en +kin sley +indie pop +homer uns +hippo crates +gre b +gold coast +fresno state +deli ri +coffee maker +clover leaf +br ouk +bo ther +am ati +al annah +achieve men +accom plices +ãĥĿãĥ¼ãĥĪ ãĥ¬ +ze tec +zare k +xenoblade chronicles +wise guy +wi ddicombe +wan khede +vitri ol +vin daloo +ty tlive +te dd +share acoke +semb ly +sd h +say it +san kar +res ver +place sto +phan euf +pel ton +no thanks +nichol l +nathan ael +man gano +leu ko +infuri ates +hypnoti ze +frustr ates +extric ation +end points +e sau +du dek +deer foot +comprehen sively +candi de +camp den +bm h +bian che +bab ies +assembly member +any e +é ¹ +zel en +wednesday want +texom awx +sundeep kishan +suc c +six words +sergior amos +rugby canada +ruck ers +pol ys +plugge din +om ura +nic eville +min koff +meow th +kul lu +jet ski +im ca +her oku +hash emi +grammar ly +fier cen +fa him +epp group +dogsare family +d swt +cr ittenden +black son +autopha gy +aler tness +val voline +tom lin +the agu +silver smith +shou ston +shor ties +ro mina +qu ina +pub quiz +provoc ations +pad dys +ne scac +n fc +michael rapaport +mic alle +life changing +geor ger +eric ally +de wor +clari dge +chri ss +car yn +bo sman +ball ack +arbut us +altam ira +âľ Ĵï¸ı +way mo +vom ited +u ther +torna dic +storm tracker +sprote ction +rob thomas +poin tof +pl ana +pa ha +montes julia +mo sher +marke to +mark son +lar ts +ker ning +julie bishop +ig ad +he ver +fahad mustafa +em cc +do ber +d rich +cul pable +crac ow +cal ri +bur ros +blen cathra +bi on +bass inet +at vs +amand ab +ac ted +ðŁij¶ ðŁı½ +ðŁIJĿ ðŁIJĿ +ze peda +womani zer +wir tz +vaish ali +u wc +u ting +sop hos +sink able +silve ster +s mail +s lit +politici zed +po ca +pediatric ians +nick cave +nemato des +melani atrump +ly tham +lam ination +ko sher +j co +in vision +in box +heuss aff +han doff +gov tech +goldkey comics +garage band +g mi +false hoods +epic ure +en dy +dy ard +che alth +bright ling +bourn ville +blow ed +birken au +baster ds +aw bb +atla sobscura +amy freeze +ðŁĩµðŁĩ · +âĥ£ , +Ì ² +z om +tromb onist +stro me +shahe ed +secon dary +savechildren uk +ro bie +ric er +reu ter +pol perro +p ns +ny phospital +norther ners +mh eller +man up +lm k +kather yn +heart worm +gau ld +futu rology +fa wns +du du +dis aron +crash bandicoot +comb ats +co ti +christi aan +ce elo +carp inter +bm ws +blood lust +yak in +tu an +tri pling +thank smom +spot sylvania +sand vik +purch aser +no where +mohabb at +mc goldrick +kap taan +it sam +in built +illhuemin ati +i ita +hir an +haiti an +hail sham +ha young +fernand inho +feliz domingo +eli ot +drawthisinyour style +dorset mag +di wan +buy er +b wk +ang olan +ai hl +ag gy +ðŁĺĥ # +ë¹ĦíĪ¬ ë¹Ħ +wbko wx +under writer +twitter ing +sun shin +sub par +start with +ri dlr +recipro cate +ra van +paradig m +ou c +nd g +mohan ty +mo zz +mass ape +lumin aire +lgb tiq +lea therette +la th +inve sto +im in +ik awa +hard wicke +from tomorrow +equ il +eco soc +e manuel +desecr ation +confe c +cam b +bel atedly +beat nik +av atas +an sell +acoustic guitar +win star +tarek fatah +super junior +som any +scic hat +salt spring +richard son +re forma +r our +q os +prep zone +ow i +moolool aba +mati syahu +la zing +kle iner +gr und +g aki +form ality +ferri by +determin ism +cut scenes +booking com +boo oooo +blu er +barclay card +al ition +af aye +adity aroy +ðŁį¾ ðŁİī +âļ½âļ½ âļ½âļ½ +âģ¦ âģ¦âģ¦@ +а ÑĤ +yo b +v rx +un tv +time form +ti ggy +the morning +sun star +stow market +ru kia +regrett able +popul arized +per dition +pat y +on ism +nobun aga +nad ella +mis ch +jack jackjohnson +iv orian +hu mer +herb streit +he iro +had ji +four square +faste st +fanta size +extor t +dor nier +design inspiration +deep mind +dav in +co location +cin ec +catter mole +anesthe siology +ana bel +am paign +all ura +yal ta +wur litzer +wp bf +thiep val +the horror +th v +sto b +qu aye +port able +peri sic +pe ut +ot way +ob je +nehadhu pia +mol dovan +log gia +lar kin +kyli ecosmetics +kari uki +jap ati +j ory +im the +gwyneth paltrow +grave site +futu ren +fen church +ero ad +endit movement +ei mear +e inf +david price +bo hn +beau maris +au drina +amazing ness +wy nyard +w ws +ven e +velaik karan +urban planning +tyn dale +theat rics +sup my +st kitts +se si +se kol +scor chers +profit eering +pi ro +peri operative +over loading +ne edi +lan sdale +itsli verpool +hood lums +hand el +ha gs +goose island +exciting times +edge computing +edgar town +d tf +clar ks +ch ads +cau gh +burn leyfc +bur an +bis muth +bella hadid +be active +bar tok +agath achristie +ack royd +ê¹Ģ ìĪĺíĺĦ +É ´ +wil lam +tam ers +st m +sk ys +shad dix +par iv +ono ghue +minu scule +mai sha +lu pi +kat ak +ka ja +jack daw +illu sive +hh sgov +gre ta +g ool +g fk +famili esto +facts matter +dwar ven +dont mes +diof avatas +car swell +bir kett +amid ala +alu card +ak il +æ ¬ +wedge wood +ti mc +tali bk +swaff ham +spla yoffs +spectacul ar +she i +sen in +satri ani +rhode sian +rashtra pati +qual icum +qu am +potter y +pon chos +pa ja +ne et +mzan si +maul er +mai sel +k mg +jackie chan +impac tin +hon ori +gun dog +fle abag +dru mheller +dr yan +do gue +dioce seof +cur tail +creative writing +chat elaine +car der +bri z +ber zer +b itu +archae o +add ons +a all +zap at +x ue +vancouver sun +van es +the bull +th hour +tel uk +spy gate +sm rt +shaw nab +sc si +sal alah +rowy so +red cros +rainbow rowell +radisson blu +r chs +pratt ville +poiti ers +pan n +oo ke +mu ddle +mor avia +michi ana +ky r +kli ff +hu ana +henry gayle +head butt +hair ball +g tn +fu rey +fram boise +evangeli zation +ec ard +destabili ze +de me +by c +badrin ath +amp ere +Ûģ ÛĴ +ver ney +uffici ale +tb wa +sof twood +soci ale +rede ems +q tv +pu bic +per ches +peoples votemarch +oni x +ol ith +oh su +novel las +mechat ronics +lunch ables +h ings +glass door +fc pa +fanexpo canada +edex cel +dict ating +courtne ym +child splay +caloun dra +birth mark +bar na +alway swith +/ + +ze is +y se +wil fork +wee ting +variet als +uof r +twee talong +sw ash +shar ad +se millon +ru bella +rap sody +phd forum +ny ce +ntv atone +no ice +ni biru +mu ma +mic hen +meaning fully +kawarthal akes +juliebishop mp +hu at +ha ff +gri eco +gibr altar +fire fly +f nb +dakah lo +buck cherry +bcfc tweets +bak ari +au dre +ash ly +andre essen +ai sha +ad duc +âļ½ï¸ı : +x uv +tu tsi +ti go +talibk weli +squir m +sound board +rehear ses +planet side +passage way +our time +os stf +ojib we +net ter +ne ac +n kn +mu ki +mo ins +matri arch +ma stani +laven ham +kiku chi +jan am +hinojo sa +fo etus +eye shadows +enig eria +ec rowd +dimit ar +did ger +di efen +defrau ding +bull sh +broom stick +back us +ar gh +xo los +squ amous +shu ai +shi ans +samo ajoe +sam ford +s llc +priyan k +pio tro +pic slip +or don +nol de +murciel ago +lu xion +lin ds +inst it +ing laterra +idoli zed +health insurance +harmon ia +h sl +financial freedom +ffici als +eu dora +con scription +clun ky +char lee +cap i +bannedbook sweek +baff le +back lot +b mcc +ascend ant +alu ddin +ðŁijĭ ðŁı» +ðŁIJ ŀ +zi v +x tr +wa ec +ve schwab +time e +thim phu +sh ree +r sca +peto sagan +palla dino +old victheatre +ner ding +micro beads +leven shul +larch mont +ja ved +ir ation +inf p +farc ical +f ars +dono hoe +dipo log +cho m +carnar von +can cha +bli ghty +ate m +wpl glocal +vive gam +ver sum +ur ne +twitch sharing +thap ar +shri ek +shaun king +ri is +re grow +raz ia +picture house +out stand +oo stende +ntn u +nav arre +mi micha +lamin ates +kilmain ham +kah le +ic a +gan za +figu ral +far miga +fa th +extru ded +central asia +buck town +bre ton +birthday staroftheday +basil icata +arri son +ðŁijĩðŁı¾ ðŁijĩðŁı¾ +wim pey +weis sman +visit maldives +vic eland +veget ative +um my +top ten +sp ss +som meli +sky diver +saw fish +partiti oning +n goron +n anga +mono grams +mercat or +mary mcdonnell +lush cosmetics +lc u +khayel itsha +ka ke +ji ya +it b +ic ff +hol dup +fur longs +eric prydz +car ys +buk ola +biennal ear +bay i +bari atric +aged care +âĻ £ï¸ı +z older +yes allwomen +western ma +vaill antuk +u indy +tric ho +tri as +the b +tar paulin +swing man +sun kissed +stump town +star tsnow +segu ri +romney ryan +real clear +pa chu +nulli fy +newh ol +mer api +lauren ti +kiss fm +kat er +jugg alos +jarre t +guil in +crun chie +climatechange isreal +bur dick +be ales +ba tho +b gb +b dt +adventure land +! ðŁĺ± +ver on +the deverakonda +sto wed +sp alletti +rhy ne +retrac ted +rail gun +pi qua +phonec ase +patri mon +nov elli +n serc +ma dani +lat am +khu t +indv snz +heine ken +guaran ty +golden age +fru mp +enne agram +do wie +digital ocean +darwin ism +cri stine +coun ten +chicken pox +carry out +buen as +bic arbon +bi onics +asci ence +) "@ +ðŁıĨðŁıĨ ðŁıĨðŁıĨðŁıĨ +çŁ ³ +à¸Ķ à¸Ķ +xx xl +woo oooo +wom ened +wa ren +vi ere +thun ter +thor se +simon baker +selfi mprovement +scul pin +sch wer +ro hr +raffa ele +ra yu +per version +pamp as +os seo +og u +nü rn +my i +mar ter +le vo +ko kan +im kona +ili ac +goo ooooo +gal antis +fed de +exce sses +compac t +citizen ry +cign al +che am +ap lace +an jos +amazon books +a av +wash board +u tti +tre ssel +stro h +sig ler +s voice +ride with +reliance ent +promethe an +pel ly +lie ber +kan azawa +k se +john cusack +ichand ra +ic ona +h ws +go parks +fire ball +fanatic ism +fal li +de ssa +de park +de mann +cur sors +cos ito +corn us +chelseaf lowershow +c ts +book maker +bne traffic +bb mme +baw try +avi v +au se +ae b +ab lec +è Ģ +è se +yng itis +ver re +ul mer +toy land +to ck +swill neverknow +sw va +stro mae +stephen hawking +sequ im +prop elling +po index +oun de +or ourke +nor vell +m zee +liber ally +le les +kim o +hit music +getwell soon +flir ted +fi red +et ta +don go +cibul kova +ci gi +ci en +ce peda +brad street +boo dle +bis cay +antiqu ed +ðŁĴĹðŁĴĹ ðŁĴĹðŁĴĹ +âı ¬ +ÑĦоÑĤог ÑĢаÑĦ +work stations +terre bonne +ta im +subli mated +stv j +so bama +sneak ily +si sk +sell s +scimit ar +s market +re ste +queen sbury +per vaiz +nat m +moh ler +meth adone +ironde quoit +ic si +gran ollers +gil der +fu jis +frie del +db fz +dark shadows +daily nation +d live +contra sted +clau dio +ch ole +canal etto +bor na +bigh orns +behind woods +arak an +air brushing +ac afe +èµ · +z wir +z ra +wor ships +truec aller +ss x +smithere ens +sket cher +re its +rapi er +poul sen +plan b +mersey rail +ligam x +l bk +kim hyunjoong +k alia +j ld +irresisti bly +insur ing +idol atry +en ice +elder scroll +dom browski +deltag oo +bryan adams +bou ff +bis sett +ay i +ari b +ali ving +al sati +ah r +un sun +twit ches +totten ham +to var +the crown +synergi stic +supmy tempo +snor ting +share tunes +righ twing +pos adas +pen rith +pantal oons +on z +moon cake +memorial cup +mel isa +le brun +la phro +just like +jac c +ja ina +ice box +gift cards +gender gap +ge il +fish guard +ene a +di mer +delam ere +bolshe viks +bland ing +bill murray +aqu in +winnet ka +watson ville +u dp +thrott ling +te pid +syn cop +st ere +se tit +ov ac +onthe wall +mussel man +midr and +maxi mizes +master ton +madon sela +m elling +lol lywood +lap ham +la throp +l ps +ith ac +ist ana +isol ates +hydro xide +hal ligan +gaz gshore +fu u +fli c +f sm +f ado +er ji +curi os +crusad er +conquist ador +chel e +bro wed +bourgeo isie +bol lard +anxi ety +am ec +al sina +ðŁĺĿ ðŁĺĿ +ðŁIJİ ðŁIJİ +ðŁ§¡ ðŁ§¡ +ìŀ Ń +âĨ Ĺï¸ı +yo gam +yellow card +welsh government +under tone +today fm +tan uki +sx sw +sun stone +slu mping +play warframe +pe ice +nole fam +my les +lukas z +ki v +kemp ire +hel den +gopin ath +foxstar hindi +ent r +deltagoo drem +dai ya +cheer full +care em +boon do +beno ist +baz zi +babys itters +as orock +artinstitu techi +ari elle +اÙĦ Ø£ +youn t +val is +uri st +twee ple +tra uma +t ck +suiko den +sco pus +ry th +round hay +re programming +pro sely +perfect day +pau le +negr on +ne co +ministry wcd +min ka +mimicha kraborty +madein canada +ly d +hi o +her peto +gi re +escu cha +equin ox +dontmes supmytempo +de gen +day dreamer +courte san +cate blanchett +bar ro +b hour +arch enemy +ad aa +. ðŁĩºðŁĩ¸ +! ðŁĺĤðŁĺĤ +ðŁĺĻ ðŁĺĻðŁĺĻ +wharfe dale +wall send +tank top +su hail +speed ball +sl cc +sim z +si gel +sau thor +salvador an +per severing +news agent +na poli +man olo +magu fuli +lac onia +kri stal +kortri jk +jan k +jac a +ft g +free all +fik ile +fan mail +ese tter +e ese +downfor what +doug jones +d appy +cur rier +croatiafullof life +costu ming +carpe ted +c series +bugab oo +bol ing +bas sem +app l +amo tt +a eu +ðŁĺĬ ðŁĺĤ +à¹Ģภķ +ts vangi +tr itt +spelling bee +small holders +ra ph +protru ding +pine al +pee wee +min en +mag is +la kin +karyo tic +kag eyama +is ca +inst alove +hy ste +hot pants +grun gy +ge ot +g naw +free holder +don te +ci al +bur don +bump kin +brun nen +ali pay +ah hhhhhhh +acram ento +) | +york university +vi burnum +un verified +tra ylor +ting a +theafric amentor +the michael +ros ann +pre teen +pre fontaine +pe tras +oz u +out sole +mo ylan +mc adam +markus feehily +mac fad +ko ichi +kin smen +heart breaks +gil son +equestrian hour +du ta +dog walking +dam eron +cosmopolit an +communic ative +chim ney +chick a +chester be +brid ger +brick works +boat life +ad ze +ðŁĽ « +ðŁĴĢðŁĴĢ ðŁĴĢðŁĴĢ +y erin +wn yt +we sty +twit tor +so g +se ob +sa ke +post uring +mer l +mandalay bay +leonardo dicaprio +kno wit +jor din +inf lexi +im x +ic ai +hisp ano +herak lion +ha thor +gyna ecology +gu lak +gib sons +gerrit sen +fund able +disen franchised +devious maids +crow es +cour ted +commu ted +col well +are port +aer ator +ya yo +y la +we at +ukr anian +truff aut +the ming +ta way +sre ed +sl ounge +sky scanner +sk nights +shirec cc +shil pians +san tu +rv ca +raashi khanna +py con +mur mur +mei ko +kur th +keep moat +hann am +flag ell +fen wick +en sion +dra win +down force +di gger +dese gregation +d jim +bucke thead +bran n +birdseye view +bel ittle +be dri +barn hill +ag assi +ðŁĴ ¢ +áµ Ĺ +yu gi +x zi +x rt +va an +ut agawa +un sinkable +twitch play +tre sa +town ley +tod rick +studi om +sc inema +rd chat +quoted daily +poun ders +pit zer +paul pogba +pack mas +pacade mic +ole ary +ok api +nic king +key tar +kay mer +immacul ately +hu leni +har b +h jk +goldenboy boxing +glent oran +f bt +eter no +edinburgh castle +ec lark +dun lin +clean ly +ci ps +chop da +burk hardt +bis bal +bil zerian +ap in +antag onists +yu ya +untapp d +tyd fil +twee ties +tol u +tn k +the cine +sun fire +sude ikis +sub side +spur n +slee pa +shop indie +rrrr rrr +ra ii +prin ted +ofthe game +le ly +ka veri +k zoo +just kidding +igu azu +helsinki uni +forts ask +fin ra +dur den +daguer reo +clu bber +cat ahou +bo vey +as ah +vr chat +thu gg +tele film +squ ints +sk Ã¥ +shock ley +ru scha +ripp ling +ragha vendra +pyro technics +progressive house +pen ter +over de +ni ds +newport beach +muer to +med way +man te +la place +k th +jo lo +gil dan +gh b +dong woon +de frosting +d how +blue gill +. ðŁ¤Ķ +ðŁı ŀ +ãħłãħłãħłãħł ãħłãħłãħłãħł +z tv +yo el +wine glass +think geek +succumb ing +se ast +schu tt +sa ez +s fl +ri jn +reform now +pit ney +pe ch +pan jab +ol ling +off sets +noct ilu +ne ga +mu bi +mot sepe +malaw ian +loo kin +le if +kr acker +it son +in au +ev aded +ed camp +ed accessible +e ich +d ju +colorador apids +clou dof +challenge accepted +bul ger +ba hut +ar keting +a oun +ðŁĴĹ # +ðŁĴĭ # +ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ðŁ¤£ +ãħ ľ +z oku +worldwar ii +wich ita +wearein puglia +ur lacher +unh cr +tranqu ille +spear fish +saunder son +sau rav +sam each +saluteto service +ruge ley +re decorated +phy te +ot so +ori sts +nex change +ne gate +me any +jere mie +holly hock +he ys +guid ry +fre sn +earth day +defl ate +con oco +choice internationalartist +check book +caric om +ble ue +baji raom +aur um +asu u +ais i +ach ille +wu vip +wishful thinking +wigan warrior +weg ner +unice findia +summ ative +recap tured +pri ve +person hood +p anga +new jersey +ne squ +mac chi +le mentary +inver urie +infra structural +independenceday india +in fringe +ham mon +goo der +gn fnr +gla ssc +gh andi +d ations +chad wick +cacophon y +ca rec +butter ball +bu gis +be ale +appic oftheweek +a store +ðŁ¤ ³ +å ģ +~ ; +yu ju +yas ssss +water down +war machine +tu ggle +ton nage +ter ly +spite ful +spang lish +sleepy time +rox ette +rome sco +ren uka +pu mbaa +popup shop +perfec tionists +palae ontology +ne ces +nba allstar +mn d +mini mus +maz u +mag lev +lea day +ka ela +jap ur +ith es +dx b +by ng +buff er +ber glund +bak in +alphon sus +ðŁĺģ ) +vi ad +train hard +thisis the +t out +river men +raz in +pla ins +peck in +ling en +kom ori +kh rush +keepbritain tidy +kate e +jit u +idle wild +heze kiah +glos birds +genetic ist +frangi pani +dev aughn +crypto graphic +c ads +bu stier +ber ber +baby z +as ari +am ass +í ħ +å± ± +~~~~ ~ +xim ena +x cited +well nes +vin ta +v gm +unci os +torre ira +then ia +tejas swi +ta inte +schrö dinger +radio activity +poo k +ou is +oo tn +medi ating +k able +iss ner +ir ri +ingui stics +in pink +in fidel +ham ble +fra s +est y +dair port +cra pe +comp lies +boat yard +batt i +an thi +am era +actu ality +west morland +we ssel +ve sely +unear ths +underwater photography +un consci +theryan adams +the hill +shu is +sed ative +ralph s +pic ardo +onther adio +o ig +o bert +mom en +ming hao +mercy me +mari kana +mac gowan +lethar gic +johno liver +har ima +haberdash ery +guzz ler +good ger +frank l +feu er +dir tier +cas l +bol sters +ay ak +am bit +! âĺº +world childrensday +west ville +un justified +u hl +turn stone +theni asharma +tatt ers +ta wi +supportsmall er +stra the +stin ker +sm ou +slam dance +skil ful +sauter nes +ram madhav +r fr +quat re +protec tor +pal am +nut ley +movie pass +mfl twitterati +margol is +mar leau +lh saa +he wer +hawai Ê»i +fellow es +f ch +etsym nt +dag ar +cheer wine +change management +bust ard +bu h +bra via +bel lec +b pp +app r +aplayer sprogram +amic ha +accomod ation +åŃ ¦ +van jones +u stin +tri x +tam ia +sward son +siar gao +shou nen +repadam schiff +rausch enberg +r ke +prop ell +pe jic +oc toberfest +o sta +new gate +mo sas +mer co +lake districtn +ko sh +kan on +jo w +indv ban +hur tin +hondar acing +ho pper +hbo boxing +geoff johns +fru sci +fis kars +felic itation +deepak chopra +ddin h +dat in +comuni dad +ch hota +carcino genic +banjar masin +aut zen +tw an +tri bals +team bts +sub buteo +sr at +se ta +sal onga +per ig +over powering +mec cano +mccl ung +mar kaz +ke ss +inve steu +impe ded +ian u +hello fresh +gtas nap +extingui shing +entom atoes +eni x +edo dyssey +e mis +country music +confront ational +bin ned +bhu v +annot ating +ambed kar +x bmc +who ah +ut sc +tou l +th birthday +super yachts +sul la +stry per +smel ting +smar tin +sal sify +roberto cavalli +pill box +pga show +narro west +mye dit +moss ley +mil y +judge mental +her universe +he ver +go leaf +global citizen +gira ffe +fluctu ating +fat ales +cura çao +clay more +cap tors +bu dak +brad pitt +bir n +ba dia +angi es +an etwork +alla gash +al tho +al it +ac aba +ab ita +yu jin +wonder ment +turbo tuesday +su che +si amo +sho peep +shopeep h +s month +pokemonsword shield +paddle board +maz har +kick started +ker ja +kear se +kan chi +k you +hoo phall +gym motivation +gro ton +flam ingo +fair bairn +eury dice +etr ics +em press +dispo sse +dev das +cy press +ce di +bat ur +bar ras +bal oney +... ðŁĺī +ðŁı ĸ +âķ ļ +ಠ¨ +yag ami +vali ente +tinie tempah +thehashtag game +ss oftball +spiegel tent +slou chy +serre mmy +sa pper +pha edra +par to +p cc +ot u +o este +northeast india +ner dalert +merri field +li ff +kookabur ras +kitch in +ital o +housel ondon +han u +h waiting +gov nl +forwar der +f ath +equi ps +conserv atories +clo velly +boost mobile +bic olor +ber ts +bald rick +art twit +accommod ates +ðŁļ ģ +âĿ£ï¸ı âĿ£ï¸ı +âĻ ¯ +z berg +yel ena +south aven +sou ther +sketch fest +sel and +sc ut +roll icking +road bike +re amer +r db +press office +na j +mother care +meet stvj +inver no +ic ould +hulla baloo +hell muth +hall marked +gri c +bath tubs +at b +ìľ ¤ +è £ +âĻ ķ +à¥ĭ _ +ÑĢ о +zen yatta +yess cotland +wh io +volksk rant +un ltd +tro oper +ti run +the cw +thal amus +tax man +st rood +sn elson +sm mw +sen ran +sec r +sch ia +photo copy +pakv nz +oak lands +o sco +nu pe +nam biar +mur t +mok sha +mam ak +lakedistrictn pa +josh lia +jay leno +inter course +gr ackle +gen ic +ge tu +ful ci +foo trest +fjor d +e cha +dhoo par +con ger +cau ght +buck wild +blo grt +band shell +azadimarch pti +amit ab +. ðŁĴĸ +% % +ðŁĽ Ģ +ëį° ìĿ´ +ëĭ¬ ìĿĺ +ãĤ¸ãĥ § +zum walt +z han +world con +wi ec +wag goner +uni vision +transl ation +the bad +temper am +sto pper +sla ger +sh q +s bj +rcb v +rb ma +rashtri ya +par mo +mano euvre +m run +lucre tia +ip sc +incol n +id ine +house hunting +gidd yup +epo ch +dern gate +dart ington +cop adelrey +co piers +chec a +catan zaro +carers week +cam phor +bustam ante +bul li +bou y +blay lock +battlefield v +bank sia +ascen ds +as san +antoni oni +americann injawarrior +am ah +alley ways +aldub meetstvj +alab ama +al amitos +worry ingly +ultra thin +u emura +tu sker +ti miso +ta kor +staphy lo +southern railuk +son ice +smoul dering +smol tz +save the +river fest +reading list +proclaim ers +po ss +ny d +ni kil +nbc svu +n of +mis behave +luxury hotels +lu kin +lipsy nc +ks la +ko ki +ju elz +ike bana +hot tie +hiker chat +hau ser +gi ang +emer g +cu shy +brat ty +bch wy +basketb al +as q +anti war +andalu sian +amherst burg +aber soch +' ] +ðŁİīðŁİī ðŁİīðŁİīðŁİī +west bridgford +v nl +so dhi +shutter fly +scar borough +sau ctions +sar far +round abouts +riseand grind +ric co +music biz +mari ko +le cker +l illa +ktn news +geel ani +fli ppy +doo kie +domin gos +close d +che o +cat rin +black star +be emer +a jun +ðŁĻĮ # +ðŁİ¬ ' +warren ton +visi bil +temple univ +t ny +sit ch +shel ove +sh ini +sequ enced +scu ola +quad ri +procrastin ator +pin kin +paper boy +naturo pathy +n ouri +milli kin +mac dowell +long man +le tras +krist offerson +kri m +ker as +ing dean +housing crisis +ham za +gra e +gh r +g sat +fac ing +eun ha +e oi +colli der +bri des +beach house +ar mit +all t +ad den +ãĥ³ãĤ ° +zacatec as +welcom escotland +wa pping +voice first +st ints +sav or +roof ing +ro shan +rec tum +out weighs +ms by +micro array +mar dy +kalaign ar +hand work +guardian ship +fore castle +fen ice +fabric ations +er rrr +ene ch +eat at +ds ound +consul ting +cm world +broad y +break dancing +bp cl +ban del +ak ere +ag pur +af lac +[ " +ðŁ¤ŀ ðŁı¾ +what make +wakand aforever +w cracing +val jean +un ge +twit cho +t li +sy o +stru thers +sha and +mar cy +ma be +l wt +heim lich +heather wick +francis cans +field work +est en +del tona +cycle way +ces spool +bur nett +barba resco +aspin all +apor te +__ . +âľ ° +zoo logist +yor ba +we ard +wall in +tur m +timmer mans +ti ong +this ss +tat ers +sw inger +son dra +si phone +secre te +pythag orean +post modernism +perfu mery +over thrown +orac lec +news man +national pizzaday +mal u +m smes +lede cky +la brin +ke elan +i spossible +holder ness +hel ge +harle ch +giov ann +f örde +er rol +epi dural +dongh yuk +detr itus +cor yn +capital tv +canon ization +blow back +berg mann +be il +ba thin +aux ili +aggi eland +ðŁıĨ . +âĿ¤ï¸ı âĺºï¸ı +yo bo +xx vii +wind ass +we sker +wcc b +war rant +tol and +tele presence +ste ren +sp ath +se sto +sau der +ry den +qantas wallabies +priya anand +paris roubaix +o dr +nick cannon +ni o +magde burg +lo thar +kenny omegam +karen ina +jon ze +je ane +iti o +heath field +ex y +den ney +courtney force +cer ys +burn aboy +beig net +bag gs +ash en +anecdo tal +wildlife art +transc ended +ta via +streng then +show place +sanji v +sal les +returno fthe +princi pia +pau lista +ok amoto +nine inch +mu tu +min smere +manjre kar +lebat ardshow +kene ally +gh or +euryth mics +dub smash +chiff chaff +chi kara +buil da +boga erts +bin nie +ber ner +asi t +ary as +anj ali +agni eszka +abhi jit +ðŁ¤¦ âĢįâĻĤï¸ı +zen on +your boy +ver ry +vanda waterfront +un trained +un original +um t +ty ro +tendin itis +stra ighter +spring ville +simon ds +shop lifters +sapere condiviso +saint paul +prowrestling ts +plum stead +pi stil +phil om +petro vic +ne vere +multic am +mo go +main ers +ma wra +kab ayan +ji ka +hall i +gar lin +fuji wara +fri ston +fast pass +dep ooja +chale ts +c illi +bre reton +best team +b ni +ag ens +whitec ourt +whistle blowing +wau n +vre eland +un dum +uk tv +tailli ght +tag ua +sé bastien +retali atory +remo dsouza +rb k +progno stic +or ic +on star +olek sandr +la al +kennyomegam anx +jer on +islam ism +insu lar +hen ning +gal indo +f ter +exple tive +docker con +design ating +dam bi +culture of +cr d +con me +chor i +car ob +bookaday uk +andy burnham +am ick +aller os +acom munity +ðŁĴĸ @ +º f +z music +yel owo +wq xr +wido do +war path +ti maru +ssc napoli +smu dging +se itan +recur sion +re homed +progen itor +pra shanth +pluto flyby +peyton list +pend ra +mus d +lati mes +kwan za +kev yn +justic ele +jazz music +in consistencies +in cas +in ator +hoy lake +half moon +dementi a +dem ure +criticalrole cosplay +cornwall is +chuk au +chhe tri +car dale +bri anc +b so +autom aton +ar tha +anti thesis +alv arez +alex ie +⼠½ +ৠĭ +yu kio +wcc b +wash cloth +vijay diwas +venuse swilliams +v li +tt ingen +tro xler +sor do +sl x +shim bun +she ene +rock a +re training +ra ha +proof reader +prolet ariat +pin ta +pedagoo friday +pe der +ortho dox +off shoot +nu gs +newyear sre +n wac +mor illo +man tas +m ure +law man +kishi moto +kenne bec +k anya +jummah mubarak +hiro mi +fan x +eye piece +esc p +eck stein +dichro ic +de aver +cra pper +cou riers +cor us +chand hok +cch l +cas key +biggreen egg +au ss +ag nus +af rance +! âĺĢï¸ı +ðŁijĮðŁı¼ ðŁijĮðŁı¼ +è le +your anon +year wood +un tie +tur nit +tor sten +thom asian +tb ats +swwap niljoshi +sop retty +simple plan +sil vana +se ppi +sat oru +s love +ph wx +mom aps +mir in +mer curio +loud wire +lexisnex is +ko do +kim mie +ki mi +ing ri +ham leys +gar misch +eu x +eng al +e wi +demon a +daily post +cul tists +colle g +c sd +c ason +body art +beer house +ald wych +alber ni +! } +yel ich +water mill +visit nc +unlock able +un attainable +u ct +traff line +tas ki +tal low +st off +space weather +snu ff +smash burger +shep p +secon do +school choice +riv lin +radic alized +pug sley +prime minister +old navy +o aths +nt pc +muja hid +migno gna +michi e +micalle f +lusit ania +looooo ool +le vies +kra ken +ive son +inte gers +immin ently +if ma +horror film +fpj b +fluid ics +f nac +es th +el b +eb bets +ci at +char it +apo ptosis +am bos +ðŁĺī âĿ¤ï¸ı +بص رÙĬ +ti ef +te i +tar ja +t ade +pd n +pascag oula +on asap +ni xie +nay e +n me +music india +mam mary +long board +krak ów +ken der +happy wednesday +girl band +fau stina +fan arts +do tting +dar m +con ium +cleve leys +broke girls +br inks +bas er +ash vili +asan try +ðŁĴ Ī +ãģķãĤĵ ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +âĹ ĸ +upp ere +twitchtv gaming +to serve +tat litu +t savo +sigh ing +sav en +royal randwick +ri ks +r tweets +play fair +p mi +nuff said +n mi +lick er +jawor ski +ing ia +ho orn +harvar dh +god child +dol men +dog and +co lette +cit go +bru g +bo asted +blu ecol +ambi dextrous +am ins +é o +tweeta pic +thur n +street dance +stau ffer +scra ppers +pu an +protec tionist +passer sby +pa ia +oxid ant +norm als +nir bhaya +navi o +nac er +ku ip +iri g +ide alist +hishammu ddinh +gro ggy +daver amsey +chukau munna +bla gden +ben ilde +barba rella +auto cratic +asqu ith +asham edly +addisab aba +xer xes +whe alth +val po +v yn +tar ry +tann adice +sw cc +san ts +re pulse +price isright +port ola +o zo +o tros +n gan +mohan shakti +mel vins +masqu reshi +mar ven +magni fi +l mi +krat z +jeff co +je v +intothe badlands +humper dinck +hu masqureshi +hu acan +gun ter +gautam gulati +gat t +e or +crissc ross +cre ech +coonawar ra +co ed +cheer fully +arn dale +app elle +æĦ Ľ +wool rich +vin dol +uk scouting +uf confox +ta ito +t ella +stu dd +si dent +shel ve +resver atrol +prestige diesels +per fi +new world +na ja +mr peter +may fly +lit ol +liberty london +kale y +ill ya +hill sides +head bangers +guard smen +existenti alism +en k +el icious +dew tour +deflec ted +ch lo +bum per +big weekend +bhu t +az kaban +argentine an +andron icus +up turn +u sports +tri vet +the heirs +thalasse mia +ter rains +t ates +sul len +ri ram +recombin ant +re decorate +pp ps +penguin day +pas ado +nottingh ill +ne ys +mugh als +mind map +lu q +ken sal +ig inla +hyster ics +hermen eu +heg depooja +h te +h li +garra way +fu bu +f age +di de +day lily +cri stal +craw lers +chandler riggs +ch eni +campan ella +caip irinha +caf f +bre tton +boutique hotel +be swick +av ens +and black +vol are +vick er +u ca +tri um +too ley +tin caps +super bad +sti fling +steel case +sh ero +sea hawk +persist ently +painst aking +ous d +negr ito +lo vi +laksh man +know lton +hol men +gol dring +gh or +event planning +copy right +contest india +cine phile +cayman islands +caf cofficial +but thurt +bre vity +bat tier +bas ile +bare illy +ba isse +ame al +al mon +ï¸ıâĥ£ @ +ãĥĹ ãĥŃ +âķIJ âķ +á´ ĩ +Ùħ ار +wh yd +uniof reading +under used +uber facts +twitchplay spokemon +test i +sta ite +sne xt +se belius +sa aho +s ll +ru lli +refu gio +qu s +pre dominant +pr ats +potre ro +period ical +passive house +of acial +moneti zing +mo ola +me igh +mandar ake +kh ta +ker win +kaz uya +k tp +issar ae +is y +ini shi +helsing borg +ham ada +gladi olus +g ort +fa sti +equ an +el fin +dough nut +disappo int +def tly +book week +black listing +ber ahino +bad water +as bo +a italia +ðŁĺĺ ðŁĺį +ÙĬ ر +wra th +wiki how +web be +vie to +vid ere +um titi +u mmmmm +trout man +thum ps +thisi sc +tab las +sub due +stra ding +sei bert +s met +releasethe snydercut +rech ts +po gue +pagan ini +oo sthu +nz xt +na har +my re +mixed reality +knight s +kam era +interi eur +infomer cial +h plove +field museum +fati mah +ent en +discord app +detri ment +coffee addict +chicag os +bou cher +boc coni +bach ia +ave bury +arsen io +ane ury +ago toronto +ðŁĶ İ +îĦĨ îĦĨ +war ung +vo g +twit t +the strain +sil ber +set ter +schi ef +rose wood +re sta +penn live +pad docks +ot al +o iler +mis diagnosed +minec raf +le ef +la way +kin ley +ki shore +karan mehra +jail ing +j angle +glass blowing +fau stus +e bru +dont cha +dfb pokal +de spot +cutt in +blan chfield +biop sies +ani zation +alec baldwin +al var +ðŁĶ¥ ðŁĴ¥ +ë £ +z au +wood sy +u haul +te oti +taste bud +stom e +spr it +so ave +sim co +si dings +rocke try +ri kerr +plan eta +par ivar +official pdc +nu eces +nineinch nails +ma zzy +lec tern +le ffe +ke illor +j vp +industrial isation +fl cl +de od +cra vens +ben tayga +bad ness +ba stos +ar bus +ak ame +ade v +. :-) +ðŁĴĿ ðŁĴĿ +ðŁĴĢ ðŁĺĤ +ðŁijĩðŁı½ ðŁijĩðŁı½ +whomade my +weald stone +vig ner +sma shers +simon schuster +sil ang +screen grab +sch aden +pre amble +pan elled +p imper +over turning +ordin arily +mindy project +micro blading +micha ell +mf gday +malvi ya +la gos +k ado +jer ky +ine t +h fs +gigab a +fr drx +forte scue +flo s +der van +crazyricha sians +check outs +bin ary +baltic sea +andre u +a oh +ðŁĺĤðŁĺĤ ðŁĺŃðŁĺŃ +Ë ¢ +ü ck +zare en +yr dsb +venkai ah +the irc +si ssi +popul ating +orange bowl +mrjames may +modern ised +ly nott +li minal +ju alan +inter agency +indefati gable +hi ze +hc dsb +havil and +gd la +fe tta +everybody in +er p +ed westwick +dal it +cu pp +critical thinking +back man +b fw +ari m +aid u +ðŁİħ ðŁİĦðŁİģ +иР» +zou ma +y von +worl dis +woo zy +vi um +urban exploration +tour guide +tom foolery +ti mmer +thri fting +t ole +su ke +sto ch +ssav elives +spoke sperson +shear smith +run nin +ro skam +pra deep +pein tre +oo h +nc ca +me ep +mal indi +m ts +la var +iv onne +in ara +front side +fanta sizing +extrac tions +enthr oned +el ve +draw back +cre atin +coral gables +chan try +black ley +behe st +zak ir +ver gil +v san +ty pal +tran mere +tor ri +to ques +tl x +ther mally +sil iguri +sei yuu +schwarz kopf +same tha +pum per +pre production +paradox es +pan j +ok ano +neck ties +naomic ampbell +mol son +middle man +mass i +mash ima +maj ik +lur ker +ld conf +la bo +k fi +h jel +g hay +don jazzy +debat ers +de vers +dag s +cut let +cole lla +can wnt +bb ctms +b ka +arm yof +--- >> +ðŁĻĭ âĢįâĻĢï¸ı +ëĶĶ ìĺ¤ +x au +would ve +vic arious +ven tre +tre anor +tra pad +touri s +su su +shorth anded +sed alia +r sb +ninj awarrior +new scientist +nd re +mun chen +mahog any +k rane +jo balert +ice gov +heske th +gored forwomen +fish bone +fi bula +endo scope +e gh +corin na +block ages +art ag +aristo cracy +. ðŁijĩ +ðŁķ Ļ +ðŁij¨âĢįðŁij©âĢįðŁij§âĢį ðŁij¦ +yn olan +va un +u zo +the elephant +ted talk +te sty +ta shan +ta ji +swis scom +sun power +scol ari +revol ver +rash ard +ram blin +ra an +prin cen +pin afore +pai ste +onther ise +naz ca +mrin poche +monte verde +mas thead +ma at +low en +kurt angle +jol ley +inter professional +guit o +gan e +fakel ove +eyeson you +do hc +dd ha +davi dal +cul tof +cre m +ch aus +cer ca +car and +be friends +bbow t +arter ton +air venture +ai der +aham sharma +aaaa aah +a ama +ðŁĺijðŁĺij ðŁĺij +ðŁĩ¨ðŁĩ ² +z han +wy dad +wra sse +we tv +w ente +vel shi +u hp +to so +spe a +re invent +pun j +pott inger +or nel +mitsu i +mb app +lulla bies +lin ville +kam inski +k anga +int angi +hpe discover +guen ther +gi zzard +econ d +di gga +chan hassen +ben adry +bed ingfield +battle ships +auto expo +amerit rade +ãĥ» ) +âĻ¥ # +âĢĭ : +tar anto +student voice +st ents +squee gee +son ography +sol f +sher rie +sac bee +s finest +red nation +py con +procor rpcrd +pin cushion +persi sting +nat con +mvp buzz +man ews +la ssa +ja jaja +ill on +ha bra +getac tive +fa ren +er h +e redcarpet +diabe tic +cra vat +cmoffice up +chor tle +care l +car phone +car luccio +baad shah +ak ua +water boys +w mtw +v room +tis land +ste ful +s ÃŃ +ruden ess +radha krishnan +post colonial +po sies +pit fall +perpi gnan +per ching +pad dington +nfu countryside +manspla ining +madin at +lett sville +lam bretta +la under +kap atid +jour i +jale bi +invicta fights +hur on +he seltine +hard boiled +grave stones +gel der +fant asma +dry land +desper ate +den otes +colon ie +cat chit +carter sville +bl b +bird bath +bab ul +ath ome +ai zaw +าภ§ +z ade +wish youwere +westhe imer +wegot this +un tethered +the vikaskhanna +ten dering +swim bikerun +sky force +sc ouring +ro sea +ram paging +pal co +out pacing +oc el +now lan +niall horan +negoti ates +my news +moment um +max illo +l pi +in my +hplove craft +hol loman +fresh start +et su +du jardin +du ffer +d acre +cr di +cosho cton +bro ach +blo tting +be do +bal asu +ar ousal +ðŁĻĭ ðŁı»âĢįâĻĢï¸ı +ðŁİ¾ ðŁİ¾ +wt cr +un reserved +u lic +tre sco +tote ms +til den +stylist magazine +spor tin +shi van +sad af +row es +raw ling +rav ages +quim by +qu ary +pra ger +or omia +ohi op +odes za +ober oi +nz warriors +mor ano +mon stro +mign ano +maj u +lit ty +ki hei +jim ny +im thereal +ham p +hair piece +fi ggis +es ong +embol dened +di oni +cu pra +cam an +call ander +bil o +amc bride +al ow +ë¯ ¸ë +zi zi +wreck ing +wo bbles +will o +wen i +uniof bath +uki ah +to kun +the matt +shutter bug +ship men +pu ddy +prah ran +post en +pon son +pe yote +pad don +p vs +nad ya +mesmeri ze +mel ayu +m pho +lar ri +kani ka +kam el +hipp est +go panthers +first lady +farm house +dont giveup +dispen sation +concealed carry +citron ella +ches nutt +bri sk +boho chic +beer festival +ba rents +auto cross +ar ndt +accep tably +ðŁIJĿðŁIJĿ ðŁIJĿ +ðŁİĤ ðŁİģ +yaroslav l +x pert +w pt +tat u +steve carell +ski umah +sal uda +rothen berg +rever ted +re shaped +r li +people sclimate +nik ah +long wood +lew i +la jolla +la chapelle +kun da +julianne moore +jichang wook +holli ster +fruit vale +for gave +flat line +fin nick +edex cel +dou te +de tract +confec tionary +char min +bru hh +bri dgers +bo im +bo ggle +bicarbon ate +bad ami +av aram +ðŁIJ ² +Ì ¸ +wolf s +whywedo research +u moja +sukhum vit +shrie king +scoo per +pin tail +ol ha +o stap +mur al +mid gard +mck ayla +le ps +in ad +if pri +hen man +gwa ii +golden eagles +geo location +freef ood +fly high +extravag ance +e ji +dar uma +care ssing +book sfor +bon nell +ble ch +bac illus +ಠ¥ +whoo per +wh oooo +vik ramad +usch amber +us kin +trojan pride +track xc +taylor sville +sy outh +stray kids +spe wed +sco ffing +ric kety +re map +pro polis +po is +plan che +on te +o bra +milit ancy +mc ferrin +man nix +l hc +kat ter +infe ction +hun s +guil herme +forthe future +faber books +end bsl +edgar do +dl x +ce sa +brick lane +australi azoo +ameli or +agn elli +Ùħ ÙĦ +ya ounde +vidy ut +u mang +travel bug +sur in +sol vay +sim plex +shu ka +se cord +sa hu +re illy +paster nak +pag a +ol lis +lo hud +kum quat +he wit +hau gen +gren del +gl k +fo tboll +f br +dr acu +devi zes +cur rington +chun ky +brood mare +be aked +baf f +aero sols +^ ~^ +ðŁĮ © +ìŀŃ ìĬ¨ +zildjian company +you ha +x f +va ar +us amb +the fox +substance painter +stable ford +si mo +red light +pat ting +pag osa +off loading +of fu +magh reb +madele ine +lad der +l alli +jun ge +j ml +j ago +in alienable +im maturity +hell enic +hate speech +frogg att +forever young +flan ks +cruise ship +craig david +clu tha +choke hold +chick adees +carriage works +cam bie +ber tin +aw it +) [ +è ¼ +ãĥ ¨ +áĥ ļ +zi us +web fest +voice mails +tr and +stre gis +sla dies +rub ino +rafi q +quit smoking +mu gu +mo ira +mati st +master race +lin nets +law l +head hunter +gou ver +golds gym +games workshop +franci acorta +fil son +eth at +cro hn +col ander +car sofinstagram +bu ta +ble st +b wp +as occer +am bur +af air +uniof adelaide +un cler +u indy +tow nie +tin ib +the timmcgraw +tau ghtme +synthe sizing +strang ler +stac ia +shar ry +se ssed +rockn roll +re pubs +ping u +ork spolice +or éal +opp enheim +oo ther +o eb +nor bury +lang ou +her cule +hard wired +green bay +for ma +fol lette +fiercen ess +em elis +den ser +de urope +dd f +dan by +dag u +cul tist +chil la +brig antine +blood sport +amazon deals +al bon +Ù ij +wor thy +tse mrinpoche +tool bar +tiv itis +ti krit +te rex +tag es +south bay +simon sinek +sam bil +rupi ah +rough trade +ro saries +ra ffy +poz ole +polar ised +phylo geny +parsipp any +os de +ol dro +obey ed +na jaf +my lene +moo sa +michel a +mariob atali +juno awards +ju ez +jab ar +inst aphoto +getit live +foo k +fm la +far rer +eviden ces +ennis cor +dopp io +dele uze +crime prevention +cleveland art +cam elia +bl acking +be ac +bar illa +ar rr +and newton +yodel ing +ym d +vi enne +tur ban +tsvangi rai +tin o +tas chen +sli med +sex toys +schneider lin +savethe planet +rhett and +rebel de +ran bu +nau tan +mu p +leish man +le jos +ke inishi +jer k +jav ur +j pa +immol ation +h wi +gr n +fried kin +fc g +euph ori +esk en +cypri an +cosmo logical +comra de +cold ness +at erial +ash man +app raised +ai ken +ðŁĩ¸ ðŁĩª +ze bo +za hara +wa dia +unitedwe stand +underwhel med +touri sma +to pr +tam ron +ra sk +pri z +per spiration +pann u +me tan +lor demusic +ki pl +kar ia +joondal up +ilo vic +for bids +fi o +desro siers +darry l +christmas shopping +bajiraom ast +as anas +ane urin +alla gash +alber ton +acos metics +ðŁıĮï¸ı âĢįâĻĤï¸ı +ðŁį µ +ç ı +wha aaaat +w pix +vege tarianism +under lining +tam al +stun ners +squad ra +spondy lo +slow cooker +ski b +si wa +sh aya +s for +ru ts +ru iz +rhine beck +nec ke +nam ba +ludo vic +lc n +kl at +john jay +jin go +j fa +ij m +ha ki +go pi +gir van +gau ss +expat life +dont stop +cre denza +co stam +cat amount +car ville +cac i +bicic leta +abbot ts +ðŁij ³ +âĿ¤ ðŁĻı +âľħ âľħ +âĻ¥âĻ¥âĻ¥âĻ¥ âĻ¥âĻ¥âĻ¥âĻ¥ +y vette +wymond ham +wack o +tu ris +the tical +the offici +strat is +sin clair +simon j +ry uji +rest i +ra gamu +porth madog +page antry +over laid +newton ian +nbas ummer +mor phy +me drano +mass oud +mam aron +lu ig +lor can +longre ach +lon ey +lions gate +ko ga +k lara +ir ma +impe dance +hydro gel +hidd ink +gore gaon +fr ink +emb laz +dg i +dark fantasy +cq uni +cor pse +colon y +collo idal +che shunt +buck shot +boy ds +ba ab +amazon smile +ag waii +aday in +% ] +ye ye +white hurst +toron ado +thaw ed +swe eny +shri gley +rival ed +rit ts +play date +pep ys +pemb ina +pas olini +ori de +mit ral +mccau l +mar ies +mac bookpro +jarre au +incre ments +hp cl +green screen +gar dn +foun dries +doo o +dispat ching +ched in +char mers +bri ano +ble ak +be veled +aw omen +asi ancup +anar ancic +ahmad inejad +yeovil ton +will fully +water tight +water son +uv acha +standardi sed +smok ymountains +sm rookies +sha ad +se mble +scar pe +sb news +re mmurd +queen sc +phen yl +petter sson +patron ize +nest ling +mor ta +mis ssa +m elli +ly can +local ities +jo bim +i oni +home builders +har f +h sh +ga sification +fre snel +flu g +e ireann +drag con +dj embe +del son +config mgr +christ ina +ca ity +c moh +business development +baltimore sun +ani um +am adri +whot els +vp debate +tsu ki +tra ste +timiso ara +sti ffer +stat o +snoo ty +shi koku +reggae music +rede ye +pur cell +pro getto +pir s +perl mutations +par tof +oooooooo oooo +national girlfriend +montgom erie +la bi +kari ba +in ard +har deman +hal in +gurup ur +gu shue +ge stern +food bloggers +en ola +dy ckman +dee zy +d illion +con wy +cis f +ch inst +cb stv +black thorne +belag avi +az lan +ay oub +at am +ang ina +alumin a +ðŁĴĦ ðŁĴĭ +ðŁįķ ðŁįķ +âŀ Ŀ +zeec ine +z apper +wild horse +u play +tt c +terry crews +sun chat +spe ke +slu sh +se eyour +peshawar attack +pel ted +pe tul +omor row +nizam uddin +neur one +mi dair +mer win +megal om +mc duck +mariska hargitay +maine coon +kran j +kev ins +inter con +identi fier +hale akala +gro win +gre ssion +frusci ante +fo zzie +eric clapton +dri ppy +dor drecht +dip set +con nick +ban yak +anderson ville +an iso +acur ry +a afp +Ï Ĩ +wx brad +wd tn +un painted +tripp ier +travel addict +tar kan +super i +spiderman farfromhome +si ddle +shu cks +san ada +ro st +ra as +poe ia +ov ate +old bury +ne yo +n é +mor pur +mix in +k fox +ine tt +in ab +horton works +h va +go sden +fay go +er de +em h +dragon force +day e +davie ss +dark net +clo the +by ker +bi ggy +audi os +ðŁĵĪ : +ðŁ¤· âĢįâĻĤï¸ı +Ñģ к +whit son +vise grad +vel led +taste makers +stefan os +sm s +over do +nul l +ne wey +min uto +mil on +makeyour mark +mac chio +lor in +lor imer +ko bra +ke cil +jcp sky +it ag +ham achi +free ways +fo v +fl inging +dig al +blo gg +bel grano +bel court +autom ne +ar rr +ale mania +ad ow +ðŁĴĻ . +ë°ķ ì§ĢíĽĪ +young ins +with ington +us amy +un following +tunn elling +tam arin +tag us +super footy +strat en +spla shy +spec tru +smart doll +po ps +out work +of icial +nap o +mir vish +metro poli +me arns +mav tv +marke tin +m wal +levenshul me +l ho +know able +ke itel +jb hifi +interce de +hatch ling +go warriors +for duk +ex ol +ed to +dreamscape ph +dor noch +cou pero +corn el +competiti ve +comm ander +col let +cla pp +centime tres +ca is +bubba watson +bro der +bren nen +blit zed +ber als +ay on +au du +aru ah +ami able +aitch ison +a pop +ðŁķ µ +æĸ ° +âĻ » +ym ama +w q +ver tu +tribe caf +trans fusions +tit es +thereal dj +taitt inger +summar ise +shor ts +sbst df +remember when +rammadhav bjp +oco ee +ma sar +le cular +lar key +king fishers +kimjon gun +ke vor +k voa +hel berg +heal d +for success +en circled +dan kie +co fc +chain link +cb j +ca chet +b ams +australian army +all startrek +¬ë ¸ +zale z +zain imam +z anna +z ala +yuz uru +well y +un reachable +u len +tom king +tin ting +tin ny +tim bre +the history +sy sco +spr itzer +spect ating +sor in +sioux land +sher gill +shad well +sarath kumar +sang akkara +raes remmurd +quick step +night clubs +meal worms +me ager +le om +it ri +hygi eni +huss ars +goe son +gid get +ft n +fat belly +ed ta +de whurst +dann o +da q +cri sler +columbi asc +ch inn +brac co +blu emo +big ny +bexley heath +ber tone +: .. +! ðŁĴĽ +âģł âģł +à® İ +اÙĦج ز +ye hey +wool ton +wang sa +twit tter +tul low +toler ates +tik tik +suga red +star girl +south wood +skybetleague one +see ps +sajj ad +sad ar +ru ki +ri dings +pin tof +pi eta +octopu ses +non invasive +n kt +mobo awards +lat ur +its whatwedo +is of +is ba +ij esse +icha el +ice dogs +hum vee +gl sen +gb chefs +frac as +f á +en vi +ea v +dw m +dori an +dar n +come down +cla si +cib sunday +c dot +bu tuan +bob weir +am ys +agar za +ac ri +åı ° +ãĤ¸ ãĤ§ +ал ÑĥÑĪ +y agi +xia omi +wizard world +vince staples +the walkingdead +so ke +sil er +sheff docfest +rhe on +retro wave +rad ke +r mac +pu tit +nissan usa +mi sper +mawra hocane +lindsey vonn +lam an +la hinch +hu uu +fu ad +flam ini +fa hy +f cu +de aton +cy lon +cr ê +chi kun +chi bis +cel is +calri ssian +cab ra +bi let +ay esh +at tock +ade wale +ú l +z uki +vi dal +thur so +sun anda +shake shack +robert son +red un +re balancing +rag ana +psilocy bin +parti r +ny te +mer rell +mel ane +matt smith +manay unk +l leyton +ju mba +gw b +guy ssss +flori dab +e bron +dout zen +cotte sloe +cavall ari +can ti +bre ll +bi gil +bc ps +ba is +ah ha +. + +ðŁij¼ ðŁı» +ë¯ ¸ +ÙĨ ج +yak ov +word camp +we ssels +uk hashtags +two some +trac ee +tag le +sun bury +spin ney +si mas +shar at +sep tembre +ran some +plexig lass +p ng +ni bali +mor tel +mahar ani +kur nool +kr qe +kl u +kick backs +ket u +karan singh +kar tini +kaleido scopic +international womansday +ice e +gre ll +ge tolympus +forthe cure +fc s +f ends +ev angel +estu aries +chro med +cer ny +catalo gued +canad as +cab are +bri mley +b ty +acom ms +âľ Ĥ +zo vic +tar ia +stereo gum +space balls +single market +si wan +satyar thi +sar rain +rela is +re stur +parachu ting +palom afaith +nike id +nau ts +mal ti +liber tarianism +leu co +le mi +jung frau +ijesse williams +fri dakahlo +fawl ty +edint fest +deli verer +bre ra +av x +amin aj +ðŁĺģ @ +çĻ º +ãģŃ ãģĵ +Ø ¬ +алÑĥÑĪ ÑĤа +wal led +tribu taries +tobias menzies +ther lands +tax co +sc ie +repri manded +pud gy +power tools +polyglo t +poll ster +pe les +p soe +or lean +moroc can +men no +mamaron eck +loril oughlin +ko kop +keepthe ban +julian castro +hand hel +gull wing +gro k +gi k +gen na +ge ir +g ddr +der ringer +d sei +cre sc +cor vet +clavic le +christma slights +chhat rapati +chap ters +can iglia +butt ress +bhu sa +( < +ราภĦา +young justice +the athletic +success story +so pot +sav y +rosne ft +ri pert +rhi mes +pw me +pol lok +nirup am +na deau +n saa +n eng +mu sprime +melan in +mag ners +khrush chev +intrac ranial +hy ori +holiday makers +goe bel +gh el +four somes +fal well +fa sia +ev ski +dw g +coning sby +cinemathe que +cbstv studios +cardo zo +cameron mustgo +assassinscre edodyssey +angeli ka +aly te +ðŁĶ ħ +ðŁIJ¾ ðŁIJ¶ +ðŁıĨ âļ½ï¸ı +ðŁ¤¦ ðŁı»âĢįâĻĢï¸ı +à¸ŀ ร +w end +viv ah +ure thane +un funded +ul ar +u ol +tur bans +tal avera +ta di +synchron ize +stil well +reve ille +re ye +p co +o ac +mono gamous +love bird +lav any +lang ar +khair a +hay lee +hair by +food borne +fix the +dru ck +docu series +con fed +city hall +chu bby +calf skin +broad gate +bit urbo +be eline +barr head +bar ti +amar th +ãĤ« ãĥ¡ +youcan doit +wh acking +vrin da +vam si +unra veled +ty kes +toy z +toron tomar +ti zing +thor sten +the gef +ss bu +sou dal +sab tv +s only +pollu tant +pedd lers +pe geeks +pas qua +out fitting +nup es +minute beachclean +mb ar +lin oleum +li sk +lar acroft +kel apa +kab uto +jun ji +jel ani +home depot +gla swe +gla dio +ger ber +follo will +er ato +ene ws +eager ness +cultiv ator +cir c +car lil +cap rio +c ge +c fg +brand in +blu eraiders +bin das +bar g +attribu table +ðŁIJij ðŁIJij +ðŁIJ¢ ðŁIJ¢ +wal singham +w da +vy pe +sushmit asen +sp ana +seven th +sam path +rhe ma +paris nice +neal on +mu lat +mbapp é +let ons +le tra +ki sha +keye tv +keinishi kori +k hari +ji mo +honor ably +ggg boxing +ekur huleni +cbc north +cathay pacific +bu ba +bri die +ax y +ani st +andre scue +americ ang +ðŁİ¤ ðŁİ¤ +wel ds +top con +tame impala +stock hol +sta edt +sa em +rin os +religi ou +rekind ling +pr an +porthar court +play az +photo de +opp er +nl tweets +mest alla +mean ey +ku rian +ivy league +gr illin +fe kir +dig itech +dau x +chro mic +cap s +bun dle +behavi our +bas so +bang i +ar kle +agric ole +aer is +ðŁıĬ âĢįâĻĤï¸ı +ðŁĮ ¯ +âĻ¡ ~ +well park +wat tage +tor menting +taun ton +tablec loths +sub machine +su eur +sonus ood +sate en +sam claflin +sachin pilot +ro p +re plant +om gggg +od do +nor dea +nev ins +nashvillec mt +melbourne city +mari ucci +mangal uru +lind ell +jake paul +i abc +hurt ling +hood lum +h re +greek life +grapp lers +global goal +gaz ed +franchi sement +fo glio +el ms +delo ach +cu uu +coupero gers +commercial property +col um +canber ra +bob tail +blan kie +bet victor +arrow verse +anni ston +zu bin +tv onetv +t mea +sme thwick +sme e +small world +skyl er +rose mcgowan +rit ers +re productions +phleg m +nonleagu ecrowd +nandam uri +nab ila +mid shipmen +mal ham +live chat +lar ain +l ancing +ki b +inadver tent +in life +higg inson +handsom ely +green lee +goo dy +go tz +g posers +fd m +encyclo paedia +en co +doug benson +d tw +con tending +circu ito +char ly +chaffe y +ball room +au ster +arch ies +a him +££ £ +ye oman +x ls +vo da +un conference +tu rok +to dt +thro yd +the moon +tal ai +symbol ically +swar agini +suffo cated +success ors +stream lines +sh aked +sei fert +ri so +provi dence +pric eline +policy making +ply mpton +phin ney +ot ar +of ootball +much h +living room +ku cha +integr ators +great dane +gg ah +ethi er +el roy +dra is +dr mike +de seo +copen hagen +caric aturi +bill kaulitz +benadry l +ben thic +bat as +aminaj mohammed +ys i +yaa as +tø p +swoo ped +sun shade +stan tial +shre ve +sel wood +se hr +sad dening +ri op +rep elled +raj ak +promis cuous +prioriti zes +platte ville +ncaa season +nbasummer league +nak shat +ment on +lie der +kare t +ka al +jav ad +ital ics +id sa +han bury +fifa u +fass binder +dj paulyd +dispon ibles +ch ex +c wi +blin ked +bio hacking +as ur +âĿ ¥ +wh ky +uttra khand +tri bble +th ak +ta ik +sin city +sco ffe +saul nier +riot ous +ric oh +raw k +rac s +olympic torch +novosel ic +news beat +mit ter +mir field +may bin +ma har +lymphe dema +love good +limb ing +lemon twittor +kle en +k sat +img models +hil aire +ha spel +gel ert +ffr f +do se +dick enson +dicho tom +del bert +dc tech +commi sion +c slewis +b ink +aur yn +ar ica +abyss inian +ðŁIJ ĵ +æ ¢ +wicom ico +w bls +vesti ge +upad hyay +un suk +ty pist +ti en +ta kuma +soap awards +shi els +saucep an +salt ine +rut land +re eland +pre dates +pp o +plac a +os asuna +mill town +mari huana +mar ro +kush alt +kis a +john j +jess op +jailavak usa +ino ble +illumin ate +heav yequipment +h ry +g th +dim ly +de sales +chin atown +bier stadt +beat les +aser rano +ab ly +ðŁıĩ ðŁı» +ye eye +vid alia +u ip +ty ron +trump ism +trouble makers +total itarianism +the arjunbijlani +termin al +ted med +sp ack +scon ces +ru ber +rapp elling +pu ppers +plan tronics +pay s +mrs funnybones +modu lator +mill y +mandi ri +m smar +laksh mirai +kam ba +kal lie +ka un +k ome +il ab +heat sink +hawk ish +half moon +gir ders +gand ang +di vis +desig ni +cl dr +ci aran +celo teh +camp agna +bestofthe best +bar ilo +ban dido +b movie +a ala +ðŁĺį ) +ðŁijĪ ðŁijĪ + ¬ +wil bert +wi at +viking river +us ine +tou cher +t ills +sque amish +spe e +shapo valov +sgo ing +saty agraha +quin cey +pryn ce +pac i +new mont +mis ing +mi gan +metall ic +melk sham +mahon y +luc kier +lets goo +lab h +la vey +l ton +ke men +jabber wo +iphon ec +ilay araja +gi vings +fatbelly bella +ever rr +engv sl +eli ze +dre e +corr ales +conjunc tivitis +chang kyun +bunde swe +bis set +bio chemist +++ ++ +ðŁ¥ Ķ +Û Ĵ +ı m +usp hl +tow cester +sc ircle +sc apa +sam yang +ro shi +rin jani +rho des +raj cheerfull +projec tiles +oul ster +o hai +ne ce +nav jot +nat suki +mark ronson +mad dog +la hori +ke agan +inter woven +inter league +i ban +hof stra +hi k +goleaf sgo +foli gno +ex y +em r +du me +cyber jaya +crew man +cor ben +cli ma +chapp aqua +cel ly +bo ya +bi fa +bank s +apo lis +an kar +west cliff +weare thepeople +vo il +uygh urs +ur kin +tt le +taras enko +silver plate +sign ment +shin obu +set ts +sanjay uvacha +refor mist +or puw +nuev os +net care +na thi +morri sey +me tuni +li vy +laverne cox +ky te +kir kin +kei ji +it sp +irr ationally +in africa +heter onor +happ pp +gram edia +goose berries +du pre +cu bus +candidat ure +ca eli +ayotzin apa +as adu +ar os +anderson paak +๠Į +war n +v cats +under pins +u bm +tj maxx +thir deye +tele graaf +tassi e +slug fest +sla de +sh eng +rolls royce +regi ments +pheno types +petti bone +palin dro +pabst blueribbon +old car +nt sa +nepen thez +nat ali +mo tom +mal ky +la peer +kreuz berg +jyo ti +it rtg +invigor ate +honey ed +f ti +emory university +duck man +doo dler +direc tx +ding le +cloak room +chat swood +bur russ +beli al +anan si +am s +ak k +ack ley +:) ))))) +ì° ¨ +âļ«ï¸ı ðŁĶ´ +âĹ Ķ +ye tte +ye ssss +x us +w sm +vic en +v um +s link +puri fies +pleas an +nic olo +min uten +maker faire +ko blenz +isi baya +inter activity +ing club +hindo stan +hand prints +grac eland +glee ful +gi ff +gh id +father ly +equ alled +dissimil ar +cow art +bra sov +bou te +apol los +angle sea +af cw +ðŁĺĺ ðŁĴĻ +under utilized +tosk ana +timbur ton +suk kot +stu por +sou s +slu t +see mar +r mi +queen sboro +pur pler +prophe tess +p tion +or ical +mou les +lun den +little wood +lindsay ell +lid ded +langui shing +kel o +infec tious +hot box +fran sisco +fck in +evil geniuses +drum kit +drug by +dar mian +cop se +conson ant +cann elloni +can tera +bl ore +bab asaheb +** * +ðŁĶ´ ðŁĶµ +ðŁIJ¶ ðŁIJ¶ðŁIJ¶ +waig uru +w no +vincent e +us rowing +un m +toron tor +till and +spra yers +sik ora +share goodness +sal ama +pi galle +pe co +official slc +noi sey +n aci +mus ick +mo en +lu dgate +kc chiefs +jit tery +interior styling +inter lo +ib d +hum safar +hick am +her riman +harvey nichols +gu ttenberg +fire fight +fa hr +emi lee +eliza jane +doo han +disney onice +der showitz +denne hy +decep ticons +d ml +cn ty +clim ited +chess board +capu ano +bol as +birmingham mail +bird fair +back lund +b anca +apu lia +anou shka +ago ddard +. ðŁĺĤðŁĺĤðŁĺĤ +ðŁİ¶ # +ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ +âĸ « +ठ¼ +worka holic +vo xbox +u daan +tur ds +tu ma +tro tz +suppor tn +stock photo +sp itt +si se +sawyer frdrx +road tor +re takes +rapp ahannock +picnic king +para pet +p nas +nordi ques +ni ente +mil agro +marit za +lu ces +lossi emouth +le ef +kyo ko +ko ppel +j lo +infini x +hu bei +godz ill +equality forall +en gulf +du uu +dopp leg +d als +bro ss +bounce back +bi weekly +bh x +bab ar +an gra +. ", +ðŁĴĥ ðŁķº +ìļ© íĻĶ +wel kom +w ya +var ghese +tri anon +time smag +tib betts +therealluke vans +ten sor +sel vam +sch open +ri zzle +re invest +pi ya +parasit ology +one one +om gom +neural gia +nat omas +mini game +mark warner +local ised +ll l +kat vond +immigr ant +ho ho +gra ig +freak ish +for mayor +fif pro +dra wl +art sy +ap w +al tec +acron is +ãĥ¼ãĥ ł +س ت +yggdra sil +upper classmen +un mask +tro ph +sk itch +sho shone +scrib bler +roof ed +re ining +phill is +o zy +mur ty +mind storms +liter atura +left field +l tb +ken ichi +k atta +gal aga +frankfur ter +fr t +fing az +down plays +dhi kr +cru s +con ning +chad li +call backs +apostro phes +adorn ment +? ðŁĺĤðŁĺĤ +é¦ Ļ +za ade +yeswe can +yellow claw +ws bradio +whel don +tori als +tip is +testar ossa +sweden in +su meet +sl sc +red skinst +re agh +nyc schools +multi racial +lu tron +ji va +hand brake +go tch +glo bies +embroi der +e iner +disney side +darke st +curragh race +cur ative +comedy night +chu chu +bur l +bry on +birch all +bar wick +b nr +aw y +an tro +americ ana +all ers +ðŁİĤðŁİĤ ðŁİĤðŁİĤ +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +ye aah +y ancy +water deep +un washed +uk as +tri alist +takor adi +sun tec +stanford med +shr iner +sa en +pg dm +pakv sl +of r +obrig ado +o tho +mini mums +langh orne +lam m +ksen ia +k and +je h +inyour phone +herman as +han sen +guadal canal +gru mbling +gor dons +friendshipin apicture +flabberga sted +eye witnesses +eun jung +drake bell +den ote +colle yville +cloud foundry +che sted +camar go +ba de +aren ado +alber tine +air soft +ç Ŀ +à¸Ļภ° +ଠ¾ +year ns +wire grass +thorough fare +ter idge +sco ped +s bay +rigo letto +ride out +reel z +quin ton +po ston +pa vi +may b +mar ae +kh lo +jun kers +jc de +j evon +ili ans +id fa +hub cap +hop kin +hig don +giac chino +exter nal +dn n +cru iting +cla wing +ci i +brav ura +bra z +beat a +anti fungal +ðŁĺ´ ðŁĺ´ +ðŁĴĻ ðŁĺį +zay to +tomking tk +thi am +theri ver +stu ber +strang ford +scent sy +ru la +polar bears +pen ess +no wor +marc maron +mar com +maker spaces +jor a +johnson ville +je te +ha ben +green army +gan on +fox hound +e gged +do logy +cer ys +beat itudes +bag i +ba sia +ati eri +acne studios +ðŁį¾ ðŁį¾ +âĻ Ķ +vampi rella +tri ver +th of +tee j +super cool +stabil ise +sit down +san ju +sag an +sa cher +ram bla +rag thakur +ra damel +puffer fish +play school +or pen +mel lows +long ingly +lin ney +la xton +k cp +high five +geom atics +ff vii +ent as +ched dar +ch azz +ceri dian +cardio logists +cap size +brew dog +bli the +becc les +as ko +acas sandra +yor o +y im +x lf +vijay television +verte brates +trans net +trans its +tik ki +the ahl +star lite +si achen +she reen +se el +saty ajit +ne ena +mam moths +kingsof leon +kat rine +jit ter +jd morgan +homo log +hl th +gro en +getz laf +g town +fluffy basil +fin ing +ex ofan +ever rrr +du ffy +dra bble +dis arming +de mian +cre oso +chro mia +casi ancup +capit ano +calgar ians +c ve +berl anti +behavi oral +bacter i +aw ani +antic o +ak oz +achi o +ac aster +ðŁĵ Ĵ +à¸Ńภļ +yar borough +wo je +wit che +w mp +v ire +ta kings +sp ats +rise up +quiz master +pah rump +nu mp +nikol aj +lu ar +kabo bs +ii hs +horse sho +hi spani +foun t +fl x +fif tieth +e gel +cu gat +cruci ble +close up +cate ch +cail lou +bott ura +bla ster +bally mun +as olo +array it +allo gic +... âĿ¤ +ðŁĺĤ ðŁĴĸ +ðŁİħ ðŁİħ +yyc traffic +we were +unic redit +un ashamedly +ta ppa +stre at +sar oj +roo ve +rhe um +red der +pir amal +ohio statefb +na sik +mo tha +mac ke +lo han +lapd hq +ip n +ing al +hyper local +hot point +honey pot +fli fe +fel ine +desper ados +dark stalkers +cth sfb +ch ora +cas ale +ban ar +and ering +aim high +adrian aci +adrianaci oci +aan am +!! ðŁİī +ðŁİī ðŁį¾ +⼠³ +vango gh +try p +st ice +st f +south bend +pl ats +pare sis +o sw +mon tra +mc on +lac ri +kho ff +inevit ability +high speed +he ena +hand rails +for tresses +fi eg +f wb +ellen sburg +dre th +dil g +cyst itis +chri sta +chauffe ured +bay lis +ahmadi yy +aerop lan +ðŁĩŃðŁĩ ¹ +ر ÛĮ +windy wilson +who doyou +war by +unfur ling +un ia +to action +thrift shop +the fat +substitu ting +smik kelsen +shape ways +re organize +re appearance +rad as +planet ary +p fg +oosthu izen +naz aire +nam u +moor ings +men a +mar jan +ma kha +lin dau +jay de +isle worth +hope lessness +hol combe +hel wani +flipgrid fever +env hist +ele f +cuteness overload +creatin ine +bun c +black sea +bhumi bol +beef heart +ban ta +az evedo +anton off +!! < +âĸ« ï¸ı +touri stic +tol man +syn ch +river land +rhettand link +over heat +od awson +mul gee +long form +ku k +intermedi ates +ink scape +in habits +ici de +hi stop +financial review +ef teling +door knob +dent su +de compose +cur ritu +co to +camp in +camar ines +bt fc +brendon urie +brea keven +bo iii +be ker +all sven +abhiman yu +a iders +x li +usa wrestling +uni west +tuscu lum +thar pe +tam ra +starbuck suk +sque aker +scor n +sa iga +ra shed +pra g +pla ice +pe ppe +pat audi +nr lgf +noplacelike home +mir ra +michel ada +melaniec music +lom u +live mint +la fontaine +ke at +je mber +it ai +inter nationalist +inst ago +i etf +ho gar +her an +gonor rhea +giam atti +fa jar +der mpath +den tro +dal y +co en +bigg a +anci en +âĢĭ # +yo enis +womenwho code +win elands +will mar +tu gging +too too +tai ji +stru e +sos borne +sk h +ring thebell +pro wein +pr out +play boi +pe on +pash up +nik off +mo town +mo hini +l alu +kar ne +kal ash +k rsna +institution alized +hay le +gan ito +flesh light +ess en +er ring +dram atic +dn as +di ski +darlo biz +crack nell +catholic stl +bsy bjp +bra ylon +ayian apa +(^_ ^) +âĸ¬âĸ¬âĸ¬âĸ¬ âĸ¬âĸ¬âĸ¬âĸ¬ +à¹ģภ¥ +wyck off +wu er +unison tweets +thermost ats +te stu +supersoul sunday +st annis +rim fire +pocket book +po ko +peace haven +music life +multic hoice +mi kol +mari os +lovel ocal +lake district +la ib +ku mail +kis smy +hero ileague +got thard +global edmonton +devotion als +der ik +d wane +crimin alize +cos entino +coach man +cer rado +bu mber +bo sse +animal lovers +agen arian +- < +ðŁĵļ ðŁĵļ +à´ ° +zig gur +winx club +wh yyyy +w day +vibr ato +ten news +te stino +taji k +t sp +t intern +si sal +rece de +po vic +new product +new life +melt water +manag er +malacañ ang +make peace +laz are +l our +kanhai ya +k ø +jor ah +holm del +ho wa +height en +head sup +handsome st +halei gh +green bank +fo st +en zo +downing town +dat sik +cu zco +craigh ead +clare balding +cal poly +brac o +biglittle lies +arte sia +af fluence +! ðŁİĦ +! '" +ðŁĽį ï¸ı +ðŁij¨âĢį ðŁİĵ +zz ang +vulgar ity +vom usic +tattoo artist +tab b +sos venezuela +somerse twt +shre dders +san ches +random izer +par tri +over draft +o shima +n aren +mc ateer +may ers +lund berg +lo ook +kni pe +inthe sun +instin ctive +in got +ilm fest +ilk ley +hoo ke +fer ne +evapor ates +eno y +dash ner +clamp down +chil a +boys and +ash mo +antico agul +": {" +! ..... +wre xham +wr o +top sham +til son +the istic +str unk +stell aris +sn app +screw sc +say hername +pi thy +p alli +osoph y +nor agami +ne phil +natural history +music ales +mls allstar +me ant +mari ju +ma ree +lovel ock +lionel richie +lawrence burg +kl int +k offi +jo is +jair o +ith as +is v +ine twork +in defensible +ill aria +hy men +gunder son +guan ac +gru bbs +gorge ou +golf week +go hil +gar ces +free tibet +fol a +exor c +emer aude +el abs +curios ity +char isse +cay la +can avan +brindi si +are view +antof agasta +am obile +ah saa +:' )) +wit tman +whi anw +whianw amos +watch band +the whisky +stone leigh +sher wani +sand pit +sam ira +ru ane +pmo i +picture perfect +p q +moor gate +mi kare +mass af +mandel brot +man nie +long bow +hel mond +ha ero +g ning +f áil +el mar +do tus +dish ear +di endo +cron ut +chol la +centime ter +bro adest +brigh am +bor rower +bodybuilding com +back ward +ak ah +âĿĹï¸ı âĿĹï¸ı +zor ba +ww mt +wheres mum +tru enorth +transcrip tional +tan kian +si mian +regin acassandra +pre cept +pin cus +pat ella +nbc dfw +my boy +marco polo +kar ylle +j di +iv or +i eri +good nigh +go el +geelong cats +engel bert +din ny +de ary +dae jun +ctvo ttawa +bur khart +boser oy +bel haven +be lew +bbc theoneshow +barbo za +aston ish +ari f +amp ong +ab sten +a hou +ðŁ Ħ +youknow you +ye pp +wy c +w kn +v anni +un made +tot ton +thre ef +the script +south am +shepher dess +set tee +ri fts +port adelaide +nocturn al +n ly +live s +kish or +issu u +in um +helf rich +hay wire +fe tti +fau stino +es ma +er mine +end tb +el ake +dwi vedi +dani pedrosa +cross road +co lima +carac ol +bow a +bbcradi oulster +bab bitt +anesthesi ologist +am illion +air race +abu elo +ab rand +ðŁĵ ¼ +ب د +yan mar +white people +we ts +wal do +w sh +voting rights +u albany +tiny house +tar zana +tai ba +switch back +stitt sville +son ico +si v +serv icio +schopen hauer +ronit boseroy +ril o +re aps +popul i +playadel carmen +plastic ine +paladins game +pal ang +over reacting +o ie +moore music +men lo +maxi me +kuruk shetra +kathr ine +ka ve +k ary +hu ynh +he igh +ha rena +en daal +eli ya +eclip sing +cruick shank +cra dock +body builders +bas anti +anti bullying +ali da +ab ac +, .... +ðŁįª ðŁįª +ðŁ¤· ðŁı¾âĢįâĻĢï¸ı +ภį +yello whammer +y van +world kindnessday +visit greece +u men +straw bs +stom ps +star light +sn k +sign board +shi man +sham anism +scav ino +rue ben +que ira +old house +odi ous +ny on +lam pe +es of +dur g +cripp les +cra ggy +col inas +cnn sotu +braver man +bil ton +back yards +b ffl +amp abay +af front +ðŁIJ Ĺ +å¼ ł +ze ig +xx xi +wo v +wi geon +vid ler +vi pin +ver acity +under developed +ul ata +tu tel +to omer +thor burn +star cast +spo iled +spear man +sound z +she etal +rever ting +paraphra sing +neva eh +mati es +i quique +hy p +hi ragana +giuli anarancic +gigaf actory +free sia +dl day +dance party +crystal healing +chit r +cal stat +boy den +alad din +ak shara +abc family +a edes +) = +( ): +yan k +watercolor painting +us afa +uniof york +twitcho sf +toronto argos +the brian +tad caster +su kumar +stv news +sper mum +re vi +ra ssa +progre s +pri sh +pon dered +ophon es +nom ar +news reader +nat ic +n ll +mil ik +mil ers +meni fee +leg co +l dot +ki djo +hor ley +gulf of +fi shin +cycli sme +cro re +cat dog +bu sing +bla q +bike dc +barra gan +ane ously +ali zed +ðŁij¸ ðŁı¼ +е ÑĤ +wol ford +warrior cats +w dd +thur low +thor sday +ter mine +tepp any +south devon +sky football +scraw led +ol k +ocr acoke +objec tif +o thman +mu gger +mean er +kay e +joy division +jayson dmx +iz awa +ir vin +ing olf +hun ker +huis man +ho bb +ha py +go jacks +ferry man +fern ández +eth no +don aire +deplo y +dan neel +co sette +cli mactic +ceph aly +bro chu +bon ec +aventu ras +ap mas +al mira +æ·±å¤ľãģ® 羣åī£ãģĬçµµæııãģį +ಠª +whit comb +west lands +var ina +tx la +tit el +swee ting +sun cor +sper ling +smith town +shoe shine +sg news +publisher swkly +per ver +p nd +ny ssa +nie man +ngr senate +monad nock +l pl +k ve +just acard +indi ra +hygieni sts +her mand +har dison +ham elin +ha agen +guardian news +grass root +gn ine +forthelove of +er ight +emblaz oned +dr ons +confront ations +cell ino +bar aboo +. ": +ìłľìĿ´ íĻ +vibe fm +vi krant +tu tt +torch bearer +t ve +sw elled +smo kie +skate boarders +shan ties +sean astin +paw ty +ow ay +nis sen +neuro imaging +music lover +mor ais +mk v +mi kan +mc v +mar j +maison ette +lt p +kn x +kagu ya +james roday +histor io +herni ated +go tg +fur ore +disser tations +char la +bureau crat +box score +bh cosmetics +all on +" } +! ': +ðŁĶ Į +zam brano +ymm fire +witch hunt +whoo pee +whang arei +w pf +vs was +tra ppers +tom mi +tiger nation +tere za +tab by +stevie wonder +si uc +sen si +prilly bie +pp fa +poindex ter +pnas news +pern icious +par men +oba id +mun ich +mu di +morpur go +monc ton +mi red +licen see +kis sables +karansingh grover +josef newgarden +j ke +gif ts +gad sby +fle shy +false flag +f online +del ac +cim goi +chut z +cattle ya +burn notice +bulldog pride +b dx +ar ula +ar q +ar da +anwar ibrahim +^ ____ +ðĿIJ¢ ðĿIJ +ìĿ´ ëĭ¬ìĿĺ +âĻ ł +âĢ Į +áµ IJ +~ . +wood sman +w sav +vel t +u dd +thegame awards +solo ing +schlumber ger +rox ana +quaker town +qu eda +pan tan +oz awa +ner vy +mik ita +mi jas +lang lois +kare lia +hutch ence +empire magazine +emeraude toubia +el ad +bott oming +aven u +arm strong +and ing +and az +ðŁij¼ ðŁı¼ +ãĤ ļ +à° Ł +zucker man +y th +wolve sup +winsor andnewton +w ati +so ane +sin ensis +ri les +re a +ra wr +pub schools +por o +ostric hes +ok onom +oceano graphic +oc cer +o oms +novo sti +ni dh +mcmur try +make re +let tieri +kur upt +kil winning +imp ure +ha jar +gun ya +git lab +g shock +edo ardo +dam p +chri sle +cause were +catahou la +car pus +c pe +bull horn +beck erman +bath es +arto flegends +amandak nox +ad ur +aber avon +ç ³ +yu cat +thorpe park +this way +tam p +tai ka +shu ga +s bag +r ving +pre forming +of amily +oak leigh +no fly +ne ven +mu ld +minim ising +lea key +law es +ky w +kok kin +kar nak +k lg +jessic ani +hel ms +har ini +har ika +gur preet +gun rights +grapp ler +eil idh +e one +con oc +casser oles +carol kirkwood +bost wick +bor laug +big boy +bell ing +armb ands +alamo gordo +!!! : +ðŁĩ¸ðŁĩ ® +íĺ ķ +was sail +ty vm +tu la +ten uous +su pr +str oup +soc sci +save timeless +sau ti +ri al +re positioning +re dra +rd pro +ogun quit +ober hausen +nas p +mtv awards +mo fa +men shockey +me to +mar cha +mar ah +lyric ism +ke qiang +iy ya +high key +green ies +grange mouth +geology page +f hl +de march +conver tibles +bron chi +bilas pur +az rael +ann ular +an jana +ambi ent +albu feira +ìĭĿ ìĬ¤ +w df +viter bo +tesser act +te ad +subsidi ze +spring fest +safe house +rich y +re em +pleasee e +pin ko +pen na +pe cha +often times +nuev as +ne id +n ary +n andy +mi sprint +man illa +lav atory +lakme fashionwk +la sci +jy p +it ai +international isation +induc ting +gom era +george strait +gang tok +eri g +enz ymatic +dog mom +di ppers +c ti +break beat +beau teous +bal eares +arch on +appalachi an +ak is +*-- * +âĿ Ľ +whit ep +whatsthe difference +wel ty +tipperary gaa +the boyz +tere sted +svit olina +suppor tw +sou ped +se ann +red hair +pino chet +oxi des +o dc +na hi +musli mb +mein l +mcman aman +mary borough +manish paul +left behind +ko zak +kil mister +kamp en +ing don +ic cb +hoar ders +goss age +flu min +fel is +ebb w +del fin +columbu screwsc +bum bum +b ents +ac larke +ðŁĺĭ # +ymoun i +water borne +walk den +va is +umb ridge +traste vere +they come +thei et +ten zing +ta as +scriptw riter +sam ra +rah way +psycho billy +police dept +pi pped +nc cu +motor trend +me shed +margre the +mar wah +ma tha +ly re +linna eus +libre office +ke vents +ir raw +ianu ragthakur +i vert +he ren +han well +ham ps +guer nica +go td +emelis ande +du ddy +dan ae +cr ampton +brian eno +blu t +auto immunity +ðŁĮ´ ðŁĮ´ +visit oslo +sain thood +sa chi +ru mmer +roblo we +prohib ited +pre ach +pepit o +oun tains +on dre +ob ello +maz in +mal ka +lead som +l orie +kohin oor +jhon ny +jac i +ir b +ip bes +hyperten sive +hu ri +glow y +fifam obile +fer dow +f gf +epic entre +di ao +del man +cc bc +bul mers +body armor +bett any +b bi +ay sha +apic ture +am all +yo shino +worldof dance +work tops +vic olu +vicolu dovico +ver adio +toy box +to se +roe hampton +ro ymouni +ren nial +rath mines +ram al +photo realistic +pawn shop +p dd +mo styn +micro electronics +lef th +kiz una +itec ture +hu ddling +gre nadine +giuse pp +fa khar +do tt +do decan +diffu sed +debor a +co zza +cho kers +childri ghts +brom ley +blue andgold +bh of +barbra streisand +austin carlile +ar la +amoun ting +alo ves +al vis +uku leles +susan ne +stun ning +stok ely +solar system +shear ling +se ge +sarang hae +sa kh +rv v +run around +roh de +receptac le +reas signed +quadrang le +q om +pur na +otter box +mck it +mary lin +leigh centurions +lee du +le xx +la pak +king kong +kil more +katvond beauty +he ta +grave digger +giving tuesday +enz ies +caul ker +c vp +borrow dale +ar ket +an esthetic +âĻ¥ âĺº +xzi bit +wol laton +wee se +uy uni +turk ington +tar kin +su rest +su darshan +ste ach +smoo t +smo tor +sa wh +rising stars +ripp le +rescin ded +re ak +rale y +r alu +plac ings +or vieto +ky ne +ko ren +k ure +i spy +hon neur +fire watch +es sec +du rai +deadliest catch +datasci entists +credit card +cox swain +corner backs +colin firth +bio tech +barilo che +ave e +ac ep +x japan +wr dw +world heritageday +wis sam +wheat sheaf +water marked +viver ito +town end +thro at +thom tillis +st lv +spu rious +sig ned +sar c +s who +remem br +press ings +oss ington +newsma ker +nar ang +liberal aus +li ffe +kal as +gre ati +ga jan +functional ities +fi ord +esper anto +edu tainment +der rida +dat t +com pare +co terie +chakravar thy +car vin +cap com +cam borne +bode gas +ben no +anir udd +an ted +al ula +al liston +zz top +yi annopoulos +wheres the +wee vils +watch the +visitgreece gr +tru ro +tram iner +thi stles +thames mead +tejasswi prakash +super b +sumit omo +stru st +shi vin +sh v +señ orita +sd schools +ro or +pom elo +partner ship +om ata +nad ja +motor boat +mi guna +maun gan +le land +joke ster +flat out +finish strong +dami ani +cru gby +con serv +chang elog +castle town +aw r +austri an +amotor show +afern andes +ach ei +ðŁĮ ij +vic pro +transduc er +tr ico +ti ro +tennews qld +talle res +sam aha +salt marsh +rou ble +rie woldt +regular ity +re dribb +raven scroft +public radio +nwo su +nodu les +mark hoppus +kur sk +im practical +harrison ford +dugg al +drau ght +col aba +boo kie +blu ffing +bis ch +bier mann +beatrix potter +be wick +basil isk +ashwin ravi +ar isa +alz association +all dog +al pen +abnormal ity +aber uni +tugg lenation +tu te +tranmere rovers +thermom eters +stin iog +ss oci +rosel yn +ra sul +pho be +pan aji +orange y +mango steen +kun ing +ke im +kady rov +jupil er +iti zen +industri als +ho pps +gre ca +gal ina +gaf fa +fon so +fast ened +fari bault +f mm +comment ate +cli matology +ci g +ba sten +austri ans +al bo +adi zero +ac en +!!!! !" +ðŁijİ ðŁijİ +ðŁĮ ĥ +ลาภĶ +wis sa +what would +wer ther +was son +vig ils +ud dy +tt as +tri alled +st annes +slike us +sli mer +sal ot +relin quish +rap star +ram zi +pas ch +ou chy +oc l +ngoron goro +new ye +michael rosen +meadow bank +levi ed +lea side +kr zy +ke uchel +j ita +ho sta +hay dn +han on +gri stle +girls night +evalu ator +ec afe +dylan n +did that +ci pe +chefjose andres +bud va +bu e +be hati +bailee madison +au rel +asynchron ous +af casiancup +ðŁIJ Ĥ +ðŁİīðŁİĬ ðŁİī +ãģ Ļ +à³ ĩ +Å Ħ +zen trum +wil len +water ston +vivo azzurro +universal studios +transplan ting +top coat +ta kata +she mar +sen do +sam paio +r dt +port moody +oneok rock +ny ah +nouvelle photode +nor mie +nan u +mic rons +le fevre +key note +kah neman +just a +jung frau +interro gating +glen brook +em is +ed ger +dista steful +dictionary com +de anne +dark ness +d ago +cé zanne +craft manship +cor des +chil lax +cash app +card stock +blur r +bb src +ayut thaya +arkham knight +acol lection +ac cia +ab ass +; __ +yard birds +w ice +un assisted +the how +t km +sud bury +see it +sadhguru jv +pon tar +pol son +per fusion +palak kad +nor iega +non stick +new supdate +nature news +mac ale +kin ser +julianne hough +joseph muscat +j pc +instac art +ilike italy +i dos +fri zzle +form alities +eu fau +ep isd +chap ati +bul on +bu ya +bla den +ben n +ar ita +appar itions +ant in +abed in +z ot +verdic ts +ta ffer +solan ki +snow making +sle ight +sch uk +rubber ized +rosen borg +rigi dity +ren al +rem pel +ray ama +pu ls +past is +mel man +mc dougal +marcas ite +maha vir +luxury home +li or +laphro aig +la familia +l ø +kw skenya +ki baki +kar mic +ju e +jeann ine +ii y +houston strong +hali l +gyo za +go tom +glu g +ghe gan +fric tionless +fic t +engi klan +end ales +dr jd +cro ome +cor ing +carni vals +caly x +caity lotz +bush ra +bladder cancer +bee hives +bang yongguk +ðŁĺĤðŁĺĤ ðŁijĮ +è¥ ¿ +ãĥĪ ãĤ +öster reich +xbox e +vil anova +tu valu +ti ge +thre shing +thomas sanders +smoo thie +sec tioned +schim mel +sam wise +pal en +p tb +not forgotten +mcco vey +lam peter +john ston +io anni +headhunter z +hat su +h wan +gian forte +ger wig +fin ks +elo u +eli ff +dyspra xia +do heny +dil se +chancell ors +ch ford +capital official +beat o +ba sit +ati enza +!!!! # +ìļ Ķ +us k +up stage +ulti maker +tower bridge +telly chakkar +squig gle +se iler +se bab +scape go +sau x +re manufactured +probosc is +poke dex +om et +officiald gispr +oc bc +no ya +n ort +mili eu +levit an +lehigh valley +hyper activity +hoi berg +gre ed +expend able +endoc annab +dol ci +do sha +devilmay cry +deschu tes +con ta +coc cin +ceremon iously +avi anca +architecture lovers +apar icio +al ga +] ( +ठĪ +á l +yellowston enps +x am +vadi velu +termin ates +spac emen +se gui +schitt screek +sc media +rel ent +pu esto +pic kerel +pi vx +n assim +kh c +jack als +ishi guro +isab ella +indi at +i mei +goo dd +gol maal +fashion news +face it +dor cas +cun y +cli brary +cent rep +cath mckenna +catchit kansas +black wall +basker ville +bahawal pur +ar bol +am har +acom pany +unear ned +transvest ite +tor ching +than ku +sul t +sine stro +sho ppin +sel le +power tv +polyphen ols +pel argon +pax ton +over achiever +modi fier +michi ko +me el +mcmur do +maungan ui +masch ine +mas l +li mate +ker mode +k mm +jessicani gri +inver cargill +indi es +in cep +iam r +hyung sik +hav arti +gwin nett +good game +gar and +fan bases +ef ans +ed w +dr pepper +demon stra +con migo +cioc col +cere bro +categori ze +both well +barre to +bar ia +bal rog +ari anne +anal ima +abat toir +aaas mtg +ðŁĩ¶ ðŁĩ¦ +ìĿ´ëĭ¬ìĿĺ ìĨĮëħĢ +м ак +zwir ner +westph alia +we play +vi mto +ti que +st out +st james +sp ps +siri sh +shav ers +sha we +ry ker +retro fitting +re boot +pre en +phalan x +oom ba +nas scom +moment when +men ingo +mas ked +ko wa +ke ets +jat engiklan +ir ked +intern alized +do vic +dei dre +clar ins +cis sy +c no +buck man +bt p +bin aryo +beel ze +bapti sts +auto repair +att oo +apho to +" ->: +âĿ¤ï¸ı ) +west life +vince mcmahon +v ass +us afric +twin ings +toast master +tkach uk +the blogger +shahid masoo +sen cha +savethe crew +sat night +river hawks +revi ver +rad har +poly gons +pir ated +organic food +nigh to +mur rum +mil house +mark ell +man v +lac son +ke uka +k bis +infe cts +huw stephens +e tho +counten ance +clar is +bo sso +bo sc +bit defender +bio char +and am +american legion +ab l +aaaaaaaa a +ðŁİ¶ ðŁİ¤ +ys gol +xal apa +west f +we want +trending now +ton college +the word +teppany aki +slap shot +senior year +sam m +ridlr mum +repatri ated +reic hert +phosp holi +op ines +mm en +mau ney +mangesh kar +love this +lo do +ka sha +jar ry +in operable +hol lie +fran ko +fa kih +en schede +ellen pompeo +du kakis +co ste +cl é +christ in +ant as +annab ella +alberto contador +ak io +y tretweets +wiganwarrior srl +vi render +together weare +sw abs +so a +sil sil +reticul ated +publi que +pool er +ph un +op b +okonom iyaki +offu tt +nov ara +mockingjay part +miami university +metall ics +le ade +kra bby +ke em +jos lin +ja ha +hom erton +great food +gla zier +gin ni +fram lingham +den nen +conferen cia +city lab +cari produk +bol asie +blue sky +app am +: , +ðŁķ Ķ +vainglor y +urband ale +urban photography +unconsci onable +un kle +ubi qu +twee tur +tro ppo +sunday blogshare +stewartha asr +stewarthaasr cng +ste ams +st joseph +slo ane +shi agenocide +say e +sat chat +s ze +red car +r nz +qu ill +paris fashionweek +michel s +malme sbury +loch leven +incre ible +ima gen +home remedies +he fe +hb f +fort collins +einf ach +ea i +dou jinshi +disin fect +cr antz +conspic uously +cole sprouse +chri scuomo +chin mayi +car pa +bread board +brass ard +blu sh +arne duncan +amu ck +aku prema +acet yl +ab dl +ðŁĺ¨ ðŁĺ¨ +z inga +yasi r +tup date +top ham +tirun elveli +the muppets +sukh dev +spani els +sof theworld +sn art +shi bata +shel drake +sa hb +ro dy +park ar +our moment +nau seating +nal c +magnific o +kopp ar +kin ane +inthe dark +i home +hon nold +he cho +gla vine +game over +fr ye +fitz simons +ferdow si +dal ÃŃ +d ming +char mingly +buc cle +bo akye +barang aroo +armi sen +ar cola +ap cc +? !! +ðŁİ ² +wool sey +wh b +vene cia +ti aa +team lh +str fc +sports women +sph illips +shay k +shan tel +shahidmasoo ddr +sepul tura +sc ally +sarcast ically +s run +ry oma +ru is +r bt +port es +mock umentary +mill creek +mik ki +kincard ine +itz ky +ir mo +ip tl +imbu ed +hot plate +gow dy +facil ity +epic ness +cu ffing +cohe rently +co akley +card well +bo ek +black list +bit main +b ently +b app +as su +ah shotel +adventure sin +ðŁıĥ ðŁĴ¨ +楽 天 +à¹ĢภĶ +á r +wi elds +tiff anys +tel mo +te me +swordar tonline +sk p +salt fish +ru dd +ro xx +recei vable +question naires +pros thodon +pre torius +personal ity +ok oro +occu pier +ns ffunded +mi sto +meathe ad +life house +kelly and +helly shah +grind stone +free kick +e be +didger idoo +cro tone +clan cashire +bu lu +bri ain +art ans +ab aco +ðŁ¤Ĺ âĿ¤ï¸ı +vec tra +union jworld +triple tt +tmr rw +ther ud +ten ses +te co +t god +sb dc +ryan higa +ritt z +paint september +nax als +mul vey +mu subi +mon the +mn t +mb ab +man in +lor ne +ling fieldpark +lans down +kuro saki +ke tv +hyste rectomy +har dees +gray skull +exam iner +es ack +digital banking +cric kle +collegi um +castle maine +can te +batt a +ar ds +ain bow +ว ย +wr ack +vin atieri +ti gue +t bird +sor ties +sla yer +sha hb +se sse +ro dd +ray y +nap ak +money team +mb ah +materi alized +lich field +lam er +ky alami +kut z +jo ol +j hoo +ir sc +en ron +down patrick +dispen sed +cor um +chateau briand +bnw photography +b wi +b gn +at war +ago on +aggie football +ðŁİ¬ ðŁİ¥ +ãĤ ģ +wy ke +w dr +un break +uk f +tru di +the gac +subsi dence +sports bar +sno tty +seque ira +sen ca +ol fo +nu bes +nie mann +mouse hole +mc queen +martinluther king +mand la +lal onde +jura do +jan in +it sour +intre pid +haemo philia +geo survey +fle xo +end homelessness +dom enic +clau dia +chan e +ce pa +ahu ff +ach ingly +âĢ¢ @ +yo gal +women power +wild fire +tili kum +tegan andsara +surro gates +sun ning +stig mas +ss op +sk unk +si son +shari fs +san kofa +repu gnant +represent ational +priyankag andhi +pric eless +pre phoops +practi ses +pan ahi +ob lak +now y +n do +metro centre +matt bevin +li thia +keele university +jin ny +ic orn +herald scotland +hef fron +he wn +happybirthday tome +gl bt +gear box +g prs +g ks +fol gers +far ou +epen ny +dev ent +derwent water +de val +david hasselhoff +ct m +chief tain +ch ten +caton sville +back ground +arjun kapoor +and more +all ways +admi rably +ac um +?! ?" +ðŁĻĬ ðŁĻĪ +ðŁĺĺ âĿ¤ï¸ı +ðŁĮŁ ⾨ +ðŁĮĻ ⾨ +wax work +un am +u sca +tril by +trapad rive +timber wolf +tam ba +sla vin +si ana +san ka +roe derer +random house +pro filer +print shop +perse cute +pe up +pe k +pat atas +nuclear bla +news queensland +nep tune +nar can +miller coors +loud mouth +knut son +kha der +jharris jayaraj +gender paygap +gate ch +fu dge +ev ar +d tech +care llo +bre cker +bo ingo +aspir ated +ag el +ìłķ ìļ©íĻĶ +vod kas +united airlines +under carriage +un played +un interested +tou pee +theli ghts +tb icycles +su go +ste warts +so ja +rom sey +pil ato +pe kinge +mra dam +mad han +keswick bootco +jazz fm +j q +inconspic uous +hi w +gro u +glit ched +ger mani +follow cii +fat t +dmu leicester +demarch elier +daily drawing +cyano type +cy mru +concur so +clive den +braam fontein +bl is +bir k +barri e +apo l +ante aters +am ri +ðŁĺ° ðŁĺ° +ìĬĪíį¼ì£¼ëĭĪ ìĸ´ +ب ÙĪ +yo pen +wur zel +war locks +ver um +thelast ship +ss nyder +sports writer +so bew +relati vism +quen cher +pan eled +nr p +nau gatuck +national comingoutday +nat atorium +make th +leeu warden +inn ards +gur dy +friends giving +free will +euror ack +creoso te +bottle men +bb clancashire +bas ques +bar well +ast bury +ar ja +ani i +ðŁĺĤ ðŁĺ³ +ðŁĩª ðŁĩª +íĻ © +undate able +thedivision game +syn cs +susann areid +stan ky +squad rons +spot ter +son equ +saras wat +ro ba +pandor as +p nl +mf n +llo ret +k allis +j se +i of +hym nal +gol u +g dynia +f bd +do fe +dj set +cyberne tic +bts festa +biglo tter +bb vacom +ball parks +bab el +after movie +zig go +whatson stage +wauwat osa +tur an +the vampirediaries +tele kom +su tera +scor k +satell ite +sar zo +sange eta +ready made +p town +only nikil +nen okkad +n ooooooo +michen er +leap fro +kwa ito +krau trock +kne ading +kal y +in icial +imag er +haid ar +guy ton +go blue +gang ster +fightfor wynonna +end rons +cott ingham +central ised +cau stralia +c sforall +bol la +bibl ically +ber ch +att oos +atri ot +ation ism +ashmo lean +al tura +ak lein +ad vil +ðŁļ § +௠Ģ +yan ked +wood shed +war man +vi vica +twee de +tg n +sic ily +rust ica +rox ane +rh shampton +pu dding +ol of +o ji +nas as +n tb +lloyd kaufman +kan ha +jewel ery +ii is +ig inals +grun er +glo bus +frey ja +founder sday +elgato gaming +do ting +del f +colon cancer +cn z +charlo ttes +caly pse +by passed +breath alyzer +bk n +bis d +big pharma +az kals +and as +all america +aaron goodwin +ðŁįĶ ðŁįŁ +yad kin +work mate +wab ash +vit ter +vil am +v xr +tom mcfly +sv end +surgical strike +staur ants +sh elia +rspb scotland +responsi ve +re ward +racec ars +primary rocks +pie tra +pay loads +p gf +om pidou +o lie +ne as +mel ford +lurk force +konta kt +kin nock +jor den +inf lux +im partiality +hau han +goo t +gal avant +ga ir +fe as +fay sal +fallow field +du bcity +d squared +cross dressing +cit rate +cere zo +carlil loyd +campbel town +bu hl +boo ka +black stock +bil oba +bec kie +b hullar +analge sic +ðŁĻıðŁı» âĿ¤ï¸ı +ðŁĸĮ ï¸ı +» » +yo an +ya seen +vi do +to stit +swal d +son ando +savi ours +re shared +nation ale +man co +love scotland +leg ally +had da +gist mania +full ers +day lesford +colorectal surgery +cine macon +blenheim palace +bal ach +ack bar +abram ovic +[ ðŁĵ¸: +ðŁĵĸ : +zeemusic company +with row +wh yyy +vas con +u ec +st ours +so arin +sm fc +shaleen malhotra +schö n +s global +ride ordie +resident advisor +ray gun +ra bility +pugets ound +out sold +onit sha +o iselle +n scs +myle ene +murder mystery +muba sher +mill saps +math s +la bu +ko el +kathy griffin +kam aru +jais wal +ide apad +hyatt sville +hal len +gun d +g wan +fu ma +education govuk +dress like +dh hs +curry wurst +choic etv +bu kuro +britt ana +brin ker +bandof brothers +ball player +alo pez +ภľ +yes network +wool dridge +w gc +vend redi +tre ze +tim farron +ti enda +tark anian +pender grass +nikol as +neighbor hood +nal anda +mull ens +mis elizajane +mf g +merito cracy +mccar tan +ki ren +justfor fun +jame so +il igan +go etze +fe ck +estate agent +ern ps +ec centricity +dv bbs +dragon age +com score +chris mas +can one +bul le +board ing +beren son +beagle freedom +ap tos +aleu tian +aldubbig boyz +ëħ ¸ +ãĥ³ ãĤ¿ +âı ³ +w ira +ven try +vas u +ul ita +ty le +stu ddard +street ball +space dotcom +s icist +pe to +pas sively +pap as +os muertos +objec t +nu pur +ner lens +nd b +n sk +mm vas +min da +mie shat +manag ment +mad smikkelsen +ma uri +lu hrmann +legal aid +le ander +kro es +k tu +jor daan +isai as +ib jjf +grass roots +glo bos +g ch +fr sa +fis cher +el mo +eeee eeeee +dian ap +cheap flights +bill o +art center +ab idal +ðŁĶ ² +wen k +thumb print +third man +therud ingroup +the players +stat eroom +sla st +shire en +sb snews +sambal pur +red ne +red head +pur ana +pou ille +pimper nel +par dy +nobelpeace prize +mash had +make dcli +ko erner +jen ko +jan u +hurtigru ten +how z +haric ot +g fm +for n +far yal +ephe mer +ed dies +do lore +de kha +dar ci +cl inte +chil cott +card holder +bw ana +bud u +bo ven +bed ard +be do +age ge +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬðŁĺĬ +ðŁijij ⾨ +work around +walk offame +uni mas +the jeremy +th picture +symboli se +stor a +speciale ffect +sn hr +sil v +sel ve +scriptw riting +r su +on iz +male y +li be +last word +guanac aste +gen entech +garden of +franco tv +formal wear +fore va +fel t +ez official +d tu +cor poreal +car ice +best practices +barri sters +barbarap alvin +avi des +attend ances +anun cia +am per +ðŁ§ ł +ì£ ¼ +z and +wi eg +tw ings +tox opla +ton sill +thisise gypt +ter ras +shi ekh +scot stoun +scol lege +royal ton +rin du +pon ga +over shadow +nus antara +nh tsa +milli seconds +memor ization +mari ani +makedcli sten +kis smar +infidel s +fi ano +envel op +defe ctor +dar ts +bt n +brock man +bran dishing +blow outs +bless thefall +big cat +b vc +agover nance +add led +ac ou +îĮ ¨ +y mp +x biz +tryan uary +stan o +spl inter +sit ra +seven dust +s gh +rumm enig +rummenig ge +ri elly +pokemongo app +pla ines +ph b +na jam +mira bella +mi raj +li pe +ld lc +kat ah +janef onda +j fs +im ou +gol lanc +fragran ce +fibaw c +f so +dig as +corn walluk +congr atu +chat tering +cb so +cat fishing +bou ghton +amu thu +alli simpson +af v +ðŁļ ĵ +ðŁĺģ ðŁĺĺ +ìķĦ ìĿ´ +yil diz +yak is +y combinator +xi ons +wil kos +wb k +tor ro +thet dn +spir alling +sen ia +scur vy +sam an +quik trip +pasqual ino +pad wa +om bs +millwall fc +mer tz +marse illes +mad der +lunch time +lu th +li gero +justi fiably +j oop +i mar +george hw +for her +eli vely +ele on +e gress +dis miss +dis engaged +de ighton +coo sa +co iling +cancer care +briar cliff +bol u +boe hringer +b ft +ay lin +are ddy +app ian +am dev +am aa +alexander wang +ðŁı ¸ +y tl +wonder boy +waitress musical +w us +virtual box +v atten +ucon nwbb +thing syou +super fine +sle ater +sig mas +sen thil +schul ich +sar ia +rapi do +ra donc +ra bo +pro pyl +polly anna +plan k +martin amcbride +kip nis +kan baru +hg tv +grave yard +glutin ous +gi pson +fre dric +fou rie +eng es +e em +dix ons +compac tor +com mode +cold fusion +ce fn +brink man +b gi +an um +an issa +yaho onews +wood ward +war da +vis or +vand alizing +v ati +un recognized +tiru mala +tac king +t mi +spe te +shr ill +schwe ppes +sach se +rho dri +photography wx +per tains +p lier +new berg +mor tad +moo die +maw lid +madein england +liberty ville +kop a +jer od +itu c +go leman +franç aise +food tank +fab inho +ep y +ep le +efflu ent +e wald +de wey +cyanobac teria +compos iting +bicent enary +bergha in +au i +african ism +* @ +zac goldsmith +wood grain +under tow +tonsill itis +thorn tons +tar aw +sp harmacy +sing lets +sho gun +sa unas +ron wyden +ris ch +po de +pekinge se +pal le +on aldo +official bwfc +nic h +nak u +modi se +mi sheard +mean green +lis bon +lig gett +kate esack +jes olo +james francotv +inter red +inte st +horse shoe +galatasar ay +er ol +ener al +com bust +chesterfield fc +break aleg +bra sco +bird lovers +be ed +bar an +bann u +ar mature +ðŁĩ¬ðŁĩ ³ +åĴ Į +âĤ¬ ) +way man +w ukong +us natarchives +trans act +top tip +su varna +sty ne +star wood +stam mer +shol lywood +p illing +overs lept +ni gra +morgen stern +mesen chymal +lagun itas +kin ah +ke ik +k usa +j are +irish music +ib mb +hr derby +hou v +glen avon +fire pit +executive goth +diet iti +di rim +di pole +craig y +cor vid +by utv +biolumin escent +bett endorf +bell amy +are ly +albace te +alab amas +ac cel +` *) +ðŁĹ£ ðŁĹ£ðŁĹ£ +âľĬ ðŁı¿ +à® ¯ +y oni +work room +woon socket +ward bound +wak u +vo ree +vest ments +tro oms +tr at +til t +tham bi +t ps +swir ly +sti mulator +pr and +mis ss +milli an +live to +lasse ter +la am +jon nie +jin day +jay aram +in ui +hinch town +hal ving +gu lo +ex ton +distor ting +death wish +chri sm +ce judo +bo ker +be ens +barn find +baj rangi +agu ayo +ad waj +ãĤ¤ ãĥ³ +é ireann +wi vind +vas anth +tm x +sympathi zers +survi ving +spo sa +span os +sece de +season finale +ro si +rep els +reli quary +real life +radhi kam +pre flight +portra yals +park as +oshi om +om art +nothing but +nicol a +mir ka +mieshat ate +me as +manufac tur +man and +ma hil +l bt +kab ar +j ns +fla s +dune din +down play +de dede +de cade +creation ists +con notation +community service +college humor +co stars +budge ted +bu colic +broo king +bo sma +bo gues +bli stered +beetro ots +an aa +americas milhist +ภĸ +wonder struck +war face +tran sis +taka o +stock stowatch +sto tts +soul ard +sk han +ro ache +rho do +re as +ra pala +pu tted +pollu x +perme ability +mul timeter +mar sha +mac u +lu tter +lor ia +lobo tomy +leon hard +keith ellison +ke diri +jon snow +jew ry +jen as +hay ward +ham ari +gher ini +ey n +dj inn +displ acing +diete tic +di ggins +dal vin +cham bliss +body boarding +big man +batt en +baton relay +anz as +za ara +yogali fe +we ise +vi pul +veuve clicquot +v uk +tom ake +sv ant +spot lighted +sky r +shil paf +she pperton +shape shifter +shal war +sen to +sam martino +s ops +roun del +pharmac ological +odeon cinemas +monic a +me anders +mak elife +kom bat +ik ar +ich mann +hump ing +hu ddleston +gul marg +gha ag +gel ang +for bidding +f np +electroly sis +een adu +dra k +ches i +cen giz +buzz saw +bo hin +be cs +ave c +amas sing +all iter +air frame +accu ms +ðŁıĢ # +zil ker +z eng +wr ks +way de +us q +thom s +thenation all +take care +sriti jha +squ aw +salonedel mobile +robust ness +quen ched +quan tic +propor tionate +pen icu +patoran king +pal y +pak ora +p min +oz one +l wr +l fk +kind les +k acy +incess antly +il ene +henry ford +hay ao +ha sa +group ings +fé lix +fen ner +es ade +eri shere +elum elu +eg ghead +e mini +du kan +dragon boat +democracy now +decre pit +de akins +dal o +cy mra +contri ved +con ny +cfl gameday +calle baut +bri ous +bal aji +as agar +art sin +a oba +ģ ) +ðŁijĭ ðŁı¼ +̶̲̥ Ìħ +zo es +wh ic +wa stel +vend ome +under cover +turn tup +tri on +ti mid +tam o +tag ine +squ el +son ee +qua shed +pp c +pi ku +overestim ate +out pace +orang enation +mon tour +mcclar non +madhu r +ma bo +love quotes +libre tto +lan ham +just add +inter viewers +hi iii +gsm np +galli um +ga en +fresh ening +dre i +colorectal cancer +bo ag +bar bel +bak al +' ', +ðŁĴķ ðŁĴŀ +year long +wn it +vodafone in +virgin ie +vi ol +tran ge +the fog +tab lon +sen dak +ren ames +re vin +rdpro cordillera +pat rolled +pat ri +pan is +pal ates +ow u +movi ec +men des +marty scurll +mana watu +lsh tm +lord taylor +koso va +ka ise +i back +hill y +h sv +global surgery +fo gel +flood water +enew york +d gaf +common er +code masters +c ty +boudre aux +ba ily +aren d +ak osi +a omg +èī º +Ã Ĺ +wom ble +with a +w tvd +votel ittle +vancouver pd +usu k +tt is +tat ars +tal at +sc ath +sandwich day +roo ghaag +pre ludes +ponson by +pho tor +p falz +owat onna +national post +nab oo +n gan +mtvawards star +lo qu +krz ysz +kind t +kin ki +kateesack hoff +join in +it sad +i ws +fle ec +fearthe turtle +f sharp +ele u +e jac +dodecan ese +do better +ch z +calli grapher +c aci +bend able +aco a +ðŁĺĵ ðŁĺĵ +zz ar +w ga +uro logist +talis ker +stra ights +sin igang +si biu +shi mo +sham us +royal welsh +row son +rose ate +rambl ers +r cts +poo ley +pett iness +overex posed +moistu rising +meh ra +marce au +marac aibo +lo omed +lo las +lets fly +la em +kick apoo +in dol +hen ne +ham tram +fort nums +fir hill +de acs +ch w +cas sian +brû lée +beyond meat +bald ricks +asdf gh +aqu in +andali o +abo l +ðŁĨ ĵ +zing ano +yaw l +vs no +voc able +vill ano +v dp +twi c +sukk ur +ss lazio +sh war +rudi mentary +rab anne +pur die +pro w +po tosi +paley center +no len +ne vers +mostre que +moor side +mill wood +ly sander +li o +ki jiji +k off +jack whitehall +ing au +heart beat +head ington +he afy +hal pert +gu ano +gr f +drone stagram +drjd rooghaag +diamond platnumz +cruci ally +col ly +channel stv +black money +bint i +ber rios +avo ir +au pdates +asi as +as sar +zapp ing +visit sunshinecoast +vic ton +vi katan +tx state +the bay +that girl +team pixel +su perfe +sour a +sli gor +si gur +shepher ds +schaden freude +red cat +ram bunc +pou let +par abel +oscill ations +nouvellephotode profil +national superheroday +merchan dis +m Ã¥ +life on +lem ke +lang it +jami ele +ig na +i frc +hu st +har rod +esp ou +devon ian +determin ant +de votes +d hal +confeder ate +chino is +book sare +bid dul +bard bits +apple pay +anna beth +ÃŁ e +w la +ther it +super leggera +sign aled +seme do +ri ba +retr ato +or son +newsc aster +negli gible +n ú +mot tram +mm flint +mathe us +mal co +gran ma +fox footy +finalfantasy xv +fallo pian +ex tempor +efl cup +disp o +dat agovernance +cor g +cf h +cart wheels +beer cat +band hav +bac s +autisma warenessday +au ke +ali yu +âĻ» ï¸ı +à° Ĺ +zoo ey +we iter +tel os +tatsu ya +swag ged +stead icam +sta ithes +smi ds +scan berra +sau lt +po ppe +percol ator +pad re +on netflix +neta porter +l antz +ku mara +ig nomin +hard copy +gold medal +ga o +ford foundation +food pics +evangeli sts +empower women +d hq +cran ey +cla ys +chief ly +cat ton +cas sano +bou chon +bos elli +boni ver +beg ru +ar tw +web pack +vivi enne +var usar +u texas +then u +the ac +tarantu las +su sy +startu paus +spar ky +s vo +retar dation +ren ergy +pur wo +postu ral +over state +mor te +mj g +me ka +master card +lovewhereyou work +loven ature +li ghty +lasc elles +ing leton +ignific ant +ho ists +ers life +cu ti +choice music +chee ze +by b +ber y +ai dy +aberne thy +ðŁĺı ðŁĺĺ +ðŁij ´ +ðŁĩ§ðŁĩ ¬ +yu sha +xer ocon +wor snop +u ba +tit ch +tion able +the fox +smo k +sen so +rohit shetty +pedic ab +pardonmy take +pa katan +pa avo +om bia +mol yb +mcgur k +mat tes +ma rella +liber sek +la palma +k latt +k bs +k alla +ji de +jablon ski +indie sel +in in +her re +head banger +ha koo +g valan +en ner +dir ks +de facing +cop as +co agulation +civit as +cath ie +braw ls +br annigan +bengal uru +bas que +band saw +and d +an chez +ac anthus +ab cac +Ø Į +white christmas +weas els +up staged +unshak able +theoffici ala +sugar hill +style book +stubborn ly +sla svegas +skin en +se vi +random house +rand oms +ra vil +r bjb +q in +persi ja +parti dos +ott bike +music matters +mol ata +mohsin khan +mike lee +ku st +kru ll +k aming +hol dthe +h ance +girl sday +em sa +eli ghting +dren ching +dread lock +dow chemical +decrimin alize +decre ed +daybreak hitz +d fat +cou ric +bha sh +aw ale +ap uto +wwe cesaro +wor sham +woo oow +winnipe saukee +whopp ers +toon town +ther olling +the gn +sar to +re sourced +pw ll +mus grove +mu ke +lo sin +llll lll +lek hi +k jrh +k dc +hockey night +hat soff +ha un +dream girl +dinesh dsouza +dend robi +cum nock +charleston shooting +chad well +cer nan +cade aux +ba chan +atten tions +al stro +ac oma +ðŁĺĺ ðŁijĮ +swee test +sport in +sony uk +sobew ff +sam mich +robin lordtaylor +rei shi +q arab +ot dh +mono tonous +me time +la zier +ki ane +kat ri +jencarlo smusic +ingo b +implant able +hope and +gurupur nima +gri fter +gor kha +give hope +g adel +flo res +ever bank +ety mo +esc ada +dream chaser +doo gie +deu lo +da el +d our +core lli +beart ooth +aziz i +au ght +angle terre +amor os +ak ai +; )" +world penguinday +tu d +tor ium +thim ble +th atta +ta her +sp ag +sou red +shape of +sche i +rub ina +rookie blue +ro jos +patoo tie +nutrition ally +mutu amadri +mutuamadri dopen +mirzas ania +mark wright +lar te +koscius ko +kn olls +kit schy +jonathan r +jawa har +hi gha +good by +desi pixer +dd un +dak ima +cicer one +can ty +cair ney +bor chetta +bm alik +bingham ton +bet sey +bernie orbust +ben nys +arm field +anni a +alla hu +aba shed +à° ¡ +yuzuru hanyu +ye en +yar sk +w bt +vic mensa +ver dad +ui dai +tra pani +tim al +sul lied +sooraj pancholi +slo wh +shin su +sen ow +scar fe +sab ella +office design +n le +mubasher lucman +mick i +li bro +juli enne +jo ef +ire lands +ingh our +hierarch ies +hailey baldwin +fri mley +favorit ism +eventi de +dv sa +domhn all +dni pro +digit our +democrati zation +dec tomy +dal awa +cre en +cos me +convey ors +ch apple +car ryo +breck in +bre mer +bell sbrewery +ba aaaa +ari ely +animal health +ê´Ģ 린 +whim per +west fall +tv channel +ton ge +thel oop +tam ir +stre aked +squ ibb +span to +slip stream +quir ke +post impressionism +oper ational +mcin nis +march é +lead ingthe +john w +gu ster +feren tz +er le +dar rah +criminal ise +cre search +cre che +coo der +chap ulte +bottlen ecks +bl é +believein yourself +bel videre +bbcy ork +all ens +al ito +ai zawa +adop tee +⾨⾨ ⾨⾨ +tac a +spray paint +smashing pumpkin +simul ates +se wanee +sac state +re agent +orno car +nesham iny +nar bonne +n ssf +n lu +my bestfriend +mu sto +mari ag +ko ve +kick flip +jor gen +gen tile +fun nest +fr and +fle dge +fa waz +en more +dirt bike +dd ar +culmin ate +comp ound +cityof sydney +bran son +beauti fied +beard less +b ils +b azz +ay une +auton omic +alli on +af cu +. âģł +ðŁ¦ Į +âŀ ķ +âķ ² +âĢ¦ âĢ¦âĢ¦ +yak ut +whatdoyou think +ving ton +truss ville +tra g +te tte +taka hiro +taip ing +share humanity +rc bc +prob ation +pp reciationday +picka xe +ottawac itizen +nu ria +nn nnnn +nicol le +n was +mc nuggets +la zers +klu ivert +j va +gl bl +gat ton +for se +for ia +el speth +dun mow +de coys +dal ziel +cypres shill +cu ticles +bri mmed +break er +boge ys +ber rye +ber kle +au sag +arus sett +ap on +aom ori +ame hta +all tech +abi el +) .# +âij ł +ye gevents +virginia beach +upro ot +triun fo +thisi show +spl iced +smack down +sh appen +sco ffs +rt k +rocket ship +ro ber +re pub +ra as +pre dawn +p cb +om v +mu tya +mo sen +mel k +mckel vie +marac ana +love shot +ki zz +ka ar +joy sticks +j sf +hether ington +ground swell +giar dino +fus illi +freaky friday +famili ar +er v +emphasi sed +dj m +di rac +del ts +de ut +convers es +clo dagh +businessc ards +brown sea +bi ge +battle ford +b be +author confession +acl fest +ãĤ¸ãĥ¥ ãĥ³ +à¸ł าภ+Å Ļ +Ä ĵ +wit i +wheat belt +vi i +vector art +up g +th si +testimonial tuesday +ta kap +symph ony +summer vacation +shaheer sheikh +sh te +rums feld +revel ers +red skin +rebec cauk +rat eliff +pred nis +pablo alboran +p dsb +optic on +mou lin +men don +leib niz +leah rebeccauk +lam ia +kedge ree +ini quity +humidi fiers +gy lfi +gun n +green tech +ger ri +galli frey +gaem gyu +financi alliteracy +fall brook +fa its +ek ta +e ot +dj ent +dece iver +de ade +cor vair +com un +char my +car pool +blo feld +baj pai +att uned +alo evera +aha shi +ãĥĩ ãĤ£ +á ĵ +yas si +y str +wwe asuka +wissa hickon +whe len +wash y +vb schools +tra ba +ta ree +screen time +qu ills +oh v +nath alia +mir wa +magi que +li ppie +lebron james +land or +j lp +iam lakshmirai +hans grohe +gro se +great artwork +gar ish +fu ga +fever ish +fe tterman +fan euil +emph atically +dan um +cu u +cow ling +conco cted +center for +brig nac +baaa ack +amar ch +al shabaab +ach arts +ðŁĶµ âļª +Ë ĭ +vo it +ver min +un readable +tre sor +thi ers +ter is +simul ation +shav n +sha c +rule sof +rock er +rivi ere +r mn +pe dis +pal abra +op acity +nd su +multi level +marin os +lg w +jake quickenden +hos king +g ph +fn q +fle ish +favour ing +fa an +eye lets +ele s +dis assembly +counter measures +chikun gunya +car one +c cl +br anc +bl l +bio gen +asan chez +arch os +alvor d +ali anza +ÑĦоÑĤогÑĢаÑĦ ии +zach ariah +spe ier +so sweet +skid daw +salahu ddin +ri endo +py t +poor na +olaju won +my our +music ality +massaf elipe +mari ano +l ge +kal yn +in site +impeach obama +iaa forg +ha dden +got the +ge malto +fas ano +event planner +en forces +dream act +dit avon +deulo feu +colqu itt +chand ram +break through +b vd +b dd +asat ru +antici patory +annex e +ðŁijij ðŁĴķ +Ø§Ø ¬ +uj ala +u may +tit alia +thenew painting +tel star +stron tium +stir chley +so ti +sky lake +ske e +seduc tively +scrim shaw +scre amo +s ba +ru blev +roo ty +return to +ren ny +pre ez +par tha +oik os +menom inee +mc chicken +lucky girl +lot teries +it sar +inform ally +illusi onists +hir ono +crutch field +craig s +coz art +bulle it +bu tera +brush strokes +at r +at plc +arab e +antam onica +ancou ver +alast air +íļ ¨ +z ma +uni ya +tru k +than javur +sv l +shaf aq +scur ry +ren de +ram trucks +rabi ot +pi dge +nürn berg +megaz ord +m gwv +kra ven +jo bber +ip sa +i fla +hol li +herm ès +hear twood +geo cache +fur i +forzan apoli +forzanapoli sempre +for greatness +fly way +fl d +first weets +domin icans +chaf ing +cf meu +ca xton +botanic garden +ban trophyhunting +af on +a alt +"} ," +Ø ¶ +zz ies +yed lin +wiven hoe +web comic +wahoo wa +vers ace +un banked +tu lear +top gear +the maine +stra vels +st ps +spa int +say brook +rss ur +plat forming +padma shri +old is +o tis +nic co +lafour che +kas ai +jer king +j ba +i flix +hind marsh +heath cliff +g war +fee han +eye of +ex um +dumb ing +duc asse +du cker +col lings +cb sa +canad ensis +basqu ecountry +art adventcalendar +ap un +anil kohli +ai doo +af air +ðŁİĥ ðŁİĥ +îIJ ĺ +ëıĻë°©ìĭł 기 +âĢ¦ âĢ¦. +wi veng +whe ats +we il +uniofe astanglia +tulo witzki +thereal ryanhiga +team tolex +tan o +summer set +start the +sh nikov +ru thin +rnc m +re ssa +r ä +pri vet +prakash raaj +pe sca +part itions +param edic +om ing +neutr al +lit itz +l nd +kool haas +kar cher +inst ate +i severything +gu ma +grou se +gn b +fu p +fu jin +fou ad +flat irons +fit etv +elles se +dre ich +derange ment +da hab +cr ana +contor tionist +chin na +change thegame +cad enza +cac a +brance day +bout in +boul anger +book able +ban ke +az y +amand la +allo way +accu m +ðŁĻĨ ðŁı» +íķ ĺ +wi shaw +wal ley +ur bann +tweet meso +tweetmeso hard +startup weekend +siyah bey +shawnab ner +sc cl +s conference +ru ch +ros ella +red skin +radi ol +pv r +porth leven +pen ciled +pac esetter +mol nar +militar ism +marcel ino +love travel +kr ys +kie hl +k oning +jo p +jin an +incen ti +in ad +h gt +greeting cards +gowdy sc +g mac +g finity +free zers +ex and +eco fashion +diffu sing +database app +chandra sekhar +cc ms +boot camps +bell eri +bar bas +as sou +art databaseapp +ani pals +alph am +aer ith +zo gh +virul ent +ts j +tru x +stru n +sl vind +si by +sch ak +sanc tus +re fundable +r win +player unknown +moo rea +mar burg +lo rescence +lo ofah +kiyo saki +kin c +ket k +j ola +inthe making +inter change +in ver +i if +gru m +grei pel +go chu +glamour ous +gh ua +ele st +confi dent +color fully +cha fee +cait riona +bu rak +braz o +b pf +air n +ack a +ab oot +ðŁĩµðŁĩ ¸ +ìº IJ +æĿ±æĸ¹ ç¥ŀ +à¸ĻภĶ +wind star +web bie +un trustworthy +tand on +sm cc +sambit swaraj +sad h +palin drome +optometri sts +o the +neu feld +ne uk +mp x +me war +knight fall +kindle deals +k na +jer myn +hh shk +guille mot +gad on +fer ne +evans drumheads +disappro ves +cp chat +cin di +agassi z +! ðŁİī +ðŁĴ¨ ðŁĴ¨ +ðŁį·ðŁį· ðŁį· +ãĥģ ãĥ£ +x scape +wilt shire +wi dge +un sanitary +uk biz +uc cess +u hoh +tu areg +top drawer +the go +st oli +sle x +scro tum +sa ja +re wired +pren ton +port more +pa hoa +nazar bayev +nar stie +nann akuprema +na stia +mo issan +melo dramatic +maz dausa +mal pas +ma rea +lin ford +lan ing +jab ari +home spun +hi stri +hell bent +global compact +ge tat +fck n +fallen kingdom +eee ek +dev summit +den ice +dan z +bur bage +bou illa +blu ed +bar tra +ali brary +agen da +! :-) +ðŁĹ Ĵï¸ı +à® Ł +ठı +Ùħ ÙĪ +you ghal +win inghour +wininghour chat +west way +warnerbro sent +w music +topa xi +the crew +st offel +spu bs +so bers +shore wood +sf dc +sav ita +redri sing +pune eth +paradox ical +par ler +oscar wilde +nca avb +na dim +memor bilia +laiki pia +ki wami +jess glynne +jar rad +ibra him +holi ka +hol ston +hat ay +gran ados +gan go +ff fff +fari ed +er tl +e wu +dissip ation +dac ty +cor dray +cm hc +ci d +capital one +bon ga +bi aggi +atten burg +asco c +ano don +zhen itsyn +w tc +w kow +tit as +te agan +tad os +tablon dean +tablondean uncios +sof ts +say yid +sap business +sam c +sal ame +reform ers +ree sh +ran kine +pr grm +pic keting +pel ini +par my +out casts +organ oids +o ag +movie making +mik kel +me ti +matta pan +m du +hol lar +hhshk mohd +gu tless +fro wning +dab ang +cred itor +cnd poli +car gos +bid deford +ben field +baz o +ay da +ar nab +am ak +ðŁij ¢ +å³ ¶ +ر اÙĨ +work books +wah da +w brc +veri fies +tu j +th ao +str ato +sophi et +son al +regi stries +pk n +p libersek +nom ercy +mid sommar +mi ocic +men ial +martingu itar +love music +laugh ter +kun o +k tf +i acc +horror fan +ge ingob +gar nering +flaw less +fantas mic +familiesto gether +ex as +entre prise +enniscor thy +end all +dro yl +disgu sts +cin ders +carpinter ia +cant illon +brightling sea +ber the +be kind +bang sar +antimicrobi al +ai duk +ãĤ Ĭ +wear ing +ve ur +u gar +the mindyproject +spirit us +sp ade +sen iority +salicy lic +ryo ta +ramin karimloo +pat chett +on tein +mish ti +ly copen +li bido +lesmis official +kal is +ire les +habl ando +gu son +gaz ans +found ationday +fin ns +edel brock +dun wich +devon te +ck p +chris rock +canad agames +bok sburg +bohemi ans +bapti zing +b ju +aziz ah +au li +ar test +alter nate +al mar +air freight +abc grandstand +ðŁĺĬ ðŁĴĹ +ç elikkol +vin z +vi ajar +un mitigated +trac i +tor turous +taver ns +tal bots +stur f +star magic +shi zzle +shack elford +se vigny +sali v +sa xi +ru thi +rte sport +pro zac +nr n +nc ba +mor pho +middle ware +mc vities +man ion +kar ly +j smith +j fr +iron ore +humb ert +horse show +horse and +ha chi +gum bel +glo at +gil an +fiord land +fin u +f gw +europ car +dÃŃ az +dre view +dra v +dont judgeme +distribu tive +ches ley +che sky +bus ily +bon gani +arm less +air boat +z ut +yi ann +tren tham +trau mas +thu y +thomas mulcair +there val +tar ma +ste ichen +starbuck s +slu gger +sland ering +skelli g +sau chie +s vod +pot ent +ou zi +ou rown +oi des +ogl ala +occupational therapy +nin a +mine ola +mascul in +ly ari +luke goss +live blog +le clair +lam pang +lag atta +in boxes +hil versum +hal eem +hag man +gsx r +gor m +gio chi +ga x +frat ello +for zaf +first avenue +fil le +ern ary +el b +dw ana +dra wl +dhar am +dev ou +chab ert +carol ers +ayr shire +ato ka +anam ari +!!!! @ +âŀ ¸ +ê s +voic esof +ut g +tomorrow world +thenationall ge +th ors +tar un +syn c +re ubens +pro ck +or row +nath ali +my job +mu tin +mo el +meille ur +mcdon ell +mb ri +maryanne hobbs +ma dy +luc ado +kel lett +ke isuke +ke il +han shin +gare tt +el sa +dr ury +dis ent +digit ale +den nett +dan foss +bi wi +ap onte +ðŁĶ· ðŁĶ¶ +zom bi +yur ts +y ingly +vali er +tf xc +stu c +stepin ac +read in +rani mukerji +qu belfast +pir o +ph ole +pf as +or as +o gi +nu x +not cool +marke teers +ma bel +m res +lycopen e +lu en +ld cs +l ary +ku lam +kil ter +jumb led +h ounded +go sto +giann ini +e gger +downing street +co ola +bra am +bike way +be ve +bar aat +bak are +and le +accre tion +y one +waterloo ville +warren point +u rock +trum pet +the hindu +synes thesia +stra ya +smock alley +sin y +sidhar th +sa hl +re plug +re han +rataj kowski +pseudom onas +private ers +pre pa +per spir +ox fords +olu wa +nd weeksary +my celi +mu ell +mk hize +mi sion +mau ghan +lo onies +kn vb +kie ren +kha bar +insi stent +idiosyn cratic +i spa +hijack ers +ground nut +glo aming +gi u +fen rir +fel da +fe ts +ed ict +dra p +del ton +crook ston +chin ko +chi we +ch ch +bor gore +bio logic +berk ley +ðŁIJ° ðŁIJ° +ت ØŃ +zi kav +y awa +w fg +ved hika +un organized +un chman +step brothers +so pp +sam j +red ington +raf museum +pu shers +present you +om ial +ne wed +nation tv +n power +mor tar +modi fied +meri wether +mac ari +loch aber +lloy ds +l mm +ku h +hin ders +grl powr +gra h +go gue +fran king +fat in +f hollande +dom aine +co pro +cadu ceus +bas seth +arsen ault +anti matter +ano de +al tis +ach t +ðŁį¾ ðŁ¥Ĥ +èĬ ± +w ando +un deterred +thel p +tan cies +speed well +sp lo +so hee +snu gly +sen so +say yed +sar ak +roz elle +renega de +or ino +nutrition month +nosh aven +nemato de +mista jam +lax alt +ku ji +key sight +k ori +j manziel +human ize +hu ac +home world +hol cim +he bb +green halgh +ga elic +fur suits +franchi se +fibr ous +facts about +elin ski +du sters +dar rene +cr itt +cor to +comic market +casso wary +cameron monaghan +ca ched +brick by +atfirst sight +abraham ic +ðŁĵħ : +ze dek +yu van +winter garden +wad hwa +train sec +the tour +the american +te ac +so blessed +simon sen +refu eled +realc frampton +r ff +pe b +mor ato +min strels +mex co +lumin aires +long mires +london zoo +la ren +j z +inspiring women +ice storm +hom ony +histor i +gry phon +global fund +gasmonkey garage +fire eye +fin techs +el brus +doc tored +cy de +com au +clark son +chuk wu +char minar +chap i +cadill acs +bringyour dogtoworkday +blow up +batt ler +ap lay +an av +amazone cho +amazon studios +ðŁĵ Ĺ +ر س +youngand hungry +unbearab ly +turbo tax +tonko tsu +t gowdysc +subsi ded +si ff +shot ton +shi ma +semin ole +sar ms +ru mer +queens lander +pre zzie +nol ita +n cia +mostreque sted +michel instar +lent on +law breakers +kale o +journe yed +jay mes +jare t +in vigil +helen yu +google chrome +dun keld +dun ga +dry dock +depre ssion +dap a +clement i +char u +cer vix +career fair +brooklyn brewery +ben ita +ben de +bees ley +bb ces +bay an +bar ta +am ram +ad gita +ãĤ ¶ +âĿ¤âĿ¤âĿ¤âĿ¤ âĿ¤âĿ¤ +zen i +ye gre +whe elock +vis wanathan +video graphers +varusar ath +us at +uri jah +u ap +tric kett +tri ana +to sun +then ational +ssi e +rumb lings +rosen blatt +pil fered +pam uk +p nm +micron utrients +mic heal +match book +mac and +lumin escence +l dr +jo wl +j fl +j blm +itch y +is na +interven es +ho ag +hill head +hi der +h xh +gr fc +gle an +gas kin +f dr +envel oped +delav al +de stress +de gan +dar u +char i +cdn pse +care takers +bunk house +bu star +bty ste +bed ell +be ak +bally clare +al by +ðŁĻĮðŁı¾ ðŁĻĮðŁı¾ðŁĻĮðŁı¾ +ðŁĺ¡ðŁĺ¡ ðŁĺ¡ðŁĺ¡ +ðŁĴĻ âļ½ï¸ı +ðŁĮ ¨ +« @ +za ghe +w abc +the we +tech forgood +tam bay +standwith pp +spartan pride +souther ner +sj d +shoal haven +shan xi +ring worm +ri led +pu mice +ptar migan +ponte vedra +pk l +one shot +mon tane +mis awa +milos raonic +lo teria +little league +leam ington +l cb +kab ar +it se +here theycome +garri ott +food fight +et si +doctr ines +death by +c xo +bumber shoot +brum by +bral ette +bal and +baf tatv +awa it +anth onym +ðŁĺĤ ðŁĴģ +ðŁĩ¸ðŁĩ ¾ +yahoo sports +wei h +uni onized +tre svant +toor ak +tic ias +thought less +si thole +sfor days +sen y +sc eo +samiz ayn +sak al +s ó +rock wood +rit enour +r mt +precari ously +por tedly +ple tcher +plate au +or raine +ob server +no che +ne meth +nas b +mire ille +me zz +mb ts +massape qua +marin a +kirk wall +jr motorsports +jabo deta +inu vik +inger many +indv ssa +hend rie +glass man +george mason +fudd ruckers +fir mament +emb l +dri pped +dental health +d age +com piles +chase z +caffe ine +bug anda +ba v +al per +ago stini +) âĻ¡ +ðŁĺį ðŁĴļ +æ¸ĭ è°· +yo ked +wis niewski +wi xt +vegas strong +va asa +uni kent +tor oro +today im +thel oo +tam ari +t anni +stu der +ssen den +son da +sol zhenitsyn +sky team +sitt ings +sho liday +sand ys +rav n +ra him +que rer +oxygen ation +nbc grimm +mur alist +math are +mar ham +ma gov +le ee +kn j +k go +iy pt +inj as +in an +ibero star +her bert +gre molata +gran th +glass er +ex ul +enjo daro +end angers +eg fr +du damel +dis que +compa q +citi open +centrep ompidou +cathr yn +by om +break ou +aw u +assist ant +ash lee +ðŁĺ¢ . +ðŁĴĸ ðŁĺį +ðŁĴª ðŁĺİ +âĹ ¦ +you matter +yo glu +win tered +wid mer +vaqu ita +vap ing +un patriotic +the tom +t ation +swa thi +show off +secret smw +say les +ric ard +repro ach +rad ham +pul la +nov ae +new bold +me lections +match box +mait ra +ma sing +m views +le gic +kel ty +jocel yn +jave dakh +j hene +ink ya +har ter +gen os +fus arium +first day +fif o +faul ks +fast food +fa eries +el stra +dan se +chey enne +c mac +bohemian rhapsody +bar kan +argy ll +annast aciam +annastaciam p +ag grieved +ðŁĻĮ ðŁİī +æĺ Ł +âĿ¤ï¸ı ðŁİĦ +ਠķ +zu erich +yu helenyu +x aver +warren buffett +univers iti +tol liver +t pain +stop motion +spo etry +snor ts +shoai bmalik +ro va +re train +prep talklive +plan eson +op ale +omg adamsaleh +no dal +nike sportswear +ner vomusic +nenokkad ine +national pumpkinday +mon tero +mc cue +louistom linson +kha dim +ju ts +jens stoltenberg +ilan ds +ije bu +how ley +ho tting +he h +gun smoke +go crimson +flat land +fer o +ep g +elder scrolls +dsb math +divin er +dine en +dem me +che h +bo tes +arse hole +ali ans +ag g +acet one +ðŁĺį ðŁĺĽ +whywe march +wh als +ur prise +ul ysse +torontomar lies +st clair +spring clean +pran ayama +pneu matics +pat ux +par athy +oil seed +nie kerk +mon dele +mi strial +m ni +lu ri +kr ld +jim mi +impressi onistic +i lex +ho day +ha kata +gur ren +grow ths +gar stang +gal ician +food stagram +foo do +flori d +fitz gibbon +ff v +f atten +en chong +des con +de filed +cur ator +clou dy +bur bridge +be tters +bar cell +ban er +all good +ah rens +ach ild +Ùģ ض +zar in +wis on +val la +utri ent +twit con +timothy sykes +sur realistic +ston ecraft +she ek +shawn formmva +save me +ro many +ravi kumar +ram bles +pink out +pac north +osc a +of action +nichol ash +nev sky +mu kil +lang an +kra emer +khan academy +kar ama +john coltrane +iz quier +ingle by +hash ish +gho e +fort worth +excu sed +dundal k +dugg ery +defence hq +blackpool fc +b ingle +air amb +af le +ðŁĺĭ ðŁĴķ +ðŁİĥðŁİĥ ðŁİĥ +ðŁį¾ðŁį¾ ðŁį¾ +} } +zo id +true tothe +thic kest +the gazette +the beach +stab bings +scri bing +screen plays +savethe arctic +regi mens +qad r +pi z +pat sy +oc m +o yl +mr jame +mortad ella +m ve +kab at +jacqu elyn +ham zah +global calgary +ged ling +ga ijin +ft l +found lostdogs +fa ur +dec affe +d mo +cor nice +chutz pah +blun ted +blak elively +bhar tiya +barne sand +back pack +ba ila +an eri +ç ij +âĿ¤ ðŁijĮ +âļ« âļª +zikav irus +y eller +wp tz +woje spn +who i +way ang +ver so +tur alist +thunder bolts +te ko +tar leton +tab let +sol apur +smithsonian mag +school lunch +ron stadt +ny primary +no strand +mon isha +milit aria +mal lam +laz ily +lamb da +krasno yarsk +joker it +jer nigan +jami ec +insectic ides +home buying +hermand ad +good latte +garden ing +fu ria +frontrun ners +flatul ence +faceli fted +f naf +even tos +emul ated +dubcity council +ditavon teese +dir nt +con ure +cabincre w +ben icio +bamboo zle +badger football +arrow writers +wh anau +wa seda +w gp +w aco +vete ments +up b +t attle +sm k +pp n +pat er +pan ter +ou rense +n aco +my rick +mol ko +mo sin +live jasmin +lede sma +labra va +la joie +l son +job site +in seoul +i ale +helm holtz +hav nt +gru ene +goss amer +france s +fire walls +colo ssal +cau pdates +bloggers blast +ben edi +barbar a +au dax +as af +alhamdulil ah +al uko +ac ed +å² ¡ +ঠª +اÙĦ Ùĥ +z ani +vikas gupta +ur sus +twit chy +tube strike +try pto +trin os +the centre +tam uc +steve yeun +sp es +rashi dat +rang ga +quin ny +poo s +po stie +pe eters +oc inema +oakridge boys +o sk +n oura +mostrequested live +marche se +maj car +lou den +lazen by +lac om +high end +hal is +gon calves +go doy +fla re +fin ial +dulce maria +dr jimmy +da ag +credenti aling +co bber +charle ville +campan a +bunk ering +buck minster +brook stone +boe tt +bi jou +ay in +art majcar +are f +act ments +ac esse +ab z +^. ^ +ðŁ¤Ķ . +wah ine +w si +twit er +town square +torre vieja +super chargers +strike sback +south town +slip cover +si kes +sho ed +raven wood +parabel lum +orme au +moon ves +mil ady +matt cardle +mark cavendish +ld v +kwe k +kre ator +kh atri +jo ba +home alone +gige conomy +for tes +ff games +di ph +despon dent +del hic +cm tawards +chemi st +catac lysm +bu cher +all rise +air bourne +agli one +af g +z he +wester ner +weard ale +was dale +votelittle mixuk +vi ator +vasec tomy +thy ssen +storyof mylife +stan lee +spor ti +sin ki +shu bham +sha bet +serpent ineuk +ser an +saf ire +sa adi +red carpet +real kvb +re try +people mag +par kers +ol lg +ok awa +na al +n gp +mirand asings +june siphone +inextric ably +ibu ki +he ch +gong chan +gaunt lets +faken ews +doc ter +doc g +do tti +deci mate +dec ayed +char twell +boon ville +bal h +ayl mer +anthem game +andre arussett +ake over +af lat +adm ittance +!!!!!!!! !!!!!!!!! +ðŁIJ ĭ +íĥ ij +ãĤ¢ ãĥ¼ãĥĪ +âĺħâĺħâĺħâĺħ âĺĨ +zey nep +yn geal +wai ves +un plugging +u spa +tul si +the wcs +spar sely +shri mpton +sel fin +scoo b +re bro +quo in +preci osa +poo chie +patient care +om arab +no wra +mss arah +mon as +lack lustre +kar ter +kal au +jav ale +jac ey +in review +ifeel slovenia +global streetart +glam berts +gh ockey +gan ics +gab bi +frustr ate +facil ities +elderscroll sonline +el din +do ga +dis armed +din ium +cor uña +caul k +bu ah +ben nu +beer garden +att lee +adol phus +ab im +ðŁĶ´ðŁĶ´ ðŁĶ´ +ðŁıĮ ï¸ı +า à¹Ģภ+world pride +wil mette +vaqu eros +tresp ass +tas c +tal bert +sy al +shar ples +schu mer +roseof tralee +rob delaney +pu pu +pr dp +plec trum +obsc uring +nash villes +miz un +mc coll +maje sties +lov age +l sbu +kind hearted +kat oomba +ka a +k iting +ja j +is cuit +hot te +hell hole +fuji fil +free the +flo gged +epi pen +dulu th +dix son +defin it +de et +clich es +chi ette +cal arts +bt sport +bou dic +ben fleet +bas in +assate ague +arth quakes +ah atchee +!!!! ) +var adero +the work +te ha +story brooke +st weeksary +sa as +ri el +replic ator +rag sdale +qu okka +prophe sied +ple ats +photo walk +pastr ana +mone ys +lomond trossachs +ler ma +le eco +lac an +hair removal +globe business +gau train +fu oco +frank land +extraterre strials +eve e +dissoci ation +d bongino +cdc whistleblower +cavali eri +by your +buter in +bi dwell +ann alisa +acceler ate +ðŁĮ ľ +ze ej +x music +x clu +wur tzbach +we scra +wa qt +u dah +table mountain +sun nis +spro perty +sk cbc +shand ling +say eed +ref n +rambunc tious +radi ouk +o bre +neuro scientists +n st +mi tha +medi acom +m trench +lang en +kon ark +ke gel +jeopar dy +hurri yat +hon shu +h my +gu ant +expe d +em ay +cra ske +chou dry +choic es +ator re +ðŁĽ ³ +é ¾ +wx yz +w bai +up shaw +ude my +the tide +the adam +terror ising +t pn +sub has +spraw led +sport trust +sp cc +sol us +see der +sa hitya +ren do +ran kin +open innovation +nasci mento +nab ors +mumb ling +movi enews +media watch +maz ars +mar abou +lym pics +is key +il ing +icri sat +human ized +hamtram ck +ha ah +gra aff +gam ecenter +fr illed +fac undo +euron ext +eu caly +espn cricinfo +do oms +chil cot +ce ms +bon y +board wal +batt elle +bad diel +am bre +altru istic +allsven skan +ago a +ðŁį £ +wreck less +wing less +water ski +veggie tales +ve ge +ta pu +sk rill +sent amu +sch ism +sac ra +ric helle +reprodu cible +prioriti zation +pais ley +ny cre +nr ly +nd fb +mourn ful +me ja +mark ballas +malaysi angp +lock able +ko er +ko dai +kneb worth +kim mich +k ree +ic hu +iam sunnydeol +handic apper +geor gy +g tt +flau bert +ess i +ecker d +dar u +cole brax +cliss old +cle mmons +city year +bro ot +black kk +anti sm +anne mari +ank lets +anim ous +am bon +adam u +; ~; +åĭ ķ +zer flin +world catday +wo hooo +wedding cake +wa vey +val en +touch é +tor ito +tit re +th anda +tam anna +tal bott +stretch able +step ford +ss afa +sk ed +sch mit +reduc tive +re brands +pul ver +proble ma +pra yag +pag er +p wl +onep unchman +no elle +mar shaw +malti poo +lineof duty +lac rosse +jones es +impractical jokers +hsi ao +guill ory +gio co +galent inesday +fed monton +discer nible +cur ates +clarine tist +car ner +bori vali +blue apron +biz tips +bho gle +bas zler +b agram +ar mi +an acho +aguas calientes +âĻ Ģ +zi z +weather all +wachu sett +vo el +tu sd +tostit os +theo cracy +the toronto +tam ucc +sequ itur +saadi q +r ones +poppy appeal +p hey +ow sla +o sen +o cot +newsc asts +mol d +lo dha +ken tish +itu l +innis fail +gott alove +gonzale z +duc ation +d jr +chou han +char as +cdw social +cap tur +bra ys +aqu inta +al ber +ad dis +ach ar +ac ela +_ ] +â¬ĩï¸ıâ¬ĩï¸ı â¬ĩï¸ıâ¬ĩï¸ı +zu b +v tt +ur sinus +un lined +transis beautiful +than ts +sy e +sb kb +rebec cal +re nia +plo d +pc sd +outer banks +out standing +monte se +mono cular +mom odu +mo dano +mb y +matt mc +marie osmond +kra sner +konstan z +k shan +jis c +jen naf +j lm +i stock +hornb lower +he ure +gy atso +film club +fe ku +energy union +duckduck go +d eller +conce als +bi pin +bh g +be ren +az mi +as microbiology +anthropo logists +ðŁĩ° ðŁĩ· +ਠ¦ +wil mott +wasi mak +up in +tric ot +tri vago +ti ah +thegood place +test ing +temper ley +smo di +ship mates +sadda haq +sab se +rin cess +ram is +quick fire +pom pom +pla skett +pl td +ow ned +over hauling +old dublintown +mono chrome +lo iter +l pp +kur unzi +khoob surat +ju doka +holy cross +ho reca +hawk smoor +gr ich +go bble +exp els +er ath +elu de +d fd +clo aking +cc tr +c mac +bon homme +bo ga +bas si +bap t +ash esi +andrew scheer +all spice +ac ma +abrac adabra +ab all +ðŁĺİ ðŁĴª +à¸ķ ลาà¸Ķ +zay as +verme ulen +updat er +tweetyour lockscreen +silver tips +s design +ru chi +quintess ence +quetz alco +pul ation +pent acle +pan ton +night jar +my d +mouss aka +mount sinai +man gle +makar ova +let smo +jeong guk +i ef +hop mancup +honor arium +ha sse +go sse +g pg +fawad akhan +fair fiel +dur u +dream time +down river +culmin ates +cul shaw +co topaxi +chlo rella +chipp endales +chatsworth house +brit awards +br ina +asymp tomatic +amar aj +ai vaz +ag at +a stern +! ãĢij +âĢ ¾ +⤵ï¸ı ⤵ï¸ı +yu ppie +women lead +vape community +un deserving +t birds +sun niest +spring wood +sp reading +slow poke +see ther +sche matics +sa zer +s grove +ru drak +ran ma +quoti dien +ober st +o han +nl rb +me o +len nard +kon ica +escu dero +elli ptic +dubl iner +do brik +dike mbe +desig nations +dam pened +cun liffe +col burn +blo go +bann erman +with stood +want to +vanc o +twit tb +tu lo +trend setters +tor ists +the drive +tau b +syri acrisis +stay lor +skit tish +site wide +silen thill +show reel +rosson eri +rip curl +revo kes +por thole +north cott +no aa +musk ok +moy les +mountainde w +mopar ornocar +mont ella +middle earth +liven ation +issu er +human izing +highway man +he ee +gr attis +go bigblue +ghost writer +food love +fo tball +fabian ski +en si +dit ton +dead shot +calvar y +brown wood +best solo +b ss +apar ker +an ty +ae z +accessi ble +aa al +. )) +! ðŁĴĭ +your i +whar ton +wee tup +u tre +truff led +tro cadero +tex ts +syndic ates +street s +stal gia +skil led +shro ve +shin geki +sauvignon blanc +ro hs +rex roth +ra us +pi ggly +pc mag +pap y +pal an +out lasts +oss ining +min ecraft +mccl ane +lec on +le ws +kon zer +kat eri +j illy +j ades +instac at +henry lau +he elan +hal dane +gor n +frangi pane +ever clear +eric afernandes +el low +de sy +cat bird +brother s +breck sville +be ha +bal kh +av anza +au sty +ar ama +an ello +an berlin +about time +!!! ' +âĿ¤ï¸ı âĢį +âľĬ âľĬâľĬ +woo dgate +winoo ski +v old +turn coat +trail run +ti psy +thibo daux +theli um +the house +ter os +stoke ontrent +sp agna +soo th +sn c +sli mmed +sid ley +schle gel +ry uk +rive ters +ra kan +quiet us +probab ilistic +popu ps +pap is +oun ge +ner vosa +nat us +motor point +mit ford +mike portnoy +med field +mayo ck +lyn wood +local music +live shere +laun dre +im kristenbell +iheart media +har ps +go aztecs +gi vin +gc saa +gar bi +far sley +em ons +dw stweets +crou ton +coo te +captiv ity +cake walk +bud weiser +billion th +bath and +app sec +andrew zimmern +aj inkya +;__ ; +visu alab +us yd +univ miami +thebody shop +team exo +swi l +shadesof blue +sh murda +senran kagura +schnit zer +ra elynn +olivi awilde +objec tionable +oak ville +nis sang +n lb +mai ko +jon o +ir relevance +hollow crown +hel pin +harb ours +elle uk +dam pening +cobble stones +brooks running +boysen berry +boston herald +be all +ane mic +all new +alder leye +alan carr +ab w +!!! :) +ðŁįĮ ðŁįĮ +æ Ķ +zu ma +z v +winter land +wild side +where the +vu j +un du +tr onik +th ando +templ ars +tart ans +syl va +stur gill +screw fix +re discovery +q os +petiti oned +per ce +pan ti +oxid ants +out buildings +olivi am +nis sa +new stead +milon akis +mc neal +jobo pening +hub caps +hel med +grant chester +frigh tens +fin land +emoun tain +em eli +ef am +easter rising +cour vo +char tering +cari oca +can ing +cam of +br acci +bol tup +bobro vsky +bend iciones +bbc scotland +at tah +ag ner +åij¨ å¹´ +zapp afaye +yul in +wx w +tur vey +the clonewars +sh enton +rashidat laib +r fd +pu gin +pu edes +plat ino +pa ik +nichol a +n ough +mou ret +mal pensa +kou n +ke von +jit bumba +ji ao +jessic acap +imperman ence +ic ket +h anne +equ inix +distr acted +dis regarded +di ah +con focal +compac ted +clo ss +carat selcaday +cal lies +cai roli +bt son +bmw usa +bee ton +ambo seli +align ments +ali ghi +ya hu +y uli +water dog +w cia +tink ler +the star +thalai van +te f +subli min +squ ashes +sol arium +saveour nhs +ra gon +power breakfast +phoenix ville +nel sons +naz riya +mani folds +m spartner +jir ga +jaco bean +j rue +ite it +hon das +gu eye +go yotes +forest day +e ton +derby uni +dding ston +dac tyl +cum bre +cork town +congre gate +cincinnati zoo +chal ke +cado gan +breeze way +bol us +bernar din +avi da +al ooza +yo jimbo +yash ica +y app +wpmoy challenge +val es +up draft +sub nautica +stic ism +ss rugby +sr ally +sli thering +sli mani +ship ka +sex tu +sci fis +samsungmobile us +ram at +r nas +queen elizabeth +prou dd +playing cards +pe tes +park gate +pan vel +ola fur +mahan ati +lm f +kie ffer +ka il +harmon izing +har is +ge onews +gam bles +egoti stical +deac oness +cu sick +cruci fy +cheese man +cat amounts +bread stick +bla g +at il +as wj +absor bers +âĿ¤ï¸ı ðŁĴľ +âļ½ï¸ı ðŁĴĻ +yo gic +wil drose +vidy arthi +v artan +uch is +tw ar +think pink +team ireland +sw on +sho orn +selec tivity +sa ola +ron ces +ro it +rep mike +ra sen +pur portedly +pu skas +pu lang +plur ality +pi ppo +phosp hor +pet co +pad locks +osho di +mun cy +like it +la ssies +ko ss +kar ns +ha ki +giro la +g ado +fiel dre +ej io +dol ley +digi day +di om +destruc to +deci bels +coder dojo +cc s +brown ell +bon ic +aw adi +ath am +as pas +arag orn +an oop +." âĢĶ@ +ðŁĴª . +ðŁijĢ . +âĶ ı +visit nland +under shirt +un becoming +trans gender +thelim it +tat anka +su omen +stpauls london +shru bbery +shi fu +s bir +re dedication +play pokemon +pen man +p hair +oy al +ot tum +nam ie +n antz +mio cene +min a +marcel le +mali on +liquid ated +lampe dusa +isleof wight +iron stone +intell fusion +he ba +ellen page +e tro +dheeraj dhoopar +dance sport +cra po +co ghlan +clearthe shelters +cic adas +chec kyour +ca relessly +breaking weather +bor sa +bestsolo breakout +ati vo +ani versary +amnesty uk +al tea +ai guille +af ka +abhi jeet +aap g +] # +ðŁijŃ ðŁijŃ +ìĬĪ ê°Ģ +zed ong +wheel ing +wh oooo +westand by +v cp +un inspiring +token ization +to velo +tea shop +strongh olds +ri yal +re ino +ravin ia +not weak +marine insight +lo yee +krishnamur ti +junior doctors +jock strap +jabodeta bek +grid locked +grant cardone +gl g +ge se +ge mann +gas works +ero ticism +dru s +confed cup +ck g +christin ec +chick end +caitlin moran +brand new +bike toworkday +bengalur ufc +be sa +as ano +ðŁĽ Ĵ +ठī +zav vi +ye ster +wood field +winter ton +ucl h +sylvi a +stev el +si bat +shep ton +shab an +service members +sep ta +sassaf ras +sa ther +raf tery +press freedomday +posit ory +pi v +otr b +nino y +ne of +nag ra +n te +motor ama +mom e +met in +m tr +lis land +lac tobac +ky iv +kil bane +kat em +insinu ating +infr inging +ho pson +green blatt +gar nets +exp ander +droyl sden +cot ash +col an +clou draiders +clau ghlin +ca jon +bc p +bae za +ad hm +ðŁĺĮ ðŁĺĮðŁĺĮ +ðŁİ¶ âĿ¤ï¸ı +âĻ ¨ +x at +waq as +up standing +sy ny +skul leero +shan be +sham ir +scare mongering +s zo +ro mulan +re clai +quadru plets +pro enza +pop vinyl +plain ville +pbb abscbn +pati o +over nights +or sini +once always +old games +nam on +mer thy +may port +mat la +lo vitz +kul ick +jessicacap shaw +j wa +it c +hou ck +happy life +grey stones +fronti spiece +found pets +fer rera +fanta st +duck sunlimited +doom ben +deniz ens +costab lanca +comi enza +cli matologist +cheat sheet +brit geosurvey +ble a +ban corp +aquil ino +ann nd +aes a +ad ai +aa sen +ðŁij ĥ +word stoliveby +vesic les +vach on +unbeliev ers +un needed +sof unny +sinfoni etta +si pp +shin ny +shel agh +same er +rs gnl +real lucylawless +r wc +r ccg +pi ps +paste magazine +ni j +me tries +mc gavin +mc broom +mal d +low rie +k ericho +jam ma +in japan +i fex +heritag es +hb h +h z +green stein +gotg vol +ga ik +front page +fi endish +electrocu tion +des sen +cro ma +con sign +clasi fic +cir i +bab er +air drie +ðŁĺ± ðŁĺĤ +zi ering +zerflin music +ye ma +yam a +wk ts +wis sen +whir ly +wedding ideas +w rens +v dm +spin ster +soli dari +sj h +she tt +scot tories +sar af +roy all +raf al +po h +pirates ofthe +pd q +omg trolls +nz veng +north point +movi da +milk men +mer ri +ma ines +lecture ship +key rings +jil ted +hypnoti zing +hydro xy +heu sen +hal con +gro sir +grays lake +good life +fyn bos +equival ency +eg go +eber hard +cub bies +cri sti +coo ts +can die +brain pop +batt enberg +avi les +am fa +. ðŁĺı +ðŁijĮ ðŁĺĬ +ðŁįģ ðŁįģ +ðŁ¦ Ĥ +ðŁ¤ IJ +ãĥ ĭãĤ +âļ¾ï¸ı âļ¾ï¸ı +was l +uk team +transcend ent +tendon itis +su shil +son iq +sig g +si sodia +shut itdown +reali stic +purpu rea +pper son +pomer anz +pay o +oral history +ol g +nonchal antly +milton on +mid summer +man deep +mal uku +life boats +hor sel +glob alized +fle shed +el ta +do gger +dishear tened +di ko +den geki +cone head +cold brew +closethe camps +chino y +cbs bigbrother +ca el +bak ayoko +as per +alex andre +al chemical +al ane +ade k +~ *~ +wheelchair tennis +waj ir +vivid cloudof +vividcloudof wat +un abashed +thisi sp +stra at +stigmati sm +sel fa +ric on +pre pper +pearle scent +pac west +or geron +o kuma +liber tine +kw t +kot la +kar ai +k sn +iz he +human istic +gui se +fo ton +er au +ell sbury +ec chi +dischar ges +dead lier +daylight saving +bow len +bal ki +bail ie +avo ice +art school +air lie +yl land +ye urope +y eni +waga h +vo ya +stat ically +spo de +sou led +socin n +scholastic uk +real hiphop +queri do +ps as +pi des +pe lee +pass enger +om ri +ol ley +ni ms +n mt +mo pac +man ang +ma stro +m kiii +lili um +ky ary +ku mud +invic tu +hotch kiss +fu si +fro tterdam +fren kie +fen land +fair lawn +et ou +dor r +delu ise +concub ine +can by +baren boim +bad bad +ambi valence +alleg an +x po +wi ggle +whit ty +vegan hour +tu dung +tom asi +to too +thy atra +thankyou thursday +t dw +t aga +ss ay +srini vasa +siddi qi +sh ly +round houseldn +ra ser +r jr +nor de +myth ical +mel kwe +lin ch +kal ah +jes ús +illa i +hel ple +goodmorning world +gabrielle u +du in +crony ism +chy ler +boot strapping +at ol +ar apa +all ent +abra zo +aam c +ðŁij ¤ +ç o +wat kin +u fl +tom ica +taste maker +soo thed +shop local +sher m +sell ars +seasi de +ry delr +pro spe +pier ces +pho tonic +pe hle +p bt +own your +op ryland +one timein +npl vic +non alcoholic +mor ado +m gc +lim ou +l uni +jo well +iter ror +ink ster +gru ppo +gross mont +good boy +gaw ad +ft z +expe diti +ers onal +epsom racecourse +em men +edmon son +cri mping +con eflower +comple ment +chis inau +cal abar +brother sosborne +beautiful bc +au be +ari i +am historymuseum +alber tus +? ': +wir t +wer m +thie baud +suc i +set suna +seo tips +ro xx +ram ire +proge sterone +peter sham +pan op +p fe +origin ale +oooo ooh +notthe fake +noshaven ovember +nor i +nom ics +nd hakoo +mm ac +mau moon +lu pin +lo effler +ling erie +leyden pride +kinder sley +kinder chat +ken o +ke strels +k br +izz at +ie tta +hyper space +home schoolers +hesit ated +h nk +gre bes +gomor rah +fox ton +fairi sd +eating disorders +dart board +da ura +cy fairisd +chri smo +chero v +cetace ans +calum worthy +ben tos +amo on +ak rapo +a forum +" ...... +âľ ´ +virgin australia +vet kin +un substantiated +u tra +tun able +ston ed +st p +sp ong +sn h +shint aro +scsu huskies +robbin sdale +pre st +peds icu +n fi +meg turney +marin el +le stat +lb v +ker sey +katiec assidy +justin welby +ip c +international womenday +hy orin +for lan +f tbl +en ny +cra dling +christmas market +chand i +buffe red +bro u +ben cic +aspen cer +arsen als +aron son +air cel +ðŁĮ Ĵ +yeg music +ush our +twee tsnow +tru tv +the magicians +t ä +subjec tivity +sson oma +share it +ri vend +rap tured +pur l +public ans +prou lx +out lay +o ggy +net scape +navi mumbai +mu amba +med x +magnu son +ken es +ka jang +k ask +jay pee +j gr +its gabrielleu +imagin arium +horri fically +hello games +head long +hal ili +gul ben +good land +foul kes +fo ch +fin ned +er day +dissip ated +deadpool movie +dakima kura +d ort +chad stone +book outure +biof ilms +baf anab +b co +as besto +ali g +actor sathish +èĭ± èª +wei der +vitru vian +th appen +tele fun +tab as +summon ers +sugar ing +steel ernation +spring water +sla very +shore lines +rund gren +ro enick +piscat away +papu anew +n car +mo gu +megap lex +matt kenseth +man than +laurenti anu +la tha +hong dae +heim at +gil ford +gh ot +fle ure +fil ton +eo in +elizabeth may +el ander +dys art +death of +by ways +brush pen +bmw pga +au kee +ant il +angu lo +amyg dala +$$ . +ðŁĺĮ âĿ¤ï¸ı +ðŁİī ⾨ +weare winter +w tic +w shh +vintage hour +ven ir +the ta +ter race +tee tering +scri ms +root beer +rik mayall +ra fc +pic ayune +over party +over blown +ostap enko +optim ally +ool ers +ob verse +noble sse +myfirst tweet +mother less +micro biologist +metro schools +m ations +lipp mann +kh en +ion ization +iam ami +homes ickness +hin ton +h ce +gw u +gel b +fur lan +forum keralam +filmmaker friday +distinc tively +crowd source +by en +blon dies +bel ay +ar boreal +ak ur +a ford +ðŁĶ ĸ +water ship +voltron legendary +ul san +thereal grimmie +te sa +stonybrook u +star bound +ss si +snew video +sh we +sc bs +sar awak +sangu ine +rough er +re release +pro day +ns be +north vancouver +north side +myfox la +mur kowski +moo sonee +monte vallo +mohan raja +men ang +mehboo b +mac aques +jan itors +home kit +fond ation +family tv +emp y +e pl +dun ston +don ner +cowden beath +brun dle +blogg ingtips +blan ka +bick ley +bau r +bas sano +au rier +ari zzo +" ..." +ðŁĻıðŁı¼ ðŁĻıðŁı¼ðŁĻıðŁı¼ +ys rcp +wilson ville +wheat field +vo gels +vig na +v sl +tweeter app +special isation +song tweeterapp +sole imani +si mula +sher win +se ia +sanat an +reinvigor ate +qu arts +pou lenc +physi ologist +pas sa +own thefuture +ov ations +ol dee +oldee ire +ni ue +n pp +n joy +matthew modine +lin h +lam jarred +ke irin +kasi ak +ha ire +h pg +gn n +galatasaray sk +fran ky +eng sub +en nui +dishe veled +discrimin ates +dh s +cultiv ars +conco ctions +con tempo +cl anc +bu cc +bo bb +b aug +architec ts +ðŁĵ ¨ +ye ow +wieder sehen +web ber +vol le +visit ma +universal credit +star rett +sport bike +scre e +san jana +rssur jewala +re ssing +pr ings +plac erville +phi de +ou sp +orche strate +or ity +norri stown +nb hatt +mo yle +mil lett +meshu ggah +math ilda +ligamx eng +kra s +kip mooremusic +k ss +ju gaad +ital i +gon dor +extra dite +euro tunnel +eu chre +differenti ator +d weck +cru tcher +cor bi +co cor +co ben +cin na +cc ny +boun ding +boo ch +bit ly +b ence +ant ini +aden uga +ðŁİ« : +whi ston +vince cable +ver ratti +tur ton +tra ppin +thisi ss +thel akes +the sea +tam ago +tal cum +tal an +suc cotash +su ria +sig ab +shine e +select men +schar f +say ulita +san j +rugg ero +ren sburg +o dish +ne te +nah da +n ris +mo haj +mathi eu +mart ineau +khadi jah +kay ano +kanchi puram +ib mi +gun ma +gh all +form h +fl acci +first lady +equ alling +corrup tible +co bi +ci vet +ci era +cab alleros +bartol omeo +av ons +anjel ica +al ci +ag t +af tn +aci dosis +ðŁ¤· ðŁı¼âĢįâĻĢï¸ı +è IJ +yyc bike +x files +wit suniversity +web pages +wa ard +val in +un set +typo logy +su wanee +stockport county +startrek cbs +stan e +shor ted +ri stin +ray ados +pre hospital +pol di +pain swick +mou n +moto america +ido wu +icec aps +god splan +fre sne +for i +fich tel +fe te +f soe +embarra sses +ebon yi +disappro ving +comic palooza +cephalo pod +cassad y +bun tu +bu shi +bri ang +ana thema +alter net +adam m +ad mission +ac nn +. ¸ +ðŁı ° +yo gri +wh ey +w cb +vel ar +symph onic +star ted +spide y +se bring +sc abs +sankal p +prate ek +pleasee ee +per icles +pater noster +par ag +ob le +nr k +mun dele +maxillo facial +ma un +m jr +m clar +keeneland sales +k live +house keepers +her ta +gu mede +gor dan +fil oil +dor mouse +disaron no +di shap +d pc +col tford +cmn hospitals +caer leon +bur gas +bi shan +bha ag +as roma +armb ar +accept ances +âĨ ĺï¸ı +vit o +var gas +sv f +super fight +ste eg +sil ke +no ia +michael fassbender +mat azz +lax airport +ku bb +impal er +illa in +hyper pigmentation +hand ers +ha ken +good vibesonly +fuen girola +fixer upper +finding dory +f ale +ep ass +e sters +dr d +de mel +co ir +climb ing +cl attenburg +chab on +apalach icola +an tiv +ame en +ach ie +a sty +women entrepreneurs +way sify +us in +union town +stopthe cull +sport f +sper formance +show match +rw th +real sociedad +readabook day +ran ey +prosen jitbumba +per kins +pas cosheriff +palli ster +nic ci +mis representation +middel burg +mau dsley +lun gi +li the +le fts +hu at +hoh ner +gim let +ge tup +food land +fo gged +eco build +dee jays +chimic hanga +bug bounty +be hl +ann ada +ag st +ag fa +adren al +ðŁijį ðŁĺģ +wr l +wor tley +william levy +w eng +un pretty +u oe +tugger anong +tho da +st michael +son amoo +si apa +shar ron +shar pies +shape shift +rn li +rick ross +ri y +ra spy +q be +pe saro +parap legic +panch kula +p summit +nin anes +mis adventure +mb atha +march es +mar gery +lod ger +letit go +knife crime +kn p +ing i +i eds +fat ca +eh ring +e by +diecast malaysia +den ia +cur rumb +cran well +cooper ative +car tes +biomechan ical +bar letta +aw right +arm ing +and or +a ash +yadi er +whe en +wh k +wat o +war ding +v ns +ura wa +un determined +u dm +type setting +traw lers +the square +t world +rob it +r top +penny stocks +nic os +mk b +maroo chy +manzan ita +mag ar +luc ile +le eming +ix tapa +hamble ton +hall statt +hair i +gym shark +g omg +dun s +dendrobi um +de perfil +cister cian +ci pla +charli ed +carac al +boriso v +av geek +arn cliffe +ano u +al wa +ðŁij ĥ +ëĿ¼ìĿ´ ê´Ģ린 +ç Ĥ +е ÑĢ +| ... +z otto +wrink ly +ultr arunning +tric hat +sy ros +str itt +spider gwen +so won +sla ppy +shekhar gupta +sel u +sa hil +pre mi +plat er +phone tics +ph led +peter son +per sol +p mpt +ow x +nis bet +na ic +mor dred +mon da +moha patra +mess rs +me ola +lo chs +ligam x +just sparkles +joyl enz +journalis m +jig ga +j ore +ig d +hitch hikers +harb ison +hallucino genic +goe de +gg ing +garl icky +freak onomics +fr sc +evangeli stic +elem chat +dread central +do er +bu ju +blood drive +beau vais +bar fly +ark low +ard ine +zi pping +z id +winter storm +wedding inspiration +ward our +tothe top +that guy +tas cam +sym pho +stu dley +seattle storm +se ey +sc up +revo king +poon am +po v +philly dotcom +one z +mar antz +m tuan +lack land +ku chi +kno tes +ki do +ju mu +jo ve +je ux +jaco bl +irish music +inst ra +indo china +hy mn +greg james +gold crest +gingi vitis +electr icals +east ers +drif field +dal liance +dag upan +cra il +comic sgate +chinst rap +ce de +categor ised +cap sizes +camill eri +but chart +brom eliad +brad well +bee day +be ds +barangay ginebra +b wr +ay so +yo plait +west fal +wee bly +un restrained +team om +tang any +spo leto +so bo +silver line +redrock sco +re boots +ramach andran +pap o +ob liqu +nico ise +nanom edicine +my plate +mongol s +mo gherini +mn ts +megal ith +mac ap +ma ggy +luis fonsi +lor g +kooten ays +kne pper +king a +kar yn +k to +j ro +i hh +hra b +hit sm +guil lem +ghe or +geome tric +feed back +falsi fied +exacerb ated +durban ville +din ky +di ep +de pop +comp tes +co sima +class dojo +choreo graphing +bub sy +bet v +ana gh +adhe red +ìĦ ľ +yu rek +work group +win di +wal lowing +valley usd +un common +te rel +sky atlantic +si gi +scra wny +question time +pim co +pe sta +pap worth +ox bridge +os wald +on z +mont ville +maid stone +logi x +li pper +jc icki +isra elite +il ac +gr illes +g fr +feature film +fal las +dw news +drew scott +dd ar +davi da +craig avon +comefroma way +brox bourne +atri athlon +appu ram +ðŁĺģ ðŁĺĬ +vacuum ed +un seen +trick y +tri ght +tor ff +tachy cardia +standar disation +st day +sje arthquakes +score boards +sco ter +ronan official +rit ch +ra gu +ps ers +popul ation +paedo philes +over supply +oc dsb +ny topinion +nicky hayden +ne stor +nau gh +n devon +morgan ite +man al +makk al +lim ps +j ellic +hayley kiyoko +harro ds +ger vinho +faking it +fab u +ess ere +crime family +ci vi +che ever +charter house +cal avera +cabbage town +butter ick +biglotter y +bene teau +ba bette +at ua +ab abes +... :-) +ðŁı ĵ +west mead +west ford +voice overs +un motivated +t ander +stand free +si biza +shop front +sha am +sc ross +sawh ney +sau gatuck +re claims +r ht +quart zite +plough man +parab ola +mon ade +molyb den +marks manship +mari ja +mahesh nbhatt +ker ch +invent or +hun ke +hr n +hou sec +gary sinise +every where +equival ents +elu des +ebook deal +e ge +do ona +dex press +daniel aruah +coch lear +circum vent +chapulte pec +champ neys +blackkk lansman +ap ie +am yl +all ys +ðŁĴĽ # +ðŁij» ðŁij» +writing prompt +val ar +un tidy +sush ant +sport chek +spad eny +s gen +read the +quarter master +prati k +po th +pat aki +ng supereagles +nc bi +mand ating +mag er +loch head +kam ar +imper vious +hel len +he u +gi ang +geni ushour +ferv our +fast break +exor di +exhilar ation +europe o +envis aged +ed gars +ear drums +diabete sday +cor tese +chri smur +can ali +bap a +ðŁij¶ ðŁı¼ +ðŁĮĪðŁĮĪ ðŁĮĪ +wark worth +vra bel +vintagec lothing +u im +time a +south view +shol t +sett le +rc g +ra ws +pre positions +pr ama +pencil art +out for +os valdo +ore y +ni oh +nabeel rajab +med gar +lif ford +kol sch +kaz uma +it sb +ilove cornwalluk +heritag emw +gra p +gall eri +for dgt +flo y +fel stead +f ony +entr ée +dy sen +custom made +cole shill +car vel +cam m +brun chc +bmr tg +blue coat +biggbos stelugu +al pena +african american +afoto deperfil +-- , +âĺĦ ï¸ı +âģ¦ @_ +zz ari +wood stock +visit the +trypto phan +ton katsu +tol d +stay in +st aci +sm ale +siber ian +runner sworld +pil on +on th +nun nery +nbc chicagopd +my ke +mp cc +milli pede +mark man +info security +il aria +hul st +horri ble +gli mmering +ge she +foam cc +eure ka +e glise +e buka +d allin +cru elest +col lo +chri sma +chi arelli +buen avista +braintu mour +atlas v +athom pson +adi za +ad ance +ze sti +yosemit enps +wa fel +ve ined +v ball +urijah faber +thanksgiving with +stu pe +sn krs +slen derman +si ku +retail tuesday +repul sed +re occurring +ra ed +qu tub +plac ido +nayanth arau +mutual funds +mi fid +mer ve +mat atu +mah wah +kri stel +ket ter +indi ant +groun dup +giftsfor him +g do +fume tti +fa sig +dak ar +d bp +croo kes +com per +clue let +cit é +chris van +cc sa +british redcross +bone head +blanchard stown +bbcradio scot +bab ington +ba is +all time +] ; +. ðŁijı +æĿ±æĸ¹ç¥ŀ èµ· +âļ½ï¸ı @ +zil ch +yn i +tri sta +ti wi +the team +sy phon +stellen bosch +selec tion +sand well +ri bcage +pest ilence +mh mm +mention someone +mb ale +made me +ka uri +hero ically +har um +gon nabe +field day +ec ats +droo g +do di +devast atingly +demarc ation +daw id +com pra +co die +ck l +chil well +cherry blossoms +cb fc +bet u +arn alds +ad ger +ðĿĺ ģ +öz demir +yogri shir +year old +yakis oba +tri ennale +to pen +ti enen +thong thursday +there ale +sá bado +sso white +so yy +pu sci +privati sed +physi os +p tf +over active +ob on +modu late +member monday +mc nairy +mai read +lex press +ken worthy +ke shia +kati emc +kap it +k icc +go omba +go blin +fo lies +far mall +eye con +dispen ses +di aph +bro y +bre r +bal combe +art scentre +applec ross +aldubi yamin +aali yah +ëĤĺ ìĿ¸ +z una +wil le +stock room +sk rona +sister patriots +sd aughter +river wood +puig demont +political prisoners +pharmaceu tical +pal atin +northumb rian +ninanes bitt +newstalk fm +mole skin +le hane +ksen ias +khor asan +kat u +jab ber +j ón +it ttt +ini sta +ic kel +hahahaha hahahah +gn ite +git te +fan cave +exc ell +encum bered +eden project +dism ount +chris webby +body positive +bc sd +ash land +ang mering +ðŁĴľðŁĴľ ðŁĴľðŁĴľðŁĴľ +ठļ +Ûģ ÛĮÚº +é ry +zzzz zzz +zet as +woolsey fire +vis ital +ve ja +uniof hull +tric kier +toon z +tom be +the of +step toe +sten o +sp ake +sho sports +sam ark +s bur +priv at +power lifter +mc sweeney +longh ouse +kitchen ette +impeach able +hy nde +gu shers +gre mio +de merit +daf ydd +carcino gen +bulb ous +budd in +boy yy +bla sto +bertr and +bandhav garh +are cibo +ar shavin +al mora +! > +ðŁĺĤðŁĺŃ ðŁĺĤðŁĺŃðŁĺĤ +çĶ » +xen ophon +ver dot +ton gar +sto ssel +sanjay dutt +road side +revi ve +rebu kes +rc v +p we +over flowed +onet ough +oik awa +ni fe +nfl onfox +miya jima +mg f +madewith love +leg aspi +lac as +kae de +k fm +hilari e +heath land +haw ker +hack y +gor ill +gl vc +gate shead +fli k +energye fficient +em mer +ele mans +eg gheads +ec rew +corri b +bot any +bol sover +bar oni +b we +alpha ville +a world +zur ita +un injured +uk zn +tri kes +ti ket +thick ened +tele scoping +ta ung +sunglasse sday +sun shines +smoo ches +sh ate +service dog +rhodod endrons +republi que +rating theraces +q urbani +p ma +no amnesty +nashvilles ounds +lucas arts +love child +lot ton +li bel +kar men +kad in +idy ll +iam johnoliver +hold out +haup t +groom sman +gri mmers +gl ay +gee chee +flap jacks +fir med +f pf +europe aid +enam elled +eco sse +ecker sley +don mar +disco theque +d mar +cad w +bri oni +ballin colli +avalan ches +alt adena +ðŁĺŃ ðŁĴĺ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤ +ðŁijĩðŁijĩ ðŁijĩðŁijĩðŁijĩ +ðŁİī ðŁį» +ðŁĩ§ðŁĩ Ń +zul fi +yogrishir amdev +wu stl +white throat +ven ere +un furnished +un changing +truck loads +the artist +summer jam +sr ite +royalwelsh show +ro don +rickie fowler +reson ance +re scheduling +r witherspoon +quar les +pin n +pil cher +p kc +p conf +omar u +official pompey +mu ting +mla denovic +mi wa +mb da +mar v +kit ana +jar l +intu bation +her metic +ha ise +gl vc +first warn +et weet +breakou tedu +bor din +bee gees +barbar ossa +as chools +an elka +amic able +americ ane +am ap +a jones +âģ£ âģ£âģ£ +zi zek +zell weger +viz caya +visit mexico +truck tuesday +trab zon +the joker +sham bolic +sali vary +rico harena +re writes +prin ny +porti mao +pha eton +par bat +p sh +norman ton +memor ise +mar got +malle able +made easy +ker ley +k pn +izhe vsk +itt f +id c +hau z +han am +fu gate +foo ters +falling water +faken ham +david labrava +dab boor +cros scut +carni vale +camer ata +bridg it +çµ IJ +âļ °ï¸ı +âĸ¶ âĸ¶ +प र +win canton +wair arapa +volcan os +veterin ary +upside down +ua allamerica +the wanted +tao yuan +t elli +szcz ec +stu g +sp c +smat tering +sel vi +se jour +scar boro +ri sc +r iner +o red +ncaas oftball +mo jom +mit sloan +mak sim +lur ay +long ton +litho graphs +ley kis +jo by +ha chette +g yor +for tress +f outs +e om +dissemin ating +datavisu alization +conoco phillips +choice awards +chitr aloka +ca afb +bur in +bis son +ben o +bbc suffolk +athur sday +aravindha sametha +ad hoc +ab lue +ðŁĮĪ ðŁĮĪ +ãĤ ģ +world class +win du +visit kent +tubri dy +tra choma +take alot +shore bird +sho ko +she aserrano +sex perience +refor med +rece ssive +re purchase +re heat +pu trid +pb con +o thr +o som +norther ner +nit rile +new relic +n endo +mwan za +mv c +moreto come +me co +lac o +hud dy +gar ners +fresh offthe +find hour +er mene +e more +dv ar +dog meat +denver post +demon ize +dan il +car sen +bouilla baisse +blo gher +at andon +alibab agroup +ðŁİĪ ðŁİĤ +ðŁį¦ ðŁį¦ +z ameen +y ata +video shoot +venkate sh +the tis +temper ing +stra thro +stodd art +sel sey +ru ises +ri ski +re house +puzz ler +pr é +po phobia +per iment +pav an +pare ja +on ur +mi sting +melkwe g +mc gee +lo omer +kumail n +gn awing +ear nt +dismiss als +degan arseguidores +dai ichi +dad u +d za +chang sub +carryon blogging +boardwal kempire +blurry face +barn wood +balo chi +b hin +amra pali +adam devine +acom pan +ab ro +~~~~~~~~ ~~~~~~~~ +wo jcicki +ve ik +user names +ton park +super bird +string fellow +stel o +sephar dic +per umal +pat ah +north ville +noctur no +naom h +nad ra +mv m +manzan illa +ku cherov +kir il +k michelle +ip al +in ster +hub bub +hr rr +gu stan +fy p +eli on +dizzy wright +disc ours +dabboor at +cu tely +brain ard +beat boxing +ban as +as kari +abo xing +you ll +world mag +w ms +urin ate +thunder snow +super bloodmoon +sm sa +skulleero z +sick les +sh inj +sesse gnon +samuel sson +ru din +road racing +river hounds +ride au +ra iz +prephoo p +porto frotterdam +polling stations +poll sters +pap el +ottawaf ury +nure yev +method ical +le ix +jame sk +in situ +ho sed +het alia +green acres +goo good +fé ile +fri bourg +frankie j +end ay +displ aces +dee red +bluestar zone +bhubanes war +beli ke +bag by +atar get +as ay +ariann ahuff +ag adh +ab ria +[ ( +âŀ¡ï¸ı âŀ¡ï¸ıâŀ¡ï¸ı +young man +weekend kav +vi das +var is +tot ter +test bed +t pp +swe tt +soo s +rut ledge +reddog susie +recer tification +priyam ani +poo t +poly vinyl +pleas ert +part ay +na iler +mount vernon +mo za +mill o +le eps +ld k +laur ita +kwas i +jay bird +is r +imag azine +il ac +grim dark +episte mology +episo dio +de colonization +chinois erie +chag as +cd v +cam bia +braz oria +br icht +blu d +bal as +athen ry +as ps +apr ès +anu radha +alt balaji +... [ +âĿ¤ï¸ı ðŁĴļ +âģ īï¸ı +ó g +ye aaaah +visit jordan +u og +u dan +tso go +trench coat +tr ite +theroy als +th atha +st illa +shaw ano +sat anist +ros lindale +red ban +ral phi +quar to +pra d +pr ang +ottawafury fc +naom ie +mysti fied +museum day +mu ff +menstru al +mediab ase +lgbt pride +leop old +ke ley +kal ank +huski es +hebden bridge +harmon ized +gye on +green lit +geo ghegan +eye opener +every child +eneed le +e ure +dian er +di ers +day fm +cru shin +credit suisse +clon dal +cle ves +ci onal +c itta +busines strip +body paint +b ous +app enz +**** *** +ðŁĺ¢ âĿ¤ï¸ı +ðŁİī ðŁijı +âľ ŀ +wheat land +westf alia +v ade +time slot +t anta +svel te +stri pey +spring summer +seeyou there +schi sto +sang er +ryan seacrest +rother y +ross ellini +r re +pu e +porter field +pj m +oo voo +o lean +nephil im +mariach is +lole sports +kim ba +jung woo +jo anc +it ll +ish ant +herma ph +heck in +hair day +gh onetv +folk tale +eo cene +el wes +ed dsworld +don nington +distor tions +digital camera +dark wing +cor tin +chi di +chat man +cath ed +catapul ted +bra g +boat load +bo ddy +blow torch +blo cky +bio bank +beat la +bb sr +au dra +andre ab +afl womens +ðŁĩ©ðŁĩ ´ +à´ ª +z eds +your story +x j +wet ness +vla dislav +vin ales +ver beek +tw irls +thor ity +t chami +super be +sug den +strat for +stormy daniels +squan dered +south bourne +semin arian +sar godha +sa ima +reven ue +re tell +rb x +rain ha +post operative +perform ative +peck ham +organ o +or ak +oh en +nyc pride +nir mal +ni ños +manic ured +manas quan +makers market +london ist +ko jo +ker to +k bd +j tc +intermitt ently +gano derma +fa ch +ev gen +equal ised +e oy +e health +e harmony +don ee +ch strfc +bint an +ar man +ajac cio +afa una +wre aked +tul sigab +ti kal +texaste ch +t ative +sweet potato +stkil dafc +spar x +slip case +shady side +sal tz +r int +pran king +phe red +par ana +od die +o we +ni lesh +nar whals +n anti +mp hil +mat in +mag asin +lu mped +kop ar +ine dible +im bo +id leness +high end +gro aning +franch is +fo ssa +fo gel +em mons +e bbing +dro ver +co ppi +chin cote +ch ory +c de +bow lin +best memoriesof +be se +ap lus +a afc +" ??? +ðŁijĮ ðŁĺĺ +âĿ¤ï¸ı âļ¾ï¸ı +à¹Ģภ§ +wre aks +w psl +unil aterally +un predictability +tu bb +tofthe year +ti o +spenny moor +snug gie +snow cone +sims bury +segas aturn +sea eagles +ro sy +retro active +qu ail +polyam ory +par tisans +od t +nuev afotodeperfil +nca agolf +na ing +music photography +mun dell +mike trout +ly sol +le doux +le awood +jit endra +hus ks +he et +hazel ton +har rod +hamp stead +gor dian +glar us +gav r +fr ing +dend ro +d ch +com pline +cit ys +character izing +cel adon +carlo ss +bur ana +bun goma +bu bby +broad beach +at g +ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ +yas uo +xfinity series +v itti +ut coach +the win +tex ans +ta kia +super heros +strang le +star child +space ship +s will +rin ella +penn relays +pede france +p nt +p lim +over bearing +off setting +n cm +more t +marin elli +makemy trip +lupe fiasco +love ireland +losange le +intu itively +go jags +gear up +g wal +esp adrille +dou jin +diam ant +d ace +c sic +bas inger +aspen institute +abcac tionnews +à¹ĥ à¸Ļภ+wel e +wee gee +wal mer +vol can +vintage findhour +un branded +thab sc +th ies +sp ars +serv i +sam ard +sabo teurs +pol icec +pal apa +olac abs +n cert +mount ford +mahin da +less er +her thabsc +ger aint +ep worth +do gara +degra w +de sc +cran mer +cn co +chand rak +bur han +buenos dias +be ki +bcb tigers +bab bage +ah gase +za w +y ls +x tr +trixie mattel +tis sue +szi get +sy co +spla yed +sd ell +ri pe +ri oli +pumper nickel +on thisdayin +ofici ales +ning en +mor bid +mc v +marcu m +lets be +l wd +khy ber +k tm +jig awa +j rd +hyper inflation +halloween town +greeng ro +disinfe cting +dis figured +dal keith +co ble +bon nes +bill mckibben +bar ç +bar uch +au dic +apol lin +ab har +william ssonoma +who ville +up state +tu tera +tilland sia +thedragon prince +the high +ta kra +t ating +super smashbros +schwe izer +sabar mati +rosco smos +remo ve +read ju +r tw +plagi arized +pi u +pen rhy +n nl +museum selfieday +mmi wg +minis kirt +man ek +ma ury +kling ons +k wi +joe star +g lowed +fol l +fl att +fi estab +eric son +eleg ant +e reader +du thie +di ano +col lis +car ami +boy kins +bor gen +bon do +bafanab afana +at ack +ar tra +acu ff +æľ Ī +yor ick +wa ar +w op +u jj +tun ica +ten orio +ta house +sum me +sel in +sec ity +sarab ande +ry d +reli ves +polynom ial +phyl icia +ol ay +nish ant +minneso tal +micro fluidics +medi agroup +make ba +mach an +long livelong +ligon ier +lam prey +karma pa +kar isma +kalk brenner +je ph +hot stove +her ath +ha stie +gro pe +gor ski +ga j +fene ch +fe ckin +f tii +ev ac +epitom ises +ch age +bran che +alim ony +⼠ı +wind shields +wi a +voltronlegendary defender +tulsigab bard +taxi driver +take action +swee et +spe yer +save ourocean +sarahk silverman +s dram +power of +phys icals +pe gi +pa ks +orec chiette +o ec +na vid +mn gop +mi ers +mel lie +lei ber +kit v +kil is +katah din +kar lo +jeff ery +imagin ator +huntington beach +hor ning +hereto help +glori aste +gloriaste inem +ghost face +fru ited +fac ci +f yn +er ac +ee h +e igg +dear ne +crow snest +compad res +charen tes +ce h +bo el +back rest +b fb +ariane space +alter na +alexand r +aldub happy +al ink +abolish ice +ze in +window less +vas anth +val entia +united for +tj x +tin ctures +thrift break +the mac +terra sse +te pco +sü d +so cr +siem pre +se sports +sar gun +reyn ol +rajasthan royals +rah mat +pro shop +phra se +phai don +per abo +p chat +ouri st +om nit +nex o +nebu lizer +nand an +more days +midwi ve +ll cr +li va +leadership matters +koto bu +ko he +jupy ter +jan ec +humer us +hindu stan +hac en +h mi +gun da +general mills +ever blades +en stars +dr ang +div is +di anthus +coll ated +car ditis +bu se +best musicvideo +au secon +alt as +ale igh +al tered +acet yl +wal ney +vintagec ar +vi pass +v qa +top billing +tac tician +t dt +t ct +suf fix +stu f +stick le +st ä +shen yang +se vend +ro sat +real talk +quil ty +pittsburgh pg +pin os +pil ly +perenni al +penni less +pacio retty +or onto +onetimein nola +off al +near ly +nau sic +mob wives +mandi ble +man ou +mal ing +jun gs +jer obo +je zz +je wett +hust led +hed man +fe tes +extr atv +dob bie +defi b +cory gardner +colum nists +charlot ter +certi fying +carne iro +book addict +blogger swanted +big fat +bethany joylenz +bassad ors +bas sam +ade el +ach ina +ľ ล +ðŁĶµ âļ½ï¸ı +ä» £ +ãĥī ãĥĥãĥĪ +ãĥ ´ +ye ver +world mcqueen +vir als +usav olleyball +to scan +time tables +thof july +te agle +takay ama +sunday brunchc +su port +solidi fies +shannon r +se cops +sch y +ru bia +ri gel +ravil ious +q ah +prof ited +pom mel +patri zia +paranor man +papuanew guinea +ou ree +nu u +neck piece +nag orno +mybe belove +mou thing +mot els +mar kelle +man city +maeste g +lu jo +jo sip +ihe anacho +hi dro +han se +go jack +fr amer +favor itas +down sview +cz ynski +contemporary painting +communic ation +bbc worldcup +ay be +... ðŁĺĤðŁĺĤ +) ," +ðŁijĮ ðŁĺİ +미ìĬ¤íĦ ° +wh darts +wal kr +vin ing +val spar +u ww +u ur +truff le +toe ic +team solomid +t tos +sub ha +so tw +sebo gier +sad dl +rob ina +re drawn +re counted +rand ell +pur slane +pt safety +pron ghorn +music ology +mur doc +micro transactions +massage therapy +mani kin +man museum +mal ley +mahersh ala +lion day +la pa +il x +huy ton +gugli elmo +gu de +for my +eu less +ess ure +elec tives +ea ste +dress ler +do et +dawn richard +dani ell +dan adel +cigar citybeer +ce ment +blue peter +bio based +be vis +b kr +arat ne +all ons +ai katsu +afri kan +ab it +ãĥĪãĤ ¥ +м и +visualab stract +un agi +ty t +tri bbles +the be +stau bach +so bbed +skel mer +scribblen auts +s fi +ride for +ric snews +red poll +r nation +quetz al +quat ernary +oce ano +noso tros +moon shiners +mika elson +mer rie +mam u +macy sparade +k ue +john wall +jay awar +ir fu +hu bli +h pc +gauri khan +feren dum +ejio for +eag lenation +batt alions +bajiraomast ani +any place +ann yeong +ang atta +af ton +:) "@ +ðŁĮ Ľ +ze ek +z ato +to ph +tin ta +te thering +sustain ment +stro ma +strading cards +still well +steven son +stepan ek +stella artois +so con +shi mada +se adogs +sabot aged +rob g +recuer do +ophy te +on nit +oc ceren +nap anee +nan os +naj wak +mis sal +marvel ing +ma hu +livel ike +ko co +ka isa +jimmy buffett +flo e +fla gler +fin dley +far row +even son +encapsul ate +el pi +dor ner +dhar an +del fino +de stro +conflic t +ch arest +canadas occeren +cad i +bj sm +being united +ban anas +audios lave +ati more +>>>> >>>>> +äº ¬ +âĿ¤ï¸ı ðŁĴ¯ +wx guy +wo akes +w ts +tro hman +tn pl +spice world +soft cover +row en +rock ingham +pul teney +plo ck +pa pped +note ven +mceach ern +manipul ations +loo keast +le mn +krist offer +jazzy b +iv ories +is am +i bus +horn ung +go in +gill man +ga iney +dis oriented +de akin +d wd +chlo ë +cannabis culture +ar oldis +anglesey scmedia +am iner +ø rn +yo v +woll stonecraft +weather live +vit a +uni do +tram adol +to inspire +sothe bys +shin ge +schne ide +sc ally +sack ings +sabc newsonline +ry bak +ro ku +reiter ating +photovolta ics +part es +north star +mu ar +moore stown +mondele z +mel ter +mark ley +mariab arti +mariabarti romo +lorela i +kkkkkkkkkkkkkkkk kkkkkkkkkkkkkkkk +ki ani +jo en +ili as +ian h +hy des +high times +hen ke +healthy choices +hat ley +grand finale +gh ali +dr strange +dog sat +dj al +dictat or +cor responds +cool more +com promis +br ø +bas son +b flay +b du +arri ver +apar ty +an ting +an schutz +af tal +ðŁijĩ # +âĺºï¸ı ðŁİī +wom b +wag staff +tv i +tss aa +shigh ered +seymour duncan +ser ra +sch lager +row ney +pll tvseries +oskar blues +ola bel +nd spdwy +mo ai +me tric +marke ts +manic ures +key stone +jor ja +jo gs +hoo kups +gre entree +gi est +geaux colonels +f ford +en b +dra is +dow sett +d wood +d fi +cost of +coral ville +city lights +chicagol ndspdwy +chic lets +ch acon +cas sy +be ps +ave z +au sf +aristo cats +ao ty +alme ida +ali ghts +ale ss +adsor ption +aac ta +[ © +ðŁįĭ ðŁįĭ +ãĥ Ĭ +à± Ģ +र à¤ķ +yum miest +wi ra +what toread +uki yoe +ton hall +ti oman +th ira +stol z +spon gy +sitt ard +sick notweak +sa recool +rott entomatoes +rock well +road tri +r dr +puertor ico +primiti ve +phan togram +par igi +mar ant +led ford +ko tel +kell yo +ke aven +hyper link +huns let +humi dex +horse box +graham stown +g alls +fr l +fa eli +du sts +down ership +de pp +da aa +cy t +cu ld +ch d +by gones +au v +ar tre +ar lin +... ). +! ðŁĺŃ +zu o +yacht life +win dup +ut knoxville +trumpre sign +toma hawks +swe ated +sw pl +stress relief +sti dham +so tt +sharemy cosplay +roger son +road trips +rhe em +plen um +peat lands +our house +on cers +old hollywood +mp inoe +maz um +lessi g +lay f +l gi +krish na +kail yn +jo ek +incub ating +ilovemy dog +hei dik +h wt +gun smith +goooooo od +g vp +fu mbling +fru strat +fis ke +f mu +esp in +encanta dia +e commerce +dal ec +cou sy +cal mac +byron bay +as ner +ar len +an anth +ðŁİī . +ãĥ¼ãĥ ł +âľ Ń +zebra head +wwe ajlee +vipass ana +trade winds +teoti huacan +ta jin +stud land +skir ball +sand ymount +resc u +repell ant +pre g +pou lin +pa quin +p bd +mondi ale +mis smal +micro grids +met formin +me ber +low carb +kw gn +kil n +kiku yu +kett les +intercep tors +fr ant +fagi oli +f bk +eri ke +doo b +d mexco +clever ness +clai mant +chatt ers +bez els +ban sko +af oul +ðŁĴª ðŁĴªðŁĴªðŁĴª +ðŁijı ðŁı¿ +zo an +wur m +viz ha +v sn +up end +twin sburg +trac ee +tower hamlets +theip aper +sun rail +sm om +sheet metal +sepat u +sel bst +scri sis +sco tti +schoo se +saturday vibes +sa pe +sa ori +regi e +oti eno +ntv today +mk ultra +micro waves +me del +man ta +mam bo +liv ro +le conte +krush na +ki ku +ke it +j mw +inhal es +industri a +heral ding +her op +ha al +gros beak +grange town +fon zie +du soleil +do si +deliver ables +dar roch +chi ve +brit athletics +be yourown +b bl +ar ge +antonio brown +a ure +âļľ ï¸ı +ÙĦ ا +zero waste +ww l +with your +windy city +wi gg +wether by +wa is +wa hy +thor ns +the over +th uli +sun kist +su bal +sport stradingcards +ski l +regen cy +ray bould +pin chot +pel ag +nikki galrani +na iro +my garden +moom bah +metro fmsa +mari k +log es +li sson +kn aus +kend i +ip so +indian ola +in tha +h sin +grey lock +gan se +fraw ley +fin epix +esh agupta +ene me +disin terested +dis jointed +cre vices +council of +cap le +calvin and +bird box +big d +bar thes +are volution +al ympics +ðŁĻĮðŁĻĮ ðŁĻĮðŁĻĮ +ãĤ ª +| -/ +zing y +zar athu +young living +xfre watch +val lotton +us as +up dike +un paved +ten ey +swk play +st james +spur snation +sound bar +soho house +sn icker +smtown global +shun ning +sen shi +sem tex +s zab +recomm ender +ram bo +radio logical +pre prints +pent ag +p bal +on ni +o sn +nom er +my story +movie quotes +movi l +mc rib +mateo guidicelli +mal apit +mac phail +lat ched +land rights +kr ann +khal eda +ked by +jo deci +harro p +gethse mane +ff dp +eyewitness wv +est elle +en demol +cmoh ry +cam illo +ble p +bio similar +bat ard +bas ant +ay ud +awesomen es +alber tope +adi ya +ðŁĴĥðŁı½ ðŁĴĥðŁı½ +ðŁį IJ +ائ ر +vat icano +va id +under pressure +slu mb +shun s +she ahan +service able +seguri dad +sau da +s gin +ri am +prep sports +pol ices +overestim ated +oko boji +nick kroll +newtown ards +marchof dimes +mar sa +lip as +kitu i +king sholm +intersper sed +inter webs +iiii i +engl ert +el nik +di kh +den v +defend theland +char an +c ics +billo frights +bern ards +beg one +ag ana +ì° ¬ +ÙĨ ÙĬ +ye ong +wheat on +vid han +ve spu +v indiesel +transform er +terra form +ta kk +t lou +t ago +sun block +span ol +scri p +rudrak sha +rhyme sayers +rest lessness +pl oring +photomon tage +periodic als +pen with +ou to +os ram +o gh +mul berries +mo wn +min u +majo red +mac aws +lon gridge +lesm is +kiren rijiju +ken way +harmon ie +growingup with +googood olls +gle ich +ford models +fo gs +eto sha +e wart +drjimmy star +cou pes +chakra barti +car ms +can not +bol stering +bha vana +auto focus +af elt +a uro +ðŁIJ ¤ +å¤ ª +Ì µ +x ab +wicker sham +tu mn +ten ch +spe ts +sage summit +run about +raw ten +rap turous +ram sar +pic kets +pantan al +oun e +nu yor +nu tting +mu bad +mor tise +mc guin +mar sdd +lu cey +lingu ists +je thro +free people +forget fulness +e zzor +dis regarding +del monico +cyber man +coldwar hist +cloud security +clo vers +clo stri +cdr kelly +brew ing +bra ssy +bor del +bill ard +be quest +b pr +apothe osis +am yn +al oma +afgh ani +ae an +adidas running +a stigmatism +a hahahahah +ðŁij©âĢį ðŁĴ» +à® ļ +who dat +whit sunday +uz ha +universal is +un garo +ty rus +tur geon +terran ova +tan war +stam pede +sport sphotography +spal omas +sam pa +revers als +re tracing +qab oos +power scourt +pi vo +petro chemicals +olive garden +official ap +nh c +mer z +marque try +m brs +kir ton +kat ra +is ser +infer red +improvis ational +hu ey +hi j +haringey council +har pa +ganon dorf +gabbi garcia +full sail +fol tz +fly laxairport +fertili ze +ec a +e gc +du plessis +di w +d mh +cut ter +condolee zza +car reno +bothvote ssnp +bit ton +bi atch +bethe a +ber tens +bar ch +bac al +at ras +ashok gehlot +artist oftheday +am and +af ai +wo den +wfu v +t pf +syl mar +subjec ting +sub woofers +steve madden +station cdrkelly +sen ter +sea star +sc f +sal wa +sa af +ro sina +red path +recy cl +preston wood +poisoni vy +percu ssive +page ant +over dosing +not atarget +nb colympics +msin spire +milli second +masa shi +mary vale +leg om +kille brew +keep talking +ist ill +io annis +icici bank +ich mond +health matters +guar ana +goun a +gon ne +ginand tonic +gas coyne +first dates +fc art +f pe +f mv +du le +discoura gement +diam o +cu mann +cougar pride +c sac +blu bber +bettere veryday +archite ttura +arachno phobia +an cha +aly x +ðŁĨļ @ +éģ ĵ +à¹ĦภĽ +w basketball +ve rena +ut ani +usair ways +tupp ence +triple talaq +travel inspiration +the gentle +tan jong +stor t +sport shub +skelmer sdale +sel igman +se aley +sb ath +rncm voice +rad boud +ra jouri +ponti anak +pic he +pic anto +pass book +p illion +ng sop +music live +mul hall +moz hi +michael vaughan +mc glo +mantel piece +laun dere +hime ji +gooch land +gif tware +gfr dofficial +ft one +fre tting +cor nett +ci oni +chal ks +cadogan hall +bro mberg +ble ep +bill iton +aubre yoday +arca dian +a ile +ðŁĴ¯ ðŁijĮ +âĺķï¸ı âĺķï¸ı +z ick +york racecourse +wolver ton +tow nies +tor tures +tol ly +thor old +territ ori +tan veer +ss at +spam med +secret service +roger waters +ri pp +relo j +r car +q mc +politi que +po inter +ous life +open stack +official wmas +my coop +mam mut +liveon news +la ity +jami el +jag iel +investo pedia +in vier +i zzi +hic h +h inews +gu c +grisel da +fidel castro +fear fully +falling skies +f lec +e mison +dag mar +clu bbed +clair mont +cell press +cany oning +canad arm +bv g +buy in +bio sci +back shall +aga ins +zer g +wester ham +v and +ui w +tour downunder +to shiro +ti ao +sun din +stje pan +ste red +so tc +sarab ia +regrett ably +re activate +potter heads +p boc +on yango +night core +n mf +n drc +mye at +my gov +multi plicity +moore head +mid am +metat arsal +meath gaa +love film +kurunzi za +jas a +j mj +j mac +im parts +home business +hac cp +gr attan +equ alization +ent als +e vey +du bb +d ch +cric kho +cookie day +constitu tions +colo strum +centa urs +camp i +arche typal +ap rs +along korn +all Äģh +- .. +ðŁļ ¢ +zu iko +zeen at +you cant +wy le +the er +siri sena +rud der +ro emer +radhikam adan +pre server +pl onk +ol at +oak y +nus rat +ne pad +my all +mu thu +mar chive +lu pins +lin don +ky ra +ksenias olo +kham is +hawaii five +har pur +green side +greatnorth run +gollanc z +fioren tino +dub ga +caro wozniacki +car illo +brexit ers +blizzard of +bird art +bel den +be urs +bar anski +bab ka +b ci +az uki +ani zed +and ani +al kan +ac worth +ac credit +y know +xy litol +wee m +the mahirakhan +t illi +swa thes +s atta +rocky horror +rain ford +ple xi +parksand rec +paedi atrics +mt c +mon ro +moissan ite +list as +kri sta +jac ek +iw ate +it news +iron work +ing team +hima wari +head count +gener ali +g pf +fo kus +de porting +david warner +d pr +cut thecable +comman deered +cen taurus +cc d +candi do +ay p +art market +ah h +turan dot +thi elen +test kitchen +tar ab +sound clash +science march +s ooooooooo +ro mping +resurrec tion +print out +post gres +pere yra +parake ets +nico tero +new releases +mtl moments +medic als +lu issuarez +li pps +ju ggles +jo si +int j +hot ell +grand theft +dur arara +diyarbak ir +digital drawing +dge tt +clean energy +cd ti +cann ula +bull winkle +bu ssell +bu kas +biddul ph +ba cha +b win +av ante +anac mo +all infor +ye tu +wr al +way togo +vir us +trait orous +thyssen krupp +the shield +the hard +tal ot +synago gues +stubborn ness +stou te +smoke screen +sh anth +se ku +sapbusiness one +rei ka +re sus +ratcha thani +ratcha sima +pol twt +op f +oci gars +mini bar +may sville +laundre tte +her ded +h ool +fuji moto +fati ha +entom bed +eagle snation +dissemin ated +cuer po +co jo +clar ington +carbon fiber +blu men +blaen au +beach am +back stabbing +assess ors +ap bs +any ar +ano te +and ino +ail party +ac supdates +ðŁĶ ° +ðŁİī ðŁĺĬ +you sif +under writers +ugand adecides +tur ro +top knot +to ren +stre tto +smi f +showusyour builds +shi vam +sex ed +sensiti vities +semio tics +sand ilands +san lam +qu avo +portu gu +pla ye +pare ido +ound theworld +myo fascial +my yy +mik al +macfad yen +ko chi +kin care +kapo lei +k sports +jag gery +j assim +irraw addy +in os +healthand wellness +golf day +ghan ds +freedom works +fah mi +europe day +errone ously +do jo +cze wska +col linson +ck ing +cimm yt +ceu ticals +ca esa +bu youts +bu ssing +boys bball +bi ber +ban es +ati one +anti hero +aneury sms +alien ating +ðŁĺĶ ðŁĺĶ +ãģ ĭãģ +âij ¡ +zogh bi +v cl +v angelis +unzi pped +under current +thebig bang +tabletop day +sk en +sie bert +se fer +santander uk +sand blasted +salt dean +ru brics +rock show +recruit ment +port chester +planeson thenet +parlo phone +paralle lo +p ade +matte o +madhan karky +kevin o +kend ama +indv seng +hollowcrown fans +ho tair +gros grain +good new +gon o +getinmy belly +ge i +free agency +fair burn +dra gracing +djfresh sa +chroni xx +chad da +can u +c wr +bla ser +all right +ai zen +ac j +abhi yaan +wilder nes +wick low +up minster +trac ers +toron tom +tm koc +t lal +speci e +ske wered +si vi +sab bey +rox anne +r di +pla sen +par ma +moh enjodaro +mikas ounds +liber ates +kry sten +kingsme ad +ke sa +junior doctorsstrike +jin u +il co +ht brunch +hr sa +ho pat +har lock +hain anese +f als +electro phoresis +eic her +ei sha +e week +disappoint ingly +de kal +cour maye +com pra +cloud native +chum my +cam u +calvin ism +bra x +biz on +bee son +bart man +bab is +author isation +aqu an +ador able +ðŁĩ¸ðŁĩ ³ +ìĨ¡ 민íĺ¸ +we ta +waff en +vor on +vi vendi +verte bra +up shur +tuss aud +su hana +sply ce +serial killer +sav sl +ram mer +raj inim +queens landers +mon of +maha bali +m wi +loven orth +liber man +leap ed +lauder hill +ko se +kh long +karen gillan +ish rat +immuno deficiency +im hotep +hed rick +he st +h anny +go res +for culture +fe ets +fab bri +eun woo +ess ler +cru x +co ren +cle land +bon ing +blake griffin +bil ang +bi phobia +barram undi +baram ulla +ash faq +ar ico +al ani +ðŁĺĨ ðŁĺĤ +ðŁįĴ ðŁįĴ +ðŁĮİ ðŁĮį +writ able +wishyouwere here +where to +velt liner +u wo +tan is +swan song +star rr +sn ina +si zer +shor o +sedg ley +sar ai +sa wesome +red fin +pra shant +phil heath +or val +nise koi +mor ath +mit m +mau le +ma um +ly ssa +ludo vico +kha an +kat ec +hun ched +hix son +high wood +ha ddington +food stuffs +em pathic +econom etrics +dr w +curritu ck +cla ssing +ck t +ce tin +car neros +can ts +ca io +bus sey +assemb ler +armad illos +andu jar +ðŁĽ ı +xplo sion +wor dt +west stigers +under painting +th au +telang anacmo +tan n +suk ma +stu bble +spor to +somer ton +skol nick +sh our +sailormoon crystal +que asy +pri mo +pin sent +pickup lucifer +ot te +nationalgri dus +mu haj +moon struck +lange vin +ky umin +kapp ap +jaz i +jan ic +j yr +hu ffy +gg u +friday facts +finger lings +f bp +e ins +de dge +christin am +bor gir +boo om +blake more +black isbeautiful +badas sery +bab aji +b acy +appell ation +ali reza +al wyn +???? ?????? +ðŁįĥ ðŁįĤ +winch combe +wash ou +vez ina +tsing tao +timon ium +thermo electric +sj maas +singularity u +ron calli +rhom bus +ra by +pulmon ary +pre cht +pla smo +over protective +one k +non traditional +ngsop engardens +nest le +medic o +ling e +le baron +kam ay +ip se +hol liston +hick sville +harmon ium +gur nard +doom tree +dh ram +crime stopper +bjer g +b vr +arthro scopy +apar is +antw an +ame ss +ðŁĺŃ ðŁĺį +ãĤĪ ãĤĬ +wil sons +un challenged +thir teen +swat ting +stone age +standwith us +softhe year +social network +sd ca +sav ona +sam en +ru ggles +roc co +rick shaws +recep tionists +reak fast +rad in +r zr +pen i +nascar onnbc +mcge ady +living with +lethal bizzle +last minute +l ch +kew pie +just dance +hi jacks +gre gar +gra fts +fric ke +floren cia +first energy +dd n +cy outh +cy mbi +confi dante +chou dary +cal lista +c low +burg led +bo pp +bill haslam +believe it +am ani +ace vic +éľ ĩ +zeal ander +wh att +uz h +turbo grafx +trailer park +thro ck +thenew school +the av +tasty tuesday +sor olla +sin on +sil ico +si banda +scott sboro +saraswat ichandra +ruff o +ran sacked +ram o +puri sima +psycho tics +pin kie +pier point +pf aff +peter dutton +nucle ic +nov ambb +nico lette +nar o +metro dome +med abad +lagh ari +kol arov +king ham +ki eth +ike bukuro +id olo +get down +figure head +daz a +cros bie +conni ff +con ner +ce urope +brum hippodrome +bj m +bhogle harsha +berry hill +be ba +ang ar +amber rose +afil ms +ðŁĺĤ ðŁĴľ +ðŁĵ ¶ +аР¹ +uri ya +tsu ko +the power +takeover tuesday +seth rich +se wick +sal vulcano +ri vu +re gr +ques nel +qu re +pre mo +power book +polymer ization +poly chrome +pj morton +ouel lette +oklahom an +ok yo +oh su +occu red +nu ages +ni ku +mus yoka +moon roof +mm hg +mc cay +mall a +kre u +ki seki +kate walsh +k pu +implo ding +ib p +fund acion +du laney +dream car +dha an +cru ral +clu edo +cha hiye +butter cups +bun doran +bon go +bet ancourt +athiy ashetty +asin ine +albertope tro +aam n +ðŁĶ ¢ +주ëħ Ħ +wa ad +vilam oura +vege tal +u ag +tao bao +swi sh +shake out +shadowhunters season +sh air +selen afor +sanc tification +s ello +s dot +s bbq +ro ti +ril los +proton mail +program ing +pol len +pla sterer +peak sn +off stage +ny ra +na ima +moor croft +mil lets +mel nyk +maxi mized +marshaw right +ley ard +ler n +le elee +lat rines +jaime camil +jagdish shetty +im ur +im measurably +home show +hallo ff +h tb +go ga +exa directioners +ex omars +ell trust +ei fel +done job +der na +dark ling +chirp chirp +chatham house +call er +brad for +bon nar +ben jy +al tan +abre wing +abduc ting +âĿ Ķ +white space +up son +type cast +trump russi +to ws +theme park +stitch fix +shi i +sat er +register tovote +q lik +pivo tal +o et +multi generational +medic ina +like app +l tte +l ingly +kur ni +ku e +king fisher +jes mond +inver aray +instig ated +in ate +hit achi +har le +ha sson +gou cher +food allergies +expen sively +diamond head +cross wind +commend ations +chri stel +che ikh +chau vet +bun do +big island +ber thed +arthr itic +ardro ssan +ar dian +ad noc +???????? ???????? +ðŁı¿ âĢįâĻĤï¸ı +ðŁ§ ¤ +ðŁ¤¬ ðŁ¤¬ +âĿ¤ï¸ı ðŁijĮ +z hon +z evon +th ound +tend encia +sock ers +so hail +sk river +short falls +shi as +se alion +scab ious +sand gate +richland two +rell eno +rei ko +r rd +nokom is +my motorhead +mul und +mor iyama +monk stown +melaniel bbh +m ander +luc kett +koo per +kin ghorn +je ppe +japan town +hoku to +genevie ve +fun n +full moon +frankiej grande +firstal er +dancing onice +chan ter +brussel sairport +breakfast ofchampions +boli varian +bicon dova +betten court +arthro plasty +ðŁĻĮ ðŁijĮ +ðŁĺĤðŁĺĤðŁĺĤ # +ðŁijĢ ðŁijĢðŁijĢðŁijĢ +ðŁ¦ģ ðŁ¦ģðŁ¦ģ +ç¾ ½ +ze on +z yl +wi ed +valenci ana +val di +u cluelet +tor adio +thra ki +thick en +t sou +sy ru +sun t +solstice publish +shim mery +sha al +se ec +ru les +resurrec ts +plastic surgeon +oye depo +over reaction +oh no +nc beer +mis rule +mic ellar +listening to +line work +lin dsley +lam eness +knight ly +ki yoshi +keep your +impresi on +ignati an +ht punjab +here wego +hand cuff +half back +futuri sts +fay emi +far u +f sg +f no +e mel +e gom +donor schoose +doni zetti +dissapo inted +dau n +crickle wood +cre es +cork airport +con serve +community day +clare more +chur che +cap shaw +bubb awallace +broad head +anne e +aly and +___ ; +âĻ ļ +âĸ ¡ +à¹Ģà¸ Ĺ +yoh ji +xau usd +work top +vol ks +vijay mallya +trend setting +sou ks +sm tx +ske eters +si rr +sa very +re wiring +quil t +prideand prejudice +patriarch ate +official bantams +o wais +nike golf +mumb ait +mo gg +michael myers +mcfly harry +lebo witz +land forms +k mf +harts field +han se +gö te +gri jal +eastere gg +dri shyam +deto fficial +daily inspiration +col lation +cheer fulness +che detofficial +breed love +bie shq +bapti sta +ðŁĻıðŁı¼ âĿ¤ï¸ı +ðŁijĮ ðŁĻĮ +íĻ į +æľ ´ +âĨĵ âĨĵ +world theatreday +vo tar +stragg lers +so yeon +slo ped +skin less +sha hs +se award +restur ant +r ks +plo ad +pau ling +par appa +oung music +no life +ne ston +n sports +mn k +mil ana +me led +marin eland +manchester pride +lund quist +low party +lar kins +lanc sccc +kim woobin +kal itta +k nar +italian ate +in yong +idyll wild +he athle +ha gan +gi one +freck led +fon nbc +fer m +febre ze +excep ting +eufau la +ethiop ians +er dington +domin ika +denbigh shire +de fro +dd d +cre el +cotedazur france +comp chem +char cha +catal ano +camo sun +bu to +bel mondo +assail ant +arthro scopic +ap fel +aali ya +îIJĴîIJĴ îIJĴ +âĶ Ĺ +âĢ¢ # +ಠ¬ +x os +wre x +wig ston +vol endam +ur sery +tru c +that one +th sh +t kt +t dl +sin b +scoun try +sam riegel +s comics +rian johnson +p mn +our way +ou sting +no tw +n ko +montra chet +mo stert +michael franti +mh w +max azria +living the +lind ac +k awa +incentivi ze +he mm +hat er +gi a +free stone +exer ted +eu storm +early music +d hr +cursed child +cross road +conse il +co quette +chill ing +cam brian +bu rien +bring ithome +blou in +biom edicine +be em +bair ns +ash ab +ar yan +anti fascist +and tobago +af acts +îĢ İ +æĻ Ĥ +youth olympics +un scientific +un repentant +uk trichat +tre bles +tinsel town +tear fund +tan ak +su sanc +ste gman +sor teo +sf g +sabo teur +roald dahl +rho den +question oftheday +presump tive +pre eclampsia +pot g +plu mpton +peace ofmind +path letics +naf a +my as +merci an +mc bee +mar ka +ma ira +ly tt +ing b +holy rood +holm gren +gom ocs +evely ne +embaras sing +em plaw +doc cubus +distr acts +cuff link +cour rier +cage theelephant +bro hm +bon sall +bar tz +af tab +af fixed +adel man +ac f +ãģ Ī +é té +yar ds +wolve srl +with style +wing chun +upd f +u ecker +tor menta +te kin +te ale +straigh tedge +star alliance +st century +spy glass +saniti zed +s indi +ros alia +ren ate +raz ek +q st +pu el +no omi +nast iness +mm viverito +mini vans +meso scale +marklevin show +luzer ne +lukash enko +lol cat +lizz i +land schaft +ko th +human oids +hoo poe +guil t +green man +gi ug +gh ly +from today +fra e +fire box +dramati st +dr oner +diy network +cover alls +counsel s +coc co +cardio thoracic +bibli o +arro ba +ann un +amgen toc +af rench +adi pec +Ä ĥ +wo wwww +w ff +vau d +ultra violence +u wt +the grdc +ten shi +tat ak +sunning dale +se et +scar diff +re order +push ing +pit ti +pic colla +pe stering +pal eta +notthefake svp +non human +ndu ja +mis steps +magnifi que +ma sta +light weights +lego set +ke ti +je g +ge dung +gag ement +fert itta +female founders +fal ke +du bo +dir ksen +din kins +de con +coo ties +concu ssed +chang bin +brian mcfadden +bel ford +begon ias +bal ch +an gra +ab log +... ". +ö n +we gman +wawa w +wav ves +u bo +take o +sto cky +son der +so ts +sle n +sky sport +sik ander +sar r +ri ana +pre order +ou p +on ore +om ax +off ramp +oc sb +mira flores +miki moto +ma res +lau rene +land sberg +ku u +ke mble +k les +joelo steen +j cb +i wak +hon o +hog weed +ho reb +hec ate +hammer time +ham pering +gossi ps +for ays +fall er +fai thin +equ ating +do deca +cott le +conom ic +con flu +chim ing +brook shire +bre an +br ani +bio geography +bay u +an zio +ðŁĶµ ðŁĶµ +ðŁı Ĥ +wy te +uuu ut +ut pa +un scr +tro om +titanic belfast +than niversary +ter schelling +sw ang +sur ge +smu dges +sin ton +scal ding +sat guru +sar py +rug ge +pur itans +ou zo +mukil teo +mechanic sburg +mall or +m ø +lim ite +lets make +lari mar +jaz ak +ir uk +init towinit +har tung +ha berman +gm fb +fri as +dysen tery +cat walks +brough ty +boston ian +ben digo +at je +ashken azi +arizon ans +appliqu es +wool ery +witch ery +travi stritt +ther un +the matildas +syracuse crunch +sun life +spl m +south london +sl ac +shur r +sa hs +rush cliffe +ren cy +reali gn +rab ada +r ld +preak ness +pl anta +over growth +mumbai kars +mpo fu +mozam bican +match s +mat ra +le well +kam in +jonathanr knight +j sr +intram urals +impressi onists +go sha +gau d +for u +for gov +fireal arm +fi xx +farmers journal +cu mber +cod fish +ch agu +celebr a +bulldo zers +blackfriday deals +ber shka +as ri +ark ell +ak ie +ad asi +ã ĭ +âĻ¥ ï¸İ +âĢĶ & +yoko suka +wi sps +westh ill +wc pt +vivi ani +vir ga +un days +tad ka +sre eram +squ in +schar gers +re introducing +phlebo tomy +peaksn valleys +pe cked +paid media +or lv +oke mos +nt p +no vic +mr peaksnvalleys +mad ama +liber ators +kam er +juliag illard +j nr +im ts +igh alo +hemorrho ids +ground hopping +go do +ge ely +fromthe field +for today +female filmmakerfriday +embezz ling +duc ato +dor ris +charlton comics +chann elled +british moths +as prey +art prints +adel ta +ìĿ´ ìĬ¹ +à¹Ģภģ +work at +wolf son +we sco +vin h +un cultured +tech y +tam ayo +sonn enberg +snar ling +sma k +se vers +s ads +prish tina +ped ne +patt ymurray +nr x +moon raker +mol an +mend elson +mcgu irk +martin omalley +kwad wo +i um +hor naday +helicop ter +gar an +en ot +discover yof +chin en +cal fire +british gq +brain waves +blue tick +ber ube +bent leigh +be aware +ave iro +are va +an unci +al sen +âľĮ âĿ¤ +x illia +wwe fanart +vigne sh +twy la +the secret +the poke +schi edam +saxophon es +pop sugar +page sen +outsi delands +nu da +nationalhot dogday +naj wa +n sac +music for +met as +lovemy life +london midland +la che +jar amillo +indoctrin ated +ib min +i dream +hellomy nameis +gn om +fur lough +fin kle +fil lu +ely xion +dug ong +do ren +ding ley +desc ents +day one +chateau neuf +charlie sheen +bread fruit +ben havn +bab oxing +ba rea +apol itical +ahu ila +afro pagesen +ad news +çİĭ åĺī +âļ¡ âļ¡ +اÙĦÙĦ Ùĩ +womend eliver +wis ma +v ro +us and +uk homeoffice +trinidad andtobago +tony stark +sweat y +stay healthy +sor chestra +smo thering +rober te +pra veen +poo m +pocaly ptic +obli ging +neg ati +na hs +n mc +man ville +ma zie +longlivelong mire +leg i +le ite +k hol +jun ker +joh anne +ja vert +j kr +inte c +hir on +heyn ckes +her mosa +hair goals +h mann +gaw ande +famil ly +fai za +dental implants +de haan +cat to +cas k +brock hampton +boon dock +as sive +arc i +aguil as +ðŁı´ ðŁı´ +åľ° éľĩ +yokai watch +wer chter +vil lette +va as +ubi quity +tor in +tol ly +stlouis rams +spo elstra +sn fonnbc +sco ggin +rahe ja +phenomen ology +ph reno +pau ley +nb ach +nati vely +napp y +mun g +major ities +lin ic +ka al +jordin sparks +gro ep +god head +fo ye +flavon oids +extro vert +dal ymount +com une +co hiba +bur sas +biz jet +bar maid +ar dan +amand as +aban ks +ðŁ¤ Ľ +âĺ ī +ਠ® +wi fi +vin tner +symph onia +sv su +stadium series +shash tag +recuper ate +ran jith +pe son +nun cio +neck wear +msport ltd +month ly +mont calm +mk tg +melo y +master softhe +mar gulies +mam noon +le bar +kro m +just jared +jessic ac +im pairs +im ited +i we +ho es +hi da +gu th +gott ago +fan sclub +e stancia +do olin +dcre birth +dash t +cou plings +compac ts +cagatayulu soyy +caesa rea +bootle gger +bar rington +al ak +å® ĺ +à´ ² +Î ³ +ya el +wi bc +wheel of +web tv +wang ari +wall en +uni onize +ther ise +the kitchen +swel come +stra dale +shannonr watts +sel anne +scap ular +san y +sa kic +ry dal +ru mped +pe mba +origin ation +ok er +mo til +mccaf ferty +max mara +mathe wson +li ken +lagun e +indiec ade +high bridge +hahah hahaha +hab toor +glass works +georgehw bush +fe ud +ez ine +emm erich +em mental +econ et +e og +dy ffr +du per +dissip ating +conversation edu +comm ittal +circu lator +bi sher +barcel o +bad azz +ay un +arm chairs +and on +ah luwalia +ภŁ +yd ney +whe eze +water lo +vi xen +vi are +ver ment +us oftball +un proven +u stad +tro xell +thank sobama +ster io +senior day +sal ir +rev amps +re married +re apply +pen umbra +pedan tic +oo sh +ok no +o gres +nz lv +no ch +newbury racing +nep sac +music ares +mo efcc +me ows +love grove +lavo ie +kevin bacon +jaco bus +j aki +infl ationary +hoo g +heg seth +gulben kian +fraser burgh +enrol lees +endo vascular +disc loses +coss acks +conju gate +comple at +call ender +broc kie +br ora +biolumin escence +belle fonte +beat s +bar sha +ax schat +atho lic +ap lan +amy acker +wi politics +wann ables +visit or +u shi +tor rey +ti fully +the shining +taxider mist +sw en +str ymon +sth attweet +rush hour +rh b +rebu ked +real betis +potat oe +phil us +petul ant +pe ten +pap oose +mon aro +mo sle +ming na +lei den +lec tro +leave your +kyan ite +kre t +k cra +jls official +inordin ate +in can +i die +huguen ot +horn castle +gluten free +fore go +emerald citycon +desecr ated +deadby daylight +danny devito +cu ero +coin marketcap +cavern ous +c fi +buenas noches +behavior aleconomics +ate am +at rip +adjudic ation +aby ssal +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +ðŁĴħ ðŁı¾ +à® ² +world pressfreedomday +wheel don +water lily +v db +ul t +u am +sin ha +shah naz +scu ffed +scu be +sac ro +rol la +recon ciled +ran noch +pl ong +no joke +no bbs +muzaffar nagar +multilingu alism +lor ber +kis son +kill shot +jo les +insu la +hondar acing +hel lion +he vc +gri e +gracie bjj +glbl ct +flaming lips +en f +el mi +disneysm mc +craw ly +cost liest +cave men +car pi +bon ar +bal ad +ash trays +ar mm +a hia +ðŁĴŠ⾨ +ðŁ¤· âĢįâĻĢï¸ı +íĥľ 민 +ãĥ ¾ +yesu das +x sw +wor rier +vo kes +vi ano +veronic as +ve ered +tr anc +tl j +ti bor +ten i +tatt n +sober look +simon pagenaud +shin igami +sei ji +s bo +ren na +palak muchhal +mu cos +mit nick +misi denti +meta verse +ma zen +kil ner +james joyce +hol ding +hik vision +go bs +garh wal +fanta stica +e za +du lac +der mic +dan dies +conju gated +cil ia +c team +bri ssett +bf n +baw den +aim less +ðŁĺ ¦ +âĺķ âĺķ +wap si +visi th +un ironically +teach foramerica +tan ey +spring brook +showand sell +show up +scra ft +ri mac +rep tile +ram ires +radi or +pure foy +pal ong +oke hampton +ne cking +mun t +milla jovovich +mc millen +malaysi atru +lu ch +lin ker +ld b +kon trol +jame sh +info graph +infinit um +ib ers +high brow +hetero chromia +he th +grant land +gra il +flat bush +fl v +edge ley +dee wane +dan ko +cr anny +comi ket +bur rs +brum mies +b uni +air way +a press +ãĤ Ĩ +win stanley +vuj icic +vote the +vali um +v dot +un forced +tra bant +tony kanaan +time shighered +the beat +sau ton +saravan an +sarah palin +ribe ira +rhi anna +per rine +party hard +paragli der +olympic games +nas er +mckel vey +m cau +kidnapp ings +khali faof +ke gv +iso des +home ent +hal ber +great job +g cp +fresh food +dom enech +curren taffairs +cra g +cor lando +butter beer +boat sthattweet +bo wn +bbc devon +bal dur +bacchan al +actu aries +ðŁ¤Ł ðŁı» +wi kia +ver dean +un cf +ty dillon +todayi learned +teu tonic +te tus +speed ers +sou ter +soon young +shah bag +sd f +sauce do +psy ang +pri y +pote et +pheno typic +phar os +par khill +osten sibly +olin ari +ne tw +ne mi +mother of +miq balkhan +mill ay +may on +matti son +masa k +lu ss +le ci +ky la +kor man +katamarayu du +j me +glu teus +ge ylang +fitness addict +fish y +dro poff +devi ants +convolu tional +chrisy oungmusic +cat ra +carbon ation +bou ghs +beat in +bar to +b pe +as chool +aqui fers +am mons +aker r +aig ner +afri forum +ad ame +ab dus +ðŁĽ ¡ +ðŁijı @ +ìŀ Ħ +ëį°ìĿ´ ìĭĿìĬ¤ +y alla +wicke duk +we itz +u pe +tren ds +trans liter +trans am +tr boxing +theo dore +tamar indo +si def +se lek +sch ä +ra st +pot tawat +plun dered +ni wa +nap les +my na +man child +kaiser chiefs +j de +irishmusic party +invisi bles +inqui red +imbe ciles +hor ace +havas u +fever tree +fe urope +f ca +ek k +cint as +cgc comics +cat elyn +car care +bun crana +birth control +bell wether +arn prior +ðŁļ IJ +ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ +ãĥīãĥĥãĥĪ çµµ +âĢĵ # +ಠŁ +{ { +worldstar hiphop +water slide +tz ler +tur us +tre ll +tommy robinson +tech jobs +ste phie +so wer +sar the +ren tice +real deal +pugli ese +our town +nu ern +novel as +nick toons +national zoo +mro lym +mel ding +mathi eson +lam berts +killer mike +kend ell +kate spadeny +indone sian +iale gis +hillar ys +ghol m +fichtel berg +dou gan +d hok +cro wing +con oce +com y +clondal kin +carav elle +cam pion +buzz es +bur slem +bra wny +bn m +barcell ona +appen dectomy +af ranc +ac chi +! ðŁ¤Ĺ +zooey deschanel +vintage showandsell +valent ini +turno ff +the x +stren ght +so har +sankran thi +ru iser +rober tb +quar ks +promp tattn +ponti ff +open suse +mtv base +mobi kwik +misdemean ors +marcu sle +magne tics +lovin leeds +kup wara +knife point +ke un +john l +it ay +inta wak +instig ating +ha bb +gray ish +geme ente +ge zi +gar onne +football index +fe is +fal es +evel ina +divo ire +dissu ade +cartoon ish +carry all +blame less +bigsky fb +bal intawak +an in +achak zai +accumul ates +( Ëĺ +âĢĵ > +³ ´ +yng wie +weare portadelaide +wat ashi +w to +w ka +victi mization +un steady +tri st +te gu +t the +sun ing +sports net +spell check +six teen +po do +mu ta +moz army +mar ta +malaysiatru ly +lon dis +infor me +hog sme +hi bbs +fisher mans +fid ler +estre la +do ee +d ss +d mca +cu l +con gen +causewere guys +camer amen +bun go +bmar czewska +blair gowrie +bio div +bett or +asth matic +am ru +aj ade +ag bon +abrew ster +: (" +ðŁij ĸ +å® ĩ +اÙĦجز ائر +win ny +v pg +v lei +up en +tu babu +track day +thierry henry +temp ts +tath lete +sta veley +sel kie +sc ampton +s berry +reyn aldo +recu r +pundit ry +pre loaded +pis cine +per gamon +par ada +oni verse +ne ame +mom ma +mc lean +matrimon io +ma spalomas +letit snow +ke ady +i is +hri shi +ho axes +henri ette +head pieces +gun ne +green house +glen cairn +gior nata +extermin ated +exi ste +econom ia +dja fro +dav ichi +cup ar +clinte astwood +cho ti +bla is +bethe match +ba hati +al thy +@ @ +ðŁĴģ ðŁĺĤ +wl m +twi sta +trul li +tra inst +tali tha +t df +si bo +seaf oo +se j +schre ck +sal dan +ren ge +re tren +r nn +r hul +r alli +prayfor boston +pp ey +pop tarts +p bf +ouse burn +of truth +no homo +night side +n wh +mor to +mor gs +mari us +lager tha +kick z +ki g +kal len +ju len +jo zy +hur lingham +huda beauty +he ter +hardy brand +gab aldon +fill in +fe de +encephal opathy +door dash +defam ing +cr ven +cor una +colt snation +co qui +away day +andrew gillum +alderleye dge +ak r += >> +* ____ +âĮ Ľ +à¸Ńภ° +vol ve +to read +ti po +termin a +tau r +stan tec +smi ther +sar ic +salv aging +pre biotic +pim lic +perth wildcats +ow an +on evoice +old boy +oak tree +n gar +metr onomy +me sis +mar ken +malaysiatruly asia +malaysi agp +loom pa +leg er +laur in +land in +kan ak +isi dore +ische mia +induc ts +in italy +i mec +grun berg +exordi umin +cy steine +cra sher +clut ched +cho wing +bran ston +bat aille +bal main +aw in +asce tic +art scouncil +aho k +adam sandler +ðŁĹ » +ze et +yu ca +way nes +ver onese +unequi vocal +to stad +the middle +ter rel +taka ful +supp ing +sta f +st fx +sm acc +selfdriving cars +revolu cion +pga of +nf caorg +neu trinos +n gun +mor is +maz embe +mall ick +ma aj +lu tes +lovelan sing +li pol +lear ts +l hu +kodi aks +ko wicz +kit tery +kill iney +kam mer +josi ah +home schooled +hoka oneone +hm fc +ha bak +gy asi +gri bble +gen na +eter na +eller slie +disney jobs +dan slott +cu ar +counter strike +compo ser +cle re +cetace an +bag piper +bacteri opha +ðŁĺ©ðŁĺ© ðŁĺ©ðŁĺ© +y hs +woo h +will man +up close +tele hit +swi jk +subscri ption +sl ang +salt and +s bakery +real kurtangle +qual trics +proce so +poly thene +obe sity +nom an +nis sin +nal ini +n sn +n acs +muh ney +morning mika +min gh +mi thing +math schat +madel aine +ma pl +louis burg +le vu +ke by +jo ely +ingle side +indye leven +healthy skin +gu ises +gofor it +go forth +fren ds +forfe ited +epidemi ological +effe ct +dayafter art +dag estan +d to +d fa +cu mp +cri stin +con founding +co che +cho cl +celeb juice +ap m +anachron ism +aar yan +ðŁIJ¾ # +zar ya +warrior wednesday +vor m +vir ate +ul zzang +stol lery +snor ted +sin gel +sar ath +sach dev +pi az +oth ate +new sam +mr j +litt len +legend a +leelan au +le mos +last minute +jig saw +in le +hunter x +gh al +flirt atious +egg shells +deion sanders +dedic ated +dece m +de ce +cor no +chit rang +carol l +capit ale +bridge man +blan che +bi jan +back inthe +aro hit +append age +annu alized +amen hotep +usl pdl +un y +un remarkable +un dra +tun ku +tra den +thor nes +ther hino +the hobbit +stemc ell +standardi ze +so dom +shatter proof +sen mikelee +savannah guthrie +saip alla +royal marines +prem giam +pastic he +ove reem +outdoor life +out classed +nu ts +mün ster +mike o +marsu pi +li bri +l sv +l aci +kin sman +kegv raja +kabb fox +jordan b +il r +humph rey +gui led +guer rilla +green street +free ware +fl inn +exasper ated +esp en +equ animity +endo fan +dru gging +dialec tical +debby ryan +cu au +cro martie +corri da +con junto +colloqui al +co king +cam inos +calvinand hobbes +c mbs +bu loh +black watch +badger cull +as syria +apho tography +ana ïs +an ami +aerom exico +ðŁĶ Ł +⼠°ï¸ı +x em +vish wanath +um r +thenext web +st roo +smol en +sa ins +ra usch +pre ck +ple ine +p tera +p ko +over tures +other world +ol ding +no strum +natur alistic +nan ook +miseric ordia +men achem +ken go +kanchan aburi +k opin +hug day +hercu lane +har boring +gar p +dg d +dab ur +cro me +cre ve +complex ed +cabernet sauvignon +bel gie +ash vik +amorph is +alien covenant +ain tre +adjust ers +acapp ella +ac cross +z es +ys mith +yor u +yaw ns +vol leys +vill ard +unab omber +to stones +stau stell +sar daar +saq ib +sa isd +ro vers +real johngreen +raff led +premgiam aren +per ish +nu un +me hak +mccon ville +latino america +lah or +kag gle +jha sanjay +j lg +inept itude +hu ch +hooligan ism +hal les +gur nee +grac es +flo ssie +fer rum +feliz miercoles +elk ton +elabor ates +dr ale +cycle chat +cri mped +cha rente +ch add +celi bacy +biop hilia +band hu +adi r +acou stically +íĿ ¬ +virtu osity +too p +spati ally +showr unners +scree ches +resto rer +ralph northam +pit cairn +pend a +paramount pics +pa cham +our an +ou tran +monu mento +maken a +mad as +lou dobbs +loc kie +li pp +l bj +k li +howtoge taway +group chat +go bo +girl meetsworld +enh of +duches ne +donthe sash +devere aux +death row +dan ske +collin a +chin edu +cha g +cab inte +blit zen +be as +bar by +auto blog +ap su +ant unes +an chester +amy lo +al esis +ãĢij # +ع ÙĬ +yard bird +y land +wil ke +wat ty +w has +un cw +too bin +tema sek +sum thin +strathro y +sto wer +sst pierre +spiritu alism +sile sia +shower head +se st +san ghis +rod ger +rd weeksary +r tv +r fe +pon do +pin yin +phari sees +pen er +nex star +my girl +musco vy +mune er +mich keegan +mcly nd +mag elang +la gasse +kw p +kis ner +josh ramsay +ho pl +herculane um +he mel +flick er +ene al +elli psis +e mag +dy sauton +death squad +deal sweek +dead pan +darwin day +craw fords +char ro +ch ali +center point +bun gy +bomb squad +beach es +be efs +ale man +adi b +yu ca +youranon news +yaaa as +watu kee +wal li +u ge +therain bow +the wall +stu die +snu gg +se kiro +real world +ray ong +photograph er +mock tail +minu et +memori alize +me hl +mbom bela +m ella +log ins +le ka +kotobu kiya +kor u +hu on +homeand familytv +glo ssed +gh ul +gabriel macht +g was +free dia +ee u +disney studios +de res +cu star +columbi ana +co iff +cc cp +car pa +c bot +bri ze +al ks +abi erto +ab baye +, + +âĺº ~ +ymoun tain +wee tz +we taski +water tower +under pinned +un belief +tvac tor +te q +stipp ling +stargat esg +spac ek +selenafor mmva +se van +s gaa +s down +s baby +rie der +redd ing +red brick +real alicecooper +re tweeter +re ti +rain tree +r mm +por ker +pl zen +pl undering +phil ander +peckin pah +northan ts +nar ula +n ise +mic f +medit ates +mc mullan +mark os +honey man +hom ed +he eler +girl ssoccer +g ws +f ws +e op +drac ut +dog walker +del gad +cy pher +chha bra +black art +bi ety +bhagav adgita +beu tiful +ber ks +bbcradi olondon +av ori +alldog smatter +air gun +address able +wells fargo +wear your +w mg +ve ux +vamp iro +tu loy +today skid +sw illy +shadow man +reve al +real deal +r wen +preck winkle +pink ney +philipp i +oak hurst +native american +mis ation +mannar ino +mal ting +lak hani +ki mora +ken ia +ic asa +herne bay +hercu lean +guzz ling +george sstpierre +fron teras +fluori dation +exc ercise +eman ci +down trodden +do tter +cor ax +chry sler +cat fight +bot te +bo in +berg dor +barbe cues +anime girl +an ind +amak hosi +al exc +air tricity +ðŁĹ Ŀ +ðŁĴ« ðŁĴ« +çĻº å£ +âĻª âĻ« +yl icious +wf cofficial +warmb lood +tom felton +tokyo ghoul +test drive +steel work +sp ahn +skib bere +sad ako +s ldn +re selling +rawten stall +r pr +pi leggi +panet tiere +ox on +nat as +n ppa +mont lake +money laundering +ml le +marath i +last week +kent st +jer ald +intelli vision +int p +ha dou +grat itude +frie za +dis member +del una +cp j +coraz ón +con fine +chur a +charli ze +cha uns +centre point +broad city +bou let +bo al +bill fish +beath ard +be dok +av ella +as sia +app ice +ail intelligence +ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį +ðŁĺĬ ðŁĺī +ðŁĵ ¥ +âĨ IJ +whoo sh +wan tt +w abbey +vom its +us band +turnar ound +tor que +sz cz +sur geon +sop inion +sam stag +sali ba +sal ary +play ground +personal growth +ov als +or leg +on no +officin alis +oc varsity +mul key +muff lerman +mapu a +le baran +lamin ator +knight fdn +ki bo +k out +k ci +iow an +inebri ated +id or +ic emen +i dia +han kar +hamidmir pak +h ran +gulf coast +graham mctavish +gr acy +gover ns +fiannaf ailparty +exy nos +esp era +elit ism +dump sters +deterior ates +cu trone +con ciliation +chor uses +chin ooks +belit ung +b wh +as ong +am dram +ì°¬ ìĹ´ +çĮ « +âĢ ı +zou is +wylde audio +w rugby +ta ina +t db +snow melt +si sko +si ris +sap in +rous se +ra si +pv d +psyched elia +pretty inpink +power shift +pol ing +ou o +oran je +one il +o ving +o gc +nieu w +new sar +never land +n walest +my bag +mv no +mur tala +muj ica +mc s +lor aine +jam nagar +j rt +image oftheday +i wk +hol gor +gt live +for climate +food pic +fik re +enda ids +elli pse +di zer +da aaaa +civil iz +cherry wood +bu ggin +brick er +boy ne +bil awal +bibl ical +bi fida +bar and +bant ay +artific ailintelligence +an ser +ah watukee +aco ach +âĻ¥ âĻ¡âĻ¥ +wedne sbury +verment ino +us rex +ther ford +ter ce +te ssie +table aux +sub contractors +sty e +six sigma +side men +sand ro +ri scoll +re ddy +raider pride +psych ometric +pro life +pal as +p sies +om are +naf ld +mis rata +mar kie +ling on +lim one +len gua +kai ba +jon axx +jo te +hr f +har tz +gra eber +go jays +go ad +ge ce +fre cce +flex or +ed inger +dot net +community radio +climat es +braw ley +boy les +boo kish +bloody mary +best memories +barnesand noble +back space +arte per +ar cia +alleg orical +! âģ£ +âĶ Ľ +اÙĦ Ø® +uof maryland +un filled +u ti +tom os +thy ne +thisi stexas +thierry neuville +ta kar +star hub +sse ur +spe ers +sim cha +shirt dress +se guro +sch ü +scar ily +ruther ford +ru v +ru skin +redar rows +re considered +oy i +onof re +oh chr +number one +nor ah +ne esh +muse et +moder no +mo ku +meet your +mat tg +ma bino +lu can +lo tro +ks worth +khalifaof islam +kendra scott +kan ae +jalli an +ik os +gym rat +flipit blue +flip flops +dur amax +drop kick +di op +david m +courtney act +cir ro +churra sco +chel sie +cambo gia +cam bus +calla o +ayush mann +apost le +ami k +am ini +air berlin +a ino +ðŁĺī ðŁĺĬ +ðŁIJ ¹ +ãģ ł +zar ina +yeeye e +yarra ville +x tian +the shilpashetty +stitch ers +six word +sco ve +sat ria +ri ek +r tu +private er +pl dr +perry ville +particul ars +one stop +no sing +miscell any +mir pur +mary lou +mart ÃŃn +mar azion +man resa +len awee +le moore +kee bler +ka is +im mo +home going +h ni +h lin +ge ma +ga seous +for dracing +ear wax +diadel osmuertos +demo tion +cri st +canadi anti +can de +cade my +burgl arized +bun ge +brock ovich +british art +bed wetting +bas o +am ater +ace hotel +aber feldy +? ": +ðŁıĥ ðŁı» +íĺ ľ +ìļ° ì§Ħ +ë± ħ +à¸ĩ à¸ĩ +yr sonair +wood berry +wob bling +weih nach +vik tori +up shot +ulcer ative +ty win +tam ithas +tamithas kov +stat ement +sta i +soul jah +shirley setia +shaw nigan +ser f +sac re +ro dri +pu tz +pu jo +press day +ph n +passive income +oura bly +omnis cient +oklahom ans +ok ri +neighbor ly +naf p +manikar nika +k gv +in vi +hell om +grat uit +fu shi +for health +extr amar +eph rata +elvis costello +eic hen +e mac +dolce vita +di eu +devol ve +d ho +d fp +car maker +brittany ferries +blon dy +ax emen +at ore +ad moni +ac q +a festival +ðŁħ ± +we ghe +val rhona +urqu ell +thre ec +the tide +siyahbey aza +shep sut +shan kumar +scen a +rwand ans +rug ge +ram butan +ph ouse +perpetu ates +o snes +no tone +nic ke +nation sleague +mid leton +michael sheen +li w +le ghorn +keep smiling +international mensday +ic entennial +i view +hon daci +h selive +green week +fal ana +energe tics +dum plin +dubga aofficial +dprin tindustry +del aine +davuto glu +cul bertson +cre vice +commu ter +colombi ais +citrix synergy +chut neys +chin as +cam girls +ca za +burden some +bu ga +biz humanrights +bhav na +are do +annab elle +aldu bin +accommod ated +wor ts +wh sv +vit tor +under lay +ta fel +son ys +smoke stack +sl á +s met +red den +re appears +q ila +pg ms +penguin india +park theatre +or dain +o ses +nic hi +musik fest +music man +meal sonwheels +mc gau +lun sford +li zumab +lan k +kw abena +known st +khal a +jamess murray +hol as +geor gia +ff ar +fer moy +femin in +en loe +ecu ador +dr ilon +disobe dient +disen chanted +dat z +cla pham +chron os +ca sei +britt an +book man +bick er +barbecu ing +az arian +artof m +apple dore +an net +an cia +ðŁĻı @ +ðŁį ¨ +ze char +wer dum +voice actor +vo lio +ve ss +the shark +tam au +sy mon +sonic drivein +shu d +s ganguly +ro tich +ro hin +rejec tions +reco ast +rebel ution +ram im +qu d +orange isthenewblack +nesqu ik +my freec +muhar raq +mr an +molybden um +men inas +media eval +mcken ney +lu iza +labor ious +kon nect +kaf r +jer u +j mb +hob bie +glen finnan +gas kins +fra gon +film school +fight scancer +di ste +con dense +burgo yne +am ach +aggre ssiveness +ðŁĴ£ ðŁĴ¥ +ðŁİ¥ ðŁİ¥ +è § +àŃ į +é lé +za f +youn ge +yo kota +war fighter +wa if +toron tonians +ti gh +the in +the avengers +termin ator +tata steel +t shep +t gh +sunid hic +simon celli +seri eb +saniti ze +san ogo +sal adin +saf ra +rece sses +r lp +pusci fer +pla ud +pan za +pan cho +offici ant +o auth +ny ama +n sr +mour nes +mil lau +mid or +miami beach +lumin ato +let arte +la pid +kre ss +iu u +ing and +g wi +flint lock +fair clough +el mbridge +dress code +domic ile +cros stalk +cooper hewitt +commissi oner +ch ah +care home +bu ggs +ay ye +ao ii +alyn ch +ðŁĮŁðŁĮŁ ðŁĮŁðŁĮŁ +ver bal +unexpec ted +tsub aki +thesun newspaper +tel lez +rdra jaofficial +rafi eh +pu er +pop sci +phe tch +personal training +p square +ox ic +over ruled +oregon coast +nine ws +national drink +mr silver +michael mas +mer ve +mat thai +mar am +machar ia +lr sd +l its +kal kan +k ch +ju eve +in kenya +hor sforth +herbi vores +har p +happen stance +han ke +fatboy slim +eus kadi +ell man +dv am +doyou even +der ren +compar tir +bulldo zed +bray don +boozy chef +bet je +ben avides +ben atia +bb ma +artofm mignola +art forum +ap les +and rich +alum n +aln mouth +ðŁĺ»ðŁĺ» ðŁĺ»ðŁĺ» +á¹ĩ a +à° ¯ +wul lie +world diabetesday +wicket keeper +unbe knownst +un ningham +track suits +sey doux +sco ff +sauchie hall +sang amon +rv hs +ridge mont +qc poli +power gen +ottaw acity +n are +mun sters +movember uk +moore field +million s +mess a +man power +m ney +ler ch +lal upra +l ere +krun g +ken i +jae beom +inf el +imp ound +h ka +go yard +go ble +gid dish +fe do +eu u +doo ku +donmar warehouse +dilip kumar +de ek +dah lin +cur tailed +cro ak +bur da +bor ah +aston martin +ard is +ar sa +am manford +abscbn sports +aalt ouniversity +? "" +ðŁĺģ ðŁĻĮ +ç¦ ı +ت ÙĪ +zi ff +ye es +water front +visi bilities +ux e +universal ity +twitch raid +tur kic +tro o +then hs +the player +tele ported +swad dling +stal kr +slow fashion +seach ange +sas usa +rub bery +remodel s +oy in +o zo +ne ee +n ght +lin dar +le ath +kal le +hert ford +hb m +gtasnap matic +fo shan +e bird +ca ius +ca haba +buck ley +bha vi +beef ing +b cel +ascen e +arcan gel +akade mie +afar mer +ðŁ¥Ĥ ðŁį¾ +âĺĤ ï¸ı +zi huat +x online +tor in +ti mus +synthe tics +splash down +sensiti ve +sd su +sau ti +satur ate +red die +reall ys +rail analysis +pyr mont +por po +plo o +pi ent +personal branding +parksandrec nbc +out lasted +or inda +myo wn +mi gori +mac lean +lun tz +lu bitsch +lon gest +life drawing +key shi +jax son +infuri ated +imp son +iate fl +har oo +feed your +ev ades +enor man +ef itness +ee z +dele m +crypto zoology +bru schi +bew ley +ando ver +abra sives +ðŁIJ Ľ +zo wie +yam mouni +want agh +trans itive +tiam owry +thsh birmingham +the os +the bigh +t dor +t cha +switch backs +ster lite +star land +so bat +sel i +se ws +s deli +ree kie +pre maturity +pre di +phi phi +oak ton +nit to +mu das +miami hurricanes +mani stee +lud ington +lethar gy +ki th +k ariz +ja ani +inthe mix +insu lators +il ter +her balism +ham bre +gü n +got game +ge di +gar rus +ff n +ed berg +du hawks +dark star +collin sworth +coates ville +cast iron +carcino gens +boo tham +augu s +ati zation +arson ists +amon tes +ì± Ħ +x el +v un +v enga +uk cyclechat +tape worm +synap ses +sul ky +sho ku +ser ang +sel ite +scu ttle +saf die +ru sev +qu anta +pan arin +outa ou +om anc +oce anc +nigeri atoday +nigeriatoday ng +neutr alized +muscle fooduk +mo halla +ming led +metallur gical +lake ontario +l jung +kun sth +kay lin +jo inte +insu bordin +insi gh +health benefits +hau pp +h mf +fox worthy +fe ingold +far mm +ex elon +english men +el frid +ef sa +de do +d ke +co sy +boo boo +belleri ve +belk nap +be ton +ard beg +air canad +! âĢĶ +âļ¡âļ¡ âļ¡ +z acu +womenin politics +wi i +volo dy +ti rico +teha chapi +tatlitu g +suppo ses +shar na +rob ford +ps one +pass iton +natural sciences +naing golan +nad al +mont ford +michael s +mer kin +luci e +liber allogic +ho ku +head banging +fur tick +fu h +french men +fam osos +expropri ation +dont stop +develop er +cu illin +cr ated +caval era +c ne +buzz tv +bul ld +bul at +bron cho +beelze bub +assy nt +american ism +ak iz +ðŁĶ¥ðŁĶ¥ ðŁĶ¥# +ðŁİĪ @ +ãĥī ãĥ© +ze bulon +z quez +wildcat pride +wee py +waste d +uniof leicester +tric ities +thu gger +sy lt +sto pe +sp dx +snar ls +sky la +shan ds +seab ourn +school mate +rt gs +ri ans +re invents +rat z +pon tel +pho e +out patients +onit suka +new comicbookday +natur ale +muse odel +mck ale +max ey +marmel yr +ld sb +lal isa +kodai kanal +jan an +irwin dale +ili b +ianh watkins +har deep +gon z +fy re +f mo +drach m +do com +day yyy +cul p +con o +co founders +buffal onews +bu bonic +bot b +be co +baske tt +az aria +authentic ate +aquat int +apost asy +aa ahhhh +ðŁĻı ðŁĴķ +zihuat anejo +youn ge +y ce +un contested +to pla +ti et +teac ake +tar ter +tan ush +swee ts +sustain ability +ste y +sense wrds +satyam ev +roman tica +ri sso +qu ip +patux ent +paolo zzi +pang olins +o ley +north lands +new start +new sand +mer ingues +man gu +liv ability +lab view +krzysz tof +kam ath +ic is +hu izen +hat shepsut +harvardh bs +granul ated +goddam mit +forzaf errari +es guerra +dun kley +dre port +drag net +do xa +dissec ts +de activating +dat es +currumb in +cel com +cau sey +cat l +buck skin +broad leaf +br aley +bis mark +bet tors +bel ge +ass wednesday +am bra +akin wun +zee shan +yu uu +wk ly +widesp read +wether sfield +wash caps +w anga +table spoons +sinab ung +si rt +si fter +se tar +se sc +remedi ed +rajinim urugan +pri ssy +preakness stakes +phu le +per du +pavili on +organic skincare +occit anie +newsmel bourne +new bern +national icecreamday +mor daunt +min ouette +mar uk +liber ation +la wny +kis co +kassi dy +intra ub +hyder abadi +gwr help +fer land +e dir +die gom +cyto logy +cru den +chrisho y +cheeri o +chann on +carpe tb +cal vi +brother ton +brooks beau +brahim çelikkol +bour ton +bo gies +au fc +areyou ready +am bala +al ker +ai ea +aa ha +ðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬ +ðŁĴŀ ðŁĴķ +ᶠ¦ +е в +zam pa +wal sham +transgre ssion +su unto +staphylo coccus +sian network +showtim ena +shil o +sharec ro +shad ab +sekol ah +ru bes +q fa +pyram idal +primiti vo +pe kanbaru +par ok +no gueira +nh rc +nbc ct +me quon +lac u +la j +k srtc +in thenews +in aug +im potence +he us +grou ting +gri ot +goto ireland +gm police +fish kill +does it +di franco +date in +cu arto +copp ice +chen e +cas si +caper naum +barric hello +bally money +assi an +adam carolla +abun da +abe di +ï¸ı . +zesti ria +we yer +w sis +tom cats +todayskid swillneverknow +tex perience +te ale +tat aki +sna han +ser on +se ck +scot trade +por twood +poe tical +peck ish +nikol aus +mol la +minnesotal ynx +mad ding +lon ge +lale ttan +ki yo +kh ich +k nur +i ker +hun ni +hither to +hay ate +gen san +f co +employ able +dump y +dois neau +digit ising +di sko +cu bo +crazy ex +cnn politics +city am +cato institute +bell town +bake shop +bag dad +agen esis +?? !? +âı © +zarathu stra +woo ley +wit ched +ven ant +track oftheday +team trump +su h +sp aren +shashtag party +shang ela +sha stra +seoul day +seac ole +sas o +retri ev +real taeyang +re ya +race hub +od st +new grange +misi ones +mb ari +liqui de +lee ch +l tt +ky l +k rol +journey tomars +it raffic +inver ts +he eee +har lan +grey lag +gi ggle +gan su +for g +finu cane +ep ath +ec ach +cymra eg +crai gie +cess ories +cell cellpress +cam shaft +c wa +bren twood +bound by +bosc astle +boo tiful +archive shashtagparty +and ddd +ah man +admini sters +ac ero +ðŁijİ ðŁı» +åį ĥ +á¹ Ľ +zhang ji +z rh +woo seok +wake field +w gu +vikramad itya +v afa +thereal mike +the mark +tan guy +sug anda +sleep walker +sleep inthe +si bu +sh ool +separ ker +saf t +rock ne +rin aldo +popein philly +paraly ze +or ms +oad by +no ver +net as +nan ami +mi do +mar as +m trs +love box +light the +li mav +jap e +jam arcus +in fol +i ad +hot springs +honey crisp +heroes ofthe +fen o +dian aross +den zil +daysof horror +cloison ne +bul well +bar bet +at sc +amazing thailand +? !!? +ðŁı´âĢį âĺłï¸ı +zon der +we ren +v ct +turke stan +tree oflife +tr ate +the book +ter cer +super con +sho toftheday +seung yeon +sen nett +sel ton +ronces valles +rac kets +ra gusa +pun ia +perpetu ated +orino co +nor seman +na jee +masa ka +marian keyes +mar un +lu ff +len ght +la valle +la hor +l hhh +kstate fb +kid sc +khur rana +kais ers +k hus +joseph ine +iam jamiefoxx +gold awards +fu miya +flyn t +fil mic +face plate +en ames +dogsof twittter +cre sted +coryn rdr +con well +car ros +capp ello +c xc +buffalo ve +bri ggan +bow ditch +body kit +bentley motors +bee cham +barne ys +bah ri +arch top +aami park +ภĵ +w imp +vo tos +sven sson +sub atomic +stry dom +star lets +siyahbeyaza ÅŁk +simpson ville +shra wal +sab onis +robu sto +poli zzi +phantas mag +peru zzi +mat or +mat as +m out +kempton parkrace +kas ur +imperson ators +ic em +green stone +girl code +fur ze +f wp +episcop alian +edge combe +di kanu +dh f +dalla ssmith +cpt traffic +ci als +cen dol +bran ham +bbc lookeast +balay ya +ar get +am et +ag old +ðŁĴĽðŁĴĽ ðŁĴĽðŁĴĽ +íİ ľ +yt creators +yo gate +women mw +whin ney +vo los +val buena +up tight +twell man +tom on +t sky +surrey cricket +sun ri +stret chers +standupto cancer +sopho cles +sli mane +sl unch +sketch card +shri ram +sewick ley +rail fan +pur u +pc masterrace +ou pacademic +ottum wa +mississi ppi +meso zoic +mar lo +lee b +kin vara +jay cees +hy slop +ha sno +good friend +gen omic +fl ic +explor ing +den k +dashi ki +cri a +cine polis +ch anced +capit ul +cand acec +bre sci +bor man +ben shephard +bbcradi olin +att ari +ate pec +ate les +at alk +anir ani +al king +al hum +agon ising +ad amo +aberdeen uni +aband hu +âĿ¤ï¸ıâĿ¤ï¸ı @ +youtube india +wir t +w mv +ver nier +v va +tum water +tn n +th yl +tex ast +ten ths +st ami +second hand +sch atten +sam sclub +sal acious +s bm +red ken +qu eville +pumpkin head +psy duck +ox ox +or nette +or ke +mash al +lu tein +lo za +laure land +kat zen +ison fire +infra structure +iber ville +gu mmer +greater anglia +graf fix +gilbert son +ghid orah +euro league +er cy +environ nement +edin ner +di jk +desmo s +desig nated +dan ila +counter balance +col yer +by usn +ambigu ously +ag baje +acre ative +' '' +ðŁĴĸ ðŁĺĺ +ðŁĩ¸ðŁĩ ° +ðŁĩ³ ðŁĩ¬ +ðĿIJ ŀ +yo thi +walkr stalkr +walk s +springer spaniel +silver bird +ser pico +scott caan +ro bre +re xton +re appeared +pre vi +pc world +pash teen +pad hao +ox ton +octa vian +nico lee +nas c +naomi aklein +mon tel +mom in +metr ically +lu neta +kj ell +j sy +j app +ham mel +ha sna +ghat kopar +ge sun +free keh +fan fics +endo genous +eco system +drive in +dementiafri ends +de as +coo gler +chou inard +char din +bruce springsteen +bre ese +better pakistan +beatle mania +bangor uni +baahubali movie +b ily +az ade +av ira +at tires +as sata +ar ison +ab roads +ab road +ðŁİĦ ⾨ +yi shun +yann ick +went zville +valentini frank +uk coach +tu cum +tp ms +tap scott +so dden +sni de +sk og +same era +restor ation +re imagine +ra wl +ra iled +quir ino +pf tompkins +perse ids +p anne +over hauls +or age +nev ents +naz a +navar atri +najwak aram +musli ms +lo sing +letsgo places +lenny kravitz +jo ver +jo cko +jeremy kyle +i yl +hol anda +giu sto +fc st +fal set +entr ancing +ekad ashi +ead pool +e spiritu +dol an +defrau ded +compad re +ci pta +che teshwar +ch ra +boo ga +bbcradiolin cs +b sa +am bang +am ane +al loy +al al +acknowledge ments +ab crural +a abe +ðŁļ ĩ +âĺĢï¸ı ðŁĮ´ +yed wards +yar nell +what sapp +wall enberg +verde jo +v ka +roy e +rela pse +rajak amaraj +racing team +ra reltd +popo vic +pett us +palad ino +oy le +no we +narcissi sts +mrsilver scott +mor sel +mon tele +mis son +malag asy +lu bbers +loc alize +live son +limav ady +l pg +kell an +jordan abrewster +joni ernst +j dl +io res +ilove this +i ding +hilli ps +gu ld +go digital +form ities +fd bloggers +evapor ating +ep b +emo ore +eman ates +dro ll +di ep +cen zo +cas sart +cam h +brit e +blood donor +berg strom +ba hi +b si +an kush +an ahan +amend es +am pi +a eds +ðŁĶ« ðŁĶ« +ðŁij»ðŁij» ðŁij» +ðŁ¥ ķ +ç aÄŁ +weird world +walkrstalkr con +ti war +there sam +theater ny +spir t +smo st +schwar zer +prospec tors +por v +peli kan +pak veng +nis i +macale ster +ma sham +kler k +kat at +jak in +itt f +hu el +hoo ke +high chair +gro ver +for children +eraser head +equ itation +deep ened +châ te +chef symon +car sten +car max +ca ppa +bro gdon +book sale +alex hirsch +ak ang +($ ): +à´ ¯ +yu go +y music +whit gift +vi dhya +uki yo +tri fles +thug sof +ther ail +them home +stall worth +spe zza +sor ies +schie ffer +sain tly +ry pien +ray leigh +ran ul +pock lington +place mats +per ine +paras ols +noo dling +my sa +monaster io +min san +mill burn +mene zes +may hem +leup old +la sky +kirk sville +hopat cong +gu tman +gr á +g alia +fun kin +f ür +eu co +en um +em itter +ear wood +dogsare love +dean cain +cyg nets +bu kid +brock en +beir ut +be hn +atte station +amni otic +ðŁĴĻðŁĴĻ ðŁĴĻðŁĴĻðŁĴĻ +za beel +x ess +why im +where sw +were wolf +vin nik +uigh ur +tur cotte +trust worthiness +to di +te ff +stein hardt +sor n +son de +sne er +sk on +she in +sar aali +red headed +pri on +piotro wski +njo ku +khun nie +hae mat +grass market +grand ville +gradu ation +go ten +global halifax +gal ax +fore ducation +f du +doppleg anger +co gent +champion stour +brun y +bray ton +bhan u +be delia +at sume +as ani +ary news +alsati an +ab du +aac sb +âĿĩ ï¸ı +à¸Ĭ à¸Ħ +whar fe +weather vane +ul ani +tomo ya +tiem pos +szy man +sty lee +stafford shire +song sof +som me +snow storms +sjc drums +set anta +ser j +rober tv +pop caan +plaud its +penicu ik +pam uk +over zealous +one more +n pt +little woods +iu pu +hyo go +humay un +ha enow +graf itti +encan ews +du v +dra goons +do cometrue +dill ane +cupp layoffs +con ra +atour nament +as vegas +ari ella +arche ologist +apo yo +ali es +êµ Ń +à¤ķ र +мак ÑĢо +val c +usa g +up north +um bia +ucla football +tor rents +sur at +sto rer +stig mata +sketch pad +sik ala +ro tti +r vad +post rock +phoenix mercury +patriot sunited +ordin al +on amission +oak leaf +nu sh +nic i +ne ater +natus vincere +nancy lee +meg afauna +mav ado +mar cal +ma wes +lo belia +leah remini +laven der +kab outit +im prints +ibm z +hee renveen +happy happy +esk dale +er w +en berger +ecor p +dun dur +dig m +ci mino +cal vino +c tt +broo kie +bo ola +black smithing +best actor +bac ke +argument ative +alexander rossi +ahu a +ðŁĴĥðŁĴĥ ðŁĴĥðŁĴĥ +ê· ľ +ye va +wdr bnews +w se +vil ly +vic tu +un interesting +toll booth +tenn ent +tab de +sussex uni +ste pla +sn l +sleep day +revital ising +re aux +ravish ndtv +q ar +play tex +pi ko +pare il +nb f +mo as +mid wood +micro environment +metro west +mc dou +mar ita +kimmy schmidt +kan gra +k ile +k arti +jim beam +ish o +hu sey +heavy metal +haw kin +green ford +gra vis +foot golf +ffe stiniog +enjoy in +embroider ies +dr ys +dip tera +dio sa +d dot +co hasset +club hectare +clay don +black history +bhand arkar +belle zza +aston villa +allrise silver +ac z +> ,< +ðŁĺģðŁĺģ ðŁĺģðŁĺģ +ðŁĩ§ðŁĩ © +âĸº âĸºâĸº +zodi ac +wil ting +w tov +w ics +v xx +v loggers +ur bino +tune stweet +tu mmy +the giant +the chris +tap is +t br +supple menting +sum ma +seize the +sat suki +sa wi +reyno so +ran go +poor nima +pi gand +pfi ster +panip at +pa ani +orth coast +neuro fibro +morgan e +mcre ynolds +mari ella +llor ar +kk ah +instant aneously +i pic +hand o +goal tenders +ge don +fried chicken +fresh produce +fly tipping +ess ences +emergency medicine +e katerinburg +dwar fare +do bbin +disintegr ating +dep th +denuclear ization +de cc +cra pp +cr ame +ci ff +cheek tow +captivity kills +c ville +brawl halla +boston schools +beh ren +akrapo vic +ðŁĺ¥ ðŁĺ¥ðŁĺ¥ +ðŁij¼ ðŁı¾ +ðŁıģðŁıģ ðŁıģðŁıģ +ठģ +y vra +y ury +ven dee +vanderbil tu +txh shoops +summer time +ss aa +spre aker +ri di +renault sportf +re charges +raw linson +ranc or +rad ja +quiz nos +pv v +pokemon letsgo +nim by +news boy +nav deep +mu v +maj ed +ma khan +la q +la aa +kri stopher +j mt +inqu iry +infant ile +hor mel +hed land +heart sof +hearing dogs +head mistress +go shen +frank s +euro p +ers ley +eri ver +econom y +e waste +di ja +cu toffs +cine mac +cheap skate +chad ne +brill ante +br ana +awesomenes stv +aval anna +vote arianagrande +tri mmers +tic oke +thorn ley +team stallion +t cn +swil son +sou les +sketch january +ran son +r ge +pl v +people are +p bk +nr cc +n annie +mr nradio +mead ville +mcel hin +mar q +mal at +ma pit +londonis open +lindsay lohan +k tc +jim rome +jenny lyn +ian m +go gogo +ger lach +fre do +fra hm +form less +deschutes beer +d chs +cross ville +clemson family +chincote ague +charity day +calver ley +bombar ding +big blue +bedri dden +back water +arrested development +arq ana +ang am +aivaz ovsky +whitec ross +we tump +tu d +tribe smen +to jo +tele vangeli +te heran +tal ke +t seng +su bir +spit fire +sm hs +ri ggers +re planting +r sh +pic ke +par malee +page views +ostr ander +o ds +northe aster +nancylee grahn +mu ke +monster a +mc phail +machin ations +mac ademy +la gonda +kris meeke +ka wor +ji e +im ers +gro lsch +gaku en +fur th +fruit fulness +fe tt +fa del +duc es +con trail +brock en +bret baier +billing sgate +bha gat +au demar +> //// +ðŁį ¥ +var u +thebigbang theory +the gifted +spoo fs +slip way +schri ft +roc team +refugees gr +q aa +prest bury +op to +nerd life +naturo path +mon ac +mini stered +mercedesbenz ind +mary lou +mari st +lo hr +kol ler +ka hala +jor dison +ji hyun +iz aak +inten sively +int age +high court +h wl +glaswe gian +gil lo +gavin rossdale +edge field +cow bells +canvas sed +canadi en +bones onfox +bha bha +bell end +battleof britain +ast an +arteper larte +ahim sa +abdic ation ++ ( +ðŁİīðŁİĪ ðŁİģ +âĺºï¸ı # +wi js +u pup +twitchtv online +tree frog +tl k +tie break +think musicindia +temperam ental +sun woo +stock piles +sp atu +sco ach +sar nie +richie hawtin +reed ley +ra bu +phy ll +om ot +metal gear +metaco gnition +man ok +kun ar +klassi k +jal op +holocaust museum +hau ghey +han nie +gre sley +flag stone +explore edmonton +er at +crun cher +crimin alizing +cove do +chandra se +cas son +cari bou +cam argue +cal zona +bu daya +band uk +anton is +ami han +ðŁĻĮðŁı½ ðŁĻĮðŁı½ðŁĻĮðŁı½ +ðŁĺ¬ ðŁĺ¬ +ðŁ¤Ķ ðŁ¤ĶðŁ¤ĶðŁ¤Ķ +ðŁ¤ ¦ +Ùģ ÙĬ +yas i +welove it +visit ing +viol as +u ted +tre vel +sympho gear +stour head +se it +se ac +scrob ble +sath letic +ri q +resc a +replic ates +raz an +pat wari +our less +n tuc +muk ha +moom ba +mid hurst +medi acorp +mc kim +matu idi +massey uni +mar cou +mal ir +ma official +m ny +le disi +l ated +kri ssy +kr illin +kid zania +kell ym +jo ga +iam vikramprabhu +hi bbing +he les +har lesden +gly nis +global news +fly swiss +ex pom +ergon om +e bisu +don ell +ci j +chel sey +cha c +best day +be van +ban anagrams +are sh +am orous +ade cco +adam as +ðŁijįðŁı» ðŁijįðŁı»ðŁijįðŁı» +ðŁĮ Ħ +à¹Ħà¸Ĺ ย +x her +wild and +u zz +tom ando +todd y +tobi ko +thebody coach +swan ton +som y +sleepinthe gardn +sj k +sho esday +shat abdi +save sharks +sa sikala +roque fort +rad lett +pink ston +pe mex +os ac +on rails +om munity +nas agoddard +murdoch mysteries +mu dge +man andthe +lu ts +look down +lett ing +let z +law ry +kevin love +k assim +jagiel ka +iucn redlist +iru mugan +if n +holl man +go pleader +gear head +for hope +fmc sa +fit o +et winning +en vivo +ebay rocteam +chlo é +chic an +carryo ver +cal era +c js +breath in +bio control +be fu +ap itals +age o +action figure +. ðŁİ¶ +âĻ ¨ +اÙĦ د +ye si +womens open +willam ette +ver ns +ta ille +stein hoff +sp x +sly ly +se abor +room ful +r blx +piran has +newsp oll +news from +ne aux +na if +mother boards +mor row +marin ades +ma fal +m scott +lud low +li ii +leon el +lam ented +ky am +kube con +kov skiy +kam au +jab hat +hermi ston +gr ata +glen don +glam or +ger n +forex signals +fabi enne +evo ked +en otes +eg or +du jun +drop head +clap trap +bani shing +bak ke +bag got +b dk +ap ar +air lock +ace e +ðŁĺįðŁĺį âĿ¤ï¸ı +ðŁĮ ĭ +world championship +vote blue +virgin america +v mug +train ier +tor ture +theroyal ballet +tele fe +tasteof london +t lu +sim bel +sho tting +ro j +rain ie +pro bando +percu ssionists +nat tie +mush u +mu iden +mel tham +mat ra +liber alization +lepre chauns +la sgo +ken jeong +hiphop gods +high water +hi e +hell blazer +hei sen +ham mill +hack enberg +green vill +furlough ed +fie star +disemb ark +de cal +dap o +dag ger +comm munity +ce iba +care t +bollywood actress +blac kett +bian chi +be set +bal briggan +ar ken +an ze +a ecc +] ] +Ú ¯ +yuca ipa +yel tsin +wit bank +va sion +tothe moon +solilo quy +setti mana +seaf aring +par rett +o q +nedbank cup +nand u +mergan ser +mel as +kissing day +kin er +ha dd +godis great +disintegr ate +der rickson +cyano gen +chloe bennet +cau e +cand al +bla ded +astr on +ap ati +ak ashi +aha doop +ad jei +â İ +when ua +vo ci +travel to +t girl +super hot +squab ble +south central +so es +saunder s +ren teria +reic hs +q tum +nh lawards +n pi +my dog +mun sch +mon ki +mc morrow +mat ka +lex masters +les fic +kash miri +jim s +integr al +ic ra +holla back +ho kie +greek town +geomor phology +ge le +gallow ay +fo al +finalfantasy xiv +eri um +en coders +devo xx +depend ability +covers ong +coke studio +car ignan +brickby brick +blo b +bird watchers +bi ocon +bar low +bally hoo +bal ai +ash brook +arl ene +and back +ðŁĻĮ ðŁı¿ +ðŁĻĨ ðŁĻĨ +ðŁijĩðŁı»ðŁijĩðŁı» ðŁijĩðŁı» +ë ¡ +æ º +yan ov +x ic +wolf blitzer +wiener mobile +weare lakota +un forgotten +un aired +tt k +syri a +sunidhic hauhan +sukho thai +style uk +st fx +spi x +si op +sau ve +sam t +saddle bred +ru ga +rtop nb +ri bes +reyn old +pr ams +po kh +phar cyde +pe v +obstruc tions +o gall +mon in +military hist +medic inal +mc bain +max ell +mag ed +ly ttle +lc cs +kokan ee +kam il +josh mcdermitt +intru ding +igi ppygrewal +har twick +hand spring +ha gu +glori fication +giang inoble +gian tess +ferr ying +eric i +ec tor +ear thing +do thraki +dis liking +di eters +de sor +cu ento +columbus day +code org +chu mb +cbee bieshq +bristle cone +brazz aville +barber ton +baha dur +auto play +arun rajakamaraj +ðŁijį ðŁĺį +ìľ Ħ +ãĥ Ľ +âĹ Ģï¸ı +Ê ¸ +yo ong +vaugh n +upstat eny +ugh hhh +twitter les +tri ver +tre eline +thre dbo +the co +than die +tel i +taxi way +stir rer +st paddysday +sor cha +shorty awards +se tu +rock ing +ren ai +rejo iced +re stre +ponti fic +pitch fork +over shoot +okin awan +nam cinema +megat rends +ma hut +long legs +land rum +ky aa +king stone +kay seri +high five +gol fcart +go key +glen bow +get together +fly in +endor ff +em mett +e azi +defaul ters +deci phered +cl ymer +ce fal +cat fish +car rol +c ici +bo len +birdo bs +big city +ati sta +ar oun +ambi ka +amalgam ated +air side +adi ga +ade boye +ad ma +abram jee +ab aliga +ðŁijı ðŁĺĤ +ðŁijį ðŁı¾ +âĺ ¾ +yess sssss +wo k +wa state +vand amme +v aa +tree hugger +thr acing +thin line +the sky +synchron y +still organ +slum dog +side show +sau con +roar ing +ravi shankar +q assim +power rangers +nor din +mustar d +muscle pharm +mun dial +mo es +mc cri +mayor soffice +masse use +manal apan +lan z +kr gv +kemp tville +i aps +h co +gu inn +gol ders +fle isch +firstaler twx +fin ke +debu gger +cam i +beer and +be moans +appliqu é +alo ren +alan shearer +abr sm +ðŁį Ľ +ëŁ ¬ë¸ +à¹Ģà¸ Ń +youha donejob +yak ub +v ff +uvam en +trespass ers +theli gh +the pleasance +syco phants +sump tu +su ae +store wide +stal ent +se ago +reflexi ones +r ts +pizz i +pat in +october yet +low den +lex ico +khil af +jy he +just go +j izz +holist ically +hobby lobby +he adey +haz arika +hay at +hak yeon +fl m +fa onews +f loc +du fc +dow sing +del as +defra ud +crystalli sed +cr annies +clo tting +cl angers +chen ango +bu ku +bron tosaurus +bo sky +black head +bir cher +belo a +balde agle +baku gan +baili ffs +ac sm +ðŁĵ· | +е к +ul ang +tom omi +ta patio +sympath ise +sw azi +surve kshan +si don +sh oney +recon cili +public works +po ck +man el +lou ren +lo hud +li les +len nan +l sarsour +ka ew +jor nal +je wson +its showtimena +inde cent +gu age +fi or +envo ys +endangered speciesday +dur sley +dor chester +don nyo +de ct +co yd +cityo fedmonton +ci ao +che atin +ch ines +cavali er +careless ness +ban at +bal bo +annu ities +ad ac +yun jae +wa hh +usc annenberg +triple threat +ten ens +te go +szczec in +stock ade +si gu +sanat orium +s lon +run ic +rhy no +re directs +principal ity +petere gan +parag ould +out doo +olm stead +o yelowo +o stro +national anthem +mer lyn +man movie +mal ate +lin dros +like minded +lid luk +kent aro +it ni +gi bi +gen om +firstdayof summer +fireemblem heroes +ent iced +e hem +do lo +design by +da ine +cruci ferous +com frey +chennai floods +carami asg +cad w +blue grass +bio active +baz e +anth ill +alam at +ak ih +ab ron +, ! +ðŁijħ ðŁijħ +ðŁ¤Ļ ðŁı¼ +åĪ Ŀ +ÑĢ оР+za inal +wh ines +weare latech +wa shu +vote james +un guarded +tho s +su panova +ster i +so close +re position +rail car +paridhi sharma +on alaska +middle east +melt down +me sse +material handling +maj nu +len ka +lau dio +lamp light +lac on +kauff man +jets am +j ru +isit octoberyet +if ers +iche tti +hou le +gender queer +gad da +form ation +flick ers +firsta id +fil oni +excav ate +envel oping +eno va +ecol ab +do yen +dema io +de winter +d hp +chickend inner +canadas militaryhist +bo wi +ben dix +be ir +bat ak +bar ged +az eez +aster oid +ami el +alien ate +akhen aten +afford ably +ae jmc +.... ... +.. < +ðŁĺ¢ðŁĺ¢ ðŁĺ¢ðŁĺ¢ +ðŁĩ³ðŁĩ ´ +war saw +victim hood +under garments +teen wolf +st ook +soldi ering +sm tp +sleu thing +sebasti ano +sar tori +rock hill +rit son +port man +pipe work +pi ala +oregon state +o dai +na hm +memori alized +mc garrett +marchi onne +malin owski +lit vin +lan ter +la brie +ko do +jad en +industri alists +hobb iton +hi jos +hawaii ans +glen side +gan grape +fur lough +fre ind +flu me +fake cases +ef t +eden vale +dev itto +detroit basketball +cpim speak +ch ho +ca er +buffo on +baj rang +ayr ton +aval os +as pin +albert ville +( )! +w pp +vern azza +vene zol +umm c +super bug +spi ff +speed week +small businesses +sign syou +science irel +ridge well +retin as +realron howard +ra ka +peshawar zalmi +pag os +our pride +ou ric +orn ery +oke ke +nsw police +nor quist +ne gus +michel son +memb ers +li wa +leather necks +la vaca +hor s +har tsville +haban os +growth hack +gr andy +ghostface killah +fiddle head +f df +ever hart +dre idel +casting call +belgi um += )))) +ãĤ Ħ +zam os +wetump ka +tre lawny +to dy +t cho +sym fony +student ship +scott sdale +sar g +robertir vine +qade er +pu ked +org one +mom i +mill bank +meat six +ma ila +lucky me +liv res +line ar +li zed +le ko +kor aku +kari ya +intimi dators +hypo glycemia +hjel m +hec paris +haver i +ham monton +ge el +fin ale +fi za +exhau sts +excre ment +etu c +diaph rag +de gen +daniel radcliffe +dak is +cri mel +colo bus +cloud waterbrew +chas er +canadianti re +ber le +banan arepublic +audemar spi +ane en +alessandro michele +ak at +af ric +ðŁı ¨ +ðŁ¤¤ ðŁ¤¤ +Ø Ń +ye syes +wait t +ve greville +uk c +travel tribe +trans sexual +touch points +tam es +swee pers +standrew sday +squa shing +squ ander +sam champion +sag res +radiof requ +pri mi +pn as +pearl thusi +patron izing +pakistan army +okoro cha +ner sville +natgeo channel +may umi +mah le +lu eders +lindi we +kri zz +king smill +ju lz +jan gh +hobby ist +heidik lum +h ti +gu edes +grac enote +gel atin +f ga +enic hols +dur on +du shanbe +drawl loween +don diablo +don ar +der win +counter intuitive +chi venation +chi da +chapters indigo +cat sprotection +ca hir +bun ks +brock u +bran cusi +ber rie +bel as +alun ageorge +# ! +ðŁĮ¸ ⾨ +ç ¿ +west world +ve ach +ti ree +tesco s +tar c +smoo ths +six fields +scam ander +san n +rad ice +queen latifah +pli able +pleasant ness +petr ino +pebble beach +otw ol +nano ha +my mind +mbo ya +man katha +ma ag +m paa +jonny bones +jn dsd +jeff hardybrand +jac arand +itsa adee +impul sively +immo kalee +if d +helple ssly +gi ppy +fair born +esthe sia +eric hie +dron field +deb tor +coiff ure +chow ki +calci omer +ca ec +bom anirani +bewil dering +beer me +bbc spotlight +bally doyle +az eri +aj inky +ach im +îIJ Ħ +Î ³ +y ri +wa ils +v de +un countable +ud g +thisis whywe +the story +spl x +si dh +qu iry +pit aya +pen alised +pag al +ober wiesenthal +o ef +ny ct +n ilo +mundele in +local produce +lign ite +lad ki +kill me +ht con +hill fort +hadi se +furio sa +fr r +enh art +emul sive +di yas +depress ingly +dee b +come dian +chi f +bc place +barak ah +avell ino +anti retroviral +ade ola +ðŁ¤ĵ ðŁ¤ĵ +zy na +zeej lf +yvra irport +x tv +vis itch +vari um +tol led +tin ia +ti dd +the weekly +the forum +tex ase +shed den +run jewels +rock pool +re doute +pyg malion +product pick +pol ity +north bay +neuro sis +neuro plasticity +lohan thony +kon ec +in af +grigor dimitrov +glori ana +girl scan +fre inds +fl w +espar garo +dur ack +dun ker +dor mand +dig vijay +dici embre +de sta +de ila +daven avarro +cor se +cor da +cohe sion +chin x +bra k +bote tour +bg v +bar now +ather stone +ari as +are ports +ar mond +ang elia +and ball +amand ine +akh shan +air heads +a discoveryof +ðŁĺĥ . +ðŁijį ðŁĴª +ðŁIJ¾ ðŁIJ¾ðŁIJ¾ +zeni th +ze be +z org +yetic oolers +wy d +was illa +wa iler +ton school +tier ra +thi stor +the challenge +te si +studio teabreak +stu ffed +stay fit +south coast +somuch fun +selen ite +sac ral +rgv zoomin +rad nor +quake con +privati zed +pat z +par ul +p wo +ol lection +ol ita +nu j +nhl network +navarre te +msu bears +mr drewscott +mand ya +mal tin +ma en +lil ac +kit ano +kar awang +k po +jo whiley +in b +holling worth +hefe weizen +gor leston +geo int +for ger +felicit aciones +fe ttes +es me +di mi +d ma +cross land +choosel ove +bet tered +bet ances +be tti +az or +aq ap +anti inflammatory +annab is +amp thill +al mon +ab riel +-------------------------------- -------- +ðŁĺŃ ðŁĺĤðŁĺĤ +à ¾ +w ttw +ver ger +venkat prabhu +thi js +tar th +sym bo +sun ray +sto les +spokesperson mod +sn n +sel cas +see f +sar na +rock hurst +quavo stuntin +oc us +naom is +mo ston +min ne +medic ate +ma do +letour neau +lemon heads +lego dimensions +kwe si +kis sable +justin mcelroy +j de +inthe know +inter loper +her bes +hen o +gisele official +for glory +fel in +esp acio +elo ves +di aw +del ved +del tas +cour ts +cha w +brush fire +brook sville +bren neman +beu ys +ad dam +ðŁijij âĿ¤ï¸ı +ðŁĩ¨ðŁĩ ¾ +® : +you rad +val ois +the con +te ck +st ny +st ille +soni agandhi +ro hat +rey mysterio +real tim +re structured +raj yam +ra thod +play mates +pic tionary +nu is +non thaburi +new tech +nca alax +national dayof +moo ji +lo wn +knigh tri +kish werm +khai dino +kal er +k sb +join ville +jodie marsh +japan e +ham di +ha upt +gro es +gla iza +ge aviation +gaw ler +fir s +euri pides +e usa +don gen +cun dy +courty ard +com ent +co ad +ch bull +cast ille +can apes +bron cos +bo tch +bo stock +bas ford +bal tazar +as n +ark wright +ap hy +adju tant +activ ate +acou stical +ðŁĺģ ðŁİī +ëĵľ ë¦ +w ld +usa p +uniof newcastle +the park +te gra +still birth +south downs +sol fe +sm illie +sle vin +sink holes +sig lo +san ha +sam pha +pet z +pac ts +oto ole +ofer tas +o sco +nr j +noi sia +nc s +nar do +more than +mont ju +mohic an +mis judged +marou bra +maj olica +liber alarts +last pass +lali gas +kla van +kir u +kin ski +ka ho +k hera +ji bril +jack hammer +is ere +impro ving +hell on +h ary +g de +fan tom +erike strada +er an +duches se +dol lie +den one +delray beach +death trap +dean wye +daily kos +co ffers +cheektow aga +cancel ation +cab bages +atp challenger +ar ouse +ar ona +andre ws +al cester +adv ancing +ðŁĺĩ ðŁĺĩ +ðŁĩª ðŁĩºðŁĩ +âĢ¦ âĺº~ +vish wa +uv f +trinida dian +travelo city +thom p +thank less +te ala +t bur +swag ger +star tle +spoiler alert +shivak umar +sc oured +rosari odawson +ren tino +pun ahou +prac ing +poo ka +pi pm +peg board +nor sk +news beeps +ndtv newsbeeps +mit os +me owing +majo rette +li der +lauren tiis +lady well +ko eln +kaz a +ka ap +ingthe future +imper ious +her mans +guar di +ginger snap +frit illaria +fran cie +extin ctions +eu budget +echop lex +dru mm +drake university +d fx +cro thers +cra s +cam cor +av net +ator ia +arama ic +alyss ahar +alber te +.. âĿ¤ +ðŁĶ Ļ +ðŁijĭ ðŁijĭðŁijĭ +Ä ħ +y ll +web app +treze guet +tothe world +ther ow +sy ke +suz anna +sultan as +suff ern +stagger ingly +son ia +sh anda +radi oc +pic sher +perit oneal +nar ain +mouse sports +mole sters +mobil ising +mish mash +midri ff +manhatt ans +maggie q +mac onie +look back +legend s +karyak artas +jor an +ib on +heis man +gru e +ge tti +fex cellence +et m +equ i +en lace +e bl +dill ards +cri se +corps man +centen arian +celoteh promo +castr ated +braw ling +bobcat nation +al brighton +ac bo +unite ch +ty y +sv t +strat ahadoop +sketch fab +shik sha +sant amar +saf flower +ros ling +par ta +on gh +nett leton +neck ar +n chc +multip liers +mu ammar +mor n +mor andi +mar ma +lan igan +kook min +kin loch +jay thewanted +ip x +im jadeja +i mee +i bec +hul se +hijack er +good charlotte +g elife +frozen four +en ine +droo d +digitale conomy +dep or +day ana +conver sion +com me +coloring book +coke zero +coffee shops +chat to +cat ena +c ally +bli shing +being maryjane +bat alla +bar win +argin ine +anim alia +af gv +âļ½ï¸ı # +âļ«ï¸ı âļªï¸ı +y strad +vand am +uniform ly +un convinced +ug r +si kandar +shan u +se poy +se mir +sb nation +pp ic +phra ya +nne di +mise ducation +lune ttes +list an +la ps +kyne ton +k nightmare +iver sen +inter min +ich ner +hod desdon +ha che +h mmmmmm +grijal va +gh illi +faryal tal +fairy tale +equi pos +energ ising +dragme down +do whatyoulove +do vers +degre e +deep ing +dam med +cs j +co chin +ci fss +chem trail +char tered +bray wyatt +bo hannon +bmovie maniacs +bi et +aff suzukicup +. ^ +ðŁĩ±ðŁĩ ¹ +ëĤ¨ ì¤Ģ +âľĮ ðŁı¾ +woo kiee +wc sh +was cana +ty lor +strangle hold +sl icked +shir l +shiel ds +sexu alas +scienti a +razz aq +ran il +pra bal +penrith panthers +pedo gate +p schools +osc ia +novonor disk +nether ton +mon archi +majum dar +lan come +kkkk kkk +kell inquinn +k offee +invier no +hunke moller +gu anci +go bulldogs +for ton +fashi oning +er za +ep ine +dro se +cul ligan +canvas ses +bun gay +bre mmer +ai ge +ðŁĶ¥ ðŁĺİ +ðŁ¥ģ ðŁ¥ģ +ìłľìĿ´íĻ ī +åħ ¬ +£ £ +week nyc +une ase +trun ner +ti gray +thi ele +ta ha +super book +star fish +spre cher +spirit awards +spin ph +skin head +si rota +se agram +schoo lies +sal oons +ragamu ffin +r dn +r bl +princen arula +prelimin aries +polit icking +pe ster +par cel +od ours +nac da +loveof mylife +l fo +kri styn +kirsten bosch +kat ar +ju bail +jarre tt +jan ab +jackson hole +j ta +ig bos +geome tries +ge hl +g ising +fa ha +der u +cracker jack +com une +car ruth +blu mberg +artif ice +al jon +!! ðŁĺĬ +íĥĢ 곤 +ye aaa +wr acking +wigg lesworth +wer der +ur vive +tv official +tru cco +trium virate +trevor project +top tips +time keeping +the ol +tat eliverpool +so ak +she affer +sh us +senor ita +s agency +ri dem +red ly +poit ou +par ul +pad d +opere tta +ol ajide +na dia +montal bano +mir ando +milli meters +man in +mammoth mountain +lok mat +lit chi +lin net +lam mas +l news +kun itz +kam rankhan +ka sher +hor st +hi stone +he most +flat top +fav elas +eep ly +dou bler +don avan +dh ing +cu li +cn es +ci opp +bin ion +banyu wangi +anti guo +ðŁĮ ¡ +âķIJâķIJ âķIJâķIJ +wizard weather +whit en +vision ary +villarreal cf +tu ria +tru ek +terri e +sti vers +sm h +sign language +shi ge +resource fulness +re directing +pr x +po to +os v +no sy +no ren +nat y +mu tai +micro fiction +metro boomin +maxi mo +manchester city +long leaf +le sli +l rg +kath ir +ji denna +hydro logical +hawk moth +gir on +flo aty +feroci ously +eli da +el bourne +ed ancer +dur ango +dhananjay ang +defl ating +daw gz +cosmo drome +cir a +cas agrande +bry den +ban presto +ay ano +athletic s +ðŁijį ðŁĺĥ +zen de +winchester bros +wh erry +wen di +we intraub +way y +voter id +vi asat +vau ght +under groun +un shine +ti mbo +stit le +scare ers +rodrigue z +rob son +rag wort +probab les +pri den +power metal +politico europe +narra bri +nan dish +n hi +matsu da +mae stra +lon go +lar c +koscius zko +kak ao +iso tonic +indv s +iloven orthcoast +hwar ang +hogsme ade +haz litt +gille speterson +ga at +f ack +ever quest +en gupta +dubu is +die thyl +desp an +danielle cormack +daniel tosh +dal len +brexite er +berkeley lab +anci enne +adri anne +ach u +zuk un +zee brugge +x da +wil by +who is +vie so +vesti ges +v apo +uu l +un selfie +ul tan +ud ta +ud hr +tre stles +timeto act +the valley +taver as +tamau lipas +subram aniam +spi rome +sh ila +sal ka +res by +rate payers +rashi di +rad res +ra x +pro ser +pr ance +photo sphere +pap aver +ob is +n anyang +my music +my exand +montal ban +mil nga +mil ilani +mb als +knowle dg +kir in +kar min +kak a +k alou +juven ile +its dre +ine ers +ic cs +hou gang +hollywood studios +ger al +gar butt +esc rito +ed enton +de vere +de kat +daf ne +character ful +chapel hill +camp fires +cage warriors +be me +bag gett +appal oosa +al et +aerop ort +ðŁı ij +zi yi +ym tn +weekende dition +weather photo +ved ha +ur mila +tri shay +torfa en +tom ac +thin i +spur ring +sophi el +slu shie +skor pion +shake able +sg f +scal pers +samark and +sam man +rose hill +proud lock +or s +open air +oneteam onedream +octane render +mu ang +mollu sk +mar wood +m skar +lam in +la zo +ku ban +k man +joel mchale +haw at +fu x +fluffy guy +flu ffed +fis erv +fa ile +f ts +ero om +eat fortheplanet +ducati motor +depar dieu +dd iction +cuer adio +crven az +clean er +claren ville +capp uc +c bridge +buzz worthy +bohin j +aph ra +an stru +an hydr +am ines +alchem y +ah san +afl finals +abvp voice ++ +++ +ìĹ Ķ +âŀ ŀ +âĿĹï¸ı # +z ef +was v +vc sk +v ava +up fight +tweet my +theri pper +th impact +talent management +sub group +sh tf +richar dro +reas signment +procrastin ated +pre existing +pic tish +pe waukee +over laps +odor less +nebu lae +muzi ek +motor i +mc fall +man fred +m mot +light years +legislat ures +leaf ed +lat ches +l nt +kofi annan +ko var +jyhe ffect +isi olo +invali des +innov ator +incan tation +hu eneme +ha boob +gy an +gall erie +g ately +frivol ity +fh wa +festi va +fa ience +euph onik +en em +di rait +da eng +cocon ino +cli braries +ci um +button hole +broad ens +birthday bash +bi vou +bbca siannetwork +baz emore +battle fron +bal mer +babys at +at atime +amon ster +amo vies +aly se +̶̲̥Ìħ ÌĬ +wil letts +ustin ov +urine town +un usual +u ef +twi gg +touch screens +thevamp scon +the bold +t lb +sty ler +sto essel +stal ley +slou ching +shel le +ser kan +scrutine ering +ro erich +ram ah +pod gorica +on film +o wh +north lake +lostand found +loc atelli +leather work +le hr +la ka +kat graham +k alian +john bolton +ingle borough +hase ena +gi ps +gal lia +fo er +dio cle +de g +dac eae +criteri on +coni ferous +car rabba +briar wood +ben alma +ay meric +avi ate +amy winehouse +abomin ations +yo go +y oooooo +wal y +wa an +universit é +ulti ma +traeger grills +to vah +theo bald +tar onga +tamir rice +ste ens +seraf ina +sat c +saar land +re activated +precision ag +par la +pann ell +octa vius +noctur nes +michel leg +me agan +mcguin ty +mc bean +maha bal +law ford +lan re +la gni +la gar +kel lam +international coffeeday +inter cooler +illu sory +ili za +her y +ha zz +gol fin +gho da +gh oops +gary clark +flatt ens +disper sing +defence less +cyanogen mod +culver city +creepi er +colorado stateu +cl td +celeri o +boston symphony +ber ate +bab ri +avn awards +au tau +arts festival +apra xia +ab els +[ ðŁĵ·: +:: :: +ðŁĸ ĭ +é rables +zan upf +wb afcofficial +vibr antly +tn w +tech expo +taun ted +tall man +skill man +skel ton +sir sa +silli man +shi ek +sc ler +sc afe +roo ter +redemp tive +re works +raj guru +pwll heli +pubg mobile +pic ka +oo dh +of eng +meningo coccal +lycan thro +j cu +home bred +gi ed +gas o +game informer +ex adata +con v +co axed +christma spresents +bov ril +bo ere +bj praj +bag chi +b ition +am aj +ale ix +ah b +achieve ment +ðŁĩ ² +ðŁ¦ij # +your game +ya ÄŁ +wh oot +west lake +ut s +un tenable +the u +sucess o +su bed +soci ation +shi raishi +seb gorka +sam ana +power fm +pla smids +pil oto +phe t +per kin +pare shrawal +o gie +no ko +newtown abbey +neu tra +nc sm +mug anda +mu dd +mi stran +mention someoneyou +maul ing +mad dock +lyn g +lipol ysis +lind quist +le flore +kine se +khat am +karma kar +intel sat +in x +hear d +hay i +gi wa +genie bouchard +gear boxes +gap year +fu mbled +e utel +dustin lynch +dic embre +decaffe inated +datasci ence +corsic ana +contrac tually +cla in +center field +ce daw +car ton +be cu +bcm houston +bad alona +audiom ack +ashe boro +ar naz +appreciation month +aph mau +an zu +alli ant +af fair +ãĤ·ãĥ§ ãĥ³ +áµ ĺ +ঠ¶ +vi ff +un civilized +tx su +transfer wise +te ju +sy leena +strat com +stab ber +ss rs +solan ke +shoe boxes +scru bby +ruffi an +rou z +rom pe +ran vir +pride in +pl z +p gy +nick kristof +navig able +nan sen +n der +myo pic +mut tering +mr ricky +micha il +mccle ary +lov ski +looo oong +lof gren +lo witz +live sey +juli ano +jeff ersons +iam fat +hel ou +he pha +epic tetus +edwar des +du quette +dire wolf +confi de +cere us +build able +boudo ir +as ala +ðŁĴªðŁı½ ðŁĴªðŁı½ +wav ers +washou gal +vill an +vijayfan strends +us v +un installed +tom wolf +thereal juicyj +the sushmitasen +super vet +stall man +sany o +sam ini +reflec tion +raj ma +ra sal +power full +pareido lia +pa es +p mh +owl city +oli vos +objec ting +o jessicanigri +northern assist +mvp school +mai ka +lumber yard +lo ld +j ir +happy new +h nic +gu aje +gre tta +fin dus +family tree +est á +ep an +elli man +dre wh +cook ham +congr ats +ca del +blo ve +alighi eri +ali ber +ad ao +acu er +actu alit +ðŁĺį ðŁijij +ðŁį« ðŁį« +âķ ¯ +ع Ùħر +ö y +wi er +west dale +vish wak +ur ich +trailerpark boys +thro m +theatre royal +su bah +seat ers +scab bard +pit re +per nell +p flag +out the +nov anation +next year +moro der +jim gaffigan +hom ura +go visit +gim mie +giff gaff +fluctu ation +fidel ity +dash wood +chipper field +cen ar +ce sarean +cath leen +bur ping +bur kini +bru gh +bare illes +bad land +ba stet +ay atra +audemarspi guet +al lum +aj c +ab ie +aa an +- ,- +âĿ ĩ +Ú© ÙĪ +you rock +y ster +wr k +von en +vir u +vas ude +ubun tu +total led +tiny url +tell me +t storms +sy rie +suk uk +sterili zed +sr sg +sol ler +sb learns +rum ple +rox burgh +rose crans +ro ko +ri serva +r ancy +public sector +peter capaldi +ou glas +objec tification +oak field +nu men +norwe gians +nissang tr +ner f +my favorite +muswell hill +much music +moon dance +modern design +mind lessly +man spreading +ly gon +luc chese +ling usamy +le sabre +le mp +lam ber +ky y +kis sf +katiec ouric +kabo cha +go i +fat man +et ti +dom ide +dist as +daily monitor +cou lton +clay ne +c maa +bridgit mendler +bom an +be ate +au w +asymp to +archae ology +apple white +ak azi +ðŁijĮðŁı» # +vintage books +video games +up w +tyour back +thecine gogue +test net +tele gram +tele commuting +tal end +sw are +sugar plum +spring vale +sp line +smar ia +slee ker +side arms +shun ting +shef vaidya +sean spicer +se mis +sd pride +rae els +pet ta +pen na +peaceand love +pan em +new sal +me out +max xis +man imal +ma stic +lastweek tonight +laem mle +ke vine +kav an +k orian +k lock +inter lagos +infer tile +in nigeria +ibar aki +hump ed +heat wave +hau ck +h ili +gt sport +grand rounds +foli ar +feature me +ew york +equal ise +ee i +e am +do wag +de face +david beckham +choosel ife +ch elios +cast ille +cas que +bin nen +big time +bang bang +ay alam +aw am +am yo +alde baran +æĸ° å® +wy ang +world lionday +window pane +ve itch +van arama +tor mund +tom ania +ti ppi +ta zz +sy p +sho twell +se if +se amen +ru apehu +r ÃŃ +probin sy +poo led +poc ke +on fire +odi ham +nove dades +med ell +mad havi +ma dr +kul i +kal ina +ka stle +iphone games +ic ap +iber dro +gv k +gratu ity +gan apati +f blchat +evacu ee +erec tus +disney animation +decrimin alization +dayton abeach +dag on +dad da +chi omega +c fu +book oftheweek +bo fors +beaut yof +badla pur +av ison +accompan ist +ab hil +:) < +ðŁijį âĿ¤ï¸ı +ëĬ Ķ +ঠĨ +zun ino +y uni +weekend wisdom +virtu alized +velve eta +vap i +up turned +under a +to plo +this flag +th street +tat ting +serv atory +schnauzer gang +san kar +ri ple +re version +raro tonga +po shan +pil sener +pe ko +p kt +odd world +la schools +kr ka +kha dr +j kl +international danceday +inspire sme +gw o +goode ve +gio van +fin lit +fili ppi +fam as +co author +caman pour +by day +bun ning +bele za +ba jac +ante ce +alyssahar ad +ðŁķ ĭ +ë Ī +大 éĺ +Å ij +} . +y lum +who sunilgrover +wetaski win +wak ayama +wach fox +viol in +vi kk +vash ti +u em +tu pole +trou per +su kira +ster anko +stanley kubrick +sf bart +se z +saraali khan +roller girls +rex burg +renzo graciebjj +rc psych +ra dian +pot torff +pon dok +parkinson suk +olap lex +now drinking +ni acin +mur do +made ira +lu mb +lon ger +loire valley +live streams +le shurr +kon trol +j miller +inj kt +gol pe +gods notdead +go khale +gam an +g ando +fe ducation +eph ron +ehren reich +dougla sville +di ur +d hen +college ville +cla stic +benig no +be any +arm ley +arca dia +ale many +adop tees +________ ___ +ãĭ ¡ +ye sto +va he +u wais +trin h +tic to +the boys +ter ias +ten ma +tau ber +si rocco +sazer ac +sas city +roy ton +raven hill +r bp +pacnorth proud +oppre ssing +og gia +national sunglassesday +mc kie +marri ot +mal appuram +loveto read +lo ti +lieb man +li ddy +last ing +kin ne +kellys later +jan z +ig m +iam valc +hay ford +hasle m +gu bler +fuku yama +extric ated +emer ita +dru mb +dj ima +dis missive +day trotter +co zier +co coro +clo set +cla ud +chi gh +cer vo +bur gs +bri st +bra es +blur bs +be eler +bap at +bag o +augu r +american muscle +alway sa +ali an +a hal +a ata +................ ... +ðŁĺ·ðŁĺ· ðŁĺ· +ðŁĮ´ðŁĮ´ ðŁĮ´ + ¹ +yash hd +we z +wan go +w web +vene gas +vanc on +v pr +usatoday sports +uni k +stream ers +ster a +sodal ite +snu ka +ske tball +sho tta +sab ic +ré sumé +rise u +ra ig +perel man +pelargon ium +p iller +orn ge +o dal +ny g +north stars +nigerian army +mq tt +mis fire +mc mann +jen ner +jay da +inher iting +highland er +har an +gli dden +gh anian +fl intri +farn worth +extreme weather +duck en +do you +dhan jani +chef tom +cat us +bo ast +bestro l +bene factors +an amika +am rut +ale gend +ak tu +aaron ovitch +த à®® +wf h +tyler thecreator +tur ris +to well +tk maxx +the buffalonews +tailli ghts +swar up +sk oll +sho chu +sen ja +ridic ules +ren stein +re connect +r vt +plec tic +myfreec ams +mid ter +micro site +mechan istic +materi als +malo los +ma gog +m tweets +lo llll +kirk us +kap s +kalon zo +kal on +k gw +jais ingh +j ach +irish whiskey +internal comms +inten tioned +hyper ventilating +ho taru +god fathers +fre eyour +fortun ato +fire fall +fin ess +e migrate +dou cette +di electric +deltar une +co sh +clari on +brook vale +bjpraj nathsingh +ðŁijĬ # +ðŁijĩðŁı» ðŁijĩðŁı» +ðŁĮ¸ ðŁĴķ +ðŁĩ¨ðŁĩ¦ ðŁĩ¨ðŁĩ¦ +리 ìĤ¬ +~ ? +ye aa +wo tton +wi spa +wi ggs +white helmets +w tmj +vy rt +vindic ator +vi ste +tv writers +tuscar awas +tu mba +tir reno +stre p +splin tered +spe irs +sp readers +south borough +shant y +sen tai +seal team +se um +schwal be +sand erson +sag arika +sa ara +rs duk +ro quette +ro bey +renfro e +promo tion +pro fusion +plow man +photo realism +paula abdul +ou verture +nebu chadne +morgan ton +mccal lion +mano tick +mak is +loc ally +lily allen +lee brice +lang port +ko yama +ker mode +il ux +ic han +ic acid +geis ler +gall inari +ful da +fly te +fing las +fin an +en ki +east field +e pping +di bella +dar ing +crimson peak +chu d +chicago an +chi klis +ched i +car net +bas swood +bas konia +ba xi +auri emma +al app +air less +accou tre +ìĦ ł + µ +wick ens +vaxx ed +urban farming +trishay earwood +ther rien +sy ork +swin doll +seon ho +senec acollege +red breast +recti fier +priyan k +priorit ised +pp as +pic cal +peup le +perme ates +pau li +pan handling +pa o +pa ic +out grew +obam ain +nai as +na ep +mis quoted +master craft +mar ak +mag a +liter ati +law dy +kor oma +ked out +jan in +halle lu +guil dof +gentle manly +fu ld +frog man +fran ck +far hat +ech ols +disp uting +da best +critical care +coti ja +ci z +card captor +boudic ca +bou cle +bar ren +ball sy +at ell +ar ata +am artin +akh bar +ðŁĺ³ðŁĺ³ ðŁĺ³ðŁĺ³ +zoey deutch +y ook +wta finals +wojci ech +van illi +un kempt +town send +thar vest +swi ggy +sod bury +slic ks +ru si +ri mi +re building +pro fastpitch +prescrip tive +pp ah +persi ans +of ws +od hi +mom and +mimic o +me j +mccl anahan +marlene king +ly anna +low man +le ffler +je red +have you +haha aa +gw ire +gro b +geo g +ga ara +fv ck +fox croft +dicken sian +di pietro +d hat +cor ne +clam bake +carbon ell +ca ia +bet amax +battlefron tii +alex salmond +agre y +adelaide kane +ad hu +acade mi +ðŁij©âĢį ðŁİĵ +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ıâŃIJï¸ı +yel le +tope leven +theophil us +sy t +sk mch +sd lc +sar do +ra ssi +point blank +outw ar +ou vert +orgul lo +ny it +nature is +mö tley +mo berly +melancho lia +mar cho +lumin ance +lau tern +lab out +kw ak +kru tch +kne els +k bm +ju suf +jockey club +jo inter +jer ri +intothe woods +implo ded +i mu +homos api +hap kido +g mv +for sure +fia worldrx +fel ts +fari d +far ma +fantasy sports +fan uc +ein ar +du y +choo sing +ccm hockey +cancer survivor +buil dit +bri gida +book tour +bew dley +be brand +ar onian +ðŁĺļ ðŁĺļ +à· ı +wolf dog +wo ols +vill ani +u kun +tupole v +ten no +tam al +stil bestrol +stem less +st baldricks +scholast ica +sau t +retro fitted +qu as +pas si +oste opath +noel fielding +myan mar +ly t +level and +ilove the +hunt music +hal ftone +gyro scope +guanci ale +glen bard +gentile schi +ge os +gay ath +gab es +freed elivery +fra gs +forsy thia +fc women +ex pository +elie bers +el da +ego ist +e par +ds bury +dl cs +d bradbery +cork city +construc tivism +con ut +cle ur +biodiv library +b ba +as che +and new +an ette +an er +? ¿ +ðŁĺ³ . +çĻ ½ +âĺĺï¸ı âĺĺï¸ı +vespu cci +vand alia +tri star +tall boy +sweat band +sunday night +st eck +shovel head +shop talk +separati sm +rivend ell +pho sis +pa chinko +obe ys +nomus limb +noah cyrus +nc g +mith ila +minecraf tedu +mc clinton +manic monday +m pesa +le ddy +lb gt +john r +jesusis lord +jesse b +insu re +in sti +im pa +hu tan +hoo ple +hol te +haroo bom +guany in +ger ontology +ful vio +fu li +ers ch +endodon tics +descrip tor +coaching family +clar isse +chi em +celer on +c gf +bogdan ovic +bo ku +birthday yyy +ba shi +att ell +as elfie +ar oll +an tastic +am bert +ad ink +a age +âļ¾ï¸ıâļ¾ï¸ı âļ¾ï¸ı +winni em +verti ser +unsig ned +translat able +ten newsadel +tall ent +tak har +stone gate +sky arts +sit aram +shi rai +seman tic +sal ting +rose mount +rac o +pieter maritzburg +pal encia +pa kai +non point +metro bank +manipul ates +man kiewicz +log ar +liver ied +kar din +k sy +indr ani +in trust +iam king +i kari +horni man +heav iness +he me +ge burt +gam in +gal lus +friday funday +fo ta +e tape +du barry +cryp t +cruel ty +compar ably +cle w +claym ation +che ah +ch ander +boy cie +black n +bel co +beat maker +bcli berals +arri go +acbo fficials +< ~ +ðŁĺĬ ðŁĺģ +ਠľ +¬ë ² +tul li +ter nal +spri ggs +so ce +sam smith +rutledge wood +robu chon +ri sha +potom ac +po tawat +pla que +patr oness +national tree +moombah ton +mm un +lyme regis +kill erton +jet pack +im posters +iamfat don +hf cs +haz aras +fit bit +enjoy the +eastere ggs +dismember ment +decarbon isation +crime sof +coffe yville +civil right +bu tyl +azi za +arn side +alex alltimelow +af it +adelaide oval +ad ad +âĿ Ģ +wicked tuna +vaccin ating +tu in +ta kagi +star ships +south fields +sing apura +shir ted +shi bori +sd learns +sau geen +saber cats +rep mark +r tc +promi sed +porter airlines +par r +p ome +ovi zioso +nou rish +ne ah +national burgerday +mou stak +mark akis +man sk +liqu i +la po +la goa +kuma on +ki zzy +ke ween +k dm +jal ali +inter scholastic +indi ain +i its +hunterx hunter +han alei +ghet toradio +g kn +fif ths +ff w +favor itos +exi de +duc ting +care x +camer ons +breast plate +break point +bhar per +beef y +azmi shabana +au bry +as cot +ann ick +andread ovizioso +agno lotti +ac delco +ab alan +âľ ³ +âķ ¯ +ya al +wunder bar +w jac +vers day +vas sell +twee gram +tourism goi +the emmys +the cur +the bma +tes se +sy rus +swee eet +slam my +sc lass +reck less +pu tyour +pre ter +over runs +oh man +of ra +nj t +ni bal +net i +minare ts +maim ed +magn animous +ma zer +m net +le stone +ko ei +kay lan +john varvatos +jj b +high light +hand fuls +guardian aus +go bearcats +gar dat +fort myers +flacci d +e sop +demb élé +chennai express +ce asar +bio synthesis +beren stain +baesystem sair +an ila +am per +alex avega +abur nett +% % +ë¹ħ ë±ħ +ä¼ ļ +ಠ° +world travel +wor mald +us mca +tyler j +tin fo +sw pg +sun sentinel +su tures +stre ett +ster k +sh le +schu ster +scam per +s yos +roc kie +pon ding +per usal +penn ell +noo tropic +mon tell +mee tha +mar tham +kuch rang +kor bel +kaji ado +i marleneking +hi gley +hi bbard +hei sts +haun ter +har der +gc sa +friend lys +fi daa +extinction r +er oo +e sign +draf tee +del illo +de red +de carlo +cooker y +construc tively +chula inn +cher ly +bou e +bm j +blo cs +atom ium +ann able +al resford +al con +abdel aziz +a hara +Ùħ اÙĨ +wiel ded +wang an +wal den +vin rana +track town +tit ano +te jash +subtrac ting +statist icians +st nyc +smackdown live +shop lifter +she ung +shaf qat +selec tric +sc ba +sad face +ré my +rur ouni +resto s +regal ado +re sound +rb m +pro fli +pre diabetes +pitch ford +pee phole +ostr aci +ok ita +ne bl +lau ria +la ffy +ky ong +jazz day +intro vert +immacul ata +how se +hospit alizations +ho tography +her dman +hard wood +go de +gh ulis +g ats +fox hole +f ellers +en acts +elizabeth banks +ee ep +ec ousins +dra ge +designi deas +delph inium +cor do +constitu tionality +can thus +cam ryn +bukid non +bri ers +aviation week +anti elab +am phi +ale f +agul has +a oc +Ùħ صر +é nez +ymur phy +yar o +x bl +warren sburg +walru ses +try fan +the martian +tele kinesis +stim son +soli h +shaw ol +rick santorum +por tor +plo tters +par vez +par sing +p mm +okon kwo +mu dgee +men cken +ld t +ko slow +klat en +kick starting +ker bs +jo co +in wardly +in significance +ilove makonnen +ig tv +i sher +ho vis +graphic novels +go aussies +ful cher +fon der +eu sew +equili brio +dogsat pollingstations +d tm +ce ta +can uk +c atia +bwo y +br aman +ay el +ash rae +art collectors +arch ery +amo a +adot com +" .@ +yu cky +un nao +team zay +ta ware +street pass +strad lin +speci ation +sk at +si sq +sal us +ravin der +peregr ines +p ama +ope x +o sor +nar di +nag er +mis fortunes +margin alia +mar gs +mak os +m sam +love art +lin zi +le gar +lam on +koi moi +je ppe +its thelittlethings +igh ty +hudson sbay +hei ke +hang ten +ham n +hac ia +g top +fore skin +f rica +embryo logy +el ounge +djafro jack +c gy +bin sky +bet wixt +ben alla +bas enji +baby love +b hang +ast r +ar av +amade o +altam onte +adida shoops +?! ' +"" """ +ðŁĺ³ # +⾨ . +xxxx xxxx +wjac tv +win ship +uniof herts +ubiqu iti +tit ration +sun and +soom ro +son at +sof ascore +so loway +sle aford +si stah +re ser +pro curing +porter ville +n kr +megam illions +lac ounty +ku za +kor in +koo zies +kill ary +jo ssa +it ta +iklan onlineshop +happy friendshipday +gul lane +gu zan +floof y +euro beat +enchan ted +ely xion +ec w +ec entre +cu bs +crucible theatre +crickho well +co geco +chiar afer +cal ve +burk ard +buffe ts +black love +atas cadero +ar nel +app x +ap lomb +ana am +al timeter +al pi +ðŁĺIJ ðŁĺIJ +ðŁĺµ ðŁĺµ +worldcup russia +wood sphd +win spear +wayne state +w spd +ver tes +ve ste +vas sa +uk biz +tol i +thor ror +tat ami +tan sy +smy th +sla gs +silver wood +rum chata +rsp ca +reme dy +ramim alek +q rp +presby tery +optimi zes +ol ena +nfl top +nbc agt +mo aarena +ma san +m pps +lit ton +len et +kw ana +ke z +ke il +kan war +ju ang +jar ritos +jack box +ir van +ir th +huski e +home grown +holiday sarecoming +haz bin +hagg ar +gir d +gard ell +fri go +for ca +fati hah +do to +dal more +d ci +cyber warfare +cil ento +chir k +che mex +born free +bat te +ban ham +austr alie +au spices +asp net +ann ale +ðŁ¥ ľ +wy ong +wood fordre +wom bles +war horse +wa aa +vesti bule +tre pi +then ext +the garden +sugar ray +seaw olf +sc aup +s victoria +ru pa +ro cin +ri ii +ram leela +plos biology +pang aea +oyster catchers +never too +nas m +n gee +mut are +mtn g +mr dan +mal ta +ma im +le tu +kar ratha +jol in +indy wrestling +hodg kinson +frank lyn +francis ca +dri ppin +dak tronics +con desa +co pps +claire richards +canni bal +caled onian +back flow +avent ures +ath ina +ar ve +angel cake +am be +ak hir +ai reland +agit ator +acol yte +a and +== == +ðŁļ º +ðŁĻıðŁı» ðŁĻıðŁı» +ðŁĺı ðŁĺī +ãĤ ĥ +âĢ¼ï¸ı @ +z off +yak in +tre g +the junoawards +terrorist attack +st ager +spe cht +somerse tccc +shap en +sen kamalaharris +se mo +sav ard +re ee +pamuk kale +nutriti onists +nov y +newyork times +naught iness +nassi f +mari ela +maiam itchell +lun din +love with +key noting +ion o +infu ser +hep worth +harry style +harmon iously +good win +g tlm +fragon ard +fin sub +fantastic fest +er rr +eg mont +du ende +disintegr ated +courty ards +burde tt +bur scough +bot vinnik +blin ker +bier garten +bethe legacy +bed bug +anthropo id +al ounge +agu iar +adver b +a aps +âĺ ¢ +ÅŁe hir +up adi +un moved +u pa +the district +tech uk +straight outt +sto kke +sp ittle +soun der +snap yourworld +smi les +sharks rugby +ser re +sedge moor +sead ragon +rhe sus +recycle d +queens bridge +pri aulx +on ramp +ok ko +nen y +n cat +michel ob +mari byrn +lifeand style +li sag +li ann +ley en +leon ar +lar b +lam pert +kom pas +kof c +katv ond +hu bbs +guv nor +gro o +gal o +fo zzy +fer man +el bow +el ad +dar ty +cor ton +co ahuila +be kin +atta i +atori al +arts jobs +art ill +ðŁĺŃ # +ಠķ +à« ĩ +woodfordre serve +whe ezy +war ners +uzo aduba +uni strathclyde +un yielding +u hmm +tun as +team green +t bo +super jet +su je +strongh oldgames +sth all +sp ao +smash box +se jong +scale model +saber tooth +room ate +ron ny +roll i +ro mulo +rahul kanwal +philadelphi a +par vin +nws spc +nol en +ni rav +na hhh +movie goers +mm romance +mid gley +marav illo +mal maison +lori da +lef twich +laur it +kor ine +kamen rider +johnson pga +infantry man +inc ites +ge an +for ro +ffici encies +fam ished +extern ship +dwigh thoward +chuck le +ce ed +cab bies +bla zed +bet ws +be zan +bag atelle +ard ner +arc tica +al ata +ag w +? / +ðŁĻ ģ +ðŁĺŃðŁĺŃ ðŁĺĤ +ðŁĺĮ ðŁĴķ +ðŁĴª ðŁĶ¥ +youn gli +yorkshire tea +x p +wayof life +vu vu +volody myr +vasund hara +var dar +traumati zing +to give +there sac +teddy bear +su thep +sor optimist +sol era +sin ar +sch litter +sc ram +sa bet +rode os +remedi os +re settled +ran ka +qui vering +north cutt +nigel slater +nex en +moog fest +mark tuan +longre ad +lees offer +kor ina +klay thompson +kar mann +jesse leesoffer +il ig +hynd man +harbor side +han neman +ground lings +gin ola +ghome shi +fish mongers +fc cincy +ex claim +every thin +ely sees +dark phoenix +cy tok +co incident +cityof culture +ci mo +cae sarean +bel len +bcel xn +bar m +ba eum +aren ta +z no +yel lowing +xher dan +wood tv +wester man +w th +vo ith +v sat +tow bars +tattoo art +ta phouse +t sim +st ner +ssan tos +spar za +ship ton +scru mpy +scorpi us +school bag +rat tray +ra zer +plann er +piratesofthe caribbean +pherom one +pet sy +p sla +ofor i +od ilon +ning news +ni fa +naf tali +my dog +msk ristin +mm urray +melissamc carthy +li kee +le strange +lapak gue +lan chester +la via +johan son +iter i +house off +hor ny +gu aido +g elli +flumin ense +fire fan +fine wine +film linc +famil yo +fab ry +ec am +eb or +culture trav +cl ung +ch ack +cf ds +butcher babies +bru isers +brebe uf +bo ree +blan keting +bhubaneswar buzz +be wilder +asser tions +amber jack +ag y +ðŁĺľ ðŁĺĺ +ðŁĴĽðŁĴļ ðŁĴĻðŁĴľ +ëª ħ +w ciu +tun gu +scotts bluff +public ised +press ly +pie zo +pale ale +nix ed +newhi phop +ndam ukong +narcis o +mo den +million aire +mand ers +low rance +law we +lar king +la vo +kid suk +in und +immer sive +i ste +haunted house +gov summit +fuse tv +fr inton +f king +ell ora +educ ative +deep ti +cole us +cl x +ck enna +chant ment +chamber music +carl sson +can ad +c sat +bo bm +bio diverse +bet tering +b kk +aishwaryar ai +ag no +af ol +a uni +ðŁ¤ĺðŁı» ðŁ¤ĺðŁı» +âĿ¤ " +xavier woodsphd +wp gc +we chsler +uplift ment +to zzi +ti ent +therain makers +the herd +terror monitor +ter ric +sud han +str in +stl today +ski ba +selec ter +san guine +salu ted +rum mel +republic fc +ree per +ra sc +proud tobe +pro va +pau to +ote dola +news dict +nat arajan +mor ison +mono kini +mcen tee +maris sa +man ar +ma bee +line webtoon +li rfc +lancaster uni +la due +kat o +kan del +in lan +ifu gao +if k +dswd serves +dri d +das ch +corn fields +circuit cat +brunch bookchallenge +bow ker +boat building +bar in +az ra +axis bank +assi ani +applic ators +aper fect +ape e +aha va +ðŁĺģ ðŁĴķ +ðŁ¦ ı +æ ® +à· Ķ +with iel +wil iam +w api +veteran s +u selection +tvweek logies +thru sting +suf tum +stu die +spo tt +sor ors +sajid nadiadwala +robin roberts +ri kishi +red legs +ray music +randy houser +pat or +pap ix +omon di +od endron +nebuchadne zzar +memorial u +maroochy dore +lu rid +li ese +l tu +kit up +johnshop kins +iam santhanam +i arc +hy wel +hot ch +hang ings +ha vel +glo cken +fri gging +fit oor +fish pond +esp re +e hm +dy ke +du q +dave bautista +creep iness +comb es +co ds +claustro phobia +card illo +book fairies +bo caue +billa bong +bass guitar +bart let +aw yer +assi stive +ar ry +ap acific +amo y +al ocal +? ðŁijĢ +ðŁĩ¬ðŁĩ§ # +íİľ íĥĢ곤 +ãĤ ¡ +zo calo +va it +uma ir +travel agent +traffic sa +ton opah +ticto cnews +tah j +ta dic +sport stech +spa strana +shan emc +sep tu +sarac en +re hm +py at +pic oftheweek +part ington +park ade +ou dt +news comau +neutr alizing +nar berth +mtv movieawards +mo bb +mark t +mak ina +leth waite +la france +l ng +ju if +is landia +ink lings +ide ale +hol ac +hat tori +hat day +g benga +faken ew +fa zed +english wine +dead spin +da ves +cory don +church gate +carri ef +cari bana +cabinte ely +bryn ner +br ach +bon ington +block heads +bbces sex +athle tica +am our +am by +am bie +ale aks +ðŁĵļ # +ðŁijī : +ðŁ¤¤ðŁ¤¤ ðŁ¤¤ +اÙĦع ر +اÙĦ ÙĨ +york swildlife +yaz d +wine enthusiast +whit man +wam aga +ville franche +ve sa +valdi via +triumph antly +tony hawk +tizi an +tem pest +tat jana +sli k +sier ras +shau sa +sarban andson +red state +radi olab +plan eth +pis ang +pat ino +or cia +ome i +nor mans +mohamed nasheed +ma key +lower town +lo di +len nart +landscape painting +kermode movie +juni pero +ivy bridge +il al +hel li +gb ong +ff k +distor ts +dis assemble +davi dv +cn tower +chro matics +castleg ar +carls jr +ðŁļ ļ +ðŁij ² +ðŁIJ± ðŁIJ± +ðŁĮļ ðŁĮļ +yaz idi +whit ener +wal green +waf u +wad desdon +w enda +typo graphical +tweetyour friendshipinapicture +tricol ore +tou ken +ste yr +stan wood +spring ishere +smo ky +sle gends +sham ans +sav pak +saniti zing +sal z +s borg +quintu plets +post script +pin elands +pas sau +oscar ssowhite +on ate +nu ove +non surgical +nir anjan +ni ña +nex tel +morning breeze +mono block +mo hi +metu chen +men age +man ca +mal ou +lo xley +leop oldo +ki u +ke mar +kam ani +k mf +jail bird +j suis +j pr +hu ell +g ve +fle mings +feren c +fe asted +dere chos +cop eland +chur i +bu sto +braw lers +aug menting +as pl +als fan +ag ente +after burner +ðŁĺįðŁĺĺ âĿ¤ï¸ı +zoo k +z t +ware gem +vv v +vol kov +vand alize +un mc +udo biz +trans bay +tele visa +syl vian +shafaq naaz +sh h +sf n +sey music +sel mer +roald dahl +pr ing +pick pocket +pa atleti +o leo +nid derdale +mo zzy +mo loch +mis aligned +mets at +m sca +likefor folow +li esl +laureland hardy +la im +kw ant +ko ber +k pt +jun ot +hus se +hon es +hin k +hagger ston +h cm +gr atia +gor l +ga iam +fim mel +feed youra +enor mity +em ley +eli ver +dt p +dravi dian +din an +deathwish coffee +co pics +ck lw +chilis jobs +ch rom +bu ys +baeum ler +av ul +èī ¦ +æĿ ij +zi pl +your best +way fare +wamaga isa +va alu +v tm +um on +tab oo +tab ard +super smashbrosultimate +rhi zo +ra pini +public theaterny +pal anti +pack in +mrpeter andre +lu gh +lat ching +l ici +kuznet sova +kir stie +jos lyn +jesse mccartney +j league +im pati +hei ko +he flin +hap tics +ha art +gre ely +good people +fr aley +escape the +er oute +energye u +dis continuing +der de +defin etly +de ba +cu neo +cow al +clu tter +ci one +cd f +car ma +cal amba +bu cu +ba sham +apil ot +ap sara +îIJ ł +wood cuts +try ing +truth fulness +the aaryan +theaaryan kartik +th ire +tao ism +sound proof +sho shana +serv is +sarbanandson wal +sany al +sabre tooth +re distribute +rath aus +qu ed +nat to +nam ak +midd les +michi gand +liri ano +lig ature +le ey +kay lee +kal yani +in get +gran it +goli ad +g cr +fle m +fla bby +fi qur +fat burger +faith nomore +ero ss +ep stein +dry ad +dist ant +dent ons +demic assiani +dam nnn +daily productpick +coffe ero +bishop jakes +bene tti +bdd sw +ant inous +aise bhi +ðŁĴ¯ðŁĴ¯ ðŁĴ¯ðŁĴ¯ +ðŁ¤¯ ðŁ¤¯ +Ø · +ze tt +wr wc +wi gh +west palmbeach +wa hala +usac nation +un dr +team ol +stack pole +sport stv +soap box +sk ind +simon harri +sap hir +ph ung +par ole +ow yn +oli vers +ni xon +mo ong +mi fune +mel ancon +mas ry +m ÃŃ +lord mayor +lev ellers +kk tv +kh any +ken si +islam opho +inciner ation +her mits +gi gli +friend swood +for king +enchan ts +cordy ceps +copp inger +circu s +che tna +car char +caf u +boon en +bar ter +at ab +ang lin +amitab hk +Ï ī +wak ame +votejames fpp +un dead +tor chy +thejeremy vine +thankyou for +ster nation +steph eng +stein man +spir al +smallbiz sat +seabir der +richar lison +rec enter +q ca +puffin books +pel icula +p onto +ostent atious +opini ones +ony x +ome z +new comic +neel um +nau tique +mul laney +marque es +mark martin +leigh j +kodan shausa +kirkus reviews +ka fir +k mp +it ts +ise o +hil dreth +here in +ha warden +g sw +fidd ler +fi be +dy in +dragon quest +dispos itions +dha dak +dand i +cre swell +choreo graph +ch ir +cfis d +cash cash +bridge hampton +bally more +athanasi us +asso cham +anai vanovic +ðŁĮ»ðŁĮ» ðŁĮ» +ह र +س ر +vin oodh +shom rim +sh rank +savi on +ron gai +res ents +re assembled +qing hai +produ cex +prin ting +pal am +p mpc +op ene +ole ksi +oak park +nb m +mus que +mi ér +mg l +maje ure +lu met +line out +life hacker +joz ef +its worthit +iti ka +is ki +inter facing +indy car +incur sions +in breeding +hurry up +hir ano +grand ads +gal lie +fer man +endome trial +e les +dor gohome +djan go +dear den +dand an +cu pped +connol ly +colour less +character art +bu stelo +brech ts +breakthe internet +brack ish +bm z +blue dot +athar va +ala id +acu tie +ach ange +> /// +; ' +! ": +zu mbo +yo do +whadd ya +ver band +tri pods +tre p +they ve +the travel +the offic +st vincent +squ ib +spo or +sphy nx +r pw +pull man +pray ag +pic cata +per is +open gov +ol ture +nem ours +mute math +mu ti +miner ality +map box +lland rin +kim davis +jail er +id f +hydro graphic +hul ks +hollen beck +ho bble +har ken +han ews +ha a +gor sein +gal ton +es boeck +du guid +derail leur +co wer +close thegap +cell ini +cameron newton +br dc +bo or +beste ver +bas smusic +bam teddy +author life +actu alization +è ½ +âľį ðŁı½ +yn ys +y lo +vap id +trump y +tow bar +teh sil +str s +stit ans +standard bred +spring boro +shar ona +shand on +sh room +rand hir +rah me +privati se +pierre bouvier +pa kar +oy al +o qu +nye rere +np ci +ni dra +newss yd +ne ef +me v +m stad +lis icki +jen ning +ion ic +im bula +ick x +hy phy +haley reinhart +germin ated +gag li +fo ckers +flu sh +e sai +e gi +dise gno +demo ed +clo e +clo bber +cant stop +bu ttes +bo han +bent all +ax p +ari ums +argon aut +_ " +ðŁķ ļ +worker sday +wis den +w cbd +u at +trutv jokers +tre w +teat re +subpo en +si ad +sen ation +sap ele +sag i +rival do +ri probin +re vises +pott sville +ny cw +nt fm +nh mrc +ne ches +mun tari +magnit sky +kann on +kade em +j stor +i qs +hy th +hy fr +hog wart +gra ving +godbless our +global citizen +girl hood +galler yof +fabric ant +everything nyc +engag ement +ed cam +dul ity +dri bbled +dr amar +deccan chronicle +colo gy +code of +cap elli +c do +ban jara +atop ic +ati e +allen by +al pe +ah ills +ðŁį» ðŁį» +ë£ ¨ +âĽĪ ï¸ı +yuv raj +vi ver +v ahs +un buttoned +the in +tex change +tar g +swad lin +super conducting +sugi moto +sta ghorn +social marketing +si dious +schmal tz +sarrain odu +santi gold +sag meister +ru pay +rough y +oun dup +ou ston +oppos able +operation smile +min os +mhi esboeck +medi auk +ll ys +kir st +ke io +kate upton +kara age +jack wilshere +gal adriel +fans bts +dirtb ags +dialec tic +devi ated +dah li +cull er +crystalli zation +cory ell +club foot +cal in +bm g +baby bel +ark adelphia +ann yc +am organ +ðŁĺŃ ) +ðŁı Ķ +ä¹ IJ +âľ ® +ym ack +yax ley +wit ney +win on +wh an +ween y +w angs +vu illard +uc am +triste sse +th c +sun dogs +state lessness +ssi g +rox borough +remin er +racer mag +ra hn +qu alia +prab ha +poppy seed +piac enza +one championship +official mopar +neutr ons +ne hi +n hai +mat tox +lynch ings +lyn am +ligan ds +laur ac +kam elot +jeffre ss +il am +hottest dayoftheyear +hockey fightscancer +hi ki +hass ani +glyco gen +esc ola +effec tor +dor ma +din als +daz z +coton ou +cigar life +chryso stom +chick asha +chee tah +bug les +bu tina +benalma dena +ax minster +am ref +all round +ai ri +a ing +? ~ +ðŁijį ðŁĺĢ +ëĶĶ ìĹIJ +د ÙĨ +yum miness +yu bin +vinyl junkie +tra ppe +tony bellew +tn ite +tb ay +summ ited +st ary +skyracing au +simonharri std +sig mar +shi flett +school craft +saliv ating +s thetic +rot man +roadto state +remain er +oli day +mon star +moder ns +marie ke +main street +ma ik +li hat +kat ze +j tg +iter ates +hereto create +goo dridge +gli de +glasgow cc +fati gued +eric john +easy going +diver gent +digital marketing +di zi +derma us +de chart +dad aab +collecti f +chuck wagon +car suk +camper down +bran k +bou lang +ballist ics +ash verse +aksh aya +ðŁĺĬ âĿ¤ï¸ı +ë¶ Ģ +winter meetings +white water +v aper +tur kistan +trump f +thel or +the starters +the fin +t na +sho cker +shi ppo +red gate +pun i +pr v +or kin +om aldini +og more +nj rotc +new scenter +mv mt +monu sco +med lock +lec los +lal anne +ky lec +kt bs +ker bal +j anya +isd strong +inter war +hyde park +hoo kin +hockey roos +hei den +göte borg +grant thornton +factor ial +equal marriage +e gar +dev tools +del mas +custom ers +case book +cam co +calstat ela +ca ho +c gd +botetour t +bol g +bear sears +avori az +argen teuil +al ac +aco tta +abudha bi +ðŁĮ± ðŁĮ± +west word +tar bert +tail back +sush mita +stric test +science march +scal ped +sar dan +sab zi +sa o +run nels +ro tham +revol ts +replic ant +r ously +po ti +pilli ga +out look +nu ba +n mm +n ello +mind thegap +mil ch +messi aen +me se +malign ancies +liza beth +la din +ka at +ju mat +joaqu im +jarry d +j rb +iom mi +invigor ated +har un +govin slee +gon do +gil let +g news +freddie gibbs +fre sher +follic ular +eric metaxas +elo ck +dumb asses +dri vel +do pp +diver gences +cymbi dium +cs russell +coke studio +ce sena +brig adoon +bre h +blood less +blaen avon +bhar u +ber ke +bat ok +ban sky +bac io +asser tiveness +amag ansett +alwaysin our +í ŀ +zon dervan +wild pipm +widespread panic +wear mouth +wak ka +under sized +un cooperative +thin ku +st pauls +sinter klaas +shro ve +ru bel +robin ho +ro is +pre med +po di +pen alty +old photos +o witz +memb ering +may fire +masc is +mag ness +ma bey +london stockexchange +len or +kod ama +jim mer +it ap +im f +ie i +iamj hud +hypothe tically +gur sky +gh un +gen ge +fore heads +fo i +fli ed +fire crest +droo p +do olan +dim and +de value +d to +d ks +cor po +cond ado +comp sci +commit tothe +cla ver +carmel lawwe +bru mb +bigi deas +big bear +berth old +audi of +assassin ations +art sed +ar mee +alim ent +? ;) +ðŁĺİ âľĮï¸ı +we bos +water skiing +ve ee +v af +un bowed +to yah +tm hs +the wolf +te sd +tand ingan +takra w +symboli zed +sukh bir +spring day +sell outs +sc ylla +samard zija +re published +pv m +pu kh +os w +or na +north way +nico let +n gai +mun shi +mg ma +meh ran +me is +luke bryan +lef twing +lapakgue com +lan kans +james b +id bi +ick ens +hello oooo +hardik pandya +gatecra sher +fr ates +fountain head +duc t +donnyo smond +don iveson +chi pe +ce va +car to +car makers +bosch etto +bon dy +bo cc +big by +benson hurst +bel os +bal len +b pb +b dy +avoc ado +av ailed +as d +an ay +work days +union dale +un packs +tw all +thra sher +tan ahashi +tai sha +suf field +star sports +sin ner +pri l +pml n +pie tahouse +pic ot +pf n +or on +on ette +monon ga +mit suru +maus am +mat tu +maruk mani +klu tz +k tar +jo bless +jerry garcia +javedakh tar +iq bal +he marukmani +haz an +hay nie +gun reformnow +grav lax +gol akers +get loud +germany tourism +garyclark jr +for dyce +fen di +eu metsat +endy mion +eg ler +eco bank +du ffin +du ale +do wel +co klat +car nie +cant ine +brad l +bau mer +baf inals +ath omas +ar gan +ar dy +al kan +ad journ +ad ders +.. ðŁĺĬ +ãĤ ı +wor leans +wo om +with nail +wim mera +usc apitol +tidal hifi +ti sd +thelast kingdom +the dirty +tez os +tendin opathy +team depot +takeit back +stann ard +sp itz +smy ths +sheer ness +sen sherrodbrown +river island +regu lus +ray ment +ran z +plu it +phthal ates +per ham +nu c +naruto shippuden +mountain life +missi o +lore en +leon alewis +keeptalking mh +karam bit +kar rie +ka iri +jehan gir +jed d +himm ler +himalay an +hesit ating +gen tian +garden ingtips +gam mal +fl ory +ellis ross +el low +dayin thelife +cross winds +chen junga +broom ball +bi ffy +bi ani +audio logist +ard namur +amg medikal +alien ist +al av +acceler ometer +!!!!!!!! !!!!!!!!!! +è ¢ +ze st +yearofthe dog +wn bc +wit w +water boarding +w gl +vicky gshore +v ite +un pretentious +u id +ty vek +the jimmy +suz u +street sville +staple ford +spring iscoming +sp lott +sound proofing +sol on +sla bour +si que +schu ld +sc astle +rubber band +read me +pup date +prow ling +ped die +oli vera +nip gaming +ninj at +nation alistic +nand itas +n bb +mtv awards +ms morgan +mat lin +kum kum +keepthe faith +in frequent +hms gofficial +hi xon +got ti +fla ils +final mente +figur ativeart +f po +doe sit +do var +decor a +coupon ing +corn elis +cal academy +bra swell +blake man +bel inda +b vt +arthro pod +am pion +ali ases +ah alli +ðŁĺı ðŁĺİ +zim my +z tao +y co +war band +vent ura +v ayne +ut ty +tun sil +tu mh +truss ardi +trump sarmy +tol ler +the mad +tam borine +stir fry +spor ad +spe ter +shoo da +shat ch +seabirder saturday +rwin p +rele c +rel o +quatre foil +pum phouse +perfor ce +pas sy +ot ani +noctur na +my thri +mitsu ki +mil stein +mb fw +mason ite +lunch room +lingu ini +len ora +lat rine +l rb +jes elnik +jerus alem +jan ani +itsdre desu +gan ar +gad is +fa heem +east west +east of +dues enberg +dro r +disobe ying +develop ing +damian marley +bor abora +blo que +be les +bar ka +bar gh +ava asi +ash in +and furious +ab idi +ab ab +ðŁİ ª +yes bank +welcom es +wan go +vul ture +vel dt +tr ally +tongar iro +ten aci +team tennis +table au +syos set +soo th +sli ema +short listing +se ka +save your +sap o +sabi ersack +royal court +pronoun ce +phil amuseum +notyour shield +nit ya +mugi sha +mike brewer +me yr +mat to +ma ku +lang land +klein isd +kan chana +javedakhtar jadu +is ss +ip sos +hoop fest +heli port +hard covers +hangten stories +fresh ened +for ti +fern wood +entit le +end z +doubt less +del roy +davidson msp +d é +count yof +contemporary artist +climat ec +chri sabiersack +boy ardee +balance d +bal akrishnan +back a +as as +ar ant +apo c +anton newcombe +anab elle +amul ya +alexanderwang ny +ah en +agbon lahor +aa ir +âĺķï¸ıâĺķï¸ı âĺķï¸ı +á Į +à© ° +zheng zhou +zag at +y ago +x os +wall onia +vector ing +uni fight +uk ta +turn about +tok os +ther oxy +ta wak +sent ul +rae burn +purwo kerto +psycho logical +pol der +pin ata +palen que +pa the +out bid +moustak as +motor head +mel is +me can +mcne ely +koraku en +john nies +jj horgan +ish ak +ii hm +hypo dermic +honky tonk +gat sby +fru g +dog stagram +disav ow +cut cliffe +costu mers +chal isa +bro ssard +brittany force +bl alock +bere ft +atur als +and ina +al j +ðŁĩ¨ðŁĩ · +wh acker +vio list +van doorne +v iti +to kay +thunder bay +shar q +setit off +se kou +sare humanrights +sar bj +s ard +ruth davidsonmsp +robit aille +recor ds +pu ch +phyl la +pastu red +papix ure +omo tor +nostal gie +ni endorf +morethan ever +kun ene +kin n +ki kwe +jab baw +irish rail +impro pri +hotel news +hil ti +har preet +hah hahaha +gv n +gla dden +gam est +fo cals +fit ts +ferdin ando +drg pradhan +cros stown +clou tier +chain saws +blue green +black tip +berth oud +beati fication +b hol +aw riter +auto tune +au ren +ash tead +alberte instein +ðŁĺŃ ðŁĻı +ä¼ ļ +án gel +z une +ya ad +what eva +weekend getaway +v online +tx st +to iling +tm ro +te bal +takam ine +t mk +sv illa +straigh tens +sti fled +sizz la +sh tick +sav our +reallys wara +que chua +par ys +ot amendi +oc les +o ded +nom adic +mr jake +monument sforall +mo go +mikare yesss +kup p +ku ant +kho i +it Ãł +isak son +is sima +huss ar +hobi day +hel in +hay wards +ha dy +greas elive +gho stinthe +fla shi +ev on +el mont +earth quake +e wwww +demary ius +dead locked +de bbi +dar linghurst +bathin da +bat ley +arch deacon +aquab ats +ally pally +ad lib +:' '' +" /" +ëıĦ ê²½ìĪĺ +year swithexo +win ged +weather bug +walker ville +ur qui +unab ash +tor tola +the farm +stran ahan +stag i +sbu x +sachsen ring +ron it +reminis ced +press on +pop ham +pel ÃŃ +ov w +oc c +nne ka +ni jin +nc ta +national wineday +michaelrosen yes +mar chin +lec a +lanc o +kodak black +ju mm +jo fa +its cominghome +il oni +hur dy +ho pel +garden shour +g summit +fore shadow +f hq +esqui vel +elyn ch +drupal con +con kers +cityof ct +chantic leer +chand u +ce at +car bun +bru ja +bow doin +bl under +be our +baz on +ðŁĺī ðŁijĮ +æĥ ħ +âŀ ł +à¸ķà¹Ĥ à¸Ĭà¸Ħ +whitt ing +up n +tx a +twel come +tu cano +treve cca +tobo gg +ste ig +sol heim +soff it +schlitter bahn +sar taj +sanc tioning +rott ingdean +road cycling +re vent +press room +pe ver +pav one +officialap cng +ny bg +nau tic +moong low +melis sas +ma ino +limou sines +lil wayne +la at +kul bhushan +ko ka +khu d +jo ist +jesseb watters +io tv +hi ddles +gt k +gro dd +em maj +down grading +djen vy +deliber ative +cri scy +cram lington +courmaye ur +coo ley +clay field +chiarafer ragni +ar ani +aggrav ate +access or +ðŁij¶ ðŁı» +ãĤ¢ ãĥĭ +âĹ ł +zi elinski +y ena +w ly +vic fires +v air +tro ost +the current +stray horn +sile stone +shadow of +secretari al +scott aukerman +san sebastian +ro ke +richa chadha +refu ted +real radi +pom fret +par dee +par ashar +p sia +mu li +mey te +mer guez +mc garvey +kathr in +john cleese +job son +jair us +ir na +intercep ting +hu dd +hor sing +ho yo +free entry +fi zzled +fanci est +eve t +eti had +er ace +en ae +eag leton +dynam o +de met +com passes +circle ville +chennai fc +can so +bobbi brown +baili ff +assi sted +albe do +ai yar +ðŁıĪ # +ä½ ľ +âĪ Ļ +wide out +v agov +ub hornsup +tracee ellisross +tla ib +tin toretto +the stroke +t ling +sw oops +su mba +su ir +spread thelove +scottish cup +radi x +qasi mi +puppy cat +psychon auts +oz aki +octag on +nen u +mu mia +middle burg +mari ans +lu mos +llan ish +legal isation +ken edy +jazz man +in exhau +i xd +hear tened +hand wala +go camels +g db +funk master +forbidden planet +f ous +ex cori +duc twork +dom in +diethyl stilbestrol +dic ing +den ims +democrati zing +cre spi +churchof england +boycot tisrael +be fikre +badbad notgood +ab cc +... âĻ¥ +! ?!?!? +ðŁij¨ ðŁı¼âĢį +ðŁĮ ĺ +yn ine +wing men +wence slas +villar ai +vesu vio +verti sing +vener ated +un schooling +um ra +u van +tul lahoma +stit an +steff ens +stand upp +ss outh +squee zer +si bb +shira stweet +sf k +scrutin ized +sch wan +scani auk +sbar ro +sai fu +rin ker +rever so +re aped +ray town +radiofrequ ency +pur ty +psy chos +predomin ately +personi fies +ov h +oc w +nhra onfox +nde bele +nc caa +me sc +mc faul +mc ad +matte is +man die +mag anda +lei sha +la brad +kuant antv +jen nal +ichi gan +ib n +hill view +gro ene +gran ton +go di +french ies +fi ke +e ggy +du oden +do cus +din go +design studio +che oil +carac alla +canon ically +bu ssy +bean pot +be dd +bal it +ar landa +ang liar +ag euk +a ï +- ! +ðŁĴĻðŁĴĽ ðŁĴĻðŁĴĽ +z k +ya haya +un impressive +un ge +tri ppy +thisday that +the iron +the advocate +tempel hof +swa ins +sun valley +strengthin numbers +stop arming +si ums +si swa +sciento logy +sam bulance +ringling bros +rich thekid +reinst ates +pollinator week +perre ault +perce ives +nichol son +my home +mate j +l ors +kö y +ke mah +ke betu +join ers +jar rah +jac s +i afc +he rer +green market +gone wild +fun neled +fro ad +en cyclical +em dr +elek syon +ecol lins +e ol +dow o +cortic oster +comple ta +city uk +ciopp ino +ci roc +char bel +bok ke +bel ies +bac up +ant davis +an thers +a key +ðŁIJ¶ âĿ¤ +y pur +wy combe +ws ls +wo hl +white girl +whit elaw +usc ollege +telefun ken +tami ami +tal y +tal uka +su ed +steve scalise +so in +snow leopard +sean m +remembr ances +re ticle +ravens flock +radio grapher +port auth +pic of +pen ryn +pay nter +pain tin +mec can +me ara +magen to +lan yon +lan di +jag ex +jade ite +her iot +go cougars +g pb +fri ary +for tier +faf atl +ev enter +du ress +din ks +cu bes +cro zet +cri c +cou illard +contra ptions +chad derton +carly pearce +c sps +bur dette +blau grana +ap ahm +anc ar +am ta +ajit pa +abstr acted +[ [ +ðŁĩ ± +ãĥ» ãĤļ +ãģ¤ ãĤģ +win ce +wedding season +w apa +til lot +ti ssier +ther monu +tall on +take downs +super corp +sat yam +sam paoli +sam iti +sal tzman +sak ina +ry on +rust am +rother hood +road maps +rim less +ri ze +recon cil +ra ibh +puppete ers +prud homme +photo copier +pe cks +northern powerhouse +no isi +mu deford +mor tally +mi h +mediterran eo +mcclat chy +m ells +low carbon +lo sc +len n +lark hall +labrador retriever +kres ge +k nap +just me +jun gy +j ire +i shootfilm +hex tall +g ages +fin dit +festive season +favourit ism +fab ians +emplo yer +dur dle +dic en +con tax +clu bland +city schools +city place +cit sci +chi bok +channel tv +ch nie +box sets +bo zza +bertu zzi +bert adores +bar tomeu +back tracks +ba auer +ap lu +ang ad +ag el +adal ah +yeng pluggedin +w ck +u ys +u ppp +trigger fish +tri sten +traverse theatre +the onion +tax a +statu esque +stand in +sp ia +si vag +romeo ville +re eb +po tus +pherom ones +on wx +oak lawn +ne ave +movie star +ment i +made to +mac lay +kin naman +ingh e +il mour +hasle tt +georg elopez +ge is +fixed gear +femen ina +fan nie +et ter +en oki +eisen staedt +dusk till +dr ine +detroit news +cro tty +co sta +christinec aine +carn forth +card games +car jacked +bull is +bou lev +bo gue +blackbear nation +arm stead +amur ali +amo tor +ambaz onia +yu kiko +webster hall +war ning +wa ian +vic ars +ver bi +universal pics +therolling stones +tab ith +sun unu +sky fox +should ve +scen eries +sarahpalin usa +rehabil itating +re stle +re if +re grouping +po ore +pa ine +og bon +nl proc +n ere +mn th +loosen ed +lett in +leave eu +laur yn +lalupra sad +jrn tr +jen ison +jelli ed +j hutch +if w +i justine +ho ys +hann elius +guide tti +flo ppies +fiq h +eli zondo +donnab raz +do el +dham aka +cri sper +bon as +billion rising +be ers +bar rons +bann an +ake d +ah lu +a ari +( !), +ðŁĸĬ ï¸ı +ðŁijŃ ðŁĴķ +ìļ° 주 +why alla +upri sings +under cooked +u fw +sp its +south mead +slo a +sisy phus +shr m +shon daland +pum meled +premier inn +phal ke +penn medicine +pel é +ol n +nun atsi +neder land +mon sey +mic drop +mel fest +man en +maho pac +lu mo +li kk +le ese +lat eness +kof xiv +ko hima +kh h +kab addi +k cci +iq a +in tosh +in cy +iberdro la +head ass +hal ab +go gol +g gi +fur babies +fi shies +fb live +ez ell +enric om +dun nock +don nas +crow ley +clo d +chu d +cham bo +cadw wales +bow ens +boss babe +bonne affaire +black wing +beast master +bay ani +bad dhan +ba ther +auton ation +---- -- +ðŁĵ² : +âŀ¡ï¸ı @ +yat sen +wing lets +ul in +trepi dation +thegn show +sugar bush +srk chennaifc +se ic +scru ise +roo se +ri mando +re ith +re installing +re gress +philly sports +par ser +oo ficial +offici a +nouve aux +n ton +my la +much acho +mc mahan +le wan +laura jane +kan goo +jeff co +ja ane +inspirational quote +indign ity +i iss +homen aje +high veld +g dd +fu lop +ende sa +dropkick murphys +decentral isation +daysof summer +cute eeee +com pos +certific ated +call is +bair n +bair dmp +aud peeps +ati ka +ac da +ðŁ¤Ĺ ðŁĺĺ +zeig ler +wick en +was c +u maine +twitter sphere +ti el +terpen e +tele performance +sw anny +sv f +su pramo +sap ul +sand akan +rough neck +rema sters +philli pines +pe king +op ta +mn dot +mcla urin +mar kan +ma sted +kore and +k link +indi gnant +glob ali +gentri fied +g aku +fuji mori +fright mare +euse bius +eman ate +du fy +dil o +de palma +dat is +curv aceous +cu co +cross man +crazyex girlfriend +cosme tologist +con senting +bull mastiff +bu shiri +brand sanderson +boondo ggle +ðŁİĬ ðŁİĬ +⼠¸ +z den +ye adon +whel ans +we del +wc p +w smith +w asse +ut is +ur n +thepar raeels +the bridge +tat atru +tail backs +steel works +sno biety +ru hl +ron icles +ration alize +photo g +p du +o tak +neuro sciences +narcis se +nam ethat +mo anin +metro town +merci fully +mccu sker +mas n +m ams +ky leg +ited fc +house ful +ho ye +heine mann +hed wi +har den +gri gori +gau har +frac tal +four ball +fok ker +estre lla +engineer inguk +electro house +education ist +dig cit +df v +crouch end +co sey +cin ch +chi z +ca steel +blood stone +bg ca +ben simon +barn storming +atul co +as ke +allu vial +ag co +ace tic +ðŁĺĤ ðŁĴĹ +ðŁĹ ¼ +ðŁijı . +ðŁij¨ ðŁı½âĢį +าภģ +ver sova +ur dd +tra baj +the blog +tem pos +tb day +super troopers +sto king +sep ang +secu isine +sauro pod +sas b +rose man +ray qu +pike sville +pa ig +ohmy girl +nl f +musque am +more in +mike bairdmp +mic hell +mi jo +med aled +mall u +m pos +lu bom +laugh arne +kar nal +inf en +hillaryfor prison +gri ms +fu si +fu cc +f ynn +ed ness +dry point +direc teur +dir ndl +del rio +de mond +de angelis +critch ley +conme bol +cam pa +brah mot +blu ray +bhak tapur +bal dess +art form +art challenge +aneur in +ad oodle +ðŁĩ¹ðŁĩ ³ +ãĥĿãĥ¼ãĥĪãĥ¬ ãĥ¼ãĥĪ +z acks +yil maz +yanis varoufakis +ya sho +un ang +trans cona +tran spires +tel enor +tar sier +talent less +ta kano +ta io +sp outing +si mic +seol hyun +scru ton +ruby onrails +rose au +pro hm +pro cli +po ie +plan ck +photo bucket +perspir ant +p km +northeast tweets +musc adet +micro fluidic +man ico +mak ki +ma ppa +lawn care +ko skinen +jet brains +iri ver +holtz claw +handker chiefs +fri sby +ef oundation +east lanc +dis engagement +delon ghi +de andre +creati vidad +cerys matthews +caram els +boy sin +bha gnani +beacon theatre +bak kie +azz arello +anima ux +ambas s +âŃIJâŃIJ âŃIJâŃIJ +Â Ń +yu li +y ts +x kr +x bo +wearec isco +wb afc +vs buf +tx stormchasers +tran i +tom ei +tam ela +super men +stef anik +stand aard +sor achi +si xt +shrou ds +short all +sh ound +separ ations +re negotiate +ra sha +pleasan tries +ple asee +plaster board +per chance +ost see +nsw votes +naval academy +mon donews +me sto +mar yl +ma po +m nek +ly ph +long stocking +local host +lead gen +l amaze +kit ab +k usi +k host +he ren +grati ot +go van +gen te +fri sian +fran nie +forever more +flu ker +em ine +don nam +dd an +cou verture +clen ch +citroen racing +chit ose +cast lec +cam bri +break free +bre ves +birch box +beach combing +ash ra +al mer +ðŁijį ðŁĺĤ +ðŁ¥ į +zer ot +za atar +wing wednesday +want z +wa chter +une qu +ul ti +tuf ts +thunder head +the super +the cup +ta ppin +side cars +seun ghyun +sebring raceway +sa qu +s stv +reas sur +pan jim +oro b +op ane +onec cps +on enews +o iling +ny mex +north rup +news comau +nbc boston +n ill +mo lise +mi miko +matthai g +mati gary +mane ka +man dias +mach inists +li dge +la dera +kal ama +je unes +jac c +ho shino +hick stead +haroobom kum +fresno grizzlies +f pt +erit re +eri on +en rolls +e plenary +do pa +dl ach +debor d +color ful +col wood +col oma +chuck schumer +cactu s +bot ley +am anita +ahsa pocalypse +." âĢĵ +ðŁĴľ ðŁĺĺ +ÑĢ оÑģ +Ê Ł +yel don +wit tier +white mugh +whitemugh alsfan +volcan o +viri dian +vac lav +unstopp able +un shakeable +tru ancy +thu ggish +thra shes +tel les +taye b +tamal pais +t gn +swar t +suit up +su tras +su pes +sten ciled +st roop +sin hal +see me +sati ate +san carlo +sab atini +rpg maker +regn um +red it +profess orial +politi fact +phy llo +pag nell +oxidi sed +oss off +now youknow +navy seals +myx philippines +mor r +moor man +mon tour +man eater +mai ya +lub na +london portauth +listen now +lind holm +lib bey +la sti +kol ly +kir shner +khu t +j woww +j kia +intervie wees +ham ba +green edge +ghost stories +fire light +eng al +dü rer +dv l +doppel gang +deaf blind +dab ooks +conce it +career builder +beso tted +ber had +ba reli +awww wwww +at kowski +art ner +ala ji +ðŁĵ Ļ +âĺĿ ðŁı½ +Ñ İ +yi h +writing tip +w utv +uk football +tw rp +terr ill +tech con +super cut +sto on +stabili zers +spa ghett +slay ton +sin ful +si keston +shof ar +shamp oos +shal ala +rupert murdoch +rich homi +redun dan +rec co +pre biotics +posta pocalyptic +our cog +nib bler +mur mansk +melis andre +mat agor +marcu se +man hasset +lic eu +lead on +kelly monaco +kel ham +ini us +igers italia +hor nac +hei ze +free wheel +feed lot +far falle +eng olf +em beds +el mann +el do +compan ys +coll ating +ca thouse +be mbridge +ball out +b corp +ai ya +ðŁĹ Ĵ +à¹ĢภĪ +zo har +zak u +whale watching +vern ation +up next +tek kers +t vos +t elli +swasti kas +sail ings +ricci one +rep tar +rah man +perk sof +os at +no thin +naomi wwe +mosko witz +merit age +mc ds +mar occo +mal treatment +mac leans +lu bav +lo ps +laro chelle +kun dali +kar jakin +k sk +j mi +inter media +ing u +in forum +her old +green port +fra kes +falken tire +excell ondon +eli ud +dru itt +dailypost wales +common ality +charle sd +cbc music +cat or +carri galine +black sweather +betje man +beh rend +au dl +atre yu +an cher +all ondon +all at +air date +afl swan +__ _: +ãĥ¯ ãĥ³ +zu ck +zeph yrs +was ay +w fu +the business +th ent +sp rep +sat or +sal ada +ren fe +rebel heart +rai ison +poly tunnel +pitts boro +phillysports birthday +newscomau hq +mat ure +mac coll +lam mer +keralafloo drelief +jay walking +james dashner +industrial strategy +horse meat +heal esville +he yo +gal ecki +fighting irish +fier stein +fear ne +escap ist +def ers +daed alus +costan zo +convic ting +cli st +celi bate +care sses +canop y +c tia +bru net +bir ger +bigre ds +ballincolli g +aus mus +aquar ia +an sh +al bino +. !!!! +ìļ° íĺĦ +âĿ¤ï¸ı ðŁĻĮ +who vians +who se +wey smith +vineyard vines +vex robotics +val mcdermid +under oath +un built +u dom +u cas +sy ll +swa e +stri pling +skir ted +she been +sea vey +scab ies +sag ada +repair man +read mission +rambling sloa +py att +philanthro pies +pat ently +out scoring +olympic team +off cla +nicole kidman +nes sun +nd k +mor aga +mahar ana +lsu football +low riders +lazy day +kle ptom +kas ka +je ena +j oos +inde scri +imc millan +il legible +hir sh +he tte +harri gan +happ iest +hacker space +gon y +fü h +fin dom +emer ia +ede weysmith +dynam ites +dejav u +co relle +can ker +ca ther +btv i +bm u +beath high +bb sr +as ing +ann amalai +alan dia +air dropped +. ,,, +æľ¬ æĹ¥ +vil ified +usur ped +ton sil +t we +t nie +svs zombies +stour port +sporad ically +slumb ering +sec md +sai dyes +rm wb +radio logic +proud principal +pho bias +pedi alyte +ong seongwu +nore aga +no limit +neu stadt +ne eti +ndi aye +nasty a +n mbs +me cs +massimo bottura +marcusle monis +mad child +ma ba +lo chan +ld in +kri sz +k anna +j hope +hy g +hornac ek +hau raki +griffin mcelroy +gli oma +free zone +fail sworth +ent ous +enamel ware +ema i +ec as +dre scher +de ment +dal os +cr p +cinemato graphers +bal ak +ar bo +ah re +ad oni +ðŁijį âĿ¤ +ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ +« ล +west phal +watch os +valky ries +v ci +ut k +up govt +thought ful +term limits +su yy +spen sgen +skane ateles +san lorenzo +ryan phillippe +rw p +roller blade +rep aving +re j +po vetkin +per ine +oxy contin +os su +opho tography +occupy wallstreet +o dun +nth qld +nb alive +men ews +mar kel +mad catz +lov ins +loren tz +le ana +lal or +jr nal +it le +in stigate +i kes +hall mar +ha ggle +gn abry +fu git +fra k +eutel sat +dysauton omia +dre west +dew berry +demetri ous +deh li +circle of +by att +bi dad +au man +ar col +ak ure +aes q +advers ities +... ðŁĺı +... "- +ðŁIJ¶ ðŁIJ± +âŃIJï¸ı @ +wat aru +tra han +tex tron +spoke smodel +shon an +sh els +scra pple +re builds +phil mont +perish ables +patt an +pam ir +ondre j +night beat +nas s +mu ffy +mononga hela +melo di +maru ti +mall galleries +ly dia +lov ells +lodg ings +lin di +larry kim +la spal +la phil +kwankwas o +kokkin akis +je en +jayalali tha +ha sling +guerre ros +footy accums +folklore thurs +don ard +ditch ling +deli sted +dai ki +civil isations +celebrity cruise +bitcoin news +biaf ra +bi bo +beha vin +az amar +asante kotoko +as eries +ar imel +an nes +ala ia +adel ina +& " +â n +zulu land +yourad f +wa hm +tweetapicture of +tra duction +t illing +sun dries +story books +scul lion +san geetha +ru af +ro gel +respir atory +re sham +protec tyour +pra desh +pier cy +pf re +pc games +no h +night shift +nh lon +nat z +ms j +marti an +lö w +kids grove +justin ian +john abraham +itsgreat uf +ired ale +herd wick +haupp auge +happy customers +ham sik +haim theband +h end +gibb ard +fa thi +en via +ele vy +divor ces +digi mon +crawfords ville +cr its +continu o +clo thier +chow chow +change up +bukas na +btr tg +bern ina +bc beer +bal de +analog ous +am way +alz scot +allagash brewing +all lllll +aig le +ad na +ØŃÙħ د +ze h +wind blown +wid ne +ver if +val enti +u inta +tra vi +thisdaythat year +swe ta +support smallbusiness +spar alympics +so ong +shi ites +sere bii +sch litz +sa idi +s dorp +ruffi ans +rome tty +rl grime +resili ence +re possessed +pur pl +pride ful +poly gon +pearld rum +pae vey +out la +ote p +orient fc +nannakuprema tho +memphis redbirds +martin schulz +mal abo +lay den +l st +kom et +jeopardy sports +j jp +inmar sat +hode idah +ho witt +hi les +gri pes +ga iters +fric ker +ev aders +euro money +economic development +e hud +drop outs +der ie +cre tins +corro ded +ch ittenden +brutal house +biopla stics +bi zzy +bar thel +atit lan +asi er +à¸ģภľà¸¥ +z era +yam cha +wak fu +w wat +vo ted +try an +travel wi +th ais +swamp thing +stylish star +sour sop +son ido +sk erry +sham bala +sh renu +round tree +riprobin williams +re taking +penny worth +p slv +news round +na ila +manc made +mal ec +ma brouk +love me +long hairdontcare +ling er +li gon +jordanb peterson +jason voorhees +intrac ellular +infr actions +indu stri +iann one +hybri dization +golden state +glycer ine +gastro paresis +gad dis +future house +fur ukawa +fi stic +euro copter +en ye +emili orivera +dr aceway +disco lored +di paolo +dag g +com andante +clean ups +br ary +blom kamp +bir m +ber go +ba tha +ayy appa +american sniper +afl fantasy +ðŁijıðŁı¾ðŁijıðŁı¾ ðŁijıðŁı¾ +ì¢ħ íĺĦ +ãģ ĵ +wgn radio +wedding flowers +tz ka +to ye +titan books +then es +tere sa +tagli ani +so wers +slit ting +sel don +sch ul +ry ton +ro sier +rhin itis +raj shahi +r de +pretz el +offcla stro +now playing +new on +neverstop learning +marro quin +mag ers +lat kes +iy as +home decor +gl x +gg tth +febru ari +especi ales +engv wi +el mers +e tre +dj z +day book +common alities +clou ding +cat sup +bharat pur +ang y +aly ci +aer ated +abrew ery +. âĢĵ +ðŁİ¶ ðŁİ¸ +yor u +wi eland +wearethe community +vi bin +var nishes +thra shers +the mag +th warts +spil lover +sno pes +sno bbery +sin color +sc amp +sb ama +sander ling +roland smartin +riv age +r mk +q et +pre frontal +phal ar +pe pi +pal min +om lin +o dem +north london +muskete er +mo honk +mat tro +mart ellus +loren zi +kwi k +kis sme +kelli giddish +kar ri +ine fficiencies +head of +good wood +go falcons +gi ffen +gerard pique +foot step +eli roth +dou b +dog patch +dill ons +did actic +devast ator +del p +das d +cin i +bow sette +boogi ecousins +bbcradi omanc +ban z +back link +b wl +apache spark +allow able +ag rad +/ > +ðŁĺĸ ðŁĺĸ +ठĽ +y abu +wr cb +wir ksworth +wg sn +war um +twir ler +travel pic +tol er +to talk +tim ken +stroop wafel +scho colate +sam ina +sam huntmusic +rose buds +py arke +polys acchar +north lane +ni hal +neel am +morning starr +mechan ix +marin er +mal heur +lec tomy +l vr +kur ri +kud zu +koppar berg +jack lin +hodg ins +ha dow +grow led +gr rrrr +got ze +gav an +fire fox +fasig tipton +eye d +erc q +dul kalam +dragonage inquisition +down sized +devon wildlife +defend daca +de sing +de am +dark side +cross y +cich lids +cat tails +be kele +angel enos +and ris +alyssa edwards +adl fringe +ãģĤ ãģ¤ãĤģ +zur ich +yuri y +yun s +vivekan and +vit or +ven eno +ti pa +the hip +tear jerker +te eling +super max +suit er +ste ing +shape shifting +se chelt +schö n +sam pe +russi e +run town +pop tv +paren theses +oxygen ated +oph en +ne ato +nach o +mu hl +macro phage +lom o +kt family +kon ic +kha jura +je se +jam ey +jack white +ilovel u +icc worldcup +hopkin ton +hhhh hhhhhh +he mis +ga ston +fab a +ear shot +dog fighting +discar ding +dead andcompany +bru tes +bri ony +braun strowman +bo by +ben ic +bang aram +ba itul +aw ala +artif ici +appreci able +air wolf +ah rexpo +agri food +ae hoops +ac ers +zi ya +z ula +whitney houston +upp ingham +thru st +thero pod +technic als +t ftp +sy na +stand art +source book +schi ano +sch orr +rous sel +regener ative +reco leta +rc ms +puma football +pr ancer +port marnock +over world +nthqld cowboys +novel ization +nat v +nare k +me bane +l ith +jud icious +jabbaw oc +inser t +her azade +heathle dger +harvest moon +geor gio +g vc +fossil free +ericjohn salut +eng in +enam oured +du y +decolon izing +cram ping +coun tered +conju gation +burn tisland +brow sed +blue stem +be ekman +ban chory +augustin ian +asu mmer +arc as +abio dun +ab rown +) âĻ¥ +ðŁĽ Į +ðŁĩ© ðŁĩ° +ãĤ«ãĥ¡ ãĥ© +âĿ¤ï¸ı ðŁijij +zol ak +vaj ra +to learn +stom p +spe ktor +spatu las +soci opathic +sno wiest +shif nal +sheryl crow +sherri hill +shaw ne +sc ph +ru dge +quasi modo +publici dad +prou sa +poo per +pak vind +outlaw z +nil giri +niel son +new look +mon ary +mat ang +loo ses +le garda +landscape architecture +lamp kin +kish war +khali d +kello ggs +kay al +jun ya +jj ig +indi ad +im ma +hell bound +hair dryer +guest room +ge k +ga ar +fakel ove +erich mond +def r +dashi ell +cute dog +colombiais magicalrealism +co q +chor o +bren dand +boulev ards +boston childrens +ar ol +andre ou +alzheim ers +adap a += / +ìĥ ģ +ç ĭ +vw bus +virtu oso +u del +tron c +to kaj +terri gal +sj su +sioux falls +se ko +se ay +schou ler +sar chive +saf arova +ro tavirus +rin ko +revel atory +quen neville +porpo ises +play ful +picto gram +obsc ures +my ka +mon treat +mckee sport +mar ler +lu ise +loveyou guys +lion snz +lati more +l mn +kurdi stan +kumb h +khan kk +kay u +kat we +jo hari +interoper able +icu eta +hor licks +hollister co +gochu jang +gary johnson +fy life +fein inger +feel er +fal ak +e im +digital payments +devon ta +deter ring +day with +davi dy +cc x +bu le +brock way +bridg end +bon gbong +bish t +bhav nagar +bg su +abra xas +ðŁĺģ âĿ¤ +å° Ķ +welling tons +wel ch +wasay jalil +vivo prokabaddi +val dis +trump in +toc queville +spiegel man +spe sh +sof christmas +sc asino +sar ki +sangh vi +rapp aport +qui jote +pon toons +pok ken +ornitho logist +o leg +ne agh +n land +n app +min us +mcro berts +mar zia +long field +la at +iam varuntej +halei wa +gro over +flav ouring +fc j +ec topic +county wide +copy book +con ifers +chi ki +centr ality +cas erta +car snapped +car gol +cal ver +c chi +b iliary +ath ul +are m +aly sia +ali do +alber tina +agar tala +ðŁijıðŁı¾ ðŁijıðŁı¾ +оÑĢÑ ħ +yn lle +y abo +whelans live +vogue paris +total afcon +time of +thelast leg +takap una +ta vira +strati fied +stir uni +star la +south fork +sou sap +sou rire +sofi ane +so dor +sequ ity +sel fe +s watch +rebel led +qu itely +proud american +paragu ayan +par ast +oren zo +open cup +om its +new love +multiv ariate +madri dista +libe skind +leg ault +laligas antander +kap ut +jackson guitars +it back +hotel direct +hen son +gram fam +girl shbo +fra zee +flor rie +flir tation +fan book +elph instone +eh is +dun ited +d ne +chin on +cellu lo +ce sca +boeing defense +bat our +bar bee +bag ot +az ali +av ratri +ari se +a ite +å¸ Į +ठĸ +wro c +world ph +wool ston +west park +uncondition al +tour nam +teo doro +supervis es +still sanders +star time +sp bs +sof rito +rock hopper +ro kh +rid ha +red alert +rain iers +poit ras +per tamina +p ation +otolaryngo logy +nise ko +ner ds +mis chka +mh ky +mate usz +man tha +ladies football +jan elle +hide aki +henri ques +he pp +he der +gra ven +gau det +fu era +forestof dean +ford canada +fl b +fan ation +fa anews +es md +en scon +em ms +du elling +dor on +dl wp +dead bolt +de pew +cross bows +croo ds +cou th +chri sch +cheri shes +cathe ters +cam ren +brah mins +bon durant +boiler plate +beech worth +be rens +bc care +arimel ber +ðŁķ Į +ðĿĹ¼ ðĿĹ +zag ging +z ord +trelle borg +tor no +terra forming +su pano +su chi +stran gulation +ster num +steph any +sny tv +skibbere en +shak u +se tc +ro sati +reno s +purpler ain +patrick sday +pa ko +oudt shoorn +nyt mag +nfl commish +mon on +lou dre +live with +laura benanti +laundere tte +itch ell +in ac +i bad +hum tv +hisp z +hare em +gravit ation +friend less +fe re +fanni bals +ellen berg +eden bridge +eddie vedder +dioni sio +de wars +dau phin +co red +boston pizza +bernal illo +bermu dian +all man +adver torial +ac ott +ðŁİ§ ðŁİ¶ +z azu +ye le +wur ly +wright stuff +wau bon +wag gle +w net +v for +uc co +u til +tony fernandes +ti mah +thoo thu +sport z +satter field +sanjay nirupam +sam al +sa ima +ro dan +rahme manuel +po pover +or c +only one +normal isation +na thank +my tv +mr michael +mo ir +middle men +meri dge +lat eline +laker idge +kr ack +ko tter +kex perience +kelly ripa +kan gol +joko y +john boyega +jet life +inter club +holocaustremem branceday +hat ton +gro ce +go wr +est el +ep md +e fin +dun das +du fresne +dro ws +dreams docometrue +dj zinhle +daw nof +continental tire +col m +coal ville +cbr n +casei h +brail sford +az pil +an anta +alfredo flores +af all +accentu ates +ðŁļ ¬ +worl di +wes thou +vs nyg +vil i +unabash edly +tx water +twitter firstfriday +ta si +stoke sley +stati stic +sig ners +sherwin williams +ri quel +reproduc tive +ramanu jan +quiet girl +quar rel +presump tuous +plant es +penn ine +over clocking +or lov +oni sion +of ilm +neuro endocrine +nen ele +mush fiqur +mrs gandhi +mis represented +mey ler +m do +liquid ators +lawy ering +ky ron +ku z +kindle books +kan ia +k andy +jo zi +jo ffe +jin ka +itsn ice +ha sk +gil martin +g tu +g greenwald +free born +fasigtipton co +europe ana +es quad +el sword +earth ling +e asons +donnabraz ile +cipri ano +ch us +cf n +bu hay +bram mer +bowel cancer +bil an +bbc sounds +bag shaw +as ae +appet ite +al franken +advo caat +ac rl +à® ¸ +Ø ¦ +zar b +yiann ch +yale town +whas sup +wer i +weare not +water spouts +u ffs +tri plic +th ula +taxon omic +t weddle +t kp +steal ers +sa irs +raj baddhan +ra thi +que ere +pre determined +pistol cliff +pist es +papp y +omin ously +jon na +j pn +gi er +gel ati +for less +faulcon er +facebook down +estate planning +ea day +don au +dit z +dism oun +delici oso +cre ighton +cont d +constip ated +buen aventura +bram well +bb cal +barra sso +alb ini +ak om +ðŁ¥ µ +âĪ ł +ठł +Ù ı +wob bler +wist fully +wind mill +win amp +wil let +war ton +vi j +ve j +u xo +tweet bot +tra ight +toon ie +teach sdgs +sport science +silver backs +seto pati +san tal +ryu ichi +retali ated +queens gate +pri pyat +patap sco +oxy codone +obi ang +nge movies +mst rooo +mor rone +mer kovskiy +mcan ally +maz ur +marche tti +mar land +magher afelt +lof thouse +la brum +l ÃŃ +kol l +kim jong +hen ao +hemat oma +ha ren +gun dlach +grass root +gla z +frater ni +flying scotsman +en light +el ang +ek lund +edgar allan +du plicity +doub let +descu ento +dat er +d princess +d add +confir mado +compu ware +cash me +car lene +cal vins +b mi +append ages +ald ale +al tere +ðŁĺĭ . +Ëĺ ) +wn ba +waldor f +vo w +vo ges +use able +us dol +up work +unc g +u hl +tur ducken +tre loar +tran spiration +thisism ay +th st +str out +show es +sher awat +sh alo +sh ags +scho harie +scaf folds +q atif +pre requisites +ol usola +nor ther +no ss +mud honey +mapu che +man gu +liqui fied +kon op +k á¹Ľ +inst amusic +impactin vesting +hoss am +hawk wind +gour ley +glory days +gh our +gar ma +g aren +fashion design +elevation church +di ar +das om +daf a +crow ned +cross way +cou to +con chords +cl unes +casi o +billie holiday +b bered +ath as +amade us +almeida theatre +adityaroy kapur +ðŁĴľ ðŁĴķ +ðŁĴª ðŁijį +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +à¸ģภ¥ +æ r +yourboy myles +yard stick +werthe im +vege table +uri jah +twc news +tinder box +the trade +te sto +sun hat +smal lish +sho petsy +shi vely +shah in +sb nd +say reville +religi on +refurbi shments +real change +ptole maic +po cari +people problems +pd ga +o ana +nic ar +nev ill +mud cats +mon tro +modi govt +ly cett +long tail +light nin +la br +key worth +joan jett +it chotels +hit raffic +hill walking +ha dar +ha as +frei heit +food art +exc itation +eng strom +eng landers +doun e +de ben +cuck oo +credenti aled +crai gie +chill wave +bi reland +bastian ich +ax stv +att stadium +ar ski +allu des +ake up +abe el +\ = +ðŁĺį ðŁĴĽ +ಠ¹ +~ !!! +zou ki +yougot this +yo kes +whit well +wat ton +vin ni +un talented +un go +terri fic +tar ka +ta cht +t pl +t elia +ss acramento +sou sse +shi amak +sco ast +road tom +ro mine +ratche ting +plu tarch +pin d +o sti +new work +muk ti +mth atha +moon dog +mo hm +merrill ville +meate ater +marching band +loe wen +lo cura +le thu +laz ari +kay ser +kan th +kal k +k hang +he stia +gregh unt +gol feu +giel gud +fraudul ently +foodin novation +food panda +fon thill +electro cardio +e bn +dog tooth +der singh +den a +de gener +ba hi +avon lea +ap ay +am da +aber lour +aaa inspector +what women +wa hs +vin italy +van taa +thumb ing +thac over +supple mented +south worth +soo jung +sm tm +sl comiccon +sha u +selfe mployed +se kh +r uru +q al +pâ té +pyarke aisebhi +producer life +pieter se +p wx +odai ba +ni zza +mo shing +ml m +milnga vie +memphis mayfire +maple ton +mand era +m cham +led wards +lands downe +lamp lighter +lam bof +kuchrang pyarkeaisebhi +kime ki +ka inen +it smo +it ong +hur ra +hra wi +how tobe +hol lin +hapha zard +habak kuk +h lat +gas con +fur riends +ft lauderdale +f bal +eu i +ec ts +ea id +doctor ates +dim mock +comic stitan +ci reland +celi ac +cb seven +car nahan +calgary herald +bharat anat +beach goers +awh h +ast oundingly +anglo phones +amus ingly +am cc +air indiain +ab stained +ab asi +aaj kamrankhan +a ú +த ல +Ùħ Ø© +u vc +the bel +sunba enim +stop killing +stain d +soli dity +shoulder to +sap hana +ren cont +pur an +pnpp io +o ty +new moon +necker chief +mo shenko +me ireles +ki on +ka pow +irish history +holland roden +helve tia +hear st +hack aday +greghunt mp +give thanks +frontier land +franc avilla +fit mom +filip inas +ep irus +ear lobe +ea sty +dray ton +de ssie +dam ar +co work +chapp ed +cano ga +bursas por +bri o +billy graham +bho sle +b wl +aw omen +autor ick +aeri alist +ae reo +ðŁĺĺ ðŁĴŀ +white hill +what makesme +wal ey +villarai gosa +ve i +v usi +un reasonably +toyo tires +thirdman records +thing iverse +th ami +survey monkey +super giant +stre vor +sound checking +som met +scar petta +roberts space +reall orraine +rajyas abha +pun ta +po om +pax prime +oren dabooks +oc tet +ndhakoo ttam +museumof art +mom ocon +mo du +medium ship +maher zain +laksh ya +k abc +impression able +ho bey +hc so +harring ay +gram in +governor tomwolf +gor man +gam esa +found ling +ff x +europe forculture +dra ig +defence forces +cy clin +cos grave +corel draw +choo sy +cbseven ingnews +cam ili +br ang +bow ral +benven uto +ben zodi +be ware +bak kam +ar le +annihil ator +am antes +ales ana +ðŁĴªðŁı» ðŁĴªðŁı»ðŁĴªðŁı» +ðŁĩ¹ ðŁĩŃ +ö n +wy che +will enhall +w anne +united against +un box +tou cans +tick tock +ti jer +st pauli +ro bron +richardo sman +red backs +rayqu aza +raff le +por tree +pol gar +pil o +pen testing +oy en +owy hee +ott b +o her +mitt en +marga ery +lauren german +ki miko +jay apal +j yn +it movie +im perfectly +if es +hol ing +hesp eler +han abi +h mw +great american +giga om +gan dia +g ys +fury road +fondaz ione +ex s +esp nw +er wan +dee wani +co bol +city asone +cgr teams +caul king +carpa thians +ca el +bundy ranch +buckingham palace +basketball cl +audit orio +aq s +androgy ny +ame et +am ae +ak asha +agio ia +afro centric +. ,,,, +à¸ķลาà¸Ķ à¸Ļ +õ es +wrest ler +vi zi +un carrier +to pol +the orossi +symph onie +sr kian +sonys antamonica +sing star +shab nam +seri alization +semin ario +se aland +sav on +sak har +rise together +red coats +ralph garman +rag na +ra bun +pw f +proce ssional +perform a +pan opticon +p illa +oh so +notin myname +nit t +net beans +muh te +mm ell +meck is +marab oli +libt ard +lef kada +kay leigh +jac q +it sab +ind l +immort alize +ho res +giveaway alert +gin nie +fat joe +et sch +es xi +elabor ated +ein en +dunker que +du toit +dri l +doncaster races +don elson +de kay +d magazine +cre ssy +cin o +cil ip +chi quito +byte coin +bre ga +box y +ble n +anti gravity +albu ll +adv aita +ac chedin +ab nam +ðŁĺĤ ðŁĶ¥ +ðŁįij ðŁįij +z huang +y acou +x net +we gener +vas o +up ended +tri l +tran ter +tk pict +sun star +stic ation +spor in +sp ast +sham miya +scann ell +savi ano +sal ing +sal en +rat to +polic ing +pisto ia +pic hi +pate k +pascu a +over riding +or able +oo loo +now a +mo fthe +min ote +metall ers +merge records +mabu se +ma don +kine matics +indi s +hu o +ho tr +hal t +hal dimand +greengro cer +fri sell +formula drift +f ism +equ aled +e bd +dnb tv +da awards +cool fm +coo tie +chin ua +bot ts +bi go +ba har +ba er +aw aren +ap lusk +anz acs +an cap +alon ce +< # +ı ÅŁ +yay yyyy +vin ita +un cluttered +tutel age +tu mp +thal ys +tailgat er +sydney tennis +swadlin cote +super copa +sunday times +su ber +stat ers +silk worm +sign writing +si stem +shawn mendes +self shot +scrip tural +sch rager +punch lines +puj ya +prolon ging +polit ica +park way +paraphra sed +p sus +organ o +octo gen +nle sdca +niec y +nenele akes +na sha +missmal ini +mis eries +mens golf +men to +me too +mc gillivray +mc callie +manic urist +ma pfre +kabir khankk +k mk +k him +joss stone +j re +ir ates +inter faith +heartbreak ingly +gó mez +gro at +gie ter +gel son +fel ice +fav icon +fal lows +ex i +drift less +dog matic +digital strategy +de stiel +cop ters +circ let +chiar oscuro +chi pol +ch afe +calv illo +c ssd +c sia +bristol rovers +bo ac +ath en +altrincham mkt +albat ros +al pen +ak owski +: (. +ا٠Ĥ +wing tips +vo anews +ven able +vat er +v vt +trac ead +sweet water +sw co +sam it +resi zed +pluri potent +ny jah +natas cha +music lovers +me khi +majik ninja +maer sk +lambert ville +kl al +ki drock +khar agpur +k ram +joun ieh +jor dang +is free +insol vent +i asi +hu set +fother gill +fd tn +fac tion +evapor ator +endocannab inoid +e migrant +drunk pete +dragon srugby +dom ine +dispo ses +de guzman +cra in +cot ts +consu elo +con formed +castle well +by p +box park +bal ut +bal akot +ajinky ara +abo b +aaron carpenter +ðŁĺİ âĿ¤ï¸ı +ðŁıĪ ðŁıĨ +ðŁ¥ ij +ãĢ ĵ +ภĺ +à¤ľ य +with al +wemy ss +var ni +twitpic your +toi delhi +thir uv +stac os +somers by +solar winds +sh hs +recuper ation +rastaf arian +philanthro py +parenting tips +pa ster +mort lake +mil bank +ma iri +luq man +loc cit +letsgo x +l pf +kal ma +ja is +isleof skye +is car +in habitat +hyper spectral +hincap ie +gen do +fu egos +es waran +depo is +da hn +d sp +d lb +commer z +chi quit +center line +cas als +border terrier +bord bia +booz allen +black catday +big boi +baz an +bas ilio +; ] +! ðŁĺįðŁĺį +ðŁĸ¤ âĿ¤ï¸ı +âĻ ¢ +á in +y oud +we di +v twx +uncann y +u mas +tw ba +tim lin +teng u +sw ades +stoo she +sor bo +seatt lec +school day +reflexi ve +public is +prabal gurung +pink news +peter facinelli +pan gos +ordin ated +or cid +nintendo direct +nct dream +mur phey +min ta +mill brae +lo lit +lie ben +lactobac illus +kra ig +ko dagu +kiane gan +kaz un +k so +j alo +itu d +ho gi +glove box +gilli e +gil ded +gam era +gag olf +fall foliage +exp elling +e rez +dio genes +club life +bremer haven +boyer town +bota fogo +batter sby +bas uki +b ca +ayushman bharat +autumn leaves +ashish sharma +ashi els +ant ine +air mass +ai ai +ac ine +ac abou +: -- +ãģŃãģĵ ãģĤãģ¤ãĤģ +wi el +whites and +w tv +unbreak able +team j +sub types +sodom y +si ki +shel ton +sha key +sang in +redundan cies +red an +pv ris +pow is +piccal illi +pic hon +pan ag +palae o +pad let +olympi acos +olivi acul +observ atories +nur sed +na ac +man so +ma aya +l ally +kap u +kail an +ju h +jabbawoc keez +iz er +irri gate +ire m +il led +hut ton +fu ne +ferv ently +fc v +er re +endofan era +dr jillstein +di pg +de uk +darrene spanto +d sch +criscy borg +chante use +bless ing +bang ka +at theraces +ash ridge +ane sh +am reading +ad lai +ðŁĨĺ ðŁĨĺ +zimmy deanie +ze ks +yak umar +tu bas +travel more +thu mped +tal ha +sylvain reynard +sph ilippines +shinsu ken +scri vens +sco ding +sch zimmydeanie +sale sharksrugby +riseand shine +ris kie +ram en +pune city +probinsy ano +pos iciones +photom ani +oren zi +op un +nach man +montag na +mandi sa +mal inda +less ens +le ute +le ele +le ashes +la sg +l tn +kem pe +jur ina +je ana +jack alltimelow +ip in +hô tel +hu cks +ham ma +gir onde +george clooney +ge om +equal rights +dprincess maja +dense tsu +dare bin +craigy ferg +conor mcgregor +com fiest +co dd +cbs denver +cb ca +calli son +bring on +bio ta +be utler +authori zes +arte contempor +argent inas +an anas +âĸ Ģ +ภ³ +yew tree +worththe wait +world sleepday +whar f +u yl +swen sen +swamin athan +sub contractor +shoulderto shoulder +sc j +sanc tify +saipalla vi +russ i +record ando +ram kumar +quil mes +queer ness +prosp ering +percu taneous +onetough nerd +oj hl +mon y +mo tional +ml se +meis sen +medit ation +marine biology +mandy moore +maj ik +lasd hq +la ith +l ph +knap sack +je ev +ir g +indian wells +gin toki +forever royal +for ky +evo sti +et zel +ef rain +der p +de chambeau +cul tus +con o +cataly ze +buy british +bam boom +azpil icueta +auburn dale +as fer +am adi +acc intouch +?! ?!?!?! +/ , +ðŁĩ¦ðŁĩ ² +whodun it +war hurst +wag ggs +vindol anda +tum mies +sun down +studi of +street fighter +sti ger +ssi ere +sheriff clarke +sens ational +sd all +sc v +ru ssa +real radio +radi onet +r bn +quadril ateral +power pop +per nam +our nhs +ob tainable +non descript +n ith +mir chi +mars den +lul z +ke ck +k hara +jam ba +his i +gy aru +girl sof +gent ler +flag ships +fer tig +euro wings +et w +es fest +eq nz +el ys +do by +chin at +ceram icist +car rental +canad aam +cam br +c prs +broad well +brand icar +bird watch +argy le +ange le +ale bri +zim m +za an +ximen anr +vir den +vic roads +v go +unf pa +un u +tracead kins +than thi +tech day +spin drift +s res +rat nam +prop iti +piazz olla +ou lis +nor itake +nom ean +margin alization +margarit aday +kil bey +kevin magnussen +jas na +janu ar +houston ians +heart bleed +hadi ah +go po +glen field +dub bel +drak engard +dj drama +deut scher +de formities +dau lton +dab ney +cruise chat +cri sty +coven ants +con sho +clex acon +ci ras +christy chibi +chicago theatre +can can +camcor ders +calciomer cato +butter bean +bre vet +blondie official +black horse +back beat +bab ur +au jour +almo dovar +ag hast +ab ailey +ãĤ¯ ãĥ© +à¸Ńภģ +yate ley +y ron +ve ren +vanv leet +utah ns +upp al +uni body +un lit +un follows +ti fton +thr in +theis mann +th id +team mma +tae tae +ta yo +stro bel +stam en +sop ra +songh ye +smart ness +shell shock +shak il +seeyou soon +secretary zinke +se garra +reck lessness +per ler +pen netta +pakistan inpics +ov um +oce ani +n dm +movie maniac +mol ana +mobil ink +mass u +maric el +mar duk +maj ima +mac ks +lun ched +lsu baseball +lipp man +lau zon +kwin ana +k wen +in nam +hul king +higu rashi +hair dos +gri zzle +go be +gal ashiels +fortu itous +fe ckless +diure tic +dap ur +consu mable +cn as +clo cal +chou ston +chic ana +char ney +cen g +cav apoo +cam igu +bu uuuuuuuu +bu football +bo ak +bnppari bas +black series +bee ley +ban deau +back streets +at tired +arab a +app é +alex x +affe c +ad ani +zon ta +you kai +yo ann +wy nd +with y +vidy asagar +tru stin +trou pes +tre mens +tom welling +the kinks +swo ggle +swif tie +sun river +seung cheol +scorpi o +school mates +schom burg +rock hounds +ra ynes +quir ky +qu or +pu glove +plasticfree july +pain management +na ha +ms build +modern ising +match room +ma stro +long street +long shore +let golf +lein time +lal as +jacobl atimore +instant pot +in secure +hv ff +heuri stic +hen lopen +gunter sville +fe ste +farou q +est ates +eas ily +cv l +crani o +courier journal +con formal +com a +co ffer +church ills +cd japan +carrief fisher +carol us +c ill +bryan ston +bour din +bil ity +big city +ben der +at oc +apollon ia +am coll +a ice +ðŁĺį ðŁĻĮ +ðŁį¸ ðŁį¸ +оÑĢÑħ иде +оÑĢÑħиде Ñı +yam yam +yak kopin +yakkopin ky +well ens +wan and +vit aco +trian gulation +tip toes +spread thel +revision ism +re vie +rat rod +pun net +phal f +pe pes +palestin ian +p sn +op ic +mill port +mie sha +mer cat +mee m +mb a +kl ick +haw co +h sin +guineap igs +gn um +e stra +cubes ats +cu an +cro se +ce mber +car min +bran caster +bo gummy +bankof ireland +ban ana +bac ar +ath u +at tucks +at palace +am ante +ad ang +ç¦ ı +zu ri +yo shino +woody ard +we stridge +v ought +un initiated +tx l +tuk wila +team dignitas +sustain er +su ps +storage wars +starr music +st weed +sav ann +sarah g +ric kast +remo vers +rat rams +r pp +pharmaco vigilance +pax son +op seaworld +omnic om +nissan leaf +new species +motor cross +mo gol +mishand ling +martyr sday +lay la +labrin th +la weather +kir ra +ki vanc +k ation +jun ctive +j jang +inside edition +in super +import ante +i ims +han off +ham ar +grav att +glos biz +gar w +en rile +diet mar +di derot +deven ish +cyn ics +chu ang +chester mere +bay bee +bagh datis +b ör +b go +au lani +ato v +am ate +adam sbeer +achrist mas +. ðŁĺĭ +% / +ðŁıĢ @ +ðŁ¥Ĭ ðŁ¥Ĭ +íĥľ ìĹ° +ঠľ +youn us +win dom +wil fredo +wi ve +whole grain +vit amix +v sg +u dit +ty mp +t mm +t caf +sw ed +st clair +selec tman +re validation +perfect ly +parrot fish +pacific northwest +orbit z +or miston +one um +oliviacul po +np cc +nor iko +mod der +micro film +meg gie +me sab +manner ism +maic hard +lucha elrey +looney tunes +kum on +j man +invali date +in consequential +i drive +house building +gü len +gir lish +gal pin +forever freo +f kat +every pony +event tech +even though +esper amos +eny imba +em mer +elev ator +dun ny +du ken +dream er +dow jones +df es +corpor ator +colo gic +clu m +clairvoy ance +castle reagh +c ile +bun so +brain z +bi ster +bbc sussex +bat ali +anu el +® ! +z drav +yo shimura +walsall fc +video tape +un fairness +tas i +tann enbaum +sti eg +stephani ej +sri divya +snell ville +side swipe +se amed +sa ada +ry lie +reth ym +progre sso +pen hali +over sea +nic olo +national rail +mi thra +mi go +marin ers +leap year +kress ley +kau er +j kenney +inter mezzo +incre dulous +hipho pp +han ash +gar con +fic es +ff le +faiz an +do ac +dis loyal +cu g +chem a +cann avaro +c bit +bron chial +braz enly +bra hm +bic ho +berrye ssa +ben jie +beis govuk +b young +b gu +ayo tte +ak m +ak iyama +ajinkyara hane +after buzztv +âľ ½ +ya akov +world skill +vi rago +un characteristically +u ric +u pu +tling it +thisis not +tech meme +subpoen aed +straw weight +south wind +sit well +side step +shu tt +serge y +ru tab +ro strum +river banks +ple gia +pi xie +phospho rescent +pete souza +paris review +oned ayat +official keef +notone more +n bo +ms v +moun ir +monte falco +mo xi +min nette +micho acan +metropolit ano +mall i +ly m +kn es +ke ish +john f +job seeker +iu pac +invite es +inexhau stible +indi g +here with +hal di +go blazers +go big +ev eld +et bu +dre dged +detroit ers +dar relli +crohn scol +col sen +clarence house +cas ado +cape verde +cap ac +by ars +boat race +blen der +bick le +bi ding +belo ved +bad la +azur lane +ardnamur chan +aqu avit +applic ation +ani si +ame a +after the +aa ic +. _ +ðŁĺ©ðŁĺ© ðŁĺ© +âķ ® +yad utta +vuni pola +tru die +tra bal +the wonder +the planet +ter abyte +tay schilling +swis so +sor ority +shus engupta +sati ated +s gottalent +revel ing +ref low +ranul ph +pirate pride +pen tax +pall one +order of +or lean +oli d +nag in +n tx +mor an +mi sti +mc gary +man jun +lyn de +long staff +lo sin +lo aded +lake field +krakat oa +kis ka +kim wilde +ki da +keke palmer +johnshopkins sph +ib ach +hen nie +happen ing +h ent +gunne dah +glady sb +gen elia +gaz eta +frat ellis +fra gi +for aday +ermene gildo +ear thob +du ino +dren nan +dor us +dal an +cri ollo +cot tawa +city west +callu ses +c gh +bu er +bra vi +an fc +ðŁĺį ðŁijħ +ë°Ķ ìĿ´ +âĺģï¸ı âĺģï¸ı +ÌĦ ) +ys ch +x all +wis co +western balkans +wau paca +uneven ly +tw ells +tric homes +to su +ti u +snape ee +sha har +sac d +rie ger +ri ada +revi led +reas ser +re den +power to +pla stik +phen ol +pen k +pati a +pat mos +par isa +pag ers +pa store +njpw world +nca atournament +muskete er +mini buses +lon ny +life son +ky gomusic +kinky boots +ji me +infiltr ator +in voluntarily +hom elands +getit done +gal á +fre ja +first take +enscon ced +emily vancamp +embroi dering +eli ott +dizz ee +dhe a +del ica +dana her +crusad ing +com ando +cog nos +clam oring +cine malaya +can mnt +c adel +bryn mawr +br onto +br iny +blu x +bas news +artof visuals +aroo stook +aral gon +and t +after school +adu tta +)))) ))) +ðŁĺį ðŁĺī +âľħâľħ âľħ +wim berly +vw camper +vin ter +v sm +usur per +unmistak ably +triffi ds +tis bury +ther iot +tcu football +sw elove +stair ways +spa rent +sommeli ers +slee per +six to +seal evel +rol lei +rit chey +richhomi equan +pra sar +popu laire +pl ang +pizz ic +nor disk +nom en +mid tj +mi versary +mal da +loath some +living legend +laryn go +inter twine +i play +harry con +hand maiden +gar ut +fre eney +fixed andfurious +extramar ital +esl proleague +emma bunton +east bay +d ze +courvo isier +clo onan +ce ee +bron zes +brandicar lile +bo boi +bluee conomy +black flag +be heard +b cla +are vival +angri est +amcoll surgeons +ali ft +ab bit +ðŁij ª +ðŁİĻ ï¸ı +writers room +wo d +west en +west borough +w dp +vidar bha +uu a +ut ada +tusc any +tivi sts +te eling +sun na +sla k +sc pa +sav oretti +sat nav +remember when +re locate +re interpretation +priv ada +prim and +pheno typing +pal ma +pa pered +ol ton +ohhill yes +nu tra +mol li +midnight texas +lu gger +kabat aan +jitter bug +im parted +hudson river +holo dom +go flames +gn ac +gaz e +ga hr +g sy +free day +fly board +fer rol +etsy vintage +en ca +done yet +demol itions +de su +cro cuses +cioccol ato +cin quec +chancell or +bo ic +bla blac +bharatanat yam +believ in +baw dy +bat boy +awild cats +as fc +ame morial +alli t +ab aga +ðŁ¦ ķ +yn on +wallace burg +ves na +un questionable +tu meric +thermonu clear +the wolf +th night +test es +squ ire +shaw lands +shantan um +sai ram +rock ledge +rever ts +r co +pun cheon +opportuni sts +nightri de +nau set +mon dale +mil ledge +mi kul +mcal eese +malware bytes +mail sport +ma vis +llan rw +koh samui +keep familiestogether +kat suki +jug end +j atra +itali ani +ib dp +hä agen +hipho particles +from the +frog mouth +essel dorf +du tty +dre ger +de gaard +de coupling +cun nin +ch ons +bur gin +brune tti +broo ds +bra den +blue shirts +alto cumulus +al ped +ak ira +aj anta +! ?) +ðŁļ ĥ +ðŁĵ ½ï¸ı +ðŁIJ» ðŁIJ» +yan sk +wn ers +what we +wen ham +too oooo +thorns fc +sy er +swa ths +suicidepreven tionday +splat ter +sco by +salz berg +s jobs +rwand air +rose bowl +rezz ed +relap sing +photo shooting +ou there +okano gan +oh dems +o logies +nas o +mskar inab +moz gov +metamorpho ses +lu ella +la vie +jo ice +ha seeb +grey goose +goo fs +fur rier +frene my +ern becher +edf energy +dy outh +die hards +di bley +darrelli ssa +carolin as +caf reeland +buc n +bu ro +bronz eville +broken shire +bra uer +bon n +battle bots +av alier +at ured +alstro emeria +; ___; +åħ ¨ +wim dive +wil kens +vine y +vine eth +viare ggio +umic hb +top stories +the specials +stang gang +ss s +sp cl +sonom acounty +sloven ija +shot ts +sche ider +sch auer +s fer +rob inette +river thames +re mco +rbg sydney +ram nation +raiison ai +rabino witz +ra baa +premi os +pon tins +pav o +pau sini +onof rio +oc ps +nik kie +n we +n tf +mir co +ma dowo +lotro family +leyton orientfc +kaisers lautern +in canada +humili ates +hon fleur +hol kar +h lb +gy un +gun gah +ge isel +g px +foreclo sed +fl aca +dutch gp +dic arlo +diam ine +di j +corn husker +cole co +camigu in +bu ti +bhagat singh +beu kes +baro ka +bar ne +bandain amc +bandainamc ous +al pu +a ben +zam mit +yu i +wa ks +upad hy +up slope +touch wood +the story +sun ray +sie sta +sbag ov +sales men +s games +ri mington +resi stances +rescu ecats +re we +ra hi +r ms +photo cred +pgat our +ob ye +nra show +now ski +ng ata +ne ela +nan ban +middle brooks +mahabal eshwar +mag non +lyn mouth +lud lam +lu glio +lampas as +kurni awan +ki ko +k ma +jump street +jis shusengupta +jai shankar +i ud +i acs +hvac r +hu gin +harrycon nickjr +gun play +gor bals +ger oni +free friday +fos sey +folklor ico +fly dubai +fen ian +face to +el si +do vish +dac us +couple s +con away +caricaturi st +cardiff met +bridge tt +bri sa +bit map +batt ambang +auto cracy +ðŁĺį ðŁ¤Ĺ +zi au +wg na +wer f +weneed diversebooks +wa aaaay +vi ja +uss f +un diluted +tr v +town sville +to st +sky running +shutt ling +seamus dever +sch all +save bangladesh +sal ander +sad owski +robert j +prin tz +par ai +paci fy +or moc +olong apo +north shore +nebrask ans +ne ft +mcgr ory +leighj alland +lap els +ku opio +kov acevic +king ship +kel and +kat ainen +ka appa +il ic +heli opolis +gote borg +gom bak +gi bs +fi ora +doug stanhope +do ds +desig ning +d ander +clock work +ci gal +cham orro +ca is +bu it +breast fed +bio safety +ay ck +avi ans +auto corrects +app arte +ðŁĺĤ , +ãĥ¼ ãĤ¹ +௠Ĭ +womens football +win ne +wh iotv +vo iture +ve stry +triglycer ides +tol leson +tip top +this sss +thel an +the steve +super charging +stephanie abrams +sp aci +sch noodle +sail ly +sab is +ru gger +rati mes +ra vo +r pn +putt ing +post in +pm lunch +plan b +pi pette +pen tre +pe quot +paraly zing +nasa juno +nano structures +myer scough +mskristin kreuk +moose head +mil lage +mal te +lam da +lam be +khan um +ker mit +insemin ation +in nn +ho bs +hel ford +gum drop +ger bils +frug ality +fiel dof +entor noi +en dos +diar ra +dead sea +dat ura +dam ai +commiss ar +cla fout +chill an +cau ayan +buccle uch +blu mer +bitt man +bb ish +b tb +au reo +ani sha +angan wadi +ang kor +ahmad u +afflu enza +ðŁĴļðŁĴļ ðŁĴļðŁĴļ +ì » +âĢ¦âĢ¦ âĢ¦âĢ¦ +zab riskie +x om +wie ber +wal berg +ven timiglia +vau cluse +un focused +tbur soni +so bey +ske han +sk ua +shau l +sax o +real shoaibmalik +p blchat +outwar dly +nofly zone +mu thu +min nis +me sop +marcan thony +macie j +ludic rously +kat vnews +kak inada +k lar +jai ho +j kovic +instrument alists +in atur +homec oo +hol mes +forthe holidays +fel sted +feeling good +e gging +duc tile +duc ale +diri gida +colour ways +celebr ando +brun ell +brain stormed +black box +berlin phil +be bek +bbc newsnight +baby boomers +allu arjun +all ato +ac ol +ab ridge +ab la +ðŁĴĻ ðŁıĪ +ðŁį° ðŁį° +åĨĻ羣 æĴ® +wool ford +wo wo +wo th +ve ka +tw f +tri gun +tr aralgon +tor x +to bruk +the legomovie +ted lieu +tand u +t fest +t enga +sky zoo +sa asports +s england +ri zzi +rhy med +r fra +plou ffe +pit i +op an +on ha +off f +mon ik +mon al +milit ar +mccaf fery +mb assy +lp live +leaf snation +lati fa +ká¹Ľ á¹£ +kod joe +kle i +kis ki +ke ble +jig gle +jeep wrangler +h dg +go bigred +ger ties +gal atea +food drive +f ani +eme lec +ely se +drum corps +do ire +demet özdemir +da her +criver side +crack pot +collision conf +chal upa +cele bs +bul gar +br once +bham citycouncil +baldess ari +arc am +ah lam +afl draft +ðŁĻı ðŁĺĬ +ðŁ¥º ðŁ¥º +å³ ¶ +wunder kind +wn z +warrant less +w pm +w mar +uval de +un deserved +u yu +tt ough +tru itt +tre ty +toyo tam +tit is +tel fair +tejash wi +stat esof +stan well +sp ada +shor tz +re styling +re primand +re compen +r inn +pra kan +polar isation +pil as +pega sister +over react +over hanging +ou uu +ou ple +on elife +nier automata +new ley +mor ang +mccol gan +mat an +lon oke +lau dat +la von +kra ys +ine ment +gu b +grey scale +go solar +galaxy sedge +fr acked +febru dairy +ex tol +edgarallan poe +dr david +die sen +di stefano +derry cityfc +debu chy +de hydrating +cu ffe +chu i +cas p +car stairs +bug ler +bucu resti +bc bg +baw ah +ardi les +ac tie +ðŁ¤Ł ðŁı¼ +yoko zuna +woo key +tru ax +tel om +tech trends +sw stheband +stor res +so ames +sni fter +sheet music +sav ana +sali x +robert deniro +ro tis +rhin elander +red shirts +ram age +pol oni +pluri bus +peter pan +pen nie +p live +over land +or feo +nic ek +mor ir +mix x +mie ke +me aghan +mazz ola +le du +land ol +kib butz +java ee +j den +ian wright +haber korn +gavi ria +gas lamp +fu ror +firstworld war +fen ny +f dg +end c +element ary +e trian +dit t +distill ate +destabili zing +dad ri +comer cio +comedy central +cav itation +c ó +boiler room +bick ford +bett ym +as anta +aj staff +adro it +accor ded +ðŁĮĬ âĺĢï¸ı +whati do +we tton +v ri +tro xy +the mi +t mag +srini vas +sport scotland +speaker phone +sho down +sh apps +scoo king +san dip +ro di +rail yard +q ari +poten za +polit buro +pi gg +perpetu ity +par lament +pal ani +over work +ourhome bts +ott is +ogall ala +ny cb +no ts +nis kay +new play +ne aarts +mc gla +ma fs +lu pone +lo bi +le red +kor ir +kel liber +john isner +illing worth +i ui +humble brag +ho gg +go yo +formula one +eras mus +enricom olinari +election ske +double trouble +dev ours +de generates +d models +cheong sam +baro que +at ower +affor ding +ad ate +ðŁĩ¨ðŁĩ ´ +wim pole +wh ish +waian ae +visit finland +turn downforwhat +sp lot +sexual health +ro cs +river stone +ri squ +poun ced +phoen icia +peace making +patient engagement +oz aukee +offic elife +news goldcoast +neuro feedback +moon ie +mcgill ic +mc cra +marti jn +labou m +la du +jon atan +ji ki +jam ali +ito day +inner space +hydro static +h ppy +ft c +friends forever +friend z +free charge +fish nets +evalu ators +esch atology +edu k +d win +cross er +crawl space +clayton kersh +check box +av otes +au da +an jal +alay na +al maz +ade sanya +ðŁĩ¬ðŁĩ§ðŁĩ¬ðŁĩ§ ðŁĩ¬ðŁĩ§ +ì ³ +âĺº ðŁĺĺ +x ray +universe today +tr ing +tor iam +toi indianews +ti sch +sing ler +sigi riya +se pi +rei ji +re all +ravens burger +ple dger +pk subban +photoof thenight +pete hegseth +pam plin +outsmar ted +net minder +mer amec +main line +ley burn +len ard +kha il +k bb +just ise +junk food +ju tting +ji k +hi h +har asses +gri zed +gra hn +god den +geop end +fuku da +fru gal +fre ear +f enti +en fp +ei th +east london +dan electro +crun ched +cod ghosts +co teau +cnbc africa +clu buk +carlo ta +bush master +belli ssimo +bel z +audi en +au chin +atta c +ast and +artex hibit +anat omic +alman ack +ðŁĺĦ . +ðŁĸ Ĵ +ðŁİīðŁİģ ðŁİĪ +âĻ¥ " +ॠĤ +yyyy yyyyy +u tters +u erto +u dio +tur fs +take my +social club +smur fette +sexy back +serv ite +science hub +sb way +sas u +rum sey +oper ade +nwan kwo +new belfast +never not +master piece +kho za +ke res +jan nie +james reid +insta size +hoar ded +hir scher +high snobiety +grun ting +gofor gold +gi vi +flu gha +eo sinop +ddf tennis +dal les +cro sse +corn starch +consho hocken +com posted +cod worldleague +ch ta +blake ley +bartol ome +bab os +anthon ynolan +air patrol +ðŁĺĬ ⾨ +wom yn +west pac +victor ino +v um +v tb +un appealing +uh v +tx wine +tusk sup +tram pling +tou re +tou quet +tostad as +tonythe sharky +th aa +tar por +tang s +su ger +stra sser +south bury +som bras +shand wick +sc ip +rout ledge +ri stretto +real ale +phillips burg +pett ine +palli ser +out station +olajide bt +nor cia +ni hl +monster palooza +mis shapen +midtj ylland +mi yako +mel ale +local ism +let live +ksi olajidebt +ko ston +k sr +jodre ll +jacob y +izz ah +gor khal +glori fies +gali lei +gal los +frizz ell +finns law +fe ms +en jol +elaz aro +don agha +di smo +dam mit +credit on +concentr ation +closing ceremony +ca ine +bay port +any u +an king +aku ya +aj pw +ab ndp +ðŁĩ¹ðŁĩ ¹ +youth voice +wy ss +wool ly +whir ly +westh ampton +walt zes +vill eroy +toronto library +torbay hour +ti thing +thumb elina +thrif ty +sub sides +steam works +ru ach +re dic +ow ler +napo ca +n olo +mess iness +me ira +kianegan wl +if taar +good girls +gi ef +gen icity +gar nier +gag op +frankfur t +e ban +creation ent +cr sng +circum navigation +christen dom +chapel hill +bor ax +beryl lium +bad der +austin basis +arca di +an bu +amber gris +adorn ments +aber aeron +_ # +ï¹ ı +ê¶ Į +å¿ « +âľħ # +âļ«ï¸ı âļ«ï¸ı +zechar iah +westy orkshire +well done +we zi +val tell +un aided +ul time +tur turro +tis me +tat sumi +tar heel +sub stations +su rab +ste eds +sp ittin +shinge kinok +shi vu +shef ali +sal ters +rome u +ro mps +ro mare +red book +rece des +react native +q ns +oli vo +ofcal dub +nu do +no bile +ne whe +mu tes +mbab azi +mati p +mag ali +lov atic +ley y +le mente +last chance +ko wal +ke ek +kav it +karas wisher +joem cel +jay dee +jab dulkalam +i id +horse tail +hit record +hill en +highe red +her li +gru en +ge te +fou lard +flyn avy +fled glings +fi ya +ferman agh +en ov +don tre +co be +cal ne +bat y +barbar amills +bak so +arba az +an pan +an ai +afric abiz +( ? +ðŁIJĬ ðŁIJĬ +âĢ¼ ï¸İ +xilin x +whynot us +we ho +war de +v are +ute p +upl and +thir sting +taw fiq +su ren +st wo +squ ab +simple ton +si fted +pra tham +pay an +pa ez +osc on +nytimes books +no stro +nev ado +nay antara +natural health +nas wamy +mo chis +mk bhd +man ey +lu ma +loveli braries +la so +kiefer ravena +kat sumi +johnny gargano +inju rious +ine o +implic itly +hu alien +hoo ped +high clere +hey violet +he ian +ha ath +gm hc +fore arms +fer ny +en vision +en raptured +emer al +e bi +corne jo +commun ing +cedar burg +ce ph +cam ero +cabo olture +blon ds +bar ish +armi stead +apri mary +apo sto +acom edy +ðŁĵ¸ # +writers block +warat ah +vers ic +tom thewanted +ther adio +sp row +siva ji +ru tten +ration ally +par te +opol itan +o jama +nose dive +none w +ni osh +na as +myceli um +mor oz +monster hunter +merri am +maum elle +lyric opera +lutter worth +kron o +ker ma +juxta poz +j sk +inj era +indi ac +hu ach +hd fc +hak ur +gug ino +gl or +ging ko +gi ggled +g isd +fl yy +fen ella +fearne cotton +far ha +f kf +export ing +explore mb +es for +ent ente +down field +dol gel +day sin +da st +crowd funder +compe ti +code breaker +cla rens +ci bolo +cater in +cari boo +book plugs +ble eker +big ge +az ian +aw restling +atul ations +apollo theater +ag itate +ab ird +ðŁĺĺ ðŁĴĹ +ðŁİĪ # +âĿĦ âĽĦ +âļ¾ï¸ı âĿ¤ï¸ı +ynlle th +yel p +whi pper +vote ph +vat ore +usat f +uh mbt +that girl +thal esgroup +stop gap +sto so +stepla dder +spare ribs +si mp +shadow play +ser in +secon ded +scot thall +rvad ine +ruck a +row der +river ine +rik ki +radi okc +ra chi +profit eers +pp q +por ium +polynom ials +ple ee +phlebo tom +pear sall +pam ela +oli vares +nor te +nano tube +mar mi +ma gui +lisam arie +le mming +la hari +ky sportsradio +kk ur +kelliber glund +k ents +joshu at +if bb +iaff newsdesk +i leen +ho ax +happ ie +gar rod +from scratch +evoc ation +drewest ate +done z +dam my +d tu +cha ar +casta ic +bon elli +bo jack +besee ch +aw ai +au tun +aly sha +ain f +af ate +íĶ Įë +ãĥķãĤ§ ãĤ¹ +ಠ³ +youare notalone +yash in +whit worth +vi rage +veri der +the godfather +tame ka +tad hg +swami vivekananda +st witter +squ alor +spruce grove +snow suit +shi ori +sh ilo +san aullah +sa wai +ro ssie +rizz le +rhy sme +peppe rell +ony i +o el +ne dd +naz ia +navrat ilova +nar an +nam u +mul berry +magno lia +mac duff +lu ego +law ren +la scru +kam ma +jimmy page +jc cc +jag z +implo res +hur rying +happybirthday louis +gu thri +ge sell +ge ee +films notdead +epine phrine +du mba +dra ya +dhy an +death valley +coo ol +contamin ating +colla zo +climate action +capital isation +bre zh +bouff ant +bor don +back haul +av tar +aho ya +ablanc a +abidi factor +!!!! !! +ðŁIJĺ ðŁIJĺ +ðŁįº ðŁįº +åĿ Ĥ +wood crest +weak ly +ver vet +ver ner +ver mel +valdis ere +uni mog +ton ny +tom tom +tl h +ter ization +tel ome +social housing +scho eman +schi er +sat ch +rhon da +rashe eda +od di +o ita +nt chat +ni aga +mir allas +matte orenzi +magi ster +kun de +ku mu +khich di +k scope +k ado +james dro +j rr +insight moments +ing is +indi af +in firm +h bi +gri st +fe g +ef ta +destin ed +de cluttering +clergy man +cler c +classic rock +cave ats +cap on +cam pe +billi ken +be zu +ayl sham +aye ga +art man +anglo phile +âļ ĸ +z mir +uc ps +tyre ek +twer kin +tre et +thwar ting +teacher problems +tat ras +syn cope +stall holders +sports law +rise above +rema de +ra gas +pre war +port side +po sti +phe onix +pett itte +pen ia +pe eves +ol shop +od ell +nu tty +nat acha +mun cle +morin ville +mol ting +mc gwire +mas lany +malm steen +lee way +l po +kill deer +k team +ja q +j aff +itweet museums +id pol +go bier +fp so +ecar ter +dream pop +dra go +don buri +digi fest +death less +de di +dan ton +cou pler +ch ng +cal das +black belt +ball nyc +ba ghe +aviation daily +and ini +ama al +am ira +aha a +ðŁĴļ @ +ਠ² +za x +v sk +un im +tu atara +tt ag +tan ehis +tanehis ico +stell acre +spir ited +serv us +scott sville +ro ches +re arrangement +pung gol +po ivre +pig pen +per ation +outstand ingly +oh bm +nn taleb +ner dist +na hum +ms state +mor tician +mel os +me ti +mar kin +ly k +luck now +ktn v +klepto cracy +kenne bunk +ju gend +jol aburnett +j jc +inou ye +ign ition +i ima +hope less +hodg kins +ge th +fir at +f dk +ex tolling +ei fert +divul ge +deter mini +dei mos +dam aris +cr aco +cor dy +clau del +ck ton +cityof boston +cere mos +carlit os +brass band +berto ia +bent a +b have +aw rence +as ci +architects jrnal +ann yt +ann nn +ameri go +ah ana +accur sed +aar av +ðŁĺĦ ðŁĺĦðŁĺĦðŁĺĦ +ðŁĴ£ ðŁĴ£ +zis sou +you reyes +wag le +vi bhushan +tv c +turn down +tt age +tr d +te fal +t ver +t smc +super structure +spi got +speed kills +sou derton +skip the +sil sbee +scri bd +sa wa +s anni +rotham sted +robertsspace ind +rigor ously +r ori +potawat omi +play out +pas ang +p elli +ouston bill +ot tica +one man +nor then +no id +nis qually +ni ved +new grounds +n ÃŃ +mor gado +mc dade +maq bool +main tainer +mag ines +lubric ated +llande ilo +lar on +kop er +khil ji +joemcel derry +inti mates +instac ool +ino hio +im acs +i fit +hot star +hij rah +he modi +guy kawasaki +gif ty +fre em +fr ater +expe dient +ergonom ically +eph oustonbill +en amore +don ts +dissoci ative +continu ou +collabor ative +cleve rest +cam pinas +budd hist +brin ley +bla in +bbc new +bay on +bar tle +ax c +aq at +ap om +amar tya +alo kesh +ag glomer +ac tros +ye olde +y itzhak +x or +teme scal +str acci +star chy +sel lo +scar p +san doz +sa hota +raw ley +radio graphers +pet ts +pet shop +pet sch +opha gy +off ically +nandam uri +muzzle loader +mo har +marco tte +lun atic +kk w +kil ted +ke vic +k fs +john krasinski +hill cats +hen ni +handson learning +grin ners +graci ela +fri dley +fe ted +do bre +demon izing +comm itt +code share +castig lione +cap onata +bud die +buck o +bra ult +bi alik +bhu j +bal een +an avenkat +. ðŁĴĹ +. âĺº +ðŁıĨ ðŁ¥ĩ +ìĿ´ì ¦Ī +ãģ¦ãĤ ĭ +z m +we tten +tz mann +tu gh +tra ut +tion of +thought fortheday +the field +super iors +stere ok +source books +sour beer +sob ha +sil berman +si ero +shu riken +shop house +shari f +see theworld +se fa +scu gog +sc rn +ryo suke +rr sp +rot fl +ri pley +re mick +r lm +queere ye +pur port +plu mas +planet fitness +orth west +of cr +neg ombo +mot ility +modul ating +mm sm +mcgu inn +mb n +man soura +limit er +li j +ká¹Ľá¹£ á¹ĩa +kry sta +krist all +kariz ma +kam pal +k sk +j ra +immigr ate +huck le +hood winked +ho tele +hitrecord joe +ha ine +gro veland +google io +funny ordie +free styling +fer g +ebay us +e hrs +dennis elazaro +dem itas +chilla xing +chick am +cher vil +ch yn +carto ony +bri sson +bookie bashing +blan ked +be held +bad akhshan +b ve +att in +arn age +angel man +andthe city +ad ve +ðŁĽ ¶ +ðŁĽ ¡ï¸ı +ðŁijĮ ðŁĶ¥ +ðŁı Ľï¸ı +ðŁ¤¦ ðŁı¼âĢįâĻĤï¸ı +à¹Ĥ à¸Ķ +ठĩ +ÙĦ بÙĨ +ze meckis +wil burys +waste iton +war farin +v yy +v spit +under dog +transa via +topp les +ten uta +tatatru sts +tac kett +ta wang +t iler +sol ara +slive at +scho tten +schap iro +san an +roo depo +ro gier +retro actively +redribb on +pa hal +ol lective +oc co +note cards +mb on +ma pr +lal ala +lach ine +kirkin til +kan ovic +kal at +k cac +jo cke +jau n +hoo ba +gol fo +giu sti +gh anta +gg ings +food tour +food stuff +fiestab owl +festo on +f ale +esp agne +er inn +el ax +eb ates +e vie +dover court +defen der +dav ros +d lg +corpor a +conju gal +con dra +compe te +cheo hospital +cas illa +bur ston +ar sha +amhar ic +amber ley +al waleed +al tr +âĸ½` )/ +ঠŁ +yar ratrams +wann eroo +wa cht +valle ly +ten na +story ville +steve o +ss entials +solic its +sho m +she athed +san juan +sac tive +rose mead +re stocks +per seve +pedago gies +over charging +olo pez +o reca +my house +must love +micro greens +ma zo +le eper +jim bo +itiner ant +ir radiation +intothe spiderverse +inter bank +halloween movie +gy uri +glori a +gi dley +gho tel +gett ys +fu cc +flow restling +fle ck +fl ander +fire itup +fe stering +faryaltal pur +er dahl +dumb foun +du esseldorf +down link +darwin ian +crissc ro +cor rina +cle fs +city market +broom hill +belling cat +bag at +b gg +ate y +as mus +alet tori +ak lan +aesthe tica +a out +ãħ ħ +ÙĬ Ùĩ +wor n +whole hearted +wan es +video conference +us coastguard +up church +tw addle +tuil agi +topp rog +team ster +susan sarandon +su mas +sreeni vasan +sol r +share aholic +shal er +se idler +sc avenger +sar sapar +sap uto +royal holloway +reinst ating +region alism +pulw ama +pri macy +pin sky +pat ellar +ouri st +nough ties +mous asi +mor fa +min elli +meur sault +med line +mas bate +mar voree +maine for +lo gro +lmk moviemaniac +liter ature +life uk +lief de +lhh ny +laval in +kome dia +ko tab +khar y +kau a +jes i +iro z +hilde brandt +her to +he yy +gov abbott +god wits +gl bc +gare t +galli ani +free ajstaff +foli o +fli ka +f issu +ev ir +e sus +den yer +dad dario +contemporary romance +col ler +coale sc +clafout is +cho wan +chare tte +cere bellum +by ford +boro silicate +bla bber +au ras +arou sing +alexander platz +adver bs +íĭ ° +zu hair +vol cs +ur bant +ter cera +te ye +swamp scott +sundar bans +stru tt +speech writer +sp rod +skid ded +silver a +shav uot +sci fic +schen ley +sam aria +sac republicfc +retrac ts +redro ck +re doubt +pier ro +pere a +oc sc +now bath +notb ad +nom os +moncri eff +meme history +mckend ree +mask er +lay ar +kha qan +keyne sian +ju dah +go green +gi om +fo cal +flour less +dry january +dis se +del hai +champion nat +bou tros +boere wors +blind sided +ay or +asap mob +ar monk +addle stone +aar de +âģ£ âłĢ +wax haw +wai pa +vr v +ver ville +tri ghts +trade center +thaic oup +t mac +stri via +strath co +stra des +sponge bob +so th +sen sed +sel alu +scru bbers +rwn j +roman off +robg macfarlane +ri gu +ri bbing +redskinst alk +pe dest +omic con +no ce +my girls +match worn +mass art +marc jacob +magnu sson +madef ur +machine gun +lor ton +lily aldridge +less ard +leg ate +l ä +ku cin +knu te +khlo é +ju p +infl ames +imp i +ic ep +hudson ville +ginni fer +for people +f yl +eli o +egg s +dir ge +ddp yoga +concentr ator +classic traction +charm bracelet +chamele ons +cas alettori +candacec bure +c sl +blan ch +ba bee +americana fest +alwayss unny +along with +aline a +ais ne +acqu aint +abb oud +ab ric +ðŁIJ Į +ðŁ¥ İ +zacu to +yello whead +wi ek +ween ey +turbo charger +takeme out +stock twits +stam aria +sonic boom +sky bridge +schu ette +sacri legi +ruby ph +regu s +quest live +py lori +purcell ville +ped ant +pe toftheday +pal mar +opale scent +ob ito +new age +neglec ts +national library +nash snewvideo +mt nug +marlon wayans +lu lla +leep ing +lec raft +landscape design +lanca shirehour +lac quers +kha yyam +ka beer +instig ator +infen wa +hemorrha gic +hell mann +hel g +gri se +graffiti ed +good health +good guys +get outand +gammal abs +fru ita +fr f +drive sober +direc tedby +del phi +cont es +clay gate +chill y +chal te +cen otes +cam bell +brett king +bobc ay +bar bier +auto clave +as able +ao ki +anic ole +ami ans +aj as +ag old +... ;) +Û ¶ +zen berg +wm hd +we imer +vir ging +verdi gris +vent ral +var ys +un realized +tto es +ton neau +thing ies +the aftermath +tac on +syno ptic +sul fu +st cc +she ab +sesqu icentennial +scul pts +scor ner +satch mo +rit as +restaur ateurs +psal mist +p ssa +nor o +neh ru +muell ert +mu sim +milag ros +meri vale +mend is +mas r +malli kar +le ea +lamb chop +kyo sho +k need +jel en +irishtimes sport +ino vich +ig at +horn beam +gare e +frank dangelo +fly leaf +fag in +e inem +dun donald +drows iness +dak in +cross words +conqui sta +congru ent +com al +cle ck +ch merkovskiy +bu she +bridg eland +bor gnine +armer ie +arch on +ap la +ane gan +ak ina +" ðŁİ¶ +ëį Ķë +wood bin +wit ten +wheresw ally +wh alum +u windsor +tit chfield +thex factor +thermo genic +ta dema +sw pa +super cells +sung jin +ste h +stag gs +son ning +sk ratch +si eck +segre gate +sal ak +sag at +saf ed +re kor +publici ze +po sit +pl g +me vag +man nin +man hwa +m â +loving kindness +l haynes +kp bs +klo tz +klg and +kl rahul +ke da +kawor u +ji k +hol lies +ghe ad +fertil isers +el itec +docu men +diamond league +condon ing +co td +co debase +chev in +british basketball +biopharmac eutical +bat ons +bai les +bab bo +autom atic +ati ku +amer a +ac tra +" âĢĵ +ðŁĺ´ ðŁĴ¤ +ðŁij© ðŁı¾âĢį +à¹ĥ ห +ॠ¥ +zo ar +wel d +un attached +tu co +tre mura +sub duction +slu shies +sla zenger +sad dled +rock pile +ris ke +realradi kaa +qatar gp +puri sts +plai stow +pat roller +pas anga +pa ils +one piece +ol athe +neighbour hood +nav ox +n alla +miamis burg +mar anda +makat on +magnifique france +ma guin +lake como +l bardugo +kiel burger +kaz umi +kar issa +jo ki +jag i +it cnews +independent film +ge est +force indiaf +for cen +fe br +farm worker +far low +educ ator +despat ched +che tto +brom eli +bro y +brexit vote +bond holders +ben nel +ba ink +avol leyball +antel opes +able ism +ðŁĩ¬ðŁĩ§ ðŁĩ¬ðŁĩ§ +Ù ¡ +а Ñģ +woo ed +wh Äģ +wat es +un ker +umbrell amovement +travel oregon +thi seid +the spec +swat ter +sarco idosis +ro ig +pres splay +pontel and +pharmac okine +oye z +optimi zations +omand m +of elia +n ood +my fav +mi seleccion +medium format +mal y +mal ar +main tain +lin sky +lilli put +leven thal +leeu wen +led lighting +lakedistrict pr +kundali bhagya +kep ner +kath mandu +joel madden +i pur +good all +gay marriage +fre und +espo s +dr ps +dncle ak +dal mia +da gh +d mrc +cro we +cover crops +concert tickets +cinquec ento +chri sley +camp fire +c qb +brett lee +body works +bio statistics +bell arine +battle toads +az sen +av p +astri de +alevel results +al be +ab dou +a athletics +. ðŁĴĽ +ðŁĮ» ðŁĮ» +ë¹Ħ ìĬ¤íĬ¸ +à¸ĩ าà¸Ļ +wind stream +wi u +western most +wear house +war bucks +travel photos +tom and +to liver +spet ses +solidi fying +si qui +si im +shopp es +shari ff +sar m +row boat +ri gon +rebe ka +rain storms +r rexpress +po kk +p ted +oz comiccon +oa key +nw sa +nc sl +nandit ath +myster ium +mclen don +mary landers +margaret cho +mal abon +lot so +lauren conrad +lar in +ko sky +ker ner +k vin +ju ha +ju ego +intermin able +il sa +hum dinger +ha diya +girl guiding +ghan em +gel ish +g pas +fokker dude +fat ties +drum head +di atom +detective pikachu +cor vettes +cop ali +common ers +che kov +card making +can pl +brit z +big day +betwe ens +be sta +bc w +ar kan +aj ga +ai as +ðŁĻĬ ðŁĻĬðŁĻĬ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +ðŁķ ¢ +ðŁĴĢðŁĴĢ ðŁĴĢðŁĴĢ +ðŁı Ķï¸ı +wa seca +vi stic +tam pax +string y +store fronts +stellacre asy +som nia +sin america +shovel er +sc lc +sant an +rit aco +restaur an +recording acad +re ys +ram shackle +r weingarten +r ital +par lez +noy ce +nhl devils +national security +n be +mur si +mun t +mohic ans +ment um +mc taggart +lu sts +lead better +lam entation +kom unews +kis mayo +kikwe te +hexag ons +he dged +green finch +fic ht +fav ase +ever sley +et am +es muellert +east village +dor ito +cz ek +collegi ans +cn comusic +char le +aviate addict +at ac +arlington va +aldu bon +acade mie +able ton +ìĬ¤ íĬ¸ +ëĵľë¦ ¼ +ı r +yaman ashi +x rf +wichita state +vit as +u ams +ting ley +ti ft +theoffic enbc +ta rena +suther land +student loans +skir k +sidharth malhotra +scot gov +sarki sian +post marked +pnc bank +pg d +patho m +paleonto logists +pack horse +orient alism +ontario travel +o ssi +npr news +nl bm +ni eld +ne ms +my style +mercado jen +meh fil +mcc une +mc stay +le ucine +kore gaon +ko lolo +ken an +keen ey +karl stad +jimmy carr +hepat ology +hee ey +heas lip +grand stands +geta fter +gar dy +gam la +gal ante +for shaw +expatri ates +exac ta +eisen stein +dx f +di alling +detal les +cramp ons +contro ver +cn blue +carri zo +cal d +bur ri +bu thele +bu skirk +bo vary +blesse dand +black pinkin +black ink +black cap +ann alyn +ai w +african art +acom pton +ac cp +ab idi +Ļ × +íı ¬ +wide man +wallpaper mag +tro tter +travel goals +thel ad +the music +sque aked +shou p +sher on +sheep dogs +shar wanand +scur rying +sant arosa +retro computing +president scup +poly p +plu ie +per ouse +nick olas +new sday +nade em +my bcu +menom onee +mcmur ry +marau ding +lizz iec +ko echlin +kasim reed +kap ital +jun kie +jenny mccarthy +j news +hur dling +harmon ics +h cac +guardian books +goes west +gaud ÃŃ +fright fully +fin au +fade away +ethiop ian +espn nfl +e fren +du ren +dovi zioso +do gh +crohnscol itis +cro oning +cristi an +conce iving +co ghill +chibok girls +cer ne +categor ical +cancer center +bus driver +bri ens +brendan schaub +barcelon eta +ban the +ban ality +alcu dia +ache son +aam er +[ ðŁĵ· +é Ĥ +âĢ¼ï¸ıâĢ¼ï¸ı âĢ¼ï¸ıâĢ¼ï¸ı +¯ ï¸ı +winniethe pooh +wander ings +u fff +tro st +tri veni +tremura sdv +tac ar +store room +star ling +sit t +septa philly +school strike +sch ist +sam anta +sa fia +ru pp +ro yor +reli sted +pescat arian +pe ats +pan sion +pad mini +ny sa +np ca +nicol ás +ne as +naz e +mis r +mis deeds +mcn ary +man ana +magazine uk +ker chief +kemp town +kal ye +k sf +ise tan +ing len +im memorial +ig b +gravit ational +got w +gospel music +google plex +ga as +fresh ly +fren che +food awards +fis u +fi delio +festo oned +dy mond +du ell +dot ard +df ds +da emons +cabine tre +blur t +bloomberg news +blood bowl +be dra +ban de +arbitr ator +alle ppey +alan ine +adjudic ator +adjourn ment +ðŁĩ¦ðŁĩ « +ðŁ¤Ļ ðŁı» +íĺ ľ +ë°©íĥĦìĨĮëħĦëĭ ¨ +âĺħ âĺĨ +âķ ² +wwe usos +walkaway from +vi ele +usab aseball +un cp +ug k +u hh +tuesday thought +too soon +somer sault +si dec +sen et +sar ap +run d +roth bard +rhiz ome +pinnac le +ost end +nbc thisisus +n pu +mc brayer +lynn field +lv rj +lep online +ken stein +kar mel +investig ational +hoo g +harry winston +gg plot +genev amotorshow +gem stone +ge ge +e jaz +ds v +do ss +di atoms +davi on +dalla ire +cuer nav +ches nut +cam phalf +bag nall +ash wani +anstru ther +amon til +alex il +alex andros +al tor +agni hotri +ðŁĩ¨ðŁĩ ® +w wn +ventric le +valu es +va shem +v rai +une a +umph rey +stay hungry +stan age +soci ete +skip jack +skat elife +scal ar +sanjay gupta +remember ance +real deancain +re thin +pum mel +pr yan +pardon ing +n win +n cos +muzaf far +mur ang +moss man +michigan state +meteor shower +melale uca +ken wright +k all +ih san +he bobiwine +har diman +genes ys +ga e +flu fighter +fe vents +fac eyour +el ho +dro ppers +devan e +cor bet +cliff top +ce de +car stens +car mi +c add +c ack +bur son +bri son +boule z +born stein +appé tit +Ë Ĺ +ı r +zombi fied +z l +x danny +white genocide +vincent vangogh +vesper ia +ug at +u hu +today y +te cs +suki yaki +stuart broad +stret chered +stra ddles +stal ac +shrenu parikh +sch wa +salvage able +s wines +ro sato +restor ative +pu ti +pre pay +on vention +o gt +nuz locke +new friends +nanditath hakur +nai b +million miler +micro gravity +metast ases +mechanic sville +maris cos +ma bility +lav ington +la vat +konstant inos +kipp ax +ker man +ke ar +k ella +juan ito +jan kowski +j wc +irrit ant +her rero +gyna ecologist +gws giants +gu mm +god sown +glory togod +gl f +gar ang +flux us +flash card +ely sia +el yon +edwar dj +dee wan +copali bertadores +clean bandit +bu sted +bry ar +besto wing +ar chri +ðŁİħ ðŁİĦ +âĿĵ âĿĵ +à¹Ģภ¡ +Ùħ تع +Ð ³ +weav ed +we transfer +vi ale +theresac aputo +task bar +superi ore +sub compact +stracci atella +small holding +sig no +si gui +sex work +s iller +revin ed +rak awa +quality time +pu battlegrounds +play music +pi dge +oppos itions +nov ates +n ml +med ill +litt ler +lee seunggi +l lew +koval am +inv ul +in adequacy +idri ss +hyper realism +hu got +hooba stank +grandi flora +fronti ers +freder ique +fast furious +dur fee +dresslike awoman +do les +do ddle +diatri be +del ange +db z +d jor +ctv canadaam +ci karang +char tier +cease less +calend ar +buzzfeed news +bon sang +bo strom +bit c +bin omial +as dale +anti psychotic +angel e +anag aram +a betting +ðŁĺĮ ðŁijĮ +ðŁIJ¶ ðŁĺį +âĢ ł +ि à¤Ĥ +wood cut +whit mire +ver laine +ut tering +u gali +thr ongs +the hate +the ab +th ate +ter acing +star maker +sri ght +son or +sky west +se anc +rohit sharma +rl cs +real king +pu tman +petiti oner +pau lino +pas qual +p bl +nigh tofthe +mus er +mul lin +mom eter +med student +marj ori +mal ena +ma doc +kim meridge +ke ven +k wale +jun inho +juju be +jalop nik +jack russell +indian ocean +il lar +idi ot +hedon istic +gopro hero +glass art +fri ese +empirestate building +dil shan +digit ise +dark throne +chamber tin +butter fingers +bulldo gnation +bu ceo +bri sky +bil dt +at tains +app in +and home +abc sharktank +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ +ìĹIJ ìĬ¤ +ãĥ³ãĤ ° +yan ez +wood mont +wh f +wewill win +un heated +twitter chat +tra um +tra fic +town ers +the devil +svo boda +skirmi shes +schumer shutdown +sarsapar illa +sa ks +s rocks +russell terrier +re ster +raf taar +pa sic +mu rai +mou rad +mor ongo +mol u +mattj willis +mar der +man ami +mac namara +lipsync battle +lasgo w +lar is +lake shore +ksh ama +kid well +k bl +ji wa +ji az +itim es +i ese +hus key +ground work +ful le +fol ky +far ook +exor ci +euco president +en sw +criminal justice +cotton bowl +consign ments +charlies angels +bridge ton +bau ghman +bary shnikov +ayles ford +au si +au an +alo sa +ai anational +ab ren +ðŁĴľ ðŁĺį +ãĢĮ # +âĹ İ +á Ħ +yon sei +yeoman ry +visit norfolk +u staad +try pod +tru dge +this oldhouse +the g +tai moor +tac cup +swar brick +stre eter +spro g +sex i +russ el +ram stein +pur ves +proxi mus +pre heat +pon i +phi lemon +pedne kar +path way +over spending +ole a +naz o +na ks +me gay +mari amen +man é +ma bs +kor te +ju rek +jay na +itv central +i sea +he mmed +hd ty +habitu ally +groun ders +grahn ort +getafter it +fram ers +eucli dean +emin ently +dmitri y +dar vill +chi ya +brown stown +bla stic +beli zean +ba elish +aw ka +as ssss +as ke +arc adis +and now +ìľ¤ íĺ¸ +ãĥ ļ +za atari +wix om +wait iti +vor ster +var mint +v dl +u mag +tu tic +tre ffen +track field +the tru +the jam +the crown +ta we +sur man +sthelen shour +south dale +sher r +people soft +pen o +parid hi +nyc go +music ale +mont morency +men ac +logan sport +lady wood +je bat +hem line +heck man +gla drags +gan ano +fo lo +find my +ever brandy +den rele +defam atory +dam odar +dag gett +count yo +con vair +chlor inated +cher ian +capit ola +cam mie +broad dus +audi ophiles +asjad nazir +ashley tisdale +ar mon +ar ino +anth uri +amor uso +aj kumar +ab ani +ðŁĻıðŁĻı ðŁĻıðŁĻıðŁĻı +âĻ¡âĻ¥ âĻ¡âĻ¥ +women on +vol nation +vi m +va beach +tw rites +thunder ball +tag bil +super bikes +star tre +spur r +sof tie +sligor overs +shack leford +sco te +sci p +sche ffler +sar ver +sab tu +ros endale +romance books +ritaco bix +re styled +re heated +re eser +que z +pre classic +pp arty +po les +pis ani +pfei fer +our land +nak ia +n pn +mur ry +mit ro +mario bros +maharash trian +l lao +koffee with +ka at +joyou sly +jak o +ig loos +hilari o +ham bur +hadou ken +gle sias +ga et +fo ggy +experien cen +execu tor +ed dington +dre ezy +dibu j +devi ance +den nish +dash croft +cul che +congre ssion +col later +chug ach +cher rie +cel gene +camp al +bundeswe hr +bu shell +bt sat +bm iller +blit zing +be ag +av it +air crash +ai u +ðŁĺį ðŁĺ± +оР½ +un li +un accountable +tim minchin +thom ase +thi eriot +thermo dynamic +th grade +tea sel +tartu fo +shul k +shaile sh +se bum +sc df +s di +redemp tion +re pp +rach ell +r sr +r sk +r ne +que zada +p ounces +os ine +onedayat atime +officiald annyt +obye zeks +ni thin +nar iman +mostre questlive +mor ales +mat tan +mal formation +make history +madra sa +ma eve +le ant +kö ping +ku dus +kow sky +kan ti +ira k +i mari +how lin +hom ophones +ho vey +ho sur +hin ter +har ber +h sm +grims by +go va +go om +gil by +gar gle +fred rickson +faryaltalpur pk +emulsive film +e jc +dumb founded +dar ah +cold harbour +co fi +chi shti +can ara +calle jon +budg am +bronx ville +best fandom +bel sky +am mu +ak ula +ac cra +^^ ; +ðŁĺ¸ ðŁĺ¸ +ठŁ +wing back +wee don +w anger +vla sic +ur phy +u din +u apb +tri ath +tette h +tess anne +tae hyung +su kh +su bl +songhye kyo +righ thand +qu illing +pp ar +padi ham +ov ski +ous ly +ne mac +nature photo +n ason +ms q +mi mms +mer lion +lin sey +land au +ko sar +knight dale +key port +je ke +jack pots +in sensitivity +her ky +hare field +gu mption +fu ca +fl inger +fe eny +dumbfoun dead +di ak +chri shar +cep bs +bow ley +bou m +bor hood +backto school +b ica +are gion +aj dinger +.. ðŁĺĤðŁĺĤ +âĿ¤ ðŁĶ¥ +yam ba +x hnews +whu employee +wel oven +we madeit +vag us +vad u +u doh +ty pa +tor aja +thecw supergirl +subo dh +stage coach +spro bs +song kh +sm ill +sh illa +san lu +sam b +ross lare +ro syth +remember the +power couple +pou liot +nar uh +museum monday +mi hir +memp hi +marig ny +man fredi +lagunitas beer +kale igh +kalan ick +hog shead +harde sty +gr ann +ex whuemployee +educ acion +east port +e ig +dy ers +doc ents +do ones +david dobrik +dark web +crow foot +cau dill +care of +beingh uman +ay ear +ash am +ðŁĶ¥ðŁĶ¥ðŁĶ¥ . +ðŁIJ ª +zir conium +yo yogi +x co +william h +what to +wetn wild +wer n +un stall +truek ofi +travel ban +tram pal +tione ering +time lessness +tar ak +ta plin +t fo +sydney kings +sono gram +sig urd +se ang +sar da +sapul pa +roor kee +ron deau +robot nik +robbies avage +reece shearsmith +re driver +ran sit +predic ated +plas mid +par oo +pan ti +ox alis +opho tonics +nik kis +n dy +mobile money +middle ham +lul ly +lo stock +liam hemsworth +laker snation +la ppin +ki ddin +k wini +jer man +irishtimes biz +inter disciplin +im personal +ibu shi +hon us +hon eo +hemodi alysis +hehe heh +happ ines +gil der +ge z +furi kake +fun ke +ever greens +de sir +cryp ton +confirm ations +cha ve +blun tly +biz news +be gets +be eny +atsu ko +arn aldo +andre y +amontil lado +all ic +all american +aar u +Ķ 리 +ðŁĶ» ðŁĶ» +ðŁĶ ļ +âĿ¤ ðŁijį +âĸ ij +was co +w ü +vincen tian +ve rett +vat ican +v js +uf v +ts xv +the bay +t pac +swe di +sue perkins +statueof liberty +spreadthel ard +sau cer +saddl ery +richard hammond +rev ich +repri sal +redro cks +re planted +pre view +pla ka +pit on +pemb rey +part ita +oun o +op seu +ol ano +noo ff +ni stel +na jaf +multi rotor +mor ya +mail room +ma athai +lu cap +lod don +live free +lip balm +lion up +lahari music +ko kan +kin u +ki mathi +kar amoja +indi gent +howie mandel +gon avy +gn r +ginger bread +gard endale +game works +fo da +fen er +eye wall +equ ated +ef d +dul se +ds ds +dolom ite +do one +cyberne tics +convey ance +ci bola +c fo +blue berry +blue beard +benzodi az +ben simmons +ban ti +autom echan +ator res +ash ba +al tes +ad heres +) > +ðŁ§ Ľ +⾨ ðŁĴĻ +⾨ ðŁĮŁ +z vi +yn b +ye ates +yaz z +wing friday +win son +wil mar +whor ter +wal lows +vin nie +sta renergy +seag rove +santac lara +san try +s vi +ru cci +ri sque +ow h +or v +oo kies +og lesby +o conom +mv cc +mur taz +mugh al +mouse kete +motor city +mi quel +mi fi +meri bel +me vents +mang ere +lac tating +kyw newsradio +kus anagi +ki dambi +kate middleton +jar vi +j school +ish in +il divo +ib pyp +huss aini +hu mo +horror films +high quality +happybirthday niall +han gover +great yarmouth +ghi bran +ghibran official +f sv +f db +espo ir +deliver able +cuth bertson +cow litz +cosplay girl +clean power +cav afy +cauca sians +bus by +bre port +bel zer +be eper +at eca +ar ain +al bay +aki va +ak infenwa +abh or +ðŁĺį , +ãĤ¢ãĤ¤ ãĤº +ÙĤ Ø· +ziggur at +wyl lie +va aste +up country +uke lele +tre ecre +tifo si +taw awa +switch able +sno bby +smar ttv +schol ten +sar desai +sam melan +ru en +romance readers +red season +re blog +raghu bar +plo tt +plear ning +pal try +orange men +n mu +mysti fying +my att +missing you +millen ial +mari est +maku eni +lé on +lu cked +loveis intheair +llanrw st +lar gent +l ances +kinkyboots bway +kine sthetic +ke sar +kathniel asap +kap an +june bug +ju mo +je tte +jack nicklaus +in ori +hou b +heigh tens +ha shed +gu ste +grand canyon +gonnam ake +gam bi +fer re +fa thering +em met +ein audi +e ue +e et +dol ma +den dy +de say +das i +cy prob +cross gates +consumer ist +cha it +calder one +cad man +bur bidge +bridgewater hall +boul ter +bol don +blue bonnets +ble ck +best en +bent wood +andre e +amar te +alexil alas +ai a +ĵ ľ +ðŁĺľ @ +ðŁĺĤ ðŁĺŃðŁĺŃ +å¸ Ĥ +ठı +uni dad +u dvar +tr ung +theblack keys +ten dai +ss lc +ss ense +sm ed +sir k +sim mba +shim mers +rei ster +reha bbed +razz matazz +q net +pomer ol +poly tech +petiti oners +nomean sno +new tech +napo let +my house +kö lsch +kron en +kare m +justin timberlake +jig ar +ink le +impact montreal +il on +ifw twa +gru el +giga watts +ger miston +gend armerie +gaik wad +ga thletics +flip the +fan tom +fal mer +face times +du rer +disp elled +char ming +camphalf blood +bron te +bou cheron +body shop +bak sh +at raffic +ar mors +alto ids +ag ged +aad hi +ðŁĺĤ ðŁĺĨ +ì§Ģ ìĪĺ +âĻ¬ âĻ¬ +à´ µ +wi ac +wall i +wak ker +w lr +twitch retwee +tuscar ora +trebu chet +the foster +the f +steam train +soci ologists +sleep apnea +sk fc +silsil abad +shev lin +sc us +salvad orian +rosan ne +roh rer +risqu é +reality check +rad ura +poie tic +po dr +pl itt +phi volcs +p side +ori o +or re +op ride +ome times +omarab dullah +nov an +notic ias +nis ar +naf sa +mis san +mirror ball +me sha +mar mion +long boarding +ken shi +kan sa +in land +hu ja +hot spurs +holo deck +hard wear +gungah lin +guj jar +fre mantle +en trusting +dehuman izing +cupca kke +cuernav aca +comra des +com odo +budd z +brew studs +big screen +beg ay +beec roft +bas kin +balt cops +b iches +b ary +ap li +amo yo +am m +ak n +agnyaath avaasi +agam em +a arey +yellow jacket +veloc ities +vaish no +unemp loy +tu tt +tru thor +tom ás +throck morton +theo tokos +te of +sy me +snu bbing +six er +sh enoy +senn heis +seas alt +sear ly +school spirit +ripp aul +ri sca +re collect +plant svszombies +pfe ffer +on thisday +o as +mcin tire +mam bazo +lon dons +lok handwala +learn spanish +kissmar c +kathryn bernardo +k lan +j mo +hust lin +ho pie +hil fe +go lead +fl anges +ficht ner +fi ac +en ham +dsl rs +deal with +david cameron +cu tion +cn nopinion +cep at +by women +bul mer +blood stock +bar ometric +atra iler +are te +ar cata +am pu +afun ny +wood winds +wheelof fortune +voter fraud +vier ge +ur uk +u mur +to al +thel ads +sou tien +snag gle +shap ely +sever son +perth now +pa zar +orgin al +net zer +my on +mu uu +mo ha +kriti kakar +konic amin +kir ana +kill init +kaha ani +jor gie +jen in +jay den +indescri bably +imperson ations +holly j +holla day +his sed +hell omag +he te +har do +germin ating +fu uuuu +fkat wigs +endodon tic +de asy +dad eland +cop co +con strain +buthele zi +bur ridge +bour ges +beli ze +bad ged +avi ator +alfon zo +ðŁijĢ # +ì¼ Ģ +z ial +yo tam +year anniversary +yaaaa ay +whis keys +v gk +trans genic +thou ston +tend ring +sun devils +su leyman +stac cato +sp lease +shaand aar +ro sell +riquel me +reg ale +redd ington +reck i +rainy days +pupp ym +pro ces +pop stars +patrick rothfuss +pas ar +p st +our vintagehour +ol ong +odem wing +nic ode +n ces +mu my +mixtape sapp +manufac turing +man citycouncil +m he +live mixtapesapp +lach man +kalin and +k cet +jo tter +itsnice that +ise au +is eng +in dot +imit ation +ill ac +hy ssop +ho ft +hi jau +handels blatt +grand al +glen nie +gin ko +flintri dge +eric colsen +ep t +endor sers +em po +demitas se +cra dles +comm ack +col abor +coal esce +clo che +chronic life +boy chuk +book silove +aver dict +ar stech +al olan +af ron +aerop uerto +ad air +ab staining +ðŁķ °ï¸ı +ãĤ¢ãĤ¤ãĤº ãĥ¯ãĥ³ +âľħ . +âĢ IJ +ysi dro +ye senia +wu erl +world championships +whit burn +wa hhh +vi van +vamp yr +uss strike +urban design +u zu +tro tta +timp ani +than di +th street +te ato +t kr +sli vesmatter +si vas +serv ici +school problems +rugby family +rin hatch +rie beeck +ri mm +pipel ine +pab io +p tu +or show +o very +nb sliveat +me si +matagor da +mas ji +mar gam +lali espos +khair pur +jer il +in consol +hu sted +happy land +ghu ll +ghol lywood +gameofthrones season +funnel ing +frat esmd +frank lloyd +flight line +fer rel +fe bu +en visage +en vie +de gc +ctr lfc +coffee and +cha a +brun ching +brand an +beth fratesmd +ben stokes +ben nies +bear paw +bal moral +bab son +as mir +aray ner +and alus +aggrav ation +ad hikari +abe che +a bey +z ong +will kommen +wa ju +vivi section +vaj ra +uf clondon +tin us +terra ssa +tas ca +tac onic +starwars battlefrontii +slá inte +semin yak +semb ilan +sask tel +saf ina +s max +ro iland +rashid khan +ram akrishnan +pat rollers +pat ras +parnas sus +park ash +northumb ria +nlbm prez +neck i +n clr +multi billion +mid vale +meag re +me har +mace wan +m gn +lap top +la donna +kill arney +ic sa +hq nigerianarmy +hide ously +her ber +hassel baink +fra k +experi encia +empren de +du opoly +die tician +damsel flies +cre asy +col unga +coast to +chum lee +capric ious +bun ds +bow don +bla que +bha vani +bal on +baham as +ary der +apple support +alexandri ava +ae th +[ âĺħ +ðŁĺħ . +ðŁĮ® ðŁĮ® +ðĿĹ® ðĿĹ +âħ ± +ÛĮ ÙĨ +zel ena +y he +withthe stars +whir ls +valken burg +uro web +uk pr +trous seau +transport govuk +tra ppings +toulou se +tool set +to the +the i +sug ababes +so der +silve stro +scott dixon +schom burg +sam pottorff +ruggi ero +promo ciones +poo p +poke ball +palmin teri +over brook +od dy +ny ala +nolen sville +nautan ki +mumbaic ityfc +milledge ville +marc bolan +lev ski +jy pe +iner gic +incongru ous +ilu stra +hu ela +hoo ghly +hel derberg +hall man +h po +glass of +ga ale +flower bed +finner ty +explo siveness +equil ateral +dusktill dawn +digi pak +co ining +chelms ford +case work +cam brai +borne an +bang abandhu +band as +att an +ast u +are em +appe asing +ë ij +Ã¥ r +y andy +x ies +u bb +thel augh +th wonder +te di +tab atha +sixth form +shi phop +santa ana +ru ination +reg gio +rd win +r po +quil ters +petro va +pant suit +oconom owoc +nay i +mh rd +metro card +mali faux +lampp osts +ko en +kixi fylife +kivanc tatlitug +king salman +kelly services +kam ik +k low +k humbu +ji h +ist van +hom g +heritages ite +here com +fil ers +din ka +commerz bank +cantab ile +brac eface +before christmas +b smith +aren yc +are nee +af h +^ )/ +: ,( +ãĤ¦ ãĤ +âľĶ âľĶ +z ira +y ha +what are +well sville +viking pride +vex ing +u criverside +tul ly +tran spen +sue de +sta z +samuel adamsbeer +sam mon +regre tful +plane te +pel angi +par due +p td +p oma +or msby +oli vas +nieu ws +metro pol +ligh ter +lesli eo +le ms +ko val +j bg +ip zig +ing well +ing reen +hrtech conf +ho ffs +hard to +haid agwaii +gon dwana +g tlds +flying lotus +fel sted +fannie gate +fab letics +f ce +do zy +data sheet +dark skin +cyber criminals +ci pr +ca es +bryanc allen +bin go +bhar athi +bab yyyyy +ali o +' ') +ðŁijĭ ðŁı¾ +ðŁİ¯ ðŁİ¯ +web sphere +tun ji +the google +su bie +stronger than +spart ans +sno gging +slan k +sing ton +sever o +sat ta +samanth abar +sam is +rollei flex +reef tank +real matt +ra zar +pit up +pho bos +over stuffed +nil giris +n ura +má r +mun dus +money lynch +mist ura +meri enda +mel fort +mccl urkin +loom ooloo +life force +le vie +jav afx +iti zed +island er +hon ig +ho twire +grac em +di pak +di ddle +con dones +clo aks +cj m +canti eri +box art +bis ous +bi spo +ber key +av ada +arrow smith +ard ingly +ali zee +aaa ahhh +.. âĻ¥ +ðŁĹ ¨ +ðŁĵ ij +íĤ¤ ìĬ¤ +ìķĦ ìĥĿìĿ¼ì¶ķíķĺíķ´ +zsl londonzoo +ym f +yin zer +y ba +way nesburg +warren ellis +w aba +vic odin +tir mid +tick lish +thewanted music +technic ality +tatter sall +strike back +steadi er +star awards +st pm +sp hi +so hl +slu mbers +rux pin +ros ic +repr ori +re capped +pur it +poly meric +our story +no pen +ni kol +nang is +nam jin +na es +museumo fcity +museumofcity ny +mrand mrss +mouth pieces +min young +me aly +marac ay +ly ss +lu tz +lo ew +karl thefog +is lets +is at +in corruptible +hin dle +hi mup +handhel ds +hair spray +gro zny +glo ssing +giam paolo +genius football +geburt stag +fron tale +f ard +eyn sham +ever itt +epi stle +enroll ments +durham cathedral +du di +dru d +di gan +dg ins +cor din +car rou +bri ve +black shear +bell man +band ori +b ami +author uproar +as cer +ar mie +ar bre +afer ro +( ´ +ðŁļ ¥ +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ðŁĴ¥ +ðŁijį ðŁĺī +๠ī +vol kan +v spc +un gan +uk natarchives +thev creations +the bus +ta ec +sy dv +sp news +sp icier +sk z +sher mer +sel am +scott disick +sc app +rain bo +pube scent +prepper talk +prece dents +pm ts +ou mar +ol ona +nationalrail enq +nati vidad +me shes +mar kowitz +majer le +lyn as +li jiang +le brons +lay ton +lar yngitis +kav os +kansas city +jade veon +international nursesday +imit ators +he eeee +ha das +gre ch +freder ico +euroleague women +em phy +em den +e town +dr oning +dopen ess +de ters +dc am +ck ay +cil wx +cali frag +cal an +bram er +board walk +ble ase +beltr ami +bb king +bathroom design +bad ajo +aj ak +ðŁļ ľ +ðŁĺ» âĿ¤ï¸ı +wy g +white hawk +w ingham +vul pes +ucc shooting +tr ys +team k +tan gram +tab comau +t sung +super f +sub titling +stown fc +sig nac +servant leadership +pop cap +plat itudes +par ow +on ero +nup tial +ni hil +ner azzurri +nab u +na dias +makere reu +le sko +kri stan +kapilshar mashow +josh dallas +jackie evancho +in sincere +impati ens +hotel school +high society +hi ace +hannibal buress +ganse voort +gaming news +fri eden +fortnite br +footbal luga +flo s +find horn +fevertree mixers +er ate +epi phan +eng aa +elo tt +east van +ea die +e migrants +dri an +das kal +daguerreo type +cha ouen +brown x +betra yer +bestjo bever +bar cia +ath ene +at ter +arch bishop +ar mini +ap aul +an sonia +å¿ ĥ +âĸ ĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸ +z man +yu vas +wa aah +v max +ume Ã¥ +tru fant +tri une +thisi shar +thegood doctor +tennesse ans +stephen ie +sph x +sol are +smu cker +shal amar +sch eck +sah ni +run cie +pra yut +poly dor +petro glyph +pardue suzanne +ottaw apolice +occas sion +neurofibro matosis +n spa +n gl +mu cci +micro systems +men na +mel ton +matil de +ly ell +loo ter +lefth ander +la sker +ku ts +kri sk +kon ig +kin na +josh duhamel +jan aki +ine er +health is +gro pius +gi bt +gar nishes +g eli +fut taim +film review +feu dal +faw ning +enter com +dolgel lau +deep u +cor bett +cho wd +body image +be are +arch iti +ar gs +angel charlie +am bs +ain sle +advi sement +ðŁĵ· # +ðŁijİ ðŁı¼ +ìł ľ +Ùħتع Ùĩ +zid is +wow selfie +wee dman +wal wal +us afootball +uni kitty +un migration +toy shop +the kingof +ten on +staw ell +som l +sno bs +she kinah +robert plant +queri da +polici a +other ness +news bayarea +new sy +nau r +mosle ms +mi mis +mel son +master stroke +mac sports +leni han +kam ryn +je x +ino z +ice wine +homopho bes +hab ism +guru kul +gu oan +gren ville +gold son +gaz oo +galli gan +fig life +ffe ct +fair way +en bush +eck lace +ea sts +did it +dhar avi +def jam +cow shed +ci me +chum ley +chry st +ca q +bor do +bojack horseman +ben ice +beauti full +base point +bar rs +aw ny +attenu ation +amazing spiderman +ali ant +al ghero +abo ston +ãĤ ³ +zak im +y ali +wx yz +wo tw +winter son +wak at +vz w +up starts +ther ise +the da +teren ure +teb butt +tan vi +ta if +supramo lecular +super speedway +stati sm +spri ghtly +sony max +slur red +sil vered +sel way +seab iscuit +sav ored +sand blasting +sa ken +ro zen +rid dims +re fitted +pi ff +parent sday +p aga +ot x +or jay +not out +mol and +lyn sey +llan elli +kit imat +kak uma +k hol +jun ks +jason manford +is anti +ili stic +il aw +holodom or +hodge podge +histor ique +hack ath +ha zem +go ed +gi ffin +gi dit +gam mar +foxsport saus +fo ord +ev it +european lms +ent rain +emmy rossum +du hh +dodeca hedron +describe your +den ting +cotton mouth +climate reality +chu ma +chi mu +cer in +car amba +bra thay +bor mio +baa ack +ar nis +ap ut +ap enas +acreative dc +a stors +a ism +zoid berg +wild nout +white bait +wal tman +ve ster +ti jd +sing leness +sil char +sh anny +se du +screen cast +rad m +rach it +polit icom +po com +play stati +open world +oc chi +nzo gs +neuro degeneration +nay ar +nan uet +na esp +muir field +mis behavin +mel asma +mc nichols +mash burn +lock smiths +khajura ho +kha chat +kalin white +is ag +inhabit ant +il def +iam x +human right +her ning +hd ds +goo sen +gol borne +glen mor +git ex +g hassan +free music +f anno +el are +drun ner +day ak +cudd alore +cre agh +co libri +cloak and +can th +bur rill +brad dy +boute flika +boul le +bor den +bomber league +bo q +benton ite +bel cher +austy nzogs +af ed +abo ts +ðŁĶ IJ +ãĤ¹ ãĤ¯ +zak opane +x na +x brl +wy dd +whe atus +wam ba +us q +ub nt +type writers +trou w +thel ens +team vic +takeyour dogtoworkday +tak appa +swell endam +suriyafan sclub +sr pt +so ori +sneaker heads +sle man +shor i +shil pash +shape shifters +se ah +purpose tour +port cullis +p inf +over achievers +ori ano +on cle +nh f +ne stea +myrt les +mudi ay +mc ki +maso chi +lingen felter +kil lem +kier ong +khali q +key arena +kas par +ka bel +k rule +john d +impres ario +hor r +hijab i +glu es +glo c +gil lam +gi de +geo chemical +garden route +flo track +enjol ras +en able +dynat race +dro more +dia vel +de crypt +cochran e +co simo +chi desai +chelsea handler +bra shear +bo ke +bo burnham +bla der +bark box +ari ens +ah rc +ðŁĻıðŁı½ ðŁĻıðŁı½ðŁĻıðŁı½ +ðŁijı ðŁĺį +ðŁİ ¢ +ìĹ ĺ +ìĦ Ń +âĿ¤ï¸ı ðŁĻıðŁı» +v expert +underground wgn +tu ckers +tr n +thereal j +the poet +ter uel +ta fe +sport suk +sk ov +shu tup +science of +sc ra +rr b +pr gm +pic cy +pen rod +over nighter +o dio +niq qa +ng f +n ssa +mz katiecassidy +mt go +mo vi +melchi or +max xi +masa yoshi +marsh land +llandrin dod +ld schurch +lam ide +l cd +he ph +harmon ization +ha sek +gt bank +gren nan +glocken spiel +gemm ill +freelo ader +financi aleducation +f hir +explore wellcome +esc an +dou ce +din do +di visible +dal t +cu ne +cry tek +coy bib +champion sle +cel os +bi wa +beer us +bal das +at cq +ark ells +ai kens +ai b +accru ed +* ' +ðŁĴĶ ðŁĺ¢ +بصرÙĬ Ùĩ +yu mmo +yo gas +yar mul +wk n +wer tz +wer bung +val anci +upro xx +uk raina +tur bidity +tin n +subtle ties +stop tpp +slim mest +slau son +si swim +sh uru +rosann apan +rosannapan sino +ram une +raf bf +pog gio +pender gast +over powers +nur u +novorossi ya +middle brook +men slax +may belle +ma po +lust ful +lim pet +l lego +kat akana +kane ko +jonah marais +ho sk +go c +gal ar +fu ck +fol le +fistic uffs +femin ino +er hs +du sh +door knobs +dishap atani +cu mia +courtau ld +conclu sively +col ina +cav o +c agua +bi ondi +anan tapur +am ena +ak ito +ak ay +aj styles +âĿ¤ ðŁİī +عÙħر اÙĨ +youth sporttrust +w aded +villet n +tweedle dee +tu bi +top scorer +thei sen +the joshuatree +t mac +sustainable living +sp la +sni ff +sim son +sik ri +sav oia +robin meade +retrac ed +po kot +pleas ingly +photo credit +ph bo +pec tor +o hashi +newport county +mon eta +mcal eer +loo ve +lic he +lemon defr +lau ge +kre ps +keepit local +jon axx +jb gill +james bay +j anda +isab ell +hunder tw +ha ii +ha chiko +gran ja +go bear +geraint thomas +fil mp +far man +fa hri +ev am +don nel +do rey +dham ma +collar bones +ch p +bun co +bronze age +bi bl +bell wood +be elite +baseball cards +aven sis +autom ator +amin ah +a é +ॠĮ +x olo +wool acombe +we wont +von erich +vi eri +var ick +tho e +sou therland +some onet +skype classroom +shor n +sc p +ry dell +ros ner +roar loud +ro wh +ro mas +pueblo s +pro tom +percol ate +perce iving +open house +mers tham +matt gaetz +mand ar +makers gonnamake +lyn brook +kri el +jo tun +jamiemc murray +j kt +i pu +hav ai +harpercollin sin +gun awan +go canes +gar side +free spirit +fer ments +fe s +en bau +emul ates +elabor ately +do of +da king +da ithi +coz mo +cla ssen +caul field +bru cer +breakfast news +bor ge +blouin artinfo +bit ting +birn baum +ber ms +ban quette +bal con +back stretch +arre ola +andre ahor +am by +ad ham +!! âĿ¤ +! ðŁĺĤðŁĺĤðŁĺĤ +ðŁ¥ ¥ +å ĸ +zachary quinto +x q +wro xham +world vegan +weal den +w sp +un ripe +un gar +tom me +timeto change +the west +the vamps +that works +tart let +sur ratt +sum ire +ste vier +ste arman +stap ley +stal ag +scor chio +ry ders +premi ère +pff ft +p timal +oven timiglia +off rance +nu uk +nex tera +mu lia +missuni verse +mil oventimiglia +magnu ms +lu ta +leslieo dom +leslieodom jr +ker nersville +karan wahi +in progress +ij ssel +ic mp +huffpo stuk +hat ted +han eke +gandu je +fuku i +franz iska +fertili zing +far zana +dundee uni +dear man +de core +day for +constell ation +collo dion +c zy +bright line +body count +ber jaya +bat an +bad am +b dc +ardo yne +ap athi +ami des +amer ks +am ery +acos grove +aco at +ðŁĺļ ðŁĴķ +ðŁĹ ³ +ðŁİ ¿ +watch me +tt m +the ultimate +terri bles +sw t +super team +st ilo +so hi +sk gaming +sar no +sac cone +ri ffle +research highlight +r me +quad rat +poe tically +ple ad +pitch wars +photo blog +parti da +o zz +n ilo +min fo +micro wave +mak hani +lily dale +let as +kal ita +inter jec +in ic +ill ana +hu et +ho arse +green head +gesch ich +gaz an +fly ingh +evacu ates +enum claw +engal uru +ence ph +en ta +drawin goftheday +diso bey +dion ne +diocle tian +der yck +del ka +colon izing +colon ist +biglottery fund +be magenta +bakh sh +badajo z +ay en +avi ones +andrew j +al rdy +al esi +ab k +. ðŁĺ¢ +* ? +ÛĮ ا +и ÑĤ +win ches +vla de +use c +ugand ac +trans world +tommy chong +tele phone +swol ves +stan sbury +simp ang +shar pay +shar king +secure the +ryo kan +rosh hashanah +ro li +ram u +ra jah +pursu ant +phu ong +per ito +nulli fication +no limit +na aman +n nt +monday night +maz al +latch ford +kol lywood +klu g +jo j +jennifer morrison +iz aya +itv calendar +ihsa state +hyacin ths +hem lock +hel y +has ler +har deep +gol drush +fri d +fam er +fab ulo +ent ic +emar osa +citizen liveat +carre rac +bronze bomber +bran ko +book week +bi ggers +b ander +ay to +astoni shment +apur pose +aber gele +* ~* +ðŁĺĶ âĿ¤ï¸ı +z aira +young k +yl c +wid de +wet more +w tennis +vu i +un bounded +to ppo +there abouts +thename is +the eagle +tangany ika +susang komen +stra x +sp enders +smu lders +sefton hour +sa gh +proje to +prodi gies +prin ze +pr al +physi ques +per rett +pas and +pan k +op rano +of dreams +new z +my choice +mur ch +ma hela +lori moreno +legal ised +le sle +lagni appe +koval chuk +ko tc +keb bel +katharine mcphee +jer ash +jar ls +jail breaking +j edin +j cole +impro ver +immigr ating +hub bard +hi es +ham am +gu shed +great taste +gl ack +gender fluid +g chs +fun ches +freak ishly +fool s +folk ways +farru ko +en sen +em elie +elite book +el bowed +edge mont +du ss +cuck old +chig non +chi asson +cheek bone +cas sar +bor am +big narstie +ben in +bas com +ash dod +and dd +alli an +aga the +a thri +ÙĦبÙĨ اÙĨ +wal she +var us +vap enation +v ink +un pad +un inhabitable +th chat +swan lake +suren dra +studentath letes +sav ino +sar te +re assemble +q we +pigg ate +pel ts +nw t +na del +n re +min tage +min shew +michael is +mat anzas +masto don +lock jaw +lock a +little steven +lil t +la gu +kr m +kla gen +kierong illen +it sag +i read +hl g +ham bone +go tribe +giar dini +g achi +fran ds +fire fighter +film challenge +exer ting +eg bert +dg ate +cy co +cloud iness +clon don +claire mont +cdn screen +bu se +brue ghel +boss day +blan ket +bird sofinstagram +bam yan +back stage +ba ah +ayesha shroff +ay oung +arth as +ak hi +ad ab +abio tic +... ðŁĺİ +Ĥâĸ Ĥ +ðŁĴŀðŁĴŀ ðŁĴŀðŁĴŀ +ðŁĮ ªï¸ı +аР½ +ziggy marley +women shockey +wc b +waist coats +vasu dev +tre bko +ti ds +sunday morning +sui vez +side man +se sac +sch lichter +satt ler +sar uman +s ã +reha sh +public power +pu shover +pre wedding +po tro +pier pont +parag on +oval tine +newh ouse +new mar +ne wr +nax al +miz ner +mal lette +lough ran +long ish +kol in +jun gian +jah res +jac q +ig an +hant si +h vr +geof froy +gelatin ous +ess am +eno shima +engel brecht +el rond +ed ling +ec m +dam it +contradic ted +cocacol aco +chero keen +chaf er +buk we +broker ages +bleed purple +asi mo +anun naki +aly ce +advance dwarfare +! ðŁĺĨ +ðŁijįðŁijį ðŁijįðŁijį +ê¹Ģ íĺĦ +ç ¨ +व स +we an +upup cronulla +uof u +unite and +tol land +th wait +tg u +tatar stan +sweet bread +sou thee +son as +sehun day +se stri +sax es +sarg syan +rickast ley +re testing +re load +pu ce +prideof britain +place bo +phil ando +os ita +open houselondon +op ine +onthis datein +now ww +ne mo +na enae +n wark +mule soft +mer horn +marquette u +lo dh +leven e +khar kov +kent on +inten tionality +inc ited +il yushin +if cc +idiosyn cra +ger vin +gear vr +fit zy +fight cancer +fam agusta +even in +er usa +en ses +econ o +dem ps +creed moor +chap ala +bro r +bol anet +banar asi +aw ski +au v +atleti english +aj ni +ah oo +ad yar +ðŁĵ¹ : +y ces +v nc +ur ru +ty ce +thr an +swach hat +sve ta +stack overflow +sol an +sof lo +silver chair +sco ggins +sco ach +sal ps +run out +re sti +re paying +poe tess +pe tula +patin kin +pa sia +of b +mu ley +mast itis +mang khut +mand arin +man ch +mach ynlleth +lo bb +life coaching +lev eller +lemon ade +kur an +juan mata +jan us +j atin +inc ised +illi dan +il and +hold fast +his ses +har ith +gregar ious +ger stein +flyo vers +fire weed +fi et +fav ourably +exu ding +denis of +dan on +cu bit +con naughton +co hoes +cas sill +cafer acer +bat u +badger monday +auto week +ash wood +a ase +! ðŁıĢ +ᶠľ +za res +yom hashoah +wear side +valanci unas +ul man +ue facom +toshi o +to pher +tinker cad +the gill +ta ino +sunny days +sub cutaneous +sell er +scant ily +sc la +rot manschool +reynol dsburg +quest nutrition +pot stocks +photogra py +over water +onelove manchester +oler ance +neuro developmental +movi miento +mel ita +mccull in +mariamen ounos +manchester arena +man zil +ly th +link öping +laurel park +la dwp +ku kri +kaz ama +kac ie +is kra +ip sy +invul ner +hyper drive +holgor sen +hol lens +hof ner +heure ux +gree kislands +girl group +fy m +for du +fon go +ff b +femini zation +faroo q +equestri agirls +e hu +drive shaft +de by +dd yer +d kb +d company +d anda +cov ado +ci enci +chop stick +cat v +ca jas +blan kly +binaryo ptions +bake along +axstv fights +an antara +ama ury +am docs +ag om +adobe xd +ðŁĴ¯ âĿ¤ï¸ı +ðŁ¤¦ âĢįâĻĢï¸ı +âľ ¡ +yn ash +y ari +william hill +wb d +walk over +ve vey +u meda +touri ster +toulou sain +tol hurst +t gp +summer readingchallenge +su kab +stra vaganza +sk aven +simul acra +show band +scra pper +sand f +san teria +ran agar +ra ghi +quick en +pen wortham +pen ick +pe tya +out crops +nebra sketball +near shore +moor hen +mo ver +mil las +may uri +mat tos +manufac tory +lic ata +kay lam +k oos +joye use +in scru +ilove heartland +hubb le +gum road +gul lies +ghostinthe shell +g ape +fon dling +fla gon +feedyoura ddiction +eve rett +dot to +der on +das uki +cover all +cor regi +coleco vision +co wra +chocol a +cf pa +car goes +blu dgeon +bla z +belu gas +alvar omaldini +ack ers +ac entral +aby ab +a hearn +vas sal +uc cio +tur ki +tu scan +trump kim +ti thes +ti fs +tech women +taxic ab +supano va +scar brough +s jam +ro mario +progressi vism +pm live +play matters +pc bc +pac ar +pa vey +p go +olan rogers +ob il +national sandwichday +missing kids +mar onite +man preet +machiav elli +li ang +le sen +kul gam +kre feld +k wood +jan zen +jack rabbits +it bp +in corri +ican ada +hypnoti sed +h js +go war +gnar led +ganano que +fore warned +fol ding +feliz viernes +fc twente +fat f +f opp +exhor tation +eff zeh +do ddle +dhar m +des boro +da ina +d de +confe x +car rell +canap é +book shop +bl anda +bbc snooker +bb pilipinas +ball inas +back spin +au strade +am lo +am am +allevi ates +alam ed +al mos +age ha +ðŁĺľ . +ì ĩ +âĻ£ ï¸ı +வ à®° +z oli +yu mmmmm +y cle +worldbook dayuk +wigan council +visu alizes +van oss +ule ma +tro icki +trans ference +traffline mum +thugg ery +tan er +t shabalala +sym bian +susang ilbert +sma de +sh ach +sc aping +save a +sal bum +royal ty +rang an +plane tofthe +patri dge +past illes +palis ade +ori ole +oom fs +nick lachey +ni go +n to +n ira +moisturi zed +miz uno +mira bel +microsoft store +mal adies +magi x +low a +logan paul +liber te +l kg +jeff probst +is ong +intrac table +has brouck +fu schia +far o +fa it +eu karyotic +erne stine +elic its +croo ke +cra c +company culture +char issa +balasu bram +at risk +aborig ines +ðŁĺĽ ðŁĺĽ +ðŁĺĢ ðŁĺĢðŁĺĢðŁĺĢ +ðŁİĢ ðŁİĢ +âļ«ï¸ı ðŁĶµ +zz i +yab ba +wr r +wit ts +virul ence +vi rar +v gs +v ache +ts wag +tromb one +tiang ong +thestroke assoc +tham rin +take on +stones our +sa ki +root sports +rand al +r tn +pr ongs +picar die +paulo aken +pash ley +par ken +ocot illo +ny f +my top +mut ate +mon oun +means business +maun dy +mastersofthe universe +lp sc +lim bic +laurajane grace +kul bhushan +kam aal +io vine +indo chine +in fielders +in dent +i ball +hok kien +head on +hallmar kies +h so +h ile +green washing +genu in +ful fils +fire red +f xx +e tre +doones bury +dg k +de car +dahl gren +cle aves +carol an +bu stan +bri ms +bra zo +blue point +bad ging +avan shi +ar vi +ani bal +andreahor wath +amon th +áµ Ĵ +wwe fastlane +wis a +willing blam +wil lock +vu ong +vox el +vee phbo +ur bain +un tam +the ma +su chen +sin ge +seth green +se co +rumb elle +robu sto +ring tail +ri yaz +re program +re ep +re assessment +pn f +over charged +ol inda +o sea +noord wijk +n gb +msc actions +mey erson +mand ap +ku ster +innov ated +hy la +heat waves +hanson music +gri es +gol u +fro mel +fer menter +fau ght +fal ooda +f arias +er for +dra sh +disc ol +desi g +co aldale +cep tional +cen tex +cas sette +car naby +bun ter +be ton +all sop +al fano +afro pop +?? @ +winstar farm +wick ford +wh h +var ic +uwe bristol +un consciousness +trans boundary +toyo da +tar ap +sty ria +spru cing +showme your +sar az +sang ita +san ja +sam achar +sali do +ru chi +rs j +rhu mba +res q +quar rels +qual y +q on +q mul +pocket knife +petro vsk +pe ur +pauloaken fold +palmet to +ni w +n flying +mor anis +lun ging +loz enge +lauter brunnen +kur ung +kerma dec +j end +inspi ral +high worth +gul let +gi ev +gh k +en ki +doppelgang ers +does burg +dil jit +dardan elles +d anta +cur belo +comd tac +bo jo +basseth ound +bac ter +as col +alapp uzha +ðŁĴķ ðŁĺĤ +ðŁij¨âĢį ðŁĴ» +with modi +wel born +vol terra +vi reo +un restored +u kem +twin n +tweetur biz +tr ounced +torre molinos +toronto pearson +tanger ines +sy dow +super chunk +stal inist +slat on +skin heads +shankar acharya +sar panch +sa be +s style +ry ar +ry ann +roo ki +r ll +q br +pure magic +pan go +p ams +over landing +or ka +opo ku +od den +migrant crisis +meg ann +me the +mck ernan +mac kem +log ism +lat rice +la hood +kings lynn +khu shi +ke mps +kac ang +k alli +ir lande +hor witz +harri smith +greek week +great place +gi psy +fu zhou +frank ish +field fare +fan shawe +en yt +don ati +di ously +cine t +chico state +car us +car nell +campan ula +breast milk +blood cancer +bhi du +beer pong +ayck bourn +arkell smusic +am boise +al fi +ae on +adu blin +accentu ated +ab ama +aaron hernandez +ðŁĴķ ðŁijĮ +ðŁij¯ ðŁĴķ +Ê Ĵ +yal c +woman ly +wit e +wel sham +vital ity +visit philly +vegas con +us an +tune z +trump now +tool ate +to bie +thru shes +the henryford +te esta +tanehisico ates +taik await +taikawait iti +steam boat +star less +spic iness +sk oll +sin siders +sem powerment +schi ppers +sam yuk +rump el +rhin eland +ren aldo +relap sed +raman singh +psycho geography +propag ated +prince ville +por osity +photom eter +pa cha +oldham hour +o eln +mumbai indians +monday funday +mikha el +micro dermabrasion +megastar chiranjeevi +mat ara +lo sey +lith onia +li em +kon oha +kim bra +kid min +kalinand myles +jer kins +jc vd +jayant sinha +ja ish +hun tel +house bound +her i +green thumb +gor an +gillian anderson +gang nam +fortu neteller +fie bre +f wi +em mas +dri vable +dress shoes +dou ches +diabo lik +cool katz +comrades race +class work +cdn film +bit bucket +be chamel +bb tv +baltimore police +ash g +arin dam +ar ul +ap sley +al sager +ais linn +acan al +? ðŁĺĬ +; # +ðŁ¤© ðŁ¤© +âĿ¤ ðŁIJ¶ +z edge +y aks +winter soldier +who you +wha thappen +vill ans +usu i +twit ta +twin kle +too m +ther midor +tex p +ste o +sj b +sher ine +sas campaigns +san er +ro mar +red minote +plat on +pet supplies +pay g +ou de +or omo +motor co +mind tree +mi ec +lon do +leon hardt +l kr +kirkintil loch +kh ouri +kbc channel +kar ima +ka ina +k dwb +justin rose +juli ani +jeff merkley +itu esday +ip ers +ie g +hyper v +hom ep +hit theroad +hi el +ham burg +gre p +flit wick +e ula +den nings +cow fish +cos ine +colton lhaynes +clen ching +ch ö +c tic +bre aze +brad leys +book smart +blood wise +ble del +bel sen +bc wildfire +aw ad +arstech nica +arashi yama +am official +am ca +a os +ðŁij¨ ðŁı¾âĢį +å°ij å¹´ +winkle man +wig go +vou ge +us kies +ul p +ton ym +tg cf +team rwb +ta iler +syru py +sun seeker +sports line +spiritu alized +ski ffle +si ds +sham im +se bald +sar ris +ru fio +romb lon +righte ously +rebel wilson +railway museum +r wang +ore x +on ti +notre dame +ne z +na shoba +moo y +mis sr +micro prompt +manhattan ville +malari aday +mac adam +luang wa +lot tery +iwant to +incapac itated +im ber +ilove mcr +iam saidharamtej +hin oday +her u +gg in +gar ver +fumi gation +foxsports west +em powered +dr ms +domin ick +den es +de safi +corin thi +conversation uk +calori fic +barley corn +ar mag +any time +allo saurus +alder grove +accoutre ments +abdul la +Ĩ ãĤ£ +ðŁĴĸðŁĴĸ ðŁĴĸðŁĴĸðŁĴĸ +ðŁĴ¯ @ +âŀ¡ï¸ı # +you dont +ye sha +y apa +wing nuts +viv int +v ted +un assail +thursday morning +the athiyashetty +sous vide +sin cil +sch ramm +sam witwer +sahi ba +sacrilegi ous +rin na +reti ef +reeper bahn +red currant +real ddp +por ate +popu lists +passi flora +oil cloth +ohio stathletics +ny ang +noor ani +nikkie tutorials +new ells +nat ak +mss oci +mi rip +metal fest +meigh an +meet inghouse +mar row +magne tized +lucy slaw +loo sens +lin tel +le sar +jon snow +jol son +jenni rivera +hand forth +game book +g bb +ex on +erock star +ent soc +elek tro +ek azarian +e ikon +dra zen +de at +dat to +d hin +cu pids +craft sy +chel a +breaking views +avon mouth +ati x +animal testing +aki shore +ad din +. } ++ ' +! ðŁijĩ +ðŁĴ ¼ +ðŁįį ðŁįį +ðŁįĥ ðŁįĥ +ðŁ¦ į +° âĢ¢ +z ele +ys p +wh ata +we sanderson +wan stead +wai the +w tr +universityof ga +un ting +u ren +ton gued +thereal buzz +tfl tph +ste ger +stanley cupfinal +sof rench +sl s +sick ert +she sha +sas an +sam plers +safe keeping +reichen bach +pal z +outh africa +oneless gun +ober yn +nation of +micror na +mat on +man ig +ma hoo +leach man +kie hl +keween aw +is db +inter locu +i spy +hor ten +hay les +gujar ate +go old +glass boro +ger not +ga tha +fi de +f wf +exal tation +erri gal +ei ko +der ain +dep ablo +d hat +cuid ado +cb ce +bur naby +bir stall +be vac +aun ch +aujour d +au sage +at tics +at kina +ar nataka +amaz ulu +al melo +al bic +wat cher +v ha +un problematic +trevor row +to kaido +sw akop +sportsc ast +so dus +slow ness +simon stown +sever a +sab io +ru hi +roun drock +ri do +rally mexico +qaw wali +ple ttenberg +pan esar +os goode +op chemtrails +nik ole +nicol aus +mu stering +monte reya +liz beth +lad son +kir una +kir ko +kenya airways +kaw ase +ka hl +k achi +jet set +j olo +izu eta +hu atulco +he yyyyy +has bro +girls nightout +ga stel +fu oris +fromthe vault +devou rer +del tat +de hydrate +day lighting +dann yo +competition time +chim ay +cantile vered +british f +boilerroom tv +bo ers +bestof british +bal sall +b mtc +az one +aw ami +avin ash +as sin +adap to +accompan iments +aa os +ðŁ¤ µ +æī ĭ +åĨĻ羣æĴ® ãģ£ +ãģ¦ ãģĦ +âŀĸ âŀĸ +Ñ į +yorkshire is +yo gab +x dddd +water slides +wad den +umb b +ther ion +syrian army +swin k +stra yer +stop yulin +slam miversary +skid row +skan ska +shopping online +shivu adda +sbli i +sanit arium +ru ess +rr g +river run +ril las +quadri plegic +pin nick +peace time +olive tti +nak ayama +m gn +li vand +kla assen +kati ele +jung lee +jumb ura +jay sean +ja en +i shin +ha ina +ha akon +gri f +fru gi +fr g +for son +for agers +esco bedo +en derby +dou bler +do bara +cry an +cor covado +cdn olympicteam +bibli a +bhar adwaj +bell tower +ay na +auti sta +, * +ðŁ¤Ļ ðŁı½ +ãĤ į +zz er +yam hill +ver sion +vande weghe +ul c +to rero +su its +street team +sho ki +severe wx +rome os +ro opa +reclai med +ph are +op ic +obam af +montereya q +megat on +mc wfc +mari adb +lu fc +labor ing +ko za +ko iv +kang nam +john paul +irfan pathan +intangi bles +imou to +i stand +home place +ho wards +halle berry +gregori us +get chu +fx cm +flo gger +fer rers +fair hurst +esk er +efra ser +diamond jubilee +de ora +cover tly +co perez +christian sburg +chri sette +ch é +carri eh +caram anga +cam illus +bur gon +bread crumb +bre izh +bbc goodfood +ask for +as wad +ap jabdulkalam +antag onistic +am jad +al mam +ak ande +adink ra +ac triz +ðŁĶµ âļ«ï¸ı +ðŁĴĻ ðŁĸ¤ +ðŁĴªðŁı¾ ðŁĴªðŁı¾ +ç ĥ +Ë Ī +é xico +ze bre +wante duk +tw oods +trivi aday +tox teth +tid dies +thu la +theofficial sbi +then i +the free +templ o +tas ers +tan f +south jersey +sou suke +sla ine +sea bees +saturday morning +ru gg +reister stown +q aim +pu jol +plant ation +pil key +physio therapists +philli pa +pashtun longmarch +par ly +ovi Äĩ +our tney +op tus +n aging +my day +multi sensory +mplo yee +mon dal +mcke chnie +lax ative +lady podsquad +kyo ku +kidney cancer +kick ing +ke iran +jeep er +je wl +jay la +iot security +influ ence +indiana fever +ij muiden +hypno therapist +hali za +graff ito +fu gu +fiji ans +exter n +ed gier +e igen +dumb ed +dick er +dae won +co housing +chab uri +bo gg +blackand gold +bal azs +ay re +av itch +au bert +angel arayner +ag nez +a ok +ç© º +ãģķ ãģı +âĨ ij +à¹ĢภĻ +zan ella +wl ky +well and +weal thier +under coat +u tin +trad able +ta pah +stra hd +sl veng +si ria +shave club +sce les +sal mo +robert glasper +rcar mitage +rak ha +ra van +pro drive +pla sma +phi sh +p eller +outside magazine +or cutt +on ard +omen i +odhi ambo +oak ham +o ai +nikola os +n music +motor coach +mccas key +macin nes +little finger +lat asha +kot ka +jo ep +jar ah +j du +iw p +ite sh +is mat +idar thritis +holli day +hal verson +ha vard +guil derland +ge ils +g vb +g sathletics +fung icides +fu mero +for pa +elling son +dor mancy +don of +dis banding +dige ster +day parade +char lam +capit alizes +cane gra +bu blé +br ingh +bi sexuals +bein ecke +bal an +bab angi +av h +august ines +ascend ancy +anishin aabe +amar ula +al able +absur dist +; ____ +ĥ ä¹ +ðŁĩ¦ðŁĩ ± +å¥ ³ +à« į +zz z +yog ya +widde combe +war i +vol ve +ul rike +tro twood +the greatescape +tha ad +tc pa +stay classy +sr il +sp hila +san abria +sab at +ry m +roberto cavalli +road shows +rep eller +railroad ing +pu ds +perme ate +penn statem +pedra za +pas sing +p nb +or nis +ny gv +nie w +mt lv +mk don +med aglia +mc beth +mc allen +lo tr +lincoln ton +lill is +laser cut +language day +ki ght +k attan +joseph son +james mcavoy +inter species +instal ments +i just +hof meyr +hite sh +het tie +he don +gorsein on +geta way +fr üh +fle dermaus +fin icky +fero pol +faber gé +f bg +excit ingly +etu des +enlar ging +el win +dun ster +de stre +de camp +dave matthew +crest line +chat win +car cross +cam bu +bree z +bo sun +b ja +aw acs +av chd +army day +ar uh +anne ke +zen do +xen arevival +wi thern +wft da +view tiful +underthe dome +tram ping +time sheets +talis ay +sycam ore +supportn wt +super villains +star gell +soul fly +so j +slow food +sig machi +sand co +salon du +sal lies +sak shi +roy ster +ri skier +re format +pau ll +pascu al +ore imo +n mm +mo ssel +mo ate +meteor garden +magne tically +mach ismo +llan gef +jer wood +jef fro +ignaz io +hyper plasia +ho ko +har n +hai den +gu ten +ge gen +gau k +forth right +foreclo sures +fin alizes +exempli fying +ep onine +elle tti +eleu thera +du ch +disaster recovery +des don +delici ou +debre cen +cool angatta +colle ton +cla sped +cit ilink +chil eno +che halis +calder cup +byd go +bus se +bonny ville +bodn ar +bifur cation +bestfandom ca +ben ko +ba qi +ay im +agamem non +.... * +... ~ +! ??? +æŃ Į +âĿ¤ï¸ı ðŁıĪ +vi ers +uz alo +uk houseoflords +tillot son +theak ston +tagbil aran +stabili zes +so de +sne deker +ski les +shan er +sen ergy +sel fy +sch ar +sal army +robusto babe +rc ade +pic ador +pear cy +pan ay +opend ays +oli vi +ntv weekendedition +ne gras +ne agle +mu cca +moneti zed +lu pino +lick in +kathy ireland +ja afar +incen sed +hail wood +great cause +goldengate bridge +gold farb +goal setting +ghost recon +ga irport +flori dal +fli ppen +fi she +far ra +en di +di staff +dah y +cri bb +cre edon +con sin +col men +co sponsored +cin donesia +brow nie +born tobe +bo gard +biffy clyro +bella vista +ba wn +aw s +alexand ru +ac opter +ac ces +aberystwy th +. ðŁĺħ +ðŁĺĤ ðŁİī +ðŁijĮ ðŁı¿ +ye eeee +yaman aka +yakin iku +weak ling +wan ji +tuss is +timeto play +sull inger +str un +sp urge +soun dary +sor te +si deb +sau ber +red day +re dy +ra che +prote c +privateer press +per lin +per ic +p shs +ode h +nbab day +mul grave +mp c +modul ated +mis steen +michi o +mevag issey +met u +mantic ore +lus combe +li vio +l bo +king smen +jj c +ichi ba +hod ler +hit less +gos set +g pk +fck oeln +fangir led +f ons +eich ler +eaz y +east vale +der ful +dau er +compos itional +cat kins +calli graphic +boy ard +bon aventura +biop ics +be such +bbcle icester +bbcal ba +av ina +alu zon +al ind +ak ry +a stringent +!! * +ðŁijĬðŁı» ðŁijĬðŁı» +ðŁİģ ðŁİĤ +ðŁİ ¡ +à² Ĺ +Ù ł +ye omans +wx ii +wo te +wi tho +wh are +vod acom +verif one +v dv +tsun amis +trav ell +trade off +tool room +stori esof +sp icc +son yes +shoed azzle +shi hab +schomburg center +sai ful +ron ni +roarloud travel +ring let +red in +rap ace +ram es +quar re +plac emat +pi gott +north jersey +ne emo +mor tons +mis direction +mick le +mi j +lead theway +le os +le mo +jitendra singh +j mp +ici ón +iam rana +i won +heel ers +heart lands +ha thi +gr ps +go griz +giuse ppe +giam battista +gener gy +ge do +g pe +eth icist +dra upadi +deleg ating +de growth +d bag +cze chs +comp toir +charle sm +bur chfield +bne i +biza sia +be ready +bds dragonflies +asli yoyo +ari ver +ar ba +appalachian trail +all hail +alge ciras +week lies +water boy +va ez +til man +thomp kins +thene therlands +su en +stalac tites +specul ates +so di +snu ffed +she reen +scotthall nwo +ri sto +ren ly +por ro +polic eug +plasen cia +odd fellows +mount joy +mo sier +manil aluzon +magen nis +ma ak +leg as +la za +katy isd +kam and +kam ali +jo key +jim ene +its no +inst illation +ideo logically +i aw +i ao +hy ams +hu berman +home wrecker +gold field +g sofa +fu or +fou z +film fareawards +fer ber +enni um +e marketer +disgu stingly +desig ned +democrati ze +cro agh +chett inad +chain ring +ce ara +candice kp +brain cancer +boom bap +bon ino +bo zak +bel more +awesome st +ad cc +aas tha +: "" +âĹ Ĩ +zepp ole +yogi babu +wide band +whoo hoo +warm ington +voc mnews +ultra sounds +twi zy +tran che +tic h +then igh +the family +t gom +sy rups +ster ns +sinu ous +shingekinok yojin +scher merhorn +ronal dre +rock s +range ela +ram il +politicom ag +pitch atpalace +ot lfp +os rs +ol dd +ok tar +ny strom +nat or +nasti ali +mis spelt +mea ford +man asi +makers mark +mahar ajas +la ddu +kir ri +ken nelly +jj author +ishqba az +inherit ors +ic fp +huntel aar +hindu rajyam +gre te +giff nock +g nu +g audio +fresno state +flori ana +fan fan +du ro +donagha dee +di bru +deb namcarey +dal at +cros scu +contu sion +commissi ons +clu cking +cimo relli +ch awal +cat sare +cas set +burun dian +burn age +brick laying +brad thor +be holden +back to +awild life +anarch ic +al ag +ab ank +a ica +ðŁĻıðŁı¾ ðŁĻıðŁı¾ðŁĻıðŁı¾ +ðŁĺ²ðŁĺ² ðŁĺ² +ðŁIJ¯ ðŁIJ¯ +ì¤ ij +âĢ ² +á rio +y stery +william devry +werder bremen +vk xip +tyran n +tren ching +tip sters +syn nara +sw right +suppre ssive +star liner +solu bility +site c +shaw nat +sardan arohit +sar kis +rene eyoung +r ÃŃo +pu jas +psycho tropic +pss sst +providen cia +pl ss +petr illo +per cen +pc cs +park town +pad ano +pachy derm +onceupon awine +natu rist +nak ama +naf s +my ki +marma duke +mait land +lu ba +letsgo peay +lefthander sday +laz lo +lave zzi +ko taro +kit z +k nt +jäger meister +joss whedon +imperson ates +haj jar +gor ving +gen au +fu to +five star +emerson college +ea org +diste mper +dau ph +cro cks +cri spy +ch ome +ce du +car vey +bo vet +bluemo on +big issue +bab oo +b hang +arche ology +ar ayana +apprais ers +ac op +ðŁĵ ® +ðŁ¤§ ðŁ¤§ +âŀ ¥ +áħ ł +wy oming +water view +war ps +vivo v +video editing +ven ceremos +us yk +urgent podr +u sia +tre stman +tb harat +sun ds +stra der +soh na +smo vie +situ ation +sim feropol +shan er +sh ying +seeyou in +se gar +se cker +roo yen +ron chi +road trippin +ren ounced +ren ji +quie ren +queensc liff +propagandi sts +pres sclub +pp opro +pitt ston +pav a +nemac olin +natu relle +mil ou +mil ani +ment alism +med star +me sni +mat tress +man ahan +lu pul +lingon berry +lewi showes +lar ga +la el +la bia +l rn +l hb +ke ce +kar is +ka ad +holac racy +hol mberg +gur t +go pe +gaz illion +gael tacht +fu tari +fo ca +flatbush zombies +fak ta +emo ji +ed by +dy dd +danadel any +cw ts +clothe spin +chop da +cbs allaccess +ca ins +c fx +bron wen +bm wx +blood letting +bilet nikoff +bike month +back tracking +artag nan +af as +yil dirim +y pf +wilke sboro +ve f +v awg +uk la +tri phop +ther itage +thar an +tem u +steno grapher +ste mple +special forces +son go +so gon +slo v +satthe table +ru ddin +rodri gue +rig sby +quint en +pro av +prize winner +pre o +pe ppe +paren thesis +onna is +one gro +on sie +omot ola +o gm +new berry +ne vil +nak ashima +n ja +mu tour +mid mowx +mic on +mic kie +mer se +menom onie +ko bus +kei sel +kaley cuoco +jointhe movement +jam fest +illi beral +hut cheson +hi ston +hazel tine +ha o +gu eu +grun wald +grig sby +gre sik +gel atine +gaale inster +every things +don ley +deten tions +davematthew sbnd +ct cs +craft speople +counting crows +connec ted +conjun ct +clinton foundation +city jet +chesapeake bay +chatter ton +car ita +can ine +bur ress +bts b +boundby blue +bbcra dmac +bas sel +bariatric surgery +ban ya +bad ou +b wp +al ara +ak ata +abduc tor +== >> +................ .... +% * +ÛĮ ÙĪ +yuv raaj +your san +world champ +wood ham +wescra ven +vin do +upri ver +tom ah +thoothu kudi +swap nil +strepto coccus +staf froom +salv ator +roof line +rel aciones +re land +pre zi +pon ton +per las +paul feig +of ac +oc elli +national bookloversday +nar alokesh +muslimb rotherhood +mull er +mu zic +monk man +manit oba +manico tti +love dit +lma ooooooo +lam ang +lac to +ker nel +k ti +intro version +i fan +gr é +gar lick +france sc +fe rens +famer tr +ec al +drown ing +d fr +cub ical +cre ak +couple t +cor b +co cho +christy clark +ce w +ce sium +c mag +buzz y +blan chette +bar que +aur at +ath ene +arri vent +arpeg gio +ang eni +ag akhan +a herne +zar beaz +wine festival +wh in +wasimak ram +waf ina +w aga +vas ai +uu h +uk ri +tra si +ton ik +th impact +syny ster +sun wolves +sri shti +smu ir +she hu +riski est +re ddin +r gd +pun it +pre ta +power pack +pol loi +pitt con +o ddle +nj morningshow +mersey side +me cc +mcelhin ney +mccar thys +market share +makeup addict +ma ula +m mos +line arity +likefor follow +kings down +ker sten +juni us +israeli pm +iah sbkb +i hub +hu sayn +hey bridge +freshoffthe boat +fra sc +faz enda +fair lie +eff i +earn shaw +eag let +duncan james +dar ton +daily quote +coo kies +ce sc +capric cio +bur s +brum pic +bie hn +battlec ry +ayrton senna +aw on +are staurant +ì ¸ +È Ļ +| âĹ +yearswith ourhomebts +xaver ian +x oom +will ington +villi ans +unassail able +un polished +u wl +track work +town usa +thelife of +the whl +the dragon +tandon raveena +t sca +sweet grass +super califrag +stabil ised +sle dder +sin ing +sin del +seis mic +seasons greetings +se futbol +sch ild +sac nas +sa ka +rohat gi +rin con +q con +pu bes +po len +per tussis +par va +orche stre +nun ney +nowh ow +ni en +nel sen +ne pom +myco bacterium +moto g +m fk +louise mensch +lan ao +kan ame +k caf +juli ssa +jose fina +j sl +ish tonka +is san +inton ation +inter group +hul bert +hou gh +hales worth +gu sti +galway hour +fre res +fag g +fa hrt +endor phin +empe zar +dad agioia +colon izers +chill ers +carrieh fletcher +car s +cali dad +brand ambassador +bear man +band anna +aw aking +austin and +assu redly +ari shaffir +analge sia +ali qui +albert dock +aizaw l +adju dged +act fl +ab sac +zit ao +zel man +ye hi +yar die +yak ov +wedd ington +wa thletics +vacu ous +v lo +use f +un labeled +un gi +ti ens +the pug +steadfast ness +star shine +son burg +soco tra +sh ays +sch mi +rencont res +rec com +property news +pret ence +post news +per roni +par que +orphan ages +nh ler +nastiali ukin +muk ta +mele hill +mee gan +md ga +mable thorpe +ll u +lator re +ky renia +ko smo +knock ers +jo bin +je melehill +hom mie +history inpics +having fun +haber sham +gon dol +gla ad +gab er +espn radio +e mon +dol lies +dhar mesh +cote divoire +coo puk +compen satory +commerci alize +berlin wall +be guiled +aper ri +alt itude +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +ðŁIJ ¡ +ÙĦ ÙĬ +yorkshire man +wk f +tu ku +thir um +tal yl +stri pers +sto ren +spiritu alist +selfies for +rum son +roano ke +reneeyoung wwe +recei ver +q or +pro petrovsk +phone book +p sm +over dosed +ou twood +oli vine +now next +moon star +mol dav +mma junkie +mi ming +man ito +man arola +leslie grace +kill joy +kash an +jon taffer +jack er +inst ag +improvis ers +hun te +gla ze +froma bove +floor board +ethe kwini +ecoun try +du mit +diaz jr +d orie +cro om +cooper age +coo ing +con oce +compart ment +bud leigh +boo throyd +bio feedback +bi ella +b ace +anti semite +an shu +album covers +al mo +ah hhhhhhhh +ag awam +af low +îIJ ij +âľĪ âľĪ +à¹ģภĶ +zu cca +wwi ii +vamp ira +up vote +tobac con +this couldbe +tenn ant +team priyanka +tarheel football +swakop mund +shi zuka +sar oj +rou ses +ran jeet +quick time +preci ousness +photo gra +pedest als +p foa +oo ool +on ghwa +o toy +newton ma +na sher +man gini +lith ops +lef thand +kur u +kot tai +kat sura +ju hl +jaqu eline +j ll +j anner +intra preneurship +hu ebner +hor loge +her zl +hein en +ha vers +gro ms +grace and +gr ze +gh hhh +gar ner +every onec +eli er +dr ington +dining room +deple te +de ul +cr ace +cou g +cont our +castell on +brit omart +bott arga +belle view +assal amu +and ras +all red +agar ic +abraham son +< $ +ðŁļ Ĩ +ðŁĴĭðŁĴĭ ðŁĴĭðŁĴĭ +âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +⾨ ðŁĺį +yomi uri +yo k +y barra +wall kill +tour series +thu mp +the tech +the cut +tem pel +te ct +sund berg +stat cast +star ship +slee plessness +shmu el +scor ps +sat u +roes elare +re ps +re ger +power team +politici zing +po ema +peer review +pang bourne +ok anagan +nz rl +ne ils +mun ni +muff lers +man handled +luther ans +learning isfun +kur ang +kron k +kre ider +kar ad +jet ties +iz abella +ith u +islam abad +irrit ability +holl inger +hob sons +hmv tweets +hel ge +glad win +gc ms +ful ks +fol som +fever ishly +facto ids +du tifully +draf tees +divisi veness +dis ley +del barton +de cen +contro le +ci mb +ch go +br injal +bil berry +bab ad +ar yl +am oral +am j +al tice +agne tha +ag re +; âĢ¦ +. ðŁĻĮ +ðŁĺ© ðŁĴķ +é m +zam bo +women for +wa jid +vap iano +vac ca +un witting +ultra vox +tra f +the movement +te kapo +te gal +te ats +tar quin +sweet pea +super duper +stam m +spider webs +somi asis +so fus +sh wed +ser ah +scre a +scal per +rei ffel +princi pe +pag et +osi jek +om c +official melb +nu x +no hate +nerd land +ne opolitan +nc dc +montele one +min ks +marin ate +lumb ini +lom ba +live science +len iro +ky ys +invisible illness +impedi ments +hy pes +ho ony +hin kes +hair net +ha yer +free thinkers +fa ena +exege sis +doom metal +dic amillo +de compressing +dal er +commissions open +colour ful +clu cas +clayne crawford +ckin chen +chu cho +chin aman +chatur bate +ch itchat +ch allen +center stage +cas amigos +caroten oids +bon da +bigh ouse +ball state +bag el +bac trian +awe igh +arizon a +anc alerts +ain ting +adi es +acce sories +ðŁĴĽ @ +ðŁıĿ ï¸ı +ðŁİĤ ðŁİĪðŁİī +ìļ°ì£¼ ìĨĮëħĢ +éĿ Ĵ +âĸ Ħ +women kickass +water s +walker ton +ventil ate +trou ty +tau rasi +su mitra +stry cova +sla den +skor janec +ship builders +ro ble +rheumato idarthritis +presu ming +pre search +pot vin +po wr +ow ine +otto bre +olds mar +o amaru +miér coles +marav illa +manju shri +lori keets +logan ville +le ben +kron a +k ls +j th +icar agua +humm el +horn swoggle +harish kalyan +hamil tons +ha pus +gu ter +gregg sulkin +gel at +gal las +g cd +frustrat ingly +fol ly +feas ance +evic z +dipp goi +devop sdays +defence minindia +con sternation +con gi +christmas giftideas +chint z +brooklyn bowl +bran islav +blo ts +bir man +bice ster +be go +bayani han +atharva amurali +al vv +afree dom +ab aesq +a jam +ðŁĺįðŁĺį # +ðŁĴ º +ðŁĩºðŁĩ¸ , +âĺºï¸ı @ +y ig +whit ened +v se +un conquered +turkey day +ter je +tal as +t mo +sw apo +su li +step brother +special edition +scot to +rour kela +roodepo ort +roh mer +ro mag +remodel led +rang pur +posto u +petro lia +pet one +pa wl +ny ayo +nu jab +nec co +name plates +mukher ji +monsanto co +mh or +maxim illian +leic am +kyle petty +jaz min +ive agh +intellectu alism +il ka +hi mer +hawkeye football +har di +happy hanukkah +happ ppy +gi mignano +gator sfb +gar ton +gar ni +g xp +far nell +fad ers +enrol ments +ene o +do ak +dd yn +coqui halla +conver sely +colla ged +chri sr +ch acko +best actress +be mpton +bar tering +awk wafina +at kinson +ambas sac +ama ia +alar mist +ak ela +abbey dale +ðŁĻ į +ðŁĺ² ðŁĺ² +ì³ IJ +y kid +x tin +x eni +woo ooooo +was sen +ut ch +the josh +tar af +tab ac +ta sik +ta homa +star com +sk k +sh ema +seri alized +scandin avian +sc primary +sai do +s green +roun tree +ros ler +project car +paw son +pat co +panch al +ofex cellence +new writing +morninge dition +mom preneur +mlb fancave +mis step +mc naught +mar ckinchen +man crush +mad ine +macer ated +lec tionary +la ffer +kunal nayyar +korean food +ko sa +kang en +k na +jo ppa +iscar iot +houston tx +hard well +gorkhal and +gig nac +gai waterhouse +g ittin +fr w +er langer +episco pal +dpan abaker +dolce tto +der bi +danielle jonas +da official +char laine +ch iso +cat sin +canadian art +caf od +brack nell +blow n +bla sko +bivou ac +bb crb +ari a +arche age +ak c +ait c +z big +xy lem +wi wt +whiteri bbon +wha kat +web zine +votethe wanteduk +visit california +un gen +turi sts +tre o +tobacco day +the women +the hub +stjohn s +south down +som thing +sl one +sk m +sam et +rick mercer +rayn ham +pronounce able +prison planet +photo journalists +p nu +over played +op is +nw sc +newmusic monday +nbs finds +much hh +mess am +mel ky +mac cas +ly r +love reading +ling am +l ence +kirk bride +kal ai +k pix +iso sceles +iron ton +ha ggling +ha ash +gur meet +grand fathered +glori ae +gad gets +express ly +dust pan +dragme down +ding bat +d ila +d drfc +crim mins +con gas +con founded +co bal +ch asseur +c sula +c suite +better late +av lon +av ine +alpac ino +all music +. âĺºï¸ı +! âļ¡ï¸ı +! ") +ðŁij¨âĢį ðŁİ¨ +ðŁį § +ëłĪ ìĿ´ +yo gas +vel in +tor rance +ti ranga +thegill sfc +team fiji +t she +sou ci +sk oy +singh vi +se ga +sad tweet +rose berry +rob ing +r tu +prote ome +petro grad +oke h +obfusc ation +ns v +nor they +ne phi +nar din +monoun saturated +mono graphs +mon stax +minig ames +mini fig +mcg ough +marketing profs +mac ys +l md +ku mba +kot ton +ker ang +kel sang +kadam pa +jr l +jon ker +jedin ak +jag gers +initi ator +haul ers +harshad chopda +hann er +grims ley +gr on +gl ickman +get te +gapp ed +free gaza +fox catcher +fin neg +f ylde +excell ent +dol orosa +dic ally +demago gue +d aga +curve leicester +cud more +cristi anor +costam esa +chri sky +challeng ing +bor omir +ble akley +blais dell +b ols +anton ius +ak ra +ad ara +ac io +âĢ¢Ì Ģ +à° ļ +Ä ĩ +won da +westh ollywood +w spa +w new +w kwk +vin der +v card +ttac ars +tourism malaysia +to tino +suj ata +su chitra +strength training +strat us +sli ppage +sky city +si ver +she bang +ry all +ram sar +race to +prote as +plu ssed +ph wo +par li +ous ers +ohl hockey +o he +o cher +mo ho +mn beer +mm is +mi ei +mcal ester +life jackets +letsgor angers +l antis +ki stler +kak apo +inspir its +ibm research +ho an +hauser wirth +hand fasting +grosse to +gold blatt +gob stopper +fer di +en isa +el achat +ecla irs +dri ed +dr joe +del ran +de icing +cp cs +constitu tional +connie britton +complain ant +catt aneo +cas se +bur fict +bit zer +bigger than +bal cones +ba sheer +ar mah +anti perspirant +ang at +ameri kk +zor bing +z ale +yearen d +w jr +vl tava +vene tia +val li +un willingness +tu fa +terror ised +strati graphy +sol ders +sigue meyte +second chance +sax by +sa hoo +roger sville +regin ae +red water +real cand +pu gwash +por tioned +polymer ase +pi ha +patt u +palla dian +oil er +nak at +mos fet +mond son +mett ler +mccar ren +light body +la ppi +kav insky +ji van +is da +hab sio +ha iz +fu kun +feed me +ex changers +dred ger +dra gos +despan yol +dal loway +d sa +d iness +conun drums +concert master +christa bel +chip board +bro me +br dg +bot con +av ox +ar iness +aph y +an sar +allyounee dise +air cooled +adri analima +________ ____ +ðŁIJ µ +ðŁı» # +ðŁİ ¦ +ya esu +wp xi +water mel +vul kan +var an +uc w +tyler hilton +travi spastrana +ta vola +ston ep +secon trol +sciencer ocks +rot ella +ron ge +ro j +reli shed +reinvigor ated +re configured +quack enbush +pram banan +pot pie +ph ry +par ador +papel bon +on pointe +moom ins +mo the +merriam webster +meng gay +man gin +li min +legali ze +lar ousse +ku pp +ksu owls +k lon +jonah hill +inter religious +impo stors +ic ta +hako date +giz mos +gin ar +gen de +ful wood +fr itsch +fi dh +en max +ecach ockey +dun ns +cran ach +code cademy +co ste +chau tala +carol iner +cape talk +cap ela +cab anat +beat boxer +audi uk +atlan tica +ast c +ap cs +anay ake +alovel is +alexander mcqueen +af gan +acadi au +âĿĦï¸ı âĺĥï¸ı +ze ena +yy cre +year th +u ow +to wered +tegu cigal +ta jima +symboli sing +survi vethe +stipul ation +stati k +son ne +si fa +sad dler +qui roga +py torch +precep tor +pra sad +positi f +popein dc +pj l +paul weller +pas c +p me +online first +obitu aries +ni gro +n anci +mur murs +mu cous +morton pride +mid ge +men eses +meda ille +matt cohen +mass spec +mark masai +mariju ana +man se +lon ilove +live sat +lat o +lam po +kri sallen +kod wa +kellan lutz +jurassic coast +jai den +highway men +hailstate bb +go tv +gine ttacars +flys fo +fin dac +festival en +expi alido +europe a +duck dynasty +dil dos +dil auren +co habitation +chi yo +cha ing +be aman +baz i +bar ash +aw akes +ar ugby +antio quia +an ele +amas amy +am eral +ðŁ¦ĭ ðŁ¦ĭ +your schoolgames +wel ke +uncann ily +ud ha +tril ingual +transfer deadlineday +thoma stown +the soul +stu ll +si rona +sem rush +ra hu +pet l +pe ix +ol it +naï ve +mor ad +mclo vin +matthi as +masc olo +mapl eridge +leniro bredo +lati fi +l ero +ky ren +k lim +k cbs +japan travel +ir yna +inter brand +healthy recipes +hat chee +har ia +hal owc +ha stert +gu di +gam bling +free zakzaky +es ler +eli ane +ea stridge +e derson +delhai ze +davi dragan +dar agon +cir clec +cell dweller +ce sare +car om +bru mm +brand es +assi sta +as shat +al gom +aggre tsuko +ðŁĻĭ ðŁı»âĢįâĻĤï¸ı +Ùħ ÙĨ +ye an +yan anda +v cm +v awine +union pay +un damaged +trou ver +tigre sses +thalas sa +switch gear +son or +shee ba +sc ali +sau gerties +san tur +rod rick +ro miley +ri als +quart z +pro mom +plat a +pit kin +par kins +oo ker +nationalgirlfriend day +nager coil +n mr +mor ose +momen tof +mis andry +med hurst +llan tri +liann ela +li bi +kerato conus +j assi +ima p +haz le +hatt en +gy u +gri et +go sar +ge fs +g out +ffi est +f live +eb sco +dun kelman +du rie +dollar shaveclub +dg g +dear g +d elling +crum pler +cor sairs +chef chaouen +chan ov +cau l +can el +cac io +buffalob isons +bluest acks +bab olat +as ant +angies list +aj ed +acry late +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį +Ú© ÛĴ +zel din +ze gers +wo ai +we ish +voye ur +ve p +united center +trun chbull +thi o +th é +taylor kinney +tam bien +subr ata +sisq o +sequ ipment +seam an +sand which +roush fenway +revo ke +relinqui shed +regar dez +realcand aceo +ra ices +r px +r gr +qu ic +prem inger +pp ort +over shadows +out paced +on looker +ncaaw restling +na ish +micro algae +lefthand ed +heis risen +hee ft +hay ato +go eth +eaves drop +du stries +didier drogba +devi ka +delic ata +de trick +daw ley +davidg andy +crazy catlady +co vic +cine matic +ch ynna +cao im +cambus lang +bull fighter +ber occa +bar ging +b chy +az es +alk alo +aliqui ppa +alamo sa +al ready +ak ab +" _ +ðŁĸIJ ï¸ı +ðŁĴĻ ðŁĴĽðŁĴĻ +äº Į +z aga +york region +wrestlec on +wol a +wes warm +wag h +u je +trape zoid +tra doc +til ghman +tiger land +tegucigal pa +tan ge +sun an +sto a +spor tivo +snow mag +sme aton +sh isa +sab es +ri ffa +pri mas +pre vin +pitt en +paul ryan +p ym +ow ers +nouve au +nih ilist +mo ves +mis al +mi dea +metal hammer +mentalhealth awarenessmonth +mc mc +man united +li more +last dayof +lar is +kom an +j sut +ilu min +i frit +ho yos +hillar ys +hal va +group think +gas per +gaff es +gab ler +ga yo +fundament alists +fi en +fe ign +far ben +e sem +e ink +dad y +da quan +cr in +chrysanthe mums +chang er +bri ere +beard gang +b th +ato saurus +as pe +ary ana +aqu afina +anne boleyn +ane h +# £ +!! !? +ðŁĺįðŁĺį âĿ¤ +ãĥ©ãĤ¤ãĥ ĸ +ଠ¿ +zab al +worldr hinoday +wil burn +unru h +uma b +thom ann +tex tu +strugg lers +stie glitz +squ alid +som ar +sol nit +soci ed +sny ders +sky park +sky activ +silver thorne +second ment +ruk mini +plun gers +pele tier +oliviam unn +nzv sa +nottingh ill +no am +net jets +nam ak +mun tu +mmo pen +me ador +mat toon +marki evicz +mand hana +lost withiel +lax er +krizz kaliko +kor aput +ki aro +kho sa +kavi tha +kan in +ja imes +j mf +is lan +impin gement +i league +hoy ts +hoo pa +gram een +gov walker +go ethe +glen na +fu tp +epic fantasy +elizabeth ton +einste ins +econom ie +ec inema +dre ssel +don ya +depress ants +d indi +crani ofacial +cole haan +chad ron +catch the +cam arena +by ard +bush wack +britt en +ben aud +bel field +baw ang +bag no +aye let +ag ry +! ðŁijĬ +ìĽ Į +zukun ft +y rold +wood hull +wolf blood +us r +ten ley +tag um +svet lan +sparkling wine +shorth aired +sd v +scott lin +sam well +sai ful +r hay +q lder +prepa red +porsche retail +pnp cordillera +partici pat +onec lu +oak hill +nor ad +newyearsre solutions +ne als +mi shi +mi mes +mi koto +med lar +lg t +kumar is +kn ish +kal ayaan +kab ah +ign ace +he iner +hat maker +hail statefb +ha sen +great outdoors +grace point +gra ver +gor ockets +gar ds +g ss +g sh +f sk +entr al +en ric +elec tioneering +dover street +dou sing +chickam auga +cer ruti +cc f +c tos +bri bie +braid wood +birdlife oz +bindi irwin +betty buckley +bar ford +bachelor abc +av ital +asu mmer +z oni +won gs +vor derman +ve ces +tu ah +ter nity +ten dered +t add +sudan uprising +stragg ler +stock holm +sl ickers +sar sfield +sa ige +s red +reality tv +rap wave +prim erica +pir ating +pettic oats +pet finder +person ne +peri sher +pat to +part way +pan avision +ott traffic +opp n +o gy +nm state +nichi wa +nax os +n js +mobile payments +ml v +me her +md ina +llan twit +lam ido +kur ama +kal ua +ka head +jol yon +its amazing +indi awith +im plac +ill omania +i ge +hei sei +haw finch +gor dita +gol gi +god by +go comics +gal anthus +fol les +fin sbury +fcbayern us +expialido cious +en ath +dru bbing +drin ks +dead man +de met +dall ara +cypri en +ca ws +by z +bram bling +bethan ie +before the +bao zi +audio engineer +ash tami +asbury park +ari bo +ar nel +aed il +actually autistic +& . +ðŁ¤£ . +ye du +xbox live +world autismawarenessday +working man +with stands +win chell +touch point +ton gass +ti dus +telome res +sub ter +star lord +skil fully +singh bjp +shar row +shad dai +semi freddo +sc abby +saturn alia +sas s +re printing +qui jano +pi quant +pa ice +oun ited +news official +mir alem +meh ro +mdpi openaccess +mari onettes +mari olopez +mafi as +mac murray +lm ere +li h +len ahe +land arch +krann ert +kirk gate +kar mal +inf ern +horseshoe tavern +hol box +gram sci +good hue +go jo +g cn +frame store +fr c +fl andre +fin anc +film house +favor iting +face friday +f type +edinburgh paper +chat tisgarh +ch seats +cc ca +car mageddon +bri se +bo diam +blo dgett +bhar athan +be positive +bau mbach +ban jul +b mac +ay ered +ani official +analy zers +alan rickman +ag il +! ? +å ķ +人 人 +ãĢ° ï¸ı +اÙĦسعÙĪد ÙĬØ© +Å¡ koda +you l +womens golf +wolf mother +wi ggin +univof standrews +u cam +ty wyn +thou sando +thang ka +tb world +sø ren +su til +spay ing +so telo +shape wear +sh ali +senator durbin +sec un +sci fri +sam mons +sal di +romanceno vels +ram ai +pulwama attack +pra sa +porte ous +pike speak +pect in +pande mics +pal lid +paco ima +one troy +oaklawn racing +ny le +na heed +mont co +milehigh basketball +medell ÃŃn +mar san +lv ad +long term +l ga +kumar sanga +kri pke +kinka id +kane ki +k lima +house man +h ns +gover ness +gas mask +fuoris alone +fiu mic +fi fer +feld mann +eur activ +er man +enor rhea +doughnut day +dista ste +did as +depress ant +dark ens +d pe +cord ura +cho te +cheap side +bu dy +boy music +bor uc +bo it +bedro omed +au stra +art week +arcan um +all ay +aljon mendoza +agen cia +ac lark +ì ¯ +ëĭ ¨ +æ ļ +âģ Ħ +will its +west chase +web dotcom +wasimakram live +victori alive +uni o +they are +thedavid cook +tele path +skul duggery +scra ig +scho tt +scan arias +san o +ren y +ram ani +rail card +perl mutter +p fr +p amp +ol é +ni shimura +missouri ans +madi un +mac ke +ma the +m side +looking for +loc key +levi than +last man +kar oline +joan rivers +jil ani +it uk +insti z +hotro ds +gob smacked +gastron om +gall ina +fair view +err body +ele mis +ead t +e bru +du tiful +dre ssed +dog pile +divin ing +dermato logists +dan ese +confe u +coc ita +che bu +can ews +ca ird +bus d +bron stein +boc con +bi pod +bedra ggled +bea hero +bar dia +aw ood +au bur +ake mi +af lex +abc new +į Ķ +ðŁĴķ ðŁĮ¸ +âĿĦâĿĦ âĿĦ +z ema +xi er +wrigley field +wire shark +v pi +un hygienic +trou ve +tox ics +together weswarm +to pa +thru sts +thi stogether +th oops +sust dev +success full +sini ster +sing topra +sido arjo +si ao +schne ier +sak thi +ru bery +ronni erad +rober ther +road man +quack ery +proté gé +pep ita +passion flower +paolo igna +pa aji +off loaded +o kai +na jam +moon beams +mi ptv +merri mack +merco sur +ma pei +lu sher +lu ol +lovethe se +loc alizing +kent ville +kay fabe +kar nad +jo ka +internationaldayof peace +immen sity +ic cr +hu ub +hu ger +hod gy +history in +hfx mooseheads +hen ny +hemp stead +hear tofthe +har aj +fi bo +f pw +everyday carry +dhan bad +dan neel +cityo fla +camel phat +bro de +bra dgate +bou ffe +bla ize +bit i +be you +bay state +ay lor +awarri or +ar cu +appreci ate +ann alee +anad olu +amrit arao +alder ton +ai rey +agood day +- _ +ðŁĴķ ðŁĴľ +ðŁijįðŁı» # +âĿ ĸ +ر ا +ب ار +webdotcom tour +wc cc +vintagec ars +vandy k +us n +une as +un ley +to wa +thofjuly weekend +ter abytes +t dam +sul fon +subsidi zing +spee dup +school teacher +safaricom ltd +ril lettes +rae es +quince añ +qui vers +por us +plus net +pase ando +occi dent +nico lec +new field +ner v +nan ba +mon os +mix ologist +man ji +mait reya +macap agal +m ris +luv ly +luth uli +lon tong +keer thi +kap it +jurisdic tional +j wu +ira j +heb burn +he z +half way +fry bread +fro man +finally mario +enner dale +doo oo +dog love +do doma +cutt inge +cre c +cou che +cloud burst +chick weed +cher ri +bon esh +barbar o +bankrupt cies +as say +arthro pods +ari b +amateur radio +alternat ely +ðŁķ· ï¸ı +âĻ¥âĻ¥âĻ¥âĻ¥ âĻ¥âĻ¥ +á k +zo eter +wren tham +wa elex +vintage shop +vien na +uri as +to chigi +the te +the halo +ta req +t ÃŃ +spelun ky +sop ore +sher ald +shee trock +seun gho +scar ter +rockymoun tain +rite ish +raj in +po ka +pay nes +pai ute +ni raj +nade shiko +miy abi +mariolopez extra +ma ghull +less ons +lar z +l wcf +kre ischer +kaz mi +kas atkina +jon tron +je ers +jal sha +jake gyllenhaal +is ar +ing lou +indone sians +inci dences +in ke +import ant +goo di +fu gard +fru mpy +fiore lli +exter ior +exporting isgreat +er tu +er ole +end aken +dra gom +demean our +dell tech +del vin +decentr alize +dau b +costi gan +copper as +conversation alist +co quito +cla x +char bonneau +chan sons +carbun cle +bydgo szcz +boro witz +ben jealous +baaaa ack +atter bury +apie terse +angio graphy +am ant +af ace +ack land +!!! .. +âļłï¸ıâļłï¸ı âļłï¸ı +é cole +wvu football +winter storm +wal wal +volu tion +van e +und ÃŃ +u ark +travel stoke +trade able +thur man +the mr +texas rangers +telom ere +tan gh +tac ops +syn cro +sput tering +sk ittle +sh eck +sab bir +red coat +rat nagiri +randomactsof kindness +ra wn +ra den +pu tu +pas aran +pan ynj +p nn +omen o +om iller +new nownext +net anya +nat es +n tx +n sd +mell ark +mas oom +loch lomond +li stos +li gi +iro ko +infor mants +ince stuous +ia eaorg +ho tin +hass les +giardini era +gam enews +g town +g dot +fictional character +fanta il +er ugby +devil driver +col ac +code pink +cobble pot +cam ano +cad ell +bu ea +bourbon nais +audit ori +au bu +apa ola +angry birds +alley ne +aid c +agrit ourism +aber rant +ë§Ī ìĿ´ +ê² Įë +è Ĥ +à¸Ħภ° +м Ñĥ +yoshi ko +xy z +wheel barrows +tol worth +the ban +sul jovic +sports day +som edays +soc cere +simm ental +shakespe ar +sentai filmworks +rw m +ru ffy +ric kets +rav aging +rab ha +promo ting +port elli +pan tani +oc cip +mr sm +montre uil +mo sely +mc millian +masco ta +logarith mic +lipp ert +leicester city +legendof korra +ld u +l hot +ko loa +ke ine +ju muah +jink x +jelly beans +jazak allah +j po +indu str +herli hy +ha kimi +guerri ero +ground s +ger alds +galvani zing +ga alive +from hell +emily thornberry +electro chemistry +el ach +dad sarmy +cru i +cru cis +coden amed +ch ava +cb z +ca ol +be strong +ban quets +astronom y +as mo +amin ophen +al po +al amein +ah manson +ah haha +acet aminophen +ðŁĴķ ðŁĺŃ +zoo plankton +zi as +ze se +world pay +wor then +won pil +ve te +tw chats +the timeisnow +te ifi +tar in +ta id +so lem +shi b +sha fi +sasusa ku +sasha apieterse +rwand a +readytor ace +ramo ji +ram but +poli mi +park house +ouri sm +not acrime +niel m +nanditas weta +mer avi +matt sorum +madeby google +macle llan +m learning +lu key +kno y +kere ta +katen ash +jolo kia +jay bilas +international beerday +inf op +ic tfc +i yo +i bee +gue sted +gud run +gri ese +gin a +g mh +fire water +ela bel +eb mt +doors down +dit i +discipl ining +der ville +decla wing +dare tobe +cymb eline +coffe ecake +climate kic +ce ja +brigham womens +bo dden +black shirts +ber icht +ar cli +anti bully +analo gs +adrian grenier +ac gme +ab abe +ðŁĩªðŁĩ ¹ +âĽĦï¸ı âĿĦï¸ı +z end +yu me +yu dk +wooo hooo +wood worth +wizard ing +whereare the +w tm +vla hos +un sportsmanlike +un satisfactory +tu gue +to paris +theshark daymond +temp i +tc v +ta onb +ta ks +set lists +scul ls +sch ing +sam warburton +salu dos +sal ta +ric ho +ri via +r alls +q ah +promp t +picku plines +pc ms +p tsd +on cbs +ok ura +ok um +nit rates +multi grain +mari insky +man metuni +ktt unstall +karan vir +intelligen ce +inst ameet +in ayat +im man +il be +hobb led +ha bu +golden heart +go jackets +glas shouses +gen teel +emphy sema +dif fie +daysof biking +dan as +co es +cleg ane +chuck norris +chiwe tel +check off +capitol hill +cap n +cad den +blood and +bindas bhidu +bhi ma +bank able +ay ur +ax eman +arshad warsi +aldine isd +>/// < +. ðŁĺĦ +ìĿ´ì ¢ħ +ãĥ³ãĥ Ľ +âĿ¤ï¸ı ðŁijį +y auch +wil mer +wa ghorn +vill eraces +uniof surrey +ul k +trage dia +sp f +sof hope +sher burn +separ ators +sci atic +sch em +s big +rol les +re organisation +priest field +pie zo +ph art +pe changa +papag ayo +out growing +of ic +niskay una +moven pick +ma hir +kra bbit +kirk herbstreit +jamie cullum +jagu ares +j beil +itv be +it in +ise d +im re +ilistic expialidocious +i inet +hurricanes andy +hands free +haha ah +gon char +go texans +glblct zn +gal leys +fl ite +fidd es +en am +el gon +diso wning +dimension data +cro oz +circas urvive +ci am +chu cke +car lor +cab er +bre hm +ben folds +bel aire +be ttes +bar ner +balu strades +ava etc +austral opi +ash lynn +as ong +are sults +arc turus +americas gp +ðŁļ´ âĢįâĻĢï¸ı +ðŁIJ ½ +z aks +yl vis +wil ho +viv ace +uniwest scotland +un manageable +un convincing +tour noi +tor p +team scorpion +tach ira +su tta +su mr +smith college +silver berg +shoot filmb +she mesh +ringo starrmusic +ri sen +red more +rac a +quintu ple +pontypri ddrfc +police woman +pero dua +pen ch +pe den +no elia +mo ch +men k +mar fan +manner isms +lo tor +light show +li ège +li ker +lc w +kram nik +ke mo +johnny swim +its morissette +intran sig +id preps +ian bohen +hunter moore +flat liners +fitz maurice +fil ho +espn nba +electro mechanical +ear drum +e pperson +deceler ation +dar b +crani osac +colli oure +cmo guj +cl gaming +christmas spirit +chris riddell +c gh +bcbg maxazria +autonom ously +ascen sion +applic ability +appar at +ali ze +ab uk +ðŁįĬðŁįĬ ðŁįĬ +ðŁĩ³ðŁĩ ¿ +zulfi kar +ye ter +yc dsb +wild cards +whis king +tom ilah +timmer man +team viewer +tai yaki +superfe st +stv l +spor tat +spo sito +son der +sch ill +rosen blum +ronnierad ke +rep ays +red action +re hana +rain out +pulse ghana +power hour +plo dding +pence in +pacham ama +oxfam gb +oom men +offer up +ni ff +luci dity +lo en +las ith +kettle bells +ju da +itsb hu +itsbhu shankumar +ish war +indy lights +hill harper +hayley atwell +hal lowell +gidit raffic +gentle men +fat ou +endocrino logist +el ica +e ie +doo fus +domen ic +dating advice +dat work +critiqu ed +coach ella +chu but +chino is +castlewell an +bu jumbura +brun ning +bren del +bo try +best show +best practice +art c +al hassan +ah so +[ ðŁĵ¸ +ðŁĺ³ ðŁĺģ +ðĿĹ Ķ +åħ¥ èį +yev geny +wynn lasvegas +wind screens +wil ayah +ven nel +unob trusive +the whitechapel +ste iz +staf fe +sheff hallam +sh anta +sas ser +sar ri +re hydrate +re ep +proto ss +pink society +pe zz +new sted +ne revolution +micro processor +michael d +met ta +meh reen +me jor +mar vi +lu croy +looooooo ol +lam ento +kristian bush +karti k +jam ai +inde e +imper dible +hol ten +hed ley +gl om +for pleasure +filipinof ood +fam osa +estim ations +e bba +dr r +cor is +clo quet +cl une +christen ings +business owners +bele ive +b fo +atmo sph +atas cocita +ash grove +arm rests +arak awa +alas sn +aintre eraces +ai o +aga wea +ðŁĺį ðŁĻĮðŁı¼ +ðŁĺĬ ! +é lie +zoes aldana +z deno +y wg +wweb attle +wwe balor +wn g +wardro be +tou ma +the silver +teenage mutant +te mbo +take it +stre sort +spec tres +span akop +soyu z +sincere ly +see konk +science direct +san tay +ridge ville +reprori ghts +re pos +rc despanyol +punche stown +play apex +pitt water +perspec tiva +or ync +nbl canada +n ky +mrjame sob +motu eka +mon gery +modi fies +make a +land ry +keo ghan +keller williams +kam bing +jackson ville +is ambard +ir bid +instagram ers +imo hq +ia q +hydro plane +herewe come +heid feld +h ny +gy imah +gn oll +fan elli +extor ting +esp news +elife style +elder ry +dsound system +ding us +de wi +collu de +clo ete +chopda harshad +chau th +but tock +brother lylove +boston ians +bj arne +bir ce +beij ing +back lighting +asi des +ðŁĵ¸ ðŁĵ¸ +ðŁĴľ âĿ¤ï¸ı +za andam +yam ahar +woo kie +whitt ling +walk up +wa wel +voteuk directioners +uz ma +utd before +ti val +thir tieth +then tic +the ginger +tack er +sy c +sulfu ric +speci esi +sp ly +sear les +sar angi +ri dic +resul tados +ren gagement +pi a +pat ois +pat gt +pap es +ol ites +oil paintings +naseeru ddin +n pe +mu tiara +moul dy +mistre at +matsu o +mar ney +maq sood +lab u +katie taylor +kak i +ipp v +ing el +i wear +himan sh +hard ware +happyn avratri +haj er +gul panag +go hard +fas b +em olli +e migrating +dull ness +divo ck +davel ackie +cor ten +ching y +chi pubschools +car rack +cabanat uan +ca illat +be quia +bar kad +autom ated +artu ro +ar ot +and thenew +ale tti +afcf ta +ab ies +ðŁļĹ ðŁļĹ +ðŁ¥ Ĵ +ðŁ¥ ĭ +�� �� +민 íĺĦ +âĸ ¿ +yan agi +wa verider +un werth +tsu kasa +tourmal et +tor books +than never +tere se +ta sered +stre ga +so cc +smrt grls +singtopra chaya +si hc +sham ba +science twitter +sch wit +sch oop +ra dia +ra bah +pur rr +pon ch +pole vault +pip kin +paw patrol +pa ku +ontheroad again +not food +nau strong +nag ato +na thletics +mo id +merthy r +mass mutual +mark waid +madhu bani +light wave +liberty u +la uro +ko sha +king size +kerat osis +kati es +kall ang +k pr +inver mere +inter na +im mingham +hu ling +han ji +guardian eco +gerst mann +ger von +gary valenciano +fe sh +favor ability +f cl +ella eyre +ec co +disp els +dir venkatprabhu +dhi ve +dab bs +clt news +civic tech +christin aperri +cer ven +cat es +bor de +ber ates +beir ne +barre t +banta yan +ar nos +an ich +am ya +ais y +?? ... +( ?), +ðŁĺĤðŁĺĤðŁĺĤ ðŁĺŃ +yel m +y bn +un quen +tz laff +truec rime +tro tted +thus ly +thin q +there all +tal in +ta vious +subver ting +subo ptimal +ssi ves +sou ths +so bukwe +shu shi +shel ping +sham mi +se pe +sav ill +ri seas +ra vines +ra gazza +provin cia +pro max +pod ca +pixar coco +pd c +pakistan day +official jld +nw ob +nin os +nay y +mun ford +mirand acosgrove +me gas +li zzi +letsgoo ilers +lar ynx +ku tta +kb fc +je ph +is ce +il ola +homes for +hat chi +haf ner +guide books +god man +fre shie +for zar +for scom +ew opinion +draft day +dj p +deut z +deleg ated +del ima +de fused +cot w +coqu imbo +capri mary +canber ratimes +back plate +aval ley +anpan man +aj ie +adity amusic +ad dl +ab ounding +( ~ +yebo sfaye +w cl +vand i +utdbefore fergie +usa wp +tu lus +thel ow +ter no +stu bhu +ste urope +skull hong +sil vano +she persisted +sever in +sche coperez +sau m +san antonio +sab by +sa ik +ru ba +root stock +resi t +real alexjones +psychop athy +pere mpu +pep co +or so +one on +north men +nihon go +nassi r +mr ddyer +mix down +mer ian +la voy +l ort +ku suma +kon eru +klar na +k fw +jinkx monsoon +ivory coast +iron dale +ira o +in bar +ilai yar +i yah +hen gist +gov andals +google earth +gesun dhe +final showdown +fabi ana +dre mel +down hills +di okno +de valued +dan ker +crin oline +co pt +ci ans +chicago tonight +ch ments +ce ux +carpool karaoke +c iss +broken ness +bou jee +bat z +austin chronicle +ask force +all nighter +agri gento +acom ic +ach opra +(âī§ âĪĩ +yan asta +welsh labour +tur man +trans ni +thehobbit movie +the sal +the oc +the men +tangi pahoa +t qm +strugg le +stati st +squ ar +si donia +seep ed +second sofsummer +sa jak +regre ssed +raj inder +ozy mandias +om ai +moon base +menta wai +martinluther king +lun go +lithu b +kir sti +juju y +jal fre +j iru +j chs +iz m +indiscri min +indie horror +im por +iheart festival +fri k +ff d +feder ici +en pt +dro oms +den feld +den den +cro on +chu mash +car me +bric ol +bra inte +black rock +banan aman +au spost +atlan tique +ase chat +arti stanbul +armie hammer +ðŁĴģ ðŁı½ +미ìĬ¤íĦ ° +é¹ ¿ +⼠µ +à¹ĥ หม +zeit kultur +wre y +withern sea +win de +vi van +va beer +v afc +tow les +tomilah ren +tar ong +tapro ot +tal madge +ta uro +swachh survekshan +su ba +style chat +sp qr +sn outs +slightly stoopid +slack lining +ser ger +romp ers +re mount +rati fies +pro tag +pintof science +pen thouses +par secs +pa al +owo sso +oster ley +need more +mug sy +morten son +meek ness +matthew perry +mart inet +mal econ +lon gan +lim mud +kno pf +ke baya +ka veh +k pm +ian to +i sit +hydro chloride +hand crafts +footballa su +fancy dress +egg ert +conceptu ally +clar is +bug z +bo ity +blue and +batt lea +aw ine +audi in +aira si +ad b +//// //// +åĥ ı +yang yang +y ade +wy rd +whit ely +war mongering +vk singh +ud hampur +ud as +tra chea +thisishar drock +te ducation +sweat in +ss ch +sna ith +slo bo +shut tering +shim mer +sent e +sad dling +rubi k +real grumpycat +rail riders +pre nz +poke shopper +pink ham +phu ket +p tu +oooooooo ooooo +oat cake +nir man +neuro psychology +mm n +missouri state +minim alists +mill pond +michael brown +merrychristma severyone +md m +mc gru +mar sabit +manolo blahnik +madal yn +mac ula +mabu za +ly fe +leg ang +lady vol +jal opy +io b +il ce +honey mooning +hi pper +ha beas +glam or +gaon kar +gal ah +event i +eso k +es lav +east chester +drum beat +doo joon +dom ore +ct tee +cross fade +chun ji +celer ator +card nation +bru ich +beverley knight +bare k +bal thus +as syrians +aphor isms +ann amaria +air disaster +ado do +ac led +íĮ ¬ +z illi +uk ay +u be +ty moshenko +tu borg +the alarm +ten pin +taxi fy +sy g +surplu ses +stad ter +som eness +sealevel rise +sche er +sch aal +rox boro +roc cat +ri gi +ravel ry +rampal arjun +ra vels +qui pped +pr sa +pj paralysis +pin sider +picad illo +phthal ate +paris airshow +ornitho logical +nbag league +nar u +nap a +mon en +mistran sl +men chaca +man jit +malign ancy +ma gie +kra kauer +kirk hope +k lang +ju die +j sb +ir chi +hom efurniture +ha sn +h Ã¥ +ghosthun ters +followthe music +flor sheim +fl amb +fi ggy +ef fu +eccle sia +das ara +cumb rae +choo choo +chatter ley +cal crutchlow +bil lows +bhat nagar +bell in +baltimore riots +bal z +bai da +arm our +ar ye +alle magne +ai z +aaron ramsey +.. âĢ¦ +ðŁĻĮ ðŁĴ¯ +ðŁĩ¹ðŁĩ ¿ +⼠ĵ +wy re +vit reous +vier nes +upp ity +under classman +tw arri +ture k +tur c +transcrip tome +the bestof +tel enet +te menos +te abag +tam bu +sp ig +skid more +ro mer +recipro cated +raci alized +pre occupation +pot gieter +poo py +ply ometrics +pk b +pil z +pe kan +opi ates +na oya +morgan library +men de +m craven +lo athed +linden hurst +ky ne +kp fk +ko chad +kent wood +jun jin +jame ela +its notre +imper cep +hi sti +her vé +grand order +family feud +esc abeche +en go +e ht +du elist +dress ing +debbie gibson +da ha +ciner ama +christyclark bc +charge sheet +car less +bys she +bis saka +be partofthe +abol itionists +aber lin +abe be +* / +ðŁĻĭ ðŁı¼âĢįâĻĢï¸ı +ðŁĶ Ĩ +Î ¸ +we ater +tutto sport +ti more +thos fansbts +style icon +steven spielberg +startrek beyond +starmagic ballfanfave +south down +si pg +sh ado +sexu alized +sel vage +sea world +se fer +sa az +s angels +run dmc +rock ford +ro strevor +ren dy +pre position +plu gin +pher able +onder wijs +of instagram +no tobaccoday +main waring +mahal axmi +maguin danao +lazare v +lam mers +kot zen +kitch engarden +jor gen +jalen rose +j boss +insta artist +fc punecity +far ish +fal kner +f atti +expun gement +esc ut +dro yal +disobey ed +dis lodge +dipi kak +di pp +deliver ing +dan u +criss angel +cor liss +co safa +cic cone +cannot wait +cafe terias +brad ys +bp brewing +bi gup +banff centre +ba azi +audi r +are ason +anag lyph +alder hey +al ima +affl ic +a ars +% ' +ðŁĺį ðŁijĮ +ðŁĮ¸ ðŁĮº +⾨ ðŁĴ« +à¹Į ) +à¸Ī ะ +y uru +wo ther +warm bier +upp pp +un coated +u mph +u icc +tur un +tu ki +tra ch +team jamaica +sye da +stubble field +stock broker +shay es +shar ath +ser as +seal ink +sc illa +ruth ven +ro ker +pett itt +pac u +op hie +neighbour ly +mumbait raffic +mr tony +med u +may ur +mar dle +mahabali puram +mag ico +luke skywalker +lloyd soflondon +leigh francis +langu id +ku bla +kiz ela +killer cat +kh are +jo ell +jay anagar +id les +hiro to +great ness +gopher football +goo wls +gon d +gloriae stefan +fron to +fil ial +est evens +er g +e bels +du wa +domest icity +diss i +desp ises +dermalog ica +cor alie +convivi al +con vers +con currency +clubb rugge +chrissi e +car rs +cam by +bu caramanga +bh fo +bertol ucci +bacter iology +au gie +app leyard +accli mated +a hotel +?? ?!! +! )) +ðŁĺĭ ðŁijĮ +ðŁıį ï¸ı +ë ĮĢ +w ops +visit dubai +uk uran +trib bett +the exorcist +tel le +teen mom +tar tt +sweet man +su zie +strange ways +sta id +sho gi +shay ne +sall is +revol ted +ran ko +polic emv +ple bes +patrick stump +paint it +ot ello +official rmt +numer ique +mr mike +mer vin +men es +mat ins +ma sn +ma saba +m chapp +leven son +le wie +kill or +kate bush +karate kid +jo shy +ital yin +ip hi +hu hu +gold corp +go ge +flexi bly +fer rato +far leigh +fan of +equ ines +el fy +dra peau +dis illusion +di van +dhu pia +dach shun +d hari +clostri dium +carmel ites +c cie +bird watcher +bam bina +bab bac +ash ah +ano int +anir ban +air park +a xion +ðŁĻĬ ðŁĺĤ +ðŁĶ ± +ðŁıĮï¸ı âĢįâĻĢï¸ı +ðŁİĪ ðŁİģ +нÑĭÐ ¹ +woo craft +we blog +un caring +uli ses +textile art +ta ym +sya allah +super set +subli mely +su ttle +stay er +ss yndrome +speci fies +sor sogon +silver light +sebastien loeb +seac liff +schlo sser +rhe ingold +resul tant +restre po +real kiefer +quin ns +princess bride +per awat +pe po +pale is +oster man +oc t +nz s +nu b +national margaritaday +n dv +mushroom head +miner alogy +mi haj +men ken +mart solf +madi kizela +ma awards +lys mic +love theatreday +lau dable +lace up +jim mi +jar im +j ynx +it to +her tel +heat ley +hal ka +h mp +gram me +gam blin +fakenews media +ell racing +du duk +doddle oddle +cot tam +christ omlin +car sand +bul g +bridge head +bod hi +bo di +bas se +bas quet +ar ash +am az +alton sterling +ak elly +ail way +after shock +af neil ++ ? +ðŁħ° ï¸ı +ãĤ ĭãģ +âĿ¤ ðŁĴļ +âĨ Ķï¸ı +⬠Ľï¸ı +yu ppies +ya j +wwebattle ground +wit tiest +wgn tv +west london +vol umen +vegan foodshare +uni e +tu ka +toby turner +tim heidecker +tar ra +tachi bana +sy zy +sweet ening +stri ved +stri ders +stealth ily +sn hu +smar ch +shim on +she dit +sha hab +se woon +sch ile +que ur +pul borough +prett ily +pra bal +pir tek +p mj +or rick +oli m +noise tte +neck beard +n fo +mu sand +mother teresa +mo khtar +mm els +mar awa +mal functioned +lu cht +lean ings +lea vey +land send +lam bourn +kin na +just voot +ir ca +indian cuisine +im debina +ho are +hather sage +hass ocks +gh oul +gastel um +famili as +f fort +ecosystem services +der vi +ch era +bom bas +blau ch +ay sia +ap ala +album cover +ade sso +acoun try +ac scc +ab khaz +a hockey +⾨ ðŁİī +× Ļ× +zo y +women scycling +wel les +we ave +war ded +vi des +ver uca +ver bose +u hc +thankyou foryour +tattoo society +stargaz ers +stal lion +sma ster +six ways +se diti +scra fts +sack boy +s ated +po ix +or se +ny nj +new media +neu l +nbaf antasy +mus burger +mssarah paulson +martin borough +mag ni +liti gator +knight sbaseball +kings gate +kick itout +kath niels +justin verlander +jay sh +iron ies +ira qi +inn keeper +hart field +har lot +had low +h br +gov pencein +gohar dorgohome +god bles +gl v +gel v +flet chers +fire house +film freeway +entrain ment +emma stone +dirty dancing +deriv ation +dal ila +confor mists +clo cal +christ us +chi h +cb ball +begru dgingly +bank sters +ball sbridge +andre sen +ador ableness +ab unga +' '@ +! '' +zam zam +yu o +u ver +te als +stereok icks +song contest +sol ak +se res +se berg +sa kata +ross ana +ri gh +resu men +rel f +rachel zoe +ph ale +passy unk +of gem +o gn +nur der +nug get +nite sh +ni hotri +na diam +multit ouch +mor y +mc y +maz ouz +li pi +lar an +kinder transport +kil kee +ke els +kab we +jave dn +infin i +id hi +hy uga +huntington sdisease +horse hair +hi rai +hay field +ha kan +gp stpete +gor sky +gi us +ge mba +fullmetal alchemist +for o +fee der +emple o +em ple +eber sole +dorse taonb +digg ory +did there +de jo +countdownto kickoff +compens ates +coinci dently +coco tte +cob den +catt ar +catac lysmic +cam hs +brew gene +bounty hunter +bo ing +be ggan +band maid +baker loo +antipo des +ðŁİī âĿ¤ +ï¸ ¶ +à® Ĩ +à ¢ +yar on +we de +w pi +thugsof hindostan +theband ghost +tex astri +swi zzz +stanis law +spr inted +speed running +samanthabar ks +rx theatre +robbie amell +rav jiani +pru st +priv é +post it +port vale +pol in +p gl +p by +otta was +one kiss +o cken +nrl knights +night scape +mortal kombat +mk stalin +milli a +mar aga +lac ounty +kyo to +kristall nacht +ko hat +kan ell +kahu ku +juli ann +jas sy +j sc +im iss +hr ough +hoo doos +helms man +hand drawn +fex pression +fast n +far ingdon +et cetera +dun luce +don nas +de min +d bt +culd rose +connec tors +combat zone +ce si +ce ol +call center +bushra gohar +bu ssed +bre saola +beque athed +b inger +ar mam +alla in +alder aan +abid in +미ìĬ¤íĦ° 미ìĬ¤íĦ° +à¹Ģภª +ൠĨ +ÙĦ اÙĦ +ze us +yeezy boost +wo tc +w aked +us latino +ur bs +un hindered +un commonly +tol son +thra x +thermo graphy +ter adio +tal kabout +stere os +sousap hone +smi the +sleaford mods +shipla p +sha e +sav inghope +saddle bag +rish nan +repo stre +ptero saur +pre gaming +prabhu deva +norfol khour +nol la +no sa +nivin official +night market +n wot +mer ano +mel illa +mar ic +maha ffey +luck enbach +lastman standing +lam esa +l ome +kit we +kh ong +junior golf +jet packs +jam mf +j ela +instru ction +hud gov +herman mashaba +hatt i +han deer +glynd wr +germany diplo +gar bine +for co +fo etal +film production +feni x +farmer smkt +fa dil +ev anna +dis sections +dil an +col usa +cla sic +chauvin ist +bow li +at rain +ali asing +ag gro +ab is +ðŁİī ðŁĺį +ä¸ ĭ +u don +tul bagh +thorn dale +te ssier +sw abi +sur p +starr cast +sock i +so wore +snu ffy +snow bound +sm oments +shrin ky +sensiti vely +sel ous +scutt led +sak hi +sab cs +que ak +pri mm +paul j +palanti r +paf os +op sec +ods al +nylon mag +multi platform +mud guard +mil sap +mi mpi +mb ak +mai kel +ma gar +ly sa +lu by +lo ja +ky on +ky ns +kh waja +kab y +jo onie +jo anna +jam z +inst adog +ing lish +inflexi ble +im material +i ferous +hor u +hierogly phic +hen day +har jo +han out +h pf +gol azo +g ml +flower show +first tweet +er ko +ent res +earthob servation +e wen +discoura ges +de codes +dar rius +cu aron +col adas +clo ke +brun tsfield +bran igan +bor neo +bbc debate +baz inga +barry m +avar ice +attle borough +at wt +apparte ment +amater asu +all orca +al ti +acab an +,,,, ,,,, +ðŁĽ İ +ðŁĺ± # +ðŁıģ ðŁıĨ +íľ ĺ +å » +à· Ģ +zo ek +xl vii +win kie +whodat nation +war den +tri ge +toriam os +thetoronto sun +sur charges +superbowl xlix +steen kamp +spac estation +sko al +ship mate +sc top +ro tter +red c +ra sam +ra jai +port slade +pietro sd +phine asand +p ous +ot ary +on dine +ny ul +n fts +mu stique +moving day +mosth aun +miss mayim +med lin +mazi buko +mark twain +mario andretti +maribyrn ong +ma drona +ly dney +lu mad +lou che +lit chat +kenne saw +ke ena +k hom +jim énez +information security +i fill +horticul turist +home bush +ho enig +her riot +he era +ha chim +gregory porter +game time +forwar ders +fl itting +faroo qi +eni ans +eclec tic +drama fever +dr ramansingh +cop thorne +congress women +conglomer ates +citym j +ci pollini +cham illionaire +carrou sel +ca hors +bruich ladd +brin ton +box nation +bother some +blumen feld +billy bragg +berzer k +beau x +bchy dro +bal ears +ath oll +ar cen +an any +aircrash series +ag elim +!! (: +ðŁĸķ ðŁı¼ +âĹ Ħ +yer ush +white privilege +well stone +volun tary +universit ario +u eg +tt ttttt +tour decor +top shelf +tack ler +su kumaran +ssh rc +sli der +shin agawa +sensation ally +se vi +ro ser +razar umi +quix otic +que iroz +pru de +pri o +phantom friday +petr ichor +per ret +parishi oner +op hore +np slsoccer +needi est +n side +mu lu +micron utrient +me tter +mc duff +man imonday +mal ouf +la zi +knott s +kapil sharma +jammf warriors +human itarianism +haz ar +har rie +fire hawk +em presa +eli seo +dor val +din ardo +cro ot +could ve +coli seo +col onic +city brew +cat oosa +calle ja +calabar zon +bun tings +braz eau +blon dell +black feet +bl ach +bernad ino +be different +bay bay +bate y +bare bones +au p +antw i +and bake +allu sions +alfon so +ak anksha +ai kin +ach el +: ~) +. ðŁĺĥ +áµĹ Ê° +z wart +ye sofficial +wood lake +waver tree +w mi +ven ango +thel abel +the flaminglips +ter relle +team england +sver d +substitu tions +st impson +spirit airlines +son ik +sling er +shru gging +shootfilmb enice +shi ho +she il +ser vos +secre ted +saldan ha +ring a +ren vy +rema pping +re shoots +re developing +ram yun +protect mueller +pri s +pap en +ornam entation +op tica +nom akeup +nieuws blad +ne ema +majo rettes +maiden graffix +magn ates +maf rica +lo chy +leg ato +le tus +kucin ich +kav ala +karisma kapoor +ka stel +jen nys +inter urban +inspector ate +ik ram +i vica +hon an +hel min +har twig +gott man +fli pped +et fo +esc olar +entit les +enter towin +edel man +dont judge +death star +crof ters +continu ous +co asting +categori zation +ca inc +by poll +bush rangers +bts m +bor ch +bladen sburg +bill c +bharat ratna +bent z +bel les +bay bears +ba hari +amber heard +am re +acom b +ðŁĺį ðŁĺľ +ðŁIJ ¨ +ðŁĩ²ðŁĩ ¹ +zi z +xia obo +ww lp +wis nu +water field +wait what +util ises +tw illing +till sonburg +tear away +tau gh +stru g +state dept +spha gnum +skÃ¥ ne +shur mur +show tv +sha ar +sex to +sc art +sa kin +resi ster +rag tag +ra ffa +r bi +quit man +pony ville +pe ets +os me +nr sc +naias detroit +n pa +mrun al +mel ges +mbr srd +martin us +maril u +mal alaf +malalaf und +la ak +kø benhavn +kentuc kians +jeff d +harve ys +ha bil +h sus +gra o +fer gal +extre mo +ety pe +ent rails +enjoy globe +education week +ec centric +easport snhl +dj carnage +demand action +dead laced +co springs +cityof to +chip man +bs india +be att +ball players +b nb +attrac tor +as salam +ach amps +ðŁĶ´âļªï¸ı âļ«ï¸ı +ðŁĴ£ðŁĴ£ ðŁĴ£ +ðŁİ¼ ðŁİ¶ +ðŁįĶ ðŁįĶ +ðŁħ ¾ +ðŁ¥ ļ +âĻ § +л е +wri mo +wi ster +waipa hu +vp ns +vi ren +under staffed +u or +taip an +stacey abrams +spread able +sol art +schle ich +salt coats +recei v +rat p +pilla ging +philly inquirer +optim o +mis carriages +mac lennan +m red +m ends +lu ana +little simz +litt mann +li ge +lad yof +ky line +ki shaadi +inscru table +i bby +hi zo +he gar +hav ells +gabrielle aplin +furi ou +fr oud +foli os +e dos +du pa +disrup tive +derby swildlife +cu pa +corporate events +code y +ci beles +chu gh +christma sc +ch ly +bol te +bo brisky +bick nell +belgian beer +babangi da +avir tue +ant manandthe +ant gupta +ak alay +ade sh +ðŁıī ðŁıī +ðŁıĥ ðŁıĥ +ðŁı « +ðŁİĦ @ +âĿ¤ ⾨ +zy go +zo ho +willow dale +wall covering +vol ar +vo ie +vit icul +vi vanco +ve mula +universal orlando +un v +thingsto doin +the flag +team bonding +sur render +ste pping +song song +siren craftbrew +singh rajput +sharma fc +shab it +sen batsu +sap i +sand pipers +sab ang +revel ator +repostre gramapp +record store +re ks +r fb +push pa +pin cer +pent agon +new saus +mis kin +mat patgt +manchester derby +le ery +klagen furt +jacqueline m +j ff +itsamazing outthere +inter sport +hermeneu tics +hackney abbott +gri mmer +good cause +glenmor angie +gett leman +futureis bright +fe tter +eras able +ep as +ent pvtltd +el ahi +du mm +du lo +dh x +day i +cul berson +ctv vancouver +create space +coffe et +christma spresent +cast res +cardio ed +boul ware +bottle cap +bbc sheffield +ani seed +angel list +agit ating +ðŁıĪ : +ઠ¨ +zomat li +woodcut wednesday +white tail +water birds +vulcan salute +vibra phone +van guards +un bc +twitter bakealong +tomlin son +to lead +thre eday +the jen +th august +terror attack +sul i +squat ted +spor ty +snar f +sk eller +sher in +sac ca +public school +ph unk +or ado +oh anian +non structurefire +nol asco +neyo compound +mtv scream +mor on +mis canthus +migr ates +mi hon +meik le +mal va +mai gret +madame tussauds +m wt +lo ath +ki mo +kag ome +ka the +je im +ic ym +iber o +goodday sac +go ksuowls +glen orchy +ge tou +found cats +fog gia +fire hose +esc ing +ed rive +descrip tors +de foli +dave and +dan j +dabboorat nani +cran fielduni +cookie monster +clu bes +char meck +cele bi +cajon valleyusd +by passes +bloom quist +black outday +be vil +bac ci +av ril +at ari +ameri star +achievemen thunt +ðŁĮİðŁĮį ðŁĮı +ðŁ¤ľ ðŁ¤Ľ +ðŁ¤· ðŁı½âĢįâĻĢï¸ı +민 íĺģ +x and +wb ss +wa ine +w set +vi mal +uuuu uuu +un believer +umbrella revolution +twit terer +tur on +theatric ally +than atos +ta em +soon est +so tt +she ir +room withaview +rising star +ra sad +pro a +po pi +phi mu +peru vians +os borne +or monde +nrl finals +no ha +nag ant +mugg sy +mor ven +mir on +min ima +mand ell +mad ams +l ws +l ft +krum lov +kronen bourg +kristian sand +kk al +kat mai +ju anda +jn f +jim dunlo +j pii +idaho bit +iam cityfc +hoo chie +hh p +heli anthus +hein ze +heid sieck +ham i +hal deman +g iller +ff ice +fabri ano +di mo +com an +ch ell +carmel ita +ca cho +bradley cooper +bo garde +ayo dele +authentic ator +audi ble +associ ative +any outh +aggre ssors +âĺĨ âĺħ +à® ¨ +yu ms +world kidney +weis sen +wa thi +ur ba +u di +u delaware +tuj he +to les +titan s +the judge +th ave +sw um +st k +st elios +spo ony +sphy sics +shonen jump +shant anu +sagu enay +ry ann +rumpel stiltskin +re funding +ray bans +ram ón +prescri bes +peter lee +per sil +penc iling +pack football +osu coach +of l +nor onha +ncis nola +nati o +mycoop food +muscul ar +mt ss +mrandmrss otto +mis érables +milit arily +middle ditch +michael cohen +me thy +matth ysse +lo ba +li stic +ken po +kal im +ju go +jac quet +ith am +il n +i amp +ho ad +halvor son +gol l +glut tonous +gli al +gar ban +game and +gal bi +frog more +floun dering +fen i +exple tives +er ism +er df +engle hart +ek elly +dilauren tis +diffe red +cri ppen +cram ond +conceptu alized +co existing +churche shour +chorley wood +character izes +castle wood +ca il +bri sas +book sto +billy burke +beck les +bamboo s +bac an +bab ae +awa res +as cona +all together +air watch +air head +ad aline +abkiba ar +[ ] +ðŁĵį - +à¸Ļภ° +à¸Ī าà¸ģ +ಠ® +¿ ¿ +v lr +uel final +tri somy +theore ms +th mic +th may +temp a +squir ts +smart ass +shen ley +sf in +sen toomey +sd npa +rape culture +r gay +quin tin +q oute +pv h +pro cks +privati zing +po lecat +pasta day +oliver os +nac omics +mendi eta +me akin +lu ling +la vasa +jo erg +jar vis +it canwait +it am +ira da +hor sham +honeye ater +he j +gran lund +go ren +fo dor +extor tionate +est ée +eras cal +enchant ments +dist antly +disen chantment +dark ish +contor tion +commerci alized +chan nibal +cahu enga +c bl +bo ons +bestin theworld +bar nette +ati ja +afro man +adu ana +absur d +ðŁĴķ âĺºï¸ı +âķ Ķ +zar ra +y ss +y rp +wym t +water treatment +u ji +tuss ock +trail side +top most +than n +stgeorge spark +steve backshall +statecap ture +sound man +sn ano +signsof spring +si var +sf bay +seung youn +see eee +san antoni +ruth enium +rolo dex +rock radio +ro gic +rit chie +ri peness +reyno sa +residen cia +ren nen +publ ico +pri maver +pe dy +oscill ators +nic colo +mon gabay +mis nomer +mind share +meso therapy +mc burney +mat u +man zi +ma et +long bridge +life proof +li fi +lea ves +l ston +kno bbly +keynote speaker +kemb la +kam my +is os +inst ills +insec t +hot line +home making +hawk shead +harri stweed +han ne +h ally +guimar aes +global shapers +fu wa +fo ghat +fe asti +fashion nova +downsyndrome day +din apoli +delhi daredevils +dam one +d vo +cran ley +comm itee +co gge +cla res +bluec rew +bin brook +ben ge +be hrs +aw fulness +as pley +ak itchen +ag andhi +acre me +ðŁĸ¼ ï¸ı +íĮ Į +ãĤ ¼ +âĻ¥ ! +ÙĨÙĪ از +wwe australia +weir ton +web casting +vor l +vacation ed +u mer +trilo gy +thisday inspace +th fleet +te pee +tar ay +tar awa +tan vir +tall madge +suit land +str u +ste eves +soundary aar +soundaryaar ajni +sn oops +shish kin +shi en +share downership +shann an +sen escence +sand ton +ronaldre agan +rise withus +rid gid +pad rino +ow ler +out score +on sunday +octogen arian +nijin sky +nav ona +my vote +muf fu +mb alu +mate os +long nose +laval lee +kn ol +kell ner +kandi vali +jumm amu +jar ir +horati o +holiday party +hill grove +happy republicday +gen is +fossil fuel +far rell +evi an +eu an +entang lements +ecur tis +dum dum +dr n +detox ing +der music +de portugal +cv cc +continu ance +com illa +co sell +chri smc +chic on +cham oy +campe stre +business awards +brush stroke +brew ers +aston merrygold +app dynamics +aler ta +ala am +ag ard +? ; +Ï ĩ +zoeter meer +xx o +x ds +world malariaday +wool loomooloo +wood chucks +wiener schnitzel +wango tango +vintag es +ve le +val lu +tu pid +trans location +tin i +thescript family +ste i +stab ile +son ically +sobie ski +slife style +silver spring +shi on +shatru ghan +samaj wadi +ron is +recu sed +raas ay +pre matur +pl om +pin atu +per anak +parv athy +pal atka +napak nowhow +my tuner +mou gins +mom blogger +men cia +mam ah +ma ini +levar burton +lem picka +jill scott +j ills +interpol ation +indu blin +im balanced +i isc +i esa +gun jan +gor sk +ge ia +flo fficial +ejer cito +ee o +dye ss +duchessof cambridge +down towns +do do +di alled +dex ys +cool kids +coc cy +co working +cityof calgary +bou rassa +boric uas +bo tta +bazaar uk +azur ite +az ula +arvind swami +arke stra +adv t +acon cagua +abo iti +x brownx +win nin +weber grills +we ka +wal tz +wa chs +urs bolt +trevel yan +tra s +tiny rebel +the modern +stay warm +spel mancollege +spanakop ita +sigur ros +seab right +se culars +sach inten +recomm ence +ra ima +r stream +pe za +ou les +obin son +nickj frost +ni wot +nb channibal +mil les +meal times +me sure +lu ken +lost inspace +lean newood +l out +kry sten +ko cher +klau dia +kevin and +k one +k far +ja ana +indv ssl +i agov +hom ology +hendri ks +h sn +gu lam +googlen ext +goal post +go yang +free comicbookday +folklor ic +erd mann +di stil +dese ret +chi dding +carol in +candle box +bush veld +bus boys +bu hle +blazed rts +bill und +bi as +barrel house +ba ster +b ended +avengers ageofultron +asi d +ap hc +after word +affe ctions +abt ballet +aaaa aaa +Ñĥ к +yoo ka +ye vo +work for +vocab ly +tyler l +tru cked +the fader +the cu +ss kin +sou they +sofrench vintage +snu ggle +si us +secon daries +scul ley +sa day +reig en +re yn +re ke +r mv +pot chef +pie tra +phoenix fc +peril ously +o calypse +ny dn +ni thi +nh strust +national ffa +muzaffar abad +messi ah +mckay lamar +maritime history +m so +level er +le vine +kon gs +kin ner +jo wls +j ino +ins worth +iam bic +happy y +ha ak +gurdas pur +gun boat +gor gie +free ship +fran che +fab ray +f rain +ecu tive +da rey +cra s +corn bury +cor dings +chu bbs +chal mette +bt ch +booker prize +bo ssie +blues music +av alli +at chaf +ar tofthe +ar ff +animat ronics +andre escu +alway z +af fords +:" ( +ðŁĩ¦ ðŁĩ¹ +âļ¡ï¸ı @ +whittle sey +wal lie +u che +troll hunters +thene ws +ta ita +ss ain +sr cm +smo sh +smo cking +sivak umar +siqui jor +sham ar +sei bel +seabor ne +sac agawea +rose bery +ro pers +rh summit +repaire r +raym undo +pro bowl +powe rexpo +pluto cracy +planes walker +pic coli +pe tawawa +pat ted +pas cha +om nam +o zomatli +neces ito +n cha +my f +mu dh +mormon ism +me ddle +mcn ally +kargil vijaydiwas +kam is +kahu lui +julie anne +jor danc +johan bbt +ig loo +ic tv +hack day +gre eeen +ge tur +flat breads +fir i +far ol +fa ja +end vaw +ence inte +eci ja +dun hill +dru mn +deser ter +de press +crink led +consoli dates +code word +coach dan +cinder ford +chrisco ons +chap els +cen i +canelo ggg +cali x +by res +bruichladd ich +bram ble +bonny rigg +bo vis +bo avista +berlin marathon +bel gard +ban ker +avi dan +aun ty +animalsin churcheshour +an diego +aldu s +academ ician +ðŁ¦ IJ +ච¸ +yr ne +year inspace +womens worldcup +westhou ghton +weekendkav aar +wave forms +wai fu +w andi +vaux hall +un popular +tro mance +travel deals +tom ita +the show +the force +th ous +ta vel +ta eng +stone mason +speak man +so bot +smart glasses +small man +skincare routine +sing lec +rutab aga +rest less +pic h +out your +nx n +nas chelli +mr sd +mollu scs +mis labeled +middle boro +mi andad +merle haggard +mckaylamar oney +may son +mad tv +log jam +light and +kis story +ke steven +kar ri +kap ta +k rang +k ck +jyp nation +juice man +joh to +jobo pportunity +inter mix +icel anders +hou sman +hof burg +ho ary +her radura +guar naschelli +gra bowski +fo k +flugha fen +fin stagram +feed stock +fa pp +f ni +est ina +else worlds +ec lassic +divor ce +dess ner +dennish of +dan ette +dan adi +cork board +christma scar +chat er +chang chun +ch ics +calab ro +cad er +bul gur +bron agh +british ness +br andre +bobcay geon +athletic club +arbor iculture +ade i +ðŁĴª @ +ðĿĹ ² +اÙĦÙĬ ÙħÙĨ +yoursan jali +yong san +ym n +wood way +whi pper +vic tro +van dien +ur sula +ul and +trumpkim summit +to cross +supanova expo +summer reads +spro gress +spo onies +somer saul +show live +shi ri +seab ury +sb scycling +sat ine +saiy ans +ru pes +rotat able +real d +potchef stroom +pirat enation +penn sau +peninsu la +pa den +p é +one music +onco logists +ne scaf +montal vo +mik ha +mexican os +meat free +lor is +lear field +king o +kali ber +jan ssens +j ingo +j adap +is man +ir radiated +inconsol able +ilove her +i know +gun volt +go karna +go down +gla dos +gair loch +frees ync +fit girl +falcon heavy +f be +ere wash +end pjparalysis +dont drown +derel iction +cut man +cristi ana +col late +cli burn +ch of +cardi opul +brain picker +border wall +birce akalay +bel ga +balfour beatty +austr a +ar vid +anc ats +am ali +aler te +ade w +aaf shar +ê¹Ģ ì§Ħ +âĺ ľ +zo eller +z apho +ya ali +william j +wen do +wel com +wef tec +vo ss +unic om +ture en +treat ment +to ib +terri fically +ta kht +syste mati +stron garm +street side +sothe by +sl ps +sin field +seemar aja +sand es +rigo berto +ridem cowboys +ra dom +propell ant +play dirty +pit z +ping tour +pau d +open bsd +olympiacos bc +nived ita +neuro sci +navas ota +my morning +my fabol +myfabol ouslife +moy nahan +max planck +marri ag +marin abay +ma pes +lar dner +lan son +ja im +inhab ited +ilove snooker +identi fiers +humb les +hex es +har simr +gri es +fundra ised +fun yuns +fort ner +eun os +eat forum +dun nott +dtp traffic +don ators +de franco +ddi qui +d pac +concer ti +co rer +co i +clich és +christ man +chio dos +castle knock +car ps +bulg akov +bud ger +bro che +brand ing +book festival +be there +b haw +b ampton +at water +al bu +akon nect +aj l +aa ke +ðŁķµï¸ı âĢįâĻĤï¸ı +ðŁ¦ ĸ +é¢ ¨ +åIJ ´ +xdanny xbrownx +x olo +ww o +wil ful +whit eland +wester berg +viral video +v schi +v cs +ty phus +tumh are +sn affle +slaughter houses +sky rail +sk amp +singul arly +si bu +shan go +sc rat +sammy hagar +rob dyrdek +ridicul ing +ri i +residen cial +rescue on +re generated +qui roz +precep ts +power house +ph oney +pet amur +pap io +oru van +official sting +o cra +ned ved +ne ipa +mynew tag +mn hs +miller music +megh ann +me issner +marvel ously +mal dive +lovi poe +logi st +lad broke +knu th +kit man +ki w +kho a +kal ing +jen nam +it ler +ho comd +hin z +green living +green baum +gra fico +giar dini +getyour rescueon +franken thaler +fiumic ino +fa wk +diur nal +co vet +clam ato +christ of +carni fex +car lee +bul ges +bor in +boo g +bo rel +bio sensors +bi fold +ate aser +arsenal ladies +alf ons +afcf ylde +adap tors +ðŁĴĿðŁĴĿ ðŁĴĿ +âģ ° +าภĩ +~ âĻ¡~ +yatsen yuk +ver ne +u son +twi g +thumb tack +ten sioner +techno park +spot swood +sp icing +sat omi +s mad +ross land +ph ela +penn sbury +over subscribed +ni ek +new stoday +muang thong +lenor mand +kul iner +ke sey +kag nihotri +jen ledger +jake and +irish art +inhibit ory +in icia +hill wood +gar o +gad os +fore saw +fif teen +execu table +dj mark +cristianor onaldo +clo bbered +ch azz +cb nnews +castel let +campbell ton +buil din +bo ze +beaver tail +asta xanthin +armit stead +algo har +afri kan +adul ation +. \ +ðŁĴĻ ðŁİī +ðŁ¥ ¶ +vs stl +vou lez +val mont +va ani +unit ar +twilling ate +ts is +to kimon +thingsto do +ter ab +te ee +stock ley +sto bart +squab bling +spl atters +sla ved +si vel +shake ology +scupp ered +practice make +parsi fal +ofthe dead +nutrac euticals +nodu le +nh simprovement +national garden +musc adine +molo ko +mo jor +ment eng +ma dox +m vollmer +lu hya +live sof +lingu istically +li venews +landscap elovers +la pid +killing bay +k vs +k ral +itsan u +it starts +inci sions +id der +hust les +ho fer +higa shi +h ø +h pt +god inez +girl shoops +galá pagos +fab rik +ev genia +ear nie +e up +du bin +din am +del os +deci mation +cwu news +cry engine +cong don +child bearing +carte sian +c nu +bur st +blue ish +bir ria +babys its +aw wh +ate le +at omics +ake chi +adelaide united +. :* +ðŁĺĦ ðŁĺį +âģ¦ âģ© +your koel +wo j +vis by +val ds +un vaccinated +teen nick +sy mes +sul liv +socialmedi aday +slo opy +sexual harassment +scarlett johansson +ready set +quebe cois +procedur ally +pro do +prim roses +power man +photo kina +photo gallery +o gie +nerd core +n ams +monmouth park +mock tails +mir an +me jo +ly nd +loyol achicago +latingram mys +ku tty +kre am +kitty cat +king splace +kind ler +jait ly +itsme leighton +is amu +ir ks +imbi bing +i for +ht fc +heer den +groun der +go thia +ge un +g elling +fur red +flipp ant +fam il +dhok la +dar ab +co sis +cen e +castle hill +boo the +bla se +bi kie +balth asar +b gl +ay ling +av neil +atlan tis +apocalyp tica +anc yl +air travel +. £ +âĿ¤ï¸ı ðŁĺĺ +âĿ¤ï¸ı ðŁĺį +wheel wednesday +weiss bier +w gi +ver ging +vair amuthu +under wing +trans mog +the resia +th ell +ten ting +tech a +su pine +strive forgreatness +stock hausen +spor tuk +sound sof +sor ana +social innovation +reu ven +redi ff +re ssler +re ms +ratt ler +punctu ate +prof tim +pro am +pirelli sport +pac ked +oso gbo +oly phant +ohi of +ogle tree +oba diah +ninjat une +mu kono +mom iji +mac manus +ma od +little bits +kir lo +kipl inger +ke dle +jun pei +joen suu +jo ao +isu zu +info world +in ty +gom i +ger hart +ge ments +followthe money +fe gan +exp ended +ef light +du pdate +dr jitendrasingh +disposse ssed +deli fe +cla vin +chap ut +cesar millan +cay ley +call aloo +caix a +c sw +britt ania +bo ssi +bed nar +bay ar +base boards +autom at +as el +alo on +aldu bt +al trock +ait anya +.... , +.. :-) +you should +wxyz detroit +white ford +wh one +venkat eswara +vac tor +un seeded +un knowing +the hotel +tex ters +sunday night +strath field +stabil ising +sports fest +spor tb +spod casts +slow dive +sleep walk +simp act +shaf fie +sel ke +sau my +ren min +redd i +red wall +recal cul +qui apo +post media +pennsylvani ans +pe ary +pat toni +param our +ou u +ou twit +os ong +os mia +octo path +model life +mis samy +miha ela +mach el +ma kaya +lu thra +let in +kum amon +king arthur +kill ick +kiaro stami +kemp ston +he av +h dm +gin day +gif te +free from +fr ary +fin ola +ferry corsten +far nes +fanni bal +do something +deutsch es +de us +cute cat +com ext +co ser +cla b +cheese making +brei vik +billie jean +anni separker +ach ance +ab ajo +aac psa +âľ Ĩ +ب Ú¾ +Äģ n +xperi az +we ert +wad ala +v adi +under funding +umichb ball +umb ral +tirmid hi +thpicture inyourphone +thanks for +ter ima +t any +squ iz +spur ts +side arm +scrutin ize +river front +ring en +renew us +re tainers +pinatu bo +paul hollywood +op killingbay +nsa ids +no love +ne om +mutt ley +multil ayered +lk ld +leed snews +kno ssos +karthi keyan +jennaf ischer +jar dines +infer ring +indiscrimin ately +indi am +ik k +i ka +house holder +hollywood musicawards +history pic +hippo campal +henri quez +hat erz +ham lyn +gri ef +good in +fil mi +fern hill +fe aster +f yr +extre ma +ear ne +e ya +dog food +dianap enty +der ailing +day le +cub stalk +comic artist +colon ised +cleve metroparks +catapul ts +carbon footprint +cap m +ca am +boule var +bou n +bal dev +art gallery +arab spring +ang poet +american airlines +ail oa +af onso +ðŁĸ¼ @ +ðŁİ İ +ðŁįĢ # +âĹ ¦ +Æ ¹ +will in +und proud +tweedle dum +tu llo +tten ham +thunder bird +suis un +su joy +strans formation +star killer +splin ters +so corro +shout out +shane warne +scot winter +sacram ents +s foster +road worthy +ri sky +reli able +ra ag +pow ley +pau le +pad ula +oller ton +og ata +nascar throwbackthursday +mush in +mile split +magnu scarl +lu re +live worx +life straw +knowyour mil +ken nan +janc is +irre vocably +ho ards +hel man +he o +great esth +ed rums +de morgen +cric ut +creek fire +cr ing +course ware +cooper atively +co generation +clu sion +ck r +che rer +cb nationals +cat tail +ca rena +c und +bru ggen +bl ms +betro thed +beach party +bal ter +asu bhash +ar dsley +anu radha +akure yri +af ell +acur ator +ab sm +âĿ¤ï¸ı ðŁıĢ +zebe dee +z wei +whistler blck +voodoo doughnut +venice biennale +vatten fall +une aten +un iter +un intelligible +twent ynine +supercalifrag ilisticexpialidocious +sun tron +stock car +speciesi sm +sp ald +sis rocks +sin noh +s folk +rock house +ri dec +rescu ecat +radio one +ra ye +pul kit +phum zile +people schoiceawards +pell erin +pad dock +over capacity +omo tor +olympic day +nt live +nam bucca +naf me +na aper +mix master +mis chief +min ori +mau ka +ma ser +ma dang +lun cheons +lili ane +li gas +la irs +king fire +kic cha +karnataka elections +jalfre zi +jak arta +intensi ves +ing g +in frequently +il h +i fans +haw kin +hau gesund +han ako +gun ston +fun di +fran xx +fa xon +es mail +el za +ed out +e ig +dull ilah +duc tions +defe cte +cro at +congre sses +con agra +chemi e +chae won +ch á +cel bridge +bar zal +bal doni +av iso +arow ana +ander matt +akit aranch +activ ators +< ---- +ðŁĩªðŁĩ ¨ +âĢ¢ _ +x ly +womenshealth mag +win and +visit york +vanoss gaming +ucan ationals +tr ps +theri ault +tfo ir +tassie keith +tai sen +steve vai +spiro graph +ski ft +sak thi +sab bat +richar de +pur ton +pro tractor +pro clamations +pre witt +pn v +peat land +oo hh +official jaden +ob inna +nzv pak +mike posner +mb log +maw gan +lothian buses +lic enced +li spector +lady leshurr +k rep +ing stone +in experience +i stom +hen ric +heis ler +gu g +gor den +gali ano +field stone +far nese +emboli zation +dho ti +demon ized +declar ative +cy m +con nah +con akry +cj tfoir +cirque dusoleil +char mouth +botu linum +bis ummit +bier ce +betterlate thannever +best buddies +arri an +and al +allato ona +ak tion +ðŁĶ½ ðŁĶ½ +ðŁijĨ ðŁı» +ðŁĮ´ # +zz ani +yogap ants +wyn num +wend el +we ichert +vir u +vi raj +v le +us ica +uof nh +ul an +tito jackson +tir ta +the urban +te tten +sung kyu +straigh taway +shail a +segam at +sal miya +road race +rhy mney +re routing +ray ray +ramadan mubarak +rac v +projectcar sgame +pou le +pom pon +per na +pack wood +nor der +new type +mit ski +ma gro +look slike +li ssy +li sn +jo tted +ji ofilm +jen is +im mor +ie b +hi pped +hed don +hac er +gre asing +gop convention +go gold +gan ge +ga str +fuj ii +fil am +feeding america +en rage +e ki +de is +cu u +cir stea +ci ani +c ju +bre ich +boot loader +bitt ner +aw alk +avac ado +ant elope +âĺĢï¸ı @ +woody att +wig town +week ley +weather spoon +wann able +unconditional love +un dignified +umat illa +tour bus +syste matic +sundance tv +strong man +spec tro +spand au +skin tight +se guidores +se btsb +sau cy +sar it +roberther javec +re direction +qaland ar +priyad ar +philando castile +petr one +pe ppard +p cap +out of +ob its +news agency +neu haus +mid den +men swear +meet in +me doc +mar get +mac room +lin gered +la van +la gg +kontrol freek +ko hei +kingdom come +kai ju +jamesbay music +j lf +ins grove +in ment +i ddle +honeo ye +he anor +griff on +gra fica +gr itt +ful mar +flan king +fact maniac +eye less +ev ene +en by +did act +demo ss +deli sting +d nr +crou ched +cote depablo +chor doverstreet +brom field +bri quettes +brahmot savam +bi ek +believe survivors +be headings +bark ads +art sat +an vers +all soul +akade mi +ðŁı ¦ +âĻ Ĥ +¨¨ ¨¨ +y sen +whistlerblck cmb +wayne brady +wan s +tin ka +summer hall +subordin ates +stan ge +spor ter +spor ch +soo bin +so gni +sig no +sa rea +rogue ales +reis man +re buy +radi al +r pio +pro create +pre formed +petre ls +ori ed +on enation +nautil us +narasi m +mex it +mer kur +list enable +just ing +inju n +hyper sensitivity +hugh hewitt +hu obi +hu gg +home security +hi elo +he yes +har on +gri dley +gre ger +gine ering +face hugger +eng s +doctor ing +det lef +cyclone idai +compos ite +club legend +chil lax +cess nock +bo para +attack man +ariel helwani +absac ape +ðŁĻĮ ðŁĴķ +ðŁĵ ķ +ðŁıĨ ðŁİī +zo bel +world heartday +wo ad +wkr p +wing let +whirly ball +we bbs +vive kagnihotri +vi varium +vel vel +vat ron +v app +um unna +um ps +tu it +ti ze +the group +terry fox +stu ary +stran ja +ssi en +soni sphere +sel ma +sd ss +sau te +s gg +ryan paevey +rosen crantz +proudly sa +priv atec +phineasand ferb +phil p +petamur gatroyd +pend ennis +ofthe year +north man +ne bulous +nb nationals +nav our +natural wine +mu cc +middle port +mas lin +maj d +lotte rer +lo ks +lo den +liz ette +kla homa +kho s +ka unda +jeon buk +inatur alist +humph rys +heer len +ha gelin +gym no +green light +gra phe +gl is +flo aters +fle cked +fe cking +fair cloth +et ine +en ning +del phia +david k +d wick +co qui +city link +cb se +camp amento +cal pers +buzz i +bru un +bin dery +ber ney +bath water +at kins +ani eri +akro tiri +. ðŁĴļ +ðŁĮ³ ðŁĮ³ +za bel +z wei +viri dis +unis ys +tere sh +su id +startrek tng +sin novation +side track +sc all +re launches +re ax +proudtobe afan +pl d +phal en +pel low +pan ter +onep age +of light +nak ata +metropolit an +melb weather +me all +loh man +leni ency +lee der +ko etter +ke al +kau ka +kas sa +jr smith +jar head +irrit ations +ir responsibility +inter varsity +infin ities +idu kki +hot cakes +historio graphy +hel mi +gu bbins +gro t +gr c +gg al +gam per +ga illar +engag ed +en sley +drewestate cigar +dess er +dang a +cro ston +cater ham +carmen ere +bru ff +bra bin +bal ta +awe a +arunach alpradesh +ared cross +angpoet nyo +andre wc +alleg ori +aal u +a ow +: "@ +!! ðŁĺĤ +ðŁĻĤ ðŁĻĤ +ðŁĺĭ ðŁĺĭðŁĺĭðŁĺĭ +ãĤŃ ãĥ£ +âĦ ĵ +z wick +youngg uns +wool fe +wild london +vi kh +uc tions +tri go +too cool +thereal rvd +the view +super cute +su blux +spe sial +sp ree +shwed agon +ser da +seman asanta +se stero +s store +rou ch +railway seva +pra gy +pit o +pi red +perfec ts +panam ax +p tes +oy als +ol son +o ved +o ao +mou lder +mon bebes +mo xon +mckin lay +mck oy +mar aj +maniac ally +mag navox +ma kara +m ry +lit us +lay lee +ko komo +ker ne +kent cricket +kad ai +ka day +io ana +innu endos +ic caworld +heroesofthe storm +hasling den +go akonnect +gi gas +gh gs +fox worth +ey yyy +espark global +e es +do bro +delav an +cyber bully +cot tee +chrismur phy +cattar augus +cab ramatta +c mes +bu fc +bce agles +ancient greece +alan hinkes +ae olus +" ðŁĺĤðŁĺĤ +ðŁĺŃ ðŁĴŀ +ðŁĴª ðŁijı +âĿ£ âĿ£ +x pac +weston birt +travel writer +tou galoo +thelittle mermaid +thak sin +t dov +spri mary +spee dos +slan dered +shi ela +sharpen ers +see u +sch um +roy don +ri era +red lips +raaj je +proff itt +pig farming +pie bald +pe tani +orchestr ator +om ha +ny anza +ning xia +ne om +mind hunter +mc grew +ma sy +lux ottica +lu chad +land reth +kan o +kah ler +iron mongery +inter ment +ing machine +ine yard +im thankfulfor +her mie +gleni ster +gill on +from software +flo c +fir min +fion nuala +fin ality +feliz sabado +fabric london +f omen +evangeli zing +ev and +eric mccormack +enfor ced +end ar +el ocke +dru ga +dianer avitch +dell inger +cv shealth +cr andon +confir a +clin ica +chop p +ce ylan +cas cara +carbon iferous +cali ban +ca zen +bran chs +boo oom +bl are +bi ji +befu ddled +as car +and star +aar ad +_ ? +ðŁĴķ ðŁIJ¶ +ðŁijĬ ðŁı¿ +ãĤ¹ãĥ ŀ +ñ o +za it +z g +wx pn +wur ster +us mc +ty balt +twil d +trump ing +topic ally +tol lywood +tod ds +thar parkar +th agallery +tel lus +tan alysis +stri d +spy ros +sop hs +sn om +slo tting +shu ang +sar ker +san kt +sam harri +re interpreted +ra fe +pizzic ato +pi eds +over shot +or not +oil spill +mun den +mo ton +mi hi +mchapp yday +mc delivery +man nnn +ma sina +link building +li at +lar ter +kon ser +kaw aii +kam ina +justin suntron +journal news +jo va +jay enge +icar us +hu zur +hall berg +half time +gitt ins +gerard butler +gel derland +gay boy +film fest +fen ster +face paint +enam els +emili aromagna +edi fying +des borough +dean heller +dar b +cnn travel +cleanpower plan +ch bosky +cau cu +bill peduto +big lia +baja j +b ø +auto biographies +ash lyn +ascri be +antico agulation +anthony horowitz +anom yces +ann er +! -- +ģà¸ģภģà¸ģภ+ðŁļ Ħ +Ù ¹ +wom xn +whl hitmen +we fts +vff vishal +vet tori +ulte gra +uci wwt +track listing +tough enough +t st +stanley cupplayoffs +snail mail +shant aram +sever yday +seti awan +scaram ouche +sau ced +sa are +s freedom +ri vard +quan tock +protec tive +portman teau +phumzile unwomen +over confidence +ov ation +mon tee +mehro tra +loccit ane +live th +li ming +kra is +josh gad +jobseeker ssa +iz abel +ic inema +ham in +goo py +fei jo +ew p +est us +do good +deeper learning +dan los +cor dia +coo kier +co key +ch affin +ce as +calgary transit +bor an +bly theville +big basket +bas ili +baj payee +awal ha +auto pia +ann aya +ac ws +absacape epic +a app +__ , +ðŁĻĬ ðŁĴķ +ðŁĺĤðŁĺĤ ðŁĴĢ +ðŁĶ¥ðŁĶ¥ # +ze stan +yah u +we irs +vou vray +voi ded +untol d +ug l +twitch stream +tutic orin +trade fair +tobogg aning +toa dies +thar u +tang led +su af +strom ness +steve dave +squir ming +slo gging +sil om +sc y +rival scamp +re locations +qu onset +poly gonal +politici ze +pember ley +pavel ski +pa j +not is +nishi ki +mothere arth +mor oney +men ina +mam bas +mal k +m cu +lor ra +lessthan jake +l ko +kor dell +kal yp +josh frydenberg +heather peace +h sf +good child +ger hardt +galli fre +farrow andball +eric balfour +el kie +dr s +dj k +diplo docus +de sailly +cynthi aeri +cynthiaeri vo +corn hill +conver ge +chaz elle +caris brooke +bri ant +breaze ale +blaz ey +bend el +b pi +atta k +ambi gram +am ii +akih ito +.. âĿ¤ï¸ı +! ðŁĺįðŁĺįðŁĺį +ðŁİ§ ðŁİ§ +ðŁį» # +âĻ Ģï¸ı +ঠ¼ +zen berger +yal da +win ders +where smy +washington state +w py +w boc +verge currency +ve les +tur fed +tu beli +tra pt +thereal swizzz +the bookseller +ste ttler +si mono +selfies aturday +river city +ri ese +relati vistic +raz dan +rath farnham +radi i +pree mie +perpetu a +op teryx +omo to +om ey +nicholas ville +my pov +my haver +mor rin +mo der +mis alignment +master killercat +mar ucci +magnuscarl sen +lu gged +low veld +lou reed +liber té +let tre +lang one +l lanes +kur umi +ko jic +ki kk +janes addiction +jac burns +j sd +i uk +hus bando +hou sat +hillen brand +heuri stics +head dresses +grill z +green roof +g suite +fukun aga +f sd +episte mo +eleanor tomlinson +east ayrshire +du rag +du hok +dor inda +donthe con +dont mess +do xie +de sa +dcu o +dar zi +cric ci +chuk ku +chis wick +central america +ced rick +carab a +bra dd +beach in +ash ak +aser vice +ak ki +ðŁķº ðŁı½ +âĶ ģ +zar ah +yo len +whow ill +wether spoon +va jani +ur gess +tsogo sun +tro pea +tot tori +tokimon sta +te gu +subscription box +strath aven +ssss ssss +shepher ding +seraf in +ri ddle +rep as +rel v +refra ined +ree du +raj as +par des +offro ading +nu ss +no stri +njor oge +navi es +mun nings +ma kurdi +liver pud +kat ju +karu izawa +jamest aylor +j ca +income tax +hel sby +h ly +gu ssie +gr anti +frog fish +fro w +endeav or +effi gies +dé j +dro pp +dread fully +do go +dh w +demo l +dat aware +da cha +coven ant +compul sively +com in +cel and +brett anomyces +boys noize +awesom econ +austin aries +asha hidi +ague final +ag ris +adhe era +accordi ons +abi er +. ðŁĺ³ +ç¾½ çĶŁ +yaw key +what it +we be +wb homeent +v nd +u ht +toyotag b +th uk +tartu ffe +sub floor +sp ga +shank land +sever na +secur itization +school holidays +ripp rince +ri zza +reak tor +rachel platten +popcorn day +poly phony +pickn pay +our is +od sc +o kes +ne olith +mythri official +mu sher +mr v +mirand akerr +me tball +ma gam +m clarke +ludd ite +leim ert +leee v +kt lamor +k aus +it pro +in ol +im printing +il more +hugh ey +hot deal +grized ale +glen shee +gen est +gan esan +gab at +elfy nevans +duckdynasty ae +doing good +dc v +dau ber +cron je +cityof melbourne +chan go +cel lists +cav in +categori zing +ca ac +burn t +boast ful +auto gas +art fund +arba az +adidas us +accredit ations +ðŁĶĬ ðŁĶĬ +ìĨĮëħĢìĭľë ĮĢ +ãĥ ® +âľĮï¸ı âľĮï¸ıâľĮï¸ı +z berger +yak ima +wound care +woke up +wil dearth +wil braham +warra gul +v any +tennis channel +team di +tarpor ley +target style +tan nen +stri stan +stabil isers +software ag +shel ford +seran goon +satyan adella +ro bri +plat zman +pi at +north bridge +mun ia +mpha sis +mosque attack +mom oko +minneso tan +min eta +mickle over +mal ki +ma pre +le amichele +lb cc +land cruiser +kas ab +k ely +it ne +int endi +il ta +i ye +hyper icum +hal am +ha dj +gram ophon +gr aca +go beyond +gd xj +findac ure +fau bourg +fair light +fabric ators +estu arine +endu ro +emb ra +electr ici +el ç +doodle bug +di ye +desp airing +del dia +de hart +d tb +d com +colom be +citizens advice +chao sium +bro man +briga deiro +born thisday +boccon cini +blu enote +bike suk +berkle ecollege +baili wick +anasta sio +allin cle +air baltic +ah mar +adel phi +\( ´ +ðŁĺª ðŁĺªðŁĺª +âĤ¬ / +à© ĭ +yar ashahidi +y ria +wim berley +wak ing +wa ren +toll gate +thunder y +tb w +tag ger +t illed +sur yah +subju gation +su sd +stend hal +stel ar +stat news +srin u +seab orn +sclu bs +sch ell +samharri sorg +salt iness +rust ington +risd greatness +reque st +reflec tors +rainbow dash +ra abta +prayag raj +positi on +police state +over wintering +orient alist +orb án +opportun ism +new sham +mccarthy ism +marl borough +mand elson +mand arina +m bro +livin ghistory +linch pin +lec avalier +lazy bones +lago m +l fm +kie wit +k ops +jaz ira +hydro gels +hull kr +hu bie +har pur +h st +guard ado +gro ene +gor ies +gna sher +ger tler +gearbox software +gad wall +fri ars +ebay seller +dr ace +dhar wad +den smore +dd x +damian lewis +counter punch +cor ran +controversi ally +cho ic +chlo elu +chill on +che shi +carbon tax +bryan dechart +berk shire +beli e +be side +bar rescue +bandi pur +baghe era +badger mbb +ast irl +asdfghj kl +aro th +anciente astirl +achi bi +ace supported +a asu +ภħ +wro ble +wood hill +will unga +welcome home +usur p +un in +ty as +team cavuto +t mt +sukk ah +sque aled +so sad +seduc er +se amu +santaclar ita +ro cc +re evaluating +pre conceptions +pli ss +palladi um +ous as +on racing +on assignment +obl iteration +morethan adodo +mir ates +melind agates +mas jid +mal do +making ithappen +lc dsoundsystem +ktlamor ningnews +kry stian +kra al +kalli o +jacob hoggard +ing all +in cr +imprison ing +implic ate +i sic +henne sy +h miller +gul ated +gu dda +grin gos +good olddays +go emon +g iler +g anta +foot man +f tw +er ba +don jon +doc sis +destruc toid +dann ys +construc tivist +cler mont +car mine +canadare members +can ar +ca zeno +c be +by example +bir ney +beve ren +ben y +bate man +bat l +basketb alls +bad ar +babbac ombe +at si +an si +ame ren +alla ire +air por +ðŁĺĬ " +ðŁĹ ¨ï¸ı +âľĮï¸ı âĿ¤ï¸ı +wol len +wec twx +wa qf +uof c +u mau +tul alip +travel and +the bookof +th re +team god +tam ashii +ta phone +syco phant +sushant singhrajput +sun iv +spro tt +siss ay +shel tie +save baloch +sanji v +sa wak +roe bling +ro jak +resi stencia +r deye +pro xy +prisma drop +poli zei +pau illac +pais leypark +oaken fold +no ps +narra been +n hat +mill ilit +mill ar +mary port +maniz ales +maithri palas +lep tin +le pe +lar oc +ki bler +kex change +kati epiper +kathryn bernardo +jancis robinson +intere strates +ij tema +i vi +hunter don +hal mstad +great things +gotom eeting +gh ur +frequ ent +flori dam +et itans +ell ines +ed ington +e bulli +dwarf ism +dn vgl +diso wns +dis assembling +di vison +de um +dann apaola +d bn +cur ro +corner stones +cor vids +c ica +bye felicia +boy fie +box uk +black ery +before thestorm +bal ck +ati ma +astri d +arri aga +amar na +ag it +abdou laye +ðŁijī ðŁı½ +ìĸ ij +ãĤ ľ +´ ) +yepp oon +y gritte +tur rentine +tl f +the water +ter on +tae gi +ta ines +swat ted +stein er +snar led +shum or +senior living +seg af +sch rock +sant angelo +s dream +roman atwood +pti family +primary day +presiden cy +police media +phlebotom ist +phan tasia +p ny +om bra +olom ouc +n illy +mu lai +milit o +mel brooks +manhattan henge +mang aka +mac world +lose it +little borough +lascru ces +kill aloe +kast uri +karim nagar +je hu +isi ah +iit tala +ig da +id v +id ar +ha za +gur khas +gunnar sson +gla xos +gen oot +for ten +ey ah +eve ready +eic ma +ec v +doll ard +den park +dab bled +cre tan +co cin +circas sian +cic lavia +ci ena +christ elle +chol as +cat love +cal me +c cha +bts v +booth bay +bb bs +ax alta +ark ady +aad hi +ðŁİ ł +zoom tv +y dr +wester ni +wal drop +vil akazi +vascul itis +tw da +to taku +time machine +ti ppers +teen sy +te trick +te hama +st any +sher ilyn +rook wood +red hook +re avis +qu aver +q alam +protector ate +pro phyl +post it +petti bon +pascual inigo +p online +opent able +nu be +no tim +no onday +mytho logies +morde chai +modu l +meg af +me che +mc elderry +mar veled +man h +m ste +life jacket +lgb thi +le dyard +land form +la ko +l de +kun kel +is am +indic ts +impeach ing +im pascualinigo +icon o +i shares +hoo pin +hocken heim +gu ta +gran fondo +gou ges +gentleman jack +gam asutra +food day +fire balls +exoner ation +em n +e zi +dp wh +desider io +d jan +congreg ating +com ber +centi pedes +catoc tin +carson daly +cag ey +beli veau +ayl ward +au b +af sc +® ¤ +wy land +wash basin +vi eng +ver us +val eo +ty ronn +toko pedia +the mm +ter ram +tand berg +stell ung +staff spolice +sm iller +slen ses +sierra boggess +sarah j +russell tovey +resi zing +rap zilla +power ofthe +plu ckers +phx cc +ni zhny +ne akers +nas reen +na aaa +murder she +moko ena +mojom agazine +mit press +mechan o +mayweather mcgregor +mar clam +kimmel center +kim rhodes +kemp ner +kee ton +jun gh +invasi ves +inter dependent +ib ps +ha ie +h ref +glaxos mith +festi vely +end papers +dren ch +daily doseof +d aki +cu lotte +cross dresser +crock ford +cop ts +co zi +christen son +charmeck schools +cb ssf +castle berry +carpen tier +canad ain +can zone +c son +buss iness +bts loveyourself +bra da +ben zi +bas sembly +barin holtz +b vp +au gur +anson mount +anit aha +aag adu +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ðŁijĮ ðŁĺı +ðŁĩ± ðŁĩ· +å® Ŀ +âľĮ âľĮ +yu asa +y ane +wthr sunrise +wood chip +wak ely +under pin +u cino +trib live +t mp +swan wick +stre ich +soo on +single use +si ol +sha itan +redribbon week +re um +raw ness +qarab ag +pro pel +pit ttweet +pau sa +patrick t +pal mistry +ode sa +nail sea +na har +moto x +moor town +mon tre +mee se +mal vinas +live smart +lique urs +lin zer +leol aporte +learn french +ld w +kol by +klon oa +juxta positions +judy garland +jcde caux +jay weatherill +jay esh +ja el +ito to +invali dated +ini st +id hun +har ro +happybirthday justinbieber +hand bill +go live +get thelook +g alea +free wheeling +fax es +eye mouth +ep ona +e or +dupon avon +dont missit +dar ran +color ism +collier schools +col chester +cn v +chrismurphy ct +c party +bre er +barrym cosmetics +bag oong +auto crat +amne siac +all night +ak off +ab ri +a show +: *** +âľĭ ðŁı» +âľħ @ +world suicidepreventionday +wl w +widne srl +wai fs +vegan life +uk is +te ahour +ta pers +t india +swithout limits +sr q +squawk cnbc +sour puss +shal ini +sec unda +sat ra +sam bas +recomm it +press meet +pin tos +pap ayas +pa yoh +ox as +oshiom hole +or cla +oak brook +novic hok +newcast ler +mv cv +mor van +mant els +man era +makar ov +m te +lu kens +lily collins +ko tt +kja er +khur sheed +ke ach +ingex cellence +immuni zed +honor thefallen +honey badger +home wardbound +hil and +gord downie +fel den +fam bam +dl mundo +coo ky +chip day +chang s +cal ea +bun ji +bra infood +bos mayor +bon line +blephar oplasty +bee man +base board +bam enda +aw memorial +avail ab +andro gen +ana is +acoun ty +? '. +ðŁĴĹ ðŁĴĻ +ðŁį ¤ +ðŁ¤£ # +ìĹ ł +âĹı âĹı +zulu eta +yoshino ya +worldkidney day +whisper ing +wb z +water colour +w one +urban ized +uppere astside +turnaround dontdrown +try pano +tiru pur +the gautamgulati +the darkness +tan am +surface pro +spino saurus +sky land +si endo +shou sem +sebastian stan +sch ini +rob kardashian +rizzle kicks +rebec car +rawling ssports +rallyfor rivers +q ia +provision ally +play doh +plasmo dium +piñ atas +philosop hy +paulsmith design +pagli acci +om gggggg +nz dusd +nil ai +nat sumi +napp ed +my ride +mud guards +mass aman +manek agandhi +lush ness +lor rain +lor an +log anc +kno blauch +kim m +kad hal +je thawks +issu ers +intimi dator +hot stuff +har ron +hai les +habitat forhumanity +h le +gill i +ge of +finneg ans +embed ded +el er +dic amba +d mcc +con dell +chine y +chan dos +chan cey +cdn media +bu ge +bre uil +big bird +argu ello +antimicrobial resistance +anti go +ad lon +ac amp +! [ +ðŁĴĽðŁĴļ ðŁĴĻ +ðŁıĥ ðŁı»âĢįâĻĢï¸ı +çīĪ æ·±å¤ľãģ®çľŁåī£ãģĬçµµæııãģį +ç ± +ঠ° +اÙĦ Ùĩ +à ° +yo gag +xl vi +wayne twp +wap da +ven ic +van guardia +vam ped +tradition alist +ti ina +sussex wildlife +sun splash +som aiya +sin den +se sar +se ha +sco bee +sar od +sal ima +sachinten dulkar +road safetyweek +redefin ing +qu abbin +pri ory +pier i +pier head +pennsau ken +pau land +par v +off shoring +nag as +n all +mut lu +mu chi +moldav ite +mid as +mey ero +mention perfection +mat amoros +magic leap +lush ltd +lu li +le vo +le aper +laker nation +kra k +kevic lifton +kel t +jurassic world +john carter +jarim atti +jarimatti wrc +ity now +insta style +im plausible +ide alized +hand son +ha shes +ge minic +gameofthrones finale +ga eta +franken weenie +fl ou +fir angi +film news +f bic +ent ices +earth week +dut chie +dino zzo +dh aliwal +dev ina +culche th +clou seau +chas ingthe +ca pos +bush walking +ben intendi +arlington natl +ar pa +ar beloa +and ym +amy leeev +ak ini +af terel +aard var +[ ' +ðŁ¦Ħ ðŁ¦Ħ +á ĸ +à¹Ģà¸ Ĺ +zayto ven +zain ab +yo weri +van fleet +ukem plaw +uk butterflies +trige minal +ti thi +ti pico +the juan +the beachboys +speake th +smallbusiness week +selec tor +sauber f +sare made +sam er +ress ata +recl ined +rac ist +pro yuvraaj +pre serve +p outs +op lay +olu mide +o tran +navy daily +national school +n oooooooo +mu sonda +mu dge +mar cellu +lon gi +lic on +le aching +kin deren +ke sq +k be +jun ger +jeff ry +je zza +je para +james blake +jam aa +ja ago +immedi acy +herd fb +gun slingers +gore tzka +ghost signs +gav roche +gart land +gar mo +full body +film music +fi fi +ev f +e gal +e banks +e ady +di fe +dfw traffic +dar nley +chou dhry +bridge view +brick layer +box en +blast off +ba oli +atal unya +ang ood +ain yc +abscon ding +ðŁĽ łï¸ı +ðŁ¥ ĸ +ðŁ¤ ¥ +íĭ ´ +à¹ĢภĬ +ÑĢоÑģ Ñģ +wak ati +vy pe +vene z +ve ia +uh in +ty rosine +tu as +tro tman +tore ba +tol ka +tim and +tig ny +thig pen +tel lem +tas krabbit +tarong azoo +tag uchi +swedi shousem +swedishousem fia +stop brexit +sr hs +sped als +sno res +ske ena +sch ange +sal leh +ru airi +rouss os +rodney atkins +q west +people pets +ori zon +obstetr ician +nwob hm +muzaffar pur +mrdavid haye +mob sters +mo fongo +mir cea +mer y +meang irl +math letics +mac neill +kron er +kill this +kai greene +ju dit +indi sci +horse play +help the +haz ar +gw d +grave side +gram atik +gonna hate +gas olina +fre search +fair ings +fad er +every simpson +est eli +est ela +ellic ott +disney junior +dirty bird +dheer aj +de grades +cu dahy +crimin alized +coren tin +con k +con forms +col den +cloud scape +clam or +ckin ney +ch evening +bra zing +bluenote records +bat tuta +baq ir +bang erz +bal der +austin town +as inghe +al endar +@ ... +ðŁĺĤ ðŁĺĭ +ðŁijı ðŁı¿ +wild card +we broot +vand aag +tor ode +tom ie +thevamp stristan +thel auren +tan jore +syn ching +stu mpf +son no +sas sari +sall natural +ron o +rethym no +repleni shes +raig ad +pub media +port sea +paper weights +p day +or ton +oni stas +ol our +obey giant +ni it +mucos al +mu dug +mi tho +marcal mond +lit en +lec c +khal eel +juli ane +ji bs +intric acy +i han +happy dog +h vs +greg pak +good nite +gn omeo +glam med +gen erics +food coma +fie sta +far nam +er ra +empres as +e met +drud gery +doit for +disembar king +did entity +chloro form +cebu ano +catt elan +car ini +car audio +can er +bul ilit +bol lo +bidad ari +bi os +bell er +ash dlmundo +ari da +ame ba +ab iz +ab im +å´ İ +yur il +wen ig +we gen +walk with +wal le +wait akere +ve za +ut m +trainst ation +tou raine +thre shers +the celtic +than s +ter raz +stephen mulhern +start list +solidari dad +shepherd stown +sfoster nyc +se spn +say le +sau sag +sal afi +rock thered +riks bank +regre ssing +r anna +push cart +play it +pack rat +pac west +orang erie +optimi stically +omis sing +nikon europe +natur alism +nan ton +mosth and +mccar rick +lik ening +lar sen +l anna +kwe ku +ker bal +kan chan +just ino +jor dand +is no +int n +imper ing +i fam +home track +haver town +green live +fron tex +fizz les +fir stin +do ce +demetri os +dave grohl +coven an +clerk ship +chrisvan hollen +buster keaton +bri den +ban fi +aw ful +ach rafieh +ðŁĺį ðŁĺļ +ìĹIJ ìĿ´ìĬ¤ +yl g +ye mpire +wire work +wbal tv +wan ed +wan e +vegetarian week +ur h +tre mendo +trans gendered +tide as +thro ad +tar ly +st thomas +sn bc +shi bain +shadowhunter schat +se ty +schwar tzel +sch u +sch ls +scat man +saf in +sab y +rubi ks +ro whouse +rip cord +rem itting +reformat ory +re ise +ramad ank +pren up +photomani pulation +opel ousas +mill street +merr itt +me tin +man teo +latitude fest +kel sall +jummamu barak +jer ked +jen nas +jabberwo cky +ja ins +j hc +ij e +ham by +grou pers +gon za +gl one +fre eu +fe int +f book +exchang ela +evic ting +en ery +en counter +dy ersburg +dri k +dis band +cur bed +conge stive +bro th +bier zo +atem ple +asil va +ap ig +alder men +al ye +aby ne +ðŁĴĻ ðŁĺĺ +ðŁijĩ ðŁı¿ +رÙĬ ÙĨ +yalo va +x large +wr ld +wizz air +war ley +vote conservative +visual kei +ut arlington +united sportscar +uncler ush +un couth +twee ples +thi splace +tad ao +ster oline +ss ays +slu t +scrn ch +sci oscia +ro thenburg +rid wan +qu ai +play day +pic atnoon +ph rma +pen et +or mer +nascar throwback +nar dw +mong kok +minic ab +megach urch +master minded +livefor music +lauren pope +kellyand michael +jay nes +ip aul +interven tionist +icab nf +ho ggs +hiday at +heart gold +harri ett +hand crafted +girl slikeus +ge ty +gastro pod +gall icabnf +fu quay +er red +elo dge +eg mond +def ile +day sleft +dat at +cre scents +coy m +columbi ans +cent eno +car acha +bur khal +br rrrrr +bf goodrich +beauty fromitaly +ban dol +antmanandthe wasp +ag os +ab han +ðŁĸķ ðŁı» +ðŁĵº @ +è ¯ +âĺĶï¸ı âĺĶï¸ı +z army +woking fc +wink worth +we wontstop +watt ack +vfl wolfsburg +twist cone +tusk en +trap door +tha ana +swag g +sti ffs +speed ily +speak ing +sak is +ro hm +red pill +ra um +r sac +r pf +pu f +per fs +over confident +ning ton +nin comp +netflix india +nas u +mura bi +monoli ths +mon to +mo hn +mi zen +map monday +man gement +le derman +kear sarge +kam ps +jam ir +in ni +hun di +hon tiveros +hetero sexuality +guid i +gor ga +gol fuk +godzill amovie +gavin free +gateway pundit +free zing +finola hughes +fer um +explo rey +ess sss +el ft +ec ca +don agh +del arosa +defaul ted +de fac +buil tto +bu sines +brown out +blue jacket +black house +ber nies +ar ango +aqu af +anti gens +al pin +ak agi +absol ve +aben omics +ab dalla +ðŁıĨðŁıĨðŁıĨðŁıĨ ðŁıĨðŁıĨðŁıĨðŁıĨ +ãĥ³ãĥ ī +wyn ne +whomademy clothes +westworld hbo +we ct +wakaflock absm +wa a +w cac +vie jas +u hb +ti ri +ten afly +spy ker +slu gged +san frecce +sam eness +s zu +s ition +ro my +rat chaburi +ram bin +rach et +pul led +prote ase +po temkin +photo synthetic +pal imp +nr tnews +non tario +net worth +mo dica +me withoutyou +manekagandhi bjp +li ph +ler and +l sw +kryp tonian +key tnc +jor ma +jobless ness +ir reconcil +hin shaw +fleish man +event management +es bjerg +equal ising +easter ns +du bia +discu ssant +colai ste +clec linic +choice scifit +boot leg +biltmore estate +be eco +bark ada +ar ou +al aki +akim bo +ad omin +_ (: +wren n +world tbday +woo ding +winter park +u mana +twel vyy +to phat +tdam erit +t jp +stra in +so is +sel iger +sal in +reli ent +refu ting +pur ch +pu rex +pre teens +poly chro +pet sathome +oo dle +olivi ach +nistel rooy +mul lane +mon tini +moham ud +mir u +medic alassn +mcham mer +mc cly +man mohan +linke ddata +leth ality +legal news +kwi atkowski +kir ks +kim ye +kerry jane +k Åį +jor ger +jk rowling +j muk +iri dium +intersec ts +inlan dempire +infl icts +humanright s +hr k +head masters +harris jofficial +ha sak +gre ased +grass fire +grain free +gonz ag +gian luigi +future offood +fri ende +fran ch +for mas +fem icide +fa wn +err orist +encro ached +ea z +dom an +defi ance +compos itor +clar kes +chan yeol +car line +bre ss +blablac ar +bl dg +beat en +bam bang +aquari um +amer medicalassn +alge meiner +al gé +after words +ach ile +ac ic +zero ing +zay at +whit te +wb sc +tyrone gaalive +the source +strip tease +singh e +si i +shu sterman +shay carl +sa thi +reni shaw +re tto +rack mount +q rl +pray formh +pos ites +pork pie +phoe bus +pav lovic +ozz fest +out sized +orom octo +notin this +neu berger +mun k +mississipp ian +meg acity +mat um +masculin ities +le vens +la sko +kl ick +kirk cud +kar men +k auto +jodre ll +j michael +it showbiz +independ ant +incar n +ic v +hondac enter +handsom eness +guru official +gott lieb +gold member +go west +fron trow +fle isher +far uk +fabri que +excu se +ef dn +eck man +dalla stown +d hau +cu ed +childrenof syria +ch l +ca sti +bur chett +bu cees +boge yman +bl w +ber inger +belitt ling +bar ti +ay le +av owed +asi mb +art ful +ao ta +ample forth +am eland +ðŁĻĭ âĢįâĻĤï¸ı +ðŁĮ¸ ðŁĮº +ìĬ¤ íĥĢ +åı į +zom ato +yarmul ke +wx yv +wen onah +wang anui +veng aboys +vegan recipes +vau dev +ultr alive +trot sky +than ka +sun deep +summerhall ery +split svilla +sp uri +slo v +scri bbly +sau rashtra +roger moore +rel ented +reinde ers +rand ers +promo tor +pre calculus +power wall +pi ot +phy salis +phy lum +pet ch +peel policemedia +orgre ave +or rell +op ress +ob elix +n ho +mon net +mi yan +maj er +mafi keng +lon abscbn +li mped +la e +kou libaly +knowledg able +ki pping +ki me +just y +jonathan rhysme +jo gged +inter ac +imper iled +hugh hefner +ho soi +han key +finger lakes +fav pic +fart lek +epil ator +enth iran +en ext +effici ents +edinburgh rugby +dog days +defin ing +de simone +de mario +david hogg +da ou +cr z +col mc +co quet +c sea +bol an +blue jackets +bipolar disorder +bhand ara +bbc motd +as are +ari ste +allegori thmic +ah ir +afi q +ðŁĽ ij +ðŁĺı " +ðŁİīðŁİĪ ðŁİĤ +ðŁİģ ðŁİģ +ðŁĮİ . +zi kr +web masters +up show +ton ko +title ix +tim bered +thor naby +te acup +sydney siege +stroo tman +stri py +shol ing +sho lidays +ru ang +roy g +rockab ye +re wind +ram zy +pots damer +polymer clay +poin tuk +photo bomber +philadelphi ans +palae onto +nar anjo +mysti kal +mtn za +mosco u +mo do +mis spellings +min new +mar son +magister ial +mag yar +mafal da +lew drp +lei per +lav ash +la follette +kno kke +kinna ird +ja res +in and +i roc +hos le +hepatitis day +gg r +gather ing +flat woods +ev eline +en close +elek tron +ele x +e ales +drown ings +dr c +dilu ting +dhanan jay +den ner +del ario +deer hoof +ctv news +cms gov +cer c +carin thia +bun ching +bu zan +br ong +bouy gues +bles sup +betsy devos +be here +aú paatleti +axi om +attemp loyee +ati p +assassin ating +alter nacomics +aacpsa wesome +ðŁijĮðŁı¼ # +ðŁ¥ ĵ +âļ Ķ +ª ¨ +witt mann +will erton +west land +tn rs +than ior +terri o +ta hira +swisso tel +swer ve +sw ir +subhash ree +stro ther +stain er +st res +sn cc +sli b +sk orea +sil kie +san toni +red wing +re packing +rabin dra +quar rying +ps ico +proto col +pre yed +pin kri +ny ac +nether world +ner ys +ne ca +monclo va +mau ger +mal functions +makh doom +ma him +kud la +kaz ee +journ alist +jing ling +jeal ou +jack daws +itson ly +invigil ator +insecure hbo +hugg able +hans berry +h nurburgring +gy ne +gun dogan +gour lay +gods word +gi al +gerry adam +geek ery +gay lord +fun fest +four fold +fore gone +focu sses +flor ham +fl ict +east cote +e ska +devo ir +def i +def alco +dar on +dam pers +cl ace +cham akh +bos na +boo kert +be sharam +bat roun +b eller +att il +asse tto +antin ori +animal art +anc inema +alien day +." ( +åIJ ī +âĶ ĵ +wen ner +weather by +v sr +tur nitin +the gop +tai bbi +so ddy +si mak +si busi +schi avo +samp son +ro ky +relapse records +r bn +q ais +pul py +pin ce +pil ani +phosphor ylation +perig ord +pa ano +nor quay +nkc schools +nh sc +movi star +mon ge +min ie +micro sco +mer lins +medi anews +mangi one +mand rill +ma demo +m kr +llanish en +lec tio +la sher +kre uz +kho on +jon ois +jar no +jamie bower +injec tion +ho sie +ho die +hig son +hi sto +happiness day +gold thwait +gi jinka +gen c +fire star +fin ovate +es at +encephal omyelitis +dy or +discover yed +dibru garh +de souza +de aring +dat alo +commit te +comedy bangbang +chu ppah +chan gi +cactu ses +broad us +boyce avenue +bhi ma +based learning +ay aki +as key +art scenter +apocaly pto +amer acad +ac ce +ab nett +ðŁĴŁ ðŁĴŁ +é ¡ +Ø´ ÙĨ +x rs +wa vered +vi gnes +ve ering +vault festival +vaul ters +vant age +unity assetstore +triti um +tri gla +to kara +terri bly +teamgod vek +tapp en +surreal art +ston ie +so cent +sm sc +sin spire +sidd hant +shiva ji +shan mugam +sexual violence +see us +satchat wc +sarab hai +ru men +rach na +pot n +parmen tier +on stad +nyc ballet +nico lear +mun ter +mon ate +mobile gaming +milk tea +mc memorialcup +mc dormand +mark wahlberg +li muru +ko ker +kirri billi +kh da +juli es +jig gs +jarls berg +jag ran +it support +insi eme +hy i +humanitarian day +houseoff raser +hor bury +hom ie +hilli ard +gur uk +gum shoe +gladi us +g fw +fl and +fast codesign +entertainment news +donate blood +desp ues +de wing +daz ed +da pet +cri sco +cee fax +car mo +buffo ons +bletch leypark +bell shill +be ssa +be mel +bbc glos +bag shot +aw rites +autom ates +aug ment +amate uri +a iche +ðŁĺį âĻ¥ï¸ı +ðŁĺĤðŁĺĤ âĿ¤ï¸ı +ðŁ¤· ðŁı»âĢįâĻĤï¸ı +íķ´ ìļĶ +z j +yaqu ina +wiki art +whit er +whis ks +vuvu zela +van ities +tyre ke +the dar +tam ura +suppor tall +star rcade +stan ek +skam france +shiv ay +shi ed +sa chiko +rural health +rim pac +real jeff +ranveer brar +pul u +proudtobeafan of +play book +o sullivan +numb ed +nostro mo +nor rell +nas anewh +nasanewh orizons +naco tweets +naaper usur +mo hua +mc whorter +mc cants +ma ung +ls st +lmfa ooooooo +life sci +le stari +le bon +lau de +la gav +l rv +katsu ya +inund ation +internationaldayof yoga +incorri gible +impregn ate +impe y +il icious +horo vitz +home ofthe +hol beach +hirsu te +gra dle +glaxosmith kline +giorgio armani +fuji xt +fro sch +febru ary +everysimpson sever +est ad +ea ve +e isa +du rocher +du er +dil ley +ddot dc +day at +charlam agne +bob saget +billiejean king +beau sallnatural +be uk +bbc sp +aspe aks +anneli ese +ðŁĻĮ ðŁĶ¥ +ðŁİī ðŁĻĮ +ðŁİ ĸ +yon ah +yo jna +yash want +x ula +wo sa +wing less +wi zzy +vennel akishore +usas oftball +under achievers +tro gir +the journal +ter nate +tb x +super position +straightoutt acompton +stein le +sr na +south bridge +smallyoutuber army +sm be +simon mayo +sie mian +sentiment ality +run yan +pro chain +pot ch +pine tree +pa che +oh sas +ober ts +nipp on +nh n +mil nes +mehl dau +medic in +mc shay +maje wski +liannela havas +las ry +la gom +karachi kings +jun agadh +ju kka +jas o +j of +high tech +hear ttour +grey hawk +green sleeves +go hounds +get money +gee zy +fonten ot +flag bearer +est our +e wer +dor é +disillusion ment +de clin +craco via +con lin +clai rol +cine matics +char pen +c shl +bi pedal +bha jan +bere an +ber ghe +bent ong +audi q +allstar weekend +algonquin colleg +ad our +acupunc turist +acqu its +ac ast +? '' ++ % +! ðŁĴ¯ +ðŁĴĹ ðŁĴķ +ðŁĴķ âĿ¤ +ðŁijıðŁijı ðŁijıðŁijıðŁijıðŁijı +ðŁ§ ¬ +æ Į +zab ka +ware heim +wad desdon +val aafshar +un fashionable +toll cross +tizi ano +the kla +thai airways +star talk +special k +so kc +sivak or +ru ched +ron killings +ron coni +renmin bi +relent less +regal films +ramblers gb +px g +ps yop +po tra +octo bre +nikki glaser +nightri ses +ni pa +news ers +nee pawa +nct zens +nationaldrink wineday +national service +nap ed +n bam +my body +mor oso +mo su +mine iro +mdc pss +may te +marsh y +mar ya +mani fen +malay ali +mag ruder +lauren gottlieb +ku bu +ku ang +keto genic +kerryjane ellis +kemp f +inter linked +i faw +i asp +hosp s +honor is +healthis wealth +ham asaki +gri sha +gre ste +gn itive +gais ano +furi ends +fem to +fal c +ero space +em pa +ef fusion +dj am +dis orienting +delici as +cringe worthy +cordi als +commun ities +col ucci +co ble +cloakand dagger +catal un +can ai +bur ges +brill antes +bode ans +ber nou +bally bunion +atl super +app ia +and han +al awi +air speed +ab ang +a ot +... ðŁijĢ +) ..... +ðŁijĮðŁı» ðŁijĮðŁı»ðŁijĮðŁı» +ðŁ¤Ķ ? +ðŁ¤£ ) +ëī´ìĿ´ ìĬ¤íĬ¸ +âĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸ ĤâĸĤâĸĤâĸĤâĸ +y up +wwe braywyatt +ver ite +uta pri +underpin nings +un mounted +un approved +umass amherst +u lukaku +trail cam +thun ger +thu raya +theart stack +the space +stream team +stratfor duponavon +sta stny +sp ts +soo oon +snack time +shep shed +sas son +sand bank +sab miller +sa char +romel ulukaku +ritu par +rhe l +rex ona +resent ful +receptac les +rece ssions +rav nica +radi ob +press box +portu gese +pen alize +pear sons +park jihoon +nou wen +nic las +neuro modulation +naz i +men n +magdal en +ma sur +loz enges +l pie +kut less +jungy onghwa +jim ena +jan usz +hex acopter +han zi +gat ley +fri gg +f xi +exclu sionary +engar cia +drum mond +dim mak +cool sculpting +con cent +cl wb +chat roulette +centr is +bull ough +body powerexpo +bertol t +b live +auber jonois +are yes +akl transport +ac chio +ab end +.. !!!! +za hawi +z nick +ym t +y rt +wout ers +wo ven +wind surfer +wgtn phoenixfc +we suppor +wad ley +w sop +vandy football +ust angs +ty lo +tu lan +tor ic +the ye +super loud +stu cker +sk ick +shibain u +sheff trees +sh abad +semi precious +seduc es +saf aree +rw by +roche ment +red house +red beard +re sen +rayu du +rat an +rapp rochement +pure gym +pic tu +phil pot +ott en +onthe edge +ne wel +milit aries +mhu ire +me son +mci vor +less ened +kit to +keep moving +jo ye +jerobo am +je tter +hoo sick +homoe opathy +ho pa +harsh ness +happ il +fren chs +floun ders +fen nelly +facilities management +fa ired +elo isa +ek du +dj d +diss enters +delux e +de classify +de ac +daw ber +dav itt +cru dup +confor ama +competen cia +circum stantial +cat sand +bun di +bt x +bar clays +ay ame +ar lie +am soil +ado ts +aber tillery +. ?? +ðŁİ ĸï¸ı +âļ½ï¸ıâļ½ï¸ı âļ½ï¸ıâļ½ï¸ı +wel z +weather head +w vs +vener acion +tv ds +transm its +tom keene +the observer +stand together +sojour ners +snar ky +silver sun +si ssies +sci pio +schut te +rmt union +re ik +pro camps +pride to +plan ahead +pixi elott +owen thal +ome tal +no cks +music box +min esq +mex i +mcqu illan +mar teen +man gga +land man +kpop snaps +kang as +kam merer +jind abyne +j jk +j hah +inter dimensional +inter changes +hull fcofficial +hu sson +hein richs +ha aland +h way +gerryadam ssf +georgemason u +gal ert +fu ka +flex ors +exo du +esco ffier +ed gel +econ oline +e hime +del ux +deepa wali +de shields +dac orum +cu bat +cu bam +cron us +const ants +con ard +cleveland dotcom +child care +char ron +capit oline +c car +bustar hymes +brow nuk +bre lla +bo din +biop olym +beau ford +bare ly +as cari +ar aka +american apparel +ah sn +a strian +.. - +ðŁĺĤ ðŁĺĿ +çļ Ħ +âĻ¡âĻ¡ âĻ¡âĻ¡âĻ¡ +wav y +vermon ters +time frames +thaana aser +sylvester turner +sol vency +si ano +shekhar ravjiani +ser hant +save shefftrees +sar chives +repet itions +rain ing +quinceañ era +profe sor +power on +pollu ter +pl ani +pass é +par cell +palatin ate +op rah +on on +olivi ap +o et +non believers +no jhl +new aygo +n sl +mtvlakpop got +mark ham +leon sis +lake man +ken fig +ke ong +kab ale +jodi stamaria +jen shad +jam bu +j nu +ise tta +irr fank +ic p +i wu +huis genoot +home is +gr anta +gal livan +ga iter +fu med +fried le +flu ffer +fi ps +feu ding +epic ally +eh x +di elle +cle on +ci one +cer mak +cast ate +cann avale +cambr ils +cal ley +c th +boo oo +bi focals +bha u +bent grass +barnar do +barn burner +at os +as scher +ar oh +ðŁĺİ ðŁ¤ĺ +ðŁĵ± ðŁĴ» +ë°ķ ë³´ +æ ¡ +âŀ ³ +whis kered +vidy abalan +ver ity +traf fik +temb isa +team liquid +swe tha +sty led +stomach ache +shor tt +seren ely +seis mology +sec ted +reve aled +receiv ables +re decoration +ra heny +q trs +pp ers +pos its +pneumo coccal +pinkie pie +phoenix open +people mover +pan th +ou ar +or ana +nu evo +ne afl +mon eda +modular ity +moderni sts +mlb tonight +min ers +loo ds +lizziec undy +kov ach +kon tos +kess ler +k for +jo ffre +ishi hara +isaiah mustafa +ho chman +here fordfc +hell gate +haunted mansion +half life +guil den +fly back +e euu +dumb ell +duc al +drop dead +dan ilo +customiz ations +cottag elife +compre ssive +chan an +ch ena +cancer society +bren ton +blogo sphere +ben ford +ban aue +avo y +as inger +as coli +angh arad +alti plano +aaaaaaaa aa +yoshin ori +wid mark +wal kar +vincent kompany +vie tti +tumul t +tru ett +tonight alive +them self +the dogs +th wnd +tantal us +tag team +summer town +sum mere +stu tter +stra fe +sot we +sha f +sc roller +samu sic +ru do +ross ington +preemp tively +po chard +pe rel +pat en +om ak +nathank ress +nak as +mur rah +mul u +more x +mer ited +men the +mega world +may as +m guggenheim +lur ked +ken mare +ich ri +home port +hol burne +histor ias +hill brow +hidden gem +gre c +gallifre yan +foto friday +flu vanna +f we +ev asi +equ alize +en rages +emp tor +die ben +den ko +cu tty +co efficients +chak u +cazeno via +by products +bro kering +bi vins +bhar ucha +bar coding +baller ini +aren berg +anas azi +aj yothi +acet ylene +è¡ Į +âľ © +yaf ai +wy omissing +world club +world chocolateday +wh impering +web kit +uc cia +swal ec +sw k +stylist ics +stream liner +snap chatted +ske en +sig na +sh ya +see man +sange et +san ne +sal oni +safety month +s din +ritu alistic +ri fic +resi dente +record z +p dvsa +oddi see +now live +mis calculation +mian wali +megan hilty +mck illop +mcel henney +mayor ga +lovemy team +lang ridge +l jn +kos ice +kooten ay +komo traffic +kim cattrall +jimmy butler +jay y +italy magazine +instit ut +inau sp +hel ias +hasak ah +halla bol +gr illi +gabri els +fur ball +fran e +fo ti +fl sa +fbr party +fay e +east man +e esti +drizz ling +deci mating +congratul ation +co axing +chin ch +boho style +berry z +ben rector +bar ware +awar ness +awaken ing +am ruta +alle man +al enka +agron omic +( ** +ðĿĹĶ ðĿĹ +ब à¤ļ +zo sia +you decide +wpu gamers +winter set +wil kinson +wiki mania +vs fs +velvel holler +umn proud +tyler florence +to dai +thu izen +ten chi +stun a +shat in +sham schar +shamschar ania +se ele +sc ald +sam bailey +ru iter +rep tom +ra ud +pro veit +plo sive +pilip inas +paul rabil +parisi ans +os waldo +omgits alia +ney ville +new haven +neveren ding +mouth bass +milk maid +mesopotam ian +matth au +madefur you +lon dinium +lead with +kxip v +kokop elli +kne ecap +kis ar +jonathanrhysme yers +janis joplin +indian town +hou ri +hop ton +he mos +gu sted +green mount +grand designs +goss ard +gi all +fur nas +four ze +en gen +ec amp +dynamic duo +delauren tiis +cin ci +chel le +car idad +candle wood +broward schools +br wanda +bir rell +beep beep +beck ton +aver ill +alway son +afric ain +a ask +âģ ± +za jac +wor zel +wood vale +wash er +wall ander +w no +vor tices +vit to +un naturally +u shing +tutu tuesday +toronto life +tor ana +to gas +th ér +testu do +team up +st pats +serrat os +sch oten +santu ario +sa pping +ro eg +queen at +pu rer +prize fighter +pop fest +pocket ful +pc cc +ori hime +o shin +ninten dogs +niel sen +nardw uar +n bak +moul ting +more ish +mary beard +ma vi +lo bbed +kle pper +iron workers +intelligen cer +ice berg +ic lub +hor crux +hee ley +har uto +h tr +got ts +gine tte +fero ze +er isa +efan art +cor a +color fabb +co dered +cleck heaton +cater ing +cal casi +c mmi +bull fight +belle isle +bbc sport +ay uda +arrhyth mias +ani morphs +angel alessandra +an gre +ðŁĮ¶ ï¸ı +âĸ ¹ +will you +weiz en +vul va +vor st +visit zimbabwe +un taxed +un intelligent +tri est +ti bby +the butcher +t pusa +sun rays +step sister +sr ly +slu ggers +sheffield shield +sham ma +rif fo +ric es +relocate revolution +reig nit +rain iest +queen ston +por denone +out é +out ta +ob jets +no co +ni ii +neuro transmitters +nak ak +mu zy +midwive srcm +me shell +may ank +maru thi +luc is +loven otts +loren z +laparo scopy +l ld +keeping people +kakao talk +in ria +i ig +hyundai india +hu sa +happy ending +hand hygiene +ham ner +gro ssi +glutam ate +germani um +farm lands +er mita +ear ning +du ppy +dra ken +doom sday +die guito +df wairport +dekal b +damian mcginty +d zi +cu erockstar +crevas se +chloelu kasiak +chis ago +charlotten c +c pre +bossho g +bla kk +belo tti +bad agry +aver ting +arcol atheatre +$ : +ðŁij ļ +ãħĭãħĭãħĭãħĭ ãħĭãħĭ +âĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸ ĤâĸĤ +yoko yama +xper ts +wend ouree +weaver ville +walk throughs +w mr +tu ke +tro yal +tid worth +tate modern +tal agang +tai ze +sze wski +sutton united +subhashree sotwe +stere olab +spe ared +simon kids +seap lanes +sav illage +sand ell +roman cer +rink u +rang eland +ran jha +ra ylan +public domain +pre ste +po sca +pesc ado +ol mec +offici ali +ny onya +national hugday +muffu letta +morphe tt +mo rel +me gad +makey makey +ko belco +kno ebels +klgand hoda +kil t +john barrowman +jc ps +international airport +himansh kohli +gro e +gou ged +georgi eva +ge te +gar ao +gan ado +fitz gerald +fis ica +essenti alism +er yan +enshr inement +ejec tor +dwan imation +cute dogs +char med +ce in +capitol records +cab u +blo emen +bi monthly +ber gan +bench ley +beau doin +bas sil +bab ay +antiqu ec +all eries +ali ko +# & +! ; +ৠĢ +x cp +women sempowerment +wi right +wheel sup +w pn +vre tta +vic hy +utter most +tho li +th our +ter nan +sublimin ally +spi eler +ser fs +schn app +rü gen +run ge +ru bis +ru bicon +ro bards +qu oll +ox ys +op hir +on health +of india +obliter ates +neuro pathic +n achi +muk tbharat +modi fiers +long shore +lak shad +israel in +in zam +il as +iam don +homes ite +gg b +ge va +g ptw +g lon +ferr at +feel ers +fanta stics +fam u +excruci atingly +el ain +e tri +dir ham +di bba +deploy able +conveni ences +ci fera +catamar ans +camp ina +bundes bank +bu cyrus +brae burn +blue steel +bill ys +bi kram +belvo ir +be aded +band ung +arnol fini +arbu th +ar salan +adi vasi +a open +Ŀ ¼ +ðĿIJ Ģ +Ï Ī +zone quotes +zephyr hills +wam pano +vodafone uk +varun kapoor +v op +v br +thin ker +thi phop +theroo kie +su mm +stick le +sti fel +squ ally +skil let +sc wc +s weekend +rush moor +regre tfully +pl eni +pick pockets +pap ilio +op u +nicek icks +ne hisi +nath ist +nar in +music blog +mr police +mon ette +mo che +mis reading +mick ael +men schen +me res +me please +manifen cours +man olas +mal us +maj ura +ma kwana +lions den +libe rec +lali bela +josh widdicombe +insi pid +hedon ist +hammar by +ha fan +gus set +gott schalk +gianluigi buffon +fy ffe +friday feelings +forest ville +fi jir +exo somes +euro sceptic +et cc +encapsul ating +en circling +en circle +ed ita +dri shti +deduc tive +damp ness +cu ta +craigie burn +cover let +cov campus +cor usc +con found +co oma +chest nut +che ban +car bery +byl sma +broad meadow +bea vers +bal ita +as ÃŃ +april dryan +app end +anthuri um +anit a +anci e +afi f +adv an +ðŁİĪðŁİĪ ðŁİĪðŁİĪ +è Ķ +x pla +workers comp +we at +v th +ultra verse +ul tran +traff ics +ton sil +ter ton +ter sedia +te gna +supp lier +su tch +sound ings +ser g +scre wy +sau ction +sand men +sac ad +red land +ra azi +pun ted +plastic waste +pig gie +phys is +perfor ation +para ñ +oti des +on thill +ol bia +ob es +no ori +nbs morningbreeze +my city +multim illionaire +mor ry +mo yers +mis sd +marni go +man nat +maku hari +ma bon +lin ski +less ening +ler wick +lear t +le ib +lao ag +kim mo +kar lin +kar ch +jag gesh +is op +icec ap +ic ds +i wish +hur dlers +help to +hau ghty +har mattan +grand slam +go sta +gland ular +gill er +gha foor +ge es +gal lic +future tech +fuller ton +fred perry +far rel +fair price +fa rel +e ades +dray cott +don tb +do sent +del aha +de aley +d ile +cre ated +cc news +cather in +bien ville +bel tsville +arizon awildcats +are valo +anishin abe +anari vo +ah rc +abe shinzo +ðŁį¿ ðŁį¿ +ëī´ìĿ´ìĬ¤íĬ¸ w +âī « +z ent +wear iness +venu gopal +vay ner +v q +tr ine +ton yo +to life +teres ita +summ iting +stro jans +sto ther +spor a +spoil age +speci alt +solom un +sn andani +sexualas sault +sen ig +sand ile +sam ani +s doitbetter +ribble sdale +raima sen +quim per +pre te +pra shad +pi atti +pare jas +pa hs +p anni +offro ad +o low +n kurunziza +mol ony +moham madi +miniature painting +matchbox twenty +man ofthe +levit z +lawand order +la burnum +l lyn +ke ttes +justine greening +ju dt +jmuk mrpolice +incur ring +i stan +i ren +house kinok +housat onic +hase be +happy yy +hammer fall +ham ont +guitar world +good body +go sn +f summit +et f +enthr all +em j +can ned +camer at +burkhal ter +bottom line +betsey johnson +ber kel +ben gu +basset law +bare si +art aud +anitaha snandani +alph ons +Ï ĥ +z eck +wire land +visit pembs +vi dar +valtell ina +v ti +train your +the kk +te ign +sto dden +smi dge +sinn fein +scre wattack +sci eng +schur z +ru x +rou l +remy ma +remo test +re awakening +py per +promote gamers +poli an +plough s +pi eper +patty sday +orang ina +on erous +okc fox +ngo c +music month +mid t +malaysia airlines +m enger +lord mayor +kil le +jelly roll +is worthit +ira ge +indian airforce +in tune +ikh wan +iam not +group board +gal eri +g tr +force friday +fiber optic +fast forward +far hadi +eti ka +equ aling +energe tically +e ship +dra kkar +deton ator +dain ik +d ps +cli me +christop er +che mb +chad mmurray +ch ite +cau da +c kel +bu land +breck land +bouton niere +benz ino +ben er +aval or +au tuni +attribu ting +at illa +ask men +aq eel +ap ri +ðŁı ķ +yas uda +wy lam +wa sit +vitali k +vitalik buterin +undÃŃ acomohoy +un iti +uc cs +tre yanasta +tol ong +to self +the c +sussex ccc +spr at +shin kai +sheffhallam uni +ru cks +ro eder +ri kka +r ti +qui ros +profit erole +pony tails +player pic +pitch side +phantasmag oria +pab los +ol dier +nic liffe +ni giri +min im +met zler +mesab oogie +men z +me hul +m wl +loo sed +llew yn +ki at +kei bler +kav in +imagine er +hul led +graham cassidy +gra sps +fro wns +flam beau +dn q +denver comiccon +de gradable +cow lick +corner shop +cornel west +collecti vism +clar as +cau ca +carre tera +car ril +bledis loe +bc am +barneys ny +b ator +au h +antan anarivo +alta vista +ali um +aggies alltheway +ag gers +ðĿIJ Ń +éĥ İ +⾨ ðŁĴĽ +wo erner +tune in +trinity laban +tra ina +therapy dog +thel ake +suble ase +strat ot +soci ol +sel fs +seger strom +rel ine +reb or +rani er +q ayy +pla yed +or la +o dissi +na sib +n sb +n mo +mubad ala +micro climate +micro burst +mercedesbenz uk +maj ella +lu chas +live band +lite speed +lin iers +liev re +lewi stown +lee v +laurel wood +lap wings +ku kui +ku ki +kri pp +kapoor fc +jamesmartin chef +jad in +inthe styleuk +imit ations +hakk asan +h df +follow back +faiz abad +fair child +f pd +elen nox +ei dul +eag ency +devol ver +dess au +day quil +dar ks +d pn +cou lier +cor io +comicbook art +cla use +chi marathon +cfe vs +buck fast +bu bb +bonni emc +bel fa +bar g +baek hyunday +ba best +azure stack +ar amex +aqui legia +angui shed +amas sacre +allot ment +a wor +@ _@ +ðŁĩºðŁĩ ² +ãħłãħłãħłãħł ãħł +ÄŁ rul +zaf ira +z aun +vandal ising +v ls +ur tic +un sociable +tu ti +tren cher +tra w +to create +teen ag +teacher toolkit +tan tric +sur yan +super speed +stur rock +stirling shire +stat ton +st ach +so ley +say ur +sau cier +road warriors +ro ci +ring central +property brothers +por os +poland ball +play making +pal vin +paintin goftheday +op tically +only thebest +next wave +nc bs +middle field +mat cher +man love +leonard cohen +lah ren +kne aded +kak enews +k undera +jodrell bank +j jackson +hypoten sion +hor ween +ho teles +herop anti +herald sunsport +guardra ils +gu ren +ger rans +em rata +em elia +du ffey +drivel ine +disc ards +dic ho +den izen +daylightsaving time +co iffe +cav olleyball +cal ved +bring erof +at aste +art sci +ani xon +ak d +agit ators +ac clamation +ìŀ ¬ë² +~ ( +wz tv +wr c +wordof theweek +war iow +wariow are +walla pop +vit ello +usp id +un dial +u eg +ty cho +trade offs +tele marketers +sun apee +stephaniej block +siddhar th +satisf yingly +sab ine +s story +rodi mus +re villa +re valuation +re her +pru eba +play ability +plac ido +neg ates +naxos records +nat ori +my family +mountain day +ml ml +mar va +macmillan coffee +lu sion +lely stad +kor b +kirch hoff +kc g +joy ner +jar at +jam bon +hes ston +grand standing +gotit free +gal ati +gain ful +g lynn +fi end +eter nals +e avis +drinking water +dran kin +dean winchester +dam ac +d workin +clio awards +chem all +car ver +capsu le +budd le +bru st +bra ven +bli x +bigbang fair +bab a +at ol +ap am +amu lets +alexand rite +al vey +ag ran +ac com +a different +< \ +ðŁĺı âĿ¤ï¸ı +ðŁĹ º +ðŁ¥ IJ +éĺ ² +é ad +zul lo +zeno bia +young dolph +ww ba +wr ittle +wp fd +webcomic chat +w ml +vec chia +un affiliated +tu mis +timm erman +teeling whiskey +team nosleep +ta urine +t magazine +spo st +sock council +sher burne +she imer +semi pro +santur ce +san chi +rul lah +pw ds +product management +pre fixes +phil o +nur selife +nt midlands +nike fuel +nascar on +nam al +mustang pride +mis managed +memor ie +ma say +ma den +llangef ni +legal ising +le moine +ko ike +kear n +kam ari +ju stu +joshu ak +it ap +inter serve +integr ale +ic key +i ex +hou ruk +har peth +har di +green infrastructure +goo finess +gand i +gam gee +fent ress +fact book +esk il +enh ancers +el eni +dun elm +dun boyne +direc te +delhi metro +del fina +com mbank +camof lauge +ca reca +book post +back up +art contemporain +agame of +ðŁĺı ) +ðŁĴļ ðŁįĢ +ðŁĴª ðŁı¿ +ðĿĹ² ðĿĹ +ze ek +z emo +yu an +youth soccer +yeg bike +x bond +woodro ffe +water safety +watch party +wakati pu +vv vvvv +vitri fied +trache al +stre es +stay lifted +sky ped +scare lli +reyn olds +ram navami +powder horn +pin wheels +pi qué +pearldrum corp +op hilia +ol loc +neck lines +n so +mun ya +mono pod +mo ster +megar an +lu si +look good +libra irie +latin os +lang tree +lal ah +la iney +kre jci +kh ary +jointe am +jo enichols +jason wu +hou l +hir anandani +hillsborough sch +happy feet +guar antor +gro sz +g sas +fresco s +fl ann +feudal ism +er mey +eich mann +eco tricity +earl swood +dog z +di mash +defaul ting +dc as +dal ma +cr ts +cor dis +comer ford +clu mping +clu blondon +chal u +ch ira +brocken hurst +bot to +bir thr +believe inthe +ban hart +b iller +az im +ay ako +as som +ar bi +andre ali +an asu +alver nia +aby dos +>>>> >>>>>> +ðŁĶ į +âĿ¤ï¸ı ðŁĺĬ +Ì · +yt ff +yi fei +woman shour +v bm +u ld +tuber ose +tory lanez +ti ums +tear fully +tal ky +sur plice +suk anya +spi on +space opera +snow plows +skip bayless +sk night +se sa +schick en +sar on +sand on +safter dark +s enders +ru stan +royg biv +ro tem +riot into +quirk iness +per vy +pat ay +par oles +par mer +omur ice +officiale fcc +national walking +nar ra +moon man +mo dders +megam ind +mcgru der +mar len +main ichi +m sta +love trump +le wes +kur anda +ku cko +kie sel +kair o +judge dredd +john s +jessicam au +je eva +j ado +ir regularly +ico sa +hu ra +hoste tler +gunder sen +g isa +fly saa +fin et +face painting +fa intest +enal davies +ek wo +damo daran +croque mb +cri enaldavies +ching lish +cab arca +bridg man +bon trager +at tia +ash tan +ar aku +ancien thistory +am my +ðŁĺľ ðŁĺį +ðŁĩ± ðŁĩº +íĪ ¬ë +é ¤ +Ùħ ØŃÙħد +y acob +working dog +weare x +vo ig +vinny guadagnino +vi vas +uro pean +the mandymoore +the human +the goat +the dcuniverse +tandu ay +stanford fball +someonet ell +sof tg +smo ore +sit aram +shar mony +sf x +sen al +seeyour city +schu ll +ricar doro +re xx +rail er +r caniglia +q opa +pis ser +pawn stars +pasi apac +ong p +o yo +nulli fied +non proliferation +mer rin +med ora +mari as +manife stos +man vel +majer us +ksh mr +khan h +ke unsuk +kat ery +k oury +jhun jhun +jay ma +hydraul ic +hr b +hend rix +gra bar +gn omon +globalgoal sun +german o +gab bie +g lim +fu le +food science +fl ensburg +fissu res +fire bug +die back +de wy +d illian +cor son +chee ba +cap tured +c sj +bush ings +bri ster +boston police +bole da +big sur +bel ah +bag ani +aver i +ated r +afilm fest +ab dur +ab assett +!! .... +ðŁĺ« ðŁĺŃ +ì§Ģ ì§Ħ +âĸĶâĸĶ âĸĶâĸĶ +د Ø© +® ï¸ı +y lon +wy p +white mud +water fall +volvo trucksuk +vo glio +vill alba +ven o +un sanctioned +u fi +u angel +the frog +swim mer +stage play +sport shall +sp angle +sie bel +shop ify +shar nab +selfa wareness +see is +sch ind +rs spirit +roswell park +rippaul walker +ren z +recor k +raw le +pray ing +poly amide +phar ah +par ation +p xp +on dra +oji bway +nov ick +mun ck +moy se +misty onpointe +militi amen +mikk eller +megac ities +mb am +ma sco +lu bb +long day +log mein +let seat +laur ale +la wang +kom bu +kill monger +kellie pickler +kee ble +kawas aki +k urs +k orie +james x +indra jith +imperson ated +hl ntv +han jin +goo drem +go win +gla sson +eve rette +ev ast +et is +ele fant +ela ing +effe minate +e ines +e bert +duck and +drum condra +dr ar +dif ford +den ison +deflec tor +cul to +creative commons +cityof tampa +bu chen +broad wood +brew ton +bra yan +bla x +bell ville +base men +at ops +ar gi +amstel veen +am ts +ðŁij¸ ðŁı¾ +ðŁİ¶ðŁİ¶ ðŁİ¶ðŁİ¶ +ðŁĮ´ ðŁĮĬ +⾨ ðŁĴľ +young victheatre +yal emed +vet triano +v us +undere mployed +tom blin +to love +tld sblearns +the kapilsharmashow +stu ta +sti ft +ste ss +ste at +stand by +splat t +spe ace +social studies +so est +snow y +skul ly +sing ling +sin india +sin h +sig er +sien kiewicz +sau mur +saf o +roam ing +ro das +ric er +rath bun +pt fc +pra th +patrimon io +our club +ot bc +onthisdatein hiphop +nove lette +nish at +nicol ls +ne ph +morg fair +man na +lo vel +li ggins +kar abo +k gosi +jolly llb +ignor amus +ie w +icu isine +ichi moku +ib k +iam queenlatifah +heine man +happ is +green wave +gram een +gib bet +friedrich shafen +free space +flu bber +fal ah +empath ise +eas ing +e toro +document arian +dar kie +cheek ily +chate let +chan o +caro lee +bur rus +blo tted +bla kel +billy ray +bar bizon +az man +avan zo +au ld +allan hawco +alc dsb +alac tic +ajitpa ifcc +ðŁĺĿ ðŁĺĤ +ðŁĺĪ ðŁĴ¯ +ë± Ģ +าภ§ +yu lee +wag tails +vor arlberg +venkatesh films +uz air +ur um +ul b +uk wu +trom so +trau matic +tran o +thor gan +thi splace +simon on +roo z +rand alls +pul sat +prepa rer +pope vil +pasi ones +partic le +par ana +pang kor +out building +old strathcona +od ong +mean ness +mat an +marclam onthill +mal abri +lorraine pascale +loo d +ku pang +jis ub +jessicamau boy +hus by +his cox +hepatitis c +he mer +hass ler +gly cine +giovin azzi +geof ilter +fi fita +fa shawn +eri des +drag neel +dra zzari +diver timento +di stin +chromo somal +cham s +capac it +cal ero +bu ike +biomime tic +beam sville +bar goed +bal ed +attend re +as cap +angel sof +ali zia +alex us +ag or +ðŁĺĤ ! +ر Ùħ +zan gief +wy socki +world skills +whit tier +walkin shaw +w ase +ve tta +van avond +uof oklahoma +tt ro +trip ty +tren italia +tra e +tom jones +the lead +tag ov +su di +smur der +sm sf +sinha bjp +single life +shivar ajkumar +sham al +sh ley +sgr ho +sb cs +sal thouse +sa en +ryo ko +ron na +rie sen +rid ere +resolu tion +resol utely +repatri ate +pr girl +pig man +parach inar +om ero +oku da +nad av +mr inal +meth us +materi ally +mani ka +lyric ists +loth brok +li shment +li anna +lack burn +kn app +ker plunk +ke pong +judicial watch +inter diction +inte x +infuri ate +holi daze +ho lein +hey hey +he trick +have an +gov rauner +gen tiles +film fare +fabul ou +elabor ation +dong guan +dieben korn +dc examiner +cv show +culler coats +cot chin +cook house +charpen tier +ch aux +cc ac +cat nap +cas sin +calcu lable +bb clondon +back doors +aller gist +ali fazal +air corps +af fan +- ] +ðŁĩ ¬ +ëŁ ° +â Ł +á ĺ +zi will +ye zidis +twir lers +transpa rently +thelon ely +thegold bergs +thefuture isnow +terce ira +tar kov +so low +snow boarders +sla pp +short land +she ffer +sch eu +sap on +saf fy +red fearn +ra sher +qu ale +pipe ttes +per di +orbital atk +open work +nuest news +nu get +non ie +mobili zes +miss jess +marinabay sands +mar loes +liven ed +lady land +knox villeraces +kara jan +kam pot +k tul +im possibilities +if sec +ici ous +ho bday +ha sting +great music +gal er +fur ter +fir ston +em itters +el ings +dun ham +devo xx +dawson screek +dari en +crew members +ch cs +c anne +boye tte +bot anists +big apple +b pride +ashford castle +anticli mactic +ano is +andro ll +anaco sti +am dav +allyouneedise cuador +abbots bury +ðŁĺį ðŁĺ© +ðŁĺĬ âĺº +ä½ IJ +âĤ¬ , +world hepatitisday +wm ds +wak ar +vp dd +visit virginia +u akron +tu pac +tor mentor +the mercury +tele wis +ta van +su ffic +steel y +squ onk +spru ced +so kol +skill sforlife +scou ps +sch ram +sal emma +sad d +rain coats +raf fia +pop tropica +po poff +pix lr +pernam buco +perempu an +pear lie +par key +over hearing +om pton +oly tic +nov ell +munt jac +mis amis +mi zzi +may tals +mar well +ma zu +lu beron +lil in +larry madowo +kv k +kc j +jer on +ja heim +ing ood +inaugur ationday +ic kie +ic ele +harvard chan +goo sander +gol ub +galvan ic +fle shing +fau ji +farma ajo +essex cricket +end slavery +edmonton esks +dip ty +dayo fre +danny john +classi er +cho k +cavali ere +capital ising +canti ague +can ic +c aging +bur fday +bo ley +bill burr +bic kel +bengal is +be bi +bal uch +bair ro +av raham +arte aga +and ar +amb johnbolton +ali se +ainsle year +ai shat +ðŁķº ðŁı¼ +íĨ ł +п од +whÄģ nau +wah habism +va sey +transit ory +tothe a +the tommy +sze ged +sunderland uk +straw bridge +sti q +starwars battlefront +st less +ssel fie +sp rad +si sd +shi b +sen blumenthal +se ibu +s ji +rp crd +rock starenergy +re activation +rcr racing +rang itoto +prednis one +poly phe +polit i +pier luigi +par scale +orit se +ore tte +new museum +ne shwar +mumb a +mn df +mcken zie +mammam iam +mal awi +leg alaid +lamo the +l stm +ko tb +khul na +kcr g +jing u +impresion ante +immuni zations +i vp +heli oc +guitar lessons +gu le +gra fik +gang lion +free thought +fr ds +fox football +fly hawks +far man +exclusi vo +er dman +el gl +egre mont +do ña +dim prov +delivery man +deek sha +dan patrick +croquemb ouche +contor ted +cer da +canadas nac +camren bicondova +ca inta +bs j +book giveaway +bin ney +bas ch +ban ville +ban kyw +attrac tively +arter io +army allamerican +anachron istic +aljaz skorjanec +al tv +aim ondo +adduc tor +âĻ ¤ +à¸Ńภ¥ +zoo keepers +zi ons +yuk imura +yourstory co +your dream +we ve +war c +w cag +ur dang +un nies +tour london +til ia +te dium +sudo crem +stick ley +snet terton +savit ri +sav eng +sathy am +redhot chilipeppers +reci eves +quicken loans +ple ural +pic ross +pe zzo +painte dby +monste renergy +mccau ghey +mal is +majum der +maggi enyt +luxury car +love island +le xx +kun un +kh it +kf bk +kettle well +kc streetcar +jen kyns +james spader +jamba juice +j law +irr ation +ine sti +imp eller +i ops +heroes inlife +her nehill +harbor view +guar ino +golden berg +ga itan +foot joy +flori dap +fan signing +expre ssi +el let +ei x +di des +der idder +cy clec +com ely +bush official +bung led +bun nell +bring thenoise +blue devil +biodiversity day +bi bbs +be free +barbour sville +avo ter +arr ative +arch bold +annamari aisland +afri yie +adidas za +achio te +aaron tveit +ðŁıĨ ! +ãĤ³ ãĤ¹ãĥĹ +า า +zi v +yq r +wit sel +whe al +wf sbnews +we there +vre de +ut ile +u ap +the ech +take overs +success quotes +st pancra +space art +skiv vies +sk atec +sen burg +selfi est +scru ms +sch aap +saf fel +sa win +s girls +s dot +r ino +pol yo +phil starnews +panop ly +ny td +nau tical +n eli +mis spent +milan designweek +low rey +ll g +lal ita +kus u +kabir singh +hunts ville +ho bon +heat onedirection +ha aaaa +gsuite edu +gowar iker +gheor ghe +gend ry +flavour some +eno te +emancip ated +egyp te +egg plants +ear lobes +del co +deade ye +de ee +cy le +cree per +chit toor +carbon ear +bur leigh +beach thursday +back benchers +auberg ines +asci ences +art land +ainsleyear hardt +aggreg ating +ac ter +abc worldnews +ðŁļ Ĭ +âĽ¹ ï¸ı +za ev +ys b +y ii +west malle +tow le +te hu +ta quito +t wines +success stories +shay ari +samo ans +romag noli +rock ridge +quote oftheweek +pre market +pol litt +pine town +patriot pride +p ingo +or azio +o ab +new steam +naz ir +nan ta +marsupi als +man ity +mal lock +mad vertiser +lu que +lau fer +jer os +jac enorman +is over +i ir +habi ba +graf ton +glee onfox +ghe alth +ger on +gender less +gau s +fan pic +ero driguez +died rich +dew drop +demo tiv +de mint +d cl +corey taylor +chees ing +blan es +be sty +banque trecords +ay oun +audio drama +ao tea +ai leen +ahlu wali +ðŁĺĤ ðŁĻĪ +е Ñģ +x ai +wy si +wai fus +victi mised +urban e +ul ama +tro mp +thu raj +tf sa +tele kinetic +taj hotels +syno psys +sy stolic +swag gie +supplic ations +su uu +sk j +se ph +sce wc +sa dist +runner s +q ine +pur dy +pur dey +prosecu torial +prophet muhammad +perseve res +oren tina +offic ine +muham ad +morethan just +moo ted +mond prop +med ved +man akin +ly onnais +lum pia +lo gh +lo ane +lef ty +le aux +lan xess +kry sty +kap al +kam mer +jon mitchell +jet setter +ini ana +in asia +her as +helio trope +hedwi gon +hat chett +han o +gri erson +food show +ely fe +e erascal +dist inguishable +degra zia +dan ecook +cur le +cre mer +consu ela +che ika +ch awx +bu fv +briand awkins +blac chyna +bi directional +bi dar +ber rics +bedo ya +be vans +b we +atur al +asi m +app ling +an ot +ale tta +above the +ðŁļĤ ðŁļĤ +ðĿIJĢ ðĿIJ +åĨĻ羣æĴ®ãģ£ ãģ¦ãĤĭ +ච» +y quem +y ig +wester nrly +w anner +up wardly +ts wift +transpen nine +track mania +tom colicchio +th ale +tg l +ter day +sun silk +storm track +solar storm +snow piercer +smo sis +sm su +sadh vi +sab ay +ro del +respec tability +rem parts +relient k +re tool +rain ier +ra so +r sb +pre jean +pok ora +ov on +omega watches +oat ley +o gon +nextgen atp +newhope club +nel e +mus ar +motion graphics +mo gen +min cer +ma int +lore ttal +ko yo +kal bi +k tuu +joan hart +itye vent +itali ane +intimid ates +im pac +ili er +hu k +hot newhiphop +hon ker +grave yards +gil well +ge birds +g sp +frit olay +fin nie +fer nes +ethi o +em rys +ed trilogy +e tribune +de smar +david lammy +ci man +cat lett +c ú +brandon lewis +bill z +bil by +benny benassi +be careful +ball ard +av cavolleyball +ard aw +ar not +al ur +afol abi +aff ton +^ ^) +: Â¥ +! âļ½ï¸ı +ðŁ¥° ðŁ¥° +íĪ¬ ê²Įë +íĪ¬ê²Įë įĶ +à´ ¸ +ÃŃ lia +z ila +yu yu +yler oux +world heritagesite +wild beerco +way an +voice acting +van morrison +utter ance +trumpp ro +trin abraxton +tri bu +treyanasta sio +tre main +ter fel +stur ge +stag ecraft +spy rothe +spring burn +so derberg +so bek +shir u +seman al +sar tori +row sell +ringo starr +proprie tors +pingu in +ph lo +ortho sis +nwin diana +nod away +no ons +nas pers +nand ed +n bal +mc phillips +mb ly +march against +lyn k +lenahe adey +le pp +kath akali +kat an +k las +jerry nadler +jenni es +jenkin town +ianm harding +huddle stone +harb anda +hand bells +hailey bury +gal y +emp at +em tothea +elge use +digital workplace +der mot +dav or +cri der +corr als +chin ook +cer to +cas itas +bo dh +blan ko +b fy +agre e +ãĥ¬ ãĤ¤ +x ur +wool la +white board +wheel s +wel liver +wa ves +w roc +volt meter +vishwak arma +vin cit +vasi lev +ul tural +u dders +tell your +teeth whitening +sirius x +sella field +seem y +seef eld +sch ama +sc s +sar avana +sahar anpur +s ool +rusev bul +ru pani +robic haud +ra wan +r fr +pis d +pen stemon +pauley p +par lement +over ripe +on ley +ol tre +nyy roro +naive ty +n eno +monom oy +micro controllers +mes dames +mae jor +ma zz +ma speth +love myself +lasci vious +la bored +la ber +l wc +krishnamur thy +kar lee +iki gai +iiii iii +i olo +i hab +haters gonnahate +h itta +gr itting +gobi gor +gesundhe it +ge dge +event s +et was +ental health +el van +don eright +do gging +dizz eerascal +diefen baker +de bla +dais uki +d had +cold stone +char nock +cast ellers +car a +caho kia +broad sword +be decked +b williams +av ena +asi ri +arba een +ap ad +al aka +afl don +aci fic +ðŁĺ® ðŁĺ® +ðŁĶ« ðŁĶ«ðŁĶ« +ðŁıĥ ðŁı¼âĢįâĻĢï¸ı +ми ÑĢ +иР¼ +zig bee +yu g +yaal artista +witche so +wg ci +week i +wab asha +w wel +w icking +visit belfast +vis sel +val arie +tur bat +tom are +the my +su mida +stor ie +ss oftware +sra banti +squel ch +spotify uk +site core +si pe +restaurant australia +rein ke +ra zy +ra dd +plat o +phiphi ohara +phi mu +pacific ocean +ou tique +op v +objec tors +nav neet +n dola +mu ggers +metroid vania +mel cher +malla ig +ma kino +lorettal ynn +lo cher +lar ity +laj pat +l fafighting +j dj +hor ikoshi +hel al +gri mez +grat ton +goodwoo dr +git anjali +freer ange +fra id +for gold +fe uro +f ending +eventu ality +entrepre nu +elo te +ear ache +e dom +duc los +diam eters +denomin ated +decat ur +de twiler +com ed +coler ain +clarkcounty sch +cit ad +ci pd +christian son +ce h +boom town +blu esc +bhu ti +bergen field +ber r +bench top +beck ers +bbc goodfood +back flips +ayles bury +av j +aux erre +aur ore +athen ahealth +astronom ia +apo yaalartista +and health +alli s +ai shah +ab amba +ðŁĶ¹ ðŁĶ¹ +ม ม +wr b +wbr cnews +vol ant +us fl +udta punjab +tw cnews +tun ics +to ten +steph ano +st kilda +sn am +sh appy +self pub +saver io +s bi +ru sch +ru fo +re usch +r pm +public service +pric hard +ph ills +pet chey +part sun +par tee +over spend +or ba +open forbusiness +oo ohhh +ohmy gosh +no pal +ngv melbourne +ne era +napolet ana +najam sethi +my self +monk house +mne monics +mc william +maple syrup +london fashionweek +lat v +l all +kil bourne +kelsey redmore +k mox +juxta pose +just say +i speak +hol tnbc +hardeep spuri +har rell +gro ho +gor o +gold finches +gol der +gol conda +gen cies +frommy heart +faof ish +fan ks +fan day +fall league +epitom ize +en ar +easy money +du buffet +dream team +discovery id +de orro +de berry +cycli st +ctr l +cit b +choreo graphic +chak akhan +cc ds +ca ap +c ws +brou illard +bok ki +blu earmy +au lia +army wp +anti och +and music +air ambulance +ad sk +access control +?? # +/ ( +ðŁħ ¿ï¸ı +éķ · +çĦ ¡ +æ ¾ +ãĤ¤ãĥ ī +zcz uk +years strong +wwt slimbridge +watch nz +vir tanen +to ggles +tie res +thrill ing +the wild +the posh +swer ved +st lawrence +sp ellers +solar pv +sharnab urgess +sav va +sand blast +sab ado +rp murphy +robert patrickt +ri ad +ree bie +real love +re saca +r ales +progressi verock +pe sco +parry sound +panam á +ortho gonal +onec up +novi kov +mur tha +mou lana +milk ha +medic ale +mb alaji +lo xton +lo sh +l ène +kin te +ki rov +ke bede +ka stles +jud ici +jane philpott +indi en +if we +hu h +hel me +har ks +ge su +flau tist +fidge t +er rani +e bon +director mbalaji +derren brown +dam age +csi miami +cp v +coqu ille +con ish +bur b +brad meltzer +bra cht +beat riz +as sport +any ones +:::: :::: +ðŁĺĤ âľĮ +æľ ¨ +ãģ£ ãģŁ +z aps +wel chs +w fo +un chain +tr us +the wire +ter al +tantal um +speed sters +snow cat +sn el +sin ta +shaz za +serge j +ser vient +se epage +save on +salvationarmy us +s laney +ro oney +ro ha +re winding +re settle +r uru +q ajar +put nam +pre packaged +om onday +oise au +o ah +nur nberg +nichi jou +nex press +neuro anatomy +ne peta +ne ddy +moon flower +me it +mc beal +mb g +lough rea +lorealparis usa +lit z +lau rea +la key +karapar aask +kal imba +inordin ately +ide en +hade stown +goven der +fu z +fro bots +fau ve +face down +experi ential +ero u +eb v +den mark +dai sey +creep ily +cp mumbaipolice +com pa +co bh +broad sides +boun dary +bo ddington +bfc dublin +beverly uangel +asin ac +ani max +all state +afric anist +ae ons +acol ytes +( ãĢĤ +ðŁĻĨ ðŁı¼ +ðŁĴľ . +ر ÙĬ +waterstone spicc +w baboxing +vin ci +ver rett +vel an +tumn us +ts q +tribe sman +the wiz +the mis +the dubaimall +tatt i +sun corp +stra pline +stam mering +st ks +spinning top +sidd ha +shilpash inde +seung woo +scapego ating +sar n +ryan lewis +ronde bosch +riffo tronic +rad ford +puke kohe +prodig y +positi va +o wari +no taro +ni ajax +mycorrhi zal +mr duncanjames +mil on +megalom aniac +mag ana +lipp in +lady killers +la fitness +kur tzman +ka hs +inciner ated +i pps +hump y +hg vs +gene ws +franklloyd wright +euro dance +epider mis +ellef son +dylan mcdermott +di ame +clam my +chir ped +ceee gebirds +car tel +burk holder +bu de +breckin ridge +bre de +bal dridge +b tt +awe urope +at nam +an ico +ale lection +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +ðŁĶ¥ ðŁıĢ +ðŁĴĢ ðŁĮ¹ +· · +ze ina +war and +vas ser +vapori zed +un roll +tori ko +tom ax +to ple +thejim cornette +te gen +tat ro +sul ting +sud americana +sporting cp +shaw aii +sham wow +sand ridge +rt pig +road safety +ramadank areem +rain goaway +q hd +pomer anians +pin ki +pers ad +per ki +parañ aque +out doing +o sto +nor semen +no ke +no ga +nh sor +net sky +ne sia +my nam +mon ferrato +mix te +mid shipman +metagen omics +mesm er +melis sal +ku va +kno tting +ki jhl +kay afm +k law +jonathan jackson +jes sen +ing ames +ihear tradi +idri ssa +iam lenovo +hij ri +gall stones +gall er +gaine sway +ga shap +g á +forte za +fh fa +eze kiel +espn fc +er b +eat sleep +dhing ra +del po +counter intelligence +core graphy +cooper ators +cons ented +cep r +cdn history +cal zaghe +c tbt +bon ham +bex tor +bergo glio +belly button +be heads +ay anda +atletic omadrid +ar tt +ap tors +anti report +am ics +ahmad is +a style ++ âĢ¦ +к и +wy rm +wt va +wit cher +wi b +whit em +vote austinmahone +van ness +ux ury +un heralded +thapp ens +tham ma +teach in +sukab umi +sub let +stoner rock +spor ti +smet ana +skills gap +sister sof +simon schuster +siden ote +sar wat +sal ade +sag gy +s alling +ric in +ren er +rebel le +ram eez +rac i +r sd +powerrangers movie +play thing +plac ers +per ico +ot ford +noctilu cent +ne ya +mist born +meng al +mcg lin +man holes +madon na +lu bs +lin ce +langu ish +lakshad weep +la pel +ku ipers +kohl schreiber +knigh thawks +keele y +ka os +journe yofficial +jayas uriya +janasen aparty +j gn +hyat tre +humanit arians +hoo ton +hamidmir geo +gree do +gre gate +fur thur +fra ise +follow andretti +fire lands +fi ving +fa ka +expedi ency +elizabeth forma +earth rise +drink responsibly +d Åį +curric ulu +ctb to +crvenaz vezda +car boy +campbell ford +buster posey +bu uuut +bra chial +bally fer +azi muth +as fa +archi v +ar leigh +american topteam +aldu barkads +alan thomas +al los +aa am +a ÅŁ +ðŁĺİ âľĮ +ðŁİĹ ðŁİĹ +ðŁĩ¹ðŁĩ ¼ +ze inab +wire duk +whati want +v res +un lincoln +tw loha +toe jam +tenn ille +st moritz +spyrothe dragon +slide shows +sivakor atala +sig nora +si byl +sh omes +segaf redo +sar ra +rock cliffe +quick book +pra japati +portsmouth uni +po ti +party goers +pap d +over views +ni asse +mul house +mc ca +master y +magn ani +ma ior +lu wak +le ura +kamau bell +justin thomas +iso pod +hot picatnoon +hank ering +handmade gifts +grilled cheese +gari funa +fon dren +fle is +fl en +fal ak +fab ri +f gb +f bb +es covedo +e disi +du lux +dri skill +dill ingham +cu bitt +cro oners +cre t +con go +cloud berry +cl ore +cau g +cardiopul monary +campaign mag +bu is +bp climited +book cover +barn old +bami yan +b ital +ave x +at aide +ark on +arco baleno +ag co +!!! ðŁĺį +ек б +y plac +wapping ers +wal on +vi ena +trust god +threes game +sher k +s musical +riseas one +reco lored +realmadri d +ram yak +ra sse +r hu +poo lesville +poco yo +pendle bury +p chidambaram +nujab es +nu tella +nh u +ne kom +moun i +moo kie +mon fort +mil ind +mei osis +marti us +manas lu +ma ff +m pr +lie tti +lego batman +le id +ladi esday +khut bah +is sei +indian wedding +illustration art +i onized +housekinok uni +hors fall +honor flight +hitch hike +hey sham +hell blade +har lin +happ n +find lay +find away +ex on +ele a +el oped +ec ards +dv usd +dru e +di ffa +co ffin +cherry tree +bol er +bi thday +bad finger +az t +au las +atthe movies +atchaf alaya +alic es +ali bis +al cac +ak uti +air co +agricul ture +ad de +a project +> . +* - +" ðŁĺĤðŁĺĤðŁĺĤ +ãħľ ãħľ +zoff any +z off +youss ou +yl ounge +wra pup +wra gg +weid ner +wc bi +wav eland +war id +vin ca +ver dade +ur am +un ce +tw fanmily +tu log +transpor table +the pc +syste ma +sur inder +stock holders +si es +she z +selec tive +schem er +sam ora +ron co +ro the +pul wam +prez ono +port ables +poly gamous +ph ac +penit ent +par aty +pa ea +over clocked +of cs +not ability +neas den +moon set +monte grappa +mini disc +min ch +midd ling +med ha +meag angood +me q +mati angi +mar jawan +m wc +lma g +lieu tenants +lesli eville +kro g +kol am +king ia +jo co +jed burgh +jaros lav +j bb +hy keham +huawe imate +hat es +grand cayman +gossip stv +fun in +for i +fer rada +enter ic +desar rollo +dead or +dan harmon +d pt +cog swell +classe ment +checkat rade +case mate +bur ts +bow line +book plates +blat chford +big mac +band aids +auti stic +any one +am rev +af fi +? ðŁĺ³ +ðŁĵ· ] +ðŁĩ²ðŁĩ ¯ +ìĺģ ìŀ¬ +willing boro +whis en +we vel +versi one +veri sty +ver ismo +traff ick +thrasher mag +talke etna +super sport +sti jl +stand down +stan te +single player +si lex +schwer in +schoon over +rin sed +r sn +per rone +pan fur +pa hat +pa changa +over dubs +or ator +omar ry +oli days +nu bra +nig guh +nie haus +niajax wwe +neuro transmitter +must achio +mc gm +man ess +madeinthe am +macro biotic +le creuset +ku y +ku coin +kon nichiwa +kiril enko +kies za +kel as +j and +intern alize +in step +in ci +hol guin +hin rich +ha german +guer rillas +gar ai +fulbright prgrm +ez el +emblem three +el ink +dolce amore +dex p +depos it +dek kers +d elli +clean sers +cer ato +cardinal nation +broad ened +bo ones +bankrup ted +avi ationday +arach nids +aper ta +ang leton +ananth kumar +amuse sme +al ster +ahmed nagar +ag akhan +ðŁĻı ðŁĺį +ðŁIJº ðŁIJº +zodi ak +wra wesome +wex tweets +vo j +vitali y +uk ltd +the katvond +th aba +sports world +sh omo +save ur +sap ere +saint arnold +rotter dam +ric heisen +re plo +radic alised +ra hr +quin cean +pumpkin spice +pu y +pron ger +project cars +prem ji +post ma +pipe stone +pe jor +pastor alists +oooooooooooooooo oooooooooooooooo +mye m +murmur ation +mu zak +mo ssel +mersal teaser +lan go +lain ie +job sfor +jee bies +je th +iri c +int vw +in hospitable +ic har +home goods +hi zb +gu u +good friends +go ke +glow ing +gla vin +gh p +ga urd +fro ese +flu vial +ffic er +explain able +ever se +ente i +du alism +dr disrespect +digital currency +clinical research +castron eves +cas ella +carter r +care uk +candi dato +c cim +btsx snl +bobby jindal +blay don +bl ount +bar tha +bal ag +baby cham +auch ter +ashab hosle +architek tur +aphor ism +an quan +ale ad +aggi ore +ðŁĮĪ ⾨ +íĿ ¬ +yuril owenthal +xen os +wal halla +wa ja +virgin islands +ving lish +un cbball +tre u +tre gan +tau bman +t enda +swachhbharat gov +super bow +stir rers +spe ake +so it +smar ine +shin er +shar leen +sambailey real +sal ati +rocky horror +roc key +re animated +proven za +pri zed +po tong +pit ney +olivier giroud +off hand +ne ath +natu rec +national doughnutday +nan ning +n xs +mercury marine +let z +laz ard +lac eration +kor ova +kok ila +know ing +katy did +kash ish +jar on +j wu +is ins +in actives +ilove hd +iam blackbear +hud speth +hero dotus +herac litus +hedwigon bway +gull foss +gen til +ge at +g men +far ag +drive tastefully +cow girl +cl c +chec kups +canni balistic +can adap +campeon as +by rom +bra infe +bon anno +bear kats +barcelona alove +bar q +author it +as ky +anu ger +ant je +adelaide hills +ad ness +ðŁ§Ł âĢįâĻĤï¸ı +ãĥª ãĥ¼ +اÙĦ ت +ze ba +zac apa +wor s +water kloof +water bed +war nes +w kamaubell +uso glu +un right +un gie +u ib +ty ro +twitch creates +tin ke +ti ppet +teenagemutant ninjaturtles +son ly +sobie c +smart card +shav a +shan awa +scro ggins +sanc timon +ralph tresvant +nick son +mobile security +min ders +me ander +mal um +mac nab +m wo +m mat +lamin ar +la grave +l wv +kier sten +juli ago +ju sko +is kander +in legis +hor sford +hiz bul +hellolove goodbye +guis borough +green biz +goddam ned +fortnite battleroyale +ferne mccann +e government +e cot +dy as +drake relays +documentary photography +diag ramming +dav ante +club sport +c gv +c bi +bull ingdon +bu ber +bram ham +boy meetsworld +blanc mange +bear skin +be well +be sth +as riel +art basel +ark ads +aran juez +anem ometer +amor im +allison holker +all out +ðŁĮ¸ðŁĮ¸ ðŁĮ¸ðŁĮ¸ +в иде +ÅĦ ski +year slater +vand ana +um theatre +ton ner +thegood fight +the hot +tempor ary +tar bell +sun screens +stu c +star ches +soa addicts +sne k +royor bison +reserve dly +regan smith +quanti fiable +pro ba +pre disposition +plane trock +pe ggle +p tur +om at +oliver heldens +ol mo +o com +no li +musco gee +monte carlo +mike sonko +middle class +mel nick +meadow sweet +mayward x +matt damon +m ols +lost girl +lec co +lang ara +lagan jae +laganjae stranja +kol asinac +ka o +iw aki +ion ut +instag ay +inglen ook +in born +hem pel +gun fighter +go devils +gil o +gan u +g morning +frei ghters +flat ness +escan aba +elli jay +dre cs +dex trose +defecte drecords +dave chappelle +cri stopher +cr m +countrymusic hof +cheese monger +cas ano +c bt +bro mo +botry tis +bau ma +bairn sdale +asadu ddin +arsen io +ano il +alco hols +ach aem +ðŁĺı . +ðŁĺĤ ... +ðŁįĤ ðŁįģ +æĢ Ŀ +༠Ħ +z fs +yo he +yankee stadium +work safe +whole food +welsham bulance +weight los +waddesdon manor +vä ster +vo isin +urban agenda +un inor +tru dell +strang ers +stoo ping +sne ering +sin ach +sihan ouk +shig ure +shek els +seab our +scott bakula +ron k +ren nais +ram pton +radion ational +pound bury +penna onb +pas alu +org sky +nw smobile +north pennaonb +non plussed +n ical +n anni +mur nau +mr rpmurphy +montene grin +love irishresearch +lino cuts +ji ro +jang keunsuk +jab ulani +indiak agame +igu al +i boschetto +hi mba +hart line +hair le +go swans +gachi bowli +fun land +fac et +epi thelium +ehr le +e wers +du va +dam bulla +da oud +cox ford +church town +cast mates +career day +bel ching +beach ampion +be amon +band ara +b to +akal pathi +aedil hai +ðŁĵ£ ðŁĵ£ðŁĵ£ +à¸Ńภģภ+ว ย +zey ne +wi fes +wac cam +var go +un ra +un organised +ti sane +threef old +spring flowers +sil vretta +sie ff +sc cs +sample sale +reg attas +rb kc +ram z +radio app +premiere night +plac ket +pet to +mur shi +mis ato +mcca ig +mbi id +mag safe +lu es +lu di +lil dicky +laye tte +kom m +knoy dart +keep init +kaf i +jose maria +johan nes +io ang +hu tong +hover flies +hol lin +hide and +heyits conrad +hey heyitsconrad +grand is +gang stas +franco eur +fol lis +fish ponds +eye shield +eye ful +e tap +e steve +dro ste +digit alized +de watering +de enday +dar in +daniel boulud +cy donia +ctv london +coreytaylor rock +cat chiest +car ano +c ten +buffoon ery +bu der +bpl boston +bootle ggers +bn sfra +bi shi +basket bol +at rack +at liens +asun glasses +ali ka +alam ocity +âĢ º +ö ller +© ï¸İ +z lo +wood ford +wol ski +wex ner +ul tim +tugue garao +trib biani +tri state +tied ye +than es +tard iness +talis mans +tagov ailoa +sym ington +supportall streamers +stay cool +star te +smer conish +ski at +sex pistols +selec table +sam aya +safe and +roc chi +ro mac +po sobiec +plac er +pi adina +pe tre +pal ash +o ort +o dium +ny ss +ny a +nic onico +new c +ne isd +naz areno +nany uki +mye ong +mu sou +mu on +mon ga +mil n +masti ffs +mal ady +mac gill +ku ih +krist perawat +ki em +khal af +k br +jan ella +j dw +inexor able +if youknow +hudders fielduni +hohen zol +hi en +harry met +guest mix +goz wift +gla b +gd ny +gbbo final +gastron omia +eric andre +enrol lee +disen franchisement +diesel punk +di thering +committothe g +cisco live +che ssie +chat to +char oen +central park +ce derberg +cad aver +bur th +brede sen +bour dieu +bern sen +barr anco +bar kat +bar bora +bag pipers +backto basics +al mos +ahmed abad +ðŁı Ń +ìŀ¬ë² Ķ +âĿ¤ ðŁĺŃ +ze olite +ye ver +y band +tech nion +tan abata +stair wells +ske w +si w +se ssional +sch wa +sc ousers +saroj ini +santa fen +sandi gan +sand rak +rug elach +rr f +roy se +ros common +rital in +recom ended +pin sp +pill man +per ahouse +pastor ius +paki stann +ou lay +ol ya +od der +north end +nit da +nas rin +mu lah +mer men +mel anom +m wai +lester holtnbc +le kin +lab be +l wwy +l bc +kyo do +kn ole +kad hal +jac kett +ja rena +isleofwight fest +irwin mitchell +indi atimes +ili ja +ia state +head scarves +he fei +grow your +gor je +gn h +gh m +frog town +fresh breakfast +float plane +far ran +ener gon +edger atedr +eating disorder +de stefano +ct il +cad en +business line +bun ko +break bot +bol l +bir thanniversary +bi japur +ber ny +bec cy +and sf +aha hha +ag is +" .# +zarbeaz b +youn gar +yoon jin +yo ji +wrestlec ade +wo ty +wha aa +we didit +war ofthe +wah habi +vic ente +ver as +uc bt +trip wire +tp mp +ti afoe +thr one +tar sus +tac t +sty mied +street league +stor g +ste eve +spi ef +son tario +sac redness +s bac +rum miku +renfro w +rc plondon +peep ed +peace able +pate kar +orcla pex +non native +nob ita +near ness +nat yam +n ells +mlb memes +mik ami +mi fa +melissa joanhart +mau die +ma kav +lynn haven +kin cade +khil afat +ke wau +kcj hoop +kav li +jire h +javel inas +jare k +itsnotre venge +itsme abir +io sh +ig ala +i vel +ho ds +hassel back +hack the +goof balls +go time +gh yll +fr nt +fen lon +faith ful +e wn +dow ding +dol lah +do stum +dis organised +di pti +deejay ing +comprehen ding +clo sers +cho ppa +ceta phil +celtic thunder +bu lut +bor relli +bnsfra ilway +bel son +bargain hunt +ari ston +ambu lance +ai ono +ack athon +, / +ðŁĺĤ ðŁĺ© +ðŁĸ¥ : +ç¾½çĶŁ çµIJ +ç¾½çĶŁçµIJ å¼ +yang gang +y csd +wasse ypur +tur u +ton le +tile hurst +the second +stu ding +stra ke +stip ends +som izi +sok cho +sick kids +sche chter +scalab rine +sam park +sach deva +s shs +rin sing +rach man +psycho analytic +prolon gs +pla iting +peven sey +param ahan +palae olithic +p ame +official ecfc +o goni +north bay +nex po +new baby +muje eb +mu go +movie fone +mer sing +mc ts +m sum +love books +le day +latel ate +ku miko +keep cal +jul lian +ice fields +hog nose +glaston bury +fs north +france se +fps bs +fau zi +ep cot +droid con +dor ota +deb u +de hydrogen +daylight savings +dan juma +cru dely +con somm +colec tivo +cc ps +calvin ist +c gp +bur jal +bricol age +bran nen +bor ra +bo ffin +basket bal +balo chi +artem chig +apro x +ap ub +ap tist +ap rc +ani ket +am bl +algonquin park +ad c +* ) +ðŁĺŃ âĿ¤ï¸ı +âĿĮâĿĮ âĿĮ +zer of +y out +wild fowl +we scott +we ide +vivac ity +unimagin ative +travel ingram +tor point +ti mat +theroyal parks +te da +swind ler +student en +starsand stripes +spu moni +spear ritt +so hot +slumber land +sidd han +shock waves +screen rant +ru da +ros eng +roof e +pren tis +porth cur +pastr nak +partner sin +paraly tic +ok tay +mountsin ainyc +mohm and +mg madvertiser +maal ouf +life forms +legiti mize +kit aro +keepingpeople safe +just announced +jugg led +jai ram +ja ket +is ky +io annina +hit makers +hard liners +h ise +galla her +frontend friday +frank linton +fra il +fay ez +fak ery +eri vera +equi v +ell ick +dwight yoakam +drum moyne +down falls +del vaux +david cassidy +cori olis +copper plate +co ggins +catch ings +bryson tiller +brig ada +bra f +blot chy +bennel ong +benedi kt +bear kat +bandain am +b sac +angela hartnett +alla re +all eni +akashi c +ab ello +åĨĻ羣æĴ®ãģ£ãģ¦ãĤĭ 人ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +ภ¶ +اÙĦ Ø· +zim v +zan go +z opa +z ellers +young bae +yi ps +xiaomi india +water wheel +w bl +vo icec +vincen eil +v ur +us lim +ti gra +the answer +th h +tal do +tail pipe +t yee +t jc +sz il +sz ab +suz aku +stanis lav +sk zn +sk ur +sco tians +sc it +sas software +saint lucia +sa ade +rf q +repet to +realestat enews +rat zinger +rain fow +rah mani +r hq +quic kies +pro pe +pin nell +palad in +nu eve +nic ott +nau de +na stier +mö n +michael keaton +mic eli +mccor mick +mann ingham +lor di +limited run +labor de +khary payton +internet day +ho stos +hi wx +grand lodge +ginar aimondo +ga stein +g po +final say +fant ine +existenti alist +eric martsolf +elie zer +electric guitar +ei de +effic acious +double lift +dhar y +derick pauls +cream sicle +cor field +com ino +cargol ux +can ig +ca prock +ca asports +c pc +bob collymore +beaz ley +athene um +ash wag +aro ya +ace day +a oficial +$$ $. +! ðŁĴŀ +ðŁĶ ī +ðŁ¥ ª +yu ichi +writing day +wevel gem +vil joen +v ona +uq am +unis dr +tip tronic +ste v +spin all +sihanouk ville +sheffield star +sha hir +sandigan bayan +sains bury +ro amer +re manufacturing +ra des +pyro clastic +public space +pric kett +photo gr +penta ho +pamban sang +ow sley +o ton +nific ent +never yield +mel ds +mc chrystal +marath oners +ma ay +landon donovan +kor at +kipp ah +kal ish +joele mbiid +jo see +jetti son +j ri +indi ap +hurri edly +hatter sley +ham are +gully boy +gov ballnyc +gol gotha +gla ucous +ger rie +g app +fri gate +elve den +e ous +down playing +dor ados +cry ption +cor rell +cher ub +ch ome +br onews +bio g +beach head +bal dry +ashley purdy +ang ella +amy ra +alvar omor +agronom ists +acu edu +ac or +ðŁĺŃðŁĺŃ âĿ¤ï¸ı +ãĤ « +à¹Ģภ¡ +z azz +yo shim +yellow ed +work it +wh iner +weare texans +wantt omarry +w marybeard +v ong +univer sum +un do +twi ste +tic kle +tee pees +tanz anians +supportyour local +ssouth africa +sol enn +soil work +sm p +skybetleague two +roit feld +rh y +re heating +r uri +plaque mines +pil ton +pag od +outand about +ori x +nic hts +mill stream +maxplanck press +marylou mcdonald +mar aca +mac books +lul led +lo der +liv ening +li gious +letter heads +legar rette +lang tang +la by +ko co +kam in +jme bbk +jacob in +igu chi +he ffer +har aam +han si +goddam nit +gang way +friendly breaks +frankie edgar +fol ge +fire trucks +fi resi +fene stration +f ones +ei gh +du kin +don alds +discipl ine +disc or +din goes +descend ant +dere kand +den ki +ctv calgary +cre ager +con drey +chatt yman +chadwick boseman +ca storm +by akuya +bru tha +bran ded +bj arke +ash gabat +anh inga +an sp +alli en +ad at +ðŁĩ © +ì§Ģ ìĽIJ +yal sa +xx viii +world soil +wetnwild beauty +vol cker +vijay ak +vide over +urb act +un traceable +ud der +u dra +tox ico +thér èse +thanior uvan +td cs +swis best +suppor tin +superstar mahesh +stream flow +strat aconf +step child +sp iteri +simply shop +shu hada +schwarz wald +sal ur +sa heed +re penting +r under +r ry +qi bla +port ly +pooja sharma +pis ano +pel is +pel ham +pay phones +paint sville +p mac +oul ter +neo cons +n oooo +mull is +mose ley +moo rer +mik ro +me ggs +mayorof la +main er +ma hou +lul lah +loch ness +lo vis +lime scale +like us +l pp +ked zie +jo vian +jimdunlo pusa +inn smouth +i stra +i ami +high cliffe +health week +h cs +gol fer +ge tag +g enda +famili arise +faces wap +ene wsroom +discover ing +dim wit +delic ous +dan z +d hur +cuc cinelli +cor stor +chen al +che ch +chart buster +be spectacled +bar mer +balder dash +back strap +b hon +as sts +am pt +allo c +all waltrip +albumo ftheday +:(( ((( +: ), +. ðŁĴĶ +ðŁĺį ðŁĴĵ +ðŁij¶ ðŁı½ +ðŁIJ· ðŁIJ· +âĸĪâĸĪ âĸĪâĸĪ +Ùħ ا +ÌµÌ ¡ +¡ # +zzzz zzzz +virtu spro +un awares +turtle tuesday +thisise dinburgh +thenew shour +te um +stark ly +spre esy +spreesy co +sor l +shay an +seren ay +scow ling +sche herazade +saf fa +ros seau +riz k +rin us +reebok classics +red letter +re negotiation +pulkit samrat +pro geria +prevention week +pit sea +pine top +photo copying +ou dh +oppos ite +objec tified +o ire +next time +neuro marketing +nbac ares +nandish sandhu +meaning ful +man ara +labor al +l mpd +ko vu +ko sho +ko kk +kidney wk +jarryd hayne +ivan o +ip k +in stil +he ti +gucci fer +gon dry +go back +girl streamers +fl annigan +extra ño +extermin ating +eee et +dri a +chapter tweets +carl fogarty +caraba ocup +cabinetre shuffle +c ssc +c spa +buck aroos +brachi osaurus +border town +bol c +bed n +becu ase +bbc tees +bbc gw +aviva stadium +asa ints +as ara +art dielle +antibully ingweek +air wave +ag ab +afric aus +( ; +ðŁļ ĥ +ðŁĺįðŁĺį . +ðŁİ ³ +ðŁ¤ ķ +zay ed +yar gentina +y mir +whit low +var on +us sen +truss elltrust +trans dev +tran socean +tonsil lectomy +think tank +the vulcansalute +tele ports +tear sfor +tamil cinema +spur ned +span kin +southern miss +samar itan +sam mie +sa huar +ru ppert +rsp bin +rspbin theeast +ri mu +record able +re ch +q ande +pre gun +pine wood +philli ppi +patt yar +pag pag +op ry +on wu +om itting +odemwing ie +narcissi stic +n komo +mul liner +maynooth uni +m ür +leon bridges +kwe sta +jo kanovic +jessic ak +jav it +instaf reebie +in cun +housel ive +gy les +go shi +friends first +fox fire +fal vey +fal com +etz le +en cana +ela ina +echocardio graphy +dr n +do gon +demol ay +dar at +dam nn +d anya +cosmo polis +core opsis +co stin +chichar ron +chay kin +camden market +cal es +cable vision +c mi +blan chet +big query +bech ler +bar os +arti s +afl w +afam ilia +abbo ts +ðŁ¥ ŀ +åĩº æ¼ +âļ ķï¸ı +yu ria +yow za +yac ine +y ber +wunder man +wood pile +watt les +vo vin +vex illo +to simplyshop +the wilson +tetsu o +tb c +tab or +star tl +sports desk +sp icks +sol ingen +sit ton +se yes +scul tural +sam aa +sal tim +s bl +reti en +ram aswamy +pre ah +plumb ed +pin ole +pier is +ou cher +o idal +night shirt +mums net +mean sall +me uk +me op +mat tt +mart elli +mam mon +llor ona +light ship +lao tian +landsc aper +kun lun +ko jak +kca argentina +kan oo +kamp us +ju shin +jay jay +islam ization +horror nights +hol is +hill yard +ham man +ha pag +h wm +gar ima +gang stance +er ges +endur ance +east mid +dutch town +du f +diab o +depos itors +del erium +cu ssing +cor deiro +cham ac +cardiff council +cam ii +c ter +c ico +bor nin +bol ting +bol ds +ber o +ber ling +belle garde +annnn nd +anas am +adal at +ad lard +ðŁij® âĢįâĻĤï¸ı +ðŁĨ Ĺ +ë¡ľ ìļ° +zer heide +y aser +who dares +war cry +uss c +urbang arden +til brook +the talk +the danny +tar ant +t max +stere rs +spar knotes +skate board +shiv aya +shi e +sab al +rain hill +rac ism +pul lup +poul try +poly count +po lit +ping ree +parl ours +para ÃŃ +pang ppopro +ow ol +ow gr +nick ers +nfl fantasy +my elin +mr jafri +mo lesworth +ml scupp +missou rian +mary knoll +manju la +ma hala +lut fi +lam ov +ky yon +kon oe +kil donan +kers lake +ira wan +inesti mable +in accuracy +ib ti +hus sen +hk racing +gri se +gre villea +go so +glen wood +gh eroes +gar my +flower pots +en aire +ei ke +dupont pioneer +desi erto +democr ac +debr aruh +dead weight +codepend ency +center nyc +by our +british farming +bra si +bergdor fs +bel on +bail able +ay anna +auro ville +att ard +ard han +archan akalpathi +ar rc +ar dent +am oroso +ìĽ Ķ +ëŁ¬ë¸ Ķ리 +ëĵľë¦¼ ìºIJ +âľĮ # +Ï ħ +val miki +trav ag +ti pit +tetten hall +tcho tch +ta wau +t fc +switch over +sub servient +steep les +sko whe +silver dome +sign um +siesta key +shor se +shek hawat +sa cher +rob breport +re produces +pre achy +pl s +photo shops +pen shurst +pattyar quette +pakv wi +no deal +nis an +net gov +nationallibrary week +my desert +mu rid +mil v +melis sad +mcly te +mbalu la +mat ahari +mand ana +man asi +london theatre +lo san +lo ll +ler ner +laluprasad rjd +l cr +l bb +iron bound +inve sco +hye sun +horse heads +hope toun +he ise +hab bo +go browns +ghost writing +genealo gist +ge aro +fron te +fer ing +feather ing +e acc +decision day +de vice +d cr +commiser ate +chin ned +che me +cephalo pods +cal amit +bra bazon +ble k +bla ire +bin os +bc ci +balde agles +athle tically +at uk +ancient rome +adam baldwin +ache let +ac che +âĿĹ âĿĹâĿĹ +Ñ Ķ +ye ou +yah i +wy at +wor kexperience +weekend fun +vin cy +up online +under performed +try p +trombone shorty +thermo couple +targe tt +south ville +sna x +smo ving +sl acked +shehu sani +sh appen +seg agenesis +redd itor +re configuration +ra ther +practic alities +por tales +person ify +papar azzo +pal at +our nameis +ong ma +o len +o chil +mot ti +michael avenatti +mf doom +medi ag +mall rats +log ous +llantri sant +lic e +le strade +john bishop +hills boro +gy gax +gram ercy +ge tex +g nat +for yourself +fel i +faz l +epp ley +dog sin +dl h +de wal +dam isi +cu eing +chint u +chi avari +chau mont +categor ia +car ny +calcasi eu +bur gen +bell in +be aky +bas ler +azu car +aw es +at chie +assetto corsa +ask ham +an tero +amar tin +am paro +ak aroa +acom ets +ðŁĺı # +ðŁķ µ +é» Ĵ +çī © +wo ah +whit enoise +victro la +un blemished +tu de +tie polo +thehip dotcom +than gam +th ax +tele porting +su mona +stone hill +spar ro +sime to +shi ur +shi kamaru +sh emp +selec cion +scar avan +sam plings +sade ghi +s ket +ru so +ri ise +recap tcha +real skipbayless +raj path +pol ara +pap acy +pad dled +official bvb +oceano grapher +new ser +nage sh +mun ky +mise z +ming a +mi shal +mi gros +mal vasia +lionsden kxip +le be +l drs +kra vis +ker is +kab ini +jol ina +jetpack joyride +jay me +istom in +is sam +ir kut +improvis ations +i would +high line +hart ke +h ran +gu ell +google plus +godd am +go ering +gen ce +gal op +fi sto +farqu har +expon ents +electro shock +east in +du plin +dav inci +dance y +cu i +coom era +chit rib +chas eric +cat te +can tanker +burn ley +bi valve +bi key +bett ye +back draft +ay ou +af al +ach ery +ach el +acap ital +ðŁijĢ ) +ðŁįºðŁįº ðŁįº +ys ss +winter bourne +whit eville +whir ter +un reservedly +tubu les +thy depark +the terminal +the matically +ten bury +tel cel +source fed +sop o +so ji +sis land +seung min +san son +sa shi +roger splace +rockof ages +red deer +ra ult +r wr +pu cca +pre destined +polyphe mus +pastor ing +opti plex +okay player +o zzi +o ssian +non ukes +nih ilistic +newcastler aces +mont verde +min ella +mil as +micro light +mi zan +mat raffic +man ie +lux o +lucifer onfox +lu ll +lipo protein +lew ington +ler adio +ku pe +kore y +kno bby +kick off +k rank +k aci +junkyard blonde +je bb +itte faq +inside out +hun gr +hop man +hay dock +gt masters +green power +gli b +ge mein +g son +freo dockers +england hour +en vies +em x +edin travel +ecan al +ec an +e ik +des demona +de arer +daysof type +corbi jn +cor mac +conni ving +chand ini +bon su +bi bis +bab ai +auti ful +at weet +at rue +ash leym +art chat +aps virginia +am lin +almo sthe +agn ès +af rin +ðŁ¤ŀ ðŁ¤ŀ +çİĭåĺī å°Ķ +âĿ¤ ðŁĴĻ +âĸ½ âī¦) +woo oh +win er +vijay ad +us ffootball +up wind +tri gon +tor q +toni oli +tom fletcher +todd haberkorn +throw ing +ta vo +syste matics +sk int +si tharaman +si pp +ser toma +sc ouser +sat anists +sassi est +sari ka +sale hi +robbin sville +rajesh m +r du +po teau +pit b +pic anha +pall u +pa wl +official nihr +nr and +ni pa +neg ative +ne ase +mar n +mafi oso +ma zy +low light +looooo ove +ld s +lambof god +l ous +l ke +kasar ani +jon gordon +jc chasez +indoctrin ate +human o +hi bsofficial +hand sup +han ratty +ge toff +gayle king +fy ah +force ps +flume music +espan ola +du che +disbur sement +diph theria +determini stic +del or +dairy land +d phil +cle eves +cil lessen +cali bur +c eller +bu ñ +bt q +bro mine +bon dsman +ben ares +as sc +and aran +all arin +aic te +ac cen +abby wambach +ab ish +. ðŁĴª +( ?). +% !! +ðŁij¯ ðŁij¯ +ðŁĮ ĵ +ðŁ¤Ĺ # +ìĨĮëħĢìĭľëĮĢ 미ìĬ¤íĦ°ë¯¸ìĬ¤íĦ° +é ³ +áħ ł +y adv +wor c +wam pler +walks britain +ured ome +tu bb +to your +to fu +ti ran +thu is +the joint +the flash +te bowing +taw se +strick lin +startl ingly +south old +som bre +sk ala +shawnat spg +shapeof water +shan kara +se her +ri poll +ri ble +re took +raw don +prohib itive +pres sed +ply ometric +pic abia +peter j +ou thern +one ws +one ofthe +nw ambulance +normal ise +noise less +new balance +min ni +mh b +meur on +mel lowed +marat ona +mad hatter +ma str +lore ena +line ages +life ok +laat ste +jun jun +jhah neu +intelligen ces +inst ers +ini ketan +hu ddles +hool ahan +ha wi +gran ville +gor der +gh era +gang won +g oms +free taheri +f ctc +exoske letons +escam illa +ed al +du roc +du ller +dari of +cou cher +cliff hangers +circul ates +can tos +camp liberty +c aguas +bru ma +bri stowe +bodh gaya +ban ski +aze ali +au chan +anim alistic +alof thotels +alen ia +al ı +akh ter +' ?! +ı ï¸ı +âĹ ¼ +âĢ ¿ +ye vent +wxpn fm +visit causeway +victor yday +ver tex +ve ve +ve dic +un knowable +u scen +trot tier +the fla +the bottlemen +than u +tag e +t center +str ø +so xford +skin nies +sk inn +she kau +sarbj it +sam eh +saf ford +ru thie +reti rements +republi ka +rad stock +ra el +qui bble +porsche tennis +po ong +po esy +photo play +of thenight +not doneyet +ning aloo +ne ak +navy pier +mv l +mueller time +move in +mo hd +mnu kteam +mlscupp layoffs +mid line +mehreen pirzada +mariecla ire +lasti mosa +krat is +ki ev +keaven y +ir responsibly +ir me +inocul ation +indu bit +hu tter +hr x +herb als +hd fs +ha bib +gu as +graci ousness +fro myour +four ways +fly te +ey er +ever thing +ev agreen +ero ach +emce ed +embal ming +elis icki +dun ton +dow se +dg pup +ded wards +cornell feeders +ci elos +cal andra +ca shout +brü cken +bot z +blue screen +bet sie +be ene +b wo +asper g +ang or +*¨ *âĢ¢ +ðŁĺŃ @ +ðŁĶ ³ +ðŁĴŀ @ +ðŁĴĭ ðŁĺĺ +ðŁıĪ ðŁĴ¯ +ì§Ģ ìĦ± +xiv preprint +world views +wood fired +west virgini +ve atch +var as +u bah +tru eno +titan pride +thyl acine +team europe +teak wood +taver ners +tam worth +sy outh +stan zas +solan um +shrin er +sepp uku +senbob casey +sa the +rummiku b +rude boy +ripper street +prin ts +premi x +pp g +ph la +pen sami +pe ma +para professional +out weighed +or ce +onomato poeia +nu tini +nail head +n kp +mr br +min dof +meyero witz +menshealth mag +mccul ly +mat unga +maru chan +maharash tra +liqu igan +li ssie +kyyon voot +ko gar +kin ver +k xl +jon foreman +jay an +ir landa +insomniac events +in saf +in fraction +ic ps +hu el +hol tzman +hispani ola +haz uki +go ggin +get kahoot +g itter +furn iss +fuer tes +fro mmer +far on +er lich +el well +ed ney +dece ives +competi zione +cimorelli band +ce ats +caval ry +ca chorro +bu ya +binib ining +ber thing +beg u +ash gate +amicha els +amateuri sh +alanthomas doyle +Ĥ ĺ +æĺ ¥ +w das +vander burgh +us acycling +tu pou +trou ts +tor ben +top kapi +texase dm +t lg +sumb awa +sports zone +spill man +skun ks +shored itch +shi prock +schle mmer +sb k +ri ggle +reali ve +raven snation +raj pal +rais ingthe +pic eno +petco park +olloc lip +oh lone +o zi +o possums +news break +nc sen +national forest +mo doc +mar the +lucy beeco +lu ker +love scenario +love cats +longlive the +longh i +lildicky tweets +ligh thou +licen sees +levit ated +lesbian ism +ko ky +kla sa +kit bag +kill la +jonny bernthal +jobs act +jer ic +jam erson +ja har +ilove gay +hypochondri ac +huff po +hub ley +ho los +hamilton ian +gy re +gt ld +gou ste +go ren +gi gir +fri pside +flet ching +exp end +entom ophagy +ei agov +eh feuro +eas ington +e via +duke dumont +dat alake +d orion +cro ll +conval escent +conting encies +coloni alist +coal fields +coa homa +co tham +chol ars +cake boss +ber row +and win +ahon y +aene id +! ðŁıĪ +ðŁijĩ . +ðŁijĢ : +æľ ī +yu shu +vogue india +twin z +thought leadership +thaanaaser ndhakoottam +su bash +stri ated +str ats +skate parks +six er +sign alled +shel drick +sabin elisicki +r risd +panam aleaks +pan fish +pan cham +ossu ary +or sa +nr hs +norm core +nght mre +mt lg +mid section +metro sexual +memori alizing +mc get +marvin humes +mandi ra +mack illop +m ally +ler ato +le ik +lau raj +la violette +ko izumi +ko est +knight sof +jo ch +je sup +j pp +izquier do +is obar +im bru +i pra +hungar ians +hiro ko +hair transplant +gro bb +go global +ging ras +gi ak +gar st +gam mage +gali l +g sl +free thinking +et ches +ense mbl +eat right +e ski +dji bril +con fab +colle gen +chaseric emusic +chang ingthe +cha hta +cav azos +carri g +can bball +burt ka +bmw group +bio sensor +best team +bernie in +bar ona +as px +app art +ap ital +anhydr ous +angeli ves +alph ard +alped huez +aero india +ad jaye +. ðŁĴŀ +---- ---> +ðŁļ ī +yu liya +young stown +x bla +winter ized +wall ac +wak eling +vital voices +v sf +v endor +un foundation +ty coons +tw ende +trans disciplinary +trade speople +the jeff +the james +ten ko +tempe stuous +tau ren +swan seab +sun bae +su ez +su crose +sty al +stopp ages +space suits +shiz uku +sh unga +sex party +repar ation +reen actors +ree king +raym und +pre vue +parten kirchen +os se +nih director +naz arian +myo sitis +my vi +mountain tops +mar cial +mar cas +mar al +manz oni +lucybeeco conut +lie tuva +kur tv +kitt itas +karlo vy +ipsos mori +insu lin +in ale +how we +hav anna +hah nemann +hab si +guilden stern +gran dia +godis love +go viks +fru ity +free kashmir +fox nashville +fou gasse +flat test +field photofriday +fiaf arn +fiafarn borough +ey talking +entren amiento +dont lie +dis believers +din is +dick o +des met +daysto gofor +day star +d ilo +countryfile mag +co system +co ati +cle anc +classi fies +byo c +bun ting +bos stones +bon fox +ber it +bbc newcastle +bag ile +ay ani +avan tasia +ary land +ap ap +andalu cÃŃa +alyn n +all art +alder wood +afterel len +ðŁĺ ¾ +ðŁķº ðŁı» +åł ± +ãĢ Ĭ +wra ss +who re +we stray +watch u +villan elle +ve sak +tun er +tor an +ti ffin +thehistory guy +the boston +the ale +that sme +tg d +tele dyne +tal eg +tai fa +surf sup +steeler shockey +spart ner +small bone +slic e +san tucci +ro ces +reid sville +registr ars +reen acted +rad dest +quin one +pyroman iac +ps b +practic a +phra sed +petty fer +pete tong +peac elove +nikit ind +nc p +my horse +mira belle +mi era +mc gar +mal lik +mad woman +long way +l sn +knock knock +kick er +jet liner +jag d +jab a +instagram mers +hu tt +hi ji +h ts +game music +fang led +ex onthebeach +eller be +e ah +dv sn +dro gon +ded mon +dan scavino +da sty +cross field +cricci eth +commerci alism +col ls +ch c +cas ar +can im +can ball +cambridge utdfc +c whm +butler u +boer se +blow dry +bic hette +ay nrand +atis faction +anaesthe tist +af jrotc +actor sequity +ðŁķº ðŁķº +ðŁĴĸ ðŁĴĻ +âĹ ī +zi ii +you might +vie we +ver ie +u rad +tw afterhours +tro wer +tri aling +tr ono +tool shed +thermost atic +tessell ation +ta ky +star fall +sle tter +silver leaf +shra der +shannon airport +se dap +sber bank +sav at +sar aramirez +ror aima +reci be +ray ne +pro pre +pin i +oh canada +north land +ng cobo +modu lus +mis ka +memor ising +mbl science +ma pping +longu euil +lit eco +lee derville +leav ening +land seer +ki dero +jin do +jil in +hue hue +hon en +har har +green and +g tw +foodin dustry +fas lane +express andstar +eli eve +ek blad +ear marks +dwind led +doherty shannen +dism aland +di iv +del fi +dat aw +conserv atively +coffe red +cityof miami +circum navigate +chapp el +cecil ie +canad achat +calz ado +bra venation +bor chers +bell ino +be parto +bab s +b bler +ayo s +areyou withus +anto griezmann +amar cord +allarin aresh +abor tionist +!! ðŁĺĺ +à® © +zbig niew +zay d +yugo sla +wul ff +wotc staff +wn f +whit ish +w gb +ve ale +vas yl +udd hav +trin sic +travelwith kids +tou k +th yn +tele work +tan sey +tal ab +super wiki +stun i +sto ch +stat eli +so gn +sj t +simm ers +si bles +screenshot ted +scott grimes +schoolo frock +scar r +sc ahill +sali l +rpcrd pro +roc ke +ro gow +rel ton +refu elled +re cut +pro ops +phreno logy +persi mmons +pe que +pa shin +or za +op ier +north town +nan ticoke +n didi +mix to +mal ene +ma gh +lo vies +lo fted +lil loo +lac lede +khun e +kate moss +k msl +java one +jam elia +j lee +j allen +irkut sk +iner tial +house maid +holo fernes +good time +f pb +em mit +elon university +dun vegan +dannyjohn jules +cyber security +cuuu ute +confir ma +cn ni +cl ery +bul ked +bs thydepark +bio reactor +bbsr buzz +bar bar +b jo +az azel +az az +annou cement +activ ity +acar ter +Ħë ² +天 å®ĺ +zodi acs +yummy food +y omo +wild storm +wi j +wh ys +vu cic +vali k +u shio +u art +ty me +tsu kuba +te yan +tab ri +station ery +slo sh +ru ti +roy all +recruit ing +rav eng +ragin cajun +rachel wood +qu tb +push kar +play mat +pilgri m +pepp eridge +pat as +param edicine +om ran +ol uw +ok uk +ocean optimism +o bia +ny sc +num é +nicode mus +naz are +nap thine +mo tz +man they +ma kovsky +lox ahatchee +letsgo g +lebe dev +l rr +jan ey +inno trans +in p +i acon +hongkong ers +hol royd +hepat o +havi sham +h nn +h lr +gur lz +fish sci +fare ed +ez us +eti xx +dur yea +dulil lah +diy ala +come home +co ar +clear inghouse +chi bs +bringerof rain +brac er +bob sinclar +bo hl +big kid +bel ch +bar qui +automobi le +atta m +at au +app ley +abu da +ðŁĺĭ âĿ¤ï¸ı +ðŁıĨ : +âŀ¡ âŀ¡ +à¹ĢภĪ +z ill +youtube ãĤĪãĤĬ +yoshi hiro +waynes world +visit somerset +vign elli +vamos rafa +usu sa +tat is +taj ine +sundaytimes za +sub t +sne p +si evers +shraddha srinath +short sighted +ser bian +screen savers +scou gars +sacram ental +rode mics +retrospec tives +ren tz +re focused +ran sack +ram anathan +pu pe +proven cal +poth older +plast isol +pan can +ome x +om us +ogun damisi +not for +ng oma +nel ley +ne yo +nassi b +my ke +mul lagh +modisar kaar +mmm bop +mign onette +mdcpss outh +mase ko +man ni +man gia +mal y +loun g +lign in +lar bert +la ska +k me +k acy +jyo tir +jur nal +joan smalls +hi rose +grand ma +good speed +ghe en +get syou +esc ala +ent oura +du art +di yan +devolver digital +der f +cuu ute +chen ab +cap illaries +bren d +bi mal +back wash +ato cha +ask acurator +ami sha +americani zed +aly si +alma den +abbi amo +aar hu +woolla hra +wam ish +von da +vo gler +un club +ty j +su ja +ste ar +sput nik +sol are +si rot +shal om +sch marzo +rev dr +rep joe +remor seful +rei ff +pum phrey +plain sman +ou tros +os ric +niagar a +nag ata +mo watt +meadow vale +maison objet +mad han +m lambo +m elly +lor ri +lom mel +la ding +kol k +kh ooper +kar nes +je uner +jane goodall +it oi +ireland cricket +in gho +hackett stown +ha si +falset tos +encom passed +em tee +doh kyungsoo +dig nan +dej loaf +de filing +datab ank +cu kor +chro mat +chocol atem +cannon balls +c crc +bromeli ads +bro xton +brian topping +bogdan ovich +bob st +bl ini +bin da +bet elgeuse +barqui simeto +back list +arnol ds +apple live +angu ages +american football +alo c +ago stin +af ree +af d +adar ling +ðŁĶ © +yemen cantwait +wis sen +wedd ell +war child +ver aging +vapori ze +v pk +the highway +ten ov +te ter +subsi ding +sensation alism +sa fir +rv m +rockthe vote +ray lewis +pulitzer prize +promis cu +princi pally +piñ as +photography islife +pap in +ou ps +once in +ogie alcasid +ny dn +nvi di +ni fl +neon ates +nab ba +n mh +mytop college +mv sales +min ako +mean wood +mck is +mb w +mayo red +marka very +lub bock +le itz +lav ina +kope cky +ker in +herbal medicine +helen s +hann ukah +gor ged +glu z +gin ther +ger sh +ger giev +gener alization +gean inec +forthe soul +fin ds +dream girl +distracted driving +disrup t +digital disruption +dewor ming +der m +demer ara +de vey +dad jokes +dachshun ds +commiser ations +chi kar +ce z +cay lee +carbon aro +cam bro +cal verton +brezh nev +boost torbay +boo b +black power +bespoke bids +bear naise +bbce mt +ballant ines +awa is +annul led +alber tac +ak ul +acompan ies +ðŁijł ðŁijł +ðŁ¤¦ ðŁı¾âĢįâĻĢï¸ı +åħ ´ +âļ½ : +ØŃ Ø³ +в од +ya ari +world downsyndromeday +wor mer +wol fy +westandby parth +wait ressing +videover anomtv +urru tia +united health +tele porter +sre invent +snow ball +sky fire +skowhe gan +sc upper +sa amy +rush ville +rim shot +ri fai +recording artist +reading is +re framed +rakuten ichiba +racing club +r nz +presidential debate +plant in +pe dra +p ck +ornel as +opho res +nikitind heer +nakshat ra +na fe +muss orgsky +mur taugh +multic loud +multi year +mp sf +mon amour +mat ri +maidstone united +luc u +ko x +kkun dra +kerman shah +keo wee +jan ee +inck arnataka +i day +hu se +her sham +hal bert +gru y +gre asers +glynd wr +gali on +front pages +fear gal +fac ie +engel berg +du wamish +downtown halifax +dove dale +dibuj ando +di kes +consomm é +con der +code black +clu broom +class icist +circle app +chrono logically +chef stable +cere bus +center fold +bur gi +bir do +bin z +ballo ting +aron off +apple ton +algi eri +alev els +ae gina +! ðŁĴĻ +ðŁļ ¦ +ðŁĺ®ðŁĺ® ðŁĺ® +wine fair +wed lock +want sto +vol ler +vir tu +vil ify +value of +v dr +under takes +und pasiapac +un collected +top flight +tibi dabo +the point +sym monds +sub sonic +specialk brook +softg els +shi val +seid man +sand lin +san francisco +sam per +ruk mini +ru ma +rm liga +ri as +remember anceday +rachel riley +qu bit +pan elli +osteo sarcoma +or ay +ok mulgee +my god +mine shaft +mickie james +mess ines +lleg ando +kre y +king ery +ke tty +kawak ami +juli ac +jab harrymet +interchange ably +inter continental +ic lass +hyper hidro +hr zn +hijab s +head butted +hal sman +hal it +gu uu +gor m +gon iners +gar ris +game dev +fuji yama +fish sketch +fin bar +es rb +endaken nytd +du h +du el +deflec ts +corvet teracing +clat ter +cheese day +ca edchat +burton albion +broad meadows +brigg ate +bornfree onekiss +bor rell +block screening +be xt +be ddington +be ason +bati stuta +barnold swick +av d +ashi q +andreali bman +aman at +alu mp +alek sey +alcac er +ðŁİī ðŁĴĻ +ë¦ ¬ë +ãĥ¼ãĥ ī +âľ¿ ) +âĵ ¥ +â¬ĩâ¬ĩ â¬ĩ +ಠ¶ +worcs warriors +woodbin eracing +wherei root +warne münde +vy apam +vis co +ve dan +ve asey +vacation ers +us iness +tr illing +tony todd +tant amount +sydney storm +ste bbins +ss rc +sport sca +spic ey +south shields +snow bank +siriusxm nascar +sham n +se et +scot lands +sac chi +road show +ran king +rajiv pratap +rajivpratap rudy +poe sie +picto grams +pahal gam +pac y +ou bre +on oda +o ey +nil son +mill in +mi w +meridi anc +medec ine +lu tion +life ok +lasc aux +lal ah +king let +k sed +joh anne +jamesdro driguez +ivan ova +io ane +in y +ill inim +ibi zar +high town +head corn +harman preet +ham amatsu +gol ondon +gi andu +gavr ilova +game keeper +fuel your +fil ename +fficial site +etsy store +epidemi ologist +d bu +crim minds +co loc +brooklyn bridge +british ers +bour nes +bo sv +bio availability +bet ina +bell anai +band ini +bal at +alma z +... ] +ðŁĴ«ðŁĴ« ðŁĴ« +âľ ¯ +¡ ! +yudk bh +young sville +wh h +wear thebear +tp gallery +the drummer +terrac ycle +team natural +stri k +sofe vil +soc dems +sn pout +sil as +she aly +sh hhhhh +ser ape +sandiego zoo +ribo flavin +remo delling +refin er +raft beer +proud sponsor +preci ate +politic ised +pi um +phone tically +phantom ofthe +pdx eats +paint box +oon light +ome one +nz stuff +nu o +not all +no st +new bedford +my nach +mr sk +mom os +miguel ito +middle ton +mic r +metal smith +mar gi +mac c +lovenorth devon +lo lu +lin tz +leav ell +la thes +ku gler +ke ssel +k sd +jugger nauts +jn moyo +hu ie +ho sier +haz rat +ha vil +gaz elles +fet zer +fall s +eu cerin +esp ad +ear plug +dress age +dispos session +discover hongkong +dior amas +diesel gate +di ener +di abolic +den y +crypto zoic +cor tez +coa stuk +clive barker +c ays +breathe carolina +breakthe taboo +ble ating +birthday wishes +bar don +avox el +apho tos +ak ka +ait cofficial +after birth +@ ' +ðŁĺŃ âĻ¥ï¸ı +ðŁĵ¢ ðŁĵ¢ðŁĵ¢ +ðŁĴķ ðŁijij +ëĵľë¦¼ìºIJ ì³IJ +ë§ ¨ +виде о +we it +vest ager +un managed +un coordinated +tol ga +tin at +tat er +swash buckler +super mini +stra wn +sten y +st ura +spo ked +skiat ook +si phoning +she diac +seraph in +sang oma +san h +sag rado +ro td +review journal +replac e +ra ia +questi oner +nor sk +nene h +nar k +mole fe +mod sun +mini on +michel ine +men deley +martin son +m grs +luxuri ant +lt da +lig ure +le we +latin x +kish twar +kind afunny +kas kus +k assem +jewel staite +jag meet +ib r +i ut +hoo pes +hol on +here tical +henri bendel +hel lions +he sham +hatch backs +hammer fest +ham taro +half on +h bday +go abuzz +glaci ernps +gen evi +gau m +fn ma +fi rel +eric ripert +e ec +doo kudu +dfat irl +cou pla +cork citycouncil +colombi ana +camp sie +cal co +bride well +bowery ballroom +bj arke +bhic how +bb g +bare foo +baby wearing +at unga +asimb aj +arab y +ali velshi +al pines +ago today +abou tit +ab ff +ðŁķ ī +ðŁİ¼ ðŁİ¼ +âľĭ âľĭ +âĢį âĢį +à¦ Ń +ü ber +à ½ +y shop +wwe chamber +women inthe +war lick +vik ash +vid hi +veik kau +val vano +val son +tune core +sunny dale +ster ry +sk nonline +sil sden +sh ene +sestri ere +sensiti ze +se amount +santo ku +rep ented +raj shri +ra ir +prof jnmoyo +po grom +planetofthe apes +pine au +per centers +pe dder +our city +official y +o ib +nis se +neversay die +ne tro +national museum +my data +muse tte +mon ach +miumi u +maur ic +ma ska +le mur +krakat au +kid swear +khu zestan +jam min +inde sit +iah sfb +heroes reborn +hal awa +guin nessi +grow ingu +gregor i +gost kowski +go dess +gi ac +gh orns +geme ente +gaj endra +fer no +feliz navidad +farm hand +f illion +excu sing +excer pted +eno teca +emo to +du puy +dong feng +devin book +datab ricks +dar yn +d formmva +cri so +crew member +con genial +comple xions +cine phile +cer ati +cap tu +brittany snow +bree den +book snaps +ati mein +ari hant +ange bote +anatom ia +amylo idosis +air shows +acam era +... ðŁĺģ +ðŁıĭï¸ı âĢįâĻĢï¸ı +ðŁĩ®ðŁĩ ¶ +yam auchi +wo su +witcheso feast +whor se +wan del +tree beard +tom ek +thre f +te end +sympathis er +sunday fishsketch +selfiefor nash +sei denberg +sa or +ro throck +rin do +re taken +pressclub dc +photogra p +persecu ting +pepp adew +o ves +nmb co +nick lin +ni ese +mthe thwa +migli ore +mi gi +mbalula fikile +matt miller +mal ory +kuber nete +koin onia +kil syth +kcaf av +kat ong +karun chandhok +jo stling +irreconcil able +insi den +in h +himan tab +gur gling +gh ole +gashap on +fun chess +fraser valley +fozzy rock +fer rata +f wt +elec tra +dir khooper +din akaran +denver nuggets +dal ecar +ct gla +corbyn besson +cap gemini +by any +bull pup +builtby bama +bix ler +bi xi +be ma +basketball er +ballo oned +ba bey +arm wrestling +angi oplasty +al astor +ðŁĶĬ : +ðŁĴĹ ðŁİī +ì¯ Ķ +ì ´ +æŀ Ĺ +æ ½ +âĺħâĺħ âĺħ +yi annis +video taping +va stu +us ury +uhur u +tor tuous +the conjuring +te bet +tar red +t swana +t jo +sy lar +swim min +swach hata +sto ats +statist acharts +stag gers +sher rard +sever ide +sekh met +sch ad +saved bythe +salvador dali +reve led +re convene +rah saan +pu lao +pla ga +ou lou +onenation oneteam +nick swardson +necess itate +muth oot +mo at +mis calculated +micro bus +metal blade +mel uck +megan boone +magic avoxel +m co +lo hia +lindsay arnold +li ub +klu gh +jant zen +jacob o +intar sia +herald hs +he ikki +he eling +halab ja +hahn dorf +glend ale +frederic lambert +france fr +ewn updates +essex wildlife +en m +elabor ating +eco village +duck y +de beers +da rell +cypri um +cycli ste +cha sen +capric orn +can ute +boston heraldhs +book making +bing es +bike week +best coast +benven uti +bayern munich +bath sheba +ban yo +ayanna pressley +ay b +autun no +au ria +and chill +alyand fila +ah hhhhhhhh +a history +wild turkey +whit el +whi pps +weare acmilan +wat rous +vo z +ver nors +ve res +vas arely +up un +un classified +tul lius +tide way +ti soy +summer underthestars +shil le +share holding +sd xc +scor ch +rady r +ra ki +q pcr +q ip +pla sa +peranak an +pel as +osh un +or oro +olemis sbsb +newsad l +newfound glory +ne vo +naku matt +n bar +music by +mon dragon +mo tus +mith ali +mil onga +mich ed +mer ril +marvell ously +mar da +m razek +lin stead +kn auf +kla k +khu l +kc na +july talk +jiang xi +jenny han +jennifer aniston +j aka +inter lake +imam hussain +ik f +hell en +h wy +goodwoodr rc +go thers +gen ny +gad dy +g ph +flu ttered +fle abane +fire land +fact set +exhal ing +en ders +dor ada +do gon +cow abunga +country house +car ledwards +breastcancer now +borgh i +bl att +be sler +be holding +barba dian +bab ita +b hind +avro homg +asan sol +aro sende +ar ps +anastasi ab +al aga +af ur += _ +ðŁijij @ +íĻĶ ìĸij +âĿ¤ âľĮ +⼠´ +you loved +yol ande +work ington +woody allen +wingless bird +wi wa +west wind +utcoach jones +up tv +un completed +tomor ro +tiny tales +thomas gibson +the baby +te te +te jon +stu lts +stra ker +star pass +sole us +so wski +snake head +shi amak +sfor change +see whati +schoen feld +s jam +roman ovs +rodri go +qui zon +que vedo +pre cap +pr ze +phil tufnell +pew sey +par atus +om gtr +neid hart +ne ad +mis shim +mend ham +marav ich +lv phantoms +luxury watches +low ens +logan lerman +li pson +leg no +kir schner +kang al +hurl burt +hornb ills +har ting +har bou +hage dorn +gop tax +gondol as +george washington +ge an +ep sa +du gger +do woon +dar ning +damon albarn +cri swell +cli ppard +chi ens +char ya +cas selman +ca etano +bé ry +bo ch +big wigs +bie ding +besse tte +berser ia +bel ton +bel lar +battlefor azeroth +ats community +at rice +asroma en +artic ular +art lab +ak mu +abstr actions +. ðŁĮ¹ +(( ((( +ðŁij¹ ðŁij¹ +ê² ¸ +ãģ µ +âľĶ ï¸İ +⬠Ľï¸ı +à¹ĢภĤ +y cl +x lii +world congress +v rain +ull rich +ugg la +ton the +toi res +tay ong +sumb astur +stein way +si rene +sd nd +sand ar +sam mer +sal mons +saint laurent +roca wear +resist trump +resi stive +rca records +pend ent +official s +nz l +nit ourist +neuro pe +nat alee +my sis +my l +multi plexes +more mi +mir ó +mil ose +migr atory +men fashion +medi afreedom +mad ilyn +loft is +lan ghe +kunun urra +kum u +jour nee +janine gutierrez +j cl +iro h +ig travel +hotstar tweets +hotell v +hor t +hor no +hehe hehehe +hard bound +haras ser +hahahahahahahaha hahaha +graci as +gla sse +gar ys +gan grel +gal eng +fre sen +exploren b +euthan ize +dy m +dragon lance +discla imers +dialo gue +dees nider +de sco +con fla +cio online +chel ps +c mr +bu di +boo ke +bo so +aure lie +anni ele +ang ill +al bomp +aber ger +__ _. +' -' +ðŁĴĺ ðŁĴĺðŁĴĺðŁĴĺ +اÙĦ ص +yu taka +yever after +will erby +wccb charlotte +ver tices +utic acomets +toast ers +thiop ian +telepath ically +surplu sph +sun nin +sun dew +srabanti smile +spectro scopic +spe icher +sorcere rs +smc gee +skin ceuticals +sk out +scy cles +sch ans +ru tile +rang anathan +quadric eps +q com +pur pp +puk kel +praise worthy +pipp ins +phar ms +pd h +p lied +or ser +obscen ities +nu bar +new sch +n vs +multip lies +mu lders +mo ye +mo gh +mil nerton +mikeo hearn +michi gander +mer rit +mens grooming +media works +meand my +ma hr +luc kie +loy d +long acre +ligh tin +khali f +key to +kauka una +kam is +john m +jack ers +is se +irri gator +i backthebirds +hunter pence +hin cha +hempstead town +frontline pbs +from target +feren gi +ethereum classic +elder care +east grinstead +e com +du pre +di i +davy hulme +dan ity +d kc +chaud huri +charlie chaplin +char trand +cap ing +cad air +bran ton +bordel aise +bat ata +basti ons +bas al +az uka +az mat +az aar +as ug +ar wa +any an +anglo saxon +agh at +aber dour +اÙĦس ÙĪد +اÙĦ Ø¥ +yyc flood +yis roel +yeou ido +women writers +wing stour +wi they +wi ddle +war dak +virg ilio +v bi +turn ham +trety akov +to taro +to gashi +ti y +through glass +ther isk +theli ons +thehockey india +tay ang +tail gate +swar ovski +sus bureau +support thearts +su ol +staustell brew +star dew +spart ners +seque stered +seal team +say yaf +sam ovar +salford devils +ren tacar +re ding +rc pi +pur year +prophyl actic +prati bha +pi anom +photo electric +pan icky +pan et +pak is +pa ree +outaou ais +on up +ob noxi +o sso +national voter +morphett ville +montreal ers +minu tes +michel rou +mi thi +lu ps +lot tie +loch nagar +lf cladies +letra set +lemu ria +le ber +labrad ors +krish nad +kint sugi +king shead +king bird +julie benz +jualan ku +info sec +in trust +head shot +hagu pit +gest uring +gav en +gar de +fan boy +fail safe +eit our +dri o +dou dou +cowboy bebop +confeder ates +che sty +char ros +br inda +be etham +bagh lan +azi oni +ator onto +art fight +ai shi +ag ot +ðŁĺį ðŁİ¶ +ðŁĴħ ðŁı» +ðŁIJ Ģ +ãĢĤ ) +ç al +zoon otic +zol ler +yus of +wre ath +word cloud +west woo +vo gu +unic ycling +un shaven +u toledo +tsu mt +trac ie +top spin +tn luk +tar ghee +tabde eli +surab hi +su hel +st assi +ss ants +spoo kiest +sh c +semper sum +seig neur +schloss berg +sch alk +sal ton +sal terton +ry x +rulesof survival +rhy mer +re iders +razorback fb +ragha va +q bert +pro logis +pre ethi +pown ers +pied mont +pat mccr +pais ano +not l +nith yam +nausic aa +nar anja +naj ma +moon shadow +min di +m musi +m bio +long ings +li geti +lear ner +kett ner +kc se +kal m +kad u +intru sions +il th +hu ka +hachette aus +guy verhofstadt +gre ts +geminic at +galli more +fru ta +fri joles +free holders +fli ghty +feld stein +feather ston +farne islands +far cry +er ite +el ss +eg is +eee ep +du ping +dr anger +down tow +do xx +destru cted +den ge +del h +dd ale +cor vi +cog ito +chi des +chauvin ism +cha ste +c ssa +burgun dian +bri ga +bill ym +ben ali +beau ti +bear doil +attitude mag +at ca +an den +americ agreat +alo e +ab ele +aay yy +a und +a bean +- > +ðŁļ´ âĢįâĻĤï¸ı +ðŁijij # +íĻĶìĸij ìĹ° +íĸ Ī +ãĥ¼ãĥ ī +yun que +ya as +y are +x country +world cu +waccam aw +visit europe +vad m +trau main +tot c +timess qu +tier garten +teaser tuesday +t fx +stro em +spacec amp +space jam +sot tawa +sor pren +son nier +soff its +sne tt +slu is +si bel +sheir gill +schwar zman +saltim bocca +ric ken +reg bo +re structures +r np +pukkel pop +pan ta +outdoor sy +oro chi +oni des +nutri a +nau ti +mrbr tg +monsterhunter world +mo ten +mal ahat +ma shre +luen ell +legen de +kr circuit +khilaf ah +kan aka +jo sa +ind berg +illustration hq +iam episd +hr ke +his ashi +hill yes +high fashion +hello world +he g +happybirthday ntr +ha worth +gre if +goon squad +gold tone +gold star +give me +gand onor +for tum +ea aci +e iti +dunnott ar +du mmer +drum step +depre cation +dan carter +cron os +crank set +cr èche +corregi dor +con lee +con cannon +clar kk +cla sps +char lot +casi raghi +bobby brown +bl andy +bas shunter +back round +at fc +ary a +arti stre +apple tini +agye man +a are +\(´ âĸ½`)/ +ðŁİ¶ @ +âĺħâĺħâĺħâĺħ âĺħ +à¨ Ĺ +whitley bay +wall coverings +vec chi +vang sness +vali ants +v ayu +tro pang +the asian +tham iz +texastri bune +tangi ers +sul le +ss rtg +sne inton +sho dan +shep stone +ross ouw +rob brydon +ram boll +pro co +plat ine +pis cis +pier ref +pc mc +oo z +on call +ol er +nak z +megaz ine +mayan smc +ma dusa +li mca +lang ston +la chance +kon z +kitz bü +king solver +jo ell +jab o +j ji +iren se +inas al +iam an +hat chim +hack learning +gé rard +greenbay packers +gr ms +gin ty +ge sti +fl ori +fid lar +faur é +fair man +ero ads +edd ard +dex trin +dem force +d weeb +cl ann +charlotten burg +cell block +cav usoglu +buchen wald +bru tali +bra zy +blo xham +ban ja +bak ri +baj al +auntie annes +ali us +ahar on +adop tables +actu ated +ab ct +ðŁ¥ ´ +æ ¹ +zi jn +zeis slenses +y oooo +y net +y gk +wy nette +weather watchnz +wa chee +voil Ãł +un dit +ty bg +ton ig +thisis me +thing sexpo +thegop jesus +thegentle author +ter mini +ta eng +sylve stre +stmartin spress +snow drift +shell harbour +sab ia +riet veld +retar dants +photo frommyheart +orator ical +noki amobile +new art +nescaf é +nc cs +n ades +mon gre +mini x +m chi +long sight +le marche +kre tsch +kin olor +kan za +jar rah +jag nation +jack theripper +j florez +iz on +instagram med +imo gene +hurst ville +how led +hazbin hotel +ham monds +gu ston +golf life +foursquare find +fig lio +f gv +em mag +drink up +downton pbs +demel za +crisp ness +country radio +consor tia +colin dale +coal mine +cine mat +centre town +capric orns +brook ins +bridge stone +bre k +bea vernation +bav aro +bak ary +an die +ðŁij Ľ +ðŁij · +ê Ń +å¹ ³ +zu sak +wi des +web ley +twel ves +ton it +thereal elvira +tha ws +tab uk +sun stroke +stand tall +ssi p +ss music +sp angles +so bre +sj r +simone tti +sim ha +shoe horn +shak thi +scientology theaftermath +sab lefish +sa inik +rsc anderlecht +ro ser +rick warren +re er +ra zo +preclu de +pr iti +por chestra +pen i +pap al +pal lete +ori an +or acing +ob h +na he +mol l +missi e +mis se +minic lip +meadow view +mcga hn +mazz u +market news +mar re +logger heads +lin deman +lear nin +land less +kur si +klo wn +ker see +k ichi +k hia +jul lien +jon gh +john mccain +jaw ani +infor med +ine se +hob nail +hi y +har jit +h sb +gent lest +gali lean +g ili +fli xton +fleisch mann +fivb volleyball +f ants +ey rie +eurovision songcontest +er gs +el bit +eidul fitr +edir ne +du man +du das +dis quiet +dige sti +dest itution +chim ilco +cep has +burjal arab +bur stein +bor da +blo hm +black hole +birthday gift +bell ona +bee zus +be aker +bam usic +ar nation +angler fish +ame era +all indi +ale kh +air india +acar penter +ac ffi +â¬Ĩï¸ı â¬Ĩï¸ı +wother spoon +wo kv +wit nes +vi aggi +van k +va ghani +thero ses +tam zin +t yo +supportsmall streams +steven j +speci alo +shaun watson +scap ula +s notes +rap music +precious metals +pic asa +p alla +op ara +no sta +neo gaf +ne un +nca i +nam ibi +min z +me thi +me kka +marse illa +mari ette +magni fies +m lo +le akes +latel ate +lat avi +kumar akom +kruger sdorp +kla ss +kinolor ber +kaz ant +kat usha +ity live +info arqana +in justice +i bt +hyun woo +hoom um +go alie +gly cae +gastroenter ologist +fa ite +epi graph +elast ics +e jo +do ernbecher +djer ba +dis engage +d ve +cu ria +courtneym elba +colin mochrie +co stain +cic lo +cher mside +chad lindberg +candel abras +ca dem +brooks bank +brandon j +bo sques +blood horse +bior xivpreprint +bi mmer +babe station +b side +ase ball +all dev +af fric +acon v +ðŁķ¹ ï¸ı +ðŁĩªðŁĩ © +âĪ © +unitedwe zag +uch t +trac ibraxton +tl railuk +ta oris +syn cre +sto kers +spind le +skop elos +sibusi so +shu bhan +se bad +sam po +receiver ship +re ats +puertor icop +prae torian +pe tanque +paul gallen +p sap +ox ic +out gunned +on io +odi sha +nur mi +nun avik +nicol ae +nag ini +mou ski +mis appropriation +min oc +mi mmo +mi mick +lar o +lanz ini +kirin yaga +khair ul +kat lyn +kar ne +job hunt +jack lyn +imperi alists +immobili zed +ie styn +i xl +hot mess +hey don +hat illo +hardrock hotellv +got season +gon etoo +go id +gare y +games youloved +ga stric +france schi +felden krais +express ways +el yar +disbur sed +dareal amberrose +d agan +clairec mc +centen arians +cam girl +big sky +ber u +bar rack +ap rakash +and tvofficial +amé lie +aman ah +am adi +aller ia +ah en +acffi orentina +accep tability +ac ct +ðŁİīðŁİī ðŁİī +ðŁĮĢ ðŁĮĢ +ðŁ¤ ² +ðŁ¤ ° +wor ley +wood mere +way up +wau conda +watch mojo +vo h +victori ana +v aga +uy gur +up skill +unitedwe win +underestim ates +u ppi +three houses +thinku hi +te ms +switch to +swee ty +sut tie +strathe arn +space bar +south tyrol +shi rou +save dallas +ruck man +ru zzz +ro stam +rj mitte +rex dale +ratch ford +priv ates +pol sky +plata forma +permanent makeup +pa eds +or ph +or donez +one m +nordic walking +nederland se +ncaas occer +national isation +nan ite +nam ita +movie time +mo kka +mary portas +mar len +ma icon +ma gui +lo chal +lo bs +lieu t +learn lap +kur ma +king snake +kee sha +kay an +kak ak +kab am +journe ys +jonmitchell itv +ilove travel +il g +hunting don +house dems +higa shi +he ade +h dn +golondon knights +gir ton +freddy krueger +fre md +fe dez +er land +enqui red +earth wind +ear dley +dove tails +diet coke +dews bury +delici osa +de go +cycl in +cy g +cur ri +cho sun +camoufla ging +black sheep +bch l +bay bridge +b chat +av ag +ameli a +am ami +agne epath +ðŁį» ðŁİī +ðŁ¤Ĺ . +z dar +yl enko +yak u +w canada +triumph ing +tre p +thi rai +ten aya +sty lec +stren ding +sq rt +sm day +si ran +shi zz +sher itage +sh after +sf opera +say something +sanctimon ious +qay yum +q ft +proc tor +prenup tial +pocom oke +phili stine +paren tage +pap ago +pa ke +over winter +ot suka +noah syndergaard +ne bel +mukun d +mif sud +manic ure +mac bride +liverpud lian +ken o +just inf +jeke vaaste +jan jua +j anny +ib ma +i ello +hermosa beach +hau gh +han ak +gum midge +ge table +garden party +fur rows +free way +fashion addict +farqu aad +electro acoustic +ekdu jekevaaste +ds ville +dor ling +do sages +di ye +dan gel +dag ibee +crypto exchange +congreg ants +bus u +bulldo ze +britt le +bi f +ay ner +auto sports +ase d +as ka +amazon basics +ag ir +aa ve +, ..." +ðŁĵĦ : +ðŁijį ðŁijĬ +éĻ ¢ +zoro astrian +yofthe year +writer sofinstagram +wick enburg +whittle sea +whites boro +vide oc +un ction +umass boston +tho de +thibo deaux +ther ick +team trudeau +stol ler +spec tra +rush en +rose s +ram la +race goers +pre cursors +pra ful +poke stop +po vo +park hotel +ontariop cparty +oj ha +o ca +nou is +new al +neg ated +n ase +my heritage +mour ner +mit is +met so +mb achelet +mayor an +manning tree +mand ing +m hall +lu ong +love jo +li therland +le vent +ku hoops +koop man +keepit real +kap iolani +jjig ae +in costabrava +ic ss +i vin +hun n +hu tter +ho kies +halvor sen +for victori +fas ching +f itur +f ady +er gon +desic cated +cow ls +cho lec +cho ix +cash more +cal om +bush y +burn twood +buch man +bb fc +arn ason +aph ili +ai man +a stri +ðŁijıðŁı»ðŁijıðŁı» ðŁijıðŁı»ðŁijıðŁı» +ðŁİģ ðŁİĪ +ðŁİ ĭ +оÑĢÑ Ĥ +za her +z wick +yar de +wi gley +w ys +unit eds +un satisfying +un pasteurized +tz aneen +tur bom +tre en +team mtkglobal +ta fo +swan igan +survivor man +surfl iner +storme mma +stin nett +steve kingia +shri veled +sea bee +sak halin +pizz eri +per gam +orland i +o tras +noti f +new business +mis representing +matur in +malag acf +long neck +lo thes +lo tf +live atthe +lat ed +lace wing +krysten ritter +keyshi acole +kany on +k ys +k pu +inve rell +inde terminate +in anda +ill iterates +il icon +iam a +hydr ator +hu uuu +house uk +hm recipes +gra zes +gior gos +frame set +faren thold +er roll +dukin field +du lin +dor dt +door knocking +do gie +dj d +destruc ting +dapper laughs +crow ed +collabor atory +coel ac +car lino +bun mi +back handed +as see +arty originals +afell ay +ab bys +a yoo +ðŁį Ī +vin it +ur an +tu big +tri stes +timmerman seu +terrori stic +tant allon +ta ward +sub h +steph y +skull cap +simon shield +simonshield cars +scho o +saw bridge +ra jani +patmccr orync +par cells +p band +oun ie +oc md +non nie +movethe sticks +me rel +mar galla +madd alena +lef tism +kr be +kassi an +jugg lers +jam ai +itv lorraine +itson us +inhib itions +impe des +hou lton +h te +gov con +gondo lier +galo shes +g pio +fox hall +followyour heart +florian sem +edinburgh fringe +dre ssy +do et +digital selling +d boss +crazy sexy +cou lom +cost asunglasses +competit on +co zens +chri slo +char o +ce cafa +brisk ly +bol aji +bill ymiller +be edle +bad things +bab yy +ay se +ave ley +aun or +au pdate +artofli ving +anno ck +amey er +ðŁİ¸ ðŁİ¤ +ðŁ¥ ħ +âģ ¿ +wysi wyg +w dg +veri dge +vai zey +transni stria +tom mison +toi les +team db +tang an +tam an +swim mingly +super heroine +sumr all +stein ke +stall ard +stag nate +spind les +south african +sig ils +senti do +san sone +sacri lege +ry mer +ru sses +republic anism +repe als +re mise +ra uma +portre ath +phi leas +paras ail +oney oung +occu pied +nur haliza +noisi est +ni antic +music videos +mr tt +mob psycho +mo phie +michael hyatt +med ju +materi el +mat tar +mar shab +m go +loth ario +lock sley +lau ter +koz lowski +ilustr ación +hal lion +ha ddin +ha bs +green shank +golmaal again +gav riel +g auth +ft as +foot lights +fictionalcharacter si +es guer +e zy +e spectacular +directedby women +dervi shes +del lave +davids ons +d tid +congress h +conceptu alizing +cape epic +cam es +bung ling +bre arley +bhan ush +bengal fc +bbcgoodfood show +aussie wine +as pera +arti um +arro gate +arac eli +antonio selas +anthrac nose +ameri prise +am man +ðŁĮº ðŁĮº +æ² ¢ +âļ« âļª +zephyr teachout +youth fulness +yeh hai +wit trock +wal dau +verbi age +valenci ennes +us nist +usnist gov +un ambiguous +ty red +truetothe blue +tom coronel +thimpact music +tek tronix +teab ags +tanu garn +st apes +spo cus +shor r +sand box +salesforce tour +saint louis +sa hra +pic ken +personali sed +peri gee +par ky +par cc +pand al +onna ire +national triviaday +nas co +n pd +mp ca +mis interpretation +marinas diamonds +mar am +mac callum +lyric al +loo tera +live stock +lev elland +ko zy +klin ik +kat onah +karanvir bohra +k nave +k mw +jewelry designer +j ør +it ami +ira i +inde p +ill in +id omeni +i eper +hong kon +ho chul +haw ay +happis burgh +hal per +gu ji +grobb elaar +gar gamel +gan sett +fi bber +festival season +fem min +fate grandorder +fac ademy +expos itor +ela unch +dun don +discre tely +de funding +de bono +dayof service +cringe y +com enius +chun king +chou pette +chance hmiller +ca key +bottle brush +bhu mika +bewilder ment +better world +ball park +bad tam +avo d +al cin +ai mi +abud get +/ \ +æķ ı +zuc carello +whack y +westy orkspolice +westmid sfire +wai spr +uje res +track ball +tinyrebel brewco +tej pal +sw coastpath +su ku +su god +stress free +sterili ze +stake out +sm show +sini esta +sil ove +shel duck +sensu ally +scott zolak +save therhino +sam vad +ry ant +riz ky +ri sed +recompen se +ra kers +r ml +pitney bowes +pic ally +pet smart +peri stal +penrhy n +pap anasam +ol fi +ok ello +nor den +nl h +net worker +nano wires +mön chengladbach +muck le +mu sser +motorcycle live +micro structure +me soli +mclar ty +lolly daskal +len sky +leather wood +la sing +l ancy +klu ge +khar ghar +ke zar +kar yotes +kal er +jag at +j twc +in vision +hydro chlor +hockenheim ring +health sci +gu anab +green wave +gods girl +go argos +fur ugby +execution ers +est v +errat ically +eric stonestreet +engv all +egg leton +du um +dramar ama +dor gan +dok tor +de mexit +de mas +cu mmer +cryo em +cric kla +concier ge +char dt +cafe oto +c illian +buy local +bro glio +bol z +barnes ville +au tol +athle tes +asimbaj waispr +arch ways +anglic an +ane ja +aedilhai mushkil +advoc acy +adamm grant +ðŁĶ¥ , +zak is +young bloods +yam ini +work man +war danadi +wardanadi adwala +up mc +tric are +thanksgiving day +ta ches +sydne yleroux +sw c +spin ner +sirot kin +silver sea +silver fox +siberian husky +si fy +sh ac +sali endo +roman os +roble do +ro hani +rival ing +reu ther +re uss +re make +ravin dran +quil len +perpetu ation +par apan +ordin ate +or ski +ol um +ol ave +ohl draft +ogc nice +o hy +non conference +no zze +ne wald +my nydd +mer folk +man tell +main ec +mae us +ma sel +love cork +likk le +lich tenberg +leeds beckett +lee za +leaf less +le det +kott ke +kno wer +ki wa +ker im +kel se +k flay +ju uu +ju tanugarn +is me +ink blot +hoag land +he an +grou pm +gol fs +fur n +franks redhot +fj ell +fabul ousness +ever long +dv n +dra enor +dopp ler +distur bia +dirt track +demp sey +dam ini +cracker barrel +con ation +cinnam inson +chin nor +cau d +cam pos +bro ga +bor re +bh b +as under +am ine +alab amians +ak wai +afton bladet +ad alovel +acci ona +ðŁĺĩ âĿ¤ï¸ı +ðŁķ¯ ï¸ı +ðŁįº ðŁį» +zab bix +yot suba +x xix +wego hard +vivek dahiya +uver world +tro ad +timp ano +tel is +sper oni +son uk +sm f +sig gi +shrie ked +school supplies +scho on +san é +s dr +rust lang +roden bach +rock art +ro sac +riseof rest +realdj premier +ra thyatra +qui ds +porthcur no +polk adots +per io +penguin books +pe pino +pdx now +ok ays +no em +mosk va +mis ms +mcgill is +matsu oka +man zoor +lu tter +lis ner +laser scanning +la chaise +ky tv +kim s +kendra wilkinson +jessi ej +hygro meter +husse ini +hu ub +hockey tv +h ba +gui der +gb d +gagli ardi +fio rella +fel tz +equal it +eco schools +eaton ville +dou ts +dou mbia +den holm +coer ce +cli mat +cardio id +campu slife +bot an +am oureux +ðŁĶ¥ ðŁijĮ +ðŁĩ®ðŁĩ¹ ðŁĩ®ðŁĩ¹ +æĸ°å® ¿ +yam al +y strad +y pr +writ enow +wish meluck +walter scott +wak il +uf bugs +tin ney +the box +thanks dad +th ung +tat ort +ta issa +swi vels +stormb orn +sk ai +sig int +sad diq +rwen zori +rudol f +reck itt +ravens bourne +profit ably +pro mes +po go +placido domingo +pit ons +pim p +pic cin +peter tatchell +pedre ra +pay rolls +objec tifying +nj dotcom +ne j +mol ca +mini mizer +mc nicol +mc are +mat they +mahi eu +ma isons +lyn nette +lo ssing +lat form +la khan +kr ama +kent ico +kahn awake +k lax +jit ney +jazz wise +is king +inver leith +inst ad +ib ile +i ok +hy ar +horse less +home boys +gun nell +greatest showman +gla zes +fri sson +flos stradamus +flexible working +far in +falsi fying +escape es +emili o +dete ctions +de val +de pa +dand ad +curi el +cp sa +cold spring +cine main +ci dium +chanel westcoast +buzzy buzz +blat che +bind weed +bart leby +balochi stan +aw sreinvent +aus grandprix +alpha go +all ora +aaa ahhhh +: ? +!!! âĿ¤ï¸ı +ðŁıĥ ðŁı½ +룬ë¸Ķ리 ì¦Ī +âĻ¡ . +âĭ Ĩ +âĨ Ķ +Ùħ ÙĬ +zy lof +zor don +zal de +y itz +wy ne +wild bill +we ke +vali m +utter ances +un willingly +un profitable +un anticipated +ti mba +tdw sport +tail spin +t dci +syste mo +sway ne +steven sville +steven ash +stanc enation +sony prousa +sky digg +sem pre +se q +sawbridge worth +sau con +sar af +san die +sahar a +rown trees +ro tolo +ran chos +r row +pony hour +nol ans +mu ji +mm rda +mb stadium +ma illard +lovetrump shate +lic or +laugha bly +kam uy +ji j +j any +inconveni enced +hud ds +hippo cratic +has sel +hanse atic +ha aha +h mrc +h mb +gyne cologic +glo ating +gen set +gel sen +gangre ne +fr ons +flu gel +fictionalcharactersi wanttomarry +fair uz +excit er +ev elled +eni ke +ele fante +ee zy +eddy stone +eco tec +dum mett +dm register +direc tionally +del orme +deer hound +dang dut +cur se +country girl +cleve rer +church ville +chand ani +chael incl +cellulo sic +cast aldi +by one +but lin +bu ehrle +bli s +beyon ce +best wishes +ber rigan +as vit +annul ment +ama q +alli seeis +aar gh +âļ½ï¸ı ðŁĶ¥ +âĺ ŀ +á Ķ +yw ca +wood worm +wis la +what su +west lock +w use +un crowned +u os +twy nd +tran ent +tiffany haddish +the knot +thankyouforyour service +te ssy +swit che +summ erishere +sp ir +so hc +snetterton msv +sli v +shre m +serv icec +seraph ina +sathy ajyothi +s llp +ri fling +ram eau +porth os +porsche forsale +por thour +pet plan +pel uso +patter son +p had +ox uni +otran to +o delirious +nor win +nitourist board +mizu ho +mishand led +ma sini +lum by +li os +le vein +kri palu +klo sterman +kit agawa +ipp on +infl ates +ili o +hydro x +hr sb +ho vel +ho skin +hat field +hann ya +ha el +h dt +grant making +gra hams +gol in +gh and +gel fand +gd pr +fri g +f anti +ex erts +eni sta +el ara +duncan trussell +down land +dellave dova +dac ey +d ctf +commun alism +cheftom kerridge +cerv elli +burnthe stage +brink mann +bok hari +blumen auer +bis aya +berg tv +bekind to +bau dr +australopi thecus +annon dale +ak anishi +ai ac +adu cati +ðŁIJ ĥ +ðŁı³ï¸ıâĢįðŁĮĪ ðŁı³ï¸ıâĢįðŁĮĪ +âľ « +ä s +za veri +yogi adityanath +vy dra +vote demilovato +vas s +v itt +tra ver +threep ence +thiru vizha +tail bone +sway amb +suz ann +suicide awareness +sugar bowl +st illed +soldier field +smu gly +sb v +rou se +re conditioning +q cs +prin ze +pmk vy +pi planter +panam acanal +over loads +or bi +olom bia +ob x +o gn +noble st +nam san +na fees +mu mab +mkdon sfc +min ky +mi ano +men or +meet our +may hall +man tv +ma vicpro +lulu hru +l ation +kwesi arthur +kate spade +kat rant +kalki kanmani +kal barri +jocel yne +jesuschri st +it ravel +information technology +indian diplomacy +hyp notic +ho tho +hinch liffe +hilton head +hell hound +ha wards +grau pel +gonebut notforgotten +ging erly +gin kel +fox ford +external ities +er rico +ee v +di mmu +de population +de muth +darkk nightrises +d dot +cl ane +char nley +cal af +brut alized +brun ches +bram alea +bey routh +are ata +ap hex +anti e +am logic +ale sia +al quds +air tricity +:' -) +( ??) +!!! , +ðŁĶĬ ðŁİ¶ +ðŁĵ ĭ +âĪ ŀ +° ðĿĺ +yal inetwork +womenin engineering +wilkin sburg +wail uku +visibil ityday +vi alli +tr onic +total biscuit +the evil +ter ang +suff ices +spencer port +san del +sag t +robo twars +ro guer +ransom riggs +py are +pr week +plu cks +pin cho +phili stines +organiz ed +ol z +ny v +not our +nikki reed +news media +n kun +mish ra +mb and +masochi stic +mar ang +ma kel +lo kay +lil bibby +lev antine +lc dr +la ettner +kere mbur +kaw aguchi +kath iel +ka sten +k rome +jay rock +inter provincial +incub ated +hundertw asser +henry viii +he eds +gre a +gov tnz +gill ick +gener alizations +fro bisher +fow ls +foo m +fnaf hs +fly ball +fir po +explore the +ev aaa +end child +dun combe +de mption +day dream +cor vina +conve yer +confu ci +colo m +clums iness +calci fication +bra sher +bbc cambs +assor tments +asgar dian +ard agh +anag alizia +ali u +af fix +adri ane +a heim +ðŁĺįðŁĺį @ +ðŁİĦðŁİħ ðŁİģ +ðŁįĢ ðŁĴļ +âĨ ©ï¸ı +worldsoil day +whodoyou collect +wasteiton me +vol le +vel lam +v hong +v adap +ur inated +upper class +under hand +town scape +tam bur +tach ometer +t do +strat orob +steve burton +snoo t +smo ker +small streamers +sla u +siva thewanted +shan te +seok min +season able +sahuar ita +sage francis +sa ham +road burn +ram bert +rally racc +ra wat +princess leia +pla stique +pen iche +pe therton +pang lao +panam games +orig ami +or questa +na shies +mu table +mon cur +mess o +men sur +massi modu +marcel o +m travis +lovemy dog +lloyd banks +lie ben +leix lip +ky dd +jab il +is ao +ic aria +hi ero +hawks worth +grave sham +gen ation +gard eng +gar owe +fox ct +flor id +ess on +eric whitacre +ent radas +elo ka +e owyn +e akin +du dgeon +du ce +demo scene +day glo +dant dm +dani elo +dab arkads +cuti ee +country club +cele k +car onde +cal allen +buy ers +buck hurst +bor ou +boett cher +batt enburg +bas singer +arnel pineda +ar nou +antim al +anne ka +alter a +alp bach +a ethel +ðŁĴĽðŁĴĽ ðŁĴĽðŁĴĽ +ðŁĴ ł +ðŁİĵ ðŁİĵ +ìĬ¹ ìľ¤ +èµ IJ +æĽ ľ +âĺĿ ðŁı¾ +ø r +ê me +yon du +wyn on +wo ssy +wel ts +vo e +torch bearers +tit lei +the cutch +tam ines +taf sir +stick iness +stal liance +spi erre +sophistic ate +sin less +shar af +se acade +schaff hausen +roller coasters +rock slide +rapo port +prun er +poly gamist +polari zer +pol avaram +pm dd +plant science +par to +onceupon atimein +nt southwest +neme chek +ne shoba +nay la +muth arika +mercu tio +me ares +ma homet +m buy +line tte +linda ikeji +lam oureux +la yo +kum ho +kla vier +king field +kap lan +kab aka +hypnoti sm +hin king +her mann +h wb +gre ymouth +four play +ewn traffic +esp ino +epiphan ies +ela sti +del orenzo +d bf +cull ompton +cu bbon +cs ny +cra dled +cost lier +confederate flag +con formation +ci legon +chapel town +chan u +capp ie +biomed ical +bal want +b sas +as cio +argu mentation +angliar uskin +ameracad peds +acce sses +abhin aya +ab ts +ðŁĺĤ ðŁĶ« +ðŁĸ¤ ⾨ +ðŁĴ» ðŁĵ± +ãģŃ ãģĵ +what matters +war of +wan jiru +vero beach +ul bricht +tyro lean +tromb ones +tre vis +the shark +taga q +suf cofficial +stom o +starbuck scanada +ssin z +snowmag gedon +ski pp +si mard +si at +ser j +senator menendez +se il +re write +re tested +ram nath +ra sco +pre destination +point es +pe azer +online safety +o chieng +nor mand +ngu gi +naz rul +national hatday +nation all +mr bo +moon less +min niem +matt lanter +mag ner +mac omber +kuri su +ku mail +kor k +ki y +jones borough +jo ep +jack savoretti +ish u +in lets +illeg ality +hb g +gar brandt +fra ying +fasci as +e gan +de etz +contr ition +cny central +clu bb +chero kee +blues rock +bj ørn +be eville +barn field +bar ako +astor ino +ar nau +ap lay +ant ó +ant é +am ck +world notobaccoday +wi f +way uu +vir i +vi brio +un happily +tiger lily +thekk ady +the jason +tam pere +sk omer +share pict +shanemc mahon +second city +scru ples +s fair +ryu kyu +rodan the +repost by +regin aking +ram kapoor +pin oftheday +phyl lo +pe afowl +op hen +on co +ol ka +ober weis +museo ideale +mon ad +mile stone +mi kay +mel chor +matt and +man ac +mal lup +m amp +lyle lovett +leea renberg +lea den +lamba alka +la hr +kun is +in famously +hen ner +har ve +har mar +go is +ff xv +evol ver +ess ilor +dz bb +dri ppings +drew barrymore +dog pound +dis avo +dg ca +del piero +deb ary +de militarized +de mean +cor de +confu singly +comp ston +cien fuegos +chi b +cc me +carri ere +car tm +can thelp +brox towe +boss y +bone tti +ble ier +biblio thek +bhar uch +ber kut +bbcgw live +au gi +at ag +anim atic +and oni +amil ano +amar ca +a ani +ðŁĴľ ðŁİī +ëĬ Ķ +è ® +z by +yas mina +wri ggling +visit abdn +up scaling +under takers +umay yad +ukgif tam +theplayer schamp +tat ry +taq wa +tales fromthe +super grass +sunny bank +stone field +ston ef +stey ning +ste ads +skincare tips +sho v +she hr +sg pc +ru mps +rizz uto +pin tor +pick away +phy to +petit e +pax os +over tone +op atija +o gwen +nee wer +necess itates +multivit amins +mu sil +mu ip +mor ros +mo ki +mix ta +min def +mee sha +marin us +ma die +lex perience +legisl ating +king i +kenyan traffic +ka idan +k harbanda +jun krat +jon m +jb pritzker +iron work +insuper able +infini x +i jaz +hyperhidro sis +head teachers +han de +ham ster +had win +h ace +go yer +girls bb +ft u +fitz geralds +fer ring +fc united +ev d +esc ol +enligh tens +enginak yurek +ele vens +edit ore +dolore shu +deaf heaven +dark sun +cycla dic +cubic les +cu buff +cre morne +commen taire +colum b +bore as +boom stick +bluet ooth +ballinas loe +bal gow +b inga +athe a +aste iner +arsenio hall +argentin agp +aren delle +al friston +ag aves +afol ayan +a oy +a jen +) ', +ó¾ ĵ +é § +ãĤ¹ãĥĹ ãĥ© +âĺº ðŁĺĬ +Ø§Ø ´ +zil lah +za hoor +y aaaaa +world humanitarianday +w up +vi le +vampi ric +tune ful +tshirt design +track ing +ton bridge +tish omingo +the coral +th axton +terribly tinytales +teacher appreciationday +swa inson +sound set +smar my +short land +san jaya +s mid +ryan holiday +ru ddington +rspb southwest +rivers dale +respec ter +re beli +rahul dravid +posit ron +pim pin +pas sey +oromo protests +organ elles +oc ar +no ahs +ne ophyte +my idol +mur dere +mor ri +menstru ating +ma fc +lucifer onnetflix +lili an +li sted +laure en +laten igh +labour party +la plante +ko zel +kitty hawk +kar hu +ju lee +jim myeat +jeep ster +jaye hanash +ir reversi +inst yle +idoli zing +ic pd +i bro +ho sni +hlf supported +her rig +he bs +galler ist +frasc ati +exten sible +exeter college +even ko +es b +ear tists +disarm hate +din ard +de criminalisation +dc pl +cynthi anixon +cont actus +colom bians +co an +chi haya +cc cs +bra ganza +bol ter +be avoter +bank stadium +ast ell +asperg illus +asi apac +ann ée +ann aa +am bal +alonce sto +all ana +aege anairlines +abet z +ðŁļ µ +ðŁĺİ ðŁĴ¯ +ر Ø© +ÑĤ а +ze hn +ys rc +whats new +was p +vi agogo +vegas news +vassil is +v dj +ush ima +un traditional +thursday night +talkin gheads +ta was +sw ch +sw abhi +supply chain +str ina +she amo +scott borchetta +sche ch +sc ouk +ron funches +redon do +qu aide +prolifer ate +play ingnow +pe ered +ol av +ny anga +np tech +national pieday +my view +miss world +medi o +mcle od +mc gil +mac ia +ma vs +ly t +lu anne +kon nen +ka sese +juan ma +ju alan +ing ate +ildef onso +hydroly zed +hu ong +himer ose +hill ard +hege monic +gri der +gra al +gi blin +game boy +flu g +fan zines +fall is +every woman +ejec tions +early childhood +di mera +dar ting +dar ted +dar ra +comm er +city traffic +cb bus +burde kin +bro ths +brett dalton +boiler ball +bedroom tax +atsu himerose +astro labe +alfred sson +ah y +ae ats +ac lassic +" [@ +ðŁĶª ðŁĶªðŁĶª +ðŁIJ¶ ðŁĴĻ +ç Ģ +ãĤ¿ ãĥ¼ +âĮļ ï¸ı: +z addy +y stems +wa inf +vesti do +vacation mode +v pro +ul le +thur rock +super nintendo +su sp +stjepan hauser +stal act +sou tache +simi en +search engine +sc ill +sas aeng +sacred geometry +rome omiller +rn ts +refr acted +real michael +rad cliff +pri es +pre uss +porch light +official sps +o logic +nascar hometrack +nap alm +nag pal +movi mento +mon chi +missing merlin +mathe ba +ma pper +m wh +lt d +lilloo et +ky p +kodan sha +kir cher +jd illa +iv ani +indi visi +in laws +in chic +ik r +i vie +houn sou +hay ton +haku ba +gu ti +ger gen +full stack +for sey +fabulous finn +degra ff +dat aran +dam ocles +da hal +da chi +cro is +clou se +che ers +centr ica +catal ana +bur chell +bry ne +blox wich +blood worth +black business +bir tday +bi beau +andre siniesta +anaam doon +alv ina +almosthe aven +all ach +al jaz +afl vic +ad dam +achi efs +ðŁijŁ ðŁijŁ +Ø§Ø ¬ +za haha +x aver +with kids +wh yyyyy +wen dig +wee ze +w taf +vion net +ver celli +van af +tvin colour +tor ist +the resi +ten leytown +tal bum +sweat box +sty li +stre sa +spoon bills +sorry im +son yo +smur fit +sin c +sidd all +si phoned +si fton +seismo logist +ro see +res wales +rally deportugal +rainbow six +que k +port coquitlam +philosop hie +philly firedept +phil brook +oto ya +nidh hi +nab u +moo cher +mister wives +mi os +merrell twins +mccas lin +mal feasance +mac naughton +ma saru +m wr +m tam +live journal +le chu +kib worth +k bw +jason plato +j rothen +inzam am +hover boards +honey combs +homin in +her ro +hard hat +ha hohe +gou die +football remembers +evi gan +du ba +dol ing +dir hams +dinner date +cu ala +crag side +chesapeake shores +cast o +car lucci +ca rella +businessc lass +bur kes +brew day +bol tz +bb bb +ball ater +babad ook +alex isonfire +adjun ct +aber fan +/ -. +ðŁĺī ðŁĺİ +ðŁij¸ ðŁı» +ìľ¤ 기 +à¸ģภ£ +yo ichi +y ı +whitecol lar +we dont +uof alabama +unner ved +twelf thnight +tu gend +tr ca +tom son +thegoogle images +theat tic +the sse +t ms +sven ska +sug ita +sm tr +sil urian +shol me +sel insgrove +san parks +s como +ru pi +rock sound +pro bus +po stand +plastic pollutes +paediatric ian +nbc timeless +mun dry +mi stle +menac ingly +mccu bbin +mather an +ma sum +lev ins +l hl +korean war +kai ji +ka whi +jb fa +j ck +intercess ory +il wu +i hp +hine sville +high on +hene cia +happ old +hang time +hand l +hammer films +halloween party +hall sville +grim ms +go ags +glen ny +g ner +g ads +fire pro +fal les +fal lah +f nv +expan ses +ex pel +environmental justice +en zy +electro physiology +ds meu +dread naught +dor in +dian ak +cu hk +cocom artin +cis me +ce g +carol ines +car star +boney kapoor +black white +bjarke ingels +bar ringer +bar log +bad ung +adre ssing +ac snano +ac crue +!! ðŁĴķ +ðŁĵ ĺ +youlike carsuk +yanke ec +won tons +wom bs +ving o +victori ously +vec chi +under perform +un rolled +uk ur +tv n +train ingday +tow v +th ooo +tb ell +t ounge +strat asys +stein ert +sr ing +sobr ino +sli iga +siss inghurst +si pa +senator reid +sen robportman +selve dge +sel mon +sel inger +say cheese +sa wat +rekind les +re route +rc de +ra gan +qu n +pro van +pivo t +pedic ures +pancra se +pa kk +out selling +our nation +oregon ians +niam ey +nation hood +n crc +msmar vel +morad abad +ml j +mine ko +mil fs +mesm eric +men chies +me gara +max ing +mat tek +laga res +kyoku shin +kl h +kirstie alley +killing it +kee sh +kat ed +josel yn +itv anglia +ine ed +incenti vise +in den +in cin +i dra +hyper links +hu uu +hash ana +gre nell +gravitational waves +grat ton +gg able +fun dthe +fan stand +exhilar ated +em mar +eli v +elder abuse +el tz +east wick +ear ings +dire kt +dev our +democr acyday +dee puk +cy ano +cu ssler +condition ally +coco oned +club wc +civil airpatrol +car luke +by fleet +bore holes +ber satu +barrel led +bar ths +ba hir +b pd +ashi shians +ar kh +ar iss +apor ter +ao ife +alwaysinour hearts +al ofa +air nz +adeni yi +adap t +. ' +" âĢĶ@ +ðŁıĢ âĿ¤ï¸ı +ðŁĨ Ļ +èµIJ ç¦ı +ãĤ» ãĥ¼ãĥ© +Ï ĩ +year son +y res +women also +wk shp +wi ya +versic olor +ve aux +us fca +twin ny +tour é +tor adora +ton tour +tho spital +surfer girl +steadfast ly +star sof +soil association +shru thi +she epi +schei fele +saddle bags +ronnie wood +re payments +ram ayan +rad wimps +quin tino +pink pop +per missive +p sei +open the +omni verse +oma ine +ohhill no +official nichols +o sher +newyearnew you +national pancakeday +my nt +mun chau +mirac ast +me zu +me self +mass enet +mar pole +lm h +liber atore +lei fer +l ling +kuns thalle +kon tak +kon nan +kk l +kirk franklin +key board +kc as +kab u +k por +jean michel +it group +is that +indi c +hor lo +hon go +hof man +head butting +h vo +green collar +gravel ly +gin blossoms +ger tz +gar din +flu tters +ff f +eth ridge +entre pen +enf ance +ed sall +ed di +du shy +dri es +dolla z +ded rick +d aka +cre mant +coun ton +cou libaly +clap back +city beat +ca stor +buj old +bli zz +blackandwhite challenge +beach bum +bayone ts +artill ery +ap or +andy mineo +amyra dastur +alber o +alas kans +ad cs +ab io +ðŁĺĦ âĿ¤ï¸ı +ðŁİĦðŁİħ ðŁı¼ +าภ² +worm wednesday +willi mon +whip sn +watermel onday +wat kinson +wal ki +vsc in +ue hara +tu itions +truec ar +traffic butter +to ks +thunder shower +thir lmere +think progress +t dc +stenc iling +sne ddon +sky liner +skol kovo +shu maker +shi rish +sch ic +samuel milby +sacrilege sunday +ridel ots +remington leith +red lining +re tr +rachelriley rr +quetzalco atl +quer ra +pyrr ha +pan abaker +pal utena +pa ku +o wego +nz vind +nytime sarts +no bly +nj pac +nar ada +n can +moor abbin +mon tt +mollu sc +mate ys +mal oo +lym sm +level design +ku tt +ko ori +k pandey +jap on +itsmo hit +it ron +inausp icious +ike ausa +hip flask +hallucin ate +ground cover +ge tor +gar ron +full circle +for food +fel onious +ever ard +eun bi +ensla ving +encan tan +eme k +eli gh +eeee eeeeee +edg cumbe +ed research +ear this +dow deswell +dis qus +delinqu ents +deidre hall +corpu z +cooper man +co ti +chu tiya +chri ster +channel ten +casser ly +card fight +boot legging +bollywood celebs +be ith +bang lore +b wy +aun ite +as om +argu able +ar ny +aqu arian +ao v +an kush +alas vegas +af obe +ë± Ģ +å¼ł èīº +york dukes +wrigh twood +wiel dy +wi fw +west michigan +wam pum +vic oladipo +v nv +type set +tu z +ton ys +threat ened +thou sing +tellem stevedave +t px +swar oop +stown hall +spring s +spo onie +sharps burg +shan ka +san gue +ricor di +referen tial +red bank +re tall +rad han +qu ba +push chair +pre scott +poin sett +parik rama +pang u +p tr +om gosh +night sof +nh v +mu bank +mo ise +mat to +mar tavis +mar quis +lu zia +lo ofa +lin ck +let tie +king sday +killor glin +kh p +ke ad +k int +juliago erges +iv anna +isla scanarias +inge vents +hypo thalamus +hou k +hel mut +gir ll +game iro +fir stamendment +far aj +dutty paul +dp show +char sadda +caronde let +carni vor +cak ery +brook field +bottom ley +border land +bo co +bless ington +black hall +beg bie +be ynon +baudr illard +bat suit +b dw +b con +azeali abanks +ati f +arjun reddy +amar r +all ington +alig ners +ald in +ag adi +aes ar +adam j +ab bess +!! âĻ¥ +ðŁĴķ ðŁIJ¾ +âĶ Ķ +zylof on +wreck it +whir ring +web mail +v ade +tyn tes +tor qu +the change +tel o +te chi +super friends +succes fully +subter fuge +spr inci +sp era +sof antastic +so sn +sho to +seag rave +se say +sardon ic +saeed ghani +sa kae +ro ject +regener on +rad ner +qu ism +put locker +psl grandprix +produc ciones +pemb ury +patho logies +pat te +oscardel arenta +oscar s +orangu tan +ok state +nav yy +multi scale +morbi han +mor di +metal music +mccor kle +maun akea +mar stons +mar quin +lu dus +live tweeting +li di +legi bility +leg old +le jog +l liber +kin zie +khush boo +katrant zou +ju tta +iowa stateu +harvardchan sph +happy sehunday +gra pho +geschich te +gen nar +gat chaman +forz aducati +fo ie +fer hat +ev onik +ecol lec +di sunity +derekand susan +de groot +crystalli ze +cran es +cow spiracy +connach t +cassi o +c sco +bun ts +bri bri +bre aley +borough bridge +back stories +ani emi +age ing +aa si +ðŁĺ¢ # +ðŁİ¤ ðŁİ¸ +íļ Į +x design +winter jam +wen sum +us ages +un bleached +tobin heath +ti amo +thereal elp +supt chat +sun cream +stratot anker +sti en +stee zy +sock en +sm cs +skul lisland +sk od +sh ttp +sciento logist +ruthi eel +rsv p +rosen quist +play sets +pesc adero +pat u +ouel let +oregon mbb +ne ka +nan king +mu f +moto ko +motil al +mobi leye +michael raymusic +man tz +live st +lec tric +lalo alcaraz +la von +keny amoore +kan ada +k ft +jor vik +jimmyeat world +jag ad +itae won +indisci pline +il ves +i mani +huy brechts +ho ki +guille mots +gar way +ga ems +fx bg +freed omo +football family +floriansem le +fine jewelry +es un +doby ns +derby shireccc +darry n +dance able +d con +cri spi +choicemusic group +bra venew +beauty blog +b gn +b elling +az tek +atte sa +asur f +astr antia +appalachi ans +ali k +algé rie +$$ $$$ +ðŁij¨ ðŁı» +ðŁIJ» â¬ĩï¸ı +ì¯Ķ ìľĦ +âĺ® ï¸ı +» »» +y rd +whir li +wel lo +visi thel +v pp +v annes +usag ym +turntab lism +trev ally +tran sect +tipp mann +thom yorke +thisdayin hiphop +t sson +standard kenya +slug go +sign posting +shar yn +shad bolt +sens orial +sal m +russ o +qr code +promis sory +pope mobile +philip sburg +pal ance +pad mas +off it +o yor +nu cky +no suke +new y +nb pa +mun awar +mis senden +mis ma +mi gun +men sa +mc cowan +mar khor +loft in +lets makeit +lalah hathaway +ki et +kad ena +kach ina +jim mys +ir mar +ig arashi +i dr +guit arri +gou cher +go wing +ghou lies +ga er +fun ston +for free +fol ha +flux es +fis d +fab india +enew ton +elli er +deador alive +das nakz +chis ato +chicagos mayor +cal ce +ca wood +c Åĵ +c jd +bury makeup +bbc scotland +ar athon +amazon fresh +am ager +alve church +ak os +ab cland +! ðŁĺĺ +ðŁļ¨ @ +ðŁĺĨ # +ðŁĵ ı +ðŁijĬ ðŁĺİ +ðŁİħ ðŁı½ +âĥ£ ! +whel k +we believein +wal ber +visu alizer +visit malta +vi vic +ver g +ui path +tru ism +tribe town +tr ills +thur ingia +teri polo +sumbastur tt +sto ichi +sim ran +si za +showtime boxing +shooter jennings +shan elo +senior care +san onymous +san field +roi dery +revi ent +rein as +re organised +po que +patag onian +pap iss +neph ritis +mussel white +mo tes +minic ooper +mike s +medi bank +march on +magic of +lode star +lockthe mallup +ke ates +kar aka +just sold +jrothen bergtv +jo yo +j xn +ingu ide +i lean +help me +han in +h pk +gru jic +gro h +gi vel +gali bier +galeng ering +force field +femini sta +farah khan +face idibia +eng adin +emerging technologies +elor za +edinburgh napier +echop ark +dun gar +die ing +d brooks +com esa +codel ottery +ch ini +cen tered +cab re +bull pens +buil dit +bucky brooks +bre genz +blom qvist +bk al +best deal +bes sey +ber bere +b hr +as cos +argent inians +aman pour +am phor +afi fest +ade o +abren ica +a itu +.... ðŁĺĤ +. ðŁĻĦ +*-- -* +å½ ¡ +âĿ¤ï¸ıðŁĩºðŁĩ¸ âĿ¤ï¸ıðŁĩºðŁĩ¸ +๠Ī +É Ļ +ü r +z ii +yay asan +y ars +wu hu +with in +wick liffe +vian ney +ven is +van ews +tun nicliffe +to one +thing ie +the print +tech e +tbird nation +tan ita +tak ada +ta relli +still here +st pattysday +st enger +squir ted +spaghett ini +sin u +si dency +seraph ine +sediti ous +re positioned +pigg ery +pie week +pear tree +palla vi +pa ppi +pa kt +outlander home +no bilis +niko lic +nca acws +nat sci +nat reswales +n brook +msd strong +mis su +mcm ldn +love greatbritain +logo tv +lc ds +laver gne +kee sian +jeon ju +inst gram +in on +in famous +hipho pdx +ge ol +gastr itis +funn ily +function alized +fresh fields +fitz hugh +fi amma +fa shanu +f pj +en no +empy rean +dor ne +doo k +dew points +def qon +de yn +de ery +de bo +dau da +dasty ari +crom ford +contamin ant +compag nia +colec o +code pen +che twynd +cend r +catch ments +car touche +br one +andre wh +alber tab +al ders +agen et +aber tay +ë´ Ħ +天å®ĺ èµIJç¦ı +⤠µ +yuri ko +wit ty +whitte more +vo dou +veikkau sliiga +vangogh museum +unquen chable +under par +tw yman +tu er +the sound +tan noy +ta ho +super fortress +sun shades +sun o +stram rahim +stal es +spoke speople +sj suryah +sil van +sho ckey +sh na +sarah colonna +s var +ry kiel +ru tting +ric hi +ran kins +ra ub +premiere pro +pr sd +poo fy +phone pe +pau pack +p nin +ophthalmo logists +om is +olym pos +odi on +o rest +men o +md gonzales +mcge ary +mc gregor +mar itz +mad he +lu mmi +live au +list less +lake port +la porta +kono suba +kaw ashima +jeanni emai +jai mie +jaf ari +iron sides +insec tweek +incis ors +in blue +ilit ary +il ong +henry ville +hen lo +heatwave uk +hamble don +ha ymon +guitar hero +g suite +fv su +fi an +fateh pur +f tball +do bi +dl su +div ac +cor lett +com eu +cold weather +chicag omarathon +cap saic +buri ed +bur sitis +bur chard +bur cham +bell ary +be here +bbc wales +aristi de +am newsers +al rosa +adop table +abcland line +ðŁĩ«ðŁĩ· ðŁĩ«ðŁĩ· +ãĥ§ ãĥ³ +âľ ĸ +âĺ ¼ +zen tai +what chamac +wb go +way haught +vri j +vo le +vin y +villa iness +vaqu ero +vag as +ur bina +up ington +tri que +tri alists +thir i +the metal +the coop +te bay +tan abe +systems thinking +summer style +sub plot +sto koe +stay blessed +so jin +side burn +shor ror +shar maa +sen ki +sen at +sat b +rouss anne +ron kon +red star +re classified +rati fying +q nb +plan o +paulo avelino +park son +over throwing +osi ris +os ric +organ i +ocon to +o stuni +ny chash +nitt any +ni obi +mal eny +littlerock steam +little caesars +lau x +land slip +l ale +ky sen +kus adasi +kler ks +kasab ian +k adapa +j dr +irving plaza +indian oil +immune system +i kin +hu ic +homo genous +hare krishna +hae matology +gyneco logical +gou veia +glen ridding +fun kof +fl anged +en unci +dragon ette +don lon +do ble +dalla glio +dahl berg +dab aby +classic porscheforsale +celi o +carter ville +caroline flack +canu so +bun del +botan ically +bon de +bio div +barry bados +bam baat +bac ary +az le +at tested +assimil ating +ar dens +ap v +andy roddick +an music +alu cha +alph en +íĪ¬ë ª¨ +zur g +zindagi kime +ye as +wis k +w lic +vers ilia +under aged +tek la +team tigershroff +ster ner +stalact ite +sql sat +sooth sayer +slot car +simpson whnt +si mal +shil in +ru ban +rie mann +ridec annondale +rep irate +rae gan +pu tu +pos thuman +par u +paper board +ou thern +organic farming +mut ating +moment smatter +me xi +master minding +lind blom +likeli est +li mani +lax mi +la sell +ku hlman +kam az +it ars +initi o +hyper thyroidism +homosapi en +her bin +har te +gu mps +gor j +gh ela +fr q +facul dade +f icio +ent weets +e idol +din sdale +demetri ou +comic boo +colom bian +cityofla svegas +cir l +chee zit +cha an +ch ans +car tas +c elive +by x +buy do +bor owski +book covers +bog ner +blood mobile +black families +bel lowing +bay ers +ati fas +apare ce +ang ell +ane ed +ali zafar +aga a +æ İ +wj la +un tung +u wanews +tl r +tinthe park +teign bridge +ted ford +swad dle +sun gha +son risa +slay ings +sky wards +singh ania +sid deley +shir lington +sheffiel dis +sh ram +sell ersville +saw chuk +samajwadi party +saf avie +sabah info +rous er +richar dy +read just +r ph +r ck +que strian +pursuit of +pro ad +porta ferry +plu ma +pin pointed +pile driver +per ales +pc x +p ta +nor land +nit itaylor +mu hyi +mtlg azette +miscre ant +min ardi +michael dell +mcg avo +maurit shuis +maser u +man zarek +m eller +lin field +lgbthi storymonth +laur ance +lar ned +la pin +l we +kun tz +kon tra +kerembur sin +ker nan +kat alog +kali hi +k west +jessic acaban +j wp +ite aser +into cable +imacele brit +iklan ok +ident ically +i av +hic ago +hib berd +hc n +harbour ing +guj ral +gold berger +glori ou +giant srl +geo stationary +freer oll +fr nsw +fl ac +face tious +ear ned +e die +door n +dick en +di ag +dan ziger +da an +cull man +cho pp +che tri +cel luc +ce es +bobs burger +bike tour +beep ing +anis achibi +anim pact +am ical +... !" +íĪ¬ëª¨ ë¡ľìļ° +à± Ĩ +ál varo +wwe payback +wo wie +wo ts +wil lets +west en +vbm plong +uth appa +un fla +u chs +tv bs +tuesday bookblog +tre mayne +tranquili zer +thehorror master +the well +the meat +tan ay +sy ko +storm chaser +stock yard +squab bles +sn az +shi powners +shere sy +seab ream +score keeper +sci ac +sat is +sar y +sal lee +ry d +ro ks +reading forpleasure +re su +punchestown race +patek philippe +open source +oc kies +o berg +neph pearls +n alban +monti el +min chin +mill vale +mid dy +mel itta +mani a +m hairi +lor ch +ll amas +lip sky +lett res +la bru +kwang soo +khabar ovsk +jual beli +je ws +jay demarcus +javit scenter +is dead +home opener +hemato poietic +hat un +ham rick +gelsen kirchen +gam bon +gal entine +fri ess +follow westwood +flu ffiest +feasti val +e itan +dur gap +dub w +du plexes +d tx +cow rie +clap ton +choose to +charli erose +ce ili +car law +bun heads +brisbane times +bo tting +beau ly +bat arang +barric ading +ballin am +ay ler +arg erich +an ri +ak ler +ae o +acu bs +abor ting +:) ... +. ðŁĻĤ +# ðŁĵ· +ãĥ ¤ +ب ر +zag mbb +y anda +wy r +writing commmunity +water land +ver ige +ven za +un acceptably +ul nar +trick les +ten er +tc palm +tat us +sy metra +spandau ballet +soun dar +soul pepper +som ely +sk ymall +shu toff +san ath +sa ap +ro zz +ra vitz +politic slive +plac ental +pe tyr +pat ry +of ili +nrl footyshow +nether landish +nak ul +my ler +mor tes +mis land +millions missing +mil som +mc mexpo +mass statepolice +m hat +lovefor dorset +lign ano +lifel ines +leg an +koh ler +kali bo +k gun +john wayne +iri de +icec reams +i key +hy den +hol le +he ino +he dra +hat ty +grove port +gley ber +gl td +galax ia +found ing +em po +elong ation +ee ts +dre wes +dis orientation +dh ss +defendthe den +cull ens +cou per +con die +commun es +co eur +clo tilde +cheni er +ch alian +cer rit +celesti al +cast ells +capital markets +bus man +bu su +bog ans +bernou lli +bbc stargazing +bad deck +ķ × +ðŁĻı ðŁĴĻ +ðŁĶ § +ìķ ¤ +è vre +® - +wn r +w lu +voting matters +vandy baseball +vand u +u ssie +tuft suniversity +try stan +trumpe ting +thorn burg +tgom agazine +teesside uni +te dat +tan ana +table scape +tab o +sweden borg +sp kr +shrimp ers +san miguel +s draft +ri mer +regr anned +red back +reb ello +reading rocks +re bo +pub con +pri sca +pel zer +pasteuri sed +p vb +over lander +ori entex +one word +ole man +ofthe cup +ny berg +neck band +nat alis +monty don +mis spell +mis construed +mic hiru +mac phee +lo vas +liam neeson +kib beh +ke mi +kay am +kari bu +jor dann +ig y +ibm wow +hyper venom +housing forall +hor tus +hope dale +hit ach +hin ault +h mx +gar amond +for tus +food wine +evan rachelwood +en ag +elie saab +du td +del ain +dau gaard +d sport +ctil burymakeup +cit v +cho ppin +carib be +carab ini +car avel +cant ley +cam eleon +bundo ora +bun ky +boo table +beat ric +ba ren +ba ini +avon books +antipo dean +am ox +agü ero +ac cp +abc brisbane +ðŁİĦðŁİħ ðŁı» +ðŁį ± +çµµæıı ãģį +ı k +ysp sculpture +y leo +ver ul +ve eran +tz ar +tro pal +th illary +sv c +surfl ine +sura karta +supp e +soft wa +snape maltings +show ery +sen su +score stream +reignit edtrilogy +red horse +rafa ela +ra gee +profession nel +pic poet +pee zy +pe thealth +pe stered +p tas +one coin +ok y +o by +neuro surgeons +ne tease +nasty gal +my e +move to +mon na +min ki +melissag orga +mazz one +laser cutting +krish nam +jo ed +j its +j ito +i ho +hun ton +hene ghan +green machine +greek food +girl son +garbi muguruza +funny videos +fivb worldleague +feijo ada +dish evelled +depor tiva +chester bennington +capitol theatre +cantanker ous +cand ra +can epa +buil din +bla w +bhanush ali +bat avi +bad land +art craft +aren s +are x +arag ongp +anu ar +ðŁĮŁðŁĮŁ ðŁĮŁðŁĮŁ +íĪ¬ëª¨ë¡ľìļ° ë°ĶìĿ´ +å± ĭ +zi j +zaz en +zan ardi +y ip +wr ung +win ns +we ready +v ng +up lo +tom ba +to ady +themichael owen +the foo +the dave +tar ini +tal uk +skim mers +sho go +shag un +sac lay +s sts +rock land +roc ers +rob z +religi osity +rec c +rb ge +qual ityof +q assam +pol mtl +po eta +patri moine +musu em +more a +mill at +mg motor +mer ola +mennon ites +mau rie +mad ar +ll susa +ko techa +kap u +ka olin +jun as +jon im +jami efraser +ja hi +iron heart +irn bru +ing li +hu gill +her st +hell sing +grosven or +groo ved +gro za +good governance +glock ner +gl b +fore shadowed +for the +fi do +ey am +ever rrrr +eri kal +embelli shing +durham birdclub +du i +du blins +drake ford +detec torists +de vises +de vic +daniel goddard +cullen bunn +constitu ting +chival rous +chal on +car vana +cand is +canad air +bot terill +blu ster +ble asdale +black shaw +bill hemmer +bige ye +beer sheba +be get +bar ts +band aging +bak uman +as sion +ar ner +anast aci +ambedkar jayanti +alaki ja +adeno carcinoma +ad ell +ab les +( = +xo chimilco +whyilove kenya +way cross +vin land +try pophobia +thrac ian +the gun +sti ers +sports writers +southland sports +slo combe +sea vey +se wall +scu damore +sc ig +sar razin +ron ay +ri yadi +ri poste +re mastering +ram das +pop matters +pon dexter +pierre gasly +perip lus +par lance +occo quan +oak en +ni ft +nebrask an +ne tra +nar vaez +n ales +multit rack +modi fieds +meal worm +mar leen +m jc +lap sang +ky uss +kil ty +kil keel +jodi picoult +is engard +intl forestday +inter action +inten sities +in soluble +ic hen +hydrogen ated +hyatt world +horlo gerie +hollywoo dun +hin man +hard iness +gul lion +go ka +gigir ules +gerald ine +ge tta +fun ic +fu gro +franklin tn +for am +fatt oush +esqu ina +elijah wood +eck hardt +drive to +de ok +day long +da er +d allon +culin a +cu f +cro sman +circular ity +chub bies +cho ko +cere bro +c wi +bi zim +bent e +be fits +bangtan boys +archang els +am live +am agi +affe ct +abbrevi ate +ðŁĶ¥ âĿ¤ +ðŁĴļ ⾨ +ðŁĮ « +âĺº ðŁĴķ +ঠ¬ +zero ed +zadi g +yyj events +yo ker +wieg and +wi en +ve snina +vascon celos +trac tion +ti a +tat ler +sun tour +sun dara +summer holiday +silve ira +sie ger +shrop shirehour +shawn johnson +saad lamjarred +ren thusi +purple heart +proje kt +pro book +out shines +ol td +obstruc tionist +norfolk broads +nj hs +nigh trace +nh primary +mun is +min in +mee eeeee +medial iteracy +ma ps +ma ill +little italy +liqu in +lec tri +le tv +lady bower +kuz co +kot ze +kor k +klerks dorp +kin naur +kim dotcom +kevor kian +karolin ska +jig en +inu ddin +inter link +in ang +im polite +i euan +huy gens +hugh son +hel my +han ahan +gri ddles +gran ey +gai ley +fu cking +fr angelico +for tran +far oes +es band +enew ton +eat aly +dontt ell +de mag +de da +de cryption +corri dos +com té +clean and +cheese maker +cele stion +capu let +bur ren +buck nor +boh ème +b dg +atre vi +as mar +archan ataide +ant iterror +anee sh +afai ers +ad busters +aar nold +.... & +Ùģ ر +ع Ø© +î le +x av +word fest +war p +wag t +w tofficial +vor tic +van at +val se +v allen +unright swire +un changeable +ul li +tro py +tran sperth +to yed +thin gever +the troubadour +the iss +t man +sy on +swag ga +super her +su llen +starwars thelastjedi +solomon islands +scri ven +sb augh +ruf ford +rock yr +ro er +remain ders +raise it +q aid +pray in +post codelottery +pin heiro +or bach +ol lan +ny history +nou man +neville southall +napole ons +my friend +mitt i +messi eurs +memor ised +mb ro +ma hat +lil it +lal al +ko sam +kill i +katery na +ji zz +jeremy renner +jar ome +j swr +indie gam +human atm +hl h +hey dar +hermaph rodite +hell onwheels +hau ge +half girlfriend +h silverstone +gre sty +go braves +ge ir +fern down +faw ley +face off +environment ca +em ers +ear bud +e zz +decent work +co sier +cap lin +bry and +bri sca +bra j +boun der +blu ep +blair sville +berk ner +aw sten +as kaman +ari k +apar na +ag da +ab ati +a they +ðŁĴĸ ðŁİī +ye ast +y kk +wv pol +wun na +w jac +uve itis +uproo ting +under croft +un interrup +un frozen +twee tz +tor v +tam ping +steril isation +step by +sports ground +sn azz +sky trax +ski bo +sir pareshrawal +shim s +samanth afaiers +rall yargentina +premier ships +photo copied +phalar ope +pe ss +os wego +obac ter +n inet +monstr ously +mis st +mi elke +mer row +maz havil +manate e +m ku +lo ree +lic ek +le bel +lam ing +kou yate +klu ger +kk g +kis sa +kid de +kear ney +kathiel gifford +john reese +jaun ts +jason inthehouse +jap chae +ise man +har rah +groe schel +gre ying +google classroom +gear up +frequ ents +for r +fon tane +foll lowparty +esof tware +en ema +ely gutierrez +eas thar +does stuff +di arist +di anova +daily dose +cor ri +clean energyeu +chim ed +cal eta +brig it +boyle heights +boo gers +big b +ben aroya +bean town +be good +bad uk +bac oor +b dom +anjel ah +ah le +adiscoveryof witches +a ints +ðŁĩ²ðŁĩ ¾ +ðŁ¤ij ðŁ¤ij +yankee town +with you +vil lew +vi my +var di +ume boshi +trans africa +ton ino +tj thyne +thomas son +thew lis +teve rett +tch rs +su es +star live +squ ito +sinthe park +se don +rush olme +red pointer +real paulwalker +razor fish +positive energy +p tab +ou z +opend at +nottm hospitals +n ze +mile so +main tainers +main bhichow +m pa +m fr +liv vy +jun kin +jas want +ina hyattworld +in ating +hr ds +hartz ell +ha unch +guillau me +guil bert +guer as +god first +gibb o +ger ade +forti eth +flor iculture +fix ate +fashion post +extravag antly +elin asvit +elinasvit olina +e soc +e east +dares bury +cour teney +corner house +conqu erable +clo ppic +cl oris +charity water +chap books +brah mos +bol duc +bobb itt +balsam o +au tis +ant ler +ang one +alison brie +al ten +al bari +aa de +a hero +a aryan +ðŁĴ¯ " +ìĦ± ê·ľ +âĦ ħ +ر ÛĮ +yennaiarind haal +x aml +westside story +war li +vul cans +villa hermosa +val sad +ur ger +u pr +tweet dcs +tropic alia +ti w +the ho +ten ancies +sx swi +stable coin +sta ats +sped ding +smith jr +sea worthy +sculp tured +schak owsky +rowland son +re ha +quick ening +pur in +outback bowl +ot ally +old football +odi yan +neu bauer +me he +lie be +lali tha +ko tak +kari ma +kal afina +jec tion +je anna +jami ek +iv f +is elin +imperi ale +im ploring +green view +geroni mo +gay lor +gar der +gar bage +free theni +flun ked +es wc +eng lands +emanu ela +eas d +desider ata +danielle fishel +d animation +cw tvd +cr k +county champ +com pras +codepend ent +cli entes +caw thon +camero ta +bling bling +bit c +belle fontaine +bec kel +bar ritt +bally na +b dn +atti yah +an vil +american ah +ali w +alge bras +agchat nz +ag wx +afer rera +adverti sment +accessto justice +? ðŁĺĤðŁĺĤðŁĺĤ +ðŁĩµðŁĩ ¦ +íĪ¬ëª¨ë¡ľìļ°ë°ĶìĿ´ íĪ¬ê²ĮëįĶ +æĽ ¸ +âŀ ½ +ਠ° +z solt +yaz id +yas sine +work shopping +water wise +vlog mas +vampire weekend +v anga +upthe blues +uni fier +un favourable +travel massive +tram ways +tr ng +ti bial +ther oot +the machine +tann ers +sub groups +su en +storm front +stay high +stag ione +ss afe +spoo fed +space force +sfr tg +roman ians +ri per +red mon +red crescent +re ur +pac is +p jc +ot ley +only louisiana +oneyoung world +og ham +nr ma +nick les +ngan nou +newcomic sday +n mea +meta sta +mcglin chey +mcge hee +mc goo +maxime bernier +kol chak +kn eller +jun d +jiofilm fareawards +j sh +j ip +il do +i ey +hey den +ham burger +haash oficial +grimez sz +go far +g tu +ek stra +e of +déj Ãł +disney princess +democr acia +del tabc +deduc ed +daim on +county fair +con iglio +cau ld +castron ovo +car v +cap tive +c sec +butterfly count +bu bo +bandi pora +b wh +b wf +ay ad +aw ns +ard ina +angie martinez +al phe +acad music +aar yn +; ... +ðŁĩºðŁĩ¸ ! +Î ¼ +y ati +wood brothers +winne mucca +willie bosshog +who what +wassen aar +ward law +walt disney +v sauce +un grounded +tric order +tom ics +to co +timessqu arenyc +stipul ated +star cruises +square pusher +soul silver +sherrie hewson +sand bagging +s gas +ruf fudd +rudy sarzo +robb i +ro fficialsite +re shoot +ran ker +raj neesh +rac eland +q nl +precipit ate +over mars +nul lar +nu ra +nan ase +msd n +micro pub +man bookerprize +mac ba +ma sud +lun ardi +leon berger +l ve +kel on +jewelry design +il dan +i kot +hol sworthy +hol beck +gl ico +gen z +gan pati +friend ster +fish finder +fille ted +fil mcity +du si +du der +dram buie +dor ange +don kin +dol ans +culture trip +cros shair +cor as +com ma +cas sina +carth ag +car sales +burk head +bubble tea +br attle +bol inas +bell erose +been leigh +bar croft +bach ar +autograph ing +at chi +ar dea +ade ts +ade sua +ðŁĺģðŁĺģ ðŁĺģðŁĺģðŁĺģ +ðŁĩ§ðŁĩ ´ +ðŁ¤ĺ ðŁ¤ĺ +ãĥ¼ ãĥ¼ +âĸ ¡ +à´ ± +zing ers +wave splatform +wa ag +vas ari +u chicago +tran scoding +top end +the golf +tex te +terry branstad +tar o +stel ly +solo thurn +ske g +silver fish +shot put +sho g +shi kar +sch ange +sc ort +sac ro +sac aja +s del +ror ke +ro bor +re casting +psori atic +popp et +plu splus +pete carroll +paulweller hq +pau gasol +pal ghar +oran more +o jh +nutr itive +neuro genesis +nau gh +mür ren +mu ggins +mis givings +milit aire +metabol ite +mcgi vern +margol in +mar ico +l wv +kirk sey +ke mayoran +kc v +k illion +jp gaultier +jes sphillips +jen ne +iam sterdam +ho cu +hic ham +ham murabi +gb highst +g willi +fran chot +fisho s +ex trinsic +es wat +equip ment +emol uments +emo tionless +earth gang +dor ridge +dogs rule +dj awadi +distr acting +dissip ates +diri gible +deflec ting +debut antes +de mus +dan soder +d and +cwa union +crusad ersfc +commen cer +case worker +bur ling +bul wer +brum bies +ben bow +batt ler +au sting +art galleryof +ano x +american pharoah +america east +am ing +akas aka +aig is +ah lers +agor aphobia +aau p +aa sha +?! ... +" | +you bears +yor ki +ye om +y ahh +whipsn ade +vec trex +vale of +v ran +ti econ +texas football +tamron hall +tag it +t chs +surf life +su mal +stru ct +spa etzle +skylar ks +skul king +se jarah +schiz o +rp bp +ro vere +ro ques +real music +rap mag +q amish +plexig las +plagi arizing +piper perabo +orr ville +o bb +my ra +mo ela +mesoli thic +mcgr ane +may theforce +m hart +lex ical +lar ouche +ku wa +kitti wake +kim chiu +jesus lovesyou +hit ler +heritage openday +hered ity +happybirthday harrystyles +han auma +ha den +grace potter +gandhi an +gallow gate +fin barr +fa asil +f chs +elen co +do that +dha dak +del and +dd t +con ci +con al +co sham +catal angp +castle island +cant wait +bucket list +boat wright +blit ar +bisch of +big love +bee ching +barri entos +americ aferrera +am bling +aig burth +ðŁĺįðŁĺĺ ðŁĺį +ðŁĺĤðŁĺŃ ðŁĺĤðŁĺŃ +ðŁĴĹ ðŁĺĺ +ë°ķ ìļ°ì§Ħ +௠Ĥ +под вод +ye pp +y su +y fc +vande grift +u oc +tw on +tony hinchcliffe +ton elli +them here +tennesse etitans +taylor r +tanker sley +tang i +sul fates +song lines +solihull hour +sk ratch +si so +seab l +scu ffles +scarl atti +sand u +san sui +roh ith +ro erig +real don +re thought +quant it +po zi +pillow cases +patricia heaton +park hyatt +opti que +od pp +och sner +nighting ales +naz ario +nab j +my fm +mmmm mmmmm +mir inda +mil roy +mein l +mater azzi +marshab lackburn +marsha ambrosius +mar isol +ma shab +lu le +labour ing +ko dy +kil birnie +kal pa +jone sc +j ato +isch gl +irre vocable +ir c +inte res +id pd +happ year +han sford +guer rill +glu te +fu uka +friday feels +for you +feelthe force +ema k +ell ondon +e schen +din u +dep tt +cop yed +colum nar +cherokeen ation +che wer +cantin flas +bristo lian +big wet +ba shy +art photography +ai ki +.. ?! +. âĻ¥ï¸ı +ðŁĺł ðŁĺł +åĨį çĶŁ +âķ ij +zee studios +zebra wood +z aida +wal demar +w sd +ver b +un characteristic +tra vie +timp act +the orange +the dorchester +t music +swis salps +su prabhat +spar ker +shi han +seduc tress +sec or +sciento logists +scan dies +roman tique +ro q +rmb aloncesto +ril les +ri sas +ren hotels +queens borough +pretty lights +pon er +patt naik +p elion +niobi um +neon atology +nag ios +mrolym pia +moon band +mon u +mon ong +misanthro pe +minim oog +me ia +mat aro +mai k +mach us +m uring +lu mping +little monsters +leann rimes +lang at +land mass +la vy +ky gov +koky anaamdoon +kirk hammett +k bp +jl h +iq iyi +infin itive +ide sof +go cavs +fran ko +fo de +ex ome +em mert +eldor ado +east wards +du gas +dr k +di awara +de pendant +culture matters +cresc enta +com fy +christopher son +bonesh aker +bel u +banan afish +au ty +ale ssa +! "" +ðŁijĪ ðŁı½ +ðŁ¤ ļ +woo dies +vulcan ized +uofl football +un wieldy +un truths +ud t +tradition alists +the meadowlands +terror ise +tar ar +sunnin ghill +su be +stevi evan +sonuk akkar +sond heimer +small z +sen berg +se bel +schir mer +sang u +sam pal +sad ak +s gurr +rock liffe +ri ou +reverber ation +reck oner +re focusing +rand corporation +ra stas +ra gione +quer cu +push up +pru center +pr icking +plant is +pi att +pd as +pas ka +park life +moss yoak +marke ta +lo ev +ligat ures +leu cadia +lam es +l js +kur us +kos suth +kore l +kne eled +kar le +kam ini +jonathan rea +jan ke +j lt +j hms +hepworth gallery +guic ruzzz +getting old +gent a +ge tyou +friedrich shain +fast cars +dol ton +dignit ary +david luiz +da rel +cyto kines +crab meat +counter weight +containeri zed +cont ango +co senza +cn trl +chris gethard +chon dr +budd h +brun dage +brook lands +bron i +bre vis +biz expo +biomed central +bin ational +beat yesterday +bar onet +bad shahi +aza dis +aster aceae +ap aka +ann an +algon a +ðŁĴķ ðŁĴĸ +å® ī +Ë Ĭ +win now +wes thigh +vo lei +vi tha +vi kapoor +ven to +van hoo +uuuu u +usas wimming +u spolitics +u bos +tru ant +trans mutation +tom waits +theatre sports +the billy +te viot +tat asky +ta ve +sub verse +sonny rollins +sho review +shadowhun ter +sand akoz +san ad +road atlanta +ric cio +rac isme +prin c +port arlington +play things +peu geo +per man +om ark +o tomo +notice board +nin ang +negre do +musc leman +mur fit +mis behaved +mini gun +micro glia +maz dac +mand vi +man bear +leon ardi +kil martin +kam es +jand k +ja bez +ishq mein +is dn +irration ality +ioang ruffudd +insol ent +i ç +i pr +i fac +hit ches +heff ner +heff alump +han ge +go valpo +gir dwood +gal vis +ent reprises +dundee unitedfc +dom esday +dissol vethe +dis believe +di santo +desal vo +colo stomy +chocolate week +cathr ine +carto graphic +can ed +bli ps +beau dry +aus able +atlan ti +ash well +ani ven +alth off +all ender +al ini +aden osine +aber foyle +ðŁĺĤ ðŁĻĬ +ðŁĴķ ðŁĴļ +ðŁı ŀï¸ı +ðŁİĤ ðŁİģðŁİī +ìĿ Ħ +ë±Ģ ë±Ģ +ãĤ¹ãĥĹãĥ© ãĥĪãĤ¥ +à¹Ģà¸ŀ ล +youre in +you ku +wor csc +white supremacy +visit lancashire +ver steeg +un desa +uk hi +u caas +trin arock +to whee +teu fel +terrori zes +sy ms +sunba ther +stron ach +streeter ville +sl una +skill sfor +shahb az +se up +scul lers +schro ders +sch wantz +sar gento +san ae +ri ghetti +revital ised +rat nani +po pple +po logne +plin ths +occip ital +nau ta +nau mann +n jea +musta fi +movie maker +mit u +mindless robot +michel inguide +may ang +mandar ins +m fc +ly th +luch adores +lido caine +les bleus +le it +kor cula +kitzbü hel +ju ls +joh ana +imp ale +i zi +i ed +hig bee +hay war +happybirthday zayn +grati fied +gne iss +g delaurentiis +fran son +food network +flw fishing +fijir ugby +fe mm +fe ile +fas si +far zad +fa aa +f mn +epic o +ent ures +elu ding +else car +dundur n +demor alizing +dago bah +d we +cros shaven +cor of +clu mpy +care ful +car bajal +bern ays +beat ntds +aven ged +asun rise +an halt +alizafar says +ah art +ðŁĺ± . +ðŁĴªðŁı¼ ðŁĴªðŁı¼ðŁĴªðŁı¼ +ðŁİĤ ðŁİīðŁİĪ +âĻ ¢ +ภ¨ +wra ith +womb well +wil hite +weigh bridge +us b +uaap cdc +trophy tour +tran sal +th abe +tele marketer +sylvani an +strat fest +strand berg +stran sit +ss mith +spo li +soori official +so jisub +so fun +skarsg Ã¥rd +sec chi +se les +sau ls +ric hy +rem sen +rece ded +ra su +ra dd +proud foot +prin ses +pot light +pnn lab +pas os +par nelli +paol onu +paolonu tini +pa ek +oy akhil +o skal +o india +o del +nu ovi +nisar gad +nat lib +mor taza +model monday +mbe ya +mash rafe +mag adheera +low sky +low ball +life buoy +labour leadership +kun u +kham oshi +jun jud +junjud api +haz els +han wha +guinnessi reland +global forum +gibb ins +escape from +el ayne +ec at +dru mmed +dis member +di baba +dare devil +d cast +crowd strike +crow thorne +clare morris +cho va +chel seab +che ka +ce zar +cat cher +car du +canon sburg +bru yn +brace well +bil lable +bee ing +barnar d +bal ers +avi o +at tune +ashion week +ash our +arde che +anjelah johnson +aishat yler +ðŁĶ¥ : +ãĤ¹ãĥĹãĥ©ãĥĪãĤ¥ ãĥ¼ãĥ³ +á´ ħ +zi b +z iller +wry golf +whipper snapper +w als +vad er +under takings +tyger berg +treecre eper +the east +tam am +sun bed +stick ney +soli daire +sk rull +shock oe +shanelo wrygolf +sen ne +schle swig +sagan aki +sacaja wea +sab ai +rod ley +ri jk +re organising +re locates +ration ed +rat tail +r tbf +r mh +polyam orous +pitt sylvania +pe ated +paramahan sa +ni evera +ne in +morning smaria +moral ising +moon spell +miniaturi zed +metho dically +md l +may field +manife st +lie bert +lech lade +lean up +lar occa +la ggan +kazi mir +jer onimo +jam una +iri she +idhun amma +ic sd +high road +hakk asan +gc pd +gator ade +gar ro +felici ana +fe igned +enbau ms +ek kle +eber hardt +eastern most +do bre +defe cting +de fusing +cyclo tron +co enzyme +chu bb +casu arina +c mail +branden burger +bi oc +asi va +ash uto +ani ban +age ar +ab un +. ðŁĺĶ +подвод нÑĭй +ze alandia +zach ery +welcome tom +uu ren +um hb +tur chin +tor mo +tin chy +ti ousness +tel ah +teenmom og +tan gos +swag gy +sw illing +ston king +sti fler +stable mate +ss d +sle h +sky walker +sketch app +si ska +shipla ke +score hero +schiz o +saun dra +sahi vikas +sag ra +ri ppy +repre ssing +rel ic +recor e +radi ant +proud papa +preston pans +practicemake sperfect +poinset tias +pierref onds +pen tru +peep s +osa urs +oni press +o ãĦ +ni j +nhat rang +n ones +my music +mv coup +mu erta +modul ates +mikeand mike +micro bio +marcu slu +maneu vered +kent land +kas sy +jer o +jeffree star +it sas +ira e +infu ses +impeach trumpnow +im pel +i jaw +hat boro +goli sano +fe il +extric ate +emper ador +emili e +ecto plasm +ec olabel +e io +dot na +din smore +dess ins +dele veraging +daily pic +dag gubati +cut back +craniosac ral +chom p +chi ke +chau dry +cargo bike +bozz io +bonec rusher +bat is +band ari +az traffic +artic le +amo del +air link +ade x +? "... +<<<< <<<< +ðŁĶ¶ ðŁĶ· +ਠª +x om +worm holes +white shark +wau pun +wade barrett +vol z +vbmplong march +un wed +uc ce +tugend hat +tric hot +thunder showers +tempor e +teacher goals +sun trust +ste ins +stav ing +sh moo +scot amb +runder ful +rock castle +robo call +roalddahl day +ric ke +reign iting +rafa ell +ra yed +ra jo +project manager +oyakhil ome +ow ska +obse sses +oat lands +navi star +natwest business +national popcornday +nas sp +mn cs +mn as +mi qu +metal lo +mer rett +me tha +mat y +marlin spark +man gi +man ette +m kc +longh and +long now +li aqat +lam son +ko han +ket cham +kel tner +job sat +jer seys +indi um +in ay +iam harishkalyan +hate ley +hare di +green arrow +fran go +foodi sta +fili bu +fil invest +fi endish +farru gia +fan ad +fair less +en us +elo c +ecuad orean +easing wold +duncan bannatyne +de ese +cytok ine +cum by +cro xley +coopuk food +chee z +ben im +bai le +au le +apic tures +ah madre +af fil +acuer do +abandoned places +ðŁĺĶ ðŁĺŃ +Ó Ŀ +подводнÑĭй миÑĢ +water buck +w ami +unitedin blue +tson gas +tin z +the affair +teng gara +tani shq +sym bology +stevievan zandt +stere k +staff pick +sahy ounie +royal acadmusic +ro ach +riz la +refin ements +red rum +re processing +re eva +rann vijay +radio logy +r ble +pro series +per ju +over lapped +onair now +observ ances +numen era +nhsor gandonor +mt leg +missteen usa +miseleccion mx +miner alization +low enthal +lo presti +lhu illier +ld an +lay ne +ki we +kewau nee +keep it +kas ol +jonsnow c +j ast +j ale +irish wildlife +indi anoil +il san +i ai +hu c +hiday ah +herecom esthe +her sch +hal don +h sps +gn ome +gg n +g wi +g abo +fu vah +flag poles +fi ala +fab y +essential classics +em all +don da +ding bats +de vens +cor ra +che tan +celluc or +c its +button wood +brow ser +bracken ridge +bloc party +bis ola +big ja +bhu m +bey er +bell am +beg at +be sant +bb w +att proam +ar ar +al ao +ab la +/ Â¥ +ðŁĻĦ ) +ðŁĺį ðŁĺĭ +ðŁIJł ðŁIJŁ +ðŁĮ ĸ +âľĭ ðŁı¼ +ye go +y xj +windo wed +wain sco +w ady +uvamen shoops +un dip +u bd +u ach +trump crimefamily +trinarock starr +toddler life +tl ach +the arvindswami +tattoo fixers +tad ley +t mall +surfact ant +sun aina +stur te +steg ner +ssi m +so an +silen cers +shar pei +se anie +ru inous +robert semma +re mar +rachel hel +rachelhel devans +qui les +plussize fashion +pl one +pin pointing +park man +pad arn +p ty +osric chau +oskal oosa +or adea +of art +nov ation +nfldraft news +moon alice +mizzou made +millon arios +melt downs +mell on +meant tobe +ma daya +lu cks +loy ello +leic s +le ks +later ally +kyl erichards +kopp elman +kanna digas +joseph capriati +ji bes +int ouch +ihear tt +hay tor +growwith stpi +ghost of +gang bang +galactic os +full sail +eul alia +er ot +east england +e oe +delicious ly +crew kerne +com pi +ck m +caver na +bon eta +beverly hilton +bee by +be ber +bank ole +asil omar +arctic council +all wood +aldub birthday +.... ..." +ðŁĺį ðŁĴ¦ +ðŁĺģ ðŁĺİ +ðŁĵĨ : +ðŁ§¡ ðŁĸ¤ +à¸Ńà¸ Ń +Ì ´ + ¤ +wis ner +w acc +ut sw +u dr +travel port +thene igh +thel ine +stor ck +spir aled +spar ling +sp ackle +sopra se +sham il +sal ata +rosen stock +red devil +re hn +r pv +py ra +py ar +pier das +pav elka +or ac +onthe way +old times +nuern berg +nola prep +never stop +moon ey +mom ot +me ul +mar n +kuni yoshi +kun ti +koz lov +kis o +kenneth cole +kat ty +jabharrymet sejal +ist at +ik ongallery +i shaq +himantab iswa +high sch +head butts +han ley +hairy bikers +good lettsville +gol s +gi olito +ge bauer +fidge ting +fen elon +extra solar +eu logies +electro cute +eis ele +ed dine +dan s +cro cker +cre wd +cr illy +colin donnell +cin eloka +child wall +cas par +bridge way +bounce tv +black ledge +bir an +bb itt +bail outs +arty r +al tin +abkhaz ia +ðŁĺŃ ðŁĴĵ +ðŁĺī ðŁĺı +ãĤ° ãĥ© +âĿ¤ï¸ı ðŁĻıðŁı¼ +âĿ¤ ðŁĺĤ +ี à¹ĭ +ye ti +wood fin +wkr g +win ker +wi ft +wi arton +whatmake syou +west gate +war of +wahy u +w ado +view port +vic kie +vi u +vi as +versi oning +va den +tri gga +the wil +the len +the fca +th off +th iran +tes da +tart arus +t kc +sy oung +stro ked +spring fling +sil vere +si q +seed sof +sche idt +scavenger hunt +sav as +sa aksh +ru die +ru bus +py g +pure foods +pro quest +pl eno +pin as +our met +or yan +one chicago +ny gard +nbc s +n inju +my nameis +my mt +movement detroit +mil f +mary popp +mary kom +mar ym +maine alden +lu tte +lon za +lin en +li isa +leav en +k com +iron hide +il os +if j +i yd +i bey +hill topper +ham al +gro per +go big +flye thiopian +fli ms +far rukh +emolli ent +el debarge +du by +doo cy +dal gle +copp aitalia +con court +coco bolo +care ssed +brang wyn +big blu +bid vest +bi ondo +bhav ya +bernar dini +balas ore +aster ia +asse h +argent a +ar cham +appel baum +ant miner +an var +amuse ments +amer urological +amali earena +alo dia +almost there +all americ +- ? +ðŁĺĥ @ +wap iti +w snc +vigne ault +uscen susbureau +ug av +u press +trichot illomania +toc tober +that guy +terrel lowens +team hisd +synth ase +swi hart +sun de +sky blue +sel znick +se folo +sefolo sha +se ds +sal aam +saf dar +runderful runners +rubik scube +ru tan +rh statton +potro juan +pitten weem +pho tome +pa ese +not man +national vegetarianweek +mu rex +member ship +meh endi +max carver +marriott intl +mar ck +magne site +m scs +lise berg +lin ge +limit less +le long +laughing stock +klu ang +kemp o +keep fighting +jas mina +i wat +hor field +hen nigan +ha ag +gun valson +guin an +glori atrevi +girls frontline +gat as +fu chi +fri er +flavor wire +fish ball +far point +ende ared +ebay uk +dun laoghaire +dis regards +del t +del potrojuan +deb p +davi di +conceptu alize +coeli ac +cheese makers +cas eload +c thul +brit vic +be tech +bb unker +ballyfer mot +bab bler +ati veness +ary digital +angel candice +ang lian +am yl +ae wrestling +adel mar +ad li +action aid +ach am +ðŁĻıðŁı½ âĿ¤ï¸ı +ðŁĻı ⾨ +ðŁĮŁ ðŁĴ« +weet suk +ver mark +ve mma +ve iling +va sek +ud mila +tre orchy +to prank +then ba +the science +ter ps +swild things +sugar beet +spyro reignitedtrilogy +sie ber +sheryl underwood +she inspiresme +sed aka +se pan +schau m +sam ael +sab on +ryn ism +roo ves +ri zo +real adam +raz il +po per +pas sio +park chat +orochi maru +oqu endo +nycw ff +neem rana +nadias awalha +mol loy +mis interpret +mir alles +minoc qua +mesc lun +m ellis +lovel ive +liber alisation +langou stine +lan vin +kubernete sio +kro on +kor ay +kk ad +kha itan +kart al +k vad +joy riding +jon antoine +jas pal +israel ic +in man +ic sc +hyper mobility +hu main +hu ac +homelo an +hit en +halit osis +h illa +gra ves +gher itage +geo duck +garages ale +fi dm +fac a +f wd +f ali +eskil stuna +east west +dro m +do bell +cross border +crisp us +contr ite +constan za +con ca +committ ment +ci for +cher now +buil tin +br arian +bo dil +bla sian +birds all +be chara +b é +atu ta +ar land +ar ka +allianz stadium +achaem enid +a aco +ðŁĺĤ âľĮï¸ı +ãģķãĤ ĵ +ye vadu +world milkday +wil den +west town +west by +weh ner +ware ing +vi aggi +v liss +track er +toom bs +tonight show +ton yy +thick ly +the vault +texa shoo +tab o +stri de +sri rang +spice works +sm pte +ship board +s feld +ru shin +ro sol +ri var +retali ates +real chalamet +re ves +re state +rand azzo +quadr illion +particul ates +om okri +oc allis +nr r +nobu o +monong alia +mis behavior +micro cephaly +medju gorje +massimodu tti +maso chism +mangal am +mag con +maced onians +long sword +le claire +lan the +ku ril +koz ma +k wwl +k wes +jer kin +iv on +im bolc +hudders field +hor tense +hin tz +gire sun +ge ting +fun kier +fu buki +fo cke +fa hr +express yourself +explorey ukon +en ns +en contra +double speak +di xi +dev as +despo tic +dec ke +dab i +cr ys +confidenti ally +colombi ano +cj mccollum +ca ston +bul ba +bre land +brah mi +bio solids +bio logical +ber rington +bax ley +barn door +bare footed +au slan +atri z +atlan tico +arm wood +anim ates +af corse +ac tavis +.. ] +âĤ¬ ! +wy theville +wind jammers +water vliet +us bank +tu ban +tom leykis +the place +taxider mied +tan guay +tad deo +sub human +su desh +st place +spy aar +spo sed +southland strong +sou chong +sno rer +sk ene +shor tener +sh aus +seavey daniel +scra ppage +sch angelives +sar ad +remo re +ram c +ra wn +profe sses +phara onic +naro oma +myst ere +mycen ae +michal ka +micha elian +metal ocalypse +med wick +mar lyn +manu va +mah fouz +m vula +lov att +lem w +lele pons +lax ey +lam beau +kian ow +jyo thi +jy h +jorgen son +jeric o +jamesp ure +jah n +iz a +harmon icas +green vale +gin der +george monbiot +gang plank +g ice +fu ggin +fresh fish +follow meand +felic itate +e still +e bit +e ath +dero ssi +de cameron +daily pick +con gis +chath iram +bow fin +bournemouth uni +bor sch +black down +beach resort +be anie +ban ani +as mita +ar ses +aqu ad +al gos +adam sville +? ðŁĺİ +" ." +ðŁĵ· ðŁĵ· +ë°Ķ ë¹Ħ +æ¨ ª +à¤ľ र +yo gat +yellow legs +x radio +tearsfor fears +sw op +sunnyside oflife +sunday dinner +stony hurst +spartan up +sj now +sivan anda +saro yan +sar p +re buffs +quint al +plac ate +pl and +pic co +phill yd +or folk +nol anews +nationaldayof prayer +mom oka +mas vingo +mandi bular +lu kel +li ppe +li pari +kwe se +kon jac +kappa alpha +k rin +james brown +horu sheresy +hen ares +gv l +gu ba +go dric +gar ve +friday vibes +free tips +food court +energy ctr +eclip tic +ec fa +dom atic +diom edes +din ars +digital is +desper tar +dark chocolate +capy baras +brexit chaos +br st +boxingh istory +ber ga +beauty ful +bang ali +bal wyn +bal ay +back house +baby care +b worldph +azz edine +ann ville +al or +aar ushi +! ðŁİĪ +! >> +ðŁĽ ©ï¸ı +ðŁĺį ... +ðŁijħðŁijħ ðŁijħ +ðŁ¤ĺðŁı¼ ðŁ¤ĺðŁı¼ +æ° ´ +චļ +yo av +yearofthe pig +x fc +win chelsea +wal pur +w car +vere enig +vau k +us lacrosse +u bie +toku gawa +the cityof +terry mcauliffe +tech republic +tar ana +stou ffer +sm ps +si mus +shro shan +she at +ridd hi +reg ar +read up +pu le +pro sumer +pro log +po ise +pm kk +phyto chemicals +pay with +pan agi +on do +olivi ad +oke reke +nicholash oult +museodel prado +mur gatroyd +munchau sen +mol ite +micro chips +mar yan +lino type +kne en +karti ka +jen kirkman +jeet endra +jaguar uk +is spyaar +in ko +ilove him +illino i +hydrochlor ic +hu ws +home opath +high grove +hell yer +har tono +har ol +gur preet +guil tless +gin oclock +gari ff +furi ous +fried berg +form ichetti +food market +ever ts +esp ie +epo que +eco logy +east anglia +dom swildthings +dam nnnn +craw daddy +commen sur +chab ahar +can ecor +box all +boing boing +black canary +bear de +bake ware +b schweinsteiger +ayush man +auckland uni +ad wa +abdic ate +???????? ??? +: ^ +ðŁĩ§ ðŁĩ¦ +ë© ´ +Ê ľ +wood lot +weare lebanon +ut sw +to ona +ticketmaster ire +the boat +te vis +sverd lovsk +stra ÃŁe +so bol +si aya +shab at +seat on +schle ck +scan ty +royal marsden +ri ki +ri bas +pole mic +pig spotter +pi mi +pi ave +pf dj +ped doc +pe urope +on kar +o wino +n ali +mun dra +mc w +mali q +luci da +lanc s +kus inews +ko y +ko ech +ke okuk +kali sta +kale em +ju icec +ise tte +io dp +inchic ore +in shape +ic ade +huck aby +hones dale +hann u +ha vet +global port +gl v +gess ner +food por +fi di +fathom events +fan cast +faceto face +ev g +epic mealtime +el sey +el oping +ek strom +ego centric +e red +duter te +dil f +der ms +dback s +dam u +da si +cy ma +cul po +cu ca +cro hns +camo zzi +caer philly +bt sout +brum ley +boy yyy +boro on +book selling +big al +bel go +bar soom +az oid +aver maet +ash bery +arun del +ar mine +ar anya +anthe mic +am mann +all smiles +ak dha +ab sr +aal st +⼠¸ +á į +yo z +yan an +y reka +witchesofeast end +wedne sfield +weare somerset +vote on +vir now +vi anna +un amid +ts live +tre gi +tonik roos +titi ously +the af +ten enbaums +tas neem +tapp reciationday +t pw +super conductivity +sub tweets +su render +stor ify +spor tier +sma son +simply thebest +side real +shiny pokemon +sher bourne +saty ricon +rule making +rid doch +ri ving +pu lli +or le +non such +my din +molca jete +mirwa iz +min yan +mile sa +mar imo +ma it +li mi +levi en +lang er +lam i +la vern +kirk wood +kir aly +k lansman +jun cos +joy pad +joy land +jo akim +it slav +irish men +ic eni +hum drum +hound mouth +ho palong +heu ston +gu u +glan mire +gas ometer +gamer tag +fried land +explic ation +end stigma +emm ure +emili aclarke +el bowing +dic kel +deli st +de oxys +cyber threats +crou p +cor reo +clark stown +charli ecox +cham ploo +celand ine +car rel +bren den +blo que +bir m +bha sha +bake sale +album launch +abo ttle +ab loy +îIJ Ķ +âĺĢ âĻ¥ +yam asaki +writ es +whodares wins +whi ther +w punj +w lb +voron oi +vo th +ventur ini +vat ica +tom u +three peat +thereal deal +the spy +th ant +super sized +sun ited +su kan +squ ely +spher ion +spd way +sker ritt +sk ook +sher borne +sen tertainment +sam is +sa kov +rou leau +roh nert +rad bury +ra var +quadri foglio +portugu esa +pipi stre +petro leu +outri ders +orn stein +occul tist +msp bj +mr sc +mou awad +ly ons +love animals +leibo witz +lam otta +kam lesh +kach a +ju tro +jeopardi zing +jasmin bhasin +ira p +interro bang +hor y +home towns +haul in +h sy +h ll +guacam elee +gu adi +gi di +freedomo fexpression +fred hutch +foun din +fore stal +for justice +file sharing +fe alty +fati ma +east ville +dream ies +dio ts +di yar +di bbs +des sus +demon puppy +co bar +chry soprase +chan soo +call out +cali gari +c fas +bo pping +bio data +bin son +bil dung +big money +bell field +bd world +bce fa +aver n +ar ice +an cur +amb ition +ale wife +ag uri +adrien broner +aarhu suni +ðŁĺį ðŁĴ¯ +âĦ¢ ! +à¹ģภķ +za j +with ings +wau seon +ugandac ranes +tux edos +tr ung +to z +tin u +thegro vela +the csp +ter f +sunds vall +sty lers +shirt waist +shel ar +secon dam +sea field +scele br +sar aya +salu brious +ru fino +ru dhram +ri ott +rc ds +rajkum mar +ra don +p ity +ot su +official gtfc +o gg +ny nj +mosthand some +mor se +moon walkers +mikha ilo +methus elah +mans ard +mal one +lumi ère +lo visa +lin as +lift ing +leg omar +k sa +josh ritter +intran ets +indent ation +hv m +hin ck +ha gg +ha dir +guide dog +gu ymon +green town +fro man +friday funny +film day +falcon srugby +en q +efferve scence +di wa +deli us +defin e +de cker +crow child +cor ks +coelac anth +chi ku +chi eri +brian ab +breast feed +brand er +brainfe eder +bra il +bol am +bassen thwaite +annihil ating +angel in +ag ala +ðŁĸ¤ # +ëĭ Ŀ +à¹Ģภŀ +zz era +za rena +x ambassadors +von der +under studies +taste ofthe +tal pur +summer all +sudden link +starfish club +spro ducts +ship ti +se ite +scot papers +sc pol +s br +rw u +rajah mundry +r dio +pur ush +policy holders +pit an +photography lovers +pal lette +oz on +ou zel +organi sts +nu hu +ne pon +muk bang +milk music +megab ytes +map quest +make s +lux or +low enstein +lake street +k dfw +jessic abiel +interpret ative +in audible +huiz enga +hil dy +herz liya +helic onia +heil bronn +head e +hal se +grid ded +gin osa +giller prize +geo tv +ge vent +gath bandhan +g eli +followmeand lufc +focu srite +fo kin +flaun ted +fist fight +el ol +desh on +demo polis +cu zin +chil lum +cataly zing +boy ko +beet z +back logs +ar il +apple man +ang ou +an jani +abu jat +.... ...@ +ðŁĴĹ ðŁĴľ +ðŁĮº @ +yoak erson +yam pa +y azi +wood loch +wol stenholme +wip joy +winner winner +wild trails +wendy davi +wendydavi stexas +war ped +vo guing +ur yn +u rock +turf care +tow ler +timpano gos +ti gran +thic um +thear can +the charlatans +te var +super delegates +su er +star there +st event +sri kak +sk iss +single because +sin chon +shadow lands +sea ice +se gi +sb g +s see +rally cars +pott sgrove +pan macmillan +onther un +onec lick +omar ket +newhol land +nbc u +na dez +musc ari +mel loyello +medic om +mcen ery +lu mut +liveonk val +l win +kro p +kogar ah +kind l +killer instinct +kap ten +justin amash +john lewis +jamespure foy +indvs ban +hydr onic +hud ler +hoo fs +home boyz +hom ero +ho c +her ms +havasu pai +hanni fin +gu avas +gr itz +glam bert +gen bosch +gadgets now +foh len +five ways +fin ian +extra it +en dimp +elong ate +ebol are +eb be +du dh +dor st +dj arum +dishon our +dee boss +daysof halloween +dan c +cullo whee +cu ts +corstor phine +comp ere +committee man +comeon england +clay mont +chin moy +child health +char cade +capsaic in +boulang erie +bosco v +blo lly +black stars +bir bigs +bigja yoakerson +bapti stery +atre ya +ascle pias +are la +ar bs +ad elie +abre w +ðŁĴĥðŁı¾ ðŁĴĥðŁı¾ +ç¾½çĶŁçµIJå¼ ¦ +âĺº ðŁĺį +ಠµ +zuc cotti +witch finder +wester feld +w azi +um ni +tri pple +tr ons +tou bkal +tom ikoshi +these days +ten dy +summer inthecity +sp sp +ser ch +ser ata +sch ola +sauer land +sar rac +saal bach +rufu shound +retail tech +reign fc +ray son +pro jets +prefe cts +party likea +pan ca +over threw +or to +open office +od walla +oak ed +nord land +net wor +nat rona +michaelian black +look north +le us +le eland +lamar athon +la fond +l suf +ko c +kal ender +id s +hynd burn +hook sett +hill side +help fulness +help find +har din +go towv +giro lamo +game plays +franki eco +fon tes +fi des +familiar isation +falken berg +elm street +dream ily +downe ast +doors open +door ly +der mann +de fazio +cur now +cu lia +com pl +cheese steaks +ca bez +brian solis +blo ss +black dog +bhuv neshwar +ba quet +az v +arab i +americ or +aco de +ac ac +ðŁĮ ¡ï¸ı +ï¸ı , +ìķ¼ ìĥĿìĿ¼ì¶ķíķĺíķ´ +ze w +youngh usband +we kiva +war ty +walkawayfrom democrats +venetian vegas +var kala +v city +us mani +thom az +thi ya +th ae +tequil aday +team wear +te ju +taylor twellman +star link +ssc supports +sphoto aday +spaw ner +sole dad +scott sburg +saw tell +s restaurant +rukmini maitra +rte pt +ron icle +radio gram +rachel e +pushaward steam +popul arize +polymorph ism +pent on +park academy +nor berg +new shd +ne vr +namo again +n ms +mä r +my level +my int +mire poix +mi mas +mcgillic uddy +mb ali +lun ds +lovel and +like the +kitt u +k young +k tour +j ook +itslav anya +inter house +incul cate +ig ai +iam raj +he ward +ha eckel +goh pu +geor di +gallo ped +fm revolution +expre ssible +egg cellent +ed onne +ed cny +donof rio +der k +delgad illo +del anie +de cky +daw at +comb iner +bur nol +break time +bran didentity +boy an +bird ville +ber io +barn ton +ar isai +al ate +ðŁıĥ ðŁı¼ +ðŁ¤Ĺ ðŁĴķ +z ro +yo gare +whatchamac allit +wallen paupack +trol lied +tren chard +transi ents +tom burke +thereal t +the island +tal umni +ta pan +swis sair +swil kins +stargate command +stand on +spon sored +spe ights +small cap +shadow ofthe +shad ri +sel t +sandakoz hi +ro chon +ray ford +projec tionist +produ its +pre press +plat ina +pak tia +padra ic +p mma +p fau +or lé +open water +official brianab +o sian +o ols +o ben +note toself +night breed +nic ho +nephro tic +nar aco +mrricky whittle +mou le +mb na +makeup geek +mai ja +ktul news +ko ji +jo v +in syaallah +iam rapaport +i wl +humor ously +han scom +gut mann +gran ato +gene w +geek wire +gardat raffic +gale ano +fossil ised +fer lin +equi ano +energy drink +en chong +emo re +eg ill +education matters +diony sos +desc endent +dele momodu +d oooo +cur rin +con is +com pr +chuck led +chester man +chess men +cau dle +car apace +capit aine +cap tor +cam eco +cam bs +buck master +buck by +bo ad +bfi player +ashwag andha +army worm +ar mata +app raising +ap ical +all ingham +å¼łèīº åħ´ +zo hra +yor gos +wing ate +water wednesday +walber swick +w add +ut din +u go +trum peters +tre ver +tar lton +taco day +spat tered +social isation +smith and +sky dance +sillu strated +shop ing +shi er +shan ina +se meru +se ib +santho sh +sai baba +ru sko +rrrr rrrr +ros man +ronkon koma +richar ddin +queensu belfast +quad ran +py ros +philli pson +pe du +onep lu +o sim +north enden +nel ms +nat itude +nam rat +naji my +mu b +mor ar +mn ps +milli pedes +memor ias +megab ike +me sos +may bury +matthe wr +man nen +mai z +ma bini +look out +like apro +lec key +la veen +ku ehn +jobs report +jku at +iw g +ive y +indubit ably +i drees +hog sback +hockey family +har lee +ham e +guadag no +glene agle +ghj nancy +gastroenter itis +for more +foam us +fly boys +europa africaus +eti enne +ee red +do oly +dh cp +del ilah +dang led +cine quest +chondro itin +chas ma +cambo dians +cali x +bwp mag +blit zes +bad u +au key +as weetz +as pr +aryas milesa +ann us +ami go +ami en +ame mories +albu min +al fond +af fen +ac lass +abu ena +abscbn ballfanfave +ðŁĺ© ðŁĺĤðŁĺĤ +ðŁĴľ ðŁĴĹ +ðŁĮ¶ ðŁĮ¶ +ç º +âĻ¥ ) +á´ į +zab ar +y st +voor trekker +tull amarine +trilo gies +the union +the outdoor +tern ary +t boss +sympathis ers +supportsmaller streamers +sun leashed +stay curious +softhe sun +so kratis +so beautiful +sap hira +safavie h +rim rock +repe chage +raff lesia +prote stan +propor tionality +prior ity +parana que +pap ri +outer most +or of +one another +ole sen +nit ra +mon dial +masto dons +ma go +lord sof +long year +libr ari +li gion +kir yat +kf st +kai kai +itb berlin +iron men +in om +hohenzol lern +har bord +ha beeb +gui on +gop nik +gem elli +gar rel +full backs +fru ctis +for going +fl acqua +fin zi +family search +ei j +ed gard +eco logies +du plantis +deer stalker +chul alongkorn +chiem see +char rette +cam tasia +but ties +bo si +bitt u +abu eva +çµ IJ +âĸ Ĵ +à¹Ģล ย +zipl ining +zi all +welles bourne +vermark ter +technic alities +sylvia earle +swag gering +sv rofficialsite +suppre ssors +stru ve +storm chasing +stan way +so logy +so jo +sl ender +sim la +sicklec ell +shko dran +sens it +say uri +sang li +sa jal +round top +re constituted +ram usic +rain raingoaway +q h +presen to +poste co +positi vism +oviya asweetz +outside isfree +oscardel ahoya +oon agh +ol ata +noise maker +nj ed +neck deepuk +nbaon abscbn +mour ne +mit ton +milose vic +mi rah +medi os +markie ff +maneuver ability +lu koil +kier ans +ker shner +i sec +hymen optera +hi bou +he ye +ha dad +gold line +flo rent +elder ly +el mer +direct mail +ding man +de sha +day man +cri sil +con spires +collo id +cla pper +chi vette +bobb yl +black ball +bla a +bhan ot +bag lioni +back tuesday +b center +av ger +am axwell +air c +ad ap +;; ;;; +ðŁĺ¶ ðŁĺ¶ +è · +ö rebro +zhangji ajie +ze hr +your city +yod elling +wq ed +world traveler +wipe homophobia +wgn morningnews +wampano ag +tol puddle +thamma sat +sur ry +stri jd +sti o +stargat enow +st arer +sorren ti +sna king +sm qure +smqure shipti +sla ir +sho yu +shi mabu +secon omy +saq lain +sali sh +sa ine +s folks +rim world +retro futurism +reci proc +real oviedo +re ko +re but +ra ziel +quincean era +prohi bido +phd advice +pete holmes +persu ades +passer ine +oz turk +outla wing +ord ina +on elife +new borough +n tc +mur muring +lytt elton +louis bourg +living wage +kr l +ke fla +kar jon +it sines +iphonec ase +indiad st +hy rax +hide yoshi +gwil ym +great places +graceland tv +gd w +gas man +faz il +fawk ner +f wisd +f bun +en co +emancip ate +ec ma +dog sled +disband ment +descen der +cre flo +cor ries +coch abamba +cle ft +cheryl strayed +caw lidge +cast ella +cand our +camelli as +by field +business objects +bur tt +bump us +body line +bit mex +belle ek +bangalore mirror +back er +atic sofc +as aka +ar nell +anz hi +allu ding +ala inf +abscon ded +¦ ¬ +zam perini +wheat kings +weather aware +victorias murfit +vic uni +train z +tin er +theech ola +the kids +ta day +sw ct +sud heer +su cia +ssmusic tweet +soci ally +sens ys +sch itz +sc ct +rock ymountains +radio leary +pat inated +partri dge +par mi +over filled +offic ina +o strom +o shaw +no reen +mo anal +mini max +miam bode +mi kay +mge xp +mega fest +me glio +man gat +maison neuve +long stone +lin tas +le hua +ko ven +kla x +kan gel +justi fications +jesu si +hor rell +hard scape +h vc +feld spar +fat lanta +f andi +envi ed +ell ner +el baz +direkt vermarkter +dense st +de vel +dartmoor npa +dar ina +cu ir +colon say +clar in +cen sored +cb ssacramento +car ting +cab an +bur berry +autodesk revit +as car +amar u +amand aw +al ness +ai der +acal cio +ðŁĻĭ ðŁı» +ðŁĶ Ī +ãĤ¸ãĤ§ ãĤ¸ãĥ¥ãĥ³ +vau dan +val et +taka shi +swa ke +sw ole +suffic ed +stra iners +sprint car +spe ts +sk oog +shille lagh +shab ira +ser ran +se watch +scri me +sap ient +santi ag +saint petersburg +s against +rex all +re stom +re released +prin touts +prematur ityday +por voo +pharmac are +per kasie +pass ant +p tz +ou den +oo ch +o eming +national bank +nath and +n tra +moving on +mor os +mo hen +mmac donald +ml baz +metho de +mess enger +me geve +marion berry +l anner +ko shy +interrup ters +ic ci +hom burg +hep ner +har rass +god ble +gil bey +ge uze +gaku in +football club +end games +en sler +eb brown +dry burgh +dram men +doubler ainbow +distracting ly +devol ver +defl ationary +debbi em +dealwith barbie +david stea +cv usd +criminal ising +cleveland birds +cla vi +chugg ington +chel seam +car mike +camp ylo +boat ing +bar bey +b caas +azadis quare +atla ses +at more +asap twelvyy +angi ec +alwaysinourhearts zaynmalik +alphabe tized +ðŁıŁ : +ðŁį ¢ +ðŁĮ ¥ +à³ Ģ +ÙĬ ÙĦ +ö lln +yon der +ye x +yas un +way n +verso books +us nationalguard +us ki +ur mia +uni g +tur bin +trus cott +toc cata +thr s +the bug +team uk +te ren +songsong couple +songkh la +son ho +smith ies +sci anna +schwit ters +saram ago +sad y +s fai +ry l +ro ty +realjoey fatone +r ly +proté g +pro viso +pro start +pra k +per cheron +par took +ow ch +oto gp +osman thus +oman air +o vas +nw f +north pole +muham ed +min ota +mg tab +mens fitness +mark kanen +mar ye +main board +mag al +madra sah +lu chi +lovel ady +lap sing +kon nie +keith harkin +kear sley +ke pala +k pp +k ameen +jw marriott +j pp +impregn able +i bye +hant u +hab iting +gunner sbury +green jackets +gol and +gh es +gestu ral +gerr ards +free agent +foodies festival +fo storia +fir ste +emb leton +ellicott ville +ed aily +down payment +don nacha +deser tion +deca icdc +de wald +de jas +cri p +cook top +cole tta +claire coffee +cha yote +carri g +by er +book ham +bob bie +ber gy +bal asaheb +at wal +ant enne +ann elise +andy ar +ai x +ðŁļ ħ +ðŁijĬðŁı¼ ðŁijĬðŁı¼ +yr ds +work fare +whir led +whin chat +wg no +vivi en +virgini atech +v art +uni ge +thelast drivein +the guardian +team bcps +take off +sthel ena +re sold +rand leman +pu ga +proud mum +pper man +post es +por res +polis ario +ph oria +per icar +per der +par it +om ana +ny ro +nw sl +nra am +non conformist +nigh tow +neat ness +ne sha +nawal el +nat rev +my love +mount view +man nan +mad flavor +ma slen +lyn skey +local memphis +lili reinhart +l kn +kwa cha +knowyour rights +kil gariff +kei yn +kay ong +jarrod farmer +jaros z +jamesx reid +ine ering +in putting +honor them +groen eveld +gotta seeit +fitz museum +fad own +esp o +en hs +econ et +e sts +e star +dinis guarda +di mo +design ates +cru mpton +centr alization +celebrity awards +car aga +british rowing +bg sconf +ber ated +be art +armor y +armag nac +arch es +ar onia +al ico +akinwun miambode +aki hiko +adi az +ac ord +." âĢķ +ðŁİ¨ # +ëĭĪ ëĭ¤ +à¹Ĥ à¸Ľ +ਠ¬ +y oma +warren ville +w ft +ve ere +union pacific +unic o +u oa +thrust master +the ovon +texasedm family +tecno logÃŃa +tanam rull +tach yon +sydney mardigras +sur an +su wa +su tanamrull +steph ania +societ yof +ske ins +sil loth +sen ora +scott m +sco pa +sas sn +ru ffa +rol ston +rhay ader +rep aved +re se +r ll +queen mary +prevention month +pre amplifier +paw lowski +parole es +nit zer +night vision +nh sm +n jr +mogol lon +mo fs +midnight sun +micro meter +mel ati +max us +mar adio +mam malo +mai oc +mac isaac +maaj id +lympho blastic +loving intan +le cky +lam brewery +krzy zewski +klu wer +keep ingthe +kat anga +judd monte +jing yu +jin xing +jab ir +ist as +hyphen ated +hear ken +he slop +gu ana +gram mars +goldeng ate +go ree +gi me +ghos al +gach anja +for ged +flyer stalk +fish on +evi denti +epitom ized +e jay +den airport +dar ude +cor nic +co bby +car rum +bobsburger sfox +black shirt +bio genesis +ber litz +balli ol +ayan ami +ark ell +ar ryn +anag ara +allianz leagues +al wan +ac land +a kel +é ed +yvr shoots +ym pia +yam anote +volta ic +ven i +un saved +tug boats +travel show +too hey +toc coa +thir un +the do +ter racing +te questa +suffra gists +spoil sport +southern cameroons +sk ok +sc illy +rud nick +ru tt +respon sable +ram rod +ram ji +rajiv gandhi +quar ta +pre match +play ford +pj hl +phal le +per saud +par ool +nypd protecting +nul and +newengland patriots +new speak +nagu ib +mwal imu +mur at +mor tlach +mor ada +mistre ating +meli ke +me ment +mccon key +mc par +mar lette +man fre +man dered +maffe i +lu so +leve es +lead beater +lea g +laroc que +kve w +ke ira +kan san +jesse welle +jam u +jake bugg +irrig ating +ib w +hyper dub +hic kie +harri ette +gol la +gar ter +ga ura +fla vius +fiore llo +fas sa +ev ened +er ney +elb philharmonie +el ore +dishon orable +diac omohoy +demoneti zed +de havilland +de at +dare us +dak id +d london +d hh +cro aking +club card +citizen weekend +choo o +cell mate +car tier +br ani +body care +bo du +bam mer +b hoot +amaz o +ag bo +! )... +zip code +z aven +wed der +wat ta +wan ti +unleash the +under scored +to sti +the cove +spits bergen +sol dat +sl t +shum lin +sel ter +sav ouring +sarac eno +sab a +rit orno +ri fa +re combination +r gruppe +qos baszler +pos tharvest +pale stina +p ite +over rule +out landers +orti z +oren go +oise aux +ofcal dub +neil patel +national teaday +nac l +n bu +msdyn crm +mor ong +mon keying +mcgavo ck +lefto ver +lebo euf +jap androids +jag off +jac keted +international ism +ink pen +indi ah +ido f +iceland ic +i follow +hour cloppic +hi aasen +heu vel +hd tvs +hat cher +har ak +ham or +hahah haa +gunner gale +gra bb +gab ion +full bright +fon zo +fb x +f agar +encan tas +eisen ach +dx racer +du gard +del k +cun ningly +coming to +clock ers +citi bike +chal le +bru jas +biggboss marathi +ba el +b hit +az ira +atlant adream +at omy +assalamu alaikum +ascen der +arte mi +aqual ung +antimicrobi als +ang elle +albat ros +^ .. +ðŁ¥³ ðŁ¥³ðŁ¥³ +îIJ Ĥ +اÙĦسÙĪد اÙĨ +ı n +ye ayy +watt pad +viter bi +u et +trisk elion +trespas ser +trap star +to kin +thirdeye blind +teyan ataylor +tanush ree +tan ko +statecap ture +star vs +ss l +sq r +silic ones +sexu alabuse +sas an +sarah spain +saint sand +roth mans +retali ating +re pulsion +re mixer +re kord +rapper swil +pun intended +pri vett +peter house +or os +noti zie +nighto wl +neuro logic +nett v +nelson ville +nad in +musand am +mun jal +mug wort +mu ska +mo zzie +mc up +maxim alist +madel aine +ma thy +lone wolf +lo sail +liverpool phil +ling cod +lg n +kowal czyk +kixi fy +kis ke +killla kill +juan cho +jaw i +it ic +ip at +ing sunday +ie j +hourly wolves +hd z +hasna in +grig son +great dismal +gooner family +good afternoon +gol de +ge pp +fé lic +fuj ilove +evosti kleague +episte mic +eper nay +ende cker +eli vers +el lec +dutch mfa +dor it +dod die +dist ills +design mag +deseret news +de crying +d nl +craw shaw +cow den +corpor atism +com modification +champag nat +canon favpic +bwl ch +buli mba +buck hannon +bor le +biggest fan +bic ton +beer lovers +baseball america +ban ishment +badger up +australian story +atic ket +anesthesi ologists +always remember +ak se +ab am +' + +' $ +! ðŁĺı +ðŁĺľ ðŁijį +ðŁĺĨ ðŁĺĨðŁĺĨðŁĺĨ +ðŁĵļ : +ðŁijı âĿ¤ï¸ı +ðŁij ¥ +ðŁĮ¹ ⾨ +ÛĮ ر +Ùģ ر +yester day +xbox p +war monger +war der +vo teen +unni krishnan +universi dade +un recorded +u vu +trici a +tree planting +tof dixie +thi stown +theo broma +tal ing +swan ley +sunday times +subsidi se +struc tured +strom lo +ss ello +spre zz +sleepa way +sch art +s spx +rw jf +rose ellen +rik rankin +re ux +re faeli +re elin +rc pch +qine tiq +preju diced +pl inking +per dita +pashin yan +our nament +open shaw +odd parents +o sk +nano second +mou lay +mo pped +mission accomplished +min j +megabike shop +mari usz +madein tyo +london art +lan gue +la ks +kir ky +kar jat +kalamaz oo +kal ancho +jared padalecki +j ind +iv rea +ioni q +ingh ana +ing erie +im bi +ik kar +hu sein +ho x +hear metoo +hay good +gy nt +grim ly +glaad awards +ga steiz +fox gloves +fest ina +farquhar son +ever age +edi fication +dow den +desp res +dan so +clif den +clean ness +cle av +che vette +chatt in +car seat +cape may +building champions +brea stre +bo karo +ble y +blank fein +bed font +bastian steel +baske try +bas sc +ap gar +andhra pradesh +alt ay +ak aka +ag ius +? ', +.... !!!! +ðŁİ¤ ðŁİ¤ +ðŁĩ¿ðŁĩ¦ ðŁĩ¿ðŁĩ¦ +ãĢı # +âĹķ âĢ¿ +âģ¦ âģ¦ +zy y +xen opho +worcsc cc +wood sen +wl st +winter break +wi rele +wh on +wax works +vote tris +velocity conf +ut l +unidenti fiable +ucan r +trump ington +ti anna +the common +the actor +team wales +sur tout +strato fortress +star creative +splat z +soe urs +smolen sk +shri kant +sel vag +ryanj newman +rudder less +ro zi +pun ning +pun ic +power station +pop an +point lessly +perri kiely +o base +nu cci +nt ca +nish ino +nat sci +my the +move set +mo gi +mlbaz fallleague +mitch elton +metam ora +mazer unner +max ton +ma dra +kun zite +kov ai +kon erko +kol l +kk rha +kip sang +key z +joanne worldtour +je mez +isspyaar kokyanaamdoon +impregn ating +horizon league +homin id +he resi +hax by +har l +hal onen +ha dro +h antz +gui don +growingup in +go izueta +gener ativeart +gen pact +ge biet +fre scas +fi yah +fa stra +essex policeuk +ero ge +ebulli ent +dr itad +dr ita +dor ina +don broco +diacon ate +devan shi +deter gents +depri ves +delpiero ale +dai do +d kw +cre eds +cra ggs +cohesion policy +christo dou +chowki dar +chast ised +champion strophy +ch m +cc mariners +carac ara +bold mere +bo los +bludge oned +black thought +barto sz +bad deley +baaz igar +ayo ade +ay ase +ar don +antó nio +ann ul +ang ers +and rae +anaphy lactic +all black +air ships +ab aqu +ðŁĮ¸ # +z schech +win etour +wic b +weather ization +vocabul aries +verizon wireless +van stone +us ap +un govern +u turn +u pi +u ai +twittb laster +tuber ville +tar pons +sway am +sprint cars +sol dotna +sne ha +sian icri +screen junkies +sci oli +sci ed +san filippo +ruby wax +ro screa +registr ationday +re stive +re iro +razor light +q amar +proftim noakes +podi um +pere tz +per sis +park ways +pa estu +one planet +nucle ya +ni ppers +mu choo +mo one +mil loy +mil ak +metr is +mer chi +mediev alists +mat exp +mali gne +loren za +leop ol +l ents +ky thera +knickerbo ckers +ke hr +jd m +j tv +igle sianicri +hu ay +hi muro +haryan to +hang ars +fo erster +fei joa +fax ed +fau j +eter nia +est rena +ep ay +emc fly +eid al +eck ington +ear wig +dun robin +dormit ories +demon et +dell acqua +dear zindagi +david l +college board +cole ford +char ak +chap atti +cay den +car don +by lines +busi ek +bran del +bradley james +bny mellon +birmingham rep +bing ed +basel ines +barbar aniven +au m +as sery +arcli ght +arad wanska +an net +ali al +ak ram +active learning +academ icians +ðŁİ ± +ãģĹ ãģŁ +ãĢ°ãĢ° ãĢ°ãĢ° +âĺĢ âĺĢ +âķ ¬ +zu mi +zu mbi +world ginday +will ard +virging alactic +value investing +us in +uniter ight +unfla ppable +uk y +trending orange +timo fey +thru sday +thor sen +thel er +that chers +systemo fadown +str indberg +ste eze +spur s +spr ts +south india +solit ary +shing al +she athing +sd hawan +roseellen dix +profu mo +portugal theman +platt smouth +panther snation +o ake +nychash tags +noo tropics +nine ty +ni harika +mn u +mis directed +mik uni +mi dol +men ounos +me dre +ly sine +lucky welive +lor na +live forthe +lemu el +lawand order +lapid ary +ke ane +kat aria +k long +justfor laughs +juli elmo +jp montoya +jonny existence +jo do +israelic rimes +infra red +hol lo +ho thead +gov murphy +g uni +fir le +ff weekend +e izo +dr ferdowsi +disinfe cted +dish ware +dimit ry +de bartolo +d hh +cool stuff +confection ers +chori ster +cell biology +cable way +cab overde +bur ch +bring themhere +brand name +bra ban +big krit +bette davis +berlin station +beng t +bbcradio solent +bayare a +ayesh acurry +as pic +antagon ism +ani plex +and park +ah ram +è Ħ +âļ¾ âļ¾ +à® ³ +wholesal ing +west away +wer c +v ch +uninor thants +thread bare +theori zing +thegrand tour +tend ril +tan ews +t zi +sy lac +sy beria +str an +sq lite +so tm +smart cares +skylar astin +sj news +sigma pi +shin ned +shill ington +sen diri +santiago de +s fanart +run rig +ru hlman +rep maxinewaters +re thankfulfor +ranger over +qu ini +pla k +pho u +pap ale +paintedby carol +ob l +ny books +no graphy +no buy +nam ic +mod afin +meh dir +mask march +mar cri +manhattan beach +mahindr arise +macdon agh +lu nette +lu beck +lisa kudrow +lafar ge +lady love +la im +la greca +kin berg +kham mam +kam per +kal ine +ho chim +her k +he ins +happy canadaday +hairspray live +gobigor gohome +gi yani +ge thu +ge ren +gal let +fun kand +financial post +er ni +er cole +ele mental +ed gy +e tic +deta inment +dehuman ization +deer wood +clive standen +cl anging +ci ona +chis elled +cey lon +can tons +breath lessness +bour don +booth scountry +blue hole +berg ere +arca chon +anthon ye +an tu +amphe tamines +amo dern +al gin +agains thumanity +ad elia +.. $ +. ðŁĺľ +ðŁĺĥ ðŁĺĥðŁĺĥðŁĺĥ +ðŁijĮ ðŁĺģ +ðŁĩ°ðŁĩ ¿ +åĽ £ +ÙĤØ· ر +zodi acal +zim bra +yes yesyes +wof ford +win rate +vo guet +vin os +val ori +under scoring +tread away +travel diary +tow sky +ti rado +ti dwell +the darkknight +tess avirtue +ta ako +stenhouse muir +star date +stan kovic +sri ram +spre miere +solo ed +shin ra +samsunggalax ys +saaf ni +s fera +roman e +ring road +pur vi +prin eville +posteco glou +pashup atin +paras port +ow www +no ar +ne aly +natural remedies +msg thefilm +mobi lec +mire ya +mart is +mal at +letsmo ve +le um +lat sondheimer +la kat +kotton mouth +kerry n +kav alier +jo chard +jim morrison +it off +in hofe +ig bo +hood rich +gerry mandered +frankie boyle +fe vers +exempl ars +er ling +eddie trunk +earth ship +e got +dit ors +der spiegel +dau mier +czech gp +con des +clar abelle +chud leigh +bur gher +bro die +bot son +black cosplay +berlin er +bell more +bal bir +ann ells +alway strump +abo des +ab salom +ðŁĺĤðŁĺĤ " +⼠ª +⤠´ï¸ı +ä l +ye veryday +yan ina +wx man +world beeday +win eclub +wh sv +van etten +valenci abasket +upon us +un conquerable +tt able +tra ini +today sor +thread needle +th starmagicball +te ks +te agarden +sub space +sty ledby +simil itude +se ol +se bas +san marino +sab ana +ry les +rt fm +rev lon +red entor +recru tement +proud parents +priscilla shirer +orni th +oraclec loud +or é +ophon ia +on erichmond +ohl rangers +o clc +nam bour +nalban dian +mv hs +mudh ens +mu ze +mothersday gifts +mehdir hasan +med ce +mau rawest +man group +living in +lam port +l iciou +kon gos +ko an +kli en +ker mis +kab in +j su +islamic art +ishqmein marjawan +ire do +il lette +horse trials +he us +harpercollin suk +hahahaha a +ha fta +gv hd +gatt aca +fearthe beard +fashion bloggers +enchan ter +em pre +dritad avanzo +dor as +div omovies +dir l +dijk stra +det mer +dese crate +dar vey +cri stie +colour fully +cel in +capital onecup +bury fcofficial +bun ratty +bse ssed +bre wed +billi epiper +bil k +bi ze +bhuban eshwar +b aga +arisai g +ar gy +ai ib +ade va +ac si +ê² ½ +ಠħ +wy playhouse +wood church +wha at +w brz +vereenig ing +tin ton +thick nesses +suz annah +sturte vant +sle monade +sho ku +shil pi +sh lo +senbob corker +ribo some +rei wa +ram sden +quin ault +pigeon forge +perv ades +pav lich +paper mill +official mcafee +o sum +o sle +nis america +negre anu +need syou +moncton wildcats +mis smy +mis firing +melo di +me ko +mastodon music +lil let +light itup +lic key +li sas +land shark +land forsale +lady superstar +la ffey +kumbh mela +kin ton +khi mar +keepp ushing +kar pathos +kan ako +inglou rious +hyper ion +houston dash +hi doe +hay loft +great ocean +gou den +gor z +gla ziers +euphe misms +eug ène +etsy jewelry +ent l +drew diplomat +dot co +dor tm +dj ed +di ffer +deccan herald +daw gnation +dain ian +cry me +cr cs +coble skill +chum ps +chri sman +choose your +chao tic +chandra shekhar +chad leclos +catch fire +car spotting +camera work +born holm +black mamba +black facts +bick erton +bell hop +bar tsch +baldwin sville +aru les +appe ased +apo se +and fairness +alt bier +ah waz +adam buxton +absen tees +a aww +. >> +ðŁĺ±ðŁĺ± ðŁĺ±ðŁĺ±ðŁĺ± +ðŁĶ¹ @ +ðŁijĩðŁı¼ðŁijĩðŁı¼ ðŁijĩðŁı¼ +ðŁİ¨ ðŁİ¨ +ðŁ¥ Į +Ùħ ج +Ø§Ø Ń +you aint +yo gotti +wonder girls +winns boro +van lis +v cfd +used car +u know +timmy trumpet +the scott +the cameron +t mnt +starvs theforce +sm elly +sl vaus +sil on +shuja at +shi d +sfor za +ser res +sch ot +sage uk +saf avid +rock lahoma +pol nare +podi atrists +peter hof +ori huela +one gerton +net suke +national sister +myla pore +my boys +mu sang +moun tup +moro goro +mor bius +moh tarma +mis diagnosis +milli ebbrown +mephi stop +lu uk +lock himup +le to +lakestreet dive +ke iron +imper me +iihm hotelschool +iconocla st +i quitos +hen dee +hat box +harri eth +har pe +gt foh +griff is +gre enough +ger o +gent illy +fu gli +f cat +e qs +dino bots +dham makaya +der ricks +de vex +day anand +david gold +dal ry +dad do +cf daawards +ce sen +cam os +c ff +bran do +boot strapped +bloo duk +bb ra +bar ua +ay ane +av radio +archi plain +ar no +aquil ina +ang ling +amwriting romance +amuse ment +ale thea +al ms +.ãĢĤ .:* +. ðŁĺįðŁĺį +ðŁĴķ ðŁĴĹ +ðŁIJIJ ðŁIJIJ +à¸ģภ£ +xx xv +xtian bautista +wreckit ralph +weekend wanderlust +we build +wap is +wad er +up mann +un savory +u dang +tre vose +tear itup +tear in +su turing +stri ations +sto wing +status quo +song ket +sinter ing +sight seers +showtime pettis +sher rington +shah ri +sele kt +ritu al +ril ton +rec i +re ge +public school +pre ggo +pen iel +pasqu ale +out posts +ns j +nike women +name tag +na ama +mun ity +mor den +miller time +mel ville +ma skin +lyn ley +lei der +lang leav +kin agr +juco product +jocke ying +jo ad +jam mie +interro gations +instruction al +imbru glia +hay tham +ham mamet +hal ina +gu ma +goron talo +gam mons +g dor +fruit land +frey cinet +fox friendsfirst +fly with +fag ot +en ma +em ale +elian enko +doloreshu erta +del hin +de matteis +d zone +cullin ane +crim six +cre mon +corn cob +col ditz +c rich +blitz boks +ational day +as av +angel ino +allyson felix +ah us +ah ur +ag lobal +!!!!!!!!!!!!!!!! !!! +ðŁıģðŁıģ ðŁıģ +ãħ ¡ +âĹ » +xfactor au +wigg inton +weta workshop +western digital +wen ning +wel lesley +we comefromaway +waubon sie +var man +v ho +tou charcade +ti gn +the oxford +t ne +sun city +stan chart +square mile +spl unk +smoke purpp +small streamer +sk aff +shal houb +self storage +sei ki +se mmes +save energy +rend all +re ee +ramach andra +ram li +ram iz +public ed +psycho logy +pil ings +phillip island +par ad +pan theon +own cloud +onthe board +on cology +ome tepe +om bija +ok ta +ny mr +ne gar +mud flats +monster sandmen +mo zza +mckin stry +mccau sland +marsh man +log sdon +light fest +lam bo +l ilo +kris bryant +kkkk kkkk +ken roth +ka am +k sc +jean sfor +jaz zie +ist d +i fera +her om +gun ns +green andgold +gram pus +gig ant +getting better +fil maker +feni more +fal ters +eur gbp +en ot +droner acing +daryl matla +cy outh +compre sses +colla do +co h +christ offer +cer aweek +cas carino +cafe coffeeday +blog to +bir ley +ben ward +bab by +ay outh +as oka +art ine +anu fac +al lier +ak we +aer ate +acru ises +aa ai +ðŁİīðŁİĬ ðŁİģ +âĺºï¸ı . +zoo atl +yi v +worldvegan day +worl die +wor don +wind break +wh orl +vote remain +visual cap +virgin mary +vi stula +utd football +ut c +uro logists +u str +tre ynolds +to golese +thu li +tex til +tand ragee +ta qi +streamer network +stock sbridge +star tin +srisri speaks +spe cular +sou se +simil kameen +sil ents +shu o +sher gar +seag le +se guin +schri st +save aca +san cocho +reassur ingly +re eta +ra shad +pumm eling +petri fying +peter frampton +pel frey +pear le +pan cha +operade paris +nu fc +novo tny +nerd hq +nen ad +mrss osbourne +mou stafa +mitchell reports +mc glone +lov at +korn acki +kkrha itai +kinagr annis +kavanaugh hearings +jal di +hospital et +han ham +go geocaching +gg as +fies ole +fcbb asket +fat u +f mofficial +er witt +elo i +dro b +deline ation +davidg ilmour +dam pener +cur sory +cr acies +copy cats +colloqui ally +cognitive computing +cher moula +catal in +camp allroad +cab azon +brook ville +bro myard +brau lio +bou ley +blan co +ben brook +balear ics +as fcofficial +ar asu +ani mage +amu ffin +amou reuse +allu ded +aco x +ach med +aar tic +( ... +ðŁĹĵ : +ðŁĶ ¯ +zor k +wer de +web perf +vo coder +vi bro +vas wani +ut us +ty oung +tre mbles +tow anda +tom sk +teen aged +tab aco +t ju +supere go +su mbur +stock piled +steel man +sse au +sni pping +sicili ano +ser ina +same time +sa osin +rumin ation +roo sen +reed sburg +re cker +pound world +pine view +philosoph ically +phantomofthe opera +ph resh +pack ers +olaf ur +mill ersburg +mika ela +mere ce +matar azzo +mark g +manit obam +malevol ence +livesof ww +legal ities +km js +kirlo skar +inste in +in dri +il ai +gry m +gil gun +gau se +g bit +fun clic +funclic kearn +fu hrmann +for bernie +foot board +fohlen elf +feuer stein +fem ke +fat back +fair bank +embar kation +elk hart +earth science +dun smuir +da jj +cross gene +cro isi +co bre +cheese cloth +centr ale +cele st +cave tt +cal pine +bundel khand +buckfast leigh +brush work +bhat ure +bernar dus +bar room +b ely +auto sales +as certain +an doh +am pera +al ha +abre ak +aat state +ðŁĴ°ðŁĴ° ðŁĴ°ðŁĴ° +à¸Ľ ระ +x amar +world prematurityday +wi ens +vat ic +v to +uv xy +truth out +tra vieso +tour memories +ten ny +shu ffler +show business +sail fishos +saf tas +sad ers +s journal +ro xo +ri angle +rhy e +rhe inland +redbull grc +re q +rath coole +r nai +pyo tr +public policy +prolifer ative +ponti ac +pon chat +pokemon xy +plain moor +phenom hoops +pasalu bong +opti musprime +oph ant +oak mtg +new paltz +nan ako +na ea +myth ic +museumof nature +mob berley +million maskmarch +mil ang +mick ens +mer id +men uhin +mar quet +mal ana +macdon alds +ma ec +lou cityfc +koiv u +kn m +king don +je el +indeci siveness +immobili en +ic are +hor nish +honda indy +hipho ped +he ssel +gü zel +google playmusic +god ons +geol soc +g callen +fro mal +flu shot +ex communicated +er id +em rs +elli sville +ed ric +ec thr +eber ly +du binsky +dome stique +do tc +diamon dand +der ose +dank memes +counter clockwise +compar ti +co authors +cmo kerala +cla rem +cj hl +che gwin +camp ton +bestfriend day +ber tucci +be ier +basti de +bar camp +anti psychotics +aktu ell +afric alive +ad mi +abdel kader +ðŁĴĶ . +⾨ " +Ä Ľ +yogotti kom +y usa +we tz +wbc sd +wal vis +video taped +val ter +tw rk +tu pe +trabal ho +toronto realestate +tor vill +this world +theright thing +tha at +terric lark +terr ariums +tech re +te urope +surbit on +stigmati zed +stagn ating +sta rena +special ity +son mp +smi tha +sleep walkers +skook um +shu go +sho we +run dle +ro lie +richmon draceway +revol vers +rend collective +ration alization +qa iser +pro gr +pro dding +plo sion +official ronnies +nj politics +natwest t +nationwide arena +my ton +muhte ÅŁ +mound sville +mon zon +mon sie +milli grams +mckin ney +mason ville +mam c +ma shi +longyear byen +leonard town +la forge +la em +kor ta +kor ps +juddmonte farms +jaw ab +jas prit +jan ez +invinci bility +independ ance +hunts man +hubb alli +hawk stone +god ly +gher bitz +gc isd +from wherei +fran zese +for folloback +ex x +emp son +em sworth +elf quest +ejec ting +east bengalfc +duplic ator +dra iman +dog star +deni ece +da ith +d by +cy lons +cu ota +critch low +cott bus +compre ssible +co ono +clau demonet +cj n +chel o +cen ac +capp iel +can asta +c mcdavid +bou se +blac ki +berch tes +bein art +armam ents +alleni verson +ach ille +abujat witter +: """ +ðŁĺŃ ðŁĺ¢ +ðŁijį ðŁĺĦ +ðŁ¥ĩ @ +zi x +z wart +vi shy +ver me +vani tha +val de +tuske gee +transpor te +tra buco +tr ate +talon sup +speedway gp +slo ps +scapp oose +sa hm +rick shaw +real d +re definition +qui kr +puro lator +proud sister +profan ities +pro plan +primeminister imrankhan +plan escape +pla sm +pioneer dj +pan head +p tah +orlé ans +on ts +oe ster +nir van +ner ules +ne vs +nale o +middle earth +mcr artgallery +ma kani +live bolanet +lig aya +lifel ine +ko loff +kfc barstool +kent mere +jit sircar +iowa state +ind ade +i au +hun k +hel meted +goer ges +go kyo +gli mmer +george soros +fur a +fron tto +febr ile +equ otes +en join +dun nett +dul quer +deliber ated +cuad rilla +col bi +cih housing +chen oa +cast legate +bumblebee trust +bridge tregan +briden stine +bo real +bernar dez +bean ey +bambaat aa +ax ford +arti es +ann ach +an ang +ami da +air pics +................ ..... +ðŁijĮ ðŁĴª +ðŁ¦ģ ðŁĴĽ +íĶĦë ł +zi ba +zambe si +z lj +yav in +whites burg +wh ately +vishnu vishal +vic ent +vi ffest +ven der +van sciver +uni st +travel thursday +til o +thalap ath +ter ni +tempe sta +tal bo +ta jani +sweet sixteen +super giants +sun n +strange musicinc +sq s +se gs +sa hana +record collection +providen ciales +po ku +pips queak +pi ola +pe ja +p music +ore wa +oden kirk +no cover +new smag +n lyonne +mu jib +more head +mont copa +mo gra +mil son +mer iam +mat ted +madra ssa +lippin cott +laure tta +la stic +kis ston +jimmy havoc +jess ops +ja at +it fest +immacul ate +hyundai wrc +htcon em +hol lett +hil burn +hack en +grif ters +grati as +gole stan +gi lead +gel led +gand u +g dad +fri sch +fc thulhu +fal abella +enic ole +eni k +ellen brook +ei shockey +edmon d +eastengland amb +dn ts +design awards +del dÃŃa +deborah meaden +david coverdale +daddario andco +d subverse +ct us +coy p +cor sham +clar inda +cep tions +carol yne +bur cu +buffal os +botu lism +bo ggles +ber ard +bajac alifornia +at sf +ang let +aber avon +aa x +'' / +ðŁijģ âĢį +ë¬ ¸ +za ke +wolfe boro +wig ton +wc p +tu ch +this guy +thermo py +the creative +t dsb +swe ed +suyy ash +suhel seth +stuart scott +spru ill +sp aper +sm alaysia +sanje eda +sal ver +sab ot +sa hrawi +roo ker +rockin rio +ro dale +re spon +raider scanberra +qui sta +polnare ff +pir ouette +per rette +per mutations +pel os +pe abo +our lady +no go +ne gre +mu tti +montero sso +mo tel +lim elight +learn chinese +l á +kul ang +kore matsu +king scourt +kar dinal +jeffd sachs +jake millermusic +it staylor +ifc films +i ach +hoo drat +hol and +hoch zeit +hemb ree +gst tnhs +glo ssy +fuzz ies +fleure ast +fer ox +farmer sville +faceli fts +etsy rt +esc orial +dt phx +disc ur +dimit ra +dex change +dance mom +cov hour +colt cabana +chi me +catastro ph +bridge ville +belvedere vodka +aw ans +ad aw +à¸Ńะ à¹Ħภ+y wam +wood cutting +whim brel +w display +vi agens +usc aa +transi ence +thi as +te dy +tat acompanies +spr ite +sk om +sc ran +sag ne +sa athi +ros ana +ri sto +rec tifying +re shammiya +ram ba +pure storage +pp w +platinum games +onec ampaign +nai ste +na hid +mo cap +mirror football +mir kwood +mer ten +mad docks +love seat +light man +lavi shed +kon achan +ko hala +kel ana +keith olbermann +kam on +joy less +ipp olito +ine ve +i orio +has bronews +gv su +gu sto +gra di +ge macht +fm sphotoaday +flori dac +fin cantieri +fi sted +fast balls +fag et +extrac tors +elu ve +dun kar +diam undial +davi dr +culture club +cul turing +ct news +cor ddry +cop an +clu brooms +cl ent +chronicle herald +ce cchini +cas cia +can ari +cal en +c pac +buck ers +bu ke +bor a +bon da +bill yel +bienven idos +bat sibat +au skas +assay as +apla stic +al win +ac tics +aa ihs +ðŁĺŃðŁĺŃ ðŁĺĤðŁĺĤ +ðŁijģ ðŁijģ +ðŁ¤¦ ðŁı»âĢįâĻĤï¸ı +éĸ ĭ +ภĺ +z ast +wesley stromberg +war road +ver net +un leavened +toy maker +town coffee +through ly +thi ef +thene edle +ten enbaum +tee total +surly brewing +superse ded +sub class +stone masons +starcreative stv +sport susa +spac elab +sof teners +snake day +smo t +sk ylon +shoo jitsircar +shal ane +sell schaft +sec ateurs +schle icher +sale em +salad ino +run keeper +rsa security +rivier anay +red and +rayn aud +puer ile +pu tian +prefer red +pig sty +park minyoung +pariv artan +ophar yngeal +one buffalo +nj ic +newye arre +new a +nc su +nadi ra +mon sta +mir ren +mac robert +low ton +louder milk +lec ity +le bow +le aman +l ú +ku hns +kraft foods +kr ls +kay ag +katrin api +kath rada +kan ai +je be +istandwith maryamrajavi +io so +ill z +il ens +holly bush +hill park +hat ful +gu to +greattaste awards +fo su +embe zzled +el ses +don ite +dj f +dis barred +de mayo +cro ons +cor ney +coal face +co ire +cnn debate +chiso x +ch h +candice accola +calab rian +buil th +bo zen +bo zell +biek sa +ball arat +ba ños +any oung +anton yl +alla ges +ali maging +af gan +z ins +w lox +ur ica +un banned +ul fa +ucla anderson +trevor ombija +synthesi ser +super blue +st ith +sse afood +spre well +shar man +sau to +sap ne +ry no +run withus +ri bot +re sna +rand on +r fh +pp ar +phul kari +parac as +ou fcofficial +opun tia +nuyor ican +no taries +nd mc +mis is +miniaturi st +millen ia +metron orth +me ike +maya ali +matthe wh +man lius +man art +main sail +mac pro +lich ten +kno twork +knightri der +ken shiro +je gede +inde ed +in aga +i ths +he aded +hali mah +fundac ion +finner an +fc go +encant ada +ego ism +edm life +dis agreeable +dia q +corusc ant +colle ens +cn wk +classi sm +cha ining +cas so +car my +bus well +brasile ira +bor ba +be healthy +aubre e +at lin +assemb lers +aldu barkad +aff ing +adjour n +! "... +ðŁĺĬ ðŁĴļ +ðŁijį âļ½ï¸ı +ðŁIJ ĸ +ðŁĮ²ðŁĮ² ðŁĮ² +ت ÙĤ +yod abuda +ye ch +wat ari +w elike +viet name +up skilling +un fair +tt india +tric ep +ten gok +technom usic +t sering +sun coast +str in +stand ley +slo vo +shalane flanagan +sen bill +sch lie +sal enow +rosal ynn +rol on +ro kh +research gate +rb z +popl ars +pl sss +pend olino +panip uri +pan handler +pack y +owen smith +os se +obe di +narc issa +nak ita +na ira +mud lark +mis sher +marke tre +man vs +loss yndrome +lock ton +little foot +lim kok +law alla +lari jani +kre isberg +kkrhaitai yaar +kend ari +ken cana +jo stle +jesse metcalfe +jel avic +jarrah dale +jakob sen +itv weather +its friday +infomer cials +ig daily +ic hou +hp discover +her ridge +hengist bury +hahahaha hah +ha gio +grom mets +greatplace towork +great value +giron afc +font bonne +fic ha +ferry bridge +fabri zi +er and +enviro ment +eliesaab world +el ham +edin bur +echo stage +east lands +d sk +cut throat +crab grass +conse crate +confection er +col lor +cogge shall +canad adev +can ongate +c jeu +bug s +brune au +bru shy +brick fields +bhutto ka +be siege +bb v +basker villes +bad girl +bab alu +av ma +archa ia +ach a +ðŁIJł ðŁIJŁ +ðŁĮ· ðŁĮ· +ล ะ +yu miko +ye sequality +yacou b +world net +woman kind +wo wt +wi zzy +westhe ad +wander lei +waf ting +vat iron +use your +uni fil +trinity learns +ton ow +to graph +tech net +stock ard +stand united +stacey solomon +specu lator +sp atriots +solu cky +sol enn +shett leston +sh ky +schi pper +scheck ter +sare awesome +san deman +sa qq +por ton +pop sters +pit tie +pe gg +panagi otis +pa ille +om ay +olympia stadion +nov ac +nol in +no excuse +neuk ölln +mu et +mer yem +mem brance +master pie +ma under +ma gos +m kg +lv m +lubric ate +lu an +lock in +li ja +land graf +lalit pur +ki bana +kash gar +jerry rice +jede diah +india at +howit zers +host ing +home decoration +herbi vorous +happy baekhyunday +ham moud +hack tivist +gool sby +future ready +fin k +fam iiy +fall ston +fair ground +er langen +er ine +dle ys +dic kiev +dah lem +da shain +cu cin +cre sco +country side +cost es +conservation org +catech esis +carli vatiron +car mon +cag giano +c sas +bridle wood +bon giorno +blood donation +bla ha +bar thel +atlantic records +ath ur +artist life +ar leen +al ady +aggrav ates +acknowledg ments +abus iness +ðŁĺİ ðŁİī +ðŁĴĥðŁı¼ ðŁĴĥðŁı¼ +ðŁijij . +ðŁijı ðŁĴª +⼠ħï¸ı +ઠµ +war ble +ur va +un b +tom oh +to pal +title holder +thin d +the state +th ills +sw police +stu hl +st rob +sp reck +so ter +so bi +sick i +si skins +sand art +sadi k +rath skeller +rat li +promiscu ity +pay nes +or by +one hunga +odi hr +ocean us +obliter ating +no ct +nate diaz +n pv +mat tea +marien platz +maajid nawaz +longh urst +l tc +kra vi +kel lum +iz ak +ili fe +i bb +hun cho +hu by +holly conrad +hi ki +heng elo +hein kel +hap iness +hain ault +ha idt +ha gin +god da +gin er +gair show +g latt +flat rock +finish the +farmers weekly +extempor aneous +echo location +dema rest +daw ud +dar ragh +cul la +cru et +crack lings +cosplay girls +consol ations +cogn ate +cle eve +churra scaria +cap il +cal cin +byu cougars +bulldo zing +book keepers +bha wani +bemo repirate +bec ke +be harie +bar thele +atifas lam +asdfghjkl ñ +armb arnation +ar line +aphex twin +ambassac ats +am v +alpha phi +ak pabio +ais c +afir st +abe ba +ðŁĺľ ðŁĺĿ +èī ¯ +§ ðĿIJ +yellow vests +uhcougar mbk +tri j +tra duc +th ena +tex mex +teng ku +tele mann +tee jay +sweet breads +strade bianche +still gotit +state fairo +statefairo ftx +stabil iser +si rr +shout a +she aven +sch rank +s gallery +rush worth +repell ents +qu itters +prospe red +prefe c +pra dip +po zzi +pim pri +phy ton +photo therapy +pen ser +oni shi +obel isks +ntu sg +ni za +new school +music and +morri sh +mex ica +mer imbula +med sci +mayo tte +maru yama +ly se +livand maddie +lin z +le vering +laur sen +lance armstrong +lam ond +l nh +kun war +kate winslet +kane brown +jo cky +jag meet +intertwin ing +inci vil +iff ley +hyper thermia +houser ules +hor adio +hiji kata +geri halliwell +ga er +foxsport saz +eye bags +elk ington +ed miston +dete stable +d sps +cycle toworkday +ctm pofficial +cru mple +coo ool +consu m +conjun ctions +cle to +citizen sunited +cian jur +cho bits +cath arina +cake bread +c fia +bri ana +bow down +bor zoi +bink ley +bec tomy +beard more +bat in +astro plus +as pens +ant t +amo wry +air ra +af ten +aer oline +ab bath +__ ) +! ðŁĺ³ +âĸ¶ï¸ı âĸ¶ï¸ı +zap iro +worldnet daily +wno tweet +west village +weare here +we stre +un glamorous +trunk show +tr nava +the ju +ta wana +stap led +soul wax +so sv +sky cable +seem ly +schi pper +sch atz +say what +ru sd +ric cardo +retro game +re vy +pyro technic +ps ils +planet coaster +pl h +pesc et +peach bowl +partylikea journalist +pal icki +p no +ow news +nith yananda +newsin vids +neuro surgical +neur orad +nel o +nature za +nat su +nac cho +na hr +mun tz +mell ons +meh rang +med ary +mcdon al +man zana +limitedrun games +lie paja +leadingthe way +law i +kri eg +itie suk +ir gend +intercess or +in ba +holiday gifts +her ma +hather ley +gold fish +girl child +ge ier +gab ou +flo etry +fi yero +feliz finde +eluve itie +ef dp +ec aatstate +e ut +dy w +door bells +do tty +dise m +de francesco +de bash +david lyons +dar ma +dal meny +cy anin +cur ti +crazysexy cool +concer tina +coffee hour +clu be +chau dhary +bower swilkins +big ham +ber ri +be cos +bbc philharmonic +bar beau +azhar uddin +au vers +ar mored +antonyl ruben +ano vich +amin os +a ahs +; / +. = +ðŁij® âĢįâĻĢï¸ı +ðŁ¥ĩ ðŁıĨ +ðŁ¥ ¦ +å¥ Ī +ãĤ ¿ +âŀ « +âĬ ķ +ÌµÌ ¨ +yellow man +womenin aviation +what getsyou +wa yo +vill ano +ul rika +u oregon +the deol +tch r +tar dif +t sao +sub su +sty l +sta den +st fagans +sp az +showme the +sensation alist +sa akash +ri ky +reece mastin +recon figure +pro cida +post traumatic +par ol +pag ham +p bj +oxi meter +official cufc +ob by +now w +nordic a +night watchman +nic omaine +new project +mâ ché +music on +mol on +mi kal +me j +made well +ma dri +lati go +la tham +l illo +knight frank +kle ber +kha war +kas dan +kan turk +it tttt +infringe ments +ide ser +hero ic +head stand +hair band +gro bler +glad stone +games radar +gal angal +frederick son +for dair +fedex field +fa awards +exacer bates +et ag +equ ick +ed ream +e gra +dut cher +dah len +com alee +cli ven +clev enger +c sub +bun do +bug bear +boy sen +black hat +ben ett +bax endale +band ila +ban tering +bai xas +any ama +annivers ay +ang irls +and then +air fares +ag la +ad hy +ach al +aan p +ðŁĮĬðŁĮĬ ðŁĮĬðŁĮĬ +wul f +wo wee +weis berg +water head +wall an +w lad +vol tex +vliss ingen +valley cats +un cia +tucum cari +tor ay +thermo set +the mar +tegr ation +steal mygirl +spider woman +sil ience +sh rum +semi annual +sch utz +sbli ii +sati ety +saafni yat +run blog +runblog run +recy cler +re authorize +puli murugan +public ise +pu ella +pro sieben +pratt institute +pp ppp +play dead +phwo ar +pe ka +paradox ically +palas zczuk +pack ing +oy ston +ouis ville +o saa +noy noy +nay oung +mccul lagh +mahan ey +lu kis +lou brutus +loe wy +lodh ran +linke dua +lego league +ld m +kumb aya +k institute +just ink +jasmin ec +jahang irk +jahangirk tareen +jack russellterrier +j afridi +iz umo +iowa statefair +her bology +fun night +fuj ioka +fla yed +figh tin +ferlin ghetti +fcunited mcr +ell ina +ea sterling +don abate +distractingly sexy +cul ls +credit able +chav arria +chant el +centri sts +cav at +care tta +c gg +bu pre +bryan brothers +brick ley +bir s +bi mba +best nigh +ber l +bedn arik +bec kia +ba hahahaha +awo olf +att ara +at your +assassinscreed origins +anandi ben +aj w +af eni +ìĻ Ħë² +̵̨ ÌĦ +w st +vla do +vis cera +ven et +va stra +twitter stake +tu ckey +trigla v +thu cy +then et +thebachelor au +the ware +the see +tb snetwork +super critical +su y +sty mie +southe aster +simpl ys +shine bright +see v +seattle symphony +sean price +sam mo +salman rushdie +safe guarded +roun dups +roof less +rheum atism +retwee et +red berry +radi ation +prev ail +pr ss +ppor tunities +pick oftheday +par terre +nigerian creatives +nfl oncbs +nam ad +mo twani +mm ers +micro aggressions +mc keen +mad son +llan id +li ep +level up +le bih +laba dee +kit ale +kis sarmy +jeep family +interview mag +ic ahn +humayun saeed +ho stiles +hh v +hemorrho id +he tta +han dog +gam me +gallo per +fer ias +fam i +f assie +ephe drine +endthe fed +dra we +d sap +cr ine +cloud native +ck ickoff +chu o +cbc to +bridal shower +brick layers +bott lerock +bon it +blessedand grateful +bjor k +beour guest +be somebody +bau n +bar ge +bal lot +b dunkelman +atx festival +atech nology +anth es +andrze j +amo or +alan de +a equ +< -< +ðŁĺį ðŁĴĻ +æŃ ¦ +ãĤ³ãĤ¹ãĥĹ ãĥ¬ +âĿ ĥ +ଠ° +ॠģ +Ù Ĵ +xxx ii +wm phoenixopen +water marks +ver vain +tá naiste +tx instruments +trues dale +te thys +tai ki +supp lan +su q +sti ka +soun darya +sece ssionist +se dna +sar lacc +roer mond +qu intel +qayy im +pr inter +pole dance +pi bil +photo show +pedre gal +pav los +nt g +nexus mods +n int +musicis legend +mule shoe +ms news +moo sic +modafin il +mo fo +med center +mcgre evy +marzi apie +marcho frobots +majo relle +mae by +mack enna +log ico +lo bbing +lin thicum +lex ia +leu chars +kumb akon +kar ai +juni at +jib con +iwm duxford +israel underfire +in ol +ieee org +i fat +hygi ene +hu dgins +healthy kids +he witt +hand spun +ha qq +grant morrison +go ber +gif tedness +getre al +ge mo +fun fetti +fuji ko +fu ffle +foxnews facts +forsk olin +fish sticks +fi roz +engra ined +ec ss +e pix +dylan thomas +dun ner +d loading +d co +cute y +crested butte +ch f +cau field +cas ca +carboxy lic +canig gia +camili zers +cab allo +bi sham +beth ke +bb ctw +ba id +au techre +au ric +ash craft +ao b +andy stanley +am bers +alison moyet +ali ki +abir d +a few +-------- -- +!!! .... +ðŁĴĹ ðŁĺį +ðŁijį ðŁı¿ +ðŁıĥðŁıĥ ðŁıĥ +ÙĩÙĩ ÙĩÙĩ +your dreams +ya jam +women shoops +win rar +wein stock +walk around +wad d +w wildlife +voteen rique +vintage bikes +v wo +ti ot +the tanmay +terraz as +stre psils +stat too +stardew valley +sr ams +squ andering +spring town +sports ound +sommer ville +soci opaths +sm j +sla gle +sko da +sil ke +she ed +sad dict +riseu pred +reli ent +region of +r ham +pur nama +puertoricop ur +pu pi +prs journal +pe sch +particip ative +palmi otti +opto genetics +openstack summit +nor v +ninju tsu +mm fa +min ya +maim onides +ma grath +lim nology +libre ville +kix brooks +king andcountry +kin ole +k mi +jess amyn +jacob grimes +instaf ollow +indi aw +iaaf worlds +hy mer +hu du +heav es +havai anas +h rad +grand prix +good hart +ged dit +gary j +foreign affairs +fl ico +fil mer +fi ats +f mm +ey l +expl ora +england golf +electr oneum +el x +eh f +drunk history +drug mart +drivein mob +den een +def els +deck hand +d bel +cp sc +ck worth +chill ingly +chau vin +chasing life +cedar ssin +ca ahoops +bru gman +broad land +boat face +biopla stic +bim by +beau desert +bau x +barbar ity +bal dies +at cs +arte ducation +ardi ente +aper ry +ali ght +ac climate +a otw +ðŁĺŃ ðŁĻĮ +æĪ ¦ +zem ski +wyn newood +wil den +vel arde +uof sc +un savoury +un civil +un achievable +ty umen +transm ountain +title holders +tic os +thei hi +te tras +te sted +sunid hi +steve mcqueen +spring break +somersaul ts +shor tens +sho whome +shi awassee +scorpi os +scer vino +rowland schools +roth schil +roger sarena +rise against +rehman malik +registr ant +qad dafi +po cos +paren te +paci fism +p tn +om aki +ol un +nucleo tide +ns agov +ni mm +nhs grampian +nd h +murshi dabad +mr sam +mo dok +mentalhealth week +mat zke +mark dayton +margare th +mar kii +manag ements +mainbhichow kidar +ma ppy +long side +lips comb +lib bie +lanc ôme +la dainian +kirkcud bright +kilkenny clg +kas auli +kar ra +kalin ic +k hairy +juliab radbury +intercess ors +he che +hatsu koi +h go +god parent +go wes +football tips +fo yle +flower beds +fi ets +fal zon +eye ofthe +expres sen +ero des +erin burnett +dunkar oos +dun huang +deri sion +deare st +de keyser +cu ira +coo pers +cister ns +cho tt +chees y +che tu +cfb hall +breakthe silence +bra gh +bowl by +boat shed +black buck +bet abrand +bay ero +banyo les +atay lor +argent inos +andrewr annells +ad dres +ðŁıĢ ðŁĴ¯ +åĭķ çĶ» +ãĥ Ī +âĹ» ï¸ı +⬠ħï¸ı +zak ia +z up +yum mie +yugi oh +you and +wi gger +weing ut +w ceu +vri end +us ando +un disciplined +televangeli st +tch ad +tam bour +syl la +sum times +stur dier +stre eth +spo int +skin nier +saint seiya +rohr bach +ratli ffr +rame kin +ram pa +public ising +pre late +pr anitha +pp ance +power stroke +pi one +par aiba +pal ar +out fielders +ou can +ou ag +os aga +orang ish +oneand only +nys dec +ninja sexparty +ner is +nel la +nc gop +nationwide kids +n cu +multi ethnic +mu kuro +mon chengladbach +mil gram +may wood +maud lin +matte i +man asseh +magic mike +lud ger +ls don +lou x +ko ester +knap weed +kin dred +jas wal +inthe wild +inter no +inher its +inform atique +inf anta +ie business +ibelievein bookfairies +hok um +handicapp ers +ha id +gul ping +gra der +ging in +gautam rode +fun gu +fore achother +fle ener +eswat ini +em wangi +e step +dry lands +dream big +de bb +dd ddddd +cro kes +co vington +christop he +carl sen +caf s +bu toh +bou gh +be stia +be back +bar men +ballinam allard +ball an +baby bump +ay ake +avail s +atay de +andre wyang +anci ens +absolu teradio +abo lism +ðŁijĨ ðŁı¼ +âļ Ĵï¸ı +âĺħ # +zak ka +za po +youth work +why iteach +whis ker +wh ib +west malling +wave guide +va hid +uni veristy +un listed +turn buckle +tren diest +the joe +tend encias +te pic +t anc +sp aw +sop ran +solym pic +so ss +sl und +sky divers +sip tu +shun suke +shar ding +sep r +sen corygardner +se dang +sci on +saafniyat sahivikas +sa hn +ru dolph +rin i +reen actments +re consideration +pat era +paper making +pa wb +p gn +or molu +nac ra +n cua +montre ux +mo zo +mis sn +metat rader +meet in +me morex +me mang +man am +maksi mc +lt col +low ry +louis theroux +longhorn network +lisac im +line smen +lesley ann +lef se +kis si +kar ras +kai muki +k ago +ire ton +iam sam +i eva +i apolitics +how lite +hoo oooo +hemis fair +hay maker +hantsi wwildlife +hal den +ha sso +granti mahara +ge würz +gar r +gal us +front court +follo back +flo rey +flamboy ance +fedor ov +fau vism +e somar +dur yo +dove cot +diver ts +devi ating +dela field +dal eville +cur seof +county show +comb atives +clo yd +chula vista +chi oggia +cel er +cappiel ow +canel ones +bre sse +bc ss +aus def +au dry +ation alism +athe art +assemb lye +are r +alber obello +ahmad abad +ðŁ¤ ¶ +âĺłï¸ı âĺłï¸ı +zo x +yeah buddy +wa heed +unfor gett +to ga +tin kered +team shabira +stre atham +ssy fy +shuk ri +shar ratt +seat ac +scottish open +saras ota +sai ki +s ÃŃ +s records +ru mania +ren unciation +pru d +pen land +pc engine +partici ple +out let +new ish +marcuslu ttrell +maker oom +macin nis +m ór +lux watch +luke mitchell +lt l +lock down +len zerheide +leaveno trace +lach hab +kri shi +korean air +knock out +khalee j +kab ira +k atti +jun ked +jeril ryan +jar lath +its ramimalek +har ms +greeng rocers +greatplaces only +gra inger +go ehring +gam esof +fluor ome +elec trum +ei ps +egom aniac +dying matters +dug outs +du se +du sable +di ox +de pose +dar ao +crore pati +concor dance +compra r +com passionately +co zad +chukku vellam +cer amide +cas sio +c ds +bro cks +brani ff +bour dais +blu hm +black en +bell woods +bell mare +battlea xe +bag sof +ath enians +astro tur +ast ilbe +arec a +aqu otes +abhin av +ðŁĮŀ # +ìĻĦë² ½ +é ī +youve got +x ue +wing rove +wil des +wal ston +w ths +vide oo +u od +tsun ami +transfer ase +trans dermal +thut mose +ther oes +tee spring +sul pice +su ica +sto day +sor ge +shan kman +resi ded +r fc +prat c +pontar dawe +planet labs +pentat onic +pen tath +par ola +paper art +pan handlers +outh waite +northumb ri +no bama +ne burg +mymt brain +multi plex +mor oka +min ia +mex ia +me theridge +masse ur +man tap +mad ley +love fest +light ner +lead belly +lc s +keh na +jö rg +itunes festival +inge urope +in red +ili ya +i strian +hu ard +hack saw +green economy +goo oooooo +gom me +fun and +fsg books +franci stown +fou lds +formu lad +elast omer +dr phil +de agles +cathe dra +cat mull +carval hal +bv barmy +bur gan +brain y +boothe el +bo cuse +bmc proteam +asi ya +arti kel +annou ce +an be +ac ase +? âłĢ +ðŁĵ ģ +ãģĦ ãģĦ +âĢĭ @ +н ов +Ì ģ +à · +yu ke +yakut sk +wur z +whatgetsyou outdoors +vali dator +under performance +tusk ers +treasure rs +together werise +thor ley +then at +th l +tamil nad +tal lison +ta affe +stie fel +ste ffi +speci alized +snapp er +sic amous +shoo kt +shari bu +sh moo +safi ya +rumin ating +rosie hw +reimbur sements +r news +r le +plant agenet +pizz arelli +pipe fish +per m +pav lo +pang arap +p enty +nowor never +nin iola +niche escapes +ni mr +new sw +neo sporin +ne wry +ne co +natural ness +morein common +moni fi +miley formmva +marche shour +mar vins +madilyn bailey +laure ano +lag wagon +l pb +ko ha +kassi ede +kan ade +k cm +ju la +j hump +international tigerday +iceland foods +human factors +hugh enden +hri day +hippoly te +hin ks +hel ene +gon line +geton board +george son +gay dar +g audi +fright night +ex ter +em z +ecur ity +dro sera +do tr +digital illustration +descen sion +deep veer +crickla de +con garee +collage art +clemson univ +change slives +centen ary +catastro phes +brac keto +bi gu +bar bad +an el +ai goo +acl festival +ðŁĺĤðŁ¤£ ðŁĺĤðŁ¤£ +âĿĦ âĿĦ +âľ ĺ +zar dly +word mark +wo ori +wight link +we care +way police +wakaflock a +upnorth live +un duly +tu thill +tri stana +tes ke +temu co +suffo cates +srilan kan +spor tac +si mul +si damo +red flag +re marking +pump in +pu issance +psychop athology +pro tos +ph h +peter ock +passage ways +participat ory +pan tano +ob on +o lot +o gether +non u +no hep +ner ney +myeong dong +my haver +mountain west +min nick +mil ow +mee totaku +md lz +manicure monday +man tr +mag or +ma dad +ll ant +len inist +lan gham +kom u +killthe bill +katy perry +jet fire +jad av +ire t +iff co +hor ic +hel ston +glass jaw +gewürz traminer +gar gi +g te +fe a +favorite things +fang asm +f wr +elk grove +elic ited +ehlersdan lossyndrome +e ade +dy fed +conco cting +clay face +chronic led +chennai yin +char coal +book recommendations +bish noi +billing shurst +bene dum +bello whead +beck oned +ban ka +bal ancer +ba ju +ayyy e +av ill +aug ments +asi atique +am mar +adopta shelter +a ines +________ ____ +ðŁijĩðŁijĩðŁijĩðŁijĩ ðŁijĩðŁijĩðŁijĩðŁijĩ +ðŁİīðŁİ ĵ +ðŁİĤ ðŁį° +ت ÙĨ +wom ent +wine week +whitt led +wal ay +ve ssel +ve iw +val lur +un did +ule scu +trun cation +tortu guero +thermo forming +tat ties +take part +tac itus +sus annah +superstar dom +stor z +ste ggles +standardi zing +st com +srikak ulam +soko lov +sli abh +shin sky +scri abin +schae ffler +salud tues +s dut +s ational +roh rabacher +ro zon +ritch son +relax ant +penet angui +peat free +peach ey +par sa +palimp sest +page boy +outri der +old castle +oil fields +nw m +nh h +ne we +nal in +n swe +my ung +mot lanthe +mor ley +missi bility +mini bike +milin kovic +metabol ic +mel zer +manga art +mac queen +m schat +lu ster +live it +li ket +leh tonen +l tw +ko lej +kk kon +ked die +jo kinen +it ep +irish food +il minster +iklan bandel +i mid +hom unculus +hin kie +h pb +glen roy +gir li +game keepers +g itation +fo scar +felly chibi +duke mbb +du err +doublec lick +docu drama +do ko +death fest +de positions +de activation +dab bler +cp bl +cover taffairs +corri do +complex mag +cleanair day +cas kets +c cleaner +bushwack ers +bolo gn +boiler football +bo rea +blunder buss +blooddonor day +bi bury +bhavi or +bb onday +barn ham +barking side +ba stin +at eliers +an ata +am bis +âĺ¢ ï¸ı +à¶ Ń +س ÙĨ +Î » +wr angell +wmn news +waterlo oroad +war iner +un american +u jung +u baldo +tro ver +transc ranial +tran sunion +tor on +to history +the quint +the fire +tearitup bts +ta vy +t ldr +sta ed +sig ny +shin nie +secondam endment +se ssa +sch lad +sav iler +sav alas +sacher torte +sac onf +s fan +run rocknroll +ru dis +rit chi +rep brian +re vine +publi k +ple be +pi est +pa ide +original music +oliv arez +og ba +o ases +nun thorpe +nincomp oop +murali tharan +mu ahaha +milk wood +mic rob +mc kidd +mc clair +mad awas +ly ford +ludo vic +lec tronic +la ppe +kno l +kim mi +killy begs +keving ates +kam ran +kaha pon +ji moh +james ra +inst ants +imper io +illy ria +i vette +hook land +home biz +hei den +hans raj +han ish +guerre iro +gary player +fox e +fl ach +ferra gam +felic ita +fas i +ex other +epi dermal +duc kies +dragon fire +din iz +delaha ye +david tutera +d illy +cu pido +coupon code +cou loir +clau dy +chi a +cdn muni +caste ism +bur ano +bonifac emwangi +bb tag +bar se +b hola +av ait +autau ga +au pt +apur va +ane choic +an sip +al chemists +adul ted +< : +ðŁĺİ ðŁĺı +æ¥ ½ +à¸ŃภĶ +Î ² +zel ina +zaven tem +yar r +wil banks +why tele +wapis kat +vs ner +ven eered +vel achery +v ul +usc g +u vs +tw ich +traut man +tran sept +ti meee +ti deswell +te knik +t mobi +super ia +stone walling +stig ler +ste iff +star field +stam baugh +spar red +spac et +sp ack +sou fri +sof l +sing lish +shi mane +sheryl sandberg +shar mil +shadow land +sha hani +roo tes +resonance fm +power sports +pern ille +paramilit aries +oc tor +o ger +nullar bor +nu groho +nor val +no ton +never where +n cra +mu zzled +mous y +mal um +ly se +loubout ins +light water +kentuc kian +kapp kvew +jake canuso +jah ren +is overparty +indi anidol +imagin ation +ic cc +i hi +hat sune +hasan minhaj +ha xe +gou d +gat chalian +fox wood +fight like +excep tional +eti os +en demol +cut work +cogn itively +clo ven +cine ws +christ ofer +chic est +chan ute +cb live +can ley +by bee +bun che +blu shed +bla si +bille vans +bi erman +beyond borders +beren ger +bar ad +back firing +audi rs +asur ya +as mussen +anastaci afan +an jum +aly goni +alexis dejoria +ìĩ ¼ +âı ² +اÙĦ ÙĪ +Ë Ļ +za popan +yugosla vian +wol i +whatwomen want +wei de +we hi +var ney +use ums +ure port +universit at +tu en +tu bo +trous dale +trans genders +town sfolk +there venant +thene we +the kings +the dog +ta illon +ta ff +swee eeet +sunny d +su gg +spu bli +sport scar +sp indler +snor kels +see scandies +scu mper +sc ult +river ford +ret tig +real bobby +re eth +pri yas +pr newswire +pl sd +paul vandyk +paestu m +nois ily +ni ve +natali ec +nar ia +mondaynight football +meso america +mcfar lin +man down +ma ari +lu sby +lu key +lochal sh +lo keren +leve que +la il +kron ik +krat er +king sheath +kil beggan +khe ir +katar zyna +jag an +ini quities +iglesianicri sto +house boats +hitt ite +hin ny +hen ninger +he men +hc so +ham ley +grimac ing +giall orossi +g mac +fur qan +fre dro +fl ds +fin eness +fear th +fail over +fa ile +eth nom +er sten +entin el +eng ale +en ak +edmund ston +edmun dmc +east brook +dj k +disco ve +dev fest +deli ve +cyan ins +cro kinole +cookier un +conco ct +comman deer +co fo +cl ines +chrisl illey +chaun cy +big al +bhu pinder +bc f +bar rela +app all +anton ello +an us +ala ine +al gor +ag ena +ad du +ðŁĺī ! +ðŁı · +ðŁĩ«ðŁĩ ¯ +าภ¡ +you togive +y ine +wer ker +voteenrique fpp +ven de +vel lir +uon bi +u mut +tragic ally +tho re +thisplace matters +the dukeof +tdamerit rade +tanz ani +tan credi +syste mically +syndic ate +surrep titiously +supp l +stone m +ssi an +spitt sburgh +so al +si ame +ser am +sco vel +s made +ru pe +rt dna +rope way +ro gie +river cruise +repos ado +re blogged +raffa ello +poly clinic +pickle back +open democracy +oldro yd +ofor igin +nor berto +ni mi +neu wirth +net working +na ac +moon lite +moog musicinc +micron auts +mc gowan +may en +mat chett +margare torr +mar te +magne sia +liquid ator +likeli ke +lady bird +la fer +korn field +ki ger +ka ay +ka abi +k ount +instaf rame +indoctrin ating +ilaiyar aaja +ideolo gue +i star +hel eng +hallo f +gwen pool +gonebut neverforgotten +go bu +gly pto +fit zy +fil omena +fe vered +escul tura +duplic itous +dramatur gy +drag strip +dit zy +dev ine +den nie +demo te +defen sa +davis ville +cre use +conden ast +ck ert +city break +ciel ito +chi leans +caterin atweets +cas ady +car ai +bun du +boot legged +back thursday +axi ata +aw adh +auto immune +as af +anton ino +ah ti +adoptdont buy +ac tes +absin th +/ * +ðŁĺĦ @ +ðŁĺĤðŁĺĤðŁĺĤ " +ðŁIJ Ĩ +íĥľ ìĸij +ಠľ +à¤ľ न +ydr com +xx xiii +x wing +whit en +we ssmith +vox els +vote eff +victi misation +van ney +uper man +un truth +tu si +towel day +tom ey +to bia +timeout newyork +ti bbles +thr ought +thebig show +the point +than son +tell ico +tar onegerton +stand asone +ss ahluwali +ssahluwali amp +sports guy +si skel +shrie ks +semrush chat +semin chin +scoo ks +ro len +requ in +rapid kl +rand fish +quad ra +personalized learning +pear ce +pa ji +ott oman +ot ara +omer ase +oli ka +oko th +ob y +nisargad atta +ni est +nanta hala +nag ul +myel itis +my til +mur f +mor rilton +mis smo +mg sv +mg mt +mcel wee +mcdonald suk +may bach +matti seman +man souri +loui seminchin +london fashionwk +lef tie +le chner +kw qc +ko ho +kai lee +jor is +jo dies +jel inek +ist ure +inno cen +in nnn +impro bab +hyper visor +htown rush +how you +hoop shabit +hitman hatton +h fi +f gl +educ ación +earth watch +dr mark +cy games +coun tess +cospla yer +co if +ch ough +c tweets +buen viernes +bryan habana +bar ao +b go +ax tell +andro scoggin +am ud +a kee +ðŁĵ ¬ +Ú ij +zz ese +yaku tia +würz burg +whoo ped +v mx +un shaken +ume ed +tubeli ght +tre mper +the monkees +sweet briar +svend sen +suyyash rai +ste ilac +steilac oom +stay ton +slu mp +simple things +simon rim +showoff bydesign +sho shan +sea houses +scott brown +scari fication +sc udder +sargas so +s gameday +ru bias +reli ved +pupu sas +possi ble +pliss ken +pedo bear +pas sy +our girl +ok tober +ok cps +nit schke +neonicotin oids +mo hom +mira bilis +mat lock +mario goetze +let d +la dles +ks fa +kefla vik +just is +jon jo +jason segel +it ro +ir regulars +io va +ins angu +im pe +hu lett +host elling +hoo ter +hof stadter +gro b +globe debate +gent ly +g anger +fo sco +esche w +elm endorf +eco logical +dl na +dil ate +desp ina +de constructs +dar denne +cover crop +cor ine +comp ilers +colo s +co habiting +clam bering +cin da +christoph ers +cal away +burn sy +buech ner +bu in +broy hill +bro oo +bom ani +blacke yed +beech mont +be sse +ba ena +at mega +ash kelon +as j +an ual +a ati +ðŁĶ¸ ðŁĶ¸ +ðŁĴķ " +ðŁĩ§ðŁĩ ¾ +ðŁ¤ŀ ðŁı¼ +yi wu +wil sey +what up +warrior games +wa al +w vt +thereal dcf +the process +su see +spe ight +spar tina +sick nesses +schar pling +sar ab +root sof +regur gitation +read aloud +pun ti +pro kop +pres byo +polar ising +po kiri +pli mpton +plas mas +pla i +phon ological +penetangui shene +pashupatin ath +organic gardening +omon sters +oh ba +nimb in +nak agawa +mish al +me dy +marcou x +man sky +mal iciously +mal dita +make theroad +mac found +lon min +lipsy nc +le toya +kun ze +ku us +ker an +jack black +ja sen +io dide +ing cancer +ind sey +hydro foil +hoyas axa +hottie oftheweek +hom inem +hollywoodun dead +hawkes bay +har ner +h gf +gr itters +ger t +fp gas +foo dy +fan cams +exeter cathedral +evangel ic +euro hockey +en ve +elou ise +dul fer +du kie +dis connect +det ente +dele k +defe ctions +come shome +col ons +chatto padhyay +beyond blue +bec q +baby center +ay as +aspir ing +ari stop +apollin aire +ðŁĴŀðŁĴŀ ðŁĴŀðŁĴŀ +ðŁİīðŁİĬ ðŁİīðŁİĬ +ðŁİĢ ðŁĴķ +âĸ¶ âĸ¶âĸ¶ +ï c +yorks biz +y aiba +where thecar +wherethecar sare +villa ggio +u iz +tin ian +thru pp +thr ou +the fan +tar kenton +street photographer +stin c +ste iger +son news +soc med +scy thian +sat in +rusty wallace +rp ms +rox eter +resul tado +quo gue +qu ne +q af +pulver ized +poloni ex +part in +pa wer +on vif +on k +not ch +nj ie +new tek +n pratc +my yyy +muham madi +men ke +mar na +manipu lators +mag at +love is +lc ss +lau ding +lanvin official +kab at +jy vä +just blaze +jo bar +je sters +jan ai +inthe us +inst ax +i ben +hil le +he cking +hapha zardly +gay ton +gam bar +far th +fad al +eric holder +duck weed +dru silla +do komi +deci r +d gh +ct fu +critic ality +ci ber +ch ando +cd q +care homes +car others +c icc +basti at +autorick shaw +at olls +and ymurray +am ang +al mera +al dea +ðŁĺī ðŁĺģ +w bi +var itek +up ham +tv sn +turn age +to iled +thorn ham +the open +the agenda +tan lines +superse des +su cr +staf fing +spe di +spac ial +south westerly +smooth jazz +sharethe lex +senior itis +sb vb +sand ero +ring send +recap turing +re xit +re ids +quie ted +pushawardsteam kisses +pu entes +process ing +por ttal +pop crush +pir u +peter ptur +per illo +pat summitt +ous music +or ani +new line +mordi alloc +mo hi +mo barak +mit ron +min k +merri mac +mercury theatre +memorab ili +mar la +mac n +lm x +llu ll +lis ap +lan olin +lac our +l fd +kuns thistor +ko kesh +kate ys +jon no +jeric ho +janegoodall inst +jal on +jae suk +ic entre +hus sein +hul lah +harsh ini +happy day +hai ro +h nl +gen co +g apping +friends notfood +er len +efe cto +do wels +dit mars +despo tism +dab rowski +clair ton +cel g +car on +car hart +cal ums +bu ang +brum field +brit birdlovers +bristol baby +bbc womanshour +barbas ol +bag man +back bencher +as uk +anti dotes +ann at +anarch y +anandiben patel +am ola +agh olor +acro wn +!! ( +! ¡ +ðŁĻĪ # +âĿĦï¸ıâĿĦï¸ı âĿĦï¸ıâĿĦï¸ı +âĺĢâĺĢ âĺĢ +zoni bali +white tails +vincen zonibali +vas sy +upadhy aya +tu ya +tre dge +trave le +to dor +time z +technology news +sustainable tourism +surfact ants +sun it +strang les +shor tridge +shelar ashish +she a +sf ile +say no +sair lines +rudhram adevi +rob schneider +ri ffin +reyk jav +resusc itated +ra heel +publici zing +pin aka +peter s +obstin ate +nt fs +mix ing +mcgon igle +mc girt +mad ball +lydi ate +loy e +low der +lex y +len o +lauren goodger +kah lon +k nd +jum illa +ju mu +jaye gi +jamie whincup +inten tional +ima x +icom edy +hondac lassic +hh b +haus man +gw and +gun gor +ger mline +gaz idis +gal ad +fr st +for fun +fit girls +fish hook +exped iting +ev t +eur jpy +eryn gium +enough said +dre ll +dispro ves +ctu local +conf rence +co pra +cath ao +car olo +cann ington +c agu +break beats +brah mas +bowdoin college +bor onia +bo boli +ber nas +baw se +bate mans +bas otho +barcel one +bailey m +an andi +alb ina +affe y +ac ares +zi zou +y stem +wood cutter +william byron +west fjords +wal b +wainsco ting +ver dure +vac as +tony dungy +toly atti +toku shima +thermopy lae +tambu wal +sushmit adev +sushmitadev mp +sto hl +stag es +sick bed +shri vastava +shakespe arian +se sion +school bus +sa ille +ru disha +remedi ate +re traces +re appearing +rally australia +pu recork +poke mont +po thos +play girl +pigeon hole +out num +oli gon +ol itz +no vis +nit z +ni f +myo b +mpl s +mech warrior +mccar ver +marypopp ins +mana hawkin +ley house +leve rett +kuch h +ker ber +k mr +jo wett +jeff vandermeer +jare tt +jack sock +iter ating +inf x +inci dent +imbu e +huach uca +ho ta +he fe +google fiber +glen burn +gets old +gabri ella +fresno bee +fra se +fire department +fahri ye +f bw +exten ded +est á +er ay +e ser +duv vad +drais lv +do dig +dev co +de ak +dam pens +dam ia +cryp tids +cra dley +cham bal +ce dis +carou sel +cab g +ben hur +be ika +attrac tant +as police +are alestate +apple store +anastaciafan ily +an shuman +an op +al cide +ðŁĴª âĿ¤ï¸ı +y thing +ww fcofficial +why so +wapp reciationday +wag gin +veronic amerrell +veg fest +us apro +under story +u ha +trenton thunder +traf ine +ti med +thir roul +theatre r +teachers matter +tar boro +sym metra +sylla bic +sylla bi +swin ner +sw afford +suk hi +sper anza +snow patrol +sheridan smith +sak aguchi +s suk +s birthday +rigu eur +regg a +reach out +re issuing +psych today +ps bhumi +play boys +pirates fb +pe vsner +pav itra +parth asar +orac les +ny m +no cent +narcole ptic +name sakes +mon ken +mol vi +meur ice +massey hall +mary ville +mani festive +ly lm +le sufi +lal wani +kassiede paiva +jyo ts +jul lie +ji raiya +j ato +insol ence +imitation game +i xi +houston ian +ho ja +hick ling +hash d +harrison ville +har gis +h sh +h frs +gil bane +friend zoned +fam ke +esguer rat +equi pedefrance +el neny +e boue +disney store +di ero +denver broncos +deca inc +dat du +cu bby +coo kislands +car ri +capp elli +bro in +brewery ommegang +br acy +bor gat +book marked +boi vin +bo tv +bo hu +band itos +back fill +am erie +ablu tion +abdic ated +aar mstrong +. // +' * +ðŁĵ ĵ +ðŁĵ ¯ +ðĿĻ ¤ +Ñģп оÑĢÑĤ +z brush +ye are +wur tz +win ry +weekend warrior +we missyou +viñ ales +vent ress +vait la +un bundling +un boun +thu an +tho w +the leftovers +the bath +th ope +th ain +texas childrens +ten sing +tb one +tages spiegel +stre k +spi rou +sp atz +soooo on +sen whitehouse +semp iter +sch amps +sales woman +rober talai +revdr barber +radio humberside +ra himi +pu pusa +pro mazda +pied montese +pay oneer +pal mb +ouag adou +or dic +obam as +nor um +nor mmacdonald +nationaltree week +narrati ve +murphys boro +mulat to +min day +librarian ship +len na +leish mani +le mus +lauri ston +lan ta +kork maz +kim yoojung +kha chanov +keesh ond +jan ki +j po +in nu +ilo gy +hun ni +ho stile +har ling +giri raj +gior ni +gener is +gel nails +fr ronconi +fore shadows +first love +fair tex +fa er +dre ds +disappro ved +culver house +cillian murphy +ch x +cf trust +carpath ia +call anan +bsn l +blu ecar +bis wa +benic assim +bath ong +bal bo +aw alla +app ea +an kers +accom pli +ac ali +a art +________ ______ +. ðŁĴĭ +. âĿ¤ï¸ı +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +ðŁĺĶ . +ãĤ¤ãĥ ī +âĿ¤ ! +worldre sources +wood cote +winter burn +wild hearts +wh attt +wan ka +vacation er +us marinecorps +un ac +u alr +toy drive +tom mies +thisisla fferty +teter boro +tali aferro +susanc alman +stor yo +steel city +ss di +solu te +sm ount +sig al +se ssler +se pak +rou le +recru te +re awaken +ran ald +ram nath +q ra +prie ster +phil vassar +pender yn +parame shwara +par ram +p ku +national ballet +muld row +mor bi +miniature art +mat amata +man ute +malak and +makin de +lucifer ian +ld d +kun du +kil led +jove tic +jack sboro +j é +iro ha +inven tiveness +inter school +ichi bi +ic helle +i ara +hex agon +hemlock grove +grand hyatt +get creative +fur uya +fre on +fli ppa +finsbury park +fangasm spn +evil doers +eu v +ebolare sponse +dj ay +depar tement +delas oul +dar low +cu boid +cristin avee +cen kuy +bri el +bou ma +bo sn +ban tu +bal og +an vils +allevi ated +addic tedto +absurd ities +ab ida +ðŁĺī âĿ¤ +ðŁĵ° | +ìĻĦë²½ íķľ +ê°ķ ìĬ¹ìľ¤ +zi ther +x seed +x imab +wwe uk +wit chy +win eland +wake hurst +und ine +try it +transforming lives +the visualart +t lim +stock pot +sony six +somerset levels +skysports boxing +shri vel +ser dar +sampal oc +s deep +rwc md +rudi ments +rosehill gardens +respon s +repp aul +re mp +re marriage +ram asamy +qui vira +propor tionally +pom pton +pil ote +op aline +objec tify +ny k +ni hari +nett le +nam er +nafp lio +murry sville +mul vi +mon tara +moanal ua +michelin tyres +mic hale +mi rella +metta worldpeace +mc cc +mb ury +matsu ura +mate i +maic ha +loy ola +limkok wing +len n +la quan +l senews +l bm +kro mer +kpor zee +karolin ska +jim norton +je ffe +hepha estus +gu ate +green law +gre er +g anc +fur o +foam posites +fil mes +fh su +fat in +far ge +fa shi +f km +esguerrat ommy +equ alizing +ear les +discord ant +crou cher +cook sey +con quests +commensur ate +cole engarcia +col son +charlo tt +burgun dy +broad field +bre th +bray shaw +bli ssett +bha sh +ben lovejoy +bb nai +barn sdall +atar axia +as oftball +aq w +antho cyanins +ðŁĴħ ðŁı¼ +ðŁıĪ âĿ¤ï¸ı +æ© ĭ +y uni +y il +wolf hall +whow ould +water proofed +visit ations +ver dant +universityof ky +un enthusiastic +ty b +trilli um +traverse city +travel tech +ton o +the score +tar da +sth elife +steve kubota +ste wa +sm older +simon helberg +sil sila +shor tie +seattle pd +salis bur +ri ho +rha bdom +renov ator +rene a +reggie watts +real bob +pu zo +produ c +power ups +ore k +on elxn +off ootball +o al +new track +ne te +naz em +music brainz +mun ira +mu tha +movie podsquad +moto guzzi +moon day +monte casino +mo thi +mindless bhavior +medi ates +manse hra +man sarovar +man ce +lydi ard +live well +lincoln wood +less a +les bury +lee z +led low +le evalley +laver ton +kle m +kaw ana +jan ela +jae hn +instap rnts +indianc inema +indian art +in in +ici zed +hosse in +histam ines +help dogs +hant s +hahahah hahaha +ha dee +green team +green slade +gno stic +gc titans +gaming life +ga es +far yab +f grfc +elç insangu +el off +eat my +dy ker +dou we +democrat shate +daver ayner +daverayner fund +danger ou +contra ven +clu bby +cityo fl +che tbaker +cb n +book binder +black magic +bbc newsline +bad ulla +b ici +av la +aubu sson +atl v +ashi m +ash mont +apu estas +appell ate +apalach ee +am ade +ag morethanever +abduc tors +^____ ^ ++ ... +ðŁĺ¡ðŁĺ¡ ðŁĺ¡ðŁĺ¡ +ðŁĸ į +ðŁĴĥ # +ðŁ¤Ĺ ðŁĺį +ìĸ´ ëĶĶìĹIJ +ãĤ»ãĥ¼ãĥ© ãĥ¼ãĥł +ØŃ ÙĦ +× ķ× +zhu kov +yerush alay +waz iri +ver such +tü bingen +twitter artexhibit +turtle dove +true bloodh +truebloodh bo +tri sha +thereal jrsmith +thau low +th ato +sydney airport +sp azz +societe generale +sling erland +selec tronics +se ige +san marino +run withthe +ros sett +ro dolph +ric an +respir ator +re probate +ra issa +r pk +qua hog +qu ite +pub chem +pr annoy +ponty clun +per tain +pepp as +pe ons +paw lak +ox shott +ou trunning +ol ver +ny asa +new burn +nag ur +muck ross +mor rendo +min et +mikul ski +meso american +melbourne vixens +medce zir +mar ilou +ma kk +leve tt +l gas +kyo ani +kun in +ko zik +kingston ian +ki bbe +jur upa +ii e +if not +i wo +hur acán +hor ne +home ground +hedge fund +gg d +genna io +gameof th +full set +forsy th +fo pp +fic ou +dre ddy +de conge +dance mom +dam pier +dad dio +cubam in +cr inoid +cityof hope +certi fiable +cenkuy gur +castell defels +casey veggies +cas sata +bupre nor +bru ces +bee bo +bay way +azzur ra +avi dly +amust fall +am cs +al uk +advo cat +adam antium +aby ad +ðŁĴ¥ ðŁĶ« +ðŁIJ¯ ðŁıĢ +ðŁ¥ĩ ðŁ¥Ī +ÙĪÙĬ ت +year so +we dem +w tw +vi dhan +us ick +unemploy able +un fail +ultr at +uconn nation +truth iness +tri maran +tr icon +sylac auga +swain wright +suman th +ston em +stereo typically +slam dunk +si yab +shoe making +shekhar kapur +screw vala +sau v +sar os +sal pointe +saffron walden +saar brücken +rosar ito +rho donite +q z +presiden cial +pre empt +por ritt +phen olic +over indulgence +oren da +onye ka +only one +never again +mutil ate +mox ey +moon shine +ming in +meh sud +mc coll +mari otti +mali ks +logi o +lo sf +life ison +length ened +l sx +knau ss +karak achat +k wal +k enda +joann ak +japan trip +inter laced +innocent drinks +harpur hey +han ko +gwan gh +gram mes +gener alize +gan jam +fran ka +fon da +fau gheen +f one +f cau +etu i +el rich +druck mann +dhru v +dav ao +dance team +danc elife +d poy +copper heads +chit arra +buff a +bronz ino +beat itude +baz a +ball gown +as tha +ann g +angel amer +am po +all anguage +(âī§ âĸ½âī¦) +ðŁĻĪ âĿ¤ +ðŁIJ¾ ðŁĴķ +ðŁ¤ ´ +ê¹ĢíĺĦ ì¤ij +âľ İ +мÑĥ з +zay nab +yassi r +wave form +wa hili +verbo ten +ve ery +ur anger +um ji +u stour +tur riff +trek kies +tibur ones +tal las +syri acivil +syno p +su vi +skin z +she rer +sf pride +sa wal +run dell +renault sport +randomhouse kids +quater mass +po liquin +pink print +pe ch +panch ito +os agie +ole ta +ol les +nup ur +north wick +no logies +ni dhi +nathan carter +nag ach +mortalkombat x +min nie +me vani +maki shima +lorett alynch +london bookfair +loc ked +livel iest +len a +knoll wood +kal ert +kal en +k ach +ju ss +joker to +jo sc +int ent +imacelebrit yau +home star +hip life +hier ophant +grist mill +green acre +good smile +go yette +gee khour +ge fil +fath ima +fairmon thotels +fa ar +ev onne +ero v +ener d +donnell an +dissolvethe union +desi rability +deep water +cubamin rex +crusad ers +county pd +cor red +congru ence +confis cates +comicboo khour +choco holic +children sphila +char man +changing places +cap leton +camp life +call inan +bunny ranch +bro mbor +bou chet +boom ed +bom mel +autonom ou +as ra +anton ina +alex fromtarget +. ðŁĴ¯ +ðŁĴĥðŁı» ðŁĴĥðŁı» +ðŁİĻ @ +ðŁĩ³ðŁĩ µ +ðĿĺ °ðĿĺ +ìĹĨ ìĿĦ +ห à¸į +ye tti +ye aaaa +yarra valley +wor le +vel ox +usav mex +twof old +true islam +thewine society +the strokes +stress less +slu twalk +skyl ä +ski ve +seraf ini +rice gum +refrac tometer +red light +pra geru +positive psychology +po bre +piri formis +pink day +pet worth +pac zki +nut meg +nur singh +nsw fires +neglec tful +natalie ben +n gen +med students +mc nicholas +mar ske +man gus +ku news +kou ki +kh r +kedle ston +karolinska inst +ju icio +jimene z +icy cling +hhhh hhhhhhh +hernan dez +gom ustangs +gol fing +gla k +gabbi adini +fu rence +flugel horn +fang ed +face thenation +españ ola +epi k +eca illat +dream hack +divisi e +dis loyalty +detroit lions +del mundo +de ine +daniel sahyounie +copy writers +comeu ppance +colbi ecaillat +col wyn +citi c +christ ingle +chen ault +chau rasia +chand ran +cannabis cup +caldic ott +blu elive +block aded +berry ville +ban kai +arri vat +ang k +alma da +ðŁĹ ¡ï¸ı +ðŁ¤ŀ ðŁı» +îIJ ij +ìķĦìĿ´ ìľł +ಠł +zaz ie +yir uma +womens institute +willi mantic +weak ley +we ser +vi goda +vern or +un dee +ugly dolls +u rad +u dot +town speople +tic at +thor st +thisis robthomas +there ef +t ck +t ague +stream fakel +ski ddle +silicon valley +si os +shin se +sc aife +sas sen +san vers +san sar +salati ga +ren at +refin ed +re ffing +re considers +rain man +po quo +po ley +pierce brosnan +pe adar +pay al +partiti oned +panther sihc +o kun +ny xl +new adult +neg ar +nat araj +murad ali +monoc acy +mo sphere +mel bur +mar gie +mand ra +m jordan +li diab +la prairie +kou rou +kan go +jaf far +j bl +il ri +il itch +hot list +hollywoo dimprov +hobo ken +heaven onearth +he eney +happy dance +h fe +gra ying +glo ball +gau ahar +fram ed +fire trap +far r +fanta sticks +fant in +fanarmy faceoff +fac cio +extro verted +em ption +elast ica +east wood +do ku +david cicilline +da a +current mood +cur lews +cross bill +conver gys +color block +cl ande +chi rac +catat onic +blind folds +bishop briggs +bis now +bertel smann +bas ma +bas f +b lear +amey aw +' ..... +ìĸ´ëĶĶìĹIJ ëıĦ +ê tre +yar der +weare nigeriancreatives +watch roh +wal ser +vibr anium +ungovern able +transparen cies +tit p +there se +te ela +tam ako +ta fari +sy st +stai the +sol man +she iks +shadow bringers +sb ca +say d +sav ina +sar ch +san marcos +san day +robri ggle +ridge dale +redbul luk +real z +rc z +radiom irchi +punch drunk +pro circuit +pri ddis +pretty man +pre me +power outage +paul sboro +pan tin +olivi as +obe id +neil n +naj era +mu jh +monarch sway +mizzou football +mir choff +milli metre +men elik +mc daag +max y +ma ppin +ma aan +lu ffa +long stre +linde mans +lik as +learn with +lan dic +kom odo +kim ya +jer ram +ja q +its better +intermitt ent +interlo chen +in ox +hor nell +hoo ten +hallo weekend +gen ious +gad di +ga hd +g ft +fun com +ful ls +feder line +ense mble +elli es +dwar fing +dr at +down played +dji phantom +dis lodged +dhive hi +dary ll +d marc +cow li +co sum +christinam ilian +cho ong +catalo ged +busines splan +bru mmer +bridge point +bor us +ber mejo +bel gica +angio genesis +amre zy +amp oule +am dra +albino kid +albari ño +al fieri +ad akar +abre wer +aaaa aaah +ðŁij£ ðŁij£ +ðĿIJ Ħ +ãģ¾ ãģĻ +zap atista +wu du +wit mer +wildlife bcn +wes sun +voy aging +v arez +ur anga +turquo ise +tr bam +tom chaplin +th appa +sw apan +stra tham +star music +soul calibur +silvers mithing +side of +sg ill +schem bri +s ative +s alla +russia invade +run up +ru ang +rob sten +ro so +recl ines +re os +rat emy +pro core +phoenix comicon +pen kridge +passi vity +pa ap +over reacted +oven ow +ot aru +on das +om exico +ohio an +official tf +od cast +nu die +neutro phil +neder lands +nd wbb +nay ef +morning drive +mi festival +mer itor +mc mee +max thieriot +m fy +la der +ki dero +k btx +jewelry making +ive ta +iri an +inver gordon +indi ranagar +in calculable +in aki +im pasto +i ur +i il +harrass ment +green peac +god frey +ger ome +ge ti +fibro blasts +farahkhan ali +fal aise +f z +duc o +document a +dc cheerleaders +colling e +cha har +ch abby +cell c +celi e +candy crush +cancel o +burgen land +bi anch +bhil wara +bbc southeast +bb waa +avger opoulos +ath enee +ander ssen +ambassad orial +amb am +ack y +abur ke +ab om +a amar +# * +ðŁĩ ´ +é ł +wor land +wool pack +vital is +valenci agp +us ila +ur anium +under berg +trop fest +trol ly +titch well +thunder cloud +ther ma +th acher +te jo +tan trum +ta kat +strib pol +stress or +snu fkin +shy ama +she an +shak un +sco delario +san mateo +ru shall +round ness +ri baj +revol ttv +rain forest +r ê +r vb +q au +pu tti +pontific ate +pit tura +pilla ged +patrick mahomes +o dori +ni was +mout inho +mott macdonald +mm s +min il +middle march +mi sha +mccam mon +mc z +marath wada +lil jon +le pers +koo koo +kin zinger +kid scan +karen kilgariff +k fcb +just asking +jar rar +hu ffer +holler ado +hersh iser +he le +h icky +gru dgingly +grouse mountain +gom pers +go pirates +gamep ass +fore father +fo gler +f forde +ex el +ep ad +enamor ado +en closing +effe l +ed leadership +e bikes +du ston +dipikak akar +d acs +cly dach +cho desh +chit tor +changi airport +ce men +card captor +brun ing +br cc +black hills +bench marked +bay side +bar tek +anthony anderson +ann marie +am rinder +ag y +ab era +^^ ~ +ðŁĻĭ ðŁĻĭ +ðŁĺŀ ðŁĺŀðŁĺŀ +ðŁĮ¸ ðŁįĥ +ðĿŶ ðĿĹ +âĢ¢Ì ħ +yerushalay im +yah an +wu c +will friedle +vau dre +usa o +unice fusa +u las +tony romo +to des +ti memor +team blake +tax season +tatasteel chess +supercar sunday +ss ential +soc i +so kka +so den +sig nofthe +shum ate +shoe bury +senso dyne +science matters +san ge +saf ood +respectthe water +refra ining +real kid +queen victoria +que ijo +propri ety +prisc ila +plain clothes +per vious +penn sylvanian +par od +pag es +pac to +over stayed +off ilth +o tec +non discrimination +nex tofficial +ner issa +nat sec +myal gice +mo wa +mir tha +mi w +medi amar +marth apli +marthapli mpton +m vo +lun de +lost prophets +lo pp +little miss +lille shall +kw am +kin dy +j inga +it chio +im n +hol lowing +hello goodbye +heat seekers +g loc +fashi o +fac tional +exhal ed +eric as +ear ch +doo ds +dev secops +deth klok +de pa +de form +da hm +chu gh +christ on +chaus sures +chak ri +cel aya +can ario +br ind +black country +bilo deau +bick ell +bi frost +beau té +b nhs +az ee +ay am +aug gie +aper ol +apas coe +anti ka +anti bully +ambi en +adam driver +% ." +ðŁĴİ âľ¨ +white cube +whit erock +war u +wa sten +use lessness +uc g +u cr +tier sen +sted chat +staun chly +stargate atlantis +stal la +som ma +solympic team +solidar ityday +socialist sunday +snow watch +sker k +sin t +si mage +sfor trump +sf v +sch ack +s ro +rosanne cash +return able +ra vitch +r gu +pu cke +pro scen +poign ancy +philipp ullman +perio dization +or lan +oo bleck +naz es +monk ton +mogo eng +melissa and +march foreurope +mad dog +luc ille +lim oux +lie sel +lan dex +kt k +juli as +journ ée +joe hockey +jeff burton +jav aid +interdisciplin arity +inteli gence +hon ma +heriot wat +ha eng +gwangh wam +good lad +gin apu +gif tever +gavi ota +frequ enting +fore shortening +food ways +extran eous +ex ti +evangelic alism +ers out +ern st +emc coy +elle schi +ear muff +dougi emcfly +diag ram +de splat +dar laston +d welt +creature design +cran more +cos mas +cor ia +congress i +con all +ci am +ce ma +cbc murdoch +categor ise +c wo +bo we +bir die +bettym white +b anti +as pac +aldubang pag +al music +afric at +ðŁ¤¡ ðŁ¤¡ +âŃIJï¸ı # +âı ¬ +worth itv +worthitv ma +wil ander +way towork +wal ing +w co +undere mployment +tyranno saur +tv k +thorn wood +thorn cliffe +thisiswhywe play +subl ingual +streamfakel ovenow +ss chool +sister s +simp lot +signi fied +sent a +se let +sd beer +school sout +sb n +sal tier +sag ay +sa hid +s alli +rosen ber +roma downey +ro sti +reven ants +raven loft +r maf +pk mn +pin ako +pepper pot +p drm +ok un +ok r +odd ness +oa week +o gaden +ny an +now ness +nca afootball +nab arro +n ks +melis sac +manoj sinhabjp +m jo +lyn don +lumb ering +lot ter +las key +kstate mbb +kooten ai +ko sarin +kiri baku +keik amara +kail i +josel ito +jiha dism +inter related +im rie +ice breakers +hu gel +hit oshi +hei sel +guic hard +go knight +go en +go ba +glen arm +flye st +ever r +eric bellinger +equalit ymc +endof theworld +electionswith ht +doren bos +do vo +do besity +disin terest +diferen tes +di lett +dam ming +cun diff +club america +chi pping +che mbur +chapar ro +cel cius +car lock +can ham +calamit ous +book tube +blan ches +bhu tia +ben zie +bar gen +audra equalitymc +attenti on +arte k +ar kit +amdra deon +allred md +afgan syah +a ei +^ ) +ðŁĺĻ ðŁĺĻ +ðŁĴĽ ðŁĮ» +æ¸ ¯ +âĹ ½ï¸ı +ÑģÑĤ в +yo gaw +y tb +y it +x med +wine chat +wil do +wat too +w lg +vic om +ure thra +under nourished +u sche +typo grapher +tycho brahe +tv uk +totten ville +to ffler +thu sian +theband perry +the ee +te mo +tail gates +ste vero +spre paration +spo oners +speci aleducation +snever die +slug gish +sister love +shru b +sher pao +she ema +send in +se uk +sab ahan +sa or +ring fort +recipe oftheweek +pur ports +poly carp +pisco po +pier son +pas sim +par faits +panchay ats +over hear +ouagadou gou +onu evo +oat man +nab ucco +must ela +mild may +mihal y +micro breweries +masta ace +m wi +lon o +lo bel +light sfor +kozlov sky +kn bc +king bach +ki it +khu fu +kgal agadi +kevin woo +ke msley +kan chenjunga +joell ortiz +indi os +imperman ent +ima g +hul sey +gu yon +green lawn +goo dest +go ater +flor um +fl its +fin cham +fel issa +eh ren +eft pos +del rey +david bisbal +cowboy cerrone +coug fb +contemp tible +comfort zone +combin atorial +childish ly +child day +cherno ff +chast ise +capital ised +ca j +c pps +bio shock +ber nadine +ber gia +beau mont +bat tel +ballyna hinch +al ick +air vistara +agye mang +ad ub +( ?!) +ðŁ¤ ļ +îIJ ħ +íĥľ ìļ© +ãĥĥãĥ Ĺ +zum thor +ysle ta +y cee +wood bine +wirt schaft +w div +vive ka +var dhan +v wd +under cutting +um alo +touch ph +thri fted +te ching +suntrust park +squeeze box +sou cy +sof ast +sk ims +sc nn +sauté ing +ryn chibi +rush limbaugh +ru mor +rose town +regu la +rec ca +rd pro +randi zuckerberg +ps fk +people make +pand u +p sps +ore ille +orange crush +one town +nu blu +nor ie +new t +na sho +n atti +mar ination +mahon ia +ma inde +lv cc +lep anga +leon ia +kor ban +kn acker +key leth +k soo +jo bur +jacob sartorius +itsin youtogive +itch iness +iot swc +in cant +hé ctor +hu len +hor ie +ho ce +happ ym +ha flinger +gro ttos +gra be +fur rowed +ex ab +du lais +dout or +can ady +campylo bacter +bic c +bekin dedu +bar det +bally bo +auth o +ashley graham +ardaw igs +ap uri +anaesthe tists +alliseeis gold +all port +ðŁĻĦ . +ðŁĺŃ ðŁĴĽ +ðŁĺį ðŁĴª +ðŁĺĪ ðŁıĪ +大éĺ ª +x seed +wä rt +wo wsers +wis ata +wee ke +w ene +voy ager +vl si +visit savannah +tre ve +trail finders +then ut +the party +tene brae +tar pley +suzanne lepage +super cluster +stephen son +spri gs +spi vak +spa day +sp hs +solheim cup +shy ly +sfor women +school kids +schen ck +sab ka +ryo ji +ry en +rome sh +retrac ting +resc ate +plat ense +pint u +photogra ghy +petr ina +per mutation +penc illing +parale gals +pal at +outre aches +open doors +ong al +ok bokki +oh g +nzv sl +normal ised +ni ort +ni ed +nam b +n achi +mr selfridge +moun tie +mol lis +mob ilities +min ard +mimick ed +mile tich +mc isaac +maythe fourth +man flu +mag gies +mad dest +lu zh +lom ita +ll dubs +lincoln hall +lik ar +ko kol +kim mage +kid sand +keo hane +job done +j ma +j hang +itv sport +is ingly +id man +hp noti +home opath +happybirthday salmankhan +gov markdayton +glo vers +fu ganda +ex ican +encro ach +edmun dson +draw pile +do iron +dimit rios +delhi dynamos +cor vin +conoce artistas +commo diti +co atta +buff ay +bee g +beat navy +be tula +az ari +avenged sevenfold +apan thers +ano j +aldubbig girlz +ae schy +ade ci +aban erjee +:' > +ðŁİŁ ðŁİŁ +ì¿ ł +æ Ĥ +âľ Ĵ +âļ ĸï¸ı +z ade +yu th +withr and +vicen cio +va ar +un block +ukin india +uj iri +tri g +transgre ssive +tra gos +tor v +ticke ts +the stars +the ss +the matrix +the duke +stocking stuffers +stafford shirehour +so dak +six nation +rs ch +ricardoro ssello +reli x +recou ping +rangers family +radion ow +pe jeta +paper white +overex posure +oniz uka +ny le +nh r +new market +naf i +n drf +n aldo +mulvi hill +me ha +mcmen amin +mcgon igal +mall ery +logan berry +lemn sissay +laun e +la fon +kre ay +kim bolton +ke pt +ke ary +k hem +john swinney +jfk library +javedn laghari +itune spodcasts +i ken +hor der +hop kirk +hl mbb +gup ta +goo i +gi onta +ghana starnews +gar ryo +gar ib +fun nell +farmer sin +ex ib +es na +edi ficio +east leigh +dw yn +doit big +con gradu +compati bles +clar a +chro site +charter is +charlie kirk +chale ur +calibr ations +cal dwell +brank some +bison pride +bankrup ting +ba fe +ausi ello +al tria +ab flood +y boy +x bi +women said +wol ter +vocal oids +vibr atory +v yach +ur banc +tu fo +trim pe +trent university +tread way +ti pping +ther aven +the mist +stan bury +st ale +ss sa +sor oka +so tn +sma h +slam et +simone simons +sergi om +schneide relec +schle reth +sab bagh +s wh +rockefeller fdn +ri mba +rey ra +qune itra +q ip +ponchat oula +peven sie +perenni ally +pad el +osom atsu +organ elle +optical society +none such +nether field +neca xa +nationalgarden ingweek +nas daq +n guy +ml ine +me scal +me io +mazum dar +marvel sdcc +len y +kyo ya +knuckle ball +kni fes +kiis fm +ki shang +ketogenic diet +juan jo +impe ding +i goe +houston police +house forsale +hon ks +heg gie +hart ington +ha sp +gu at +gu am +good without +glori oso +ger b +geome trically +fox ing +fl oro +fend t +fairchild garden +f assett +ever sheds +enchan te +eli zalde +ed markey +ec ar +di martino +di atom +dhal sim +dead y +cuck mere +crypto pia +co authored +chir ic +chip stead +cas as +br ymo +boo sie +bo sarts +bo ki +bla w +bi ped +baz in +bar oud +az raq +az ara +ash h +as par +are pas +anticli max +another one +acadi an +a ite +ðŁİ¶ ðŁĴķ +ðŁĩ¨ðŁĩ ± +ðŁ¤ ľ +ì² Ń +while black +wh ern +v ws +u stain +tru stuk +top secret +today yy +tn news +thucy dides +sp fx +si solak +shak y +sh net +rufu swainwright +ru bal +re buffed +rc memories +ra aga +quincy djones +q ed +pu kul +prince ssc +presidenti elle +pre da +por fi +po can +pi atek +period poverty +pen light +ol wen +nitro circus +mu ddin +movie reviews +mono culture +mill bury +mid ori +merry n +mel lowing +medal monday +ma goffin +m ld +liti gators +krish olden +kno p +kla ssy +kf dx +kalancho e +jyvä skylä +jam ón +israel icon +ina ina +in anna +i bar +hot d +gumb oot +gri gory +greatest generation +ge da +fujin on +fuji x +fric assee +formul ary +flead h +finn mark +feren c +es gr +ent p +ele azar +egg old +ecumen ism +e gle +duvvad ajag +down draft +doet inchem +dis ambigu +dan reynolds +cy c +crox teth +coatta ils +co wappreciationday +clo ve +chil ika +c conf +bud in +bon us +biz kaia +bisson nette +beard life +awas thi +as mit +as ket +ap lanet +allen west +ac ads +ab v +^ / +? ðŁĺģ +:-) :-):-) +âĿ¤ ðŁijįðŁijį +ze ke +wild land +whytele afe +wat tie +w tt +vibr ated +ve chain +tuan ku +thereal p +the wright +tenter field +ted nugent +t mv +sof honor +simon i +shutu pand +shobha a +sex change +scul ture +run tagit +rock ineve +ro zz +richarddin atale +richard garriott +rajkumar hirani +radha krishna +pugli si +politici zation +pinch beck +over think +os loff +ok ker +oc tol +nostal gi +mortar board +mi stake +mary na +mar on +lu gu +love leam +li acou +le grand +l fd +kate flannery +k fa +it ou +iran freedom +il agan +holein one +hob son +henley regatta +hari krishna +hans brough +gustav sson +grizz ley +good things +go yen +giandu ja +ge tn +game faqs +g sd +feliz diadel +f vg +f oma +ep w +earth strong +dun nell +drjoe abah +dortm under +cour trooms +ce uta +cape k +cap les +buprenor phine +blues ville +blon o +biophy sical +bill bailey +be dene +batter son +bal r +baker mckenzie +aw yers +ave yron +au solympicteam +asteroid day +anuger ah +al shabab +al gie +ain sdale +ach elle +a hed +zi yon +vu du +volu me +visiti ow +ven den +u tha +trend hunter +tore lli +tom islav +the web +ter ada +sugi zo +sol va +sker ry +seren ity +sempiter nal +seismo graph +robe spierre +road er +ro da +reverber ate +retrospec tively +red s +pre hen +po trait +po in +plat for +pi voted +pet an +per nah +parkinson sdisease +p mn +ox a +ou ard +ol ah +norway mfa +north borough +nor ceca +neel ak +n dom +myhaver photography +mu ara +mobile dev +mid c +mer ak +melo dia +me con +max am +man gom +major leagu +long last +lo hse +lfc tv +kle iman +kit ne +ju ge +jesse tyler +iy am +italian art +ir th +ice house +hill arious +hello fresh +hamilton college +haen ni +fanc on +extermin ators +experim enter +everyday is +european elections +ess ai +e wel +dy land +dramati zation +dr ra +dh m +complex con +co ate +chá vez +cay ton +car nau +bronx zoo +bo pinion +black strap +bee ches +bang ash +balac lavas +b cash +au rie +ar ayan +apoor va +alder sgate +ae mia +adi aries +ab aca +ðŁĽ ¥ +ðŁļ² ðŁļ² +ðŁĺĬ ðŁĻĮ +ðŁĴĩ ðŁı¼ +ðŁ¥ĩ ðŁ¥ĩ +çĻºå£ ² +âĶ ĥ +za ini +x company +whatiwant most +wam bui +wal ima +w cr +vi gyan +vai dy +usc gc +usc c +upp ort +unse rer +unisouth wales +un watchable +tom arnold +tit u +timo teo +the odd +templ er +te hel +tb d +superstar rajinikanth +stick ler +ste phi +state house +staf fies +spur se +soto grande +snapp in +sm g +shug borough +se ish +scher tz +sar at +saath iya +s fin +rose bush +ri sser +reci a +re forged +rat oath +rafale deal +popu lus +pollu tion +poker news +po blen +pic col +par acycling +nbab allot +mummi fication +micro credit +mas sport +man rique +make chester +mah endran +lennox town +lake placid +kunis ada +kray zie +kezi ah +kd lang +jame shar +ix a +ity council +isol ator +ingo ts +indoor oo +indi atravel +in heaven +imp ish +icom os +hy uck +ho ppy +hi ei +health talk +harmon ised +getmoney out +gb g +gay timesmag +g da +first legoleague +far ry +eth je +epi stles +eco l +dunn ville +dre we +door frame +dino bot +co asto +cha dian +cast rol +bluel ight +big daddy +bernade tte +be true +bar la +bacter io +back scratcher +b music +ati i +ast man +andyburnham mp +aitu taki +aerop orto +adam curry +abdul lah +abbey theatre +ðŁ¦ħ ðŁ¦ħ +구 구 +ಠ¯ +ঠ¸ +ym fa +ya red +weiz mann +war my +walkthe moonband +van c +vac c +tom m +the mummy +tess it +tell ings +teamdi data +survivethe ark +starvstheforce sofevil +snu ffle +snow don +sin que +sch ach +ri ah +retin oblastoma +ren mark +rein aldo +re establish +pride day +philli pp +panch gani +paedo philia +ous and +ok abe +oc ic +nu mpy +no brainer +nick le +ne eth +nair amb +moun te +mc math +maui jim +mar ring +mak ana +ma sr +m div +ly ch +loveyour melon +looknorth bbc +liken oother +le breton +kri spie +koech ner +jessie ware +iklan in +ig we +hindu ja +het chy +hed ger +haw thorne +gravit ated +googl enews +gbo wee +gas son +fsm py +free event +ess l +emor ies +ebon i +du iker +dre scue +doit yourself +di emen +derek carr +demysti fied +cultiv ates +cr cc +corr alled +colqu houn +ci ro +cha shma +car se +calci fied +cack le +ca shinin +bry ony +brad berry +blossom sband +bin tur +battlec ruiser +bat umi +avak ian +au dette +ann apolis +ang t +aiya ary +abut ment +$ ? +ðĿIJ ¡ +ãĤ¦ ãĤ£ +wi fis +wha aaa +wax editorial +w bbj +vis a +ultr arun +uk pubs +turntab list +traf low +toy show +to seland +ti fft +tequ era +tan sley +talla poo +talk sport +taka hata +sy am +spring fever +spon don +spike ball +sou tine +so bered +slo vers +she saidyes +sd ks +sal onika +rol ph +roger stv +rhodo chrosite +regular show +re culver +quiz timemor +puppy bowl +psy lli +princeton u +pri eta +pol ony +pneu moni +pipistre lle +phili pe +pan ah +oooo ol +om ance +ny havn +no ff +nhl bi +mu stan +mu dding +mot tos +moro zov +mick mars +metam aterials +mer ick +me ili +mck enzies +mc millon +marsh aling +makechester proud +love goldenheart +lo red +leg work +korta jarena +kateys agal +jic ama +it sac +influx db +industry news +ide ac +ice ice +hulu si +histor yo +hem ric +gu rel +go bearcat +gen maicha +for four +fearless records +f ö +f stvl +end note +dug anda +drum chapel +draught sman +dom ingue +def a +daf ridi +d bs +cyano gen +cor bel +copp ery +con oco +chain saw +cau then +by g +bun ka +brombor ough +break ie +book awards +back of +astr alis +ar mill +ar adia +amar ante +ac eves +" !!!! +! âĿ¤ï¸ıâĿ¤ï¸ı +ðŁĴŀ # +رÙĬ Ø§Ø +Ð µ +z the +yan ag +y thon +whitting stall +we sties +ur vivor +uphol sterer +up h +un char +u fd +twee ty +trin comalee +ti ar +thexfactor usa +thecoop way +the vet +that game +testam ents +tc f +tam bun +sus ans +su maya +sser ver +smith wick +shiv dasani +senig allia +sen randpaul +seag ull +satur no +sand inista +sa jan +rub bishes +po legate +plain well +omer ta +oku bo +og l +ncan ationals +murrum bi +model ismo +michael urie +melli fluous +media awards +massi mino +mar ja +man handling +madein nigeria +ma sami +list serv +liqui date +lin nell +lesm cke +lazz ara +laura dern +l bof +kr cg +k jer +izz et +is ser +hel pe +hard aker +ha bba +gü ell +gwanghwam un +gro ttoes +gover no +go eth +give us +gi gli +ghis laine +g football +fore taste +far yal +faith hill +es days +eidal fitr +ed ong +e im +dra ko +diffic ul +del port +db ms +dari ush +dam ekelly +craft work +cor relating +col back +cine magic +chinch wad +champion sday +ch ems +cawlidge hawkey +candid ats +c engage +butter ly +bush wacker +ber gu +be ek +ausv sl +au fen +alco co +ad woa +( ???) +# < +ðŁĴ Ĵ +ðŁijİ ðŁijİ +åľ Ĵ +å£ « +âłĢâłĢâłĢâłĢ âłĢ +âĵ ľï¸ı +you view +winnerwinner chickendinner +whe ater +wh iti +wes thuizen +vsc ancer +virtu o +ver ghese +up ci +twitter smarter +tu junga +sudhan shu +stop tb +sp acy +small sat +sel ayang +seatac airport +sa im +ro hl +rigon deaux +rec tification +re mu +r tos +r ra +pur gat +pur ba +pug ilist +promotional products +pri e +poe tica +po tala +po sso +pejor ative +paragli ders +on broadway +oh dear +nn u +ner ina +mer ger +mede ski +me les +max illary +math works +mal har +lu uu +long shaw +llanid loes +liber dade +le vey +klo ster +kamal ha +kal si +junior contract +jom alon +impropri ety +il ondon +hur ontario +hen wood +har umi +gulfof mexico +guilty pleasure +girlsin tech +girl stalk +gau cho +franchis or +foxsports det +for mel +fit na +fairtrade fortnight +e ire +dreddy tennis +do oney +desh ler +del oris +cork screws +cop al +co zz +clo gher +cir ce +cindy capo +chlor ite +cal cot +brown live +bro fur +boy ssoccer +bel al +asse tto +ask with +angar cia +amal hotra +al faj +aja hn +aguas nash +ðŁĴľðŁĴľðŁĴľðŁĴľ ðŁĴľðŁĴľðŁĴľðŁĴľ +ðŁij¨ ðŁı» +yse ali +wwww www +varsity cup +under achieving +twitter friends +tri de +till ing +thep slt +the thing +the dead +t sy +str acism +stepby step +sprow ston +spor tiv +six ers +sie gal +sf ball +seraf in +q tc +proven cher +power lunch +plur alistic +play with +percol ating +oun tain +omin ion +obl ate +noo d +nizam abad +nit rite +nakat omi +n alo +me gu +marketing agency +lo ess +liph ook +len sculture +l lew +l cu +ku en +ki ppers +kalon ji +just cause +jessamyn duke +jat ya +icen ine +hydro meter +huuu ge +hard ly +gru l +google india +genealo gists +gam ache +fw ends +fresen ius +fil mi +esp inal +ep sb +enam eling +el ar +e ala +disc oloured +deri ded +derekcarr qb +deno ting +das sey +cru ce +cross over +c spc +bu hat +british monarchy +bo zar +bo bov +blu emotion +bar dock +bar ada +ban kni +aw wal +ash burnham +ans combe +am joy +agu stina +ag ould +aeschy lus +accident als +. "... +ðŁĺįðŁĺįðŁĺį # +ðŁĴĭ ðŁĴĦ +âĪł )_ +ঠķ +y talkies +wi shy +w gw +virgin active +vindic ate +vin ous +us ba +ting ting +tin aturner +the defenders +texashoo ps +tail less +sur ya +stray dogs +sto dgy +sr p +siguemeyte sigo +shu mba +sher rin +shah jahan +sarah geronimo +round wood +ro blin +reykjav ÃŃk +reinvigor ating +reic hardt +re work +re khta +pro stration +po ona +obase ki +nr can +ni os +ngu rah +nex tup +nepon set +my bestfriends +mon dad +min dedly +mikha il +mer k +med tech +me harry +luxu ria +loo by +lo phus +le ja +kyung soo +kel lar +kath thi +kab ul +jo sap +ja ars +illi um +hydro dynamics +hot n +hornet pride +hilde sheim +hi mig +gold medal +go sch +gl itz +gin us +ger ards +gare the +free wheelin +frank ton +foodpor rn +esc ap +end malaria +eag lets +dougie poynter +don ahoe +dol fin +die antwoord +denver outlaws +deniso hare +daw sonville +cph ftw +concur rence +co omes +cand ace +call an +braun ton +book stall +black forest +bil d +bike show +beaver brook +be fallen +at kin +arma gh +ai vo +ðŁĻĮ ðŁĺĤ +ðŁĴī ðŁĴīðŁĴī +ðŁĮ © +ëį° ìĿ´ +âļ ĺ +wil ight +whern side +vexillo logy +uni as +uin tah +tyr whitt +tu lipa +too sweet +ton glen +thisi si +stich ting +so horadio +sher way +shar rock +senti miento +sch rier +salem or +sal y +rich ten +ri bault +play throughs +pi pp +pet ko +p inga +or chester +now shera +nar re +mtn ng +mor iya +mis ophonia +megan follows +me ades +mcglo in +mayan sfx +make better +la plagne +kru mm +kas al +ka pok +jamesb valentine +israelicon flict +hu sain +housing day +he dera +hass all +gran holm +go air +geet anjali +fuvah mulah +flag man +fik ri +ex ley +enjo ining +endor fer +emo ir +diss ension +dismant les +deptof ed +cy bele +curl pro +chen na +ch rit +ca hoon +c ire +buffal ony +bou dica +bo ff +bla by +bill and +b isou +ast ound +art style +anti phon +air munro +ad of +abel ardo +aa di +! âĻ¥ï¸ı +ðŁĴ¿ ðŁĴ¿ +z enga +wu bb +wou l +world rowing +word books +west more +wee z +we belos +wayne coyne +vel outé +trans lu +the ki +the bible +th ale +telegraph travel +teal over +sopp ressata +snh strust +sk ola +sh ary +set lock +seg menting +rivieranay arit +rig ney +reg ner +refin es +realkevin nash +r q +quad rants +pra soon +per otti +one thing +oklahom acity +offthe beaten +nicol am +ne ara +mj m +magick al +macle od +long lines +lo licon +lew k +lali gaen +la foret +l jones +krist ofer +koe hn +kin ahan +keigh ley +kan ab +k pe +janef allon +is w +infinite warfare +individu alised +ic wa +i spor +hy patia +hy bels +ho cker +her dy +heim dall +hag ans +haare tz +gugu lethu +fri ggen +fore seen +for tino +fidd les +fel ina +endemol shine +en snared +ema snhstrust +duck pin +disability rights +dad agiri +cur wen +cosmopolit ans +ci pher +choc cy +cau li +calam ine +brim field +breaking the +bou man +boston cannons +boon sboro +black smiths +bistro s +bing u +bin ta +bel air +baru ah +b son +azira phale +axel sen +aviation history +aspen snowmass +am bat +allan mcnish +abstract artist +aboiti z +ðŁıĮï¸ı âĢįâĻĤï¸ı +ðŁıĪ ðŁĴĻ +ðŁĩµðŁĩ ¾ +å¿« ä¹IJ +ãģ ¿ +Ø « +Ê ķ +wärt sil +world musicday +who syour +v hd +utt ara +under garment +twit pod +tun de +town afc +tn cs +ti wary +ti pu +the hu +t fk +sy ariah +sweet leaf +swar m +stu tzman +stock land +stab at +sky ferreira +sinhal ese +sin opec +si eu +shu ey +shi ppy +sens go +s itti +rob stown +ren table +reli t +release day +red ol +pv hs +protect ing +pi pits +pe ton +patrick starrr +pam grier +p isode +on thel +official rufc +obli ges +o boro +no cona +niem and +mosi ah +mont vale +mo oned +me ir +mas ke +mara sigan +ma alik +lu gia +la weekly +la flamme +kin di +ken non +ke bun +kar ur +incre mentally +haz lett +hatsune miku +han dedness +ham pion +hack le +gy r +gu inee +gor rie +goldeng lobe +go sensgo +gnar ls +gh pl +g ines +fw end +forten berry +fo gelman +films bywomen +field turf +el tham +dwt sirl +drac aena +de maria +crime wave +creati vo +coun sell +com busted +cohe sity +coffe elove +chill is +ch ancy +candy crush +c ée +brighton seo +bor rel +bmw motor +bha vesh +ben splatt +be kal +atel ink +as pac +as ama +aren a +arab israeliconflict +anti fragile +andu in +an je +amand ashi +alliter acyday +ah and +adid asu +acu ña +ðŁĸĸ ðŁı¼ +ðŁĵĢ ðŁĵĢ +ðŁĴļ ðŁĴľ +ðŁIJ¼ ðŁIJ¼ +ðŁİī ðŁĴľ +æĪ ij +youth sports +yas sssss +wu i +willem se +whi tham +whakat ane +water aiduk +wash fellowship +vie v +u va +tor doff +thomas power +sxm thehighway +swap na +stren g +sk ot +siwon choi +sie gel +senator burr +sap utra +s bisd +ry ant +residente vil +renov ates +pre ssions +pre operative +po vey +pimlic orc +pedic abs +pedal ed +operation ally +nyc mayorsoffice +nd h +mu ty +mis cast +milan ello +meso sphere +mean whi +mcgu ffey +mar ois +lou vain +lo bed +learn german +lau dato +la story +koz lov +kate bosworth +ka eding +jol ted +jen ac +jare th +jaga dish +iy er +ingeni ously +in fier +hil mar +hawk stalk +h tn +gw ana +gu shi +grim ley +go terps +globu lin +ga via +frnd z +fran con +fli bs +fili ppa +fight the +fe sting +fan meeting +f sf +ezra klein +espad rilles +ell roy +dot te +don en +do yeon +dictator ships +diab olo +deli as +concre ting +coach tom +bo young +bo lex +blue tec +as st +are scue +ar rs +ar preps +aqu ÃŃ +apla za +ander as +alali brary +ajed rez +________ _______ +// < +ðŁĩ¯ ðŁĩ² +ãĤ º +yow amu +x th +wyn koop +wind surfers +whitworth art +whisky day +visit bristol +viking cruises +vic traffic +v rij +uw sp +un tangling +un ops +un ds +tv live +tu tee +tto win +thr illa +tb it +tardi grade +tar ring +t bex +spinning fields +sky deck +sitaram yechury +shoo touts +sali f +rod da +regurgit ated +ram ez +rabbin ical +picto graph +phil brick +om ake +ok hla +net tie +ne ster +nat gallery +nan terre +mum taz +monop rint +mo ggy +mn gr +mix nine +mari ek +malk mus +mal tz +lou den +lic hen +ks music +krat ts +ki ro +isthe problem +interfer on +ill enium +ig ate +hump backs +hot toys +hir si +hi rayama +hast ings +har pal +ha iz +ha islip +green chemistry +gre nou +glam cricket +ge thi +gar ita +flamen ca +film strip +f pg +f gn +ene ts +ely as +ejec ts +den de +dc policedept +dan diya +d scanning +d endi +cs music +craigh all +community college +castro tx +campbell river +cam eo +cal shot +bre slau +bor al +bla gdon +bike to +beat bama +ange bot +amjoy show +am hs +ali bre +aen gus +: £ +-____ _- +ðŁĺIJðŁĺIJ ðŁĺIJ +ðŁIJ © +ÑĢоÑģÑģ иÑı +yucat án +vu h +vol tac +veg gi +tor ians +tho le +thing sabout +the paul +that sa +ter hune +tel p +ta war +sub type +stra iler +sto kley +stal y +son arika +smartcity expo +small ness +sm t +sk itter +sig am +shivam bu +she pley +set to +scouncil uk +run gs +rob illard +ric kert +repl y +re qs +raf san +r re +pur porting +pic acho +photo copies +pere go +pedi ment +pal anca +pak man +pag ination +on j +oda at +o denton +new roz +multi view +mtv u +mosh ood +manoj tiwar +maggi es +m ool +ludwig sburg +lique faction +leh man +kuy per +kar nazes +k see +juniper networks +james acaster +its bristolbaby +ise f +ine yards +in cel +huf worldwide +hss wi +hsin chu +heu ser +he tton +harmon isation +gry ffin +gr aco +goldsmith suol +gitt ens +ge ith +flood plains +fe en +exacerb ating +douche bags +do de +dill man +diamondand silk +de itch +cradle offilth +cor ti +carry themhome +bri mmer +bio bio +berry farm +bang sa +athlon sports +ap tac +ap athy +amit ra +ale quinox +agre ssive +accli mation +ðŁĴ¡ ðŁĴ¡ +ðŁİ¶ ðŁİ§ +ðŁ¥° âĿ¤ï¸ı +x ist +wood man +whe ads +well ston +wave front +vasi l +un solvable +ull mann +ug t +u gal +u arkansas +thejuan williams +swa deshi +st bl +south yorksbiz +so cc +sil v +si kk +service dogs +serafin owicz +semi ah +se mir +rou steing +puer tas +philly mayor +perio dismo +p daf +owen benjamin +okum ura +o esn +nutrac eutical +nu bians +ni pah +már quez +mur li +moon bow +moj ica +mine workers +midter melections +mene fee +melan son +mc tom +may sa +li ska +length ens +lady boss +l ro +kost ka +juke box +jones ville +in oki +howto trainyour +harmar superstar +hag akure +ha ch +guine afowl +greath ouse +glan z +gay don +game jobs +fu yu +fr ancy +fle dging +fl ours +femin azi +f xs +emma willis +ell ard +ei ht +du ed +dispro ved +dese greg +dat o +cr ater +citizen ship +burak deniz +brew ster +break away +bo wale +blake slee +bi gor +bel mullet +baloch genocide +ao ib +am enable +ali ando +ac ros +a shem +: , +( .) +ðŁĺį ðŁ¤¤ +ðŁı ¥ +â µ +zug spitze +xi xi +window sinsiders +wig town +weather watcher +wayfare rs +w re +vas ilis +vac ates +tiger up +ti mp +ther ocks +the challenge +te kk +taylor guitars +surrey life +sto ppin +ssc ricket +spo se +solution tree +semy on +re ber +ram co +pre tension +pre ike +port way +pig my +pen rhos +pe ci +par dub +packaging design +orch ha +nun c +nobuy uki +new fie +national champs +mo tability +mi global +mer ay +meet bros +medal en +me ki +makh ura +lur ve +london lgbtpride +letsgo dodgers +kle ys +key one +k vn +jig me +j rn +j mr +iphonex s +insom nisa +indooroo pilly +indeci pherable +i asc +houseof cubs +hoge styn +hei fetz +hare hills +ha fsa +greeng ate +gre ss +gir ma +gh earts +fl oria +exagger ates +ev re +ep festival +eliver ance +disco vers +dag ang +consequ ent +complex e +by ward +ban yan +ay un +attenti veness +arch viz +ar lette +apu blic +and ong +an ae +aldub maiden +ad it +actu alize +ac tis +aamir liaquat +ðŁĺį ðŁ¥° +ðŁħ° ðŁĨ +âľĶï¸ı âľĶï¸ıâľĶï¸ı +âķ ® +z de +wrong ness +wood chips +wal ke +vum chealth +ver kho +vape shop +van esa +vai ko +tra wick +tor ti +tobaccon ist +to pl +tim westwood +thousando aks +ther is +terrence j +technicol our +te gern +stru tter +strait jacket +spl center +shakey graves +sa stro +s ddc +run meb +ro setti +revel le +re shuffles +rash omon +ra baul +queen b +praise god +panor am +oin uma +oe ttinger +ny dia +nxt takeover +na sia +n roll +n nd +my best +morning show +ml td +mikey way +mass ena +lun ged +long live +litvin enko +law society +l hh +koe itec +ko dai +kick starts +ki gur +its been +ileague official +ide sai +ic ast +hel icon +hei sey +guest post +gor achelle +gorachelle ann +gine bra +gaw an +for der +flagell ation +five a +fin esse +epic cosmos +el ow +eight ball +dramati zed +donald duck +di dio +design milk +dar lene +curtin uni +cougar nation +convul sing +co sn +ceph alic +cav orting +cap el +ca isse +busc aglia +br ts +book storeday +baf a +ati ds +ar ling +appall ingly +agri busines +adu rai +á´ Ģ +world snakeday +wing y +warren sville +usav s +upp et +u sportsca +tru ll +toplo ader +thi z +the her +tas nim +su pts +soph more +sin ai +sil vas +se asia +sd oodle +sa ed +res or +pre seli +pr h +pope scu +pc sk +our schools +or du +op roud +oak ie +now www +new all +mov ingly +michael annett +mer amy +mahogany lox +lyn es +lin csc +li gety +lett ice +l vi +kha si +ken rick +kah ana +joanc rawford +jo achim +jab rill +itsad og +incarn ated +i fri +hy ong +heee ey +happine s +had denham +guer rier +geb hardt +funkand soul +franco phile +for lease +fing alcoco +esi ason +employee experience +eb ace +e miko +der as +d design +cu ms +cro whurst +co omer +cmb yn +chim bor +che min +chandi mal +car swithoutlimits +busines sperson +big ay +bat tic +au j +astor i +anne aling +anc ru +al g +ag ba +african us +a seem +:- )))) +à¹ĥ à¸Ī +Í ¡ +xi es +wit tig +wise guys +virgin iam +vir chakra +vel ux +ut ton +un guided +ubi q +u calgary +twy cross +twe at +tra van +tibetan buddhism +tem be +sthe series +ste ffy +serv pro +secul arists +sanctuary cities +roy an +ri ems +res ounds +raven scraig +rash guard +ranc ourt +raise d +qu ent +qu atu +punjab is +prize money +positi vely +pe ste +pba onespn +parmi gian +oy ale +over coats +ol abs +nca ab +musa fir +mm tc +mey rick +metal album +merry lands +mechan ization +me gh +mat eri +mad aba +macewan u +lu king +lu dden +liber ians +lem nos +langu ag +ku tti +klein man +keat ley +k de +jo j +jan se +irr ational +inf l +ie b +id in +ic et +i herb +hispanici ze +hij jah +hepat oc +head ship +hallo ck +hal lie +gur ps +gu fc +globe trotting +g sv +fur mint +fugli en +fit food +femmin ile +f cra +ers rock +dwind les +dv g +dos gaming +dimension ality +denti st +dee g +de mont +dar yle +corne as +contain ership +cent um +cas os +can ción +campe se +bul ski +brockle bank +biz nasty +beat son +bas le +bal derson +b my +as g +ann ua +aeter na +ab senti +ðŁĺĦ ðŁĺĤ +ðŁĺ© ðŁĻĮ +ðŁıIJ ðŁıIJ +ðŁĮ·ðŁĮ· ðŁĮ· +ê·¸ë ŀ +ÙĬ Ùħ +y news +xx oo +whirli gig +web star +waynetwp super +wat teau +twitter mirror +tre esof +tigo bba +tame side +sub junctive +stru dwick +ssi mon +ss aturday +sla b +sigh thound +sie gen +sey more +semin aries +seem s +samurai jack +sam ma +s sudan +s ros +rohit roy +rail cats +pose able +popp leton +pas cack +pan handles +oo se +nice guy +negre te +n ssn +n ough +ma kak +lo thians +live underpar +la kiss +la ha +kon ner +kochad aii +ko sinski +jeremy mckinnon +j bonamassa +iv ins +hue vember +houri hane +hop f +hok itika +ho xie +hide out +hee bie +he cla +hamlet fc +hal kidiki +fre de +fm kenya +flori das +fat cat +dulwich hamletfc +dri skell +drac aen +dou bs +demir tas +dem socialists +dc tid +creative scots +conserv ator +co ko +co by +clay born +castell ani +cas sa +car fag +ca elum +black monday +billy bob +ber ndt +ber mingham +bed was +bally shannon +au ba +ascen so +ar mel +amaz i +........ # +åIJ § +âĶ ĵ +âĢįâĻ Ĥ +ze th +ys lam +wool shed +wol fal +wal shy +w gt +voi vod +vie ux +vic ini +veri sign +vare jao +valu er +un cas +ty nes +town line +tiktik tik +tidd ly +the villain +tallapoo sa +t lax +sydne yo +stein berger +star base +spider sona +sp ini +snit ches +shapp y +se kt +sc ag +sasi kumar +samanth a +roe hampton +robust ly +referen dums +re invested +ra ghe +r á +quin livan +pul ford +proven zano +pras lin +portsmouth nh +plo rers +play more +plas monic +pil ar +peter sagal +pang an +pae onia +osor no +orel lana +on repeat +og maco +nz vaus +now den +notinthis lifetime +neil sen +nathan varni +mo sta +mam usic +mal wa +l jp +l chat +kun di +kare ga +kan ald +jo swinson +ji rou +jar k +ja ig +ite k +inter costal +indi stin +incogn ita +incivil ity +hydro codone +hocu spocus +ho pin +ha sen +go jordan +go figure +gm fus +gl unch +gi unta +gepp etto +gang ly +ga hara +enamel pin +en gro +egg less +ec ru +ea sements +durham nc +dun sfold +down range +double bass +da day +cy on +cra ine +cover story +conson ants +coach jim +co quet +clu bo +cliff central +citym all +chair persons +cav our +carls bad +canvas ser +can dia +cab elo +ca ille +brun elleschi +bore scope +bear ing +ba sile +b cra +ati q +arch s +aber crom +ðŁĴĹ ðŁĴļ +ðŁĴģ ðŁı¾ +ç ¶ +е д +zi yad +zah le +wr angles +woooo ow +wit old +westend live +von age +v fp +un toward +ulti max +tre lli +tie gs +the difference +tera hertz +tallas see +tah qu +sten ation +spe tt +song jihyo +si ong +sask power +sadi sts +ruffin o +rooster s +rom puy +rogow sky +read vocates +ra dom +quin ney +queenof scots +print ings +prayfor southkorea +pen coed +pei poli +pad den +open cv +o ab +noar lunga +nihal ani +nect ars +mu dras +milan esa +mient us +mev lana +mazdar aceway +mat tock +marquin hos +marine inst +ma uli +ma ja +llll llll +j rr +inte mper +indiab ulls +ind travel +in service +im melt +holy day +hardik patel +hack ley +green hithe +gan ic +formula ic +fin domme +figu eras +father andson +f ww +end ricks +ear ing +duvvadajag annad +du bbin +dru ms +din ary +dick heads +daf fs +craig kielburger +cra ik +chi hay +cas inor +can an +c gpa +bristo lold +bj u +bi ben +bear ance +bay nton +bag ge +ay la +as afa +are ena +ane ka +am zing +allen and +alam gir +af ound +a egyp +ðŁİĤ # +ðŁĮ¸ ðŁĮ· +ëĦ ¤ +à¹ģล ะ +wer ke +wehr lein +we igl +v anny +ush mm +tr illed +timeto shine +the worst +texas forever +ta vor +t ads +swoo pes +sumb ar +stor row +spol icy +so ontario +sme uk +sm l +sheskindahot musicvideo +sanjay azad +sa am +roz as +rock androll +ric kon +restric tor +respect for +quart ile +pul o +pu sey +pr on +pope franci +pon ts +paro died +o shea +nr genergy +nov um +no words +my m +musco vado +mom ir +mn or +min omonsters +mi zo +lucifer season +llan os +leishmani asis +lat u +lacri mosa +kk box +kid brooke +kan es +justicele agu +jung min +ji eun +jennifer nettles +jan ah +itur be +is foreveryone +inge cho +im t +hy am +horse back +hol dover +hocke ssin +gold leaf +girl strip +galle ons +fs midwest +fie vel +femini zed +fe herty +equ idad +el stern +eat the +durham cricket +dragon fruit +dima io +did st +destabil ise +de ok +dat al +dardan elle +coach able +cere bellar +byom kesh +bu bi +bro cket +bra instem +bin tulu +bin dass +bi asa +be vs +bas ah +ba hau +ba arba +asian et +ash na +ag rand +ðŁĽ £ +ðŁĻĮ âĿ¤ +ðŁĮ Ĺ +æĴ® å½ +âĻ¡ ) +âĺº . +woocraft scs +wino grand +what t +wed s +wd ney +watt bike +was sa +vit ational +v mr +us ko +un varnished +tu que +tsu chiya +tr ama +total war +tidal x +thanksgivingwith blackfamilies +ten ge +teacher scollege +switche shop +sul phu +stre lit +stil inski +st bri +sse airtricity +south hams +sour ness +sky i +sj e +shu ma +shail endra +shab elle +semi automatic +schlad ming +sc use +sc dp +sagu aros +sa ami +ror attack +robert marawa +ro tr +ro em +releg ate +redondo beach +purple army +pu tsch +pro pos +pro lapse +prized raw +pre disposed +pol unin +pay scale +pau w +pad locked +otta wab +opp as +never getsold +n cre +more fun +michal ak +meramy akrishnan +medi aboy +mb ira +maul s +malayalam review +love wildlife +lordof the +lidiab asti +la shawn +kumbakon am +keep americagreat +kann o +kamal hassan +ist h +indigen ously +iac occa +hoo pinsider +home field +holiday spirit +holi es +hershey pa +heim an +ha kun +gonz aga +gon tier +go cat +gay dos +gab in +fy rom +fe verything +endo carditis +en ita +e ev +dog mas +dis rael +da via +d ún +d mt +cá diz +cow gill +cl news +cheru bim +canoe ist +by fuglien +by doing +bu sed +brux ism +blazer nation +bio scope +bad girl +avi ds +assu age +ar ouses +apost olate +andre ward +an ura +alvaromor ata +ðŁļ ĸ +ÙĨ ÛģÛĮÚº +z ool +yep live +y ke +wunder lich +wow app +viol inists +ul haq +the resident +the age +tfl s +team cap +te yes +te bo +te alight +tau n +swa ine +suf croot +sufcroot shall +sto ken +spraw ls +spra ining +sof apaka +shots fired +semb awang +sel ft +scienti fique +sch wabe +sar nies +sap er +salv atore +read indie +ra at +preike stolen +popo va +pin ney +opar di +omgom gomg +old paths +ne ese +mo jokerto +mein ers +ma inst +lil as +li hue +legisl ated +le mmons +ld nairamb +laudat osi +lanca sters +lan thi +kontak te +knit ter +ki vi +khaleej times +kha war +ju árez +joe the +jason r +is sy +i fru +humber stone +ho tta +hi jas +han kin +hallam shire +guine y +gopo lov +gom oo +gom an +gamer oom +fords theatre +f gd +ex os +er melo +er f +dy ad +dol gopolov +dic er +deser ves +dep to +den huys +deduc ting +day es +dani yal +d tz +convul sions +cil acap +ci ri +check mark +ch aba +carter reynolds +bruce wayne +book mark +boo kexpo +bla ue +ballybo fey +b ma +ar shi +am yn +am vca +ag race +actualit é +# ) +!!! ??? +ðŁĺį ðŁĺĦ +ðŁijij ðŁIJĿ +ðŁıĥ ðŁı»âĢįâĻĤï¸ı +ðŁį´ âĿ¤ðŁijįðŁijį +ðŁ¤ijðŁ¤ij ðŁ¤ij +à« Ģ +zy x +yah shua +wsu cougfb +wolf hounds +winniem andela +white gold +wa key +video juegos +ve sti +uni leiden +tx grizzled +ts ne +the hope +the chief +than ky +th ral +tau riel +t flofficial +super sprint +su ro +sja day +si rc +shra ger +sha hn +sh lita +san ita +sali eri +s gairshow +s bb +roll pride +richarde grant +reagan rays +raza q +ra fal +qui etude +pu long +priorit isation +popp en +pit stops +pin dar +penhali gon +pe acoat +parthi v +pa ean +our country +ouar zaz +opp ur +ni um +mul roy +monterey bay +mod bus +missamy childs +mephistop heles +me gang +me dulla +mag ent +lit aford +lidiabasti anich +les age +land guard +labra da +ku tt +kodo txgrizzled +ko tatsu +ko bby +joy ceme +jac ana +ite asy +initi ations +in this +ili ani +hé lène +hur tt +hur ns +hi bari +habi bie +ha fod +h fb +gran ata +goat man +go vardhan +glac é +gi de +gal vatron +fu ster +fl out +eric wareheim +duvvadajagannad ham +do ggg +devon seron +der bez +de anc +d wain +cut the +con sensys +communic ado +chal kidiki +ch appa +c za +bot olph +barthele my +ban jar +atta wapiskat +at tax +ar vs +and rena +aman si +allo fus +agar ajan +ad ms +ðŁĴ² ðŁĴ² +ðŁİī ! +ðŁĮ Ĥ +zim bardo +z ev +yot tawa +yahoo live +xander berkeley +wo de +visi oned +u og +twir led +turbo fan +timeto fly +thun dered +thearcan agame +the fire +the bb +te et +ta wh +swee tums +sun danese +sp lish +snake pit +sidd arth +shot by +shop lift +sheamo isture +shar mel +sam bu +saint msg +ro dge +resolu te +ren to +recal cit +que eni +qu ili +q am +putin rf +prun ty +pla stica +pla gue +park lane +oligon ucle +o ingo +ne ch +nab awi +my nba +muse umm +ms ds +mihaj lovic +maxi mu +max chilton +mari ote +march esi +mam ey +lal bagh +kov ski +kj ell +kendal calling +just icec +ju re +jamaic ap +jab oo +ist p +isi o +invest ing +hypo xic +hyou ka +hay hurst +happy newyears +ham madi +gur inder +grin ded +giam bi +gi be +er mah +en yi +en ie +du charme +dis locate +desic cant +dat l +dam ping +da inese +connor army +coachdan mullen +clause witz +cel li +boat right +ble wett +bit ar +birk beck +belitt led +backin black +ay yyyyy +avanti ka +arin ze +aparthe idweek +animal crossing +an berra +allison b +al vida +ail leurs +acec ourse +ab org +aal to +æĻ º +ãĤ¢ ãĥ« +à¸Ħภ§ +à ® +yon hap +wrong doings +william and +west cork +warwick castle +w vc +vi rendra +univ groningen +union ization +under reported +to pd +tiger air +tic o +thing sin +team effort +sydney trains +supple ment +stru an +straf ford +steal in +stalag mites +smoke stacks +sic ure +scatter gories +sc avo +say z +sanjayazad sln +sand more +sag rad +raw kus +ra hel +ra ds +pro fe +pe cor +pal meri +oo ohh +nu aimi +nar vik +n alls +n afc +miz ell +miro ir +minniem ouse +michal ski +mer credi +menstru ationmatters +mctom inay +mcfar land +marine science +mar ston +luci enne +le mm +l ss +l ome +kimso hyun +ke ur +k uper +joon as +jerus ale +j ti +ishi gaki +intere st +ina itis +he si +hau ssmann +go choctaws +gi ese +folk rock +flour noy +fau t +every thin +ever day +eup en +ent endres +el ke +ec ancer +du ker +doppelgän gers +dial up +designated survivor +dela et +darkest dungeon +cogni zance +cityof hamilton +cecili o +cabo chons +ca ed +braz illian +bol ler +boboi boy +bishop scourt +bet ta +ber nama +be beto +b xb +avis itor +aro ad +ak un +ag lecam +af oods +ade ep +ache ampong +aash ish +--- >>> +*: ãĥ»ãĤļ +ðŁĩ¨ðŁĩ¦ . +ðŁ¤£ðŁ¤£ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ðŁ¤£ðŁ¤£ +è ´ +zeet amil +yu ne +wri f +wil ner +whatson antv +war te +vander pumprules +us jn +tr tworld +ting gal +thisi sthelife +sy yc +superblue bloodmoon +sundar pichai +stereophon ic +sql pass +sl so +sis fur +she erin +sharon vanetten +sha adi +sarwat valim +sal al +s walk +ruth men +quiztimemor ning +quit te +pucke red +pre menstrual +post cards +porsch enewsroom +po wel +pin oys +party with +offthe grid +o wain +o esophageal +nu minous +national tequiladay +national filmawards +na stro +my vancouver +mo sport +mas vidal +mar zi +mano el +mal oy +m chi +lock en +lil nas +lang side +key pads +kas u +kam asi +jar ia +jan owicz +ink off +ib mi +hr vy +hierogly ph +guide to +green line +graphi que +grandtheft auto +gor ney +gomoo sego +god rich +go hogs +gan ley +gab en +futuren hs +fare share +el swick +ebr pd +dy scal +dro vers +don kor +domin ions +dilla hunt +dc ps +day trader +cÅĵ ur +cowli shaw +con ed +c zu +bryan bros +brumbies rugby +bleed orange +berwick shire +ay ev +atl ant +at é +at rack +astra khan +ard al +am able +aless andria +agri ppa +ade kun +ad ak +abag nale +" ): +ðŁĶ¥ ðŁĴª +ðŁĵŀ : +è° · +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı@ +zu zana +zo zo +z idan +wy ld +willi ger +wil ms +wheel wright +vol tas +uni das +ty d +twee die +tra ub +tol ar +tiber i +thim bles +thelauren graham +thel one +thatsmy dodge +than ga +tan ey +syl ph +sty x +stan ne +ss bm +sou to +so han +sig erson +shimabu kuro +sh kov +sco smetics +schne iders +ram bler +r vw +pul ly +protein world +pran av +polari ze +phil co +p mg +p ami +op re +op an +of peace +ny senate +nou mea +north co +nac elle +na shi +mygov india +mumb led +mother sbaugh +masa ko +ma thes +m wy +m dg +loi shua +lee jonghyun +knick stape +juli ere +jose fine +jan sch +jamesra hendry +j ng +it amar +i beacon +hot dog +hoo e +hi mal +hert zog +hel plines +hand stands +gr annis +global ised +gab ardine +g weru +fred ricks +fo t +eye hate +ev cen +en eco +en dian +eli ason +electroly tic +el puig +eidol on +ed dings +drin kal +dre ric +dar vin +dani al +dan ser +clutter buck +ci k +che eta +cele br +board masters +bo bol +bi ao +ber te +back britishfarming +baby gift +at tie +ar drey +ann aw +all indiab +aj r +. ðŁİī +ðŁĮ Ĺ +íķĺ ìĿ´ +yan bu +yadv ashem +will acy +ward ley +vine et +ve eder +v tt +usafric abf +tx educhat +traffic crash +todd whitaker +ti dur +thr oneof +thisi shome +taylor momsen +t sports +t jackson +swiss re +surviv ability +sul is +sublux ation +stagn ated +sno g +sk telecom +size well +ship builder +sharks za +sam sam +saint patricksday +sacchar ine +rye ong +runner bliss +rose bay +roger stv +ran x +quoti dian +qip co +pub crawl +produc tiv +pri vee +pre y +pram ila +pra bu +past ner +own voices +oliviach ow +official rezz +nil am +night bird +mo tter +mo tet +mith ril +me guro +mc niven +mau g +mar gy +man music +lou bet +lion sclub +lar ock +l bb +ko caeli +kitt i +kid slit +khamoshi yan +ker messe +kac zynski +jane ane +imogen heap +hol douts +hel oise +gu ria +goka iger +goal mouth +glamour maguk +flower photography +fire station +fern tree +fam es +extracur ricul +eve leigh +electro plating +dup date +dun bar +dubl inohio +do i +dia stolic +den ham +da ang +cthul hu +co don +clean tech +ca haya +c ses +bu ma +bread and +bing crosby +ber ridge +base plate +ball erin +bal fron +asseen in +ashley monroe +aq r +anil kumble +am dry +alo es +allmy children +alad in +adam richman +aap ka +ðŁĻĮðŁĻĮ ðŁĻĮðŁĻĮðŁĻĮ +è© ± +å ¢ +ãĥ¢ ãĥĩ +zo ya +you then +yor kies +y ster +woj cik +while youwere +wel cher +weight lessness +web marketing +wake fulness +vibe magazine +ventrilo quism +utt aran +ur on +transpor ter +tho pe +the ken +the junglebook +th awk +terab ithia +tb in +super storm +stru ly +stellenbosch uni +squ ote +spor um +shon telle +shad rack +servici os +schisto somiasis +sch rade +sau to +reci ation +re discovers +pul leys +plat en +pensac ola +pencil sketch +pah lawan +osucoach meyer +opp er +o eh +mullagh more +mis behaviour +mil dest +mall in +madmax furyroad +mabino gi +loko ja +lic enti +l ru +kkkon colors +ker fuffle +kar thika +joseph us +ith appen +institution alization +ingate stone +iffe rent +idol producer +he iss +happy valley +ham at +h sas +geton mylevel +g cap +ful k +free to +foie gras +fly pal +fc stpauli +end yk +ehl inger +dub v +dou that +doc week +din din +die hard +die bold +di sharmony +dhan raj +deco ders +danny pudi +da ik +collar oy +clean beauty +cin zia +children sla +car share +ca che +busines speople +bt as +br kfst +bor ris +blick ling +bill inge +bell port +be sta +bau com +az c +ar sons +ap ak +anim ism +angkor wat +ang pilipino +ang am +andi ka +albu mart +ðŁĺİ ðŁijĬ +ðŁĮ ĸ +ðŁ¤ª ðŁ¤ªðŁ¤ª +ìĹIJ íĶĦ +京 éĥ +ت Ùħ +á r +yani v +wind jammer +wil ayat +week uk +us movie +un recognised +tu cuman +toi mumbai +til ford +thom asians +the ys +the forum +tetra hedron +tat weets +sunny slope +sub stratum +su bba +stubhu bcenter +stro mal +strengthen er +star ched +sri jit +sig fox +shrew sweb +show boating +scry pt +sag af +rox anna +ri ft +re ju +puertor ican +ps lon +pro meth +pin ball +pend se +pat tim +outw ards +ol atun +of ir +obl ation +nu ku +ner fed +naughty america +n ä +mw ana +mv rp +miss this +merchandis ers +mascar ol +magen ta +m sha +lu sted +lou ps +life crisis +ley endecker +levan ter +les miz +le tha +le brock +lc bern +lcbern alo +l icia +ke ko +justin baldoni +ju sco +joe bob +jeff coat +intere ss +inter bike +im no +id hu +hh d +hetero sexuals +hesit ancy +head way +guillo che +go wa +gag genau +free app +fon z +file system +fe stin +f mw +eu st +escal ope +equal sfreedom +enjoy illinois +ef fin +du sen +dro pin +drew ry +dis order +destabili zation +de sain +daysuntil qatar +daily quotes +custome rengagement +cic cio +buder im +book con +bay h +ax im +att enders +ak su +ak chod +aim es +aero gel +! ðŁİĤ +ðŁİī ðŁĴĸ +ðŁ¤· ðŁı¾âĢįâĻĤï¸ı +ìķĦìĿ´ ëĵ¤ +구구 ëĭ¨ +á´ ľ +z are +wurz els +wsu pullman +wing sof +whyi march +wan de +vill an +un reality +tru sh +trop med +treasure hunt +the thing +the acc +tetra zzini +ter ris +team titleist +tac eae +ta here +synchron izing +swoo plife +strand bookstore +steril ised +steel yard +star set +st sci +spy master +spring forward +sp ils +soulful house +social responsibility +sme ar +siss ons +sid grauman +sibb ald +shin wari +rsv ps +rough guides +roh tang +riems dyk +resin ous +rehabil itative +regurgit ate +regal movies +rainbow laces +ra ffel +pur fleet +princess diana +power systems +post menopausal +pope ye +pere grin +pan isse +pall bearer +ober land +ob st +new biggin +music scene +mun oz +morning ton +mish ere +metho dist +mel ani +make som +mac iel +m laden +lux ton +lmm fao +lie big +leeu w +ko hen +kad hi +jovan ovic +john piper +jeppe sen +it ak +io ve +in dre +huang shan +hans zimmer +han afi +hagg ai +ha rel +gri mmy +gra us +give th +gen ove +ge saffel +gar n +functional medicine +fri ede +framer ate +fo yers +felipe melo +fault line +faul ted +encryp ting +eby zio +devel ope +deser ting +deni alism +den si +deep dream +dan the +dag ny +cyno sure +cherry ville +char line +cancer bats +bur un +bram lett +boroon dara +booth by +ber gg +ban ken +bal int +ayo shi +attune ment +ar lfc +ant witter +annas ophia +acry lamide +abc de +aas l +[ ðŁĵ·] +ðŁĵ± # +èĭ±èª ŀ +zy gote +ze ts +yed chat +ye sler +yay ay +w sk +vaudre uil +vaness amerrell +v magazine +usch o +up gradation +turt len +ton ly +thir deye +team english +te very +syco ph +stri ped +staf fan +smo cks +sketch book +shu ker +sch nur +rob bert +ro ft +reve rently +refr acting +recer tified +ra yer +py romania +pix i +pente cost +parod ying +pan ka +omidy ar +offici albull +officialbull srl +no kian +myas thenia +mur g +mu stre +miti gates +minne ola +mend onca +mem ling +mc george +mb en +marke aton +le ches +laur inaitis +la ak +kol lar +kirk ley +ki xx +kelly ville +iwan rheon +institu ting +im brettdalton +hy pom +home studio +gn awed +forum nyc +fli ghted +flec ks +fl anger +fab four +edu ar +durham college +dom mett +digni dad +digital singlemarket +cultiv ators +cu zz +crewd son +creative review +cole brook +cl é +cel la +ce to +cas ares +capacit ance +bru ton +bla sco +bla sberg +big show +berg son +bel don +bblo fficial +bacteriopha ge +aqu id +anti pasti +amp oules +ag ym +afl crow +adden brooke +> "@ +ðŁĶµ âļª +åħ¥èį · +ت س +zal man +y une +xi bal +wood brook +wo burn +web isodes +war g +v dub +unci ations +twilight sparkle +troll tunga +tele sur +sy ston +studi es +stro mer +stral sund +stpat sfc +stand o +soviet union +snow falls +sle iman +slan ting +sixword story +sh oneys +sarcast ic +ruba diri +road nats +regi o +ray u +promo tocross +prati que +po prock +pear man +pe go +paul pierce +param oun +p nh +ou as +oli mar +odel rosario +ob ay +o san +nauti yal +mo ze +mau ch +m ssa +love songs +les que +lanca strian +kun ai +key t +kar amel +k ne +jonah ray +jo t +jim al +j kd +info graphie +if only +iden hout +huub design +humb lest +high boy +gul liver +gros mont +golds berry +go tr +girl probz +fro thing +fri en +floss moor +fal tered +explo sively +exemp ts +ex itos +es en +erin dale +enr anta +elstern wick +eleven se +elec tor +dig deep +de brah +david muir +dav alos +d cn +cow ens +confe ss +con traflow +chittor garh +chiro po +chima era +cer veris +cani bus +cal gon +cabare t +brandre th +bicker staff +ball i +bac cano +ati fs +atel ateshow +at resia +assn chat +anglo gold +andalu z +an kari +amdry zen +amadeus itgroup +alv ador +accred iting +ðŁı ® +ãĤ¿ ãĤ¤ +âľ ĵ +à¹ĥ à¸Ļ +ار ÛĮ +¥ o +yo wie +yan et +world wi +wigg ler +war lingham +w iry +vande mataram +vaish navi +urva shira +uch us +tour ne +toot sies +thess aly +the kenny +tgi fridays +tail piece +symbo list +suppor tin +sub sp +sp ons +sim kins +shar mon +sf wa +sar apascoe +s march +rev ans +reid hoffman +psy c +poison ings +phospholi pid +pain lessly +pa zzi +oto ño +orl pride +om il +oc cam +nur kic +ns dc +ni mue +ne ith +nah ant +mon ito +mom ina +mmb ht +missjess wright +minchin hampton +metro park +me trix +mau ra +mar ras +mani ax +mand uka +loin cloth +liber tas +lessi smore +lacer ations +kl are +kingofthe monsters +k rock +j lr +j jab +iti zens +in scribe +house martins +hesit ates +haydock races +hay makers +gra ils +glas vegas +gha uri +g pe +fran sen +for tomorrow +fle che +f nl +es war +encamp ments +ekstra klasa +eco logic +dor rance +dom ici +devi ent +cund all +collie buddz +co sin +circul ars +christian bale +cell therapy +carnau ba +capri les +cai ley +buff ington +boo dles +bo pe +biz rt +bir on +big dataanalytics +bey az +be hera +bal ne +arnau lt +apprehen sions +ani ela +ak kians +agh y +aa i +! ðŁį» +ðŁĻĮðŁı» ðŁĻĮðŁı» +ðŁij©âĢį ðŁĶ¬ +ðŁı ĭ +о ÑģÑĤ +youn gen +x one +willo spre +willospre ay +wil da +weareall harry +wahi awa +vel á +var una +van helsing +union chapel +un bound +tr ach +thread gill +the sharksza +the fish +ten chu +tellu ride +taver ner +tand ridge +ta ren +t los +t ente +stur key +steve case +sru thi +spiritu als +so lex +silver mine +sch wager +sarfar aza +redcros scanada +ra sch +py i +pru ri +pro lo +pepp ering +penguin awarenessday +ostraci zed +of ia +occupy dc +nh on +na sta +n cic +mo ty +mid america +michelrou xjr +mh ra +mam tay +maha shivaratri +madison beer +m zuzu +lul ling +lore t +lof africa +line as +ler on +lennox lewis +kri z +kle z +kh ancats +k outa +jurassicworld fallenkingdom +j rp +iv ka +iter ary +is lah +ino ids +imp ong +id wal +hoch uli +he intz +har ford +hae jin +h medabad +geith ner +gat ers +gam bled +fox hill +five star +emerson barrett +ell ena +ek ins +dj quik +confla gration +commu tative +cinemain my +ches ney +chen once +cer vez +celo sia +cas is +butt resses +birthday present +back down +as phal +ang aa +ambro ise +amandashi res +alpha phi +adam antly +! ðŁIJ¾ +ðŁĩ©ðŁĩ °: +ا٠Ĥ +â tre +zlat ko +you ville +yach tsman +y ic +xan the +whizz ing +whisen hunt +when they +wap ato +vicuni wgtn +ure sh +tul se +theo logically +theav club +swoo pe +swee tromance +star musicph +socialmedi atips +sk ul +sic b +shan aya +sequ entially +sekar ang +secretary meity +sat ana +santi bernardez +sa hay +s vic +rt j +rout t +rot j +ro ge +por lock +pis atower +pin kel +pel ota +pau se +outdoor photomag +on time +old trafford +o ore +no isier +national watermelonday +nan ai +movie twit +motor plex +mor wen +mon ceau +mom mys +milli metres +mat ina +magic man +mag tang +ly cam +love eeeee +llan do +little mermaid +lang ga +keith haring +katy b +joaquin castrotx +jo casta +jacqueline fernandez +jackson ms +j bj +istandwith ahmed +ir reverence +in shorts +hy er +hv n +huic hol +grizz nation +gra vy +gg v +gesaffel stein +fren chi +fir ings +f de +ent wine +elimin ation +ed gley +ec dc +dra gger +do sto +dis illusion +dal in +da we +cul zean +cro ats +contra bass +con ex +cap rica +bur zum +bridg ton +bri ant +brau tigan +bou bou +beau soleil +be ate +bat th +bag ong +awh hh +as ae +andy bell +amphi bia +amaz ov +alfie boe +ðŁįĬ ðŁįĬ +à´ ķ +ø ya +wu or +wonder bra +well com +wc zyk +w elive +ver ba +uniteand conquer +uni oslo +tz comics +travel india +trad cat +tra kai +to hru +tik kun +the girls +the far +tele play +team unity +te mora +taraw era +tan field +swachhbharat mission +sw aging +st ecker +ss ave +spine farm +sne yd +sn sh +sk atal +scu zz +sch lock +sb swinner +sa ale +rural crime +river dale +rel pool +ra jut +pre selection +pantal eo +nun zio +neutr alizes +nav otas +na jah +mu de +mtn l +moun a +mon agas +mind the +michaele aston +lu cus +lol lo +lo ftheday +lee h +le ta +laun dries +l losa +ktn lifeandstyle +kro k +kp fa +ko tigobba +kkw beauty +kappa sigma +iti ate +ing us +ice prince +hu mic +haywar dgallery +ha good +gre sham +gran town +goo die +glaz ersout +general news +gen gar +gell ert +flying dog +fire bird +far ri +fa hm +ey en +er hard +epile p +emo tor +dev araj +dev akshi +de colonize +couch base +coil over +cine mam +chiz uru +cellu litis +calom baris +bu fo +bre it +bill rancic +awa g +assemb lages +archam bault +ak ola +agne se +ach ines +ðŁĺľ âĿ¤ï¸ı +ðŁĵ¸ - +Ñĩ аÑģ +zet land +yel ps +wak en +vel den +vall ée +us lims +uni kent +tizi ana +thisi sco +them icky +theat ro +the frank +tam inas +ss am +sk now +sh aki +sed atives +sal ai +s rush +robin thicke +re organise +re nova +raz avi +rambo donkeykong +r anny +que ene +quag ga +power pc +po sie +peyton manning +pal frey +ori k +ok tib +o dess +nipa win +neutr alise +my cin +mesti zo +maz andaran +man ston +mamtay patnaik +lun da +lady ship +ko hi +ko chan +kav y +ka hane +jud kins +joo won +jak bar +ja si +inn keepers +in ness +hi vos +hal k +hackath ons +gue strooms +gu mmo +gas light +gal en +g bbf +future day +frick ley +flipk art +fi ef +fcgo aofficial +es fahan +edge fest +ed policy +eccle sall +earth moving +din da +diero ten +darby shire +dam os +credit score +col u +cic le +che main +bul o +bul lett +bron zy +bio m +beta wi +ben icia +bellamy young +bb bots +ball ons +baarba arde +at na +ar bury +ant age +anit adon +am cu +allu du +abil ty +ab ach +?! ?!! +ðŁijī ðŁı¾ +âĸł âĸł +âģ£âģ£ âģ£âģ£ +á´ ´ +z inho +yasi elpuig +wolfs bane +wma onedirection +wand sworth +video drome +vas ool +union square +un compromised +un adilla +uconn mbb +to shin +thatgame company +th air +tal dÃŃ +sugar land +star tv +st cuth +spi ra +spe zial +small hd +sho reline +se dm +sa dek +ross ella +ros aparks +regen bogen +president kovind +pre ico +polit icos +par ds +pa wh +oregon state +oppre sses +old pic +of fe +nu u +nr c +nic d +nay ak +monop oli +mill and +metro land +men il +mcpar land +mckend ry +matthew j +masi si +mar ge +mal acan +macmillan coffeemorning +ma thai +lü beck +lifeok tv +lie bling +le gio +laspal mas +kle mmer +ker st +kar ki +ju bal +john b +jenny packham +j ü +j oux +iv w +inti mate +incant ations +humph ry +hu mored +hu ggers +hagg adah +gra do +goo dre +fin min +ewe cisme +el ady +du tra +down south +dog spotting +dish water +cust is +cou peville +coono or +constan cio +coccy x +cam ira +bron zing +bran dish +borg warner +bom berg +blue mont +blue friday +big ness +be mine +bbvacom pass +baf tac +ash can +and care +altr arunning +allindiab akchod +acar ter +. )" +ðŁĶ ĵ +ðŁĮ¤ ï¸ı +ìĿ ¸ë +yvon near +yehrish taky +war randy +vive andalucia +ver ulam +valent ini +vale ting +ur be +tragically hip +tr ouncing +to tara +that ss +thali domide +ter ol +te ki +tam m +sto c +sti jn +sp rees +sou maya +sm oment +sloven e +science ctr +rob itu +robitu ssin +ritu ximab +ris borough +rein ert +rames waram +qui ero +quebec ers +pe ca +p ach +or man +onward lu +online gaming +old skool +no worries +my son +mor itz +mo li +mir zap +manojtiwar imp +mah out +le von +laps ley +krist tps +kor f +kom mer +kno ch +kan aya +kab u +jin ho +ist itu +invari ant +ineffec tual +indy indians +i bex +hon aker +high lin +hes burgh +hear tedness +h winkler +h fr +go dd +foo kin +ey ards +engli shri +en sayo +disin i +deser tisland +deb it +de camped +cumb res +courteney cox +cos kie +cordi ale +consul tancies +church goers +cf z +centr alize +cap as +canthelp falling +cam mell +bi vens +bbc berkshire +bb u +barnab y +ban te +av ore +astro samantha +asf andyar +arti an +army navy +apo phis +amund son +alcor con +ain tv +african fashion +afl tiger +ab oud +a this +# ? +ðŁĺİ ðŁĻĮ +ðŁĺĤ ðŁIJ¶ +ðŁĮ Ķ +ì¹ ĺ +è ĥ +ز ÙĬ +writ ting +wakeup call +wainf leet +verizon fios +undu latus +tro ad +theme park +taylor swif +tat um +t che +sy am +swo boda +sw bts +steel ers +so se +sm le +sk icks +side as +schie hallion +sar copen +sam hsa +ri ev +repaire rs +rat zenberger +raise droyal +quent intar +qual ys +pu stu +proscen ium +pressu rec +pj bowles +pang ako +pa peete +oãĦ ¥o +op ment +olo f +ok ur +nwalest weetsuk +no xu +ncl b +montju ic +milli kan +mikkeller beer +men tof +mass eria +marsh on +mar aton +male gaon +mahan adi +lyo to +lot ti +litu ation +life form +lee hi +leak ages +ku lu +knight pride +ke dron +jun it +julian clary +jud son +instig ators +hu gu +hot wife +hope college +home work +he ge +hc ska +gur dji +gargan ey +g tbicycles +firesi de +fides z +farn ham +eu ijin +erec ts +emb olden +em enike +dou dna +don di +disfru tando +dho far +dev dutt +dd orp +dani ka +cy bill +cson ka +corn dog +concer ned +chee ta +cc fest +c jb +by am +bux us +bre anne +brad don +bossi er +blue water +barre iro +b politics +auburn tigers +arden ne +ar ven +app ened +am om +aivo arm +/ /# +ðŁĴĭðŁĴĭ ðŁĴĭðŁĴĭ +ðŁij©âĢį âĿ¤ï¸ıâĢį +ðŁ¤¦ ðŁı¼âĢįâĻĢï¸ı +çĿ Ģ +ãģ ® +w sav +vegas weather +upanish ads +uniof brighton +under handed +un j +too wong +tath agata +sympathi zes +sudan massacre +stop tober +ste yn +sol or +shi moga +shark science +sen ya +sen rehmanmalik +sch wi +saqib saleem +sad ams +s vil +romp in +risk on +repri sed +rele m +re possession +re buil +rapi des +r ri +pir ouettes +oxic lean +open ai +older people +oaken shield +now icki +note pads +north fleet +nor ville +nawalel zoghbi +n mireland +mott ola +mind scape +min ter +milit ari +men kes +lympho cytes +lsuf ball +lesar cs +kow ens +kh ir +keaton stromberg +ke tsu +ke ay +jar ano +istandwith israel +iron i +iceprince zamani +ice field +hil ma +hammer sley +gay athri +gabrielle doug +g leave +fu ssell +fro mc +free ebook +f sh +exofan art +epi thet +ei dos +discipline equalsfreedom +democratshate america +defence day +curt smith +ctc mediaboy +crew men +con on +colorado live +ci arab +chem society +car rero +bur rough +boo ke +bobro ss +bj w +benedic ta +beat nuts +be ed +bank ston +bad gering +art trail +apo stol +anky losing +angel ov +an heuser +alfaj iri +agn ello +abim bola +aa ve +? ¿? +.. ?" +ðŁĺ³ ðŁĺ± +ðŁij° ðŁı» +ðŁIJ§ ðŁIJ§ +ðŁĮ¶ @ +à¹Ĥ à¸Ń +zak ariya +yu yu +ys r +ye dder +x pt +wich man +wcc w +vr la +tr ès +tom kaulitz +theceltic manor +tam baram +sun way +sp j +skatal ites +shak o +sean an +s oooooooooo +ro sca +q un +punjabi rooh +peru vian +peri vale +pe ameal +pap on +pa quito +one towatch +nic ols +nh lers +national tacoday +nate berkus +my sjaday +modern life +mis kat +mar mol +manzan ar +mahi dol +london birds +liz mcclarnon +kochadaii yaan +king swear +kin ta +kar y +jami em +j crossover +it smi +islam ujeres +instra gram +hems well +healthye ats +hav licek +haile mariam +ha itham +h kust +gun an +gul man +grze gorz +grena dian +grav ell +gli ac +glam ori +gentri fying +g illa +forge din +for theride +fe thul +f ttp +ez ri +explo rec +epilepsy awareness +ekkle sia +drexel univ +doni phan +don nee +do onan +digis coping +de sau +daisy ridley +da j +culture days +cul ley +compreh ended +cho g +chic hay +cel les +ce devita +ca ver +bun ited +bun cee +br ice +benne teau +ben sley +bb k +bal dur +aw aka +arri ola +ap ko +ane ela +andrew schulz +allahu akbar +!! ~ +çĮ « +âĺº / +youtube spac +your quote +yn l +yag nik +xeno gears +x mm +where of +u vas +u cn +txh sfb +tw ila +trum ple +ter res +tempor ally +tani sha +sten cil +stand withrand +sque aler +sla vs +si pi +ser vic +sel ah +sebasti andan +sebastiandan zig +se tya +schu ur +ryan vaughan +russ ert +ru sk +ro derick +raven symone +rac is +r bs +provoc atively +phor aone +pet ted +part time +outh florida +ob ong +nic orette +next chapter +naraco orte +my burgh +musc ling +mur ine +movie tv +mirac leman +mi b +mcpart land +martin j +mart one +mal aka +makar ska +lu kman +louise redknapp +lor ong +livel iness +ld pe +lach lan +kud zi +khar an +jo bbing +jas par +hog town +heriotwat tuni +hazle hurst +h ca +grosse st +gn oni +el niño +ek afka +e ja +du guay +dr atch +dirt car +del val +deathwish inc +de coupled +co hl +chen yuk +cha eng +bri slington +blood sugar +bezu idenhout +ball rooms +bal ak +at cc +as kins +ap mc +alphon stourism +alham dullilah +al don +akiz aka +ak ville +agu ez +ad ae +ac inemas +a hahahahahaha +ðŁĺĬ ðŁijı +îIJ ķ +ãĤ»ãĥ¼ãĥ©ãĥ¼ãĥł ãĥ¼ãĥ³ +ãĢij ãĢIJ +âŀĸâŀĸ âŀĸ +ภ¨ +zi yech +zi bah +yogate acher +yi pee +wärtsil ä +would you +wo za +weather field +wa edu +v ith +transduc ers +the silent +tait ung +sylvan as +spic inemas +sne e +sky dive +sbswinner shour +sad ashi +robin tunney +ren yc +re vul +ramire z +quit aine +queen rania +qu har +pro era +pro creation +pop tart +plo tt +p agu +ono dera +nursing school +ne men +natalie grant +na gh +mun ising +mu lino +mit so +mind fully +mc griddles +mc feely +mart lesham +mac ou +m spaint +lun cheon +lovel orn +lin ky +lat tices +laf ite +kashmir bleeds +joesp ub +jenni woww +j era +isur ppo +ine aux +in nately +ig ley +heisman trophy +hanni gram +hamble n +grac ec +geo science +gend ron +g pc +fu d +form alized +fiendish ly +fall game +evry one +et yuk +enye ama +discol our +detroit tigers +custo des +commerci alizing +clau rak +chel i +cer am +cap ability +cam ier +bra sen +bio process +beauty queen +bb claurak +bargh outi +bal amory +av ra +ar hi +ano dyne +afri kaner +a sexuality +é ns +win elife +white friars +wan z +wall ner +w enty +victori aday +v ris +ur ania +under achiever +ultr atech +twer ton +turk cell +tro th +tne du +ti ote +taminas nuka +tab ilis +sy rin +sum pah +streetfighter v +stone arena +star cross +sma shin +ske ws +seren ata +scrutin izing +sand tats +ro cc +ric ca +ri kara +reviv alists +rannvijay singha +r bm +portu mna +por s +phin sup +pen ni +out shot +oju kwu +no hotch +no cs +niz wa +n cla +my lor +music notes +motor craft +mon chele +mit am +mission possible +metaph oric +masi yiwa +manfro muncle +mackin ac +longboard stuff +len ox +le blond +le ask +lap kus +ku fuor +ku ai +knuckle heads +kis wahili +jim mer +jessica simpson +jar ama +j day +hlat ahai +hell scape +health fdn +haus mann +hacksaw ridge +h enty +gurdji eff +gentle woman +geno typing +game trailers +gal vin +freak azoid +fi ware +fcc ps +fashion trends +f anni +esp r +elo ves +early voting +ds q +drain pipe +di fy +daniel platzman +cott man +co tuit +canal plus +butterfly fish +bot elho +blackgirl nerds +bi man +betsy devo +best meow +bbc wiltshire +avan am +aspi red +ar yeh +ap mso +aor aki +andreab ocelli +anah ita +ake hlatahai +ai weiwei +agra bah +ag ada +affili atelink +ðŁ¤ ½ +å Ĥ +ãĤŃ ãĥª +ม à¸ŀ +Ø´ Ûģ +zan ussi +ys avage +y anna +wf sullivan +weare messi +wdy tya +wat usi +visit rwanda +vanilla ice +u rious +turtle beach +tu eindhoven +trique tra +tri duum +tipsfor new +thetommy dreamer +tema gami +svi zzera +suppo sing +step well +ste gall +sound box +socon sports +sho whouse +sh urt +sel fridge +sche els +rob be +ren derman +re ff +ray field +rasp bian +pu u +pressuri sed +pran it +piggy backing +par vo +pai stenation +one es +nivin pauly +nau voo +n ti +n sh +my calvins +mkt gnation +min ns +mi af +meg apolis +madawas ka +ma irie +lit mag +le them +kier ra +key akizaka +ke ila +kayak fishing +kar gentina +jä rvi +jeff mauro +jacoby shaddix +j itter +ish ta +in case +homos assa +hitch in +hi as +hat tie +gh n +ful well +foun dress +for newindia +entoura ge +embar goed +em era +ely sian +egg sy +ed wyn +e mina +dv all +disintegr ates +din na +dg classics +del toid +ded man +davi dicke +dar am +danc zuk +cur to +cro i +critic schoiceawards +cour onne +confu cian +ci ba +chi oma +cent in +camel bak +cac o +c ni +by bike +bi ii +auburn u +aru z +arro whead +ar ace +angar dens +alle gre +aaaa ay +ðŁĺĦ ðŁİī +ðŁ¤ĶðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ +ëĮĢ íľĺ +âļł âļł +âĻ¥ âĺĢâĻ¥ +аР¼ +zo sa +yy ates +youtu betv +young jeezy +you the +yo shimi +y iss +xseed games +wel che +vis y +vermin tide +ty outh +tro on +tou sa +toa sties +to bu +the forks +tap low +tab ular +sur ma +stanley cup +sta o +sprezz atura +spit toon +so bra +sk ane +shou rie +shoebury ness +shal lower +send help +scrutin ise +sanctuary asia +rsac onference +retic ent +red nose +re marketing +ragg amuffin +proudof you +pres age +pot ters +phe be +pescat ore +paradise papers +orient alis +oo die +non no +nin na +musi ka +mother lode +mire les +minu it +mi des +mf h +merri on +mediev alist +masterpie cepbs +marie mont +maki as +lu ssier +little girl +li des +lan than +krispy kre +kle infeld +kiyom izu +kah in +iy ad +it awamba +iso cial +in patients +ime che +ig li +hol bert +ho ty +hi jama +hi iiii +heresi es +head hunting +hand sof +hag strom +gre z +geck o +fur tive +free scale +fre ey +fei ffer +fal le +f dle +ever man +euro bike +eti ology +et canada +ep son +engle field +el ook +din gos +cul hane +cro fting +cour tier +community spirit +co fer +christ fic +chim er +canar sie +camp day +cali bers +c gw +bu mbu +boer boel +bir nam +bi enne +bern heim +beau sejour +b appi +azi kiwe +avoce ts +as mode +arch diocesan +anu bhav +ak ri +aha fo +agar ci +ad os +acar los +ac em +# [ +ðŁį» ðŁįº +âŀĸâŀĸâŀĸâŀĸâŀĸâŀĸâŀĸâŀĸ âŀĸâŀĸâŀĸâŀĸ +yoga inspiration +x z +wer sching +visual ised +visitu ganda +victori aave +ve sel +un troubled +ump teenth +u acc +trolley bus +transp ire +tradcat knight +tr indade +the moon +td bank +tai sho +stro bo +stra ka +st g +southern stars +so quel +sky scape +shi ina +shar jeel +sha ped +sent i +sam csmusic +s loo +rumin ations +rt ve +rol f +ro ding +ric ke +rhi zom +revolution ist +recogn itions +real pigfarming +rble ipzig +r tb +pyro lysis +pv z +pre pubescent +power puff +pic hler +pd w +parti t +paralle lism +pam an +ob eng +nwm waypolice +no watch +no guera +nicole byer +ncb wa +nam it +mu res +mot son +mik ro +mi dy +mersey travel +man den +lee hom +kvo awx +kun try +kul am +kingsme adow +ki ang +ker ala +jus reign +j asons +is chae +imper atives +ij at +iced coffee +huck le +homecoo ked +his ss +herald leader +glend enning +gey sir +gener ator +gar bi +gal chenyuk +ga thon +flori dian +fel ker +f tb +estre ll +er rington +end ine +elli avram +ec khar +duchessof sussex +dry cleaning +dire tta +del tab +death stranding +de gnan +dan as +dalla sschools +ctb b +constra ining +coin drop +chic ho +cfl s +catson twitter +carol decker +care quality +call sthe +c cre +bubble uk +bre c +bin ondo +bel inelli +basel itz +baal bek +ast ars +apach ekafka +ann woll +allo yed +ale sha +ack ay +abhay deol +.... :) +ðŁĴĻ ðŁIJ¾ +ðŁĴķ ) +ðŁĮ ĩ +åĴ ² +à° ° +ö calan +ziller tal +zaid hamid +z dar +yu kie +win kelman +william zabka +wei qi +van ç +val let +u tau +twitch retweets +travel awards +trans dayof +tiru pathi +tan felix +t seva +suzu miya +stormb re +stormb lood +sto h +stan thorpe +sp ang +sou lar +snow blind +sira gusa +sign boards +shir at +she said +seic aday +sd lplive +scher r +sapp eal +russ y +rol leston +rod inger +rik ke +re gin +re enter +r nh +r ari +r allo +pon oka +politic snation +ober to +obam a +noxu bee +new scenter +near sighted +nai vely +mis leads +micro blog +manbear pig +ma dia +lo kal +legion ary +lawrence town +kir sti +kalam unda +jic ha +jh us +jennab ush +inciner ate +immuno therapies +hi ff +he wwo +har twood +hage man +gre ce +goo fy +glack ens +gin ghs +geelong addy +games don +fren sham +et ce +ent rain +em elianenko +eastern ghouta +dream ing +cosm in +com ence +clough jordan +cho zen +chicken foot +certi fies +cambridge analytica +brook house +bow sher +billyray cyrus +barrac lough +au han +ard ingen +ap ron +ann am +anj ir +anim alier +amer chemsociety +al gar +ah la +aff ney +adore delano +adam ferrara +about you +a eta +! ðŁ¤© +« à¸Ļ +with holds +wis fb +visitt ampabay +vermon ter +twit z +tw ays +turn bc +ttur tles +tt t +troy ave +tro ya +tom delonge +thestar kenya +thereal daytime +than et +te mberg +son orous +sok onews +sobot ka +sla sher +sf as +service women +sent sov +roar lions +ro dos +rith vik +reef ers +re zone +re tread +re gev +ranch itti +prin cel +pr j +porcel ain +peop lenow +pent ine +pe conic +oun try +open door +o town +nusrat chirps +nsw waratahs +nom ex +music tech +mo ville +mir tz +mh ingis +mass o +maris cal +man ero +magand ang +mac mahon +ma ah +lt museum +lolo lolololol +lin dow +le pa +lappe enranta +lanc elo +ko bi +kin ne +katy turnbc +ka inan +k yser +jessic ad +jer se +ix s +immun ity +hye sung +hut chin +hu tu +hel imed +hay ati +har n +gir paten +ged don +fro ot +en antio +e strange +diver ge +dis membering +dha hran +desk og +de wdney +cre gg +cor ry +conceiv ably +commonwealth sec +coal inga +cinde rella +chi pp +champag nes +cal man +c shs +burge s +bre don +bran ly +bombay sunshine +bli xen +blanch flower +ben cy +bal da +b ated +ax um +auchin leck +ar cing +appare ls +apho bic +aor us +antic om +annies land +am wa +al labout +actor dougjones +a high +ðŁĻĭ ðŁı½ +ðŁĺĬ ðŁijį +âĻ¥ï¸ıâĻ¥ï¸ı âĻ¥ï¸ıâĻ¥ï¸ı +ઠĤ +whi ppets +western cape +wazz u +wate rer +vigil antism +ven go +vasude van +vap elyfe +v lf +usp icious +unionchapel uk +un quiet +tre forest +tothe future +the train +tain ment +su raksha +snit zel +shaku hachi +sam pan +safe t +s frs +rh in +resc ape +recor dist +quarri ed +pom pa +pol tava +photo chromic +philippine star +pan niers +p fo +op ole +only the +omor phs +om id +o valle +not ill +not arized +ni ku +ne wi +nbad league +nas ag +n ju +mor ire +mohawk college +mo reco +max ims +man awa +madal ena +mad hur +ma pple +m jj +m cap +leather goods +kry pto +kou ign +koo yong +kil cullen +ke aly +kc ca +kal impong +ju el +john no +ittake savillage +iti k +it stime +ing res +ic and +ho tei +hen rich +hato ful +guerri eri +golf chat +golden hobiday +g crf +fronten acs +flor ina +ers dorf +emanci pator +do dos +dn rr +dal hart +courtau ld +counterfe iters +cle v +cendr illon +ce ano +cade te +butler mbb +bru hhh +bla kley +bi valves +bas cule +bal lista +b wal +army team +apprehen ding +antho logy +ala jones +ai ha +" = +ðŁĺ¬ ðŁĺĤ +ðŁĩ²ðŁĩ ° +îģ ĸ +ãĥ¼ ãĤ¢ +âģ© ) +ÑĩаÑģ Ñĭ +zen as +ys man +yer ry +wy lambrewery +whib ley +wey land +wawa see +wah ls +w valderrama +vt beer +traxx as +themicky dolenz +the weekend +te mas +swing ing +style guide +stop es +spotthe shuttle +sn stuff +sm ilo +shaw in +scrutin ised +screw tape +schom berg +sch or +sav anna +sa hel +s derot +rin ce +rene auberjonois +rei mann +raj endran +r nd +pru e +pre sta +pis ci +peli gro +p ct +ou sts +oriz aba +open water +onto pp +noise makers +no kian +national ised +moon stones +mil one +middles boro +mi sic +mar ve +manis calco +man alive +machiavelli an +lo blolly +likefor likes +lambeau field +la blab +kn aus +kasabian hq +jo vana +isthis reallife +ini ki +hel ig +hae user +gre yer +ga ethje +for bearance +fly high +fit show +fin ley +f ng +ex ito +eith ne +dul ly +dul cet +dracaen awines +doo fen +do dder +digital spy +devin der +desi ree +dejavu love +def min +cy hi +cigar illo +chun soft +chitec t +chir lane +bur bank +bo ite +blue print +blaze pizza +bir doftheday +bendel acreme +ba ÅŁ +az navour +ay inde +asian food +as aad +archa ea +anton yms +ang ham +an ett +aldub dejavulove +absor bable +a dism +) ). +ç ® +Î ¸ +é tienne +z akes +ys g +you uuuuuu +yor n +ye eun +xi umin +xen osaga +william son +wheat stone +wex po +wdy wt +wall street +wa aaa +vo tel +tric orn +tof ed +the angel +swa thed +sulph ur +stormb rian +south kensington +sout doors +segui mos +science and +school safety +sb last +saty endar +satyendar jain +sar al +ruru madrid +ru lez +rev jjackson +rei der +re packaging +r music +quentintar antino +pu lak +pop con +pati dar +pam ore +pakv sa +paddle fish +om nis +ohiostate hoops +of all +oakland raiders +nu on +normal ising +nin an +ni az +new life +naac pi +muzz les +mu min +mo pani +miy ano +minim ised +me ji +mccly mont +mar illa +lu gg +lou dand +lleg ar +lets dance +lar aine +la sports +kon ami +kel kar +kan sa +kal ym +isle ta +islamic finance +intra operative +hope solo +hog ans +ho ic +hi z +henry rollins +hell bender +gur bani +gun ga +geis inger +garban zo +free bobiwine +fo ci +fen se +evo te +eucaly pt +er yn +er skin +el makias +de standaard +croy als +cow slip +commer ical +citadel le +chri sley +chihay af +castle milk +car drona +cap ron +broad stone +bri anger +behind bars +aur on +ath saust +anjan aom +anae mic +ambu shes +*____ * +ðŁĺĺ ðŁĺī +ðŁĴĭ ðŁĺį +ม า +wi ba +vyach eslav +vel ha +usl pro +tre go +the ating +team sideline +sun yani +summer ton +social business +sloven ia +shim za +roosen daal +ro gal +rivalry week +repet itively +redbul lair +re investing +re engineering +ragn bone +radio graph +radio day +racer back +pon ferrada +pic ta +philadelphia eagles +per les +pent ameter +pan tie +ou ttv +ok leg +ok at +oil seeds +noi seymusic +no ths +nieder sachsen +negli gee +nau seated +n mh +musk ox +mukun dan +mud larking +monu mentally +monte sano +mis cavi +mind bender +mi ele +mel en +mc fe +manmohan singh +make lele +m tor +lilnas x +li etz +lanthi mos +keem star +kan aa +k ander +jimmy barnes +intre sting +high point +helpin gothers +heart ly +guj rati +gu rashi +gi zzi +ga elle +frontpage stoday +fra y +fool hardy +foo ts +fly withus +fi ume +fathersday gifts +fabi ani +ek berg +dz ong +dram ani +digital diplomacy +del ite +community matters +channel seed +cbc p +bus boy +boru ssi +bo band +blogger bees +beam line +balla gh +balbo apark +aw adhi +as ntm +adv a +] âĢ¦ +( + +æĹ ħ +áµ ĩ +á´ µ +Õ ¡ +zig go +z ta +womenin history +what ttt +web server +vla ardingen +ve eck +upper classman +uni sphere +tu zla +tu aran +trul lo +tor me +the sky +th kapoor +team bc +tam ago +stin do +ster mann +stan wick +spee dier +sol tan +sn are +siddhan thkapoor +se kai +sd news +sche ffer +scan avino +sansk riti +rud beckia +ru fa +ro barts +radiof ree +ra zing +pur wak +psychop harmac +pre recorded +pou illy +plat formers +per ino +parok ya +park stone +park jimin +param aribo +p nut +oxid ase +not today +next step +new week +na tha +mit ri +mi ha +mer lino +mehrang arh +mari ama +ma sab +m ative +laur in +lam kin +kor yo +kor sakov +kor bin +kem pe +kei go +keep texas +ka aya +jennabush hager +jangh yuk +individu alistic +il sen +il a +hor lick +hongkong protests +hill ery +happy dussehra +gur us +god hra +gli m +gle aners +geor gen +gav ilan +fre ars +for bade +flo es +fidd ly +empir ically +el iti +ea stri +disp assion +dindi gul +denni stoun +deer skin +cru ller +che sson +c gu +bora hansgrohe +bir gitta +bel ur +bel leau +be moaning +b cy +algin ate +al ae +ab elli +; ') +ðŁĴ¯ðŁĴ¯ ðŁĴ¯ðŁĴ¯ +ðŁijĢ @ +ðŁĮ ° +îģ Ĺ +âĸ ¼ +ุ à¹ī +É Ľ +wroble wski +women fashion +wimp ykid +whiteri bbonday +weizen bock +ve i +tru ee +ton ers +thepc clondon +te ed +tas is +stadi umau +spotify playlist +sper ms +sken worthy +sk omo +simple ment +sikor ski +sig nore +she tty +sch ur +sch eller +sara hin +sa stra +river cats +real cider +re ily +protestan tism +po conom +phetch aburi +per ia +pac t +or tal +o bras +new lin +ne hal +nar inder +mv choops +moc tane +mo cker +melodi festivalen +med alla +me lek +mcken zi +marcell in +lu pines +lo pped +leed scity +la ibach +kitak its +karan ja +johnjay college +jo or +ja key +international mountainday +iclass ical +hype m +hy sics +hash browns +guy brush +guay nabo +fuk rey +frit illaries +fran ki +flight path +feil ding +fav o +exotic cars +early morning +dive sting +dave east +cé dric +cul ham +cobra kai +che pauk +ce ti +cat aldo +canon bury +cake boss +by as +burtonalbion fc +brit a +bit rate +beh nke +be vins +be falls +be bel +b pm +atom os +at boshoff +ap helion +an tequera +an sin +ah il +agameof tones +afel lows +ab atic +ab aad +a weee +// âĢ¦ +. ðŁĺĮ +ðŁ¦ ¸ +world govsummit +wo ss +with ou +wil fully +van ja +un addressed +turnit up +troglody te +touch of +tor ney +therealmike epps +thegreat outdoors +the h +te ven +tan dem +syrian children +sub system +street outlaws +strat agem +stop trump +stigmati ze +stephen marley +squaw king +sport scenter +sin namon +shar ry +sen jeffmerkley +screen prints +sch elle +sap ariba +rou ghead +roo tless +rivers of +ri sco +rhy olite +re integrate +radham ohan +quin nell +purple bricks +prod ded +power slave +phi er +paralym pians +pa ther +ou an +orke stra +nudi branchs +national chocolateday +nar la +mu ta +mo q +mili aromagna +mile sluna +meren da +manipul atives +magento imagine +lor al +lin dahl +ligh ten +life as +lhot se +lgb ti +leaf cutter +ky let +ku tu +kn u +kiel ty +kex press +jer m +jak ku +inter locked +hone sty +hi yori +hau ff +goknight sgo +go kwan +fore foot +figh ters +el lement +ei du +ed icion +e ita +degen kolb +de cried +d old +costel loe +cher n +chaud hari +chang won +chancell ery +cameron diaz +cam illu +by t +blu to +blackgirl scode +bi den +ben nis +ax ess +artemchig vin +ari q +anti semites +annen berg +amir ul +ami don +af ine +ad hd +action on +ðŁĴģ ðŁı½âĢįâĻĢï¸ı +ðĿĻ¤ ðĿĻ +⼠ħ +âĹ Ķ +âĢ¢âĢ¢ âĢ¢âĢ¢ +x inc +well ings +we ga +vas arja +val ero +v lan +usa ha +twitch online +tu ille +transpo sed +topp dogg +tele casted +tatt le +tar ang +swind ling +sum mum +subha dra +stran millis +sof italy +so can +sil kair +si eving +shiro gane +sep timi +s vidler +ru bra +ro tonda +re men +rapid deal +py thian +public transit +prostatec tomy +pmp st +pav lyu +pathfinder rpg +p nca +or dsall +op fun +nau l +nau er +moto gp +mossel bay +men en +mariote stino +mar to +mar git +mar con +madein india +liby ans +le we +lang ella +khi zr +kawhi leonard +justice reform +john sentamu +indi ag +iban ezofficial +iam up +hot fix +hor sfield +homestead er +hm treasury +hiro aki +harpsic hor +hal y +gw enda +guide posts +gt chat +gladi ator +gi gu +fan cher +f zs +ew t +emirate sair +ed mundo +dna day +dept ford +deep ender +de ichmann +dariof ranchitti +d mas +d mag +cul ross +crohnscolitis uk +controver tible +concentration camps +collegi al +cla rett +chill n +chihayaf uru +cher s +cheat day +chach ki +cast agna +car rico +car load +c ck +business forsale +bn ld +black girl +bay ona +ana res +amin ta +al tag +ak ilah +ab ic +ðŁĺĺ ) +ðŁĺIJ ðŁĺĤ +ðŁij©âĢį ðŁį³ +ðŁı ĺ +ðĿIJ İ +ê· ¼ +Î · +yaaaa as +vel tins +vag ner +un tu +un quote +ty er +tsu yoshi +tro va +trail blazer +tit illating +thin kin +theware xo +thes ats +the ca +termin ations +sudha kar +star ley +sr pg +sj h +shre ck +shameon you +shah ed +sar kari +s jo +ru sthall +rol lon +rish na +revealed rec +re distributed +pull down +pu po +pro col +police dogs +phra gm +phil ando +pesc ador +perki omen +paraÃŃ so +oke x +oc ell +north westerly +nohotch nowatch +new tons +net z +ne uland +monster products +ml is +mir jam +mine ol +make amovie +leic spolice +leci thin +leaf green +lare rtu +lad broke +knox rocks +kell yl +ka ghan +ir ac +hye jin +hidden gems +her in +he ati +h pv +gone girl +goldent icket +gil ts +g won +fr ampton +fe ig +fal la +f va +f peck +engel hardt +du mans +digital artwork +deri ving +deep rai +de activates +crum mock +cric k +cow bird +cor nes +co cu +ch ely +bun kie +bravo wwhl +bo canegra +bies mulders +berchtes gaden +be moan +bath spa +bar tered +au tz +amar go +ahrc press +af te +advi see +aas ld +. ðŁĺĴ +ç· ļ +âĢĵâĢĵâĢĵâĢĵ âĢĵâĢĵâĢĵâĢĵ +Ø´ رÛĮ +é tat +y anny +withdra wal +was san +warrandy te +w ason +un worn +un sent +u vi +tween ies +tw oc +time stamps +thankyou jesus +television acad +tele phonic +t line +sz ymon +super size +st davids +spo s +sonali bendre +slur ring +sku dai +signsyou re +short lists +sher d +shar meen +sens itisation +sen nen +se uro +sd w +s guy +ru ster +rou x +roc zen +ro eland +rimm ellondon +ray al +psych ometrics +po hang +peng elly +p wned +ospre ay +osc i +oo hl +o borne +ni dd +narrow band +mo ded +micro surgery +mentalhealth month +mb tc +m anderson +ljung berg +kla ar +kha as +just keep +ju beir +jr v +jami elaing +jam ena +itt ner +immun g +ic sid +ho pia +hipp y +he itz +flower friday +fi gg +fair ouz +ex trude +english bulldog +ene w +ellip ses +ech al +ds gn +divyan katri +discolour ation +cumbri an +cour y +col ma +clair sville +chi v +castro ville +cam hnews +cafe bar +c fra +bro ilers +bl inged +ba yo +az off +aki moto +ai jobs +ac bc +abu elita +@ @@ +) ,... +! âŃIJï¸ı +yo il +yaros lav +x vs +wolves races +who i +way nel +un cor +un ceremoniously +tu omas +top ten +tom perez +tom ales +to pos +tk ss +thesse hydro +tex ti +tan doh +tac tful +ta kako +sy st +superse de +super naturally +sun ga +sultan pur +stre aty +str ung +spread love +sp ul +soweto derby +sin itta +sin dustry +shah nawaz +save theday +sab ri +robinson cano +recycl ers +recu sal +razz ano +ratat at +rainf alls +railroad ers +r fruit +quin ine +puer co +pokemont cg +phoenix es +pha ed +per rins +pent acles +paynes ville +pan ja +pac ademy +orre ry +opfun kill +oni musha +on no +o att +neo tropical +ne vi +ne kop +n guni +moon cakes +moby dick +mo est +mi msy +mesni l +meh wish +mazz ei +materi alist +mar im +lumber kings +lost found +lion heart +ko aa +kevinand bean +ker shim +kay kay +joe strummer +inu it +inst anti +in communicado +icenine kills +hik mah +hal sparks +ha igh +h fg +gle ek +ge ducation +gar io +gan gof +g listen +g kids +finding nemo +fighter wing +femen il +fee q +fal ter +f nn +eye s +ever leigh +end gbv +elevense shour +ei sts +dima pur +dilip kpandey +deep tech +de haven +ctv kitchener +cri bbing +cra pp +cm ts +climate finance +cham bly +cgp grey +cd nimm +cc isd +cart ago +byu hoops +buu ut +boris kodjoe +bon dar +bl s +bi hh +benaz ir +bar son +bal dri +bal ado +assinibo ia +arch enemy +afloo ds +afgh ani +ab stra +a jol +ðŁĻıðŁĻı ðŁĻı +ðŁķº ðŁĴĥ +ðŁı Ļï¸ı +ðŁı Ļ +ëĭĿ 맨 +ãĤ·ãĥ ¥ +âĶĥ âĶĥ +ya str +y aboy +words withfriends +wo lowitz +wit kowski +wick en +w by +vogue williams +vil helm +victoriaave yard +ve u +v gn +urban ites +tv sports +torn illo +toi business +ticketmaster uk +thisisla bor +this be +the vegan +the pursuit +the edit +tam pon +tal ay +sword smen +su par +sta ab +ss ong +song books +sli gh +sli ding +sik sika +si um +sho igu +see doil +scre ative +schre yer +sahi wal +s ican +s dublin +ro pp +resu l +pur pura +pu du +power list +po zzo +pect ations +pat labor +on nie +om es +norse up +no place +my coskie +mu lan +mi asma +me sk +mar ui +mangesh kar +magh era +lyce umtheatre +ly ns +lud vig +la sk +ke ever +kalyp so +jeff gerstmann +israel mfa +in seong +ifwe date +i fra +ho way +harlequin books +har gett +grape vine +god backs +globe arts +gil dea +gat esville +games nostalgia +extingui shes +en um +dro gue +dep ted +denni spra +de valuing +curriculu ms +cur ta +corbin wwe +coo ber +conf ounds +cli mac +chrise u +chis eling +chi at +c frs +buy tolet +bu ssa +book ended +bethe sda +bear ss +baron corbinwwe +bar dwell +bam bo +au ton +ati erra +ark dg +amin aticsofc +ace us +:* :* +ðŁĮ¸ ðŁĮ¼ +ðŁĮ¸ @ +ìľ Ħ +zin c +york beer +x vx +wow wee +wm ms +wednesday morning +walter boro +unfor getable +tor che +tom kinson +toks vig +te airra +t sos +sub tweeting +st wm +spad den +sol na +shar bour +set the +salt box +s like +royalo ak +ro keby +river cottage +ri mouski +rep gh +rat dog +raj veer +r lg +q assem +pv lonabscbn +pv cs +pre zzies +pr riya +pose hn +plor ation +pit so +petroleu mmin +per rot +people over +pedestrian ised +path art +p afa +ox ana +ott tanak +osw al +on twitter +nol ito +niag ra +neo soul +naw af +nano crystals +movie history +lor os +loch side +larra bee +lang sung +kol len +khali fah +jewelry lovers +internet archive +iima hmedabad +highend audio +helens vale +hate week +han sson +ha ig +ha ad +ha ack +gula bi +gu skenworthy +going global +genetic ists +gam bang +food trip +en jo +eli b +elan valley +ec cl +dun nigan +dreamgirl hema +doctrin al +diy sos +destre han +del coronado +def s +death lok +da ia +cs rs +copi ah +col wick +coccin elle +cla ws +chelse alfc +ce tus +car keys +campbell claret +buydo texpress +bu ma +brig itta +bb clocal +battle ments +availab lenow +as os +artemchigvin tse +arm bureau +aqu is +anodi sed +ami um +alas dair +aho y +ab bado +ab ating +_ ~ +$ ' +ðŁĺĨ ) +Ø§Ø µ +za hi +wo reit +who a +wall enda +usc ellular +up es +under written +un dyed +u verse +tü rk +tu pdates +trumppro test +tra vers +tip sare +tha ip +ter mer +tam en +tag along +t ppa +syriacivil def +sun ghoon +slee pless +sle ssor +segreg ating +scot ney +sat lanta +sandra oh +sal vi +sakura jima +ross ell +rosen stiel +ro go +ran au +ra pan +qui era +psi onic +prometh azine +program ming +prakash an +potus geeks +portra itist +popefranci sph +pod sincolor +pin ni +ph f +pe mi +out strip +or ail +oktib beha +naacpi mage +mur amasa +mr nick +mot mot +mon aca +momo chi +miumi u +militar isation +mes ra +maximo park +matthe wl +mapre duce +lucian aberger +lost pets +lif ted +kul tura +kran ji +kle o +kil ic +jrl web +johna thon +jayawar dene +jammu kashmir +ir f +ipa day +hu hn +hil tz +hi go +gun by +gru ver +gou nod +go trojans +gn k +glen ys +ger asi +gamesdon equick +food co +fit c +fantasi afest +fan boost +exasper ation +ent ally +eleanor j +el te +east midlands +dun drod +diver sionary +dimp act +dev is +das raghubar +cur lin +cotte rell +com illas +clo ds +city view +cindy crawford +chew ton +chel seag +cen c +buxton brewery +bull nose +brig itte +bore k +black clover +ben do +bar ot +as mine +arne son +alternati va +allisonb janney +alberta ferretti +ala inde +al agna +agu ada +ag ay +ag ari +aberavon rfc +:) )))))) +ðŁij» ðŁİĥ +ðŁıį ï¸ı +ìĹijìĨ Į +z ea +your city +world fish +wild heart +wi kis +wc choops +water house +vu elo +vill i +vill amor +valent inos +val co +v kontakte +ux ury +tra ill +to shack +th ampi +tchotch ke +tat an +tal liance +tab ora +ta sha +stun de +sto you +stay true +short crust +se gers +se dar +sa je +ru iner +ros ch +root sy +ro des +rc stweets +rai ji +pre ading +pli o +play your +phon ology +peter hollens +pai hia +or ri +oker lund +occul tation +o dy +nadi az +n map +mon ten +mk d +mental ities +mar mon +mad hat +mac each +love mk +logi k +listen up +lets roll +ki pla +kath bum +ju gar +josh henderson +jose fa +jen nac +iron ical +ipad mini +internation alliteracyday +infier no +indie game +in expressible +ilovemy cat +homogene ity +ho ww +ham mon +gö ttingen +gross mann +gran ule +good acre +fron twoman +fowler pga +exclu siva +eco le +do dgson +despi erta +dent sply +defe rent +dam acy +coo tes +coffeewith acop +cm hr +chim my +cam on +c tot +bur nes +brahman andam +black spot +be witch +back stag +auto bant +anjanaom kashyap +ambas sade +ak ley +afl hawk +af ly +ac wa +aa ad +a jaz +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺį +ðŁĺ© . +ðŁĴ¡ # +ðŁĩºðŁĩ ¬ +ìļ° 리 +âĿ¤ï¸ı ðŁĸ¤ +âĿ¤âĿ¤âĿ¤âĿ¤ âĿ¤âĿ¤âĿ¤ +âĢ¦ ." +à¸ĻภĦ +zoro astri +yu ma +yeon dan +ye ayyy +x me +x bet +wra ppings +wor dings +wen digo +weight man +way yyyy +vli eland +v ites +un tainted +ul lr +ton yi +tee total +tam alajones +stro y +star lights +space ibiza +soun de +soft skills +sob server +sne x +silver hawks +shropshire star +shout cast +sch lag +saf fire +s wn +rajeev khandelwal +quick draw +queens speech +pa que +overestim ating +old time +nu al +non lge +nio br +nigh tinthe +mos shart +matthew santoro +mass ad +marou lis +mammamiam usical +mam ma +makeup forever +m lauer +lu gging +lt frb +lost cats +logan air +la sser +kristi ann +kitt in +king arturo +kil gour +ke pri +kay es +jon bonjovi +jj cc +iss ler +isa itl +ilove the +ic alli +i dents +hof en +godsown country +goat skin +glenn howerton +glaci ation +gen asis +fortune cookie +fat ou +far be +faith and +fa reg +eyewitness news +engad ine +ee ve +e ung +e jb +dra isaitl +down to +dar ity +danny mcfly +dajj al +curi al +cure ttage +coo kin +consci ences +centuri on +calli han +bot nets +booksare mybag +bol dman +bod ø +bnld italia +bit burger +beg ley +beati fic +bay stars +bac laran +ash tons +ar pit +again stracism +ag li +adulter ation +ðŁĺİðŁĺİ ðŁĺİðŁĺİ +ðŁĶ¥ ðŁĻĮ +ðŁİī ðŁıĨ +ðŁĩ®ðŁĩ³ ðŁĩ®ðŁĩ³ +íĹ Ī +ìĭ ł +zab ala +yehrishtaky akehlatahai +yastr zemski +willem stad +vul fpeck +vol vos +var lamov +union station +tt olle +travis mcelroy +tra sporti +tra qu +ta kin +ta cho +sty ling +stress awarenessday +stop childabuse +stick land +stampe ding +spets naz +sol amente +soe toro +sne st +sky ways +si rio +sec tioning +scoo bies +sch achter +saha ba +roun dy +rose music +robert shaw +reverber ating +real j +probab ly +preci ado +pou pon +param aham +ouarzaz ate +opening day +nie buhr +nh sscot +ne ds +nan ometer +n atta +my club +mi way +me chag +lysi strata +liken esses +lear d +ko zi +ken sho +kelli her +kay bee +k lr +jon quil +ja veri +indira gandhi +implac able +ima bad +hill yer +herst mon +haun tings +harima u +hal tand +haku oki +gul barga +growth hub +glo ver +gil as +gi anyar +fort son +food facts +extru sions +est eves +ero les +en gr +en gender +effec tual +e zer +dieroten bullen +dho bi +dem ith +decapit ating +danko jones +dam ania +d te +cou sine +conserv atorium +co len +co biesmulders +ci offi +chin naswamy +cheri sh +charlie brown +caru anagalizia +car leigh +capit alistic +broc ke +bi aly +belcher town +be chet +bangkok post +back road +az ek +ann ae +al vor +aa q +< == +ðŁıĬ âĢįâĻĢï¸ı +í ij +åħ ī +âľĮï¸ı ðŁĺİ +à¹ģภĽ +zab ul +yay i +wit te +whone ed +whiskey myers +vigner on +ve do +vall on +us y +the move +tech week +tafa hl +stove pipe +stone masonry +stol tz +stern ly +stephan jenkins +slam dunk +skind red +sin tered +sanmarino gp +ru thy +ru ge +rind t +ren ouncing +re tton +re introduces +re assembling +re animation +pyg mies +push kin +pso as +prince william +preston sburg +pr wx +posi donia +pol at +pick meup +pay asam +park board +on w +on nn +ny ak +noo k +nic eties +ne vel +nati xis +n reid +murph twn +mon el +medi af +me gi +mayday parade +man que +lin ne +latin a +labyrin thine +kir it +ke hl +kaina atar +joyceme yer +jo vita +jo de +jeho vah +jason aaron +jar ratt +iso propyl +hu men +hir ata +hard fest +gw tweets +gun t +gru pos +gru it +gre ssive +gra bb +gon ia +golden voice +gi stics +gb doc +garda sil +future isnow +forsa king +fo sun +farm fresh +er zur +elizabeth taylor +e learning +dyffr yn +downtown la +dd b +dawg sontop +david chang +d bl +chuck berry +cat cafe +carab obo +can sa +camp and +c giar +brit tafahl +brae head +begin ning +bc s +atlan tean +as oke +ar network +ar ale +al meda +ag ray +af shan +ad ais +ace re +accordi onist +( -) +æ° Ĺ +ãĥķãĤ©ãĥŃ ãĥ¼ +âľĬ ðŁĴ¯ +ÙĦ Ùħ +woj tek +wi recard +white stown +white girl +vish wa +urum qi +ur ora +un ceasing +today news +to ils +thomas cook +test able +sub section +stav ro +sp ta +sound view +sof ties +ske ets +sk loof +shop style +shir anui +sharmel sheikh +sh yn +sett ing +sean patrick +sax mun +satt va +saras wathi +santor a +san angelo +saf ak +sa ji +sa ari +s done +rou ler +re assembly +ram ie +ra utela +ra bih +poquo son +per gi +penguin classics +pendi entes +over statement +outside thebox +ore stes +ofcaldub koto +obnoxi ously +nic ca +ne vel +nag er +n man +n korea +n kem +moth balls +mis understands +min ang +max thewanted +ma î +lu ki +lovel ine +lol jk +loath es +liti gants +lime crime +li spe +leonardo davinci +le uk +le coq +laur at +lat vians +ku sum +kin ard +kast ner +k plc +indiana univ +incentivi zing +ic ruise +hv dc +hin chin +happy veteransday +future bass +front men +fried l +finn hour +fin ck +every child +enders game +embal med +el wyn +ec us +ec astle +e qc +dexter ous +destiny schild +deck le +death inparadise +d ric +cv x +cul two +cine plex +chak wal +cem paka +carlo sm +bul man +bro iling +bi shopp +bay ridge +bay ram +bay ah +barrow lands +barra za +b ür +auck landers +ar notts +ap inv +anastasi ades +amy lee +al fre +acri mony +abujatwitter community +! *** +âļ½ ðŁıĨ +zoo pla +zib bet +youn us +wine studio +we bassembly +w enger +vi irs +vas an +vander voort +u az +tri ang +tram pa +topol ino +time keeper +tic ide +theal thy +ted dibiase +tal lia +ta tham +steve letarte +stenc illed +ss ri +sor guk +son er +sil ky +sc olo +sal il +sabo l +s stl +round hill +ricci ardi +reag ans +psylli um +pipe band +pi ques +pen nock +pand it +pachu co +outsmar ting +out numbering +oscill ate +oo om +officialo afc +objec tor +new space +ne way +national hand +mo ch +miscavi ge +mc millin +mavis staples +mat field +mal lets +mad z +mack le +maaa an +ley afc +lee ton +leaveno doubt +le llo +ko sten +kil learn +kid wai +jaga dish +it ss +ili ff +ib sa +hun nic +howtotrainyour dragon +hoo kah +haw khurst +gr ity +gopal akrishnan +go los +garden hire +fore tell +fore sta +fitt ler +em rick +eli en +eiz amusica +echever ia +eaton town +dushy ant +dee dee +de silva +day soff +dark star +cup cakeday +cu chi +cu caracha +cric buzz +craft fair +cove guardians +corne ille +comet landing +coiffe ur +cili p +chuck anut +christian music +cha b +bre on +brandon flowers +boxing day +book promo +bh ool +be fall +bat la +bag ani +back bench +am berg +affin ities +ðŁİ¾ðŁİ¾ ðŁİ¾ +ðŁĩ¬ðŁĩ³ ðŁĩ¬ðŁĩ³ +룰 ëĭĿ맨 +âĨ ł +Ê· Ê° +zi ker +ze enew +yose op +x ul +wen die +voice over +vit aisland +ver du +ve oli +ve h +van ita +uf h +tum he +tour ner +thisis london +tac ony +ta vis +t kb +t its +subro to +spiro sis +sou tah +son taran +sh ichi +seven oak +seas pray +scep tics +sar noff +sacred heart +ru tin +ro ter +refe ction +red team +re miss +ram ones +radical islam +ra eng +r hoops +q adri +prosthodon tics +pok kiri +pi oli +peper on +pedan try +paw trol +pad res +p shr +outand about +open ssl +om mi +ol t +o eno +nu d +north cliffe +nbcdfw weather +national runningday +n dd +mu ur +mt lal +mo shi +mee thi +match week +manig ault +mand ara +man ok +ma hood +m rap +lul ls +lipp ies +li sc +lepto spirosis +leather neck +lau thor +la sek +kyam bogo +kreay shawn +kra ddick +kean university +k bh +ju gs +jo bli +jig ging +jc su +jack knifed +j cre +it anagar +island wide +in delhi +in controvertible +haltand catchfire +h än +grove town +great team +gl hf +gimna sio +gig ichibi +ger des +foam ems +flu gz +fl er +fish kin +fini ster +fer mat +fa eces +ent ury +emul sions +ell ys +cummer bund +conval escing +con form +ci encias +chri sh +che ee +change d +cat ron +carre ira +car hire +bf si +better idge +bang alow +att ys +ar ath +and then +amox icillin +amandab ynes +am eric +alternative medicine +agon cillo +ag ila +adomin icana +acor ah +abduc ts +ðŁİīðŁİģ ðŁİĤ +âĹ Ĭ +âĸ«ï¸ı âĸ«ï¸ı +young day +wei mann +w go +un workable +um kc +tv awards +the tre +the jack +tel ok +super lot +stro zzi +st als +squ atch +sports memorabilia +spar kas +sm ca +sk ata +scoff law +sar ki +ru aha +rick sen +practic able +poblen ou +plant power +piece meal +phi rek +pen pals +padma bhushan +p marca +p inger +oro go +oni er +ob ata +o gan +norman whiteside +ne atoday +nascar goeswest +napp i +mus ico +mo p +me ep +mc clay +ma drug +liv ni +len sman +le coq +lax mik +lan kershim +kim taehyung +keven ingecho +kaf ka +ka steel +ka ino +jedediah bila +jar ro +jap het +inf eld +indecent ly +in adequ +ice man +ic q +holm wood +hog fish +hatt an +hass anandani +ha go +ha gens +gun ov +gool wa +fri eze +free time +fore lock +fonte yn +financial brand +fern bank +family love +f ator +exother mic +ev or +essential mix +es ling +en hage +eg be +dm z +discoun ters +der am +dave morrissey +cri spin +cour tre +cor keveningecho +comor bid +choice hotels +ces ario +ca bel +c spi +bou vet +body by +blue coats +bhat i +beat us +be smart +bangladesh is +balo gh +b cle +as mp +ambro gio +amar ley +al dc +air transat +af rl +ab ud +âĿ¤ âĺº +áIJ ħ +Ù Ģ +Ì²Ì ħ +zom boy +zapho d +yon atan +xi omara +worldd omination +wl bt +whiteri bbon +w fc +viper room +vign ale +val o +tu bule +tu berlin +thu ll +taip ans +sur mise +stor ico +sta hl +st n +st anced +smex aminer +sip smith +si rs +shin awat +shalo tt +sche epers +sc bc +sasol sa +sam sa +roy d +rib chester +rhy dian +re assessing +pro vine +poo ch +pom on +pic ado +pi ma +pg x +periodon tist +pd ca +pann acotta +p win +om avirus +ni dal +neon trees +nal co +my viking +muj uru +mtlal ouettes +mo zes +mo ren +mcv itie +mat thes +mani x +mac donnell +ma zen +ma ass +luis ito +lord stown +light painting +levan te +legal advice +le iria +lap angan +la ze +la the +krish n +kol on +ken sal +kare y +kam ya +kad lec +jig arth +itsadog slife +is ae +iron birds +hyu koh +hm nation +he ssler +has na +gore tex +good memories +gonetoo soon +goe tta +gel dings +g mm +fr yman +foot ballis +eric a +end sleigh +ek tar +ec lub +ear p +corn market +concur red +coet zer +co ker +clap board +ciz re +cic m +care e +ca ep +bush baby +bub bler +bread basket +bre p +brazilian jiujitsu +bod ger +bi sho +beng kulu +b itti +av eline +architi zer +angel ine +all c +æĽ ² +yl u +xon r +world snooker +west cliffe +ux mal +toto wa +too early +tipp ler +ting z +thin c +tf tuesday +t itt +supporting women +sru hle +som nam +sm itty +sly ke +slur ping +si miles +semir amis +see ma +sec ca +sc g +rite sh +riseupred hawks +rho dy +res nik +qu wain +pá irc +pun kand +propag andhi +principal ities +prem chand +pr zy +potenti ometer +pol ke +plu gand +pf nicholls +park field +pall ant +ophi uchus +one happy +news reel +ner r +nat geom +na ah +n to +mu ffle +mi at +manchu ria +man ai +mac ross +ma sand +lockthe gate +lo far +le pak +kemp sville +kather yn +iucn congress +irrepar ably +hi kaye +hack able +gund agai +goodwithout god +go teamusa +ge milang +funko pops +fix it +ero i +eel am +dun raven +dro st +dra ther +dorn birn +design and +dar wyn +cut ler +co bit +chimbor azo +car oftheday +car ned +brown trout +broc kett +bre vard +bluer ising +bin n +bidvest wits +ball ena +b hari +auto drome +asphyxi ation +ar dr +appet ito +anitadon gre +am ayi +alla scala +all ontheboard +absten tion +ðŁļĢ # +ðŁĺī ðŁĴķ +çĽ ® +âĿĮ âĿĮ +áµ Ī +wyn jones +wsf cs +wrong ful +winand yourein +vill on +vel ho +urtic aria +travel writing +thework isworthit +th ye +svf social +submer ging +strelit zia +stevie j +sr va +sport speople +sport park +spiderman movie +souven ir +sott sass +shel le +senti ence +scotu spick +sco bie +samu dra +sa amy +rope walk +rickie fowlerpga +quadr ille +qu k +q nx +pr itt +planetrock radio +paul ricard +pag el +pack ag +our people +ormer od +oppos itional +opol itical +olafur arnalds +ne ared +naz gul +naveen jindal +my four +morning run +mor by +mcin ally +mari eee +mangeshkar lata +mall ord +malaw ians +makeme smile +lol lar +lindy hop +lar do +lam as +l rd +klax ons +jim jefferies +j tag +iter ature +house and +homeboyz radio +ho omin +hari prriya +hand guard +grou ted +gretsch drums +gre go +gi do +friend satthetable +football archive +f nc +en ate +em cr +e iland +dor ayaki +dioce se +daniel craig +cz k +custar d +co var +co ban +chetri sunil +cher no +canone os +c train +bub nagar +bo jana +bo dley +betra yals +beh rouz +bas sing +bar bas +bank rolled +bag no +assan te +as j +anni elennox +alab adi +ðŁĻı ðŁĩºðŁĩ¸ +ðŁĺħ ðŁĺħðŁĺħðŁĺħ +ðŁĶ® ⾨ +ðŁĴ§ ðŁĴ§ +ðŁij ± +ìĤ ° +âĺĢï¸ı . +त र +ı oÄŁlu +z up +ys l +wol v +wi da +we k +ver mette +ve trano +val on +up skirt +unt ying +u rena +ts ong +tro v +toy collector +thomas ina +the ts +the stoneroses +tf boys +ten th +telang an +ta ite +swas ft +su wa +strip elife +sport paleis +soccere x +sm pls +sin to +shop ads +sher ratt +ser ous +screen ers +scot th +sch wag +sai ko +ry croft +ro dders +rein ier +range ley +qual ino +poign antly +picador books +ph orm +pete foramerica +pescet arian +pan war +pa edo +pa chter +ornam ented +orac er +opp erman +nefer tari +nan amusic +mur ton +mu mias +mol lo +mitri one +mill field +michael muhney +mediacom poser +mass ari +lu iz +licen se +ley on +la style +kamp ar +josi elong +intra week +inner circle +iit bombay +id una +hydro dynamic +homes ites +gri mey +google cardboard +gi gged +gal sworthy +for animals +eu com +don nay +doing right +dil opho +dad aism +cu v +chi pola +callsthe heart +bristolold vic +bhattachar jee +assembly fest +anything ispossible +actor karuna +a arti +. ðŁ¤Ĺ +## ## +ðŁĺĤ âľĭ +ðĿIJ ĵ +ãģ ¨ +z nik +witch of +winter berg +wh summit +well ard +vel oute +u ter +tv dsbmath +tur p +treat day +to inton +thevoice kids +thev ra +ther ich +the intercept +the coach +tel aviv +tam sen +tal garth +super impose +str fans +sr sg +spar kes +skitch en +shel ob +se ant +sakur acon +s offer +ru dman +ru ah +ronnie o +rocksolid show +ric carton +rabbin ic +puz der +punch line +pu ft +pot latch +phil os +pacqui ao +over hear +orn ella +opend ns +national nutritionmonth +music hub +mu ybridge +moody grams +missing cat +micro plastic +me il +mari ecurie +mar williamson +mar al +man za +loch gelly +lifein pieces +li q +len ni +leader ships +lat robe +la ie +kor y +kong skullisland +jane goodall +jak cloth +intra ocular +in ves +im zaheer +hurricane season +hol lowell +hahahahah haha +gu ac +great people +gon go +fr acing +flight attendant +ferry boat +fern ley +far ang +ex im +ex hum +erec tor +ef ford +e gle +dry stone +doy enne +discred iting +dia hann +delta zeta +dam ascene +cor less +con scripted +colmc ille +col ca +chase water +carden al +canad apost +cam wowapp +cab ane +bud de +brian w +bra ding +bor st +avan ade +ask ell +ashi da +arche o +ar lo +ander sen +amazing racec +am brin +alt stadt +alam gi +aig adesign +adu que +!! ðŁĺįðŁĺį +ðŁ§¡ ðŁĴĻ +ðŁ¤Ł ðŁı½ +ìĨĮë ¯ +æĥħ åł± +âĺij ï¸ı +é dou +zahaha did +ym un +yel e +with congress +v dara +undoub ted +u din +tri pler +today sdoodle +ti ang +think and +the villa +the jay +ten se +tart u +ta kai +sycoph ancy +superlot biz +sub verted +stickle back +sky hook +sho tter +shar in +sg br +se uro +sb hs +sardon yx +sarah jane +sal er +rou e +retirement planning +ren dle +prison er +portal berni +paulricard track +param ore +para professionals +pap akura +pali ka +p tech +orion books +ne dv +n ali +mondad ori +mis fires +mil verton +mex borough +met service +man gg +mam i +m fah +lyn ds +lpr nyc +let girls +landsc apers +krish nan +ki raz +kd nuggets +kalym nos +kab badi +ka ira +k achi +ju vie +ji zzle +jab bing +infantry men +ib botson +hunter valley +gu z +grow up +gre nadi +glo scricket +glee ks +giri raj +gel in +ge ste +fre twork +fore bears +flor ance +fitz gibbons +f natic +ess m +embaras syour +electro plated +dur ness +dne pr +demon ology +daily gist +cycling weekly +cour ser +cor net +col td +closing bell +cip are +ch nl +capric or +cal houn +bur rito +bouti qa +bor ys +boo zman +bk stg +bj k +bi osis +bal en +anthony jeselnik +alem bic +accentu ating +... / +ðŁĶ¥ ⾨ +ðŁĮ ĺ +zom bo +yard girl +yar is +wisconsin ite +wid dop +west van +we sen +waste basket +uper y +united coaches +uncle blazer +un ready +un perturbed +tor tue +ton igh +tim scott +thiop hene +the tribune +the blackpanther +tend re +tat au +sumbur gh +subo tic +su eno +stre icher +st cloud +soulcalibur vi +sof light +sing en +sig man +selfiest ick +scott skomo +sad an +s fax +s den +rock field +reep ham +redd war +q ais +presiden tever +prefec tural +pol li +peri helion +panto ja +pais aje +olli elocke +ohi a +ocel ots +obe isance +nor is +ni shar +ni dho +ne fer +nancy drew +my brother +modern monday +min ories +mill ard +mc mick +matted mondson +mat kearney +masa aki +mar le +mam ared +mali c +lo chee +lay away +lau di +lam he +kungfu panda +kong sberg +kan z +kac per +jo gos +jam huri +j rod +is for +indiav spakistan +in coherently +im erick +hun nid +hu it +ht cone +hin demith +he pb +hat ake +han aa +gutt man +gopal an +gal les +fer mo +fashion jewelry +far mar +eug bil +esc c +ego istic +dy stro +din ve +der ya +dend rites +cw batb +county fa +cork ery +con descension +complic ates +co gno +co ffy +cherry belle +chelt festivals +chef ou +call acu +cal as +bou los +bobbyl lew +black ening +bishop ston +be zer +be bb +bacal hau +ba ath +b pw +at enas +assi stir +as bel +andhrapradesh cm +and c +an ap +ade al +* $ +ðŁĺĬ ðŁĺĦ +ðŁijĩðŁı¼ ðŁijĩðŁı¼ +ãĥ³ãĤ º +ãĥ ¡ +wor ton +wool er +wil bur +wick man +we it +we iner +wc sd +visit brussels +vian sk +u ah +tu gu +to kyu +thre ss +ter williger +te ef +ta va +syno logy +suomen linna +sunray sia +su lit +stewar desses +ska ite +shop ko +shin do +shi maz +she ads +shan shan +se yed +scream fest +science fair +sathi sh +sas ummit +re yer +raffa ella +purwak arta +promp tlist +pet portrait +pawl enty +paw circle +pauly shore +parat ransit +par thian +papen brook +ndr rmc +mul ligat +mc dreamy +man jeet +mahesh sharma +lap ine +lam onte +key card +ker oro +kaz ak +karl urban +kainaatar ora +jur gen +jimal khalili +james dean +j radio +inform ality +in authentic +imit ator +i haven +hin dman +healthy hair +har oun +gu ast +graveyard carz +gal lucci +gab b +fi roz +farm houses +entry ways +enjo bs +electric picnic +eddi emurphy +eagle ville +duff town +don nelley +desi sto +date able +dak otas +confi ding +co des +city rail +chi angra +charlies loth +cer t +cc funkandsoul +cas sels +carequality comm +car mello +bru e +bie gel +bachelorette abc +awe want +astr alis +app design +all td +air band +ad hs +actu alized +ðŁĹ ¡ +ðŁĸ ±ï¸ı +ðŁİĬ ðŁİĪ +ðĿĹ ¢ +áµ ĸ +ॠĥ +zoy sia +zeenew shindi +yav uz +xx l +worldof warships +wink news +westh off +web tour +we te +wal lawalla +wah peton +w ben +vol and +victory beer +vicky rubadiri +ven eration +up éry +u anl +tw iss +tur alists +tron foundation +to fun +tm ills +tin am +the legendofzelda +th ays +ter lingua +tamer amowry +taldÃŃ acomohoy +supre ma +sound ness +so su +sj l +shin sen +scot amfa +rene wal +re animate +py les +pra ja +per nice +pedr ito +pat ong +pa ura +p ello +over laying +or way +onthe green +not as +nin ful +nil i +nav o +natural hair +nan chang +must weed +museu marchive +mun sey +mill ers +mc ternan +mc spadden +mant illa +magic mike +ma dailygist +lü neburg +loren zana +ll np +le iv +kel burn +kayag nik +katie holmes +k mel +ju ric +jessica jung +jan ak +io d +i ffic +hu ai +htc tryouts +hitthe floor +hi rap +hi bi +hel low +goul den +gold ber +frie de +fire baugh +fin nis +euro pac +en livened +dul thood +dhru vasarja +de duce +david h +d si +cruis ing +crimestopper suk +chun li +cho le +chery shev +cat orce +casper vandien +bristol nhs +bre m +bo ij +bo gal +blr citytraffic +birch ington +beti bacha +bbcscotland news +att kisson +asym phony +anti histamine +anind ya +amp suk +al dez +ak pan +acer bic +- "@ +ðŁĺı @ +ë¸ Į +åł ´ +âĪ Ĵ +ย ว +اÙĦب ØŃ +wo aa +wic kett +wi on +wedding decor +voj vod +vent spils +ven ray +vel uwe +v ahan +ur key +up staging +une scap +tro yal +thi em +the ug +so car +si h +shu mai +sen ai +secre tions +se ssi +rup tures +rubi doux +restore thevra +refriger ants +qu illin +pu mba +probowl vote +privati sing +pra i +police family +po ya +po cho +oldman crew +ok ent +nomin ative +no pf +net label +nei va +n itti +mulligat awny +men asha +mar gao +manpower group +man sory +man son +magal ur +lun eng +love oz +lal ang +la on +la ire +ku duro +knot fest +kele la +kamer on +john newman +jerky xp +io ta +inter jet +ich thys +iam lenaheadey +iam cannabis +henry holt +ham ban +h mua +gur riel +gour i +g berlanti +fl n +fior ano +fight ingfor +eu taw +enz as +end ly +el c +eco watch +dyscal culia +duba ic +don mar +dil se +di ola +di kt +defl ates +de sensitized +convul sive +con la +chowd ary +ce bit +cb ct +car not +brianger vais +black hawk +ber lingo +bel is +bar jatya +ay k +aphrodi si +any who +ann d +am lw +ak s +ai jaz +ac oo +aaa ad +ðŁĩ¯ðŁĩ ´ +your heart +whati learned +wear redday +war moth +walshy fire +ver lo +varney co +van ill +us g +un balance +tre ec +toscan ini +titlei steurope +tick ell +texts anta +stay focused +stat ecraft +star garyen +so bhi +smex y +smart contract +sids riram +shi geto +shawin igan +share alittle +sells vintage +se ba +schi emer +ru su +real cj +qnl z +projec toftheday +power plants +pover a +poll star +play box +phili pham +phel tzcomics +parallelo gram +p she +over shadowing +outer hebs +olf club +oil paint +nicomaine at +naw abs +mor cheeba +mel fi +matador records +mat tr +london bronco +list ener +le ist +kim guilfoyle +ke wanee +ke ffi +karti keya +kaad hal +jo ynt +jeff botl +je gs +j re +j omin +j illa +instaf it +ib times +hull fc +hiro mu +hex agram +heff ley +hand carved +gri ego +gen aro +funic ello +from dusktilldawn +food friday +follow ing +fit and +feli u +etsy sellsvintage +eli braries +du bi +dis qualifies +ding li +design junction +deni grate +davidar quette +david b +daddys gurl +d jam +cryogen ics +con fit +commun itech +colorad oriver +ck ner +chetu mal +charliec rist +ce sky +cdn screenawards +cb ase +carto graphers +can tatas +bwa hahaha +bun yip +brigh twood +bracketo logy +blan ke +bet tel +barcel os +balla deer +bab us +ba eday +att ilio +art world +ar zu +anthropom orphism +anim ales +an ath +am ua +alye ska +ad com +(âī§âĪĩ âī¦) +âĢĶ -> +à¹ģภŁ +zel aya +z ann +x pose +wra iders +wc w +um laut +transpo se +tr é +they ll +tat sun +tal isa +sz ky +sx moctane +sob chak +sig is +shu cking +shol ders +sen ju +sel borne +se bor +se acroft +scottish highlands +saad at +rpo online +rn tata +richar da +rebel hearttour +rc sd +ra ich +r alo +pun kd +poster design +phosp hates +pett way +personal care +pe gram +panther s +pad ley +pa zza +over working +op ala +ol czyk +nic ke +neu meier +natu ren +national cookieday +mor re +maruti suzuki +lob dell +liter acyday +lip kin +lic orne +la bre +la bial +la ad +krish nar +ken ting +kemp en +kc caug +ka id +ju stre +j reed +itu din +it au +insinu ate +ich rist +hu lot +hpnoti q +home school +hay don +harrieth arman +hand lin +hal ilovic +h sts +gur purab +gamers gate +g mod +g beng +franch itti +flashi kat +figue iredo +fen ske +enter o +eas dale +dh v +d sn +cru dit +corin thia +conference usa +coloni alists +co agul +chocta what +chie fex +charlesd ickens +charle swood +cen o +call ery +cal football +bul ley +brigh twell +bree am +bol ding +bi plab +betsydevo sed +bar il +bankni fty +bani shes +ban yo +ban iyas +ba reminer +asking for +as pho +app sych +ap late +and day +ade ga +aber corn +ðŁij¸ ðŁı½ +ìĭ Ŀ +âķ ¥ +ye pa +ya eger +wrest lin +world history +window sserver +way faring +ward man +vers ac +upen patel +umass lowell +u my +u local +theodor ou +the weekend +ten ergy +tce cardinals +tas min +ta ira +srijit speaketh +smo d +slv sa +sland erous +sk cv +shag gy +sen an +se ber +rich burg +re uses +ray toro +rav ager +raim ond +proser pine +pol lok +par ami +pap illary +oy uki +osme ña +or thy +op ac +old mane +oldmane bro +ob scu +non conforming +natl prep +nam as +myri ad +mud flap +mono theism +metan oia +medi en +me xt +martinlutherking jr +map making +make out +ma hay +li gnes +li ba +lesleyann brandt +le ki +laver cup +lar ner +ku zu +joe dotie +its agreat +isa ia +intermittent fasting +in china +hy dride +gub bio +gher bo +ge tover +gd live +gar re +flame throwers +exab eliebers +enchong dee +emo de +edg iness +easter brook +dow ler +dor ing +director ship +deadly sins +curling canada +cu tes +contempl ations +cl enden +car oni +bra ggs +botan ico +blog paws +blackink crew +betibacha obe +batt lers +ban bury +avi atrix +av and +anu bhav +ann ya +anheuser busch +alo dge +alamgi rizvi +age ek +ac ti +ac rack +ðŁĻıðŁı¾ âĿ¤ï¸ı +çĽ ® +ಠ¸ +à° ľ +zhong shan +ww elive +worl dexpo +woo p +wai hi +vo t +vil lo +vid éo +vas cu +vagu eness +v usd +under value +u see +tut bury +trache ostomy +to kam +tir ith +there sa +tauto logy +synchron isation +swan berg +su ad +spl urged +sp lan +sound less +smile more +sitt we +shil ajit +sh agreen +schoo le +sab ry +ryan reynolds +roth child +rbge hort +rafsan jani +ra shan +qu be +psycho analyst +proprio ception +profess r +pro me +pp d +porcup ine +po stel +pang aniban +oubli er +ol die +moo gle +mo sler +memo ire +mc di +math ed +lon drina +lla ve +live action +le ering +law ers +ku ehne +kirk dale +julio jones +jez reel +jetti son +ip ag +induc tor +ici um +ichthyo saur +hyster ic +hu pp +hr lich +houston heights +home away +he int +hat ting +ha aga +guildof finefood +gidd iness +ghost buster +gen next +fou dre +fo res +floren zi +feu illes +fer ret +fe ve +euro a +ep is +enrique gil +end les +e tim +didd ly +dic taphone +de mining +cubat ravel +co calico +clarkk ent +chor tle +cb ellracing +bul las +bren o +birth marks +be jarano +bbcradio stoke +b tho +b kt +audi ere +atp masters +as sp +ap his +ang ol +ang ere +alex s +al tran +aic p +ag gia +ðŁĺĩ ðŁĻı +ðŁ¤Ķ @ +á´ Ħ +ze spri +ye ws +y go +world run +wind lesham +we uro +vogel song +vho enen +us olympic +ur ning +u sun +twitchretwee tr +tv network +tor rez +ti be +ther see +ther itz +teresh kova +tak firi +sy ra +splot ches +spaw n +snow barry +sight lines +si mc +shilo h +se wu +schi ff +saviler ow +san wa +ro ten +rizzoli isle +re eee +rat ap +r factor +qualityof life +pur posing +pul man +pu tten +procreate app +post i +phi the +pa quet +official charts +nor west +nit ef +nick groff +nev ada +mono hydrate +mon ckton +molson coors +mod ellers +mc tell +maj ithia +low cost +louisian agov +lahore qalandars +l be +ju lly +jeux video +j man +island records +intothe wild +im ss +ili brary +ig inal +hydro logic +hou chen +hoo ter +hom am +hi madas +her ve +her ren +hel loo +heinz vhoenen +he ey +has z +h mer +gel inas +g elliott +french quarter +forthe weekend +for honor +florian opolis +fa sho +dun t +du aa +dom itian +dj max +din k +descon to +d johnsonpga +clear skin +but u +bras sey +body board +bk v +bit pay +bigro ck +bene detta +bar ony +bal art +bad uy +ba q +at ami +as umi +andrew christian +an fa +an ever +am usa +al medalen +air busa +aero space +ðŁİĦðŁİĦ ðŁİĦðŁİĦ +ë¬ ´ +å¼ ¾ +ér ôme +zuc chero +yer caud +work table +we win +wb sdcc +vill ena +viewfromthe office +usta z +upenpatel world +un ah +uit enhage +tyler cowen +thereal morrison +thame side +ter na +tar if +tani guchi +taf ter +swag man +sunday service +stre b +sky forge +simm o +shinawat ra +sety rn +sar kis +s bank +real men +re ja +prime au +plum tree +pardub ice +pal mera +or phi +off setyrn +nak heel +multi stakeholder +michel is +mc cafe +mary stown +mar bach +mad in +mac ungie +love that +lo hn +lo ffici +li thic +les niak +legi onnaire +lee z +l lah +l ity +kor ang +ko vil +kho j +ke olis +kar ls +kab o +jordan stown +jeni fer +je maine +ire x +inquirer biz +ing ly +indul gences +i abc +hor ological +har ring +halcy on +haj ji +gru s +gra vois +gis d +gi js +getur riskon +ga jar +from space +flash forward +fight news +fer ried +fast fact +fair water +eski mo +er be +eich ner +do reen +dhi vy +david afrench +darren hayes +cyclon ef +cth k +cre f +cowh er +cor wen +copp elia +cool ing +coco ons +char vel +car lie +bro sh +bri dles +breaking dawn +bre snahan +bel grade +bandof horses +bailey may +ark ley +ar kra +agreat bigcity +ag ip +adi da +ad rs +aac ps +ðŁIJ¬ ðŁIJ¬ +ï » +éĽ » +ç Ĭ +y ati +world blooddonorday +wn es +wha thapp +weare the +uw badgers +uni ak +under lies +ulti mates +tw oway +tso tsi +triangu lum +thermal take +theover tunes +thehow sofus +thec rick +tameramowry two +t shir +symph on +surfer sparadise +suncorp stadium +styles p +stick ball +sp rockets +sne tball +sna c +smart mobility +side steps +self worth +second chances +scri be +sar mad +sam ms +saakash vili +s mullen +redefin e +re tooled +rale kha +raider strong +pur view +pu gn +pranit asubhash +perme ated +par apar +nov ations +nigh tin +never surrender +nc ell +mole hill +metho w +mar tes +mar sy +manek shaw +male m +ma zo +ma had +ma ddin +m qa +m jj +m ji +lu ter +love it +lin ha +lifel ock +le twin +le gh +kathmandu post +jou e +institutional ised +im ac +illumin ators +iko kaz +home fortheholidays +hel pu +he chos +hau liers +har nish +guer neville +gre gan +gon dal +go outside +gh and +gen thaler +gabri ell +gaben fica +g list +fox sportsnews +fly tpa +flan eur +et in +esc rit +end at +dump the +dor aville +dam aris +dale steyn +dal ley +cran brook +cor oll +come up +cla de +chol mon +chess base +che es +cein ture +carre gabenfica +cann ell +c sem +box games +bou squet +biz dev +benef icent +autom obili +ass ant +ar ati +ann ell +an tri +ambassad ress +agra ham +adren aline +ðŁĺįðŁĺį âĿ¤âĿ¤ +ðŁıĢðŁıĢ ðŁıĢðŁıĢ +ðŁĨ Ķ +æµ · +ãħ Ĥ +ãĥĪ ãĥª +ãģķãģı ãĤī +๠ģ +zell ner +z ill +ysleta isd +yach trock +xylo to +wolf hard +west combe +wedding inspo +war hammer +wal de +vir sa +victim less +van go +v uganda +v hl +uk bff +ts agov +tri ble +travan core +to ton +to rey +time scales +theword alive +the team +tex ashi +talkto pd +sulphu ric +su rer +so len +sing la +schlu pp +rubb ers +ru y +ra fc +r anta +py rus +prat ts +pos iting +pof icial +poetry society +pn brock +phragm ites +penn ard +peach y +par atriathlon +ost p +oro ville +o gh +ny mf +niki for +ndu bz +moz fest +mon cks +ment ally +mc comas +maytheforce bewithyou +may flies +mat ram +maqu illage +mag is +lyn dale +lucap as +lou rens +leu ci +le xie +le shan +le ota +law enforce +latingram my +lat ou +la a +kubla i +kpm guk +kitch ener +juicy couture +join there +jedi diah +jay and +jan ky +italian style +ig as +hul kam +horni manmuseum +himm at +hil lock +hen shah +hail southern +grass ington +gore ski +geno types +fleet management +flav ell +fiction alized +fi du +fascin atingly +far allon +f ma +ett inger +er music +el lam +du leep +drive insured +dar pan +dae mun +cy bil +cur rys +contin o +con oci +co ent +chand rika +ch ulo +cen ser +caw thra +bu shi +brandy well +bou ie +block ading +bla do +bin ky +ben be +bel gis +bau ti +bau le +baro ss +bar ite +asseenin columbus +alm shouses +ag ol +achristmas story +ìľł ëħ¸ +ë³´ ìĿ´ +ê¹Ģ ìĦĿì§Ħ +ಠ¿ +ye tta +yaf fe +wordof god +wen zhou +val astro +ur f +unitedwe dream +ula res +uit p +tit ley +tiff ani +tag on +sugar and +stru mp +stri m +stri ker +stem day +sme er +sine w +sha hada +sh older +sey dou +sahar awi +rol lup +ro stro +re im +rac quets +qu ise +ppro ject +pp ina +pol let +pis an +pha res +pak sha +our town +or kest +opp ement +oh mi +ogbon na +o ord +o brist +nin ot +ne bel +nas ca +ms fc +mol ts +mohom bi +mel hores +mal en +maithri pala +ly da +lon da +liquid ating +lick ers +less or +leard blockade +le mbo +le ash +ku fi +kam illa +kad abra +ion izer +in cle +i all +hyper text +hill town +high tea +hang u +hadd am +gut tural +gu ap +gran adilla +gr ö +goff stown +gee king +g pro +fum fum +freetheni pple +floo dgate +flat man +evil hag +euro control +epitom ised +edinburgh zoo +ed ance +dro ad +diatom aceous +di ds +datascience ctrl +dar ko +comb in +co ye +cc cu +buch tel +bogo sian +big nell +ben harper +ay ar +au teuil +as ol +arc and +am att +aga g +af forestation +ae w +aal apor +ðŁĻĪ ) +ðŁĶ Ħ +áIJ Ľ +yu mmi +yj hd +wy ant +win eco +wil pf +wil lumb +whaaaaa at +wa inaina +volunteer day +trasi meno +titu t +ti so +thetalk cbs +the jump +the homeof +tender izer +tech geeks +team ucm +ta en +sw asan +suu kyi +spra gg +spo c +sien asaints +si rois +shoton oneplus +shere met +shel man +sheepi shly +serendipit ously +seaf ire +scott gshore +scen ography +scarlett moffatt +sale town +ru salka +roman ization +ridg ely +re constructions +re classification +rap mon +quebec city +pronoun ces +pre ti +pre defined +po pin +play on +petrol heads +pen er +pa hl +or dre +of music +octavi aspencer +mr tommy +moon lights +min ott +mik los +mase go +mari af +maha yek +ly kan +lin dal +leg ant +lat eran +lamba sted +kom mt +kit amura +kill cliff +kap itan +k ace +it suki +is il +insur gentes +inf lorescence +ich it +ich ay +hu an +history day +hi eu +hai kyu +grand mom +gir alda +gi en +getto know +frederik sen +forger ies +fly day +fl stateparks +fin twit +feeling blessed +fay outh +f ya +ev elin +era worth +ent deck +e dea +dwn twn +duf fus +dro om +dic he +der mody +delhin cr +de stra +dal gety +d cau +cyber sec +cour te +conten tedly +congres sperson +col tart +brown lie +bre x +bour ret +bog side +bo esch +bel phe +barber ini +ar bab +am mer +acham pion +absor bency +:' ' +ðŁĺĦ âĿ¤ +ðŁĴĶ # +æľ Ľ +ä¼ Ĭ +âľĮ ðŁı¿ +âĸ« ï¸ı@ +á zar +zehn der +yer kes +xero x +ww wf +women surgeons +wit n +we star +vinay aka +ven ator +vari ously +un translatable +u fl +u dan +ty rr +tren tuni +tott ington +the savoy +thak or +tegern see +sur ve +smithfield foods +shi u +scott baio +saur usrex +salon pas +safety tips +run offs +restaur ation +real world +r ón +quan tumb +q ml +pulmon ology +pir atas +pi ra +oven den +on ny +ollan tay +oc weekly +o hit +musth aves +mur tabak +mul ford +muhammadi yah +mu dv +mon roy +mon geau +mo liere +mis sle +middle ages +mic ke +mc p +mb fwa +m di +lumin ate +londonbronco srl +lil ting +lau rid +lare my +kow tow +kom ando +kar z +kamalha asan +k gu +journalis mis +jo shab +intre sted +iff r +ic ty +harro dsburg +hard inge +hange ul +gott acat +gl n +gam bir +fre richs +forever with +fex po +fay az +expan se +etch ells +et ool +eluci date +el rick +east devon +distribu torship +dis affected +diar rhe +deme o +deenday al +dean s +de martini +coldwar kids +children shospital +chee zy +char cot +cas andra +car reg +cap turing +brown lee +brother ali +beck ner +bas ak +b boy +auto psies +au stal +agu da +' !" +ðŁĺİ ) +âľ ¶ +é ri +zoe trope +z alab +yoshi oka +yogal ove +x cond +winter ville +wild west +wedder burn +vi varo +uryn hill +un colored +the calm +the arti +teatro allascala +te ays +tas bih +t smith +t lo +t elic +sunday selfie +sun u +summari zation +spell book +smu ckers +sli e +sksk sk +skill share +skatec anada +sicure zza +shou ka +sel by +sc te +s again +ro loff +ren ouf +rema pped +red u +reci o +re val +plu mp +ph yl +petre scue +pest ana +part age +paper man +paper magazine +on tiveros +on enew +ne ira +nare lle +mi one +manip uri +man md +male ek +mal ays +lu bang +lo ic +le ur +lar tigue +la vis +ke sel +jal y +is ob +in ski +ice do +hoof ddorp +here come +hau ts +ha or +h mph +h dr +green ing +go pala +gad son +fon i +fed con +f hc +emer aude +dunnell on +dub ya +don don +dhu l +devinder raiji +dak hla +dais aku +ch lin +cavat appi +cast away +cap rio +ca strate +bur nit +bond holder +blo tches +birch grove +bir tles +bevil acqua +bet wn +bbc merseyside +bas ak +bapti st +b cus +at ake +anok hi +annes ley +amber ly +al eck +aero star +ador kable +adel leo +ack lam +abscbn ball +"" "" +ðŁĺ£ ðŁĺ£ðŁĺ£ +ðŁij¯ âĿ¤ï¸ı +ðŁĩ®ðŁĩ © +æĽ ´ +ÙĪ Úº +Ùĩ ÙĬ +رÙħ ض +zel mer +y atim +wr on +wies er +wheel jack +weiz hou +weare weber +vene dig +ven ham +u chu +twitter party +tru elove +trade craft +to ve +thestor mishere +stri bling +spel thorne +sk anda +selfre g +schla pp +round tables +road hog +rish tey +ray more +raj in +qu ity +qian long +ps media +produc toftheday +pre port +pou ched +pis d +phi beta +pat o +pand ita +pal le +p ni +oyor ooms +ove ts +one humanity +omni directional +omidy arnetwork +northumbria uni +non members +nidho gg +mur willumb +mtb life +mtam aryland +morg ann +men folk +mel len +markr pellegrino +mariab rink +mar tock +man abu +mainten ance +lor ds +li fy +lean in +lan awwe +kw q +killthis love +kil mallock +kag erou +k wc +ix p +is que +hol men +ho is +haven lust +guaj ira +get north +ge healthcare +fox ley +fle eces +divas confessions +desperate housewives +del phic +dame wine +d jay +coun tin +city kitchener +ce smma +cay es +carlin ville +campo ree +c ge +bluestar media +bene ath +ben askren +bee bee +beat niks +b ghs +ath iya +asi acup +ash el +ase sino +alainf cae +akh ya +aj ar +admon ition +ad harshini +å°ij女 åīį +ãĥ³ãĥ Ī +your vote +xia oming +wux ia +wat lington +wal esa +w fr +ve el +va shish +un settle +un dr +ug wu +ty rel +tr nc +thu gga +thorn dike +thal loween +tan a +tam la +t ma +str ating +stock trading +ss g +sp ick +so tu +snow mobiles +sing son +shy lock +sfor the +sfor peace +ser mon +schiz oid +sar gentina +sam arth +rac coon +qui ddick +pur s +psychedelic art +pro europe +perme ating +pere grym +our finland +ori ello +o guri +o dus +ni obe +net scout +natural products +natural ised +nato summit +mt dc +ms rachel +misssaig onuk +mis represent +metac ar +medi are +maree ba +march mont +mal pass +mal ite +loveu all +london city +local auckland +lo key +lma ker +ley endas +lau g +lancelo teh +kul wicki +khat ter +kas son +je tz +iv m +itf db +iso k +impeach ment +ik lanceloteh +hu eso +house hold +hor den +hani fa +gras strack +gam ine +free ski +fibro id +fi ds +ffe y +f fo +eur on +ernest moniz +enforced disappearances +endimp unity +dro medary +don nelly +dod son +de ba +dal edc +dak o +cur tice +cran ky +confi ance +con tender +citi sports +circu mv +cash for +carra way +cal pis +bro mas +bre al +bour ses +bo za +black jack +ben ni +being boycie +bale stier +baker mayfield +arctic circle +ar sht +angu l +anal i +allo p +al nico +al murray +ag ol +a africa +! ðŁĺĪ +ðŁij Ĵ +yo soy +whizz er +vijayak anth +tyn wald +tre volution +tre han +trac tive +tit li +thir dly +thin lizzy +th form +telang ana +sympathi zing +sub d +stu be +stgeorge s +sterili zing +soul train +ske p +shiva ay +shinsen gumi +self made +scal gary +sb j +sal ice +ri vie +reverber ates +rang elands +ral ston +rad tke +q oq +proven çal +pro khor +pi miento +perfec ter +paw ley +pack pride +oak bank +o hr +o are +no problem +news net +news bud +new sit +nav deep +nap ier +n da +my du +me ineke +master system +major league +mac nee +ma quo +ma cha +len zi +kn ack +kid lington +kat arin +kalye serye +kad al +kab ba +jud ds +ip on +ing net +in he +ikokaz ike +i pod +hy on +hu cker +heinz field +heck lers +harmon town +gul berg +go bel +gar mon +free trade +floridag ators +fan sided +escapethe fate +e bell +dox ology +deb es +day club +d ally +contex tually +conom ic +com and +clai borne +cit rul +chu ll +christma sy +cavall ini +cavall aro +cas sville +cap n +brown hill +bou ldin +blau w +birch field +bio dome +behren dt +bc v +barri ere +bar cs +bal last +b wn +austinand ally +au mont +amil car +adri any +aa sif +& && +ðŁĽ ¢ +à© Ī +Ñ Ħ +yvonnear naud +work sfor +wishi was +willy wonka +vi vos +vann in +tylerj blackburn +tow son +toler ation +to ffice +tlax cala +tfl ers +star maa +stan ko +stall er +ss diski +square d +snug bucket +si skind +shaw nee +scotamb service +sardaar gabbarsingh +san tonio +s med +roy bal +ro goff +rizzoliisle stnt +ring le +reptil ia +record keeping +pren up +poster paper +photo sby +pad dler +pa ho +outra m +out ag +our d +osso buco +omni vorous +od zilla +new blogpost +n kc +music ologist +multi sports +mor iches +mitchell vii +mil ken +mem u +me kas +mc chesney +mat tam +lun ny +locke din +lake city +kristen ledlow +ki thar +jubil ate +joy as +jal aluddin +jal ade +inuk titut +intensi fier +inocul ated +house fly +hi biya +hash em +har away +ha peville +gen ii +gaw xcond +gand ara +g way +fried mann +free wifi +fivb women +fine gan +far amir +estab rook +epau lettes +ent se +en nie +dur ations +dru mma +dil ys +dg wick +dar nall +ct k +cow ans +contact center +col y +co wed +clam pett +chuk w +chronic ler +chil led +chem ed +chappa quiddick +ch th +cen mag +campan elli +caldic ot +butter fish +bur gen +bol sena +bike ways +bi yori +az m +auchter arder +ate me +aren ds +alani z +ai ge +adi sa +ad ream +actu aliz +ab sent +... " +. ** +ðŁĩ¬ âļ½ +ðŁĩ¬âļ½ ðŁĩ± +ä¸ Ģ +âĨ Ļï¸ı +âģł âģł# +ਠµ +ÙĦ ÛĮ +Ì ³ +win tney +wi hm +wan ee +walmart strikers +us ss +tv drama +tro vatore +tor pey +tops field +tito ortiz +tic khill +thoo ver +thiscouldbe us +thelu cky +theband musical +tech tips +team razer +tc v +take offs +t sev +sub national +street ly +stay hydrated +spo to +spee do +sier ung +shu shu +shel li +serious fun +sens ori +sd h +say ani +save gaza +rock ery +rcl ens +pä rt +poy thress +poy dras +popp lewell +pet renko +oxen free +old n +official baileym +no tam +nkotb sb +murwillumb ah +mitso takis +mill wright +maz on +lt gov +lo quillo +le icht +lar us +kul dip +kanti pur +japan times +jamie kennedy +j wh +iam jojo +house sitting +hou traffic +hotair balloon +ho ian +h jr +gocat sgo +go wo +gis day +funeral care +fri m +fire lli +fer rule +feminis mis +every night +em sp +em ount +elec table +dor rit +domestic ate +dik sha +desmar ais +de ason +d juma +cz ars +crest fallen +chew tonglen +can oa +cad dis +bro zovic +bov ington +boo kex +bod kin +bo whead +bluet sunami +bel fiore +ban tul +av z +ast olfo +appellate twitter +aon tro +anti histamines +anant kumar +alton brownlive +alic ja +ale ye +al kas +air frames +ç Ń +âĺ ľ +Ùħ د +ye syou +wü rt +wy ff +wed more +versail les +ve b +u ddingston +tor ise +toni thecat +todayim wearing +tiger style +ti pps +the osophy +the most +tabri zi +sy ma +swa ins +sw ca +stac kexchange +st ex +space channel +sno ddy +sne h +septe t +scutt ling +sal zman +s min +ros al +real romadowney +ran ade +radi ative +ra bie +proto culture +presidenti rl +po ddar +phari see +pec ado +om ad +ocre gister +niobr ara +new telegraph +my i +muhaj ir +mudv ayne +mradam scott +montp ellier +missuni verso +mile ena +mezu zah +md p +mar mo +mad is +lu pit +love by +load star +lo che +lev ana +kyle kinane +ku fa +kraken rum +kne ale +kenne ally +kam as +itch er +it smar +ironwork thursday +internacion al +inag arten +imper ishable +ig ma +hon ka +home time +home economics +high nesses +har gre +h aces +gol fon +gi app +fro ebel +flo rek +flavour ings +five fold +fir str +eight fold +eckhar ttolle +dr f +dor it +don ta +di se +den ou +demo is +del im +datt atreya +darb hanga +cu k +csu mb +cran leigh +cran bury +cot in +choctawhat chee +chamele one +capp adonna +c plusplus +bur gum +buch wald +brie fest +breastfeed ingweek +au sd +ari alv +ali venews +aj na +! ðŁĮŀ +ðŁĶ Ń +ðŁij¨âĢį âļķï¸ı +åį Ĺ +ठĿ +zoom zoom +yn ich +wor sfold +wol itzer +watch word +warr iner +wad low +vÃŃ deo +ver dy +upper most +uni for +un ami +twee dle +tro pon +tril log +trans kei +time warp +th amar +stud illo +strick ler +street scapes +stim ming +spren ger +sm tickets +slo sangeles +sherry ontopp +sen ter +schem ers +sch acht +saw fly +san j +sal tram +safe cofield +rumin ant +rosen wald +ron ja +ron g +rev athi +red olu +reb sbsb +re flog +rafi k +quen elle +porphy ry +pisco tty +per vs +pale strina +omis sions +nord stro +nagach aitanya +mp naveenjindal +mosqu era +morin aga +min ski +mi ras +medi an +med scape +many ara +li bret +lewis ville +lev ated +league intheworld +laryn geal +kar ski +kap tur +john muir +jhump a +iso bel +ind say +hurricanes rugby +hu ila +hol ga +hit music +hearing aids +hahahaha hahahahah +gri quas +gold stream +goe demorgen +go ads +glam sham +geoc ities +gent ing +ge sellschaft +gaillar dia +fum fumfum +fedex forum +eo valdi +energi zes +drag sters +derby shi +dena fil +demonstr ative +dav ro +cu mnor +cis l +ci aldini +christ off +chin di +charle son +chander i +car adoc +canop ic +cal usa +by culla +bure aus +brit tri +br ini +bower bird +bor chardt +black sad +black people +bic oastal +bc bs +bad die +bab o +ba shar +av r +av aaz +ar roman +angel ini +alz forum +ake redolu +ak ame +ag ma +adjour ns +adam west +ðŁĴĻ ðŁĴĹ +Å ij +you zhny +yama shiro +xer is +x ania +wing nut +wing dings +wee eeee +wad ge +wach tel +vil lu +vic pol +ver ducci +v wt +v aka +twop ad +tu ney +town son +tinke rer +thereal kiss +the jake +thar mon +terriclark music +suf is +su roor +stu dia +stron geurope +storm bringer +sti k +stateof decay +sseairtricity lg +ss quare +spor tre +special collections +spe idel +sol di +slugger nation +seam os +saf tey +s ying +ra yel +ra jon +prun ella +pis sy +pas u +p be +ox hey +on usa +obam acare +o cl +nonlge progs +nis sim +news dc +nba history +mur naghan +mi pim +men ses +man utd +m che +lit man +leg on +lan sley +la si +jon bellion +jang ly +j wala +istandwith pp +inter news +hawaii fb +hard knocks +har sin +h jh +gö tze +gul da +go bbo +ger ona +garag erock +fu mar +fr acti +fly frontier +fightfor iowa +fal lof +esp en +eritre ans +emil yos +emc donald +em mitt +ear vin +douce ur +don nan +don ation +do en +dj akarta +design lab +del tad +del lo +damas us +d under +conse jo +clock maker +cl ancey +cic lo +chis en +capp elletti +boot legs +be iner +bar bat +bagh el +bac kedby +bab alik +baarbaarde kho +avic enna +av ner +arach ne +angel ababy +ad ss +ad ry +... ðŁĺĴ +ðŁĩ¨ðŁĩ © +ö k +zo ethe +you will +ye at +y alu +wn v +win ched +water beach +wan k +vibrant gujarat +ven ner +v mm +ut am +university sa +un buried +tourisma us +tor valds +tor reon +tit led +timber man +then ats +th aven +th april +syd filmfest +su arez +sthe best +stepp es +ste ier +sr kfc +spin master +sol aria +sleepy hollow +sj f +side effects +sh att +school book +san ko +samo thrace +sab ir +sa wah +romantic ize +ramesh waram +pur vi +preten ses +pow ter +perri go +per plex +pe si +pe akers +paul brandt +pastu re +pang eran +pablo picasso +pa xon +out burger +oun ders +ou di +occul tism +noth appy +nh tsagov +n nc +myfox ny +mye yes +mun guia +mori bund +morgan field +mis k +med calf +mar ans +mal en +lover ly +laure us +kinder hook +katherine kellyl +kar amazov +k shat +ju ist +jeth malani +jad oo +j ist +j emma +itch in +interior designers +in capacity +hoop z +hemen way +harri ss +hari pur +happil yeverafter +hanson sauctions +ha fer +gon ç +go tyourback +go bots +gin ning +fury fc +est oo +energy star +don keys +di ast +desi rables +dave doyle +d hoop +chil oe +cedarssin ai +car so +callaway golfeu +bwa haha +bri zen +blue origin +blu sher +binghamton u +betty who +bernab e +benjam ins +ben sen +beg on +av us +as om +arqu ite +arielle kebbel +anatom ist +ag ga +ad re +acro ce +ab dali +! ðŁijĢ +ðŁļ´ ðŁı» +ðŁĵ IJ +ëij IJ +ê ½ +ãĤ± ãĥ¢ +âĨIJ # +à¸Ļภķ +ÅŁ a +zdrav kost +yan ka +x tend +womenin horror +winter halter +vikram bhatt +vascul arization +un us +u media +trump s +transliter ation +tos sup +thi eme +thames water +tel lement +tal ita +susan cain +sugar creek +su ar +stylist ically +statue ttes +star ker +sn ice +sil ay +semiah moo +seam stresses +ri ma +rei ver +rat m +prosthe ses +pre date +pil chuck +photo sportnz +peter mansbridge +peder nales +pe led +ou ris +ole sya +northeast ward +night call +neur oradi +mun ns +mor ad +miss india +mesh ach +mcgee han +mass aso +mark zuckerberg +mare lli +mam baday +mah mou +m sic +m da +lo wey +lo ks +limerick clg +lily whites +le pr +lake george +kit siang +kay seri +kap aun +kam pa +k je +juli ed +in sensible +impac thub +haaretz com +h nb +green keeper +grac ey +gin tonic +gi dea +gemeente museum +gam bill +fr cs +flores ville +flit croft +fire bomb +fahriye evcen +f ch +exal ting +eu foria +el st +el ord +eir com +dowag iac +dog sitting +discur sive +depreci ating +daily motivation +cur tly +cuck field +coroll ary +colle tage +co balt +cho isi +chil las +chiangra i +ches ed +cav en +ca yoglu +bisp ham +b unions +arch dale +arag ones +anu mber +and only +amor tization +ambu shing +am ania +agu erra +adidasu prising +acces shollywood +abdi rahman +ab ow +ãģĨ ãģ +âĨ ¬ +zi u +zach ry +z ole +wk bn +wicken heiser +whe ed +weed ore +wa aaah +visitu tah +viny ladd +vic mignogna +vand ellas +va vel +usd chf +tu lu +trust towbars +tow trusttowbars +tobe your +thetribune chd +thereal joebob +thei acp +the pit +tanu ja +star bomb +sr h +snow dog +simco ecounty +shun ted +shaw tv +sh off +sarcopen ia +ru sts +roger io +rodolph e +ro skill +re dedicated +pron ovi +press news +poo h +phy tic +petro c +paper clips +pa wned +p tb +p sch +ouss ama +occit ane +new construction +neel ofa +nd cs +nai docweek +na den +musician life +mu hr +mount lake +metal detecting +mc nay +marque e +lymp stone +lisam arie +lewi sp +lang sford +kwest adakar +kramer girl +kati ep +jes see +jae jin +is may +im pri +ido lish +ho que +hail storms +goo k +goo domen +glimp sed +gio conda +gi path +gev rey +furry tails +fox baltimore +for sett +foll ome +far chitecture +f th +f dc +ey ring +es af +endor p +drumn bass +dri vers +dre p +do dy +dispar age +dilopho saurus +dbel twrites +d for +co scarelli +chi quis +cha oyang +celebr ants +black diamond +astro physical +assun ta +arkra zor +aristop hanes +archon sec +aqu is +apple pie +ance l +amazing grace +all ou +al sea +ak ert +adjunct professr +abo lishes +a pod +a anchal +.. ;) +âģ© : +y anno +ximen aduque +wro e +went zel +weap onize +water shed +w tvr +vel oce +u idaho +tweet ad +trigon ometric +tou ght +thyroid cancer +they callme +thermo polis +ther ington +space apps +snow dogs +smither man +shami sen +ser kan +sch outen +ry ce +roger stone +ro sko +rep ousse +real gilbert +re offending +racha ele +ra hab +r pharms +qui o +pu pae +presbyo pia +petr cech +offer ta +o fro +notim pressed +nor is +nil and +ne pt +natalie portman +myfav murder +msla urynhill +mil spouse +mende sarmy +mbio journal +mati ang +man k +luke pasqualino +lope zo +lo ge +le kh +lam bat +lago di +la jong +ko stov +kee sept +kay sville +isol ationism +innov atively +immuni ze +im hoff +idi omatic +i ves +husk orkut +hu ds +ho bart +har tofdixie +h dcp +gram ado +gir dles +gau hati +g slv +french ay +four che +for sa +fin don +film score +fici ent +evil twin +evan halen +eisen man +dx c +doris day +donat elli +dmy tro +deepender shooda +davi dax +cur fews +cryp ton +crank worx +corning ware +common est +commi sioner +coent rao +choose kind +choice summer +char gé +centra irl +career goals +calder ón +c tec +by re +bur styn +better future +bern inger +bel ka +beaver dam +b how +aptac sm +aphili ppe +amp ly +ame ans +am alive +all meansall +ali ers +ail and +a indonesia +________ _____ +ðŁĺ¹ðŁĺ¹ ðŁĺ¹ðŁĺ¹ +ðŁĴķ ðŁİ¶ +âĸª âĸª +Ùħ س +za atar +walk men +w ör +vis u +vin ni +video in +val ry +us outhflorida +un corrected +uma ir +u oe +tribute to +transfer talk +therun way +thebig issue +ter zo +ter ious +silver lining +si ss +seattle u +sarato gas +rit eaid +rent ola +rasal khaimah +rap allo +ple sio +pit v +pet stagram +pap adi +over by +or bo +oo st +onda atje +of change +nun chaku +nottinghill carnival +nc gov +natali ya +n ourse +my nottingham +musici d +multic ast +mobile first +mm j +mitho on +mirzap ur +mck er +mam ou +m key +luca still +lo siento +lin ate +letter men +lec tura +le maitre +kra kowski +kol usola +kim bell +kill bill +ke aggy +karl towns +ka hr +k pf +ji ffy +jam sil +iwe ala +isti klal +ingex tinction +iac eae +hurst bourne +high jump +hi miko +he ilig +goj evich +gly fada +gen n +fluor ine +fair head +epi stol +eon ni +easter ners +disin vestment +din of +dhivy adharshini +cre asing +cod ling +chri si +chees man +cer vera +cd tv +cardi gan +bread winners +bonni es +bon nett +bne storm +blu cher +black alicious +bla gojevich +ber thel +ballin robe +assn at +ashok selvan +anu ja +ambul ation +akal amusic +aho i +academ yof +; )! +! "# +ðŁĻĭ ðŁı¼ +ðŁĺĤ ðŁĺį +ð ĵ +é Ĺ +å®ĩ å® +âŀ¡ï¸ı â¬ħï¸ı +âľĶï¸ı # +âĻ § +âĹ¼ ï¸ı +اÙĦ ÙĬÙĪÙħ +youn an +yom kippur +wv lt +wies ner +white plains +when callstheheart +wee der +u loom +traver so +to wolfpack +teuk ury +ten jin +tele x +summar ily +stat work +speci ous +space ksc +sof joy +sis back +shen k +shark skin +sha ikh +sh oll +scho oners +sal mahayek +sac rum +s beach +rose anna +ride along +ricky skaggs +ri blets +remb ert +realkid poker +r bb +pub blic +pro le +pri ley +pp is +po ha +os setia +om ms +o ker +ni ketan +ni ghted +ng media +nam iss +my friend +mu ere +model ers +mo ssa +militar ised +metv startrek +mel amed +mc fee +mary queenofscots +madein uk +lucapas qualino +lit le +lim kitsiang +letthem stay +lark field +korn heiser +kn wn +ju bin +jigarth anda +james the +j ne +j gd +io st +inter missions +ingu inal +incarcer ate +in offensive +ideolo gues +id k +icahn mountsinai +hyper sport +ho dag +handof hope +hand anovic +han eef +ham dard +h cfc +guar ani +gu mmy +gratu ities +grand ly +graci ano +googl enew +gi ons +funny man +french toast +explore spaceksc +deniz li +de wx +davedoyle mma +cr pg +cle ang +chang zhou +cathe terization +catch pole +cake shop +ca rel +bur ys +bug fixes +bray ford +brand shatch +bo che +bi dens +bard sey +baeday alden +ba asha +b mb +ay et +athel stan +as cat +art smia +aro ssa +arkrazor backs +arc angelo +ar lon +af ball +> = +ðŁıĨ ðŁijı +íĹ Į +âĺºï¸ı ðŁĴĻ +ม าร +ил ан +zerot olerance +youtube channel +ye wande +yarra wonga +war sash +vote katniss +v neck +v mf +under passes +ulster gaa +tremb led +ton is +ther ave +theq arena +thau vin +sym metries +superf icially +strike apose +st azione +speci alist +sp su +skam italia +sheremet yevo +sgt pepper +se journal +salt ford +rupa huq +roc as +reuter spictures +report ing +ren k +redu cere +red panda +phithe takappa +p inet +nowon air +neu illy +nephro logist +mo tw +mike will +mide ast +meadow dale +mar kin +man teno +mal len +mac ario +ma sika +lovel ife +long well +local beer +leed smot +lay field +kom arov +ko ech +kitak yushu +kenny rogers +ju ho +j ent +i var +hira eth +hemer ocallis +har r +happybirthday srk +hann s +ha ass +green juice +good ness +galve ston +g ll +fru g +fou quet +fo aled +fi ma +faf ner +en gie +en feld +emascul ated +easthar lem +dn ssec +di stancia +di siac +degener acy +dau be +daphne oz +cloud land +chy stryder +chad mendes +cal trans +bre vi +book bub +bobb les +bis nis +big mouth +be za +autu m +ask dr +aldubarkad spreparation +alau ddin +ðŁĴŀ ðŁĺį +ðŁijĢ ðŁĴ¦ +⾨ ðŁĮĻ +yuru yuri +yo ong +yo gaf +worldof wonder +work work +whiskey town +wall work +vol stead +verdic chio +vat anen +un match +typi fies +tte okbokki +trou ville +tribe of +tre va +tra g +tom parker +the style +th ilo +te aand +su jit +su ic +su bu +sow den +small batch +simple pleasures +show masters +short stops +ser gent +secul arist +scor members +sap sec +sand ag +salli saw +ryan serhant +rubin report +ro te +ro la +richmond kickers +reza aslan +revol ved +reha shing +reedtimmer accu +razor smack +ram lee +radhar ani +rad hi +quali es +principal sinaction +pri ed +pre condition +pen so +out sell +ny ck +no zawa +nicky morgan +ni ka +neuro physiology +neeti mohan +ne dc +natural light +my lifeis +mu c +mr tom +mom in +mo je +mi ette +mdcps north +mckel din +may le +marr iner +manoeuv ring +man sel +makesom enoise +m itali +lo quat +liquid mayhem +lill ington +la veau +kulbhushan jadhav +ku lim +khel o +kala handi +julie chen +jud icially +jol anda +jay akumar +it sti +indi v +in saan +in fi +ick enham +hms qnlz +he res +hal abi +gr ackles +go di +gn an +global tv +gha jini +gang ed +gan ti +folk festival +fo aling +flo gging +fis c +fight forthe +felicit ations +fa ille +eri reland +emmy kinney +eiri k +ebony mag +dor fer +dg love +de man +dat en +dar ingly +dap to +collect sideshow +classical guitar +chichester ft +celebr ates +car lease +bund chen +br yer +boxer dog +blog con +bi modal +ba ati +arra yed +app s +ah it +ðŁı · +ðŁİ¶ ðŁİ¤ +ðŁĩºðŁĩ¸ : +ðŁ§ ¦ +íĶĦë¡ľëĵĢ ìĬ¤ +åŁ İ +ãģ£ ãģ¦ +र ह +Ê ³ +} : +yoshi moto +wn du +white party +weare mkto +urbant vuganda +transform ers +to death +the following +teddy b +taylor wimpey +tax slayer +tas lim +tableau public +stock photography +star i +ssur gery +soli dus +simon etta +sathletic shoes +sarah millican +rv h +russell p +ro hi +reign cane +realmatt lucas +rd bms +raku go +ra fat +promo si +pepp apig +patron a +pam per +or un +nor ia +nct fanart +nc sl +mc swain +mc gui +mandre ll +mal ave +mag ics +lud mila +logi stica +lipstick day +league mag +latin as +las well +lan ni +kor ch +knaus gaard +kid son +kal t +j fw +it tuesday +is mart +ing gi +ind itex +imagin atively +ic s +hoo ge +hf pa +halo ween +groove shark +gol dy +go jhl +givingtuesday ca +g weedore +forever leedsmot +featured concerts +fan u +f ci +ex clu +ever after +equ atorial +eg more +dup online +dong saeng +dirk gently +di ddy +den ter +d printers +d hole +cush nie +cruci al +cre tech +cre sa +cliffcentral com +chapter house +channing posters +career teched +cardcaptor sakura +car ney +ca jal +c wre +breath lessly +breakfast club +brachy therapy +bol in +bo sw +bay city +asi ad +arkan san +arie ge +andre wn +aldubhappy baedayalden +al ward +ahu bble +affl alo +ad yen +ach aille +^^ " +? ...... +ðŁĶ¥ðŁĶ¥ðŁĶ¥ @ +ðŁIJ¶ # +ðŁĩ¸ ðŁĩ¦ +ëĦ ¤ +ãĥ¬ ãĤ¹ +z te +z quad +z atar +ye ileen +wide field +wearethe arsenal +w ru +vo tem +ub hai +tu tta +trump lies +tro ve +travel card +tor cida +toge thers +tmb g +time zones +thro cks +thre epenny +thisi sac +tbin chat +studi ously +stagi aire +spro m +son et +sit z +sh j +sel ondon +scitech museum +sai shopkins +ricko wen +raw ski +rail freight +rachael ray +qu ade +purpose tour +pu h +prehen sile +pharmacokine tics +persian gulf +per cale +patient sfirst +pan cas +pab lito +orland ounited +o chi +nv g +next week +ne gie +ne ff +ne bext +morgan freeman +mix e +minnew aska +me p +mat aji +mashi rafael +ma atv +ler os +lapi dus +kh iri +kaz mir +jann atul +ja que +j vp +ishqba aaz +irrit ants +ir m +insi debates +imperme able +im mobility +higu chi +hallow en +gul food +grease monkey +grandcanyon nps +goto southafrica +glass work +gan apathy +g ses +fu dgy +fo gging +flee ce +fi ord +falcon i +fab ra +entom ological +em ami +ek iti +dx cc +deno sau +cyber stalking +crohns disease +cro es +cn mi +clapper board +chris thile +ch isol +cf ca +carib beans +ca zares +bo vell +bed well +bb w +barrow afc +ali sta +adhe rents +ad han +ãģ ¥ +âļ¾ï¸ı ðŁĴĻ +zelda thon +ze ze +wro th +women rights +wel ches +wald wick +ul rik +u iw +the predator +tax march +t song +süd tirol +supply chains +supere xclusive +sub frame +stoichi ometry +spe akin +som ato +so yo +sl fp +sky rockets +sh azz +sam po +sa che +rochdale hornets +rid wan +realmike wilbon +r ce +pu sam +princi p +port patrick +pl x +pe ve +patriot league +pa ston +pa sen +pa har +outw ard +oop sie +ol ero +o sak +nom es +no bi +nel les +na ipaul +multi versity +momo iro +mo sco +mit ter +mic u +megat jp +may aguez +mar ucci +man gusta +m world +luth ria +lock step +linn he +length wise +le sher +la presse +la kers +la barbera +kom ar +kla i +kasab lanka +ir um +ir lemb +intern day +inadequ acies +ima genes +ich kov +icalli han +i fixit +hulk buster +honi ara +homony ms +home staging +holly woodland +hill day +guy ane +gun ton +green sborough +gordon hayward +good girl +goo bers +go guins +glen mont +giz mo +gi velife +ga aclub +for ni +fish wick +fer l +fel tman +ethan vansciver +erne sts +er agon +emilyos ment +ella henderson +ee zer +ed wyn +down turns +dont crack +dd ino +day stil +dav o +cra ins +consumer reports +conspiracy theory +conqui stad +colmen ares +coach d +citi ess +charge dup +chal ices +cease lessly +ce tt +cav ers +cake and +br angelina +bo kan +bil ty +big star +baz il +az l +az ha +avail able +atx weather +arca ea +anthon yo +ang ga +aler mo +aid i +age uk +ag ganis +adekun legold +accou stic +,,,, ,,, +ðŁĺĬ âĺºï¸ı +ðŁijįðŁijį ðŁijįðŁijįðŁijį +íĥľ íĺĦ +å¯ º +าภª +ü re +youn ghollywood +y ia +with me +wi eden +why wellington +well played +we cker +vier tel +vi var +vas antha +vall one +vad hana +u kip +ts ss +trouble shooter +tre ece +travag anza +tomas berdych +thon dan +thal amic +thack ray +te du +stade toulousain +ssc s +sor bus +social protection +sme m +sho tover +seen u +rho dy +read yyy +pur éed +post codes +perce val +pawh uska +patti smith +over dressed +oui met +oooooooo oooooo +ollantay tambo +od da +occur ing +no aas +mun ic +modern fam +mob in +mk p +missing no +mik ko +mi sen +mayweather v +mate er +madhuri ma +ma sin +lough nane +logan square +lo or +les seps +lab oured +la font +kra h +kad jar +k tx +jos é +jon gh +john lock +jeff dunham +ist i +in yc +iheartt ally +ho ima +hedley online +hau denosau +gro yne +gor inchem +goo ssens +food city +fau zia +exp consulting +expconsulting es +elem mathchat +egi dio +edwyn collins +eat well +dym church +dof the +detro i +den z +defer ral +daphne caruanagalizia +concent rix +comorbid ities +cat elynn +bt ls +brid well +bra wijaya +boy ar +beli z +barro wraiders +bal dini +bai jiu +awa ited +aw tg +aw aki +auster litz +atra de +archae opteryx +adjudic ators +ach oli +a hy +ľ ëĵľ +z wave +your home +wo ahh +winning wednesday +westvirgini au +wan ge +wa sif +vi elle +vec tored +tx politics +tomor i +tin chystryder +thenight manager +theatre uk +stur minster +southwark cathed +schmal z +sarban es +sant illan +sam l +ring mer +ri et +rath gar +rant oul +radhamohan bjp +pun ked +planet comicon +phan tic +paul polman +os am +oet ker +o es +nott age +ne ven +multi use +mon agh +mira beau +mille miglia +micro biological +meetthe artist +medi agu +loe wen +l sr +l sh +ke aney +ka ÅŁk +jag jit +i dig +hex um +haz ama +gou ter +gentle mens +g sfc +fra sca +fr ö +flower stagram +esc ro +ell inger +ed corlando +dro oping +dor mice +ding er +dies fc +de balt +debalt seve +daw yck +darao briain +d age +co hosted +cla u +ci alis +chocol aty +chin may +cac ia +bret bielema +brahman yam +bott en +blanc as +black on +bla d +bey ers +beir ness +bab bs +anne cy +angi er +ana huac +ale gg +agger scricket +ag lew +aer u +âĮļ ï¸ı +zap ruder +z burg +xx xiv +vir gina +v ong +that boy +tele casts +tc margate +tar di +sun ye +su er +stani er +squar tet +sickkids news +si mad +shoe bill +sepul cher +sarahm gellar +sach ems +sa ura +rich woods +ress ources +real sway +reagan omics +re tellings +re marque +ra ijin +quer cetin +pyro graphy +punkand stuff +principal es +plat oons +pl ari +pin der +oz ge +over populated +ny gaard +neu romancer +nativeamerican heritagemonth +nap aracing +nach t +muriel bowser +motor mouth +mon tt +mo is +mercer sburg +maz ama +manj ari +mal c +m js +lu vr +lin oleic +kwang min +kir n +ju u +japanese art +j li +itso kay +itsmohit sehgal +ipp f +inag ur +im planting +ic tp +hil den +havean iceday +har by +han cox +gro fers +grand niece +glo p +glasgow uni +gladi atorial +fm drive +fi on +feeling festive +fair wood +f legg +er col +em rich +e bc +dr ongo +defe o +de wolf +de ux +day ang +cycle tour +cur ate +cor avin +co dsall +circuit ous +che ena +cate rer +cart lidge +can y +brook green +boo gaard +bol ick +blue bear +bin ding +bi ms +bale wa +ayurve dic +auto express +app ena +ang ai +alo gic +aj in +agu er +addic t +ad tech +aco e +ðŁĴª ðŁijĮ +ðŁijĬ ðŁijį +ç ¸ +ã ı +âĺħâĺħâĺħâĺħ : +âĺĢï¸ı âĿ¤ï¸ı +zar korel +xi en +wil kes +wfm z +wap akon +wak elin +video tron +vass allo +v wap +us military +un graded +uk ho +tusc umbia +tsumt sum +toro company +tool kits +tomar ket +thondan kani +thisis lany +ter fs +tang lin +sura u +stock wood +spor tireland +spar sh +som alian +sidd ons +shel a +sham ers +sg vn +sf symphony +selvar aj +seb agai +sant illi +rumin ants +rt ls +rr v +richardy ap +rex ford +qi ong +precipit ous +pat ta +paget paget +over abundance +olimpi ja +nu dged +nu dge +non pareil +noi settes +n ni +musi q +mur rells +mu ds +mon tac +mir s +mingh ella +maric hal +makebetter happen +ma eyoung +ludd ites +luc ban +lou reiro +lo tos +ku mano +kre ta +kha dka +jess on +je sh +jane te +in news +her javec +helioc entric +head rick +hackney wick +h lundqvist +guil lot +grun dig +grin drod +grimac es +g sma +forest fire +fin chel +explor ation +ex upéry +eraser heads +dvent ures +dun g +dor rington +dj tira +deser ters +der rek +cur du +ct buh +cra iova +colle dge +children shealth +caren cro +cal lup +c twx +brock university +br antly +big fan +beyourown boss +ben na +beautiful game +bb curdu +bat kid +barbi ere +backin time +ay sen +as cher +as aram +albatros scam +aire uropa +ag ac +adom ah +ac rm +ðŁĺĺ âĿ¤ +ðŁİ ½ +ÙĦ اÙħ +yassi zzle +wine growers +wilhelm sen +who dini +wer oll +water fowl +wai alua +w shs +vine sauce +vi lest +urban ecology +u ssi +twit ness +tro gon +touch down +techno logic +tar chives +ta eler +sudar san +stump towncoffee +stre amy +spar go +sou ra +sni k +sk ow +schmid t +sam ah +sab atino +running uk +ro gge +public education +pu ber +pri zep +pied ad +p ting +nebra ska +naz imabad +naj ran +mun di +mo ed +mitchel stown +mimi kyu +mil ke +mi yam +mann ering +manjun ath +mac iver +m ten +lyn g +la gat +klein burg +kay ako +jor dache +johnnewman music +john waters +jasmin walia +indiat vnews +iey asu +hu moured +ho fers +ham brick +gurdas maan +great comet +gamer gram +ford trucks +fi lem +fal ck +f ys +f ct +er tel +eleanorj calder +duche sses +drought lander +digital leader +di parole +dend rum +demor alized +demar com +cray ford +cp x +cosum nes +cir colo +calli ance +cal zada +braun stone +bott lings +boo ya +black men +bhu pathi +bestin the +bailey lakings +au fman +aspir a +as lef +ariad na +ar tec +apple pencil +angelcake pics +ad dd +ab mb +ðŁĺĤðŁĺĤðŁĺĤ ðŁĺŃðŁĺŃðŁĺŃ +ë ¡ľ +ç³ » +âŀ ¤ +à· ĥ +ت ÙĬ +zing erman +x eter +wright stown +woo sung +whit elock +war bling +wa chau +ve ctis +us en +ty burn +top dog +tb v +t sel +swim swam +sud afed +spectro photometer +spare parts +space exploration +south ard +smart cities +shi raz +shar an +se inen +scu tt +scout ing +sac i +rubi x +ro milly +rev engers +re marry +raghun ath +ra ver +pv da +ps itt +prescri bers +poc so +po ppo +pl zzzz +pj py +ph ua +par asy +pac em +p nj +p crm +over charge +opening soon +of ilm +o ton +ni archos +ne gin +national bossday +mzansi magic +multi state +midge ure +mb asketball +mathi as +married atfirstsight +mar low +malcol mb +ly ak +kre utz +kiri akou +kinka jou +kei thing +kean sburg +karmal oop +kalam kari +k netz +k alem +james blunt +intra squad +iller time +holo graphy +hi roh +hal tom +gri maud +glovers ville +franki ekazarian +flock hart +facial recognition +everyonec an +ere k +ep at +ec lac +earth sea +dug gie +dub fire +drew lachey +dont forget +do vid +direc ts +descendant softhesun +degu station +daniel marven +dales man +da rena +d nab +cr ary +compac ting +cle wiston +ci ones +ci ety +cat andu +carabini eri +business model +bp mn +blan ck +be ok +b hog +aye shat +apar ra +am th +alkal inity +a peoples +ÃŃ m +yu uka +yeas ayer +xmen movies +west garth +wapakon eta +vi shesh +uss ocom +tu tup +tu mon +tri poto +tor oro +tor is +therise of +thereal russellp +the progressives +terre stris +teo chew +tar ahu +tae jin +stan fill +stag gies +spn famiiy +spectacular nwt +sketch bet +sin love +sho dge +shin ies +seku low +se gui +say egh +sar dana +samanth as +rescu eme +renn sport +refugee schief +re double +rat pack +randy moss +prith vival +pric ed +power lessness +pierre pont +phosp hat +perpetr ation +pave se +parab éns +pa ole +p wb +on duty +official psl +no zaki +no wing +ne wart +na via +mu tism +modu lators +mi hir +marypoppins returns +map maker +madi ha +ma ben +longer term +logarith ms +le amy +lake hurst +ladi ators +ku shida +kate mansi +ju ster +jan ele +j heri +j hen +iso ch +ir leach +inde mni +ichi kawa +iam mr +hopl ite +hank green +gretchen carlson +gine st +ginapu stor +ford ing +fashion finds +fa den +ess ent +en ationalpark +dun given +dontcrack underpressure +dom brov +dil fer +der mis +de very +cynthi abailey +cu lum +con signing +cocor ahs +chortle town +cho ise +cheap ness +ce fas +cc bvb +cal pur +cabinet maker +cab bag +c ba +belphe gor +bag gers +av c +av am +art ford +are ola +anton iom +antal yaspor +and rada +afilm fareawards +ab ingdon +ðŁijı ðŁıĨ +âķ± âķ± +ÑĤа илан +ÑĤаилан д +ã es +yl td +wo er +whit marsh +waldor fa +voltac atalunya +vander hoof +ut me +un mastered +truman sburg +the merry +the hype +tele fon +super volcano +spad aro +sin kers +ser ral +se pak +schön brunn +scen es +sam bit +sal ter +roundrock isd +river way +reali gned +re qd +push forward +pu sch +powder ham +pie man +pi era +pen alosa +oreilly media +on dcp +of shame +o gee +no dui +new beverly +natlib scot +national policeweek +namad gi +n tom +mu du +mor ti +mon ton +min jung +mel bour +medi acity +mcgra il +mc kiernan +mazz oni +martin imonday +mar tech +ma ven +m fo +lliber tat +letter forms +le the +lar aza +king g +kids activities +k liz +judd monte +john king +jere bko +jak un +jab arda +improvis es +i versity +i hn +home theater +hoki enation +hick en +har king +gu igno +gb pjpy +g pw +francis can +fo tor +feels goodman +dragon fly +dr p +dl ls +dhe yar +depreci ate +demon ization +del ap +de ads +dd ca +d hee +cur tailing +culture l +collecti ble +co sma +clay ne +chrono graphs +che re +chas sagne +ch one +cab ras +bren da +bluecol lar +bbc so +basti da +bam bi +ballet day +balder as +bal zer +avi dheyar +archer field +anti mony +anna akana +amo on +ally girl +alco y +albu men +albu maday +ac rum +ðŁIJ¸ ðŁIJ¸ +ðŁ¥ Ħ +ë§ Ī +ç Ĩ +Ø® اÙĨ +мÑĥз Ñĭ +zo ie +your allypally +xian lim +westwoo done +wein man +war fighters +vul pix +un compressed +un acknowledged +tshep o +troglody tes +toli sso +tho tep +thisisp vris +thed appy +the esh +thats what +thanksgiving week +tal er +take backthe +takam atsu +sx ii +suki waterhouse +smol ders +slopp ily +skin health +she arers +shawnmendes the +shar jah +shak shuka +scrap the +scho eller +saveour seas +salary man +run asone +roy c +ri fat +revoke article +red sonja +re bb +rand b +ra strick +ra son +quar shie +pre so +pre k +pot coin +pol ansky +pleasee eee +peter scanavino +periodon titis +pe dley +pat aky +parvo virus +p bhushan +ow y +omi ami +official ghsa +north central +nie bla +nhlon nbcsports +new era +neko case +n tia +muswell brook +mom oh +mik kelson +microne edling +michael wsmith +mer in +mckin zie +mc wane +mark dice +mari pos +mar os +mag adi +ler ouge +le pus +lam berti +kno pp +ki kki +ki hu +ke dai +katheryn winnick +k ko +jon montag +jamiele ecurtis +ir well +infu sion +imp ru +im purity +im par +hy tner +hu ta +hs bc +hoag y +his sy +himm el +hey erdahl +hersh man +heir loom +healthy diet +he v +harts dale +har uno +gro tte +gom on +goel bjp +ge mili +fuzz ing +french wine +free state +fore vs +food park +fon o +fay oum +f gg +dessert day +david harewood +data analysis +d music +cyn wyd +cycl orama +cras sula +cor dele +chag ford +cecil erichards +catelynn lowell +cas u +cas sock +brevi ary +brave souls +boss u +bi ram +bha jans +balmoral show +bal boni +b under +aver e +artscouncil ni +ar ji +an san +an ali +ail eron +agu er +ag ical +aaaaaa and +a versa +ðŁIJ´ ðŁIJ´ +ðŁ§ ¹ +ðŁ¥ºðŁ¥º ðŁ¥º +ðĿĹ ĺ +xxx vi +ww mtnews +wood thorpe +whar ves +wel over +wag ener +vsco de +very proud +un justifiable +un burnt +ue matsu +u ef +tulip siddiq +ts ys +tri shul +trampal ji +tol tec +teacher prize +tai shi +syn crude +sunshine coasto +su sty +south offrance +sha aan +seper ated +savat age +sau veur +sam mies +sal az +s dag +ri bet +re twit +re ines +queen bee +pun to +pp ke +persu ader +pay n +pantom imes +oun try +or ko +open mic +only you +ny stag +nairo bian +my jasper +mor ny +mor ioka +michaele mann +mean smore +man ha +loy ally +loc atie +lam pre +la thi +l luis +king scote +ke mer +kaz imi +k naw +jakeand amir +it uri +in competency +hispanici magines +hen rye +he dd +he aping +hair port +ha sui +h sct +gur um +glo e +gh ard +gggg ggg +gate am +forest school +fle te +fla shover +eschen bach +erd rich +ej ad +eden derry +dy y +du su +du bc +dialec tics +del acor +defi lement +de sus +de ob +dan ede +dal arna +daddy shome +cross keys +cro mer +concili atory +col po +chri spine +cham pe +c ation +but true +brock ington +brecon beacon +brad ner +blur ted +blum spew +blom berg +bha gal +ber an +bel grad +baf tag +at allah +artic lev +arru da +army rotc +an tt +am mo +alit rophy +alam enti +aed an +ad w +ðŁĺij ðŁĺĤ +ê¹Ģì§Ħ ìļ° +м оР+Î · +z na +yun o +yu da +ym outh +working mom +wild water +whit lock +wedding fair +w woof +w sn +vo dianova +un seemly +twitter storm +tf h +textile design +t dx +straight ness +soci opolitical +shek hin +sh ung +seabour n +se aways +rock away +re zende +raj shah +quant cast +psychopharmac ology +pietr angelo +phil odendron +phe x +pengu inte +pend ence +peer j +paho kee +pa pe +od awara +net books +ner gal +neh gov +mtvbase africa +mill street +micro scale +meh wish +max ence +mash rou +mand ingo +lu ers +lokay ukta +labor atorio +kalon gan +kac z +jim inday +jan a +jagmeet singh +jack knife +inside ferrari +in hand +i vies +hi mb +hatchim als +har ang +gau mont +gar bled +fiz dale +fig tree +fidd lin +fanci ulli +fal tan +emily y +e bbs +div yak +dis da +davidax elrod +d lm +cle ws +chri qui +chatur anga +cedar cove +catch oftheday +bush whacker +building bridges +british birds +brak pan +bo snow +black swan +bi sha +bel bin +bal lester +bab bb +ase ema +am z +am diabete +am bia +ag ito +acaci as +% âĢ¦ +ðŁĺĬ ðŁĴŀ +ðŁĺĨ . +ìĺ¤ ìķĦìĿ´ +æĿ ¥ +æ ¶ +âĢįâĻ Ģ +° ! +xxxx xxxxx +xim enez +wyn berg +wom bat +wi ed +wan king +viadu ckworth +up lifts +ulla dulla +u ea +twent yman +traut mann +trade centre +towno fficial +top cat +to losa +theori zed +thenewe uropean +the torocompany +ted to +tac it +t mux +student debt +spn chi +serv ais +sen zo +saw ada +sale sians +sal twell +sa q +ro do +ri spoli +reel sofficial +re join +re home +ram lila +rak ish +purple day +pre fabs +plym stock +plot lib +pi azon +petrol head +pav on +palm tree +pal med +pa sek +p wu +ori go +one planet +nikk il +nc ad +nas ahubble +n the +mobb deep +mo bike +mira sorvino +mha iri +mechan ised +mb mbam +matta rella +mat z +manz anares +mall ari +mag dy +lo veridge +limb ed +le panto +l pm +ko suke +kelly sue +jun in +jay apura +it sco +io les +im monen +ici mod +heu res +heteronor mative +helpto buy +har tog +gu yot +gly col +ghu rair +gh in +ger tner +genoci de +gain ax +fri erson +fitness journey +fer mor +feature ttes +emc gee +el chapo +e kin +dor rell +don air +dog gie +der mer +den eb +de schi +dali da +criso stomo +council members +cornelis sen +coo lie +colli gan +codi fied +clan destin +chuk ar +cho wa +chen in +chat ard +char vet +char ged +c mma +bute town +buech el +budget travel +bel gaum +bb cred +bar ce +bah ri +bab alola +az ion +awal sh +aus stellung +as rock +alvar o +aless andr +akademi ks +ai wa +ahmadre za +aditi rathore +ðŁĻĮ @ +ðŁİīðŁİĪ ðŁİĬ +ðŁ¤ĺðŁı» # +íĶĦë¡ľëĵĢìĬ¤ x +yar ov +xpla p +wvprep fb +wicker park +wa chowski +vinay avidheyar +var os +va ide +us ace +urvashira utela +upheav als +un learned +tre i +tre as +toread or +thedavid crosby +the cloud +temple man +team on +tb ayne +tad lock +swiss made +stu mbo +stu arth +squ ill +spaci ousness +sol arec +slopp iness +sle ben +she x +se be +roy lilley +re kka +re ev +raz van +ran maru +rabbit mq +qalam oun +pre bble +pi at +perfor ate +pat ara +par ga +pam lico +pam ilya +over steer +onelast time +o tun +ni mrat +nfl kickoff +nest lings +my name +mother languageday +mini o +meyer hoff +men dips +mar iss +mal formations +m prnews +lyth goe +lopezo brador +le lia +le kha +last nite +la duke +kyr sten +kv ass +kur gan +ku kul +ks giving +klu b +keny aredcross +jou et +jewl ery +janasen ani +it sat +it oh +is ara +interce ding +inge ducation +ic ron +i priyank +hebb al +hand sup +h he +gyeon ggi +gor rell +global new +gig wise +garni delia +fun belt +fon taines +fold out +feel better +eu dat +eri elive +english town +elph ick +ed guy +eaz ye +eagle sham +e one +demonstra bly +de ya +dab bles +ctv wpg +cl on +chu ter +charlied aniels +cf kargentina +buñ uel +body language +bleak ness +beso in +bent aleb +beat it +be ab +back off +b nai +b co +b chockey +avec chia +auk land +astronom ically +as wang +ar ric +apilot seye +api zza +amar ina +alph ac +ad lv +achi mota +=_ = +ì² ľ +åŃIJ éŁ +z ary +yy cevents +yehu di +wol man +wild wednesday +wasi kowska +visit goldcoast +vi sting +unity tips +techno logie +sul phide +stre at +sovere igns +shar n +sh ats +seven logics +seton hall +screen printed +san cha +sa kuma +ra ymer +pu review +pre amps +pr cc +poké mongo +perfor ations +pe wee +pare jo +over man +ot z +oh the +oh saasports +mohit sehgal +meh med +mcfad yen +mark lanegan +marc garneau +man j +madri gals +luxembour gish +lp wan +lookat my +life ontheroad +kin dra +khar ge +kei ichi +kai grun +kaigrun witz +isu net +insinu ates +ii ed +ih me +hewit son +hc sd +gro tta +go wri +gau ck +gandol fo +gab c +g ach +fro mn +forest whitaker +fe k +family medicine +energys aving +ec sc +ear wolf +dont nod +dj mix +dis ki +dir lingusamy +dand eli +dainik bhaskar +cork city +con cisely +college basketball +clear ly +cla yo +chu giak +cho sin +chi kin +care for +brunel uni +bio systems +betibachaobe ti +bach rach +az ami +at socialmedi +ash elf +as cott +as cal +an tae +am rav +alpham ale +alli want +alle go +ak sel +$ \ +ðŁĺĽ ðŁĺį +ðŁijģâĢį ðŁĹ¨ +ðŁ¤ Ĵ +âĶĪâĶĪ âĶĪâĶĪ +zon ne +white tip +what about +weing art +un ceded +turner prize +times live +time scale +ther os +tg k +ter centenary +talyl lyn +syl viaduckworth +swing arm +substance designer +su td +su mber +stor rington +space govuk +sp et +sl bc +skate shop +sharepict chibi +sent ries +seewhati didthere +san ssou +sammy wilk +sam bha +red row +re power +ramnath kovind +profun dity +poly phia +pero gies +per vad +pan kow +o estrogen +nor tel +no break +niagar aparks +nh mla +nc se +murrumbi dgee +mo val +mnc tv +mis matches +mi ket +mex chat +mat plotlib +marco g +man nu +malacan ang +ma stung +log ers +lj mu +lis sette +lign um +lan cement +la gran +kristy na +kristiann airn +kam ila +k du +jyo tish +jud gen +jin xx +itu n +itu ation +ipp atel +intrigu ingly +inte bnlditalia +im ple +ice music +hun ziker +hi bees +hend ren +hd k +haver straw +h ico +gr r +geh ring +gar dot +foun taine +flo ret +fertil ised +fer net +felicit ates +fat rophy +etsy sale +epo ca +eh v +earl sfield +dwee zil +dunhill links +doll houses +dis respects +digital sales +dietiti ans +de spots +de shaunwatson +dak u +cr tv +count mein +const anta +co rella +clin k +chuck wendig +bri sco +blac keyed +bhak ta +benbe cula +ben nion +bar go +ba sto +astralis gg +andrea petkovic +ame z +al awine +afoo tball +a issa +:' ') +.. ??? +!!! < +ðŁijįðŁı» @ +ðĿIJİ ðĿIJ +é Ĭ +ãħłãħłãħłãħł ãħłãħł +âĻ¥ ~ +âĺºï¸ıâĺºï¸ı âĺºï¸ıâĺºï¸ı +öl nir +ê t +~ âĻª +yu bikey +yellow fever +y ato +wrigley ville +wr fc +williamand mary +wh arncliffe +war mest +wang chuk +wall dorf +wa ju +urban ity +up ending +trach tenberg +to sachat +ti ar +tho orn +the tls +te fillin +su in +stiff en +ss wiss +spru e +sol la +snow cap +snoo ks +skyblue fc +silk screened +shi rob +se bright +school sport +sarang ani +sa po +revel a +re quote +ra ppe +r ó +pyrene an +pend ine +paul k +par go +panam acity +painting silove +ot an +order now +olivi ers +nws seattle +neuro toxin +n scorp +movietv techgeeks +morning coffee +mor tales +mi ral +me demb +margare tha +march itec +mar cano +manz ini +lion sclubs +limp bizkit +ker pen +kel mscott +jjab rams +j atta +itv wales +ici m +i septaphilly +hu eco +holm strom +ho sein +ho ola +hit c +hi ley +hat ice +happyear thday +gurmeet choudhary +grown ish +gro aned +go canada +ger sen +gau cher +gar bag +gango tri +fu jitsu +foo bar +fire hawks +fer dy +fat berg +far rand +face plates +equin or +epp endorf +edchat nz +dur m +disch em +demol ition +dee z +copper belt +com pres +colored pencil +cog burn +clinton fdn +chisol m +cedarcove tv +cat zingano +can son +cam ba +brant daugherty +az aad +austin isd +at ours +astro boy +asak ura +ap ier +annual report +and dean +amal aysia +alphabe tic +albi rex +ahed tamimi +aden tro +ad har +abo tt +ðŁij©âĢį ðŁı« +ච½ +à ½ +york sambulance +yo cum +yin z +wye valley +winch more +westpac stadium +weather caster +water marking +v precords +upthe dubs +uky p +tw ts +trit ic +tourde yorkshire +thesm sd +theori ze +the weirdworld +sunshinecoasto z +stur gill +steak n +spiegel online +sper kin +siri kit +she han +se aming +sc rabb +save hannibal +rosal ine +right scon +ren du +red card +rang sit +rak shak +rac ingextinction +prin toctober +pre ppin +pre cis +ppe al +pow assan +poo ds +polychro mos +pir bright +piezo electric +perfect as +patt an +pat os +p elling +on li +oh sen +nn h +ngw sd +nd ale +nar dini +n infa +n ating +muhl ach +motivational quote +monster high +miam ipd +mer aj +meanwhi lein +lucas cruikshank +ligh tof +leapfro gging +kremlin russia +kan angill +ka or +ine au +hunnic utt +hundred ths +he ger +hay seed +gra byour +fleis chman +fen berg +fa herty +econet zimbabwe +dt by +differenti als +del ma +death valley +cp ca +clear cut +che kk +cer ium +cann ata +boycott nfl +bookweek scot +bby awards +bay ing +baske tof +ball ance +ay on +ar sh +and you +anastasi ya +amé ric +all ying +ali ke +ala ura +al mont +ad zuki +ach mad +a als +:" "> +æĪ IJ +⾨⾨ ⾨⾨⾨ +ච¯ +ઠ¤ +zi zi +zac chae +yom bo +y q +wq am +whit emountains +voteblue to +vol turi +us bankstadium +unil incoln +und mhockey +umbrella academy +uc v +tri mb +tourism week +time les +tile fish +the amy +tart ine +tang ina +tan ith +states manship +snet tisham +smu ggles +smir nov +sky copter +septimi us +schu maker +sch all +ruth lessness +ru ffins +red cap +red bus +randall stown +rad ziwill +powere dge +pol ari +periodic table +pager ank +owl boy +over print +ong ate +no bler +naz eer +national doctorsday +mor well +moe ed +min dyour +ment as +mclaren vale +max joseph +mat tz +mary mary +mapper ley +manu shic +mandi bles +mahal akshmi +ma ek +lith os +lat terly +lam onica +kö nen +konzer thaus +kir rie +kingdom of +king aroy +kess ock +kam aal +kai ja +jonesc apo +jim jonescapo +jackier obinson +ja siri +j bf +ism raceway +is sf +ing space +hou renergy +hindr ances +hay dee +hann is +h fuji +gen erico +gar ak +filli p +fe ssenden +fan boying +enor me +em placement +ec tin +dow l +dont miss +dms guild +divis adero +di sher +demarcom urray +debau ched +cs ds +cont actor +com ingof +cher iton +ce mpire +bo ilies +bo dd +blade andsoul +black all +bbclocal ite +av ito +au riga +asa hi +arizon adot +anton ine +andre s +amar ket +( âĢ¢ +ðŁĴĶðŁĴĶ ðŁĴĶðŁĴĶ +æľĢ æĸ° +ãĢį âĪł)_ +⼠º +à¸Ńะà¹Ħภ£ +£ ¨ +zef firelli +yyj arts +yu mmmm +yar darm +ya semin +x ri +world tv +wild lings +wi dgeon +whel ks +we stra +vir ile +up selling +tru enorth +time forchange +thor ning +the montydon +thai day +th june +tele mundo +surrep titious +substanti ate +su dip +steph breakfast +steier mark +steel heart +st dm +spar ta +shu ja +sha ista +sequ in +se tubal +salisbury cath +rubb ings +rollsroyce cars +re formulated +re ath +quanti fies +pur ity +pro pan +po stre +par abol +op ent +on ye +neil son +neal mccoy +my protein +mx f +mue stra +mr george +mou at +morpho genesis +modic um +mo dic +misidenti fied +michael jordan +mia universe +mer n +melbur nians +mel ded +man tooth +man kin +mac master +lou cks +litt leneck +la sk +kri sto +kpr clocal +kipla gat +ki gali +juan fran +jared kushner +jab ong +idoli zes +idesof march +i the +hun ny +howtogetaway abc +hospital isation +hn tb +hiz bullah +har pal +han sel +gy da +gun dar +gordon stoun +go bows +gerry mander +gang aa +friday focus +fly half +el h +eco school +ea sia +domain name +doing business +desh one +der ic +deni ability +debt free +day u +d itta +cush endall +cun nington +cud joe +cu ssons +cor rode +con gos +christma seve +cat rin +cast ag +carfag no +car ballo +caci que +c saba +buil ders +box of +bom beck +boe ken +beparto fit +bel lotti +barber life +b zh +b fa +autumn statement +ark hu +ard ha +arch a +ar h +analog photography +alban i +ak bari +aeron aut +ad cruz +aa viation +a abb +? ): +ðŁĺİ ðŁĺĺ +ðŁĺį ðŁĺŃðŁĺį +ê± ¸ +ล า +Ñ Ķ +zon go +zakhar chenko +y pn +won do +women sbball +wb tourlondon +wann see +vo well +vig eland +un sympathetic +un ilife +un coupling +um bel +tivo li +thibau t +the arts +techno crats +te ti +tal ente +sugar rush +sto i +st immung +spring has +spirit of +speed art +southern mis +snoo zin +sil ene +shul kin +shu pe +shoul dering +sh su +sen dero +se ery +scare dy +roy moore +ro vi +rann fl +qi yah +poly chae +phi pp +partic k +origin ators +oleksi ak +ne shat +n irs +mur ri +mr porter +morgan ville +mon dy +mike schiemer +mi fi +met zen +me ers +mari do +mar nock +man olis +m ny +luncheon ette +lud lum +lincol ns +le akers +ku bler +ko viÄĩ +kit tredge +killing sworth +ki hara +ju mble +ju cy +jay lin +jackand jack +j hr +ital yday +ish afoundation +ir regardless +ir ani +iono sphere +inter states +iman gel +ifi were +human ness +hri sto +ho ess +hick ox +gv m +goback modi +gill ings +gil key +ged ney +full time +fluoro carbon +fail ure +ex arch +eric hard +ent rapped +elliot ts +el zhi +eh ner +duci dni +du par +digg ler +diff rent +democrati sation +dc s +david love +datdu debp +culp ability +coffee bean +co yl +co ston +clean seas +chak de +capri sun +cad dis +bu ari +bry her +brock ley +bro ich +bonniemc kee +bo ey +blin kers +bel and +bari atrics +bar ad +bagu ley +at large +arri vo +and wine +all ter +ak tien +ag ario +abi erta +ab ike +aad c +ðŁĩ ° +ᶠł +à¦ Ĺ +édou ard +ze ist +yout u +yor chard +y azz +wo bbled +with syria +weather authority +we heart +wan chai +vo ynich +usk asing +un selfishly +un encumbered +ul ly +ts arist +tofthe month +te cla +te americ +sp hil +sneaker pedia +sku b +si kit +short lived +sch rodinger +sas kag +river kings +reson ators +re ordering +rashmi kamand +random ize +push button +pri ons +pre party +portrait challenge +phil lauri +pha go +people with +pee ked +pat man +oste ology +onthe spot +ontari ondp +onair romeo +omni pollo +nuclearbla steu +nu un +nsc c +mor lock +model trains +mccl urg +maxi mization +man ser +man jit +man booker +lud wi +lit as +lisal oeb +lin enews +leop ar +lennon nme +lb su +lag man +la skar +ko lod +kingdom comed +ke uk +kap uskasing +kan eda +kal kaska +k jl +john sburg +idoli sed +ide v +i muk +hind head +hem nes +ha ins +gazette er +future s +fox x +fox man +fore going +fjord norway +first snow +ff ington +expun ged +esp in +esh re +end humantrafficking +en tailed +embarrass your +ele an +dro x +drmike murdock +dow ska +di radio +def jam +deben ham +danede haan +cor darrelle +community garden +col clough +cochin ita +clear out +church man +chil lest +ch aley +cas sel +c siriano +brook sby +bron y +bo cking +blind cat +bi aus +benig ni +bat ton +baskin robbins +bang ko +bag gie +axi oms +aund h +as ba +artu ria +ango stura +and real +amwriting scifi +adobe premiere +absr dnews +abse con +: âłĢ +ðŁĵ½ ï¸ı: +æ ¤ +z auber +workout motivation +wood stown +will hill +we ste +ve don +var ta +under wear +under insured +un gal +u mofficial +tri ot +tou rers +ton go +tiv at +tish man +tic s +ti gnor +the time +the ic +tej eda +te emo +tatu aje +t acy +sö der +sur anne +space museum +sou lection +soci ali +sm ys +sky watcher +sense of +secret garden +sde france +s studio +rho dolite +rene au +recru iter +ran vir +ra oul +protom artyr +proof of +produc tive +priz ren +pretty woman +pe can +park chester +par in +opp ong +music studio +mun z +mis laid +minu ets +michael angelo +mic o +mathis fun +mar wick +mal fi +maeyoung classic +lee anne +l cem +kn h +ki ren +ki am +job vacancy +iwant one +ip cpr +inge xpo +ilu sion +il ament +ifl science +hutch ens +he parin +haryan vi +ha sani +gleneagle shotel +gir lup +ginny goodwin +fu z +frit ts +fin ito +felicity huffman +fan sn +fair hill +encroach ments +el of +e town +dö ner +dow dell +der ksen +de pasquale +czecho slovakian +cox ed +coming up +cholmon deley +centime tre +caz adores +cambi um +bur dwan +bun z +bug ü +bu gli +br amb +bo ell +blu rofficial +black fire +belle vu +beauti fu +b mbf +b cause +augu stan +atal ant +al shaq +airdrie onians +a experience +-- ( +ðŁļ¶ âĢįâĻĤï¸ı +ðŁĺŃ ðŁĺĺ +ðŁĮ¿ðŁĮ¿ ðŁĮ¿ +ìķĦìĿ´ ìĺ¤ìķĦìĿ´ +ê³ ł +âĹ ĺ +ঠ¹ +е л +ál bum +y vel +ww jd +wrath ful +wil i +wal is +vampire the +v ys +un molested +ul ars +tri aled +train to +tok ki +to tty +tn leg +tech land +team red +tar jei +summer bee +steam newrelease +ss ow +soor ma +somers worth +simulation friday +sie grist +sho velling +shag ging +servic ed +sax a +rom ford +roch dal +riv alling +ret te +regre sa +real martin +ras gulla +pru frock +picto graphs +pi ad +phal anges +parachu tist +paddy mcguinness +pa iz +out eni +oo zed +ny arla +nic anor +natu ur +muse i +mu ddling +mu ad +mr teller +mo sis +mitro vica +mispron ounced +mele ch +mechag odzilla +me es +mar soc +mali m +lon avla +lin sanity +le usm +lam elo +lake garda +kir at +kal ka +jo bi +indianoil cl +in brotherhood +hippo griff +hee jin +ham worthy +green spring +gor oth +gil ham +ge bran +gast rectomy +fe stu +es miles +easy recipes +du mble +dj shadow +dennispra ger +d ils +crimin ologist +cork coco +cop i +compa ñ +come stible +chou teau +chi uni +chagu anas +cas ali +bur sle +bruce willis +book mail +black lab +bint aro +benefit uk +ben dera +av or +at us +angu sti +akhi lesh +adam ski +activ ites +ðŁĴĹðŁĴĹ ðŁĴĹðŁĴĹðŁĴĹ +ðŁĮ± # +ê± ¸ +ç§ ĭ +æ´ ¾ +âļ½ï¸ı ðŁijį +yo shis +ww d +wis d +wage red +vishnu vardhan +vis cardi +ve k +universit ät +underthe sea +tin plate +thewhisky agogo +thec ct +the writ +terry pratchett +tech ne +team no +team fortress +tch ouk +st birthday +squ am +slim ited +sli din +skillet music +shopp ers +self defence +saxmun dham +sa ipa +s van +ru sky +rosel awn +rene sas +reen actor +re classify +radnor shire +pupp i +po dunk +plu med +plat ja +pic tus +perpe rennial +par sec +pan chi +p ined +ou saf +ori el +om al +oldd ominion +now available +no st +nga io +neu chatel +nepen thes +nca af +national french +mo dc +mid nighter +micha elek +michaelek lund +mc nee +macron utrients +ly kes +looooooo ool +lim my +li bin +land shut +lab ine +la ar +kron wall +katat onia +kad ha +jonath and +j annis +it stony +inner visions +immort elle +imer ini +ig ur +homedecor ideas +him ley +hert shour +hawk sley +hard point +har perperennial +han auer +gyp sophila +gradu ationday +gow land +girl gang +fy f +franç oise +foli ate +flogging molly +fil adel +enjoy ably +empor io +echel on +e zi +dun cle +dr michael +dp ms +daysof blackcosplay +dau gav +darren shan +d se +cri sing +cri bbins +contamin ates +cn traveller +clipper nation +cinde rel +ch ye +castell ana +carly aquilino +c vs +breath itt +brass eri +boston comiccon +bor delon +blon din +better makeroom +benedic to +bathy metry +bal tz +bac carin +au gen +aster y +asic samerica +as thana +alekh ine +acci esfc +( [ +ðŁļ ® +ðŁijį ðŁijĮ +ðŁ¥ £ +çĶ ° +âĢ¢ ~ +z s +z ette +young boy +yan ko +women supportingwomen +wat l +w bir +virgini awoolf +veer u +ultr amodern +tu ur +trun cate +tru ef +tri pe +tof te +te cn +tape worms +tac tix +ta pley +sut til +strong side +stratfor don +srisri u +spec kle +sp art +sim cox +shannon bream +shal it +sc lay +sam r +ryan leslie +royal visit +rond out +rol lovers +roc codi +reis z +re dragon +rath down +r thk +qu ello +pre science +pen ha +pen do +patt ani +ou thouses +on nnn +oftheyear hma +ob t +nigel barker +new church +nau s +nan tuc +nadiaz anelli +n spra +n mn +mustang nation +multi drug +monster monday +mon ch +mo yam +migrant sday +micro blogging +mel robbins +medi vac +mecklen burgh +me dak +max pain +lun i +lubav itch +lock ridge +liver disease +leed suni +l arios +kil twalk +ken naugh +ke mlu +katsu shika +kat anning +juxta posing +je ay +jaz baa +jal gaon +jac co +ilike samizayn +ide c +hic hi +happ s +h km +h ci +gyna ecological +gow ans +gottacat chemall +good work +gene wilder +g tourney +fu qing +fresh prince +farn don +famili arization +fairground snola +e chev +dul verton +deer hurst +dam ie +cro z +cou par +correspon dences +compe tion +coc ci +chu uya +chin skiy +chenonce au +cbr ne +car na +c cat +bu suttil +box fish +bon jovi +bin u +berk off +be ere +be com +bbc surrey +bai ze +b johnson +astro physicists +aden auer +acqui esce +acqu it +acomic con +ðŁļ ķ +ðĿĹ ľ +ä¿ ¡ +è te +yaari yan +va ghela +use fully +up asana +trudeau mustgo +transport ation +tor ock +ton kinese +ti os +thusi ast +theatre ldn +teab agging +taym or +take a +super powered +sun birds +stru b +stefano gabbana +stand on +sp liff +sp ersons +sp ds +sl int +sidd al +sher pas +roa sted +rid ler +ri gid +rheu matologist +quis ition +proro deo +prophe sies +pro static +prithvival labh +preste igne +perfect gift +peniel shin +paw ling +pan get +osle isure +osiris rex +neon ate +national trust +mrs browns +mgsv tpp +merci an +may nor +mar cher +maquo keta +man by +mall inson +lo kk +lis gar +la sley +la ho +kwq cnews +kir ya +ke vo +k lee +ju se +jebe diah +jagu are +ja ib +ing man +igh tham +iam dr +i osis +hu day +ha yy +gwand shows +gun safety +gov mattbevin +gondo liers +gilded balloon +gen u +gar di +g fd +for hillary +flo ssy +flo bots +feel ies +elvis duran +elrey theatre +edi ger +dri ss +dram as +deton ates +de broy +dad os +d sey +coy gig +chro mis +charge back +chapelhill shooting +canadian opera +cal vet +ca hier +buro happold +bu ton +bru ery +brawl stars +bra ine +border patrol +birmingham pride +beth nal +bait fish +asqu ared +ar ue +aon ach +aldubin italyday +al dia +aksh ar +ablu tions +ðŁĵĮ # +ðŁıĢ ðŁĴĻ +ᣠĴ +zacchae us +worldbook night +wil fried +west king +wat ere +wasi lewski +vent ers +trac on +tony pandy +thene ed +sy re +swe ene +sw offord +super majority +super mac +sun it +suje eth +style by +stu voice +state oforigin +ske wing +sj sj +shey enne +sen ge +school master +sch itt +saf mradio +ro secity +ric kowens +rei vers +r saf +puru lia +prep star +pon tos +photo album +pharo ahe +on at +omni potence +office dog +o townofficial +o cul +native breeds +nam askar +nach richten +my fwc +mor phia +margare ta +ma aaaa +lon gy +lin ka +lang worthy +kra hn +kale v +instac ar +inser m +hyper ventilation +hopen ot +hale wood +hah ne +gre aser +grand tour +grac eville +gon zo +go via +go bel +fun atwork +free mind +forbe stech +fold sofhonor +fiji airways +end ry +emo sh +elly se +elizabeth warren +ec is +dush ku +drinkal sace +down with +dit or +dialo gic +dg and +devop ssummit +democratic debate +dele tions +del sol +death rock +dat acom +dal zell +cute off +compu ter +ci vita +chum phon +chemain us +californi adre +bro ten +bou ch +bosch global +bor r +bon ta +bhatt arai +bhar vard +becau sey +be scot +bal ks +bal ama +bad chicks +ay ato +at rade +as kim +arro yo +agil bert +adam cole +acou sa +ac ist +a eda +ðŁĺ° ðŁĺ°ðŁĺ° +ðŁijĮðŁijĮ ðŁijĮðŁijĮ +ðŁ¤¦ ðŁı½âĢįâĻĢï¸ı +íĶĦëł Įë +ا Ûģ +zeyne pab +zan ski +zack y +worlds best +wool ard +women schampion +wom bourne +williamj hague +will l +wad ham +vari ances +va q +v angel +ukcoach calipari +uconn football +u oy +tten nis +tre sco +to pre +thisi sn +ther ob +terran ce +tam ie +swa im +sun foil +still withher +st aser +spoke smen +spencer ville +south wards +sheldon ian +seac at +saltwater fish +room ed +roman owski +rob art +receip t +re shuffling +rd grade +razor blade +ran j +pyrr hic +play dough +pide mia +par tey +par menter +p batour +ou in +oooo ps +on ald +om ish +music in +mur ci +mo gami +mispr inted +misanthro py +mine ers +me iling +mark land +m smith +liv cathedral +lex an +le ane +la fia +ko daly +kit chee +kir sch +kids first +jan cic +ite mized +ini go +img academy +icosa hedron +human itas +ho soda +hi was +he wes +greatcomet bway +german yin +genuin eness +gentle mans +gaz al +gau zy +fun tastic +fm j +filmin dustry +fbun ational +fa enza +est at +enjo ined +eh den +earth sci +e ffa +drew gulak +dow ni +do ti +div y +der oy +demon ic +cy cad +crowned heads +con text +con nally +clu te +christi any +cf g +catandu anes +canecor so +bt me +brussels attacks +briti an +book art +block house +bett in +balu sters +bacchan alia +bab ich +b awards +ash eville +ap ix +ago da +a eu +- âłĢ +ðŁĻĤ ðŁĻĤðŁĻĤ +âĿĦï¸ı # +âĻ¡ " +âķ² âķ² +ઠķ +ÅĤ o +yemen crisis +whe ath +water conservation +wa iner +vir das +vic ent +viad uc +vermel ho +vent agli +vas ko +vamp y +union station +twin ed +turn again +tunder ground +tu lio +tu azon +tomy future +the brand +the better +th ark +taun us +tal aq +tak acs +t sle +syracuse chiefs +swith amazon +stu gotz +stom orrow +sri mad +sm kc +simpl yaj +shan em +se os +se alions +sanc lemente +s meal +roby ns +rib oud +repri sals +recalcit rant +re states +quar ies +q eh +promo cion +plo it +play listed +pine grove +pin edale +party in +paraly zes +open call +op tionally +offici ates +num erically +now lin +nov itiate +new designers +neer u +ne mec +myco plasma +mister giuntoli +mil stead +marcel kittel +mag ician +m fat +look man +lat ah +lang i +la ville +la ury +ktv tamil +kra vet +kor ona +kis d +ki ai +jim breuer +jax on +indi g +hight stown +hei der +hard wa +ham ida +ha jj +ha insworth +greatday tobe +ge bre +gabbie show +friend sforlife +flori bunda +ferment ers +euro sport +es el +epic urus +engel mann +elo cution +dor tch +dj jazzy +da gher +d á +d illian +cuti eee +cup w +crp findia +consul ta +com res +collective evol +ci dg +chur ns +chumb awamba +char lier +chap eron +cf adden +ce arch +bru mmell +box ed +book talk +bla upun +be ja +bar acoa +back sliding +aver ts +audit or +at gm +apri ze +an day +amelior ate +alo se +addis combe +ab bate +: _ +% ( +ðŁļ¶ ðŁļ¶ +ðŁĺ³ ðŁĻĪ +ðŁĵļ ðŁĵĸ +ðŁĴĻ âļ¾ï¸ı +íķ © +ãĥ©ãĥĸ ãĥ©ãĤ¤ãĥĸ +Å¡ a +wroc ÅĤaw +workplace safety +wex gaa +wakat obi +unc charlotte +u ws +twic ulate +truthor dare +til lett +ti ma +thisi swa +ther y +thanks forthe +tal ley +syco phantic +subsidi sing +stopthe bans +standard bank +sri xon +spring awakening +spin out +sp outed +son ship +si ma +shra van +shel p +seok ang +sak ha +s enda +ro ki +relinqui shing +recre o +re vers +re interprets +ram se +ra so +preservation ists +pp ms +pac bio +p nj +oh pa +ob it +ny rb +nottoo young +nonleagu epaper +nichol ls +new wave +nancy sinatra +n phs +n lo +mum life +mou vement +motor park +mo dak +mo ama +ming kki +mik kel +mick o +mi mar +mi drash +meli ora +mcmick en +match boxes +mase field +mac donald +ly all +leot ards +lasvegas shooting +la hav +kon st +keeptexas red +juli usc +jointeam alpha +jar mo +j ila +inner city +in ala +ig uns +hy les +heartsof oak +hear ses +haram ain +hamilton island +guatem alans +gil boa +gh d +gb hockey +fri berg +flori daf +fe tid +extrapol ation +estac ado +erne sto +eddi evanhalen +dragonball fighterz +dragon stone +div aio +diame trically +df bharvard +decrimin alizing +dais ley +d ites +ch ko +cebu ana +cas elli +carri acou +cardo za +ca pet +bur qas +bru it +bridle way +br yl +bir tley +be toys +bais den +ax xx +astru c +as ophia +as lo +artist center +ani moto +af shin +adam stown +abra sions +ðŁĻı ðŁĺĩ +ðŁĺ¢ðŁĺ¢ ðŁĺ¢ðŁĺ¢ +ðŁıĬ ðŁı¼ +ðŁİģ ðŁİīðŁİĪ +íĶĦëłĮë ĵľ +ë°° ì§Ħìĺģ +ê³ł ë§Ī +à¹Ģ à¸Ńภ+young buck +vol com +ver itas +vam shi +ty mon +twi xt +twe ener +toxopla sma +tom riley +toby keith +tho sp +the de +te tt +tales of +suicide squad +spend in +slo at +sling sby +sky one +sjo gren +school innigeria +rv f +ru ffs +rtx rt +rfd tv +reden bacher +re past +rat as +rai b +quer cia +pu ku +principe ssa +presiden to +po tage +po stive +over acting +or uk +of lu +ode brecht +naruh ina +myprotein uk +mckis sick +matriarch al +mar ito +mand vi +madrug ada +ling ling +kin ko +kai bab +kaf a +k wave +jon favs +je g +janh vi +inau t +im pulse +ilove u +ih in +hoo oo +hoo ge +honey z +heck mond +he ita +hallucin atory +gu zzo +green horn +girard perregaux +gee zers +gadge try +fri son +foot way +erotic romance +ent onces +en trance +en shrine +el mina +ec centr +du mer +domestic workers +dok lam +dj danny +dis quieting +dis continuation +din is +digitalleader sa +diap ering +deleg iti +dav adi +d me +cow dray +copp ens +con tru +clair vaux +cf ps +cav endish +cate chi +car ina +car agh +buster love +boy shoops +bhu pend +aw restaurants +auto body +atlan te +articul ates +arri etty +an tero +amur thy +alde burgh +aic ha +adel hills +academ ical +ab negation +(^ ^) +ĸ ðĿĻ +âĨ Ĺ +Ë Ĩ +yoak land +yl ancs +yakov lev +x terra +wheeling nailers +wendy davis +vintage traffic +vari ate +transdayof visibility +to pley +the gabbieshow +tan in +swimswam news +sven son +substanti ation +stat man +st birthday +sportsday hs +so frock +sixnation srugby +sin motion +sd sc +saraha ines +ro vs +ring land +recircul ating +ray com +rav ings +rail hawks +q ri +program mer +ox chambo +ot ss +on thisdayinhistory +o dum +mo graph +mind set +mic om +mar thar +manne quin +man ak +mac gillivray +lg p +lam est +knu dson +klai peda +kenny florian +ka ap +just because +jose altuve +ivan chuk +irish film +ico splay +i lea +hu ffing +horton ville +he izer +haudenosau nee +hanover ian +h atim +guer rera +gotham ist +goe tia +glyndwr uni +glaci ated +gl hs +git ana +gec dsb +gal mudug +fizz er +fin c +febu ary +fatt ened +explore your +es ns +ep ting +dot pict +dom usic +dis locations +dd newslive +danis ordo +dai fuku +daener y +curb appeal +clear lake +cbc toronto +cat ley +case break +carned dau +carla hall +bye e +build series +book sof +bollywood flashback +bloor dale +az umi +aw newyork +at ag +as oli +as ok +artist oftheyearhma +ard ening +anton ym +anthropom orphi +anne of +anatom y +anast asi +an new +alice keeler +alber talli +alai ka +al anc +accru al +ðŁijģ ï¸ı +ðŁİģðŁİģ ðŁİģ +ë IJ +é»Ħ åŃIJéŁ +اÙĦ Ùģ +yuvraj singh +yi ff +x mend +wood workers +wing ert +wanti rna +wal ch +vol pi +vit abio +virtuo sic +vir na +vici ous +val ise +un availability +tx sen +tx f +tir zah +ting u +time code +ti ant +the bige +tardi grades +tafo ya +super conductor +su ta +stra db +stradb ally +stand byme +song do +sm tg +skylar grey +sa pperton +ry er +rush koff +rural women +recon figuring +re life +raun ds +rajon rondo +port ents +pon tard +pok é +poetr yeileen +pic one +photo weather +patron ise +patric kk +pastor a +pandor a +pand avas +ot sego +omni vores +ok san +ofthe future +numb ingly +nor by +ni mh +new snl +neph jc +my poetryeileen +mus ice +min as +micro waved +micro chip +mev simi +mccar roll +mc nerney +mash ed +mark w +liv v +l annon +ks ss +kno win +kigur umi +kho jaly +k fans +jun gler +ji han +ja quet +ja ay +isti gh +ip sf +inau ghton +hmrc govuk +heme path +have eru +harrystyle slive +hang outfest +ha ddy +gru newald +gou de +gli ese +glam rock +gla dy +gif tw +gar ms +forti ssimo +for girls +fon zi +follow spree +family matters +extram ile +er adi +entro pic +emo cione +ef its +ee es +durgap ur +dom ar +dillian whyte +di bles +derri ere +de young +conquistad ors +con cour +chip sets +chim o +chi vo +chester races +chang jo +can ticle +bur gle +braban tia +bc tv +battic aloa +bas ks +bar ve +bal raj +asympto tic +asbury park +amar illa +ald ar +agr itech +abet ted +> , +ðŁĴĢ # +ðŁĩŃ ðŁĩ· +ëĤ ´ +ãĤ¯ ãĥª +ÛĮ Ùħ +y wc +x ms +wire line +wee h +ventagli diparole +ve ining +v show +tv patrol +tt ol +tri pof +ting les +tho tel +te sch +splat tering +song pop +son dre +som one +slowh and +si man +school room +sch ek +sarah h +sain ty +sad day +rubi u +rosal inda +rig our +rat tigan +radio graphic +public enemy +post le +posse ssor +poi esis +pn k +photo synth +parap sychology +par due +pad mé +op ie +ond kar +o swin +noo h +nj tv +newcastle upon +nag ore +n gala +mu sco +mode ste +mid oriya +mi uccia +media art +matri archy +mary kay +malak off +makeit rain +light up +li ath +lehigh ton +ku lik +kozlovsky d +kiz ito +king swinford +kenner ly +kees maat +kar g +k roo +k love +just transition +jo zy +j fe +innovator s +ing arden +inf ur +hor vitz +holo type +hof meister +gregg sofficial +gre if +go zags +gl antz +gg es +gb ta +g wer +g mit +forrest griffin +farra go +eviscer ated +europe ana +dura bles +du bber +drug gie +dig nam +dan ville +d hir +coa sted +co ill +ckin non +caul drons +cambu ur +ca zz +bou dhan +bio gen +benid or +believe that +ba ai +aw ash +ask twitter +ar mers +anonym ous +ana erob +al the +al pro +al adee +afranc is +action jackson +! '. +ħ ¸ +ðŁIJį ðŁIJį +îĦ ħ +ìĦ¸ ìłķ +çĻ ¾ +à¹ĥ à¸Ĭ +à¸ķ à¸Ńà¸Ļ +à¸Ĥ à¸Ńà¸ĩ +رÙħض اÙĨ +y is +würt temberg +wokeup likethis +wind horst +wim wear +wc n +wc m +wan ag +vow les +viva an +visit korea +vi ed +vel oz +vain queur +uv ella +under sheriff +tuesday shoesday +tuber ous +traf studios +too fan +those who +the ar +teh rani +te pes +summer love +states b +sta ste +spal ted +sno bbish +shar kie +shannon poe +sh ick +se pik +sc m +say id +san sad +sa hu +s dorff +royal airforce +roth ley +remb lant +re share +plu shy +play on +pg itte +pent z +pe aker +paid leave +p bo +over hangs +oo dy +olemiss rebels +ock ham +observ ator +nel da +necess itated +n nuh +morning walk +mol as +min de +mill ner +manufactur inguk +manta she +malcol mn +maj ili +ma ute +look n +le ora +label le +kü bler +ku ha +kis sthe +ki ara +joannak rupa +j scott +ital a +irish cancer +inter library +indy statefair +in berlin +ichin ose +hyper dimension +hs j +houston flood +harrell sllc +ha kim +gup te +grenadi ers +greatesth its +game pro +fu rio +fly spicejet +fire work +fini stere +ffd pgitte +fed soc +fe bs +fanc ourt +enki du +dream wave +don wood +devdutt myth +defer ring +de jah +dand elion +d hinchcliffe +con gra +clemson tigers +ch retien +ch itten +ch assi +ceru tti +ce b +canvas back +call eri +cad wellpark +cab infe +bro aches +bon nin +bk d +bin chy +az ules +ays garth +ay ee +athen s +as mbs +aru ban +ari ffin +ar uk +am rap +all inclusive +all hiphop +al die +air tran +afro basket +abor ts +ab ounded +@ - +ðŁĺĤ ðŁĺĴ +ðŁĴĽ âĿ¤ +ðŁij ² +ðŁ¦ Ĵ +ðŁ¤ ¾ +ðĿĻ ĸðĿĻ +ë´ Ħ +é»ĦåŃIJéŁ ¬ +âĿ¤ï¸ı ðŁĴĭ +âĢ³ , +ÛĮ ر +Í Ł +zi arat +yoshi o +xia oping +x wa +wether ill +welcome tomy +w wn +voc ational +ve endam +v league +usp sa +un questioned +un counted +ucla health +ty le +tor rilla +thé âtre +tas o +taran is +tampabay rowdies +tal u +stro mat +start les +st ca +spend thrift +snu gs +sm m +slobo dan +ser fdom +se fo +scifi fri +science spo +sanger institute +roccodi spirito +rc p +random ization +plac ita +pioneer woman +pau wels +pate ley +palo alton +onetown oneteam +ohi om +obi m +o gni +nw sc +night in +new sm +naz ri +mrdan walker +mothersday gift +monstr ance +mi stery +mashre q +ma or +lene han +kur land +ku bik +ki kim +k gl +joey ryan +jason mohammad +jamaic a +interior style +indiegam elover +im modest +ik aw +i hat +hyper cube +hump ty +holme sdale +hod kinson +hk jc +hemi spheric +guigno l +granad areports +gran te +glon ass +g djb +fra port +forte an +foresth ill +fli pping +flam b +feed thefuture +experiment ally +estim ators +er manno +eo sio +e bury +divine mercy +distinc tiveness +diaspor ic +delhic apitals +dc wx +cv h +csu sm +cray ton +coon an +colom bo +chris ber +chak an +chai fetz +c fw +buy ck +bri j +bre mbo +bou zouki +be sty +barry wyman +as me +art glass +arrog antly +apologi a +any as +antony cotton +amon te +amar is +am ining +al aphilippe +after hour +ad resse +accor ding +ðŁĴĥðŁı»ðŁĴĥðŁı» ðŁĴĥðŁı» +ðŁIJIJðŁIJIJ ðŁIJIJ +⾨ ðŁijij +Ã¥ le +zu manity +you make +yearswith out +ye lection +yas ar +wine pairing +wi ffle +weekly chris +wee ee +vo tol +var vara +ud ta +touri smb +thi steam +that matters +temer loh +sw mrs +strac zynski +so ory +so hrab +sky lit +sho red +san tis +rip saw +retro s +rem nick +re breather +re attach +re appointed +q tip +po cruises +pen at +patho logic +par ichay +pang ong +neiln mukesh +nawab shah +mye verything +mor na +mo hun +men inga +mc gibbon +mar ins +mann ington +mall o +ly rik +lor dan +litter ally +lamar que +ko haku +kew science +ket ches +k pol +juli ef +jap onic +it ches +ir ano +inver keithing +inclusion ary +imti azali +il ite +high flyer +happy times +hant scricket +hani ya +han kinson +h ma +go lovin +get n +füh rer +fou che +fl on +exoner ate +entic ement +engv snz +englishri viera +emp tive +e fre +dz mm +do don +di pan +der mo +de hn +dale e +dak o +cun o +ctvnews vi +con vocations +ck x +chriso cearch +charn wood +cathao irleach +capit ulate +cac p +cab over +c elife +bun ya +brue gger +book ers +bon nier +bang olufsen +autisma warenessmonth +arc gis +and relton +an fer +amo han +amdiabete sassn +af oo +ab m +a stig +a jai +ðŁĺ¢ âĿ¤ +ë³´ìĿ´ íĶĦëłĮëĵľ +âĿ¤ï¸ı ðŁĴĻâĿ¤ï¸ı +ह म +zoom ies +zo is +zeynepab dullah +your welcome +you them +year with +yard sale +whathapp ened +wedd ingo +wc ps +w shed +ving t +vic ary +utu san +us arm +ugar te +tom ate +to ten +tin da +ti angu +thomas mtv +thelu mineers +su ya +steven rinella +speci fiers +ske ete +sk ru +sign posted +se bts +sch ut +sar son +santi am +sam ahan +safe hands +ry ou +rock xx +ro get +revul sion +resi d +re tooling +re ple +radi ophonic +r fafighting +pol lies +pf aff +patriot sfight +party bus +pal afox +pack age +ott ley +o zan +nor k +nice comms +nh se +nevere ver +multi point +mu kesh +movie posters +molen beek +mil ward +mid nap +mick jenkins +men tosa +medemb lik +mber gen +mb ag +mas ar +manu fc +mane ki +makh ni +maced on +love gwendoline +li sowski +lafour cade +l cb +kol lection +key biscayne +kar son +k dr +jo bo +j ope +insu fficiently +inol vid +ing on +ing lives +inde mann +hy pn +huuuu ge +high tide +hel mick +hari ini +har alson +ha wes +gy ar +gaslamp quarter +fer ulic +farah zeynepabdullah +fa aa +ex pul +es al +equatorial guinea +eo tters +empire ofthe +elie be +e wo +don ts +deme sne +de paola +cu o +convers as +convers ant +claiborne farm +chlorop last +chi odo +chand an +can ape +bur net +brutal ities +bio chem +bin ford +biennalear te +bibi ana +bernas coni +azz ura +au ber +ar pan +anuradha pura +anton ini +an kur +alcal de +al gha +aflo or +'' '' +ðŁĺı ðŁĺıðŁĺıðŁĺı +Ù¾ ت +zy g +yy ceats +wol v +wi di +whale bone +weare mg +water week +villa vicencio +ver in +va jazz +tre the +thisweek in +thefutureis female +the chef +ter rel +te waar +t pr +sujeeth sign +su lay +spon d +south wales +so ami +sh ko +se idl +sc rowder +sag s +sad u +s science +row les +rockingham uk +returno f +qu avers +pro mesa +police women +po kok +pet terson +pa as +or jan +only badchicks +off broadway +nwc fl +nsw labor +noble woman +no control +nic asio +my sskin +music to +mo hin +mil ord +michal is +mckit trick +mari ann +manit ob +m sau +love yall +letgirls learn +lakel ouise +kuro o +kni ghting +kel laway +kashi wa +judi dench +jon benet +jessiej decker +janet lynne +in dro +ilo vela +il ar +icon forhire +i blis +ho eness +go ans +fun palaces +fre port +finn skata +fil ion +fare ast +ev y +elasti girl +ei fs +digital uk +di gue +di bb +dar gan +czar ina +cy rene +cre ggan +cr da +cor ra +con nelly +chan y +ce elo +caper ca +boysand poets +border lines +bol lettieri +blue plaque +bar isan +b wc +b life +avi ano +av ang +auden shaw +amoe bamusic +american eagle +acro phobia +ðŁĺĤ ðŁijĮðŁı» +ðŁĺ± ðŁĺŃ +ðŁĺ« ðŁĺĤ +ðŁĵ Ķ +ðŁ§ľ âĢįâĻĢï¸ı +å¿ Ĺ +âľĪï¸ı # +áĬ ł +н а +with confidence +wh ooooo +vu ren +vik ki +vie ille +uru guay +univers als +uk garage +uigh urs +trans mute +tr ys +ti dd +theme parks +thaic ave +tent acled +t suru +style inspo +stani forth +si yah +shar ris +shah zada +sell ick +selfcare sunday +schwe iger +scar olyn +sas co +sak ta +sa jal +rosar ia +ric kett +r ale +q asem +power apps +pir ri +peter crouch +peculi arities +pap ineau +over stay +out strips +orange man +opto electronics +ny er +now isthe +no win +neu star +mur rays +mon serrat +moisturi zes +mg u +mcgi v +mat ai +mam un +main ieri +lu ft +lolo jones +land marked +lam anna +juli antina +jallian wala +isee you +in ki +in icio +id om +hurst pierpoint +heat ing +had lee +grey water +greatest leagueintheworld +gorgon io +good beer +go girl +globalnew sto +ga si +fording bridge +fla vel +f ally +estro gens +ell ingham +elder of +ef ya +ed lund +dod dridge +di ger +de mentors +dah lan +cuad ros +cu x +cro fter +chikar apro +charl bury +cape coral +canadian army +border s +blow ing +big gies +beng tsson +bel in +bank rate +av ary +ast ros +aspin wall +ash by +as wan +arrow root +animation dev +amazing places +allegi ances +air box +aff ton +acu shnet +aaa al +a jaz +ðŁĴĸ . +ðĿĺ ¢ +ðĿĺ¢ ðĿĺ +ìĥĿ ìĿ¼ +âľĮ @ +âĺºï¸ı ðŁĴĸ +à¸Ħภ£ +Ê Ķ +ô n +y nares +wmn hist +weid mann +we sh +wb g +voi ding +vis ca +vast o +val ette +uni que +unex citing +ty ana +tv fest +tutankham en +tt chelps +trip to +to see +tell eria +team nl +team gaspari +taly bont +swo ons +sugar cubes +sub bu +sto ya +slo fficial +sla ver +shire brook +sham okin +schu ett +san oma +sail cloth +royalairforce uk +ro ids +r gu +proto typical +pro bly +pork belly +pon orogo +plesio saur +plau sibility +pin chas +perro tta +peori achiefs +pen ic +pe kar +pathan amth +pas ch +parks rec +pa quita +newcastleupon tyne +neal brennan +n ze +minic omic +metro stars +me ghana +me chas +marketing land +marit imo +mari ad +magic fm +ma hel +loon athe +lo ei +life inthe +lemon de +le mmer +kro ft +juliet landau +jinder mahal +inti mated +infe sting +ind ilens +in compatibility +ii ight +i irc +hydrogen ation +hooke don +ho wol +hen in +har ip +great schools +grapp led +go bind +geb bia +garri gan +gareth cliff +fou n +fashion model +fab ula +exal ts +ex upery +em oments +dur rell +dou bloons +dl rcc +dis figu +de tt +cur dled +cre swick +conceptu alization +colour ings +claus sen +citys c +cis ne +ciné ma +cb cradio +c ón +c sio +bus boysandpoets +bru hits +bluelive smtr +bloem endaal +bi dd +bh lib +bed sit +be twood +be tong +bbcred button +as man +ar twiculate +ap ca +anth on +am ran +am orosa +am molite +alo to +alfa jor +al tro +ahu t +agu adilla +ade pitan +aag pbl +ðŁĻĮðŁı¾ ðŁĻĮðŁı¾ +ðŁĺĬ ðŁĺİ +ðŁĺĤ ðŁĻĦ +ðŁIJ¶ ðŁIJ¾ +ðŁįĵðŁįĵ ðŁįĵ +ï ĥ +ç ½ +â ¸ +zar doz +yougo girl +ymc as +wr acked +women tech +win ther +win big +wifis funeral +wel low +we we +visit noosa +un tiring +un shackled +un romantic +un pronounceable +tur keye +tric losan +tothe people +to wy +to di +thriller writers +the muppet +the money +ten ser +tan an +tack ling +stron gest +stri e +ss bn +snow bell +shu jaa +sho tel +sett in +save daredevil +sarawak ian +sac rosan +sacrosan ct +rspb minsmere +rotor craft +revol ve +re ordered +pu st +propert yoftheweek +por twine +ponder ous +plateau ed +pel z +pan mure +or ville +or dov +or am +oo sten +of ans +nor gay +niek ro +ni blett +nati v +msu ext +monsie ur +mj ölnir +miss gem +miner vois +mer pol +men ier +may enne +mati vity +mascot te +mar uf +mar ani +mad h +mad decent +m night +living ston +law lz +laut aro +kill menow +kad en +jfk airport +jc mo +jagann adh +j gp +itstony bennett +ine qu +ig aming +hyper cholester +hofstra u +gosn ells +gold hawk +gior giom +gil crease +flo m +fla pped +fau chon +es m +epo c +endeav our +emmy lou +echi um +dontb omb +do ering +den ly +demi gods +daw i +darab ont +cycle ways +colori sts +colonial pride +cold cut +co ilovers +cli ving +chow chilla +cho pped +chin ar +chicagom usical +chat elain +car mouche +buffe ted +black letter +bir dr +bi utiful +bev ington +belly ache +bel lu +beh ring +bal lew +b bog +azz opardi +asymp tote +arnold schwarzenegger +apr incipal +am ole +a ans +ðŁĻı ðŁij¼ +ðŁĴŁðŁĴŁ ðŁĴŁ +ðŁĴĻ ðŁ§¡ +íĥĢ ìĺ¤ +ãĥ¼ãĥ Ń +zo oniverse +yeee ah +winter games +white horn +wand wmusic +wal sch +vote thomasmtv +vill amaria +usd learns +ubiquit in +triglycer ide +ton n +tit in +threl fall +the mars +ta ft +str ously +ste cher +sor oti +snarky puppy +sha han +schre ier +sc ai +say i +saw amura +ru der +ref ill +redbullair race +re matches +pro sec +pray ingfor +pp ur +pointe dly +pay kel +pau sch +pat ni +parti zan +parrot let +paren thetical +pac elli +out fall +odu sports +obstruc ts +mun du +milleni um +mel gar +mar dyke +mame town +mam aw +ly cans +love sick +loose strife +lin tels +le uc +lawson official +lau toka +l ura +kour nikova +kindafunny vids +k ns +ju dic +john williams +inter solar +in kosi +ic orp +hb ddar +har ton +ha fe +green burgh +gom ti +gi gha +gar gan +ga shed +ga aru +g mn +fu trell +foxnew ssunday +floridian creat +fire news +far rant +fal ci +expression less +esof tball +endodon tist +ele gies +elderof ziyon +eg ba +denis coderre +decad al +dar ting +cro pre +cro pp +ci vit +ci az +cho wn +charlize africa +cdn tv +candycrush saga +blood stains +big machine +bert kreischer +below deck +bellin zona +bbc somerset +baltimore uprising +ay lestone +at official +alle m +ale ena +ai z +ah sa +abyss inia +ab hisar +>> @ +ðŁķ¶ ï¸ı +åĿ Ĥ +ൠĩ +zy gous +x rays +wal don +voc ation +valen cian +uwh uskies +uni sport +un hurried +umb c +tu dou +triplic ate +tr é +tony kanal +tmobi learena +the tony +tel com +stra dio +stom y +song writer +shoo ky +sheil agun +sheilagun nreid +seraf ino +scho enen +sam bhar +rspb birders +rival smike +red act +reck ell +ready for +re distributing +re dedicate +quotes forlife +pit ying +pf sense +paul weller +pa zo +p mu +ou vindo +oren burg +nws bayarea +nonchal ance +nolla ig +nihon bashi +nicol ay +newfound landers +neph rite +mushroom ing +miko yan +metho dis +mer cnews +mende leev +mcl ars +mccri mmon +mc sherry +mag loire +ly ster +low boy +lo pam +lam pley +kul p +know it +kier on +kar ly +kab lam +jol liffe +jay baer +iri st +iri ga +instinc tual +inclin ations +in line +high note +herni as +he yarnold +hartz ler +happen shere +fun day +exp ounds +et nowlive +ers baseball +en et +emili ana +em ps +el ice +e hi +dun smore +dra enei +dor mont +distribu ted +der v +cta flash +craig smith +constitu tionalism +con soled +choose cruz +cer ta +cel led +carrie ann +but ner +bro mel +ble ddyn +bienven u +bbc music +bar kov +back kk +ba injal +ay ak +av endre +auto logous +at ago +arru pe +anze kopitar +any am +anantkumar h +akshar dham +afternoon tea +afric om +advance auto +ad era +achrist mascarol +ac fc +aar thi +aa official ++ +, +ðŁĺĺ ðŁĴķ +ðŁĺ» ðŁĴķ +ðŁİīðŁİĤ ðŁİĪ +æĻ ¯ +å¤ ı +à¸ĩ à¸Ĺ +world rugby +west brom +wag gy +umph rey +u chicagop +tweetapicture that +trade shows +tr ze +tepp ei +tcr no +takar azuka +tac c +tab e +t set +super res +style tip +stocking stuffer +stanley park +sn v +silk worms +shish ir +shim omura +seattle children +sarrac enia +sandal wood +sal low +sa pped +rol linson +rick wood +rest ling +recipe blog +r tty +quarter deck +pres sphoto +pil at +pe asantry +pav in +parasit ism +or ison +o zer +ny cd +nmm greenwich +nhsm illion +newlook fashion +nan or +nam anana +myco toxins +mira sol +mika il +mccoll ough +maj ka +ma shiro +lrb ht +low ville +low ns +lou lou +lo pen +lim be +lack adais +la sto +la pride +kram ers +koss off +kingdomcomed eliverance +kindergar ten +ke ef +kaye adams +ka ve +juli amichaels +joyce didonato +jon batiste +jo ginder +jo ear +ira bad +ily ich +ich wein +iback thenats +han lin +h sing +gran ulation +gra velle +gou dy +gor ging +get fit +frank warren +footb ath +fl outing +fisher folk +fall winter +extracurricul ars +eugen emir +eugenemir man +em mit +ek c +eic hel +dis associate +dhol era +dark soul +craig millar +conte h +col locations +cod champs +christmas special +cheer sport +cc tv +call anish +ca ppo +bob seger +bin do +bin ch +bill simmons +bhak ti +belo v +be mani +bar uchel +avail ble +at se +at anas +asjad nazir +ase f +aren adublin +akro polis +abo ah +*:ãĥ»ãĤļ ⾧ +ðŁĻĬ ðŁĺį +ðŁĻĤ # +ðŁij¼ ðŁĻı +ðŁıİ ðŁĴ¨ +ðŁĩ± : +æĹ¥ ãģ® +âĢĶ âĢ¦ +à¸Ī ร +öz ge +ï a +zamor ano +zai us +za habi +yy xx +yay ayay +x ol +wool ridge +who cares +way ner +wat ters +val lum +u hmmm +traffic chief +trafficchief ng +thru xton +tholi prema +thene c +the walking +the tournament +te on +tales ofthe +take astand +takan ori +stateofthe union +sor table +sonal chauhan +ship wrights +see a +sar ria +sampo erna +sahar are +sah len +rock i +resi duals +re hire +rashmikamand anna +ra del +queri ed +putin atwar +propag ator +pre heating +pow ner +postand courier +po cs +plan cha +plain tive +pedro za +pay am +op as +on our +ol g +oh boy +official wexgaa +ny j +noto kay +night clubbing +nd p +nathan s +my at +mumb ail +monarchi st +megay acht +medical research +man cs +mali ha +mal et +lou donville +lost teddy +legion fx +lanz amiento +kpop ers +kmf dm +kindergar tener +kidero evans +khan e +kc wx +juliet telewis +jazz times +iwon a +inter actively +infiltr ators +image awards +ice age +hor ch +hc ps +gri mas +ge birge +gang star +friday mood +forevery thing +for amini +foot plate +ferru ccio +extor ted +espor te +esk ay +er rata +ejec ta +east bank +dry skin +doom patrol +dombrov skis +det ling +desi rous +dell tech +cy presses +cros scountry +cre stron +compar te +cli q +ci amis +chri mbo +chibi usa +camerat rap +cairn sgbr +c plp +brown rigg +brad t +bou w +bluem tn +bloss er +blau er +bit w +be itar +bc stx +base bal +bar ani +baini marama +bai ji +ar ow +anac apa +agribusines stalk +afric aine +................ ...... +ðŁĺį ðŁ¤© +ðŁĶµ ðŁĺĪ +ðŁİī ðŁĴĹ +ðŁ¤ µ +åѦ éĻ¢ +⼠© +à¹Ģภĭ +zz one +zapp one +z gan +yeah hhhhh +whole sale +w us +v sts +usaid transforms +ure thral +upanish ad +unite c +u dit +tim bo +templ ating +swachhat ahi +sw osu +suwa idi +suppos ition +st ma +speed boats +sol ti +sk ö +scottish canals +sab z +ro sin +riv ka +rask ass +rafc gy +ra sto +qu ing +pw res +pro di +press ler +poplar ville +pol lens +ph re +pal y +pakar my +oz ona +over seas +orda z +or ite +onero vers +ny we +no ye +nis ka +new world +mr j +mo rella +mo berg +michelinguide uk +mi ria +mez cal +marilyn ne +macer ata +lth trust +loo kahead +loc q +line berger +leek wang +leed ong +lan des +la opera +kri er +kel ani +ke di +kac chako +jen naw +jay park +jas wim +j soc +indian summer +ima gic +huber tus +hou tte +hot press +gom pa +ghat ge +frick collection +for sman +fol o +fisher cats +fifa e +etsy teamunity +eric an +eras mo +elm hurst +dollar general +dic om +dele vi +dehuman ize +degan wy +dau g +d cb +cu dd +cru cero +cord mn +cor vette +convers aciones +conceptu alart +ci gale +ci bia +christmas dinner +chill o +cedric alexander +carra ig +c ung +bush meat +bon ello +bo eser +bli k +biomole cular +bio graphic +bha agam +best buy +ber thon +beck a +be om +bbc norfolk +bar th +audi bly +att ar +assi stindo +ash ell +aro che +ar kad +aper to +ami e +am ese +al sip +adhe rent +ab ondthatcantbebroken +ðŁijį ðŁĺĺ +èij ī +è ĵ +à« ĭ +ı z +ym el +world triathlon +wind blade +volu si +vle uten +vintagetraffic usa +vaness o +vac tress +ur la +un il +terrori zer +tad worth +t sw +syl vanesso +subju gated +stel zer +st q +ssss sssss +sf alls +seren aryder +see ff +seas ickness +sav arese +sar keesian +sane wwe +ru i +ros é +rob m +ric hert +rc u +raj ar +rail ton +pun akha +pri sco +precipit ated +positi vel +pos ity +plat ting +pitt sbvb +pin ho +pi ph +people matter +ore e +ordov ician +or sett +on twitch +ol pc +nu cor +nightinthe woods +never leave +natu rism +nas w +nano bots +more ra +mis za +mign ons +met ar +mello tron +mc pd +marcjacob sintl +lostin translation +lon line +lo isa +learning disability +lak l +lak is +kw anten +ku bra +kim zolciak +khu shal +kairi sanewwe +ka dir +joeyryan online +jeet bo +je k +jake pittsbvb +ja imel +itsme marcog +iron bowl +ir da +ine miliaromagna +hug your +hosp icec +ho ttt +ho ar +hiro kazu +help ful +hawaiian air +han ae +haider alabadi +ha eun +gen cia +game zone +fre de +first place +fast way +expo sed +evely ne +end el +emerson drive +el met +ecu baseball +dou ll +clin micro +cl w +chuck liddell +ceil ing +ca zy +bur rowes +bry k +brizen orton +brac ket +blunt ness +bishop stown +betti epage +ben nell +bel ters +bap ak +az leg +aw onder +avi ka +ascend ance +artist sofinstagram +annel ies +angel ayee +andru w +all sup +air fryer +ahmed patel +afford ances +> ) +ðŁij¼ ðŁı» +ðŁIJ ¿ +ðŁįī ðŁįī +ï¹ ¡ +ઠ¸ +شرÛĮ Ùģ +zindagikime hak +yz ma +what culture +well travelled +we missyou +villa verde +ut x +ultr amar +u efi +tv illa +truste eship +threl keld +theamerican sfx +techno terrorist +te guh +tart ar +tamsen fadal +stother t +stor mon +shat tered +scrip ture +scar ff +s sci +s bla +ro ton +rithvik dhanjani +ren es +refra ins +refr act +raj skub +pvr cinemas +pur ges +prote omic +plim soll +pix ley +pc bb +pace makers +p suv +p bafinals +nin kovich +nicolas cage +nic eness +new quay +narrow boats +nadez hda +n ter +mutil ating +mon aya +mobile punch +mizu hara +michael shanks +mic ks +mc que +mati z +mas nor +mar ar +maggiel awson +luxury watch +lug nut +ling le +liacou ras +le mahieu +law firms +lam oriello +ku cera +klar man +kill ough +kemp en +kas erne +kar aj +k crew +jul liard +johnny g +joh nette +jeffrey combs +jake shears +j walsh +inve ste +insane cham +insanecham pwres +ifly mia +ic ture +hp celebration +hol lick +hodder books +hi ren +her ping +hard body +ha go +gour ami +frag mentary +fr s +fe deli +err day +eng ates +el sworth +el ster +ec lips +e mig +dream less +designer sguild +del pozo +daf na +cress kill +cosmo sdb +comman deering +coal port +check point +ch acousa +caw dor +buck den +bour bons +bb mp +bad enoch +asjadnazir sexylist +amphi polis +a acs +???????? ???? +ðŁĴļ ðŁĸ¤ +ðŁİĦ ðŁİī +à¯į _ +z ol +youthen voy +yan kees +wo tan +wish bones +wind starcruises +who woreit +wah ed +unic anberra +udd hav +tx stars +tri ppi +treva than +ti ful +thread ripper +tho p +ta irport +su hani +spring fiel +spl ant +spaceship two +son paper +so apo +sm koneru +shak en +sh ood +sh dop +sasi kumar +sar lat +sar gam +refer rer +re tractor +re appointment +pnca rena +pin z +pau die +pa wesome +our in +oc v +o lowo +nar va +nam ics +my hero +monopoli ze +mom afilm +modi fiable +mid mo +mi el +mercado libre +lon nie +lia ising +li ker +les ch +le tyour +lam brecht +lagav ulin +lac ofd +kne ast +kh eng +ke j +ke aley +jun to +jay sus +jadav pur +j wilson +j jones +j hum +ir pur +im j +ikar ia +hu ahin +hon ora +hispan ia +hex is +hersh berger +her schel +hel zberg +han ok +hal berd +ha ggin +h jal +green island +gr ttweets +gautam rode +fnd tn +fluore sc +ferro vial +fc splayoffs +expand ers +ethical hour +emocione scan +el sie +eid as +e ying +e ah +dumfries and +duck ula +diop side +dh alls +den yse +deme trio +dead space +de twe +de ignan +cynic ally +cram ton +chil ds +chead le +charter schools +cath ays +ca icedo +bé ar +bur chill +bot in +bor odin +big data +bel isle +be joy +back kkk +bab bit +ba jar +aw la +atlanta fx +at tridge +as ali +arya stark +art us +arroman ches +an ies +al sadd +ais for +ai dy +actu ation +ab ula +ðŁĺį âĺº +ðŁĺĤ ðŁĴĺ +ðŁİĻ : +çij ľ +ãħİ ãħİãħİ +zo han +zah raa +z wan +yn elson +yank sonyes +winter warmer +wheeler dealers +warand peace +war same +viv ab +tri phala +trend z +traci iguns +toyo tas +time keepers +the odd +su liman +su hr +su are +stepan akert +steal z +son owal +so yer +sko glund +sie ges +shutt ler +sep timus +se eta +scar ra +scab iosa +saxi frage +sand ers +s magic +rit ory +resent ments +re location +puff balls +pu yo +peregr ine +pd ash +on cidium +official pvfc +official pes +n spc +my tton +monop lane +mono coque +marmo zets +manu bennett +mang la +lymp ne +lund university +lr ath +ll ly +leyon hjelm +leon hart +leg alism +lees ung +le pton +lac onic +kw es +kapp as +jj rockxx +implic ates +ii faut +ia sen +hot yoga +happybirthday liam +ham lisch +h na +gyn lais +griffin shockey +gravit ating +grace jones +g thr +fel ici +fac ce +escape ment +en ballet +elic iting +el mb +e tomi +du rell +destined tobeyour +dam our +dal u +dab ke +da stan +cre sson +cn sa +chap manu +cen bank +cand ling +calum best +brent butt +bo ger +bie hl +better off +b kl +auxili aries +arbitr ators +ap cu +andri od +alu ae +. ðŁĴ¥ +à° ħ +Ø§Ø ® +zan in +yo sh +with y +witchof gric +war dy +wain scott +vir ility +vice president +v mu +une z +un delivered +to vo +tin dal +then ra +tele vise +tal ex +ta wad +sturgill simpson +stu arts +stal king +speed line +spani en +sol in +snow blower +sel k +se alife +saf ta +sa karya +ry land +root sport +robre iner +ric t +refresh ers +queen sugar +q wop +pre conception +per dida +pe plo +pe ay +par amotor +one us +om ron +of arms +nicole tti +new stuff +nat oma +my u +my co +mu iz +more life +moom oo +mol alla +mi leg +mercy ful +mckin ley +matthar vey +mark logic +mamared carpet +mal on +lor as +lom adia +leg ler +lec tro +lar go +lar don +kwe se +ku le +kom bo +k oper +journo request +jad is +isak ov +iri dology +ing ness +individu alist +i she +hor sell +hol ub +hij ra +her ford +hac i +gay romance +fre tted +fe zz +farmto school +ext enders +eric j +ecou tez +eck mann +ear lof +de values +daw na +dal ys +cro oz +coul ro +costab ile +comb ichrist +chin ni +cher one +che fe +charli eco +carav ana +cano eists +can ak +c trip +bye ee +bun b +bridge of +boom boom +bok rugby +bit ties +big things +betibachaobeti padhao +bas ri +bapti stry +bag lan +at ley +astra galus +arcen eaux +ali ona +aa ÅŁk +, '' +ðŁĺģ ðŁĺľ +ðŁĮ¹ ðŁĴĢðŁĮ¹ +ðŁĮ ¯ +ìľ ¼ +âĿ¤ï¸ı ðŁ¤Ĺ +âĿ ģ +yeon jun +without limits +wire cutter +wing s +wear pink +wal brook +wa sters +val v +un supportive +un drinkable +trevor jackson +tough er +to paz +theyoung turks +ther am +the police +the grove +the fieldhouse +the bush +tewaar aton +te ter +tanger ine +su jan +sp lines +sla voj +sigma hq +shu ld +shu ben +shen mue +shan am +schmel zer +sc x +san ai +salam on +sak on +rott we +rh us +ree zy +red line +red cliff +randal stown +rag am +progressive metal +pon da +pnin ator +ple uro +playoff final +pitch man +photo catalytic +people whom +patre se +pat erson +p flp +on elmstreet +omari hardwick +ny ard +north wales +new shindi +neuro psychiatric +nece sit +native americans +nat ter +mor as +missi ves +mis su +mirac leon +mic odelrosario +mi pi +mcil wain +mar oto +mar imar +lly welyn +leit rim +kn wa +kir ara +kevin m +kat as +kaf u +juliere ichwein +jour dain +jingo ism +jean grey +iy ar +hol loween +heil man +har in +gri bbin +gra j +getat me +gerald orivera +geo x +fu sca +fant agio +es now +em yr +egyp tologist +ed na +ec cw +dige stive +di zi +det ama +dark lines +dan cere +cos se +cor fe +comicbook men +coly tic +claudia jordan +cityof perth +cine a +cham paran +cep k +catalan referendum +cad enas +burk hard +bur ga +brook lynn +bron ski +bri anne +brett kissel +bon omo +beg ich +ayo ze +avo gad +au sout +aren aof +aontro im +anec i +amand ase +alle le +al vi +ai v +a ves +!! ⾨ +ðŁĺľ ðŁİī +ðŁĺį ðŁĺĩ +ðŁĴ¯ ðŁıĪ +ä¹ ĭ +z ation +voye uri +vj se +ver da +union saustralia +ty hafan +tul lio +things done +thi er +te men +te at +tab ler +ta veta +sy en +swi wx +suss kind +steve wal +specu lum +soft works +so thern +shu bha +shant ung +serkan cayoglu +sel lar +sclo thes +sc so +sc ira +savedbythe bell +santo shi +s garden +ro ev +ro ba +retweeet please +ren ton +prim rosehill +president duterte +polit ec +patri mony +p ctto +over holt +over drawn +ov adia +onthe water +nli reland +nfldraft scout +new smy +nail sworth +my kha +msam ber +mostbeautiful faces +monof ilament +mo lex +mississipp iriver +meister singer +maur its +mat ua +mari anor +mal vin +mal to +live authentic +laurel hurst +lan kesh +ki bby +ken ly +ke bab +janel parrish +is qua +install ation +igno u +i ipa +hol zman +heron dale +gun ships +gran do +gill ylancs +fur ler +for humboldt +flo rette +fire y +figure drawing +far g +ex horts +ent ano +eccle shall +driver les +drive on +dove cote +denzel washington +d ä +comm vault +coll ingham +cer atop +camoufla ges +bristol news +bristol city +bocc accio +blythe wood +bk twtr +bay lon +bass musician +basil an +ban za +baham ians +as sche +as l +aren ts +arbit ral +am iss +ach ra +ac press +ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺį +ðŁĩ« ðŁĩ® +ë°ķ ìĭł +âĿ¤ ðŁĴľ +y utu +wr p +wny wxguy +wiley updates +west wing +we urope +vojvod ina +vit tori +vir gie +vie tto +umphrey smcgee +um bus +tx milesplit +trespas sed +titan sof +time sunion +thest style +thener d +tam popo +ta ppers +sv ball +super user +sun dari +su sand +strath spey +stan chion +sta e +sl trib +silver side +sig gy +shop fronts +sar it +samsun gs +sakhar ov +sa ie +reg n +rec ent +re re +r head +pron ghorns +pin cers +p te +onlinec asino +nzv ban +ny liberty +motivational speaker +mol der +mil air +maw dd +marin aro +mar yo +manto vani +manj hi +mandra gora +man dos +mal loch +lynd sey +loffici el +lin tott +li gation +kick as +ki bet +katiemc grath +karti ka +joo hyuk +jinxx bvb +iz aguirre +irregular ity +inter breed +intel pro +int ents +imuk biz +hu uuuu +guil lemb +gradu ands +glan bia +gat ot +football friday +fle cha +find me +fel ly +fe ste +fe aver +euro furence +egg cup +effu sive +eagle hawk +dye sebel +diagon alley +deer field +de iv +dan ka +craig ella +country duty +cou lon +corning museum +comedynight swith +collec tives +cn sphoto +clari dges +ci bona +cher if +che se +character ise +cav ender +bra chio +br acks +booksto read +boij mans +ble m +bi zer +battle on +bat lle +baker va +bai doa +bad blood +back boris +atap ult +asi mone +al uska +ðŁĺįðŁĺįðŁĺį . +ðŁijij âĿ¤ +° ï¸ı +younus algohar +wom anist +wer neth +vs bos +vil ification +vi all +v roman +un sullied +un making +tr inians +tol booth +tele casting +taylormade tour +tau k +szyman ski +sz ka +steph ani +sse arena +sou tho +silent disco +sho taro +shah ram +se ath +scout master +sar dina +sam ani +rick world +ric cardi +ri gaud +renee graziano +rant zen +ps le +pro saic +poly unsaturated +pa jaro +p scs +online poker +ohiop yle +of cl +ober ge +o ei +nev schulman +ne dra +memor ably +mb am +lo gr +lnp fail +leu ch +leaveyour mark +le di +kin hai +ken burns +kemp es +keeping you +k roy +john leguizamo +james mar +itsall goo +ik hil +hyper dunk +husky nation +hu lud +hr in +hol landia +hick forco +heures dumans +hallucino gens +gu detama +gian marco +fren chal +fox searchlight +fivb men +farm pics +expun ge +este ban +end family +easter house +diaspor as +depress o +demar re +dave mcclure +dar l +daf tar +cro che +cour tly +cor ie +col ine +co tai +co flood +cn alive +chen yi +chasu ble +cast in +cafe cito +burger week +bri gs +bra ds +bot as +bor gman +bhil ai +bestnigh tever +beauty tip +bal ada +ato jr +anno ying +am la +al cides +ais in +ag lass +@ .@ +* ] +' ". +ëŀ © +å¤ ľ +ãĢĭ ãĢĭ +د ÙĪ +zen nor +x pe +wych avon +wom ad +wol fies +what car +well spent +w daz +vag aries +transpo sition +track man +to lima +the orie +tand f +sun sout +sun gei +stradi vari +spic ers +sn ur +smithson iann +sarah jaswim +sal ine +ryan hunter +rid ger +rico che +riche mont +re spire +puri foy +pe kalongan +pe et +paro di +pan try +p th +oc pd +obam aday +o wa +nu ked +noman sland +nekop ara +neg ating +mu lago +morecambe bay +montt remblant +mm film +mello dy +mehwish hayat +mega shot +may sles +mat tera +manag ing +ma dala +litho graphic +li gia +kon kon +kitti wakes +ki yomi +khel oindia +ker ins +kate beirness +kap ag +kam asutra +jay len +int ell +hour glass +hex ham +haz im +haupt bahnhof +hal ston +hal d +grand ches +grandches stour +ger akan +gam brel +fracti ous +form alize +forgi one +fl andres +fa kku +ex claiming +enig ma +emili eder +emilieder avin +dy ar +du ri +dough ty +dob son +desecr ating +dar ri +d hir +crou te +cr nas +coyo acan +comeon you +col ón +cling mans +che sser +charity shop +cer via +campe sinos +bu ckey +bri j +bo yet +bertrand piccard +bas sie +back breaking +bab ineaux +azu lejos +as cp +andy grammer +amba reesh +amat suri +alu va +a jose +ðŁĺģ ðŁĺī +ðŁĺ¯ ðŁĺ¯ +ðŁĴĵ ðŁĴĵðŁĴĵðŁĴĵ +ðŁİĬ ðŁİīðŁİĪ +ðŁį ² +ìĿ´ ëĮĢíľĺ +ãĢ ħ +ت Ùģ +Ã¥ r +z waan +z our +youn gre +x fce +womenin ag +whatdoyou mean +wat sapp +walker booksuk +w cdma +v annan +usa ir +un subtle +trout beck +trend micro +tor turers +tol li +thereal gokwan +the mary +the jon +thaw kins +ten so +taleg gio +ta hoe +supportlocal music +strato cumulus +sthe world +stephen mangan +speci alizations +spe el +spani er +sonny digital +snow berry +smar tt +sloane stephens +serv ative +sch is +sanantoni ofc +rum ley +rot tie +rna seq +rin at +riff raff +regal es +reen acts +re dress +rajat tokas +r alli +quiztimemorning swithamazon +quis ling +pro social +pres sey +pra p +poom sae +physi atry +pat more +p skov +or rr +ontheroad with +oneteam one +omy ces +o hhhhhhh +no bill +ni sei +n mp +my night +mp cs +mon essen +mo one +mnight shyamalan +minister io +mile le +mifi dii +meander ings +mccar ley +mb oldo +man atsu +m pire +lenny gaspari +kiran shaw +kir ko +john berman +j la +incentivi zed +in viol +in attentive +in accurately +iii ight +iam ond +hockey hub +ha ast +gray wolf +google drive +gar dein +fire andrescue +far hi +estrange ment +enew sletter +ell acruz +e fr +dul han +don giovanni +do ÄŁ +djim on +dd hi +creative mornings +corred or +congre ve +com hairle +clau diab +clar os +cinemainmy genes +cholec yst +chite ttura +chil mark +chi ya +cassa day +can lon +cal mar +bri se +bra ked +bob white +black country +atl ay +athar va +architec tsuk +anth o +anime con +alph anu +alp eng +allu de +alac rity +agri goi +afloo d +ade bate +ad tran +ðŁıĦ âĢįâĻĢï¸ı +ðŁĮ Ĵ +ว ม +ø y +yogur ts +wedding present +weare stv +wau ters +walkthe talk +ve ith +under powered +un spent +um braco +ty rian +tur aco +tsu taya +troy bakerva +tricia helfer +tre esur +toi world +thur s +team nuh +tau be +tasmani ans +tamago yaki +take that +t dg +swa ppable +stur gis +stewar tuk +stare down +ss badal +spr ats +sla shers +shee ted +shanty town +sha ji +set swana +saf ai +rep l +reg ge +rebu ts +re tz +radio deejay +r sl +property forsale +promom yshop +profit eroles +pon cho +plu shes +play for +pat mcgrath +pas su +orchestr ates +nyarla thotep +nut field +nunatsi avut +north up +norfolk va +nat asa +n maa +myviking story +must ache +munster gaa +mess inger +megan n +med ill +marsh lands +marau der +mar yan +mam ed +mag ura +ly die +lu ddington +lo iseau +li fo +lec lub +lang try +l mics +l mb +kyri akos +key club +kay i +karmen uvella +karao ke +kali uchis +kal ima +juvenile justice +josap hat +jan am +j mattmiller +j ali +ian us +hiroh ito +high est +hel s +he hir +hb hai +ham me +ha thletics +gu ts +gog com +glen campbell +ger ak +gany an +g ela +g day +frand sen +flat white +ez pass +esp ina +eri der +en claves +em mm +dran ath +dig ang +di leo +defe rence +dead lands +de marcu +culi acan +cru gby +cro sland +cor bitt +coco on +cardi stry +car ona +bunb trillog +bu ttle +borne misza +bor ley +bogal usa +ben savage +bay ly +baby steps +b inet +az ir +ational guard +ati dora +and are +alic eroberts +af sp +a strup +ðŁĽ ¸ +ðŁĴģ ðŁĴģðŁĴģ +ðŁĩµðŁĩ ¯ +ze ee +you meatsix +x ers +x amas +x adee +wing sscotland +vol lrath +us wa +ur ts +uneas iness +under lie +tucson newsnow +trues dell +troo dos +ti ro +thisis roc +the original +the incredibles +te do +taylor swift +staur ant +st illing +so j +skee ball +sher ie +shak shouka +sen io +scher rer +scand alized +rou garou +ring way +rever ies +re mer +pp ap +philom ath +pc sreeram +ou asport +ortho pe +oral b +only way +odd job +nz post +nic le +natural news +nat ation +muh sin +mis behaves +mcfe tridge +magu ey +lo ga +let r +kh qa +ker on +kel tie +kash kari +jud ger +jazz ercise +janh vikapoor +jak art +it ment +is chool +ilike it +homestead ers +hol mer +hi zon +head man +ha zes +girl gamer +ge gen +garth tander +forma zione +finn forchange +fing ers +fifae worldcup +fal o +fa sta +fa e +ey n +extern als +eve leth +er satz +e ip +du gs +dow land +dis dain +di kan +del phos +def uniak +deck ers +dal go +cumber batch +cp su +coro coro +con ade +citad els +ci otti +chitten ango +chil o +check mates +chai kin +castle ton +caram ac +car week +cann ing +c ly +bur rowed +bro sseau +bor re +ble akest +bianch ini +bi yombo +bi sta +be mba +bar gnani +au chen +as ays +arci mboldo +ar agua +alky l +affl ict +ðŁĴŀ ðŁĺĺ +ðŁİ¤ ðŁİ¼ +ðŁĩ ¼ +ðŁ§ ģ +ëŀ ľëĵľ +zim zalab +xibal ba +wt bs +wi fey +wheel man +wat in +visite den +vam ily +usic festival +usc lay +u um +u ille +twit terers +twarri or +tur pentine +tu sker +tron aut +tric entennial +trade marking +to iv +tam ira +tabas sum +symphonic metal +suz lon +staf fies +spor tives +spa ins +silver io +shu hei +shi ii +sha hin +sen ough +self magazine +sandwich ing +sal ma +rory stewartuk +richmond park +ri skin +ri ber +regurgit ating +red tour +recali br +ran del +pur va +prophet sof +play towin +perturb ation +ped lar +pas chall +p rus +oxygen os +obre gon +nu v +ni avar +nav a +nad die +na hal +mtu kudzi +meng ele +me dyo +mat tocks +martham ac +malavi ka +lu la +lot ton +loks atta +lev it +leon ov +le master +lan cey +l vo +kristian sen +kris shdop +knowyour status +khar if +ket el +keegan mkey +karapar aaÅŁk +juilli ard +jellic oe +incen dio +ik lant +ig liano +if sa +idol bday +hu bo +hen nen +harsh man +happy vday +had dix +gum pert +gu ff +good news +gill ray +general ised +g sis +g dt +fu lof +freck leton +esp y +ef alls +dy r +donald sonville +dino sau +di bi +descar gar +deltag amma +deconge stant +de commission +dau r +daily show +da ig +d guy +color adan +code breakers +chu gg +che u +champag nel +bu tyou +brdg stonearena +bo iiii +blue wave +bi shara +be kka +bal tus +bach ir +ba rend +athle ti +ar cho +ar bore +apeoples journey +ann cleeves +amor is +alone together +afflic tions +adam sss +a hal +] ? +) < +ðŁį ĸ +ðĿIJ Ī +z ando +yam un +xab ial +wy verns +winter lude +wil pon +wastel ands +war di +vote dem +voor de +vi ke +var don +use recycle +twi bi +tsun a +trum puk +tre kon +to an +the adventure +tf ny +stech nology +stay humble +smilo don +smart sheet +small things +silver stonec +shahi dafridi +se ik +saqq ara +sam mut +sal umi +sal ita +ru ok +ro tan +ro mm +ro jer +re fitting +ravi zacharias +rain bird +ra oka +quest for +puc cio +prairi eville +pra chidesai +por bandar +pitt ance +pir in +paramaham sa +pacha uri +os burn +on al +ob vio +ny ab +nsc w +north wards +multit ool +mtv star +mrsbrowns boys +mollu sc +moeed nj +mixed martialarts +men slacrosse +mari ash +mahar astra +lu gogo +london jazz +lindsay jones +laim beer +la vand +kou delka +kant é +julie tta +iw sc +itunes movies +it like +ho by +har ar +h wood +gra j +go yt +gi aa +fu seli +frederick md +fol f +father john +farewell tour +fai the +ex por +emo ting +eliza veta +effec tive +e studios +du hon +dont end +dog leg +direction less +davi da +dav ar +dang it +cuttinge dge +contac to +confl ate +citizen four +chester hour +car lier +can asto +bör se +bul taco +blu ffer +bio processing +bas adi +azu read +apor tugal +apo ker +ann av +ankylo sing +amur ray +amand af +alan walker +acl tv +ðŁĺį âĿ¤ï¸ıðŁĺĺ +ðŁIJįðŁIJį ðŁIJį +âĺ ĩ +z ameer +youtube music +x cellence +wel ham +weare indigenous +vapori zing +usarm yeurope +un mixed +tyrr hen +tyr ant +tiv ities +the vfl +tamer hosny +takay uki +sympo siums +sym metrically +subor bital +stun twoman +spag nola +sofe urope +shal u +sh mirtz +se ales +schil dren +sag enda +ron dell +repre nd +redu cible +re deployment +r dan +prize winning +plau sibly +pit tock +phil taylor +pan ji +ome gap +ni renberg +ng d +never see +nebl aruz +ne ele +navar ino +nag pur +mis sin +mini atur +min um +mikewill madeit +mic as +mccre adie +mc whirter +mc bath +ma ju +m newsradio +long don +lit vin +let sen +leaving cert +lauren koslow +kyun ki +kw un +klu ane +kimber lee +kess inger +jun aluska +ju baland +ju ab +joh ancru +jeron teng +jac e +i aap +hedgehog society +happy family +go thard +ger wyn +fur pals +fu ori +freak nik +fe ssler +f me +exemp ting +en fer +eigen mann +e golf +dont shoot +disson ant +dec agon +dataf low +dai shi +cru it +cross y +cr pc +civic si +che to +che apo +cess ary +cbc thenational +calaf ate +c sg +béar naise +breaking barriers +braun er +boost vibes +biennalear chitettura +beer wah +bassmusician mag +bar io +aza adi +authent ics +ar mc +aparna dixit +ant acid +an dex +aman ullah +am erson +all that +( / +ðŁĻĦ # +ðŁĮ¿ # +ðŁĮ§ ï¸ı +î Ķ +âĿ¤ï¸ı ðŁĵļ +âĿ¤ ðŁİ¶ +Ì « +zz oli +yp silon +worldwide web +weare warriors +ward ha +w fl +viol on +vie len +vad uz +un unu +tx plants +ti bbits +thunder wolves +temple more +tem bro +sw sh +surviv able +super conductors +star times +st robes +square enix +spark led +sole a +skunk works +sibling love +shrove tuesday +shol m +shikar pur +sam trans +sa astr +ryo hei +rose and +rit annia +remark able +recuer da +re examine +ps bl +prophe sying +pro pul +prasar bharati +ph vegas +pet abytes +pandor ica +on os +ol ak +nyct subway +nigel farage +ni verville +nag ler +mouni roy +momot aro +mn ch +ml stadium +misper ceptions +mer u +mer mhart +masch era +martin heinrich +marque se +lo ys +lauri ers +lab ella +la ven +krist offer +kle mm +kingshead thtr +kar go +kam akhya +jon pardi +jo bbers +jj redick +jar os +jane austen +ireland amtv +im my +icol lection +holm fir +haz ari +had dow +hackney council +h tu +go q +gary numan +fill an +ff rench +fen ris +fast jet +ero adtrip +elast in +eddi ep +ed cs +dustin poirier +dun gan +domin go +dis bands +den of +country lifemag +con tem +cl b +church radio +chrisky le +chri sar +chai wala +car boot +bron k +bra gi +bol lin +bob vand +bds fail +back yar +aur icula +askingfor afriend +appro vingly +anaesthe tics +amrav ati +amic ably +alief isd +academy brix +ab ai +) ãĢį +ì ¢ +á´ ĺ +Ø· بÙĬ +ž i +y alo +wil co +werthe imer +w ace +vm vc +visit lisboa +un selfishness +un rolling +un followers +umu ahia +uj world +u daya +tz is +twitter art +touris mus +ther ium +theother artfair +the taste +the mira +the call +tak am +ta deo +stre ch +stopp able +stop suicide +sto janovic +steve krohn +sle ip +sit g +shirob ako +selfie expert +scol ombia +sce aux +say yes +sarah mclachlan +rey ne +relap ses +rel in +redemp torist +re hearing +qu ev +puk aki +pharmaco genomics +perse baya +pathanamth itta +pal ko +opp i +one oh +olivi eri +office depot +odell brewing +non agon +nh out +national internday +nagas wara +muk wege +missgem collins +michel lea +mi as +mcdou gald +mar king +man isa +lost lightfest +lightitup blue +le ut +kingh enry +king kong +kil coyne +kat elynn +kali das +ka el +k ku +jen carfagno +jan nik +j izo +itsu ku +i ant +hou elle +hideand seek +helenclar knz +hah haa +great work +glau cus +ghul am +gaf as +free base +flavor flav +finger printed +expen sive +etsy trending +epau lette +eat right +dun man +doo dad +disin fo +devol ves +dele terious +danny glover +dalgle ish +crow le +cross mag +co zzi +co intelpro +clums ily +city tshwane +cinema scope +ci vari +castle man +cas al +build that +brasen ose +bblo gger +ba the +b pn +aw on +av int +ast us +ari ley +aran manai +ar sd +aontroim gaa +ammon ites +ag usan +adi po +> __ +.. ðŁĻı +à¥ĭ à¤Ĥ +z rx +yor am +wwe thebigshow +wi ds +well sboro +war museum +waldorfa storia +w enge +vi borg +ut sa +u lisse +tz ou +ty ers +ton tine +telegram dotcom +taren as +tam ala +sw oon +super glue +strike out +son ika +sme a +sho guns +sel kirk +se fi +scen ted +sauti sol +s beauty +ro sleh +rim sky +right most +ri emer +r jl +qu bits +pre sta +poem trail +ple b +peter boro +pep to +pas ca +paren teau +over reaching +outh unger +officeof ssbadal +ne wa +n pf +music business +moy ra +mi yyah +melchi zedek +mcne illy +mayor alty +masnor ioles +magn animity +lo dy +liz gillies +library journal +lecoq sportif +kt nuk +ksr nv +king lear +jes c +j degrom +isth mian +inter zone +ine music +indo siar +ik könen +i annucci +hur rr +hondaci vic +har riton +gri mbergen +global artisth +geo ca +gen iu +gar ston +funk houser +ever wood +er vine +disinfect ants +discover your +de famed +de cai +dancing withthestars +cur tiz +cruis eline +crow son +coo ver +clu bc +claws out +cent a +cas sim +caperca illie +bun an +br v +boyn ton +bou twell +bo ke +bi ent +be creative +barit ones +ash lar +as ahd +arnol fini +ard ently +architek ten +ar rangers +aman resorts +al tri +air worthy +air conditioned +against cancer +ag bu +ðŁĺį âĺĢï¸ı +ðŁĶ´âļª ðŁĶµ +ðŁĴª âĿ¤ +ðŁĮ´ ðŁĮŀ +ãĥĭãĤ ³ +âľĬ âľĬ +Ì ¯ +ér rez +ye tto +y hoo +y abe +war isan +vive ko +verul amium +vanguard news +us ra +ur ing +ur bis +un answerable +turbo diesel +ton ie +tol ani +thisis america +thirty one +the handmaidstale +test accio +te mis +super cat +squid billies +spr in +spe an +sla pper +si vo +si stani +shu le +sheab utter +shaw mut +sev res +senyor atidora +se tau +ro so +ride shimano +reic helt +re interpret +rbg canada +quin tos +q y +pw ba +poise tt +pi beta +per rie +optome tric +om onia +officially dale +ny gma +nobill nobreak +niavar dalos +nen es +nce atalk +n ado +megh wal +mc murphy +mati ja +mar ving +mal liance +ma des +looking glass +leather y +laser hairremoval +l rl +koffeewith karan +ko fori +kofori dua +kliz an +kil ims +khur d +ka hu +jr w +johnc mcginley +john h +jing yi +info com +idiosyncra sies +i thin +i olan +houelle becq +hay dar +haik us +ger aci +gar rin +fu dan +far fan +evolu cion +dumfriesand galloway +dj j +diar maid +demois elles +de yo +coo lock +chi zik +chann ell +cates by +carson kressley +c ink +bur dening +bra cho +bn pa +bludge oning +beauty salon +bb celeb +bar cal +ayre some +as dru +aro ss +ap ley +any o +al styne +al amb +agi ft +af bf +ac x +ðŁİīðŁİī ðŁİīðŁİīðŁİīðŁİī +áĥ ¦ +اÙĦÙĩ ÙĦاÙĦ +zo tti +ystrad gynlais +wi gle +ver mes +valle dupar +umb rage +trans fe +thur l +tg is +tex ture +ten ere +telli gence +tech f +takeme home +ta fc +sto gie +sn aring +silver classic +ser hat +seapor t +satyamev jayate +sat inwood +sat ins +ry un +ru beng +romantic izing +ro lin +procu re +prah lad +plari del +pen ley +pe koe +param speak +ozz ie +omnam ah +omen ico +ol az +o ip +nm fcofficial +ner ul +naj ia +multi modality +mul larkey +mul her +margin alisation +march al +manti ses +m life +leban king +le ola +la fuente +kor le +ki est +ken ickie +k mbappe +jik lian +jait dp +j churchradio +inge agle +in dor +ik pe +hop ital +hal ak +hac ket +go hawaii +gen ève +gar row +gar finkel +gam ely +filadel fia +fight ins +evil legas +er st +ec inemas +dncle aks +disembar ked +demp o +danneel harris +cv f +cre matory +cinder block +catal ina +car lyon +brown y +body power +bo quer +bli fe +barber o +av ram +ausv wi +at mo +apo loo +andy serkis +ali ano +ak dong +ag twitter +affe e +ðŁĴĸ ðŁĴŀ +ðŁıĢ . +ðŁħ± ï¸ı +ðŁ¤ º +ë°ķë³´ ê² +ãģ® åĽ½ +zhen ya +zach arie +xo yo +wra bel +wait wait +viro qua +vijay an +vi veros +vet tai +ve ut +vance joy +upro arious +un tag +ud ell +u thman +ts ars +toys betoys +ti meeee +the york +tay port +sub divisions +spor tw +spart annation +southern miss +sla ys +sh ange +se ke +scriptw riters +sch nu +sar l +sac ross +sac anime +sa stre +sa inted +s sps +rol an +re attached +ralu ca +rai ffe +public sch +premon itions +point and +phantom opera +pa ha +out world +out d +official c +offic efurniture +o tu +nong shim +new game +nerd camp +na iman +mull ica +mlb fc +mis in +maurici om +matta poisett +master man +maris ela +let toysbetoys +lei dy +lanc elin +lam ott +lac asse +kri zz +kir on +jozy altidore +jig nesh +j allow +int al +in chief +iffic ult +hotho thot +har ma +gry bau +globalartisth ma +glassof bubbly +f ong +eyel ash +exol selcaday +endow ments +en raging +ed mv +e ab +desig no +der yn +der rius +daniel la +dam ara +cou pe +constric ted +confin ing +conce tta +coma stia +com ac +coach bill +co ale +clau diag +cityo fo +chandra pur +ch anti +cedar town +cards againsthumanity +cap uch +can ol +bts world +bru eg +brand t +bitcoin talk +bcu hb +bat week +ay aw +astu rian +ast and +ar nim +appe al +ano des +am ang +alex change +akh mat +ak sar +ag utter +ac cone +) '. +ðŁĺĤ ðŁijĮðŁı¼ +ðŁĩ§ðŁĩ ¸ +îIJĴîIJĴ îIJĴîIJĴ +èª ŀ +âĬ Ļ +zz le +zen aida +yo hannes +xadee journalist +volei bol +vit rio +vitrio lic +veronic aroth +up fest +unmatch able +trumpuk visit +triple talaq +tim ryan +temp at +te sfaye +t weak +sw aby +suz ana +supply co +str acker +ste marie +stab by +sta verton +sq ld +sou la +snow board +shinj iro +shim kus +ser an +sb ks +sant amari +ryanhunter reay +run tastic +run skg +rip nelsonmandela +regi us +raoka vitha +rajag opal +radio surgery +quili brium +quaide azam +pride parade +pon nu +pollok shields +plat y +pa ju +p and +over bridge +or ona +ophy l +online auction +nus ingapore +nc pc +murphys law +mur uga +mu shishi +mu hur +mtv uk +moving quickly +mo ora +mis d +mcget tigan +mam ar +ly st +lu carelli +lex ile +lass wade +lamar ca +la vaux +kir alı +kings things +kim berlin +kar ura +ka aa +jin x +ji my +jadap smith +it gets +irre deemable +inno gy +illa warra +hydroxy cut +ho cr +hef ce +hand al +haeun dae +grass field +gha ffar +fu leight +flat ly +fi as +eu il +es adler +employee benefits +elin coln +editor schoice +drawe veryday +dor ji +donat elife +dl f +depaul u +de tik +dand unn +cur rie +cor sini +con tort +compet ently +cod man +cla hrc +cic carelli +chaper oned +bur nin +buc to +blan kety +blacklist irgc +betterthan yours +benedic tus +ball an +ate man +alexander skarsgard +acceler ant +a ach +. ðŁĺĨ +**** **** +ìĦ± ìļ´ +ëł ¤ +âij ¢ +ÌĦ âĸ½ +you too +yo sp +y it +xen akis +woo sh +wonder women +wicked wednesday +wi max +way days +watch oftheday +w ends +v me +u af +tw illiams +tu llis +tru by +thu l +tech comm +tai you +si su +scip y +scho ch +sch es +sc lera +sc ali +salam u +sa dist +robert smith +ro miti +rich ton +reminis cen +redol ent +re arranges +ransom ed +pinch punch +pilgri mages +peter king +pan americana +pakistann ature +p gb +over top +ornitho logists +ol aya +off wht +nasa earth +mé rida +my man +murdershe wrote +mu do +mor ikawa +moo i +money bag +model e +mix show +mit ra +mind fullness +min iso +meravi glia +mer an +me ols +mazz eo +martin o +mari en +mar acle +luck less +lotu ses +logic pro +ligh trail +lifeat sea +lets doit +lee kuan +lebo res +lar s +lamborgh ini +kunsthistor isches +kmc kidd +jo ely +ir retriev +in play +home renovation +hom eles +here eee +hem lines +grave tt +gr ane +gov garyjohnson +ge tsu +french polynesia +free world +four cade +fa es +f ap +everyday iloveyou +entrepre nuer +em iller +earth month +ea seof +drogh eda +dpan ikkar +dolby atmos +dirty water +devon franklin +death wing +dad s +d oud +curi eux +cowboy fb +colour ation +classic film +chri sn +ching on +chin sky +cher aw +ch acal +cav snation +camp bestival +call ard +blue dot +bere jiklian +baris al +awa ji +att inam +arizon afball +arbaaz skhan +ap nic +ame th +al ts +al fam +ade goke +:: . +ðŁıĪ @ +ðŁıĩ ðŁı¼ +âĿ¤ï¸ıâĿ¤ï¸ı # +à¸²à¸ Ĺ +zing ara +you have +wo ahhh +wind less +vietname se +vampirethe masquerade +v hope +ur ss +un noticeable +uk photoshow +tur gut +transcrip tomics +tr bin +tor ano +themac belfast +the pack +the bachelorette +ter ium +td kr +swif ty +suppos itory +sportscenter ph +sports blitz +sofar sounds +shu ck +sasikumar dir +san th +san joy +ri ess +reasonswhy welove +re tells +re programmed +ram p +plot line +play more +pharmaceu tics +pay pigs +over strand +ot ville +omni presence +ola fu +okon jo +ocon taldo +occident alis +obedi ently +ob elli +museum next +muscle car +montan ez +mis ner +mi hara +mess ick +medi atour +me ret +m for +le tti +kinok uniya +ke ion +ka hanam +kahanam oku +jim erson +jame stown +jam et +jaguar usa +it sgo +inst ay +impre ss +human i +holliday sburg +health policy +he mma +hardwell onair +haiku jam +haha hh +gucc icruise +gom en +gold mining +glen ferrie +gennar ocontaldo +g star +fox rock +fly fishing +flo o +fin alization +falsi fication +fa hadh +esta dio +equit ably +enumer ation +emil yy +em elt +ear m +dom ineering +devop sday +deborah annwoll +dam ba +da beast +chu bs +chik magalur +carsand coffee +bx tch +bri quette +bi bio +banff np +bad hairday +attenu ator +arnold palmer +apple day +andre au +americor ps +alph ington +aham ed +ag amma +adobe illustrator +adi e +ðŁĺı ðŁĴķ +ðŁĺįðŁĺį âĿ¤ï¸ıâĿ¤ï¸ı +åĦ ª +« ม +zdar sky +zah ira +yo ku +yeare ver +yar mol +yach ting +wy bor +work arounds +win free +whowhat wear +wey den +wer nick +waw g +vallu var +vallab hbhai +v crs +tran scriptions +tic hin +ti ron +the hills +th im +teh ri +team messi +sty rke +stro h +spirome try +spald ing +sothebys realty +sling in +shrews berry +shipy ard +sher ali +sha stri +se less +schem mel +sch movies +sc magazine +safe schools +riz wan +resna is +rec rimin +read more +re ws +quan ah +prince sse +pokh ran +pel u +pel otas +paw ning +pat eros +over spill +over burdened +oun is +on fleek +olatun ji +official camogie +ni ji +ng lifestyle +newton more +neu e +my d +multi mode +mis cues +mis carried +michael chiklis +micha ux +matt taven +luci fans +lucas oil +lostgirl series +li sat +leekuan yew +lakeland terrier +ki pru +kate garraway +kapp ak +jonny wilkinson +jo edon +jn co +jas in +irishcancer soc +im seth +ig ing +hurricane prep +hinchin brook +ha vo +gyne comastia +gigu ere +ghan oush +fy ing +footbal lau +fine man +fig c +fatherjohn misty +digital agency +d now +d line +curmu dge +csu sb +con comit +clay man +chi omega +cher ni +charle mont +cer van +can ino +camil amendes +brown barrie +bomb proof +bk club +bintur ong +best es +bend or +bam bini +back home +av ity +at omar +art ane +armi jo +angelique kidjo +al avi +ad hoo +a university +^ âĸ½ +ðŁĺį ðŁĻĮðŁı» +ðŁijįðŁı¼ ðŁijįðŁı¼ +âĢº âĢº +àª Ĺ +zimzalab im +zi um +yy ours +ye stur +wi vedi +we sa +vs jax +v official +un free +u hhhhhh +turf way +tuney ards +tun ny +tread le +transp acific +titus oneil +titusoneil wwe +thereal kmckidd +theposh official +thegreat gatsby +the basement +taj mahal +suf fixes +spiritual awakening +spike island +soccer dotcom +so hu +sky bus +sk ater +shar ara +se tag +rz esz +rte gaa +road tripping +ripp ed +rick springfield +rich ter +ra uk +q do +pom be +play harder +penc illed +p man +own town +odd s +ns ls +nko sa +ng am +n wed +mu ssa +mr darcy +micro car +mar wari +malm stro +mac jc +ma ung +m wo +lin in +lake view +l summit +ku ja +kirrie muir +keepfighting michael +k maw +jhal akon +ja ja +islam istheproblem +ic title +i ven +hudson weather +hov land +hol steins +hipp ea +har dens +gwy dir +gw ladys +guy zz +god manchester +go ti +g ors +fu jin +flash tweet +first grade +fac daniels +f kd +en sing +e wing +du shi +det sky +del mont +day stom +dan h +d sena +d land +cy fair +cu mali +craft bizparty +cour son +coffee script +cho han +chi ons +cep tive +car doftheday +campe ona +buli mic +bri mbank +boudhan ath +boom slang +boom bayah +bo ty +bo the +bis wa +bin us +baby led +b hh +as par +arkh angel +amnesty usa +alve olar +alle stree +ali a +afro house +adrian edmondson +actu ate +actress life +(( (: +íĻ© 민íĺĦ +èĭ± ä¼ļ +ج Ùħ +z andi +yose ary +yaf antasy +whipp any +wbb mnewsradio +wall bridge +w you +vy rn +var chives +un ak +tritic ale +tre sem +too hot +the journey +ten anted +tall ship +ta pio +t morello +syn ched +step family +stay la +staser aintv +springhas sprung +sla ppin +sl mc +sko ch +shad ings +sh kin +sergio garcia +sav vas +s gi +ru pali +ron del +ri va +requis ition +rendle sham +recon quista +raven scourt +rah mah +ragh eb +ra ir +qur an +puro resu +por gs +pneu ma +periodon tics +pen alise +one ering +od ourless +o cker +nu bbin +naf is +my hill +min v +mil azzo +me gau +manu rewa +mad h +mac es +lil ja +libby schaaf +large sse +laid ley +l trs +kur saal +kome diab +kfm za +j antz +hand cart +ha ining +gu yoseary +goldenstate warriors +gil les +ger ads +games master +flu o +flavor ings +er onen +energy access +en ro +ek im +eco logie +eart fair +e stel +disney channel +dgand yofficial +den isa +del vecchio +dead rising +cu bi +cro eso +credit cards +cor oz +cli ocup +chew able +chamber land +cad dell +book out +bla bla +bis p +bill yon +bhar ti +bec ton +ba ret +atro x +ani us +all rise +adve ction +? ðŁ¤Ķ +<<<< << +( ^- +åħ ī +ൠĭ +world fest +with heart +wat kiss +vinyladd ict +vino gra +ve dt +vander pool +uss ain +ur mi +un sophisticated +un desirables +un approachable +ty rie +tu lp +trust me +traumain formed +thedead daisies +the jagmeetsingh +thar bour +sy ro +sy chi +stop avn +steel town +st francis +shu ichi +shav ing +second home +scar f +sas ural +sandwell council +sab lan +rohan bopanna +resc ities +recor riendo +ration alist +pie stewa +peep z +pap ato +pal af +pag odas +oh s +neel ima +mother ingsunday +monet ary +massaso it +lö f +lu lay +loch ner +lo vering +leedu chat +kyungsoo day +kj show +kil ala +kh ate +kay ani +ka ist +jhalak reloaded +ing post +in opportune +in concert +image sof +ike men +halloween time +go tem +glend inning +gender bend +g afford +etihad stadiumau +el by +easter monday +don nap +diamon t +deano gorman +da sia +ctvmorning live +cron kit +cir lik +chri scar +chin ta +chi ado +cam bon +c dre +brandonj routh +bon doc +bo olin +bid ston +bi kela +begum nadiya +bab an +aun gier +artiston instagram +aqu arist +annihil ates +anim enyc +ald abra +aj m +air dro +ad ome +abor tive +ðŁĺ¬ # +ðŁĴĻ ðŁıĢ +ðŁij¼ ðŁı½ +ðŁij¨âĢį ðŁı« +ðŁİ Ĵ +à® ´ +र द +zeecine awards +wildflower ctr +water management +wai kiki +von taze +voice top +unlv mbb +un born +ul er +tweeta photo +trans acting +the jeep +thai pusam +tech sters +tasty trade +sw ich +stom ata +sta th +st mt +sol l +sla thered +side y +segu ros +sc ps +sc chat +sar ag +rn k +rend ts +reel foot +ray ana +raisingthe bar +ra pin +pu tri +prophe tic +press burger +por ty +pla xico +per r +per ps +pal matum +over age +outeni qua +oro driguez +ordin aire +open farm +nj dv +nike uk +nau ck +nar thex +nacion ales +mor ti +mone ill +moneill sf +med spa +mcclo y +mc grain +mat rons +maryam nawaz +mar gate +m ds +lease holders +laura osnes +lamin itis +l den +ko stya +kis mat +kii ara +ju bb +james r +il ham +ico in +hy me +hou rigan +heart landon +heart fulness +harper adam +google pixel +gon u +girly things +gin i +gillian jacobs +genoci des +fon dled +fi ster +fashion gram +f tr +explo der +exib ition +ex ilis +esh our +english language +edd zeko +don ot +do led +disal low +david giuntoli +dalla ss +council day +con boy +chan thaburi +cerebro vascular +cast ros +calgary police +c ÃŃ +bromel ain +bridal market +billy crystal +bij li +batman vs +ba ise +ammy virk +adil hussain +aap ki +.... ..! +ðŁĴ¦ ðŁĴ¦ðŁĴ¦ðŁĴ¦ +ä¹ ĥ +âĿ¤ï¸ı ðŁĴį +འ² +à± ª +z suz +z sc +z ali +wor g +willie geist +war shaw +want s +vi vat +ve co +vanat chathiram +van brugh +usic live +ureport ke +unchar ted +un too +uil texas +toulouse fc +tom oka +ti jani +the town +ted leo +techno polis +te itel +t zi +swith un +studio city +slovenia info +sl q +sky bar +si raj +shain anc +schipper ke +rolling loud +ro dro +rio ter +re populate +rain or +ragaz ze +rac tice +point les +plot kin +player stribune +pic ador +peewee herman +pay roll +pac os +our stars +ou sia +other kin +or rrr +ob trusive +north well +no ongar +nin jas +nic hd +newpor tri +ner dgasm +ne res +national dessertday +mu vy +mayweather pacquiao +maximili ano +masti mania +mar sy +man ap +mal ate +makelife aride +ly y +ly ria +leas owe +lea quitaine +lay un +last chance +lan ow +la quan +l porchestra +ks v +kor ach +kil dee +k nation +ju iz +johnnyg weir +jes sel +j arri +ital ks +inun date +inter weave +ing rati +ib is +i aido +ho ste +har ket +happ p +han amaru +ha ug +great memories +go tyj +go ines +gilber to +geek dom +galax ya +et ana +este fania +ellip ticals +e stro +e ecs +dur gin +dontbomb syria +do dsworth +dead beats +curric u +ct politics +configur ator +colly er +collegi ality +colec cion +co su +co dex +clau ss +ck ler +cityand colour +chi mie +cardiff bay +capital stb +callo fcthulhu +bruhits zach +bru yere +bray brook +braintu mor +bolly mastimania +bo gar +beth lem +ben ro +ben ck +bang ura +bal adi +ati as +ar curi +ankylosing spondylitis +anc elife +an eda +ambi ental +alpin ist +alexandro v +abo lishment +ab dash +a hin +.... ..? +! ðŁĶ¥ +ðŁĺĤðŁĺĤ ðŁĺį +ðŁĶ ī +ðŁĴľ ðŁĴĸ +ê·¸ëŀ ¨ +ãĥĸ ãĥ© +ãĥ³ãĥ Ĩ +à¸Ķ à¸ĸ +za ghari +ym b +yc nature +xabial onso +weather watchers +ve gal +us womensopen +unob tainable +un volunteers +u selections +u donis +twer ks +tr na +tish james +timber frame +thistle down +the stage +the palm +the champ +th ely +tee zy +tax cut +sé rie +ston efly +ssun stein +ss om +shum way +se bas +sale sian +sak shim +ry se +ry pt +rotor adar +red hook +re creations +ran jan +pu dd +pom poms +pl tw +pemb scoast +pe als +parliam en +palindro mes +p sim +over rides +onep ur +nik laus +nen ownews +mueller investigation +micro bead +mehra uli +mb alula +materi alizes +lyre bird +liv ant +lightning deal +ku roi +kp j +ko va +kims convenience +kh of +katzen berg +k under +jour dandunn +jnj news +jim ura +jeep thing +jamesp oulter +jamesmar ster +jag ga +ich ry +ia fe +i sport +huntingdon shire +horseand hound +hope fulness +honor ing +hazel dean +happy tweet +h fo +guti érrez +gu ery +grace church +go explore +ghes quiere +funky town +fly catchers +euro centric +et witter +eli sts +ed sheer +drink bodyarmor +dispro ving +definit ely +decath lete +debt trap +dan sen +d memories +cw f +csn chicago +costu m +com pre +climateaction now +chlor o +chall en +cec ina +cas ssunstein +canon usapro +cad wal +bro force +assas in +art crew +alve ar +al toon +agon isingly +aga st +. )! +- + +ðŁĺĤ ðŁĺĪ +ðĿĻ ļ +ë ķ +ç ¤ +à¹ģภ¡ +zom bs +zof ia +yan key +wokv news +wiz bi +wi pro +we buk +warwick adavis +viz wizbi +villa franca +vie tor +venice filmfestival +vehe ment +uefa championsleague +tsne deker +tri eu +toro idal +ther ight +theli st +the arto +thank youn +temer ity +swachh india +sth d +sta a +ss wim +spin abi +son oda +sock sout +so dding +simul acrum +simpl on +sign posts +side walls +shak yam +setau ket +sat ine +sand storms +sa adia +s vet +s grace +run o +r ham +pro long +pre tori +po veda +phir se +pe gues +patro clus +pal omo +over lake +out magazine +our heroes +oun ce +ori ordan +on gus +ol le +of sted +ny heter +no conversion +night spot +n ols +my twitter +mood iness +mo pa +mo dems +medit ators +me spo +max ted +max pro +mal olo +mail let +mag nier +loo ka +lo sar +lear month +landmark game +l wk +infer no +in gos +hy nd +ht ml +he wa +har ap +hal ore +gra yer +gold thorpe +gal gos +fusil ier +frei berg +forward together +fm wales +fat ai +ez idi +exp consulting +etv scandal +elevate gg +doofen shmirtz +do we +di restra +defin iti +de ely +dan ke +d smith +creepy puppet +cot man +confl ating +chinese theatres +chan tier +celebr atin +cashme recat +caravan serai +car photography +camp ello +c pos +bryce papenbrook +brand tsnedeker +boo te +bobvand illen +blon der +bizim hikaye +bil ko +bha id +bensen ville +bear sted +bath i +balla chu +aw oman +aven e +arts district +art u +apcu kingdom +anti static +andro b +alphanu meric +almer ÃŃa +agit prop +ad reno +ðŁĺĬ ðŁĺĭ +ðŁ¥ Ľ +ðĿĹ¢ ðĿĹ +ë¹ Ľ +âĹ ĩ +ze bu +yor lando +yoff ish +x anthi +work up +wetherby hour +waveform gaming +v ory +us nsa +us ami +un ning +tri on +travel noire +th burne +tb ptv +tan zi +sy b +sun trap +summers lam +sr ch +spl ant +shot left +shan tou +sapp arel +sanssou ci +sal ang +rod kar +rho desi +rh cp +re fashioned +qui am +pumpkin seed +puma southafrica +pre ux +prati ma +pog dogs +platt ner +pint size +phen mj +pau lan +pau dybala +ost ler +on style +on salenow +ol oroso +ni ddry +new burg +nast ase +mu six +moi sten +mixer streamer +mar mal +makav eli +ma fa +lou n +long ter +lon done +lime wire +la en +kor ba +kom olika +kk crvenazvezda +kav ana +jur gensen +jamesmarster sof +it sma +ip sen +ino saur +ing love +ic ml +ic amp +i wt +hi dup +gre bel +gra dings +gr antly +gl ück +gilli gan +gar aki +for freedom +ev als +esh t +du ga +do gin +dc sd +ct z +cow man +cor ke +convey ancer +contribu tory +chiric ahua +catal in +carstar phenmj +car yl +c pi +bu bur +brun dle +brown sboro +boston fire +body painting +bethnal green +ber line +bedd gelert +be zz +bakers field +avi acion +australi as +agu ita +age ant +aeroline as +$$ , +! ðŁįĢ +Ĥ ĺ +ðŁĻĮðŁı» âĿ¤ï¸ı +ðŁIJŁðŁIJŁ ðŁIJŁ +ðŁĩ±ðŁĩ ¾ +ìĺ¨ ìľł +ä» ĭ +人 rt +âĥ£ âŀĸ +ya day +x slasvegas +watch face +w ge +voyeuri stic +vo va +visual merchandising +ver wood +vas ile +up y +twopad docks +tweet link +ther ace +te be +ta ar +t go +sun danc +sting less +sosn icaragua +singleuse plastic +share file +se acombe +scre ek +sargas sum +sarai va +s enga +roosevel ts +right sarehumanrights +rejected jokes +reg icide +re ais +ra sika +ra kh +q urban +pul ps +pom pe +plat ano +pic tori +peter roskam +pase kand +pasekand paul +pan oz +no it +new mom +nd ma +napp anee +mur mu +motor hour +morph ism +men ted +matra ding +math art +mark duplass +mariob autista +man official +man gel +mal low +mal formed +madein scotland +m seto +lo bato +live jazz +lin ac +li salam +la ge +ko ken +key strokes +juan man +john elway +jagu a +indeb tedness +in ri +in conveniently +huar aches +host as +hook worms +hil ip +heartsofoak gh +haw kinson +greg jennings +gom m +god flesh +glos sum +ghar ana +freed land +flori dat +fl atlay +fin ola +fin ear +feder ation +fan euil +ent en +en quanto +efan club +edmundmc millen +distant worlds +derek theler +dc te +da esh +cu dnt +creati ven +cra ver +coton easter +confor mance +clar kie +ciar án +chrisber ghd +chi w +chekk achi +ch ach +cav anagh +cas pers +brew is +beck ys +be hm +baz za +asap nat +ar rrr +ani sta +aneurin barnard +ameans business +alex slemonade +ale cology +ad sit +a hil +ðŁij ŀ +ðŁİ¤ ðŁİ¶ +âĺ ¢ +zi el +yu ku +yam ec +xi ah +worldskill suk +wn ation +wh omes +vw beetle +vo sloo +vale ska +ul va +ucht dorf +u cros +ucros spop +tough man +thought bubbleuk +terp stra +tay m +syzy gy +swe mfa +sun ni +sun king +summ ering +stone island +steph curry +ss ch +sp insters +shim o +seen ur +sap cp +san jo +sac valley +ra bal +program ma +presidenti alelection +pic on +piano forte +pegas o +orchi daceae +open space +op rint +ocean arium +n fm +more years +mor umbi +mezz o +meu len +me up +mal dive +ma kau +lovemy family +louren co +lo hen +liket kit +kick son +kel u +kare lian +ju nee +john travolta +jersey boy +it yo +ing ian +imper o +hud hud +height ening +happ ily +gri fo +green tech +ge dsb +gaz asi +g gh +french town +fest ool +ex is +esc apology +empire strikesback +el ga +el fen +eck ner +dra che +donne come +donnecome cipare +dj booth +di jual +decor ah +daf f +cul o +cro ghan +covar rubias +cold case +cas cio +car canet +cap alaba +bur meister +bo lete +blu ess +blo k +bioshock infinite +bg motogp +bbc nottingham +baybay in +az oulay +ay eeeee +appetite for +anti ga +anil ine +angk lung +ak ap +aids walk +ah p +ag rf +!! - +ðŁĺ¨ ðŁĺ¨ +ðŁĴª ðŁĴ¯ +ðŁijij ðŁĴĸ +ðŁIJ¦ ðŁIJ¦ +ðŁıĥ ðŁı½âĢįâĻĢï¸ı +ðŁĮ¬ ï¸ı +ðŁ¤© ðŁĺį +ìĿ´ì Ķ +âŀ ¢ +ÙĤ ÙĪ +ün den +yt ff +yoko gawa +years agotoday +yacht club +winner sh +white people +weyer haeuser +wab o +vzw buzz +vine sh +umm i +u vula +u cia +traditional home +the sen +thanks givin +templ in +tal ca +tai led +syl wia +swift key +stru m +sk yn +shadow cat +sees outh +seb divo +se mu +scy cle +sar daar +san guin +rani khet +r hames +quar tey +pre emption +poldark tv +plebe ian +per ations +pen ang +pawn brokers +ox blood +onto logies +od ni +nuf fiel +north pennines +nico tortorella +mur gh +mir amax +mil bury +micro biomes +mega projects +mcgla shan +maurice benard +marr show +man cino +lo as +live inthe +legobatman movie +leaves den +le thaw +lake town +kru is +kin ison +ka fue +jen kins +jaide ep +im prisons +ichi ello +hil aria +hi mani +grizzly bear +greg grunberg +gor gas +gn z +glan ford +gam elab +gage town +fc academy +fanta st +expen ding +evan sdale +emp angeni +emily slist +em presses +ell park +eli hu +dul lahan +dr jason +day atthe +dan ariely +dam sels +csi ro +comm ing +clare mont +civari ze +ciut at +city farm +cis gender +che did +chantic leers +cal care +bur tons +bon ton +bom et +bl ough +ben haenow +ben ben +begu sarai +bee de +bak an +bag ua +as cc +ar cy +anni ster +anglic ans +alle les +al ute +agu delo +afl north +adri aan +ade osun +ab bv +ðŁĻĪ ðŁĻĪðŁĻĪðŁĻĪ +ðŁĵ· ðŁİ¥ +ì°½ 민 +å¾ IJ +z ino +youth summit +your call +wn cwx +with nature +wi eck +way station +watchme work +voor hies +views for +veri thanam +v fest +us new +ura sawa +tv j +troop a +tri fluorome +trans duction +tom or +tin dle +the press +te oh +tan zer +super ted +stor i +sth our +steam punk +sl oman +shil pi +shamp oo +sh ash +sex periment +san sevi +rugg eri +ron ix +rob shaw +ri vette +rc slt +raghe balama +rachel doesstuff +que ster +propeller head +pra z +potenti ality +po groms +ple e +pinup girl +petz old +pattim urin +pat ang +or ley +oce ancity +nur gle +nun dah +nowisthe time +nor wood +nomuslimb anever +ni vas +nef yn +mous avi +middle village +mey hem +medal of +maxim ises +mart elly +marianor ajoy +malmstro meu +ma hur +ma fu +lu bber +lovel ords +long ship +le pel +lat l +la chowski +l alab +karab akh +k oral +jewellery quarter +jay mewes +j dc +iri zarry +ini sterio +infection prevention +in eland +icy cle +iclassical com +ic re +hou f +hoo fed +hol dren +hek mat +grote sk +great apes +go thel +gi at +gadel maleh +frenchal ps +fore telling +film tv +every ones +england v +e ppo +do ar +dc is +cool um +competen ces +colon ise +co bi +city year +ci anci +child protection +candle mas +bullet proof +boat y +blv ck +being coop +band q +bally mal +bagsof help +au llo +anirudd ha +angeli e +am gu +am ab +al annam +abo i +aban kers +aaaaaaaa aaa +. âłĢâłĢ +ðŁį´ # +ìĦ ł +é¤ ¨ +à¸łà¸²à¸ ŀ +zan es +z illa +yor dano +wahe eda +w stc +volcan ology +vic eroy +trum pian +tric ycle +tre ys +tomo hiro +the profit +take s +swi pe +sumptu ously +stock holder +stead iness +star log +ss ilver +sre i +sp igen +sor presa +son ye +soccer six +small talk +sky cricket +sie wert +shimaz aki +shiki gami +sco ble +santay ana +rup turing +rose well +rosco ff +root stech +roman zi +redwhite andblue +raic hur +rai han +pv p +pp u +pharmaceutical medicine +pey to +pa zuzu +pa vic +of en +obam ba +ob om +nor lin +no confidence +nn sa +nk ana +ni pe +n cep +multic enter +mouth guard +mi az +meop ham +md v +may nes +mau de +mari jke +lun aire +lifeof desiigner +le comte +laine y +ksh b +kom in +kir win +kane wwe +jic ek +isal oni +isa ach +iran election +in attention +iit madras +hype app +hero in +hepatoc ellular +har key +gwilli mbury +gu altieri +go camel +glycer ol +gk union +gangnam style +fre ke +fre el +fel brigg +faz ura +faul ds +fau rot +ev aa +erskin eville +epic reads +echel ons +easter seal +dr ane +doc teur +disney hyperion +diplom at +diade troit +de wees +dd lc +dar ci +dant as +danne mora +cout u +chor lton +chic ou +car fest +cap sizing +c micon +british spiders +brac keted +bom ar +bo ber +blog paw +bay les +ba chill +ba cher +ath ousand +ar leta +amuk erji +ale ague +ad ames +acc tourney +ab rit +??? ?" +ðŁĺįðŁĺįðŁĺį ðŁĺĺðŁĺĺðŁĺĺ +ðŁĵļ ðŁĵļ +ðŁĴ© ðŁĴ© +ðŁİĵðŁİĵ ðŁİĵ +zapo tec +ye pes +xx yyxx +wit kin +whipp oor +vu lus +vivi ano +visit japan +var den +vanqui sher +van u +v dsl +tu xie +trivi athursday +travel africa +to kaji +teve rest +tekno logi +tech nis +ta vit +syncop ated +sw ool +super couple +sundar ban +sub mitter +stru ff +star force +sportsc asters +sp ä +smriti vidyarthi +sig sauer +shi bu +sequo ias +sc b +saul williams +sare k +rosen kavalier +ron in +ren toul +pra de +pink history +pic hardo +phyto ph +pen za +pal et +ourland our +or li +one year +nr lakl +new urbanagenda +ne iges +ne els +n stom +mom ota +made at +lyn donville +loonathe world +lo anees +liquid lipstick +lay men +lar avel +kim bo +kar thick +k sf +k ren +jun yeol +jav elin +inequ itable +imag ing +ida home +iam dbanj +hagg en +gul mohar +glar ingly +gd antes +g elo +fu xin +fren kel +frank ness +fr ys +followthe cow +electro magnetism +dublin marathon +dubai marina +desig ne +deb nam +dare ss +dar rion +continuous delivery +clarem bee +clar dy +clan william +citi bikenyc +cebu anos +c figueres +bu mming +bra p +bra ben +bier ut +beauty bloggers +beauty and +beaker head +balon mano +bal derton +back slash +back nt +ba ilo +b flo +award winner +au te +at sb +assou line +as oc +armand de +and dddd +ali jo +agu ardiente +advis or +ad alli +a events +ðŁĻĮðŁı¼ ðŁĻĮðŁı¼ +Ù ¢ +ع ر +zai batsu +z if +yog scast +www blogs +wake andbake +wag gon +vol aris +valeri o +val ores +v yan +ulster museum +transhuman ist +thri ving +thr ur +there for +the island +tc ja +taw dry +tam riel +sugar hut +stay well +stan bic +sp idad +spidad mitchell +solenn heussaff +simil ar +seo joon +sas ne +sand ino +ru zzle +restar ta +reli asson +re ten +raz ones +r khan +pri js +pp ls +pou lain +po ire +pin points +petre lla +peer age +pec kem +peace forchange +pas kal +part is +parasit oid +or adell +onz ales +ober st +nps official +nph w +nicole and +nichi ren +national loveyourpetday +mu za +mol ysis +miri anag +mirianag rassi +michelle phan +materi alizing +masa o +margare tta +make m +mag as +le ire +le ab +labor sec +ketu pat +kam ble +k croyals +jor hat +jo su +jeffreestar cosmetics +jak er +j lab +ishq subhanallah +induc er +im and +ilo cano +ick worth +ibm security +hi po +hand knit +ha em +go peng +gim sswiss +ga etan +flipk ens +fil led +eye onthe +es la +encoura ger +ely ons +dy i +dr sarah +dough boys +discover able +del imitation +dear born +cr sh +contemp tuous +coat rack +co xe +cl inger +ce fc +capp rentice +cam pero +californi awild +bur wash +bu stour +bu sker +breakfastclub am +bo dak +bit trex +barri ga +back packer +b wt +b sg +at onal +asu xx +art matters +arro char +ap sphysics +antic lock +ant sula +alo isi +ak ins +aer yn +" << +ðŁĴĭ ⾨ +ðŁİĦ âĿĦï¸ı +ðŁĮ ¨ï¸ı +ãĤ° ãĥ©ãĥĸ +âĢ¼ï¸ıâĢ¼ï¸ı âĢ¼ï¸ıâĢ¼ï¸ı +د بÙĬ +ä ki +zah ran +xe han +x force +wi za +who tel +weekend read +web toons +weare jcps +wc ps +w str +vermic om +veg ard +unzi ata +ulti mas +ul trafine +ucl ambb +twin ks +time outs +the gallery +th ence +swift ness +suc at +strato varius +stats zone +sn cla +sexy sunday +severy body +sep toria +seenur amasamy +scorpion fish +sco s +run neth +ric ki +rhi an +retic ulum +res foundation +red lion +po sco +pba ontv +parro toftheday +pap hi +pan ela +o zeri +northeast news +no wa +museu mc +motor land +mccl ary +mate ship +madhu du +maceach ern +lin ec +lie man +lette rer +le aches +lamb ily +la sairport +kon st +kle berg +kle a +kaushal army +kait lynn +kafka esque +ju raj +john carpenter +jay mcguiness +jacque storres +j sp +irish design +ire zumi +ingui de +impe tuous +id ade +ich ella +icec ap +holi bobs +hir schi +hil ts +he dren +harrypotter film +gyeong bok +gor nal +go recki +go further +gla as +garri gle +gan ap +ga illi +fren k +forte an +fish lock +fin det +fer ron +fanni emae +exfoli ator +ex claims +es mee +enti es +emmitt smith +emb ree +ell it +drummond ville +dr d +dogs nyc +diversity intech +dil um +de pon +comment ated +clu ff +chi dori +che fan +categori zes +car dell +canasto ta +bride shead +bri v +bri jesh +bre search +bold ly +bli ppar +black stone +biz women +before i +be dou +be ag +at ong +ar ges +analo gues +amare lo +abhan sali +- [( +è Ļ +æµ ľ +âŀ ° +âĶĬ âĶĬ +à² Ń +zax by +xin yi +wr ps +whiskey wednesday +wak iso +vici ou +vegan recipe +tt g +trivi umofficial +the cbso +thati sall +th iri +tee zer +tav leen +t sak +subtrac ted +sto le +steam town +ss rn +sop io +sm fm +she ppy +ser pa +se ato +scand rick +sc ir +saw telle +sac and +ro mig +rent schler +red lobster +realclear politics +re growing +rat v +projec ts +prin sen +pri show +pe lec +paw fect +pal merton +ota ki +ot di +or tunity +onthis date +ol alla +ogden sburg +of rac +ni ggling +my slf +muhyi ddin +mini figs +mikaela shiffrin +mend inger +men ot +mat es +magi sta +made inspain +ma kalu +lympho cytic +li kowski +lauren holly +lau be +kur ti +krish namo +kh uri +kh ill +j ör +j haveri +isol ationist +intothe unknown +innocent i +iheart country +horn dean +heartlandon cbc +habil itation +ha ole +ha ffner +green landic +gly de +ge ot +gario ch +fundament al +fro ots +for one +flat fish +fayouth cup +faul k +fathom s +espre sso +el na +eke inde +diversity matters +di ther +dan ja +dam ita +custom s +coa sta +chou ler +chen ery +chembur studio +cc np +cassio bury +carla harvey +cam pin +by les +buc cos +bu yon +bou lay +book tours +bene gal +bap tise +balti erra +ball ymurphy +at rac +as itis +ar vest +antagon izing +an tre +an amne +all youcan +alam in +adventu res +adel boden +ade b +ad jani +acer ola +ðŁĺħ ðŁĺį +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ðŁĴ¸ ðŁĴ¸ +ðŁIJ¯ ðŁıĪ +ðŁ¥ Ĺ +ì² ł +ãģķãģıãĤī åѦéĻ¢ +áħł áħł +ystr dy +workington town +walk outday +wakar usa +virtu emoir +video tapes +ves sey +v line +usatf outdoors +tu e +tk allday +thiem domi +theevil within +swa ard +su ster +str atten +stam ey +st live +sr ks +slo ck +sle ee +ship ston +shi rakawa +shaf ting +sbu f +sau sa +sar ja +saint motel +rockwood nyc +ro ys +rif kind +recircul ation +re shuffled +ratt us +ration alism +rand fontein +rais i +pul sation +pra x +piazz ale +p tm +p my +p few +over diagnosis +no tobacco +no fur +nav an +nas uwt +moer aki +mobile phone +men zi +men tira +men go +mel illo +mc café +maggiel indemann +made it +macdon ell +lo rene +liven ow +landing page +kyrsten sinema +kuro ki +kier nan +kid dush +kai ley +jo sten +jad er +ir za +in situ +ice breaking +ic tures +hy nix +hu huhuhu +hopenot out +hippea strum +hg hair +heis en +he ddle +hat chards +ham mock +granth am +gary oldman +gansett beer +gall antly +gal lion +fur thered +franco phones +fin ne +el iner +dol and +do ire +dne ews +demo gorgon +dementi auk +dayof pink +cv spharmacy +cor adio +co be +clyde side +clarine ts +chung king +chocolatec ake +chi antic +che c +cer ney +car pel +c sdp +bren au +bol on +black swan +bill ye +ber magui +bel loc +beart ooth +bal aram +ate gui +ast it +ash s +ascor bic +as pers +are mos +ar vid +ar rakis +an sf +amar anthus +al biz +agu st +adul terer +ðŁĺįðŁĺĺ ðŁĴķ +ðŁĺĤ ðŁĺ´ +Æ Ģ +zet tai +z c +yo a +wwii museum +walking tour +w kt +visit ms +vide ophone +ver k +uach tar +try m +to bler +ther ace +teslamo del +tech navio +taver ne +tail coat +ta quitos +stil ted +shal f +sec conf +scra wl +schri stian +saifu ddin +rosat om +ron delle +rhu dd +rebeccas ugar +re iman +quin tiles +quer é +pur don +pun tag +pro by +pre conditions +plu mer +photo gravure +phac elia +pep tic +patty jenks +pal let +open vpn +obo ist +o vale +nz x +nor ges +newsp olitics +nasa insight +narr ators +n je +musi que +msh saa +mr music +mmk maymay +mine fields +mdc p +mb laq +maxim ised +mati bo +mar cle +ma pi +ly ri +lowe red +love thy +lord minion +le froy +kri stal +jubi le +jack olan +jack nicholson +island peeps +inter county +ik ke +i win +horten sia +hor sman +hil len +he bel +he aux +hawes water +harmon ising +hard drive +hamban tota +hai k +grey town +grenou ille +gar ance +fur con +fresco baldi +film forumnyc +father like +ey in +et x +equ itter +el ens +ejac ulation +drake hogestyn +deyoung museum +dere r +deal withit +cy be +cri xus +countdown to +co chem +cl ancy +ced illo +catapul ting +bu gat +brigg s +bra ai +bow eni +beyon der +best buds +beauty by +bar a +back dated +b too +awe st +at its +ase k +arm streaty +ar amide +any a +an fal +all winner +alici avi +ðŁĺķ ðŁĺķðŁĺķ +ðŁĺ¬ ) +ï¸ıâĥ£ / +âĻ¥ï¸ı . +z atch +your life +wrecking ball +who youare +we streamers +we gs +w fyi +vite bsk +vis njic +veld hoven +vaun wilmott +us bp +un pleasantness +u stennis +tonyi ommi +toe hold +to bio +thisis what +the barn +team yoshiki +sven sk +subur gatory +sto chowa +stereo scope +st maarten +spit ality +spa stic +sor ters +sno we +sl fl +sight line +sh antz +ser gal +seng ineland +sa ung +rober ge +ring lets +ri ed +red wolves +raj ab +r km +q nd +preten tiousness +pine do +peter hook +pastry chef +par ce +on kiss +nzo ia +no vio +nie g +nfl redzone +nag ara +nab er +mookie betts +ma sten +lu ganda +love field +li j +letsgo warriors +leone an +le tham +krist novoselic +kin da +kas uri +kali ka +jupit us +jonah nro +j ide +im ing +il te +holy cow +hemorrha ging +harris ville +har less +hand cuffing +hamp us +hal ala +hagan ai +gulli bility +gran ado +glass blower +gareth malone +fu u +friday nigh +free e +fr ate +foxsport ssw +fore warning +fm hs +flu te +fer rar +dri mn +down shift +doire gaa +depreci ated +daenery stargaryen +crush monday +cmicon tent +clenden in +church down +chi kka +chalo bah +ch war +car cosa +car ap +bur ness +brick leberry +blun den +bi pa +bemid ji +believe movie +bc sn +ba ot +b prd +aur al +atomar aullo +astro cytes +asi andream +art brut +armandde brignac +al ge +aap kad +aa ap +.. ðŁĺįðŁĺį +!!!!!!!!!!!!!!!! !!!! +æķ £ +å®Ŀ çŁ³ +ÑĦ ле +ʸ áµĴáµ +z hai +ya official +woj nar +with h +winter weather +water bird +wait omo +w dam +vol kova +vi hara +v kook +us ms +up field +travel ers +trail magazine +trac tions +token ism +tobo lowsky +thequeen bean +tel stra +t atten +stutter heim +sto ichkov +sporting news +sp atch +sna k +sky liners +six tus +simp kin +shortland street +shag ari +sep tal +sen markey +salta ire +sair at +sad dar +sa sebo +ro an +rene es +real ricky +ps ons +ppc chat +po red +plent yoffish +people analytics +pensacola beach +party list +pad er +p ère +outandabout scotland +ornam entals +orn dorff +oracle database +or nately +oldschool rs +occlu ded +novo a +nine wells +nic astro +new spring +multic ore +more land +mo azzam +mescal ine +mell anox +maw lana +mart lets +malcolmn ance +mag as +lp tnhs +loh mann +lo were +ligh the +lat ins +lar usso +lam ont +laf rica +ko g +kf z +jay cutler +j klf +its complicated +inv ader +ingh ands +ing ate +in sets +in day +iam writing +hungar ian +homelo ans +hel lebores +had nt +great deals +gouver neur +glade water +ge su +galar za +fc family +extrapol ate +enic ole +el ver +d family +crypto twitter +content creator +cole moore +clear brook +chol lo +chatt ooga +ch ry +celebr ities +ce sme +catastroph ically +car ducci +cann an +cadi eux +c cia +c bos +bur atai +builda bear +bo fam +bn ha +bird wat +bio available +betten hausen +barbar ous +avogad ro +ashi elds +ap tx +ap lace +amphi bi +alli reland +ain scough +_ ... +?? .. +ðŁĵ° âŀ¡ï¸ı +ðŁİĨ ðŁİĨ +ðŁĮ² ðŁĮ² +رÙĬØ§Ø ¶ +аР´ +zad ok +z ado +young band +y ilan +wunder land +white boy +wheel an +wend ling +wade yar +wa yofthe +vo ise +vidy ar +ven era +usaid gh +u shanka +tropon in +the stra +the english +the boris +tatt va +ste agall +star sailor +sou tient +smur ray +signofthe times +shi ka +schaff ner +scani agroup +sand boxes +rough shod +romantic ized +romanceno vel +rochester rhinos +ro mul +rit ter +rebeccam inkoff +ran fur +pop choiceawards +po boy +pipe fitter +per aza +pe mp +ount down +ok aku +o ecs +o budu +nun aw +nunaw ading +nosy crow +mu mmer +monarch ies +min oring +michael jai +michaeljai white +mccar ney +march break +lu oyang +loc alist +les se +ki din +kal aw +jan ek +iwas aki +ile ostomy +i ally +honor club +high performance +hd w +hat v +hahahaha hahha +gy or +green idge +green backs +grant kirkhope +gra p +glass ell +gla ize +gied royc +get outthere +gema yel +gat z +fun niest +free access +freddie highmore +fisch ler +epas cott +epascott pruitt +ep yc +en tearth +easy recipe +dur kee +drogheda united +doc x +dis u +de mars +david zwirner +david spade +dat os +dalhou sieu +cv v +craigella chie +cosmo logist +congreg ated +ci gano +chal mer +bu ste +brew crew +bli xt +blat z +bal aban +b itta +auto somal +au ghters +ataste of +asi es +art yom +ark itek +antoni ob +adv ant +ab antu +aa aw +, ? +ðŁıĩ ðŁıĩ +ðŁįĢðŁįĢ ðŁįĢðŁįĢ +ðŁĩºðŁĩ¸ ðŁĩ¨ðŁĩ¦ +íĭ´ íĥij +ç© º +âĺºï¸ı ðŁijĮ +à¸Ļะ à¸Ħะ +мÑĥзÑĭ ка +zag ora +ye esh +y ce +wrink leintime +woman iser +western union +wee tin +weare ready +vou ches +volu mab +vikram vedha +velá zquez +u ark +tr v +toshi ko +tole dano +to wey +to de +tie de +thin ners +sylve on +studio canal +statecapture inquiry +sso good +soufri ere +snoo per +sj v +site map +shilpaf am +sharealittle sun +se hs +scic luna +sche ide +salem wgna +rugged ness +rub ins +ri ppers +re flation +rav ina +rat ana +raf finity +radi als +que tte +pro om +pregn ant +pra sh +pep so +pen pal +par da +pam m +paide ia +or it +one goro +olvi do +numan cia +ni volumab +neversay never +n dez +muvy z +music bank +mu gu +mr scot +mor and +mon than +mollusc monday +mic e +merchi ston +mati sts +marvel legends +mar oun +mahil acongress +mache tes +mac omb +ma den +ma dai +lieu tenant +lauri emit +kerber os +kan ungo +jawa an +it ng +in sar +ima izumi +ide es +iam john +hpp lay +han ous +hal us +gu ler +grounds keepers +gour cuff +gay men +gam ali +gall and +g ba +fujis awa +fireemblem threehouses +f sw +european art +esp aces +el eri +egg ar +edmonton expo +e ka +dy ersville +dor rie +dong wan +dewal ttough +dev ac +deter rents +del one +de so +dale m +d group +czecho slovak +cw the +customer journey +crystal bridges +convivi ality +classicrock mag +change iscoming +can ice +cal is +bu res +bru cie +bro din +bren an +bota iku +be fearless +ban j +baking championship +avon more +asi mmons +as mith +appenz ell +apo plectic +ao y +anticoagul ant +anti thetical +anne frank +am ined +alu cas +\ " +! ðŁijij +ðŁĻ İ +ðŁİ¸ ðŁİ¸ +ðŁįİ ðŁįİ +ðŁĮ ¦ +âĸ ³ +young love +winn dixie +wf es +wend ler +vinayavidheyar ama +ug p +tru glio +ti elemans +thorough fares +the os +the magnificent +tal ar +syna esthesia +sv cc +sul zer +su enos +stop ed +sof ya +shim amura +sebastian bach +sab riel +sa chem +s itta +rs gb +rosleh tinen +ris don +rew ery +re activating +quint us +poppy pride +plumb ing +pend leton +park slope +or tner +op ress +of nature +nm true +nirav modi +nil da +nhv natural +new mr +naz z +month sof +mis interpreting +mil sim +mer cola +medve sc +mat tern +man mademo +manmademo on +mac l +m sw +m kh +ly l +long muir +lion babe +label mates +la urus +l nh +l ne +kin gham +ke saath +kan ojo +kal ay +john john +joanne wheatley +jay inslee +iv ry +invali dation +ingel heim +inge vent +imperial nhs +ij ff +ig worldclub +ia ia +huac achina +histop athology +heckmond wike +harness racing +hack worth +ha skin +government za +goli ath +gilmore girls +gh ome +gee ta +gav ino +gang an +fredd ys +fad hil +est ren +eno chian +education ally +eb ke +e ather +dex ters +dev as +de wolfe +comp ter +cdn paralympics +callo whill +cal thorpe +boo ps +blaupun kt +be sola +bathspa uni +bair ro +ay yub +auto detailing +aureli en +atre us +at eng +ar dini +angel ita +an zo +aham m +acry l +a aba +ðŁļ ª +ðŁĩ¬ âļ½âļ½âļ½âļ½ +ðŁĩ¬âļ½âļ½âļ½âļ½ ðŁĩ±: +رÙĬ ا +zu i +zor ah +you ri +yazidi genocide +wis c +win ed +wc pss +vla da +visit southdevon +veli yi +ve irs +uu ya +tir tha +tir rell +tim horton +than asis +tar mac +sub versi +stopthe hate +so po +sl h +sch aper +satur ating +sand worm +rotten broadway +riden our +retren chment +ren ick +reed hastings +ra chana +poor am +phonec ases +phic hit +or deals +om git +oath breaker +new caledonia +ne mes +nbaallstar to +multi story +mi rab +manfred weber +mal ama +lon gy +llan fair +liver nois +lettu ce +leon is +leaf ly +lanc spolice +kor on +kon di +kn acks +kin man +keepit wild +kas er +kali sto +iz har +horse fly +helsinki airport +hand tools +hal an +gru be +gor oyals +gobi ge +game changer +fr yers +ford son +fle voland +exp ound +european parliament +essi e +el anor +do jima +diamon dd +dave berry +dab olu +cw janethevirgin +cv hs +csu football +cour tice +com hghair +ciut adella +christian keyes +c puc +bra gger +br ame +bel tz +bad ai +autom ob +auto club +auror ahu +ate st +as kl +as aro +ar sle +aphrodisi acs +anti pathy +an field +amy dumas +am atu +alou ette +aller dale +al inda +accen ting +ac tc +a wit +! ?! +ðŁįŃ ðŁį¬ +ãĥŀ ãĤ¯ãĥŃ +âĿ¤ ðŁĺįðŁĺĺ +zen der +young stars +yat ton +yar on +x tian +world balletday +wildcat nation +weir ded +w kar +vu v +ving e +un fixed +um el +tugger ah +tuck well +tu mk +traffic jam +tho ta +the ph +testic ular +tel t +teach meet +super family +summer party +staf fa +sp ouse +smashing pumpkins +sil bert +shad rach +semi otic +sa whorse +sa pps +rust ler +rudi mental +rosen berger +roots music +retr or +regi sta +re seeding +re buff +r dium +quir is +pron ti +politi ken +pol hill +phil bert +pa sion +orson welles +organ elle +om un +old photo +ojib wa +oberst dorf +nws norman +nikon nofilter +nike soccer +new hair +neonicotin oid +natu ur +nates ilver +mul downey +mousekete er +mock asin +mmm gorgeous +mid wales +mid sole +liver poole +lion sxii +len ham +lady hawke +la vers +l hw +kuchi pudi +ko les +kir stend +kho dor +kam asi +kaappa an +k rell +ju ca +jhalakon colors +itali aricci +international museumday +inte zaar +inexor ably +industri alised +igo ta +ich art +icef all +ic ol +hyper real +humidi fication +ho jicha +hin aholics +hel ia +he pi +hat ari +gun stock +grin spoon +gov za +gl ancy +for far +floren tin +fish uk +fish bourne +exorc ised +er ris +epping forest +eco c +dyne ema +double fine +dine tte +dine out +die gol +di blasio +design build +dermat ological +del age +dece dent +de el +data quality +coy f +combu sti +co sas +cl ts +cl sa +ci fuentes +chillax in +ceelo green +car ve +car milla +capital and +bun te +brook man +brompton bicycle +boul ting +bir twistle +bid co +bech stein +bal doyle +bachill erato +bab c +bab ago +an sbach +allo dds +ahut chinson +ac ey +abh brows +ðŁ¤¦ ðŁı½âĢįâĻĤï¸ı +âŃIJï¸ı ⾨ +âĿ¤ï¸ı ðŁĺįðŁĺĺ +zom er +yoak um +wsw v +worl dday +wil ier +whig ham +wag len +waglen ikhil +vn dev +visit dorset +uofl baseball +tull amore +track master +tome asure +toge t +tip sand +tin u +timp anist +thisi su +the halls +the bar +sun glas +study english +sti x +standar ds +sta il +ssel f +sper rys +space week +sp nn +sig ne +si mca +si ani +sha dia +se ann +sare en +sap ir +sale straining +sal in +saha ja +row den +ron ning +rock aways +riky rickworld +regin apolice +refu se +rau fareg +raufareg besola +r st +qui enes +q ura +preemp ted +pil chard +paren ti +pad o +p tak +ot int +ol phin +ohl sson +oce t +noy noy +nipi gon +newton abbot +mu kt +ment one +manzoor pashteen +madel a +m sco +lo ing +limb less +lethaw aii +lethawaii happen +lab as +knur led +kn agar +kel ing +kal u +k iti +k illian +just sayno +jel gava +jeansfor genes +itsm illertime +itsd anny +im v +hi a +grin dle +gir r +g winn +flo p +fil iz +family holiday +fam elab +eye works +eri ley +en livening +ed ress +echo arena +e ens +drag queens +dock less +di meo +del grosso +deci mus +dau lat +dam ron +cudd ler +crê pe +crowne plaza +club houses +cer ci +carolin apanthers +ca vies +bsc sd +bru jer +bru ins +boss ert +bet ches +bel reddevils +be est +bar chetta +ban ov +at nagar +arthi story +arre stees +ar fon +app leeduchat +ag etv +af m +ach rome +ab de +a quest +ðŁĺĦ ðŁĺĬ +ðŁĵ ¿ +ãĤŃ ãĥ¥ +áµ Ī +འ¦ +zz st +zi ppor +xfinity racing +vishak ha +video gam +vid ant +ul kom +u coms +tu am +trans figured +total dhamaal +tom ies +to stan +thrill jockey +thistle tweet +thin kaboutit +tham ilton +tank less +sz w +syr crunch +swed bank +stem pel +skoll wf +sis lands +sigh thill +side saddle +sidd hi +sick ens +si y +shec an +sh pong +seventeen mag +sen te +san iga +sabar imal +rosam ond +rescueme ohio +red star +re cher +ration alizing +radio carbon +portland me +pha e +ot directo +ore o +oo bi +on des +olo ber +olafu reliasson +no wata +no confidence +ner va +nau ert +natali a +mysti k +myo dd +mun desley +moz art +mor p +misanthro pic +mining indaba +micro economics +meat wad +me ggan +mari et +man singh +main tainable +maersk line +luc chesi +lou vered +lo ken +lightsfor liberty +korch noi +ko tal +keepit up +ke ating +justin colemoore +jere mi +j pd +j mpd +ing é +info blox +indiscre tions +ig ent +hol croft +hob house +ho tor +histor ica +ham macher +halla han +h inging +gun as +guj ju +guillemb al +guillembal ague +gi aro +gg day +foxwood sct +ew ski +ew adi +enter al +energy fc +en yo +emprende dores +eil idh +early momentsmatter +dun murry +din di +di mos +derby dell +deplo res +deon dre +crossmag len +comic fest +color run +co cas +ci dent +chun ni +christiany elich +categor isation +carrick macross +bur dine +blu stein +blaster jaxx +big ley +bernar deschi +ben enden +beau bourg +be ssel +bar camp +bar ay +ban ded +ba shir +asse tte +asam blea +ary ab +arenaof valor +architec t +ar matrading +alphabet success +ah ma +aeru ginosa +... +) ": +ó¾ Į +ðŁĺĬðŁĺĬ ðŁĺĬ +ðŁĺĤ âĺºï¸ı +ðŁĸ¤ ðŁĴĽ +ðŁij¨ðŁı»âĢį ðŁį³ +ðŁĮ¸ ðŁĮ¼ +ó k +whereare they +wed eliver +war ily +w ame +visit italy +village voice +vasilev skiy +vanguard ngr +van aqua +ur on +univers ite +uni westminster +twee tt +tt ly +trung pa +tr ino +to power +ti ae +te ana +taver ham +tam ora +tal ese +ta ps +sweet waterbrew +stjohn ssmith +steph ane +stay wild +stac ys +sou kup +snap artcrew +skerry vore +shud dering +shap ira +ser vir +secre twars +se shadri +scott j +rsh yr +row blanchard +pro sport +pho tometry +per center +pas sos +pancas ila +pag ibig +pad mal +ore illes +odon to +nottooyoung torun +nom in +nikol aj +ni eri +nau shad +mun ghana +mu tsu +mq vist +moto cycle +min kus +min do +marri ot +made ine +lon gla +lip sey +light sc +li skova +letsgo tothe +leck hampton +lane way +lacer ated +ky ros +kn oop +ke lem +k hem +k ando +juri sts +jo i +jagu war +ix elles +it sky +it mo +islam isation +ir ud +ing ale +ine very +in black +imman ent +hu se +hero e +hender shot +he tti +hart shorn +ham dy +hadd adi +ha glund +gustav son +gray wolf +global foundries +gil merton +gener alists +flat water +field stud +felic itas +fasti dious +far ney +f music +exit poll +ew ert +espou sed +elf yn +eki den +ed henry +ec ri +eas me +dynam obim +dwar ven +don nal +do therightthing +died onthisday +dfa records +del boy +dare you +da aay +cyber risk +cow drey +cost as +comm eng +cocac ola +co git +classi fier +cla dy +christopher lee +cheri moya +cap en +c py +but first +br bird +bodh ran +bin yamin +bestteam ever +benck iser +be thri +bang ar +bad ler +ayeshat akia +as com +ap ag +ama waterways +ali via +al za +ag grey +abo dy +abdomin oplasty +# âĿ¤ +! âľĮï¸ı +ðŁĽ © +ðŁĺĤ ðŁĴŀ +ðŁijĢ ⾨ +âĿ ĭ +à© Ń +z ns +wer king +wedding anniversary +wealth tech +water way +ve wy +tv f +tru cke +tropo sphere +trees for +train in +todd barry +tit os +tic ky +thisi sirish +thi o +the hype +ten niel +team alex +table mountain +swi mathon +sun studio +sun dials +sound wave +sj v +sir in +sgo p +sco v +saras wati +salman khan +saar aa +ryand un +run g +ri xton +ren ren +rat es +ran ken +r se +q wq +q sr +presta shop +pre ety +pow les +podi atric +pic her +phosphor ous +phon es +pet unias +pear ling +patoran kingfire +pac c +octa ves +oath keepers +novem bro +north wind +norah jones +ne zha +ne elix +nat ron +muse umb +miamid ade +mediamar kt +mat tum +mar acan +mait ri +mah end +love itor +lisalam panelli +les band +lass ico +ku ehl +kk t +khand a +kent wildlife +ji brin +jeopar dized +is ay +instap assport +independent artist +im bb +il ma +ian jamespoulter +i wk +hy rum +ho quiam +he ff +gor am +gon orth +gb p +game ready +gal erie +future world +fortune mpw +flu ent +fire brigade +fac king +euro athletics +easter holidays +disfigu rement +dhe ena +der ci +ded chat +dad asaheb +d kp +coulro phobia +cot tes +coach p +coach k +cl ouser +cine mas +cher ui +charity week +cha hine +bi gear +beverly wilshire +bethany hamilton +bent en +bath on +ballachu lish +bal tica +asp net +as kem +aperfect circle +amreading romance +ah is +adap tions +a beautiful +% !!! +ðŁĺĬ ðŁijįðŁı» +ðŁĮł ðŁĮł +ë°ķìĭł íĺľ +å ¡ +චº +zu cco +zan jeer +x ango +wowo wow +wh ill +wal de +villa gelife +ustad h +un screwed +try hard +tro ms +torrance coombs +thi ra +the whole +t sy +sur facep +supere xcited +studio tour +stop it +sil vic +si bi +shakespeare lives +self development +sare not +ryan bingham +rossi ya +road sides +right s +reboun ders +re posts +queen spark +pur se +pro gess +pres rajapaksa +postpon ements +pinchpunch post +per ie +pel in +pe ther +parti als +parach ute +oro adshow +organic beauty +op hora +ontari on +nuclear power +nj d +ni aaa +next bigthing +net news +nb football +nan se +na ach +my car +mo ppet +mo gador +mira belli +mil ah +michel barnier +men teri +men otti +marun ouchi +margar ito +m ram +luke combs +luci en +lu tton +lock ley +live export +li sandro +li bert +lgb trights +le toy +laver stoke +last call +lake ofthe +lac alle +kuno ichi +klin gh +king lake +kh s +kar pinski +jo sy +jamest cobbler +irish tv +ieb ckenya +i ast +husey in +hop god +hom ophone +ha ake +gy ra +gy aan +grit tier +great deal +got tes +gor st +glad ney +fv k +fu do +fit ty +fish market +expo sure +eval ue +esken azi +em eline +el berg +e mia +dri essen +don official +don kis +disin cen +di gha +di gest +dac re +cup sleeve +costar ic +church planting +chriss ununu +chelse al +cann ings +can opus +caloo sa +bü nd +brian j +br oun +best sho +bare tta +bar tos +ba ee +b sr +ap rn +am prog +al atina +ak shi +aber dyfi +ðŁij©âĢį âļķï¸ı +âĻ¨ ï¸ı +âĺºï¸ı ðŁĴĹ +ا٠ģ +ÑĪ моР+ÑĪмоР± +ÑĦле ÑĪмоб +ç ay +yor dan +y aqui +with love +westfield london +welove music +water bottle +wanted wednesday +vi zier +vi b +v rr +uttarak hand +un realistically +tur p +town sley +tor ne +top oulos +tolu ene +that searth +tam pin +table topr +sv end +stoparming israel +ste pin +stars bbl +so cap +sin city +sean pertwee +se young +sat cher +ry nn +rudy mancuso +ru pe +ray man +queen radio +publicis groupe +pop concerts +poo tie +pix ography +penny royal +pen knife +peer support +par sh +or ts +om eric +nw schicago +nic ulescu +mycen aean +mor do +ment z +meet meat +marshall ampsuk +margin alize +mana fest +man nes +loc avore +liver si +liver sedge +lan zhou +la bette +kor f +ko smo +ko hi +kiraz mevsimi +ki eu +khaled azia +key and +kevino lear +kal orama +jume irah +jon jon +jennal dewan +iba secretariat +hois ington +hermi esadler +h bos +h bc +gun ite +ground less +greta vanfleet +ger th +ge mas +galla gh +futureof mining +fur u +funke akindele +fra ss +farmer sday +f ptp +ezekiel mutua +ex ped +en ner +em tech +dy le +dy c +du gongs +dru gre +dream land +doub table +con sett +co ches +cn at +chri smar +chir la +char co +c sis +buk hara +bhaagam ath +ber nice +bed stead +bana gher +baj rang +bah rami +aust mus +atri be +astre a +ard in +ann unci +an v +ami z +alter nation +alis sa +alex y +af dn +abo yne +- :) +ðŁĺ»ðŁĺ» ðŁĺ»ðŁĺ» +ðŁĺ³ðŁĺ³ ðŁĺ³ðŁĺ³ +ðŁij¼ ðŁij¼ +ðŁĮºðŁĮº ðŁĮº +ðŁĩ ½ +âĿ¤ï¸ı ðŁij¯ +zam alek +wren ched +wra ys +war precords +valken swaard +up stream +um ri +tuk ur +to fficiel +thu rai +thor selfies +th march +terry gilliam +taylor gang +survivor au +stree twalker +stacy london +spur n +sprad lin +slv ban +sky raider +skipthe dishes +sig sauer +shaw kan +sec schoolinnigeria +sar am +sap nextgen +sad er +s folly +rune quest +rumple stiltskin +rick hoffman +redd ine +reclin ers +re gaz +re collecting +raw vegan +ra ww +ra se +poster art +po partist +pluto tv +pitti uomo +pi eve +ph ou +peng u +pand ajay +pal ert +oz ma +om ino +ok olona +official b +nicolo di +nh lan +neu ss +nba xmas +nationalpark week +msc cruise +montre aler +model town +mens conference +luci ani +lizar do +le ston +l vp +ku stom +kha w +kele mentary +ke du +kanhai yakumar +kan ellis +jiu jitsu +jimmy sheirgill +jay mi +itsme deaner +isthmian league +inge borg +ing rat +ine en +ima gens +ily as +hiwas see +hi mi +haus of +guit are +gues thouses +goodnigh tt +good loe +go gos +glycae mia +gi shu +gardner md +gan ji +fur rer +freec ell +fre twell +fe spa +f lection +exoluxion in +exasper ating +ew ington +eni um +dy al +diver ticul +distric theating +desen ho +demon te +daily life +cron aca +cousine au +con nan +cl nolan +car ral +c jk +brew pubs +braintumour org +bra cha +bom bast +ber ling +bella houston +atb financial +art price +are well +ag ib +ae gis +a you +ðŁĺĬ âľĮ +ðŁıĥ ðŁı½ +á´ ¼ +à® ī +Ø· ÙĦ +zzzz zzzz +xen omorphs +wu zhen +women ofthe +walk üre +vitamin b +v no +under nutrition +un popularity +tri fling +tre bbi +the film +tch as +tab er +suther lin +su co +stock still +sp ams +south pole +sor ti +slau ghters +ske ma +side chain +se shat +scam orza +sam pat +sab har +rival s +religi onof +rebeli on +reaper mini +reading agency +r cd +projekt red +pher i +perip ate +peculi arly +over stating +nyle dimarco +naval history +mur shid +monochrome photography +mik maq +mew seum +meetthe maker +may nil +mar indu +m clay +kuy kendall +ku ber +ku bam +kro gh +klu te +kit trell +kin tail +kan at +jungfrau joch +jd ff +jacquelinem jos +i ster +i pi +hun g +holly r +holiday giftguide +ho stin +hisp ana +hi awassee +harak iri +gue sser +gett n +gam brell +gal va +fol an +faze ley +faneuil hall +family reunion +ex pe +endear ingly +ec su +dy skine +dramatur g +dine ren +dent ition +del py +del ing +defac ement +dd k +dan sk +d hall +d alli +customer care +crudit és +cre x +cod ger +cas sells +bush kill +bre ady +bon jour +blogpaw schat +ble eder +bistro t +bel staff +beg ins +baltus rol +back k +authori zations +andre rieu +alici af +ain ley +admon ished +absolut ly +: ^ +ðŁıĨðŁıĨðŁıĨðŁıĨ ðŁıĨðŁıĨ +ðŁİµ ðŁİµðŁİµ +îĢ ij +æķ ij +ä» £ +your love +yorkshire terrier +yon as +x g +wre athed +womensmar chon +wart burg +volks fest +uofl wbb +un ama +tin ari +tessell ate +tanisha amukerji +ta zo +sugarray leonard +stur geon +sto u +stigmati zing +stay safe +squ amish +sle a +signal man +shi bani +shav asana +serv ando +sakshim alik +rp motorsports +roman die +robert mondavi +robby gordon +rhyth mand +rest day +repro health +realmichael kay +re ineke +rads one +ra ymon +r br +quiz night +qu illiam +pun ks +pul lovers +publicenemy ftp +prophetsof rage +project lit +ple yel +pil ates +pi kin +philip glass +perkin elmer +per rott +paphi ope +pa ay +ov na +orlando shooting +orlando bloom +oral care +oo j +on n +ol fe +ol dis +occup yoakland +obje tivo +nag l +my g +mul lett +mu hl +movie stars +montan er +mir pur +mi bf +meteo gib +mayored lee +masc aren +maim ing +madr yn +longu eville +llan wern +liv miami +lein en +lanes borough +k vp +judy blume +juanman santos +jo wa +j tweets +j ce +isi zulu +innov azione +indi afirst +im lay +ich on +hor an +here andnow +guar aldi +gu len +green flash +gra gg +goldie hawn +god scountry +go jay +gal ley +fromthe groundup +forestry commeng +flor ine +fan wars +exmoor np +ek ar +ecw press +dro ite +dit mas +diss enter +disney nature +conceptu alised +con volution +coachella valley +co ber +civil service +ci j +chefou cauld +center fielder +catoc tin +ca official +brendan cole +bren ta +boat swain +blan ching +biz talk +ber ic +beauty world +bbc facup +auden cia +as ahutchinson +arav alli +amor pho +alcohol ic +ak int +academic swith +< =- +éĺ² å¼¾ +éĺ²å¼¾ å°ijå¹´ +âĦĥ , +à° ¬ +zab aglione +wind row +welcome week +wb kb +voc i +vision aire +uni da +ua em +typhoon display +trow dy +traveler schamp +thesavoy london +thenation aluae +sy scoin +su v +su tah +stanlee comikaze +sky science +sjo erd +silk men +sco c +sam su +sam eth +ru dan +roo sts +rela ying +red volution +re paints +quad cities +pu pillage +pro challenge +power books +pear n +palm desert +oro ck +oran allo +olic ity +o gra +no chill +nicolodi daria +nico ya +multi show +mu ter +mo ws +mi jares +matthewk heafy +massac ring +mac art +lunch boxes +lu ber +little brown +la berge +l alande +ker jak +kaz ak +kas u +kad ai +k van +k brown +jö nk +jm gardnermd +jerry lentz +jas df +jad zia +insi dere +ing gris +infinity thegame +in bath +i yan +hu kum +hoe hn +hirez studios +hi jazi +hi ers +hermo sas +hel u +greek mythology +gravity rush +gor in +go wd +global runningday +ghan af +genie francis +gam brin +ful co +fu tch +fu king +frog let +fm qs +fest nyc +factsabout me +et xwx +en h +earth work +dy sm +dra a +displac ements +dish man +dest america +cush wake +clay aiken +ci ii +cen as +cast d +bomba dil +bog ast +blin dd +blau velt +birch mount +bing su +best coversong +beren saat +beat airpollution +az nar +au trey +attenti onal +arti equitter +anton du +amlw ch +allsaint sday +al ake +agri ppina +ðŁĺĮ ðŁĺĮ +ðŁķº ðŁı¾ +âĻ¡ # +âĪ Ĥ +Ð ¼ +wy vern +wol ken +wal esc +v ult +uper girl +union budget +un dar +ultimate warrior +tumk ur +tu ska +tro c +top story +to home +ti gno +the interview +temple stowe +tehel ka +tech ne +team kitty +tan za +syfy tv +summa recon +succul ent +stra ying +stjohnssmith sq +stasi areport +sr ms +sportsp ersons +spor te +sor an +son ate +sli mes +seu mas +ser ow +scal zo +saskat oon +sarah drew +sanjay leel +ring rose +rest ing +rehome hour +quarre ling +qc times +pontard dulais +pla sse +pharmaco therapy +pen ya +peep al +pat ou +ostro wski +opier adio +o thering +o gers +o cam +nigh tie +nicom ir +nasa history +minim alist +med len +me men +marseilla ise +magnu mpi +maf fia +lu tt +love her +ll vm +liter atur +lakshmi manchu +lady bay +kling berg +keral ites +kauff mann +ka he +jane mari +jag jaguwar +it amil +inter nist +id alp +iam d +hustle hard +hone gger +ho ws +healthcare forall +har leen +hahaha ah +gye ong +ground hogs +green keeping +gor uck +fu sible +frank caliendo +final ride +fertil isation +fer mata +fen burg +fau zi +fang amer +falli ble +ent group +ei u +dogs rock +disappro vingly +deton ating +cub ator +cor mega +college hockey +coal ash +cleveland browns +char g +cham ak +can las +caer au +cad ca +buck enham +bra ska +ber beris +bel ville +bbcwales news +bac one +ba im +b vov +b dj +aç ores +ato logical +ar qi +ambassad orship +ale storm +^ ^^ +ŀ ĺ +ðŁĺħ # +ðŁĵ» @ +ìł ģ +ë°ķë³´ê² Ģ +yetto come +year sin +xehan ort +wy f +who les +welling united +web chat +wal let +ves alius +ven oms +ve dre +vas ili +vall ance +usman ov +ur gence +u mble +ty co +transfor mable +toron tonow +tmobile careers +ti jn +thrombo cyto +thin h +thewomen stour +ther mae +the tank +techno phobe +teameric sson +summer break +subju gate +stone fc +spo ker +shard london +see torontonow +se tembro +se gh +scrap books +scoo ts +sci kit +sch or +sad ler +rhe ingau +rgv wx +rem hq +redletter media +red together +reci ous +real john +raz ine +rahe en +qpr v +pu reed +psla fc +pon tes +peri shes +pel aez +pal erts +oiland gas +ofthe seas +office max +modern baseball +meth ylene +matriarch s +masse ffect +mal com +makin wa +lough gall +lon gg +lister iosis +libr arie +lean ing +lazi o +lam u +lal oo +l by +ko zo +king ussie +kil rush +ken ward +ken ard +kazimi erz +joey lawrence +jasmin evillegas +jac key +it ree +iron age +incis or +inar team +hr ys +hopen othate +hog sett +he ili +hal let +gid do +ger vasi +ge sf +gall man +gall inu +for abetter +fi ven +f bre +exp ounding +es an +er vice +en sa +en ingly +emmy awards +elms ford +elg ouna +el ya +e dul +dk v +die ren +dee waan +ct in +con ecu +comor bidity +college signingday +cli j +cd projektred +carr fire +ca jam +business coach +bu be +bruck heimer +bob sledding +bhag wati +bakhti ari +bailli eu +bag gott +bab il +ato ken +ann ers +amit sadh +agar uda +ad w +ðŁĩ °: +çľ Į +ç« Ļ +ãĥĩ ãĤ¶ +âĺĥï¸ı âĿĦï¸ı +à« ģ +zz aro +yu rika +ye asts +women sashes +wo to +wmp dogs +wc tv +wall on +ur dan +upcoming releases +under lings +tyntes field +toronto symphony +to pen +the challengecup +sya oran +svan doorne +struc turalism +stam ens +sta ip +st b +spit ball +sl ama +sim simi +sim plic +sid he +sex trafficking +sa hai +rupauls dragcon +run ralphi +runralphi erun +rip ens +repar tee +rep co +red for +re percussion +rack ley +pu ren +prin tvilla +pever il +pen edes +pas k +pal au +orlando sentinel +one ocean +off cl +nom i +nin omiya +nh p +nant asket +na pel +mun ga +mubar akan +mu is +moving imagen +movingimagen yc +mor goth +mopar chat +mindbody spirit +mes illa +maur oranallo +marvel led +managed services +man sor +male ev +lu le +lok manya +lin ny +laver ock +ki sta +kan ta +jw marriott +jo ab +ji had +it ok +isi s +ish ti +iam lp +hot lines +holling shead +holden graber +high gate +hel wig +heil ong +gu al +go tr +gel denhuys +ge hrke +g gu +fun gi +fu raffinity +fron ing +f ended +europeana eu +esken azi +ep cs +en ciso +emul lin +e zo +dor oro +di visi +decon gest +daph nia +cu yam +cu sk +cook out +con dom +cambo dge +bri ard +board shorts +az riel +au teurs +attach ment +ar vel +am pradio +am fam +al cor +aal smeer +ðŁķ¸ ï¸ı +ðŁĴħ ðŁĴħ +ðŁ¤ŀ ðŁı½ +ãĢĤ ãĢĤ +à¸Ļ à¸Ļ +Ë Ļ +é d +zag li +y ns +y appy +wr g +wf council +war ming +ver den +veganrecipe hour +trevi thick +tough ening +tou rettes +toni storm +to ome +the boat +tay som +swag at +sur inder +sub divided +still water +steve tignor +st vincent +snow flake +smart water +show mance +sho lo +scratch ings +sal divar +riev aulx +rei des +ray uki +rache le +propo fol +proc tologist +pown all +pha ilin +pew ds +pens wordbooks +pe cora +pan telleria +orange wood +ora for +onlin enews +odon ata +o ires +now open +novi grad +nose worthy +nois y +nicomir allegro +netapp insight +nat c +ms ar +mro dofficial +mo cambo +megal opolis +mc mansion +marci a +mang ling +man marzi +making of +made tomeasure +leop ardi +knott ingley +kishore kumar +kinky bootsuk +king sley +kin tore +khu ra +kevin max +kaz eem +k haw +ja aye +infectious diseases +in with +ilovelu cy +horror family +ho stal +hh mi +h bbtv +gor acing +golden boy +garret son +gamer life +gal asso +fushi gi +fu tah +fre dol +form als +food safety +fi rish +fair phone +es am +em trains +el ver +ed app +di anat +dhe ere +de sul +da pest +corrobor ate +corri gan +color me +civ milair +cine timee +ch ich +ces spit +c andre +bro dies +bre g +bha in +barry hearn +babyled weaning +ba el +aw ai +athan s +arnau d +ar amos +aq il +appor tion +anc illa +allu sion +agu do +after burners +abc newsbrisbane +aar c +ðŁķ ĸ +å¼ µ +yne kev +yadav tejashwi +y abby +whim sically +wh ou +we sel +wayne sermon +war speaks +wander n +uru an +under dark +u uponline +u od +tze dek +ts z +trin itarian +tri fon +too ke +todays special +timp f +ti so +thing swe +theo cratic +the unit +sun news +stra thy +sto x +spi er +sho liday +ser very +sc dc +sayye sto +save ur +sa ko +rr v +rouge mont +regul arization +reas sert +re dron +ram nagar +py mat +pru den +pil fering +pan zero +pad mash +opent ennis +nys fair +niy kee +nightmare beforechristmas +nicoleand bri +n ber +mrscot teddy +mosthaun ted +mm al +michi okaku +men ti +mama hal +maha sabha +lyt chett +lov ley +lo ade +lie zel +kin zer +jah nke +jaboo wins +j emmy +ital ymfa +it shappening +is aw +instru mented +ilean adcruz +hur stresort +highlin enyc +hang nail +gor din +gon oodle +go dragons +georgi ev +garden birdwatch +gar diners +game board +g elli +forre stal +five point +fit es +ff be +fat ta +fa zak +endemolshine ind +embarrass ments +eli ud +ea se +dw ade +dump site +dou ga +donal dd +dj uri +debru ynekev +cure ton +cuff ley +crack heads +cra is +cov ell +cor c +copp ock +can de +c agon +burn outs +brü hl +brun twood +blon d +beer lover +bber ing +bb coxford +bassi sts +barone zaza +bar chester +bag al +bab b +azte cempire +ausout backnt +auction at +ate f +as rar +arkhu hro +arab ella +aldub you +:- \ +/ .\ +ðŁĺĦ ðŁĺĺ +ðŁĩºðŁĩ¸ ðŁĻı +éĺ²å¼¾å°ijå¹´ åĽ£ +à¸ķ ร + ¥ +zoo cbs +z all +yuvas ena +waz ed +ve idt +val ery +vad undee +us asa +tz ed +tt ourism +timel ine +tiger n +th inf +tetra drachm +syno vial +summer camp +stro heim +squir rela +sin ked +shar ada +samu ele +saan ich +rio ted +ri pro +queens berry +psycho somatic +pre treatment +pr il +pla isance +pickup shadowhunters +over stretched +ov ni +ostr ac +ok orafor +oc z +nikit in +neutro phils +nau seam +monic as +mis on +mini stre +mercat us +md x +mal usi +mad hura +ma shan +lu iss +live sound +li iiii +lau v +kil ledit +kap ler +ilove wpb +ibu solih +i wb +hull city +hayward sheath +ha bano +guthri e +ground nuts +green all +god father +gobier no +giar dia +gand hara +gag liano +fr ings +fox field +ff u +feel in +fe compendium +fau teuil +every little +ethno botany +et zion +esp alier +end sars +dum dum +drif twood +disin her +disch ord +digital globe +digi bytecoin +dethr oning +de mur +day job +das sler +dark wood +dan forth +commercial ise +com us +colon nades +chiff re +chem ung +cbc falc +cap ut +cap ucine +can tus +caix inha +bikini body +bha bie +batt y +av aya +ar bat +apoloo hno +anesthe tist +am bati +alo ves +aker shus +ait ken +afan art +abay omi +... ðŁĴķ +! ðŁĮ¹ +ðŁĩ¦ ðŁĩª +éĸ ¢ +ا٠ĥ +z radio +yz ors +wra yzors +wr hs +wp sproud +what more +wer x +we p +vish wan +uw madison +un suspected +tri fari +todd dammit +tegr ity +tee shirt +sx s +swear ingen +sugar foot +str ouse +sof steel +shy glizzy +scu stom +sb se +sant acon +saint field +sahy adri +saaraa alto +row se +rod taylor +ren tin +re formatted +publ ick +pro enzas +proenzas chouler +pon gal +polaro id +pic tou +pex els +peop l +pay per +parthasar athy +outre mont +or cl +optic h +ol hos +oc ket +nes j +n cats +my favourite +musk aan +montal to +mon forte +mon ami +mom bas +mi shi +me on +married life +man madhudu +lunch box +lovefor sam +long port +licht steiner +lawandorder svu +later als +kunst museum +ku czynski +korn gold +k run +k ramp +jeopardi se +j ati +ip es +iop sych +io annou +integral yoga +ini ka +igen berg +ifc center +i ppy +i hrc +hynd land +har bert +great nature +gre engage +giug no +girls that +ghost sign +getex cited +ge henna +gast ineau +garden city +frankieco cozza +fr itch +fat was +far me +ey res +etri gan +eti da +ent on +en ak +eg w +e spirito +e rebor +e migr +dis orientated +derby museums +davide igenberg +countryfile live +corin thos +constric ting +co stest +classi st +cla rey +cherui yot +ch he +castate parks +cassandra sleee +c gt +bun gend +bott ineau +bor gne +blood good +bleed in +berg gren +baku go +av ely +at ang +ari fin +aquan aut +amph it +aldub for +alco ves +agha doe +agent ur +abric ation +abri stol +ableton live +ab yan +èģ´ãģĦ ãģ¦ãģĦ +âĢ¦ ... +Ùĩ ا +³´ ìĿ´ì¦Ī +zin chenko +you willbe +wv ua +white wood +vu yo +vitabio tics +v ont +v aren +u mina +tro eg +travel quote +tr yn +ti zzy +thrombo tic +thol yoke +the fight +tar vin +taj iri +syke sville +straight outta +stock brokers +stobart group +ster ility +sta h +sky sportnz +singlec ell +sidel ining +shou sing +shi jiaz +seon gnam +seanpatrick flanery +se sam +scream ers +sch moe +scar nival +saharare porters +rr u +ro sab +right stuf +rake shroshan +rake em +r yoga +pul let +publi us +pro finet +por at +pol vo +pic cini +phi delt +per icos +pau low +owen sound +north end +niykee heaton +newsar ama +new release +neutr alised +n co +n clc +move theneedle +mort lock +model railway +mo cho +mis sour +migra ine +mewseum monday +mati gnon +mat ula +mat tawa +man eesh +lu vu +kyo jin +kurtv ile +kne ec +kir kenes +ker rie +k pcb +jones music +jo ema +jer oen +jen lisa +j vs +j anni +ix ia +intelli j +inte xt +int acct +ing my +indic ud +ignit er +hor dern +hol ms +hin ching +harvey weinstein +h ff +groo k +green burg +great times +grad life +gopher hockey +gi galert +ge sucht +gael scoil +gab p +g me +g afc +fy inglife +fortn it +forfe iting +fol i +fo day +film life +fiel den +ff ert +empath y +ek or +ecre ek +e mond +dra vet +dje mba +dis qualifying +diam anti +cush wake +commo dore +com unity +chrissi efit +che ff +centrifu ges +cal vert +brief er +bridge fc +bran de +ber minat +benef ic +be ziers +bam berger +ba jan +azi one +ax elle +as cl +are gbe +arch stl +arapa ima +ar round +anyou neversee +ann ago +ank ole +am bula +allin with +ali on +aap ko +! ðŁİīðŁİī +ðŁĴķ ðŁij¯ +ðŁıĨ ðŁıĢ +âģ Ħ +Î ¾ +zoom car +ysle ta +yaz awa +yar wood +woo do +wkr c +white horse +whatilearned today +whati slife +wh dh +wat ain +vol quez +viol encia +un moving +un luckily +tra versed +tommy wiseau +tom asso +todddammit kerns +ti mate +the zoo +the vic +the amas +ten ews +tc w +tal bot +stan es +spast icity +sm soc +sla unch +si ang +shi pper +sheik hu +shar pless +sf m +schoon maker +sales manship +ry thm +rotar act +romu aldez +retail design +rescin ding +rcmp mb +ran sacking +q ic +psin sp +program matically +phone mic +pequ annock +pe a +pc game +paras auro +ous ley +one iric +of x +objec tivism +nz inga +nwa chukwu +neck pain +n aper +myodd balls +much hhh +mr h +moom oos +mob ilit +miro slava +millin ocket +middle grade +mel co +mcdon ogh +maroon dah +marit ima +long ine +liver adio +les bi +le me +le frak +lady boy +kat zman +jo da +jen is +j ss +itsuku shima +is ap +ili z +igh ty +identity theft +hiphopp antsula +hel ichry +healthcare it +han au +ham park +gu jar +gp cr +go gulls +gang war +gal low +fu rie +fbal lus +father son +ec assidy +e zzard +dur row +du vets +doub leg +dor na +ding a +dev aki +del homme +daga ati +corn ella +cinephile photo +chamber of +cam mack +bungend ore +bun o +bott band +blood money +bit d +bend ita +bar ah +av ad +aust ins +arvin dg +ar od +anti doping +ant ar +ali ster +al vie +ai ps +aerop onics +adidas fballus +\ ( +. âľĮ +ðŁİ¼ ðŁİ¤ +é Ĵ +yo gad +yel verton +wol pert +wld life +wi ggers +wat amu +waffle crew +vere em +thunder nation +ten sioning +te cla +te cha +tang ential +tan ke +tall ships +step wise +sor ong +sn d +smy lie +silicon hbo +sil vey +shu mmels +shan ter +seton hall +se ble +scar abs +scal ps +saumy atandon +sang ay +roysoc med +revolu tapp +relax er +relax ationday +rege x +readju sting +ra kel +r jc +qui res +publi shable +pleni potenti +piti fully +par takes +oy ler +over hyped +ou ise +osa ki +olober syko +ni bel +newed ition +mv ci +mu cker +mt ps +monte agle +mobi lebanking +mello phone +megab yte +manga studio +lover ugby +london npc +lit fest +lind blad +leff erts +le dgers +lc cc +lauren lapkus +lamo ille +lam bourne +kry ten +khodor kovsky +kal enjin +jo suke +jefferson town +jc zyk +ip man +interior decorating +instam oment +idhunamma aalu +i dontw +hun do +hether sett +hau ghnessy +ha ith +h iso +gwyn ne +gu ck +gra un +gra ub +gob bling +glenfidd ich +gi jón +gi bill +fri is +fl sen +fire tv +fe delin +fc ps +eu refre +eo incol +entomo logists +enni g +du th +du melo +drop zone +dining nc +depu y +de stry +de eded +danc o +couple ts +concu ssion +col chic +cl onic +chil ena +chat tel +char mian +can be +cafe press +bt spor +bren ner +brae side +bonnie and +bear mccreary +bang on +ba stow +ba die +av ta +anti fouling +amrap ali +ak ota +accessori zed +ac rid +ab big += ' +ðŁļ Ľ +ðŁij° ðŁı¼ +ðŁij©âĢį ðŁİĵ +ðŁİī ðŁĴĥ +ðŁĮ ĵ +âĿ¤ï¸ıðŁĺĬ ðŁİĤ +Ê ĺ +ye won +yak ko +wr m +worl wide +wor x +wi elder +water proofs +vivac c +vi rens +unequ aled +tl picks +tiger sfamily +the mc +tex eira +tellem jaye +te are +tanner foust +ta are +t enta +story map +stockhol ms +standardi se +st rock +speci a +ski pper +siem ens +se ai +sdg action +rone ttes +richard j +ri ata +pel meni +peking duk +pe ffer +pas sin +pab on +ot n +oo on +one gative +ny autoshow +nj enga +niki fyinglife +new i +new car +nb alive +nanow ire +mt fc +morbid elli +marqu ina +marindu que +man gwana +lyric belfast +luzh niki +lu sive +lid combe +lib man +li ban +leve ret +latch for +lan go +l spraggan +kel si +joshab bottband +jim sterling +janemari elynch +international kissingday +id wp +i yaz +hungry house +ho ppa +heb buli +hd ms +happy pride +grand teton +gr rm +gold box +gang i +game strong +gam i +g fg +fu ente +fen oli +fal sa +eye brow +erri ble +er hardt +encant ado +em slie +edu coach +ed itio +echof ox +drew seeley +dol lis +di ene +der ay +daw it +dan an +cryogen ically +cre o +cra bbers +corrobor ated +cooki ed +citrus bowl +che b +chander paul +cham plain +car forsale +canyon fire +caloosa hatchee +bumb ashi +bl undering +billie faiers +be intheknow +bbc cov +ba bes +b ason +ay er +autom arketing +auto crats +atal ji +arri vab +antoni osab +amo c +amerikk ka +am ax +albat rosses +al ha +ail i +ah wah +aga h +affil ate +abri el +ab ase +ab ak +ðŁļĻ ðŁĴ¨ +ðŁĶ¥ ðŁİ¶ +ðŁİµ ðŁİ¶ +ðŁįª ðŁįª +ðŁ¤Ļ ðŁ¤Ļ +ì² ľ +èĪ ŀ +ਠ¸ +yl va +wo chen +western ghats +wal kleys +viveko beroi +urban iak +ultra europe +tun nell +trail way +the mbi +the evening +texas monthly +super fici +su di +squ ill +south ayrshire +soft wares +sny man +smer ch +smallstreamers connect +sk rein +silver hill +sh andi +sen sen +sea power +sat anas +sare gama +sa ren +row ville +rosen zweig +rich gang +reserv as +red bulle +re mon +q ds +prag matics +pr ound +piece hall +persuasi ons +performance art +os mania +on paper +o ae +nor il +nfl sunday +na jar +mr joe +mn timberwolves +mm cr +mel by +meghan mccain +mc moran +max g +maup assant +marriage bootcamp +margare triver +mac eda +m sal +lieben berg +leys in +le dg +la ster +kis san +kar and +johny hendricks +ji ocare +jean loup +je ant +jacob s +isa beau +intersec ted +hrishi kesh +hockey town +ho ca +hin richs +her nameis +hel enab +heart and +har ra +han z +hal di +ha sk +gun sout +godzilla kingofthemonsters +git eau +game digital +fof ana +exo genous +esc at +erzur um +digic el +deri paska +de soto +crew mates +cor ail +copper smith +consig liere +con cho +ch ingly +cau very +carra geen +candle mass +cal k +c chooks +bru mis +british summertime +bram cote +bb ar +b pr +avent ine +auctionat alive +ashley judd +ankit lal +ale cia +aimee mann +aham eed +agon zalez +abdash soul +ab ert +ðŁ§ļ âĢįâĻĢï¸ı +è½ ī +å®ĿçŁ³ ãģ®åĽ½ +âľį ðŁı¾ +âľ ª +è que +your majesty +wrps rockland +with dean +willi emullin +whole wheat +whenin rome +weather alert +wab ara +wa then +vijay deverakonda +up tuks +u or +ti sing +thoughtful thursday +the shardlondon +the digital +sur realists +sun ol +stormbre aker +stim me +ster ic +stein hauer +staip ans +sru d +sportsc asting +sports massage +sin ne +si guen +shi ppen +seet trading +save themall +sat er +sa igh +sa hr +s ft +ru as +ro mil +respon se +pu ca +propri o +pro vable +pri der +plan as +pham ous +perpetr ating +pel ita +pe ddled +parasy te +pan tha +out witted +out ils +ous seau +ot k +ol ake +ny ad +nor mann +no edit +nag araj +mire la +mi eux +mega house +me rend +mary rose +marc ille +manushic hh +mad man +m jin +lo set +lim pets +len hart +leg ance +lacey chabert +koin ange +kle ve +kesh asuxx +kc traffic +kab uk +k nac +jur mala +jaun diced +invali dates +ini photo +ilm wx +ido los +ic at +hum vees +happy yyy +grub street +go zips +go bbled +ger minal +gen re +gand his +followyour dreams +flore sta +fellow ship +eyewitness nyc +evangel os +eskenazi health +es ad +elle decor +do bel +del his +cri sfield +conge aled +comp diningnc +chicken wings +chae bae +cer atops +car michael +cairn staipans +cade tsuk +brax tons +bour seettrading +book direct +bon if +blin ka +bil our +bick more +bei sel +beau bien +beach walk +backto you +at midnight +ak hand +ad ari +aardvar ks +] @ +ðŁij® ðŁı» +âĿ¤ï¸ı ðŁĴľ +âĢ¢ *¨*âĢ¢ +Ø® ت +ÅĦ sk +yo shis +yat ai +yale britishart +woody inho +whel chel +wake ham +volvo trucks +vol land +vl tn +verti ginous +val met +v wf +united by +timb its +thy mus +thur day +the village +the face +tas kin +suc cour +sub mission +su mar +social ised +snu ffer +slav yansk +sj fc +show choir +sha ren +sas ki +s biz +real monarchs +re gol +ram mandir +ra bab +r vaidya +puer tom +poldark pbs +pninator nai +philosop hic +pay zer +parasauro lophus +paphiope dilum +otra sheffield +organiz ational +or tona +ole mia +od deven +obfusc ate +ny fwm +north london +no deal +nm fc +na ak +myfour cats +mul ga +monte go +model sown +mod cloth +mic an +met alist +mega structure +mcgoo han +marth ac +m acked +lu bumbashi +la ich +kup chak +ko bolds +ki pps +ki and +kev ich +kap uso +k wat +jet se +je j +j nd +j ml +itsal way +it um +ing rate +in expensively +hyn chus +holmen kollen +ho berman +ha inaut +grader ocks +gen ies +ge mat +francis cus +foxsports go +follow er +flat pack +fabi ano +ex clamations +epistol ary +eoincol fer +ema m +ek deewaan +ecu piratesfb +do stana +diverticul itis +discover la +disciplin arian +di benedetto +de weese +day togo +davey havok +comedy show +colo ssi +co win +clande boye +chang elings +castan o +canadapost corp +bu jang +bre slow +borge ousmusic +bin x +ber hampur +benson henderson +bas ka +artsc entre +armor ial +antigu abar +antic y +ant olin +anony me +almost famous +allo h +all thebest +aj ola +afternoontea week +^^ * +ðŁıĬ ðŁı» +ðŁı ĺï¸ı +ðŁĮļ ðŁĮļ +éº » +âľį ï¸ı: +âĢ¢ âłĢ +young ji +you se +y wood +wo begon +white marsh +whi ppy +where with +wf nz +wester lies +ween ies +we ard +wash outs +waron yemen +vi bert +var u +valley fire +v rt +uc its +tu me +travel quotes +travel agents +trade talk +themira gelv +super novae +stol t +ster ols +shereen bhan +scri pta +sanjay manjrekar +s ved +reve re +pretty boy +predic ate +port colborne +pin zon +pin el +pic tet +pas richa +pan talla +outag amie +on fc +nissan uk +newsp ic +new shoes +neve u +ner fs +nen go +nac i +mose by +mon hegan +mom ento +mo hr +misse dit +mete pec +meen u +mcin tyre +mat shummels +maje ski +mah y +mah lon +lycam obile +lumin al +lis ation +le vison +laurid sen +lar khill +lam ina +l brut +kou m +king ricochet +kin ged +kildare gaa +kell man +kc pe +kay ley +kal pat +jar vie +inst l +hob good +ho gle +he sh +hall inan +gyeongbok gung +gou k +gaz ipur +g ny +fulton coschools +front als +football league +films official +faul ting +fau s +extor tionist +erin cruz +engine ersweek +eless ness +dox ey +dis comforts +dio u +dazz lingly +cut throats +comedynightswith kapil +cle liam +chine sel +ce duna +cat olica +car ya +brexit deal +bo swell +blun kett +bill u +ber ges +ben sen +batchel der +barbic an +bar the +b bu +av athi +autum ne +au vsi +ator sk +ass ja +ar lan +amu sique +all one +ahon a +af shar +! ðŁĺĿ +ðŁĴŀ ðŁĴĸ +ðŁijĢ ðŁĺį +á´ ¥ +yol andi +yogare treat +ym ous +ym cofficial +xoxox ox +williemullin snh +wido wers +wex ham +westhigh land +war is +wa ay +uranium one +un gli +ud ham +u calgar +tweet likeagirl +tur lough +thunder cracker +thor is +ten sity +tang ere +tan pa +tal mu +suni el +sm sd +show addy +shop girl +shankar raja +sha fie +sextu plets +scroo ged +sc alling +sans tha +sag ara +rovere to +rossi o +rec all +prairie chasers +pepe aguilar +papill omavirus +pan ico +oo ey +odal isque +notan other +nol lywood +nikon owner +nightshift beer +nesj loch +mú sic +my rie +my great +my fan +mun caster +mon ier +mipim world +mat suz +man je +lo pilato +lew dness +kul m +kom iks +klingh offer +kepp ler +justin hartley +just love +jer zy +it oo +ilo pez +hermi da +harjit sajjan +h alling +gun ne +guer illa +go dan +girls not +gearo id +ge us +gar u +gale abrewer +fri endo +fish fry +et cs +esoter ica +elek tro +duck bill +dot me +distill ery +disfru ta +diamon dring +deu sto +defence men +de anda +dd firish +day tripper +d windle +d gaming +crui secontrol +cruelty free +cropre dy +cran nog +conval escence +col beck +cb buk +cat tery +budd hi +bot tes +bolt action +black dress +black beauty +beque ath +be son +bar bag +bante ay +bang ko +at tests +artific er +arn ley +aregbe sola +ap sc +an ot +alphon se +alab amade +acci ones +abat to +aat r +a ill +ðŁļĹ : +ðŁļ© ðŁļ© +ðŁĴª ðŁĻĮ +ðŁį©ðŁį© ðŁį© +éĢ ± +ãĤ¤ãĥ© ãĤ¹ãĥĪ +âľĶï¸ı . +â¬ĩ â¬ĩ +ঠ¾ +whit etv +wat ing +vin ilo +vagab onds +un welcoming +un wary +un masks +tux edo +tru g +tp wd +touri sme +touch my +too ker +toast day +to lo +the clown +teng ger +ten ino +tatt nall +t ö +sum mith +sug ata +stwee tings +stu pas +stewar ton +spray tan +sop ron +sion i +shi moda +shash lik +seh ri +sc ure +sab iha +rush cutters +rudimental uk +roy ds +rober tg +ridley scott +re ap +ranz kyle +r breich +popular ised +pol onium +po kor +perplex ity +part time +paris marathon +padman ab +osco da +oregon football +oo of +om kara +ny pa +north wood +new school +ner music +neh len +nationalvoter registrationday +nation alized +mur mer +mu u +mand ela +m sti +lw lies +le ggins +la haye +kon adu +kokol ondon +kir tley +kh oops +ke sho +jon ation +ja ane +j du +is mrm +irreversi bly +insubordin ation +insafi ans +ignati eff +hor ns +holt mann +henni ker +heilong jiang +head and +harperadam suni +h elling +gu ap +great service +good kind +gol spie +getting ready +ger be +ge deon +fuzz ed +free picks +four tet +fon ics +flann agan +fire r +fethul lah +feline friday +fan wood +f mea +em rah +dil shad +den sification +de bar +crit ch +che il +cene rentola +caram bola +cal lip +cabinfe ver +business strategy +bri stling +bre ann +biz markie +bio ethanol +big ler +bab ers +b movie +az gov +asphy xia +aqu aman +apple edu +ani ze +an az +am h +alamo bowl +ah iri +adar sh +? âĢĶ +... ðŁĺħ +-- ' +! ', +ðŁĺŃ ðŁĴĢ +ðŁĴĥ ðŁİī +ðŁij½ðŁij½ ðŁij½ +ìĹ ł +æ § +å² ¡ +ÙĥÙĦ ÙĨا +zu an +x ara +wolfal icemusic +wm tw +willo wh +wik icom +wi ggling +w inge +vill al +verdad era +veg f +ush l +tv s +the leader +the hive +tequil as +tag al +tack lers +sur feit +style blog +steins gate +star finder +ss ure +so ireland +sla pdash +sko dauk +sequ atchie +sebastian rulli +schem bechler +sal mag +revan che +repl ou +re evaluated +raf brizenorton +r fo +po lem +pat waris +p cn +only on +nulli fies +muscle tech +mul lein +mou li +mont illa +mo tho +mil ap +miguel cabrera +medi adays +mc cambridge +long life +li mbers +let aba +le yo +labu an +kemb awalker +ke len +kcr anews +kcc afc +kar lie +je evi +jason dufner +jagadish bliss +jag ged +it sv +ir one +ipriyank sharmaa +inter ac +i vig +hil sa +he ttinger +harpers bazaar +gun show +gu lli +growingup black +groupp lc +group set +gro ad +gri saille +greet ers +gor go +goodto great +go colts +gener o +gc morningdrive +gav yn +fitz water +fel ty +es ol +es ada +erry body +eeee ek +dingh ies +dhru vanatchathiram +dem party +cy syll +con rado +change severything +chal oner +chal am +cer titude +cdn sci +car quest +car lina +cabez as +bus stop +bob saniga +bo sko +bmo harris +betty white +b sor +az tlan +aruban etworks +arti stique +ar mando +apoor v +all gä +alice a +air cargo +ag ur +adalovel aceday +abot anist +abac us +a otd +a ayi +a ara +ðŁĴ©ðŁĴ© ðŁĴ© +ê¹Ģ íĥľíĺķ +âĢ¦ ( +zor ya +zoo z +x liv +wyan dot +ve tt +unic y +u waterloo +u ja +tren a +toy spotting +tower hamlet +tl ry +thorn berrys +thom ast +telekom wall +te shima +t pk +sur mount +strange music +stead ying +standwith israel +stam ets +spell caster +spedi zioni +sn ac +sin bad +silver oak +seb aceous +sau dagar +salam is +ride to +remo teness +ready togo +re asure +radioc itizen +qu ie +q magazine +pymat uning +pul sars +pu spa +profit eer +pro du +presu mes +pitt ard +peak auto +parathy roid +over consumption +ou tre +oli vers +ofthe sea +o thon +nov ae +not me +norder ney +ni me +ni hilo +network security +need ville +ne yed +narasim han +musix match +motor able +mo ger +mini mization +min ting +men ia +mcil rath +marinel itter +madi keri +lu stre +live world +lever hulme +le lang +le com +knight stown +ki evan +khim ki +kee pers +ka ag +judith light +jeremy piven +jas mith +j ji +ioc media +inver loch +im plan +il vo +ij ssel +ibrahi ma +her ff +helli well +ham mes +ha chem +greet z +greek wine +great york +friday feature +fo gg +flax man +fal chuk +fail ing +escu do +ery x +ehr hardt +dur man +dubu c +dis connected +dev arak +der yk +dar g +d green +cu ello +counter acting +cor win +construc tionist +ci brian +canadian open +breath nach +boling broke +blacke verything +black well +black catsrule +bergh olt +ber lay +begin shere +beaut ys +be eee +asi at +as selin +artgalleryof nsw +ani kan +angel haze +an gol +amal ick +adhi kravi +abar ca +a inc +ðŁĴ¯ âľĶï¸ı +ðŁIJİðŁIJİ ðŁIJİ +çĽ ¸ +æĭ ¡ +ÃŃ rez +´ âĪĢ +zam asu +ye aaaaah +wrestle mani +win driver +waffle day +vin italy +video gamer +ver wo +val ya +univer selle +tv w +tumble down +ts it +tra urig +tr ond +token ized +the doctor +tg sports +sy ch +sport pe +social ites +sham en +shakyam uni +septe m +seib old +salesforce devs +saha j +s vin +restaurant news +red hawk +rebutt als +re titled +rajiv message +prolifer ating +plumb ago +pe tu +pe leg +pd st +pale face +over running +onon dag +nn pa +netro ots +n illa +my bb +mon duk +moisturi zers +mohan lal +mo pen +mil kovich +man aka +maid ment +mah ina +lance storm +l me +kleptom aniac +kingsc ross +king stree +kindle book +jay buma +jaybuma om +jas inski +ivin ghoe +im r +ifbb pro +ic ent +huf nagel +hit sville +h th +gy am +good witch +go titans +go ffman +gear shift +fy b +fre ds +forex trader +fol s +fad l +eun an +ess chau +esp ouse +en sky +eagle man +e store +dre wett +draw tober +dj h +dizz bee +dece ase +death stalker +cra il +cour noyer +coun seled +color ant +cl wyd +chon ors +ceram ica +celi k +career tech +bryan clauson +boooo om +bol li +blooming daledc +bl ico +be tway +be muse +basili que +avi shai +astro photo +apal ace +anti ageing +anodi zing +anne hathaway +anjan avj +alvv ays +almu dena +ac ai +aak nopf +? ðŁĺŃ +; $ +ðŁĴ· ðŁĴ· +âŃ ķ +âĿ¤ï¸ı ðŁĺİ +âĿ § +ร าย +Û Ķ +Ë Ĭ +ya o +wo y +winter in +winston salem +wi king +warhorse studios +usur ping +ur ca +uni fy +un ita +un alienable +u ht +u asin +traitor trump +trac s +tr illian +tourde suisse +tomas elli +tinari wen +tiddly winks +three js +the pig +thank ss +th rap +techno cracy +te ake +super mare +summer bash +su dip +stri al +sto key +sk up +simad wasim +sha fik +see ster +se mer +s ical +ruhr gebiet +ron sexsmith +rochelle humes +ro tondo +red z +red pill +real news +re processed +raj hi +pomer antz +pho cu +pete y +per al +paulma sonnews +patho logy +palam pur +over rules +o had +nv h +northant shour +no cera +natali ep +mun ari +mu hd +mtu td +migu ero +me cum +mccour y +manushichh illar +mancity women +main landers +madelaine petsch +love hate +llll llll +lic ey +li mm +li dell +let tera +legoland florida +lag ell +la gaan +kvad rat +ku ff +ko san +kevinolear ytv +johnnie walker +jer g +jenna jameson +iran talks +im bert +illiam son +il ynx +ichthyo logy +horse sofinstagram +hor ti +holo han +gw mag +gujar att +grrm speaking +gre ave +gla xo +gh pl +gaz ania +gain fully +g ics +freedom caucus +foto sred +exu ded +es war +entre met +electric ity +el itch +ei le +ei k +ear le +e bs +dry bar +di metro +deta ching +dath lete +cze wski +common s +coach k +ce mil +can as +c summit +bur treynolds +bur row +bu ana +bharathan en +beer tography +bat tist +bas ca +auto chrome +audi os +arvi at +ap lit +an ley +alim entation +aliciavi kander +alegg ero +ak ids +... ðŁĻı +)))) )))) +$$ ! +ðŁĺĬ ðŁĴĽ +ðŁĺ±ðŁĺ± ðŁĺ± +ðŁĩ¿ðŁĩ ² +ðŁ¤¸ âĢįâĻĢï¸ı +âĿ¤ï¸ı ðŁĴĹ +à¸Ļภģ +ಠ£ +à¥Ī à¤Ĥ +ze itz +wyn ford +wr its +walla hi +votedem sout +vic parkacademy +v md +up voted +u tong +u ani +tr yn +todayin prog +there bel +tamiz h +take walks +t fm +swann anoa +stre cords +st ns +spec new +space marines +socialmedi am +sd x +scroll work +sai fi +s gbv +rin dge +railroad er +pu ddicombe +pop mech +plan thealth +piti ed +pav lik +out lasting +nff network +new ell +ne trunner +nd pldr +mu sson +mor ass +mon sef +miracle treatday +mcki bbin +maz ara +kron berg +km ss +kis on +khan de +keepit public +kam sa +k san +just ici +j co +is ymf +inter fax +ick en +hast ens +ha ka +h tweets +gre sford +ge trowdy +g lines +fu zion +fu gs +ft nqt +fre a +fo sho +flo rea +ever body +et attoo +er stalk +ent rap +empor da +ellen son +el aval +ekdeewaan atha +ee er +ea stover +e ion +drumb agus +dr al +dor rian +domest ica +dine fwr +digital learning +de baser +david ferrer +dareto achieve +da oust +croco dil +crit ica +cos ch +corks redfm +cooper ator +che me +ce deno +canary island +ca ele +brit ton +bil angpilipino +bhan ja +ben souda +bed stuy +bed der +bac sin +aval ok +arti stin +art sctr +arri age +appro che +ankush loveuall +and alex +aldub ftnqt +advance qld +ac ckickoff +ðŁ¦ Ķ +ìĹ IJ +zul ki +yel a +yayo tte +world whiskyday +wo whead +wira djuri +wim bush +wee ty +wav ell +vijay tv +vibr ators +vel indre +va ine +ucu strike +tze hn +ti german +thin ness +sunset strip +sun records +sul than +speak ership +sne ach +sl r +sky harbor +si pri +shut en +sho bu +she ilah +search andrescue +sch euer +saving system +sav ar +s shop +rut kowski +run streak +ronde santis +r pas +r ala +quadri ga +prof dev +pork ys +or dem +offer te +o hene +nw sw +noaa fisheries +ner am +ne so +n sel +mitchel musso +mega project +mccre ady +mc peak +mas ke +mary lander +mar ris +ma es +lu gh +lovel ove +lohen grin +lev ick +leigh anne +leeds museums +lazz aro +kv bohra +kra fty +kom mer +kim jun +kha e +kempe gowda +kad ı +juli ahb +jeopardi zes +jdff fn +jaw ara +jason reynolds +jar boe +indie films +il mu +hinching brooke +har ahan +hai me +h enton +grac in +goon di +girl school +gar bett +fore tells +eye em +et g +elimination chamber +eff endi +e hhhh +dy bbuk +dor ries +dom pet +dir ilis +dioce ses +defaul ter +cron an +cring y +copp icing +cle atus +clam ouring +ci elo +ced chat +career center +care ening +capit ole +can nock +cal state +busy ness +brown thomas +brown ing +bou ille +bj ör +bio y +bergen county +be abin +back pages +bab lu +aw sum +aval ance +av anna +arti k +antoniosab atojr +anan avarro +alu ckett +all ene +ak or +ach tzehn +ac anth +abou bakar +ðŁĺĤ ðŁĴĶ +ðŁĴķ ðŁİĢ +ê´ ij +ê° IJ +za hid +yel les +wra c +wer q +weis ser +vit ek +visu ally +vill ach +vandana shiva +ur win +uniof glos +tweet suk +too short +til les +thenext one +the grind +ter ai +ten sai +tele vising +tann is +strath allan +solihull moors +so lie +sj hs +sier re +shop at +sex posed +resu me +ram lal +ra zzo +ra uh +ra ggi +quatu or +pulled pork +pon za +ovo xo +nokian etworks +nbs frontline +nation alization +n qf +my lanta +monoli th +mish kin +marshall u +mal enko +ma ire +luxur yy +lu chino +lps leads +log ne +lah bati +kno tto +ke me +ke ito +kalani thi +jönk öping +jab ot +j cg +inthe woods +inter lace +in reallife +in kers +illusion ary +iam super +horri fies +hi ebert +hex ane +here comes +hep cat +henning sen +he dden +hb wx +halt whistle +ha ine +h kl +h bu +greatocean road +gi gan +gi ardina +geek out +gas prices +fra sers +for thood +first champ +extre mo +espou sing +er ci +eng irl +ds band +dram atical +disa strously +dil ruba +de freitas +daugav pils +daress alaam +dam ine +clean technica +classic motor +christopher sean +chif ley +chan teur +chag os +cesen atico +car chives +cancer prevention +camillu ddington +bwn sft +bro dgar +bo it +bil ad +ban ki +bam bara +bachelor nation +babie jaan +ba rer +b sh +aurorahu skies +ashi k +as de +ar uknews +anu ma +anamne sis +all ant +alex the +ab ug +ðŁĴķðŁĴķ ðŁĴķðŁĴķðŁĴķðŁĴķ +ðŁijĮ ⾨ +ðŁı ľ +ðŁĮ ģ +ë§ IJ +âļ Ľï¸ı +à® ± +Ùĥ Ùħ +z iro +wonderwoman film +wash spirit +ward han +w va +volve remos +user research +unstopp ables +un youthenvoy +tnt vote +tie breakers +tic hy +theblue coat +the pro +th alab +te tes +tan dil +sw u +stunt men +stat erooms +sri aurobindo +spear heads +sos fanart +sma drid +sig mak +shung ite +shelove sme +see hofer +samu raic +reg la +redro ses +rad agast +queen stennis +pink villa +pin jarra +pad an +p tofed +p list +oy j +oswald twistle +oh u +nu va +notor acism +not that +nor wic +ne ste +miz utani +mis sma +mick legate +med lab +ma sto +m ve +m pre +lit o +ley music +lev ator +le pine +la ket +kwant len +kor dei +kk onen +ki ppa +kell yayotte +jim james +j assie +it ories +io h +indian river +hom ilies +hom enor +he mmo +he do +harland williams +hang i +hak ama +hac ke +ha cha +gray bar +glob alizing +glen mark +girl ll +gi essen +gar hi +g wang +fri jns +freedom pop +f op +f bisd +ecuad orians +ea stre +e toe +dro me +dri vin +dk b +dispro ven +dir co +db sk +days gone +dan aroh +danaroh rabacher +da ira +cuse football +ct fletcher +coch lea +clan sman +chit on +chandra a +cente redness +cci w +cat kin +carpet cleaning +c mcs +bur ritt +bur rit +bur ps +bristol council +breconbeacon snp +br ous +bmoharris bank +bittrex exchange +bhagal pur +berg ü +bath chron +bad boye +ay ami +auto shrine +atlantic ocean +ash ur +any where +and ora +an r +alabamade ptofed +al smtl +al goa +ag ener +ad missible +aci do +ac sc +(** *) +ðŁĺį .. +xoxox oxo +wood cliff +wa hhhh +wa ala +vi official +val da +un revealed +un ct +uc ina +tucker ton +top or +tony t +the cottag +that momentwhen +sã opaulo +ss and +sp ates +soar ing +slu mming +shy la +shir king +shak ha +ser ino +sequ al +sea quest +scum villain +san chita +samsunggalax y +sakura ba +sag ot +rosen field +reclaim temples +re vent +re housing +radi ot +ra wer +ra kia +q music +publi ka +primary school +pic say +pha i +ph ans +pg tips +pelo tonia +panzero tti +pad auk +pa pps +p fl +oster ville +nu oc +nrl storm +nor c +nid drie +nationalcheese burgerday +nation alize +nan og +nag ma +my rin +monifi eth +mat twal +mar kups +mag ery +ma zie +m dotnews +ll cs +linsey godfrey +la don +klu mp +kings road +kher son +jon te +je had +jagi ell +inter ludes +instagram aviation +inspir a +ingeni eur +in z +ike me +hotel ympia +gu agua +glo ved +glo ats +gh ook +gam pel +fuel band +free guys +fre un +filip iniana +fil ipa +field view +felic ite +eye sof +en strom +eat local +easter by +e je +e est +dys morphic +dur ston +doge ver +dinner party +dic kie +delhi governance +dave matthews +dar waza +cor zine +coc chi +cirro cumulus +ci err +cfl draft +cash cow +call al +bunting ford +border security +bobby flay +blood wood +blo on +bible verses +bham bri +belo it +bay da +as microbe +ar lyn +ar ada +an derer +amandase ales +aerom edical +a angan +-------- - +(*´ âĪĢ +' - +ðŁĻĮðŁı¼ âĿ¤ï¸ı +ðŁ§¡ðŁ§¡ ðŁ§¡ +æ¸ £ +âĻ ¿ï¸ı +ñ an +yu kino +yim by +worlds best +wood chester +wonder la +wire tapped +was ley +wal shy +visit wiltshire +u av +tun aji +tororo sso +ti sd +thalai vi +tay lore +tan ev +tag esschau +ta vener +swift water +swayam sevak +stat ecollege +sm oooo +slu dgy +shirat aki +shic ooks +shell no +senbill nelson +scro ps +roar for +rider nation +reza ian +reu el +ren nan +reis inger +reflec tance +recali bration +re eeee +railway men +queré taro +pri am +pre stel +pran dial +post grads +po too +plas matics +peace full +organic chemistry +onther ange +o zeki +nvi dia +nuit blanche +nau fal +n mw +musik verein +murmu red +millen colin +meth ley +me ggy +matti oli +maha veer +mah mut +magno liam +lo cum +live work +li pi +li le +let son +lavany ab +larkin poe +la isse +kro ko +kor is +kal ev +k snt +k mox +jojo lion +jersey boys +jamai raja +iz nik +hypn ago +huawei mobile +hr weather +hol son +highway sengland +heteronor mativity +her berger +gy ps +gp b +go vikings +gat wick +gal al +g vs +fra zer +fotogra fo +foot ages +fogel berg +fashion diaries +europe union +eti seo +eni elsen +en no +emr gency +ell and +du bie +drac onic +di rec +di du +delta pi +deci de +dav its +dar shana +cu ta +ctfletcher isymf +cra zing +clam per +chup ke +champion sof +cap city +cand ys +cafe bustelo +c phi +but chie +bro ca +box hq +black deser +bin dle +bi si +beautifu lon +ball python +bali united +av elli +aster son +arkhangel sk +ar ail +antan en +amc clain +ak au +agen i +ag ril +afric ageo +adult work +adri atic +ðŁĺĤ ðŁijĬ +ðŁijĮðŁı½ ðŁijĮðŁı½ +âķ Ĺ +à¹ģภģ +wyff news +wwe backlash +we fly +wal deck +wa ira +video art +vaill ancourt +uri ah +upto date +up en +un ay +u shahidi +tree of +then g +thelonely island +the pig +the houseof +the first +the bronx +the bobby +tere x +temper aments +tar bes +tab u +street party +st catharines +sport said +sou bise +side hustle +si donie +septic art +seal ers +sanjayleel abhansali +sam jarvis +sam bassador +saiful lah +ross more +rish on +rise borough +ri jo +referen da +rdr hw +rdrhw ke +ra glak +que eze +qld maroons +pun tarenas +pulwam ater +porta ventura +orang i +or fc +od ham +o sma +nikon canada +new blood +mul led +missan dei +man cer +mamed yarov +maha kumbh +ly co +low fat +lov ington +lou ds +lost planet +lor rain +local syr +loc ascio +lo ker +lat vi +la thletics +kope ch +kk f +khan al +kaw amura +kang ta +ka arora +k out +k bye +k block +ju che +ip eline +inter regional +inhal ers +il ynn +howard winn +holocau stuk +ha gue +gu mbs +gu ddu +go fer +gear talk +gani zation +function alization +fri dman +forthe culture +fight likeagirl +fiba europe +fel trin +far ner +du sun +divers ities +dig na +di grassi +dela uro +del is +datt ilo +ct as +cre b +cost i +cook stove +child labor +ch haya +cer as +castell an +cam ming +callfor papers +c music +bul keley +buffer chat +bre tz +bla i +bird studie +big bend +bet to +best deals +belfast trust +behindthe scene +beard day +b koepka +astro family +ap ki +ap ath +and in +an sty +americanc rime +ak dn +/ âĢ¦/ +!! ). +ðŁĵļ âĿ¤ï¸ı +ðŁĩ¦ ðŁĩºðŁĩ +å® ¶ +Äį iÄĩ +z ater +yaq oob +yach ts +xcel energyctr +woll man +wit zel +where it +wh w +w uss +vichy sso +un assembled +u an +tttt tttt +tin ed +the same +th und +test is +tar quin +sudan ese +su jin +str acing +stati k +star zz +sleep well +silicon beach +sig nes +secur itisation +sad lier +rs one +rondesantis fl +red band +ray han +ray bucknell +qu ain +pris matic +po dia +plo de +pitch er +phirek baar +par doe +pap onmusic +ou tran +om men +nvi vo +night ma +newham london +nar done +na has +n carb +mustlove dogsnyc +moyam ee +more h +morales nbc +moral monday +mo hana +miner alo +micro dot +lough or +london underground +lin ked +ley endo +le ke +lam otte +l rc +krysty na +kore sh +kor do +kim on +ke bang +k boo +jat in +jan maat +j cf +island peeps +is gusting +ione sco +horticul turalist +her mia +hau f +gre ate +gre aney +geot agging +gan ador +g kg +felicit ating +faf nir +eye dea +eu greenweek +e gon +dun stan +duba itour +dol lmaker +deod ori +dahl strom +cy ne +coz ying +coor paroo +bun nic +brumis brill +bli sh +bla ker +bi bek +bertol ini +bear up +barri o +bad religion +atheist pics +argu in +arah ma +ar bil +anupam apar +an laby +am ans +ðŁĺĤ ðŁijĢ +ðŁİĤ ðŁİī +ðŁ¥³ ðŁİī +æ ¡ľ +å· ¥ +ãħ ħ +you love +ye osu +wool lard +with rome +whit ted +waz owski +vam usic +v icia +ut karsh +unis an +un hq +un disguised +ud fa +trebbi ano +treason summit +tot tering +tiny desk +thef dci +t mu +t ellier +sun dazed +sullivan stapleton +sublime withrome +sto on +star tyour +springe tt +spec tat +sow mya +soun ion +sm ommy +shu ghes +shin agar +sar oundtheworld +samp aguita +sal tney +sacri sty +repri mands +refu sals +ra yos +pu glaas +po den +phone bank +pet lovers +pat kar +party wear +palom ares +pab a +os nat +operation alize +ob son +nat museum +nap er +mustafi zur +mrat andhan +misogyni sts +medical school +mb b +mat tryan +manus amoa +make you +ma seno +ma iri +m shs +lollo brigida +lo ssy +legend re +le tra +la sha +kon ig +kin loss +khu zdar +ke ba +kay ah +jointhe hunt +jo equ +jennifer lopez +jazz guitar +jack reacher +ja ho +insta stories +hyper converged +hor miguero +hi ep +her zig +henry mcmaster +hard life +gü rel +gra uer +gott al +glo bo +gar nham +fv glive +fro moz +fo tom +fever few +er berg +ent zel +du ology +dri scol +dit c +di mages +di ante +deli ke +del li +dejec tion +deci siveness +de presse +dazz le +cu cur +cre ll +corpor at +constitution alist +complete streets +co ziest +chop tank +chast ises +care sse +bun naha +bull s +breath ers +bol ong +block stream +bi zi +better life +ban deira +au det +at tern +arric hannel +apportion ment +ap ti +angelas canlon +amin ata +am roth +all me +ag rant +a akin +ä¼ Ŀ +zz acnn +z scaler +ye dition +yak ub +wg x +wene ed +w kw +vacation rentals +tran z +toy town +to gar +thick ener +the sopranos +tent en +tar getting +talent acquisition +ta wil +swar ts +sw and +susan ville +suno cor +stan chions +sputnik int +spay and +so bu +sig ala +shu ter +shigh school +sg len +seri ksen +sensiti zing +sd summit +salv adore +sai bot +saber metrics +s my +ra avan +pod fix +pier zynski +perri man +pensami ento +oro der +oranje stad +ober lin +nul led +nol enation +new fane +n jac +n alu +mush ers +mou lins +mo il +mo he +militari stic +mah boob +lima hl +lift bridge +labr ys +la dolcevita +kur tz +kram atorsk +know sley +king makers +kas am +juli aroberts +je der +in ae +har jit +hack berry +ground works +greg sestero +gre ninja +golden era +go duhawks +giu giaro +girar dot +get ter +ger ken +gem sona +gang adhar +gam ingh +g wich +fish cakes +far fetched +fal sify +evidenti ary +dy spe +dun gloe +dpt student +dmv hoops +demar ini +del wp +dar lin +czar necki +customer service +clt food +cilli zzacnn +centin ela +cc bl +cag op +bu achaille +brü cke +bru gby +brit ts +bret michaels +brave st +booboo stewart +blow out +black tie +bi h +bert strips +ber ano +beforeyou go +be rel +b ance +ato cracy +ati oneu +at alan +ast age +ar li +ap lang +and ung +ambiti ously +aint no +aid il +aged don +ad avies +aba journal +ðŁĻĮ ðŁĴª +ðŁĺľ ðŁĺľðŁĺľðŁĺľ +ðŁĶ¸ @ +ðŁĩ¨ðŁĩ ¿ +ðŁ§ĺ âĢįâĻĢï¸ı +ðŁ¥ĩ # +íĶ ¼ +ë° Ķë +ä¸ ĥ +âĿ¤ï¸ı ðŁĮĪ +young leaders +yamaham otor +women inste +wir h +ve dette +user interface +unboun ders +ulster grandprix +ty ran +tri pathy +til tro +the hi +the cle +terry fox +templ enew +susan n +sp aeth +sop hy +soko lowski +skan sen +sit z +sik hi +shar am +sham balla +shadow banned +sent om +sel d +sd ick +sch emed +sal eyards +riff age +re submit +re mora +re hired +radi ans +ra unch +princeton upress +powder magazine +post traumatic +pi eno +pflu eger +parade of +pakv sind +pa q +on etwork +ome de +o esophagus +nys dot +not forsale +nom is +nol on +n gara +mu cosa +mou lya +mer q +me giddo +maxg schneider +mat tc +mal herbe +maison avendre +ma sumi +lucas dirt +laquan mcdonald +lan gholm +kam os +ka ash +k do +ju bei +ittake stwo +induc tors +ib sf +horror core +homo erotic +henness y +hase ul +hang out +hal m +gui ao +giriraj singhbjp +fuse box +free zy +faro ese +faci ally +ex ocet +enlar ges +emrgency kittens +el ul +dysp nea +dream girls +district speed +digital twin +deple tes +danny sullivan +dal ab +county durham +col v +col lum +coffeet able +chicago symphony +cherno byl +caz ador +capric e +cam utd +cal mes +bur mester +bur kha +boli de +bo wne +ber c +ben aras +bal ewadi +ase er +as obi +ang g +amp adu +all ones +aha asan +!! âĢ¦ +èĩª æķij +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı# +Å¡ i +y xe +whoa jordie +werewolf wednesday +v achi +un cut +tom boyish +the legal +ten de +swanse acity +stig ma +st lukes +spiri don +spectro graph +soch aux +sn omg +shaz ad +sha henshah +sd awson +salford cityfc +s north +s back +riaz or +rallye montecarlo +rad wan +pu ting +pter anodon +port ant +popul ationday +po ple +pic fair +phyl lida +path mark +pastor alist +pap ar +p inging +or tom +oooo oooh +onor ris +nor aho +nom nom +netball worldcup +natash aleggero +mysteri esof +my happyplace +move it +moto corp +miss them +mile high +mh p +mezz otint +meme monday +mediat ama +margare ts +madhu sudan +m duk +lucre cia +lu bis +local ise +line mates +ligu rian +land onorris +lac er +l varchives +ku ne +kour i +ko kane +ko ban +kis lyak +kin daichi +kennedy nation +ke ister +kashmir floods +joh no +it vs +istitu to +ise kai +ioanni dis +inter val +ing var +in britain +immun ological +igu anodon +he mme +hart pur +gru bbing +gre gari +gor ki +g lead +free beacon +flu es +fl l +fin try +fa xe +explore victoria +eugeni des +ee ks +e frem +div yadutta +di pa +di orio +dee v +crypto spor +creative market +cor undum +coo pride +conspir atorial +congr ates +commerci alised +citizen suk +christian slater +chamber layne +cath kin +cake pops +boxerdog union +boss logic +boar drooms +bo fill +bellec i +bbca q +bali ka +bale ful +b gt +audi southafrica +atom ica +arab net +ar vis +anam era +ai guil +agu stin +adel le +ade vine +aber cynon +^ *) +. ðŁıĨ +-_ -" +! = +ðŁĴĽðŁĴļ ðŁĴĽ +ðŁIJ ģ +ì¸ Ħ +ঠ¡ +न म +x media +worm ley +wedding dresses +we own +w ps +von tae +uto pias +un salted +un ation +un adorned +ty m +ty che +tou la +tou l +thuli madonsela +ten aga +te jay +t bird +sy nec +suk ses +stron aut +stock mann +st w +spark fun +sleek makeup +sk w +si dr +shanth nu +seed bed +se mba +sau thentic +sat oko +sam gye +sa heli +rye o +roar withpride +riverfront times +reclaimed wood +reali gning +qu ants +pul le +price waterhouse +preserv ationist +pp op +padmal akshmi +ox fords +otta viani +nuis ances +nu tan +nsc ad +no filters +neuroradi ology +nan aman +mutu ality +model mgmt +mod pizza +memor able +me tten +mary rose +marthamac callum +mar wat +mai ze +lyne ham +lin sley +li fan +kro onstad +kre ms +ki dding +keffi yeh +kar ley +jman rara +jeff brazier +jeal ously +ip ython +inebri ation +incumb ency +ik ko +hun ches +ht cafe +helichry sum +he sj +gren dizer +gc v +gat ineau +fly together +find sorguk +fag us +ex ss +elaw ler +eg n +eck elibrary +di eck +deaf ened +dddd dddd +dal ston +customers atisfaction +curi ae +colorad ans +coach mike +co sc +clar kin +chuck palahniuk +chron am +chav annes +cau ser +ca rele +bur u +bren gle +bren de +bly leven +bhagav an +bh hs +bein eckelibrary +b ta +ati an +ast en +app raise +anticoagul ants +an ings +age a +adri anj +ac ros +ðŁĻĥðŁĻĥ ðŁĻĥðŁĻĥ +ðŁĶ¥ðŁĶ¥ @ +ðŁİīðŁİīðŁİīðŁİī ðŁİīðŁİīðŁİīðŁİī +ðŁĩ¹ ðŁĩ¹ +å¨ ģ +ม าà¸ģ +ठħ +you cef +yo hei +ulkom inisterio +turn blad +tren chant +thel as +the velve +the futureof +te he +tal manac +sv ill +su gru +stony brook +star sfc +ss ay +sports nutrition +solo lastyle +six point +sime on +silver agetv +sho well +seeyou again +se ena +samira wiley +saf l +ro ent +read women +ram ÃŃrez +product development +pre installed +pietra santa +peristal tic +past it +parami yer +pal ette +pai gey +oberge fell +nu ñez +nkosa zana +nas ser +nam aqu +muscul ature +mor tu +million ai +mii foto +mick conlan +mav ro +lackadais ical +l án +kro gan +karl skrona +journalismis notacrime +jimin hofe +islandpeeps birthdays +incre dulity +honest ly +her si +health systems +haj du +gu pton +great clips +gott es +go lia +ghost land +fitz carral +faru qi +fabric live +f he +everythin ge +eu dy +ere tz +eli yahu +eigh ty +ec pr +dz hok +du kat +diaz epam +de regulated +de hydrator +danny kanell +d total +cycl ine +can filmday +broadway com +brigan te +box cars +bombar dier +boin net +bo witz +bi agi +ber toni +bay le +bab li +av illage +audio technica +arm end +apo thecar +andre greipel +ambul anc +adil ray +ad nate +ðŁĻĤ . +ðŁĺį ðŁijĮðŁı» +ðŁĶĿ ðŁĶĿ +渣 åıį +渣åıį æ´¾ +渣åıįæ´¾ èĩªæķij +人 渣åıįæ´¾èĩªæķij +ziau ddin +well wishers +vyrn wy +volcan ism +volcan ic +video love +van meter +ul brich +uh manoa +twenty something +tri delta +to vic +the sportshub +th onet +tex hibit +stau dt +starry night +south pacific +sof ie +smart phone +sl h +sky ride +sir specialists +shing led +sh sm +secon ding +se any +sau jani +san miguel +road rage +ro dent +rhyth mically +radiocitizen fm +ra ymi +q j +presi dium +perfec tion +pa illard +outfit grid +out t +ok ayy +nú mero +ny ghoops +ny akun +nile postnews +ner ine +ne ate +nanse mond +nag ap +my mc +must reads +muni er +moo rel +mo sconi +mit tag +min tel +mh chat +me pratap +mass aged +marri yum +mark ku +marchitec ts +maha vishnu +mag ph +mad dox +lot z +lea ke +ld week +la force +kor bol +korbol orbo +ja jafri +itsallgoo dep +imin love +huss am +hollyj green +hirsch horn +hi do +hen party +heartw alk +gu revich +green ie +gre lle +gom ers +gan bare +g wx +g burg +fr m +fos sum +fil mcenter +feel z +fe dri +fash nerd +facility management +ero yal +ermah gerd +er so +elev ated +el ay +ec ken +dur rett +dream stime +dh any +defend ing +def our +dec ay +dani elli +cyclonef ani +co wan +caw ston +catte drale +carbon ell +breastre construction +bois vert +bhu van +ban ews +as le +ar ren +ar jan +app ar +aom ine +antag onize +andrew scott +am exico +aircanad acentre +ain z +agi letd +aften posten +af thunderbirds +abhil ash +; )))) +ðŁİ¾ ðŁİ¾ +ðŁį ® +ìŀ ī +éģ ¸ +æĶ ¾ +åĬ ł +âĻ¥ï¸ı # +ঠ¨ +zo tac +za ine +ym pathetic +yab ooks +wx pn +woo fs +wild thing +war gamer +vir ani +v its +us of +under body +u kie +tsu shima +tri pl +trabaj ando +tejas wini +te os +tb alls +tash amed +tar kington +tamar aw +taitt inger +t pay +t cl +suss an +supersport fc +st francis +springh ouse +spas modic +sonic mania +shear waters +sharp stown +ser hiy +sem plice +se vo +salam at +rubber duck +roo sa +rocke ts +ribo somes +ri bon +rac ci +projec tx +prescri ber +pendi dikan +payday loans +paol achi +pan te +off sides +north walest +norfolk show +nevers leeps +my favorit +must ad +moel fre +mirac les +me der +mayor kun +mass ac +margo twall +loy le +loyle carner +love flowers +lindsay mendez +light sticks +kur ram +kirstend unst +kar aw +kami akin +k him +jessic aes +isi ana +io anna +in vocations +in ma +ike barinholtz +iamraj choco +iam lindsayjones +hyper trophic +hummel stown +hiro o +hill enburg +he bb +ha utes +h anian +gur ram +giving soul +gh raib +gabou rey +g sburg +g ite +fy ne +freak in +for him +follo train +fire boat +fi stu +ffi on +ferr ite +fe stac +fe bbra +fall acious +f ê +f twd +esto u +escri va +er sa +ent j +efin itely +dol son +diplom atically +dhanush fans +dewsbury rams +degre ed +dan il +cn sc +charless oule +cham bery +carrageen an +bun kered +bre lade +bran dished +berser kers +bb ad +banger ter +baj aur +b sy +auburn football +akh laq +adv i +abh ors +. ðŁĺįðŁĺįðŁĺį +ðŁĶ Ģ +ðŁĴĸ ðŁĴľ +å¸Į æľĽ +yin yang +y way +willowh erb +whereis adtr +wait itu +w lv +vin ces +vi zzini +vers ity +ver icks +ur v +twi p +trustthe plan +tj es +ten newsmelb +tax us +tang mere +sun music +stor min +stor ation +sti pa +spro ductions +sol ms +singh ji +sid malhotra +si eves +shijiaz huang +sh vili +sh eller +set as +sel ine +se tra +sd am +scifi actor +san tha +sab ourin +sab bah +rte soccer +rough est +ro stec +ro mil +repeat ability +rejoin der +reducere userecycle +re interpreting +py charm +photor tg +philly chic +phill amarr +patri zio +numan official +northan ger +nor anda +ni pes +next cloud +ner s +nandit adas +nanai mo +na hhhh +mutt day +mend ous +mediab ias +mary katrantzou +marsh wood +markh enry +mar ami +man ou +lo ong +letsgo heat +lee schools +lax atives +latchfor devans +lang age +la ing +la chs +l latchfordevans +kom al +kho khar +kel vedon +ka io +juxta poses +jo josi +jagran news +ir ambulance +instagram down +hon k +hill croft +helle buyck +he mis +harmon izes +gun it +gu é +gru bbin +grac ies +go ire +go flashes +go crows +gel ora +gamer retweeters +fer ial +f sk +ec ymru +eaton ton +dubai worldcup +domin ate +din as +dic ec +declin ation +dbt india +day tour +dat elin +counter measure +cle ghorn +cent i +cancer moonshot +c mrf +buy british +bon zi +black ferns +birdstudie scan +bhatt a +ben to +belk bowl +bar ison +autom at +atal aya +ar cona +anten natv +alex bowman +akwe sasne +adore e +ablu e +ab lock +a ÃŃ +ðŁĺĺ ðŁijį +ðŁ¤Ķ ) +âĶ Ķ +win dians +wellcom elibrary +wee ter +wan chope +wad is +w week +up your +toshi ya +tomat ometer +to dom +tal u +ta haw +swwap nil +sweater weather +sul cata +stru ble +ss cho +sonequ a +so bo +shari fa +sham ilton +sex tra +scan di +san andreas +sagitt al +s graffito +rhudd lan +red hour +radi ata +pwe ase +phil ology +p ells +p cin +ortho graphic +or bea +octo pizzo +nade ch +my friends +mur re +mohun bagan +mo dit +missi le +mir u +med star +me hs +mass dcr +mac er +lukas ulic +lu isi +locker room +leeann womack +le edy +l sjnews +kol om +kar ao +jae hyo +itsabout time +ij er +iafe ssia +hu ot +hu bert +hom icide +herni ation +her f +har uko +gun dry +gu ri +grit stone +gries bach +greenman fest +gou w +golden yearsof +go pers +giorgiom oroder +gilead sciences +gh l +get stealz +fun i +fru its +front stretch +fri skies +foxy gen +fion n +fex ile +ferguson october +fair lady +f pu +exac ts +eugen iafessia +epal ace +ei ras +eco leman +do bler +deutsche telekom +desay uno +dead fall +d hand +cun anan +crimin alised +colonial williamsburg +chi haru +cherrybelle indo +challenge yourself +chad bourne +ch eva +ch azy +cdw festival +carolin ed +box nationtv +bottle shop +book facefriday +book buzz +bonda renko +bon voyage +boardof directors +bo sk +bit of +bigg ame +bi jl +between the +batter ie +az tex +athe istic +as nr +ar ki +annap aquin +ange rer +amp p +alcin dor +al tab +above thelaw +_ -_ +ðŁijįðŁı» ðŁijįðŁı» +ðŁ¤· ðŁı½âĢįâĻĤï¸ı +ðŁ¤¦ ðŁı¾âĢįâĻĤï¸ı +ìĿ´ìĶ ½ +⾨ ðŁĮ¸ +âĺ¹ï¸ı âĺ¹ï¸ı +âĹĸ | +ØŃ Ø± +z b +wildbill photo +wild wood +wi ecki +whippoor will +west law +weihnach ten +weall bleedblue +vu u +video clips +utt amav +unge rer +un desired +tw alls +the lead +tere ssa +te ich +tchouk ball +tabletopr pg +sur fcity +sudarsan sand +stan away +sque ez +soli psi +sm yl +shot crete +sen alexander +semit railer +scill onian +sat anic +s endra +queensugar own +progno stication +phil trum +pervad ing +per vasi +paris jackson +par ous +paloalton tw +palindro mic +ote y +onceuponatimein hollywood +off beat +off as +ob as +noraho donnell +non ito +mul tan +mortg ag +mer nda +melo dica +mari za +mar ki +mand ino +maddie and +live on +liti gious +kul len +ku mag +kni p +king sisland +kin neil +kather ina +kani mozhi +jan ick +is eries +iam bohemia +hon ored +hin ze +hell s +hatcher ies +har da +hammer smith +gun s +gre ases +gel ada +gefil te +front door +fri m +fother ingham +foo de +fleuri eu +fieldre cording +fer ric +ext asy +exas cale +est ados +er can +en voi +ell sberg +el vina +el sen +ec pa +early year +dub awi +do yon +direstra its +dg f +detwe iler +destruc tor +de castro +craw leytown +corpor atocracy +cor biere +contor tions +conecu h +comm ending +colle gues +char ging +chakravar ty +btsout castd +bi gos +beagle facts +be ani +bath nes +axi al +astru d +aru th +aragon ite +apr ili +ap aw +antiguabar buda +android app +amor ya +akhmat ova +ad ine +ac ot +aam c +ðŁĻĭðŁĻĭ ðŁĻĭ +ðŁĹĿ ï¸ı +ðŁĴ¯ ðŁĻĮ +ðŁijŃ âĿ¤ï¸ı +ðŁ§ ¸ +âĺĿ ðŁı¼ +à¸Ńม ส +ঠ® +Ú© پت +کپت اÙĨ +ай д +zen ia +y ami +woo dring +wee ked +wa aw +w bap +ver din +van diver +us br +un ruffled +ttly teala +traqu air +thrill ingly +the cool +ten ax +tai v +supp lant +state street +st tropez +squir ting +sports media +sn cb +sik lan +should be +shi pra +sgo t +sedi mentation +saw ley +satur nia +rock wiz +ro chefoucauld +river hawk +rene ged +record label +re aney +ran ney +pur dah +pre ordering +pla bs +paolachi occhi +oxidi zing +over spent +oko lie +official tulisa +ob serve +nystag mus +non duality +newstalk zb +net mediatama +ne ung +nazar é +my kitchen +mu toh +mon ads +mis ur +mi metic +mei sha +me thi +mc as +mau boy +marien assar +mac kidsbooks +ly dd +luka ther +lov ell +ler che +leekwang soo +lam phun +kirst jen +kansa shistory +k wwl +k ello +juli ec +john cross +jn rs +jac anews +iu chi +ion izing +invest is +introduc er +ing omar +iconocla stic +hm givingsoul +heph zibah +hen ze +hell ll +hakkasan lv +gus beef +guiller mo +geode tic +geek s +gd ha +game gear +ga day +fre ema +fox hunting +firstalert ct +fe kete +entren ch +enni als +ela dies +education fest +e wer +drau ghts +dog u +disp late +denou ement +dag i +d ack +cynthi ana +cyber tron +cur ated +ctvmorning wpg +cottes more +contextu alized +con acher +comman do +ck k +chin nery +cent aurea +bünd chen +bv n +bry ana +bole lli +bit shares +bish ley +bisd pride +bevac qua +best wood +barbar alee +bar rowland +arnotts dublin +arac elyar +antondu beke +alyci adeb +akl council +ah vaz +activ ated +aa sia +ãĥ ĺ +ãĤ°ãĥ©ãĥĸ ãĥ« +you do +y cat +wom mack +wgx anews +way laid +w bell +vu ght +vel via +v fm +ut saf +uss ari +us by +tur ku +tunaji bu +tran shuman +topo data +to ka +thedaily meal +the very +the out +te anu +tc prepzone +tas si +tacho graph +swis consin +swi veling +super coppa +ster k +steakn shake +ss aint +sports fan +splatter house +sk filmsofficial +si akam +scary farm +sar iska +s good +run ton +ross o +reverber ations +resh mi +r ba +proto zoa +pil son +ouri f +orientex press +one way +on ow +ok han +ofrac osmetics +oble zada +o group +nuclear weapons +noki amo +ne sters +narcissistic abuse +nab ard +n ta +mosh pit +mispron ouncing +miam il +mi va +megal omania +masi ello +mariano divaio +mar ve +ma uk +li sel +le azes +lamp man +l alife +konz ert +kin donesia +kh ater +keyand peele +job st +jeremy mjordan +jal ouse +itsc old +it done +ill ys +house trained +hf ma +hel ove +hay u +hann ay +guer tin +guay abera +gu sher +gaurav kapur +gary gulman +fu thark +fire pro +fil mand +fil lup +fansn stars +f pack +examiner com +evo shield +et ang +ess ington +en eu +eli zab +ear mark +dru ce +dissoci ate +diag ne +den isle +deb icki +de sfile +dau k +dam es +csic yber +cros stour +crafty chaching +cr rc +conversation us +cole tti +chy trid +chriswe id +cheri e +cbit weets +career sin +car land +campus rec +buch ner +brain tumor +blue tones +birth of +bio geochemistry +bh or +believ eland +be j +be greater +bao babs +baftac ymru +av ene +attan asio +ater ra +arrow fieldstud +ard ell +archenemy metal +ar or +apoorv amehta +anticlock wise +ans gar +and en +alto adige +alo st +al lots +af low +ac w +a story +) !!!! +ðŁĻĬ âĿ¤ï¸ı +ÑĢоР¼ +zu baida +z ima +z ager +yong bosch +yade j +wood side +whati sschool +wf my +wewill waitfor +war ford +usu f +tu chman +the fall +th fan +templenew sam +temple ogue +tat raffic +t dr +sy ne +sucr alose +stan dees +st ites +spon ging +spi ffing +sky music +skew ering +sit rep +sister ship +sis sel +sin éad +shog goth +sho b +seed sman +rspca qld +roy blunt +robber y +river plate +rin th +ride the +ric asoli +renthusi asm +re spo +re man +r anco +q ala +pr iciest +pr fm +pomer ania +plu n +pic hincha +peperon cino +pent ennis +pay in +ott weather +nom inet +national nurses +na ves +n xi +n gah +muslim pro +mud died +mu ffed +mon ark +mol lison +minim ises +mic t +me lectro +maxim alism +marn grook +macau lay +ma or +ma gri +m wu +lu zzi +long boards +like toknow +lia ise +lev ada +lauren cimorelli +lan us +lan dish +la valier +karyak arta +k len +johnny yongbosch +johncross mirror +jessicaes anchez +jeong min +jeet music +itor loseit +indign ities +ill s +il div +i bang +hox ha +hol lo +hoce ima +ho bia +high ton +hat man +h ici +h anc +gosp artans +good hew +go sox +git ano +gg m +gamesof thrones +four teen +for throad +folger library +figh ton +fi bra +ever son +ev att +elton john +eli eve +ele mento +eel grass +dream theater +dog ge +de gla +cy mo +crush ingly +cott ons +cot ler +cor m +co stal +can ari +caf o +ca cha +bushy park +bryan ferry +brun sw +break room +bow fishing +bishops stortford +be safe +bav o +bau ch +band olero +badrin ath +bab un +art mag +archer fx +am far +allman brothers +alan ritchson +ail ity +ager wal +ae ther +adul yadej +ace comiccon +ac rrm +!! âĿ¤ï¸ıâĿ¤ï¸ı +ðŁķ £ +ðŁ§¡ ðŁ§¡ +æĸ° èģ +ಠ¦ +Ø§Ø ³ +zu a +zar qa +zapat illas +yakima valley +xia olin +we bane +wa hey +w ttc +veto ing +tribecaf ilmfest +trex ate +to kor +theoxford mail +than ol +tend ance +tatt y +t lu +surviv alists +sty lis +stjohn su +st fb +st ard +spre ston +spinal cordin +sp ry +sop ris +somni um +smil k +smart data +slam online +skel os +signore lli +shi rey +senator timscott +scrutin ising +scarlet blue +sandra bullock +ru bai +ros setto +rock tober +ro dt +rip curlpro +rebu king +rapi ds +pil chards +phenom eno +pent lands +pe ee +pau los +param par +ot ps +ot actic +one dog +on gole +nh t +newton grange +naj at +n itec +n dio +moff it +mo bbs +mitch albom +min oso +mid south +mcgau ghey +mc garrigle +mc callister +mb el +mal tepe +love golf +loqu acious +libraryof bham +lea hey +jac are +j np +inthe game +imangel abassett +id leg +i heid +holly gshore +hel ado +he gg +gu ld +gro ysman +gri schuk +gri dman +gori zia +gif ford +gd p +fy vie +fu kn +foo dd +focu srs +fender gbi +f ä +f te +evan oblezada +ev amarie +esch eric +ech t +don tour +do illon +corn forth +complement arity +co drington +citywinery nyc +ch id +cbs boston +caball ito +brown fields +briano driscoll +bor dir +blakk rasta +bj t +bio steel +ber tsch +ball point +avalon hollywood +arma geddon +ap atosaurus +andy milonakis +an jar +alham bra +al ising +ade deji +ac cultur +abal os +ðŁĺĤ ðŁĺĦ +ðŁį IJ +å Ķ +न ह +Ù İ +Ñĥ Ñģ +zel dab +zap atos +zad ran +za idan +wubb zy +wrays bury +wolfies mom +wie demann +weare hiring +water course +w tem +vitamin water +vaxx ers +vander griff +unfail ingly +tun dras +tri vera +token pay +thisi sanfield +terrible towel +symbi ont +sunset sunday +stro mbo +stra bis +some time +sohail khan +smu sh +ski dder +si we +shoes andcare +sclero therapy +scint illa +s book +ruben diazjr +rome e +roisin murphy +ri singer +results with +restom od +r si +priest man +pre ll +po es +plate aus +plan te +pi va +perman ency +pastit sio +paran al +oar fish +northumb rian +no en +neil ston +ne mor +national walkoutday +national agday +n power +myfox houston +melting pot +mcal pin +marque elv +m ggs +lu co +live feed +linds borg +like fatherlike +lid strom +letsmakeit awkward +leigh onsea +koreand rama +koll witz +kitchen decor +ker naghan +kappap hi +kail as +jomalon elondon +jemi ma +inst are +ik ha +ice bridge +hrithi k +homen agem +holy grail +hockey day +henry danger +hel lier +har av +group ama +grote squely +groes beck +gri ddled +green smith +gilli ard +gi ggly +ghet tos +ghe gola +fro gotd +fraterni zing +for nature +fil oli +fb family +falcon ers +entang led +encu entr +el p +ed fest +dri g +doro thee +dg trends +des lauriers +demp sie +deid rick +davematthews band +con tiki +comingof age +coming out +chrisweid manufc +cfc w +car vell +can tone +camden fringe +c gk +bull whip +bu isson +bro der +bran scombe +bel ang +beg int +be de +ar vato +ann one +ane williams +andy priaulx +aly pse +agerwal nidhhi +af arm +ack worth +abq topes +.. ðŁĺİ +ðĿIJĦ ðĿIJ +ë ģ +ঠ¼ +ya ws +x dr +work and +wm of +wern her +vo to +vel opark +vaness am +us ky +tto vino +tre ally +time and +thr onged +the palace +the eric +tex cellence +sustran sscot +superstar life +sphin xes +speight stown +smur f +sma sters +sin uk +sang i +san ni +ru sted +ru brik +roger k +rewar i +rest lers +repar ative +rajar am +punc turing +pun go +psy chol +plagiar ised +phytoph thora +phoo ey +peace ably +pan ax +paleo diet +oni ght +on oa +offici alle +o intments +ns ford +no ell +niku man +ni alla +nag ambie +nadin en +music as +multi spectral +michael berry +metal smithing +melissa fumero +mca chicago +max xie +max ene +maris sam +mari oc +mambo ibiza +mam mu +mac fellow +ma ww +lou vre +lat onia +lar gest +kor ah +kin an +keri keri +keepit simple +ke tut +jun gen +joom eara +j pj +it snick +ifam edia +hoo gland +ho per +high speed +healthe deng +h Äģ +guy sssss +guy fawkes +gru ss +gop chairwoman +gl ers +gil strap +general issimo +futu ro +fun sies +forthe boys +fo sse +fivb grandprix +fe dup +fav ell +ey re +exhor ting +excel ent +elix irs +el wha +ein mal +eh ret +dulwich college +dro zd +der ide +de mic +dcy oung +danielj gillies +cu la +corte ge +com unica +ch mp +canvas lms +cant lie +button willow +bu low +birdwat chie +at oning +asi am +as ki +apay ao +anand an +an amosa +am ily +alyciadeb namcarey +ale es +ah m +action shot +a ung +ðŁĻĢðŁĻĢ ðŁĻĢ +ðŁĺ©ðŁĺ© ðŁĺ©ðŁĺ©ðŁĺ© +ðŁĶ´ @ +ðŁĴ ¶ +ìĥĿìĿ¼ ì¶ķíķĺ +å ŀ +ൠĤ +z aa +yyj traffic +y aaaaaaaa +x sos +world boxing +will ink +west leigh +vy se +vs galang +vichysso ise +vic times +vesti gial +un raced +ul na +trifon ov +torture report +tir so +ti j +thul in +the deal +th ach +taylor r +tam aqua +ta rell +ta etiseo +sun co +sullen berger +sug andha +stu ka +steve woz +sp aul +sophi a +sl b +skidmore college +short coming +shan ked +setag aya +sere vi +sand bars +re purchased +re publish +ram el +que ally +psycho drama +premier boxing +portsmouth news +pin kies +phyton utrients +pen ile +ot chouston +oj ala +ny mets +non white +ni a +neuro ma +musik messe +mu stan +msamber priley +miw band +me ade +maravillo so +mar um +ma kit +m no +love twitter +lm h +lik ability +les den +kn apping +kingsisland pr +ki ess +ki dron +ju ans +jam ala +jalo ta +jad yn +jab iru +irish town +infiltr ates +immigr an +ih or +highland games +her p +hard scrabble +habit ability +ha gd +gro ynes +great news +gra phology +glit ching +fraun hofer +foot long +folkl orist +fo ire +fernan dez +fener o +fe strail +el mas +eileen fisher +dolant win +diam antina +dev days +cur cuma +cor tazar +chri seriksen +chand ana +cha il +cather iner +bu tti +bro g +bro berg +bosco bel +bo dog +blaz ey +bestplace to +barist alife +bar ata +ball ito +bairro alto +bai x +asdfgh jk +art fx +anubhav sinha +ant lered +amo slee +aly th +administr ative +ad uri +above water +(... )" +ðŁĸ¤ ðŁĴĻ +ðŁĶ¥ ðŁijĢ +ë² Ī +æ ħ +⾨ : +à¸Ħว าม +zul te +yogaw ith +ye atman +y ro +x bt +womens rugby +wet plate +wes farmers +wal gett +vu ka +vivam exico +varad arajan +valla dares +up stair +under write +un palatable +uc merced +u wi +twil d +ture brevi +tri estina +the shoe +te bbit +ta ac +ste ddy +sr iti +sony liv +son iam +soli do +smar ti +smacc dub +si ron +she eler +self harm +se alth +scrit turebrevi +sar va +sapp hic +sa hur +ron din +ro anne +ris ner +rie hl +rec ant +rct council +pur rp +proxim ate +post workout +phylo genetics +photonic swest +opol ice +op ined +nucle arenergy +nh w +nan jiani +n itta +mom an +maî tre +mar ren +man asa +lumix uk +lu pica +lt ps +liti gate +lenny henry +lam arche +kol ha +kin taro +kilo watts +keen en +k holi +juju bee +jojosi wa +jan ma +jackson rathbone +itv racing +intu os +impe x +iifaut savam +ig bt +hou tbay +he yl +hass le +gy rating +gur don +gro ome +gre ns +goldcoast suns +ghou li +fu gao +fri gga +fo go +family life +factor ization +ex hume +espe jo +equi distant +eccentric ities +eb k +e ha +dunman way +do rel +dharma puri +develope ment +devel o +de jean +dating me +crop sey +corr ingham +cord ner +coquet tish +coon skin +conoc er +concert goers +colle tte +col ling +cj fl +civit avecchia +chri shol +cherry belle +catechi sts +carol peletier +car ifta +brett kavanaugh +bre cht +bor inqu +bol lington +boden kirk +bo sie +blu st +black bird +battist elli +baser unning +bad astronomer +at aris +andrewr sorkin +allthings mayo +air max +ad sby +ac tres +] ]] +! âĿ¤ +ðŁĶ¹ # +ðŁĩ³ðŁĩ ¦ +ìĭľ ìļ° +ì½ Ķ +人渣åıįæ´¾èĩªæķij ç³» +âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢ +zim cricke +yuvan shankarraja +ye tt +y entl +world market +will sasso +wil shaw +whoare you +waters meet +vit ry +vac s +v lb +u hq +tun khan +tu sh +tren tharmon +tre ta +tre law +tranqu illo +toki doki +tn r +tichin aarnold +ther anch +the di +ter tulia +tak aki +stipul ations +st mike +spla yer +sphoto grapher +speci osa +sp offord +sor b +son tv +so ichiro +so dmg +sn sd +smal en +sic her +she believes +shari ef +sh anthi +salom ons +salam one +rhein metall +ren sen +regulat ory +rc pch +ran dol +ran cocas +power women +papato etoe +oce and +nothing to +nomin ate +nebu las +mom an +mish con +michael kiwanuka +melani stic +me ddled +mckin nie +man sha +malte se +m hy +ly ly +ly cia +loveoz ya +lma oooooooo +le za +lau ch +la boe +kyle hebert +ku cing +kpop fanart +kee so +ke ema +kal uuya +jo d +jim ma +jb laudio +jau zofficial +ja ic +inver kip +integr ative +indi rty +hypercholester olemia +hoo m +hertz berg +herstmon ceux +helle borus +hau ght +h wee +grou lx +gre gy +grazi ella +graphic art +go bruno +glit tered +gay ness +game tography +ga hhh +foot patrol +food processing +flag stones +femme fatale +fc bb +eun uchs +et cher +er mac +dynasty warriors +double days +diaphrag matic +dear rings +co ing +chou chou +cho ksi +chini ot +castle blayney +bu cherer +brig man +bridesma id +br aless +bo tsford +bo ffo +beth houf +beth behrs +bbc thevoiceuk +baw l +base uk +ba sir +avoy ages +assi me +animal league +amit ri +affl icts +ad ded +ac yl +.... .( +ðŁĺ» ðŁĺ½ +ðŁĸ į +ðŁĴ« # +ðĿĻ ŀ +âĿ£ï¸ı âĿ£ï¸ıâĿ£ï¸ı +âĻ¥ ' +âĻ Ĥ +س ÙĬ +z enda +z ard +win ema +win drush +whites ands +west port +wash stand +wa haha +vander cook +umen yi +uc can +transport news +todayin moviehistory +tod dr +to kel +terry androb +swind led +sw oll +super jail +sty mon +stou dt +star power +spir iting +sol ares +smo kies +skip ton +signi fier +si or +sho lom +shaw bury +rtn ba +ren as +re awakened +rdeye girl +ranvir shorey +rais ina +ra on +quality assurance +qu agli +q pf +py rac +pu sser +prince charles +pra yuth +power trains +plagi arist +par wan +pap uan +out stripping +ot itis +or ugby +open cl +ol sen +officiale gl +o sus +o brador +nstom ar +nor bert +non compliance +nic hole +nat pe +nam usic +my dream +mv v +musketeer seurope +mor u +ml rs +miy awaki +melissamc bride +md ant +mc com +margotwall strom +mace wen +london artfair +li bb +lec ito +le then +lb ma +ksn news +kr b +kir choff +joh ri +jay araman +jam ul +iz abela +inter connected +ingrat itude +inew snow +ik onic +hy lan +house breaking +hard worker +han go +ha ston +go wild +glo ves +gie thoorn +fried richs +freshman advice +football news +fetch am +fau recia +farm boy +fan chant +ey f +eric acampbell +em me +eh sa +egoti sm +dot tir +design indaba +deep kaur +cro ssett +cps reds +cou sens +cosmic consciousness +con ine +cleliam ussari +chin ensis +chee ch +ch iri +ch aco +cast ler +care bears +cap ella +busine ssi +brack ins +bor ic +belle mare +beati fied +bal ke +bak re +bad y +atta way +astor ga +aspe cies +arsen ess +ari ff +angel ito +amiz han +after image +ðŁĩ¬ðŁĩ§ ðŁĩºðŁĩ¸ +ðĿĹ µ +à¸Ńภ¥ +à¤Ń à¤Ĺ +zy ı +zaf ra +you thre +x ux +x team +whereyou live +wex med +well l +walk highlands +vla ams +vienne tta +uro logic +une qually +udemy coupon +u gur +twitter takeover +trethe wey +tre me +tororo sso +ti sl +think like +the cab +th feb +tain ting +spo hn +spho enix +sp cs +soup kitchen +snod land +smith h +slo dge +sin thenfl +si dor +shoes ource +seg mental +sea weeds +ryth me +rothschil ds +rejuven ates +redress al +read yourworld +re solve +re med +randolph harris +proprio direct +poles itter +play style +pe ming +pdx music +pal aro +oz ora +oyin bo +oni official +omnis cience +newcastle herald +ner diest +national parkcity +n acre +min era +melani escro +mam etz +magi stral +live better +line age +li even +lau l +lab one +kentish town +kas o +jal pa +iran air +inter dental +ing time +ind welling +imple menter +heal ty +ham idou +hachim an +graf ana +grab ner +ghm conline +ga un +fre se +fre enas +fr yar +faw cett +faroo qui +ex hort +espn seattle +ep aul +ent is +ennis more +enf j +enantio selective +disen franchise +dik ko +devo to +dev camp +des jar +daniel agger +cran borne +con tends +cob ley +clun kers +cincy childrens +chrissy costanza +ce aser +cau dwell +bulgar iofficial +bu har +bu ettner +bow ler +boots riley +bi aly +bhi da +bha sk +be zier +basse terre +bac co +as kia +aro adshow +annex ing +and ys +amar jeet +am cor +al stott +aj green +against trump +afri end +ðŁĽ Ģ +íģ¬ 리 +âľĬðŁı¾ âľĬðŁı¾ +âĸªï¸ı âĸªï¸ı +áµ į +zen ko +writing prompts +wowo win +wood carver +won line +wh ith +weare james +vs nyj +vortic ity +vhong x +ur sul +uo chester +treasury mog +tra kker +toad flax +tivi dale +tiki barber +tick ner +the bull +teil hard +team got +tash ir +take control +swee zy +survivor cbs +sur lerouge +stra us +stay ner +so ad +silver point +shor thai +sho eracing +scott mgi +scottmgi mple +school pr +sc sd +saw dust +safdar jung +rugged maniac +rudi ger +ri aan +real lisamarie +re doubtable +que rel +pul sen +pul po +process o +pre mratandhan +prag matist +powder ham +peplo e +pe ine +p kane +oul son +op é +ones i +one fc +no dy +nishi oka +naves ink +nationalschool walkout +nar umi +nan oro +musee orsay +mont se +misez surlerouge +mc tiernan +mc elli +mark azi +man tou +mal vina +maketheroad ny +mah in +luc re +lon grich +legionof boom +le zz +lar wood +kum in +ku so +ko diaq +key west +kaz emi +katelyn tarver +k ourt +juli eg +john hurt +jason witten +jam my +jaars veld +infl ame +ii hm +ian ism +hum buckers +hon ble +ho ps +he dy +hair and +gy r +gurum urthy +goal u +gla ube +gin acar +geo coding +gator s +g anim +fre yr +fotogra fi +fotogra f +fil des +fam ines +fac u +extru ding +evil queen +eve myles +eu mundi +emis saries +echofox gg +driving test +despon dency +dec ile +de dal +dan ticat +cran field +cosmopolitan uk +cc ts +care mark +call ington +bur ley +bu uuuu +breakfast show +big gardenbirdwatch +bi ju +berg sabc +bb qing +bacsin szky +b ner +b inning +ashley y +app same +annex es +anat ol +am bre +al anal +akint ola +ahe gao +aflat oxin +af tv +acade me +abu dapest +abi asi +ðŁij½ ðŁij½ +ìľłëħ¸ ìľ¤íĺ¸ +âĿ¤ï¸ı ðŁĻıðŁı½ +âĵ ľ +zimcricke tv +ye ssssssss +ye sh +winston churchill +virtu ality +vap il +ur wa +unra velled +umenyi ora +ul aris +turn back +try somethingnew +tou hy +timb ale +the adelaideoval +than q +taf fair +ta imur +su id +sty gian +storm frank +stoken ewington +squad up +socio cultural +scott moir +saw ston +sarkis sian +sa kal +rival sons +ri velin +ras berry +randomactsof kindnessday +r mg +quality control +qu yen +pro foto +pri sa +porsch esauce +podu machi +pete dun +per vez +pe ir +pave ment +pat cham +pasquale totaro +parklife fest +paras ke +ous sef +ni un +never be +nam as +na ston +n intex +mu kho +mosi mann +modern home +mis singh +mis sel +menin black +meg son +mc cs +maz el +manv sale +mal achi +magnet ite +mac cag +lisar inna +leh tinen +l slofficial +kill ingly +ken suke +kat el +kab al +jol yon +jen net +jack posobiec +ja yo +j rees +iz ar +isha an +iran elections +house ch +hou sings +hol ls +health workers +gta vonline +green eyes +gover ner +gok delivers +gn cc +gi meno +gg ler +gesh wari +gene editing +gay oom +gar ment +g anti +fris bie +fo ad +fil mon +febbra io +fam i +fad den +essence mag +du sek +dread noughts +dor ton +dir rell +desp ising +daw on +damas o +dam bro +cumu lus +crop top +cri ssy +cre mate +contextu alize +coach b +ci ana +chir ality +chester fiel +char lotta +ch atime +cag nes +cab ourg +bu tan +british swimming +book bag +bir bs +big bro +bibliothe ca +bc rich +bate aux +baden horst +ba official +b awi +aur icular +arbuth not +ap assion +ann unziata +an ker +alista iro +ali bhai +ale so +aj et +ahar u +ac ts +. , +- ______ ++ £ +ðŁĺĭ ðŁį´ +ðŁĴĻ ðŁĴķ +ðŁıİ ï¸ı +ðŁį ¡ +ðŁ¤® ðŁ¤® +æĸ°èģ ŀ +âı º +ಠļ +youth power +wick steed +west fiel +wer un +vincent price +vani er +uu uh +ug bu +ucc ello +ther ing +thal afan +th jan +tele m +tast ico +su kha +star day +stand ridge +st ello +st austell +soun dre +sou my +sli berty +sin isa +shill ingford +she sh +shak ila +selet ar +secur itas +schmel ing +sau lo +roger craigsmith +ri obamba +revi vals +regal os +reas ures +rapha els +q erim +publi sh +pi got +phi bes +pe muda +pavlo vian +pau lam +over ran +ontari an +of o +occi dent +ny tt +nintendo ds +newpor trfc +neko atsume +nd as +multi role +mr cp +mo preps +metaboli ze +meccan ica +mccre esh +material design +maqu illa +mad bum +ma bou +m sleg +lolli pop +letoy aluckett +leaf ing +lady vols +l dap +key dets +kevin saunderson +kesh et +kentuc kiana +kartika aryan +karak orum +k sis +k itting +john mellencamp +jo leon +jess ore +jay shree +itstaylor yall +is at +invulner able +inoc ente +ingen io +i ys +human os +hot ti +hive mind +high am +hi fk +hall er +go local +gauth am +future learn +fun dingh +fr n +forthroad bridge +finn art +er vices +er obinson +enthr onement +ent wick +end om +earth skyscience +dug gee +drar aut +don thug +dj mo +dis aggregated +dhy ana +dhau la +demar ai +decep tions +dayof giving +dance wear +cryp sis +common ground +co digo +city pgh +chin cha +chican ery +cat ala +carolin amud +carolinamud cats +cal ex +cac ao +c vw +bulgar ians +brooke henderson +broad mead +bois set +blob fish +bing aman +bbc sportsound +bau douin +bal un +ba ws +av aris +auditi onees +at vi +at ena +aravindha sameth +arac ely +apol icy +anthro pome +andy mientus +and all +am blin +agricul tura +ado or +ac nes +above ground +# % +! âļ¾ï¸ı +ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ðŁļ¨ +ðŁĺĺ " +å µ +âĪ ļ +Å ¾ +á e +wustl med +wil ken +wel burn +wal lowa +vra iment +var num +ur j +um be +turtlen ecks +trump budget +tri pa +trape zius +tingu ely +time splitters +thisisd urham +the hip +te sori +tb sofficial +tachi kawa +syn tactic +syn ge +sweet land +su mon +sten ting +sober ly +si val +shop if +shields fc +shield maiden +seren issima +seish un +secre tos +sci acca +scand y +sa uro +s diner +ron johnson +rep kevin +rear guard +real politik +peter greste +pet ard +pamban sa +p mik +osh park +oneoh trix +one ofus +nx umalo +nw l +northant sccc +no war +no rell +no ire +ni mble +neg ley +ne sian +my nah +mwa haha +musicis mylife +modern day +mo zar +mo har +mlb pa +mind q +mic mac +mf is +metho trexate +mari ane +m frost +len zie +lari ver +ky lar +kut ter +knock aert +ki shin +kak ai +ji ah +jel utong +it carlow +iron monger +il ga +iconocla sm +hen ery +hell spawn +haworth ia +har bi +ham bly +hail u +gyeong ju +gra ef +goooooo ood +fom ent +fo glia +fel ino +fam oso +ey vind +exorc ising +epi thets +elli son +electrocu ting +elas mo +e hehe +dou rif +do xford +di bang +de mentor +cotedazur now +con rail +compar ator +colson whitehead +cat alu +care en +camer ino +bur se +bry z +breit ner +bledis lo +bla ise +biome thane +ba iser +b amp +aver il +ambassador power +all mets +al acrosse +ak utagawa +abigail spencer +ab dn +// : +ðŁĺłðŁĺł ðŁĺł +ðŁijijðŁijij ðŁijijðŁijij +ëĭ ĺ +æ£ ® +âĿ ¦ +è dre +yan u +xma sparty +x lu +wy prk +wg st +western sydney +wak ana +w tam +vizi anagaram +vers ini +vander waal +tunkhan nock +toyn bee +tie out +te phra +sy nucle +sy mmes +sun care +sub in +sub contracting +stre ssed +stopthe debttrap +ss w +sper m +spee die +soci ability +small youtubers +sm x +skan sk +sinu so +shri mper +sheff council +seh ban +samaritan spurse +salish sea +sal ted +s jones +rid out +red bourn +ram shaw +predic aments +pn pi +plo ys +pitch forks +pet tai +pen ampang +pajar ito +otter burn +ot ice +oro ss +one ills +nieu wen +mr b +mo esha +mmmm mmmm +mat us +ma homie +louden swain +lipo somal +lal af +lag ana +la vette +ko bler +king and +khay al +kh ri +kat wijk +kai sen +jun ia +jelly man +jeff bullas +jash n +iri sd +ingh a +hire me +hey sel +helm sdale +hake em +haber mas +h ounding +gregg rosenthal +gary leff +garden centre +foto g +forza italia +fibro blast +fell running +fee ley +fe k +eve of +evangel inel +ero deo +er tiga +elo gic +elly awards +elef ther +eg shore +edward tufte +ecol lector +ebon ics +east nor +dungeon master +dragon ite +dig in +dhol akia +dev day +dental hygiene +defro ster +dataware house +dam avand +dal ers +cu ppy +cu entas +crew mate +colon ising +code foramerica +clip stone +citiess ky +ci at +cheese cake +cdn crown +candel ario +bunnic ula +bron wyn +bra edon +boi leau +ban co +bal alaika +attic arace +atticarace wyprk +ary newsofficial +angelsof fur +and h +an ick +amera action +alli ums +ali assime +ac rif +ðŁĺĭ ðŁĺĤ +ðŁĺĤðŁĺĤ ðŁĺ© +ðŁĴľ âĿ¤ +ðŁĴĻ ðŁĴĸ +ðŁ¤ĵ # +ëĿ¼ìĿ´ íĬ¸ +âĿ¤ï¸ı ðŁĺĩ +âĨ ĺ +Ú ¡ +wom e +wc pd +wall onne +w le +ver ti +vel ia +v ung +urdan eta +un likeable +u mia +tur l +trail head +toyo tan +the oph +tarot cards +tanan lam +su hoday +steely dan +st century +sport sturf +spin offs +sphero id +sper ry +spartans will +smo yer +sk j +sion yc +sch latter +sat am +san jac +roman ian +reminiscen ces +re animator +raw ling +ra tho +priyadar shan +prabha karan +po rec +pi tha +peoplewhom ademy +par minder +p liz +p ities +onem illion +off y +noril sk +nor rington +ne tti +ne ma +nav ys +national aviationday +mcne aly +mauriciom acri +ma ssing +little dragon +liss at +lin go +lead byexample +le ix +lar ia +l bo +ko tha +kho ya +khan om +kaz en +k radio +jyr ki +juma anewilliams +joe mantegna +ji v +its ellacruz +it trivedi +ipp olita +ic tsi +hoch stein +hipho pawards +grow ur +grac iosa +gloom haven +gag an +fore seeing +fi lets +feature less +fa ial +eviltwin brewing +er au +ei rene +edge mere +ed surge +e are +dracon is +downtown ptbo +dj clue +dine o +dem at +del tron +decrimin alized +dante basco +crou ches +cra shed +cr inan +counter acts +contest able +cnblue gt +citrul line +christo logy +chris bosh +chas ms +caring bah +car ll +bur rage +bru ticus +boxer vijender +bo water +bo letus +black enterprise +bi asi +bear sden +band ha +baby go +b br +arvindg aur +arrivat w +ar asan +apic ulture +ant y +ali zafar +ali stas +alex constancio +al imi +ajin omoto +air fields +acci on +abar are +aar ohi +a preci +... âłĢ +ðŁĺģ âľĮ +ðŁĺ½ ðŁĺ½ +ðŁ¤£ðŁ¤£ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ +æŃ £ +ãĤ¤ãĥ « +ãģ¨ ãģĨ +yyyy yyyyyy +woo duk +wi bowo +wh on +warcraft movie +voter suppression +tud denham +truck n +trav ails +tom ska +timm is +the wiggles +the beauty +terr r +tal kis +ta ja +stu l +star flyer +stam u +stalag mite +st aley +ssi de +ss ongs +sp ahr +slow downs +shil don +shi rayuki +sexu alised +scul ly +sch l +sar re +ru pal +rn zaf +redhour ben +race ways +ra ichu +queen sof +que te +promo cional +premi xed +practic als +plan ica +ph rom +paradi so +p mt +over stepped +or loff +nz ta +na an +mplo yers +mosthandsome faces +mo edas +mis dproud +mey cauayan +mc vicker +matt kemp +mak ura +magic ofthecup +maci ek +love actually +lipo ic +li mber +levi mitchell +lake house +la dan +l bci +kul in +kor net +knu x +kentu ck +kab inett +ka strup +jun hong +jone ss +ji yala +jak el +jaimie alexander +j ats +ipp r +in may +in ji +il ja +ic tafrica +hy bpa +hour fitness +hoh mann +hare woodhouse +h lt +gro win +gramophon emag +graceand frankie +glo ttis +gigan det +gic lée +ger ri +gcse results +games aus +ga shes +funny pictures +fron tieres +friday morning +fo ard +fit i +fish mas +fi stral +fc zenit +event sin +esp anya +emporio armani +el ene +e comm +dre mil +don no +dof theweek +do ye +do che +dh anya +dela hunty +decarbon ization +dd avis +dcyoung fly +corruption case +commerce gov +co darmy +co creation +chi d +cf m +cest lavie +britanni c +body suits +boc cioni +be evers +be eni +bbc shropshire +bal aam +bad stuber +aspr illa +arth us +annam arie +animal liberation +alistairo vereem +ab attle +ðŁĶ¸ ðŁĶ¹ +ðŁĨĺ # +ç¾ İ +wgn america +west van +wad don +wad den +vrou wen +victor ial +valeri y +valen za +v rush +v inter +un ti +u hs +tx ts +ttm success +tre garon +tre blinka +train able +thisisco ventry +th acker +t pv +t pas +sustainable finance +string band +spen son +sm day +sk ole +sham bhu +sf ontein +seong woo +se sam +scol ts +sanc ta +sa ire +ross marquand +renew timeless +red hood +ramyak rishnan +quadru ped +publ ically +pra ya +petro ssian +perfu mer +p ccw +ourlandour news +nwa as +nor aen +n anne +myk ki +mykki blanco +musc led +morgan spurlock +monclo a +mm une +miti e +michael h +michael b +metv batman +meer schaum +marcu scooks +marak wet +m wbb +long niddry +live updates +leader less +lan phier +l xc +kup fer +kre utz +kev adamsss +karnataka world +k pae +ju ku +ji ddu +jday golf +jan ka +j ies +hye res +hu sni +hollow knight +ho yne +head house +har laxton +gym wear +gul ates +groom bridge +global britain +gh ita +gere ja +geophy sicist +geno ise +exop lane +eco sphere +early learning +dre we +direc tness +digit als +denti stry +dell matchplay +dark skies +cv payne +counter tenor +coun tri +costume design +const ancy +cn try +cityo fatlanta +chukw uma +cheshire police +cele stino +car freeday +cad dick +c dos +bul losa +bravo andy +bramb illa +boz os +bosw ellia +borough fc +boad icea +bo soms +biz boost +bau hin +ba ars +b sl +avi dson +au glaize +attend ance +asdru bal +ar ar +apar napkin +ap enn +all ornothing +air plan +afl ori +adi m +ab last +aaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaa +:) âĻ¥ +ðŁĺį ðŁijĮðŁı¼ +ðŁijĮ " +ðŁIJ° ðŁIJ°ðŁIJ° +ав ÑĤ +î t +z ior +yon ex +yo ver +yestur day +wun sch +won o +wl bz +web casts +warner music +war re +wale ed +wag i +vesp asian +veliyi dai +var roa +tro ilus +traffic scotland +tmc lebanon +tla que +this ssss +tenov usc +tar nishing +stu dly +star tet +sleigh s +silk wood +sil ves +shiv puri +scr unch +s va +realbobby roode +real sarathkumar +rd mas +race for +r ci +pride fest +podumachi goalu +po theads +pnpi fugao +play backs +plane tor +patr onal +party yy +pale tta +p ssi +os sett +oce ph +neopla sms +na ic +n dy +mut tered +morri ston +min cha +mi igwe +mh day +men ai +mang ler +mah mu +madri distas +ma ham +little port +kha pa +kaz aam +jis r +jimmy bullard +jen ners +jan ia +jagann atha +in cs +il ir +iklan barison +ike auk +i ak +hypere mesis +hu maim +hotel belair +hot chner +hiphop history +hijab day +hepat ica +harvey fierstein +hampton court +hammer down +habit ants +h nn +grand hotel +gra eae +gr ing +ger rish +gab onese +fur freefriday +fss ai +folk song +fo amped +fil mc +fdm group +faw zi +es wari +ed leaders +dutch bros +dis contents +descu bre +der abad +depresse dd +dat ass +da et +cysyll te +craig lockhart +cot ter +com ba +coast walk +chis um +chees elo +chaf in +cha sti +bus er +broad institute +bre snan +bon nici +bol año +bo vines +bledislo ecup +bl ang +bharat natyam +ben it +beabin ene +bar bato +asi onal +areth usa +ar tos +allgä u +ag p +acu ba +* ---- +ðŁĺį ðŁĻĬ +ðŁIJ¶ : +ðŁį ¶ +ìłľ ëĭĪ +âĢĶâĢĶâĢĶâĢĶ âĢĶ +á Ĵ +ਠķ +zay ousaf +youth month +youn kers +y po +y enko +x liii +wigw ams +wal nu +wa aaa +viaduc ts +verkho vna +un principled +typo logies +tor telli +tin d +there stless +thecottag eneedle +the aus +tem pu +taye diggs +tam ana +take five +stry pes +stra ddled +ste geman +st chat +springer nature +sper mato +speed weeks +sk otti +shu gart +sfor good +sehban azim +scoti abank +sch an +scal pels +sc id +sas c +saf c +s music +ru pauls +restor an +razor bill +r mef +purch asable +pu cker +poly hedral +pimi enta +pen arth +paul smith +parachu ted +paci fics +pa wel +ov hd +outd channel +op tout +o ep +novel isation +north fork +noraen pure +nicholas sparks +nevel son +nay sayer +money talks +mi mir +megal o +maz da +marke tability +looking ood +london grammar +lland y +like that +lef son +la gta +l les +korbolorbo jeetbo +kho isan +kam andi +jettison ed +jefferson ian +ittake s +itso knotto +isth mian +im cc +idec tomy +id sa +hoo vering +hitch hiked +her rings +health ug +hand cut +han ap +hammer ton +h ph +h kr +gujaratt ourism +gre u +gla uber +gho sh +gar madon +future past +flo ve +fleet week +fire bombing +ene mas +en ow +ele men +eg eland +ed wi +east ney +east dulwich +drag ana +dj mustard +deep am +d ın +cre ag +cor owa +chuck comeau +chop shop +chloe fashion +catal pa +cas a +cal ac +bun tin +bree zed +boho jewelry +boer sma +bl ine +big gies +bi joy +belen enses +bat themusical +bar ito +balla gha +bal ala +asu ppor +arbor io +ano les +annalyn ne +ame dia +ambl yo +amazon primeday +allegh eny +all mendinger +all black +aaf london +:' ))) +Ī : +ðŁĺĪ ðŁĺĪðŁĺĪðŁĺĪ +ðŁĮ ©ï¸ı +ë³ µ +åº ĥ +âĻ« âĻ© +york cityfc +yemi aladee +x au +wiv pak +win ward +willi ford +whiti anga +wee ping +warriors ground +ver sus +v ly +up votes +tra verses +town head +tour ne +top up +ti money +tho ver +thewe bbyawards +theatre works +thau sen +tb ath +taxi ed +state ivlp +spring fashion +speed ed +social mobility +slur pees +si regar +shiv angi +shawne merriman +sepan x +sch aut +sat ar +sand strom +san bernadino +rose bruford +ri bisi +rhetor ics +retail therapy +requ ena +relax in +raji b +preten sions +pre ponder +pois oner +pis mo +pen tire +par olin +p crg +ony m +offer tory +ode tta +ob ando +not chback +normali zes +norfolk county +nkn kk +neph rectomy +mymorning jacket +mirpur khas +mani than +mal indo +liber ato +leeu win +later alus +lar ries +kirk up +kinder garden +khawar ij +ker kyra +k rac +juli ana +jone z +jax jones +jaket austin +itsti meto +is thebest +ingh all +ine ssa +in oo +illi gan +ii ia +i fr +hood wink +hide outs +hel enium +heck uva +health summit +hand saw +gene v +gan tries +ga ily +ful kerson +fro llo +for goes +for ages +flic omovies +first love +feder man +fc groningen +farmers guardian +eru dition +els mann +el wick +eichel berger +e online +drawing aday +dot day +dock weiler +dit v +dak en +dah le +cy farth +comhghair deas +com hal +cataly se +caer laverock +bottom sup +bobble head +bo hot +bir gitte +bien al +awe bb +avon dale +arte san +arra ial +ar bel +andrew mcmahon +an tron +an el +al let +ai zu +ad mir +ad av +ab ao +ðŁĺĶ ðŁĺ¢ +ðŁĺĪ ðŁĺĤ +ðŁİ¬ ðŁİ¬ +ëįĶ ìĩ¼ +åĩºæ¼ Ķ +ãĢ Ī +zim toti +y cp +wood burning +who weare +waller racing +vijay goelbjp +valle es +usab mnt +universit elaval +ultra wide +ttan dem +translu cency +tori bio +to phobia +tist mgmt +then ana +thecalm zone +the guy +the chri +ter ming +ten napel +team sesh +tai b +sve ti +sur u +sugar daddy +steel making +sta ver +shirt friday +ship week +shaw ns +sel v +salondu bourget +sa kari +running man +ro fe +revolu t +regg ina +recon dite +rc vd +rav allo +pr ar +poly morph +polar plunge +pla intext +pi shin +peter ose +pen ola +p lit +oro sso +one music +off white +ny penn +no tion +ne jad +nc soft +music mondays +moet blindcat +mid sized +mi rian +mer cure +mcgiv ney +mb lue +man ka +make music +mait lis +m ó +ly cée +luci dum +lu minor +lo pes +line less +larry blustein +lab neh +la galaxy +l tf +kur une +krat on +kra it +kos gei +kno we +king angi +kil lester +kaw agoe +jungfrau region +jon culshaw +joe walsh +jin soul +jennie garth +j érôme +irk some +ic in +human ization +hu mph +hoge school +herom anoj +he mam +han sal +ha wala +ha voline +gz chef +great fashionfinds +glo scathedral +getting married +ge tenough +fundra ising +free h +event inglive +escal era +eri eotters +dul cinea +dor bz +dong daemun +demago gues +demago guery +dallas lovefield +da ai +crunch ies +conservator ship +co ple +cla es +cees ay +bur kle +bur kitt +big pond +bi mb +bell one +beer bods +be ghe +baltimore county +backward ness +at le +ascle pius +as ger +angel olsen +anc re +allo ween +ai h +ace attorney +! ) +ðŁĴķ ðŁijŃ +çŁ ¥ +ر د +á ras +zoo svictoria +zo ster +ye she +west la +west elm +welove shilpashinde +vol vulus +vick erman +ur ge +un modified +twit ts +tweetad run +tu bac +trac coon +to kill +thegold bergsabc +tar oko +tan x +tal ton +summerof love +spo oling +so ss +silver stream +shari fah +rosen dahl +roman jancic +re ste +raphaels barge +q k +pur merend +public protector +privacy matters +po ggi +phant asma +pe eth +padilla bela +ot acon +olympic park +olu femi +ol ap +ojh lofficial +ness on +nam eh +n cart +mutual fund +mo sso +mitsu ko +missing people +memo irist +man ick +magne tometer +mag ination +live ira +lets do +leather craft +l enti +kumb hal +kon yaspor +king span +kay lyn +jon lovett +joer ger +janee spenson +iso ara +is k +iq ra +ing re +in coherence +hoshi arpur +horror fans +homoe opathic +hen rys +ha val +gul ley +gre ferendum +grace less +grace e +gra spop +girlsnot brides +ger main +fro mt +fr wy +foodand cosplay +fiel dy +fa hn +end ro +elvis duran +draw dinovember +double think +do sen +dl hughley +deline ate +def ene +de sam +davi dre +d top +cus rise +county sheriff +cor ts +cor sican +congradu lations +con ch +collo ids +col lon +co soc +cleveland artmuseum +circum polar +chy pre +chris martin +cherly chibi +chan in +cartm ell +car hop +canvas print +bra es +bobm ckenzie +bob marley +be ville +baby animals +bab ic +aw ade +au byn +arm let +ar dyn +al tc +?! ?!" +ðŁĶ¥ ðŁĺĪ +ðŁĴķ ðŁĴĻ +ðŁijĮðŁı¼ ðŁijĮðŁı¼ðŁijĮðŁı¼ +ðŁıįï¸ı ðŁıİï¸ı +é¾ į +âĢ ķ +á´ ı +xy mox +x lk +wood smen +wi fu +vis ity +vel vets +vallab h +valent a +v pm +twit pics +tut ti +the sheep +the edge +tand ing +stur div +stan stead +ss wans +southern california +south point +sil ento +shinse gae +shen yue +sa che +s but +ryu jin +rivu let +ripon cathedral +rep lika +rel ph +red mill +rc cs +qu ast +q wp +pro ffer +preston steve +pr yer +power grid +postu late +porto fla +po styour +po ble +pend rive +pal oo +over staying +osteo spermum +non smoker +no son +nithyam enen +nick diaz +new chapter +nav aho +moz con +mortal instruments +mongo loid +mini mathur +mene my +memori alizes +mc cu +max lucado +ma sai +low carbon +long more +l vac +konta veit +kenny g +kawar than +janet varney +inter gender +instagram mable +inge agles +ine ducation +hebrew u +heat culture +harde eville +har tal +hack tivism +haber dasher +green book +gran at +gom avs +glend ening +free from +foto speed +fh g +ffbe ww +fe iner +en dia +ela sto +ef eld +edexcel maths +ea ina +duba it +du gin +diferen cia +dic ted +dec icco +dance hall +cé cile +cuyam aca +cu bi +cru achan +corri eri +com ent +co enen +chen kova +cay de +by all +bur ro +bron son +blue october +bis leri +birdof prey +bio steel +bil kent +bad er +au sk +ast ellas +asian art +asi r +as aa +app ended +andyburnham gm +an diamo +all ard +ale urope +albic ans +afternoon express +ab ms +ab arab +ðŁĺµ ðŁĺµðŁĺµ +ðŁijıðŁijıðŁijıðŁijı ðŁijıðŁijıðŁijıðŁijı +íĦ ° +ë² Ħë +ÙħÛĮ Úº +Ø µ +|âĹ Ĺ +zumi ez +zu elo +zin aida +yuki hiro +yu qi +yajam ana +wood in +winthe dark +wave music +von leh +var de +v ram +uw sp +un thinking +un defeat +tram el +track less +tlaque paque +tion ately +threeday sgrace +thedukeof york +theater shooting +tex ter +tahqu itz +sylve stris +sustainab lec +spir alled +sock game +so di +sk imp +si ii +shot ley +shor ta +sh games +seal skin +sco d +sap hire +sant inof +sam us +sad vocacy +ry sz +ry hope +rela yed +red point +ray hudson +rainbow fish +ra gg +q x +pu zha +pr ange +po ile +ple ssy +play it +penn sville +pen nin +par ijat +p ts +osu wexmed +om ed +ole v +ol faction +odd balls +obli vi +oak engates +noo tka +nnnn nnn +newh ope +nar da +mort decai +mor sy +mor rice +money inthe +mer ca +melaniescro fano +me rec +mawdd ach +mat ley +masji ds +mary ann +manik andan +m pho +lou l +litt let +legally blonde +langu r +lam acq +kri pa +kount ze +kk as +kem merer +kel sen +kc mo +ka ichi +judge ship +jo sse +jame shet +ir ally +iphonex smax +io hk +inconspic uously +hum zayousaf +help us +he tch +hadley wickham +gold mark +go iter +global music +gh um +gau din +fro de +for me +fioren tini +fastn loud +fa im +ee ight +dine sen +di emon +defibrill ation +de vis +dare rising +d pu +cwre ign +curb you +croco dile +counter insurgency +cocoro cha +chuk chi +chil cotin +cf cs +ce berano +carb ines +car dew +captain hook +buck tail +bro se +bre gat +bra sch +blue day +berlin ers +bene field +bar mby +ascri bed +arnold sports +ar rings +angou leme +andy c +amo sun +aman zimtoti +al britton +aj la +adair sville +acre ative +a beach +_ < +. * +ðŁĴģ ðŁĴķ +åĵ ¡ +à¸Ħ à¸Ļภ+zamalek sc +yearend sale +x vid +world populationday +whi story +wen z +we got +watkin sville +wak awaka +votol atino +venew shoes +ve iga +v amovie +uk baseball +ud murt +tw ane +tre sor +then ff +the writer +the andy +ten ille +techno gym +tag oe +sun ne +stu bai +sto ically +squig gly +spreck els +speech writing +spayand neuter +shon ours +sharman joshi +semic ircle +seed and +scre es +scott stapp +school sweek +row ley +ring side +reli e +relent les +rb cs +rand olf +quercu sbooks +public history +pu tte +proudly southafrican +pol lin +pod genie +pli moth +phy no +oligo poly +oil price +of qual +nick swisher +ne vik +nay ana +mis si +michaelberry sho +mccul ley +map ada +man ische +man er +mai er +magin ot +mag das +lun ney +li zation +li ong +lewis and +len sed +le hrman +le fort +kurune gala +kof u +ker lin +ken ingau +kar ap +justi fied +jhope day +jay co +ir ama +infra sound +impact live +illi p +i spo +humaim amalick +hei ji +hahaha aa +hage mann +h unie +guys borough +greenvill enews +godre j +gen tiana +gautamrode team +g burg +friday flashback +franken heimer +fo ibles +femto second +esche ws +epoch times +ek hu +ei en +dos box +disney d +diar yo +di methyl +deep ellum +debla sionyc +debat er +dar kangel +dar cis +dan dekar +dan abrams +da hir +cull in +cri velli +cour cy +cor tel +chrisley knowsbest +ce dro +catter all +brad burn +bol dinsider +bigbrother ca +bic ameral +ben tiu +beforeyou exit +b fly +b alian +army tage +arjun official +anyan wu +ameli o +alo vely +ak arjunofficial +aj pur +ah w +acon cert +aadhi official +ðŁļ¶ âĢįâĻĢï¸ı +ðŁĴĻðŁĴļ ðŁĴĽ +åĥ ı +âĹķ ) +à®İ ன +yu ill +yha official +wimbledon final +wc ba +water logging +vo lim +vampire academy +uru zgan +un drip +thum or +suppos itories +stopthe violence +starwar sep +spe ttac +spati o +space shuttle +sono ita +somo sporto +so hyun +sneaker snstuff +slee man +sko vic +sin spired +sil vassa +si vers +showaddy waddy +sh ingen +sen ri +sco tathletics +ru cian +research impac +realjoey b +re ema +re ath +ralph macchio +rail head +pre ller +pre ggers +pottawat omie +pick oftheweek +peter sson +pas es +paris motorshow +over time +ogo pogo +nw lc +ni da +nau i +msf tedu +mr g +morethan agame +mic i +mccor mack +math letes +marriage equ +lur kin +love bts +look good +lo ben +liversi dge +line arly +le eu +lamar re +kre we +kop ing +ko gi +kn abe +khu tba +ken nys +kas sab +k vapil +k rates +joel creasey +ji ocinema +inv ar +intu itive +inad missible +impregn ates +ilo gical +iced tea +iam abotanist +holiday home +ho de +hard tail +gun barrel +green music +gre vy +goooo o +gas cony +flix bus +fatin sl +far uq +evi leye +esz ter +ent eng +enni stymon +el itch +dre ier +dranath tagore +do ddington +disdain ful +digitali sierung +di af +dest iney +del sin +de itsch +de code +data storage +d pb +cor dle +congression al +con script +community shield +commis so +clo t +bud worth +bor ovets +book sin +blue throat +blackwomen didthat +bir atnagar +bernar dsville +ber chem +beau chemin +be ppu +batmanvs superman +baseball canada +backthe pack +back tracked +ba ard +annamari abiasi +angel alan +an iche +ame v +ambigu ities +alek sei +akar ni +ahs freakshow +ah ack +acar thy +_ ^ +[ +] +ðŁĵļðŁĵļ ðŁĵļ +ðŁijĢðŁijĢ ðŁijĢðŁijĢ +ðŁ¤£ðŁ¤£ ðŁ¤£ +ë¡ľ ìłľ +âĺĨâĺĨ âĺĨâĺĨâĺĨ +âĢ¦ âĢ¦.. +âĢ¢Ì ģ) +youare loved +ye ds +x pl +wr oughton +waiting for +w luk +vic h +val halla +v tv +usopen cup +un sorted +uk volkswagen +turbo chargers +trout dale +ton na +to cco +timo thee +the bone +team ceec +suj atha +sto renvy +stje pan +sti pes +steen burgen +stay gold +ssa ints +sou tar +sno whour +sm sp +sli ghted +skotti eyoung +she sa +sha ab +seo inguk +seab ikes +se yi +screen sho +sant elli +sagrad afamilia +sabhar wal +ros set +regre ssions +proto star +pricewaterhouse coopers +pho bla +pele liu +pegas i +parathy ro +oo hhh +ong ata +oneon one +oli fants +nkem diche +need leman +my writings +my my +ms b +mr inal +mou f +molly mauk +mmun ol +mirren fc +min ichiello +milit are +mic cosu +metro tech +meridi ana +mee ce +medi ations +mead er +manu ka +maith ili +maccab iah +luch alibre +laurel schuett +lam lash +l ri +kripp arrian +kore aboo +kl n +ke phart +kang an +jazz dotorg +jay r +jam mer +hoch schule +heure use +headline pg +har mos +gre edy +gott aget +go bowling +geode sy +gam mas +ga ited +frontiers man +fish back +fair brother +eval yn +euro bond +esch aton +emal ick +el via +ehr mann +ed fu +ec le +ear gasm +dimetro don +crimesof grindelwald +coyo tes +co zi +chuck y +char ing +cat ul +by rum +buzz kill +bran te +bowhun ter +bor ton +black owned +be urope +ba sim +ba ic +atic i +ate en +associ ati +archit rave +aracelyar ambula +appar at +ankylo saurus +amar sh +am rish +all ari +al tin +al thorp +air train +ðŁĺįðŁĺįðŁĺį @ +ðŁİĤ ðŁİīðŁİģ +ðŁ¤ŀ ðŁı¾ +âĺº # +à° ® +Äį a +zen imax +yl en +with thebest +wh ood +west de +wego tem +web apps +wal czak +w mw +ut martin +under desk +un does +ug al +u bp +tweetapictureof your +tun ning +tsn bobmckenzie +to ji +the blue +tender loins +ten ures +teitel baum +sug awara +streat ley +strabis mus +str yn +squee zy +spec tre +sp afford +south afric +slay age +sil kin +sie ben +si sse +sars gaard +rule set +ro eper +rich wood +read yyyy +re ges +raw materials +ram bis +ral f +rac q +pra dy +pp cli +pon yo +philosophi zing +phil by +ph ron +peter hickman +petedun ney +petedunney xb +pend ers +p ée +o herty +nott ur +nic ee +nh week +ner vou +n br +mou l +melo che +mcgu ckin +mat tre +marie fre +mar sone +mal practices +lucas digrassi +lightning network +leg ged +leg are +la reunion +l bh +kol ton +knotts berryfarm +keepingit real +kai sha +join me +jo of +jam tour +ja ia +j collins +iwak uni +ish peming +is af +invision app +infe stations +huy ghe +home inspection +heil tsuk +hat te +greatyork show +gre sini +gram marian +gor os +good karma +golden child +goeth als +german e +garrin cha +fu jit +for tb +fla vian +fine ssed +fair funding +f ng +ey non +er hu +economic growth +e he +dive sted +dinner tonight +dia thecat +dd ya +das u +cultiv ate +com ox +college colors +cnc pts +clean in +claw diathecat +chekkachi van +celest ite +can tal +c engage +bull fights +buck lin +bronco sports +bot ello +bird softwitter +be hr +basile us +b nu +azu buike +ay al +aw con +aviation geek +athletics weekly +ashken azy +arro wheads +ar bon +ar boleda +ar bogast +am artinez +am ap +alu so +alten ango +allo graft +al bie +aege an +admoni shes +( ãĥ» +ðŁĺĦ ðŁĴķ +ðŁĺĤ ðŁĺħ +ðŁĺĢ ) +ðŁĶĶ ðŁĶĶ +म न +ÏĦ he +world liness +winnie harlow +wence sla +war ga +wall aroo +vote anc +vogue uk +very play +un ar +toile tek +thenu mbers +tell er +tan on +super ba +struc tur +strength and +spar khill +soular tistmgmt +situ ate +si ón +sheep ish +sexu alization +sen thomtillis +secondary schoolmemories +seam ers +se dimen +schoo ley +san key +s faf +s atti +re zo +re ig +re affirmation +rain sy +rail fans +qual itye +pu rie +prinze ssin +pre peration +plow shares +pla ited +ping ame +peñ arol +peripate tic +penalty rates +part yof +pa olog +orthope dist +orang eroom +od nb +o va +ntv newsnl +nin aag +ninaag dal +ni eva +ncaaw bb +na or +my phone +multi hull +mudge er +mosqu ito +miguel ferrer +mack trucks +macart ney +ma belle +liss itzky +ligab bva +life changing +lazare tto +law fare +land side +la ppy +ko chs +knight swood +kisar agi +ketu rah +ka tho +ju by +josh devin +joshdevin edrums +jack hammers +ja ise +invest ec +infection control +indie pub +i kut +hydroly sis +hu tz +hot and +hen ni +gu mmed +goo b +go flyers +gior gione +geo scientists +fu sz +flat ted +fiest y +fer us +far th +fai roz +excep tion +ent rop +embal se +elfon ashelf +ef w +ec cs +digger land +diffu ses +des sel +deal sof +de dic +cá ceres +cyber ark +cultu red +cryp topsy +consu elos +comi furo +cobal amin +clari dge +carden ales +callip ers +callacu tieout +ca ireland +c xl +c tid +bru cker +broken lizard +bofam l +bb ck +bb capprentice +band la +ban erji +ay han +avalan che +ase va +as ato +artificial inteligence +armedforce s +arant xa +arad hana +ar j +anton opoulos +anor th +allu re +adidas soccer +ðŁĺ« ðŁĺį +ðŁĴµðŁĴµ ðŁĴµðŁĴµ +íķĺ ìĦ±ìļ´ +âĿĦï¸ı âĽĦï¸ı +⬠ľï¸ı +оР± +yu rio +ys by +y ili +y ates +xbox uk +wil ts +we tan +way lon +wat to +voel ker +utt ley +uncas ville +tum se +tu dy +to hono +the pioneerwoman +the misfits +th ag +taw fik +t town +t for +suit elife +st leonards +ss mh +sports radio +sonequ amg +sol ons +sen ility +sen ado +se dinburgh +salv aje +saint es +s belike +rol fing +right towork +reo speedwagon +ra bs +r hh +quik pro +qantas airways +po sed +plom acy +pinto fotografÃŃa +pin ault +pav lovsk +patag oni +pan starrs +os garage +oc d +nue stra +nj sp +naci miento +n ack +mu thi +mu ling +moyamee haa +motor car +moth balled +mortgage broker +mohand as +ming tsai +midnap ore +ment zer +megan amram +maneuver able +man none +mal ing +lu ddy +lex xx +lat ers +laff an +la famili +kwes é +killing me +keep corbyn +kat anas +kam bi +jay wick +itsagreat day +is bt +is awa +ili festyle +iconocla sts +ic osmetics +ho ak +he med +gri ppy +gil kes +fried chicken +fol x +fle urie +five guys +faun af +f pc +f out +er bb +er anow +emo cracy +ed an +e ji +du cote +do oling +discover overberg +digit alliteracy +di eta +delta state +dean morgan +dari ya +cr ys +cou lsdon +consu mp +con signer +co vin +cart land +cakeboss buddy +bu hari +btsloveyourself tour +bru hl +blood stain +bill browder +bell x +awar ner +as pher +as ghar +art daily +argan oil +ar kush +apu lian +apol it +anz alone +andre ak +an sk +an lonsdale +alzheimers disease +alex honnold +al can +af phq +af fine +aci ar +accu satory +____ ___ +! ðŁĻĪ +! ðŁĺħ +ðŁĻĮðŁı½ ðŁĻĮðŁı½ +ðŁĺĵ ðŁĺĵðŁĺĵ +ìļ ´ +ìĹ ´ +âļªï¸ı âļªï¸ı +à®ķ à®® +É Ľ +zer land +z elig +yaa asss +y rn +wester field +wen ye +we play +war ders +visit sweden +vam ani +un icity +tto loso +tro ss +thro mb +teof ilo +teenag efanclub +tc mi +tavit ulle +symboli sed +stron k +staple hurst +stack ers +spas sky +sore head +so suke +skag it +sigmar gabriel +she ild +schur ch +sama atv +road sof +r fu +quit ted +pu san +project mangement +pla x +piercethe vic +pe ery +pan u +ohi sto +officials blessing +of death +o bie +nun nally +nt ate +naomis cott +n dia +mo xie +medit ator +mc crum +maynil ad +mariefre ttoloso +lule Ã¥ +liz otte +lawy ers +kid ap +ke ta +kart ell +k ery +justgo shoot +juic ery +ju ggy +jor jasmith +jmichael tatum +jed york +ix ora +in ata +id one +iam jer +huday dah +hu sam +hooge veen +hitch en +hich ki +guide dog +growur startup +gali za +fault lines +fau det +ever glow +escu char +esc s +elvis es +else vier +e then +dreamtheater net +doc tson +din ara +dil ara +defe atist +czer ny +cw tch +cul leton +cour tin +chur cher +chri stu +chitrang ada +card assian +can am +c wallerracing +by choice +brom ham +brite eye +boon ton +biop sycho +bezan is +basketof deplorables +b flo +auto harp +ap adilla +anthony cumia +an jem +amc talkingdead +al bia +air foil +> ; += )) +ðŁĹ ¿ +ðŁij ľ +ย ย +à¥ĩ à¤Ĥ_ +Ê ĥ +youngand therestless +york town +yearen d +waw ild +vitam ine +v hi +ut pal +uni sex +tur ma +tu pi +trafalgar square +title town +thrap ston +thisi sr +th man +tech update +team sasha +teake ttle +tay miyyah +tan ish +sye da +super nanny +st ent +splay house +sou der +son ger +solo preneur +so bral +sla gging +sivi glia +si min +shri ft +shr tampa +shen long +shamit abh +seri alisation +sen na +se where +scy thes +sat uan +rudy giuliani +ru ly +rosen do +road y +ro sequ +ri ghting +ri eti +ri dis +rhu le +retro horror +rel ents +r mbr +pun kie +pul ped +powderham castle +pou f +pot es +ph ritis +out moded +om et +nomin ally +no sler +no sleep +ne za +nau t +nap es +na hai +mystic seaport +mou stach +mj keenan +mistransl ation +miamidade county +megan tic +lun aleso +lev asseur +kin ny +ka atru +jk tourism +james franco +inten se +ingh ot +ilu stration +ili v +ij t +hub bucket +hry v +home tips +heli um +heb ner +gu tu +gle w +g jr +ford india +fire arm +ff c +fcau gsburg +er ste +ele e +e consultancy +du bay +dramati ze +dec red +de metris +dance co +dae kim +ctv winnipeg +cru ijff +cor mick +complain ants +com ico +cj spiller +ci oran +chain rings +broward county +belgis che +bel va +barat aria +bam av +bal lasts +bad luck +b mus +ashley mcbryde +ash down +ash bridge +arkan sa +ar oon +anna beth +ad dai +a bear +======== ==== +ðŁļ İ +ðŁĺį ðŁĴį +ðŁĴļ . +ðĿĺ Ģ +ãĥĹãĥŃ ãĥ¬ãĤ¹ +¬ ´ +zi ad +yu v +world t +weare mumbai +wast aken +wand bc +voxel art +vis ayan +vi bha +vau ban +upper deck +tur nhout +tur gid +tre xp +tot land +to grapher +till i +ti mu +thereal xpac +te ste +tau fik +tat amag +tainte d +t pot +sug aya +stre ater +stat i +srisri in +sport sau +south indian +sk ers +sinthe city +sim ko +silver link +shoe fie +shar dul +ser ravallo +selfie time +sear ls +scott rogowsky +rune stone +ru elle +rober th +ri perton +pv f +promoting women +progen itors +pro pe +pro fuse +priv ated +pre raphaelite +prand elli +porfi rio +porcup ines +plu ck +planet x +persi ja +palla volo +notbe infringed +norse mythology +nim rud +ngay ong +nevad ans +nego cios +nb hd +n italo +multi band +monument our +mn gt +misssaig on +miss america +mersey police +megali ths +mc f +max keiser +mali gning +maha vir +len awai +lap is +kra sny +kir ui +kil bourn +ki maka +just girlythings +juniat acollege +jo sua +jimo heir +inish more +indiscre et +implo red +impercep tible +illegal aliens +high clere +heav ed +h mos +groo vies +gre search +gr v +goo fed +going strong +gems bok +ge za +gar ant +frye burg +friez eartfair +evangelinel illy +esk ar +epi da +english ness +el ser +el ani +dres den +donthug cacti +d town +cornell mba +conden sers +co sponsors +cityof toronto +cine family +christ of +chim eric +chennai rain +cheeselo versday +cfa institute +castig lioni +caitlyn jenner +cad enet +bru gger +bra sse +bgg con +bbc questiontime +barg ello +balear ic +b hy +aur aria +atro pical +ar mco +aqu inn +aje sh +ai ro +ah met +aga o +ðŁĺĭ @ +ðŁį ļ +èĭ±ä¼ļ 話 +ç« ĭ +âľĸï¸ı âľĸï¸ı +© : +yofthe seas +yas uk +yas sa +xxx holic +writer wednesday +wh p +vil ain +uvm vermont +upgrade yourworld +un spectacular +uk oug +ug dsb +tw angy +turtle back +tt om +thestra infx +ten aglia +tb u +tam amo +t ú +suble tte +straigh teners +stor ag +spor ades +spar ql +sn ark +shi ve +sharon lawrence +ser ry +scand ale +save theworld +s sex +ri béry +res ented +remun er +reign ites +raveng lass +ra anj +quir inale +py re +pu jol +prate ek +poo jab +per ic +pay bill +paradigm shift +ouro ceans +ota valo +nyc gov +noris ring +nintend ouk +nat ya +nas ugbu +n ka +myel ination +mr chris +monkey island +mo stest +miles morales +mike pence +medi asummit +mal er +maiden hair +maddieand tae +lu ÃŃs +los ada +long case +le phants +ld ny +king sx +kd v +jet port +j alo +ira ivi +ing show +ing america +indi ameansbusiness +ig ad +ideser venewshoes +hoax er +historic preservation +heavy weight +happy memorialday +handel sman +hak am +gj allar +gastro post +gam betta +future star +football season +field school +fay yaz +famili arizing +exercise works +enam elling +en stone +ember js +electro therapy +edwardj olmos +ec ma +eb don +e badi +dex com +democrati zed +daco its +da xe +d ft +cux haven +cupp les +corpor atist +cor nick +coal brookdale +cn ooc +ci enti +children underattack +chic co +cardio vascular +californiawild fires +buss mann +bu cadi +broad heath +bri zzi +brady haran +bor des +bo en +blue hour +bibliothe ek +bi thell +bi gamy +ba sted +avery brewingco +aspir ator +armb ruster +ap aper +ang y +an vi +an sascity +ale ko +ah rq +íķĺìĿ´ ëĿ¼ìĿ´íĬ¸ +èĦ ĩ +æĿİ æķı +æ± Ł +åİŁ å® +ãĥ Ł +áķ Ĺ +à³ Ĥ +Æ ° +® ) +x posed +wre sting +west bury +von k +vic trix +vamp y +usc upstate +us enet +ur w +tu chova +trumple aks +tong ans +thestor yo +thaicave rescue +ten nison +tele graphs +tejas vi +te sta +takeover day +tab ur +t pt +sumed ang +stoltz fus +star ro +star fm +st mag +spir ates +snaggle puss +sm iting +si mel +shar min +schuyl erville +roberts dale +ricci rivero +research day +rachel dolezal +put ney +proud fan +pr jct +poli shed +pk gs +param edical +pa ko +ordin ations +or poreal +onno ghen +on asap +official steps +of ire +nie res +ni mona +next stop +nat la +ms gt +mir ta +mhat re +men angle +mcro bbie +mc kees +mc frs +mb abane +maha patra +lur ching +li gao +lent i +lenawai the +la fave +konec ranes +kirk cousins +kiralı kaÅŁk +kawak ubo +kat unews +karn ali +joh ne +jim bob +jessi es +jan fam +jac i +j md +is las +inter reg +heyit scarolyn +hen low +hari kota +hand bag +gulf coast +goo derham +gla uca +for mars +filo sofia +esche wing +eman ation +eliud kipchoge +dÃŃ adel +dou chey +dor king +din or +desp ain +den k +defen sie +dan bilzerian +cre ami +cle fts +circum venting +ci ega +card ston +car lifestyle +candid ate +buff lehead +bronx nation +brian schatz +boul den +bou ska +born and +bogdan ov +black label +birch bark +bio tope +biblio graphies +bb bb +bay ad +bas rah +bar ahona +ban kia +avi gdor +aro on +arab idol +and redrum +anan arama +an ur +afro disiac +af oa +ac cardi +abu ll +aborig ine +abo d +ab d +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃ +ðŁĺĺðŁĺĺ ðŁĺįðŁĺį +ðŁĴį ðŁĴį +ðŁį¬ ðŁįŃ +ðŁĩ ¸ +îĮ ¨ +س Ùģر +تص ÙĪÙĬ +ار ات +zbrush central +zaf er +yun is +yun amusic +yo kel +year nings +ye vich +ye as +winter fashion +win ford +wid dowson +whack ers +wendi emalick +wbr c +val ances +tot ley +thisisd avina +thenana aba +thec wu +the wedding +the register +tenovusc ancer +tau p +tan redron +tanredron cal +super capacitors +sti mac +soft pedia +sic ecream +sheikhu pura +sd or +sco wl +san den +sal vin +ro vira +ro versi +rize spor +righ twing +re pa +rc cc +ram cinemas +r gn +quarter finalist +qld labor +qamish li +premratandhan payo +poul tice +pas son +pan neer +pac eu +p ite +out matched +ol medo +of osu +new usc +ncaa icehockey +molly quinn +mo ta +michel led +me som +mcgre al +mazz ini +matchday image +masar yk +manish mischief +ma fu +little st +lind seys +ley hall +le schi +lc as +latime sopinion +lam pa +ky lee +kra jicek +koz elek +kon z +keen est +kale mia +k jel +juliusc aesar +ja ha +isma ily +inter dict +ingra ssia +in sha +in mac +ib are +hush puppies +hoy te +hom ma +hol tren +han en +hack neyed +ha doo +go kin +girl talk +gab oury +fund aci +fueledby ramen +fi zzy +ferru cci +ferra gosto +felsted school +farm market +famili esc +expul sions +evolu zione +endocr ine +east burn +e braeden +dysm enorrhea +dre ver +dont look +dj ou +dis organization +develop mental +defin able +cruci fixes +cot f +condem nable +clu mped +chy ron +chro ma +cent ar +ce gep +carav aning +broad foot +brix worth +braw ner +br d +bellige rence +bear r +barry island +bar us +bal erno +bal co +b tweets +at it +arach tober +amazon giveaway +ais ling +aiac obelli +afun eral +afgh anist +aduri z +adel itas +( !). +ðŁĴĩ ðŁı» +ðŁĮ´ âĺĢï¸ı +ðŁ§ Ķ +ðŁ¤ĵ ðŁ¤ĵðŁ¤ĵ +ðŁ¤¢ ðŁ¤¢ +ìĬ¤íĥĢ ê·¸ëŀ¨ +ì¹ ´ +ãĢ°ï¸ı ãĢ°ï¸ı +âľĪï¸ı âľĪï¸ı +اÙĦبØŃ Ø±ÙĬÙĨ +zi yar +ze me +yer im +wo hoooo +vo bis +villu puram +vijay alakshmi +vi res +v rat +tun ggal +tropic als +tri bology +trans fered +tobykeith music +to v +to chi +thur ston +tex asto +tere k +sze to +super cheap +stat u +sou rav +so what +shpong le +shink awa +shawn na +sf d +selfie olympics +scru ff +sc sk +sc lassics +raw ong +rangi ora +r Ä« +pp od +popo cate +polit icon +personal shopper +pel ayo +pau city +over shooting +ortho graphy +ny land +number less +nitalo wey +nip muc +nahai wrimo +nac all +mun dy +mor osi +mis uses +matthewl illard +mar les +mag alies +m phs +lime bike +legend aries +labru sca +la se +l bi +ks ack +kre wel +ker li +johancru yff +joakim noah +jameshet field +iz anagi +iy aa +iam laceychabert +hy pon +hor ak +hoo pla +hawai inewsnow +han shika +gur k +group news +grl powr +greed ily +grad ations +google photos +goo oooooooooooooooo +golden retrievers +gig guide +gg olf +gener alizing +fun kiest +frail ties +flower girl +f stone +er roll +en ninful +el repgh +ec ross +dubo ce +doo bie +din ma +dil dhadak +der reen +dallast nt +daily calm +d chen +cu yler +cro teau +cre on +cr nc +con scripts +compen sator +col azione +coastal living +co ley +chekkachivan thav +ce volleyball +cap acious +cad burys +ca stries +bunnaha bhain +bike towork +bhand up +ber tinelli +baske twomen +banc or +ban anab +baira va +bab ul +asho ka +as gr +apar o +all ly +all bright +alessandr ini +aku mara +ai ste +afg anistan +ad ame +ab salon +; } +!! ðŁĺĤðŁĺĤ +ðŁĮº ðŁĮ¸ +ìĽIJ íĺ¸ +ãģ Ľ +zi zz +zi k +yor ke +wx mafc +wwe bige +wo wk +win today +win co +whati f +wer kin +vide tte +vajazz le +ud h +u mineko +ty ng +tu pelo +trans vaal +total cafcl +thrur pg +the revolution +th parallel +ter cel +tal man +ta be +swam ping +super micro +su nexpress +stigmati zation +steiger wald +spur ling +sports writing +sport sclub +spoiler tv +si ru +si dle +shel bs +sharon jones +shari alaw +sen deros +sec nation +sap r +ro dden +researchimpac teu +redding power +rashi ki +r mit +que brada +puffin ess +prince albert +pp aca +pop health +point lessness +pleas ence +phoenix lp +pau lar +pat rouille +pantal one +p cl +ny che +nu dgee +neer atan +nas ar +nach es +nab bing +n he +n ck +mu umu +morning listening +moon watch +ml jet +miskat onic +mis u +milk fish +mi u +mg tow +meh di +mar athe +lucasoil stadium +london breed +lizard men +live say +lifes better +liam fox +led widge +lavanyab hardwa +kre wella +kne eler +kings north +keepcal mand +joy al +josep he +jon ni +jo edi +jeff tweedy +instagram stories +ingen ue +ine learn +how lers +hipho partist +he mera +he mant +he kate +hann ahs +gold bergs +glu cagon +gad u +gabri ele +gabby giffords +from where +friez elondon +fre she +fo shay +fernet branca +fer r +farm show +er se +end coal +echi dnas +eastlanc srly +earthwind fire +duct tape +du bas +dru cken +drive thrurpg +dick ov +di gos +devarak onda +david johnston +dab omb +conis brough +comple xo +colle geny +clu mber +chester tweetsuk +che bet +ceano thus +ce ding +cas lon +career development +caoim he +bru te +borde leau +birdwatch extra +bengal cat +bel aying +beau s +bbcsouth weather +batt in +base ballis +bapp amorya +az d +att r +an anias +akshar ahaasan +abhi jit +ðŁĺĦ ) +ðŁĺĤ "@ +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +young music +yo ke +win chesters +whok illed +weare cue +vo gl +vi ji +ver asani +uss k +un manly +ul ite +ugand a +tri vikram +tran scom +tor ye +time snews +theaf ccl +thank god +tham esp +tele toon +tat to +ta fa +su athletics +stop commoncore +st s +spire a +speed hunters +slu sser +sir na +si put +shaaan xo +selen is +scra zy +scam mell +sar tor +sam ik +safer internetday +russ els +rogu ish +rl tw +rixton official +rivie res +richard marx +re inet +re constitution +re configurable +ra pson +priyad arsh +pot c +pen american +pat tullo +par acha +p head +ori shas +one wtc +ocu lus +norman lamb +nick vujicic +ni ed +new products +nar sis +nab l +moy cullen +moo o +million cup +micha eli +mi dian +mb aye +malay o +m trainier +m ser +loveyou all +liti gating +lind ab +lehigh u +launch party +latch key +lanca shirec +lan dover +kra thong +ko tz +kint bury +ke iz +jubin nautiyal +ip mi +inten sion +hou la +hou ght +her ber +helge son +helge sen +he ireann +harry appreciationday +gimb als +gill ingham +giftof life +gel ber +ge iser +gal z +fanta sized +ey ard +ext ella +enter ta +endless summer +eat fresh +dyne vor +drunk ards +dream ers +do ps +dig able +designby humans +dart ford +confer ment +cold front +co health +claire fontaine +citiessky lines +chin amcclain +chan dio +chae bol +cec ily +caric atured +cardiff met +car ven +cameron mathison +cab ourn +buen camino +brundle f +brazo sport +brand ao +bio art +bil stein +beacon house +ba her +añ ejo +auto group +author amish +as avi +aram kon +antiquec lique +ann eli +anand tech +amne si +ambro se +aji bade +af raz +aber lady +aaron son +a etc +ðŁĺį âĿ¤âĿ¤ +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +ðŁĩ·ðŁĩ ¼ +ðŁĥ ı +íķľ ë¹Ī +ìĶ ¨ +Ê ° +zen book +zato ichi +yo or +y alty +x html +womens facup +wild horse +weather wax +we some +war like +wal tons +voteblueto saveamerica +val emount +v ml +ut kal +tric home +tresp asses +tool ong +thor aco +thesquare ball +thehard way +theatre awards +the drew +thar anga +sy reeta +sur ridge +sto inis +splend ens +sp lanet +soultrain awards +sophiel ady +som ec +smart buildings +smar a +sing post +shoshan abean +shak ya +sh agu +scott pelley +sat an +sal yer +saint snation +s vein +s comedy +rober ta +ro der +rebel o +rebecca jarvis +raiffe isen +pul te +ps v +prompt advant +pro visioned +pre iss +portrait november +port ela +plan itia +pic xania +philipham monduk +path on +par ata +pan hard +ns live +novo gratz +nh b +neeratan den +nam ida +nam es +mv agusta +mun in +money gram +micro climates +mement omori +medieval art +media ev +marqu es +mark smen +mar nie +mad magazine +machine intelligence +ma pun +ly c +lum sky +lu me +lewisham council +len inism +kordo fan +ki as +kel oid +kathleen madigan +jud iciously +jose fin +jessic as +je ti +jay z +jan as +jac en +j ver +ir craft +in th +illi ans +if adnews +hopgod friday +half price +guitarri sta +grlpowr chat +great indian +golf show +god win +gobble gobble +go za +gl te +gigab ytes +g lat +g brow +frank reich +fp jr +essi g +en theo +ed p +ed iti +ec ruz +eb st +eastern cape +duke gang +dover athletic +di emer +demo dex +decryp ted +d bh +crusaders rugby +clam our +choic ed +chicago cubs +carai bes +bren chley +bree zer +born thisway +ben av +ay umu +avs nigeria +antoniob anderas +am art +all eno +al sh +adi vision +aber porth +abad don +ab hy +a bee +' ..." +ðŁĶµ âļ½ï¸ı +ðŁĵ ī +ê¹Ģ ëıĻ +ี à¹ī +ÙģÙĦسط ÙĬÙĨ +ا Úº +zwart ble +young spubs +wc pw +volk off +vijayan pinarayi +val parai +un produced +un likable +ulcer ative +u can +tä nak +toxopla smosis +tor ren +tine a +thegirls musical +texas strong +tari que +tami roman +take away +ta st +t js +sx melectro +sweat pink +suresh chandraa +sound mag +si ya +sex tape +sebad oh +scott porter +satter white +sar anya +ros ada +ro senior +recomend ado +por tholes +pon tin +plio cene +pe quod +patho fexile +pak hi +p du +off world +news busters +nat vanlis +na un +my fdot +moun troyal +micro fiche +miami bookfair +mi q +mh sc +me sabi +me ddle +mau ch +math lab +materi alised +marqu ita +mar tucci +macou pin +lleg an +limit ers +leonard cheshire +lavo z +koin news +king si +kick offs +ki am +kad aramkon +jo sho +jenni em +je wer +jac ey +j pi +isaac mizrahi +ir thday +in comparably +ima ke +hou de +ho cked +hit btc +high seas +heer babu +he dd +har kavy +ham irpur +gy c +gretchen whitmer +great guy +graff erty +golf travel +giftfor him +gh un +ger as +gau vin +gar agi +gand alf +g di +g ades +früh ling +fri derz +frederick town +fing al +fibaeurope cup +fanta sist +esti mable +ellic ottcity +ek d +ed xonline +e apc +du eled +don gs +dog treats +dish nation +die h +dev ault +des apare +dent es +davidg ill +dar cel +dad bod +da di +d km +craig lowndes +cor ris +capetown cityfc +cam poo +cal d +cairn terrier +cad ence +c mg +bwal ya +bw g +bragg art +br ica +bor nes +bon um +bon der +bm ttandem +blackdeser tonline +be vents +bb ell +bat dad +ather ton +at witter +ask am +arnolfini arts +andri ana +andre an +amo reno +amit abha +alan tudyk +al van +al ace +ad be +ab ayan +? âĿ¤ï¸ı +ðŁĺĤ ðŁĴª +ðŁĵ² | +ðŁijĮ ðŁĺĭ +ðŁIJİ ðŁIJİ +ì¤Ģ íĺķ +zom usic +z ouch +yyc music +yu saku +wwen eville +wo ggle +wick reme +westfal en +weigh troom +vr ps +vote jkt +v th +ty ard +tu ks +truckn driver +travel ist +touch wiz +thero bot +ther ag +thelaugh factory +ter ma +tel for +tel erad +teach ag +tatamag ouche +summer glau +sub humans +southern star +shaf ak +seawol ve +scu do +scorch trials +sat na +sant en +sanc on +san ghar +ruok day +rumb led +rogal and +revolver mag +railroad ed +queens land +pumpkin carving +pre sonus +pr request +pop dust +po gona +parl ons +on scott +om ir +ok mesonet +nu wara +no vica +nigh tt +nh dems +newton ville +new beer +navar one +nahu el +n md +must i +morning star +model cars +mo peds +miyaz awa +michaele merson +mi yoshi +me tron +mat angi +maker studios +madon naf +lub be +losin j +livel ounge +lead up +ladies g +kre ut +kin sey +ke et +kar wan +k bb +joven es +is bnpa +industrial music +inde acon +in dominus +henson company +hay market +hat ori +ha skel +gy al +gravit on +grant ley +good looking +glu teal +ge ffen +freak indeacon +flin der +flick inger +fi ved +femin amiss +far sighted +erlen meyer +do gen +dal beattie +daily herald +cu ong +cru mley +court sey +could nt +cotton seed +co traffic +chin ky +cefas govuk +cam mi +cam bourne +c sed +c festival +bur gee +bro cante +breakfast with +brac keting +bli brary +biggest loser +bethesda studios +barry sanders +bal in +azamar avoyages +auctione ering +att park +ar ni +app rised +ani festo +al nassr +al loc +ai mar +adu bai +adam woodyatt +ach on +>> # +ðŁĺ¥ ðŁĺ¥ +çģ « +ãģ © +а ÑĢÑ +zoethe ball +zam ana +zab ludo +xia oyu +ww cc +wre tch +wi leman +white read +wal kab +viktori ya +vikk star +un trusted +tsa on +tran stv +the knick +thar aka +t drs +sw brail +swbrail riders +sw all +susu mu +star sonice +stap ha +spring well +spinabi fida +soci os +sn ard +smar ini +sk use +sit us +ship wright +sep i +sene cio +se vin +scho or +schill inger +sc sn +satis factor +sachsen hausen +s girl +ry uki +rugge dized +rosic rucian +roman britain +rl j +reas sign +rac ia +proud father +po je +plu mping +plenipotenti ary +pick les +phosp hati +phi b +ph enter +pepso dent +partick thistle +pan ge +otter dam +or re +olym pe +nut free +not ta +norther nontario +nor fleet +noir summer +multi focal +mudgeer aba +mrporter live +movi eland +morning live +mo ssi +mir us +minu tiae +min usma +metam orph +medic os +mc glade +match es +mal amu +mahar lika +ma ppings +lik on +ler k +koro vin +kaz insky +jobo aler +jac kiel +in memory +i io +howto basic +horn beck +hm hs +hel muth +hay dens +hashem ite +happy makarsankranti +ham strung +hacket tofficial +gon calo +golden temple +ge ist +gator zon +fredo santana +fl acs +first gen +fil omeno +f heis +ev intage +en ali +ed iciones +dragon gate +dr j +do ina +disper ses +di wal +dev hynes +deni grating +del rey +de duplication +davido tunga +dab lam +d str +crow medicine +cot grave +com ate +col drain +coffe es +city westminster +chieftain cy +cherry blossom +c sto +burgh leyhouse +bu er +brom well +bir bal +berry tmr +bered seered +ben the +bc x +bar atta +bam se +bal lade +aw is +av adi +as mat +as cu +armi da +app ric +anasu ya +all llllll +ale theia +aldub xe +aldu be +ail lu +against allodds +adamcole pro +accoun tab +a ane +" ...# +ðŁ¤ ² +ãĥĸ ãĥ¬ +à³ ģ +z day +youth for +yoga pose +wain ui +wad ada +w clc +var ona +urin alysis +underdesk loser +under glaze +umich medicine +ull mark +ukrain ian +tzed akah +tw ash +trump sters +think landscape +theoutdoor city +the town +the sar +thath omas +tez pur +ter no +tas lima +tar ta +tafa wa +ta arab +super mariobros +subhash ghai +sub contract +spublic ations +social rights +slo ppy +ski jumping +sher born +sehs thebest +scuder i +sar ova +sal ah +s room +roman ova +rock on +regaz zoni +r kt +produ k +predi lection +plaster work +pi ura +pen alizing +pan ta +pan enka +pal mares +old english +oax acan +notredam ec +not fair +no vik +nic kie +nast assja +n said +myfox tampabay +mom usic +moistu rise +medi amo +mc quay +mas dar +mani yah +makeit real +ma kay +m jakbar +lpool council +lowest price +lass ic +lar mer +la win +ko conews +kn apper +key stroke +k rook +joburg theatre +jack the +hk ir +hil der +herre ra +hei jden +he ley +haz ra +hann ad +gregari ous +german shorthai +games manship +gam p +gal u +g sy +g ico +fo b +fe to +fate ha +exagger ations +esur ance +eigh th +ec wa +e thology +drug war +dru gab +donkeys anctuary +diap hanous +di maria +de cri +dai quiris +cor day +con leth +come at +co bix +cobix reyes +cc sf +cap radi +c bo +bsn sports +bot ching +bin n +bike chi +ap ke +annas ui +anc as +an aka +almir ante +al bies +ait ana +ad alberto +abar bera +. ,. +ðŁijī ðŁijī +ðĿIJĪ ðĿIJ +éĢ ļ +ãĥ³ãĥ ģ +ye siam +were the +wehave wewill +we ki +wa ht +vul gare +ura uganda +up gradable +uof regina +un righteous +u stra +twitch affilate +twee ds +tv guide +top coder +tom ber +the virdas +thankyou u +ter man +tele visual +team ajaydevgn +tali ban +tale of +tac tic +swains boro +supri sing +su raiya +stu lsa +stir rings +steph anos +stand pipe +space coast +solart vnews +snoqual mi +sn live +sl un +sh ole +se wak +science advances +sche ff +sch roth +scal ene +sal atin +saf ran +sab r +road runner +ri smo +resemb lances +replic able +replac ement +re trained +re draiders +r fef +pra chu +pol an +pesh wa +ongh ere +on ova +ome a +o dgers +new shead +ne eli +national lipstickday +n ram +n han +my heart +mur nane +multic olour +mtr cb +monte mayor +miguel tanfelix +mck ell +mark d +mar m +mango tsfield +ma bie +m brundlef +lulu lemon +louisi ana +log books +lev itt +leff ingwell +la villa +ku nedo +kra de +ken ma +karate ka +k gp +jame er +jagad guru +it ouch +ish h +indul ges +ilig ence +i prit +hymn als +ho ppo +histor ico +harmon izer +grant sville +google exper +gm fc +gemat ria +gal breath +fun c +fraction ated +fox newspolitics +fi h +fan light +engag ing +en esis +ec oli +diferen te +dhen kanal +de moralising +dayo ade +day tuesday +cy m +cra zzy +conver gences +colou red +cine spia +chis os +cer ullo +carno taurus +caric hampion +capp ellini +c pyne +bu land +bri atore +bradleys fight +bracci ano +br ck +be atie +bam asb +bag na +bad day +ay ran +au do +at ole +astro logers +around hampshire +archi vo +aqu azzura +ant ourism +almir on +airbus ds +adeci mal +acid house +abell ana +a dis +.. ~ +! ðŁİĥ +ðŁĴŀ ⾨ +ðŁĴĸ ðŁĴĭ +ðŁıı ðŁıı +ðĿĺ ¦ +⾨ ðŁĺĦ +âĺºï¸ı ðŁĴľ +ب ÙĬØ© +zee man +x cfl +will mott +wikicom mons +vibr ance +vi ste +une motional +un truthful +tom segura +to pac +the bill +ter update +team sa +te icher +tam plin +tailgat ers +ta haj +super normal +stat evb +sn mp +si men +shap er +se vil +sc as +sat akarni +sanit ised +sam bro +saf wan +rider strong +re color +rath fr +quade cooper +public diplomacy +pronoun cement +positive coach +plebe ians +plane te +ph le +pent yr +pen ko +pe red +patron ised +orgre ave +ok l +o je +nor ra +non et +nas rullah +museu mart +mou tier +mosas aur +monopoli zing +mm schocolate +mizz en +mis l +mal ott +m con +lud wick +looo ok +long den +liver ight +litt a +lit te +lion nation +lever ton +lefthand brewing +le petit +kul iner +kro pp +kid sday +khali fe +kh ound +ke shi +kat la +kasar agod +kas lo +jer am +jef ford +info sy +ho adley +himach al +heritage fun +heller town +hass ine +hard anger +hallo ates +hall yu +ha sit +h sw +gy p +gron din +graub ünden +gra af +gon dar +go girls +glas lyn +ge thealthy +gan k +ga hh +form o +form ative +forgi vable +flu season +finding carter +ferrar ir +euthan ised +err orists +emb aj +eliz acarthy +ed unn +eat clean +e max +drive safely +don ghan +dom ec +do ts +divers i +deven o +dea thanniversary +dave ed +dance onfox +dair ying +d town +cyto skeleton +ct surgery +consumm ation +consoli dations +comrade ship +col ch +chun ga +c pu +busy body +bu den +bro chette +biz school +bi bb +begley jr +beautiful places +bbcradi of +bar thez +bah ram +bachel der +ar que +al iso +agi bbons +agglomer ation +ag unn +ag news +achen bach +ach or +abo gados +ab ster +ab azar +aar au +a shot +ìł Ŀ +ãĥķ ãĥ¬ +âĩ © +ÙĪ ا +zo va +wou k +wn p +windy city +wego again +wedd in +wat esgroup +vinod khanna +ve kic +v lam +unex amined +unbeliev ing +un labelled +tur nal +transc athe +tram mel +topra k +told story +todd ington +thu ms +the union +tau fiq +tab an +super intelligence +story behind +stcuth bert +sta w +spell checker +sl rs +sk news +si pple +sh kapoor +sear a +saturdaynight live +satine phoenix +s yo +s meets +row lf +ro say +richard coles +ren c +red dot +rajag op +pust aka +pneumoni ae +pi ase +petron as +parl ors +paolog ent +paologent iloni +overex tended +over wrought +ou attara +oned bestfans +o teri +ni ère +ni gr +ne a +nal gonda +n wtf +mor tification +mo tala +miz rachi +milen io +mil bourne +mh world +meridianc u +me yn +me tri +mat ariki +mail and +mad villain +love food +lo des +letsgotothe ex +lec ricket +la sen +la fe +kt va +kon ink +kir t +khal fan +it ter +ira ja +indo logy +in bal +i ae +huffpost live +hot beds +hom eland +heaven officialsblessing +health ed +groen links +gro be +grace fulness +gau ff +fundam ent +fr anny +feed lots +farmto fork +fa bel +eve yone +etru ria +esk om +ere whon +enov aaw +dr ough +down s +dis kette +dhi raj +design miami +deck i +date time +dash lane +dasch le +dam ay +cra zz +coy q +confi dants +chain maille +chad dadon +bun ited +braehead clan +blue sea +billys later +berlin a +ber it +behere now +bbcin tune +bareminer als +bal luk +bab ii +au ll +ation alist +at tu +at reides +asah ina +as rs +arts offl +app state +americ o +aller y +al az +. ðŁĮŁ +" ....... +ðŁĻĨ ðŁĻĨðŁĻĨ +ðŁĺı ðŁĶ¥ +ðŁĺ¬ . +ðŁĮ¿ ðŁĮ¿ +èģ´ãģĦãģ¦ãģĦ ãĤĭ +âŃIJï¸ı : +âĻ¥ï¸ı @ +âĨ ª +zipp speed +yel ahan +winter sale +watch me +vir di +vic ari +val demar +tu tton +trum pusa +triti ya +tornado warning +tomp kin +to fin +then fb +theli ber +thel ab +the est +th rum +teu fel +taxcut sand +swind ells +ston i +sto povers +steve perry +ster r +star flower +ss q +sound track +sigh isoara +shutt led +shen stone +shaf aq +sense making +seis mometer +seapor ts +seaman ship +sal cido +sab u +s thouse +s arti +rub chinskiy +ri poff +rho dia +retro fits +rep jerrynadler +rec entre +purrr fect +puff puff +ple eeee +pit roda +phra o +per se +partsun known +paraly se +pal mira +paddington bear +paci ous +of london +north pole +naga oka +most awaited +mor ini +mor arji +monast ir +mo wins +mike crapo +mar janovic +m chat +lindac ohn +li zza +lever ly +legg ere +latch mere +kuus amo +ku uga +kle ist +kha yr +kevin mitnick +kazant zakis +kam araj +ka shu +jos buttler +j sk +iolan the +intere sante +inter fans +incu bus +im richardyap +im broglio +hydro logist +house boy +hol ling +hk sar +hills comedy +here ee +har thill +hal yard +gov of +gimna sia +gie sen +gg lobal +gag tweets +g mtv +g board +fu sed +fru tos +fight net +fan sday +exc elle +eph rem +ele me +e tron +down slope +dg w +desro ches +desk top +derby uni +deli jah +dar gis +crime drama +crevas ses +colo simo +cla vell +chiantic lassico +can nel +camise tas +business continuity +bur lesque +bun ched +budger igar +bu ist +brun skill +brin ker +bert man +berber ian +bar onial +balu ster +bab ymama +au mf +as del +and cream +alannam asterson +: ðŁijĩ +Ķë ¸ +ðŁĩ¬ðŁĩ ³ +ðŁ¥ § +é¦ Ļ +zwartble sie +zo omi +woo ot +whoop whoop +whitec astle +wa as +virtual tour +var dan +v uuren +unter gang +tu twiler +trevor jd +tree brewing +togetherfor yes +ti ye +the ke +ter vuren +tar pon +tail ment +tab ligh +sund quist +summer fashion +spreadlove positivity +spec news +si dec +shri venham +septu agenarian +sales management +ri u +resp ighi +recep tivity +real fpjr +rational isation +quin s +quar tering +quad rennial +pope inus +pol onius +pl sql +piet sch +pan fur +over simplified +out smarts +ourcountry red +oris kany +new cifera +ne ons +national muttday +n ji +myx musicawards +my erson +mu women +mr t +mohen jo +ml bonfox +mississipp ians +micro chipping +mic keys +mi yoko +mc gowen +mc atee +mar uko +mailand guardian +magneto sphere +mac ritchie +liz ia +life below +letting go +lets roar +lea o +lab life +la et +l anni +kok stad +ker ast +k met +jas raj +jame shut +jab u +j brown +iz ola +iam intel +houseof lies +histo grams +hal ite +good rum +gbrow ingteam +fu ÃŁ +flugz eugbil +flugzeugbil dde +flamin goes +fifty three +feile belfast +fe ir +fa sig +espn greeny +dothe work +dont get +dominick cruz +disal vo +dis ables +digi dhan +di deas +deliciously ella +deaf lympics +dai ji +cur cio +con sole +cis cou +chekkachivanthav aanam +cheer sto +casca bel +cal dron +bro dick +bird dog +ber meo +bel onghere +ar p +al mod +al ahly +aga c +abe ille +ðŁĴİ # +ðŁĩ¨ðŁĩ µ +çIJ ĥ +ç¬ ij +æĽ ² +özge gürel +zin ni +yaku pov +y for +x si +wright state +worldar chery +win chen +well head +vis erys +vin da +un glazed +to cant +tocant ins +tin ky +thi ther +themeat ly +temp ter +teg mark +take charge +tac tless +suppor tive +subsi st +su ha +stil acosmetics +ssal on +spar da +smarty rs +sky sox +shutt lesworth +seismic ity +scol lective +schnit t +sch ib +sc d +sar anda +santan acarlos +sang ma +sak ala +ris ation +rein ders +ram fam +prayfor peace +pra sadam +pleas uredome +pf leger +pe f +patmcgrath real +our pain +oss ining +on ear +omer uo +oil man +official alw +ob sse +norde ste +neutr alizer +nav s +national pet +n andy +ms mith +morris art +morphe me +mic limerick +mari b +man cos +lagar to +kish en +kin lochleven +ke ffer +kamasi w +kak eru +just ment +john nys +jamshed pur +jagga jasoos +jager meister +j nan +j ff +irantalks vienna +iphi genia +inas much +in appropriateness +il ve +iamami whoami +how z +horror hound +holly holm +hitmusic only +hell yeah +hel los +haye z +harsh deepkaur +harrogate town +ham er +g tz +foo tre +finds friday +fin edon +feminamiss india +ev aristo +eu th +eth icon +epi gram +end ays +ed begleyjr +e wha +dimm itt +denni ston +daysof oscar +dal ot +ct la +christmas decorations +chee sey +ceram ist +cbc sby +cattle man +camping world +bru hn +brad stock +birthday month +birmingham weare +bird flu +bir sa +bill kristol +bh ana +bel ice +bb log +bar kin +az b +aturi smo +ath lone +aspe tt +arnou x +anamari ecox +an una +am munitions +all blackeverything +agricul tural +af ce +' ?? +ðŁĻĪ . +ðŁĺī ðŁĺľ +ðŁĸ ± +çĢ ¬ +~~ ~ +zuk erman +zippor ah +witcher game +whor un +wee tie +wal thall +vel as +v hong +us nft +us asunrise +un sheltered +un ani +type casting +trans metropolitan +the guy +super tanker +suggesti vely +ss m +spol icing +sm itten +slo cal +slash dot +skybet league +skam ania +sing am +sig gers +si mes +shar ansky +sh loka +se dro +schi frin +sat yrs +reve sby +retail news +ran chom +rackete er +polon sky +ple dgers +pit u +phenomen al +pe tah +pari stech +or lin +ocean acidification +obu asi +now toronto +newberry library +neder lander +n acks +my father +mss ociety +mount field +mor tier +mo in +mischiev ously +micro service +micro green +mc gown +matth agan +lur kers +lowes water +love wilko +lmfa oooooooo +kwan ghee +ko tv +kau fusi +kati em +kat ori +kat ar +kare en +jorger amos +j ems +ital design +isp wp +india historypic +ignomin y +if v +hostel ry +hope world +honor our +hon d +he schel +h alie +gri sman +goondi windi +get motivated +geek girl +ge tar +fire bombed +feder ico +family guy +excel sis +eve tt +estac ada +era edta +du kraine +dr l +dou ai +denisle ary +dar la +cryo lipolysis +cruel ties +cre ady +collecti vist +codeof vets +cobra golf +co gen +club med +cloi stered +cla gue +chit osan +chau dh +championsle aguefinal +cham pur +ce mal +carpenter sville +bry na +brumb augh +broad green +brat en +bram ante +bob sledder +black lick +bi alo +bhar gav +bhan j +bbc northampton +bay at +barrela ged +bal som +baby ariel +b gh +az nude +associ ate +as ound +ar coding +alainde botton +af es +aapkad haram +** . +!! ðŁĺģ +ðŁijı ðŁĺĬ +ðŁ¤Ł ðŁı¾ +íĸ ī +é Ī +æĸ ĩ +~~~~ ~~ +zu zanna +wo wwwww +wickreme singhe +wau chope +voteuk mahomies +vis arjan +vir l +vintag ep +ve rest +ur sel +unve il +uncle bob +ud n +tu ite +tr nd +total access +the phantom +the gin +the fashion +the at +ten ter +tasty poem +sutton coldfield +stream side +stre mme +stop fundingh +sports file +speed man +sp hen +so arer +sla dy +sin ter +shi st +schne pp +sch umi +sat pura +salt spring +saj jan +s arie +rogo zin +pre volution +ponto ise +po hlman +petter sen +pd schargers +pas quier +pac ers +pac eman +p bg +our de +oste op +oscar trial +o gura +nsu kka +northant spolice +natgeom ag +mo shiri +mis quoting +mercury prize +memorial hermann +mdant sane +mad dies +mac gruber +lud milla +lu sail +lin ie +lighthouse day +lich man +li ze +lazz ari +lay zie +lar ussa +kon ings +knob creek +ki si +kevin vonerich +kayseri spor +jin ni +iso de +is may +iprit amofficial +ine ws +implement ers +ig slovenia +hy dr +hof stetter +he ma +h shanghai +green planet +glu can +ghi bran +form alism +flower oftheday +falak numa +fac il +eric topol +elo ise +ei go +ec statically +east siders +eagle son +du er +drama beans +don skoy +don ata +desafi o +de za +de ste +dani her +d Ãł +cre atu +craig melvin +cr sp +con j +comhal tas +clar us +city radio +ci fer +cho let +chlor ination +chin ad +ch itti +catal di +car al +capacity building +cagli ostro +bullet storm +bu ssiere +brisban ecity +bride and +brendand assey +brac ts +bo vard +blue bull +black heath +best man +bern stein +bamboom usiclive +ath nico +at rain +ar non +appreh ends +amanda holden +am ason +alleg ories +aha dra +active yes +-------- ---- +ðŁĺģ ðŁĺĦ +ðŁijı âĿ¤ +ëĭ¤ ìĿ´ +ঠļ +اÙĦ ر +п ÑĢ +ç ons +youvegot mail +yemen ite +year sofe +yamahar acing +x op +x fighters +wood lice +wim an +wan kel +vin cat +uplift ing +un rehearsed +ub co +tom fitton +the ek +the drake +thab iso +tes las +tag er +super sonic +stro ve +stonec old +sine ws +sieg ler +shi rov +shi ren +sc liffe +salon en +sac cessories +republic adominicana +rep your +reload able +real isations +read missions +re genesis +re confirmed +rasto gi +ram jet +raja beta +rajabeta sharad +qu itec +pra kritikakar +polic ar +pol en +po wri +pipe smoking +pint ado +photo aday +pet abyte +pend ry +pen lee +paw z +p fen +p bw +overex cited +open cast +oh god +nishi kawa +nish ad +nikonowner mag +nikol aev +nas d +mun che +mu ahahaha +miss americ +mil icevic +microne sian +me mes +mc craw +matthew berrytmr +mak am +macul ata +m commerce +litt ell +lit em +lies beth +ki hn +ken ning +ke van +kamalhaasan fans +josel u +jo ssel +jim stweetings +jeet kunedo +j sch +iy engar +iv ars +incur s +impul sivity +impal ing +ili sts +if msa +ic bms +hor i +hin da +head abovewater +he ye +haz y +gul zar +guil la +gu stas +gor get +google docs +good thing +gameofth ones +gamali el +fu ssed +floridal ottery +fe ma +far away +fac er +fab bro +expec tedly +en circles +ele wis +egg en +ef ren +easter seals +ear ful +dun ya +dou rado +dilo renzo +diade ma +deco ction +dead heading +de tre +custo dio +cuer vos +crisscro ssing +cor ral +combo ver +columbu sohio +collecti v +co yo +co vel +cn nee +cho cor +char ityevent +ch avan +brass ware +boy land +bo twin +bla gg +big play +badboye m +bad illa +ba jas +b gd +ardu c +aposto los +apo state +antiterror ism +amar ah +alex iso +al men +al bright +adam jones +ad do +ad die +ðŁĺĺ ðŁ¤Ĺ +ðŁĮµ ðŁĮµ +ðĿĻ ļ +å°ij 女 +é mile +ç i +zion nps +z ian +yan tai +wr inging +won young +wis ler +will ferrell +west lothian +wah ro +w ky +w autu +vin us +victori ous +vee jay +up fitness +ulcerative colitis +uk ku +u ÃŃ +trac ism +tiru chi +the curren +temple of +tak bir +ta wn +t ver +super crawl +stop ford +soledad obrien +sing ita +simul casting +she her +se il +sc apho +save gotham +saturdaymorning cartoons +sars fields +ridge top +ri stic +rener gracie +ram apo +rach ita +r vo +productiv ity +pp sh +popocate petl +pi edmon +period stories +pav é +paul blackthorne +pachel bels +or tolan +op tane +olom ide +nw ts +newon folksy +neuro logists +nein quarterly +nass nigeria +na qi +n rol +n ooooooooo +mysteriesof laura +mun dos +motor mistress +montac ute +mo chi +milan luthria +mens shoes +mam asap +madison square +mac taggart +ma wr +lo boc +li wanag +lev ites +legg ero +lac y +kus al +kitt anning +key largo +kemp is +kam bli +ju iciest +john lithgow +jo bert +jameson whiskey +ic auk +hu llo +hom mel +hl inger +ha su +h reat +gu sman +green lights +go pen +ga thi +freakindeacon friday +fonda theatre +flamen co +fic ation +feel better +eyehate god +esk rima +eric ks +er ag +england netball +dis continuity +daniel daekim +d so +cryptom ining +crate red +coon abarab +coonabarab ran +confit ure +com pris +collegi ate +co ssa +cleanup day +cere brum +ce qa +car donald +canc ún +c newslive +btoo om +bso azad +bra zed +blue wings +biom aterial +beer geek +aun er +ash is +ash ely +as ob +art class +arkan oid +arig atou +angr yorchard +anc c +am irah +ah all +ago sta +a ep += @ +* .. +ðŁİ¶ âĿ¤ +ðŁĮ² ðŁĮ³ +ê´ ľ +ãĥ´ ãĤ¡ +âĶģâĶģâĶģâĶģ âĶģâĶģâĶģâĶģ +ॠ¤ +عÙĬ د +zam ba +za pote +yuku stun +yelahan ka +yan jun +wizzy jr +wilson tennis +ve rey +v qg +un cy +un conventionally +ultra hd +ud k +tul lian +th att +ter c +tax ation +syl ver +swat ara +sune dison +sunday read +stri pes +stopfundingh ate +stein itz +startup grind +ssin ce +spi elt +smar athi +sle epi +slack line +shrou ding +shi o +shi geo +s ry +ross lyn +roman kemp +rll racing +rick on +rep barbaralee +ratt y +pugn acious +pit ot +pig face +photo by +pad more +or lovsky +ol ten +ol itis +nz t +nin is +mo she +mel nik +mel bs +mecan oo +me flo +mcmee kin +mb sings +mahal ingam +m sch +luther ville +london town +lilli an +lewisp ugh +lend ing +lees ang +lay asmine +laur is +last year +land skrona +l gu +key f +keon jhar +kel ner +katrinapi erson +kadı köy +jum bos +jose i +i fra +hy land +huss a +hun grier +huck ster +hondac rv +hamble tonian +haji won +gyneco logists +gou let +gol go +general aviation +gc s +gbag bo +fragi lex +fl anno +fc business +fasten ings +fag un +exi stance +eura si +est ecker +equal su +epr df +dou ses +diet l +defend our +de plore +dav pope +dark sky +dar ger +dam pa +collector ate +co sp +climax es +cho com +char n +cassand re +carbox amide +car ref +car naval +ca zij +by k +bruce prichard +bowie forever +boulevar dier +blues brothers +bluec ross +bil bao +bev hills +be venting +babest ationtv +bab ay +atz maut +arc illa +araf ah +ar ag +ann ane +am etal +ale em +ald win +ah hhhhhhhhh +af branco +acce ssp +;- ). +ðŁĺĭ ðŁĺĺ +ðŁı µ +ìĦ¸ ìļĶ +기 ìłģ +âĿ¤ ' +âľĮðŁı» âľĮðŁı» +âĺº ðŁijį +ö kull +yas mani +x lb +wo da +vote searly +voltag e +uka itis +tw ald +ton earm +thof en +the antondubeke +tessell ations +swit z +sturdiv ant +spi v +spec ts +soul sby +slo we +sj giants +side mount +si et +si delights +shrub sole +sc ities +sav aged +sare thebest +sarah grafferty +saint mirrenfc +ross all +rocin ha +robin ince +river monsters +qu ente +pur lo +pneumo thorax +pilipinas debates +pere c +pc mr +par um +paper back +pan til +nu ma +nb apra +nano composites +na shik +mt gart +mr schu +michellea z +megau pload +med line +matt le +maram ures +mar torano +mandel baum +ma ipo +m ty +m thood +lu cc +lifein apicture +lepi dolite +le ser +lan ville +l fn +kil rea +kil duff +kay in +kar war +k mp +jonas blue +jin woon +japan expo +ins berg +inge ye +infinite simal +ind ar +inam illion +husk isson +hou ss +her ter +hen nes +h fp +guitar uk +gom el +god ating +foo trace +finn jones +field ps +fe ad +fate stay +emirate spalace +elvisduran show +ele igh +dysp horic +dau bed +dal le +collo quia +co hle +clari fications +cla v +choir master +chandrase karan +cer d +cdn tech +carry cot +can ina +cajam arca +ca ino +business school +brown hills +box borough +bor ak +big west +bet d +ben well +barber motorpark +bal by +ba aba +az epam +av isa +atur ns +arav ali +am amos +a ção +! ðŁĴ« +ðŁĮĬ ðŁĮ´ +ìĿ´ìĬ¹ íĽĪ +æķ ° +âľ Ń +á´ ¬ +wi sen +wherearethey now +weston supermare +war do +w xc +vertical farming +v tu +ussk iteam +ur rea +under construction +tus shkapoor +trans versal +ton n +todd terry +the kate +tas os +subli mity +subed ar +stop smoking +stipp led +spo ol +son gever +so haib +smu n +skirk ja +shen zhou +shaun w +shaun avon +set suko +san ct +sair force +s than +quir rell +prog metal +pro mat +pr ati +port age +poli dori +phi us +per ls +paper towns +pa un +p wy +p mh +p boro +oz s +owler report +orak po +or dine +oli gom +oc ton +ny gren +now play +not done +no ory +no kes +nikifor ov +net ballers +ness man +nam aqua +nae em +movie actress +middle wood +mha feez +met life +melani ep +megan bata +meaghan rath +mari ac +mar steller +lw anga +lou kas +line t +like an +li ag +laure tte +land holders +la stampa +l bg +kk d +kim jonghyun +ken za +kat yal +kali sto +k mb +justice forall +ju mana +jan ek +is oc +ike chukwu +icon ference +ic ome +i mortal +hydro philic +hou da +hotro d +hind march +heritag er +hat pin +greenpeace uk +gre x +goo ge +giga watt +gerry dick +gay weho +gas conade +for ding +folklor ama +fitness girl +eurefre sults +err le +eric decker +emu eagles +ele giac +e use +e king +dore mus +dom biv +cu tis +crank bait +cranes bill +clou ds +clothing brand +cling man +california adventure +but chart +bur gu +bu scan +boul mer +boston logan +bdn wheatkings +bar um +b zz +ay ziggy +ath o +ash kan +ash ankar +art museums +ar lovski +ap gov +and aya +ajen ews +afro z +adam hillscomedy +ac grayling +abish mathew +ðŁķ µï¸ı +îĢ Ī: +åľ ¨ +اÙĦع اÙĦÙħ +za q +z anni +yum mies +you uuuuuuuu +writer sclub +whole someness +wg ms +wear red +wales comiccon +w cd +vit ar +vater drumsticks +van am +un winnable +uht red +ub co +tx u +thy pe +tawad ros +ta jh +swi de +sveng ali +stroke play +stre ator +stir k +sol die +slo f +sf sketchfest +sebastian kurz +s bootcamp +ro cred +rel le +rain i +qua shing +qu or +py xis +produc ers +prakash raj +pir and +patt il +pasqu ini +pari eur +p mik +outnum bered +one show +om h +oik ou +o tram +nam pak +my cu +mut ineers +mosqu eda +montre zl +mi thra +mi roh +meji as +med aka +manchester fire +main frames +mahoo sive +macn cheese +ma quettes +luc ke +lowes racing +london eye +lo lllll +laff ite +ki seop +k wid +jody watley +jo a +ja akko +ipl final +iphone sia +in memoriam +ihave adream +huiz ar +horse woman +hom eric +ho ef +hm ms +hel las +hart mut +har mandir +gu pt +gra flex +gon char +gh u +gadget show +flamin ia +faz lur +fa rell +ether ic +esh un +er rs +ecma script +dream chasers +dawg sup +dan el +cyto pathology +cx nats +cul verts +corner gas +columbi arecords +co ens +clé ment +chill ingham +cha is +bur la +budd leia +bree de +bou lud +borden town +bo ving +bit ored +billy gardell +bbcradiof oyle +bbcradi okent +bas ell +bal me +ax id +aren dal +ap ine +an tak +an ky +ak osu +air langga +ad au +' & +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +ðŁĮŀ ðŁĮĬ +ìļ ± +é£ Ľ +è¢ « +âĢ ¡ +yaf fa +wyn ton +woo delijah +winchen don +whereyou at +whatson in +walk together +voten ow +vi seu +val las +un di +ulster bank +u dub +ty bee +truecol ors +track man +tosc ani +ti kes +ten penny +ten no +tem pests +sword sup +stra ph +st cw +sportb ild +spec ters +sophielady deparis +slipper y +shing ler +ser ved +sei gel +sal amand +sai etam +sa are +ruang bin +ru gani +ros elli +rock os +rishi kapoor +reic her +rd top +rachi eskar +rachieskar sten +power trip +pope bars +per rier +pas chim +ourtown ourteam +om igu +note card +ner vi +national chocolate +narrati vely +na ie +n kab +montan ari +me ha +man gel +ma hu +ma am +m ras +lord shiva +li ah +leg ado +learning together +lan cel +lagun as +la pe +la bra +krewel layasmine +kra ak +kh ruangbin +ker stin +ken ney +kellogg sus +kan ani +kai mana +k line +johno ates +jacc journals +j js +inn outburger +ie fun +hum are +honori fic +hog wood +history today +happy spring +hair pins +green fingers +geis mar +ge wand +fun ner +fresh radio +free education +freddi eroach +extro verts +euro pride +enation ale +elm wood +el ac +edy the +dumb arton +drop shipping +dragon pride +dorm ition +di el +desi gual +davis son +dash ti +dar yn +dal am +d wr +cu scus +ct fcofficial +cry wolf +corri endo +com yn +colla bro +co ge +co bram +co bos +clin ker +chees i +ch ci +cag ed +bulldog ge +bo ken +bluedot festival +bb alli +bar nt +baek sang +av anagh +annon ces +ancestry hour +alp ade +ak aw +air fest +aga ints +acce sori +acade mi +' ~ +ðŁĻı ðŁĴĸ +ðŁĺįðŁĺį ðŁijĮ +ðŁĴ° ðŁĴ¸ +ðŁĴªðŁı¼ # +ðŁĴªðŁı» # +ðŁijį ðŁĩºðŁĩ¸ +âĢĶâĢĶ - +ziggo dome +your struly +yoon kook +whom st +wheath ampstead +walnut creek +visit neworleans +vin ik +victor io +vas co +vallec as +ut aro +un das +ugly ducklings +u sparalympics +try sail +thereal redman +the cc +the catch +team elite +ta kato +swaraj ya +supportw restlers +suni dos +steve earle +sp ms +so gn +sksk sk +securethe border +s giving +ross moor +rosel ine +rock line +ri yan +recal gary +rec itations +read by +razorback bsb +rath drum +rak hee +qu im +pri ors +pri dgen +pre calc +prat ama +play field +perpetu al +perpetr ate +pembro lizumab +pari etal +parac el +oxford street +op ere +o sin +nyul angone +nut job +ntw ales +new gate +near sightedness +ne wapp +nal le +nahu atl +mor nig +mo gov +miy ata +mini ato +mc caleb +mass incarceration +man tu +mak ka +ma shie +lincoln motorco +left most +la hs +l fe +kru mping +kolly wud +kira kira +keys borough +ker zner +ke ad +kathy najimy +kappak app +jo st +islington bc +ingh i +ilove monet +ier ce +ic hal +i aquinta +hom os +hoho kam +hin kson +hawk shead +hahahahahahahaha hahahaha +gre mory +grazi ani +gi rod +ghanaf aofficial +galvan ising +fur se +foramini fera +food hall +fe ku +fe igns +excre tion +ex py +euro barometer +esc anada +epida urus +epic research +end ora +eh burun +e weather +du bu +disambigu ation +desen zano +de couple +d stl +craw for +courte ously +cor teo +cor bally +cooper ations +clare ss +cing ular +chir inos +childhoo dobesity +cann ady +by ler +bucadi beppo +bru tus +brand design +bl itt +big baby +bethe force +beh naz +beauty products +bbcsport scot +base balli +ban ki +b drms +ay bar +av lent +aus media +ar ghhh +ar dg +ap tnnews +anne applebaum +ak if +ait zaz +agul ati +admon ish +abelli o +ab ic +:) !! +ðŁĺĹ ðŁĺĹ +ãĤ¤ãĥ Ĭ +z illi +weg mann +wat seka +view bug +vien tos +victor wanyama +var daman +va sek +uy uki +uy ajola +urin ates +tun ned +tu lane +tranquili zers +to victory +tj dillashaw +timand sid +tho lidays +the piecehall +ted wheeler +tb s +tam o +tal anoa +supportthe swiss +sun web +su ff +statik selekt +staffies aturday +sri harikota +square up +snar ks +sm ate +sh eart +sax lokk +sat su +sas ana +santinof ontana +sa st +ru the +rr rawlings +ros sen +ron go +roel of +real denise +raz ole +provinci al +primer os +pot ties +pin ions +patron ized +pas ukan +p smith +ou ji +open signal +onepur suit +ol am +oh sc +ocean port +o pro +nouri el +nano pore +n bi +mr cc +mr ancelotti +moulin rouge +mon opol +mj biz +mic dro +mi mmi +mer anti +meg myers +matsu take +mak ge +learning disabilities +lanes boro +lamb ada +lam ers +lac to +la thers +la hn +ku lak +ke fka +jeanne ret +jazz nation +it cell +indic ting +i wai +ho dy +hec ke +ha uk +gwen n +grow ler +global trade +glaze brook +gend arme +gen nes +gen ee +ge xpo +fu er +from nov +for bach +fly guy +fivea anews +film director +feen stra +famil ys +etsu tough +eru th +eris kay +episo devi +ense i +emb run +dÅį terra +dor able +dj john +dha de +deu el +dal awang +da aaay +cul bert +cosmic gate +com ity +chi ana +chau bey +charl ene +cd cs +catt in +cat ts +canadas wonderland +busines spro +bur ts +brecken ridge +br aryn +borgh ini +bol lock +big room +bi king +bee flamb +beartooth band +be cht +baw dsey +barri ster +bar dic +babylon ians +ato dos +ati ger +artistre sidency +aristi des +arch ons +aquin nah +alesha official +ade kar +abar nes +ðŁĻ į +ðŁĹ Ĵ +ðŁij¶ ðŁij¶ +ðŁIJ¥ ðŁIJ¥ +ðĿĺ Ĥ +âĸ ĵ +ภľ +ภ© +yu ja +yl t +yan es +ya ha +wwee volution +widen er +white gate +wautu th +was sen +vy jay +voci ferous +vivienne westwood +villar rica +vili fying +ven nes +vast a +up allnight +univers iteit +un encrypted +tip the +thermo fisher +the ecnl +take meto +t bon +suc cor +stor mi +stan wix +spl ans +spar r +sna sh +sm caen +skipp ack +shireen mazari +shan ola +shanola hampton +sh pe +se gol +science innov +schu errle +sato shil +saietam hankar +sag arma +roy ndtv +rogerk ver +richar dayoade +refin ers +rec ency +rational ise +qay amat +q ps +prit char +prannoy royndtv +perri ello +paulow nia +patern alism +paro dy +ox man +no shoes +ni aid +national relaxationday +nail design +my work +mw amba +mou sc +morwen na +mit smr +mcga w +mas set +mar ai +maple leaf +man ea +m ity +liber ata +let there +le fe +land ini +la paglia +ku hlmann +king glass +king air +ke c +kal las +k schi +jol lie +jeho vahs +is so +is forever +is aca +iowa hawkeyes +info blaze +i zza +i bru +hot man +home fires +hau ter +happy jiminday +gric huk +gri eve +gold ar +go ian +ginacar ano +gad h +g police +francis ville +fer gie +evalu ative +ent ure +enamel pins +eleph ante +duc tal +du rov +dou bl +din om +diet ary +de vol +de is +construc t +comence m +ci hi +chit ter +chickas aw +chef slife +cel ty +cel anese +cau sative +camel toe +califor niag +buff o +brighton museums +black hearts +bian co +bac chae +baby shower +azu cena +at asco +as ai +arap aho +anni bale +anne music +ancho ress +an boy +alo x +al bam +ad alia +ab akery +. âĢ¢ +ðŁĺı ðŁĺĪ +ðŁĴª ðŁıĨ +ðŁİĵ # +ìħ ĺ +æĽ´ æĸ° +ãĤ £ +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢ +âĿĮ âŃķï¸ı +áµ ķ +à° ¿ +اÙĦ ار +айд ан +za ke +yan ick +wu ld +woo oooooo +wis casset +westen ra +well house +warner music +visit singapore +vinyl mation +vic hai +vau tour +twitch ell +triath lons +topdrawer soccer +throm bus +the stormers +telerad yo +sta ston +sports marketing +spor tn +sli mmy +sk un +sk aro +siff ert +shi ong +she aths +schind ler +salvationarmy uk +sacred heart +sa ev +s fts +roll humps +rodri quez +research lab +regi er +rebellion racing +real jeffrey +ptc punjabi +prep xtra +portu gues +pic us +pha ya +perkins will +peng en +pe il +pavi thra +panto graph +overs atur +ontari os +ob olus +oasi s +o keefe +o date +nuc football +ni block +newzealand shooting +new pic +ne maha +nbapra yers +nb forum +natsci wk +nati vist +mut ter +multi strada +mor awi +mollu sks +micro organism +met zinger +meit ner +med anta +mayo l +mattb ourne +mar saxlokk +ma juli +m lo +m hq +lym phoe +lu fisto +ky lene +kom pakt +kirch berg +kim jiwon +kim jaejoong +kati el +kali m +kali kimaka +k sco +iyan la +ikar uga +hu u +hog ben +hay ami +gujar ati +graywolf press +gra ds +gor is +fron tal +for s +flow in +fir stuk +fiel dy +exper twitness +exhum ation +estro bel +ep silon +ee w +dotte rel +dock land +diss a +digi ov +del bonis +de math +de aky +dd able +dave hill +cosi danews +cos sette +cooking channel +coo pe +coconut grove +coale scence +claress ashields +civit an +ch asse +ch alco +candid at +bush heritage +bu kitt +btsport football +bor tz +blm national +best giftever +berlay mont +ber bera +ba ofeng +athanasi ou +art se +are iner +apenn ines +am undi +alysi areiner +alo tta +al preps +akh dar +akak atie +ahadra zamir +acram pton +ac custom +? ..." +... ðŁĺįðŁĺįðŁĺį +ðŁļ Ń +ðŁĩ¹ðŁĩ ´ +à´ ļ +Ä § +yut ani +wont forget +var ty +upri ghts +tu pa +trape zo +ton ites +tile work +thor sten +tere sina +tech live +sud ley +style dotcom +strate gi +stephen kamos +spar ke +snel grove +sharpe ville +semis onic +sbse urovision +sal sac +sa ho +s wnt +revenge ofthe +rat ory +rancher ia +q tr +piran esi +picture show +pi as +penetr ator +pattan aik +paloaltontw ks +oul try +os mani +ophy tes +ome gas +ole ds +ol inium +of texas +nfl honors +nc wts +nbak icks +n newi +mug la +monop rice +mon áe +mol fetta +mohawk austin +mo lest +mo apa +mir jana +middle school +micro graph +mecon opsis +mc menemy +mau romanzi +man gam +mal lah +mac graw +ly kos +low brow +lock n +li mas +leighton buzzard +lang lais +lale h +lac ity +kuch era +kid sto +kent in +kell yosbourne +kander steg +journe yof +johnny sauter +jenny slate +je wison +je ane +jaun pur +jame se +israeli apartheidweek +innovators mindset +infe ctive +infan cia +indigen es +in scri +iklan laku +i bike +hypo thermic +hyper loop +hy mers +har ge +guar ini +great lakes +good friday +gf wc +g pu +florid alife +fer nan +feel my +fat emi +export ation +expediti ously +evidence based +ess y +eg ge +ed oll +ec ml +dundalk stadium +down hole +do fc +deno ted +del tic +db q +d ki +couples therapy +cm da +cine bl +chth onic +chelt litfest +charity jobs +casio pea +by ington +buck nell +bu kavu +brink worth +bor do +bog of +bifur cated +bhagy ashree +bha sa +beech nut +bb clondon +battle creek +aux in +att itudinal +apu esta +angel adu +and counting +ðŁĺĪ # +ðŁĵ½ : +ðŁij¯ ðŁij¯ðŁij¯ +ìķĦ ëĿ¼ +ëįĶë ³´ìĿ´ì¦Ī +ç ¯ +âľħ , +âĺĢ ðŁĮ´ +à¤ı à¤ķ +za anse +z wer +yon tour +yo bs +yearofthe bird +year old +xx xo +women against +weapon ization +water berg +was me +vail mtn +un circumcised +ub ong +u equalsu +the movie +teh seen +taxcutsand jobsact +tar u +ta wheed +super kick +sub tler +star ves +staff picks +squ inty +spla ys +sof fic +sof america +sher mano +sher loc +sheffield sharks +shap ley +shahi da +sha hr +sh ve +ser ps +scifis unday +science isfun +sat ra +saint louis +ro kko +reyn or +re tai +pro jared +preven tive +prest wood +pre cooked +port us +pont cysyllte +pod fest +pet sitting +perturb ator +pepper mint +part agas +pal ay +ori e +oneof my +no pressure +naray anas +my soul +my hre +murdere ss +mund sson +moun ts +montour sville +messer smith +men with +medical tourism +med chem +me azza +marsh alled +man oran +man ase +les ss +le auge +lati han +lat ini +lass ical +kelving rove +kat barrell +kam ak +jor da +jesper sen +jason momoa +jal ep +ira c +ic able +i earn +han az +hacker rank +grow ingthe +grand aughter +giant stalk +gat ch +g aca +furio so +for al +flight plan +first dogon +fir dau +final score +er dal +emer ic +east carolina +duc at +du enas +dra gunov +dilett ante +di aa +dete sted +de ferment +david baldacci +cu kes +cu entos +coy gib +cos way +cof state +co let +chin apa +chimp sin +cat enary +cardu elis +buck walter +blue gills +blair stown +ben nevis +bday in +auggie pride +atwater village +as kra +army football +alis al +alebri jes +ala fia +ade adepitan +a itis +... ", +ðŁĺį ðŁĴĥ +ðŁIJ¢ ðŁIJ¢ +åº Ĺ +ãĥŁ ãĤ¢ +ãģĻ ãĤĭ +âĨ ĺ +ย à¸ĩ +ب ÙĦ +z andra +ye su +ye ho +winnipeg sd +win cities +wil m +wil dy +waz za +wal ters +victory venkatesh +vi el +vel ika +vej le +us the +tyrrhen ian +trze wik +tri xx +travis fimmel +transvest ites +tr ant +timefor wiltshire +thebachelorette finale +the vijaymallya +team rubicon +tax cuts +tab ou +stopthe pressure +spu tum +son ni +sj m +semi h +sat chell +sapphi relv +sa kon +s sum +ryan sheckler +rugby romania +rn cm +rebel des +pipp in +paul kingston +park let +paint in +on zalez +omni vore +official foxes +nive thathomas +next conf +newyork jets +mongol rally +monday nigh +mol teni +modi for +mill sand +member news +mechat ronic +mayak oba +marriageequ aility +man servant +ma rey +m str +ly udmila +lego starwars +land race +l kl +kirky ard +kir ani +kil ic +keiyn anlonsdale +kan awa +kalisto wwe +jonjo oneill +jon ne +johnny gill +jin hyuk +jd bc +james ville +jag d +ja ip +j clark +j any +iz elos +it rust +in british +ig ins +ib v +i pec +homolog ation +holi er +head ford +happybirthday suriya +h ki +gli wice +fro dd +fon dre +flag football +ey nsford +en casing +edy ta +earnest ness +dun ja +dro pou +dnrr ts +dit official +district champs +dinesh karthik +din ajpur +dere kk +dend rite +dehuman ized +craft scounciluk +counsel or +cou zens +conven to +con uk +commu tation +colon izer +co xy +ch ré +car star +can h +cam ac +black musicmonth +bhadra k +bein sportsusa +bean er +ballan trae +balag tas +bad alamenti +back ings +ay ork +as gar +all sorts +aldu bun +akin ola +ai ir +ach ap +abz love +absol ved +abol ition +a heart +(- _ +!! ??? +ðŁĺĥ ðŁĺĦ +ðŁĺº ðŁĺº +ðŁĩ¨ ðŁĩº +ðŁĨĺðŁĨĺ ðŁĨĺðŁĨĺ +âĹ¾ ï¸ı +à¸į า +à¤Ń à¤ķ +z sas +y igal +y ho +x sweat +won th +westlife music +we ahps +vikh roli +un certified +trish stratus +trishstratus com +tran slivesmatter +tor q +top cow +tho resby +sydney derby +sun bel +stol le +sti verne +stere otactic +stage hands +st ful +son ge +sl qld +show stopping +show room +shakun tala +sergi us +senator leahy +sed ski +seble febvre +se gues +sb meunier +satisfactor ily +salut ary +salmon arm +sacro iliac +rugby club +rif fi +rfr driven +red squirrel +rabin dranathtagore +pror sum +presiden cies +premi o +pe trick +pay master +oo hs +onehappy island +on radionow +on dra +old city +ni ma +nd ong +my le +multiple xing +morning ireland +monday madness +mo zi +me urs +martin scorsese +ly nieg +lou ella +little more +legac yof +le estrobel +lancashirec are +la skin +la sch +khe da +kang er +kal icious +ka be +j clayfield +ire alestate +ing season +ing ay +infinity ward +inciner ators +hv ff +hun wx +hol royd +ha gh +guine an +grand hotel +gr itty +gol fe +forhonor game +fomen ting +fla grantly +finn aly +fat so +far ley +ey d +ent ra +ec fr +earth changes +e gli +dvs bjp +dutch sassenach +dun ne +dod gy +dip day +dichotom ous +day ao +da ja +cul ter +crooz ef +crewe alex +content creation +con very +com sat +climatec entral +cin dere +choir boy +chen es +c grand +bri stly +brew erton +bragg ing +boosie official +bol on +bisp hen +barbour ville +au tent +at exas +armin ia +ari bbons +$ " +ðŁĺĬ ðŁijįðŁı» +ðŁĶµ ðŁĶµ +ðŁİ¥ ðŁİ¥ +âĿ¤ï¸ı ðŁĻĪ +ઠ¿ +Ùħ ØŃ +zip lock +zind ag +zen y +z hin +yo hanna +yan chep +xen u +ww h +wild horn +wil tz +whist ling +watch nerd +vj fan +ve eee +v alia +u vp +trans rightsarehumanrights +trac kies +tr ong +toyo tat +top bloke +tin dale +tic kers +theodo sius +thel word +t Äģ +t xi +t life +super furry +sum ner +stra thal +stan sberry +stag ey +soci alistic +simple minds +sim guru +shi pley +sch mo +sc ath +sal kinstitute +riyad h +ripp led +region alli +re gran +rab wah +quan trill +pu ya +pu tre +pp en +por tait +po yer +plang lobal +ox eye +out ly +nu mark +now ar +nord kapp +ni kh +never forget +ne spress +na ilogical +mu kul +mr sm +mo hale +mmor pgs +melli fera +mediaev alis +med hi +md phd +mascaren has +mallikar jun +ma sha +lor dand +listen ings +lets work +les doggg +lac of +kum is +klu gman +ke ch +kathleen lights +kar d +jyo tika +justy na +jimmy tatro +jere m +je eta +jamal khashoggi +jac y +ivani sevic +indistin ct +inci pient +ic kes +hush puppy +ht city +ho en +he kla +guer os +food history +flash tvwriters +fa uns +exten sibility +every can +euro tour +er da +ent ini +elitch gardens +ef endi +dublin ladiesg +dj c +dismoun ted +depu tized +d ze +curbyou renthusiasm +cook stoves +comeback home +co di +co chine +cierr aramirez +christma sat +chri sv +chap ul +castell icycling +bruce buffer +brownlee tri +bra der +boeh mer +bezer ra +believein science +bek end +band es +ban kin +ay ey +as crs +argu elles +ar ild +aniso tropic +ame i +altere go +afate hi +adri aan +ador as +. __. ++ +. +ðŁĺİ ! +ðŁıĭï¸ı âĢįâĻĤï¸ı +ðŁ§ ¨ +ப த +yo hn +yaf oods +woo de +with asmile +wild dog +wex for +wacky wednesday +vinogra dov +vici ousness +van pool +van il +vali dations +vachi er +urban ag +ur ate +up ward +unsun gheroes +tom oki +tit mouse +the full +th ah +taarab t +stopp ing +steve kornacki +sports bet +smar ta +six flag +shrey ast +shashi kapoor +sd pi +rose mond +rha egar +rei ver +ran ter +rac gp +r nt +quanti fied +py bus +pre dating +pras tha +pi rogue +pay porte +p stn +outra dio +out size +on ita +o ez +nt k +norr köping +ncp speaks +nasty y +napp ingday +nad z +nab f +mul enga +msc athy +mon tie +mo gensen +mn gov +mira bal +mg book +menshealth week +men gi +melane sian +lc willi +l ound +kap liskova +kak u +john force +joann ac +jam elle +isal oniofficial +illustration oftheday +il las +ik oma +id gi +hyun suk +hoo ker +home built +hein le +h ml +h itec +gun geon +grü ner +greater manchester +gojay sgo +girlscout cookies +gi elen +gda ÅĦsk +gateway msp +gand ini +for migration +flat worm +finalfantasy vii +fet tuc +ey yc +esp ence +erry day +ero yal +el t +dubu isson +dro ma +dre x +dont frack +dom on +do ree +diony sius +dic icco +daysof giveaways +cour ter +comuni dad +co ag +clow ne +clemen za +clack mannan +civil s +cate chist +cash less +carlos vives +call toaction +c jv +bu suk +bu soga +breaking dawn +brain awarenessweek +boeing lovers +birch mere +bic hir +benef ice +bed does +bc swx +awwwww www +au pe +ati shi +as prin +army cadetsuk +ar um +american us +ame me +amber rudd +air print +a hahahahahah +!!!!!!!!!!!!!!!! !!!!! +ðŁĺį ðŁĺĮ +ï¸ıâĥ£ âŀĸ +ãĢ ī +âı ¯ +ë ns +zha o +z hir +womensmarchon washington +we yl +wc x +walk outs +w ampus +vote selena +vainglory game +un blinking +u clou +twat ter +trelli ses +track way +tom ino +thewire magazine +theli brary +th ly +term ly +te vans +tam pers +summer of +stereo scopy +star in +spatio temporal +son u +so called +sl on +sigma beauty +shopp rs +shopprs drugmart +sevasto va +se eu +sally kohn +ror onoa +ri zz +ri xon +redband society +red cedar +re imagin +rathfr iland +raman ath +r vel +quo ter +qu ico +pur itan +pu ss +protec te +pro lac +prin cy +pr icked +pioneer sports +pi hl +pc m +paint balling +ow boy +oper cula +oli mpo +newfound land +net w +multi polar +ms me +mo hinder +metten berger +lympho id +logi stically +lock land +lek ker +ker io +jas curtissmith +jap heth +jam rud +issa char +in schools +ic ba +hil mi +hereto fore +happybirthday tomhiddleston +happy camper +hamilton police +grrr graphics +gp news +gore tober +getin the +gar anti +for sch +first nation +firel ord +fe o +exi le +everything is +escheric hia +emmit sburg +elec teds +egal itarianism +ed scifest +eag u +ds india +down burst +dj sam +dit ties +demo tt +d zn +countyo fla +cor rente +colom b +cla ssc +civic engagement +circuit spa +cer ita +cb spittsburgh +case ys +cad walla +buildthat wall +bu rian +bsnl corporate +bru dder +brian dozier +brahma stra +bor y +bo gos +blu eridge +bio log +benzo yl +be lek +bach pan +ask fc +ar haus +aq pk +animal s +andrew garfield +alex marquez +afl cat +ðŁķµï¸ı âĢįâĻĢï¸ı +ðŁijĩðŁijĩðŁijĩðŁijĩ ðŁijĩðŁijĩ +ðŁıĭ ï¸ı +ãĤ¿ ãĤª +ãģ· ãĤĮ +yu ha +yo in +y ooooooo +wish master +wis bb +win noch +wiffle ball +wh ch +was now +war sop +wak ening +w gy +ver gel +vasek pospisil +us gs +ty rer +ty lers +twilight zone +thre estoo +the dani +terri irwin +te ez +sr anieri +spla sher +sopra steria +skysports golf +si bil +shreyast alpade +shipp eo +shel lie +sg haze +serv ile +sens orium +sd ons +sark cess +sant al +sag rad +sa wang +rey mond +regent street +re clu +rayal aseema +r kd +quantumb reak +py pchat +pu lev +pryn ne +pran am +placebo world +pj vfl +pis ky +phoebe j +paul malignaggi +pau lap +par cheesi +oli an +oc ci +obse qui +o hel +nxi vm +noo b +ner ja +neil gaiman +naf to +my on +mutu alism +mur d +mil dert +micro brew +merz bow +mel don +mehe kf +mary lee +lun dell +lucer omexico +lo oooooooooooooooo +le val +le mann +kru dd +ke dua +kal itta +justinbieber updates +jo stens +jae won +jack ingram +iz anami +iv t +it our +iri sranieri +ing tons +incon el +illi m +ili kazi +icon gress +iam humayunsaeed +i ley +howit smade +hiphop culture +highfive day +he tzel +he mic +hat oum +ha aaaa +gun du +goog lead +g gie +furiou spete +food matters +fire break +ev ang +esk ridge +en rico +electoral college +ef nd +demon io +das ari +dan akil +daily bread +crosscu tters +cot ours +commen taires +cineplex movies +chu ke +chop house +chae un +cathy newman +can to +c illiers +byrne offic +but su +bush ranger +build april +blockchain news +blephar itis +belle ville +band aran +bal ram +autumne quinox +at ate +arrivab ene +anfer nee +and ance +an aw +amnesty online +aldi shodge +ald n +alberta party +aha w +ac na +abhi jeet +aashi q +. ðŁĴ« +ðŁĹ ij +ðŁķ IJ +ðĿĺ ª +ï¸ıâĥ£ % +ç ¢ +åħ « +âĿ¤ï¸ı âľĮï¸ı +É ´ +ysby ty +wolf of +wire taps +win nen +weigh ton +vis wanath +vare se +var sh +v mo +u paz +ton t +toga dia +titmouse inc +tir pitz +the goo +the en +ter psic +tax returns +tal mage +tal espin +syl vio +sword and +sty mest +street games +stateli braryn +statelibraryn sw +star lit +stan collymore +spaz zing +sk ender +schlu ss +ricky martin +ri bon +rep ta +relat edness +ream ers +re pointing +re grouped +rd brwanda +privi le +primiti ves +politic shour +plato oning +philharmon iker +pe cz +outra ging +outnumbered fnc +ound stone +ome coming +nh ss +ne ills +natu real +mustang monday +mus kets +mul ally +min ty +meth il +me pauloavelino +margol yes +mar sal +mar lean +manage ability +man joo +mal volio +louise hay +lo cky +ln dont +like sfor +lein art +laurent fabius +lau rak +lang staff +lake head +l fb +kö ni +kut u +kimjong kook +kim chee +kaz oku +jyrki katainen +just wann +indiebook sblast +hyper glycemia +hun ath +hub ner +hu en +hel wan +h pl +gru dging +goth ams +gir orosa +gf dl +geoc ello +fuen labrada +fluor inated +fal chion +fad ell +ev am +el or +ed urne +ecm records +dos barth +die sem +demon ess +de michelis +dale jarrett +cu sa +cre atec +coach t +cher an +centri petal +centre for +cc cu +cañ ada +carpetb agger +cap elin +camel ford +californi achrome +buz by +bru cel +brit onedirection +blow y +bett pride +beth hart +basso onist +bahau ddin +baaaaa ack +atleti smo +assemblye lections +asha ikh +as usual +ari ola +arac al +and harry +and alou +amo dels +affl icting +ad ric +ad ow +................ ....... +.. ðŁĺ³ +" âĿ¤ï¸ı +ðĿŁ ı +îIJ į +ìĭľìļ° 민 +âĻ¥ï¸ıâĻ¥ï¸ı âĻ¥ï¸ıâĻ¥ï¸ı +âķ Ŀ +๠Ħ +ÛĮ Ûģ +zom b +yas elf +wonder ous +we oy +viva x +victori afalls +v do +under world +ultra suede +trafford centre +thecameron boyce +the killing +the captain +tech awards +tabby cat +sw gn +stron omy +stre s +sto be +steer age +statedept spox +spo em +speci aled +son oma +social saturday +sli fer +show t +scar th +saf adi +roxeter aribbons +roman us +rock paper +rock mart +redbulle sports +red fox +ram ar +rade macher +pyri dine +pslon espn +ps ja +pra der +porttal bot +play like +pern illa +pat miletich +passe ig +park haejin +pag ode +pack mensbball +pac ini +osc e +oh sehun +od ka +octo bers +oc sd +no vos +new amsterdam +na she +n he +mz ilikazi +my future +mun ny +mu as +mrmark millar +miccosu kee +matur ities +malay alee +make ssense +mai sie +mah nke +magdal eno +madon nina +mac ul +local love +list ing +life lesson +li ot +letter boxing +lat ri +lamar z +kha rel +kapp adel +ka ard +k camp +johnny orlando +jo geshwari +it secure +is yettocome +ir vana +igh ted +ic r +hoo kahs +hell zarmy +happybirthday ssmb +go bison +gille smarini +gill mor +gau dette +future proof +fondaz ion +e tech +dtop beautyworld +diam anté +di zen +dar cie +crown royal +con ro +col er +coffee houses +cho dy +chi ao +cer y +cash master +boys lax +black rod +black holes +black dog +big ass +big as +beau pre +aw p +aw oo +au cd +asi ant +as cj +as app +artbasel miami +amo ss +aly ona +ach ro +aa shi +!! ðŁĶ¥ +ðŁĺį âĿ¤ï¸ıâĿ¤ï¸ı +ðŁĺ± âĿ¤ï¸ı +ðŁĶĿðŁĶĿ ðŁĶĿ +ðŁ§ ª +ê´ ij +ãĥ¼ãĥ ŀ +ãģ¡ ãĤĥ +स म +é on +zu ehl +zent angle +yu sh +yoko ham +yn books +yamaham otogp +x sd +wu yi +white oak +vs v +vend redi +vashish tha +ty oung +twit r +ti ssa +thelast airbender +the sz +the post +the fanatic +the batman +the barn +that ches +ter ah +tech tip +tam ente +take two +synucle in +sy strom +summer learning +sug u +still births +state visit +spir ing +sou ley +sol at +sir ken +sham arie +sd urham +sale ha +sab ena +rosenber gradio +roll chos +rock ymtn +rev richardcoles +re formatting +ra abe +pyaar ke +pu dhu +property investment +pron k +prem peh +poly morphic +plains boro +pi fy +paridhi official +owen sville +our blues +ortho tic +orange county +one hit +oncein alifetime +official fpl +och il +noordinary park +need more +na jm +n wi +my voice +mul loy +monopoli stic +minne dosa +mesqu ita +men za +me sures +marketing derby +mark bam +mari aton +magni ficient +madagas can +ma bille +ly tic +liquid metal +line game +l xi +koo b +kap an +k wo +k kinen +juice bar +jim town +jason taylor +jaku bowski +interro gator +insomni acs +in orbit +in cube +ilu lissat +il ala +hu p +hotel chocolat +high castle +hi ggle +harhar mahadev +har tt +hal cones +hain pyaarke +h eneral +guineap ig +greg cipes +gra hi +gordon sville +ghost light +george s +gari bay +gander bal +flex time +fish scale +fight newsasia +ferru ginous +et trick +ei ge +eber hart +e ile +dzhok har +du mars +direction ers +dinner with +delor aine +delac our +de merits +dari usz +dan anda +cor iginal +com port +cityof stjohns +chu eca +ch ya +bul ling +buddy holly +bru gman +bri enz +bri ans +break sthe +brazil vs +bon field +bod mer +black ham +beer news +bath lifemag +barber shop +ban y +as chaf +ane gada +and yl +always keepfighting +afghanist anyouneversee +acqu i +aco ach +ac red +abb ington +^ ; +ðŁĻĪ ðŁĺĺ +ðŁĺī " +ðŁĺĪ âĿ¤ï¸ı +ðŁĮĪ âĿ¤ï¸ı +ìĶ¨ ìĬ¤íĥĢ +åħ ĥ +âĿķ âĿķ +z ica +wiz ened +win co +white boy +whi les +video today +ves z +var um +unite foreurope +unic um +typhoon team +twcnews roc +tren holm +toiletek prem +tart aglia +ta ints +sun daze +stor ian +steff london +ste ez +so hr +sher gold +shepp ard +sean j +sealteam cbs +sd mc +scott derrickson +schwar ze +sant olan +saad hariri +s aper +rep ú +rein car +recou ped +re mon +raf typhoonteam +prokhor ov +probation ers +predic tion +pla sterers +pic public +pel sall +pe dium +park hyungsik +pac ke +p ten +or theast +op sal +op art +old fashioned +oh snap +of oundation +nu k +nsc n +noc news +nl w +nikki benz +nik phillips +ni gri +ne ek +nar singh +n ulty +mö bius +mur komen +muj taba +mt bs +mobili er +mo tti +min aur +mil grom +mei jer +meet me +me len +matt kindt +marchin march +madhu kishwar +lo fa +liv tyler +lilli putian +ligh thi +li bo +lfc tour +leone tti +lend ingclub +l te +ky k +kristy n +kar asu +k love +itsecure gamer +inv itee +inter face +in churches +im hyunsik +hyo sung +hillsong united +hiberni anfc +hann um +h wd +grime ys +green leaf +gom anga +gof theweek +ger v +ger ring +geof fre +fun hau +fra ss +fox holes +food academy +flu dd +ferr one +fau stian +fal zone +fairfiel dhalls +es net +enqu iring +ear flap +dud don +du pain +dou h +don quixote +de dan +dam in +dak shina +cro co +craw lies +cli m +che pe +ch ona +ce bo +cary atids +cartoon saloon +capta insp +cape zio +c tica +buil ten +blavat nik +bigsky mbb +bb najia +aw st +ato vic +arch icad +aniche be +alz ado +ali mi +ale agu +aco aches +, £ +ðŁĵ § +ðŁĴ° # +ðŁį» @ +ðŁĮį ðŁĮİ +ðŁĩ¿ðŁĩ ¼ +ðŁĩºðŁĩ¸ ðŁĩ¬ðŁĩ§ +ãĤ¦ãĤ © +âĺķ âĺķâĺķ +âĸ¬ âĸ¬ +window pub +will an +wester lund +wemb ury +wei z +un wired +u ih +trump world +tradition alism +tomor ro +ter apia +tan nic +swa the +stri ppy +st kitchen +st ator +spi aggia +so ay +sing appen +shermano aks +sha ima +selek tah +schir ra +sali ence +ro castle +rick steves +rhy n +regenerative medicine +rahu lr +ra zy +positivecoach us +pos is +pir aten +pi enza +phoebej tonkin +pey roux +penny mordaunt +penguin book +over comer +ott mar +orange shirtday +or dos +open to +open society +ofsted news +nomin ator +nialla ppreciationday +new ent +nab or +n ão +my banff +musk er +music box +mun dt +mtn training +mol in +miz pah +mind control +mer sch +mean green +marlon brando +market day +man ica +löf ven +lusit ano +loyal sock +l bhf +ku f +kri hanoff +kindergar tens +kgal ema +ker f +keg worth +kal ba +jonm chu +je ggings +itu mbi +i isd +hur ries +ho del +hashi motos +happy mondays +greenflash beer +gjallar horn +fun t +fu ssing +freu denberg +evening chron +evan escent +en stadion +en li +en eng +emer y +eddi ggs +eber ron +dys regulation +dumfries shire +drive srt +down pipes +dom ode +do vi +dick ory +deal in +dave eddiggs +cyfarth fa +cryp ts +cro ix +cro cody +conju gates +cma openaccess +clo va +ciarab ravo +choic est +cho es +chel on +celesti als +car acci +cape hart +buy itnow +bur nishing +bugs bunny +broad band +bra ue +bon usu +blasphe mer +bestin travel +baz oo +azu mah +at bristol +asitis official +asi asoci +apost ates +annual meeting +and ito +amar an +alys sum +alltogether now +allinwith chris +akh gar +aj opinion +ais yah +ade ma +abi bi +ab dy +[... ]" +.. ðŁĺĺ +ðŁļ£ âĢįâĻĢï¸ı +ðŁĻĪ ðŁĻĬ +ðŁĶ¥ ðŁĺĤ +ìķ ł +à± ģ +° - +zea xanthin +your quote +yak umo +wy rick +weare bangalore +we mo +war lal +wak rah +vien nois +veri fications +uw gb +tusc on +tt ank +troy trojan +tos link +til is +the struts +the square +tax ila +tahaj jud +syring a +syd al +stra sberg +stor ino +sreed har +sport news +south la +software developer +sk off +si ona +shangha inese +shack ney +scou gar +rv sed +rockstar spud +rm sothebys +ri pl +proprie torship +pro ss +photograph ically +phenter mine +phase out +pe gging +paul deveno +part ys +p wa +out put +out ines +or que +ok one +nyakun dih +nu suk +nove m +new profi +net weaver +ne ot +nb sat +napalm records +musical uk +moven pick +moss op +mo so +mi eri +mets rewind +meta search +merry man +meh tab +mar clay +maiden head +litur gies +letsgo flyers +lechu ga +lari more +lanter ne +land trust +laber into +klein hans +kidap awan +kat chi +kam boj +kal isz +k é +ju bba +jorgeramos news +j sw +iron bark +ine wa +in oran +ideac ellular +hey ne +hex adecimal +hemo dynamic +he ssen +haydn jones +hand bills +gru ene +grow the +gretsch usa +gooo al +good toknow +go sho +go sei +go il +freeall arrested +for bury +fin cen +file maker +fear rts +evolu tionists +es ben +engle bert +eli ak +dur can +dit er +dante wada +dan rather +daler mehndi +d jane +cy world +comp ell +clo ves +cic lista +chol angio +charlotter usse +car bor +cap oue +buzz r +bur goo +bra sse +bol dini +boh dan +billion aire +bha sker +bemel mans +beach vb +barbar acrampton +bar ik +aval kyrie +au brac +as cal +appare l +ak ick +aa o +\ âĺº/ +) ðŁĺĤ +ìĻ ķ +âĮ ¨ï¸ı +Õ¡ Õ +zen n +yo te +y allop +wo tt +weird beard +w pc +vogel sang +vand or +ultra sa +trump colluded +triple m +timid ity +tian men +three word +thebold type +th ops +th oooo +tess gerritsen +tejas wi +taylor kitsch +tar k +swi ffer +su hsd +started fromthe +sr f +sou dha +soon ish +son theroad +soder strom +sig ar +sennheis erusa +sch ley +sant ner +sa way +s johnson +ru lon +resi sti +raj kapoor +rad key +plow right +pic keted +pha d +per cept +per alejo +pema quid +patrio tic +paras ympathetic +pak tika +org is +orange amps +ol au +o jt +nice day +nat cap +nandamuri balakrishna +n indies +mor ghulis +monk seaton +mazel tov +mat ura +mariaton tini +man si +man kins +mali shka +male fic +mal tag +mak ran +mae gan +ma resca +love theatre +lord swood +loch gil +lily hammer +licht man +li kers +li bia +li bi +ley bridge +la vag +l ams +kon itz +kn aggs +kar lis +kam at +kal uga +kair at +ka on +jo brani +jim irsay +ja the +i sim +hywel dda +horn buckle +hi za +hekmat yar +gu energy +gratuit ously +go rebels +give way +ghay al +fishing trip +fis chetti +far da +fabi en +eus kal +es com +eco sia +du ar +denomin ators +del bene +de hesa +coup de +cor gi +constra ins +co kie +chiri qui +chesney hawkes +change your +central bank +cb university +case mates +carra untoo +ca podi +boy stown +bloo dier +ble an +bio remediation +ber til +bar tali +bar ryo +bal ko +b marshall +aw inner +aus geo +ath es +ash ami +as ako +aquaf aba +alle mands +ak havan +agno sticism +afl q +afl power +ab sar +ab ong +ðŁĺĥ ðŁĺį +ê¹Ģ ì¢ħ +Ú© ÙĪ +اÛĮ راÙĨ +ä ger +z wari +z q +young king +yo joe +y fg +wpl g +wmtw tv +weare south +vm wa +viscer ally +val ore +uni part +the storyof +the crystal +ta fen +t jr +sure tte +suffolk wildlife +su thers +su mut +squ anders +springh ead +so rey +sin fully +simm s +seme a +se phor +sarang ha +sal sha +saga ins +red turn +ram us +radi onica +pre me +polon aise +po els +playstati oneu +pi hu +phan art +palu stris +pal misano +pab udget +outdoor play +out music +ont liberal +old friends +ok amura +ode tte +nu star +news readers +neural network +n lighten +n bbj +my artwork +mscathy gonzaga +movie s +moen ch +mit tee +mi halik +menis cal +mag ine +mach loop +lon garm +live veryplay +lit era +lingu ica +lavat ories +lau ber +lat ona +lang ata +lake huron +knu d +kla ssic +kin nikuman +kad dish +jo dee +jap antour +jan ssen +is cc +interior inspo +inst al +indian ambb +in mortales +i vens +humor less +head cover +harvar dg +happy birth +hani f +haha i +gur gaon +gun smithing +great white +gra ben +good read +gim let +gg ae +germanshorthai redpointer +geor geous +g jer +g adam +flun ky +fi p +fat en +execu tors +ethno logy +est alk +el abour +ef arms +e je +dood lin +dof fro +do ted +deutsch en +determin ate +de itz +cre pe +corn u +coo tam +continu ities +columbia journ +classic films +claire holt +cl ario +châ tel +chief srugby +chal ker +ch the +center parc +caroliner hea +capric ho +can cun +can aday +cam pp +ber land +ber dan +ban chan +bab uji +ba aa +austin healey +armani exchange +ar jen +anemon efish +andre ana +andreana vedo +alu x +absten tions +aac tas +\\ \ +! ðŁĺ¡ +ðŁĺİ ðŁĮ´ +ðŁĺģ ! +ðŁĹ ¯ +ðŁĴª ðŁıĢ +âĿĹï¸ı @ +âľĮðŁı¼ # +yn hs +y gl +wise shopper +whatmakesme happy +way bill +vo key +vo isins +vampi rism +uw f +unce asingly +un mentioned +un impeded +ugly sweater +uc chi +u winnipeg +tran sur +tom ok +the odds +tex tes +tac tfully +syd fest +stopbrexit save +stin co +steven mnuchin +sor tium +solom ita +so tn +silvers miths +silke borg +schotten stein +san zo +sam winchester +rust led +ru xton +ru tt +roy ston +rival schallenge +rish fa +rise again +rig ours +ri or +repre zent +refe reed +r antanen +pwn age +pure michigan +pro mark +prithvi theatre +pride aux +pre spa +pre edy +polyphen ol +pieceof me +personi fying +palit choke +pa kor +over flying +oo ow +nifty warehouse +ne aq +nay py +nak usp +n ør +muumu u +mukun da +mor ng +month long +michael smith +metho dists +mem ri +mcallen isd +markj weather +mahindr aracing +ma wer +ma bus +lc clondon +ku leuven +klo of +kir ill +kant ner +kalin owski +k ichi +juven al +joe manchin +jesus freak +jenn colella +jem al +iwant clips +inflam mable +in ic +if nt +ida ireland +hudson sbay +hert smere +heati son +hayden byerly +han nover +h di +gre tton +giff ard +ghis lain +ge sser +gan ton +funhau steam +fun t +fuch sias +four five +fonten elle +fiber art +fc basel +family values +et pt +eri ff +earl xsweat +e map +dy er +do jos +die ffen +de files +david c +da ji +cou shatta +chi sholm +che sh +channel uk +cc dc +cash time +car fest +calder on +cald beck +c india +bway con +bre ssler +bi bis +berg quist +beal ach +bay shore +bartol om +badtam eez +az abu +ati sh +appeti sing +anti balas +andre ja +anand amayi +almaz an +alit abattle +ali sher +alexand ro +akame gak +ai ves +acon roy +ach ef +absur dism +abhishek bachchan +... ðŁijį +! ...... +ðŁĴĺ ðŁĺį +ðŁĴĥðŁı½ ðŁĴĥðŁı½ +ðŁij ¢ +ðŁıĢ ðŁĴª +îĦĨ îĦĨ +ë¶ Ī +ê° ľ +Í Ī +wind ber +what abou +wen jun +we o +ver bas +valle lunga +ush kowitz +urin ary +uni directional +twin brook +twic elight +tom udall +to doro +threestoo ges +ther im +the je +the cam +the broad +the bol +th nk +th band +teng en +tam bora +tai yo +t ft +summer festival +su santo +stark weather +sor bara +skin ks +sil denafil +shuben acadie +se mm +se j +san ilac +sam ant +salesforce ohana +sain tramrahim +said hanshika +sadhguru quotes +s mic +s key +roev wade +rif fe +re constructs +pura skar +profun do +pres nell +pra vasi +pol kas +po gs +pink hair +pepit one +pa de +p ú +orca dian +oni rose +oman is +o hed +nu age +not vivoree +no ty +nico tin +newsc lt +nct zen +nascar salutes +mrs gif +mrsgif letcher +movie actor +mour vedre +mo gha +micron ized +mi asto +me myself +max illa +matsu shima +mato sin +mandy rose +mak ens +mag ala +madele ines +ma vens +ma snow +loch end +living my +lease back +land sman +kyr sten +krish nagiri +kei ko +kap ur +kangaroo island +kade tt +ka stur +k dei +just my +jonas son +jimmer fredette +jerry can +intro biz +inf n +i era +i dium +hy del +hul stone +history matters +han dovers +hampton roads +greif swald +gold ner +gim bel +gau ci +ga res +form labs +forde ast +fil ma +fedor as +fau stine +fanta sizes +fa oi +f dn +f bn +etou ffee +entre at +en ature +elis sa +el ddis +ecol ts +demonstra ble +de regulate +de my +de and +daz dillinger +dallas comiccon +dach stein +d nam +custom shop +cuis ine +cox sac +corof in +containeri zation +com modus +ci gi +celebs godating +carrieann inaba +cap stan +campo bello +cal ama +caf fenero +bus sum +brown ells +brooklands museu +bha sma +benji bananas +bedro ck +be jewelled +be awesome +avi spa +av go +atla irport +armen trout +anikan onirose +andr é +and ur +and ale +amc kee +ab radley +a jac +ðŁĺŃ ðŁĺĤðŁĺŃðŁĺĤ +ðŁIJŁ ðŁIJŁ +ðŁĮ¸ ðŁįĥ +âĿ¤ ðŁĮ¹ +⾨ ðŁĴĹ +âļ¾ï¸ı : +иРµ +z ue +you ro +wolf song +win ecountry +wi eden +whispering bob +wal las +vinyl meplease +umi zoomi +twit te +tv at +tul fo +tribun a +tom sula +to travel +ti zzle +thisisirish food +thi amine +syd nee +supp leness +su has +sonic maxpro +somnam bu +snow line +sme x +small caps +sky high +silk road +shiv aj +shibu tani +sem la +seaw alls +seatt let +sea otter +schi ffman +s ftp +rosal ba +revent on +rec sys +re facing +r ni +plo eg +pe skov +ou trank +ott en +ong niel +one man +o sten +new mutants +ne onics +monk land +men sclothing +melane sia +medi mmune +mcga han +mary kill +mark uk +mar win +major can +magal haes +madam ex +machine tools +ma bius +lle gamos +land art +lady beard +kur up +kun gla +kend zior +k khh +je ev +it startshere +in music +in ish +igers france +hyp mic +house hotel +home chef +here fords +he hee +hay am +has bara +happ i +gu ffey +gitex techweek +git ane +ger gely +geo storm +gate keeping +gat ting +gal oob +fu ly +from heaven +for deco +feni ans +fantas ylg +fair pay +euro satory +emmas later +down able +dow en +di za +df j +der aa +de mu +dan er +daaa amn +cross on +con gs +civic a +circum navigating +champur rado +cham ling +cham ar +celebr itye +carrerac up +bun nie +bli ssed +bant z +bally mena +baby cakes +are e +antro bus +anal o +amph lett +al bro +ai ki +ah sd +. ðŁ¤£ +( ^^ +! | +ðŁĸĸ ðŁı» +ï· » +ì² ¸ +âĺºï¸ı ðŁĺĬ +à´ £ +Í Ī + ¶ +zind abaad +yur man +ys ay +wool folk +wine shop +wigan warriors +we u +wau ke +vi ole +vam o +un no +tylero akley +tu mi +tree less +tor ra +timo f +ti zi +themy scira +theben forster +the south +the hollow +tel ma +te vita +tar quini +ta kaya +t sou +sub genre +stell aracal +ss occer +sno win +simon says +show you +sep tima +sch moke +save bsoazad +sau de +saddle up +s dogs +run ciman +row en +row botham +rm hs +ri stor +reco do +re portable +re groups +re eagency +ra shaad +quick quotes +pyroman cer +puj ari +pu go +prosely tizing +pirand ello +pick pocketing +pha sic +ph ryne +peugeot sport +petro u +peter thiel +perform in +pe trac +pani agua +pac ke +pa hari +p ge +ou risme +od l +noval iches +newcastle hosps +new country +neil d +navy blues +natural medicine +mor atti +moon bin +mihon ishida +mic hon +mesh el +mck endrick +mar stellaracal +man ak +mach aca +lin thorpe +lei dos +laur diy +lamon gan +l wt +ku sa +kol hs +kis ch +ki ano +keith richards +kan sans +k upa +k club +jon kortajarena +jo ico +j bt +insta quote +ineffec tiveness +ignomin ious +ici ousness +hy yh +hoo yah +hippoly ta +health month +hal las +hagi asophia +h wi +gob blers +gn clive +gnclive well +girl sss +gan z +gad olinium +g ör +fy ffest +friday freebie +free kesha +first look +fin acle +far maid +fall river +fala hee +em mets +e kin +don julio +cran berry +coal mining +cliff avril +clas ica +church land +chugh tai +christ offerson +chinese art +chi veon +car acol +cap tian +campe sina +ca kra +bre z +black lives +bit wise +beh nam +bed ale +barry allen +bar ral +balne ario +bal krishna +badrinath ki +back road +auto dro +attle foracause +as sail +arte mi +apartment sfor +ap ba +anand skfc +aldubb attleforacause +agu ard +ad ino +ach eron +abram ov +ab ente +ðŁĺĤðŁĺĤðŁĺĤ ðŁijĮ +ðŁĵ· © +ðŁĮŀ . +ðŁĮ¼ðŁĮ¼ ðŁĮ¼ +ìĿ´ì¢ħ ìĦĿ +ãĢ½ï¸ı ðŁıĢ +zy gon +zelmer low +zak arian +zabludo wicz +y th +woo snam +won derer +w trf +w sa +vocali ze +v sop +usc s +uni kl +un tried +uclou vain +tu gger +tre gs +transcathe ter +tom rw +tom men +time slots +thursday treat +tho dari +then aked +the record +the hive +teentit an +te brau +tailor made +sur ti +sun art +step children +standupp addle +stan bridge +sr lfc +sportat orium +sense mble +sec ta +seabourn cruise +salomon running +safe space +s foods +ru ine +redwood city +re settling +re fa +ran ong +ralli art +q outes +pocon o +piero gies +pi ppy +perfect fit +pand as +p forz +ox igen +or co +ofic ina +north africa +no dame +nikk ic +nicol led +monch hichi +mon daw +mo vers +minim inter +min aya +milos z +medic aid +matosin hos +mark jin +mariash river +main aand +lyon dell +luc ci +lemb ah +lace work +la king +kschi thra +konop ka +ko tta +ko ek +ki bra +kay le +kann adiga +int nl +infr inges +in on +im ready +heavy duty +head lee +hcp sarea +gur s +gor dano +go squirrels +go getit +gilligan sisland +gil breath +fri ant +fr ath +fa thead +es rd +el j +ed elson +ec lass +dv antage +down towno +domic iliary +do ber +di enes +devo y +debbie allen +dang ly +curious er +crystal ball +cre de +coor ong +cokestudio africa +click ers +church warden +char twell +chamele on +car ica +cad aques +brown bag +brock worth +bo ere +blackpanther party +bla ker +bin der +big ride +big give +bha vi +becau ser +ballagha derreen +bah ra +bag y +ay aku +atter see +athar vaa +angel sinamerica +anc afe +an sara +amsterdam se +am elle +almod ó +ali ot +ad amp +ac tioned +ac ron +ac el +a ana ++ ). +ðŁ§ ŀ +íĶĮë Ŀ¼ +⼠± +ÅĤ a +yung blud +yo gend +wick y +weir racing +wedem boyz +wah ba +w ame +vishak ha +veen endaal +vee bha +ur schel +theros ary +ther ink +theo walcott +terrac ina +ten yc +tempor ality +tele path +teacup club +te ems +tc bc +tafel musik +sydneyo perahouse +strathal byn +stee les +splen di +span ic +sp olympic +sou treach +so tr +skylar king +shar ica +shand ur +sfu sd +se cho +saving places +sarah shahi +sansevi eria +sab aq +s deb +rosen bloom +ro jas +respe k +redbull za +re tra +re housed +ra ham +r do +pepe jeans +out growth +on fd +on aga +nurder hsv +ni ç +nhs digital +my ron +my ne +my lfc +mw ca +mu rawski +mthe mbu +mon stress +mil ledge +mcca in +maxi priest +matan uska +masay uki +mal hi +ma kabayan +ly rica +lol wut +local art +lef in +lead on +le cu +la it +kyiv post +kup i +ki anna +kcr gwx +joke day +jo ser +jeong in +jam bs +jalpa iguri +j hay +is ud +ingof the +igre ja +ic ure +i ones +hunni ford +hi mura +gui yang +guar do +guan aco +grat on +grameen phone +gossi py +googleexper tuk +gla zers +ge ers +fun s +friende ver +fri so +frees ample +free pick +fleadh cheoil +fit nes +familiesc lose +evi dential +eu h +es ung +episcop alians +em mott +ef conline +ear wigs +dougla ston +directs elling +dem swork +del onte +deadly class +de jeuner +de caro +dc shoes +darke sthour +da aaa +cra u +continuou simprovement +confuci anism +comb ate +co fi +cleo bury +cilli zza +chiz uki +chicken hour +cay abyab +cancer treatment +c src +c ml +by ung +buzz cut +bro war +bro l +bre cher +black by +billy tolley +bee zer +bam l +bam a +bake club +backedby aib +az hs +aro b +ap ass +anthonye tuggle +another magazine +an art +allegre tto +aftershock comix +ach hedin +aber tay +! ðŁĴĺ +ðŁĻĤ ðŁĻĥ +ðŁIJ± ðŁIJ±ðŁIJ± +ðŁİŁï¸ı ðŁİŁï¸ı +âĿ¤ ðŁĩºðŁĩ¸ +е м +z anda +youthem powerment +yl unch +yam assey +women with +winkel mann +wh ay +weis ner +water polo +war master +vis cabarca +vir ta +ven ia +utter back +un fussy +uk orchids +tour neur +time shift +ter kel +tay son +tamar ins +ta ipa +superbowl lii +steph i +spol sky +sor okin +soldie red +sk og +shi ken +se hs +schulich school +say ing +sagarma tha +ry leigh +rocred wings +rock n +remor seless +reed bed +re deployed +pro tips +playstation vr +pel key +parapar au +palit oy +pa heli +oz amiz +ox alate +official willow +official triumph +oc tors +non commercial +ne do +nai z +mrtommy land +model kit +men z +me costa +may ash +mato logists +maroo chy +ma hera +lucky manzano +ltgov delhi +lou rie +lin derman +leuci stic +leez agibbons +leeh sien +le ino +law making +law are +l zr +kri ge +kollywud cinema +kirkle esc +khar is +karai kudi +kapp el +jud moo +jb mauney +jay walker +j ini +itsar ush +inter vista +ine f +i six +how ler +guardian witness +guaj ardo +glow up +gis bergen +gigli otti +gertru dis +gaming pc +fran ti +fluctu ates +fishn tips +ff wd +fab aceae +fa illa +emmaslater dance +el ac +du dleys +du bbs +dremil yamassey +dog boe +de use +de oband +de kton +daniel padilla +dak shin +da hisar +d fc +corbin bleu +city bus +choisi won +ch fi +cel entano +bse india +brockle hurst +bro dus +brit actor +britactor sfan +born free +blogger life +black burne +bird land +bell labs +be fell +bb cr +bal laugh +au nee +astar oth +arag ami +app ens +an american +alzheimer sday +almodó var +al port +air ings +adeni z +acol lusion +ach ary +________ __ +ðŁĺ© ðŁĴĶ +ðŁĶµâļªï¸ı ðŁĶµâļªï¸ı +ðŁĶ¥ " +ðĿij Ĵ +ìµľ ìĬ¹ +è¡ ĵ +å®® èĦĩ +âľĮðŁı» # +اÙĨ ÙĬ +yogur tland +yarmol enko +yankeec andle +y sabel +wri ddhi +wire image +wessel mann +war daz +vis vim +uttamav illain +uchicagop ress +ubc tbirds +ty ms +tu larosa +tre bor +toyo tasa +tou reiffel +tor mey +toff oli +timber lands +tiger football +thisi sse +thero se +thelaw society +the ba +ter zi +tananlam az +sub o +stage it +spokane indians +socal gas +sj games +si vi +si dd +seta pak +savi ation +sav arin +roar ke +ro saleen +rel an +re gnier +raiz awilson +r dh +py ré +plate a +pavan wadeyar +pas sa +par ki +papad akis +panneer selvam +pand anus +orange ade +o stara +o hau +nostal gically +nicolled wallace +nde geocello +nam po +my president +mont ages +mis sa +mel bre +medline plus +mcken nitt +mat en +mariek ondo +mar oni +mar ma +ma kan +livepd fantasylg +ladies fc +l yoko +ku kush +kor angi +kom ple +ko g +kkun drra +kensington wm +ken oother +kapil mishra +k anner +jabarda sth +ic td +horn book +ha pand +grigor yan +git u +gg f +georgin io +freep sports +fred matiangi +fly fish +floren cio +fla thead +fl ers +first group +face spics +ew snow +eri ght +er got +ene sco +elek trik +e ick +dt cc +drum life +dol t +deod har +de tracts +cricket nation +coming back +cochine al +cnd world +ch our +cdw corp +can ora +call o +bu duc +brisbanecity qld +brett dennen +bi eta +bed wyn +bed narek +bar bu +backing green +b icon +ashley banjo +ar tel +an tron +an bieding +albor z +aj mal +ahl berg +abil is +abatto irs +ðŁİ© ðŁİ© +ê´ Ģ +ó ria +z art +york city +yard goats +wl ns +win nowing +win de +wilden stein +wild water +weare marshall +we thu +watch fam +washington ville +wage theft +wac ke +vocali zations +under manned +un zip +trag icomedy +tow boat +to kimeki +thor ton +thelead cnn +tar om +tann eries +sur co +sreed haran +sp inde +sony xperia +social science +smo te +sma shedit +sivas spor +shop era +shaunw keaveny +shar bino +shadow box +se malam +schro er +saturn awards +sam at +sal ameh +sac ré +roast y +ro kin +respe to +re dis +radio graphs +q ni +prescrip tion +peter parker +ox ox +oun slow +oakham ales +nor mies +nitto tire +nigh a +nd grade +nati vism +my cause +mur derby +mon arda +miss jillscott +mindful monday +middle weights +mickey hart +melody jkt +me tsu +mcfar lane +masa ku +marchfor truth +maj e +mainaand kingangi +lwk md +lec l +lans downe +lafarge holcim +ladu ree +la ina +la ffin +kwame alexander +kum manam +kro kus +kem boi +ke vitch +ke iser +kathy raven +karun akaran +jeky lland +je ga +jar lena +irri gators +in quests +in ni +ic ot +homeaway fromhome +ho way +hilari on +heu mann +he ur +harnessracing fz +happybirthday prabhas +ham bo +grybau skaite +gran ter +grammy museum +goe ttingen +girl ss +gigan tea +geor dies +fv cking +fromm ay +fran kies +fou cher +fit ba +evic ts +evangeli ze +er ol +enter ovirus +eleph anti +e eva +driverles scars +dream work +doit right +dis arms +de funded +de criminalise +ddfirish open +dat en +dar ach +daniel sen +dani alve +dance plus +d brand +cy d +cory barlog +conglomer ation +colle c +coach works +clarine t +chitra koot +chir ur +chandram ouli +c vi +burton wood +brek ke +blu et +bid ness +barry manilow +avery music +audi gier +attack uk +ar rabbi +ar ao +ar amid +anc tified +an society +amaz one +am ooo +allenand unwin +air bn +aggie pride +acc football +ac ini +abkibaar modisarkaar +^ = +ðŁĺį ðŁĴĸ +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +ðŁĴª ðŁĺį +ðŁij¸ ðŁij¸ +ì° ® +ë ®¤ +ãĤ³ ãĥ¼ãĥ +âĺ ¯ï¸ı +zi ak +z wicker +working families +win dex +westernsydney u +var in +u ep +turkey hunting +tre covery +tour mnl +the mill +temer aire +sur jit +sub mit +stan konia +spinalcordin jury +south en +sor dell +son david +simon books +si ron +si bert +scot x +scoo kies +scare fest +santafen m +sanc ti +s sea +russiainvade dukraine +rs no +rose will +richard hawley +ram my +prosecu tes +procli vity +presidency za +por ur +pod saveamerica +pag cor +pa yot +ott arts +og na +o sports +nieuwe marlean +newsal bany +ne villeg +ne mea +muradali shah +mr cs +mother house +mont and +mill sap +men indee +mal asak +mako ssa +make animpact +lute fisk +lot w +li sk +li i +legionof superheroes +late comer +lat tice +lake mba +l bg +kri sa +kid life +khid mat +kelli wardaz +kari byron +kappaalpha order +k layton +jubi lees +josh malina +it ama +invest ni +internet billofrights +infec tiously +illi quid +ic mr +hy son +hot an +hen rico +heming ford +hellolove goodbye +hear taches +head erfor +gulf shores +greg j +gre eley +gor goroth +goe tt +gh in +gc sb +gaunt lett +fu xk +for ag +figure heads +feet sfriday +fantasy league +f ki +ext ols +elimin ators +eli ber +ele af +echev arria +ead weard +e anna +dz rh +dun levy +duken ation +dream fall +dor ham +dk weirracing +dj sli +dan quah +d crising +cyto plasm +cri stela +crank case +count downs +corticoster oids +con con +co readvocates +cla va +chry stia +chiw enga +charle smil +ch hath +cefal u +capp ucino +cantab rian +c wr +c atters +by uw +br or +boye tt +bir git +biopsycho social +best picture +bell icose +bbc newsbeat +bath ampton +bat alha +bang arang +bandit ry +au tour +assemb lea +artificial grass +archri val +ap fel +andreja pejic +an sal +amu jer +amph icar +american hero +am sel +am angala +adapto gen +. ðŁĺĩ +ðŁİī ðŁİ¶ +ðĿĹ ¦ +ðĿij ĸ +íģ¬ ëĤĺ +íģ¬ëĤĺ íģ +ëĵ ¤ +« « +yy ys +ya hy +x roads +whyi write +waite mata +vidyar thee +varieg ata +val gus +vaj rayana +utilit arianism +usic ology +undp nepal +ul rich +uba id +tur genev +tracy morgan +tr pg +the weather +the french +territ ory +terr atech +temp t +tele suren +telesuren glish +te odo +sv hs +style inspiration +student sfirst +stu ttered +stoo ped +ss art +spi o +sigsauer inc +sharon stone +ser ban +se akay +science center +saras wat +sandi fer +sam billings +sal mi +sak sham +rub én +room nyc +ric keys +ri gas +rei ley +radi ore +py are +punk newwave +promul gated +prob ity +prat ley +pin en +ph anie +pan in +official somo +oel wein +nws boston +no thern +netflix andchill +nbam emes +nay er +mylfc matchdayimage +my daily +my bad +multic ellular +moul ton +mer redin +men hir +meach em +mcclu sky +mal ong +luv v +looking up +logarith m +life sinked +li scard +leehsien loong +lauter bach +la pua +ko cian +kil ob +ki pedia +kar aca +k hub +jo zo +jami ed +j bf +iti me +immun ohisto +hollow ay +helic arrier +han kyu +gas ca +gallery nucleus +fore finger +fo dera +fast ener +f ttc +exer tions +ev ren +elast omers +eis ler +egh radio +ed mc +eclec ticism +dramatic tvactress +dogge rel +dile k +dic tum +dem townhall +de eming +dani o +daily artapp +d pu +cre ese +coton de +coo len +come tothe +columb arium +color ants +cio glu +chev rons +cher ini +campa ña +call ousness +bur kart +bran dishes +brain port +bps official +book design +bess borough +bed knobs +bea del +be toftheday +bas sm +b ics +aw ow +at tr +at tala +asi us +as gupta +around theworld +ando ther +amal thea +alter cations +ale u +al j +ail or +ag rill +acon lin +achi eng +abc perth +ab k +..... !! +... ðŁĻĦ +ðŁĻĮ ⾨ +ðŁĺĤðŁĺĤ ðŁĺĺ +ðĿĹ § +è¡ Į +âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +Ãł n +zi val +yuz uki +yo sa +x tz +warmest dayoftheyear +wall man +wab co +vesper tine +ver hagen +vaidy anathan +uts engage +uni one +uk ko +ud yo +ubis of +u ble +tow and +too tie +too kes +ton us +theother palace +theor yo +tham marat +team parieur +team marco +tart us +tarant ella +tar sem +supportsmaller streams +subtrac tive +string ers +stay ers +st patrick +spil sby +spati o +sor ay +slat kin +si pos +share alike +sel zer +schill aci +schan zer +ru lz +rott nest +ren ter +re start +rashe ed +quasi moto +pol ack +plac id +party poker +partic u +par ri +pall ant +paga dian +pa zz +open mind +onu cle +om ix +odu ba +oc transpo +nu zz +nevilleg aunt +nelli gan +nathan caliendo +mur ga +mor iz +monta ña +moj govuk +mc gorry +masseffect andromeda +man tia +maj ima +lu tea +lode stone +lef kowitz +laur amer +la stra +la quon +ku rashiki +kingston uni +key logger +kar upp +kali dou +just married +ju yal +john daly +ine ver +inconveni ences +holtren frew +ho efer +hasan uddin +gr rrrrr +gen eric +gab ap +fredrik stad +fra ile +fl anagan +first book +f mcc +eri ko +ell ende +ee sha +du mo +down cast +do bry +divyankatri pathi +dip onegoro +desi perkins +david le +cryp tic +cort land +cootam undra +colli o +cla vel +cin tra +ci rio +ce ann +cau dal +cary grant +can struction +by hilton +budd leja +bo gho +bl art +bis mil +birdr inging +bilingu als +biggies malls +be kar +be careful +bc boston +bar sky +bag naia +av eli +art books +around thenfl +ant farm +amand am +al over +agra deci +ach he +ab ella +a beauty +a ade +... âĻ¡ +! ðŁ¤£ +ðŁĶĽ ðŁĶĿ +ðŁİ¥ # +ìĻ Ģ +âĿ¤ , +âĺķï¸ı ðŁIJ¸ +Ùİ Ùij +zar os +wj hg +wind turbine +wide format +whit nall +whist led +wans beck +uniof greenwich +under my +u afootball +twitter arty +tun ick +tric ycles +tri ss +to fur +thankyou lord +terra zza +ter mas +tellu rium +tal abani +ta uri +ta official +supportw yo +squ anto +sp edchat +sor na +shin da +shi row +sh ym +scraw ler +scam bridge +salmag undi +ru derman +rit as +ricar dol +redbull ring +real racing +par x +pack able +onthe table +officiali rishfa +ny ana +nump ty +n ellie +mrschu reads +mi ak +makge olli +mahel ajay +mac ondo +lumi ere +live itup +legiti mizing +lamor na +lam ington +ksh mrmusic +kit kat +kin i +kim itsme +kelsey grammer +kav adhu +ji ren +ji rayu +ji ley +jer rold +isra r +inter line +insur rec +inocul ate +ino v +inf urt +in ther +in skip +ill ings +hul hu +hs live +hossein panahi +ho sford +hfx gov +here ward +hello kitty +han afu +hal flings +had do +gy ratory +goog learts +god ber +gen nie +gail kimitsme +futureis clean +footre sts +flip class +firstdogon moon +fiji water +fant v +et one +esof twitter +en ze +el ittle +ed ris +econom e +ec rs +dr pol +dog man +dirty south +dikt at +dichotom ies +deb harkness +danse use +daga anbieding +d wor +cut down +cumbri auni +crossy road +cros sen +cot to +compare the +com ley +col a +ci le +cc mfc +casc adel +cas ap +cab ella +bu chs +brugman sia +braz ell +bir dies +biblio therapy +behnaz akhgar +b spencer +az al +autum ns +arqi va +ar z +ar ques +andri ukaitis +an ini +an al +am rap +ain da +ahwah nee +adi alogue +abo xer +ab dal +... ðŁĺŃ +) . +ðŁĺĺ âĺºï¸ı +ðŁĴį âĿ¤ï¸ı +ðŁİīðŁİģ ðŁİĪ +ðŁ§IJ ðŁ§IJ +ðŁ¥ĩðŁ¥Ī ðŁ¥ī +ê´ľ ì°® +ãĥIJ ãĥ³ãĥī +âĢĵ ... +म र +Ú© Ø´ +yam ah +versi ones +usa rec +under ling +um g +turi sm +tune n +tom greenlive +tetra pak +tessell ated +tan auan +tak ami +tablo id +sub domain +student nurse +stu hr +stu bbins +strath more +ssoci al +sociol inguistics +sk la +shrews morris +shou ty +sel vin +sch unk +sa ww +s ago +rose tta +rene ef +religionof peace +refu els +reduc tase +redon da +real tristan +rad or +r ning +projec tion +profun dis +pop surrealism +plym ou +pin on +pil ley +pe mc +open weight +once more +om n +om loop +official itm +ny kv +nucle o +nove cento +nim mayash +nie miec +ni had +ni ge +ni eve +nes sus +nd sufootball +natur inaughton +nash y +nar m +mr hs +motley fool +moren te +mongre ls +mol k +mcelli gott +mark mcmorris +mani sharma +mahesh war +mahar aj +lis se +li pan +lav ant +lar ı +kar avan +kal inda +ka aris +k dramas +jul quen +ju mah +john nosta +jethro tull +jar o +it begins +inve ctive +inthe middle +instruc tables +ing bottom +in sincerity +im it +hurl but +hock omo +health grades +he mat +happy jinday +great read +gh f +ge stede +gaur ilan +g biffle +fx ck +frank ly +for charity +falci parum +explore tocreate +exfoli ates +estad ouni +en id +em cer +dylan wang +dull stroom +dete sts +daysof happiness +coo oool +cle te +cl bv +chitt y +chap leau +catch me +bush c +bronchi olitis +broad street +bo kor +big il +beltr ame +bbc panorama +bb bz +bauhin ia +bal ey +b jr +awe sum +aqu ilo +antimal arial +anti k +angrybirds movie +amon dru +al mac +ahor ra +ab os +ðŁĴķ ðŁĻĪ +ðŁĴ¯ ! +ðŁijı ðŁĻı +ðŁijĢ " +ðŁıĪ ðŁĶ¥ +ðĿĻŀ ðĿĻ +ðĿijĸ ðĿij +âĸ Ķ +Ùĥ ر +и ÑĤе +zor ds +zeit lin +ystr day +yn p +xiumin day +women folk +wind pipe +wel ding +we pa +wa ac +vladimir putin +vital ogy +uni z +unex pressed +un dressing +u tube +u alber +tor tora +tony denison +thor ny +thereal autoblog +thejeep boss +the flying +story corps +stie ber +ste mp +so al +sin fin +shiamak official +shenmue hd +sf aulkner +semantic web +sarac en +sar tain +sammy watkins +sak ya +sac town +s dept +ritu ally +ri shab +ri oux +ree de +realestate investor +rat ers +quad er +q cd +pre dated +portu k +plan chette +pla iner +pink tober +pilo thouse +par anj +packer scamp +outre ach +on elu +obli gate +npl nsw +nott jmiller +northco teuk +ni ga +ne leg +my sad +must die +mul tani +muen ch +msd honi +miner alized +mi ked +melbourne rebels +mand saur +macro monday +macleod lisa +ma bon +lunch special +love fool +lo sch +list in +lew ys +laurin burg +lamin ck +laid ler +kn auer +kingsc lere +kelly hoppen +ke mber +k heim +je anie +jan edu +jahn joyce +ja ey +j nl +j fb +it ra +irish athletics +invest ingin +ice pick +iam nathancarter +ia edchat +hutter ite +hong qiao +homi letics +hand ball +ham burglar +ha eger +group suk +gos well +gop shutdown +glycol ysis +glo ben +gi aco +gerring ong +ge bra +gar do +fruit and +fein berg +fat ma +f ager +erit age +er la +end ment +ei jun +dro ege +down hearted +domode dovo +di mock +di gression +dham mapada +dell in +daniele wski +cre aking +cour tiers +cortin as +cook with +contextu alizing +ci pe +child actor +chi usa +cent conf +ce ducation +carol i +candy floss +can adam +cab ri +blue stockings +big hair +ber lyn +battle ship +bass fishntips +aure ole +as quare +artscentre melb +arti ste +ard glass +ap ari +an holt +alph on +alham dol +al ano +aju da +abq journal +abil aa +aar ya +ðŁļĢðŁļĢ ðŁļĢðŁļĢ +ðŁĺĴ ðŁĺĴðŁĺĴðŁĺĴ +ðŁĺįðŁĺį ðŁĺŃ +ðŁĴĭ ðŁĴĸ +ðŁİĤ âĿ¤ï¸ı +ìĪĺ ì§Ģ +ç» Ł +åı £ +à¸Ĺ ำ +اÙģ Ø© +ÑĢом айдан +youknowyou lovethem +women wh +w tr +uninterrup tible +un treatable +uk g +uc susa +tyne dale +tri ston +tim mies +thener d +the breakfastclub +tel er +tail pipes +suren dran +sparkle horse +spac enews +soton bloggers +sne ers +sm lb +shopif yl +sch one +sar us +sale able +sa kay +rugby team +reviv alist +readabook sa +re sund +queen y +propul sive +prom out +pol sk +po stol +petron io +pecz wolle +pate y +palm spring +our councilday +ound le +oti um +or pik +or ne +opera holland +onlin eradio +ok ane +oj simpson +obe tten +o war +nw ssan +nor afatehi +nfl trainingcamp +ne agoe +nbaf reeagency +n vr +mosque shooting +monster girl +miumiu official +may ben +mares me +maic har +mag li +m din +lyondell basell +lo docom +le em +le corbusier +lande cho +land lines +ladies coffeehour +kn filters +kim es +kihu en +ker shaw +ker no +ju bbly +jeremy shada +jeep neys +jare cki +ja jang +isag enix +intere sse +indy fuel +hi ggi +hec kel +har io +h á +grav ina +go kart +gis ella +gir llll +ge res +gam bi +gab r +fu jimura +frog men +forthe union +ff acts +fe iler +fatta hamin +famili ars +evelyne brochu +euro dollar +eu scienceinnov +eri zed +eri ously +eosinop hilic +edward sharpe +e ppie +e jig +e gil +dy fed +dued iligence +don nat +do ges +dent i +den ili +de pil +day in +data point +dan acar +conspiracy theories +clo ying +cl andon +choc taw +charger pride +ce se +carab iners +c scc +ble e +bi planes +be zal +bat as +bar ic +bali kavadhu +awu mia +apriv ate +ad fa +acrif ice +ðŁĻĪ ðŁĻī +ðŁĩ· ðŁĩºðŁĩ +ðŁĩ²ðŁĩ ² +ìĹIJìĿ´ íĭ° +éĿ ¢ +Ùģ ت +Ø® ÙĪ +Ø« ÙĤ +zyl ka +ys w +ye sor +yar ai +ya hia +wheat croft +wap ello +want in +vo p +vir ushka +ven yc +use lessly +un tagged +tw en +tsu ji +tre zor +tn ks +thelast word +thefla bar +team r +strongman burner +stra ks +stoy show +spor tv +som ani +sof er +sneaker holics +shore ham +shar nbrook +sc broncos +says thanks +sarah jan +ru pesh +roc que +ran sparent +quarter maine +proven ce +power wolf +ph onic +peter reckell +perturb ations +perth saint +periscope tv +pere stro +party like +partnership working +par le +p vo +ori fic +on thames +on se +od deven +nt pol +my job +mon sun +moment a +mo hawke +mj h +mississ au +minority report +miner alisation +min cing +mil ius +max in +market smith +mar griet +mai ley +long town +lisan andy +lion t +lam born +lack o +kyo ka +kiku sharda +kad okawa +jehovah swit +j ú +j heel +institutional isation +ili on +i yogibabu +hu gest +green bonds +gra ze +gra da +get surrey +gell horn +gat ron +fuel ledby +freddie mac +flye ia +fer oz +f official +exoplane tapp +ex one +erin andrews +entren ching +eltonjohn dotcom +dz ire +drug policy +dre bin +decor s +de classification +dalecar negie +da than +cryo sphere +crooked media +creative coding +concert series +cel t +ce si +bra zza +border line +book ofthemonth +bobby deol +bo vespa +blue marble +bit ola +ber man +bench mark +bel man +bar bap +bad illo +az ore +at ering +and one +an dere +amdav ad +amb h +amazing world +ale ment +al verson +al caz +ac tr +ab caustralia +aash to +ðŁļ ¤ +ðŁİħðŁı¼ ðŁİĦ +ðŁİĤ ðŁį° +ðŁĩ²ðŁĩ¯ ðŁĩ²ðŁĩ¯ +ðŁĩ µ +ãĤ¹ãĤ¯ ãĥķãĤ§ãĤ¹ +âĢ¦ /âĢ¦/ +zz ap +young sheldon +ym piad +wyn and +women at +willi g +we cam +wan less +wald ner +vil ar +vi stap +vb hardwaj +vag h +us now +uri arte +ur baine +tru ssed +tru del +to god +titansof cosplay +timb res +thisi smo +think different +the empty +thames and +tec tonic +tat yan +tal aat +studi ob +star mall +spanish wine +space plane +sonyo pentennis +sonic youth +som osc +solfe ggio +smar tie +siame se +shore side +sho tof +she han +shark friday +sh man +serv ator +sen dit +saw bone +save forever +sage steele +s burning +rohit vbhardwaj +rock centernyc +river head +ricer ca +restin power +raise theroof +present ation +prepar ando +pose fx +plain smen +pic turi +photome tric +pen alizes +paint ourcountryred +out land +ou lihan +ont sciencectr +off man +ny saf +nun obetten +nix es +nik khil +nav orro +na ini +mw ff +msu bear +mont au +mittel stand +mi ahamm +medi apro +marcus rashford +male fic +ly sette +lunatic fringe +lover anch +lik elier +landol akes +ku bas +ko djia +kel an +jo bling +ji ayi +j simpson +iñ aki +im fact +ical cio +holy prophet +hk n +harms worth +happpp py +h gst +govisit donegal +gear hart +ge mc +fur r +fromthe heart +freedom for +free bet +first data +episode ix +emoun tain +drimn agh +dni propetrovsk +di ffs +dev yani +desol ated +cyto toxic +cro pland +cou pa +co yy +christi ano +char ring +cfas ummit +cel lier +catt olica +cas ely +car ron +ca they +c suf +c family +business world +bu ong +boo ooo +bluebull srugby +best cover +ber tini +b po +b hide +azam garh +arul nith +anne hill +anight club +amo u +ak sha +air lifting +ab baf +ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ +âĻ Ŀ +| âĢ¢ +| " +youn gent +ye lem +x mpp +wuor nos +wrong doers +worldwar z +worl don +visi ón +ver su +up one +u cks +tweeds muir +twd season +tu fn +travis mills +tran sasia +tour an +tot teridge +tin man +ti ë +thelad bible +the code +thd specialt +thdspecialt yops +te poz +t way +t cb +sydney harbour +sura gi +stro zier +stay stron +star bird +squi shing +south yarra +small streamer +skan ks +sk imo +shey i +shaw kat +sha di +sece ded +se de +scul thorpe +scal endar +say antika +saras vati +sar afina +rtel atelateshow +roberts bridge +ri ser +retro game +red dragon +receipt bank +re sour +re nier +ra fan +pli ant +pinstripe bowl +picof day +pear ly +paladins art +paci fied +our planet +oikou mene +norman scat +nfl gameday +newzealand terroristattack +nat bookfest +n ées +n dogo +mur ra +mog wai +mo kel +mo bot +mitch ells +min ner +mer rick +men il +mee go +mat v +mat eu +malate sta +lund by +lon glo +less lie +leip sic +ku las +kit by +ke ala +kan kar +jeffrey deanmorgan +jan an +j iri +inter aksyon +in articulate +hibern ates +hfd cathedral +hemost asis +heidi montag +harps well +gri mble +glu ckman +gif tv +gerard cosmetics +fordair show +ford fiesta +flying lizard +fa zel +endic ott +em boss +elen aof +el row +el al +div ada +disp lease +dis neys +digital inclusion +dif fi +daniel pink +dam aja +dab u +curi g +cur vil +compli mentday +chicou timi +cep heus +cc am +casey stoney +calpur nia +by polls +bryl creem +bre mont +box elder +boom afoo +book tweeter +bolly spy +big land +bho pal +bend y +bemore bulldog +auto didact +at will +ann ayya +al thy +al ila +af zal +achil lea +aap ki +. âĺĢï¸ı +-------------------------------- ---------------- +ðŁĴĹ ðŁIJ¾ +ìĨ IJ +人渣åıįæ´¾èĩªæķijç³» 绣 +Æ Ĵ +¨¨¨¨¨¨¨¨ ¨¨¨¨ +zwar te +zu biri +zo gby +zah ra +your style +yes yet +yash ar +wei den +veloci pede +van doorn +use dom +up setters +unmiss media +un amused +u ise +ty to +tru triciahelfer +trans gress +ton bori +thum amina +the sergiogarcia +th planet +targe ted +sy kora +sw j +suppre ssants +stree ting +st patricks +sports network +sle at +shiv amo +serj tankian +seago ville +s oper +roes ler +riv kin +rin king +rel ite +red l +re go +rc pe +ray rice +que ss +puntag orda +poetry club +pl w +pet teri +parac lete +p texp +oviya army +otta way +ole k +nrl south +ng w +n ber +morro co +mic ca +meinl cymbals +mar kova +manji mup +manav gat +malai kaarora +made lein +mad ingley +mad ill +mad diec +macau gp +m sian +logro ño +little wood +leon arda +kol la +ko stic +keep grinding +jung koo +julien solomita +juilliard school +jaime murray +itsoknotto beok +ir furugby +iphonex r +interrup ter +iam kevingates +hypoten use +holm quist +histri onic +h gw +guildhall school +guant á +ground out +good trouble +goian ia +go pis +gen ix +form alin +film friday +fe tus +evry thing +eudat ap +estor il +eri sta +ep ines +emil ys +elisa bet +eli el +edward norton +ecor re +echever ria +ear ther +e kywx +dramati sation +do tan +dism ally +dia gh +di photo +den el +de ko +dann yo +dal bir +cudd yer +con fort +community first +clanc ys +charlesmil ander +cau tioning +carre ra +cad le +by noe +bro eck +brian may +blue family +bit me +bil ayer +bie bers +bi ene +bhu pen +beit bridge +bat ala +bas smaster +bapti sing +bad ak +b ico +ar trave +anu sh +ano tti +ang ley +analy tically +amor gos +amanda seyfried +ama hal +akamegak ill +air craf +adi son +[ ..] +.. ðŁĺĤðŁĺĤðŁĺĤ +## ## +ðŁĴĹ ðŁĴĭ +âĺº @ +âĦ ĵ +zen er +yeezy season +workat bmo +wil cox +weare lions +water foot +wat more +vintage finery +vanqui shing +ucbt la +tw b +tra ks +tiru vann +theatric als +the peoples +the fresh +the culture +terry ville +ter ate +syncop ation +subo dh +su steren +styx theband +spir anac +sl pp +ski e +shur tle +shu bin +scor chers +scorchers bbl +scam bill +rø de +ry uu +run day +royal nottingham +roseng arten +roo ters +ro ved +restorative justice +rec d +ram k +produc ts +pral ines +po hn +phon te +perry farrell +opp en +om entum +olivi as +ol inger +oil prices +nucle ation +noo ksack +nikkhil advani +nem rut +muzz in +muzi ek +mul ligan +mont seny +monster shockey +money lifers +mo sk +mitt a +mi thi +mchen ry +may il +mate ch +mar jane +mak ris +mac aluso +ma bley +m mol +lions roar +limb u +legend sneverdie +ku si +kqed news +king crimson +kar ok +kane shiro +k jal +ju gni +jm sdf +inform ación +in adequately +i bid +hend ryx +heat world +hard head +gu lags +grand vintagefinery +ghar afa +gar zon +gallinu le +futu h +fried rice +frey tag +forever faster +fluid ly +fli kis +flat sharkfriday +fer it +f tir +ev ng +er kan +e od +dé but +dz mm +dyou th +dogre scu +dixie chicks +disc ordia +di ge +defen siveness +dead of +dac arter +com eta +cm madhyapradesh +chi architecture +chennairain shelp +change agent +ce sta +categor ising +camp illo +c cam +bw ca +brendon hartley +breeze wood +bon soir +blue diamond +black pride +black moor +bla sé +bicycle day +bibek debroy +bergü zarkorel +ber nd +bel apur +beav an +bali ke +az ri +aud ley +att ini +atmosph eric +ash bar +as ale +arulnith itamil +arte verywhere +arom ero +appell ations +ante y +alexiso hanian +alder maston +ah g +acon ite +aard wolf +åĽŀ æķ° +åĨįçĶŁ åĽŀæķ° +öster sund +é ire +à ¿ +zoek ravitz +z inner +yak ushima +wood art +william morris +whites ell +west ling +we sti +wall onie +ver ka +van badham +uni fortheunion +u mac +to cs +tiv ation +the tapi +the hsf +thal ic +tele babad +tc boe +tar kwa +tan trik +tac chini +sustainable seafood +su chy +stin ky +spini fex +somer sethour +skil ton +sepul ch +sci enza +sang ster +road and +res su +relation ships +re so +rand on +raim und +radi kal +qu ater +q lowes +purpose less +pow pow +pix s +petri fied +paul fox +pan american +pa korn +p canada +ow ls +ou ton +orig nal +on tour +omo bile +odi ya +o tunga +nur ture +nrl bulldog +ness week +mon em +mo akley +mma worldseries +mazz arri +mayor gregor +making it +lo aiza +lie bing +ler ay +leng let +la vac +kon tor +ko thi +kill star +khan e +jo konta +jer wood +inner strength +in cirlik +in ap +im un +illion aires +iihf hockey +hu mmm +heather smusical +he arers +hard talk +happy min +gu sev +gre athe +gayath rie +fli es +feno kee +engineer sday +enamor ada +du lli +dro yal +dewy ze +depart amento +denili quin +danielle peazer +danialve sd +danacar vey +cymb al +concur ring +co ins +clari fier +chol ar +chi el +chasti sement +bug zy +boon dox +body positivity +bloodh ound +bla go +bi agini +bath bomb +baby parts +b pt +awe bster +ash all +arr ancar +aq a +apal encia +allegh en +agains thate +adventure dub +adsby google +! ðŁĮ¸ +ðŁĺ³ ðŁĺĤðŁĺĤ +ðŁĺ³ @ +ðŁĴĥðŁĴĥ ðŁĴĥðŁĴĥðŁĴĥ +æģ © +ä» Ļ +à¸Ħภ¥ +Ï Ł +~ ' +yo gar +xmas gifts +wmc actionnews +wearit pink +uof california +ukho zi +uk coastwalk +u hud +tu shy +triste za +trek segafredo +trance music +ten sioned +team wendy +take warning +sto dge +star dock +st joe +st fest +srish ty +sri hari +sony pic +sol arc +siyab onga +sirr fs +signor ia +she el +shar q +sel z +see tha +school shooting +sali k +sal aah +sad atx +rockos modernlife +rit suko +ri mas +reinde er +re applied +rat te +pri ses +pp sells +pir on +pavlyu chenkova +papri kash +palazz olo +pac kexpo +p alike +ousp lants +opent ype +o vp +novis ad +ne wex +nat ore +my jam +moun tallison +mou stapha +moo somin +mo state +mo chizuki +mi sere +mar able +madeto order +lut senko +longstre th +lon hro +league acs +le so +lac tose +l voe +l nm +ku ze +kriti k +kir kelementary +kidsin museums +kc n +jermaine dupri +jazz giants +inter weaving +instap undit +inqu ilab +illinim bb +i pupa +hyper sensitive +hoch berg +hig nett +hi shoot +hernan es +handmade withlove +hand shaking +halloff amer +gun sand +gudi padwa +gre v +grange gorman +goldber gabc +get xo +fl anne +fi dalgo +feder alists +es rc +elf cosmetics +ef ell +eccle sfield +e off +dove tailed +diplom atic +di spiriting +dead locks +david haydnjones +darren aronofsky +dak ini +critt all +cor sair +compli ance +committee woman +chili bowl +cher rapun +chab rol +cash money +cambridge up +calde ira +bu zek +bol tzmann +blood worm +bird wing +bil lah +bhag want +ber ko +ato ols +at d +asi mha +asa pro +ap ta +an erley +aha ve +access oires +ab elson +ðŁĻı ðŁĺĺ +çĪ ± +âĢ ¹ +ب Ø© +Ñĥ ÑĢ +zo tero +zach aria +z ura +young dems +ye hia +ya x +wyo sports +world day +wir th +wil les +whatsin your +vet college +us ke +upaz ila +unk rich +uni fies +uk nighted +u pike +troop ingthe +travel holic +toom any +thisgirl canuk +theak ston +ter yl +ten o +techo bloc +tali on +t de +supri yo +sunny and +sul rich +su dd +st croix +spark s +space program +space ghost +somer ford +smash mouth +sm wnyc +sin dic +shiv shankar +shear man +schmoe down +ru ark +ribon ucle +real dlhughley +radial first +rac ters +quan go +prome sas +pow a +politec nico +pol itti +playboy plus +play z +phil health +petr arca +official rfdtv +ob ay +nigel walsh +nicol lette +nic col +ni gar +natural capital +my kindof +monic ac +miner strong +mi zo +mg mre +mer in +mend acious +mcgin lay +may apur +matthewk oma +materials science +mark downs +mac al +m power +long branch +lochgil phead +llamas oft +live performance +ley ne +lac us +l sl +kumbhal garh +ku dai +ke char +kari mov +kap ila +ka inos +jone goro +john wesley +joequ esada +jam y +iwant my +it l +impre cise +ii fym +ig noble +id aniel +hue ys +housen y +hau er +hart sock +guan eng +gu entzel +gr rm +gone withthe +gn x +give back +gaw thorpe +g dc +frees kiing +fli eger +fer riter +exc ite +ev anovich +eter n +energye ast +dz ic +dou gal +dhan ak +day party +daren th +d tek +cut worm +cunnin gham +cu fi +cri men +crave online +cool ly +collin sjr +clo ture +clavic les +chelt scifest +carnaby london +calcare ous +buffe l +bs gyn +brig and +bo vey +bo il +bi eler +beverly hill +baz alge +bazalge tte +barn house +back plane +baby shambles +ba stani +b nar +atthe beach +ation ists +at aman +articul ations +argo sy +ap oria +animation history +an ted +adventuresof sabrina +abdul kadir +ðŁĺĸ ðŁĺĸðŁĺĸ +ðŁĺĴ " +ðŁĹ ij +ðŁijĬ @ +âĢ³ ) +âĢ¢ | +è ce +zo ila +zak i +zait sev +za har +wonth aggi +wcs district +wai roa +wag staffe +virgin radio +vietnam war +uwh arrie +uss enate +under state +ugly sweater +tw bb +tux ford +tsu gumi +ts ve +trophy less +tri ppie +tor ok +tooth fairy +tom baugh +toib engaluru +to wards +time is +ti mex +thun berg +theland news +tell ing +tee garden +tc palm +tb al +tantrum jas +tagteam titles +strong and +stiff key +sport sand +sou k +si kar +sher inian +shaz aming +sharon tate +sh lomi +sc inemas +sap stat +sag acious +sa ison +rut les +roun drock +rother field +rosel ia +robert liefeld +regi one +refor mists +ram adoss +punks notdead +priyadarsh ini +power bar +pon tian +phyno fino +percy jackson +pa ek +p ds +op hrys +oko h +nor ds +mugg ings +mu barak +mr ts +mou sa +mo pan +mitch inson +mis ao +mi thai +mall er +malcolmb arrett +mainde foe +madison wi +lp family +long drive +lettu ces +letsgo fish +letsgo bluejays +lahore blast +la yl +la wr +la had +knee bone +ken yc +ke yeast +kc z +just jane +jack averymusic +j music +iso pods +in chi +il f +ho sey +guanab ara +gir nar +fleec ed +fla key +fieldwork friday +ex bury +eu onymus +en snare +el acrosse +ek po +ebay deals +dge y +dee sh +daun te +dam usic +dac ascos +cou lomb +cotsw ol +coffe ec +cityof york +christodou lou +chapp atte +cav ill +capo bianco +bul kier +bru jo +brit pod +bristol museum +brady bunch +botanical garden +bom e +bol len +beau ce +be sit +be achi +ban ach +bal tas +bal jit +azhar ali +ated atbirth +ar mon +ang ellic +all ready +aerop onic +advis ory +acul ture +acri mon +aa aj +ĥâĸĥâĸĥâĸĥâĸ ĥâĸĥâĸĥâĸĥâĸ +ðŁĴĻ ðŁĺĬ +ðŁı ¢ +âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +Î ¹ +än der +zet seguin +za im +ym g +yam en +xfiles revival +womenin football +whe ely +visit nsw +vill amil +var ur +uj fm +tur nings +tri lemma +tomas ino +ting in +ti em +thrombo embolism +thenext level +thanks you +tehseen p +taylor ville +tank girl +super mamc +stipul ates +st jinx +som d +snoo py +sloven ly +sini akova +shi garaki +shi bas +se aga +sawbone shex +salsha abilaa +sa has +ru dess +relief ph +quarry men +quadru plet +prin tr +pos thu +port ent +poli she +po ffo +pi b +ph arre +person ae +pel inka +pe psic +os mar +op tom +nr v +nerdy bookclub +nelli safb +ne zetseguin +nayanth ar +nav sari +mus thear +mon toro +mon er +men shair +men ashe +meflo quine +mazar ine +man esar +mack aye +maccab i +mac eration +lance hoyt +kol ten +ko hut +kne ipp +ki dul +keep cup +kbr shanthnu +kart arpur +kapil dev +jj project +ji ho +jared keeso +jam session +jail ene +jag at +israel houghton +isix hosa +ingex pert +im kbrshanthnu +herne hill +heck ingbottom +head water +grue somely +gregarious gus +gov tof +gored foxes +giri ja +ghat ak +gh anim +gaz coombes +gastro esophageal +gam le +gab ino +fry selectronics +for paws +for gan +flat shoes +firepro world +financial advisor +fi it +fe bo +fashion revolution +farn combe +exam inees +es op +dun shau +dontbuy thesun +di val +dday live +dark man +cole k +clayo quot +city comiccon +ci han +chop ta +chi am +catal oni +cap ensis +braue rei +bra ying +blue stein +billye ichner +benazir bhutto +bel ami +band elier +b plant +b mm +atx tvs +arbor vitae +andrew bird +and j +americane wsroom +alitabattle angel +ali h +album review +agu ars +agen der +afl footyshow +aegyp ti +aed rummer +ace to +a ahhhh +a abc +< -> +ðŁĺ» # +ðŁĵ Ł +ðŁĴľ ðŁĺĬ +âľ Ĺ +ص ب +zil los +zay lea +ys an +ye ea +y kids +y ge +wondo lowski +wn n +when the +web deals +wash post +upgrade able +underground hiphop +under stating +ukbiz lunch +ucc as +ty k +tu mut +toni morrison +themercury comau +th ast +tgi bf +techno cratic +susan oo +submer sion +su at +sty nes +stel co +start growthhack +splin ting +speake asies +smtown engsub +sh uru +ser ota +schum peter +sc atters +satra pi +s mino +ric key +rh show +re ino +re conversion +rco bsgyn +qu ynh +py mble +pure blood +pu pil +priyank sharma +ppe dup +plough shares +petal inews +on field +nswe ducation +nh scrisis +newscenter maine +na id +n be +mr tim +mm ts +mic nice +meg awith +med ine +manu script +locomo tor +lin tang +ley la +laureus sport +lane ways +kukush kin +ko lob +kl k +ken de +kaz em +ka ura +joke oftheday +joint address +j das +iy ana +is and +infirm ities +incongru ity +ibmin sight +hulkam ania +hu mawards +how live +hor stman +hin akhan +health coach +have it +hardrock cafe +ha sel +grim stoyshow +good sell +gods love +fro ilan +fm sport +fisher price +final cut +fb heroes +faul tin +fa kers +ex oxo +emph is +emo cion +elenaof avalor +ed codes +east bourne +dun dru +du ur +dog park +digital day +deve aux +del gada +deep space +day byday +daveand busters +da er +cur sos +cruis elife +crop science +cott aging +cor duff +conflu ences +cohe siveness +cel an +cat experience +caprio tti +cag iva +caf ferty +bus bee +buildseries nyc +bu et +bru el +bookof mormon +bo cage +blax land +blavat sky +benav ente +be sic +banc assurance +automotive industry +aston university +approxim ations +amy adams +am mal +alex ius +alber thall +aj arian +ac cc +. ðŁijĢ +. âľĮï¸ı +ðŁĺŃ ðŁĺįâĿ¤ï¸ı +ðŁĺ³ ) +ðŁIJ « +ðŁİīðŁİĤ ðŁİģ +ñ ol +zale ski +z aib +youth hockeyhub +yacht club +woof woof +with cnbctv +wil born +whit efly +west water +west texas +wen gie +weare ky +water colorist +wa ik +w mca +w be +vou ching +visual isations +var ro +v ants +utt lesford +us ing +un feeling +u fp +tory belleci +tiruvann amalai +ti schler +texas music +test drive +te pa +t wh +super malls +sun s +style sheet +spectro meters +sol do +sign ano +sci pol +scher zo +sar b +sanguine tti +san a +samani ego +salaf is +sa wal +ro ep +rit am +rising fc +reaction aries +rang zen +rado van +ra fer +po tty +petro logy +pel ters +par sed +opor tuni +oke fenokee +o ic +nou ak +nor doffro +naz ari +navo daya +nat andalex +nai adi +mun chie +mor uya +monu mental +mir am +mill house +mi mar +met c +mergan sers +mb un +martial law +maro cain +local artist +late junction +l cac +kur kova +krewella jahan +kias oul +kendall ville +jobsat sap +jay atv +jav elina +jar nail +jake snake +jad wal +ir rera +im mobilization +ill ich +heat pumps +har im +grou pltd +go bbi +gin sider +gen l +gautam i +fuji feed +fra is +for mby +food systems +feel on +fast ford +eye inthesky +equestri ans +ef vater +dream it +dh ani +dest inee +derek blasberg +de santo +de mel +dce aglecam +dau di +darth maul +d mi +cw fc +contin i +christen sen +chocl ate +chil duk +chatter is +challenge mtv +chall oner +ca estecker +buz dar +bo als +bloomberg nef +blon del +bil gi +bidden den +bey eler +ber ti +be filter +bc so +bath couk +ban cos +awami league +aw ls +atra vels +annu alised +angelamer kel +and ray +amorpho phallus +amai zing +agh oulihan +ac b +abid sherali +\ : +ðŁĺŃðŁĺŃ ðŁĺį +ðŁĺĮ ðŁĺį +ðĿĺ Ģ +ãģĦ ãģ +âĿ¤ï¸ı ðŁİ¸ +à¹Ĥ ม +غ ز +اÙĦ اÙħ +zu er +z lin +your face +yo hio +yn k +wissen schaft +who afro +white sox +whati wore +west norwood +wak amatsu +wa gh +w abush +vat an +uno suke +ulster wildlife +trul ly +tor il +ti sc +this weekend +thel in +thekenny johnson +theatric ality +tapa chula +tab ulation +ta waf +sunder bans +strong room +stop p +sser vice +snow drifts +sisse ton +shriner shosp +sherri saum +sheil aedrummer +shar m +schul tze +sch ow +saving species +sahl berg +ru sin +romul ans +ro tte +re der +railway stoday +ra gging +quarter backing +qu ashes +prolet arian +por che +play renegade +photo sensitive +pasteuri zation +party city +parkh rc +pa ed +os bournes +oo ks +octa hedron +oce ang +nw mt +nelly furtado +ne si +nase by +ms sen +mr sc +mon ell +mon acan +mom entu +me aker +mayur bhanj +manische witz +mag sa +mac abe +m cree +ludwig shafen +luckywelive hawaii +lodocom ello +lil lahi +lik ha +liber ate +leonard peltier +lc sw +lai kuanlin +l boro +kier vi +ki prop +kerast ase +kcb group +kay yem +kal dor +k gf +jx mmi +jvp live +jo ee +jnd newin +jn tu +ite u +inter nists +indra dhanush +ilo tt +id lesband +hou mous +hex en +hend aye +hani m +ha patra +gwent police +go ar +gli atti +gla snow +gir lof +ger land +garethe mery +freethe children +flow cytometry +flam ini +fe ye +est agram +ent reco +eng les +electro magnet +dst govza +drugab use +dri pper +dl bcl +dese gregate +democrat sare +day today +davidal angrier +david w +dae han +cornell birds +colin cowherd +co ia +co el +cloud busting +clat sop +ci endo +chilling adventuresofsabrina +cha itra +ch kala +central valley +can teens +cam t +caffe y +brueg gemann +bren ham +brace less +boy ss +blo wh +bit on +big rig +big bluerising +bharathanen enu +beauty awards +bar nicle +ballo onist +bag mati +ba rex +aur ic +au ta +asburypark press +ap ada +ande cho +ald inga +albion parkhrc +af ic +________________ _ +( _ +" --- +ðŁij º +ëĤ ł +æ¸ĭ è°· +ઠ® +ب ÛĮ +z eli +ye ley +yan o +women slibrary +willi son +whole food +whitt all +white shell +w lo +visit nh +tz ipi +tomat ina +toledo zoo +thu bb +then ow +then ex +thelong dark +tey hainpyaarke +tar buck +ta bet +t san +suc ci +stu xnet +specu laas +sock puppet +smi leys +sl mpd +shi pre +shannon od +shannonod komo +sgvn sports +se mar +scream official +sc cg +sau dara +sar athi +ry uko +righ ton +rig don +realmike fox +rass lin +radion z +r carter +quote sabout +purlo ined +pueblo bonito +projec tive +pnp benguet +pin ia +piase cki +phil star +pain free +paddy considine +oo ley +onmy way +ok ream +official ntas +of ar +occi dente +o ssia +nz greens +nov is +my chal +music magazine +msc cruises +ms as +monte squi +mon stru +mk malarkey +minof healthug +metax a +metamor ph +me sta +mazdar acing +max okream +mal gudi +maas ante +m doc +love guernsey +lee goldbergabc +lar ale +kom achi +knit ted +kee fe +joanne clifton +jingo istic +jh pie +jg quintel +jam shoro +jahang ir +indonesi agaruda +in sofar +ho thouse +hiphop news +helpin ghand +he ree +ham on +h murray +goun damani +glen rock +gi gh +floral design +film ation +fili povic +ex pl +eun jiwon +eo connor +dun church +dream theater +dirty heads +dicky johnson +di si +davidd avis +cup ation +cou va +corn elio +cor nexchange +che mun +cham bana +cb cy +catalu ña +carolo ates +carlo scon +carbon ates +car ras +c smt +businessc ard +boy ton +bo stik +bleep ed +baltimore ravens +bad ley +auror amax +audit ore +atribe called +arse holes +arr anca +ar saigh +alnwick gazette +ak ino +air bnb +adventure awaits +ach on +ab copen +;____ ; +! & +ðŁĺį âĿ¤ï¸ıðŁĺį +ðŁĺĬ ðŁIJ¶ +âĿ¤ï¸ı ðŁijı +à¹ĢภĦภ+ä n +zy lo +zel az +zam belli +yoshi e +women slax +wiz kids +whitman comics +was u +war ta +viv ant +vin u +vast atin +under city +twicelight sin +torfa en +tik vah +thisishow we +the sc +the official +the legendof +th q +tatt ler +sweeney todd +swan scombe +sw oc +suga day +su azo +studio time +stou dam +stop islam +steep ly +st bcbeer +smy ly +sky boxes +sk yo +shesthe man +sh Åį +se ak +sd in +schoen aerts +scam ps +salute our +s vea +rol lei +rock fall +ro mo +right ness +rh hs +rec afe +rang y +racing usa +q waq +pá draig +pro pp +prachu ap +poly chromatic +place res +pidge y +parthi epan +over it +oppur tunity +oo ak +one on +omat osis +o shii +o asys +nye timber +nunobetten court +nor dha +night lights +news gathering +nar annual +na ig +na ee +n clt +muss ina +mud bound +mu co +moz eliak +mousc ron +mor ges +monso onal +mj d +migo satl +michel isz +michael aconlin +melen chon +marykill speople +marvel s +mad chester +ma iga +loong abba +lincsc athedral +lad bible +kir kk +keeping familiesclose +ke ino +k hil +joedon rooney +ji om +jessi ka +janet mock +jac at +itsme angelie +itsmeangelie ofc +ing change +infer nos +independ anceday +ima ik +illustr ator +hut cheon +head wrap +hang ar +gun buster +gu end +go vikes +gla zer +gis elle +franklin ville +foo ks +focu sed +flu ting +ff mpeg +fabu list +encyclo pedic +el ak +eeeeeeee eee +ec tasis +earn history +ea si +du gald +dramati sts +don adoni +dog ar +divyak ho +digi pack +day breakers +dar da +daniel biss +cra ze +cour town +coste ssey +cooking class +chocol ati +chester ville +chester see +chast ising +charles stur +carne secca +can tin +buland shahr +book clubs +blo ater +black label +big bro +bed kar +be yo +baliunited shop +ayud ham +astri x +as agi +arthur s +art less +antan stead +annel amott +anis arahma +anime today +ang ono +amik kr +alpeng low +almam ater +air bushc +adri ve +ac athletics +!!!! !!" +ðŁļ¨ðŁļ¨ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ðŁļ¨ðŁļ¨ +ðŁĺĪ ðŁıĢ +ðŁĺĩ # +ðŁĺģ ðŁijı +ðŁĶ¥ðŁĶ¥ . +ðŁĩ²ðŁĩ » +ëĪ Ī +ãĥĹ ãĥª +âķŃ âķ® +Å µ +ze idler +ya hara +with that +winter reise +weal den +wake ford +visionary art +v india +un heeded +un decorated +u vo +towand abraxton +tour nai +tlal pan +tilling hast +thur a +theun toldstory +the diamond +tah niah +tab qa +stu me +st stephen +sportsin kansas +shrimp sofficial +sf vae +sem ler +scotland rhshow +sari kaya +sanit ary +sa via +sa aya +s gu +ry stal +rock umentary +ro woon +ro stro +ro stered +rescu eme +rachael ray +pyrac antha +pulver ize +pu late +promo zione +poo jak +pin aco +perfect match +par rs +p tit +onthe air +one wyoming +ol ar +ohed chat +nyche althy +nu thatches +ni elle +nett uno +nb storm +nandamuri kalyan +mussel burgh +musco vite +mud jug +mo ated +millstreet brew +marti ka +makeyour fringe +mahar anap +mag daily +louder than +loop back +lon ers +le ya +laura bell +laurabell bundy +ko ger +kingscross n +kha os +keral ite +ker cher +kelly music +kel me +karne val +joshu agates +jim and +jas ong +itsa jeepthing +isol ators +int fi +indol ent +imagine ers +ic ole +hu meral +home of +hillary shealth +gun di +gr atulations +gior nale +gat ty +freer adio +flx wine +fluori dated +fi da +father christmas +fair lop +ec ran +dragon slayer +dr paul +do ink +derry berry +decor dova +david lloy +cra han +cla sping +chor hai +chint amani +che state +chatter ji +chao imh +chak ma +carrol lisd +cam anach +caf fi +c mk +c jones +buzz ing +bran well +bracken bury +bo twood +blom field +bla x +bemore chill +bed post +bas ant +ap lc +ansp ach +anisi mov +al chemist +ac credit +ab da +ab ali +a strong +ðŁĻıðŁı¼ ðŁĻıðŁı¼ +ðŁĴµ ðŁĴµ +æĽľ æĹ¥ +âĿ¤ï¸ı ðŁĺģ +âĥ£ % +ö mer +ze ppo +yi mou +ww norton +womeninthe arts +wom ex +wi ze +where on +waltham forest +vinyl record +vil oria +vicom te +ven dee +uta at +un ha +un alaska +ultimat ums +uli sses +u schi +twiz tid +tubabu stun +tri kala +travel manitoba +town site +tony danza +tn f +thom mo +thetrade shub +the missing +the max +ten ements +summ icron +staffordshire bullterrier +sp global +sole bury +sit all +si delight +shr u +selfiewith daughter +see tickets +sar agar +sany u +run ing +royal blooduk +richard madden +regi stro +red ale +ra be +quins rugbyunion +psycho therapists +pre eran +poke mons +pocket watch +please dont +photo ssmh +pe texpo +pau ll +patrio tically +padam see +onthe mic +on wood +on sports +om durman +nye pi +nouak chott +nati oggi +nan twich +mtv lebanon +moor thy +mono drama +mo state +mo realive +milwauke etool +mb loggers +may nard +maverick mack +mathi ascor +mat twil +mark knopfler +macand cheese +louisi anat +lolit afashion +lo ey +leit ao +leaving neverland +latavi us +l rn +kuri yama +kreu zer +koso ko +ko pec +kar oli +justin roiland +jump shot +jess alyn +jacket pride +ja hannam +ipu parliament +ingui stic +ik wy +hur v +hi ral +hesj edal +hel an +hei hachi +he ming +har bored +guantá namo +gam is +fr in +flip flop +fiat friday +fen cers +famili ares +euphe mia +ell l +el anna +e guchi +divul ges +disco teca +didyou know +devon shire +destabil ising +dent su +de tto +danger zone +da sein +cy matics +crand ell +comer mx +colossal con +clark dale +clan lupi +chil dof +cher itage +chaudhary mos +charli ea +castle derg +castle comer +cast ilian +can zoni +c cea +black butler +beng haz +bbce urovision +bar wa +b sales +aw ali +avoc at +augu stu +ani x +andre c +amil ies +am ti +alay kum +ais beautiful +ack enzie +abr antes +! ðŁĶ¥ðŁĶ¥ +ðŁĩ®ðŁĩ¹ # +ðŁĩ¨ðŁĩ¦ ' +ìĹIJìĿ´íĭ° ì¦Ī +ãģ Ŀ +âĿ¤ï¸ı : +ᶠł +à¹Ģภļ +à· Ļ +ó t +yeg biz +y tm +world wired +wol ffe +wil lesden +wi od +wel le +vl cc +ushe rette +unis sen +un constrained +u pert +twoo fus +tu loy +tu ffs +tsu en +travel photographer +tra inning +tor ridge +tokam ak +thor gerson +thol land +the vinyl +thal mic +th bday +texas roadhouse +taxonom ies +talmu dic +synop ses +super talent +sumbasturtt ooo +sumbasturttooo sday +state ofe +ssi ers +soul is +smu k +silver alert +se zer +se hat +scots man +sat elite +san alytics +saddle brook +s mets +russ diemon +rspb norfolk +rspbnorfolk linc +rou bini +ronnie fieg +rio jaw +reic hel +rei ki +rare pic +r risd +quo test +qld premier +prow lers +pre ve +plant breeding +pic torials +photogra hy +pe ya +pat ro +over simplification +our planet +osun decides +oo di +oe dema +nkab inde +ner ang +music ally +more so +mon now +mod bury +mit sun +minim i +minic ar +min son +min dover +meto we +melt zer +medi bang +m chan +ly de +ludo vica +lovethi splace +love construction +lored ana +lesc aut +leonard nimoy +lee music +lawenforce men +lan o +lali berte +l do +kush n +krono squartet +kit ching +kimberley jwalsh +ki ddle +kan aloa +jossel yn +jo vem +jes see +je witt +jani kowski +janella salvador +jane eyre +it speter +in ie +il kay +id led +hus ar +h dh +goog let +go wers +glac iology +gil le +gi x +food guide +fol ke +fnb sa +fle isch +finger picking +f cie +dz um +dyskine sia +dundee fc +double ornothing +disfru tar +din kel +desig no +deodor ants +del ille +de veau +da ay +cut more +cub scouts +cook county +common weal +clwyd tweets +circe anna +chin maya +chilli da +ce dros +cat mint +carri on +buse ireann +braun stein +bobs redmill +bir e +belvo irst +bel ing +ban kes +b iti +autor oute +an ory +all hallow +al maj +aguil ar +af fy +adri aen +ad sk +ad me +.... : +ðŁĻĢ ðŁĻĢ +âľ ¡ï¸ı +zar o +yu uko +ye u +ww c +world teamtennis +wi jn +whit sett +wesle ys +watson iot +walt frazier +w bn +vin en +vi ggen +valley field +vak il +ut la +ump ed +ul ys +twee ti +travel deeper +topdrawer london +tiny house +till ers +thi stv +ten ka +tarta kovsky +tac ek +t sparks +sutt les +survi vin +sucess ful +stau ber +spra shanth +sonnen ch +sn itching +sm supermalls +sli vers +sla bajo +shun ts +seanan mcguire +scav olini +sc ake +saraj cox +sandy ford +sand fire +sal sas +saiy uki +rosemary shrager +rip muhammadali +ri ya +rhon dd +ren ta +ren old +reen ie +rec chia +re education +r ferl +quality improvement +pur ps +pull inger +pro lapsed +peta india +pe pero +pe ahen +paste bin +oun taine +oto ko +ni ver +newport folkfest +never y +music i +mu sina +mon twood +modular synth +mississ inewa +mcin doe +mari yam +mar os +mar ing +mand ola +macca bee +loe we +lo cash +liven good +le rena +lavo isier +lam bi +koo ser +ko leos +khush want +k lun +k cee +jamesh askell +ja afari +ithac acollege +ira sci +indy to +idu bbbz +huar az +ho plr +hit sugaya +hi va +her bo +han lan +ha sc +gre tag +grand baby +gor ls +gor aiders +geni y +gargan tua +full kit +flat lands +fel id +falken spotting +ezz at +extrac orporeal +en larger +eat slee +dor si +dev log +demon ised +dash board +dar bari +dan sa +da ij +d mart +d conf +cu bing +county gov +collo quy +cat boy +castig lia +calix to +calend ers +ca via +burn sday +bud gies +bs fc +brainte aser +br ata +book bug +bo ies +blue grey +bloomberg dotorg +birthr ate +bibli ophiles +bi ra +bi ggles +au rion +at ation +asmode us +apart ners +ao i +anz ca +antho cyanin +analge sics +am maasante +alone together +agrit erra +adel ica +! ðŁĴĥ +ðŁįĴ @ +ðŁįĭðŁįĭ ðŁįĭ +ðĿĺ ¦ +ë²Ī ì§ +ñ e +zlo ty +zel f +ze ich +ye agles +yam it +ya ke +wwt worldwide +wri thing +webber naturals +wc f +ur za +uof denver +underestim ation +un oh +troopingthe colour +tre mont +tok ina +thelittle idiot +the global +termin ologies +teas ley +ta kt +sw yn +strato volcano +spokes man +spectro gram +somo za +smo squito +sme ad +sm ta +sj susp +shim my +schin kel +salvad orean +sag ged +ru man +ridge ley +rich thofen +reyn es +repudi ation +reg no +re ade +pure gold +pier paolo +part low +oyster mouth +ottawa hospital +oster tag +osh kosh +om ah +o city +neuro typical +mull aly +mu larkey +mogwai band +michigand ers +mel ua +me ppel +me iz +mars global +mark richt +mani er +ma sat +luce scu +live ga +livega aresults +lind h +li zam +league one +lancashire day +lanc shospitals +la fd +kush waha +kri os +ko hr +kick s +kati ek +kar ad +ju hi +jodie whittaker +jha adu +jan sport +jackie morrisart +j angling +irish hockey +invol vements +ing peace +immun ologist +i yam +humber side +hu gz +heem skerk +hed vig +healing crystals +he yn +gü ne +guad ar +gor zata +galent ines +g dr +fresh meat +finne more +ferr arif +fer mi +feel it +faithe vans +facebook ads +ek land +echo esof +ec atepec +duke sof +digital nomads +diane sawyer +dhaula giri +denter tainer +david schwimmer +cy f +cruz ado +cor saire +coastto coast +clon curry +charlie baker +chao tix +cel ina +cavanagh tom +c ttw +business day +bur ki +buch la +bronchi ectasis +bro sius +bor Ã¥s +black gold +big ass +ber de +bel ittles +beauty chic +az on +ash p +arti fact +andrew bogut +alexandri av +ain hoa +a amo +:- )" +ìŬìŀIJ ìķĦìĿ´ëĵ¤ +ãĢ ķ +âĿ¤ï¸ı ðŁĴĸ +yoshi kawa +world pressphoto +work with +wonder swan +wildwater stu +whata view +water stone +walker artcenter +w pr +volu bilis +ver no +veebha anand +vag rancy +tumble weeds +tubabu yukustun +tri shap +travel gumbo +tor rente +tor por +ti sca +thel y +thebusiness show +temp tation +te gid +tann ahill +sweet life +sure shra +strep tomy +spr ou +spark ly +sonic fox +sombre ros +sof nature +sob tians +so gard +sla vica +sky light +sksk sks +skill sshow +sk top +simi on +sime k +side one +sfe ir +ser rate +senator cardin +se wanee +sco tian +sch leg +scatter brain +satoshil ite +sand town +ry dell +ronnie oddo +road sbrewing +rishab h +restaur ante +ren ounces +redar row +rachel boston +pter osaurs +psycho logies +poo bah +picture h +phrase book +pepp ino +p kn +ocal an +no bin +newar knj +nelson piquet +ndom bele +n pw +n nab +mount juliet +mondaw min +moderni stic +mo chan +mindbody soul +mindbody green +midnight madness +michael irvin +me dair +matt makens +mar ny +mag adan +lol rt +listento this +le drew +lam poons +ki goma +karlo slabajo +ka et +just inj +jazz radio +janak puri +is scr +il tm +hu mn +hin de +hef ti +han ly +han ayo +ha fa +go grow +gibr al +ger ace +gau t +from dec +freen hs +fr ann +floo w +fab by +easty orkshire +du pleader +dra il +dian eneal +di wak +davi dru +craz yyy +coulom be +concor di +cl unie +cho key +char len +cha erin +central pictureh +cc mv +cat fished +carloscon dit +buy black +butchart gardens +brin apalencia +bir cham +bi ff +bhagav atam +beta thetapi +beng tson +bell roy +bare bone +bancos antander +b horsetrials +as cents +arth quake +ar matures +animal sasia +ancient art +aloe blacc +ah music +actof kindness +acceler ometers +ac lock +aa ar +... ðŁĺĺ +ðŁĹ£ ðŁĹ£ +å¤ ļ +ãĥŀ ãĥ³ +âĽ³ï¸ı âĽ³ï¸ı +âĺģï¸ı âĺģï¸ıâĺģï¸ı +à¹Ĩ à¹Ĩ +ยย ยย +z v +yo wen +y endi +y bn +wood working +winder mere +whoo kid +walker stalker +vivic afox +visit orlando +vie wing +vertic ality +ve ille +van horn +up there +uof m +uf ford +to logists +the men +suppor ter +su amico +stele com +stefan molyneux +star z +sram road +spun out +spen guins +spann ers +smoke out +smith mp +sma kers +si sts +shan el +sh ales +segre to +seen onmy +scint ill +sche iner +sanlu iso +sakura gi +s ga +rutherford ton +rubber ised +pu asa +prayfor syria +port credit +pon ent +point break +pir sig +personal brand +peri winkles +per mit +ped lars +pam bondi +open wrt +ome body +odi dev +nov as +nott z +north face +nordoffro bbins +non standard +noki a +nishar awal +ninot chka +newss vc +neg aunee +nav ed +n sen +mytwitter anniversary +mumbai metro +move more +moon rock +minim isation +micro management +met to +memo ires +mcga hee +maxi on +ly cian +lovethi steam +loud mouth +losf eliz +lopes rising +limp sfield +like ability +lan ken +kn or +kir ari +kids games +kid problems +keala settle +karish ma +jo ongi +jin bei +jfl mtl +jamie chung +iw amoto +insec tiv +injury pics +ight club +ht delhi +hon ge +heck ert +han ai +ha iri +h music +gu ill +gru mbled +gra fi +gote ana +glo winthedark +gil das +fun dus +fred vanvleet +fre as +france diplo +fo el +fix edin +fas sler +faha dh +fabi of +f é +espa illat +en fren +el rancho +el nido +el arry +ee x +dro op +dontbuy aticket +dis believing +din nington +deray davis +dem ander +dash pay +crazy horse +cor moran +cold feet +clu bre +cloak coin +clay pole +clan king +cine rea +child lessness +chat ur +cel sus +ce dentertainer +car ris +cal fire +bun inyong +bt group +brø nd +bru hs +bring itback +borro wings +booster club +bli t +bho sale +bar de +bad ha +bach alo +aw oo +angelalan sbury +ana q +alm shouse +ald well +ad dae +acec ar +ac ats += ))))) +ðŁĴ ¾ +ðŁij¨âĢį ðŁĶ¬ +ðŁIJ ¡ +ðŁİĬðŁİĬ ðŁİĬ +ðŁįªðŁįª ðŁįª +âĺĿ ðŁı» +اÙĨ ÛĮ +yu a +ys auce +yaa dein +y nasty +xmas jumperday +x ende +wkc dogs +win tle +westerni zed +welcometo fife +wearable art +vol f +v orders +unborn livesmatter +un inviting +ultra chat +tyn drum +trump taxscam +troeg sbeer +trave sties +tom ove +time suk +thisweek abc +ther uf +then asem +the blues +thatcher ism +tender foot +teacher training +super cili +stol per +stif les +ster rett +sta el +spl endo +space walks +so zo +so be +skelton sophie +she kar +shaw sheen +shar bat +scienti sm +schoolo flaw +sand nes +sa stry +ruby tandoh +ru sal +roger bezanis +ridge view +r ttowin +r nu +qu ach +q esh +pv z +pol ster +physio logically +peter kin +pe prally +pat ar +paci fying +ou gars +om n +olu wa +of thel +oci ation +oat cakes +o ja +nw fl +no stell +nhsc trust +ne utdfc +nbc sandiego +nar ges +nak aka +myri am +monte jo +mis ael +milak unis +mero ving +me hi +matte son +mathiascor mann +mal content +mah ana +ma zzi +m xc +m rad +lou sa +lob sang +lind asu +liat ris +li jk +ldn ent +lamb dal +la har +kres se +kra z +kot ze +kathe vans +ka than +ka det +jim cameron +j achat +iso topic +iow acubs +inthe usa +inspira si +ici onal +ib tiss +hoursof spa +hou gaard +homin ids +home makers +herz berg +hello ooooo +han am +hamilton ians +hal lux +h mis +greeng ables +gr itti +glyco sy +gl ace +gabby logan +frank town +fb nation +fazak erley +exti rp +en cwx +emiratesair line +emili op +ell ac +dolant wins +do blo +disbur se +devi ates +den cia +del amar +de aries +cor ton +colo colo +codi fy +christma scards +by word +bryo phytes +bro g +boath ouses +bare illy +baldri ge +ar field +anc ook +alway sab +ï¸ı âĻ +ìķ Ī +åIJį åı¤ +à® ľ +zap ier +ys london +x bit +wy kes +wsc atlanta +work flo +wool loongabba +wied lin +wake ful +vitru vius +vijay awards +vel ive +van ss +tur ri +tokyo pop +the pretty +the orlando +that sthe +team c +t zy +t byd +sun catcher +stephen colletti +stal inism +spag nu +so tis +snowmob ilers +shop my +sho ward +she kel +sharma bjp +ser zh +scru mv +sat un +sas one +s los +s line +s bt +ru ski +quarter finalists +quand t +pun kt +pp an +pin acol +pepper idge +opti ks +officialtf gm +nottm playhouse +nor dan +no ter +newtech network +naf t +n mmu +multi beam +motor ing +montesqui eu +minof culture +minofculture goi +mime sis +micror nas +mi ani +mazz anti +masa an +mal bork +m wak +ley green +le ther +la ging +kurt schlichter +ki ke +kal ona +ju dai +janele eves +jale el +itv westcountry +is ci +inthe box +impu dent +ih re +iamking promise +hy pixel +hugh i +hau dio +han kerson +gru sin +grand view +gra vid +godd d +go puram +gas sy +galactic elliot +fra ger +fow le +fon er +fest spiele +fandom btsarmy +esc aflow +dispropor tional +des barres +den ker +cred ito +cr ite +consu ls +cl aman +chimpsin socks +cheer up +che shirec +charliebaker ma +chang kat +caleb shomo +bra sh +bor ra +black work +bharat bandh +bha d +bell let +avi vauk +atra ircraft +arvid sson +ano d +anjun adeep +anim ate +angela white +alco pop +ade en +ac coya +" .- +ðŁĴľ ðŁĸ¤ +íķ ł +ìľ¤ ìķĦ +åĽ Ľ +âĿ¤ ðŁIJ¾ +ൠį +़ ा +© × +zan aco +yw pd +yu va +y rian +wood works +we stor +wak an +vel iko +vanguard ngr +vanguardngr news +vanc leave +un learning +uk crafter +uih lein +tur ay +tum mel +trelaw ney +tour nee +tho ckey +therealt boz +thereal mattkemp +te un +ta sik +swag gg +steph end +star ring +spe ace +souley mane +soo o +sonic forces +sinful sunday +simp er +silli er +shop uk +shon ibare +shay kh +shan kha +sham shad +sever ally +sef covic +se ow +scar is +scale bound +sandi acre +salty ard +ru ta +ru gosa +ronald say +rn wn +resusc itating +ratche tness +ranc agua +proudto bean +propor tionately +posh mark +plu tus +petro ff +peter gallagher +pers ico +pas crell +par amo +oun ion +orgreave justice +oar sman +no iz +nca avolleyball +nca atennis +navy seal +n lex +my scar +mor io +minn ich +mil aj +micro tia +michael jackson +ment is +mein en +matt bennett +m qi +lub wx +lo tan +leo burnett +legalize marijuana +le scu +kuchi ki +kon omi +kha sh +key amo +kel y +kare em +kal vin +in humanely +in coming +iam sandraoh +hydro electricity +high wycombe +her dsman +hel dt +healthy habits +hand printed +ham re +hait ink +guter man +gg davidjohnston +finger hut +fel tbicycles +euron cap +er and +emu fb +ed din +dran ks +di wata +desro chers +ddin shah +dayton flyers +dam ed +cry onics +cream tea +chev rier +che wy +castle berry +car leen +canni zzaro +cal c +cadi gan +buddy hield +bor rero +bn ppm +blove is +beast men +be fi +bash kor +az ed +audible uk +aten cion +asab fb +ant sy +andri us +ana am +allen ge +ag onies +ag ian +ad dax +actin ic +ach ats +acci dente +* âĢ¦ +) âĿ¤ï¸ı +ðŁİĦðŁİħ ðŁı» +âĢ ¥ +Äį ek +yoo hoo +yar wanda +wo hn +wi mey +warren dale +waja hat +vidy ape +uoft medicine +unct ad +un scramble +tri fid +travel ph +trade deadline +top starnews +the ophile +terror tuesday +tan imals +subscri bes +su chard +stone cutters +stev elu +star darshan +st mark +southpoint lv +sin fulness +shi bley +sher lyn +shepar d +sham o +sentom cotton +seems legit +secre tof +school snc +sche ster +scam era +sar ny +sal tal +saint anselm +s wn +raf fi +pu gh +proudto workatbmo +projec tre +pratt and +pm ct +pie za +ph m +peten ajarian +peop lemw +pentag onal +p wt +om pt +old spice +ol au +nyu stern +north park +news journal +new sen +near by +nas lofficial +nag ase +n gh +mor th +moder nam +mi len +melissaand joey +meli odas +me cham +man zar +man asu +ma br +m roz +ly u +lili enthal +kyo suke +ku fr +kra viz +kk c +jäger bomb +julian marley +jrr tolkien +join aap +je pang +jam ar +iv ri +in most +il ys +il ok +i give +he ymo +hak ko +h tcu +green skeeper +gravity x +gol dies +go inghome +gh arib +gart side +gab ru +fly y +fil by +fi dan +fet life +fay ad +fant ino +eno ir +ema son +em itchell +eli p +el angana +ear thers +dun igan +dor f +do j +ditt mer +dex po +cre aky +corri mal +coraz on +con fab +chang an +cha amp +cer vi +caer u +bur le +bull ington +bry anna +broom all +blo ks +billy porter +ben koku +bell ringing +bel ou +be informed +balu arte +bad sha +b surveillance +b gi +atasco sa +armend ariz +ake ley +ak ers +ag od +ag grandi +af rc +act itud +abe sigye +aa fia +? ðŁĺľ +ðŁĺĥ âĿ¤ï¸ı +ðŁĺĤ .. +ðŁķ İ +ðŁĶ¥ ðŁĴ¥ +ðŁ¤Ń ðŁ¤Ń +ìŀ¥ ìĿ´ìĶ½ +ìĽ Į +ì ¦ +æľ ± +ãĥ¼ãĥ IJ +âĨ © +à¹Ģภģ +world alzheimersday +wine review +willmott dixon +welcom ing +warri gal +waifu wednesday +w acs +von ian +vinyl collector +valu eless +tvguide magazine +tru eee +troy aikman +trap soul +ton ature +tic toc +theti den +theroyal opera +there bel +thekings fund +the feed +that swy +thal f +team tuesday +team psg +te bogo +talis manic +swar na +supriyo babul +sulay maniyah +stubb ington +straw man +stoudam ire +ssel berghe +sp ak +son amo +sm ic +shack ling +scott k +school trip +scandal ously +ryu junyeol +rukh sana +ri sin +reic hard +reflexi vity +red lines +raim undo +r ki +r kg +r ills +ps j +pre ndre +pra dera +plat in +pir atical +pig nat +picke ters +philippe gilbert +ph um +pe otone +pap olitics +pap ad +padmash ree +p sms +over stepping +osu v +ooster beek +ocam l +o izo +nc po +national pastaday +nat aly +nan kana +nag ari +n fum +myco toxin +mr h +mo ayush +mitch gerads +mis uponus +mike j +mendo za +mass generalnews +mar kes +ma kinson +m sy +m ll +lu u +lin gotto +light itup +lie bes +liciou sto +le pper +le compte +lady tron +lac aze +lab corp +ku rien +kle infeld +kin nard +k sp +japanese chin +j williams +is b +ine te +ima am +hong seok +hell yes +health food +ham med +h bogo +h barnes +guer cio +gi puz +gal is +g tc +futureof mobility +fu miko +fre jus +france sinha +flu mes +fal am +entreprenu ership +elek tronik +elast o +ed widge +early risers +dutch nat +du breuil +dont panic +do xy +digital divide +di ack +deuse x +de it +de cle +dark matter +dac apo +cow pens +conten to +coach d +clackmannan shire +cin zano +chin y +child safety +cc stca +castille jo +bo jonegoro +bmw champs +bio engineered +big dayout +beer advocate +baycity rollers +bashkor tostan +band master +ban sal +bailey sprize +bag nell +avo ices +august ine +atu cker +at avi +as kincare +ari v +ari ana +ar ten +ar pad +anne tta +angi olini +ang ini +alex ail +alber tan +agor aphobic +ago g +acar r +ab ration +ðŁijIJ ðŁı¼ +ðŁıħ ðŁıħ +ëĵ ł +ê³łë§Ī ìĽĮ +zoo sk +zam in +z eca +ysrc party +wux ian +wonder lust +win fromwithin +w chl +vanhoo ser +van ak +urbang ardening +twee zer +top speed +toi fa +to ghe +tic hen +thesam icallihan +then bhd +the plac +th ate +te sl +tar kowski +swan ee +sub plots +star ac +sse m +south church +shet ler +share thedream +shar my +shane helm +shanehelm scom +sham sher +sh amy +sewing bee +sen o +security council +se tif +se ap +sco bar +sand alphon +rosen gren +ri ina +pro pa +prime performer +pri d +pod cas +par mley +par ise +pail waan +open jdk +op lus +on hot +obi dos +oak v +night club +nec primeperformer +national nappingday +nadal ind +mush taq +mure san +mo dan +miln row +mil ing +meyer land +marang oni +mal ope +maje sty +luci ano +lobo tom +lean or +laro ja +lab and +khar al +katie qlowes +kar adeniz +k beauty +jon ty +joann alum +jiz an +jak elong +it railer +is sou +indie gaming +ill ero +ile v +i sho +huguen ots +hr block +hoch schild +heart this +har bisson +gu zheng +gu li +graven ey +go stags +gno sticism +glasgo wc +gc punknewwave +gar field +for c +foodre volution +food mag +fle sch +finan za +fin ning +fieldof dreams +extr atropical +estadouni dense +est il +ero ski +er mer +election pakistan +el mes +eh c +ed ar +e tec +donnell rawlings +don kiss +don avon +dil raju +den mead +dar le +dal um +cyber goth +cra ige +cow rote +cortel you +ci flacs +chir stmas +che in +challenging stardarshan +catholic newssvc +calvin ists +c xr +brou weri +bro sse +bir gunj +big wig +ber lage +bell oni +be ze +be sting +api er +anat oli +allo tt +ali ko +alcon bury +al ver +adobe symp +ab rin +@ _____ +ðŁĺŃ âľ¨ +ðŁįĬ ðŁıĪ +éĻ Ī +æĿ ¨ +ä¿ Ĭ +کر ÙĪ +» ðĿĹ +zi bo +wrp stoday +wey bourne +wbc baseball +water town +vo lei +viol ino +vers ation +vend itti +vee redi +value walk +valky rie +tü v +twick enham +tweetyour seat +tra ven +tow yn +too ie +tmr kt +tin sider +thereal stylesp +the quietus +talis ca +ta aa +suppor ts +sun omiya +su el +stru tters +stpaul saints +squirrela ppreciationday +speak up +sour cing +so rell +smee th +sle zak +singul arities +simul ink +shor ne +se iner +se gan +sawak ened +s was +s enger +ru mi +roman ized +rin der +rhy d +rayne spark +pushawards donkiss +pu yi +pro max +pres anctified +pou rover +pom bo +pi kappaphi +phe l +perver sely +patriot sawakened +o canada +nil an +namak kal +nam ja +mull ery +mt cc +ms ss +mol ls +mil ner +mi datlantic +mend on +medic aleducation +medi acon +mc nichol +mb raves +matricul ated +mat thu +marcin iak +magic weekend +ma assen +lun amaya +lu skin +loc all +le ury +lawn mowers +kum bia +ku wabara +knap ton +klo veradio +kizz abesigye +kirkby lonsdale +king e +kap ar +kalev ala +kad ay +k me +jumu iya +joey kramer +jhar su +j nc +ittf world +invinci ble +insinu ated +initi ald +indra prastha +ide e +ic v +hob goblins +harrogate hour +ha ab +gu lab +green party +god des +gir ouard +gb ball +gas pe +funny bone +fore sti +fla k +fin back +fan tome +fairfax county +fa ecal +f mtv +eli zed +el khound +eeve elu +ec lo +dur ley +dun ord +dublin pride +duba iland +drainthe deepstate +dpc dsb +doun reay +delight ful +del mon +daf trucksuk +crê pes +cruci form +creati va +countyof gp +cot trill +corpor ately +cooper union +chi ayi +che doke +ce of +catul lus +capodi monte +callum smith +bu cheon +brun elle +bren a +brand l +bis sell +bis cay +big dreams +bha gira +bearcat nation +be tw +bau dry +bath wick +bangerz tour +att lers +an tum +ameli ar +all am +agelim it +aga pe +ag l +ach ari +abstr acting +a hini +( âĹı +ðŁİħðŁı» ðŁİĦ +zo boomafoo +zeetv jodhaakbar +wur d +woody harrelson +wol bachia +white way +we ma +wash dc +walker hayes +w assim +val orem +v atika +us vi +turkeye lections +tro binson +thegreatest showman +the oscars +the core +telegraph news +tarheel nation +tar ge +tan it +ta zz +syd ni +surve il +sul fites +sub soil +str m +sten nett +sonthe hill +so sick +snorkel ers +skit ouring +showyour stripes +sh kar +sexu ales +se alift +scifit v +same sex +s mag +royal jordanian +rou k +rosen heim +rosen ama +rosenama junas +ron quillo +rock solid +rit zy +re orient +re negotiating +r atta +prote aceae +pronoun cements +pp s +plot lines +plen itude +on p +oliviap alermo +oakland county +northeaster ly +nikhil chinapa +night cliff +nic oti +next topmodel +near buy +nas ution +mote ts +mill woods +migun amig +micha elia +mi kazuki +mez quita +mar ls +mar lee +magicmike xxl +ma gre +london calling +lo wit +live phish +lies matter +li viu +larry king +land use +kristi ana +kib bie +khu lic +kavanaugh now +kassi us +inked girls +ic gc +hughi efury +house sitter +hon asan +hir ingnow +helg ason +hear ty +head stall +hartford courant +hang leton +ham lett +halton police +gro es +griffon ramsey +goulet pens +general isation +fore stier +fi ere +event prof +et ats +et at +et ags +eric thomas +embroide rers +el sner +eet ings +easter island +depu is +delhi ites +dan brown +dal riada +cosmopolit anism +con cious +colling wood +cl at +chinese medicine +cephalo pod +cdn press +cau stin +bull frogs +bruno tonioli +broad cloth +bombay bicycle +blood brothers +bic ultural +beren berg +ber ro +bd smovement +bar in +augu ay +ash es +ash combe +aschaf fenburg +ar groupsuk +app l +ap na +annecy festival +altam irano +aj w +aga inste +abster go +ðŁķ ¡ +ðŁĶ¥ ðŁĴ¯ +ðŁij¦ âĢį +åĭ Ł +ãģĹ ãģ¦ +âĽĦ âĿĦ +à¹Ĥà¸Ľ ร +à¹ģà¸ Ĺ +youth homelessness +y da +whid bey +wat onga +visith ampshire +vi rion +vg com +vel ux +uri en +ukcrafter shour +tur ville +tur rican +tsong khapa +tourdecor se +timber ners +tier no +ti rl +thirty something +thermo plastics +thelo af +tb q +tal aash +ta kuma +swan son +superbowl xlviii +spencer boldman +sort sintl +sleepyhollow fox +sla vik +si dro +shet terly +scroun ged +san mate +safe haven +run du +ruff ner +rod ina +ride ox +rep ens +rec chi +re zeki +prophe tically +plan um +ping ry +parrs boro +paraparau mu +p cori +or rinhatch +olive tte +oli vella +official blue +of ashion +nush rat +nun an +nie man +ne rea +ne grin +naq sh +nant garw +na hlhockey +na anum +mycause my +my journey +modern warfare +mitt agong +mis appropriated +min der +middlesex uni +micro tubules +mgmre sortsintl +mer na +maru game +malm strom +mait ake +macmillan kidsuk +lemon is +lance field +kra g +ko sty +ko li +km fm +kad aga +kacz marek +ivel isse +iti er +inter bay +intelli j +in sou +ili um +i imc +hrtech world +hoo fer +hit parade +hill crest +hapand leonard +hair growth +hail stone +haha aha +gurun anak +grac ed +gemini ds +gd st +gal v +for umb +flo tte +flash man +fl outed +fent on +fe tters +family goals +fa wnl +eviden cing +escal ada +em j +el r +eff lorescence +di ant +dev lin +depend ants +death ly +dark and +d bo +crimin alizes +crashed ice +cor robo +cop us +cm chat +ci ot +cast en +carlo sainz +british pieweek +brad wall +bor on +bird shot +biomole cules +bi bby +bestteam in +balle tic +bagat sing +bac ton +au ghtme +at the +as afo +ar ness +aqual ad +apit alist +anu media +americanc rime +ali ah +aldub thering +air worthiness +aegon championships +abu il +aa aan +ðŁĻĮ ðŁĻı +ðĿĺª ðĿĺ +å¤ ª +âĿ¤ï¸ıðŁĴĻ âĿ¤ï¸ıðŁĴĻ +âĺĢï¸ıâĺĢï¸ı âĺĢï¸ıâĺĢï¸ı +âĬ Ļ +à¹ģภģภ+zyg munt +y ula +wy lie +work space +wild scotland +weather tech +wean ling +viz quel +vit olo +vi gg +ver day +usu l +umich hockey +uk health +ty pennington +ton et +thisis fusion +ther saorg +the wee +the village +tele conferencing +tab lature +sv vsd +sun burns +sul tra +star my +spring isd +spat ter +smac cus +sku ld +sizz lin +sin space +shu ji +sfor you +scol lide +sche ana +roth gar +roo ke +rohan mehra +radi oman +rach els +ra ditional +que star +pra des +post secret +pon dy +pi lea +oto scope +ol kata +official sslazio +of books +ny jv +nord berg +nice attack +nax alite +nar ducci +nad asurf +n sula +my babies +msh sl +mom of +moist ened +mit r +min di +mid wales +mel ena +meg u +me urer +mayward for +margare tho +man ov +mal ouda +mah ra +mackin lay +luxor lv +loun gers +lo pers +lim pid +ligab an +ligaban comermx +land wehr +l alive +kumar u +khulic hana +kho si +khaz anah +kell am +jewer ly +jenniem calpine +interven tionism +ini sti +in volu +husky pride +he tt +he aver +harri sfaulkner +handmade jewellery +h laudi +gü nter +guadar rama +greeting card +goo dge +glen ns +ge tb +gann icus +fun ck +fountain sabbey +fficial page +fa thy +escut cheon +eric ans +emb erton +ei der +ecor nish +dj ze +dj david +dil bar +dia internacional +deob andi +del ice +dej oun +degre esof +de mountable +dang le +daffo dil +cp gs +corre os +cocc inea +chub buck +chic lana +cassad aga +bul lock +bu erk +bru v +bom ar +bh vr +bass rush +ban kon +ax ons +all inthe +aleu tians +agar nier +af und +act for +ab ct +a septic +ðŁĶĿ # +ðŁĵ ľ +åĴ² èī¯ +âĽı ï¸ı +âĺİ ï¸ı: +âĢ¢Ìħ _ +ص ÙĪرة +ин а +zel jko +your plate +xylo phones +xu an +wire uk +w kc +visit norway +vann elli +tone bell +to car +ti gi +thi el +the tachi +the doctors +teryl rothery +tauto u +sunsout gunsout +sunday live +sun od +su chus +st mun +sou li +sn ts +smom entum +si saket +sfra hm +sean sch +scott adam +sch ank +regin a +recycle sday +ratche t +rapo so +ram page +prof david +ppsells babyparts +policy maker +pol lex +plus blogs +pion ate +pe avine +pan ache +otta war +os by +or be +npa ie +neapol itan +natural is +nar r +nal im +nail sinc +na thuram +n lg +moto red +montan astate +meik le +marqu am +ma whinney +lu ria +loyol amar +lone some +live aboard +lection fraud +koval ainen +ki anand +kath imerini +kar dia +kad av +jur ra +jugend stil +jim parsons +ji ye +ji goku +janete van +jackierobinson day +ishin omaki +indu sind +indiant ellyawards +hus n +ho din +him zahawi +hel goland +hal mahera +hack neye +grind time +granul arity +gon injas +gobel ins +go pa +gab led +fu ke +free bets +fra bbits +fior ucci +fam z +fake facts +express js +every corner +euro scepticism +eu karyotes +es w +erik solheim +ephemer is +ellen ville +elec ciones +eg ner +dundru mtc +ducati uk +du stria +dou ge +den nys +demarcu sware +del luk +deduc tibles +decade of +debash is +de muro +cumbri ans +cor abi +con ures +col ter +char ri +ceremon ially +ce tin +catar aqui +casc ina +cas eros +carrauntoo hil +carr aro +capitul ated +brown coats +br cnews +bong bong +blood root +bir bhum +big dog +bi gr +be wafa +bar gy +awe som +au zon +aspho del +arro el +ar ace +ann erley +al annah +ah jussi +ag op +aber rations +aan and +ðŁĻıðŁı¾ . +ä¹ ħ +âĿ¤ï¸ı âĺĢï¸ı +zen kmm +y the +wood z +wood burner +wol fin +wic cans +who p +wetter ling +wer ki +wel len +weid ler +wartho gs +virtu osi +vi aj +veteran sday +utre ch +unicorn day +un seasoned +un mas +tu dela +tre acher +tou rage +tororosso spy +tit res +thi g +thelon gest +the budget +table tennis +supt king +sub structure +sri devil +sridevil ives +sridevilives forever +sportac cord +sloven sko +sha shan +sequ el +senergy drink +sal vac +sal guero +ryandun gey +rter ugby +rou st +retin ue +reti rez +recre ationally +q bl +proco pio +pro digi +pri matologist +pram ukh +phobla cht +pets alive +perver ting +personality disorder +paulo grady +pat chen +pachu lia +p drc +over step +oneof ourown +now all +new show +nat aly +nasag lenn +nanomat erial +nan ing +monast icism +mohandas pai +mobile al +mo yn +min ju +mh clg +medi us +maracan ã +mac m +lur d +loin ie +lobby day +lil ya +les b +laer dal +kyle brandt +ky naston +know thyself +kk k +kenner ley +ken ichiro +kemer ovo +kear ly +juli er +ioc prev +insu rable +industri ally +indian ad +in ot +i play +hur ford +high park +heine mann +hay akawa +hann elore +gol dglove +gi lets +ge ph +fun mi +free india +fowl kes +foresth ills +for ap +for africa +feelthe burn +fal guni +es man +eni ola +e special +dis contented +de personalization +david the +dad aist +con ow +clow nish +classic horror +cj leblanc +chri sf +cho dron +chee siest +chau sson +chan dos +chall inor +ch ado +canter o +candre va +cal stock +cad avers +buon arro +brandon heath +bra hm +block ades +ben ayoun +bad ab +ba artman +b girl +assisted living +are op +anast aciam +amusement park +amath us +alec bradley +ab dal +ðŁķ Ŀ +ðŁ¤Ļ ðŁı¾ +ãĥ ĸ +à° µ +௠ģ +zsas z +z ok +yoshim itsu +ym x +wj bf +wella pro +we aried +wal kal +vital signs +vini fera +urock radionet +tvweek mag +ti ii +the viperroom +the gift +the festival +the angry +te ds +tce lectronic +tai k +super collider +stü ssy +stre ats +ster ritory +stat ev +ssi one +spar xxx +space invaders +so ini +sheep adoodle +sexiest manalive +semper fidelis +sec ant +se sotho +scol aire +sal onica +sa repta +s monday +ry de +ribble valley +re ssed +re deploy +rail pictures +ra sm +q ag +potat os +post lethwaite +pl iner +pd news +parli ment +park boyoung +or ito +opp ermann +ooster hout +official donegal +nr ms +nor walk +nin ak +nick swag +nad himzahawi +na ho +n á +mt pa +monet arily +mo tac +miss vogueuk +mi amis +metr onews +mc curtain +massey ferguson +mar azzi +man liest +mame tro +mag si +mac alister +lu bez +lo salam +lie shout +legen ius +l wow +kri korian +kan zaki +kam pu +ka ther +ka ida +joss elin +jo su +jes shar +jeff erys +jdr f +japan gov +jackolan tern +j gi +itscold outside +intellij idea +hypnoti zes +hur u +hub bert +hu zoor +hu sen +hondac ar +hill i +har leys +h wv +gu se +grin berg +glu on +giant games +gi ously +ger hard +g gh +fre eness +flor a +fleet line +fit forlife +ev ak +europe echecs +el op +dor sa +deep veer +decem bre +david burtka +dave meltzer +ctb ase +cru ll +crit care +coron ado +coachtom herman +clark university +cityo ff +ci fa +chin chin +changeagent sa +catherine tresa +cate che +cas ula +cas kale +caroline manzo +bull riding +breck in +boving don +bot kin +bon ner +bo ie +black welder +bi yani +begum pet +bbc spoty +bad guy +ay tas +atv show +as ml +art deco +apod aca +am aa +alu z +ake sson +ak sy +ag aya +aero tropolis +ac ire +ac d +a ang +! --> +ðŁĺĺ ðŁĴĺ +ðŁĺį ðŁĺĪ +ðŁĶij ðŁĶij +ðŁİ¸ðŁİ¸ ðŁİ¸ +ðŁįĤ ðŁįĥ +âľ µ +Ùĥ Ø© +ع Ø´ +é l +youre ye +yacht smen +x fire +writer life +wilmington nc +w ne +vv pat +vroom vroom +visit richmond +v cc +untam ed +un accustomed +tusk ys +to eat +thom astur +theatre news +ter amo +taq iyya +supper sunny +sue anna +stan doff +spi key +sp uppet +solit ario +sm itt +sic ard +shurtle ff +sell wood +scout scanada +scott age +scien cel +rudolf schenker +ring sidec +ravi shed +pride seeds +popul arizing +pin elli +pfe il +pe skin +pav oni +patric ian +over doing +oli sa +of arevolution +od ley +nut case +nu us +nu fbfamily +nou ll +nor df +non committal +nin der +nil sfrahm +nes see +nc isla +nation scup +n ment +my army +mon naie +min oo +micro aggression +mechanic ville +mcl ane +mam mill +makeupforever us +ll nl +larcen ciel +koni shi +ko tto +ko sk +jiha di +jezz ine +jewish chron +jeh lum +jam rock +jack fish +hw ys +hoy les +ho el +healthy ireland +havan aclub +hate story +hash t +haiti en +gwal tney +gun fights +gre han +go wings +gha ith +ger vase +gemin id +ge u +gal im +g ct +fre chette +fr yn +fight stick +feltrin elli +fast car +en hor +el pais +ec bc +du bonnet +dragmedown musicvideo +drag ali +dou bloon +do eee +di ks +depon ia +dece m +deal z +davis jr +david suzuki +dat ar +dan agould +damekelly holmes +d hun +cycl onep +cur sus +cra pped +cool runnings +conden ses +coco abeach +co ie +bur ghs +boy ata +bound forglory +bor stal +bo bin +bin ley +biblio graphic +bethe sd +bel fer +beg awan +bbvacom pass +barbap apa +ath let +aspher ical +asic s +art smith +ang ely +am ancio +alway son +aero tech +adul terers +ach inery +acclimati se +ab util +-. -" +ðŁĺĺ ðŁĺįâĿ¤ +ðŁİĢ ðŁİĢ +âĵ Ķ +y anti +xx music +win don +visu ali +vid mar +vic is +va ad +urbann ax +un sr +ukin theusa +thisday ing +thewilson center +thereal b +the pink +tape o +tann ersville +takam ina +ta isuke +t soi +sylvan us +super humans +stu ckin +simel ane +shu ddle +sex scandal +seriou seats +sa official +s alive +ruz icka +royal freenhs +rosequ artz +roo tin +rock box +robert m +realm royale +real remyma +rawn sley +radhi ka +ra ee +puppete ering +psych ical +pro positioned +prick ly +preacher amc +pr z +poshan abhiyaan +playbook athlete +pim enta +peri ence +penn sboro +pen cak +pb ks +panch ali +ov ando +ou uuu +ot ch +op r +onthe farm +ong kong +ol szewski +nj con +nas sa +na ag +myhouse idea +mtu hky +mm ilive +mentionsomeoneyou rethankfulfor +meg acon +mann ino +man field +mal ins +mal alay +m itic +luv vie +lu to +living uk +let my +led lights +le zz +la ves +la er +kroko dil +konicamin olta +kir mani +ker mes +kelly hu +kam io +k wei +juego de +jim henson +jan netty +insider tweets +ine us +in chon +huck berry +horseback riding +her ault +he arer +har uki +good wyn +gon zi +gil ardi +gaunt let +gar lock +fur ni +from me +fle xbox +fiore tti +fence post +fac ed +e board +e anes +domestic violence +dissoci ating +dem u +de briefed +davemeltzer won +copp el +com tesse +cle lia +chiropo dist +cate ley +carlin i +can ongate +caed mon +ca thal +bru d +brett cateley +bran sford +book qw +black street +blabber mouth +bi gup +ax illary +auto gly +anan sie +an wr +an ax +amblyo pia +amar deep +aji mobi +air show +aesthe te +ad oes +ac loud +ab culture +ðŁĺĺðŁĺĺ ðŁĺį +ðŁĮ Ĩ +æĺİ æĹ¥ +åģ ¶ +âĻª âĻªâĻª +ÙĨÙĪ از +yr self +youss ouf +ylon agarcia +yeee haaa +xl center +xende sktop +wl k +vote sforwomen +volk man +vo ssen +union bank +unicy clist +tro is +tri ppe +tr b +ton kawa +tin su +three words +thom an +the wave +the party +te ann +tarahu mara +sy lum +swee thome +stick in +steep ing +stavro pol +sound transit +sophi atown +slike these +sk n +sk ille +simply santafe +simon parkin +si pri +si dero +si benik +ser dar +sch ronicle +sander stead +sabah tourism +s thorpe +s ates +ry sselberghe +red way +rec tus +re ba +ravi pudi +ravens thorpe +rath more +ranfur ly +rad itya +ra pra +ra pacious +quan zhou +prinsen gracht +photo ed +phon on +phenomen ons +people smosquito +pd mf +pdmf nb +particip a +par malat +papp ano +palaeonto logist +oli var +ol ink +o fu +npr freshair +noe mie +niç oise +ng tindia +nc statec +n toko +mun tashir +ml bn +marches afashion +marcel hirscher +macgy ver +mac chia +lung fish +leh to +kon tiki +kk as +ki pro +kerry on +kendra para +ivan ovich +inte gra +hirsch man +hin tze +hemi plegia +har ned +happyhappy labs +hanafu da +halam adri +h ure +greece central +gra gson +goodby enewton +goodbyenewton trees +go with +go scots +go cubs +gb x +football museum +fly boy +fil enames +fearthe fin +fast web +f muganda +ext ents +eure kalert +escaflow ne +erra zu +ero bbins +entourage movie +enni es +ea fifamobile +dur utti +dri se +dimp led +di ard +de coloni +de blois +dak shin +crink les +cra ving +chak otay +casano vas +cancer uk +can id +brin sley +brian stann +bravest warriors +bobb ys +black guard +bay anis +bassen dean +bas eload +bar si +bar ato +bac onf +aw ong +aru iz +armad afc +ap rice +ang ame +and aman +an ani +algar ve +acar on +??? !!!! +? !!!!! +:) :) +..... !!! +**************** **************** +ðŁij§ ðŁı» +ðŁİ ı +ðŁħ°ðŁĨ ĸ +ê·ľ íĺĦ +ê³ ¨ +âĿ¤ï¸ı ðŁĴĽðŁĴĻ +âĿ¤ ðŁijij +اÙĦ رÙĬاض +yumm my +yi fang +yas ui +xx s +xer oroadshow +white knights +watch on +w wh +var key +v ril +ut kar +ushl draft +us x +un enforceable +trekon linegame +toiv onen +the bear +th sc +tex ana +tand ang +sw is +sun gazing +suha imi +suf fused +subb u +stgeorge groves +stephanie sheh +stell arton +stel ton +spur pose +sp fc +som pting +slush hq +shu dders +shor rock +sh ring +se belum +sclero tinia +sav ban +sand ham +san ral +sadi ya +rv g +resilient cities +repre sen +reha shed +re spu +ran ee +ram rahim +pur ani +profli gate +pride chs +pretty boy +photo grams +persian food +pen men +ori ol +or ale +on ii +ome where +official allegri +of riday +oak bay +now akowski +nick mangwana +nep tun +nd lea +nb fc +nat ale +much ly +most beautiful +missal ex +mis sl +mass roots +marqu art +man oh +lu hh +london winefair +loe ws +live it +launch pad +la senza +kin nick +ker inci +kel k +kar lov +kan oa +jec t +iw lca +is ya +invent ory +institution alize +ine w +incre el +ifyou can +house judiciary +hoss ack +holroyd howe +hil ditch +happ ier +hammer son +hail mary +go ward +gm si +glo fish +ghot ki +gh im +ger ges +gee kin +gav increel +gal ov +g mu +fortean times +flyn as +flesh god +feis al +fe well +fareshare uk +extra dition +eu funded +es ame +eras ure +dy ah +down light +do ki +do dgin +discover the +digital print +desau tels +deloitte us +de ports +countr ys +coo ee +commu te +cob ber +co pai +cli ente +choo sen +choices ong +che ska +chalk paint +cele stin +cathedr ale +car leasing +ca vil +ca chaca +bé la +brum mel +box ley +bour goin +bot swan +bongbong marcos +bon te +black field +b ville +az ette +autonomous driving +au ob +are ynolds +anu grah +andre wro +an ter +an se +an m +alap ai +al mand +ak bars +ah scult +ah ine +aglew ings +af feldt +ae g +ad ush +action uk +abscon der +! ðŁİģ +ðŁijį ) +è İ +ห ล +Ø ¤ +zing t +x tension +ws ferries +wo yz +will ers +wi an +walpur gis +wac ket +w mt +visit dartmoor +vil na +vi mala +u eli +tougher together +tothe max +torin ofc +thi ong +theris enation +then ic +tammy baldwin +synec doche +swanseab ay +sw ingle +strath peffer +steadi ed +ste arn +ss afety +spo well +space shi +sou live +son s +social ism +sec nav +seapor tmrkt +seabir d +scra pp +saf fold +ro pical +rmh c +ric eless +ri sp +red sky +rath aur +ram ban +race check +princess of +pressuri zation +porsch ec +plun ket +pavlo vic +paracel sus +or go +oo chee +om oon +obstetr icians +nushrat bharucha +ni fe +nes golf +ne reus +nd preps +nar n +museumc ats +montpellier hsc +mont one +meddle some +makin en +macro scopic +m dd +ly oness +locum tenens +lee ching +laurit zen +kü n +kaw ara +k gaf +jurassic june +jk jk +jeff stinco +jann arden +jackier obinson +j atim +iamjer maindefoe +i aye +hunie pop +hov de +hfx seaportmrkt +heat seeker +hatt in +ha ssi +h wd +gutt macher +gri ha +gi rolles +geor geor +geer twil +ga jap +g pdf +flyo ver +fj all +ex positions +emirates nbd +em n +drum beats +dropou thillary +drach ma +door ley +doc tober +den dera +deal maker +curmudge only +crook ham +county fire +coun ties +cosme tically +core values +col dly +co we +co group +clay travis +ce fr +carbonell nestor +capital factory +camanach d +cal ore +buy ingit +brighton hove +bou lot +blue box +blessed ness +bel liveau +beck ler +bbc sportsday +bandar ban +ban ter +az oo +aven port +ave t +aren ga +arch uk +ar é +ant acids +andre ev +amin ute +alexail acad +alber tini +ak hali +ab ish +, _ +, '" +( .@ +éĽ Ĩ +âĿĵ âĿĵ +âĢĶ ' +Ì Ģ +Ã¥le sund +zu banski +zel alem +zar ine +zar autz +yp young +yal u +y io +wood pigeon +won de +winne shiek +whe ee +well don +wall now +w pu +view park +val pro +twin kly +traffick ing +tipsare vic +thur m +thi az +theat r +the three +thailand news +team blue +t vix +t qs +sureshra ina +stu min +stock fish +stable mates +sq d +spelun king +spar ano +sn whs +smith sburg +sleip nir +sin aia +sin a +sher locks +she w +sel b +sec nielsen +schol ly +sch ellenberg +saving abel +sau vie +sath letic +sajal aly +safaricom plc +sa pper +rowh ouses +ross mann +revan th +retro spection +repre sses +relinqui shes +red squirrel +re thinking +qui zz +q alam +pw tc +protec tively +probation er +pres stitute +pre sto +pom pano +politic isation +point swest +pla smon +per mai +pe mbe +pau leen +pang lima +palmo il +p fk +over write +or nc +oc s +nub lar +north well +no senergydrink +nine tte +nikol ina +nickswag ypyoung +nick naming +nhs leadership +nadi gar +n world +mykitchen rules +my husband +moreno valley +mo ers +mike dirnt +men stennis +mclo one +may i +mat za +magdas zubanski +machin eries +luke kuechly +lu ken +lu er +lock chain +lie bermann +lati um +la wro +l blogger +ky leigh +kp tv +ke ston +ke edy +ka elyn +k te +jonim itchell +iz ingly +inter locks +il ahi +hyacin the +house master +ho bar +hindi imposition +her omoto +hab lar +gu sm +gra e +glass making +ger old +future decoded +fur ano +fra yne +fox terrier +ff xiii +fent yx +fen cing +explo iters +eu ws +eng ill +dw mtweets +dige sters +de wis +dangerous wom +danger mouse +d do +cs ba +croy de +cre te +chi bu +chat on +ch live +cece winans +calu met +bu dig +br ama +berner ay +bed clothes +be san +bayanis andiego +average hunter +aust int +append ices +anti racism +americ andad +ame sh +al cona +afre zza +afloor test +ade mo +ac ohen +ðŁĻı ðŁĩ®ðŁĩ³ +ðŁĺĢðŁĺĢ ðŁĺĢðŁĺĢ +ðŁįĵ ðŁįĵ +ðŁį¸ ðŁį¸ðŁį¸ +íģ¬ëĤĺíģ ° +ë¥ ´ +Ạ¡ +ห ร +yul ong +wor boys +wolf ers +win ick +willi ston +vote forchange +visit snowdonia +vento aureo +tom mor +timore se +thekingof queens +the cove +the club +tamar isk +t je +sw sb +sw l +sunking brewing +sub by +spring is +splinter ing +slo bber +skep tic +sir tomjones +sid lowe +sh wrs +seun gh +saint leo +sa wano +ri par +rel lik +reh mat +real jonghyun +readyfor hillary +ra elyn +prolong ation +pre views +piccin ini +pic ea +penit ence +pe ditions +pak ula +or ai +oliver io +o tide +nh cw +new battle +national assembly +nac ott +n ttw +moreco wbell +michigan tech +mayak ovsky +matil dam +mass challenge +man inder +mali kyo +malikyo ba +lymphoe dema +lor ac +lly r +llibertat preso +lig ny +libraries week +lam ey +lam be +lab at +la duma +kre pt +ko dos +killthe trade +kelsen rally +kaatru veliyidai +ka im +jose canseco +jen te +jay asurya +jay anta +jani k +jaj pur +j mac +ish ima +iren aeus +inter cal +in sley +in bangkok +illumin ed +husker fbnation +hil dr +high ams +henrik stenson +hej duk +han pti +hac ke +h ct +grail lot +gow anda +gott alo +girl love +gar çons +gangster ism +fulle rene +fin t +fel o +est court +entdeck t +em ill +ele ssp +edinbur ght +dry ads +dramati sed +dol vett +di oxin +di ame +cultu relle +cul turi +cu uuu +cru sin +cl volley +chil koot +cha aaa +cell reports +castel vetrano +bus news +bursle don +boxer bond +book posse +bon u +bon temps +bla zin +bates burg +bashi rah +bas ar +az ha +assi r +ar me +ap hl +anti serum +anthony bourdain +antece dents +ane us +amy ra +amik kelsenrally +allian zarena +ale v +ad ma +abull dogs +aa sher +,, / +( âģ¦@ +ðŁİ¶ðŁİ¶ ðŁİ¶ðŁİ¶ +ðŁİ į +ðŁįĶ ðŁįŁ +è ĸ +åĢ ī +âĿ ľ +ج ÙĪ +y pu +wx by +wirele s +win diest +weare international +wang xian +wain scot +w top +vhs india +van os +up setter +un wholesome +u ov +u ou +turbo jet +tsur ugi +tru lia +tor ino +to er +thrott le +the office +the har +the fix +testim oni +teacher friends +taec cool +stra ss +spe ters +sinthe sky +sier pinski +shichi mi +she is +shangha ima +sales enablement +sack cloth +s gro +s followparty +routledge books +roque brune +roo th +re uk +rally x +r xr +q school +pro les +pin al +pendu la +pas ku +outdoor fun +oooooooo ooooooo +oh gov +obscen ely +obi ano +obas i +nu ffin +non conformity +no ten +nar cis +mus ick +mun daring +mormon probs +mor ale +mis sam +mend elian +me gui +man ley +lili angarcia +leapfro gged +lamb swool +ker newe +kenya power +katak lysm +juve derm +joyce caroloates +jim wkyt +jean not +jackie o +it slit +it ac +isof ix +ir ão +inj akarta +im ats +humphrey bogart +hom etour +hinter land +herit ability +haway thelads +harmon ise +har ik +gy umri +gun makers +gre glou +gio i +gastri que +g vw +fun home +fren do +fort mcmurray +for youth +fix able +essenti a +ess lingen +es news +eric vergne +er minator +ep chihuahuas +ent le +engine shed +eller ton +el sayed +ebol a +e issa +dy mph +du monde +dragon lord +do gan +dil utes +die for +desertisland discs +den een +deceler ating +country boy +cor ser +cop tic +cly ffe +cher ri +casam ance +cas ona +bukitt inggi +brujer ia +book quotes +boardwalk hall +bio terrorism +bend all +behe moth +bak assi +au clair +ast mh +arou ca +app ending +alpha deltapi +ales und +ale ksa +aldubxe bloveis +alchem illa +across the +ach ing +? !# +!!! * +ðŁĺģ âľĮï¸ı +ðŁĴĽðŁĴĻðŁĴľ ðŁĴļâĿ¤ +ðŁĴĶ ðŁĺĶ +ðŁıĴ ðŁ¥ħ +ðŁ¤¤ ðŁĺį +à¸Ńภ° +zuc chi +york dale +yan a +womenin law +wine mag +walkin stown +vote dem +violet chachki +usat sport +unitedin orange +umb reon +ultram ar +ukbusiness lunch +uef ayouth +tyson foods +ttro pez +tsh ering +toy ne +ton geren +tim the +ti anti +the market +the key +tf con +tero th +team sailer +t reader +swi ley +swach hta +sur sok +summ ilux +storytell er +ssou ths +sp res +sonymusic india +smu ller +sly james +slo viansk +sis rael +she hab +sha j +senyor a +sch aff +scan al +remington arms +remb au +rati gan +ra smu +ra gen +r hanews +r fe +pym nts +prow res +pra iz +pr ss +pi ents +phel ps +pe gan +pavlovic nbcs +pab owl +p ton +over heads +ou standing +os f +on scene +offici o +occa sion +ober heim +ob cs +nis man +ni sta +nbad raft +nat ics +nanopore conf +mur rey +mu tasa +mu dgal +mt sen +mis represents +mis classification +min oa +mi at +manne h +lun guk +lov ly +let scher +len asia +lad ha +l ff +l ario +kon ichi +kerri son +keep working +k deleon +k cur +juju tsu +joseph jett +jau har +jat inder +jann ation +insinu ation +ingh orse +indiam art +honey pots +healthand fitness +haw era +hare brained +han ge +h tb +great gatsby +gott ardo +goodnightt witter +golf ball +go ias +glow sticks +glow stick +ge etv +gb hour +gar net +g pf +fur ze +fuer zas +fu gee +fri endof +frauen kirche +forec ourts +for tun +fal me +esp guitar +epistemo logical +enumer ate +elast omeric +eh s +ed w +dulwich gallery +dublin horseshow +don lan +digiov anni +deer park +daily bot +cs gazette +cotonde tulear +col ber +clear way +celebration of +cbc sask +caul kins +cartoon hangover +carol vorders +care full +car meli +by ne +buonarro ti +bum stead +bre el +bras stown +brad by +bored oms +blow fly +bloody scotland +blo q +betro thal +beng old +be ara +basil don +barbar acom +ba set +ba his +ato records +at ok +aru sh +ar si +antron brown +aman zo +amalgam ate +alleno very +ali ghting +ale au +al vy +agu std +aerop uer +ae expo +adul tedu +ad ate +acl are +ab illion +ðŁĺįðŁĺŃ ðŁĺįðŁĺŃ +ðŁ¥ĩðŁ¥ĩ ðŁ¥ĩ +îĮ ª +å½ © +à¹ĥ à¸Ī +your vote +yo liverpool +yam l +world cafe +won kyu +women fortrump +wit tes +wilton musichall +whid don +wh oring +ward ine +w bbm +van tho +val verde +val ory +v nu +ut mb +us sey +under brush +tv mohandaspai +trapezo idal +tom az +ther mali +the kitchn +the dilipkumar +the aaf +th are +team bnn +team ashishians +ta kkar +t sin +sub heading +st lucie +spacec am +smo del +sile sian +shawn stockman +shak oor +score board +sara watkins +san siro +sai pem +rt pi +rou shy +repudi ate +remed ying +reli ve +re ik +pic poul +pc s +pan jab +p kushn +or ding +onther ocks +ocv updates +obfusc ated +ob ad +oab aab +no ac +nester ov +nb h +nak amichi +na jim +mickey avalon +melissar auch +me chi +mcla gan +mc girr +magn itudes +ma ws +ma sp +m monogram +lit rpg +leg alizes +lar a +la ppe +kid swb +key shot +kar lan +k lime +juda ic +jenny lewis +jenna elfman +inclu sively +im ls +i set +homony m +ho pley +hir ta +her lings +haynes ville +happy anniversary +guar neri +gu o +gro the +google home +gon i +global health +glade sville +gl c +georgel ucas +foot light +fluff ball +fcn antes +fan army +extrapol ated +exacerb ation +er sclub +emil ym +east sussex +early start +dy dd +du etting +dry ga +dom tar +divul ging +di po +dez ma +desol at +den er +csed week +cor zo +co tulla +clark mp +cher ney +chan del +cant waitfor +ca ha +bu ghead +brush wood +bombay times +blueridge parkway +blu eroom +bikel anes +big bear +bharath iraja +beav en +b fore +awi ki +auto bus +author sofinstagram +athen sga +asi mon +ash rafi +arrabbi ata +ann curry +ambi ga +alkal ine +algorith mically +al si +al dou +afric acom +abre go +abe dian +:) .. +ðŁĴĻðŁĴļ ðŁĴĽðŁĴľ +ðŁĴĥ @ +ðŁijĮðŁijĮ ðŁijĮðŁijĮðŁijĮ +å¯ Į +ãħ ĩ +yy o +y iz +wr ona +world changers +water gate +wal eshour +vla ar +veoli auk +usace hq +us amateur +uru bamba +up welling +ul ts +uefayouth league +tupac shakur +trishap aytas +trio works +travelo gues +trampol ene +tra ister +tor con +toni and +ton der +toiletekprem katha +today is +thec gf +tham esc +tennis australia +tat lı +tart lets +swift current +su te +speed wy +spac eneedle +soci été +snap shot +sme w +shat to +shan atics +senator wong +sd ny +schar lie +sc indi +samgye opsal +sam gyimah +ry usei +ry l +ro seee +rhe o +re organizes +rdan ational +pun ahou +pre fabrication +power glide +por tw +plan ted +pete doherty +pa chan +ou mu +on love +not given +north wich +niantic labs +newworld order +mr josh +mr f +million views +metas ploit +marucci sports +mark burnett +markburnett tv +marie hamn +mar nell +mar lowe +mar ise +malasak it +mal nourishment +mad lang +m wf +ly nah +lot fi +lion smusic +li ths +lewis capaldi +learning analytics +lead o +las dun +l arian +ku ss +ku dat +key ano +ke ely +k ch +jueve s +jet charter +jay mohr +ja eden +istigh far +isles app +ios app +inx ile +intrepid museum +iklan terbaru +hill billy +helic obacter +health data +harro wer +hannah spearritt +gu wop +grow yourown +gratu ito +grape seed +gland ore +gin ato +gho stre +geon osis +fifty shade +fas cic +far lane +extre mely +eliti sts +ed ps +dw l +dungeon family +djima vic +din ghouse +deep spac +de gate +daw nzpost +davidlove photog +dar ner +crump sall +cre gan +cochlear implant +cheru bic +chel ation +chase masterson +ch under +cellu lar +canaryisland sen +bsor ules +bro seley +blk perspectives +behavi orist +barne sy +augh rim +aqu atica +amat ata +am one +allameric agame +________ ________ +! ⾨ +ðŁĻĭâĢįâĻĢï¸ı ðŁĻĭâĢįâĻĤï¸ı +ðŁĸķ ðŁı½ +ðŁijİðŁijİ ðŁijİ +ðŁİĻ ï¸ı@ +ðŁİĨ ðŁİĩ +ì Ł +⾨ âľĶ +âĻ¥ï¸ı ⾨ +âĸº # +à¸Ļภª +ج اÙħ +ö l +z wel +xfactor final +wood bury +wild child +wan tit +wa urn +viol on +ve gam +v nl +uro logy +ur araka +upone aglewings +unc wilmington +unanim ity +ubi que +transgender ism +tipping point +thinking outloud +the church +tech nip +team blackberry +tavis smiley +tam lyn +swart berg +style me +ste ds +ste acher +sp acc +solor zano +sin ghs +side m +sha be +set by +seri ally +sean hayes +satur ates +san wool +sal ar +saints row +ru stin +ru ffing +rock face +road warrior +reprezent radio +reno omokri +reboot liberty +pronovi as +pra kriti +polar ities +pete sessions +perth and +pas sp +oo iman +onemore time +one yearof +okone do +ojama jo +o week +nit zan +ngi reland +negr oni +n fre +mus lera +mu squ +mortg aging +monsta xin +mm wave +mitch y +ming les +mil utta +memphis fb +melissa ordway +may fest +man repeller +m mie +m le +ly sa +legiti mized +la ffs +knowledge management +kiernan shipka +khal is +kawarthan ow +jean ericvergne +jason bourne +jack o +ja ked +ja ima +inf am +in sky +homep ages +ho vered +ho sch +hi the +herto genbosch +he gerty +hall marking +gyor ko +gul ick +gu apa +gre gs +good foryou +gen berger +gandalf wasme +ful bourn +fru gally +fpv racing +food fact +flo y +flo rets +fe dele +el vin +effi ong +een age +eddi reader +eag ar +div vy +distill ates +deb ello +day sss +dani eller +cork chamber +cold water +cla ggett +chick y +ce oil +capability brown +camero onians +california fires +calcu tt +cal dey +brian azzarello +bren er +boys brigade +bmwmotor ra +blen kin +bio compatible +binge watching +bin nington +big bos +ber ating +basal tic +babun aidu +as ph +anthology film +angh el +al cos +ai fam +acro pora +ab berton +ðŁĺįðŁĺį ðŁĺįðŁĺĺ +ðŁĺĤ ðŁ¤Ķ +ìķĦ íĬ¸ +æĭ¡ æķ£ +ÙĦ ÛĮ +ا٠ĩ +yl p +yad u +what sinthe +westmid shour +web socket +voo c +vm ro +viv adelrio +victori ao +veen stra +ve dran +v ch +ul ing +uk business +tur ron +trin ita +times magazine +thibau d +thewrit elist +the face +th st +ter za +team nike +ta share +swan bourne +svt foe +steph on +statueof unity +st nd +speci fier +spagnu olo +so hlhockey +small faces +sin till +shus kies +shoo fly +shakti rajan +shahi dul +sd pd +schul enburg +sch tick +sawal ha +sal soul +sag acity +s vig +royalvisit canada +rox by +roeth ke +reson ances +re eni +ram blin +pwe tty +pri mark +pramo d +polo club +plu ghole +photo chemistry +phillips academy +pete sohlhockey +pe dir +ost friesland +oke chukwu +noynoy aquino +now streaming +nive les +nikon india +neo classicism +negro ponte +ne sd +nbc nightshift +na thu +n tis +n pas +n bb +n acon +my ah +mur ari +mubar ik +mo jis +missk atie +mis wak +mirror sedge +min ow +men jadi +melb derby +masch io +mar ji +mal ine +ma quis +ly onnaise +lier se +late y +large format +kour a +kost ner +king sdale +kick the +ken gen +k bbq +jw j +justicefor benghazi +juse yo +jur nee +jasminec ain +jacqu elin +inthe clouds +id lo +hss vca +honey wood +hockey isforeveryone +he bron +ha seo +h ke +gold wing +gold mines +girl slax +gi ya +garri ga +forest dale +foot action +flash game +fiat chrysler +felipe calderon +facto tum +ew stv +ev as +er kin +emiliop ucci +elock hart +ego yan +ebel ing +e iders +discer ned +demor alize +darting tonhall +damaris cotta +dairy month +cutthroat kitchen +cu bas +correspon ded +cin ar +che ssies +canton io +bowie baysox +blue chip +blair ite +bili moria +be yourbest +bb s +ban karena +ba shaw +armc andy +an chi +amberrudd hr +alex bracing +ab ashi +ðŁļ´ ðŁı¼ +ðŁĺĺ ðŁĺģ +ðŁĺ© ðŁĺŃ +ðŁİ¬ # +ðŁĮį . +ðŁĮ¾ ðŁĮ¾ +ãħ Ĭ +ñ ana +zz ato +zer din +zer be +zan ele +z aco +xxx viii +wych wood +wha a +week endo +we cantwait +viennois erie +vide op +v laminck +uta ware +un enviable +ul le +tran shu +torye lectionfraud +topo logies +tomato e +timy the +the change +ten ali +tech cc +super bock +stra uch +ss mb +sri vastav +spor tawards +sp robz +sof ÃŃa +so hi +slow travel +sic ut +si ring +sea fishing +sc ea +sbut d +sain ttropez +saaksh isra +roust about +roadtrip tv +ro chambeau +rf k +ren a +reggie bush +rege hr +re stuar +rain men +rail budget +proble mas +pon zio +perfume genius +per loff +pap azian +ou tu +ossi pee +or vis +oh hey +o zzy +nv sd +north leach +nfl freeagency +na os +myo c +mur alists +mu scaria +mo ton +mo gens +midnight red +me ins +matt sson +mark field +map info +mang ino +lucre zia +love qub +louisianat ravel +law ther +lat u +lam pson +la ppa +krisa quino +kohi stan +kit i +john nyo +iti sprashanth +hul ley +hedon ic +hass i +hackneye mpire +greens bor +gam po +futureready libs +forti fying +follow in +fle ek +flag ship +fer id +feel like +fashion week +ey b +evel ynn +entertain ingly +embe ddings +dhan u +depor te +day pack +dar rin +d pict +consul ates +conow ingo +chi yoda +cas spi +carbon ic +car ota +call ic +c da +bryo zoan +bo sio +bir dy +bab bu +aye she +av ui +archang el +ar val +aqu igley +ap lic +anti ago +an sen +ak asi +ad owns +ad fw +ac unity +:( "@ +į ° +ðŁļĹ ðŁĴ¨ +ðŁĴģ ðŁı»âĢįâĻĤï¸ı +ìĤ¬ë ¬´ +ìĤ¬ë¬´ ìĹĺ +ãģĬ ãĤģ +à´ ³ +تص ÙħÙĬ +yasuk uni +worl dy +wool ens +wentworth miller +wemy ss +we stri +waf s +vol tige +vo et +vin c +vil de +ve aled +urban wildlife +up f +uni vienna +u kyo +twitter uk +truck fest +tro caire +trail ering +too vey +togar ashi +tn ell +tin h +the jo +the hunter +the heat +tex turi +terra za +ter im +ter batas +telfor dutd +tech nation +te gh +sy ren +sud duth +submer ge +su fis +stre gi +sten zel +stech ford +st assen +splendo red +smar thomes +slumber party +sim sim +shee sha +shant iniketan +sf ballet +semen ov +schalk wyk +say es +saty amev +sap skzn +s woman +rubeng allego +ro iz +reher sal +re analysis +rainor shine +radio aire +qu mran +port lethen +por gie +plough mans +pine iro +pat nap +pap en +palar ong +over up +om achenko +nor gaard +ni dia +new bridge +national ise +mul lum +moscow mitch +mil ko +meri weather +me official +mazz etti +marian rivera +map úa +mand ai +man at +maan karate +ma sya +lore to +lop ate +latelate toyshow +lambda chi +kwa hu +kr n +kim sey +khat ami +judgen ap +joshu as +jami elynn +jami ah +j adel +it matters +ison zo +inter med +inco terms +in mind +i anni +hoo ft +hi way +hay on +hands comb +han ske +half adams +guimar as +gos lin +gian tuk +get tested +gaw k +g ww +g stephanopoulos +foe ticide +fil in +exist ences +excell encies +evam arc +ene spanol +en ext +el tiempo +dra ugr +ditt mar +disco tek +day day +dar a +dan icim +d chat +cow ering +comple tly +color adop +chou sing +chima ira +ch n +cel ite +cas bah +caro emerald +can ta +calm down +buden holzer +brink manship +boywith luv +boh len +blogger suk +bedtime story +basketbal laus +bak ht +b craw +ato gether +asho kk +as cb +art dubai +architec tes +aram m +ar pin +ant middleton +ani poke +andro logy +alexand rina +alex a +ajax capetown +* \(^ +ðŁĺĢ ðŁĺģ +ðŁĺ¿ ðŁĺ¿ +ìĪĺ íĺ¸ +모 모 +âĺºï¸ı ðŁĺĤ +à¸Ńภ¡ +à° ¶ +Ø§Ø · +yun an +worley parsons +workflo whq +witney carson +win z +we ymouth +vu cevic +vou liag +vin expo +vic gov +venkat raman +venezi ano +urgent care +udyo kta +ud yog +twee abondthatcantbebroken +tl aloc +timel ord +ticket webuk +thorn leigh +thor sby +the press +the kevin +tech forum +tau malolo +tara weeh +tan jung +t strong +sy kess +swag gart +sw amy +storybehind myscar +stein haus +sr ini +sphy g +sonamo hapatra +son en +sn ck +sle v +silic osis +sextu ple +sev aks +se infel +sc avenge +s veri +roadsof mumbai +road ton +rho diola +re entering +raymond ville +rang arajan +quir rel +q sfp +preeti karao +polit is +per versity +pat tee +oc ele +oak wood +o scott +ny drock +nature medicine +n ads +memory monday +mak tab +ma thu +lobb an +land bou +lago sians +l day +kra bbe +karan vir +jof frey +joe thomas +intercon hotels +inter web +inform acion +indeli bly +in training +il ho +hydro geology +hun ch +hell and +hahah hahahaha +ha kai +h sia +gre port +girlsin science +gen ga +fo tis +fo sbury +flanne lette +financial domination +festivalof lights +eu ets +emanu elle +eas ilocks +driven by +dream iest +dr seuss +doy ley +dorse techo +don health +dj whookid +disav owed +dic o +desi o +dean karnazes +day music +daw illiams +crowd cube +craig leith +cor bell +constant inos +col ler +co fee +co as +cme group +cho ck +catt ell +cat news +cast ner +card room +carab ins +cali bri +bor sato +ben ighted +bea sting +be yoglu +basti dores +bah man +ba rest +avell aneda +atur days +aspl und +as oe +arquite tura +ard vark +arctic frontiers +anewh ope +ai ku +adjac ency +ad um +academicswith cats +aber daron +aaron yan +ðŁĩ¬ðŁĩ§ @ +éŁ ³ +ãı Į +zor aida +yig it +wes ther +way de +water melon +wac sports +wa fula +villano vau +vicis situ +var nado +vap elove +utel aforever +ur ro +universal ism +un constitutionally +tu fan +tru iden +trij icon +topal ov +tomor ph +tomoh on +tn ght +tit lan +tit anium +thomastur goose +tho ct +this winter +thing for +theruf friderz +the ashes +tattle tale +tarte ist +tam arab +sym pathetically +student problems +stop knifecrime +spo ints +sne ider +singh is +si menon +seman as +scri bble +sc ros +sas a +sar b +sahib zada +sahara occidental +s gc +s des +row let +rc jh +rare breed +ram anan +raising awareness +rain sford +rai sen +radu lov +proc tors +pres spass +pre season +po ori +plu ot +pennstatem hky +pal en +oxfords bs +oun ified +operaholland pk +nt w +nor tec +nikkis anderson +ne xx +nav agio +na xx +na imi +muzi k +mur aco +mu hle +mother nature +moneyinthe bank +mn dnr +mirror monday +mer ville +mcne ice +mayor slyjames +marq ise +mal var +main aure +lux us +luck yday +lu per +loisa andalio +lo si +linux foundation +l ct +ky aaa +kotze bue +kirko bangz +kier sey +keh rer +k gra +john key +jhpie go +jeanne de +jav adekar +janak pur +j ni +iti oned +inv ited +international teaday +ian ziering +i boy +hid aka +heck scher +heather dubrow +hd palooza +hallo way +gu apas +gr hs +gotoireland us +good for +fuel ing +fl ö +ferdin and +fan support +evamarc ille +er ca +emabiggest fan +em f +el bridge +eco ins +dumer vil +duck s +dove awards +double standards +discover greece +dhe er +desig nin +dec ca +dd insi +dab bin +curvil inear +counterfe its +cob hc +coachj franklin +ck w +chandra shekar +cebit aus +cabernet franc +by sea +buy social +bucc al +bu sco +brody jenner +brec bassinger +bou illet +bf hardline +ber halter +bent eng +be your +bas c +bag anac +atten bury +att aboy +as kea +ari ano +ann nnnn +alimi ballard +ad mn +acher on +> ... +" "@ +ðŁĺĺ ðŁĺľ +ðŁĴķðŁĴķðŁĴķðŁĴķ ðŁĴķðŁĴķðŁĴķðŁĴķ +íĺķ ìĽIJ +ë· Ķ +âļª âļ« +ઠľ +zin nias +zale wski +yogag irl +wyr ley +woyz eck +worth morealive +world sight +won u +wil drose +wei hen +vu ko +vendredi lecture +tz on +twinpeak s +tur non +total football +top golf +thur sley +thr on +the pi +thatsmy girl +terrific tuesday +tal uk +symp a +sunny beach +su deley +ste idl +star uk +spi ano +sperkin sdnd +soule ater +sop rou +solfe ge +smo or +sketch noting +sidec inema +shubhan gi +sh indler +sed its +se dat +sd chat +scotti e +sap inker +sab ba +river park +ri mowa +reason ableness +readju sted +ran jana +quer as +py ong +pu ggy +promo products +prayforthe world +phosphat ase +ph v +pau lone +open studio +nr sv +nose k +nis ch +ng wa +new rules +ne scafe +my wolfsong +morphe tt +mon n +mon kee +mini atura +micdro premix +mesop orous +mc pher +mc coo +mbl season +marin ka +mal asada +m do +luni z +lse plc +love youu +lo ew +libret tist +letter sto +lair g +lac on +kou ga +kid shealth +kend riya +kay han +kau shiki +kan de +june jo +ju styn +john inne +johninne scentre +jo bad +jharsu guda +jer ri +jame spear +jal na +infl iction +ine gypt +incarcer ating +im hof +i asb +hel pre +head casey +headcasey mike +hallucino gen +hal lettsville +h ilife +goo din +go sparks +global summit +ger an +freethe arctic +fre aker +for trose +for summer +fe her +fcu trecht +face time +ext gen +er awan +enew man +ellef sondavid +el il +ed ball +duc ted +dr marty +doh ring +dhar ti +des ford +dent in +dau lat +climate breakdown +ci wf +chri sperkinsdnd +chill er +chevening fco +can ids +bro mides +bid dies +bham donkeys +bed in +be idou +be honest +bat en +bare la +bac ademy +b ment +b dl +anastaciam usic +an erjee +am pex +al fas +al ben +ak hir +aed as +adam aofficial +ack in +achievement unlocked +abutil on +abdel rahman +aar sbl +?? "@ +! ðŁĶ¥ðŁĶ¥ðŁĶ¥ +! ðŁijī +! :( +ðŁĺį ðŁĺĻ +ðŁİĬ ðŁİī +ë§ ¤ +ë¦ ´ +åĵ ģ +ze ma +youngen terprise +yaqu b +xavier daniel +wit tic +win st +wag ar +vovin am +vor ov +vijay filmaker +ver ilog +vapor ware +uri m +up ends +tre sham +tre mpe +total y +tor ie +today at +thimble weed +the chase +swit chers +swati jaihind +swap meet +stru ms +stock photos +stin ch +speed prayers +southernmiss fb +so ca +simon webbe +shep newsteam +ser via +sean cardo +sam ish +saakshisra wat +sa otome +rough rider +rond ina +roadand track +righ tist +ric c +renew ui +ren lagann +re treat +rc psychic +rae bareli +py x +pron o +present ando +poo jai +pau gh +pat roberts +ovi de +over development +ometal lic +nfl pabowl +nep tunes +mt dna +mother and +moore senate +mjin nocent +mistre s +mike tirico +med itech +marky ramone +maquilla je +low theband +love tour +lift back +li arin +latt in +labor atorium +ky bourbon +kr ach +ken sit +kazakh stan +kati ed +kaan ur +jones jr +jac kel +it ain +isser lis +interrup tus +international youthday +insan ity +indo chinese +im mbm +ig awa +hydro graphy +hi mitsu +herak les +her se +hend all +he ssel +havil ah +happy jhopeday +ham ina +h wu +h wp +h kt +gy res +guy ra +gun sense +gul fair +greglou ganis +grease paint +ghaz ala +gang neung +gambrin us +foodis medicine +fon ua +fi ii +felice herrig +ero icab +ero ars +er lend +engie group +ell p +eli sir +einstein maga +efra im +earl ham +e peeps +dr marth +do towsky +dis love +dim witted +de juan +daniel s +dak os +d scc +cri an +contr alto +commun is +clergy men +christmas music +chi ded +chest nu +chazz palminteri +cd na +castle martyr +cab by +brighton pride +bm v +big and +bel mon +be oplay +bat sky +bag amoyo +australian shepherd +audio boom +ath boy +as chi +arcadia unified +ap ure +ap ate +anc ities +amand ac +alife dotowsky +al san +ake elah +af sar +adam rank +. ;) +" ¡ +!! ðŁİīðŁİī +ðŁĴļðŁĴļ ðŁĴļðŁĴļðŁĴļ +ðŁİīðŁİĬ ðŁİĪ +ðŁĮ¼ ðŁĮ¼ +ðŁ§Ł âĢįâĻĢï¸ı +íĭ° ìķĦëĿ¼ +éī Ħ +âĿ¤ï¸ı ðŁĴª +ᥠ± +о ÑĤ +your car +york theatre +ye sil +yam ma +ya hog +wt fff +writers festival +worldfood prize +wor dle +we yer +vo xx +vendee globe +var ico +vancity buzz +under development +tran scultural +tor neo +tor mentors +tiltro tor +thisisse th +thisisseth sblog +the bonni +th year +tayl an +tahaw y +sw fclive +super soft +suburban ites +str ator +speaker series +sp sm +sou ad +slo per +sla visa +skipp ered +sketch cards +show pieces +sho wi +shin han +sf hs +seg no +se chs +school meals +schel de +sau rian +sag ami +ry p +rump f +rou lston +ro ffey +riz al +renega des +ren alovelis +regionalli ga +region ale +rb k +ravin dersingh +ra ux +pu plovers +pro scribed +present an +pre heated +pr illy +poul son +po ids +pm dc +pik app +photo video +phi bbs +pc j +par ash +pa tha +p rum +p drm +our community +oph ils +ny rd +nonprofit day +non resident +nin ny +ne ira +nat oli +nalim bachia +my app +muke sham +msccruise susa +morning rush +monagh angaa +mol let +mile posts +mi thu +meaning lessness +mcken ny +mathi sen +masterchef sheep +mar one +mamat abanerjee +ly dden +lon ie +lo bat +lin foot +lgb thistory +lec ats +kilmac ud +kerry katona +ke ens +karol sevilla +kam akshi +jrod fromoz +jose cuervo +jai my +interior designideas +ino ise +inder jit +hon gi +hill erman +hemam alini +hayley mcqueen +hand maids +hal kett +gui z +gross ers +gre el +gib let +gay les +fou sey +fli k +female entrepreneur +feels badman +fanni bal +f staste +erc p +ent el +en in +eli kes +ei ichiro +ed postcards +ec kankar +duc s +dor mammu +domin ik +dogand puplovers +disp leasing +dge l +del fonics +del at +de ase +d pm +cu cks +cronkit enews +congress muktbharat +color smarathi +col avita +clen buter +cipri an +cbc news +carleton college +car nal +black nose +bior xiv +bhav nalimbachia +best t +be chard +bar ma +babs draws +b fest +az atho +avon rep +att ac +asph alt +ap sf +anton etti +annane trebko +ang om +ameth asone +am oner +ah hh +ag proud +affor dance +ad joint +: ~ +//// // +$ ] +! :)) +ðŁĴį ðŁĴİ +ðŁijįðŁı½ ðŁijįðŁı½ +zar da +yuk ito +work athome +women scienceday +wit s +where withal +vic ta +vas elines +v any +utaware ru +us fda +un competitive +un believe +twoo odley +tw ofer +truck driver +tri stin +trans acted +threep wood +the yyam +the shoe +the il +tari kh +tali m +sussex lifemag +sus open +sugi mori +su long +stati stik +sta this +spring side +smo ss +set su +sch aer +sc ana +sat oh +sar la +santa anita +sambu cus +saig on +ru perts +ru ia +roman i +robb ins +real muto +reading festival +read justment +provision als +pro kabaddi +ppl summit +poe mas +pig tail +pdp nigeria +party yyy +ow lets +news sport +neptun o +naray ang +nan tong +nai ad +mur rin +mo scon +ml h +merq ury +mc t +marcus butler +makha chkala +lucky man +loe hr +ll cc +liveyour dream +lin acre +lime stone +lifeof an +li ffs +lesmiz bway +leh ri +learn coding +le user +lan l +lake eri +lado ga +ko chunni +k dd +justicele ague +jen nar +iyanla vanzant +im pi +i stre +i ib +hondac anada +hey ns +har diest +hannah bronfman +h pr +h fc +gw m +guar in +greatbritish menu +ghu ggi +ghost story +gec dsb +ge ats +gaslight anthem +garci aparra +friend sday +fl ye +fis chl +fent anil +fair vote +ery ri +episco po +eng cro +electrocardio gram +ek mainaure +dun ces +duc ators +dub nyk +distru stful +deton ators +deceler ate +debbie reynolds +de mais +day z +d th +cuar ón +cu pich +cros well +courts matter +cool kid +chol ar +cha que +catholic faith +cap olitics +c pride +by land +bran sholme +bla zek +bicy clec +be calmed +bas combe +bae jin +bad on +auri fil +audi op +attenu ated +ar ki +ar dara +anti theism +ames waran +am paio +al isms +ag api +abo g +!!! ?? +! âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +ðŁĴĻ ðŁĴļðŁĴĽ +ðŁĩ´ ðŁĩ² +ìĨ Ķë +미ë Ĥĺ +é İ +åį ģ +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +z abu +ye young +ye ji +y ma +wild boyz +weald stonefc +vote ariana +ven ne +v bn +ut zon +uise du +ufo table +ufo sighting +uff ington +tu le +tt itz +troph ication +travel awesome +transcendent alism +tra pper +tik va +ti sed +thisis england +teenwolf season +tech support +sundar c +suare zg +su sy +stron gheart +street food +story bots +sto c +st gile +srsg vac +southeast ward +soccer saturday +so zone +smid t +sm city +sli mey +sin claire +sd reader +scare d diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/Cargo.toml new file mode 100644 index 0000000..4d496f9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/Cargo.toml @@ -0,0 +1,45 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Automatic differentiation backend for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-autodiff" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-autodiff" +documentation = "https://docs.rs/burn-autodiff" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std", "tracing"] +std = ["dep:parking_lot"] +export_tests = [] # check checkpointer is_empty in tests + +tracing = [ + "dep:tracing", + "burn-std/tracing", + "burn-backend/tracing", +] + +[dependencies] +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } + + +derive-new = { workspace = true } +spin = { workspace = true } +parking_lot = { workspace = true, optional = true } +log = { workspace = true } +hashbrown = { workspace = true } +num-traits = { workspace = true } +portable-atomic = { workspace = true } +tracing = { workspace = true, optional = true, features = ["default"] } + + +[package.metadata.docs.rs] +features = ["default"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/README.md b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/README.md new file mode 100644 index 0000000..0859a5c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/README.md @@ -0,0 +1,8 @@ +# Burn Autodiff + +> [Burn](https://github.com/tracel-ai/burn) autodiff backend + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-autodiff.svg)](https://crates.io/crates/burn-autodiff) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-autodiff/blob/master/README.md) + +For now only first order reverse mode autodiff is supported. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/backend.rs new file mode 100644 index 0000000..bd8c531 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/backend.rs @@ -0,0 +1,138 @@ +use crate::{ + checkpoint::strategy::{CheckpointStrategy, NoCheckpointing}, + grads::Gradients, + tensor::AutodiffTensor, +}; +use alloc::{format, string::String}; +use burn_backend::{ + backend::{AutodiffBackend, Backend, ExecutionError}, + tensor::{BoolTensor, IntTensor, QuantizedTensor}, +}; +use core::marker::PhantomData; + +/// Enable auto-differentiation on a backend. +/// +/// This works as a backend decorator, extending the functionality of any backend with +/// backpropagation. +#[derive(Clone, Copy, Debug, Default)] +pub struct Autodiff { + _b: PhantomData, + _checkpoint_strategy: PhantomData, +} + +impl Backend for Autodiff { + type Device = B::Device; + + type FloatTensorPrimitive = AutodiffTensor; + type FloatElem = B::FloatElem; + + type IntTensorPrimitive = B::IntTensorPrimitive; + type IntElem = B::IntElem; + + type BoolTensorPrimitive = B::BoolTensorPrimitive; + type BoolElem = B::BoolElem; + + type QuantizedTensorPrimitive = B::QuantizedTensorPrimitive; + + fn ad_enabled(_device: &Self::Device) -> bool { + true + } + + fn name(device: &Self::Device) -> String { + format!("autodiff<{}>", B::name(device)) + } + + fn seed(device: &B::Device, seed: u64) { + B::seed(device, seed) + } + + fn sync(device: &B::Device) -> Result<(), ExecutionError> { + B::sync(device) + } + + fn memory_persistent_allocations Output>( + device: &Self::Device, + input: Input, + func: Func, + ) -> Output { + B::memory_persistent_allocations(device, input, func) + } + + fn memory_cleanup(device: &Self::Device) { + B::memory_cleanup(device) + } + + fn staging<'a, Iter>(data: Iter, device: &Self::Device) + where + Iter: Iterator, + { + B::staging(data, device); + } + + fn supports_dtype(device: &Self::Device, dtype: burn_std::DType) -> bool { + B::supports_dtype(device, dtype) + } + + fn dtype_usage(device: &Self::Device, dtype: burn_std::DType) -> burn_backend::DTypeUsageSet { + B::dtype_usage(device, dtype) + } +} + +impl AutodiffBackend for Autodiff { + type InnerBackend = B; + type Gradients = Gradients; + + fn backward(tensor: AutodiffTensor) -> Gradients { + tensor.backward() + } + + fn grad(tensor: &AutodiffTensor, grads: &Gradients) -> Option { + tensor.grad(grads) + } + + fn grad_remove( + tensor: &AutodiffTensor, + grads: &mut Gradients, + ) -> Option { + tensor.grad_remove(grads) + } + fn inner(tensor: AutodiffTensor) -> B::FloatTensorPrimitive { + tensor.primitive + } + + fn from_inner(tensor: B::FloatTensorPrimitive) -> AutodiffTensor { + AutodiffTensor::new(tensor) + } + + fn grad_replace( + tensor: &AutodiffTensor, + grads: &mut Self::Gradients, + grad: B::FloatTensorPrimitive, + ) { + tensor.grad_replace(grads, grad); + } + + fn int_inner(tensor: IntTensor) -> IntTensor { + tensor + } + + fn bool_inner(tensor: BoolTensor) -> BoolTensor { + tensor + } + + fn int_from_inner(tensor: IntTensor) -> IntTensor { + tensor + } + + fn bool_from_inner(tensor: BoolTensor) -> BoolTensor { + tensor + } + + fn q_inner(tensor: QuantizedTensor) -> QuantizedTensor { + tensor + } + + fn q_from_inner(tensor: QuantizedTensor) -> QuantizedTensor { + tensor + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/base.rs new file mode 100644 index 0000000..4181640 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/base.rs @@ -0,0 +1,82 @@ +use super::{ + retro_forward::RetroForwards, + state::{BackwardStates, State}, +}; +use crate::collections::HashMap; +use crate::graph::NodeId; + +use alloc::{vec, vec::Vec}; + +#[derive(new, Debug)] +/// Links a [NodeId] to its autodiff graph [NodeRef] +pub(crate) struct NodeTree { + map: HashMap>, +} + +impl NodeTree { + /// Gives the parents of the node in the autodiff graph + pub(crate) fn parents(&self, node_id: &NodeId) -> Option> { + self.map.get(node_id).cloned() + } +} + +#[derive(new, Debug)] +/// Struct responsible of fetching the output for a node in the autodiff graph during a backward pass +pub struct Checkpointer { + backward_states: BackwardStates, + retro_forwards: RetroForwards, + node_tree: NodeTree, +} + +impl Checkpointer { + /// Gives the output of the given node, by recursively asking parents to compute themselves + /// or give their pre-computed tensors. + pub fn retrieve_node_output(&mut self, node_id: NodeId) -> T + where + T: Clone + Send + 'static, + { + self.topological_sort(node_id).into_iter().for_each(|node| { + self.retro_forwards + .execute_retro_forward(node, &mut self.backward_states) + }); + + self.backward_states.get_state::(&node_id) + } + + /// Sorts the ancestors of NodeId in a way such that all parents come before their children + /// Useful to avoid recursivity later when mutating the states + /// + /// The sort on a compute bound state or a memory bound that is already computed is trivial. + /// The match on State::Computed also serves as a stopping criterion for the sort, + /// we don't need to look higher than that during recursivity. + fn topological_sort(&self, node_id: NodeId) -> Vec { + match self.backward_states.get_state_ref(&node_id) { + Some(state) => match state { + State::Recompute { n_required: _ } => { + let mut sorted = Vec::new(); + let parents = self.node_tree.parents(&node_id).unwrap(); + for parent_node in parents { + let parent_sorted = self.topological_sort(parent_node); + for ps in parent_sorted { + if !sorted.contains(&ps) { + sorted.push(ps) + } + } + } + sorted.push(node_id); + sorted + } + State::Computed { + state_content: _, + n_required: _, + } => vec![node_id], + }, + None => panic!("Node {node_id:?} is not in the backward_states. "), + } + } + + /// Checks if checkpointer has been drained adequately. Useful for testing + pub fn is_empty(&self) -> bool { + self.backward_states.is_empty() && self.retro_forwards.is_empty() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/builder.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/builder.rs new file mode 100644 index 0000000..3ecfca1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/builder.rs @@ -0,0 +1,304 @@ +use crate::{ + collections::HashMap, + graph::{ComputingProperty, NodeId}, + tensor::AutodiffTensor, +}; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use burn_backend::Backend; +use core::any::Any; + +use super::{ + base::{Checkpointer, NodeTree}, + retro_forward::{RetroForward, RetroForwards}, + state::{BackwardStates, State}, +}; + +#[derive(Debug)] +/// Determines if a node should checkpoint its computed output or its retro_forward for recomputation +/// The action is normally created by the child of the node, once the node is determined to be needed +pub enum CheckpointingAction { + /// The node's already computed output should be saved + Computed { + /// The node + node_id: NodeId, + /// The node's output + state_content: Box, + }, + /// The node should recompute itself when asked + Recompute { + /// The node + node_id: NodeId, + /// How the node should recompute itself + retro_forward: Arc, + }, +} + +// TODO: Remove that when proper client server. +unsafe impl Send for CheckpointingAction {} + +impl CheckpointingAction { + /// Utility function to access the id of the node of the checkpointing action + pub fn id(&self) -> NodeId { + match self { + CheckpointingAction::Computed { + node_id: node_ref, + state_content: _, + } => *node_ref, + CheckpointingAction::Recompute { + node_id: node_ref, + retro_forward: _, + } => *node_ref, + } + } +} + +#[derive(new, Debug, Default)] +/// Accumulates checkpoints as checkpointing actions during the forward pass, +/// and builds a checkpointer right before the backward pass +pub struct CheckpointerBuilder { + explicit_actions: Vec, + backup_actions: Vec, +} + +/// Determines if a checkpoint should impact the n_required values (Main) +/// or if it should just keep the state in case it's required (Backup) +/// +pub(crate) enum ActionType { + /// Explicit actions have been explicitly requested by some operation to retrieve their state + Explicit, + /// Backup actions are not always needed. They exist to save the output of an operation + /// whose child is memory bound, in case the state is indirectly needed when computing + /// the child's retro_forward. If no explicit action ever asks for the child's output, then + /// the backup output will go out of scope when the checkpointer is built. + Backup, +} + +impl CheckpointerBuilder { + pub(crate) fn checkpoint( + &mut self, + tensor: &AutodiffTensor, + action_type: ActionType, + ) { + let action_list = match action_type { + ActionType::Explicit => &mut self.explicit_actions, + ActionType::Backup => &mut self.backup_actions, + }; + match &tensor.node.properties { + ComputingProperty::ComputeBound | ComputingProperty::Ambiguous => { + action_list.push(CheckpointingAction::Computed { + node_id: tensor.node.id, + state_content: Box::new(tensor.primitive.clone()), + }) + } + ComputingProperty::MemoryBound { retro_forward } => { + action_list.push(CheckpointingAction::Recompute { + node_id: tensor.node.id, + retro_forward: retro_forward.clone(), + }) + } + } + } + + pub(crate) fn extend(&mut self, other: CheckpointerBuilder) { + for other_action in other.explicit_actions { + self.explicit_actions.push(other_action) + } + for other_unsure in other.backup_actions { + self.backup_actions.push(other_unsure) + } + } + + pub(crate) fn build(self, node_tree: NodeTree) -> Checkpointer { + let mut backward_states_map = HashMap::new(); + let mut retro_forwards_map = HashMap::new(); + + // Find recursion stopping points + let stop_nodes: Vec = self.find_stop_nodes(); + + // We start by identifying how many times each node will be required. + let n_required_map = self.build_n_required_map(&node_tree, stop_nodes); + + // Then we checkpoint the nodes with the corresponding n_required value + self.insert_checkpoints( + &mut backward_states_map, + &mut retro_forwards_map, + n_required_map, + ); + + Checkpointer::new( + BackwardStates::new(backward_states_map), + RetroForwards::new(retro_forwards_map), + node_tree, + ) + } + + fn find_stop_nodes(&self) -> Vec { + let mut stop_nodes = Vec::default(); + for action in self + .explicit_actions + .iter() + .chain(self.backup_actions.iter()) + { + match action { + CheckpointingAction::Computed { + node_id: node_ref, + state_content: _, + } => stop_nodes.push(*node_ref), + CheckpointingAction::Recompute { + node_id: _, + retro_forward: _, + } => {} + } + } + stop_nodes + } + + fn build_n_required_map( + &self, + node_tree: &NodeTree, + stop_nodes: Vec, + ) -> HashMap { + let mut n_required_map = HashMap::::default(); + + for action in self.explicit_actions.iter() { + match action { + CheckpointingAction::Computed { + node_id: node_ref, + state_content: _, + } => { + let id = *node_ref; + match n_required_map.remove(&id) { + Some(n) => { + n_required_map.insert(id, n + 1); + } + None => { + n_required_map.insert(id, 1); + } + }; + } + CheckpointingAction::Recompute { + node_id: node_ref, + retro_forward: _, + } => { + let id = *node_ref; + Self::update_n_required_of_parents( + id, + &mut n_required_map, + node_tree, + &stop_nodes, + ); + } + } + } + + n_required_map + } + + fn insert_checkpoints( + mut self, + backward_states_map: &mut HashMap, + retro_forward_map: &mut HashMap>, + n_required_map: HashMap, + ) { + // We do not loop over checkpointing actions anymore because they can contain + // duplicates or miss some that are in backup. We loop over the n_required_map + // from which we use the ids to find them again in the checkpointing actions + for (node_id, n_required) in n_required_map { + // We find the checkpointing action for node_id. It's likely in checkpointing_actions + // so we check there first, otherwise it will be in backup. + // Technically it can be there several times but can never be of both types, so we can assume the first we find is fine + + let action = match self + .explicit_actions + .iter() + .position(|action| action.id() == node_id) + { + Some(pos) => self.explicit_actions.remove(pos), + None => { + let pos = self + .backup_actions + .iter() + .position(|action| action.id() == node_id); + self.backup_actions.remove(pos.unwrap_or_else(|| { + panic!("Node {:?} is needed but never checkpointed", &node_id) + })) + } + }; + + match action { + CheckpointingAction::Computed { + node_id: _, + state_content, + } => { + self.checkpoint_compute(backward_states_map, node_id, state_content, n_required) + } + CheckpointingAction::Recompute { + node_id: _, + retro_forward, + } => self.checkpoint_lazy( + backward_states_map, + retro_forward_map, + node_id, + retro_forward, + n_required, + ), + }; + } + } + + fn update_n_required_of_parents( + id: NodeId, + n_required_map: &mut HashMap, + node_tree: &NodeTree, + stop_nodes: &Vec, + ) { + match n_required_map.remove(&id) { + Some(n) => { + n_required_map.insert(id, n + 1); + } + None => { + n_required_map.insert(id, 1); + if !stop_nodes.contains(&id) + && let Some(parents) = node_tree.parents(&id) + { + for p in parents { + Self::update_n_required_of_parents( + p, + n_required_map, + node_tree, + stop_nodes, + ); + } + } + } + } + } + + fn checkpoint_compute( + &self, + backward_states_map: &mut HashMap, + node_id: NodeId, + state_content: Box, + n_required: usize, + ) { + backward_states_map.insert( + node_id, + State::Computed { + state_content, + n_required, + }, + ); + } + + fn checkpoint_lazy( + &self, + backward_states_map: &mut HashMap, + retro_forward_map: &mut HashMap>, + node_id: NodeId, + retro_forward: Arc, + n_required: usize, + ) { + retro_forward_map.insert(node_id, retro_forward); + backward_states_map.insert(node_id, State::Recompute { n_required }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/mod.rs new file mode 100644 index 0000000..a67952b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/mod.rs @@ -0,0 +1,9 @@ +/// Checkpointer module +pub mod base; +pub(crate) mod builder; +/// RetroForward module +pub mod retro_forward; +/// BackwardStates module +pub mod state; +/// CheckpointStrategy module +pub mod strategy; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/retro_forward.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/retro_forward.rs new file mode 100644 index 0000000..003ad79 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/retro_forward.rs @@ -0,0 +1,116 @@ +use crate::collections::HashMap; +use crate::graph::NodeId; + +use alloc::sync::Arc; +use core::fmt::Debug; + +use super::state::{BackwardStates, State}; + +/// Definition of the forward function of a node, called during retropropagation only. +/// This is different from the normal forward function because it reads and writes from +/// the [BackwardStates] map instead of having a clear function signature. +pub trait RetroForward: Debug + Send + 'static { + /// Applies the forward pass for retropropagation. + fn forward(&self, states: &mut BackwardStates, out_node: NodeId); +} + +#[derive(new, Debug)] +/// Links [NodeId]s to their corresponding [RetroForward] +pub(crate) struct RetroForwards { + map: HashMap>, +} + +impl RetroForwards { + /// Executes the [RetroForward] for a given [NodeId] if the node's + /// [State] is [State::Recompute], otherwise does nothing. + pub(crate) fn execute_retro_forward( + &mut self, + node_id: NodeId, + backward_states: &mut BackwardStates, + ) { + if let State::Recompute { n_required: _ } = backward_states + .get_state_ref(&node_id) + .unwrap_or_else(|| panic!("Should find node {node_id:?}")) + { + // Retro forwards are always used only once because afterwards their state is computed + let retro_forward = self.map.remove(&node_id).unwrap(); + retro_forward.forward(backward_states, node_id); + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.map.is_empty() + } +} + +#[macro_export] +/// Creates a RetroForward struct for unary scalar operations +macro_rules! retro_unary_scalar { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug, Clone)] + struct $name { + lhs_id: NodeId, + rhs: Scalar, + _backend: PhantomData, + } + + impl RetroForward for $name { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let lhs = states.get_state::(&self.lhs_id); + let out = $ops(lhs, self.rhs); + states.save(out_node, out) + } + } + }; +} + +#[macro_export] +/// Creates a RetroForward struct for unary scalar operations +macro_rules! retro_unary { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug, Clone)] + struct $name { + input_id: NodeId, + _backend: PhantomData, + } + + impl RetroForward for $name { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = $ops(input); + states.save(out_node, out) + } + } + }; +} + +#[macro_export] +/// Creates a RetroForward struct for binary operations +macro_rules! retro_binary { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug, Clone)] + struct $name { + lhs_id: NodeId, + rhs_id: NodeId, + _backend: PhantomData, + } + + impl RetroForward for $name { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let lhs = states.get_state::(&self.lhs_id); + let rhs = states.get_state::(&self.rhs_id); + let out = $ops(lhs, rhs); + states.save(out_node, out) + } + } + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/state.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/state.rs new file mode 100644 index 0000000..28b9578 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/state.rs @@ -0,0 +1,144 @@ +use core::any::Any; + +use crate::collections::HashMap; +use crate::graph::NodeId; +use alloc::boxed::Box; + +/// In order to accept arbitrary node output in the same hashmap, we need to upcast them to any. +pub(crate) type StateContent = Box; + +#[derive(Debug)] +/// The state contained at one node. Encapsulates the node output if precomputed, +/// or clearly asks that it needs to be recomputed from the parents. +/// Also keeps track of the number of times the state is required so it can be removed +/// from the map of states on its last use. +pub(crate) enum State { + /// The state was not checkpointed, will need to recompute it from the node's parents + Recompute { n_required: usize }, + /// The state was checkpointed or computed during retropropagation and can be directly accessed + Computed { + state_content: StateContent, + n_required: usize, + }, +} + +impl State { + /// Returns a reference to the (not yet) downcasted node output, if checkpointed + pub(crate) fn to_state_content(&self) -> &StateContent { + match self { + State::Recompute { n_required: _ } => { + unreachable!( + "Can't get state content of recompute state. A child has likely been accessed before its parents." + ) + } + State::Computed { + state_content, + n_required: _, + } => state_content, + } + } + + /// Returns a (not yet) downcasted node output, if checkpointed + pub(crate) fn into_state_content(self) -> StateContent { + match self { + State::Recompute { n_required: _ } => { + unreachable!( + "Can't get state content of recompute state. A child has likely been accessed before its parents." + ) + } + State::Computed { + state_content, + n_required: _, + } => state_content, + } + } + + /// Returns the number of time the state is required + pub(crate) fn n_required(&self) -> usize { + match self { + State::Recompute { n_required } => *n_required, + State::Computed { + state_content: _, + n_required, + } => *n_required, + } + } +} + +#[derive(new, Default, Debug)] +/// Links [NodeId]s to their current state +pub struct BackwardStates { + map: HashMap, +} + +impl BackwardStates { + /// Returns the output in the state of the given [NodeId], + /// and decrements the number of times this state is required. + /// This function always gives ownership of the output, but will clone it if needed for further uses. + pub fn get_state(&mut self, node_id: &NodeId) -> T + where + T: Clone + Send + 'static, + { + // Fetch the state and decrement its number of required + let state = self.map.remove(node_id).unwrap(); + let remaining_n_required = state.n_required() - 1; + + // Downcast the state to whatever it is supposed to be + // If still needed after giving ownership, we copy it back to the hashmap + if remaining_n_required > 0 { + let new_stored_state = match state { + State::Recompute { n_required: _ } => unreachable!(), + State::Computed { + state_content, + n_required: _, + } => State::Computed { + state_content, + n_required: remaining_n_required, + }, + }; + + let downcasted = new_stored_state + .to_state_content() + .downcast_ref::() + .unwrap() + .clone(); + + self.insert_state(*node_id, new_stored_state); + + downcasted + } else { + let downcasted = state.into_state_content().downcast::().unwrap(); + *downcasted + } + } + + /// Returns a reference to the [State] of the given node + /// Useful when we need [State] information without needing the underlying tensor + pub(crate) fn get_state_ref(&self, node_id: &NodeId) -> Option<&State> { + self.map.get(node_id) + } + + /// Associates a [State] to its [NodeId] + pub(crate) fn insert_state(&mut self, node_id: NodeId, state: State) { + self.map.insert(node_id, state); + } + + /// Saves the output to the state of the given [NodeId]. + pub fn save(&mut self, node_id: NodeId, saved_output: T) + where + T: Clone + Send + 'static, + { + let n_required = self.get_state_ref(&node_id).unwrap().n_required(); + self.insert_state( + node_id, + State::Computed { + state_content: Box::new(saved_output), + n_required, + }, + ); + } + + pub(crate) fn is_empty(&self) -> bool { + self.map.is_empty() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/strategy.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/strategy.rs new file mode 100644 index 0000000..aae9449 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/checkpoint/strategy.rs @@ -0,0 +1,102 @@ +use core::fmt::Debug; + +use burn_backend::Backend; + +use crate::{graph::ComputingProperty, tensor::AutodiffTensor}; +use alloc::sync::Arc; + +use super::{ + builder::{ActionType, CheckpointerBuilder}, + retro_forward::RetroForward, +}; + +/// Strategy for the amount of checkpointing to do during autodiff +pub trait CheckpointStrategy: Clone + Copy + Debug + Default + Send + Sync + 'static { + /// May modify the compute property depending on the strategy + fn compute_property(retro_forward: R) -> ComputingProperty; + + /// Checkpoints parents if necessary in the strategy + fn checkpoint_parents<'a, B2, A>( + parents: A, + builder: &mut CheckpointerBuilder, + ) -> Result<(), CheckpointingError> + where + B2: Backend, + A: IntoIterator>; +} + +#[derive(Debug)] +/// Error that can happen when trying to checkpoint a tensor. +pub enum CheckpointingError { + /// When a parent is untracked, we can't easily checkpoint its state, since we don't know the + /// requirements in advanced. + UntrackedParent, +} + +#[derive(Clone, Copy, Debug, Default)] +/// All operations are considered compute bound, notwithstanding how they are marked +pub struct NoCheckpointing {} + +impl CheckpointStrategy for NoCheckpointing { + /// An operation marked as memory bound is actually compute bound. + fn compute_property(_retro_forward: R) -> ComputingProperty { + ComputingProperty::ComputeBound + } + + /// An operation marked as memory bound is actually compute bound. + /// It's therefore useless to checkpoint the parents + fn checkpoint_parents<'a, B2, A>( + _parents: A, + _builder: &mut CheckpointerBuilder, + ) -> Result<(), CheckpointingError> + where + B2: Backend, + A: IntoIterator>, + { + // Nothing to do here + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, Default)] +/// Operation properties are as they are marked (compute or memory bound) +pub struct BalancedCheckpointing {} + +impl CheckpointStrategy for BalancedCheckpointing { + /// An operation marked as memory bound is memory bound. + /// When memory bound, an operation needs to save its RetroForward + fn compute_property(retro_forward: R) -> ComputingProperty { + ComputingProperty::MemoryBound { + retro_forward: Arc::new(retro_forward), + } + } + + /// An operation marked as memory bound is really memory bound. + /// Since the operation may not checkpoint its parents but may need them indirectly + /// if asked to recompute itself, the method needs to know the parent tensors to maybe checkpoint them + fn checkpoint_parents<'a, B2, A>( + parents: A, + builder: &mut CheckpointerBuilder, + ) -> Result<(), CheckpointingError> + where + B2: Backend, + A: IntoIterator>, + { + let mut can_checkpoint = true; + + for tensor in parents.into_iter() { + if let crate::graph::Requirement::None = tensor.node.requirement { + can_checkpoint = false; + } else { + builder.checkpoint(tensor, ActionType::Backup); + } + } + + if !can_checkpoint { + *builder = CheckpointerBuilder::default(); + return Err(CheckpointingError::UntrackedParent); + } + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/grads.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/grads.rs new file mode 100644 index 0000000..e76d576 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/grads.rs @@ -0,0 +1,85 @@ +use burn_backend::{ + Backend, TensorMetadata, TensorPrimitive, + tensor::{FloatTensor, TensorContainer}, +}; + +use crate::{ + NodeId, + graph::{NodeRef, Requirement}, + tensor::AutodiffTensor, +}; + +/// Gradient identifier. +pub type GradID = u64; + +/// Gradients container used during the backward pass. +pub struct Gradients { + container: TensorContainer, +} + +impl Gradients { + /// Creates a new gradients container. + pub fn new(root_node: NodeRef, root_tensor: FloatTensor) -> Self { + let mut gradients = Self { + container: TensorContainer::new(), + }; + gradients.register::( + root_node.id, + B::float_ones( + root_tensor.shape(), + &B::float_device(&root_tensor), + root_tensor.dtype().into(), + ), + ); + gradients + } + + /// Consumes the gradients for a given tensor. + /// + /// Each tensor should be consumed exactly 1 time if its gradients are only required during the + /// backward pass, otherwise, it may be consume multiple times. + pub fn consume(&mut self, node: &NodeRef) -> FloatTensor { + match node.requirement { + Requirement::Grad => self + .container + .get::(&node.id.value) + .map(|tensor| tensor.tensor()) + .expect("Can't consume the gradients before they are registered at least once."), + Requirement::GradInBackward => self + .container + .remove::(&node.id.value) + .map(|tensor| tensor.tensor()) + .expect("Can't consume the gradients before they are registered at least once."), + Requirement::None => panic!("Trying to consume the gradients for an untracked tensor"), + } + } + + /// Removes a grad tensor from the container. + pub fn remove(&mut self, tensor: &AutodiffTensor) -> Option> { + self.container + .remove::(&tensor.node.id.value) + .map(|tensor| tensor.tensor()) + } + + /// Gets a grad tensor from the container. + pub fn get(&self, tensor: &AutodiffTensor) -> Option> { + self.container + .get::(&tensor.node.id.value) + .map(|tensor| tensor.tensor()) + } + + /// Register a grad tensor in the container. + /// + /// If the tensor already exists, add both tensors together before saving the result. + pub fn register(&mut self, node_id: NodeId, value: FloatTensor) { + if let Some(tensor_old) = self.container.remove::(&node_id.value) { + self.container.register::( + node_id.value, + TensorPrimitive::Float(B::float_add(value, tensor_old.tensor())), + ); + } else { + self.container + .register::(node_id.value, TensorPrimitive::Float(value)); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/base.rs new file mode 100644 index 0000000..63bdcb4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/base.rs @@ -0,0 +1,17 @@ +use super::NodeId; +use crate::{checkpoint::base::Checkpointer, grads::Gradients, graph::Parent}; +use alloc::boxed::Box; + +/// Backward step for reverse mode autodiff. +pub trait Step: Send + core::fmt::Debug { + /// Executes the step and consumes it. + fn step(self: Box, grads: &mut Gradients, checkpointer: &mut Checkpointer); + /// Depth of the operation relative to the first node added to a graph. + fn depth(&self) -> usize; + /// The node associated to the step. + fn node(&self) -> NodeId; + /// The parents of the node associated to the step. + fn parents(&self) -> &[Parent]; +} + +pub type StepBoxed = Box; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/mod.rs new file mode 100644 index 0000000..7e78bed --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/mod.rs @@ -0,0 +1,9 @@ +mod base; +mod node; +mod requirement; + +pub mod traversal; + +pub use base::*; +pub use node::*; +pub use requirement::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/node.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/node.rs new file mode 100644 index 0000000..a854409 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/node.rs @@ -0,0 +1,87 @@ +use alloc::{sync::Arc, vec::Vec}; + +#[cfg(target_has_atomic = "64")] +use core::sync::atomic::{AtomicU64, Ordering}; +#[cfg(not(target_has_atomic = "64"))] +use portable_atomic::{AtomicU64, Ordering}; + +use crate::checkpoint::retro_forward::RetroForward; +use crate::runtime::AutodiffClientImpl; + +use super::Requirement; + +#[derive(Debug, Clone)] +pub enum ComputingProperty { + ComputeBound, + MemoryBound { + retro_forward: Arc, + }, + Ambiguous, // Maybe autotune someday +} + +/// This is safe only because we only call RetroForward on the autodiff server. +/// Therefore, the trait will never be used by multiple threads at the same time. +/// +/// TODO: Find a way to avoid cloning the compute property, which will remove the need to add the +/// Arc, which will make (dyn RetroForward) safely implement Send. +unsafe impl Send for ComputingProperty {} +/// unsafe Sync is required because Send is only implemented for Arc, not Arc. +unsafe impl Sync for ComputingProperty {} + +/// A node contains graph metadata and should be used wrapped in an Arc for cheap cloning. +#[derive(new, Debug)] +pub struct Node { + pub parents: Vec, + pub order: usize, + pub id: NodeId, + pub requirement: Requirement, + pub properties: ComputingProperty, + pub client: AutodiffClientImpl, +} +pub type NodeRef = Arc; + +#[derive(new, Debug, Clone, PartialEq, Eq)] +pub struct Parent { + pub id: NodeId, +} + +impl Node { + /// Returns the [node](Node) only if gradients are required. + pub fn clone_if_require_grad(self: &Arc) -> Option { + match self.requirement.is_none() { + true => None, + false => Some(self.clone()), + } + } +} + +/// Unique identifier generated for each node. +#[derive(Clone, Hash, PartialEq, Eq, Debug, Copy)] +pub struct NodeId { + /// The integer representation of the id + pub value: u64, +} + +impl core::fmt::Display for NodeId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("NodeId({})", self.value)) + } +} + +impl NodeId { + /// Create a unique [node id](NodeId). + pub fn new() -> Self { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let value = COUNTER.fetch_add(1, Ordering::Relaxed); + if value == u64::MAX { + panic!("NodeId overflowed"); + } + Self { value } + } +} + +impl Default for NodeId { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/requirement.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/requirement.rs new file mode 100644 index 0000000..ce7291c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/requirement.rs @@ -0,0 +1,38 @@ +use super::NodeRef; + +/// Requirement for each tensor in the graph. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Requirement { + /// Operations that require gradients. + Grad, + /// Operations that require gradients only for backprop. + GradInBackward, + /// Operations that don't need gradients, therefore not to be included in the graph. + None, +} + +impl Requirement { + /// Returns true if gradients are not required. + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } + /// Returns the right requirement from a list of nodes. + pub fn from_nodes(nodes: &[NodeRef]) -> Self { + if nodes.len() == 1 { + return nodes[0].requirement.infer(&Requirement::None); + } + + nodes + .iter() + .map(|node| node.requirement) + .reduce(|acc, requirement| requirement.infer(&acc)) + .unwrap_or(Requirement::None) + } + + fn infer(&self, other: &Self) -> Self { + match self.is_none() && other.is_none() { + true => Self::None, + false => Self::GradInBackward, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/traversal.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/traversal.rs new file mode 100644 index 0000000..0a0afcb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/graph/traversal.rs @@ -0,0 +1,74 @@ +use super::{Step, StepBoxed}; +use crate::{ + NodeId, + collections::{HashMap, HashSet}, + graph::Parent, +}; +use alloc::vec::Vec; + +/// Breadth for search algorithm. +pub struct BreadthFirstSearch; + +pub trait TraversalItem { + fn id(&self) -> NodeId; + fn parents(&self) -> &[Parent]; + fn parent_nodes(&self) -> Vec { + self.parents().iter().map(|p| p.id).collect() + } +} + +impl BreadthFirstSearch { + /// Traverse the graph of backward steps from a root node. + pub fn traverse( + &self, + root_id: NodeId, + root_step: I, + steps: &mut HashMap, + mut callback: F, + ) where + F: FnMut(NodeId, I), + I: TraversalItem, + { + let mut visited = HashSet::new(); + let mut parents = Vec::new(); + + visited.insert(root_id); + parents.append(&mut root_step.parent_nodes()); + + callback(root_id, root_step); + + while let Some(id) = parents.pop() { + let step = match steps.remove(&id) { + Some(step) => step, + None => continue, + }; + + let step_node = step.id(); + let step_parents = step.parent_nodes(); + + if visited.contains(&step_node) { + continue; + } + + visited.insert(step_node); + + for id in step_parents.iter() { + if !visited.contains(id) { + parents.push(*id); + } + } + + callback(step_node, step); + } + } +} + +impl TraversalItem for StepBoxed { + fn id(&self) -> NodeId { + Step::node(self.as_ref()) + } + + fn parents(&self) -> &[Parent] { + Step::parents(self.as_ref()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/lib.rs new file mode 100644 index 0000000..ad19d6f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/lib.rs @@ -0,0 +1,43 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! # Burn Autodiff +//! +//! This autodiff library is a part of the Burn project. It is a standalone crate +//! that can be used to perform automatic differentiation on tensors. It is +//! designed to be used with the Burn Tensor crate, but it can be used with any +//! tensor library that implements the `Backend` trait. + +#[macro_use] +extern crate derive_new; + +extern crate alloc; + +/// Checkpoint module. +pub mod checkpoint; +/// Gradients module. +pub mod grads; +/// Operation module. +pub mod ops; + +pub(crate) mod graph; +// Exported for backend extension +pub use graph::NodeId; +pub(crate) mod tensor; +pub(crate) mod utils; + +mod backend; + +pub(crate) mod runtime; + +pub use backend::*; + +/// A facade around for HashMap and HashSet. +/// This avoids elaborate import wrangling having to happen in every module. +mod collections { + #[cfg(not(feature = "std"))] + pub use hashbrown::{HashMap, HashSet}; + #[cfg(feature = "std")] + pub use std::collections::{HashMap, HashSet}; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/activation.rs new file mode 100644 index 0000000..4be3912 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/activation.rs @@ -0,0 +1,167 @@ +use core::marker::PhantomData; + +use crate::{ + Autodiff, + checkpoint::{ + base::Checkpointer, retro_forward::RetroForward, state::BackwardStates, + strategy::CheckpointStrategy, + }, + grads::Gradients, + graph::NodeId, + ops::{Backward, Ops, OpsKind, unary}, + retro_unary, +}; +use burn_backend::{Backend, ops::ActivationOps, tensor::FloatTensor}; + +impl ActivationOps> for Autodiff { + fn gelu(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Gelu; + + retro_unary!(RetroGelu, B::gelu); + + impl Backward for Gelu { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + + unary::(ops.parents, ops.node, grads, |grad| { + B::gelu_backward(input, grad) + }); + } + } + + match Gelu + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroGelu::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::gelu(tensor.primitive.clone())) + } + OpsKind::UnTracked(prep) => prep.finish(B::gelu(tensor.primitive)), + } + } + + fn relu(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Relu; + + retro_unary!(RetroRelu, B::relu); + + impl Backward for Relu { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let state = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + B::relu_backward(state, grad) + }); + } + } + + match Relu + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroRelu::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::relu(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::relu(tensor.primitive)), + } + } + + fn sigmoid(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sigmoid; + + retro_unary!(RetroSigmoid, B::sigmoid); + + impl Backward for Sigmoid { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + let output = B::sigmoid(input); + unary::(ops.parents, ops.node, grads, |grad| { + B::sigmoid_backward(output, grad) + }); + } + } + + match Sigmoid + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSigmoid::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::sigmoid(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::sigmoid(tensor.primitive)), + } + } + + fn log_sigmoid(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct LogSigmoid; + + retro_unary!(RetroLogSigmoid, B::log_sigmoid); + + impl Backward for LogSigmoid { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + + unary::(ops.parents, ops.node, grads, |grad| { + B::log_sigmoid_backward(input, grad) + }); + } + } + + match LogSigmoid + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroLogSigmoid::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::log_sigmoid(tensor.primitive.clone())) + } + OpsKind::UnTracked(prep) => prep.finish(B::log_sigmoid(tensor.primitive)), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/backward.rs new file mode 100644 index 0000000..548acd7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/backward.rs @@ -0,0 +1,88 @@ +use super::{Ops, OpsPrep}; +use crate::{ + checkpoint::{base::Checkpointer, builder::CheckpointerBuilder, strategy::CheckpointStrategy}, + grads::Gradients, + graph::{ComputingProperty, NodeRef, Requirement}, + utils::duplicate, +}; +use burn_backend::Backend; + +/// Trait for all operations. +/// +/// # Notes +/// +/// Concrete types implementing this trait should not have any state. +/// If a state is necessary during the backward pass, +/// they should be declared with the associated type 'State'. +pub trait Backward: Send + core::fmt::Debug +where + Self: Sized + 'static, + B: Backend, +{ + /// Associated type to compute the backward pass. + type State: Clone + Send + core::fmt::Debug + 'static; + + /// The backward pass. + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ); + + /// Prepare the backward ops. + fn prepare( + self, + nodes: [NodeRef; N], + ) -> OpsPrep { + let requirement = Requirement::from_nodes(&nodes); + OpsPrep::new( + nodes, + requirement, + self, + ComputingProperty::Ambiguous, // If not specified we start with ambiguous + CheckpointerBuilder::default(), + ) + } +} + +/// Execute a binary operation during the backward step. +pub fn binary( + parents: [Option; 2], + node: NodeRef, + grads: &mut Gradients, + func_lhs: FLhs, + func_rhs: FRhs, +) where + B: Backend, + FLhs: FnOnce(B::FloatTensorPrimitive) -> B::FloatTensorPrimitive, + FRhs: FnOnce(B::FloatTensorPrimitive) -> B::FloatTensorPrimitive, +{ + let [grad_4lhs, grad_4rhs] = duplicate(&parents, Some(grads.consume::(&node))); + let [node_lhs, node_rhs] = parents; + + if let Some(node) = node_lhs { + let grad = func_lhs(grad_4lhs.unwrap()); + grads.register::(node.id, grad) + } + + if let Some(node) = node_rhs { + let grad = func_rhs(grad_4rhs.unwrap()); + grads.register::(node.id, grad) + } +} + +/// Execute a unary operation during the backward step. +pub fn unary(parents: [Option; 1], node: NodeRef, grads: &mut Gradients, func: F) +where + B: Backend, + F: FnOnce(B::FloatTensorPrimitive) -> B::FloatTensorPrimitive, +{ + let [parent_node] = parents; + let grad = grads.consume::(&node); + + if let Some(node) = parent_node { + let grad = func(grad); + grads.register::(node.id, grad) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/base.rs new file mode 100644 index 0000000..403effc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/base.rs @@ -0,0 +1,317 @@ +use super::Backward; +use crate::{ + checkpoint::{ + base::Checkpointer, + builder::{ActionType, CheckpointerBuilder}, + retro_forward::RetroForward, + strategy::CheckpointStrategy, + }, + grads::Gradients, + graph::{ComputingProperty, NodeId, NodeRef, Parent, Requirement, Step}, + tensor::AutodiffTensor, +}; +use alloc::boxed::Box; +use burn_backend::{Backend, TensorMetadata, tensor::FloatTensor}; +use burn_std::Shape; +use core::marker::PhantomData; + +/// Operation in preparation. +/// +/// Each mode has its own set of functions to minimize cloning for unused backward states. +#[derive(new)] +pub struct OpsPrep { + nodes: [NodeRef; N], + requirement: Requirement, + backward: Backward, + compute_property: ComputingProperty, + checkpointer_builder: CheckpointerBuilder, + checkpoint_strategy: PhantomData, + phantom_backend: PhantomData, + phantom_state: PhantomData, + marker: PhantomData, +} + +/// Operation is initialized +pub struct Init; +/// Operation has been tagged as memory bound +pub struct MemoryBound; +/// Memory bound operation has received its RetroForward +pub struct MemoryBoundRetroForward; +/// Operation's compute property is fixed +pub struct ComputePropertyDone; +/// Tracked operation tag. +pub struct Tracked; +/// Untracked operation tag. +pub struct UnTracked; + +impl OpsPrep +where + B: Backend, + BO: Backward, +{ + /// Indicates that the operation is compute bound, meaning its computation + /// is heavy and should not be recomputed + pub fn compute_bound(self) -> OpsPrep { + OpsPrep::new( + self.nodes, + self.requirement, + self.backward, + ComputingProperty::ComputeBound, + self.checkpointer_builder, + ) + } + + /// Indicates that the operation is memory bound, meaning its computation + /// is light and can be recomputed + pub fn memory_bound(self) -> OpsPrep { + OpsPrep::new( + self.nodes, + self.requirement, + self.backward, + self.compute_property, + self.checkpointer_builder, + ) + } +} + +impl OpsPrep +where + B: Backend, + BO: Backward, + C: CheckpointStrategy, +{ + /// Registers the retro forward, if needed + pub fn retro_forward( + self, + retro_forward: R, + ) -> OpsPrep { + OpsPrep::new( + self.nodes, + self.requirement, + self.backward, + C::compute_property(retro_forward), + self.checkpointer_builder, + ) + } +} + +impl OpsPrep +where + B: Backend, + BO: Backward, + C: CheckpointStrategy, +{ + /// Checkpoints the parents, if needed + pub fn parents<'a, B2, A>(mut self, parents: A) -> OpsPrep + where + B2: Backend, + A: IntoIterator>, + { + let compute_property = match C::checkpoint_parents(parents, &mut self.checkpointer_builder) + { + Ok(..) => self.compute_property, + Err(..) => ComputingProperty::ComputeBound, + }; + + OpsPrep::new( + self.nodes, + self.requirement, + self.backward, + compute_property, + self.checkpointer_builder, + ) + } +} + +impl OpsPrep +where + B: Backend, + BO: Backward, +{ + /// Prepare a stateless operation. + pub fn stateless(self, output: FloatTensor) -> AutodiffTensor { + match self.stateful() { + OpsKind::Tracked(prep) => prep.finish((), output), + OpsKind::UnTracked(prep) => prep.finish(output), + } + } +} + +impl OpsPrep +where + B: Backend, + S: Clone + Send + core::fmt::Debug + 'static, + BO: Backward, +{ + /// Prepare an operation that requires a state during the backward pass. + pub fn stateful(self) -> OpsKind { + match self.requirement.is_none() { + false => OpsKind::Tracked(OpsPrep::new( + self.nodes, + self.requirement, + self.backward, + self.compute_property, + self.checkpointer_builder, + )), + true => OpsKind::UnTracked(OpsPrep::new( + self.nodes, + self.requirement, + self.backward, + self.compute_property, + self.checkpointer_builder, + )), + } + } +} + +impl OpsPrep +where + B: Backend, + S: Clone + Send + core::fmt::Debug + 'static, + BO: Backward, +{ + /// Finish the preparation of an untracked operation and returns the output tensor. + pub fn finish(self, output: FloatTensor) -> AutodiffTensor { + let output = AutodiffTensor::from_parents( + output, + &self.nodes, + self.requirement, + self.compute_property, + ); + let parents = self.nodes.map(|node| node.clone_if_require_grad()); + let ops = Ops::new(parents, output.node.clone(), ()); + + // We register the ops in the graph even if untracked, otherwise memory bound operations + // that have an untracked parent would not be able to retrieve it + output.register_step(UntrackedOpsStep::new(ops), self.checkpointer_builder) + } +} + +impl OpsPrep +where + B: Backend, + S: Clone + Send + core::fmt::Debug + 'static, + BO: Backward, +{ + /// Finish the preparation of a tracked operation and returns the output tensor. + pub fn finish(self, state: S, output: FloatTensor) -> AutodiffTensor { + let output = AutodiffTensor::from_parents( + output, + &self.nodes, + self.requirement, + self.compute_property, + ); + let parents = self.nodes.map(|node| node.clone_if_require_grad()); + let ops = Ops::new(parents, output.node.clone(), state); + + output.register_step(OpsStep::new(ops, self.backward), self.checkpointer_builder) + } + + /// Checkpoints the tensor + pub fn checkpoint(&mut self, tensor: &AutodiffTensor) -> NodeId { + self.checkpointer_builder + .checkpoint(tensor, ActionType::Explicit); + + tensor.node.id + } +} + +/// Enum used before finishing tracked and untracked operations. +pub enum OpsKind { + /// Tracked operation preparation. + Tracked(OpsPrep), + /// Untracked operation preparation. + UnTracked(OpsPrep), +} + +/// Operation containing its parent nodes, its own node and the backward step state. +#[derive(new, Debug)] +pub struct Ops { + /// Parents nodes. + pub parents: [Option; N], + /// The node. + pub node: NodeRef, + /// The state. + pub state: S, +} + +/// Operation implementing backward [step](Step) with type erasing. +#[derive(new, Debug)] +struct OpsStep +where + B: Backend, + T: Backward, + SB: Clone + Send + core::fmt::Debug + 'static, +{ + ops: Ops, + backward: T, + phantom: PhantomData, +} + +impl Step for OpsStep +where + B: Backend, + T: Backward, + SB: Clone + Send + core::fmt::Debug + 'static, +{ + fn step(self: Box, grads: &mut Gradients, checkpointer: &mut Checkpointer) { + self.backward.backward(self.ops, grads, checkpointer); + } + + fn node(&self) -> NodeId { + self.ops.node.id + } + + fn parents(&self) -> &[Parent] { + &self.ops.node.parents + } + + fn depth(&self) -> usize { + self.ops.node.order + } +} + +#[derive(new, Debug)] +struct UntrackedOpsStep { + ops: Ops<(), N>, +} + +impl Step for UntrackedOpsStep { + fn step(self: Box, _grads: &mut Gradients, _checkpointer: &mut Checkpointer) { + // Nothing to do + } + + fn node(&self) -> NodeId { + self.ops.node.id + } + + fn parents(&self) -> &[Parent] { + &self.ops.node.parents + } + fn depth(&self) -> usize { + self.ops.node.order + } +} + +/// Make sure the grad tensor has the given shape. +/// +/// If broadcasting happened during the forward pass, the gradients will be sum along the +/// broadcasted dimension. +pub fn broadcast_shape(mut grad: FloatTensor, shape: &Shape) -> FloatTensor { + let shape_grad = grad.shape(); + let ndims = shape_grad.num_dims(); + + for i in 0..ndims { + if shape_grad[i] != shape[i] { + if shape[i] != 1 { + panic!( + "Invalid broadcast shapes: Next grad shape {:?}, Previous grad shape {:?}. {}", + shape, shape_grad, "Expected the shape of the next grad to be 1." + ); + } + grad = B::float_sum_dim(grad, i); + } + } + + grad +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/bool_tensor.rs new file mode 100644 index 0000000..ac3a171 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/bool_tensor.rs @@ -0,0 +1,161 @@ +use crate::{Autodiff, checkpoint::strategy::CheckpointStrategy, tensor::AutodiffTensor}; +use alloc::vec::Vec; + +use burn_backend::{ + Backend, ExecutionError, Scalar, TensorData, + ops::BoolTensorOps, + tensor::{BoolTensor, Device, IntTensor}, +}; +use burn_std::Shape; + +impl BoolTensorOps for Autodiff { + fn bool_from_data(data: TensorData, device: &Device) -> BoolTensor { + B::bool_from_data(data, device) + } + + async fn bool_into_data(tensor: BoolTensor) -> Result { + B::bool_into_data(tensor).await + } + + fn bool_into_int(tensor: BoolTensor) -> IntTensor { + B::bool_into_int(tensor) + } + + fn bool_to_device(tensor: BoolTensor, device: &Device) -> BoolTensor { + B::bool_to_device(tensor, device) + } + + fn bool_device(tensor: &BoolTensor) -> Device { + B::bool_device(tensor) + } + + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor { + B::bool_reshape(tensor, shape) + } + + fn bool_slice(tensor: BoolTensor, slices: &[burn_std::Slice]) -> BoolTensor { + B::bool_slice(tensor, slices) + } + + fn bool_empty(shape: Shape, device: &Device) -> BoolTensor { + B::bool_empty(shape, device) + } + + fn bool_zeros(shape: Shape, device: &Device) -> BoolTensor { + B::bool_zeros(shape, device) + } + + fn bool_ones(shape: Shape, device: &Device) -> BoolTensor { + B::bool_ones(shape, device) + } + + fn bool_slice_assign( + tensor: BoolTensor, + slices: &[burn_std::Slice], + value: BoolTensor, + ) -> BoolTensor { + B::bool_slice_assign(tensor, slices, value) + } + + fn bool_cat(tensors: Vec>, dim: usize) -> BoolTensor { + B::bool_cat(tensors, dim) + } + + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + B::bool_equal(lhs, rhs) + } + + fn bool_not(tensor: BoolTensor) -> BoolTensor { + B::bool_not(tensor) + } + + fn bool_and(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + B::bool_and(lhs, rhs) + } + + fn bool_or(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + B::bool_or(lhs, rhs) + } + + fn bool_xor(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + B::bool_xor(lhs, rhs) + } + + fn bool_into_float(tensor: BoolTensor) -> as Backend>::FloatTensorPrimitive { + AutodiffTensor::new(B::bool_into_float(tensor)) + } + + fn bool_swap_dims( + tensor: as Backend>::BoolTensorPrimitive, + dim1: usize, + dim2: usize, + ) -> as Backend>::BoolTensorPrimitive { + B::bool_swap_dims(tensor, dim1, dim2) + } + + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + B::bool_permute(tensor, axes) + } + + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + B::bool_flip(tensor, axes) + } + + async fn bool_argwhere(tensor: BoolTensor) -> IntTensor { + B::bool_argwhere(tensor).await + } + + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor { + B::bool_expand(tensor, shape) + } + + fn bool_repeat_dim(tensor: BoolTensor, dim: usize, times: usize) -> BoolTensor { + B::bool_repeat_dim(tensor, dim, times) + } + + fn bool_unfold( + tensor: BoolTensor, + dim: usize, + size: usize, + step: usize, + ) -> BoolTensor { + B::bool_unfold(tensor, dim, size, step) + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + source: BoolTensor, + ) -> BoolTensor { + B::bool_mask_where(tensor, mask, source) + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + B::bool_mask_fill(tensor, mask, value) + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + B::bool_gather(dim, tensor, indices) + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + B::bool_scatter_or(dim, tensor, indices, value) + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + B::bool_equal_elem(lhs, rhs) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/int_tensor.rs new file mode 100644 index 0000000..91b4d80 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/int_tensor.rs @@ -0,0 +1,406 @@ +use crate::{Autodiff, checkpoint::strategy::CheckpointStrategy, tensor::AutodiffTensor}; +use alloc::vec::Vec; + +use burn_backend::{ + Backend, Distribution, ExecutionError, Scalar, TensorData, + ops::IntTensorOps, + tensor::{BoolTensor, Device, IntTensor}, +}; +use burn_std::{IntDType, Shape}; + +impl IntTensorOps for Autodiff { + fn int_from_data(data: TensorData, device: &Device) -> IntTensor { + B::int_from_data(data, device) + } + + async fn int_into_data(tensor: IntTensor) -> Result { + B::int_into_data(tensor).await + } + + fn int_to_device(tensor: IntTensor, device: &Device) -> IntTensor { + B::int_to_device(tensor, device) + } + + fn int_device(tensor: &IntTensor) -> Device { + B::int_device(tensor) + } + + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor { + B::int_reshape(tensor, shape) + } + + fn int_slice(tensor: IntTensor, slices: &[burn_std::Slice]) -> IntTensor { + B::int_slice(tensor, slices) + } + + fn int_empty( + shape: Shape, + device: & as Backend>::Device, + dtype: IntDType, + ) -> IntTensor { + B::int_empty(shape, device, dtype) + } + + fn int_slice_assign( + tensor: IntTensor, + slices: &[burn_std::Slice], + value: IntTensor, + ) -> IntTensor { + B::int_slice_assign(tensor, slices, value) + } + + fn int_cat(tensors: Vec>, dim: usize) -> IntTensor { + B::int_cat(tensors, dim) + } + + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + B::int_equal(lhs, rhs) + } + + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + B::int_equal_elem(lhs, rhs) + } + + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::int_add(lhs, rhs) + } + + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::int_add_scalar(lhs, rhs) + } + + fn int_clamp_min(tensor: IntTensor, min: Scalar) -> IntTensor { + B::int_clamp_min(tensor, min) + } + + fn int_clamp_max(tensor: IntTensor, max: Scalar) -> IntTensor { + B::int_clamp_max(tensor, max) + } + + fn int_clamp(tensor: IntTensor, min: Scalar, max: Scalar) -> IntTensor { + B::int_clamp(tensor, min, max) + } + + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::int_sub(lhs, rhs) + } + + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::int_sub_scalar(lhs, rhs) + } + + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::int_mul(lhs, rhs) + } + + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::int_mul_scalar(lhs, rhs) + } + + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::int_div(lhs, rhs) + } + + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::int_div_scalar(lhs, rhs) + } + + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::int_remainder(lhs, rhs) + } + + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::int_remainder_scalar(lhs, rhs) + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::int_matmul(lhs, rhs) + } + + fn int_neg(tensor: IntTensor) -> IntTensor { + B::int_neg(tensor) + } + + fn int_zeros(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + B::int_zeros(shape, device, dtype) + } + + fn int_ones(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + B::int_ones(shape, device, dtype) + } + + fn int_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: IntDType, + ) -> IntTensor { + B::int_full(shape, fill_value, device, dtype) + } + + fn int_sum(tensor: IntTensor) -> IntTensor { + B::int_sum(tensor) + } + + fn int_sum_dim(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_sum_dim(tensor, dim) + } + + fn int_mean(tensor: IntTensor) -> IntTensor { + B::int_mean(tensor) + } + + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_mean_dim(tensor, dim) + } + + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_cumsum(tensor, dim) + } + + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_cumprod(tensor, dim) + } + + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_cummin(tensor, dim) + } + + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_cummax(tensor, dim) + } + + fn int_repeat_dim(tensor: IntTensor, dim: usize, times: usize) -> IntTensor { + B::int_repeat_dim(tensor, dim, times) + } + + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + B::int_greater(lhs, rhs) + } + + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + B::int_greater_elem(lhs, rhs) + } + + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + B::int_greater_equal(lhs, rhs) + } + + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + B::int_greater_equal_elem(lhs, rhs) + } + + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + B::int_lower(lhs, rhs) + } + + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + B::int_lower_elem(lhs, rhs) + } + + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + B::int_lower_equal(lhs, rhs) + } + + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + B::int_lower_equal_elem(lhs, rhs) + } + + fn int_gather(dim: usize, tensor: IntTensor, indices: IntTensor) -> IntTensor { + B::int_gather(dim, tensor, indices) + } + + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + B::int_scatter_add(dim, tensor, indices, value) + } + + fn int_select(tensor: IntTensor, dim: usize, indices: IntTensor) -> IntTensor { + B::int_select(tensor, dim, indices) + } + + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + B::int_select_add(tensor, dim, indices, value) + } + + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + value: IntTensor, + ) -> as Backend>::IntTensorPrimitive { + B::int_mask_where(tensor, mask, value) + } + + fn int_mask_fill( + tensor: IntTensor, + mask: BoolTensor, + value: Scalar, + ) -> as Backend>::IntTensorPrimitive { + B::int_mask_fill(tensor, mask, value) + } + + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_argmax(tensor, dim) + } + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_argmin(tensor, dim) + } + fn int_max(tensor: B::IntTensorPrimitive) -> B::IntTensorPrimitive { + B::int_max(tensor) + } + fn int_max_dim(tensor: B::IntTensorPrimitive, dim: usize) -> B::IntTensorPrimitive { + B::int_max_dim(tensor, dim) + } + fn int_max_dim_with_indices( + tensor: B::IntTensorPrimitive, + dim: usize, + ) -> (B::IntTensorPrimitive, B::IntTensorPrimitive) { + B::int_max_dim_with_indices(tensor, dim) + } + fn int_min(tensor: B::IntTensorPrimitive) -> B::IntTensorPrimitive { + B::int_min(tensor) + } + fn int_min_dim(tensor: B::IntTensorPrimitive, dim: usize) -> B::IntTensorPrimitive { + B::int_min_dim(tensor, dim) + } + fn int_min_dim_with_indices( + tensor: B::IntTensorPrimitive, + dim: usize, + ) -> (B::IntTensorPrimitive, B::IntTensorPrimitive) { + B::int_min_dim_with_indices(tensor, dim) + } + fn int_abs(tensor: B::IntTensorPrimitive) -> B::IntTensorPrimitive { + B::int_abs(tensor) + } + fn int_into_float( + tensor: as Backend>::IntTensorPrimitive, + ) -> as Backend>::FloatTensorPrimitive { + AutodiffTensor::new(B::int_into_float(tensor)) + } + + fn int_swap_dims( + tensor: as Backend>::IntTensorPrimitive, + dim1: usize, + dim2: usize, + ) -> as Backend>::IntTensorPrimitive { + B::int_swap_dims(tensor, dim1, dim2) + } + + fn int_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> IntTensor { + B::int_random(shape, distribution, device) + } + + fn int_arange(range: core::ops::Range, device: &Device) -> IntTensor { + B::int_arange(range, device) + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + B::int_permute(tensor, axes) + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + B::int_flip(tensor, axes) + } + + fn int_sign(tensor: IntTensor) -> IntTensor { + B::int_sign(tensor) + } + + fn int_prod(tensor: IntTensor) -> IntTensor { + B::int_prod(tensor) + } + + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_prod_dim(tensor, dim) + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + B::int_expand(tensor, shape) + } + + fn int_sort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + B::int_sort(tensor, dim, descending) + } + + fn int_sort_with_indices( + tensor: IntTensor, + dim: usize, + descending: bool, + ) -> (IntTensor, IntTensor) { + B::int_sort_with_indices(tensor, dim, descending) + } + + fn int_argsort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + B::int_argsort(tensor, dim, descending) + } + + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::bitwise_and(lhs, rhs) + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::bitwise_and_scalar(lhs, rhs) + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::bitwise_or(lhs, rhs) + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::bitwise_or_scalar(lhs, rhs) + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::bitwise_xor(lhs, rhs) + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::bitwise_xor_scalar(lhs, rhs) + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + B::bitwise_not(tensor) + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::bitwise_left_shift(lhs, rhs) + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::bitwise_left_shift_scalar(lhs, rhs) + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::bitwise_right_shift(lhs, rhs) + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::bitwise_right_shift_scalar(lhs, rhs) + } + + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor { + B::int_cast(tensor, dtype) + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + B::int_unfold(tensor, dim, size, step) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/maxmin.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/maxmin.rs new file mode 100644 index 0000000..22e4f4b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/maxmin.rs @@ -0,0 +1,27 @@ +use super::{Backward, Ops, unary}; +use crate::{checkpoint::base::Checkpointer, grads::Gradients}; +use burn_backend::{Backend, TensorMetadata}; +use burn_std::Shape; + +#[derive(Debug)] +pub(crate) struct MaxMinDim; + +impl Backward for MaxMinDim { + type State = (B::IntTensorPrimitive, Shape, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + let (indices, shape, dim) = ops.state; + let device = B::float_device(&grad); + let dtype = grad.dtype(); + let zeros = B::float_zeros(shape, &device, dtype.into()); + + B::float_scatter_add(dim, zeros, indices, grad) + }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/mod.rs new file mode 100644 index 0000000..b57184e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/mod.rs @@ -0,0 +1,15 @@ +mod activation; +mod backward; +mod base; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; + +pub(crate) mod maxmin; +pub(crate) mod sort; + +pub use backward::*; +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/module.rs new file mode 100644 index 0000000..496d65d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/module.rs @@ -0,0 +1,1828 @@ +use crate::Autodiff; +use crate::checkpoint::base::Checkpointer; +use crate::checkpoint::strategy::CheckpointStrategy; +use crate::grads::Gradients; +use crate::graph::NodeId; +use crate::ops::{Backward, Ops, unary}; +use crate::tensor::AutodiffTensor; + +use burn_backend::Backend; +use burn_backend::ops::attention::attention_fallback; +use burn_backend::ops::*; +use burn_backend::tensor::{FloatTensor, IntTensor}; + +use super::OpsKind; + +impl ModuleOps> for Autodiff { + fn embedding(weights: AutodiffTensor, indices: IntTensor) -> AutodiffTensor { + #[derive(Debug)] + struct Embedding; + + impl Backward for Embedding { + type State = (B::FloatTensorPrimitive, IntTensor); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (weights, indices) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + B::embedding_backward(weights, grad, indices) + }); + } + } + + match Embedding + .prepare::([weights.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (weights.primitive.clone(), indices.clone()), + B::embedding(weights.primitive, indices), + ), + OpsKind::UnTracked(prep) => prep.finish(B::embedding(weights.primitive, indices)), + } + } + + fn embedding_backward( + _weights: AutodiffTensor, + _output: AutodiffTensor, + _indices: IntTensor, + ) -> AutodiffTensor { + panic!("Can't differentiate embedding backward."); + } + + fn conv1d( + x: AutodiffTensor, + weight: AutodiffTensor, + bias: Option>, + options: ConvOptions<1>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct Conv1DWithBias; + #[derive(Debug)] + struct Conv1DNoBias; + + impl Backward for Conv1DWithBias { + type State = (NodeId, NodeId, NodeId, ConvOptions<1>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + let bias = checkpointer.retrieve_node_output::(bias_state); + + if let Some(node) = node_x { + let grad = B::conv1d_x_backward( + x.clone(), + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv1d_weight_backward(x.clone(), weight, grad.clone(), options); + grads.register::(node.id, grad) + } + if let Some(node) = node_bias { + let grad = B::conv1d_bias_backward(x, bias, grad); + grads.register::(node.id, grad) + } + } + } + + impl Backward for Conv1DNoBias { + type State = (NodeId, NodeId, ConvOptions<1>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + + if let Some(node) = node_x { + let grad = B::conv1d_x_backward( + x.clone(), + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv1d_weight_backward(x, weight, grad, options); + grads.register::(node.id, grad) + } + } + } + match bias { + Some(bias) => match Conv1DWithBias + .prepare::([x.node.clone(), weight.node.clone(), bias.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + prep.finish( + (x_state, weight_state, bias_state, options.clone()), + B::conv1d(x.primitive, weight.primitive, Some(bias.primitive), options), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv1d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + )), + }, + None => match Conv1DNoBias + .prepare::([x.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + prep.finish( + (x_state, weight_state, options.clone()), + B::conv1d(x.primitive, weight.primitive, None, options), + ) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::conv1d(x.primitive, weight.primitive, None, options)) + } + }, + } + } + + fn conv_transpose1d( + x: AutodiffTensor, + weight: AutodiffTensor, + bias: Option>, + options: ConvTransposeOptions<1>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct ConvTranspose1DWithBias; + #[derive(Debug)] + struct ConvTranspose1DNoBias; + + impl Backward for ConvTranspose1DWithBias { + type State = (NodeId, NodeId, NodeId, ConvTransposeOptions<1>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + let bias = checkpointer.retrieve_node_output::(bias_state); + + if let Some(node) = node_x { + let grad = B::conv_transpose1d_x_backward( + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv_transpose1d_weight_backward( + x.clone(), + weight, + grad.clone(), + options, + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_bias { + let grad = B::conv_transpose1d_bias_backward(x, bias, grad); + grads.register::(node.id, grad) + } + } + } + + impl Backward for ConvTranspose1DNoBias { + type State = (NodeId, NodeId, ConvTransposeOptions<1>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + + if let Some(node) = node_x { + let grad = B::conv_transpose1d_x_backward( + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv_transpose1d_weight_backward(x, weight, grad, options); + grads.register::(node.id, grad) + } + } + } + + match bias { + Some(bias) => match ConvTranspose1DWithBias + .prepare::([x.node.clone(), weight.node.clone(), bias.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + prep.finish( + (x_state, weight_state, bias_state, options.clone()), + B::conv_transpose1d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv_transpose1d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + )), + }, + None => match ConvTranspose1DNoBias + .prepare::([x.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + prep.finish( + (x_state, weight_state, options.clone()), + B::conv_transpose1d(x.primitive, weight.primitive, None, options), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv_transpose1d( + x.primitive, + weight.primitive, + None, + options, + )), + }, + } + } + + fn conv2d( + x: AutodiffTensor, + weight: AutodiffTensor, + bias: Option>, + options: ConvOptions<2>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct Conv2DWithBias; + #[derive(Debug)] + struct Conv2DNoBias; + + impl Backward for Conv2DWithBias { + type State = (NodeId, NodeId, NodeId, ConvOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + let bias = checkpointer.retrieve_node_output::(bias_state); + + if let Some(node) = node_x { + let grad = B::conv2d_x_backward( + x.clone(), + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = + B::conv2d_weight_backward(x.clone(), weight.clone(), grad.clone(), options); + grads.register::(node.id, grad) + } + if let Some(node) = node_bias { + let grad = B::conv2d_bias_backward(x, bias, grad); + grads.register::(node.id, grad) + } + } + } + + impl Backward for Conv2DNoBias { + type State = (NodeId, NodeId, ConvOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + + if let Some(node) = node_x { + let grad = B::conv2d_x_backward( + x.clone(), + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv2d_weight_backward(x, weight, grad, options); + grads.register::(node.id, grad) + } + } + } + + match bias { + Some(bias) => match Conv2DWithBias + .prepare::([x.node.clone(), weight.node.clone(), bias.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + prep.finish( + (x_state, weight_state, bias_state, options.clone()), + B::conv2d(x.primitive, weight.primitive, Some(bias.primitive), options), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv2d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + )), + }, + None => match Conv2DNoBias + .prepare::([x.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + prep.finish( + (x_state, weight_state, options.clone()), + B::conv2d(x.primitive, weight.primitive, None, options), + ) + } + + OpsKind::UnTracked(prep) => { + prep.finish(B::conv2d(x.primitive, weight.primitive, None, options)) + } + }, + } + } + + fn deform_conv2d( + x: AutodiffTensor, + offset: AutodiffTensor, + weight: AutodiffTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct DeformConv2DWithMaskWithBias; + #[derive(Debug)] + struct DeformConv2DWithMaskNoBias; + #[derive(Debug)] + struct DeformConv2DNoMaskWithBias; + #[derive(Debug)] + struct DeformConv2DNoMaskNoBias; + + impl Backward for DeformConv2DWithMaskWithBias { + type State = (NodeId, NodeId, NodeId, NodeId, NodeId, DeformConvOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_offset, node_weight, node_mask, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, offset_state, weight_state, mask_state, bias_state, options) = + ops.state; + let x = checkpointer.retrieve_node_output(x_state); + let offset = checkpointer.retrieve_node_output(offset_state); + let weight = checkpointer.retrieve_node_output(weight_state); + let mask = Some(checkpointer.retrieve_node_output(mask_state)); + let bias = Some(checkpointer.retrieve_node_output(bias_state)); + + let backward = + B::deform_conv2d_backward(x, offset, weight, mask, bias, grad, options); + + if let Some(node) = node_x { + grads.register::(node.id, backward.x_grad) + } + if let Some(node) = node_offset { + grads.register::(node.id, backward.offset_grad) + } + if let Some(node) = node_weight { + grads.register::(node.id, backward.weight_grad) + } + if let Some(node) = node_mask { + grads.register::(node.id, backward.mask_grad.unwrap()) + } + if let Some(node) = node_bias { + grads.register::(node.id, backward.bias_grad.unwrap()) + } + } + } + + impl Backward for DeformConv2DWithMaskNoBias { + type State = (NodeId, NodeId, NodeId, NodeId, DeformConvOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_offset, node_weight, node_mask] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, offset_state, weight_state, mask_state, options) = ops.state; + let x = checkpointer.retrieve_node_output(x_state); + let offset = checkpointer.retrieve_node_output(offset_state); + let weight = checkpointer.retrieve_node_output(weight_state); + let mask = Some(checkpointer.retrieve_node_output(mask_state)); + + let backward = + B::deform_conv2d_backward(x, offset, weight, mask, None, grad, options); + + if let Some(node) = node_x { + grads.register::(node.id, backward.x_grad) + } + if let Some(node) = node_offset { + grads.register::(node.id, backward.offset_grad) + } + if let Some(node) = node_weight { + grads.register::(node.id, backward.weight_grad) + } + if let Some(node) = node_mask { + grads.register::(node.id, backward.mask_grad.unwrap()) + } + } + } + + impl Backward for DeformConv2DNoMaskWithBias { + type State = (NodeId, NodeId, NodeId, NodeId, DeformConvOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_offset, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, offset_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output(x_state); + let offset = checkpointer.retrieve_node_output(offset_state); + let weight = checkpointer.retrieve_node_output(weight_state); + let bias = Some(checkpointer.retrieve_node_output(bias_state)); + + let backward = + B::deform_conv2d_backward(x, offset, weight, None, bias, grad, options); + + if let Some(node) = node_x { + grads.register::(node.id, backward.x_grad) + } + if let Some(node) = node_offset { + grads.register::(node.id, backward.offset_grad) + } + if let Some(node) = node_weight { + grads.register::(node.id, backward.weight_grad) + } + if let Some(node) = node_bias { + grads.register::(node.id, backward.bias_grad.unwrap()) + } + } + } + + impl Backward for DeformConv2DNoMaskNoBias { + type State = (NodeId, NodeId, NodeId, DeformConvOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_offset, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, offset_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output(x_state); + let offset = checkpointer.retrieve_node_output(offset_state); + let weight = checkpointer.retrieve_node_output(weight_state); + + let backward = + B::deform_conv2d_backward(x, offset, weight, None, None, grad, options); + + if let Some(node) = node_x { + grads.register::(node.id, backward.x_grad) + } + if let Some(node) = node_offset { + grads.register::(node.id, backward.offset_grad) + } + if let Some(node) = node_weight { + grads.register::(node.id, backward.weight_grad) + } + } + } + + match (mask, bias) { + (Some(mask), Some(bias)) => match DeformConv2DWithMaskWithBias + .prepare::([ + x.node.clone(), + offset.node.clone(), + weight.node.clone(), + mask.node.clone(), + bias.node.clone(), + ]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let offset_state = prep.checkpoint(&offset); + let weight_state = prep.checkpoint(&weight); + let mask_state = prep.checkpoint(&mask); + let bias_state = prep.checkpoint(&bias); + prep.finish( + ( + x_state, + offset_state, + weight_state, + mask_state, + bias_state, + options.clone(), + ), + B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + Some(mask.primitive), + Some(bias.primitive), + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + Some(mask.primitive), + Some(bias.primitive), + options, + )), + }, + (Some(mask), None) => match DeformConv2DWithMaskNoBias + .prepare::([ + x.node.clone(), + offset.node.clone(), + weight.node.clone(), + mask.node.clone(), + ]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let offset_state = prep.checkpoint(&offset); + let weight_state = prep.checkpoint(&weight); + let mask_state = prep.checkpoint(&mask); + prep.finish( + ( + x_state, + offset_state, + weight_state, + mask_state, + options.clone(), + ), + B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + Some(mask.primitive), + None, + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + Some(mask.primitive), + None, + options, + )), + }, + (None, Some(bias)) => match DeformConv2DNoMaskWithBias + .prepare::([ + x.node.clone(), + offset.node.clone(), + weight.node.clone(), + bias.node.clone(), + ]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let offset_state = prep.checkpoint(&offset); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + prep.finish( + ( + x_state, + offset_state, + weight_state, + bias_state, + options.clone(), + ), + B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + None, + Some(bias.primitive), + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + None, + Some(bias.primitive), + options, + )), + }, + (None, None) => match DeformConv2DNoMaskNoBias + .prepare::([x.node.clone(), offset.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let offset_state = prep.checkpoint(&offset); + let weight_state = prep.checkpoint(&weight); + prep.finish( + (x_state, offset_state, weight_state, options.clone()), + B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + None, + None, + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::deform_conv2d( + x.primitive, + offset.primitive, + weight.primitive, + None, + None, + options, + )), + }, + } + } + + fn deform_conv2d_backward( + _x: AutodiffTensor, + _offset: AutodiffTensor, + _weight: AutodiffTensor, + _mask: Option>, + _bias: Option>, + _output_grad: AutodiffTensor, + _options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + panic!("Can't differentiate deform conv 2d backward."); + } + + fn conv_transpose2d( + x: AutodiffTensor, + weight: AutodiffTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct ConvTranspose2DWithBias; + #[derive(Debug)] + struct ConvTranspose2DNoBias; + + impl Backward for ConvTranspose2DWithBias { + type State = (NodeId, NodeId, NodeId, ConvTransposeOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + let bias = checkpointer.retrieve_node_output::(bias_state); + + if let Some(node) = node_x { + let grad = B::conv_transpose2d_x_backward( + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv_transpose2d_weight_backward( + x.clone(), + weight, + grad.clone(), + options, + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_bias { + let grad = B::conv_transpose2d_bias_backward(x, bias, grad); + grads.register::(node.id, grad) + } + } + } + + impl Backward for ConvTranspose2DNoBias { + type State = (NodeId, NodeId, ConvTransposeOptions<2>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + + if let Some(node) = node_x { + let grad = B::conv_transpose2d_x_backward( + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv_transpose2d_weight_backward(x, weight, grad, options); + grads.register::(node.id, grad) + } + } + } + + match bias { + Some(bias) => match ConvTranspose2DWithBias + .prepare::([x.node.clone(), weight.node.clone(), bias.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + + prep.finish( + (x_state, weight_state, bias_state, options.clone()), + B::conv_transpose2d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv_transpose2d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + )), + }, + None => match ConvTranspose2DNoBias + .prepare::([x.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + + prep.finish( + (x_state, weight_state, options.clone()), + B::conv_transpose2d(x.primitive, weight.primitive, None, options), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv_transpose2d( + x.primitive, + weight.primitive, + None, + options, + )), + }, + } + } + + fn conv3d( + x: AutodiffTensor, + weight: AutodiffTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct Conv3DWithBias; + #[derive(Debug)] + struct Conv3DNoBias; + + impl Backward for Conv3DWithBias { + type State = (NodeId, NodeId, NodeId, ConvOptions<3>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + let bias = checkpointer.retrieve_node_output::(bias_state); + + if let Some(node) = node_x { + let grad = B::conv3d_x_backward( + x.clone(), + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = + B::conv3d_weight_backward(x.clone(), weight.clone(), grad.clone(), options); + grads.register::(node.id, grad) + } + if let Some(node) = node_bias { + let grad = B::conv3d_bias_backward(x, bias, grad); + grads.register::(node.id, grad) + } + } + } + + impl Backward for Conv3DNoBias { + type State = (NodeId, NodeId, ConvOptions<3>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + + if let Some(node) = node_x { + let grad = B::conv3d_x_backward( + x.clone(), + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv3d_weight_backward(x, weight, grad, options); + grads.register::(node.id, grad) + } + } + } + + match bias { + Some(bias) => match Conv3DWithBias + .prepare::([x.node.clone(), weight.node.clone(), bias.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + prep.finish( + (x_state, weight_state, bias_state, options.clone()), + B::conv3d(x.primitive, weight.primitive, Some(bias.primitive), options), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv3d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + )), + }, + None => match Conv3DNoBias + .prepare::([x.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + prep.finish( + (x_state, weight_state, options.clone()), + B::conv3d(x.primitive, weight.primitive, None, options), + ) + } + + OpsKind::UnTracked(prep) => { + prep.finish(B::conv3d(x.primitive, weight.primitive, None, options)) + } + }, + } + } + + fn conv_transpose3d( + x: AutodiffTensor, + weight: AutodiffTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> AutodiffTensor { + #[derive(Debug)] + struct ConvTranspose3DWithBias; + #[derive(Debug)] + struct ConvTranspose3DNoBias; + + impl Backward for ConvTranspose3DWithBias { + type State = (NodeId, NodeId, NodeId, ConvTransposeOptions<3>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight, node_bias] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, bias_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + let bias = checkpointer.retrieve_node_output::(bias_state); + + if let Some(node) = node_x { + let grad = B::conv_transpose3d_x_backward( + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv_transpose3d_weight_backward( + x.clone(), + weight, + grad.clone(), + options, + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_bias { + let grad = B::conv_transpose3d_bias_backward(x, bias, grad); + grads.register::(node.id, grad) + } + } + } + + impl Backward for ConvTranspose3DNoBias { + type State = (NodeId, NodeId, ConvTransposeOptions<3>); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_x, node_weight] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, weight_state, options) = ops.state; + let x = checkpointer.retrieve_node_output::(x_state); + let weight = + checkpointer.retrieve_node_output::(weight_state); + + if let Some(node) = node_x { + let grad = B::conv_transpose3d_x_backward( + weight.clone(), + grad.clone(), + options.clone(), + ); + grads.register::(node.id, grad) + } + if let Some(node) = node_weight { + let grad = B::conv_transpose3d_weight_backward(x, weight, grad, options); + grads.register::(node.id, grad) + } + } + } + + match bias { + Some(bias) => match ConvTranspose3DWithBias + .prepare::([x.node.clone(), weight.node.clone(), bias.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + let bias_state = prep.checkpoint(&bias); + + prep.finish( + (x_state, weight_state, bias_state, options.clone()), + B::conv_transpose3d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv_transpose3d( + x.primitive, + weight.primitive, + Some(bias.primitive), + options, + )), + }, + None => match ConvTranspose3DNoBias + .prepare::([x.node.clone(), weight.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let weight_state = prep.checkpoint(&weight); + + prep.finish( + (x_state, weight_state, options.clone()), + B::conv_transpose3d(x.primitive, weight.primitive, None, options), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::conv_transpose3d( + x.primitive, + weight.primitive, + None, + options, + )), + }, + } + } + + // TODO: Support a custom unfold4d operation by overriding the default implementation. + // + // We don't override it now because the fold operation isn't available for the backward pass. + // This implies that when autodiff is enabled, custom unfold operations defined by backends + // won't be used. Instead, the conv2d operation with custom weights matrix will be used. + // Therefore, the conv2d backward pass will be used for the unfold4d backward pass. + // + // fn unfold4d( + // x:AutodiffTensor, + // kernel_size: [usize; 2], + // options: UnfoldOptions, + // ) -> AutodiffTensor { + // todo!() + // } + + fn avg_pool1d( + x: AutodiffTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> AutodiffTensor { + #[derive(Debug)] + struct AvgPool1D; + + impl Backward for AvgPool1D { + type State = (NodeId, usize, usize, usize, bool, bool); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + let (x_state, kernel_size, stride, padding, count_include_pad, ceil_mode) = + ops.state; + let x = checkpointer.retrieve_node_output(x_state); + + if let Some(node) = node_parent { + let grad = B::avg_pool1d_backward( + x, + grad, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ); + grads.register::(node.id, grad); + } + } + } + + match AvgPool1D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + prep.finish( + ( + x_state, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ), + B::avg_pool1d( + x.primitive.clone(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::avg_pool1d( + x.primitive, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + )), + } + } + + fn avg_pool2d( + x: AutodiffTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> AutodiffTensor { + #[derive(Debug)] + struct AvgPool2D; + + impl Backward for AvgPool2D { + type State = (NodeId, [usize; 2], [usize; 2], [usize; 2], bool, bool); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + let (x_state, kernel_size, stride, padding, count_include_pad, ceil_mode) = + ops.state; + let x = checkpointer.retrieve_node_output(x_state); + + if let Some(node) = node_parent { + let grad = B::avg_pool2d_backward( + x, + grad, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ); + grads.register::(node.id, grad); + } + } + } + + match AvgPool2D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + prep.finish( + ( + x_state, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ), + B::avg_pool2d( + x.primitive.clone(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::avg_pool2d( + x.primitive, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + )), + } + } + + fn avg_pool2d_backward( + _x: AutodiffTensor, + _grad: AutodiffTensor, + _kernel_size: [usize; 2], + _stride: [usize; 2], + _padding: [usize; 2], + _count_include_pad: bool, + _ceil_mode: bool, + ) -> AutodiffTensor { + panic!("Can't differentiate avg pool 2d backward."); + } + + fn max_pool1d( + x: AutodiffTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> AutodiffTensor { + match MaxPool1D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let output = B::max_pool1d_with_indices( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + prep.finish( + ( + x_state, + output.indices, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ), + output.output, + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::max_pool1d( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + )), + } + } + + fn max_pool1d_with_indices( + x: AutodiffTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> MaxPool1dWithIndices { + match MaxPool1D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let output = B::max_pool1d_with_indices( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + + let output_tensor = prep.finish( + ( + x_state, + output.indices.clone(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ), + output.output, + ); + + MaxPool1dWithIndices::new(output_tensor, output.indices) + } + OpsKind::UnTracked(prep) => { + let output = B::max_pool1d_with_indices( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + let output_tensor = prep.finish(output.output); + + MaxPool1dWithIndices::new(output_tensor, output.indices) + } + } + } + + fn max_pool1d_with_indices_backward( + x: AutodiffTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + output_grad: AutodiffTensor, + indices: IntTensor, + ) -> MaxPool1dBackward { + let output = B::max_pool1d_with_indices_backward( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + output_grad.primitive, + indices, + ); + MaxPool1dBackward::new(AutodiffTensor::new(output.x_grad)) + } + + fn max_pool2d( + x: AutodiffTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> AutodiffTensor { + match MaxPool2D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let output = B::max_pool2d_with_indices( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + prep.finish( + ( + x_state, + output.indices, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ), + output.output, + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::max_pool2d( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + )), + } + } + + fn max_pool2d_with_indices( + x: AutodiffTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices { + match MaxPool2D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + + let output = B::max_pool2d_with_indices( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + + let output_tensor = prep.finish( + ( + x_state, + output.indices.clone(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ), + output.output, + ); + + MaxPool2dWithIndices::new(output_tensor, output.indices) + } + OpsKind::UnTracked(prep) => { + let output = B::max_pool2d_with_indices( + x.primitive, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + let output_tensor = prep.finish(output.output); + + MaxPool2dWithIndices::new(output_tensor, output.indices) + } + } + } + + fn max_pool2d_with_indices_backward( + _x: AutodiffTensor, + _kernel_size: [usize; 2], + _stride: [usize; 2], + _padding: [usize; 2], + _dilation: [usize; 2], + _ceil_mode: bool, + _output_grad: AutodiffTensor, + _indices: IntTensor, + ) -> MaxPool2dBackward { + panic!("Can't differentiate max pool2d with indices backward."); + } + fn adaptive_avg_pool1d(x: AutodiffTensor, output_size: usize) -> AutodiffTensor { + #[derive(Debug)] + struct AdaptiveAvgPool1D; + + impl Backward for AdaptiveAvgPool1D { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + let state = checkpointer.retrieve_node_output(ops.state); + + if let Some(node) = node_parent { + let grad = B::adaptive_avg_pool1d_backward(state, grad); + grads.register::(node.id, grad); + } + } + } + + match AdaptiveAvgPool1D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + prep.finish(x_state, B::adaptive_avg_pool1d(x.primitive, output_size)) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::adaptive_avg_pool1d(x.primitive, output_size)) + } + } + } + + fn adaptive_avg_pool2d(x: AutodiffTensor, output_size: [usize; 2]) -> AutodiffTensor { + #[derive(Debug)] + struct AdaptiveAvgPool2D; + + impl Backward for AdaptiveAvgPool2D { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + let state = checkpointer.retrieve_node_output(ops.state); + + if let Some(node) = node_parent { + let grad = B::adaptive_avg_pool2d_backward(state, grad); + grads.register::(node.id, grad); + } + } + } + + match AdaptiveAvgPool2D + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + prep.finish(x_state, B::adaptive_avg_pool2d(x.primitive, output_size)) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::adaptive_avg_pool2d(x.primitive, output_size)) + } + } + } + + fn adaptive_avg_pool2d_backward( + _x: AutodiffTensor, + _grad: AutodiffTensor, + ) -> as Backend>::FloatTensorPrimitive { + panic!("Can't differentiate adaptive avg pool2d backward."); + } + + fn interpolate( + x: AutodiffTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> AutodiffTensor { + #[derive(Debug)] + struct Interpolate; + impl Backward for Interpolate { + type State = (NodeId, [usize; 2], InterpolateOptions); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + + let (x_state, output_size, options) = ops.state; + let state = checkpointer.retrieve_node_output(x_state); + + if let Some(node) = node_parent { + let grad = B::interpolate_backward(state, grad, output_size, options); + grads.register::(node.id, grad); + } + } + } + + match Interpolate + .prepare::([x.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let x_state = prep.checkpoint(&x); + let output = B::interpolate(x.primitive.clone(), output_size, options.clone()); + prep.finish((x_state, output_size, options), output) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::interpolate(x.primitive, output_size, options)) + } + } + } + + fn interpolate_backward( + _x: FloatTensor>, + _grad: FloatTensor>, + _output_size: [usize; 2], + _options: InterpolateOptions, + ) -> as Backend>::FloatTensorPrimitive { + panic!("Can't differentiate interpolate backward."); + } + + fn attention( + query: FloatTensor>, + key: FloatTensor>, + value: FloatTensor>, + mask: Option>>, + attn_bias: Option>>, + options: AttentionModuleOptions, + ) -> FloatTensor> { + attention_fallback::(query, key, value, mask, attn_bias, options) + } +} + +#[derive(Debug)] +struct MaxPool1D; + +impl Backward for MaxPool1D { + type State = (NodeId, IntTensor, usize, usize, usize, usize, bool); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + let (x_state, indices, kernel_size, stride, padding, dilation, ceil_mode) = ops.state; + let x = checkpointer.retrieve_node_output(x_state); + + if let Some(node) = node_parent { + let grad = B::max_pool1d_with_indices_backward( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + grad, + indices, + ); + + grads.register::(node.id, grad.x_grad); + } + } +} + +#[derive(Debug)] +struct MaxPool2D; + +impl Backward for MaxPool2D { + type State = ( + NodeId, + IntTensor, + [usize; 2], + [usize; 2], + [usize; 2], + [usize; 2], + bool, + ); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let [node_parent] = ops.parents; + let grad = grads.consume::(&ops.node); + let (x_state, indices, kernel_size, stride, padding, dilation, ceil_mode) = ops.state; + let x = checkpointer.retrieve_node_output(x_state); + + if let Some(node) = node_parent { + let grad = B::max_pool2d_with_indices_backward( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + grad, + indices, + ); + + grads.register::(node.id, grad.x_grad); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/qtensor.rs new file mode 100644 index 0000000..0238de4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/qtensor.rs @@ -0,0 +1,106 @@ +use burn_backend::{ + Backend, ExecutionError, TensorData, + ops::QTensorOps, + tensor::{ + Device, FloatTensor, IntTensor, QuantizedTensor, + quantization::QuantizationParametersPrimitive, + }, +}; +use burn_std::{QuantScheme, Shape}; + +use crate::{Autodiff, checkpoint::strategy::CheckpointStrategy}; + +impl QTensorOps for Autodiff { + fn q_from_data(_data: TensorData, _device: &Device) -> QuantizedTensor { + todo!() + } + + fn quantize( + _tensor: FloatTensor, + _scheme: &QuantScheme, + _qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + todo!() // required for QAT + } + + fn quantize_dynamic( + _tensor: FloatTensor, + _scheme: &QuantScheme, + ) -> QuantizedTensor { + todo!() + } + + fn dequantize(_tensor: QuantizedTensor) -> FloatTensor { + todo!() + } + + fn q_device(tensor: &QuantizedTensor) -> Device { + B::q_device(tensor) + } + + fn q_to_device( + _tensor: QuantizedTensor, + _device: &Device, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_reshape(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + B::q_reshape(tensor, shape) + } + + async fn q_into_data(tensor: QuantizedTensor) -> Result { + B::q_into_data(tensor).await + } + + fn q_swap_dims( + _tensor: QuantizedTensor, + _dim1: usize, + _dim2: usize, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_permute(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_flip(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_gather( + _dim: usize, + _tensor: QuantizedTensor, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_select( + _tensor: QuantizedTensor, + _dim: usize, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_slice( + _tensor: QuantizedTensor, + _slices: &[burn_std::Slice], + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_argmax(tensor: QuantizedTensor, dim: usize) -> IntTensor { + B::q_argmax(tensor, dim) + } + + fn q_argmin(tensor: QuantizedTensor, dim: usize) -> IntTensor { + B::q_argmin(tensor, dim) + } + + fn q_expand(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/sort.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/sort.rs new file mode 100644 index 0000000..4f1831f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/sort.rs @@ -0,0 +1,27 @@ +use super::{Backward, Ops, unary}; +use crate::{checkpoint::base::Checkpointer, grads::Gradients}; +use burn_backend::{Backend, TensorMetadata}; +use burn_std::Shape; + +#[derive(Debug)] +pub(crate) struct SortDim; + +impl Backward for SortDim { + type State = (B::IntTensorPrimitive, Shape, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + let (indices, shape, dim) = ops.state; + let device = B::float_device(&grad); + let dtype = grad.dtype(); + let zeros = B::float_zeros(shape, &device, dtype.into()); + + B::float_scatter_add(dim, zeros, indices, grad) + }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/tensor.rs new file mode 100644 index 0000000..66f00bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/tensor.rs @@ -0,0 +1,3480 @@ +use alloc::{boxed::Box, vec, vec::Vec}; +use core::marker::PhantomData; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports, reason = "required on aarch64, unused on x86_64")] +use num_traits::float::Float; + +use crate::{ + Autodiff, + checkpoint::{ + base::Checkpointer, builder::CheckpointerBuilder, retro_forward::RetroForward, + state::BackwardStates, strategy::CheckpointStrategy, + }, + grads::Gradients, + graph::{ComputingProperty, NodeId, NodeRef, Parent, Requirement, Step}, + ops::{Backward, Ops, OpsKind, binary, broadcast_shape, unary}, + retro_binary, retro_unary, retro_unary_scalar, + tensor::AutodiffTensor, + utils::duplicate, +}; + +use burn_backend::{ + Backend, ExecutionError, TensorData, TensorMetadata, + ops::FloatTensorOps, + tensor::{BoolTensor, Device, FloatTensor, IntTensor}, +}; +use burn_backend::{Scalar, ops::unfold::calculate_unfold_windows}; +use burn_std::{FloatDType, Shape, Slice}; + +use super::maxmin::MaxMinDim; + +// Unsqueeze op on primitive. +fn unsqueeze_like( + tensor: B::FloatTensorPrimitive, + shape: Shape, +) -> B::FloatTensorPrimitive { + let ndims_out = shape.num_dims(); + let shape = tensor.shape(); + let ndims_in = shape.num_dims(); + + let mut dims = vec![1; ndims_out]; + let num_ones = ndims_out - ndims_in; + dims[num_ones..(ndims_in + num_ones)].copy_from_slice(&shape[..ndims_in]); + + B::float_reshape(tensor, Shape::from(dims)) +} + +impl FloatTensorOps for Autodiff { + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(data), + fields(?data.shape, ?data.dtype) + ))] + fn float_from_data(data: TensorData, device: &Device) -> FloatTensor { + AutodiffTensor::new(B::float_from_data(data, device)) + } + + fn float_random( + shape: Shape, + distribution: burn_backend::Distribution, + device: &Device, + ) -> FloatTensor { + AutodiffTensor::new(B::float_random(shape, distribution, device)) + } + + fn float_zeros(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + AutodiffTensor::new(B::float_zeros(shape, device, dtype)) + } + + fn float_ones(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + AutodiffTensor::new(B::float_ones(shape, device, dtype)) + } + + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensor), + fields( + from = ?tensor.node, + shape = ?tensor.shape(), + dtype = ?tensor.dtype(), + ) + ))] + async fn float_into_data(tensor: FloatTensor) -> Result { + B::float_into_data(tensor.primitive).await + } + + fn float_device(tensor: &FloatTensor) -> Device { + B::float_device(&tensor.primitive) + } + + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensor), + fields( + from = ?tensor.node, + shape = ?tensor.shape(), + dtype = ?tensor.dtype(), + ) + ))] + fn float_to_device(tensor: FloatTensor, device: &Device) -> FloatTensor { + #[derive(Debug)] + struct ToDevice; + + impl Backward for ToDevice { + type State = B::Device; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + B::float_to_device(grad, &ops.state) + }); + } + } + + match ToDevice + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let device_old = B::float_device(&tensor.primitive); + prep.finish(device_old, B::float_to_device(tensor.primitive, device)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_to_device(tensor.primitive, device)), + } + } + + fn float_empty(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + AutodiffTensor::new(B::float_empty(shape, device, dtype)) + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Add; + + retro_binary!(RetroAdd, B::float_add); + + impl Backward for Add { + type State = (Shape, Shape); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape_lhs, shape_rhs) = ops.state; + + binary::( + ops.parents, + ops.node, + grads, + |grad| broadcast_shape::(grad, &shape_lhs), + |grad| broadcast_shape::(grad, &shape_rhs), + ); + } + } + + match Add + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .memory_bound() + .retro_forward(RetroAdd::::new(lhs.node.id, rhs.node.id)) + .parents([&lhs, &rhs]) + .stateful() + { + OpsKind::Tracked(preps) => preps.finish( + (lhs.primitive.shape(), rhs.primitive.shape()), + B::float_add(lhs.primitive, rhs.primitive), + ), + OpsKind::UnTracked(preps) => preps.finish(B::float_add(lhs.primitive, rhs.primitive)), + } + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + #[derive(Debug)] + struct AddScalar; + + retro_unary_scalar!(RetroAddScalar, B::float_add_scalar); + + impl Backward for AddScalar { + type State = (); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| grad); + } + } + + AddScalar + .prepare::([lhs.node.clone()]) + .memory_bound() + .retro_forward(RetroAddScalar::::new(lhs.node.id, rhs)) + .parents([&lhs]) + .stateless(B::float_add_scalar(lhs.primitive, rhs)) + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sub; + + retro_binary!(RetroSub, B::float_sub); + + impl Backward for Sub { + type State = (Shape, Shape); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape_lhs, shape_rhs) = ops.state; + + binary::( + ops.parents, + ops.node, + grads, + |grad| broadcast_shape::(grad, &shape_lhs), + |grad| broadcast_shape::(B::float_neg(grad), &shape_rhs), + ); + } + } + + match Sub + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .memory_bound() + .retro_forward(RetroSub::::new(lhs.node.id, rhs.node.id)) + .parents([&lhs, &rhs]) + .stateful() + { + OpsKind::Tracked(preps) => preps.finish( + (lhs.primitive.shape(), rhs.primitive.shape()), + B::float_sub(lhs.primitive, rhs.primitive), + ), + OpsKind::UnTracked(preps) => preps.finish(B::float_sub(lhs.primitive, rhs.primitive)), + } + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + #[derive(Debug)] + struct SubScalar; + + retro_unary_scalar!(RetroSubScalar, B::float_sub_scalar); + + impl Backward for SubScalar { + type State = (); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| grad); + } + } + + SubScalar + .prepare::([lhs.node.clone()]) + .memory_bound() + .retro_forward(RetroSubScalar::::new(lhs.node.id, rhs)) + .parents([&lhs]) + .stateless(B::float_sub_scalar(lhs.primitive, rhs)) + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Mul; + + retro_binary!(RetroMul, B::float_mul); + + impl Backward for Mul { + type State = (Option, Option, BinaryOpsBroadcast); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (lhs, rhs, broadcast) = ops.state; + let lhs = lhs.map(|lhs| checkpointer.retrieve_node_output(lhs)); + let rhs = rhs.map(|rhs| checkpointer.retrieve_node_output(rhs)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + let grad = B::float_mul(grad, rhs.unwrap()); + broadcast.backward_lhs::(grad) + }, + |grad| { + let grad = B::float_mul(grad, lhs.unwrap()); + broadcast.backward_rhs::(grad) + }, + ); + } + } + + let lhs_tracked = lhs.is_tracked(); + let rhs_tracked = rhs.is_tracked(); + let broadcast = BinaryOpsBroadcast::new::(&lhs.primitive, &rhs.primitive); + + match Mul + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .memory_bound() + .retro_forward(RetroMul::::new(lhs.node.id, rhs.node.id)) + .parents([&lhs, &rhs]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let lhs_state = rhs_tracked.then(|| prep.checkpoint(&lhs)); + let rhs_state = lhs_tracked.then(|| prep.checkpoint(&rhs)); + + prep.finish( + (lhs_state, rhs_state, broadcast), + B::float_mul(lhs.primitive, rhs.primitive), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_mul(lhs.primitive, rhs.primitive)), + } + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + #[derive(Debug)] + struct MulScalar; + + retro_unary_scalar!(RetroMulScalar, B::float_mul_scalar); + + impl Backward for MulScalar { + type State = Scalar; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + B::float_mul_scalar(grad, ops.state) + }); + } + } + + match MulScalar + .prepare::([lhs.node.clone()]) + .memory_bound() + .retro_forward(RetroMulScalar::::new(lhs.node.id, rhs)) + .parents([&lhs]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish(rhs, B::float_mul_scalar(lhs.primitive, rhs)), + OpsKind::UnTracked(prep) => prep.finish(B::float_mul_scalar(lhs.primitive, rhs)), + } + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Div; + + retro_binary!(RetroDiv, B::float_div); + + impl Backward for Div { + type State = (Option, Option, BinaryOpsBroadcast); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (lhs, rhs, broadcast) = ops.state; + let lhs = lhs.map(|lhs| checkpointer.retrieve_node_output(lhs)); + let rhs = rhs.map(|rhs| checkpointer.retrieve_node_output(rhs)); + let [rhs_4lhs, rhs_4rhs] = duplicate(&ops.parents, rhs); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + let rhs = rhs_4lhs.unwrap(); + let value = B::float_recip(rhs); + let grad = B::float_mul(grad, value); + + broadcast.backward_lhs::(grad) + }, + |grad| { + let rhs = rhs_4rhs.unwrap(); + let lhs = lhs.unwrap(); + let value = + B::float_div(B::float_neg(lhs), B::float_powi_scalar(rhs, 2.into())); + let grad = B::float_mul(grad, value); + + broadcast.backward_rhs::(grad) + }, + ); + } + } + + let lhs_tracked = lhs.is_tracked(); + let rhs_tracked = rhs.is_tracked(); + let broadcast = BinaryOpsBroadcast::new::(&lhs.primitive, &rhs.primitive); + + match Div + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .memory_bound() + .retro_forward(RetroDiv::::new(lhs.node.id, rhs.node.id)) + .parents([&lhs, &rhs]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let lhs_state = rhs_tracked.then(|| prep.checkpoint(&lhs)); + let rhs_state = (lhs_tracked || rhs_tracked).then(|| prep.checkpoint(&rhs)); + + prep.finish( + (lhs_state, rhs_state, broadcast), + B::float_div(lhs.primitive, rhs.primitive), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_div(lhs.primitive, rhs.primitive)), + } + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + #[derive(Debug)] + struct DivScalar; + + retro_unary_scalar!(RetroDivScalar, B::float_div_scalar); + + impl Backward for DivScalar { + type State = Scalar; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + let tmp = 1.0 / ops.state.elem::(); + B::float_mul_scalar(grad, tmp.into()) + }); + } + } + + match DivScalar + .prepare::([lhs.node.clone()]) + .memory_bound() + .retro_forward(RetroDivScalar::::new(lhs.node.id, rhs)) + .parents([&lhs]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish(rhs, B::float_div_scalar(lhs.primitive, rhs)), + OpsKind::UnTracked(prep) => prep.finish(B::float_div_scalar(lhs.primitive, rhs)), + } + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Rem; + + retro_binary!(RetroRem, B::float_remainder); + + impl Backward for Rem { + type State = (Option, Option, BinaryOpsBroadcast); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (lhs, rhs, broadcast) = ops.state; + let lhs = lhs.map(|lhs| checkpointer.retrieve_node_output(lhs)); + let rhs = rhs.map(|rhs| checkpointer.retrieve_node_output(rhs)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + // remainder(x, y) = x - floor(x / y) * y + // partial(x - floor(x / y) * y, x) = 1 + broadcast.backward_lhs::(grad) + }, + |grad| { + // partial(x - floor(x / y) * y, y) = - floor(x / y) + let rhs = rhs.unwrap(); + let lhs = lhs.unwrap(); + let value = B::float_neg(B::float_floor(B::float_div(lhs, rhs))); + let grad = B::float_mul(grad, value); + broadcast.backward_rhs::(grad) + }, + ); + } + } + + let lhs_tracked = lhs.is_tracked(); + let rhs_tracked = rhs.is_tracked(); + let broadcast = BinaryOpsBroadcast::new::(&lhs.primitive, &rhs.primitive); + + match Rem + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .memory_bound() + .retro_forward(RetroRem::::new(lhs.node.id, rhs.node.id)) + .parents([&lhs, &rhs]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let lhs_state = rhs_tracked.then(|| prep.checkpoint(&lhs)); + let rhs_state = (lhs_tracked || rhs_tracked).then(|| prep.checkpoint(&rhs)); + + prep.finish( + (lhs_state, rhs_state, broadcast), + B::float_remainder(lhs.primitive, rhs.primitive), + ) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::float_remainder(lhs.primitive, rhs.primitive)) + } + } + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + #[derive(Debug)] + struct RemainderScalar; + + retro_unary_scalar!(RetroRemainderScalar, B::float_remainder_scalar); + + impl Backward for RemainderScalar { + type State = (); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| grad); + } + } + + RemainderScalar + .prepare::([lhs.node.clone()]) + .memory_bound() + .retro_forward(RetroRemainderScalar::::new(lhs.node.id, rhs)) + .parents([&lhs]) + .stateless(B::float_remainder_scalar(lhs.primitive, rhs)) + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Matmul; + + impl Backward for Matmul { + type State = (Option, Option, BinaryOpsBroadcast); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (lhs, rhs, broadcast) = ops.state; + let lhs = lhs.map(|lhs| checkpointer.retrieve_node_output(lhs)); + let rhs = rhs.map(|rhs| checkpointer.retrieve_node_output(rhs)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + let rhs = B::float_transpose(rhs.unwrap()); + let grad = B::float_matmul(grad, rhs); + + broadcast.backward_lhs::(grad) + }, + |grad| { + let lhs = B::float_transpose(lhs.unwrap()); + let grad = B::float_matmul(lhs, grad); + + broadcast.backward_rhs::(grad) + }, + ); + } + } + + let lhs_tracked = lhs.is_tracked(); + let rhs_tracked = rhs.is_tracked(); + let broadcast = BinaryOpsBroadcast::new::(&lhs.primitive, &rhs.primitive); + + match Matmul + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let lhs_state = rhs_tracked.then(|| prep.checkpoint(&lhs)); + let rhs_state = lhs_tracked.then(|| prep.checkpoint(&rhs)); + prep.finish( + (lhs_state, rhs_state, broadcast), + B::float_matmul(lhs.primitive, rhs.primitive), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_matmul(lhs.primitive, rhs.primitive)), + } + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + #[derive(Debug)] + struct Cross; + + impl Backward for Cross { + type State = (Option, Option, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (lhs_id, rhs_id, dim) = ops.state; + let lhs = lhs_id.map(|id| checkpointer.retrieve_node_output(id)); + let rhs = rhs_id.map(|id| checkpointer.retrieve_node_output(id)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| B::float_cross(rhs.unwrap(), grad, dim), + |grad| B::float_cross(grad, lhs.unwrap(), dim), + ); + } + } + + let lhs_tracked = lhs.is_tracked(); + let rhs_tracked = rhs.is_tracked(); + + match Cross + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(mut prep) => { + let lhs_state = rhs_tracked.then(|| prep.checkpoint(&lhs)); + let rhs_state = lhs_tracked.then(|| prep.checkpoint(&rhs)); + prep.finish( + (lhs_state, rhs_state, dim), + B::float_cross(lhs.primitive, rhs.primitive, dim), + ) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::float_cross(lhs.primitive, rhs.primitive, dim)) + } + } + } + + fn float_neg(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Neg; + + retro_unary!(RetroNeg, B::float_neg); + + impl Backward for Neg { + type State = (); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| B::float_neg(grad)); + } + } + + Neg.prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroNeg::::new(tensor.node.id)) + .parents([&tensor]) + .stateless(B::float_neg(tensor.primitive)) + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Recip; + + retro_unary!(RetroRecip, B::float_recip); + + impl Backward for Recip { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let tensor = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + let tmp = B::float_powi_scalar(tensor, (-2).into()); + let value = B::float_neg(tmp); + + B::float_mul(grad, value) + }); + } + } + + match Recip + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroRecip::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_recip(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_recip(tensor.primitive)), + } + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + #[derive(Debug)] + struct SwapDim; + + #[derive(new, Debug)] + struct RetroSwapDims { + input_id: NodeId, + dim1: usize, + dim2: usize, + _backend: PhantomData, + } + + impl RetroForward for RetroSwapDims { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = B::float_swap_dims(input, self.dim1, self.dim2); + states.save(out_node, out) + } + } + + impl Backward for SwapDim { + type State = (usize, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (dim1, dim2) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + B::float_swap_dims(grad, dim2, dim1) + }); + } + } + + match SwapDim + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSwapDims::::new(tensor.node.id, dim1, dim2)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (dim1, dim2), + B::float_swap_dims(tensor.primitive, dim1, dim2), + ), + OpsKind::UnTracked(prep) => { + prep.finish(B::float_swap_dims(tensor.primitive, dim1, dim2)) + } + } + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + #[derive(Debug)] + struct PermuteDim; + + #[derive(new, Debug)] + struct RetroPermuteDims { + input_id: NodeId, + axes: Vec, + _backend: PhantomData, + } + + impl RetroForward for RetroPermuteDims { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = B::float_permute(input, &self.axes); + states.save(out_node, out) + } + } + + impl Backward for PermuteDim { + type State = Vec; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let axes = ops.state; + + let mut inverse = vec![0usize; axes.len()]; + axes.iter() + .enumerate() + .for_each(|(i, &axis)| inverse[axis] = i); + + unary::(ops.parents, ops.node, grads, |grad| { + B::float_permute(grad, &inverse) + }); + } + } + + match PermuteDim + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroPermuteDims::::new(tensor.node.id, axes.to_vec())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => { + prep.finish(axes.to_vec(), B::float_permute(tensor.primitive, axes)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_permute(tensor.primitive, axes)), + } + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + #[derive(Debug)] + struct FlipDim; + + #[derive(new, Debug)] + struct RetroFlipDims { + input_id: NodeId, + axes: Vec, + _backend: PhantomData, + } + + impl RetroForward for RetroFlipDims { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = B::float_flip(input, &self.axes); + states.save(out_node, out) + } + } + + impl Backward for FlipDim { + type State = Vec; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let axes = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + B::float_flip(grad, &axes) + }); + } + } + + match FlipDim + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroFlipDims::::new(tensor.node.id, axes.to_vec())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => { + prep.finish(axes.to_vec(), B::float_flip(tensor.primitive, axes)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_flip(tensor.primitive, axes)), + } + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + #[derive(Debug)] + struct ReshapeDim; + + #[derive(new, Debug)] + struct RetroReshape { + input_id: NodeId, + shape: Shape, + _backend: PhantomData, + } + + impl RetroForward for RetroReshape { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = B::float_reshape(input, self.shape.clone()); + states.save(out_node, out) + } + } + + impl Backward for ReshapeDim { + type State = (Shape, Shape); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape_original, shape) = ops.state; + let ndims_out = shape.num_dims(); + + unary::(ops.parents, ops.node, grads, |grad| { + let shape_grad = grad.shape(); + let mut grad = grad; + + for i in 0..ndims_out { + if shape[i] == 1 && shape_grad[i] != 1 { + grad = B::float_sum_dim(grad, i); + } + } + + B::float_reshape(grad, shape_original) + }); + } + } + + match ReshapeDim + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroReshape::::new(tensor.node.id, shape.clone())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.shape(), shape.clone()), + B::float_reshape(tensor.primitive, shape), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_reshape(tensor.primitive, shape)), + } + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + #[derive(Debug)] + struct Gather; + + impl Backward for Gather { + type State = (usize, IntTensor, Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (dim, indices, shape, device) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + let zeros = B::float_zeros(shape, &device, grad.dtype().into()); + B::float_scatter_add(dim, zeros, indices, grad) + }); + } + } + + match Gather + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + ( + dim, + indices.clone(), + tensor.primitive.shape(), + B::float_device(&tensor.primitive), + ), + B::float_gather(dim, tensor.primitive, indices), + ), + OpsKind::UnTracked(prep) => { + prep.finish(B::float_gather(dim, tensor.primitive, indices)) + } + } + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + #[derive(Debug)] + struct Scatter; + + impl Backward for Scatter { + type State = (usize, IntTensor); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (dim, indices) = ops.state; + let [_, indices_4rhs] = duplicate(&ops.parents, Some(indices)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| grad, + |grad| B::float_gather(dim, grad, indices_4rhs.unwrap()), + ); + } + } + + match Scatter + .prepare::([tensor.node, value.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (dim, indices.clone()), + B::float_scatter_add(dim, tensor.primitive, indices, value.primitive), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_scatter_add( + dim, + tensor.primitive, + indices, + value.primitive, + )), + } + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + ) -> FloatTensor { + #[derive(Debug)] + struct Select; + + #[derive(new, Debug)] + struct RetroSelect { + input_id: NodeId, + dim: usize, + indices: IntTensor, + } + + impl RetroForward for RetroSelect { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = B::float_select(input, self.dim, self.indices.clone()); + states.save(out_node, out) + } + } + + impl Backward for Select { + type State = (usize, IntTensor, Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (dim, indices, shape, device) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + let zeros = B::float_zeros(shape, &device, grad.dtype().into()); + B::float_select_add(zeros, dim, indices, grad) + }); + } + } + + match Select + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSelect::::new(tensor.node.id, dim, indices.clone())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + ( + dim, + indices.clone(), + tensor.primitive.shape(), + B::float_device(&tensor.primitive), + ), + B::float_select(tensor.primitive, dim, indices), + ), + OpsKind::UnTracked(prep) => { + prep.finish(B::float_select(tensor.primitive, dim, indices)) + } + } + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + #[derive(Debug)] + struct IndexSelectDimAssign; + + #[derive(new, Debug)] + struct RetroSelectAssign { + tensor_id: NodeId, + dim: usize, + indices: IntTensor, + value_id: NodeId, + } + + impl RetroForward for RetroSelectAssign { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let tensor = states.get_state::(&self.tensor_id); + let value = states.get_state::(&self.value_id); + let out = B::float_select_add(tensor, self.dim, self.indices.clone(), value); + states.save(out_node, out) + } + } + + impl Backward for IndexSelectDimAssign { + type State = (usize, IntTensor); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (dim, indices) = ops.state; + + binary::( + ops.parents, + ops.node, + grads, + |grad| grad, + |grad| B::float_select(grad, dim, indices), + ); + } + } + + match IndexSelectDimAssign + .prepare::([tensor.node.clone(), value.node.clone()]) + .memory_bound() + .retro_forward(RetroSelectAssign::::new( + tensor.node.id, + dim, + indices.clone(), + value.node.id, + )) + .parents([&tensor, &value]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (dim, indices.clone()), + B::float_select_add(tensor.primitive, dim, indices, value.primitive), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_select_add( + tensor.primitive, + dim, + indices, + value.primitive, + )), + } + } + + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor { + #[derive(Debug)] + struct Index; + + #[derive(new, Debug)] + struct RetroSlice { + tensor_id: NodeId, + slices: Vec, + _backend: PhantomData, + } + + impl RetroForward for RetroSlice { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let tensor = states.get_state::(&self.tensor_id); + let out = B::float_slice(tensor, &self.slices); + states.save(out_node, out) + } + } + + impl Backward for Index { + type State = (Vec, Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (slices, shape, device) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + let zeros = B::float_zeros(shape, &device, grad.dtype().into()); + B::float_slice_assign(zeros, &slices, grad) + }); + } + } + + match Index + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSlice::::new(tensor.node.id, slices.to_vec())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + ( + slices.to_vec(), + tensor.primitive.shape(), + B::float_device(&tensor.primitive), + ), + B::float_slice(tensor.primitive, slices), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_slice(tensor.primitive, slices)), + } + } + + fn float_slice_assign( + tensor: FloatTensor, + slices: &[Slice], + value: FloatTensor, + ) -> FloatTensor { + #[derive(Debug)] + struct SliceAssign; + + #[derive(new, Debug)] + struct RetroSliceAssign { + tensor_id: NodeId, + slices: Vec, + value_id: NodeId, + _backend: PhantomData, + } + + impl RetroForward for RetroSliceAssign { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let tensor = states.get_state::(&self.tensor_id); + let value = states.get_state::(&self.value_id); + let out = B::float_slice_assign(tensor, &self.slices, value); + states.save(out_node, out) + } + } + + impl Backward for SliceAssign { + type State = (Vec, Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (slices, shape_rhs, device) = ops.state; + let [slices_4lhs, slices_4rhs] = duplicate(&ops.parents, Some(slices)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + let zeros = B::float_zeros(shape_rhs, &device, grad.dtype().into()); + B::float_slice_assign(grad, &slices_4lhs.unwrap(), zeros) + }, + |grad| B::float_slice(grad, &slices_4rhs.unwrap()), + ); + } + } + + match SliceAssign + .prepare::([tensor.node.clone(), value.node.clone()]) + .memory_bound() + .retro_forward(RetroSliceAssign::::new( + tensor.node.id, + slices.to_vec(), + value.node.id, + )) + .parents([&tensor, &value]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + ( + slices.to_vec(), + value.primitive.shape(), + B::float_device(&value.primitive), + ), + B::float_slice_assign(tensor.primitive, slices, value.primitive), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_slice_assign( + tensor.primitive, + slices, + value.primitive, + )), + } + } + + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + source: FloatTensor, + ) -> FloatTensor { + #[derive(Debug)] + struct MaskWhere; + + impl Backward for MaskWhere { + type State = (BoolTensor, Shape, Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (mask, shape_lhs, shape_rhs, device) = ops.state; + let [mask_4lhs, mask_4rhs] = duplicate(&ops.parents, Some(mask)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + let zeros = B::float_zeros(shape_lhs.clone(), &device, grad.dtype().into()); + let grad = B::float_mask_where(grad, mask_4lhs.unwrap(), zeros); + + broadcast_shape::(grad, &shape_lhs) + }, + |grad| { + let zeros = B::float_zeros(shape_rhs.clone(), &device, grad.dtype().into()); + let grad = B::float_mask_where(zeros, mask_4rhs.unwrap(), grad); + + broadcast_shape::(grad, &shape_rhs) + }, + ); + } + } + + match MaskWhere + .prepare::([tensor.node, source.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + ( + mask.clone(), + tensor.primitive.shape(), + source.primitive.shape(), + B::float_device(&source.primitive), + ), + B::float_mask_where(tensor.primitive, mask, source.primitive), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_mask_where( + tensor.primitive, + mask, + source.primitive, + )), + } + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor { + #[derive(Debug)] + struct MaskFill; + + impl Backward for MaskFill { + type State = BoolTensor; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + B::float_mask_fill(grad, ops.state, 0f32.into()) + }); + } + } + + match MaskFill + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + mask.clone(), + B::float_mask_fill(tensor.primitive, mask, value), + ), + OpsKind::UnTracked(prep) => { + prep.finish(B::float_mask_fill(tensor.primitive, mask, value)) + } + } + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + B::float_equal(lhs.primitive, rhs.primitive) + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + B::float_equal_elem(lhs.primitive, rhs) + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + B::float_greater(lhs.primitive, rhs.primitive) + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + B::float_greater_elem(lhs.primitive, rhs) + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + B::float_greater_equal(lhs.primitive, rhs.primitive) + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + B::float_greater_equal_elem(lhs.primitive, rhs) + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + B::float_lower(lhs.primitive, rhs.primitive) + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + B::float_lower_elem(lhs.primitive, rhs) + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + B::float_lower_equal(lhs.primitive, rhs.primitive) + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + B::float_lower_equal_elem(lhs.primitive, rhs) + } + + fn float_is_nan(tensor: FloatTensor) -> BoolTensor { + B::float_is_nan(tensor.primitive) + } + + fn float_is_inf(tensor: FloatTensor) -> BoolTensor { + B::float_is_inf(tensor.primitive) + } + + fn float_detach(tensor: FloatTensor) -> FloatTensor { + // When we detach a tensor, we remove it from the graph, but we still want to keep the + // `require_grad` setting. + let is_require_grad = Self::float_is_require_grad(&tensor); + let tensor = AutodiffTensor::new(tensor.primitive); + + match is_require_grad { + true => tensor.require_grad(), + false => tensor, + } + } + + fn float_set_require_grad(tensor: FloatTensor, require_grad: bool) -> FloatTensor { + if require_grad { + return tensor.require_grad(); + } + + AutodiffTensor::new(tensor.primitive) + } + + fn float_is_require_grad(tensor: &FloatTensor) -> bool { + matches!(tensor.node.requirement, Requirement::Grad) + } + + fn float_mean(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Mean; + + impl Backward for Mean { + type State = Shape; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + let shape = ops.state; + let val = 1_f64 / shape.num_elements() as f64; + let ones = B::float_ones(shape, &B::float_device(&grad), grad.dtype().into()); + let val = B::float_mul_scalar(ones, val.into()); + + let grad = unsqueeze_like::(grad, val.shape()); + B::float_mul(val, grad) + }); + } + } + + match Mean.prepare::([tensor.node]).compute_bound().stateful() { + OpsKind::Tracked(prep) => { + prep.finish(tensor.primitive.shape(), B::float_mean(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_mean(tensor.primitive)), + } + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sum; + + impl Backward for Sum { + type State = Shape; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + let val = + B::float_ones(ops.state, &B::float_device(&grad), grad.dtype().into()); + + let grad = unsqueeze_like::(grad, val.shape()); + B::float_mul(val, grad) + }); + } + } + + match Sum.prepare::([tensor.node]).compute_bound().stateful() { + OpsKind::Tracked(prep) => { + prep.finish(tensor.primitive.shape(), B::float_sum(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_sum(tensor.primitive)), + } + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(Debug)] + struct MeanDim; + + impl Backward for MeanDim { + type State = (Shape, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape, dim) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + let val = 1_f64 / shape[dim] as f64; + let ones = B::float_ones(shape, &B::float_device(&grad), grad.dtype().into()); + let val = B::float_mul_scalar(ones, val.into()); + + let grad = B::float_sum_dim(grad, dim); + B::float_mul(val, grad) + }); + } + } + + match MeanDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.shape(), dim), + B::float_mean_dim(tensor.primitive, dim), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_mean_dim(tensor.primitive, dim)), + } + } + + fn float_sum_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(Debug)] + struct SumDim; + + impl Backward for SumDim { + type State = (Shape, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape, dim) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + let ones = B::float_ones(shape, &B::float_device(&grad), grad.dtype().into()); + let grad = B::float_sum_dim(grad, dim); + + B::float_mul(ones, grad) + }); + } + } + + match SumDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.shape(), dim), + B::float_sum_dim(tensor.primitive, dim), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_sum_dim(tensor.primitive, dim)), + } + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(Debug)] + struct CumSum; + + impl Backward for CumSum { + type State = (Shape, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (_shape, dim) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + // Gradient of cumsum is cumsum of gradient in reverse + let grad_reversed = B::float_flip(grad.clone(), &[dim]); + let grad_cumsum = B::float_cumsum(grad_reversed, dim); + B::float_flip(grad_cumsum, &[dim]) + }); + } + } + + match CumSum + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.shape(), dim), + B::float_cumsum(tensor.primitive, dim), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_cumsum(tensor.primitive, dim)), + } + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(Debug)] + struct CumProd; + + impl Backward for CumProd { + type State = (B::FloatTensorPrimitive, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (input, dim) = ops.state; + let output = B::float_cumprod(input.clone(), dim); + + unary::(ops.parents, ops.node, grads, |grad| { + // Gradient of cumprod using negative step slicing + // Formula: grad_input[i] = sum_{j>=i}(grad_output[j] * output[j] / input[i]) + // = (1 / input[i]) * sum_{j>=i}(grad_output[j] * output[j]) + // = (1 / input) * reverse_cumsum(grad * output) + // + // LIMITATION: This produces NaN when input contains zeros. + // A proper zero-safe implementation requires more sophisticated algorithms + // (see PyTorch's cumprod_backward or JAX's associative_scan approach). + // TODO: Implement zero-safe gradient computation. + // See: https://github.com/tracel-ai/burn/issues/3864 + + let grad_times_output = B::float_mul(grad, output.clone()); + + // Create slices to reverse along the specified dimension + let shape = grad_times_output.shape(); + let mut slices = vec![Slice::full(); shape.num_dims()]; + slices[dim] = Slice::with_step(0, None, -1); + + // Reverse, cumsum, reverse back using negative step slicing + let grad_reversed = B::float_slice(grad_times_output, &slices); + let grad_cumsum = B::float_cumsum(grad_reversed, dim); + let grad_result = B::float_slice(grad_cumsum, &slices); + + B::float_div(grad_result, input) + }); + } + } + + match CumProd + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.clone(), dim), + B::float_cumprod(tensor.primitive, dim), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_cumprod(tensor.primitive, dim)), + } + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(Debug)] + struct CumMin; + + impl Backward for CumMin { + type State = (B::FloatTensorPrimitive, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (input, dim) = ops.state; + let output = B::float_cummin(input.clone(), dim); + + unary::(ops.parents, ops.node, grads, |grad| { + // Gradient flows to the input positions that produced each output + // Use scatter to accumulate gradients (scatter does sum reduction) + + let shape = input.shape(); + let device = B::float_device(&input); + let dim_size = shape[dim] as i64; + + // Create indices [0, 1, 2, ...] along the dimension + let arange_1d = B::int_arange(0..dim_size, &device); + + // Reshape to broadcast along the specified dimension + let mut arange_shape = vec![1; shape.num_dims()]; + arange_shape[dim] = dim_size as usize; + let arange = B::int_reshape(arange_1d, Shape::from(arange_shape)); + + // Expand to match input shape + let arange = B::int_expand(arange, shape.clone()); + + // Find where cummin[i] == input[i] (these are source positions) + let is_source = B::float_equal(output.clone(), input.clone()); + let is_source_int = B::bool_into_int(is_source); + + // Mask: where is_source, use index; else 0 + let masked_indices = B::int_mul(arange, is_source_int); + + // Cummax propagates the last valid (non-zero) index forward + let source_indices = B::int_cummax(masked_indices, dim); + + // Scatter gradients to source positions (sum reduction) + let zeros = B::float_zeros(shape, &device, grad.dtype().into()); + B::float_scatter_add(dim, zeros, source_indices, grad) + }); + } + } + + match CumMin + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.clone(), dim), + B::float_cummin(tensor.primitive, dim), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_cummin(tensor.primitive, dim)), + } + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(Debug)] + struct CumMax; + + impl Backward for CumMax { + type State = (B::FloatTensorPrimitive, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (input, dim) = ops.state; + let output = B::float_cummax(input.clone(), dim); + + unary::(ops.parents, ops.node, grads, |grad| { + // Gradient flows to the input positions that produced each output + // Use scatter to accumulate gradients (scatter does sum reduction) + + let shape = input.shape(); + let device = B::float_device(&input); + let dim_size = shape[dim] as i64; + + // Create indices [0, 1, 2, ...] along the dimension + let arange_1d = B::int_arange(0..dim_size, &device); + + // Reshape to broadcast along the specified dimension + let mut arange_shape = vec![1; shape.num_dims()]; + arange_shape[dim] = dim_size as usize; + let arange = B::int_reshape(arange_1d, Shape::from(arange_shape)); + + // Expand to match input shape + let arange = B::int_expand(arange, shape.clone()); + + // Find where cummax[i] == input[i] (these are source positions) + let is_source = B::float_equal(output.clone(), input.clone()); + let is_source_int = B::bool_into_int(is_source); + + // Mask: where is_source, use index; else 0 + let masked_indices = B::int_mul(arange, is_source_int); + + // Cummax propagates the last valid (non-zero) index forward + let source_indices = B::int_cummax(masked_indices, dim); + + // Scatter gradients to source positions (sum reduction) + let zeros = B::float_zeros(shape, &device, grad.dtype().into()); + B::float_scatter_add(dim, zeros, source_indices, grad) + }); + } + } + + match CumMax + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.clone(), dim), + B::float_cummax(tensor.primitive, dim), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_cummax(tensor.primitive, dim)), + } + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor { + B::float_argmax(tensor.primitive, dim) + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor { + B::float_argmin(tensor.primitive, dim) + } + + fn float_exp(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Exp; + + retro_unary!(RetroExp, B::float_exp); + + impl Backward for Exp { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + let output = B::float_exp(input); + unary::(ops.parents, ops.node, grads, |grad| { + B::float_mul(grad, output) + }); + } + } + + match Exp + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroExp::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_exp(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_exp(tensor.primitive)), + } + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Log; + + retro_unary!(RetroLog, B::float_log); + + impl Backward for Log { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + let value = B::float_recip(input); + B::float_mul(grad, value) + }); + } + } + + match Log + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroLog::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_log(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_log(tensor.primitive)), + } + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Log1P; + + retro_unary!(RetroLog1P, B::float_log1p); + + impl Backward for Log1P { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + let value = B::float_add_scalar(input, 1f32.into()); + let value = B::float_recip(value); + + B::float_mul(grad, value) + }); + } + } + + match Log1P + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroLog1P::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_log1p(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_log1p(tensor.primitive)), + } + } + + fn float_powf_scalar_impl(tensor: FloatTensor, value: Scalar) -> FloatTensor { + #[derive(Debug)] + struct PowfScalar; + + #[derive(new, Debug)] + struct RetroPowfScalar { + lhs_id: NodeId, + rhs: f64, + _backend: PhantomData, + } + + impl RetroForward for RetroPowfScalar { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let lhs = states.get_state::(&self.lhs_id); + let out = B::float_powf_scalar(lhs, self.rhs.into()); + states.save(out_node, out) + } + } + + impl Backward for PowfScalar { + type State = (NodeId, f64); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (tensor_id, value) = ops.state; + let tensor = checkpointer.retrieve_node_output(tensor_id); + + unary::(ops.parents, ops.node, grads, |grad| { + let tmp = B::float_powf_scalar(tensor, (value - 1.).into()); + let value = B::float_mul_scalar(tmp, value.into()); + + B::float_mul(grad, value) + }); + } + } + + match PowfScalar + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroPowfScalar::::new(tensor.node.id, value.elem())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = (prep.checkpoint(&tensor), value.elem()); + prep.finish(state, B::float_powf_scalar(tensor.primitive, value)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_powf_scalar(tensor.primitive, value)), + } + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sqrt; + + retro_unary!(RetroSqrt, B::float_sqrt); + + impl Backward for Sqrt { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + let value = B::float_div_scalar( + B::float_powf_scalar(input, (-0.5).into()), + 2f32.into(), + ); + + B::float_mul(grad, value) + }); + } + } + + match Sqrt + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSqrt::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_sqrt(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_sqrt(tensor.primitive)), + } + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Abs; + + retro_unary!(RetroAbs, B::float_abs); + + impl Backward for Abs { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let tensor: B::FloatTensorPrimitive = checkpointer.retrieve_node_output(ops.state); + let state = B::float_sign(tensor); + unary::(ops.parents, ops.node, grads, |grad| { + B::float_mul(grad, state) + }); + } + } + + match Abs + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAbs::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_abs(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_abs(tensor.primitive)), + } + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Cos; + + retro_unary!(RetroCos, B::float_cos); + + impl Backward for Cos { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + let value = B::float_neg(B::float_sin(input)); + + B::float_mul(grad, value) + }); + } + } + + match Cos + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroCos::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_cos(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_cos(tensor.primitive)), + } + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sin; + + retro_unary!(RetroSin, B::float_sin); + + impl Backward for Sin { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let state = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + let value = B::float_cos(state); + B::float_mul(grad, value) + }); + } + } + + match Sin + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSin::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_sin(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_sin(tensor.primitive)), + } + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Tanh; + + retro_unary!(RetroTanh, B::float_tanh); + + impl Backward for Tanh { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + let state = B::float_tanh(input); + unary::(ops.parents, ops.node, grads, |grad| { + let value = B::float_add_scalar( + B::float_neg(B::float_powi_scalar(state, 2.into())), + 1f32.into(), + ); + B::float_mul(grad, value) + }); + } + } + + match Tanh + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroTanh::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_tanh(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_tanh(tensor.primitive)), + } + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Cosh; + + retro_unary!(RetroCosh, B::float_cosh); + + impl Backward for Cosh { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + B::float_mul(grad, B::float_sinh(input)) + }); + } + } + + match Cosh + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroCosh::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_cosh(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_cosh(tensor.primitive)), + } + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sinh; + + retro_unary!(RetroSinh, B::float_sinh); + + impl Backward for Sinh { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + B::float_mul(grad, B::float_cosh(input)) + }); + } + } + + match Sinh + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSinh::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_sinh(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_sinh(tensor.primitive)), + } + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Tan; + + retro_unary!(RetroTan, B::float_tan); + + impl Backward for Tan { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + let tan_x = B::float_tan(input); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx tan(x) = 1 + tan^2(x) + let tan_sq = B::float_powi_scalar(tan_x, 2.into()); + B::float_mul(grad, B::float_add_scalar(tan_sq, 1f32.into())) + }); + } + } + + match Tan + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroTan::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_tan(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_tan(tensor.primitive)), + } + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Asin; + + retro_unary!(RetroAsin, B::float_asin); + + impl Backward for Asin { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx asin(x) = 1/sqrt(1 - x^2) + let x_sq = B::float_powi_scalar(input, 2.into()); + let denom = B::float_sqrt(B::float_add_scalar(B::float_neg(x_sq), 1f32.into())); + B::float_mul(grad, B::float_recip(denom)) + }); + } + } + + match Asin + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAsin::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_asin(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_asin(tensor.primitive)), + } + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Acos; + + retro_unary!(RetroAcos, B::float_acos); + + impl Backward for Acos { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx acos(x) = -1/sqrt(1 - x^2) + let x_sq = B::float_powi_scalar(input, 2.into()); + let denom = B::float_sqrt(B::float_add_scalar(B::float_neg(x_sq), 1f32.into())); + let value = B::float_neg(B::float_recip(denom)); + B::float_mul(grad, value) + }); + } + } + + match Acos + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAcos::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_acos(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_acos(tensor.primitive)), + } + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Atan; + + retro_unary!(RetroAtan, B::float_atan); + + impl Backward for Atan { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx atan(x) = 1/(1 + x^2) + let x_sq = B::float_powi_scalar(input, 2.into()); + let value = B::float_recip(B::float_add_scalar(x_sq, 1f32.into())); + B::float_mul(grad, value) + }); + } + } + + match Atan + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAtan::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_atan(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_atan(tensor.primitive)), + } + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Asinh; + + retro_unary!(RetroAsinh, B::float_asinh); + + impl Backward for Asinh { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx asinh(x) = 1/sqrt(x^2 + 1) + let x_sq = B::float_powi_scalar(input, 2.into()); + let value = + B::float_recip(B::float_sqrt(B::float_add_scalar(x_sq, 1f32.into()))); + B::float_mul(grad, value) + }); + } + } + + match Asinh + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAsinh::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_asinh(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_asinh(tensor.primitive)), + } + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Acosh; + + retro_unary!(RetroAcosh, B::float_acosh); + + impl Backward for Acosh { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx acosh(x) = 1/sqrt(x^2 - 1) + let x_sq = B::float_powi_scalar(input, 2.into()); + let value = + B::float_recip(B::float_sqrt(B::float_sub_scalar(x_sq, 1f32.into()))); + B::float_mul(grad, value) + }); + } + } + + match Acosh + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAcosh::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_acosh(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_acosh(tensor.primitive)), + } + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Atanh; + + retro_unary!(RetroAtanh, B::float_atanh); + + impl Backward for Atanh { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let input = checkpointer.retrieve_node_output(ops.state); + unary::(ops.parents, ops.node, grads, |grad| { + // d/dx atanh(x) = 1/(1 - x^2) + let x_sq = B::float_powi_scalar(input, 2.into()); + let value = + B::float_recip(B::float_add_scalar(B::float_neg(x_sq), 1f32.into())); + B::float_mul(grad, value) + }); + } + } + + match Atanh + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroAtanh::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_atanh(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_atanh(tensor.primitive)), + } + } + + fn float_atan2(y: FloatTensor, x: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Atan2; + + retro_binary!(RetroAtan2, B::float_atan2); + + impl Backward for Atan2 { + type State = (Option, Option, BinaryOpsBroadcast); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (y_id, x_id, broadcast) = ops.state; + let y = y_id.map(|id| checkpointer.retrieve_node_output(id)); + let x = x_id.map(|id| checkpointer.retrieve_node_output(id)); + let [y_4y, y_4x] = duplicate(&ops.parents, y); + let [x_4y, x_4x]: [Option>; 2] = duplicate(&ops.parents, x); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + // d/dy atan2(y, x) = x/(x^2 + y^2) + let y = y_4y.unwrap(); + let x = x_4y.unwrap(); + let x_sq = B::float_powi_scalar(x.clone(), 2.into()); + let y_sq = B::float_powi_scalar(y, 2.into()); + let denom = B::float_add(x_sq, y_sq); + let value = B::float_div(x, denom); + let grad = B::float_mul(grad, value); + + broadcast.backward_lhs::(grad) + }, + |grad| { + // d/dx atan2(y, x) = -y/(x^2 + y^2) + let y = y_4x.unwrap(); + let x = x_4x.unwrap(); + let x_sq = B::float_powi_scalar(x, 2.into()); + let y_sq = B::float_powi_scalar(y.clone(), 2.into()); + let denom = B::float_add(x_sq, y_sq); + let value = B::float_neg(B::float_div(y, denom)); + let grad = B::float_mul(grad, value); + + broadcast.backward_rhs::(grad) + }, + ); + } + } + + let y_tracked = y.is_tracked(); + let x_tracked = x.is_tracked(); + let broadcast = BinaryOpsBroadcast::new::(&y.primitive, &x.primitive); + + match Atan2 + .prepare::([y.node.clone(), x.node.clone()]) + .memory_bound() + .retro_forward(RetroAtan2::::new(y.node.id, x.node.id)) + .parents([&y, &x]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let is_tracked = y_tracked || x_tracked; + let y_state = is_tracked.then(|| prep.checkpoint(&y)); + let x_state = is_tracked.then(|| prep.checkpoint(&x)); + + prep.finish( + (y_state, x_state, broadcast), + B::float_atan2(y.primitive, x.primitive), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_atan2(y.primitive, x.primitive)), + } + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Round; + retro_unary!(RetroRound, B::float_round); + + impl Backward for Round { + type State = (Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape, device) = ops.state; + unary::(ops.parents, ops.node, grads, |grad| { + B::float_zeros(shape, &device, grad.dtype().into()) + }) + } + } + + match Round + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroRound::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(preps) => preps.finish( + (tensor.primitive.shape(), B::float_device(&tensor.primitive)), + B::float_round(tensor.primitive), + ), + OpsKind::UnTracked(preps) => preps.finish(B::float_round(tensor.primitive)), + } + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Floor; + retro_unary!(RetroFloor, B::float_floor); + + impl Backward for Floor { + type State = (Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape, device) = ops.state; + unary::(ops.parents, ops.node, grads, |grad| { + B::float_zeros(shape, &device, grad.dtype().into()) + }) + } + } + + match Floor + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroFloor::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(preps) => preps.finish( + (tensor.primitive.shape(), B::float_device(&tensor.primitive)), + B::float_floor(tensor.primitive), + ), + OpsKind::UnTracked(preps) => preps.finish(B::float_floor(tensor.primitive)), + } + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Ceil; + retro_unary!(RetroCeil, B::float_ceil); + + impl Backward for Ceil { + type State = (Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape, device) = ops.state; + unary::(ops.parents, ops.node, grads, |grad| { + B::float_zeros(shape, &device, grad.dtype().into()) + }) + } + } + + match Ceil + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroCeil::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(preps) => preps.finish( + (tensor.primitive.shape(), B::float_device(&tensor.primitive)), + B::float_ceil(tensor.primitive), + ), + OpsKind::UnTracked(preps) => preps.finish(B::float_ceil(tensor.primitive)), + } + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Trunc; + retro_unary!(RetroTrunc, B::float_trunc); + + impl Backward for Trunc { + type State = (Shape, B::Device); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape, device) = ops.state; + unary::(ops.parents, ops.node, grads, |grad| { + B::float_zeros(shape, &device, grad.dtype().into()) + }) + } + } + + match Trunc + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroTrunc::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(preps) => preps.finish( + (tensor.primitive.shape(), B::float_device(&tensor.primitive)), + B::float_trunc(tensor.primitive), + ), + OpsKind::UnTracked(preps) => preps.finish(B::float_trunc(tensor.primitive)), + } + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Erf; + + retro_unary!(RetroErf, B::float_erf); + + impl Backward for Erf { + type State = NodeId; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| { + let ops = checkpointer.retrieve_node_output(ops.state); + let exponent = B::float_neg(B::float_powi_scalar(ops, 2.into())); + let numerator = B::float_mul_scalar(B::float_exp(exponent), 2.0.into()); + let denominator = core::f64::consts::PI.sqrt().into(); + let value = B::float_div_scalar(numerator, denominator); + + B::float_mul(grad, value) + }); + } + } + + match Erf + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroErf::::new(tensor.node.id)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let state = prep.checkpoint(&tensor); + prep.finish(state, B::float_erf(tensor.primitive)) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_erf(tensor.primitive)), + } + } + + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + #[derive(new, Debug)] + struct CatStep { + nodes: Vec>, + // The dimension of each tensor along the dim dimension. + // This indicates the number of dimension concatenated for each tensor. + dim_sizes: Vec, + output: NodeRef, + phantom: PhantomData, + dim: usize, + parents: Vec, + } + + impl Step for CatStep { + fn step(self: Box, grads: &mut Gradients, _checkpointer: &mut Checkpointer) { + let grad = grads.consume::(&self.output); + let ranges_template: Vec<_> = grad.shape().iter().map(|&v| 0..v).collect(); + + self.nodes + .into_iter() + .zip(self.dim_sizes) + .scan(0, |offset, (node_opt, dim_size)| { + let start = *offset; + let end = start + dim_size; + *offset = end; + Some(node_opt.map(|node| (node, start, end))) + }) + .flatten() + .for_each(|(node, start, end)| { + let mut ranges = ranges_template.clone(); + ranges[self.dim] = start..end; + + let slices: Vec = ranges + .iter() + .map(|r| Slice::new(r.start as isize, Some(r.end as isize), 1)) + .collect(); + grads.register::(node.id, B::float_slice(grad.clone(), &slices)); + }); + } + + fn node(&self) -> NodeId { + self.output.id + } + + fn parents(&self) -> &[Parent] { + &self.parents + } + fn depth(&self) -> usize { + self.output.order + } + } + + let mut nodes = Vec::with_capacity(tensors.len()); + let mut primitives = Vec::with_capacity(tensors.len()); + let mut dim_sizes = Vec::with_capacity(tensors.len()); + + tensors.into_iter().for_each(|tensor| { + dim_sizes.push(tensor.primitive.shape()[dim]); + nodes.push(tensor.node); + primitives.push(tensor.primitive); + }); + + let requirement = Requirement::from_nodes(&nodes); + + // For simplicity, this operation does not checkpoint anything + let cat_computing_property = ComputingProperty::Ambiguous; + let checkpointer_builder = CheckpointerBuilder::default(); + + let output = B::float_cat(primitives, dim); + if requirement.is_none() { + return AutodiffTensor::from_parents( + output, + &nodes, + requirement, + cat_computing_property, + ); + } + + let output = + AutodiffTensor::from_parents(output, &nodes, requirement, cat_computing_property); + + let mut parents = Vec::new(); + + let nodes = nodes + .into_iter() + .map(|node| node.clone_if_require_grad()) + .collect::>(); + for node in nodes.iter().flatten() { + parents.push(Parent { id: node.id }); + } + let ops = CatStep::::new(nodes, dim_sizes, output.node.clone(), dim, parents); + output.register_step(ops, checkpointer_builder) + } + + fn float_max_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + match MaxMinDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let shape = tensor.primitive.shape(); + let (tensor, index) = B::float_max_dim_with_indices(tensor.primitive, dim); + prep.finish((index, shape, dim), tensor) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_max_dim(tensor.primitive, dim)), + } + } + fn float_max_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + match MaxMinDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let shape = tensor.primitive.shape(); + let (tensor, index) = B::float_max_dim_with_indices(tensor.primitive, dim); + let tensor = prep.finish((index.clone(), shape, dim), tensor); + + (tensor, index) + } + OpsKind::UnTracked(prep) => { + let (tensor, index) = B::float_max_dim_with_indices(tensor.primitive, dim); + let tensor = prep.finish(tensor); + + (tensor, index) + } + } + } + + fn float_min_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + match MaxMinDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let shape = tensor.primitive.shape(); + let (tensor, index) = B::float_min_dim_with_indices(tensor.primitive, dim); + prep.finish((index, shape, dim), tensor) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_min_dim(tensor.primitive, dim)), + } + } + fn float_min_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + match MaxMinDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let shape = tensor.primitive.shape(); + let (tensor, index) = B::float_min_dim_with_indices(tensor.primitive, dim); + let tensor = prep.finish((index.clone(), shape, dim), tensor); + + (tensor, index) + } + OpsKind::UnTracked(prep) => { + let (tensor, index) = B::float_min_dim_with_indices(tensor.primitive, dim); + let tensor = prep.finish(tensor); + + (tensor, index) + } + } + } + + fn float_into_int(tensor: FloatTensor) -> as Backend>::IntTensorPrimitive { + B::float_into_int(tensor.primitive) + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct PowF; + + retro_binary!(RetroPowf, B::float_powf); + + impl Backward for PowF { + type State = (NodeId, NodeId, BinaryOpsBroadcast); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + checkpointer: &mut Checkpointer, + ) { + let (lhs_id, rhs_id, broadcast) = ops.state; + let lhs: B::FloatTensorPrimitive = checkpointer.retrieve_node_output(lhs_id); + let rhs: B::FloatTensorPrimitive = checkpointer.retrieve_node_output(rhs_id); + + // Both lhs and rhs are needed for both lhs and rhs gradients, but we clone them + // the number of times required by the parents specification. + let [rhs_4lhs, rhs_4rhs] = duplicate(&ops.parents, Some(rhs)); + let [lhs_4lhs, lhs_4rhs] = duplicate(&ops.parents, Some(lhs)); + + binary::( + ops.parents, + ops.node, + grads, + |grad| { + //rhs*(lhs.val**(rhs-1))*grad + let rhs1 = rhs_4lhs.unwrap(); + let rhs2 = rhs1.clone(); + let lhs = lhs_4lhs.unwrap(); + + let tmp = B::float_powf(lhs, B::float_sub_scalar(rhs1, 1.0.into())); + let value = B::float_mul(tmp, rhs2); + let grad = B::float_mul(grad, value); + + broadcast.backward_lhs::(grad) + }, + |grad| { + //lhs**rhs * ln(lhs) * grad + let rhs = rhs_4rhs.unwrap(); + let lhs1 = lhs_4rhs.unwrap(); + let lhs2 = lhs1.clone(); + let tmp = B::float_powf(lhs1, rhs); + let value = B::float_mul(tmp, B::float_log(lhs2)); + let grad = B::float_mul(grad, value); + + broadcast.backward_rhs::(grad) + }, + ); + } + } + + let broadcast = BinaryOpsBroadcast::new::(&lhs.primitive, &rhs.primitive); + + match PowF + .prepare::([lhs.node.clone(), rhs.node.clone()]) + .memory_bound() + .retro_forward(RetroPowf::::new(lhs.node.id, rhs.node.id)) + .parents([&lhs, &rhs]) + .stateful() + { + OpsKind::Tracked(mut prep) => { + let lhs_state = prep.checkpoint(&lhs); + let rhs_state = prep.checkpoint(&rhs); + prep.finish( + (lhs_state, rhs_state, broadcast), + B::float_powf(lhs.primitive, rhs.primitive), + ) + } + OpsKind::UnTracked(prep) => prep.finish(B::float_powf(lhs.primitive, rhs.primitive)), + } + } + + fn float_sign(tensor: FloatTensor) -> FloatTensor { + #[derive(Debug)] + struct Sign; + + retro_unary!(RetroSign, B::float_sign); + + impl Backward for Sign { + type State = (); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + unary::(ops.parents, ops.node, grads, |grad| + // Always return 0 because the derivative of the sign function + // does not contribute to gradient updates in a meaningful way. + B::float_mul_scalar(grad, 0f32.into())); + } + } + + Sign.prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroSign::::new(tensor.node.id)) + .parents([&tensor]) + .stateless(B::float_sign(tensor.primitive)) + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + // D1: tensor, D2: shape + #[derive(Debug)] + struct ExpandDim; + + #[derive(new, Debug)] + struct RetroExpand { + input_id: NodeId, + shape: Shape, + _backend: PhantomData, + } + + impl RetroForward for RetroExpand { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let input = states.get_state::(&self.input_id); + let out = B::float_expand(input, self.shape.clone()); + states.save(out_node, out) + } + } + + impl Backward for ExpandDim { + type State = (Shape, Shape); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape_in, shape_out) = ops.state; + let ndims_in = shape_in.num_dims(); + let ndims_out = shape_out.num_dims(); + + let mut shape_expanded = vec![1; ndims_out]; + + debug_assert!(ndims_out >= ndims_in); + + for i in 0..ndims_in { + shape_expanded[i + (ndims_out - ndims_in)] = shape_in[i]; + } + + unary::(ops.parents, ops.node, grads, |grad| { + let shape_grad = grad.shape(); + let mut grad = grad; + + #[allow(clippy::needless_range_loop)] + for i in 0..ndims_out { + if shape_expanded[i] == 1 && shape_grad[i] != 1 { + grad = B::float_sum_dim(grad, i); + } + } + + B::float_reshape(grad, shape_in) + }); + } + } + + match ExpandDim + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroExpand::::new(tensor.node.id, shape.clone())) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.shape(), shape.clone()), + B::float_expand(tensor.primitive, shape), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_expand(tensor.primitive, shape)), + } + } + + fn float_sort(tensor: FloatTensor, dim: usize, descending: bool) -> FloatTensor { + match super::sort::SortDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let shape = tensor.primitive.shape(); + let (tensor, indices) = + B::float_sort_with_indices(tensor.primitive, dim, descending); + prep.finish((indices, shape, dim), tensor) + } + OpsKind::UnTracked(prep) => { + prep.finish(B::float_sort(tensor.primitive, dim, descending)) + } + } + } + + fn float_sort_with_indices( + tensor: FloatTensor, + dim: usize, + descending: bool, + ) -> (FloatTensor, IntTensor) { + match super::sort::SortDim + .prepare::([tensor.node]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => { + let shape = tensor.primitive.shape(); + let (tensor, indices) = + B::float_sort_with_indices(tensor.primitive, dim, descending); + let tensor = prep.finish((indices.clone(), shape, dim), tensor); + + (tensor, indices) + } + OpsKind::UnTracked(prep) => { + let (tensor, indices) = + B::float_sort_with_indices(tensor.primitive, dim, descending); + let tensor = prep.finish(tensor); + + (tensor, indices) + } + } + } + + fn float_argsort(tensor: FloatTensor, dim: usize, descending: bool) -> IntTensor { + B::float_argsort(tensor.primitive, dim, descending) + } + + fn float_repeat_dim(tensor: FloatTensor, dim: usize, times: usize) -> FloatTensor { + #[derive(Debug)] + struct Repeat; + + #[derive(new, Debug)] + struct RetroRepeat { + tensor_id: NodeId, + dim: usize, + times: usize, + _backend: PhantomData, + } + + impl RetroForward for RetroRepeat { + fn forward(&self, states: &mut BackwardStates, out_node: NodeId) { + let tensor = states.get_state::(&self.tensor_id); + let out = B::float_repeat_dim(tensor, self.dim, self.times); + states.save(out_node, out) + } + } + + impl Backward for Repeat { + type State = (usize, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (dim, times) = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + let mut dims = grad.shape(); + let orig_dim_size = dims[dim] / times; + if orig_dim_size > 1 { + dims[dim] = orig_dim_size; + let orig_dims = dims.clone(); + dims.insert(dim + 1, times); // shape [..., orig_dim_size, times, ...] + let grad = B::float_reshape(grad, dims); + let grad = B::float_sum_dim(grad, dim + 1); // sum over repeat times + B::float_reshape(grad, orig_dims) + } else { + B::float_sum_dim(grad, dim) + } + }); + } + } + + match Repeat + .prepare::([tensor.node.clone()]) + .memory_bound() + .retro_forward(RetroRepeat::::new(tensor.node.id, dim, times)) + .parents([&tensor]) + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (dim, times), + B::float_repeat_dim(tensor.primitive, dim, times), + ), + OpsKind::UnTracked(prep) => { + prep.finish(B::float_repeat_dim(tensor.primitive, dim, times)) + } + } + } + + fn float_cast(tensor: FloatTensor, dtype: burn_std::FloatDType) -> FloatTensor { + #[derive(Debug)] + struct Cast; + + impl Backward for Cast { + type State = FloatDType; + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let dtype = ops.state; + + unary::(ops.parents, ops.node, grads, |grad| { + B::float_cast(grad, dtype) + }); + } + } + + match Cast + .prepare::([tensor.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + tensor.dtype().into(), + B::float_cast(tensor.primitive, dtype), + ), + OpsKind::UnTracked(prep) => prep.finish(B::float_cast(tensor.primitive, dtype)), + } + } + + // TODO: Implement float_prod and float_sum + // https://github.com/tracel-ai/burn/issues/1458 + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + #[derive(Debug)] + struct Unfold; + + impl Backward for Unfold { + type State = (Shape, usize, usize, usize); + + fn backward( + self, + ops: Ops, + grads: &mut Gradients, + _checkpointer: &mut Checkpointer, + ) { + let (shape_in, dim, size, step) = ops.state; + let windows = calculate_unfold_windows(shape_in[dim], size, step); + + unary::(ops.parents, ops.node, grads, |grad| { + let device = B::float_device(&grad); + let mut grad_input = + B::float_zeros(shape_in.clone(), &device, grad.dtype().into()); + + if windows == 0 { + return grad_input; + } + + let ndims_in = shape_in.num_dims(); + let ndims_out = grad.shape().num_dims(); + + let mut target_shape = shape_in.clone(); + target_shape[dim] = size; + + for window_idx in 0..windows { + let mut slices_out = vec![Slice::new(0, None, 1); ndims_out]; + let start = window_idx * step; + let end = start + size; + slices_out[dim] = + Slice::new(window_idx as isize, Some((window_idx + 1) as isize), 1); + + let window_grad = B::float_slice(grad.clone(), &slices_out); + + let last_axis = ndims_out - 1; + let mut permutation: Vec = (0..dim).collect(); + permutation.push(last_axis); + permutation.extend(dim + 1..last_axis); + permutation.push(dim); + + let window_grad = B::float_permute(window_grad, &permutation); + let window_grad = B::float_reshape(window_grad, target_shape.clone()); + + let mut slices_in = vec![Slice::new(0, None, 1); ndims_in]; + slices_in[dim] = Slice::new(start as isize, Some(end as isize), 1); + + let current = B::float_slice(grad_input.clone(), &slices_in); + let updated = B::float_add(current, window_grad); + grad_input = B::float_slice_assign(grad_input, &slices_in, updated); + } + + grad_input + }); + } + } + + match Unfold + .prepare::([tensor.node.clone()]) + .compute_bound() + .stateful() + { + OpsKind::Tracked(prep) => prep.finish( + (tensor.primitive.shape(), dim, size, step), + B::float_unfold(tensor.primitive, dim, size, step), + ), + OpsKind::UnTracked(prep) => { + prep.finish(B::float_unfold(tensor.primitive, dim, size, step)) + } + } + } +} + +#[derive(Debug, Clone)] +enum BinaryOpsBroadcast { + Broadcasted(Shape, Shape), + None, +} + +impl BinaryOpsBroadcast { + fn new(lhs: &B::FloatTensorPrimitive, rhs: &B::FloatTensorPrimitive) -> Self { + let shape_lhs = lhs.shape(); + let shape_rhs = rhs.shape(); + let ndims = shape_lhs.num_dims(); + + for i in 0..ndims { + if shape_rhs[i] != shape_lhs[i] { + return Self::Broadcasted(shape_lhs, shape_rhs); + } + } + + Self::None + } + + fn backward_lhs(&self, grad: B::FloatTensorPrimitive) -> B::FloatTensorPrimitive { + match self { + BinaryOpsBroadcast::Broadcasted(lhs, _rhs) => broadcast_shape::(grad, lhs), + BinaryOpsBroadcast::None => grad, + } + } + + fn backward_rhs(&self, grad: B::FloatTensorPrimitive) -> B::FloatTensorPrimitive { + match self { + BinaryOpsBroadcast::Broadcasted(_lhs, rhs) => broadcast_shape::(grad, rhs), + BinaryOpsBroadcast::None => grad, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/transaction.rs new file mode 100644 index 0000000..17a07a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/ops/transaction.rs @@ -0,0 +1,24 @@ +use burn_backend::{ + Backend, ExecutionError, + ops::{TransactionOps, TransactionPrimitive}, +}; + +use crate::{Autodiff, checkpoint::strategy::CheckpointStrategy}; + +impl TransactionOps for Autodiff { + async fn tr_execute( + transaction: TransactionPrimitive, + ) -> Result { + B::tr_execute(TransactionPrimitive::new( + transaction + .read_floats + .into_iter() + .map(|t| t.primitive) + .collect(), + transaction.read_qfloats, + transaction.read_ints, + transaction.read_bools, + )) + .await + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/client.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/client.rs new file mode 100644 index 0000000..dc8a661 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/client.rs @@ -0,0 +1,18 @@ +use crate::{ + checkpoint::builder::CheckpointerBuilder, + grads::Gradients, + graph::StepBoxed, + tensor::{AutodiffTensor, NodeRefCount}, +}; +use burn_backend::Backend; + +/// Client used to communicate with the autodiff server. +pub trait AutodiffClient: Send + Clone { + /// Register a new step. + fn register(&self, node_id: NodeRefCount, step: StepBoxed, actions: CheckpointerBuilder); + /// Call backpropagation from the given tensor. + fn backward(&self, tensor: AutodiffTensor) -> Gradients; +} + +/// Client implementation in used. +pub type AutodiffClientImpl = super::graph::GraphMutexClient; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/graph.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/graph.rs new file mode 100644 index 0000000..1f95e49 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/graph.rs @@ -0,0 +1,335 @@ +use super::{AutodiffClient, server::AutodiffServer}; +use crate::{ + NodeId, + checkpoint::builder::CheckpointerBuilder, + grads::Gradients, + graph::{Parent, StepBoxed}, + runtime::server::NodeCleaner, + tensor::{AutodiffTensor, NodeRefCount}, +}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use burn_backend::Backend; +use hashbrown::{HashMap, HashSet}; + +#[cfg(feature = "std")] +use parking_lot::{Mutex, MutexGuard}; + +#[cfg(not(feature = "std"))] +use spin::{Mutex, MutexGuard}; + +/// A client for managing multiple graphs using mutex-based synchronization. +/// +/// The biggest benefit of using this client implementation is that each graph can modify its own +/// data without blocking other graphs, which is essential for multi-device training. +/// +/// # Notes +/// +/// The [AutodiffServer] fully supports multiple graphs with sharing nodes, however those type of +/// graphs will be stored under a single mutex-protected graph by the client, limiting +/// parallelisation. +#[derive(Clone, new, Debug)] +pub struct GraphMutexClient; + +/// Manages a collection of graphs, mapping [node ids](NodeId) to their respective graph. +/// +/// The `GraphLocator` is responsible for selecting and merging graphs based on their IDs and parent +/// dependencies, ensuring proper synchronization and server allocation. +/// +/// # Notes +/// +/// Multiple node ids can point to the same graph, where the autodiff graph is stored. +#[derive(Default)] +pub struct GraphLocator { + graphs: HashMap>, + /// We keep a mapping of each original node id (graph id) => all nodes that point to that graph. + /// This is to ensure that when merging graphs, we correctly move all previous graphs to + /// the new merged one. + keys: HashMap>, +} + +/// Represents a single computation graph with a mutex-protected server. +/// +/// Each `Graph` contains an [AutodiffServer] and the original [NodeId] where the server was +/// first created. +pub(crate) struct Graph { + origin: NodeId, + state: Mutex, +} + +#[derive(Default)] +struct GraphState { + server: AutodiffServer, +} + +impl core::fmt::Debug for Graph { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Graph") + .field("origin", &self.origin) + .finish() + } +} + +static STATE: Mutex> = Mutex::new(None); + +impl GraphMutexClient { + /// Retrieves or creates a graph for the given [NodeId] and parent dependencies. + /// + /// # Parameters + /// - `node`: The unique identifier for the stream. + /// - `parents`: A slice of parent nodes that the stream depends on. + /// + /// # Returns + /// An `Arc` representing the selected or newly created stream. + fn graph(node: NodeId, parents: &[Parent]) -> Arc { + let mut state = STATE.lock(); + + match state.as_mut() { + Some(locator) => locator.select(node, parents), + None => { + let mut locator = GraphLocator::default(); + let stream = locator.select(node, parents); + *state = Some(locator); + stream + } + } + } +} + +impl AutodiffClient for GraphMutexClient { + fn register(&self, node_id_ref: NodeRefCount, step: StepBoxed, actions: CheckpointerBuilder) { + let node_id = *node_id_ref; + let graph = GraphMutexClient::graph(node_id, step.parents()); + let mut state = graph.state.lock(); + + state.server.register(node_id_ref, step, actions); + } + + fn backward(&self, root: AutodiffTensor) -> Gradients { + let node_id = root.node.id; + let graph = GraphMutexClient::graph(root.node.id, &[]); + + let grads = Gradients::new::(root.node, root.primitive); + let grads = { + let mut state = graph.state.lock(); + state.server.backward::(grads, node_id) + }; // lock released + + GraphCleaner::cleanup_orphaned_entries(); + + grads + } +} + +struct GraphCleaner<'a> { + guard: MutexGuard<'a, Option>, +} + +impl<'a> GraphCleaner<'a> { + fn cleanup_orphaned_entries() { + let graphs = { + // Get the available graphs and release the lock + match STATE.lock().as_ref() { + Some(state) => state.graphs.clone(), + None => return, + } + }; + + let mut should_remove = Vec::new(); + for graph in graphs.values() { + { + let mut guard = graph.state.lock(); + // Double safety: in case it was marked as no longer useful, but other + // nodes are still relevant, we only check which nodes can safely be removed. + if !guard.server.maybe_useful() { + guard + .server + .free_unused_roots(|node| should_remove.push(*node)); + } + } + } + + if !should_remove.is_empty() { + let mut state = STATE.lock(); + if let Some(state) = state.as_mut() { + for node in should_remove { + state.remove_entry(&node); + } + } + } + } +} + +impl<'a> NodeCleaner for GraphCleaner<'a> { + fn init() -> Self { + let guard = STATE.lock(); + Self { guard } + } + + fn clean(&mut self, node: &NodeId) { + if let Some(state) = self.guard.as_mut() { + state.remove_entry(node); + } + } +} + +impl GraphLocator { + /// Selects a single graph for the given [NodeId], considering parent dependencies. + /// + /// If multiple graphs are found, they are merged into a single one. + /// + /// # Parameters + /// - `node`: The node ID of the graph to select. + /// - `parents`: A slice of parent nodes that the graph depends on. + /// + /// # Returns + /// + /// An `Arc` representing the selected or merged graph. + pub(crate) fn select(&mut self, node: NodeId, parents: &[Parent]) -> Arc { + match self.analyse(node, parents) { + GraphAnalysis::NoCollision(graph) => { + if graph.origin != node { + self.graphs.insert(node, graph.clone()); + self.register_key(graph.origin, node); + } + + graph + } + GraphAnalysis::Collisions(graphs) => self.merge(node, graphs), + } + } + + /// Analyses the graph for a given node and its parents, returning the associated `GraphAnalysis`. + fn analyse(&mut self, node: NodeId, parents: &[Parent]) -> GraphAnalysis { + // If no parents, there is no collision, therefore a single graph is ok. + if parents.is_empty() { + let graph = match self.graphs.get(&node) { + Some(val) => val.clone(), + None => self.new_graph(node), + }; + return GraphAnalysis::NoCollision(graph); + }; + + // We collect all graphs of parents and of the current node based on their origin node id. + let mut graphs = HashMap::>::new(); + + if let Some(val) = self.graphs.get(&node) { + graphs.insert(val.origin, val.clone()); + } + + for parent in parents { + match self.graphs.get(&parent.id) { + Some(graph) => graphs.insert(graph.origin, graph.clone()), + None => continue, + }; + } + + if graphs.is_empty() { + return match self.graphs.get(&node) { + Some(old) => GraphAnalysis::NoCollision(old.clone()), + None => GraphAnalysis::NoCollision(self.new_graph(node)), + }; + } + + if graphs.len() == 1 { + return GraphAnalysis::NoCollision(graphs.drain().next().unwrap().1); + } + + GraphAnalysis::Collisions(graphs) + } + + /// Merges multiple graphs associated with a node into a single graph. + fn merge(&mut self, node: NodeId, mut graphs: HashMap>) -> Arc { + let mut graphs = graphs.drain().map(|g| g.1); + + let main = graphs.next().expect("At least one graph"); + self.register_key(main.origin, node); + + let mut state = main.state.lock(); + + for graph in graphs { + self.merge_two(&mut state, &main, graph); + } + + self.graphs.insert(main.origin, main.clone()); + self.graphs.insert(node, main.clone()); + + core::mem::drop(state); + + main + } + + /// Registers a key for a given origin node. + fn register_key(&mut self, origin: NodeId, key: NodeId) { + if !self.keys.contains_key(&origin) { + // Ensure an entry exists for this origin + self.keys.insert(origin, HashSet::new()); + } + + if origin != key { + // Register this node to point to the origin graph + self.keys.get_mut(&origin).unwrap().insert(key); + } + } + + /// Merges two graphs by combining their states and updating graph mappings. + fn merge_two(&mut self, main_state: &mut GraphState, main: &Arc, merged: Arc) { + let mut locked = merged.state.lock(); + let mut state_old = GraphState::default(); + core::mem::swap(&mut state_old, &mut locked); + main_state.server.extend(state_old.server); + + // Re-map merged origin to the main graph + self.graphs.insert(merged.origin, main.clone()); + + // Move all keys (node IDs) from the merged graph to the main graph + if let Some(locator_keys) = self.keys.remove(&merged.origin) { + for k in locator_keys.iter() { + self.graphs.insert(*k, main.clone()); + } + + let locator_keys_main = self + .keys + .get_mut(&main.origin) + .expect("Should be init before the merge."); + locator_keys_main.extend(locator_keys); + } + } + + /// Creates a new graph for a given node. + fn new_graph(&mut self, origin: NodeId) -> Arc { + let graph = Arc::new(Graph { + origin, + state: Mutex::new(GraphState::default()), + }); + self.graphs.insert(origin, graph.clone()); + self.keys.insert(origin, HashSet::new()); + graph + } + + fn remove_entry(&mut self, node: &NodeId) { + if let Some(graph) = self.graphs.remove(node) { + let mut remove = false; + + if let Some(entry) = self.keys.get_mut(&graph.origin) { + entry.remove(node); + if entry.is_empty() { + remove = true; + } + } + + if remove { + self.keys.remove(&graph.origin); + } + } + } +} + +/// Represents the analysis result of graph operations for a given node and its parents. +#[derive(Debug)] +enum GraphAnalysis { + /// No collision detected, contains the graph associated with the node. + NoCollision(Arc), + /// Collision detected, contains a map of node IDs to their associated graphs. + Collisions(HashMap>), +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/memory_management.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/memory_management.rs new file mode 100644 index 0000000..a333ffd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/memory_management.rs @@ -0,0 +1,294 @@ +use crate::{ + NodeId, + collections::{HashMap, HashSet}, + graph::Parent, + tensor::NodeRefCount, +}; +use alloc::{borrow::ToOwned, sync::Arc, vec, vec::Vec}; +use core::mem; + +#[derive(Default, Debug)] +pub struct GraphMemoryManagement { + nodes: HashMap>, + leaves: HashSet, + statuses: HashMap, +} + +#[derive(Debug, Clone, PartialEq)] +enum NodeMemoryStatus { + Useful, + Unavailable, + Unknown, +} + +impl GraphMemoryManagement { + pub fn extend(&mut self, other: Self) { + self.nodes.extend(other.nodes); + self.leaves.extend(other.leaves); + self.statuses.extend(other.statuses); + } + + /// Register a new node with its parent. + pub fn register(&mut self, node: NodeRefCount, parents: &[Parent]) { + let node_id = *node.as_ref(); + + for parent in parents.iter() { + self.leaves.remove(&parent.id); + } + + self.leaves.insert(node_id); + self.nodes + .insert(node, parents.iter().map(|p| p.id).collect()); + } + + /// Free the node from the state. + pub fn consume_node(&mut self, node_id: NodeId) { + if !self.is_referenced(node_id) { + self.leaves.remove(&node_id); + self.nodes.remove(&node_id); + } + } + + /// Free all nodes whose backward call has become impossible + /// + /// This function goes into three steps, which must happen for all leaves + /// before going into the next step. Then it deletes what can be safely deleted + pub(crate) fn free_unavailable_nodes(&mut self, mut on_free_graph: impl FnMut(&NodeId)) { + let leaves = self.leaves.clone(); + let mut new_leaves = HashSet::new(); + let mut deletables = Vec::new(); + + // When consuming nodes with a backward pass, some other backward passes become + // unavailable because some of their parents have been consumed. They are + // identified here. + for leaf in leaves.clone() { + self.unavailable_propagation(leaf); + } + + // Among the available nodes that remain, some may be useless if no + // available node with a tensor reference exist in their descendance. + // But some may seem useless from some leaf but be useful from another one, + // hence the need to iterate on all leaves. + self.useful_propagation(leaves.clone()); + + // New leaves are the roots of a useful backward sub-tree. + // Deletables are everything not marked as useful. + for leaf in leaves { + self.identify_leaves_and_deletables(leaf, &mut new_leaves, &mut deletables); + } + + // Replace leaves by the new ones and delete everything not useful anymore + mem::swap(&mut self.leaves, &mut new_leaves); + + self.clear_unused_roots(&mut deletables); + + self.statuses.clear(); + for node_to_delete in deletables { + self.nodes.remove(&node_to_delete); + on_free_graph(&node_to_delete) + } + } + + pub(crate) fn free_unused_roots(&mut self, mut on_free_graph: impl FnMut(&NodeId)) { + let mut deletables = Vec::new(); + self.clear_unused_roots(&mut deletables); + + for node_id in deletables { + self.nodes.remove(&node_id); + on_free_graph(&node_id); + } + } + + fn clear_unused_roots(&self, to_delete: &mut Vec) { + for (id, parents) in self.nodes.iter() { + let is_useful = matches!( + self.statuses.get(id.as_ref()), + Some(NodeMemoryStatus::Useful) + ); + + // Check if parents are either empty or absent from self.nodes + let parents_absent = parents.iter().all(|p| !self.nodes.contains_key(p)); + + if !is_useful && Arc::strong_count(id) == 1 && parents_absent { + to_delete.push(*id.as_ref()) + } + } + } + + fn unavailable_propagation(&mut self, node_id: NodeId) -> NodeMemoryStatus { + // If already visited + if let Some(status) = self.statuses.get(&node_id) { + return status.clone(); + } + + match self.nodes.get(&node_id).cloned() { + // If node exists and any of its parents is unavailable, it is unavailable as well + // If node exists but the parents vec is empty, it is a tensor that never had parents; + // the status remains unknown + Some(parents) => { + let mut node_status = NodeMemoryStatus::Unknown; + for parent in parents { + let parent_status = self.unavailable_propagation(parent); + if let NodeMemoryStatus::Unavailable = parent_status { + node_status = NodeMemoryStatus::Unavailable; + } + } + self.statuses.insert(node_id, node_status.clone()); + node_status + } + // If node does not exist, it was + // deleted, so this and all its descendants are unavailable + None => { + self.statuses.insert(node_id, NodeMemoryStatus::Unavailable); + NodeMemoryStatus::Unavailable + } + } + } + + fn useful_propagation(&mut self, leaves: HashSet) { + // Accumulate visited nodes + let mut explored = HashSet::new(); + let mut tagged_useful = HashSet::new(); + + // Queue of nodes to visit + let mut to_tag_useful = PopNodeSet::default(); + let mut to_explore = PopNodeSet::new(leaves); + + // Utility function to iterate over a node's parents + let parents = |node_id| { + self.nodes + .get(&node_id) + .cloned() + .unwrap_or_default() + .into_iter() + }; + + loop { + // Pop a node id, greedily looking at tag_useful ones first + let (node_id, status) = match to_tag_useful.pop() { + Some(node_id) => (node_id, NodeMemoryStatus::Useful), + None => match to_explore.pop() { + Some(node_id) => { + let node_status = self + .statuses + .get(&node_id) + .expect("All nodes should have received a status during unavailable_propagation") + .to_owned(); + + if let NodeMemoryStatus::Unknown = node_status { + match self.is_referenced(node_id) { + true => (node_id, NodeMemoryStatus::Useful), + false => (node_id, NodeMemoryStatus::Unknown), + } + } else { + (node_id, node_status) + } + } + None => { + // There are no nodes in the queues anymore + break; + } + }, + }; + + match status { + NodeMemoryStatus::Useful => { + tagged_useful.insert(node_id); + for parent in parents(node_id) { + // The node can be explored, as long as it's not already tagged useful + if !(tagged_useful.contains(&parent) || to_tag_useful.contains(&parent)) { + to_tag_useful.insert(parent); + } + } + } + _ => { + explored.insert(node_id); + for parent in parents(node_id) { + if !(explored.contains(&parent) || to_explore.contains(&parent)) { + to_explore.insert(parent); + } + } + } + } + + self.statuses.insert(node_id, status); + } + } + + fn identify_leaves_and_deletables( + &self, + leaf_id: NodeId, + new_leaves: &mut HashSet, + to_delete: &mut Vec, + ) { + let mut visited = HashSet::new(); + let mut to_visit = vec![leaf_id]; + + while let Some(node_id) = to_visit.pop() { + visited.insert(node_id); + + match self + .statuses + .get(&node_id) + .expect("Node should have status") + { + NodeMemoryStatus::Useful => { + new_leaves.insert(node_id); + } + _ => { + to_delete.push(node_id); + + for parent in self + .nodes + .get(&node_id) + .cloned() + .unwrap_or_default() + .into_iter() + { + if !visited.contains(&parent) { + to_visit.push(parent); + } + } + } + }; + } + } + + fn is_referenced(&self, node_id: NodeId) -> bool { + match self.nodes.get_key_value(&node_id) { + Some((key, _value)) => Arc::strong_count(key) > 1, + None => panic!("Node should be in the nodes map"), + } + } + + pub(crate) fn maybe_useful(&self) -> bool { + self.nodes.keys().any(|node| Arc::strong_count(node) > 1) + } +} + +/// Wrapper over hash set for fast popping of any node +#[derive(new, Default)] +struct PopNodeSet { + hash_set: HashSet, +} + +impl PopNodeSet { + #[inline(always)] + fn pop(&mut self) -> Option { + self.hash_set + .iter() + .next() + .copied() + .and_then(|node_id| self.hash_set.take(&node_id)) + } + + #[inline(always)] + fn contains(&self, node_id: &NodeId) -> bool { + self.hash_set.contains(node_id) + } + + #[inline(always)] + fn insert(&mut self, node_id: NodeId) { + self.hash_set.insert(node_id); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/mod.rs new file mode 100644 index 0000000..fde3a5f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/mod.rs @@ -0,0 +1,6 @@ +mod client; +mod memory_management; +mod server; + +pub mod graph; +pub use client::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/server.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/server.rs new file mode 100644 index 0000000..d3e7c41 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/runtime/server.rs @@ -0,0 +1,143 @@ +use super::memory_management::GraphMemoryManagement; +use crate::{ + NodeId, + checkpoint::{ + base::{Checkpointer, NodeTree}, + builder::CheckpointerBuilder, + }, + collections::HashMap, + grads::Gradients, + graph::{StepBoxed, traversal::BreadthFirstSearch}, + tensor::NodeRefCount, +}; +use alloc::vec::Vec; + +#[derive(Default)] +pub struct AutodiffServer { + steps: HashMap, + actions_builder: HashMap, + memory_management: GraphMemoryManagement, +} + +/// Defines how nodes are clean. +pub trait NodeCleaner { + /// Initialize a new cleaner. + fn init() -> Self; + /// Cleans a single [node](NodeId). + fn clean(&mut self, node: &NodeId); +} + +impl AutodiffServer { + pub fn extend(&mut self, other: AutodiffServer) { + self.steps.extend(other.steps); + self.actions_builder.extend(other.actions_builder); + self.memory_management.extend(other.memory_management); + } + + pub fn register(&mut self, rc: NodeRefCount, step: StepBoxed, actions: CheckpointerBuilder) { + let parents = step.parents(); + let node_id = *rc.as_ref(); + + self.memory_management.register(rc, parents); + + self.steps.insert(node_id, step); + self.actions_builder.insert(node_id, actions); + } + + pub fn backward(&mut self, grads: Gradients, node_id: NodeId) -> Gradients { + let step = self.steps.remove(&node_id).expect( + "Node should have a step registered, did you forget to call \ + `Tensor::register_grad` on the tensor where you need gradients?", + ); + let builder = self.actions_builder.remove(&node_id).unwrap(); + + let mut consumed = Vec::new(); + let (tape, checkpointer) = self.build_tape(node_id, step, builder, &mut consumed); + + let gradients = Self::execute_steps(tape, grads, checkpointer); + + // Cleanup + let mut cleaner = NC::init(); + self.memory_management + .free_unavailable_nodes(|node_id: &NodeId| { + self.steps.remove(node_id); + self.actions_builder.remove(node_id); + NC::clean(&mut cleaner, node_id); + }); + for node_id in consumed { + cleaner.clean(&node_id) + } + + gradients + } + + pub(crate) fn free_unused_roots(&mut self, mut on_free_graph: impl FnMut(&NodeId)) { + self.memory_management.free_unused_roots(|node_id| { + self.steps.remove(node_id); + self.actions_builder.remove(node_id); + on_free_graph(node_id); + }); + } + + fn build_tape( + &mut self, + node: NodeId, + node_step: StepBoxed, + mut builder: CheckpointerBuilder, + consumed: &mut Vec, + ) -> (Vec>, Checkpointer) { + let mut tape = (0..node_step.depth()) + .map(|_| Vec::with_capacity(1)) + .collect::>(); + + let mut tree = HashMap::default(); + + BreadthFirstSearch.traverse(node, node_step, &mut self.steps, |id, step| { + self.memory_management.consume_node(id); + // Clean up consumed node + consumed.push(id); + + let depth = step.depth(); + + if depth == 0 { + return; + } + + if let Some(steps) = tape.get_mut(depth - 1) { + let parents = step.parents().iter().map(|p| p.id).filter(|s| *s != id); + tree.insert(id, parents.collect()); + steps.push(step); + } + + if let Some(node_builder) = self.actions_builder.remove(&id) { + builder.extend(node_builder); + } + }); + + let checkpointer = builder.build(NodeTree::new(tree)); + + (tape, checkpointer) + } + + fn execute_steps( + tape: Vec>, + mut grads: Gradients, + mut checkpointer: Checkpointer, + ) -> Gradients { + tape.into_iter().rev().for_each(|steps| { + steps + .into_iter() + .for_each(|step| step.step(&mut grads, &mut checkpointer)) + }); + + // For checkpointing tests + #[cfg(feature = "export_tests")] + assert!(checkpointer.is_empty()); + + grads + } + + pub(crate) fn maybe_useful(&self) -> bool { + self.memory_management.maybe_useful() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/tensor.rs new file mode 100644 index 0000000..7218d50 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/tensor.rs @@ -0,0 +1,189 @@ +use crate::{ + checkpoint::{base::Checkpointer, builder::CheckpointerBuilder}, + grads::Gradients, + graph::{ComputingProperty, Node, NodeId, NodeRef, Parent, Requirement, Step}, + runtime::{AutodiffClient, AutodiffClientImpl}, +}; +use alloc::{boxed::Box, sync::Arc, vec}; +use burn_backend::{Backend, TensorMetadata}; + +#[derive(Debug, Clone)] +pub struct AutodiffTensor { + pub primitive: B::FloatTensorPrimitive, + pub node: NodeRef, + pub rc: NodeRefCount, +} + +impl TensorMetadata for AutodiffTensor { + fn dtype(&self) -> burn_std::DType { + self.primitive.dtype() + } + + fn shape(&self) -> burn_std::Shape { + self.primitive.shape() + } + + fn rank(&self) -> usize { + self.primitive.rank() + } +} + +pub type NodeRefCount = Arc; + +#[derive(new, Debug)] +pub(crate) struct RootStep { + node: NodeRef, +} + +impl Step for RootStep { + fn step(self: Box, _grads: &mut Gradients, _checkpointer: &mut Checkpointer) { + // Nothing to do + } + + fn node(&self) -> NodeId { + self.node.id + } + + fn parents(&self) -> &[Parent] { + &self.node.parents + } + + fn depth(&self) -> usize { + self.node.order + } +} + +impl AutodiffTensor { + /// Create a new leaf tensor. + pub fn new(primitive: B::FloatTensorPrimitive) -> Self { + let id = NodeId::new(); + let node: NodeRef = Node::new( + vec![], + 0, + id, + Requirement::None, + ComputingProperty::Ambiguous, + AutodiffClientImpl::new(), + ) + .into(); + + Self { + rc: Arc::new(node.id), + primitive, + node: node.clone(), + } + } + + pub fn is_tracked(&self) -> bool { + !self.node.requirement.is_none() + } + + /// Mark the tensor as requiring gradients. + /// + /// # Panics + /// + /// It panics if the tensor is not a leaf. + pub fn require_grad(mut self) -> Self { + match self.node.requirement { + Requirement::Grad => self, + Requirement::GradInBackward => { + panic!("Can't convert a non leaf tensor into a tracked tensor") + } + Requirement::None => { + self.node = Node::new( + vec![], + 0, + self.node.id, + Requirement::Grad, + self.node.properties.clone(), + self.node.client.clone(), + ) + .into(); + let step = RootStep::new(self.node.clone()); + + self.register_step(step, CheckpointerBuilder::default()) + } + } + } + + /// Create a tensor from parent infos. + pub fn from_parents( + primitive: B::FloatTensorPrimitive, + parent_nodes: &[NodeRef], + requirement: Requirement, + computing_properties: ComputingProperty, + ) -> Self { + let order = parent_nodes + .iter() + .map(|node| node.order) + .reduce(usize::max) + .unwrap_or(0) + + 1; + + let client = parent_nodes + .first() + .map(|node| node.client.clone()) + .unwrap_or_else(AutodiffClientImpl::new); + + let node: NodeRef = Node::new( + parent_nodes + .iter() + .filter_map(|node| node.clone_if_require_grad()) + .map(|node| Parent::new(node.id)) + .collect(), + order, + NodeId::new(), + requirement, + computing_properties, + client, + ) + .into(); + + Self { + rc: Arc::new(node.id), + primitive, + node, + } + } + + /// Register a step into a graph for that tensor. + /// + /// # Warning + /// + /// This should be called only once per tensor. + pub fn register_step( + self, + step_that_created_the_tensor: S, + actions: CheckpointerBuilder, + ) -> Self { + self.node.client.register( + self.rc.clone(), + Box::new(step_that_created_the_tensor), + actions, + ); + self + } + + pub fn into_primitive(self) -> B::FloatTensorPrimitive { + self.primitive + } + + pub fn backward(self) -> Gradients { + let client = self.node.client.clone(); + + AutodiffClient::backward::(&client, self) + } + + pub fn grad(&self, grads: &Gradients) -> Option { + grads.get::(self) + } + + pub fn grad_remove(&self, grads: &mut Gradients) -> Option { + grads.remove::(self) + } + + pub fn grad_replace(&self, grads: &mut Gradients, grad: B::FloatTensorPrimitive) { + grads.remove::(self); + grads.register::(self.node.id, grad); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/utils.rs b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/utils.rs new file mode 100644 index 0000000..1f3549c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-autodiff/src/utils.rs @@ -0,0 +1,25 @@ +use alloc::vec::Vec; + +use crate::graph::NodeRef; +/// Duplicate the given object for each node that requires gradients. +/// +/// # Notes +/// +/// This is useful since you don't have to keep N cloned references alive event if just 1 node +/// will be updated. +/// +/// If the object is a tensor and if one reference exists, it can be updated inplace. +pub fn duplicate( + nodes: &[Option; N], + obj: Option, +) -> [Option; N] { + nodes + .iter() + .map(|node| match node { + Some(_) => obj.clone(), + None => None, + }) + .collect::>() + .try_into() + .unwrap() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/.cargo/config.toml b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/.cargo/config.toml new file mode 100644 index 0000000..63cdd77 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/.cargo/config.toml @@ -0,0 +1,10 @@ +[alias] +test-cpu = "test --release --no-default-features --features cpu,std" +test-cuda = "test --release --no-default-features --features cuda,std" +test-ndarray = "test --release --no-default-features --features ndarray,std" +test-rocm = "test --release --no-default-features --features rocm,std" +test-router = "test --release --no-default-features --features router,std" +test-tch = "test --release --no-default-features --features tch,std" +test-wgpu = "test --release --no-default-features --features wgpu,std" +test-vulkan = "test --release --no-default-features --features vulkan,std" +test-metal = "test --release --no-default-features --features metal,std" diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/Cargo.toml new file mode 100644 index 0000000..37a9c13 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/Cargo.toml @@ -0,0 +1,120 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Tensor tests for Burn backends" +documentation = "https://docs.rs/burn-backend-tests" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-backend-tests" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-backend-tests" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [ + "burn-tensor/default", + "burn-autodiff/default", + # Backends (default not enabled for CubeCL backends as it activates fusion) + "burn-cpu?/default", + "burn-ndarray?/default", + "burn-tch?/default", + # Default + "ndarray", + "std", +] +std = [ + "burn-tensor/std", + "burn-autodiff/std", + # Backends + "burn-cpu?/std", + "burn-ndarray?/std", + "burn-wgpu?/std", + "burn-router?/std", + "burn-cuda?/std", + "burn-rocm?/std", +] + +tracing = [ + "cubecl?/tracing", + "burn-tensor/tracing", + "burn-autodiff/tracing", + # Backends + "burn-cpu?/tracing", + "burn-ndarray?/tracing", + "burn-wgpu?/tracing", + "burn-router?/tracing", + "burn-cuda?/tracing", + "burn-rocm?/tracing", +] + +# Backends +cuda = ["burn-cuda", "quantization", "cube"] +rocm = ["burn-rocm", "quantization", "cube"] +ndarray = ["burn-ndarray", "quantization"] +tch = ["burn-tch"] +vulkan = ["wgpu", "burn-wgpu/vulkan"] +webgpu = ["wgpu", "burn-wgpu/webgpu"] +metal = ["wgpu", "burn-wgpu/metal"] +wgpu = ["burn-wgpu", "quantization", "cube"] +cpu = ["burn-cpu", "cube"] +router = ["burn-router", "ndarray", "burn-wgpu"] + +autotune = [ + "burn-wgpu?/autotune", + "burn-cuda?/autotune", + "burn-rocm?/autotune", + "burn-cpu?/autotune", +] +autotune-checks = [ + "burn-wgpu?/autotune-checks", + "burn-cuda?/autotune-checks", + "burn-rocm?/autotune-checks", + "burn-cpu?/autotune-checks", +] + +# CubeCL backends +cube = [ + "cubecl", + "cubek", + "autotune", + "burn-fusion", + "burn-cubecl", + "burn-ndarray", +] + +# Test configs +quantization = [] + +[dependencies] +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", default-features = false } +burn-tensor-testgen = { path = "../burn-tensor-testgen", version = "=0.21.0-pre.2" } + +# Backends +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2", default-features = false, features = [ + "export_tests", +] } +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-cpu = { path = "../burn-cpu", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-rocm = { path = "../burn-rocm", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2", optional = true, default-features = false, features = [ + "export_tests", +] } +burn-router = { path = "../burn-router", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true, default-features = false } + +# To wrap `Fusion +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } +burn-cubecl = { path = "../burn-cubecl", version = "=0.21.0-pre.2", optional = true, features = [ + "fusion", +] } + +num-traits = { workspace = true } +serial_test = { workspace = true } + +cubecl = { workspace = true, optional = true } +cubek = { workspace = true, features = ["random"], optional = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/README.md b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/README.md new file mode 100644 index 0000000..3480c75 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/README.md @@ -0,0 +1,111 @@ +# Burn Backend Tests + +This crate provides a comprehensive suite of tests for Burn backends, covering: + +- Tensor operations: [tests/tensor/](./tests/tensor/) +- Autodiff: [tests/autodiff/](./tests/autodiff/) +- (Optional) CubeCL kernels correctness: [tests/cubecl/](./tests/cubecl/) + +## Running Tests + +The `TestBackend` is selected via feature flags. Use the provided shorthand commands for +convenience: + +```sh +# Cpu +cargo test-cpu +# Cuda +cargo test-cuda +# Rocm +cargo test-rocm +# Wgpu / WebGpu +cargo test-wgpu +# Vulkan +cargo test-vulkan +# Metal +cargo test-metal +# Router +cargo test-router + +# NdArray +cargo test-ndarray +# LibTorch +cargo test-tch +``` + +By default, `cargo test` fail-fast across integration test binaries. When one integration test +binary fails, Cargo does not run the remaining test binaries. If you want to run all test binaries +regardless of failures, pass `--no-fail-fast`, for example: + +```sh +cargo test-cuda --no-fail-fast +``` + +## Structure + +- `tests/tensor.rs`: Tensor tests +- `tests/autodiff.rs`: Autodiff tests +- `tests/fusion.rs`: Fusion backend tests wrapping tensor and autodiff tests +- `tests/cubecl.rs`: CubeCL kernel tests + +Each test module assumes exactly one `FloatElemType`, `IntElemType`, and `TestBackend` in scope. + +### Common Modules + +- `common/backend.rs`: Backend type definitions +- `common/tensor.rs`: Reusable tensor test suite, split across float, int and bool tensor kinds +- `common/autodiff.rs`: Reusable autodiff test suite, with and without checkpointing + +### Test Reusability + +This crate uses a pattern of parameterized test modules to run the same tests with different +configurations (backends, dtypes, etc.): + +1. **Type aliases define the configuration**: Each test scope declares `FloatElemType`, + `IntElemType`, and `TestBackend` +1. **`#[path = "..."]` references shared modules**: Points to test files outside the normal module + hierarchy, e.g. `"common/tensor.rs"` +1. **`include!()` imports test code**: Test modules are included multiple times with different type + configurations +1. **`use super::*;`** propagates types down the module tree: Each level re-exports parent types so + deeply nested tests have access to the configured types + +For example, `common/tensor.rs` can be included with `FloatElemType = f32` for base tests, then +included again with `FloatElemType = f16` for half-precision tests, running the same test suite +twice with different dtypes. + +## Adding New Tests + +Add test modules under `tests/tensor/`, `tests/autodiff/`, or `tests/cubecl` respectively. They will +automatically run for all required configurations. + +For tensor tests, make sure to add the test to each relevant tensor kind: + +- `tensor/bool`: boolean tensor tests +- `tensor/float`: float tensor tests +- `tensor/int`: integer tensor tests + +**Guidelines:** + +Import types with `use super::*;` at the top of each module and use the types defined in +`common/backend.rs`: + +```rust +/// Collection of types used across tests +pub use burn_autodiff::Autodiff; +pub use burn_tensor::Tensor; +pub type TestBackend = ...; + +pub type TestTensor = Tensor; +pub type TestTensorInt = Tensor; +pub type TestTensorBool = Tensor; + +pub type FloatElem = burn_tensor::ops::FloatElem; +pub type IntElem = burn_tensor::ops::IntElem; + +pub type TestAutodiffBackend = Autodiff; +pub type TestAutodiffTensor = Tensor; +``` + +Tests will automatically run with default dtypes and any variants (f16, bf16, etc.) based on the +backend configuration. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/src/lib.rs new file mode 100644 index 0000000..23870d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/src/lib.rs @@ -0,0 +1,22 @@ +extern crate alloc; + +#[cfg(feature = "std")] +pub use burn_tensor_testgen::might_panic; + +/// Generate a test module with custom floating element types. +#[macro_export] +macro_rules! test_float_elem_variant { + ($modname:ident, $float:ty, $module:literal, [$($feat:literal),* $(,)?]) => { + #[cfg(all(test, any($(feature = $feat),*)))] + mod $modname { + pub type FloatElemType = $float; + #[allow(unused)] + pub use super::IntElemType; + + mod ty { + include!("backend.rs"); + include!($module); + } + } + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff.rs new file mode 100644 index 0000000..644ad2f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff.rs @@ -0,0 +1,20 @@ +//! Burn autodiff tests. + +#![allow( + clippy::single_range_in_vec_init, + clippy::duplicate_mod, + reason = "false positive" +)] +extern crate alloc; + +pub type FloatElemType = f32; +#[allow(unused)] +pub type IntElemType = i32; + +#[path = "common/backend.rs"] +mod backend; +pub use backend::*; + +#[allow(clippy::module_inception)] +#[path = "common/autodiff.rs"] +mod autodiff; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/abs.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/abs.rs new file mode 100644 index 0000000..ffe48b7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/abs.rs @@ -0,0 +1,58 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance, cast::ToElement}; + +#[test] +fn should_diff_abs() { + let data_1 = TensorData::from([[0.0, -1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, -10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().abs()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[71.0, 107.0], [71.0, 107.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[84.0, 42.0], [90.0, 54.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_abs_no_nans() { + let data_1 = TensorData::from([[6.0, 7.0], [9.0, -10.0]]); + let data_2 = TensorData::from([[0.0, -1.0], [3.0, 4.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().abs()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[1.0, 7.0], [1.0, 7.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[0.0, -15.0], [-3.0, -3.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let contains_nan = grad_2.contains_nan(); + assert!(!contains_nan.into_scalar().to_bool()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool1d.rs new file mode 100644 index 0000000..edeb1ef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool1d.rs @@ -0,0 +1,50 @@ +use super::*; +use burn_tensor::module::adaptive_avg_pool1d; +use burn_tensor::{Shape, Tolerance}; + +#[test] +fn test_avg_pool1d_simple() { + let test = AdaptiveAvgPool1dTestCase { + batch_size: 1, + channels: 2, + length: 5, + output_size: 3, + }; + + test.assert_output(TestTensor::from_floats( + [[ + [0.5000, 0.83333, 0.33333, 0.83333, 0.5000], + [0.5000, 0.83333, 0.33333, 0.83333, 0.5000], + ]], + &Default::default(), + )); +} + +struct AdaptiveAvgPool1dTestCase { + batch_size: usize, + channels: usize, + length: usize, + output_size: usize, +} + +impl AdaptiveAvgPool1dTestCase { + fn assert_output(self, x_grad: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.length]); + let device = Default::default(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = adaptive_avg_pool1d(x.clone(), self.output_size); + let grads = output.backward(); + let x_grad_actual = x.grad(&grads).unwrap(); + + x_grad.to_data().assert_approx_eq::( + &x_grad_actual.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool2d.rs new file mode 100644 index 0000000..3a6848b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/adaptive_avgpool2d.rs @@ -0,0 +1,96 @@ +use super::*; +use burn_tensor::module::adaptive_avg_pool2d; +use burn_tensor::{Shape, Tolerance}; + +#[test] +fn test_avg_pool2d_simple() { + let test = AdaptiveAvgPool2dTestCase { + batch_size: 1, + channels: 2, + height: 5, + width: 3, + output_size_1: 3, + output_size_2: 2, + }; + + test.assert_output(TestTensor::from_floats( + [[ + [ + [0.2500, 0.5000, 0.2500], + [0.41667, 0.83333, 0.41667], + [0.16667, 0.33333, 0.16667], + [0.41667, 0.83333, 0.41667], + [0.2500, 0.5000, 0.2500], + ], + [ + [0.2500, 0.5000, 0.2500], + [0.41667, 0.83333, 0.41667], + [0.16667, 0.33333, 0.16667], + [0.41667, 0.83333, 0.41667], + [0.2500, 0.5000, 0.2500], + ], + ]], + &Default::default(), + )); +} + +#[test] +fn test_avg_pool2d_output_1() { + let test = AdaptiveAvgPool2dTestCase { + batch_size: 1, + channels: 1, + height: 4, + width: 8, + output_size_1: 1, + output_size_2: 1, + }; + + test.assert_output(TestTensor::from_floats( + [[[ + [ + 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, + ], + [ + 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, + ], + [ + 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, + ], + [ + 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, 0.03125, + ], + ]]], + &Default::default(), + )); +} + +struct AdaptiveAvgPool2dTestCase { + batch_size: usize, + channels: usize, + height: usize, + width: usize, + output_size_1: usize, + output_size_2: usize, +} + +impl AdaptiveAvgPool2dTestCase { + fn assert_output(self, x_grad: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let device = Default::default(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = adaptive_avg_pool2d(x.clone(), [self.output_size_1, self.output_size_2]); + let grads = output.backward(); + let x_grad_actual = x.grad(&grads).unwrap(); + + x_grad.to_data().assert_approx_eq::( + &x_grad_actual.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/add.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/add.rs new file mode 100644 index 0000000..5bdc4ea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/add.rs @@ -0,0 +1,74 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_add() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_floats([2.0, 5.0], &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_floats([4.0, 1.0], &device).require_grad(); + + let tensor_3 = tensor_1.clone() + tensor_2.clone(); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([1.0, 1.0]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([1.0, 1.0]), false); + tensor_3 + .to_data() + .assert_eq(&TensorData::from([6.0, 6.0]), false); +} + +#[test] +fn should_diff_add_scalar() { + let data = TensorData::from([2.0, 10.0]); + + let tensor = TestAutodiffTensor::<1>::from_data(data, &Default::default()).require_grad(); + let tensor_out = tensor.clone().add_scalar(5.0); + let grads = tensor_out.backward(); + + let grad = tensor.grad(&grads).unwrap(); + + grad.to_data() + .assert_eq(&TensorData::from([1.0, 1.0]), false); + tensor_out + .into_data() + .assert_eq(&TensorData::from([7.0, 15.0]), false); +} + +#[test] +fn test_add_complex_1() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[2.0, 2.0], [2.0, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().add(tensor_2.clone()); + let tensor_5 = tensor_4 + .add(tensor_3) + .add_scalar(5.0) + .add(tensor_1.clone()) + .add(tensor_2.clone()); + let tensor_6 = tensor_1.clone().add(tensor_5); + + let grads = tensor_6.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[3.0, 3.0], [3.0, 3.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[2.0, 2.0], [2.0, 2.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/aggregation.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/aggregation.rs new file mode 100644 index 0000000..95f0c36 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/aggregation.rs @@ -0,0 +1,138 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_mean() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.mean().unsqueeze()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[3.5, 9.5], [3.5, 9.5]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[-0.75, -0.75], [3.0, 3.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_sum_1() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.sum().unsqueeze()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[14.0, 38.0], [14.0, 38.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[-3.0, -3.0], [12.0, 12.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_sum_2() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.clone().sum_dim(1); + let tensor_5 = tensor_4.mul(tensor_3); + + let grads = tensor_5.sum().backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[494.0, 722.0], [2990.0, 4370.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[690.0, 690.0], [958.0, 958.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_mean_dim() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.mean_dim(1).unsqueeze()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[4.0, 36.0], [3.0, -17.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[9.0, 9.0], [35.5, 35.5]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_sum_dim() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.sum_dim(1).unsqueeze()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[8.0, 72.0], [6.0, -34.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[18.0, 18.0], [71.0, 71.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool1d.rs new file mode 100644 index 0000000..f42dcfa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool1d.rs @@ -0,0 +1,102 @@ +use super::*; +use burn_tensor::module::avg_pool1d; +use burn_tensor::{Shape, Tolerance}; + +#[test] +fn test_avg_pool1d_simple() { + let test = AvgPool1dTestCase { + batch_size: 1, + channels: 1, + kernel_size: 3, + padding: 0, + stride: 1, + length: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from_floats( + [[[0.33333, 0.66667, 1.0000, 1.0000, 0.66667, 0.33333]]], + &Default::default(), + )); +} + +#[test] +fn test_avg_pool1d_complex() { + let test = AvgPool1dTestCase { + batch_size: 1, + channels: 2, + kernel_size: 3, + padding: 1, + stride: 2, + length: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from_floats( + [[ + [0.33333, 0.66667, 0.33333, 0.66667, 0.33333, 0.33333], + [0.33333, 0.66667, 0.33333, 0.66667, 0.33333, 0.33333], + ]], + &Default::default(), + )); +} + +#[test] +fn test_avg_pool1d_complex_dont_count_pad() { + let test = AvgPool1dTestCase { + batch_size: 1, + channels: 2, + kernel_size: 3, + padding: 1, + stride: 2, + length: 6, + count_include_pad: false, + }; + + test.assert_output(TestTensor::from_floats( + [[ + [0.5000, 0.83333, 0.33333, 0.66667, 0.33333, 0.33333], + [0.5000, 0.83333, 0.33333, 0.66667, 0.33333, 0.33333], + ]], + &Default::default(), + )); +} + +struct AvgPool1dTestCase { + batch_size: usize, + channels: usize, + kernel_size: usize, + padding: usize, + stride: usize, + length: usize, + count_include_pad: bool, +} + +impl AvgPool1dTestCase { + fn assert_output(self, x_grad: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.length]); + let device = Default::default(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = avg_pool1d( + x.clone(), + self.kernel_size, + self.stride, + self.padding, + self.count_include_pad, + false, + ); + let grads = output.backward(); + let x_grad_actual = x.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + x_grad + .to_data() + .assert_approx_eq::(&x_grad_actual.into_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool2d.rs new file mode 100644 index 0000000..9d99158 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/avgpool2d.rs @@ -0,0 +1,129 @@ +use super::*; +use burn_tensor::module::avg_pool2d; +use burn_tensor::{Shape, Tolerance}; + +#[test] +fn test_avg_pool2d_simple() { + let test = AvgPool2dTestCase { + batch_size: 1, + channels: 1, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + height: 6, + width: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from_floats( + [[[ + [0.11111, 0.22222, 0.33333, 0.33333, 0.22222, 0.11111], + [0.22222, 0.44444, 0.66667, 0.66667, 0.44444, 0.22222], + [0.33333, 0.66667, 1.00000, 1.00000, 0.66667, 0.33333], + [0.33333, 0.66667, 1.00000, 1.00000, 0.66667, 0.33333], + [0.22222, 0.44444, 0.66667, 0.66667, 0.44444, 0.22222], + [0.11111, 0.22222, 0.33333, 0.33333, 0.22222, 0.11111], + ]]], + &Default::default(), + )); +} + +#[test] +fn test_avg_pool2d_complex() { + let test = AvgPool2dTestCase { + batch_size: 1, + channels: 1, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 1, + padding_2: 2, + stride_1: 1, + stride_2: 2, + height: 4, + width: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from_floats( + [[[ + [0.33333, 0.33333, 0.33333, 0.33333, 0.33333, 0.33333], + [0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000], + [0.33333, 0.33333, 0.33333, 0.33333, 0.33333, 0.33333], + ]]], + &Default::default(), + )); +} + +#[test] +fn test_avg_pool2d_complex_dont_include_pad() { + let test = AvgPool2dTestCase { + batch_size: 1, + channels: 1, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 1, + padding_2: 2, + stride_1: 1, + stride_2: 2, + height: 4, + width: 6, + count_include_pad: false, + }; + + test.assert_output(TestTensor::from_floats( + [[[ + [0.6250, 0.6250, 0.41667, 0.41667, 0.6250, 0.6250], + [0.8750, 0.8750, 0.58333, 0.58333, 0.8750, 0.8750], + [0.8750, 0.8750, 0.58333, 0.58333, 0.8750, 0.8750], + [0.6250, 0.6250, 0.41667, 0.41667, 0.6250, 0.6250], + ]]], + &Default::default(), + )); +} + +struct AvgPool2dTestCase { + batch_size: usize, + channels: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + stride_1: usize, + stride_2: usize, + height: usize, + width: usize, + count_include_pad: bool, +} + +impl AvgPool2dTestCase { + fn assert_output(self, x_grad: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let device = Default::default(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = avg_pool2d( + x.clone(), + [self.kernel_size_1, self.kernel_size_2], + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + self.count_include_pad, + false, + ); + let grads = output.backward(); + let x_grad_actual = x.grad(&grads).unwrap(); + + x_grad.to_data().assert_approx_eq::( + &x_grad_actual.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/backward.rs new file mode 100644 index 0000000..424bbdf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/backward.rs @@ -0,0 +1,24 @@ +use super::*; +use burn_tensor::{Int, Tensor, TensorData, module::embedding}; + +#[test] +fn test_embedding_backward() { + let weights = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TensorData::from([[0, 1], [1, 1]]); + let x = TensorData::from([ + [[1.0, 2.0], [4.0, 5.0], [3.0, 4.0]], + [[4.0, 5.0], [8.0, 5.0], [1.0, 9.0]], + ]); + let device = Default::default(); + let weights = Tensor::::from_data(weights, &device).require_grad(); + let indices = Tensor::::from_data(indices, &device); + let x = Tensor::::from_data(x, &device).require_grad(); + + let output = embedding(weights.clone(), indices); + let output = output.matmul(x); + let grads = output.backward(); + + let grad = weights.grad(&grads).unwrap(); + grad.to_data() + .assert_eq(&TensorData::from([[3., 9., 7.], [21., 35., 27.]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/bridge.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/bridge.rs new file mode 100644 index 0000000..44851d1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/bridge.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::{DType, Distribution, Tensor}; + +#[test] +fn test_full_precision() { + let device = Default::default(); + let x1 = Tensor::::random([32, 32], Distribution::Default, &device) + .require_grad(); + let x2 = Tensor::::random([32, 32], Distribution::Default, &device) + .require_grad(); + let dtype = x1.dtype(); + + let x3 = x1.clone().cast(DType::F32); + let x4 = x2.clone().cast(DType::F32); + + let x5 = x3.matmul(x4); + let x6 = x5.cast(dtype); + let x7 = x6 * x1.clone() / x2.clone(); + + let grads = x7.backward(); + + let x1_grad = x1.grad(&grads); + let x2_grad = x2.grad(&grads); + + assert!(x1_grad.is_some()); + assert!(x2_grad.is_some()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/broadcast.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/broadcast.rs new file mode 100644 index 0000000..4dd2057 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/broadcast.rs @@ -0,0 +1,56 @@ +use super::*; + +#[test] +fn mul_broadcast() { + test_ops_broadcast_backward(|x, y| x * y); +} + +#[test] +fn div_broadcast() { + test_ops_broadcast_backward(|x, y| x / y); +} + +#[test] +fn sub_broadcast() { + test_ops_broadcast_backward(|x, y| x - y); +} + +#[test] +fn add_broadcast() { + test_ops_broadcast_backward(|x, y| x + y); +} + +#[test] +fn matmul_broadcast() { + test_ops_broadcast_backward(|x, y| x.matmul(y)); +} + +#[test] +fn mask_where_broadcast() { + test_ops_broadcast_backward(|x, y| { + let cond = y.clone().equal_elem(4); + x.mask_where(cond, y) + }); +} + +fn test_ops_broadcast_backward(func: F) +where + F: Fn(TestAutodiffTensor<3>, TestAutodiffTensor<3>) -> TestAutodiffTensor<3>, +{ + let device = Default::default(); + let w = TestAutodiffTensor::zeros([16, 5, 5], &device).require_grad(); + let x = TestAutodiffTensor::zeros([4, 5, 5], &device).require_grad(); + + // Slice isn't a broadcastable operation, so it will fail when the previous backward pass + // of an operation that support broadcast doesn't support it during the backward pass. + let y = func(w.clone().slice([0..1]), x.clone()); + + // Will panic if broadcast isn't supported! + let grads = y.backward(); + + let w_grad = w.grad(&grads).unwrap(); + let x_grad = x.grad(&grads).unwrap(); + + assert_eq!(w_grad.shape(), w.shape()); + assert_eq!(x_grad.shape(), x.shape()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cast.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cast.rs new file mode 100644 index 0000000..3e699b8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cast.rs @@ -0,0 +1,28 @@ +// Skip on metal - F64 not supported +#![cfg(all(feature = "std", not(feature = "metal")))] + +use super::*; +use burn_backend_tests::might_panic; +use burn_tensor::{DType, Tensor, TensorData}; + +#[might_panic(reason = "Unsupported precision for fusion")] +#[test] +fn cast_keeps_gradient_flow() { + let device = Default::default(); + + let x = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ) + .require_grad(); + + let y = x.clone().cast(DType::F64); + let z = y.sum(); + + let grads = z.backward(); + let grad_x = x.grad(&grads).unwrap(); + + grad_x + .to_data() + .assert_eq(&TensorData::from([[1., 1.], [1., 1.]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cat.rs new file mode 100644 index 0000000..e3e43e8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cat.rs @@ -0,0 +1,110 @@ +use super::*; + +use burn_tensor::Tolerance; + +#[test] +fn should_diff_cat() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_data([[2.0, -1.0], [5.0, 2.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::<2>::from_data([[5.0, 4.0], [-1.0, 4.0]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let mut tensor_1_list = Vec::new(); + let mut tensor_2_list = Vec::new(); + + for i in 0..2 { + tensor_1_list.push(tensor_1.clone().slice([i..i + 1])); + tensor_2_list.push(tensor_2.clone().slice([i..i + 1])); + } + + let tensor_1_cat = TestAutodiffTensor::cat(tensor_1_list.clone(), 0); + let tensor_2_cat = TestAutodiffTensor::cat(tensor_2_list.clone(), 0); + + let tensor_3_cat = tensor_1_cat.clone().matmul(tensor_2_cat.clone()); + let grads = tensor_3_cat.backward(); + + let grad_1_slice_1 = tensor_1.grad(&grads).unwrap().slice([0..1]); + let grad_1_slice_2 = tensor_1.grad(&grads).unwrap().slice([1..2]); + + let grad_2_slice_1 = tensor_2.grad(&grads).unwrap().slice([0..1]); + let grad_2_slice_2 = tensor_2.grad(&grads).unwrap().slice([1..2]); + + grad_1 + .clone() + .slice([0..1]) + .to_data() + .assert_approx_eq::(&grad_1_slice_1.to_data(), Tolerance::default()); + grad_1 + .slice([1..2]) + .to_data() + .assert_approx_eq::(&grad_1_slice_2.to_data(), Tolerance::default()); + + grad_2 + .clone() + .slice([0..1]) + .to_data() + .assert_approx_eq::(&grad_2_slice_1.to_data(), Tolerance::default()); + grad_2 + .slice([1..2]) + .to_data() + .assert_approx_eq::(&grad_2_slice_2.to_data(), Tolerance::default()); +} + +#[test] +fn should_diff_cat_more_than_1_dim() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_data([[2.0, -1.0], [5.0, 2.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::<2>::from_data([[5.0, 4.0], [-1.0, 4.0], [4.0, 1.0]], &device) + .require_grad(); + + // Concat a tensor [2, 2] with another tensor [3, 2] along dim 0. + // The resulting tensor should be [5, 2] + let tensor_3 = TestAutodiffTensor::cat(vec![tensor_1.clone(), tensor_2.clone()], 0); + assert_eq!(tensor_3.dims(), [5, 2]); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + assert_eq!(tensor_1.dims(), grad_1.dims()); + assert_eq!(tensor_2.dims(), grad_2.dims()); +} + +#[test] +fn should_slice_grads_correctly_when_some_inputs_not_tracked() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data([[1.0]], &device).require_grad(); // tracked + let tensor_2 = TestAutodiffTensor::<2>::from_data([[10.0, 20.0]], &device); // not tracked + let tensor_3 = + TestAutodiffTensor::<2>::from_data([[100.0, 200.0, 300.0]], &device).require_grad(); // tracked + + let cat = TestAutodiffTensor::cat( + vec![tensor_1.clone(), tensor_2.clone(), tensor_3.clone()], + 1, + ); + + // Make gradient per column unique so wrong slicing shows up. + let weights = TestAutodiffTensor::<2>::from_data([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]], &device); + let loss = (cat * weights).sum(); + + let grads = loss.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_3 = tensor_3.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&burn_tensor::TensorData::from([[1.0]]), false); + grad_3 + .to_data() + .assert_eq(&burn_tensor::TensorData::from([[4.0, 5.0, 6.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/ceil.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/ceil.rs new file mode 100644 index 0000000..04416a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/ceil.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_ceil() { + let data = TensorData::from([ + [-1.9751, 0.0714, 0.0643, 0.2406], + [-1.3172, 0.1252, -0.1119, -0.0127], + ]); + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data, &device).require_grad(); + let tensor_2 = tensor_1.clone().ceil(); + let grads = tensor_2.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + + grad_1.to_data().assert_eq( + &TensorData::from([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/checkpoint.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/checkpoint.rs new file mode 100644 index 0000000..dc89ea1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/checkpoint.rs @@ -0,0 +1,215 @@ +use super::*; +use burn_tensor::{Bool, Tensor, TensorData}; + +#[test] +fn test_autodiff_checkpoint_complicated_computation() { + let data_0 = TensorData::from([[0.0, 7.0], [7.0, 7.0]]); + let data_1 = TensorData::from([[0.1, 7.0], [7.0, 7.0]]); + let data_2 = TensorData::from([[0.2, 7.0], [7.0, 7.0]]); + let data_3 = TensorData::from([[0.3, 7.0], [7.0, 7.0]]); + let data_4 = TensorData::from([[0.4, 7.0], [7.0, 7.0]]); + + let device = Default::default(); + let tensor_0 = TestAutodiffTensor::<2>::from_data(data_0, &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + let tensor_4 = TestAutodiffTensor::from_data(data_4, &device).require_grad(); + + let tensor_5 = compute_bound_eager(tensor_0, tensor_1); + let tensor_6 = compute_bound_lazy(tensor_2, tensor_3.clone()); + let tensor_7 = memory_bound_eager(tensor_3, tensor_4); + let tensor_8 = compute_bound_lazy(tensor_6, tensor_7.clone()); + let tensor_9 = memory_bound_eager_scalar(tensor_7, 11.); + let tensor_10 = memory_bound_lazy(tensor_5, tensor_8.clone()); + let tensor_11 = memory_bound_lazy(tensor_8, tensor_9); + let tensor_12 = compute_bound_lazy(tensor_10, tensor_11); + + assert_checkpoint(tensor_12); +} + +#[test] +fn test_autodiff_checkpoint_with_missing_requirement() { + let data_0 = TensorData::from([[0.0, 7.0], [7.0, 7.0]]); + let data_1 = TensorData::from([[0.1, 7.0], [7.0, 7.0]]); + + let device = Default::default(); + let tensor_0 = TestAutodiffTensor::<2>::from_data(data_0, &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device); // does not require_grad + + let tensor_2 = memory_bound_eager(tensor_0, tensor_1); + let tensor_3 = memory_bound_eager_scalar(tensor_2.clone(), 11.); + let tensor_4 = memory_bound_eager_scalar(tensor_2.clone(), 11.); + let tensor_5 = compute_bound_lazy(tensor_3, tensor_4); + let tensor_6 = compute_bound_eager_scalar(tensor_5.clone(), 11.); + let tensor_7 = memory_bound_eager(tensor_5, tensor_2); + let tensor_8 = memory_bound_eager(tensor_6, tensor_7); + + assert_checkpoint(tensor_8); +} + +#[test] +fn test_autodiff_checkpoint_with_many_duplicates() { + let data_0 = TensorData::from([[4.0, 7.0], [7.0, 7.0]]); + + let device = Default::default(); + let tensor_0 = TestAutodiffTensor::<2>::from_data(data_0, &device).require_grad(); + + let tensor_1 = memory_bound_eager(tensor_0.clone(), tensor_0.clone()); + let tensor_2 = compute_bound_eager(tensor_0.clone(), tensor_0.clone()); + let tensor_3 = memory_bound_lazy(tensor_0.clone(), tensor_0.clone()); + let tensor_4 = compute_bound_lazy(tensor_0.clone(), tensor_0.clone()); + + let tensor_5 = memory_bound_eager(tensor_1.clone(), tensor_0.clone()); + let tensor_6 = memory_bound_eager(tensor_0.clone(), tensor_5.clone()); + let tensor_7 = compute_bound_lazy(tensor_3.clone(), tensor_5.clone()); + let tensor_8 = compute_bound_eager(tensor_4.clone(), tensor_2.clone()); + let tensor_9 = memory_bound_lazy(tensor_6, tensor_7); + let tensor_10 = memory_bound_eager(tensor_0, tensor_9); + let tensor_11 = memory_bound_eager_scalar(tensor_10, 9.); + let tensor_12 = compute_bound_lazy(tensor_8, tensor_11); + + assert_checkpoint(tensor_12); +} + +#[test] +fn test_autodiff_checkpoint_with_long_chain_of_eager_memory_bound() { + let data_0 = TensorData::from([[0.0, 7.0], [7.0, 7.0]]); + let data_1 = TensorData::from([[0.1, 7.0], [7.0, 7.0]]); + let data_2 = TensorData::from([[0.2, 7.0], [7.0, 7.0]]); + let data_3 = TensorData::from([[0.3, 7.0], [7.0, 7.0]]); + let data_4 = TensorData::from([[0.4, 7.0], [7.0, 7.0]]); + + let device = Default::default(); + let tensor_0 = TestAutodiffTensor::<2>::from_data(data_0, &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + let tensor_4 = TestAutodiffTensor::from_data(data_4, &device).require_grad(); + + let tensor_5 = memory_bound_eager(tensor_0, tensor_1.clone()); + let tensor_6 = memory_bound_eager(tensor_5, tensor_2); + let tensor_7 = memory_bound_eager(tensor_6, tensor_3); + let tensor_8 = memory_bound_eager(tensor_7, tensor_4); + let tensor_9 = memory_bound_lazy(tensor_8, tensor_1); + + assert_checkpoint(tensor_9) +} + +#[test] +fn test_autodiff_checkpoint_half_sub_graph_not_tracked() { + let data_0 = TensorData::from([[0.0, 7.0], [7.0, 7.0]]); + let data_1 = TensorData::from([[0.1, 7.0], [7.0, 7.0]]); + let data_2 = TensorData::from([[0.2, 7.0], [7.0, 7.0]]); + let data_3 = TensorData::from([[0.3, 7.0], [7.0, 7.0]]); + let data_4 = TensorData::from([[0.4, 7.0], [7.0, 7.0]]); + let data_5 = TensorData::from([[0.5, 7.0], [7.0, 7.0]]); + + let device = Default::default(); + let tensor_0 = TestAutodiffTensor::<2>::from_data(data_0, &device); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + let tensor_4 = TestAutodiffTensor::from_data(data_4, &device).require_grad(); + let tensor_5 = TestAutodiffTensor::from_data(data_5, &device).require_grad(); + + let tensor_6 = memory_bound_lazy(tensor_0, tensor_1); + let tensor_7 = compute_bound_eager(tensor_6, tensor_2); + + let tensor_8 = memory_bound_eager(tensor_3, tensor_4); + let tensor_9 = compute_bound_lazy(tensor_8, tensor_5); + + let tensor_10 = compute_bound_lazy(tensor_7, tensor_9); + + assert_checkpoint(tensor_10); +} + +#[test] +fn test_autodiff_checkpoint_very_complex() { + let data_0 = TensorData::from([[0.0, 7.0], [7.0, 7.0]]); + let data_1 = TensorData::from([[0.1, 7.0], [7.0, 7.0]]); + let data_2 = TensorData::from([[0.2, 7.0], [7.0, 7.0]]); + let data_3 = TensorData::from([[0.3, 7.0], [7.0, 7.0]]); + let data_4 = TensorData::from([[0.4, 7.0], [7.0, 7.0]]); + + let device = Default::default(); + let tensor_0 = TestAutodiffTensor::<2>::from_data(data_0, &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + let tensor_4 = TestAutodiffTensor::from_data(data_4, &device).require_grad(); + + let tensor_5 = memory_bound_eager_scalar(tensor_0, 8.); + let tensor_6 = memory_bound_lazy(tensor_5.clone(), tensor_1.clone()); + let tensor_7 = compute_bound_lazy(tensor_6.clone(), tensor_6); + let tensor_8 = memory_bound_lazy(tensor_1.clone(), tensor_5.clone()); + let tensor_9 = memory_bound_eager_scalar(tensor_7.clone(), 7.); + let tensor_10 = compute_bound_eager(tensor_5, tensor_8); + let tensor_11 = memory_bound_eager(tensor_2.clone(), tensor_9); + let tensor_12 = memory_bound_lazy(tensor_2.clone(), tensor_2); + let tensor_13 = compute_bound_eager(tensor_10.clone(), tensor_11); + let tensor_14 = compute_bound_eager_scalar(tensor_3, 8.); + let tensor_15 = compute_bound_lazy(tensor_4, tensor_12); + let tensor_16 = memory_bound_lazy(tensor_10, tensor_7); + let tensor_17 = compute_bound_lazy(tensor_13, tensor_1); + let tensor_18 = memory_bound_eager(tensor_15, tensor_16); + let tensor_19 = compute_bound_eager(tensor_14, tensor_17); + let tensor_20 = memory_bound_lazy(tensor_18, tensor_19); + let tensor_21 = memory_bound_eager_scalar(tensor_20, 8.); + + assert_checkpoint(tensor_21) +} + +fn assert_checkpoint(tensor: TestAutodiffTensor) { + // Assert is not explicit here, but the test can fail + // - when a tensor is actually required more than n_required, it won't be found and will panic + // - when a tensor is actually required less than n_required, the backward states map won't be + // empty and will fail the assertion within the backward code, same for retro_forwards + tensor.backward(); +} + +// Does not save its state and does not need its parents +fn memory_bound_eager( + tensor_a: TestAutodiffTensor, + tensor_b: TestAutodiffTensor, +) -> TestAutodiffTensor { + tensor_a.add(tensor_b) +} +fn memory_bound_eager_scalar( + tensor_a: TestAutodiffTensor, + b: f32, +) -> TestAutodiffTensor { + tensor_a.add_scalar(b) +} + +// Saves its own state and does not need its parents +fn compute_bound_eager( + tensor_a: TestAutodiffTensor, + tensor_b: TestAutodiffTensor, +) -> TestAutodiffTensor { + let mask = Tensor::::empty(tensor_a.shape(), &tensor_a.device()); + tensor_a.mask_where(mask, tensor_b) +} +fn compute_bound_eager_scalar( + tensor_a: TestAutodiffTensor, + b: f32, +) -> TestAutodiffTensor { + let mask = Tensor::::empty(tensor_a.shape(), &tensor_a.device()); + tensor_a.mask_fill(mask, b) +} + +// Does not save its state and needs its parents +fn memory_bound_lazy( + tensor_a: TestAutodiffTensor, + tensor_b: TestAutodiffTensor, +) -> TestAutodiffTensor { + tensor_a.mul(tensor_b) +} + +// Saves its own state and needs its parents +fn compute_bound_lazy( + tensor_a: TestAutodiffTensor, + tensor_b: TestAutodiffTensor, +) -> TestAutodiffTensor { + tensor_a.matmul(tensor_b) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/complex.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/complex.rs new file mode 100644 index 0000000..ac25cc5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/complex.rs @@ -0,0 +1,81 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_full_complex_1() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.matmul(tensor_1.clone()); + let tensor_5 = tensor_4.mul(tensor_2.clone()); + + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[593., 463.0], [487.0, 539.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[734.0, 294.0], [1414.0, 242.0]]), false); +} + +#[test] +fn should_diff_full_complex_2() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.matmul(tensor_1.clone()); + let tensor_5 = tensor_4.add_scalar(17.0).add(tensor_2.clone()); + + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[166.0, 110.0], [212.0, 156.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[113.0, 141.0], [33.0, 41.0]]), false); +} + +#[test] +fn should_diff_full_complex_3() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.matmul(tensor_1.clone()); + let tensor_5 = tensor_4.clone().sub(tensor_2.clone()); + let tensor_6 = tensor_5.add(tensor_4); + + let grads = tensor_6.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[332.0, 220.0], [424.0, 312.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[223.0, 279.0], [63.0, 79.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv1d.rs new file mode 100644 index 0000000..c54e154 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv1d.rs @@ -0,0 +1,277 @@ +use super::*; +use burn_tensor::{Shape, Tolerance, module::conv1d, ops::ConvOptions}; + +#[test] +fn test_conv1d_basic() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 1, + groups: 1, + length: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[14., 24., 24., 18.], [26., 42., 42., 30.]], + [[14., 24., 24., 18.], [26., 42., 42., 30.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[30., 44., 36.], [54., 76., 60.]], + [[30., 44., 36.], [54., 76., 60.]], + ], + &device, + ), + bias: TestTensor::from_floats([8., 8.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv1d_different_channels() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 3, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 1, + groups: 1, + length: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[39., 63., 63., 45.], [57., 90., 90., 63.]], + [[39., 63., 63., 45.], [57., 90., 90., 63.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[30., 44., 36.], [54., 76., 60.]], + [[30., 44., 36.], [54., 76., 60.]], + [[30., 44., 36.], [54., 76., 60.]], + ], + &device, + ), + bias: TestTensor::from_floats([8., 8., 8.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv1d_with_padding() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 2, + stride: 1, + dilation: 1, + groups: 1, + length: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[24., 24., 24., 24.], [42., 42., 42., 42.]], + [[24., 24., 24., 24.], [42., 42., 42., 42.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[44., 44., 44.], [76., 76., 76.]], + [[44., 44., 44.], [76., 76., 76.]], + ], + &device, + ), + bias: TestTensor::from_floats([12., 12.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv1d_with_stride() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 2, + dilation: 1, + groups: 1, + length: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[8., 16., 8., 10.], [14., 28., 14., 16.]], + [[8., 16., 8., 10.], [14., 28., 14., 16.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[10., 20., 24.], [18., 36., 40.]], + [[10., 20., 24.], [18., 36., 40.]], + ], + &device, + ), + bias: TestTensor::from_floats([4., 4.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv1d_dilation() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 2, + groups: 1, + length: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[6., 8., 8., 10.], [12., 14., 14., 16.]], + [[6., 8., 8., 10.], [12., 14., 14., 16.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[8., 22., 14.], [16., 38., 22.]], + [[8., 22., 14.], [16., 38., 22.]], + ], + &device, + ), + bias: TestTensor::from_floats([4., 4.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv1d_groups() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 1, + groups: 2, + length: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[1., 3., 3., 3.], [7., 12., 12., 9.]], + [[1., 3., 3., 3.], [7., 12., 12., 9.]], + ], + &device, + ), + weight: TestTensor::from_floats([[[30., 44., 36.]], [[54., 76., 60.]]], &device), + bias: TestTensor::from_floats([8., 8.], &device), + }; + test.assert_grads(grads); +} + +struct Conv1dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size: usize, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + length: usize, +} + +struct Grads { + x: TestTensor<3>, + weight: TestTensor<3>, + bias: TestTensor<1>, +} + +impl Conv1dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.length]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size, + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<3, _>(shape_weight) + .into_data(), + &device, + ) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + + let output = conv1d( + x.clone(), + weight.clone(), + Some(bias.clone()), + ConvOptions::new([self.stride], [self.padding], [self.dilation], self.groups), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + let tolerance = Tolerance::default(); + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), tolerance); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), tolerance); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv2d.rs new file mode 100644 index 0000000..bb6c8ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv2d.rs @@ -0,0 +1,962 @@ +use super::*; +use burn_tensor::{Shape, Tolerance, module::conv2d, ops::ConvOptions}; + +#[test] +fn test_conv2d_basic() { + let test = Conv2dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [88., 138., 138., 96.], + [150., 234., 234., 162.], + [150., 234., 234., 162.], + [112., 174., 174., 120.], + ], + [ + [160., 246., 246., 168.], + [258., 396., 396., 270.], + [258., 396., 396., 270.], + [184., 282., 282., 192.], + ], + ], + [ + [ + [88., 138., 138., 96.], + [150., 234., 234., 162.], + [150., 234., 234., 162.], + [112., 174., 174., 120.], + ], + [ + [160., 246., 246., 168.], + [258., 396., 396., 270.], + [258., 396., 396., 270.], + [184., 282., 282., 192.], + ], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[378., 516., 396.], [552., 752., 576.], [450., 612., 468.]], + [[666., 900., 684.], [936., 1264., 960.], [738., 996., 756.]], + ], + [ + [[378., 516., 396.], [552., 752., 576.], [450., 612., 468.]], + [[666., 900., 684.], [936., 1264., 960.], [738., 996., 756.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([32., 32.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_different_channels() { + let test = Conv2dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [240., 369., 369., 252.], + [387., 594., 594., 405.], + [387., 594., 594., 405.], + [276., 423., 423., 288.], + ], + [ + [348., 531., 531., 360.], + [549., 837., 837., 567.], + [549., 837., 837., 567.], + [384., 585., 585., 396.], + ], + ], + [ + [ + [240., 369., 369., 252.], + [387., 594., 594., 405.], + [387., 594., 594., 405.], + [276., 423., 423., 288.], + ], + [ + [348., 531., 531., 360.], + [549., 837., 837., 567.], + [549., 837., 837., 567.], + [384., 585., 585., 396.], + ], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[378., 516., 396.], [552., 752., 576.], [450., 612., 468.]], + [[666., 900., 684.], [936., 1264., 960.], [738., 996., 756.]], + ], + [ + [[378., 516., 396.], [552., 752., 576.], [450., 612., 468.]], + [[666., 900., 684.], [936., 1264., 960.], [738., 996., 756.]], + ], + [ + [[378., 516., 396.], [552., 752., 576.], [450., 612., 468.]], + [[666., 900., 684.], [936., 1264., 960.], [738., 996., 756.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([32., 32., 32.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_different_kernel_size() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [116., 180., 192., 132.], + [198., 306., 324., 222.], + [198., 306., 324., 222.], + [148., 228., 240., 164.], + ], + [ + [212., 324., 336., 228.], + [342., 522., 540., 366.], + [342., 522., 540., 366.], + [244., 372., 384., 260.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [27., 45., 54., 39.], + [52., 84., 96., 68.], + [51., 81., 90., 63.], + ], + [ + [123., 189., 198., 135.], + [180., 276., 288., 196.], + [147., 225., 234., 159.], + ], + ], + [ + [ + [27., 45., 54., 39.], + [52., 84., 96., 68.], + [51., 81., 90., 63.], + ], + [ + [123., 189., 198., 135.], + [180., 276., 288., 196.], + [147., 225., 234., 159.], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([12., 12.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_different_padding() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 2, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [138., 138., 138., 138.], + [234., 234., 234., 234.], + [234., 234., 234., 234.], + [174., 174., 174., 174.], + ], + [ + [246., 246., 246., 246.], + [396., 396., 396., 396.], + [396., 396., 396., 396.], + [282., 282., 282., 282.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[66., 66., 66.], [120., 120., 120.], [114., 114., 114.]], + [[258., 258., 258.], [376., 376., 376.], [306., 306., 306.]], + ], + [ + [[66., 66., 66.], [120., 120., 120.], [114., 114., 114.]], + [[258., 258., 258.], [376., 376., 376.], [306., 306., 306.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([24., 24.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_different_width() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 5, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [88., 138., 138., 138., 96.], + [150., 234., 234., 234., 162.], + [150., 234., 234., 234., 162.], + [112., 174., 174., 174., 120.], + ], + [ + [160., 246., 246., 246., 168.], + [258., 396., 396., 396., 270.], + [258., 396., 396., 396., 270.], + [184., 282., 282., 282., 192.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[78., 105., 90.], [144., 190., 160.], [138., 180., 150.]], + [[318., 405., 330.], [464., 590., 480.], [378., 480., 390.]], + ], + [ + [[78., 105., 90.], [144., 190., 160.], [138., 180., 150.]], + [[318., 405., 330.], [464., 590., 480.], [378., 480., 390.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([20., 20.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_stride_2() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 2, + stride_2: 2, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 6, + width: 6, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [26., 52., 26., 52., 26., 28.], + [52., 104., 52., 104., 52., 56.], + [26., 52., 26., 52., 26., 28.], + [52., 104., 52., 104., 52., 56.], + [26., 52., 26., 52., 26., 28.], + [32., 64., 32., 64., 32., 34.], + ], + [ + [44., 88., 44., 88., 44., 46.], + [88., 176., 88., 176., 88., 92.], + [44., 88., 44., 88., 44., 46.], + [88., 176., 88., 176., 88., 92.], + [44., 88., 44., 88., 44., 46.], + [50., 100., 50., 100., 50., 52.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[56., 84., 90.], [84., 126., 135.], [120., 180., 189.]], + [[200., 300., 306.], [300., 450., 459.], [336., 504., 513.]], + ], + [ + [[56., 84., 90.], [84., 126., 135.], [120., 180., 189.]], + [[200., 300., 306.], [300., 450., 459.], [336., 504., 513.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([9., 9.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_different_stride() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 3, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 8, + width: 8, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [50., 78., 78., 78., 78., 78., 78., 54.], + [62., 96., 96., 96., 96., 96., 96., 66.], + [38., 60., 60., 60., 60., 60., 60., 42.], + [50., 78., 78., 78., 78., 78., 78., 54.], + [62., 96., 96., 96., 96., 96., 96., 66.], + [38., 60., 60., 60., 60., 60., 60., 42.], + [50., 78., 78., 78., 78., 78., 78., 54.], + [62., 96., 96., 96., 96., 96., 96., 66.], + ], + [ + [86., 132., 132., 132., 132., 132., 132., 90.], + [98., 150., 150., 150., 150., 150., 150., 102.], + [74., 114., 114., 114., 114., 114., 114., 78.], + [86., 132., 132., 132., 132., 132., 132., 90.], + [98., 150., 150., 150., 150., 150., 150., 102.], + [74., 114., 114., 114., 114., 114., 114., 78.], + [86., 132., 132., 132., 132., 132., 132., 90.], + [98., 150., 150., 150., 150., 150., 150., 102.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[434., 504., 448.], [567., 660., 588.], [735., 852., 756.]], + [ + [1330., 1528., 1344.], + [1911., 2196., 1932.], + [2079., 2388., 2100.], + ], + ], + [ + [[434., 504., 448.], [567., 660., 588.], [735., 852., 756.]], + [ + [1330., 1528., 1344.], + [1911., 2196., 1932.], + [2079., 2388., 2100.], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([24., 24.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_dilation_2() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 2, + dilation_2: 2, + groups: 1, + height: 6, + width: 6, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [18., 38., 38., 42., 42., 22.], + [42., 88., 88., 96., 96., 50.], + [42., 88., 88., 96., 96., 50.], + [54., 112., 112., 120., 120., 62.], + [54., 112., 112., 120., 120., 62.], + [30., 62., 62., 66., 66., 34.], + ], + [ + [36., 74., 74., 78., 78., 40.], + [78., 160., 160., 168., 168., 86.], + [78., 160., 160., 168., 168., 86.], + [90., 184., 184., 192., 192., 98.], + [90., 184., 184., 192., 192., 98.], + [48., 98., 98., 102., 102., 52.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[63., 102., 90.], [192., 280., 228.], [225., 318., 252.]], + [[387., 534., 414.], [624., 856., 660.], [549., 750., 576.]], + ], + [ + [[63., 102., 90.], [192., 280., 228.], [225., 318., 252.]], + [[387., 534., 414.], [624., 856., 660.], [549., 750., 576.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([16., 16.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_different_dilation() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 2, + dilation_2: 3, + groups: 1, + height: 6, + width: 6, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [18., 0., 20., 20., 0., 22.], + [42., 0., 46., 46., 0., 50.], + [42., 0., 46., 46., 0., 50.], + [54., 0., 58., 58., 0., 62.], + [54., 0., 58., 58., 0., 62.], + [30., 0., 32., 32., 0., 34.], + ], + [ + [36., 0., 38., 38., 0., 40.], + [78., 0., 82., 82., 0., 86.], + [78., 0., 82., 82., 0., 86.], + [90., 0., 94., 94., 0., 98.], + [90., 0., 94., 94., 0., 98.], + [48., 0., 50., 50., 0., 52.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[18., 51., 33.], [60., 140., 80.], [72., 159., 87.]], + [[126., 267., 141.], [204., 428., 224.], [180., 375., 195.]], + ], + [ + [[18., 51., 33.], [60., 140., 80.], [72., 159., 87.]], + [[126., 267., 141.], [204., 428., 224.], [180., 375., 195.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([8., 8.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_groups() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 5, + width: 5, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [0., 1., 3., 3., 2.], + [3., 8., 15., 12., 7.], + [9., 21., 36., 27., 15.], + [9., 20., 33., 24., 13.], + [6., 13., 21., 15., 8.], + ], + [ + [9., 19., 30., 21., 11.], + [21., 44., 69., 48., 25.], + [36., 75., 117., 81., 42.], + [27., 56., 87., 60., 31.], + [15., 31., 48., 33., 17.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [[[54., 63., 72.], [99., 108., 117.], [144., 153., 162.]]], + [[[279., 288., 297.], [324., 333., 342.], [369., 378., 387.]]], + ], + &device, + ), + bias: TestTensor::from_floats([9., 9.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_groups_stride_2() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 4, + channels_out: 4, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 2, + stride_2: 2, + dilation_1: 1, + dilation_2: 1, + groups: 4, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [4., 8., 4., 5.], + [8., 16., 8., 10.], + [4., 8., 4., 5.], + [7., 14., 7., 8.], + ], + [ + [13., 26., 13., 14.], + [26., 52., 26., 28.], + [13., 26., 13., 14.], + [16., 32., 16., 17.], + ], + [ + [22., 44., 22., 23.], + [44., 88., 44., 46.], + [22., 44., 22., 23.], + [25., 50., 25., 26.], + ], + [ + [31., 62., 31., 32.], + [62., 124., 62., 64.], + [31., 62., 31., 32.], + [34., 68., 34., 35.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [[[5., 10., 12.], [10., 20., 24.], [18., 36., 40.]]], + [[[21., 42., 44.], [42., 84., 88.], [50., 100., 104.]]], + [[[37., 74., 76.], [74., 148., 152.], [82., 164., 168.]]], + [[[53., 106., 108.], [106., 212., 216.], [114., 228., 232.]]], + ], + &device, + ), + bias: TestTensor::from_floats([4., 4., 4., 4.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_groups_different_channels() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 6, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 3, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [9., 20., 24., 13.], + [24., 52., 60., 32.], + [36., 76., 84., 44.], + [21., 44., 48., 25.], + ], + [ + [45., 92., 96., 49.], + [96., 196., 204., 104.], + [108., 220., 228., 116.], + [57., 116., 120., 61.], + ], + [ + [81., 164., 168., 85.], + [168., 340., 348., 176.], + [180., 364., 372., 188.], + [93., 188., 192., 97.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [[[10., 14., 18.], [26., 30., 34.], [42., 46., 50.]]], + [[[10., 14., 18.], [26., 30., 34.], [42., 46., 50.]]], + [[[74., 78., 82.], [90., 94., 98.], [106., 110., 114.]]], + [[[74., 78., 82.], [90., 94., 98.], [106., 110., 114.]]], + [[[138., 142., 146.], [154., 158., 162.], [170., 174., 178.]]], + [[[138., 142., 146.], [154., 158., 162.], [170., 174., 178.]]], + ], + &device, + ), + bias: TestTensor::from_floats([4., 4., 4., 4., 4., 4.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_complex() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 2, + kernel_size_2: 3, + padding_1: 1, + padding_2: 2, + stride_1: 1, + stride_2: 2, + dilation_1: 2, + dilation_2: 3, + groups: 1, + height: 4, + width: 5, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [36., 39., 0., 39., 42.], + [81., 87., 0., 87., 93.], + [81., 87., 0., 87., 93.], + [45., 48., 0., 48., 51.], + ], + [ + [54., 57., 0., 57., 60.], + [117., 123., 0., 123., 129.], + [117., 123., 0., 123., 129.], + [63., 66., 0., 66., 69.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[15., 42., 27.], [30., 72., 42.]], + [[75., 162., 87.], [90., 192., 102.]], + ], + [ + [[15., 42., 27.], [30., 72., 42.]], + [[75., 162., 87.], [90., 192., 102.]], + ], + [ + [[15., 42., 27.], [30., 72., 42.]], + [[75., 162., 87.], [90., 192., 102.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([8., 8., 8.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv2d_groups_stride_2_no_pad() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 4, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 2, + stride_2: 2, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [0., 1., 2., 0.], + [3., 4., 5., 0.], + [6., 7., 8., 0.], + [0., 0., 0., 0.], + ], + [ + [9., 10., 11., 0.], + [12., 13., 14., 0.], + [15., 16., 17., 0.], + [0., 0., 0., 0.], + ], + [ + [18., 19., 20., 0.], + [21., 22., 23., 0.], + [24., 25., 26., 0.], + [0., 0., 0., 0.], + ], + [ + [27., 28., 29., 0.], + [30., 31., 32., 0.], + [33., 34., 35., 0.], + [0., 0., 0., 0.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[0., 1., 2.], [4., 5., 6.], [8., 9., 10.]], + [[16., 17., 18.], [20., 21., 22.], [24., 25., 26.]], + ], + [ + [[32., 33., 34.], [36., 37., 38.], [40., 41., 42.]], + [[48., 49., 50.], [52., 53., 54.], [56., 57., 58.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([1., 1.], &device), + }; + test.assert_grads(grads); +} + +struct Conv2dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + stride_1: usize, + stride_2: usize, + dilation_1: usize, + dilation_2: usize, + groups: usize, + height: usize, + width: usize, +} + +struct Grads { + x: TestTensor<4>, + weight: TestTensor<4>, + bias: TestTensor<1>, +} + +impl Conv2dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size_1, + self.kernel_size_2, + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<4, _>(shape_weight) + .into_data(), + &device, + ) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = conv2d( + x.clone(), + weight.clone(), + Some(bias.clone()), + ConvOptions::new( + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + [self.dilation_1, self.dilation_2], + self.groups, + ), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + let tolerance = Tolerance::rel_abs(0.01, 0.01); + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), tolerance); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), tolerance); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv3d.rs new file mode 100644 index 0000000..f6740bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv3d.rs @@ -0,0 +1,690 @@ +use super::*; +use burn_tensor::{Shape, Tolerance, module::conv3d, ops::ConvOptions}; + +#[test] +fn test_conv3d_basic() { + let test = Conv3dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 1, + depth: 4, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [ + [536., 816., 816., 552.], + [840., 1278., 1278., 864.], + [840., 1278., 1278., 864.], + [584., 888., 888., 600.], + ], + [ + [912., 1386., 1386., 936.], + [1422., 2160., 2160., 1458.], + [1422., 2160., 2160., 1458.], + [984., 1494., 1494., 1008.], + ], + [ + [912., 1386., 1386., 936.], + [1422., 2160., 2160., 1458.], + [1422., 2160., 2160., 1458.], + [984., 1494., 1494., 1008.], + ], + [ + [680., 1032., 1032., 696.], + [1056., 1602., 1602., 1080.], + [1056., 1602., 1602., 1080.], + [728., 1104., 1104., 744.], + ], + ], + [ + [ + [968., 1464., 1464., 984.], + [1488., 2250., 2250., 1512.], + [1488., 2250., 2250., 1512.], + [1016., 1536., 1536., 1032.], + ], + [ + [1560., 2358., 2358., 1584.], + [2394., 3618., 3618., 2430.], + [2394., 3618., 3618., 2430.], + [1632., 2466., 2466., 1656.], + ], + [ + [1560., 2358., 2358., 1584.], + [2394., 3618., 3618., 2430.], + [2394., 3618., 3618., 2430.], + [1632., 2466., 2466., 1656.], + ], + [ + [1112., 1680., 1680., 1128.], + [1704., 2574., 2574., 1728.], + [1704., 2574., 2574., 1728.], + [1160., 1752., 1752., 1176.], + ], + ], + ], + [ + [ + [ + [536., 816., 816., 552.], + [840., 1278., 1278., 864.], + [840., 1278., 1278., 864.], + [584., 888., 888., 600.], + ], + [ + [912., 1386., 1386., 936.], + [1422., 2160., 2160., 1458.], + [1422., 2160., 2160., 1458.], + [984., 1494., 1494., 1008.], + ], + [ + [912., 1386., 1386., 936.], + [1422., 2160., 2160., 1458.], + [1422., 2160., 2160., 1458.], + [984., 1494., 1494., 1008.], + ], + [ + [680., 1032., 1032., 696.], + [1056., 1602., 1602., 1080.], + [1056., 1602., 1602., 1080.], + [728., 1104., 1104., 744.], + ], + ], + [ + [ + [968., 1464., 1464., 984.], + [1488., 2250., 2250., 1512.], + [1488., 2250., 2250., 1512.], + [1016., 1536., 1536., 1032.], + ], + [ + [1560., 2358., 2358., 1584.], + [2394., 3618., 3618., 2430.], + [2394., 3618., 3618., 2430.], + [1632., 2466., 2466., 1656.], + ], + [ + [1560., 2358., 2358., 1584.], + [2394., 3618., 3618., 2430.], + [2394., 3618., 3618., 2430.], + [1632., 2466., 2466., 1656.], + ], + [ + [1112., 1680., 1680., 1128.], + [1704., 2574., 2574., 1728.], + [1704., 2574., 2574., 1728.], + [1160., 1752., 1752., 1176.], + ], + ], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [ + [4590., 6156., 4644.], + [6264., 8400., 6336.], + [4806., 6444., 4860.], + ], + [ + [6696., 8976., 6768.], + [9120., 12224., 9216.], + [6984., 9360., 7056.], + ], + [ + [5454., 7308., 5508.], + [7416., 9936., 7488.], + [5670., 7596., 5724.], + ], + ], + [ + [ + [8046., 10764., 8100.], + [10872., 14544., 10944.], + [8262., 11052., 8316.], + ], + [ + [11304., 15120., 11376.], + [15264., 20416., 15360.], + [11592., 15504., 11664.], + ], + [ + [8910., 11916., 8964.], + [12024., 16080., 12096.], + [9126., 12204., 9180.], + ], + ], + ], + [ + [ + [ + [4590., 6156., 4644.], + [6264., 8400., 6336.], + [4806., 6444., 4860.], + ], + [ + [6696., 8976., 6768.], + [9120., 12224., 9216.], + [6984., 9360., 7056.], + ], + [ + [5454., 7308., 5508.], + [7416., 9936., 7488.], + [5670., 7596., 5724.], + ], + ], + [ + [ + [8046., 10764., 8100.], + [10872., 14544., 10944.], + [8262., 11052., 8316.], + ], + [ + [11304., 15120., 11376.], + [15264., 20416., 15360.], + [11592., 15504., 11664.], + ], + [ + [8910., 11916., 8964.], + [12024., 16080., 12096.], + [9126., 12204., 9180.], + ], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([128., 128.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv3d_complex() { + let test = Conv3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 2, + kernel_size_2: 3, + kernel_size_3: 4, + padding_1: 1, + padding_2: 2, + padding_3: 3, + stride_1: 1, + stride_2: 2, + stride_3: 3, + dilation_1: 2, + dilation_2: 3, + dilation_3: 4, + groups: 1, + depth: 5, + height: 6, + width: 7, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [ + [0., 147., 0., 0., 0., 150., 0.], + [0., 159., 0., 0., 0., 162., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 159., 0., 0., 0., 162., 0.], + [0., 171., 0., 0., 0., 174., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 330., 0., 0., 0., 336., 0.], + [0., 354., 0., 0., 0., 360., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 354., 0., 0., 0., 360., 0.], + [0., 378., 0., 0., 0., 384., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 330., 0., 0., 0., 336., 0.], + [0., 354., 0., 0., 0., 360., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 354., 0., 0., 0., 360., 0.], + [0., 378., 0., 0., 0., 384., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 330., 0., 0., 0., 336., 0.], + [0., 354., 0., 0., 0., 360., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 354., 0., 0., 0., 360., 0.], + [0., 378., 0., 0., 0., 384., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 183., 0., 0., 0., 186., 0.], + [0., 195., 0., 0., 0., 198., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 195., 0., 0., 0., 198., 0.], + [0., 207., 0., 0., 0., 210., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + ], + [ + [ + [0., 219., 0., 0., 0., 222., 0.], + [0., 231., 0., 0., 0., 234., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 231., 0., 0., 0., 234., 0.], + [0., 243., 0., 0., 0., 246., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 474., 0., 0., 0., 480., 0.], + [0., 498., 0., 0., 0., 504., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 498., 0., 0., 0., 504., 0.], + [0., 522., 0., 0., 0., 528., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 474., 0., 0., 0., 480., 0.], + [0., 498., 0., 0., 0., 504., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 498., 0., 0., 0., 504., 0.], + [0., 522., 0., 0., 0., 528., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 474., 0., 0., 0., 480., 0.], + [0., 498., 0., 0., 0., 504., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 498., 0., 0., 0., 504., 0.], + [0., 522., 0., 0., 0., 528., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + [ + [0., 255., 0., 0., 0., 258., 0.], + [0., 267., 0., 0., 0., 270., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 267., 0., 0., 0., 270., 0.], + [0., 279., 0., 0., 0., 282., 0.], + [0., 0., 0., 0., 0., 0., 0.], + ], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [ + [0., 256., 272., 0.], + [0., 624., 656., 0.], + [0., 368., 384., 0.], + ], + [ + [0., 424., 440., 0.], + [0., 960., 992., 0.], + [0., 536., 552., 0.], + ], + ], + [ + [ + [0., 1096., 1112., 0.], + [0., 2304., 2336., 0.], + [0., 1208., 1224., 0.], + ], + [ + [0., 1264., 1280., 0.], + [0., 2640., 2672., 0.], + [0., 1376., 1392., 0.], + ], + ], + ], + [ + [ + [ + [0., 256., 272., 0.], + [0., 624., 656., 0.], + [0., 368., 384., 0.], + ], + [ + [0., 424., 440., 0.], + [0., 960., 992., 0.], + [0., 536., 552., 0.], + ], + ], + [ + [ + [0., 1096., 1112., 0.], + [0., 2304., 2336., 0.], + [0., 1208., 1224., 0.], + ], + [ + [0., 1264., 1280., 0.], + [0., 2640., 2672., 0.], + [0., 1376., 1392., 0.], + ], + ], + ], + [ + [ + [ + [0., 256., 272., 0.], + [0., 624., 656., 0.], + [0., 368., 384., 0.], + ], + [ + [0., 424., 440., 0.], + [0., 960., 992., 0.], + [0., 536., 552., 0.], + ], + ], + [ + [ + [0., 1096., 1112., 0.], + [0., 2304., 2336., 0.], + [0., 1208., 1224., 0.], + ], + [ + [0., 1264., 1280., 0.], + [0., 2640., 2672., 0.], + [0., 1376., 1392., 0.], + ], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([10., 10., 10.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv3d_groups_stride_2_no_pad() { + let test = Conv3dTestCase { + batch_size: 1, + channels_in: 4, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 0, + padding_2: 0, + padding_3: 0, + stride_1: 2, + stride_2: 2, + stride_3: 2, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 2, + depth: 4, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [ + [0., 1., 2., 0.], + [3., 4., 5., 0.], + [6., 7., 8., 0.], + [0., 0., 0., 0.], + ], + [ + [9., 10., 11., 0.], + [12., 13., 14., 0.], + [15., 16., 17., 0.], + [0., 0., 0., 0.], + ], + [ + [18., 19., 20., 0.], + [21., 22., 23., 0.], + [24., 25., 26., 0.], + [0., 0., 0., 0.], + ], + [ + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + ], + ], + [ + [ + [27., 28., 29., 0.], + [30., 31., 32., 0.], + [33., 34., 35., 0.], + [0., 0., 0., 0.], + ], + [ + [36., 37., 38., 0.], + [39., 40., 41., 0.], + [42., 43., 44., 0.], + [0., 0., 0., 0.], + ], + [ + [45., 46., 47., 0.], + [48., 49., 50., 0.], + [51., 52., 53., 0.], + [0., 0., 0., 0.], + ], + [ + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + ], + ], + [ + [ + [54., 55., 56., 0.], + [57., 58., 59., 0.], + [60., 61., 62., 0.], + [0., 0., 0., 0.], + ], + [ + [63., 64., 65., 0.], + [66., 67., 68., 0.], + [69., 70., 71., 0.], + [0., 0., 0., 0.], + ], + [ + [72., 73., 74., 0.], + [75., 76., 77., 0.], + [78., 79., 80., 0.], + [0., 0., 0., 0.], + ], + [ + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + ], + ], + [ + [ + [81., 82., 83., 0.], + [84., 85., 86., 0.], + [87., 88., 89., 0.], + [0., 0., 0., 0.], + ], + [ + [90., 91., 92., 0.], + [93., 94., 95., 0.], + [96., 97., 98., 0.], + [0., 0., 0., 0.], + ], + [ + [99., 100., 101., 0.], + [102., 103., 104., 0.], + [105., 106., 107., 0.], + [0., 0., 0., 0.], + ], + [ + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + ], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [[0., 1., 2.], [4., 5., 6.], [8., 9., 10.]], + [[16., 17., 18.], [20., 21., 22.], [24., 25., 26.]], + [[32., 33., 34.], [36., 37., 38.], [40., 41., 42.]], + ], + [ + [[64., 65., 66.], [68., 69., 70.], [72., 73., 74.]], + [[80., 81., 82.], [84., 85., 86.], [88., 89., 90.]], + [[96., 97., 98.], [100., 101., 102.], [104., 105., 106.]], + ], + ], + [ + [ + [[128., 129., 130.], [132., 133., 134.], [136., 137., 138.]], + [[144., 145., 146.], [148., 149., 150.], [152., 153., 154.]], + [[160., 161., 162.], [164., 165., 166.], [168., 169., 170.]], + ], + [ + [[192., 193., 194.], [196., 197., 198.], [200., 201., 202.]], + [[208., 209., 210.], [212., 213., 214.], [216., 217., 218.]], + [[224., 225., 226.], [228., 229., 230.], [232., 233., 234.]], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([1., 1.], &device), + }; + test.assert_grads(grads); +} + +struct Conv3dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + kernel_size_3: usize, + padding_1: usize, + padding_2: usize, + padding_3: usize, + stride_1: usize, + stride_2: usize, + stride_3: usize, + dilation_1: usize, + dilation_2: usize, + dilation_3: usize, + groups: usize, + depth: usize, + height: usize, + width: usize, +} + +struct Grads { + x: TestTensor<5>, + weight: TestTensor<5>, + bias: TestTensor<1>, +} + +impl Conv3dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let shape_x = Shape::new([ + self.batch_size, + self.channels_in, + self.depth, + self.height, + self.width, + ]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size_1, + self.kernel_size_2, + self.kernel_size_3, + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<5, _>(shape_weight) + .into_data(), + &device, + ) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<5, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = conv3d( + x.clone(), + weight.clone(), + Some(bias.clone()), + ConvOptions::new( + [self.stride_1, self.stride_2, self.stride_3], + [self.padding_1, self.padding_2, self.padding_3], + [self.dilation_1, self.dilation_2, self.dilation_3], + self.groups, + ), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + let tolerance = Tolerance::default(); + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), tolerance); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), tolerance); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose1d.rs new file mode 100644 index 0000000..28cb8bb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose1d.rs @@ -0,0 +1,292 @@ +use super::*; +use burn_tensor::{Shape, Tolerance, module::conv_transpose1d, ops::ConvTransposeOptions}; + +#[test] +fn test_conv_transpose1d_basic() { + let test = ConvTranspose1dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: 3, + padding: 0, + padding_out: 0, + stride: 1, + dilation: 1, + groups: 1, + size: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[15.0, 15.0, 15.0, 15.0], [51.0, 51.0, 51.0, 51.0]], + [[15.0, 15.0, 15.0, 15.0], [51.0, 51.0, 51.0, 51.0]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[44.0, 44.0, 44.0], [44.0, 44.0, 44.0]], + [[76.0, 76.0, 76.0], [76.0, 76.0, 76.0]], + ], + &device, + ), + bias: TestTensor::from_floats([12., 12.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose1d_padding() { + let test = ConvTranspose1dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: 3, + padding: 2, + padding_out: 0, + stride: 1, + dilation: 1, + groups: 1, + size: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[7., 12., 8., 3.], [19., 36., 32., 15.]], + [[7., 12., 8., 3.], [19., 36., 32., 15.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[26., 22., 18.], [26., 22., 18.]], + [[42., 38., 34.], [42., 38., 34.]], + ], + &device, + ), + bias: TestTensor::from_floats([4., 4.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose1d_stride() { + let test = ConvTranspose1dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: 3, + padding: 0, + padding_out: 0, + stride: 2, + dilation: 1, + groups: 1, + size: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[15., 15., 15., 15.], [51., 51., 51., 51.]], + [[15., 15., 15., 15.], [51., 51., 51., 51.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[44., 44., 44.], [44., 44., 44.]], + [[76., 76., 76.], [76., 76., 76.]], + ], + &device, + ), + bias: TestTensor::from_floats([18., 18.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose1d_stride_padding_out() { + let test = ConvTranspose1dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: 3, + padding: 0, + padding_out: 1, + stride: 2, + dilation: 1, + groups: 1, + size: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[15., 15., 15., 15.], [51., 51., 51., 51.]], + [[15., 15., 15., 15.], [51., 51., 51., 51.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[44., 44., 44.], [44., 44., 44.]], + [[76., 76., 76.], [76., 76., 76.]], + ], + &device, + ), + bias: TestTensor::from_floats([20., 20.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose1d_dilation() { + let test = ConvTranspose1dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: 3, + padding: 0, + padding_out: 0, + stride: 1, + dilation: 2, + groups: 1, + size: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [[15., 15., 15., 15.], [51., 51., 51., 51.]], + [[15., 15., 15., 15.], [51., 51., 51., 51.]], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[44., 44., 44.], [44., 44., 44.]], + [[76., 76., 76.], [76., 76., 76.]], + ], + &device, + ), + bias: TestTensor::from_floats([16., 16.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose1d_complex() { + let test = ConvTranspose1dTestCase { + batch_size: 2, + channels: [2, 4], + kernel_size: 3, + padding: 1, + padding_out: 1, + stride: 2, + dilation: 2, + groups: 2, + size: 8, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [12.0, 15.0, 15.0, 15.0, 15.0, 15.0, 15.0, 15.0], + [36.0, 51.0, 51.0, 51.0, 51.0, 51.0, 51.0, 51.0], + ], + [ + [12.0, 15.0, 15.0, 15.0, 15.0, 15.0, 15.0, 15.0], + [36.0, 51.0, 51.0, 51.0, 51.0, 51.0, 51.0, 51.0], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [[168.0, 184.0, 184.0], [168.0, 184.0, 184.0]], + [[280.0, 312.0, 312.0], [280.0, 312.0, 312.0]], + ], + &device, + ), + bias: TestTensor::from_floats([36.0, 36.0, 36.0, 36.0], &device), + }; + test.assert_grads(grads); +} + +struct ConvTranspose1dTestCase { + batch_size: usize, + channels: [usize; 2], + kernel_size: usize, + padding: usize, + padding_out: usize, + stride: usize, + dilation: usize, + groups: usize, + size: usize, +} + +struct Grads { + x: TestTensor<3>, + weight: TestTensor<3>, + bias: TestTensor<1>, +} + +impl ConvTranspose1dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let shape_x = Shape::new([self.batch_size, self.channels[0], self.size]); + let shape_weight = Shape::new([ + self.channels[0], + self.channels[1] / self.groups, + self.kernel_size, + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<3, _>(shape_weight) + .into_data(), + &device, + ) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels[1] as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = conv_transpose1d( + x.clone(), + weight.clone(), + Some(bias.clone()), + ConvTransposeOptions::new( + [self.stride], + [self.padding], + [self.padding_out], + [self.dilation], + self.groups, + ), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), Tolerance::default()); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose2d.rs new file mode 100644 index 0000000..795cf10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose2d.rs @@ -0,0 +1,706 @@ +use super::*; +use burn_tensor::{Shape, Tolerance, module::conv_transpose2d, ops::ConvTransposeOptions}; + +#[test] +fn test_conv_transpose2d_basic() { + let test = ConvTranspose2dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: [3, 3], + padding: [0, 0], + padding_out: [0, 0], + stride: [1, 1], + dilation: [1, 1], + groups: 1, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [153., 153., 153., 153.], + [153., 153., 153., 153.], + [153., 153., 153., 153.], + [153., 153., 153., 153.], + ], + [ + [477., 477., 477., 477.], + [477., 477., 477., 477.], + [477., 477., 477., 477.], + [477., 477., 477., 477.], + ], + ], + [ + [ + [153., 153., 153., 153.], + [153., 153., 153., 153.], + [153., 153., 153., 153.], + [153., 153., 153., 153.], + ], + [ + [477., 477., 477., 477.], + [477., 477., 477., 477.], + [477., 477., 477., 477.], + [477., 477., 477., 477.], + ], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[752., 752., 752.], [752., 752., 752.], [752., 752., 752.]], + [[752., 752., 752.], [752., 752., 752.], [752., 752., 752.]], + ], + [ + [ + [1264., 1264., 1264.], + [1264., 1264., 1264.], + [1264., 1264., 1264.], + ], + [ + [1264., 1264., 1264.], + [1264., 1264., 1264.], + [1264., 1264., 1264.], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([72., 72.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_padding() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [1, 1], + kernel_size: [3, 3], + padding: [1, 2], + padding_out: [0, 0], + stride: [1, 1], + dilation: [1, 1], + groups: 1, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[[ + [13., 24., 20., 9.], + [15., 27., 21., 9.], + [15., 27., 21., 9.], + [7., 12., 8., 3.], + ]]], + &device, + ), + weight: TestTensor::from_floats( + [[[[63., 57., 51.], [68., 60., 52.], [39., 33., 27.]]]], + &device, + ), + bias: TestTensor::from_floats([8.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_stride() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [1, 1], + kernel_size: [3, 3], + padding: [0, 0], + padding_out: [0, 0], + stride: [2, 3], + dilation: [1, 1], + groups: 1, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[[ + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + ]]], + &device, + ), + weight: TestTensor::from_floats( + [[[[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]]]], + &device, + ), + bias: TestTensor::from_floats([108.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_stride_padding_out() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [1, 1], + kernel_size: [3, 3], + padding: [0, 0], + padding_out: [1, 2], + stride: [2, 3], + dilation: [1, 1], + groups: 1, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[[ + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + ]]], + &device, + ), + weight: TestTensor::from_floats( + [[[[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]]]], + &device, + ), + bias: TestTensor::from_floats([140.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_dilation() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [1, 1], + kernel_size: [3, 3], + padding: [0, 0], + padding_out: [0, 0], + stride: [1, 1], + dilation: [2, 3], + groups: 1, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[[ + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + ]]], + &device, + ), + weight: TestTensor::from_floats( + [[[[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]]]], + &device, + ), + bias: TestTensor::from_floats([80.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_channels() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [2, 3], + kernel_size: [3, 3], + padding: [0, 0], + padding_out: [0, 0], + stride: [1, 1], + dilation: [1, 1], + groups: 1, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [351., 351., 351., 351.], + [351., 351., 351., 351.], + [351., 351., 351., 351.], + [351., 351., 351., 351.], + ], + [ + [1080., 1080., 1080., 1080.], + [1080., 1080., 1080., 1080.], + [1080., 1080., 1080., 1080.], + [1080., 1080., 1080., 1080.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]], + [[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]], + [[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]], + ], + [ + [[376., 376., 376.], [376., 376., 376.], [376., 376., 376.]], + [[376., 376., 376.], [376., 376., 376.], [376., 376., 376.]], + [[376., 376., 376.], [376., 376., 376.], [376., 376., 376.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([36., 36., 36.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_kernel_size() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [1, 1], + kernel_size: [3, 5], + padding: [0, 0], + padding_out: [0, 0], + stride: [1, 1], + dilation: [1, 1], + groups: 1, + size: [6, 6], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[[ + [105., 105., 105., 105., 105., 105.], + [105., 105., 105., 105., 105., 105.], + [105., 105., 105., 105., 105., 105.], + [105., 105., 105., 105., 105., 105.], + [105., 105., 105., 105., 105., 105.], + [105., 105., 105., 105., 105., 105.], + ]]], + &device, + ), + weight: TestTensor::from_floats( + [[[ + [630., 630., 630., 630., 630.], + [630., 630., 630., 630., 630.], + [630., 630., 630., 630., 630.], + ]]], + &device, + ), + bias: TestTensor::from_floats([80.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_groups() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [2, 2], + kernel_size: [3, 3], + padding: [0, 0], + padding_out: [0, 0], + stride: [1, 1], + dilation: [1, 1], + groups: 2, + size: [4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + [36., 36., 36., 36.], + ], + [ + [117., 117., 117., 117.], + [117., 117., 117., 117.], + [117., 117., 117., 117.], + [117., 117., 117., 117.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [[[120., 120., 120.], [120., 120., 120.], [120., 120., 120.]]], + [[[376., 376., 376.], [376., 376., 376.], [376., 376., 376.]]], + ], + &device, + ), + bias: TestTensor::from_floats([36., 36.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_complex_no_groups() { + let test = ConvTranspose2dTestCase { + batch_size: 2, + channels: [2, 3], + kernel_size: [3, 5], + padding: [1, 2], + padding_out: [1, 2], + stride: [2, 3], + dilation: [2, 3], + groups: 1, + size: [6, 8], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [600., 735., 735., 735., 735., 735., 735., 735.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + ], + [ + [1680., 2085., 2085., 2085., 2085., 2085., 2085., 2085.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + ], + ], + [ + [ + [600., 735., 735., 735., 735., 735., 735., 735.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + [810., 990., 990., 990., 990., 990., 990., 990.], + ], + [ + [1680., 2085., 2085., 2085., 2085., 2085., 2085., 2085.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + [2430., 3015., 3015., 3015., 3015., 3015., 3015., 3015.], + ], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [5320., 6040., 6040., 6040., 6040.], + [6048., 6864., 6864., 6864., 6864.], + [6048., 6864., 6864., 6864., 6864.], + ], + [ + [5320., 6040., 6040., 6040., 6040.], + [6048., 6864., 6864., 6864., 6864.], + [6048., 6864., 6864., 6864., 6864.], + ], + [ + [5320., 6040., 6040., 6040., 6040.], + [6048., 6864., 6864., 6864., 6864.], + [6048., 6864., 6864., 6864., 6864.], + ], + ], + [ + [ + [8680., 9880., 9880., 9880., 9880.], + [10080., 11472., 11472., 11472., 11472.], + [10080., 11472., 11472., 11472., 11472.], + ], + [ + [8680., 9880., 9880., 9880., 9880.], + [10080., 11472., 11472., 11472., 11472.], + [10080., 11472., 11472., 11472., 11472.], + ], + [ + [8680., 9880., 9880., 9880., 9880.], + [10080., 11472., 11472., 11472., 11472.], + [10080., 11472., 11472., 11472., 11472.], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([896., 896., 896.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_complex_no_groups_2() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [4, 2], + kernel_size: [2, 3], + padding: [1, 2], + padding_out: [1, 2], + stride: [2, 3], + dilation: [1, 2], + groups: 1, + size: [10, 10], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [30., 42., 42., 42., 42., 42., 42., 42., 42., 42.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [48., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + ], + [ + [78., 114., 114., 114., 114., 114., 114., 114., 114., 114.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + [144., 210., 210., 210., 210., 210., 210., 210., 210., 210.], + ], + [ + [126., 186., 186., 186., 186., 186., 186., 186., 186., 186.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + [240., 354., 354., 354., 354., 354., 354., 354., 354., 354.], + ], + [ + [174., 258., 258., 258., 258., 258., 258., 258., 258., 258.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + [336., 498., 498., 498., 498., 498., 498., 498., 498., 498.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [[4455., 4905., 4905.], [4500., 4950., 4950.]], + [[4455., 4905., 4905.], [4500., 4950., 4950.]], + ], + [ + [[12555., 13905., 13905.], [13500., 14950., 14950.]], + [[12555., 13905., 13905.], [13500., 14950., 14950.]], + ], + [ + [[20655., 22905., 22905.], [22500., 24950., 24950.]], + [[20655., 22905., 22905.], [22500., 24950., 24950.]], + ], + [ + [[28755., 31905., 31905.], [31500., 34950., 34950.]], + [[28755., 31905., 31905.], [31500., 34950., 34950.]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([570., 570.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose2d_complex_groups() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels: [4, 2], + kernel_size: [2, 3], + padding: [1, 2], + padding_out: [1, 2], + stride: [2, 3], + dilation: [1, 2], + groups: 2, + size: [10, 10], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [9., 12., 12., 12., 12., 12., 12., 12., 12., 12.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + [12., 15., 15., 15., 15., 15., 15., 15., 15., 15.], + ], + [ + [21., 30., 30., 30., 30., 30., 30., 30., 30., 30.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + [36., 51., 51., 51., 51., 51., 51., 51., 51., 51.], + ], + [ + [33., 48., 48., 48., 48., 48., 48., 48., 48., 48.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + [60., 87., 87., 87., 87., 87., 87., 87., 87., 87.], + ], + [ + [45., 66., 66., 66., 66., 66., 66., 66., 66., 66.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + [84., 123., 123., 123., 123., 123., 123., 123., 123., 123.], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [[[4455., 4905., 4905.], [4500., 4950., 4950.]]], + [[[12555., 13905., 13905.], [13500., 14950., 14950.]]], + [[[20655., 22905., 22905.], [22500., 24950., 24950.]]], + [[[28755., 31905., 31905.], [31500., 34950., 34950.]]], + ], + &device, + ), + bias: TestTensor::from_floats([570., 570.], &device), + }; + test.assert_grads(grads); +} + +struct ConvTranspose2dTestCase { + batch_size: usize, + channels: [usize; 2], + kernel_size: [usize; 2], + padding: [usize; 2], + padding_out: [usize; 2], + stride: [usize; 2], + dilation: [usize; 2], + groups: usize, + size: [usize; 2], +} + +struct Grads { + x: TestTensor<4>, + weight: TestTensor<4>, + bias: TestTensor<1>, +} + +impl ConvTranspose2dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let shape_x = Shape::new([ + self.batch_size, + self.channels[0], + self.size[0], + self.size[1], + ]); + let shape_weight = Shape::new([ + self.channels[0], + self.channels[1] / self.groups, + self.kernel_size[0], + self.kernel_size[1], + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<4, _>(shape_weight) + .into_data(), + &device, + ) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels[1] as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let output = conv_transpose2d( + x.clone(), + weight.clone(), + Some(bias.clone()), + ConvTransposeOptions::new( + self.stride, + self.padding, + self.padding_out, + self.dilation, + self.groups, + ), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + let tolerance = Tolerance::permissive(); + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), tolerance); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), tolerance); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose3d.rs new file mode 100644 index 0000000..ed6ac9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/conv_transpose3d.rs @@ -0,0 +1,711 @@ +use super::*; +use burn_tensor::{Shape, Tolerance, module::conv_transpose3d, ops::ConvTransposeOptions}; + +#[test] +fn test_conv_transpose3d_basic() { + let test = ConvTranspose3dTestCase { + batch_size: 2, + channels: [2, 2], + kernel_size: [3, 3, 3], + padding: [0, 0, 0], + padding_out: [0, 0, 0], + stride: [1, 1, 1], + dilation: [1, 1, 1], + groups: 1, + size: [4, 4, 4], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + ], + [ + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + ], + ], + [ + [ + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + [ + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + [13.250001, 13.250001, 13.250001, 13.250001], + ], + ], + [ + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + [ + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + [40.249992, 40.249992, 40.249992, 40.249992], + ], + ], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [ + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + ], + [ + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + ], + [ + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + ], + ], + [ + [ + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + ], + [ + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + ], + [ + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + [47.750000, 47.750000, 47.750000], + ], + ], + ], + [ + [ + [ + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + ], + [ + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + ], + [ + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + ], + ], + [ + [ + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + ], + [ + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + ], + [ + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + [79.750000, 79.750000, 79.750000], + ], + ], + ], + ], + &device, + ), + bias: TestTensor::from_floats([432., 432.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_conv_transpose3d_complex_groups() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels: [4, 2], + kernel_size: [2, 3, 4], + padding: [1, 2, 3], + padding_out: [1, 2, 3], + stride: [2, 3, 4], + dilation: [1, 2, 3], + groups: 2, + size: [6, 6, 6], + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [ + [1.250000, 1.625000, 1.625000, 1.625000, 1.625000, 1.625000], + [1.687500, 2.187500, 2.187500, 2.187500, 2.187500, 2.187500], + [1.687500, 2.187500, 2.187500, 2.187500, 2.187500, 2.187500], + [1.687500, 2.187500, 2.187500, 2.187500, 2.187500, 2.187500], + [1.687500, 2.187500, 2.187500, 2.187500, 2.187500, 2.187500], + [1.687500, 2.187500, 2.187500, 2.187500, 2.187500, 2.187500], + ], + [ + [1.750000, 2.250000, 2.250000, 2.250000, 2.250000, 2.250000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + ], + [ + [1.750000, 2.250000, 2.250000, 2.250000, 2.250000, 2.250000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + ], + [ + [1.750000, 2.250000, 2.250000, 2.250000, 2.250000, 2.250000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + ], + [ + [1.750000, 2.250000, 2.250000, 2.250000, 2.250000, 2.250000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + ], + [ + [1.750000, 2.250000, 2.250000, 2.250000, 2.250000, 2.250000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + [2.250000, 2.875000, 2.875000, 2.875000, 2.875000, 2.875000], + ], + ], + [ + [ + [2.750000, 3.625000, 3.625000, 3.625000, 3.625000, 3.625000], + [3.937500, 5.187500, 5.187500, 5.187500, 5.187500, 5.187500], + [3.937500, 5.187500, 5.187500, 5.187500, 5.187500, 5.187500], + [3.937500, 5.187500, 5.187500, 5.187500, 5.187500, 5.187500], + [3.937500, 5.187500, 5.187500, 5.187500, 5.187500, 5.187500], + [3.937500, 5.187500, 5.187500, 5.187500, 5.187500, 5.187500], + ], + [ + [4.750000, 6.250000, 6.250000, 6.250000, 6.250000, 6.250000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + ], + [ + [4.750000, 6.250000, 6.250000, 6.250000, 6.250000, 6.250000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + ], + [ + [4.750000, 6.250000, 6.250000, 6.250000, 6.250000, 6.250000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + ], + [ + [4.750000, 6.250000, 6.250000, 6.250000, 6.250000, 6.250000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + ], + [ + [4.750000, 6.250000, 6.250000, 6.250000, 6.250000, 6.250000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + [6.750000, 8.875000, 8.875000, 8.875000, 8.875000, 8.875000], + ], + ], + [ + [ + [4.250000, 5.625000, 5.625000, 5.625000, 5.625000, 5.625000], + [6.187500, 8.187500, 8.187500, 8.187500, 8.187500, 8.187500], + [6.187500, 8.187500, 8.187500, 8.187500, 8.187500, 8.187500], + [6.187500, 8.187500, 8.187500, 8.187500, 8.187500, 8.187500], + [6.187500, 8.187500, 8.187500, 8.187500, 8.187500, 8.187500], + [6.187500, 8.187500, 8.187500, 8.187500, 8.187500, 8.187500], + ], + [ + [ + 7.750000, 10.250000, 10.250000, 10.250000, 10.250000, 10.250000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + ], + [ + [ + 7.750000, 10.250000, 10.250000, 10.250000, 10.250000, 10.250000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + ], + [ + [ + 7.750000, 10.250000, 10.250000, 10.250000, 10.250000, 10.250000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + ], + [ + [ + 7.750000, 10.250000, 10.250000, 10.250000, 10.250000, 10.250000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + ], + [ + [ + 7.750000, 10.250000, 10.250000, 10.250000, 10.250000, 10.250000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + [ + 11.250000, 14.875000, 14.875000, 14.875000, 14.875000, 14.875000, + ], + ], + ], + [ + [ + [5.750000, 7.625000, 7.625000, 7.625000, 7.625000, 7.625000], + [ + 8.437500, 11.187500, 11.187500, 11.187500, 11.187500, 11.187500, + ], + [ + 8.437500, 11.187500, 11.187500, 11.187500, 11.187500, 11.187500, + ], + [ + 8.437500, 11.187500, 11.187500, 11.187500, 11.187500, 11.187500, + ], + [ + 8.437500, 11.187500, 11.187500, 11.187500, 11.187500, 11.187500, + ], + [ + 8.437500, 11.187500, 11.187500, 11.187500, 11.187500, 11.187500, + ], + ], + [ + [ + 10.750000, 14.250000, 14.250000, 14.250000, 14.250000, 14.250000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + ], + [ + [ + 10.750000, 14.250000, 14.250000, 14.250000, 14.250000, 14.250000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + ], + [ + [ + 10.750000, 14.250000, 14.250000, 14.250000, 14.250000, 14.250000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + ], + [ + [ + 10.750000, 14.250000, 14.250000, 14.250000, 14.250000, 14.250000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + ], + [ + [ + 10.750000, 14.250000, 14.250000, 14.250000, 14.250000, 14.250000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + [ + 15.750000, 20.875000, 20.875000, 20.875000, 20.875000, 20.875000, + ], + ], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [[ + [ + [18.663193, 22.309027, 22.309027, 22.309027], + [21.875000, 26.145834, 26.145834, 26.145834], + [21.875000, 26.145834, 26.145834, 26.145834], + ], + [ + [19.270832, 23.020834, 23.020834, 23.020834], + [22.500000, 26.875002, 26.875002, 26.875002], + [22.500000, 26.875002, 26.875002, 26.875002], + ], + ]], + [[ + [ + [49.913193, 59.809029, 59.809029, 59.809029], + [59.375000, 71.145836, 71.145836, 71.145836], + [59.375000, 71.145836, 71.145836, 71.145836], + ], + [ + [56.770836, 68.020836, 68.020836, 68.020836], + [67.500000, 80.875000, 80.875000, 80.875000], + [67.500000, 80.875000, 80.875000, 80.875000], + ], + ]], + [[ + [ + [81.163193, 97.309029, 97.309029, 97.309029], + [96.875000, 116.145828, 116.145828, 116.145828], + [96.875000, 116.145828, 116.145828, 116.145828], + ], + [ + [94.270828, 113.020828, 113.020828, 113.020828], + [112.500000, 134.875000, 134.875000, 134.875000], + [112.500000, 134.875000, 134.875000, 134.875000], + ], + ]], + [[ + [ + [112.413200, 134.809021, 134.809021, 134.809021], + [134.375000, 161.145828, 161.145828, 161.145828], + [134.375000, 161.145828, 161.145828, 161.145828], + ], + [ + [131.770844, 158.020828, 158.020828, 158.020828], + [157.500000, 188.875000, 188.875000, 188.875000], + [157.500000, 188.875000, 188.875000, 188.875000], + ], + ]], + ], + &device, + ), + bias: TestTensor::from_floats([5346., 5346.], &device), + }; + test.assert_grads(grads); +} + +struct ConvTranspose3dTestCase { + batch_size: usize, + channels: [usize; 2], + kernel_size: [usize; 3], + padding: [usize; 3], + padding_out: [usize; 3], + stride: [usize; 3], + dilation: [usize; 3], + groups: usize, + size: [usize; 3], +} + +struct Grads { + x: TestTensor<5>, + weight: TestTensor<5>, + bias: TestTensor<1>, +} + +impl ConvTranspose3dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let shape_x = Shape::new([ + self.batch_size, + self.channels[0], + self.size[0], + self.size[1], + self.size[2], + ]); + let shape_weight = Shape::new([ + self.channels[0], + self.channels[1] / self.groups, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<5, _>(shape_weight.clone()) + .into_data(), + &device, + ) + .div_scalar(shape_weight.num_elements() as f32) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels[1] as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<5, _>(shape_x.clone()) + .into_data(), + &device, + ) + .div_scalar(shape_x.num_elements() as f32) + .require_grad(); + let output = conv_transpose3d( + x.clone(), + weight.clone(), + Some(bias.clone()), + ConvTransposeOptions::new( + self.stride, + self.padding, + self.padding_out, + self.dilation, + self.groups, + ), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + let tolerance = Tolerance::permissive(); + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), tolerance); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), tolerance); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross.rs new file mode 100644 index 0000000..6accb00 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross.rs @@ -0,0 +1,103 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[cfg(feature = "std")] +use burn_backend_tests::might_panic; + +#[test] +fn backward_basic() { + let device = Default::default(); + let a = TestAutodiffTensor::<2>::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ) + .require_grad(); + let b = TestAutodiffTensor::<2>::from_data( + TensorData::from([[4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]), + &device, + ) + .require_grad(); + + // Simple cross product; grad is a vector of ones. + let c = a.clone().cross(b.clone(), 1); + let grads = c.backward(); + + let a_grad = a.grad(&grads).unwrap().to_data(); + let b_grad = b.grad(&grads).unwrap().to_data(); + + // For a: b×grad_out, where grad_out = [1,1,1] + let expected_a = TensorData::from([[-1.0, 2.0, -1.0], [-1.0, 2.0, -1.0]]); + // For b: grad_out×a + let expected_b = TensorData::from([[1.0, -2.0, 1.0], [1.0, -2.0, 1.0]]); + + a_grad.assert_approx_eq::(&expected_a, Tolerance::default()); + b_grad.assert_approx_eq::(&expected_b, Tolerance::default()); +} + +#[test] +fn backward_after_sum() { + let device = Default::default(); + let a = TestAutodiffTensor::<2>::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ) + .require_grad(); + let b = TestAutodiffTensor::<2>::from_data( + TensorData::from([[4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]), + &device, + ) + .require_grad(); + + // Sum reduces to scalar, but the gradient should be the same. + let c = a.clone().cross(b.clone(), 1).sum(); + let grads = c.backward(); + + let a_grad = a.grad(&grads).unwrap().to_data(); + let b_grad = b.grad(&grads).unwrap().to_data(); + + let expected_a = TensorData::from([[-1.0, 2.0, -1.0], [-1.0, 2.0, -1.0]]); + let expected_b = TensorData::from([[1.0, -2.0, 1.0], [1.0, -2.0, 1.0]]); + + a_grad.assert_approx_eq::(&expected_a, Tolerance::default()); + b_grad.assert_approx_eq::(&expected_b, Tolerance::default()); +} + +#[cfg(feature = "std")] +#[might_panic(reason = "not implemented: Cross product on non-last dimension")] +#[test] +fn different_dim() { + // Also check when the cross is along a different dimension (e.g. dim 0). + let device = Default::default(); + let a_raw = [[1.0, 4.0, 7.0], [2.0, 5.0, 8.0], [3.0, 6.0, 9.0]]; + let b_raw = [[9.0, 6.0, 3.0], [8.0, 5.0, 2.0], [7.0, 4.0, 1.0]]; + + let a = TestTensor::<2>::from_data(TensorData::from(a_raw), &device); + let b = TestTensor::<2>::from_data(TensorData::from(b_raw), &device); + // Cross along dim 0. Some backends (for example CubeCL) may not support + // cross on non-last dimensions and will intentionally panic with a + // message like "Cross product on non-last dimension not yet implemented". + // In that case we treat the panic as a skipped test for that backend. + let out = a.cross(b.clone(), 0); + + // Manually compute cross of each column vector using raw arrays + let expected = [ + [ + a_raw[1][0] * b_raw[2][0] - a_raw[2][0] * b_raw[1][0], + a_raw[1][1] * b_raw[2][1] - a_raw[2][1] * b_raw[1][1], + a_raw[1][2] * b_raw[2][2] - a_raw[2][2] * b_raw[1][2], + ], + [ + a_raw[2][0] * b_raw[0][0] - a_raw[0][0] * b_raw[2][0], + a_raw[2][1] * b_raw[0][1] - a_raw[0][1] * b_raw[2][1], + a_raw[2][2] * b_raw[0][2] - a_raw[0][2] * b_raw[2][2], + ], + [ + a_raw[0][0] * b_raw[1][0] - a_raw[1][0] * b_raw[0][0], + a_raw[0][1] * b_raw[1][1] - a_raw[1][1] * b_raw[0][1], + a_raw[0][2] * b_raw[1][2] - a_raw[1][2] * b_raw[0][2], + ], + ]; + + out.to_data() + .assert_approx_eq::(&TensorData::from(expected), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross_entropy.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross_entropy.rs new file mode 100644 index 0000000..b734146 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cross_entropy.rs @@ -0,0 +1,33 @@ +use super::*; +use burn_tensor::{Tensor, TensorData, Tolerance, loss}; + +#[test] +fn test_cross_entropy_loss_grad() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + let data_targets = TensorData::from([[0.8, 0.2], [0.9, 0.1]]); + + let device = Default::default(); + let tensor_1 = Tensor::::from_data(data_1, &device).require_grad(); + let tensor_2 = Tensor::::from_data(data_2, &device).require_grad(); + let tensor_targets = + Tensor::::from_data(data_targets, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = loss::cross_entropy_with_logits(tensor_3, tensor_targets); + + let grads = tensor_4.backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::permissive(); + let expected = TensorData::from([[0.26553, 0.26553], [0.44954, 0.44954]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[-1.34863, 1.34863], [-2.06371, 2.06371]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummax.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummax.rs new file mode 100644 index 0000000..889d033 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummax.rs @@ -0,0 +1,117 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_cummax() { + // Simple test to verify cummax gradients work + let device = Default::default(); + let tensor = TestAutodiffTensor::<1>::from_data(TensorData::from([1.0, 3.0, 2.0]), &device) + .require_grad(); + + let output = tensor.clone().cummax(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 2.0, 0.0] + let expected = TensorData::from([1.0, 2.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummax_2d() { + // Test 2D cummax gradients + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data( + TensorData::from([[1.0, 3.0, 2.0], [2.0, 5.0, 4.0]]), + &device, + ) + .require_grad(); + + let output = tensor.clone().cummax(1); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [[1.0, 2.0, 0.0], [1.0, 2.0, 0.0]] + let expected = TensorData::from([[1.0, 2.0, 0.0], [1.0, 2.0, 0.0]]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummax_duplicate_values() { + // Test with duplicate maximum values - critical edge case + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([1.0, 3.0, 3.0, 2.0]), &device) + .require_grad(); + + let output = tensor.clone().cummax(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // input: [1.0, 3.0, 3.0, 2.0] + // cummax: [1.0, 3.0, 3.0, 3.0] + // PyTorch reference: [1.0, 1.0, 2.0, 0.0] + // Position 2 gets grad from itself + position 3 + let expected = TensorData::from([1.0, 1.0, 2.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummax_all_same() { + // Test with all same values + let device = Default::default(); + let tensor = TestAutodiffTensor::<1>::from_data(TensorData::from([2.0, 2.0, 2.0]), &device) + .require_grad(); + + let output = tensor.clone().cummax(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 1.0, 1.0] + // Each position matches cummax, so each gets its own gradient + let expected = TensorData::from([1.0, 1.0, 1.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummax_increasing() { + // Test with increasing sequence + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([1.0, 2.0, 3.0, 4.0]), &device) + .require_grad(); + + let output = tensor.clone().cummax(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 1.0, 1.0, 1.0] + // Each position is a new maximum + let expected = TensorData::from([1.0, 1.0, 1.0, 1.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummax_2d_duplicates() { + // Test 2D with duplicate values + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data( + TensorData::from([[1.0, 3.0, 3.0, 2.0], [2.0, 5.0, 5.0, 4.0]]), + &device, + ) + .require_grad(); + + let output = tensor.clone().cummax(1); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [[1.0, 1.0, 2.0, 0.0], [1.0, 1.0, 2.0, 0.0]] + let expected = TensorData::from([[1.0, 1.0, 2.0, 0.0], [1.0, 1.0, 2.0, 0.0]]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummin.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummin.rs new file mode 100644 index 0000000..b1fc8b0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cummin.rs @@ -0,0 +1,117 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_cummin() { + // Simple test to verify cummin gradients work + let device = Default::default(); + let tensor = TestAutodiffTensor::<1>::from_data(TensorData::from([3.0, 2.0, 4.0]), &device) + .require_grad(); + + let output = tensor.clone().cummin(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 2.0, 0.0] + let expected = TensorData::from([1.0, 2.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummin_2d() { + // Test 2D cummin gradients + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data( + TensorData::from([[3.0, 2.0, 4.0], [5.0, 1.0, 3.0]]), + &device, + ) + .require_grad(); + + let output = tensor.clone().cummin(1); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [[1.0, 2.0, 0.0], [1.0, 2.0, 0.0]] + let expected = TensorData::from([[1.0, 2.0, 0.0], [1.0, 2.0, 0.0]]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummin_duplicate_values() { + // Test with duplicate minimum values - critical edge case + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([3.0, 2.0, 2.0, 4.0]), &device) + .require_grad(); + + let output = tensor.clone().cummin(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // input: [3.0, 2.0, 2.0, 4.0] + // cummin: [3.0, 2.0, 2.0, 2.0] + // PyTorch reference: [1.0, 1.0, 2.0, 0.0] + // Position 2 gets grad from itself + position 3 + let expected = TensorData::from([1.0, 1.0, 2.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummin_all_same() { + // Test with all same values + let device = Default::default(); + let tensor = TestAutodiffTensor::<1>::from_data(TensorData::from([2.0, 2.0, 2.0]), &device) + .require_grad(); + + let output = tensor.clone().cummin(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 1.0, 1.0] + // Each position matches cummin, so each gets its own gradient + let expected = TensorData::from([1.0, 1.0, 1.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummin_decreasing() { + // Test with decreasing sequence + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([5.0, 4.0, 3.0, 2.0]), &device) + .require_grad(); + + let output = tensor.clone().cummin(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 1.0, 1.0, 1.0] + // Each position is a new minimum + let expected = TensorData::from([1.0, 1.0, 1.0, 1.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cummin_2d_duplicates() { + // Test 2D with duplicate values + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data( + TensorData::from([[3.0, 2.0, 2.0, 4.0], [5.0, 1.0, 1.0, 3.0]]), + &device, + ) + .require_grad(); + + let output = tensor.clone().cummin(1); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [[1.0, 1.0, 2.0, 0.0], [1.0, 1.0, 2.0, 0.0]] + let expected = TensorData::from([[1.0, 1.0, 2.0, 0.0], [1.0, 1.0, 2.0, 0.0]]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumprod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumprod.rs new file mode 100644 index 0000000..fb471f3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumprod.rs @@ -0,0 +1,132 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_cumprod() { + // Simple test to verify cumprod gradients work + let device = Default::default(); + let tensor = TestAutodiffTensor::<1>::from_data(TensorData::from([2.0, 3.0, 4.0]), &device) + .require_grad(); + + let output = tensor.clone().cumprod(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [16.0, 10.0, 6.0] + let expected = TensorData::from([16.0, 10.0, 6.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cumprod_2d() { + // Test 2D cumprod gradients + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ) + .require_grad(); + + let output = tensor.clone().cumprod(1); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [[9.0, 4.0, 2.0], [36.0, 28.0, 20.0]] + let expected = TensorData::from([[9.0, 4.0, 2.0], [36.0, 28.0, 20.0]]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +// TODO: The following tests are currently ignored due to a known limitation +// in the cumprod gradient implementation. The current implementation uses +// division (grad / input), which produces NaN when the input contains zeros. +// +// A proper fix requires implementing a zero-safe algorithm using exclusive +// cumulative products (similar to PyTorch's cumprod_backward or JAX's +// associative_scan approach). This is a non-trivial implementation that +// requires careful handling of cumulative products in both forward and +// reverse directions. +// +// See: https://github.com/tracel-ai/burn/issues/3864 +// +// References: +// - PyTorch: https://github.com/pytorch/pytorch (cumprod_backward) +// - JAX PR #2596: Parallel prefix scan implementation +// - TensorFlow Issue #3862: tf.cumprod's gradient produces nans given zeros + +#[test] +#[ignore = "cumprod gradient with zeros not yet implemented - produces NaN due to division by zero"] +fn should_diff_cumprod_zero_in_middle() { + // Test cumprod with zero in the middle - edge case for division + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([2.0, 0.0, 3.0, 4.0]), &device) + .require_grad(); + + let output = tensor.clone().cumprod(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 32.0, 0.0, 0.0] + let expected = TensorData::from([1.0, 32.0, 0.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[ignore = "cumprod gradient with zeros not yet implemented - produces NaN due to division by zero"] +fn should_diff_cumprod_zero_at_start() { + // Test cumprod with zero at the beginning + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([0.0, 2.0, 3.0, 4.0]), &device) + .require_grad(); + + let output = tensor.clone().cumprod(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [33.0, 0.0, 0.0, 0.0] + let expected = TensorData::from([33.0, 0.0, 0.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[ignore = "cumprod gradient with zeros not yet implemented - produces NaN due to division by zero"] +fn should_diff_cumprod_zero_at_end() { + // Test cumprod with zero at the end + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([2.0, 3.0, 4.0, 0.0]), &device) + .require_grad(); + + let output = tensor.clone().cumprod(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [16.0, 10.0, 6.0, 24.0] + let expected = TensorData::from([16.0, 10.0, 6.0, 24.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[ignore = "cumprod gradient with zeros not yet implemented - produces NaN due to division by zero"] +fn should_diff_cumprod_multiple_zeros() { + // Test cumprod with multiple zeros + let device = Default::default(); + let tensor = + TestAutodiffTensor::<1>::from_data(TensorData::from([2.0, 0.0, 3.0, 0.0, 5.0]), &device) + .require_grad(); + + let output = tensor.clone().cumprod(0); + let grads = output.sum().backward(); + let grad = tensor.grad(&grads).unwrap(); + + // PyTorch reference: [1.0, 8.0, 0.0, 0.0, 0.0] + let expected = TensorData::from([1.0, 8.0, 0.0, 0.0, 0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumsum.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumsum.rs new file mode 100644 index 0000000..288b300 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/cumsum.rs @@ -0,0 +1,89 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_cumsum_dim0() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.cumsum(0); + let tensor_5 = tensor_1.clone().mul(tensor_4); + let grads = tensor_5.sum().backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + // Expected gradients computed with PyTorch + let expected = TensorData::from([[-14.0, 24.0], [17.0, 6.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[3.0, 10.0], [-1.0, 37.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cumsum_dim1() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.cumsum(1); + let tensor_5 = tensor_1.clone().mul(tensor_4); + let grads = tensor_5.sum().backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + // Expected gradients computed with PyTorch + let expected = TensorData::from([[1.0, 69.0], [-13.0, -28.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[18.0, 13.0], [71.0, 58.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_cumsum_complex() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.clone().cumsum(1); + let tensor_5 = tensor_4.mul(tensor_3); + + let grads = tensor_5.sum().backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + // Expected gradients computed with PyTorch + let expected = TensorData::from([[371.0, 542.0], [2246.0, 3281.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[507.0, 528.0], [704.0, 733.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/deform_conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/deform_conv2d.rs new file mode 100644 index 0000000..ff0c5ae --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/deform_conv2d.rs @@ -0,0 +1,1804 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Shape, module::deform_conv2d, ops::DeformConvOptions}; + +#[test] +fn test_deform_conv2d_basic() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [0.000, 6.0678, 14.2071, 12.2477], + [11.2292, 33.7937, 50.1555, 44.0561], + [17.9294, 57.2174, 85.1505, 79.1840], + [18.0220, 73.6263, 126.8184, 151.6910], + ], + [ + [0.000, 8.9783, 20.7620, 17.7888], + [16.2326, 48.7386, 71.7961, 62.5845], + [25.3808, 80.5195, 119.0949, 110.0938], + [25.0567, 101.8461, 174.3329, 206.6013], + ], + ]], + &device, + ), + offset: TestTensor::from_floats( + [[ + [[0.000, 15.0000], [30.000, 45.0000]], + [[0.000, 3.7500], [7.5000, 11.2500]], + [[62.6667, 78.3333], [94.0000, 109.6667]], + [[15.6667, 19.5833], [23.5000, 27.4167]], + [[130.6667, 104.1250], [163.3333, 122.2732]], + [[32.6667, -492.9583], [40.8333, -787.1620]], + [[204.0000, 221.0000], [238.0000, 255.0000]], + [[51.0000, 55.2500], [59.5000, 63.7500]], + [[282.6667, 300.3333], [318.0000, 335.6667]], + [[70.6667, 75.0833], [79.5000, 83.9167]], + [[366.6667, 144.3750], [403.3333, 146.4121]], + [[91.6667, -1788.9860], [100.8333, -2392.7456]], + [[456.0000, 475.0000], [-2718.6250, -2953.2188]], + [[114.0000, 118.7500], [37.7361, 37.4063]], + [[550.6667, 570.3334], [-3404.5139, -3672.5312]], + [[137.6667, 142.5833], [28.6806, 27.5197]], + [[650.6667, 27.9584], [-4174.3657, -59.7509]], + [[162.6667, -3991.0139], [14.4028, -298.7557]], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [0.7029, 2.8356, 5.1067], + [12.7492, 19.4745, 17.8345], + [22.0687, 25.9156, 14.6394], + ], + [ + [3.3696, 12.6134, 19.2671], + [36.7492, 50.5856, 43.5506], + [50.8774, 56.3292, 30.7470], + ], + ], + [ + [ + [0.7029, 2.8356, 5.1067], + [12.7492, 19.4745, 17.8345], + [22.0687, 25.9156, 14.6394], + ], + [ + [3.3696, 12.6134, 19.2671], + [36.7492, 50.5856, 43.5506], + [50.8774, 56.3292, 30.7470], + ], + ], + [ + [ + [0.7029, 2.8356, 5.1067], + [12.7492, 19.4745, 17.8345], + [22.0687, 25.9156, 14.6394], + ], + [ + [3.3696, 12.6134, 19.2671], + [36.7492, 50.5856, 43.5506], + [50.8774, 56.3292, 30.7470], + ], + ], + ], + &device, + ), + mask: TestTensor::from_floats( + [[ + [[1303.5000, 1447.8750], [1862.2500, 2006.6250]], + [[1571.1666, 1721.9581], [2154.7500, 2305.5417]], + [[1857.4999, 1396.7151], [2465.9167, 1753.2246]], + [[2315.5000, 2479.1250], [2948.7502, 3112.3750]], + [[2645.1665, 2815.2085], [3303.2500, 3473.2917]], + [[2993.5000, 1150.0625], [3676.4165, 1300.4055]], + [[3531.5000, 3714.3752], [1150.1876, 1148.4744]], + [[3923.1665, 4112.4585], [794.3865, 770.0470]], + [[4333.5000, 181.4101], [368.3260, 4.2679]], + ]], + &device, + ), + bias: TestTensor::from_floats([4., 4., 4.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_deform_conv2d_batched() { + let test = Conv2dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [ + [ + [ + [0.000, 3.4604, 8.7539, 6.8080], + [8.4661, 24.0784, 35.4610, 26.4276], + [19.5988, 51.0406, 68.4389, 53.4993], + [17.4698, 47.9106, 67.3808, 56.6063], + ], + [ + [0.000, 5.1185, 12.7803, 9.8796], + [12.1957, 34.5728, 50.4616, 37.3777], + [27.4521, 71.1227, 94.5778, 73.4724], + [24.1147, 65.8443, 91.8995, 76.7475], + ], + ], + [ + [ + [6.3750, 19.3553, 26.4935, 22.5650], + [17.0026, 57.8088, 85.5580, 78.0746], + [20.7334, 86.5793, 139.4667, 136.4133], + [16.8126, 103.0225, 186.4502, 206.9613], + ], + [ + [9.5625, 28.8786, 39.1137, 32.9178], + [25.1984, 85.0747, 124.6941, 112.5691], + [30.0242, 124.2863, 198.6056, 192.4489], + [23.5826, 143.4660, 257.8752, 283.2587], + ], + ], + ], + &device, + ), + + offset: TestTensor::from_floats( + [ + [ + [[0.000, 7.5000], [15.0000, 22.5000]], + [[0.000, 1.8750], [3.7500, 5.6250]], + [[31.3333, 39.1667], [47.0000, 54.8333]], + [[7.8333, 9.7917], [11.7500, 13.7083]], + [[65.3333, 62.7813], [81.6667, 75.4849]], + [[16.3333, -237.8021], [20.4167, -381.7280]], + [[102.0000, 110.5000], [119.0000, 127.5000]], + [[25.5000, 27.6250], [29.7500, 31.8750]], + [[141.3333, 150.1667], [159.0000, 167.8333]], + [[35.3333, 37.5417], [39.7500, 41.9583]], + [[183.3333, 132.3438], [201.6667, 142.0197]], + [[45.8333, -839.6840], [50.4167, -1133.4155]], + [[228.0000, 237.5000], [-1336.1562, -1452.1173]], + [[57.0000, 59.3750], [40.3090, 41.4141]], + [[275.3333, 285.1667], [-1670.5034, -1802.9244]], + [[68.8333, 71.2917], [44.0451, 44.9841]], + [[325.3333, 174.7396], [-2045.1747, -1090.4585]], + [[81.3333, -1844.0659], [46.8090, -1150.2101]], + ], + [ + [[270.000, 277.5000], [285.0000, 292.5000]], + [[67.5000, 69.3750], [71.2500, 73.1250]], + [[313.3333, 321.1667], [329.0000, 336.8333]], + [[78.3333, 80.2917], [82.2500, 84.2083]], + [[359.3333, 130.1563], [375.6667, 130.6099]], + [[89.8333, -4312.7603], [93.9167, -4893.6035]], + [[408.0000, 416.5000], [425.0000, 433.5000]], + [[102.0000, 104.1250], [106.2500, 108.3750]], + [[459.3333, 468.1667], [477.0000, 485.8333]], + [[114.8333, 117.0417], [119.2500, 121.4583]], + [[513.3334, 97.9688], [531.6667, 93.8947]], + [[128.3333, -6720.3926], [132.9167, -7504.5405]], + [[570.000, 579.5000], [-7971.8438, -8251.0850]], + [[142.5000, 144.8750], [22.4965, 21.8203]], + [[629.3333, 639.1667], [-8948.2334, -9249.6641]], + [[157.3333, 159.7917], [15.7743, 14.8695]], + [[691.3333, 14.6145], [-9992.9453, -70.4040]], + [[172.8333, -9818.5234], [7.4132, -352.0222]], + ], + ], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [77.7195, 89.8692, 69.0213], + [121.0760, 137.0775, 92.2989], + [100.0212, 106.5561, 61.1851], + ], + [ + [112.3862, 131.6470, 103.8793], + [177.0760, 200.1887, 138.2681], + [149.5922, 158.7074, 94.3991], + ], + ], + [ + [ + [77.7195, 89.8692, 69.0213], + [121.0760, 137.0775, 92.2989], + [100.0212, 106.5561, 61.1851], + ], + [ + [112.3862, 131.6470, 103.8793], + [177.0760, 200.1887, 138.2681], + [149.5922, 158.7074, 94.3991], + ], + ], + [ + [ + [77.7195, 89.8692, 69.0213], + [121.0760, 137.0775, 92.2989], + [100.0212, 106.5561, 61.1851], + ], + [ + [112.3862, 131.6470, 103.8793], + [177.0760, 200.1887, 138.2681], + [149.5922, 158.7074, 94.3991], + ], + ], + ], + &device, + ), + mask: TestTensor::from_floats( + [ + [ + [[1299.7499, 1439.4375], [1849.1249, 1988.8125]], + [[1528.0834, 1673.9791], [2101.8750, 2247.7708]], + [[1771.7500, 1624.9811], [2369.9583, 2099.5039]], + [[2183.7500, 2342.0625], [2806.3750, 2964.6875]], + [[2464.0833, 2628.6042], [3111.1250, 3275.6458]], + [[2759.7500, 1979.2551], [3431.2085, 2390.0286]], + [[3241.7498, 3418.6873], [2415.3589, 2500.8682]], + [[3574.0835, 3757.2292], [2394.3889, 2471.7510]], + [[3921.7500, 2095.5293], [2345.9363, 1199.5048]], + ], + [ + [[5957.2500, 6096.9375], [6506.6250, 6646.3125]], + [[6392.5835, 6538.4790], [6966.3750, 7112.2705]], + [[6843.2500, 2443.8982], [7441.4585, 2550.9199]], + [[7462.2505, 7620.5625], [8084.8745, 8243.1875]], + [[7949.5835, 8114.1045], [8596.6250, 8761.1465]], + [[8452.2500, 1591.6719], [9123.7080, 1589.9454]], + [[9141.2500, 9318.1875], [1414.3584, 1375.1803]], + [[9680.5840, 9863.7285], [949.0560, 897.3544]], + [[10235.2500, 213.4454], [428.2699, 2.4790]], + ], + ], + &device, + ), + bias: TestTensor::from_floats([8., 8., 8.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_deform_conv2d_different_kernel_size() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [14.558521, 27.249609, 37.382030, 36.039406], + [33.151936, 60.480656, 81.264656, 78.618156], + [57.520061, 108.623283, 153.413559, 170.072998], + [54.706184, 102.596664, 144.367157, 162.643570], + ], + [ + [25.836353, 48.088451, 65.249161, 62.103317], + [56.805233, 102.995605, 136.983124, 131.120911], + [96.105408, 179.790192, 250.550934, 272.668793], + [90.210945, 167.567917, 232.847275, 257.934692], + ], + ]], + &device, + ), + offset: TestTensor::from_floats( + [[ + [ + [0.0e+00, 5.355903e+00, 1.171528e+01], + [3.124999e-01, 8.000000e+00, 1.000000e+01], + [7.500000e-01, 1.400000e+01, 1.600000e+01], + [1.312500e+00, 2.000000e+01, 2.200000e+01], + ], + [ + [0.0e+00, 1.736104e-03, 6.944418e-03], + [1.606250e+01, 2.000000e+00, 2.500000e+00], + [4.425000e+01, 3.500000e+00, 4.000000e+00], + [8.456250e+01, 5.000000e+00, 5.500000e+00], + ], + [ + [6.745834e+01, 7.996479e+01, 9.353048e+01], + [3.166667e+01, 3.377778e+01, 3.588889e+01], + [3.800000e+01, 4.011111e+01, 4.222223e+01], + [4.433333e+01, 4.644444e+01, 4.855556e+01], + ], + [ + [5.277777e-01, 5.955827e-01, 6.670526e-01], + [7.916667e+00, 8.444445e+00, 8.972222e+00], + [9.500000e+00, 1.002778e+01, 1.055556e+01], + [1.108333e+01, 1.161111e+01, 1.213889e+01], + ], + [ + [1.547778e+02, 1.751640e+02, 1.518874e+02], + [6.000000e+01, 6.222223e+01, 4.989969e+01], + [6.666666e+01, 6.888889e+01, 5.432098e+01], + [7.333334e+01, 7.555556e+01, 5.860340e+01], + ], + [ + [2.222223e+00, 2.363040e+00, -3.360339e+01], + [1.500000e+01, 1.555556e+01, -2.277485e+02], + [1.666667e+01, 1.722222e+01, -3.231605e+02], + [1.833333e+01, 1.888889e+01, -4.320448e+02], + ], + [ + [2.641250e+02, 2.021189e+02, 0.0e+00], + [9.100000e+01, 6.481482e+01, 0.0e+00], + [9.800000e+01, 6.863078e+01, 0.0e+00], + [1.050000e+02, 7.230093e+01, 0.0e+00], + ], + [ + [5.250000e+00, -7.268316e+01, 0.0e+00], + [2.275000e+01, -3.346296e+02, 0.0e+00], + [2.450000e+01, -4.611053e+02, 0.0e+00], + [2.625000e+01, -6.017269e+02, 0.0e+00], + ], + [ + [4.400000e+01, 1.197778e+02, 1.222222e+02], + [4.804860e+01, 1.271111e+02, 1.295556e+02], + [5.225000e+01, 1.344444e+02, 1.368889e+02], + [-3.138958e+02, -8.007446e+02, -8.507313e+02], + ], + [ + [3.377778e+02, 2.994445e+01, 3.055556e+01], + [4.848542e+02, 3.177778e+01, 3.238889e+01], + [6.467500e+02, 3.361111e+01, 3.422222e+01], + [4.909653e+02, 2.239892e+01, 2.265992e+01], + ], + [ + [1.533333e+02, 1.558889e+02, 1.584444e+02], + [1.610000e+02, 1.635556e+02, 1.661111e+02], + [1.686667e+02, 1.712222e+02, 1.737778e+02], + [-9.952491e+02, -1.054551e+03, -1.115134e+03], + ], + [ + [3.833333e+01, 3.897222e+01, 3.961111e+01], + [4.025000e+01, 4.088889e+01, 4.152778e+01], + [4.216667e+01, 4.280556e+01, 4.344445e+01], + [2.433767e+01, 2.453511e+01, 2.472810e+01], + ], + [ + [1.920000e+02, 1.946667e+02, 8.907407e+01], + [2.000000e+02, 2.026667e+02, 9.054632e+01], + [2.080000e+02, 2.106667e+02, 9.185186e+01], + [-1.272938e+03, -1.343509e+03, -5.811921e+02], + ], + [ + [4.800000e+01, 4.866667e+01, -7.413704e+02], + [5.000000e+01, 5.066667e+01, -9.788981e+02], + [5.200000e+01, 5.266667e+01, -1.232593e+03], + [2.531250e+01, 2.543518e+01, -6.388311e+02], + ], + [ + [2.333333e+02, 8.772182e+01, 0.0e+00], + [2.416667e+02, 8.827161e+01, 0.0e+00], + [2.500000e+02, 8.864776e+01, 0.0e+00], + [-1.587216e+03, -5.535372e+02, 0.0e+00], + ], + [ + [5.833333e+01, -9.011902e+02, 0.0e+00], + [6.041667e+01, -1.179988e+03, 0.0e+00], + [6.250000e+01, -1.475625e+03, 0.0e+00], + [2.489150e+01, -6.213175e+02, 0.0e+00], + ], + [ + [1.964444e+02, 2.802222e+02, 2.831111e+02], + [2.055625e+02, 2.888889e+02, 2.917778e+02], + [-1.173472e+03, -1.679611e+03, -1.771290e+03], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [1.144889e+03, 7.005556e+01, 7.077778e+01], + [1.469646e+03, 7.222223e+01, 7.294444e+01], + [5.029167e+02, 2.298823e+01, 2.295062e+01], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [3.240000e+02, 3.270000e+02, 3.300000e+02], + [3.330000e+02, 3.360000e+02, 3.390000e+02], + [-1.931469e+03, -2.034961e+03, -2.139958e+03], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [8.100000e+01, 8.175000e+01, 8.250000e+01], + [8.325000e+01, 8.400000e+01, 8.475000e+01], + [1.959376e+01, 1.946614e+01, 1.933334e+01], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [3.733333e+02, 3.764445e+02, 4.480865e+01], + [3.826667e+02, 3.857778e+02, 4.185955e+01], + [-2.313792e+03, -2.431276e+03, -2.392101e+02], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [9.333333e+01, 9.411111e+01, -1.904932e+03], + [9.566667e+01, 9.644444e+01, -2.344715e+03], + [1.429166e+01, 1.406212e+01, -3.417283e+02], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [4.253333e+02, 1.636843e+01, 0.0e+00], + [4.350000e+02, 1.217279e+01, 0.0e+00], + [-2.738517e+03, -4.792887e+01, 0.0e+00], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [1.063333e+02, -2.178747e+03, 0.0e+00], + [1.087500e+02, -2.670679e+03, 0.0e+00], + [6.947917e+00, -1.629574e+02, 0.0e+00], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [1.856041, 7.203409, 12.833395, 11.969448], + [24.236776, 40.125511, 41.396423, 27.642044], + [43.613083, 57.508926, 46.093338, 25.174383], + ], + [ + [6.989914, 26.580338, 42.618557, 37.501404], + [75.623192, 116.925674, 113.288368, 72.567764], + [112.724869, 139.826447, 107.653435, 56.799385], + ], + ], + [ + [ + [1.856041, 7.203409, 12.833395, 11.969448], + [24.236776, 40.125511, 41.396423, 27.642044], + [43.613083, 57.508926, 46.093338, 25.174383], + ], + [ + [6.989914, 26.580338, 42.618557, 37.501404], + [75.623192, 116.925674, 113.288368, 72.567764], + [112.724869, 139.826447, 107.653435, 56.799385], + ], + ], + ], + &device, + ), + mask: TestTensor::from_floats( + [[ + [ + [0.0e+00, 2.677941e+00, 5.857617e+00], + [4.015623e+01, 7.759999e+02, 8.492499e+02], + [6.637500e+01, 1.067750e+03, 1.141000e+03], + [9.865628e+01, 1.359500e+03, 1.432750e+03], + ], + [ + [6.745831e+01, 7.688924e+01, 8.684974e+01], + [8.387916e+02, 9.161111e+02, 9.934306e+02], + [1.146750e+03, 1.224069e+03, 1.301389e+03], + [1.454708e+03, 1.532028e+03, 1.609347e+03], + ], + [ + [1.547778e+02, 1.716607e+02, 1.460455e+02], + [9.861667e+02, 1.067556e+03, 8.756536e+02], + [1.310333e+03, 1.391722e+03, 1.110864e+03], + [1.634500e+03, 1.715889e+03, 1.339339e+03], + ], + [ + [2.641250e+02, 1.993876e+02, 0.0e+00], + [1.144875e+03, 8.365740e+02, 0.0e+00], + [1.485250e+03, 1.056253e+03, 0.0e+00], + [1.825625e+03, 1.268859e+03, 0.0e+00], + ], + [ + [3.800000e+02, 1.047861e+03, 1.137389e+03], + [5.276354e+02, 1.404444e+03, 1.493972e+03], + [6.826807e+02, 1.761028e+03, 1.850555e+03], + [5.038855e+02, 1.256341e+03, 1.304936e+03], + ], + [ + [1.123500e+03, 1.217097e+03, 1.310694e+03], + [1.496292e+03, 1.589889e+03, 1.683486e+03], + [1.869083e+03, 1.962681e+03, 2.056278e+03], + [1.146700e+03, 1.190136e+03, 1.232930e+03], + ], + [ + [1.300000e+03, 1.397667e+03, 6.512036e+02], + [1.689000e+03, 1.786667e+03, 8.072734e+02], + [2.078000e+03, 2.175667e+03, 9.552593e+02], + [1.060781e+03, 1.097745e+03, 4.656539e+02], + ], + [ + [1.487833e+03, 5.672195e+02, 0.0e+00], + [1.893042e+03, 6.972655e+02, 0.0e+00], + [2.298250e+03, 8.188910e+02, 0.0e+00], + [9.472098e+02, 3.238781e+02, 0.0e+00], + ], + [ + [1.216444e+03, 1.792806e+03, 1.898611e+03], + [1.536448e+03, 2.214222e+03, 2.320028e+03], + [5.177084e+02, 7.256571e+02, 7.493920e+02], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [1.897500e+03, 2.007375e+03, 2.117250e+03], + [2.335125e+03, 2.445000e+03, 2.554875e+03], + [5.591096e+02, 5.750975e+02, 5.903336e+02], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [2.119333e+03, 2.233278e+03, 2.654414e+02], + [2.573167e+03, 2.687111e+03, 2.907444e+02], + [3.856317e+02, 3.924502e+02, 3.737657e+01], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + [ + [2.352500e+03, 9.009851e+01, 0.0e+00], + [2.822542e+03, 7.854909e+01, 0.0e+00], + [1.785990e+02, 2.930897e+00, 0.0e+00], + [0.0e+00, 0.0e+00, 0.0e+00], + ], + ]], + &device, + ), + bias: TestTensor::from_floats([12., 12.], &device), + }; + test.assert_grads(grads); +} + +#[test] +fn test_deform_conv2d_different_padding() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 2, + padding_2: 3, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + let device = Default::default(); + let grads = Grads { + x: TestTensor::from_floats( + [[ + [ + [60.633026, 60.906506, 61.179493, 61.451954], + [122.557770, 123.088188, 123.618599, 124.149033], + [126.801132, 127.331535, 127.861938, 128.392365], + [131.044434, 131.574875, 132.105286, 132.635712], + ], + [ + [102.000595, 102.497604, 102.993835, 103.489281], + [198.932983, 199.830597, 200.728210, 201.625870], + [206.113968, 207.011627, 207.909256, 208.806870], + [213.294952, 214.192627, 215.090271, 215.987930], + ], + ]], + &device, + ), + // => Position 788: 10.421875 != 10.0546875 + // diff (rel = +1.79e-2, abs = +3.67e-1), tol (rel = +1.00e-2, abs = +9.77e-4) + offset: TestTensor::from_floats( + [[ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 0.0, 0.0, 0.895062, 14.760561, 17.604168, 20.698063, 22.200424, 0.0, + ], + [ + 0.0, 0.0, 0.687500, 9.500000, 10.0, 10.500000, 10.108797, 0.0, + ], + [ + 0.0, 0.0, 1.113426, 13.500000, 14.000000, 14.499999, 13.645835, 0.0, + ], + [ + 0.0, 0.0, 1.613426, 17.500000, 18.000000, 18.500000, 17.108795, 0.0, + ], + [ + 0.0, + 0.0, + -12.395836, + -122.399445, + -130.752319, + -139.355469, + -131.526810, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 0.0, 0.0, 0.154321, 0.017506, 0.020833, 0.024450, -0.387539, 0.0, + ], + [ + 0.0, 0.0, 24.187502, 2.375000, 2.500000, 2.625000, -37.863422, 0.0, + ], + [ + 0.0, 0.0, 48.057869, 3.375000, 3.500000, 3.625000, -66.770836, 0.0, + ], + [ + 0.0, + 0.0, + 80.02312, + 4.375000, + 4.500000, + 4.625000, + -103.752319, + 0.0, + ], + [ + 0.0, + 0.0, + 113.215271, + 5.107495, + 5.219907, + 5.332031, + -139.725891, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 0.0, 14.206017, 83.017586, 92.379395, 102.010040, 90.356323, 0.0, 0.0, + ], + [ + 0.0, 6.504737, 35.444443, 35.981483, 36.518517, 29.978970, 0.0, 0.0, + ], + [ + 0.0, 7.668316, 39.740742, 40.277779, 40.814816, 33.071907, 0.0, 0.0, + ], + [ + 0.0, 8.911458, 44.037037, 44.574074, 45.111111, 36.085281, 0.0, 0.0, + ], + [ + 0.0, + -57.523048, + -274.267914, + -289.547089, + -305.095093, + -248.578552, + 0.0, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 0.0, 9.749230, 0.955354, 0.980994, 1.006945, -13.930464, 0.0, 0.0, + ], + [ + 0.0, + 96.046921, + 8.861111, + 8.995371, + 9.129629, + -129.920715, + 0.0, + 0.0, + ], + [ + 0.0, + 147.434769, + 9.935185, + 10.069445, + 10.203704, + -186.718735, + 0.0, + 0.0, + ], + [ + 0.0, + 207.494781, + 11.009259, + 11.143518, + 11.277778, + -252.188889, + 0.0, + 0.0, + ], + [ + 0.0, + 226.050003, + 10.153355, + 10.252030, + 10.350393, + -266.255280, + 0.0, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 44.224964, 159.898483, 176.651901, 193.692688, 146.270813, 0.0, 0.0, 0.0, + ], + [ + 19.050755, 64.870377, 65.444443, 66.018517, 46.553150, 0.0, 0.0, 0.0, + ], + [ + 21.049385, 69.462967, 70.037033, 70.611115, 49.104595, 0.0, 0.0, 0.0, + ], + [ + 23.133059, 74.055557, 74.629631, 75.203705, 51.570988, 0.0, 0.0, 0.0, + ], + [ + -141.200272, + -445.302155, + -468.381012, + -491.747223, + -341.553131, + 0.0, + 0.0, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 35.665298, 3.505739, 3.556735, 3.608062, -48.756947, 0.0, 0.0, 0.0, + ], + [ + 181.404663, + 16.217594, + 16.361111, + 16.504629, + -238.136124, + 0.0, + 0.0, + 0.0, + ], + [ + 263.888885, + 17.365742, + 17.509258, + 17.652779, + -326.403656, + 0.0, + 0.0, + 0.0, + ], + [ + 355.643341, + 18.513889, + 18.657408, + 18.800926, + -423.941345, + 0.0, + 0.0, + 0.0, + ], + [ + 318.709198, + 14.359658, + 14.441552, + 14.523109, + -369.819580, + 0.0, + 0.0, + 0.0, + ], + ], + [ + [ + 0.0, 0.0, 88.846703, 237.478439, 261.731201, 286.289917, 182.508713, 0.0, + ], + [ + 0.0, 0.0, 37.688015, 94.722221, 95.333328, 95.944450, 57.441605, 0.0, + ], + [ + 0.0, 0.0, 40.562500, 99.611107, 100.222229, 100.833336, 59.410744, 0.0, + ], + [ + 0.0, 0.0, 43.527519, 104.500000, 105.111107, 105.722221, 61.289349, 0.0, + ], + [ + 0.0, + 0.0, + -258.324371, + -618.353943, + -649.340271, + -680.632507, + -397.101013, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, + 0.0, + 76.229431, + 7.564093, + 7.641718, + 7.719699, + -102.792252, + 0.0, + ], + [ + 0.0, + 0.0, + 272.015167, + 23.680555, + 23.833332, + 23.986113, + -351.944214, + 0.0, + ], + [ + 0.0, + 0.0, + 386.062500, + 24.902777, + 25.055557, + 25.208334, + -472.147888, + 0.0, + ], + [ + 0.0, + 0.0, + 509.978149, + 26.125000, + 26.277777, + 26.430555, + -602.219971, + 0.0, + ], + [ + 0.0, + 0.0, + 378.410248, + 17.123661, + 17.187500, + 17.250984, + -436.000732, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, 157.623291, 331.938538, 365.283356, 398.952606, 205.988480, 0.0, 0.0, + ], + [ + 0.0, 66.495949, 130.925934, 131.574066, 132.222229, 64.435974, 0.0, 0.0, + ], + [ + 0.0, 70.396835, 136.111115, 136.759262, 137.407410, 65.672256, 0.0, 0.0, + ], + [ + 0.0, 74.393753, 141.296295, 141.944458, 142.592606, 66.812523, 0.0, 0.0, + ], + [ + 0.0, + -432.798035, + -827.492065, + -867.978455, + -908.789368, + -425.074158, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, + 140.150024, + 14.043960, + 14.152921, + 14.262260, + -187.656906, + 0.0, + 0.0, + ], + [ + 0.0, + 386.813873, + 32.731483, + 32.893517, + 33.055557, + -494.779602, + 0.0, + 0.0, + ], + [ + 0.0, + 538.926697, + 34.027779, + 34.189816, + 34.351852, + -653.421875, + 0.0, + 0.0, + ], + [ + 0.0, + 701.505859, + 35.324074, + 35.486115, + 35.648151, + -822.530640, + 0.0, + 0.0, + ], + [ + 0.0, + 416.044586, + 18.903570, + 18.944647, + 18.985338, + -476.728790, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 249.876541, 435.868500, 479.178772, 522.832031, 207.919815, 0.0, 0.0, 0.0, + ], + [ + 105.417015, 170.611115, 171.296295, 171.981476, 64.750000, 0.0, 0.0, 0.0, + ], + [ + 110.441696, 176.092590, 176.777771, 177.462952, 65.156044, 0.0, 0.0, 0.0, + ], + [ + 115.567902, 181.574066, 182.259247, 182.944458, 65.460571, 0.0, 0.0, 0.0, + ], + [ + -662.743530, + -1056.641846, + -1107.501953, + -1158.704712, + -409.510162, + 0.0, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 227.160507, + 22.982454, + 23.125793, + 23.269531, + -303.112030, + 0.0, + 0.0, + 0.0, + ], + [ + 518.495178, + 42.652779, + 42.824074, + 42.995369, + -657.157410, + 0.0, + 0.0, + 0.0, + ], + [ + 712.252380, + 44.023148, + 44.194443, + 44.365738, + -857.817200, + 0.0, + 0.0, + 0.0, + ], + [ + 917.074036, + 45.393517, + 45.564812, + 45.736115, + -1069.541626, + 0.0, + 0.0, + 0.0, + ], + [ + 416.581482, + 18.997831, + 19.013102, + 19.027966, + -475.031525, + 0.0, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, 0.0, 151.750259, 210.166672, 210.888885, 211.611099, 57.506927, 0.0, + ], + [ + 0.0, 0.0, 157.929276, 215.944443, 216.666672, 217.388901, 57.052204, 0.0, + ], + [ + 0.0, 0.0, 164.215271, 221.722229, 222.444458, 223.166672, 56.490482, 0.0, + ], + [ + 0.0, + 0.0, + -931.783752, + -1285.353760, + -1346.555908, + -1408.119385, + -346.739044, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, + 0.0, + 655.669983, + 52.541668, + 52.722221, + 52.902775, + -824.946777, + 0.0, + ], + [ + 0.0, + 0.0, + 890.972473, + 53.986111, + 54.166668, + 54.347225, + -1067.525024, + 0.0, + ], + [ + 0.0, + 0.0, + 1137.937500, + 55.430557, + 55.611115, + 55.791668, + -1321.765625, + 0.0, + ], + [ + 0.0, + 0.0, + 375.580566, + 17.180984, + 17.169498, + 17.157579, + -425.993713, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, 213.521454, 256.629639, 257.388885, 258.148132, 41.652927, 0.0, 0.0, + ], + [ + 0.0, 221.015625, 262.703705, 263.462982, 264.222229, 40.176598, 0.0, 0.0, + ], + [ + 0.0, 228.622284, 268.777802, 269.537048, 270.296295, 38.587788, 0.0, 0.0, + ], + [ + 0.0, + -1285.466797, + -1554.254517, + -1627.530640, + -1701.186646, + -228.291397, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, + 823.380554, + 64.157410, + 64.347221, + 64.537033, + -1028.532715, + 0.0, + 0.0, + ], + [ + 0.0, + 1107.296509, + 65.675926, + 65.865746, + 66.055557, + -1320.097534, + 0.0, + 0.0, + ], + [ + 0.0, + 1403.473022, + 67.194450, + 67.384262, + 67.574074, + -1623.922974, + 0.0, + 0.0, + ], + [ + 0.0, + 288.151398, + 13.201796, + 13.158524, + 13.114797, + -323.577820, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 288.790131, 306.574066, 307.370361, 308.166656, 15.734239, 0.0, 0.0, 0.0, + ], + [ + 297.696838, 312.944427, 313.740723, 314.537048, 13.138914, 0.0, 0.0, 0.0, + ], + [ + 306.721527, 319.314819, 320.111115, 320.907410, 10.425544, 0.0, 0.0, 0.0, + ], + [ + -1711.543213, + -1844.013062, + -1930.236572, + -2016.858643, + -46.846100, + 0.0, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 1011.358093, + 76.643517, + 76.842590, + 77.041664, + -1255.045654, + 0.0, + 0.0, + 0.0, + ], + [ + 1347.466431, + 78.236107, + 78.435181, + 78.634262, + -1599.175903, + 0.0, + 0.0, + 0.0, + ], + [ + 1696.433350, + 79.828705, + 80.027779, + 80.226852, + -1956.164917, + 0.0, + 0.0, + 0.0, + ], + [ + 146.703568, + 6.690874, + 6.612756, + 6.534196, + -159.277222, + 0.0, + 0.0, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + ]], + &device, + ), + weight: TestTensor::from_floats( + [ + [ + [ + [10.341997, 22.988085, 35.634174], + [46.920216, 59.566299, 72.212387], + [80.881615, 92.591522, 104.158524], + ], + [ + [29.213360, 68.837769, 108.462166], + [143.825104, 183.449509, 223.073944], + [228.029373, 256.751740, 283.807098], + ], + ], + [ + [ + [10.341997, 22.988085, 35.634174], + [46.920216, 59.566299, 72.212387], + [80.881615, 92.591522, 104.158524], + ], + [ + [29.213360, 68.837769, 108.462166], + [143.825104, 183.449509, 223.073944], + [228.029373, 256.751740, 283.807098], + ], + ], + ], + &device, + ), + mask: TestTensor::from_floats( + [[ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 0.0, 0.0, 0.447531, 7.380288, 8.802088, 10.349031, 11.100212, 0.0, + ], + [ + 0.0, 0.0, 44.343754, 584.937439, 639.250000, 693.562439, 683.262756, 0.0, + ], + [ + 0.0, 0.0, 68.390068, 803.437561, 857.750000, 912.062500, 874.698059, 0.0, + ], + [ + 0.0, + 0.0, + 96.473381, + 1021.937500, + 1076.250000, + 1130.562500, + 1062.095947, + 0.0, + ], + [ + 0.0, + 0.0, + 121.302101, + 1168.487915, + 1218.373779, + 1268.134888, + 1169.444702, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 0.0, 13.084491, 75.860909, 83.767761, 91.809029, 80.728188, 0.0, 0.0, + ], + [ + 0.0, 118.950417, 649.486084, 707.821777, 766.157410, 658.076599, 0.0, 0.0, + ], + [ + 0.0, + 170.660782, + 884.171265, + 942.506958, + 1000.842651, + 837.809326, + 0.0, + 0.0, + ], + [ + 0.0, + 226.707260, + 1118.856445, + 1177.192261, + 1235.527710, + 1013.205933, + 0.0, + 0.0, + ], + [ + 0.0, + 234.939651, + 1106.213867, + 1153.415649, + 1200.482666, + 966.248901, + 0.0, + 0.0, + ], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [ + 42.524002, 153.045700, 168.319275, 183.736511, 138.144653, 0.0, 0.0, 0.0, + ], + [ + 207.319611, 718.432800, 780.791626, 843.150391, 619.975037, 0.0, 0.0, 0.0, + ], + [ + 290.277802, + 969.303223, + 1031.661987, + 1094.020752, + 784.421631, + 0.0, + 0.0, + 0.0, + ], + [ + 377.871063, + 1220.173584, + 1282.532471, + 1344.891235, + 944.233032, + 0.0, + 0.0, + 0.0, + ], + [ + 328.083038, + 1025.494995, + 1069.130615, + 1112.622192, + 766.054932, + 0.0, + 0.0, + 0.0, + ], + ], + [ + [ + 0.0, 0.0, 88.238174, 235.055206, 258.194336, 281.486389, 178.858536, 0.0, + ], + [ + 0.0, 0.0, 305.575500, 789.868042, 856.250061, 922.631897, 572.466064, 0.0, + ], + [ + 0.0, + 0.0, + 421.809021, + 1056.923584, + 1123.305542, + 1189.687500, + 719.598816, + 0.0, + ], + [ + 0.0, + 0.0, + 542.976746, + 1323.979248, + 1390.361206, + 1456.743042, + 861.797302, + 0.0, + ], + [ + 0.0, + 0.0, + 393.291565, + 934.439697, + 974.010376, + 1013.428101, + 586.924011, + 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, 157.214920, 330.227448, 362.473419, 394.881653, 203.374420, 0.0, 0.0, + ], + [ + 0.0, + 424.340576, + 867.495361, + 937.900452, + 1008.305542, + 505.640503, + 0.0, + 0.0, + ], + [ + 0.0, + 578.894897, + 1150.736084, + 1221.141235, + 1291.546265, + 630.414001, + 0.0, + 0.0, + ], + [ + 0.0, + 738.682495, + 1433.976929, + 1504.381958, + 1574.787109, + 749.954346, + 0.0, + 0.0, + ], + [ + 0.0, 429.912781, 816.507507, 850.771973, 884.873779, 411.152588, 0.0, 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 249.876541, 434.964233, 477.198730, 519.604675, 206.215576, 0.0, 0.0, 0.0, + ], + [ + 560.309326, + 949.520813, + 1023.949097, + 1098.377319, + 422.458344, + 0.0, + 0.0, + 0.0, + ], + [ + 756.768127, + 1248.946777, + 1323.375000, + 1397.803223, + 521.289001, + 0.0, + 0.0, + 0.0, + ], + [ + 958.759216, + 1548.372803, + 1622.800903, + 1697.229248, + 614.587402, + 0.0, + 0.0, + 0.0, + ], + [ + 428.833923, 679.269775, 707.346252, 735.250916, 258.169373, 0.0, 0.0, 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, + 0.0, + 707.671387, + 1033.687378, + 1112.138916, + 1190.590210, + 328.295044, + 0.0, + ], + [ + 0.0, + 0.0, + 947.779419, + 1349.298584, + 1427.750000, + 1506.201416, + 399.438080, + 0.0, + ], + [ + 0.0, + 0.0, + 1193.718872, + 1664.909668, + 1743.361084, + 1821.812500, + 464.749847, + 0.0, + ], + [ + 0.0, 0.0, 388.737854, 532.503540, 553.962891, 575.241089, 140.658310, 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 0.0, + 880.797302, + 1124.393555, + 1206.868042, + 1289.342651, + 209.627625, + 0.0, + 0.0, + ], + [ + 0.0, + 1169.882812, + 1456.189819, + 1538.664429, + 1621.138916, + 247.754730, + 0.0, + 0.0, + ], + [ + 0.0, + 1465.098755, + 1787.986084, + 1870.460571, + 1952.935181, + 279.751526, + 0.0, + 0.0, + ], + [ + 0.0, 297.330719, 356.362152, 369.893524, 383.234344, 50.974621, 0.0, 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [ + 1074.567993, + 1219.497681, + 1305.995361, + 1392.493042, + 71.162437, + 0.0, + 0.0, + 0.0, + ], + [ + 1416.214722, + 1567.479126, + 1653.976929, + 1740.474609, + 72.689949, + 0.0, + 0.0, + 0.0, + ], + [ + 1764.290771, + 1915.460571, + 2001.958496, + 2088.456055, + 67.787628, + 0.0, + 0.0, + 0.0, + ], + [ + 151.018372, 160.055023, 164.776138, 169.298447, 3.865937, 0.0, 0.0, 0.0, + ], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + ]], + &device, + ), + bias: TestTensor::from_floats([48., 48.], &device), + }; + test.assert_grads(grads); +} + +struct Conv2dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + stride_1: usize, + stride_2: usize, + dilation_1: usize, + dilation_2: usize, + groups: usize, + offset_groups: usize, + height: usize, + width: usize, +} + +struct Grads { + x: TestTensor<4>, + offset: TestTensor<4>, + weight: TestTensor<4>, + mask: TestTensor<4>, + bias: TestTensor<1>, +} + +impl Conv2dTestCase { + fn assert_grads(self, expected_grads: Grads) { + let out_height = + (self.height + 2 * self.padding_1 - self.dilation_1 * (self.kernel_size_1 - 1) - 1) + / self.stride_1 + + 1; + let out_width = + (self.width + 2 * self.padding_2 - self.dilation_2 * (self.kernel_size_2 - 1) - 1) + / self.stride_2 + + 1; + + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let shape_offset = Shape::new([ + self.batch_size, + 2 * self.offset_groups * self.kernel_size_1 * self.kernel_size_2, + out_height, + out_width, + ]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size_1, + self.kernel_size_2, + ]); + let shape_mask = Shape::new([ + self.batch_size, + self.offset_groups * self.kernel_size_1 * self.kernel_size_2, + out_height, + out_width, + ]); + let device = Default::default(); + let weight = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<4, _>(shape_weight) + .into_data(), + &device, + ) + .require_grad(); + let bias = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + &device, + ) + .require_grad(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + let offset = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_offset.num_elements() as i64, &device) + .reshape::<4, _>(shape_offset.clone()) + .into_data(), + &device, + ) + .div_scalar(shape_offset.num_elements() as f32) + .require_grad(); + + let mask = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_mask.num_elements() as i64, &device) + .reshape::<4, _>(shape_mask.clone()) + .into_data(), + &device, + ) + .div_scalar(shape_mask.num_elements() as f32) + .require_grad(); + + let output = deform_conv2d( + x.clone(), + offset.clone(), + weight.clone(), + Some(mask.clone()), + Some(bias.clone()), + DeformConvOptions::new( + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + [self.dilation_1, self.dilation_2], + self.groups, + self.offset_groups, + ), + ); + let grads = output.backward(); + + // Assert + let x_grad_actual = x.grad(&grads).unwrap(); + let offset_grad_actual = offset.grad(&grads).unwrap(); + let weight_grad_actual = weight.grad(&grads).unwrap(); + let mask_grad_actual = mask.grad(&grads).unwrap(); + let bias_grad_actual = bias.grad(&grads).unwrap(); + + // Relative is set to 5%, which is much higher than typical numerical test tolerances. + // This is due to the complexity of the deformable convolution operation. + // Unlike regular conv2d, which samples from fixed integer grid positions, + // deformable conv2d samples input values at fractional offset locations (learned offsets). + // These non-integer positions require bilinear interpolation to estimate the input value. + // Gradients computed through all these floating-point operations can compound numerical differences. + let tolerance = Tolerance::relative(0.5); + + println!("Testing bias"); + expected_grads + .bias + .to_data() + .assert_approx_eq::(&bias_grad_actual.to_data(), tolerance); + println!("Testing input"); + expected_grads + .x + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), tolerance); + println!("Testing offset"); + expected_grads + .offset + .to_data() + .assert_approx_eq::(&offset_grad_actual.to_data(), tolerance); + println!("Testing mask"); + expected_grads + .mask + .to_data() + .assert_approx_eq::(&mask_grad_actual.to_data(), tolerance); + println!("Testing weight"); + expected_grads + .weight + .to_data() + .assert_approx_eq::(&weight_grad_actual.to_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/div.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/div.rs new file mode 100644 index 0000000..8e1ac09 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/div.rs @@ -0,0 +1,105 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_div() { + let data_1 = TensorData::from([1.0, 7.0]); + let data_2 = TensorData::from([4.0, 7.0]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().div(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([0.25, 0.14285715]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([-0.0625, -0.14285715]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_div_scalar() { + let data = TensorData::from([1.0, 7.0]); + + let tensor = TestAutodiffTensor::<1>::from_data(data, &Default::default()).require_grad(); + let tensor_out = tensor.clone().div_scalar(4.0); + + let grads = tensor_out.backward(); + let grad = tensor.grad(&grads).unwrap(); + + grad.to_data() + .assert_eq(&TensorData::from([0.25, 0.25]), false); +} + +#[test] +fn test_div_complex_1() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[2.0, 2.0], [2.0, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().div(tensor_2.clone()); + let tensor_5 = tensor_4.div(tensor_3.clone()); + + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + let grad_3 = tensor_3.grad(&grads).unwrap(); + + let expected = TensorData::from([[0.1250, 0.07142857], [0.25, 0.16666667]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[-0.03125, -0.07142857], [-1.6250, 0.16666667]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + let expected = TensorData::from([[-0.0625, -0.25], [-1.6250, 0.25]]); + grad_3 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_div_complex_2() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.div(tensor_2.clone()); + + let grads = tensor_4.backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_absolute(2e-3); + let expected = TensorData::from([[2.00, 2.92857146], [1.36666667, 2.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[0.08333334, 0.09591837], [-0.05555558, -0.06714284]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/erf.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/erf.rs new file mode 100644 index 0000000..5b3d27e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/erf.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_erf() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().erf()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[32.0, 32.0], [32.0, 32.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[8.0, 8.0], [8.0, 8.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/exp.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/exp.rs new file mode 100644 index 0000000..d567265 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/exp.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_exp() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().exp()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default(); + let expected = TensorData::from([[54.5991, 27.4746], [54.5991, 27.4746]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[-5.4598e+01, -9.1188e-04], [2.9556e+01, 8.0342e+01]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/expand.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/expand.rs new file mode 100644 index 0000000..8a82716 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/expand.rs @@ -0,0 +1,39 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_expand() { + // Python code to generate the test case values + // import torch + // x1 = torch.tensor([4.0, 7.0, 2.0, 3.0], requires_grad=True) + // x2 = torch.tensor([2.0, 4.5, 7.0, 3.0], requires_grad=True) + // y = x1.expand(4, 4) + // z = (x2 * y).sum() + // z.backward() + // print("x1", x1.grad) + // print("x2", x2.grad) + + let device = Default::default(); + + let data_1 = TensorData::from([4.0, 7.0, 2.0, 3.0]); + let tensor_1 = TestAutodiffTensor::<1>::from_data(data_1, &device).require_grad(); + + let data_2 = TensorData::from([2.0, 4.5, 7.0, 3.0]); + let tensor_2 = TestAutodiffTensor::<1>::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().expand([4, 4]); + + // Use unsqueeze to make tensor_2 have the same shape as tensor_3 + let tensor_4 = tensor_2.clone().unsqueeze().mul(tensor_3).sum(); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([8., 18., 28., 12.]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([16., 28., 8., 12.]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/flip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/flip.rs new file mode 100644 index 0000000..fac5f99 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/flip.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_flip() { + let data_1 = TensorData::from([[[1.0, 7.0], [2.0, 3.0]]]); // 1x2x2 + let data_2 = TensorData::from([[[3.0, 2.0, 7.0], [3.0, 3.2, 1.0]]]); // 1x2x3 + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<3>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_2.clone().flip([1, 2]); + let tensor_4 = tensor_1.clone().matmul(tensor_3); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + grad_1 + .into_data() + .assert_approx_eq::(&TensorData::from([[[7.2, 12.0], [7.2, 12.0]]]), tolerance); // 1x2x2 + grad_2.into_data().assert_approx_eq::( + &TensorData::from([[[10.0, 10.0, 10.0], [3.0, 3.0, 3.0]]]), + tolerance, + ); // 1x2x3 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/floor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/floor.rs new file mode 100644 index 0000000..fc35a66 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/floor.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_floor() { + let data = TensorData::from([ + [-1.9751, 0.0714, 0.0643, 0.2406], + [-1.3172, 0.1252, -0.1119, -0.0127], + ]); + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data, &device).require_grad(); + let tensor_2 = tensor_1.clone().floor(); + let grads = tensor_2.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + + grad_1.to_data().assert_eq( + &TensorData::from([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gather_scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gather_scatter.rs new file mode 100644 index 0000000..7fb2295 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gather_scatter.rs @@ -0,0 +1,99 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, Int, Tensor, TensorData}; + +#[test] +fn test_gather_grad() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::from_data( + TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]), + &device, + ) + .require_grad(); + let indices = Tensor::::from_data( + TensorData::from([[2, 1, 0, 1, 2], [1, 0, 2, 1, 0]]), + &device, + ); + + let tensor_2 = tensor_1.clone().matmul(tensor_1.clone().transpose()); + let tensor_3 = tensor_1.clone().gather(1, indices); + let tensor_4 = tensor_2.matmul(tensor_3); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + + grad_1.to_data().assert_eq( + &TensorData::from([[94., 150., 187.], [242., 305., 304.]]), + false, + ); +} + +#[test] +fn test_scatter_grad() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::from_data( + TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]), + &device, + ) + .require_grad(); + let values = TestAutodiffTensor::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ) + .require_grad(); + let indices = Tensor::::from_data( + TensorData::from([[2, 1, 0], [2, 0, 1]]), + &device, + ); + + let tensor_2 = tensor_1.clone().matmul(tensor_1.clone().transpose()); + let tensor_3 = tensor_1 + .clone() + .scatter(1, indices, values.clone(), IndexingUpdateOp::Add); + let tensor_4 = tensor_2.matmul(tensor_3); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = values.grad(&grads).unwrap(); + + grad_1.to_data().assert_eq( + &TensorData::from([[127., 181., 235.], [226., 316., 406.]]), + false, + ); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[19., 19., 19.], [64., 64., 64.]]), false); +} + +#[test] +fn test_scatter_add_grad_partial_indices() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::from_data(TensorData::from([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]]), &device) + .require_grad(); + let tensor_2 = + TestAutodiffTensor::from_data(TensorData::from([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]]), &device) + .require_grad(); + let values = + TestAutodiffTensor::from_data(TensorData::from([[4.0, 5.0, 6.0]]), &device).require_grad(); + let indices = + Tensor::::from_data(TensorData::from([[2, 1, 0]]), &device); + + let tensor_3 = tensor_1.clone().mul(tensor_2); + let tensor_4 = tensor_3 + .clone() + .scatter(1, indices, values.clone(), IndexingUpdateOp::Add); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = values.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[1., 2., 3., 4., 5., 6.]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[1., 1., 1.]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gelu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gelu.rs new file mode 100644 index 0000000..ddc4582 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gelu.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance, activation}; + +#[test] +fn should_diff_gelu() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_floats([[0.0, 1.0], [-3.0, 4.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::from_floats([[6.0, -0.5], [9.0, 10.0]], &device).require_grad(); + + let x = tensor_1.clone().matmul(activation::gelu(tensor_2.clone())); + let x = tensor_1.clone().matmul(x); + let grads = x.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::permissive(); + let expected = TensorData::from([[1.46281, 1.46281], [48.22866, 153.46280]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[-15.0000, -1.98757], [17.0000, 17.0000]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gradients.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gradients.rs new file mode 100644 index 0000000..2cef9f1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/gradients.rs @@ -0,0 +1,24 @@ +use super::*; +use burn_tensor::{Distribution, activation}; + +#[test] +fn should_update_tensor_when_grad_replace() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::random([32, 32], Distribution::Default, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::random([32, 32], Distribution::Default, &device); + + let x = tensor_1.clone().matmul(activation::gelu(tensor_2)); + let mut grads = x.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + + let grad_1_updated = + TestAutodiffTensor::random([32, 32], Distribution::Default, &device).require_grad(); + tensor_1.grad_replace(&mut grads, grad_1_updated.clone().inner()); + + let grad_1_new = tensor_1.grad(&grads).unwrap(); + + assert_ne!(grad_1_new.to_data(), grad_1.into_data()); + assert_eq!(grad_1_new.into_data(), grad_1_updated.into_data()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log.rs new file mode 100644 index 0000000..4839dca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log.rs @@ -0,0 +1,30 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_diff_log() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().log()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + let expected = TensorData::from([[60.2652, 72.3130], [60.2652, 72.3130]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[22.8614, 24.5043], [24.5729, 26.8507]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log1p.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log1p.rs new file mode 100644 index 0000000..1042080 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log1p.rs @@ -0,0 +1,28 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_log1p() { + let tensor_1 = TestAutodiffTensor::<2>::from([[0.0, 1.0], [3.0, 4.0]]).require_grad(); + let tensor_2 = TestAutodiffTensor::from([[6.0, 7.0], [9.0, 10.0]]).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().log1p()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + let expected = TensorData::from([[64.80622101, 75.49362183], [64.80622101, 75.49362183]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[22.92208481, 24.47565651], [24.72780228, 26.86416626]]); + + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log_sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log_sigmoid.rs new file mode 100644 index 0000000..0ce77b1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/log_sigmoid.rs @@ -0,0 +1,19 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn should_diff_log_sigmoid() { + let data = TensorData::from([[0.8762, -0.1423], [-300., 200.]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data, &device).require_grad(); + let tensor_2 = activation::log_sigmoid(tensor_1.clone()); + let grads = tensor_2.backward(); + + let grad = tensor_1.grad(&grads).unwrap(); + + let expected = TensorData::from([[0.293966, 0.535515], [1.000000, 0.000000]]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mask.rs new file mode 100644 index 0000000..8b64182 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mask.rs @@ -0,0 +1,65 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Bool, Tensor, TensorData}; + +#[test] +fn should_diff_mask_fill() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let mask = TensorData::from([[true, false], [false, true]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let mask = Tensor::::from_bool(mask, &device); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.mask_fill(mask, 2.0); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[7.0, 3.0], [4.0, 2.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[2.0, 1.0], [3.0, 7.0]]), false); +} + +#[test] +fn should_diff_mask_where() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::from_data([[1.0, 7.0], [2.0, 3.0]], &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data([[4.0, 7.0], [2.0, 3.0]], &device).require_grad(); + let tensor_3 = + TestAutodiffTensor::from_data([[8.8, 9.8], [10.8, 11.8]], &device).require_grad(); + let mask = + Tensor::::from_data([[true, false], [false, true]], &device); + + let tensor_4 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_5 = tensor_4.clone().matmul(tensor_3.clone()); + let tensor_6 = tensor_5.mask_where(mask, tensor_3.clone()); + let grads = tensor_6.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + let grad_3 = tensor_3.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + let expected = TensorData::from([[121.8, 55.0], [110.8, 50.0]]); + grad_1 + .into_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[27.4, 33.4], [95.0, 115.0]]); + grad_2 + .into_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[15., 18.], [23., 29.]]); + grad_3 + .into_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/matmul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/matmul.rs new file mode 100644 index 0000000..ab148ab --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/matmul.rs @@ -0,0 +1,83 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_matmul() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[11.0, 5.0], [11.0, 5.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[3.0, 3.0], [10.0, 10.0]]), false); + tensor_3 + .to_data() + .assert_eq(&TensorData::from([[18.0, 28.0], [14.0, 23.0]]), false); +} + +#[test] +fn test_matmul_complex_1() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[2.0, 2.0], [2.0, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_5 = tensor_4.matmul(tensor_3); + + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[44.0, 20.0], [44.0, 20.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[56.0, 56.0], [16.0, 16.0]]), false); +} + +#[test] +fn test_matmul_complex_2() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[2.0, 2.0], [2.0, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_5 = tensor_4.matmul(tensor_3.clone()); + let tensor_6 = tensor_1.clone().matmul(tensor_5); + + let grads = tensor_6.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[800.0, 792.0], [360.0, 592.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[264., 264.0], [344.0, 344.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxmin.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxmin.rs new file mode 100644 index 0000000..2222587 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxmin.rs @@ -0,0 +1,82 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_max_dim() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_floats([[1.0, 7.0], [-2.0, -3.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::from_floats([[4.0, -7.0], [2.0, 3.0]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.max_dim(1).unsqueeze()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[50.0, 34.0], [40.0, -10.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[8.0, 10.0], [56.0, 15.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_min_dim() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_floats([[1.0, 7.0], [-2.0, -3.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::from_floats([[4.0, -7.0], [2.0, 3.0]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.min_dim(1).unsqueeze()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[-42.0, 38.0], [-34.0, -24.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[10.0, 8.0], [15.0, 56.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_min_dim_3d_dim1() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<3>::from_floats([[[1.0, 7.0], [-2.0, -3.0]]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::<3>::from_floats([[[4., -7.], [2., 3.]]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().mul(tensor_2.clone()); + let tensor_4 = tensor_3.min_dim(1); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[[0., -7.], [2., 0.]]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[[0., 7.], [-2., -0.]]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool1d.rs new file mode 100644 index 0000000..0a4c9e1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool1d.rs @@ -0,0 +1,134 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::module::max_pool1d; + +#[test] +fn test_max_pool1d_simple() { + let kernel_size = 4; + let padding = 0; + let stride = 1; + let dilation = 1; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[0.9861, 0.5474, 0.4477, 0.0732, 0.3548, 0.8221]]], + &device, + ) + .require_grad(); + let x_grad_expected = + TestAutodiffTensor::<3>::from_floats([[[1., 1., 0., 0., 0., 1.]]], &device); + + let output = max_pool1d(x.clone(), kernel_size, stride, padding, dilation, false); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_with_dilation() { + let kernel_size = 4; + let padding = 0; + let stride = 1; + let dilation = 2; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + 0.5388, 0.0676, 0.7122, 0.8316, 0.0653, 0.9154, 0.1536, 0.9089, 0.8016, 0.7518, 0.2073, + 0.0501, 0.8811, 0.5604, 0.5075, 0.4384, 0.9963, 0.9698, 0.4988, 0.2609, 0.3391, 0.2230, + 0.4610, 0.5365, 0.6880, + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<3>::from_floats( + [[[ + 0., 0., 1., 0., 0., 3., 0., 1., 2., 1., 0., 0., 2., 0., 0., 0., 4., 4., 0., 0., 0., 0., + 0., 0., 1., + ]]], + &device, + ); + + let output = max_pool1d(x.clone(), kernel_size, stride, padding, dilation, false); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_complex() { + let kernel_size = 4; + let padding = 0; + let stride = 1; + let dilation = 1; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + 0.5388, 0.0676, 0.7122, 0.8316, 0.0653, 0.9154, 0.1536, 0.9089, 0.8016, 0.7518, 0.2073, + 0.0501, 0.8811, 0.5604, 0.5075, 0.4384, 0.9963, 0.9698, 0.4988, 0.2609, 0.3391, 0.2230, + 0.4610, 0.5365, 0.6880, + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<3>::from_floats( + [[[ + 0., 0., 0., 2., 0., 4., 0., 2., 1., 0., 0., 0., 4., 0., 0., 0., 4., 1., 1., 0., 0., 0., + 1., 1., 1., + ]]], + &device, + ); + + let output = max_pool1d(x.clone(), kernel_size, stride, padding, dilation, false); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_complex_with_padding() { + let kernel_size = 4; + let padding = 2; + let stride = 1; + let dilation = 1; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + 0.5388, 0.0676, 0.7122, 0.8316, 0.0653, 0.9154, 0.1536, 0.9089, 0.8016, 0.7518, 0.2073, + 0.0501, 0.8811, 0.5604, 0.5075, 0.4384, 0.9963, 0.9698, 0.4988, 0.2609, 0.3391, 0.2230, + 0.4610, 0.5365, 0.6880, + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<3>::from_floats( + [[[ + 1., 0., 1., 2., 0., 4., 0., 2., 1., 0., 0., 0., 4., 0., 0., 0., 4., 1., 1., 0., 0., 0., + 1., 1., 3., + ]]], + &device, + ); + + let output = max_pool1d(x.clone(), kernel_size, stride, padding, dilation, false); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool2d.rs new file mode 100644 index 0000000..adf9ce7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/maxpool2d.rs @@ -0,0 +1,271 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::module::max_pool2d; + +#[test] +fn test_max_pool2d_simple_1() { + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 0; + let padding_2 = 0; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 1; + let dilation_2 = 1; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + [0.2479, 0.6386, 0.3166, 0.5742], + [0.7065, 0.1940, 0.6305, 0.8959], + [0.5416, 0.8602, 0.8129, 0.1662], + [0.3358, 0.3059, 0.8293, 0.0990], + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<4>::from_floats( + [[[ + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 2.0], + [0.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + ]]], + &device, + ); + + let output = max_pool2d( + x.clone(), + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_simple_2() { + let kernel_size_1 = 2; + let kernel_size_2 = 2; + let padding_1 = 1; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 1; + let dilation_2 = 1; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + [0.2479, 0.6386, 0.3166, 0.5742], + [0.7065, 0.1940, 0.6305, 0.8959], + [0.5416, 0.8602, 0.8129, 0.1662], + [0.3358, 0.3059, 0.8293, 0.0990], + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<4>::from_floats( + [[[ + [1., 3., 0., 2.], + [3., 0., 0., 4.], + [1., 4., 0., 1.], + [2., 0., 3., 1.], + ]]], + &device, + ); + + let output = max_pool2d( + x.clone(), + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_with_dilation() { + let kernel_size_1 = 2; + let kernel_size_2 = 2; + let padding_1 = 1; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 2; + let dilation_2 = 2; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + [0.2479, 0.6386, 0.3166, 0.5742], + [0.7065, 0.1940, 0.6305, 0.8959], + [0.5416, 0.8602, 0.8129, 0.1662], + [0.3358, 0.3059, 0.8293, 0.0990], + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<4>::from_floats( + [[[ + [0., 0., 0., 0.], + [1., 1., 1., 2.], + [0., 4., 4., 0.], + [0., 1., 2., 0.], + ]]], + &device, + ); + + let output = max_pool2d( + x.clone(), + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_complex() { + let kernel_size_1 = 4; + let kernel_size_2 = 2; + let padding_1 = 2; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + let device = Default::default(); + let x = TestAutodiffTensor::from_floats( + [[[ + [0.5388, 0.0676, 0.7122, 0.8316, 0.0653], + [0.9154, 0.1536, 0.9089, 0.8016, 0.7518], + [0.2073, 0.0501, 0.8811, 0.5604, 0.5075], + [0.4384, 0.9963, 0.9698, 0.4988, 0.2609], + [0.3391, 0.2230, 0.4610, 0.5365, 0.6880], + ]]], + &device, + ) + .require_grad(); + let x_grad_expected = TestAutodiffTensor::<4>::from_floats( + [[[ + [0., 0., 0., 3., 0.], + [4., 0., 2., 1., 0.], + [0., 0., 0., 0., 0.], + [2., 4., 0., 0., 0.], + [0., 0., 0., 0., 2.], + ]]], + &device, + ); + + let output = max_pool2d( + x.clone(), + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_ceil_mode() { + // Test ceil_mode=true with gradient computation + // Using 1x1x6x6 input with kernel 3x3, stride 2x2, padding 0 + // Floor mode: output 2x2 + // Ceil mode: output 3x3 + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 0; + let padding_2 = 0; + let stride_1 = 2; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + let device = Default::default(); + // Input (values 1-36): + let x = TestAutodiffTensor::from_floats( + [[[ + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + [7.0, 8.0, 9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0, 17.0, 18.0], + [19.0, 20.0, 21.0, 22.0, 23.0, 24.0], + [25.0, 26.0, 27.0, 28.0, 29.0, 30.0], + [31.0, 32.0, 33.0, 34.0, 35.0, 36.0], + ]]], + &device, + ) + .require_grad(); + + // Expected gradients for ceil_mode output 3x3: + // Output positions and their max value positions: + // (0,0): max at (2,2)=15 -> grad[2,2] += 1 + // (0,1): max at (2,4)=17 -> grad[2,4] += 1 + // (0,2): max at (2,5)=18 -> grad[2,5] += 1 + // (1,0): max at (4,2)=27 -> grad[4,2] += 1 + // (1,1): max at (4,4)=29 -> grad[4,4] += 1 + // (1,2): max at (4,5)=30 -> grad[4,5] += 1 + // (2,0): max at (5,2)=33 -> grad[5,2] += 1 + // (2,1): max at (5,4)=35 -> grad[5,4] += 1 + // (2,2): max at (5,5)=36 -> grad[5,5] += 1 + let x_grad_expected = TestAutodiffTensor::<4>::from_floats( + [[[ + [0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0.], + [0., 0., 1., 0., 1., 1.], + [0., 0., 0., 0., 0., 0.], + [0., 0., 1., 0., 1., 1.], + [0., 0., 1., 0., 1., 1.], + ]]], + &device, + ); + + let output = max_pool2d( + x.clone(), + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + true, + ); + let grads = output.backward(); + + // Asserts + let x_grad_actual = x.grad(&grads).unwrap(); + x_grad_expected + .to_data() + .assert_approx_eq::(&x_grad_actual.to_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/memory_management.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/memory_management.rs new file mode 100644 index 0000000..f33196b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/memory_management.rs @@ -0,0 +1,290 @@ +use super::*; +use burn_tensor::{Tensor, TensorData}; + +#[test] +fn test_mm_independent_trees() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // First tree + let tensor_0 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_4 = tensor_0 * tensor_1; + let tensor_5 = tensor_2 * tensor_3; + let tensor_6 = tensor_4 * tensor_5; + + // Second tree + let tensor_7 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_8 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_9 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_10 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_11 = tensor_7.clone() * tensor_8.clone(); + let tensor_12 = tensor_9.clone() * tensor_10.clone(); + let tensor_13 = tensor_11 * tensor_12; + + let _grads = tensor_6.backward(); + let grads = tensor_13.backward(); + + assert!(tensor_7.grad(&grads).is_some()); + assert!(tensor_8.grad(&grads).is_some()); + assert!(tensor_9.grad(&grads).is_some()); + assert!(tensor_10.grad(&grads).is_some()); +} + +#[test] +#[should_panic] +fn test_mm_crossover_trees_root_unavailable() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // First tree + let tensor_0 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_4 = tensor_0 * tensor_1; + let tensor_5 = tensor_2 * tensor_3; + let tensor_6 = tensor_4.clone() * tensor_5; + + // Second tree + let tensor_7 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_8 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_9 = tensor_7.clone() * tensor_8.clone(); + let tensor_10 = tensor_4 * tensor_9; + + let _grads = tensor_6.backward(); + let _grads = tensor_10.backward(); +} + +#[test] +fn test_mm_crossover_trees_with_referred_subtree() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // First tree + let tensor_0 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_4 = tensor_0 * tensor_1; + let tensor_5 = tensor_2 * tensor_3; + let tensor_6 = tensor_4.clone() * tensor_5; + + // Second tree + let tensor_7 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_8 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_9 = tensor_7.clone() * tensor_8.clone(); + let _tensor_10 = tensor_4 * tensor_9.clone(); + + let _grads = tensor_6.backward(); + let _grads = tensor_9.backward(); +} + +#[test] +fn test_mm_three_crossover_trees_last_still_usable() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // First tree + let tensor_0 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_4 = tensor_0 * tensor_1; + let tensor_5 = tensor_2 * tensor_3; + let tensor_6 = tensor_4 * tensor_5.clone(); + + // Third tree + let tensor_7 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_8 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_9 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_10 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_11 = tensor_7 * tensor_8; + let tensor_12 = tensor_9 * tensor_10; + let tensor_13 = tensor_11 * tensor_12.clone(); + + // Second tree (in between) + let _tensor_14 = tensor_5 * tensor_12; + + let _grads = tensor_6.backward(); + let _grads = tensor_13.backward(); +} + +#[test] +#[should_panic] +fn test_mm_three_crossover_trees_middle_one_unavailable() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // First tree + let tensor_0 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_4 = tensor_0 * tensor_1; + let tensor_5 = tensor_2 * tensor_3; + let tensor_6 = tensor_4 * tensor_5.clone(); + + // Third tree + let tensor_7 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_8 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_9 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_10 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_11 = tensor_7 * tensor_8; + let tensor_12 = tensor_9 * tensor_10; + let _tensor_13 = tensor_11 * tensor_12.clone(); + + // Second tree (in between) + let tensor_14 = tensor_5 * tensor_12; + + let _grads = tensor_6.backward(); + let _grads = tensor_14.backward(); +} + +#[test] +fn test_mm_self_referencing_tree() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // First tree + let tensor_0 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_1 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data.clone(), &device).require_grad(); + + let tensor_3 = tensor_0 * tensor_1; + let tensor_5 = tensor_2 * tensor_3.clone(); + let tensor_6 = tensor_3 * tensor_5; + + let _grads = tensor_6.backward(); +} + +#[test] +fn test_mm_with_non_impacting_detach() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + let tensor_1 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_2 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_3 = Tensor::::from_data(data, &device).require_grad(); + + let tensor_4 = tensor_1.clone() * tensor_2.clone(); + let tensor_5 = tensor_4.detach() * tensor_3.clone(); + + let grads = tensor_5.backward(); + assert!(tensor_3.grad(&grads).is_some()); +} + +#[test] +fn test_mm_with_missing_require_grad_after_cleanup() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + let tensor_1 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_2 = Tensor::::from_data(data.clone(), &device); + let tensor_3 = Tensor::::from_data(data.clone(), &device); + + let tensor_4 = tensor_1.clone() * tensor_2.clone(); + let tensor_5 = tensor_4 * tensor_3.clone(); + + // Trivial backward, just to trigger cleanup + Tensor::::from_data(data, &device) + .require_grad() + .backward(); + + let grads = tensor_5.backward(); + assert!(tensor_1.grad(&grads).is_some()); + assert!(tensor_2.grad(&grads).is_none()); + assert!(tensor_3.grad(&grads).is_none()); +} + +#[test] +fn test_mm_with_detach_after_cleanup() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + let tensor_1 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_2 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_3 = + Tensor::::from_data(data.clone(), &device).require_grad(); + + let tensor_4 = tensor_1.clone() * tensor_2.clone(); + let tensor_5 = tensor_4 * tensor_3.clone().detach(); + + // Trivial backward, just to trigger cleanup + Tensor::::from_data(data, &device) + .require_grad() + .backward(); + + let grads = tensor_5.backward(); + assert!(tensor_1.grad(&grads).is_some()); + assert!(tensor_2.grad(&grads).is_some()); + assert!(tensor_3.grad(&grads).is_none()); +} + +#[test] +#[should_panic] +fn test_mm_deletables_propagate_well() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + let tensor_0 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_1 = + Tensor::::from_data(data.clone(), &device).require_grad(); + + let tensor_2 = tensor_0 * tensor_1; + let tensor_3 = tensor_2.clone().exp(); + let _tensor_4 = tensor_3.clone().log(); + + let _grads = tensor_2.backward(); + + // We are testing that after backward on tensor_2, not only the leaf tensor_4 is deleted, but + // the intermediate tensor_3 as well + let _grads = tensor_3.backward(); +} + +#[test] +fn test_mm_node_explored_once_can_still_be_tagged_as_useful_when_found_again_deeper() { + let data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let device = Default::default(); + + // The test has 50% chance of starting with leaf tensor_8 instead of tensor_4, which is not informative + // By repeating it many times it becomes almost impossible that it passes if it shouldn't + for _ in 0..12 { + let tensor_0 = + Tensor::::from_data(data.clone(), &device).require_grad(); + let tensor_1 = + Tensor::::from_data(data.clone(), &device).require_grad(); + + let tensor_2 = tensor_1.clone().exp(); + let tensor_3 = tensor_0.exp(); + let _tensor_4 = tensor_3.clone() * tensor_2.clone(); + let tensor_5 = tensor_2.exp(); + let tensor_6 = tensor_5.exp(); + let tensor_7 = tensor_6.exp(); + let tensor_8 = tensor_7.exp(); + + // tensor_2 should be tagged unknown through the leaf tensor_4, then useful through the leaf tensor_8 + // which should happen after because tensor_2 is deeper from tensor_8 point of view and we're in breadth first search + tensor_3.backward(); + let grads = tensor_8.backward(); + + assert!(tensor_1.grad(&grads).is_some()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mod.rs new file mode 100644 index 0000000..2f6696b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mod.rs @@ -0,0 +1,74 @@ +#[allow(unused_imports)] // required for re-included modules +pub use super::*; + +mod abs; +mod adaptive_avgpool1d; +mod adaptive_avgpool2d; +mod add; +mod aggregation; +mod avgpool1d; +mod avgpool2d; +mod backward; +mod bridge; +mod broadcast; +mod cast; +mod cat; +mod ceil; +mod checkpoint; +mod complex; +mod conv1d; +mod conv2d; +mod conv3d; +mod conv_transpose1d; +mod conv_transpose2d; +mod conv_transpose3d; +mod cross; +mod cross_entropy; +mod cummax; +mod cummin; +mod cumprod; +mod cumsum; +mod deform_conv2d; +mod div; +mod erf; +mod exp; +mod expand; +mod flip; +mod floor; +mod gather_scatter; +mod gelu; +mod gradients; +mod log; +mod log1p; +mod log_sigmoid; +mod mask; +mod matmul; +mod maxmin; +mod maxpool1d; +mod maxpool2d; +mod memory_management; +mod mul; +mod multithread; +mod nearest_interpolate; +mod neg; +mod nonzero; +mod permute; +mod pow; +mod recip; +mod relu; +mod remainder; +mod repeat_dim; +mod reshape; +mod round; +mod select; +mod sigmoid; +mod sign; +mod slice; +mod slice_assign; +mod softmax; +mod sort; +mod sqrt; +mod sub; +mod transpose; +mod trig; +mod unfold; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mul.rs new file mode 100644 index 0000000..982227f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/mul.rs @@ -0,0 +1,68 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_mul() { + let data_1 = TensorData::from([1.0, 7.0]); + let data_2 = TensorData::from([4.0, 7.0]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data(data_1.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2.clone(), &device).require_grad(); + + let tensor_3 = tensor_1.clone().mul(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let _grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_eq(&data_2, false); + tensor_3 + .into_data() + .assert_eq(&TensorData::from([4.0, 49.0]), false); +} + +#[test] +fn should_diff_mul_scalar() { + let data = TensorData::from([2.0, 5.0]); + + let tensor = TestAutodiffTensor::<1>::from_data(data, &Default::default()).require_grad(); + let tensor_out = tensor.clone().mul_scalar(4.0); + + let grads = tensor_out.backward(); + let grad = tensor.grad(&grads).unwrap(); + + tensor_out + .into_data() + .assert_eq(&TensorData::from([8.0, 20.0]), false); + grad.to_data() + .assert_eq(&TensorData::from([4.0, 4.0]), false); +} + +#[test] +fn test_mul_complex_1() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[2.0, 2.0], [2.0, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().mul(tensor_2.clone()); + let tensor_5 = tensor_4.mul(tensor_3); + let tensor_6 = tensor_1.clone().mul(tensor_5); + + let grads = tensor_6.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[16.0, 196.0], [104.0, -36.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[2.0, 98.0], [338.0, 18.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/multithread.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/multithread.rs new file mode 100644 index 0000000..62c3efc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/multithread.rs @@ -0,0 +1,88 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_behave_the_same_with_multithread() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let with_move = || { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2.clone(), &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.clone().matmul(tensor_2.clone()); + let tensor_5 = tensor_4.matmul(tensor_3); + + // Task 1 + let tensor_1_cloned = tensor_1.clone(); + let tensor_2_cloned = tensor_2.clone(); + let tensor_5_cloned = tensor_5.clone(); + + let first_call = move || { + let tensor_6_1 = tensor_5_cloned.matmul(tensor_2_cloned); + tensor_6_1.matmul(tensor_1_cloned) + }; + + // Task 2 + let tensor_1_cloned = tensor_1.clone(); + let tensor_2_cloned = tensor_2.clone(); + let tensor_5_cloned = tensor_5; + + let second_call = move || { + let tensor_6_2 = tensor_5_cloned.matmul(tensor_1_cloned); + tensor_6_2.matmul(tensor_2_cloned) + }; + + let tensor_7_1_handle = std::thread::spawn(first_call); + let tensor_7_2_handle = std::thread::spawn(second_call); + + let tensor_7_1 = tensor_7_1_handle.join().unwrap(); + let tensor_7_2 = tensor_7_2_handle.join().unwrap(); + let tensor_8 = tensor_7_1.matmul(tensor_7_2); + + let grads = tensor_8.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + (grad_1, grad_2) + }; + let without_move = || { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1.clone(), &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2.clone(), &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.clone().matmul(tensor_2.clone()); + let tensor_5 = tensor_4.matmul(tensor_3); + + // Task 1 + let tensor_6_1 = tensor_5.clone().matmul(tensor_2.clone()); + let tensor_7_1 = tensor_6_1.matmul(tensor_1.clone()); + + // Task 2 + let tensor_6_2 = tensor_5.matmul(tensor_1.clone()); + let tensor_7_2 = tensor_6_2.matmul(tensor_2.clone()); + + let tensor_8 = tensor_7_1.matmul(tensor_7_2); + + let grads = tensor_8.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + (grad_1, grad_2) + }; + + let (grad_1, grad_2) = without_move(); + let (grad_1_moved, grad_2_moved) = with_move(); + + grad_1 + .into_data() + .assert_approx_eq::(&grad_1_moved.into_data(), Tolerance::default()); + grad_2 + .into_data() + .assert_approx_eq::(&grad_2_moved.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nearest_interpolate.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nearest_interpolate.rs new file mode 100644 index 0000000..bf0156f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nearest_interpolate.rs @@ -0,0 +1,97 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::interpolate; +use burn_tensor::ops::{InterpolateMode, InterpolateOptions}; + +#[test] +fn test_upsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 2, + channels: 1, + height: 7, + width: 5, + height_out: 8, + width_out: 7, + }; + + test.assert_output(TestTensor::from([ + [[ + [4., 2., 4., 2., 2.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + ]], + [[ + [4., 2., 4., 2., 2.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + [2., 1., 2., 1., 1.], + ]], + ])); +} + +#[test] +fn test_downsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 8, + width: 8, + height_out: 4, + width_out: 6, + }; + + test.assert_output(TestTensor::from([[[ + [1., 1., 1., 0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [1., 1., 1., 0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [1., 1., 1., 0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [1., 1., 1., 0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + ]]])); +} + +struct InterpolateTestCase { + batch_size: usize, + channels: usize, + height: usize, + width: usize, + height_out: usize, + width_out: usize, +} + +impl InterpolateTestCase { + fn assert_output(self, x_grad: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let device = Default::default(); + let x = TestAutodiffTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &x_grad.device()) + .reshape::<4, _>(shape_x) + .into_data(), + &device, + ) + .require_grad(); + + let output = interpolate( + x.clone(), + [self.height_out, self.width_out], + InterpolateOptions::new(InterpolateMode::Nearest), + ); + + let grads = output.backward(); + let x_grad_actual = x.grad(&grads).unwrap(); + + x_grad + .to_data() + .assert_approx_eq::(&x_grad_actual.into_data(), Tolerance::permissive()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/neg.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/neg.rs new file mode 100644 index 0000000..e160a99 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/neg.rs @@ -0,0 +1,26 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_neg() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().neg()); + let tensor_4 = tensor_3.neg(); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[11.0, 5.0], [11.0, 5.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[3.0, 3.0], [10.0, 10.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nonzero.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nonzero.rs new file mode 100644 index 0000000..d54f2ea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/nonzero.rs @@ -0,0 +1,41 @@ +use super::*; +use burn_tensor::{Bool, Tensor, TensorData}; + +#[test] +fn should_diff_nonzero() { + let data_1 = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let data_2 = TensorData::from([-1.0, 1.0]); + let mask = TensorData::from([[false, true], [true, false]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::<1>::from_data(data_2, &device).require_grad(); + + // Multi-dimensional tensor indexing isn't really supported yet so the easiest way to do + // this is to flatten the mask and tensor to get proper indexing. Anyway the returned tensor would + // have dimensions different from the input, so this is somewhat equivalent. + let mask = Tensor::::from_bool(mask, &device).flatten::<1>(0, 1); + let indices = mask.nonzero(); + let tensor_3 = tensor_1 + .clone() + .flatten::<1>(0, 1) + .select(0, indices[0].clone()); + + // Vector dot product not supported (only 2D matmuls) so unsqueeze for test purposes + let tensor_4 = tensor_2 + .clone() + .unsqueeze_dim::<2>(0) + .matmul(tensor_3.unsqueeze_dim(1)); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[0.0, -1.0], [1.0, 0.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([2.0, 3.0]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/permute.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/permute.rs new file mode 100644 index 0000000..d25f0c2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/permute.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_permute() { + let data_1 = TensorData::from([[[1.0, 7.0], [2.0, 3.0]]]); // 1x2x2 + let data_2 = TensorData::from([[[1.0, 7.0], [3.2, 2.0], [3.0, 3.0]]]); // 1x3x2 + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_2.clone().permute([0, 2, 1]); + let tensor_4 = tensor_1.clone().matmul(tensor_3); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + grad_1 + .into_data() + .assert_approx_eq::(&TensorData::from([[[7.2, 12.0], [7.2, 12.0]]]), tolerance); // 1x2x2 + grad_2.into_data().assert_approx_eq::( + &TensorData::from([[[3.0, 10.0], [3.0, 10.0], [3.0, 10.0]]]), + tolerance, + ); // 1x3x2 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/pow.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/pow.rs new file mode 100644 index 0000000..c199b03 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/pow.rs @@ -0,0 +1,93 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_powf_scalar() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().powf_scalar(0.4)); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(2e-3); + let expected = TensorData::from([[68.0, 79.0328], [68.0, 79.0328]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[23.5081, 25.2779], [26.0502, 28.6383]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn should_diff_powf() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data([2.0, 7.0], &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data([4.0, 2.0], &device).require_grad(); + + let tensor_3 = tensor_1.clone().powf(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([32.0, 14.0]); + grad_1 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([11.09035, 95.34960]); + grad_2 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([16.0, 49.0]); + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_powf_with_untracked_lhs() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data([2.0, 7.0], &device); + let tensor_2 = TestAutodiffTensor::from_data([4.0, 2.0], &device).require_grad(); + + let tensor_3 = tensor_1.clone().powf(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([11.09035, 95.34960]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_powf_with_untracked_rhs() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data([2.0, 7.0], &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data([4.0, 2.0], &device); + + let tensor_3 = tensor_1.clone().powf(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + + let expected = TensorData::from([32.0, 14.0]); + grad_1 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/recip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/recip.rs new file mode 100644 index 0000000..9cb0ea3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/recip.rs @@ -0,0 +1,22 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_recip() { + let data = TensorData::from([2.0, 5.0, 0.4]); + + let tensor = TestAutodiffTensor::<1>::from_data(data, &Default::default()).require_grad(); + let tensor_out = tensor.clone().recip(); + + let grads = tensor_out.backward(); + let grad = tensor.grad(&grads).unwrap(); + + tensor_out + .into_data() + .assert_eq(&TensorData::from([0.5, 0.2, 2.5]), false); + grad.to_data().assert_approx_eq::( + &TensorData::from([-0.25, -0.04, -6.25]), + Tolerance::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/relu.rs new file mode 100644 index 0000000..68e200f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/relu.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::{TensorData, activation}; + +#[test] +fn should_diff_relu() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, -7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = activation::relu(tensor_3); + let tensor_5 = tensor_4.matmul(tensor_2.clone()); + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[-47.0, 9.0], [-35.0, 15.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[15.0, 13.0], [-2.0, 39.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/remainder.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/remainder.rs new file mode 100644 index 0000000..c7eaec6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/remainder.rs @@ -0,0 +1,41 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_remainder() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data( + TensorData::from([ + 0.9742, 0.3676, 0.0905, 0.8066, 0.7072, 0.7883, 0.6987, 0.1560, 0.7179, 0.7874, 0.9032, + 0.1845, + ]), + &device, + ) + .require_grad(); + let tensor_2 = TestAutodiffTensor::<1>::from_data( + TensorData::from([ + 0.3357, 0.0285, 0.4115, 0.5511, 0.8637, 0.3593, 0.3885, 0.2569, 0.0936, 0.7172, 0.4792, + 0.4898, + ]), + &device, + ) + .require_grad(); + let tensor_3 = tensor_1.clone().remainder(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([ + -2.0, -12.0, -0.0, -1.0, -0.0, -2.0, -1.0, -0.0, -7.0, -1.0, -1.0, -0.0, + ]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/repeat_dim.rs new file mode 100644 index 0000000..0442513 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/repeat_dim.rs @@ -0,0 +1,44 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_repeat() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0], [2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_2.clone().repeat_dim(1, 3); + + let tensor_3 = tensor_1.matmul(tensor_3); + let grads = tensor_3.backward(); + + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_2 + .to_data() + .assert_eq(&TensorData::from([[-3.0], [12.0]]), false); +} + +#[test] +fn should_diff_repeat_multi_dim() { + let data_1 = TensorData::from([[1.0, 7.0], [-2.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 2.0], [2.0, 4.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_2.clone().repeat_dim(1, 3); + + let tensor_3 = tensor_1.matmul(tensor_3); + let grads = tensor_3.backward(); + + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_2 + .to_data() + .assert_eq(&TensorData::from([[-3.0, -3.0], [12.0, 12.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/reshape.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/reshape.rs new file mode 100644 index 0000000..02b4d3c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/reshape.rs @@ -0,0 +1,26 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_reshape() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([4.0, 7.0, 2.0, 3.0]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::<1>::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_2.clone().reshape([2, 2]); + let tensor_4 = tensor_1.clone().matmul(tensor_3); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[11.0, 5.0], [11.0, 5.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([3.0, 3.0, 10.0, 10.0]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/round.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/round.rs new file mode 100644 index 0000000..611e2c0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/round.rs @@ -0,0 +1,20 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_round() { + let data = TensorData::from([ + [-1.9751, 0.0714, 0.0643, 0.2406], + [-1.3172, 0.1252, -0.1119, -0.0127], + ]); + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data.clone(), &device).require_grad(); + let tensor_2 = tensor_1.clone().round(); + let grads = tensor_2.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + grad_1.to_data().assert_eq( + &TensorData::from([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/select.rs new file mode 100644 index 0000000..d857f14 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/select.rs @@ -0,0 +1,89 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, Int, Tensor, TensorData}; + +#[test] +fn test_select_grad() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data( + TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]), + &device, + ) + .require_grad(); + let indices = + Tensor::::from_data(TensorData::from([1, 0]), &device); + + let tensor_2 = tensor_1.clone().matmul(tensor_1.clone().transpose()); + let tensor_3 = tensor_1.clone().select(0, indices); + let tensor_4 = tensor_2.matmul(tensor_3); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + + grad_1.into_data().assert_eq( + &TensorData::from([[109., 148., 187.], [37., 58., 79.]]), + false, + ); +} + +#[test] +fn test_select_add_grad() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data( + TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]), + &device, + ) + .require_grad(); + let values = TestAutodiffTensor::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ) + .require_grad(); + let indices = + Tensor::::from_data(TensorData::from([1, 0]), &device); + + let tensor_2 = tensor_1.clone().matmul(tensor_1.clone().transpose()); + let tensor_3 = + tensor_1 + .clone() + .select_assign(0, indices, values.clone(), IndexingUpdateOp::Add); + let tensor_4 = tensor_2.matmul(tensor_3); + + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = values.grad(&grads).unwrap(); + + grad_1.into_data().assert_eq( + &TensorData::from([[127., 199., 271.], [172., 244., 316.]]), + false, + ); + grad_2 + .into_data() + .assert_eq(&TensorData::from([[64., 64., 64.], [19., 19., 19.]]), false); +} + +#[test] +fn test_select_add_grad_different_shapes() { + let device = Default::default(); + + let indices: Tensor = Tensor::from_ints([1], &device); + let x: Tensor = Tensor::ones([1, 1], &device).require_grad(); + let y = Tensor::ones([2, 1], &device).require_grad(); + + let w = y + .clone() + .select_assign(0, indices, x.clone(), IndexingUpdateOp::Add); + let w = w.matmul(y.clone().transpose()); + + let grads = w.backward(); + let x_grad = x.grad(&grads).unwrap(); + let y_grad = y.grad(&grads).unwrap(); + + x_grad + .into_data() + .assert_eq(&TensorData::from([[2.0]]), false); + y_grad + .into_data() + .assert_eq(&TensorData::from([[5.0], [5.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sigmoid.rs new file mode 100644 index 0000000..d4096a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sigmoid.rs @@ -0,0 +1,35 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn should_diff_sigmoid() { + let data = TensorData::from([0.8762]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data(data, &device).require_grad(); + let tensor_2 = activation::sigmoid(tensor_1.clone()); + let grads = tensor_2.backward(); + + let grad = tensor_1.grad(&grads).unwrap(); + + let expected = TensorData::from([0.207549]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn small_neg_val_should_not_cause_grad_overflow() { + let data = TensorData::from([-90.0]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data(data, &device).require_grad(); + let tensor_2 = activation::sigmoid(tensor_1.clone()); + let grads = tensor_2.backward(); + + let grad = tensor_1.grad(&grads).unwrap(); + + let expected = TensorData::from([0.0]); + grad.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sign.rs new file mode 100644 index 0000000..63621b8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sign.rs @@ -0,0 +1,42 @@ +use super::*; +use burn_tensor::TensorData; + +/// Example using the sign function with PyTorch: +// >>> import torch +// >>> # Create a tensor with requires_grad=True +// >>> x = torch.tensor([-2.0, -1.0, 0.0, 1.0, 2.0], requires_grad=True) +// >>> # Forward pass: Apply the sign function +// >>> y = torch.sign(x) +// >>> print("Forward pass:") +// Forward pass: +// >>> print("x:", x) +// x: tensor([-2., -1., 0., 1., 2.], requires_grad=True) +// >>> print("y:", y) +// y: tensor([-1., -1., 0., 1., 1.], grad_fn=) +// >>> # Compute the loss (just an example) +// >>> loss = y.sum() +// >>> # Backward pass: Compute the gradients +// >>> loss.backward() +// >>> print("\nBackward pass:") +// Backward pass: +// >>> print("x.grad:", x.grad) +// x.grad: tensor([0., 0., 0., 0., 0.]) + +#[test] +fn should_diff_sign() { + let data = TensorData::from([-2.0, -1.0, 0.0, 1.0, 2.0]); + + let device = Default::default(); + let x = TestAutodiffTensor::<1>::from_data(data, &device).require_grad(); + + let y = x.clone().sign(); + + let loss = y.clone().sum(); + let grads = loss.backward(); + let grad = x.grad(&grads).unwrap(); + + y.to_data() + .assert_eq(&TensorData::from([-1., -1., 0., 1., 1.]), false); + grad.to_data() + .assert_eq(&TensorData::from([0., 0., 0., 0., 0.]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice.rs new file mode 100644 index 0000000..fb89bbf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice.rs @@ -0,0 +1,67 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_matmul_with_slice() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0, 100.0], [2.0, 3.0, 15.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_2.clone().slice([0..2, 0..2]); + let tensor_4 = tensor_1.clone().matmul(tensor_3); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[11.0, 5.0], [11.0, 5.0]]), false); + grad_2.to_data().assert_eq( + &TensorData::from([[3.0, 3.0, 0.0], [10.0, 10.0, 0.0]]), + false, + ); +} + +#[test] +fn should_diff_matmul_with_slice_stepped() { + use burn_tensor::s; + let data_1 = TensorData::from([[1.0, 7.0], [100.0, 100.0], [2.0, 3.0], [100.0, 100.0]]); + let data_2 = TensorData::from([[4.0, 100.0, 7.0, 100.0], [2.0, 100.0, 3.0, 15.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().slice(s![0..;2, 0..2]); // [[1., 7.], [2., 3.]] + let tensor_4 = tensor_2.clone().slice(s![0..2, 0..;2]); // [[4., 7.], [2., 3.]] + let tensor_5 = tensor_3.clone().matmul(tensor_4); + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_eq( + &TensorData::from([[11., 5.], [0., 0.], [11., 5.], [0., 0.]]), + false, + ); + grad_2.to_data().assert_eq( + &TensorData::from([[3., 0., 3., 0.], [10., 0., 10., 0.]]), + false, + ); +} + +#[test] +fn should_panic_on_slice_with_step() { + use burn_tensor::s; + + let data = TensorData::from([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]]); + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data(data, &device).require_grad(); + + // This should panic because step is 2 + let _sliced = tensor.slice(s![.., 0..4; 2]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice_assign.rs new file mode 100644 index 0000000..07a3b3d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/slice_assign.rs @@ -0,0 +1,163 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_matmul_with_slice_assign() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_assigned = TensorData::from([[9.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_assigned = TestAutodiffTensor::from_data(data_assigned, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_3.slice_assign([0..1, 0..1], tensor_assigned); + let tensor_5 = tensor_4.matmul(tensor_1.clone()); + + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[58.0, 38.0], [118.0, 82.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[16.0, 15.0], [24.0, 50.0]]), false); +} + +#[test] +fn should_diff_matmul_with_slice_assign_complex() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[9.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_5 = tensor_2.clone().slice([0..1, 0..1]); + let tensor_6 = tensor_5.mul(tensor_3.clone()); + let tensor_7 = tensor_4.slice_assign([0..1, 0..1], tensor_6); + let tensor_8 = tensor_7.matmul(tensor_1.clone()); + + let grads = tensor_8.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + let grad_3 = tensor_3.grad(&grads).unwrap(); + + grad_3 + .to_data() + .assert_eq(&TensorData::from([[32.0]]), false); + grad_1 + .to_data() + .assert_eq(&TensorData::from([[85.0, 65.0], [118.0, 82.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[88.0, 15.0], [24.0, 50.0]]), false); +} + +#[test] +fn slice_assign_diff_should_give_same_results_as_cat() { + let data_1 = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[5.0, 6.0], [7.0, 8.0]]); + let data_3 = TensorData::from([[14.0, 97.0, 100.0, 9.0], [2.0, 3.0, 15.0, 7.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device); + + let slice_assign_output = TestAutodiffTensor::zeros([2, 4], &Default::default()); + let slice_assign_output = slice_assign_output.slice_assign([0..2, 0..2], tensor_1.clone()); + let slice_assign_output = slice_assign_output.slice_assign([0..2, 2..4], tensor_2.clone()); + let slice_assign_output = slice_assign_output / tensor_3.clone(); + + let cat_output = TestAutodiffTensor::cat(vec![tensor_1.clone(), tensor_2.clone()], 1); + let cat_output = cat_output / tensor_3; + + slice_assign_output + .to_data() + .assert_approx_eq::(&cat_output.to_data(), Tolerance::default()); + + let slice_assign_grads = slice_assign_output.backward(); + let cat_grads = cat_output.backward(); + + let slice_assign_grad_1 = tensor_1.grad(&slice_assign_grads).unwrap(); + let slice_assign_grad_2 = tensor_2.grad(&slice_assign_grads).unwrap(); + let cat_grad_1 = tensor_1.grad(&cat_grads).unwrap(); + let cat_grad_2 = tensor_2.grad(&cat_grads).unwrap(); + + slice_assign_grad_1 + .to_data() + .assert_approx_eq::(&cat_grad_1.to_data(), Tolerance::default()); + slice_assign_grad_2 + .to_data() + .assert_approx_eq::(&cat_grad_2.to_data(), Tolerance::default()); +} + +#[test] +fn should_diff_slice_assign_with_step() { + use burn_tensor::s; + let data = TensorData::from([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]); + let value_data = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data(data, &device).require_grad(); + let value = TestAutodiffTensor::<2>::from_data(value_data, &device).require_grad(); + + // Assign with step=2 + let result = tensor.clone().slice_assign(s![.., 0..4; 2], value.clone()); + let result = result * 2.0; // Scale to create gradients + let grads = result.backward(); + + let grad_tensor = tensor.grad(&grads).unwrap(); + let grad_value = value.grad(&grads).unwrap(); + + // The gradient for tensor should be 2.0 everywhere except the assigned positions + grad_tensor.to_data().assert_eq( + &TensorData::from([[0.0, 2.0, 0.0, 2.0], [0.0, 2.0, 0.0, 2.0]]), + false, + ); + // The gradient for value should be 2.0 at all positions + grad_value + .to_data() + .assert_eq(&TensorData::from([[2.0, 2.0], [2.0, 2.0]]), false); +} + +#[test] +fn should_diff_slice_assign_with_negative_step() { + use burn_tensor::s; + + let data = TensorData::from([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]); + let value_data = TensorData::from([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]]); + let device = Default::default(); + let tensor = TestAutodiffTensor::<2>::from_data(data, &device).require_grad(); + let value = TestAutodiffTensor::<2>::from_data(value_data, &device).require_grad(); + + // Assign with step=-1 (reverse order, all elements) + let result = tensor.clone().slice_assign(s![.., ..;-1], value.clone()); + let result = result * 2.0; // Scale to create gradients + let grads = result.backward(); + + let grad_tensor = tensor.grad(&grads).unwrap(); + let grad_value = value.grad(&grads).unwrap(); + + // The gradient for tensor should be 0 since all values were replaced + grad_tensor.to_data().assert_eq( + &TensorData::from([[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]), + false, + ); + // The gradient for value should be 2.0 at all positions + grad_value.to_data().assert_eq( + &TensorData::from([[2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/softmax.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/softmax.rs new file mode 100644 index 0000000..9a36af4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/softmax.rs @@ -0,0 +1,90 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Tensor, TensorData, activation}; + +#[test] +fn test_softmax_grad() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + let device = Default::default(); + let tensor_1 = Tensor::::from_data(data_1, &device).require_grad(); + let tensor_2 = Tensor::::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = activation::softmax(tensor_3, 1).matmul(tensor_2.clone()); + + let grads = tensor_4.backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[1.179665, 1.179661], [0.005462, 0.005463]]); + + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(0.05, 0.5)); + + let expected = TensorData::from([[0.253469, 0.286237], [0.528630, 2.931664]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(0.05, 0.05)); +} + +#[test] +fn test_log_softmax_grad() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + let device = Default::default(); + let tensor_1 = Tensor::::from_data(data_1, &device).require_grad(); + let tensor_2 = Tensor::::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = activation::log_softmax(tensor_3, 1).matmul(tensor_2.clone()); + + let grads = tensor_4.backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[-4.3939, -4.3939], [-12.9709, -12.9709]]); + // f16 gradients from log-softmax + matmul amplify error, so we increase the tolerance + // to account for limited precision and large representable step sizes in this range. + let tolerance = Tolerance::permissive().set_half_precision_relative(6e-2); + + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[30.5984, -47.2267], [55.9631, -56.5914]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_quiet_softmax_grad() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = Tensor::::from_data(data_1, &device).require_grad(); + let tensor_2 = Tensor::::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = activation::softmax(tensor_3, 1).matmul(tensor_2.clone()); + + let grads = tensor_4.backward(); + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[1.179665, 1.179661], [0.005462, 0.005463]]); + + // Precision is quite bad yet on softmax grad especially with half precision. + let tolerance = Tolerance::rel_abs(0.5, 0.2); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[0.253469, 0.286237], [0.528630, 2.931664]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sort.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sort.rs new file mode 100644 index 0000000..61d1fab --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sort.rs @@ -0,0 +1,82 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_sort() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_floats([[1.0, 7.0], [-2.0, -3.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::from_floats([[4.0, -7.0], [2.0, 3.0]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.sort(1)); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[35.0, 35.0], [-1.0, -8.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[11.0, 7.0], [55.0, 16.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_sort_with_indices() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<2>::from_floats([[1.0, 7.0], [-2.0, -3.0]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::from_floats([[4.0, -7.0], [2.0, 3.0]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let (values, _indices) = tensor_3.sort_with_indices(1); + let tensor_4 = tensor_1.clone().mul(values); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[35.0, 35.0], [-1.0, -8.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[11.0, 7.0], [55.0, 16.0]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_diff_sort_3d_dim1() { + let device = Default::default(); + let tensor_1 = + TestAutodiffTensor::<3>::from_floats([[[1.0, 7.0], [-2.0, -3.0]]], &device).require_grad(); + let tensor_2 = + TestAutodiffTensor::from_floats([[[4.0, -7.0], [2.0, 3.0]]], &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone()); + let tensor_4 = tensor_1.clone().mul(tensor_3.sort(1)); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let expected = TensorData::from([[[-1., -8.], [-27., 37.]]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[[-4., -17.], [-17., -42.]]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sqrt.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sqrt.rs new file mode 100644 index 0000000..4263c15 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sqrt.rs @@ -0,0 +1,31 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_sqrt() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().sqrt()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + let expected = TensorData::from([[82.112640, 99.083275], [82.112640, 99.083275]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[30.309311, 33.120457], [34.581974, 38.769463]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sub.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sub.rs new file mode 100644 index 0000000..5be6786 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/sub.rs @@ -0,0 +1,73 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_diff_sub() { + let data_1 = TensorData::from([2.0, 5.0]); + let data_2 = TensorData::from([4.0, 1.0]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<1>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().sub(tensor_2.clone()); + let grads = tensor_3.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([1.0, 1.0]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([-1.0, -1.0]), false); + + tensor_3 + .into_data() + .assert_eq(&TensorData::from([-2.0, 4.0]), false); +} + +#[test] +fn should_diff_sub_scalar() { + let data = TensorData::from([2.0, 10.0]); + let tensor = TestAutodiffTensor::<1>::from_data(data, &Default::default()).require_grad(); + let tensor_out = tensor.clone().sub_scalar(5.0); + let grads = tensor_out.backward(); + + let grad = tensor.grad(&grads).unwrap(); + + grad.to_data() + .assert_eq(&TensorData::from([1.0, 1.0]), false); + tensor_out + .into_data() + .assert_eq(&TensorData::from([-3.0, 5.0]), false); +} + +#[test] +fn test_sub_complex_1() { + let data_1 = TensorData::from([[1.0, 7.0], [13.0, -3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + let data_3 = TensorData::from([[2.0, 2.0], [2.0, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1.clone().sub(tensor_2.clone()); + let tensor_5 = tensor_4.sub(tensor_3).sub_scalar(5.0); + let tensor_6 = tensor_1.clone().sub(tensor_5); + + let grads = tensor_6.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1 + .to_data() + .assert_eq(&TensorData::from([[0.0, 0.0], [0.0, 0.0]]), false); + grad_2 + .to_data() + .assert_eq(&TensorData::from([[1.0, 1.0], [1.0, 1.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/transpose.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/transpose.rs new file mode 100644 index 0000000..8b7cd66 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/transpose.rs @@ -0,0 +1,60 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_transpose() { + let data_1 = TensorData::from([[1.0, 7.0], [2.0, 3.0]]); + let data_2 = TensorData::from([[4.0, 7.0], [2.0, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().transpose()); + let tensor_4 = tensor_3.transpose(); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[6.0, 10.0], [6.0, 10.0]]), + Tolerance::default(), + ); + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[3.0, 10.0], [3.0, 10.0]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_swap_dims() { + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<3>::from_floats( + [[[0.0, 1.0], [3.0, 4.0]], [[6.0, 7.0], [9.0, 10.0]]], + &device, + ) + .require_grad(); + let tensor_2 = TestAutodiffTensor::from_floats( + [[[1.0, 4.0], [2.0, 5.0]], [[7.0, 10.0], [8.0, 11.0]]], + &device, + ) + .require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().swap_dims(0, 2)); + let tensor_4 = tensor_3.matmul(tensor_2.clone().swap_dims(1, 2)); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[[66., 78.], [66., 78.]], [[270., 306.], [270., 306.]]]), + Tolerance::default(), + ); + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[[22., 286.], [28., 316.]], [[172., 652.], [190., 694.]]]), + Tolerance::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/trig.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/trig.rs new file mode 100644 index 0000000..a588887 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/trig.rs @@ -0,0 +1,371 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_diff_cos() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().cos()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + // Metal has less precise trigonometric functions + let tolerance = Tolerance::default().set_half_precision_relative(1e-2); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[26.8063, -27.7870], [26.8063, -27.7870]]), + tolerance, + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[9.222064, -39.123375], [-28.721354, 49.748356]]), + tolerance, + ); +} + +#[test] +fn should_diff_sin() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().sin()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + // Metal has less precise trigonometric functions + let tolerance = Tolerance::default().set_half_precision_relative(1e-2); + + let expected = TensorData::from([[8.8500, -4.9790], [8.8500, -4.9790]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[38.668987, 44.194775], [-59.97261, -80.46094]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn should_diff_tanh() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[6.0, 7.0], [9.0, 10.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().tanh()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + let tolerance = Tolerance::default().set_half_precision_relative(8e-3); + let expected = TensorData::from([[32.0, 32.0], [32.0, 32.0]]); + grad_1 + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[8.00092, 8.000153], [8.000003, 7.999995]]); + grad_2 + .to_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn should_diff_cosh() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[0.5, 1.0], [1.5, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().cosh()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[7.092221, 16.696301], [7.092221, 16.696301]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[17.489855, 27.484539], [39.409813, 86.910278]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_sinh() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[0.5, 1.0], [1.5, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().sinh()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[4.894847, 15.887931], [4.894847, 15.887931]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[17.284000, 28.412029], [39.302979, 87.498329]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_tan() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[0.5, 1.0], [0.3, 0.8]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().tan()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[2.532602, 1.596607], [2.532602, 1.596607]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[9.028598, 14.489801], [18.038082, 21.151270]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_asin() { + let data_1 = TensorData::from([[0.0, 0.1], [0.3, 0.4]]); + let data_2 = TensorData::from([[0.2, 0.3], [0.5, 0.6]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().asin()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[0.435841, 0.969651], [0.435841, 0.969651]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[0.475300, 0.668141], [0.701834, 1.100658]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_acos() { + let data_1 = TensorData::from([[0.0, 0.1], [0.3, 0.4]]); + let data_2 = TensorData::from([[0.2, 0.3], [0.5, 0.6]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().acos()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[2.077433, 1.543624], [2.077433, 1.543624]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[0.781337, 0.588496], [0.554804, 0.155979]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_atan() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[0.5, 1.0], [1.5, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().atan()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[3.444365, 5.349211], [3.444365, 5.349211]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[9.904911, 11.554912], [10.199631, 11.391938]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_asinh() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[0.5, 1.0], [1.5, 2.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().asinh()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[3.806625, 6.844869], [3.806625, 6.844869]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[11.442373, 14.842072], [14.022551, 17.688538]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_acosh() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[1.5, 2.0], [2.5, 3.0]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().acosh()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[10.611752, 15.178907], [10.611752, 15.178907]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[20.112753, 20.247547], [20.402235, 22.487328]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_atanh() { + let data_1 = TensorData::from([[0.0, 0.1], [0.3, 0.4]]); + let data_2 = TensorData::from([[0.2, 0.3], [0.5, 0.6]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + + let tensor_3 = tensor_1.clone().matmul(tensor_2.clone().atanh()); + let tensor_4 = tensor_3.matmul(tensor_2.clone()); + let grads = tensor_4.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[0.441838, 1.037115], [0.441838, 1.037115]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[0.491723, 0.698110], [0.772763, 1.298805]]), + Tolerance::default(), + ); +} + +#[test] +fn should_diff_atan2() { + let data_1 = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + let data_2 = TensorData::from([[0.5, 1.0], [1.5, 2.0]]); + let data_3 = TensorData::from([[1.0, 0.5], [2.0, 1.5]]); + + let device = Default::default(); + let tensor_1 = TestAutodiffTensor::<2>::from_data(data_1, &device).require_grad(); + let tensor_2 = TestAutodiffTensor::from_data(data_2, &device).require_grad(); + let tensor_3 = TestAutodiffTensor::from_data(data_3, &device).require_grad(); + + let tensor_4 = tensor_1 + .clone() + .matmul(tensor_2.clone().atan2(tensor_3.clone())); + let tensor_5 = tensor_4.matmul(tensor_2.clone()); + let grads = tensor_5.backward(); + + let grad_1 = tensor_1.grad(&grads).unwrap(); + let grad_2 = tensor_2.grad(&grads).unwrap(); + let grad_3 = tensor_3.grad(&grads).unwrap(); + + grad_1.to_data().assert_approx_eq::( + &TensorData::from([[4.570492, 4.210785], [4.570492, 4.210785]]), + Tolerance::default(), + ); + + grad_2.to_data().assert_approx_eq::( + &TensorData::from([[8.208448, 8.808449], [10.357923, 12.157923]]), + Tolerance::default(), + ); + + grad_3.to_data().assert_approx_eq::( + &TensorData::from([[-1.8, -8.4], [-1.8, -5.6]]), + Tolerance::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/unfold.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/unfold.rs new file mode 100644 index 0000000..d1e5d11 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/autodiff/unfold.rs @@ -0,0 +1,18 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn unfold_backward_accumulates_overlaps() { + let device = Default::default(); + let x = TestAutodiffTensor::<2>::from_data([[1.0, 2.0, 3.0, 4.0]], &device).require_grad(); + + let y = x.clone().unfold::<3, _>(1, 2, 1); + let loss = y.sum(); + + let grads = loss.backward(); + let grad_x = x.grad(&grads).unwrap(); + + grad_x + .to_data() + .assert_eq(&TensorData::from([[1., 2., 2., 1.]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/autodiff.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/autodiff.rs new file mode 100644 index 0000000..0723224 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/autodiff.rs @@ -0,0 +1,35 @@ +// Burn autodiff tests, reusable with element types. + +pub use super::*; + +#[path = "../autodiff/mod.rs"] +mod base; + +mod checkpointing { + pub use super::*; + use burn_autodiff::checkpoint::strategy::BalancedCheckpointing; + + // Override type def + pub type TestAutodiffBackend = Autodiff; + pub type TestAutodiffTensor = Tensor; + + include!("../autodiff/mod.rs"); +} + +use burn_backend_tests::test_float_elem_variant; + +// NOTE: this currently doesn't test checkpointing with different dtypes +test_float_elem_variant!( + f16, + burn_tensor::f16, + "../autodiff/mod.rs", + ["vulkan", "cuda", "rocm", "metal"] +); + +// TODO: bf16 not yet supported on any backend for full test suite +// test_float_elem_variant!( +// bf16, +// burn_tensor::bf16, +// "../autodiff/mod.rs", +// [] // ["cuda", "rocm"] TODO, ["vulkan"] only supports bf16 for matmul, metal/wgpu doesn't support bf16 +// ); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/backend.rs new file mode 100644 index 0000000..c0e7a8f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/backend.rs @@ -0,0 +1,47 @@ +// Re-export +use super::FloatElemType; + +// Default +#[cfg(feature = "ndarray")] +pub type TestBackend = burn_ndarray::NdArray; + +#[cfg(feature = "tch")] +pub type TestBackend = burn_tch::LibTorch; + +#[cfg(feature = "cuda")] +pub type TestBackend = burn_cuda::Cuda; + +#[cfg(feature = "rocm")] +pub type TestBackend = burn_rocm::Rocm; + +#[cfg(feature = "wgpu")] +pub type TestBackend = burn_wgpu::Wgpu; + +#[cfg(feature = "cpu")] +pub type TestBackend = burn_cpu::Cpu; + +#[cfg(feature = "router")] +pub type TestBackend = burn_router::BackendRouter< + burn_router::DirectByteChannel<(burn_ndarray::NdArray, burn_wgpu::Wgpu)>, +>; + +/// Collection of types used across tests +#[allow(unused)] +pub mod prelude { + pub use burn_autodiff::Autodiff; + pub use burn_tensor::Tensor; + + use super::*; + pub type TestTensor = Tensor; + pub type TestTensorInt = Tensor; + pub type TestTensorBool = Tensor; + + pub type FloatElem = burn_tensor::ops::FloatElem; + pub type IntElem = burn_tensor::ops::IntElem; + + pub type TestAutodiffBackend = Autodiff; + pub type TestAutodiffTensor = Tensor; +} + +#[allow(unused)] +pub use prelude::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/tensor.rs new file mode 100644 index 0000000..79264d3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/common/tensor.rs @@ -0,0 +1,39 @@ +// Burn backend tensor tests, reusable with element types. + +pub use super::*; + +#[path = "../tensor/clone_invariance.rs"] +mod clone_invariance; + +#[cfg(feature = "std")] +#[path = "../tensor/multi_threads.rs"] +mod multi_threads; + +// Default float dtype +#[path = "../tensor/float/mod.rs"] +mod float; + +// Default integer dtype +#[path = "../tensor/int/mod.rs"] +mod int; + +// Default bool dtype +#[path = "../tensor/bool/mod.rs"] +mod bool; + +use burn_backend_tests::test_float_elem_variant; + +test_float_elem_variant!( + f16, + burn_tensor::f16, + "../tensor/float/mod.rs", + ["vulkan", "cuda", "rocm", "metal"] +); + +// TODO: bf16 not yet supported on any backend for full test suite +// test_float_elem_variant!( +// bf16, +// burn_tensor::bf16, +// "../tensor/float/mod.rs", +// [] // ["cuda", "rocm"] TODO, ["vulkan"] only supports bf16 for matmul, metal/wgpu doesn't support bf16 +// ); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl.rs new file mode 100644 index 0000000..dfad1ec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl.rs @@ -0,0 +1,17 @@ +//! CubeCL kernel tests. + +#[cfg(feature = "cube")] +#[path = "."] +mod cube { + type FloatElemType = f32; + type IntElemType = i32; + + mod backend { + include!("common/backend.rs"); + pub type ReferenceBackend = burn_ndarray::NdArray; + } + pub use backend::*; + + #[path = "cubecl/mod.rs"] + mod kernel; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/avg_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/avg_pool2d.rs new file mode 100644 index 0000000..a54c9bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/avg_pool2d.rs @@ -0,0 +1,96 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{ + Distribution, Tensor, TensorPrimitive, backend::Backend, module, ops::ModuleOps, +}; + +#[test] +fn avg_pool2d_should_match_reference_backend() { + let tensor = Tensor::::random( + [32, 32, 32, 32], + Distribution::Default, + &Default::default(), + ); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let kernel_size = [3, 4]; + let stride = [1, 2]; + let padding = [1, 2]; + let count_include_pad = true; + + let pooled = module::avg_pool2d( + tensor, + kernel_size, + stride, + padding, + count_include_pad, + false, + ); + let pooled_ref = module::avg_pool2d( + tensor_ref, + kernel_size, + stride, + padding, + count_include_pad, + false, + ); + + pooled + .into_data() + .assert_approx_eq::(&pooled_ref.into_data(), Tolerance::default()); +} + +#[test] +fn avg_pool2d_backward_should_match_reference_backend() { + let device = Default::default(); + + TestBackend::seed(&device, 0); + ReferenceBackend::seed(&Default::default(), 0); + + let tensor = Tensor::::random([32, 32, 32, 32], Distribution::Default, &device); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let kernel_size = [3, 3]; + let stride = [1, 1]; + let padding = [1, 1]; + let count_include_pad = true; + + let shape_out = module::avg_pool2d( + tensor.clone(), + kernel_size, + stride, + padding, + count_include_pad, + false, + ) + .shape(); + let grad_output = + Tensor::::random(shape_out, Distribution::Default, &Default::default()); + let grad_output_ref = + Tensor::::from_data(grad_output.to_data(), &Default::default()); + + let grad: Tensor = + Tensor::from_primitive(TensorPrimitive::Float(TestBackend::avg_pool2d_backward( + tensor.into_primitive().tensor(), + grad_output.into_primitive().tensor(), + kernel_size, + stride, + padding, + count_include_pad, + false, + ))); + let grad_ref: Tensor = Tensor::from_primitive(TensorPrimitive::Float( + ReferenceBackend::avg_pool2d_backward( + tensor_ref.into_primitive().tensor(), + grad_output_ref.into_primitive().tensor(), + kernel_size, + stride, + padding, + count_include_pad, + false, + ), + )); + + grad.into_data() + .assert_approx_eq::(&grad_ref.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/bernoulli.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/bernoulli.rs new file mode 100644 index 0000000..d1894d2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/bernoulli.rs @@ -0,0 +1,48 @@ +use super::*; + +use serial_test::serial; + +use core::f32; + +use burn_tensor::{Distribution, Shape, Tensor, backend::Backend}; + +use cubek::random::{assert_number_of_1_proportional_to_prob, assert_wald_wolfowitz_runs_test}; + +#[test] +#[serial] +fn number_of_1_proportional_to_prob() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let shape: Shape = [40, 40].into(); + let prob = 0.7; + + let tensor = + Tensor::::random(shape.clone(), Distribution::Bernoulli(prob), &device) + .into_data(); + + let numbers = tensor + .as_slice::<::FloatElem>() + .unwrap(); + + assert_number_of_1_proportional_to_prob(numbers, prob as f32); +} + +#[test] +#[serial] +fn wald_wolfowitz_runs_test() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let shape = Shape::new([512, 512]); + let device = Default::default(); + let tensor = Tensor::::random(shape, Distribution::Bernoulli(0.5), &device); + + let data = tensor.into_data(); + let numbers = data + .as_slice::<::FloatElem>() + .unwrap(); + + // High bound slightly over 1 so 1.0 is included in second bin + assert_wald_wolfowitz_runs_test(numbers, 0., 1.1); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cast.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cast.rs new file mode 100644 index 0000000..f9322f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cast.rs @@ -0,0 +1,45 @@ +use super::*; +use burn_tensor::{Int, Tensor, TensorData}; + +#[test] +fn should_cast_int_to_float() { + const START: usize = 0; + const END: usize = 100; + + let device = Default::default(); + let tensor = Tensor::::arange(START as i64..END as i64, &device); + + let data_int = tensor.to_data(); + let data_int = data_int.as_slice::().unwrap(); + let data_float = tensor.float().into_data(); + let data_float = data_float.as_slice::().unwrap(); + + for i in START..END { + assert_eq!(data_int[i], i as i32); + assert_eq!(data_float[i], i as f32); + } +} + +#[test] +fn should_cast_bool_to_int() { + let device = Default::default(); + + let tensor_1 = Tensor::::from_floats([[1., 0., 3.], [0., 0., 900.]], &device); + let tensor_2: Tensor = tensor_1.clone().greater_elem(0.0).int(); + + tensor_2 + .to_data() + .assert_eq(&TensorData::from([[1, 0, 1], [0, 0, 1]]), false); +} + +#[test] +fn should_cast_bool_to_float() { + let device = Default::default(); + + let tensor_1 = Tensor::::from_floats([[1., 0., 3.], [0., 0., 900.]], &device); + let tensor_2: Tensor = tensor_1.clone().greater_elem(0.0).float(); + + tensor_2 + .to_data() + .assert_eq(&TensorData::from([[1., 0., 1.], [0., 0., 1.]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cat.rs new file mode 100644 index 0000000..e25bb19 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cat.rs @@ -0,0 +1,42 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, backend::Backend}; + +#[test] +fn cat_should_match_reference_backend_dim0() { + test_same_as_reference([6, 256], 2, 0); +} + +#[test] +fn cat_should_match_reference_backend_dim1() { + test_same_as_reference([6, 256], 2, 1); +} + +#[test] +fn cat_should_support_uneven_launch() { + test_same_as_reference([1, 137], 2, 0); +} + +fn test_same_as_reference(shape: [usize; 2], num_tensors: usize, dim: usize) { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let tensors = (0..num_tensors) + .map(|_| { + Tensor::::random(shape, Distribution::Default, &Default::default()) + }) + .collect::>(); + let tensors_ref = tensors + .iter() + .map(|tensor| { + Tensor::::from_data(tensor.to_data(), &Default::default()) + }) + .collect::>(); + + let tensor = Tensor::::cat(tensors, dim); + let tensor_ref = Tensor::::cat(tensors_ref, dim); + + tensor + .into_data() + .assert_approx_eq::(&tensor_ref.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/clamp.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/clamp.rs new file mode 100644 index 0000000..7ccdc23 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/clamp.rs @@ -0,0 +1,20 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor}; + +#[test] +fn clamp_should_match_reference() { + let input = Tensor::::random( + [1, 5, 32, 32], + Distribution::Default, + &Default::default(), + ); + let input_ref = Tensor::::from_data(input.to_data(), &Default::default()); + + let output = input.clamp(0.3, 0.7); + + output.into_data().assert_approx_eq::( + &input_ref.clamp(0.3, 0.7).into_data(), + Tolerance::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/contiguous.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/contiguous.rs new file mode 100644 index 0000000..570607d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/contiguous.rs @@ -0,0 +1,40 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Int, Tensor}; + +#[test] +pub fn into_contiguous_match_reference_backend_1() { + for shape in [ + [4, 4, 4, 4], + [32, 42, 24, 48], + [8, 3, 7, 4], + [1, 4, 1, 1], + [1, 32, 256, 128], + ] { + let num_elems = shape.iter().product::() as i64; + let tensor: Tensor = + Tensor::::arange(0..num_elems, &Default::default()) + .reshape(shape) + .float(); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + + for (i, j) in get_combinations(shape.len()) { + let view = tensor.clone().swap_dims(i, j); + let view_ref = tensor_ref.clone().swap_dims(i, j); + let data = view.into_data(); + let data_ref = view_ref.into_data(); + + data_ref.assert_approx_eq::(&data, Tolerance::default()); + } + } +} + +fn get_combinations(n: usize) -> impl Iterator { + // Iterate from 0 up to n + (0..n).flat_map(move |i| { + // For each i, iterate from i + 1 up to n + // This ensures no repeats (i == j) and no duplicates (j, i) + (i + 1..n).map(move |j| (i, j)) + }) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv2d.rs new file mode 100644 index 0000000..f3b9141 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv2d.rs @@ -0,0 +1,78 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, module}; + +#[test] +fn conv2d_should_match_reference_backend() { + let test_device = Default::default(); + let input = + Tensor::::random([6, 16, 32, 32], Distribution::Default, &test_device); + let weight = + Tensor::::random([12, 8, 3, 3], Distribution::Default, &test_device); + let bias = Tensor::::random([12], Distribution::Default, &test_device); + let ref_device = Default::default(); + + let input_ref = Tensor::::from_data(input.to_data(), &ref_device); + let weight_ref = Tensor::::from_data(weight.to_data(), &ref_device); + let bias_ref = Tensor::::from_data(bias.to_data(), &ref_device); + + let options = burn_tensor::ops::ConvOptions::new([2, 3], [2, 3], [2, 3], 2); + + let output = module::conv2d(input, weight, Some(bias), options.clone()); + let output_ref = module::conv2d(input_ref, weight_ref, Some(bias_ref), options); + + output + .into_data() + .assert_approx_eq::(&output_ref.into_data(), Tolerance::default()); +} + +#[test] +fn conv2d_should_match_reference_backend_implicit() { + let test_device = Default::default(); + let input = + Tensor::::random([4, 16, 6, 6], Distribution::Default, &test_device); + let weight = + Tensor::::random([16, 16, 3, 3], Distribution::Default, &test_device); + let bias = Tensor::::random([16], Distribution::Default, &test_device); + let ref_device = Default::default(); + + let input_ref = Tensor::::from_data(input.to_data(), &ref_device); + let weight_ref = Tensor::::from_data(weight.to_data(), &ref_device); + let bias_ref = Tensor::::from_data(bias.to_data(), &ref_device); + + let options = burn_tensor::ops::ConvOptions::new([1, 1], [2, 2], [1, 1], 1); + + let output = module::conv2d(input, weight, Some(bias), options.clone()); + let output_ref = module::conv2d(input_ref, weight_ref, Some(bias_ref), options); + + let tolerance = Tolerance::default(); + output + .into_data() + .assert_approx_eq::(&output_ref.into_data(), tolerance); +} + +/// Regression test for bias loader in new implicit GEMM +#[test] +fn conv2d_should_match_reference_backend_bias_regression() { + let test_device = Default::default(); + let input = Tensor::::random([1, 1, 1, 1], Distribution::Default, &test_device); + let weight = + Tensor::::random([32, 1, 3, 3], Distribution::Default, &test_device); + let bias = Tensor::::random([32], Distribution::Default, &test_device); + let ref_device = Default::default(); + + let input_ref = Tensor::::from_data(input.to_data(), &ref_device); + let weight_ref = Tensor::::from_data(weight.to_data(), &ref_device); + let bias_ref = Tensor::::from_data(bias.to_data(), &ref_device); + + let options = burn_tensor::ops::ConvOptions::new([1, 1], [1, 1], [1, 1], 1); + + let output = module::conv2d(input, weight, Some(bias), options.clone()).permute([0, 2, 3, 1]); + let output_ref = + module::conv2d(input_ref, weight_ref, Some(bias_ref), options).permute([0, 2, 3, 1]); + + let tolerance = Tolerance::default(); + output + .into_data() + .assert_approx_eq::(&output_ref.into_data(), tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv3d.rs new file mode 100644 index 0000000..f424176 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv3d.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, module}; + +#[test] +fn conv3d_should_match_reference_backend() { + let test_device = Default::default(); + let input = + Tensor::::random([6, 16, 32, 32, 32], Distribution::Default, &test_device); + let weight = + Tensor::::random([12, 8, 3, 3, 3], Distribution::Default, &test_device); + let bias = Tensor::::random([12], Distribution::Default, &test_device); + let ref_device = Default::default(); + + let input_ref = Tensor::::from_data(input.to_data(), &ref_device); + let weight_ref = Tensor::::from_data(weight.to_data(), &ref_device); + let bias_ref = Tensor::::from_data(bias.to_data(), &ref_device); + + let options = burn_tensor::ops::ConvOptions::new([2, 3, 4], [2, 3, 4], [2, 3, 4], 2); + + let output = module::conv3d(input, weight, Some(bias), options.clone()); + let output_ref = module::conv3d(input_ref, weight_ref, Some(bias_ref), options); + + output + .into_data() + .assert_approx_eq::(&output_ref.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose2d.rs new file mode 100644 index 0000000..edfc0ad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose2d.rs @@ -0,0 +1,48 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, backend::Backend, module}; + +#[test] +fn conv_transpose2d_should_match_reference_backend() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let height = 8; + let width = 8; + let in_channels = 8; + let out_channels = 8; + let batch_size = 32; + let kernel_size_0 = 3; + let kernel_size_1 = 3; + let options = burn_tensor::ops::ConvTransposeOptions::new([1, 1], [1, 1], [0, 0], [1, 1], 1); + + let test_device = Default::default(); + let input = Tensor::::random( + [batch_size, in_channels, height, width], + Distribution::Default, + &test_device, + ); + let weight = Tensor::::random( + [ + in_channels, + out_channels / options.groups, + kernel_size_0, + kernel_size_1, + ], + Distribution::Default, + &test_device, + ); + let bias = + Tensor::::random([out_channels], Distribution::Default, &test_device); + let ref_device = Default::default(); + let input_ref = Tensor::::from_data(input.to_data(), &ref_device); + let weight_ref = Tensor::::from_data(weight.to_data(), &ref_device); + let bias_ref = Tensor::::from_data(bias.to_data(), &ref_device); + + let output = module::conv_transpose2d(input, weight, Some(bias), options.clone()); + let output_ref = module::conv_transpose2d(input_ref, weight_ref, Some(bias_ref), options); + + output + .into_data() + .assert_approx_eq::(&output_ref.into_data(), Tolerance::rel_abs(0.01, 0.02)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose3d.rs new file mode 100644 index 0000000..4c5494d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/conv_transpose3d.rs @@ -0,0 +1,51 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, backend::Backend, module}; + +#[test] +fn conv_transpose3d_should_match_reference_backend() { + let test_device = Default::default(); + TestBackend::seed(&test_device, 0); + + let depth = 8; + let height = 8; + let width = 8; + let in_channels = 8; + let out_channels = 8; + let batch_size = 32; + let kernel_size_0 = 3; + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let options = + burn_tensor::ops::ConvTransposeOptions::new([1, 1, 1], [1, 1, 1], [0, 0, 0], [1, 1, 1], 1); + + let input = Tensor::::random( + [batch_size, in_channels, depth, height, width], + Distribution::Default, + &test_device, + ); + let weight = Tensor::::random( + [ + in_channels, + out_channels / options.groups, + kernel_size_0, + kernel_size_1, + kernel_size_2, + ], + Distribution::Default, + &test_device, + ); + let bias = + Tensor::::random([out_channels], Distribution::Default, &test_device); + let ref_device = Default::default(); + let input_ref = Tensor::::from_data(input.to_data(), &ref_device); + let weight_ref = Tensor::::from_data(weight.to_data(), &ref_device); + let bias_ref = Tensor::::from_data(bias.to_data(), &ref_device); + + let output = module::conv_transpose3d(input, weight, Some(bias), options.clone()); + let output_ref = module::conv_transpose3d(input_ref, weight_ref, Some(bias_ref), options); + + output + .into_data() + .assert_approx_eq::(&output_ref.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cross.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cross.rs new file mode 100644 index 0000000..e4fb322 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/cross.rs @@ -0,0 +1,159 @@ +use super::*; +use burn_tensor::Tensor; +use burn_tensor::Tolerance; + +#[test] +fn test_cross_product() { + let device = Default::default(); + // Test with well-known orthogonal vectors for clearer validation + let a = Tensor::::from_data([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], &device); + let b = Tensor::::from_data([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], &device); + + let result = a.cross(b, 1); + // For orthogonal unit vectors: + // i × j = k + // j × k = i + let expected = Tensor::::from_data([[0.0, 0.0, 1.0], [1.0, 0.0, 0.0]], &device); + + // Use Tolerance for floating-point comparisons + let tolerance = Tolerance::::default(); + result + .to_data() + .assert_approx_eq(&expected.to_data(), tolerance); +} + +#[test] +fn test_cross_product_zeros() { + let device = Default::default(); + // Test cross product with zero vector - should always give zero vector + let a = Tensor::::from_data([[2.0, 3.0, 4.0]], &device); + let b = Tensor::::zeros([1, 3], &device); + + let result = a.cross(b, 1); + let expected = Tensor::::zeros([1, 3], &device); + + // For zeros, we can use exact equality or a very tight tolerance + let tolerance = Tolerance::::default(); + result + .to_data() + .assert_approx_eq(&expected.to_data(), tolerance); +} + +#[test] +fn test_cross_product_batch() { + let device = Default::default(); + // Test typical cross product computations in batch + let a = Tensor::::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let b = Tensor::::from_data([[4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], &device); + + let result = a.cross(b, 1); + // Cross products: + // [1,2,3] × [4,5,6] = [-3,6,-3] + // [4,5,6] × [7,8,9] = [-3,6,-3] + let expected = + Tensor::::from_data([[-3.0, 6.0, -3.0], [-3.0, 6.0, -3.0]], &device); + + let tolerance = Tolerance::::default(); + result + .to_data() + .assert_approx_eq(&expected.to_data(), tolerance); +} + +#[test] +#[should_panic] +fn test_cross_product_invalid_dimension() { + let device = Default::default(); + let a = Tensor::::zeros([1, 4], &device); + let b = Tensor::::zeros([1, 4], &device); + + let _ = a.cross(b, 1); +} + +#[test] +fn test_cross_product_parallel_vectors() { + let device = Default::default(); + // Test cross product of parallel vectors (should be zero) + let a = Tensor::::from_data([[1.0, 2.0, 3.0]], &device); + let b = Tensor::::from_data([[2.0, 4.0, 6.0]], &device); // b = 2 * a + + let result = a.cross(b, 1); + let expected = Tensor::::zeros([1, 3], &device); + + let tolerance = Tolerance::::default(); + result + .to_data() + .assert_approx_eq(&expected.to_data(), tolerance); +} + +#[test] +fn test_cross_product_3d_tensor() { + let device = Default::default(); + // Test with 3D tensor (batch of matrices) + let a = Tensor::::from_data( + [ + [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + ], + &device, + ); + + let b = Tensor::::from_data( + [ + [[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + [[4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], + ], + &device, + ); + + let result = a.cross(b, 2); // Cross on last dimension + let expected = Tensor::::from_data( + [ + [[0.0, 0.0, 1.0], [1.0, 0.0, 0.0]], + [[-3.0, 6.0, -3.0], [-3.0, 6.0, -3.0]], + ], + &device, + ); + + let tolerance = Tolerance::::default(); + result + .to_data() + .assert_approx_eq(&expected.to_data(), tolerance); +} + +// Test to verify that padding doesn't affect results +#[test] +fn test_cross_product_with_padding_awareness() { + let device = Default::default(); + // Create tensors that would span multiple 4-element blocks + // This tests that the padding doesn't corrupt adjacent data + let a = Tensor::::from_data( + [ + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], // Two vectors: [1,2,3] and [4,5,6] + ], + &device, + ); + + let b = Tensor::::from_data( + [ + [7.0, 8.0, 9.0, 10.0, 11.0, 12.0], // Two vectors: [7,8,9] and [10,11,12] + ], + &device, + ); + + // Reshape to have proper 3-element vectors in last dimension + let a_reshaped = a.reshape([2, 3]); + let b_reshaped = b.reshape([2, 3]); + + let result = a_reshaped.cross(b_reshaped, 1); + + // Expected cross products: + // [1,2,3] × [7,8,9] = [-6,12,-6] + // [4,5,6] × [10,11,12] = [-6,12,-6] + let expected = + Tensor::::from_data([[-6.0, 12.0, -6.0], [-6.0, 12.0, -6.0]], &device); + + let tolerance = Tolerance::::default(); + result + .to_data() + .assert_approx_eq(&expected.to_data(), tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/gather.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/gather.rs new file mode 100644 index 0000000..328b8bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/gather.rs @@ -0,0 +1,44 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Int, Shape, Tensor, backend::Backend}; + +#[test] +fn gather_should_work_with_multiple_workgroups_dim0() { + test_same_as_ref([6, 256], 0); +} + +#[test] +fn gather_should_work_with_multiple_workgroups_dim1() { + test_same_as_ref([6, 256], 1); +} + +fn test_same_as_ref(shape: [usize; D], dim: usize) { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let max = shape[dim]; + let shape = Shape::new(shape); + let tensor = + Tensor::::random(shape.clone(), Distribution::Default, &Default::default()); + let indices = Tensor::::from_data( + Tensor::::random( + [shape.num_elements()], + Distribution::Uniform(0., max as f64), + &Default::default(), + ) + .into_data(), + &Default::default(), + ) + .reshape(shape); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let indices_ref = + Tensor::::from_data(indices.to_data(), &Default::default()); + + let actual = tensor.gather(dim, indices); + let expected = tensor_ref.gather(dim, indices_ref); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_fill.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_fill.rs new file mode 100644 index 0000000..d36e0c2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_fill.rs @@ -0,0 +1,64 @@ +use super::*; +use burn_cubecl::kernel::{MaskFillStrategy, mask_fill}; +use burn_tensor::Tolerance; +use burn_tensor::{Bool, Distribution, Element, Tensor, TensorPrimitive, backend::Backend}; +use cubecl::prelude::InputScalar; + +#[test] +fn mask_fill_should_match_reference_backend() { + let (tensor, mask, tensor_ref, mask_ref) = inputs_mask_fill(); + let dtype_bool = <::BoolElem as Element>::dtype(); + let dtype_ft = ::dtype(); + + let actual = Tensor::::from_primitive(TensorPrimitive::Float(mask_fill( + tensor.into_primitive().tensor(), + mask.into_primitive(), + InputScalar::new(4.0, dtype_ft), + MaskFillStrategy::Readonly, + dtype_bool, + ))); + let expected = tensor_ref.mask_fill(mask_ref, 4.0); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[test] +fn mask_fill_inplace_should_match_reference_backend() { + let (tensor, mask, tensor_ref, mask_ref) = inputs_mask_fill(); + let dtype_bool = <::BoolElem as Element>::dtype(); + let dtype_ft = ::dtype(); + + let actual = Tensor::::from_primitive(TensorPrimitive::Float(mask_fill::<_>( + tensor.into_primitive().tensor(), + mask.into_primitive(), + InputScalar::new(4.0, dtype_ft), + MaskFillStrategy::Inplace, + dtype_bool, + ))); + let expected = tensor_ref.mask_fill(mask_ref, 4.0); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[allow(clippy::type_complexity)] +fn inputs_mask_fill() -> ( + Tensor, + Tensor, + Tensor, + Tensor, +) { + let test_device = Default::default(); + let tensor = Tensor::::random([2, 6, 256], Distribution::Default, &test_device); + let mask = + Tensor::::random([2, 6, 256], Distribution::Uniform(0., 1.), &test_device) + .lower_equal_elem(0.5); + let ref_device = Default::default(); + let tensor_ref = Tensor::::from_data(tensor.to_data(), &ref_device); + let mask_ref = Tensor::::from_data(mask.to_data(), &ref_device); + + (tensor, mask, tensor_ref, mask_ref) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_where.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_where.rs new file mode 100644 index 0000000..ab7ad4e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mask_where.rs @@ -0,0 +1,80 @@ +use super::*; +use burn_cubecl::kernel::{MaskWhereStrategy, mask_where}; +use burn_tensor::Tolerance; +use burn_tensor::{Bool, Distribution, Element, Tensor, TensorPrimitive, backend::Backend}; + +#[test] +fn mask_where_should_match_reference_backend() { + let (tensor, value, mask, tensor_ref, value_ref, mask_ref) = inputs_mask_where(); + + let actual = tensor.mask_where(mask, value); + let expected = tensor_ref.mask_where(mask_ref, value_ref); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} +#[test] +fn mask_where_inplace_lhs_should_match_reference_backend() { + let (tensor, value, mask, tensor_ref, value_ref, mask_ref) = inputs_mask_where(); + let dtype_bool = <::BoolElem as Element>::dtype(); + + let actual = Tensor::::from_primitive(TensorPrimitive::Float(mask_where::<_>( + tensor.into_primitive().tensor(), + mask.into_primitive(), + value.into_primitive().tensor(), + MaskWhereStrategy::InplaceLhs, + dtype_bool, + ))); + let expected = tensor_ref.mask_where(mask_ref, value_ref); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[test] +fn mask_where_inplace_rhs_should_match_reference_backend() { + let (tensor, value, mask, tensor_ref, value_ref, mask_ref) = inputs_mask_where(); + let dtype_bool = <::BoolElem as Element>::dtype(); + + let actual = Tensor::::from_primitive(TensorPrimitive::Float(mask_where::<_>( + tensor.into_primitive().tensor(), + mask.into_primitive(), + value.into_primitive().tensor(), + MaskWhereStrategy::InplaceRhs, + dtype_bool, + ))); + let expected = tensor_ref.mask_where(mask_ref, value_ref); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[allow(clippy::type_complexity)] +fn inputs_mask_where() -> ( + Tensor, + Tensor, + Tensor, + Tensor, + Tensor, + Tensor, +) { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let tensor = Tensor::::random([2, 6, 256], Distribution::Default, &device); + let value = Tensor::::random([2, 6, 256], Distribution::Default, &device); + let mask = + Tensor::::random([2, 6, 256], Distribution::Uniform(0., 1.), &device) + .lower_equal_elem(0.5); + + let device_ref = Default::default(); + let tensor_ref = Tensor::::from_data(tensor.to_data(), &device_ref); + let value_ref = Tensor::::from_data(value.to_data(), &device_ref); + let mask_ref = Tensor::::from_data(mask.to_data(), &device_ref); + mask.to_data().assert_eq(&mask_ref.to_data(), false); + + (tensor, value, mask, tensor_ref, value_ref, mask_ref) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d.rs new file mode 100644 index 0000000..dd93c01 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d.rs @@ -0,0 +1,52 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, module}; + +#[test] +pub fn max_pool2d_should_match_reference_backends() { + let tensor = Tensor::::random( + [32, 32, 32, 32], + Distribution::Default, + &Default::default(), + ); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let kernel_size = [3, 3]; + let stride = [2, 2]; + let padding = [1, 1]; + let dilation = [1, 1]; + + let pooled = module::max_pool2d(tensor, kernel_size, stride, padding, dilation, false); + let pooled_ref = module::max_pool2d(tensor_ref, kernel_size, stride, padding, dilation, false); + + pooled + .into_data() + .assert_approx_eq::(&pooled_ref.into_data(), Tolerance::default()); +} + +#[test] +pub fn max_pool2d_with_indices_should_match_reference_backend() { + let tensor = Tensor::::random( + [32, 32, 32, 32], + Distribution::Default, + &Default::default(), + ); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let kernel_size = [3, 3]; + let stride = [2, 2]; + let padding = [1, 1]; + let dilation = [1, 1]; + + let (pooled, indices) = + module::max_pool2d_with_indices(tensor, kernel_size, stride, padding, dilation, false); + let (pooled_ref, indices_ref) = + module::max_pool2d_with_indices(tensor_ref, kernel_size, stride, padding, dilation, false); + + pooled + .into_data() + .assert_approx_eq::(&pooled_ref.into_data(), Tolerance::default()); + indices + .into_data() + .assert_eq(&indices_ref.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d_backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d_backward.rs new file mode 100644 index 0000000..a1198a9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/max_pool2d_backward.rs @@ -0,0 +1,67 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor, TensorPrimitive, module, ops::ModuleOps}; + +#[test] +pub fn max_pool2d_with_indices_backward_should_match_reference_backend() { + let test_device = Default::default(); + let tensor = + Tensor::::random([32, 32, 32, 32], Distribution::Default, &test_device); + let grad_output = + Tensor::::random([32, 32, 16, 16], Distribution::Default, &test_device); + let ref_device = Default::default(); + let tensor_ref = Tensor::::from_data(tensor.to_data(), &ref_device); + let grad_output_ref = + Tensor::::from_data(grad_output.to_data(), &ref_device); + let kernel_size = [3, 3]; + let stride = [2, 2]; + let padding = [1, 1]; + let dilation = [1, 1]; + + let (_, indices) = module::max_pool2d_with_indices( + tensor.clone(), + kernel_size, + stride, + padding, + dilation, + false, + ); + let (_, indices_ref) = module::max_pool2d_with_indices( + tensor_ref.clone(), + kernel_size, + stride, + padding, + dilation, + false, + ); + let grad = TestBackend::max_pool2d_with_indices_backward( + tensor.into_primitive().tensor(), + kernel_size, + stride, + padding, + dilation, + false, + grad_output.into_primitive().tensor(), + indices.into_primitive(), + ) + .x_grad; + let grad_ref = ReferenceBackend::max_pool2d_with_indices_backward( + tensor_ref.into_primitive().tensor(), + kernel_size, + stride, + padding, + dilation, + false, + grad_output_ref.into_primitive().tensor(), + indices_ref.into_primitive(), + ) + .x_grad; + + Tensor::::from_primitive(TensorPrimitive::Float(grad)) + .into_data() + .assert_approx_eq::( + &Tensor::::from_primitive(TensorPrimitive::Float(grad_ref)) + .into_data(), + Tolerance::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mod.rs new file mode 100644 index 0000000..df5c3df --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/mod.rs @@ -0,0 +1,30 @@ +// #[allow(unused_imports)] // required for re-included modules +pub use super::*; + +mod avg_pool2d; +mod bernoulli; +mod cast; +mod cat; +mod clamp; +mod contiguous; +mod conv2d; +mod conv3d; +mod conv_transpose2d; +mod conv_transpose3d; +mod cross; +mod gather; +mod mask_fill; +mod mask_where; +mod max_pool2d; +mod max_pool2d_backward; +mod normal; +mod quantization; +mod reduce; +mod repeat_dim; +mod scatter; +mod select; +mod select_assign; +mod slice; +mod slice_assign; +mod unary; +mod uniform; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/normal.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/normal.rs new file mode 100644 index 0000000..ff6192e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/normal.rs @@ -0,0 +1,36 @@ +use super::*; +use burn_tensor::{Distribution, Shape, Tensor, backend::Backend}; +use cubek::random::{assert_mean_approx_equal, assert_normal_respects_68_95_99_rule}; +use serial_test::serial; + +#[test] +#[serial] +fn empirical_mean_close_to_expectation() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let shape = [100, 100]; + let mean = 10.; + let tensor = Tensor::::random(shape, Distribution::Normal(mean, 2.), &device) + .into_data(); + let numbers = tensor.as_slice::().unwrap(); + + assert_mean_approx_equal(numbers, mean as f32); +} + +#[test] +#[serial] +fn normal_respects_68_95_99_rule() { + // https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule + let shape: Shape = [1000, 1000].into(); + let device = Default::default(); + let mu = 0.; + let s = 1.; + let tensor = + Tensor::::random(shape.clone(), Distribution::Normal(mu, s), &device) + .into_data(); + + let numbers = tensor.as_slice::().unwrap(); + + assert_normal_respects_68_95_99_rule(numbers, mu as f32, s as f32); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/quantization.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/quantization.rs new file mode 100644 index 0000000..4145726 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/quantization.rs @@ -0,0 +1,240 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{ + Shape, Tensor, + backend::Backend, + quantization::{QuantLevel, QuantScheme, QuantStore, QuantValue}, +}; + +fn should_quantize_dequantize_symmetric_arange>( + value: QuantValue, + store: QuantStore, + shape: S, +) { + let shape = shape.into(); + assert_eq!(shape.rank(), 2); // 2D tests + + let scheme = QuantScheme::default().with_value(value).with_store(store); + let scheme_ref = scheme.clone().with_store(QuantStore::Native); + + let input: Tensor = + Tensor::arange(0..shape.num_elements() as i64, &Default::default()) + .float() + .reshape(shape); + let input_ref = Tensor::::from_data(input.to_data(), &Default::default()); + + let output = input.quantize_dynamic(&scheme); + let output_ref = input_ref.quantize_dynamic(&scheme_ref); + + output.to_data().assert_eq(&output_ref.to_data(), false); + + let output = output.dequantize(); + let output_ref = output_ref.dequantize(); + + output + .into_data() + .assert_approx_eq::(&output_ref.to_data(), Tolerance::default()); +} + +fn should_quantize_dequantize_symmetric_per_block_arange>( + value: QuantValue, + block_size: usize, + store: QuantStore, + shape: S, +) { + let scheme = QuantScheme::default() + .with_value(value) + .with_level(QuantLevel::block([block_size as u8])) + .with_store(store); + let scheme_ref = scheme.clone().with_store(QuantStore::Native); + + let shape = shape.into(); + let input: Tensor = + Tensor::arange(0..shape.num_elements() as i64, &Default::default()) + .float() + .reshape(shape); + let input_ref = Tensor::::from_data(input.to_data(), &Default::default()); + + let output = input.quantize_dynamic(&scheme); + let output_ref = input_ref.quantize_dynamic(&scheme_ref); + + output.to_data().assert_eq(&output_ref.to_data(), false); + + let output = output.dequantize(); + let output_ref = output_ref.dequantize(); + + output + .into_data() + .assert_approx_eq::(&output_ref.to_data(), Tolerance::default()); +} + +fn should_quantize_dequantize_symmetric_per_block( + value: QuantValue, + block_size: usize, + store: QuantStore, +) { + let scheme = QuantScheme::default() + .with_value(value) + .with_level(QuantLevel::block([block_size as u8])) + .with_store(store); + let scheme_ref = scheme.clone().with_store(QuantStore::Native); + + let input = Tensor::::from_floats( + [ + [ + -1.8, -1.0, 0.0, 0.5, -1.8, -1.0, 0.0, 0.5, 0.01, 0.025, 0.03, 0.04, 0.01, 0.025, + 0.03, 0.04, + ], + [ + 1.8, 1.0, 0.0, -0.5, 1.8, 1.0, 0.0, -0.5, -0.01, -0.025, -0.03, -0.04, -0.01, + -0.025, -0.03, -0.04, + ], + ], + &Default::default(), + ); + let input_ref = Tensor::::from_data(input.to_data(), &Default::default()); + + let output = input.quantize_dynamic(&scheme); + let output_ref = input_ref.quantize_dynamic(&scheme_ref); + + output.to_data().assert_eq(&output_ref.to_data(), false); + + let output = output.dequantize(); + let output_ref = output_ref.dequantize(); + + output + .into_data() + .assert_approx_eq::(&output_ref.to_data(), Tolerance::default()); +} + +fn supports_native() -> bool { + let name = ::name(&Default::default()); + // TODO: Proper checks for i8 support. + name.contains("cuda") + || name.contains("rocm") + || name.contains("hip") + || name.contains("vulkan") + || name.contains("spirv") + || name.contains("metal") + || name.contains("msl") +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q8s_packed() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q8S, QuantStore::PackedU32(0), [8, 16]) +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q8f_packed() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q8F, QuantStore::PackedU32(0), [8, 16]) +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q4s_packed() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q4S, QuantStore::PackedU32(0), [8, 16]) +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q4f_packed() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q4F, QuantStore::PackedU32(0), [8, 16]) +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q2s_packed() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q2S, QuantStore::PackedU32(0), [8, 16]) +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q2f_packed() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q2F, QuantStore::PackedU32(0), [8, 16]) +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_q8s_packed() { + should_quantize_dequantize_symmetric_per_block(QuantValue::Q8S, 8, QuantStore::PackedU32(0)) +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_q4s_packed() { + should_quantize_dequantize_symmetric_per_block(QuantValue::Q4S, 8, QuantStore::PackedU32(0)) +} + +#[test] +#[should_panic = "Block size must be divisible by 16"] +fn should_panic_when_block_size_cannot_store_num_quants() { + // num_quants in u32 = 32 bits / 2 bits = 16 + should_quantize_dequantize_symmetric_per_block(QuantValue::Q2S, 8, QuantStore::PackedU32(0)) +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_q2s_packed() { + should_quantize_dequantize_symmetric_per_block(QuantValue::Q2S, 16, QuantStore::PackedU32(0)) +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_q8s_native() { + if supports_native() { + should_quantize_dequantize_symmetric_arange(QuantValue::Q8S, QuantStore::Native, [32, 32]) + } +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_q8s_native() { + if supports_native() { + should_quantize_dequantize_symmetric_per_block(QuantValue::Q8S, 8, QuantStore::Native) + } +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_arange_q8s_packed() { + should_quantize_dequantize_symmetric_per_block_arange( + QuantValue::Q8S, + 32, + QuantStore::PackedU32(0), + [32, 32], + ) +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_arange_q8s_native() { + if supports_native() { + should_quantize_dequantize_symmetric_per_block_arange( + QuantValue::Q8S, + 32, + QuantStore::Native, + [32, 32], + ) + } +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_128x256_q8s_native() { + if supports_native() { + should_quantize_dequantize_symmetric_per_block_arange( + QuantValue::Q8S, + 32, + QuantStore::Native, + [128, 256], + ) + } +} +#[test] +fn should_quantize_dequantize_symmetric_arange_128x256_q8s_packed() { + should_quantize_dequantize_symmetric_per_block_arange( + QuantValue::Q8S, + 32, + QuantStore::PackedU32(0), + [128, 256], + ) +} + +#[test] +#[should_panic = "Can't store in u32"] +fn should_panic_when_shape_cannot_store_quants() { + let device = Default::default(); + let scheme = QuantScheme::default(); + + let _tensor_1 = + Tensor::::from_floats([[1.0, 6.35], [2.0, 3.0], [1.0, 3.0]], &device) + .quantize_dynamic(&scheme); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/reduce.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/reduce.rs new file mode 100644 index 0000000..f48f3eb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/reduce.rs @@ -0,0 +1,135 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor}; + +const RANK: usize = 4; +const SHAPE: [usize; RANK] = [2, 4, 8, 16]; + +#[test] +fn reduction_argmax_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + for dim in 0..RANK { + tensor + .clone() + .argmax(dim) + .into_data() + .assert_eq(&tensor_ref.clone().argmax(dim).into_data(), false); + } +} + +#[test] +fn reduction_argmin_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + for dim in 0..RANK { + tensor + .clone() + .argmin(dim) + .into_data() + .assert_eq(&tensor_ref.clone().argmin(dim).into_data(), false); + } +} + +#[test] +fn reduction_mean_dim_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + for dim in 0..RANK { + tensor + .clone() + .mean_dim(dim) + .into_data() + .assert_approx_eq::( + &tensor_ref.clone().mean_dim(dim).into_data(), + Tolerance::default(), + ); + } +} + +#[test] +fn reduction_mean_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + tensor + .clone() + .mean() + .into_data() + .assert_approx_eq::( + &tensor_ref.clone().mean().into_data(), + Tolerance::default(), + ); +} + +#[test] +fn reduction_prod_dim_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + for dim in 0..RANK { + tensor + .clone() + .prod_dim(dim) + .into_data() + .assert_approx_eq::( + &tensor_ref.clone().prod_dim(dim).into_data(), + Tolerance::default(), + ); + } +} + +#[test] +fn reduction_prod_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + tensor + .clone() + .prod() + .into_data() + .assert_approx_eq::( + &tensor_ref.clone().prod().into_data(), + Tolerance::default(), + ); +} + +#[test] +fn reduction_sum_dim_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + for dim in 0..RANK { + tensor + .clone() + .sum_dim(dim) + .into_data() + .assert_approx_eq::( + &tensor_ref.clone().sum_dim(dim).into_data(), + Tolerance::default(), + ); + } +} + +#[test] +fn reduction_sum_should_match_reference_backend() { + let tensor = + Tensor::::random(SHAPE, Distribution::Default, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + tensor + .clone() + .sum() + .into_data() + .assert_approx_eq::(&tensor_ref.clone().sum().into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/repeat_dim.rs new file mode 100644 index 0000000..6065ec1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/repeat_dim.rs @@ -0,0 +1,71 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor}; + +#[test] +fn repeat_dim_0_few_times() { + let tensor = + Tensor::::random([1, 6, 6], Distribution::Default, &Default::default()); + let dim = 0; + let times = 4; + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + + let actual = tensor.repeat_dim(dim, times); + let expected = tensor_ref.repeat_dim(dim, times); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[test] +fn repeat_dim_1_few_times() { + let tensor = + Tensor::::random([6, 1, 6], Distribution::Default, &Default::default()); + let dim = 1; + let times = 4; + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + + let actual = tensor.repeat_dim(dim, times); + let expected = tensor_ref.repeat_dim(dim, times); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[test] +fn repeat_dim_2_few_times() { + let tensor = + Tensor::::random([6, 6, 1], Distribution::Default, &Default::default()); + let dim = 2; + let times = 4; + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + + let actual = tensor.repeat_dim(dim, times); + let expected = tensor_ref.repeat_dim(dim, times); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +#[test] +fn repeat_dim_2_many_times() { + let tensor = + Tensor::::random([10, 10, 1], Distribution::Default, &Default::default()); + let dim = 2; + let times = 200; + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + + let actual = tensor.repeat_dim(dim, times); + let expected = tensor_ref.repeat_dim(dim, times); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/scatter.rs new file mode 100644 index 0000000..6f7ef90 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/scatter.rs @@ -0,0 +1,66 @@ +use super::*; +use burn_tensor::{Distribution, Int, Tensor, backend::Backend}; +use burn_tensor::{IndexingUpdateOp, Tolerance}; + +#[test] +fn scatter_should_work_with_multiple_workgroups_2d_dim0() { + same_as_reference_same_shape(0, [256, 32]); +} + +#[test] +fn scatter_should_work_with_multiple_workgroups_2d_dim1() { + same_as_reference_same_shape(1, [32, 256]); +} + +#[test] +fn scatter_should_work_with_multiple_workgroups_3d_dim0() { + same_as_reference_same_shape(0, [256, 6, 6]); +} + +#[test] +fn scatter_should_work_with_multiple_workgroups_3d_dim1() { + same_as_reference_same_shape(1, [6, 256, 6]); +} + +#[test] +fn scatter_should_work_with_multiple_workgroups_3d_dim2() { + same_as_reference_same_shape(2, [6, 6, 256]); +} + +#[test] +fn scatter_should_work_with_multiple_workgroups_diff_shapes() { + same_as_reference_diff_shape(1, [32, 128], [32, 1]); +} + +fn same_as_reference_diff_shape( + dim: usize, + shape1: [usize; D], + shape2: [usize; D], +) { + let test_device = Default::default(); + TestBackend::seed(&test_device, 0); + + let tensor = Tensor::::random(shape1, Distribution::Default, &test_device); + let value = Tensor::::random(shape2, Distribution::Default, &test_device); + let indices = Tensor::::random( + [shape2.iter().product::()], + Distribution::Uniform(0., shape2[dim] as f64), + &test_device, + ) + .reshape(shape2); + let ref_device = Default::default(); + let tensor_ref = Tensor::::from_data(tensor.to_data(), &ref_device); + let value_ref = Tensor::::from_data(value.to_data(), &ref_device); + let indices_ref = Tensor::::from_data(indices.to_data(), &ref_device); + + let actual = tensor.scatter(dim, indices, value, IndexingUpdateOp::Add); + let expected = tensor_ref.scatter(dim, indices_ref, value_ref, IndexingUpdateOp::Add); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} + +fn same_as_reference_same_shape(dim: usize, shape: [usize; D]) { + same_as_reference_diff_shape(dim, shape, shape); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select.rs new file mode 100644 index 0000000..4fea799 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Int, Tensor}; + +#[test] +fn select_should_work_with_multiple_workgroups() { + let tensor = + Tensor::::random([6, 256], Distribution::Default, &Default::default()); + let indices = Tensor::::arange(0..100, &Default::default()); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let indices_ref = + Tensor::::from_data(indices.to_data(), &Default::default()); + + let actual = tensor.select(1, indices); + let expected = tensor_ref.select(1, indices_ref); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select_assign.rs new file mode 100644 index 0000000..4ff498a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/select_assign.rs @@ -0,0 +1,54 @@ +use super::*; +use burn_tensor::{Distribution, Int, Tensor, backend::Backend}; +use burn_tensor::{IndexingUpdateOp, Tolerance}; + +#[test] +fn select_add_should_work_with_multiple_workgroups_2d_dim0() { + select_add_same_as_ref(0, [256, 6]); +} + +#[test] +fn select_add_should_work_with_multiple_workgroups_2d_dim1() { + select_add_same_as_ref(1, [6, 256]); +} + +#[test] +fn select_add_should_work_with_multiple_workgroups_3d_dim0() { + select_add_same_as_ref(0, [256, 6, 6]); +} + +#[test] +fn select_add_should_work_with_multiple_workgroups_3d_dim1() { + select_add_same_as_ref(1, [6, 256, 6]); +} + +#[test] +fn select_add_should_work_with_multiple_workgroups_3d_dim2() { + select_add_same_as_ref(2, [6, 6, 256]); +} + +fn select_add_same_as_ref(dim: usize, shape: [usize; D]) { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let tensor = + Tensor::::random(shape, Distribution::Default, &Default::default()); + let value = Tensor::::random(shape, Distribution::Default, &Default::default()); + let indices = Tensor::::random( + [shape[dim]], + Distribution::Uniform(0., shape[dim] as f64), + &Default::default(), + ); + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let value_ref = Tensor::::from_data(value.to_data(), &Default::default()); + let indices_ref = + Tensor::::from_data(indices.to_data(), &Default::default()); + + let actual = tensor.select_assign(dim, indices, value, IndexingUpdateOp::Add); + let expected = tensor_ref.select_assign(dim, indices_ref, value_ref, IndexingUpdateOp::Add); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice.rs new file mode 100644 index 0000000..bc5cdab --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice.rs @@ -0,0 +1,19 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, Tensor}; + +#[test] +fn slice_should_work_with_multiple_workgroups() { + let tensor = + Tensor::::random([6, 256], Distribution::Default, &Default::default()); + let indices = [3..5, 45..256]; + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + + let actual = tensor.slice(indices.clone()); + let expected = tensor_ref.slice(indices); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice_assign.rs new file mode 100644 index 0000000..915cb0a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/slice_assign.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::{Distribution, Tensor, Tolerance}; + +#[test] +fn slice_assign_should_work_with_multiple_workgroups() { + let tensor = + Tensor::::random([6, 256], Distribution::Default, &Default::default()); + let value = + Tensor::::random([2, 211], Distribution::Default, &Default::default()); + let indices = [3..5, 45..256]; + let tensor_ref = + Tensor::::from_data(tensor.to_data(), &Default::default()); + let value_ref = Tensor::::from_data(value.to_data(), &Default::default()); + + let actual = tensor.slice_assign(indices.clone(), value); + let expected = tensor_ref.slice_assign(indices, value_ref); + + expected + .into_data() + .assert_approx_eq::(&actual.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/unary.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/unary.rs new file mode 100644 index 0000000..82a1c03 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/unary.rs @@ -0,0 +1,24 @@ +use super::*; +use burn_tensor::Tensor; + +#[test] +fn tanh_should_not_have_numerical_bugs_on_macos() { + fn tanh_one_value(input: f32) -> f32 { + let tensor = Tensor::::ones([1], &Default::default()) * input; + let output = tensor.tanh().into_primitive(); + Tensor::::from_primitive(output) + .into_data() + .as_slice() + .unwrap()[0] + } + + let ok = tanh_one_value(43.0); // metal tanh gives 1.0 which is the right answer + let zero = tanh_one_value(44.0); // metal tanh gives zero when within 43.67..44.36 + let nan = tanh_one_value(45.0); // metal tanh gives nan when over 44.36 + let neg = tanh_one_value(-45.0); // metal works correctly here + + assert!(!ok.is_nan() && ok == 1.0); + assert!(!zero.is_nan() && zero == 1.0); + assert!(!nan.is_nan() && nan == 1.0); + assert!(!neg.is_nan() && neg == -1.0); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/uniform.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/uniform.rs new file mode 100644 index 0000000..5fbd782 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/cubecl/uniform.rs @@ -0,0 +1,113 @@ +use super::*; +use burn_tensor::{Distribution, Int, Shape, Tensor, backend::Backend}; +use burn_tensor::{ElementConversion, Tolerance}; + +use serial_test::serial; + +use cubek::random::{assert_at_least_one_value_per_bin, assert_wald_wolfowitz_runs_test}; + +#[test] +#[serial] +fn values_all_within_interval_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let shape = [24, 24]; + + let tensor = Tensor::::random(shape, Distribution::Default, &device); + tensor + .to_data() + .assert_within_range::(0.elem()..1.elem()); +} + +#[test] +#[serial] +fn values_all_within_interval_uniform() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let shape = [24, 24]; + + let tensor = Tensor::::random(shape, Distribution::Uniform(5., 17.), &device); + tensor + .to_data() + .assert_within_range::(5.elem()..17.elem()); +} + +#[test] +#[serial] +fn at_least_one_value_per_bin_uniform() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let shape = [64, 64]; + + let tensor = Tensor::::random(shape, Distribution::Uniform(-5., 10.), &device) + .into_data(); + let numbers = tensor.as_slice::().unwrap(); + + assert_at_least_one_value_per_bin(numbers, 3, -5., 10.); +} + +#[test] +#[serial] +fn runs_test() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let shape = Shape::new([512, 512]); + let tensor = + Tensor::::random(shape, Distribution::Default, &device).into_data(); + + let numbers = tensor.as_slice::().unwrap(); + + assert_wald_wolfowitz_runs_test(numbers, 0., 1.); +} + +#[test] +#[serial] +fn int_values_all_within_interval_uniform() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let shape = Shape::new([20, 20]); + let tensor: Tensor = Tensor::random(shape, Distribution::Default, &device); + + let data_float = tensor.float().into_data(); + + data_float.assert_within_range(0..255); +} + +#[test] +#[serial] +fn at_least_one_value_per_bin_int_uniform() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let shape = Shape::new([64, 64]); + + let tensor: Tensor = + Tensor::random(shape, Distribution::Uniform(-10.0, 10.0), &device); + + let data_float = tensor.float().into_data(); + + let numbers = data_float.as_slice::().unwrap(); + + assert_at_least_one_value_per_bin(numbers, 10, -10., 10.); +} + +#[test] +fn should_not_fail_on_non_float_autotune() { + let device = Default::default(); + let tensor_1 = Tensor::::from_floats([[1., 2., 3.], [3., 4., 5.]], &device); + + // Autotune of all (reduce) on lower_equal_elem's output calls uniform distribution + tensor_1.lower_equal_elem(1.0).all(); +} + +#[test] +#[serial] +fn test_seed_reproducibility() { + let device = Default::default(); + TestBackend::seed(&device, 42); + let t1 = TestTensor::<1>::random([5], Distribution::Default, &device); + TestBackend::seed(&device, 42); + let t2 = TestTensor::<1>::random([5], Distribution::Default, &device); + + t1.into_data() + .assert_approx_eq::(&t2.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/mod.rs new file mode 100644 index 0000000..a312427 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/mod.rs @@ -0,0 +1 @@ +mod reduce_broadcasted; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/reduce_broadcasted.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/reduce_broadcasted.rs new file mode 100644 index 0000000..84e6103 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fused_ops/reduce_broadcasted.rs @@ -0,0 +1,158 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance, backend::Backend}; + +#[test] +fn test_reduce_broadcasted_1() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_read = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_write = TestTensorInt::<1>::arange(0..4, &device) + .reshape([4, 1]) + .float(); + + // Forces previous tensors to be materialized. + TestBackend::sync(&device).unwrap(); + + let x = tensor + fused_on_read.clone(); + let x = x.sum_dim(1); + + let x = x + fused_on_write; + + // Broadcast + let end = x + fused_on_read; + let actual = end.into_data(); + let expected = TensorData::from([ + [56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0], + [193.0, 194.0, 195.0, 196.0, 197.0, 198.0, 199.0, 200.0], + [330.0, 331.0, 332.0, 333.0, 334.0, 335.0, 336.0, 337.0], + [467.0, 468.0, 469.0, 470.0, 471.0, 472.0, 473.0, 474.0], + ]); + actual.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_reduce_broadcasted_2() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_read = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_write = TestTensorInt::<1>::arange(16..48, &device) + .reshape([4, 8]) + .float(); + // Second fuse on read + let y = TestTensorInt::<1>::arange(32..64, &device) + .reshape([4, 8]) + .float(); + + // Forces previous tensors to be materialized. + TestBackend::sync(&device).unwrap(); + + let x = tensor + fused_on_read.clone(); + let x = x.sum_dim(1); + let x = x + fused_on_write; + let x = x.mean_dim(1); + + let end = x + y; + TestBackend::sync(&device).unwrap(); + + let actual = end.into_data(); + let expected = TensorData::from([ + [107.5, 108.5, 109.5, 110.5, 111.5, 112.5, 113.5, 114.5], + [251.5, 252.5, 253.5, 254.5, 255.5, 256.5, 257.5, 258.5], + [395.5, 396.5, 397.5, 398.5, 399.5, 400.5, 401.5, 402.5], + [539.5, 540.5, 541.5, 542.5, 543.5, 544.5, 545.5, 546.5], + ]); + actual.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_reduce_broadcasted_3() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_read = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_write = TestTensorInt::<1>::arange(0..4, &device) + .reshape([4, 1]) + .float(); + // Second fuse on read + let y = TestTensorInt::<1>::arange(32..64, &device) + .reshape([4, 8]) + .float(); + + // Forces previous tensors to be materialized. + TestBackend::sync(&device).unwrap(); + + let x = tensor + fused_on_read.clone(); + let x = x.sum_dim(1); + + let x = x + fused_on_write; + + // Broadcast + let x = x + fused_on_read; + // Second reduce + let x = x.mean_dim(1); + + let end = x + y; + let actual = end.into_data(); + let expected = TensorData::from([ + [91.5, 92.5, 93.5, 94.5, 95.5, 96.5, 97.5, 98.5], + [236.5, 237.5, 238.5, 239.5, 240.5, 241.5, 242.5, 243.5], + [381.5, 382.5, 383.5, 384.5, 385.5, 386.5, 387.5, 388.5], + [526.5, 527.5, 528.5, 529.5, 530.5, 531.5, 532.5, 533.5], + ]); + actual.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_reduce_broadcasted_4_reused_partial() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_read = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + let fused_on_write = TestTensorInt::<1>::arange(0..4, &device) + .reshape([4, 1]) + .float(); + let y = TestTensorInt::<1>::arange(32..64, &device) + .reshape([4, 8]) + .float(); + + // Forces previous tensors to be materialized. + TestBackend::sync(&device).unwrap(); + + // In fusion we have to create a global buffer to keep the intermediate data for now. + let x_previous = tensor + fused_on_read; + let x = x_previous.clone().sum_dim(1); + + let x = x * fused_on_write; + + // Broadcast + let x = x + x_previous; + // Second reduce + let x = x.mean_dim(1); + + // Second fuse on read + let end = x + y; + let actual = end.into_data(); + let expected = TensorData::from([ + [39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0], + [247.0, 248.0, 249.0, 250.0, 251.0, 252.0, 253.0, 254.0], + [711.0, 712.0, 713.0, 714.0, 715.0, 716.0, 717.0, 718.0], + [ + 1431.0, 1432.0, 1433.0, 1434.0, 1435.0, 1436.0, 1437.0, 1438.0, + ], + ]); + actual.assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fusion.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fusion.rs new file mode 100644 index 0000000..272d3de --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/fusion.rs @@ -0,0 +1,41 @@ +//! Burn tensor and autodiff tests for CubeCL backends with fusion enabled. + +#![allow( + clippy::single_range_in_vec_init, + clippy::duplicate_mod, + reason = "false positive" +)] +extern crate alloc; + +#[cfg(feature = "cube")] +#[path = "."] +mod fusion { + pub type FloatElemType = f32; + pub type IntElemType = i32; + + #[path = "common/backend.rs"] + mod backend; + pub use backend::prelude::*; + + // NOTE: + // We re-include the tensor and autodiff test suites after overriding `TestBackend` + // with `Fusion`. This intentionally duplicates module names and test + // logic to execute the same tests under fusion. + pub type TestBackend = burn_fusion::Fusion; + pub type TestTensor = Tensor; + pub type TestTensorInt = Tensor; + pub type TestTensorBool = Tensor; + + // Tensor tests + mod tensor { + include!("common/tensor.rs"); + } + + // Autodiff tests + mod autodiff { + include!("common/autodiff.rs"); + } + + // Fusion tests + include!("fused_ops/mod.rs"); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor.rs new file mode 100644 index 0000000..1d0befe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor.rs @@ -0,0 +1,15 @@ +//! Burn backend tensor tests. + +#![allow(clippy::single_range_in_vec_init, reason = "false positive")] +extern crate alloc; + +pub type FloatElemType = f32; +#[allow(unused)] +pub type IntElemType = i32; + +#[path = "common/backend.rs"] +mod backend; +pub use backend::*; + +#[path = "common/tensor.rs"] +mod tensor; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/mod.rs new file mode 100644 index 0000000..5d0928d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/mod.rs @@ -0,0 +1,3 @@ +pub use super::*; // re-export test types + +mod ops; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/all.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/all.rs new file mode 100644 index 0000000..e7439d6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/all.rs @@ -0,0 +1,34 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_all() { + let tensor = TestTensorBool::<2>::from([[false, true, false], [true, true, true]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensorBool::<2>::from([[true, true, true], [true, true, true]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_all_dim() { + let tensor = TestTensorBool::<2>::from([[false, true, false], [true, true, true]]); + let data_actual = tensor.all_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_all_with_bool_from_lower_equal() { + let tensor1 = TestTensor::<2>::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]) + 1e-6; + let tensor2 = TestTensor::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]) + 1e-6; + + let ge = tensor1.lower_equal(tensor2); + let all = ge.clone().all(); + + TensorData::from([true]).assert_eq(&all.clone().into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/any.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/any.rs new file mode 100644 index 0000000..fd414fa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/any.rs @@ -0,0 +1,23 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_any() { + let tensor = TestTensorBool::<2>::from([[false, false, false], [true, true, false]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensorBool::<2>::from([[false, false, false], [false, false, false]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_any_dim() { + let tensor = TestTensorBool::<2>::from([[false, false, false], [true, true, false]]); + let data_actual = tensor.any_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/argwhere_nonzero.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/argwhere_nonzero.rs new file mode 100644 index 0000000..2e2cb26 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/argwhere_nonzero.rs @@ -0,0 +1,107 @@ +use super::*; +use alloc::vec::Vec; +use burn_tensor::{Shape, TensorData}; + +#[test] +fn test_argwhere_1d() { + let tensor = TestTensorBool::<1>::from([false, true, false, true, true]); + let output = tensor.argwhere(); + + output + .into_data() + .assert_eq(&TensorData::from([[1], [3], [4]]), false); +} + +#[test] +fn test_argwhere_2d() { + let tensor = TestTensorBool::<2>::from([[false, false], [false, true], [true, true]]); + let output = tensor.argwhere(); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 1], [2, 0], [2, 1]]), false); +} + +#[test] +fn test_argwhere_3d() { + let tensor = TestTensorBool::<3>::from([ + [[false, false, false], [false, true, false]], + [[true, false, true], [true, true, false]], + ]); + let output = tensor.argwhere(); + + output.into_data().assert_eq( + &TensorData::from([[0, 1, 1], [1, 0, 0], [1, 0, 2], [1, 1, 0], [1, 1, 1]]), + false, + ); +} + +#[test] +fn test_nonzero_1d() { + let tensor = TestTensorBool::<1>::from([false, true, false, true, true]); + let data_actual = tensor + .nonzero() + .into_iter() + .map(|t| t.into_data()) + .collect::>(); + + assert_eq!(data_actual.len(), 1); + data_actual[0].assert_eq(&TensorData::from([1, 3, 4]), false); +} + +#[test] +fn test_nonzero_2d() { + // 2-D tensor + let tensor = TestTensorBool::<2>::from([[false, false], [false, true], [true, true]]); + let data_actual = tensor + .nonzero() + .into_iter() + .map(|t| t.into_data()) + .collect::>(); + let data_expected = [TensorData::from([1, 2, 2]), TensorData::from([1, 0, 1])]; + + assert_eq!(data_actual.len(), 2); + for (idx, actual) in data_actual.iter().enumerate() { + actual.assert_eq(&data_expected[idx], false) + } +} + +#[test] +fn test_nonzero_3d() { + // 3-D tensor + let tensor = TestTensorBool::<3>::from([ + [[false, false, false], [false, true, false]], + [[true, false, true], [true, true, false]], + ]); + let data_actual = tensor + .nonzero() + .into_iter() + .map(|t| t.into_data()) + .collect::>(); + let data_expected = [ + TensorData::from([0, 1, 1, 1, 1]), + TensorData::from([1, 0, 0, 1, 1]), + TensorData::from([1, 0, 2, 0, 1]), + ]; + + assert_eq!(data_actual.len(), 3); + for (idx, actual) in data_actual.iter().enumerate() { + actual.assert_eq(&data_expected[idx], false) + } +} + +#[test] +fn test_nonzero_empty() { + let tensor = TestTensorBool::<1>::from([false, false, false, false, false]); + let output = tensor.nonzero(); + + assert_eq!(output.len(), 0); +} + +#[test] +fn test_argwhere_empty() { + let tensor = TestTensorBool::<1>::from([false, false, false, false, false]); + let output = tensor.argwhere(); + + assert_eq!(output.shape(), Shape::new([0, 1])); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/cat.rs new file mode 100644 index 0000000..3dcda65 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/cat.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_cat_ops_bool() { + let device = Default::default(); + let tensor_1 = TestTensorBool::<2>::from_data([[false, true, true]], &device); + let tensor_2 = TestTensorBool::<2>::from_data([[true, true, false]], &device); + + let output = Tensor::cat(vec![tensor_1, tensor_2], 0); + + output.into_data().assert_eq( + &TensorData::from([[false, true, true], [true, true, false]]), + false, + ); +} + +#[test] +fn should_support_cat_with_empty_tensor_bool() { + let device = Default::default(); + let tensor_1 = TestTensorBool::<2>::from_data([[true, false, true]], &device); + let tensor_2: TestTensorBool<2> = TestTensorBool::empty([1, 0], &device); + + let output = Tensor::cat(vec![tensor_1, tensor_2], 1); + + output + .into_data() + .assert_eq(&TensorData::from([[true, false, true]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/comparison.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/comparison.rs new file mode 100644 index 0000000..2c971ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/comparison.rs @@ -0,0 +1,71 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_bool_equal() { + let data_1 = TensorData::from([[false, true, true], [true, false, true]]); + let data_2 = TensorData::from([[false, false, true], [false, true, true]]); + let device = Default::default(); + let tensor_1 = TestTensorBool::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorBool::<2>::from_data(data_2, &device); + + let data_actual_cloned = tensor_1.clone().equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.equal(tensor_2); + + let data_expected = TensorData::from([[true, false, true], [false, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn should_support_bool_not_equal() { + let data_1 = TensorData::from([[false, true, true], [true, false, true]]); + let data_2 = TensorData::from([[false, false, true], [false, true, true]]); + let device = Default::default(); + let tensor_1 = TestTensorBool::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorBool::<2>::from_data(data_2, &device); + + let data_actual_cloned = tensor_1.clone().not_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.not_equal(tensor_2); + + let data_expected = TensorData::from([[false, true, false], [true, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn should_support_bool_not() { + let data_1 = TensorData::from([[false, true, true], [true, true, false]]); + let tensor_1 = TestTensorBool::<2>::from_data(data_1, &Default::default()); + + let data_actual_cloned = tensor_1.clone().bool_not(); + let data_actual_inplace = tensor_1.bool_not(); + + let data_expected = TensorData::from([[true, false, false], [false, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_bool_equal_elem() { + let tensor_1 = TestTensorBool::<2>::from([[true, false, true], [false, true, false]]); + + let data_actual_cloned = tensor_1.clone().equal_elem(false); + let data_actual_inplace = tensor_1.equal_elem(false); + + let data_expected = TensorData::from([[false, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_bool_not_equal_elem() { + let tensor_1 = TestTensorBool::<2>::from([[true, false, true], [false, true, false]]); + + let data_actual_cloned = tensor_1.clone().not_equal_elem(true); + let data_actual_inplace = tensor_1.not_equal_elem(true); + + let data_expected = TensorData::from([[false, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/create_like.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/create_like.rs new file mode 100644 index 0000000..accd268 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/create_like.rs @@ -0,0 +1,34 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_zeros_like() { + let tensor = TestTensorBool::<3>::from([ + [[false, true, false], [true, true, true]], + [[false, false, false], [true, true, false]], + ]); + + let tensor = tensor.zeros_like(); + let expected = TensorData::from([ + [[false, false, false], [false, false, false]], + [[false, false, false], [false, false, false]], + ]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_ones_like() { + let tensor = TestTensorBool::<3>::from([ + [[false, true, false], [true, true, true]], + [[false, false, false], [true, true, false]], + ]); + + let tensor = tensor.ones_like(); + let expected = TensorData::from([ + [[true, true, true], [true, true, true]], + [[true, true, true], [true, true, true]], + ]); + + tensor.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/expand.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/expand.rs new file mode 100644 index 0000000..50310ca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/expand.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn expand_2d_bool() { + let tensor = TestTensorBool::<1>::from([false, true, false]); + let expanded_tensor = tensor.expand([3, 3]); + + let expected_data = TensorData::from([ + [false, true, false], + [false, true, false], + [false, true, false], + ]); + + expanded_tensor.into_data().assert_eq(&expected_data, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/flip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/flip.rs new file mode 100644 index 0000000..bda4941 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/flip.rs @@ -0,0 +1,33 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn flip_bool() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .greater_elem(10); + + let flipped = tensor.clone().flip([0, 2]); + + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).flip((0, 2)).gt(10) + let data_expected = TensorData::from([ + [ + [true, true, true, true], + [true, true, true, true], + [true, true, true, true], + ], + [ + [false, false, false, false], + [false, false, false, false], + [true, false, false, false], + ], + ]); + + flipped.into_data().assert_eq(&data_expected, false); + + // Test with no flip + let flipped = tensor.clone().flip([]); + tensor.into_data().assert_eq(&flipped.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/full.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/full.rs new file mode 100644 index 0000000..11c0b2e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/full.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_tensor_full() { + let device = Default::default(); + let bool_tensor = TestTensorBool::<2>::full([2, 2], true, &device); + bool_tensor + .into_data() + .assert_eq(&TensorData::from([[true, true], [true, true]]), false); + + let bool_tensor = TestTensorBool::<2>::full([2, 2], false, &device); + bool_tensor + .into_data() + .assert_eq(&TensorData::from([[false, false], [false, false]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/gather_scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/gather_scatter.rs new file mode 100644 index 0000000..b7b6b70 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/gather_scatter.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, TensorData}; + +#[test] +fn should_scatter_1d_bool() { + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, false], &device); + let values = TestTensorBool::from_data([false, true, true], &device); + let indices = TestTensorInt::from_ints([1, 0, 2], &device); + + let output = tensor.scatter(0, indices, values, IndexingUpdateOp::Add); + + output + .into_data() + .assert_eq(&TensorData::from([true, false, true]), false); +} + +#[test] +fn should_gather_1d_dim0_bool() { + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, false], &device); + let indices = TestTensorInt::from_ints([1, 1, 0, 1, 2], &device); + + let output = tensor.gather(0, indices); + + output + .into_data() + .assert_eq(&TensorData::from([false, false, true, false, false]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/init.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/init.rs new file mode 100644 index 0000000..6255d06 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/init.rs @@ -0,0 +1,31 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_bool_empty() { + let shape = [2, 2]; + let tensor = TestTensorBool::<2>::empty(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()) +} + +#[test] +fn should_support_bool_zeros() { + let shape = [2, 2]; + let tensor = TestTensorBool::<2>::zeros(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()); + + tensor + .into_data() + .assert_eq(&TensorData::from([[false, false], [false, false]]), false); +} + +#[test] +fn should_support_bool_ones() { + let shape = [2, 2]; + let tensor = TestTensorBool::<2>::ones(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()); + + tensor + .into_data() + .assert_eq(&TensorData::from([[true, true], [true, true]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/logical.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/logical.rs new file mode 100644 index 0000000..3ce4328 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/logical.rs @@ -0,0 +1,49 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_bool_and() { + let tensor1 = TestTensorBool::<2>::from([[false, true, false], [true, false, true]]); + let tensor2 = TestTensorBool::<2>::from([[true, true, false], [false, false, true]]); + let data_actual = tensor1.bool_and(tensor2).into_data(); + let data_expected = TensorData::from([[false, true, false], [false, false, true]]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_bool_or() { + let tensor1 = TestTensorBool::<2>::from([[false, true, false], [true, false, true]]); + let tensor2 = TestTensorBool::<2>::from([[true, true, false], [false, false, true]]); + let data_actual = tensor1.bool_or(tensor2).into_data(); + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_bool_xor() { + let tensor1 = TestTensorBool::<2>::from([[false, true, false], [true, false, true]]); + let tensor2 = TestTensorBool::<2>::from([[true, true, false], [false, false, true]]); + let data_actual = tensor1.bool_xor(tensor2).into_data(); + let data_expected = TensorData::from([[true, false, false], [true, false, false]]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_bool_or_vec() { + let device = Default::default(); + let tensor1 = TestTensorBool::<1>::full([256], 0, &device); + let tensor2 = TestTensorBool::<1>::full([256], 1, &device); + let data_actual = tensor1.bool_or(tensor2).into_data(); + let data_expected = TensorData::from([true; 256]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_bool_and_vec() { + let device = Default::default(); + let tensor1 = TestTensorBool::<1>::full([256], 0, &device); + let tensor2 = TestTensorBool::<1>::full([256], 1, &device); + let data_actual = tensor1.bool_and(tensor2).into_data(); + let data_expected = TensorData::from([false; 256]); + data_expected.assert_eq(&data_actual, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mask.rs new file mode 100644 index 0000000..243df0b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mask.rs @@ -0,0 +1,30 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_bool_mask_where_ops() { + let device = Default::default(); + let tensor = TestTensorBool::<2>::from_data([[true, false], [false, false]], &device); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + let value = + TestTensorBool::<2>::from_data(TensorData::from([[false, true], [true, false]]), &device); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([[false, false], [false, false]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_bool_mask_fill_ops() { + let device = Default::default(); + let tensor = TestTensorBool::<2>::from_data([[false, true], [false, false]], &device); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + + let output = tensor.mask_fill(mask, true); + let expected = TensorData::from([[true, true], [false, true]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mod.rs new file mode 100644 index 0000000..a7b31f1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/mod.rs @@ -0,0 +1,26 @@ +pub use super::*; // re-export test types + +mod all; +mod any; +mod argwhere_nonzero; +mod cat; +mod comparison; +mod create_like; +mod expand; +mod flip; +mod full; +mod gather_scatter; +mod init; +mod logical; +mod mask; +mod movedim; +mod permute; +mod repeat; +mod repeat_dim; +mod reshape; +mod select; +mod stack; +mod take; +mod transpose; +mod tri_mask; +mod unfold; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/movedim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/movedim.rs new file mode 100644 index 0000000..fba3f4b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/movedim.rs @@ -0,0 +1,56 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn movedim_bool() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .greater_elem(10); + + let permuted = tensor.clone().movedim(0, 2); + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim(0, 2).gt(10) + let expected = TensorData::from([ + [[false, true], [false, true], [false, true], [false, true]], + [[false, true], [false, true], [false, true], [false, true]], + [[false, true], [false, true], [false, true], [true, true]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().movedim(0, -1); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().movedim(0, 0); + permuted.into_data().assert_eq(&tensor.into_data(), false); +} + +#[test] +fn vec_input_bool() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .greater_elem(10); + + let permuted = tensor.clone().movedim(vec![0, 1], vec![1, 0]); + // from pytorch + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim([0, 1], [1, 0]).gt(10) + let expected = TensorData::from([ + [[false, false, false, false], [true, true, true, true]], + [[false, false, false, false], [true, true, true, true]], + [[false, false, false, true], [true, true, true, true]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axes + let permuted = tensor.clone().movedim(vec![-3, -2], vec![-2, -3]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axes + let permuted = tensor.clone().movedim(vec![0, 1], vec![0, 1]); + permuted.into_data().assert_eq(&tensor.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/permute.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/permute.rs new file mode 100644 index 0000000..ef095c0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/permute.rs @@ -0,0 +1,31 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn permute_bool() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .greater_elem(10); + + let permuted = tensor.clone().permute([2, 1, 0]); + + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).permute(2, 1, 0).gt(10) + let expected = TensorData::from([ + [[false, true], [false, true], [false, true]], + [[false, true], [false, true], [false, true]], + [[false, true], [false, true], [false, true]], + [[false, true], [false, true], [true, true]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().permute([-1, 1, 0]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().permute([0, 1, 2]); + permuted.into_data().assert_eq(&tensor.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat.rs new file mode 100644 index 0000000..ba0bd33 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat.rs @@ -0,0 +1,64 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_bool_repeat_ops_one_dimension() { + let data = TensorData::from([[true, false, false]]); + let tensor = TestTensorBool::<2>::from_data(data, &Default::default()); + + let output = tensor.repeat(&[4, 1, 1]); + let expected = TensorData::from([ + [true, false, false], + [true, false, false], + [true, false, false], + [true, false, false], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_bool_repeat_on_many_dimension() { + let data = TensorData::from([ + [[false, true], [true, false]], + [[true, true], [false, false]], + ]); + let tensor = TestTensorBool::<3>::from_data(data, &Default::default()); + + let output = tensor.repeat(&[2, 3, 2]); + let expected = TensorData::from([ + [ + [false, true, false, true], + [true, false, true, false], + [false, true, false, true], + [true, false, true, false], + [false, true, false, true], + [true, false, true, false], + ], + [ + [true, true, true, true], + [false, false, false, false], + [true, true, true, true], + [false, false, false, false], + [true, true, true, true], + [false, false, false, false], + ], + [ + [false, true, false, true], + [true, false, true, false], + [false, true, false, true], + [true, false, true, false], + [false, true, false, true], + [true, false, true, false], + ], + [ + [true, true, true, true], + [false, false, false, false], + [true, true, true, true], + [false, false, false, false], + [true, true, true, true], + [false, false, false, false], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat_dim.rs new file mode 100644 index 0000000..c3b33a8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/repeat_dim.rs @@ -0,0 +1,34 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_bool_repeat_ops() { + let data = TensorData::from([[true, false, false]]); + let tensor = TestTensorBool::<2>::from_data(data, &Default::default()); + + let output = tensor.repeat_dim(0, 4); + let expected = TensorData::from([ + [true, false, false], + [true, false, false], + [true, false, false], + [true, false, false], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_bool_repeat_on_dims_larger_than_1() { + let data = TensorData::from([ + [[false, true], [true, false]], + [[true, true], [false, false]], + ]); + let tensor = TestTensorBool::<3>::from_data(data, &Default::default()); + + let output = tensor.repeat_dim(1, 2); + let expected = TensorData::from([ + [[false, true], [true, false], [false, true], [true, false]], + [[true, true], [false, false], [true, true], [false, false]], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/reshape.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/reshape.rs new file mode 100644 index 0000000..7652aef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/reshape.rs @@ -0,0 +1,13 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_reshape_bool() { + let data = TensorData::from([false, true, false]); + let tensor = TestTensorBool::<1>::from_data(data, &Default::default()); + + let output = tensor.clone().reshape([1, 3]); + let expected = TensorData::from([[false, true, false]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/select.rs new file mode 100644 index 0000000..05e30a0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/select.rs @@ -0,0 +1,241 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, TensorData}; + +#[test] +fn should_select_bool_tensor_1d() { + // Test that select works for boolean tensors + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, true], &device); + let indices = TestTensorInt::from_data([0, 2, 1, 0], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([true, true, false, true]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_bool_tensor_2d() { + // Test that select works for boolean 2D tensors + let device = Default::default(); + let tensor = + TestTensorBool::<2>::from_data([[true, false, true], [false, true, false]], &device); + let indices = TestTensorInt::from_data([1, 0], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([[false, true, false], [true, false, true]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_bool_tensor() { + // Test that select_add works for boolean tensors + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, true], &device); + let values = TestTensorBool::<1>::from_data([false, true], &device); + let indices = TestTensorInt::from_data([0, 2], &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + // Note: select_add uses sum reduction, so: + // index 0: true OR false = true + // index 2: true OR true = true + // index 1: false (unchanged) + let expected = TensorData::from([true, false, true]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_bool_overlapping_indices() { + // Test accumulation behavior with overlapping indices + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([false, true], &device); + let indices = TestTensorInt::from_data([0, 0], &device); + let values = TestTensorBool::<1>::from_data([true, false], &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + // Index 0: false OR true OR false = true + let expected = TensorData::from([true, true]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_bool_false_to_true_case() { + // Test false OR true = true + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([false], &device); + let indices = TestTensorInt::from_data([0], &device); + let values = TestTensorBool::<1>::from_data([true], &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([true]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_bool_true_or_true_accumulation() { + // Test multiple true accumulations + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false], &device); + let indices = TestTensorInt::from_data([0, 0, 0], &device); + let values = TestTensorBool::<1>::from_data([true, true, true], &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([true, false]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_match_default_implementation_behavior() { + // Verify optimized implementation matches original default logic + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, true], &device); + let indices = TestTensorInt::from_data([0, 1, 0], &device); + let values = TestTensorBool::<1>::from_data([false, true, true], &device); + + let optimized_result = + tensor + .clone() + .select_assign(0, indices.clone(), values.clone(), IndexingUpdateOp::Add); + + // Manual default implementation logic + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + + optimized_result + .into_data() + .assert_eq(&default_result.into_data(), false); +} + +#[test] +fn should_select_add_bool_overlapping_indices_vs_default() { + // Test overlapping indices against default implementation + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([false, true], &device); + let indices = TestTensorInt::from_data([0, 0], &device); + let values = TestTensorBool::<1>::from_data([true, false], &device); + + let optimized_result = + tensor + .clone() + .select_assign(0, indices.clone(), values.clone(), IndexingUpdateOp::Add); + + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + + optimized_result + .into_data() + .assert_eq(&default_result.into_data(), false); +} + +#[test] +fn should_select_add_bool_true_or_true_accumulation_vs_default() { + // Test multiple true accumulations against default implementation + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false], &device); + let indices = TestTensorInt::from_data([0, 0, 0], &device); + let values = TestTensorBool::<1>::from_data([true, true, true], &device); + + let optimized_result = + tensor + .clone() + .select_assign(0, indices.clone(), values.clone(), IndexingUpdateOp::Add); + + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + + optimized_result + .into_data() + .assert_eq(&default_result.into_data(), false); +} + +#[test] +fn should_select_add_bool_false_to_true_case_vs_default() { + // Test false OR true case against default implementation + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([false], &device); + let indices = TestTensorInt::from_data([0], &device); + let values = TestTensorBool::<1>::from_data([true], &device); + + let optimized_result = + tensor + .clone() + .select_assign(0, indices.clone(), values.clone(), IndexingUpdateOp::Add); + + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + + optimized_result + .into_data() + .assert_eq(&default_result.into_data(), false); +} + +#[test] +fn should_select_add_bool_tensor_vs_default() { + // Test existing basic case against default implementation + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, true], &device); + let indices = TestTensorInt::from_data([0, 2], &device); + let values = TestTensorBool::<1>::from_data([false, false], &device); + + let optimized_result = + tensor + .clone() + .select_assign(0, indices.clone(), values.clone(), IndexingUpdateOp::Add); + + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + + optimized_result + .into_data() + .assert_eq(&default_result.into_data(), false); +} + +#[test] +#[should_panic(expected = "Tensors are not eq")] +fn should_fail_if_replacement_semantics_were_used() { + // Test that framework uses accumulation, not replacement + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true], &device); + let indices = TestTensorInt::from_data([0], &device); + let values = TestTensorBool::<1>::from_data([false], &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let replacement_expected = TensorData::from([false]); + + output.into_data().assert_eq(&replacement_expected, false); +} + +#[test] +#[should_panic(expected = "Tensors are not eq")] +fn should_fail_if_replacement_semantics_were_used_vs_default() { + // Test that default implementation also uses accumulation, not replacement + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true], &device); + let indices = TestTensorInt::from_data([0], &device); + let values = TestTensorBool::<1>::from_data([false], &device); + + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + let replacement_expected = TensorData::from([false]); + + default_result + .into_data() + .assert_eq(&replacement_expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/stack.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/stack.rs new file mode 100644 index 0000000..aea179e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/stack.rs @@ -0,0 +1,15 @@ +use super::*; +use alloc::vec; +use burn_tensor::{Tensor, TensorData}; + +#[test] +fn should_support_stack_ops_bool() { + let device = Default::default(); + let tensor_1 = TestTensorBool::<2>::from_data([[false, true, true]], &device); + let tensor_2 = TestTensorBool::<2>::from_data([[true, true, false]], &device); + + let output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[false, true, true]], [[true, true, false]]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/take.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/take.rs new file mode 100644 index 0000000..f11e29d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/take.rs @@ -0,0 +1,41 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_take_bool_tensor() { + // Test take with boolean tensors + let device = Default::default(); + let tensor = TestTensorBool::<2>::from_data([[true, false], [false, true]], &device); + let indices = TestTensorInt::<1>::from_data([1, 0], &device); + + let output = tensor.take::<1, 2>(0, indices); + let expected = TensorData::from([[false, true], [true, false]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_bool_tensor_with_2d_indices() { + // Test take with boolean tensors - output will be 3D + let device = Default::default(); + let tensor = TestTensorBool::<2>::from_data( + [ + [true, false, true], + [false, true, false], + [true, true, false], + ], + &device, + ); + + // 2D indices - shape [2, 2] + let indices = TestTensorInt::<2>::from_data([[0, 2], [1, 0]], &device); + let output = tensor.take::<2, 3>(0, indices); + + // Expected: shape [2, 2, 3] + let expected = TensorData::from([ + [[true, false, true], [true, true, false]], + [[false, true, false], [true, false, true]], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/transpose.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/transpose.rs new file mode 100644 index 0000000..51d44bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/transpose.rs @@ -0,0 +1,41 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_transpose_bool() { + let tensor = TestTensorBool::<3>::from_data( + [ + [[false, true, false], [false, false, false]], + [[false, false, true], [false, false, true]], + ], + &Default::default(), + ); + + let output = tensor.transpose(); + let expected = TensorData::from([ + [[false, false], [true, false], [false, false]], + [[false, false], [false, false], [true, true]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_swap_dims_bool() { + let tensor = TestTensorBool::<3>::from_data( + [ + [[false, true, false], [false, false, false]], + [[false, false, true], [false, false, true]], + ], + &Default::default(), + ); + + let output = tensor.swap_dims(0, 2); + let expected = TensorData::from([ + [[false, false], [false, false]], + [[true, false], [false, false]], + [[false, true], [false, true]], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/tri_mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/tri_mask.rs new file mode 100644 index 0000000..dfe3fa0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/tri_mask.rs @@ -0,0 +1,94 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn square_diag() { + let device = Default::default(); + let data_expected = TensorData::from([ + [false, true, true], + [true, false, true], + [true, true, false], + ]); + let tensor = TestTensorBool::<2>::diag_mask([3, 3], 0, &device); + tensor.into_data().assert_eq(&data_expected, false); +} + +#[test] +fn square_diag_offset() { + let device = Default::default(); + let data_expected = + TensorData::from([[true, false, true], [true, true, false], [true, true, true]]); + let tensor = TestTensorBool::<2>::diag_mask([3, 3], 1, &device); + tensor.into_data().assert_eq(&data_expected, false); +} + +#[test] +fn square_tri_upper() { + let device = Default::default(); + let data_expected = TensorData::from([ + [false, false, false], + [true, false, false], + [true, true, false], + ]); + let tensor = TestTensorBool::<2>::triu_mask([3, 3], 0, &device); + tensor.into_data().assert_eq(&data_expected, false); +} + +#[test] +fn square_tri_upper_offset() { + let device = Default::default(); + let data_expected = TensorData::from([ + [true, false, false], + [true, true, false], + [true, true, true], + ]); + let tensor = TestTensorBool::<2>::triu_mask([3, 3], 1, &device); + tensor.into_data().assert_eq(&data_expected, false); +} + +#[test] +fn square_tri_lower() { + let device = Default::default(); + + let data_expected = TensorData::from([ + [false, true, true], + [false, false, true], + [false, false, false], + ]); + let tensor = TestTensorBool::<2>::tril_mask([3, 3], 0, &device); + tensor.into_data().assert_eq(&data_expected, false); +} + +#[test] +fn square_tri_lower_offset() { + let device = Default::default(); + + let data_expected = TensorData::from([ + [true, true, true], + [false, true, true], + [false, false, true], + ]); + let tensor = TestTensorBool::<2>::tril_mask([3, 3], -1, &device); + tensor.into_data().assert_eq(&data_expected, false); +} + +#[test] +fn rect_diag() { + let device = Default::default(); + let data_expected = TensorData::from([ + [false, true, true, true], + [true, false, true, true], + [true, true, false, true], + ]); + let tensor = TestTensorBool::<2>::diag_mask([3, 4], 0, &device); + tensor.into_data().assert_eq(&data_expected, false); + + let data_expected = TensorData::from([ + [false, true, true], + [true, false, true], + [true, true, false], + [true, true, true], + ]); + let tensor = TestTensorBool::<2>::diag_mask([4, 3], 0, &device); + tensor.into_data().assert_eq(&data_expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/unfold.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/unfold.rs new file mode 100644 index 0000000..fb02534 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/bool/ops/unfold.rs @@ -0,0 +1,36 @@ +use super::*; +use burn_tensor::Distribution; +use burn_tensor::s; + +#[test] +fn test_unfold_bool() { + let device = Default::default(); + + let input = + TestTensor::<3>::random([2, 6, 6], Distribution::Default, &device).greater_elem(0.5); + + let dim = 1; + let size = 3; + let step = 2; + let actual: TestTensorBool<4> = input.clone().unfold(dim, size, step); + + let expected = TestTensorBool::<4>::empty([2, 2, 6, 3], &device) + .slice_assign( + s![.., 0, .., ..], + input + .clone() + .slice(s![.., 0..3, ..]) + .swap_dims(1, 2) + .unsqueeze_dim::<4>(1), + ) + .slice_assign( + s![.., 1, .., ..], + input + .clone() + .slice(s![.., 2..5, ..]) + .swap_dims(1, 2) + .unsqueeze_dim::<4>(1), + ); + + actual.to_data().assert_eq(&expected.to_data(), true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/clone_invariance.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/clone_invariance.rs new file mode 100644 index 0000000..806f960 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/clone_invariance.rs @@ -0,0 +1,761 @@ +/// This module tests whether basic tensor operations remain invariant when performed on clones, +/// meaning that cloning input tensors won't affect the results. +/// +/// Those are relevant tests because backends may employ unsafe optimizations to reuse tensor data +/// and use different kernels in such cases. We ensure that the results are consistent regardless +/// of the approach and that the input tensors are not modified when cloned. +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::activation::{ + gelu, log_sigmoid, log_softmax, mish, relu, sigmoid, silu, softmax, softplus, tanh, +}; +use burn_tensor::{Distribution, IndexingUpdateOp, TensorData}; + +pub trait CloneInvarianceTest { + type Args; + + fn args(&self) -> Self::Args; + + fn run(&self, args: &Self::Args, inplace: bool) -> TensorData; + + fn check(&self) { + let args = self.args(); + let out = self.run(&args, false); + let out_inplace = self.run(&args, true); + + out.assert_approx_eq::(&out_inplace, Tolerance::default()); + } +} + +macro_rules! clone_invariance_test { + (unary: $name:ident, ops_float: $ops:expr) => { + #[test] + #[allow(non_snake_case)] + fn $name() { + struct $name; + + impl CloneInvarianceTest<2> for $name { + type Args = TensorData; + + fn args(&self) -> Self::Args { + TestTensor::<2>::random([32, 32], Distribution::Default, &Default::default()) + .into_data() + .convert::() + } + + fn run(&self, args: &Self::Args, inplace: bool) -> TensorData { + let lhs = TestTensor::from_data(args.clone(), &Default::default()); + + if inplace { + $ops(lhs).into_data().convert::() + } else { + let out = $ops(lhs.clone()).into_data().convert::(); + lhs.into_data() + .assert_approx_eq::(args, Tolerance::default()); + out + } + } + } + + CloneInvarianceTest::<2>::check(&$name); + } + }; + + (binary: $name:ident, ops_float: $ops:expr) => { + #[test] + #[allow(non_snake_case)] + fn $name() { + struct $name; + + impl CloneInvarianceTest<2> for $name { + type Args = (TensorData, TensorData); + + fn args(&self) -> Self::Args { + let device = Default::default(); + ( + TestTensor::<2>::ones([32, 32], &device) + .into_data() + .convert::(), + // Avoid div by zero. + TestTensor::<2>::ones([32, 32], &device) + .into_data() + .convert::(), + ) + } + + fn run(&self, (lhs_arg, rhs_arg): &Self::Args, inplace: bool) -> TensorData { + let device = Default::default(); + let lhs = TestTensor::from_data(lhs_arg.clone(), &device); + let rhs = TestTensor::from_data(rhs_arg.clone(), &device); + + if inplace { + $ops(lhs, rhs).into_data().convert::() + } else { + let out = $ops(lhs.clone(), rhs.clone()).into_data().convert::(); + + lhs.into_data() + .assert_approx_eq::(lhs_arg, Tolerance::default()); + rhs.into_data() + .assert_approx_eq::(rhs_arg, Tolerance::default()); + + out + } + } + } + + CloneInvarianceTest::<2>::check(&$name); + } + }; + + (unary: $name:ident, ops_int: $ops:expr) => { + #[test] + #[allow(non_snake_case)] + fn $name() { + struct $name; + + impl CloneInvarianceTest<2> for $name { + type Args = TensorData; + + fn args(&self) -> Self::Args { + TestTensor::<2>::random( + [32, 32], + Distribution::Uniform(0.0, 50.0), + &Default::default(), + ) + .into_data() + .convert::() + } + + fn run(&self, args: &Self::Args, inplace: bool) -> TensorData { + let lhs = TestTensorInt::from_data(args.clone(), &Default::default()); + + if inplace { + $ops(lhs).into_data().convert::() + } else { + let out = $ops(lhs.clone()).into_data().convert::(); + lhs.into_data() + .convert::() + .assert_approx_eq::(args, Tolerance::default()); + out + } + } + } + + CloneInvarianceTest::<2>::check(&$name); + } + }; + + (binary: $name:ident, ops_int: $ops:expr) => { + #[test] + #[allow(non_snake_case)] + fn $name() { + struct $name; + + impl CloneInvarianceTest<2> for $name { + type Args = (TensorData, TensorData); + + fn args(&self) -> Self::Args { + let device = Default::default(); + ( + TestTensor::<2>::random([32, 32], Distribution::Uniform(0., 50.), &device) + .into_data() + .convert::(), + // Avoid div by zero. + TestTensor::<2>::random([32, 32], Distribution::Uniform(1., 51.), &device) + .into_data() + .convert::(), + ) + } + + fn run(&self, (lhs_arg, rhs_arg): &Self::Args, inplace: bool) -> TensorData { + let device = Default::default(); + let lhs = TestTensorInt::from_data(lhs_arg.clone(), &device); + let rhs = TestTensorInt::from_data(rhs_arg.clone(), &device); + + if inplace { + $ops(lhs, rhs).into_data().convert::() + } else { + let out = $ops(lhs.clone(), rhs.clone()).into_data().convert::(); + + lhs.into_data() + .convert::() + .assert_approx_eq::(lhs_arg, Tolerance::default()); + rhs.into_data() + .convert::() + .assert_approx_eq::(rhs_arg, Tolerance::default()); + + out + } + } + } + + CloneInvarianceTest::<2>::check(&$name); + } + }; +} + +mod float { + use super::*; + + // Unary ops + clone_invariance_test!( + unary: AddScalar, + ops_float: |tensor: TestTensor<2>| tensor.add_scalar(2.0) + ); + clone_invariance_test!( + unary: SubScalar, + ops_float: |tensor: TestTensor<2>| tensor.sub_scalar(2.0) + ); + clone_invariance_test!( + unary: DivScalar, + ops_float: |tensor: TestTensor<2>| tensor.div_scalar(2.0) + ); + clone_invariance_test!( + unary: MulScalar, + ops_float: |tensor: TestTensor<2>| tensor.mul_scalar(2.0) + ); + clone_invariance_test!( + unary: PowScalar, + ops_float: |tensor: TestTensor<2>| tensor.powf_scalar(2.0) + ); + clone_invariance_test!( + unary: Square, + ops_float: |tensor: TestTensor<2>| tensor.square() + ); + clone_invariance_test!( + unary: Sqrt, + ops_float: |tensor: TestTensor<2>| tensor.sqrt() + ); + clone_invariance_test!( + unary: Exp, + ops_float: |tensor: TestTensor<2>| tensor.exp() + ); + clone_invariance_test!( + unary: Neg, + ops_float: |tensor: TestTensor<2>| tensor.neg() + ); + clone_invariance_test!( + unary: MeanDim, + ops_float: |tensor: TestTensor<2>| tensor.mean_dim(1) + ); + clone_invariance_test!( + unary: SumDim, + ops_float: |tensor: TestTensor<2>| tensor.sum_dim(1) + ); + clone_invariance_test!( + unary: Sum, + ops_float: |tensor: TestTensor<2>| tensor.sum().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: Mean, + ops_float: |tensor: TestTensor<2>| tensor.mean().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: Clamp, + ops_float: |tensor: TestTensor<2>| tensor.clamp(-2., 2.) + ); + clone_invariance_test!( + unary: ClampMin, + ops_float: |tensor: TestTensor<2>| tensor.clamp_min(-2.) + ); + clone_invariance_test!( + unary: ClampMax, + ops_float: |tensor: TestTensor<2>| tensor.clamp_max(2.) + ); + clone_invariance_test!( + unary: Abs, + ops_float: |tensor: TestTensor<2>| tensor.abs() + ); + clone_invariance_test!( + unary: Cos, + ops_float: |tensor: TestTensor<2>| tensor.cos() + ); + clone_invariance_test!( + unary: Sin, + ops_float: |tensor: TestTensor<2>| tensor.sin() + ); + clone_invariance_test!( + unary: Tan, + ops_float: |tensor: TestTensor<2>| tensor.tan() + ); + clone_invariance_test!( + unary: Log, + ops_float: |tensor: TestTensor<2>| tensor.log() + ); + clone_invariance_test!( + unary: Log1P, + ops_float: |tensor: TestTensor<2>| tensor.log1p() + ); + clone_invariance_test!( + unary: SwapDims, + ops_float: |tensor: TestTensor<2>| tensor.swap_dims(0, 1) + ); + clone_invariance_test!( + unary: Transpose, + ops_float: |tensor: TestTensor<2>| tensor.transpose() + ); + clone_invariance_test!( + unary: Slice, + ops_float: |tensor: TestTensor<2>| tensor.slice([0..12, 12..24]) + ); + clone_invariance_test!( + unary: Erf, + ops_float: |tensor: TestTensor<2>| tensor.erf() + ); + clone_invariance_test!( + unary: EqualElem, + ops_float: |tensor: TestTensor<2>| tensor.equal_elem(0.5) + ); + clone_invariance_test!( + unary: NotEqualElem, + ops_float: |tensor: TestTensor<2>| tensor.not_equal_elem(0.5) + ); + clone_invariance_test!( + unary: GreaterElem, + ops_float: |tensor: TestTensor<2>| tensor.greater_elem(0.5) + ); + clone_invariance_test!( + unary: GreaterEqualElem, + ops_float: |tensor: TestTensor<2>| tensor.greater_equal_elem(0.5) + ); + clone_invariance_test!( + unary: LowerElem, + ops_float: |tensor: TestTensor<2>| tensor.lower_elem(0.5) + ); + clone_invariance_test!( + unary: LowerEqualElem, + ops_float: |tensor: TestTensor<2>| tensor.lower_equal_elem(0.5) + ); + clone_invariance_test!( + unary: Argmax, + ops_float: |tensor: TestTensor<2>| tensor.argmax(0) + ); + clone_invariance_test!( + unary: Argmin, + ops_float: |tensor: TestTensor<2>| tensor.argmin(0) + ); + clone_invariance_test!( + unary: Max, + ops_float: |tensor: TestTensor<2>| tensor.max().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: Min, + ops_float: |tensor: TestTensor<2>| tensor.min().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: MaxDim, + ops_float: |tensor: TestTensor<2>| tensor.max_dim(1) + ); + clone_invariance_test!( + unary: MaxDimWithIndices, + ops_float: |tensor: TestTensor<2>| tensor.max_dim_with_indices(1).0 + ); + clone_invariance_test!( + unary: MinDimWithIndices, + ops_float: |tensor: TestTensor<2>| tensor.min_dim_with_indices(1).0 + ); + clone_invariance_test!( + unary: MinDim, + ops_float: |tensor: TestTensor<2>| tensor.min_dim(1) + ); + clone_invariance_test!( + unary: Repeat, + ops_float: |tensor: TestTensor<2>| { + tensor.reshape([1, 32, 32]).repeat_dim(0, 4).reshape([4 * 32, 32]) + } + ); + clone_invariance_test!( + unary: Reshape, + ops_float: |tensor: TestTensor<2>| { + let shape = tensor.shape(); + let new_shape = [shape.num_elements(), 1]; + tensor.reshape(new_shape) + } + ); + clone_invariance_test!( + unary: Gatter, + ops_float: |tensor: TestTensor<2>| { + let shape = tensor.shape(); + let indices = TestTensorInt::ones(shape, &Default::default()); + tensor.gather(0, indices) + } + ); + clone_invariance_test!( + unary: Select, + ops_float: |tensor: TestTensor<2>| { + let indices = TestTensorInt::from_ints([1, 2, 0, 5], &Default::default()); + tensor.select(0, indices) + } + ); + clone_invariance_test!( + unary: MaskFill, + ops_float: |tensor: TestTensor<2>| { + let mask = tensor.clone().greater_elem(0.5); + tensor.mask_fill(mask, 77.0) + } + ); + + // Activation + clone_invariance_test!( + unary: Softmax, + ops_float: |tensor: TestTensor<2>| softmax(tensor, 1) + ); + clone_invariance_test!( + unary: LogSoftmax, + ops_float: |tensor: TestTensor<2>| log_softmax(tensor, 1) + ); + clone_invariance_test!( + unary: Sigmoid, + ops_float: |tensor: TestTensor<2>| sigmoid(tensor) + ); + clone_invariance_test!( + unary: LogSigmoid, + ops_float: |tensor: TestTensor<2>| log_sigmoid(tensor) + ); + clone_invariance_test!( + unary: Relu, + ops_float: |tensor: TestTensor<2>| relu(tensor) + ); + clone_invariance_test!( + unary: Gelu, + ops_float: |tensor: TestTensor<2>| gelu(tensor) + ); + clone_invariance_test!( + unary: Mish, + ops_float: |tensor: TestTensor<2>| mish(tensor) + ); + clone_invariance_test!( + unary: Silu, + ops_float: |tensor: TestTensor<2>| silu(tensor) + ); + clone_invariance_test!( + unary: Softplus, + ops_float: |tensor: TestTensor<2>| softplus(tensor, 1.0) + ); + clone_invariance_test!( + unary: Tanh, + ops_float: |tensor: TestTensor<2>| tanh(tensor) + ); + + // Binary ops + clone_invariance_test!( + binary: Add, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.add(rhs) + ); + clone_invariance_test!( + binary: Sub, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.sub(rhs) + ); + clone_invariance_test!( + binary: Div, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.div(rhs) + ); + clone_invariance_test!( + binary: Mul, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.mul(rhs) + ); + clone_invariance_test!( + binary: Matmul, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.matmul(rhs) + ); + clone_invariance_test!( + binary: Equal, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.equal(rhs) + ); + clone_invariance_test!( + binary: Greater, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.greater(rhs) + ); + clone_invariance_test!( + binary: GreaterEqual, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.greater_equal(rhs) + ); + clone_invariance_test!( + binary: Lower, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.lower(rhs) + ); + clone_invariance_test!( + binary: LowerEqual, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| lhs.lower_equal(rhs) + ); + clone_invariance_test!( + binary: Cat, + ops_float: |lhs: TestTensor<2>, rhs: TestTensor<2>| { + let lhs = lhs.reshape([1usize, 32, 32]); + let rhs = rhs.reshape([1usize, 32, 32]); + + TestTensor::cat(vec![lhs, rhs], 0).reshape([64, 32]) + } + ); + clone_invariance_test!( + binary: Scatter, + ops_float: |tensor: TestTensor<2>, values: TestTensor<2>| { + let shape = tensor.shape(); + let indices = TestTensorInt::ones(shape, &Default::default()); + tensor.scatter(0, indices, values, IndexingUpdateOp::Add) + } + ); + clone_invariance_test!( + binary: SliceAssign, + ops_float: |tensor: TestTensor<2>, values: TestTensor<2>| { + tensor.slice_assign([0..12, 12..24], values.slice([12..24, 0..12])) + } + ); + clone_invariance_test!( + binary: MaskWhere, + ops_float: |tensor: TestTensor<2>, values: TestTensor<2>| { + let mask = tensor.clone().greater_elem(0.5); + tensor.mask_where(mask, values) + } + ); + clone_invariance_test!( + binary: SelectAssign, + ops_float: |tensor: TestTensor<2>, values: TestTensor<2>| { + let indices = TestTensorInt::from_ints([1, 2, 0, 5], &Default::default()); + let values = values.select(0, indices.clone()); + tensor.select_assign(0, indices, values, IndexingUpdateOp::Add) + } + ); +} + +mod int { + use super::*; + + // Unary ops + clone_invariance_test!( + unary: AddScalar, + ops_int: |tensor: TestTensorInt<2>| tensor.add_scalar(2.0) + ); + clone_invariance_test!( + unary: SubScalar, + ops_int: |tensor: TestTensorInt<2>| tensor.sub_scalar(2.0) + ); + clone_invariance_test!( + unary: DivScalar, + ops_int: |tensor: TestTensorInt<2>| tensor.div_scalar(2.0) + ); + clone_invariance_test!( + unary: MulScalar, + ops_int: |tensor: TestTensorInt<2>| tensor.mul_scalar(2.0) + ); + clone_invariance_test!( + unary: Neg, + ops_int: |tensor: TestTensorInt<2>| tensor.neg() + ); + clone_invariance_test!( + unary: MeanDim, + ops_int: |tensor: TestTensorInt<2>| tensor.mean_dim(1) + ); + clone_invariance_test!( + unary: SumDim, + ops_int: |tensor: TestTensorInt<2>| tensor.sum_dim(1) + ); + clone_invariance_test!( + unary: Sum, + ops_int: |tensor: TestTensorInt<2>| tensor.sum().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: Mean, + ops_int: |tensor: TestTensorInt<2>| tensor.mean().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: Clamp, + ops_int: |tensor: TestTensorInt<2>| tensor.clamp(-2., 2.) + ); + clone_invariance_test!( + unary: ClampMin, + ops_int: |tensor: TestTensorInt<2>| tensor.clamp_min(-2.) + ); + clone_invariance_test!( + unary: ClampMax, + ops_int: |tensor: TestTensorInt<2>| tensor.clamp_max(2.) + ); + clone_invariance_test!( + unary: Abs, + ops_int: |tensor: TestTensorInt<2>| tensor.abs() + ); + clone_invariance_test!( + unary: SwapDims, + ops_int: |tensor: TestTensorInt<2>| tensor.swap_dims(0, 1) + ); + clone_invariance_test!( + unary: Transpose, + ops_int: |tensor: TestTensorInt<2>| tensor.transpose() + ); + clone_invariance_test!( + unary: Slice, + ops_int: |tensor: TestTensorInt<2>| tensor.slice([0..12, 12..24]) + ); + clone_invariance_test!( + unary: EqualElem, + ops_int: |tensor: TestTensorInt<2>| tensor.equal_elem(25) + ); + clone_invariance_test!( + unary: NotEqualElem, + ops_int: |tensor: TestTensorInt<2>| tensor.not_equal_elem(25) + ); + clone_invariance_test!( + unary: GreaterElem, + ops_int: |tensor: TestTensorInt<2>| tensor.greater_elem(25) + ); + clone_invariance_test!( + unary: GreaterEqualElem, + ops_int: |tensor: TestTensorInt<2>| tensor.greater_equal_elem(25) + ); + clone_invariance_test!( + unary: LowerElem, + ops_int: |tensor: TestTensorInt<2>| tensor.lower_elem(25) + ); + clone_invariance_test!( + unary: LowerEqualElem, + ops_int: |tensor: TestTensorInt<2>| tensor.lower_equal_elem(25) + ); + clone_invariance_test!( + unary: Argmax, + ops_int: |tensor: TestTensorInt<2>| tensor.argmax(0) + ); + clone_invariance_test!( + unary: Argmin, + ops_int: |tensor: TestTensorInt<2>| tensor.argmin(0) + ); + clone_invariance_test!( + unary: Max, + ops_int: |tensor: TestTensorInt<2>| tensor.max().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: Min, + ops_int: |tensor: TestTensorInt<2>| tensor.min().unsqueeze::<2>() + ); + clone_invariance_test!( + unary: MaxDim, + ops_int: |tensor: TestTensorInt<2>| tensor.max_dim(1) + ); + clone_invariance_test!( + unary: MaxDimWithIndices, + ops_int: |tensor: TestTensorInt<2>| tensor.max_dim_with_indices(1).0 + ); + clone_invariance_test!( + unary: MinDimWithIndices, + ops_int: |tensor: TestTensorInt<2>| tensor.min_dim_with_indices(1).0 + ); + clone_invariance_test!( + unary: MinDim, + ops_int: |tensor: TestTensorInt<2>| tensor.min_dim(1) + ); + clone_invariance_test!( + unary: Repeat, + ops_int: |tensor: TestTensorInt<2>| { + tensor.reshape([1, 32, 32]).repeat_dim(0, 4).reshape([4 * 32, 32]) + } + ); + clone_invariance_test!( + unary: Reshape, + ops_int: |tensor: TestTensorInt<2>| { + let shape = tensor.shape(); + let new_shape = [shape.num_elements(), 1]; + tensor.reshape(new_shape) + } + ); + clone_invariance_test!( + unary: Gatter, + ops_int: |tensor: TestTensorInt<2>| { + let shape = tensor.shape(); + let indices = TestTensorInt::ones(shape, &Default::default()); + tensor.gather(0, indices) + } + ); + clone_invariance_test!( + unary: Select, + ops_int: |tensor: TestTensorInt<2>| { + let indices = TestTensorInt::from_ints([1, 2, 0, 5], &Default::default()); + tensor.select(0, indices) + } + ); + clone_invariance_test!( + unary: MaskFill, + ops_int: |tensor: TestTensorInt<2>| { + let mask = tensor.clone().greater_elem(0.5); + tensor.mask_fill(mask, 77.0) + } + ); + + // Binary ops + clone_invariance_test!( + binary: Add, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.add(rhs) + ); + clone_invariance_test!( + binary: Sub, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.sub(rhs) + ); + clone_invariance_test!( + binary: Div, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.div(rhs) + ); + clone_invariance_test!( + binary: Mul, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.mul(rhs) + ); + clone_invariance_test!( + binary: Equal, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.equal(rhs) + ); + clone_invariance_test!( + binary: NotEqual, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.not_equal(rhs) + ); + clone_invariance_test!( + binary: Greater, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.greater(rhs) + ); + clone_invariance_test!( + binary: GreaterEqual, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.greater_equal(rhs) + ); + clone_invariance_test!( + binary: Lower, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.lower(rhs) + ); + clone_invariance_test!( + binary: LowerEqual, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| lhs.lower_equal(rhs) + ); + clone_invariance_test!( + binary: Cat, + ops_int: |lhs: TestTensorInt<2>, rhs: TestTensorInt<2>| { + let lhs = lhs.reshape([1usize, 32, 32]); + let rhs = rhs.reshape([1usize, 32, 32]); + + TestTensorInt::cat(vec![lhs, rhs], 0).reshape([64, 32]) + } + ); + clone_invariance_test!( + binary: Scatter, + ops_int: |tensor: TestTensorInt<2>, values: TestTensorInt<2>| { + let shape = tensor.shape(); + let indices = TestTensorInt::ones(shape, &Default::default()); + tensor.scatter(0, indices, values, IndexingUpdateOp::Add) + } + ); + clone_invariance_test!( + binary: SliceAssign, + ops_int: |tensor: TestTensorInt<2>, values: TestTensorInt<2>| { + tensor.slice_assign([0..12, 12..24], values.slice([12..24, 0..12])) + } + ); + clone_invariance_test!( + binary: MaskWhere, + ops_int: |tensor: TestTensorInt<2>, values: TestTensorInt<2>| { + let mask = tensor.clone().greater_elem(0.5); + tensor.mask_where(mask, values) + } + ); + clone_invariance_test!( + binary: SelectAssign, + ops_int: |tensor: TestTensorInt<2>, values: TestTensorInt<2>| { + let indices = TestTensorInt::from_ints([1, 2, 0, 5], &Default::default()); + let values = values.select(0, indices.clone()); + tensor.select_assign(0, indices, values, IndexingUpdateOp::Add) + } + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/celu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/celu.rs new file mode 100644 index 0000000..49b4812 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/celu.rs @@ -0,0 +1,34 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_celu_d2() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [-3.0, 0.5]]); + + let output = activation::celu(tensor, 1.0); + // celu(1, 1) = 1 + // celu(7, 1) = 7 + // celu(-3, 1) = 1 * (exp(-3) - 1) = -0.950213 + // celu(0.5, 1) = 0.5 + let expected = TensorData::from([[1.0, 7.0], [-0.950213, 0.5]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_celu_with_alpha() { + let tensor = TestTensor::<1>::from([0.0, -1.0, -2.0]); + + let output = activation::celu(tensor, 2.0); + // celu(0, 2) = 0 + // celu(-1, 2) = 2 * (exp(-0.5) - 1) = -0.786939 + // celu(-2, 2) = 2 * (exp(-1) - 1) = -1.264241 + let expected = TensorData::from([0.0, -0.786939, -1.264241]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/elu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/elu.rs new file mode 100644 index 0000000..f38cc07 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/elu.rs @@ -0,0 +1,32 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_elu() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::elu(tensor, 1.0); + // elu(1, 1) = 1, elu(7, 1) = 7, elu(13, 1) = 13 + // elu(-3, 1) = 1 * (exp(-3) - 1) = -0.950213 + let expected = TensorData::from([[1.0, 7.0], [13.0, -0.950213]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_elu_alpha() { + let tensor = TestTensor::<1>::from([0.0, -1.0, -2.0]); + + let output = activation::elu(tensor, 2.0); + // elu(0, 2) = 2*(exp(0)-1) = 0 + // elu(-1, 2) = 2*(exp(-1)-1) = 2*(-0.632121) = -1.264241 + // elu(-2, 2) = 2*(exp(-2)-1) = 2*(-0.864665) = -1.729329 + let expected = TensorData::from([0.0, -1.264241, -1.729329]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/gelu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/gelu.rs new file mode 100644 index 0000000..cad440a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/gelu.rs @@ -0,0 +1,20 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_gelu() { + let tensor = TestTensor::<2>::from([[ + 0.5447, 0.9809, 0.4114, 0.1398, 0.8045, 0.4103, 0.2388, 0.5262, 0.6677, 0.6737, + ]]); + let output = activation::gelu(tensor); + let expected = TensorData::from([[ + 0.3851, 0.8207, 0.2714, 0.0777, 0.6351, 0.2704, 0.1419, 0.3687, 0.4993, 0.5051, + ]]); + + // Low precision to allow approximation implementation using tanh + output.into_data().assert_approx_eq::( + &expected, + Tolerance::default().set_half_precision_absolute(2e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/glu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/glu.rs new file mode 100644 index 0000000..f8e46fa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/glu.rs @@ -0,0 +1,28 @@ +use super::*; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_glu_d3() { + let tensor = TestTensor::<3>::from([[ + [ + -0.5710, -1.3416, 1.9128, -0.8257, -0.1331, -1.4804, -0.6281, -0.6115, + ], + [ + 0.0267, -1.3834, 0.2752, 0.7844, -0.3549, -0.4274, 0.3290, -0.5459, + ], + [ + -1.6347, -2.0908, 1.8801, 0.3541, 0.2237, 1.0377, 2.4850, 0.3490, + ], + ]]); + + let output = activation::glu(tensor, 2); + + output.into_data().assert_approx_eq::( + &TensorData::from([[ + [-0.2665, -0.2487, 0.6656, -0.2904], + [0.0110, -0.5461, 0.1601, 0.2877], + [-0.9084, -1.5439, 1.7355, 0.2077], + ]]), + Default::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/hard_sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/hard_sigmoid.rs new file mode 100644 index 0000000..c1af47d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/hard_sigmoid.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_hard_sigmoid() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::hard_sigmoid(tensor, 0.2, 0.5); + let expected = TensorData::from([[0.7, 1.0], [1.0, 0.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_hard_sigmoid_overflow() { + let tensor = TestTensor::<1>::from([FloatElem::MAX, FloatElem::MIN]); + + let output = activation::hard_sigmoid(tensor, 0.2, 0.5); + let expected = TensorData::from([1.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/leaky_relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/leaky_relu.rs new file mode 100644 index 0000000..e1009cb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/leaky_relu.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_leaky_relu_d2() { + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, -4.0, 5.0]]); + + let output = activation::leaky_relu(tensor, 0.01); + + // Account for conversion errors if `FloatType != f32` + output.into_data().assert_approx_eq::( + &TensorData::from([[0.0, -0.01, 2.0], [3.0, -0.04, 5.0]]), + Tolerance::default(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/log_sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/log_sigmoid.rs new file mode 100644 index 0000000..62c94d4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/log_sigmoid.rs @@ -0,0 +1,37 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{ElementConversion, TensorData, activation}; + +#[test] +fn test_log_sigmoid() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::log_sigmoid(tensor); + let expected = TensorData::from([[-3.132617e-1, -9.114665e-4], [-2.260327e-6, -3.0485873]]); + + let tolerance = Tolerance::rel_abs(0.01, 0.0001); + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_log_sigmoid_numerical_stability() { + let tensor = TestTensor::<1>::from([300.0, -300.0]); + + let output = activation::log_sigmoid(tensor); + + // For large negative values, the previous implementation −log(1 + exp(−x)) would give -inf + let expected = TensorData::from([0.0, -300.0]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let tensor = TestTensor::<1>::from([FloatElem::MAX, FloatElem::MIN]); + let output = activation::log_sigmoid(tensor); + let expected = TensorData::from([0.elem(), FloatElem::MIN]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mish.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mish.rs new file mode 100644 index 0000000..2cb2f20 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mish.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_mish() { + let tensor = TestTensor::<2>::from([[-0.4240, -0.9574, -0.2215], [-0.5767, 0.7218, -0.1620]]); + + let output = activation::mish(tensor); + let expected = TensorData::from([ + [-0.19709, -0.30056, -0.11714], + [-0.24132, 0.58235, -0.08877], + ]); + + // Metal has less precise trigonometric functions (tanh inside mish) + let tolerance = Tolerance::default().set_half_precision_relative(1e-2); + + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mod.rs new file mode 100644 index 0000000..be70b93 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/mod.rs @@ -0,0 +1,22 @@ +use super::*; + +mod celu; +mod elu; +mod gelu; +mod glu; +mod hard_sigmoid; +mod leaky_relu; +mod log_sigmoid; +mod mish; +mod prelu; +mod quiet_softmax; +mod relu; +mod selu; +mod sigmoid; +mod silu; +mod softmax; +mod softmin; +mod softplus; +mod softsign; +mod tanh_activation; +mod thresholded_relu; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/prelu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/prelu.rs new file mode 100644 index 0000000..f4d1c35 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/prelu.rs @@ -0,0 +1,101 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_prelu_2_dimension() { + let data = [ + [-1.1, 0.0, 1.2, 0.25, -5.4], + [-4.567, 0.56, -1.55, 99.9, 0.0], + ]; + let tensor = TestTensor::<2>::from(data); + let output = activation::prelu(tensor, TestTensor::from([0.5, 0.25, 0.0, -0.8, -0.4])); + let expected = TensorData::from([ + [-0.5500, 0.0000, 1.2000, 0.2500, 2.1600], + [-2.2835, 0.5600, -0.0000, 99.9000, -0.0000], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} +#[test] +fn test_prelu_2_dimension_scalar_weight() { + let data = [ + [-1.1, 0.0, 1.2, 0.25, -5.4], + [-4.567, 0.56, -1.55, 99.9, 0.0], + ]; + let tensor = TestTensor::<2>::from(data); + let output = activation::prelu(tensor, TestTensor::from([-0.8])); + let expected = TensorData::from([ + [0.8800, -0.0000, 1.2000, 0.2500, 4.3200], + [3.6536, 0.5600, 1.2400, 99.9000, -0.0000], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_prelu_positives() { + // Check that positives are untouched + let data = [[ + 0.5447, 0.9809, 0.4114, 0.1398, 0.8045, 0.4103, 0.2388, 0.5262, 0.6677, 0.6737, + ]]; + let tensor = TestTensor::<2>::from(data); + let output = activation::prelu(tensor, TestTensor::from([0.25])); + let expected = TensorData::from(data); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_prelu_zero_weight() { + // test that with weight 0 it behaves as relu + let data = [-1.1, 0.0, 1.2, 0.25, -5.4]; + let tensor = TestTensor::<1>::from(data); + let output = activation::prelu(tensor, TestTensor::from([0.0])); + let expected = TensorData::from([0.0, 0.0, 1.2, 0.25, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_prelu_some_weight() { + // test that with some non zero weight it works like leaky relu + let data = [-1.1, 0.0, 1.2, 0.25, -5.4]; + let tensor = TestTensor::<1>::from(data); + let output = activation::prelu(tensor, TestTensor::from([0.5])); + let expected = TensorData::from([-0.550, 0.0, 1.20, 0.250, -2.70]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[should_panic] +fn test_prelu_single_dim_multi_weight() { + // should panic because the data has only 1 channel + let data = [-1.1, 2.0, 1.2, 0.25, -5.4]; + let tensor = TestTensor::<1>::from(data); + let data_actual = + activation::prelu(tensor, TestTensor::from([0.5, -0.25, 0.0, 0.5, -1.0])).into_data(); + let data_expected = TensorData::from([-0.550, 0.0, 1.20, 0.250, -2.70]); + data_expected.assert_approx_eq::(&data_actual, Tolerance::default()); +} + +#[test] +#[should_panic] +fn test_prelu_multi_dim_wrong_weights() { + let data = [ + [-1.1, 0.0, 1.2, 0.25, -5.4], + [-4.567, 0.56, -1.55, 99.9, 0.0], + ]; + let tensor = TestTensor::<2>::from(data); + let _ = activation::prelu(tensor, TestTensor::from([-0.8, 0.1])); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/quiet_softmax.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/quiet_softmax.rs new file mode 100644 index 0000000..32e0385 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/quiet_softmax.rs @@ -0,0 +1,15 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_quiet_softmax_d2() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::quiet_softmax(tensor, 1); + let expected = TensorData::from([[2.47e-03, 9.975e-01], [1.0, 1.1254e-07]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/relu.rs new file mode 100644 index 0000000..2e91a24 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/relu.rs @@ -0,0 +1,13 @@ +use super::*; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_relu_d2() { + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, -4.0, 5.0]]); + + let output = activation::relu(tensor); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 0.0, 2.0], [3.0, 0.0, 5.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/selu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/selu.rs new file mode 100644 index 0000000..a17109a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/selu.rs @@ -0,0 +1,37 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_selu() { + // selu(x) = gamma * x if x > 0, gamma * alpha * (exp(x) - 1) if x <= 0 + // alpha = 1.6733, gamma = 1.0507 + let tensor = TestTensor::<2>::from([[0.0, 1.0, -1.0], [2.0, -2.0, 0.5]]); + + let output = activation::selu(tensor); + + // Expected values computed from the formula: + // selu(0.0) = 1.0507 * 1.6733 * (exp(0) - 1) = 0.0 + // selu(1.0) = 1.0507 * 1.0 = 1.0507 + // selu(-1.0) = 1.0507 * 1.6733 * (exp(-1) - 1) = 1.7581 * (0.3679 - 1) = -1.1113 + // selu(2.0) = 1.0507 * 2.0 = 2.1014 + // selu(-2.0) = 1.0507 * 1.6733 * (exp(-2) - 1) = 1.7581 * (0.1353 - 1) = -1.5202 + // selu(0.5) = 1.0507 * 0.5 = 0.5254 + let expected = TensorData::from([[0.0, 1.0507, -1.1113], [2.1014, -1.5202, 0.5254]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_selu_zero() { + let tensor = TestTensor::<1>::from([0.0]); + + let output = activation::selu(tensor); + let expected = TensorData::from([0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/sigmoid.rs new file mode 100644 index 0000000..aead0f5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/sigmoid.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_sigmoid() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::sigmoid(tensor); + let expected = TensorData::from([[0.731059, 0.999089], [0.999998, 0.047426]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_sigmoid_overflow() { + let tensor = TestTensor::<1>::from([FloatElem::MAX, FloatElem::MIN]); + + let output = activation::sigmoid(tensor); + let expected = TensorData::from([1.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/silu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/silu.rs new file mode 100644 index 0000000..da62121 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/silu.rs @@ -0,0 +1,15 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_silu() { + let tensor = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let output = activation::silu(tensor); + let expected = TensorData::from([[0.73106, 1.76159], [2.85772, 3.92806]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmax.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmax.rs new file mode 100644 index 0000000..6055252 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmax.rs @@ -0,0 +1,15 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_softmax_d2() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::softmax(tensor, 1); + let expected = TensorData::from([[2.472623e-03, 9.975274e-01], [1.0, 1.125352e-07]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmin.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmin.rs new file mode 100644 index 0000000..e513c45 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softmin.rs @@ -0,0 +1,15 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_softmin_d2() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::softmin(tensor, 1); + let expected = TensorData::from([[9.975274e-01, 2.472623e-03], [1.125352e-07, 1.0000]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softplus.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softplus.rs new file mode 100644 index 0000000..a0911e2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softplus.rs @@ -0,0 +1,28 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_softplus_d2() { + let tensor = TestTensor::<2>::from([[-0.4240, -0.9574, -0.2215], [-0.5767, 0.7218, -0.1620]]); + + let output = activation::softplus(tensor.clone(), 1.0); + let expected = TensorData::from([ + [0.503453, 0.324898, 0.588517], + [0.445806, 1.117805, 0.615424], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let output = activation::softplus(tensor, 2.0); + let expected = TensorData::from([ + [0.178232, 0.068737, 0.247990], + [0.137132, 0.827771, 0.272106], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softsign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softsign.rs new file mode 100644 index 0000000..a4c1334 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/softsign.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_softsign() { + let tensor = TestTensor::<2>::from([[1.0, 7.0], [13.0, -3.0]]); + + let output = activation::softsign(tensor); + let expected = TensorData::from([[0.5, 0.875], [0.928571, -0.75]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_softsign_zero() { + let tensor = TestTensor::<1>::from([0.0]); + + let output = activation::softsign(tensor); + let expected = TensorData::from([0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/tanh_activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/tanh_activation.rs new file mode 100644 index 0000000..2015672 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/tanh_activation.rs @@ -0,0 +1,15 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_tanh() { + let tensor = TestTensor::<2>::from([[1., 2.], [3., 4.]]); + + let output = activation::tanh(tensor); + let expected = TensorData::from([[0.761594, 0.964028], [0.995055, 0.999329]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/thresholded_relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/thresholded_relu.rs new file mode 100644 index 0000000..d1be8eb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/activation/thresholded_relu.rs @@ -0,0 +1,26 @@ +use super::*; +use burn_tensor::{TensorData, activation}; + +#[test] +fn test_thresholded_relu_d2() { + // alpha = 1.0 (ONNX default): x if x > 1.0, else 0 + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, 1.0, 0.5]]); + + let output = activation::thresholded_relu(tensor, 1.0); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 0.0, 2.0], [3.0, 0.0, 0.0]]), false); +} + +#[test] +fn test_thresholded_relu_d2_alpha() { + // alpha = 0.5: x if x > 0.5, else 0 + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, 0.5, 0.6]]); + + let output = activation::thresholded_relu(tensor, 0.5); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 0.0, 2.0], [3.0, 0.0, 0.6]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/affine_grid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/affine_grid.rs new file mode 100644 index 0000000..9a67b35 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/affine_grid.rs @@ -0,0 +1,82 @@ +use super::*; +use burn_tensor::grid::affine_grid_2d; + +fn create_identity_transform(batch_size: usize) -> TestTensor<3> { + // Identity affine transform (batch_size, 2, 3) + TestTensor::<3>::from([[[1f32, 0., 0.], [0., 1., 0.]]]).expand([batch_size, 2, 3]) +} + +#[test] +fn test_affine_grid_identity() { + let batch_size = 1; + let channels = 1; + let height = 2; + let width = 2; + + let transform = create_identity_transform(batch_size); + + let output = affine_grid_2d(transform, [batch_size, channels, height, width]); + + // Expected normalized coords: + // [-1, -1], [ 1,-1] + // [-1, 1], [ 1, 1] + let expected = TestTensor::<4>::from([[ + [[-1f32, -1f32], [1f32, -1f32]], + [[-1f32, 1f32], [1f32, 1f32]], + ]]); + + output.into_data().assert_eq(&expected.into_data(), false); +} + +#[test] +fn test_affine_grid_scaling() { + let batch_size = 1; + let channels = 1; + let height = 2; + let width = 2; + + let scale = 2.0f32; + let transform = TestTensor::<3>::from([[[scale, 0., 0.], [0., scale, 0.]]]); + + let output = affine_grid_2d(transform, [batch_size, channels, height, width]); + + // Expect scaled coordinates from normalized grid, so coords * 2 + let expected = TestTensor::<4>::from([[ + [[-2f32, -2f32], [2f32, -2f32]], + [[-2f32, 2f32], [2f32, 2f32]], + ]]); + + output.into_data().assert_eq(&expected.into_data(), false); +} + +#[test] +fn test_affine_grid_translation() { + let batch_size = 1; + let channels = 1; + let height = 2; + let width = 2; + + // Translate by 0.5 in x and -0.5 in y (normalized coords) + let tx = 0.5f32; + let ty = -0.5f32; + + let transform = TestTensor::<3>::from([[[1.0, 0.0, tx], [0.0, 1.0, ty]]]); + + let output = affine_grid_2d(transform, [batch_size, channels, height, width]); + + // Expected coordinates: + // Original normalized coords are [-1,1] in x and y + // After translation, each coordinate shifts by tx and ty + // So points become: + // [-1 + 0.5, -1 - 0.5] = [-0.5, -1.5] + // [ 1 + 0.5, -1 - 0.5] = [1.5, -1.5] + // [-1 + 0.5, 1 - 0.5] = [-0.5, 0.5] + // [ 1 + 0.5, 1 - 0.5] = [1.5, 0.5] + + let expected = TestTensor::<4>::from([[ + [[-0.5f32, -1.5f32], [1.5f32, -1.5f32]], + [[-0.5f32, 0.5f32], [1.5f32, 0.5f32]], + ]]); + + output.into_data().assert_eq(&expected.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/meshgrid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/meshgrid.rs new file mode 100644 index 0000000..5b382b8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/meshgrid.rs @@ -0,0 +1,153 @@ +use super::*; +use burn_tensor::BasicOps; +use burn_tensor::Tensor; +use burn_tensor::TensorData; +use burn_tensor::backend::Backend; +use burn_tensor::grid::{ + GridIndexing, GridOptions, GridSparsity, IndexPos, meshgrid, meshgrid_stack, +}; + +fn assert_tensors_equal( + actual: &[Tensor; N], + expected: &[Tensor; N], +) where + K: BasicOps, +{ + for (a, e) in actual.iter().zip(expected.iter()) { + a.clone() + .into_data() + .assert_eq(&e.clone().into_data(), true); + } +} + +#[test] +fn test_meshgrid() { + let x = TestTensor::<1>::from([1, 2, 3, 4]); + let y = TestTensor::<1>::from([5, 6]); + let z = TestTensor::<1>::from([7, 8]); + + let grid_shape = [x.dims()[0], y.dims()[0], z.dims()[0]]; + + // 3D, Dense, Matrix + assert_tensors_equal( + &meshgrid(&[x.clone(), y.clone(), z.clone()], GridOptions::default()), + &[ + x.clone().reshape([4, 1, 1]).expand(grid_shape), + y.clone().reshape([1, 2, 1]).expand(grid_shape), + z.clone().reshape([1, 1, 2]).expand(grid_shape), + ], + ); + assert_tensors_equal( + &meshgrid(&[x.clone(), y.clone(), z.clone()], GridSparsity::Dense), + &[ + x.clone().reshape([4, 1, 1]).expand(grid_shape), + y.clone().reshape([1, 2, 1]).expand(grid_shape), + z.clone().reshape([1, 1, 2]).expand(grid_shape), + ], + ); + assert_tensors_equal( + &meshgrid(&[x.clone(), y.clone(), z.clone()], GridIndexing::Matrix), + &[ + x.clone().reshape([4, 1, 1]).expand(grid_shape), + y.clone().reshape([1, 2, 1]).expand(grid_shape), + z.clone().reshape([1, 1, 2]).expand(grid_shape), + ], + ); + + // 3D, Sparse, Matrix + assert_tensors_equal( + &meshgrid( + &[x.clone(), y.clone(), z.clone()], + GridOptions { + indexing: GridIndexing::Matrix, + sparsity: GridSparsity::Sparse, + }, + ), + &[ + x.clone().reshape([4, 1, 1]), + y.clone().reshape([1, 2, 1]), + z.clone().reshape([1, 1, 2]), + ], + ); + assert_tensors_equal( + &meshgrid(&[x.clone(), y.clone(), z.clone()], GridSparsity::Sparse), + &[ + x.clone().reshape([4, 1, 1]), + y.clone().reshape([1, 2, 1]), + z.clone().reshape([1, 1, 2]), + ], + ); + + // 3D, Dense, Cartesian + assert_tensors_equal( + &meshgrid(&[x.clone(), y.clone(), z.clone()], GridIndexing::Cartesian), + &[ + x.clone() + .reshape([4, 1, 1]) + .expand(grid_shape) + .swap_dims(0, 1), + y.clone() + .reshape([1, 2, 1]) + .expand(grid_shape) + .swap_dims(0, 1), + z.clone() + .reshape([1, 1, 2]) + .expand(grid_shape) + .swap_dims(0, 1), + ], + ); + + // 3D, Sparse, Cartesian + assert_tensors_equal( + &meshgrid( + &[x.clone(), y.clone(), z.clone()], + GridOptions::new(GridIndexing::Cartesian, GridSparsity::Sparse), + ), + &[ + x.clone().reshape([4, 1, 1]).swap_dims(0, 1), + y.clone().reshape([1, 2, 1]).swap_dims(0, 1), + z.clone().reshape([1, 1, 2]).swap_dims(0, 1), + ], + ); + assert_tensors_equal( + &meshgrid( + &[x.clone(), y.clone(), z.clone()], + GridOptions { + indexing: GridIndexing::Cartesian, + sparsity: GridSparsity::Sparse, + }, + ), + &[ + x.clone().reshape([4, 1, 1]).swap_dims(0, 1), + y.clone().reshape([1, 2, 1]).swap_dims(0, 1), + z.clone().reshape([1, 1, 2]).swap_dims(0, 1), + ], + ); +} + +#[test] +fn test_meshgrid_stack() { + let tensors = [ + TestTensor::from([0.5, 1.0, 2.5]), + TestTensor::from([0.5, 1.0]), + ]; + + let result: Tensor<_, 3> = meshgrid_stack(&tensors, IndexPos::First); + result.to_data().assert_eq( + &TensorData::from([ + [[0.5, 0.5], [1.0, 1.0], [2.5, 2.5]], + [[0.5, 1.0], [0.5, 1.0], [0.5, 1.0]], + ]), + false, + ); + + let result: Tensor<_, 3> = meshgrid_stack(&tensors, IndexPos::Last); + result.to_data().assert_eq( + &TensorData::from([ + [[0.5, 0.5], [0.5, 1.0]], + [[1.0, 0.5], [1.0, 1.0]], + [[2.5, 0.5], [2.5, 1.0]], + ]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/mod.rs new file mode 100644 index 0000000..e2f874c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/grid/mod.rs @@ -0,0 +1,4 @@ +use super::*; + +pub(crate) mod affine_grid; +pub(crate) mod meshgrid; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/cosine_similarity.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/cosine_similarity.rs new file mode 100644 index 0000000..d86b4b9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/cosine_similarity.rs @@ -0,0 +1,100 @@ +use super::*; +use burn_tensor::{ElementConversion, Tolerance}; +use burn_tensor::{TensorData, linalg}; + +#[test] +fn test_cosine_similarity_basic() { + // Create test tensors + let x1 = TestTensor::<2>::from([[1.0, 2.0, 3.0], [0.5, 1.5, 2.5]]); + let x2 = TestTensor::<2>::from([[1.5, 2.5, 3.5], [0.7, 1.7, 2.7]]); + + // Test cosine similarity along dimension 1 + let expected = TensorData::from([[0.99983203], [0.99987257]]); + linalg::cosine_similarity(x1.clone(), x2.clone(), 1, None) + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + // Test with explicit epsilon + linalg::cosine_similarity(x1.clone(), x2.clone(), 1, Some(1e-8.elem::())) + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_cosine_similarity_orthogonal() { + // Create orthogonal vectors + let x1 = TestTensor::<2>::from([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]); + let x2 = TestTensor::<2>::from([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]); + + // Orthogonal vectors should have cosine similarity of 0 + let expected = TensorData::from([[0.0], [0.0]]); + linalg::cosine_similarity(x1, x2, 1, None) + .into_data() + .assert_eq(&expected, false); +} + +#[test] +fn test_cosine_similarity_parallel() { + // Create parallel vectors + let x1 = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let x2 = TestTensor::<2>::from([[2.0, 4.0, 6.0], [8.0, 10.0, 12.0]]); + + // Parallel vectors should have cosine similarity of 1 + let expected = TensorData::from([[1.0], [1.0]]); + linalg::cosine_similarity(x1, x2, 1, None) + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_cosine_similarity_opposite() { + // Create opposite direction vectors + let x1 = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let x2 = TestTensor::<2>::from([[-1.0, -2.0, -3.0], [-4.0, -5.0, -6.0]]); + + // Opposite vectors should have cosine similarity of -1 + let expected = TensorData::from([[-1.0], [-1.0]]); + linalg::cosine_similarity(x1, x2, 1, None) + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_cosine_similarity_different_dimension() { + // Test with a 3D tensor + let x1 = TestTensor::<3>::from([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]); + let x2 = TestTensor::<3>::from([[[2.0, 3.0], [4.0, 5.0]], [[6.0, 7.0], [8.0, 9.0]]]); + + // Test along dimension 2 + let expected = TensorData::from([[[0.9959688], [0.9958376]], [[0.9955946], [0.9955169]]]); + + // sensitive to rounding in dot/norm; loosen f16 tolerance + let tolerance = Tolerance::default().set_half_precision_relative(7e-3); + + linalg::cosine_similarity(x1.clone(), x2.clone(), 2, None) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + // Test with negative dimension (-1 is the last dimension, which is 2 in this case) + linalg::cosine_similarity(x1.clone(), x2.clone(), -1, None) + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_cosine_similarity_near_zero() { + // Test with near-zero vectors + let x1 = TestTensor::<2>::from([[1e-10, 2e-10, 3e-10], [4e-10, 5e-10, 6e-10]]); + let x2 = TestTensor::<2>::from([[2e-10, 4e-10, 6e-10], [8e-10, 10e-10, 12e-10]]); + + // Update the expected values based on the actual implementation behavior + let expected = TensorData::from([[0.0028], [0.0154]]); + + // Smaller values result in NaN on metal f16 + let epsilon = Some(FloatElem::from_elem(1e-2)); + let tolerance = Tolerance::absolute(0.2); + + linalg::cosine_similarity(x1, x2, 1, epsilon) + .into_data() + .assert_approx_eq::(&expected, tolerance); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/diag.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/diag.rs new file mode 100644 index 0000000..c8e0407 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/diag.rs @@ -0,0 +1,259 @@ +use super::*; +use burn_tensor::{TensorData, linalg::diag}; + +#[test] +fn test_diag_2d_square() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + let expected = TensorData::from([1.0, 4.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_2d_tall() { + let device = Default::default(); + // 4x2 matrix (tall) - min(4,2) = 2 diagonal elements + let tensor = + TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // Result should have shape [2] with values [1.0, 4.0] + let expected = TensorData::from([1.0, 4.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_2d_wide() { + let device = Default::default(); + // 2x4 matrix (wide) - min(2,4) = 2 diagonal elements + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // Result should have shape [2] with values [1.0, 6.0] + let expected = TensorData::from([1.0, 6.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_3d_batch_square() { + let device = Default::default(); + // Batch of 2 matrices, each 2x2 + let tensor = TestTensor::<3>::from_data( + [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], + &device, + ); + let result = diag::<_, 3, 2, _>(tensor); + // Result should have shape [2, 2] + let expected = TensorData::from([[1.0, 4.0], [5.0, 8.0]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_3d_batch_tall() { + let device = Default::default(); + // Batch of 2 matrices, each 3x2 (tall) + let tensor = TestTensor::<3>::from_data( + [ + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], + [[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]], + ], + &device, + ); + let result = diag::<_, 3, 2, _>(tensor); + // Result should have shape [2, 2] - min(3,2) = 2 diagonal elements each + let expected = TensorData::from([[1.0, 4.0], [7.0, 10.0]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_3d_batch_wide() { + let device = Default::default(); + // Batch of 2 matrices, each 2x3 (wide) + let tensor = TestTensor::<3>::from_data( + [ + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]], + ], + &device, + ); + let result = diag::<_, 3, 2, _>(tensor); + // Result should have shape [2, 2] - min(2,3) = 2 diagonal elements each + let expected = TensorData::from([[1.0, 5.0], [7.0, 11.0]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_4d_batch_channel_square() { + let device = Default::default(); + // [batch=2, channel=2, rows=2, cols=2] + let tensor = TestTensor::<4>::from_data( + [ + [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], + [[[9.0, 10.0], [11.0, 12.0]], [[13.0, 14.0], [15.0, 16.0]]], + ], + &device, + ); + let result = diag::<_, 4, 3, _>(tensor); + // Result should have shape [2, 2, 2] + let expected = TensorData::from([[[1.0, 4.0], [5.0, 8.0]], [[9.0, 12.0], [13.0, 16.0]]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_4d_batch_channel_tall() { + let device = Default::default(); + // [batch=2, channel=1, rows=3, cols=2] + let tensor = TestTensor::<4>::from_data( + [ + [[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]], + [[[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]]], + ], + &device, + ); + let result = diag::<_, 4, 3, _>(tensor); + // Result should have shape [2, 1, 2] - min(3,2) = 2 diagonal elements each + let expected = TensorData::from([[[1.0, 4.0]], [[7.0, 10.0]]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_4d_batch_channel_wide() { + let device = Default::default(); + // [batch=1, channel=2, rows=2, cols=4] + let tensor = TestTensor::<4>::from_data( + [[ + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], + [[9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]], + ]], + &device, + ); + let result = diag::<_, 4, 3, _>(tensor); + // Result should have shape [1, 2, 2] - min(2,4) = 2 diagonal elements each + let expected = TensorData::from([[[1.0, 6.0], [9.0, 14.0]]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_1x1() { + let device = Default::default(); + // Single element matrix + let tensor = TestTensor::<2>::from_data([[5.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // Should return [5.0] with shape [1] + let expected = TensorData::from([5.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_single_row() { + let device = Default::default(); + // Single row matrix + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // min(1,3) = 1, should return [1.0] with shape [1] + let expected = TensorData::from([1.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_single_column() { + let device = Default::default(); + // Single column matrix + let tensor = TestTensor::<2>::from_data([[1.0], [2.0], [3.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // min(3,1) = 1, should return [1.0] with shape [1] + let expected = TensorData::from([1.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_zeros() { + let device = Default::default(); + // Matrix with zeros on diagonal + let tensor = TestTensor::<2>::from_data([[0.0, 1.0], [2.0, 0.0]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // Should extract diagonal: [0.0, 0.0] + let expected = TensorData::from([0.0, 0.0]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_batch_single_element() { + let device = Default::default(); + // Batch with single element matrices + let tensor = TestTensor::<3>::from_data([[[5.0]], [[7.0]]], &device); + let result = diag::<_, 3, 2, _>(tensor); + // Should return [[5.0], [7.0]] with shape [2, 1] + let expected = TensorData::from([[5.0], [7.0]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_batch_mixed_zeros() { + let device = Default::default(); + // Batch with mixed zero and non-zero diagonal elements + let tensor = TestTensor::<3>::from_data( + [[[1.0, 2.0], [3.0, 0.0]], [[0.0, 5.0], [6.0, 7.0]]], + &device, + ); + let result = diag::<_, 3, 2, _>(tensor); + // Should return [[1.0, 0.0], [0.0, 7.0]] with shape [2, 2] + let expected = TensorData::from([[1.0, 0.0], [0.0, 7.0]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_int_tensor() { + let device = Default::default(); + // Test with integer tensor + let tensor = TestTensorInt::<2>::from_data([[1, 2], [3, 4]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // Result should have shape [2] with values [1, 4] + let expected = TensorData::from([1, 4]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_diag_int_3x3() { + let device = Default::default(); + // Test with 3x3 integer matrix + let tensor = TestTensorInt::<2>::from_data([[1, 2, 3], [4, 5, 6], [7, 8, 9]], &device); + let result = diag::<_, 2, 1, _>(tensor); + // Result should have shape [3] with values [1, 5, 9] + let expected = TensorData::from([1, 5, 9]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn test_diag_1d_should_panic() { + let device = Default::default(); + // 1D tensor should panic - diagonal requires at least 2 dimensions + let tensor = TestTensor::<1>::from_data([1.0, 2.0, 3.0], &device); + let _result = diag::<_, 1, 0, _>(tensor); +} + +#[test] +#[should_panic] +fn test_diag_wrong_output_rank_should_panic() { + let device = Default::default(); + // Providing wrong output rank should panic + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let _result = diag::<_, 2, 2, _>(tensor); // Should be 2,1 not 2,2 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/lu_decomposition.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/lu_decomposition.rs new file mode 100644 index 0000000..7894a3a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/lu_decomposition.rs @@ -0,0 +1,113 @@ +use super::*; +use burn_tensor::{ + Distribution, Shape, TensorData, Tolerance, cast::ToElement, linalg::lu_decomposition, s, +}; + +#[test] +fn test_lu_2x2_decomposition() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[4.0, 3.0], [6.0, 3.0]], &device); + let (result, _permutations) = lu_decomposition(tensor); + let expected = TensorData::from([[6.0, 3.0], [2.0 / 3.0, 1.0]]); + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_lu_3x3_decomposition() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [[0.0, 5.0, 22.0 / 3.0], [4.0, 2.0, 1.0], [2.0, 7.0, 9.0]], + &device, + ); + let (result, permutations) = lu_decomposition(tensor); + let expected = TestTensor::<2>::from_data( + [ + [4.0, 2.0, 1.0], + [0.5, 6.0, 8.5], + [0.0, 0.8333333, 0.25000048], + ], + &device, + ); + let expected_permutations = TensorData::from([1, 2, 0]); + permutations + .into_data() + .assert_eq(&expected_permutations, false); + + let tolerance = Tolerance::default().set_half_precision_absolute(5e-3); + result + .into_data() + .assert_approx_eq::(&expected.into_data(), tolerance); +} + +#[test] +#[should_panic] +fn test_lu_singular_matrix() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [2.0, 4.0]], &device); + let _result = lu_decomposition(tensor); +} + +#[test] +#[should_panic] +fn test_lu_non_square_matrix() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let _result = lu_decomposition(tensor); +} + +#[test] +fn test_lu_1x1_element_matrix() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[5.0]], &device); + let (result, _permutations) = lu_decomposition(tensor); + let expected = TensorData::from([[5.0]]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_lu_identity_matrix() { + let device = Default::default(); + + let tensor = TestTensor::<2>::eye(4, &device); + let (result, _permutations) = lu_decomposition(tensor); + let expected = TestTensor::<2>::eye(4, &device); + result.into_data().assert_eq(&expected.into_data(), true); +} + +#[test] +fn test_lu_50x50_random_matrix() { + let device = Default::default(); + let size = 50; + let distribution = Distribution::Uniform(0.0, 1.0); + let tensor = TestTensor::<2>::random(Shape::new([size, size]), distribution, &device); + let (result, permutations) = lu_decomposition(tensor.clone()); + // Reconstruct the original matrix from L and U + let mut l = TestTensor::<2>::eye(size, &device); + let mut u = TestTensor::<2>::zeros(Shape::new([size, size]), &device); + + for i in 0..size { + for j in 0..size { + if i > j { + l = l.slice_assign(s![i, j], result.clone().slice(s![i, j])); + } else { + u = u.slice_assign(s![i, j], result.clone().slice(s![i, j])); + } + } + } + // Construct the permutation matrix P from the permutation vector + let mut p = TestTensor::<2>::zeros(Shape::new([size, size]), &device); + for i in 0..size { + let perm_index = permutations.clone().slice(s![i]).into_scalar().to_usize(); + p = p.slice_assign( + s![perm_index, i], + TestTensor::<2>::from_data([[1.0]], &device), + ); + } + + // Verify that P * L * U reconstructs the original matrix + let reconstructed = p.matmul(l).matmul(u); + reconstructed + .into_data() + .assert_approx_eq::(&tensor.into_data(), Tolerance::permissive()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/matvec.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/matvec.rs new file mode 100644 index 0000000..3825f2f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/matvec.rs @@ -0,0 +1,102 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance, linalg}; + +#[test] +fn test_matvec_basic_float() { + let device = Default::default(); + let matrix = TestTensor::<2>::from_floats([[1.0, 2.0], [3.0, 4.0]], &device); + let vector = TestTensor::<1>::from_floats([5.0, 6.0], &device); + + let result = linalg::matvec::(matrix, vector); + let expected = TensorData::from([17.0, 39.0]); + + result + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_matvec_basic_int() { + let device = Default::default(); + let matrix = TestTensorInt::<2>::from_ints([[2, 0, -1], [1, 3, 2]], &device); + let vector = TestTensorInt::<1>::from_ints([3, -2, 4], &device); + + let result = linalg::matvec::(matrix, vector); + let expected = TensorData::from([2, 5]); + + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_matvec_batched() { + let device = Default::default(); + let matrix = TestTensor::<3>::from_floats( + [ + [[1.0, 0.0, 2.0], [3.0, 1.0, -1.0]], + [[-2.0, 1.0, 0.0], [0.5, -1.5, 2.0]], + ], + &device, + ); + let vector = TestTensor::<2>::from_floats([[1.0, -1.0, 0.5], [2.0, 0.0, -1.0]], &device); + + let result = linalg::matvec::(matrix, vector); + let expected = TensorData::from([[2.0, 1.5], [-4.0, -1.0]]); + + result + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_matvec_vector_broadcasts_over_batches() { + let device = Default::default(); + let matrix = TestTensor::<3>::from_floats( + [ + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + [[-1.0, 0.0, 2.0], [3.0, 1.0, -2.0]], + ], + &device, + ); + let vector = TestTensor::<2>::from_floats([[1.0, 0.0, -1.0]], &device); + + let result = linalg::matvec::(matrix, vector); + let expected = TensorData::from([[-2.0, -2.0], [-3.0, 5.0]]); + + result + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_matvec_matrix_broadcasts_over_vector_batches() { + let device = Default::default(); + let matrix = TestTensor::<3>::from_floats([[[1.0, 0.0, 2.0], [3.0, -1.0, 1.0]]], &device); + let vector = TestTensor::<2>::from_floats([[2.0, 1.0, 0.0], [1.0, -1.0, 3.0]], &device); + + let result = linalg::matvec::(matrix, vector); + let expected = TensorData::from([[2.0, 5.0], [7.0, 7.0]]); + + result + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[should_panic] +fn test_matvec_invalid_inner_dim_panics() { + let device = Default::default(); + let matrix = TestTensor::<2>::zeros([2, 3], &device); + let vector = TestTensor::<1>::zeros([4], &device); + + let _ = linalg::matvec::(matrix, vector); +} + +#[test] +#[should_panic] +fn test_matvec_mismatched_batches_panics() { + let device = Default::default(); + let matrix = TestTensor::<3>::zeros([2, 3, 4], &device); + let vector = TestTensor::<2>::zeros([3, 4], &device); + + let _ = linalg::matvec::(matrix, vector); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/mod.rs new file mode 100644 index 0000000..495dbd8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/mod.rs @@ -0,0 +1,9 @@ +use super::*; + +pub(crate) mod cosine_similarity; +pub(crate) mod diag; +pub(crate) mod lu_decomposition; +pub(crate) mod matvec; +pub(crate) mod outer; +pub(crate) mod trace; +pub(crate) mod vector_norm; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/outer.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/outer.rs new file mode 100644 index 0000000..6b8e66a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/outer.rs @@ -0,0 +1,262 @@ +use super::*; +use burn_tensor::{ElementConversion, Tolerance}; +use burn_tensor::{TensorData, linalg}; + +// ---------- Vector (D=1, R=2) tests ---------- + +#[test] +fn test_outer_basic() { + let u = TestTensor::<1>::from([1.0, 2.0, 3.0]); + let v = TestTensor::<1>::from([4.0, 5.0]); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::from([[4.0, 5.0], [8.0, 10.0], [12.0, 15.0]]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_shapes_only() { + let device = Default::default(); + let u = TestTensor::<1>::zeros([3], &device); + let v = TestTensor::<1>::zeros([5], &device); + let out = linalg::outer::(u, v); + assert_eq!(out.shape().dims(), [3, 5]); +} + +#[test] +fn test_outer_asymmetry_and_shapes() { + let u = TestTensor::<1>::from([1.0, 2.0]); + let v = TestTensor::<1>::from([3.0, 4.0, 5.0]); + + let uv = linalg::outer::(u.clone(), v.clone()); + let vu = linalg::outer::(v, u); + + assert_eq!(uv.shape().dims(), [2, 3]); + assert_eq!(vu.shape().dims(), [3, 2]); +} + +#[test] +fn test_outer_zero_left() { + let device = Default::default(); + let u = TestTensor::<1>::zeros([3], &device); + let v = TestTensor::<1>::from([7.0, 8.0]); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::zeros::([3, 2]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_zero_right() { + let device = Default::default(); + let u = TestTensor::<1>::from([1.0, -2.0, 3.0]); + let v = TestTensor::<1>::zeros([4], &device); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::zeros::([3, 4]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_signs() { + let u = TestTensor::<1>::from([-1.0, 2.0]); + let v = TestTensor::<1>::from([3.0, -4.0]); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::from([[-3.0, 4.0], [6.0, -8.0]]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_integer_inputs() { + let u = TestTensorInt::<1>::from([1, 2, 3]); + let v = TestTensorInt::<1>::from([4, 5]); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::from([[4, 5], [8, 10], [12, 15]]); + + out.assert_eq(&expected, false); +} + +#[test] +fn test_outer_equivalence_to_matmul() { + let u = TestTensor::<1>::from([1.0, 2.0, 3.0]); + let v = TestTensor::<1>::from([4.0, 5.0]); + + let out = linalg::outer::(u.clone(), v.clone()).into_data(); + + let u2 = u.reshape([3, 1]); + let v2 = v.reshape([1, 2]); + let out_matmul = u2.matmul(v2).into_data(); + + out.assert_approx_eq::(&out_matmul, Tolerance::default()); +} + +#[test] +fn test_outer_vector_identity_right_mult() { + let u = TestTensor::<1>::from([2.0, -1.0]); + let v = TestTensor::<1>::from([3.0, 4.0]); + let w = TestTensor::<1>::from([5.0, 6.0]); + + let uv = linalg::outer::(u.clone(), v.clone()); + let left = uv.matmul(w.clone().reshape([2, 1])).reshape([2]); + + let v_dot_w = v.dot(w); + let right = u * v_dot_w; + + left.into_data() + .assert_approx_eq::(&right.into_data(), Tolerance::default()); +} + +#[test] +fn test_outer_length_one_vectors() { + let u = TestTensor::<1>::from([3.0]); + let v = TestTensor::<1>::from([4.0, 5.0, 6.0]); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::from([[12.0, 15.0, 18.0]]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_large_values() { + let big = 1.0e10; + let u = TestTensor::<1>::from([big, -big]); + let v = TestTensor::<1>::from([big, big]); + + let out = linalg::outer::(u, v).into_data(); + let expected = TensorData::from([[big * big, big * big], [-big * big, -big * big]]); + + let tol = Tolerance::relative(1e-6).set_half_precision_relative(1e-3); + out.assert_approx_eq::(&expected, tol); +} + +#[test] +fn test_outer_nan_propagation() { + let u = TestTensor::<1>::from([f32::NAN, 2.0]); + let v = TestTensor::<1>::from([3.0, 4.0]); + + let out = linalg::outer::(u, v).into_data(); + + let s: &[FloatElem] = out + .as_slice::() + .expect("outer nan_propagation: as_slice failed"); + + assert!(s[0].is_nan()); + assert!(s[1].is_nan()); + assert_eq!(s[2], 6.0f32.elem::()); + assert_eq!(s[3], 8.0f32.elem::()); +} + +// ---------- Batched (D=2, R=3) tests ---------- + +#[test] +fn test_outer_batched_basic() { + let x = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + let y = TestTensor::<2>::from([[5.0, 6.0, 7.0], [8.0, 9.0, 10.0]]); + let out = linalg::outer::(x, y).into_data(); + + let expected = TensorData::from([ + [[5.0, 6.0, 7.0], [10.0, 12.0, 14.0]], + [[24.0, 27.0, 30.0], [32.0, 36.0, 40.0]], + ]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_batched_shapes() { + let device = Default::default(); + let x = TestTensor::<2>::zeros([3, 4], &device); + let y = TestTensor::<2>::zeros([3, 5], &device); + let out = linalg::outer::(x, y); + assert_eq!(out.shape().dims(), [3, 4, 5]); +} + +#[test] +fn test_outer_batched_zero_left() { + let device = Default::default(); + let x = TestTensor::<2>::zeros([2, 3], &device); + let y = TestTensor::<2>::from([[7.0, 8.0], [9.0, 10.0]]); + let out = linalg::outer::(x, y).into_data(); + + let expected = TensorData::zeros::([2, 3, 2]); + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_batched_zero_right() { + let device = Default::default(); + let x = TestTensor::<2>::from([[1.0, -2.0, 3.0], [4.0, 5.0, -6.0]]); + let y = TestTensor::<2>::zeros([2, 4], &device); + let out = linalg::outer::(x, y).into_data(); + + let expected = TensorData::zeros::([2, 3, 4]); + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_batched_signs() { + let x = TestTensor::<2>::from([[-1.0, 2.0], [3.0, -4.0]]); + let y = TestTensor::<2>::from([[3.0, -4.0], [-5.0, 6.0]]); + let out = linalg::outer::(x, y).into_data(); + + let expected = TensorData::from([[[-3.0, 4.0], [6.0, -8.0]], [[-15.0, 18.0], [20.0, -24.0]]]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_outer_batched_equivalence_to_per_sample_outer() { + let x = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + let y = TestTensor::<2>::from([[5.0, 6.0, 7.0], [8.0, 9.0, 10.0]]); + let batched = linalg::outer::(x.clone(), y.clone()); + + for b in 0..2 { + let idx = TestTensorInt::<1>::from([b]); + + let xb2d = x.clone().select(0, idx.clone()); // (1, m) + let yb2d = y.clone().select(0, idx); // (1, n) + + let dims_x: [usize; 2] = xb2d.shape().dims(); + let dims_y: [usize; 2] = yb2d.shape().dims(); + let (m, n) = (dims_x[1], dims_y[1]); + + let per = linalg::outer::(xb2d.reshape([m]), yb2d.reshape([n])); + + let bat3d = batched.clone().select(0, TestTensorInt::<1>::from([b])); // (m, n) + + let per_len = per.shape().num_elements(); + let per_flat = per.reshape([per_len]).into_data(); + + let bat_len = bat3d.shape().num_elements(); + let bat_flat = bat3d.reshape([bat_len]).into_data(); + + bat_flat.assert_approx_eq::(&per_flat, Tolerance::default()); + } +} + +#[test] +#[should_panic] +fn test_outer_batched_mismatched_batches_panics() { + let device = Default::default(); + let x = TestTensor::<2>::zeros([2, 3], &device); + let y = TestTensor::<2>::zeros([3, 4], &device); + let _ = linalg::outer::(x, y); +} + +#[test] +fn test_outer_dim() { + let u = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + let v = TestTensor::<2>::from([[4.0, 5.0], [5.0, 6.0]]); + + let out = linalg::outer_dim::(u, v, 0).into_data(); + let expected = TensorData::from([[[4.0, 10.0], [5.0, 12.0]], [[12.0, 20.0], [15.0, 24.0]]]); + + out.assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/trace.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/trace.rs new file mode 100644 index 0000000..2e2cc96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/trace.rs @@ -0,0 +1,114 @@ +use super::*; +use burn_tensor::linalg::trace; + +#[test] +fn test_trace_2d_square() { + let device = Default::default(); + let tensor = + TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], &device); + let result = trace::<_, 2, 1>(tensor); + let expected = TestTensor::<1>::from_data([15.0], &device); // 1 + 5 + 9 = 15 + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_2d_rectangular_wide() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], &device); + let result = trace::<_, 2, 1>(tensor); + let expected = TestTensor::<1>::from_data([7.0], &device); // 1 + 6 = 7 + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_2d_rectangular_tall() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], &device); + let result = trace::<_, 2, 1>(tensor); + let expected = TestTensor::<1>::from_data([5.0], &device); // 1 + 4 = 5 + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_3d_batch() { + let device = Default::default(); + let tensor = TestTensor::<3>::from_data( + [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], + &device, + ); + + let result = trace::<_, 3, 2>(tensor); + let expected = TestTensor::<2>::from_data([[5.0], [13.0]], &device); // [1+4=5, 5+8=13] + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_4d_batch() { + let device = Default::default(); + let tensor = TestTensor::<4>::from_data( + [[ + // Batch 0, Channel 0 + [[1.0, 2.0], [3.0, 4.0]], + // Batch 0, Channel 1 + [[5.0, 6.0], [7.0, 8.0]], + ]], + &device, + ); + + let result = trace::<_, 4, 3>(tensor); + let expected = TestTensor::<3>::from_data([[[5.0], [13.0]]], &device); + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_single_element() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[42.0]], &device); + let result = trace::<_, 2, 1>(tensor); + let expected = TestTensor::<1>::from_data([42.0], &device); + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_zeros() { + let device = Default::default(); + let tensor = TestTensor::<2>::zeros([3, 3], &device); + let result = trace::<_, 2, 1>(tensor); + let expected = TestTensor::<1>::from_data([0.0], &device); + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +fn test_trace_negative_values() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[-1.0, 2.0], [3.0, -4.0]], &device); + let result = trace::<_, 2, 1>(tensor); + let expected = TestTensor::<1>::from_data([-5.0], &device); // -1 + (-4) = -5 + + assert_eq!(result.to_data(), expected.to_data()); +} + +#[test] +#[should_panic] +fn test_trace_1d_should_panic() { + let device = Default::default(); + // 1D tensor should panic - trace requires at least 2 dimensions + let tensor = TestTensor::<1>::from_data([1.0, 2.0, 3.0], &device); + let _result = trace::<_, 1, 0>(tensor); +} + +#[test] +#[should_panic] +fn test_trace_wrong_output_rank_should_panic() { + let device = Default::default(); + // Providing wrong output rank should panic + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let _result = trace::<_, 2, 2>(tensor); // Should be 2,1 not 2,2 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/vector_norm.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/vector_norm.rs new file mode 100644 index 0000000..51c4ab6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/linalg/vector_norm.rs @@ -0,0 +1,241 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use burn_tensor::backend::Backend; +use burn_tensor::linalg; + +#[test] +fn test_max_min_abs() { + let x = TestTensor::<2>::from([[1., 2.], [3., 4.]]); + + let expected = TestTensor::<2>::from([[3., 4.]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::LInf, 0) + .into_data() + .assert_eq(&expected, true); + linalg::max_abs_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + + let expected = TestTensor::<2>::from([[1., 2.]]).into_data(); + linalg::vector_norm(x.clone(), -f64::INFINITY, 0) + .into_data() + .assert_eq(&expected, true); + linalg::vector_norm(x.clone(), f64::NEG_INFINITY, 0) + .into_data() + .assert_eq(&expected, true); + linalg::min_abs_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + + let expected = TestTensor::<2>::from([[2.], [4.]]).into_data(); + linalg::vector_norm(x.clone(), f64::INFINITY, 1) + .into_data() + .assert_eq(&expected, true); + linalg::max_abs_norm(x.clone(), 1) + .into_data() + .assert_eq(&expected, true); + + let expected = TestTensor::<2>::from([[1.], [3.]]).into_data(); + linalg::vector_norm(x.clone(), -f64::INFINITY, 1) + .into_data() + .assert_eq(&expected, true); + linalg::vector_norm(x.clone(), f64::NEG_INFINITY, 1) + .into_data() + .assert_eq(&expected, true); + linalg::min_abs_norm(x, 1) + .into_data() + .assert_eq(&expected, true); + + // Test with integer tensor + let z = TestTensorInt::<2>::from([[1, 2], [3, 4]]); + + linalg::max_abs_norm(z.clone(), 0) + .into_data() + .assert_eq(&TestTensorInt::<2>::from([[3, 4]]).into_data(), true); + linalg::max_abs_norm(z.clone(), 1) + .into_data() + .assert_eq(&TestTensorInt::<2>::from([[2], [4]]).into_data(), true); + + linalg::min_abs_norm(z.clone(), 0) + .into_data() + .assert_eq(&TestTensorInt::<2>::from([[1, 2]]).into_data(), true); + linalg::min_abs_norm(z, 1) + .into_data() + .assert_eq(&TestTensorInt::<2>::from([[1], [3]]).into_data(), true); +} + +#[test] +fn test_l0_norm() { + let x = TestTensor::<2>::from([[1.0, -2.0, 0.], [0.0, 0., 4.]]); + + let expected = TestTensor::<2>::from([[1., 1., 1.]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::L0, 0) + .into_data() + .assert_eq(&expected, true); + linalg::l0_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + + let expected = TestTensor::<2>::from([[2.], [1.]]).into_data(); + linalg::vector_norm(x.clone(), 0.0, 1) + .into_data() + .assert_eq(&expected, true); + linalg::l0_norm(x.clone(), 1) + .into_data() + .assert_eq(&expected, true); + + // Test with integer tensor + let z = TestTensorInt::<2>::from([[1, -2, 0], [0, 0, 4]]); + + linalg::l0_norm(z.clone(), 0) + .into_data() + .assert_eq(&TestTensor::<2>::from([[1, 1, 1]]).int().into_data(), true); + linalg::l0_norm(z.clone(), 1) + .into_data() + .assert_eq(&TestTensor::<2>::from([[2], [1]]).int().into_data(), true); +} + +#[test] +fn test_l1_norm() { + let x = TestTensor::<2>::from([[1., 2.], [3., 4.]]); + + let expected = TestTensor::<2>::from([[4.0, 6.0]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::L1, 0) + .into_data() + .assert_eq(&expected, true); + linalg::l1_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + + let expected = TestTensor::<2>::from([[3.0], [7.0]]).into_data(); + linalg::vector_norm(x.clone(), 1.0, 1) + .into_data() + .assert_eq(&expected, true); + linalg::l1_norm(x.clone(), 1) + .into_data() + .assert_eq(&expected, true); +} + +#[test] +fn test_lp_norm() { + let x = TestTensor::<2>::from([[1., -2., 0.], [0., 3., 4.]]); + let tolerance = Tolerance::relative(1e-5).set_half_precision_relative(2e-3); + + fn lp_norm_naive( + x: Tensor, + p: f64, + dim: usize, + ) -> Tensor { + x.abs().powf_scalar(p).sum_dim(dim).powf_scalar(1. / p) + } + + // Arbitrary P + let expected = TestTensor::<2>::from([[1.0, 3.2710664, 4.0]]).into_data(); + linalg::vector_norm(x.clone(), 3, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + linalg::lp_norm(x.clone(), 3., 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + // L0 + let expected = TestTensor::<2>::from([[1., 2., 1.]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::L0, 0) + .into_data() + .assert_eq(&expected, true); + linalg::l0_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + linalg::lp_norm(x.clone(), 0.0, 0) + .into_data() + .assert_eq(&expected, true); + + // L1 + let expected = TestTensor::<2>::from([[1.0, 5.0, 4.0]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::L1, 0) + .into_data() + .assert_eq(&expected, true); + linalg::l1_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + lp_norm_naive(x.clone(), 1.0, 0) + .into_data() + .assert_eq(&expected, true); + linalg::lp_norm(x.clone(), 1.0, 0) + .into_data() + .assert_eq(&expected, true); + + // L2 + let expected = TestTensor::<2>::from([[1.0, 3.6055512, 4.0]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::L2, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + linalg::l2_norm(x.clone(), 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + lp_norm_naive(x.clone(), 2.0, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + linalg::lp_norm(x.clone(), 2.0, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + // LInf + let expected = TestTensor::<2>::from([[1.0, 3.0, 4.0]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::LInf, 0) + .into_data() + .assert_eq(&expected, true); + linalg::max_abs_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + linalg::lp_norm(x.clone(), f64::INFINITY, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + // LNegInf + let expected = TestTensor::<2>::from([[0.0, 2.0, 0.0]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::LNegInf, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + linalg::min_abs_norm(x.clone(), 0) + .into_data() + .assert_eq(&expected, true); + linalg::lp_norm(x.clone(), f64::NEG_INFINITY, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_l2_norm() { + let x = TestTensor::<2>::from([[1., 2.], [3., 4.]]); + let tolerance = Tolerance::relative(1e-5).set_half_precision_relative(1e-3); + + let expected = TestTensor::<2>::from([[3.16227766, 4.47213595]]).into_data(); + linalg::vector_norm(x.clone(), linalg::Norm::L2, 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + linalg::l2_norm(x.clone(), 0) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TestTensor::<2>::from([[2.23606798], [5.0]]).into_data(); + linalg::vector_norm(x.clone(), 2.0, 1) + .into_data() + .assert_approx_eq::(&expected, tolerance); + linalg::l2_norm(x.clone(), 1) + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_normalize() { + let x = TestTensor::<2>::from([[1., 2.], [3., 4.]]); + + let expected = TensorData::from([[1. / 4., 2. / 6.], [3. / 4., 4. / 6.]]); + let output = linalg::vector_normalize(x.clone(), 1.0, 0, 0.25).into_data(); + output.assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([[1. / 5., 2. / 6.], [3. / 5., 4. / 6.]]); + let output = linalg::vector_normalize(x.clone(), 1.0, 0, 5.0).into_data(); + output.assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/mod.rs new file mode 100644 index 0000000..b5d2650 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/mod.rs @@ -0,0 +1,13 @@ +#[allow(unused_imports)] +pub use super::*; // re-export test types + +mod activation; +mod grid; +mod linalg; +mod module; +mod ops; +mod primitive; +mod stats; + +#[cfg(feature = "quantization")] +mod quantization; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool1d.rs new file mode 100644 index 0000000..4a9b284 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool1d.rs @@ -0,0 +1,70 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::adaptive_avg_pool1d; + +#[test] +fn test_adaptive_avg_pool1d_simple() { + let test = AdaptiveAvgPool1dTestCase { + batch_size: 1, + channels: 2, + length: 8, + length_out: 4, + }; + + test.assert_output(TestTensor::from([[ + [0.5, 2.5, 4.5, 6.5], + [8.5, 10.5, 12.5, 14.5], + ]])); +} + +#[test] +fn test_adaptive_avg_pool1d_dyn_filter_size() { + let test = AdaptiveAvgPool1dTestCase { + batch_size: 1, + channels: 2, + length: 7, + length_out: 3, + }; + + test.assert_output(TestTensor::from([[[1.0, 3.0, 5.0], [8.0, 10.0, 12.0]]])); +} + +#[test] +fn test_adaptive_avg_pool1d_bigger_output() { + let test = AdaptiveAvgPool1dTestCase { + batch_size: 1, + channels: 2, + length: 4, + length_out: 8, + }; + + test.assert_output(TestTensor::from([[ + [0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0], + [4.0, 4.0, 5.0, 5.0, 6.0, 6.0, 7.0, 7.0], + ]])); +} + +struct AdaptiveAvgPool1dTestCase { + batch_size: usize, + channels: usize, + length: usize, + length_out: usize, +} + +impl AdaptiveAvgPool1dTestCase { + fn assert_output(self, y: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.length]); + let device = Default::default(); + let x = TestTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ); + let output = adaptive_avg_pool1d(x, self.length_out); + + y.into_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool2d.rs new file mode 100644 index 0000000..baeb034 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/adaptive_avgpool2d.rs @@ -0,0 +1,101 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::adaptive_avg_pool2d; + +#[test] +fn test_adaptive_avg_pool2d_simple() { + let test = AdaptiveAvgPool2dTestCase { + batch_size: 1, + channels: 2, + height: 8, + width: 6, + height_out: 4, + width_out: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [3.5000, 4.5000, 6.5000, 7.5000], + [15.5000, 16.5000, 18.5000, 19.5000], + [27.5000, 28.5000, 30.5000, 31.5000], + [39.5000, 40.5000, 42.5000, 43.5000], + ], + [ + [51.5000, 52.5000, 54.5000, 55.5000], + [63.5000, 64.5000, 66.5000, 67.5000], + [75.5000, 76.5000, 78.5000, 79.5000], + [87.5000, 88.5000, 90.5000, 91.5000], + ], + ]])); +} + +#[test] +fn test_adaptive_avg_pool2d_dyn_filter_size() { + let test = AdaptiveAvgPool2dTestCase { + batch_size: 1, + channels: 2, + height: 5, + width: 7, + height_out: 3, + width_out: 2, + }; + + test.assert_output(TestTensor::from([[ + [[5.0000, 8.0000], [15.5000, 18.5000], [26.0000, 29.0000]], + [[40.0000, 43.0000], [50.5000, 53.5000], [61.0000, 64.0000]], + ]])); +} + +#[test] +fn test_adaptive_avg_pool2d_bigger_output() { + let test = AdaptiveAvgPool2dTestCase { + batch_size: 1, + channels: 2, + height: 4, + width: 3, + height_out: 5, + width_out: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [0.0000, 0.5000, 1.5000, 2.0000], + [1.5000, 2.0000, 3.0000, 3.5000], + [4.5000, 5.0000, 6.0000, 6.5000], + [7.5000, 8.0000, 9.0000, 9.5000], + [9.0000, 9.5000, 10.5000, 11.0000], + ], + [ + [12.0000, 12.5000, 13.5000, 14.0000], + [13.5000, 14.0000, 15.0000, 15.5000], + [16.5000, 17.0000, 18.0000, 18.5000], + [19.5000, 20.0000, 21.0000, 21.5000], + [21.0000, 21.5000, 22.5000, 23.0000], + ], + ]])); +} + +struct AdaptiveAvgPool2dTestCase { + batch_size: usize, + channels: usize, + height: usize, + width: usize, + height_out: usize, + width_out: usize, +} + +impl AdaptiveAvgPool2dTestCase { + fn assert_output(self, y: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &y.device()) + .reshape::<4, _>(shape_x) + .into_data(), + ); + let output = adaptive_avg_pool2d(x, [self.height_out, self.width_out]); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/attention.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/attention.rs new file mode 100644 index 0000000..5bee85e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/attention.rs @@ -0,0 +1,372 @@ +use super::*; +use burn_tensor::Distribution; +use burn_tensor::Tolerance; +use burn_tensor::module::attention; +use burn_tensor::module::attention_fallback; +use burn_tensor::ops::AttentionModuleOptions; + +#[test] +fn test_attention_no_mask() { + // Skip on metal with f16 - flash attention returns zeros + // Enable once this issue is fixed: https://github.com/tracel-ai/burn/issues/4325 + #[cfg(feature = "metal")] + if core::any::TypeId::of::() == core::any::TypeId::of::() { + return; + } + + let num_batches = 1; + let num_heads = 1; + let seq_q = 128; + let seq_kv = 128; + let head_dim = 64; + let val_dim = 64; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_q, head_dim], + Distribution::Uniform(0., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_kv, head_dim], + Distribution::Uniform(0., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_kv, val_dim], + Distribution::Uniform(0., 1.), + &Default::default(), + ); + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + None, + None, + Default::default(), + ); + + let expected = + attention_fallback::(query, key, value, None, None, Default::default()); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} + +#[test] +fn test_attention_custom_scale() { + let [num_batches, num_heads, seq_len, head_dim] = [1, 2, 16, 32]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + + let options = AttentionModuleOptions { + scale: Some(0.1), + ..Default::default() + }; + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + None, + None, + options, + ); + + let expected = attention_fallback::(query, key, value, None, None, options); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} + +#[test] +fn test_attention_attn_bias() { + let [num_batches, num_heads, seq_len, head_dim] = [1, 2, 16, 32]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let bias = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, seq_len], + Distribution::Uniform(-0.5, 0.5), + &Default::default(), + ); + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + None, + Some(bias.clone()), + Default::default(), + ); + + let expected = + attention_fallback::(query, key, value, None, Some(bias), Default::default()); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} + +#[test] +fn test_attention_softcap() { + let [num_batches, num_heads, seq_len, head_dim] = [1, 2, 16, 32]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + + let options = AttentionModuleOptions { + softcap: Some(50.0), + ..Default::default() + }; + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + None, + None, + options, + ); + + let expected = attention_fallback::(query, key, value, None, None, options); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} + +#[test] +fn test_attention_is_causal() { + let [num_batches, num_heads, seq_len, head_dim] = [2, 4, 16, 32]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + + let options = AttentionModuleOptions { + is_causal: true, + ..Default::default() + }; + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + None, + None, + options, + ); + + let expected = attention_fallback::(query, key, value, None, None, options); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} + +/// Cross-attention: seq_q != seq_k, with causal masking and additive bias. +#[test] +fn test_attention_cross_attention_with_bias() { + let [num_batches, num_heads, seq_q, seq_k, head_dim] = [2, 2, 8, 24, 32]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_q, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_k, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_k, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let bias = TestTensor::<4>::random( + [num_batches, num_heads, seq_q, seq_k], + Distribution::Uniform(-0.5, 0.5), + &Default::default(), + ); + + let options = AttentionModuleOptions { + is_causal: true, + ..Default::default() + }; + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + None, + Some(bias.clone()), + options, + ); + + let expected = attention_fallback::(query, key, value, None, Some(bias), options); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} + +/// Regression: softcap must be applied before -inf masking. +/// With causal masking, position 0 can only attend to itself, so output[0] == value[0]. +/// If softcap were applied after masking, tanh(-inf/softcap) = -softcap (finite), +/// and the masked position would leak into the output. +#[test] +fn test_attention_softcap_preserves_causal_mask() { + let [num_batches, num_heads, seq_len, head_dim] = [1, 1, 4, 8]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + + let options = AttentionModuleOptions { + softcap: Some(20.0), + is_causal: true, + ..Default::default() + }; + + let output = attention_fallback::(query, key, value.clone(), None, None, options); + + // With causal masking, position 0 can only attend to itself (softmax = [1, 0, 0, 0]). + // So output[..., 0, :] must equal value[..., 0, :]. + let output_row0 = output.slice([0..1, 0..1, 0..1, 0..head_dim]); + let value_row0 = value.slice([0..1, 0..1, 0..1, 0..head_dim]); + + output_row0 + .into_data() + .assert_approx_eq::(&value_row0.into_data(), Tolerance::relative(1e-5)); +} + +/// Combined: mask + bias + custom scale + softcap together. +#[test] +fn test_attention_all_options() { + let [num_batches, num_heads, seq_len, head_dim] = [2, 2, 16, 32]; + + let query = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let key = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let value = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, head_dim], + Distribution::Uniform(-1., 1.), + &Default::default(), + ); + let bias = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, seq_len], + Distribution::Uniform(-0.5, 0.5), + &Default::default(), + ); + // Create a random bool mask by thresholding a uniform float tensor + let mask = TestTensor::<4>::random( + [num_batches, num_heads, seq_len, seq_len], + Distribution::Uniform(0., 1.), + &Default::default(), + ) + .greater_elem(0.7); + + let options = AttentionModuleOptions { + scale: Some(0.05), + softcap: Some(30.0), + is_causal: true, + }; + + let output = attention( + query.clone(), + key.clone(), + value.clone(), + Some(mask.clone()), + Some(bias.clone()), + options, + ); + + let expected = + attention_fallback::(query, key, value, Some(mask), Some(bias), options); + + output.into_data().assert_approx_eq::( + &expected.into_data(), + Tolerance::rel_abs(1e-2, 1e-3).set_half_precision_relative(1e-1), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool1d.rs new file mode 100644 index 0000000..8dc2170 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool1d.rs @@ -0,0 +1,168 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::avg_pool1d; + +#[test] +fn test_avg_pool1d_simple() { + let test = AvgPool1dTestCase { + batch_size: 1, + channels: 1, + kernel_size: 3, + padding: 0, + stride: 1, + length: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from([[[1., 2., 3., 4.]]])); +} + +#[test] +fn test_avg_pool1d_complex() { + let test = AvgPool1dTestCase { + batch_size: 1, + channels: 2, + kernel_size: 3, + padding: 1, + stride: 2, + length: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from([[ + [0.33333, 2.0000, 4.0000], + [4.33333, 8.0000, 10.0000], + ]])); +} + +#[test] +fn test_avg_pool1d_complex_dont_count_pad() { + let test = AvgPool1dTestCase { + batch_size: 1, + channels: 2, + kernel_size: 3, + padding: 1, + stride: 2, + length: 6, + count_include_pad: false, + }; + + test.assert_output(TestTensor::from([[ + [0.5000, 2.0000, 4.0000], + [6.5000, 8.0000, 10.0000], + ]])); +} + +struct AvgPool1dTestCase { + batch_size: usize, + channels: usize, + kernel_size: usize, + padding: usize, + stride: usize, + length: usize, + count_include_pad: bool, +} + +impl AvgPool1dTestCase { + fn assert_output(self, y: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.length]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &y.device()) + .reshape::<3, _>(shape_x) + .into_data(), + ); + let output = avg_pool1d( + x, + self.kernel_size, + self.stride, + self.padding, + self.count_include_pad, + false, + ); + + y.to_data().assert_approx_eq::( + &output.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + } +} + +#[test] +fn test_avg_pool1d_ceil_mode() { + // Test ceil_mode=true produces larger output when input doesn't divide evenly by stride + // Input: 1x1x6 (values 0-5), kernel: 3, stride: 2, padding: 0 + // Floor mode: output = (6-3)/2+1 = 2 elements + // Ceil mode: output = ceil((6-3)/2)+1 = ceil(1.5)+1 = 3 elements + let x = TestTensor::from([[[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]]]); + + // With ceil_mode=false (floor): output is 2 elements + // Window 0: avg(0,1,2) = 1 + // Window 1: avg(2,3,4) = 3 + let y_floor = TestTensor::<3>::from([[[1.0, 3.0]]]); + + let output_floor = avg_pool1d( + x.clone(), + 3, // kernel_size + 2, // stride + 0, // padding + true, // count_include_pad + false, + ); + + y_floor.to_data().assert_approx_eq::( + &output_floor.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + + // With ceil_mode=true: output is 3 elements + // Window 0: avg(0,1,2) = 1 + // Window 1: avg(2,3,4) = 3 + // Window 2: avg(4,5) = 4.5 (partial window, count_include_pad=false divides by 2) + let y_ceil = TestTensor::<3>::from([[[1.0, 3.0, 4.5]]]); + + let output_ceil = avg_pool1d( + x, 3, // kernel_size + 2, // stride + 0, // padding + false, // count_include_pad=false to get correct average for partial window + true, + ); + + y_ceil.to_data().assert_approx_eq::( + &output_ceil.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); +} + +#[test] +fn test_avg_pool1d_ceil_mode_count_include_pad() { + // Test count_include_pad=true + ceil_mode=true interaction for 1D + // When ceil_mode creates windows that extend beyond the padded input: + // - count_include_pad=true should count positions within padded bounds (not ceil_mode extensions) + // + // Input: 1x1x6, kernel 3, stride 2, padding 1, ceil_mode=true + // Output is 4 elements + let x = TestTensor::from([[[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]]]); + + // Expected PyTorch output with padding=1, ceil_mode=true, count_include_pad=true: + // Window 0: positions -1,0,1 -> values 0,0,1 (0 is padding) / 3 = 0.333 + // Window 1: positions 1,2,3 -> values 1,2,3 / 3 = 2.0 + // Window 2: positions 3,4,5 -> values 3,4,5 / 3 = 4.0 + // Window 3: positions 5,6,7 -> only 5 is valid, 6 is padding, 7 is ceil_mode extension + // value 5 / 2 (only 2 positions within padded bounds) = 2.5 + let expected = TestTensor::<3>::from([[[0.3333, 2.0, 4.0, 2.5]]]); + + let output = avg_pool1d( + x, 3, // kernel_size + 2, // stride + 1, // padding + true, // count_include_pad=true + true, // ceil_mode=true + ); + + expected.to_data().assert_approx_eq::( + &output.into_data(), + Tolerance::default().set_half_precision_relative(1e-2), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool2d.rs new file mode 100644 index 0000000..233a5d6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/avgpool2d.rs @@ -0,0 +1,220 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::avg_pool2d; + +#[test] +fn test_avg_pool2d_simple() { + let test = AvgPool2dTestCase { + batch_size: 1, + channels: 1, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + height: 6, + width: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from([[[ + [7., 8., 9., 10.], + [13., 14., 15., 16.], + [19., 20., 21., 22.], + [25., 26., 27., 28.], + ]]])); +} + +#[test] +fn test_avg_pool2d_complex() { + let test = AvgPool2dTestCase { + batch_size: 1, + channels: 1, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 1, + padding_2: 2, + stride_1: 1, + stride_2: 2, + height: 4, + width: 6, + count_include_pad: true, + }; + + test.assert_output(TestTensor::from([[[ + [1.1667, 3.0000, 4.3333, 2.5000], + [3.2500, 7.5000, 9.5000, 5.2500], + [6.2500, 13.5000, 15.5000, 8.2500], + [5.1667, 11.0000, 12.3333, 6.5000], + ]]])); +} + +#[test] +fn test_avg_pool2d_complex_dont_include_pad() { + let test = AvgPool2dTestCase { + batch_size: 1, + channels: 1, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 1, + padding_2: 2, + stride_1: 1, + stride_2: 2, + height: 4, + width: 6, + count_include_pad: false, + }; + + test.assert_output(TestTensor::from([[[ + [3.5000, 4.5000, 6.5000, 7.5000], + [6.5000, 7.5000, 9.5000, 10.5000], + [12.5000, 13.5000, 15.5000, 16.5000], + [15.5000, 16.5000, 18.5000, 19.5000], + ]]])); +} + +struct AvgPool2dTestCase { + batch_size: usize, + channels: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + stride_1: usize, + stride_2: usize, + height: usize, + width: usize, + count_include_pad: bool, +} + +impl AvgPool2dTestCase { + fn assert_output(self, y: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &y.device()) + .reshape::<4, _>(shape_x) + .into_data(), + ); + let output = avg_pool2d( + x, + [self.kernel_size_1, self.kernel_size_2], + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + self.count_include_pad, + false, + ); + + y.to_data().assert_approx_eq::( + &output.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + } +} + +#[test] +fn test_avg_pool2d_ceil_mode() { + // Test ceil_mode=true produces larger output when input doesn't divide evenly by stride + // Input: 1x1x6x6 (values 0-35), kernel: 3x3, stride: 2x2, padding: 0x0 + // Floor mode: output = (6-3)/2+1 = 2 x 2 + // Ceil mode: output = ceil((6-3)/2)+1 = ceil(1.5)+1 = 3 x 3 + let x = TestTensor::from([[[ + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], + [6.0, 7.0, 8.0, 9.0, 10.0, 11.0], + [12.0, 13.0, 14.0, 15.0, 16.0, 17.0], + [18.0, 19.0, 20.0, 21.0, 22.0, 23.0], + [24.0, 25.0, 26.0, 27.0, 28.0, 29.0], + [30.0, 31.0, 32.0, 33.0, 34.0, 35.0], + ]]]); + + // With ceil_mode=false (floor): output is 2x2 + // Window (0,0): avg(0,1,2,6,7,8,12,13,14) = avg(63) = 7 + // Window (0,1): avg(2,3,4,8,9,10,14,15,16) = avg(81) = 9 + // Window (1,0): avg(12,13,14,18,19,20,24,25,26) = avg(171) = 19 + // Window (1,1): avg(14,15,16,20,21,22,26,27,28) = avg(189) = 21 + let y_floor = TestTensor::<4>::from([[[[7.0, 9.0], [19.0, 21.0]]]]); + + let output_floor = avg_pool2d( + x.clone(), + [3, 3], + [2, 2], + [0, 0], + true, // count_include_pad + false, + ); + + y_floor.to_data().assert_approx_eq::( + &output_floor.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); + + // With ceil_mode=true: output is 3x3 + // The extra windows at the edge include partial/padded regions + // When count_include_pad=false, only actual values are averaged + // Window (0,2): positions (0:3, 4:6) -> values 4,5,10,11,16,17 -> avg = 10.5 + // Window (1,2): positions (2:5, 4:6) -> values 16,17,22,23,28,29 -> avg = 22.5 + // Window (2,0): positions (4:6, 0:3) -> values 24,25,26,30,31,32 -> avg = 28 + // Window (2,1): positions (4:6, 2:5) -> values 26,27,28,32,33,34 -> avg = 30 + // Window (2,2): positions (4:6, 4:6) -> values 28,29,34,35 -> avg = 31.5 + let y_ceil = + TestTensor::<4>::from([[[[7.0, 9.0, 10.5], [19.0, 21.0, 22.5], [28.0, 30.0, 31.5]]]]); + + let output_ceil = avg_pool2d( + x, + [3, 3], + [2, 2], + [0, 0], + false, // count_include_pad=false to avoid dividing by full kernel size + true, + ); + + y_ceil.to_data().assert_approx_eq::( + &output_ceil.into_data(), + Tolerance::default().set_half_precision_relative(1e-3), + ); +} + +#[test] +fn test_avg_pool2d_ceil_mode_count_include_pad() { + // Test count_include_pad=true + ceil_mode=true interaction + // When ceil_mode creates windows that extend beyond the padded input: + // - count_include_pad=true should count positions within padded bounds (not ceil_mode extensions) + // + // For input 6x6, kernel 3, stride 2, padding 1, ceil_mode=true: + // - Output is 4x4 + // - Corner (3,3) window covers positions beyond even the user padding + // - Expected: 35/4 = 8.75 (divides by count of positions within padded bounds) + + let x = TestTensor::from([[[ + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], + [6.0, 7.0, 8.0, 9.0, 10.0, 11.0], + [12.0, 13.0, 14.0, 15.0, 16.0, 17.0], + [18.0, 19.0, 20.0, 21.0, 22.0, 23.0], + [24.0, 25.0, 26.0, 27.0, 28.0, 29.0], + [30.0, 31.0, 32.0, 33.0, 34.0, 35.0], + ]]]); + + // Expected PyTorch output with padding=1, ceil_mode=true, count_include_pad=true + // Note: corner (3,3) = 8.75 = 35/4, not 35/9 + let expected = TestTensor::<4>::from([[[ + [1.5556, 3.3333, 4.6667, 2.6667], + [8.3333, 14.0000, 16.0000, 8.5000], + [16.3333, 26.0000, 28.0000, 14.5000], + [10.1667, 16.0000, 17.0000, 8.7500], + ]]]); + + let output = avg_pool2d( + x, + [3, 3], + [2, 2], + [1, 1], + true, // count_include_pad=true + true, // ceil_mode=true + ); + + expected.to_data().assert_approx_eq::( + &output.into_data(), + Tolerance::default().set_half_precision_relative(1e-2), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bicubic_interpolate.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bicubic_interpolate.rs new file mode 100644 index 0000000..794bd97 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bicubic_interpolate.rs @@ -0,0 +1,196 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::interpolate; +use burn_tensor::ops::{InterpolateMode, InterpolateOptions}; + +#[test] +fn test_upsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 2, + channels: 1, + height: 7, + width: 5, + height_out: 8, + width_out: 7, + }; + + test.assert_output(TestTensor::from([ + [[ + [0.0000, 0.5741, 1.3704, 2.0000, 2.6296, 3.4259, 4.0000], + [4.0015, 4.5755, 5.3718, 6.0015, 6.6311, 7.4274, 8.0015], + [8.3528, 8.9268, 9.7231, 10.3528, 10.9824, 11.7787, 12.3528], + [ + 12.7697, 13.3438, 14.1400, 14.7697, 15.3993, 16.1956, 16.7697, + ], + [ + 17.2303, 17.8044, 18.6007, 19.2303, 19.8600, 20.6562, 21.2303, + ], + [ + 21.6472, 22.2213, 23.0176, 23.6472, 24.2769, 25.0731, 25.6472, + ], + [ + 25.9986, 26.5726, 27.3689, 27.9986, 28.6282, 29.4245, 29.9986, + ], + [ + 30.0000, 30.5741, 31.3704, 32.0000, 32.6296, 33.4259, 34.0000, + ], + ]], + [[ + [ + 35.0000, 35.5741, 36.3704, 37.0000, 37.6296, 38.4259, 39.0000, + ], + [ + 39.0015, 39.5755, 40.3718, 41.0015, 41.6311, 42.4274, 43.0015, + ], + [ + 43.3528, 43.9269, 44.7231, 45.3528, 45.9824, 46.7787, 47.3528, + ], + [ + 47.7697, 48.3438, 49.1400, 49.7697, 50.3993, 51.1956, 51.7697, + ], + [ + 52.2303, 52.8044, 53.6007, 54.2303, 54.8600, 55.6562, 56.2303, + ], + [ + 56.6472, 57.2213, 58.0176, 58.6472, 59.2769, 60.0731, 60.6472, + ], + [ + 60.9986, 61.5726, 62.3689, 62.9986, 63.6282, 64.4245, 64.9986, + ], + [ + 65.0000, 65.5741, 66.3704, 67.0000, 67.6296, 68.4259, 69.0000, + ], + ]], + ])); +} + +#[test] +fn test_downsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 45, + width: 14, + height_out: 4, + width_out: 6, + }; + + test.assert_output(TestTensor::from([[[ + [0.0000, 2.5760, 5.2480, 7.7520, 10.4240, 13.0000], + [204.8148, 207.3908, 210.0628, 212.5668, 215.2388, 217.8148], + [411.1852, 413.7612, 416.4331, 418.9371, 421.6091, 424.1852], + [616.0000, 618.576, 621.2479, 623.7519, 626.4239, 629.0000], + ]]])); +} + +#[test] +fn test_1d_bicubic() { + // Initialize the model without weights (because the exported file does not contain them) + let device = Default::default(); + + // Run the model + let input = TestTensor::<3>::from_floats( + [[[1.5410, -0.2934, -2.1788, 0.5684, -1.0845, -1.3986]]], + &device, + ); + + let input = input.unsqueeze_dim(2); + + let output = interpolate( + input, + [1, 9], + InterpolateOptions::new(InterpolateMode::Bicubic), + ); + + assert_eq!(output.dims(), [1, 1, 1, 9]); + + // assert output data does not contain NaN + assert!( + !output + .clone() + .to_data() + .as_slice::() + .unwrap() + .iter() + .any(|&x| x.is_nan()), + "interpolate output contains NaN" + ); + + TestTensor::<4>::from([[[[ + 1.541, 0.5747652, -1.010614, -2.197787, -0.8269969, 0.59609234, -0.5803058, -1.3792794, + -1.3986, + ]]]]) + .to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} +struct InterpolateTestCase { + batch_size: usize, + channels: usize, + height: usize, + width: usize, + height_out: usize, + width_out: usize, +} + +impl InterpolateTestCase { + fn assert_output(self, y: TestTensor<4>) { + self.assert_output_with_align_corners(y, true); + } + + fn assert_output_with_align_corners(self, y: TestTensor<4>, align_corners: bool) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &y.device()) + .reshape::<4, _>(shape_x) + .into_data(), + ); + let output = interpolate( + x, + [self.height_out, self.width_out], + InterpolateOptions::new(InterpolateMode::Bicubic).with_align_corners(align_corners), + ); + + let tolerance = Tolerance::permissive(); + y.to_data() + .assert_approx_eq::(&output.into_data(), tolerance); + } +} + +#[test] +fn test_upsample_half_pixel() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 4, + width: 4, + height_out: 8, + width_out: 8, + }; + + test.assert_output_with_align_corners( + TestTensor::from([[[ + [ + -0.5273, -0.2305, 0.2461, 0.875, 1.2812, 1.9102, 2.3867, 2.6836, + ], + [ + 0.6602, 0.957, 1.4336, 2.0625, 2.4688, 3.0977, 3.5742, 3.8711, + ], + [ + 2.5664, 2.8633, 3.3398, 3.9688, 4.375, 5.0039, 5.4805, 5.7773, + ], + [5.082, 5.3789, 5.8555, 6.4844, 6.8906, 7.5195, 7.9961, 8.293], + [6.707, 7.0039, 7.4805, 8.1094, 8.5156, 9.1445, 9.6211, 9.918], + [ + 9.2227, 9.5195, 9.9961, 10.625, 11.0312, 11.6602, 12.1367, 12.4336, + ], + [ + 11.1289, 11.4258, 11.9023, 12.5312, 12.9375, 13.5664, 14.043, 14.3398, + ], + [ + 12.3164, 12.6133, 13.0898, 13.7188, 14.125, 14.7539, 15.2305, 15.5273, + ], + ]]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bilinear_interpolate.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bilinear_interpolate.rs new file mode 100644 index 0000000..2ca5ac6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/bilinear_interpolate.rs @@ -0,0 +1,270 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::module::interpolate; +use burn_tensor::ops::{InterpolateMode, InterpolateOptions}; +use burn_tensor::{DType, Shape}; + +#[test] +fn test_upsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 2, + channels: 1, + height: 7, + width: 5, + height_out: 8, + width_out: 7, + }; + + test.assert_output(TestTensor::from([ + [[ + [0.0000, 0.6667, 1.3333, 2.0000, 2.6667, 3.3333, 4.0000], + [4.2857, 4.9524, 5.6190, 6.2857, 6.9524, 7.6190, 8.2857], + [8.5714, 9.2381, 9.9048, 10.5714, 11.2381, 11.9048, 12.5714], + [ + 12.8571, 13.5238, 14.1905, 14.8571, 15.5238, 16.1905, 16.8571, + ], + [ + 17.1429, 17.8095, 18.4762, 19.1429, 19.8095, 20.4762, 21.1429, + ], + [ + 21.4286, 22.0952, 22.7619, 23.4286, 24.0952, 24.7619, 25.4286, + ], + [ + 25.7143, 26.3810, 27.0476, 27.7143, 28.3810, 29.0476, 29.7143, + ], + [ + 30.0000, 30.6667, 31.3333, 32.0000, 32.6667, 33.3333, 34.0000, + ], + ]], + [[ + [ + 35.0000, 35.6667, 36.3333, 37.0000, 37.6667, 38.3333, 39.0000, + ], + [ + 39.2857, 39.9524, 40.6190, 41.2857, 41.9524, 42.6190, 43.2857, + ], + [ + 43.5714, 44.2381, 44.9048, 45.5714, 46.2381, 46.9048, 47.5714, + ], + [ + 47.8571, 48.5238, 49.1905, 49.8571, 50.5238, 51.1905, 51.8571, + ], + [ + 52.1429, 52.8095, 53.4762, 54.1429, 54.8095, 55.4762, 56.1429, + ], + [ + 56.4286, 57.0952, 57.7619, 58.4286, 59.0952, 59.7619, 60.4286, + ], + [ + 60.7143, 61.3810, 62.0476, 62.7143, 63.3810, 64.0476, 64.7143, + ], + [ + 65.0000, 65.6667, 66.3333, 67.0000, 67.6667, 68.3333, 69.0000, + ], + ]], + ])); +} + +#[test] +fn test_downsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 45, + width: 14, + height_out: 4, + width_out: 6, + }; + + test.assert_output(TestTensor::from([[[ + [0.0, 2.6, 5.2, 7.8, 10.4, 13.], + [205.3333, 207.9333, 210.5333, 213.1333, 215.7333, 218.3333], + [410.6667, 413.2667, 415.8667, 418.4667, 421.0667, 423.6667], + [616., 618.6, 621.2, 623.8, 626.4, 629.], + ]]])); +} + +#[test] +fn test_1d_bilinear() { + // Initialize the model without weights (because the exported file does not contain them) + let device = Default::default(); + + // Run the model + let input = TestTensor::<3>::from_floats( + [[[1.5410, -0.2934, -2.1788, 0.5684, -1.0845, -1.3986]]], + &device, + ); + + let input = input.unsqueeze_dim(2); + + let output = interpolate( + input, + [1, 9], + InterpolateOptions::new(InterpolateMode::Bilinear), + ); + + assert_eq!(output.dims(), [1, 1, 1, 9]); + + // assert output data does not contain NaN + assert!( + !output + .clone() + .to_data() + .as_slice::() + .unwrap() + .iter() + .any(|&x| x.is_nan()), + "interpolate output contains NaN" + ); + + TestTensor::<4>::from([[[[ + 1.541f32, + 0.39450002, + -0.76475, + -1.943125, + -0.80520004, + 0.36178753, + -0.671275, + -1.2022874, + -1.3986, + ]]]]) + .to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_interpolate_coord_float_precision_boundary() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 28, + width: 4, + height_out: 24, + width_out: 2, + }; + + test.assert_output(TestTensor::from([[[ + [0.0, 3.0], + [4.6956, 7.6956], + [9.3913, 12.3913], + [14.0869, 17.0869], + [18.7826, 21.7826], + [23.4782, 26.4782], + [28.1739, 31.1739], + [32.8695, 35.8695], + [37.5652, 40.5652], + [42.2608, 45.2608], + [46.9565, 49.9565], + [51.6521, 54.6521], + [56.3478, 59.3478], + [61.0434, 64.0434], + [65.7391, 68.7391], + [70.4347, 73.4347], + [75.1304, 78.1304], + [79.8260, 82.8260], + [84.5217, 87.5217], + [89.2173, 92.2173], + [93.9130, 96.9130], + [98.6086, 101.6086], + [103.3043, 106.3043], + [108.0, 111.0], + ]]])); +} + +#[test] +fn should_interpolate_cast() { + let device = Default::default(); + let shape_x = Shape::new([1, 1, 4, 4]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + ) + .cast(DType::F32); // ok for f32 backends, casts dtype for f16 tests + let output = interpolate( + x, + [8, 8], + InterpolateOptions::new(InterpolateMode::Bilinear), + ); + + let expected = TestTensor::<4>::from([[[ + [0.0, 0.42857, 0.8571, 1.2857, 1.7142, 2.1428, 2.5714, 3.0], + [1.7142, 2.1428, 2.5714, 3.0, 3.4285, 3.8571, 4.2857, 4.7142], + [3.4285, 3.8571, 4.2857, 4.7142, 5.1428, 5.5714, 6.0, 6.4285], + [5.1428, 5.5714, 6.0, 6.4285, 6.8571, 7.2857, 7.7142, 8.1428], + [6.8571, 7.2857, 7.7142, 8.1428, 8.5714, 9.0, 9.4285, 9.8571], + [ + 8.5714, 9.0, 9.4285, 9.8571, 10.2857, 10.7142, 11.1428, 11.5714, + ], + [ + 10.2857, 10.7142, 11.1428, 11.5714, 12.0, 12.4285, 12.8571, 13.2857, + ], + [ + 12.0, 12.4285, 12.8571, 13.2857, 13.7142, 14.1428, 14.5714, 15.0, + ], + ]]]); + + let tolerance = Tolerance::permissive(); + output + .into_data() + .assert_approx_eq::(&expected.into_data(), tolerance); +} + +struct InterpolateTestCase { + batch_size: usize, + channels: usize, + height: usize, + width: usize, + height_out: usize, + width_out: usize, +} + +impl InterpolateTestCase { + fn assert_output(self, y: TestTensor<4>) { + self.assert_output_with_align_corners(y, true); + } + + fn assert_output_with_align_corners(self, y: TestTensor<4>, align_corners: bool) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &y.device()) + .reshape::<4, _>(shape_x) + .into_data(), + ); + let output = interpolate( + x, + [self.height_out, self.width_out], + InterpolateOptions::new(InterpolateMode::Bilinear).with_align_corners(align_corners), + ); + + let tolerance = Tolerance::permissive(); + y.to_data() + .assert_approx_eq::(&output.into_data(), tolerance); + } +} + +#[test] +fn test_upsample_half_pixel() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 4, + width: 4, + height_out: 8, + width_out: 8, + }; + + test.assert_output_with_align_corners( + TestTensor::from([[[ + [0.0, 0.25, 0.75, 1.25, 1.75, 2.25, 2.75, 3.0], + [1.0, 1.25, 1.75, 2.25, 2.75, 3.25, 3.75, 4.0], + [3.0, 3.25, 3.75, 4.25, 4.75, 5.25, 5.75, 6.0], + [5.0, 5.25, 5.75, 6.25, 6.75, 7.25, 7.75, 8.0], + [7.0, 7.25, 7.75, 8.25, 8.75, 9.25, 9.75, 10.0], + [9.0, 9.25, 9.75, 10.25, 10.75, 11.25, 11.75, 12.0], + [11.0, 11.25, 11.75, 12.25, 12.75, 13.25, 13.75, 14.0], + [12.0, 12.25, 12.75, 13.25, 13.75, 14.25, 14.75, 15.0], + ]]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv1d.rs new file mode 100644 index 0000000..711805a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv1d.rs @@ -0,0 +1,138 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::conv1d; +use burn_tensor::ops::ConvOptions; + +#[test] +fn test_conv1d_simple() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 1, + groups: 1, + length: 4, + }; + + test.assert_output(TestTensor::from([ + [[43., 67., 82., 49.], [104., 176., 227., 158.]], + [[139., 187., 202., 113.], [392., 584., 635., 414.]], + ])); +} + +#[test] +fn test_conv1d_dilation() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 2, + groups: 1, + length: 4, + }; + + test.assert_output(TestTensor::from([ + [[62., 38.], [159., 111.]], + [[158., 102.], [447., 367.]], + ])); +} + +#[test] +fn test_conv1d_groups() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + stride: 1, + dilation: 1, + groups: 2, + length: 4, + }; + + test.assert_output(TestTensor::from([ + [[2., 5., 8., 3.], [42., 63., 75., 47.]], + [[26., 29., 32., 11.], [114., 159., 171., 103.]], + ])); +} + +#[test] +fn test_conv1d_complex() { + let test = Conv1dTestCase { + batch_size: 2, + channels_in: 3, + channels_out: 4, + kernel_size: 3, + padding: 1, + stride: 2, + dilation: 1, + groups: 1, + length: 4, + }; + + test.assert_output(TestTensor::from_floats( + [ + [[171., 294.], [415., 781.], [659., 1268.], [903., 1755.]], + [[495., 726.], [1387., 2185.], [2279., 3644.], [3171., 5103.]], + ], + &Default::default(), + )); +} + +struct Conv1dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size: usize, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + length: usize, +} + +impl Conv1dTestCase { + fn assert_output(self, y: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.length]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size, + ]); + let device = Default::default(); + let weight = TestTensor::from_data( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<3, _>(shape_weight) + .into_data(), + &device, + ); + let bias = TestTensor::from_data( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + &device, + ); + let x = TestTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ); + let output = conv1d( + x, + weight, + Some(bias), + ConvOptions::new([self.stride], [self.padding], [self.dilation], self.groups), + ); + + let tolerance = Tolerance::relative(1e-5).set_half_precision_relative(1e-3); + y.to_data() + .assert_approx_eq::(&output.into_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv2d.rs new file mode 100644 index 0000000..9718e0c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv2d.rs @@ -0,0 +1,652 @@ +use super::*; +use alloc::{vec, vec::Vec}; +use burn_tensor::Shape; +use burn_tensor::activation::gelu; +use burn_tensor::module::conv2d; +use burn_tensor::ops::ConvOptions; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn test_conv2d_simple() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [1196., 1796., 1916., 1264.], + [1881., 2793., 2946., 1923.], + [2313., 3405., 3558., 2307.], + [1424., 2072., 2156., 1380.], + ], + [ + [2709., 4173., 4509., 3065.], + [4582., 7006., 7483., 5056.], + [5878., 8914., 9391., 6304.], + [4089., 6177., 6477., 4333.], + ], + ]])); +} + +#[test] +fn test_conv2d_simple_implicit() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 1, + channels_out: 16, + kernel_size_1: 4, + kernel_size_2: 4, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 5, + width: 5, + }; + + test.assert_output(TestTensor::from([[ + [ + [666., 916., 1030., 774.], + [1124., 1500., 1620., 1190.], + [1604., 2100., 2220., 1610.], + [990., 1264., 1330., 936.], + ], + [ + [1531., 2165., 2471., 1927.], + [2757., 3805., 4181., 3207.], + [4197., 5685., 6061., 4587.], + [3295., 4433., 4691., 3529.], + ], + [ + [2396., 3414., 3912., 3080.], + [4390., 6110., 6742., 5224.], + [6790., 9270., 9902., 7564.], + [5600., 7602., 8052., 6122.], + ], + [ + [3261., 4663., 5353., 4233.], + [6023., 8415., 9303., 7241.], + [9383., 12855., 13743., 10541.], + [7905., 10771., 11413., 8715.], + ], + [ + [4126., 5912., 6794., 5386.], + [7656., 10720., 11864., 9258.], + [11976., 16440., 17584., 13518.], + [10210., 13940., 14774., 11308.], + ], + [ + [4991., 7161., 8235., 6539.], + [9289., 13025., 14425., 11275.], + [14569., 20025., 21425., 16495.], + [12515., 17109., 18135., 13901.], + ], + [ + [5856., 8410., 9676., 7692.], + [10922., 15330., 16986., 13292.], + [17162., 23610., 25266., 19472.], + [14820., 20278., 21496., 16494.], + ], + [ + [6721., 9659., 11117., 8845.], + [12555., 17635., 19547., 15309.], + [19755., 27195., 29107., 22449.], + [17125., 23447., 24857., 19087.], + ], + [ + [7586., 10908., 12558., 9998.], + [14188., 19940., 22108., 17326.], + [22348., 30780., 32948., 25426.], + [19430., 26616., 28218., 21680.], + ], + [ + [8451., 12157., 13999., 11151.], + [15821., 22245., 24669., 19343.], + [24941., 34365., 36789., 28403.], + [21735., 29785., 31579., 24273.], + ], + [ + [9316., 13406., 15440., 12304.], + [17454., 24550., 27230., 21360.], + [27534., 37950., 40630., 31380.], + [24040., 32954., 34940., 26866.], + ], + [ + [10181., 14655., 16881., 13457.], + [19087., 26855., 29791., 23377.], + [30127., 41535., 44471., 34357.], + [26345., 36123., 38301., 29459.], + ], + [ + [11046., 15904., 18322., 14610.], + [20720., 29160., 32352., 25394.], + [32720., 45120., 48312., 37334.], + [28650., 39292., 41662., 32052.], + ], + [ + [11911., 17153., 19763., 15763.], + [22353., 31465., 34913., 27411.], + [35313., 48705., 52153., 40311.], + [30955., 42461., 45023., 34645.], + ], + [ + [12776., 18402., 21204., 16916.], + [23986., 33770., 37474., 29428.], + [37906., 52290., 55994., 43288.], + [33260., 45630., 48384., 37238.], + ], + [ + [13641., 19651., 22645., 18069.], + [25619., 36075., 40035., 31445.], + [40499., 55875., 59835., 46265.], + [35565., 48799., 51745., 39831.], + ], + ]])); +} + +#[test] +fn test_conv2d_implicit_padded_in_channels() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 16, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [4521., 6753., 7014., 4635.], + [6858., 10197., 10548., 6939.], + [7830., 11601., 11952., 7839.], + [5007., 7383., 7590., 4953.], + ], + [ + [10516., 15988., 16735., 11278.], + [16822., 25507., 26587., 17875.], + [19738., 29827., 30907., 20719.], + [13594., 20506., 21199., 14188.], + ], + [ + [16511., 25223., 26456., 17921.], + [26786., 40817., 42626., 28811.], + [31646., 48053., 49862., 33599.], + [22181., 33629., 34808., 23423.], + ], + [ + [22506., 34458., 36177., 24564.], + [36750., 56127., 58665., 39747.], + [43554., 66279., 68817., 46479.], + [30768., 46752., 48417., 32658.], + ], + [ + [28501., 43693., 45898., 31207.], + [46714., 71437., 74704., 50683.], + [55462., 84505., 87772., 59359.], + [39355., 59875., 62026., 41893.], + ], + [ + [34496., 52928., 55619., 37850.], + [56678., 86747., 90743., 61619.], + [67370., 102731., 106727., 72239.], + [47942., 72998., 75635., 51128.], + ], + [ + [40491., 62163., 65340., 44493.], + [66642., 102057., 106782., 72555.], + [79278., 120957., 125682., 85119.], + [56529., 86121., 89244., 60363.], + ], + [ + [46486., 71398., 75061., 51136.], + [76606., 117367., 122821., 83491.], + [91186., 139183., 144637., 97999.], + [65116., 99244., 102853., 69598.], + ], + [ + [52481., 80633., 84782., 57779.], + [86570., 132677., 138860., 94427.], + [103094., 157409., 163592., 110879.], + [73703., 112367., 116462., 78833.], + ], + [ + [58476., 89868., 94503., 64422.], + [96534., 147987., 154899., 105363.], + [115002., 175635., 182547., 123759.], + [82290., 125490., 130071., 88068.], + ], + [ + [64471., 99103., 104224., 71065.], + [106498., 163297., 170938., 116299.], + [126910., 193861., 201502., 136639.], + [90877., 138613., 143680., 97303.], + ], + [ + [70466., 108338., 113945., 77708.], + [116462., 178607., 186977., 127235.], + [138818., 212087., 220457., 149519.], + [99464., 151736., 157289., 106538.], + ], + [ + [76461., 117573., 123666., 84351.], + [126426., 193917., 203016., 138171.], + [150726., 230313., 239412., 162399.], + [108051., 164859., 170898., 115773.], + ], + [ + [82456., 126808., 133387., 90994.], + [136390., 209227., 219055., 149107.], + [162634., 248539., 258367., 175279.], + [116638., 177982., 184507., 125008.], + ], + [ + [88451., 136043., 143108., 97637.], + [146354., 224537., 235094., 160043.], + [174542., 266765., 277322., 188159.], + [125225., 191105., 198116., 134243.], + ], + [ + [94446., 145278., 152829., 104280.], + [156318., 239847., 251133., 170979.], + [186450., 284991., 296277., 201039.], + [133812., 204228., 211725., 143478.], + ], + ]])); +} + +#[test] +fn test_conv2d_groups_channels_out() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 16, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [73., 121., 154., 103.], + [171., 258., 294., 186.], + [279., 402., 438., 270.], + [139., 187., 202., 113.], + ], + [ + [164., 284., 371., 266.], + [415., 664., 781., 538.], + [739., 1132., 1249., 838.], + [518., 782., 851., 564.], + ], + [ + [255., 447., 588., 429.], + [659., 1070., 1268., 890.], + [1199., 1862., 2060., 1406.], + [897., 1377., 1500., 1015.], + ], + [ + [346., 610., 805., 592.], + [903., 1476., 1755., 1242.], + [1659., 2592., 2871., 1974.], + [1276., 1972., 2149., 1466.], + ], + [ + [437., 773., 1022., 755.], + [1147., 1882., 2242., 1594.], + [2119., 3322., 3682., 2542.], + [1655., 2567., 2798., 1917.], + ], + [ + [528., 936., 1239., 918.], + [1391., 2288., 2729., 1946.], + [2579., 4052., 4493., 3110.], + [2034., 3162., 3447., 2368.], + ], + [ + [619., 1099., 1456., 1081.], + [1635., 2694., 3216., 2298.], + [3039., 4782., 5304., 3678.], + [2413., 3757., 4096., 2819.], + ], + [ + [710., 1262., 1673., 1244.], + [1879., 3100., 3703., 2650.], + [3499., 5512., 6115., 4246.], + [2792., 4352., 4745., 3270.], + ], + [ + [5793., 8865., 9330., 6335.], + [9467., 14450., 15134., 10250.], + [11303., 17186., 17870., 12062.], + [7971., 12099., 12546., 8457.], + ], + [ + [6460., 9892., 10411., 7074.], + [10575., 16152., 16917., 11466.], + [12627., 19212., 19977., 13494.], + [8926., 13558., 14059., 9484.], + ], + [ + [7127., 10919., 11492., 7813.], + [11683., 17854., 18700., 12682.], + [13951., 21238., 22084., 14926.], + [9881., 15017., 15572., 10511.], + ], + [ + [7794., 11946., 12573., 8552.], + [12791., 19556., 20483., 13898.], + [15275., 23264., 24191., 16358.], + [10836., 16476., 17085., 11538.], + ], + [ + [8461., 12973., 13654., 9291.], + [13899., 21258., 22266., 15114.], + [16599., 25290., 26298., 17790.], + [11791., 17935., 18598., 12565.], + ], + [ + [9128., 14000., 14735., 10030.], + [15007., 22960., 24049., 16330.], + [17923., 27316., 28405., 19222.], + [12746., 19394., 20111., 13592.], + ], + [ + [9795., 15027., 15816., 10769.], + [16115., 24662., 25832., 17546.], + [19247., 29342., 30512., 20654.], + [13701., 20853., 21624., 14619.], + ], + [ + [10462., 16054., 16897., 11508.], + [17223., 26364., 27615., 18762.], + [20571., 31368., 32619., 22086.], + [14656., 22312., 23137., 15646.], + ], + ]])); +} + +#[test] +fn test_conv2d_groups() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 5, + width: 5, + }; + + test.assert_output(TestTensor::from([[ + [[312., 348., 384.], [492., 528., 564.], [672., 708., 744.]], + [ + [3724., 3841., 3958.], + [4309., 4426., 4543.], + [4894., 5011., 5128.], + ], + ]])); +} + +#[test] +fn test_conv2d_groups_multiple_channels() { + let test = Conv2dTestCase { + batch_size: 1, + channels_in: 4, + channels_out: 4, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 5, + width: 5, + }; + + test.assert_output(TestTensor::from([[ + [ + [4035., 4188., 4341.], + [4800., 4953., 5106.], + [5565., 5718., 5871.], + ], + [ + [10030., 10507., 10984.], + [12415., 12892., 13369.], + [14800., 15277., 15754.], + ], + [ + [56075., 56876., 57677.], + [60080., 60881., 61682.], + [64085., 64886., 65687.], + ], + [ + [78270., 79395., 80520.], + [83895., 85020., 86145.], + [89520., 90645., 91770.], + ], + ]])); +} + +#[test] +fn test_conv2d_complex() { + let test = Conv2dTestCase { + batch_size: 2, + channels_in: 3, + channels_out: 4, + kernel_size_1: 3, + kernel_size_2: 2, + padding_1: 1, + padding_2: 2, + stride_1: 2, + stride_2: 3, + dilation_1: 1, + dilation_2: 2, + groups: 1, + height: 4, + width: 5, + }; + + test.assert_output(TestTensor::from([ + [ + [[1845., 3789., 1926.], [3210., 6465., 3228.]], + [[4276., 9082., 4789.], [8071., 16834., 8737.]], + [[6707., 14375., 7652.], [12932., 27203., 14246.]], + [[9138., 19668., 10515.], [17793., 37572., 19755.]], + ], + [ + [[5445., 10629., 5166.], [8070., 15645., 7548.]], + [[14356., 28882., 14509.], [22651., 45454., 22777.]], + [[23267., 47135., 23852.], [37232., 75263., 38006.]], + [[32178., 65388., 33195.], [51813., 105072., 53235.]], + ], + ])); +} + +struct Conv2dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + stride_1: usize, + stride_2: usize, + dilation_1: usize, + dilation_2: usize, + groups: usize, + height: usize, + width: usize, +} + +impl Conv2dTestCase { + fn assert_output(self, y: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size_1, + self.kernel_size_2, + ]); + let device = Default::default(); + let weight = TestTensor::from( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<4, _>(shape_weight) + .into_data(), + ); + let bias = TestTensor::from( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + ); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + ); + let output = conv2d( + x, + weight, + Some(bias), + ConvOptions::new( + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + [self.dilation_1, self.dilation_2], + self.groups, + ), + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + } +} + +#[rustfmt::skip] // param values are too long + fn conv2d_weight() -> TensorData { + TensorData::new( + vec![0.048065186, -0.3059082, -0.10345459, -0.34643555, -0.20788574, -0.021072388, 0.13745117, -0.05102539, 0.024536133, -0.16479492, -0.19519043, 0.27270508, 0.17700195, -0.33764648, -0.08239746, -0.27929688, 0.17321777, -0.1315918, 0.04574585, -0.17980957, -0.33569336, 0.27612305, 0.30004883, -0.28979492, -0.17297363, -0.021759033, -0.27148438, 0.005657196, 0.29956055, -0.06958008, -0.29345703, -0.14440918, 0.10827637, -0.13305664, -0.20239258, 0.24890137, -0.1541748, -0.20019531, -0.2854004, 0.17016602, 0.07861328, -0.09075928, 0.30908203, -0.00013422966, 0.29589844, 0.15258789, -0.25708008, 0.20422363, -0.2529297, 0.07891846, -0.19506836, 0.23571777, 0.27124023, 0.17370605, -0.16992188, -0.23522949, 0.14648438, -0.09576416, -0.18310547, 0.21044922, -0.08911133, -0.2541504, -0.2775879, -0.2064209, -0.16271973, -0.048919678, -0.03555298, -0.11639404, 0.09661865, -0.10241699, 0.08929443, 0.2866211], + [8, 1, 3, 3], + ) + } + +#[test] +fn test_conv2d_binary_broadcasted() { + let device = Default::default(); + let x = TestTensor::<4>::full([1, 1, 28, 28], -0.42421296, &device); + + // conv2d -> batchnorm -> activation + let weight = TestTensor::from_data(conv2d_weight(), &device); + let bias = TestTensor::from([ + 0.082336426, + -0.049591064, + 0.0031795502, + 0.00095653534, + 0.02357483, + 0.005569458, + 0.07525635, + 0.056396484, + ]); + + // channels: [1, 8], kernel_size: [3, 3], stride: [1, 1], dilation: [1, 1], groups: 1, padding: [0, 0] + let opt = ConvOptions::new([1, 1], [0, 0], [1, 1], 1); + let x = conv2d(x, weight, Some(bias), opt); + + // simulate batchnorm binary ops with broadcasted params + let gamma = TestTensor::<1>::from([ + 1.0048828, 0.9902344, 1.0185547, 0.97558594, 1.0097656, 0.97802734, 1.0009766, 1.0146484, + ]); + let beta = TestTensor::<1>::from([ + 0.026290894, + 0.0007505417, + 0.006134033, + 0.02418518, + 0.07373047, + 0.020507813, + 0.01902771, + 0.02003479, + ]); + let mean = TestTensor::<1>::from([ + 0.029159546, + -0.08673096, + -0.03894043, + -0.01108551, + 0.032440186, + 0.03237915, + 0.013839722, + 0.04397583, + ]) + .reshape([1, 8, 1, 1]); + let var = TestTensor::<1>::from([ + 0.67089844, 0.29956055, 0.5209961, 0.1862793, 0.30419922, 0.21313477, 0.7504883, 0.26342773, + ]) + .reshape([1, 8, 1, 1]); + + let std = var.add_scalar(1e-5).sqrt(); + let x = x.sub(mean); + let x = x.div(std); + let x = x.mul(gamma.reshape([1, 8, 1, 1])); + let x = x.add(beta.reshape([1, 8, 1, 1])); + + let x = gelu(x); + + let expected: Vec = [ + 0.36432067f32, + 0.34909567, + 0.30684796, + 0.13217466, + -0.018471397, + -0.1389876, + 0.39402074, + 0.12394252, + ] + .iter() + .flat_map(|&v| core::iter::repeat_n(v, 676)) + .collect(); + let expected = TensorData::new(expected, [1, 8, 26, 26]); + + x.into_data().assert_approx_eq::( + &expected, + Tolerance::default().set_half_precision_absolute(1e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv3d.rs new file mode 100644 index 0000000..1143edd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv3d.rs @@ -0,0 +1,299 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::conv3d; +use burn_tensor::ops::ConvOptions; + +#[test] +fn test_conv3d_simple() { + let test = Conv3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 1, + depth: 4, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [ + [29980.0, 44860.0, 45640.0, 30324.0], + [45072.0, 67380.0, 68496.0, 45468.0], + [48096.0, 71844.0, 72960.0, 48396.0], + [31780.0, 47428.0, 48136.0, 31900.0], + ], + [ + [47292.0, 70548.0, 71556.0, 47400.0], + [70335.0, 104823.0, 106254.0, 70317.0], + [74223.0, 110547.0, 111978.0, 74061.0], + [48552.0, 72240.0, 73140.0, 48324.0], + ], + [ + [58236.0, 86676.0, 87684.0, 57960.0], + [85887.0, 127719.0, 129150.0, 85293.0], + [89775.0, 133443.0, 134874.0, 89037.0], + [58344.0, 86640.0, 87540.0, 57732.0], + ], + [ + [36148.0, 53620.0, 54184.0, 35692.0], + [52740.0, 78144.0, 78936.0, 51936.0], + [54900.0, 81312.0, 82104.0, 54000.0], + [35260.0, 52156.0, 52648.0, 34580.0], + ], + ], + [ + [ + [66701.0, 100589.0, 102665.0, 68773.0], + [102745.0, 154861.0, 157921.0, 105733.0], + [110953.0, 167101.0, 170161.0, 113845.0], + [75413.0, 113525.0, 115529.0, 77261.0], + ], + [ + [112741.0, 169693.0, 172645.0, 115441.0], + [172396.0, 259372.0, 263719.0, 176266.0], + [184060.0, 276760.0, 281107.0, 187786.0], + [124369.0, 186937.0, 189781.0, 126733.0], + ], + [ + [144421.0, 216925.0, 219877.0, 146737.0], + [219052.0, 328924.0, 333271.0, 222346.0], + [230716.0, 346312.0, 350659.0, 233866.0], + [154897.0, 232441.0, 235285.0, 156877.0], + ], + [ + [100517.0, 150821.0, 152681.0, 101789.0], + [151885.0, 227833.0, 230569.0, 153673.0], + [159229.0, 238777.0, 241513.0, 160921.0], + [106541.0, 159725.0, 161513.0, 107589.0], + ], + ], + ]])); +} + +#[test] +fn test_conv3d_groups() { + let test = Conv3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 0, + padding_2: 0, + padding_3: 0, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 2, + depth: 5, + height: 5, + width: 5, + }; + + test.assert_output(TestTensor::from([[ + [ + [ + [15219., 15570., 15921.], + [16974., 17325., 17676.], + [18729., 19080., 19431.], + ], + [ + [23994., 24345., 24696.], + [25749., 26100., 26451.], + [27504., 27855., 28206.], + ], + [ + [32769., 33120., 33471.], + [34524., 34875., 35226.], + [36279., 36630., 36981.], + ], + ], + [ + [ + [172819., 173899., 174979.], + [178219., 179299., 180379.], + [183619., 184699., 185779.], + ], + [ + [199819., 200899., 201979.], + [205219., 206299., 207379.], + [210619., 211699., 212779.], + ], + [ + [226819., 227899., 228979.], + [232219., 233299., 234379.], + [237619., 238699., 239779.], + ], + ], + ]])); +} + +#[test] +fn test_conv3d_complex() { + let test = Conv3dTestCase { + batch_size: 2, + channels_in: 3, + channels_out: 4, + kernel_size_1: 4, + kernel_size_2: 3, + kernel_size_3: 2, + padding_1: 1, + padding_2: 2, + padding_3: 3, + stride_1: 2, + stride_2: 3, + stride_3: 4, + dilation_1: 1, + dilation_2: 2, + dilation_3: 3, + groups: 1, + depth: 4, + height: 5, + width: 6, + }; + + test.assert_output(TestTensor::from([ + [ + [ + [[149148., 299070., 149850.], [147636., 295758., 148050.]], + [[150660., 301014., 150282.], [147420., 294246., 146754.]], + ], + [ + [[351325., 709903., 358507.], [357589., 722143., 364483.]], + [[391717., 789607., 397819.], [396253., 798391., 402067.]], + ], + [ + [[553502., 1120736., 567164.], [567542., 1148528., 580916.]], + [[632774., 1278200., 645356.], [645086., 1302536., 657380.]], + ], + [ + [[755679., 1531569., 775821.], [777495., 1574913., 797349.]], + [[873831., 1766793., 892893.], [893919., 1806681., 912693.]], + ], + ], + [ + [ + [[408348., 810990., 402570.], [393876., 781758., 387810.]], + [[370980., 735174., 364122.], [354780., 702486., 347634.]], + ], + [ + [ + [1077085., 2154943., 1077787.], + [1070389., 2141263., 1070803.], + ], + [ + [1078597., 2156887., 1078219.], + [1070173., 2139751., 1069507.], + ], + ], + [ + [ + [1745822., 3498896., 1753004.], + [1746902., 3500768., 1753796.], + ], + [ + [1786214., 3578600., 1792316.], + [1785566., 3577016., 1791380.], + ], + ], + [ + [ + [2414559., 4842849., 2428221.], + [2423415., 4860273., 2436789.], + ], + [ + [2493831., 5000313., 2506413.], + [2500959., 5014281., 2513253.], + ], + ], + ], + ])); +} + +struct Conv3dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + kernel_size_3: usize, + padding_1: usize, + padding_2: usize, + padding_3: usize, + stride_1: usize, + stride_2: usize, + stride_3: usize, + dilation_1: usize, + dilation_2: usize, + dilation_3: usize, + groups: usize, + depth: usize, + height: usize, + width: usize, +} + +impl Conv3dTestCase { + fn assert_output(self, y: TestTensor<5>) { + let shape_x = Shape::new([ + self.batch_size, + self.channels_in, + self.depth, + self.height, + self.width, + ]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size_1, + self.kernel_size_2, + self.kernel_size_3, + ]); + let device = Default::default(); + let weight = TestTensor::from( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<5, _>(shape_weight) + .into_data(), + ); + let bias = TestTensor::from( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + ); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<5, _>(shape_x) + .into_data(), + ); + let output = conv3d( + x, + weight, + Some(bias), + ConvOptions::new( + [self.stride_1, self.stride_2, self.stride_3], + [self.padding_1, self.padding_2, self.padding_3], + [self.dilation_1, self.dilation_2, self.dilation_3], + self.groups, + ), + ); + + let tolerance = Tolerance::relative(1e-5).set_half_precision_relative(2e-3); + y.to_data() + .assert_approx_eq::(&output.into_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose1d.rs new file mode 100644 index 0000000..10b80a5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose1d.rs @@ -0,0 +1,145 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::conv_transpose1d; +use burn_tensor::ops::ConvTransposeOptions; + +#[test] +fn test_conv_transpose1d_diff_channels() { + let test = ConvTranspose1dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 2, + kernel_size: 3, + padding: 1, + padding_out: 0, + stride: 1, + dilation: 1, + groups: 1, + length: 4, + }; + + test.assert_output(TestTensor::from([[ + [270., 453., 516., 387.], + [352., 589., 679., 505.], + ]])); +} + +#[test] +fn test_conv_transpose1d_stride() { + let test = ConvTranspose1dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + padding_out: 1, + stride: 2, + dilation: 1, + groups: 1, + length: 4, + }; + + test.assert_output(TestTensor::from([[ + [28., 62., 36., 78., 44., 94., 52., 62.], + [41., 93., 55., 121., 69., 149., 83., 93.], + ]])); +} + +#[test] +fn test_conv_transpose1d_dilation() { + let test = ConvTranspose1dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + padding_out: 0, + stride: 1, + dilation: 2, + groups: 1, + length: 4, + }; + + test.assert_output(TestTensor::from([[ + [30., 64., 78., 76., 94., 52.], + [49., 101., 127., 113., 143., 77.], + ]])); +} + +#[test] +fn test_conv_transpose1d_groups() { + let test = ConvTranspose1dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size: 3, + padding: 1, + padding_out: 0, + stride: 1, + dilation: 1, + groups: 2, + length: 4, + }; + + test.assert_output(TestTensor::from_floats( + [[[0., 1., 4., 7.], [32., 59., 71., 59.]]], + &Default::default(), + )); +} + +struct ConvTranspose1dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size: usize, + padding: usize, + padding_out: usize, + stride: usize, + dilation: usize, + groups: usize, + length: usize, +} + +impl ConvTranspose1dTestCase { + fn assert_output(self, y: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.length]); + let shape_weights = Shape::new([ + self.channels_in, + self.channels_out / self.groups, + self.kernel_size, + ]); + let device = Default::default(); + let weights = TestTensor::from_data( + TestTensorInt::arange(0..shape_weights.num_elements() as i64, &device) + .reshape::<3, _>(shape_weights) + .into_data(), + &device, + ); + let bias = TestTensor::from_data( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + &device, + ); + let x = TestTensor::from_data( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<3, _>(shape_x) + .into_data(), + &device, + ); + let output = conv_transpose1d( + x, + weights, + Some(bias), + ConvTransposeOptions::new( + [self.stride], + [self.padding], + [self.padding_out], + [self.dilation], + self.groups, + ), + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose2d.rs new file mode 100644 index 0000000..ec8d069 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose2d.rs @@ -0,0 +1,361 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::conv_transpose2d; +use burn_tensor::ops::ConvTransposeOptions; + +#[test] +fn test_conv_transpose2d_simple_1() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 1, + channels_out: 1, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + padding_out_1: 0, + padding_out_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[[[5.0, 11.0], [23.0, 29.0]]]])); +} + +#[test] +fn test_conv_transpose2d_simple_2() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + padding_out_1: 0, + padding_out_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [9855., 15207., 15738., 10797.], + [16290., 25119., 25956., 17793.], + [18486., 28467., 29304., 20061.], + [13593., 20913., 21498., 14703.], + ], + [ + [11854., 18286., 18979., 13012.], + [19612., 30223., 31303., 21439.], + [22456., 34543., 35623., 24355.], + [16456., 25288., 26035., 17782.], + ], + [ + [13853., 21365., 22220., 15227.], + [22934., 35327., 36650., 25085.], + [26426., 40619., 41942., 28649.], + [19319., 29663., 30572., 20861.], + ], + ]])); +} + +#[test] +fn test_conv_transpose2d_simple_3() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 1, + channels_out: 1, + kernel_size_1: 2, + kernel_size_2: 2, + padding_1: 0, + padding_2: 0, + padding_out_1: 0, + padding_out_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[[ + [0.0, 0.0, 1.0], + [0.0, 4.0, 6.0], + [4.0, 12.0, 9.0], + ]]])); +} + +#[test] +fn test_conv_transpose2d_stride_2() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 1, + channels_out: 1, + kernel_size_1: 2, + kernel_size_2: 2, + padding_1: 0, + padding_2: 0, + padding_out_1: 0, + padding_out_2: 0, + stride_1: 2, + stride_2: 2, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[[ + [0.0, 0.0, 0.0, 1.0], + [0.0, 0.0, 2.0, 3.0], + [0.0, 2.0, 0.0, 3.0], + [4.0, 6.0, 6.0, 9.0], + ]]])); +} + +#[test] +fn test_conv_transpose2d_dilation_2() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + padding_out_1: 1, + padding_out_2: 1, + stride_1: 1, + stride_2: 1, + dilation_1: 2, + dilation_2: 2, + groups: 1, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[ + [ + [126., 116., 136., 124., 146.], + [108., 88., 114., 92., 120.], + [156., 140., 166., 148., 176.], + [126., 100., 132., 104., 138.], + [186., 164., 196., 172., 206.], + ], + [ + [217., 189., 227., 197., 237.], + [163., 125., 169., 129., 175.], + [247., 213., 257., 221., 267.], + [181., 137., 187., 141., 193.], + [277., 237., 287., 245., 297.], + ], + ]])); +} + +#[test] +fn test_conv_transpose2d_stride2_out_padding() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + padding_out_1: 1, + padding_out_2: 1, + stride_1: 2, + stride_2: 2, + dilation_1: 1, + dilation_2: 1, + groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [352., 728., 378., 780., 404., 832., 430., 452.], + [784., 1616., 836., 1720., 888., 1824., 940., 992.], + [456., 936., 482., 988., 508., 1040., 534., 564.], + [992., 2032., 1044., 2136., 1096., 2240., 1148., 1216.], + [560., 1144., 586., 1196., 612., 1248., 638., 676.], + [1200., 2448., 1252., 2552., 1304., 2656., 1356., 1440.], + [664., 1352., 690., 1404., 716., 1456., 742., 788.], + [784., 1598., 816., 1662., 848., 1726., 880., 926.], + ], + [ + [497., 1035., 541., 1123., 585., 1211., 629., 651.], + [1145., 2373., 1233., 2549., 1321., 2725., 1409., 1461.], + [673., 1387., 717., 1475., 761., 1563., 805., 835.], + [1497., 3077., 1585., 3253., 1673., 3429., 1761., 1829.], + [849., 1739., 893., 1827., 937., 1915., 981., 1019.], + [1849., 3781., 1937., 3957., 2025., 4133., 2113., 2197.], + [1025., 2091., 1069., 2179., 1113., 2267., 1157., 1203.], + [1145., 2337., 1195., 2437., 1245., 2537., 1295., 1341.], + ], + ]])); +} + +#[test] +fn test_conv_transpose2d_groups_2() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 1, + padding_2: 1, + padding_out_1: 0, + padding_out_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[ + [[5., 11.], [23., 29.]], + [[236., 258.], [302., 324.]], + ]])); +} + +#[test] +fn test_conv_transpose2d_groups_different_channels() { + let test = ConvTranspose2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 6, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + padding_out_1: 0, + padding_out_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + groups: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[ + [ + [0.0000e+00, 0.0000e+00, 1.0000e+00, 2.0000e+00], + [0.0000e+00, 5.0000e+00, 1.1000e+01, 1.1000e+01], + [6.0000e+00, 2.3000e+01, 2.9000e+01, 2.3000e+01], + [1.2000e+01, 3.2000e+01, 3.7000e+01, 2.4000e+01], + ], + [ + [1.0000e+00, 1.0000e+01, 1.1000e+01, 1.2000e+01], + [1.9000e+01, 6.0000e+01, 6.6000e+01, 4.8000e+01], + [2.5000e+01, 7.8000e+01, 8.4000e+01, 6.0000e+01], + [3.1000e+01, 7.8000e+01, 8.3000e+01, 5.2000e+01], + ], + [ + [2.0000e+00, 2.0000e+01, 2.1000e+01, 2.2000e+01], + [3.8000e+01, 1.1500e+02, 1.2100e+02, 8.5000e+01], + [4.4000e+01, 1.3300e+02, 1.3900e+02, 9.7000e+01], + [5.0000e+01, 1.2400e+02, 1.2900e+02, 8.0000e+01], + ], + [ + [1.1100e+02, 2.5000e+02, 2.5900e+02, 1.4800e+02], + [2.8500e+02, 6.3400e+02, 6.5600e+02, 3.6600e+02], + [3.1500e+02, 7.0000e+02, 7.2200e+02, 4.0200e+02], + [2.0100e+02, 4.3800e+02, 4.5100e+02, 2.4800e+02], + ], + [ + [1.4800e+02, 3.3200e+02, 3.4100e+02, 1.9400e+02], + [3.7600e+02, 8.3300e+02, 8.5500e+02, 4.7500e+02], + [4.0600e+02, 8.9900e+02, 9.2100e+02, 5.1100e+02], + [2.5600e+02, 5.5600e+02, 5.6900e+02, 3.1200e+02], + ], + [ + [1.8500e+02, 4.1400e+02, 4.2300e+02, 2.4000e+02], + [4.6700e+02, 1.0320e+03, 1.0540e+03, 5.8400e+02], + [4.9700e+02, 1.0980e+03, 1.1200e+03, 6.2000e+02], + [3.1100e+02, 6.7400e+02, 6.8700e+02, 3.7600e+02], + ], + ]])); +} + +struct ConvTranspose2dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + padding_out_1: usize, + padding_out_2: usize, + stride_1: usize, + stride_2: usize, + dilation_1: usize, + dilation_2: usize, + groups: usize, + height: usize, + width: usize, +} + +impl ConvTranspose2dTestCase { + fn assert_output(self, y: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let shape_weights = Shape::new([ + self.channels_in, + self.channels_out / self.groups, + self.kernel_size_1, + self.kernel_size_2, + ]); + let device = Default::default(); + let weights = TestTensor::from( + TestTensorInt::arange(0..shape_weights.num_elements() as i64, &device) + .reshape::<4, _>(shape_weights) + .into_data(), + ); + let bias = TestTensor::from( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + ); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x) + .into_data(), + ); + let output = conv_transpose2d( + x, + weights, + Some(bias), + ConvTransposeOptions::new( + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + [self.padding_out_1, self.padding_out_2], + [self.dilation_1, self.dilation_2], + self.groups, + ), + ); + + y.into_data() + .assert_approx_eq::(&output.into_data(), Tolerance::rel_abs(1e-1, 0.01)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose3d.rs new file mode 100644 index 0000000..49911a6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/conv_transpose3d.rs @@ -0,0 +1,749 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::conv_transpose3d; +use burn_tensor::ops::ConvTransposeOptions; + +#[test] +fn test_conv_transpose3d_simple_1() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 1, + channels_out: 1, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + padding_out_1: 0, + padding_out_2: 0, + padding_out_3: 0, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 1, + depth: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[[ + [[96., 124.], [180., 208.]], + [[348., 376.], [432., 460.]], + ]]])); +} +#[test] +fn test_conv_transpose3d_simple_2() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + padding_out_1: 0, + padding_out_2: 0, + padding_out_3: 0, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 1, + depth: 4, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [ + [238452., 360588., 363756., 244488.], + [367929., 556353., 561186., 377163.], + [380745., 575685., 580518., 390123.], + [261192., 394896., 398172., 267564.], + ], + [ + [394083., 595827., 600822., 403749.], + [607635., 918648., 926262., 622404.], + [627831., 949104., 956718., 642816.], + [430353., 650529., 655686., 440523.], + ], + [ + [447075., 675747., 680742., 457317.], + [688419., 1040472., 1048086., 704052.], + [708615., 1070928., 1078542., 724464.], + [485073., 733041., 738198., 495819.], + ], + [ + [328656., 496632., 500124., 335892.], + [505611., 763983., 769302., 516645.], + [519723., 785259., 790578., 530901.], + [355428., 536988., 540588., 363000.], + ], + ], + [ + [ + [286729., 433489., 437629., 294061.], + [442288., 668620., 674911., 453466.], + [458992., 693784., 700075., 470314.], + [314653., 475573., 479821., 322321.], + ], + [ + [474274., 716842., 723295., 485884.], + [730837., 1104544., 1114345., 748522.], + [756865., 1143748., 1153549., 774766.], + [518320., 783208., 789823., 530434.], + ], + [ + [542818., 820090., 826543., 555004.], + [834949., 1261360., 1271161., 853498.], + [860977., 1300564., 1310365., 879742.], + [588592., 889048., 895663., 601282.], + ], + [ + [397669., 600637., 605101., 406201.], + [611074., 922906., 929683., 624052.], + [629074., 950014., 956791., 642196.], + [429625., 648769., 653341., 438493.], + ], + ], + [ + [ + [335006., 506390., 511502., 343634.], + [516647., 780887., 788636., 529769.], + [537239., 811883., 819632., 550505.], + [368114., 556250., 561470., 377078.], + ], + [ + [554465., 837857., 845768., 568019.], + [854039., 1290440., 1302428., 874640.], + [885899., 1338392., 1350380., 906716.], + [606287., 915887., 923960., 620345.], + ], + [ + [638561., 964433., 972344., 652691.], + [981479., 1482248., 1494236., 1002944.], + [1013339., 1530200., 1542188., 1035020.], + [692111., 1045055., 1053128., 706745.], + ], + [ + [466682., 704642., 710078., 476510.], + [716537., 1081829., 1090064., 731459.], + [738425., 1114769., 1123004., 753491.], + [503822., 760550., 766094., 513986.], + ], + ], + ]])); +} + +#[test] +fn test_conv_transpose3d_stride_2() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 1, + channels_out: 1, + kernel_size_1: 2, + kernel_size_2: 2, + kernel_size_3: 2, + padding_1: 0, + padding_2: 0, + padding_3: 0, + padding_out_1: 0, + padding_out_2: 0, + padding_out_3: 0, + stride_1: 2, + stride_2: 2, + stride_3: 2, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 1, + depth: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[[ + [ + [0., 0., 0., 1.], + [0., 0., 2., 3.], + [0., 2., 0., 3.], + [4., 6., 6., 9.], + ], + [ + [0., 0., 4., 5.], + [0., 0., 6., 7.], + [8., 10., 12., 15.], + [12., 14., 18., 21.], + ], + [ + [0., 4., 0., 5.], + [8., 12., 10., 15.], + [0., 6., 0., 7.], + [12., 18., 14., 21.], + ], + [ + [16., 20., 20., 25.], + [24., 28., 30., 35.], + [24., 30., 28., 35.], + [36., 42., 42., 49.], + ], + ]]])); +} + +#[test] +fn test_conv_transpose3d_dilation_2() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + padding_out_1: 1, + padding_out_2: 1, + padding_out_3: 1, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 2, + dilation_2: 2, + dilation_3: 2, + groups: 1, + depth: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[ + [ + [ + [810., 776., 832., 796., 854.], + [756., 712., 774., 728., 792.], + [876., 836., 898., 856., 920.], + [810., 760., 828., 776., 846.], + [942., 896., 964., 916., 986.], + ], + [ + [720., 660., 734., 672., 748.], + [606., 536., 616., 544., 626.], + [762., 696., 776., 708., 790.], + [636., 560., 646., 568., 656.], + [804., 732., 818., 744., 832.], + ], + [ + [1008., 956., 1030., 976., 1052.], + [918., 856., 936., 872., 954.], + [1074., 1016., 1096., 1036., 1118.], + [972., 904., 990., 920., 1008.], + [1140., 1076., 1162., 1096., 1184.], + ], + [ + [846., 768., 860., 780., 874.], + [696., 608., 706., 616., 716.], + [888., 804., 902., 816., 916.], + [726., 632., 736., 640., 746.], + [930., 840., 944., 852., 958.], + ], + [ + [1206., 1136., 1228., 1156., 1250.], + [1080., 1000., 1098., 1016., 1116.], + [1272., 1196., 1294., 1216., 1316.], + [1134., 1048., 1152., 1064., 1170.], + [1338., 1256., 1360., 1276., 1382.], + ], + ], + [ + [ + [1405., 1317., 1427., 1337., 1449.], + [1243., 1145., 1261., 1161., 1279.], + [1471., 1377., 1493., 1397., 1515.], + [1297., 1193., 1315., 1209., 1333.], + [1537., 1437., 1559., 1457., 1581.], + ], + [ + [1099., 985., 1113., 997., 1127.], + [877., 753., 887., 761., 897.], + [1141., 1021., 1155., 1033., 1169.], + [907., 777., 917., 785., 927.], + [1183., 1057., 1197., 1069., 1211.], + ], + [ + [1603., 1497., 1625., 1517., 1647.], + [1405., 1289., 1423., 1305., 1441.], + [1669., 1557., 1691., 1577., 1713.], + [1459., 1337., 1477., 1353., 1495.], + [1735., 1617., 1757., 1637., 1779.], + ], + [ + [1225., 1093., 1239., 1105., 1253.], + [967., 825., 977., 833., 987.], + [1267., 1129., 1281., 1141., 1295.], + [997., 849., 1007., 857., 1017.], + [1309., 1165., 1323., 1177., 1337.], + ], + [ + [1801., 1677., 1823., 1697., 1845.], + [1567., 1433., 1585., 1449., 1603.], + [1867., 1737., 1889., 1757., 1911.], + [1621., 1481., 1639., 1497., 1657.], + [1933., 1797., 1955., 1817., 1977.], + ], + ], + ]])); +} + +#[test] +fn test_conv_transpose3d_stride2_out_padding() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + padding_out_1: 1, + padding_out_2: 1, + padding_out_3: 1, + stride_1: 2, + stride_2: 2, + stride_3: 2, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 1, + depth: 2, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [ + [ + [2144., 4366., 2224., 4526., 2304., 4686., 2384., 2422.], + [4584., 9324., 4744., 9644., 4904., 9964., 5064., 5148.], + [2464., 5006., 2544., 5166., 2624., 5326., 2704., 2750.], + [5224., 10604., 5384., 10924., 5544., 11244., 5704., 5804.], + [2784., 5646., 2864., 5806., 2944., 5966., 3024., 3078.], + [5864., 11884., 6024., 12204., 6184., 12524., 6344., 6460.], + [3104., 6286., 3184., 6446., 3264., 6606., 3344., 3406.], + [3272., 6628., 3358., 6800., 3444., 6972., 3530., 3592.], + ], + [ + [5280., 10716., 5440., 11036., 5600., 11356., 5760., 5868.], + [ + 11152., 22616., 11472., 23256., 11792., 23896., 12112., 12344., + ], + [5920., 11996., 6080., 12316., 6240., 12636., 6400., 6524.], + [ + 12432., 25176., 12752., 25816., 13072., 26456., 13392., 13656., + ], + [6560., 13276., 6720., 13596., 6880., 13916., 7040., 7180.], + [ + 13712., 27736., 14032., 28376., 14352., 29016., 14672., 14968., + ], + [7200., 14556., 7360., 14876., 7520., 15196., 7680., 7836.], + [7632., 15432., 7804., 15776., 7976., 16120., 8148., 8304.], + ], + [ + [3424., 6926., 3504., 7086., 3584., 7246., 3664., 3734.], + [7144., 14444., 7304., 14764., 7464., 15084., 7624., 7772.], + [3744., 7566., 3824., 7726., 3904., 7886., 3984., 4062.], + [7784., 15724., 7944., 16044., 8104., 16364., 8264., 8428.], + [4064., 8206., 4144., 8366., 4224., 8526., 4304., 4390.], + [8424., 17004., 8584., 17324., 8744., 17644., 8904., 9084.], + [4384., 8846., 4464., 9006., 4544., 9166., 4624., 4718.], + [4648., 9380., 4734., 9552., 4820., 9724., 4906., 5000.], + ], + [ + [4000., 8096., 4098., 8292., 4196., 8488., 4294., 4364.], + [8368., 16928., 8564., 17320., 8760., 17712., 8956., 9104.], + [4392., 8880., 4490., 9076., 4588., 9272., 4686., 4764.], + [9152., 18496., 9348., 18888., 9544., 19280., 9740., 9904.], + [4784., 9664., 4882., 9860., 4980., 10056., 5078., 5164.], + [ + 9936., 20064., 10132., 20456., 10328., 20848., 10524., 10704., + ], + [5176., 10448., 5274., 10644., 5372., 10840., 5470., 5564.], + [5440., 10982., 5544., 11190., 5648., 11398., 5752., 5846.], + ], + ], + [ + [ + [3009., 6149., 3143., 6417., 3277., 6685., 3411., 3449.], + [6529., 13321., 6797., 13857., 7065., 14393., 7333., 7417.], + [3545., 7221., 3679., 7489., 3813., 7757., 3947., 3993.], + [7601., 15465., 7869., 16001., 8137., 16537., 8405., 8505.], + [4081., 8293., 4215., 8561., 4349., 8829., 4483., 4537.], + [8673., 17609., 8941., 18145., 9209., 18681., 9477., 9593.], + [4617., 9365., 4751., 9633., 4885., 9901., 5019., 5081.], + [4785., 9707., 4925., 9987., 5065., 10267., 5205., 5267.], + ], + [ + [7873., 16009., 8141., 16545., 8409., 17081., 8677., 8785.], + [ + 16769., 34065., 17305., 35137., 17841., 36209., 18377., 18609., + ], + [8945., 18153., 9213., 18689., 9481., 19225., 9749., 9873.], + [ + 18913., 38353., 19449., 39425., 19985., 40497., 20521., 20785., + ], + [ + 10017., 20297., 10285., 20833., 10553., 21369., 10821., 10961., + ], + [ + 21057., 42641., 21593., 43713., 22129., 44785., 22665., 22961., + ], + [ + 11089., 22441., 11357., 22977., 11625., 23513., 11893., 12049., + ], + [ + 11521., 23317., 11801., 23877., 12081., 24437., 12361., 12517., + ], + ], + [ + [5153., 10437., 5287., 10705., 5421., 10973., 5555., 5625.], + [ + 10817., 21897., 11085., 22433., 11353., 22969., 11621., 11769., + ], + [5689., 11509., 5823., 11777., 5957., 12045., 6091., 6169.], + [ + 11889., 24041., 12157., 24577., 12425., 25113., 12693., 12857., + ], + [6225., 12581., 6359., 12849., 6493., 13117., 6627., 6713.], + [ + 12961., 26185., 13229., 26721., 13497., 27257., 13765., 13945., + ], + [6761., 13653., 6895., 13921., 7029., 14189., 7163., 7257.], + [7025., 14187., 7165., 14467., 7305., 14747., 7445., 7539.], + ], + [ + [5729., 11607., 5881., 11911., 6033., 12215., 6185., 6255.], + [ + 12041., 24381., 12345., 24989., 12649., 25597., 12953., 13101., + ], + [6337., 12823., 6489., 13127., 6641., 13431., 6793., 6871.], + [ + 13257., 26813., 13561., 27421., 13865., 28029., 14169., 14333., + ], + [6945., 14039., 7097., 14343., 7249., 14647., 7401., 7487.], + [ + 14473., 29245., 14777., 29853., 15081., 30461., 15385., 15565., + ], + [7553., 15255., 7705., 15559., 7857., 15863., 8009., 8103.], + [7817., 15789., 7975., 16105., 8133., 16421., 8291., 8385.], + ], + ], + ]])); +} + +#[test] +fn test_conv_transpose3d_groups_2() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 2, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 1, + padding_2: 1, + padding_3: 1, + padding_out_1: 0, + padding_out_2: 0, + padding_out_3: 0, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 2, + depth: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[ + [[[96., 124.], [180., 208.]], [[348., 376.], [432., 460.]]], + [ + [[2997., 3089.], [3273., 3365.]], + [[3825., 3917.], [4101., 4193.]], + ], + ]])); +} + +#[test] +fn test_conv_transpose3d_groups_different_channels() { + let test = ConvTranspose3dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 6, + kernel_size_1: 3, + kernel_size_2: 3, + kernel_size_3: 3, + padding_1: 0, + padding_2: 0, + padding_3: 0, + padding_out_1: 0, + padding_out_2: 0, + padding_out_3: 0, + stride_1: 1, + stride_2: 1, + stride_3: 1, + dilation_1: 1, + dilation_2: 1, + dilation_3: 1, + groups: 2, + depth: 2, + height: 2, + width: 2, + }; + + test.assert_output(TestTensor::from([[ + [ + [ + [0., 0., 1., 2.], + [0., 5., 11., 11.], + [6., 23., 29., 23.], + [12., 32., 37., 24.], + ], + [ + [0., 13., 23., 21.], + [30., 96., 124., 86.], + [66., 180., 208., 134.], + [66., 161., 179., 107.], + ], + [ + [36., 103., 113., 75.], + [138., 348., 376., 230.], + [174., 432., 460., 278.], + [138., 323., 341., 197.], + ], + [ + [72., 166., 175., 100.], + [192., 433., 455., 255.], + [222., 499., 521., 291.], + [144., 318., 331., 182.], + ], + ], + [ + [ + [1., 28., 29., 30.], + [55., 168., 174., 120.], + [61., 186., 192., 132.], + [67., 168., 173., 106.], + ], + [ + [109., 284., 294., 184.], + [355., 853., 881., 519.], + [391., 937., 965., 567.], + [283., 648., 666., 378.], + ], + [ + [145., 374., 384., 238.], + [463., 1105., 1133., 663.], + [499., 1189., 1217., 711.], + [355., 810., 828., 468.], + ], + [ + [181., 410., 419., 236.], + [463., 1028., 1050., 580.], + [493., 1094., 1116., 616.], + [307., 670., 683., 372.], + ], + ], + [ + [ + [2., 56., 57., 58.], + [110., 331., 337., 229.], + [116., 349., 355., 241.], + [122., 304., 309., 188.], + ], + [ + [218., 555., 565., 347.], + [680., 1610., 1638., 952.], + [716., 1694., 1722., 1000.], + [500., 1135., 1153., 649.], + ], + [ + [254., 645., 655., 401.], + [788., 1862., 1890., 1096.], + [824., 1946., 1974., 1144.], + [572., 1297., 1315., 739.], + ], + [ + [290., 654., 663., 372.], + [734., 1623., 1645., 905.], + [764., 1689., 1711., 941.], + [470., 1022., 1035., 562.], + ], + ], + [ + [ + [651., 1388., 1405., 750.], + [1485., 3150., 3188., 1690.], + [1539., 3264., 3302., 1750.], + [873., 1840., 1861., 982.], + ], + [ + [1695., 3578., 3620., 1910.], + [3789., 7967., 8059., 4233.], + [3921., 8243., 8335., 4377.], + [2181., 4566., 4616., 2416.], + ], + [ + [1875., 3956., 3998., 2108.], + [4185., 8795., 8887., 4665.], + [4317., 9071., 9163., 4809.], + [2397., 5016., 5066., 2650.], + ], + [ + [1191., 2490., 2515., 1316.], + [2613., 5450., 5504., 2870.], + [2691., 5612., 5666., 2954.], + [1473., 3062., 3091., 1608.], + ], + ], + [ + [ + [868., 1848., 1865., 994.], + [1972., 4177., 4215., 2231.], + [2026., 4291., 4329., 2291.], + [1144., 2408., 2429., 1280.], + ], + [ + [2236., 4713., 4755., 2505.], + [4978., 10452., 10544., 5530.], + [5110., 10728., 10820., 5674.], + [2830., 5917., 5967., 3119.], + ], + [ + [2416., 5091., 5133., 2703.], + [5374., 11280., 11372., 5962.], + [5506., 11556., 11648., 6106.], + [3046., 6367., 6417., 3353.], + ], + [ + [1516., 3166., 3191., 1668.], + [3316., 6909., 6963., 3627.], + [3394., 7071., 7125., 3711.], + [1852., 3846., 3875., 2014.], + ], + ], + [ + [ + [1085., 2308., 2325., 1238.], + [2459., 5204., 5242., 2772.], + [2513., 5318., 5356., 2832.], + [1415., 2976., 2997., 1578.], + ], + [ + [2777., 5848., 5890., 3100.], + [6167., 12937., 13029., 6827.], + [6299., 13213., 13305., 6971.], + [3479., 7268., 7318., 3822.], + ], + [ + [2957., 6226., 6268., 3298.], + [6563., 13765., 13857., 7259.], + [6695., 14041., 14133., 7403.], + [3695., 7718., 7768., 4056.], + ], + [ + [1841., 3842., 3867., 2020.], + [4019., 8368., 8422., 4384.], + [4097., 8530., 8584., 4468.], + [2231., 4630., 4659., 2420.], + ], + ], + ]])); +} + +struct ConvTranspose3dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + kernel_size_3: usize, + padding_1: usize, + padding_2: usize, + padding_3: usize, + padding_out_1: usize, + padding_out_2: usize, + padding_out_3: usize, + stride_1: usize, + stride_2: usize, + stride_3: usize, + dilation_1: usize, + dilation_2: usize, + dilation_3: usize, + groups: usize, + depth: usize, + height: usize, + width: usize, +} + +impl ConvTranspose3dTestCase { + fn assert_output(self, y: TestTensor<5>) { + let shape_x = Shape::new([ + self.batch_size, + self.channels_in, + self.depth, + self.height, + self.width, + ]); + let shape_weights = Shape::new([ + self.channels_in, + self.channels_out / self.groups, + self.kernel_size_1, + self.kernel_size_2, + self.kernel_size_3, + ]); + let device = Default::default(); + let weights = TestTensor::from( + TestTensorInt::arange(0..shape_weights.num_elements() as i64, &device) + .reshape::<5, _>(shape_weights) + .into_data(), + ); + let bias = TestTensor::from( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + ); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<5, _>(shape_x) + .into_data(), + ); + let output = conv_transpose3d( + x, + weights, + Some(bias), + ConvTransposeOptions::new( + [self.stride_1, self.stride_2, self.stride_3], + [self.padding_1, self.padding_2, self.padding_3], + [self.padding_out_1, self.padding_out_2, self.padding_out_3], + [self.dilation_1, self.dilation_2, self.dilation_3], + self.groups, + ), + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/deform_conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/deform_conv2d.rs new file mode 100644 index 0000000..c3c6698 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/deform_conv2d.rs @@ -0,0 +1,438 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::module::deform_conv2d; +use burn_tensor::ops::DeformConvOptions; +use burn_tensor::{Shape, Tensor}; + +#[test] +fn test_deform_conv2d_simple() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 5, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [[0.9074, 0.6387], [0.5160, 0.4196]], + [[2.4259, 1.8008], [1.5449, 1.3112]], + [[3.9444, 2.9629], [2.5738, 2.2027]], + [[5.4629, 4.1250], [3.6027, 3.0943]], + [[6.9814, 5.2871], [4.6316, 3.9859]], + ]])); +} + +#[test] +fn test_deform_conv2d_batched() { + let test = DeformConv2dTestCase { + batch_size: 2, + channels_in: 3, + channels_out: 5, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([ + [ + [[0.215466, 0.192846], [0.193407, 0.175496]], + [[0.725073, 0.675926], [0.687746, 0.648506]], + [[1.234679, 1.159006], [1.182085, 1.121516]], + [[1.744286, 1.642086], [1.676423, 1.594526]], + [[2.253892, 2.125167], [2.170762, 2.067536]], + ], + [ + [[1.652976, 1.136937], [0.984030, 0.718403]], + [[4.836801, 3.472453], [3.177263, 2.418021]], + [[8.020626, 5.807969], [5.370497, 4.117639]], + [[11.204453, 8.143486], [7.563731, 5.817256]], + [[14.388277, 10.479003], [9.756965, 7.516875]], + ], + ])) +} + +#[test] +fn test_deform_conv2d_weight_groups() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 6, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 3, + offset_groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [[0.101823, 0.065756], [0.046691, 0.036233]], + [[0.412523, 0.336674], [0.306863, 0.282386]], + [[1.307585, 1.024152], [0.902454, 0.800008]], + [[1.840507, 1.458072], [1.299371, 1.158781]], + [[3.402235, 2.634555], [2.305198, 2.014265]], + [[4.157379, 3.231476], [2.838861, 2.485659]], + ]])) +} + +#[test] +fn test_deform_conv2d_offset_groups() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 3, + channels_out: 6, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 3, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [[1.0794, 0.7676], [0.7209, 0.5337]], + [[2.7059, 2.0216], [1.9740, 1.5419]], + [[4.3325, 3.2755], [3.2271, 2.5501]], + [[5.9590, 4.5295], [4.4802, 3.5582]], + [[7.5855, 5.7835], [5.7333, 4.5664]], + [[9.2120, 7.0375], [6.9864, 5.5746]], + ]])) +} + +#[test] +fn test_deform_conv2d_different_kernel_size() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 4, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [[1.0669], [0.6329]], + [[2.9741], [2.0383]], + [[4.8812], [3.4437]], + ]])) +} + +#[test] +fn test_deform_conv2d_different_padding_size() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 2, + padding_2: 3, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [ + [ + 0.199779, 0.376176, 0.528501, 0.605256, 0.384365, 0.198675, 0.048145, 0.000000, + ], + [ + 0.287923, 0.551719, 0.777562, 0.890479, 0.580469, 0.304325, 0.079554, 0.000000, + ], + [ + 0.372947, 0.721405, 1.013668, 1.151988, 0.756444, 0.393098, 0.101582, 0.000000, + ], + [ + 0.132138, 0.324872, 0.495372, 0.584617, 0.453122, 0.250084, 0.075703, 0.000000, + ], + [ + 0.059332, 0.160658, 0.244789, 0.297057, 0.239464, 0.132701, 0.047114, 0.000000, + ], + [ + 0.014338, 0.051338, 0.078303, 0.094190, 0.081278, 0.041954, 0.014506, 0.000000, + ], + ], + [ + [ + 0.766652, 1.164805, 1.521938, 1.711110, 1.230500, 0.807579, 0.450423, 0.333333, + ], + [ + 0.981162, 1.601005, 2.152534, 2.440920, 1.745547, 1.091843, 0.536749, 0.333333, + ], + [ + 1.196386, 2.044845, 2.785330, 3.152243, 2.242613, 1.351308, 0.604905, 0.333333, + ], + [ + 0.669465, 1.178133, 1.644096, 1.902188, 1.573183, 1.033924, 0.553577, 0.333333, + ], + [ + 0.495048, 0.786124, 1.039796, 1.204721, 1.052342, 0.743887, 0.483380, 0.333333, + ], + [ + 0.378767, 0.498209, 0.592867, 0.654230, 0.615487, 0.488202, 0.390890, 0.333333, + ], + ], + [ + [ + 1.333524, 1.953435, 2.515375, 2.816964, 2.076636, 1.416483, 0.852701, 0.666667, + ], + [ + 1.674402, 2.650291, 3.527507, 3.991360, 2.910625, 1.879361, 0.993943, 0.666667, + ], + [ + 2.019825, 3.368286, 4.556992, 5.152499, 3.728782, 2.309520, 1.108229, 0.666667, + ], + [ + 1.206791, 2.031395, 2.792820, 3.219759, 2.693245, 1.817763, 1.031452, 0.666667, + ], + [ + 0.930765, 1.411590, 1.834802, 2.112385, 1.865221, 1.355072, 0.919646, 0.666667, + ], + [ + 0.743195, 0.945081, 1.107431, 1.214270, 1.149695, 0.934451, 0.767274, 0.666667, + ], + ], + ]])) +} + +#[test] +fn test_deform_conv2d_different_stride() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 2, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 1, + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [[1.0647], [0.5783]], + [[2.9289], [1.8829]], + [[4.7931], [3.1875]], + ]])) +} + +#[test] +fn test_deform_conv2d_different_dilation() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 2, + weight_groups: 1, + offset_groups: 1, + height: 5, + width: 5, + }; + + test.assert_output(TestTensor::<4>::from([[ + [[0.6162], [0.7611], [0.4666]], + [[1.8578], [2.2684], [1.6208]], + [[3.0994], [3.7757], [2.7749]], + ]])) +} + +#[test] +fn test_deform_conv2d_different_width() { + let test = DeformConv2dTestCase { + batch_size: 1, + channels_in: 2, + channels_out: 3, + kernel_size_1: 3, + kernel_size_2: 3, + padding_1: 0, + padding_2: 0, + stride_1: 1, + stride_2: 1, + dilation_1: 1, + dilation_2: 1, + weight_groups: 1, + offset_groups: 1, + height: 6, + width: 4, + }; + + test.assert_output(TestTensor::<4>::from([[ + [ + [0.8909, 0.6016], + [1.0697, 0.7186], + [1.2618, 0.8433], + [0.6424, 0.5032], + ], + [ + [2.4670, 1.8168], + [2.9529, 2.1497], + [3.4805, 2.5090], + [2.0925, 1.7411], + ], + [ + [4.0432, 3.0321], + [4.8362, 3.5809], + [5.6992, 4.1746], + [3.5425, 2.9790], + ], + ]])) +} + +struct DeformConv2dTestCase { + batch_size: usize, + channels_in: usize, + channels_out: usize, + kernel_size_1: usize, + kernel_size_2: usize, + padding_1: usize, + padding_2: usize, + stride_1: usize, + stride_2: usize, + dilation_1: usize, + dilation_2: usize, + weight_groups: usize, + offset_groups: usize, + height: usize, + width: usize, +} + +impl DeformConv2dTestCase { + fn assert_output(self, y: Tensor) { + let out_height = + (self.height + 2 * self.padding_1 - self.dilation_1 * (self.kernel_size_1 - 1) - 1) + / self.stride_1 + + 1; + let out_width = + (self.width + 2 * self.padding_2 - self.dilation_2 * (self.kernel_size_2 - 1) - 1) + / self.stride_2 + + 1; + + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let shape_weight = Shape::new([ + self.channels_out, + self.channels_in / self.weight_groups, + self.kernel_size_1, + self.kernel_size_2, + ]); + let shape_offset = Shape::new([ + self.batch_size, + self.kernel_size_1 * self.kernel_size_2 * self.offset_groups * 2, + out_height, + out_width, + ]); + let shape_mask = Shape::new([ + self.batch_size, + self.kernel_size_1 * self.kernel_size_2 * self.offset_groups, + out_height, + out_width, + ]); + let device = Default::default(); + let weight = TestTensor::<4>::from( + TestTensorInt::arange(0..shape_weight.num_elements() as i64, &device) + .reshape::<4, _>(shape_weight.clone()) + .into_data(), + ) + .div_scalar(shape_weight.num_elements() as f32); + let bias = TestTensor::<1>::from( + TestTensorInt::arange(0..self.channels_out as i64, &device).into_data(), + ) + .div_scalar(self.channels_out as f32); + let x = TestTensor::<4>::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &device) + .reshape::<4, _>(shape_x.clone()) + .into_data(), + ) + .div_scalar(shape_x.num_elements() as f32); + let offset = TestTensor::<4>::from( + TestTensorInt::arange(0..shape_offset.num_elements() as i64, &device) + .reshape::<4, _>(shape_offset.clone()) + .into_data(), + ) + .div_scalar(shape_offset.num_elements() as f32); + let mask = TestTensor::<4>::from( + TestTensorInt::arange(0..shape_mask.num_elements() as i64, &device) + .reshape::<4, _>(shape_mask.clone()) + .into_data(), + ) + .div_scalar(shape_mask.num_elements() as f32); + + let output = deform_conv2d( + x, + offset, + weight, + Some(mask), + Some(bias), + DeformConvOptions::new( + [self.stride_1, self.stride_2], + [self.padding_1, self.padding_2], + [self.dilation_1, self.dilation_2], + self.weight_groups, + self.offset_groups, + ), + ); + + let tolerance = Tolerance::permissive(); + y.to_data() + .assert_approx_eq::(&output.into_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/forward.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/forward.rs new file mode 100644 index 0000000..5a9d097 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/forward.rs @@ -0,0 +1,18 @@ +use super::*; +use burn_tensor::{TensorData, module::embedding}; + +#[test] +fn test_embedding_forward() { + let weights = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TensorData::from([[0, 1], [1, 1]]); + let weights = TestTensor::<2>::from(weights); + let indices = TestTensorInt::<2>::from(indices); + + let output = embedding(weights, indices); + let expected = TensorData::from([ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[3.0, 4.0, 5.0], [3.0, 4.0, 5.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/linear.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/linear.rs new file mode 100644 index 0000000..07f06dc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/linear.rs @@ -0,0 +1,59 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use burn_tensor::module::linear; + +#[test] +fn test_linear_1d() { + let weight = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let x = TestTensor::<1>::from([1.0, 2.0]); + let output = linear(x, weight, None); + + let expected = TensorData::from([7.0, 10.0]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(1e-5)); +} + +#[test] +fn test_linear_1d_one_element_output() { + let weight = TestTensor::<2>::from([[3.0], [4.0]]); + + let x = TestTensor::<1>::from([1.0, 2.0]); + let output = linear(x, weight, None); + + let expected = TensorData::from([11.0]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(1e-5)); +} + +#[test] +fn test_linear_forward_no_bias() { + let weight = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let x = TestTensor::<3>::from([[[1.0, 2.0], [3.0, 4.0]], [[-1.0, -2.0], [-3.0, -4.0]]]); + + let output = linear(x, weight, None); + + let expected = TensorData::from([[[7.0, 10.0], [15.0, 22.0]], [[-7.0, -10.0], [-15.0, -22.0]]]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(1e-5)); +} + +#[test] +fn test_linear_forward_with_bias() { + let weight = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + let bias = Some(TestTensor::<1>::from([1.0, -1.0])); + + let x = TestTensor::<3>::from([[[1.0, 2.0], [3.0, 4.0]], [[-1.0, -2.0], [-3.0, -4.0]]]); + + let output = linear(x, weight, bias); + + let expected = TensorData::from([[[8.0, 9.0], [16.0, 21.0]], [[-6.0, -11.0], [-14.0, -23.0]]]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(1e-5)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool1d.rs new file mode 100644 index 0000000..fdfee13 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool1d.rs @@ -0,0 +1,155 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use burn_tensor::module::{max_pool1d, max_pool1d_with_indices}; + +#[test] +fn test_max_pool1d_simple() { + let kernel_size = 3; + let padding = 0; + let stride = 1; + let dilation = 1; + + let x = TestTensor::from([[ + [0.9861, 0.5474, 0.4477, 0.0732, 0.3548, 0.8221], + [0.8148, 0.5474, 0.9490, 0.7890, 0.5537, 0.5689], + ]]); + let y = TestTensor::<3>::from([[ + [0.9861, 0.5474, 0.4477, 0.8221], + [0.949, 0.949, 0.949, 0.789], + ]]); + + let output = max_pool1d(x, kernel_size, stride, padding, dilation, false); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_different_padding_stride_kernel() { + let kernel_size = 3; + let padding = 1; + let stride = 2; + let dilation = 1; + + let x = TestTensor::from([[[0.6309, 0.6112, 0.6998, 0.4708]]]); + let y = TestTensor::<3>::from([[[0.6309, 0.6998]]]); + + let output = max_pool1d(x, kernel_size, stride, padding, dilation, false); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_with_neg() { + let kernel_size = 3; + let padding = 1; + let stride = 1; + let dilation = 1; + + let x = TestTensor::from([[[-0.6309, -0.6112, -0.6998, -0.4708]]]); + let y = TestTensor::<3>::from([[[-0.6112, -0.6112, -0.4708, -0.4708]]]); + + let output = max_pool1d(x, kernel_size, stride, padding, dilation, false); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_with_dilation() { + let kernel_size = 2; + let padding = 1; + let stride = 1; + let dilation = 2; + + let x = TestTensor::from([[ + [0.9861, 0.5474, 0.4477, 0.0732, 0.3548, 0.8221], + [0.8148, 0.5474, 0.9490, 0.7890, 0.5537, 0.5689], + ]]); + let y = TestTensor::<3>::from([[ + [0.5474, 0.9861, 0.5474, 0.4477, 0.8221, 0.3548], + [0.5474, 0.9490, 0.7890, 0.9490, 0.7890, 0.5537], + ]]); + + let output = max_pool1d(x, kernel_size, stride, padding, dilation, false); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool1d_with_indices() { + let kernel_size = 2; + let padding = 0; + let stride = 1; + let dilation = 1; + + let x = TestTensor::from([[[0.2479, 0.6386, 0.3166, 0.5742]]]); + let indices = TensorData::from([[[1, 1, 3]]]); + let y = TestTensor::<3>::from([[[0.6386, 0.6386, 0.5742]]]); + + let (output, output_indices) = + max_pool1d_with_indices(x, kernel_size, stride, padding, dilation, false); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + output_indices.into_data().assert_eq(&indices, false); +} + +#[test] +fn test_max_pool1d_complex() { + let kernel_size = 4; + let padding = 2; + let stride = 1; + let dilation = 1; + + let x = TestTensor::from([[[0.5388, 0.0676, 0.7122, 0.8316, 0.0653]]]); + let indices = TensorData::from([[[0, 2, 3, 3, 3, 3]]]); + let y = TestTensor::<3>::from([[[0.5388, 0.7122, 0.8316, 0.8316, 0.8316, 0.8316]]]); + + let (output, output_indices) = + max_pool1d_with_indices(x, kernel_size, stride, padding, dilation, false); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + output_indices.into_data().assert_eq(&indices, false); +} + +#[test] +fn test_max_pool1d_ceil_mode() { + // Test ceil_mode=true produces larger output when input doesn't divide evenly by stride + // Input: 1x1x6, kernel: 3, stride: 2, padding: 0 + // Floor mode: output = (6-3)/2+1 = 2 elements + // Ceil mode: output = ceil((6-3)/2)+1 = ceil(1.5)+1 = 3 elements + let kernel_size = 3; + let padding = 0; + let stride = 2; + let dilation = 1; + + let x = TestTensor::from([[[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]]]); + + // With ceil_mode=false (floor): output is 2 elements + // Window 0: positions [0:3] -> max(1,2,3) = 3 + // Window 1: positions [2:5] -> max(3,4,5) = 5 + let y_floor = TestTensor::<3>::from([[[3.0, 5.0]]]); + + let output_floor = max_pool1d(x.clone(), kernel_size, stride, padding, dilation, false); + + y_floor + .to_data() + .assert_approx_eq::(&output_floor.into_data(), Tolerance::default()); + + // With ceil_mode=true: output is 3 elements + // Window 0: positions [0:3] -> max(1,2,3) = 3 + // Window 1: positions [2:5] -> max(3,4,5) = 5 + // Window 2: positions [4:7] -> max(5,6) = 6 (partial window) + let y_ceil = TestTensor::<3>::from([[[3.0, 5.0, 6.0]]]); + + let output_ceil = max_pool1d(x, kernel_size, stride, padding, dilation, true); + + y_ceil + .to_data() + .assert_approx_eq::(&output_ceil.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool2d.rs new file mode 100644 index 0000000..325d5bb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/maxpool2d.rs @@ -0,0 +1,523 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use burn_tensor::module::{max_pool2d, max_pool2d_with_indices}; + +#[test] +fn test_max_pool2d_simple() { + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 1; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 1; + let dilation_2 = 1; + + let x = TestTensor::from([ + [ + [ + [0.9861, 0.5474, 0.4477, 0.0732, 0.3548, 0.8221], + [0.8148, 0.5474, 0.9490, 0.7890, 0.5537, 0.5689], + [0.5986, 0.2059, 0.4897, 0.6136, 0.2965, 0.6182], + [0.1485, 0.9540, 0.4023, 0.6176, 0.7111, 0.3392], + [0.3703, 0.0472, 0.2771, 0.1868, 0.8855, 0.5605], + [0.5063, 0.1638, 0.9432, 0.7836, 0.8696, 0.1068], + ], + [ + [0.8872, 0.0137, 0.1652, 0.5505, 0.6127, 0.6473], + [0.1128, 0.0888, 0.1152, 0.5456, 0.6199, 0.7947], + [0.5911, 0.7781, 0.7256, 0.6578, 0.0989, 0.9149], + [0.5879, 0.5189, 0.6561, 0.0578, 0.7025, 0.6426], + [0.9590, 0.0325, 0.6455, 0.6248, 0.2009, 0.1544], + [0.7339, 0.1369, 0.6598, 0.5528, 0.6775, 0.1572], + ], + ], + [ + [ + [0.6853, 0.6439, 0.4639, 0.5573, 0.2723, 0.5910], + [0.5419, 0.7729, 0.6743, 0.8956, 0.2997, 0.9546], + [0.0334, 0.2178, 0.6917, 0.4958, 0.3357, 0.6584], + [0.7358, 0.9074, 0.2462, 0.5159, 0.6420, 0.2441], + [0.7602, 0.6297, 0.6073, 0.5937, 0.8037, 0.4881], + [0.8859, 0.0974, 0.3954, 0.6763, 0.1078, 0.7467], + ], + [ + [0.2991, 0.5012, 0.8024, 0.7653, 0.9378, 0.7952], + [0.7393, 0.2336, 0.9521, 0.2719, 0.8445, 0.0454], + [0.6479, 0.9822, 0.7905, 0.0318, 0.2474, 0.0628], + [0.9955, 0.7591, 0.4140, 0.3215, 0.4349, 0.1527], + [0.8064, 0.0164, 0.4002, 0.2024, 0.6128, 0.5827], + [0.5368, 0.7895, 0.8727, 0.7793, 0.0910, 0.3421], + ], + ], + ]); + let y = TestTensor::<4>::from([ + [ + [ + [0.9861, 0.9861, 0.9490, 0.9490, 0.8221, 0.8221], + [0.9861, 0.9861, 0.9490, 0.9490, 0.8221, 0.8221], + [0.9540, 0.9540, 0.9540, 0.9490, 0.7890, 0.7111], + [0.9540, 0.9540, 0.9540, 0.8855, 0.8855, 0.8855], + [0.9540, 0.9540, 0.9540, 0.9432, 0.8855, 0.8855], + [0.5063, 0.9432, 0.9432, 0.9432, 0.8855, 0.8855], + ], + [ + [0.8872, 0.8872, 0.5505, 0.6199, 0.7947, 0.7947], + [0.8872, 0.8872, 0.7781, 0.7256, 0.9149, 0.9149], + [0.7781, 0.7781, 0.7781, 0.7256, 0.9149, 0.9149], + [0.9590, 0.9590, 0.7781, 0.7256, 0.9149, 0.9149], + [0.9590, 0.9590, 0.6598, 0.7025, 0.7025, 0.7025], + [0.9590, 0.9590, 0.6598, 0.6775, 0.6775, 0.6775], + ], + ], + [ + [ + [0.7729, 0.7729, 0.8956, 0.8956, 0.9546, 0.9546], + [0.7729, 0.7729, 0.8956, 0.8956, 0.9546, 0.9546], + [0.9074, 0.9074, 0.9074, 0.8956, 0.9546, 0.9546], + [0.9074, 0.9074, 0.9074, 0.8037, 0.8037, 0.8037], + [0.9074, 0.9074, 0.9074, 0.8037, 0.8037, 0.8037], + [0.8859, 0.8859, 0.6763, 0.8037, 0.8037, 0.8037], + ], + [ + [0.7393, 0.9521, 0.9521, 0.9521, 0.9378, 0.9378], + [0.9822, 0.9822, 0.9822, 0.9521, 0.9378, 0.9378], + [0.9955, 0.9955, 0.9822, 0.9521, 0.8445, 0.8445], + [0.9955, 0.9955, 0.9822, 0.7905, 0.6128, 0.6128], + [0.9955, 0.9955, 0.8727, 0.8727, 0.7793, 0.6128], + [0.8064, 0.8727, 0.8727, 0.8727, 0.7793, 0.6128], + ], + ], + ]); + + let output = max_pool2d( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_different_padding_stride_kernel() { + let kernel_size_1 = 3; + let kernel_size_2 = 1; + let padding_1 = 1; + let padding_2 = 0; + let stride_1 = 1; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + let x = TestTensor::from([[[ + [0.6309, 0.6112, 0.6998], + [0.4708, 0.9161, 0.5402], + [0.4577, 0.7397, 0.9870], + [0.6380, 0.4352, 0.5884], + [0.6277, 0.5139, 0.4525], + [0.9333, 0.9846, 0.5006], + ]]]); + let y = TestTensor::<4>::from([[[ + [0.6309, 0.6998], + [0.6309, 0.9870], + [0.6380, 0.9870], + [0.6380, 0.9870], + [0.9333, 0.5884], + [0.9333, 0.5006], + ]]]); + + let output = max_pool2d( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_with_neg() { + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 1; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 1; + let dilation_2 = 1; + + let x = TestTensor::from([[[ + [0.6309, 0.6112, 0.6998], + [0.4708, 0.9161, 0.5402], + [0.4577, 0.7397, 0.9870], + [0.6380, 0.4352, 0.5884], + [0.6277, 0.5139, 0.4525], + [0.9333, 0.9846, 0.5006], + ]]]) + .neg(); + let y = TestTensor::<4>::from([[[ + [-0.4708, -0.4708, -0.5402], + [-0.4577, -0.4577, -0.5402], + [-0.4352, -0.4352, -0.4352], + [-0.4352, -0.4352, -0.4352], + [-0.4352, -0.4352, -0.4352], + [-0.5139, -0.4525, -0.4525], + ]]]); + + let output = max_pool2d( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_with_dilation() { + let kernel_size_1 = 2; + let kernel_size_2 = 2; + let padding_1 = 0; + let padding_2 = 0; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 2; + let dilation_2 = 2; + + let x = TestTensor::from([[[ + [0.9861, 0.9861, 0.9490, 0.9490, 0.8221, 0.8221], + [0.9861, 0.9861, 0.9490, 0.9490, 0.8221, 0.8221], + [0.9540, 0.9540, 0.9540, 0.9490, 0.7890, 0.7111], + [0.9540, 0.9540, 0.9540, 0.8855, 0.8855, 0.8855], + [0.9540, 0.9540, 0.9540, 0.9432, 0.8855, 0.8855], + [0.5063, 0.9432, 0.9432, 0.9432, 0.8855, 0.8855], + ]]]); + let y = TestTensor::<4>::from([[[ + [0.9861, 0.9861, 0.9540, 0.9490], + [0.9861, 0.9861, 0.9540, 0.9490], + [0.9540, 0.9540, 0.9540, 0.9490], + [0.9540, 0.9540, 0.9540, 0.9432], + ]]]); + + let output = max_pool2d( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_with_indices() { + let kernel_size_1 = 2; + let kernel_size_2 = 2; + let padding_1 = 1; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 1; + let dilation_1 = 1; + let dilation_2 = 1; + + let x = TestTensor::from([[[ + [0.2479, 0.6386, 0.3166, 0.5742], + [0.7065, 0.1940, 0.6305, 0.8959], + [0.5416, 0.8602, 0.8129, 0.1662], + [0.3358, 0.3059, 0.8293, 0.0990], + ]]]); + let indices = TensorData::from([[[ + [0, 1, 1, 3, 3], + [4, 4, 1, 7, 7], + [4, 9, 9, 7, 7], + [8, 9, 9, 14, 11], + [12, 12, 14, 14, 15], + ]]]); + let y = TestTensor::<4>::from([[[ + [0.2479, 0.6386, 0.6386, 0.5742, 0.5742], + [0.7065, 0.7065, 0.6386, 0.8959, 0.8959], + [0.7065, 0.8602, 0.8602, 0.8959, 0.8959], + [0.5416, 0.8602, 0.8602, 0.8293, 0.1662], + [0.3358, 0.3358, 0.8293, 0.8293, 0.0990], + ]]]); + + let (output, output_indices) = max_pool2d_with_indices( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + output_indices.into_data().assert_eq(&indices, false); +} + +#[test] +fn test_max_pool2d_complex() { + let kernel_size_1 = 4; + let kernel_size_2 = 2; + let padding_1 = 2; + let padding_2 = 1; + let stride_1 = 1; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + let x = TestTensor::from([[[ + [0.5388, 0.0676, 0.7122, 0.8316, 0.0653], + [0.9154, 0.1536, 0.9089, 0.8016, 0.7518], + [0.2073, 0.0501, 0.8811, 0.5604, 0.5075], + [0.4384, 0.9963, 0.9698, 0.4988, 0.2609], + [0.3391, 0.2230, 0.4610, 0.5365, 0.6880], + ]]]); + let indices = TensorData::from([[[ + [5, 7, 3], + [5, 7, 3], + [5, 16, 3], + [5, 16, 8], + [15, 16, 24], + [15, 16, 24], + ]]]); + let y = TestTensor::<4>::from([[[ + [0.9154, 0.9089, 0.8316], + [0.9154, 0.9089, 0.8316], + [0.9154, 0.9963, 0.8316], + [0.9154, 0.9963, 0.8016], + [0.4384, 0.9963, 0.688], + [0.4384, 0.9963, 0.688], + ]]]); + let (output, output_indices) = max_pool2d_with_indices( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + output_indices.into_data().assert_eq(&indices, false); +} + +#[test] +fn test_max_pool2d_ceil_mode() { + // Test ceil_mode=true which produces larger output when input doesn't divide evenly by stride + // Using 1x1x6x6 with kernel 3x3, stride 2x2, padding 0: + // Floor mode: output = (6+0-1*(3-1)-1)/2+1 = 3/2+1 = 2 x 2 + // Ceil mode: output = ceil(3/2)+1 = 2+1 = 3 x 3 + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 0; + let padding_2 = 0; + let stride_1 = 2; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + // Input (values 1-36 arranged row by row): + // col: 0 1 2 3 4 5 + // row 0: 1 2 3 4 5 6 + // row 1: 7 8 9 10 11 12 + // row 2: 13 14 15 16 17 18 + // row 3: 19 20 21 22 23 24 + // row 4: 25 26 27 28 29 30 + // row 5: 31 32 33 34 35 36 + let x = TestTensor::from([[[ + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + [7.0, 8.0, 9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0, 17.0, 18.0], + [19.0, 20.0, 21.0, 22.0, 23.0, 24.0], + [25.0, 26.0, 27.0, 28.0, 29.0, 30.0], + [31.0, 32.0, 33.0, 34.0, 35.0, 36.0], + ]]]); + + // With ceil_mode=false (floor): output is 2x2 + // (0,0): rows 0-2, cols 0-2 -> max(1,2,3,7,8,9,13,14,15) = 15 + // (0,1): rows 0-2, cols 2-4 -> max(3,4,5,9,10,11,15,16,17) = 17 + // (1,0): rows 2-4, cols 0-2 -> max(13,14,15,19,20,21,25,26,27) = 27 + // (1,1): rows 2-4, cols 2-4 -> max(15,16,17,21,22,23,27,28,29) = 29 + let y_floor = TestTensor::<4>::from([[[[15.0, 17.0], [27.0, 29.0]]]]); + + let output_floor = max_pool2d( + x.clone(), + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + false, + ); + + y_floor + .to_data() + .assert_approx_eq::(&output_floor.into_data(), Tolerance::default()); + + // With ceil_mode=true: output is 3x3 + // Extra windows at edges use only available input values (padded with -inf for max pooling) + // (0,0): rows 0-2, cols 0-2 -> max = 15 + // (0,1): rows 0-2, cols 2-4 -> max = 17 + // (0,2): rows 0-2, cols 4-5 -> max(5,6,11,12,17,18) = 18 + // (1,0): rows 2-4, cols 0-2 -> max = 27 + // (1,1): rows 2-4, cols 2-4 -> max = 29 + // (1,2): rows 2-4, cols 4-5 -> max(17,18,23,24,29,30) = 30 + // (2,0): rows 4-5, cols 0-2 -> max(25,26,27,31,32,33) = 33 + // (2,1): rows 4-5, cols 2-4 -> max(27,28,29,33,34,35) = 35 + // (2,2): rows 4-5, cols 4-5 -> max(29,30,35,36) = 36 + let y_ceil = + TestTensor::<4>::from([[[[15.0, 17.0, 18.0], [27.0, 29.0, 30.0], [33.0, 35.0, 36.0]]]]); + + let output_ceil = max_pool2d( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + true, + ); + + y_ceil + .to_data() + .assert_approx_eq::(&output_ceil.into_data(), Tolerance::default()); +} + +#[test] +fn test_max_pool2d_ceil_mode_with_indices() { + // Test ceil_mode=true with indices to verify correct index calculation + // when pooling windows extend beyond original input bounds + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 0; + let padding_2 = 0; + let stride_1 = 2; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + // Input 6x6 (indices 0-35 in row-major order): + // row 0: 0 1 2 3 4 5 + // row 1: 6 7 8 9 10 11 + // row 2: 12 13 14 15 16 17 + // row 3: 18 19 20 21 22 23 + // row 4: 24 25 26 27 28 29 + // row 5: 30 31 32 33 34 35 + let x = TestTensor::from([[[ + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], + [6.0, 7.0, 8.0, 9.0, 10.0, 11.0], + [12.0, 13.0, 14.0, 15.0, 16.0, 17.0], + [18.0, 19.0, 20.0, 21.0, 22.0, 23.0], + [24.0, 25.0, 26.0, 27.0, 28.0, 29.0], + [30.0, 31.0, 32.0, 33.0, 34.0, 35.0], + ]]]); + + // With ceil_mode=true: output is 3x3 + // (0,0): rows 0-2, cols 0-2 -> max at index 14 + // (0,1): rows 0-2, cols 2-4 -> max at index 16 + // (0,2): rows 0-2, cols 4-5 -> max at index 17 + // (1,0): rows 2-4, cols 0-2 -> max at index 26 + // (1,1): rows 2-4, cols 2-4 -> max at index 28 + // (1,2): rows 2-4, cols 4-5 -> max at index 29 + // (2,0): rows 4-5, cols 0-2 -> max at index 32 + // (2,1): rows 4-5, cols 2-4 -> max at index 34 + // (2,2): rows 4-5, cols 4-5 -> max at index 35 + let expected_values = + TestTensor::<4>::from([[[[14.0, 16.0, 17.0], [26.0, 28.0, 29.0], [32.0, 34.0, 35.0]]]]); + let expected_indices = TensorData::from([[[[14i64, 16, 17], [26, 28, 29], [32, 34, 35]]]]); + + let (output, output_indices) = max_pool2d_with_indices( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + true, + ); + + expected_values + .to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + output_indices + .into_data() + .assert_eq(&expected_indices, false); +} + +#[test] +fn test_max_pool2d_ceil_mode_with_indices_and_padding() { + // Test ceil_mode=true with padding and indices to verify correct index calculation + // This exercises the case where both user padding and ceil_mode extra padding apply + let kernel_size_1 = 3; + let kernel_size_2 = 3; + let padding_1 = 1; + let padding_2 = 1; + let stride_1 = 2; + let stride_2 = 2; + let dilation_1 = 1; + let dilation_2 = 1; + + // Input 5x5 (indices 0-24 in row-major order): + // row 0: 0 1 2 3 4 + // row 1: 5 6 7 8 9 + // row 2: 10 11 12 13 14 + // row 3: 15 16 17 18 19 + // row 4: 20 21 22 23 24 + let x = TestTensor::from([[[ + [0.0, 1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0, 9.0], + [10.0, 11.0, 12.0, 13.0, 14.0], + [15.0, 16.0, 17.0, 18.0, 19.0], + [20.0, 21.0, 22.0, 23.0, 24.0], + ]]]); + + // With padding=1, ceil_mode=true: + // Effective input is 7x7 (5 + 2*1) + // Output size: ceil((5 + 2*1 - 3) / 2) + 1 = ceil(4/2) + 1 = 3 + // + // Windows (with -inf padding at boundaries): + // (0,0): rows -1 to 1, cols -1 to 1 -> valid: (0,0) to (1,1), max at (1,1)=6 + // (0,1): rows -1 to 1, cols 1 to 3 -> max at (1,3)=8 + // (0,2): rows -1 to 1, cols 3 to 5 -> max at (1,4)=9 + // (1,0): rows 1 to 3, cols -1 to 1 -> max at (3,1)=16 + // (1,1): rows 1 to 3, cols 1 to 3 -> max at (3,3)=18 + // (1,2): rows 1 to 3, cols 3 to 5 -> max at (3,4)=19 + // (2,0): rows 3 to 5, cols -1 to 1 -> max at (4,1)=21 + // (2,1): rows 3 to 5, cols 1 to 3 -> max at (4,3)=23 + // (2,2): rows 3 to 5, cols 3 to 5 -> max at (4,4)=24 + let expected_values = + TestTensor::<4>::from([[[[6.0, 8.0, 9.0], [16.0, 18.0, 19.0], [21.0, 23.0, 24.0]]]]); + let expected_indices = TensorData::from([[[[6i64, 8, 9], [16, 18, 19], [21, 23, 24]]]]); + + let (output, output_indices) = max_pool2d_with_indices( + x, + [kernel_size_1, kernel_size_2], + [stride_1, stride_2], + [padding_1, padding_2], + [dilation_1, dilation_2], + true, + ); + + expected_values + .to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + output_indices + .into_data() + .assert_eq(&expected_indices, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/mod.rs new file mode 100644 index 0000000..f2a9cf2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/mod.rs @@ -0,0 +1,22 @@ +use super::*; + +mod adaptive_avgpool1d; +mod adaptive_avgpool2d; +mod attention; +mod avgpool1d; +mod avgpool2d; +mod bicubic_interpolate; +mod bilinear_interpolate; +mod conv1d; +mod conv2d; +mod conv3d; +mod conv_transpose1d; +mod conv_transpose2d; +mod conv_transpose3d; +mod deform_conv2d; +mod forward; +mod linear; +mod maxpool1d; +mod maxpool2d; +mod nearest_interpolate; +mod unfold4d; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/nearest_interpolate.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/nearest_interpolate.rs new file mode 100644 index 0000000..890bedb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/nearest_interpolate.rs @@ -0,0 +1,127 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::interpolate; +use burn_tensor::ops::{InterpolateMode, InterpolateOptions}; + +#[test] +fn test_upsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 2, + channels: 1, + height: 7, + width: 5, + height_out: 8, + width_out: 7, + }; + + test.assert_output(TestTensor::from([ + [[ + [0., 0., 1., 2., 2., 3., 4.], + [0., 0., 1., 2., 2., 3., 4.], + [5., 5., 6., 7., 7., 8., 9.], + [10., 10., 11., 12., 12., 13., 14.], + [15., 15., 16., 17., 17., 18., 19.], + [20., 20., 21., 22., 22., 23., 24.], + [25., 25., 26., 27., 27., 28., 29.], + [30., 30., 31., 32., 32., 33., 34.], + ]], + [[ + [35., 35., 36., 37., 37., 38., 39.], + [35., 35., 36., 37., 37., 38., 39.], + [40., 40., 41., 42., 42., 43., 44.], + [45., 45., 46., 47., 47., 48., 49.], + [50., 50., 51., 52., 52., 53., 54.], + [55., 55., 56., 57., 57., 58., 59.], + [60., 60., 61., 62., 62., 63., 64.], + [65., 65., 66., 67., 67., 68., 69.], + ]], + ])); +} + +#[test] +fn test_downsample_interpolation() { + let test = InterpolateTestCase { + batch_size: 1, + channels: 1, + height: 45, + width: 14, + height_out: 4, + width_out: 6, + }; + + test.assert_output(TestTensor::from([[[ + [0., 2., 4., 7., 9., 11.], + [154., 156., 158., 161., 163., 165.], + [308., 310., 312., 315., 317., 319.], + [462., 464., 466., 469., 471., 473.], + ]]])); +} + +#[test] +fn test_1d_nearest() { + // Initialize the model without weights (because the exported file does not contain them) + let device = Default::default(); + + // Run the model + let input = TestTensor::<3>::from_floats( + [[[1.5410, -0.2934, -2.1788, 0.5684, -1.0845, -1.3986]]], + &device, + ); + + let input = input.unsqueeze_dim(2); + + let output = interpolate( + input, + [1, 9], + InterpolateOptions::new(InterpolateMode::Nearest), + ); + assert_eq!(output.dims(), [1, 1, 1, 9]); + + // assert output data does not contain NaN + assert!( + !output + .clone() + .to_data() + .as_slice::() + .unwrap() + .iter() + .any(|&x| x.is_nan()), + "interpolate output contains NaN" + ); + + TestTensor::<4>::from([[[[ + 1.541, 1.541, -0.2934, -2.1788, -2.1788, 0.5684, -1.0845, -1.0845, -1.3986, + ]]]]) + .to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); +} + +struct InterpolateTestCase { + batch_size: usize, + channels: usize, + height: usize, + width: usize, + height_out: usize, + width_out: usize, +} + +impl InterpolateTestCase { + fn assert_output(self, y: TestTensor<4>) { + let shape_x = Shape::new([self.batch_size, self.channels, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &y.device()) + .reshape::<4, _>(shape_x) + .into_data() + .convert::(), + ); + let output = interpolate( + x, + [self.height_out, self.width_out], + InterpolateOptions::new(InterpolateMode::Nearest), + ); + + y.to_data() + .assert_approx_eq::(&output.into_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/unfold4d.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/unfold4d.rs new file mode 100644 index 0000000..ed20837 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/module/unfold4d.rs @@ -0,0 +1,132 @@ +use super::*; +use burn_tensor::Shape; +use burn_tensor::Tolerance; +use burn_tensor::module::unfold4d; +use burn_tensor::ops::UnfoldOptions; + +#[test] +fn test_unfold4d_shape() { + let test = Unfold4dTestCase { + batch_size: 2, + channels_in: 5, + kernel_size: [2, 3], + padding: [0, 0], + stride: [1, 1], + dilation: [1, 1], + height: 3, + width: 4, + }; + + test.assert_shape([2, 30, 4]); +} + +#[test] +fn test_unfold4d_simple() { + let test = Unfold4dTestCase { + batch_size: 1, + channels_in: 2, + kernel_size: [2, 2], + padding: [0, 0], + stride: [1, 1], + dilation: [1, 1], + height: 4, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [0., 1., 2., 4., 5., 6., 8., 9., 10.], + [1., 2., 3., 5., 6., 7., 9., 10., 11.], + [4., 5., 6., 8., 9., 10., 12., 13., 14.], + [5., 6., 7., 9., 10., 11., 13., 14., 15.], + [16., 17., 18., 20., 21., 22., 24., 25., 26.], + [17., 18., 19., 21., 22., 23., 25., 26., 27.], + [20., 21., 22., 24., 25., 26., 28., 29., 30.], + [21., 22., 23., 25., 26., 27., 29., 30., 31.], + ]])); +} + +#[test] +fn test_unfold4d_complex() { + let test = Unfold4dTestCase { + batch_size: 1, + channels_in: 2, + kernel_size: [2, 3], + padding: [0, 1], + stride: [1, 2], + dilation: [1, 2], + height: 3, + width: 4, + }; + + test.assert_output(TestTensor::from([[ + [0., 0.], + [1., 5.], + [3., 7.], + [0., 0.], + [5., 9.], + [7., 11.], + [0., 0.], + [13., 17.], + [15., 19.], + [0., 0.], + [17., 21.], + [19., 23.], + ]])); +} + +struct Unfold4dTestCase { + batch_size: usize, + channels_in: usize, + kernel_size: [usize; 2], + padding: [usize; 2], + stride: [usize; 2], + dilation: [usize; 2], + height: usize, + width: usize, +} + +impl Unfold4dTestCase { + fn assert_shape(self, expected_shape: [usize; 3]) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &Default::default()) + .reshape::<4, _>(shape_x) + .into_data() + .convert::(), + ); + + let output = unfold4d( + x, + self.kernel_size, + UnfoldOptions::new(self.stride, self.padding, self.dilation), + ); + + assert_eq!( + output.shape().as_slice(), + expected_shape, + "Expected shape doesn't match the actual shape" + ); + } + + fn assert_output(self, expected: TestTensor<3>) { + let shape_x = Shape::new([self.batch_size, self.channels_in, self.height, self.width]); + let x = TestTensor::from( + TestTensorInt::arange(0..shape_x.num_elements() as i64, &Default::default()) + .reshape::<4, _>(shape_x) + .into_data(), + ); + + let output = unfold4d( + x, + self.kernel_size, + UnfoldOptions::new(self.stride, self.padding, self.dilation), + ); + + let tolerance = Tolerance::default() + .set_half_precision_relative(2e-3) + .set_half_precision_absolute(2e-3); + output + .into_data() + .assert_approx_eq::(&expected.into_data(), tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/abs.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/abs.rs new file mode 100644 index 0000000..bd7eed8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/abs.rs @@ -0,0 +1,13 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_abs_ops_float() { + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, 4.0, -5.0]]); + + let output = tensor.abs(); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/add.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/add.rs new file mode 100644 index 0000000..2e1ea5d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/add.rs @@ -0,0 +1,118 @@ +use super::*; +use burn_tensor::{TensorData, backend::Backend}; + +#[test] +fn test_add_d2() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::from([[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]]); + + let output = tensor_1 + tensor_2; + + output.into_data().assert_eq( + &TensorData::from([[6.0, 8.0, 10.0], [12.0, 14.0, 16.0]]), + false, + ); +} + +#[test] +fn test_add_broadcast() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0]]); + let tensor_2 = TestTensor::from([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]); + + let output = tensor_1 + tensor_2; + + output.into_data().assert_eq( + &TensorData::from([[3.0, 5.0, 7.0], [6.0, 8.0, 10.0]]), + false, + ); +} + +#[test] +fn test_add_different_strides_rhs() { + // We need to execute an operation after `from data` to trigger inplace in some backends. + // Which is the operation that might be problematic in this case. + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0], [2.0, 3.0]]) * 1; + let tensor_2 = TestTensor::from([[4.0, 5.0], [6.0, 7.0]]) * 1; + + let output = tensor_1 + tensor_2.transpose(); + + output + .into_data() + .assert_eq(&TensorData::from([[4.0, 7.0], [7.0, 10.0]]), false); +} + +#[test] +fn test_add_different_strides_lhs() { + // We need to execute an operation after `from data` to trigger inplace in some backends. + // Which is the operation that might be problematic in this case. + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0], [2.0, 3.0]]) * 1; + let tensor_2 = TestTensor::from([[4.0, 5.0], [6.0, 7.0]]) * 1; + + let output = tensor_1.transpose() + tensor_2; + + output + .into_data() + .assert_eq(&TensorData::from([[4.0, 7.0], [7.0, 10.0]]), false); +} + +#[test] +fn test_add_different_strides_broadcast() { + // We need to execute an operation after `from data` to trigger inplace in some backends. + // Which is the operation that might be problematic in this case. + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0], [2.0, 3.0]]) * 1; + let tensor_2 = TestTensor::from([[4.0, 5.0]]) * 1; + + let output = tensor_1.transpose() + tensor_2; + + output + .into_data() + .assert_eq(&TensorData::from([[4.0, 7.0], [5.0, 8.0]]), false); +} + +#[test] +fn should_support_add_scalar_ops() { + let scalar = 2.0; + let tensor = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor + scalar; + + output + .into_data() + .assert_eq(&TensorData::from([[2.0, 3.0, 4.0], [5.0, 6.0, 7.0]]), false); +} + +#[test] +fn add_maybe_fused_not_contiguous() { + let tensor1 = TestTensorInt::arange(0..8, &Default::default()).float(); + let tensor2 = TestTensorInt::arange(8..16, &Default::default()).float(); + let tensor1 = tensor1.reshape([2, 4]); + let tensor2 = tensor2.reshape([4, 2]); + let tensor2 = tensor2.swap_dims(0, 1); + + TestBackend::sync(&tensor2.device()).unwrap(); + + let output = tensor1 + tensor2; + + output.into_data().assert_eq( + &TensorData::from([[8.0, 11.0, 14.0, 17.0], [13.0, 16.0, 19.0, 22.0]]), + false, + ); +} + +#[test] +fn add_maybe_fused_not_contiguous_broadcasted() { + let tensor1 = TestTensorInt::arange(0..8, &Default::default()).float(); + let tensor2 = TestTensorInt::arange(8..10, &Default::default()).float(); + let tensor1 = tensor1.reshape([2, 4]); + let tensor2 = tensor2.reshape([1, 2]); + let tensor2 = tensor2.swap_dims(0, 1); + + TestBackend::sync(&tensor2.device()).unwrap(); + + let output = tensor2 + tensor1; + + output.into_data().assert_eq( + &TensorData::from([[8.0, 9.0, 10.0, 11.0], [13.0, 14.0, 15.0, 16.0]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/aggregation.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/aggregation.rs new file mode 100644 index 0000000..e24b9d8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/aggregation.rs @@ -0,0 +1,460 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use burn_tensor::backend::Backend; + +#[test] +fn test_should_mean() { + let tensor = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.mean(); + let expected = TensorData::from([15.0 / 6.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_should_sum() { + let tensor = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sum(); + + output + .into_data() + .assert_eq(&TensorData::from([15.0]), false); +} + +#[test] +fn test_should_sum_dim_maybe_fused() { + let tensor = TestTensor::<2>::from([[5.0], [-12.0]]); + let tensor1 = TestTensor::<2>::from([[2.0, 3.0], [-1.0, -5.0]]); + let ones = TestTensor::<2>::ones([2, 2], &Default::default()); + let _x = ones.clone() * tensor; + let y = ones * tensor1; + + let output = y.clone().sum_dim(1); + output + .into_data() + .assert_eq(&TensorData::from([[5.0], [-6.0]]), false); + + // Negative Indexing. + let output = y.clone().sum_dim(-1); + output + .into_data() + .assert_eq(&TensorData::from([[5.0], [-6.0]]), false); +} + +#[test] +fn test_should_mean_last_dim() { + let tensor = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.clone().mean_dim(1); + let expected = TensorData::from([[3.0 / 3.0], [12.0 / 3.0]]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + // Negative Indexing. + let output = tensor.clone().mean_dim(-1); + let expected = TensorData::from([[3.0 / 3.0], [12.0 / 3.0]]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_should_sum_last_dim() { + let tensor = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sum_dim(1); + + output + .into_data() + .assert_eq(&TensorData::from([[3.0], [12.0]]), false); +} + +#[test] +fn test_should_sum_first_dim() { + let tensor = TestTensor::<2>::from([[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]]); + + let output = tensor.sum_dim(0); + + output + .into_data() + .assert_eq(&TensorData::from([[7.0, 3.0, 5.0]]), false); +} + +#[test] +fn test_should_mean_first_dim() { + let tensor = TestTensor::<2>::from([[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]]); + + let output = tensor.mean_dim(0); + + output.into_data().assert_eq( + &TensorData::from([[7.0 / 2.0, 3.0 / 2.0, 5.0 / 2.0]]), + false, + ); +} + +#[test] +fn test_should_sum_mid_dim_3d_non_contiguous_1() { + let tensor = TestTensor::<3>::from([ + [[2.0, 4.0, 1.0], [7.0, -5.0, 3.0]], + [[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]], + ]); + + let output = tensor.swap_dims(0, 2).sum_dim(1); + + output.into_data().assert_eq( + &TensorData::new(vec![9.0, 7.0, -1.0, 3.0, 4.0, 5.0], [3, 1, 2]), + false, + ); +} + +#[test] +fn test_should_sum_mid_dim_3d_non_contiguous_2() { + let tensor = TestTensor::<3>::from([ + [[2.0, 4.0, 1.0], [7.0, -5.0, 3.0]], + [[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]], + ]); + + let output = tensor.swap_dims(0, 1).sum_dim(1); + + output.into_data().assert_eq( + &TensorData::new(vec![5.0, 5.0, 3.0, 11.0, -3.0, 6.0], [2, 1, 3]), + false, + ); +} + +#[test] +fn test_prod_float() { + let tensor = TestTensor::<2>::from([[2.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let output = tensor.prod(); + + // 2 * 1 * 2 * 3 * 4 * 5 = 240 but we need to check the precision because of the float + let expected = TensorData::from([240.0]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let tensor_with_zero = TestTensor::<2>::from([[2.0, 0.0, 2.0], [3.0, 4.0, 5.0]]); + let output = tensor_with_zero.prod(); + + output + .into_data() + .assert_eq(&TensorData::from([0.0]), false); +} + +#[test] +fn test_prod_dim_float() { + let tensor = TestTensor::<2>::from([[2.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let output = tensor.prod_dim(1); + let expected = TensorData::from([[4.0], [60.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let tensor_with_zero = TestTensor::<2>::from([[2.0, 0.0, 2.0], [3.0, 4.0, 5.0]]); + let output = tensor_with_zero.prod_dim(1); + let expected = TensorData::from([[0.0], [60.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_sum_dim_2d() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + let output = tensor.clone().sum_dim(1); + let expected = TensorData::from([[3.], [12.]]); + + output.into_data().assert_eq(&expected, false); + + let output = tensor.sum_dim(0); + let expected = TensorData::from([[3., 5., 7.]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dims_2d() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + tensor + .clone() + .sum_dims(&[1]) + .to_data() + .assert_eq(&TensorData::from([[3.], [12.]]), false); + + tensor + .clone() + .sum_dims(&[-1]) + .to_data() + .assert_eq(&TensorData::from([[3.], [12.]]), false); + + tensor + .clone() + .sum_dims(&[0, 1]) + .to_data() + .assert_eq(&TensorData::from([[15.]]), false); +} + +#[test] +fn test_sum_and_squeeze_dims() { + let tensor = TestTensor::<3>::from_floats( + [ + [[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], + [[9.0, 2.0, 5.0], [5.0, 7.0, 7.0]], + ], + &Default::default(), + ); + + tensor + .sum_dims_squeeze::<1, _>(&[0, 1]) + .to_data() + .assert_eq(&TensorData::from([20., 16., 21.]), false); +} + +#[test] +fn test_sum_dim_1_reshape_maybe_fused() { + let tensor = TestTensorInt::arange(0..9, &Default::default()).float(); + TestBackend::sync(&tensor.device()).unwrap(); + + let output = tensor.reshape([3, 3]) + 2; + let output = output.sum_dim(1); + let expected = TensorData::from([[9.0], [18.0], [27.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_1_swap_dims_maybe_fused() { + let tensor = TestTensorInt::arange(0..9, &Default::default()).float(); + let tensor = tensor.reshape([3, 3]); + TestBackend::sync(&tensor.device()).unwrap(); + + let output = tensor.swap_dims(0, 1) + 2; + let output = output.sum_dim(1); + let expected = TensorData::from([[15.0], [18.0], [21.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_2_reshape_maybe_fused_broadcast() { + let tensor = TestTensorInt::arange(0..9, &Default::default()).float(); + TestBackend::sync(&tensor.device()).unwrap(); + + let output = tensor.reshape([1, 3, 3]) + 2; + let output = output.sum_dim(2); + let expected = TensorData::from([[[9.0], [18.0], [27.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_2_maybe_fused_on_write() { + let tensor_1 = TestTensorInt::arange(0..8, &Default::default()).float(); + let tensor_2 = TestTensorInt::arange(10..12, &Default::default()).float(); + let tensor_1 = tensor_1.reshape([1, 2, 4]); + let tensor_2 = tensor_2.reshape([1, 2, 1]); + TestBackend::sync(&tensor_1.device()).unwrap(); + + let output = (tensor_1 + tensor_2.clone()).sum_dim(2) + tensor_2; + TestBackend::sync(&output.device()).unwrap(); + let expected = TensorData::from([[[56.0], [77.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_3_maybe_fused_on_read_not_contiguous() { + let tensor_1 = TestTensorInt::arange(0..8, &Default::default()).float(); + let tensor_2 = TestTensorInt::arange(16..24, &Default::default()).float(); + + let tensor_1 = tensor_1.reshape([4, 2, 1]); + let tensor_1 = tensor_1.swap_dims(0, 2); + + let tensor_2 = tensor_2.reshape([1, 4, 2]); + let tensor_2 = tensor_2.swap_dims(1, 2); + TestBackend::sync(&tensor_1.device()).unwrap(); + + let output = (tensor_1 + tensor_2).sum_dim(2); + let expected = TensorData::from([[[88.0], [96.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_4_maybe_fused_on_read_not_contiguous_mixed() { + let tensor_1 = TestTensorInt::arange(0..8, &Default::default()).float(); + let tensor_2 = TestTensorInt::arange(16..24, &Default::default()).float(); + let tensor_3 = TestTensorInt::arange(32..40, &Default::default()).float(); + + let tensor_1 = tensor_1.reshape([4, 2, 1]); + let tensor_3 = tensor_3.reshape([1, 2, 4]); + let tensor_1 = tensor_1.swap_dims(0, 2); + + let tensor_2 = tensor_2.reshape([1, 4, 2]); + let tensor_2 = tensor_2.swap_dims(1, 2); + TestBackend::sync(&tensor_1.device()).unwrap(); + + let output = (tensor_3 + tensor_1 + tensor_2).sum_dim(2); + let expected = TensorData::from([[[222.0], [246.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_5_maybe_fused_on_read_not_contiguous_mixed() { + let tensor_1 = TestTensorInt::arange(0..8, &Default::default()).float(); + let tensor_2 = TestTensorInt::arange(16..24, &Default::default()).float(); + let tensor_3 = TestTensorInt::arange(32..40, &Default::default()).float(); + + let tensor_1 = tensor_1.reshape([4, 2, 1]); + let tensor_3 = tensor_3.reshape([1, 2, 4]); + let tensor_1 = tensor_1.swap_dims(0, 2); + + let tensor_2 = tensor_2.reshape([1, 4, 2]); + let tensor_2 = tensor_2.swap_dims(1, 2); + TestBackend::sync(&tensor_1.device()).unwrap(); + + let output = (tensor_3 + tensor_1 + tensor_2).sum_dim(1); + let expected = TensorData::from([[[102.0, 112.0, 122.0, 132.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_6_maybe_fused_on_read_not_contiguous_broadcasted() { + let tensor_1 = TestTensorInt::arange(0..32, &Default::default()).float(); + let tensor_2 = TestTensorInt::arange(0..8, &Default::default()).float(); + + let tensor_1 = tensor_1.reshape([4, 2, 2, 2]); + let tensor_1 = tensor_1.swap_dims(3, 2); + let tensor_1 = tensor_1.swap_dims(1, 2); + + let tensor_2 = tensor_2.reshape([1, 2, 2, 2]); + + TestBackend::sync(&tensor_1.device()).unwrap(); + let sum = tensor_2.clone().sum_dim(0); + let sum = sum.sum_dim(1); + let sum = sum.sum_dim(2); + + TestBackend::sync(&tensor_1.device()).unwrap(); + + let _tmp = sum.clone() + 2; + let output = (tensor_1 + tensor_2 + sum).sum_dim(1); + let expected = TensorData::from([ + [[[29.0, 43.0], [41.0, 55.0]]], + [[[45.0, 59.0], [57.0, 71.0]]], + [[[61.0, 75.0], [73.0, 87.0]]], + [[[77.0, 91.0], [89.0, 103.0]]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sum_dim_7_maybe_fused_on_read_reshaped() { + let tensor_1 = TestTensorInt::arange(0..16, &Default::default()).float(); + + let tensor_1 = tensor_1.reshape([4, 4]); + + TestBackend::sync(&tensor_1.device()).unwrap(); + + let reshaped = tensor_1.reshape([1, 4, 4]); + let tmp = reshaped + 5.0; + let output = tmp.sum_dim(2); + let expected = TensorData::from([[[26.0], [42.0], [58.0], [74.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_mean_dim_fused_on_read_on_write() { + // https://github.com/tracel-ai/burn/issues/3987 + let device = Default::default(); + let x = TestTensor::ones([128, 32, 1], &device); + + let weight = TestTensor::ones([1, 32, 1], &device); + let options = burn_tensor::ops::ConvOptions::new([1], [0], [1], 1); + let x = burn_tensor::module::conv1d(x, weight, None, options); + let global = x.clone().powi_scalar(2).sum_dim(2).add_scalar(1e-5).sqrt(); + let norm = global.clone().div(global.mean_dim(1)); + let x = x.clone().mul(norm).add(x); + + let out = x.sum(); + + out.into_data() + .assert_eq(&TensorData::from([8192.0]), false); +} + +#[test] +fn test_mean_dim_2d() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + let output = tensor.clone().mean_dim(1); + let expected = TensorData::from([[1.], [4.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let output = tensor.mean_dim(0); + let expected = TensorData::from([[1.5, 2.5, 3.5]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_mean_dims_2d() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + tensor + .clone() + .mean_dims(&[1]) + .to_data() + .assert_eq(&TensorData::from([[1.], [4.]]), false); + + tensor + .clone() + .mean_dims(&[-1]) + .to_data() + .assert_eq(&TensorData::from([[1.], [4.]]), false); + + tensor + .clone() + .mean_dims(&[0, 1]) + .to_data() + .assert_eq(&TensorData::from([[2.5]]), false); +} + +#[test] +fn test_multiple_reduce_dims_permuted() { + // Regression test for https://github.com/tracel-ai/burn/issues/4461 + let tensor = TestTensorInt::arange(0..2 * 2 * 256, &Default::default()) + .float() + .reshape([2, 2, 256]); + + let output = tensor + .permute([1, 2, 0]) + .mean_dim(0) + .mean_dim(1) + .squeeze_dims::<1>(&[0, 1]); + + output + .into_data() + .assert_approx_eq::(&TensorData::from([255.5, 767.5]), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/all.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/all.rs new file mode 100644 index 0000000..a71643a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/all.rs @@ -0,0 +1,18 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_all() { + let tensor = TestTensor::<2>::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_all_dim() { + let tensor = TestTensor::<2>::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]); + let data_actual = tensor.all_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/any.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/any.rs new file mode 100644 index 0000000..b4a7efa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/any.rs @@ -0,0 +1,58 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_any() { + // test float tensor + let tensor = TestTensor::<2>::from([[0.0, 0.0, 0.0], [1.0, -1.0, 0.0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensor::<2>::from([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); + + // test int tensor + let tensor = TestTensorInt::<2>::from([[0, 0, 0], [1, -1, 0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensorInt::<2>::from([[0, 0, 0], [0, 0, 0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); + + // test bool tensor + let tensor = TestTensorBool::<2>::from([[false, false, false], [true, true, false]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensorBool::<2>::from([[false, false, false], [false, false, false]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_any_dim() { + let tensor = TestTensor::<2>::from([[0.0, 0.0, 0.0], [1.0, -1.0, 0.0]]); + let data_actual = tensor.any_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); + + // test int tensor + let tensor = TestTensorInt::<2>::from([[0, 0, 0], [1, -1, 0]]); + let data_actual = tensor.any_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); + + // test bool tensor + let tensor = TestTensorBool::<2>::from([[false, false, false], [true, true, false]]); + let data_actual = tensor.any_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/arg.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/arg.rs new file mode 100644 index 0000000..d7d2e57 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/arg.rs @@ -0,0 +1,46 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_argmax_2d_dim0() { + let tensor = TestTensor::<2>::from([[10.0, 11.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.argmax(0); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 0, 1]]), false); +} + +#[test] +fn test_argmin_2d_dim0() { + let tensor = TestTensor::<2>::from([[10.0, 11.0, 2.0], [30.0, 4.0, 5.0]]); + + let output = tensor.argmin(0); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 0]]), false); +} + +#[test] +fn test_argmax_2d_dim1() { + let tensor = TestTensor::<2>::from([[10.0, 11.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.argmax(1); + + output + .into_data() + .assert_eq(&TensorData::from([[1], [2]]), false); +} + +#[test] +fn test_argmin_2d_dim1() { + let tensor = TestTensor::<2>::from([[10.0, 11.0, 2.0], [30.0, 4.0, 5.0]]); + + let output = tensor.argmin(1); + + output + .into_data() + .assert_eq(&TensorData::from([[2], [1]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cast.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cast.rs new file mode 100644 index 0000000..3e2b00c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cast.rs @@ -0,0 +1,51 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{DType, TensorData}; + +#[test] +fn cast_float_to_bool() { + let tensor1 = TestTensor::<2>::from([[0.0, 43.0, 0.0], [2.0, -4.2, 31.33]]); + let data_actual = tensor1.bool().into_data(); + let data_expected = TensorData::from([[false, true, false], [true, true, true]]); + data_actual.assert_eq(&data_expected, false); +} + +#[test] +fn cast_float_to_int() { + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.4, 5.5, 6.6]]).int(); + let expected = TensorData::from([[1, 2, 3], [4, 5, 6]]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn cast_int_to_float_tensor() { + let tensor = TestTensorInt::<2>::from([[1, 2, 3], [4, 5, 6]]).float(); + + let expected = TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn cast_bool_to_float_tensor() { + let tensor = TestTensorBool::<2>::from([[true, false, true], [false, false, true]]).float(); + + let expected = TensorData::from([[1., 0., 1.], [0., 0., 1.]]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn cast_float_precision() { + let data = TensorData::from([[1.0, 2.0, 3.0], [4.4, 5.5, 6.6]]); + let tensor = TestTensor::<2>::from(data.clone()); + + let output = tensor.cast(DType::F32); + + assert_eq!(output.dtype(), DType::F32); + // Use precision 2 for parameterized tests in f16 and bf16 + output + .into_data() + .assert_approx_eq::(&data, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cat.rs new file mode 100644 index 0000000..7c2d12b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cat.rs @@ -0,0 +1,148 @@ +use super::*; +use alloc::vec::Vec; +use burn_tensor::Tolerance; +use burn_tensor::{DType, TensorData}; + +#[test] +fn should_support_cat_ops_2d_dim0() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0]], &device); + let tensor_2 = TestTensor::from_data([[4.0, 5.0, 6.0]], &device); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_cat_ops_2d_dim1() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0]], &device); + let tensor_2 = TestTensor::from_data([[4.0, 5.0, 6.0]], &device); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 1); + let expected = TensorData::from([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_cat_ops_3d() { + let device = Default::default(); + let tensor_1 = TestTensor::<3>::from_data([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]]], &device); + let tensor_2 = TestTensor::from_data([[[4.0, 5.0, 6.0]]], &device); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]], [[4.0, 5.0, 6.0]]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[should_panic] +fn should_panic_when_dimensions_are_not_the_same() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0]], &device); + let tensor_2 = TestTensor::from_data([[4.0, 5.0]], &device); + + TestTensor::cat(vec![tensor_1, tensor_2], 0).into_data(); +} + +#[test] +#[should_panic] +fn should_panic_when_list_of_vectors_is_empty() { + let tensor: Vec> = vec![]; + TestTensor::cat(tensor, 0).into_data(); +} + +#[test] +#[should_panic] +fn should_panic_when_cat_exceeds_dimension() { + let device = Default::default(); + let tensor_1 = TestTensor::<3>::from_data([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]]], &device); + let tensor_2 = TestTensor::from_data([[[4.0, 5.0, 6.0]]], &device); + + TestTensor::cat(vec![tensor_1, tensor_2], 3).into_data(); +} + +#[test] +fn should_support_cat_ops_cast_dtype() { + let device = Default::default(); + // ok for f32 backends, casts dtype for f16 tests + let tensor_1 = TestTensor::<3>::from_data([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]]], &device) + .cast(DType::F32); + let tensor_2 = TestTensor::from_data([[[4.0, 5.0, 6.0]]], &device).cast(DType::F32); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]], [[4.0, 5.0, 6.0]]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_cat_with_empty_tensor() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0]], &device); + let tensor_2: TestTensor<2> = TestTensor::empty([1, 0], &device); // Empty tensor with size 0 on dim 1 + + // Concatenating with an empty tensor should just return the non-empty tensor + let output = TestTensor::cat(vec![tensor_1.clone(), tensor_2], 1); + let expected = TensorData::from([[1.0, 2.0, 3.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_cat_with_empty_tensor_first() { + let device = Default::default(); + let tensor_1: TestTensor<2> = TestTensor::empty([1, 0], &device); // Empty tensor + let tensor_2 = TestTensor::<2>::from_data([[4.0, 5.0, 6.0]], &device); + + // Empty tensor first, then non-empty + let output = TestTensor::cat(vec![tensor_1, tensor_2.clone()], 1); + let expected = TensorData::from([[4.0, 5.0, 6.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_cat_with_multiple_empty_tensors() { + let device = Default::default(); + let tensor_1: TestTensor<2> = TestTensor::empty([2, 0], &device); + let tensor_2 = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let tensor_3: TestTensor<2> = TestTensor::empty([2, 0], &device); + let tensor_4 = TestTensor::<2>::from_data([[5.0], [6.0]], &device); + + // Mix of empty and non-empty tensors + let output = TestTensor::cat(vec![tensor_1, tensor_2, tensor_3, tensor_4], 1); + let expected = TensorData::from([[1.0, 2.0, 5.0], [3.0, 4.0, 6.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_cat_all_empty_tensors() { + let device = Default::default(); + let tensor_1: TestTensor<2> = TestTensor::empty([2, 0], &device); + let tensor_2: TestTensor<2> = TestTensor::empty([2, 0], &device); + + // All empty tensors should produce an empty tensor + let output = TestTensor::cat(vec![tensor_1, tensor_2], 1); + + assert_eq!(output.shape().as_slice(), [2, 0]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/ceil.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/ceil.rs new file mode 100644 index 0000000..8164d3e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/ceil.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_ceil_ops() { + let data = TensorData::from([[24.0423, 87.9478, 76.1838], [59.6929, 43.8169, 94.8826]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.ceil(); + let expected = TensorData::from([[25., 88., 77.], [60., 44., 95.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/chunk.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/chunk.rs new file mode 100644 index 0000000..1c93ee6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/chunk.rs @@ -0,0 +1,85 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_chunk_evenly_divisible() { + let tensors = TestTensorInt::arange(0..12, &Default::default()) + .float() + .chunk(6, 0); + assert_eq!(tensors.len(), 6); + + let expected = [ + TensorData::from([0, 1]), + TensorData::from([2, 3]), + TensorData::from([4, 5]), + TensorData::from([6, 7]), + TensorData::from([8, 9]), + TensorData::from([10, 11]), + ]; + + for (index, tensor) in tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_chunk_not_evenly_divisible() { + let tensors = TestTensorInt::arange(0..11, &Default::default()) + .float() + .chunk(6, 0); + assert_eq!(tensors.len(), 6); + + let expected = [ + TensorData::from([0, 1]), + TensorData::from([2, 3]), + TensorData::from([4, 5]), + TensorData::from([6, 7]), + TensorData::from([8, 9]), + TensorData::from([10]), + ]; + + for (index, tensor) in tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_chunk_not_evenly_divisible_remains_several() { + let tensors = TestTensorInt::arange(0..100, &Default::default()) + .float() + .chunk(8, 0); + assert_eq!(tensors.len(), 8); + + let expected = [13, 13, 13, 13, 13, 13, 13, 9]; + + for (index, tensor) in tensors.iter().enumerate() { + assert_eq!(tensor.shape()[0], expected[index]); + } +} + +#[test] +fn test_chunk_not_divisible() { + let tensors = TestTensorInt::arange(0..6, &Default::default()) + .float() + .chunk(7, 0); + assert_eq!(tensors.len(), 6); + + let expected = [ + TensorData::from([0]), + TensorData::from([1]), + TensorData::from([2]), + TensorData::from([3]), + TensorData::from([4]), + TensorData::from([5]), + ]; + + for (index, tensor) in tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +#[should_panic] +fn test_invalid_dim() { + let _tensors = TestTensorInt::arange(0..12, &Default::default()).chunk(6, 1); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/clamp.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/clamp.rs new file mode 100644 index 0000000..9c15c00 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/clamp.rs @@ -0,0 +1,81 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn clamp_min() { + let device = Default::default(); + // test float tensor + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &device); + + let output = tensor.clamp_min(2.0); + + output + .into_data() + .assert_eq(&TensorData::from([[2.0, 2.0, 2.0], [3.0, 4.0, 5.0]]), false); + + // test int tensor + let data = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let tensor = TestTensorInt::<2>::from_data(data, &device); + let output = tensor.clamp_min(2); + + output + .into_data() + .assert_eq(&TensorData::from([[2, 2, 2], [3, 4, 5]]), false); +} + +#[test] +fn clamp_max() { + let device = Default::default(); + // test float tensor + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &device); + + let output = tensor.clamp_max(2.0); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 1.0, 2.0], [2.0, 2.0, 2.0]]), false); + + // test int tensor + let data = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let tensor = TestTensorInt::<2>::from_data(data, &device); + let output = tensor.clamp_max(4); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 2], [3, 4, 4]]), false); +} + +#[test] +fn clamp_min_max() { + let device = Default::default(); + // test float tensor + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &device); + let output = tensor.clamp(1.0, 4.0); + + output + .into_data() + .assert_eq(&TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 4.0]]), false); + + // test int tensor + let data = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let tensor = TestTensorInt::<2>::from_data(data, &device); + let output = tensor.clamp(1, 4); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 1, 2], [3, 4, 4]]), false); +} + +#[test] +fn clamp_min_max_vec_should_compile() { + let input = TestTensor::<2>::ones([2, 4], &Default::default()); + let output = input.clamp(0., 0.5); + + output.into_data().assert_eq( + &TensorData::from([[0.5, 0.5, 0.5, 0.5], [0.5, 0.5, 0.5, 0.5]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/close.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/close.rs new file mode 100644 index 0000000..1f177ef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/close.rs @@ -0,0 +1,40 @@ +use super::*; +use burn_tensor::{DEFAULT_ATOL, DEFAULT_RTOL, TensorData}; + +#[test] +fn test_is_close() { + let tensor1 = TestTensor::<2>::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]); + let tensor2 = TestTensor::from([[0.0, 1.0, 0.0], [1.0, -1.0, 3.0]]) + 1e-9; + + let data_actual = tensor1 + .clone() + .is_close(tensor2.clone(), None, None) + .into_data(); + let defaults_expected = TensorData::from([[true, true, true], [true, true, false]]); + defaults_expected.assert_eq(&data_actual, false); + + // Using the defaults. + let data_actual = tensor1 + .is_close(tensor2, Some(DEFAULT_RTOL), Some(DEFAULT_ATOL)) + .into_data(); + defaults_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_all_close() { + let tensor1 = TestTensor::<2>::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]); + let tensor2 = TestTensor::from([[0.0, 1.0, 0.0], [1.0, -1.0, 3.0]]) + 1e-9; + assert!(!tensor1.clone().all_close(tensor2.clone(), None, None)); + + let tensor2 = TestTensor::from([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]) + 1e-9; + assert!(tensor1.all_close(tensor2, None, None)); + + // non finite values + let inf_plus = TestTensor::<2>::from([[f32::INFINITY]]); + let one = TestTensor::<2>::from([[1.]]); + let inf_minus = TestTensor::<2>::from([[-f32::INFINITY]]); + assert!(!inf_plus.clone().all_close(inf_minus.clone(), None, None)); + assert!(!one.clone().all_close(inf_minus.clone(), None, None)); + assert!(!one.all_close(inf_plus.clone(), None, None)); + assert!(inf_plus.clone().all_close(inf_plus, None, None)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/comparison.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/comparison.rs new file mode 100644 index 0000000..4e91abe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/comparison.rs @@ -0,0 +1,303 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_equal_inf() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0], [f32::INFINITY, 4.0, f32::NEG_INFINITY]]); + let data_2 = TensorData::from([[1.0, 1.0, 1.0], [f32::INFINITY, 3.0, f32::NEG_INFINITY]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let data_actual_cloned = tensor_1.clone().equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.equal(tensor_2); + + let data_expected = TensorData::from([[false, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_not_equal_inf() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0], [3.0, f32::INFINITY, 5.0]]); + let data_2 = TensorData::from([[1.0, 1.0, 1.0], [f32::INFINITY, 3.0, f32::NEG_INFINITY]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let data_actual_cloned = tensor_1.clone().not_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.not_equal(tensor_2); + + let data_expected = TensorData::from([[true, false, true], [true, true, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_equal() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::<2>::from([[1.0, 1.0, 1.0], [4.0, 3.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.equal(tensor_2); + + let data_expected = TensorData::from([[false, true, false], [false, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_not_equal() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::<2>::from([[1.0, 1.0, 1.0], [4.0, 3.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().not_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.not_equal(tensor_2); + + let data_expected = TensorData::from([[true, false, true], [true, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_equal_elem() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 2.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().equal_elem(2); + let data_actual_inplace = tensor_1.equal_elem(2); + + let data_expected = TensorData::from([[false, false, true], [false, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_not_equal_elem() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 2.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().not_equal_elem(2); + let data_actual_inplace = tensor_1.not_equal_elem(2); + + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn greater_elem() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().greater_elem(4); + let data_actual_inplace = tensor_1.greater_elem(4); + + let data_expected = TensorData::from([[false, false, false], [false, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater_equal_elem() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().greater_equal_elem(4.0); + let data_actual_inplace = tensor_1.greater_equal_elem(4.0); + + let data_expected = TensorData::from([[false, false, false], [false, true, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::<2>::from([[1.0, 1.0, 1.0], [4.0, 3.0, 50.0]]); + + let data_actual_cloned = tensor_1.clone().greater(tensor_2.clone()); + let data_actual_inplace = tensor_1.greater(tensor_2); + + let data_expected = TensorData::from([[false, false, true], [false, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater_equal() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::<2>::from([[1.0, 1.0, 1.0], [4.0, 3.0, 50.0]]); + + let data_actual_cloned = tensor_1.clone().greater_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.greater_equal(tensor_2); + + let data_expected = TensorData::from([[false, true, true], [false, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower_elem() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().lower_elem(4.0); + let data_actual_inplace = tensor_1.lower_elem(4.0); + + let data_expected = TensorData::from([[true, true, true], [true, false, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower_equal_elem() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor_1.clone().lower_equal_elem(4.0); + let data_actual_inplace = tensor_1.lower_equal_elem(4.0); + + let data_expected = TensorData::from([[true, true, true], [true, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::<2>::from([[1.0, 1.0, 1.0], [4.0, 3.0, 50.0]]); + + let data_actual_cloned = tensor_1.clone().lower(tensor_2.clone()); + let data_actual_inplace = tensor_1.lower(tensor_2); + + let data_expected = TensorData::from([[true, false, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower_equal() { + let tensor_1 = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = TestTensor::<2>::from([[1.0, 1.0, 1.0], [4.0, 3.0, 50.0]]); + + let data_actual_cloned = tensor_1.clone().lower_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.lower_equal(tensor_2); + + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater_broadcast() { + // Test broadcasting with shape [1, 4] vs [4, 4] + let device = Default::default(); + let data_1 = TensorData::from([[1.0, 2.0, 3.0, 4.0]]); + let data_2 = TensorData::from([ + [0.5, 1.5, 2.5, 3.5], + [1.5, 2.5, 3.5, 4.5], + [2.5, 3.5, 4.5, 5.5], + [3.5, 4.5, 5.5, 6.5], + ]); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let result = tensor_1.greater(tensor_2); + + let expected = TensorData::from([ + [true, true, true, true], + [false, false, false, false], + [false, false, false, false], + [false, false, false, false], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_greater_equal_broadcast() { + // Test broadcasting with shape [4, 1] vs [1, 4] + let device = Default::default(); + let data_1 = TensorData::from([[1.0], [2.0], [3.0], [4.0]]); + let data_2 = TensorData::from([[1.0, 2.0, 3.0, 4.0]]); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let result = tensor_1.greater_equal(tensor_2); + + let expected = TensorData::from([ + [true, false, false, false], + [true, true, false, false], + [true, true, true, false], + [true, true, true, true], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_lower_broadcast() { + // Test broadcasting mimicking CLIP pattern: [1, 5] vs [5, 1] + let device = Default::default(); + let data_1 = TensorData::from([[0.0, 1.0, -1.0, 2.0, -2.0]]); + let data_2 = TensorData::from([[0.5], [1.5], [-0.5], [-1.5], [2.5]]); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let result = tensor_1.lower(tensor_2); + + let expected = TensorData::from([ + [true, false, true, false, true], + [true, true, true, false, true], + [false, false, true, false, true], + [false, false, false, false, true], + [true, true, true, true, true], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_lower_equal_broadcast() { + // Test broadcasting with shape [1, 1] vs [2, 4] + let device = Default::default(); + let data_1 = TensorData::from([[2.5]]); + let data_2 = TensorData::from([[1.0, 2.0, 3.0, 4.0], [2.0, 2.5, 3.0, 3.5]]); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let result = tensor_1.lower_equal(tensor_2); + + let expected = TensorData::from([[false, false, true, true], [false, true, true, true]]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_equal_broadcast() { + // Test broadcasting with different ranks + let device = Default::default(); + let data_1 = TensorData::from([[2.0], [3.0], [4.0]]); + let data_2 = TensorData::from([[2.0, 3.0, 4.0, 2.0]]); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let result = tensor_1.equal(tensor_2); + + let expected = TensorData::from([ + [true, false, false, true], + [false, true, false, false], + [false, false, true, false], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_not_equal_broadcast() { + // Test broadcasting with shape [3, 1] vs [1, 3] + let device = Default::default(); + let data_1 = TensorData::from([[1.0], [2.0], [3.0]]); + let data_2 = TensorData::from([[1.0, 2.0, 3.0]]); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let result = tensor_1.not_equal(tensor_2); + + let expected = TensorData::from([ + [false, true, true], + [true, false, true], + [true, true, false], + ]); + expected.assert_eq(&result.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/create_like.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/create_like.rs new file mode 100644 index 0000000..dad9926 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/create_like.rs @@ -0,0 +1,57 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Distribution, TensorData}; + +#[test] +fn should_support_zeros_like() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let tensor = tensor.zeros_like(); + let expected = TensorData::from([[[0., 0., 0.], [0., 0., 0.]], [[0., 0., 0.], [0., 0., 0.]]]); + + tensor + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_ones_like() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let tensor = tensor.ones_like(); + let expected = TensorData::from([[[1., 1., 1.], [1., 1., 1.]], [[1., 1., 1.], [1., 1., 1.]]]); + + tensor + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_randoms_like() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let tensor = tensor.random_like(Distribution::Uniform(0.99999, 1.)); + let expected = TensorData::from([[[1., 1., 1.], [1., 1., 1.]], [[1., 1., 1.], [1., 1., 1.]]]); + + tensor + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cross.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cross.rs new file mode 100644 index 0000000..a0bc5cb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cross.rs @@ -0,0 +1,101 @@ +use super::*; +use burn_tensor::TensorData; + +#[cfg(feature = "std")] +use burn_backend_tests::might_panic; + +#[test] +fn test_cross_3d_last_dim() { + let tensor_1 = TestTensor::<2>::from([[1.0, 3.0, -5.0], [2.0, -1.0, 4.0]]); + let tensor_2 = TestTensor::from([[4.0, -2.0, 1.0], [3.0, 5.0, -2.0]]); + + let output = tensor_1.cross(tensor_2, -1); + + output.into_data().assert_eq( + &TensorData::from([[-7.0, -21.0, -14.0], [-18.0, 16.0, 13.0]]), + false, + ); +} + +#[test] +fn test_cross_3d_non_contiguous_last_dim() { + let tensor_1 = TestTensor::<2>::from([[1.0, 3.0, -5.0], [2.0, -1.0, 4.0]]); + let tensor_2 = TestTensor::from([[4.0, 3.0], [-2.0, 5.0], [1.0, -2.0]]); + + let output = tensor_1.cross(tensor_2.permute([1, 0]), -1); + + output.into_data().assert_eq( + &TensorData::from([[-7.0, -21.0, -14.0], [-18.0, 16.0, 13.0]]), + false, + ); +} + +#[cfg(feature = "std")] +#[might_panic(reason = "not implemented: Cross product on non-last dimension")] +#[test] +fn test_cross_3d_dim0() { + let tensor_1 = TestTensor::<2>::from([[1.0, 0.0], [0.0, 1.0], [0.0, 0.0]]); + let tensor_2 = TestTensor::from([[0.0, 1.0], [0.0, 0.0], [1.0, 0.0]]); + + let output = tensor_1.cross(tensor_2, 0); + + output.into_data().assert_eq( + &TensorData::from([[0.0, 0.0], [-1.0, 0.0], [0.0, -1.0]]), + false, + ); +} + +#[test] +fn test_cross_3d_broadcast() { + let tensor_1 = TestTensor::<2>::from([[1.0, 3.0, -5.0]]); + let tensor_2 = TestTensor::from([[4.0, -2.0, 1.0], [3.0, 5.0, -2.0]]); + + let output = tensor_1.cross(tensor_2, -1); + + output.into_data().assert_eq( + &TensorData::from([[-7.0, -21.0, -14.0], [19.0, -13.0, -4.0]]), + false, + ); +} + +#[test] +fn test_cross_4d_last_dim() { + let tensor_1 = TestTensor::<3>::from([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]]); + let tensor_2 = TestTensor::from([[[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]]); + + let output = tensor_1.cross(tensor_2, -1); + + output.into_data().assert_eq( + &TensorData::from([[[0.0, 0.0, 1.0], [1.0, 0.0, 0.0]]]), + false, + ); +} + +// Helper to compute expected cross product for 2-D (N × 3) tensors. +fn manual_cross(a: &[[f32; 3]], b: &[[f32; 3]]) -> Vec<[f32; 3]> { + a.iter() + .zip(b.iter()) + .map(|(x, y)| { + [ + x[1] * y[2] - x[2] * y[1], + x[2] * y[0] - x[0] * y[2], + x[0] * y[1] - x[1] * y[0], + ] + }) + .collect() +} + +#[test] +fn forward_matches_manual_cross() { + let a_raw = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]; + let b_raw = [[7.0, 8.0, 9.0], [1.0, 0.0, -1.0]]; + let a = TestTensor::<2>::from(a_raw); + let b = TestTensor::<2>::from(b_raw); + + let out = a.cross(b.clone(), 1); + let expected_vec = manual_cross(&a_raw, &b_raw); + let expected: [[f32; 3]; 2] = [expected_vec[0], expected_vec[1]]; + + out.into_data() + .assert_eq(&TensorData::from(expected), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cumulative.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cumulative.rs new file mode 100644 index 0000000..69017df --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/cumulative.rs @@ -0,0 +1,152 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_cumsum_float_dim_0() { + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + let output = tensor.cumsum(0); + + output + .into_data() + .assert_eq(&TensorData::from([[1.0, 2.0, 3.0], [5.0, 7.0, 9.0]]), false); +} + +#[test] +fn test_cumsum_float_dim_1() { + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + let output = tensor.cumsum(1); + + output.into_data().assert_eq( + &TensorData::from([[1.0, 3.0, 6.0], [4.0, 9.0, 15.0]]), + false, + ); +} + +#[test] +fn test_cumsum_non_contiguous() { + let tensor = TestTensor::<2>::from([[1., 2.], [3., 4.]]).swap_dims(0, 1); + + let output = tensor.cumsum(1); + + output + .into_data() + .assert_eq(&TensorData::from([[1., 4.], [2., 6.]]), false); +} + +#[test] +fn test_cumsum_float_3d() { + let tensor = TestTensor::<3>::from([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]); + + let output = tensor.cumsum(2); + + output.into_data().assert_eq( + &TensorData::from([[[1.0, 3.0], [3.0, 7.0]], [[5.0, 11.0], [7.0, 15.0]]]), + false, + ); +} + +#[test] +fn test_cumprod_float_dim_0() { + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + let output = tensor.cumprod(0); + + output.into_data().assert_eq( + &TensorData::from([[1.0, 2.0, 3.0], [4.0, 10.0, 18.0]]), + false, + ); +} + +#[test] +fn test_cumprod_float_dim_1() { + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + let output = tensor.cumprod(1); + + output.into_data().assert_eq( + &TensorData::from([[1.0, 2.0, 6.0], [4.0, 20.0, 120.0]]), + false, + ); +} + +#[test] +fn test_cumprod_float_3d() { + let tensor = TestTensor::<3>::from([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]); + + let output = tensor.cumprod(2); + + output.into_data().assert_eq( + &TensorData::from([[[1.0, 2.0], [3.0, 12.0]], [[5.0, 30.0], [7.0, 56.0]]]), + false, + ); +} + +#[test] +fn test_cummin_float_dim_0() { + let tensor = TestTensor::<2>::from([[3.0, 1.0, 4.0], [2.0, 5.0, 1.0]]); + + let output = tensor.cummin(0); + + output + .into_data() + .assert_eq(&TensorData::from([[3.0, 1.0, 4.0], [2.0, 1.0, 1.0]]), false); +} + +#[test] +fn test_cummin_float_dim_1() { + let tensor = TestTensor::<2>::from([[3.0, 1.0, 4.0], [2.0, 5.0, 1.0]]); + + let output = tensor.cummin(1); + + output + .into_data() + .assert_eq(&TensorData::from([[3.0, 1.0, 1.0], [2.0, 2.0, 1.0]]), false); +} + +#[test] +fn test_cummin_float_3d() { + let tensor = TestTensor::<3>::from([[[4.0, 2.0], [3.0, 1.0]], [[5.0, 6.0], [7.0, 8.0]]]); + + let output = tensor.cummin(2); + + output.into_data().assert_eq( + &TensorData::from([[[4.0, 2.0], [3.0, 1.0]], [[5.0, 5.0], [7.0, 7.0]]]), + false, + ); +} + +#[test] +fn test_cummax_float_dim_0() { + let tensor = TestTensor::<2>::from([[3.0, 1.0, 4.0], [1.0, 5.0, 2.0]]); + + let output = tensor.cummax(0); + + output + .into_data() + .assert_eq(&TensorData::from([[3.0, 1.0, 4.0], [3.0, 5.0, 4.0]]), false); +} + +#[test] +fn test_cummax_float_dim_1() { + let tensor = TestTensor::<2>::from([[3.0, 1.0, 4.0], [1.0, 5.0, 2.0]]); + + let output = tensor.cummax(1); + + output + .into_data() + .assert_eq(&TensorData::from([[3.0, 3.0, 4.0], [1.0, 5.0, 5.0]]), false); +} + +#[test] +fn test_cummax_float_3d() { + let tensor = TestTensor::<3>::from([[[1.0, 3.0], [2.0, 4.0]], [[5.0, 2.0], [6.0, 1.0]]]); + + let output = tensor.cummax(2); + + output.into_data().assert_eq( + &TensorData::from([[[1.0, 3.0], [2.0, 4.0]], [[5.0, 5.0], [6.0, 6.0]]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/div.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/div.rs new file mode 100644 index 0000000..c19e6aa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/div.rs @@ -0,0 +1,49 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_div_ops() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data_2 = TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let output = tensor_1 / tensor_2; + let expected = TensorData::from([[0.0, 1.0, 1.0], [1.0, 1.0, 1.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_div_broadcast() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0]]); + let data_2 = TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let output = tensor_1 / tensor_2; + + output.into_data().assert_eq( + &TensorData::from([[0.0, 1.0, 1.0], [0.0, 0.25, 0.4]]), + false, + ); +} + +#[test] +fn should_support_div_scalar_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let scalar = 2.0; + let device = Default::default(); + let tensor = TestTensor::<2>::from_data(data, &device); + + let output = tensor / scalar; + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 0.5, 1.0], [1.5, 2.0, 2.5]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/dot.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/dot.rs new file mode 100644 index 0000000..33c03e5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/dot.rs @@ -0,0 +1,35 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_float() { + let device = Default::default(); + let tensor_1 = TestTensor::<1>::from_data([1.0, 2.0, 3.0], &device); + let tensor_2 = TestTensor::<1>::from_data([0.0, -1.0, 4.0], &device); + + let output = tensor_1.dot(tensor_2); + let expected = TensorData::from([10.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int() { + let device = Default::default(); + let tensor_1 = TestTensor::<1>::from_data([1, 2, 3], &device); + let tensor_2 = TestTensor::<1>::from_data([0, -1, 4], &device); + + let output = tensor_1.dot(tensor_2); + let expected = TensorData::from([10]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn test_panics_for_different_sizes() { + let device = Default::default(); + let tensor_1 = TestTensor::<1>::from_data([1, 2], &device); + let tensor_2 = TestTensor::<1>::from_data([1, 2, 3], &device); + let _output = tensor_1.dot(tensor_2); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/erf.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/erf.rs new file mode 100644 index 0000000..76ab84e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/erf.rs @@ -0,0 +1,34 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_erf_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.erf(); + let expected = TensorData::from([[0.0000, 0.8427, 0.99532], [0.99998, 1.0000, 1.0000]]); + + output.into_data().assert_approx_eq::( + &expected, + Tolerance::default().set_half_precision_absolute(2e-3), + ); +} + +#[test] +fn should_support_erf_ops_with_negative_number() { + let data = TensorData::from([[-0.056, -0.043, -0.089], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.erf(); + let expected = TensorData::from([ + [-0.06312324, -0.048490416, -0.10016122], + [0.99998, 1.0000, 1.0000], + ]); + + output.into_data().assert_approx_eq::( + &expected, + Tolerance::default().set_half_precision_absolute(3e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/exp.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/exp.rs new file mode 100644 index 0000000..3ece101 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/exp.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_exp_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.exp(); + let expected = TensorData::from([[1.0, 2.71830, 7.3891], [20.0855, 54.5981, 148.4132]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/expand.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/expand.rs new file mode 100644 index 0000000..caa8d9c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/expand.rs @@ -0,0 +1,80 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn expand_2d() { + let tensor = TestTensor::<1>::from_floats([1.0, 2.0, 3.0], &Default::default()); + let output = tensor.expand([3, 3]); + + output.into_data().assert_eq( + &TensorData::from([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [1.0, 2.0, 3.0]]), + false, + ); + + let tensor = TestTensor::<1>::from_floats([4.0, 7.0, 2.0, 3.0], &Default::default()); + let output = tensor.expand([2, 4]); + + output.into_data().assert_eq( + &TensorData::from([[4.0, 7.0, 2.0, 3.0], [4.0, 7.0, 2.0, 3.0]]), + false, + ); +} + +#[test] +fn expand_3d() { + let tensor = TestTensor::<2>::from_floats([[1.0, 2.0], [3.0, 4.0]], &Default::default()); + let output = tensor.expand([3, 2, 2]); + let expected = TensorData::from([ + [[1.0, 2.0], [3.0, 4.0]], + [[1.0, 2.0], [3.0, 4.0]], + [[1.0, 2.0], [3.0, 4.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn expand_higher_dimensions() { + let tensor = TestTensor::<2>::from_floats([[1.0, 2.0, 3.0, 4.0]], &Default::default()); + let output = tensor.expand([2, 3, 4]); + let expected = TensorData::from([ + [ + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + ], + [ + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn expand_sum_3d() { + let tensor = TestTensor::<2>::from_floats([[1.0, 2.0], [3.0, 4.0]], &Default::default()); + let output = tensor.expand([3, 2, 2]).sum_dim(0); + let expected = TensorData::from([[[3.0, 6.0], [9.0, 12.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn broadcast_single() { + let tensor = TestTensor::<1>::from_floats([1.0], &Default::default()); + let output = tensor.expand([2, 3]); + + output + .into_data() + .assert_eq(&TensorData::from([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]), false); +} + +#[test] +#[should_panic] +fn should_fail_expand_incompatible_shapes() { + let tensor = TestTensor::<1>::from_floats([1.0, 2.0, 3.0], &Default::default()); + let _expanded_tensor = tensor.expand([2, 2]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/finite.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/finite.rs new file mode 100644 index 0000000..eb21c9a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/finite.rs @@ -0,0 +1,23 @@ +use super::*; + +#[test] +fn is_finite() { + let all_finite = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let all_finite_expected = TestTensorBool::<2>::from([[true, true, true], [true, true, true]]); + + let with_inf_nan = TestTensor::<2>::from([ + [0.0, f32::INFINITY, f32::NAN], + [f32::NEG_INFINITY, f32::NAN, 5.0], + ]); + let with_inf_nan_expected = + TestTensorBool::<2>::from([[true, false, false], [false, false, true]]); + + all_finite_expected + .into_data() + .assert_eq(&all_finite.is_finite().into_data(), false); + + with_inf_nan + .is_finite() + .into_data() + .assert_eq(&with_inf_nan_expected.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flatten.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flatten.rs new file mode 100644 index 0000000..f34a96c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flatten.rs @@ -0,0 +1,64 @@ +use super::*; +use burn_tensor::Shape; + +/// Test if the function can successfully flatten a 4D tensor to a 1D tensor. +#[test] +fn should_flatten_to_1d() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let flattened_tensor: TestTensor<1> = tensor.flatten(0, 3); + let expected_shape = Shape::new([120]); + assert_eq!(flattened_tensor.shape(), expected_shape); +} + +/// Test if the function can successfully flatten the middle dimensions of a 4D tensor. +#[test] +fn should_flatten_middle() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let flattened_tensor: TestTensor<3> = tensor.flatten(1, 2); + let expected_shape = Shape::new([2, 12, 5]); + assert_eq!(flattened_tensor.shape(), expected_shape); +} + +/// Test if the function can successfully flatten the first dimensions of a 4D tensor. +#[test] +fn should_flatten_begin() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let flattened_tensor: TestTensor<2> = tensor.flatten(0, 2); + let expected_shape = Shape::new([24, 5]); + assert_eq!(flattened_tensor.shape(), expected_shape); +} + +/// Test if the function can successfully flatten the last dimensions of a 4D tensor. +#[test] +fn should_flatten_end() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let flattened_tensor: TestTensor<2> = tensor.flatten(1, 3); + let expected_shape = Shape::new([2, 60]); + assert_eq!(flattened_tensor.shape(), expected_shape); +} + +/// Test if the function can flatten negative indices. +#[test] +fn should_flatten_end_negative_indices() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let flattened_tensor: TestTensor<2> = tensor.flatten(-3, -1); + let expected_shape = Shape::new([2, 60]); + assert_eq!(flattened_tensor.shape(), expected_shape); +} + +/// Test if the function panics when the start dimension is greater than the end dimension. +#[test] +#[should_panic] +fn should_flatten_panic() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let _flattened_tensor: TestTensor<2> = tensor.flatten(2, 0); +} + +#[test] +#[should_panic] +fn not_enough_destination_dimension() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 5, 15]), &Default::default()); + let flattened_tensor: TestTensor<1> = tensor.flatten(1, 2); + let expected_shape = Shape::new([75]); + assert_eq!(flattened_tensor.shape(), expected_shape); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flip.rs new file mode 100644 index 0000000..4448ea6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/flip.rs @@ -0,0 +1,48 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn flip_float() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .float(); + + let flipped = tensor.clone().flip([0, 2]); + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).flip((0, 2)).float() + let expected = TensorData::from([ + [ + [15., 14., 13., 12.], + [19., 18., 17., 16.], + [23., 22., 21., 20.], + ], + [[3., 2., 1., 0.], [7., 6., 5., 4.], [11., 10., 9., 8.]], + ]); + + flipped.into_data().assert_eq(&expected, false); + + // Test with no flip + let flipped = tensor.clone().flip([]); + tensor.into_data().assert_eq(&flipped.into_data(), false); +} + +#[test] +#[should_panic] +fn flip_duplicated_axes() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with a duplicated axis + let _ = tensor.clone().flip([0, 0, 1]); +} + +#[test] +#[should_panic] +fn flip_out_of_bound_axis() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with an out of bound axis + let _ = tensor.clone().flip([3, 0, 1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/floor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/floor.rs new file mode 100644 index 0000000..1c29e9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/floor.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_floor_ops() { + let data = TensorData::from([[24.0423, 87.9478, 76.1838], [59.6929, 43.8169, 94.8826]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.floor(); + let expected = TensorData::from([[24., 87., 76.], [59., 43., 94.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/fmod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/fmod.rs new file mode 100644 index 0000000..5962f31 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/fmod.rs @@ -0,0 +1,295 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{ElementConversion, TensorData}; + +#[allow(unused_imports)] // f16 +use num_traits::Float; + +#[test] +fn should_support_fmod_ops() { + let dividend = TensorData::from([[5.3, -5.3], [7.5, -7.5]]); + let divisor = TensorData::from([[2.0, 2.0], [3.0, 3.0]]); + + let dividend_tensor = TestTensor::<2>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<2>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([[1.3, -1.3], [1.5, -1.5]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_fmod_scalar() { + let data = TensorData::from([5.3, -5.3, 7.5, -7.5]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.fmod_scalar(2.0); + let expected = TensorData::from([1.3, -1.3, 1.5, -1.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_positive_dividend_positive_divisor() { + let dividend = TensorData::from([10.0, 7.5, 3.8, 1.2]); + let divisor = TensorData::from([3.0, 2.0, 1.5, 0.7]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([1.0, 1.5, 0.8, 0.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_negative_dividend() { + let dividend = TensorData::from([-10.0, -7.5, -3.8, -1.2]); + let divisor = TensorData::from([3.0, 2.0, 1.5, 0.7]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([-1.0, -1.5, -0.8, -0.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_mixed_signs() { + let dividend = TensorData::from([5.3, -5.3, 5.3, -5.3]); + let divisor = TensorData::from([2.0, 2.0, -2.0, -2.0]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + // fmod result has same sign as dividend + let expected = TensorData::from([1.3, -1.3, 1.3, -1.3]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_infinity_dividend() { + // If x is ±∞ and y is not NaN, NaN is returned + let dividend = TensorData::from([ + f32::INFINITY, + f32::NEG_INFINITY, + f32::INFINITY, + f32::NEG_INFINITY, + ]); + let divisor = TensorData::from([2.0, 3.0, -2.0, -3.0]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let data = output.into_data(); + let values = data.as_slice::().unwrap(); + + // All results should be NaN + assert!(values[0].is_nan(), "fmod(inf, 2.0) should be NaN"); + assert!(values[1].is_nan(), "fmod(-inf, 3.0) should be NaN"); + assert!(values[2].is_nan(), "fmod(inf, -2.0) should be NaN"); + assert!(values[3].is_nan(), "fmod(-inf, -3.0) should be NaN"); +} + +#[test] +fn should_handle_zero_divisor() { + // If y is ±0 and x is not NaN, NaN should be returned + let dividend = TensorData::from([5.3, -5.3, 0.0, 1.0]); + let divisor = TensorData::from([0.0, -0.0, 0.0, -0.0]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let data = output.into_data(); + let values = data.as_slice::().unwrap(); + + // All results should be NaN + assert!(values[0].is_nan(), "fmod(5.3, 0.0) should be NaN"); + assert!(values[1].is_nan(), "fmod(-5.3, -0.0) should be NaN"); + assert!(values[2].is_nan(), "fmod(0.0, 0.0) should be NaN"); + assert!(values[3].is_nan(), "fmod(1.0, -0.0) should be NaN"); +} + +#[test] +fn should_handle_infinity_divisor() { + // If y is ±∞ and x is finite, x is returned + let dividend = TensorData::from([5.3, -5.3, 0.0, -0.0]); + let divisor = TensorData::from([ + f32::INFINITY, + f32::NEG_INFINITY, + f32::INFINITY, + f32::NEG_INFINITY, + ]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([5.3, -5.3, 0.0, -0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_nan_arguments() { + // If either argument is NaN, NaN is returned + let dividend = TensorData::from([f32::NAN, 5.3, f32::NAN, 0.0]); + let divisor = TensorData::from([2.0, f32::NAN, f32::NAN, 3.0]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let data = output.into_data(); + let values = data.as_slice::().unwrap(); + + assert!(values[0].is_nan(), "fmod(NaN, 2.0) should be NaN"); + assert!(values[1].is_nan(), "fmod(5.3, NaN) should be NaN"); + assert!(values[2].is_nan(), "fmod(NaN, NaN) should be NaN"); + assert!(!values[3].is_nan(), "fmod(0.0, 3.0) should be 0.0"); +} + +#[test] +fn should_handle_negative_zero() { + // If x is -0 and y is greater than zero, either +0 or -0 may be returned + let dividend = TensorData::from([-0.0_f32]); + let divisor = TensorData::from([2.0_f32]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let data = output.into_data(); + let values = data.as_slice::().unwrap(); + + // Result should be zero (either +0 or -0 is acceptable) + assert_eq!( + values[0], + 0.0f32.elem::(), + "fmod(-0, 2.0) should be zero" + ); +} + +#[test] +fn should_support_fmod_broadcasting_2d() { + // Test broadcasting: 1x2 with 3x2 + let dividend = TensorData::from([[5.3, -5.3]]); // Shape: 1x2 + let divisor = TensorData::from([[2.0, 2.0], [3.0, 3.0], [1.5, 1.5]]); // Shape: 3x2 + + let dividend_tensor = TestTensor::<2>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<2>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([ + [1.3, -1.3], // 5.3 % 2.0, -5.3 % 2.0 + [2.3, -2.3], // 5.3 % 3.0, -5.3 % 3.0 + [0.8, -0.8], // 5.3 % 1.5, -5.3 % 1.5 + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_fmod_broadcasting_3d() { + // Test broadcasting: 1x1x3 with 2x1x3 + let dividend = TensorData::from([[[5.0, -7.0, 8.0]]]); // Shape: 1x1x3 + let divisor = TensorData::from([[[3.0, 3.0, 3.0]], [[4.0, 4.0, 4.0]]]); // Shape: 2x1x3 + + let dividend_tensor = TestTensor::<3>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<3>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([ + [[2.0, -1.0, 2.0]], // 5.0 % 3.0, -7.0 % 3.0, 8.0 % 3.0 + [[1.0, -3.0, 0.0]], // 5.0 % 4.0, -7.0 % 4.0, 8.0 % 4.0 + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_fmod_scalar_broadcasting() { + // Test scalar operation with different shapes + let data = TensorData::from([[5.3, -5.3, 7.5], [-7.5, 10.0, -10.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.fmod_scalar(3.0); + let expected = TensorData::from([[2.3, -2.3, 1.5], [-1.5, 1.0, -1.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_edge_case_values() { + // Test various edge cases + let dividend = TensorData::from([0.0, -0.0, 1e-10, -1e-10, 10.0, -10.0]); + let divisor = TensorData::from([1.0, 1.0, 1.0, 1.0, 3.0, 3.0]); + + let dividend_tensor = TestTensor::<1>::from_data(dividend, &Default::default()); + let divisor_tensor = TestTensor::<1>::from_data(divisor, &Default::default()); + + let output = dividend_tensor.fmod(divisor_tensor); + let expected = TensorData::from([0.0, -0.0, 1e-10, -1e-10, 1.0, -1.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_special_scalar_cases() { + // Test scalar operations with special values + let data = TensorData::from([5.3, -5.3, 0.0, -0.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + // Test with infinity divisor + let output_inf = tensor.clone().fmod_scalar(f32::INFINITY); + let expected_inf = TensorData::from([5.3, -5.3, 0.0, -0.0]); + output_inf + .into_data() + .assert_approx_eq::(&expected_inf, Tolerance::default()); + + // Test with very small divisor + // Doesn't work if the test divisor is subnormal + if FloatElem::MIN_POSITIVE > 1e-5f32.elem::() { + return; + } + + let output_small = tensor.clone().fmod_scalar(1e-5); + let data = output_small.into_data(); + let values = data.as_slice::().unwrap(); + + // let expected = TensorData::from([0.0, 0.0, 0.0, 0.0]); + + // Results should be very small remainders + assert!(values[0].abs() < 1e-5f32.elem::()); + assert!(values[1].abs() < 1e-5f32.elem::()); + assert_eq!(values[2], 0.0f32.elem::()); + assert_eq!(values[3], 0.0f32.elem::()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/full.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/full.rs new file mode 100644 index 0000000..87d824c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/full.rs @@ -0,0 +1,28 @@ +use super::*; +use burn_tensor::{DType, TensorData}; + +#[test] +fn test_data_full() { + let tensor = TensorData::full([2, 3], 2.0); + + tensor.assert_eq(&TensorData::from([[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]), false); +} + +#[test] +fn test_tensor_full() { + let device = Default::default(); + let tensor = TestTensor::<2>::full([2, 3], 2.1, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([[2.1, 2.1, 2.1], [2.1, 2.1, 2.1]]), false); +} + +#[test] +fn test_tensor_full_options() { + let tensor = TestTensor::<2>::full([2, 3], 2.1, (&Default::default(), DType::F32)); + assert_eq!(tensor.dtype(), DType::F32); + + tensor + .into_data() + .assert_eq(&TensorData::from([[2.1, 2.1, 2.1], [2.1, 2.1, 2.1]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/gather_scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/gather_scatter.rs new file mode 100644 index 0000000..aa9db44 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/gather_scatter.rs @@ -0,0 +1,173 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, TensorData}; + +#[test] +fn should_gather_1d_dim0() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_floats([0.0, 1.0, 2.0], &device); + let indices = TestTensorInt::from_ints([1, 1, 0, 1, 2], &device); + + let output = tensor.gather(0, indices); + + output + .into_data() + .assert_eq(&TensorData::from([1.0, 1.0, 0.0, 1.0, 2.0]), false); +} + +#[test] +fn should_gather_2d_dim0() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_ints([[0, 1, 0], [1, 0, 1]], &device); + + let output = tensor.gather(0, indices); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 4.0, 2.0], [3.0, 1.0, 5.0]]), false); +} + +#[test] +fn should_gather_2d_dim1() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_ints([[2, 1, 0, 0], [2, 0, 1, 2]], &device); + + let output = tensor.gather(1, indices); + + output.into_data().assert_eq( + &TensorData::from([[2.0, 1.0, 0.0, 0.0], [5.0, 3.0, 4.0, 5.0]]), + false, + ); +} + +#[test] +fn should_gather_3d_dim1() { + let device = Default::default(); + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &device, + ); + let indices = + TestTensorInt::from_ints([[[1, 0, 0], [0, 1, 0]], [[0, 0, 1], [0, 1, 1]]], &device); + + let output = tensor.gather(1, indices); + let expected = TensorData::from([ + [[3.0, 1.0, 2.0], [0.0, 4.0, 2.0]], + [[6.0, 7.0, 11.0], [6.0, 10.0, 11.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_gather_2d_only_1dim() { + let device = Default::default(); + let tensor = TestTensor::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::<2>::from_ints([[1, 2]], &device).reshape([2, 1]); + + let output = tensor.gather(1, indices); + + output + .into_data() + .assert_eq(&TensorData::from([[1.0], [5.0]]), false); +} + +#[test] +fn should_scatter_add_1d() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_floats([0.0, 0.0, 0.0], &device); + let values = TestTensor::from_floats([5.0, 4.0, 3.0], &device); + let indices = TestTensorInt::from_ints([1, 0, 2], &device); + + let output = tensor.scatter(0, indices, values, IndexingUpdateOp::Add); + + output + .into_data() + .assert_eq(&TensorData::from([4.0, 5.0, 3.0]), false); +} + +#[test] +fn should_scatter_add_2d_dim0() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_floats([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], &device); + let values = TestTensor::from_floats([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let indices = TestTensorInt::from_ints([[1, 0, 1], [1, 1, 0]], &device); + + let output = tensor.scatter(0, indices, values, IndexingUpdateOp::Add); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 2.0, 6.0], [5.0, 5.0, 3.0]]), false); +} + +#[test] +fn should_scatter_add_2d_dim1() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_floats([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], &device); + let values = TestTensor::from_floats([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let indices = TestTensorInt::from_ints([[1, 0, 2], [1, 2, 0]], &device); + + let output = tensor.scatter(1, indices, values, IndexingUpdateOp::Add); + + output + .into_data() + .assert_eq(&TensorData::from([[2.0, 1.0, 3.0], [6.0, 4.0, 5.0]]), false); +} + +#[test] +fn should_scatter_add_3d_dim1() { + let device = Default::default(); + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &device, + ); + let values = TestTensor::from_floats( + [ + [[12.0, 13.0, 14.0], [15.0, 16.0, 17.0]], + [[18.0, 19.0, 20.0], [21.0, 22.0, 23.0]], + ], + &device, + ); + let indices = + TestTensorInt::from_ints([[[1, 0, 0], [0, 1, 0]], [[0, 0, 1], [0, 1, 1]]], &device); + + let output = tensor.scatter(1, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([ + [[15.0, 14.0, 33.0], [15.0, 20.0, 5.0]], + [[45.0, 26.0, 8.0], [9.0, 32.0, 54.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_scatter_add_2d_dim1_diff_shape() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_floats([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], &device); + let values = TestTensor::from_floats([[1.0], [4.0]], &device); + let indices = TestTensorInt::from_ints([[1], [2]], &device); + + let output = tensor.scatter(1, indices, values, IndexingUpdateOp::Add); + + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 1.0, 0.0], [0.0, 0.0, 4.0]]), false); +} + +#[test] +#[should_panic] +fn scatter_should_panic_on_mismatch_of_shapes() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_floats([0.0, 0.0, 0.0], &device); + let values = TestTensor::from_floats([5.0, 4.0], &device); + let indices = TestTensorInt::from_ints([1, 0, 2], &device); + + tensor.scatter(0, indices, values, IndexingUpdateOp::Add); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/grid_sample.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/grid_sample.rs new file mode 100644 index 0000000..79f71c7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/grid_sample.rs @@ -0,0 +1,126 @@ +use super::*; +use burn_tensor::{ + TensorData, Tolerance, + ops::{GridSampleOptions, GridSamplePaddingMode, InterpolateMode}, +}; + +/// Tests grid_sample_2d with default options (align_corners=false, zeros padding). +/// +/// For a 3x3 input with grid coordinates: +/// - (0.0, 0.0) maps to pixel (1.0, 1.0) -> center pixel = 4.0 +/// - (-1.0, 0.25) maps to pixel (-0.5, 1.375) -> partially out of bounds +/// - (1.0, 1.0) maps to pixel (2.5, 2.5) -> corner, partially out of bounds +/// - (0.2, -0.8) maps to pixel (1.3, 0.3) -> interpolates around center-top +#[test] +fn should_grid_sample_2d_default() { + let device = Default::default(); + let tensor = TestTensor::<4>::from_floats( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + &device, + ); + let grid = TestTensor::<4>::from_floats( + [[[[0.0, 0.0], [-1.0, 0.25]], [[1.0, 1.0], [0.2, -0.8]]]], + &device, + ); + + let output = tensor.grid_sample_2d(grid, InterpolateMode::Bilinear); + + // Expected values computed with PyTorch grid_sample(align_corners=False, padding_mode='zeros') + let expected = TensorData::from([[[[4.0, 2.0625], [2.0, 1.04]]]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +/// Tests grid_sample_2d with align_corners=true and border padding. +/// +/// This is the original Burn semantics before the API change. +#[test] +fn should_grid_sample_2d_align_corners_border() { + let device = Default::default(); + let tensor = TestTensor::<4>::from_floats( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + &device, + ); + let grid = TestTensor::<4>::from_floats( + [[[[0.0, 0.0], [-1.0, 0.25]], [[1.0, 1.0], [0.2, -0.8]]]], + &device, + ); + + let options = GridSampleOptions::new(InterpolateMode::Bilinear) + .with_padding_mode(GridSamplePaddingMode::Border) + .with_align_corners(true); + let output = tensor.grid_sample_2d(grid, options); + + // Expected values computed with PyTorch grid_sample(align_corners=True, padding_mode='border') + let expected = TensorData::from([[[[4.0, 3.75], [8.0, 1.8]]]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +/// Tests out-of-bounds grid coordinates with zeros padding. +/// Grid coordinate (0.0, -2.0) maps to pixel (1.0, -2.5) which is completely out of bounds. +#[test] +fn should_pad_zeros_grid_sample_2d() { + let device = Default::default(); + let tensor = TestTensor::<4>::from_floats( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + &device, + ); + let grid = TestTensor::<4>::from_floats([[[[0.0, -2.0]]]], &device); + + let output = tensor.grid_sample_2d(grid, GridSampleOptions::default()); + + // With zeros padding, out-of-bounds samples return 0 + let expected = TensorData::from([[[[0.0]]]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +/// Tests out-of-bounds grid coordinates with border padding. +#[test] +fn should_pad_border_grid_sample_2d() { + let device = Default::default(); + let tensor = TestTensor::<4>::from_floats( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + &device, + ); + let grid = TestTensor::<4>::from_floats([[[[0.0, -2.0]]]], &device); + + let options = GridSampleOptions::new(InterpolateMode::Bilinear) + .with_padding_mode(GridSamplePaddingMode::Border); + let output = tensor.grid_sample_2d(grid, options); + + // With border padding, out-of-bounds coordinates are clamped to border + // Grid (0.0, -2.0) with align_corners=false: pixel (1.0, -2.5) -> clamped to (1.0, 0.0) = 1.0 + let expected = TensorData::from([[[[1.0]]]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +/// Tests bilinear interpolation with reflection padding. +#[test] +fn should_pad_reflection_grid_sample_2d() { + let device = Default::default(); + let tensor = TestTensor::<4>::from_floats( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + &device, + ); + let grid = TestTensor::<4>::from_floats( + [[[[0.0, 0.0], [-1.0, 0.25]], [[1.0, 1.0], [0.2, -0.8]]]], + &device, + ); + + let options = GridSampleOptions::new(InterpolateMode::Bilinear) + .with_padding_mode(GridSamplePaddingMode::Reflection); + let output = tensor.grid_sample_2d(grid, options); + + // Expected values computed with PyTorch F.grid_sample(mode='bilinear', padding_mode='reflection', align_corners=False) + let expected = TensorData::from([[[[4.0, 4.125], [8.0, 1.3]]]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/inf.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/inf.rs new file mode 100644 index 0000000..f0c6cef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/inf.rs @@ -0,0 +1,21 @@ +use super::*; + +#[test] +fn is_inf() { + let no_inf = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let no_inf_expected = TestTensorBool::<2>::from([[false, false, false], [false, false, false]]); + + let with_inf = + TestTensor::<2>::from([[0.0, f32::INFINITY, 2.0], [f32::NEG_INFINITY, 4.0, 5.0]]); + let with_inf_expected = TestTensorBool::<2>::from([[false, true, false], [true, false, false]]); + + no_inf + .is_inf() + .into_data() + .assert_eq(&no_inf_expected.into_data(), false); + + with_inf + .is_inf() + .into_data() + .assert_eq(&with_inf_expected.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/init.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/init.rs new file mode 100644 index 0000000..eaa7b58 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/init.rs @@ -0,0 +1,62 @@ +use super::*; +use burn_tensor::{DType, TensorData}; + +#[test] +fn should_support_float_empty() { + let shape = [2, 2]; + let tensor = TestTensor::<2>::empty(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()) +} + +#[test] +fn should_support_float_empty_options() { + let shape = [2, 2]; + let tensor = TestTensor::<2>::empty(shape, (&Default::default(), DType::F32)); + assert_eq!(tensor.shape(), shape.into()) +} + +#[test] +fn should_support_float_zeros() { + let shape = [2, 2]; + let tensor = TestTensor::<2>::zeros(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()); + + tensor + .into_data() + .assert_eq(&TensorData::from([[0., 0.], [0., 0.]]), false); +} + +#[test] +fn should_support_float_zeros_options() { + let shape = [2, 2]; + let tensor = TestTensor::<2>::zeros(shape, (&Default::default(), DType::F32)); + assert_eq!(tensor.shape(), shape.into()); + assert_eq!(tensor.dtype(), DType::F32); + + tensor + .into_data() + .assert_eq(&TensorData::from([[0., 0.], [0., 0.]]), false); +} + +#[test] +fn should_support_float_ones() { + let shape = [2, 2]; + let tensor = TestTensor::<2>::ones(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()); + + tensor + .into_data() + .assert_eq(&TensorData::from([[1., 1.], [1., 1.]]), false); +} + +#[test] +fn should_support_float_ones_options() { + let shape = [2, 2]; + let tensor = TestTensor::<2>::ones(shape, (&Default::default(), DType::F32)); + assert_eq!(tensor.shape(), shape.into()); + assert_eq!(tensor.dtype(), DType::F32); + + tensor + .into_data() + .assert_eq(&TensorData::from([[1., 1.], [1., 1.]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/iter_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/iter_dim.rs new file mode 100644 index 0000000..49897e5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/iter_dim.rs @@ -0,0 +1,246 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_1d_iter_last_item() { + let data = [1, 2, 3, 4]; + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_ints(data, &device); + tensor + .iter_dim(0) + .last() + .unwrap() + .into_data() + .assert_eq(&TensorData::from([4]), false); +} + +#[test] +#[should_panic] +fn test_too_high_dimension() { + TestTensor::<1>::zeros([10], &Default::default()).iter_dim(1); +} + +#[test] +fn test_transposed() { + let data = [ + [1., 2., 3., 1., 2.], + [4., 5., 6., 1., 2.], + [7., 8., 9., 1., 2.], + ]; + let tensor = TestTensor::<2>::from_floats(data, &Default::default()); + let lhs = tensor.clone().slice([1..2, 0..5]); + let rhs = tensor.transpose().iter_dim(1).nth(1).unwrap(); + assert_eq!( + lhs.into_data().as_slice::().unwrap(), + rhs.into_data().as_slice::().unwrap() + ); +} + +#[test] +fn test_2d_iter_dim() { + let tensor = + TestTensor::<2>::from_data([[3.0, 4.9, 2.0], [2.0, 1.9, 3.0]], &Default::default()); + + let mut iter = tensor.iter_dim(0); + + let iter1 = iter.next().unwrap(); + iter1 + .into_data() + .assert_eq(&TensorData::from([[3.0, 4.9, 2.0]]), false); + + let iter2 = iter.next().unwrap(); + iter2 + .into_data() + .assert_eq(&TensorData::from([[2.0, 1.9, 3.0]]), false); + + assert!(iter.next().is_none()); +} + +#[test] +fn test_2d_iter_dim1() { + let tensor = + TestTensor::<2>::from_data([[3.0, 4.9, 2.0], [2.0, 1.9, 3.0]], &Default::default()); + + let mut iter = tensor.iter_dim(1); + + let iter1 = iter.next().unwrap(); + iter1 + .into_data() + .assert_eq(&TensorData::from([[3.0], [2.0]]), false); + + let iter2 = iter.next().unwrap(); + iter2 + .into_data() + .assert_eq(&TensorData::from([[4.9], [1.9]]), false); + + let iter3 = iter.next().unwrap(); + iter3 + .into_data() + .assert_eq(&TensorData::from([[2.0], [3.0]]), false); + + assert!(iter.next().is_none()); +} + +#[test] +fn test_3d_iter_dim() { + let tensor = TestTensor::<3>::from([[ + [1., 2., 3., 1., 2.], + [4., 5., 6., 1., 2.], + [7., 8., 9., 1., 2.], + ]]); + + let mut iter = tensor.clone().iter_dim(0); + + let iter1 = iter.next().unwrap(); + iter1.into_data().assert_eq(&tensor.into_data(), true); + + assert!(iter.next().is_none()); +} + +#[test] +fn test_3d_iter_dim1() { + let tensor = TestTensor::<3>::from([[ + [1., 2., 3., 1., 2.], + [4., 5., 6., 1., 2.], + [7., 8., 9., 1., 2.], + ]]); + + let mut iter = tensor.iter_dim(1); + + let iter1 = iter.next().unwrap(); + iter1 + .into_data() + .assert_eq(&TensorData::from([[[1., 2., 3., 1., 2.]]]), false); + + let iter2 = iter.next().unwrap(); + iter2 + .into_data() + .assert_eq(&TensorData::from([[[4., 5., 6., 1., 2.]]]), false); + + let iter3 = iter.next().unwrap(); + iter3 + .into_data() + .assert_eq(&TensorData::from([[[7., 8., 9., 1., 2.]]]), false); + + assert!(iter.next().is_none()); +} + +#[test] +fn test_3d_iter_dim2() { + let tensor = TestTensor::<3>::from([[ + [1., 2., 3., 1., 2.], + [4., 5., 6., 1., 2.], + [7., 8., 9., 1., 2.], + ]]); + + let mut iter = tensor.iter_dim(2); + + let iter1 = iter.next().unwrap(); + iter1 + .into_data() + .assert_eq(&TensorData::from([[[1.], [4.], [7.]]]), false); + + let iter2 = iter.next().unwrap(); + iter2 + .into_data() + .assert_eq(&TensorData::from([[[2.], [5.], [8.]]]), false); + + let iter3 = iter.next().unwrap(); + iter3 + .into_data() + .assert_eq(&TensorData::from([[[3.], [6.], [9.]]]), false); + + let iter4 = iter.next().unwrap(); + iter4 + .into_data() + .assert_eq(&TensorData::from([[[1.], [1.], [1.]]]), false); + + let iter5 = iter.next().unwrap(); + iter5 + .into_data() + .assert_eq(&TensorData::from([[[2.], [2.], [2.]]]), false); + + assert!(iter.next().is_none()); +} + +#[test] +fn test_iteration_over_low_dim() { + let data = [[ + [1., 2., 3., 1., 2.], + [4., 5., 6., 1., 2.], + [7., 8., 9., 1., 2.], + ]]; + + let tensor = TestTensor::<3>::from_floats(data, &Default::default()); + + let lhs = tensor.iter_dim(2).nth(1).unwrap(); + let rhs = TestTensor::<1>::from([2., 5., 8.]); + assert_eq!( + lhs.into_data().as_slice::().unwrap(), + rhs.into_data().as_slice::().unwrap() + ); +} + +#[test] +fn test_iter_dim_double_end() { + let input = TestTensorInt::<1>::arange(0..(4 * 6 * 3), &Default::default()).reshape([4, 6, 3]); + let mut iter = input.iter_dim(1); + + let ele0 = TensorData::from([[[0, 1, 2]], [[18, 19, 20]], [[36, 37, 38]], [[54, 55, 56]]]); + let ele1 = TensorData::from([[[3, 4, 5]], [[21, 22, 23]], [[39, 40, 41]], [[57, 58, 59]]]); + let ele2 = TensorData::from([[[6, 7, 8]], [[24, 25, 26]], [[42, 43, 44]], [[60, 61, 62]]]); + let ele3 = TensorData::from([ + [[9, 10, 11]], + [[27, 28, 29]], + [[45, 46, 47]], + [[63, 64, 65]], + ]); + let ele4 = TensorData::from([ + [[12, 13, 14]], + [[30, 31, 32]], + [[48, 49, 50]], + [[66, 67, 68]], + ]); + let ele5 = TensorData::from([ + [[15, 16, 17]], + [[33, 34, 35]], + [[51, 52, 53]], + [[69, 70, 71]], + ]); + + iter.next().unwrap().into_data().assert_eq(&ele0, false); + iter.next_back() + .unwrap() + .into_data() + .assert_eq(&ele5, false); + iter.next_back() + .unwrap() + .into_data() + .assert_eq(&ele4, false); + iter.next().unwrap().into_data().assert_eq(&ele1, false); + iter.next().unwrap().into_data().assert_eq(&ele2, false); + iter.next().unwrap().into_data().assert_eq(&ele3, false); + assert!(iter.next().is_none()); + assert!(iter.next_back().is_none()); +} + +#[test] +fn test_iter_dim_single_element() { + let input = TestTensorInt::<1>::arange(0..(4 * 3), &Default::default()).reshape([4, 1, 3]); + + let mut iter = input.clone().iter_dim(1); + iter.next() + .unwrap() + .into_data() + .assert_eq(&input.clone().into_data(), false); + assert!(iter.next_back().is_none()); + assert!(iter.next().is_none()); + + let mut iter = input.clone().iter_dim(1); + iter.next_back() + .unwrap() + .into_data() + .assert_eq(&input.clone().into_data(), false); + assert!(iter.next().is_none()); + assert!(iter.next_back().is_none()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log.rs new file mode 100644 index 0000000..127e1bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log.rs @@ -0,0 +1,20 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_log_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.log(); + let expected = TensorData::from([ + [-f32::INFINITY, 0.0, core::f32::consts::LN_2], + [1.09861, 1.38629, 1.60944], + ]); + + output.into_data().assert_approx_eq::( + &expected, + Tolerance::default().set_half_precision_relative(1e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log1p.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log1p.rs new file mode 100644 index 0000000..289b264 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/log1p.rs @@ -0,0 +1,20 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_exp_log1p() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.log1p(); + let expected = TensorData::from([ + [0.0, core::f32::consts::LN_2, 1.09861], + [1.38629, 1.60944, 1.79176], + ]); + + output.into_data().assert_approx_eq::( + &expected, + Tolerance::default().set_half_precision_relative(1e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mask.rs new file mode 100644 index 0000000..63109cb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mask.rs @@ -0,0 +1,165 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_mask_fill_swap_dims() { + let device = Default::default(); + let tensor_1 = TestTensorInt::arange(0..16, &device).float(); + let tensor_1 = tensor_1.reshape([2, 2, 4]); + let tensor_1 = tensor_1.swap_dims(0, 2); + + let mask = tensor_1.clone().lower_equal_elem(5.0); + let output = tensor_1.clone().mask_fill(mask, -5.0); + + let expected = TensorData::from([ + [[-5.0, 8.0], [-5.0, 12.0]], + [[-5.0, 9.0], [-5.0, 13.0]], + [[-5.0, 10.0], [6.0, 14.0]], + [[-5.0, 11.0], [7.0, 15.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_mask_where_ops() { + let device = Default::default(); + let tensor = TestTensor::from_data([[1.0, 7.0], [2.0, 3.0]], &device); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + let value = TestTensor::<2>::from_data(TensorData::from([[1.8, 2.8], [3.8, 4.8]]), &device); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([[1.8, 7.0], [2.0, 4.8]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_mask_where_broadcast() { + let device = Default::default(); + // When broadcasted, the input [[2, 3], [4, 5]] is repeated 4 times + let tensor = TestTensorInt::<1>::arange(2..6, &device).reshape([1, 2, 2]); + let mask = TestTensorBool::<3>::from_bool( + TensorData::from([ + [[true, false], [false, true]], + [[false, true], [true, false]], + [[false, false], [false, false]], + [[true, true], [true, true]], + ]), + &device, + ); + let value = TestTensor::<3>::ones([4, 2, 2], &device); + + let output = tensor.float().mask_where(mask, value); + let expected = TensorData::from([ + [[1., 3.], [4., 1.]], + [[2., 1.], [1., 5.]], + [[2., 3.], [4., 5.]], + [[1., 1.], [1., 1.]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_mask_where_broadcast_value_small() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(2..4, &device).float(); + let mask = TestTensorBool::<1>::from_bool(TensorData::from([true, false]), &device); + let value = TestTensor::<1>::ones([1], &device); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([1., 3.]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_handle_mask_where_nans() { + let device = Default::default(); + let tensor = TestTensor::from_data( + [ + [f32::NAN, f32::NAN, f32::NAN], + [f32::NAN, f32::NAN, f32::NAN], + [f32::NAN, f32::NAN, f32::NAN], + ], + &device, + ); + let mask = TestTensorBool::<2>::from_bool( + TensorData::from([ + [true, true, true], + [true, true, false], + [false, false, false], + ]), + &device, + ); + let value = TestTensor::<2>::from_data( + TensorData::from([[0.9, 0.8, 0.7], [0.6, 0.5, 0.4], [0.3, 0.2, 0.1]]), + &device, + ); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([ + [0.9, 0.8, 0.7], + [0.6, 0.5, f32::NAN], + [f32::NAN, f32::NAN, f32::NAN], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_mask_fill_ops() { + let device = Default::default(); + let tensor = TestTensor::from_data([[1.0, 7.0], [2.0, 3.0]], &device); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + + let output = tensor.mask_fill(mask, 2.0); + let expected = TensorData::from([[2.0, 7.0], [2.0, 2.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_mask_fill_broadcasted() { + let device = Default::default(); + let tensor = TestTensor::zeros([1, 4, 2, 2], &device); + let mask = TestTensorBool::<4>::from_bool( + TensorData::from([[[[true, false], [false, true]]]]), + &device, + ); + + let output = tensor.mask_fill(mask, 2.0); + let expected = TensorData::from([[ + [[2., 0.], [0., 2.]], + [[2., 0.], [0., 2.]], + [[2., 0.], [0., 2.]], + [[2., 0.], [0., 2.]], + ]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn float_mask_fill_infinite() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [f32::NEG_INFINITY, f32::NEG_INFINITY], + [f32::NEG_INFINITY, f32::NEG_INFINITY], + ], + &device, + ); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + + let output = tensor.mask_fill(mask, 10.0f32); + let expected = TensorData::from([[10f32, f32::NEG_INFINITY], [f32::NEG_INFINITY, 10f32]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/matmul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/matmul.rs new file mode 100644 index 0000000..3ae7c3d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/matmul.rs @@ -0,0 +1,275 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::{ElementConversion, Tolerance, backend::Backend}; + +#[test] +fn test_float_matmul_d2() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_floats([[1.0, 7.0], [2.0, 3.0], [1.0, 5.0]], &device); + let tensor_2 = TestTensor::from_floats([[4.0, 7.0, 5.0], [2.0, 3.0, 5.0]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[18.0, 28.0, 40.0], [14.0, 23.0, 25.0], [14.0, 22.0, 30.0]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_d3() { + let device = Default::default(); + let tensor_1 = TestTensor::<3>::from_floats([[[1.0, 7.0], [2.0, 3.0]]], &device); + let tensor_2 = TestTensor::from_floats([[[4.0, 7.0], [2.0, 3.0]]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[[18.0, 28.0], [14.0, 23.0]]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_broadcast_1() { + let device = Default::default(); + let tensor_1 = TestTensor::<3>::from_floats([[[1.0, 7.0], [2.0, 3.0]]], &device); + let tensor_2 = TestTensor::from_floats( + [[[4.0, 7.0], [2.0, 3.0]], [[2.0, 5.0], [6.0, 3.0]]], + &device, + ); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[[18.0, 28.0], [14.0, 23.0]], [[44.0, 26.0], [22.0, 19.0]]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_broadcast_4d() { + let device = Default::default(); + // [2, 1, 2, 2] + let tensor_1 = TestTensor::<4>::from_floats( + [[[[1.0, 7.0], [2.0, 3.0]]], [[[2.0, 5.0], [6.0, 3.0]]]], + &device, + ); + // [1, 2, 2, 2] + let tensor_2 = TestTensor::from_floats( + [[[[9.0, 8.0], [1.0, 4.0]], [[2.0, 7.0], [3.0, 5.0]]]], + &device, + ); + + // [2, 1, 2, 2] @ [1, 2, 2, 2] -> [2, 2, 2, 2] + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [[[16.0, 36.0], [21.0, 28.0]], [[23.0, 42.0], [13.0, 29.0]]], + [[[23.0, 36.0], [57.0, 60.0]], [[19.0, 39.0], [21.0, 57.0]]], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_simple_1() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_floats([[5.0, 14.0], [14.0, 50.0]], &device); + let tensor_2 = TestTensor::from_floats([[3.0, 4.0, 5.0], [0.0, 1.0, 2.0]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[15.0, 34.0, 53.0], [42.0, 106.0, 170.0]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_4_3() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_floats( + [[0., 1., 2., 3.], [4., 5., 6., 7.], [8., 9., 10., 11.]], + &device, + ); + let tensor_2 = TestTensor::from_floats( + [[0., 1., 2.], [4., 5., 6.], [8., 9., 10.], [12., 13., 14.]], + &device, + ); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[56., 62., 68.], [152., 174., 196.], [248., 286., 324.]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_batch_vec_mat() { + let device = Default::default(); + + // [..., B, 1, K] = [3, 1, 2] + let tensor_1 = + TestTensor::<3>::from_floats([[[1.0, 7.0]], [[2.0, 3.0]], [[1.0, 5.0]]], &device); + + // [..., 1, K, N] = [1, 2, 3] + let tensor_2 = TestTensor::<3>::from_floats([[[4.0, 7.0, 5.0], [2.0, 3.0, 5.0]]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + + // [..., B, 1, N] = [3, 1, 3] + let expected = TensorData::from([ + [[18.0, 28.0, 40.0]], + [[14.0, 23.0, 25.0]], + [[14.0, 22.0, 30.0]], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_trivial() { + let device = Default::default(); + + let tensor_1 = TestTensorInt::<1>::arange(0..16, &device) + .reshape([4, 4]) + .float(); + + let tensor_3 = tensor_1.clone().matmul(tensor_1); + + tensor_3.into_data().assert_approx_eq::( + &TensorData::from([ + [56., 62., 68., 74.], + [152., 174., 196., 218.], + [248., 286., 324., 362.], + [344., 398., 452., 506.], + ]), + Tolerance::default(), + ); +} + +#[test] +fn test_float_matmul_trivial_transposed() { + let device = Default::default(); + + let tensor_1 = TestTensorInt::<1>::arange(0..16, &device) + .reshape([4, 4]) + .float(); + + let tensor_3 = tensor_1.clone().matmul(tensor_1.transpose()); + + tensor_3.into_data().assert_approx_eq::( + &TensorData::from([ + [14., 38., 62., 86.], + [38., 126., 214., 302.], + [62., 214., 366., 518.], + [86., 302., 518., 734.], + ]), + Tolerance::default(), + ); +} + +/// Regression test for batch bug in fused matmul +#[test] +fn test_float_matmul_vecmat_transposed_fused() { + let device = Default::default(); + + let batch1 = 1; + let batch2 = 2; + let batch = batch1 * batch2; + let seq_length = 3; + let d_model = 32; + + // Guard int arange limits + #[allow(clippy::unnecessary_cast)] + if (IntElem::MAX as i64) < seq_length * d_model * batch { + return; + } + if FloatElem::MAX.elem::() < 269493.0 { + return; + } + + let weight: TestTensor<4> = TestTensorInt::arange(0..d_model * batch, &device) + .reshape([batch1, batch2, 1, d_model]) + .float(); + let signal: TestTensor<4> = TestTensorInt::arange(0..seq_length * d_model * batch, &device) + .reshape([batch1, batch2, seq_length, d_model]) + .float(); + + TestBackend::sync(&device).unwrap(); + let weight = weight.transpose(); + let out = signal.matmul(weight) + 5; + let expected = TensorData::from([[ + [[10421.0], [26293.0], [42165.0]], + [[172213.0], [220853.0], [269493.0]], + ]]); + expected.assert_approx_eq(&out.into_data(), Tolerance::::strict()); +} + +#[test] +fn test_float_matmul_4_8() { + let device = Default::default(); + + let tensor_1 = TestTensorInt::<1>::arange(0..32, &device) + .reshape([4, 8]) + .float(); + + let tensor_3 = tensor_1.clone().matmul(tensor_1.transpose()); + + tensor_3.into_data().assert_approx_eq::( + &TensorData::from([ + [140., 364., 588., 812.], + [364., 1100., 1836., 2572.], + [588., 1836., 3084., 4332.], + [812., 2572., 4332., 6092.], + ]), + Tolerance::default(), + ); +} + +#[test] +fn test_float_matmul_simple_2() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_floats([[1.0, 2.0, 3.0, 4.0]], &device); + let tensor_2 = TestTensor::from_floats([[3.0], [4.0], [5.0], [6.0]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[50.0]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_float_matmul_simple_3() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_floats( + [[3., 3., 3.], [4., 4., 4.], [5., 5., 5.], [6., 6., 6.]], + &device, + ); + let tensor_2 = TestTensor::from_floats( + [[1., 2., 3., 4.], [1., 2., 3., 4.], [1., 2., 3., 4.]], + &device, + ); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [9., 18., 27., 36.], + [12., 24., 36., 48.], + [15., 30., 45., 60.], + [18., 36., 54., 72.], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn float_should_panic_when_inner_dimensions_are_not_equal() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_floats([[3., 3.], [4., 4.], [5., 5.], [6., 6.]], &device); + let tensor_2 = TestTensor::from_floats( + [[1., 2., 3., 4.], [1., 2., 3., 4.], [1., 2., 3., 4.]], + &device, + ); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [9., 18., 27., 36.], + [12., 24., 36., 48.], + [15., 30., 45., 60.], + [18., 36., 54., 72.], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/maxmin.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/maxmin.rs new file mode 100644 index 0000000..e4e3d37 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/maxmin.rs @@ -0,0 +1,269 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_max_dim_2d() { + let f = TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + f.clone() + .max_dim(0) + .into_data() + .assert_eq(&TensorData::from([[3., 4., 5.]]), false); + + f.clone() + .max_dim(1) + .into_data() + .assert_eq(&TensorData::from([[2.], [5.]]), false); + + // Negative Index + f.clone() + .max_dim(-1) + .into_data() + .assert_eq(&TensorData::from([[2.], [5.]]), false); + + // Regression Test: https://github.com/tracel-ai/burn/issues/3139 + let z = f.clone().int(); + z.clone() + .max_dim(0) + .into_data() + .assert_eq(&TensorData::from([[3, 4, 5]]), false); + z.clone() + .max_dim(1) + .into_data() + .assert_eq(&TensorData::from([[2], [5]]), false); +} + +#[test] +fn test_max_dims_2d() { + let f = TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + f.clone() + .max_dims(&[0]) + .into_data() + .assert_eq(&TensorData::from([[3., 4., 5.]]), false); + + f.clone() + .max_dims(&[-2]) + .into_data() + .assert_eq(&TensorData::from([[3., 4., 5.]]), false); + + f.clone() + .max_dims(&[0, 1]) + .into_data() + .assert_eq(&TensorData::from([[5.]]), false); +} + +#[test] +fn test_max_dim_with_indices_2d_with_dim_0th() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + // Positive, Negative Index + for idx in [0, -2] { + let (output, index) = tensor.clone().max_dim_with_indices(idx); + + let output_expected = TensorData::from([[3., 4., 5.]]); + let index_expected = TensorData::from([[1, 1, 1]]); + + output.into_data().assert_eq(&output_expected, false); + index.into_data().assert_eq(&index_expected, false); + } +} + +#[test] +fn test_max_dim_with_indices_2d() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + let (output, index) = tensor.max_dim_with_indices(1); + + let output_expected = TensorData::from([[2.], [5.]]); + let index_expected = TensorData::from([[2], [2]]); + + output.into_data().assert_eq(&output_expected, false); + index.into_data().assert_eq(&index_expected, false); +} + +#[test] +fn test_max_dim_2d_with_0th_dim() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + let output = tensor.max_dim(0); + let expected = TensorData::from([[3., 4., 5.]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_max_pair() { + let a = TestTensor::<1>::from_floats([1.0, 2.0, 3.0, 4.0], &Default::default()); + let b = TestTensor::from_floats([2.0, 1.0, 4.0, 5.0], &Default::default()); + + let output = a.max_pair(b); + let expected = TensorData::from([2.0, 2.0, 4.0, 5.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_min_dim_2d() { + let f = TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + f.clone() + .min_dim(0) + .into_data() + .assert_eq(&TensorData::from([[0., 1., 2.]]), false); + + f.clone() + .min_dim(1) + .into_data() + .assert_eq(&TensorData::from([[0.], [3.]]), false); + + // Negative Index + f.clone() + .min_dim(-1) + .into_data() + .assert_eq(&TensorData::from([[0.], [3.]]), false); + + // Regression Test: https://github.com/tracel-ai/burn/issues/3139 + let z = f.int(); + z.clone() + .min_dim(0) + .into_data() + .assert_eq(&TensorData::from([[0, 1, 2]]), false); + z.clone() + .min_dim(1) + .into_data() + .assert_eq(&TensorData::from([[0], [3]]), false); +} + +#[test] +fn test_min_dims_2d() { + let f = TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + f.clone() + .min_dims(&[0]) + .into_data() + .assert_eq(&TensorData::from([[0., 1., 2.]]), false); + + f.clone() + .min_dims(&[-2]) + .into_data() + .assert_eq(&TensorData::from([[0., 1., 2.]]), false); + + f.clone() + .min_dims(&[0, 1]) + .into_data() + .assert_eq(&TensorData::from([[0.]]), false); +} + +#[test] +fn test_min_dim_with_indices_2d() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + let (output, index) = tensor.min_dim_with_indices(1); + + let output_expected = TensorData::from([[0.], [3.]]); + let index_expected = TensorData::from([[0], [0]]); + + output.into_data().assert_eq(&output_expected, false); + index.into_data().assert_eq(&index_expected, false); +} + +#[test] +fn test_min_dim_2d_with_0th_dim() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + let output = tensor.min_dim(0); + let expected = TensorData::from([[0., 1., 2.]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_min_dim_with_indices_2d_with_0th_dim() { + let tensor = + TestTensor::<2>::from_floats([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &Default::default()); + + // Positive, Negative Index + for idx in [0, -2] { + let (output, index) = tensor.clone().min_dim_with_indices(idx); + + let output_expected = TensorData::from([[0., 1., 2.]]); + let index_expected = TensorData::from([[0, 0, 0]]); + + output.into_data().assert_eq(&output_expected, false); + index.into_data().assert_eq(&index_expected, false); + } +} + +#[test] +fn test_min_pair() { + let a = TestTensor::<1>::from_floats([1.0, 2.0, 3.0, 4.0], &Default::default()); + let b = TestTensor::from_floats([2.0, 1.0, 4.0, 5.0], &Default::default()); + + let output = a.min_pair(b); + let expected = TensorData::from([1.0, 1.0, 3.0, 4.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_max_abs() { + let tensor = TestTensor::<2>::from_floats([[0., 1., -2.], [-5., 6., 1.]], &Default::default()); + + let output = tensor.max_abs(); + let expected = TensorData::from([6.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_max_abs_dim_2d_dim_0() { + let tensor = TestTensor::<2>::from_floats([[0., 1., -2.], [-5., 6., 1.]], &Default::default()); + + let output = tensor.clone().max_abs_dim(0); + let expected = TensorData::from([[5., 6., 2.]]); + output.into_data().assert_eq(&expected, false); + + // Negative Index + let output = tensor.clone().max_abs_dim(-2); + let expected = TensorData::from([[5., 6., 2.]]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_max_abs_dims_2d() { + let tensor = TestTensor::<2>::from_floats([[0., 1., -2.], [-5., 6., 1.]], &Default::default()); + + tensor + .clone() + .max_abs_dims(&[0]) + .into_data() + .assert_eq(&TensorData::from([[5., 6., 2.]]), false); + + tensor + .clone() + .max_abs_dims(&[-2]) + .into_data() + .assert_eq(&TensorData::from([[5., 6., 2.]]), false); + + tensor + .clone() + .max_abs_dims(&[0, 1]) + .into_data() + .assert_eq(&TensorData::from([[6.]]), false); +} + +#[test] +fn test_max_abs_dim_2d_dim_1() { + let tensor = TestTensor::<2>::from_floats([[0., 1., -2.], [-5., 6., 1.]], &Default::default()); + + let output = tensor.max_abs_dim(1); + let expected = TensorData::from([[2.], [6.]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mod.rs new file mode 100644 index 0000000..94c81f1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mod.rs @@ -0,0 +1,76 @@ +use super::*; + +mod abs; +mod add; +mod aggregation; +mod all; +mod any; +mod arg; +mod cast; +mod cat; +mod ceil; +mod chunk; +mod clamp; +mod close; +mod comparison; +mod create_like; +mod cross; +mod cumulative; +mod div; +mod dot; +mod erf; +mod exp; +mod expand; +mod finite; +mod flatten; +mod flip; +mod floor; +mod fmod; +mod full; +mod gather_scatter; +mod grid_sample; +mod inf; +mod init; +mod iter_dim; +mod log; +mod log1p; +mod mask; +mod matmul; +mod maxmin; +mod movedim; +mod mul; +mod nan; +mod narrow; +mod neg; +mod one_hot; +mod padding; +mod permute; +mod powf; +mod powf_scalar; +mod prod; +mod random; +mod recip; +mod remainder; +mod repeat; +mod repeat_dim; +mod reshape; +mod round; +mod select; +mod sign; +mod slice; +mod slice_assign; +mod sort_argsort; +mod split; +mod sqrt; +mod square; +mod squeeze; +mod stack; +mod sub; +mod take; +mod topk; +mod transaction; +mod transpose; +mod tri; +mod trig; +mod trunc; +mod unfold; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/movedim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/movedim.rs new file mode 100644 index 0000000..49ee98a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/movedim.rs @@ -0,0 +1,123 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn movedim_float() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .float(); + + let permuted = tensor.clone().movedim(0, 2); + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim(0, 2).float() + let expected = TensorData::from([ + [[0., 12.], [1., 13.], [2., 14.], [3., 15.]], + [[4., 16.], [5., 17.], [6., 18.], [7., 19.]], + [[8., 20.], [9., 21.], [10., 22.], [11., 23.]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().movedim(0, -1); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().movedim(0, 0); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} + +#[test] +fn vec_input_float() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .float(); + + let permuted = tensor.clone().movedim(vec![0, 1], vec![1, 0]); + // from pytorch + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim([0, 1], [1, 0]).float() + let expected = TensorData::from([ + [[0., 1., 2., 3.], [12., 13., 14., 15.]], + [[4., 5., 6., 7.], [16., 17., 18., 19.]], + [[8., 9., 10., 11.], [20., 21., 22., 23.]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axes + let permuted = tensor.clone().movedim(vec![-3, -2], vec![-2, -3]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axes + let permuted = tensor.clone().movedim(vec![0, 1], vec![0, 1]); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} + +#[test] +fn different_input_types() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .float(); + + let permuted = tensor.clone().movedim(0_usize, 2_i32); + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim(0, 2).float() + let expected = TensorData::from([ + [[0., 12.], [1., 13.], [2., 14.], [3., 15.]], + [[4., 16.], [5., 17.], [6., 18.], [7., 19.]], + [[8., 20.], [9., 21.], [10., 22.], [11., 23.]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().movedim(0_usize, -1); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().movedim(0_i32, 0_usize); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} + +#[test] +#[should_panic] +fn edge_different_sizes() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with a repeated axis + let _ = tensor.clone().movedim(vec![0, 1], vec![0]); +} + +#[test] +#[should_panic] +fn edge_out_of_bound_axis() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with an out of bound axis + let _ = tensor.clone().movedim(0, 100); +} + +#[test] +#[should_panic] +fn edge_vec_is_not_a_set() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with a repeated axis + let _ = tensor.clone().movedim(vec![0, 1, 1, 1, 1], vec![0, 0, 1]); +} + +#[test] +#[should_panic] +fn edge_out_of_bound_axis_vec() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with an out of bound axis + let _ = tensor.clone().movedim(vec![0, 100], vec![0, 1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mul.rs new file mode 100644 index 0000000..32d6483 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/mul.rs @@ -0,0 +1,54 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_mul_ops() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data_2 = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0.0, 1.0, 4.0], [9.0, 16.0, 25.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_mul_broadcast() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0]]); + let data_2 = TensorData::from([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0.0, 4.0, 10.0], [0.0, 7.0, 16.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_mul_broadcast_2_dims() { + let device = Default::default(); + let tensor_1 = TestTensor::<1>::from_data([0.0, 1.0, 2.0], &device).reshape([3, 1]); + let tensor_2 = TestTensor::<1>::from_data([3.0, 4.0, 5.0], &device).reshape([1, 3]); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0.0, 0.0, 0.0], [3.0, 4.0, 5.0], [6.0, 8.0, 10.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_mul_scalar_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let scalar = 2.0; + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor * scalar; + let expected = TensorData::from([[0.0, 2.0, 4.0], [6.0, 8.0, 10.0]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/nan.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/nan.rs new file mode 100644 index 0000000..599a1fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/nan.rs @@ -0,0 +1,24 @@ +use super::*; +use burn_tensor::cast::ToElement; + +#[test] +fn is_nan() { + let no_nan = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let no_nan_expected = TestTensorBool::<2>::from([[false, false, false], [false, false, false]]); + + let with_nan = TestTensor::<2>::from([[0.0, f32::NAN, 2.0], [f32::NAN, 4.0, 5.0]]); + let with_nan_expected = TestTensorBool::<2>::from([[false, true, false], [true, false, false]]); + + assert_eq!(no_nan_expected.into_data(), no_nan.is_nan().into_data()); + + assert_eq!(with_nan_expected.into_data(), with_nan.is_nan().into_data()); +} + +#[test] +fn contains_nan() { + let no_nan = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + assert!(!no_nan.contains_nan().into_scalar().to_bool()); + + let with_nan = TestTensor::<2>::from([[0.0, f32::NAN, 2.0], [3.0, 4.0, 5.0]]); + assert!(with_nan.contains_nan().into_scalar().to_bool()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/narrow.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/narrow.rs new file mode 100644 index 0000000..ec47e34 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/narrow.rs @@ -0,0 +1,98 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Shape, TensorData}; + +#[test] +fn test_narrow_1() { + let tensor = TestTensor::<2>::from_data( + TensorData::from([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]), + &Default::default(), + ); + + let output = tensor.clone().narrow(0, 0, 2); + let expected = TensorData::from([[1., 2., 3.], [4., 5., 6.]]); + + assert_eq!(output.shape(), Shape::from([2, 3])); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_narrow_2() { + let tensor = TestTensor::<2>::from_data( + TensorData::from([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]), + &Default::default(), + ); + + let output = tensor.clone().narrow(1, 1, 2); + let expected = TensorData::from([[2., 3.], [5., 6.], [8., 9.]]); + assert_eq!(output.shape(), Shape::from([3, 2])); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_narrow_3() { + let device = &Default::default(); + let shape = Shape::new([8, 8]); + let tensor = TestTensorInt::arange(0..shape.num_elements() as i64, device) + .reshape::<2, _>(shape) + .float(); + + let output = tensor.clone().narrow(0, 3, 4); + let expected = TensorData::from([ + [24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], + [32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0], + [40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0], + [48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0], + ]); + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_dim() { + let tensor = TestTensor::<2>::from_data( + TensorData::from([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]), + &Default::default(), + ); + + let _output = tensor.narrow(2, 0, 2); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_start() { + let tensor = TestTensor::<2>::from_data( + TensorData::from([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]), + &Default::default(), + ); + + let _output = tensor.narrow(0, 3, 2); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_zero_length() { + let tensor = TestTensor::<2>::from_data( + TensorData::from([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]), + &Default::default(), + ); + + let _output = tensor.narrow(0, 1, 0); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_length() { + let tensor = TestTensor::<2>::from_data( + TensorData::from([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]), + &Default::default(), + ); + + let _output = tensor.narrow(0, 0, 4); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/neg.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/neg.rs new file mode 100644 index 0000000..01c9aaa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/neg.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_neg_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.neg(); + let expected = TensorData::from([[-0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]).convert::(); + + // -0.0 is represented differently than 0.0 so we make sure the values are the same in f32 + assert_eq!( + output + .into_data() + .convert::() + .as_slice::() + .unwrap(), + expected.as_slice::().unwrap() + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/one_hot.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/one_hot.rs new file mode 100644 index 0000000..60430ad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/one_hot.rs @@ -0,0 +1,71 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn float_should_support_one_hot() { + let tensor = TestTensor::<1>::from([0.0, 1.0, 4.0]); + let one_hot_tensor: TestTensor<2> = tensor.one_hot(5); + let expected = TensorData::from([ + [1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0], + ]); + one_hot_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn float_should_support_one_hot_index() { + let tensor = TestTensor::<1>::from([2.0]); + let one_hot_tensor: TestTensor<2> = tensor.one_hot::<2>(10); + let expected = TensorData::from([[0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]); + one_hot_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn float_one_hot_should_panic_when_index_exceeds_number_of_classes() { + let tensor = TestTensor::<1>::from([5.0]); + let _result: TestTensor<2> = tensor.one_hot(5); +} + +#[test] +#[should_panic] +fn float_one_hot_should_panic_when_number_of_classes_is_zero() { + let tensor = TestTensor::<1>::from([0.0]); + let _result: TestTensor<2> = tensor.one_hot(0); +} + +#[test] +fn one_hot_fill_with_negative_axis_and_indices() { + let tensor = TestTensor::<2>::from([[0, 2], [1, -1]]); + let expected = TensorData::from([ + [[5.0, 0.0, 0.0], [0.0, 0.0, 5.0]], + [[0.0, 5.0, 0.0], [0.0, 0.0, 5.0]], + ]); + + let one_hot_tensor: TestTensor<3> = tensor.one_hot_fill(3, 5.0, 0.0, -1); + + one_hot_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn one_hot_fill_with_negative_indices() { + let tensor = TestTensor::<1>::from([0.0, -7.0, -8.0]); + let expected = TensorData::from([ + [3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ]); + + let one_hot_tensor: TestTensor<2> = tensor.one_hot_fill(10, 3.0, 1.0, 1); + + one_hot_tensor.into_data().assert_eq(&expected, false); +} + +#[should_panic] +#[test] +fn one_hot_fill_should_panic_when_axis_out_range_of_rank() { + let tensor = TestTensor::<2>::from([[0.0, 2.0], [1.0, -1.0]]); + + let _one_hot_tensor: TestTensor<3> = tensor.one_hot_fill(2, 5.0, 0.0, 3); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/padding.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/padding.rs new file mode 100644 index 0000000..fa2101d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/padding.rs @@ -0,0 +1,470 @@ +use super::*; +use burn_tensor::{TensorData, ops::PadMode}; + +#[test] +fn padding_constant_2d_test() { + let unpadded_floats: [[f32; 3]; 2] = [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]; + let tensor = TestTensor::<2>::from(unpadded_floats); + + let padded_tensor = tensor.pad((2, 2, 2, 2), 1.1); + + let expected = TensorData::from([ + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 0.0, 1.0, 2.0, 1.1, 1.1], + [1.1, 1.1, 3.0, 4.0, 5.0, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_constant_4d_test() { + let unpadded_floats = [[[[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]]]]; + let tensor = TestTensor::<4>::from(unpadded_floats); + + let padded_tensor = tensor.pad((2, 2, 2, 2), 1.1); + + let expected = TensorData::from([[[ + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 0.0, 1.0, 1.1, 1.1], + [1.1, 1.1, 2.0, 3.0, 1.1, 1.1], + [1.1, 1.1, 4.0, 5.0, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + ]]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_constant_asymmetric_test() { + let unpadded_floats = [[[[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]]]]; + let tensor = TestTensor::<4>::from(unpadded_floats); + + let padded_tensor = tensor.pad((2, 1, 4, 3), 1.1); + + let expected = TensorData::from([[[ + [1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 0.0, 1.0, 1.1], + [1.1, 1.1, 2.0, 3.0, 1.1], + [1.1, 1.1, 4.0, 5.0, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1], + ]]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_2d_test() { + // Test reflect padding on a 2D tensor + // Input: [[1, 2, 3], [4, 5, 6]] + // With padding (1, 1, 1, 1): + // - Top: reflect row 1 -> [4, 5, 6] + // - Bottom: reflect row 0 -> [1, 2, 3] + // - Left: reflect col 1 + // - Right: reflect col 1 + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + let padded_tensor = tensor.pad((1, 1, 1, 1), PadMode::Reflect); + + // Expected: reflect excludes the edge value + // Before padding height: [[1,2,3], [4,5,6]] + // After top pad (reflect row at index 1): [[4,5,6], [1,2,3], [4,5,6]] + // After bottom pad (reflect row at index 1 from end): [[4,5,6], [1,2,3], [4,5,6], [1,2,3]] + // Then pad width similarly + let expected = TensorData::from([ + [5.0, 4.0, 5.0, 6.0, 5.0], + [2.0, 1.0, 2.0, 3.0, 2.0], + [5.0, 4.0, 5.0, 6.0, 5.0], + [2.0, 1.0, 2.0, 3.0, 2.0], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_width_only_test() { + // Test reflect padding on width dimension only + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0, 4.0]]); + + let padded_tensor = tensor.pad((2, 2, 0, 0), PadMode::Reflect); + + // Input: [1, 2, 3, 4] + // Reflect left 2: take indices [1, 2] = [2, 3], flip = [3, 2] + // Reflect right 2: take indices [1, 2] from end = [2, 3], flip = [3, 2] + // Result: [3, 2, 1, 2, 3, 4, 3, 2] + let expected = TensorData::from([[3.0, 2.0, 1.0, 2.0, 3.0, 4.0, 3.0, 2.0]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_4d_test() { + // Test reflect padding on 4D tensor (common for images: NCHW) + let tensor = TestTensor::<4>::from([[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]]]); + + let padded_tensor = tensor.pad((1, 1, 1, 1), PadMode::Reflect); + + let expected = TensorData::from([[[ + [5.0, 4.0, 5.0, 6.0, 5.0], + [2.0, 1.0, 2.0, 3.0, 2.0], + [5.0, 4.0, 5.0, 6.0, 5.0], + [8.0, 7.0, 8.0, 9.0, 8.0], + [5.0, 4.0, 5.0, 6.0, 5.0], + ]]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_edge_2d_test() { + // Test edge padding on a 2D tensor + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + let padded_tensor = tensor.pad((1, 1, 1, 1), PadMode::Edge); + + // Edge padding replicates the boundary values + let expected = TensorData::from([ + [1.0, 1.0, 2.0, 3.0, 3.0], + [1.0, 1.0, 2.0, 3.0, 3.0], + [4.0, 4.0, 5.0, 6.0, 6.0], + [4.0, 4.0, 5.0, 6.0, 6.0], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_edge_width_only_test() { + // Test edge padding on width dimension only + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0, 4.0]]); + + let padded_tensor = tensor.pad((2, 3, 0, 0), PadMode::Edge); + + // Input: [1, 2, 3, 4] + // Left 2: [1, 1] + // Right 3: [4, 4, 4] + // Result: [1, 1, 1, 2, 3, 4, 4, 4, 4] + let expected = TensorData::from([[1.0, 1.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 4.0]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_edge_4d_test() { + // Test edge padding on 4D tensor + let tensor = TestTensor::<4>::from([[[[1.0, 2.0], [3.0, 4.0]]]]); + + let padded_tensor = tensor.pad((1, 1, 1, 1), PadMode::Edge); + + let expected = TensorData::from([[[ + [1.0, 1.0, 2.0, 2.0], + [1.0, 1.0, 2.0, 2.0], + [3.0, 3.0, 4.0, 4.0], + [3.0, 3.0, 4.0, 4.0], + ]]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_constant_default_test() { + // Test default PadMode (Constant with 0.0) + let tensor = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let padded_tensor = tensor.pad((1, 1, 1, 1), PadMode::default()); + + let expected = TensorData::from([ + [0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 0.0], + [0.0, 3.0, 4.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_max_valid_test() { + // Test reflect padding at maximum valid size (dim_size - 1) + // For a 4-element dimension, max valid padding is 3 + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0, 4.0]]); + + // Padding of 3 on left is valid for width=4 (3 < 4) + let padded_tensor = tensor.pad((3, 3, 0, 0), PadMode::Reflect); + + // Input: [1, 2, 3, 4] + // Reflect left 3: take indices [1, 2, 3] = [2, 3, 4], flip = [4, 3, 2] + // Reflect right 3: take indices [0, 1, 2] = [1, 2, 3], flip = [3, 2, 1] + // Result: [4, 3, 2, 1, 2, 3, 4, 3, 2, 1] + let expected = TensorData::from([[4.0, 3.0, 2.0, 1.0, 2.0, 3.0, 4.0, 3.0, 2.0, 1.0]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_asymmetric_test() { + // Test asymmetric reflect padding + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]); + + // Asymmetric padding: left=2, right=1, top=1, bottom=2 + let padded_tensor = tensor.pad((2, 1, 1, 2), PadMode::Reflect); + + let expected = TensorData::from([ + [6.0, 5.0, 4.0, 5.0, 6.0, 5.0], + [3.0, 2.0, 1.0, 2.0, 3.0, 2.0], + [6.0, 5.0, 4.0, 5.0, 6.0, 5.0], + [9.0, 8.0, 7.0, 8.0, 9.0, 8.0], + [6.0, 5.0, 4.0, 5.0, 6.0, 5.0], + [3.0, 2.0, 1.0, 2.0, 3.0, 2.0], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic(expected = "Reflect padding")] +fn padding_reflect_exceeds_dimension_test() { + // Test that reflect padding panics when padding >= dim_size + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0]]); + + // Padding of 3 on width=3 should panic (3 >= 3, need padding < dim_size) + let _ = tensor.pad((3, 0, 0, 0), PadMode::Reflect); +} + +#[test] +fn padding_edge_asymmetric_test() { + // Test asymmetric edge padding + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + // Asymmetric padding: left=2, right=1, top=3, bottom=1 + let padded_tensor = tensor.pad((2, 1, 3, 1), PadMode::Edge); + + let expected = TensorData::from([ + [1.0, 1.0, 1.0, 2.0, 3.0, 3.0], + [1.0, 1.0, 1.0, 2.0, 3.0, 3.0], + [1.0, 1.0, 1.0, 2.0, 3.0, 3.0], + [1.0, 1.0, 1.0, 2.0, 3.0, 3.0], + [4.0, 4.0, 4.0, 5.0, 6.0, 6.0], + [4.0, 4.0, 4.0, 5.0, 6.0, 6.0], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_zero_padding_test() { + // Test that zero padding returns the original tensor unchanged + let tensor = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let padded_constant = tensor.clone().pad((0, 0, 0, 0), PadMode::Constant(5.0)); + let padded_reflect = tensor.clone().pad((0, 0, 0, 0), PadMode::Reflect); + let padded_edge = tensor.clone().pad((0, 0, 0, 0), PadMode::Edge); + + let expected = TensorData::from([[1.0, 2.0], [3.0, 4.0]]); + padded_constant.into_data().assert_eq(&expected, false); + padded_reflect.into_data().assert_eq(&expected, false); + padded_edge.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_empty_tensor_constant_test() { + // Test constant padding on an empty tensor (zero-sized dimension) + // This should work - creates a tensor filled with the constant value + let tensor: TestTensor<2> = TestTensor::empty([0, 3], &Default::default()); + + // Padding an empty height dimension with constant should create a tensor of just padding + let padded = tensor.pad((0, 0, 2, 2), 1.0); + + // Result should be 4x3 (0 + 2 + 2 = 4 rows) + assert_eq!(padded.dims(), [4, 3]); + + let expected = TensorData::from([ + [1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic(expected = "edge padding")] +fn padding_empty_tensor_edge_panics_test() { + // Test that edge padding panics on empty tensor + let tensor: TestTensor<2> = TestTensor::empty([0, 3], &Default::default()); + + // Edge padding on zero-sized dimension should panic + let _ = tensor.pad((0, 0, 1, 1), PadMode::Edge); +} + +#[test] +#[should_panic(expected = "Reflect padding")] +fn padding_empty_tensor_reflect_panics_test() { + // Test that reflect padding panics on empty tensor + let tensor: TestTensor<2> = TestTensor::empty([0, 3], &Default::default()); + + // Reflect padding on zero-sized dimension should panic + let _ = tensor.pad((0, 0, 1, 1), PadMode::Reflect); +} + +// --- Tests for N-dimensional padding using (before, after) pairs --- + +#[test] +fn padding_constant_pairs_2d_test() { + // Same as padding_constant_2d_test but using the new pairs API + let tensor = TestTensor::<2>::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + // [(row_before, row_after), (col_before, col_after)] + let padded_tensor = tensor.pad([(2, 2), (2, 2)], 1.1); + + let expected = TensorData::from([ + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 0.0, 1.0, 2.0, 1.1, 1.1], + [1.1, 1.1, 3.0, 4.0, 5.0, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + [1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1], + ]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_constant_single_dim_test() { + // Pad only the last dimension + let tensor = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let padded_tensor = tensor.pad([(1, 1)], 0.0); + + let expected = TensorData::from([[0.0, 1.0, 2.0, 0.0], [0.0, 3.0, 4.0, 0.0]]); + padded_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_constant_all_dims_4d_test() { + // Pad all 4 dimensions of a 4D tensor (batch, channel, height, width) + // Input: shape [1, 1, 2, 2] + let tensor = TestTensor::<4>::from([[[[1.0, 2.0], [3.0, 4.0]]]]); + + // Pad: batch(1,1), channel(1,1), height(0,0), width(0,0) + let padded = tensor.pad([(1, 1), (1, 1), (0, 0), (0, 0)], 0.0); + + // Shape should be [3, 3, 2, 2] + assert_eq!(padded.dims(), [3, 3, 2, 2]); + + let expected = TensorData::from([ + [ + [[0.0, 0.0], [0.0, 0.0]], + [[0.0, 0.0], [0.0, 0.0]], + [[0.0, 0.0], [0.0, 0.0]], + ], + [ + [[0.0, 0.0], [0.0, 0.0]], + [[1.0, 2.0], [3.0, 4.0]], + [[0.0, 0.0], [0.0, 0.0]], + ], + [ + [[0.0, 0.0], [0.0, 0.0]], + [[0.0, 0.0], [0.0, 0.0]], + [[0.0, 0.0], [0.0, 0.0]], + ], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_constant_batch_dim_only_test() { + // Pad only the batch dimension of a 3D tensor [N, H, W] + let tensor = TestTensor::<3>::from([[[1.0, 2.0], [3.0, 4.0]]]); + + // 3 pairs for 3 dims: batch(1,1), height(0,0), width(0,0) + let padded = tensor.pad([(1, 1), (0, 0), (0, 0)], -1.0); + + assert_eq!(padded.dims(), [3, 2, 2]); + + let expected = TensorData::from([ + [[-1.0, -1.0], [-1.0, -1.0]], + [[1.0, 2.0], [3.0, 4.0]], + [[-1.0, -1.0], [-1.0, -1.0]], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_pairs_test() { + // Reflect padding using pairs API + let tensor = TestTensor::<2>::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]); + + let padded = tensor.pad([(1, 1), (1, 1)], PadMode::Reflect); + + let expected = TensorData::from([ + [5.0, 4.0, 5.0, 6.0, 5.0], + [2.0, 1.0, 2.0, 3.0, 2.0], + [5.0, 4.0, 5.0, 6.0, 5.0], + [8.0, 7.0, 8.0, 9.0, 8.0], + [5.0, 4.0, 5.0, 6.0, 5.0], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_edge_pairs_test() { + // Edge padding using pairs API + let tensor = TestTensor::<2>::from([[1.0, 2.0], [3.0, 4.0]]); + + let padded = tensor.pad([(1, 1), (1, 1)], PadMode::Edge); + + let expected = TensorData::from([ + [1.0, 1.0, 2.0, 2.0], + [1.0, 1.0, 2.0, 2.0], + [3.0, 3.0, 4.0, 4.0], + [3.0, 3.0, 4.0, 4.0], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_reflect_batch_dim_3d_test() { + // Reflect pad the batch dimension of a 3D tensor [N, H, W] + // Input shape: [3, 1, 2] - 3 batches, 1 row, 2 cols + let tensor = TestTensor::<3>::from([[[1.0, 2.0]], [[3.0, 4.0]], [[5.0, 6.0]]]); + + // Pad batch dim with reflect(1, 1), no spatial padding + let padded = tensor.pad([(1, 1), (0, 0), (0, 0)], PadMode::Reflect); + + assert_eq!(padded.dims(), [5, 1, 2]); + + // Reflect on batch: [3,4] [1,2] [3,4] [5,6] [3,4] + let expected = TensorData::from([ + [[3.0, 4.0]], + [[1.0, 2.0]], + [[3.0, 4.0]], + [[5.0, 6.0]], + [[3.0, 4.0]], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +fn padding_edge_batch_dim_3d_test() { + // Edge pad the batch dimension of a 3D tensor + let tensor = TestTensor::<3>::from([[[1.0, 2.0]], [[3.0, 4.0]]]); + + let padded = tensor.pad([(2, 1), (0, 0), (0, 0)], PadMode::Edge); + + assert_eq!(padded.dims(), [5, 1, 2]); + + let expected = TensorData::from([ + [[1.0, 2.0]], + [[1.0, 2.0]], + [[1.0, 2.0]], + [[3.0, 4.0]], + [[3.0, 4.0]], + ]); + padded.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic(expected = "Padding has")] +fn padding_too_many_pairs_panics_test() { + let tensor = TestTensor::<2>::from([[1.0, 2.0]]); + + // 3 pairs for a 2D tensor should panic + let _ = tensor.pad([(1, 1), (1, 1), (1, 1)], 0.0); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/permute.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/permute.rs new file mode 100644 index 0000000..c9a2982 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/permute.rs @@ -0,0 +1,58 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn permute_float_a() { + let tensor = TestTensor::<1>::from([ + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ]) + .reshape([2, 2, 4]); + + let permuted = tensor.clone().permute([2, 1, 0]); + + let expected = TensorData::from([ + [[0., 8.], [4., 12.]], + [[1., 9.], [5., 13.]], + [[2., 10.], [6., 14.]], + [[3., 11.], [7., 15.]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().permute([-1, 1, 0]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().permute([0, 1, 2]); + permuted.into_data().assert_eq(&tensor.into_data(), false); +} + +#[test] +fn permute_float() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device) + .reshape([2, 3, 4]) + .float(); + + let permuted = tensor.clone().permute([2, 1, 0]); + + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).permute(2, 1, 0).float() + let expected = TensorData::from([ + [[0., 12.], [4., 16.], [8., 20.]], + [[1., 13.], [5., 17.], [9., 21.]], + [[2., 14.], [6., 18.], [10., 22.]], + [[3., 15.], [7., 19.], [11., 23.]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().permute([-1, 1, 0]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().permute([0, 1, 2]); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf.rs new file mode 100644 index 0000000..7271d7a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf.rs @@ -0,0 +1,105 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_powf_ops() { + let data = TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + let pow = TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 2.0]]); + let tensor_pow = TestTensor::<2>::from_data(pow, &Default::default()); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[1.0, 1.0, 4.0], [27.0, 256.0, 25.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_neg_power() { + let data = TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + let pow = TensorData::from([[-0.95, -0.67, -0.45], [-0.24, -0.5, -0.6]]); + let tensor_pow = TestTensor::<2>::from_data(pow, &Default::default()); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[1., 1., 0.73204285], [0.76822936, 0.5, 0.38073079]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_neg_values_with_even_power() { + let data = TensorData::from([[1.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + let pow = TensorData::from([[2.0, 2.0, 4.0], [4.0, 4.0, 2.0]]); + let tensor_pow = TestTensor::<2>::from_data(pow, &Default::default()); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[1.0, 1.0, 16.0], [81.0, 256.0, 25.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_neg_values_with_odd_power() { + let data = TensorData::from([[1.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + let pow = TensorData::from([[3.0, 3.0, 3.0], [3.0, 3.0, 3.0]]); + let tensor_pow = TestTensor::<2>::from_data(pow, &Default::default()); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[1.0, -1.0, -8.0], [-27.0, -64.0, -125.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_powf_broadcasted() { + let device = Default::default(); + let tensor_1 = TestTensor::<1>::from_floats([2.0, 3.0, 4.0], &device); + let tensor_2 = TestTensor::from_floats([1.0], &device); + + // Broadcast rhs + let output = tensor_1.clone().powf(tensor_2.clone()); + output + .into_data() + .assert_approx_eq::(&tensor_1.to_data(), Tolerance::default()); + + // Broadcast lhs + let output = tensor_2.powf(tensor_1); + output + .into_data() + .assert_approx_eq::(&TensorData::from([1.0, 1.0, 1.0]), Tolerance::default()); +} + +fn outer(a: TestTensor<1>, b: TestTensor<1>) -> TestTensor<2> { + a.unsqueeze_dim::<2>(1) * b.unsqueeze_dim::<2>(0) +} + +#[test] +fn should_support_powf_scalar_tensor() { + let device = Default::default(); + let head_dim = 64; + let seq_len = 1024; + let base = 10000; + + let channel_range = TestTensorInt::arange_step(0..head_dim as i64, 2, &device).float(); + let base = TestTensor::<1>::from_data([base as f32], &device); + let inv_freq = base.powf(-channel_range / head_dim as f32); + + let t = TestTensorInt::arange(0..seq_len as i64, &device).float(); + + let freqs = outer(t, inv_freq); + + let _cos = freqs.clone().cos(); + let _sin = freqs.sin(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf_scalar.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf_scalar.rs new file mode 100644 index 0000000..1b19589 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/powf_scalar.rs @@ -0,0 +1,55 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_powf_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.powf_scalar(0.71); + let expected = TensorData::from([[0.0, 1.0, 1.6358], [2.1815, 2.67586, 3.13522]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_neg_power() { + let data = TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.powf_scalar(-0.33); + let expected = TensorData::from([[1.0, 1.0, 0.79553646], [0.695905, 0.6328783, 0.58794934]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_neg_values_with_even_power() { + let data = TensorData::from([[0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.powf_scalar(4.0); + let expected = TensorData::from([[0.0, 1.0, 16.0], [81.0, 256.0, 625.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_neg_values_with_odd_power() { + let data = TensorData::from([[0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.powf_scalar(3.0); + let expected = TensorData::from([[0.0, -1.0, -8.0], [-27.0, -64.0, -125.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/prod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/prod.rs new file mode 100644 index 0000000..dc9156e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/prod.rs @@ -0,0 +1,48 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_prod_float() { + let tensor_1 = TestTensor::<2>::from([[-5.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor_1.prod(); + + output + .into_data() + .assert_eq(&TensorData::from([-600.0]), false); +} + +#[test] +fn test_prod_dim_2d() { + let f = TestTensor::<2>::from([[-5.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + f.clone() + .prod_dim(1) + .into_data() + .assert_eq(&TensorData::from([[-10.0], [60.0]]), false); + + f.clone() + .prod_dim(-1) + .into_data() + .assert_eq(&TensorData::from([[-10.0], [60.0]]), false); +} + +#[test] +fn test_prod_dims_2d() { + let f = TestTensor::<2>::from([[-5.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + f.clone() + .prod_dims(&[1]) + .into_data() + .assert_eq(&TensorData::from([[-10.0], [60.0]]), false); + + f.clone() + .prod_dims(&[-1]) + .into_data() + .assert_eq(&TensorData::from([[-10.0], [60.0]]), false); + + f.clone() + .prod_dims(&[0, 1]) + .into_data() + .assert_eq(&TensorData::from([[-600.0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/random.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/random.rs new file mode 100644 index 0000000..30f53e5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/random.rs @@ -0,0 +1,53 @@ +use super::*; +use burn_tensor::{Distribution, ElementConversion, TensorData, Tolerance, backend::Backend}; + +#[test] +fn rand_default() { + let tensor = TestTensor::<1>::random([20], Distribution::Default, &Default::default()); + + // check that the tensor is within the range of [0..1) (1 is exclusive) + // the conversion can ceil the value if `FloatElem` is less precise than f32 + let low = 0.elem::(); + let high = 1.elem::(); + if FloatElem::EPSILON.elem::() > f32::EPSILON { + tensor.into_data().assert_within_range_inclusive(low..=high); + } else { + tensor.into_data().assert_within_range(low..high); + } +} + +#[test] +fn rand_uniform() { + let tensor = TestTensor::<1>::random([20], Distribution::Uniform(4., 5.), &Default::default()); + let low = 4.elem::(); + let high = 5.elem::(); + + if FloatElem::EPSILON.elem::() > f32::EPSILON { + tensor.into_data().assert_within_range_inclusive(low..=high); + } else { + tensor.into_data().assert_within_range(low..high); + } +} + +#[test] +fn rand_bernoulli() { + let tensor = TestTensor::<1>::random([20], Distribution::Bernoulli(1.), &Default::default()); + + tensor.into_data().assert_eq( + &TensorData::new::(vec![1.elem(); 20], [20]), + true, + ); +} + +#[test] +#[ignore] // TODO: mark serial for backends that handle the same devices (e.g. fusion)? +fn test_seed_reproducibility() { + let device = Default::default(); + TestBackend::seed(&device, 42); + let t1 = TestTensor::<1>::random([5], Distribution::Default, &device); + TestBackend::seed(&device, 42); + let t2 = TestTensor::<1>::random([5], Distribution::Default, &device); + + t1.into_data() + .assert_approx_eq::(&t2.into_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/recip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/recip.rs new file mode 100644 index 0000000..689b023 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/recip.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_recip_ops() { + let data = TensorData::from([[0.5, 1.0, 2.0], [3.0, -4.0, -5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.recip(); + let expected = TensorData::from([[2.0, 1.0, 0.5], [0.33333, -0.25, -0.2]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/remainder.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/remainder.rs new file mode 100644 index 0000000..e48eb92 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/remainder.rs @@ -0,0 +1,241 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +/// From https://pytorch.org/docs/stable/generated/torch.remainder.html +#[test] +fn should_support_remainder_basic() { + let device = Default::default(); + let lhs = + TestTensor::<1>::from_data(TensorData::from([-3.0, -2.0, -1.0, 1.0, 2.0, 3.0]), &device); + let rhs = TestTensor::<1>::from_data(TensorData::from([2.0, 3.0, 1.0, 2.0, 1.0, 3.0]), &device); + let output = lhs.remainder(rhs); + let expected = TensorData::from([1.0, 1.0, -0.0, 1.0, 0.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_remainder_basic_scalar() { + let data = TensorData::from([-3.0, -2.0, -1.0, 1.0, 2.0, 3.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.remainder_scalar(2.0); + let expected = TensorData::from([1.0, 0.0, 1.0, 1.0, 0.0, 1.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_remainder_float() { + let device = Default::default(); + let lhs = TestTensor::<1>::from_data(TensorData::from([1.0, 2.0, 3.0, 4.0, 5.0]), &device); + let rhs = TestTensor::<1>::from_data( + TensorData::from([1.4233, 2.7313, 0.2641, 1.9651, 0.5897]), + &device, + ); + let output = lhs.remainder(rhs); + let expected = TensorData::from([1.0, 2.0, 0.0949, 0.0698, 0.2824]); + + // Metal has less precise remainder function + let tolerance = Tolerance::default() + .set_half_precision_relative(1e-2) + .set_half_precision_absolute(2e-3); + + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +/// Also from https://pytorch.org/docs/stable/generated/torch.remainder.html +#[test] +fn should_support_remainder_float_scalar() { + let data = TensorData::from([1.0, 2.0, 3.0, 4.0, 5.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.clone().remainder_scalar(-1.5); + let expected = TensorData::from([-0.5, -1.0, 0.0, -0.5, -1.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_be_zero() { + let device = Default::default(); + let lhs = TestTensor::<1>::from_data(TensorData::from([0.0, 0.0, 0.0]), &device); + let rhs = TestTensor::<1>::from_data(TensorData::from([3.5, -2.1, 1e-4]), &device); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([0.0, 0.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_be_zero_scalar() { + let data = TensorData::from([0.0, 0.0, 0.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.clone().remainder_scalar(3.5); + let expected = TensorData::from([0.0, 0.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_have_no_remainder() { + let device = Default::default(); + let lhs = TestTensor::<1>::from_data( + // Previous values failed on some vulkan backends (driver bug?) + // TensorData::from([-1.4843, 1.1350, -2.1563, 1.0862, 0.5, 3.6587]), + TensorData::from([-1.0, 1.5, -2.0, 2.5, 0.5, 4.0]), + &device, + ); + let rhs = TestTensor::<1>::from_data( + // TensorData::from([1.4843, 1.1350, 2.1563, 1.0862, 0.5, 3.6587]), + TensorData::from([1.0, 1.5, 2.0, 2.5, 0.5, 4.0]), + &device, + ); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([-0., 0., -0., 0., 0., 0.]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_have_no_remainder_scalar() { + let data = TensorData::from([-4.0, 4.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.remainder_scalar(4.0); + let expected = TensorData::from([-0.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_be_negative() { + let device = Default::default(); + + let lhs = TestTensor::<1>::from_data(TensorData::from([-7.0, -3.0, 2.0, 6.0]), &device); + let rhs = TestTensor::<1>::from_data(TensorData::from([-2.5, -2.1, -1.5, -3.25]), &device); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([-2.0, -0.9, -1.0, -0.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_be_negative_scalar() { + let data = TensorData::from([-7.0, -3.0, 2.0, 6.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.clone().remainder_scalar(-2.5); + let expected = TensorData::from([-2.0, -0.50, -0.50, -1.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_fp_dividends() { + let data = TensorData::from([-7.5, -2.5, 2.5, 7.5]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.remainder_scalar(3.0); + let expected = TensorData::from([1.5, 0.5, 2.5, 1.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + // for tensor.remainder case, tests above have already covered float point dividend cases +} + +#[test] +fn should_support_large_divisor() { + let device = Default::default(); + + let lhs = TestTensor::<1>::from_data( + TensorData::from([-1.0, 1.0, -1.5, 1.5, -1.0, 1.0, -1.5, 1.5]), + &device, + ); + let rhs = TestTensor::<1>::from_data( + TensorData::from([10.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, -10.0]), + &device, + ); + let output = lhs.remainder(rhs); + let expected = TensorData::from([9.0, 1.0, 8.5, 1.5, -1.0, -9.0, -1.5, -8.5]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_large_divisor_scalar() { + let data = TensorData::from([-1.0, 1.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.remainder_scalar(10.0); + let expected = TensorData::from([9.0, 1.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_remainder_op() { + let device = Default::default(); + let lhs = + TestTensor::<1>::from_data(TensorData::from([-3.0, -2.0, -1.0, 1.0, 2.0, 3.0]), &device); + let rhs = TestTensor::<1>::from_data(TensorData::from([2.0, 3.0, 1.0, 2.0, 1.0, 3.0]), &device); + + let output = lhs % rhs; + let expected = TensorData::from([1.0, 1.0, -0.0, 1.0, 0.0, 0.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_remainder_scalar_op() { + let data = TensorData::from([-3.0, -2.0, -1.0, 1.0, 2.0, 3.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor % 2.0; + let expected = TensorData::from([1.0, 0.0, 1.0, 1.0, 0.0, 1.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat.rs new file mode 100644 index 0000000..be843ef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat.rs @@ -0,0 +1,108 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_repeat_ops_one_dimension() { + let data = TensorData::from([[0.0f32, 1.0f32, 2.0f32]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.repeat(&[4, 1, 1]); + let expected = TensorData::from([ + [0.0f32, 1.0f32, 2.0f32], + [0.0f32, 1.0f32, 2.0f32], + [0.0f32, 1.0f32, 2.0f32], + [0.0f32, 1.0f32, 2.0f32], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_float_repeat_repeating_on_many_dimensions() { + let data = TensorData::from([ + [[1.0f32, 2.0f32], [3.0f32, 4.0f32]], + [[5.0f32, 6.0f32], [7.0f32, 8.0f32]], + [[9.0f32, 10.0f32], [11.0f32, 12.0f32]], + [[13.0f32, 14.0f32], [15.0f32, 16.0f32]], + ]); + let tensor = TestTensor::<3>::from_data(data, &Default::default()); + + let output = tensor.repeat(&[2, 3, 2]); + let expected = TensorData::from([ + [ + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + ], + [ + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + ], + [ + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + ], + [ + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + ], + [ + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + ], + [ + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + ], + [ + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + ], + [ + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_repeat_0_times_empty() { + let tensor = TestTensor::<3>::ones([2, 3, 4], &Default::default()); + + let output = tensor.repeat(&[1, 0, 2]); + + assert_eq!(output.shape(), [2, 0, 8].into()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat_dim.rs new file mode 100644 index 0000000..267ae38 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/repeat_dim.rs @@ -0,0 +1,166 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_repeat_ops() { + let data = TensorData::from([[0.0f64, 1.0f64, 2.0f64]]); + let tensor = TestTensor::<2>::from_data(data.clone(), &Default::default()); + + let output = tensor.repeat_dim(0, 4); + let expected = TensorData::from([ + [0.0f32, 1.0f32, 2.0f32], + [0.0f32, 1.0f32, 2.0f32], + [0.0f32, 1.0f32, 2.0f32], + [0.0f32, 1.0f32, 2.0f32], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_float_repeat_on_dims_larger_than_1() { + let data = TensorData::from([ + [[1.0f32, 2.0f32], [3.0f32, 4.0f32]], + [[5.0f32, 6.0f32], [7.0f32, 8.0f32]], + [[9.0f32, 10.0f32], [11.0f32, 12.0f32]], + [[13.0f32, 14.0f32], [15.0f32, 16.0f32]], + ]); + let tensor = TestTensor::<3>::from_data(data, &Default::default()); + + let output = tensor.repeat_dim(2, 2); + let expected = TensorData::from([ + [ + [1.0f32, 2.0f32, 1.0f32, 2.0f32], + [3.0f32, 4.0f32, 3.0f32, 4.0f32], + ], + [ + [5.0f32, 6.0f32, 5.0f32, 6.0f32], + [7.0f32, 8.0f32, 7.0f32, 8.0f32], + ], + [ + [9.0f32, 10.0f32, 9.0f32, 10.0f32], + [11.0f32, 12.0f32, 11.0f32, 12.0f32], + ], + [ + [13.0f32, 14.0f32, 13.0f32, 14.0f32], + [15.0f32, 16.0f32, 15.0f32, 16.0f32], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn repeat_dim_swap_dims_1() { + let tensor = TestTensorInt::arange(0..16, &Default::default()).float(); + + let tensor = tensor.reshape([4, 1, 4]); + let tensor = tensor.swap_dims(0, 2); + let output = tensor.repeat_dim(1, 4); + + let expected = TensorData::from([ + [ + [0.0, 4.0, 8.0, 12.0], + [0.0, 4.0, 8.0, 12.0], + [0.0, 4.0, 8.0, 12.0], + [0.0, 4.0, 8.0, 12.0], + ], + [ + [1.0, 5.0, 9.0, 13.0], + [1.0, 5.0, 9.0, 13.0], + [1.0, 5.0, 9.0, 13.0], + [1.0, 5.0, 9.0, 13.0], + ], + [ + [2.0, 6.0, 10.0, 14.0], + [2.0, 6.0, 10.0, 14.0], + [2.0, 6.0, 10.0, 14.0], + [2.0, 6.0, 10.0, 14.0], + ], + [ + [3.0, 7.0, 11.0, 15.0], + [3.0, 7.0, 11.0, 15.0], + [3.0, 7.0, 11.0, 15.0], + [3.0, 7.0, 11.0, 15.0], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn repeat_dim_swap_dims_2() { + let tensor = TestTensorInt::arange(0..16, &Default::default()).float(); + + let tensor = tensor.reshape([2, 2, 1, 4]); + let tensor = tensor.swap_dims(0, 1); + let output = tensor.repeat_dim(2, 4); + + let expected = TensorData::from([ + [ + [ + [0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0], + ], + [ + [8.0, 9.0, 10.0, 11.0], + [8.0, 9.0, 10.0, 11.0], + [8.0, 9.0, 10.0, 11.0], + [8.0, 9.0, 10.0, 11.0], + ], + ], + [ + [ + [4.0, 5.0, 6.0, 7.0], + [4.0, 5.0, 6.0, 7.0], + [4.0, 5.0, 6.0, 7.0], + [4.0, 5.0, 6.0, 7.0], + ], + [ + [12.0, 13.0, 14.0, 15.0], + [12.0, 13.0, 14.0, 15.0], + [12.0, 13.0, 14.0, 15.0], + [12.0, 13.0, 14.0, 15.0], + ], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn repeat_dim_swap_dims_3() { + let tensor = TestTensorInt::arange(0..16, &Default::default()).float(); + + let tensor = tensor.reshape([1, 2, 2, 4]); + let tensor = tensor.swap_dims(0, 2); + let tensor = tensor.swap_dims(1, 3); + let output = tensor.repeat_dim(2, 4); + + let expected = TensorData::from([ + [ + [[0.0, 8.0], [0.0, 8.0], [0.0, 8.0], [0.0, 8.0]], + [[1.0, 9.0], [1.0, 9.0], [1.0, 9.0], [1.0, 9.0]], + [[2.0, 10.0], [2.0, 10.0], [2.0, 10.0], [2.0, 10.0]], + [[3.0, 11.0], [3.0, 11.0], [3.0, 11.0], [3.0, 11.0]], + ], + [ + [[4.0, 12.0], [4.0, 12.0], [4.0, 12.0], [4.0, 12.0]], + [[5.0, 13.0], [5.0, 13.0], [5.0, 13.0], [5.0, 13.0]], + [[6.0, 14.0], [6.0, 14.0], [6.0, 14.0], [6.0, 14.0]], + [[7.0, 15.0], [7.0, 15.0], [7.0, 15.0], [7.0, 15.0]], + ], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_repeat_dim_0_times_empty() { + let tensor = TestTensor::<3>::ones([2, 3, 4], &Default::default()); + + let output = tensor.repeat_dim(2, 0); + + assert_eq!(output.shape(), [2, 3, 0].into()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/reshape.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/reshape.rs new file mode 100644 index 0000000..4b1a772 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/reshape.rs @@ -0,0 +1,90 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_rank() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + assert_eq!(tensor.rank(), 1); + + let data = TensorData::from([[0.0, 1.0, 2.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + assert_eq!(tensor.rank(), 2); +} + +#[test] +fn should_support_reshape_1d() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.clone().reshape([1, 3]); + let expected = TensorData::from([[0.0, 1.0, 2.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_reshape_2d() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.clone().reshape([6]); + let expected = TensorData::from([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_dim_infererence() { + let data = TensorData::from([ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0], + ]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + // Infer the dimension via -1 + let reshaped = tensor.clone().reshape([2, -1]); + assert_eq!(reshaped.shape(), [2, 6].into()); + + // Infer the dimension via 0 (keep from the source) and -1 (infer) + let reshaped = reshaped.reshape([0, 2, -1]); + assert_eq!(reshaped.shape(), [2, 2, 3].into()); + + // This is effectively as if we did a flatten + let reshaped = tensor.clone().reshape([-1]); + assert_eq!(reshaped.shape(), [12].into()); + + // Keeping the first dimension the same (using 0) + let reshaped = tensor.clone().reshape([0, 3]); + assert_eq!(reshaped.shape(), [4, 3].into()); +} + +#[test] +fn should_not_corrupt_after_slice() { + let zeros = TestTensor::<1>::zeros([2], &Default::default()); + zeros.clone().slice([1..2]).reshape([1]).exp(); + + // May lead to zeroes being equal to [0.0, 1.0] + zeros.into_data().assert_eq( + &TestTensor::<1>::zeros([2], &Default::default()).to_data(), + true, + ); +} + +#[test] +#[should_panic] +fn multiple_neg_ones() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + let _data_actual = tensor.reshape([-1, -1]).into_data(); +} + +#[test] +#[should_panic] +fn neg_value() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + let _data_actual = tensor.reshape([-2, -1]).into_data(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/round.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/round.rs new file mode 100644 index 0000000..f6161da --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/round.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_round_ops() { + let data = TensorData::from([[24.0423, 87.9478, 76.1838], [59.6929, 43.8169, 94.8826]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.round(); + let expected = TensorData::from([[24., 88., 76.], [60., 44., 95.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_round_ties_even() { + let data = TensorData::from([1.5, 2.5, 3.5, 4.5, 5.5, 6.5]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.round(); + let expected = TensorData::from([2., 2., 4., 4., 6., 6.]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/select.rs new file mode 100644 index 0000000..ebcc356 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/select.rs @@ -0,0 +1,235 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, TensorData}; + +#[test] +fn should_select_1d() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([0.0, 1.0, 2.0], &device); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([1.0, 1.0, 0.0, 1.0, 2.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_2d_dim0_same_num_dim() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_data([1, 0], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([[3.0, 4.0, 5.0], [0.0, 1.0, 2.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_2d_dim0_more_num_dim() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_data([1, 0, 1, 1], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([ + [3.0, 4.0, 5.0], + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [3.0, 4.0, 5.0], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_2d_dim0_vec() { + let device = Default::default(); + let tensor = + TestTensor::<2>::from_data([[0.0, 1.0], [2.0, 3.0], [4.0, 5.0], [6.0, 7.0]], &device); + let indices = TestTensorInt::from_data([1, 0, 3, 2], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([[2.0, 3.0], [0.0, 1.0], [6.0, 7.0], [4.0, 5.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_2d_dim1() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &device); + + let output = tensor.select(1, indices); + let expected = TensorData::from([[1.0, 1.0, 0.0, 1.0, 2.0], [4.0, 4.0, 3.0, 4.0, 5.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_1d() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([0.0, 1.0, 2.0], &device); + let values = TestTensor::from_data([5.0, 4.0, 3.0, 2.0, 1.0], &device); + let indices = TestTensorInt::from_data(TensorData::from([1, 1, 0, 1, 2]), &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([3.0, 12.0, 3.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_1d_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data([7, 8, 9], &device); + let values = TestTensorInt::from_data([5, 4, 3, 2, 1], &device); + let indices = TestTensorInt::from_data(TensorData::from([1, 1, 0, 1, 2]), &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([10, 19, 10]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_2d_dim0() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let values = TestTensor::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let indices = TestTensorInt::from_data(TensorData::from([1, 0]), &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([[4.0, 6.0, 8.0], [4.0, 6.0, 8.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_2d_dim1() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let values = TestTensor::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let indices = TestTensorInt::from_data(TensorData::from([1, 0, 2]), &device); + + let output = tensor.select_assign(1, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([[2.0, 2.0, 5.0], [8.0, 8.0, 11.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_3d_dim1_vec() { + let device = Default::default(); + let tensor = TestTensor::<3>::from_data( + [ + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]], + [[-1.0, -2.0], [-3.0, -4.0], [-5.0, -6.0], [-7.0, -8.0]], + ], + &device, + ); + let indices = TestTensorInt::from_data([1, 0, 3, 2], &device); + + let output = tensor.select(1, indices); + let expected = TensorData::from([ + [[3.0, 4.0], [1.0, 2.0], [7.0, 8.0], [5.0, 6.0]], + [[-3.0, -4.0], [-1.0, -2.0], [-7.0, -8.0], [-5.0, -6.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn should_select_panic_invalid_dimension() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &device); + + tensor.select(10, indices); +} + +#[test] +fn should_match_default_implementation_behavior() { + // Verify optimized implementation matches original default logic + let device = Default::default(); + let tensor = TestTensorBool::<1>::from_data([true, false, true], &device); + let indices = TestTensorInt::from_data([0, 1, 0], &device); + let values = TestTensorBool::<1>::from_data([false, true, true], &device); + + let optimized_result = + tensor + .clone() + .select_assign(0, indices.clone(), values.clone(), IndexingUpdateOp::Add); + + // Manual default implementation logic + let int_tensor = tensor.int(); + let int_values = values.int(); + let assigned = int_tensor.select_assign(0, indices, int_values, IndexingUpdateOp::Add); + let default_result = assigned.greater_elem(0); + + optimized_result + .into_data() + .assert_eq(&default_result.into_data(), false); +} + +#[test] +fn should_select_with_negative_dim_2d() { + // Test using negative dimension indexing on 2D tensor + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::from_data([1, 0, 2], &device); + + // Using -1 should refer to the last dimension (dim 1) + let output_neg = tensor.clone().select(-1, indices.clone()); + let output_pos = tensor.select(1, indices); + + // Both should produce the same result + output_neg + .into_data() + .assert_eq(&output_pos.into_data(), false); +} + +#[test] +fn should_select_add_with_negative_dim_2d() { + // Test select_add with negative dimension on 2D tensor + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let values = TestTensor::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let indices = TestTensorInt::from_data([0, 2], &device); + + // Using -1 should refer to the last dimension (dim 1) + let output_neg = + tensor + .clone() + .select_assign(-1, indices.clone(), values.clone(), IndexingUpdateOp::Add); + let output_pos = tensor.select_assign(1, indices, values, IndexingUpdateOp::Add); + + output_neg + .into_data() + .assert_eq(&output_pos.into_data(), false); +} + +#[test] +#[should_panic] +fn should_panic_select_negative_dim_out_of_bounds() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let indices = TestTensorInt::from_data([0, 1], &device); + + // This should panic because -3 is out of bounds for a 2D tensor + tensor.select(-3, indices); +} + +#[test] +#[should_panic] +fn should_panic_select_add_negative_dim_out_of_bounds() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let values = TestTensor::from_data([[5.0], [6.0]], &device); + let indices = TestTensorInt::from_data([0], &device); + + // This should panic because -3 is out of bounds for a 2D tensor + tensor.select_assign(-3, indices, values, IndexingUpdateOp::Add); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sign.rs new file mode 100644 index 0000000..6b71edc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sign.rs @@ -0,0 +1,12 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_sign_ops_float() { + let tensor = TestTensor::<2>::from([[-0.2, -1.0, 2.0], [3.0, 0.0, -5.0]]); + + let output = tensor.sign(); + let expected = TensorData::from([[-1.0, -1.0, 1.0], [1.0, 0.0, -1.0]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice.rs new file mode 100644 index 0000000..ade65b5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice.rs @@ -0,0 +1,591 @@ +use super::*; +use burn_tensor::{ElementConversion, Slice, TensorData, s}; + +#[test] +fn should_support_slice_dim_1d() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data.clone(), &Default::default()); + + // Test with range (negative index) + let output = tensor.clone().slice_dim(0, -2..); + output + .into_data() + .assert_eq(&TensorData::from([1.0, 2.0]), false); + + // Test with Slice directly + let slice = Slice::new(1, None, 1); // equivalent to 1.. + let output = tensor.slice_dim(0, slice); + output + .into_data() + .assert_eq(&TensorData::from([1.0, 2.0]), false); +} + +#[test] +#[should_panic(expected = "The provided dimension exceeds the tensor dimensions")] +fn should_panic_when_slice_dim_1d_bad_dim() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data.clone(), &Default::default()); + + let _output = tensor.slice_dim(1, 1..); +} + +#[test] +fn should_support_slice_dim_2d() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data.clone(), &Default::default()); + + let output = tensor.slice_dim(1, 1..); + output + .into_data() + .assert_eq(&TensorData::from([[1.0, 2.0], [4.0, 5.0]]), false); +} + +#[test] +fn should_support_slice_dim_with_step() { + let data = TensorData::from([[0.0, 1.0, 2.0, 3.0], [4.0, 5.0, 6.0, 7.0]]); + let tensor = TestTensor::<2>::from_data(data.clone(), &Default::default()); + + // Test 1: Slice dimension 1 with step=2 using s! macro + let output = tensor.clone().slice_dim(1, s![0..4;2]); + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 2.0], [4.0, 6.0]]), false); + + // Test 2: Slice dimension 1 with step=2 using Slice directly + let slice = Slice::new(0, Some(4), 2); + let output = tensor.slice_dim(1, slice); + output + .into_data() + .assert_eq(&TensorData::from([[0.0, 2.0], [4.0, 6.0]]), false); +} + +#[test] +fn should_support_slice_dim_with_negative_step() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data.clone(), &Default::default()); + + // Slice dimension 1 with negative step (reverse columns) + let output = tensor.slice_dim(1, s![..;-1]); + output + .into_data() + .assert_eq(&TensorData::from([[2.0, 1.0, 0.0], [5.0, 4.0, 3.0]]), false); +} + +#[test] +fn should_support_full_sliceing_1d() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data.clone(), &Default::default()); + + let output = tensor.slice([0..3]); + + output.into_data().assert_eq(&data, false); +} + +#[test] +fn should_support_full_sliceing_vec() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data.clone(), &Default::default()); + + let slices: Vec = vec![(0..2).into()]; + + let output = tensor.clone().slice(&slices); + output.into_data().assert_eq(&data, false); + + let output = tensor.slice([0..2, 0..3]); + output.into_data().assert_eq(&data, false); +} + +#[test] +fn should_support_partial_sliceing_1d() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.slice([1..3]); + let expected = TensorData::from([1.0, 2.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_full_sliceing_2d() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data.clone(), &Default::default()); + + let output = tensor.clone().slice([0..2]); + output.into_data().assert_eq(&data, false); + + let output = tensor.slice([0..2, 0..3]); + output.into_data().assert_eq(&data, false); +} + +#[test] +fn should_support_partial_sliceing_2d() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.slice([0..2, 0..2]); + let expected = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_range_first_dim() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.slice(0..1); + let expected = TensorData::from([[0.0, 1.0, 2.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_partial_sliceing_3d() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let output = tensor.slice([1..2, 1..2, 0..2]); + let expected = TensorData::from([[[9.0, 10.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_partial_sliceing_3d_non_contiguous() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let output = tensor.transpose().slice([1..2, 1..2, 0..2]); + let expected = TensorData::from([[[7.0, 10.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_1d() { + let data = TensorData::from([0.0, 1.0, 2.0]); + + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.slice_fill([0..2], -1.0); + let expected = TensorData::from([-1.0, -1.0, 2.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_vec() { + let data = TensorData::from([0.0, 1.0, 2.0]); + + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let slices: Vec = vec![(0..2).into()]; + + let output = tensor.slice_fill(&slices, -1.0); + let expected = TensorData::from([-1.0, -1.0, 2.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_cast_f32() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device).cast(burn_tensor::DType::F32); + + tensor + .slice_fill(s![0..2], 1.0) + .into_data() + .assert_eq(&TensorData::from([1.0, 1.0, 2.0]), false); +} + +// Skip on metal - F64 not supported +#[cfg(not(feature = "metal"))] +#[test] +fn should_support_slice_fill_cast_f64() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device).cast(burn_tensor::DType::F64); + + tensor + .slice_fill(s![0..2], 1.0) + .into_data() + .assert_eq(&TensorData::from([1.0, 1.0, 2.0]), false); +} + +#[test] +fn should_support_slice_fill_1d_neg() { + let data = TensorData::from([0.0, 1.0, 2.0]); + + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + + let output = tensor.slice_fill([-1..], -1.0); + let expected = TensorData::from([0.0, 1.0, -1.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_2d() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let device = Default::default(); + let tensor = TestTensor::<2>::from_data(data, &device); + + let output = tensor.slice_fill([1..2, 0..2], -1.0); + let expected = TensorData::from([[0.0, 1.0, 2.0], [-1.0, -1.0, 5.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_with_positive_step() { + let device = Default::default(); + + // Test 1D tensor with step + let tensor = TestTensor::<1>::zeros([10], &device); + let output = tensor.slice_fill(s![0..10;2], 5.0); + let expected = TensorData::from([5.0, 0.0, 5.0, 0.0, 5.0, 0.0, 5.0, 0.0, 5.0, 0.0]); + output.into_data().assert_eq(&expected, false); + + // Test 2D tensor with step on first dimension + let tensor = TestTensor::<2>::zeros([4, 4], &device); + let output = tensor.slice_fill(s![0..4;2, ..], 3.0); + let expected = TensorData::from([ + [3.0, 3.0, 3.0, 3.0], + [0.0, 0.0, 0.0, 0.0], + [3.0, 3.0, 3.0, 3.0], + [0.0, 0.0, 0.0, 0.0], + ]); + output.into_data().assert_eq(&expected, false); + + // Test 2D tensor with step on second dimension + let tensor = TestTensor::<2>::zeros([3, 6], &device); + let output = tensor.slice_fill(s![.., 0..6;3], 2.0); + let expected = TensorData::from([ + [2.0, 0.0, 0.0, 2.0, 0.0, 0.0], + [2.0, 0.0, 0.0, 2.0, 0.0, 0.0], + [2.0, 0.0, 0.0, 2.0, 0.0, 0.0], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_with_negative_step() { + let device = Default::default(); + + // Test 1D tensor with negative step (reverse fill) + let tensor = TestTensor::<1>::from_data([1.0, 2.0, 3.0, 4.0, 5.0], &device); + let output = tensor.slice_fill(s![0..5;-1], 10.0); + // Should reverse the indices [4,3,2,1,0] and fill them with 10.0 + let expected = TensorData::from([10.0, 10.0, 10.0, 10.0, 10.0]); + output.into_data().assert_eq(&expected, false); + + // Test 2D tensor with negative step + let tensor = + TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], &device); + let output = tensor.slice_fill(s![.., 0..3;-2], -1.0); + // Should fill columns in reverse order with step 2: indices 2, 0 + let expected = TensorData::from([[-1.0, 2.0, -1.0], [-1.0, 5.0, -1.0], [-1.0, 8.0, -1.0]]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_fill_with_mixed_steps() { + let device = Default::default(); + + // Test 2D tensor with mixed positive and negative steps + let tensor = TestTensor::<2>::zeros([4, 6], &device); + let output = tensor.slice_fill(s![0..4;2, 0..6;-3], 7.0); + // Step 2 on dim 0 selects rows 0, 2 + // Step -3 on dim 1 with range 0..6 reverses and takes every 3rd: indices [5, 2] + let expected = TensorData::from([ + [0.0, 0.0, 7.0, 0.0, 0.0, 7.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 7.0, 0.0, 0.0, 7.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ]); + output.into_data().assert_eq(&expected, false); + + // Test 3D tensor with steps + let tensor = TestTensor::<3>::zeros([2, 4, 4], &device); + let output = tensor.slice_fill(s![.., 0..4;2, 0..4;-2], 1.0); + // Step 2 on dim 1 selects rows 0, 2 + // Step -2 on dim 2 with range 0..4 reverses and takes every 2nd: indices [3, 1] + let expected_slice = [ + [0.0, 1.0, 0.0, 1.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 1.0], + [0.0, 0.0, 0.0, 0.0], + ]; + let expected = TensorData::from([expected_slice, expected_slice]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn clamp_when_slice_exceeds_dimension() { + let tensor = TestTensor::<1>::from([0.0, 1.0, 2.0]); + let data = tensor.to_data(); + + let output = tensor.slice([0..4]); + output.into_data().assert_eq(&data, true); +} + +#[test] +fn negative_dimensions() { + let tensor = TestTensor::<2>::from([[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data = tensor.to_data(); + + // Clamping to the tensor dimensions + let output = tensor.clone().slice([0..4, 0..4]); + output.into_data().assert_eq(&data, true); + + // Negative dimensions + let output = tensor.clone().slice([0..1, 0..1]); + let data = TensorData::from([[0.elem::()]]); + output.into_data().assert_eq(&data, true); + + let output = tensor.slice(s![0..-1, 0..-2]); + output.into_data().assert_eq(&data, true); +} + +#[test] +fn missing_dimensions() { + let tensor = TestTensor::<2>::from([[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data = tensor.to_data(); + + // Clamping to the tensor dimensions + let output = tensor.clone().slice([0..4, 0..4]); + output.into_data().assert_eq(&data, true); + + // Negative dimensions + let data = TensorData::from([[0.elem::()]]); + let output = tensor.clone().slice(s![0..-1, 0..-2]); + output.into_data().assert_eq(&data, true); + + // Missing dimensions + let output = tensor.clone().slice(s![0..1, ..]); + let data = TensorData::from([[0.0f32, 1.0, 2.0]]); + output.into_data().assert_eq(&data, false); + + let output = tensor.clone().slice(s![.., 0..2]); + let data = TensorData::from([[0.0f32, 1.0], [3.0, 4.0]]); + output.into_data().assert_eq(&data, false); + + let output = tensor.clone().slice([.., ..]); + let data = TensorData::from([[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]]); + output.into_data().assert_eq(&data, false); +} + +#[test] +fn should_slice_aggregation_result() { + let tensor = TestTensor::<1>::from([0.0, 1.0, 2.0]).mean(); + + let output = tensor.clone().slice([(0..1)]); + output.into_data().assert_eq(&tensor.into_data(), true); +} + +#[test] +#[should_panic] +fn should_panic_when_slice_with_too_many_dimensions() { + let tensor = TestTensor::<1>::from([0.0, 1.0, 2.0]); + + let _output = tensor.slice([0..1, 0..1]); +} + +#[test] +fn should_support_descending_slice_as_empty() { + // Like PyTorch, x[3:1] should return an empty tensor, not panic + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.slice(s![2..1]); + + // Should produce an empty tensor with shape [0] + assert_eq!(output.dims(), [0]); +} + +#[test] +fn should_support_empty_slice() { + // ONNX models can have empty slices where start == end + // This should produce a tensor with size 0 in that dimension + let data = TensorData::from([0.0, 1.0, 2.0]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.slice([1..1]); + + // Should produce an empty tensor with shape [0] + assert_eq!(output.dims(), [0]); +} + +#[test] +fn should_support_empty_slice_2d() { + // Test empty slice on 2D tensor + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + // Empty slice on first dimension + let output = tensor.clone().slice([1..1, 0..3]); + assert_eq!(output.dims(), [0, 3]); + + // Empty slice on second dimension + let output = tensor.slice([0..2, 2..2]); + assert_eq!(output.dims(), [2, 0]); +} + +#[test] +fn test_slice_with_positive_step() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + &device, + ); + + // Test step=2 along first dimension + let sliced = tensor.clone().slice([s![0..3;2]]); + let expected = TensorData::from([[1.0, 2.0, 3.0, 4.0], [9.0, 10.0, 11.0, 12.0]]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=2 along second dimension + let sliced = tensor.clone().slice(s![.., 0..4;2]); + let expected = TensorData::from([[1.0, 3.0], [5.0, 7.0], [9.0, 11.0]]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=2 along both dimensions + let sliced = tensor.clone().slice(s![0..3;2, 0..4;2]); + let expected = TensorData::from([[1.0, 3.0], [9.0, 11.0]]); + sliced.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_with_negative_step() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + &device, + ); + + // Test step=-1 along first dimension (reverse rows) + let sliced = tensor.clone().slice([s![0..3;-1]]); + let expected = TensorData::from([ + [9.0, 10.0, 11.0, 12.0], + [5.0, 6.0, 7.0, 8.0], + [1.0, 2.0, 3.0, 4.0], + ]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=-1 along second dimension (reverse columns) + let sliced = tensor.clone().slice(s![.., 0..4;-1]); + let expected = TensorData::from([ + [4.0, 3.0, 2.0, 1.0], + [8.0, 7.0, 6.0, 5.0], + [12.0, 11.0, 10.0, 9.0], + ]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=-2 along first dimension + let sliced = tensor.clone().slice([s![0..3;-2]]); + let expected = TensorData::from([[9.0, 10.0, 11.0, 12.0], [1.0, 2.0, 3.0, 4.0]]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=-2 along second dimension + let sliced = tensor.clone().slice(s![.., 0..4;-2]); + let expected = TensorData::from([[4.0, 2.0], [8.0, 6.0], [12.0, 10.0]]); + sliced.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_with_mixed_steps() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + &device, + ); + + // Test positive step along first dimension, negative along second + let sliced = tensor.clone().slice(s![0..3;2, 0..4;-1]); + let expected = TensorData::from([[4.0, 3.0, 2.0, 1.0], [12.0, 11.0, 10.0, 9.0]]); + sliced.into_data().assert_eq(&expected, false); + + // Test negative step along first dimension, positive along second + let sliced = tensor.clone().slice(s![0..3;-1, 0..4;2]); + let expected = TensorData::from([[9.0, 11.0], [5.0, 7.0], [1.0, 3.0]]); + sliced.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_with_steps_1d() { + let device = Default::default(); + let tensor = + TestTensor::<1>::from_data([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], &device); + + // Test positive step + let sliced = tensor.clone().slice([s![0..10;2]]); + let expected = TensorData::from([1.0, 3.0, 5.0, 7.0, 9.0]); + sliced.into_data().assert_eq(&expected, false); + + // Test negative step + let sliced = tensor.clone().slice([s![0..10;-1]]); + let expected = TensorData::from([10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]); + sliced.into_data().assert_eq(&expected, false); + + // Test negative step with partial range + let sliced = tensor.clone().slice([s![2..8;-2]]); + let expected = TensorData::from([8.0, 6.0, 4.0]); + sliced.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_with_steps_3d() { + let device = Default::default(); + let tensor = TestTensor::<3>::from_data( + [ + [[1.0, 2.0], [3.0, 4.0]], + [[5.0, 6.0], [7.0, 8.0]], + [[9.0, 10.0], [11.0, 12.0]], + [[13.0, 14.0], [15.0, 16.0]], + ], + &device, + ); + + // Test step=2 along first dimension + let sliced = tensor.clone().slice(s![0..4;2, .., ..]); + let expected = TensorData::from([[[1.0, 2.0], [3.0, 4.0]], [[9.0, 10.0], [11.0, 12.0]]]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=-1 along all dimensions + let sliced = tensor.clone().slice(s![0..4;-1, 0..2;-1, 0..2;-1]); + let expected = TensorData::from([ + [[16.0, 15.0], [14.0, 13.0]], + [[12.0, 11.0], [10.0, 9.0]], + [[8.0, 7.0], [6.0, 5.0]], + [[4.0, 3.0], [2.0, 1.0]], + ]); + sliced.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice_assign.rs new file mode 100644 index 0000000..8799f5d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/slice_assign.rs @@ -0,0 +1,366 @@ +use super::*; +use burn_tensor::{Slice, TensorData, s}; + +#[test] +fn should_support_slice_assign_1d() { + let data = TensorData::from([0.0, 1.0, 2.0]); + let data_assigned = TensorData::from([10.0, 5.0]); + + let device = Default::default(); + let tensor = TestTensor::<1>::from_data(data, &device); + let tensor_assigned = TestTensor::<1>::from_data(data_assigned, &device); + + let output = tensor.slice_assign([0..2], tensor_assigned); + let expected = TensorData::from([10.0, 5.0, 2.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_assign_2d() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data_assigned = TensorData::from([[10.0, 5.0]]); + + let device = Default::default(); + let tensor = TestTensor::<2>::from_data(data, &device); + let tensor_assigned = TestTensor::<2>::from_data(data_assigned, &device); + + let output = tensor.slice_assign([1..2, 0..2], tensor_assigned); + let expected = TensorData::from([[0.0, 1.0, 2.0], [10.0, 5.0, 5.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_assign_vec() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data_assigned = TensorData::from([[10.0, 5.0]]); + + let device = Default::default(); + let tensor = TestTensor::<2>::from_data(data, &device); + let tensor_assigned = TestTensor::<2>::from_data(data_assigned, &device); + + let slices: Vec = vec![1..2, 0..2].into_iter().map(Slice::from).collect(); + + let output = tensor.slice_assign(&slices, tensor_assigned); + let expected = TensorData::from([[0.0, 1.0, 2.0], [10.0, 5.0, 5.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn slice_assign_now_supports_non_unit_step() { + let device = Default::default(); + // Create tensors where the shapes match for stepped slicing + let tensor = TestTensor::<2>::ones([4, 4], &device); + // With step=2 on first dim, we select indices 0 and 2, so we need a [2, 4] values tensor + let values = TestTensor::<2>::zeros([2, 4], &device); + + // This now works because slice_assign supports steps != 1 + // We use s! macro to create a slice with step=2 + let result = tensor.slice_assign(s![0..3;2, ..], values); + + // Verify the result: rows 0 and 2 should be zeros, rows 1 and 3 should be ones + let expected = TensorData::from([ + [0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 1.0], + ]); + result.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_with_positive_step_1d() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], &device); + let values = TestTensor::<1>::from_data([10.0, 20.0, 30.0], &device); + + // Assign to indices 0, 2, 4 (step=2) + let output = tensor.slice_assign([s![0..6;2]], values); + let expected = TensorData::from([10.0, 2.0, 20.0, 4.0, 30.0, 6.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_with_positive_step_2d() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ], + &device, + ); + + // Assign to rows 0, 2 (step=2) + let values = TestTensor::<2>::from_data( + [[100.0, 101.0, 102.0, 103.0], [200.0, 201.0, 202.0, 203.0]], + &device, + ); + let output = tensor.clone().slice_assign([s![0..4;2]], values); + let expected = TensorData::from([ + [100.0, 101.0, 102.0, 103.0], + [5.0, 6.0, 7.0, 8.0], + [200.0, 201.0, 202.0, 203.0], + [13.0, 14.0, 15.0, 16.0], + ]); + output.into_data().assert_eq(&expected, false); + + // Assign to columns 0, 2 (step=2) + let values = TestTensor::<2>::from_data( + [ + [100.0, 200.0], + [101.0, 201.0], + [102.0, 202.0], + [103.0, 203.0], + ], + &device, + ); + let output = tensor.clone().slice_assign(s![.., 0..4;2], values); + let expected = TensorData::from([ + [100.0, 2.0, 200.0, 4.0], + [101.0, 6.0, 201.0, 8.0], + [102.0, 10.0, 202.0, 12.0], + [103.0, 14.0, 203.0, 16.0], + ]); + output.into_data().assert_eq(&expected, false); + + // Assign with step=2 on both dimensions + let values = TestTensor::<2>::from_data([[100.0, 200.0], [300.0, 400.0]], &device); + let output = tensor.slice_assign(s![0..4;2, 0..4;2], values); + let expected = TensorData::from([ + [100.0, 2.0, 200.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [300.0, 10.0, 400.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_with_negative_step_1d() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], &device); + let values = TestTensor::<1>::from_data([60.0, 50.0, 40.0, 30.0, 20.0, 10.0], &device); + + // Assign in reverse order (step=-1) + let output = tensor.slice_assign([s![0..6;-1]], values); + let expected = TensorData::from([10.0, 20.0, 30.0, 40.0, 50.0, 60.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_with_negative_step_2d() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + &device, + ); + + // Assign to rows in reverse order (step=-1) + let values = TestTensor::<2>::from_data( + [ + [30.0, 31.0, 32.0, 33.0], + [20.0, 21.0, 22.0, 23.0], + [10.0, 11.0, 12.0, 13.0], + ], + &device, + ); + let output = tensor.clone().slice_assign([s![0..3;-1]], values); + let expected = TensorData::from([ + [10.0, 11.0, 12.0, 13.0], + [20.0, 21.0, 22.0, 23.0], + [30.0, 31.0, 32.0, 33.0], + ]); + output.into_data().assert_eq(&expected, false); + + // Assign to columns in reverse order (step=-1) + let values = TestTensor::<2>::from_data( + [ + [40.0, 30.0, 20.0, 10.0], + [80.0, 70.0, 60.0, 50.0], + [120.0, 110.0, 100.0, 90.0], + ], + &device, + ); + let output = tensor.clone().slice_assign(s![.., 0..4;-1], values); + let expected = TensorData::from([ + [10.0, 20.0, 30.0, 40.0], + [50.0, 60.0, 70.0, 80.0], + [90.0, 100.0, 110.0, 120.0], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_with_mixed_steps() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0], + ], + &device, + ); + + // Positive step along rows, negative along columns + let values = TestTensor::<2>::from_data( + [[100.0, 101.0, 102.0, 103.0], [200.0, 201.0, 202.0, 203.0]], + &device, + ); + let output = tensor.clone().slice_assign(s![0..4;2, 0..4;-1], values); + let expected = TensorData::from([ + [103.0, 102.0, 101.0, 100.0], + [5.0, 6.0, 7.0, 8.0], + [203.0, 202.0, 201.0, 200.0], + [13.0, 14.0, 15.0, 16.0], + ]); + output.into_data().assert_eq(&expected, false); + + // Negative step along rows, positive along columns + let values = TestTensor::<2>::from_data( + [ + [100.0, 200.0], + [101.0, 201.0], + [102.0, 202.0], + [103.0, 203.0], + ], + &device, + ); + let output = tensor.slice_assign(s![0..4;-1, 0..4;2], values); + let expected = TensorData::from([ + [103.0, 2.0, 203.0, 4.0], + [102.0, 6.0, 202.0, 8.0], + [101.0, 10.0, 201.0, 12.0], + [100.0, 14.0, 200.0, 16.0], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_3d_with_steps() { + let device = Default::default(); + let tensor = TestTensor::<3>::from_data( + [ + [[1.0, 2.0], [3.0, 4.0]], + [[5.0, 6.0], [7.0, 8.0]], + [[9.0, 10.0], [11.0, 12.0]], + [[13.0, 14.0], [15.0, 16.0]], + ], + &device, + ); + + // Test step=2 along first dimension + let values = TestTensor::<3>::from_data( + [ + [[100.0, 101.0], [102.0, 103.0]], + [[200.0, 201.0], [202.0, 203.0]], + ], + &device, + ); + let output = tensor.clone().slice_assign(s![0..4;2, .., ..], values); + let expected = TensorData::from([ + [[100.0, 101.0], [102.0, 103.0]], + [[5.0, 6.0], [7.0, 8.0]], + [[200.0, 201.0], [202.0, 203.0]], + [[13.0, 14.0], [15.0, 16.0]], + ]); + output.into_data().assert_eq(&expected, false); + + // Test step=-1 along all dimensions + let values = TestTensor::<3>::from_data( + [ + [[400.0, 399.0], [398.0, 397.0]], + [[396.0, 395.0], [394.0, 393.0]], + [[392.0, 391.0], [390.0, 389.0]], + [[388.0, 387.0], [386.0, 385.0]], + ], + &device, + ); + let output = tensor.slice_assign(s![0..4;-1, 0..2;-1, 0..2;-1], values); + let expected = TensorData::from([ + [[385.0, 386.0], [387.0, 388.0]], + [[389.0, 390.0], [391.0, 392.0]], + [[393.0, 394.0], [395.0, 396.0]], + [[397.0, 398.0], [399.0, 400.0]], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_partial_with_steps() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0, 5.0], + [6.0, 7.0, 8.0, 9.0, 10.0], + [11.0, 12.0, 13.0, 14.0, 15.0], + [16.0, 17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0, 25.0], + ], + &device, + ); + + // Assign to a subset with step=2 + let values = TestTensor::<2>::from_data([[100.0, 200.0], [300.0, 400.0]], &device); + let output = tensor.slice_assign(s![1..4;2, 1..4;2], values); + let expected = TensorData::from([ + [1.0, 2.0, 3.0, 4.0, 5.0], + [6.0, 100.0, 8.0, 200.0, 10.0], + [11.0, 12.0, 13.0, 14.0, 15.0], + [16.0, 300.0, 18.0, 400.0, 20.0], + [21.0, 22.0, 23.0, 24.0, 25.0], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_assign_empty_range() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let values: TestTensor<2> = TestTensor::empty([2, 0], &device); + + // Empty slice assignment (start == end) should be a no-op + let output = tensor.clone().slice_assign([0..2, 1..1], values); + let expected = TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_assign_empty_range_1d() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([1.0, 2.0, 3.0, 4.0, 5.0], &device); + let values: TestTensor<1> = TestTensor::empty([0], &device); + + // Empty slice assignment should return tensor unchanged + let output = tensor.clone().slice_assign([2..2], values); + let expected = TensorData::from([1.0, 2.0, 3.0, 4.0, 5.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_assign_single_dim_slice() { + let device = Default::default(); + let x = TestTensor::<3>::ones([2, 3, 1], &device); + let values = TestTensor::<3>::zeros([1, 3, 1], &device); + + let output = x.slice_assign(s![1], values); + + output.into_data().assert_eq( + &TensorData::from([[[1.0], [1.0], [1.0]], [[0.0], [0.0], [0.0]]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sort_argsort.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sort_argsort.rs new file mode 100644 index 0000000..7da93cd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sort_argsort.rs @@ -0,0 +1,216 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_sort_1d_float() { + let tensor = TestTensor::<1>::from([ + 0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 199.412, 4., 0.99, 3., -8.1, + ]); + + // Sort along dim=0 + let values = tensor.sort(0); + + let values_expected = TensorData::from([ + -8.1, -0.3, -0.21, 0., 0.5, 0.94, 0.99, 1.2, 2.1, 2.3, 3., 4., 199.412, + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); +} + +#[test] +fn test_argsort_1d_float() { + let tensor = TestTensor::<1>::from([ + 0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 199.412, 4., 0.99, 3., -8.1, + ]); + + // Sort along dim=0 + let indices = tensor.argsort(0); + + let indices_expected = TensorData::from([12, 6, 2, 3, 0, 5, 10, 1, 4, 7, 11, 9, 8]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_with_indices_descending_float() { + // 1D + let tensor = TestTensor::<1>::from([ + 0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 199.412, 4., 0.99, 3., -8.1, + ]); + + // Sort along dim=0 + let (values, indices) = tensor.sort_descending_with_indices(0); + + let values_expected = TensorData::from([ + 199.412, 4., 3., 2.3, 2.1, 1.2, 0.99, 0.94, 0.5, 0., -0.21, -0.3, -8.1, + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + let indices_expected = TensorData::from([8, 9, 11, 7, 4, 1, 10, 5, 0, 3, 2, 6, 12]); + indices.into_data().assert_eq(&indices_expected, false); + + // 2D + let tensor = TestTensor::<3>::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, 4.], [0.99, 3., -8.1]], + ]); + + // Sort along dim=1 + let (values, indices) = tensor.sort_descending_with_indices(1); + + let values_expected = TensorData::from([ + [[0., 2.1, 0.94], [-0.5, 1.2, -0.21]], + [[0.99, 3., 4.], [-0.3, 2.3, -8.1]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + let indices_expected = TensorData::from([[[1, 1, 1], [0, 0, 0]], [[1, 1, 0], [0, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_float() { + let tensor = TestTensor::<3>::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, 4.], [0.99, 3., -8.1]], + ]); + + // Sort along dim=0 + let values = tensor.clone().sort(0); + + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, -8.1]], + [[-0.3, 2.3, 4.], [0.99, 3., 0.94]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + // Sort along dim=1 + let values = tensor.clone().sort(1); + + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, -8.1], [0.99, 3., 4.]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + // Sort along dim=2 + let values = tensor.sort(2); + + let values_expected = TensorData::from([ + [[-0.5, -0.21, 1.2], [0., 0.94, 2.1]], + [[-0.3, 2.3, 4.], [-8.1, 0.99, 3.]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); +} + +#[test] +fn test_sort_with_indices_float() { + let tensor = TestTensor::<3>::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, 4.], [0.99, 3., -8.1]], + ]); + + // Sort along dim=0 + let (values, indices) = tensor.clone().sort_with_indices(0); + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, -8.1]], + [[-0.3, 2.3, 4.], [0.99, 3., 0.94]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + let indices_expected = TensorData::from([[[0, 0, 0], [0, 0, 1]], [[1, 1, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=1 + let (values, indices) = tensor.clone().sort_with_indices(1); + + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, -8.1], [0.99, 3., 4.]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + let indices_expected = TensorData::from([[[0, 0, 0], [1, 1, 1]], [[0, 0, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=2 + let (values, indices) = tensor.sort_with_indices(2); + + let values_expected = TensorData::from([ + [[-0.5, -0.21, 1.2], [0., 0.94, 2.1]], + [[-0.3, 2.3, 4.], [-8.1, 0.99, 3.]], + ]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + let indices_expected = TensorData::from([[[0, 2, 1], [0, 2, 1]], [[0, 1, 2], [2, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_argsort_float() { + let tensor = TestTensor::<3>::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, 4.], [0.99, 3., -8.1]], + ]); + + // Sort along dim=0 + let indices = tensor.clone().argsort(0); + + let indices_expected = TensorData::from([[[0, 0, 0], [0, 0, 1]], [[1, 1, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=1 + let indices = tensor.clone().argsort(1); + + let indices_expected = TensorData::from([[[0, 0, 0], [1, 1, 1]], [[0, 0, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=2 + let indices = tensor.argsort(2); + + let indices_expected = TensorData::from([[[0, 2, 1], [0, 2, 1]], [[0, 1, 2], [2, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_float_nan() { + let tensor = TestTensor::<2>::from([[-0.5, f32::NAN], [0., 0.94], [-0.3, f32::NAN]]); + + // Sort along dim=0 + let values = tensor.sort(0); + + let values_expected = TensorData::from([[-0.5, 0.94], [-0.3, f32::NAN], [0., f32::NAN]]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); +} + +#[test] +fn test_sort_descending_1d() { + let tensor = TestTensor::<1>::from([1., 2., 3., 4., 5.]); + + // Sort along dim=0 + let values = tensor.sort_descending(0); + + let values_expected = TensorData::from([5., 4., 3., 2., 1.]); + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/split.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/split.rs new file mode 100644 index 0000000..7607b75 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/split.rs @@ -0,0 +1,206 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_split_evenly_divisible() { + let device = Default::default(); + let tensors = + TestTensor::<2>::from_data([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11]], &device); + + let split_tensors = tensors.split(2, 0); + assert_eq!(split_tensors.len(), 3); + + let expected = [ + TensorData::from([[0, 1], [2, 3]]), + TensorData::from([[4, 5], [6, 7]]), + TensorData::from([[8, 9], [10, 11]]), + ]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_split_not_evenly_divisible() { + let device = Default::default(); + let tensors = TestTensor::<2>::from_data([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]], &device); + + let split_tensors = tensors.split(2, 0); + assert_eq!(split_tensors.len(), 3); + + let expected = [ + TensorData::from([[0, 1], [2, 3]]), + TensorData::from([[4, 5], [6, 7]]), + TensorData::from([[8, 9]]), + ]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_split_along_dim1() { + let device = Default::default(); + let tensors = TestTensor::<2>::from_data([[0, 1, 2], [3, 4, 5]], &device); + + let split_tensors = tensors.split(2, 1); + assert_eq!(split_tensors.len(), 2); + + let expected = [ + TensorData::from([[0, 1], [3, 4]]), + TensorData::from([[2], [5]]), + ]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_split_split_size_larger_than_tensor_size() { + let device = Default::default(); + let tensors = TestTensor::<1>::from_data([0, 1, 2, 3, 4], &device); + + let split_tensors = tensors.split(10, 0); + assert_eq!(split_tensors.len(), 1); + + let expected = [TensorData::from([0, 1, 2, 3, 4])]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_split_with_zero_split_size_zero_tensor_size() { + let device = Default::default(); + let empty_array: [i32; 0] = []; + let tensors = TestTensor::<1>::from_data(empty_array, &device); + + let split_tensors = tensors.split(0, 0); + assert_eq!(split_tensors.len(), 0); +} + +#[test] +fn test_split_zero_sized_tensor() { + let device = Default::default(); + let empty_array: [i32; 0] = []; + let tensors = TestTensor::<1>::from_data(empty_array, &device); + + let split_tensors = tensors.split(1, 0); + assert_eq!(split_tensors.len(), 0); +} + +#[test] +#[should_panic( + expected = "split_size must be greater than 0 unless the tensor size along the dimension is 0." +)] +fn test_split_with_zero_split_size_non_zero_tensor() { + let device = Default::default(); + let tensors = TestTensor::<1>::from_data([0, 1, 2, 3, 4], &device); + + let _split_tensors = tensors.split(0, 0); +} + +#[test] +#[should_panic(expected = "Given dimension is greater than or equal to the tensor rank.")] +fn test_split_invalid_dim() { + let device = Default::default(); + let tensors = TestTensor::<1>::from_data([0, 1, 2], &device); + + let _split_tensors = tensors.split(1, 2); +} + +#[test] +fn test_split_3d_tensor_along_dim0() { + let device = Default::default(); + let tensors = TestTensor::<3>::from_data( + [ + [[0, 1], [2, 3]], + [[4, 5], [6, 7]], + [[8, 9], [10, 11]], + [[12, 13], [14, 15]], + ], + &device, + ); + + let split_tensors = tensors.split(2, 0); + assert_eq!(split_tensors.len(), 2); + + let expected = [ + TensorData::from([[[0, 1], [2, 3]], [[4, 5], [6, 7]]]), + TensorData::from([[[8, 9], [10, 11]], [[12, 13], [14, 15]]]), + ]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_split_3d_tensor_along_dim1() { + let device = Default::default(); + let tensors = TestTensor::<3>::from_data( + [[[0, 1], [2, 3], [4, 5]], [[6, 7], [8, 9], [10, 11]]], + &device, + ); + + let split_tensors = tensors.split(2, 1); + assert_eq!(split_tensors.len(), 2); + + let expected = [ + TensorData::from([[[0, 1], [2, 3]], [[6, 7], [8, 9]]]), + TensorData::from([[[4, 5]], [[10, 11]]]), + ]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +fn test_split_with_sizes() { + let device = Default::default(); + let tensors = TestTensor::<1>::from_data([0, 1, 2, 3, 4, 5], &device); + + let split_tensors = tensors.split_with_sizes(vec![2, 3, 1], 0); + assert_eq!(split_tensors.len(), 3); + + let expected = [ + TensorData::from([0, 1]), + TensorData::from([2, 3, 4]), + TensorData::from([5]), + ]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} + +#[test] +#[should_panic( + expected = "The sum of split_sizes must equal the tensor size along the specified dimension." +)] +fn test_split_with_sizes_invalid_sum() { + let device = Default::default(); + let tensors = TestTensor::<1>::from_data([0, 1, 2, 3, 4, 5], &device); + + let _split_tensors = tensors.split_with_sizes(vec![2, 2, 1], 0); +} + +#[test] +fn test_split_with_sizes_zero_length() { + let device = Default::default(); + let tensors = TestTensor::<1>::from_data([0, 1, 2], &device); + + let split_tensors = tensors.split_with_sizes(vec![0, 1, 2], 0); + assert_eq!(split_tensors.len(), 2); + + let expected = [TensorData::from([0]), TensorData::from([1, 2])]; + + for (index, tensor) in split_tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sqrt.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sqrt.rs new file mode 100644 index 0000000..05ba326 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sqrt.rs @@ -0,0 +1,18 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use core::f32::consts::SQRT_2; + +#[test] +fn should_support_sqrt_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.sqrt(); + let expected = TensorData::from([[0.0, 1.0, SQRT_2], [1.73205, 2.0, 2.2360]]); + + output.into_data().assert_approx_eq::( + &expected, + Tolerance::relative(1e-4).set_half_precision_relative(1e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/square.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/square.rs new file mode 100644 index 0000000..994282b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/square.rs @@ -0,0 +1,17 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_sqrt_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.square(); + let expected = TensorData::from([[0.0, 1.0, 4.0], [9.0, 16.0, 25.0]]); + + output.into_data().assert_approx_eq::( + &expected, + Tolerance::relative(1e-4).set_half_precision_relative(1e-3), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/squeeze.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/squeeze.rs new file mode 100644 index 0000000..0aead04 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/squeeze.rs @@ -0,0 +1,207 @@ +use super::*; +use burn_tensor::Shape; + +/// Test if the function can successfully squeeze the size 1 dimension of a 3D tensor. +#[test] +fn should_squeeze_dim() { + let tensor = TestTensor::<3>::ones(Shape::new([2, 1, 4]), &Default::default()); + let squeezed_tensor: TestTensor<2> = tensor.squeeze_dim(1); + let expected_shape = Shape::new([2, 4]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +#[test] +fn should_squeeze() { + let tensor = TestTensor::<3>::ones(Shape::new([2, 1, 4]), &Default::default()); + let squeezed_tensor: TestTensor<2> = tensor.squeeze(); + let expected_shape = Shape::new([2, 4]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +/// Test if the function can successfully squeeze the first size 1 dimension of a 4D tensor. +#[test] +fn should_squeeze_first() { + let tensor = TestTensor::<4>::ones(Shape::new([1, 3, 4, 5]), &Default::default()); + let squeezed_tensor: TestTensor<3> = tensor.squeeze_dim(0); + let expected_shape = Shape::new([3, 4, 5]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} +/// Test if the function can successfully squeeze the last size 1 dimension of a 4D tensor. +#[test] +fn should_squeeze_last() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 1]), &Default::default()); + let squeezed_tensor: TestTensor<3> = tensor.squeeze_dim(3); + let expected_shape = Shape::new([2, 3, 4]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} +/// Test if the function panics when the squeezed dimension is not of size 1. +#[test] +#[should_panic] +fn should_squeeze_panic() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let _squeezed_tensor: TestTensor<3> = tensor.squeeze_dim(2); +} + +/// Test if the function works with an empty slice +#[test] +fn should_squeeze_dims_with_empty_slice() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 1, 3]), &Default::default()); + let squeezed_tensor: TestTensor<1> = tensor.squeeze_dims(&[]); + let expected_shape = Shape::new([3]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +#[test] +fn should_squeeze_all_dims() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 3, 1]), &Default::default()); + let squeezed_tensor: TestTensor<1> = tensor.squeeze(); + let expected_shape = Shape::new([3]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +/// Test if the function works with positive indices +#[test] +fn should_squeeze_dims_with_positive_indices() { + let tensor = TestTensor::<4>::ones(Shape::new([1, 3, 1, 5]), &Default::default()); + let squeezed_tensor: TestTensor<2> = tensor.squeeze_dims(&[0, 2]); + let expected_shape = Shape::new([3, 5]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +/// Test if the function works with negative indices +#[test] +fn should_squeeze_dims_with_negative_indices() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 1, 3, 1]), &Default::default()); + let squeezed_tensor: TestTensor<2> = tensor.squeeze_dims(&[-3, -1]); + let expected_shape = Shape::new([2, 3]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +/// Test to make sure the function panics if a non-singleton dimension is squeezed +#[test] +#[should_panic] +fn should_squeeze_dims_work_if_non_singleton() { + let tensor = TestTensor::<3>::ones(Shape::new([2, 3, 4]), &Default::default()); + let squeezed_tensor: TestTensor<3> = tensor.squeeze_dims(&[1]); + let expected_shape = Shape::new([2, 3, 4]); + assert_eq!(squeezed_tensor.shape(), expected_shape); +} + +#[test] +#[should_panic] +fn should_panic_squeeze_consumes_all_singleton() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 3, 1]), &Default::default()); + let _squeezed_tensor: TestTensor<2> = tensor.squeeze(); // output rank should be 1 +} + +/// Test to make sure the function panics if too many dimensions are requested to be squeezed +#[test] +#[should_panic] +fn should_squeeze_dims_panic_on_too_many_dimensions() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 1, 1]), &Default::default()); + let _: TestTensor<1> = tensor.squeeze_dims(&[0, 1, 2]); +} + +/// Test to make sure function panics if dimensions are mismatched +#[test] +#[should_panic] +fn should_squeeze_dims_dimension_mismatch_panic() { + let tensor = TestTensor::<4>::ones(Shape::new([1, 3, 1, 5]), &Default::default()); + let _: TestTensor<3> = tensor.squeeze_dims(&[0, 2]); +} + +/// Test if the function can successfully unsqueeze the size 1 dimension at the specified position of a 3D tensor. +#[test] +fn should_unsqueeze_dim() { + let tensor = TestTensor::<3>::ones(Shape::new([2, 4, 1]), &Default::default()); + let unsqueezed_tensor: TestTensor<4> = tensor.unsqueeze_dim(1); + let expected_shape = Shape::new([2, 1, 4, 1]); + assert_eq!(unsqueezed_tensor.shape(), expected_shape); +} + +/// Test if the function can successfully unsqueeze the first size 1 dimension of a 4D tensor. +#[test] +fn should_unsqueeze_dim_first() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let unsqueezed_tensor: TestTensor<5> = tensor.unsqueeze_dim(0); + let expected_shape = Shape::new([1, 2, 3, 4, 5]); + assert_eq!(unsqueezed_tensor.shape(), expected_shape); +} + +/// Test if the function can successfully unsqueeze the last size 1 dimension of a 4D tensor. +#[test] +fn should_unsqueeze_dim_last() { + let tensor = TestTensor::<4>::ones(Shape::new([5, 4, 3, 2]), &Default::default()); + let unsqueezed_tensor: TestTensor<5> = tensor.unsqueeze_dim(4); + let expected_shape = Shape::new([5, 4, 3, 2, 1]); + assert_eq!(unsqueezed_tensor.shape(), expected_shape); +} + +/// Test if the function panics when the unsqueezed dimension is out of bounds. +#[test] +#[should_panic] +fn should_unsqueeze_dim_panic() { + let tensor = TestTensor::<4>::ones(Shape::new([2, 3, 4, 5]), &Default::default()); + let _unsqueezed_tensor: TestTensor<5> = tensor.unsqueeze_dim(5); +} + +#[test] +fn should_unsqueeze_dims_support_dim_inference() { + let input_tensor = TestTensor::<3>::ones(Shape::new([3, 4, 5]), &Default::default()); + let output_tensor = input_tensor.unsqueeze_dims::<5>(&[1, -2]); + let expected_shape = Shape::new([3, 1, 4, 1, 5]); + assert_eq!(output_tensor.shape(), expected_shape); +} + +#[test] +fn should_unsqueeze_dims_handle_first_last() { + let input_tensor = TestTensor::<3>::ones(Shape::new([3, 4, 5]), &Default::default()); + let output_tensor = input_tensor.unsqueeze_dims::<5>(&[0, 4]); + let expected_shape = Shape::new([1, 3, 4, 5, 1]); + assert_eq!(output_tensor.shape(), expected_shape); +} + +#[test] +fn should_unsqueeze_dims_work_with_single_dim() { + //bruh, just call unsqueeze_dim + let input_tensor = TestTensor::<3>::ones(Shape::new([3, 4, 5]), &Default::default()); + let output_tensor: TestTensor<4> = input_tensor.unsqueeze_dims(&[1]); + let expected_shape = Shape::new([3, 1, 4, 5]); + assert_eq!(output_tensor.shape(), expected_shape); +} + +#[test] +fn should_unsqueeze_dims_multiple_trailing_negatives() { + let input_tensor = TestTensor::<3>::ones(Shape::new([3, 4, 5]), &Default::default()); + let output_tensor: TestTensor<6> = input_tensor.unsqueeze_dims(&[0, -1, -1]); + let expected_shape = Shape::new([1, 3, 4, 5, 1, 1]); + assert_eq!(output_tensor.shape(), expected_shape); +} + +#[test] +#[should_panic] +fn should_unsqueeze_dims_panic() { + let input_tensor = TestTensor::<3>::ones(Shape::new([3, 4, 5]), &Default::default()); + let _output_tensor: TestTensor<5> = input_tensor.unsqueeze_dims(&[0, -6]); +} + +#[test] +#[should_panic] +fn squeeze_all_singleton_not_supported() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 1, 1]), &Default::default()); + let _ = tensor.squeeze::<0>(); +} + +#[test] +#[should_panic] +fn squeeze_dim_singleton_not_supported() { + let tensor = TestTensor::<1>::ones(Shape::new([1]), &Default::default()); + let _ = tensor.squeeze_dim::<0>(0); +} + +#[test] +#[should_panic] +fn squeeze_dims_all_singleton_not_supported() { + let tensor = TestTensor::<3>::ones(Shape::new([1, 1, 1]), &Default::default()); + let _ = tensor.squeeze_dims::<0>(&[0, 1, 2]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/stack.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/stack.rs new file mode 100644 index 0000000..63d8c85 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/stack.rs @@ -0,0 +1,69 @@ +use super::*; +use alloc::{vec, vec::Vec}; +use burn_tensor::{Tensor, TensorData}; + +#[test] +fn should_support_stack_ops_2d_dim0() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0]], &device); + let tensor_2 = TestTensor::from_data([[4.0, 5.0, 6.0]], &device); + + let output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[1.0, 2.0, 3.0]], [[4.0, 5.0, 6.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_stack_ops_2d_dim1() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0]], &device); + let tensor_2 = TestTensor::from_data([[4.0, 5.0, 6.0]], &device); + + let output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 1); + let expected = TensorData::from([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_stack_ops_3d() { + let device = Default::default(); + let tensor_1 = TestTensor::<3>::from_data([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]]], &device); + let tensor_2 = TestTensor::from_data([[[4.0, 5.0, 6.0]], [[4.1, 5.1, 6.1]]], &device); + + let output = Tensor::stack::<4>(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([ + [[[1.0000, 2.0000, 3.0000]], [[1.1000, 2.1000, 3.1000]]], + [[[4.0000, 5.0000, 6.0000]], [[4.1000, 5.1000, 6.1000]]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn should_panic_when_dimensions_are_not_the_same() { + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0]], &device); + let tensor_2 = TestTensor::from_data([[4.0, 5.0]], &device); + + let _output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 0); +} + +#[test] +#[should_panic] +fn should_panic_when_list_of_vectors_is_empty() { + let tensors: Vec> = vec![]; + let _output = Tensor::stack::<3>(tensors, 0); +} + +#[test] +#[should_panic] +fn should_panic_when_stack_exceeds_dimension() { + let device = Default::default(); + let tensor_1 = TestTensor::<3>::from_data([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]]], &device); + let tensor_2 = TestTensor::from_data([[[4.0, 5.0, 6.0]]], &device); + + let _output = Tensor::stack::<4>(vec![tensor_1, tensor_2], 3); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sub.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sub.rs new file mode 100644 index 0000000..d390caf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/sub.rs @@ -0,0 +1,42 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_sub_ops() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data_2 = TensorData::from([[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let output = tensor_1 - tensor_2; + let expected = TensorData::from([[-6.0, -6.0, -6.0], [-6.0, -6.0, -6.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sub_broadcast() { + let data_1 = TensorData::from([[0.0, 1.0, 2.0]]); + let data_2 = TensorData::from([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]); + let device = Default::default(); + let tensor_1 = TestTensor::<2>::from_data(data_1, &device); + let tensor_2 = TestTensor::<2>::from_data(data_2, &device); + + let output = tensor_1 - tensor_2; + let expected = TensorData::from([[-3.0, -3.0, -3.0], [-6.0, -6.0, -6.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_sub_scalar_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let scalar = 2.0; + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor - scalar; + let expected = TensorData::from([[-2.0, -1.0, 0.0], [1.0, 2.0, 3.0]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/take.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/take.rs new file mode 100644 index 0000000..db27d77 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/take.rs @@ -0,0 +1,206 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_take_1d() { + // Test that take works with 1D indices + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([0.0, 1.0, 2.0], &device); + let indices = TestTensorInt::<1>::from_data([1, 1, 0, 1, 2], &device); + + let output = tensor.take::<1, 1>(0, indices); + let expected = TensorData::from([1.0, 1.0, 0.0, 1.0, 2.0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_2d_dim0() { + // Test take on 2D tensor along dimension 0 + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::<1>::from_data([1, 0, 1, 1], &device); + + let output = tensor.take::<1, 2>(0, indices); + let expected = TensorData::from([ + [3.0, 4.0, 5.0], + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [3.0, 4.0, 5.0], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_2d_dim1() { + // Test take on 2D tensor along dimension 1 + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::<1>::from_data([2, 0, 1], &device); + + let output = tensor.take::<1, 2>(1, indices); + let expected = TensorData::from([[2.0, 0.0, 1.0], [5.0, 3.0, 4.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn take_and_select_should_be_equivalent() { + // Verify that take and select produce identical results + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + &device, + ); + let indices = TestTensorInt::<1>::from_data([2, 0, 1, 1], &device); + + let result_take = tensor.clone().take::<1, 2>(0, indices.clone()); + let result_select = tensor.select(0, indices); + + let take_data = result_take.into_data(); + let select_data = result_select.into_data(); + + take_data.assert_eq(&select_data, false); +} + +#[test] +fn should_take_with_2d_indices() { + // Test take with 2D indices - output will be 3D with shape [2, 2, 4] + let device = Default::default(); + let tensor = TestTensor::<2>::from_data( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + &device, + ); + + // 2D indices to select along dimension 0 - shape [2, 2] + let indices = TestTensorInt::<2>::from_data([[0, 2], [1, 0]], &device); + let output = tensor.take::<2, 3>(0, indices); + + // Expected: shape [2, 2, 4] - indices shape replaces dim 0 + let expected = TensorData::from([ + [[1.0, 2.0, 3.0, 4.0], [9.0, 10.0, 11.0, 12.0]], + [[5.0, 6.0, 7.0, 8.0], [1.0, 2.0, 3.0, 4.0]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_with_2d_indices_dim1() { + // Test take with 2D indices along dimension 1 - output will be 3D with shape [2, 2, 2] + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], &device); + + // 2D indices to select along dimension 1 - shape [2, 2] + let indices = TestTensorInt::<2>::from_data([[0, 3], [2, 1]], &device); + let output = tensor.take::<2, 3>(1, indices); + + // Expected: shape [2, 2, 2] - indices shape replaces dim 1 + let expected = TensorData::from([[[1.0, 4.0], [3.0, 2.0]], [[5.0, 8.0], [7.0, 6.0]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_3d_tensor() { + // Test take with 3D tensor - output will be 4D with shape [2, 2, 2, 2] + let device = Default::default(); + let tensor = TestTensor::<3>::from_data( + [ + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], + [[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]], + ], + &device, + ); + + // 2D indices to select along dimension 1 - shape [2, 2] + let indices = TestTensorInt::<2>::from_data([[0, 2], [1, 0]], &device); + let output = tensor.take::<2, 4>(1, indices); + + // Expected: shape [2, 2, 2, 2] - indices shape replaces dim 1 + let expected = TensorData::from([ + [[[1.0, 2.0], [5.0, 6.0]], [[3.0, 4.0], [1.0, 2.0]]], + [[[7.0, 8.0], [11.0, 12.0]], [[9.0, 10.0], [7.0, 8.0]]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_with_3d_indices() { + // Test take with 3D indices - output will be 4D + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + + // 3D indices to select along dimension 1 - shape [2, 2, 2] + let indices = TestTensorInt::<3>::from_data([[[0, 2], [1, 0]], [[2, 1], [0, 2]]], &device); + let output = tensor.take::<3, 4>(1, indices); + + // Expected: shape [2, 2, 2, 2] - indices shape replaces dim 1 + let expected = TensorData::from([ + [[[1.0, 3.0], [2.0, 1.0]], [[3.0, 2.0], [1.0, 3.0]]], + [[[4.0, 6.0], [5.0, 4.0]], [[6.0, 5.0], [4.0, 6.0]]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn should_panic_take_invalid_dimension() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], &device); + let indices = TestTensorInt::<1>::from_data([1, 0], &device); + + // This should panic because dimension 10 is out of bounds + tensor.take::<1, 2>(10, indices); +} + +#[test] +fn should_take_with_single_index() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let indices = TestTensorInt::<1>::from_data([1], &device); + + let output = tensor.take::<1, 2>(0, indices); + let expected = TensorData::from([[4.0, 5.0, 6.0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_with_negative_dim_2d() { + // Test using negative dimension indexing on 2D tensor + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let indices = TestTensorInt::<1>::from_data([2, 0, 1], &device); + + // Using -1 should refer to the last dimension (dim 1) + let output_neg = tensor.clone().take::<1, 2>(-1, indices.clone()); + let output_pos = tensor.take::<1, 2>(1, indices); + + // Both should produce the same result + let neg_data = output_neg.into_data(); + let pos_data = output_pos.into_data(); + neg_data.assert_eq(&pos_data, false); +} + +#[test] +#[should_panic] +fn should_panic_take_negative_dim_out_of_bounds() { + let device = Default::default(); + let tensor = TestTensor::<2>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let indices = TestTensorInt::<1>::from_data([0, 1], &device); + + // This should panic because -3 is out of bounds for a 2D tensor + tensor.take::<1, 2>(-3, indices); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/topk.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/topk.rs new file mode 100644 index 0000000..3672586 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/topk.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_topk_with_indices_3d() { + let tensor = + TestTensor::<3>::from([[[1., 4., 7.], [2., 5., 6.]], [[3., 0., 9.], [8., 2., 7.]]]); + + let (values, indices) = tensor.topk_with_indices(2, /*dim*/ 2); + + let values_expected = TensorData::from([[[7., 4.], [6., 5.]], [[9., 3.], [8., 7.]]]); + + values + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::default()); + + let indices_expected = TensorData::from([[[2, 1], [2, 1]], [[2, 0], [0, 2]]]); + + indices.into_data().assert_eq(&indices_expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transaction.rs new file mode 100644 index 0000000..ac9dbbf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transaction.rs @@ -0,0 +1,31 @@ +use super::*; +use burn_tensor::Transaction; + +// https://github.com/tracel-ai/burn/issues/4021 +#[test] +fn should_support_transaction() { + let rows = 261120; + let cols = 408; + + let device = Default::default(); + + let j = TestTensor::<2>::zeros([rows, cols], &device); + let jt = j.clone().transpose(); + + let g = jt.matmul(j); + + let g = g.transpose(); + let expected = g.to_data(); + + assert_eq!(g.shape().dims(), [cols, cols]); + + // Fails + let [data] = Transaction::default() + .register(g) + .execute() + .try_into() + .unwrap(); + + // check byte equality + assert_eq!(data, expected); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transpose.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transpose.rs new file mode 100644 index 0000000..5885513 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/transpose.rs @@ -0,0 +1,116 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_transpose_ops() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + // Check the .t() alias. + let output = tensor.t(); + + let expected = TensorData::from([ + [[0.0, 3.0], [1.0, 4.0], [2.0, 5.0]], + [[6.0, 9.0], [7.0, 10.0], [8.0, 11.0]], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_transpose_maybe_fused_with_one() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + let ones = TestTensor::<3>::ones([1, 1, 1], &Default::default()); + + let output = tensor.transpose(); + let expected = TensorData::from([ + [[0.0, 3.0], [1.0, 4.0], [2.0, 5.0]], + [[6.0, 9.0], [7.0, 10.0], [8.0, 11.0]], + ]); + let expected_ones = TensorData::from([[[1.0]]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + ones.into_data() + .assert_approx_eq::(&expected_ones, Tolerance::default()); +} + +#[test] +fn should_support_swap_dims_no_op() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let output = tensor.swap_dims(0, 0); + let expected = TensorData::from([ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_swap_dims() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let output = tensor.swap_dims(0, 2); + let expected = TensorData::from([ + [[0.0, 6.0], [3.0, 9.0]], + [[1.0, 7.0], [4.0, 10.0]], + [[2.0, 8.0], [5.0, 11.0]], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_swap_dims_neg_index() { + let tensor = TestTensor::<3>::from_floats( + [ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ], + &Default::default(), + ); + + let output = tensor.swap_dims(-3, -1); + let expected = TensorData::from([ + [[0.0, 6.0], [3.0, 9.0]], + [[1.0, 7.0], [4.0, 10.0]], + [[2.0, 8.0], [5.0, 11.0]], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/tri.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/tri.rs new file mode 100644 index 0000000..3325e10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/tri.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_triu() { + let tensor = TestTensor::<2>::from([[1., 1., 1.], [1., 1., 1.], [1., 1., 1.]]); + let output = tensor.triu(0); + let expected = TensorData::from([[1., 1., 1.], [0., 1., 1.], [0., 0., 1.]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_triu_positive_diagonal() { + let tensor = TestTensor::<2>::from([[1, 1, 1], [1, 1, 1], [1, 1, 1]]); + + let output = tensor.triu(1); + let expected = TensorData::from([[0, 1, 1], [0, 0, 1], [0, 0, 0]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trig.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trig.rs new file mode 100644 index 0000000..610b5a2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trig.rs @@ -0,0 +1,242 @@ +#![allow(clippy::approx_constant)] + +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use core::f32::consts::{FRAC_PI_2, FRAC_PI_3, FRAC_PI_4, FRAC_PI_6, FRAC_PI_8, PI}; + +#[test] +fn should_support_cos_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.cos(); + let expected = TensorData::from([[1.0, 0.54030, -0.41615], [-0.98999, -0.65364, 0.28366]]); + + // Metal has less precise trigonometric functions + let tolerance = Tolerance::default().set_half_precision_relative(1e-2); + + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn should_support_cosh_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.cosh(); + let expected = TensorData::from([[1.0000, 1.5431, 3.7622], [10.0677, 27.3082, 74.2099]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_sin_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.sin(); + let expected = TensorData::from([[0.0, 0.841471, 0.909297], [0.141120, -0.756802, -0.958924]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_sinh_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.sinh(); + let expected = TensorData::from([[0.0000, 1.1752, 3.6269], [10.0179, 27.2899, 74.2032]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_tan_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.tan(); + let expected = TensorData::from([[0.0, 1.557408, -2.185040], [-0.142547, 1.157821, -3.380515]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_tanh_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.tanh(); + let expected = TensorData::from([[0.0, 0.761594, 0.964028], [0.995055, 0.999329, 0.999909]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_asin_ops() { + let data = TensorData::from([[0.0, 0.5, 0.707107], [-0.5, -0.707107, -1.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.asin(); + let expected = TensorData::from([[0.0, 0.523599, 0.785398], [-0.523599, -0.785398, -1.570796]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_acos_ops() { + let data = TensorData::from([[0.0, 0.5, 0.707107], [-0.5, -0.707107, -1.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.acos(); + let expected = TensorData::from([ + [1.570796, 1.047198, 0.785398], + [2.094395, 2.356194, 3.141593], + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_atan_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.atan(); + let expected = TensorData::from([[0.0, 0.785398, 1.107149], [1.249046, 1.325818, 1.373401]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_asinh_ops() { + let data = TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.asinh(); + let expected = TensorData::from([[0.0, 0.881374, 1.443635], [1.818446, 2.094713, 2.312438]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_acosh_ops() { + let data = TensorData::from([[1.0, 1.5, 2.0], [3.0, 4.0, 5.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.acosh(); + let expected = TensorData::from([[0.0, 0.962424, 1.316958], [1.762747, 2.063437, 2.292432]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_atanh_ops() { + let data = TensorData::from([[0.0, 0.5, 0.707107], [-0.5, -0.707107, -0.9]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.atanh(); + let expected = TensorData::from([[0.0, 0.549306, 0.881374], [-0.549306, -0.881374, -1.472219]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_atan2_ops() { + let y = TensorData::from([[0.0, 1.0, 1.0], [-1.0, -1.0, 0.0]]); + let x = TensorData::from([[1.0, 1.0, 0.0], [1.0, 0.0, -1.0]]); + + let y_tensor = TestTensor::<2>::from_data(y, &Default::default()); + let x_tensor = TestTensor::<2>::from_data(x, &Default::default()); + + let output = y_tensor.atan2(x_tensor); + let expected = TensorData::from([[0.0, 0.785398, 1.570796], [-0.785398, -1.570796, 3.141593]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_deg2rad_ops() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_floats( + [ + 0.0, 22.5, 30.0, 45.0, 60.0, 90.0, 135.0, 180.0, 270.0, 360.0, + ], + &device, + ); + + let output = tensor.deg2rad(); + let expected = TensorData::from([ + 0.0f32, + FRAC_PI_8, + FRAC_PI_6, + FRAC_PI_4, + FRAC_PI_3, + FRAC_PI_2, + 0.75 * PI, + PI, + 1.5 * PI, + 2.0 * PI, + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_support_rad2deg_ops() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_floats( + [ + 0.0, + FRAC_PI_8, + FRAC_PI_6, + FRAC_PI_4, + FRAC_PI_3, + FRAC_PI_2, + PI, + 1.5 * PI, + 2.0 * PI, + -FRAC_PI_3, + ], + &device, + ); + + let output = tensor.rad2deg(); + let expected = TensorData::from([ + 0.0f32, 22.5, 30.0, 45.0, 60.0, 90.0, 180.0, 270.0, 360.0, -60.0, + ]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trunc.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trunc.rs new file mode 100644 index 0000000..d29692c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/trunc.rs @@ -0,0 +1,67 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{ElementConversion, TensorData}; + +#[test] +fn should_support_trunc_ops() { + let data = TensorData::from([[2.3, -1.7, 0.5], [-0.5, 3.9, -4.2]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.trunc(); + let expected = TensorData::from([[2.0, -1.0, 0.0], [0.0, 3.0, -4.0]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_truncate_positive_values_like_floor() { + let data = TensorData::from([1.7, 2.9, 3.1, 4.5]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.trunc(); + let expected = TensorData::from([1.0, 2.0, 3.0, 4.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_truncate_negative_values_like_ceil() { + let data = TensorData::from([-1.7, -2.9, -3.1, -4.5]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.trunc(); + let expected = TensorData::from([-1.0, -2.0, -3.0, -4.0]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn should_handle_special_cases() { + // Test special IEEE 754 cases + let data = TensorData::from([0.0, -0.0, f32::INFINITY, f32::NEG_INFINITY, f32::NAN]); + let tensor = TestTensor::<1>::from_data(data, &Default::default()); + + let output = tensor.trunc(); + let values = output.into_data().as_slice::().unwrap().to_vec(); + + // Check positive zero + assert_eq!(values[0], 0.0f32.elem::()); + assert!(values[0].is_sign_positive()); + + // Check negative zero is preserved + assert_eq!(values[1], 0.0f32.elem::()); + assert!(values[1].is_sign_negative()); + + // Check infinity is preserved + assert!(values[2].is_infinite() && values[2].is_sign_positive()); + assert!(values[3].is_infinite() && values[3].is_sign_negative()); + + // Check NaN is preserved + assert!(values[4].is_nan()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/unfold.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/unfold.rs new file mode 100644 index 0000000..5169d92 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/ops/unfold.rs @@ -0,0 +1,35 @@ +use super::*; +use burn_tensor::Distribution; +use burn_tensor::s; + +#[test] +fn test_unfold_float() { + let device = Default::default(); + + let input = TestTensor::<3>::random([2, 6, 6], Distribution::Default, &device); + + let dim = 1; + let size = 3; + let step = 2; + let actual: TestTensor<4> = input.clone().unfold(dim, size, step); + + let expected = TestTensor::<4>::empty([2, 2, 6, 3], &device) + .slice_assign( + s![.., 0, .., ..], + input + .clone() + .slice(s![.., 0..3, ..]) + .swap_dims(1, 2) + .unsqueeze_dim::<4>(1), + ) + .slice_assign( + s![.., 1, .., ..], + input + .clone() + .slice(s![.., 2..5, ..]) + .swap_dims(1, 2) + .unsqueeze_dim::<4>(1), + ); + + actual.to_data().assert_eq(&expected.to_data(), true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/primitive.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/primitive.rs new file mode 100644 index 0000000..d1c1bdd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/primitive.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::{Element, Shape}; + +#[test] +fn should_support_float_dtype() { + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, 4.0, -5.0]]).into_primitive(); + + assert_eq!( + burn_tensor::TensorMetadata::shape(&tensor), + Shape::new([2, 3]) + ); + assert_eq!( + burn_tensor::TensorMetadata::dtype(&tensor), + FloatElem::dtype() // default float elem type + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/calibration.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/calibration.rs new file mode 100644 index 0000000..4124fd4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/calibration.rs @@ -0,0 +1,51 @@ +use super::*; +use burn_tensor::{ + TensorData, + ops::QuantizedTensor, + quantization::{Calibration, QTensorPrimitive, QuantLevel, QuantValue, compute_range}, +}; + +// NOTE: The scheme variant fields are not important for calibration, only the "main" variant (e.g., per-tensor) +#[test] +fn min_max_calibration_range_per_tensor() { + let tensor = TestTensor::<1>::from_floats([-1.8, -1.0, 0.0, 0.5], &Default::default()); + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let range = compute_range(&scheme, &tensor, &Calibration::MinMax); + + range + .min + .into_data() + .assert_eq(&TensorData::from([-1.8]), false); + range + .max + .into_data() + .assert_eq(&TensorData::from([0.5]), false); +} + +#[test] +fn min_max_calibration_range_per_block() { + let tensor = TestTensor::<2>::from_floats( + [ + [-1.8, -1.0, 0.0, 0.5], + [1.8, 1.0, 0.0, -0.5], + [0.01, 0.02, 0.03, 0.04], + [-0.01, -0.02, -0.03, -0.04], + ], + &Default::default(), + ); + let scheme = QuantizedTensor::::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([4])); + + let range = compute_range(&scheme, &tensor, &Calibration::MinMax); + + range + .min + .into_data() + .assert_eq(&TensorData::from([[-1.8], [-0.5], [0.01], [-0.04]]), false); + range + .max + .into_data() + .assert_eq(&TensorData::from([[0.5], [1.8], [0.04], [-0.01]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/data.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/data.rs new file mode 100644 index 0000000..4fed023 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/data.rs @@ -0,0 +1,39 @@ +use super::*; +use alloc::vec; +use burn_tensor::quantization::{QTensorPrimitive, QuantLevel, QuantValue}; +use burn_tensor::{TensorData, ops::QuantizedTensor}; + +#[test] +fn should_support_per_tensor_symmetric_int8() { + let data = TensorData::quantized( + vec![-127i8, -71, 0, 35], + [4], + QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S), + &[0.014_173_228], + ); + let tensor = TestTensor::<1>::from_data(data.clone(), &Default::default()); + + let q_data = tensor.into_data(); + q_data.assert_eq(&data, true); + + let tensor = TestTensor::<1>::from_data(q_data.clone(), &Default::default()); + + tensor.into_data().assert_eq(&q_data, true); +} + +#[test] +fn should_support_per_block_symmetric_int8() { + let data = TensorData::quantized( + vec![ + -127i8, -71, 0, 35, -127i8, -71, 0, 35, -32, -63, -95, -127, -32, -63, -95, -127, + ], + [16], + QuantizedTensor::::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([8])), + &[0.014_173_228, 0.000_314_96], + ); + let tensor = TestTensor::<1>::from_data(data.clone(), &Default::default()); + + tensor.into_data().assert_eq(&data, true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/mod.rs new file mode 100644 index 0000000..668ce76 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/mod.rs @@ -0,0 +1,48 @@ +pub use super::*; // re-export test types + +mod calibration; +mod data; +mod ops; +mod scheme; + +/// Quantized tensor utilities +pub mod qtensor { + use core::marker::PhantomData; + + use burn_tensor::quantization::QuantLevel; + + use burn_tensor::{ + Tensor, TensorData, + backend::Backend, + quantization::{QTensorPrimitive, QuantValue}, + }; + + pub struct QTensor { + b: PhantomData, + } + + impl QTensor { + /// Creates a quantized int8 tensor from the floating point data using the default quantization scheme + /// (i.e., per-tensor symmetric quantization). + pub fn int8>(floats: F) -> Tensor { + Self::int8_symmetric(floats) + } + + /// Creates a quantized int8 tensor from the floating point data using blocks of size 16 + pub fn int8_block>(floats: F) -> Tensor { + Tensor::from_floats(floats, &Default::default()).quantize_dynamic( + &::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([16])), + ) + } + + /// Creates a quantized int8 tensor from the floating point data using per-tensor symmetric quantization. + pub fn int8_symmetric>(floats: F) -> Tensor { + Tensor::from_floats(floats, &Default::default()).quantize_dynamic( + &::default_scheme() + .with_value(QuantValue::Q8S), + ) + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/abs.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/abs.rs new file mode 100644 index 0000000..ab3d630 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/abs.rs @@ -0,0 +1,19 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_abs_ops() { + let tensor = QTensor::::int8([[0.0, -1.0, 2.0], [3.0, 4.0, -5.0]]); + + let output = tensor.abs(); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]), + Tolerance::absolute(1e-1), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/add.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/add.rs new file mode 100644 index 0000000..b1285d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/add.rs @@ -0,0 +1,106 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_add_d2() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]]); + + let output = tensor_1 + tensor_2; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[6.0, 8.0, 10.0], [12.0, 14.0, 16.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_add_broadcast() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0]]); + let tensor_2 = QTensor::::int8([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]); + + let output = tensor_1 + tensor_2; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[3.0, 5.0, 7.0], [6.0, 8.0, 10.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_add_different_strides_rhs() { + // We need to execute an operation after `from data` to trigger inplace in some backends. + // Which is the operation that might be problematic in this case. + let tensor_1 = QTensor::::int8([[0.0, 1.0], [2.0, 3.0]]) * 1; + let tensor_2 = QTensor::::int8([[4.0, 5.0], [6.0, 7.0]]) * 1; + + let output = tensor_1 + tensor_2.transpose(); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[4.0, 7.0], [7.0, 10.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_add_different_strides_lhs() { + // We need to execute an operation after `from data` to trigger inplace in some backends. + // Which is the operation that might be problematic in this case. + let tensor_1 = QTensor::::int8([[0.0, 1.0], [2.0, 3.0]]) * 1; + let tensor_2 = QTensor::::int8([[4.0, 5.0], [6.0, 7.0]]) * 1; + + let output = tensor_1.transpose() + tensor_2; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[4.0, 7.0], [7.0, 10.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_add_different_strides_broadcast() { + // We need to execute an operation after `from data` to trigger inplace in some backends. + // Which is the operation that might be problematic in this case. + let tensor_1 = QTensor::::int8([[0.0, 1.0], [2.0, 3.0]]) * 1; + let tensor_2 = QTensor::::int8([[4.0, 5.0]]) * 1; + + let output = tensor_1.transpose() + tensor_2; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[4.0, 7.0], [5.0, 8.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_support_add_scalar_ops() { + let scalar = 2.0; + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor + scalar; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[2.0, 3.0, 4.0], [5.0, 6.0, 7.0]]), + Tolerance::absolute(1e-1), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/aggregation.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/aggregation.rs new file mode 100644 index 0000000..b79e910 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/aggregation.rs @@ -0,0 +1,166 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_should_mean() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.mean(); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&TensorData::from([15.0 / 6.0]), Tolerance::absolute(1e-1)); +} + +#[test] +fn test_should_sum() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sum(); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&TensorData::from([15.0]), Tolerance::absolute(1e-1)); +} + +#[test] +fn test_should_mean_last_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.mean_dim(1); + let expected = TensorData::from([[3.0 / 3.0], [12.0 / 3.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_should_sum_last_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sum_dim(1); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[3.0], [12.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_should_sum_first_dim() { + let tensor = QTensor::::int8([[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]]); + + let output = tensor.sum_dim(0); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[7.0, 3.0, 5.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_should_mean_first_dim() { + let tensor = QTensor::::int8([[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]]); + + let output = tensor.mean_dim(0); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[7.0 / 2.0, 3.0 / 2.0, 5.0 / 2.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_should_sum_mid_dim_3d_non_contiguous_1() { + let tensor = QTensor::::int8([ + [[2.0, 4.0, 1.0], [7.0, -5.0, 3.0]], + [[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]], + ]); + + let output = tensor.swap_dims(0, 2).sum_dim(1); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::new(vec![9.0, 7.0, -1.0, 3.0, 4.0, 5.0], [3, 1, 2]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_should_sum_mid_dim_3d_non_contiguous_2() { + let tensor = QTensor::::int8([ + [[2.0, 4.0, 1.0], [7.0, -5.0, 3.0]], + [[3.0, 1.0, 2.0], [4.0, 2.0, 3.0]], + ]); + + let output = tensor.swap_dims(0, 1).sum_dim(1); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::new(vec![5.0, 5.0, 3.0, 11.0, -3.0, 6.0], [2, 1, 3]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn test_prod_float() { + let tensor = QTensor::::int8([[2.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.prod(); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&TensorData::from([240.0]), Tolerance::rel_abs(1e-1, 1e-1)); + + let tensor_with_zero = QTensor::::int8([[2.0, 0.0, 2.0], [3.0, 4.0, 5.0]]); + let output = tensor_with_zero.prod(); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&TensorData::from([0.0]), Tolerance::rel_abs(1e-1, 1e-1)); +} + +#[test] +fn test_prod_dim_float() { + let tensor = QTensor::::int8([[2.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.prod_dim(1); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[4.0], [60.0]]), + Tolerance::absolute(1e-1), + ); + + let tensor_with_zero = QTensor::::int8([[2.0, 0.0, 2.0], [3.0, 4.0, 5.0]]); + let output = tensor_with_zero.prod_dim(1); + let expected = TensorData::from([[0.0], [60.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/all.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/all.rs new file mode 100644 index 0000000..3ac81be --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/all.rs @@ -0,0 +1,24 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_all() { + let tensor = QTensor::::int8([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([false]); + assert_eq!(data_expected, data_actual); + + let tensor = QTensor::::int8([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([true]); + assert_eq!(data_expected, data_actual); +} + +#[test] +fn test_all_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 0.0], [1.0, -1.0, 1.0]]); + let data_actual = tensor.all_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + assert_eq!(data_expected, data_actual); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/any.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/any.rs new file mode 100644 index 0000000..47c7562 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/any.rs @@ -0,0 +1,25 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_any() { + let tensor = QTensor::::int8([[0.0, 0.0, 0.0], [1.0, -1.0, 0.0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([true]); + assert_eq!(data_expected, data_actual); + + let tensor = QTensor::::int8([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([false]); + assert_eq!(data_expected, data_actual); +} + +#[test] +fn test_any_dim() { + let tensor = QTensor::::int8([[0.0, 0.0, 0.0], [1.0, -1.0, 0.0]]); + + let data_actual = tensor.any_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + assert_eq!(data_expected, data_actual); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/arg.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/arg.rs new file mode 100644 index 0000000..be03103 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/arg.rs @@ -0,0 +1,47 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_argmax_2d_dim0() { + let tensor = QTensor::::int8([[10.0, 11.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.argmax(0); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 0, 1]]), false); +} + +#[test] +fn test_argmin_2d_dim0() { + let tensor = QTensor::::int8([[10.0, 11.0, 2.0], [30.0, 4.0, 5.0]]); + + let output = tensor.argmin(0); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 0]]), false); +} + +#[test] +fn test_argmax_2d_dim1() { + let tensor = QTensor::::int8([[10.0, 11.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.argmax(1); + + output + .into_data() + .assert_eq(&TensorData::from([[1], [2]]), false); +} + +#[test] +fn test_argmin_2d_dim1() { + let tensor = QTensor::::int8([[10.0, 11.0, 2.0], [30.0, 4.0, 5.0]]); + + let output = tensor.argmin(1); + + output + .into_data() + .assert_eq(&TensorData::from([[2], [1]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cat.rs new file mode 100644 index 0000000..e12fcb1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cat.rs @@ -0,0 +1,65 @@ +use super::qtensor::*; +use super::*; +use alloc::vec; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_cat_ops_2d_dim0() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0, 6.0]]); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_cat_ops_2d_dim1() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0, 6.0]]); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 1); + let expected = TensorData::from([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_cat_ops_3d() { + let tensor_1 = QTensor::::int8([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]]]); + let tensor_2 = QTensor::::int8([[[4.0, 5.0, 6.0]]]); + + let output = TestTensor::cat(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[1.0, 2.0, 3.0]], [[1.1, 2.1, 3.1]], [[4.0, 5.0, 6.0]]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +#[should_panic] +fn should_panic_when_dimensions_are_not_the_same() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0]]); + + let _output = TestTensor::cat(vec![tensor_1, tensor_2], 0); +} + +#[test] +#[should_panic] +fn should_panic_when_cat_exceeds_dimension() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0, 6.0]]); + + let _output = TestTensor::cat(vec![tensor_1, tensor_2], 3); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/ceil.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/ceil.rs new file mode 100644 index 0000000..e4703d7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/ceil.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_ceil_ops() { + let tensor = + QTensor::::int8([[24.0423, 87.9478, 76.1838], [59.6929, 43.8169, 94.8826]]); + + let output = tensor.ceil(); + let expected = TensorData::from([[25., 88., 77.], [60., 44., 96.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(1e-1, 1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/chunk.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/chunk.rs new file mode 100644 index 0000000..243a31c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/chunk.rs @@ -0,0 +1,98 @@ +use super::qtensor::*; +use super::*; +use alloc::vec::Vec; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_chunk_evenly_divisible() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let tensors: Vec> = tensor.chunk(3, 0); + assert_eq!(tensors.len(), 3); + + let expected = [ + TensorData::from([0., 1.]), + TensorData::from([2., 3.]), + TensorData::from([4., 5.]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +fn test_chunk_not_evenly_divisible() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + + let tensors: Vec> = tensor.chunk(4, 0); + assert_eq!(tensors.len(), 4); + + let expected = [ + TensorData::from([0., 1.]), + TensorData::from([2., 3.]), + TensorData::from([4., 5.]), + TensorData::from([6.]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +fn test_chunk_not_divisible() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let tensors: Vec> = tensor.chunk(7, 0); + assert_eq!(tensors.len(), 6); + + let expected = [ + TensorData::from([0.]), + TensorData::from([1.]), + TensorData::from([2.]), + TensorData::from([3.]), + TensorData::from([4.]), + TensorData::from([5.]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +fn test_chunk_multi_dimension() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]]); + + let tensors: Vec> = tensor.chunk(2, 1); + assert_eq!(tensors.len(), 2); + + let expected = [ + TensorData::from([[0., 1., 2.]]), + TensorData::from([[3., 4., 5.]]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +#[should_panic] +fn test_invalid_dim() { + let _tensors = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]).chunk(6, 1); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/clamp.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/clamp.rs new file mode 100644 index 0000000..dde9892 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/clamp.rs @@ -0,0 +1,49 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn clamp_min() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.clamp_min(2.0); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[2.0, 2.0, 2.0], [3.0, 4.0, 5.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn clamp_max() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.clamp_max(2.0); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 1.0, 2.0], [2.0, 2.0, 2.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn clamp_min_max() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.clamp(1.0, 4.0); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[1.0, 1.0, 2.0], [3.0, 4.0, 4.0]]), + Tolerance::absolute(1e-1), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cos.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cos.rs new file mode 100644 index 0000000..9121a77 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cos.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_cos_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.cos(); + let expected = TensorData::from([[1.0, 0.5403, -0.4161], [-0.9899, -0.6536, 0.2836]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cosh.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cosh.rs new file mode 100644 index 0000000..63d5318 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/cosh.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_cosh_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.cosh(); + let expected = TensorData::from([[1.0000, 1.5431, 3.7622], [10.0677, 27.3082, 74.2100]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/div.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/div.rs new file mode 100644 index 0000000..aae2ad5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/div.rs @@ -0,0 +1,50 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_div_ops() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor_1 / tensor_2; + let expected = TensorData::from([[0.0, 1.0, 1.0], [1.0, 1.0, 1.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_div_broadcast() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0]]); + let tensor_2 = QTensor::::int8([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor_1 / tensor_2; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 1.0, 1.0], [0.0, 0.25, 0.4]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_support_div_scalar_ops() { + let scalar = 2.0; + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor / scalar; + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 0.5, 1.0], [1.5, 2.0, 2.5]]), + Tolerance::absolute(1e-1), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/erf.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/erf.rs new file mode 100644 index 0000000..d1231db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/erf.rs @@ -0,0 +1,33 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_erf_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.erf(); + let expected = TensorData::from([[0.0000, 0.8427, 0.9953], [1.0000, 1.0000, 1.0000]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_erf_ops_with_negative_number() { + let tensor = QTensor::::int8([[-0.056, -0.043, -0.089], [3.0, 4.0, 5.0]]); + + let output = tensor.erf(); + let expected = TensorData::from([ + [-0.06312324, -0.048490416, -0.10016122], + [1.0000, 1.0000, 1.0000], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/exp.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/exp.rs new file mode 100644 index 0000000..a105970 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/exp.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_exp_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.exp(); + let expected = TensorData::from([[1.0, 2.71830, 7.3891], [20.0855, 54.5981, 148.4132]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/expand.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/expand.rs new file mode 100644 index 0000000..fb9178c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/expand.rs @@ -0,0 +1,112 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn expand_2d() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0]); + let output = tensor.expand([3, 3]); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [1.0, 2.0, 3.0]]), + Tolerance::absolute(1e-1), + ); + + // Quantized [4.0, 7.0, 2.0, 3.0] + let tensor = QTensor::::int8([4.0, 7.0, 2.0, 3.0]); + let output = tensor.expand([2, 4]); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[4.0, 7.0, 2.0, 3.0], [4.0, 7.0, 2.0, 3.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn expand_3d() { + let tensor = QTensor::::int8([[1.0, 2.0], [3.0, 4.0]]); + + let output = tensor.expand([3, 2, 2]); + let expected = TensorData::from([ + [[1.0, 2.0], [3.0, 4.0]], + [[1.0, 2.0], [3.0, 4.0]], + [[1.0, 2.0], [3.0, 4.0]], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn expand_higher_dimensions() { + let tensor = QTensor::::int8([[1.0, 2.0, 3.0, 4.0]]); + + let output = tensor.expand([2, 3, 4]); + let expected = TensorData::from([ + [ + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + ], + [ + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0], + ], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn broadcast_single() { + let tensor = QTensor::::int8([1.0]); + + let output = tensor.expand([2, 3]); + + output + .dequantize() + .into_data() + .assert_eq(&TensorData::from([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]), false); +} + +#[test] +#[should_panic] +fn should_fail_expand_incompatible_shapes() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0]); + let _expanded_tensor = tensor.expand([2, 2]); +} + +#[test] +fn should_all_negative_one() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0]); + + let output = tensor.expand([2, -1]); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[1., 2., 3.], [1., 2., 3.]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +#[should_panic] +fn should_panic_negative_one_on_non_existing_dim() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0]); + let _expanded_tensor = tensor.expand([-1, 3]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/flip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/flip.rs new file mode 100644 index 0000000..9e54451 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/flip.rs @@ -0,0 +1,39 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn flip_float() { + let tensor = QTensor::::int8([[[0.0, 1.0, 2.0]], [[3.0, 4.0, 5.0]]]); + + let flipped = tensor.clone().flip([0, 2]); + let expected = TensorData::from([[[5., 4., 3.]], [[2., 1., 0.]]]); + + flipped + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); + + // Test with no flip + let flipped = tensor.clone().flip([]); + tensor.into_data().assert_eq(&flipped.into_data(), true); +} + +#[test] +#[should_panic] +fn flip_duplicated_axes() { + let tensor = QTensor::::int8([[[0.0, 1.0, 2.0]], [[3.0, 4.0, 5.0]]]); + + // Test with a duplicated axis + let _ = tensor.flip([0, 0, 1]); +} + +#[test] +#[should_panic] +fn flip_out_of_bound_axis() { + let tensor = QTensor::::int8([[[0.0, 1.0, 2.0]], [[3.0, 4.0, 5.0]]]); + + // Test with an out of bound axis + let _ = tensor.clone().flip([3, 0, 1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/floor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/floor.rs new file mode 100644 index 0000000..b4f1ff1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/floor.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_floor_ops() { + let tensor = + QTensor::::int8([[24.0423, 87.9478, 76.1838], [59.6929, 43.8169, 94.8826]]); + + let output = tensor.floor(); + let expected = TensorData::from([[24., 87., 76.], [59., 43., 95.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(1e-1, 1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/gather_scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/gather_scatter.rs new file mode 100644 index 0000000..7ed4573 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/gather_scatter.rs @@ -0,0 +1,198 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::IndexingUpdateOp; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_gather_1d_dim0() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + let indices = TestTensorInt::from_ints([1, 1, 0, 1, 2], &Default::default()); + + let output = tensor.gather(0, indices); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([1.0, 1.0, 0.0, 1.0, 2.0]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_gather_2d_dim0() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::from_ints([[0, 1, 0], [1, 0, 1]], &Default::default()); + + let output = tensor.gather(0, indices); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 4.0, 2.0], [3.0, 1.0, 5.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_gather_2d_dim1() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::from_ints([[2, 1, 0, 0], [2, 0, 1, 2]], &Default::default()); + + let output = tensor.gather(1, indices); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[2.0, 1.0, 0.0, 0.0], [5.0, 3.0, 4.0, 5.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_gather_3d_dim1() { + let tensor = QTensor::::int8([ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ]); + let indices = TestTensorInt::from_ints( + [[[1, 0, 0], [0, 1, 0]], [[0, 0, 1], [0, 1, 1]]], + &Default::default(), + ); + + let output = tensor.gather(1, indices); + let expected = TensorData::from([ + [[3.0, 1.0, 2.0], [0.0, 4.0, 2.0]], + [[6.0, 7.0, 11.0], [6.0, 10.0, 11.0]], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_gather_2d_only_1dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::<2>::from_ints([[1, 2]], &Default::default()).reshape([2, 1]); + + let output = tensor.gather(1, indices); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[1.0], [5.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_scatter_1d() { + let tensor = QTensor::::int8([0.0, 0.0, 0.0]); + let values = QTensor::::int8([5.0, 4.0, 3.0]); + let indices = TestTensorInt::from_ints([1, 0, 2], &Default::default()); + + let output = tensor.scatter(0, indices, values, IndexingUpdateOp::Add); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([4.0, 5.0, 3.0]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_scatter_2d_dim0() { + let tensor = QTensor::::int8([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]); + let values = QTensor::::int8([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let indices = TestTensorInt::from_ints([[1, 0, 1], [1, 1, 0]], &Default::default()); + + let output = tensor.scatter(0, indices, values, IndexingUpdateOp::Add); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 2.0, 6.0], [5.0, 5.0, 3.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_scatter_2d_dim1() { + let tensor = QTensor::::int8([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]); + let values = QTensor::::int8([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let indices = TestTensorInt::from_ints([[1, 0, 2], [1, 2, 0]], &Default::default()); + + let output = tensor.scatter(1, indices, values, IndexingUpdateOp::Add); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[2.0, 1.0, 3.0], [6.0, 4.0, 5.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +fn should_scatter_3d_dim1() { + let tensor = QTensor::::int8([ + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], + [[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]], + ]); + let values = QTensor::::int8([ + [[12.0, 13.0, 14.0], [15.0, 16.0, 17.0]], + [[18.0, 19.0, 20.0], [21.0, 22.0, 23.0]], + ]); + let indices = TestTensorInt::from_ints( + [[[1, 0, 0], [0, 1, 0]], [[0, 0, 1], [0, 1, 1]]], + &Default::default(), + ); + + let output = tensor.scatter(1, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([ + [[15.0, 14.0, 33.0], [15.0, 20.0, 5.0]], + [[45.0, 26.0, 8.0], [9.0, 32.0, 54.0]], + ]); + + // Set higher tolerance (0.2) due to larger de/quantization errors + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(2e-1)); +} + +#[test] +fn should_scatter_2d_dim1_diff_shape() { + let tensor = QTensor::::int8([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]); + let values = QTensor::::int8([[1.0], [4.0]]); + let indices = TestTensorInt::from_ints([[1], [2]], &Default::default()); + + let output = tensor.scatter(1, indices, values, IndexingUpdateOp::Add); + + output + .dequantize() + .into_data() + .assert_approx_eq::( + &TensorData::from([[0.0, 1.0, 0.0], [0.0, 0.0, 4.0]]), + Tolerance::absolute(1e-1), + ); +} + +#[test] +#[should_panic] +fn scatter_should_panic_on_mismatch_of_shapes() { + let tensor = QTensor::::int8([0.0, 0.0, 0.0]); + let values = QTensor::::int8([1.0, 4.0]); + let indices = TestTensorInt::from_ints([1, 0, 2], &Default::default()); + + tensor.scatter(0, indices, values, IndexingUpdateOp::Add); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log.rs new file mode 100644 index 0000000..bdbb81c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log.rs @@ -0,0 +1,20 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_log_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.log(); + let expected = TensorData::from([ + [-f32::INFINITY, 0.0, core::f32::consts::LN_2], + [1.0986, 1.3862, 1.6094], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log1p.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log1p.rs new file mode 100644 index 0000000..0649558 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/log1p.rs @@ -0,0 +1,20 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_exp_log1p() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.log1p(); + let expected = TensorData::from([ + [0.0, core::f32::consts::LN_2, 1.0986], + [1.3862, 1.6094, 1.7917], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/map_comparison.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/map_comparison.rs new file mode 100644 index 0000000..d8f28f8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/map_comparison.rs @@ -0,0 +1,157 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_equal() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[0.0, 1.0, 1.0], [3.0, 5.0, 4.0]]); + + let data_actual_cloned = tensor_1.clone().equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.equal(tensor_2); + + let data_expected = TensorData::from([[true, true, false], [true, false, false]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_not_equal() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[0.0, 1.0, 1.0], [3.0, 5.0, 4.0]]); + + let data_actual_cloned = tensor_1.clone().not_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.not_equal(tensor_2); + + let data_expected = TensorData::from([[false, false, true], [false, true, true]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +#[ignore = "quantization equality with float element is undefined"] +fn test_equal_elem() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 2.0, 5.0]]); + + let data_actual_cloned = tensor.clone().equal_elem(2); + let data_actual_inplace = tensor.equal_elem(2); + + let data_expected = TensorData::from([[false, false, true], [false, true, false]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +#[ignore = "quantization equality with float element is undefined"] +fn test_not_equal_elem() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 2.0, 5.0]]); + + let data_actual_cloned = tensor.clone().not_equal_elem(2); + let data_actual_inplace = tensor.not_equal_elem(2); + + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +#[ignore = "quantization equality with float element is undefined"] +fn test_greater_elem() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor.clone().greater_elem(4); + let data_actual_inplace = tensor.greater_elem(4); + + let data_expected = TensorData::from([[false, false, false], [false, false, true]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_greater_equal_elem() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor.clone().greater_equal_elem(4.0); + let data_actual_inplace = tensor.greater_equal_elem(4.0); + + let data_expected = TensorData::from([[false, false, false], [false, true, true]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_greater() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[0.0, 1.0, 1.0], [3.0, 5.0, 4.0]]); + + let data_actual_cloned = tensor_1.clone().greater(tensor_2.clone()); + let data_actual_inplace = tensor_1.greater(tensor_2); + + let data_expected = TensorData::from([[false, false, true], [false, false, true]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_greater_equal() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 1.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 5.0, 4.0]]); + + let data_actual_cloned = tensor_1.clone().greater_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.greater_equal(tensor_2); + + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_lower_elem() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor.clone().lower_elem(4.0); + let data_actual_inplace = tensor.lower_elem(4.0); + + let data_expected = TensorData::from([[true, true, true], [true, false, false]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +#[ignore = "quantization equality with float element is undefined"] +fn test_lower_equal_elem() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let data_actual_cloned = tensor.clone().lower_equal_elem(4.0); + let data_actual_inplace = tensor.lower_equal_elem(4.0); + + let data_expected = TensorData::from([[true, true, true], [true, true, false]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_lower() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 1.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 5.0, 4.0]]); + + let data_actual_cloned = tensor_1.clone().lower(tensor_2.clone()); + let data_actual_inplace = tensor_1.lower(tensor_2); + + let data_expected = TensorData::from([[false, false, true], [false, true, false]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} + +#[test] +fn test_lower_equal() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[0.0, 1.0, 1.0], [3.0, 5.0, 4.0]]); + + let data_actual_cloned = tensor_1.clone().lower_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.lower_equal(tensor_2); + + let data_expected = TensorData::from([[true, true, false], [true, true, false]]); + assert_eq!(data_expected, data_actual_cloned.into_data()); + assert_eq!(data_expected, data_actual_inplace.into_data()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mask.rs new file mode 100644 index 0000000..9e65bf5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mask.rs @@ -0,0 +1,39 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_mask_where_ops() { + let tensor = QTensor::::int8([[1.0, 7.0], [2.0, 3.0]]); + let mask = TestTensorBool::<2>::from_bool( + TensorData::from([[true, false], [false, true]]), + &Default::default(), + ); + let value = QTensor::::int8([[1.8, 2.8], [3.8, 4.8]]); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([[1.8, 7.0], [2.0, 4.8]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_mask_fill_ops() { + let tensor = QTensor::::int8([[1.0, 7.0], [2.0, 3.0]]); + let mask = TestTensorBool::<2>::from_bool( + TensorData::from([[true, false], [false, true]]), + &Default::default(), + ); + + let output = tensor.mask_fill(mask, 2.0); + let expected = TensorData::from([[2.0, 7.0], [2.0, 2.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/maxmin.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/maxmin.rs new file mode 100644 index 0000000..3d7574a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/maxmin.rs @@ -0,0 +1,149 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_max_dim_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.max_dim(1); + let expected = TensorData::from([[2.], [5.]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn test_max_dim_with_indices_2d_with_dim_0th() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let (output, index) = tensor.max_dim_with_indices(0); + + let output_expected = TensorData::from([[3., 4., 5.]]); + let index_expected = TensorData::from([[1, 1, 1]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&output_expected, Tolerance::rel_abs(2e-2, 1e-2)); + index.into_data().assert_eq(&index_expected, false); +} + +#[test] +fn test_max_dim_with_indices_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let (output, index) = tensor.max_dim_with_indices(1); + + let output_expected = TensorData::from([[2.], [5.]]); + let index_expected = TensorData::from([[2], [2]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&output_expected, Tolerance::rel_abs(2e-2, 1e-2)); + index.into_data().assert_eq(&index_expected, false); +} + +#[test] +fn test_min_dim_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.min_dim(1); + + let expected = TensorData::from([[0.], [3.]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn test_min_dim_with_indices_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let (output, index) = tensor.min_dim_with_indices(1); + + let output_expected = TensorData::from([[0.], [3.]]); + let index_expected = TensorData::from([[0], [0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&output_expected, Tolerance::rel_abs(2e-2, 1e-2)); + index.into_data().assert_eq(&index_expected, false); +} + +#[test] +fn test_min_dim_2d_with_0th_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.min_dim(0); + let expected = TensorData::from([[0., 1., 2.]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn test_max_dim_2d_with_0th_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.max_dim(0); + let expected = TensorData::from([[3., 4., 5.]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn test_min_dim_with_indices_2d_with_0th_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let (output, index) = tensor.min_dim_with_indices(0); + + let output_expected = TensorData::from([[0., 1., 2.]]); + let index_expected = TensorData::from([[0, 0, 0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&output_expected, Tolerance::rel_abs(2e-2, 1e-2)); + index.into_data().assert_eq(&index_expected, false); +} + +#[test] +fn test_maximum_pair() { + let a = QTensor::::int8([1.0, 5.0, 3.0, 4.0]); + let b = QTensor::::int8([2.0, 1.0, 4.0, 5.0]); + + let output = a.max_pair(b); + let expected = TensorData::from([2.0, 5.0, 4.0, 5.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn test_minimum_pair() { + let a = QTensor::::int8([1.0, 5.0, 3.0, 4.0]); + let b = QTensor::::int8([2.0, 1.0, 4.0, 5.0]); + + let output = a.min_pair(b); + let expected = TensorData::from([1.0, 1.0, 3.0, 4.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mod.rs new file mode 100644 index 0000000..a3cc89f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mod.rs @@ -0,0 +1,50 @@ +pub use super::*; + +mod abs; +mod add; +mod aggregation; +mod all; +mod any; +mod arg; +mod cat; +mod ceil; +mod chunk; +mod clamp; +mod cos; +mod cosh; +mod div; +mod erf; +mod exp; +mod expand; +mod flip; +mod floor; +mod gather_scatter; +mod log; +mod log1p; +mod map_comparison; +mod mask; +mod maxmin; +mod mul; +mod narrow; +mod neg; +mod permute; +mod powf; +mod powf_scalar; +mod recip; +mod remainder; +mod repeat_dim; +mod reshape; +mod round; +mod select; +mod sin; +mod sinh; +mod slice; +mod sort_argsort; +mod split; +mod sqrt; +mod stack; +mod sub; +mod tan; +mod tanh; +mod topk; +mod transpose; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mul.rs new file mode 100644 index 0000000..b0e4ff0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/mul.rs @@ -0,0 +1,60 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_mul_ops() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = tensor_1.clone(); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0.0, 1.0, 4.0], [9.0, 16.0, 25.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(5e-2, 1e-2)); +} + +#[test] +fn test_mul_broadcast() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0]]); + let tensor_2 = QTensor::::int8([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0.0, 4.0, 10.0], [0.0, 7.0, 16.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn test_mul_broadcast_2_dims() { + let tensor_1 = QTensor::::int8([[0.0], [1.0], [2.0]]); + let tensor_2 = QTensor::::int8([[3.0, 4.0, 5.0]]); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0.0, 0.0, 0.0], [3.0, 4.0, 5.0], [6.0, 8.0, 10.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_mul_scalar_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let scalar = 2.0; + + let output = tensor * scalar; + let expected = TensorData::from([[0.0, 2.0, 4.0], [6.0, 8.0, 10.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/narrow.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/narrow.rs new file mode 100644 index 0000000..da2faca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/narrow.rs @@ -0,0 +1,58 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{Shape, TensorData}; + +#[test] +fn test_narrow() { + let tensor = QTensor::::int8([[1., 2., 3.], [7., 8., 9.], [13., 14., 15.]]); + + let output = tensor.clone().narrow(0, 0, 2); + let expected = TensorData::from([[1., 2., 3.], [7., 8., 9.]]); + + assert_eq!(output.shape(), Shape::from([2, 3])); + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); + + let output = tensor.narrow(1, 1, 2); + let expected = TensorData::from([[2., 3.], [8., 9.], [14., 15.]]); + assert_eq!(output.shape(), Shape::from([3, 2])); + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_dim() { + let tensor = QTensor::::int8([[1., 2., 3.], [7., 8., 9.], [13., 14., 15.]]); + + let _output = tensor.narrow(2, 0, 2); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_start() { + let tensor = QTensor::::int8([[1., 2., 3.], [7., 8., 9.], [13., 14., 15.]]); + + let _output = tensor.narrow(0, 3, 2); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_zero_length() { + let tensor = QTensor::::int8([[1., 2., 3.], [7., 8., 9.], [13., 14., 15.]]); + + let _output = tensor.narrow(0, 1, 0); +} + +#[test] +#[should_panic] +fn test_narrow_invalid_length() { + let tensor = QTensor::::int8([[1., 2., 3.], [7., 8., 9.], [13., 14., 15.]]); + + let _output = tensor.narrow(0, 0, 4); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/neg.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/neg.rs new file mode 100644 index 0000000..c9ca933 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/neg.rs @@ -0,0 +1,19 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_neg_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.neg(); + let expected = TensorData::from([[-0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]).convert::(); + + // -0.0 is represented differently than 0.0 so we make sure the values are the same in f32 + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/permute.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/permute.rs new file mode 100644 index 0000000..ae82b05 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/permute.rs @@ -0,0 +1,66 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn permute_float() { + let tensor = QTensor::::int8([ + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ]) + .reshape([2, 2, 4]); + + let permuted = tensor.clone().permute([2, 1, 0]); + + let expected = TensorData::from([ + [[0., 8.], [4., 12.]], + [[1., 9.], [5., 13.]], + [[2., 10.], [6., 14.]], + [[3., 11.], [7., 15.]], + ]); + + permuted + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(1e-1, 1e-1)); + + // Test with negative axis + let permuted = tensor.clone().permute([-1, 1, 0]); + permuted + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(1e-1, 1e-1)); + + // Test with the same axis + let permuted = tensor.clone().permute([0, 1, 2]); + permuted + .dequantize() + .into_data() + .assert_approx_eq::( + &tensor.dequantize().into_data(), + Tolerance::rel_abs(1e-4, 1e-4), // dequant error should be the same + ); +} + +#[test] +#[should_panic] +fn edge_repeated_axes() { + let tensor = QTensor::::int8([ + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ]) + .reshape([2, 2, 4]); + + // Test with a repeated axis + let _ = tensor.permute([0, 0, 1]); +} + +#[test] +#[should_panic] +fn edge_out_of_bound_axis() { + let tensor = QTensor::::int8([ + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ]) + .reshape([2, 2, 4]); + + // Test with an invalid axis + let _ = tensor.permute([3, 0, 1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf.rs new file mode 100644 index 0000000..f051124 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf.rs @@ -0,0 +1,60 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_powf_ops() { + let tensor = QTensor::::int8([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_pow = QTensor::::int8([[1.0, 1.0, 2.0], [3.0, 4.0, 2.0]]); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[1.0, 1.0, 4.0], [27.0, 256.0, 25.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} + +#[test] +fn should_support_neg_power() { + let tensor = QTensor::::int8([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_pow = QTensor::::int8([[-0.95, -0.67, -0.45], [-0.24, -0.5, -0.6]]); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[1., 1., 0.73204285], [0.76822936, 0.5, 0.38073079]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} + +#[test] +fn should_support_neg_values_with_even_power() { + let tensor = QTensor::::int8([[0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor_pow = QTensor::::int8([[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[0.0, 1.0, 4.0], [9.0, 16.0, 25.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} + +#[test] +fn should_support_neg_values_with_odd_power() { + let tensor = QTensor::::int8([[0.0, -1.0, -2.0], [-3.0, -4.0, -4.0]]); + let tensor_pow = QTensor::::int8([[3.0, 3.0, 3.0], [3.0, 3.0, 3.0]]); + + let output = tensor.powf(tensor_pow); + let expected = TensorData::from([[0.0, -1.0, -8.0], [-27.0, -64.0, -64.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf_scalar.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf_scalar.rs new file mode 100644 index 0000000..f6e7f29 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/powf_scalar.rs @@ -0,0 +1,56 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_powf_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.powf_scalar(0.71); + let expected = TensorData::from([[0.0, 1.0, 1.6358], [2.182, 2.6759, 3.1352]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} + +#[test] +fn should_support_neg_power() { + let tensor = QTensor::::int8([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.powf_scalar(-0.33); + let expected = TensorData::from([[1.0, 1.0, 0.79553646], [0.695905, 0.6328783, 0.58794934]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} + +#[test] +fn should_support_neg_values_with_even_power() { + let tensor = QTensor::::int8([[0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + + let output = tensor.powf_scalar(2.0); + let expected = TensorData::from([[0., 1., 4.], [9., 16., 25.]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} + +#[test] +fn should_support_neg_values_with_odd_power() { + let tensor = QTensor::::int8([[0.0, -1.0, -2.0], [-3.0, -4.0, -4.0]]); + + let output = tensor.powf_scalar(3.0); + let expected = TensorData::from([[0.0, -1.0, -8.0], [-27.0, -64.0, -64.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(4e-2, 1e-2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/recip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/recip.rs new file mode 100644 index 0000000..791a736 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/recip.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_recip_ops() { + let tensor = QTensor::::int8([[0.5, 1.0, 2.0], [3.0, -4.0, -5.0]]); + + let output = tensor.recip(); + let expected = TensorData::from([[2.0, 1.0, 0.5], [0.33333, -0.25, -0.2]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/remainder.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/remainder.rs new file mode 100644 index 0000000..ba5b445 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/remainder.rs @@ -0,0 +1,208 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_remainder_basic() { + let lhs = QTensor::::int8([-3.0, -2.0, -1.0, 1.0, 2.0, 2.0]); + let rhs = QTensor::::int8([2.0, 3.0, 1.0, 2.0, 1.0, 2.0]); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([1., 1., 0., 1., 0., 0.]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +#[ignore = "quantization remainder with float element is undefined"] +fn should_support_remainder_basic_scalar() { + let tensor = QTensor::::int8([-3.0, -2.0, -1.0, 1.0, 2.0, 3.0]); + + let output = tensor.remainder_scalar(2.0); + let expected = TensorData::from([1.0, 0.0, 1.0, 1.0, 0.0, 1.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_remainder_float() { + let lhs = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + let rhs = QTensor::::int8([1.4233, 2.7313, 0.2641, 1.9651, 0.5897]); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([1., 2., 0.0949, 0.0698, 0.2824]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_remainder_float_scalar() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + + let output = tensor.remainder_scalar(-1.5); + let expected = TensorData::from([-0.5, -1.0, 0.0, -0.5, -1.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_be_zero() { + let lhs = QTensor::::int8([0.0, 0.0, 0.0]); + let rhs = QTensor::::int8([3.5, -2.1, 1.5]); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([0.0, 0.0, 0.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_be_zero_scalar() { + let tensor = QTensor::::int8([0.0, 0.0, 0.0]); + + let output = tensor.remainder_scalar(3.5); + let expected = TensorData::from([0.0, 0.0, 0.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_have_no_remainder() { + let lhs = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + let rhs = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([0.0, 0.0, 0.0, 0.0, 0.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_have_no_remainder_scalar() { + let tensor = QTensor::::int8([4.0, 4.0]); + + let output = tensor.remainder_scalar(4.0); + let expected = TensorData::from([0.0, 0.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_be_negative() { + let lhs = QTensor::::int8([-7.0, -3.0, 2.0, 6.0]); + let rhs = QTensor::::int8([-2.5, -2.1, -1.5, -3.25]); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([-2., -0.9, -1., -0.5]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_be_negative_scalar() { + let tensor = QTensor::::int8([-7.0, -3.0, 2.0, 6.0]); + + let output = tensor.remainder_scalar(-2.5); + let expected = TensorData::from([-2.0, -0.50, -0.50, -1.5]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_fp_dividends() { + let tensor = QTensor::::int8([-7.5, -2.5, 2.5, 7.5]); + + let output = tensor.remainder_scalar(3.0); + let expected = TensorData::from([1.5, 0.5, 2.5, 1.5]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_large_divisor() { + let lhs = QTensor::::int8([-1.0, 1.0, -1.5, 1.5, -1.0, 1.0, -1.5, 1.5]); + let rhs = QTensor::::int8([10.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, -10.0]); + + let output = lhs.remainder(rhs); + let expected = TensorData::from([9., 1., 8.5, 1.5, -1., -9., -1.5, -8.5]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_large_divisor_scalar() { + let tensor = QTensor::::int8([-1.0, 1.0]); + + let output = tensor.remainder_scalar(10.0); + let expected = TensorData::from([9.0, 1.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_remainder_op() { + let lhs = QTensor::::int8([-3.0, -2.0, -1.0, 1.0, 2.0, 2.0]); + let rhs = QTensor::::int8([2.0, 3.0, 1.0, 2.0, 1.0, 2.0]); + + let output = lhs % rhs; + let expected = TensorData::from([1., 1., 0., 1., 0., 0.]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +#[ignore = "quantization remainder with float element is undefined"] +fn should_support_remainder_scalar_op() { + let tensor = QTensor::::int8([-3.0, -2.0, -1.0, 1.0, 2.0, 3.0]); + + let output = tensor % 2.0; + let expected = TensorData::from([1.0, 0.0, 1.0, 1.0, 0.0, 1.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/repeat_dim.rs new file mode 100644 index 0000000..7dd681d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/repeat_dim.rs @@ -0,0 +1,42 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_support_repeat_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0, 3.0]]); + + let output = tensor.repeat_dim(0, 4); + let expected = TensorData::from([ + [0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::permissive()); +} + +#[test] +fn should_support_repeat_on_dims_larger_than_1() { + let tensor = QTensor::::int8([ + 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., + ]) + .reshape([4, 2, 2]); + + let output = tensor.repeat_dim(2, 2); + let expected = TensorData::from([ + [[0., 1., 0., 1.], [2., 3., 2., 3.]], + [[4., 5., 4., 5.], [6., 7., 6., 7.]], + [[8., 9., 8., 9.], [10., 11., 10., 11.]], + [[12., 13., 12., 13.], [14., 15., 14., 15.]], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(1e-1, 1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/reshape.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/reshape.rs new file mode 100644 index 0000000..3e93cdf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/reshape.rs @@ -0,0 +1,79 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn should_support_reshape_1d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0, 3.0]]); + + let output = tensor.clone().reshape([1, 4]); + let expected = TensorData::from([[0.0, 1.0, 2.0, 3.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_reshape_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.clone().reshape([6]); + let expected = TensorData::from([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_dim_infererence() { + let tensor = QTensor::::int8([ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + ]) + .reshape([4, 3]); + + // Infer the dimension via -1 + let reshaped = tensor.clone().reshape([2, -1]); + assert_eq!(reshaped.shape(), [2, 6].into()); + + // Infer the dimension via 0 (keep from the source) and -1 (infer) + let reshaped = reshaped.reshape([0, 2, -1]); + assert_eq!(reshaped.shape(), [2, 2, 3].into()); + + // This is effectively as if we did a flatten + let reshaped = tensor.clone().reshape([-1]); + assert_eq!(reshaped.shape(), [12].into()); + + // Keeping the first dimension the same (using 0) + let reshaped = tensor.clone().reshape([0, 3]); + assert_eq!(reshaped.shape(), [4, 3].into()); +} + +#[test] +fn should_not_corrupt_after_slice() { + let zeros = QTensor::::int8([0.0, 0.0]); + zeros.clone().slice([1..2]).reshape([1]).exp(); + + // May lead to zeroes being equal to [0.0, 1.0] + zeros.dequantize().into_data().assert_eq( + &TestTensor::<1>::zeros([2], &Default::default()).to_data(), + true, + ); +} + +#[test] +#[should_panic] +fn multiple_neg_ones() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + let _ = tensor.reshape([-1, -1]); +} + +#[test] +#[should_panic] +fn neg_value() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + let _ = tensor.reshape([-2, -1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/round.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/round.rs new file mode 100644 index 0000000..fc04f36 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/round.rs @@ -0,0 +1,32 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_round_ops() { + let tensor = + QTensor::::int8([[24.0423, 87.9478, 76.1838], [59.6929, 43.8169, 94.8826]]); + + let output = tensor.round(); + let expected = TensorData::from([[24., 88., 76.], [60., 44., 95.]]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_round_ties_even() { + // NOTE: round ties to even only affects values that are exact halfway from ceil/floor, so quantization + // errors can impact this. This basically only guarantees the values for the max value in the range since + // it is always represented correctly. + let tensor = QTensor::::int8([5.5]); + + let output = tensor.round(); + let expected = TensorData::from([6.]); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/select.rs new file mode 100644 index 0000000..0f7332e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/select.rs @@ -0,0 +1,120 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::IndexingUpdateOp; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_select_1d() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0]); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &Default::default()); + + let output = tensor.select(0, indices); + let expected = TensorData::from([1.0, 1.0, 0.0, 1.0, 2.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_select_2d_dim0_same_num_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::from_data([1, 0], &Default::default()); + + let output = tensor.select(0, indices); + let expected = TensorData::from([[3.0, 4.0, 5.0], [0.0, 1.0, 2.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_select_2d_dim0_more_num_dim() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::from_data([1, 0, 1, 1], &Default::default()); + + let output = tensor.select(0, indices); + let expected = TensorData::from([ + [3.0, 4.0, 5.0], + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [3.0, 4.0, 5.0], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_select_2d_dim1() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &Default::default()); + + let output = tensor.select(1, indices); + let expected = TensorData::from([[1.0, 1.0, 0.0, 1.0, 2.0], [4.0, 4.0, 3.0, 4.0, 5.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_select_assign_1d() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + let values = QTensor::::int8([5.0, 4.0, 3.0, 2.0, 1.0]); + let indices = TestTensorInt::from_data(TensorData::from([1, 1, 0, 1, 2]), &Default::default()); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([3.0, 12.0, 3.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_select_assign_2d_dim0() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let values = tensor.clone(); + let indices = TestTensorInt::from_data(TensorData::from([1, 0]), &Default::default()); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([[3.0, 5.0, 7.0], [3.0, 5.0, 7.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_select_assign_2d_dim1() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let values = tensor.clone(); + let indices = TestTensorInt::from_data(TensorData::from([1, 0, 2]), &Default::default()); + + let output = tensor.select_assign(1, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([[1.0, 1.0, 4.0], [7.0, 7.0, 10.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +#[should_panic] +fn should_select_panic_invalid_dimension() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &Default::default()); + + tensor.select(10, indices); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sin.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sin.rs new file mode 100644 index 0000000..5c4e645 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sin.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_sin_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sin(); + let expected = TensorData::from([[0.0, 0.8414, 0.9092], [0.1411, -0.7568, -0.9589]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sinh.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sinh.rs new file mode 100644 index 0000000..1962db4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sinh.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_sinh_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sinh(); + let expected = TensorData::from([[0.0000, 1.1752, 3.6269], [10.0179, 27.2899, 74.2032]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(3e-2, 1e-2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/slice.rs new file mode 100644 index 0000000..01fc9bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/slice.rs @@ -0,0 +1,229 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::{Tolerance, s}; + +#[test] +fn should_support_full_sliceing_1d() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0]); + let data = tensor.to_data(); + + let output = tensor.slice([0..4]); + + output.into_data().assert_eq(&data, false); +} + +#[test] +fn should_support_partial_sliceing_1d() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0]); + + let output = tensor.slice([1..3]); + let expected = TensorData::from([1.0, 2.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_full_sliceing_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data = tensor.to_data(); + + let output = tensor.clone().slice([0..2]); + output.into_data().assert_eq(&data, true); + + let output = tensor.slice([0..2, 0..3]); + output.into_data().assert_eq(&data, true); +} + +#[test] +fn should_support_partial_sliceing_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.slice([0..2, 0..2]); + let expected = TensorData::from([[0.0, 1.0], [3.0, 4.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_partial_sliceing_3d() { + let tensor = QTensor::::int8([ + [[0., 1., 2., 3.], [4., 5., 6., 7.]], + [[8., 9., 10., 11.], [12., 13., 14., 15.]], + ]); + + let output = tensor.slice([1..2, 1..2, 0..2]); + let expected = TensorData::from([[[12.0, 13.0]]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_partial_sliceing_3d_non_contiguous() { + let tensor = QTensor::::int8([ + [[0., 1., 2., 3.], [4., 5., 6., 7.]], + [[8., 9., 10., 11.], [12., 13., 14., 15.]], + ]); + + let output = tensor.transpose().slice([1..2, 1..2, 0..2]); + let expected = TensorData::from([[[9.0, 13.0]]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn should_support_slice_assign_1d() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + let tensor_assigned = QTensor::::int8([10.0, 5.0]); + + let output = tensor.slice_assign([0..2], tensor_assigned); + let expected = TensorData::from([10.0, 5.0, 2.0]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_slice_assign_2d() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_assigned = QTensor::::int8([[10.0, 5.0]]); + + let output = tensor.slice_assign([1..2, 0..2], tensor_assigned); + let expected = TensorData::from([[0.0, 1.0, 2.0], [10.0, 5.0, 5.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn slice_should_not_corrupt_potentially_inplace_operations() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + let tensor = tensor.clone().slice([0..3]) + tensor.clone().slice([2..5]); + + let expected = TensorData::from([4., 6., 8.]); + + tensor + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn slice_assign_should_not_corrupt_potentially_inplace_operations() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + let values = QTensor::::int8([10., 20., 30.]); + + let tensor_1 = tensor.clone().slice_assign([0..3], values); + let tensor_2 = tensor + 2; + + let expected = TensorData::from([10., 20., 30., 4., 5.]); + + tensor_1 + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); + + let expected = TensorData::from([3., 4., 5., 6., 7.]); + + tensor_2 + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn clamp_when_slice_exceeds_dimension() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + let data = tensor.to_data(); + + let output = tensor.slice([0..4]); + output.into_data().assert_eq(&data, true); +} + +#[test] +fn negative_dimensions() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data = tensor.to_data(); + + // Clamping to the tensor dimensions + let output = tensor.clone().slice([0..4, 0..4]); + output.into_data().assert_eq(&data, true); + + // Negative dimensions + let output = tensor.clone().slice([0..1, 0..1]); + let data = TensorData::from([[0.0f32]]); + output + .dequantize() + .into_data() + .assert_approx_eq::(&data, Tolerance::rel_abs(2e-2, 1e-2)); + + let output = tensor.slice(s![0..-1, 0..-2]); + output + .dequantize() + .into_data() + .assert_approx_eq::(&data, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +fn missing_dimensions() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let data = tensor.to_data(); + + // Clamping to the tensor dimensions + let output = tensor.clone().slice([0..4, 0..4]); + output.into_data().assert_eq(&data, true); + + // Negative dimensions + let data = TensorData::from([[0.0f32]]); + let output = tensor.clone().slice(s![0..-1, 0..-2]); + output + .dequantize() + .into_data() + .assert_approx_eq::(&data, Tolerance::rel_abs(2e-2, 1e-2)); + + // Missing dimensions + let output = tensor.clone().slice(s![0..1, ..]); + let data = TensorData::from([[0.0f32, 1.0, 2.0]]); + output + .dequantize() + .into_data() + .assert_approx_eq::(&data, Tolerance::rel_abs(2e-2, 1e-2)); + + let output = tensor.clone().slice(s![.., 0..2]); + let data = TensorData::from([[0.0f32, 1.0], [3.0, 4.0]]); + output + .dequantize() + .into_data() + .assert_approx_eq::(&data, Tolerance::rel_abs(2e-2, 1e-2)); + + let output = tensor.clone().slice([.., ..]); + let data = TensorData::from([[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]]); + output + .dequantize() + .into_data() + .assert_approx_eq::(&data, Tolerance::rel_abs(2e-2, 1e-2)); +} + +#[test] +#[should_panic] +fn should_panic_when_slice_with_too_many_dimensions() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0]); + + let _output = tensor.slice([0..1, 0..1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sort_argsort.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sort_argsort.rs new file mode 100644 index 0000000..e77ea41 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sort_argsort.rs @@ -0,0 +1,225 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_sort_1d_float() { + // Quantized [0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 5.2, 4., 0.99, 3., -8.1] + let tensor = QTensor::::int8([ + 0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 5.2, 4., 0.99, 3., -8.1, + ]); + + // Sort along dim=0 + let values = tensor.sort(0); + + let values_expected = TensorData::from([ + -8.1, -0.3, -0.21, 0., 0.5, 0.94, 0.99, 1.2, 2.1, 2.3, 3., 4., 5.2, + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_argsort_1d_float() { + let tensor = QTensor::::int8([ + 0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 5.2, 4., 0.99, 3., -8.1, + ]); + + // Sort along dim=0 + let indices = tensor.argsort(0); + + let indices_expected = TensorData::from([12, 6, 2, 3, 0, 5, 10, 1, 4, 7, 11, 9, 8]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_with_indices_descending_float() { + // 1D + let tensor = QTensor::::int8([ + 0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 5.2, 4., 0.99, 3., -8.1, + ]); + + // Sort along dim=0 + let (values, indices) = tensor.sort_descending_with_indices(0); + + let values_expected = TensorData::from([ + 5.2, 4., 3., 2.3, 2.1, 1.2, 0.99, 0.94, 0.5, 0., -0.21, -0.3, -8.1, + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + let indices_expected = TensorData::from([8, 9, 11, 7, 4, 1, 10, 5, 0, 3, 2, 6, 12]); + indices.into_data().assert_eq(&indices_expected, false); + + // 3D + // Quantized [-0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 4., 0.99, 3., -8.1] + let tensor = QTensor::::int8([ + -0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 4., 0.99, 3., -8.1, + ]) + .reshape([2, 2, 3]); + + // Sort along dim=1 + let (values, indices) = tensor.sort_descending_with_indices(1); + + let values_expected = TensorData::from([ + [[0., 2.1, 0.94], [-0.5, 1.2, -0.21]], + [[0.99, 3., 4.], [-0.3, 2.3, -8.1]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + let indices_expected = TensorData::from([[[1, 1, 1], [0, 0, 0]], [[1, 1, 0], [0, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_float() { + let tensor = QTensor::::int8([ + -0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 4., 0.99, 3., -8.1, + ]) + .reshape([2, 2, 3]); + + // Sort along dim=0 + let values = tensor.clone().sort(0); + + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, -8.1]], + [[-0.3, 2.3, 4.], [0.99, 3., 0.94]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + // Sort along dim=1 + let values = tensor.clone().sort(1); + + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, -8.1], [0.99, 3., 4.]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + // Sort along dim=2 + let values = tensor.sort(2); + + let values_expected = TensorData::from([ + [[-0.5, -0.21, 1.2], [0., 0.94, 2.1]], + [[-0.3, 2.3, 4.], [-8.1, 0.99, 3.]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_sort_with_indices_float() { + let tensor = QTensor::::int8([ + -0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 4., 0.99, 3., -8.1, + ]) + .reshape([2, 2, 3]); + + // Sort along dim=0 + let (values, indices) = tensor.clone().sort_with_indices(0); + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, -8.1]], + [[-0.3, 2.3, 4.], [0.99, 3., 0.94]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + let indices_expected = TensorData::from([[[0, 0, 0], [0, 0, 1]], [[1, 1, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=1 + let (values, indices) = tensor.clone().sort_with_indices(1); + + let values_expected = TensorData::from([ + [[-0.5, 1.2, -0.21], [0., 2.1, 0.94]], + [[-0.3, 2.3, -8.1], [0.99, 3., 4.]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + let indices_expected = TensorData::from([[[0, 0, 0], [1, 1, 1]], [[0, 0, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=2 + let (values, indices) = tensor.sort_with_indices(2); + + let values_expected = TensorData::from([ + [[-0.5, -0.21, 1.2], [0., 0.94, 2.1]], + [[-0.3, 2.3, 4.], [-8.1, 0.99, 3.]], + ]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + let indices_expected = TensorData::from([[[0, 2, 1], [0, 2, 1]], [[0, 1, 2], [2, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_argsort_float() { + let tensor = QTensor::::int8([ + -0.5, 1.2, -0.21, 0., 2.1, 0.94, -0.3, 2.3, 4., 0.99, 3., -8.1, + ]) + .reshape([2, 2, 3]); + + // Sort along dim=0 + let indices = tensor.clone().argsort(0); + + let indices_expected = TensorData::from([[[0, 0, 0], [0, 0, 1]], [[1, 1, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=1 + let indices = tensor.clone().argsort(1); + + let indices_expected = TensorData::from([[[0, 0, 0], [1, 1, 1]], [[0, 0, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=2 + let indices = tensor.argsort(2); + + let indices_expected = TensorData::from([[[0, 2, 1], [0, 2, 1]], [[0, 1, 2], [2, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_descending_1d() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + + // Sort along dim=0 + let values = tensor.sort_descending(0); + + let values_expected = TensorData::from([5., 4., 3., 2., 1.]); + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/split.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/split.rs new file mode 100644 index 0000000..99e56c0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/split.rs @@ -0,0 +1,151 @@ +use super::qtensor::*; +use super::*; +use alloc::vec; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_split_evenly_divisible() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let tensors = tensor.split(2, 0); + assert_eq!(tensors.len(), 3); + + let expected = [ + TensorData::from([0., 1.]), + TensorData::from([2., 3.]), + TensorData::from([4., 5.]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +fn test_split_not_evenly_divisible() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + + let tensors = tensor.split(2, 0); + assert_eq!(tensors.len(), 4); + + let expected = [ + TensorData::from([0., 1.]), + TensorData::from([2., 3.]), + TensorData::from([4., 5.]), + TensorData::from([6.]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +fn test_split_along_dim1() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let tensors = tensor.split(2, 1); + assert_eq!(tensors.len(), 2); + + let expected = [ + TensorData::from([[0., 1.], [3., 4.]]), + TensorData::from([[2.], [5.]]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +fn test_split_split_size_larger_than_tensor_size() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let tensors = tensor.split(10, 0); + assert_eq!(tensors.len(), 1); + + let expected = [TensorData::from([0.0, 1.0, 2.0, 3.0, 4.0, 5.0])]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +#[should_panic( + expected = "split_size must be greater than 0 unless the tensor size along the dimension is 0." +)] +fn test_split_with_zero_split_size_non_zero_tensor() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let _ = tensor.split(0, 0); +} + +#[test] +#[should_panic(expected = "Given dimension is greater than or equal to the tensor rank.")] +fn test_split_invalid_dim() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let _ = tensor.split(1, 2); +} + +#[test] +fn test_split_with_sizes() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let tensors = tensor.split_with_sizes(vec![2, 3, 1], 0); + assert_eq!(tensors.len(), 3); + + let expected = [ + TensorData::from([0., 1.]), + TensorData::from([2., 3., 4.]), + TensorData::from([5.]), + ]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} + +#[test] +#[should_panic( + expected = "The sum of split_sizes must equal the tensor size along the specified dimension." +)] +fn test_split_with_sizes_invalid_sum() { + let tensor = QTensor::::int8([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); + + let _ = tensor.split_with_sizes(vec![2, 2, 1], 0); +} + +#[test] +fn test_split_with_sizes_zero_length() { + let tensor = QTensor::::int8([0.0, 2.0, 5.0]); + + let tensors = tensor.split_with_sizes(vec![0, 1, 2], 0); + assert_eq!(tensors.len(), 2); + + let expected = [TensorData::from([0.]), TensorData::from([2., 5.])]; + + for (index, tensor) in tensors.into_iter().enumerate() { + tensor + .dequantize() + .to_data() + .assert_approx_eq::(&expected[index], Tolerance::absolute(1e-1)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sqrt.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sqrt.rs new file mode 100644 index 0000000..fad492e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sqrt.rs @@ -0,0 +1,18 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; +use core::f32::consts::SQRT_2; + +#[test] +fn should_support_sqrt_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.sqrt(); + let expected = TensorData::from([[0.0, 1.0, SQRT_2], [1.73205, 2.0, 2.2360]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/stack.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/stack.rs new file mode 100644 index 0000000..da490a8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/stack.rs @@ -0,0 +1,69 @@ +use super::qtensor::*; +use super::*; +use alloc::vec; +use burn_tensor::Tensor; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_stack_ops_2d_dim0() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0, 6.0]]); + + let output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[1.0, 2.0, 3.0]], [[4.0, 5.0, 6.0]]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_stack_ops_2d_dim1() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0, 6.0]]); + + let output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 1); + let expected = TensorData::from([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_stack_ops_3d() { + let tensor_1 = QTensor::::int8([[[1.0, 2.0, 3.0]], [[3.0, 2.0, 1.0]]]); + let tensor_2 = QTensor::::int8([[[4.0, 5.0, 6.0]], [[6.0, 5.0, 4.0]]]); + + let output = Tensor::stack::<4>(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([ + [[[1.0, 2.0, 3.0]], [[3.0, 2.0, 1.0]]], + [[[4.0, 5.0, 6.0]], [[6.0, 5.0, 4.0]]], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +#[should_panic] +fn should_panic_when_dimensions_are_not_the_same() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 5.0]]); + + let _output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 0); +} + +#[test] +#[should_panic] +fn should_panic_when_stack_exceeds_dimension() { + let tensor_1 = QTensor::::int8([[[1.0, 2.0, 3.0]], [[3.0, 2.0, 1.0]]]); + let tensor_2 = QTensor::::int8([[[4.0, 5.0, 6.0]]]); + + let _output = Tensor::stack::<4>(vec![tensor_1, tensor_2], 3); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sub.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sub.rs new file mode 100644 index 0000000..1024e7d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/sub.rs @@ -0,0 +1,46 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_sub_ops() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_2 = QTensor::::int8([[6.0, 7.0, 8.0], [9.0, 10.0, 11.0]]); + + let output = tensor_1 - tensor_2; + let expected = TensorData::from([[-6.0, -6.0, -6.0], [-6.0, -6.0, -6.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_sub_broadcast() { + let tensor_1 = QTensor::::int8([[0.0, 1.0, 2.0]]); + let tensor_2 = QTensor::::int8([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]); + + let output = tensor_1 - tensor_2; + let expected = TensorData::from([[-3.0, -3.0, -3.0], [-6.0, -6.0, -6.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_sub_scalar_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let scalar = 2.0; + + let output = tensor - scalar; + let expected = TensorData::from([[-2.0, -1.0, 0.0], [1.0, 2.0, 3.0]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(2e-2, 1e-2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tan.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tan.rs new file mode 100644 index 0000000..cf5532f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tan.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_tan_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.tan(); + let expected = TensorData::from([[0.0, 1.5574, -2.1850], [-0.1425, 1.1578, -3.3805]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tanh.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tanh.rs new file mode 100644 index 0000000..2b507dd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/tanh.rs @@ -0,0 +1,17 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_tanh_ops() { + let tensor = QTensor::::int8([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + + let output = tensor.tanh(); + let expected = TensorData::from([[0.0, 0.7615, 0.9640], [0.9950, 0.9993, 0.9999]]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/topk.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/topk.rs new file mode 100644 index 0000000..8a4349b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/topk.rs @@ -0,0 +1,72 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_topk_1d() { + let tensor = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + + let values = tensor.topk(3, /*dim*/ 0); + let expected = TensorData::from([5., 4., 3.]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_topk() { + let tensor = QTensor::::int8([ + [[1., 4., 7.], [2., 5., 6.]], + [[3., 0., 9.], [8., 2., 7.]], + ]); + + let values = tensor.topk(2, /*dim*/ 2); + let expected = TensorData::from([[[7., 4.], [6., 5.]], [[9., 3.], [8., 7.]]]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn test_topk_with_indices() { + // 1D + let tensor = QTensor::::int8([1.0, 2.0, 3.0, 4.0, 5.0]); + + let (values, indices) = tensor.topk_with_indices(3, /*dim*/ 0); + + let values_expected = TensorData::from([5., 4., 3.]); + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::permissive()); + + let indices_expected = TensorData::from([4, 3, 2]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_topk_with_indices_3d() { + // 3D + let tensor = QTensor::::int8([ + [[1., 4., 7.], [2., 5., 6.]], + [[3., 0., 9.], [8., 2., 7.]], + ]); + + let (values, indices) = tensor.topk_with_indices(2, /*dim*/ 2); + + let values_expected = TensorData::from([[[7., 4.], [6., 5.]], [[9., 3.], [8., 7.]]]); + + values + .dequantize() + .into_data() + .assert_approx_eq::(&values_expected, Tolerance::absolute(1e-1)); + + let indices_expected = TensorData::from([[[2, 1], [2, 1]], [[2, 0], [0, 2]]]); + + indices.into_data().assert_eq(&indices_expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/transpose.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/transpose.rs new file mode 100644 index 0000000..6419130 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/extended/transpose.rs @@ -0,0 +1,43 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn should_support_transpose_ops() { + let tensor = QTensor::::int8([ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + ]) + .reshape([2, 2, 3]); + + let output = tensor.transpose(); + let expected = TensorData::from([ + [[0.0, 3.0], [1.0, 4.0], [2.0, 5.0]], + [[6.0, 9.0], [7.0, 10.0], [8.0, 11.0]], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} + +#[test] +fn should_support_swap_dims() { + let tensor = QTensor::::int8([ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + ]) + .reshape([2, 2, 3]); + + let output = tensor.swap_dims(0, 2); + let expected = TensorData::from([ + [[0.0, 6.0], [3.0, 9.0]], + [[1.0, 7.0], [4.0, 10.0]], + [[2.0, 8.0], [5.0, 11.0]], + ]); + + output + .dequantize() + .into_data() + .assert_approx_eq::(&expected, Tolerance::absolute(1e-1)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/matmul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/matmul.rs new file mode 100644 index 0000000..5a7f32c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/matmul.rs @@ -0,0 +1,219 @@ +use super::qtensor::*; +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +#[ignore] +fn test_matmul_vectors() { + let tensor_1 = QTensor::::int8([[1.0, 2.0, 3.0, 6.35]]); + let tensor_2 = QTensor::::int8([[12.7], [4.0], [5.0], [1.0]]); + + let tensor_3 = tensor_1.matmul(tensor_2); + + let expected = TensorData::from([[42.05]]); + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +#[ignore] +fn test_matmul_2d() { + let tensor_1 = QTensor::::int8([[1.0, 6.35], [2.0, 3.0], [1.0, 3.0]]); + let tensor_2 = QTensor::::int8([[4.0, 8.0, 12.7], [2.0, 3.0, 6.0]]); + let tensor_3 = tensor_1.matmul(tensor_2); + + let expected = TensorData::from([[16.7, 27.05, 50.8], [14., 25., 43.4], [10., 17., 30.7]]); + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +fn test_matmul_2d_aligned() { + let tensor_1 = QTensor::::int8([ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ]); + let tensor_2 = QTensor::::int8([ + [2.0, 0.0, 1.0, 0.0], + [1.0, 2.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 0.0], + [1.0, 0.0, 0.0, 1.0], + ]); + let tensor_3 = tensor_1.matmul(tensor_2); + + let expected = TensorData::from([ + [8.0, 7.0, 7.0, 4.0], + [24.0, 19.0, 19.0, 8.0], + [40.0, 31.0, 31.0, 12.0], + ]); + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +fn test_matmul_2d_aligned_fused() { + let tensor_1 = QTensor::::int8([ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ]); + let tensor_2 = QTensor::::int8([ + [2.0, 0.0, 1.0, 0.0], + [1.0, 2.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 0.0], + [1.0, 0.0, 0.0, 1.0], + ]); + let tensor_3 = tensor_1.matmul(tensor_2); + let tensor_4 = tensor_3 / 2.0; + + let expected = TensorData::from([ + [4.0, 3.5, 3.5, 2.0], + [12.0, 9.5, 9.5, 4.0], + [20.0, 15.5, 15.5, 6.0], + ]); + tensor_4 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +#[ignore] +fn test_matmul_3d() { + let tensor_1 = QTensor::::int8([[[1.0, 6.35], [2.0, 3.0]]]); + let tensor_2 = QTensor::::int8([[[12.7, 4.0], [2.0, 3.0]]]); + + let tensor_3 = tensor_1.matmul(tensor_2); + + let expected = TensorData::from([[[25.4, 23.05], [31.4, 17.0]]]); + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +#[ignore] +fn test_matmul_broadcast_4d() { + let tensor_1 = + QTensor::::int8([[[[1.0, 7.0], [2.0, 3.0]]], [[[2.0, 5.0], [6.0, 3.0]]]]); + let tensor_2 = + QTensor::::int8([[[[9.0, 8.0], [1.0, 4.0]], [[2.0, 7.0], [3.0, 5.0]]]]); + + // [2, 1, 2, 2] @ [1, 2, 2, 2] -> [2, 2, 2, 2] + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [[[16.0, 36.0], [21.0, 28.0]], [[23.0, 42.0], [13.0, 29.0]]], + [[[23.0, 36.0], [57.0, 60.0]], [[19.0, 39.0], [21.0, 57.0]]], + ]); + + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +#[ignore] +fn test_matmul_broadcast() { + let tensor_1 = QTensor::::int8([[[1.0, 7.0], [2.0, 3.0]]]); + let tensor_2 = + QTensor::::int8([[[4.0, 7.0], [2.0, 3.0]], [[2.0, 5.0], [6.0, 3.0]]]); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[[18.0, 28.0], [14.0, 23.0]], [[44.0, 26.0], [22.0, 19.0]]]); + + tensor_3 + .into_data() + .assert_approx_eq::(&expected, Tolerance::relative(2e-2)); +} + +#[test] +#[should_panic] +fn should_panic_when_inner_dimensions_are_not_equal() { + let tensor_1 = QTensor::::int8([[3., 3.], [4., 4.], [5., 5.], [6., 6.]]); + let tensor_2 = + QTensor::::int8([[1., 2., 3., 4.], [1., 2., 3., 4.], [1., 2., 3., 4.]]); + + let _ = tensor_1.matmul(tensor_2); +} + +#[test] +fn test_matmul_lhs_float_rhs_quantized() { + // Simulates a typical workflow with linear layers (e.g., transformers), where the rhs + // represents the weights. The lhs might be a float if a previous operation did not propagate + // the quantization. We still want to perform an efficient matmul with quantized weights. + let tensor_1 = TestTensor::<2>::from([ + [1.0, 6.35, 2.0, 3.0], + [2.0, 3.0, 4.0, 5.0], + [1.0, 3.0, 5.0, 7.0], + ]); + let tensor_2 = QTensor::::int8([ + [4.0, 8.0, 12.7, 1.6], + [2.0, 3.0, 6.0, 4.0], + [1.0, 5.0, 9.0, 2.5], + [3.0, 7.0, 11.0, 0.5], + ]); + let tensor_3 = tensor_1.matmul(tensor_2); + + let expected = TensorData::from([ + [27.7, 58.05, 101.8, 33.5], + [33., 80., 134.4, 27.7], + [36., 91., 152.7, 29.6], + ]); + let output = tensor_3.into_data(); + output.assert_approx_eq::(&expected, Tolerance::default()); + + // Default quantization scheme does not propagate quantization with matmul + assert!(output.dtype.is_float()); +} + +#[test] +fn test_matmul_mixed_block_scale() { + let tensor_1 = TestTensor::<2>::from([ + [1.0, 6.35, 2.0, 3.0], + [2.0, 3.0, 4.0, 5.0], + [1.0, 3.0, 5.0, 7.0], + ]); + let tensor_2 = QTensor::::int8_block([ + [ + 6.110, 4.0, 9.360, 7.850, 0.630, 1.770, 0.430, 7.550, 9.690, 3.560, 2.920, 9.130, + 3.390, 0.510, 1.620, 1.460, + ], + [ + 6.140, 8.260, 5.660, 5.610, 7.070, 3.050, 9.890, 5.520, 1.350, 3.810, 5.630, 0.250, + 0.350, 8.860, 3.610, 6.240, + ], + [ + 8.810, 4.620, 7.420, 8.110, 2.560, 4.710, 5.730, 8.980, 1.170, 6.090, 4.140, 3.610, + 4.960, 9.720, 5.710, 1.470, + ], + [ + 2.260, 9.640, 6.320, 6.980, 9.860, 1.030, 8.340, 1.570, 4.140, 4.760, 4.590, 6.400, + 5.350, 1.430, 4.960, 1.180, + ], + ]); + let tensor_3 = tensor_1.matmul(tensor_2); + + let expected = TensorData::from([ + [ + 69.499, 94.611, 79.101, 80.633, 80.225, 33.647, 99.711, 65.272, 33.022, 54.213, 60.721, + 37.138, 31.582, 80.501, 50.843, 47.564, + ], + [ + 77.180, 99.460, 96.980, 99.870, 82.010, 36.680, 95.150, 75.430, 48.810, 66.710, 62.240, + 65.450, 54.420, 73.630, 61.710, 33.420, + ], + [ + 84.400, 119.360, 107.680, 114.090, 103.660, 41.680, 117.130, 80.0, 48.570, 78.760, + 72.640, 72.730, 66.690, 85.700, 75.720, 35.790, + ], + ]); + let output = tensor_3.into_data(); + output.assert_approx_eq::(&expected, Tolerance::permissive()); + + // Default quantization scheme does not propagate quantization with matmul + assert!(output.dtype.is_float()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/mod.rs new file mode 100644 index 0000000..c3bb6a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/mod.rs @@ -0,0 +1,8 @@ +pub use super::*; + +mod matmul; +mod quantize; + +// TODO: re-enable for cubecl backends when inputs are valid for packed U32 storage +#[cfg(feature = "ndarray")] +mod extended; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/quantize.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/quantize.rs new file mode 100644 index 0000000..71200a5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/ops/quantize.rs @@ -0,0 +1,209 @@ +use super::*; +use alloc::{vec, vec::Vec}; +use burn_tensor::quantization::{ + QParams, QTensorPrimitive, QuantLevel, QuantScheme, QuantStore, QuantValue, + QuantizationParameters, QuantizedBytes, +}; +use burn_tensor::{DType, Element, TensorData}; +use burn_tensor::{Tolerance, ops::QuantizedTensor}; + +fn get_q_params(data: TensorData) -> QParams> { + let num_elements = data.num_elements(); + let scheme = if let DType::QFloat(scheme) = data.dtype { + scheme + } else { + unreachable!() + }; + let q_bytes = QuantizedBytes { + bytes: data.into_bytes(), + scheme, + num_elements, + }; + q_bytes.into_vec_i8().1 +} + +#[test] +fn should_support_quantize_symmetric_int8() { + // Strict equality was based on full precision + if !matches!(FloatElem::dtype(), DType::F32) { + return; + } + let device = Default::default(); + let tensor = TestTensor::<1>::from_floats([-1.8, -1.0, 0.0, 0.5], &device); + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + let qparams = QuantizationParameters { + scales: TestTensor::from_floats([0.014_173_228], &device), + }; + + let x_q = tensor.clone().quantize(&scheme, qparams); + + let x_q_data = x_q.to_data(); + let expected = TensorData::quantized( + vec![-127i8, -71, 0, 35], + [4], + scheme.with_store(QuantStore::Native), + &[0.014_173_228], // scale + ); + + // Values equality + x_q_data.assert_eq(&expected, false); + + // Quantization parameters check + let qparams = get_q_params(x_q_data); + let expected = get_q_params(expected); + assert_eq!(qparams.scales.len(), 1); + // TODO: check scales + assert_eq!(qparams.scales, expected.scales); + + // Dequantize + let x = x_q.dequantize(); + + x.into_data() + .assert_approx_eq::(&tensor.into_data(), Tolerance::rel_abs(1e-1, 1e-2)); +} + +#[test] +fn should_support_quantize_dynamic_int8() { + let device = Default::default(); + // NOTE: we use fully representable values since different backend implementations could differ slightly + // due to rounding discrepancies + let tensor = TestTensor::<1>::from_floats([5., 0., 4., -12.7], &device); + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let x_q = tensor.quantize_dynamic(&scheme); + + let expected = TensorData::quantized( + vec![50i8, 0, 40, -127], + [4], + scheme.with_store(QuantStore::Native), + &[0.1], // scale + ); + + x_q.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_quantize_dequantize_symmetric_single_with_transform() { + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + let input = TestTensorInt::<1>::arange(0..32, &Default::default()).float(); + + let quant = input.quantize_dynamic(&scheme); + let result = quant * 10; + + let data = result.into_data(); + let expected = [ + 0.0, 9.76378, 19.52756, 29.29134, 39.05512, 48.818897, 61.02362, 70.7874, 80.551186, + 90.31496, 100.07874, 109.84252, 119.60631, 129.37009, 139.13387, 148.89764, 161.10237, + 170.86615, 180.62991, 190.39369, 200.15749, 209.92126, 219.68504, 229.44882, 239.21262, + 248.97638, 261.1811, 270.9449, 280.70865, 290.47244, 300.23624, 310.0, + ]; + data.assert_approx_eq::(&TensorData::from(expected), Tolerance::permissive()); +} + +#[test] +fn should_quantize_dequantize_symmetric_arange_16x16() { + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let input: TestTensor<2> = TestTensorInt::arange(0..256, &Default::default()) + .float() + .div_scalar(256.) + .reshape([16, 16]); + + let output = input.clone().quantize_dynamic(&scheme); + let output = output.dequantize(); + + output.into_data().assert_approx_eq::( + &input.into_data(), + Tolerance::absolute(1e-1).set_relative(1e-2), + ); +} + +#[test] +fn should_quantize_dequantize_symmetric_per_block_arange_16x16() { + let scheme = QuantizedTensor::::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([2, 16])); + + let input: TestTensor<2> = TestTensorInt::arange(0..256, &Default::default()) + .float() + .div_scalar(256.) + .reshape([16, 16]); + + let output = input.clone().quantize_dynamic(&scheme); + let output = output.dequantize(); + + output.into_data().assert_approx_eq::( + &input.into_data(), + Tolerance::absolute(1e-1).set_relative(1e-2), + ); +} + +fn should_quantize_transposed(tensor: Tensor, scheme: QuantScheme) { + let tensor_t = tensor.clone().transpose(); + + let output = tensor_t.quantize_dynamic(&scheme).dequantize().transpose(); + + tensor.into_data().assert_approx_eq::( + &output.into_data(), + Tolerance::absolute(1e-1).set_relative(1e-2), + ); +} + +#[test] +fn should_quantize_symmetric_int8_transposed_8x32() { + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let tensor = TestTensorInt::arange(0..256, &Default::default()) + .float() + .div_scalar(256.) + .reshape([8, 32]); + should_quantize_transposed(tensor, scheme); +} + +#[test] +fn should_quantize_symmetric_int8_transposed_48x64() { + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let tensor = TestTensorInt::arange(0..3072, &Default::default()) + .float() + .div_scalar(3072.) + .reshape([48, 64]); + should_quantize_transposed(tensor, scheme); +} + +#[test] +fn should_quantize_symmetric_per_block_int8_transposed_32x64() { + let scheme = QuantizedTensor::::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([32])); + + let tensor = TestTensorInt::arange(0..2048, &Default::default()) + .float() + .div_scalar(2048.) + .reshape([32, 64]); + should_quantize_transposed(tensor, scheme); +} + +#[test] +fn should_quantize_symmetric_int8_permuted_batch_dims() { + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let tensor = TestTensorInt::arange(0..2048, &Default::default()) + .float() + .div_scalar(2048.) + .reshape([2, 4, 8, 32]); + + // Permute [0,1,2,3] -> [1,2,0,3] + // This rearranges batch dims but keeps packed dim in place + let tensor_permuted = tensor.clone().permute([1, 2, 0, 3]); + + let output = tensor_permuted + .quantize_dynamic(&scheme) + .dequantize() + .permute([2, 0, 1, 3]); // reverse permutation + + tensor.into_data().assert_approx_eq::( + &output.into_data(), + Tolerance::absolute(1e-1).set_relative(1e-2), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/scheme.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/scheme.rs new file mode 100644 index 0000000..f479073 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/quantization/scheme.rs @@ -0,0 +1,71 @@ +use super::*; +use burn_tensor::Tolerance; +use burn_tensor::{ + Element, TensorData, + ops::QuantizedTensor, + quantization::{CalibrationRange, QTensorPrimitive, QuantLevel, QuantValue, compute_q_params}, +}; + +#[test] +fn per_tensor_symmetric_int8() { + let device = Default::default(); + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + let range = CalibrationRange { + min: TestTensor::<1>::from_floats([0.5], &device), + max: TestTensor::<1>::from_floats([1.8], &device), + }; + + let qparams = compute_q_params(&scheme, range); + + qparams + .scales + .into_data() + .assert_approx_eq::(&TensorData::from([0.014_173_23]), Tolerance::default()); +} + +#[test] +fn per_block_symmetric_int8() { + let device = Default::default(); + let scheme = QuantizedTensor::::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([4])); + let range = CalibrationRange { + min: TestTensor::<1>::from_floats([-1.8, -0.5, 0.01, -0.04], &device), + max: TestTensor::<1>::from_floats([0.5, 1.8, 0.04, -0.01], &device), + }; + + let qparams = compute_q_params(&scheme, range); + + qparams.scales.into_data().assert_approx_eq::( + &TensorData::from([0.014_173_23, 0.014_173_23, 0.000_314_96, 0.000_314_96]), + Tolerance::default(), + ); +} + +#[test] +fn quant_scheme_should_inhibit_by_default() { + let device = Default::default(); + let scheme = QuantizedTensor::::default_scheme().with_value(QuantValue::Q8S); + + let tensor_1 = TestTensor::<2>::from_floats( + [[1.0, 6.35, 0., 0.], [2.0, 3.0, 0., 0.], [1.0, 3.0, 0., 0.]], + &device, + ) + .quantize_dynamic(&scheme); + let _tensor_2 = TestTensor::<2>::from_floats( + [ + [4.0, 8.0, 12.7, 0.], + [2.0, 3.0, 6.0, 0.], + [0., 0., 0., 0.], + [0., 0., 0., 0.], + ], + &device, + ) + .quantize_dynamic(&scheme); + + // let tensor_3 = tensor_1.clone().matmul(tensor_2); + // assert_eq!(tensor_3.to_data().dtype, FloatElem::dtype()); + + let tensor_4 = tensor_1.add_scalar(1.); + assert_eq!(tensor_4.to_data().dtype, FloatElem::dtype()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/cov.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/cov.rs new file mode 100644 index 0000000..e0b4075 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/cov.rs @@ -0,0 +1,66 @@ +use super::*; +use burn_tensor::{TensorData, Tolerance}; + +#[test] +fn test_cov_1() { + let data = TensorData::from([[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.cov(1, 1); + let expected = + TensorData::from([[2.48917, -1.73333], [-1.73333, 15.33333]]).convert::(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_cov_4() { + let data = TensorData::from([[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.cov(1, 0); + let expected = TensorData::from([[1.86687, -1.30000], [-1.30000, 11.5]]).convert::(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_cov_2() { + let data = TensorData::from([[0.5, 1.8], [0.2, -2.0], [3.0, -4.0], [5.0, 0.0]]); + let tensor = TestTensor::<2>::from_data(data, &Default::default()); + + let output = tensor.cov(1, 1); + let expected = TensorData::from([ + [0.845, -1.43, -4.55, -3.25], + [-1.43, 2.42, 7.7, 5.5], + [-4.55, 7.7, 24.5, 17.5], + [-3.25, 5.5, 17.5, 12.5], + ]) + .convert::(); + + let tolerance = Tolerance::default().set_half_precision_relative(1e-3); + output + .into_data() + .assert_approx_eq::(&expected, tolerance); +} + +#[test] +fn test_cov_3() { + let data = TensorData::from([ + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + ]); + let device = Default::default(); + let tensor = TestTensor::<3>::from_data(data, &device); + let data_actual = tensor.cov(0, 1).into_data(); + let data_expected = TestTensor::<3>::zeros([4, 4, 4], &device).to_data(); + data_expected.assert_approx_eq::(&data_actual, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/display.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/display.rs new file mode 100644 index 0000000..31ea29e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/display.rs @@ -0,0 +1,338 @@ +use super::*; +use burn_tensor::backend::Backend; +use burn_tensor::{Element, Shape, TensorData}; + +type FloatElem = ::FloatElem; +type IntElem = ::IntElem; + +// Floating point values might not match for other precisions +fn skip_precision_not_f32() -> bool { + core::any::TypeId::of::() != core::any::TypeId::of::() +} + +#[test] +fn test_display_2d_int_tensor() { + let int_data = TensorData::from([[1, 2, 3], [4, 5, 6], [7, 8, 9]]); + let tensor_int = TestTensorInt::<2>::from_data(int_data, &Default::default()); + + let output = format!("{}", tensor_int); + let expected = format!( + r#"Tensor {{ + data: +[[1, 2, 3], + [4, 5, 6], + [7, 8, 9]], + shape: [3, 3], + device: {:?}, + backend: {:?}, + kind: "Int", + dtype: "{dtype}", +}}"#, + tensor_int.device(), + TestBackend::name(&tensor_int.device()), + dtype = core::any::type_name::(), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_2d_float_tensor() { + if skip_precision_not_f32() { + return; + } + + let float_data = TensorData::from([[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [7.7, 8.8, 9.9]]); + let tensor_float = TestTensor::<2>::from_data(float_data, &Default::default()); + + let output = format!("{}", tensor_float); + let expected = format!( + r#"Tensor {{ + data: +[[1.1, 2.2, 3.3], + [4.4, 5.5, 6.6], + [7.7, 8.8, 9.9]], + shape: [3, 3], + device: {:?}, + backend: {:?}, + kind: "Float", + dtype: "f32", +}}"#, + tensor_float.device(), + TestBackend::name(&tensor_float.device()), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_2d_bool_tensor() { + let bool_data = TensorData::from([ + [true, false, true], + [false, true, false], + [false, true, true], + ]); + let tensor_bool = TestTensorBool::<2>::from_data(bool_data, &Default::default()); + + let output = format!("{}", tensor_bool); + let expected = format!( + r#"Tensor {{ + data: +[[true, false, true], + [false, true, false], + [false, true, true]], + shape: [3, 3], + device: {:?}, + backend: {:?}, + kind: "Bool", + dtype: {:?}, +}}"#, + tensor_bool.device(), + TestBackend::name(&tensor_bool.device()), + ::BoolElem::dtype().name(), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_3d_tensor() { + let data = TensorData::from([ + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], + [[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]], + ]); + let tensor = TestTensorInt::<3>::from_data(data, &Default::default()); + + let output = format!("{}", tensor); + let expected = format!( + r#"Tensor {{ + data: +[[[1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12]], + [[13, 14, 15, 16], + [17, 18, 19, 20], + [21, 22, 23, 24]]], + shape: [2, 3, 4], + device: {:?}, + backend: {:?}, + kind: "Int", + dtype: "{dtype}", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + dtype = core::any::type_name::(), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_4d_tensor() { + let data = TensorData::from([ + [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]], + [[[13, 14, 15], [16, 17, 18]], [[19, 20, 21], [22, 23, 24]]], + ]); + + let tensor = TestTensorInt::<4>::from_data(data, &Default::default()); + + let output = format!("{}", tensor); + let expected = format!( + r#"Tensor {{ + data: +[[[[1, 2, 3], + [4, 5, 6]], + [[7, 8, 9], + [10, 11, 12]]], + [[[13, 14, 15], + [16, 17, 18]], + [[19, 20, 21], + [22, 23, 24]]]], + shape: [2, 2, 2, 3], + device: {:?}, + backend: {:?}, + kind: "Int", + dtype: "{dtype}", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + dtype = core::any::type_name::(), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_tensor_summarize_1() { + let tensor = TestTensor::<4>::zeros(Shape::new([2, 2, 2, 1000]), &Default::default()); + + let output = format!("{}", tensor); + let expected = format!( + r#"Tensor {{ + data: +[[[[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]]]], + shape: [2, 2, 2, 1000], + device: {:?}, + backend: {:?}, + kind: "Float", + dtype: "{dtype}", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + dtype = FloatElem::dtype().name(), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_tensor_summarize_2() { + let tensor = TestTensor::<4>::zeros(Shape::new([2, 2, 20, 100]), &Default::default()); + + let output = format!("{}", tensor); + let expected = format!( + r#"Tensor {{ + data: +[[[[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0]]]], + shape: [2, 2, 20, 100], + device: {:?}, + backend: {:?}, + kind: "Float", + dtype: "{dtype}", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + dtype = FloatElem::dtype().name(), + ); + assert_eq!(output, expected); +} + +#[test] +fn test_display_tensor_summarize_3() { + let tensor = TestTensor::<4>::zeros(Shape::new([2, 2, 200, 6]), &Default::default()); + + let output = format!("{}", tensor); + let expected = format!( + r#"Tensor {{ + data: +[[[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ... + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]]], + shape: [2, 2, 200, 6], + device: {:?}, + backend: {:?}, + kind: "Float", + dtype: "{dtype}", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + dtype = FloatElem::dtype().name(), + ); + assert_eq!(output, expected); +} +#[test] +fn test_display_precision() { + if skip_precision_not_f32() { + return; + } + + let tensor = TestTensor::<2>::full([1, 1], 0.123456789, &Default::default()); + + let output = format!("{}", tensor); + let expected = format!( + r#"Tensor {{ + data: +[[0.12345679]], + shape: [1, 1], + device: {:?}, + backend: {:?}, + kind: "Float", + dtype: "f32", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + ); + assert_eq!(output, expected); + + // CAN'T DO THIS BECAUSE OF GLOBAL STATE + // let print_options = PrintOptions { + // precision: Some(3), + // ..Default::default() + // }; + // set_print_options(print_options); + + let tensor = TestTensor::<2>::full([3, 2], 0.123456789, &Default::default()); + + // Set precision to 3 + let output = format!("{:.3}", tensor); + + let expected = format!( + r#"Tensor {{ + data: +[[0.123, 0.123], + [0.123, 0.123], + [0.123, 0.123]], + shape: [3, 2], + device: {:?}, + backend: {:?}, + kind: "Float", + dtype: "f32", +}}"#, + tensor.device(), + TestBackend::name(&tensor.device()), + ); + assert_eq!(output, expected); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/eye.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/eye.rs new file mode 100644 index 0000000..13ac86f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/eye.rs @@ -0,0 +1,17 @@ +use super::*; + +#[test] +fn test_eye_float() { + let device = Default::default(); + let tensor = TestTensor::<2>::from([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]); + let rhs = TestTensor::<2>::eye(3, &device); + assert_eq!(tensor.to_data(), rhs.to_data()); +} + +#[test] +fn test_eye_int() { + let device = Default::default(); + let tensor = TestTensorInt::<2>::from([[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + let rhs = TestTensorInt::<2>::eye(3, &device); + assert_eq!(tensor.to_data(), rhs.to_data()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/median.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/median.rs new file mode 100644 index 0000000..ad4b4ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/median.rs @@ -0,0 +1,92 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_median_even() { + let tensor = TestTensor::<2>::from_data( + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + &Default::default(), + ); + + let median_actual_1 = tensor.clone().median(1); + let median_expected_1 = TensorData::from([[0.2], [0.0]]).convert::(); + median_actual_1 + .into_data() + .assert_eq(&median_expected_1, false); + + let median_actual_0 = tensor.median(0); + let median_expected_0 = TensorData::from([[0.5, -4.0, 0.2, -2.0]]).convert::(); + median_actual_0 + .into_data() + .assert_eq(&median_expected_0, false); +} + +#[test] +fn test_median_odd() { + let tensor = TestTensor::<2>::from_data( + [ + [0.5, 1.8, 0.2, -2.0, 1.0], + [3.0, -4.0, 5.0, 0.0, -1.0], + [5.0, -5.0, 1.0, 3.0, -2.0], + ], + &Default::default(), + ); + + let median_actual_1 = tensor.clone().median(1); + let median_expected_1 = TensorData::from([[0.5], [0.0], [1.0]]).convert::(); + median_actual_1 + .into_data() + .assert_eq(&median_expected_1, false); + + let median_actual_0 = tensor.median(0); + let median_expected_0 = TensorData::from([[3.0, -4.0, 1.0, 0.0, -1.0]]).convert::(); + median_actual_0 + .into_data() + .assert_eq(&median_expected_0, false); +} + +#[test] +fn test_median_with_indices() { + let device = Default::default(); + let tensor = TestTensor::<1>::from_data([3.0, 1.0, 2.0], &device); + // median = 2, original index = 2 + let (values, indices) = tensor.median_with_indices(0); + values + .into_data() + .assert_eq(&TensorData::from([2.0]), false); + indices + .into_data() + .assert_eq(&TensorData::from([2i64]), false); + + let tensor = TestTensor::<2>::from_data([[5.0, 1.0, 3.0], [2.0, 8.0, 4.0]], &device); + // Along dim 1: + // Row 0: median = 3, original index = 2 + // Row 1: median = 4, original index = 2 + let (values, indices) = tensor.median_with_indices(1); + values + .into_data() + .assert_eq(&TensorData::from([[3.0], [4.0]]), false); + indices + .into_data() + .assert_eq(&TensorData::from([[2i64], [2i64]]), false); +} + +#[test] +fn test_median_all_elements() { + let tensor = TestTensor::<2>::from_data( + [ + [0.5, 1.8, 0.2, -2.0, 1.0], + [3.0, -4.0, 5.0, 0.0, -1.0], + [5.0, -5.0, 1.0, 3.0, -2.0], + ], + &Default::default(), + ); + + // Sorted: [-5, -4, -2, -2, -1, 0, 0.2, 0.5, 1, 1, 1.8, 3, 3, 5, 5] + let dims = tensor.dims().len(); + let flattened_tensor: Tensor<_, 1> = tensor.flatten(0, dims - 1); + let result = flattened_tensor.median(0); + result + .into_data() + .assert_eq(&TensorData::from([0.5]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/mod.rs new file mode 100644 index 0000000..dd5ee4c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/mod.rs @@ -0,0 +1,7 @@ +pub use super::*; // re-export test types + +mod cov; +mod display; +mod eye; +mod median; +mod var; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/var.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/var.rs new file mode 100644 index 0000000..a0a901a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/float/stats/var.rs @@ -0,0 +1,69 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_var() { + let tensor = TestTensor::<2>::from_data( + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + &Default::default(), + ); + + let output = tensor.var(1); + let expected = TensorData::from([[2.4892], [15.3333]]).convert::(); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_var_mean() { + let tensor = TestTensor::<2>::from_data( + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + &Default::default(), + ); + + let (var, mean) = tensor.var_mean(1); + + let var_expected = TensorData::from([[2.4892], [15.3333]]).convert::(); + let mean_expected = TensorData::from([[0.125], [1.]]).convert::(); + + var.into_data() + .assert_approx_eq::(&var_expected, Tolerance::default()); + mean.into_data() + .assert_approx_eq::(&mean_expected, Tolerance::default()); +} + +#[test] +fn test_var_bias() { + let tensor = TestTensor::<2>::from_data( + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + &Default::default(), + ); + + let output = tensor.var_bias(1); + let expected = TensorData::from([[1.86688], [11.5]]).convert::(); + + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_var_mean_bias() { + let tensor = TestTensor::<2>::from_data( + [[0.5, 1.8, 0.2, -2.0], [3.0, -4.0, 5.0, 0.0]], + &Default::default(), + ); + + let (var, mean) = tensor.var_mean_bias(1); + + let var_expected = TensorData::from([[1.86688], [11.5]]).convert::(); + let mean_expected = TensorData::from([[0.125], [1.]]).convert::(); + + var.into_data() + .assert_approx_eq::(&var_expected, Tolerance::default()); + mean.into_data() + .assert_approx_eq::(&mean_expected, Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/mod.rs new file mode 100644 index 0000000..a071d89 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/mod.rs @@ -0,0 +1,4 @@ +pub use super::*; // re-export test types + +mod ops; +mod primitive; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/abs.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/abs.rs new file mode 100644 index 0000000..56b8138 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/abs.rs @@ -0,0 +1,24 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_abs_ops_int() { + let tensor = TestTensorInt::<2>::from([[0, -1, 2], [3, 4, -5]]); + + let output = tensor.abs(); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 2], [3, 4, 5]]), false); +} + +#[test] +fn should_support_abs_ops_int_signed_min() { + let tensor = TestTensorInt::<2>::from([[IntElem::MIN]]); + + let output = tensor.abs(); + + output + .into_data() + .assert_eq(&TensorData::from([[IntElem::MIN]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/add.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/add.rs new file mode 100644 index 0000000..288be96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/add.rs @@ -0,0 +1,66 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_add_d2_int() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::from([[6, 7, 8], [9, 10, 11]]); + + let output = tensor_1 + tensor_2; + + output + .into_data() + .assert_eq(&TensorData::from([[6, 8, 10], [12, 14, 16]]), false); +} + +#[test] +fn test_add_broadcast_int() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2]]); + let tensor_2 = TestTensorInt::from([[3, 4, 5], [6, 7, 8]]); + + let output = tensor_1 + tensor_2; + + output + .into_data() + .assert_eq(&TensorData::from([[3, 5, 7], [6, 8, 10]]), false); +} + +#[test] +fn should_support_add_scalar_ops_int() { + let scalar = 2; + let tensor = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let output = tensor + scalar; + + output + .into_data() + .assert_eq(&TensorData::from([[2, 3, 4], [5, 6, 7]]), false); +} + +#[test] +fn scalar_add_not_contiguous() { + let tensor = TestTensorInt::<1>::arange(0..32, &Default::default()).float(); + let tensor = tensor.reshape([1, 4, 4, 2]).permute([0, 3, 1, 2]); + + let tensor = tensor.slice([0..1, 0..2, 0..4, 0..4]); + let before = tensor.clone(); + + let after = tensor.add_scalar(0.0); + + before + .into_data() + .assert_approx_eq::(&after.into_data(), Default::default()); +} + +#[test] +fn scalar_add_not_contiguous_int() { + let tensor = TestTensorInt::<1>::arange(0..32, &Default::default()); + let tensor = tensor.reshape([1, 4, 4, 2]).permute([0, 3, 1, 2]); + + let tensor = tensor.slice([0..1, 0..2, 0..4, 0..4]); + let before = tensor.clone(); + + let after = tensor.add_scalar(0); + + before.into_data().assert_eq(&after.into_data(), true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/aggregation.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/aggregation.rs new file mode 100644 index 0000000..0d1d37e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/aggregation.rs @@ -0,0 +1,81 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_should_mean_int() { + let tensor = TestTensorInt::<2>::from([[2, 2, 2], [3, 4, 5]]); + + let output = tensor.mean(); + + output.into_data().assert_eq(&TensorData::from([3]), false); +} + +#[test] +fn test_should_mean_last_dim_int() { + let tensor = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let output = tensor.mean_dim(1); + + output + .into_data() + .assert_eq(&TensorData::from([[1], [4]]), false); +} + +#[test] +fn test_should_sum_last_dim_int() { + let tensor = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let output = tensor.sum_dim(1); + + output + .into_data() + .assert_eq(&TensorData::from([[3], [12]]), false); +} + +#[test] +fn test_should_sum_int() { + let tensor = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let output = tensor.sum(); + + output.into_data().assert_eq(&TensorData::from([15]), false); +} + +#[test] +#[ignore = "Not implemented for all backends yet"] +fn test_prod_int() { + let tensor = TestTensorInt::<2>::from([[2, 1, 2], [3, 4, 5]]); + let output = tensor.prod(); + + output + .into_data() + .assert_eq(&TensorData::from([240]), false); + + let tensor_with_zero = TestTensorInt::<2>::from([[2, 0, 2], [3, 4, 5]]); + let output = tensor_with_zero.prod(); + + output.into_data().assert_eq(&TensorData::from([0]), false); +} + +#[test] +#[ignore = "Not implemented for all backends yet"] +fn test_prod_dim_int() { + let tensor = TestTensorInt::<2>::from([[2, 1, 2], [3, 4, 5]]); + let output = tensor.prod_dim(1); + output + .into_data() + .assert_eq(&TensorData::from([[4], [60]]), false); + + let tensor_with_zero = TestTensorInt::<2>::from([[2, 0, 2], [3, 4, 5]]); + let output = tensor_with_zero.prod_dim(1); + output + .into_data() + .assert_eq(&TensorData::from([[0], [60]]), false); + + // Negative Indexing. + let tensor_with_zero = TestTensorInt::<2>::from([[2, 0, 2], [3, 4, 5]]); + let output = tensor_with_zero.prod_dim(-1); + output + .into_data() + .assert_eq(&TensorData::from([[0], [60]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/all.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/all.rs new file mode 100644 index 0000000..490874c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/all.rs @@ -0,0 +1,23 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_all() { + let tensor = TestTensorInt::<2>::from([[0, 1, 0], [1, -1, 1]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensorInt::<2>::from([[1, 1, 1], [1, 1, 1]]); + let data_actual = tensor.all().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_all_dim() { + let tensor = TestTensorInt::<2>::from([[0, 1, 0], [1, -1, 1]]); + let data_actual = tensor.all_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/any.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/any.rs new file mode 100644 index 0000000..967129f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/any.rs @@ -0,0 +1,23 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_any() { + let tensor = TestTensorInt::<2>::from([[0, 0, 0], [1, -1, 0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([true]); + data_expected.assert_eq(&data_actual, false); + + let tensor = TestTensorInt::<2>::from([[0, 0, 0], [0, 0, 0]]); + let data_actual = tensor.any().into_data(); + let data_expected = TensorData::from([false]); + data_expected.assert_eq(&data_actual, false); +} + +#[test] +fn test_any_dim() { + let tensor = TestTensorInt::<2>::from([[0, 0, 0], [1, -1, 0]]); + let data_actual = tensor.any_dim(1).into_data(); + let data_expected = TensorData::from([[false], [true]]); + data_expected.assert_eq(&data_actual, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange.rs new file mode 100644 index 0000000..e521b2a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange.rs @@ -0,0 +1,32 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::backend::Backend; + +#[test] +fn test_arange() { + let device = ::Device::default(); + + let tensor = TestTensorInt::<1>::arange(2..5, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([2, 3, 4]), false); + + // Test arange with negative numbers + let tensor = TestTensorInt::<1>::arange(-10..-5, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([-10, -9, -8, -7, -6]), false); + + let tensor = TestTensorInt::<1>::arange(-3..0, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([-3, -2, -1]), false); + + // Test arange with a mix of positive and negative numbers + let tensor = TestTensorInt::<1>::arange(-2..3, &device); + tensor + .clone() + .into_data() + .assert_eq(&TensorData::from([-2, -1, 0, 1, 2]), false); + assert_eq!(tensor.device(), device); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange_step.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange_step.rs new file mode 100644 index 0000000..cdfe70f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arange_step.rs @@ -0,0 +1,45 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::backend::Backend; + +#[test] +fn test_arange_step() { + let device = ::Device::default(); + + // Test correct sequence of numbers when the range is 0..9 and the step is 1 + let tensor = TestTensorInt::<1>::arange_step(0..9, 1, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([0, 1, 2, 3, 4, 5, 6, 7, 8]), false); + + // Test correct sequence of numbers when the range is 0..3 and the step is 2 + let tensor = TestTensorInt::<1>::arange_step(0..3, 2, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([0, 2]), false); + + // Test correct sequence of numbers when the range is 0..2 and the step is 5 + let tensor = TestTensorInt::<1>::arange_step(0..2, 5, &device); + tensor.into_data().assert_eq(&TensorData::from([0]), false); + + // Test correct sequence of numbers when the range includes negative numbers + let tensor = TestTensorInt::<1>::arange_step(-3..3, 2, &device); + tensor + .into_data() + .assert_eq(&TensorData::from([-3, -1, 1]), false); + + let tensor = TestTensorInt::<1>::arange_step(-5..1, 5, &device); + tensor + .clone() + .into_data() + .assert_eq(&TensorData::from([-5, 0]), false); + assert_eq!(tensor.device(), device); +} + +#[test] +#[should_panic] +fn should_panic_when_step_is_zero() { + let device = ::Device::default(); + // Test that arange_step panics when the step is 0 + let _tensor = TestTensorInt::<1>::arange_step(0..3, 0, &device); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arg.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arg.rs new file mode 100644 index 0000000..9ba36af --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/arg.rs @@ -0,0 +1,24 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_argmax_2d_dim0_int() { + let tensor = TestTensorInt::<2>::from([[10, 11, 2], [3, 4, 5]]); + + let output = tensor.argmax(0); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 0, 1]]), false); +} + +#[test] +fn test_argmin_2d_dim0_int() { + let tensor = TestTensorInt::<2>::from([[10, 11, 2], [30, 4, 5]]); + + let output = tensor.argmin(0); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 0]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/bitwise.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/bitwise.rs new file mode 100644 index 0000000..cfdfc15 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/bitwise.rs @@ -0,0 +1,173 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_apply_bitwise_and_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let tensor_2 = TestTensorInt::from([[6, 7, 8], [9, 10, 15]]); + + let output = tensor_1.bitwise_and(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([[2, 4, 0], [9, 2, 8]]), false); +} + +#[test] +fn should_apply_bitwise_and_1d() { + let tensor_1 = TestTensorInt::<1>::from([13, 7]); + let tensor_2 = TestTensorInt::from([11, 3]); + + let output = tensor_1.bitwise_and(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([9, 3]), false); +} + +#[test] +fn should_apply_bitwise_and_scalar_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let scalar = 5; + + let output = tensor_1.bitwise_and_scalar(scalar); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 4, 5], [1, 1, 0]]), false); +} + +#[test] +fn should_apply_bitwise_not_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + + let output = tensor_1.bitwise_not(); + + output + .into_data() + .assert_eq(&TensorData::from([[-4, -5, -6], [-10, -4, -9]]), false); +} + +#[test] +fn should_apply_bitwise_or_scalar_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let scalar = 5; + + let output = tensor_1.bitwise_or_scalar(scalar); + + output + .into_data() + .assert_eq(&TensorData::from([[7, 5, 5], [13, 7, 13]]), false); +} + +#[test] +fn should_apply_bitwise_or_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let tensor_2 = TestTensorInt::from([[6, 7, 8], [9, 10, 15]]); + + let output = tensor_1.bitwise_or(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([[7, 7, 13], [9, 11, 15]]), false); +} + +#[test] +fn should_apply_bitwise_or_1d() { + let tensor_1 = TestTensorInt::<1>::from([13, 7]); + let tensor_2 = TestTensorInt::from([11, 3]); + + let output = tensor_1.bitwise_or(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([15, 7]), false); +} + +#[test] +fn should_apply_bitwise_xor_scalar_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let scalar = 5; + + let output = tensor_1.bitwise_xor_scalar(scalar); + + output + .into_data() + .assert_eq(&TensorData::from([[6, 1, 0], [12, 6, 13]]), false); +} + +#[test] +fn should_apply_bitwise_xor_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let tensor_2 = TestTensorInt::from([[6, 7, 8], [9, 10, 15]]); + + let output = tensor_1.bitwise_xor(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([[5, 3, 13], [0, 9, 7]]), false); +} + +#[test] +fn should_apply_bitwise_xor_1d() { + let tensor_1 = TestTensorInt::<1>::from([13, 7]); + let tensor_2 = TestTensorInt::from([11, 3]); + + let output = tensor_1.bitwise_xor(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([6, 4]), false); +} + +#[test] +fn should_apply_bitwise_left_shift_2d() { + if (IntElem::MAX as u32) < 512 { + return; + } + + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let tensor_2 = TestTensorInt::from([[1, 2, 3], [4, 5, 6]]); + + let output = tensor_1.bitwise_left_shift(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([[6, 16, 40], [144, 96, 512]]), false); +} + +#[test] +fn should_apply_bitwise_left_shift_scalar_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let scalar = 2; + + let output = tensor_1.bitwise_left_shift_scalar(scalar); + + output + .into_data() + .assert_eq(&TensorData::from([[12, 16, 20], [36, 12, 32]]), false); +} + +#[test] +fn should_apply_bitwise_right_shift_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let tensor_2 = TestTensorInt::from([[1, 2, 3], [4, 5, 6]]); + + let output = tensor_1.bitwise_right_shift(tensor_2); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 1, 0], [0, 0, 0]]), false); +} + +#[test] +fn should_apply_bitwise_right_shift_scalar_2d() { + let tensor_1 = TestTensorInt::<2>::from([[3, 4, 5], [9, 3, 8]]); + let scalar = 2; + + let output = tensor_1.bitwise_right_shift_scalar(scalar); + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 1], [2, 0, 2]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cartesian_grid.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cartesian_grid.rs new file mode 100644 index 0000000..fac950e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cartesian_grid.rs @@ -0,0 +1,21 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::backend::Backend; + +#[test] +fn test_cartesian_grid() { + let device = ::Device::default(); + + // Test a single element tensor + let tensor: TestTensorInt<2> = TestTensorInt::<1>::cartesian_grid([1], &device); + tensor + .into_data() + .assert_eq(&TensorData::from([[0]]), false); + + // Test for a 2x2 tensor + let tensor: TestTensorInt<3> = TestTensorInt::<2>::cartesian_grid([2, 2], &device); + tensor.into_data().assert_eq( + &TensorData::from([[[0, 0], [0, 1]], [[1, 0], [1, 1]]]), + false, + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cast.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cast.rs new file mode 100644 index 0000000..42e3c7e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cast.rs @@ -0,0 +1,30 @@ +use super::*; +use burn_tensor::{DType, TensorData}; + +#[test] +fn cast_int_to_bool() { + let tensor1 = TestTensorInt::<2>::from([[0, 43, 0], [2, -4, 31]]); + let data_actual = tensor1.bool().into_data(); + let data_expected = TensorData::from([[false, true, false], [true, true, true]]); + data_actual.assert_eq(&data_expected, false); +} + +#[test] +fn cast_bool_to_int_tensor() { + let tensor = TestTensorBool::<2>::from([[true, false, true], [false, false, true]]).int(); + + let expected = TensorData::from([[1, 0, 1], [0, 0, 1]]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn cast_int_precision() { + let data = TensorData::from([[1, 2, 3], [4, 5, 6]]); + let tensor = TestTensorInt::<2>::from(data.clone()); + + let output = tensor.cast(DType::I32); + + assert_eq!(output.dtype(), DType::I32); + output.into_data().assert_eq(&data, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cat.rs new file mode 100644 index 0000000..640ed06 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cat.rs @@ -0,0 +1,28 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_cat_ops_int() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data([[1, 2, 3]], &device); + let tensor_2 = TestTensorInt::<2>::from_data([[4, 5, 6]], &device); + + let output = Tensor::cat(vec![tensor_1, tensor_2], 0); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 3], [4, 5, 6]]), false); +} + +#[test] +fn should_support_cat_with_empty_tensor_int() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data([[1, 2, 3]], &device); + let tensor_2: TestTensorInt<2> = TestTensorInt::empty([1, 0], &device); + + let output = Tensor::cat(vec![tensor_1, tensor_2], 1); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 3]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/chunk.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/chunk.rs new file mode 100644 index 0000000..e040fad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/chunk.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_chunk_multi_dimension() { + let tensors = + TestTensorInt::<2>::from_data(TensorData::from([[0, 1, 2, 3]]), &Default::default()) + .chunk(2, 1); + assert_eq!(tensors.len(), 2); + + let expected = [TensorData::from([[0, 1]]), TensorData::from([[2, 3]])]; + + for (index, tensor) in tensors.iter().enumerate() { + tensor.to_data().assert_eq(&expected[index], false); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/comparison.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/comparison.rs new file mode 100644 index 0000000..41aca86 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/comparison.rs @@ -0,0 +1,267 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_equal() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::<2>::from([[1, 1, 1], [4, 3, 5]]); + + let data_actual_cloned = tensor_1.clone().equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.equal(tensor_2); + + let data_expected = TensorData::from([[false, true, false], [false, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_not_equal() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::<2>::from([[1, 1, 1], [4, 3, 5]]); + + let data_actual_cloned = tensor_1.clone().not_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.not_equal(tensor_2); + + let data_expected = TensorData::from([[true, false, true], [true, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_equal_elem() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 2, 5]]); + + let data_actual_cloned = tensor_1.clone().equal_elem(2); + let data_actual_inplace = tensor_1.equal_elem(2); + + let data_expected = TensorData::from([[false, false, true], [false, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_not_equal_elem() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 2, 5]]); + + let data_actual_cloned = tensor_1.clone().not_equal_elem(2); + let data_actual_inplace = tensor_1.not_equal_elem(2); + + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn greater_elem() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let data_actual_cloned = tensor_1.clone().greater_elem(4); + let data_actual_inplace = tensor_1.greater_elem(4); + + let data_expected = TensorData::from([[false, false, false], [false, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater_equal_elem() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let data_actual_cloned = tensor_1.clone().greater_equal_elem(4); + let data_actual_inplace = tensor_1.greater_equal_elem(4); + + let data_expected = TensorData::from([[false, false, false], [false, true, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::<2>::from([[1, 1, 1], [4, 3, 50]]); + + let data_actual_cloned = tensor_1.clone().greater(tensor_2.clone()); + let data_actual_inplace = tensor_1.greater(tensor_2); + + let data_expected = TensorData::from([[false, false, true], [false, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater_equal() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::<2>::from([[1, 1, 1], [4, 3, 50]]); + + let data_actual_cloned = tensor_1.clone().greater_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.greater_equal(tensor_2); + + let data_expected = TensorData::from([[false, true, true], [false, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower_elem() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let data_actual_cloned = tensor_1.clone().lower_elem(4); + let data_actual_inplace = tensor_1.lower_elem(4); + + let data_expected = TensorData::from([[true, true, true], [true, false, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower_equal_elem() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + let data_actual_cloned = tensor_1.clone().lower_equal_elem(4); + let data_actual_inplace = tensor_1.lower_equal_elem(4); + + let data_expected = TensorData::from([[true, true, true], [true, true, false]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::<2>::from([[1, 1, 1], [4, 3, 50]]); + + let data_actual_cloned = tensor_1.clone().lower(tensor_2.clone()); + let data_actual_inplace = tensor_1.lower(tensor_2); + + let data_expected = TensorData::from([[true, false, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_lower_equal() { + let tensor_1 = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + let tensor_2 = TestTensorInt::<2>::from([[1, 1, 1], [4, 3, 50]]); + + let data_actual_cloned = tensor_1.clone().lower_equal(tensor_2.clone()); + let data_actual_inplace = tensor_1.lower_equal(tensor_2); + + let data_expected = TensorData::from([[true, true, false], [true, false, true]]); + data_expected.assert_eq(&data_actual_cloned.into_data(), false); + data_expected.assert_eq(&data_actual_inplace.into_data(), false); +} + +#[test] +fn test_greater_broadcast() { + // Test broadcasting with shape [1, 4] vs [4, 4] + let device = Default::default(); + let data_1 = TensorData::from([[1, 2, 3, 4]]); + let data_2 = TensorData::from([ + [0.5, 1.5, 2.5, 3.5], + [1.5, 2.5, 3.5, 4.5], + [2.5, 3.5, 4.5, 5.5], + [3.5, 4.5, 5.5, 6.5], + ]); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let result = tensor_1.greater(tensor_2); + + let expected = TensorData::from([ + [true, true, true, true], + [false, false, false, false], + [false, false, false, false], + [false, false, false, false], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_greater_equal_broadcast() { + // Test broadcasting with shape [4, 1] vs [1, 4] + let device = Default::default(); + let data_1 = TensorData::from([[1], [2], [3], [4]]); + let data_2 = TensorData::from([[1, 2, 3, 4]]); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let result = tensor_1.greater_equal(tensor_2); + + let expected = TensorData::from([ + [true, false, false, false], + [true, true, false, false], + [true, true, true, false], + [true, true, true, true], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_equal_broadcast() { + // Test broadcasting with different ranks + let device = Default::default(); + let data_1 = TensorData::from([[2], [3], [4]]); + let data_2 = TensorData::from([[2, 3, 4, 2]]); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let result = tensor_1.equal(tensor_2); + + let expected = TensorData::from([ + [true, false, false, true], + [false, true, false, false], + [false, false, true, false], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_not_equal_broadcast() { + // Test broadcasting with shape [3, 1] vs [1, 3] + let device = Default::default(); + let data_1 = TensorData::from([[1], [2], [3]]); + let data_2 = TensorData::from([[1, 2, 3]]); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let result = tensor_1.not_equal(tensor_2); + + let expected = TensorData::from([ + [false, true, true], + [true, false, true], + [true, true, false], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_int_greater_broadcast() { + let device = Default::default(); + let data_1 = TensorData::from([[1i32, 2, 3]]); + let data_2 = TensorData::from([[0i32], [2], [4]]); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let result = tensor_1.greater(tensor_2); + + let expected = TensorData::from([ + [true, true, true], + [false, false, true], + [false, false, false], + ]); + expected.assert_eq(&result.into_data(), false); +} + +#[test] +fn test_int_lower_equal_broadcast() { + let device = Default::default(); + let data_1 = TensorData::from([[2i32], [4]]); + let data_2 = TensorData::from([[1i32, 2, 3]]); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let result = tensor_1.lower_equal(tensor_2); + + let expected = TensorData::from([[false, true, true], [false, false, false]]); + expected.assert_eq(&result.into_data(), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/create_like.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/create_like.rs new file mode 100644 index 0000000..c674971 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/create_like.rs @@ -0,0 +1,22 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_zeros_like() { + let tensor = TestTensorInt::<3>::from([[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]]); + + let tensor = tensor.zeros_like(); + let expected = TensorData::from([[[0, 0, 0], [0, 0, 0]], [[0, 0, 0], [0, 0, 0]]]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_ones_like() { + let tensor = TestTensorInt::<3>::from([[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]]); + + let tensor = tensor.ones_like(); + let expected = TensorData::from([[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1]]]); + + tensor.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cumulative.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cumulative.rs new file mode 100644 index 0000000..6af39a9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/cumulative.rs @@ -0,0 +1,90 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_cumsum_int_dim_0() { + let tensor = TestTensorInt::<2>::from([[1, 2, 3], [4, 5, 6]]); + + let output = tensor.cumsum(0); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 3], [5, 7, 9]]), false); +} + +#[test] +fn test_cumsum_int_dim_1() { + let tensor = TestTensorInt::<2>::from([[1, 2, 3], [4, 5, 6]]); + + let output = tensor.cumsum(1); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 3, 6], [4, 9, 15]]), false); +} + +#[test] +fn test_cumprod_int_dim_0() { + let tensor = TestTensorInt::<2>::from([[1, 2, 3], [4, 5, 6]]); + + let output = tensor.cumprod(0); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 3], [4, 10, 18]]), false); +} + +#[test] +fn test_cumprod_int_dim_1() { + let tensor = TestTensorInt::<2>::from([[1, 2, 3], [4, 5, 6]]); + + let output = tensor.cumprod(1); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 6], [4, 20, 120]]), false); +} + +#[test] +fn test_cummin_int_dim_0() { + let tensor = TestTensorInt::<2>::from([[3, 1, 4], [2, 5, 1]]); + + let output = tensor.cummin(0); + + output + .into_data() + .assert_eq(&TensorData::from([[3, 1, 4], [2, 1, 1]]), false); +} + +#[test] +fn test_cummin_int_dim_1() { + let tensor = TestTensorInt::<2>::from([[3, 1, 4], [2, 5, 1]]); + + let output = tensor.cummin(1); + + output + .into_data() + .assert_eq(&TensorData::from([[3, 1, 1], [2, 2, 1]]), false); +} + +#[test] +fn test_cummax_int_dim_0() { + let tensor = TestTensorInt::<2>::from([[3, 1, 4], [1, 5, 2]]); + + let output = tensor.cummax(0); + + output + .into_data() + .assert_eq(&TensorData::from([[3, 1, 4], [3, 5, 4]]), false); +} + +#[test] +fn test_cummax_int_dim_1() { + let tensor = TestTensorInt::<2>::from([[3, 1, 4], [1, 5, 2]]); + + let output = tensor.cummax(1); + + output + .into_data() + .assert_eq(&TensorData::from([[3, 3, 4], [1, 5, 5]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/div.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/div.rs new file mode 100644 index 0000000..167b3a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/div.rs @@ -0,0 +1,45 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_div_ops_int() { + let data_1 = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let data_2 = TensorData::from([[1, 1, 2], [1, 1, 2]]); + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let output = tensor_1 / tensor_2; + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 1], [3, 4, 2]]), false); +} + +#[test] +fn test_div_broadcast_int() { + let data_1 = TensorData::from([[0, 1, 2]]); + let data_2 = TensorData::from([[1, 1, 2], [3, 4, 5]]); + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let output = tensor_1 / tensor_2; + + output + .into_data() + .assert_eq(&TensorData::from([[0, 1, 1], [0, 0, 0]]), false); +} + +#[test] +fn should_support_div_scalar_ops_int() { + let data = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let scalar = 2; + let tensor = TestTensorInt::<2>::from_data(data, &Default::default()); + + let output = tensor / scalar; + + output + .into_data() + .assert_eq(&TensorData::from([[0, 0, 1], [1, 2, 2]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/expand.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/expand.rs new file mode 100644 index 0000000..a311ec5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/expand.rs @@ -0,0 +1,41 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn expand_2d_int() { + let tensor = TestTensorInt::<1>::from([1, 2, 3]); + let output = tensor.expand([3, 3]); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 3], [1, 2, 3], [1, 2, 3]]), false); +} + +#[test] +fn should_all_negative_one() { + let tensor = TestTensorInt::<1>::from([1, 2, 3]); + let output = tensor.expand([2, -1]); + + output + .into_data() + .assert_eq(&TensorData::from([[1, 2, 3], [1, 2, 3]]), false); +} + +#[test] +#[should_panic] +fn should_panic_negative_one_on_non_existing_dim() { + let tensor = TestTensorInt::<1>::from([1, 2, 3]); + let _expanded_tensor = tensor.expand([-1, 3]); +} + +/// Regression test for https://github.com/tracel-ai/burn/issues/2091 +#[test] +fn inplace_op_after_expand() { + let tensor = TestTensorInt::<1>::from([1, 2, 3]); + let mut output = tensor.expand([2, 3]); + output = output + 1; + + output + .into_data() + .assert_eq(&TensorData::from([[2, 3, 4], [2, 3, 4]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/flip.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/flip.rs new file mode 100644 index 0000000..c302322 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/flip.rs @@ -0,0 +1,22 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn flip_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + let flipped = tensor.clone().flip([0, 2]); + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).flip((0, 2)) + let expected = TensorData::from([ + [[15, 14, 13, 12], [19, 18, 17, 16], [23, 22, 21, 20]], + [[3, 2, 1, 0], [7, 6, 5, 4], [11, 10, 9, 8]], + ]); + + flipped.into_data().assert_eq(&expected, false); + + // Test with no flip + let flipped = tensor.clone().flip([]); + assert_eq!(tensor.into_data(), flipped.into_data()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/full.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/full.rs new file mode 100644 index 0000000..4541795 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/full.rs @@ -0,0 +1,11 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_tensor_full() { + let device = Default::default(); + let int_tensor = TestTensorInt::<2>::full([2, 2], 2, &device); + int_tensor + .into_data() + .assert_eq(&TensorData::from([[2, 2], [2, 2]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/gather_scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/gather_scatter.rs new file mode 100644 index 0000000..ca8d281 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/gather_scatter.rs @@ -0,0 +1,69 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, TensorData}; + +#[test] +fn should_gather_1d_dim0_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_ints([5, 6, 7], &device); + let indices = TestTensorInt::from_ints([1, 1, 0, 1, 2], &device); + + let output = tensor.gather(0, indices); + + output + .into_data() + .assert_eq(&TensorData::from([6, 6, 5, 6, 7]), false); +} + +#[test] +fn should_gather_indices_broadcasted() { + let device = Default::default(); + + let batch_size = 3; + let fft_size = 4; + let shape = [batch_size, fft_size, 2]; + let x = TestTensorInt::arange( + 0..shape.iter().product::() as i64, + &Default::default(), + ) + .reshape(shape); + let idx = TestTensorInt::<1>::from_ints([0, 2, 1, 3], &device); + + let expected = TestTensorInt::<3>::from([ + [[0, 1], [4, 5], [2, 3], [6, 7]], + [[8, 9], [12, 13], [10, 11], [14, 15]], + [[16, 17], [20, 21], [18, 19], [22, 23]], + ]) + .into_data(); + + // Case 1: gather dim 2 + let perm = idx + .clone() + .reshape([1, 1, fft_size]) + .repeat_dim(0, batch_size) + .repeat_dim(1, 2); + + let input = x.clone().permute([0, 2, 1]); + let out = input.gather(2, perm).permute([0, 2, 1]); + + out.into_data().assert_eq(&expected, true); + + // Case 2: gather directly on dim 1 + let perm = idx.reshape([1, fft_size, 1]).repeat_dim(0, batch_size); + let out2 = x.gather(1, perm.repeat_dim(2, 2)); + + out2.into_data().assert_eq(&expected, true); +} + +#[test] +fn should_scatter_add_1d_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_ints([0, 0, 0], &device); + let values = TestTensorInt::from_ints([5, 4, 3], &device); + let indices = TestTensorInt::from_ints([1, 0, 2], &device); + + let output = tensor.scatter(0, indices, values, IndexingUpdateOp::Add); + + output + .into_data() + .assert_eq(&TensorData::from([4, 5, 3]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/init.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/init.rs new file mode 100644 index 0000000..b95fa38 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/init.rs @@ -0,0 +1,31 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_int_empty() { + let shape = [2, 2]; + let tensor = TestTensorInt::<2>::empty(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()) +} + +#[test] +fn should_support_int_zeros() { + let shape = [2, 2]; + let tensor = TestTensorInt::<2>::zeros(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()); + + tensor + .into_data() + .assert_eq(&TensorData::from([[0, 0], [0, 0]]), false); +} + +#[test] +fn should_support_int_ones() { + let shape = [2, 2]; + let tensor = TestTensorInt::<2>::ones(shape, &Default::default()); + assert_eq!(tensor.shape(), shape.into()); + + tensor + .into_data() + .assert_eq(&TensorData::from([[1, 1], [1, 1]]), false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mask.rs new file mode 100644 index 0000000..2f734e8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mask.rs @@ -0,0 +1,56 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_mask_where_broadcast_int() { + let device = Default::default(); + // When broadcasted, the input [[2, 3], [4, 5]] is repeated 4 times + let tensor = TestTensorInt::<1>::arange(2..6, &device).reshape([1, 2, 2]); + let mask = TestTensorBool::<3>::from_bool( + TensorData::from([ + [[true, false], [false, true]], + [[false, true], [true, false]], + [[false, false], [false, false]], + [[true, true], [true, true]], + ]), + &device, + ); + let value = TestTensorInt::<3>::ones([4, 2, 2], &device); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([ + [[1, 3], [4, 1]], + [[2, 1], [1, 5]], + [[2, 3], [4, 5]], + [[1, 1], [1, 1]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_int_mask_where_ops() { + let device = Default::default(); + let tensor = TestTensorInt::<2>::from_data([[1, 7], [2, 3]], &device); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + let value = TestTensorInt::<2>::from_data(TensorData::from([[8, 9], [10, 11]]), &device); + + let output = tensor.mask_where(mask, value); + let expected = TensorData::from([[8, 7], [2, 11]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_int_mask_fill_ops() { + let device = Default::default(); + let tensor = TestTensorInt::<2>::from_data([[1, 7], [2, 3]], &device); + let mask = + TestTensorBool::<2>::from_bool(TensorData::from([[true, false], [false, true]]), &device); + + let output = tensor.mask_fill(mask, 9); + let expected = TensorData::from([[9, 7], [2, 9]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/matmul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/matmul.rs new file mode 100644 index 0000000..c01af14 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/matmul.rs @@ -0,0 +1,203 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_int_matmul_d2() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_ints([[1, 7], [2, 3], [1, 5]], &device); + let tensor_2 = TestTensorInt::<2>::from_ints([[4, 7, 5], [2, 3, 5]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[18, 28, 40], [14, 23, 25], [14, 22, 30]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_d3() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<3>::from_ints([[[1, 7], [2, 3]]], &device); + let tensor_2 = TestTensorInt::<3>::from_ints([[[4, 7], [2, 3]]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[[18, 28], [14, 23]]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_broadcast_1() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<3>::from_ints([[[1, 7], [2, 3]]], &device); + let tensor_2 = TestTensorInt::from_ints([[[4, 7], [2, 3]], [[2, 5], [6, 3]]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[[18, 28], [14, 23]], [[44, 26], [22, 19]]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_broadcast_4d() { + let device = Default::default(); + // [2, 1, 2, 2] + let tensor_1 = TestTensorInt::<4>::from_ints([[[[1, 7], [2, 3]]], [[[2, 5], [6, 3]]]], &device); + // [1, 2, 2, 2] + let tensor_2 = TestTensorInt::from_ints([[[[9, 8], [1, 4]], [[2, 7], [3, 5]]]], &device); + + // [2, 1, 2, 2] @ [1, 2, 2, 2] -> [2, 2, 2, 2] + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [[[16, 36], [21, 28]], [[23, 42], [13, 29]]], + [[[23, 36], [57, 60]], [[19, 39], [21, 57]]], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_simple_1() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_ints([[5, 14], [14, 25]], &device); + let tensor_2 = TestTensorInt::from_ints([[3, 4, 5], [0, 1, 2]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[15, 34, 53], [42, 81, 120]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_4_3() { + if (IntElem::MAX as u32) < 324 { + return; + } + + let device = Default::default(); + let tensor_1 = + TestTensorInt::<2>::from_ints([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], &device); + let tensor_2 = + TestTensorInt::from_ints([[0, 1, 2], [4, 5, 6], [8, 9, 10], [12, 13, 14]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[56, 62, 68], [152, 174, 196], [248, 286, 324]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_trivial() { + if (IntElem::MAX as u32) < 506 { + return; + } + + let device = Default::default(); + + let tensor_1 = TestTensorInt::<1>::arange(0..16, &device).reshape([4, 4]); + + let tensor_3 = tensor_1.clone().matmul(tensor_1); + + tensor_3.into_data().assert_eq( + &TensorData::from([ + [56, 62, 68, 74], + [152, 174, 196, 218], + [248, 286, 324, 362], + [344, 398, 452, 506], + ]), + false, + ); +} + +#[test] +fn test_int_matmul_trivial_transposed() { + if (IntElem::MAX as u32) < 734 { + return; + } + + let device = Default::default(); + + let tensor_1 = TestTensorInt::<1>::arange(0..16, &device).reshape([4, 4]); + + let tensor_3 = tensor_1.clone().matmul(tensor_1.transpose()); + + tensor_3.into_data().assert_eq( + &TensorData::from([ + [14, 38, 62, 86], + [38, 126, 214, 302], + [62, 214, 366, 518], + [86, 302, 518, 734], + ]), + false, + ); +} + +#[test] +fn test_int_matmul_4_8() { + if (IntElem::MAX as u32) < 6092 { + return; + } + + let device = Default::default(); + + let tensor_1 = TestTensorInt::<1>::arange(0..32, &device).reshape([4, 8]); + + let tensor_3 = tensor_1.clone().matmul(tensor_1.transpose()); + + tensor_3.into_data().assert_eq( + &TensorData::from([ + [140, 364, 588, 812], + [364, 1100, 1836, 2572], + [588, 1836, 3084, 4332], + [812, 2572, 4332, 6092], + ]), + false, + ); +} + +#[test] +fn test_int_matmul_simple_2() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_ints([[1, 2, 3, 4]], &device); + let tensor_2 = TestTensorInt::from_ints([[3], [4], [5], [6]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([[50]]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_int_matmul_simple_3() { + let device = Default::default(); + let tensor_1 = + TestTensorInt::<2>::from_ints([[3, 3, 3], [4, 4, 4], [5, 5, 5], [6, 6, 6]], &device); + let tensor_2 = TestTensorInt::from_ints([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [9, 18, 27, 36], + [12, 24, 36, 48], + [15, 30, 45, 60], + [18, 36, 54, 72], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn int_should_panic_when_inner_dimensions_are_not_equal() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_ints([[3, 3], [4, 4], [5, 5], [6, 6]], &device); + let tensor_2 = TestTensorInt::from_ints([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], &device); + + let tensor_3 = tensor_1.matmul(tensor_2); + let expected = TensorData::from([ + [9, 18, 27, 36], + [12, 24, 36, 48], + [15, 30, 45, 60], + [18, 36, 54, 72], + ]); + + tensor_3.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mod.rs new file mode 100644 index 0000000..da798c5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mod.rs @@ -0,0 +1,47 @@ +pub use super::*; // re-export test types + +mod abs; +mod add; +mod aggregation; +mod all; +mod any; +mod arange; +mod arange_step; +mod bitwise; +mod cartesian_grid; +mod cast; +mod cat; +mod chunk; +mod comparison; +mod create_like; +mod cumulative; +mod div; +mod expand; +mod flip; +mod full; +mod gather_scatter; +mod init; +mod mask; +mod matmul; +mod movedim; +mod mul; +mod one_hot; +mod permute; +mod random; +mod remainder; +mod repeat; +mod repeat_dim; +mod reshape; +mod roll; +mod select; +mod sign; +mod slice; +mod slice_assign; +mod sort_argsort; +mod stack; +mod sub; +mod take; +mod topk; +mod transpose; +mod tri; +mod unfold; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/movedim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/movedim.rs new file mode 100644 index 0000000..613e37e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/movedim.rs @@ -0,0 +1,52 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn movedim_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + let permuted = tensor.clone().movedim(0, 2); + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim(0, 2) + let expected = TensorData::from([ + [[0, 12], [1, 13], [2, 14], [3, 15]], + [[4, 16], [5, 17], [6, 18], [7, 19]], + [[8, 20], [9, 21], [10, 22], [11, 23]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().movedim(0, -1); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().movedim(0, 0); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} + +#[test] +fn vec_input_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + let permuted = tensor.clone().movedim(vec![0, 1], vec![1, 0]); + // from pytorch + // import torch; torch.arange(0, 24).reshape(2, 3, 4).movedim([0, 1], [1, 0]) + let expected = TensorData::from([ + [[0, 1, 2, 3], [12, 13, 14, 15]], + [[4, 5, 6, 7], [16, 17, 18, 19]], + [[8, 9, 10, 11], [20, 21, 22, 23]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axes + let permuted = tensor.clone().movedim(vec![-3, -2], vec![-2, -3]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axes + let permuted = tensor.clone().movedim(vec![0, 1], vec![0, 1]); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mul.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mul.rs new file mode 100644 index 0000000..5d7b261 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/mul.rs @@ -0,0 +1,42 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_mul_ops_int() { + let data_1 = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let data_2 = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0, 1, 4], [9, 16, 25]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_mul_broadcast_int() { + let data_1 = TensorData::from([[0, 1, 2]]); + let data_2 = TensorData::from([[3, 4, 5], [6, 7, 8]]); + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let output = tensor_1 * tensor_2; + let expected = TensorData::from([[0, 4, 10], [0, 7, 16]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_mul_scalar_ops_int() { + let data = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let scalar = 2; + let tensor = TestTensorInt::<2>::from_data(data, &Default::default()); + + let output = tensor * scalar; + let expected = TensorData::from([[0, 2, 4], [6, 8, 10]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/one_hot.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/one_hot.rs new file mode 100644 index 0000000..cd6c0aa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/one_hot.rs @@ -0,0 +1,59 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn int_should_support_one_hot() { + let tensor = TestTensorInt::<1>::from([0, 1, 4]); + let one_hot_tensor: TestTensorInt<2> = tensor.one_hot(5); + let expected = TensorData::from([[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 1]]); + one_hot_tensor.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn int_one_hot_should_panic_when_index_exceeds_number_of_classes() { + let tensor = TestTensorInt::<1>::from([5]); + let _result: TestTensorInt<2> = tensor.one_hot(5); +} + +#[test] +#[should_panic] +fn int_one_hot_should_panic_when_number_of_classes_is_zero() { + let tensor = TestTensorInt::<1>::from([2]); + let _result: TestTensorInt<2> = tensor.one_hot(0); +} + +#[test] +fn one_hot_fill_with_positive_axis_and_indices() { + let tensor = TestTensorInt::<2>::from([[1, 9], [2, 4]]); + let expected = TensorData::from([ + [ + [1, 1], + [3, 1], + [1, 1], + [1, 1], + [1, 1], + [1, 1], + [1, 1], + [1, 1], + [1, 1], + [1, 3], + ], + [ + [1, 1], + [1, 1], + [3, 1], + [1, 1], + [1, 3], + [1, 1], + [1, 1], + [1, 1], + [1, 1], + [1, 1], + ], + ]); + + let one_hot_tensor: TestTensorInt<3> = tensor.one_hot_fill(10, 3.0, 1.0, 1); + + one_hot_tensor.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/permute.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/permute.rs new file mode 100644 index 0000000..3f008f8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/permute.rs @@ -0,0 +1,49 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn permute_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + let permuted = tensor.clone().permute([2, 1, 0]); + + // from pytorch: + // import torch; torch.arange(0, 24).reshape(2, 3, 4).permute(2, 1, 0) + let expected = TensorData::from([ + [[0, 12], [4, 16], [8, 20]], + [[1, 13], [5, 17], [9, 21]], + [[2, 14], [6, 18], [10, 22]], + [[3, 15], [7, 19], [11, 23]], + ]); + + permuted.into_data().assert_eq(&expected, false); + + // Test with negative axis + let permuted = tensor.clone().permute([-1, 1, 0]); + permuted.into_data().assert_eq(&expected, false); + + // Test with the same axis + let permuted = tensor.clone().permute([0, 1, 2]); + permuted.into_data().assert_eq(&tensor.into_data(), true); +} + +#[test] +#[should_panic] +fn edge_repeated_axes() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with a repeated axis + let _ = tensor.clone().permute([0, 0, 1]); +} + +#[test] +#[should_panic] +fn edge_out_of_bound_axis() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(0..24, &device).reshape([2, 3, 4]); + + // Test with a repeated axis + let _ = tensor.clone().permute([3, 0, 1]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/random.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/random.rs new file mode 100644 index 0000000..83138c1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/random.rs @@ -0,0 +1,18 @@ +use super::*; +use burn_tensor::{Distribution, ElementConversion}; + +#[test] +fn rand_uniform_int() { + let low = 0.; + let high = 5.; + + let tensor = TestTensorInt::<1>::random( + [100_000], + Distribution::Uniform(low, high), + &Default::default(), + ); + + tensor + .into_data() + .assert_within_range::(low.elem()..high.elem()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/remainder.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/remainder.rs new file mode 100644 index 0000000..526ffea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/remainder.rs @@ -0,0 +1,27 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_int_remainder_basic() { + let data = TensorData::from([-3, -2, -1, 1, 2, 3]); + let device = Default::default(); + let lhs = TestTensorInt::<1>::from_data(data, &device); + + let rhs = TestTensorInt::from_data(TensorData::from([2, 3, 1, 2, 1, 3]), &device); + let output = lhs.remainder(rhs); + let expected = TensorData::from([1, 1, -0, 1, 0, 0]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_int_remainder_basic_scalar() { + let data = TensorData::from([-3, -2, -1, 1, 2, 3]); + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data(data, &device); + + let output = tensor.remainder_scalar(2); + let expected = TensorData::from([1, 0, 1, 1, 0, 1]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat.rs new file mode 100644 index 0000000..44e488a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat.rs @@ -0,0 +1,100 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_int_repeat_ops_one_dimension() { + let data = TensorData::from([[0i32, 1i32, 2i32]]); + let tensor = TestTensorInt::<2>::from_data(data, &Default::default()); + + let output = tensor.repeat(&[4, 1, 1]); + let expected = TensorData::from([ + [0i32, 1i32, 2i32], + [0i32, 1i32, 2i32], + [0i32, 1i32, 2i32], + [0i32, 1i32, 2i32], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_int_repeat_on_many_dims() { + let data = TensorData::from([ + [[1i32, 2i32], [3i32, 4i32]], + [[5i32, 6i32], [7i32, 8i32]], + [[9i32, 10i32], [11i32, 12i32]], + [[13i32, 14i32], [15i32, 16i32]], + ]); + let tensor = TestTensorInt::<3>::from_data(data, &Default::default()); + + let output = tensor.repeat(&[2, 3, 2]); + + let expected = TensorData::from([ + [ + [1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32], + [1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32], + [1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32], + ], + [ + [5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32], + [5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32], + [5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32], + ], + [ + [9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32], + [9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32], + [9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32], + ], + [ + [13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32], + [13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32], + [13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32], + ], + [ + [1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32], + [1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32], + [1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32], + ], + [ + [5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32], + [5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32], + [5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32], + ], + [ + [9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32], + [9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32], + [9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32], + ], + [ + [13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32], + [13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32], + [13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat_dim.rs new file mode 100644 index 0000000..fd29a32 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/repeat_dim.rs @@ -0,0 +1,46 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_int_repeat_ops() { + let data = TensorData::from([[0, 1, 2]]); + let tensor = TestTensorInt::<2>::from_data(data, &Default::default()); + + let output = tensor.repeat_dim(0, 4); + let expected = TensorData::from([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_int_repeat_on_dims_larger_than_1() { + let data = TensorData::from([ + [[1i32, 2i32], [3i32, 4i32]], + [[5i32, 6i32], [7i32, 8i32]], + [[9i32, 10i32], [11i32, 12i32]], + [[13i32, 14i32], [15i32, 16i32]], + ]); + let tensor = TestTensorInt::<3>::from_data(data, &Default::default()); + + let output = tensor.repeat_dim(2, 3); + let expected = TensorData::from([ + [ + [1i32, 2i32, 1i32, 2i32, 1i32, 2i32], + [3i32, 4i32, 3i32, 4i32, 3i32, 4i32], + ], + [ + [5i32, 6i32, 5i32, 6i32, 5i32, 6i32], + [7i32, 8i32, 7i32, 8i32, 7i32, 8i32], + ], + [ + [9i32, 10i32, 9i32, 10i32, 9i32, 10i32], + [11i32, 12i32, 11i32, 12i32, 11i32, 12i32], + ], + [ + [13i32, 14i32, 13i32, 14i32, 13i32, 14i32], + [15i32, 16i32, 15i32, 16i32, 15i32, 16i32], + ], + ]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/reshape.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/reshape.rs new file mode 100644 index 0000000..ba9dee1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/reshape.rs @@ -0,0 +1,179 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_reshape_maybe_fused_1() { + let tensor = TestTensorInt::arange(0..32, &Default::default()); + let tensor0 = TestTensorInt::zeros([8, 4, 8], &Default::default()); + let tensor1 = tensor.clone().reshape([1, 4, 8]); + let output = tensor0 + tensor1; + + let expected = TensorData::from([ + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30, 31], + ], + ]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_reshape_maybe_fused_2() { + let tensor = TestTensorInt::<3>::from_data([[[0, 2], [1, 2]]], &Default::default()); + let tensor1 = tensor.reshape([2, 2, 1]); + let tensor2 = TestTensorInt::<3>::full([2, 2, 4], 4, &Default::default()); + let output = tensor2 + tensor1; + + let expected_tensor1 = + TensorData::from([[[4, 4, 4, 4], [6, 6, 6, 6]], [[5, 5, 5, 5], [6, 6, 6, 6]]]); + output.into_data().assert_eq(&expected_tensor1, false); +} + +#[test] +fn should_support_reshape_maybe_fused_3() { + let tensor = TestTensorInt::<3>::from_data([[[0, 2], [1, 2]]], &Default::default()); + let tensor1 = tensor.reshape([2, 2, 1]); + let _tensor2 = TestTensorInt::<3>::full([2, 2, 3], 5, &Default::default()); + + let expected_tensor1 = TensorData::from([[[0], [2]], [[1], [2]]]); + tensor1.into_data().assert_eq(&expected_tensor1, false); +} + +#[test] +fn should_support_reshape_maybe_fused_4() { + let tensor = TestTensorInt::<3>::from_data([[[0, 2], [1, 2]]], &Default::default()); + let tensor2 = TestTensorInt::<3>::full([2, 2, 4], 4, &Default::default()); + let tensor2 = tensor2.swap_dims(0, 1); + let tensor1 = tensor.reshape([2, 2, 1]); + let output = tensor2 + tensor1; + + let expected_tensor1 = + TensorData::from([[[4, 4, 4, 4], [6, 6, 6, 6]], [[5, 5, 5, 5], [6, 6, 6, 6]]]); + output.into_data().assert_eq(&expected_tensor1, false); +} + +#[test] +fn should_support_reshape_maybe_fused_5() { + let tensor = TestTensorInt::<3>::from_data([[[0], [1], [2], [3]]], &Default::default()); + let tensor1 = tensor.clone().reshape([2, 1, 2]); + let tensor2 = TestTensorInt::<3>::full([2, 4, 2], 0, &Default::default()); + let output = tensor2.clone() + tensor1 + tensor.clone(); + + let expected_tensor1 = TensorData::from([ + [[0, 1], [1, 2], [2, 3], [3, 4]], + [[2, 3], [3, 4], [4, 5], [5, 6]], + ]); + output.into_data().assert_eq(&expected_tensor1, false); +} + +#[test] +fn should_support_reshape_maybe_fused_6() { + let device = Default::default(); + + let tensor1 = TestTensorInt::arange(0..32, &device); + let tensor1 = tensor1.reshape([2, 4, 4]); + + let tensor2 = TestTensorInt::arange(0..16, &device); + let tensor2 = tensor2.reshape([1, 4, 4]); + + let tensor3 = TestTensorInt::arange(0..8, &device); + let tensor3 = tensor3.reshape([4, 1, 2]); + let tensor3 = tensor3.swap_dims(0, 2); + + let out = tensor1 + tensor2 + tensor3; + + let expected = TensorData::from([ + [ + [0, 4, 8, 12], + [8, 12, 16, 20], + [16, 20, 24, 28], + [24, 28, 32, 36], + ], + [ + [17, 21, 25, 29], + [25, 29, 33, 37], + [33, 37, 41, 45], + [41, 45, 49, 53], + ], + ]); + out.to_data().assert_eq(&expected, false); +} + +// Skip on metal - cubecl autotune error +// Enable once this issue is fixed: https://github.com/tracel-ai/burn/issues/4327 +#[cfg(not(feature = "metal"))] +#[test] +fn should_support_multiple_reshapes_cloned_tensor() { + let device = Default::default(); + + let lhs = TestTensorInt::<1>::arange(0..4, &device).reshape([2, 2]); + // fusion should preserve correct strides when operating on the same tensor + let rhs = lhs.clone(); + + let lhs = lhs.reshape([2, 2, 1]); + let rhs = rhs.reshape([1, 2, 2]); + + let p = lhs.mul(rhs); + + let s = p.sum_dim(1); + + let out = s.reshape([2, 2]); + + out.into_data() + .assert_eq(&TensorData::from([[2, 3], [6, 11]]), false); +} + +#[test] +fn should_support_reshape_int() { + let data = TensorData::from([0, 1, 2]); + let tensor = TestTensorInt::<1>::from_data(data, &Default::default()); + + let output = tensor.clone().reshape([1, 3]); + let expected = TensorData::from([[0, 1, 2]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/roll.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/roll.rs new file mode 100644 index 0000000..e53e928 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/roll.rs @@ -0,0 +1,108 @@ +use super::*; +use burn_tensor::TensorData; + +#[ignore = "0 size resources are not yet supported"] +#[test] +fn test_roll_empty() { + let device = Default::default(); + let input = TestTensorInt::<2>::zeros([12, 0], &device); + + let result = input.clone().roll(&[1, 2], &[0, 1]); + + assert_eq!(&*result.shape(), &[12, 0]); + + // TODO: Rolling an empty tensor should return the same empty tensor; + // but we have no way to compare tensor references yet. +} + +#[test] +fn test_roll() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + // No-op shift: + input + .clone() + .roll(&[0, 0], &[0, 1]) + .to_data() + .assert_eq(&input.clone().to_data(), false); + + input + .clone() + .roll(&[1, -1], &[0, 1]) + .to_data() + .assert_eq(&TensorData::from([[5, 3, 4], [2, 0, 1]]), false); + + input + .clone() + .roll(&[-1, 1], &[1, 0]) + .to_data() + .assert_eq(&TensorData::from([[5, 3, 4], [2, 0, 1]]), false); + + input + .clone() + .roll(&[2 * 32 + 1, 3 * (-400) - 1], &[0, 1]) + .to_data() + .assert_eq(&TensorData::from([[5, 3, 4], [2, 0, 1]]), false); +} + +#[should_panic] +#[test] +fn test_roll_dim_too_big() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + // Attempting to roll on a dimension that doesn't exist should panic + let _d = input.roll(&[1], &[2]); +} + +#[should_panic] +#[test] +fn test_roll_dim_too_small() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + // Attempting to roll on a dimension that doesn't exist should panic + let _d = input.roll(&[1], &[-3]); +} + +#[should_panic] +#[test] +fn test_roll_shift_size_mismatch() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + // Attempting to roll with a shift size that doesn't match the number of dimensions should panic + let _d = input.roll(&[1, 2], &[0]); +} + +#[test] +fn test_roll_dim() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + input + .clone() + .roll_dim(1, 0) + .to_data() + .assert_eq(&TensorData::from([[3, 4, 5], [0, 1, 2]]), false); + + input + .clone() + .roll_dim(-1, 1) + .to_data() + .assert_eq(&TensorData::from([[2, 0, 1], [5, 3, 4]]), false); +} + +#[should_panic] +#[test] +fn test_roll_dim_dim_too_big() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + // Attempting to roll on a dimension that doesn't exist should panic + let _d = input.roll_dim(1, 2); +} + +#[should_panic] +#[test] +fn test_roll_dim_dim_too_small() { + let input = TestTensorInt::<2>::from([[0, 1, 2], [3, 4, 5]]); + + // Attempting to roll on a dimension that doesn't exist should panic + let _d = input.roll_dim(1, -3); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/select.rs new file mode 100644 index 0000000..ef1d5c4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/select.rs @@ -0,0 +1,38 @@ +use super::*; +use burn_tensor::{IndexingUpdateOp, TensorData}; + +#[test] +fn should_select_1d_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data([5, 6, 7], &device); + let indices = TestTensorInt::from_data([1, 1, 0, 1, 2], &device); + + let output = tensor.select(0, indices); + let expected = TensorData::from([6, 6, 5, 6, 7]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_select_add_1d_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data([7, 8, 9], &device); + let values = TestTensorInt::from_data([5, 4, 3, 2, 1], &device); + let indices = TestTensorInt::from_data(TensorData::from([1, 1, 0, 1, 2]), &device); + + let output = tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); + let expected = TensorData::from([10, 19, 10]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn should_panic_select_add_invalid_num_indices() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data([0; 12], &device); + let values = TestTensorInt::from_data([1; 12], &device); + let indices = TestTensorInt::from_data(TensorData::from([1]), &device); + + tensor.select_assign(0, indices, values, IndexingUpdateOp::Add); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sign.rs new file mode 100644 index 0000000..db4f6f6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sign.rs @@ -0,0 +1,12 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_sign_ops_int() { + let tensor = TestTensorInt::<2>::from([[-2, -1, 2], [3, 0, -5]]); + + let output = tensor.sign(); + let expected = TensorData::from([[-1, -1, 1], [1, 0, -1]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice.rs new file mode 100644 index 0000000..2e725bf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice.rs @@ -0,0 +1,29 @@ +use super::*; +use burn_tensor::{TensorData, s}; + +#[test] +fn slice_should_not_corrupt_potentially_inplace_operations() { + let tensor = TestTensorInt::<1>::from([1, 2, 3, 4, 5]); + let tensor = tensor.clone().slice([0..3]) + tensor.clone().slice([2..5]); + + let expected = TensorData::from([4, 6, 8]); + + tensor.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_int_tensor_with_steps() { + let device = Default::default(); + let tensor = + TestTensorInt::<2>::from_data([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], &device); + + // Test step=2 along first dimension + let sliced = tensor.clone().slice([s![0..3;2]]); + let expected = TensorData::from([[1i32, 2, 3, 4], [9, 10, 11, 12]]); + sliced.into_data().assert_eq(&expected, false); + + // Test step=-1 along second dimension + let sliced = tensor.clone().slice(s![.., 0..4;-1]); + let expected = TensorData::from([[4i32, 3, 2, 1], [8, 7, 6, 5], [12, 11, 10, 9]]); + sliced.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice_assign.rs new file mode 100644 index 0000000..5e5b1f6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/slice_assign.rs @@ -0,0 +1,55 @@ +use super::*; +use burn_tensor::{TensorData, s}; + +#[test] +fn slice_assign_should_not_corrupt_potentially_inplace_operations() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data([1, 2, 3, 4, 5], &device); + let values = TestTensorInt::<1>::from_data([10, 20, 30], &device); + let tensor_1 = tensor.clone().slice_assign([0..3], values); + let tensor_2 = tensor + 2; + + let expected = TensorData::from([10, 20, 30, 4, 5]); + + tensor_1.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([3, 4, 5, 6, 7]); + + tensor_2.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_slice_assign_int_tensor_with_steps() { + let device = Default::default(); + let tensor = + TestTensorInt::<2>::from_data([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], &device); + + // Test step=2 along first dimension + let values = + TestTensorInt::<2>::from_data([[100, 101, 102, 103], [200, 201, 202, 203]], &device); + let output = tensor.clone().slice_assign([s![0..3;2]], values); + let expected = TensorData::from([[100i32, 101, 102, 103], [5, 6, 7, 8], [200, 201, 202, 203]]); + output.into_data().assert_eq(&expected, false); + + // Test step=-1 along second dimension + let values = TestTensorInt::<2>::from_data( + [[40, 30, 20, 10], [80, 70, 60, 50], [120, 110, 100, 90]], + &device, + ); + let output = tensor.slice_assign(s![.., 0..4;-1], values); + let expected = TensorData::from([[10i32, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]); + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_slice_assign_empty_range_int() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::from_data([1, 2, 3, 4, 5], &device); + let values: TestTensorInt<1> = TestTensorInt::empty([0], &device); + + // Empty slice assignment for int tensor + let output = tensor.clone().slice_assign([3..3], values); + let expected = TensorData::from([1i32, 2, 3, 4, 5]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sort_argsort.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sort_argsort.rs new file mode 100644 index 0000000..21322db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sort_argsort.rs @@ -0,0 +1,153 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_sort_1d_int() { + // Skip with u8 + if (IntElem::MAX as u32) < 1000u32 { + return; + } + + let tensor = TestTensorInt::<1>::from([1, 4, 7, 2, 5, 6, 3, 0, 9, 8, 2, 8, -10, 42, 1000]); + + // Sort along dim=0 + let values = tensor.sort(0); + let values_expected = TensorData::from([-10, 0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 8, 9, 42, 1000]); + + values.into_data().assert_eq(&values_expected, false); +} + +#[test] +fn test_argsort_1d_int() { + // Skip with u8 + if (IntElem::MAX as u32) < 1000u32 { + return; + } + + let tensor = TestTensorInt::<1>::from([1, 4, 7, 2, 5, 6, 3, 0, 9, 8, -10, 42, 1000]); + + // Sort along dim=0 + let indices = tensor.argsort(0); + let indices_expected = TensorData::from([10, 7, 0, 3, 6, 1, 4, 5, 2, 9, 8, 11, 12]); + + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_with_indices_descending_int() { + // Skip with u8 + if (IntElem::MAX as u32) >= 1000u32 { + // 1D + let tensor = TestTensorInt::<1>::from([1, 4, 7, 2, 5, 6, 3, 0, 9, 8, -10, 42, 1000]); + + // Sort along dim=0 + let (values, indices) = tensor.sort_descending_with_indices(0); + + let values_expected = TensorData::from([1000, 42, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -10]); + values.into_data().assert_eq(&values_expected, false); + + let indices_expected = TensorData::from([12, 11, 8, 9, 2, 5, 4, 1, 6, 3, 0, 7, 10]); + indices.into_data().assert_eq(&indices_expected, false); + } + + // 2D + let tensor = TestTensorInt::<3>::from([[[1, 4, 7], [2, 5, 6]], [[3, 0, 9], [8, 2, 8]]]); + + // Sort along dim=1 + let (values, indices) = tensor.sort_descending_with_indices(1); + + let values_expected = TensorData::from([[[2, 5, 7], [1, 4, 6]], [[8, 2, 9], [3, 0, 8]]]); + values.into_data().assert_eq(&values_expected, false); + + let indices_expected = TensorData::from([[[1, 1, 0], [0, 0, 1]], [[1, 1, 0], [0, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_int() { + let tensor = TestTensorInt::<3>::from([[[1, 4, 7], [2, 5, 6]], [[3, 0, 9], [8, 2, 8]]]); + + // Sort along dim=0 + let values = tensor.clone().sort(0); + + let values_expected = TensorData::from([[[1, 0, 7], [2, 2, 6]], [[3, 4, 9], [8, 5, 8]]]); + values.into_data().assert_eq(&values_expected, false); + + // Sort along dim=1 + let values = tensor.clone().sort(1); + + let values_expected = TensorData::from([[[1, 4, 6], [2, 5, 7]], [[3, 0, 8], [8, 2, 9]]]); + values.into_data().assert_eq(&values_expected, false); + + // Sort along dim=2 + let values = tensor.sort(2); + + let values_expected = TensorData::from([[[1, 4, 7], [2, 5, 6]], [[0, 3, 9], [2, 8, 8]]]); + values.into_data().assert_eq(&values_expected, false); +} + +#[test] +fn test_sort_with_indices_int() { + let tensor = TestTensorInt::<3>::from([[[1, 4, 7], [2, 5, 6]], [[3, 0, 9], [7, 2, 8]]]); + + // Sort along dim=0 + let (values, indices) = tensor.clone().sort_with_indices(0); + + let values_expected = TensorData::from([[[1, 0, 7], [2, 2, 6]], [[3, 4, 9], [7, 5, 8]]]); + values.into_data().assert_eq(&values_expected, false); + + let indices_expected = TensorData::from([[[0, 1, 0], [0, 1, 0]], [[1, 0, 1], [1, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=1 + let (values, indices) = tensor.clone().sort_with_indices(1); + + let values_expected = TensorData::from([[[1, 4, 6], [2, 5, 7]], [[3, 0, 8], [7, 2, 9]]]); + values.into_data().assert_eq(&values_expected, false); + + let indices_expected = TensorData::from([[[0, 0, 1], [1, 1, 0]], [[0, 0, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=2 + let (values, indices) = tensor.sort_with_indices(2); + + let values_expected = TensorData::from([[[1, 4, 7], [2, 5, 6]], [[0, 3, 9], [2, 7, 8]]]); + values.into_data().assert_eq(&values_expected, false); + + let indices_expected = TensorData::from([[[0, 1, 2], [0, 1, 2]], [[1, 0, 2], [1, 0, 2]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_argsort_int() { + let tensor = TestTensorInt::<3>::from([[[1, 4, 7], [2, 5, 6]], [[3, 0, 9], [7, 2, 8]]]); + + // Sort along dim=0 + let indices = tensor.clone().argsort(0); + + let indices_expected = TensorData::from([[[0, 1, 0], [0, 1, 0]], [[1, 0, 1], [1, 0, 1]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=1 + let indices = tensor.clone().argsort(1); + + let indices_expected = TensorData::from([[[0, 0, 1], [1, 1, 0]], [[0, 0, 1], [1, 1, 0]]]); + indices.into_data().assert_eq(&indices_expected, false); + + // Sort along dim=2 + let indices = tensor.argsort(2); + + let indices_expected = TensorData::from([[[0, 1, 2], [0, 1, 2]], [[1, 0, 2], [1, 0, 2]]]); + indices.into_data().assert_eq(&indices_expected, false); +} + +#[test] +fn test_sort_descending_1d() { + let tensor = TestTensorInt::<1>::from([1, 2, 3, 4, 5]); + + // Sort along dim=0 + let values = tensor.sort_descending(0); + + let values_expected = TensorData::from([5, 4, 3, 2, 1]); + values.into_data().assert_eq(&values_expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/stack.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/stack.rs new file mode 100644 index 0000000..ebda5ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/stack.rs @@ -0,0 +1,33 @@ +use super::*; +use alloc::vec; +use burn_tensor::{Tensor, TensorData}; + +#[test] +fn should_support_stack_ops_int() { + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data([[1, 2, 3]], &device); + let tensor_2 = TestTensorInt::<2>::from_data([[4, 5, 6]], &device); + + let output = Tensor::stack::<3>(vec![tensor_1, tensor_2], 0); + let expected = TensorData::from([[[1, 2, 3]], [[4, 5, 6]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_generate_row_major_layout() { + let device = Default::default(); + let tensor = TestTensorInt::<1>::arange(1..25, &device).reshape([4, 6]); + let zeros = TestTensorInt::zeros([4, 6], &device); + let intersperse = + Tensor::stack::<3>([tensor.clone(), zeros.clone()].to_vec(), 2).reshape([4, 12]); + + let expected = TensorData::from([ + [1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0], + [7, 0, 8, 0, 9, 0, 10, 0, 11, 0, 12, 0], + [13, 0, 14, 0, 15, 0, 16, 0, 17, 0, 18, 0], + [19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24, 0], + ]); + + intersperse.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sub.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sub.rs new file mode 100644 index 0000000..0a2a760 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/sub.rs @@ -0,0 +1,42 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_sub_ops_int() { + let data_1 = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let data_2 = TensorData::from([[6, 7, 8], [9, 10, 11]]); + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let output = tensor_1 - tensor_2; + let expected = TensorData::from([[-6, -6, -6], [-6, -6, -6]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_sub_broadcast_int() { + let data_1 = TensorData::from([[0, 1, 2]]); + let data_2 = TensorData::from([[3, 4, 5], [6, 7, 8]]); + let device = Default::default(); + let tensor_1 = TestTensorInt::<2>::from_data(data_1, &device); + let tensor_2 = TestTensorInt::<2>::from_data(data_2, &device); + + let output = tensor_1 - tensor_2; + let expected = TensorData::from([[-3, -3, -3], [-6, -6, -6]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_sub_scalar_ops_int() { + let data = TensorData::from([[0, 1, 2], [3, 4, 5]]); + let scalar = 2; + let tensor = TestTensorInt::<2>::from_data(data, &Default::default()); + + let output = tensor - scalar; + let expected = TensorData::from([[-2, -1, 0], [1, 2, 3]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/take.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/take.rs new file mode 100644 index 0000000..d9c28ef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/take.rs @@ -0,0 +1,31 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_take_int_tensor() { + // Test take with integer tensors + let device = Default::default(); + let tensor = TestTensorInt::<2>::from_data([[10, 20, 30], [40, 50, 60]], &device); + let indices = TestTensorInt::<1>::from_data([1, 0], &device); + + let output = tensor.take::<1, 2>(0, indices); + let expected = TensorData::from([[40, 50, 60], [10, 20, 30]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_take_int_tensor_with_2d_indices() { + // Test take with integer tensors - output will be 3D + let device = Default::default(); + let tensor = TestTensorInt::<2>::from_data([[10, 20, 30], [40, 50, 60], [70, 80, 90]], &device); + + // 2D indices - shape [2, 2] + let indices = TestTensorInt::<2>::from_data([[0, 2], [2, 1]], &device); + let output = tensor.take::<2, 3>(0, indices); + + // Expected: shape [2, 2, 3] + let expected = TensorData::from([[[10, 20, 30], [70, 80, 90]], [[70, 80, 90], [40, 50, 60]]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/topk.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/topk.rs new file mode 100644 index 0000000..eb71617 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/topk.rs @@ -0,0 +1,59 @@ +use super::*; +use burn_tensor::TensorData; +use burn_tensor::Tolerance; + +#[test] +fn test_topk_1d() { + // Int + let tensor = TestTensorInt::<1>::from([1, 2, 3, 4, 5]); + + let values = tensor.topk(3, /*dim*/ 0); + let expected = TensorData::from([5, 4, 3]); + + values.into_data().assert_eq(&expected, false); + + // Float + let tensor = TestTensor::<1>::from([1., 2., 3., 4., 5.]); + + let values = tensor.topk(3, /*dim*/ 0); + let expected = TensorData::from([5., 4., 3.]); + + values + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_topk() { + // 3D Int + let tensor = TestTensorInt::<3>::from([[[1, 4, 7], [2, 5, 6]], [[3, 0, 9], [8, 2, 8]]]); + + let values = tensor.topk(2, /*dim*/ 2); + let expected = TensorData::from([[[7, 4], [6, 5]], [[9, 3], [8, 8]]]); + + values.into_data().assert_eq(&expected, false); + + // 3D Float + let tensor = + TestTensor::<3>::from([[[1., 4., 7.], [2., 5., 6.]], [[3., 0., 9.], [8., 2., 8.]]]); + + let values = tensor.topk(2, /*dim*/ 2); + let expected = TensorData::from([[[7., 4.], [6., 5.]], [[9., 3.], [8., 8.]]]); + + values + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); +} + +#[test] +fn test_topk_with_indices_1d() { + let tensor = TestTensorInt::<1>::from([1, 2, 3, 4, 5]); + + let (values, indices) = tensor.topk_with_indices(3, /*dim*/ 0); + + let values_expected = TensorData::from([5, 4, 3]); + values.into_data().assert_eq(&values_expected, false); + + let indices_expected = TensorData::from([4, 3, 2]); + indices.into_data().assert_eq(&indices_expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/transpose.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/transpose.rs new file mode 100644 index 0000000..7de0b0d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/transpose.rs @@ -0,0 +1,28 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn should_support_transpose_ops_int() { + let tensor = TestTensorInt::<3>::from_data( + [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]], + &Default::default(), + ); + + let output = tensor.transpose(); + let expected = TensorData::from([[[0, 3], [1, 4], [2, 5]], [[6, 9], [7, 10], [8, 11]]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn should_support_swap_dims_int() { + let tensor = TestTensorInt::<3>::from_data( + [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]], + &Default::default(), + ); + + let output = tensor.swap_dims(0, 2); + let expected = TensorData::from([[[0, 6], [3, 9]], [[1, 7], [4, 10]], [[2, 8], [5, 11]]]); + + output.into_data().assert_eq(&expected, false); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/tri.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/tri.rs new file mode 100644 index 0000000..c62de88 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/tri.rs @@ -0,0 +1,85 @@ +use super::*; +use burn_tensor::TensorData; + +#[test] +fn test_triu_negative_diagonal() { + let tensor = TestTensorInt::<2>::from([[1, 1, 1], [1, 1, 1], [1, 1, 1]]); + + let output = tensor.triu(-1); + let expected = TensorData::from([[1, 1, 1], [1, 1, 1], [0, 1, 1]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_triu_batch_tensors() { + let tensor = TestTensorInt::<4>::from([ + [[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]], + [[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]], + ]); + let output = tensor.triu(1); + let expected = TensorData::from([ + [[[0, 1, 1, 1], [0, 0, 1, 1], [0, 0, 0, 1], [0, 0, 0, 0]]], + [[[0, 1, 1, 1], [0, 0, 1, 1], [0, 0, 0, 1], [0, 0, 0, 0]]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn test_triu_too_few_dims() { + let tensor = TestTensorInt::<1>::from([1, 2, 3]); + let _output = tensor.triu(0); +} + +#[test] +fn test_tril() { + let tensor = TestTensor::<2>::from([[1., 1., 1.], [1., 1., 1.], [1., 1., 1.]]); + let output = tensor.tril(0); + let expected = TensorData::from([[1., 0., 0.], [1., 1., 0.], [1., 1., 1.]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_tril_positive_diagonal() { + let tensor = TestTensorInt::<2>::from([[1, 1, 1], [1, 1, 1], [1, 1, 1]]); + + let output = tensor.tril(1); + let expected = TensorData::from([[1, 1, 0], [1, 1, 1], [1, 1, 1]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_tril_negative_diagonal() { + let tensor = TestTensorInt::<2>::from([[1, 1, 1], [1, 1, 1], [1, 1, 1]]); + + let output = tensor.tril(-1); + let expected = TensorData::from([[0, 0, 0], [1, 0, 0], [1, 1, 0]]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +fn test_tril_batch_tensors() { + let tensor = TestTensorInt::<4>::from([ + [[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]], + [[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]], + ]); + let output = tensor.tril(1); + let expected = TensorData::from([ + [[[1, 1, 0, 0], [1, 1, 1, 0], [1, 1, 1, 1], [1, 1, 1, 1]]], + [[[1, 1, 0, 0], [1, 1, 1, 0], [1, 1, 1, 1], [1, 1, 1, 1]]], + ]); + + output.into_data().assert_eq(&expected, false); +} + +#[test] +#[should_panic] +fn test_tril_too_few_dims() { + let tensor = TestTensorInt::<1>::from([1, 2, 3]); + let _output = tensor.tril(0); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/unfold.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/unfold.rs new file mode 100644 index 0000000..779d37e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/ops/unfold.rs @@ -0,0 +1,39 @@ +use super::*; +use burn_tensor::Distribution; +use burn_tensor::s; + +#[test] +fn test_unfold_int() { + // Distribution::Default samples from [0, 255) + if (IntElem::MAX as u32) < 255 - 1 { + return; + } + let device = Default::default(); + + let input = TestTensorInt::<3>::random([2, 6, 6], Distribution::Default, &device); + + let dim = 1; + let size = 3; + let step = 2; + let actual: TestTensorInt<4> = input.clone().unfold(dim, size, step); + + let expected = TestTensorInt::<4>::empty([2, 2, 6, 3], &device) + .slice_assign( + s![.., 0, .., ..], + input + .clone() + .slice(s![.., 0..3, ..]) + .swap_dims(1, 2) + .unsqueeze_dim::<4>(1), + ) + .slice_assign( + s![.., 1, .., ..], + input + .clone() + .slice(s![.., 2..5, ..]) + .swap_dims(1, 2) + .unsqueeze_dim::<4>(1), + ); + + actual.to_data().assert_eq(&expected.to_data(), true); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/primitive.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/primitive.rs new file mode 100644 index 0000000..820a328 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/int/primitive.rs @@ -0,0 +1,16 @@ +use super::*; +use burn_tensor::{Element, Shape}; + +#[test] +fn should_support_int_dtype() { + let tensor = TestTensorInt::<2>::from([[0, -1, 2], [3, 4, -5]]).into_primitive(); + + assert_eq!( + burn_tensor::TensorMetadata::shape(&tensor), + Shape::new([2, 3]) + ); + assert_eq!( + burn_tensor::TensorMetadata::dtype(&tensor), + IntElem::dtype() // default int elem type + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/mod.rs new file mode 100644 index 0000000..afc9ae0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/mod.rs @@ -0,0 +1,10 @@ +pub use super::*; // re-export test types + +mod clone_invariance; +#[cfg(feature = "std")] +mod multi_threads; + +// Data types +mod bool; +mod float; +mod int; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/multi_threads.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/multi_threads.rs new file mode 100644 index 0000000..3af06e3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend-tests/tests/tensor/multi_threads.rs @@ -0,0 +1,171 @@ +use super::*; +use core::time::Duration; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; + +struct MultiThreadTestSettings { + num_threads: usize, + // The number of operations that are applied while the tensor is still alive and has a + // reference count > 1 on the new thread. + num_ops_alive: usize, + // The number of operations that are applied after the tensor is consumed for the last time. + num_ops_consumed: usize, + // Number of operations that needs to execute before continuing execution on the main thread. + sleep_before: Duration, + sleep_alive: Duration, + sleep_consumed: Duration, + // If the output is dropped, otherwise it will be consumed by an operation. + dropped: bool, +} + +#[test] +fn should_handle_multi_threads_dropped() { + run_multi_thread_test(MultiThreadTestSettings { + num_threads: 3, + num_ops_alive: 5, + num_ops_consumed: 5, + sleep_before: Duration::from_millis(100), + sleep_alive: Duration::from_millis(100), + sleep_consumed: Duration::from_millis(100), + dropped: true, + }) +} + +#[test] +fn should_handle_multi_threads_consumed() { + run_multi_thread_test(MultiThreadTestSettings { + num_threads: 3, + num_ops_alive: 5, + num_ops_consumed: 5, + sleep_before: Duration::from_millis(100), + sleep_alive: Duration::from_millis(100), + sleep_consumed: Duration::from_millis(100), + dropped: false, + }) +} + +#[test] +fn should_handle_multi_threads_drop_no_wait() { + run_multi_thread_test(MultiThreadTestSettings { + num_threads: 3, + num_ops_alive: 5, + num_ops_consumed: 5, + sleep_before: Duration::from_millis(100), + sleep_alive: Duration::from_millis(100), + sleep_consumed: Duration::from_millis(100), + dropped: true, + }) +} + +#[test] +fn should_handle_multi_threads_consumed_no_wait() { + run_multi_thread_test(MultiThreadTestSettings { + num_threads: 3, + num_ops_alive: 5, + num_ops_consumed: 5, + sleep_before: Duration::from_millis(100), + sleep_alive: Duration::from_millis(100), + sleep_consumed: Duration::from_millis(100), + dropped: false, + }) +} + +#[test] +fn should_handle_multi_threads_no_async_op() { + run_multi_thread_test(MultiThreadTestSettings { + num_threads: 3, + num_ops_alive: 0, + num_ops_consumed: 0, + sleep_before: Duration::from_millis(100), + sleep_alive: Duration::from_millis(100), + sleep_consumed: Duration::from_millis(100), + dropped: false, + }) +} + +// Skip on metal - flaky (works when ran alone) +// Enable once this issue is fixed: https://github.com/tracel-ai/burn/issues/4328 +#[cfg(not(feature = "metal"))] +#[test] +fn should_handle_multi_threads_no_async_op_no_wait() { + run_multi_thread_test(MultiThreadTestSettings { + num_threads: 3, + num_ops_alive: 0, + num_ops_consumed: 0, + sleep_before: Duration::from_millis(0), + sleep_alive: Duration::from_millis(100), + sleep_consumed: Duration::from_millis(100), + dropped: false, + }) +} + +fn run_multi_thread_test(settings: MultiThreadTestSettings) { + let tensor = TestTensor::<2>::from([[0.0, -1.0, 2.0], [3.0, 4.0, -5.0]]); + + let mut joined = Vec::with_capacity(settings.num_threads); + + let counter_alive = Arc::new(AtomicU32::new(0)); + let counter_consumed = Arc::new(AtomicU32::new(0)); + + for i in 0..settings.num_threads { + let tensor_moved = tensor.clone(); + let ca_moved = counter_alive.clone(); + let cc_moved = counter_consumed.clone(); + + let handle = std::thread::spawn(move || { + let mut base = tensor_moved.clone(); + std::thread::sleep(settings.sleep_before); + + if settings.num_ops_alive == 0 && settings.num_ops_consumed == 0 { + core::mem::drop(tensor_moved); + core::mem::drop(base); + } else { + if settings.num_ops_alive > 1 { + for j in 0..(settings.num_ops_alive - 1) { + base = tensor_moved.clone() + j as u32; + ca_moved.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(settings.sleep_alive); + } + } + + base = base * tensor_moved + i as u32; + ca_moved.fetch_add(1, Ordering::Relaxed); + + for n in 0..settings.num_ops_consumed { + base = base + n as i32; + cc_moved.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(settings.sleep_consumed); + } + let _data = base.into_data(); + } + }); + joined.push(handle); + } + + fn wait(counter: Arc, limit: usize) { + loop { + let counter_curr = counter.load(Ordering::Relaxed); + if counter_curr as usize >= limit { + break; + } else { + std::thread::sleep(Duration::from_millis(10)); + } + } + } + + wait(counter_alive, settings.num_ops_alive); + wait(counter_consumed, settings.num_ops_consumed); + + if settings.dropped { + core::mem::drop(tensor); + } else { + let t = tensor * 2.0; + let _t = t.into_data(); + } + + for j in joined { + j.join().unwrap(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-backend/Cargo.toml new file mode 100644 index 0000000..cd3da5f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/Cargo.toml @@ -0,0 +1,46 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Core backend interfaces and data structures for executing tensor operations in Burn." +documentation = "https://docs.rs/burn-backend" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-backend" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-backend" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std"] +doc = ["default"] +std = ["rand/std", "num-traits/std", "burn-std/std", "cubecl?/std"] + +tracing = ["burn-std/tracing", "cubecl/tracing"] + +cubecl = ["dep:cubecl", "burn-std/cubecl"] +cubecl-cuda = ["cubecl", "cubecl/cuda"] +cubecl-hip = ["cubecl", "cubecl/hip"] +cubecl-wgpu = ["cubecl", "cubecl/wgpu"] +cubecl-cpu = ["cubecl", "cubecl/cpu"] + +[dependencies] +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } +cubecl = { workspace = true, optional = true, default-features = false } + +bytemuck = { workspace = true, features = ["extern_crate_alloc"] } +derive-new = { workspace = true } +enumset = { workspace = true } +hashbrown = { workspace = true } +num-traits = { workspace = true } +rand = { workspace = true, default-features = false } +rand_distr = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +rand = { workspace = true, features = ["thread_rng"] } +paste = { workspace = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/README.md b/crates/stable-diffusion-burn/burn-crates/burn-backend/README.md new file mode 100644 index 0000000..e79c1b5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/README.md @@ -0,0 +1,4 @@ +# Burn Backend + +This crate includes the core backend interfaces and data structures for executing tensor operations +in Burn. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/base.rs new file mode 100644 index 0000000..3cba938 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/base.rs @@ -0,0 +1,391 @@ +use burn_std::DType; +pub use burn_std::backtrace::BackTrace; + +use alloc::string::String; +use enumset::{EnumSet, EnumSetType}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::element::Element; +use crate::ops::*; +use crate::tensor::{BoolTensor, FloatTensor, IntTensor, QuantizedTensor}; +use crate::{QTensorPrimitive, TensorData, TensorMetadata}; + +use super::DeviceOps; + +/// This trait defines all types and functions needed for a backend to be used with burn. +/// +/// ## Design +/// +/// This trait aims to be as unopinionated as possible and allows implementations to define +/// their own types and patterns. Therefore, there are few pre-defined abstractions baked +/// into this trait. +/// +/// Backends must define their own tensor types for each data type: `float`, `int`, and `bool`. +/// Since we minimize assumptions, we chose to separate these types, as they are used in +/// different contexts. However, some backends may have a generic tensor type that is used +/// for all data types. +/// +/// ### Eager Mode +/// +/// Because burn supports dynamic graphs, the backend trait is designed around kernel +/// implementations that can be called without any mutable context or graph. This may not be +/// ideal for backends that want to configure their computational graphs and execute them +/// multiple times. +/// +/// To implement this kind of backend, channels could be used to communicate with a backend +/// server thread to build the computation graphs and re-execute the ones that are repeated, +/// with some form of cache. Once that pattern has matured, a graph mode backend trait could +/// be extracted from it, allowing other backends of the same kind to be quickly integrated +/// with burn. This pattern could also be used to create an operation fusion trait, which +/// allows backends to define what kind of graph structures can be fused into one operation. +/// +/// ### Multi-Threaded +/// +/// Backend tensor types are all `Clone` + `Send`, which allows them to be safely +/// sent between threads. It is recommended to wrap tensors with [Arc](alloc::sync::Arc), +/// which avoids copying the tensor's buffer. Note that it is still possible to mutate and +/// reuse tensors' buffer without locking; see the next section on the Mutable API. +/// +/// ### Mutable API +/// +/// There is no mutable or inplace operation API to implement, but that does not mean that +/// backends cannot support them. Using [try_unwrap](alloc::sync::Arc::try_unwrap) and +/// [get_mut](alloc::sync::Arc::get_mut) allows backends to have access to an owned or mutable +/// reference to their tensor buffer data structure if the tensor is not shared. In that case, +/// backends can dispatch to their owned inplace operations for better performance. +/// +/// ## Documentation +/// +/// Most of the documentation for each function can be found on the user API +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// struct in the `burn-tensor` crate. +/// For modules, public functions are often created, which can be used by `burn-core` modules. +pub trait Backend: + FloatTensorOps + + BoolTensorOps + + IntTensorOps + + ModuleOps + + ActivationOps + + QTensorOps + + TransactionOps + + Clone + + Default + + Sized + + Send + + Sync + + core::fmt::Debug + + 'static +{ + /// Device type. + type Device: DeviceOps; + + /// Tensor primitive to be used for all float operations. + type FloatTensorPrimitive: TensorMetadata + 'static; + /// Default float element type. + type FloatElem: Element; + + /// Tensor primitive to be used for all int operations. + type IntTensorPrimitive: TensorMetadata + 'static; + /// Int element type. + type IntElem: Element; + + /// Tensor primitive to be used for all bool operations. + type BoolTensorPrimitive: TensorMetadata + 'static; + /// Tensor primitive to be used for all bool operations. + type BoolElem: Element; + + /// Tensor primitive to be used for all quantized operations. + type QuantizedTensorPrimitive: TensorMetadata + QTensorPrimitive + 'static; + + /// If autodiff is enabled. + fn ad_enabled(_device: &Self::Device) -> bool { + false + } + + /// Sets the current allocation mode to persistent. + #[allow(unused_variables)] + fn memory_persistent_allocations Output>( + device: &Self::Device, + input: Input, + func: Func, + ) -> Output { + func(input) + } + + /// Manually triggers a memory cleanup on the given device. + #[allow(unused_variables)] + fn memory_cleanup(device: &Self::Device) {} + + /// Name of the backend. + fn name(device: &Self::Device) -> String; + + /// Seeds the backend on the specified device. + /// + /// There is no guarantee that only the specified device will be seeded, but it is guaranteed + /// that at least the specified device will be seeded. + /// + /// In all cases, this should ensure deterministic execution for a single-threaded program. + fn seed(device: &Self::Device, seed: u64); + + /// Sync the backend, ensure that all computation are finished. + fn sync(_device: &Self::Device) -> Result<(), ExecutionError> { + Ok(()) + } + + /// Marks the given data as being used as a staging buffer for transfer between CPU and + /// accelerators like GPUs. + /// + /// The given data might be transferred to pinned memory or another format to improve data transfer + /// speed. + fn staging<'a, Iter>(_data: Iter, _device: &Self::Device) + where + Iter: Iterator, + { + } + + /// Whether the type is fully supported by the specified device for general operations. + /// + /// A type is considered supported if it can be used for the full suite of tensor + /// operations, including storage, conversion, and basic arithmetic. + /// + /// Returning `false` does not necessarily mean the device cannot handle the type at all. + /// For instance, a device might support a type only for specialized hardware + /// acceleration (e.g., matrix multiplication) but lack general arithmetic support. Such + /// types should return `false` here as they are not globally supported. + fn supports_dtype(device: &Self::Device, dtype: DType) -> bool { + Self::dtype_usage(device, dtype).is_superset(DTypeUsage::general()) + } + + /// Returns the [DTypeUsageSet] for the given [DType] on the specified device. + fn dtype_usage(device: &Self::Device, dtype: DType) -> DTypeUsageSet; +} + +/// An error that can happen when syncing a device. +#[derive(Error, Serialize, Deserialize)] +pub enum ExecutionError { + /// A generic error happened during execution. + /// + /// The backtrace and context information should be included in the reason string. + #[error("An error happened during execution\nCaused by:\n {reason}")] + WithContext { + /// The reason of the error. + reason: String, + }, + /// A generic error happened during execution thrown in the Burn project. + /// + /// The full context isn't captured by the string alone. + #[error("An error happened during execution\nCaused by:\n {reason}")] + Generic { + /// The reason of the error. + reason: String, + /// The backtrace. + #[serde(skip)] + backtrace: BackTrace, + }, +} + +impl core::fmt::Debug for ExecutionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{self}")) + } +} + +/// Trait that allows a backend to support autodiff. +pub trait AutodiffBackend: Backend { + /// The inner backend type. + type InnerBackend: Backend; + + /// Gradients type. + type Gradients: Send; + + /// Backward pass. + /// + /// # Arguments + /// + /// * `tensor` - The tensor is the last node of computational graph where the gradients are computed. + /// + /// # Returns + /// + /// The gradients. + fn backward(tensor: FloatTensor) -> Self::Gradients; + + /// Returns the gradients of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to extract the gradients from. + /// + /// # Returns + /// + /// An optional tensor containing the gradient. + fn grad( + tensor: &FloatTensor, + grads: &Self::Gradients, + ) -> Option>; + + /// Pops the gradients of a tensor and returns them. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to pop the gradients from. + /// * `grads` - The gradients. + /// + /// # Returns + /// + /// An optional tensor containing the given gradients. + fn grad_remove( + tensor: &FloatTensor, + grads: &mut Self::Gradients, + ) -> Option>; + + /// Replace the gradients of a tensor with the one provided. + /// + /// If no gradient existed for the provided tensor, register it. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to pop the gradients from. + /// * `grads` - The gradients. + /// * `grad` - The updated grad tensor. + fn grad_replace( + tensor: &FloatTensor, + grads: &mut Self::Gradients, + grad: FloatTensor, + ); + + /// Returns the tensor with inner backend type. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the inner backend tensor for. + /// + /// # Returns + /// + /// The inner backend tensor. + fn inner(tensor: FloatTensor) -> FloatTensor; + + /// Returns the tensor with inner backend type. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the inner backend tensor for. + /// + /// # Returns + /// + /// The inner backend tensor. + fn int_inner(tensor: IntTensor) -> IntTensor; + + /// Returns the tensor with inner backend type. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the inner backend tensor for. + /// + /// # Returns + /// + /// The inner backend tensor. + fn bool_inner(tensor: BoolTensor) -> BoolTensor; + + /// Returns the tensor with inner backend type. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the inner backend tensor for. + /// + /// # Returns + /// + /// The inner backend tensor. + fn q_inner(tensor: QuantizedTensor) -> QuantizedTensor; + + /// Converts the inner backend tensor to the autodiff backend tensor. + /// + /// # Arguments + /// + /// * `tensor` - The inner backend tensor to convert. + /// + /// + /// # Returns + /// + /// The autodiff backend tensor. + fn from_inner(tensor: FloatTensor) -> FloatTensor; + + /// Converts the inner backend tensor to the autodiff backend tensor. + /// + /// # Arguments + /// + /// * `tensor` - The inner backend tensor to convert. + /// + /// + /// # Returns + /// + /// The autodiff backend tensor. + fn int_from_inner(tensor: IntTensor) -> IntTensor; + + /// Converts the inner backend tensor to the autodiff backend tensor. + /// + /// # Arguments + /// + /// * `tensor` - The inner backend tensor to convert. + /// + /// + /// # Returns + /// + /// The autodiff backend tensor. + fn bool_from_inner(tensor: BoolTensor) -> BoolTensor; + + /// Converts the inner backend tensor to the autodiff backend tensor. + /// + /// # Arguments + /// + /// * `tensor` - The inner backend tensor to convert. + /// + /// + /// # Returns + /// + /// The autodiff backend tensor. + fn q_from_inner(tensor: QuantizedTensor) -> QuantizedTensor; +} + +/// Describes how a data type can be used on a given device. +/// +/// A data type may be supported for different classes of operations. Not all +/// data types that appear in hardware or kernel implementations are suitable +/// for general-purpose tensor operations. +#[derive(Debug, EnumSetType)] +pub enum DTypeUsage { + /// The type can be stored in device memory and converted to and from + /// other supported data types. + Storage, + /// The type supports general-purpose arithmetic and common tensor + /// operations (e.g. elementwise ops, reductions, etc.). + Arithmetic, + /// The type is supported by hardware-accelerated execution paths. + /// + /// This typically indicates support for accelerator-backed compute units (e.g., tensor + /// cores executing MMA instructions) for high-performance operations such as matrix + /// multiplication and operations that lower to it. + /// + /// # Notes + /// - A type can be both [`Arithmetic`](DTypeUsage::Arithmetic) and + /// [`Accelerated`](DTypeUsage::Accelerated) if it supports general-purpose operations + /// *and* accelerated paths. + /// - If a type is marked as `Accelerated` but not `Arithmetic`, it is not + /// suitable for general-purpose tensor operations and may only be used + /// in specific accelerated operations. + /// + /// `Accelerated` is a **flag**, not a detailed descriptor. It does not enumerate which + /// operations are accelerated or which accelerator features are available. + Accelerated, +} + +/// A set of [DTypeUsage] representing the total capabilities of a data type on a device. +pub type DTypeUsageSet = EnumSet; + +impl DTypeUsage { + /// Returns the usage set required for general-purpose tensor support. + pub fn general() -> DTypeUsageSet { + DTypeUsage::Storage | DTypeUsage::Arithmetic + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/device.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/device.rs new file mode 100644 index 0000000..c94e29c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/device.rs @@ -0,0 +1,17 @@ +pub use burn_std::device::*; + +/// Device trait for all burn backend devices. +pub trait DeviceOps: Clone + Default + PartialEq + Send + Sync + core::fmt::Debug + Device { + /// Returns the [device id](DeviceId). + fn id(&self) -> DeviceId { + self.to_id() + } + + /// Returns the inner device without autodiff enabled. + /// + /// For most devices this is a no-op that returns `self`. For autodiff-enabled + /// devices, this returns the underlying inner device. + fn inner(&self) -> &Self { + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/mod.rs new file mode 100644 index 0000000..f16fc6d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/mod.rs @@ -0,0 +1,10 @@ +mod base; +mod device; +mod primitive; + +pub use base::*; +pub use device::*; +pub use primitive::*; + +/// Backend operations on tensors. +pub mod ops; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/activation.rs new file mode 100644 index 0000000..e8fb7ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/activation.rs @@ -0,0 +1,279 @@ +use crate::tensor::FloatTensor; +use crate::{Backend, Scalar, TensorMetadata}; +use core::f64::consts::SQRT_2; + +/// Activation function operations. +/// +/// This trait let backend implementations override activation functions for better performance. +pub trait ActivationOps { + /// Applies the LeakyReLU activation function. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `negative_slope` - The negative_slope value that values smaller than 0 are multiplied with. + /// + /// # Returns + /// + /// The output tensor. + fn leaky_relu(tensor: FloatTensor, negative_slope: Scalar) -> FloatTensor { + let mask = B::float_lower_elem(tensor.clone(), 0f32.into()); + let scaled_tensor = B::float_mul_scalar(tensor.clone(), negative_slope); + + // Update the tensor where the values are `< 0` by `tensor * negative_slope`. + B::float_mask_where(tensor, mask, scaled_tensor) + } + + /// Applies the ReLU activation function. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The output tensor. + fn relu(tensor: FloatTensor) -> FloatTensor { + let mask = B::float_lower_equal_elem(tensor.clone(), 0f32.into()); + + B::float_mask_fill(tensor, mask, 0f32.into()) + } + + /// Applies the ReLU activation function backward. + /// + /// # Arguments + /// + /// * `output` - The output tensor. + /// + /// # Returns + /// + /// The gradient. + fn relu_backward(output: FloatTensor, grad: FloatTensor) -> FloatTensor { + let mask = B::float_lower_equal_elem(output, 0f32.into()); + + B::float_mask_fill(grad, mask, 0.into()) + } + + /// Applies the Gelu activation function. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The output tensor. + fn gelu(tensor: FloatTensor) -> FloatTensor { + let x = B::float_div_scalar(tensor.clone(), SQRT_2.into()); + let x = B::float_erf(x); + let x = B::float_add_scalar(x, 1f32.into()); + let x = B::float_mul(tensor, x); + + B::float_div_scalar(x, 2f32.into()) + } + /// Applies the PReLu activation function. + /// # Arguments + /// * `tensor` - The input tensor + /// * `alpha` - The weight tensor + fn prelu(tensor: FloatTensor, alpha: FloatTensor) -> FloatTensor { + let mask = B::float_lower_elem(tensor.clone(), 0f32.into()); + let scaled_tensor = B::float_mul(tensor.clone(), alpha); + B::float_mask_where(tensor, mask, scaled_tensor) + } + + /// Applies the Gelu activation function backward. + /// + /// # Arguments + /// + /// * `x` - The tensor. + /// * `grad` - The gradient. + /// + /// # Returns + /// + /// The output tensor. + fn gelu_backward(x: FloatTensor, grad: FloatTensor) -> FloatTensor { + // Derivative of the approximate gelu implementation based on tanh. + + let constant_1 = 0.0356774; + let constant_2 = 0.797885; + let constant_3 = 0.0535161; + let constant_4 = 0.398942; + + let x3 = B::float_powi_scalar(x.clone(), 3.into()); + + let c1 = B::float_mul_scalar(x3.clone(), constant_1.into()); + let c2 = B::float_mul_scalar(x.clone(), constant_2.into()); + let c3 = B::float_mul_scalar(x3, constant_3.into()); + let c4 = B::float_mul_scalar(x, constant_4.into()); + + let inner1 = B::float_add(c1, c2); + let inner2 = B::float_add(c3, c4); + + let tanh = B::float_tanh(inner1); + + let sech = B::float_powi_scalar(tanh.clone(), 2.into()); + let sech = B::float_neg(sech); + let sech = B::float_add_scalar(sech, 1.into()); + + let y1 = B::float_mul_scalar(tanh, 0.5.into()); + let y2 = B::float_mul(inner2, sech); + let y2 = B::float_add_scalar(y2, 0.5.into()); + let y = B::float_add(y1, y2); + + B::float_mul(y, grad) + } + + /// Applies the Sigmoid activation function. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The output tensor. + fn sigmoid(tensor: FloatTensor) -> FloatTensor { + let dtype = tensor.dtype(); + let tensor_full = B::float_cast(tensor, burn_std::FloatDType::F32); + let tensor_tmp = B::float_exp(B::float_neg(B::float_log(B::float_add_scalar( + B::float_exp(B::float_neg(tensor_full)), + 1.0.into(), + )))); + + B::float_cast(tensor_tmp, dtype.into()) + } + + /// Applies the Sigmoid activation function backward. + /// + /// # Arguments + /// + /// * `output` - The output tensor of the sigmoid function. + /// * `grad` - The gradient. + /// + /// # Returns + /// + /// The output tensor. + fn sigmoid_backward(output: FloatTensor, grad: FloatTensor) -> FloatTensor { + let value = B::float_mul( + output.clone(), + B::float_add_scalar(B::float_neg(output), 1.0.into()), + ); + B::float_mul(value, grad) + } + + /// Applies the hard Sigmoid activation function. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `alpha` - The alpha value that the tensor is multiplied with. + /// * `beta` - The beta value that is added to the tensor + /// + /// # Returns + /// + /// The output tensor. + fn hard_sigmoid(tensor: FloatTensor, alpha: Scalar, beta: Scalar) -> FloatTensor { + let dtype = tensor.dtype(); + let tensor_full = B::float_cast(tensor, burn_std::FloatDType::F32); + + let tensor_tmp = B::float_clamp( + B::float_add_scalar(B::float_mul_scalar(tensor_full, alpha), beta), + 0.0.into(), + 1.0.into(), + ); + + B::float_cast(tensor_tmp, dtype.into()) + } + + /// Applies the LogSigmoid activation function. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The output tensor. + fn log_sigmoid(tensor: FloatTensor) -> FloatTensor { + // To avoid overflow, we use the log-sum-exp trick. + // + // ```ignore + // log(sigmoid(x)) = log(1/(1 + exp(-x))) + // = log(1) - log(1 + exp(-x)) + // = -log(1 + exp(-x)) + // = -log(exp(0) + exp(-x)) + // ``` + // The `exp(t)` of even a moderate-magnitude positive number can be astronomically huge, so we + // subtract the `max(t, 0)` of each value (where `t = -x` in this case). This results in the + // following equivalence: + // ```ignore + // log(sigmoid(x)) = -(max(-x, 0) + log(exp(-max(-x, 0)) + exp(-x - max(-x, 0)))) + // ``` + // + // This extends the range of values for which we obtain accurate results. + + // max(-x, 0) + let tensor_neg = B::float_neg(tensor); + let mask = B::float_lower_elem(tensor_neg.clone(), 0f32.into()); + let max_elem = B::float_mask_fill(tensor_neg.clone(), mask, 0f32.into()); + let max_elem_neg = B::float_neg(max_elem.clone()); + + // z = exp(-max(-x, 0)) + exp(-x - max(-x, 0)) + let z = B::float_add( + B::float_exp(max_elem_neg.clone()), + B::float_exp(B::float_sub(tensor_neg, max_elem.clone())), + ); + + // -max(-x, 0) - log(-z) + B::float_sub(max_elem_neg, B::float_log(z)) + } + + /// Applies the LogSigmoid activation function backward. + /// + /// # Arguments + /// + /// * `x` - The input tensor. + /// * `grad` - The gradient. + /// + /// # Returns + /// + /// The output gradient. + fn log_sigmoid_backward(x: FloatTensor, grad: FloatTensor) -> FloatTensor { + // Derivative of -max(-x, 0) - log(exp(-max(-x, 0)) - exp(-x - max(-x, 0)))) is + // -max_derive - (-max_derive * exp(-max(-x, 0)) + (-1 - max_derive) * exp(-x - max(-x, 0))) / z + // where z = exp(-max(-x, 0)) + exp(-x - max(-x, 0)) + // + // This simplifies to: + // -max_derive - (z-1)/z if x is >= 0 + // -max_derive + (z-1)/z if x is < 0 + + let shape = x.shape(); + let dtype = x.dtype(); + let device = B::float_device(&x); + + // max(-x, 0) + let x_neg = B::float_neg(x); + let mask = B::float_lower_elem(x_neg.clone(), 0f32.into()); // -x < 0 or x >= 0 + let max_elem = B::float_mask_fill(x_neg.clone(), mask.clone(), 0f32.into()); + + // z = exp(-max(-x, 0)) + exp(-x - max(-x, 0)) + let z = B::float_add( + B::float_exp(B::float_neg(max_elem.clone())), + B::float_exp(B::float_sub(x_neg, max_elem)), + ); + + // Derivative of max(-x, 0) is 1 if x < 0 or 0 if x >= 0 + let ones = B::float_ones(shape, &device, dtype.into()); + let max_derive = B::float_mask_fill(ones.clone(), mask.clone(), 0f32.into()); + let sign = B::float_mask_fill(ones.clone(), mask, (-1f32).into()); + + // grad * (max_derive - sign * (1 - (1 / z))) + B::float_mul( + grad, + B::float_sub( + max_derive, + B::float_mul(sign, B::float_sub(ones, B::float_recip(z))), + ), + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/argwhere.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/argwhere.rs new file mode 100644 index 0000000..a2ecc22 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/argwhere.rs @@ -0,0 +1,56 @@ +use crate::tensor::{Device, IntTensor}; +use crate::{Backend, TensorData, element::ElementConversion}; +use alloc::vec::Vec; +use burn_std::Shape; + +/// Compute the indices of the elements that are non-zero, grouped by element. +/// +/// # Arguments +/// +/// * `data` - The input tensor data. +/// +/// # Returns +/// +/// A 2D tensor containing the indices of all non-zero elements of the given tensor. +/// Each row contains the indices of a non-zero element. +/// +/// # Remarks +/// +/// This is a fallback solution that used only when the backend doesn't have the corresponding implementation. +/// Ideally, it is supposed to be implemented by the backend and the backend implementation will be resolved +/// by static dispatch. It is not designed for direct usage by users, and not recommended to import +/// or use this function directly. +pub fn argwhere_data(data: TensorData, device: &Device) -> IntTensor { + let dims = &data.shape; + let ndims = dims.len(); + let count_nonzero = data.iter::().filter(|&v| v).count(); + + /// Converts a flat index into a vector of indices for the specified tensor shape + fn unravel_index(index: usize, shape: &[usize]) -> Vec { + shape + .iter() + .rev() + .scan(index, |i, size| { + let dim_idx = *i % size; + *i /= size; + Some((dim_idx as i64).elem()) + }) + .collect::>() + .into_iter() + .rev() + .collect() + } + + let indices = data + .iter::() + .enumerate() + .filter_map(|(index, v)| if v { Some(index) } else { None }) + .map(|index| unravel_index::(index, dims)) + .collect::>() + .concat(); + + B::int_from_data( + TensorData::new(indices, Shape::new([count_nonzero, ndims])), + device, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/bool_tensor.rs new file mode 100644 index 0000000..514ddb1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/bool_tensor.rs @@ -0,0 +1,563 @@ +use super::{ + argwhere::argwhere_data, cat::cat_with_slice_assign, repeat_dim::repeat_with_slice_assign, +}; +use crate::tensor::{Bool, BoolTensor, Device, FloatTensor, IntTensor}; +use crate::{Backend, TensorData, TensorMetadata}; +use crate::{ExecutionError, Scalar}; +use alloc::vec::Vec; +use burn_std::{Shape, Slice}; +use core::future::Future; + +/// Bool Tensor API for basic operations, see +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// for documentation on each function. +pub trait BoolTensorOps { + /// Creates a new bool tensor. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The boolean tensor with the given shape. + fn bool_empty(shape: Shape, device: &Device) -> BoolTensor; + + /// Creates a new bool tensor filled false. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The boolean tensor filled with false. + fn bool_zeros(shape: Shape, device: &Device) -> BoolTensor; + + /// Creates a new bool tensor filled true. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The boolean tensor filled with true. + fn bool_ones(shape: Shape, device: &Device) -> BoolTensor; + + /// Converts the tensor to a data structure. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The data structure with the tensor's data. + fn bool_into_data( + tensor: BoolTensor, + ) -> impl Future> + Send; + + /// Creates a tensor from the data structure. + /// + /// # Arguments + /// + /// * `data` - The data structure. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the data. + fn bool_from_data(data: TensorData, device: &Device) -> BoolTensor; + + /// Converts bool tensor to int tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The int tensor with the same data as the bool tensor. + fn bool_into_int(tensor: BoolTensor) -> IntTensor; + + /// Converts bool tensor to float tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The float tensor with the same data as the bool tensor. + fn bool_into_float(tensor: BoolTensor) -> FloatTensor; + + /// Gets the device of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The device of the tensor. + fn bool_device(tensor: &BoolTensor) -> Device; + + /// Moves the tensor to the device. + fn bool_to_device(tensor: BoolTensor, device: &Device) -> BoolTensor; + + /// Reshapes the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `shape` - The new shape. + /// + /// # Returns + /// + /// The tensor with the new shape. + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor; + + /// Gets the values from the tensor for the given ranges. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `slices` - The slices specifying ranges and steps for each dimension. + /// + /// # Returns + /// + /// The tensor with the values for the given slices. + /// + /// # Note + /// + /// Empty slices (where start >= end) are handled at the high-level tensor API and will not + /// be passed to this method. Backend implementations do not need to handle empty slices. + fn bool_slice(tensor: BoolTensor, slices: &[Slice]) -> BoolTensor; + + /// Sets the values in the tensor for the given ranges. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `ranges` - The ranges to set the values for. + /// * `value` - The values to set. + /// + /// # Returns + /// + /// The tensor with the values set for the given ranges. + /// + /// # Note + /// + /// Empty slice assignments (where any slice range produces 0 elements) are handled at the + /// high-level tensor API and will not be passed to this method. Backend implementations do + /// not need to handle empty slice assignments. + fn bool_slice_assign( + tensor: BoolTensor, + slices: &[Slice], + value: BoolTensor, + ) -> BoolTensor; + + /// Fills the tensor with values from the value tensor if the mask is true at the given + /// indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `mask` - The mask. + /// * `value` - The value tensor. + /// + /// # Returns + /// + /// The tensor with the values filled. + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor; + + /// Fills the tensor with the given value if the mask is true at the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `mask` - The mask. + /// * `value` - The value. + /// + /// # Returns + /// + /// The tensor with the values filled. + fn bool_mask_fill(tensor: BoolTensor, mask: BoolTensor, value: Scalar) -> BoolTensor; + + /// Gather elements from the tensor at the given indices. + /// + /// # Arguments + /// + /// * `dim` - The dimension to gather from. + /// * `tensor` - The tensor. + /// * `indices` - The indices. + fn bool_gather(dim: usize, tensor: BoolTensor, indices: IntTensor) -> BoolTensor; + + /// Scatter a given value to the tensor at the given indices using boolean or reduction. + /// + /// # Arguments + /// + /// * `dim` - The dimension to scatter to. + /// * `tensor` - The tensor. + /// * `indices` - The indices. + /// * `value` - The value. + /// + /// # Returns + /// + /// The tensor with the values scattered. + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor; + + /// Select tensor elements along the given dimension corresponding to the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices of the elements to select. + /// + /// # Returns + /// + /// The tensor with the selected elements. + fn bool_select(tensor: BoolTensor, dim: usize, indices: IntTensor) -> BoolTensor { + // Default implementation: convert to int, select, then convert back to bool + let int_tensor = B::bool_into_int(tensor); + let selected = B::int_select(int_tensor, dim, indices); + B::int_equal_elem(selected, 1.into()) + } + + /// Assign the selected elements along the given dimension corresponding to the given indices + /// to the given value using sum reduction. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to assign the values to. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices of the elements to assign. + /// * `value` - The values to assign. + /// + /// # Returns + /// + /// The tensor with the assigned values. + fn bool_select_or( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + // Default implementation: convert to int, select_assign, then convert back to bool + let int_tensor = B::bool_into_int(tensor); + let int_values = B::bool_into_int(value); + let assigned = B::int_select_add(int_tensor, dim, indices, int_values); + // After select_assign with sum reduction, any non-zero value should be true + B::int_greater_elem(assigned, 0.into()) + } + + /// Repeats one dimension of the tensor a given number of times along that dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension to repeat. + /// * `times` - The number of times to repeat the dimension. + /// + /// # Returns + /// + /// The tensor with the dimension repeated. + fn bool_repeat_dim(tensor: BoolTensor, dim: usize, times: usize) -> BoolTensor { + repeat_with_slice_assign::(tensor, dim, times) + } + + /// Concatenates the tensors along the given dimension. + /// + /// # Arguments + /// + /// * `tensors` - The tensors to concatenate. + /// * `dim` - The dimension to concatenate along. + /// + /// # Returns + /// + /// The tensor with the tensors concatenated along the given dimension. + /// + /// # Note + /// + /// Empty tensors (where the concatenation dimension has size 0) are filtered out at the + /// high-level tensor API and will not be passed to this method. Backend implementations do + /// not need to handle empty tensors. + fn bool_cat(tensors: Vec>, dim: usize) -> BoolTensor { + cat_with_slice_assign::(tensors, dim) + } + + /// Equates the two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor with the result of the equate. + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor; + + /// Element-wise non-equality comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor with the result of the comparison. + fn bool_not_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + let equal_tensor = B::bool_equal(lhs, rhs); + B::bool_not(equal_tensor) + } + + /// Element-wise equality comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor; + + /// Element-wise non-equality comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn bool_not_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + let equal_tensor = B::bool_equal_elem(lhs, rhs); + B::bool_not(equal_tensor) + } + + /// Inverses boolean values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The tensor with the result of the negation. + fn bool_not(tensor: BoolTensor) -> BoolTensor; + + /// Executes the logical and (`&&`) operation on two boolean tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor with the result of the logical and. + fn bool_and(tensor: BoolTensor, rhs: BoolTensor) -> BoolTensor; + + /// Executes the logical or (`||`) operation on two boolean tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor with the result of the logical or. + fn bool_or(tensor: BoolTensor, rhs: BoolTensor) -> BoolTensor; + + /// Element-wise exclusive or. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor with the result of the comparison. + fn bool_xor(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + Self::bool_not_equal(lhs, rhs) + } + + /// Transposes a bool tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to transpose. + /// + /// # Returns + /// + /// The transposed tensor. + fn bool_transpose(tensor: BoolTensor) -> BoolTensor { + let ndims = tensor.shape().num_dims(); + Self::bool_swap_dims(tensor, ndims - 2, ndims - 1) + } + + /// Swaps two dimensions of a bool tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to swap the dimensions of. + /// * `dim1` - The first dimension to swap. + /// * `dim2` - The second dimension to swap. + /// + /// # Returns + /// + /// The tensor with the dimensions swapped. + fn bool_swap_dims(tensor: BoolTensor, dim1: usize, dim2: usize) -> BoolTensor; + + /// Permutes the dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to permute the dimensions of. + /// * `axes` - The new order of the dimensions. + /// # Returns + /// + /// The tensor with the dimensions permuted. + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor; + + /// Reverse the order of elements in a tensor along the given axes. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to reverse. + /// * `axes` - The axes to reverse. + /// + /// The tensor with the elements reversed. + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor; + + /// Tests if any element in the boolean `tensor` evaluates to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with a single element, True if any element in the tensor is True, False otherwise. + fn bool_any(tensor: BoolTensor) -> BoolTensor { + let sum = B::int_sum(B::bool_into_int(tensor)); + B::int_greater_elem(sum, 0.into()) + } + + /// Tests if any element in the boolean `tensor` evaluates to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if any element along this dim in the input + /// evaluates to True, False otherwise. + fn bool_any_dim(tensor: BoolTensor, dim: usize) -> BoolTensor { + let sum = B::int_sum_dim(B::bool_into_int(tensor), dim); + B::int_greater_elem(sum, 0.into()) + } + + /// Tests if all elements in the boolean `tensor` evaluate to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with a single element, True if all elements in the input tensor + /// evaluate to True, False otherwise. + fn bool_all(tensor: BoolTensor) -> BoolTensor { + let num_elems = tensor.shape().num_elements() as i64; + let sum = B::int_sum(B::bool_into_int(tensor)); + B::int_equal_elem(sum, num_elems.into()) + } + + /// Tests if all elements in the boolean `tensor` evaluate to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if all elements along this dim in the input + /// evaluates to True, False otherwise. + fn bool_all_dim(tensor: BoolTensor, dim: usize) -> BoolTensor { + let num_elems = tensor.shape()[dim] as i64; + let sum = B::int_sum_dim(B::bool_into_int(tensor), dim); + B::int_equal_elem(sum, num_elems.into()) + } + + /// Compute the indices of the elements that are non-zero, grouped by element. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A 2D tensor containing the indices of all non-zero elements of the given tensor. + /// Each row contains the indices of a non-zero element. + fn bool_argwhere(tensor: BoolTensor) -> impl Future> + 'static + Send { + async { + // Size of each output tensor is variable (= number of nonzero elements in the tensor). + // Reading the data to count the number of truth values might cause sync but is required. + let device = B::bool_device(&tensor); + let data = B::bool_into_data(tensor) + .await + .expect("Can read the data without error"); + argwhere_data::(data, &device) + } + } + + /// Broadcasts the bool `tensor` to the given `shape`. + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor; + + /// Unfold windows along a dimension. + /// + /// Returns a view of the tensor with all complete windows of size `size` in dimension `dim`; + /// where windows are advanced by `step` at each index. + /// + /// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor to unfold; of shape ``[pre=..., dim shape, post=...]`` + /// * `dim` - the selected dim. + /// * `size` - the size of each unfolded window. + /// * `step` - the step between each window. + /// + /// # Returns + /// + /// A tensor view with shape ``[pre=..., windows, size, post=...]``. + fn bool_unfold(tensor: BoolTensor, dim: usize, size: usize, step: usize) -> BoolTensor; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/cat.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/cat.rs new file mode 100644 index 0000000..fb0906d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/cat.rs @@ -0,0 +1,40 @@ +use crate::{ + Backend, TensorMetadata, + tensor::{BasicOps, TensorKind}, +}; +use alloc::vec::Vec; +use burn_std::Slice; + +pub(crate) fn cat_with_slice_assign + BasicOps>( + tensors: Vec, + dim: usize, +) -> K::Primitive { + let first_tensor = tensors.first().expect("Tensors should not be empty"); + let mut shape = first_tensor.shape(); + let device = K::device(first_tensor); + let dtype = first_tensor.dtype(); + + let output_dim_length: usize = tensors.iter().map(|tensor| tensor.shape()[dim]).sum(); + shape[dim] = output_dim_length; + + let mut tensor_output = K::empty(shape.clone(), &device, dtype); + + let indices_select_all = shape.iter().map(|d| 0..*d).collect::>(); + + let mut output_index = 0; + for tensor in tensors { + let mut indices = indices_select_all.clone(); + let tensor_dim_length = tensor.shape()[dim]; + indices[dim] = output_index..output_index + tensor_dim_length; + output_index += tensor_dim_length; + + // Convert ranges to Slice + let slices: Vec = indices + .iter() + .map(|r| Slice::new(r.start as isize, Some(r.end as isize), 1)) + .collect(); + tensor_output = K::slice_assign(tensor_output, &slices, tensor); + } + + tensor_output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/int_tensor.rs new file mode 100644 index 0000000..db0e550 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/int_tensor.rs @@ -0,0 +1,1372 @@ +use super::cat::cat_with_slice_assign; +use super::repeat_dim::repeat_with_slice_assign; +use super::sort::{argsort, sort, sort_with_indices}; +use crate::tensor::{BoolTensor, Device, FloatTensor, Int, IntElem, IntTensor}; +use crate::{Backend, Distribution, TensorData, TensorMetadata, element::ElementConversion}; +use crate::{ExecutionError, Scalar}; +use alloc::vec::Vec; +use burn_std::{IntDType, Shape, Slice}; +use core::ops::Range; + +/// Int Tensor API for basic and numeric operations, see +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// for documentation on each function. +pub trait IntTensorOps { + /// Creates a new int tensor. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The integer tensor with the given shape. + fn int_empty(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor; + + /// Converts the tensor to a data structure. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The data structure with the tensor's data. + fn int_into_data( + tensor: IntTensor, + ) -> impl Future> + Send; + + /// Creates a tensor from the data structure. + /// + /// # Arguments + /// + /// * `data` - The data structure. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the data. + fn int_from_data(data: TensorData, device: &Device) -> IntTensor; + + /// Gets the device of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The device of the tensor. + fn int_device(tensor: &IntTensor) -> Device; + + /// Moves the tensor to the given device. + fn int_to_device(tensor: IntTensor, device: &Device) -> IntTensor; + + /// Reshapes the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `shape` - The new shape. + /// + /// # Returns + /// + /// The tensor with the new shape. + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor; + + /// Gets the element at the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `slices` - The slices specifying ranges and steps for each dimension. + /// + /// # Returns + /// + /// The elements at the given indices. + /// + /// # Note + /// + /// Empty slices (where start >= end) are handled at the high-level tensor API and will not + /// be passed to this method. Backend implementations do not need to handle empty slices. + fn int_slice(tensor: IntTensor, slices: &[Slice]) -> IntTensor; + + /// Sets the values in the tensor for the given ranges. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `ranges` - The ranges to set the values for. + /// + /// # Returns + /// + /// The tensor with the values set for the given ranges. + /// + /// # Note + /// + /// Empty slice assignments (where any slice range produces 0 elements) are handled at the + /// high-level tensor API and will not be passed to this method. Backend implementations do + /// not need to handle empty slice assignments. + fn int_slice_assign( + tensor: IntTensor, + slices: &[Slice], + value: IntTensor, + ) -> IntTensor; + + /// Converts int tensor to float tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The int tensor with the same data as the float tensor. + fn int_into_float(tensor: IntTensor) -> FloatTensor; + + /// Fills the tensor with values from the value tensor if the mask is true at the given + /// indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `mask` - The mask. + /// * `value` - The value tensor. + /// + /// # Returns + /// + /// The tensor with the values filled. + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + value: IntTensor, + ) -> IntTensor; + + /// Fills the tensor with the given value if the mask is true at the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `mask` - The mask. + /// * `value` - The value. + /// + /// # Returns + /// + /// The tensor with the values filled. + fn int_mask_fill(tensor: IntTensor, mask: BoolTensor, value: Scalar) -> IntTensor; + + /// Gather elements from the tensor at the given indices. + /// + /// # Arguments + /// + /// * `dim` - The dimension to gather from. + /// * `tensor` - The tensor. + /// * `indices` - The indices. + fn int_gather(dim: usize, tensor: IntTensor, indices: IntTensor) -> IntTensor; + + /// Scatter a given value to the tensor at the given indices using sum reduction. + /// + /// # Arguments + /// + /// * `dim` - The dimension to scatter to. + /// * `tensor` - The tensor. + /// * `indices` - The indices. + /// * `value` - The value. + /// + /// # Returns + /// + /// The tensor with the values scattered. + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor; + + /// Select tensor elements along the given dimension corresponding to the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices. + /// + /// # Returns + /// + /// The tensor with the selected elements. + fn int_select(tensor: IntTensor, dim: usize, indices: IntTensor) -> IntTensor; + + /// Assign the selected elements along the given dimension corresponding to the given indices + /// to the given value using sum reduction. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices. + /// * `value` - The value. + /// + /// # Returns + /// + /// The tensor with the selected elements assigned to the given value. + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor; + + /// Repeats the tensor along the given dimension the given number of times. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension to repeat. + /// * `times` - The number of times to repeat. + /// + /// # Returns + /// + /// The tensor with the given dimension repeated the given number of times. + fn int_repeat_dim(tensor: IntTensor, dim: usize, times: usize) -> IntTensor { + repeat_with_slice_assign::(tensor, dim, times) + } + + /// Concatenates the given tensors along the given dimension. + /// + /// # Arguments + /// + /// * `tensors` - The tensors. + /// * `dim` - The dimension to concatenate along. + /// + /// # Returns + /// + /// The concatenated tensor. + /// + /// # Note + /// + /// Empty tensors (where the concatenation dimension has size 0) are filtered out at the + /// high-level tensor API and will not be passed to this method. Backend implementations do + /// not need to handle empty tensors. + fn int_cat(tensors: Vec>, dim: usize) -> IntTensor { + cat_with_slice_assign::(tensors, dim) + } + + /// Element-wise equality comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor; + + /// Element-wise non-equality comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_not_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let equal_tensor = B::int_equal(lhs, rhs); + B::bool_not(equal_tensor) + } + + /// Element-wise equality comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor; + + /// Element-wise non-equality comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_not_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let equal_tensor = B::int_equal_elem(lhs, rhs); + B::bool_not(equal_tensor) + } + + /// Element-wise greater than comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor; + + /// Element-wise greater than comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor; + + /// Element-wise greater than or equal comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor; + + /// Element-wise greater than or equal comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor; + + /// Element-wise less than comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor; + + /// Element-wise less than comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor; + + /// Element-wise less than or equal comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor; + + /// Element-wise less than or equal comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The boolean tensor with the result of the comparison. + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor; + + // ==== NUMERIC ==== // + + /// Element-wise addition. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of the addition. + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Element-wise addition with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of the addition. + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Element-wise power with a IntTensor. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side IntTensor. + /// * `rhs` - The right-hand side IntTensor. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the power of the elements of `rhs`. + fn int_powi(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + B::float_into_int(B::float_powi(B::int_into_float(lhs), rhs)) + } + + /// Element-wise power with a floatTensor. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side floatTensor. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. Result is an IntTensor. + fn int_powf(lhs: IntTensor, rhs: FloatTensor) -> IntTensor { + B::float_into_int(B::float_powf(B::int_into_float(lhs), rhs)) + } + + /// Element-wise power with a scalar. + /// + /// # Backend Implementors Note + /// + /// A number of common exponent cases can be implemented with operations + /// which are much cheaper than generic exponentiation. + /// + /// This (`Backend` impl overridable) operation handles generic optimizations + /// for several common integer exponent cases; and then dispatches to + /// the (`Backend` impl overridable) [`Self::int_powi_scalar_impl`] + /// operation to handle the generic case. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. + fn int_powi_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let exp = rhs.elem::(); + match exp { + 0 => Self::int_ones(lhs.shape(), &B::int_device(&lhs), lhs.dtype().into()), + 1 => lhs, + 2 => Self::int_mul(lhs.clone(), lhs), + _ => Self::int_powi_scalar_impl(lhs, rhs), + } + } + + /// Element-wise power with a scalar. + /// + /// # Backend Implementors Note + /// + /// This is the generic implementation of integer exponentiation + /// called by [`Self::int_powi_scalar`] in the fallback case. + /// + /// By default, this performs a relatively expensive conversion to float, + /// exponentiation in float, and conversion back to int. + /// This reduces the minimal operation set for `Backend`s, + /// at the cost of performance. + /// + /// This is a good target for specialized optimizations in `Backend` implementations. + /// + /// As a general rule, this should not be called directly. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. + fn int_powi_scalar_impl(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::float_into_int(B::float_powi_scalar_impl(B::int_into_float(lhs), rhs)) + } + + /// Element-wise power with a floatTensor. + /// + /// Handles a number of special cases, then calls [`Self::int_powf_scalar_impl`]. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. Result is an IntTensor. + fn int_powf_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + // TODO: remove int powf which has weird semantics + if let Some(exp) = rhs.try_as_integer() { + Self::int_powi_scalar(lhs, exp) + } else { + Self::int_powf_scalar_impl(lhs, rhs) + } + } + + /// Element-wise power with a floatTensor. + /// + /// Fallback handler for [`Self::int_powf_scalar`]. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. Result is an IntTensor. + fn int_powf_scalar_impl(lhs: IntTensor, rhs: Scalar) -> IntTensor { + B::float_into_int(B::float_powf_scalar_impl(B::int_into_float(lhs), rhs)) + } + + /// Clamps a tensor under a minimum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn int_clamp_min(tensor: IntTensor, min: Scalar) -> IntTensor { + let mask = Self::int_lower_elem(tensor.clone(), min); + Self::int_mask_fill(tensor, mask, min) + } + + /// Clamps a tensor over a maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn int_clamp_max(tensor: IntTensor, max: Scalar) -> IntTensor { + let mask = Self::int_greater_elem(tensor.clone(), max); + Self::int_mask_fill(tensor, mask, max) + } + + /// Clamps a tensor between a minimum and maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn int_clamp(tensor: IntTensor, min: Scalar, max: Scalar) -> IntTensor { + Self::int_clamp_min(Self::int_clamp_max(tensor, max), min) + } + + /// Element-wise subtraction. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of the subtraction. + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Element-wise subtraction with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of the subtraction. + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Element-wise multiplication. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of the multiplication. + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Element-wise multiplication with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of the multiplication. + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Element-wise division. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of the division. + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Element-wise division with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of the division. + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Element-wise modulus. + /// + /// # Arguments + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of applying the modulus of the scalar to the tensor. + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Element-wise modulus with a scalar. + /// + /// # Arguments + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of applying the modulus of the scalar to the tensor. + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Multiplies two tensors together using matrix multiplication. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of multiplying the two tensors together using matrix multiplication. + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Element-wise negation. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to negate. + /// + /// # Returns + /// + /// The negated tensor. + fn int_neg(tensor: IntTensor) -> IntTensor { + Self::int_mul_scalar(tensor, (-1).into()) + } + + /// Creates a tensor of zeros. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor of zeros. + fn int_zeros(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + Self::int_from_data(TensorData::full_dtype(shape, 0, dtype.into()), device) + } + + /// Creates a tensor of ones. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor of ones. + fn int_ones(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + Self::int_from_data(TensorData::full_dtype(shape, 1, dtype.into()), device) + } + + /// Creates a tensor filled with given value. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `fill_value` - The value with which to fill the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor filled with given value + fn int_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: IntDType, + ) -> IntTensor { + Self::int_from_data( + TensorData::full_dtype(shape, fill_value, dtype.into()), + device, + ) + } + + /// Sums all elements in the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// + /// # Returns + /// + /// The sum of all elements in the tensor. + fn int_sum(tensor: IntTensor) -> IntTensor; + + /// Sums all elements in the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// * `dim` - The dimension to sum along. + /// + /// # Returns + /// + /// The sum of all elements in the tensor along the dimension. + fn int_sum_dim(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Computes the product of all elements in the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the product of. + /// + /// # Returns + /// + /// The product of all elements in the tensor. + fn int_prod(tensor: IntTensor) -> IntTensor; + + /// Computes the product of all elements in the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the product of. + /// * `dim` - The dimension to compute the product along. + /// + /// # Returns + /// + /// The product of all elements in the tensor along the dimension. + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Computes the mean of all elements in the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the mean of. + /// + /// # Returns + /// + /// The mean of all elements in the tensor. + fn int_mean(tensor: IntTensor) -> IntTensor { + let num_elems = tensor.shape().num_elements() as i64; + B::int_div_scalar(B::int_sum(tensor), num_elems.into()) + } + + /// Computes the mean of all elements in the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the mean of. + /// + /// # Returns + /// + /// The mean of all elements in the tensor along the dimension. + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Computes the cumulative sum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative sum of. + /// * `dim` - The dimension along which to compute the cumulative sum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the cumulative sum + /// of all elements up to and including that position along the dimension. + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Computes the cumulative product of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative product of. + /// * `dim` - The dimension along which to compute the cumulative product. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the cumulative product + /// of all elements up to and including that position along the dimension. + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Computes the cumulative minimum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative minimum of. + /// * `dim` - The dimension along which to compute the cumulative minimum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the minimum + /// of all elements up to and including that position along the dimension. + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Computes the cumulative maximum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative maximum of. + /// * `dim` - The dimension along which to compute the cumulative maximum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the maximum + /// of all elements up to and including that position along the dimension. + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Gets the indices of the maximum elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum indices of. + /// * `dim` - The dimension to get the maximum indices along. + /// + /// # Returns + /// + /// The indices of the maximum elements along the dimension. + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Gets the indices of the minimum elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum indices of. + /// * `dim` - The dimension to get the minimum indices along. + /// + /// # Returns + /// + /// The indices of the minimum elements along the dimension. + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor; + + /// Gets the maximum element in the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum element of. + /// + /// # Returns + /// + /// The maximum element in the tensor. + fn int_max(tensor: IntTensor) -> IntTensor { + let shape = tensor.shape(); + let tensor = B::int_reshape(tensor, Shape::new([shape.num_elements()])); + + B::int_max_dim(tensor, 0) + } + + /// Gets the maximum element in the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum element of. + /// * `dim` - The dimension to get the maximum element along. + /// + /// # Returns + /// + /// The maximum element in the tensor along the dimension. + fn int_max_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let index = B::int_argmax(tensor.clone(), dim); + B::int_gather(dim, tensor, index) + } + + /// Gets the maximum elements and corresponding indices along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements and indices of. + /// * `dim` - The dimension to get the maximum elements and indices along. + /// + /// # Returns + /// + /// The maximum elements and corresponding indices along the dimension. + fn int_max_dim_with_indices(tensor: IntTensor, dim: usize) -> (IntTensor, IntTensor) { + let index = B::int_argmax(tensor.clone(), dim); + let values = B::int_gather(dim, tensor, index.clone()); + + (values, index) + } + + /// Gets the maximum absolute element in the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum element of. + /// + /// # Returns + /// + /// The maximum element in the tensor. + fn int_max_abs(tensor: IntTensor) -> IntTensor { + let shape = tensor.shape(); + let tensor = B::int_reshape(tensor, Shape::new([shape.num_elements()])); + + B::int_max_abs_dim(tensor, 0) + } + + /// Gets the maximum absolute element in the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum element of. + /// * `dim` - The dimension to get the maximum element along. + /// + /// # Returns + /// + /// The maximum element in the tensor along the dimension. + fn int_max_abs_dim(tensor: IntTensor, dim: usize) -> IntTensor { + B::int_max_dim(B::int_abs(tensor), dim) + } + + /// Gets the minimum element in the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum element of. + /// + /// # Returns + /// + /// The minimum element in the tensor. + fn int_min(tensor: IntTensor) -> IntTensor { + let shape = tensor.shape(); + let tensor = B::int_reshape(tensor, Shape::new([shape.num_elements()])); + + B::int_min_dim(tensor, 0) + } + + /// Gets the minimum elements in the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum element of. + /// * `dim` - The dimension to get the minimum element along. + /// + /// # Returns + /// + /// The minimum element in the tensor along the dimension. + fn int_min_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let index = B::int_argmin(tensor.clone(), dim); + B::int_gather(dim, tensor, index) + } + + /// Gets the minimum elements and corresponding indices along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements and indices of. + /// * `dim` - The dimension to get the minimum elements and indices along. + /// + /// # Returns + /// + /// The minimum elements and corresponding indices along the dimension. + fn int_min_dim_with_indices(tensor: IntTensor, dim: usize) -> (IntTensor, IntTensor) { + let indices = B::int_argmin(tensor.clone(), dim); + let values = B::int_gather(dim, tensor, indices.clone()); + + (values, indices) + } + + /// Returns a new tensor with absolute values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take absolute value of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with absolute values. + fn int_abs(tensor: IntTensor) -> IntTensor; + + /// Transposes an int tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to transpose. + /// + /// # Returns + /// + /// The transposed tensor. + fn int_transpose(tensor: IntTensor) -> IntTensor { + let ndims = tensor.shape().num_dims(); + Self::int_swap_dims(tensor, ndims - 2, ndims - 1) + } + + /// Swaps two dimensions of an int tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to swap the dimensions of. + /// * `dim1` - The first dimension to swap. + /// * `dim2` - The second dimension to swap. + /// + /// # Returns + /// + /// The tensor with the dimensions swapped. + fn int_swap_dims(tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor; + + /// Permutes the dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to permute the dimensions of. + /// * `axes` - The new order of the dimensions. + /// # Returns + /// + /// The tensor with the dimensions permuted. + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor; + + /// Reverse the order of elements in a tensor along the given axes. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to reverse. + /// * `axes` - The axes to reverse. + /// + /// The tensor with the elements reversed. + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor; + + /// Creates a new int tensor with random values. + /// + /// # Arguments + /// * `shape` - The shape of the tensor. + /// * `distribution` - The distribution to sample from. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the given shape and random values. + fn int_random(shape: Shape, distribution: Distribution, device: &Device) -> IntTensor; + + /// Creates a new tensor with values from the given range with the given step size. + /// + /// # Arguments + /// + /// * `range` - The range of values. + /// * `step` - The step size. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the given values. + fn int_arange_step(range: Range, step: usize, device: &Device) -> IntTensor { + let value = range + .step_by(step) + .map(|i| i.elem()) + .collect::>>(); + let shape = Shape::new([value.len()]); + let data = TensorData::new(value, shape); + B::int_from_data(data, device) + } + + /// Creates a new tensor with values from the given range. + /// + /// # Arguments + /// + /// * `range` - The range of values. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the given values. + /// + /// # Remarks + /// + /// Uses `arange_step` with a step size of 1 under the hood. + fn int_arange(range: Range, device: &Device) -> IntTensor { + Self::int_arange_step(range, 1, device) + } + + /// Tests if any element in the int `tensor` evaluates to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with a single element, True if any element in the tensor is True, False otherwise. + fn int_any(tensor: IntTensor) -> BoolTensor { + let bool_tensor = B::int_equal_elem(tensor, 0.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::int_sum(B::bool_into_int(bool_tensor)); + B::int_greater_elem(sum, 0.into()) + } + + /// Tests if any element in the int `tensor` evaluates to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if any element along this dim in the input + /// evaluates to True, False otherwise. + fn int_any_dim(tensor: IntTensor, dim: usize) -> BoolTensor { + let bool_tensor = B::int_equal_elem(tensor, 0.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::int_sum_dim(B::bool_into_int(bool_tensor), dim); + B::int_greater_elem(sum, 0.into()) + } + + /// Tests if all elements in the int `tensor` evaluate to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with a single element, True if all elements in the input tensor + /// evaluate to True, False otherwise. + fn int_all(tensor: IntTensor) -> BoolTensor { + let num_elems = tensor.shape().num_elements() as i64; + let bool_tensor = B::int_equal_elem(tensor, 0.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::int_sum(B::bool_into_int(bool_tensor)); + B::int_equal_elem(sum, num_elems.into()) + } + + /// Tests if all elements in the int `tensor` evaluate to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if all elements along this dim in the input + /// evaluates to True, False otherwise. + fn int_all_dim(tensor: IntTensor, dim: usize) -> BoolTensor { + let num_elems = tensor.shape()[dim] as i64; + let bool_tensor = B::int_equal_elem(tensor, 0.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::int_sum_dim(B::bool_into_int(bool_tensor), dim); + B::int_equal_elem(sum, num_elems.into()) + } + + /// Returns the signs of the int `tensor`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to extract the signs from. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` containing the signs of the elements of `tensor`. + fn int_sign(tensor: IntTensor) -> IntTensor { + let dtype = tensor.dtype(); + let zeros = B::int_zeros(tensor.shape(), &B::int_device(&tensor), dtype.into()); + let less_than_zero = B::int_lower_elem(tensor.clone(), 0.into()); + let greater_than_zero = B::int_greater_elem(tensor, 0.into()); + + let mut result = B::int_mask_fill(zeros, less_than_zero, (-1).into()); + result = B::int_mask_fill(result, greater_than_zero, 1.into()); + result + } + + /// Broadcasts the int `tensor` to the given `shape`. + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor; + + /// Sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where the elements are sorted by value. + fn int_sort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + sort::(tensor, dim, descending) + } + + /// Sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor and corresponding indices, where + /// the elements are sorted by value and the indices map back to the original input tensor. + fn int_sort_with_indices( + tensor: IntTensor, + dim: usize, + descending: bool, + ) -> (IntTensor, IntTensor) { + sort_with_indices::(tensor, dim, descending) + } + + /// Returns the indices that sort the elements of the input `tensor` by value + /// along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor the indices map back to the original input tensor. + fn int_argsort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + argsort::(tensor, dim, descending) + } + + /// Bitwise AND operation for Int Tensors + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Bitwise AND operation for Int Tensors with a scalar + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Bitwise OR operation for Int Tensors + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Bitwise OR operation for Int Tensors with a scalar + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Bitwise XOR operation for Int Tensors + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Bitwise XOR operation for Int Tensors with a scalar + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Bitwise NOT operation for Int Tensors + fn bitwise_not(tensor: IntTensor) -> IntTensor; + + /// Bitwise left shift operation for Int Tensors + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Bitwise left shift operation for Int Tensors with a scalar + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Bitwise right shift operation for Int Tensors + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor; + + /// Bitwise right shift operation for Int Tensors with a scalar + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor; + + /// Converts a tensor to another integer data type. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to convert. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// A tensor with the same values as `tensor` but in the target integer data type. + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor; + + /// Unfold windows along a dimension. + /// + /// Returns a view of the tensor with all complete windows of size `size` in dimension `dim`; + /// where windows are advanced by `step` at each index. + /// + /// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor to unfold; of shape ``[pre=..., dim shape, post=...]`` + /// * `dim` - the selected dim. + /// * `size` - the size of each unfolded window. + /// * `step` - the step between each window. + /// + /// # Returns + /// + /// A tensor view with shape ``[pre=..., windows, size, post=...]``. + fn int_unfold(tensor: IntTensor, dim: usize, size: usize, step: usize) -> IntTensor; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/mod.rs new file mode 100644 index 0000000..0485608 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/mod.rs @@ -0,0 +1,20 @@ +mod activation; +mod bool_tensor; +mod int_tensor; +mod modules; +mod qtensor; +mod tensor; +mod transaction; + +pub(crate) mod argwhere; +pub(crate) mod cat; +pub(crate) mod repeat_dim; +pub(crate) mod sort; + +pub use activation::*; +pub use bool_tensor::*; +pub use int_tensor::*; +pub use modules::*; +pub use qtensor::*; +pub use tensor::*; +pub use transaction::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/attention.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/attention.rs new file mode 100644 index 0000000..f3ddf4c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/attention.rs @@ -0,0 +1,107 @@ +use core::f32; +#[allow(unused_imports)] +use num_traits::Float as _; + +use burn_std::Shape; + +use crate::{ + Backend, TensorMetadata, + ops::AttentionModuleOptions, + tensor::{BoolTensor, FloatTensor}, +}; + +/// Computes softmax(QKᵗ * scale) · V using separate kernels. +/// Serves as a fallback when FlashAttention is not used. +pub fn attention_fallback( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, +) -> FloatTensor { + if let Some(softcap) = options.softcap { + assert!(softcap > 0.0, "softcap must be positive, got {softcap}"); + } + + // Attention scores: A = QKᵗ * scale + let query_shape = query.shape().dims::<4>(); + let scale = options + .scale + .unwrap_or_else(|| 1.0 / (*query_shape.last().unwrap() as f64).sqrt()); + let transposed_key = B::float_transpose(key); + let qk = B::float_matmul(query, transposed_key); + let attention_scores = B::float_mul_scalar(qk, scale.into()); + + // Softcap: softcap * tanh(scores / softcap) + // Applied to raw logits before any -inf masking, so that tanh does not + // map -inf to a finite value (which would break masking semantics). + let attention_scores = if let Some(softcap) = options.softcap { + let scaled = B::float_div_scalar(attention_scores, softcap.into()); + let tanh = B::float_tanh(scaled); + B::float_mul_scalar(tanh, softcap.into()) + } else { + attention_scores + }; + + // Bool masking + let attention_scores = if let Some(mask) = mask { + B::float_mask_fill(attention_scores, mask, f32::NEG_INFINITY.into()) + } else { + attention_scores + }; + + // Causal masking: mask positions where col > row (future positions) + let attention_scores = if options.is_causal { + let causal_mask = build_causal_mask::(&attention_scores); + B::float_mask_fill(attention_scores, causal_mask, f32::NEG_INFINITY.into()) + } else { + attention_scores + }; + + // Additive bias (ALiBi, relative position biases, etc.) + let attention_scores = if let Some(bias) = attn_bias { + B::float_add(attention_scores, bias) + } else { + attention_scores + }; + + // Softmax: S = softmax(A) + let max_per_dim = B::float_max_dim(attention_scores.clone(), 3); + let minus_max = B::float_sub(attention_scores, max_per_dim); + let numerator = B::float_exp(minus_max); + let sum_exp = B::float_sum_dim(numerator.clone(), 3); + let softmax = B::float_div(numerator, sum_exp); + + // Context: S · V + B::float_matmul(softmax, value) +} + +/// Builds a causal (upper-triangular) bool mask where `true` means "mask this position". +/// Shape: [batch_size, num_heads, seq_q, seq_k], masking positions where col > row. +fn build_causal_mask(attention_scores: &FloatTensor) -> BoolTensor { + let device = B::float_device(attention_scores); + let scores_shape = attention_scores.shape().dims::<4>(); + let [batch_size, num_heads, seq_q, seq_k] = scores_shape; + + // row indices [seq_q, 1] and col indices [1, seq_k] + // Offset col indices so that the causal boundary aligns at the bottom-right corner, + // which handles cross-attention (seq_k > seq_q) correctly. + let offset = seq_k as i64 - seq_q as i64; + let rows = B::int_reshape( + B::int_arange(0..seq_q as i64, &device), + Shape::new([seq_q, 1]), + ); + let cols = B::int_reshape( + B::int_arange(0..seq_k as i64, &device), + Shape::new([1, seq_k]), + ); + + // mask where col > row + offset (upper triangle) + let rows_shifted = B::int_add_scalar(rows, offset.into()); + let mask_2d = B::int_lower(rows_shifted, cols); + + // Reshape to [1, 1, seq_q, seq_k] then expand to [batch_size, num_heads, seq_q, seq_k] + let mask_4d = B::bool_reshape(mask_2d, Shape::new([1, 1, seq_q, seq_k])); + B::bool_expand(mask_4d, Shape::new([batch_size, num_heads, seq_q, seq_k])) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/base.rs new file mode 100644 index 0000000..2147b60 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/base.rs @@ -0,0 +1,1132 @@ +use super::{conv, pool}; +use crate::ops::unfold::unfold4d_using_conv2d; +use crate::tensor::{BoolTensor, FloatTensor, IntTensor}; +use crate::{Backend, ElementConversion, TensorMetadata}; +use burn_std::Shape; +use core::num::NonZeroUsize; + +/// Gradient computed during the backward pass for each tensor used by [conv2d](ModuleOps::conv2d). +#[derive(new)] +pub struct Conv2dBackward { + /// Gradient. + pub x_grad: FloatTensor, + + /// Weights gradient. + pub weights_grad: FloatTensor, + + /// Bias gradient. + pub bias_grad: Option>, +} + +/// Gradient computed during the backward pass for each tensor used by [deform_conv2d](ModuleOps::deform_conv2d). +#[derive(new)] +pub struct DeformConv2dBackward { + /// Gradient. + pub x_grad: FloatTensor, + + /// Offset gradient. + pub offset_grad: FloatTensor, + + /// Weights gradient. + pub weight_grad: FloatTensor, + + /// Mask gradient. + pub mask_grad: Option>, + + /// Bias gradient. + pub bias_grad: Option>, +} + +/// Gradient computed during the backward pass for each tensor used by [conv3d](ModuleOps::conv3d). +#[derive(new)] +pub struct Conv3dBackward { + /// Gradient. + pub x_grad: FloatTensor, + + /// Weights gradient. + pub weights_grad: FloatTensor, + + /// Bias gradient. + pub bias_grad: Option>, +} + +/// Gradient computed during the backward pass for each tensor used by [max_pool1d](ModuleOps::max_pool1d). +#[derive(new)] +pub struct MaxPool1dBackward { + /// Gradient. + pub x_grad: FloatTensor, +} + +/// Results from [max_pool1d](ModuleOps::max_pool1d_with_indices). +#[derive(new)] +pub struct MaxPool1dWithIndices { + /// The output tensor. + pub output: FloatTensor, + + /// The indices tensor. + pub indices: IntTensor, +} + +/// Gradient computed during the backward pass for each tensor used by [max_pool2d](ModuleOps::max_pool2d). +#[derive(new)] +pub struct MaxPool2dBackward { + /// Gradient. + pub x_grad: FloatTensor, +} + +/// Results from [max_pool2d](ModuleOps::max_pool2d_with_indices). +#[derive(new)] +pub struct MaxPool2dWithIndices { + /// The output tensor. + pub output: FloatTensor, + + /// The indices tensor. + pub indices: IntTensor, +} + +/// Check that the parameter value is non-zero. +// NOTE: for now we keep usize but we could refactor the parameters to hold `NonZeroUsize`. +pub(crate) fn check_nonzero(value: usize, msg: &str) -> usize { + NonZeroUsize::new(value).expect(msg); + value +} + +/// Convolution options. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct ConvOptions { + /// Stride (non-zero). + pub stride: [usize; N], + + /// Padding. + pub padding: [usize; N], + + /// Dilation (non-zero). + pub dilation: [usize; N], + + /// Groups (non-zero). + pub groups: usize, +} + +impl ConvOptions { + /// Constructs a new `ConvOptions`. + pub fn new( + stride: [usize; N], + padding: [usize; N], + dilation: [usize; N], + groups: usize, + ) -> Self { + Self { + stride: stride.map(|s| check_nonzero(s, "stride must be non-zero")), + padding, + dilation: dilation.map(|d| check_nonzero(d, "dilation must be non-zero")), + groups: check_nonzero(groups, "groups must be non-zero"), + } + } +} + +/// Convolution options with support for asymmetric padding. +/// +/// Wraps [`ConvOptions`] (which represents symmetric padding for the backend op) +/// and adds optional asymmetric padding. When asymmetric padding is specified, +/// the functional convolution layer applies an explicit pad operation before +/// dispatching to the backend. +/// +/// Implements `From>` for backward compatibility. +#[derive(Debug, Clone)] +pub struct PaddedConvOptions { + /// The underlying convolution options for the backend. + pub options: ConvOptions, + /// Padding at the end of each dimension (e.g., bottom/right for 2D). + /// If `None`, padding is symmetric (same as `options.padding`). + /// If `Some`, specifies different end-padding per dimension. + pub padding_end: Option<[usize; N]>, +} + +impl PaddedConvOptions { + /// Creates options with asymmetric padding. + /// + /// `padding_start` is stored in `ConvOptions::padding`. + /// `padding_end` specifies the end padding per dimension. + pub fn asymmetric( + stride: [usize; N], + padding_start: [usize; N], + padding_end: [usize; N], + dilation: [usize; N], + groups: usize, + ) -> Self { + let options = ConvOptions::new(stride, padding_start, dilation, groups); + if padding_start == padding_end { + Self { + options, + padding_end: None, + } + } else { + Self { + options, + padding_end: Some(padding_end), + } + } + } + + /// Returns true if padding is asymmetric. + pub fn is_asymmetric(&self) -> bool { + self.padding_end.is_some() + } +} + +impl From> for PaddedConvOptions { + fn from(options: ConvOptions) -> Self { + Self { + options, + padding_end: None, + } + } +} + +/// Convolution options. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct DeformConvOptions { + /// Stride (non-zero). + pub stride: [usize; N], + + /// Padding. + pub padding: [usize; N], + + /// Dilation (non-zero). + pub dilation: [usize; N], + + /// Weight Groups (non-zero). + pub weight_groups: usize, + + /// Offset Groups (non-zero). + pub offset_groups: usize, +} + +impl DeformConvOptions { + /// Constructs a new `DeformConvOptions`. + pub fn new( + stride: [usize; N], + padding: [usize; N], + dilation: [usize; N], + weight_groups: usize, + offset_groups: usize, + ) -> Self { + Self { + stride: stride.map(|s| check_nonzero(s, "stride must be non-zero")), + padding, + dilation: dilation.map(|d| check_nonzero(d, "dilation must be non-zero")), + weight_groups: check_nonzero(weight_groups, "weight groups must be non-zero"), + offset_groups: check_nonzero(offset_groups, "offset groups must be non-zero"), + } + } +} + +/// Transposed convolution options. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct ConvTransposeOptions { + /// Stride (non-zero). + pub stride: [usize; N], + + /// Padding. + pub padding: [usize; N], + + /// Padding out. + pub padding_out: [usize; N], + + /// Dilation (non-zero). + pub dilation: [usize; N], + + /// Groups (non-zero). + pub groups: usize, +} + +impl ConvTransposeOptions { + /// Constructs a new `ConvTransposeOptions`. + pub fn new( + stride: [usize; N], + padding: [usize; N], + padding_out: [usize; N], + dilation: [usize; N], + groups: usize, + ) -> Self { + Self { + stride: stride.map(|s| check_nonzero(s, "stride must be non-zero")), + padding, + padding_out, + dilation: dilation.map(|d| check_nonzero(d, "dilation must be non-zero")), + groups: check_nonzero(groups, "groups must be non-zero"), + } + } +} + +/// Unfold operation options. +#[derive(Debug, Clone)] +pub struct UnfoldOptions { + /// The number of positions to slide over the input tensor in each dimension. + /// A stride of `[1, 1]` will slide the kernel one pixel at a time. + pub stride: [usize; 2], + + /// The number of zero-padding pixels added to each side of the input tensor in each dimension. + pub padding: [usize; 2], + + /// The spacing between the blocks (patches) in the original input tensor. + pub dilation: [usize; 2], +} + +impl UnfoldOptions { + /// Constructs a new `UnfoldOptions`. + pub fn new(stride: [usize; 2], padding: [usize; 2], dilation: [usize; 2]) -> Self { + Self { + stride: stride.map(|s| check_nonzero(s, "stride must be non-zero")), + padding, + dilation: dilation.map(|d| check_nonzero(d, "dilation must be non-zero")), + } + } +} + +/// Algorithm used for upsampling. +#[derive(new, Debug, Clone, serde::Deserialize, serde::Serialize)] +pub enum InterpolateMode { + /// Nearest-neighbor interpolation. + /// + Nearest, + + /// Bilinear interpolation. + /// + Bilinear, + + /// Bicubic interpolation. + /// + Bicubic, +} + +/// Interpolation options. +#[derive(Debug, Clone)] +pub struct InterpolateOptions { + /// Algorithm used for upsampling. + pub mode: InterpolateMode, + /// If `true`, the input and output tensors are aligned by their corner pixels. + /// If `false`, half-pixel coordinate mapping is used instead. + pub align_corners: bool, +} + +impl InterpolateOptions { + /// Create new interpolate options with the given mode. + /// Defaults to `align_corners = true`. + pub fn new(mode: InterpolateMode) -> Self { + Self { + mode, + align_corners: true, + } + } + + /// Set align_corners. + pub fn with_align_corners(mut self, align_corners: bool) -> Self { + self.align_corners = align_corners; + self + } +} + +/// Padding mode for grid sampling when coordinates are out of bounds. +/// +/// Matches PyTorch's `padding_mode` parameter in `grid_sample`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Deserialize, serde::Serialize)] +pub enum GridSamplePaddingMode { + /// Fill with zeros for out-of-bounds coordinates. + #[default] + Zeros, + /// Clamp coordinates to the border (use nearest edge value). + Border, + /// Reflect coordinates at the boundary. + Reflection, +} + +/// Options for grid sampling operations. +#[derive(Debug, Clone)] +pub struct GridSampleOptions { + /// Interpolation mode (bilinear, nearest, or bicubic). + pub mode: InterpolateMode, + /// Padding mode for out-of-bounds coordinates. + pub padding_mode: GridSamplePaddingMode, + /// If `true`, grid values of -1 and 1 correspond to the corner pixels. + /// If `false`, they correspond to the corner points of the corner pixels + /// (i.e., -1 maps to -0.5 and 1 maps to size - 0.5 in pixel coordinates). + pub align_corners: bool, +} + +impl Default for GridSampleOptions { + fn default() -> Self { + Self { + mode: InterpolateMode::Bilinear, + padding_mode: GridSamplePaddingMode::Zeros, + align_corners: false, + } + } +} + +impl From for GridSampleOptions { + fn from(value: InterpolateMode) -> Self { + GridSampleOptions::new(value) + } +} + +impl GridSampleOptions { + /// Create new grid sample options with the given interpolation mode. + /// + /// Uses default values for padding_mode (Zeros) and align_corners (false). + pub fn new(mode: InterpolateMode) -> Self { + Self { + mode, + ..Default::default() + } + } + + /// Set the padding mode. + pub fn with_padding_mode(mut self, padding_mode: GridSamplePaddingMode) -> Self { + self.padding_mode = padding_mode; + self + } + + /// Set align_corners. + pub fn with_align_corners(mut self, align_corners: bool) -> Self { + self.align_corners = align_corners; + self + } +} + +/// Padding mode for tensor pad operations. +/// +/// Defines how values are filled when padding a tensor beyond its original boundaries. +/// Padding can be applied to any dimension of a tensor. +/// +/// # Modes +/// +/// - [`Constant`](PadMode::Constant): Fill with a specified value (default: 0.0) +/// - [`Reflect`](PadMode::Reflect): Mirror values at boundary, excluding edge (requires padding < dim_size) +/// - [`Edge`](PadMode::Edge): Replicate boundary values +#[derive(Debug, Clone, Copy, PartialEq, serde::Deserialize, serde::Serialize)] +pub enum PadMode { + /// Fill padded regions with a constant value. + /// + /// # Example + /// For tensor `[1, 2, 3]` with padding 2 on the left and value 0: + /// Result: `[0, 0, 1, 2, 3]` + Constant(f32), + + /// Reflect values at the boundary, excluding the edge value. + /// + /// Padding must be less than the dimension size (i.e., `padding < dim_size`). + /// + /// # Example + /// For tensor `[1, 2, 3, 4]` with padding 2 on the left: + /// Result: `[3, 2, 1, 2, 3, 4]` (reflects from index 1, not 0) + Reflect, + + /// Replicate the edge values. + /// + /// # Example + /// For tensor `[1, 2, 3, 4]` with padding 2 on the left: + /// Result: `[1, 1, 1, 2, 3, 4]` + Edge, +} + +impl Default for PadMode { + fn default() -> Self { + PadMode::Constant(0.0) + } +} + +impl From for PadMode { + fn from(value: E) -> Self { + PadMode::Constant(value.elem()) + } +} + +/// Gradient computed during the backward pass for each tensor used by [interpolate](ModuleOps::interpolate). +#[derive(new)] +pub struct InterpolateBackward { + /// Gradient. + pub x_grad: FloatTensor, +} + +/// Options for [attention](ModuleOps::attention). +#[derive(Debug, Clone, Copy, Default, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct AttentionModuleOptions { + /// Custom scale factor applied to QK^T. When `None`, defaults to `1/sqrt(head_dim)`. + pub scale: Option, + + /// Soft capping applied before softmax: `softcap * tanh(scores / softcap)`. + /// Used by Gemma-2 and similar models. Must be positive when set. + pub softcap: Option, + + /// When `true`, applies causal (autoregressive) masking so that each query position + /// can only attend to key positions at or before it. This is more efficient than + /// passing an explicit lower-triangular bool mask because backends can use optimized + /// kernel paths (e.g. flash attention with causal mode). + pub is_causal: bool, +} + +/// Module operations trait. +pub trait ModuleOps { + /// Embedding operation. + /// + /// # Arguments + /// + /// * `weights` - The embedding weights. + /// * `indices` - The indices tensor. + /// + /// # Returns + /// + /// The output tensor. + fn embedding(weights: FloatTensor, indices: IntTensor) -> FloatTensor { + let [batch_size, seq_length] = indices.shape().dims(); + let [_, d_model] = weights.shape().dims(); + + let indices = B::int_reshape(indices, Shape::new([batch_size * seq_length])); + let output = B::float_select(weights, 0, indices); + + B::float_reshape(output, Shape::new([batch_size, seq_length, d_model])) + } + + /// Embedding backward operation. + /// + /// # Arguments + /// + /// * `weights` - The embedding weights. + /// * `output_grad` - The output gradient. + /// * `indices` - The indices tensor. + /// + /// # Returns + /// + /// The gradient. + fn embedding_backward( + weights: FloatTensor, + output_grad: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + let [batch_size, seq_length] = indices.shape().dims(); + let [n_embeddings, d_model] = weights.shape().dims(); + let device = B::float_device(&weights); + let dtype = output_grad.dtype(); + + let indices = B::int_reshape(indices, Shape::new([batch_size * seq_length])); + let output_grad = + B::float_reshape(output_grad, Shape::new([batch_size * seq_length, d_model])); + let grad = B::float_zeros(Shape::new([n_embeddings, d_model]), &device, dtype.into()); + + B::float_select_add(grad, 0, indices, output_grad) + } + /// One dimensional convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, length]`, + /// weight: `[channels_out, channels_in, kernel_size]`, + /// bias: `[channels_out]`, + fn conv1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<1>, + ) -> FloatTensor { + conv::conv1d_from_conv2d::(x, weight, bias, options) + } + /// Backward pass for the [conv1d](ModuleOps::conv1d) operation, returning the gradient for `x`. + fn conv1d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, + ) -> FloatTensor { + conv::conv1d_x_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv1d](ModuleOps::conv1d) operation, returning the gradient for `weight`. + fn conv1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, + ) -> FloatTensor { + conv::conv1d_weight_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv1d](ModuleOps::conv1d) operation, returning the gradient for `bias`. + fn conv1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + conv::conv1d_bias_backward::(x, bias, output_grad) + } + /// Two dimensional convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, height, width]`, + /// weight: `[channels_out, channels_in, kernel_size_1, kernel_size_2]`, + /// bias: `[channels_out]`, + fn conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<2>, + ) -> FloatTensor; + /// Backward pass for the [conv2d](ModuleOps::conv2d) operation, returning the gradient for `x`. + fn conv2d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, + ) -> FloatTensor { + conv::conv2d_x_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv2d](ModuleOps::conv2d) operation, returning the gradient for `weight`. + fn conv2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, + ) -> FloatTensor { + conv::conv2d_weight_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv2d](ModuleOps::conv2d) operation, returning the gradient for `bias`. + fn conv2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + conv::conv2d_bias_backward::(x, bias, output_grad) + } + + /// Two dimensional deformable convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, height, width]`, + /// weight: `[channels_out, channels_in, kernel_size_1, kernel_size_2]`, + /// bias: `[channels_out]`, + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> FloatTensor; + /// Backward pass for the [deform_conv2d](ModuleOps::deform_conv2d) operation. + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: DeformConvOptions<2>, + ) -> DeformConv2dBackward; + + /// Three dimensional convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, depth, height, width]`, + /// weight: `[channels_out, channels_in, kernel_size_1, kernel_size_2, kernel_size_3]`, + /// bias: `[channels_out]`, + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> FloatTensor; + /// Backward pass for the [conv3d](ModuleOps::conv3d) operation, returning the gradient for `x`. + fn conv3d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, + ) -> FloatTensor { + conv::conv3d_x_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv3d](ModuleOps::conv3d) operation, returning the gradient for `weight`. + fn conv3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, + ) -> FloatTensor { + conv::conv3d_weight_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv3d](ModuleOps::conv3d) operation, returning the gradient for `bias`. + fn conv3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + conv::conv3d_bias_backward::(x, bias, output_grad) + } + /// One dimensional transposed convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, length]`, + /// weight: `[channels_in, channels_out, length]`, + /// bias: `[channels_out]`, + fn conv_transpose1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<1>, + ) -> FloatTensor { + conv::conv_transpose1d_from_conv_transpose2d::(x, weight, bias, options) + } + /// Backward pass for the [conv transpose 1d](ModuleOps::conv_transpose1d) operation, returning the gradient for `x`. + fn conv_transpose1d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<1>, + ) -> FloatTensor { + conv::conv_transpose1d_x_backward::(weight, output_grad, options) + } + /// Backward pass for the [conv transpose 1d](ModuleOps::conv_transpose1d) operation, returning the gradient for `weight`. + fn conv_transpose1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<1>, + ) -> FloatTensor { + conv::conv_transpose1d_weight_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv transpose 1d](ModuleOps::conv_transpose1d) operation, returning the gradient for `bias`. + fn conv_transpose1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + conv::conv_transpose1d_bias_backward::(x, bias, output_grad) + } + + /// Two dimensional transposed convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, height, width]`, + /// weight: `[channels_in, channels_out, kernel_size_1, kernel_size_2]`, + /// bias: `[channels_out]`, + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> FloatTensor; + /// Backward pass for the [conv transpose 2d](ModuleOps::conv_transpose2d) operation, returning the gradient for `x`. + fn conv_transpose2d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + conv::conv_transpose2d_x_backward::(weight, output_grad, options) + } + /// Backward pass for the [conv transpose 2d](ModuleOps::conv_transpose2d) operation, returning the gradient for `weight`. + fn conv_transpose2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + conv::conv_transpose2d_weight_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv transpose 2d](ModuleOps::conv_transpose2d) operation, returning the gradient for `bias`. + fn conv_transpose2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + conv::conv_transpose2d_bias_backward::(x, bias, output_grad) + } + + /// Three dimensional transposed convolution. + /// + /// # Shapes + /// + /// x: `[batch_size, channels_in, height, width]`, + /// weight: `[channels_in, channels_out, kernel_size_1, kernel_size_2, kernel_size_3]`, + /// bias: `[channels_out]`, + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> FloatTensor; + /// Backward pass for the [conv transpose 3d](ModuleOps::conv_transpose3d) operation, returning the gradient for `x`. + fn conv_transpose3d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + conv::conv_transpose3d_x_backward::(weight, output_grad, options) + } + /// Backward pass for the [conv transpose 3d](ModuleOps::conv_transpose3d) operation, returning the gradient for `weight`. + fn conv_transpose3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + conv::conv_transpose3d_weight_backward::(x, weight, output_grad, options) + } + /// Backward pass for the [conv transpose 3d](ModuleOps::conv_transpose3d) operation, returning the gradient for `bias`. + fn conv_transpose3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + conv::conv_transpose3d_bias_backward::(x, bias, output_grad) + } + + /// Four-dimensional unfolding. + /// + /// # Shapes + /// + /// * x: ``[batch_size, channels_in, height, width]``, + /// * returns: ``[batch_size, channels_in * kernel_size_1 * kernel_size_2, number of blocks]``, + fn unfold4d( + x: FloatTensor, + kernel_size: [usize; 2], + options: UnfoldOptions, + ) -> FloatTensor { + if options.padding == [0, 0] && options.dilation == [1, 1] { + let blocks = B::float_unfold(x, 2, kernel_size[0], options.stride[0]); + let blocks = B::float_unfold(blocks, 3, kernel_size[1], options.stride[1]); + + // batch, channels, h_blocks, w_blocks, h_kern, w_kern + + let blocks = B::float_permute(blocks, &[0, 1, 4, 5, 2, 3]); + let shape = blocks.shape(); + + // batch, channels, h_kern, w_kern, h_blocks, w_blocks + + B::float_reshape( + blocks, + [ + shape[0], + shape[1] * shape[2] * shape[3], + shape[4] * shape[5], + ] + .into(), + ) + } else { + unfold4d_using_conv2d::(x, kernel_size, options) + } + } + + /// One dimensional avg pooling. + /// + /// # Shapes + /// + /// x: [batch_size, channels, length], + fn avg_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + pool::avg_pool1d_from_2d::( + x, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ) + } + /// Backward pass for the [avg pooling 1d](ModuleOps::avg_pool1d) operation. + fn avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + pool::avg_pool1d_backward_from_2d::( + x, + grad, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ) + } + /// Two dimensional avg pooling. + /// + /// # Shapes + /// + /// x: [batch_size, channels, height, width], + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor; + /// Backward pass for the [avg pooling 2d](ModuleOps::avg_pool2d) operation. + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor; + /// Two dimensional adaptive avg pooling. + /// + /// # Shapes + /// + /// x: [batch_size, channels, height, width], + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor; + /// Backward pass for the [adaptive avg pooling 2d](ModuleOps::adaptive_avg_pool2d) operation. + fn adaptive_avg_pool2d_backward(x: FloatTensor, grad: FloatTensor) -> FloatTensor; + /// One dimensional adaptive avg pooling. + /// + /// # Shapes + /// + /// x: [batch_size, channels, length], + fn adaptive_avg_pool1d(x: FloatTensor, output_size: usize) -> FloatTensor { + pool::adaptive_avg_pool1d_from_2d::(x, output_size) + } + /// Backward pass for the [adaptive avg pooling 1d](ModuleOps::adaptive_avg_pool1d) operation. + fn adaptive_avg_pool1d_backward(x: FloatTensor, grad: FloatTensor) -> FloatTensor { + pool::adaptive_avg_pool1d_backward_from_2d::(x, grad) + } + /// One dimensional max pooling. + /// + /// # Shapes + /// + /// x: [batch_size, channels, length], + fn max_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> FloatTensor { + pool::max_pool1d_from_2d::(x, kernel_size, stride, padding, dilation, ceil_mode) + } + + /// One dimensional max pooling with indices. + /// + /// # Shapes + /// + /// x: [batch_size, channels, height, width], + fn max_pool1d_with_indices( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> MaxPool1dWithIndices { + pool::max_pool1d_with_indices_from_2d::( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + } + /// Backward pass for the [max pooling 1d](ModuleOps::max_pool1d_with_indices) operation. + #[allow(clippy::too_many_arguments)] + fn max_pool1d_with_indices_backward( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool1dBackward { + pool::max_pool1d_with_indices_backward_from_2d::( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + output_grad, + indices, + ) + } + + /// Two dimensional max pooling. + /// + /// # Shapes + /// + /// x: [batch_size, channels, height, width], + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor; + + /// Two dimensional max pooling with indices. + /// + /// # Shapes + /// + /// x: [batch_size, channels, height, width], + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices; + /// Backward pass for the [max pooling 2d](ModuleOps::max_pool2d_with_indices) operation. + #[allow(clippy::too_many_arguments)] + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool2dBackward; + + /// Down/up samples the input. + /// + /// # Shapes + /// + /// x: `[batch_size, channels, height, width]`, + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor; + + /// Backward pass for the [interpolate](ModuleOps::interpolate) operation. + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor; + + /// Computes scaled dot-product attention: softmax(QKᵗ * scale) · V, + /// where scale defaults to 1/sqrt(head_dim). Optionally applies masking, + /// additive bias, causal masking, and softcap to the attention scores. + /// + /// # Arguments + /// - `query`: Query tensor of shape `[batch_size, num_heads, seq_len_q, head_dim]` + /// - `key`: Key tensor of shape `[batch_size, num_heads, seq_len_k, head_dim]` + /// - `value`: Value tensor of shape `[batch_size, num_heads, seq_len_k, val_dim]` + /// - `mask`: Optional boolean mask of shape `[batch_size, num_heads, seq_len_q, seq_len_k]`, + /// where `true` indicates positions to mask (i.e. set to -inf before softmax). + /// - `attn_bias`: Optional float tensor of shape `[batch_size, num_heads, seq_len_q, seq_len_k]` + /// added to the attention scores before softmax (e.g. ALiBi, relative position biases). + /// - `options`: Additional attention options (custom scale, softcap, causal masking). + /// + /// # Returns + /// A tensor of shape `[batch_size, num_heads, seq_len_q, val_dim]` + /// representing the attended context per head. + /// + /// # Note + /// This implementation does not support dropout and is intended for inference or + /// use cases where dropout is not needed. + fn attention( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, + ) -> FloatTensor; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic = "stride must be non-zero"] + fn conv_options_stride_zero() { + let _opt = ConvOptions::new([0, 1], [0, 0], [1, 1], 1); + } + + #[test] + #[should_panic = "dilation must be non-zero"] + fn conv_options_dilation_zero() { + let _opt = ConvOptions::new([1, 1], [0, 0], [0, 0], 1); + } + + #[test] + #[should_panic = "groups must be non-zero"] + fn conv_options_groups_zero() { + let _opt = ConvOptions::new([1, 1], [0, 0], [1, 1], 0); + } + + #[test] + #[should_panic = "stride must be non-zero"] + fn conv_transpose_options_stride_zero() { + let _opt = ConvTransposeOptions::new([0, 1], [0, 0], [0, 0], [1, 1], 1); + } + + #[test] + #[should_panic = "dilation must be non-zero"] + fn conv_transpose_options_dilation_zero() { + let _opt = ConvTransposeOptions::new([1, 1], [0, 0], [0, 0], [0, 0], 1); + } + + #[test] + #[should_panic = "groups must be non-zero"] + fn conv_transpose_options_groups_zero() { + let _opt = ConvTransposeOptions::new([1, 1], [0, 0], [0, 0], [1, 1], 0); + } + + #[test] + #[should_panic = "stride must be non-zero"] + fn deform_conv_options_stride_zero() { + let _opt = DeformConvOptions::new([0, 1], [0, 0], [1, 1], 1, 1); + } + + #[test] + #[should_panic = "dilation must be non-zero"] + fn deform_conv_options_dilation_zero() { + let _opt = DeformConvOptions::new([1, 1], [0, 0], [0, 0], 1, 1); + } + + #[test] + #[should_panic = "weight groups must be non-zero"] + fn deform_conv_options_weights_groups_zero() { + let _opt = DeformConvOptions::new([1, 1], [0, 0], [1, 1], 0, 1); + } + + #[test] + #[should_panic = "offset groups must be non-zero"] + fn deform_conv_options_offset_groups_zero() { + let _opt = DeformConvOptions::new([1, 1], [0, 0], [1, 1], 1, 0); + } + + #[test] + #[should_panic = "stride must be non-zero"] + fn unfold_options_stride_zero() { + let _opt = UnfoldOptions::new([0, 1], [0, 0], [1, 1]); + } + + #[test] + #[should_panic = "dilation must be non-zero"] + fn unfold_options_dilation_zero() { + let _opt = UnfoldOptions::new([1, 1], [0, 0], [0, 0]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/conv.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/conv.rs new file mode 100644 index 0000000..a4e0666 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/conv.rs @@ -0,0 +1,1408 @@ +#![allow(clippy::single_range_in_vec_init)] +use super::{ConvOptions, ConvTransposeOptions}; +use crate::{Backend, TensorMetadata, tensor::FloatTensor}; +use burn_std::{MetadataError, Shape, Slice}; + +use alloc::{vec, vec::Vec}; +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Calculate the expected output shape `[batch_size, channels_out, spatial_dims, ..]` for a pooling operation. +pub fn calculate_pool_output_shape( + in_shape: &Shape, + kernel_size: &[usize; N], + stride: &[usize; N], + padding: &[usize; N], + dilation: &[usize; N], + ceil_mode: bool, +) -> Result { + if in_shape.rank() != N + 2 { + return Err(MetadataError::RankMismatch { + left: in_shape.rank(), + right: N + 2, + }); + } + + let mut out_shape = in_shape.clone(); + // Spatial dims + for (i, size_i) in out_shape[2..].iter_mut().enumerate() { + *size_i = calculate_pool_output_size( + kernel_size[i], + stride[i], + padding[i], + dilation[i], + *size_i, + ceil_mode, + ); + } + + Ok(out_shape) +} + +/// Calculate the expected output shape `[batch_size, channels_out, spatial_dims, ..]` for a convolution. +pub fn calculate_conv_output_shape( + in_shape: &Shape, + weight_shape: &Shape, + stride: &[usize; N], + padding: &[usize; N], + dilation: &[usize; N], +) -> Result { + if weight_shape.rank() != N + 2 { + return Err(MetadataError::RankMismatch { + left: weight_shape.rank(), + right: N + 2, + }); + } + + if in_shape.rank() != N + 2 { + return Err(MetadataError::RankMismatch { + left: in_shape.rank(), + right: N + 2, + }); + } + + let kernel_size = &weight_shape[2..]; + + let mut out_shape = in_shape.clone(); + // Spatial dims + for (i, size_i) in out_shape[2..].iter_mut().enumerate() { + *size_i = + calculate_conv_output_size(kernel_size[i], stride[i], padding[i], dilation[i], *size_i); + } + // Output channels + out_shape[1] = weight_shape[0]; + + Ok(out_shape) +} + +/// Calculate the expected output shape `[batch_size, channels_out, spatial_dims, ..]` for a transposed convolution. +pub fn calculate_conv_transpose_output_shape( + in_shape: &Shape, + weight_shape: &Shape, + stride: &[usize; N], + padding: &[usize; N], + padding_out: &[usize; N], + dilation: &[usize; N], + groups: usize, +) -> Result { + if weight_shape.rank() != N + 2 { + return Err(MetadataError::RankMismatch { + left: weight_shape.rank(), + right: N + 2, + }); + } + + if in_shape.rank() != N + 2 { + return Err(MetadataError::RankMismatch { + left: in_shape.rank(), + right: N + 2, + }); + } + + let kernel_size = &weight_shape[2..]; + + let mut out_shape = in_shape.clone(); + // Spatial dims + for (i, size_i) in out_shape[2..].iter_mut().enumerate() { + *size_i = calculate_conv_transpose_output_size( + kernel_size[i], + stride[i], + padding[i], + padding_out[i], + dilation[i], + *size_i, + ); + } + // Output channels + out_shape[1] = weight_shape[1] * groups; + + Ok(out_shape) +} + +/// Calculate the expected padding size required when applying a convolution. +pub fn calculate_conv_padding( + kernel_size: usize, + stride: usize, + size_in: usize, + size_out: usize, +) -> usize { + let kernel_size = kernel_size as f32; + let stride = stride as f32; + let size_in = size_in as f32; + let size_out = size_out as f32; + + let padding = stride * (size_out - 1.) - size_in + kernel_size; + let padding = (padding / 2.).ceil(); + + padding as usize +} + +/// Calculate the expected output size when doing a convolution operation. +pub fn calculate_conv_output_size( + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + size_in: usize, +) -> usize { + (size_in + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + 1 +} + +/// Calculate the expected output sizes when doing a convolution operation. +pub fn calculate_conv_output_sizes( + kernel_size: &[usize], + stride: &[usize], + padding: &[usize], + dilation: &[usize], + size_in: &[usize], +) -> Vec { + size_in + .iter() + .enumerate() + .map(|(i, size_in)| { + calculate_conv_output_size(kernel_size[i], stride[i], padding[i], dilation[i], *size_in) + }) + .collect() +} + +/// Calculate the expected output size when doing a transposed convolution operation. +pub fn calculate_conv_transpose_output_size( + kernel_size: usize, + stride: usize, + padding: usize, + padding_out: usize, + dilation: usize, + size_in: usize, +) -> usize { + (size_in - 1) * stride + (dilation * (kernel_size - 1) + 1) + padding_out - 2 * padding +} + +/// Calculate the expected output size when doing a pooling operation. +/// +/// # Arguments +/// +/// * `kernel_size` - Size of the pooling kernel +/// * `stride` - Stride of the pooling operation +/// * `padding` - Padding applied to input +/// * `dilation` - Dilation of the pooling kernel +/// * `size_in` - Input size (height or width) +/// * `ceil_mode` - If true, use ceiling instead of floor for output size calculation. +/// This allows the last pooling window to go out-of-bounds if needed. +pub fn calculate_pool_output_size( + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + size_in: usize, + ceil_mode: bool, +) -> usize { + let numerator = size_in + 2 * padding - dilation * (kernel_size - 1) - 1; + if ceil_mode { + // Ceiling division: (a + b - 1) / b + numerator.div_ceil(stride) + 1 + } else { + // Floor division (default) + numerator / stride + 1 + } +} + +/// Calculate the [1D convolution](crate::ops::ModuleOps::conv1d) backward pass, returning the gradient for `x`. +pub(crate) fn conv1d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, +) -> FloatTensor { + let weight_shape = weight.shape(); + + let [_batch_size, _, length_in] = x.shape().dims(); + let [_batch_size, _channels_out, length_out] = output_grad.shape().dims(); + let [_, _, kernel_size] = weight_shape.dims(); + + let padding_out = calculate_padding_out( + kernel_size, + options.stride[0], + options.padding[0], + options.dilation[0], + length_in, + length_out, + ); + + B::conv_transpose1d( + output_grad, + weight, + None, + ConvTransposeOptions::new( + options.stride, + options.padding, + [padding_out], + options.dilation, + options.groups, + ), + ) +} + +/// Calculate the [1D convolution](crate::ops::ModuleOps::conv1d) backward pass, returning the gradient for `weight`. +pub(crate) fn conv1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, +) -> FloatTensor { + let weight_dtype = weight.dtype(); + let weight_shape = weight.shape(); + let weight_device = B::float_device(&weight); + + match options.groups == 1 { + true => conv1d_weight_grad_no_groups::(x, output_grad, weight_shape, options), + false => conv1d_weight_grad_groups::( + x, + B::float_zeros(weight_shape, &weight_device, weight_dtype.into()), + output_grad, + options, + ), + } +} + +/// Calculate the [1D convolution](crate::ops::ModuleOps::conv1d) backward pass, returning the gradient for `bias`. +pub(crate) fn conv1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, +) -> FloatTensor { + let [batch_size, _, _length_in] = x.shape().dims(); + let [_batch_size, channels_out, length_out] = output_grad.shape().dims(); + + let grad = B::float_swap_dims(output_grad, 0, 1); + let grad = B::float_reshape(grad, Shape::new([channels_out, batch_size * length_out])); + let grad = B::float_sum_dim(grad, 1); + + B::float_reshape(grad, bias.shape()) +} + +/// Calculate the [2D convolution](crate::ops::ModuleOps::conv2d) backward pass, returning the gradient for `x`. +pub(crate) fn conv2d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, +) -> FloatTensor { + let weight_shape = weight.shape(); + + let [_batch_size, _channels_in, height_in, width_in] = x.shape().dims(); + let [_, _, height_out, width_out] = output_grad.shape().dims(); + let [_channels_out, _, kernel_size_1, kernel_size_2] = weight_shape.dims(); + + let padding_1_out = calculate_padding_out( + kernel_size_1, + options.stride[0], + options.padding[0], + options.dilation[0], + height_in, + height_out, + ); + let padding_2_out = calculate_padding_out( + kernel_size_2, + options.stride[1], + options.padding[1], + options.dilation[1], + width_in, + width_out, + ); + + B::conv_transpose2d( + output_grad, + weight, + None, + ConvTransposeOptions::new( + options.stride, + options.padding, + [padding_1_out, padding_2_out], + options.dilation, + options.groups, + ), + ) +} + +/// Calculate the [2D convolution](crate::ops::ModuleOps::conv2d) backward pass, returning the gradient for `weight`. +pub(crate) fn conv2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, +) -> FloatTensor { + let weight_dtype = weight.dtype(); + let weight_shape = weight.shape(); + let weight_device = B::float_device(&weight); + + match options.groups == 1 { + true => conv2d_weight_grad_no_groups::(x, output_grad, weight_shape, options), + false => conv2d_weight_grad_groups::( + x, + B::float_zeros(weight_shape, &weight_device, weight_dtype.into()), + output_grad, + options, + ), + } +} + +/// Calculate the [2D convolution](crate::ops::ModuleOps::conv2d) backward pass, returning the gradient for `bias`. +pub(crate) fn conv2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, +) -> FloatTensor { + let [batch_size, _, _, _] = x.shape().dims(); + let [_, channels_out, height_out, width_out] = output_grad.shape().dims(); + + let grad = B::float_swap_dims(output_grad, 0, 1); + let grad = B::float_reshape( + grad, + Shape::new([channels_out, batch_size * height_out * width_out]), + ); + let grad = B::float_sum_dim(grad, 1); + + B::float_reshape(grad, bias.shape()) +} + +/// Calculate the [3D convolution](crate::ops::ModuleOps::conv3d) backward pass, returning the gradient for `x`. +pub(crate) fn conv3d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, +) -> FloatTensor { + let weight_shape = weight.shape(); + + let [_batch_size, _channels_in, depth_in, height_in, width_in] = x.shape().dims(); + let [_, _, depth_out, height_out, width_out] = output_grad.shape().dims(); + let [ + _channels_out, + _, + kernel_size_1, + kernel_size_2, + kernel_size_3, + ] = weight_shape.dims(); + + let padding_1_out = calculate_padding_out( + kernel_size_1, + options.stride[0], + options.padding[0], + options.dilation[0], + depth_in, + depth_out, + ); + let padding_2_out = calculate_padding_out( + kernel_size_2, + options.stride[1], + options.padding[1], + options.dilation[1], + height_in, + height_out, + ); + let padding_3_out = calculate_padding_out( + kernel_size_3, + options.stride[2], + options.padding[2], + options.dilation[2], + width_in, + width_out, + ); + + B::conv_transpose3d( + output_grad, + weight, + None, + ConvTransposeOptions::new( + options.stride, + options.padding, + [padding_1_out, padding_2_out, padding_3_out], + options.dilation, + options.groups, + ), + ) +} + +/// Calculate the [3D convolution](crate::ops::ModuleOps::conv3d) backward pass, returning the gradient for `weight`. +pub(crate) fn conv3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, +) -> FloatTensor { + let weight_dtype = weight.dtype(); + let weight_shape = weight.shape(); + let weight_device = B::float_device(&weight); + + match options.groups == 1 { + true => conv3d_weight_grad_no_groups::(x, output_grad, weight_shape, options), + false => conv3d_weight_grad_groups::( + x, + B::float_zeros(weight_shape, &weight_device, weight_dtype.into()), + output_grad, + options, + ), + } +} + +/// Calculate the [3D convolution](crate::ops::ModuleOps::conv3d) backward pass, returning the gradient for `bias`. +pub(crate) fn conv3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, +) -> FloatTensor { + let [batch_size, _channels_in, _depth_in, _height_in, _width_in] = x.shape().dims(); + let [_, channels_out, depth_out, height_out, width_out] = output_grad.shape().dims(); + + let grad = B::float_swap_dims(output_grad, 0, 1); + let grad = B::float_reshape( + grad, + Shape::new([ + channels_out, + batch_size * depth_out * height_out * width_out, + ]), + ); + let grad = B::float_sum_dim(grad, 1); + + B::float_reshape(grad, bias.shape()) +} + +/// Calculate the [1D convolution transpose](crate::ops::ModuleOps::conv_transpose1d) backward pass, returning the gradient for `x`. +pub(crate) fn conv_transpose1d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<1>, +) -> FloatTensor { + B::conv1d( + output_grad, + weight, + None, + ConvOptions::new( + options.stride, + options.padding, + options.dilation, + options.groups, + ), + ) +} + +/// Calculate the [1D convolution transpose](crate::ops::ModuleOps::conv_transpose1d) backward pass, returning the gradient for `weight`. +pub(crate) fn conv_transpose1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<1>, +) -> FloatTensor { + let weight_dtype = weight.dtype(); + let weight_shape = weight.shape(); + let weight_device = B::float_device(&weight); + + match options.groups == 1 { + true => conv_transpose1d_weight_grad_no_groups::(x, output_grad, weight_shape, options), + false => conv_transpose1d_weight_grad_groups::( + x, + B::float_zeros(weight_shape, &weight_device, weight_dtype.into()), + output_grad, + options, + ), + } +} + +/// Calculate the [1D convolution transpose](crate::ops::ModuleOps::conv_transpose1d) backward pass, returning the gradient for `bias`. +pub(crate) fn conv_transpose1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, +) -> FloatTensor { + let [batch_size, _channels_in, _] = x.shape().dims(); + let [_, channels_out, length_out] = output_grad.shape().dims(); + + let grad = B::float_swap_dims(output_grad, 0, 1); + let grad = B::float_reshape(grad, Shape::new([channels_out, batch_size * length_out])); + let grad = B::float_sum_dim(grad, 1); + + B::float_reshape(grad, bias.shape()) +} + +/// Calculate the [2D convolution transpose](crate::ops::ModuleOps::conv_transpose2d) backward pass, returning the gradient for `x`. +pub(crate) fn conv_transpose2d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<2>, +) -> FloatTensor { + B::conv2d( + output_grad, + weight, + None, + ConvOptions::new( + options.stride, + options.padding, + options.dilation, + options.groups, + ), + ) +} + +/// Calculate the [2D convolution transpose](crate::ops::ModuleOps::conv_transpose2d) backward pass, returning the gradient for `weight`. +pub(crate) fn conv_transpose2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<2>, +) -> FloatTensor { + let weight_dtype = weight.dtype(); + let weight_shape = weight.shape(); + let weight_device = B::float_device(&weight); + + match options.groups == 1 { + true => conv_transpose2d_weight_grad_no_groups::(x, output_grad, weight_shape, options), + false => conv_transpose2d_weight_grad_groups::( + x, + B::float_zeros(weight_shape, &weight_device, weight_dtype.into()), + output_grad, + options, + ), + } +} + +/// Calculate the [2D convolution transpose](crate::ops::ModuleOps::conv_transpose2d) backward pass, returning the gradient for `bias`. +pub(crate) fn conv_transpose2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, +) -> FloatTensor { + let [batch_size, _channels_in, _, _] = x.shape().dims(); + let [_, channels_out, height_out, width_out] = output_grad.shape().dims(); + + let grad = B::float_swap_dims(output_grad, 0, 1); + let grad = B::float_reshape( + grad, + Shape::new([channels_out, batch_size * height_out * width_out]), + ); + let grad = B::float_sum_dim(grad, 1); + + B::float_reshape(grad, bias.shape()) +} + +/// Calculate the [3D convolution transpose](crate::ops::ModuleOps::conv_transpose3d) backward pass, returning the gradient for `x`. +pub(crate) fn conv_transpose3d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<3>, +) -> FloatTensor { + B::conv3d( + output_grad, + weight, + None, + ConvOptions::new( + options.stride, + options.padding, + options.dilation, + options.groups, + ), + ) +} + +/// Calculate the [3D convolution transpose](crate::ops::ModuleOps::conv_transpose3d) backward pass, returning the gradient for `weight`. +pub(crate) fn conv_transpose3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<3>, +) -> FloatTensor { + let weight_dtype = weight.dtype(); + let weight_shape = weight.shape(); + let weight_device = B::float_device(&weight); + + match options.groups == 1 { + true => conv_transpose3d_weight_grad_no_groups::(x, output_grad, weight_shape, options), + false => conv_transpose3d_weight_grad_groups::( + x, + B::float_zeros(weight_shape, &weight_device, weight_dtype.into()), + output_grad, + options, + ), + } +} + +/// Calculate the [3D convolution transpose](crate::ops::ModuleOps::conv_transpose3d) backward pass, returning the gradient for `bias`. +pub(crate) fn conv_transpose3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, +) -> FloatTensor { + let [batch_size, _channels_in, _, _, _] = x.shape().dims(); + let [_, channels_out, depth_out, height_out, width_out] = output_grad.shape().dims(); + + let grad = B::float_swap_dims(output_grad, 0, 1); + let grad = B::float_reshape( + grad, + Shape::new([ + channels_out, + batch_size * depth_out * height_out * width_out, + ]), + ); + let grad = B::float_sum_dim(grad, 1); + + B::float_reshape(grad, bias.shape()) +} + +/// Execute a 1D convolution using a 2D convolution. +pub(crate) fn conv1d_from_conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<1>, +) -> FloatTensor { + let [channels_out, _channels_in, kernel_size] = weight.shape().dims(); + let [batch_size, channels_in, length_in] = x.shape().dims(); + + let weight = B::float_reshape( + weight, + Shape::new([channels_out, channels_in / options.groups, kernel_size, 1]), + ); + let x = B::float_reshape(x, Shape::new([batch_size, channels_in, length_in, 1])); + + let tensor = B::conv2d( + x, + weight, + bias, + ConvOptions::new( + [options.stride[0], 1], + [options.padding[0], 0], + [options.dilation[0], 1], + options.groups, + ), + ); + let [batch_size, channels_out, height_out, _weight_out] = tensor.shape().dims(); + B::float_reshape(tensor, Shape::from([batch_size, channels_out, height_out])) +} + +/// Execute a 1D transposed convolution using a 2D transposed convolution. +pub(crate) fn conv_transpose1d_from_conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<1>, +) -> FloatTensor { + let [channels_in, channels_out, kernel_size] = weight.shape().dims(); + let [batch_size, _channels_in, length_in] = x.shape().dims(); + + let weight = B::float_reshape( + weight, + Shape::new([channels_in, channels_out, kernel_size, 1]), + ); + let x = B::float_reshape(x, Shape::new([batch_size, channels_in, length_in, 1])); + + let tensor = B::conv_transpose2d( + x, + weight, + bias, + ConvTransposeOptions::new( + [options.stride[0], 1], + [options.padding[0], 0], + [options.padding_out[0], 0], + [options.dilation[0], 1], + options.groups, + ), + ); + let [batch_size, channels_out, height_out, _weight_out] = tensor.shape().dims(); + B::float_reshape(tensor, Shape::from([batch_size, channels_out, height_out])) +} + +fn conv1d_weight_grad_no_groups( + x: FloatTensor, + output_grad: FloatTensor, + weight_shape: Shape, + options: ConvOptions<1>, +) -> FloatTensor { + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + let weight_grad_swapped = B::conv1d( + x_swapped, + output_grad_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + let mut weight_grad = B::float_swap_dims(weight_grad_swapped, 0, 1); + + if weight_grad.shape() != weight_shape { + let slices = vec![ + Slice::from(0..weight_shape[0]), + Slice::from(0..weight_shape[1]), + Slice::from(0..weight_shape[2]), + ]; + weight_grad = B::float_slice(weight_grad, &slices); + } + weight_grad +} + +fn conv2d_weight_grad_no_groups( + x: FloatTensor, + output_grad: FloatTensor, + weight_shape: Shape, + options: ConvOptions<2>, +) -> FloatTensor { + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + let weight_grad_swapped = B::conv2d( + x_swapped, + output_grad_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + let mut weight_grad = B::float_swap_dims(weight_grad_swapped, 0, 1); + + if weight_grad.shape() != weight_shape { + let slices = vec![ + Slice::from(0..weight_shape[0]), + Slice::from(0..weight_shape[1]), + Slice::from(0..weight_shape[2]), + Slice::from(0..weight_shape[3]), + ]; + weight_grad = B::float_slice(weight_grad, &slices); + } + weight_grad +} + +fn conv3d_weight_grad_no_groups( + x: FloatTensor, + output_grad: FloatTensor, + weight_shape: Shape, + options: ConvOptions<3>, +) -> FloatTensor { + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + let weight_grad_swapped = B::conv3d( + x_swapped, + output_grad_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + let mut weight_grad = B::float_swap_dims(weight_grad_swapped, 0, 1); + + if weight_grad.shape() != weight_shape { + let slices = vec![ + Slice::from(0..weight_shape[0]), + Slice::from(0..weight_shape[1]), + Slice::from(0..weight_shape[2]), + Slice::from(0..weight_shape[3]), + Slice::from(0..weight_shape[4]), + ]; + weight_grad = B::float_slice(weight_grad, &slices); + } + weight_grad +} + +fn conv1d_weight_grad_groups( + x: FloatTensor, + mut weight_grad: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, +) -> FloatTensor { + let [channels_out, increment_ci, kernel_size] = weight_grad.shape().dims(); + let increment_co = channels_out / options.groups; + + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let x_slice = vec![Slice::new( + start_idx_ci as isize, + Some(end_idx_ci as isize), + 1, + )]; + let x = B::float_slice(x_swapped.clone(), &x_slice); + let grad_slice = vec![Slice::new( + start_idx_co as isize, + Some(end_idx_co as isize), + 1, + )]; + let grad = B::float_slice(output_grad_swapped.clone(), &grad_slice); + let mut weight_grad_tmp = B::conv1d( + x, + grad, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + weight_grad_tmp = B::float_swap_dims(weight_grad_tmp, 0, 1); + weight_grad = B::float_slice_assign( + weight_grad, + &[ + Slice::from(start_idx_co..end_idx_co), + Slice::from(0..increment_ci), + Slice::from(0..kernel_size), + ], + weight_grad_tmp, + ); + } + + weight_grad +} + +fn conv2d_weight_grad_groups( + x: FloatTensor, + mut weight_grad: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, +) -> FloatTensor { + let [channels_out, increment_ci, kernel_size_1, kernel_size_2] = weight_grad.shape().dims(); + let increment_co = channels_out / options.groups; + + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let x_slice = vec![Slice::new( + start_idx_ci as isize, + Some(end_idx_ci as isize), + 1, + )]; + let x = B::float_slice(x_swapped.clone(), &x_slice); + let grad_slice = vec![Slice::new( + start_idx_co as isize, + Some(end_idx_co as isize), + 1, + )]; + let grad = B::float_slice(output_grad_swapped.clone(), &grad_slice); + let mut weight_grad_tmp = B::conv2d( + x, + grad, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + weight_grad_tmp = B::float_swap_dims(weight_grad_tmp, 0, 1); + let [_, _, kernel_size_1_tmp, kernel_size_2_tmp] = weight_grad_tmp.shape().dims(); + + if kernel_size_1_tmp != kernel_size_1 || kernel_size_2_tmp != kernel_size_2 { + let slices = vec![ + Slice::from(0..increment_co), + Slice::from(0..increment_ci), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + ]; + weight_grad_tmp = B::float_slice(weight_grad_tmp, &slices); + } + + weight_grad = B::float_slice_assign( + weight_grad, + &[ + Slice::from(start_idx_co..end_idx_co), + Slice::from(0..increment_ci), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + ], + weight_grad_tmp, + ); + } + + weight_grad +} + +fn conv3d_weight_grad_groups( + x: FloatTensor, + mut weight_grad: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, +) -> FloatTensor { + let [ + channels_out, + increment_ci, + kernel_size_1, + kernel_size_2, + kernel_size_3, + ] = weight_grad.shape().dims(); + let increment_co = channels_out / options.groups; + + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let x_slice = vec![Slice::new( + start_idx_ci as isize, + Some(end_idx_ci as isize), + 1, + )]; + let x = B::float_slice(x_swapped.clone(), &x_slice); + let grad_slice = vec![Slice::new( + start_idx_co as isize, + Some(end_idx_co as isize), + 1, + )]; + let grad = B::float_slice(output_grad_swapped.clone(), &grad_slice); + let mut weight_grad_tmp = B::conv3d( + x, + grad, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + weight_grad_tmp = B::float_swap_dims(weight_grad_tmp, 0, 1); + let [ + _, + _, + kernel_size_1_tmp, + kernel_size_2_tmp, + kernel_size_3_tmp, + ] = weight_grad_tmp.shape().dims(); + + if kernel_size_1_tmp != kernel_size_1 + || kernel_size_2_tmp != kernel_size_2 + || kernel_size_3_tmp != kernel_size_3 + { + let slices = vec![ + Slice::from(0..increment_co), + Slice::from(0..increment_ci), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + Slice::from(0..kernel_size_3), + ]; + weight_grad_tmp = B::float_slice(weight_grad_tmp, &slices); + } + + weight_grad = B::float_slice_assign( + weight_grad, + &[ + Slice::from(start_idx_co..end_idx_co), + Slice::from(0..increment_ci), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + Slice::from(0..kernel_size_3), + ], + weight_grad_tmp, + ); + } + + weight_grad +} + +fn conv_transpose1d_weight_grad_no_groups( + x: FloatTensor, + output_grad: FloatTensor, + weight_shape: Shape, + options: ConvTransposeOptions<1>, +) -> FloatTensor { + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + let weight_grad_swapped = B::conv1d( + output_grad_swapped, + x_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + let mut weight_grad = B::float_swap_dims(weight_grad_swapped, 0, 1); + + let grad_shape = weight_grad.shape(); + if grad_shape != weight_shape { + let slices = vec![ + Slice::from(0..weight_shape[0]), + Slice::from(0..weight_shape[1]), + Slice::from(0..weight_shape[2]), + ]; + weight_grad = B::float_slice(weight_grad, &slices); + } + weight_grad +} + +fn conv_transpose2d_weight_grad_no_groups( + x: FloatTensor, + output_grad: FloatTensor, + weight_shape: Shape, + options: ConvTransposeOptions<2>, +) -> FloatTensor { + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + let weight_grad_swapped = B::conv2d( + output_grad_swapped, + x_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + let mut weight_grad = B::float_swap_dims(weight_grad_swapped, 0, 1); + + let grad_shape = weight_grad.shape(); + if grad_shape != weight_shape { + let slices = vec![ + Slice::from(0..weight_shape[0]), + Slice::from(0..weight_shape[1]), + Slice::from(0..weight_shape[2]), + Slice::from(0..weight_shape[3]), + ]; + weight_grad = B::float_slice(weight_grad, &slices); + } + weight_grad +} + +fn conv_transpose3d_weight_grad_no_groups( + x: FloatTensor, + output_grad: FloatTensor, + weight_shape: Shape, + options: ConvTransposeOptions<3>, +) -> FloatTensor { + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + let weight_grad_swapped = B::conv3d( + output_grad_swapped, + x_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + let mut weight_grad = B::float_swap_dims(weight_grad_swapped, 0, 1); + + let grad_shape = weight_grad.shape(); + if grad_shape != weight_shape { + let slices = vec![ + Slice::from(0..weight_shape[0]), + Slice::from(0..weight_shape[1]), + Slice::from(0..weight_shape[2]), + Slice::from(0..weight_shape[3]), + Slice::from(0..weight_shape[4]), + ]; + weight_grad = B::float_slice(weight_grad, &slices); + } + weight_grad +} + +fn conv_transpose1d_weight_grad_groups( + x: FloatTensor, + mut weight_grad: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<1>, +) -> FloatTensor { + let [channels_in, increment_co, kernel_size] = weight_grad.shape().dims(); + let increment_ci = channels_in / options.groups; + + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let x_slice = vec![Slice::new( + start_idx_ci as isize, + Some(end_idx_ci as isize), + 1, + )]; + let x = B::float_slice(x_swapped.clone(), &x_slice); + let grad_slice = vec![Slice::new( + start_idx_co as isize, + Some(end_idx_co as isize), + 1, + )]; + let grad = B::float_slice(output_grad_swapped.clone(), &grad_slice); + let mut weight_grad_tmp = B::conv1d( + grad, + x, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + weight_grad_tmp = B::float_swap_dims(weight_grad_tmp, 0, 1); + let [_, _, kernel_size_tmp] = weight_grad_tmp.shape().dims(); + + if kernel_size_tmp != kernel_size { + let slices = vec![ + Slice::from(0..increment_ci), + Slice::from(0..increment_co), + Slice::from(0..kernel_size), + ]; + weight_grad_tmp = B::float_slice(weight_grad_tmp, &slices); + } + + weight_grad = B::float_slice_assign( + weight_grad, + &[ + Slice::from(start_idx_ci..end_idx_ci), + Slice::from(0..increment_co), + Slice::from(0..kernel_size), + ], + weight_grad_tmp, + ); + } + + weight_grad +} + +fn conv_transpose2d_weight_grad_groups( + x: FloatTensor, + mut weight_grad: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<2>, +) -> FloatTensor { + let [channels_in, increment_co, kernel_size_1, kernel_size_2] = weight_grad.shape().dims(); + let increment_ci = channels_in / options.groups; + + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let x_slice = vec![Slice::new( + start_idx_ci as isize, + Some(end_idx_ci as isize), + 1, + )]; + let x = B::float_slice(x_swapped.clone(), &x_slice); + let grad_slice = vec![Slice::new( + start_idx_co as isize, + Some(end_idx_co as isize), + 1, + )]; + let grad = B::float_slice(output_grad_swapped.clone(), &grad_slice); + let mut weight_grad_tmp = B::conv2d( + grad, + x, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + weight_grad_tmp = B::float_swap_dims(weight_grad_tmp, 0, 1); + let [_, _, kernel_size_1_tmp, kernel_size_2_tmp] = weight_grad_tmp.shape().dims(); + + if kernel_size_1_tmp != kernel_size_1 || kernel_size_2_tmp != kernel_size_2 { + let slices = vec![ + Slice::from(0..increment_ci), + Slice::from(0..increment_co), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + ]; + weight_grad_tmp = B::float_slice(weight_grad_tmp, &slices); + } + + weight_grad = B::float_slice_assign( + weight_grad, + &[ + Slice::from(start_idx_ci..end_idx_ci), + Slice::from(0..increment_co), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + ], + weight_grad_tmp, + ); + } + + weight_grad +} + +fn conv_transpose3d_weight_grad_groups( + x: FloatTensor, + mut weight_grad: FloatTensor, + output_grad: FloatTensor, + options: ConvTransposeOptions<3>, +) -> FloatTensor { + let [ + channels_in, + increment_co, + kernel_size_1, + kernel_size_2, + kernel_size_3, + ] = weight_grad.shape().dims(); + let increment_ci = channels_in / options.groups; + + let x_swapped = B::float_swap_dims(x, 0, 1); + let output_grad_swapped = B::float_swap_dims(output_grad, 0, 1); + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let x_slice = vec![Slice::new( + start_idx_ci as isize, + Some(end_idx_ci as isize), + 1, + )]; + let x = B::float_slice(x_swapped.clone(), &x_slice); + let grad_slice = vec![Slice::new( + start_idx_co as isize, + Some(end_idx_co as isize), + 1, + )]; + let grad = B::float_slice(output_grad_swapped.clone(), &grad_slice); + let mut weight_grad_tmp = B::conv3d( + grad, + x, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + ); + weight_grad_tmp = B::float_swap_dims(weight_grad_tmp, 0, 1); + let [ + _, + _, + kernel_size_1_tmp, + kernel_size_2_tmp, + kernel_size_3_tmp, + ] = weight_grad_tmp.shape().dims(); + + if kernel_size_1_tmp != kernel_size_1 + || kernel_size_2_tmp != kernel_size_2 + || kernel_size_3_tmp != kernel_size_3 + { + let slices = vec![ + Slice::from(0..increment_ci), + Slice::from(0..increment_co), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + Slice::from(0..kernel_size_3), + ]; + weight_grad_tmp = B::float_slice(weight_grad_tmp, &slices); + } + weight_grad = B::float_slice_assign( + weight_grad, + &[ + Slice::from(start_idx_ci..end_idx_ci), + Slice::from(0..increment_co), + Slice::from(0..kernel_size_1), + Slice::from(0..kernel_size_2), + Slice::from(0..kernel_size_3), + ], + weight_grad_tmp, + ); + } + + weight_grad +} + +fn calculate_padding_out( + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + size_in: usize, + size_out: usize, +) -> usize { + if stride <= 1 { + return 0; + } + + let out = 1 + + ((size_in + 2 * padding - dilation * (kernel_size - 1) - 1) as f64 / stride as f64).ceil() + as usize; + i64::max(0, out as i64 - size_out as i64) as usize +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_calculate_output_size_1() { + let kernel_size = 3; + let stride = 1; + let padding = 1; + let size_in = 3; + let dilation = 1; + + let size_out = calculate_conv_output_size(kernel_size, stride, padding, dilation, size_in); + + assert_eq!(size_out, 3); + } + + #[test] + fn test_calculate_output_size_2() { + let kernel_size = 5; + let stride = 2; + let padding = 3; + let size_in = 27; + let dilation = 1; + + let size_out = calculate_conv_output_size(kernel_size, stride, padding, dilation, size_in); + + assert_eq!(size_out, 15); + } + + #[test] + fn test_calculate_output_size_3() { + let kernel_size = 5; + let stride = 2; + let padding = 3; + let size_in = 27; + let dilation = 2; + + let size_out = calculate_conv_output_size(kernel_size, stride, padding, dilation, size_in); + + assert_eq!(size_out, 13); + } + + #[test] + fn test_calculate_same_padding_1() { + let kernel_size = 3; + let stride = 1; + let size_in = 3; + let dilation = 1; + + let padding = calculate_conv_padding(kernel_size, stride, size_in, size_in); + let size_out = calculate_conv_output_size(kernel_size, stride, padding, dilation, size_in); + + assert_eq!(size_in, size_out, "Expected size"); + } + + #[test] + fn test_calculate_same_padding_2() { + let kernel_size = 3; + let stride = 2; + let size_in = 7; + let dilation = 1; + + let padding = calculate_conv_padding(kernel_size, stride, size_in, size_in); + let size_out = calculate_conv_output_size(kernel_size, stride, padding, dilation, size_in); + + assert_eq!(size_in, size_out, "Expected size"); + } + + #[test] + fn test_calculate_output_padding_1() { + let kernel_size = 3; + let stride = 2; + let size_in = 7; + let size_out = 10; + let dilation = 1; + + let padding = calculate_conv_padding(kernel_size, stride, size_in, size_out); + let size_out_expected = + calculate_conv_output_size(kernel_size, stride, padding, dilation, size_in); + + assert_eq!(size_out, size_out_expected, "Expected size"); + } + + #[test] + fn test_expect_conv2d_output_shape() { + // in channels: 3 + // out channels: 8 + // size in: [27, 3] + // kernel size: [5, 3] + let stride = [2, 1]; + let padding = [3, 1]; + let dilation = [2, 1]; + let shape = calculate_conv_output_shape( + &Shape::new([12, 3, 27, 3]), + &Shape::new([8, 3, 5, 3]), + &stride, + &padding, + &dilation, + ) + .unwrap(); + assert_eq!(shape, Shape::new([12, 8, 13, 3])) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/grid_sample.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/grid_sample.rs new file mode 100644 index 0000000..3a33b5c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/grid_sample.rs @@ -0,0 +1,312 @@ +use crate::{ + Backend, TensorMetadata, + ops::{GridSampleOptions, GridSamplePaddingMode, InterpolateMode}, + tensor::FloatTensor, +}; +use alloc::vec; +use burn_std::{Shape, Slice}; + +/// Reference implementation of grid_sample_2d that supports all options. +/// +/// # Arguments +/// +/// * `tensor` - The tensor being sampled from, must be contiguous with shape (N, C, H_in, W_in) +/// * `grid` - A tensor of locations, with shape (N, H_out, W_out, 2). Values are [-1, 1]. +/// A [x = -1, y = -1] means top-left, and [x = 1, y = 1] means bottom-right +/// * `options` - Grid sampling options +/// +/// # Returns +/// +/// A tensor with shape (N, C, H_out, W_out) +pub fn float_grid_sample_2d_ref( + tensor: FloatTensor, + grid: FloatTensor, + options: GridSampleOptions, +) -> FloatTensor { + match options.mode { + InterpolateMode::Bilinear => float_grid_sample_2d_bilinear::( + tensor, + grid, + options.padding_mode, + options.align_corners, + ), + _ => todo!( + "Default implementation for grid_sample_2d with {:?} unimplemented", + options.mode + ), + } +} + +/// Bilinear grid sampling implementation. +fn float_grid_sample_2d_bilinear( + tensor: FloatTensor, + grid: FloatTensor, + padding_mode: GridSamplePaddingMode, + align_corners: bool, +) -> FloatTensor { + let n = tensor.shape()[0]; + let c = tensor.shape()[1]; + let h_in = tensor.shape()[2]; + let w_in = tensor.shape()[3]; + let h_out = grid.shape()[1]; + let w_out = grid.shape()[2]; + let spatial_in = h_in * w_in; + let spatial_out = h_out * w_out; + + // Separate x and y coordinates from grid + // shape: (N, H_out, W_out, 1) + let grid_x_slice = vec![ + Slice::new(0, Some(n as isize), 1), + Slice::new(0, Some(h_out as isize), 1), + Slice::new(0, Some(w_out as isize), 1), + Slice::new(0, Some(1), 1), + ]; + let grid_y_slice = vec![ + Slice::new(0, Some(n as isize), 1), + Slice::new(0, Some(h_out as isize), 1), + Slice::new(0, Some(w_out as isize), 1), + Slice::new(1, Some(2), 1), + ]; + + let grid_x = B::float_slice(grid.clone(), &grid_x_slice); + let grid_x = B::float_reshape(grid_x, Shape::new([n, 1, h_out, w_out])); + let grid_y = B::float_slice(grid.clone(), &grid_y_slice); + let grid_y = B::float_reshape(grid_y, Shape::new([n, 1, h_out, w_out])); + + // Convert normalized grid coordinates [-1, 1] to pixel coordinates + let w_in_f = w_in as f64; + let h_in_f = h_in as f64; + + let (grid_x, grid_y) = if align_corners { + // align_corners=true: x_pixel = (x_norm + 1) * (width - 1) / 2 + // Maps -1 to 0 and 1 to width - 1 + let grid_x = B::float_add_scalar(grid_x, 1f32.into()); + let grid_x = B::float_mul_scalar(grid_x, ((w_in_f - 1.0) / 2.0).into()); + + let grid_y = B::float_add_scalar(grid_y, 1f32.into()); + let grid_y = B::float_mul_scalar(grid_y, ((h_in_f - 1.0) / 2.0).into()); + + (grid_x, grid_y) + } else { + // align_corners=false: x_pixel = (x_norm + 1) * width / 2 - 0.5 + // Maps -1 to -0.5 and 1 to width - 0.5 + let grid_x = B::float_add_scalar(grid_x, 1f32.into()); + let grid_x = B::float_mul_scalar(grid_x, (w_in_f / 2.0).into()); + let grid_x = B::float_sub_scalar(grid_x, 0.5f32.into()); + + let grid_y = B::float_add_scalar(grid_y, 1f32.into()); + let grid_y = B::float_mul_scalar(grid_y, (h_in_f / 2.0).into()); + let grid_y = B::float_sub_scalar(grid_y, 0.5f32.into()); + + (grid_x, grid_y) + }; + + // Apply padding mode to coordinates + let (grid_x, grid_y) = match padding_mode { + GridSamplePaddingMode::Border => { + // Clamp coordinates to valid range [0, size-1] + let grid_x = B::float_clamp(grid_x, 0f32.into(), ((w_in - 1) as f32).into()); + let grid_y = B::float_clamp(grid_y, 0f32.into(), ((h_in - 1) as f32).into()); + (grid_x, grid_y) + } + GridSamplePaddingMode::Reflection => { + // Reflect coordinates at boundaries + let grid_x = reflect_coordinates::(grid_x, w_in_f, align_corners); + let grid_y = reflect_coordinates::(grid_y, h_in_f, align_corners); + (grid_x, grid_y) + } + GridSamplePaddingMode::Zeros => { + // Keep coordinates as-is, we'll mask out-of-bounds later + (grid_x, grid_y) + } + }; + + // Get floor indices for the four corners + let grid_x_floored = B::float_floor(grid_x.clone()); + let grid_y_floored = B::float_floor(grid_y.clone()); + + // Compute interpolation weights (fractional part) + let x_frac = B::float_sub(grid_x.clone(), grid_x_floored.clone()); + let y_frac = B::float_sub(grid_y.clone(), grid_y_floored.clone()); + + // Convert to integer indices + let x0 = B::float_into_int(grid_x_floored.clone()); + let y0 = B::float_into_int(grid_y_floored.clone()); + let x1 = B::float_into_int(B::float_add_scalar(grid_x_floored, 1f32.into())); + let y1 = B::float_into_int(B::float_add_scalar(grid_y_floored, 1f32.into())); + + // Create masks for out-of-bounds coordinates (only used for zeros padding) + let (mask_00, mask_01, mask_10, mask_11) = if padding_mode == GridSamplePaddingMode::Zeros { + let x0_valid = B::int_greater_equal_elem(x0.clone(), 0.into()); + let x0_valid = B::bool_and( + x0_valid, + B::int_lower_elem(x0.clone(), (w_in as i32).into()), + ); + let x1_valid = B::int_greater_equal_elem(x1.clone(), 0.into()); + let x1_valid = B::bool_and( + x1_valid, + B::int_lower_elem(x1.clone(), (w_in as i32).into()), + ); + let y0_valid = B::int_greater_equal_elem(y0.clone(), 0.into()); + let y0_valid = B::bool_and( + y0_valid, + B::int_lower_elem(y0.clone(), (h_in as i32).into()), + ); + let y1_valid = B::int_greater_equal_elem(y1.clone(), 0.into()); + let y1_valid = B::bool_and( + y1_valid, + B::int_lower_elem(y1.clone(), (h_in as i32).into()), + ); + + ( + Some(B::bool_and(x0_valid.clone(), y0_valid.clone())), + Some(B::bool_and(x0_valid.clone(), y1_valid.clone())), + Some(B::bool_and(x1_valid.clone(), y0_valid)), + Some(B::bool_and(x1_valid, y1_valid)), + ) + } else { + (None, None, None, None) + }; + + // Clamp indices to valid range for gather + let x0_clamped = B::int_clamp(x0, 0.into(), ((w_in - 1) as i32).into()); + let x1_clamped = B::int_clamp(x1, 0.into(), ((w_in - 1) as i32).into()); + let y0_clamped = B::int_clamp(y0, 0.into(), ((h_in - 1) as i32).into()); + let y1_clamped = B::int_clamp(y1, 0.into(), ((h_in - 1) as i32).into()); + + // Linear indices: idx = y * W_in + x + let w_in_scalar: i32 = w_in as i32; + let idx_00 = B::int_add( + B::int_mul_scalar(y0_clamped.clone(), w_in_scalar.into()), + x0_clamped.clone(), + ); + let idx_01 = B::int_add( + B::int_mul_scalar(y1_clamped.clone(), w_in_scalar.into()), + x0_clamped, + ); + let idx_10 = B::int_add( + B::int_mul_scalar(y0_clamped, w_in_scalar.into()), + x1_clamped.clone(), + ); + let idx_11 = B::int_add( + B::int_mul_scalar(y1_clamped, w_in_scalar.into()), + x1_clamped, + ); + + // [N, 1, H_out, W_out] -> [N, 1, H_out * W_out] + let idx_00 = B::int_reshape(idx_00, Shape::new([n, 1, spatial_out])); + let idx_01 = B::int_reshape(idx_01, Shape::new([n, 1, spatial_out])); + let idx_10 = B::int_reshape(idx_10, Shape::new([n, 1, spatial_out])); + let idx_11 = B::int_reshape(idx_11, Shape::new([n, 1, spatial_out])); + + // [N, 1, spatial] -> [N, C, spatial] + let idx_00 = B::int_expand(idx_00, Shape::new([n, c, spatial_out])); + let idx_01 = B::int_expand(idx_01, Shape::new([n, c, spatial_out])); + let idx_10 = B::int_expand(idx_10, Shape::new([n, c, spatial_out])); + let idx_11 = B::int_expand(idx_11, Shape::new([n, c, spatial_out])); + + let tensor_flat = B::float_reshape(tensor, Shape::new([n, c, spatial_in])); + + let sample_00 = B::float_gather(2, tensor_flat.clone(), idx_00); + let sample_01 = B::float_gather(2, tensor_flat.clone(), idx_01); + let sample_10 = B::float_gather(2, tensor_flat.clone(), idx_10); + let sample_11 = B::float_gather(2, tensor_flat, idx_11); + + // Reshape samples to (N, C, H_out, W_out) + let sample_00 = B::float_reshape(sample_00, Shape::new([n, c, h_out, w_out])); + let sample_01 = B::float_reshape(sample_01, Shape::new([n, c, h_out, w_out])); + let sample_10 = B::float_reshape(sample_10, Shape::new([n, c, h_out, w_out])); + let sample_11 = B::float_reshape(sample_11, Shape::new([n, c, h_out, w_out])); + + // Apply masks for zeros padding (set out-of-bounds samples to 0) + let (sample_00, sample_01, sample_10, sample_11) = + if padding_mode == GridSamplePaddingMode::Zeros { + let mask_00 = mask_00.unwrap(); + let mask_01 = mask_01.unwrap(); + let mask_10 = mask_10.unwrap(); + let mask_11 = mask_11.unwrap(); + + let mask_00_inv = B::bool_not(mask_00); + let mask_00_inv = B::bool_reshape(mask_00_inv, Shape::new([n, 1, h_out, w_out])); + let mask_00_inv = B::bool_expand(mask_00_inv, Shape::new([n, c, h_out, w_out])); + let mask_01_inv = B::bool_not(mask_01); + let mask_01_inv = B::bool_reshape(mask_01_inv, Shape::new([n, 1, h_out, w_out])); + let mask_01_inv = B::bool_expand(mask_01_inv, Shape::new([n, c, h_out, w_out])); + let mask_10_inv = B::bool_not(mask_10); + let mask_10_inv = B::bool_reshape(mask_10_inv, Shape::new([n, 1, h_out, w_out])); + let mask_10_inv = B::bool_expand(mask_10_inv, Shape::new([n, c, h_out, w_out])); + let mask_11_inv = B::bool_not(mask_11); + let mask_11_inv = B::bool_reshape(mask_11_inv, Shape::new([n, 1, h_out, w_out])); + let mask_11_inv = B::bool_expand(mask_11_inv, Shape::new([n, c, h_out, w_out])); + + ( + B::float_mask_fill(sample_00, mask_00_inv, 0f32.into()), + B::float_mask_fill(sample_01, mask_01_inv, 0f32.into()), + B::float_mask_fill(sample_10, mask_10_inv, 0f32.into()), + B::float_mask_fill(sample_11, mask_11_inv, 0f32.into()), + ) + } else { + (sample_00, sample_01, sample_10, sample_11) + }; + + // Compute bilinear interpolation weights + let one_minus_x = B::float_neg(x_frac.clone()); + let one_minus_x = B::float_add_scalar(one_minus_x, 1f32.into()); + + let one_minus_y = B::float_neg(y_frac.clone()); + let one_minus_y = B::float_add_scalar(one_minus_y, 1f32.into()); + + let weight_00 = B::float_mul(one_minus_x.clone(), one_minus_y.clone()); + let weight_01 = B::float_mul(one_minus_x.clone(), y_frac.clone()); + let weight_10 = B::float_mul(x_frac.clone(), one_minus_y); + let weight_11 = B::float_mul(x_frac, y_frac); + + // Bilinear interpolation + let result = B::float_mul(sample_00, weight_00); + let result = B::float_add(result, B::float_mul(sample_01, weight_01)); + let result = B::float_add(result, B::float_mul(sample_10, weight_10)); + + B::float_add(result, B::float_mul(sample_11, weight_11)) +} + +/// Reflect coordinates at boundaries using a triangle wave pattern. +/// +/// For align_corners=true: reflects within [0, size-1] +/// For align_corners=false: reflects within [-0.5, size-0.5] +fn reflect_coordinates( + coords: FloatTensor, + size: f64, + align_corners: bool, +) -> FloatTensor { + let (min_val, max_val) = if align_corners { + (0.0f32, (size - 1.0) as f32) + } else { + (-0.5f32, (size - 0.5) as f32) + }; + + let span = max_val - min_val; + if span <= 0.0 { + // Edge case: size is 1, just return min_val everywhere + let zeros = B::float_mul_scalar(coords, 0f32.into()); + return B::float_add_scalar(zeros, min_val.into()); + } + + // Triangle wave formula: span - |((x mod 2*span) - span)| + min_val + let period = 2.0 * span; + + // x = abs(coord - min_val) + let x = B::float_sub_scalar(coords, min_val.into()); + let x = B::float_abs(x); + + // x_mod = x - floor(x / period) * period + let x_div = B::float_div_scalar(x.clone(), period.into()); + let x_div_floor = B::float_floor(x_div); + let x_mod = B::float_sub(x, B::float_mul_scalar(x_div_floor, period.into())); + + // result = span - abs(x_mod - span) + min_val + let diff = B::float_sub_scalar(x_mod, span.into()); + let abs_diff = B::float_abs(diff); + let reflected = B::float_sub_scalar(abs_diff, span.into()); + let reflected = B::float_neg(reflected); + B::float_add_scalar(reflected, min_val.into()) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/mod.rs new file mode 100644 index 0000000..7c5949f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/mod.rs @@ -0,0 +1,18 @@ +/// Module with convolution operations. +pub mod conv; + +/// Module with attention operations. +pub mod attention; + +/// Module with unfold operations. +pub mod unfold; + +/// Module with pooling operations. +pub mod pool; + +/// Module for grid_sample operations +pub mod grid_sample; + +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/pool.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/pool.rs new file mode 100644 index 0000000..1cd2c2f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/pool.rs @@ -0,0 +1,176 @@ +use crate::tensor::{FloatTensor, IntTensor}; +use crate::{Backend, TensorMetadata}; +use burn_std::Shape; + +use super::{MaxPool1dBackward, MaxPool1dWithIndices}; + +pub(crate) fn avg_pool1d_from_2d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, +) -> FloatTensor { + let [batch_size, channels, length] = x.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, length, 1])); + let x = B::avg_pool2d( + x, + [kernel_size, 1], + [stride, 1], + [padding, 0], + count_include_pad, + ceil_mode, + ); + + let [batch_size, channels, length, _] = x.shape().dims(); + + B::float_reshape(x, Shape::from([batch_size, channels, length])) +} + +pub(crate) fn avg_pool1d_backward_from_2d( + x: FloatTensor, + grad: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, +) -> FloatTensor { + let [batch_size, channels, length_in] = x.shape().dims(); + let [_, _, length_out] = grad.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, length_in, 1])); + let grad_x = B::float_reshape(grad, Shape::from([batch_size, channels, length_out, 1])); + + let grad_x = B::avg_pool2d_backward( + x, + grad_x, + [kernel_size, 1], + [stride, 1], + [padding, 0], + count_include_pad, + ceil_mode, + ); + + B::float_reshape(grad_x, Shape::from([batch_size, channels, length_in])) +} + +pub(crate) fn adaptive_avg_pool1d_from_2d( + x: FloatTensor, + output_size: usize, +) -> FloatTensor { + let [batch_size, channels, length] = x.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, length, 1])); + let x = B::adaptive_avg_pool2d(x, [output_size, 1]); + + let [batch_size, channels, length, _] = x.shape().dims(); + + B::float_reshape(x, Shape::from([batch_size, channels, length])) +} + +pub(crate) fn adaptive_avg_pool1d_backward_from_2d( + x: FloatTensor, + grad: FloatTensor, +) -> FloatTensor { + let [batch_size, channels, length_in] = x.shape().dims(); + let [_, _, length_out] = grad.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, length_in, 1])); + let grad_x = B::float_reshape(grad, Shape::from([batch_size, channels, length_out, 1])); + + let grad_x = B::adaptive_avg_pool2d_backward(x, grad_x); + + B::float_reshape(grad_x, Shape::from([batch_size, channels, length_in])) +} + +pub(crate) fn max_pool1d_from_2d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, +) -> FloatTensor { + let [batch_size, channels, length] = x.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, length, 1])); + let x = B::max_pool2d( + x, + [kernel_size, 1], + [stride, 1], + [padding, 0], + [dilation, 1], + ceil_mode, + ); + + let [batch_size, channels, length, _] = x.shape().dims(); + + B::float_reshape(x, Shape::from([batch_size, channels, length])) +} + +pub(crate) fn max_pool1d_with_indices_from_2d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, +) -> MaxPool1dWithIndices { + let [batch_size, channels, length] = x.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, 1, length])); + let x = B::max_pool2d_with_indices( + x, + [1, kernel_size], + [1, stride], + [0, padding], + [1, dilation], + ceil_mode, + ); + let [batch_size, channels, _, length] = x.output.shape().dims(); + let output = B::float_reshape(x.output, Shape::from([batch_size, channels, length])); + let indices = B::int_reshape(x.indices, Shape::from([batch_size, channels, length])); + MaxPool1dWithIndices::new(output, indices) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn max_pool1d_with_indices_backward_from_2d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, +) -> MaxPool1dBackward { + let [batch_size, channels, length_in] = x.shape().dims(); + let [_, _, length_out] = output_grad.shape().dims(); + + let x = B::float_reshape(x, Shape::from([batch_size, channels, length_in, 1])); + let grad_x = B::float_reshape( + output_grad, + Shape::from([batch_size, channels, length_out, 1]), + ); + let indices = B::int_reshape(indices, Shape::from([batch_size, channels, length_out, 1])); + + let grad_x = B::max_pool2d_with_indices_backward( + x, + [kernel_size, 1], + [stride, 1], + [padding, 0], + [dilation, 1], + ceil_mode, + grad_x, + indices, + ) + .x_grad; + + MaxPool1dBackward::new(B::float_reshape( + grad_x, + Shape::from([batch_size, channels, length_in]), + )) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/unfold.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/unfold.rs new file mode 100644 index 0000000..124c2d7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/modules/unfold.rs @@ -0,0 +1,146 @@ +use super::{ConvOptions, UnfoldOptions}; +use crate::tensor::FloatTensor; +use crate::{Backend, TensorData, TensorMetadata, element::ElementConversion}; +use alloc::vec; +use alloc::vec::Vec; +use burn_std::Shape; + +/// Constructs a special weight tensor used for unfolding. +/// +/// # Notes +/// +/// The idea behind using convolution for unfolding is to leverage the sliding window mechanism of +/// convolution. By creating a weight tensor with ones in a particular pattern, we are able to borrow +/// the convolution operation's mechanism as it moves across the input tensor, picking up the desired +/// values in the pattern of the unfolding operation. +pub(crate) fn create_unfolding_weight( + in_channels: usize, + kernel_size: [usize; 2], + device: &B::Device, +) -> FloatTensor { + let shape = Shape::new([ + in_channels * kernel_size[0] * kernel_size[1], + in_channels, + kernel_size[0], + kernel_size[1], + ]); + + let mut strides = [0; 4]; + let mut current = 1; + shape.iter().enumerate().rev().for_each(|(index, val)| { + strides[index] = current; + current *= val; + }); + + let num_elements = shape.num_elements(); + + let mut weight: Vec = vec![0.0.elem(); num_elements]; + + for k in 0..in_channels { + for i in 0..kernel_size[0] { + for j in 0..kernel_size[1] { + let output_channel = k * kernel_size[0] * kernel_size[1] + i * kernel_size[1] + j; + let index = + output_channel * strides[0] + k * strides[1] + i * strides[2] + j * strides[3]; + + weight[index] = 1.elem(); + } + } + } + + B::float_from_data(TensorData::new(weight, shape), device) +} + +/// Compute the unfold4d operation using the conv2d operations. +pub(crate) fn unfold4d_using_conv2d( + x: FloatTensor, + kernel_size: [usize; 2], + options: UnfoldOptions, +) -> FloatTensor { + let [_batch_size, in_channels, _in_height, _in_width] = x.shape().dims(); + let weight = create_unfolding_weight::(in_channels, kernel_size, &B::float_device(&x)); + let unfolded = B::conv2d( + x, + weight, + None, + ConvOptions::new(options.stride, options.padding, options.dilation, 1), + ); + + let [batch_size, channels_out, out_height, out_width] = unfolded.shape().dims(); + + B::float_reshape( + unfolded, + Shape::new([batch_size, channels_out, out_height * out_width]), + ) +} + +/// Calculate the number of unfolding windows that can be extracted from a dimension of given size. +pub fn calculate_unfold_windows(dim_size: usize, window_size: usize, step_size: usize) -> usize { + assert!(step_size > 0); + let x = dim_size + step_size; + if x < window_size { + 0 + } else { + (x - window_size) / step_size + } +} + +/// Calculate the output shape for an unfold operation. +/// +/// The operation yields a view with all complete windows of size `size` in dimension `dim`; +/// where windows are advanced by `step` at each index. +/// +/// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. +/// +/// # Arguments +/// +/// * `shape` - The input shape to unfold; of shape ``[pre=..., dim shape, post=...]`` +/// * `dim` - the dimension to unfold. +/// * `size` - the size of each unfolded window. +/// * `step` - the step between each window. +/// +/// # Returns +/// +/// A shape with ``[pre=..., windows, post=..., size]``. +pub fn calculate_unfold_shape>( + shape: S, + dim: usize, + size: usize, + step: usize, +) -> Shape { + let mut shape = shape.into(); + let d_shape = shape[dim]; + let windows = calculate_unfold_windows(d_shape, size, step); + shape[dim] = windows; + shape.push(size); + + shape +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_calculate_unfold_windows() { + assert_eq!(calculate_unfold_windows(2, 5, 1), 0); + + assert_eq!(calculate_unfold_windows(2, 3, 1), 0); + assert_eq!(calculate_unfold_windows(3, 3, 1), 1); + assert_eq!(calculate_unfold_windows(4, 3, 1), 2); + assert_eq!(calculate_unfold_windows(5, 3, 1), 3); + + assert_eq!(calculate_unfold_windows(2, 3, 2), 0); + assert_eq!(calculate_unfold_windows(3, 3, 2), 1); + assert_eq!(calculate_unfold_windows(4, 3, 2), 1); + assert_eq!(calculate_unfold_windows(5, 3, 2), 2); + } + + #[test] + fn test_calculate_unfold_shape() { + assert_eq!( + calculate_unfold_shape([2, 6, 6], 1, 3, 2), + Shape::new([2, 2, 6, 3]) + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/qtensor.rs new file mode 100644 index 0000000..503ad10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/qtensor.rs @@ -0,0 +1,1373 @@ +use alloc::vec::Vec; +use burn_std::{ + Shape, Slice, + quantization::{QuantPropagation, QuantScheme}, +}; + +use crate::{ + Backend, ExecutionError, QTensorPrimitive, TensorData, TensorMetadata, TensorPrimitive, +}; +use crate::{ + Scalar, + tensor::{ + BoolTensor, Device, FloatTensor, IntTensor, QuantizedTensor, + quantization::{ + Calibration, QuantizationParametersPrimitive, compute_q_params, compute_range, + }, + }, +}; + +/// Automatically applies `dequantization -> float operation -> quantization`. +/// +/// Used for tensor ops that should always return a quantized output. +#[macro_export] +macro_rules! dequant_op_quant { + // Binary tensor float op w/ lhs & rhs + ( + ty $ty:ty, float_op $float_op:expr, $t1:expr, $t2:expr + ) => {{ + // Heuristic: prioritize lhs scheme + let scheme = $t1.scheme().clone(); + + let t1_f = <$ty>::dequantize($t1); + let t2_f = <$ty>::dequantize($t2); + #[allow(clippy::redundant_closure_call)] + let out_f = $float_op(t1_f, t2_f); + + <$ty>::quantize_dynamic(out_f, &scheme) + }}; + // Unary tensor float op + ( + ty $ty:ty, float_op $float_op:expr, $tensor:expr + ) => {{ + let scheme = $tensor.scheme().clone(); + + let tensor_f = <$ty>::dequantize($tensor); + #[allow(clippy::redundant_closure_call)] + let out_f = $float_op(tensor_f); + + <$ty>::quantize_dynamic(out_f, &scheme) + }}; +} + +/// Automatically applies `dequantization -> float operation [-> quantization]`. +/// +/// The output quantization step is optional. +/// It is only performed when the input quantization scheme is propagated. +#[macro_export] +macro_rules! dequant_op_flow { + // Binary tensor float op w/ lhs & rhs + ( + ty $ty:ty, float_op $float_op:expr, $t1:expr, $t2:expr + ) => {{ + // Heuristic: prioritize lhs scheme + let scheme = $t1.scheme().clone(); + let propagation = $t1.propagation(); + + let t1_f = <$ty>::dequantize($t1); + let t2_f = <$ty>::dequantize($t2); + #[allow(clippy::redundant_closure_call)] + let out_f = $float_op(t1_f, t2_f); + + match propagation { + QuantPropagation::Propagate => { + TensorPrimitive::QFloat(<$ty>::quantize_dynamic(out_f, &scheme)) + } + QuantPropagation::Inhibit => TensorPrimitive::Float(out_f), + } + }}; + // Unary tensor float op + ( + ty $ty:ty, float_op $float_op:expr, $tensor:expr + ) => {{ + let scheme = $tensor.scheme().clone(); + let propagation = $tensor.propagation(); + + let tensor_f = <$ty>::dequantize($tensor); + #[allow(clippy::redundant_closure_call)] + let out_f = $float_op(tensor_f); + + match propagation { + QuantPropagation::Propagate => { + TensorPrimitive::QFloat(<$ty>::quantize_dynamic(out_f, &scheme)) + } + QuantPropagation::Inhibit => TensorPrimitive::Float(out_f), + } + }}; +} + +/// Operations on quantized tensors. +/// +/// # Return Type Semantics +/// +/// The return type of each operation indicates how quantization is handled: +/// +/// ## [`QuantizedTensor`] +/// If the method returns a `QuantizedTensor`, the operation is expected to preserve the quantized +/// representation. Implementations should avoid dequantizing when possible to maintain performance. +/// For example, shape or layout changes such as expand or transpose preserve quantization. +/// +/// *Note: while this currently doesn't affect the quantized tensor parameters (only per-tensor is +/// supported at the time of writing), other quantization levels (e.g., per-block) may require re-ordering +/// the quantization parameters to match the new layout.* +/// +/// +/// ## [`TensorPrimitive`] +/// If the method returns a `TensorPrimitive` enum, the return type should align with propagation +/// strategy specified in the quantization scheme. The output should remain quantized ([`TensorPrimitive::QFloat`]) +/// returned in floating-point form ([`TensorPrimitive::Float`]). +/// +/// This distinction allows for fine-grained control over mixed-precision flows while still operating +/// through a unified API. +pub trait QTensorOps { + /// Creates a new tensor from the data structure. + /// + /// # Arguments + /// + /// * `data` - The data structure. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the given data. + fn q_from_data(data: TensorData, device: &Device) -> QuantizedTensor; + + /// Convert the tensor to a lower precision data type based on the quantization scheme and parameters. + fn quantize( + tensor: FloatTensor, + scheme: &QuantScheme, + qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor; + + /// Dynamically convert the tensor to a lower precision data type based on the quantization scheme. + fn quantize_dynamic(tensor: FloatTensor, scheme: &QuantScheme) -> QuantizedTensor { + // Dynamically compute min/max tensor range and qparams before quantizing + let (min, max) = compute_range::(scheme, tensor.clone(), &Calibration::MinMax); + let qparams = compute_q_params(scheme, min, max); + Self::quantize(tensor, scheme, qparams) + } + + /// Convert the tensor back to a higher precision data type. + fn dequantize(tensor: QuantizedTensor) -> FloatTensor; + + /// Gets the device of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The device of the tensor. + fn q_device(tensor: &QuantizedTensor) -> Device; + + /// Moves the tensor to the given device. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `device` - The device to move the tensor to. + /// + /// # Returns + /// + /// The tensor on the given device. + fn q_to_device(tensor: QuantizedTensor, device: &Device) -> QuantizedTensor; + + /// Reshapes a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to reshape. + /// * `shape` - The new shape of the tensor. + /// + /// # Returns + /// + /// The tensor with the new shape. + fn q_reshape(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor; + + /// Converts the tensor to a data structure. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The data structure with the tensor's data. + fn q_into_data( + tensor: QuantizedTensor, + ) -> impl Future> + Send; + + /// Detaches a tensor from the computation graph. + fn q_detach(tensor: QuantizedTensor) -> QuantizedTensor { + // Should only be overridden by autodiff backends. + tensor + } + + /// Sets the `require_grad` flag of a tensor. + fn q_set_require_grad(tensor: QuantizedTensor, _require_grad: bool) -> QuantizedTensor { + // Should only be overridden by autodiff backends. + tensor + } + + /// Returns the `require_grad` flag of a tensor. + fn q_is_require_grad(_tensor: &QuantizedTensor) -> bool { + // Should only be overridden by autodiff backends. + false + } + + /// Broadcasts the `tensor` to the given `shape`. + fn q_expand(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor; + + /// Transposes a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to transpose. + /// + /// # Returns + /// + /// The transposed tensor. + fn q_transpose(tensor: QuantizedTensor) -> QuantizedTensor { + let ndims = tensor.shape().num_dims(); + Self::q_swap_dims(tensor, ndims - 2, ndims - 1) + } + + /// Swaps two dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to swap the dimensions of. + /// * `dim1` - The first dimension to swap. + /// * `dim2` - The second dimension to swap. + /// + /// # Returns + /// + /// The tensor with the dimensions swapped. + fn q_swap_dims(tensor: QuantizedTensor, dim1: usize, dim2: usize) -> QuantizedTensor; + + /// Permutes the dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to permute the dimensions of. + /// * `axes` - The new order of the dimensions. + /// # Returns + /// + /// The tensor with the dimensions permuted. + fn q_permute(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor; + + /// Reverse the order of elements in a tensor along the given axes. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to reverse. + /// * `axes` - The axes to reverse. + /// + /// The tensor with the elements reversed. + fn q_flip(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor; + + /// Select tensor elements along the given dimension corresponding for the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices to select. + /// + /// # Returns + /// + /// The selected elements. + fn q_select( + tensor: QuantizedTensor, + dim: usize, + indices: IntTensor, + ) -> QuantizedTensor; + + /// Select tensor elements corresponding to the given slices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `slices` - The slices specifying ranges and steps for each dimension. + /// + /// # Returns + /// + /// The selected elements in a new tensor. + fn q_slice(tensor: QuantizedTensor, slices: &[Slice]) -> QuantizedTensor; + + /// Gather elements from a tensor. + /// + /// # Arguments + /// + /// * `dim` - The dimension to gather from. + /// * `tensor` - The tensor to gather from. + /// * `indices` - The indices to gather. + /// + /// # Returns + /// + /// The gathered elements. + fn q_gather( + dim: usize, + tensor: QuantizedTensor, + indices: IntTensor, + ) -> QuantizedTensor { + // Default implementation. Backends can gather on the quantized values when supported. + dequant_op_quant!( + ty Self, + float_op |tensor| B::float_gather(dim, tensor, indices), + tensor + ) + } + + /// Repeat the tensor along the given dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension to repeat. + /// * `times` - The number of times to repeat the dimension. + /// + /// # Returns + /// + /// The tensor with the given dimension repeated. + fn q_repeat_dim(tensor: QuantizedTensor, dim: usize, times: usize) -> QuantizedTensor { + dequant_op_quant!( + ty Self, + float_op |tensor| B::float_repeat_dim(tensor, dim, times), + tensor + ) + } + + /// Adds two tensors together. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The result of adding the two tensors together. + fn q_add(lhs: QuantizedTensor, rhs: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |lhs, rhs| B::float_add(lhs, rhs), + lhs, + rhs + ) + } + + /// Adds a scalar to a tensor. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The result of adding the scalar to the tensor. + fn q_add_scalar(lhs: QuantizedTensor, rhs: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_add_scalar(tensor, rhs), + lhs + ) + } + + /// Clamps a tensor under a minimum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn q_clamp_min(tensor: QuantizedTensor, min: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_clamp_min(tensor, min), + tensor + ) + } + + /// Clamps a tensor over a maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn q_clamp_max(tensor: QuantizedTensor, max: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_clamp_max(tensor, max), + tensor + ) + } + + /// Clamps a tensor between a minimum and maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn q_clamp(tensor: QuantizedTensor, min: Scalar, max: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_clamp(tensor, min, max), + tensor + ) + } + + /// Subtracts two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The result of subtracting the two tensors. + fn q_sub(lhs: QuantizedTensor, rhs: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |lhs, rhs| B::float_sub(lhs, rhs), + lhs, + rhs + ) + } + + /// Subtracts a scalar from a tensor. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The result of subtracting the scalar from the tensor. + fn q_sub_scalar(lhs: QuantizedTensor, rhs: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_sub_scalar(tensor, rhs), + lhs + ) + } + + /// Multiplies two tensors together element-wise. + fn q_mul(lhs: QuantizedTensor, rhs: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |lhs, rhs| B::float_mul(lhs, rhs), + lhs, + rhs + ) + } + + /// Multiplies a tensor by a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The result of multiplying the tensor by the scalar. + fn q_mul_scalar(lhs: QuantizedTensor, rhs: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_mul_scalar(tensor, rhs), + lhs + ) + } + + /// Divides two tensors element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The result of dividing the two tensors. + fn q_div(lhs: QuantizedTensor, rhs: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |lhs, rhs| B::float_div(lhs, rhs), + lhs, + rhs + ) + } + + /// Divides a tensor by a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The result of dividing the tensor by the scalar. + fn q_div_scalar(lhs: QuantizedTensor, rhs: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_div_scalar(tensor, rhs), + lhs + ) + } + + /// Multiplies two tensors together using matrix multiplication. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The result of multiplying the two tensors together using matrix multiplication. + fn q_matmul(lhs: TensorPrimitive, rhs: TensorPrimitive) -> TensorPrimitive { + let mut propagation = QuantPropagation::Inhibit; + let mut scheme = QuantScheme::default(); + let lhs = match lhs { + TensorPrimitive::Float(lhs) => lhs, + TensorPrimitive::QFloat(lhs) => { + propagation = lhs.propagation(); + scheme = *lhs.scheme(); + Self::dequantize(lhs) + } + }; + let rhs = match rhs { + TensorPrimitive::Float(rhs) => rhs, + TensorPrimitive::QFloat(rhs) => { + propagation = rhs.propagation(); + scheme = *rhs.scheme(); + Self::dequantize(rhs) + } + }; + + let out_f = B::float_matmul(lhs, rhs); + match propagation { + QuantPropagation::Propagate => { + TensorPrimitive::QFloat(::quantize_dynamic(out_f, &scheme)) + } + QuantPropagation::Inhibit => TensorPrimitive::Float(out_f), + } + } + + /// Negates a tensor element-wise. + fn q_neg(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_neg(tensor), + tensor + ) + } + + /// Calculates the reciprocals element-wise + fn q_recip(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_recip(tensor), + tensor + ) + } + + /// Sum of all elements in a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// + /// # Returns + /// + /// A scalar tensor with the sum of all elements in `tensor`. + fn q_sum(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_sum(tensor), + tensor + ) + } + + /// Sum of all elements in a tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// * `dim` - The dimension along which to sum. + /// + /// # Returns + /// + /// A tensor with the sum of all elements in `tensor` along `dim`. + fn q_sum_dim(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_sum_dim(tensor, dim), + tensor + ) + } + + /// Product of all elements in a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to product. + /// + /// # Returns + /// + /// A scalar tensor with the product of all elements in `tensor`. + fn q_prod(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_prod(tensor), + tensor + ) + } + + /// Product of all elements in a tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to product. + /// + /// # Returns + /// + /// A tensor with the product of all elements in `tensor` along `dim`. + fn q_prod_dim(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_prod_dim(tensor, dim), + tensor + ) + } + + /// Mean of all elements in a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to mean. + /// + /// # Returns + /// + /// A scalar tensor with the mean of all elements in `tensor`. + fn q_mean(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_mean(tensor), + tensor + ) + } + + /// Mean of all elements in a tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to mean. + /// * `dim` - The dimension along which to mean. + /// + /// # Returns + /// + /// A tensor with the mean of all elements in `tensor` along `dim`. + fn q_mean_dim(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_mean_dim(tensor, dim), + tensor + ) + } + + /// Computes the cumulative sum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative sum of. + /// * `dim` - The dimension along which to compute the cumulative sum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the cumulative sum + /// of all elements up to and including that position along the dimension. + fn q_cumsum(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_cumsum(tensor, dim), + tensor + ) + } + + /// Computes the cumulative product of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative product of. + /// * `dim` - The dimension along which to compute the cumulative product. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the cumulative product + /// of all elements up to and including that position along the dimension. + fn q_cumprod(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_cumprod(tensor, dim), + tensor + ) + } + + /// Computes the cumulative minimum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative minimum of. + /// * `dim` - The dimension along which to compute the cumulative minimum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the minimum + /// of all elements up to and including that position along the dimension. + fn q_cummin(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_cummin(tensor, dim), + tensor + ) + } + + /// Computes the cumulative maximum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative maximum of. + /// * `dim` - The dimension along which to compute the cumulative maximum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the maximum + /// of all elements up to and including that position along the dimension. + fn q_cummax(tensor: QuantizedTensor, dim: usize) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_cummax(tensor, dim), + tensor + ) + } + + /// Returns a new tensor with exponential values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to exponentiate. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with exponential values. + fn q_exp(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_exp(tensor), + tensor + ) + } + + /// Returns a new tensor with natural logarithm values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the logarithm of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with natural logarithm values. + fn q_log(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_log(tensor), + tensor + ) + } + + /// Returns a new tensor with logarithm values of (1 + Xi). + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the logarithm of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with logarithm values of (1 + Xi). + fn q_log1p(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_log1p(tensor), + tensor + ) + } + + /// Element-wise power with another tensor. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the power of the elements of `rhs`. + fn q_powf(lhs: QuantizedTensor, rhs: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |lhs, rhs| B::float_powf(lhs, rhs), + lhs, + rhs + ) + } + + /// Element-wise power with an IntTensor. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side floatTensor. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. Result is an IntTensor. + fn q_powi(lhs: QuantizedTensor, rhs: IntTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_powi(tensor, rhs), + lhs + ) + } + + /// Element-wise power with an int scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. + fn q_powi_scalar(lhs: QuantizedTensor, rhs: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_powi_scalar(tensor, rhs), + lhs + ) + } + + /// Element-wise power with a float scalar. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to exponentiate. + /// * `value` - The exponent. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with values raised to the power of `value`. + fn q_powf_scalar(tensor: QuantizedTensor, value: Scalar) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_powf_scalar(tensor, value), + tensor + ) + } + + /// Returns a new tensor with square root values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the square root of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with square root values. + fn q_sqrt(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_sqrt(tensor), + tensor + ) + } + + /// Returns a new tensor with absolute values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take absolute value of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with absolute values. + fn q_abs(tensor: QuantizedTensor) -> QuantizedTensor { + dequant_op_quant!( + ty Self, + float_op |tensor| B::float_abs(tensor), + tensor + ) + } + + /// Returns a new tensor with cosine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the cosine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with cosine values. + fn q_cos(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_cos(tensor), + tensor + ) + } + + /// Returns a new tensor with sine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the sine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with sine values. + fn q_sin(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_sin(tensor), + tensor + ) + } + + /// Returns a new tensor with tangent values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the tangent of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with tangent values. + fn q_tan(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_tan(tensor), + tensor + ) + } + + /// Returns a new tensor with hyperbolic cosine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the hyperbolic cosine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with hyperbolic cosine values. + fn q_cosh(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_cosh(tensor), + tensor + ) + } + + /// Returns a new tensor with hyperbolic sine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the hyperbolic sine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with hyperbolic sine values. + fn q_sinh(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_sinh(tensor), + tensor + ) + } + + /// Returns a new tensor with hyperbolic tangent values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the hyperbolic tangent of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with hyperbolic tangent values. + fn q_tanh(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_tanh(tensor), + tensor + ) + } + + /// Returns a new tensor with the error function values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the error function of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with error function values. + fn q_erf(tensor: QuantizedTensor) -> TensorPrimitive { + dequant_op_flow!( + ty Self, + float_op |tensor| B::float_erf(tensor), + tensor + ) + } + + /// Concatenates tensors along a dimension. + /// + /// # Arguments + /// + /// * `tensors` - The tensors to concatenate. + /// * `dim` - The dimension along which to concatenate. + /// + /// # Returns + /// + /// A tensor with the concatenated tensors along `dim`. + fn q_cat(tensors: Vec>, dim: usize) -> QuantizedTensor { + // Heuristic: prioritize first tensor scheme + let scheme = *tensors.first().unwrap().scheme(); + + let tensor_f = tensors + .into_iter() + .map(|tensor| Self::dequantize(tensor)) + .collect(); + + let out_f = B::float_cat(tensor_f, dim); + + Self::quantize_dynamic(out_f, &scheme) + } + + /// Gets the indices of the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the indices of the maximum elements of `tensor` along `dim`. + fn q_argmax(tensor: QuantizedTensor, dim: usize) -> IntTensor { + // Default implementation. Backends can sort on the int values since qparams remain the same. + let tensor_f = Self::dequantize(tensor); + B::float_argmax(tensor_f, dim) + } + + /// Gets the indices of the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// * `dim` - The dimension along which to get the minimum elements. + /// + /// # Returns + /// + /// A tensor with the indices of the minimum elements of `tensor` along `dim`. + fn q_argmin(tensor: QuantizedTensor, dim: usize) -> IntTensor { + // Default implementation. Backends can sort on the int values since qparams remain the same. + let tensor_f = Self::dequantize(tensor); + B::float_argmin(tensor_f, dim) + } + + /// Gets the maximum element of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// + /// # Returns + /// + /// A tensor with the maximum element of `tensor`. + fn q_max(tensor: QuantizedTensor) -> QuantizedTensor { + let shape = tensor.shape(); + let tensor = B::q_reshape(tensor, Shape::new([shape.num_elements()])); + + B::q_max_dim(tensor, 0) + } + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the maximum elements of `tensor` along `dim`. + fn q_max_dim(tensor: QuantizedTensor, dim: usize) -> QuantizedTensor { + let index = B::q_argmax(tensor.clone(), dim); + + B::q_gather(dim, tensor, index) + } + + /// Gets the maximum elements of a tensor along an axis and their indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tuple with the maximum elements of `tensor` along `dim` and their indices. + fn q_max_dim_with_indices( + tensor: QuantizedTensor, + dim: usize, + ) -> (QuantizedTensor, IntTensor) { + let index = B::q_argmax(tensor.clone(), dim); + let values = B::q_gather(dim, tensor, index.clone()); + + (values, index) + } + + /// Gets the minimum element of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// + /// # Returns + /// + /// A tensor with the minimum element of `tensor`. + fn q_min(tensor: QuantizedTensor) -> QuantizedTensor { + let shape = tensor.shape(); + let tensor = B::q_reshape(tensor, Shape::new([shape.num_elements()])); + + B::q_min_dim(tensor, 0) + } + + /// Gets the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// * `dim` - The dimension along which to get the minimum elements. + /// + /// # Returns + /// + /// A tensor with the minimum elements of `tensor` along `dim`. + fn q_min_dim(tensor: QuantizedTensor, dim: usize) -> QuantizedTensor { + let index = B::q_argmin(tensor.clone(), dim); + + B::q_gather(dim, tensor, index) + } + + /// Gets the minimum elements of a tensor along an axis and their indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// * `dim` - The dimension along which to get the minimum elements. + /// + /// # Returns + /// + /// A tuple with the minimum elements of `tensor` along `dim` and their indices. + fn q_min_dim_with_indices( + tensor: QuantizedTensor, + dim: usize, + ) -> (QuantizedTensor, IntTensor) { + let index = B::q_argmin(tensor.clone(), dim); + let values = B::q_gather(dim, tensor, index.clone()); + + (values, index) + } + + /// Gets the maximum element of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// + /// # Returns + /// + /// A tensor with the maximum element of `tensor`. + fn q_max_abs(tensor: QuantizedTensor) -> QuantizedTensor { + let shape = tensor.shape(); + let tensor = B::q_reshape(tensor, Shape::new([shape.num_elements()])); + + B::q_max_abs_dim(tensor, 0) + } + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the maximum elements of `tensor` along `dim`. + fn q_max_abs_dim(tensor: QuantizedTensor, dim: usize) -> QuantizedTensor { + let index = B::q_argmax(B::q_abs(tensor.clone()), dim); + + B::q_gather(dim, tensor, index) + } + + /// Tests if any element in the `tensor` evaluates to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with a single element, True if any element in the tensor is True, False otherwise. + fn q_any(tensor: QuantizedTensor) -> BoolTensor { + let tensor_f = Self::dequantize(tensor); + B::float_any(tensor_f) + } + + /// Tests if any element in the float `tensor` evaluates to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if any element along this dim in the + /// input evaluates to True, False otherwise. + fn q_any_dim(tensor: QuantizedTensor, dim: usize) -> BoolTensor { + let tensor_f = Self::dequantize(tensor); + B::float_any_dim(tensor_f, dim) + } + + /// Tests if all elements in the `tensor` evaluate to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with a single element, True if all elements in the input tensor + /// evaluate to True, False otherwise. + fn q_all(tensor: QuantizedTensor) -> BoolTensor { + let tensor_f = Self::dequantize(tensor); + B::float_all(tensor_f) + } + + /// Tests if all elements in the `tensor` evaluate to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if all elements along this dim in the input + /// evaluates to True, False otherwise. + fn q_all_dim(tensor: QuantizedTensor, dim: usize) -> BoolTensor { + let tensor_f = Self::dequantize(tensor); + B::float_all_dim(tensor_f, dim) + } + + /// Sort the elements of the input `tensor` by value in along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where the elements are sorted by value. + fn q_sort(tensor: QuantizedTensor, dim: usize, descending: bool) -> QuantizedTensor { + // Default implementation. Backends can sort on the int values since qparams remain the same. + dequant_op_quant!( + ty Self, + float_op |tensor| B::float_sort(tensor, dim, descending), + tensor + ) + } + + /// Sort the elements of the input `tensor` by value in along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor and corresponding indices, where + /// the elements are sorted by value and the indices map back to the original input tensor. + fn q_sort_with_indices( + tensor: QuantizedTensor, + dim: usize, + descending: bool, + ) -> (QuantizedTensor, IntTensor) { + // Default implementation. Backends can sort on the int values since qparams remain the same. + let scheme = *tensor.scheme(); + + let tensor_f = Self::dequantize(tensor); + let (out_f, indices) = B::float_sort_with_indices(tensor_f, dim, descending); + + (Self::quantize_dynamic(out_f, &scheme), indices) + } + + /// Returns the indices that sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor the indices map back to the original input tensor. + fn q_argsort(tensor: QuantizedTensor, dim: usize, descending: bool) -> IntTensor { + // Default implementation. Backends can sort on the int values since qparams remain the same. + let tensor_f = Self::dequantize(tensor); + B::float_argsort(tensor_f, dim, descending) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/repeat_dim.rs new file mode 100644 index 0000000..2955539 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/repeat_dim.rs @@ -0,0 +1,39 @@ +use crate::{ + Backend, TensorMetadata, + tensor::{BasicOps, TensorKind}, +}; +use alloc::vec::Vec; +use burn_std::Slice; + +pub(crate) fn repeat_with_slice_assign + BasicOps>( + tensor: K::Primitive, + dim: usize, + times: usize, +) -> K::Primitive { + let shape = tensor.shape(); + let device = K::device(&tensor); + let dtype = tensor.dtype(); + + let original_dim_length = shape[dim]; + let shape = shape.repeat(dim, times).unwrap(); + + let mut tensor_output = K::empty(shape.clone(), &device, dtype); + + let indices_select_all = shape.iter().map(|d| 0..*d).collect::>(); + + let mut output_index = 0; + for _ in 0..times { + let mut indices = indices_select_all.clone(); + indices[dim] = output_index..output_index + original_dim_length; + output_index += original_dim_length; + + // Convert ranges to Slice + let slices: Vec = indices + .iter() + .map(|r| Slice::new(r.start as isize, Some(r.end as isize), 1)) + .collect(); + tensor_output = K::slice_assign(tensor_output, &slices, tensor.clone()); + } + + tensor_output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/sort.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/sort.rs new file mode 100644 index 0000000..be76850 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/sort.rs @@ -0,0 +1,377 @@ +use core::cmp::Ordering; + +use crate::{ + Backend, DType, TensorData, + element::{ElementConversion, ElementOrdered}, + tensor::{BasicOps, IntElem, IntTensor}, +}; +use alloc::{vec, vec::Vec}; +use burn_std::reader::try_read_sync; +use burn_std::{bf16, f16}; + +/// Macro used to dispatch sort operations based on dtype. +macro_rules! sort_dispatch_dtype { + ($fn:ident, $data:ident, $($args:expr),*) => { + match $data.dtype { + DType::F64 => $fn::($data, $($args),*), + DType::F32 | DType::Flex32 => $fn::($data, $($args),*), + DType::F16 => $fn::($data, $($args),*), + DType::BF16 => $fn::($data, $($args),*), + DType::I64 => $fn::($data, $($args),*), + DType::I32 => $fn::($data, $($args),*), + DType::I16 => $fn::($data, $($args),*), + DType::I8 => $fn::($data, $($args),*), + DType::U64 => $fn::($data, $($args),*), + DType::U32 => $fn::($data, $($args),*), + DType::U16 => $fn::($data, $($args),*), + DType::U8 => $fn::($data, $($args),*), + DType::Bool | DType::QFloat(_) => unimplemented!("not supported for sorting operations"), + } + }; +} + +/// Sort the elements of the input `tensor` by value along a given dimension. +/// +/// This sort is unstable (i.e., may reorder equal elements). +/// +/// # Arguments +/// +/// * `tensor` - The input tensor. +/// * `dim` - The axis along which to sort. +/// * `descending` - The sorting order. +/// +/// # Returns +/// +/// A tensor with the same shape as the input tensor, where the elements are sorted by value. +/// +/// # Remarks +/// +/// This is a fallback solution that used only when the backend doesn't have the corresponding implementation. +/// Ideally, it is supposed to be implemented by the backend and the backend implementation will be resolved +/// by static dispatch. It is not designed for direct usage by users, and not recommended to import +/// or use this function directly. +pub fn sort>( + tensor: K::Primitive, + dim: usize, + descending: bool, +) -> K::Primitive { + let device = K::device(&tensor); + let msg = "Failed to synchronously read tensor data. This operation is not supported until this backend has a GPU sorting implementation."; + let data = try_read_sync(K::into_data_async(tensor)) + .expect(msg) + .expect(msg); + + let data = sort_dispatch_dtype!(sort_data, data, dim, descending); + K::from_data(data, &device) +} + +pub fn sort_data( + mut data: TensorData, + dim: usize, + descending: bool, +) -> TensorData { + let dims = data.shape.clone(); + let data_slice = data.as_mut_slice().unwrap(); + if dims.len() == 1 { + // 1D sort + data_slice.sort_unstable_by(|&a, &b| compare(&a, &b, descending)); + } else { + sort_slice::(data_slice, &dims, dim, None, false, descending); + } + + data +} + +/// Sort the elements of the input `tensor` by value along a given dimension. +/// +/// This sort is unstable (i.e., may reorder equal elements). +/// +/// # Arguments +/// +/// * `tensor` - The input tensor. +/// * `dim` - The axis along which to sort. +/// * `descending` - The sorting order. +/// +/// # Returns +/// +/// A tensor with the same shape as the input tensor and corresponding indices, where +/// the elements are sorted by value and the indices map back to the original input tensor. +/// +/// # Remarks +/// +/// This is a fallback solution that used only when the backend doesn't have the corresponding implementation. +/// Ideally, it is supposed to be implemented by the backend and the backend implementation will be resolved +/// by static dispatch. It is not designed for direct usage by users, and not recommended to import +/// or use this function directly. +pub fn sort_with_indices>( + tensor: K::Primitive, + dim: usize, + descending: bool, +) -> (K::Primitive, IntTensor) { + let device = K::device(&tensor); + let msg = "Failed to synchronously read tensor data. This operation is not supported until this backend has a GPU sorting implementation."; + let data = try_read_sync(K::into_data_async(tensor)) + .expect(msg) + .expect(msg); + + let (values, indices) = sort_dispatch_dtype!(sort_data_with_indices, data, dim, descending); + + ( + K::from_data(values, &device), + B::int_from_data(indices, &device), + ) +} + +fn sort_data_with_indices( + mut data: TensorData, + dim: usize, + descending: bool, +) -> (TensorData, TensorData) { + let dims = data.shape.clone(); + let mut indices_data = dim_indices::(&dims, dim); + let data_slice = data.as_mut_slice().unwrap(); + if dims.len() == 1 { + // 1D sort + indices_data.sort_unstable_by(|&a, &b| { + compare( + &data_slice[a.elem::() as usize], + &data_slice[b.elem::() as usize], + descending, + ) + }); + + // Permute data in-place by the sorted indices + let mut indices = indices_data + .clone() + .iter() + .map(|i| i.elem::() as usize) + .collect::>(); + for idx in 0..indices.len() { + if indices[idx] != idx { + let mut current_idx = idx; + loop { + let target_idx = indices[current_idx]; + indices[current_idx] = current_idx; + if indices[target_idx] == target_idx { + // correct position + break; + } + + // Permute data by indices + data_slice.swap(current_idx, target_idx); + current_idx = target_idx; + } + } + } + } else { + sort_slice::( + data_slice, + &dims, + dim, + Some(&mut indices_data), + true, + descending, + ); + } + + (data, TensorData::new(indices_data, dims)) +} + +/// Returns the indices that sort the elements of the input `tensor` along a given dimension. +/// +/// This sort is unstable (i.e., may reorder equal elements). +/// +/// # Arguments +/// +/// * `tensor` - The input tensor. +/// * `dim` - The axis along which to sort. +/// * `descending` - The sorting order. +/// +/// # Returns +/// +/// A tensor with the same shape as the input tensor the indices map back to the original input tensor. +/// +/// # Remarks +/// +/// This is a fallback solution that used only when the backend doesn't have the corresponding implementation. +/// Ideally, it is supposed to be implemented by the backend and the backend implementation will be resolved +/// by static dispatch. It is not designed for direct usage by users, and not recommended to import +/// or use this function directly. +pub fn argsort>( + tensor: K::Primitive, + dim: usize, + descending: bool, +) -> IntTensor { + let device = K::device(&tensor); + let msg = "Failed to synchronously read tensor data. This operation is not supported until this backend has a GPU sorting implementation."; + let data = try_read_sync(K::into_data_async(tensor)) + .expect(msg) + .expect(msg); + + let data = sort_dispatch_dtype!(argsort_data, data, dim, descending); + B::int_from_data(data, &device) +} + +fn argsort_data( + mut data: TensorData, + dim: usize, + descending: bool, +) -> TensorData { + let dims = data.shape.clone(); + let mut indices_data = dim_indices::(&dims, dim); + if dims.len() == 1 { + // 1D sort + let slice = data.as_slice::().unwrap(); + indices_data.sort_unstable_by(|&a, &b| { + compare( + &slice[a.elem::() as usize], + &slice[b.elem::() as usize], + descending, + ) + }); + } else { + sort_slice::( + data.as_mut_slice().unwrap(), + &dims, + dim, + Some(&mut indices_data), + false, + descending, + ); + } + + TensorData::new(indices_data, dims) +} + +/// Sort the elements by value along a given dimension. +/// +/// When `indices` are not provided, the `data` is sorted. +/// Otherwise, the `indices` are sorted based on the value of the elements in `data`, +/// and if `permute_both` is enabled then the data is also sorted. +/// +/// This sort is unstable (i.e., may reorder equal elements). +fn sort_slice( + data: &mut [E], + dims: &[usize], + dim: usize, + mut indices: Option<&mut [IntElem]>, + permute_both: bool, + descending: bool, +) { + let ndims = dims.len(); + let strides = compute_strides(dims); + // Dimensions to access elements to sort + let mut sort_dims = dims.to_vec(); + sort_dims[dim] = 1; + let strides_out = compute_strides(&sort_dims); + + // Number of groups to sort + let num_sorts: usize = dims + .iter() + .enumerate() + .filter(|&(i, _)| i != dim) + .map(|(_, d)| d) + .product(); + + // TODO: run each sort in parallel + // run_par!(|| { + // iter_range_par!(0, num_sorts).for_each(|id| {...}) + for id in 0..num_sorts { + let mut index_offset = 0; + let mut stride_dim = 0; + let mut shape_dim = 0; + for d in 0..ndims { + let stride_input = strides[d]; + let stride_output = strides_out[d]; + let shape_output = sort_dims[d]; + + let num_block = id / stride_output % shape_output; + + if d != dim { + index_offset += num_block * stride_input; + } else { + let shape_input = dims[d]; + stride_dim = stride_input; + shape_dim = shape_input; + index_offset += num_block; + } + } + + // For each group, sort the indices based on the element values + // NOTE: Sorting methods like `sort_unstable_by` are in-place but we need to sort + // different views/groups of the underlying data, so the swap is performed on the elements + // of the (flat index, element value) collection. + let mut elements = (0..shape_dim) + .map(|d| { + let flat_index = d * stride_dim + index_offset; + let elem = data[flat_index]; + (d, flat_index, elem) + }) + .collect::>(); + + elements.sort_unstable_by(|&(_, _, a), &(_, _, b)| compare(&a, &b, descending)); + + // Permute data in-place by the sorted indices + for idx in 0..elements.len() { + if elements[idx].0 != idx { + let mut current_idx = idx; + loop { + let target_idx = elements[current_idx].0; + elements[current_idx].0 = current_idx; + if elements[target_idx].0 == target_idx { + // correct position + break; + } + + if indices.is_none() || permute_both { + // Permute data by indices + data.swap(elements[current_idx].1, elements[target_idx].1); + } + + if let Some(ref mut indices_data) = indices { + // Permute data element indices + indices_data.swap(elements[current_idx].1, elements[target_idx].1); + } + + current_idx = target_idx; + } + } + } + } +} + +/// Computes the steps for each dimension when traversing an array. +fn compute_strides(dims: &[usize]) -> Vec { + let mut strides = vec![0; dims.len()]; + let mut current = 1; + + dims.iter().enumerate().rev().for_each(|(index, val)| { + strides[index] = current; + current *= val; + }); + + strides +} + +/// Generates the indices for each element along the specified dimension. +fn dim_indices(dims: &[usize], dim: usize) -> Vec> { + if dims.len() == 1 { + (0..dims[dim]) + .map(|i| (i as i64).elem::>()) + .collect::>() + } else { + // Dimension indices tensor + let numel_leading_dims: usize = dims[..dim].iter().product(); + let numel_trailing_dims: usize = dims[dim + 1..].iter().product(); + (0..dims[dim]) + .map(|i| [(i as i64).elem::>()].repeat(numel_trailing_dims)) + .collect::>() + .concat() + .repeat(numel_leading_dims) + } +} + +/// Compare two elements +fn compare(a: &E, b: &E, descending: bool) -> Ordering { + if descending { b.cmp(a) } else { a.cmp(b) } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/tensor.rs new file mode 100644 index 0000000..18245b7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/tensor.rs @@ -0,0 +1,1650 @@ +use super::cat::cat_with_slice_assign; +use super::grid_sample::float_grid_sample_2d_ref; +use super::repeat_dim::repeat_with_slice_assign; +use super::sort::{argsort, sort, sort_with_indices}; +use crate::ops::GridSampleOptions; +use crate::tensor::{BoolTensor, Device, Float, FloatTensor, IntTensor}; +use crate::{Backend, Distribution, TensorData}; +use crate::{ExecutionError, Scalar, TensorMetadata, TensorPrimitive}; +use alloc::vec::Vec; +use burn_std::{FloatDType, Shape, Slice}; + +/// Operations on float tensors. +pub trait FloatTensorOps { + /// Creates a new tensor from the data structure. + /// + /// # Arguments + /// + /// * `data` - The data structure. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the given data. + fn float_from_data(data: TensorData, device: &Device) -> FloatTensor; + + /// Creates a new tensor with random values. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `distribution` - The distribution to sample from. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// The tensor with the given shape and random values. + fn float_random(shape: Shape, distribution: Distribution, device: &Device) + -> FloatTensor; + + /// Creates a new tensor with zeros. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor with the given shape and zeros. + fn float_zeros(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + Self::float_from_data(TensorData::full_dtype(shape, 0., dtype.into()), device) + } + + /// Creates a new tensor with ones. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor with the given shape and ones. + fn float_ones(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + Self::float_from_data(TensorData::full_dtype(shape, 1., dtype.into()), device) + } + + /// Creates a tensor filled with given value. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `fill_value` - The value with which to fill the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor filled with given value + fn float_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: FloatDType, + ) -> FloatTensor { + Self::float_from_data( + TensorData::full_dtype(shape, fill_value, dtype.into()), + device, + ) + } + + /// Converts the tensor to a data structure. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The data structure with the tensor's data. + fn float_into_data( + tensor: FloatTensor, + ) -> impl Future> + Send; + + /// Gets the device of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The device of the tensor. + fn float_device(tensor: &FloatTensor) -> Device; + + /// Moves the tensor to the given device. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `device` - The device to move the tensor to. + /// + /// # Returns + /// + /// The tensor on the given device. + fn float_to_device(tensor: FloatTensor, device: &Device) -> FloatTensor; + + /// Converts float tensor to int tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The int tensor with the same data as the float tensor. + fn float_into_int(tensor: FloatTensor) -> IntTensor; + + /// Creates an empty tensor with the given shape. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The empty tensor with the given shape. + fn float_empty(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor; + + /// Repeat the tensor along the given dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension to repeat. + /// * `times` - The number of times to repeat the dimension. + /// + /// # Returns + /// + /// The tensor with the given dimension repeated. + fn float_repeat_dim(tensor: FloatTensor, dim: usize, times: usize) -> FloatTensor { + repeat_with_slice_assign::(TensorPrimitive::Float(tensor), dim, times).tensor() + } + + /// Adds two tensors together. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of adding the two tensors together. + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Adds a scalar to a tensor. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of adding the scalar to the tensor. + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor; + + /// Clamps a tensor under a minimum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn float_clamp_min(tensor: FloatTensor, min: Scalar) -> FloatTensor { + // Default implementation + let mask = Self::float_lower_elem(tensor.clone(), min); + B::float_mask_fill(tensor, mask, min) + } + + /// Clamps a tensor over a maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn float_clamp_max(tensor: FloatTensor, max: Scalar) -> FloatTensor { + // Default implementation + let mask = Self::float_greater_elem(tensor.clone(), max); + B::float_mask_fill(tensor, mask, max) + } + + /// Clamps a tensor between a minimum and maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// The clamped tensor. + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + // Default implementation + Self::float_clamp_min(Self::float_clamp_max(tensor, max), min) + } + + /// Subtracts two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of subtracting the two tensors. + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Subtracts a scalar from a tensor. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of subtracting the scalar from the tensor. + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor; + + /// Multiplies two tensors together element-wise. + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Multiplies a tensor by a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of multiplying the tensor by the scalar. + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor; + + /// Divides two tensors element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of dividing the two tensors. + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Divides a tensor by a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of dividing the tensor by the scalar. + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor; + + /// Computes the remainder of division between two tensors element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The element-wise remainder when dividing `lhs` by `rhs`. + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Computes the modulus of a tensor given a scalar. + /// + /// # Arguments + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The result of applying the modulus of the scalar to the tensor. + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor; + + /// Multiplies two tensors together using matrix multiplication. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The result of multiplying the two tensors together using matrix multiplication. + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Computes the cross product of two tensors along a given dimension. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// * `dim` - The dimension to compute the cross product along. + /// + /// # Returns + /// + /// The cross product of the two tensors. + fn float_cross(lhs: FloatTensor, rhs: FloatTensor, dim: usize) -> FloatTensor; + + /// Negates a tensor element-wise. + fn float_neg(tensor: FloatTensor) -> FloatTensor { + Self::float_mul_scalar(tensor, (-1f32).into()) + } + + /// Calculates the reciprocals element-wise + fn float_recip(tensor: FloatTensor) -> FloatTensor; + + /// Transposes a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to transpose. + /// + /// # Returns + /// + /// The transposed tensor. + fn float_transpose(tensor: FloatTensor) -> FloatTensor { + let ndims = tensor.shape().num_dims(); + Self::float_swap_dims(tensor, ndims - 2, ndims - 1) + } + + /// Swaps two dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to swap the dimensions of. + /// * `dim1` - The first dimension to swap. + /// * `dim2` - The second dimension to swap. + /// + /// # Returns + /// + /// The tensor with the dimensions swapped. + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor; + + /// Permutes the dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to permute the dimensions of. + /// * `axes` - The new order of the dimensions. + /// # Returns + /// + /// The tensor with the dimensions permuted. + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor; + + /// Reverse the order of elements in a tensor along the given axes. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to reverse. + /// * `axes` - The axes to reverse. + /// + /// The tensor with the elements reversed. + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor; + + /// Reshapes a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to reshape. + /// * `shape` - The new shape of the tensor. + /// + /// # Returns + /// + /// The tensor with the new shape. + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor; + + /// Gather elements from a tensor. + /// + /// # Arguments + /// + /// * `dim` - The dimension to gather from. + /// * `tensor` - The tensor to gather from. + /// * `indices` - The indices to gather. + /// + /// # Returns + /// + /// The gathered elements. + fn float_gather(dim: usize, tensor: FloatTensor, indices: IntTensor) -> FloatTensor; + + /// Scatter elements into a tensor using sum reduction. + /// + /// # Arguments + /// + /// * `dim` - The dimension to scatter into. + /// * `tensor` - The tensor to scatter into. + /// * `indices` - The indices to scatter into. + /// * `value` - The value to scatter. + /// + /// # Returns + /// + /// The tensor with the scattered elements. + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor; + + /// Select tensor elements along the given dimension corresponding for the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices to select. + /// + /// # Returns + /// + /// The selected elements. + fn float_select(tensor: FloatTensor, dim: usize, indices: IntTensor) -> FloatTensor; + + /// Assign the selected elements along the given dimension corresponding for the given indices + /// to the given value using sum reduction. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `dim` - The dimension to select from. + /// * `indices` - The indices to select. + /// * `value` - The value to assign. + /// + /// # Returns + /// + /// The tensor with the selected elements assigned to the given value. + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor; + + /// Select tensor elements corresponding to the given slices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `slices` - The slices specifying ranges and steps for each dimension. + /// + /// # Returns + /// + /// The selected elements in a new tensor. + /// + /// # Note + /// + /// Empty slices (where start >= end) are handled at the high-level tensor API and will not + /// be passed to this method. Backend implementations do not need to handle empty slices. + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor; + + /// Assign the selected elements corresponding to the given slices to the given value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `ranges` - The ranges to select. + /// * `value` - The value to assign. + /// + /// # Returns + /// + /// The tensor with the selected elements assigned to the given value. + /// + /// # Note + /// + /// Empty slice assignments (where any slice range produces 0 elements) are handled at the + /// high-level tensor API and will not be passed to this method. Backend implementations do + /// not need to handle empty slice assignments. + fn float_slice_assign( + tensor: FloatTensor, + slices: &[Slice], + value: FloatTensor, + ) -> FloatTensor; + + /// Update the given tensor with the value tensor where the mask is true. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `mask` - The boolean mask to select with. + /// * `value` - The value to assign to the selected elements from the value tensor. + /// + /// # Returns + /// + /// The tensor with the selected elements assigned to the given value. + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + value: FloatTensor, + ) -> FloatTensor; + + /// Update the given tensor with the value where the mask is true. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `mask` - The boolean mask to select with. + /// * `value` - The value to assign to the selected elements. + /// + /// # Returns + /// + /// The tensor with the selected elements assigned to the given value. + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor; + + /// Equal comparison of two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor; + + /// Element-wise non-equality comparison. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_not_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let equal_tensor = B::float_equal(lhs, rhs); + B::bool_not(equal_tensor) + } + + /// Equal comparison of a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor; + + /// Element-wise non-equality comparison with a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_not_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let equal_tensor = B::float_equal_elem(lhs, rhs); + B::bool_not(equal_tensor) + } + + /// Greater than comparison of two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor; + + /// Greater than comparison of a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor; + + /// Greater than or equal comparison of two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor; + + /// Greater than or equal comparison of a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor; + + /// Less than comparison of two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor; + + /// Less than comparison of a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor; + + /// Less than or equal comparison of two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor; + + /// Less than or equal comparison of a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the result of the comparison. + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor; + + /// Detaches a tensor from the computation graph. + fn float_detach(tensor: FloatTensor) -> FloatTensor { + // Should only be overridden by autodiff backends. + tensor + } + + /// Sets the `require_grad` flag of a tensor. + fn float_set_require_grad(tensor: FloatTensor, _require_grad: bool) -> FloatTensor { + // Should only be overridden by autodiff backends. + tensor + } + + /// Returns the `require_grad` flag of a tensor. + fn float_is_require_grad(_tensor: &FloatTensor) -> bool { + // Should only be overridden by autodiff backends. + false + } + + /// Sum of all elements in a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// + /// # Returns + /// + /// A scalar tensor with the sum of all elements in `tensor`. + fn float_sum(tensor: FloatTensor) -> FloatTensor; + + /// Sum of all elements in a tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// * `dim` - The dimension along which to sum. + /// + /// # Returns + /// + /// A tensor with the sum of all elements in `tensor` along `dim`. + fn float_sum_dim(tensor: FloatTensor, dim: usize) -> FloatTensor; + + /// Product of all elements in a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to product. + /// + /// # Returns + /// + /// A scalar tensor with the product of all elements in `tensor`. + fn float_prod(tensor: FloatTensor) -> FloatTensor { + // Product of all elements in a tensor + B::float_exp(B::float_sum(B::float_log(tensor))) + } + + /// Product of all elements in a tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to product. + /// + /// # Returns + /// + /// A tensor with the product of all elements in `tensor` along `dim`. + fn float_prod_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + // Product of all elements in a tensor along a dimension + B::float_exp(B::float_sum_dim(B::float_log(tensor), dim)) + } + + /// Mean of all elements in a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to mean. + /// + /// # Returns + /// + /// A scalar tensor with the mean of all elements in `tensor`. + fn float_mean(tensor: FloatTensor) -> FloatTensor { + let num_elems = tensor.shape().num_elements() as f32; + B::float_div_scalar(B::float_sum(tensor), num_elems.into()) + } + + /// Mean of all elements in a tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to mean. + /// * `dim` - The dimension along which to mean. + /// + /// # Returns + /// + /// A tensor with the mean of all elements in `tensor` along `dim`. + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor; + + /// Computes the cumulative sum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative sum of. + /// * `dim` - The dimension along which to compute the cumulative sum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the cumulative sum + /// of all elements up to and including that position along the dimension. + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor; + + /// Computes the cumulative product of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative product of. + /// * `dim` - The dimension along which to compute the cumulative product. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the cumulative product + /// of all elements up to and including that position along the dimension. + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor; + + /// Computes the cumulative minimum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative minimum of. + /// * `dim` - The dimension along which to compute the cumulative minimum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the minimum + /// of all elements up to and including that position along the dimension. + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor; + + /// Computes the cumulative maximum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative maximum of. + /// * `dim` - The dimension along which to compute the cumulative maximum. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the maximum + /// of all elements up to and including that position along the dimension. + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor; + + /// Converts a tensor to another floating point data type. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to convert. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// A tensor with the same values as `tensor` but in the target floating point data type. + fn float_cast(tensor: FloatTensor, dtype: FloatDType) -> FloatTensor; + + /// Returns a new tensor with exponential values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to exponentiate. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with exponential values. + fn float_exp(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with natural logarithm values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the logarithm of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with natural logarithm values. + fn float_log(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with logarithm values of (1 + Xi). + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the logarithm of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with logarithm values of (1 + Xi). + fn float_log1p(tensor: FloatTensor) -> FloatTensor; + + /// Element-wise power with a FloatTensor. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side tensor. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the power of the elements of `rhs`. + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Element-wise power with an IntTensor. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side floatTensor. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. Result is an IntTensor. + fn float_powi(lhs: FloatTensor, rhs: IntTensor) -> FloatTensor { + Self::float_powf(lhs, B::int_into_float(rhs)) + } + + /// Raises a tensor to the power of an int scalar. + /// + /// # Backend Implementors Note + /// + /// A number of common exponent cases can be implemented with operations + /// which are much cheaper than generic exponentiation. + /// + /// This (`Backend` impl overridable) operation handles generic optimizations + /// for several common integer exponent cases; and then dispatches to + /// the (`Backend` impl overridable) [`Self::float_powi_scalar_impl`] + /// operation to handle the generic case. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. + fn float_powi_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + match rhs.elem::() { + 0 => Self::float_ones(lhs.shape(), &B::float_device(&lhs), lhs.dtype().into()), + 1 => lhs, + 2 => B::float_mul(lhs.clone(), lhs), + -1 => Self::float_recip(lhs), + -2 => Self::float_recip(B::float_mul(lhs.clone(), lhs)), + _ => Self::float_powi_scalar_impl(lhs, rhs), + } + } + + /// Raises a tensor to the power of an int scalar. + /// + /// # Backend Implementors Note + /// + /// This is the generic implementation of integer exponentiation + /// called by [`Self::float_powi_scalar`] in the fallback case. + /// + /// As a general rule, this should not be called directly. + /// + /// # Arguments + /// + /// * `lhs` - The left-hand side tensor. + /// * `rhs` - The right-hand side scalar. + /// + /// # Returns + /// + /// The elements of `lhs` raised to the value of `rhs`. + fn float_powi_scalar_impl(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + // Avoid a recursive loop by deferring directly to float_powf_scalar_impl. + Self::float_powf_scalar_impl(lhs, rhs) + } + + /// Returns a new tensor with values raised to the power of float `value`. + /// + /// # Backend Implementors Note + /// + /// This (`Backend` impl overridable) operation dispatches integer exponentiation + /// to [`Self::float_powi_scalar`], and the remaining non-integer exponent cases to + /// the (`Backend` impl overridable) [`Self::float_powf_scalar_impl`] + /// operation to handle the generic case. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to exponentiate. + /// * `value` - The exponent. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with values raised to the power of `value`. + fn float_powf_scalar(tensor: FloatTensor, value: Scalar) -> FloatTensor { + if let Some(exp) = value.try_as_integer() { + Self::float_powi_scalar(tensor, exp) + } else { + Self::float_powf_scalar_impl(tensor, value) + } + } + + /// Returns a new tensor with values raised to the power of float `value`. + /// + /// # Backend Implementors Note + /// + /// This is the generic implementation of integer exponentiation + /// called by [`Self::float_powf_scalar`] in the fallback case. + /// + /// This is the minimal required support a `Backend` must implement + /// for exponentiation. + /// + /// As a general rule, this should not be called directly. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to exponentiate. + /// * `value` - The exponent. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with values raised to the power of `value`. + fn float_powf_scalar_impl(tensor: FloatTensor, value: Scalar) -> FloatTensor; + + /// Returns a new tensor with square root values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the square root of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with square root values. + fn float_sqrt(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with absolute values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take absolute value of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with absolute values. + fn float_abs(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with cosine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the cosine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with cosine values. + fn float_cos(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with sine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the sine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with sine values. + fn float_sin(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with tangent values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the tangent of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with tangent values. + fn float_tan(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with hyperbolic cosine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the hyperbolic cosine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with hyperbolic cosine values. + fn float_cosh(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with hyperbolic sine values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the hyperbolic sine of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with hyperbolic sine values. + fn float_sinh(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with hyperbolic tangent values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the hyperbolic tangent of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with hyperbolic tangent values. + fn float_tanh(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with inverse cosine values. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with inverse cosine values. + fn float_acos(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with inverse hyperbolic cosine values. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with inverse hyperbolic cosine values. + fn float_acosh(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with inverse sine values. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with inverse sine values. + fn float_asin(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with inverse hyperbolic sine values. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with inverse hyperbolic sine values. + fn float_asinh(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with the inverse tangent values. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with the inverse tangent values. + fn float_atan(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with the inverse hyperbolic tangent values. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with the inverse hyperbolic tangent values. + fn float_atanh(tensor: FloatTensor) -> FloatTensor; + + /// Returns a tensor with the four-quadrant inverse tangent values of `y` and `x`. + /// + /// # Arguments + /// + /// * `lhs` - The tensor with y coordinates. + /// * `rhs` - The tensor with x coordinates. + /// + /// # Returns + /// + /// A tensor with the four-quadrant inverse tangent values. + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with rounded values. + /// + /// This function should implement the [round half to even](https://en.wikipedia.org/wiki/Rounding#Rounding_half_to_even) + /// strategy, with halfway cases rounded to the nearest even integer value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to be rounded. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with rounded values. + fn float_round(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with floored values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to be floored. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with floored values. + fn float_floor(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with ceiled values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to be ceiled. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with ceiled values. + fn float_ceil(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with truncated values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to be truncated. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with truncated values. + fn float_trunc(tensor: FloatTensor) -> FloatTensor; + + /// Returns a new tensor with the error function values. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to take the error function of. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` with error function values. + fn float_erf(tensor: FloatTensor) -> FloatTensor; + + /// Concatenates tensors along a dimension. + /// + /// # Arguments + /// + /// * `tensors` - The tensors to concatenate. + /// * `dim` - The dimension along which to concatenate. + /// + /// # Returns + /// + /// A tensor with the concatenated tensors along `dim`. + /// + /// # Note + /// + /// Empty tensors (where the concatenation dimension has size 0) are filtered out at the + /// high-level tensor API and will not be passed to this method. Backend implementations do + /// not need to handle empty tensors. + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + cat_with_slice_assign::( + tensors.into_iter().map(TensorPrimitive::Float).collect(), + dim, + ) + .tensor() + } + + /// Gets the indices of the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the indices of the maximum elements of `tensor` along `dim`. + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor; + + /// Gets the indices of the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// * `dim` - The dimension along which to get the minimum elements. + /// + /// # Returns + /// + /// A tensor with the indices of the minimum elements of `tensor` along `dim`. + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor; + + /// Gets the maximum element of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// + /// # Returns + /// + /// A tensor with the maximum element of `tensor`. + fn float_max(tensor: FloatTensor) -> FloatTensor { + let shape = tensor.shape(); + let tensor = B::float_reshape(tensor, Shape::new([shape.num_elements()])); + + B::float_max_dim(tensor, 0) + } + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the maximum elements of `tensor` along `dim`. + fn float_max_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + let index = B::float_argmax(tensor.clone(), dim); + + B::float_gather(dim, tensor, index) + } + + /// Gets the maximum elements of a tensor along an axis and their indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tuple with the maximum elements of `tensor` along `dim` and their indices. + fn float_max_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + let index = B::float_argmax(tensor.clone(), dim); + let values = B::float_gather(dim, tensor, index.clone()); + + (values, index) + } + + /// Gets the minimum element of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// + /// # Returns + /// + /// A tensor with the minimum element of `tensor`. + fn float_min(tensor: FloatTensor) -> FloatTensor { + let shape = tensor.shape(); + let tensor = B::float_reshape(tensor, Shape::new([shape.num_elements()])); + + B::float_min_dim(tensor, 0) + } + + /// Gets the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// * `dim` - The dimension along which to get the minimum elements. + /// + /// # Returns + /// + /// A tensor with the minimum elements of `tensor` along `dim`. + fn float_min_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + let index = B::float_argmin(tensor.clone(), dim); + + B::float_gather(dim, tensor, index) + } + + /// Gets the minimum elements of a tensor along an axis and their indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements of. + /// * `dim` - The dimension along which to get the minimum elements. + /// + /// # Returns + /// + /// A tuple with the minimum elements of `tensor` along `dim` and their indices. + fn float_min_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + let index = B::float_argmin(tensor.clone(), dim); + let values = B::float_gather(dim, tensor, index.clone()); + + (values, index) + } + + /// Gets the maximum absolute element of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// + /// # Returns + /// + /// A tensor with the maximum element of `tensor`. + fn float_max_abs(tensor: FloatTensor) -> FloatTensor { + let shape = tensor.shape(); + let tensor = B::float_reshape(tensor, Shape::new([shape.num_elements()])); + + B::float_max_abs_dim(tensor, 0) + } + + /// Gets the maximum absolute elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements of. + /// * `dim` - The dimension along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the maximum elements of `tensor` along `dim`. + fn float_max_abs_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + B::float_max_dim(B::float_abs(tensor), dim) + } + + /// Tests if any element in the float `tensor` evaluates to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with a single element, True if any element in the tensor is True, False otherwise. + fn float_any(tensor: FloatTensor) -> BoolTensor { + let bool_tensor = B::float_equal_elem(tensor, 0f32.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::float_sum(B::bool_into_float(bool_tensor)); + B::float_greater_elem(sum, 0f32.into()) + } + + /// Tests if any element in the float `tensor` evaluates to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if any element along this dim in the + /// input evaluates to True, False otherwise. + fn float_any_dim(tensor: FloatTensor, dim: usize) -> BoolTensor { + let bool_tensor = B::float_equal_elem(tensor, 0f32.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::float_sum_dim(B::bool_into_float(bool_tensor), dim); + B::float_greater_elem(sum, 0f32.into()) + } + + /// Tests if all elements in the float `tensor` evaluate to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with a single element, True if all elements in the input tensor + /// evaluate to True, False otherwise. + fn float_all(tensor: FloatTensor) -> BoolTensor { + let num_elems = tensor.shape().num_elements() as f32; + let bool_tensor = B::float_equal_elem(tensor, 0f32.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::float_sum(B::bool_into_float(bool_tensor)); + B::float_equal_elem(sum, num_elems.into()) + } + + /// Tests if all elements in the float `tensor` evaluate to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same size as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if all elements along this dim in the input + /// evaluates to True, False otherwise. + fn float_all_dim(tensor: FloatTensor, dim: usize) -> BoolTensor { + let num_elems = tensor.shape()[dim] as f32; + let bool_tensor = B::float_equal_elem(tensor, 0f32.into()); + let bool_tensor = B::bool_not(bool_tensor); + let sum = B::float_sum_dim(B::bool_into_float(bool_tensor), dim); + B::float_equal_elem(sum, num_elems.into()) + } + + /// Returns the signs of the float `tensor`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to extract the signs from. + /// + /// # Returns + /// + /// A tensor with the same shape as `tensor` containing the signs of the elements of `tensor`. + fn float_sign(tensor: FloatTensor) -> FloatTensor { + let zeros = B::float_zeros( + tensor.shape(), + &B::float_device(&tensor), + tensor.dtype().into(), + ); + let less_than_zero = B::float_lower_elem(tensor.clone(), 0f32.into()); + let greater_than_zero = B::float_greater_elem(tensor, 0f32.into()); + + let mut result = B::float_mask_fill(zeros, less_than_zero, (-1f32).into()); + result = B::float_mask_fill(result, greater_than_zero, 1f32.into()); + result + } + + /// Broadcasts the float `tensor` to the given `shape`. + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor; + + /// Sort the elements of the input `tensor` by value in along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where the elements are sorted by value. + fn float_sort(tensor: FloatTensor, dim: usize, descending: bool) -> FloatTensor { + sort::(TensorPrimitive::Float(tensor), dim, descending).tensor() + } + + /// Sort the elements of the input `tensor` by value in along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor and corresponding indices, where + /// the elements are sorted by value and the indices map back to the original input tensor. + fn float_sort_with_indices( + tensor: FloatTensor, + dim: usize, + descending: bool, + ) -> (FloatTensor, IntTensor) { + let (values, indices) = + sort_with_indices::(TensorPrimitive::Float(tensor), dim, descending); + (values.tensor(), indices) + } + + /// Returns the indices that sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor the indices map back to the original input tensor. + fn float_argsort(tensor: FloatTensor, dim: usize, descending: bool) -> IntTensor { + argsort::(TensorPrimitive::Float(tensor), dim, descending) + } + + /// Samples tensor as a two-dimensional spatial grid of (possibly multi-channel) values, + /// using the given locations in [-1, 1]. + /// + /// # Arguments + /// + /// * `tensor` - The tensor being sampled from, must be contiguous with shape (N, C, H_in, W_in) + /// * `grid` - A tensor of locations, with shape (N, H_out, W_out, 2). Values are [-1, 1]. + /// A [x = -1, y = -1] means top-left, and [x = 1, y = 1] means bottom-right + /// * `options` - Grid sampling options (mode, padding_mode, align_corners) + /// + /// # Returns + /// + /// A tensor with shape (N, C, H_out, W_out) + fn float_grid_sample_2d( + tensor: FloatTensor, + grid: FloatTensor, + options: GridSampleOptions, + ) -> FloatTensor { + float_grid_sample_2d_ref::(tensor, grid, options) + } + + /// Unfold windows along a dimension. + /// + /// Returns a view of the tensor with all complete windows of size `size` in dimension `dim`; + /// where windows are advanced by `step` at each index. + /// + /// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor to unfold; of shape ``[pre=..., dim shape, post=...]`` + /// * `dim` - the selected dim. + /// * `size` - the size of each unfolded window. + /// * `step` - the step between each window. + /// + /// # Returns + /// + /// A tensor view with shape ``[pre=..., windows, size, post=...]``. + fn float_unfold(tensor: FloatTensor, dim: usize, size: usize, step: usize) + -> FloatTensor; + + /// Returns a new tensor with boolean elements indicating whether each element of the input is NaN. + /// + /// # Returns + /// + /// A boolean tensor where `true` indicates NaN and `false` indicates a non-NaN value. + fn float_is_nan(tensor: FloatTensor) -> BoolTensor { + // Check if the input tensor is NaN by comparing it to itself + // NaN is the only value that is not equal to itself + B::float_not_equal(tensor.clone(), tensor) + } + + /// Returns a new tensor with boolean elements indicating whether each element of the input is infinite (either +INF or -INF). + /// + /// # Returns + /// + /// A boolean tensor where `true` indicates that the value is infinite + fn float_is_inf(tensor: FloatTensor) -> BoolTensor { + B::float_equal_elem(B::float_abs(tensor), f64::INFINITY.into()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/transaction.rs new file mode 100644 index 0000000..5f2814f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/ops/transaction.rs @@ -0,0 +1,139 @@ +use alloc::vec::Vec; +use core::future::Future; + +use crate::tensor::{BoolTensor, FloatTensor, IntTensor, QuantizedTensor}; +use crate::{Backend, ExecutionError, TensorData, TensorPrimitive}; + +enum Order { + Float(usize), + QFloat(usize), + Int(usize), + Bool(usize), +} + +#[derive(Default)] +/// Contains all tensor primitives that are going to be read. +pub struct TransactionPrimitive { + /// Float tensors. + pub read_floats: Vec>, + /// Quantized tensors. + pub read_qfloats: Vec>, + /// Int tensors. + pub read_ints: Vec>, + /// Bool tensors. + pub read_bools: Vec>, + orders: Vec, +} + +#[derive(Default)] +/// Contains all [data](TensorData) related to a [transaction](TransactionPrimitive). +pub struct TransactionPrimitiveData { + /// Float tensor data. + pub read_floats: Vec, + /// Quantized tensor data. + pub read_qfloats: Vec, + /// Int tensor data. + pub read_ints: Vec, + /// Bool tensor data. + pub read_bools: Vec, +} + +/// Operations that are sync by nature and that can be batch together in transactions to improve +/// compute utilization with efficient laziness. +pub trait TransactionOps { + /// Executes a [transaction](TransactionPrimitive) and return its + /// [data](TransactionPrimitiveData). + fn tr_execute( + transaction: TransactionPrimitive, + ) -> impl Future> + Send { + async move { + let mut floats = Vec::new(); + let mut qfloats = Vec::new(); + let mut ints = Vec::new(); + let mut bools = Vec::new(); + + for t in transaction.read_floats { + floats.push(B::float_into_data(t).await?); + } + for t in transaction.read_qfloats { + qfloats.push(B::q_into_data(t).await?); + } + for t in transaction.read_ints { + ints.push(B::int_into_data(t).await?); + } + for t in transaction.read_bools { + bools.push(B::bool_into_data(t).await?); + } + + Ok(TransactionPrimitiveData { + read_floats: floats, + read_qfloats: qfloats, + read_ints: ints, + read_bools: bools, + }) + } + } +} + +impl TransactionPrimitive { + /// Creates a new transaction. + pub fn new( + read_floats: Vec>, + read_qfloats: Vec>, + read_ints: Vec>, + read_bools: Vec>, + ) -> Self { + Self { + read_floats, + read_qfloats, + read_ints, + read_bools, + orders: Vec::default(), + } + } + /// Executes the transaction asynchronously and returns the [data](TensorData) in the same order + /// in which they were [registered](crate::tensor::BasicOps::register_transaction). + pub async fn execute_async(mut self) -> Result, ExecutionError> { + let mut orders = Vec::new(); + core::mem::swap(&mut orders, &mut self.orders); + let result = B::tr_execute(self).await?; + + let mut floats: Vec<_> = result.read_floats.into_iter().map(Some).collect(); + let mut qfloats: Vec<_> = result.read_qfloats.into_iter().map(Some).collect(); + let mut ints: Vec<_> = result.read_ints.into_iter().map(Some).collect(); + let mut bools: Vec<_> = result.read_bools.into_iter().map(Some).collect(); + + Ok(orders + .into_iter() + .map(|order| match order { + Order::Float(index) => floats.get_mut(index).unwrap().take().unwrap(), + Order::QFloat(index) => qfloats.get_mut(index).unwrap().take().unwrap(), + Order::Int(index) => ints.get_mut(index).unwrap().take().unwrap(), + Order::Bool(index) => bools.get_mut(index).unwrap().take().unwrap(), + }) + .collect::>()) + } + + pub(crate) fn register_float(&mut self, tensor: TensorPrimitive) { + match tensor { + TensorPrimitive::Float(tensor) => { + self.orders.push(Order::Float(self.read_floats.len())); + self.read_floats.push(tensor); + } + TensorPrimitive::QFloat(tensor) => { + self.orders.push(Order::QFloat(self.read_qfloats.len())); + self.read_qfloats.push(tensor); + } + } + } + + pub(crate) fn register_int(&mut self, tensor: IntTensor) { + self.orders.push(Order::Int(self.read_ints.len())); + self.read_ints.push(tensor); + } + + pub(crate) fn register_bool(&mut self, tensor: BoolTensor) { + self.orders.push(Order::Bool(self.read_bools.len())); + self.read_bools.push(tensor); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/primitive.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/primitive.rs new file mode 100644 index 0000000..bdf66eb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/backend/primitive.rs @@ -0,0 +1,77 @@ +use crate::Backend; +use burn_std::quantization::{QuantAcc, QuantPropagation, QuantScheme}; +use burn_std::{DType, Shape}; + +#[derive(Debug, Clone)] +/// A primitive tensor representation. +pub enum TensorPrimitive { + /// Float tensor primitive. + Float(B::FloatTensorPrimitive), + /// Quantized float tensor primitive. + QFloat(B::QuantizedTensorPrimitive), +} + +impl TensorPrimitive { + /// Returns the full tensor representation. + pub fn tensor(self) -> B::FloatTensorPrimitive { + match self { + Self::QFloat(tensor) => B::dequantize(tensor), + Self::Float(tensor) => tensor, + } + } +} + +impl TensorMetadata for TensorPrimitive { + fn dtype(&self) -> DType { + match self { + TensorPrimitive::Float(tensor) => tensor.dtype(), + TensorPrimitive::QFloat(tensor) => tensor.dtype(), + } + } + + fn shape(&self) -> Shape { + match self { + TensorPrimitive::Float(tensor) => tensor.shape(), + TensorPrimitive::QFloat(tensor) => tensor.shape(), + } + } + + fn rank(&self) -> usize { + match self { + TensorPrimitive::Float(tensor) => tensor.rank(), + TensorPrimitive::QFloat(tensor) => tensor.rank(), + } + } +} + +/// Tensor metadata trait for tensor primitive. +pub trait TensorMetadata: Clone + Send + Sync + core::fmt::Debug { + /// The dtype of the tensor. + fn dtype(&self) -> DType; + /// The shape of the tensor. + fn shape(&self) -> Shape; + + /// The number of dimensions of the tensor. + fn rank(&self) -> usize { + self.shape().num_dims() + } +} + +/// Quantized tensor primitive. +pub trait QTensorPrimitive { + /// Returns the quantization settings for the given tensor. + fn scheme(&self) -> &QuantScheme; + /// The precision used for the accumulation in various kernels. + fn acc_precision(&self) -> QuantAcc { + QuantAcc::F32 + } + /// How quantization is propagated during computation. + fn propagation(&self) -> QuantPropagation { + QuantPropagation::Inhibit + } + + /// Returns the default tensor quantization scheme. + fn default_scheme() -> QuantScheme { + QuantScheme::default() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/compare.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/compare.rs new file mode 100644 index 0000000..1845dea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/compare.rs @@ -0,0 +1,427 @@ +use alloc::format; +use alloc::string::String; +use burn_std::{DType, bf16, f16}; +use num_traits::{Float, ToPrimitive}; + +use super::TensorData; +use crate::{Element, ElementOrdered}; + +/// The tolerance used to compare to floating point numbers. +/// +/// Generally, two numbers `x` and `y` are approximately equal if +/// +/// ```text +/// |x - y| < max(R * (|x + y|), A) +/// ``` +/// +/// where `R` is the relative tolerance and `A` is the absolute tolerance. +/// +/// +/// The most common way to initialize this struct is to use `Tolerance::::default()`. +/// In that case, the relative and absolute tolerances are computed using an heuristic based +/// on the EPSILON and MIN_POSITIVE values of the given floating point type `F`. +/// +/// Another common initialization is `Tolerance::::rel_abs(1e-4, 1e-5).set_half_precision_relative(1e-2)`. +/// This will use a sane default to manage values too close to 0.0 and +/// use different relative tolerances depending on the floating point precision. +#[derive(Debug, Clone, Copy)] +pub struct Tolerance { + relative: F, + absolute: F, +} + +impl Default for Tolerance { + fn default() -> Self { + Self::balanced() + } +} + +impl Tolerance { + /// Create a tolerance with strict precision setting. + pub fn strict() -> Self { + Self { + relative: F::from(0.00).unwrap(), + absolute: F::from(64).unwrap() * F::min_positive_value(), + } + } + /// Create a tolerance with balanced precision setting. + pub fn balanced() -> Self { + Self { + relative: F::from(0.005).unwrap(), // 0.5% + absolute: F::from(1e-5).unwrap(), + } + } + + /// Create a tolerance with permissive precision setting. + pub fn permissive() -> Self { + Self { + relative: F::from(0.01).unwrap(), // 1.0% + absolute: F::from(0.01).unwrap(), + } + } + /// When comparing two numbers, this uses both the relative and absolute differences. + /// + /// That is, `x` and `y` are approximately equal if + /// + /// ```text + /// |x - y| < max(R * (|x + y|), A) + /// ``` + /// + /// where `R` is the `relative` tolerance and `A` is the `absolute` tolerance. + pub fn rel_abs(relative: FF, absolute: FF) -> Self { + let relative = Self::check_relative(relative); + let absolute = Self::check_absolute(absolute); + + Self { relative, absolute } + } + + /// When comparing two numbers, this uses only the relative difference. + /// + /// That is, `x` and `y` are approximately equal if + /// + /// ```text + /// |x - y| < R * max(|x|, |y|) + /// ``` + /// + /// where `R` is the relative `tolerance`. + pub fn relative(tolerance: FF) -> Self { + let relative = Self::check_relative(tolerance); + + Self { + relative, + absolute: F::from(0.0).unwrap(), + } + } + + /// When comparing two numbers, this uses only the absolute difference. + /// + /// That is, `x` and `y` are approximately equal if + /// + /// ```text + /// |x - y| < A + /// ``` + /// + /// where `A` is the absolute `tolerance`. + pub fn absolute(tolerance: FF) -> Self { + let absolute = Self::check_absolute(tolerance); + + Self { + relative: F::from(0.0).unwrap(), + absolute, + } + } + + /// Change the relative tolerance to the given one. + pub fn set_relative(mut self, tolerance: FF) -> Self { + self.relative = Self::check_relative(tolerance); + self + } + + /// Change the relative tolerance to the given one only if `F` is half precision. + pub fn set_half_precision_relative(mut self, tolerance: FF) -> Self { + if core::mem::size_of::() == 2 { + self.relative = Self::check_relative(tolerance); + } + self + } + + /// Change the relative tolerance to the given one only if `F` is single precision. + pub fn set_single_precision_relative(mut self, tolerance: FF) -> Self { + if core::mem::size_of::() == 4 { + self.relative = Self::check_relative(tolerance); + } + self + } + + /// Change the relative tolerance to the given one only if `F` is double precision. + pub fn set_double_precision_relative(mut self, tolerance: FF) -> Self { + if core::mem::size_of::() == 8 { + self.relative = Self::check_relative(tolerance); + } + self + } + + /// Change the absolute tolerance to the given one. + pub fn set_absolute(mut self, tolerance: FF) -> Self { + self.absolute = Self::check_absolute(tolerance); + self + } + + /// Change the absolute tolerance to the given one only if `F` is half precision. + pub fn set_half_precision_absolute(mut self, tolerance: FF) -> Self { + if core::mem::size_of::() == 2 { + self.absolute = Self::check_absolute(tolerance); + } + self + } + + /// Change the absolute tolerance to the given one only if `F` is single precision. + pub fn set_single_precision_absolute(mut self, tolerance: FF) -> Self { + if core::mem::size_of::() == 4 { + self.absolute = Self::check_absolute(tolerance); + } + self + } + + /// Change the absolute tolerance to the given one only if `F` is double precision. + pub fn set_double_precision_absolute(mut self, tolerance: FF) -> Self { + if core::mem::size_of::() == 8 { + self.absolute = Self::check_absolute(tolerance); + } + self + } + + /// Checks if `x` and `y` are approximately equal given the tolerance. + pub fn approx_eq(&self, x: F, y: F) -> bool { + // See the accepted answer here + // https://stackoverflow.com/questions/4915462/how-should-i-do-floating-point-comparison + + // This also handles the case where both a and b are infinity so that we don't need + // to manage it in the rest of the function. + if x == y { + return true; + } + + let diff = (x - y).abs(); + let max = F::max(x.abs(), y.abs()); + + diff < self.absolute.max(self.relative * max) + } + + fn check_relative(tolerance: FF) -> F { + let tolerance = F::from(tolerance).unwrap(); + assert!(tolerance <= F::one()); + tolerance + } + + fn check_absolute(tolerance: FF) -> F { + let tolerance = F::from(tolerance).unwrap(); + assert!(tolerance >= F::zero()); + tolerance + } +} + +impl TensorData { + /// Asserts the data is equal to another data. + /// + /// # Arguments + /// + /// * `other` - The other data. + /// * `strict` - If true, the data types must the be same. + /// Otherwise, the comparison is done in the current data type. + /// + /// # Panics + /// + /// Panics if the data is not equal. + #[track_caller] + pub fn assert_eq(&self, other: &Self, strict: bool) { + if strict { + assert_eq!( + self.dtype, other.dtype, + "Data types differ ({:?} != {:?})", + self.dtype, other.dtype + ); + } + + match self.dtype { + DType::F64 => self.assert_eq_elem::(other), + DType::F32 | DType::Flex32 => self.assert_eq_elem::(other), + DType::F16 => self.assert_eq_elem::(other), + DType::BF16 => self.assert_eq_elem::(other), + DType::I64 => self.assert_eq_elem::(other), + DType::I32 => self.assert_eq_elem::(other), + DType::I16 => self.assert_eq_elem::(other), + DType::I8 => self.assert_eq_elem::(other), + DType::U64 => self.assert_eq_elem::(other), + DType::U32 => self.assert_eq_elem::(other), + DType::U16 => self.assert_eq_elem::(other), + DType::U8 => self.assert_eq_elem::(other), + DType::Bool => self.assert_eq_elem::(other), + DType::QFloat(q) => { + // Strict or not, it doesn't make sense to compare quantized data to not quantized data for equality + let q_other = if let DType::QFloat(q_other) = other.dtype { + q_other + } else { + panic!("Quantized data differs from other not quantized data") + }; + + // Data equality mostly depends on input quantization type, but we also check level + if q.value == q_other.value && q.level == q_other.level { + self.assert_eq_elem::(other) + } else { + panic!("Quantization schemes differ ({q:?} != {q_other:?})") + } + } + } + } + + #[track_caller] + fn assert_eq_elem(&self, other: &Self) { + let mut message = String::new(); + if self.shape != other.shape { + message += format!( + "\n => Shape is different: {:?} != {:?}", + self.shape, other.shape + ) + .as_str(); + } + + let mut num_diff = 0; + let max_num_diff = 5; + for (i, (a, b)) in self.iter::().zip(other.iter::()).enumerate() { + if !a.eq(&b) { + // Only print the first 5 different values. + if num_diff < max_num_diff { + message += format!("\n => Position {i}: {a} != {b}").as_str(); + } + num_diff += 1; + } + } + + if num_diff >= max_num_diff { + message += format!("\n{} more errors...", num_diff - max_num_diff).as_str(); + } + + if !message.is_empty() { + panic!("Tensors are not eq:{message}"); + } + } + + /// Asserts the data is approximately equal to another data. + /// + /// # Arguments + /// + /// * `other` - The other data. + /// * `tolerance` - The tolerance of the comparison. + /// + /// # Panics + /// + /// Panics if the data is not approximately equal. + #[track_caller] + pub fn assert_approx_eq(&self, other: &Self, tolerance: Tolerance) { + let mut message = String::new(); + if self.shape != other.shape { + message += format!( + "\n => Shape is different: {:?} != {:?}", + self.shape, other.shape + ) + .as_str(); + } + + let iter = self.iter::().zip(other.iter::()); + + let mut num_diff = 0; + let max_num_diff = 5; + + for (i, (a, b)) in iter.enumerate() { + //if they are both nan, then they are equally nan + let both_nan = a.is_nan() && b.is_nan(); + //this works for both infinities + let both_inf = + a.is_infinite() && b.is_infinite() && ((a > F::zero()) == (b > F::zero())); + + if both_nan || both_inf { + continue; + } + + if !tolerance.approx_eq(F::from(a).unwrap(), F::from(b).unwrap()) { + // Only print the first 5 different values. + if num_diff < max_num_diff { + let diff_abs = ToPrimitive::to_f64(&(a - b).abs()).unwrap(); + let max = F::max(a.abs(), b.abs()); + let diff_rel = diff_abs / ToPrimitive::to_f64(&max).unwrap(); + + let tol_rel = ToPrimitive::to_f64(&tolerance.relative).unwrap(); + let tol_abs = ToPrimitive::to_f64(&tolerance.absolute).unwrap(); + + message += format!( + "\n => Position {i}: {a} != {b}\n diff (rel = {diff_rel:+.2e}, abs = {diff_abs:+.2e}), tol (rel = {tol_rel:+.2e}, abs = {tol_abs:+.2e})" + ) + .as_str(); + } + num_diff += 1; + } + } + + if num_diff >= max_num_diff { + message += format!("\n{} more errors...", num_diff - 5).as_str(); + } + + if !message.is_empty() { + panic!("Tensors are not approx eq:{message}"); + } + } + + /// Asserts each value is within a given range. + /// + /// # Arguments + /// + /// * `range` - The range. + /// + /// # Panics + /// + /// If any value is not within the half-open range bounded inclusively below + /// and exclusively above (`start..end`). + pub fn assert_within_range(&self, range: core::ops::Range) { + for elem in self.iter::() { + if elem.cmp(&range.start).is_lt() || elem.cmp(&range.end).is_ge() { + panic!("Element ({elem:?}) is not within range {range:?}"); + } + } + } + + /// Asserts each value is within a given inclusive range. + /// + /// # Arguments + /// + /// * `range` - The range. + /// + /// # Panics + /// + /// If any value is not within the half-open range bounded inclusively (`start..=end`). + pub fn assert_within_range_inclusive( + &self, + range: core::ops::RangeInclusive, + ) { + let start = range.start(); + let end = range.end(); + + for elem in self.iter::() { + if elem.cmp(start).is_lt() || elem.cmp(end).is_gt() { + panic!("Element ({elem:?}) is not within range {range:?}"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_assert_appox_eq_limit() { + let data1 = TensorData::from([[3.0, 5.0, 6.0]]); + let data2 = TensorData::from([[3.03, 5.0, 6.0]]); + + data1.assert_approx_eq::(&data2, Tolerance::absolute(3e-2)); + data1.assert_approx_eq::(&data2, Tolerance::absolute(3e-2)); + } + + #[test] + #[should_panic] + fn should_assert_approx_eq_above_limit() { + let data1 = TensorData::from([[3.0, 5.0, 6.0]]); + let data2 = TensorData::from([[3.031, 5.0, 6.0]]); + + data1.assert_approx_eq::(&data2, Tolerance::absolute(1e-2)); + } + + #[test] + #[should_panic] + fn should_assert_approx_eq_check_shape() { + let data1 = TensorData::from([[3.0, 5.0, 6.0, 7.0]]); + let data2 = TensorData::from([[3.0, 5.0, 6.0]]); + + data1.assert_approx_eq::(&data2, Tolerance::absolute(1e-2)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/mod.rs new file mode 100644 index 0000000..cf5d2dc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/mod.rs @@ -0,0 +1,5 @@ +mod compare; +mod tensor; + +pub use compare::*; +pub use tensor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/tensor.rs new file mode 100644 index 0000000..d9a8237 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/data/tensor.rs @@ -0,0 +1,815 @@ +use core::f32; + +use alloc::boxed::Box; +use alloc::format; +use alloc::string::String; +use alloc::vec::Vec; +use bytemuck::{AnyBitPattern, CheckedBitPattern, Zeroable, cast_mut, checked::CheckedCastError}; +use rand::Rng; +use thiserror::Error; + +use crate::Scalar; +use crate::distribution::Distribution; +use crate::element::{Element, ElementConversion}; +use burn_std::tensor::DType; +use burn_std::{Bytes, QuantLevel, QuantMode, QuantScheme, QuantValue, QuantizedBytes, bf16, f16}; + +/// Data structure for tensors. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TensorData { + /// The values of the tensor (as bytes). + pub bytes: Bytes, + + /// The shape of the tensor. + pub shape: Vec, + + /// The data type of the tensor. + pub dtype: DType, +} + +impl TensorData { + /// Creates a new tensor data structure. + pub fn new>>(value: Vec, shape: S) -> Self { + // Ensure shape is valid + let shape = shape.into(); + Self::check_data_len(&value, &shape); + + Self { + bytes: Bytes::from_elems(value), + shape, + dtype: E::dtype(), + } + } + + /// Creates a new quantized tensor data structure. + pub fn quantized>>( + value: Vec, + shape: S, + scheme: QuantScheme, + qparams: &[f32], + ) -> Self { + let shape = shape.into(); + Self::check_data_len(&value, &shape); + + let q_bytes = QuantizedBytes::new(value, scheme, qparams); + + Self { + bytes: q_bytes.bytes, + shape, + dtype: DType::QFloat(q_bytes.scheme), + } + } + + /// Creates a new tensor data structure from raw bytes. + pub fn from_bytes>>(bytes: Bytes, shape: S, dtype: DType) -> Self { + Self { + bytes, + shape: shape.into(), + dtype, + } + } + + /// Creates a new tensor data structure from raw bytes stored in a vector. + /// + /// Prefer [`TensorData::new`] or [`TensorData::quantized`] over this method unless you are + /// certain that the bytes representation is valid. + pub fn from_bytes_vec>>(bytes: Vec, shape: S, dtype: DType) -> Self { + Self { + bytes: Bytes::from_bytes_vec(bytes), + shape: shape.into(), + dtype, + } + } + + // Check that the input vector contains a correct number of elements + fn check_data_len(data: &[E], shape: &Vec) { + let expected_data_len = Self::numel(shape); + let num_data = data.len(); + assert_eq!( + expected_data_len, num_data, + "Shape {shape:?} is invalid for input of size {num_data:?}", + ); + } + + /// Returns the immutable slice view of the tensor data. + pub fn as_slice(&self) -> Result<&[E], DataError> { + if E::dtype() == self.dtype { + match E::dtype() { + // The only way to create a bool `TensorData` with invalid values is by unsafely modifying + // the dtype. This should be considered unsafe to begin with, so we unsafely cast bool + // to u8 to skip bit validation. Validation iterates through the entire vector, so it's slow. + DType::Bool => { + let slice = bytemuck::checked::try_cast_slice::<_, u8>(&self.bytes) + .map_err(DataError::CastError)?; + Ok(unsafe { core::mem::transmute::<&[u8], &[E]>(slice) }) + } + _ => bytemuck::checked::try_cast_slice(&self.bytes).map_err(DataError::CastError), + } + } else { + Err(DataError::TypeMismatch(format!( + "Invalid target element type (expected {:?}, got {:?})", + self.dtype, + E::dtype() + ))) + } + } + + /// Returns the mutable slice view of the tensor data. + /// + /// # Panics + /// If the target element type is different from the stored element type. + pub fn as_mut_slice(&mut self) -> Result<&mut [E], DataError> { + if E::dtype() == self.dtype { + match E::dtype() { + // The only way to create a bool `TensorData` with invalid values is by unsafely modifying + // the dtype. This should be considered unsafe to begin with, so we unsafely cast bool + // to u8 to skip bit validation. Validation iterates through the entire vector, so it's slow. + DType::Bool => { + let slice = bytemuck::checked::try_cast_slice_mut::<_, u8>(&mut self.bytes) + .map_err(DataError::CastError)?; + Ok(unsafe { core::mem::transmute::<&mut [u8], &mut [E]>(slice) }) + } + _ => bytemuck::checked::try_cast_slice_mut(&mut self.bytes) + .map_err(DataError::CastError), + } + } else { + Err(DataError::TypeMismatch(format!( + "Invalid target element type (expected {:?}, got {:?})", + self.dtype, + E::dtype() + ))) + } + } + + /// Returns the tensor data as a vector of scalar values. + pub fn to_vec(&self) -> Result, DataError> { + Ok(self.as_slice()?.to_vec()) + } + + /// Returns the tensor data as a vector of scalar values. + pub fn into_vec(self) -> Result, DataError> { + // This means we cannot call `into_vec` for QFloat + if E::dtype() != self.dtype { + return Err(DataError::TypeMismatch(format!( + "Invalid target element type (expected {:?}, got {:?})", + self.dtype, + E::dtype() + ))); + } + + match E::dtype() { + // The only way to create a bool `TensorData` with invalid values is by unsafely modifying + // the dtype. This should be considered unsafe to begin with, so we unsafely cast bool + // to u8 to skip bit validation. Validation iterates through the entire vector, so it's slow. + DType::Bool => { + let vec = self.into_vec_unchecked::()?; + Ok(unsafe { core::mem::transmute::, Vec>(vec) }) + } + _ => self.into_vec_unchecked(), + } + } + + /// Returns the tensor data as a vector of scalar values. Does not check dtype. + fn into_vec_unchecked(self) -> Result, DataError> { + let mut me = self; + me.bytes = match me.bytes.try_into_vec::() { + Ok(elems) => return Ok(elems), + Err(bytes) => bytes, + }; + + // The bytes might have been deserialized and allocated with a different align. + // In that case, we have to memcopy the data into a new vector, more suitably allocated + Ok(bytemuck::checked::try_cast_slice(me.as_bytes()) + .map_err(DataError::CastError)? + .to_vec()) + } + + /// Returns an iterator over the values of the tensor data. + pub fn iter(&self) -> Box + '_> { + if E::dtype() == self.dtype { + Box::new(bytemuck::checked::cast_slice(&self.bytes).iter().copied()) + } else { + match self.dtype { + DType::I8 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &i8| e.elem::()), + ), + DType::I16 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &i16| e.elem::()), + ), + DType::I32 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &i32| e.elem::()), + ), + DType::I64 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &i64| e.elem::()), + ), + DType::U8 => Box::new(self.bytes.iter().map(|e| e.elem::())), + DType::U16 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &u16| e.elem::()), + ), + DType::U32 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &u32| e.elem::()), + ), + DType::U64 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &u64| e.elem::()), + ), + DType::BF16 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &bf16| e.elem::()), + ), + DType::F16 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &f16| e.elem::()), + ), + DType::F32 | DType::Flex32 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &f32| e.elem::()), + ), + DType::F64 => Box::new( + bytemuck::checked::cast_slice(&self.bytes) + .iter() + .map(|e: &f64| e.elem::()), + ), + // bool is a byte value equal to either 0 or 1 + DType::Bool => Box::new(self.bytes.iter().map(|e| e.elem::())), + DType::QFloat(scheme) => match scheme { + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + value: + QuantValue::Q8F + | QuantValue::Q8S + // Represent sub-byte values as i8 + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S, + .. + } => { + // Quantized int8 values + let q_bytes = QuantizedBytes { + bytes: self.bytes.clone(), + scheme, + num_elements: self.num_elements(), + }; + let (values, _) = q_bytes.into_vec_i8(); + + Box::new( + values + .iter() + .map(|e: &i8| e.elem::()) + .collect::>() + .into_iter(), + ) + } + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + value: + QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1, + .. + } => { + unimplemented!("Not yet implemented for iteration"); + } + }, + } + } + } + + /// Returns the rank (the number of dimensions). + pub fn rank(&self) -> usize { + self.shape.len() + } + + /// Returns the total number of elements of the tensor data. + pub fn num_elements(&self) -> usize { + Self::numel(&self.shape) + } + + fn numel(shape: &[usize]) -> usize { + shape.iter().product() + } + + /// Populates the data with random values. + pub fn random>>( + shape: S, + distribution: Distribution, + rng: &mut R, + ) -> Self { + let shape = shape.into(); + let num_elements = Self::numel(&shape); + let mut data = Vec::with_capacity(num_elements); + + for _ in 0..num_elements { + data.push(E::random(distribution, rng)); + } + + TensorData::new(data, shape) + } + + /// Populates the data with zeros. + pub fn zeros>>(shape: S) -> TensorData { + let shape = shape.into(); + let num_elements = Self::numel(&shape); + let mut data = Vec::::with_capacity(num_elements); + + for _ in 0..num_elements { + data.push(0.elem()); + } + + TensorData::new(data, shape) + } + + /// Populates the data with ones. + pub fn ones>>(shape: S) -> TensorData { + let shape = shape.into(); + let num_elements = Self::numel(&shape); + let mut data = Vec::::with_capacity(num_elements); + + for _ in 0..num_elements { + data.push(1.elem()); + } + + TensorData::new(data, shape) + } + + /// Populates the data with the given value + pub fn full>>(shape: S, fill_value: E) -> TensorData { + let shape = shape.into(); + let num_elements = Self::numel(&shape); + let mut data = Vec::::with_capacity(num_elements); + for _ in 0..num_elements { + data.push(fill_value) + } + + TensorData::new(data, shape) + } + + /// Populates the data with the given value + pub fn full_dtype, S: Into>>( + shape: S, + fill_value: E, + dtype: DType, + ) -> TensorData { + let fill_value = fill_value.into(); + match dtype { + DType::F64 => Self::full::(shape, fill_value.elem()), + DType::F32 | DType::Flex32 => Self::full::(shape, fill_value.elem()), + DType::F16 => Self::full::(shape, fill_value.elem()), + DType::BF16 => Self::full::(shape, fill_value.elem()), + DType::I64 => Self::full::(shape, fill_value.elem()), + DType::I32 => Self::full::(shape, fill_value.elem()), + DType::I16 => Self::full::(shape, fill_value.elem()), + DType::I8 => Self::full::(shape, fill_value.elem()), + DType::U64 => Self::full::(shape, fill_value.elem()), + DType::U32 => Self::full::(shape, fill_value.elem()), + DType::U16 => Self::full::(shape, fill_value.elem()), + DType::U8 => Self::full::(shape, fill_value.elem()), + DType::Bool => Self::full::(shape, fill_value.elem()), + DType::QFloat(_) => unreachable!(), + } + } + + /// Converts the data to a different element type. + pub fn convert(self) -> Self { + self.convert_dtype(E::dtype()) + } + + /// Converts the data to a different element type. + pub fn convert_dtype(self, dtype: DType) -> Self { + if dtype == self.dtype { + self + } else if dtype.size() == self.dtype.size() + && !matches!(self.dtype, DType::Bool | DType::QFloat(_)) + && !matches!(dtype, DType::Bool | DType::QFloat(_)) + { + match self.dtype { + DType::F64 => self.convert_inplace_dtype::(dtype), + DType::F32 | DType::Flex32 => self.convert_inplace_dtype::(dtype), + DType::F16 => self.convert_inplace_dtype::(dtype), + DType::BF16 => self.convert_inplace_dtype::(dtype), + DType::I64 => self.convert_inplace_dtype::(dtype), + DType::I32 => self.convert_inplace_dtype::(dtype), + DType::I16 => self.convert_inplace_dtype::(dtype), + DType::I8 => self.convert_inplace_dtype::(dtype), + DType::U64 => self.convert_inplace_dtype::(dtype), + DType::U32 => self.convert_inplace_dtype::(dtype), + DType::U16 => self.convert_inplace_dtype::(dtype), + DType::U8 => self.convert_inplace_dtype::(dtype), + DType::Bool | DType::QFloat(_) => unreachable!(), + } + } else { + match self.dtype { + DType::F64 => self.convert_clone_dtype::(dtype), + DType::F32 | DType::Flex32 => self.convert_clone_dtype::(dtype), + DType::F16 => self.convert_clone_dtype::(dtype), + DType::BF16 => self.convert_clone_dtype::(dtype), + DType::I64 => self.convert_clone_dtype::(dtype), + DType::I32 => self.convert_clone_dtype::(dtype), + DType::I16 => self.convert_clone_dtype::(dtype), + DType::I8 => self.convert_clone_dtype::(dtype), + DType::U64 => self.convert_clone_dtype::(dtype), + DType::U32 => self.convert_clone_dtype::(dtype), + DType::U16 => self.convert_clone_dtype::(dtype), + DType::U8 => self.convert_clone_dtype::(dtype), + DType::Bool => self.convert_clone_dtype::(dtype), + DType::QFloat(_) => unreachable!(), + } + } + } + + fn convert_inplace_dtype(self, dtype: DType) -> Self { + match dtype { + DType::F64 => self.convert_inplace::(), + DType::F32 | DType::Flex32 => self.convert_inplace::(), + DType::F16 => self.convert_inplace::(), + DType::BF16 => self.convert_inplace::(), + DType::I64 => self.convert_inplace::(), + DType::I32 => self.convert_inplace::(), + DType::I16 => self.convert_inplace::(), + DType::I8 => self.convert_inplace::(), + DType::U64 => self.convert_inplace::(), + DType::U32 => self.convert_inplace::(), + DType::U16 => self.convert_inplace::(), + DType::U8 => self.convert_inplace::(), + DType::Bool | DType::QFloat(_) => unreachable!(), + } + } + + fn convert_inplace( + mut self, + ) -> Self { + for x in bytemuck::cast_slice_mut::<_, Current>(&mut self.bytes) { + let t: Target = x.elem(); + let x = cast_mut::<_, Target>(x); + *x = t; + } + + self.dtype = Target::dtype(); + + self + } + + fn convert_clone_dtype(self, dtype: DType) -> Self { + match dtype { + DType::F64 => self.convert_clone::(), + DType::F32 | DType::Flex32 => self.convert_clone::(), + DType::F16 => self.convert_clone::(), + DType::BF16 => self.convert_clone::(), + DType::I64 => self.convert_clone::(), + DType::I32 => self.convert_clone::(), + DType::I16 => self.convert_clone::(), + DType::I8 => self.convert_clone::(), + DType::U64 => self.convert_clone::(), + DType::U32 => self.convert_clone::(), + DType::U16 => self.convert_clone::(), + DType::U8 => self.convert_clone::(), + DType::Bool => self.convert_clone::(), + DType::QFloat(_) => unreachable!(), + } + } + + fn convert_clone( + self, + ) -> Self { + let this = bytemuck::checked::cast_slice::<_, Current>(&self.bytes); + let mut out: Vec = ::alloc::vec![Zeroable::zeroed(); self.num_elements()]; + + for (x, out) in this.iter().zip(&mut out) { + *out = x.elem(); + } + + Self::new(out, self.shape) + } + + /// Returns the data as a slice of bytes. + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// Returns the bytes representation of the data. + pub fn into_bytes(self) -> Bytes { + self.bytes + } +} + +impl From<[E; A]> for TensorData { + fn from(elems: [E; A]) -> Self { + TensorData::new(elems.to_vec(), [A]) + } +} + +impl From<[usize; A]> for TensorData { + fn from(elems: [usize; A]) -> Self { + TensorData::new(elems.iter().map(|&e| e as i64).collect(), [A]) + } +} + +impl From<&[usize]> for TensorData { + fn from(elems: &[usize]) -> Self { + let mut data = Vec::with_capacity(elems.len()); + for elem in elems.iter() { + data.push(*elem as i64); + } + + TensorData::new(data, [elems.len()]) + } +} + +impl From<&[E]> for TensorData { + fn from(elems: &[E]) -> Self { + let mut data = Vec::with_capacity(elems.len()); + for elem in elems.iter() { + data.push(*elem); + } + + TensorData::new(data, [elems.len()]) + } +} + +impl From<[[E; B]; A]> for TensorData { + fn from(elems: [[E; B]; A]) -> Self { + let mut data = Vec::with_capacity(A * B); + for elem in elems.into_iter().take(A) { + for elem in elem.into_iter().take(B) { + data.push(elem); + } + } + + TensorData::new(data, [A, B]) + } +} + +impl From<[[[E; C]; B]; A]> + for TensorData +{ + fn from(elems: [[[E; C]; B]; A]) -> Self { + let mut data = Vec::with_capacity(A * B * C); + + for elem in elems.into_iter().take(A) { + for elem in elem.into_iter().take(B) { + for elem in elem.into_iter().take(C) { + data.push(elem); + } + } + } + + TensorData::new(data, [A, B, C]) + } +} + +impl + From<[[[[E; D]; C]; B]; A]> for TensorData +{ + fn from(elems: [[[[E; D]; C]; B]; A]) -> Self { + let mut data = Vec::with_capacity(A * B * C * D); + + for elem in elems.into_iter().take(A) { + for elem in elem.into_iter().take(B) { + for elem in elem.into_iter().take(C) { + for elem in elem.into_iter().take(D) { + data.push(elem); + } + } + } + } + + TensorData::new(data, [A, B, C, D]) + } +} + +impl + From<[[[[[Elem; E]; D]; C]; B]; A]> for TensorData +{ + fn from(elems: [[[[[Elem; E]; D]; C]; B]; A]) -> Self { + let mut data = Vec::with_capacity(A * B * C * D * E); + + for elem in elems.into_iter().take(A) { + for elem in elem.into_iter().take(B) { + for elem in elem.into_iter().take(C) { + for elem in elem.into_iter().take(D) { + for elem in elem.into_iter().take(E) { + data.push(elem); + } + } + } + } + } + + TensorData::new(data, [A, B, C, D, E]) + } +} +impl core::fmt::Display for TensorData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let fmt = match self.dtype { + DType::F64 => format!("{:?}", self.as_slice::().unwrap()), + DType::F32 | DType::Flex32 => format!("{:?}", self.as_slice::().unwrap()), + DType::F16 => format!("{:?}", self.as_slice::().unwrap()), + DType::BF16 => format!("{:?}", self.as_slice::().unwrap()), + DType::I64 => format!("{:?}", self.as_slice::().unwrap()), + DType::I32 => format!("{:?}", self.as_slice::().unwrap()), + DType::I16 => format!("{:?}", self.as_slice::().unwrap()), + DType::I8 => format!("{:?}", self.as_slice::().unwrap()), + DType::U64 => format!("{:?}", self.as_slice::().unwrap()), + DType::U32 => format!("{:?}", self.as_slice::().unwrap()), + DType::U16 => format!("{:?}", self.as_slice::().unwrap()), + DType::U8 => format!("{:?}", self.as_slice::().unwrap()), + DType::Bool => format!("{:?}", self.as_slice::().unwrap()), + DType::QFloat(scheme) => match scheme { + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + value: + QuantValue::Q8F + | QuantValue::Q8S + // Display sub-byte values as i8 + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S, + .. + } => { + format!("{:?} {scheme:?}", self.iter::().collect::>()) + }, + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + value: + QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1, + .. + } => { + unimplemented!("Can't format yet"); + } + }, + }; + f.write_str(fmt.as_str()) + } +} + +/// The things that can go wrong when manipulating tensor data. +#[derive(Debug, Error)] +pub enum DataError { + /// Failed to cast the values to a specified element type. + #[error("Failed to cast values to the specified element type.\nError:\n {0}")] + CastError(CheckedCastError), + /// Invalid target element type. + #[error("{0}")] + TypeMismatch(String), +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use rand::{ + SeedableRng, + rngs::{StdRng, SysRng}, + }; + + #[test] + fn should_have_rank() { + let shape = [3, 5, 6]; + let data = TensorData::random::( + shape, + Distribution::Default, + &mut StdRng::try_from_rng(&mut SysRng).unwrap(), + ); + + assert_eq!(data.rank(), 3); + } + + #[test] + fn into_vec_should_yield_same_value_as_iter() { + let shape = [3, 5, 6]; + let data = TensorData::random::( + shape, + Distribution::Default, + &mut StdRng::try_from_rng(&mut SysRng).unwrap(), + ); + + let expected = data.iter::().collect::>(); + let actual = data.into_vec::().unwrap(); + + assert_eq!(expected, actual); + } + + #[test] + #[should_panic] + fn into_vec_should_assert_wrong_dtype() { + let shape = [3, 5, 6]; + let data = TensorData::random::( + shape, + Distribution::Default, + &mut StdRng::try_from_rng(&mut SysRng).unwrap(), + ); + + data.into_vec::().unwrap(); + } + + #[test] + fn should_have_right_num_elements() { + let shape = [3, 5, 6]; + let num_elements: usize = shape.iter().product(); + let data = TensorData::random::( + shape, + Distribution::Default, + &mut StdRng::try_from_rng(&mut SysRng).unwrap(), + ); + + assert_eq!(num_elements, data.bytes.len() / 4); // f32 stored as u8s + assert_eq!(num_elements, data.as_slice::().unwrap().len()); + } + + #[test] + fn should_have_right_shape() { + let data = TensorData::from([[3.0, 5.0, 6.0]]); + assert_eq!(data.shape, vec![1, 3]); + + let data = TensorData::from([[4.0, 5.0, 8.0], [3.0, 5.0, 6.0]]); + assert_eq!(data.shape, vec![2, 3]); + + let data = TensorData::from([3.0, 5.0, 6.0]); + assert_eq!(data.shape, vec![3]); + } + + #[test] + fn should_convert_bytes_correctly() { + let mut vector: Vec = Vec::with_capacity(5); + vector.push(2.0); + vector.push(3.0); + let data1 = TensorData::new(vector, vec![2]); + + let factor = core::mem::size_of::() / core::mem::size_of::(); + assert_eq!(data1.bytes.len(), 2 * factor); + assert_eq!(data1.bytes.capacity(), 5 * factor); + } + + #[test] + fn should_convert_bytes_correctly_inplace() { + fn test_precision() { + let data = TensorData::new((0..32).collect(), [32]); + for (i, val) in data + .clone() + .convert::() + .into_vec::() + .unwrap() + .into_iter() + .enumerate() + { + assert_eq!(i as u32, val.elem::()) + } + } + test_precision::(); + test_precision::(); + test_precision::(); + test_precision::(); + } + + macro_rules! test_dtypes { + ($test_name:ident, $($dtype:ty),*) => { + $( + paste::paste! { + #[test] + fn [<$test_name _ $dtype:snake>]() { + let full_dtype = TensorData::full_dtype([2, 16], 4, <$dtype>::dtype()); + let full = TensorData::full::<$dtype, _>([2, 16], 4.elem()); + assert_eq!(full_dtype, full); + } + } + )* + }; +} + + test_dtypes!( + should_create_with_dtype, + bool, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f16, + bf16, + f32, + f64 + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/distribution.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/distribution.rs new file mode 100644 index 0000000..d16ebc1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/distribution.rs @@ -0,0 +1,125 @@ +//! Random value distributions used to initialize and populate tensor data. + +use rand::{Rng, RngExt, distr::StandardUniform}; + +use super::element::{Element, ElementConversion}; + +/// Distribution for random value of a tensor. +#[derive(Debug, Default, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum Distribution { + /// Uniform distribution from 0 (inclusive) to 1 (exclusive). + #[default] + Default, + + /// Bernoulli distribution with the given probability. + Bernoulli(f64), + + /// Uniform distribution `[low, high)`. + Uniform(f64, f64), + + /// Normal distribution with the given mean and standard deviation. + Normal(f64, f64), +} + +/// Distribution sampler for random value of a tensor. +#[derive(new)] +pub struct DistributionSampler<'a, E, R> +where + StandardUniform: rand::distr::Distribution, + E: rand::distr::uniform::SampleUniform, + R: Rng, +{ + kind: DistributionSamplerKind, + rng: &'a mut R, +} + +/// Distribution sampler kind for random value of a tensor. +pub enum DistributionSamplerKind +where + StandardUniform: rand::distr::Distribution, + E: rand::distr::uniform::SampleUniform, +{ + /// Standard distribution. + Standard(rand::distr::StandardUniform), + + /// Uniform distribution. + Uniform(rand::distr::Uniform), + + /// Bernoulli distribution. + Bernoulli(rand::distr::Bernoulli), + + /// Normal distribution. + Normal(rand_distr::Normal), +} + +impl DistributionSampler<'_, E, R> +where + StandardUniform: rand::distr::Distribution, + E: rand::distr::uniform::SampleUniform, + E: Element, + R: Rng, +{ + /// Sames a random value from the distribution. + pub fn sample(&mut self) -> E { + match &self.kind { + DistributionSamplerKind::Standard(distribution) => self.rng.sample(distribution), + DistributionSamplerKind::Uniform(distribution) => self.rng.sample(distribution), + DistributionSamplerKind::Bernoulli(distribution) => { + if self.rng.sample(distribution) { + 1.elem() + } else { + 0.elem() + } + } + DistributionSamplerKind::Normal(distribution) => self.rng.sample(distribution).elem(), + } + } +} + +impl Distribution { + /// Creates a new distribution sampler. + /// + /// # Arguments + /// + /// * `rng` - The random number generator. + /// + /// # Returns + /// + /// The distribution sampler. + pub fn sampler(self, rng: &'_ mut R) -> DistributionSampler<'_, E, R> + where + R: Rng, + E: Element + rand::distr::uniform::SampleUniform, + StandardUniform: rand::distr::Distribution, + { + let kind = match self { + Distribution::Default => { + DistributionSamplerKind::Standard(rand::distr::StandardUniform {}) + } + Distribution::Uniform(low, high) => DistributionSamplerKind::Uniform( + rand::distr::Uniform::new(low.elem::(), high.elem::()).unwrap(), + ), + Distribution::Bernoulli(prob) => { + DistributionSamplerKind::Bernoulli(rand::distr::Bernoulli::new(prob).unwrap()) + } + Distribution::Normal(mean, std) => { + DistributionSamplerKind::Normal(rand_distr::Normal::new(mean, std).unwrap()) + } + }; + + DistributionSampler::new(kind, rng) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_distribution_default() { + let dist: Distribution = Default::default(); + + assert_eq!(dist, Distribution::Default); + assert_eq!(Distribution::default(), Distribution::Default); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/base.rs new file mode 100644 index 0000000..8703964 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/base.rs @@ -0,0 +1,295 @@ +use core::cmp::Ordering; +use rand::Rng; + +use crate::distribution::Distribution; +use burn_std::{DType, bf16, f16}; + +#[cfg(feature = "cubecl")] +use burn_std::flex32; + +use super::cast::ToElement; + +/// Core element trait for tensor values. +/// +/// This trait defines the minimal set of capabilities required for a type to be +/// stored and manipulated as a tensor element across all backends. +pub trait Element: + ToElement + + ElementRandom + + ElementConversion + + ElementEq + + ElementLimits + + bytemuck::CheckedBitPattern + + bytemuck::NoUninit + + bytemuck::Zeroable + + core::fmt::Debug + + core::fmt::Display + + Default + + Send + + Sync + + Copy + + 'static +{ + /// The dtype of the element. + fn dtype() -> DType; +} + +/// Ordered element trait for tensor values. +/// +/// This trait extends [`Element`] with ordering semantics, enabling comparison +/// and order-dependent operations in generic Rust implementations. +/// +/// Backends that implement these operations entirely at the device level do +/// not rely on this trait. It only constrains the scalar type for generic Rust code. +pub trait ElementOrdered: Element + ElementComparison {} + +/// Element conversion trait for tensor. +pub trait ElementConversion { + /// Converts an element to another element. + /// + /// # Arguments + /// + /// * `elem` - The element to convert. + /// + /// # Returns + /// + /// The converted element. + fn from_elem(elem: E) -> Self; + + /// Converts and returns the converted element. + fn elem(self) -> E; +} + +/// Element trait for random value of a tensor. +pub trait ElementRandom { + /// Returns a random value for the given distribution. + /// + /// # Arguments + /// + /// * `distribution` - The distribution to sample from. + /// * `rng` - The random number generator. + /// + /// # Returns + /// + /// The random value. + fn random(distribution: Distribution, rng: &mut R) -> Self; +} + +/// Element trait for equality of a tensor. +pub trait ElementEq { + /// Returns whether `self` and `other` are equal. + fn eq(&self, other: &Self) -> bool; +} + +/// Element ordering trait. +pub trait ElementComparison { + /// Returns and [Ordering] between `self` and `other`. + fn cmp(&self, other: &Self) -> Ordering; +} + +/// Element limits trait. +pub trait ElementLimits { + /// The minimum representable value + const MIN: Self; + /// The maximum representable value + const MAX: Self; +} + +/// Macro to implement the element trait for a type. +#[macro_export] +macro_rules! make_element { + ( + ty $type:ident, + convert $convert:expr, + random $random:expr, + cmp $cmp:expr, + dtype $dtype:expr + ) => { + make_element!(ty $type, convert $convert, random $random, cmp $cmp, dtype $dtype, min $type::MIN, max $type::MAX); + }; + ( + ty $type:ident, + convert $convert:expr, + random $random:expr, + cmp $cmp:expr, + dtype $dtype:expr, + min $min:expr, + max $max:expr + ) => { + impl Element for $type { + #[inline(always)] + fn dtype() -> burn_std::DType { + $dtype + } + } + impl ElementEq for $type { + fn eq(&self, other: &Self) -> bool { + self == other + } + } + + impl ElementConversion for $type { + #[inline(always)] + fn from_elem(elem: E) -> Self { + #[allow(clippy::redundant_closure_call)] + $convert(&elem) + } + #[inline(always)] + fn elem(self) -> E { + E::from_elem(self) + } + } + + impl ElementRandom for $type { + fn random(distribution: Distribution, rng: &mut R) -> Self { + #[allow(clippy::redundant_closure_call)] + $random(distribution, rng) + } + } + + impl ElementComparison for $type { + fn cmp(&self, other: &Self) -> Ordering { + let a = self.elem::<$type>(); + let b = other.elem::<$type>(); + #[allow(clippy::redundant_closure_call)] + $cmp(&a, &b) + } + } + + impl ElementLimits for $type { + const MIN: Self = $min; + const MAX: Self = $max; + } + + impl ElementOrdered for $type {} + + }; +} + +make_element!( + ty f64, + convert ToElement::to_f64, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &f64, b: &f64| a.total_cmp(b), + dtype DType::F64 +); + +make_element!( + ty f32, + convert ToElement::to_f32, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &f32, b: &f32| a.total_cmp(b), + dtype DType::F32 +); + +make_element!( + ty i64, + convert ToElement::to_i64, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &i64, b: &i64| Ord::cmp(a, b), + dtype DType::I64 +); + +make_element!( + ty u64, + convert ToElement::to_u64, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &u64, b: &u64| Ord::cmp(a, b), + dtype DType::U64 +); + +make_element!( + ty i32, + convert ToElement::to_i32, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &i32, b: &i32| Ord::cmp(a, b), + dtype DType::I32 +); + +make_element!( + ty u32, + convert ToElement::to_u32, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &u32, b: &u32| Ord::cmp(a, b), + dtype DType::U32 +); + +make_element!( + ty i16, + convert ToElement::to_i16, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &i16, b: &i16| Ord::cmp(a, b), + dtype DType::I16 +); + +make_element!( + ty u16, + convert ToElement::to_u16, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &u16, b: &u16| Ord::cmp(a, b), + dtype DType::U16 +); + +make_element!( + ty i8, + convert ToElement::to_i8, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &i8, b: &i8| Ord::cmp(a, b), + dtype DType::I8 +); + +make_element!( + ty u8, + convert ToElement::to_u8, + random |distribution: Distribution, rng: &mut R| distribution.sampler(rng).sample(), + cmp |a: &u8, b: &u8| Ord::cmp(a, b), + dtype DType::U8 +); + +make_element!( + ty f16, + convert ToElement::to_f16, + random |distribution: Distribution, rng: &mut R| { + let sample: f32 = distribution.sampler(rng).sample(); + f16::from_elem(sample) + }, + cmp |a: &f16, b: &f16| a.total_cmp(b), + dtype DType::F16 +); +make_element!( + ty bf16, + convert ToElement::to_bf16, + random |distribution: Distribution, rng: &mut R| { + let sample: f32 = distribution.sampler(rng).sample(); + bf16::from_elem(sample) + }, + cmp |a: &bf16, b: &bf16| a.total_cmp(b), + dtype DType::BF16 +); + +#[cfg(feature = "cubecl")] +make_element!( + ty flex32, + convert |elem: &dyn ToElement| flex32::from_f32(elem.to_f32()), + random |distribution: Distribution, rng: &mut R| { + let sample: f32 = distribution.sampler(rng).sample(); + flex32::from_elem(sample) + }, + cmp |a: &flex32, b: &flex32| a.total_cmp(b), + dtype DType::Flex32, + min flex32::from_f32(f16::MIN.to_f32_const()), + max flex32::from_f32(f16::MAX.to_f32_const()) +); + +make_element!( + ty bool, + convert ToElement::to_bool, + random |distribution: Distribution, rng: &mut R| { + let sample: u8 = distribution.sampler(rng).sample(); + bool::from_elem(sample) + }, + cmp |a: &bool, b: &bool| Ord::cmp(a, b), + dtype DType::Bool, + min false, + max true +); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/cast.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/cast.rs new file mode 100644 index 0000000..6d56064 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/cast.rs @@ -0,0 +1,706 @@ +use core::mem::size_of; + +use burn_std::{bf16, f16}; + +/// A generic trait for converting a value to a number. +/// Adapted from num_traits::ToPrimitive to support [bool]. +/// +/// A value can be represented by the target type when it lies within +/// the range of scalars supported by the target type. +/// For example, a negative integer cannot be represented by an unsigned +/// integer type, and an `i64` with a very high magnitude might not be +/// convertible to an `i32`. +/// On the other hand, conversions with possible precision loss or truncation +/// are admitted, like an `f32` with a decimal part to an integer type, or +/// even a large `f64` saturating to `f32` infinity. +/// +/// The methods *panic* when the value cannot be represented by the target type. +pub trait ToElement { + /// Converts the value of `self` to an `isize`. + #[inline] + fn to_isize(&self) -> isize { + ToElement::to_isize(&self.to_i64()) + } + + /// Converts the value of `self` to an `i8`. + #[inline] + fn to_i8(&self) -> i8 { + ToElement::to_i8(&self.to_i64()) + } + + /// Converts the value of `self` to an `i16`. + #[inline] + fn to_i16(&self) -> i16 { + ToElement::to_i16(&self.to_i64()) + } + + /// Converts the value of `self` to an `i32`. + #[inline] + fn to_i32(&self) -> i32 { + ToElement::to_i32(&self.to_i64()) + } + + /// Converts the value of `self` to an `i64`. + fn to_i64(&self) -> i64; + + /// Converts the value of `self` to an `i128`. + /// + /// The default implementation converts through `to_i64()`. Types implementing + /// this trait should override this method if they can represent a greater range. + #[inline] + fn to_i128(&self) -> i128 { + i128::from(self.to_i64()) + } + + /// Converts the value of `self` to a `usize`. + #[inline] + fn to_usize(&self) -> usize { + ToElement::to_usize(&self.to_u64()) + } + + /// Converts the value of `self` to a `u8`. + #[inline] + fn to_u8(&self) -> u8 { + ToElement::to_u8(&self.to_u64()) + } + + /// Converts the value of `self` to a `u16`. + #[inline] + fn to_u16(&self) -> u16 { + ToElement::to_u16(&self.to_u64()) + } + + /// Converts the value of `self` to a `u32`. + #[inline] + fn to_u32(&self) -> u32 { + ToElement::to_u32(&self.to_u64()) + } + + /// Converts the value of `self` to a `u64`. + fn to_u64(&self) -> u64; + + /// Converts the value of `self` to a `u128`. + /// + /// The default implementation converts through `to_u64()`. Types implementing + /// this trait should override this method if they can represent a greater range. + #[inline] + fn to_u128(&self) -> u128 { + u128::from(self.to_u64()) + } + + /// Converts the value of `self` to an `f16`. Overflows may map to positive + /// or negative infinity. + #[inline] + fn to_f16(&self) -> f16 { + f16::from_f32(self.to_f32()) + } + + /// Converts the value of `self` to an `bf16`. Overflows may map to positive + /// or negative infinity. + #[inline] + fn to_bf16(&self) -> bf16 { + bf16::from_f32(self.to_f32()) + } + + /// Converts the value of `self` to an `f32`. Overflows may map to positive + /// or negative infinity. + #[inline] + fn to_f32(&self) -> f32 { + ToElement::to_f32(&self.to_f64()) + } + + /// Converts the value of `self` to an `f64`. Overflows may map to positive + /// or negative infinity. + /// + /// The default implementation tries to convert through `to_i64()`, and + /// failing that through `to_u64()`. Types implementing this trait should + /// override this method if they can represent a greater range. + #[inline] + fn to_f64(&self) -> f64 { + ToElement::to_f64(&self.to_u64()) + } + + /// Converts the value of `self` to a bool. + /// Rust only considers 0 and 1 to be valid booleans, but for compatibility, C semantics are + /// adopted (anything that's not 0 is true). + /// + /// The default implementation tries to convert through `to_i64()`, and + /// failing that through `to_u64()`. Types implementing this trait should + /// override this method if they can represent a greater range. + #[inline] + fn to_bool(&self) -> bool { + ToElement::to_bool(&self.to_u64()) + } +} + +macro_rules! impl_to_element_int_to_int { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> $DstT { + let min = $DstT::MIN as $SrcT; + let max = $DstT::MAX as $SrcT; + if size_of::<$SrcT>() <= size_of::<$DstT>() || (min <= *self && *self <= max) { + *self as $DstT + } else { + panic!( + "Element cannot be represented in the target type: {:?}({:?}) => {:?}", + core::any::type_name::<$SrcT>(), + self, + core::any::type_name::<$DstT>(), + ) + } + } + )*} +} + +macro_rules! impl_to_element_int_to_uint { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> $DstT { + let max = $DstT::MAX as $SrcT; + if 0 <= *self && (size_of::<$SrcT>() <= size_of::<$DstT>() || *self <= max) { + *self as $DstT + } else { + panic!( + "Element cannot be represented in the target type: {:?}({:?}) => {:?}", + core::any::type_name::<$SrcT>(), + self, + core::any::type_name::<$DstT>(), + ) + } + } + )*} +} + +macro_rules! impl_to_element_int { + ($T:ident) => { + impl ToElement for $T { + impl_to_element_int_to_int! { $T: + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + } + + impl_to_element_int_to_uint! { $T: + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + } + + #[inline] + fn to_f32(&self) -> f32 { + *self as f32 + } + #[inline] + fn to_f64(&self) -> f64 { + *self as f64 + } + #[inline] + fn to_bool(&self) -> bool { + *self != 0 + } + } + }; +} + +impl_to_element_int!(isize); +impl_to_element_int!(i8); +impl_to_element_int!(i16); +impl_to_element_int!(i32); +impl_to_element_int!(i64); +impl_to_element_int!(i128); + +macro_rules! impl_to_element_uint_to_int { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> $DstT { + let max = $DstT::MAX as $SrcT; + if size_of::<$SrcT>() < size_of::<$DstT>() || *self <= max { + *self as $DstT + } else { + panic!( + "Element cannot be represented in the target type: {:?}({:?}) => {:?}", + core::any::type_name::<$SrcT>(), + self, + core::any::type_name::<$DstT>(), + ) + } + } + )*} +} + +macro_rules! impl_to_element_uint_to_uint { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> $DstT { + let max = $DstT::MAX as $SrcT; + if size_of::<$SrcT>() <= size_of::<$DstT>() || *self <= max { + *self as $DstT + } else { + panic!( + "Element cannot be represented in the target type: {:?}({:?}) => {:?}", + core::any::type_name::<$SrcT>(), + self, + core::any::type_name::<$DstT>(), + ) + } + } + )*} +} + +macro_rules! impl_to_element_uint { + ($T:ident) => { + impl ToElement for $T { + impl_to_element_uint_to_int! { $T: + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + } + + impl_to_element_uint_to_uint! { $T: + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + } + + #[inline] + fn to_f32(&self) -> f32 { + *self as f32 + } + #[inline] + fn to_f64(&self) -> f64 { + *self as f64 + } + #[inline] + fn to_bool(&self) -> bool { + *self != 0 + } + } + }; +} + +impl_to_element_uint!(usize); +impl_to_element_uint!(u8); +impl_to_element_uint!(u16); +impl_to_element_uint!(u32); +impl_to_element_uint!(u64); +impl_to_element_uint!(u128); + +macro_rules! impl_to_element_float_to_float { + ($SrcT:ident : $( fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + fn $method(&self) -> $DstT { + // We can safely cast all values, whether NaN, +-inf, or finite. + // Finite values that are reducing size may saturate to +-inf. + *self as $DstT + } + )*} +} + +macro_rules! float_to_int_unchecked { + // SAFETY: Must not be NaN or infinite; must be representable as the integer after truncating. + // We already checked that the float is in the exclusive range `(MIN-1, MAX+1)`. + ($float:expr => $int:ty) => { + unsafe { $float.to_int_unchecked::<$int>() } + }; +} + +macro_rules! impl_to_element_float_to_signed_int { + ($f:ident : $( $(#[$cfg:meta])* fn $method:ident -> $i:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> $i { + // Float as int truncates toward zero, so we want to allow values + // in the exclusive range `(MIN-1, MAX+1)`. + if size_of::<$f>() > size_of::<$i>() { + // With a larger size, we can represent the range exactly. + const MIN_M1: $f = $i::MIN as $f - 1.0; + const MAX_P1: $f = $i::MAX as $f + 1.0; + if *self > MIN_M1 && *self < MAX_P1 { + return float_to_int_unchecked!(*self => $i); + } + } else { + // We can't represent `MIN-1` exactly, but there's no fractional part + // at this magnitude, so we can just use a `MIN` inclusive boundary. + const MIN: $f = $i::MIN as $f; + // We can't represent `MAX` exactly, but it will round up to exactly + // `MAX+1` (a power of two) when we cast it. + const MAX_P1: $f = $i::MAX as $f; + if *self >= MIN && *self < MAX_P1 { + return float_to_int_unchecked!(*self => $i); + } + } + panic!("Float cannot be represented in the target signed int type") + } + )*} +} + +macro_rules! impl_to_element_float_to_unsigned_int { + ($f:ident : $( $(#[$cfg:meta])* fn $method:ident -> $u:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> $u { + // Float as int truncates toward zero, so we want to allow values + // in the exclusive range `(-1, MAX+1)`. + if size_of::<$f>() > size_of::<$u>() { + // With a larger size, we can represent the range exactly. + const MAX_P1: $f = $u::MAX as $f + 1.0; + if *self > -1.0 && *self < MAX_P1 { + return float_to_int_unchecked!(*self => $u); + } + } else { + // We can't represent `MAX` exactly, but it will round up to exactly + // `MAX+1` (a power of two) when we cast it. + // (`u128::MAX as f32` is infinity, but this is still ok.) + const MAX_P1: $f = $u::MAX as $f; + if *self > -1.0 && *self < MAX_P1 { + return float_to_int_unchecked!(*self => $u); + } + } + panic!("Float cannot be represented in the target unsigned int type") + } + )*} +} + +macro_rules! impl_to_element_float { + ($T:ident) => { + impl ToElement for $T { + impl_to_element_float_to_signed_int! { $T: + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + } + + impl_to_element_float_to_unsigned_int! { $T: + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + } + + impl_to_element_float_to_float! { $T: + fn to_f32 -> f32; + fn to_f64 -> f64; + } + + #[inline] + fn to_bool(&self) -> bool { + *self != 0.0 + } + } + }; +} + +impl_to_element_float!(f32); +impl_to_element_float!(f64); + +impl ToElement for f16 { + #[inline] + fn to_i64(&self) -> i64 { + Self::to_f32(*self).to_i64() + } + #[inline] + fn to_u64(&self) -> u64 { + Self::to_f32(*self).to_u64() + } + #[inline] + fn to_i8(&self) -> i8 { + Self::to_f32(*self).to_i8() + } + #[inline] + fn to_u8(&self) -> u8 { + Self::to_f32(*self).to_u8() + } + #[inline] + fn to_i16(&self) -> i16 { + Self::to_f32(*self).to_i16() + } + #[inline] + fn to_u16(&self) -> u16 { + Self::to_f32(*self).to_u16() + } + #[inline] + fn to_i32(&self) -> i32 { + Self::to_f32(*self).to_i32() + } + #[inline] + fn to_u32(&self) -> u32 { + Self::to_f32(*self).to_u32() + } + #[inline] + fn to_f16(&self) -> f16 { + *self + } + #[inline] + fn to_f32(&self) -> f32 { + Self::to_f32(*self) + } + #[inline] + fn to_f64(&self) -> f64 { + Self::to_f64(*self) + } + #[inline] + fn to_bool(&self) -> bool { + *self != f16::from_f32_const(0.0) + } +} + +impl ToElement for bf16 { + #[inline] + fn to_i64(&self) -> i64 { + Self::to_f32(*self).to_i64() + } + #[inline] + fn to_u64(&self) -> u64 { + Self::to_f32(*self).to_u64() + } + #[inline] + fn to_i8(&self) -> i8 { + Self::to_f32(*self).to_i8() + } + #[inline] + fn to_u8(&self) -> u8 { + Self::to_f32(*self).to_u8() + } + #[inline] + fn to_i16(&self) -> i16 { + Self::to_f32(*self).to_i16() + } + #[inline] + fn to_u16(&self) -> u16 { + Self::to_f32(*self).to_u16() + } + #[inline] + fn to_i32(&self) -> i32 { + Self::to_f32(*self).to_i32() + } + #[inline] + fn to_u32(&self) -> u32 { + Self::to_f32(*self).to_u32() + } + #[inline] + fn to_bf16(&self) -> bf16 { + *self + } + #[inline] + fn to_f32(&self) -> f32 { + Self::to_f32(*self) + } + #[inline] + fn to_f64(&self) -> f64 { + Self::to_f64(*self) + } + #[inline] + fn to_bool(&self) -> bool { + *self != bf16::from_f32_const(0.0) + } +} + +#[cfg(feature = "cubecl")] +impl ToElement for burn_std::flex32 { + #[inline] + fn to_i64(&self) -> i64 { + Self::to_f32(*self).to_i64() + } + #[inline] + fn to_u64(&self) -> u64 { + Self::to_f32(*self).to_u64() + } + #[inline] + fn to_i8(&self) -> i8 { + Self::to_f32(*self).to_i8() + } + #[inline] + fn to_u8(&self) -> u8 { + Self::to_f32(*self).to_u8() + } + #[inline] + fn to_i16(&self) -> i16 { + Self::to_f32(*self).to_i16() + } + #[inline] + fn to_u16(&self) -> u16 { + Self::to_f32(*self).to_u16() + } + #[inline] + fn to_i32(&self) -> i32 { + Self::to_f32(*self).to_i32() + } + #[inline] + fn to_u32(&self) -> u32 { + Self::to_f32(*self).to_u32() + } + #[inline] + fn to_f32(&self) -> f32 { + Self::to_f32(*self) + } + #[inline] + fn to_f64(&self) -> f64 { + Self::to_f64(*self) + } + #[inline] + fn to_bool(&self) -> bool { + *self != burn_std::flex32::from_f32(0.0) + } +} + +impl ToElement for bool { + #[inline] + fn to_i64(&self) -> i64 { + *self as i64 + } + #[inline] + fn to_u64(&self) -> u64 { + *self as u64 + } + #[inline] + fn to_i8(&self) -> i8 { + *self as i8 + } + #[inline] + fn to_u8(&self) -> u8 { + *self as u8 + } + #[inline] + fn to_i16(&self) -> i16 { + *self as i16 + } + #[inline] + fn to_u16(&self) -> u16 { + *self as u16 + } + #[inline] + fn to_i32(&self) -> i32 { + *self as i32 + } + #[inline] + fn to_u32(&self) -> u32 { + *self as u32 + } + #[inline] + fn to_f32(&self) -> f32 { + self.to_u8() as f32 + } + #[inline] + fn to_f64(&self) -> f64 { + self.to_u8() as f64 + } + #[inline] + fn to_bool(&self) -> bool { + *self + } +} + +mod tests { + #[allow(unused_imports)] + use super::*; + + #[test] + fn to_element_float() { + let f32_toolarge = 1e39f64; + assert_eq!(f32_toolarge.to_f32(), f32::INFINITY); + assert_eq!((-f32_toolarge).to_f32(), f32::NEG_INFINITY); + assert_eq!((f32::MAX as f64).to_f32(), f32::MAX); + assert_eq!((-f32::MAX as f64).to_f32(), -f32::MAX); + assert_eq!(f64::INFINITY.to_f32(), f32::INFINITY); + assert_eq!((f64::NEG_INFINITY).to_f32(), f32::NEG_INFINITY); + assert!((f64::NAN).to_f32().is_nan()); + } + + #[test] + #[should_panic] + fn to_element_signed_to_u8_underflow() { + let _x = (-1i8).to_u8(); + } + + #[test] + #[should_panic] + fn to_element_signed_to_u16_underflow() { + let _x = (-1i8).to_u16(); + } + + #[test] + #[should_panic] + fn to_element_signed_to_u32_underflow() { + let _x = (-1i8).to_u32(); + } + + #[test] + #[should_panic] + fn to_element_signed_to_u64_underflow() { + let _x = (-1i8).to_u64(); + } + + #[test] + #[should_panic] + fn to_element_signed_to_u128_underflow() { + let _x = (-1i8).to_u128(); + } + + #[test] + #[should_panic] + fn to_element_signed_to_usize_underflow() { + let _x = (-1i8).to_usize(); + } + + #[test] + #[should_panic] + fn to_element_unsigned_to_u8_overflow() { + let _x = 256.to_u8(); + } + + #[test] + #[should_panic] + fn to_element_unsigned_to_u16_overflow() { + let _x = 65_536.to_u16(); + } + + #[test] + #[should_panic] + fn to_element_unsigned_to_u32_overflow() { + let _x = 4_294_967_296u64.to_u32(); + } + + #[test] + #[should_panic] + fn to_element_unsigned_to_u64_overflow() { + let _x = 18_446_744_073_709_551_616u128.to_u64(); + } + + #[test] + fn to_element_int_to_float() { + assert_eq!((-1).to_f32(), -1.0); + assert_eq!((-1).to_f64(), -1.0); + assert_eq!(255.to_f32(), 255.0); + assert_eq!(65_535.to_f64(), 65_535.0); + } + + #[test] + fn to_element_float_to_int() { + assert_eq!((-1.0).to_i8(), -1); + assert_eq!(1.0.to_u8(), 1); + assert_eq!(1.8.to_u16(), 1); + assert_eq!(123.456.to_u32(), 123); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/mod.rs new file mode 100644 index 0000000..c1f7884 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/mod.rs @@ -0,0 +1,10 @@ +//! Traits and helpers for working with element types and conversions. + +mod base; +mod scalar; + +/// Tensor element casting. +pub mod cast; + +pub use base::*; +pub use scalar::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/scalar.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/scalar.rs new file mode 100644 index 0000000..1b90770 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/element/scalar.rs @@ -0,0 +1,105 @@ +use burn_std::{DType, bf16, f16}; +use num_traits::ToPrimitive; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +use crate::{Element, ElementConversion}; + +/// A scalar element. +#[derive(Clone, Copy, Debug)] +#[allow(missing_docs)] +pub enum Scalar { + Float(f64), + Int(i64), + UInt(u64), + Bool(bool), +} + +impl Scalar { + /// Creates a scalar with the specified data type. + /// + /// # Note + /// [`QFloat`](DType::QFloat) scalars are represented as float for element-wise operations. + pub fn new(value: E, dtype: &DType) -> Self { + if dtype.is_float() | matches!(dtype, &DType::QFloat(_)) { + Self::Float(value.elem()) + } else if dtype.is_int() { + Self::Int(value.elem()) + } else if dtype.is_uint() { + Self::UInt(value.elem()) + } else if dtype.is_bool() { + Self::Bool(value.elem()) + } else { + unimplemented!("Scalar not supported for {dtype:?}") + } + } + + /// Converts and returns the converted element. + pub fn elem(self) -> E { + match self { + Self::Float(x) => x.elem(), + Self::Int(x) => x.elem(), + Self::UInt(x) => x.elem(), + Self::Bool(x) => x.elem(), + } + } + + /// Returns the exact integer value, if valid. + pub fn try_as_integer(&self) -> Option { + match self { + Scalar::Float(x) => (x.floor() == *x).then(|| Self::Int(x.to_i64().unwrap())), + Scalar::Int(_) | Scalar::UInt(_) => Some(*self), + Scalar::Bool(x) => Some(Scalar::Int(*x as i64)), + } + } +} + +macro_rules! impl_from_scalar { + ($($ty:ty => $variant:ident),+ $(,)?) => { + $( + impl From<$ty> for Scalar { + fn from(value: $ty) -> Self { + Scalar::$variant(value.elem()) + } + } + )+ + }; +} + +impl_from_scalar! { + f64 => Float, f32 => Float, f16 => Float, bf16 => Float, + i64 => Int, i32 => Int, i16 => Int, i8 => Int, + u64 => UInt, u32 => UInt, u16 => UInt, u8 => UInt, bool => Bool, +} + +// CubeCL requirement +impl ToPrimitive for Scalar { + fn to_i64(&self) -> Option { + match self { + Scalar::Float(x) => x.to_i64(), + Scalar::UInt(x) => x.to_i64(), + Scalar::Int(x) => Some(*x), + Scalar::Bool(x) => Some(*x as i64), + } + } + + fn to_u64(&self) -> Option { + match self { + Scalar::Float(x) => x.to_u64(), + Scalar::UInt(x) => Some(*x), + Scalar::Int(x) => x.to_u64(), + Scalar::Bool(x) => Some(*x as u64), + } + } + + fn to_f64(&self) -> Option { + match self { + Scalar::Float(x) => Some(*x), + Scalar::UInt(x) => x.to_f64(), + Scalar::Int(x) => x.to_f64(), + Scalar::Bool(x) => (*x as u8).to_f64(), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/lib.rs new file mode 100644 index 0000000..84e5e1d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/lib.rs @@ -0,0 +1,122 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! This library provides the core types that define how Burn tensor data is represented, stored, and interpreted. + +#[macro_use] +extern crate derive_new; + +extern crate alloc; + +mod data; +pub use data::*; + +pub mod distribution; +pub use distribution::*; +pub mod element; +pub use element::*; + +/// [`Backend`] trait and required types. +pub mod backend; +pub use backend::*; + +/// Backend tensor primitives and operations. +pub mod tensor; + +// Re-exported types +pub use burn_std::reader::*; // Useful so that backends don't have to add `burn_std` as a dependency. +pub use burn_std::{ + AllocationProperty, Bytes, DType, FloatDType, IntDType, bf16, f16, stream_id::StreamId, +}; + +/// Shape definition. +pub mod shape { + pub use burn_std::shape::*; +} +pub use shape::*; + +/// Slice utilities. +pub mod slice { + pub use burn_std::{s, slice::*}; +} +pub use slice::*; + +/// Indexing utilities. +pub mod indexing { + pub use burn_std::indexing::*; +} +pub use indexing::*; + +/// Quantization data representation. +pub mod quantization { + pub use crate::tensor::quantization::*; + pub use burn_std::quantization::{ + BlockSize, QuantLevel, QuantMode, QuantParam, QuantPropagation, QuantScheme, QuantStore, + QuantValue, QuantizedBytes, + }; +} + +#[cfg(feature = "cubecl-wgpu")] +mod cube_wgpu { + use crate::backend::DeviceOps; + use cubecl::wgpu::WgpuDevice; + + impl DeviceOps for WgpuDevice {} +} + +#[cfg(feature = "cubecl-cuda")] +mod cube_cuda { + use crate::backend::DeviceOps; + use cubecl::cuda::CudaDevice; + + impl DeviceOps for CudaDevice {} +} + +#[cfg(feature = "cubecl-cpu")] +mod cube_cpu { + use crate::backend::DeviceOps; + use cubecl::cpu::CpuDevice; + + impl DeviceOps for CpuDevice {} +} + +#[cfg(feature = "cubecl-hip")] +mod cube_hip { + use crate::backend::DeviceOps; + use cubecl::hip::AmdDevice; + + impl DeviceOps for AmdDevice {} +} + +/// Convenience macro to link to the `burn-tensor` docs for this crate version. +/// +/// Usage: +/// ```rust,ignore +/// # use burn_backend::doc_tensor; +/// doc_tensor!(); // Links to `Tensor` struct +/// doc_tensor!("zeros"); // Links to `Tensor::zeros` method +/// ``` +#[macro_export] +macro_rules! doc_tensor { + () => { + concat!( + "[`Tensor`](https://docs.rs/burn-tensor/", + env!("CARGO_PKG_VERSION"), + "/burn_tensor/struct.Tensor.html)" + ) + }; + + ($method:literal) => { + concat!( + "[`Tensor::", + $method, + "`](", + "https://docs.rs/burn-tensor/", + env!("CARGO_PKG_VERSION"), + "/burn_tensor/struct.Tensor.html#method.", + $method, + ")" + ) + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/alias.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/alias.rs new file mode 100644 index 0000000..7ca7c4b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/alias.rs @@ -0,0 +1,23 @@ +use crate::backend::Backend; + +// We provide some type aliases to improve the readability of using associated types without +// having to use the disambiguation syntax. + +/// Device type used by the backend. +pub type Device = ::Device; + +/// Float element type used by backend. +pub type FloatElem = ::FloatElem; +/// Integer element type used by backend. +pub type IntElem = ::IntElem; +/// Boolean element type used by backend. +pub type BoolElem = ::BoolElem; + +/// Float tensor primitive type used by the backend. +pub type FloatTensor = ::FloatTensorPrimitive; +/// Integer tensor primitive type used by the backend. +pub type IntTensor = ::IntTensorPrimitive; +/// Boolean tensor primitive type used by the backend. +pub type BoolTensor = ::BoolTensorPrimitive; +/// Quantized tensor primitive type used by the backend. +pub type QuantizedTensor = ::QuantizedTensorPrimitive; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/container.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/container.rs new file mode 100644 index 0000000..7e4eb0d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/container.rs @@ -0,0 +1,92 @@ +use alloc::boxed::Box; +use core::any::Any; + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; +#[cfg(not(feature = "std"))] +use hashbrown::HashMap; + +#[cfg(feature = "std")] +use std::collections::HashMap; + +use crate::{TensorPrimitive, backend::Backend}; + +/// Contains tensor of arbitrary dimension. +#[derive(Debug)] +pub struct TensorContainer { + tensors: HashMap>, +} + +impl Default for TensorContainer +where + ID: core::hash::Hash + PartialEq + Eq + core::fmt::Debug, +{ + fn default() -> Self { + Self::new() + } +} + +impl TensorContainer +where + ID: core::hash::Hash + PartialEq + Eq + core::fmt::Debug, +{ + /// Create an empty container. + pub fn new() -> Self { + Self { + tensors: HashMap::new(), + } + } + + /// Get a tensor with the given ID. + pub fn get(&self, id: &ID) -> Option> + where + B: Backend, + { + let grad = self.tensors.get(id)?; + + let tensor = grad + .downcast_ref::>() + // .map(|primitive| Tensor::::from_primitive(primitive.clone())) + .unwrap(); + + Some(tensor.clone()) + } + + /// Register a new tensor for the given ID. + /// + /// # Notes + /// + /// If a tensor is already registered for the given ID, it will be replaced. + pub fn register(&mut self, id: ID, value: TensorPrimitive) + where + B: Backend, + { + self.tensors.insert(id, Box::new(value)); + } + + /// Remove a tensor for the given ID and returns it. + pub fn remove(&mut self, id: &ID) -> Option> + where + B: Backend, + { + self.tensors + .remove(id) + .map(|item| *item.downcast::>().unwrap()) + // .map(|primitive| Tensor::from_primitive(*primitive)) + } + + /// The number of tensors registered. + pub fn len(&self) -> usize { + self.tensors.len() + } + + /// If any tensor is contained. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get id of every tensor in the container + pub fn ids(&self) -> Vec<&ID> { + self.tensors.keys().collect() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/kind.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/kind.rs new file mode 100644 index 0000000..b907714 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/kind.rs @@ -0,0 +1,44 @@ +use crate::{Backend, TensorMetadata, TensorPrimitive}; + +/// A type-level representation of the kind of a float tensor +#[derive(Clone, Debug)] +pub struct Float; + +/// A type-level representation of the kind of a int tensor. +#[derive(Clone, Debug)] +pub struct Int; + +/// A type-level representation of the kind of a bool tensor. +#[derive(Clone, Debug)] +pub struct Bool; + +/// A type-level representation of the kind of a tensor. +/// Metadata access is lazy. +pub trait TensorKind: Clone + core::fmt::Debug { + /// The primitive type of the tensor. + type Primitive: TensorMetadata; + + /// The name of the tensor kind. + fn name() -> &'static str; +} + +impl TensorKind for Float { + type Primitive = TensorPrimitive; + fn name() -> &'static str { + "Float" + } +} + +impl TensorKind for Int { + type Primitive = B::IntTensorPrimitive; + fn name() -> &'static str { + "Int" + } +} + +impl TensorKind for Bool { + type Primitive = B::BoolTensorPrimitive; + fn name() -> &'static str { + "Bool" + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/mod.rs new file mode 100644 index 0000000..992ca50 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/mod.rs @@ -0,0 +1,12 @@ +mod alias; +mod container; +mod kind; +mod ops; + +pub use alias::*; +pub use container::*; +pub use kind::*; +pub use ops::*; + +/// Tensor quantization module. +pub mod quantization; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/autodiff.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/autodiff.rs new file mode 100644 index 0000000..029f304 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/autodiff.rs @@ -0,0 +1,49 @@ +use crate::{ + AutodiffBackend, + tensor::{BasicOps, TensorKind}, +}; + +/// Trait that list all operations that can be applied on all tensors on an autodiff backend. +/// +/// # Warnings +/// +/// This is an internal trait, use the public API provided by the +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// struct. +pub trait BasicAutodiffOps: BasicOps + BasicOps { + /// Inner primitive tensor. + type InnerKind: BasicOps; + + /// Returns the inner tensor without the autodiff information. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("inner"))] + #[cfg_attr(not(doc), doc = "`Tensor::inner`")] + /// function, which is more high-level and designed for public use. + fn inner( + tensor: >::Primitive, + ) -> >::Primitive; + + /// Convert a tensor to the autodiff backend. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("from_inner"))] + #[cfg_attr(not(doc), doc = "`Tensor::from_inner`")] + /// function, which is more high-level and designed for public use. + fn from_inner( + inner: >::Primitive, + ) -> >::Primitive; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/base.rs new file mode 100644 index 0000000..7e0ff0b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/base.rs @@ -0,0 +1,807 @@ +use alloc::vec::Vec; +use burn_std::{DType, Shape, Slice}; + +use crate::{ + Backend, ExecutionError, Scalar, TensorData, TensorMetadata, + element::Element, + ops::TransactionPrimitive, + tensor::{IndexingUpdateOp, IntTensor, TensorKind}, +}; + +/// Trait that list all operations that can be applied on all tensors. +/// +/// # Warnings +/// +/// This is an internal trait, use the public API provided by the +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// struct. +pub trait BasicOps: TensorKind { + /// The type of the tensor elements. + type Elem: Element; + + /// Creates an empty tensor with the given shape. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device on which the tensor will be allocated. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The empty tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For creating empty tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("empty"))] + #[cfg_attr(not(doc), doc = "`Tensor::empty`")] + /// function, which is more high-level and designed for public use. + fn empty(shape: Shape, device: &B::Device, dtype: DType) -> Self::Primitive; + + /// Creates a tensor filled with zeros. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device on which the tensor will be allocated. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor filled with zeros. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For creating a tensor filled with zeros, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("zeros"))] + #[cfg_attr(not(doc), doc = "`Tensor::zeros`")] + /// function, which is more high-level and designed for public use. + fn zeros(shape: Shape, device: &B::Device, dtype: DType) -> Self::Primitive; + + /// Creates a tensor filled with ones. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device on which the tensor will be allocated. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor filled with ones. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For creating a tensor filled with ones, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("ones"))] + #[cfg_attr(not(doc), doc = "`Tensor::ones`")] + /// function, which is more high-level and designed for public use. + fn ones(shape: Shape, device: &B::Device, dtype: DType) -> Self::Primitive; + + /// Creates a tensor of the given shape where each element is equal to the provided value. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `fill_value` - The value with which to fill the tensor. + /// * `device` - The device on which the tensor will be allocated. + /// * `dtype` - The target data type. + /// + /// # Returns + /// + /// The tensor filled with the specified value. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For creating full tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("full"))] + #[cfg_attr(not(doc), doc = "`Tensor::full`")] + /// function, which is more high-level and designed for public use. + fn full(shape: Shape, fill_value: Scalar, device: &B::Device, dtype: DType) -> Self::Primitive; + + /// Reshapes the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `shape` - The new shape of the tensor. + /// + /// # Returns + /// + /// The reshaped tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For reshaping a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("reshape"))] + #[cfg_attr(not(doc), doc = "`Tensor::reshape`")] + /// function, which is more high-level and designed for public use. + fn reshape(tensor: Self::Primitive, shape: Shape) -> Self::Primitive; + + /// Transposes a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to transpose. + /// + /// # Returns + /// + /// The transposed tensor. + fn transpose(tensor: Self::Primitive) -> Self::Primitive; + + /// Swaps two dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to swap the dimensions of. + /// * `dim1` - The first dimension to swap. + /// * `dim2` - The second dimension to swap. + /// + /// # Returns + /// + /// The tensor with the dimensions swapped. + fn swap_dims(tensor: Self::Primitive, dim1: usize, dim2: usize) -> Self::Primitive; + + /// Permutes the dimensions of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to permute the dimensions of. + /// * `axes` - The new order of the dimensions. + /// + /// # Returns + /// + /// The tensor with the dimensions permuted. + fn permute(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive; + + /// Flips the tensor along the given axes. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to flip. + /// * `axes` - The axes to flip the tensor along. + /// + /// # Returns + /// + /// The tensor with the axes flipped. + fn flip(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive; + + /// Select tensor elements corresponding to the given slices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `slices` - The slices specifying ranges and steps for each dimension. + /// + /// # Returns + /// + /// The selected elements. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For selecting elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("slice"))] + #[cfg_attr(not(doc), doc = "`Tensor::slice`")] + /// function, which is more high-level and designed for public use. + fn slice(tensor: Self::Primitive, slices: &[Slice]) -> Self::Primitive; + + /// Assigns the given value to the tensor elements corresponding to the given slices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `slices` - The slices specifying which elements to assign, including support for steps. + /// * `value` - The value to assign. + /// + /// # Returns + /// + /// The tensor with the assigned values. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For assigning values to elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("slice_assign"))] + #[cfg_attr(not(doc), doc = "`Tensor::slice_assign`")] + /// function, which is more high-level and designed for public use. + fn slice_assign( + tensor: Self::Primitive, + slices: &[Slice], + value: Self::Primitive, + ) -> Self::Primitive; + + /// Select tensor elements along the given dimension corresponding to the given indices. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select from. + /// * `dim` - The dimension along which to select. + /// * `indices` - The indices of the elements to select. + /// + /// # Returns + /// + /// The selected tensor elements. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For selecting elements from a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("select"))] + #[cfg_attr(not(doc), doc = "`Tensor::select`")] + /// function, which is more high-level and designed for public use. + fn select(tensor: Self::Primitive, dim: usize, indices: IntTensor) -> Self::Primitive; + + /// Assign the selected elements along the given dimension corresponding to the given indices + /// from the value tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to assign elements to. + /// * `dim` - The axis along which to assign elements. + /// * `indices` - The indices of the elements to assign. + /// * `values` - The values to assign to the tensor. + /// * `update` - The operation used to update the existing values at the indexed positions (e.g., add). + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is taken from the + /// corresponding element of the input tensor at the corresponding index along the specified axis, + /// except for the elements at the specified indices, which are taken from the corresponding + /// element of the values tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For assigning elements to a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("select_assign"))] + #[cfg_attr(not(doc), doc = "`Tensor::select_assign`")] + /// function, which is more high-level and designed for public use. + fn select_assign( + tensor: Self::Primitive, + dim: usize, + indices: IntTensor, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive; + + /// Selects elements from a tensor based on a boolean mask. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to select elements from if the corresponding element of the mask is true. + /// * `mask` - The boolean mask to use for selecting elements. + /// * `source` - The tensor to select elements from when the corresponding element of the mask is false. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensors, where each element is taken from the + /// corresponding element of the left hand side tensor if the corresponding element of the mask + /// is true, and from the corresponding element of the right hand side tensor otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For selecting elements from a tensor based on a boolean mask, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("mask_where"))] + #[cfg_attr(not(doc), doc = "`Tensor::mask_where`")] + /// function, which is more high-level and designed for public use. + fn mask_where( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + source: Self::Primitive, + ) -> Self::Primitive; + + /// Fills elements of a tensor based on a boolean mask. + /// + /// # Arguments + /// + /// * `tensor` - The tensor where will be overwritten with the value + /// when the corresponding element of the mask is true. + /// * `mask` - The boolean mask to use for filling elements. + /// * `value` - The value to fill elements with when the corresponding element of the mask is true. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensors, where each element is taken from the + /// corresponding element unmodified if the corresponding element of the mask is false, and + /// filled with the value otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For filling elements of a tensor based on a boolean mask, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("mask_fill"))] + #[cfg_attr(not(doc), doc = "`Tensor::mask_fill`")] + /// function, which is more high-level and designed for public use. + fn mask_fill( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + value: Scalar, + ) -> Self::Primitive; + + /// Gathers elements from a tensor along an axis. + /// + /// # Arguments + /// + /// * `dim` - The axis along which to gather elements. + /// * `tensor` - The tensor to gather elements from. + /// * `indices` - The indices of the elements to gather. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is taken from the + /// corresponding element of the input tensor at the corresponding index along the specified axis. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For gathering elements from a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("gather"))] + #[cfg_attr(not(doc), doc = "`Tensor::gather`")] + /// function, which is more high-level and designed for public use. + fn gather(dim: usize, tensor: Self::Primitive, indices: IntTensor) -> Self::Primitive; + + /// Scatters elements into a tensor along an axis. + /// + /// # Arguments + /// + /// * `dim` - The axis along which to scatter elements. + /// * `tensor` - The tensor to scatter elements into. + /// * `indices` - The indices of the elements to scatter. + /// * `values` - The values to scatter into the tensor. + /// * `update` - The operation used to update the existing values at the indexed positions (e.g., add). + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is taken from the + /// corresponding element of the input tensor at the corresponding index along the specified axis, + /// except for the elements at the specified indices, which are taken from the corresponding + /// element of the values tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For scattering elements into a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("scatter"))] + #[cfg_attr(not(doc), doc = "`Tensor::scatter`")] + /// function, which is more high-level and designed for public use. + fn scatter( + dim: usize, + tensor: Self::Primitive, + indices: IntTensor, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive; + + /// Returns the device on which the tensor is allocated. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The device on which the tensor is allocated. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the device of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("device"))] + #[cfg_attr(not(doc), doc = "`Tensor::device`")] + /// function, which is more high-level and designed for public use. + fn device(tensor: &Self::Primitive) -> B::Device; + + /// Moves the tensor to the given device. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `device` - The device on which the tensor will be moved. + /// + /// # Returns + /// + /// The tensor on the given device. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For moving a tensor to a device, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("to_device"))] + #[cfg_attr(not(doc), doc = "`Tensor::to_device`")] + /// function, which is more high-level and designed for public use. + #[allow(clippy::wrong_self_convention)] + fn to_device(tensor: Self::Primitive, device: &B::Device) -> Self::Primitive; + + /// Extracts the data from the tensor asynchronously. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The data of the tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For extracting the data of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("into_data"))] + #[cfg_attr(not(doc), doc = "`Tensor::into_data`")] + /// function, which is more high-level and designed for public use. + #[allow(clippy::wrong_self_convention)] + fn into_data_async( + tensor: Self::Primitive, + ) -> impl Future> + Send; + + /// Read the data from the tensor using a transaction. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + fn register_transaction(tr: &mut TransactionPrimitive, tensor: Self::Primitive); + + /// Creates a tensor from the given data. + /// + /// # Arguments + /// + /// * `data` - The data of the tensor. + /// * `device` - The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// The tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For creating a tensor from data, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("from_data"))] + #[cfg_attr(not(doc), doc = "`Tensor::from_data`")] + /// function, which is more high-level and designed for public use. + fn from_data(data: TensorData, device: &B::Device) -> Self::Primitive; + /// Creates a tensor from the given data enforcing the given data type. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For creating a tensor from data, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("from_data_dtype"))] + #[cfg_attr(not(doc), doc = "`Tensor::from_data_dtype`")] + /// function, which is more high-level and designed for public use. + fn from_data_dtype(data: TensorData, device: &B::Device, dtype: DType) -> Self::Primitive; + + /// Repeat the tensor along the given dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// * `dim` - The dimension along which the tensor will be repeated. + /// * `times` - The number of times the tensor will be repeated. + /// + /// # Returns + /// + /// The repeated tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For repeating a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("repeat_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::repeat_dim`")] + /// function, which is more high-level and designed for public use. + fn repeat_dim(tensor: Self::Primitive, dim: usize, times: usize) -> Self::Primitive; + + /// Concatenates the given tensors along the given dimension. + /// + /// # Arguments + /// + /// * `vectors` - The tensors to concatenate. + /// * `dim` - The dimension along which the tensors will be concatenated. + /// + /// # Returns + /// + /// The concatenated tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For concatenating tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("cat"))] + #[cfg_attr(not(doc), doc = "`Tensor::cat`")] + /// function, which is more high-level and designed for public use. + fn cat(vectors: Vec, dim: usize) -> Self::Primitive; + + /// Equates the given tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor of booleans indicating whether the corresponding elements are equal. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For equating tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("equal"))] + #[cfg_attr(not(doc), doc = "`Tensor::equal`")] + /// function, which is more high-level and designed for public use. + fn equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Element-wise equality between two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors, where each element is true if the + /// corresponding elements of the input tensors are equal, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise equality between two tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("equal_elem"))] + #[cfg_attr(not(doc), doc = "`Tensor::equal_elem`")] + /// function, which is more high-level and designed for public use. + fn equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive; + + /// Applies element-wise non-equality comparison between the given tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The tensor of booleans indicating whether the corresponding elements are equal. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For non-equality comparison of tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("not_equal"))] + #[cfg_attr(not(doc), doc = "`Tensor::not_equal`")] + /// function, which is more high-level and designed for public use. + fn not_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Element-wise non-equality between two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors, where each element is true if the + /// corresponding elements of the input tensors are equal, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise non-equality between two tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("not_equal_elem"))] + #[cfg_attr(not(doc), doc = "`Tensor::not_equal_elem`")] + /// function, which is more high-level and designed for public use. + fn not_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive; + + /// Returns the name of the element type. + fn elem_type_name() -> &'static str { + core::any::type_name::() + } + + /// Returns the tensor data type. + fn dtype(tensor: &Self::Primitive) -> DType { + tensor.dtype() + } + + /// Tests if any element in the `tensor` evaluates to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with a single element, True if any element in the input tensor evaluates to True, False otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("any"))] + #[cfg_attr(not(doc), doc = "`Tensor::any`")] + /// function, which is more high-level and designed for public use. + fn any(tensor: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Tests if any element in the tensor evaluates to True along a given dimension dim. + /// + /// # Arguments + /// + /// * tensor - The tensor to test. + /// * dim - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor with the same size as input tensor, except in the dim axis where the size is 1. + /// Returns True if any element in the input tensor along the given dimension evaluates to True, False otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("any_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::any_dim`")] + /// function, which is more high-level and designed for public use. + fn any_dim(tensor: Self::Primitive, dim: usize) -> B::BoolTensorPrimitive; + + /// Tests if all elements in the `tensor` evaluate to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with a single element, True if all elements in the input tensor evaluates to True, False otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("all"))] + #[cfg_attr(not(doc), doc = "`Tensor::all`")] + /// function, which is more high-level and designed for public use. + fn all(tensor: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Tests if all elements in the `tensor` evaluate to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. + /// + /// # Returns + /// + /// A boolean tensor with the same size as input `tensor`, except in the `dim` axis where the size is 1. + /// Returns True if all elements in the input tensor along the given dimension evaluate to True, False otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("all_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::all_dim`")] + /// function, which is more high-level and designed for public use. + fn all_dim(tensor: Self::Primitive, dim: usize) -> B::BoolTensorPrimitive; + + /// Broadcasts the given tensor to the specified shape. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to broadcast. + /// * `shape` - The shape to broadcast to. + /// + /// # Returns + /// + /// The broadcasted tensor. + fn expand(tensor: Self::Primitive, shape: Shape) -> Self::Primitive; + + /// Unfold windows along a dimension. + /// + /// Returns a view of the tensor with all complete windows of size `size` in dimension `dim`; + /// where windows are advanced by `step` at each index. + /// + /// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. + /// + /// # Warning + /// + /// For the `ndarray` and `candle` backends; this is not a view but a full copy. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor to unfold; of shape ``[pre=..., dim shape, post=...]`` + /// * `dim` - the dimension to unfold. + /// * `size` - the size of each unfolded window. + /// * `step` - the step between each window. + /// + /// # Returns + /// + /// A tensor view with shape ``[pre=..., windows, post=..., size]``. + fn unfold(tensor: Self::Primitive, dim: usize, size: usize, step: usize) -> Self::Primitive; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/bool.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/bool.rs new file mode 100644 index 0000000..f43542e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/bool.rs @@ -0,0 +1,218 @@ +use alloc::vec::Vec; +use burn_std::{DType, Shape, Slice}; + +use crate::{ + AutodiffBackend, Backend, ExecutionError, Scalar, TensorData, + element::Element, + ops::TransactionPrimitive, + tensor::{BasicAutodiffOps, BasicOps, Bool, Device, IndexingUpdateOp, IntTensor, TensorKind}, +}; + +impl BasicOps for Bool { + type Elem = B::BoolElem; + + fn empty(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + if dtype != Self::Elem::dtype() { + panic!("Expected bool data type, got {dtype:?}"); + } + B::bool_empty(shape, device) + } + + fn zeros(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + if dtype != Self::Elem::dtype() { + panic!("Expected bool data type, got {dtype:?}"); + } + B::bool_zeros(shape, device) + } + fn ones(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + if dtype != Self::Elem::dtype() { + panic!("Expected bool data type, got {dtype:?}"); + } + B::bool_ones(shape, device) + } + + fn full(shape: Shape, fill_value: Scalar, device: &Device, dtype: DType) -> Self::Primitive { + if dtype != Self::Elem::dtype() { + panic!("Expected bool data type, got {dtype:?}"); + } + if fill_value.elem() { + B::bool_ones(shape, device) + } else { + B::bool_zeros(shape, device) + } + } + + fn register_transaction(tr: &mut TransactionPrimitive, tensor: Self::Primitive) { + tr.register_bool(tensor); + } + + fn reshape(tensor: Self::Primitive, shape: Shape) -> Self::Primitive { + B::bool_reshape(tensor, shape) + } + + fn transpose(tensor: Self::Primitive) -> Self::Primitive { + B::bool_transpose(tensor) + } + + fn swap_dims(tensor: Self::Primitive, dim1: usize, dim2: usize) -> Self::Primitive { + B::bool_swap_dims(tensor, dim1, dim2) + } + + fn slice(tensor: Self::Primitive, slices: &[Slice]) -> Self::Primitive { + B::bool_slice(tensor, slices) + } + + fn slice_assign( + tensor: Self::Primitive, + slices: &[Slice], + value: Self::Primitive, + ) -> Self::Primitive { + B::bool_slice_assign(tensor, slices, value) + } + + fn select(tensor: Self::Primitive, dim: usize, indices: IntTensor) -> Self::Primitive { + B::bool_select(tensor, dim, indices) + } + + fn select_assign( + tensor: Self::Primitive, + dim: usize, + indices: IntTensor, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive { + match update { + IndexingUpdateOp::Add => B::bool_select_or(tensor, dim, indices, values), + } + } + + fn mask_where( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + source: Self::Primitive, + ) -> Self::Primitive { + B::bool_mask_where(tensor, mask, source) + } + + fn mask_fill( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + value: Scalar, + ) -> Self::Primitive { + B::bool_mask_fill(tensor, mask, value) + } + + fn gather( + dim: usize, + tensor: Self::Primitive, + indices: B::IntTensorPrimitive, + ) -> Self::Primitive { + B::bool_gather(dim, tensor, indices) + } + + fn scatter( + dim: usize, + tensor: Self::Primitive, + indices: B::IntTensorPrimitive, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive { + match update { + IndexingUpdateOp::Add => B::bool_scatter_or(dim, tensor, indices, values), + } + } + + fn device(tensor: &Self::Primitive) -> Device { + B::bool_device(tensor) + } + + fn to_device(tensor: Self::Primitive, device: &Device) -> Self::Primitive { + B::bool_to_device(tensor, device) + } + + async fn into_data_async(tensor: Self::Primitive) -> Result { + B::bool_into_data(tensor).await + } + + fn from_data(data: TensorData, device: &Device) -> Self::Primitive { + B::bool_from_data(data.convert::(), device) + } + + fn from_data_dtype(data: TensorData, device: &Device, _dtype: DType) -> Self::Primitive { + // Bool tensors have exactly one representation per backend, so the + // requested dtype is irrelevant. Convert to `B::BoolElem` directly. + B::bool_from_data(data.convert::(), device) + } + + fn repeat_dim(tensor: Self::Primitive, dim: usize, times: usize) -> Self::Primitive { + B::bool_repeat_dim(tensor, dim, times) + } + + fn equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::bool_equal(lhs, rhs) + } + + fn not_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::bool_not_equal(lhs, rhs) + } + + fn equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::bool_equal_elem(lhs, rhs) + } + + fn not_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::bool_not_equal_elem(lhs, rhs) + } + + fn cat(vectors: Vec, dim: usize) -> Self::Primitive { + B::bool_cat(vectors, dim) + } + + fn any(tensor: Self::Primitive) -> B::BoolTensorPrimitive { + B::bool_any(tensor) + } + + fn any_dim(tensor: Self::Primitive, dim: usize) -> B::BoolTensorPrimitive { + B::bool_any_dim(tensor, dim) + } + + fn all(tensor: Self::Primitive) -> B::BoolTensorPrimitive { + B::bool_all(tensor) + } + + fn all_dim(tensor: Self::Primitive, dim: usize) -> B::BoolTensorPrimitive { + B::bool_all_dim(tensor, dim) + } + + fn permute(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive { + B::bool_permute(tensor, axes) + } + + fn expand(tensor: Self::Primitive, shape: Shape) -> Self::Primitive { + B::bool_expand(tensor, shape) + } + + fn flip(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive { + B::bool_flip(tensor, axes) + } + + fn unfold(tensor: Self::Primitive, dim: usize, size: usize, step: usize) -> Self::Primitive { + B::bool_unfold(tensor, dim, size, step) + } +} + +impl BasicAutodiffOps for Bool { + type InnerKind = Bool; + + fn inner( + tensor: >::Primitive, + ) -> ::InnerBackend>>::Primitive { + B::bool_inner(tensor) + } + + fn from_inner( + inner: ::InnerBackend>>::Primitive, + ) -> >::Primitive { + B::bool_from_inner(inner) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/float.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/float.rs new file mode 100644 index 0000000..9ef1d95 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/float.rs @@ -0,0 +1,700 @@ +use alloc::vec::Vec; +use burn_std::{DType, Shape, Slice}; + +use crate::{ + AutodiffBackend, Backend, Distribution, ExecutionError, Scalar, TensorData, TensorPrimitive, + ops::TransactionPrimitive, + tensor::{ + BasicAutodiffOps, BasicOps, Device, Float, IndexingUpdateOp, IntTensor, Numeric, Ordered, + TensorKind, + }, +}; + +macro_rules! q_bin_ops { + ($lhs:ident, $rhs:ident, $op:ident, $q_op:ident) => { + match ($lhs, $rhs) { + (TensorPrimitive::Float(lhs), TensorPrimitive::Float(rhs)) => { + TensorPrimitive::Float(B::$op(lhs, rhs)) + } + (TensorPrimitive::QFloat(lhs), TensorPrimitive::QFloat(rhs)) => B::$q_op(lhs, rhs), + (TensorPrimitive::QFloat(lhs), TensorPrimitive::Float(rhs)) => { + TensorPrimitive::Float(B::$op(B::dequantize(lhs), rhs)) + } + (TensorPrimitive::Float(lhs), TensorPrimitive::QFloat(rhs)) => { + TensorPrimitive::Float(B::$op(lhs, B::dequantize(rhs))) + } + } + }; +} + +impl BasicOps for Float { + type Elem = B::FloatElem; + + fn empty(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + TensorPrimitive::Float(B::float_empty(shape, device, dtype.into())) + } + + fn zeros(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + TensorPrimitive::Float(B::float_zeros(shape, device, dtype.into())) + } + fn ones(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + TensorPrimitive::Float(B::float_ones(shape, device, dtype.into())) + } + + fn full(shape: Shape, fill_value: Scalar, device: &Device, dtype: DType) -> Self::Primitive { + TensorPrimitive::Float(B::float_full(shape, fill_value, device, dtype.into())) + } + + fn register_transaction(tr: &mut TransactionPrimitive, tensor: Self::Primitive) { + tr.register_float(tensor); + } + + fn reshape(tensor: Self::Primitive, shape: Shape) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_reshape(tensor, shape)) + } + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_reshape(tensor, shape)), + } + } + + fn transpose(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_transpose(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_transpose(tensor)), + } + } + + fn swap_dims(tensor: Self::Primitive, dim1: usize, dim2: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_swap_dims(tensor, dim1, dim2)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_swap_dims(tensor, dim1, dim2)) + } + } + } + + fn slice(tensor: Self::Primitive, slices: &[Slice]) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_slice(tensor, slices)) + } + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_slice(tensor, slices)), + } + } + + fn slice_assign( + tensor: Self::Primitive, + slices: &[Slice], + value: Self::Primitive, + ) -> Self::Primitive { + TensorPrimitive::Float(B::float_slice_assign( + tensor.tensor(), + slices, + value.tensor(), + )) + } + + fn select(tensor: Self::Primitive, dim: usize, indices: IntTensor) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_select(tensor, dim, indices)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_select(tensor, dim, indices)) + } + } + } + + fn select_assign( + tensor: Self::Primitive, + dim: usize, + indices: IntTensor, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive { + // Select assign is ambiguous for QFloat + match update { + IndexingUpdateOp::Add => TensorPrimitive::Float(B::float_select_add( + tensor.tensor(), + dim, + indices, + values.tensor(), + )), + } + } + + fn mask_where( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + source: Self::Primitive, + ) -> Self::Primitive { + TensorPrimitive::Float(B::float_mask_where(tensor.tensor(), mask, source.tensor())) + } + + fn mask_fill( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + value: Scalar, + ) -> Self::Primitive { + TensorPrimitive::Float(B::float_mask_fill(tensor.tensor(), mask, value)) + } + + fn gather(dim: usize, tensor: Self::Primitive, indices: IntTensor) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_gather(dim, tensor, indices)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_gather(dim, tensor, indices)) + } + } + } + + fn scatter( + dim: usize, + tensor: Self::Primitive, + indices: IntTensor, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive { + match update { + IndexingUpdateOp::Add => TensorPrimitive::Float(B::float_scatter_add( + dim, + tensor.tensor(), + indices, + values.tensor(), + )), + } + } + + fn device(tensor: &Self::Primitive) -> Device { + match tensor { + TensorPrimitive::Float(tensor) => B::float_device(tensor), + TensorPrimitive::QFloat(tensor) => B::q_device(tensor), + } + } + + fn to_device(tensor: Self::Primitive, device: &Device) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_to_device(tensor, device)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_to_device(tensor, device)) + } + } + } + + async fn into_data_async(tensor: Self::Primitive) -> Result { + match tensor { + TensorPrimitive::Float(tensor) => B::float_into_data(tensor).await, + TensorPrimitive::QFloat(tensor) => B::q_into_data(tensor).await, + } + } + + fn from_data(data: TensorData, device: &Device) -> Self::Primitive { + match &data.dtype { + DType::QFloat(_scheme) => TensorPrimitive::QFloat(B::q_from_data(data, device)), + _ => TensorPrimitive::Float(B::float_from_data(data.convert::(), device)), + } + } + + fn from_data_dtype(data: TensorData, device: &Device, dtype: DType) -> Self::Primitive { + match dtype { + DType::QFloat(_scheme) => { + TensorPrimitive::QFloat(B::q_from_data(data.convert_dtype(dtype), device)) + } + _ if dtype.is_float() => { + TensorPrimitive::Float(B::float_from_data(data.convert_dtype(dtype), device)) + } + _ => panic!("Expected float dtype, got {dtype:?}"), + } + } + + fn repeat_dim(tensor: Self::Primitive, dim: usize, times: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_repeat_dim(tensor, dim, times)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_repeat_dim(tensor, dim, times)) + } + } + } + + fn cat(vectors: Vec, dim: usize) -> Self::Primitive { + match vectors.first().unwrap() { + TensorPrimitive::Float(_) => TensorPrimitive::Float(B::float_cat( + vectors.into_iter().map(|tensor| tensor.tensor()).collect(), + dim, + )), + TensorPrimitive::QFloat(_) => TensorPrimitive::QFloat(B::q_cat( + vectors + .into_iter() + .map(|tensor| { + if let TensorPrimitive::QFloat(t) = tensor { + t + } else { + panic!("Concatenation only works with vector of QFloat") + } + }) + .collect(), + dim, + )), + } + } + + fn equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_equal(lhs.tensor(), rhs.tensor()) + } + + fn not_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_not_equal(lhs.tensor(), rhs.tensor()) + } + + fn equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::float_equal_elem(lhs.tensor(), rhs) + } + + fn not_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::float_not_equal_elem(lhs.tensor(), rhs) + } + + fn any(tensor: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_any(tensor.tensor()) + } + + fn any_dim(tensor: Self::Primitive, dim: usize) -> B::BoolTensorPrimitive { + B::float_any_dim(tensor.tensor(), dim) + } + + fn all(tensor: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_all(tensor.tensor()) + } + + fn all_dim(tensor: Self::Primitive, dim: usize) -> B::BoolTensorPrimitive { + B::float_all_dim(tensor.tensor(), dim) + } + + fn permute(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_permute(tensor, axes)) + } + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_permute(tensor, axes)), + } + } + + fn expand(tensor: Self::Primitive, shape: Shape) -> Self::Primitive { + TensorPrimitive::Float(B::float_expand(tensor.tensor(), shape)) + } + + fn flip(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_flip(tensor, axes)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_flip(tensor, axes)), + } + } + + fn unfold(tensor: Self::Primitive, dim: usize, size: usize, step: usize) -> Self::Primitive { + TensorPrimitive::Float(B::float_unfold(tensor.tensor(), dim, size, step)) + } +} + +impl Numeric for Float { + fn add(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + q_bin_ops!(lhs, rhs, float_add, q_add) + } + + fn add_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + match lhs { + TensorPrimitive::Float(lhs) => TensorPrimitive::Float(B::float_add_scalar(lhs, rhs)), + TensorPrimitive::QFloat(lhs) => B::q_add_scalar(lhs, rhs), + } + } + + fn sub(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + q_bin_ops!(lhs, rhs, float_sub, q_sub) + } + + fn sub_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + match lhs { + TensorPrimitive::Float(lhs) => TensorPrimitive::Float(B::float_sub_scalar(lhs, rhs)), + TensorPrimitive::QFloat(lhs) => B::q_sub_scalar(lhs, rhs), + } + } + + fn div(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + q_bin_ops!(lhs, rhs, float_div, q_div) + } + + fn div_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + match lhs { + TensorPrimitive::Float(lhs) => TensorPrimitive::Float(B::float_div_scalar(lhs, rhs)), + TensorPrimitive::QFloat(lhs) => B::q_div_scalar(lhs, rhs), + } + } + fn remainder(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + TensorPrimitive::Float(B::float_remainder(lhs.tensor(), rhs.tensor())) + } + + fn remainder_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + TensorPrimitive::Float(B::float_remainder_scalar(lhs.tensor(), rhs)) + } + + fn mul(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + q_bin_ops!(lhs, rhs, float_mul, q_mul) + } + + fn mul_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + match lhs { + TensorPrimitive::Float(lhs) => TensorPrimitive::Float(B::float_mul_scalar(lhs, rhs)), + TensorPrimitive::QFloat(lhs) => B::q_mul_scalar(lhs, rhs), + } + } + fn neg(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_neg(tensor)), + TensorPrimitive::QFloat(tensor) => B::q_neg(tensor), + } + } + + fn sum(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_sum(tensor)), + TensorPrimitive::QFloat(tensor) => B::q_sum(tensor), + } + } + + fn sum_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_sum_dim(tensor, dim)), + TensorPrimitive::QFloat(tensor) => B::q_sum_dim(tensor, dim), + } + } + + fn prod(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_prod(tensor)), + TensorPrimitive::QFloat(tensor) => B::q_prod(tensor), + } + } + + fn prod_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_prod_dim(tensor, dim)) + } + TensorPrimitive::QFloat(tensor) => B::q_prod_dim(tensor, dim), + } + } + + fn mean(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_mean(tensor)), + TensorPrimitive::QFloat(tensor) => B::q_mean(tensor), + } + } + + fn mean_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_mean_dim(tensor, dim)) + } + TensorPrimitive::QFloat(tensor) => B::q_mean_dim(tensor, dim), + } + } + + fn cumsum(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_cumsum(tensor, dim)), + TensorPrimitive::QFloat(tensor) => B::q_cumsum(tensor, dim), + } + } + + fn cumprod(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_cumprod(tensor, dim)), + TensorPrimitive::QFloat(tensor) => B::q_cumprod(tensor, dim), + } + } + + fn abs(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_abs(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_abs(tensor)), + } + } + + fn powf(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + q_bin_ops!(lhs, rhs, float_powf, q_powf) + } + + fn powf_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + match lhs { + TensorPrimitive::Float(lhs) => TensorPrimitive::Float(B::float_powf_scalar(lhs, rhs)), + TensorPrimitive::QFloat(lhs) => B::q_powf_scalar(lhs, rhs), + } + } + + fn powi(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + q_bin_ops!(lhs, rhs, float_powf, q_powf) + } + + fn powi_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + match lhs { + TensorPrimitive::Float(lhs) => TensorPrimitive::Float(B::float_powi_scalar(lhs, rhs)), + TensorPrimitive::QFloat(lhs) => B::q_powi_scalar(lhs, rhs), + } + } + + fn random(shape: Shape, distribution: Distribution, device: &Device) -> Self::Primitive { + TensorPrimitive::Float(B::float_random(shape, distribution, device)) + } + + fn sign(tensor: Self::Primitive) -> Self::Primitive { + TensorPrimitive::Float(B::float_sign(tensor.tensor())) + } + + /// Applies the matrix multiplication operation. + /// + /// `C = AB` + /// + /// # Panics + /// + /// If the two tensors don't have a compatible shape. + fn matmul(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + match (lhs, rhs) { + (TensorPrimitive::Float(lhs), TensorPrimitive::Float(rhs)) => { + TensorPrimitive::Float(B::float_matmul(lhs, rhs)) + } + (lhs, rhs) => B::q_matmul(lhs, rhs), + } + } +} +impl Ordered for Float { + fn sort(tensor: Self::Primitive, dim: usize, descending: bool) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_sort(tensor, dim, descending)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_sort(tensor, dim, descending)) + } + } + } + + fn sort_with_indices( + tensor: Self::Primitive, + dim: usize, + descending: bool, + ) -> (Self::Primitive, IntTensor) { + match tensor { + TensorPrimitive::Float(tensor) => { + let (values, indices) = B::float_sort_with_indices(tensor, dim, descending); + (TensorPrimitive::Float(values), indices) + } + TensorPrimitive::QFloat(tensor) => { + let (values, indices) = B::q_sort_with_indices(tensor, dim, descending); + (TensorPrimitive::QFloat(values), indices) + } + } + } + + fn argsort(tensor: Self::Primitive, dim: usize, descending: bool) -> IntTensor { + match tensor { + TensorPrimitive::Float(tensor) => B::float_argsort(tensor, dim, descending), + TensorPrimitive::QFloat(tensor) => B::q_argsort(tensor, dim, descending), + } + } + + fn cummin(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_cummin(tensor, dim)), + TensorPrimitive::QFloat(tensor) => B::q_cummin(tensor, dim), + } + } + + fn cummax(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_cummax(tensor, dim)), + TensorPrimitive::QFloat(tensor) => B::q_cummax(tensor, dim), + } + } + + fn greater(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_greater(lhs.tensor(), rhs.tensor()) + } + + fn greater_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::float_greater_elem(lhs.tensor(), rhs) + } + + fn greater_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_greater_equal(lhs.tensor(), rhs.tensor()) + } + + fn greater_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::float_greater_equal_elem(lhs.tensor(), rhs) + } + + fn lower(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_lower(lhs.tensor(), rhs.tensor()) + } + + fn lower_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::float_lower_elem(lhs.tensor(), rhs) + } + + fn lower_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::float_lower_equal(lhs.tensor(), rhs.tensor()) + } + + fn lower_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::float_lower_equal_elem(lhs.tensor(), rhs) + } + + fn argmax(tensor: Self::Primitive, dim: usize) -> IntTensor { + match tensor { + TensorPrimitive::Float(tensor) => B::float_argmax(tensor, dim), + TensorPrimitive::QFloat(tensor) => B::q_argmax(tensor, dim), + } + } + + fn argmin(tensor: Self::Primitive, dim: usize) -> IntTensor { + match tensor { + TensorPrimitive::Float(tensor) => B::float_argmin(tensor, dim), + TensorPrimitive::QFloat(tensor) => B::q_argmin(tensor, dim), + } + } + + fn max(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_max(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_max(tensor)), + } + } + + fn max_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_max_dim(tensor, dim)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_max_dim(tensor, dim)), + } + } + + fn max_dim_with_indices( + tensor: Self::Primitive, + dim: usize, + ) -> (Self::Primitive, IntTensor) { + match tensor { + TensorPrimitive::Float(tensor) => { + let (values, indices) = B::float_max_dim_with_indices(tensor, dim); + (TensorPrimitive::Float(values), indices) + } + TensorPrimitive::QFloat(tensor) => { + let (values, indices) = B::q_max_dim_with_indices(tensor, dim); + (TensorPrimitive::QFloat(values), indices) + } + } + } + + fn min(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_min(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_min(tensor)), + } + } + + fn min_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_min_dim(tensor, dim)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_min_dim(tensor, dim)), + } + } + + fn min_dim_with_indices( + tensor: Self::Primitive, + dim: usize, + ) -> (Self::Primitive, IntTensor) { + match tensor { + TensorPrimitive::Float(tensor) => { + let (values, indices) = B::float_min_dim_with_indices(tensor, dim); + (TensorPrimitive::Float(values), indices) + } + TensorPrimitive::QFloat(tensor) => { + let (values, indices) = B::q_min_dim_with_indices(tensor, dim); + (TensorPrimitive::QFloat(values), indices) + } + } + } + + fn clamp(tensor: Self::Primitive, min: Scalar, max: Scalar) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_clamp(tensor, min, max)) + } + TensorPrimitive::QFloat(tensor) => B::q_clamp(tensor, min, max), + } + } + + fn clamp_min(tensor: Self::Primitive, min: Scalar) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_clamp_min(tensor, min)) + } + TensorPrimitive::QFloat(tensor) => B::q_clamp_min(tensor, min), + } + } + + fn clamp_max(tensor: Self::Primitive, max: Scalar) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_clamp_max(tensor, max)) + } + TensorPrimitive::QFloat(tensor) => B::q_clamp_max(tensor, max), + } + } + + fn max_abs(tensor: Self::Primitive) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::float_max_abs(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_max_abs(tensor)), + } + } + + fn max_abs_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_max_abs_dim(tensor, dim)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_max_abs_dim(tensor, dim)) + } + } + } +} + +impl BasicAutodiffOps for Float { + type InnerKind = Float; + + fn inner( + tensor: >::Primitive, + ) -> ::InnerBackend>>::Primitive { + match tensor { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::inner(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_inner(tensor)), + } + } + + fn from_inner( + inner: ::InnerBackend>>::Primitive, + ) -> >::Primitive { + match inner { + TensorPrimitive::Float(tensor) => TensorPrimitive::Float(B::from_inner(tensor)), + TensorPrimitive::QFloat(tensor) => TensorPrimitive::QFloat(B::q_from_inner(tensor)), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/int.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/int.rs new file mode 100644 index 0000000..63ad724 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/int.rs @@ -0,0 +1,426 @@ +use alloc::vec::Vec; +use burn_std::{DType, Shape, Slice}; + +use crate::{ + AutodiffBackend, Backend, Distribution, ExecutionError, Scalar, TensorData, + ops::TransactionPrimitive, + tensor::{ + BasicAutodiffOps, BasicOps, BoolTensor, Device, IndexingUpdateOp, Int, IntTensor, Numeric, + Ordered, TensorKind, + }, +}; + +impl BasicOps for Int { + type Elem = B::IntElem; + + fn empty(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + B::int_empty(shape, device, dtype.into()) + } + + fn zeros(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + B::int_zeros(shape, device, dtype.into()) + } + fn ones(shape: Shape, device: &Device, dtype: DType) -> Self::Primitive { + B::int_ones(shape, device, dtype.into()) + } + + fn full(shape: Shape, fill_value: Scalar, device: &Device, dtype: DType) -> Self::Primitive { + B::int_full(shape, fill_value, device, dtype.into()) + } + + fn register_transaction(tr: &mut TransactionPrimitive, tensor: Self::Primitive) { + tr.register_int(tensor); + } + + fn reshape(tensor: Self::Primitive, shape: Shape) -> Self::Primitive { + B::int_reshape(tensor, shape) + } + + fn transpose(tensor: Self::Primitive) -> Self::Primitive { + B::int_transpose(tensor) + } + + fn swap_dims(tensor: Self::Primitive, dim1: usize, dim2: usize) -> Self::Primitive { + B::int_swap_dims(tensor, dim1, dim2) + } + + fn slice(tensor: Self::Primitive, slices: &[Slice]) -> Self::Primitive { + B::int_slice(tensor, slices) + } + + fn slice_assign( + tensor: Self::Primitive, + slices: &[Slice], + value: Self::Primitive, + ) -> Self::Primitive { + B::int_slice_assign(tensor, slices, value) + } + + fn select(tensor: Self::Primitive, dim: usize, indices: IntTensor) -> Self::Primitive { + B::int_select(tensor, dim, indices) + } + + fn select_assign( + tensor: Self::Primitive, + dim: usize, + indices: IntTensor, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive { + match update { + IndexingUpdateOp::Add => B::int_select_add(tensor, dim, indices, values), + } + } + + fn mask_where( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + source: Self::Primitive, + ) -> Self::Primitive { + B::int_mask_where(tensor, mask, source) + } + + fn mask_fill( + tensor: Self::Primitive, + mask: B::BoolTensorPrimitive, + value: Scalar, + ) -> Self::Primitive { + B::int_mask_fill(tensor, mask, value) + } + + fn gather( + dim: usize, + tensor: Self::Primitive, + indices: B::IntTensorPrimitive, + ) -> Self::Primitive { + B::int_gather(dim, tensor, indices) + } + + fn scatter( + dim: usize, + tensor: Self::Primitive, + indices: B::IntTensorPrimitive, + values: Self::Primitive, + update: IndexingUpdateOp, + ) -> Self::Primitive { + match update { + IndexingUpdateOp::Add => B::int_scatter_add(dim, tensor, indices, values), + } + } + + fn device(tensor: &Self::Primitive) -> Device { + B::int_device(tensor) + } + + fn to_device(tensor: Self::Primitive, device: &Device) -> Self::Primitive { + B::int_to_device(tensor, device) + } + + async fn into_data_async(tensor: Self::Primitive) -> Result { + B::int_into_data(tensor).await + } + + fn from_data(data: TensorData, device: &Device) -> Self::Primitive { + B::int_from_data(data.convert::(), device) + } + + fn from_data_dtype(data: TensorData, device: &Device, dtype: DType) -> Self::Primitive { + if !dtype.is_int() { + panic!("Expected int dtype, got {dtype:?}") + } + + B::int_from_data(data.convert_dtype(dtype), device) + } + + fn repeat_dim(tensor: Self::Primitive, dim: usize, times: usize) -> Self::Primitive { + B::int_repeat_dim(tensor, dim, times) + } + + fn equal(lhs: Self::Primitive, rhs: Self::Primitive) -> BoolTensor { + B::int_equal(lhs, rhs) + } + + fn not_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> BoolTensor { + B::int_not_equal(lhs, rhs) + } + + fn equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::int_equal_elem(lhs, rhs) + } + + fn not_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::int_not_equal_elem(lhs, rhs) + } + + fn cat(vectors: Vec, dim: usize) -> Self::Primitive { + B::int_cat(vectors, dim) + } + + fn any(tensor: Self::Primitive) -> BoolTensor { + B::int_any(tensor) + } + + fn any_dim(tensor: Self::Primitive, dim: usize) -> BoolTensor { + B::int_any_dim(tensor, dim) + } + + fn all(tensor: Self::Primitive) -> BoolTensor { + B::int_all(tensor) + } + + fn all_dim(tensor: Self::Primitive, dim: usize) -> BoolTensor { + B::int_all_dim(tensor, dim) + } + + fn permute(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive { + B::int_permute(tensor, axes) + } + + fn expand(tensor: Self::Primitive, shape: Shape) -> Self::Primitive { + B::int_expand(tensor, shape) + } + + fn flip(tensor: Self::Primitive, axes: &[usize]) -> Self::Primitive { + B::int_flip(tensor, axes) + } + + fn unfold(tensor: Self::Primitive, dim: usize, size: usize, step: usize) -> Self::Primitive { + B::int_unfold(tensor, dim, size, step) + } +} + +impl Numeric for Int { + fn add(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_add(lhs, rhs) + } + fn add_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_add_scalar(lhs, rhs) + } + fn sub(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_sub(lhs, rhs) + } + fn sub_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_sub_scalar(lhs, rhs) + } + fn div(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_div(lhs, rhs) + } + fn div_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_div_scalar(lhs, rhs) + } + fn remainder(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_remainder(lhs, rhs) + } + fn remainder_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_remainder_scalar(lhs, rhs) + } + fn mul(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_mul(lhs, rhs) + } + fn mul_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_mul_scalar(lhs, rhs) + } + fn neg(tensor: Self::Primitive) -> Self::Primitive { + B::int_neg(tensor) + } + + fn sum(tensor: Self::Primitive) -> Self::Primitive { + B::int_sum(tensor) + } + + fn sum_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_sum_dim(tensor, dim) + } + + fn prod(tensor: Self::Primitive) -> Self::Primitive { + B::int_prod(tensor) + } + + fn prod_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_prod_dim(tensor, dim) + } + + fn mean(tensor: Self::Primitive) -> Self::Primitive { + B::int_mean(tensor) + } + fn mean_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_mean_dim(tensor, dim) + } + fn cumsum(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_cumsum(tensor, dim) + } + fn cumprod(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_cumprod(tensor, dim) + } + + fn abs(tensor: Self::Primitive) -> Self::Primitive { + B::int_abs(tensor) + } + + fn powf(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_powf(lhs, B::int_into_float(rhs)) + } + + fn powf_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_powf_scalar(lhs, rhs) + } + + fn powi(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_powi(lhs, rhs) + } + + fn powi_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive { + B::int_powi_scalar(lhs, rhs) + } + + fn random(shape: Shape, distribution: Distribution, device: &Device) -> Self::Primitive { + B::int_random(shape, distribution, device) + } + + fn sign(tensor: Self::Primitive) -> Self::Primitive { + B::int_sign(tensor) + } + + /// Applies the matrix multiplication operation. + /// + /// `C = AB` + /// + /// # Panics + /// + /// If the two tensors don't have a compatible shape. + fn matmul(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive { + B::int_matmul(lhs, rhs) + } +} + +impl Ordered for Int { + fn sort(tensor: Self::Primitive, dim: usize, descending: bool) -> Self::Primitive { + B::int_sort(tensor, dim, descending) + } + + fn sort_with_indices( + tensor: Self::Primitive, + dim: usize, + descending: bool, + ) -> (Self::Primitive, IntTensor) { + B::int_sort_with_indices(tensor, dim, descending) + } + + fn argsort(tensor: Self::Primitive, dim: usize, descending: bool) -> IntTensor { + B::int_argsort(tensor, dim, descending) + } + + fn cummin(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_cummin(tensor, dim) + } + + fn cummax(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_cummax(tensor, dim) + } + + fn greater(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::int_greater(lhs, rhs) + } + + fn greater_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::int_greater_elem(lhs, rhs) + } + + fn greater_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::int_greater_equal(lhs, rhs) + } + + fn greater_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::int_greater_equal_elem(lhs, rhs) + } + + fn lower(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::int_lower(lhs, rhs) + } + + fn lower_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::int_lower_elem(lhs, rhs) + } + + fn lower_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive { + B::int_lower_equal(lhs, rhs) + } + + fn lower_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive { + B::int_lower_equal_elem(lhs, rhs) + } + + fn argmax(tensor: Self::Primitive, dim: usize) -> IntTensor { + B::int_argmax(tensor, dim) + } + + fn argmin(tensor: Self::Primitive, dim: usize) -> IntTensor { + B::int_argmin(tensor, dim) + } + + fn max(tensor: Self::Primitive) -> Self::Primitive { + B::int_max(tensor) + } + + fn max_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_max_dim(tensor, dim) + } + + fn max_dim_with_indices( + tensor: Self::Primitive, + dim: usize, + ) -> (Self::Primitive, IntTensor) { + B::int_max_dim_with_indices(tensor, dim) + } + + fn max_abs(tensor: Self::Primitive) -> Self::Primitive { + B::int_max_abs(tensor) + } + + fn max_abs_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_max_abs_dim(tensor, dim) + } + + fn min(tensor: Self::Primitive) -> Self::Primitive { + B::int_min(tensor) + } + + fn min_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive { + B::int_min_dim(tensor, dim) + } + + fn min_dim_with_indices( + tensor: Self::Primitive, + dim: usize, + ) -> (Self::Primitive, IntTensor) { + B::int_min_dim_with_indices(tensor, dim) + } + + fn clamp(tensor: Self::Primitive, min: Scalar, max: Scalar) -> Self::Primitive { + B::int_clamp(tensor, min, max) + } + + fn clamp_min(tensor: Self::Primitive, min: Scalar) -> Self::Primitive { + B::int_clamp_min(tensor, min) + } + + fn clamp_max(tensor: Self::Primitive, max: Scalar) -> Self::Primitive { + B::int_clamp_max(tensor, max) + } +} + +impl BasicAutodiffOps for Int { + type InnerKind = Int; + + fn inner( + tensor: >::Primitive, + ) -> ::InnerBackend>>::Primitive { + B::int_inner(tensor) + } + + fn from_inner( + inner: ::InnerBackend>>::Primitive, + ) -> >::Primitive { + B::int_from_inner(inner) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/mod.rs new file mode 100644 index 0000000..2174836 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/mod.rs @@ -0,0 +1,21 @@ +mod autodiff; +mod base; +mod bool; +mod float; +mod int; +mod numeric; +mod ordered; + +pub use autodiff::*; +pub use base::*; +pub use numeric::*; +pub use ordered::*; + +/// Computation to be used to update the existing values in indexed assignment operations (scatter/select). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum IndexingUpdateOp { + // Assign, + /// Performs an addition. + Add, + // Mul +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/numeric.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/numeric.rs new file mode 100644 index 0000000..16fea87 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/numeric.rs @@ -0,0 +1,556 @@ +use burn_std::Shape; + +use crate::{Backend, Distribution, Scalar, element::Element, tensor::BasicOps}; + +/// Trait that list all operations that can be applied on all numerical tensors. +/// +/// # Warnings +/// +/// This is an internal trait, use the public API provided by the +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// struct. +pub trait Numeric: BasicOps +where + Self::Elem: Element, +{ + /// Adds two tensors together. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The sum of the two tensors. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For adding tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("add"))] + #[cfg_attr(not(doc), doc = "`Tensor::add`")] + /// function, which is more high-level and designed for public use. + fn add(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Adds a scalar to a tensor element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The sum of the tensor and the scalar. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For adding a scalar to a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("add_scalar"))] + #[cfg_attr(not(doc), doc = "`Tensor::add_scalar`")] + /// function, which is more high-level and designed for public use. + fn add_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Subtracts two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The difference of the two tensors. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For subtracting tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sub"))] + #[cfg_attr(not(doc), doc = "`Tensor::sub`")] + /// function, which is more high-level and designed for public use. + fn sub(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Subtracts a scalar from a tensor element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The difference of the tensor and the scalar. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For subtracting a scalar from a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sub_scalar"))] + #[cfg_attr(not(doc), doc = "`Tensor::sub_scalar`")] + /// function, which is more high-level and designed for public use. + fn sub_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Divides two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The quotient of the two tensors. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For dividing tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("div"))] + #[cfg_attr(not(doc), doc = "`Tensor::div`")] + /// function, which is more high-level and designed for public use. + fn div(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Divides a tensor by a scalar element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The quotient of the tensor and the scalar. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For dividing a tensor by a scalar, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("div_scalar"))] + #[cfg_attr(not(doc), doc = "`Tensor::div_scalar`")] + /// function, which is more high-level and designed for public use. + fn div_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Computes the modulo element-wise. The result is the *signed* remainder of the division and its absolute value is + /// less than that of the divisor. + /// + /// # Arguments + /// + /// * `lhs` - The dividend. + /// * `rhs` - The divisor. + /// + /// # Returns + /// + /// The modulo of the input tensor with the divisor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For performing the modulo operation, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("remainder"))] + #[cfg_attr(not(doc), doc = "`Tensor::remainder`")] + /// function, which is more high-level and designed for public use. + fn remainder(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Computes the modulo element-wise. The result is the *signed* remainder of the division and its absolute value is + /// less than that of the divisor. + /// + /// # Arguments + /// + /// * `lhs` - The dividend. + /// * `rhs` - The divisor. + /// + /// # Returns + /// + /// The modulo of the input tensor with the divisor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For performing the modulo operation, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("remainder_scalar"))] + #[cfg_attr(not(doc), doc = "`Tensor::remainder_scalar`")] + /// function, which is more high-level and designed for public use. + fn remainder_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Multiplies two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// The product of the two tensors. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For multiplying tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("mul"))] + #[cfg_attr(not(doc), doc = "`Tensor::mul`")] + /// function, which is more high-level and designed for public use. + fn mul(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Multiplies a tensor by a scalar element-wise. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// The product of the tensor and the scalar. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For multiplying a tensor by a scalar, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("mul_scalar"))] + #[cfg_attr(not(doc), doc = "`Tensor::mul_scalar`")] + /// function, which is more high-level and designed for public use. + fn mul_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Negates a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to negate. + /// + /// # Returns + /// + /// The negated tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For negating a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("neg"))] + #[cfg_attr(not(doc), doc = "`Tensor::neg`")] + /// function, which is more high-level and designed for public use. + fn neg(tensor: Self::Primitive) -> Self::Primitive; + + /// Returns the signs of the elements of a tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor. + /// + /// # Returns + /// + /// The signs of the elements of the tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the signs of the elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sign"))] + #[cfg_attr(not(doc), doc = "`Tensor::sign`")] + /// function, which is more high-level and designed for public use. + fn sign(tensor: Self::Primitive) -> Self::Primitive; + + /// Sums all the elements of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// + /// # Returns + /// + /// The sum of all the elements of the tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For summing all the elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sum"))] + #[cfg_attr(not(doc), doc = "`Tensor::sum`")] + /// function, which is more high-level and designed for public use. + fn sum(tensor: Self::Primitive) -> Self::Primitive; + + /// Sums all the elements of the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to sum. + /// * `dim` - The dimension along which to sum. + /// + /// # Returns + /// + /// The sum of all the elements of the tensor along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For summing all the elements of a tensor along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sum_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::sum_dim`")] + /// function, which is more high-level and designed for public use. + fn sum_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Computes the product of all the elements of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the product of. + /// + /// # Returns + /// + /// The product of all the elements of the tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the product of all the elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("prod"))] + #[cfg_attr(not(doc), doc = "`Tensor::prod`")] + /// function, which is more high-level and designed for public use. + fn prod(tensor: Self::Primitive) -> Self::Primitive; + + /// Computes the product of all the elements of the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the product of. + /// * `dim` - The dimension along which to compute the product. + /// + /// # Returns + /// + /// The product of all the elements of the tensor along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the product of all the elements of a tensor along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("prod_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::prod_dim`")] + /// function, which is more high-level and designed for public use. + fn prod_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Computes the mean of all the elements of the tensor. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the mean of. + /// + /// # Returns + /// + /// The mean of all the elements of the tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the mean of all the elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("mean"))] + #[cfg_attr(not(doc), doc = "`Tensor::mean`")] + /// function, which is more high-level and designed for public use. + fn mean(tensor: Self::Primitive) -> Self::Primitive; + + /// Computes the mean of all the elements of the tensor along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the mean of. + /// * `dim` - The dimension along which to compute the mean. + /// + /// # Returns + /// + /// The mean of all the elements of the tensor along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the mean of all the elements of a tensor along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("mean_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::mean_dim`")] + /// function, which is more high-level and designed for public use. + fn mean_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Computes the cumulative sum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative sum of. + /// * `dim` - The dimension along which to compute the cumulative sum. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is the cumulative sum + /// of all elements up to and including that position along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the cumulative sum of elements along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("cumsum"))] + #[cfg_attr(not(doc), doc = "`Tensor::cumsum`")] + /// function, which is more high-level and designed for public use. + fn cumsum(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Computes the cumulative product of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative product of. + /// * `dim` - The dimension along which to compute the cumulative product. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is the cumulative product + /// of all elements up to and including that position along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the cumulative product of elements along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("cumprod"))] + #[cfg_attr(not(doc), doc = "`Tensor::cumprod`")] + /// function, which is more high-level and designed for public use. + fn cumprod(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Calculate absolute value on all elements of a tensor + /// + /// # Arguments + /// + /// * `tensor` - The tensor to apply abs to. + /// + /// # Returns + /// + /// A tensor with absolute values. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For calculating abs of the elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("abs"))] + #[cfg_attr(not(doc), doc = "`Tensor::abs`")] + /// function, which is more high-level and designed for public use. + fn abs(tensor: Self::Primitive) -> Self::Primitive; + + /// Element-wise power of a tensor to a float tensor + /// + /// # Arguments + /// * `tensor` - The tensor to apply power to. + /// * `power` - The power to apply to the tensor. + fn powf(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Element-wise power of a tensor + /// + /// # Arguments + /// * `tensor` - The tensor to apply power to. + /// * `power` - The power to apply to the tensor. + fn powi(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; + + /// Element-wise power of a tensor to a scalar float + /// + /// # Arguments + /// * `tensor` - The tensor to apply power to. + /// * `power` - The power to apply to the tensor. + fn powf_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Element-wise power of a tensor to a scalar int + /// + /// # Arguments + /// * `tensor` - The tensor to apply power to. + /// * `power` - The power to apply to the tensor. + fn powi_scalar(lhs: Self::Primitive, rhs: Scalar) -> Self::Primitive; + + /// Create a random tensor. + /// + /// # Arguments + /// + /// * `shape` - The shape of the output tensor. + /// * `distribution` - The distribution used to sample. + /// * `device` - The device to use. + /// + /// # Returns + /// + /// A new tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("random"))] + #[cfg_attr(not(doc), doc = "`Tensor::random`")] + /// function, which is more high-level and designed for public use. + fn random(shape: Shape, distribution: Distribution, device: &B::Device) -> Self::Primitive; + + /// Applies the matrix multiplication operation. + /// + /// ```math + /// C = AB + /// ``` + fn matmul(lhs: Self::Primitive, rhs: Self::Primitive) -> Self::Primitive; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/ordered.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/ordered.rs new file mode 100644 index 0000000..46b7208 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/ops/ordered.rs @@ -0,0 +1,650 @@ +use crate::{ + Backend, Scalar, + tensor::{IntTensor, Numeric}, +}; + +/// Trait that list all operations that can be applied on all numerical tensors +/// whose elements have a well-defined ordering. +/// +/// This includes operations such as comparisons, minimum/maximum reductions, +/// and other order-dependent computations that are not strictly valid for all numerical +/// types. +/// +/// # Warnings +/// +/// This is an internal trait, use the public API provided by the +#[cfg_attr(doc, doc = crate::doc_tensor!())] +#[cfg_attr(not(doc), doc = "`Tensor`")] +/// struct. +pub trait Ordered: Numeric { + /// Sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where the elements are sorted by value. + /// + /// # Remarks + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sort"))] + #[cfg_attr(not(doc), doc = "`Tensor::sort`")] + /// function, which is more high-level and designed for public use. + fn sort(tensor: Self::Primitive, dim: usize, descending: bool) -> Self::Primitive; + + /// Sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor and corresponding indices, where + /// the elements are sorted by value and the indices map back to the original input tensor. + /// + /// # Remarks + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For sorting the elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("sort_with_indices"))] + #[cfg_attr(not(doc), doc = "`Tensor::sort_with_indices`")] + /// function, which is more high-level and designed for public use. + fn sort_with_indices( + tensor: Self::Primitive, + dim: usize, + descending: bool, + ) -> (Self::Primitive, IntTensor); + + /// Returns the indices that sort the elements of the input `tensor` by value along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `tensor` - The input tensor. + /// * `dim` - The axis along which to sort. + /// * `descending` - The sorting order. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor the indices map back to the original input tensor. + /// + /// # Remarks + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// Users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("argsort"))] + #[cfg_attr(not(doc), doc = "`Tensor::argsort`")] + /// function, which is more high-level and designed for public use. + fn argsort(tensor: Self::Primitive, dim: usize, descending: bool) -> IntTensor; + + /// Computes the cumulative minimum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative minimum of. + /// * `dim` - The dimension along which to compute the cumulative minimum. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is the minimum + /// of all elements up to and including that position along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the cumulative minimum of elements along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("cummin"))] + #[cfg_attr(not(doc), doc = "`Tensor::cummin`")] + /// function, which is more high-level and designed for public use. + fn cummin(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Computes the cumulative maximum of elements along a dimension. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to compute the cumulative maximum of. + /// * `dim` - The dimension along which to compute the cumulative maximum. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is the maximum + /// of all elements up to and including that position along the specified dimension. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For computing the cumulative maximum of elements along a dimension, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("cummax"))] + #[cfg_attr(not(doc), doc = "`Tensor::cummax`")] + /// function, which is more high-level and designed for public use. + fn cummax(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Element-wise greater than comparison between two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors, where each element is true if the + /// corresponding element of the left hand side tensor is greater than the corresponding element + /// of the right hand side tensor, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise greater than comparison between two tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("greater"))] + #[cfg_attr(not(doc), doc = "`Tensor::greater`")] + /// function, which is more high-level and designed for public use. + fn greater(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Element-wise greater than comparison between a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensor, where each element is true if the + /// corresponding element of the left hand side tensor is greater than the right hand side + /// scalar, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise greater than comparison between a tensor and a scalar, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("greater_elem"))] + #[cfg_attr(not(doc), doc = "`Tensor::greater_elem`")] + /// function, which is more high-level and designed for public use. + fn greater_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive; + + /// Element-wise greater than or equal comparison between two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors, where each element is true if the + /// corresponding element of the left hand side tensor is greater than or equal to the + /// corresponding element of the right hand side tensor, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise greater than or equal comparison between two tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("greater_equal"))] + #[cfg_attr(not(doc), doc = "`Tensor::greater_equal`")] + /// function, which is more high-level and designed for public use. + fn greater_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Element-wise greater than or equal comparison between a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensor, where each element is true if the + /// corresponding element of the left hand side tensor is greater than or equal to the right + /// hand side scalar, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise greater than or equal comparison between a tensor and a scalar, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("greater_equal_elem"))] + #[cfg_attr(not(doc), doc = "`Tensor::greater_equal_elem`")] + /// function, which is more high-level and designed for public use. + fn greater_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive; + + /// Element-wise less than comparison between two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors, where each element is true if the + /// corresponding element of the left hand side tensor is less than the corresponding element of + /// the right hand side tensor, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise less than comparison between two tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("lower"))] + #[cfg_attr(not(doc), doc = "`Tensor::lower`")] + /// function, which is more high-level and designed for public use. + fn lower(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Element-wise less than comparison between a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensor, where each element is true if the + /// corresponding element of the left hand side tensor is less than the right hand side scalar, + /// and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise less than comparison between a tensor and a scalar, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("lower_elem"))] + #[cfg_attr(not(doc), doc = "`Tensor::lower_elem`")] + /// function, which is more high-level and designed for public use. + fn lower_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive; + + /// Element-wise less than or equal comparison between two tensors. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side tensor. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors, where each element is true if the + /// corresponding element of the left hand side tensor is less than or equal to the corresponding + /// element of the right hand side tensor, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise less than or equal comparison between two tensors, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("lower_equal"))] + #[cfg_attr(not(doc), doc = "`Tensor::lower_equal`")] + /// function, which is more high-level and designed for public use. + fn lower_equal(lhs: Self::Primitive, rhs: Self::Primitive) -> B::BoolTensorPrimitive; + + /// Element-wise less than or equal comparison between a tensor and a scalar. + /// + /// # Arguments + /// + /// * `lhs` - The left hand side tensor. + /// * `rhs` - The right hand side scalar. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensor, where each element is true if the + /// corresponding element of the left hand side tensor is less than or equal to the right hand + /// side scalar, and false otherwise. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For element-wise less than or equal comparison between a tensor and a scalar, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("lower_equal_elem"))] + #[cfg_attr(not(doc), doc = "`Tensor::lower_equal_elem`")] + /// function, which is more high-level and designed for public use. + fn lower_equal_elem(lhs: Self::Primitive, rhs: Scalar) -> B::BoolTensorPrimitive; + + /// Gets the indices of the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `dim` - The axis along which to get the indices of the maximum elements. + /// * `tensor` - The tensor to get the indices of the maximum elements from. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is the index of the + /// maximum element of the input tensor at the corresponding index along the specified axis. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the indices of the maximum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("argmax"))] + #[cfg_attr(not(doc), doc = "`Tensor::argmax`")] + /// function, which is more high-level and designed for public use. + fn argmax(tensor: Self::Primitive, dim: usize) -> IntTensor; + + /// Gets the indices of the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `dim` - The axis along which to get the indices of the minimum elements. + /// * `tensor` - The tensor to get the indices of the minimum elements from. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor, where each element is the index of the + /// minimum element of the input tensor at the corresponding index along the specified axis. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the indices of the minimum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("argmin"))] + #[cfg_attr(not(doc), doc = "`Tensor::argmin`")] + /// function, which is more high-level and designed for public use. + fn argmin(tensor: Self::Primitive, dim: usize) -> IntTensor; + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `dim` - The axis along which to get the maximum elements. + /// + /// # Returns + /// + /// A single-element tensor containing the maximum element of the input tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the maximum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("max"))] + #[cfg_attr(not(doc), doc = "`Tensor::max`")] + /// function, which is more high-level and designed for public use. + fn max(tensor: Self::Primitive) -> Self::Primitive; + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements from. + /// * `dim` - The axis along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the same rank as the input tensor, but the given dim set to a shape of 1. + /// Each element is the maximum element of the corresponding input dim. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the maximum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("max_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::max_dim`")] + /// function, which is more high-level and designed for public use. + fn max_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements from. + /// * `dim` - The axis along which to get the maximum elements. + /// + /// # Returns + /// + /// A tuple containing the maximum element of the input tensor, and a tensor with the same shape + /// as the input tensor, where each element is the index of the maximum element of the input tensor + /// at the corresponding index along the specified axis. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the maximum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("max_dim_with_indices"))] + #[cfg_attr(not(doc), doc = "`Tensor::max_dim_with_indices`")] + /// function, which is more high-level and designed for public use. + fn max_dim_with_indices(tensor: Self::Primitive, dim: usize) + -> (Self::Primitive, IntTensor); + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `dim` - The axis along which to get the maximum elements. + /// + /// # Returns + /// + /// A single-element tensor containing the maximum absolute element of the input tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the maximum absolute elements of a tensor, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("max_abs"))] + #[cfg_attr(not(doc), doc = "`Tensor::max_abs`")] + /// function, which is more high-level and designed for public use. + fn max_abs(tensor: Self::Primitive) -> Self::Primitive; + + /// Gets the maximum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the maximum elements from. + /// * `dim` - The axis along which to get the maximum elements. + /// + /// # Returns + /// + /// A tensor with the same rank as the input tensor, but the given dim set to a shape of 1. + /// Each element is the maximum absolute element of the corresponding input dim. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the maximum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("max_abs_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::max_abs_dim`")] + /// function, which is more high-level and designed for public use. + fn max_abs_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Gets the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements from. + /// + /// # Returns + /// + /// A single-element tensor containing the minimum element of the input tensor. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the minimum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("min"))] + #[cfg_attr(not(doc), doc = "`Tensor::min`")] + /// function, which is more high-level and designed for public use. + fn min(tensor: Self::Primitive) -> Self::Primitive; + + /// Gets the minimum elements of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements from. + /// * `dim` - The axis along which to get the minimum elements. + /// + /// # Returns + /// + /// A tensor with the same rank as the input tensor, but the given dim set to a shape of 1. + /// Each element is the minimum element of the corresponding input dim. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the minimum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("min_dim"))] + #[cfg_attr(not(doc), doc = "`Tensor::min_dim`")] + /// function, which is more high-level and designed for public use. + fn min_dim(tensor: Self::Primitive, dim: usize) -> Self::Primitive; + + /// Gets the minimum elements and indices of a tensor along an axis. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to get the minimum elements from. + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensor and corresponding indices, where + /// each element is the minimum element of the input tensor at the corresponding index + /// along the specified axis. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users, and not recommended to import + /// or use this function directly. + /// + /// For getting the minimum elements of a tensor along an axis, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("min_dim_with_indices"))] + #[cfg_attr(not(doc), doc = "`Tensor::min_dim_with_indices`")] + /// function, which is more high-level and designed for public use. + fn min_dim_with_indices(tensor: Self::Primitive, dim: usize) + -> (Self::Primitive, IntTensor); + + /// Clamp the tensor between the given min and max values. + /// + /// # Arguments + /// + /// * `min` - The minimum value. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// A new tensor with the values clamped between the given min and max values. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users. + /// + /// For clamping a tensor between the given min and max values, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("clamp"))] + #[cfg_attr(not(doc), doc = "`Tensor::clamp`")] + /// function, which is more high-level and designed for public use. + fn clamp(tensor: Self::Primitive, min: Scalar, max: Scalar) -> Self::Primitive; + + /// Clamps a tensor under a minimum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// + /// # Returns + /// + /// A new tensor with the values clamped under the given min value. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users. + /// + /// For clamping a tensor under a minimum value, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("clamp_min"))] + #[cfg_attr(not(doc), doc = "`Tensor::clamp_min`")] + /// function, which is more high-level and designed for public use. + fn clamp_min(tensor: Self::Primitive, min: Scalar) -> Self::Primitive; + + /// Clamps a tensor over a maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// A new tensor with the values clamped over the given max value. + /// + /// # Remarks + /// + /// This is a low-level function used internally by the library to call different backend functions + /// with static dispatch. It is not designed for direct usage by users. + /// + /// For clamping a tensor over a maximum value, users should prefer the + #[cfg_attr(doc, doc = crate::doc_tensor!("clamp_max"))] + #[cfg_attr(not(doc), doc = "`Tensor::clamp_max`")] + /// function, which is more high-level and designed for public use. + fn clamp_max(tensor: Self::Primitive, max: Scalar) -> Self::Primitive; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/calibration.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/calibration.rs new file mode 100644 index 0000000..e26c483 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/calibration.rs @@ -0,0 +1,5 @@ +/// Calibration method used to compute the quantization range mapping. +pub enum Calibration { + /// Computes quantization range mapping based on the min and max values. + MinMax, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/mod.rs new file mode 100644 index 0000000..bd1860b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/mod.rs @@ -0,0 +1,7 @@ +mod calibration; +mod parameters; +mod scheme; + +pub use calibration::*; +pub use parameters::*; +pub use scheme::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/parameters.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/parameters.rs new file mode 100644 index 0000000..5b50882 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/parameters.rs @@ -0,0 +1,15 @@ +use crate::Backend; + +pub use burn_std::quantization::{QParamTensor, QParams}; + +/// The quantization parameters primitive. +/// +/// # Remarks +/// +/// This is a low-level struct used internally by the library to provide the quantization parameters +/// to the backends. It is not designed for direct usage by users, and not recommended to import +/// or use this struct directly. +pub struct QuantizationParametersPrimitive { + /// The scaling factor. + pub scales: B::FloatTensorPrimitive, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/scheme.rs b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/scheme.rs new file mode 100644 index 0000000..c986798 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-backend/src/tensor/quantization/scheme.rs @@ -0,0 +1,70 @@ +pub use burn_std::{QPARAM_ALIGN, params_shape}; +use burn_std::{QuantLevel, QuantMode, QuantScheme, Shape}; + +use super::{Calibration, QuantizationParametersPrimitive}; +use crate::{Backend, TensorMetadata}; + +/// Compute the quantization range mapping. +pub fn compute_range( + scheme: &QuantScheme, + tensor: B::FloatTensorPrimitive, + calibration: &Calibration, +) -> (B::FloatTensorPrimitive, B::FloatTensorPrimitive) { + match calibration { + Calibration::MinMax => match scheme.level { + QuantLevel::Tensor => (B::float_min(tensor.clone()), B::float_max(tensor)), + QuantLevel::Block(block_size) => { + let block_elems = block_size.num_elements(); + let shape = tensor.shape(); + let numel = shape.num_elements(); + + assert_eq!( + numel % block_elems, + 0, + "Tensor {shape:?} must be evenly divisible by block size {block_elems}" + ); + + let num_blocks = numel / block_elems; + + let params_shape = params_shape(&shape, scheme.level); + + let blocks = B::float_reshape(tensor, Shape::new([num_blocks, block_elems])); + let blocks_min = + B::float_reshape(B::float_min_dim(blocks.clone(), 1), params_shape.clone()); + let blocks_max = B::float_reshape(B::float_max_dim(blocks, 1), params_shape); + (blocks_min, blocks_max) + } + }, + } +} + +/// Compute the quantization parameters. +pub fn compute_q_params( + scheme: &QuantScheme, + min: B::FloatTensorPrimitive, + max: B::FloatTensorPrimitive, +) -> QuantizationParametersPrimitive { + match scheme { + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + .. + } => { + // Quantized range `[a, b]` + let (a, b) = scheme.value.range(); + + // Compute scale to convert an input value in range `[-alpha, alpha]` + let min_abs = B::float_abs(min); + let max_abs = B::float_abs(max); + + // `min_abs.max_pair(max_abs)` + let mask = B::float_lower(min_abs.clone(), max_abs.clone()); + let values_range = + B::float_mul_scalar(B::float_mask_where(min_abs, mask, max_abs), 2f32.into()); + + QuantizationParametersPrimitive { + scales: B::float_div_scalar(values_range, (b - a).into()), + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-candle/Cargo.toml new file mode 100644 index 0000000..3e5aa73 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/Cargo.toml @@ -0,0 +1,44 @@ +[package] +authors = ["louisfd "] +categories = ["science"] +description = "[Deprecated] Candle backend for the Burn framework - use burn-cubecl, burn-ndarray, or burn-tch instead" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-candle" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-candle" +documentation = "https://docs.rs/burn-candle" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std"] +std = [] +doc = ["default"] +tracing = [ + "burn-backend/tracing", + "burn-std/tracing", +] + +cuda = ["candle-core/cuda"] +metal = ["candle-core/metal"] +accelerate = ["candle-core/accelerate"] + +[dependencies] +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } +# For rand utils and stub mutex +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } + +candle-core = { workspace = true } +derive-new = { workspace = true } + +[dev-dependencies] +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", default-features = false, features = [ +] } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/README.md b/crates/stable-diffusion-burn/burn-crates/burn-candle/README.md new file mode 100644 index 0000000..a1102ed --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/README.md @@ -0,0 +1,14 @@ +# Burn Candle Backend + +> **Deprecated:** This crate is deprecated as of `0.21.0-pre.2` and will be removed in a future release. +> Please migrate to one of the actively maintained backends: +> - **CubeCL backends** (CUDA, ROCm, Vulkan, Metal, WebGPU) for GPU acceleration +> - **NdArray** for portable CPU execution +> - **LibTorch** (`burn-tch`) for a mature CPU/GPU backend + +This crate provides a backend for [Burn](https://github.com/tracel-ai/burn) based on the [Candle](https://github.com/huggingface/candle) framework. + +## Feature Flags + +- `cuda` - Cuda GPU device (NVIDIA only) +- `accelerate` - Accelerate framework (macOS only) diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/backend.rs new file mode 100644 index 0000000..57159ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/backend.rs @@ -0,0 +1,300 @@ +use std::marker::PhantomData; + +use burn_backend::{ + BackTrace, Backend, DType, DTypeUsage, DeviceId, DeviceOps, ExecutionError, QTensorPrimitive, + tensor::Device, +}; +use burn_std::{ + rand::{SeedableRng, StdRng}, + stub::Mutex, +}; +use candle_core::{DeviceLocation, backend::BackendDevice}; + +use crate::{ + CandleTensor, IntoDType, + element::{CandleElement, FloatCandleElement, IntCandleElement}, +}; + +/// Tensor backend that uses the [candle](candle_core) crate for executing tensor operations. +/// +/// It is compatible with a wide range of hardware configurations, including CPUs and GPUs +/// that support CUDA or Metal. Additionally, the backend can be compiled to `wasm` when using the CPU. +#[derive(Clone, Default, Debug)] +pub struct Candle +where + F: FloatCandleElement, + I: IntCandleElement, +{ + _float: PhantomData, + _int: PhantomData, +} + +// Seed for CPU device +pub(crate) static SEED: Mutex> = Mutex::new(None); + +pub(crate) fn get_seeded_rng() -> StdRng { + let mut seed = SEED.lock().unwrap(); + seed.take().unwrap_or_else(burn_std::rand::get_seeded_rng) +} + +pub(crate) fn set_seeded_rng(rng_seeded: StdRng) { + let mut seed = SEED.lock().unwrap(); + *seed = Some(rng_seeded); +} + +/// The device type for the candle backend. +#[derive(Clone, Debug, PartialEq, Eq)] +/// The device struct when using the `candle` backend. +/// +/// To create a Cuda or Metal device from the index, use the associated methods to create the variant: +/// ```no_run +/// use burn_candle::CandleDevice; +/// +/// // Create a Cuda device from its index +/// let device = CandleDevice::cuda(0); +/// // Create a Metal device from its index +/// let device = CandleDevice::metal(0); +/// ``` +#[derive(Default)] +pub enum CandleDevice { + /// CPU device. + #[default] + Cpu, + + /// Cuda device with the given index. The index is the index of the Cuda device in the list of + /// all Cuda devices found on the system. + Cuda(CudaDevice), + + /// Metal device with the given index. The index is the index of the Metal device in the list of + /// all Metal devices found on the system. + Metal(MetalDevice), +} + +impl CandleDevice { + /// Create a Cuda device with the given index. + /// The index is the index of the Cuda device in the list of all Cuda devices found on the system. + pub fn cuda(index: usize) -> Self { + CandleDevice::Cuda(CudaDevice { + device: candle_core::CudaDevice::new(index).unwrap(), + index, + }) + } + + /// Create a Metal device with the given index. + /// The index is the index of the Metal device in the list of all Metal devices found on the system. + pub fn metal(index: usize) -> Self { + CandleDevice::Metal(MetalDevice { + device: candle_core::MetalDevice::new(index).unwrap(), + index, + }) + } + + pub(crate) fn set_seed(&self, seed: u64) { + match self { + CandleDevice::Cpu => { + // candle_core::cpu_backend::CpuDevice.set_seed(seed).unwrap(); + // Candle does not support seeding the CPU rng so we use a global seed + let rng = StdRng::seed_from_u64(seed); + set_seeded_rng(rng); + } + CandleDevice::Cuda(cuda_device) => cuda_device.device.set_seed(seed).unwrap(), + CandleDevice::Metal(metal_device) => metal_device.device.set_seed(seed).unwrap(), + } + } +} + +#[derive(Clone, Debug)] +/// A Cuda device for the `candle` backend. +pub struct CudaDevice { + pub(crate) device: candle_core::CudaDevice, + /// The index of the Cuda device in the list of all devices on the system. + pub index: usize, +} + +impl PartialEq for CudaDevice { + fn eq(&self, other: &Self) -> bool { + self.device.same_device(&other.device) && self.index == other.index + } +} + +impl Eq for CudaDevice {} + +#[derive(Clone, Debug)] +/// A Metal device for the `candle` backend. +pub struct MetalDevice { + pub(crate) device: candle_core::MetalDevice, + /// The index of the Metal device in the list of all devices on the system. + pub index: usize, +} + +impl PartialEq for MetalDevice { + fn eq(&self, other: &Self) -> bool { + self.device.same_device(&other.device) && self.index == other.index + } +} + +impl Eq for MetalDevice {} + +impl From for candle_core::Device { + fn from(device: CandleDevice) -> Self { + match device { + CandleDevice::Cpu => candle_core::Device::Cpu, + CandleDevice::Cuda(device) => candle_core::Device::Cuda(device.device), + CandleDevice::Metal(device) => candle_core::Device::Metal(device.device), + } + } +} + +impl From for CandleDevice { + fn from(device: candle_core::Device) -> Self { + match device.location() { + DeviceLocation::Cpu => CandleDevice::Cpu, + DeviceLocation::Cuda { gpu_id } => { + if let candle_core::Device::Cuda(device) = device { + CandleDevice::Cuda(CudaDevice { + device, + index: gpu_id, + }) + } else { + panic!("Expected CUDA device."); + } + } + DeviceLocation::Metal { gpu_id } => { + if let candle_core::Device::Metal(device) = device { + CandleDevice::Metal(MetalDevice { + device, + index: gpu_id, + }) + } else { + panic!("Expected Metal device."); + } + } + } + } +} + +impl burn_backend::Device for CandleDevice { + fn to_id(&self) -> burn_backend::DeviceId { + match self { + CandleDevice::Cuda(device) => DeviceId::new(0, device.index as u32), + CandleDevice::Metal(device) => DeviceId::new(1, device.index as u32), + CandleDevice::Cpu => DeviceId::new(2, 0), + } + } + + fn from_id(device_id: DeviceId) -> Self { + match device_id.type_id { + 0 => CandleDevice::cuda(device_id.index_id as usize), + 1 => CandleDevice::metal(device_id.index_id as usize), + _ => CandleDevice::Cpu, + } + } + + fn device_count(type_id: u16) -> usize { + // TODO: Fix that + 1 + } +} +impl DeviceOps for CandleDevice {} + +impl Backend for Candle { + type Device = CandleDevice; + + type FloatTensorPrimitive = CandleTensor; + type FloatElem = F; + + type IntTensorPrimitive = CandleTensor; + type IntElem = I; + + type BoolTensorPrimitive = CandleTensor; + type BoolElem = u8; + + type QuantizedTensorPrimitive = CandleTensor; + + fn ad_enabled(_device: &Self::Device) -> bool { + false + } + + fn name(device: &Self::Device) -> String { + match device { + CandleDevice::Cpu => "candle", + CandleDevice::Cuda(..) => "candle", + CandleDevice::Metal(..) => "candle", + } + .to_string() + } + + fn seed(device: &CandleDevice, seed: u64) { + device.set_seed(seed); + } + + fn sync(device: &Device) -> Result<(), ExecutionError> { + let device: candle_core::Device = (device.clone()).into(); + + match device { + candle_core::Device::Cpu => (), + candle_core::Device::Cuda(device) => { + #[cfg(feature = "cuda")] + device + .synchronize() + .map_err(|err| ExecutionError::Generic { + reason: format!("Can't sync the cuda device: {err}"), + backtrace: BackTrace::capture(), + })?; + } + candle_core::Device::Metal(device) => { + // For some reason, device.wait_until_completed() does not seem to work, + // and neither does writing and reading a value with into_data + return Err(ExecutionError::Generic { + reason: + "Device synchronization unavailable with Metal device on Candle backend" + .into(), + backtrace: BackTrace::capture(), + }); + } + } + + Ok(()) + } + + fn dtype_usage(device: &Self::Device, dtype: DType) -> burn_backend::DTypeUsageSet { + if dtype.try_into_dtype().is_ok() { + burn_backend::DTypeUsage::general() + } else { + burn_backend::DTypeUsageSet::empty() + } + } +} + +#[cfg(test)] +mod tests { + use burn_std::QuantScheme; + + use super::*; + + #[test] + fn should_support_dtypes() { + type B = Candle; + let device = Default::default(); + + assert!(B::supports_dtype(&device, DType::F64)); + assert!(B::supports_dtype(&device, DType::F32)); + assert!(B::supports_dtype(&device, DType::Flex32)); + assert!(B::supports_dtype(&device, DType::F16)); + assert!(B::supports_dtype(&device, DType::BF16)); + assert!(B::supports_dtype(&device, DType::I64)); + assert!(B::supports_dtype(&device, DType::U32)); + assert!(B::supports_dtype(&device, DType::U8)); + assert!(B::supports_dtype(&device, DType::I32)); + assert!(B::supports_dtype(&device, DType::I16)); + + assert!(!B::supports_dtype(&device, DType::U64)); + assert!(!B::supports_dtype(&device, DType::U16)); + assert!(!B::supports_dtype(&device, DType::I8)); + assert!(!B::supports_dtype(&device, DType::Bool)); + assert!(!B::supports_dtype( + &device, + DType::QFloat(QuantScheme::default()) + )); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/element.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/element.rs new file mode 100644 index 0000000..bc78c13 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/element.rs @@ -0,0 +1,32 @@ +use std::borrow::Borrow; + +use burn_backend::{Element, bf16, f16}; +use candle_core::{FloatDType, Tensor, WithDType}; + +/// Candle element +pub trait CandleElement: Element + WithDType {} +/// Candle float element +pub trait FloatCandleElement: CandleElement + FloatDType {} +/// Candle int element +pub trait IntCandleElement: CandleElement {} + +impl CandleElement for f64 {} +impl FloatCandleElement for f64 {} + +impl CandleElement for f32 {} +impl FloatCandleElement for f32 {} + +impl CandleElement for f16 {} +impl FloatCandleElement for f16 {} + +impl CandleElement for bf16 {} +impl FloatCandleElement for bf16 {} + +impl CandleElement for u8 {} +impl IntCandleElement for u8 {} + +impl CandleElement for u32 {} +impl IntCandleElement for u32 {} + +impl CandleElement for i64 {} +impl IntCandleElement for i64 {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/lib.rs new file mode 100644 index 0000000..5f20447 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/lib.rs @@ -0,0 +1,27 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![allow(unused)] // TODO remove when backend filled +#![deprecated( + since = "0.21.0-pre.2", + note = "burn-candle is deprecated and will be removed in a future release. Use burn-cubecl (CUDA/ROCm/Vulkan/Metal/WebGPU), burn-ndarray, or burn-tch instead." +)] + +//! Burn Candle Backend +//! +//! **Deprecated:** This backend is deprecated and will be removed in a future release. +//! Please migrate to one of the actively maintained backends: +//! - CubeCL backends (CUDA, ROCm, Vulkan, Metal, WebGPU) for GPU acceleration +//! - NdArray for portable CPU execution +//! - LibTorch (`burn-tch`) for a mature CPU/GPU backend + +#[macro_use] +extern crate derive_new; + +mod backend; +mod element; +mod ops; +mod tensor; + +pub use backend::*; +pub use element::*; +pub use tensor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/activation.rs new file mode 100644 index 0000000..80be26c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/activation.rs @@ -0,0 +1,17 @@ +use burn_backend::{ops::ActivationOps, tensor::FloatTensor}; + +use crate::{ + Candle, CandleTensor, + element::{CandleElement, FloatCandleElement, IntCandleElement}, + tensor, +}; + +impl ActivationOps for Candle { + fn gelu(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.gelu().unwrap()) + } + + fn relu(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.relu().unwrap()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/base.rs new file mode 100644 index 0000000..ba2ea6c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/base.rs @@ -0,0 +1,572 @@ +use std::cmp::max; +use std::marker::PhantomData; + +use crate::{ + Candle, CandleDevice, CandleTensor, + element::{CandleElement, FloatCandleElement, IntCandleElement}, +}; +use burn_backend::{ + BackTrace, Backend, Distribution, ExecutionError, Slice, bf16, f16, + ops::unfold::{calculate_unfold_shape, calculate_unfold_windows}, +}; +use burn_backend::{Element, Shape, TensorData, TensorMetadata}; +use candle_core::{Layout, WithDType}; + +use super::tensor; + +pub fn cpu_random(shape: Shape, distribution: Distribution) -> TensorData { + let mut rng = crate::get_seeded_rng(); + let data = TensorData::random::(shape, distribution, &mut rng); + crate::set_seeded_rng(rng); + data +} + +pub fn cat(tensors: Vec, dim: usize) -> CandleTensor { + let tensors: Vec = tensors.into_iter().map(|t| t.tensor).collect(); + CandleTensor::new(candle_core::Tensor::cat(&tensors, dim).unwrap()) +} + +pub fn from_data(data: TensorData, device: &CandleDevice) -> CandleTensor { + CandleTensor::from_data::(data, device.clone()) +} +pub fn into_data(tensor: CandleTensor) -> Result { + fn tensor_data_from_dtype( + tensor: &CandleTensor, + ) -> Result { + let data = tensor + .tensor + .flatten_all() + .map_err(|err| ExecutionError::Generic { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + })? + .to_vec1::() + .map_err(|err| ExecutionError::Generic { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + })?; + Ok(TensorData::new(data, tensor.shape())) + } + + match tensor.tensor.dtype() { + candle_core::DType::BF16 => tensor_data_from_dtype::(&tensor), + candle_core::DType::F16 => tensor_data_from_dtype::(&tensor), + candle_core::DType::F32 => tensor_data_from_dtype::(&tensor), + candle_core::DType::F64 => tensor_data_from_dtype::(&tensor), + candle_core::DType::U8 => tensor_data_from_dtype::(&tensor), + candle_core::DType::U32 => tensor_data_from_dtype::(&tensor), + candle_core::DType::I16 => tensor_data_from_dtype::(&tensor), + candle_core::DType::I32 => tensor_data_from_dtype::(&tensor), + candle_core::DType::I64 => tensor_data_from_dtype::(&tensor), + other => todo!("{other:?} not yet supported"), + } +} + +pub fn to_device(tensor: CandleTensor, device: &CandleDevice) -> CandleTensor { + CandleTensor::new(tensor.tensor.to_device(&(device.clone()).into()).unwrap()) +} + +pub fn empty(shape: Shape, device: &CandleDevice, dtype: candle_core::DType) -> CandleTensor { + zeros(shape, device, dtype) +} + +pub fn zeros(shape: Shape, device: &CandleDevice, dtype: candle_core::DType) -> CandleTensor { + CandleTensor::new( + candle_core::Tensor::zeros(shape.to_vec(), dtype, &(device.clone()).into()).unwrap(), + ) +} + +pub fn ones(shape: Shape, device: &CandleDevice, dtype: candle_core::DType) -> CandleTensor { + CandleTensor::new( + candle_core::Tensor::ones(shape.to_vec(), dtype, &(device.clone()).into()).unwrap(), + ) +} + +pub fn swap_dims(mut tensor: CandleTensor, dim1: usize, dim2: usize) -> CandleTensor { + CandleTensor::new(tensor.tensor.transpose(dim1, dim2).unwrap()) +} + +pub fn permute(tensor: CandleTensor, axes: &[usize]) -> CandleTensor { + CandleTensor::new(tensor.tensor.permute(axes).unwrap()) +} + +pub fn flip(tensor: CandleTensor, axes: &[usize]) -> CandleTensor { + // FIXME: Replace with an appropriate method when Candle provides one. + let mut tensor = tensor.tensor; + for &axis in axes { + // Ensure tensor is contiguous before index_select (required by Candle) + tensor = tensor.contiguous().unwrap(); + + let indexes = candle_core::Tensor::arange_step( + tensor.dim(axis).unwrap() as i64 - 1, + -1, + -1, + tensor.device(), + ) + .unwrap(); + tensor = tensor.index_select(&indexes, axis).unwrap(); + } + + CandleTensor::new(tensor) +} + +pub fn reshape(tensor: CandleTensor, shape: Shape) -> CandleTensor { + CandleTensor::new(tensor.tensor.reshape(shape.to_vec()).unwrap()) +} + +pub fn device(tensor: &CandleTensor) -> CandleDevice { + tensor.tensor.device().clone().into() +} + +pub fn shape(tensor: &CandleTensor) -> Shape { + tensor.shape() +} + +pub fn slice(tensor: CandleTensor, ranges: &[std::ops::Range]) -> CandleTensor { + let mut narrow_tensor = tensor.tensor; + for (i, range) in ranges.iter().enumerate().take(ranges.len()) { + narrow_tensor = narrow_tensor + .narrow(i, range.start, range.end - range.start) + .unwrap() + } + CandleTensor::new(narrow_tensor) +} + +pub fn slice_with_steps(tensor: CandleTensor, slices: &[Slice]) -> CandleTensor { + let mut result_tensor = tensor.tensor; + + for (dim, slice) in slices.iter().enumerate() { + if slice.step == 1 { + // Use narrow for step=1 (more efficient) + // Convert slice to range using tensor shape + let dim_size = result_tensor.dim(dim).unwrap(); + let range = slice.to_range(dim_size); + let start = range.start; + let length = range.end - range.start; + result_tensor = result_tensor.narrow(dim, start, length).unwrap(); + } else { + // Use index_select for step != 1 + let dim_size = result_tensor.dim(dim).unwrap(); + let range = slice.to_range(dim_size); + let start = range.start; + let end = range.end; + let step = slice.step; + + // Generate indices based on step direction + let indices_vec = if step > 0 { + // Forward stepping + let step_usize = step as usize; + (start..end).step_by(step_usize).collect::>() + } else { + // Backward stepping (negative step) + let step_usize = step.unsigned_abs(); + // Start from end-1 and go backwards + let mut indices = Vec::new(); + let mut idx = end - 1; + while idx >= start && idx < end { + // Check for underflow + indices.push(idx); + if idx >= step_usize { + idx -= step_usize; + } else { + break; + } + } + indices + }; + + // Convert indices to tensor and use index_select + let indices_len = indices_vec.len(); + let device = result_tensor.device(); + let indices = candle_core::Tensor::from_vec( + indices_vec.iter().map(|&x| x as u32).collect::>(), + indices_len, + device, + ) + .unwrap(); + + result_tensor = result_tensor.index_select(&indices, dim).unwrap(); + } + } + + CandleTensor::new(result_tensor) +} + +pub fn slice_assign(tensor: CandleTensor, slices: &[Slice], value: CandleTensor) -> CandleTensor { + // Check if all slices have step=1 (candle's native slice_assign requirement) + let all_unit_steps = slices.iter().all(|s| s.step == 1); + + if all_unit_steps { + // Convert Slice to Range for candle's native slice_assign + let ranges: Vec> = slices + .iter() + .enumerate() + .map(|(dim, slice)| { + let dim_size = tensor.tensor.dim(dim).unwrap_or(usize::MAX); + slice.to_range(dim_size) + }) + .collect(); + + CandleTensor::new(tensor.tensor.slice_assign(&ranges, &value.tensor).unwrap()) + } else { + // Implement slice_assign with steps using scatter operations + slice_assign_with_steps_workaround(tensor, slices, value) + } +} + +/// Implements slice_assign for non-unit steps using index operations +fn slice_assign_with_steps_workaround( + tensor: CandleTensor, + slices: &[Slice], + value: CandleTensor, +) -> CandleTensor { + let shape = tensor.shape(); + let ndims = shape.num_dims(); + let device = tensor.tensor.device(); + + // Generate indices for each dimension based on slice specifications + let indices_per_dim = generate_slice_indices(slices, &shape); + + // Early return if no elements to assign + let total_elements: usize = indices_per_dim.iter().map(|v| v.len()).product(); + if total_elements == 0 { + return tensor; + } + + // Flatten tensors and get metadata + let value_flat = value.tensor.flatten_all().unwrap(); + let strides = tensor.tensor.stride(); + let tensor_shape = tensor.tensor.dims(); + + // Use a macro to handle different dtypes without code duplication + macro_rules! apply_slice_assign { + ($dtype:ty, $to_vec_fn:ident) => {{ + let mut tensor_vec: Vec<$dtype> = + tensor.tensor.flatten_all().unwrap().$to_vec_fn().unwrap(); + let value_vec: Vec<$dtype> = value_flat.$to_vec_fn().unwrap(); + + // Apply assignments using cartesian product of indices + for (value_idx, &value) in value_vec.iter().enumerate() { + let flat_idx = compute_flat_index(value_idx, &indices_per_dim, &strides); + if flat_idx < tensor_vec.len() { + tensor_vec[flat_idx] = value; + } + } + + candle_core::Tensor::from_vec(tensor_vec, tensor_shape, device).unwrap() + }}; + } + + use candle_core::DType; + let result = match tensor.tensor.dtype() { + DType::F32 => apply_slice_assign!(f32, to_vec1), + DType::F64 => apply_slice_assign!(f64, to_vec1), + DType::I64 => apply_slice_assign!(i64, to_vec1), + DType::U32 => apply_slice_assign!(u32, to_vec1), + DType::U8 => apply_slice_assign!(u8, to_vec1), + _ => panic!( + "Unsupported dtype {:?} for slice_assign with steps", + tensor.tensor.dtype() + ), + }; + + CandleTensor::new(result) +} + +/// Generate indices for each dimension based on slice specifications +fn generate_slice_indices(slices: &[Slice], tensor_dims: &[usize]) -> Vec> { + let ndims = tensor_dims.len(); + let mut indices_per_dim = Vec::with_capacity(ndims); + + // Process provided slices + for (dim_idx, slice) in slices.iter().enumerate() { + let dim_size = tensor_dims[dim_idx]; + let range = slice.to_range(dim_size); + let indices = generate_stepped_indices(range.start, range.end, slice.step); + indices_per_dim.push(indices); + } + + // Fill remaining dimensions with full ranges + for &dim_size in tensor_dims.iter().skip(slices.len()) { + indices_per_dim.push((0..dim_size).collect()); + } + + indices_per_dim +} + +/// Generate indices for a single dimension with stepping +fn generate_stepped_indices(start: usize, end: usize, step: isize) -> Vec { + if step > 0 { + // Forward stepping + (start..end).step_by(step as usize).collect() + } else if step < 0 { + // Backward stepping: start from end-1 and go backwards + let step_size = step.unsigned_abs(); + let mut indices = Vec::new(); + let mut idx = end.saturating_sub(1); + + while idx >= start && idx < end { + indices.push(idx); + if idx >= step_size { + idx -= step_size; + } else { + break; + } + } + indices + } else { + // This branch should never be reached since step is validated to be non-zero + panic!("Step cannot be zero") + } +} + +/// Compute flat index from multi-dimensional indices using cartesian product logic +fn compute_flat_index( + value_idx: usize, + indices_per_dim: &[Vec], + strides: &[usize], +) -> usize { + let mut flat_idx = 0; + let mut remainder = value_idx; + + // Convert value_idx to multi-dimensional indices and compute flat tensor index + for dim in (0..indices_per_dim.len()).rev() { + let dim_size = indices_per_dim[dim].len(); + let idx_in_dim = remainder % dim_size; + remainder /= dim_size; + + let actual_idx = indices_per_dim[dim][idx_in_dim]; + flat_idx += actual_idx * strides[dim]; + } + + flat_idx +} + +pub fn narrow(tensor: CandleTensor, dim: usize, start: usize, length: usize) -> CandleTensor { + let tensor = tensor.tensor.narrow(dim, start, length); + match tensor { + Ok(tensor) => CandleTensor::new(tensor), + Err(e) => panic!("error narrow from Candle"), + } +} + +pub fn chunk(tensor: CandleTensor, chunks: usize, dim: usize) -> Vec { + let tensors = tensor.tensor.chunk(chunks, dim); + match tensors { + Ok(tensors) => tensors.into_iter().map(CandleTensor::new).collect(), + Err(e) => panic!("error chunk from Candle"), + } +} + +pub fn expand(tensor: CandleTensor, shape: Shape) -> CandleTensor { + CandleTensor::new(tensor.tensor.broadcast_as(shape.to_vec()).unwrap()) +} + +pub fn unfold(tensor: CandleTensor, dim: usize, size: usize, step: usize) -> CandleTensor { + let result_shape = calculate_unfold_shape(tensor.shape(), dim, size, step); + let windows = result_shape[dim]; + + let mut select_ranges = tensor.shape().into_ranges(); + let new_axis = select_ranges.len(); + + let mut stack = Vec::with_capacity(windows); + for widx in 0..windows { + let start = widx * step; + let end = start + size; + select_ranges[dim] = start..end; + + let mut window_slice = slice(tensor.clone(), &select_ranges); + + window_slice = swap_dims(window_slice, dim, new_axis); + let window_slice = CandleTensor::new(window_slice.tensor.unsqueeze(new_axis).unwrap()); + + stack.push(window_slice); + } + cat(stack, dim) +} + +pub fn sign(tensor: CandleTensor) -> CandleTensor { + CandleTensor::new(tensor.tensor.sign().unwrap()) +} + +pub fn mask_where_broadcasted( + tensor: CandleTensor, + mask: CandleTensor, + value: CandleTensor, +) -> CandleTensor { + let shape = tensor + .tensor + .shape() + .broadcast_shape_binary_op(mask.tensor.shape(), "where_cond") + .unwrap(); + + let mut tensor = tensor.tensor; + let mut mask = mask.tensor; + let mut value = value.tensor; + + if shape != *tensor.shape() { + tensor = tensor.broadcast_as(shape.clone()).unwrap(); + } + if shape != *mask.shape() { + mask = mask.broadcast_as(shape.clone()).unwrap(); + } + if shape != *value.shape() { + value = value.broadcast_as(shape).unwrap(); + } + + CandleTensor::new(mask.where_cond(&value, &tensor).unwrap()) +} + +pub fn cross(lhs: CandleTensor, rhs: CandleTensor, dim: usize) -> CandleTensor { + let shape_lhs = lhs.shape(); + let shape_rhs = rhs.shape(); + let ndims = shape_lhs.num_dims(); + + // Broadcast the shapes except along dim + let mut broadcast_shape = vec![0; ndims]; + for (i, item) in broadcast_shape.iter_mut().enumerate().take(ndims) { + if i == dim { + *item = shape_lhs[i]; + } else { + let l = shape_lhs[i]; + let r = shape_rhs[i]; + if l == r { + *item = l; + } else if l == 1 { + *item = r; + } else if r == 1 { + *item = l; + } else { + panic!("Tensors are not broadcastable along dimension {}", i); + } + } + } + + // Broadcast lhs and rhs + let lhs_broadcast = if shape_lhs == Shape::from(broadcast_shape.clone()) { + lhs + } else { + expand(lhs, Shape::from(broadcast_shape.clone())) + }; + let rhs_broadcast = if shape_rhs == Shape::from(broadcast_shape.clone()) { + rhs + } else { + expand(rhs, Shape::from(broadcast_shape.clone())) + }; + + // Now, move dim to the last dimension + let mut perm = (0..ndims).collect::>(); + perm.remove(dim); + perm.push(dim); + + let lhs_permuted = permute(lhs_broadcast, &perm); + let rhs_permuted = permute(rhs_broadcast, &perm); + + // Reshape to (*, 3) + let total_elements = lhs_permuted.shape().num_elements(); + let batch_size = total_elements / 3; + let lhs_reshaped = reshape(lhs_permuted, Shape::new([batch_size, 3])); + let rhs_reshaped = reshape(rhs_permuted, Shape::new([batch_size, 3])); + + // Extract components using narrow and squeeze + let lhs_0 = CandleTensor::new( + lhs_reshaped + .tensor + .narrow(1, 0, 1) + .unwrap() + .squeeze(1) + .unwrap(), + ); + let lhs_1 = CandleTensor::new( + lhs_reshaped + .tensor + .narrow(1, 1, 1) + .unwrap() + .squeeze(1) + .unwrap(), + ); + let lhs_2 = CandleTensor::new( + lhs_reshaped + .tensor + .narrow(1, 2, 1) + .unwrap() + .squeeze(1) + .unwrap(), + ); + let rhs_0 = CandleTensor::new( + rhs_reshaped + .tensor + .narrow(1, 0, 1) + .unwrap() + .squeeze(1) + .unwrap(), + ); + let rhs_1 = CandleTensor::new( + rhs_reshaped + .tensor + .narrow(1, 1, 1) + .unwrap() + .squeeze(1) + .unwrap(), + ); + let rhs_2 = CandleTensor::new( + rhs_reshaped + .tensor + .narrow(1, 2, 1) + .unwrap() + .squeeze(1) + .unwrap(), + ); + + // Compute cross product components + let result_0 = CandleTensor::new( + lhs_1 + .tensor + .mul(&rhs_2.tensor) + .unwrap() + .sub(&lhs_2.tensor.mul(&rhs_1.tensor).unwrap()) + .unwrap(), + ); + let result_1 = CandleTensor::new( + lhs_2 + .tensor + .mul(&rhs_0.tensor) + .unwrap() + .sub(&lhs_0.tensor.mul(&rhs_2.tensor).unwrap()) + .unwrap(), + ); + let result_2 = CandleTensor::new( + lhs_0 + .tensor + .mul(&rhs_1.tensor) + .unwrap() + .sub(&lhs_1.tensor.mul(&rhs_0.tensor).unwrap()) + .unwrap(), + ); + + // Stack the components + let result_0_unsqueezed = CandleTensor::new(result_0.tensor.unsqueeze(1).unwrap()); + let result_1_unsqueezed = CandleTensor::new(result_1.tensor.unsqueeze(1).unwrap()); + let result_2_unsqueezed = CandleTensor::new(result_2.tensor.unsqueeze(1).unwrap()); + let result = cat( + vec![ + result_0_unsqueezed, + result_1_unsqueezed, + result_2_unsqueezed, + ], + 1, + ); + + // Reshape back to the broadcast shape with dim at the end + let mut result_shape = broadcast_shape; + result_shape.remove(dim); + result_shape.push(3); + let result_reshaped = reshape(result, Shape::from(result_shape)); + + // Permute back + let mut inv_perm = vec![0; ndims]; + for (i, &p) in perm.iter().enumerate() { + inv_perm[p] = i; + } + permute(result_reshaped, &inv_perm) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/bool_tensor.rs new file mode 100644 index 0000000..3432570 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/bool_tensor.rs @@ -0,0 +1,212 @@ +use burn_backend::{ + BackTrace, DType, ExecutionError, Scalar, Shape, Slice, TensorData, TensorMetadata, + ops::BoolTensorOps, + tensor::{BoolTensor, Device, FloatTensor, IntTensor}, +}; + +use crate::{ + Candle, CandleTensor, + element::{CandleElement, FloatCandleElement, IntCandleElement}, +}; + +use super::base::{expand, permute, unfold}; + +impl BoolTensorOps for Candle { + fn bool_empty(shape: Shape, device: &Device) -> BoolTensor { + super::base::empty(shape, device, candle_core::DType::U8) + } + + fn bool_zeros(shape: Shape, device: &Device) -> BoolTensor { + super::base::zeros(shape, device, candle_core::DType::U8) + } + + fn bool_ones(shape: Shape, device: &Device) -> BoolTensor { + super::base::ones(shape, device, candle_core::DType::U8) + } + + async fn bool_into_data(tensor: BoolTensor) -> Result { + let x: Vec = tensor + .tensor + .flatten_all() + .map_err(|err| ExecutionError::Generic { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + })? + .to_vec1() + .map_err(|err| ExecutionError::Generic { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + })?; + + let y = x.iter().map(|b| !matches!(b, 0)).collect(); + + Ok(TensorData::new(y, tensor.shape())) + } + + fn bool_from_data(data: TensorData, device: &Device) -> BoolTensor { + match data.dtype { + DType::U8 => super::base::from_data::(data, device), + _ => unimplemented!("Unsupported dtype for `bool_from_data`"), + } + } + + fn bool_into_int(tensor: BoolTensor) -> IntTensor { + CandleTensor::new(tensor.tensor.to_dtype(I::DTYPE).unwrap()) + } + + fn bool_into_float(tensor: BoolTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.to_dtype(F::DTYPE).unwrap()) + } + + fn bool_device(tensor: &BoolTensor) -> Device { + super::base::device(tensor) + } + + fn bool_to_device(tensor: BoolTensor, device: &Device) -> BoolTensor { + super::base::to_device(tensor, device) + } + + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor { + super::base::reshape(tensor, shape) + } + + fn bool_slice(tensor: BoolTensor, slices: &[Slice]) -> BoolTensor { + super::base::slice_with_steps(tensor, slices) + } + + fn bool_slice_assign( + tensor: BoolTensor, + slices: &[Slice], + value: BoolTensor, + ) -> BoolTensor { + super::base::slice_assign(tensor, slices, value) + } + + fn bool_cat(tensors: Vec>, dim: usize) -> BoolTensor { + super::base::cat(tensors, dim) + } + + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.eq(&rhs_broadcast).unwrap()) + } + + fn bool_not(tensor: BoolTensor) -> BoolTensor { + let x = (candle_core::Tensor::zeros_like(&tensor.tensor).unwrap()); + CandleTensor::new(tensor.tensor.eq(&x).unwrap()) + } + + fn bool_and(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + let x = candle_core::Tensor::ones_like(&lhs.tensor).unwrap(); + CandleTensor::new(lhs.tensor.add(&rhs.tensor).unwrap().gt(&x).unwrap()) + } + + fn bool_or(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .add(&rhs.tensor) + .unwrap() + .clamp(0u32, 1u32) + .unwrap(), + ) + } + + fn bool_swap_dims(tensor: BoolTensor, dim1: usize, dim2: usize) -> BoolTensor { + super::base::swap_dims(tensor, dim1, dim2) + } + + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + super::base::permute(tensor, axes) + } + + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + super::base::flip(tensor, axes) + } + + fn bool_select( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + ) -> BoolTensor { + CandleTensor::new(tensor.tensor.index_select(&indices.tensor, dim).unwrap()) + } + + fn bool_select_or( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + CandleTensor::new( + tensor + .tensor + .index_add(&indices.tensor, &value.tensor, dim) + .unwrap(), + ) + } + + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor { + expand(tensor, shape) + } + + fn bool_unfold( + tensor: BoolTensor, + dim: usize, + size: usize, + step: usize, + ) -> BoolTensor { + unfold(tensor, dim, size, step) + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + super::base::mask_where_broadcasted(tensor, mask, value) + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + CandleTensor::new( + mask.tensor + .where_cond( + &super::candle_utils::fill_like::(value.elem(), &tensor.tensor), + &tensor.tensor, + ) + .unwrap(), + ) + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + let tensor = tensor.tensor.contiguous().unwrap(); + let indices = indices.tensor.contiguous().unwrap(); + CandleTensor::new(tensor.gather(&indices, dim).unwrap()) + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + CandleTensor::new( + tensor + .tensor + .scatter_add(&indices.tensor, &value.tensor, dim) + .unwrap(), + ) + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new(lhs.tensor.eq(rhs.elem::()).unwrap()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/candle_utils.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/candle_utils.rs new file mode 100644 index 0000000..957da6e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/candle_utils.rs @@ -0,0 +1,46 @@ +use candle_core::{DType, Device, Shape, Tensor}; + +use crate::element::CandleElement; + +pub(crate) fn fill>( + value: E, + shape: S, + dtype: DType, + device: &Device, +) -> Tensor { + let values = (Tensor::ones((1), dtype, device).unwrap() * value.elem::()).unwrap(); + values.expand(shape).unwrap() +} + +pub(crate) fn fill_like(value: E, reference_tensor: &Tensor) -> Tensor { + fill( + value, + reference_tensor.shape(), + reference_tensor.dtype(), + reference_tensor.device(), + ) +} + +/// Broadcasts two tensors to a common shape for comparison operations +pub(crate) fn broadcast_for_comparison( + lhs: &Tensor, + rhs: &Tensor, +) -> Result<(Tensor, Tensor), candle_core::Error> { + let broadcast_shape = lhs + .shape() + .broadcast_shape_binary_op(rhs.shape(), "comparison")?; + + let lhs = if broadcast_shape != *lhs.shape() { + lhs.broadcast_as(&broadcast_shape)? + } else { + lhs.clone() + }; + + let rhs = if broadcast_shape != *rhs.shape() { + rhs.broadcast_as(&broadcast_shape)? + } else { + rhs.clone() + }; + + Ok((lhs, rhs)) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/int_tensor.rs new file mode 100644 index 0000000..848e2f4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/int_tensor.rs @@ -0,0 +1,521 @@ +use burn_backend::{ + DType, Distribution, ElementConversion, ExecutionError, IntDType, Scalar, Shape, Slice, + TensorData, + ops::{FloatTensorOps, IntTensorOps}, + tensor::{Bool, BoolTensor, Device, FloatTensor, IntElem, IntTensor}, +}; + +use crate::{ + Candle, CandleDevice, CandleTensor, IntoDType, + element::{CandleElement, FloatCandleElement, IntCandleElement}, +}; + +use super::base::{cpu_random, expand, permute, sign, unfold}; + +impl IntTensorOps for Candle { + fn int_empty(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + super::base::empty(shape, device, dtype.into_dtype()) + } + + async fn int_into_data(tensor: IntTensor) -> Result { + super::base::into_data(tensor) + } + + fn int_from_data(data: TensorData, device: &Device) -> IntTensor { + match data.dtype { + DType::I64 => super::base::from_data::(data, device), + DType::U32 => super::base::from_data::(data, device), + DType::U8 => super::base::from_data::(data, device), + _ => unimplemented!("Unsupported dtype for `int_from_data`"), + } + } + + fn int_device(tensor: &IntTensor) -> Device { + super::base::device(tensor) + } + + fn int_to_device(tensor: IntTensor, device: &Device) -> IntTensor { + super::base::to_device(tensor, device) + } + + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor { + super::base::reshape(tensor, shape) + } + + fn int_slice(tensor: IntTensor, slices: &[Slice]) -> IntTensor { + super::base::slice_with_steps(tensor, slices) + } + + fn int_slice_assign( + tensor: IntTensor, + slices: &[Slice], + value: IntTensor, + ) -> IntTensor { + super::base::slice_assign(tensor, slices, value) + } + + fn int_into_float(tensor: IntTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.to_dtype(F::DTYPE).unwrap()) + } + + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + source: IntTensor, + ) -> IntTensor { + super::base::mask_where_broadcasted(tensor, mask, source) + } + + fn int_mask_fill( + tensor: IntTensor, + mask: BoolTensor, + value: Scalar, + ) -> IntTensor { + CandleTensor::new( + mask.tensor + .where_cond( + &super::candle_utils::fill_like::(value.elem(), &tensor.tensor), + &tensor.tensor, + ) + .unwrap(), + ) + } + + fn int_gather( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + ) -> IntTensor { + let tensor = tensor.tensor.contiguous().unwrap(); + let indices = indices.tensor.contiguous().unwrap(); + CandleTensor::new(tensor.gather(&indices, dim).unwrap()) + } + + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + CandleTensor::new( + tensor + .tensor + .scatter_add(&indices.tensor, &value.tensor, dim) + .unwrap(), + ) + } + + fn int_select( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + ) -> IntTensor { + CandleTensor::new(tensor.tensor.index_select(&indices.tensor, dim).unwrap()) + } + + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + CandleTensor::new( + tensor + .tensor + .index_add(&indices.tensor, &value.tensor, dim) + .unwrap(), + ) + } + + fn int_cat(tensors: Vec>, dim: usize) -> IntTensor { + super::base::cat(tensors, dim) + } + + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.eq(&rhs_broadcast).unwrap()) + } + + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new(lhs.tensor.eq(rhs.elem::()).unwrap()) + } + + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.gt(&rhs_broadcast).unwrap()) + } + + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .gt(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.ge(&rhs_broadcast).unwrap()) + } + + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .ge(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.lt(&rhs_broadcast).unwrap()) + } + + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .lt(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.le(&rhs_broadcast).unwrap()) + } + + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .le(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + CandleTensor::new(lhs.tensor.broadcast_add(&rhs.tensor).unwrap()) + } + + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + CandleTensor::new((lhs.tensor + rhs.elem::()).unwrap()) + } + + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + CandleTensor::new(lhs.tensor.broadcast_sub(&rhs.tensor).unwrap()) + } + + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + CandleTensor::new((lhs.tensor - rhs.elem::()).unwrap()) + } + + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + CandleTensor::new(lhs.tensor.broadcast_mul(&rhs.tensor).unwrap()) + } + + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + CandleTensor::new((lhs.tensor * rhs.elem::()).unwrap()) + } + + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + CandleTensor::new(lhs.tensor.broadcast_div(&rhs.tensor).unwrap()) + } + + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + // Candle implements scalar a/b as a * (1/b). With ints 1/b is rounded to 0 so we always obtain 0. + panic!("Not supported by Candle") + } + + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + CandleTensor::new( + (lhs.tensor.clone() + - lhs + .tensor + .broadcast_div(&rhs.tensor) + .unwrap() + .broadcast_mul(&rhs.tensor) + .unwrap()) + .unwrap(), + ) + } + + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + // Same problem as int_div_scalar. + panic!("Not supported by Candle") + } + + fn int_zeros(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + CandleTensor::new( + candle_core::Tensor::zeros( + shape.to_vec(), + dtype.into_dtype(), + &(device.clone()).into(), + ) + .unwrap(), + ) + } + + fn int_ones(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + CandleTensor::new( + candle_core::Tensor::ones(shape.to_vec(), dtype.into_dtype(), &(device.clone()).into()) + .unwrap(), + ) + } + + fn int_sum(tensor: IntTensor) -> IntTensor { + let sum = tensor.tensor.sum_all().unwrap().to_scalar::().unwrap(); + CandleTensor::from_data::( + TensorData::new([sum].into(), [1]), + Self::int_device(&tensor), + ) + } + + fn int_sum_dim(tensor: IntTensor, dim: usize) -> IntTensor { + CandleTensor::new(tensor.tensor.sum_keepdim(dim).unwrap()) + } + + fn int_prod(tensor: IntTensor) -> IntTensor { + todo!( + "prod is not implemented for Candle IntTensor (see https://github.com/tracel-ai/burn/issues/1454)" + ) + } + + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + todo!( + "prod_int is not implemented for Candle IntTensor (see https://github.com/tracel-ai/burn/issues/1454)" + ) + } + + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor { + // Candle implements scalar a/b as a * (1/b). With ints 1/b is rounded to 0 so we always obtain 0. + panic!("Not supported by Candle") + } + + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor { + // Candle's cumsum doesn't support integer types, so we convert to float, + // compute cumsum, and convert back to int + let dtype = tensor.tensor.dtype(); + let tensor_float = tensor.tensor.to_dtype(candle_core::DType::F32).unwrap(); + let result_float = tensor_float.cumsum(dim).unwrap(); + CandleTensor::new(result_float.to_dtype(dtype).unwrap()) + } + + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor { + // Convert to float for computation, then convert back + let dtype = tensor.tensor.dtype(); + let tensor_float = tensor.tensor.to_dtype(candle_core::DType::F32).unwrap(); + + let result_float = super::utils::cumulative_with_op(&tensor_float, dim, |prev, curr| { + prev.broadcast_mul(curr) + }); + CandleTensor::new(result_float.to_dtype(dtype).unwrap()) + } + + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor { + // Convert to float for computation, then convert back + let dtype = tensor.tensor.dtype(); + let tensor_float = tensor.tensor.to_dtype(candle_core::DType::F32).unwrap(); + + let result_float = super::utils::cumulative_with_op(&tensor_float, dim, |prev, curr| { + prev.broadcast_minimum(curr) + }); + CandleTensor::new(result_float.to_dtype(dtype).unwrap()) + } + + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor { + let result = super::utils::cumulative_with_op(&tensor.tensor, dim, |prev, curr| { + prev.broadcast_maximum(curr) + }); + CandleTensor::new(result) + } + + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor { + CandleTensor::new( + tensor + .tensor + .argmax_keepdim(dim) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap(), + ) + } + + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor { + CandleTensor::new( + tensor + .tensor + .argmin_keepdim(dim) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap(), + ) + } + + fn int_abs(tensor: IntTensor) -> IntTensor { + // Ugly type conversion here as Candle does not support unary ops on ints + match tensor.tensor.dtype() { + candle_core::DType::U8 | candle_core::DType::U32 => tensor, + candle_core::DType::I64 => CandleTensor::new( + tensor + .tensor + .to_dtype(F::DTYPE) + .unwrap() + .abs() + .unwrap() + .to_dtype(candle_core::DType::I64) + .unwrap(), + ), + _ => unreachable!(), + } + } + + fn int_swap_dims(tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor { + super::base::swap_dims(tensor, dim1, dim2) + } + + fn int_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> IntTensor { + if let CandleDevice::Cpu = device { + let distribution = if distribution == Distribution::Default { + Distribution::Uniform(0.0, 255.0) + } else { + distribution + }; + // Use our own seed since candle doesn't support it on CPU + return Self::int_from_data(cpu_random::(shape, distribution), device); + } + + let shape = shape.to_vec(); + let device = &(device.clone()).into(); + match distribution { + Distribution::Default => CandleTensor::new( + candle_core::Tensor::rand(0.elem::(), 255.elem::(), shape, device) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap(), + ), + Distribution::Bernoulli(prob) => CandleTensor::new( + candle_core::Tensor::rand(0.elem::(), 1.elem::(), shape.clone(), device) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap() + .lt(&super::candle_utils::fill(prob, shape, I::DTYPE, device)) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap(), + ), + Distribution::Uniform(from, to) => CandleTensor::new( + candle_core::Tensor::rand(from.elem::(), to.elem::(), shape, device).unwrap(), + ), + Distribution::Normal(mean, std) => CandleTensor::new( + candle_core::Tensor::randn(mean.elem::(), std.elem::(), shape, device) + .unwrap(), + ), + } + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + super::base::permute(tensor, axes) + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + super::base::flip(tensor, axes) + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + expand(tensor, shape) + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + unfold(tensor, dim, size, step) + } + + fn int_sign(tensor: IntTensor) -> IntTensor { + sign(tensor) + } + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + unimplemented!("bitwise_and is not implemented for Candle IntTensor"); + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unimplemented!("bitwise_and_scalar is not implemented for Candle IntTensor"); + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + unimplemented!("bitwise_or is not implemented for Candle IntTensor"); + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unimplemented!("bitwise_or_scalar is not implemented for Candle IntTensor"); + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + unimplemented!("bitwise_xor is not implemented for Candle IntTensor"); + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unimplemented!("bitwise_xor_scalar is not implemented for Candle IntTensor"); + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + unimplemented!("bitwise_not is not implemented for Candle IntTensor"); + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + unimplemented!("bitwise_left_shift is not implemented for Candle IntTensor"); + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + unimplemented!("bitwise_right_shift is not implemented for Candle IntTensor"); + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unimplemented!("bitwise_left_shift_scalar is not implemented for Candle IntTensor"); + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unimplemented!("bitwise_right_shift_scalar is not implemented for Candle IntTensor"); + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let lhs = Self::int_into_float(lhs); + let rhs = Self::int_into_float(rhs); + + let out = Self::float_matmul(lhs, rhs); + Self::float_into_int(out) + } + + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor { + let dtype = dtype.into_dtype(); + + if tensor.tensor.dtype() == dtype { + tensor + } else { + CandleTensor::new(tensor.tensor.to_dtype(dtype).unwrap()) + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/mod.rs new file mode 100644 index 0000000..49de10c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/mod.rs @@ -0,0 +1,10 @@ +mod activation; +mod base; +mod bool_tensor; +mod candle_utils; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; +mod utils; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/module.rs new file mode 100644 index 0000000..552079e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/module.rs @@ -0,0 +1,327 @@ +use burn_backend::{ + Shape, + ops::{ + ConvOptions, ConvTransposeOptions, DeformConv2dBackward, DeformConvOptions, + InterpolateMode, InterpolateOptions, MaxPool2dBackward, MaxPool2dWithIndices, ModuleOps, + UnfoldOptions, attention::attention_fallback, + }, + tensor::{FloatTensor, IntTensor}, +}; +use candle_core::ToUsize2; + +use crate::{ + Candle, CandleTensor, + element::{CandleElement, FloatCandleElement, IntCandleElement}, + ops::base::reshape, +}; + +impl ModuleOps for Candle { + fn conv1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<1>, + ) -> FloatTensor { + let conv = x + .tensor + .conv1d( + &weight.tensor, + options.padding[0], + options.stride[0], + options.dilation[0], + options.groups, + ) + .unwrap(); + CandleTensor::new(match bias { + Some(bias) => conv + .broadcast_add(&bias.tensor.unsqueeze(1).unwrap()) + .unwrap(), + None => conv, + }) + } + + fn conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<2>, + ) -> FloatTensor { + assert!( + options.dilation[0] == options.dilation[1] + && options.padding[0] == options.padding[1] + && options.stride[0] == options.stride[1], + "Candle does not support per dimension options in convolutions" + ); + let conv = x + .tensor + .conv2d( + &weight.tensor, + options.padding[0], + options.stride[0], + options.dilation[0], + options.groups, + ) + .unwrap(); + CandleTensor::new(match bias { + Some(bias) => conv + .broadcast_add( + &bias + .tensor + .unsqueeze(0) + .unwrap() + .unsqueeze(2) + .unwrap() + .unsqueeze(3) + .unwrap(), + ) + .unwrap(), + None => conv, + }) + } + + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> FloatTensor { + unimplemented!("Candle does not support deformable convolutions") + } + + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + unimplemented!("Candle does not support deformable convolutions") + } + + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> FloatTensor { + panic!("Candle does not support 3D convolutions"); + } + + fn conv_transpose1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<1>, + ) -> FloatTensor { + let conv_transpose = x + .tensor + .conv_transpose1d( + &weight.tensor, + options.padding[0], + options.padding_out[0], + options.stride[0], + options.dilation[0], + options.groups, + ) + .unwrap(); + CandleTensor::new(match bias { + Some(bias) => conv_transpose + .broadcast_add(&bias.tensor.unsqueeze(0).unwrap().unsqueeze(2).unwrap()) + .unwrap(), + None => conv_transpose, + }) + } + + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + assert!( + options.dilation[0] == options.dilation[1] + && options.padding[0] == options.padding[1] + && options.padding_out[0] == options.padding_out[1] + && options.stride[0] == options.stride[1], + "Candle does not support per dimension options in transposed convolutions" + ); + assert!( + options.groups == 1, + "Candle does not support groups in transposed convolutions" + ); + let conv_transpose = x + .tensor + .conv_transpose2d( + &weight.tensor, + options.padding[0], + options.padding_out[0], + options.stride[0], + options.dilation[0], + ) + .unwrap(); + CandleTensor::new(match bias { + Some(bias) => conv_transpose + .broadcast_add( + &bias + .tensor + .unsqueeze(0) + .unwrap() + .unsqueeze(2) + .unwrap() + .unsqueeze(3) + .unwrap(), + ) + .unwrap(), + None => conv_transpose, + }) + } + + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + panic!("Candle does not support 3D transposed convolutions"); + } + + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + assert!( + padding[0] == 0 && padding[1] == 0, + "Candle does not support padding in pooling" + ); + assert!( + count_include_pad, + "Candle does not support excluding pad count in pooling" + ); + assert!(!ceil_mode, "Candle does not support ceil_mode in pooling"); + CandleTensor::new( + x.tensor + .avg_pool2d_with_stride((kernel_size[0], kernel_size[1]), (stride[0], stride[1])) + .unwrap(), + ) + } + + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + _ceil_mode: bool, + ) -> FloatTensor { + panic!("avg_pool2d_backward is not supported by Candle") + } + + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor { + assert!( + padding[0] == 0 && padding[1] == 0, + "Candle does not support padding in pooling" + ); + assert!( + dilation[0] == 1 && dilation[1] == 1, + "Candle does not support dilation in pooling" + ); + assert!(!ceil_mode, "Candle does not support ceil_mode in pooling"); + CandleTensor::new( + x.tensor + .max_pool2d_with_stride((kernel_size[0], kernel_size[1]), (stride[0], stride[1])) + .unwrap(), + ) + } + + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + _ceil_mode: bool, + ) -> MaxPool2dWithIndices> { + panic!("max_pool2d_with_indices is not supported by Candle") + } + + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + _ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool2dBackward> { + panic!("max_pool2d_with_indices_backward is not supported by Candle") + } + + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor { + panic!("adaptive_avg_pool2 is not supported by Candle") + } + + fn adaptive_avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + panic!("adaptive_avg_pool2d_backward is not supported by Candle") + } + + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + let tensor = match options.mode { + InterpolateMode::Nearest => x + .tensor + .upsample_nearest2d(output_size[0], output_size[1]) + .unwrap(), + InterpolateMode::Bilinear => { + panic!("bilinear interpolation is not supported by Candle") + } + InterpolateMode::Bicubic => { + panic!("bicubic interpolation is not supported by Candle") + } + }; + + CandleTensor::new(tensor) + } + + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + panic!("interpolate_backward is not supported by Candle") + } + + fn attention( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: burn_backend::ops::AttentionModuleOptions, + ) -> FloatTensor { + attention_fallback::(query, key, value, mask, attn_bias, options) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/qtensor.rs new file mode 100644 index 0000000..1c87e81 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/qtensor.rs @@ -0,0 +1,88 @@ +use burn_backend::{ + Backend, DType, ExecutionError, Shape, Slice, TensorData, + ops::QTensorOps, + quantization::{QuantScheme, QuantizationParametersPrimitive}, + tensor::{Device, FloatTensor, IntTensor, QuantizedTensor}, +}; + +use crate::{ + Candle, + element::{FloatCandleElement, IntCandleElement}, +}; + +impl QTensorOps for Candle { + fn q_from_data(data: TensorData, device: &Device) -> QuantizedTensor { + unimplemented!() + } + + fn quantize( + _tensor: FloatTensor, + _scheme: &QuantScheme, + _qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + unimplemented!() + } + + fn dequantize(_tensor: QuantizedTensor) -> FloatTensor { + unimplemented!() + } + + fn q_device(_tensor: &QuantizedTensor) -> Device { + unimplemented!() + } + + fn q_to_device( + _tensor: QuantizedTensor, + _device: &Device, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_reshape(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } + + async fn q_into_data(tensor: QuantizedTensor) -> Result { + unimplemented!() + } + + fn q_swap_dims( + _tensor: QuantizedTensor, + _dim1: usize, + _dim2: usize, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_permute(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_flip(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_gather( + _dim: usize, + _tensor: QuantizedTensor, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_select( + _tensor: QuantizedTensor, + _dim: usize, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_slice(_tensor: QuantizedTensor, _slices: &[Slice]) -> QuantizedTensor { + unimplemented!() + } + + fn q_expand(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/tensor.rs new file mode 100644 index 0000000..3517daf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/tensor.rs @@ -0,0 +1,612 @@ +use std::borrow::Borrow; + +use burn_backend::{ + DType, Distribution, ElementConversion, ExecutionError, FloatDType, Scalar, Shape, Slice, + TensorData, bf16, f16, + ops::FloatTensorOps, + tensor::{BoolTensor, Device, FloatElem, FloatTensor, IntTensor}, +}; +use candle_core::{Tensor, backend::BackendStorage, shape}; + +use crate::{ + Candle, CandleDevice, CandleTensor, IntoDType, + element::{CandleElement, FloatCandleElement, IntCandleElement}, +}; + +use super::base::{cpu_random, expand, permute, sign, unfold}; + +impl FloatTensorOps for Candle { + fn float_from_data(data: TensorData, device: &Device) -> CandleTensor { + match data.dtype { + DType::F64 => super::base::from_data::(data, device), + DType::F32 => super::base::from_data::(data, device), + DType::F16 => super::base::from_data::(data, device), + DType::BF16 => super::base::from_data::(data, device), + _ => unimplemented!("Unsupported dtype for `float_from_data`"), + } + } + + fn float_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> FloatTensor { + if let CandleDevice::Cpu = device { + // Use our own seed since candle doesn't support it on CPU + return Self::float_from_data(cpu_random::(shape, distribution), device); + } + + let shape = shape.to_vec(); + let device = &(device.clone()).into(); + match distribution { + Distribution::Default => CandleTensor::new( + candle_core::Tensor::rand(0.elem::(), 1.elem::(), shape, device) + .unwrap() + .to_dtype(F::DTYPE) + .unwrap(), + ), + Distribution::Bernoulli(prob) => CandleTensor::new( + candle_core::Tensor::rand(0.elem::(), 1.elem::(), shape.clone(), device) + .unwrap() + .to_dtype(F::DTYPE) + .unwrap() + .lt(&super::candle_utils::fill(prob, shape, F::DTYPE, device)) + .unwrap() + .to_dtype(F::DTYPE) + .unwrap(), + ), + Distribution::Uniform(from, to) => CandleTensor::new( + candle_core::Tensor::rand(from.elem::(), to.elem::(), shape, device).unwrap(), + ), + Distribution::Normal(mean, std) => CandleTensor::new( + candle_core::Tensor::randn(mean.elem::(), std.elem::(), shape, device) + .unwrap(), + ), + } + } + + async fn float_into_data(tensor: CandleTensor) -> Result { + super::base::into_data(tensor) + } + + fn float_device(tensor: &CandleTensor) -> Device { + super::base::device(tensor) + } + + fn float_to_device(tensor: CandleTensor, device: &Device) -> CandleTensor { + super::base::to_device(tensor, device) + } + + fn float_into_int(tensor: CandleTensor) -> IntTensor { + CandleTensor::new(tensor.tensor.to_dtype(I::DTYPE).unwrap()) + } + + fn float_empty(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + super::base::empty(shape, device, dtype.into_dtype()) + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + CandleTensor::new(lhs.tensor.broadcast_add(&rhs.tensor).unwrap()) + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + CandleTensor::new((lhs.tensor + rhs.elem::()).unwrap()) + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + CandleTensor::new(lhs.tensor.broadcast_sub(&rhs.tensor).unwrap()) + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + CandleTensor::new((lhs.tensor - rhs.elem::()).unwrap()) + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + CandleTensor::new(lhs.tensor.broadcast_mul(&rhs.tensor).unwrap()) + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + CandleTensor::new((lhs.tensor * rhs.elem::()).unwrap()) + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + CandleTensor::new(lhs.tensor.broadcast_div(&rhs.tensor).unwrap()) + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + CandleTensor::new((lhs.tensor / rhs.elem::()).unwrap()) + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + CandleTensor::new( + (lhs.tensor.clone() + - lhs + .tensor + .broadcast_div(&rhs.tensor) + .unwrap() + .floor() + .unwrap() + .broadcast_mul(&rhs.tensor) + .unwrap()) + .unwrap(), + ) + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + // In PyTorch, remainder can also be defined as torch.remainder(a, b) == a - a.div(b, rounding_mode="floor") * b + let rhs_val = rhs.elem::(); + let division_result = (lhs.tensor.clone() / rhs_val).unwrap().floor().unwrap(); + let product = division_result * rhs_val; + + CandleTensor::new((lhs.tensor - product).unwrap()) + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let lhs_contiguous = if !lhs.tensor.is_contiguous() { + lhs.tensor.contiguous().unwrap() + } else { + lhs.tensor + }; + let rhs_contiguous = if !rhs.tensor.is_contiguous() { + rhs.tensor.contiguous().unwrap() + } else { + rhs.tensor + }; + CandleTensor::new(lhs_contiguous.broadcast_matmul(&rhs_contiguous).unwrap()) + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + super::base::cross(lhs, rhs, dim) + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + super::base::swap_dims(tensor, dim1, dim2) + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + super::base::reshape(tensor, shape) + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + let tensor = tensor.tensor.contiguous().unwrap(); + let indices = indices.tensor.contiguous().unwrap(); + CandleTensor::new(tensor.gather(&indices, dim).unwrap()) + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + CandleTensor::new( + tensor + .tensor + .scatter_add(&indices.tensor, &value.tensor, dim) + .unwrap(), + ) + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + ) -> FloatTensor { + CandleTensor::new(tensor.tensor.index_select(&indices.tensor, dim).unwrap()) + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + CandleTensor::new( + tensor + .tensor + .index_add(&indices.tensor, &value.tensor, dim) + .unwrap(), + ) + } + + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor { + super::base::slice_with_steps(tensor, slices) + } + + fn float_slice_assign( + tensor: FloatTensor, + slices: &[Slice], + value: FloatTensor, + ) -> FloatTensor { + super::base::slice_assign(tensor, slices, value) + } + + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + value: FloatTensor, + ) -> FloatTensor { + super::base::mask_where_broadcasted(tensor, mask, value) + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor { + let value = super::candle_utils::fill_like::(value.elem(), &tensor.tensor); + super::base::mask_where_broadcasted(tensor, mask, CandleTensor::new(value)) + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.eq(&rhs_broadcast).unwrap()) + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new(lhs.tensor.eq(rhs.elem::()).unwrap()) + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.gt(&rhs_broadcast).unwrap()) + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .gt(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.ge(&rhs_broadcast).unwrap()) + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .ge(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.lt(&rhs_broadcast).unwrap()) + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .lt(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let (lhs_broadcast, rhs_broadcast) = + super::candle_utils::broadcast_for_comparison(&lhs.tensor, &rhs.tensor).unwrap(); + CandleTensor::new(lhs_broadcast.le(&rhs_broadcast).unwrap()) + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + CandleTensor::new( + lhs.tensor + .le(&super::candle_utils::fill_like::( + rhs.elem(), + &lhs.tensor, + )) + .unwrap(), + ) + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + let sum = tensor.tensor.sum_all().unwrap().to_scalar::().unwrap(); + CandleTensor::from_data::( + TensorData::new([sum].into(), [1]), + Self::float_device(&tensor), + ) + } + + fn float_sum_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + CandleTensor::new(tensor.tensor.sum_keepdim(dim).unwrap()) + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + CandleTensor::new(tensor.tensor.mean_keepdim(dim).unwrap()) + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + CandleTensor::new(tensor.tensor.cumsum(dim).unwrap()) + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + let result = super::utils::cumulative_with_op(&tensor.tensor, dim, |prev, curr| { + prev.broadcast_mul(curr) + }); + CandleTensor::new(result) + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + let result = super::utils::cumulative_with_op(&tensor.tensor, dim, |prev, curr| { + prev.broadcast_minimum(curr) + }); + CandleTensor::new(result) + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + let result = super::utils::cumulative_with_op(&tensor.tensor, dim, |prev, curr| { + prev.broadcast_maximum(curr) + }); + CandleTensor::new(result) + } + + fn float_exp(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.exp().unwrap()) + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.log().unwrap()) + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new((tensor.tensor + 1.).unwrap().log().unwrap()) + } + + fn float_powf_scalar_impl(tensor: FloatTensor, value: Scalar) -> FloatTensor { + CandleTensor::new(tensor.tensor.powf(value.elem::()).unwrap()) + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.sqrt().unwrap()) + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.abs().unwrap()) + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.cos().unwrap()) + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + // cosh(x) = (e^x + e^(-x)) / 2 + let exp_x = tensor.tensor.exp().unwrap(); + CandleTensor::new(((exp_x.clone() + exp_x.recip().unwrap()).unwrap() / 2.0).unwrap()) + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.sin().unwrap()) + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + // sinh(x) = (e^x - e^(-x)) / 2 + let exp_x = tensor.tensor.exp().unwrap(); + CandleTensor::new(((exp_x.clone() - exp_x.recip().unwrap()).unwrap() / 2.0).unwrap()) + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new((tensor.tensor.sin().unwrap() / tensor.tensor.cos().unwrap()).unwrap()) + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.tanh().unwrap()) + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + // acos(x) = PI/2 - asin(x) + let neg_asin_x = Self::float_neg(Self::float_asin(tensor)); + Self::float_add_scalar(neg_asin_x, core::f64::consts::FRAC_PI_2.into()) + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + // acosh(x) = ln(x + sqrt(x^2 - 1)) + let x_squared = Self::float_powi_scalar(tensor.clone(), 2.into()); + let x_sq_minus_one = Self::float_sub_scalar(x_squared, 1f64.into()); + let sqrt_term = Self::float_sqrt(x_sq_minus_one); + Self::float_log(Self::float_add(tensor, sqrt_term)) + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + // asin(x) = atan(x / sqrt(1 - x^2)) + let x_squared = Self::float_powi_scalar(tensor.clone(), 2.into()); + let one_minus_x_sq = Self::float_add_scalar(Self::float_neg(x_squared), 1f64.into()); + let sqrt_term = Self::float_sqrt(one_minus_x_sq); + Self::float_atan(Self::float_div(tensor, sqrt_term)) + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + // asinh(x) = ln(x + sqrt(x^2 + 1)) + let x_squared = Self::float_powi_scalar(tensor.clone(), 2.into()); + let x_sq_plus_one = Self::float_add_scalar(x_squared, 1f64.into()); + let sqrt_term = Self::float_sqrt(x_sq_plus_one); + Self::float_log(Self::float_add(tensor, sqrt_term)) + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + // atan(x) = asin(x / sqrt(1 + x^2)) + let x_squared = Self::float_powi_scalar(tensor.clone(), 2.into()); + let one_plus_x_sq = Self::float_add_scalar(x_squared, 1f64.into()); + let sqrt_term = Self::float_sqrt(one_plus_x_sq); + Self::float_asin(Self::float_div(tensor, sqrt_term)) + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + // atanh(x) = ln((1 + x) / (1 - x)) / 2 + let num = (1.0 + tensor.tensor.clone()).unwrap(); + let denom = (1.0 - tensor.tensor).unwrap(); + CandleTensor::new(((num / denom).unwrap().log().unwrap() / 2.0).unwrap()) + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + // atan2(y, x) = 2 * atan(y / (sqrt(x^2 + y^2) + x)) + let x_squared = Self::float_powi_scalar(rhs.clone(), 2.into()); + let y_squared = Self::float_powi_scalar(lhs.clone(), 2.into()); + let r = Self::float_sqrt(Self::float_add(x_squared, y_squared)); + let ratio = Self::float_div(lhs, Self::float_add(r, rhs)); + Self::float_mul_scalar(Self::float_atan(ratio), 2f64.into()) + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + let inner = |tensor: FloatTensor| -> candle_core::Result> { + // implements round_to_even for consistent behavior vs libtorch + // https://github.com/pytorch/pytorch/blob/main/torch/csrc/jit/runtime/register_ops_utils.h#L65-L67 + + let floor_a = tensor.tensor.floor()?; + let frac_part = tensor.tensor.sub(&floor_a)?; + + let half = (candle_core::Tensor::ones_like(&tensor.tensor)? * 0.5)?; + let mask_half = frac_part.eq(&half)?; + let half_tensor = tensor.tensor.mul(&half)?; + let rounded_half = half_tensor.round()?; + let doubled = + rounded_half.mul(&(candle_core::Tensor::ones_like(&tensor.tensor)? * 2.0)?)?; + let standard_round = tensor.tensor.round()?; + Ok(CandleTensor::new( + mask_half.where_cond(&doubled, &standard_round)?, + )) + }; + inner(tensor).unwrap() + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.floor().unwrap()) + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.ceil().unwrap()) + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + // truncate(x) = ⌊x⌋ if x ≥ 0, and ⌈x⌉ if x < 0 + // This preserves the sign of zero and handles all special cases correctly + let is_negative = tensor.tensor.lt(0.0).unwrap(); + let floored = tensor.tensor.floor().unwrap(); + let ceiled = tensor.tensor.ceil().unwrap(); + CandleTensor::new(is_negative.where_cond(&ceiled, &floored).unwrap()) + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.erf().unwrap()) + } + + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + super::base::cat(tensors, dim) + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor { + CandleTensor::new( + tensor + .tensor + .argmax_keepdim(dim) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap(), + ) + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor { + CandleTensor::new( + tensor + .tensor + .argmin_keepdim(dim) + .unwrap() + .to_dtype(I::DTYPE) + .unwrap(), + ) + } + + fn float_clamp_max(tensor: FloatTensor, max: Scalar) -> FloatTensor { + CandleTensor::new(tensor.tensor.minimum(max.elem::()).unwrap()) + } + + fn float_clamp_min(tensor: FloatTensor, min: Scalar) -> FloatTensor { + CandleTensor::new(tensor.tensor.maximum(min.elem::()).unwrap()) + } + + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + CandleTensor::new( + tensor + .tensor + .clamp(min.elem::(), max.elem::()) + .unwrap(), + ) + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + CandleTensor::new(tensor.tensor.recip().unwrap()) + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + //broadcast_pow is in main but not yet published + //note: probably replace once pow once 0.3.3 is out + //see: https://github.com/huggingface/candle/pull/1583/files#diff-6319fa1e16dadc4c7b4e25698139703d93b70f30a1f8e2ac0999978e39efaa81R2594 + + CandleTensor::new( + rhs.tensor + .broadcast_mul(&lhs.tensor.log().unwrap()) + .unwrap() + .exp() + .unwrap(), + ) + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + super::base::permute(tensor, axes) + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + super::base::flip(tensor, axes) + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + expand(tensor, shape) + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + unfold(tensor, dim, size, step) + } + + fn float_sign(tensor: FloatTensor) -> FloatTensor { + sign(tensor) + } + + fn float_cast(tensor: FloatTensor, dtype: FloatDType) -> FloatTensor { + let dtype = dtype.into_dtype(); + + if tensor.tensor.dtype() == dtype { + tensor + } else { + CandleTensor::new(tensor.tensor.to_dtype(dtype).unwrap()) + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/transaction.rs new file mode 100644 index 0000000..e4025a7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/transaction.rs @@ -0,0 +1,11 @@ +use burn_backend::{ + Backend, + ops::{TransactionOps, TransactionPrimitive}, +}; + +use crate::{ + Candle, + element::{FloatCandleElement, IntCandleElement}, +}; + +impl TransactionOps for Candle {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/utils.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/utils.rs new file mode 100644 index 0000000..7686018 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/ops/utils.rs @@ -0,0 +1,29 @@ +/// Helper function for cumulative operations in Candle backend +/// +/// This function reduces code duplication for cumulative operations (cumprod, cummin, cummax) +/// which all follow the same pattern of slicing, applying an operation, and concatenating. +/// +/// # Arguments +/// +/// * `tensor` - The input tensor +/// * `dim` - The dimension along which to apply the cumulative operation +/// * `op` - A closure that takes two tensor references and produces a result tensor +pub fn cumulative_with_op(tensor: &candle_core::Tensor, dim: usize, op: F) -> candle_core::Tensor +where + F: Fn(&candle_core::Tensor, &candle_core::Tensor) -> candle_core::Result, +{ + let dim_size = tensor.dims()[dim]; + let mut slices = Vec::with_capacity(dim_size); + + // First slice is the initial value + slices.push(tensor.narrow(dim, 0, 1).unwrap()); + + // Apply cumulative operation + for i in 1..dim_size { + let curr = tensor.narrow(dim, i, 1).unwrap(); + let result = op(&slices[i - 1], &curr).unwrap(); + slices.push(result); + } + + candle_core::Tensor::cat(&slices, dim).unwrap() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-candle/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/tensor.rs new file mode 100644 index 0000000..ad7689d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-candle/src/tensor.rs @@ -0,0 +1,114 @@ +use burn_backend::{DType, FloatDType, IntDType, Shape, quantization::QuantScheme}; +use burn_backend::{Element, QTensorPrimitive, TensorData, TensorMetadata}; + +use crate::{CandleDevice, element::CandleElement}; + +/// A tensor that uses the candle backend. +#[derive(Debug, Clone)] +pub struct CandleTensor { + pub(crate) tensor: candle_core::Tensor, +} + +impl TensorMetadata for CandleTensor { + fn dtype(&self) -> DType { + match self.tensor.dtype() { + candle_core::DType::U8 => DType::U8, + candle_core::DType::U32 => DType::U32, + candle_core::DType::I64 => DType::I64, + candle_core::DType::BF16 => DType::BF16, + candle_core::DType::F16 => DType::F16, + candle_core::DType::F32 => DType::F32, + candle_core::DType::F64 => DType::F64, + candle_core::DType::I16 => DType::I16, + candle_core::DType::I32 => DType::I32, + other => todo!("{other:?} not yet supported"), + } + } + + fn shape(&self) -> Shape { + Shape::from(self.tensor.dims().to_vec()) + } + + fn rank(&self) -> usize { + self.tensor.dims().len() + } +} + +impl QTensorPrimitive for CandleTensor { + fn scheme(&self) -> &QuantScheme { + unimplemented!("Quantization is not supported") + } +} + +impl CandleTensor { + /// Create a new tensor. + pub fn new(tensor: candle_core::Tensor) -> Self { + Self { tensor } + } + + /// Creates a new tensor from data and a device. + /// + /// # Arguments + /// + /// * `data` - The tensor's data. + /// * `device` - The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// A new tensor. + pub fn from_data(data: TensorData, device: CandleDevice) -> Self { + let candle_shape: candle_core::Shape = data.shape.clone().into(); + let tensor = candle_core::Tensor::from_slice( + data.as_slice::().unwrap(), + candle_shape, + &device.into(), + ); + Self::new(tensor.unwrap()) + } +} + +pub(crate) trait IntoDType { + fn try_into_dtype(self) -> Result; + + fn into_dtype(self) -> candle_core::DType + where + Self: Sized, + { + self.try_into_dtype().unwrap() + } +} + +impl IntoDType for IntDType { + fn try_into_dtype(self) -> Result { + let dtype: DType = self.into(); + dtype.try_into_dtype() + } +} + +impl IntoDType for FloatDType { + fn try_into_dtype(self) -> Result { + let dtype: DType = self.into(); + dtype.try_into_dtype() + } +} + +impl IntoDType for DType { + fn try_into_dtype(self) -> Result { + match self { + DType::F64 => Ok(candle_core::DType::F64), + DType::F32 => Ok(candle_core::DType::F32), + DType::Flex32 => Ok(candle_core::DType::F32), + DType::F16 => Ok(candle_core::DType::F16), + DType::BF16 => Ok(candle_core::DType::BF16), + DType::I64 => Ok(candle_core::DType::I64), + DType::U32 => Ok(candle_core::DType::U32), + DType::U8 => Ok(candle_core::DType::U8), + DType::I16 => Ok(candle_core::DType::I16), + DType::I32 => Ok(candle_core::DType::I32), + // DType::Bool => Ok(candle_core::DType::U8), + _ => Err(candle_core::Error::Msg(format!( + "Unsupported dtype {self:?}" + ))), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-collective/Cargo.toml new file mode 100644 index 0000000..9018562 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/Cargo.toml @@ -0,0 +1,73 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Backend extension for collective calculations." +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "collective"] +license.workspace = true +name = "burn-collective" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-collective" +documentation = "https://docs.rs/burn-collective" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [] +doc = [] +tracing = [ + "dep:tracing", + "burn-std/tracing", + "burn-tensor/tracing", + "burn-communication/tracing", + "burn-ndarray?/tracing", + "burn-wgpu?/tracing", + "burn-cuda?/tracing", +] +orchestrator = ["burn-communication/websocket"] + +# Backends for testing +test-ndarray = ["burn-ndarray"] +test-wgpu = ["burn-wgpu", "burn-wgpu/webgpu"] +test-metal = ["burn-wgpu", "burn-wgpu/metal"] +test-vulkan = ["burn-wgpu", "burn-wgpu/vulkan"] +test-cuda = ["burn-cuda"] + +[dependencies] +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", default-features = true } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = true } + +log = { workspace = true } + +burn-communication = { path = "../burn-communication", version = "=0.21.0-pre.2", features = [ + "data-service", + "websocket", +] } +tokio = { workspace = true, features = [ + "rt-multi-thread", + "sync", + "signal", + "time", + "tracing", +] } +serde = { workspace = true, features = ["derive"] } +rmp-serde = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true, optional = true } + +# Tests +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2", optional = true } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true } +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true } + +[dev-dependencies] +serial_test = { workspace = true } + + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/README.md b/crates/stable-diffusion-burn/burn-crates/burn-collective/README.md new file mode 100644 index 0000000..ade26a6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/README.md @@ -0,0 +1,139 @@ +# burn-collective + +Collective operations on tensors + +The following collective operation are implemented: + +- `all-reduce` + Aggregates a tensor between all peers, and distributes the result to all peers. + Different strategies can be used on the local and global levels. The result can only be + returned when all peers have called the all-reduce. +- `reduce` + Aggregates a tensor from all peers onto one peer, called the "root" +- `broadcast` + Copies a tensor from one peer to all other peers in the collective. + +Peers must call `register` before calling any other operation. +The total number of devices on the node, or nodes in the collective, must be known ahead of time. + +In many libraries like NCCL and PyTorch, participating units are called "ranks". +This name is confusing in the context of tensors, so in burn-collective the participating units +are called "peers". + +*`reduce` and `broadcast` are not yet implemented for multi-node contexts* + +## Local and Global + +Internally, there are two levels to the collective operations: local and global. Operations are done on the local level, then optionally on the global level. + +| Local | Global | +|-----------------------------------------------|-----------------------------------------------| +| Intra-node (typically within one machine) | Inter-node (typically across machies) | +| Participants are threads (one per peer/GPU) | Participants are processes (one per node) | +| Communication depends on backend | Network peer-to-peer communication | +| Local server is launched automatically | Global coordinator must be launched manually | +| Local server does the aggregation | Nodes do the operations themselves | + +For global operations (ie. with multiple nodes), there must be a global orchestrator available. +Start one easily with `burn_collective::start_global_orchestrator()`. + +On the global level, nodes use the `burn_communication::data_service::TensorDataService` to +expose and download tensors in a peer-to-peer manner, in order to be independent. + +## Components + +The following are the important pieces of the collective operations system. + +| Term | One per... | Meaning +|--------------------------------|---------------|---------------------------------------------------------- +| Local Collective Client | Peer/thread | Requests operations to the Local Collective Server +| Local Collective Server | Node/process | Does local-level ops for threads in this process. In the case of global operations, passes operations on to the Global Collective Client. +| Global Collective Client | Node/process | Does global-level ops for this node. Registers and requests strategies from the Global Collective Orchestrator. +| Global Collective Orchestrator | Collective | Responds to the Global Collective Client from each node. Responsible for aggregation strategies. + +## Strategies + +Different strategies can be used on the local and global level. + +### Centralized + +An arbitrary peer is designated as the "root", and all others are transferred to the root's device. +The operation is done on that device. +The resulting tensor then sent to each peer. + +### Tree + +Tensors in groups of N are aggregated together. This is done recursively until only one tensor +remains. The strategy tries to put devices of the same type closer in the tree. +When N=2, this is like a binary tree reduce. +The resulting tensor then sent to each peer + +### Ring + +See this good explanation: + +The tensors are sliced into N parts, where N is the number of tensors to aggregate. +Then, the slices are sent around in a series of cycles and aggregated until every tensor's slices +is a sum of the other corresponding slices. + +In the case where the tensors are too small to split into N slices, a fallback algorithm is used. +For now, the fallback is a binary tree. + +(p=3, n=3) + +o->o o +o o->o +o o o-> + +o 1->o +o o 1-> +1->o o + +o 1 2-> +2->o 1 +1 2->o + +3 1 2 +2 3 1 +1 2 3 + +(This is essentially a reduce-scatter) + +3->x x +x 3->x +x x 3-> + +3 3->x +x 3 3-> +3->x 3 + +3 3 3-> +3->3 3 +3 3->3 + +3 3 3 +3 3 3 +3 3 3 + +(This is essentially an all-gather) + +This is done so that every peer is both sending and receiving data at any moment. +This is an important part of this strategy's advantages. + +The ring strategy takes full advantage of the bandwidth available. The latency scales with the +number of peers. + +So when the tensors are very small, or when the number of peers is very large, the latency is more +important in the ring strategy, and a tree algorithm is better. Otherwise, the ring algorithm is +the better. + +In multi-node contexts, use of the Ring strategy in the local level may be less +advantageous. With multiple nodes, the global all-reduce step is enabled, and its result +is redistributed to all devices. +The Ring strategy inherently distributes the result, which in this context would not be necessary. + +It is recommended to use the Ring strategy at the global level + +### Double binary tree + + diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/Cargo.toml new file mode 100644 index 0000000..7ba74d6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "burn-collective-multinode-tests" +version.workspace = true +edition.workspace = true +license.workspace = true + +[features] +default = ["ndarray"] +ndarray = ["burn/ndarray"] + +[dependencies] +burn = { path = "../../burn", default-features = false, features = ["std"] } +burn-std = { path = "../../burn-std", default-features = false } +burn-collective = { path = "..", features = ["orchestrator"] } +burn-communication = { path = "../../burn-communication" } + +tokio = { workspace = true, features = ["rt-multi-thread", "process"] } + +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +interprocess = "2.3.1" +rmp-serde = { workspace = true } +tokio-util = { workspace = true, features = ["codec"] } +tokio-serde = { version = "0.9.0", features = ["messagepack"] } +futures = { workspace = true } + + +[[bin]] +name = "global" +path = "src/bin/global.rs" + +[[bin]] +name = "node" +path = "src/bin/node.rs" diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/README.md b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/README.md new file mode 100644 index 0000000..7768452 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/README.md @@ -0,0 +1,32 @@ +# Integration test for burn collective operations with multiple nodes and devices. + +Run `cargo run --bin test_launcher` + +There are 3 binaries: + +## node.rs + +Launches `n` threads each simulating a different device. Currently the backend is NdArray, +so everything is CPU. The program takes a file with configurations and input data. + +## global.rs + +Runs the global orchestrator, who is responsible for responding to global collective operation +requests. In the case of an all-reduce, the orchestrator responds with a strategy for reducing, +and the node can do the reduction independently. + +## test_launcher.rs + +Generates input data, calculates the expected results, and launches the nodes each with their +own inputs in a separate file. + +The topology is [4, 4, 4, 4]. This means 4 nodes are launched, +each with 4 threads (for each device). + +The global orchestrator (`global.rs`) is also launched. + +## Output + +The outputs and inputs for each node and the orchestrator are written to the `target/test_files` folder + +If the nodes or orchestrator stall, there is a timeout. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/global.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/global.rs new file mode 100644 index 0000000..38a99aa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/global.rs @@ -0,0 +1,19 @@ +//! Global orchestrator +//! +//! Launches the orchestrator that responds to global collective operations for nodes for the +//! integration test +//! +//! This is necessary for any node who needs global collective operations + +use std::env; + +#[tokio::main] +/// Start the global orchestrator on the port given as first arg +pub async fn main() { + let args: Vec = env::args().collect(); + let port = args[1].parse::().expect("invalid port"); + + // Launch the global orchestrator, which will listen and respond to global collective op + // requests from nodes + burn_collective::start_global_orchestrator(port).await; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/node.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/node.rs new file mode 100644 index 0000000..2393d00 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/node.rs @@ -0,0 +1,157 @@ +use burn::{ + backend::NdArray, + prelude::Backend, + tensor::{Tensor, TensorPrimitive, Tolerance}, +}; +use burn_collective::{ + CollectiveConfig, PeerId, ReduceOperation, all_reduce, finish_collective, register, + reset_collective, +}; +use burn_collective_multinode_tests::shared::{NodeTest, NodeTestResult, TENSOR_RANK}; +use std::{ + env, + sync::mpsc::SyncSender, + time::{Duration, Instant}, +}; +use tokio::net::TcpStream; + +use futures::{SinkExt, StreamExt}; +use std::thread::JoinHandle; +use tokio_serde::formats::MessagePack; +use tokio_util::codec::LengthDelimitedCodec; + +type TestBackend = NdArray; + +/// Framed TCP connection channel +type TestChannel = tokio_serde::Framed< + tokio_util::codec::Framed, + NodeTest, + NodeTestResult, + MessagePack, +>; + +/// Start a node that will test all-reduce +/// Args are the following: +/// - launcher endpoint +#[tokio::main] +pub async fn main() { + let args: Vec = env::args().collect(); + + let launcher_addr = args[1].clone(); + + let socket = TcpStream::connect(launcher_addr).await.unwrap(); + let length_delimited = tokio_util::codec::Framed::new(socket, LengthDelimitedCodec::new()); + let mut socket: TestChannel = tokio_serde::Framed::new( + length_delimited, + MessagePack::::default(), + ); + + // Loop: receive, do test, send result + while let Some(Ok(test)) = socket.next().await { + println!("Received test: {test:?}"); + + let result = run_test::(&test); + + // send the result back + socket.send(result).await.expect("failed to send Result"); + } + + println!("Server closed connection; exiting."); +} + +/// Runs a test for one node +fn run_test(test_input: &NodeTest) -> NodeTestResult { + reset_collective::(); + + // Channel for results + let (result_send, result_recv) = std::sync::mpsc::sync_channel(32); + + // Launch a thread for each "device" + let handles = launch_threads::(test_input.clone(), result_send); + + // Receive results + let mut durations = vec![]; + let tol: Tolerance = Tolerance::balanced(); + for _ in 0..test_input.device_count { + // Assert all results are equal to each other as well as expected result + let (tensor, duration) = result_recv.recv().unwrap(); + test_input.expected.assert_approx_eq(&tensor.to_data(), tol); + + durations.push(duration); + } + + // Threads finish + for handle in handles { + let _ = handle.join(); + } + + NodeTestResult { + success: true, + durations, + } +} + +/// Launch a thread for each device, and run the all-reduce +fn launch_threads( + test_input: NodeTest, + result_send: SyncSender<(Tensor, Duration)>, +) -> Vec> { + let mut handles = vec![]; + for id in 0..test_input.device_count { + // Launch a thread to test + + // Put all the parameters in the config + let config = CollectiveConfig::default() + .with_num_devices(test_input.device_count) + .with_global_address(test_input.global_address.clone()) + .with_node_address(test_input.node_address.clone()) + .with_data_service_port(test_input.data_service_port) + .with_num_nodes(test_input.node_count) + .with_global_all_reduce_strategy(test_input.global_strategy) + .with_local_all_reduce_strategy(test_input.local_strategy); + + // Inputs and outputs for the test + let tensor_data = test_input.inputs[id].clone(); + let tensor = Tensor::::from_data(tensor_data, &B::Device::default()); + let result_send = result_send.clone(); + + let handle = std::thread::spawn(move || { + run_peer::( + id.into(), + config, + tensor, + result_send, + test_input.all_reduce_op, + ) + }); + handles.push(handle); + } + + handles +} + +/// Runs a thread in the all-reduce test. +pub fn run_peer( + id: PeerId, + config: CollectiveConfig, + input: Tensor, + output: SyncSender<(Tensor, Duration)>, + all_reduce_op: ReduceOperation, +) { + // Register the device + register::(id, input.device(), config).unwrap(); + + let start = Instant::now(); + + // All-reduce + let input = input.into_primitive().tensor(); + let tensor = all_reduce::(id, input, all_reduce_op).unwrap(); + let tensor = Tensor::::from_primitive(TensorPrimitive::Float(tensor)); + + let duration = start.elapsed(); + + // Send result + output.send((tensor, duration)).unwrap(); + + finish_collective::(id).unwrap(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/test_launcher.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/test_launcher.rs new file mode 100644 index 0000000..1f915fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/bin/test_launcher.rs @@ -0,0 +1,354 @@ +use burn::tensor::TensorData; +use burn_communication::Address; +use futures::{SinkExt, StreamExt}; +use std::{ + fmt::Display, + fs::{self, File}, + str::FromStr, + time::{Duration, Instant}, + vec, +}; +use tokio::net::TcpListener; +use tokio_serde::formats::MessagePack; +use tokio_util::codec::LengthDelimitedCodec; + +use burn::{backend::NdArray, prelude::Backend, tensor::Tensor}; +use burn_collective::{AllReduceStrategy, ReduceOperation}; +use burn_collective_multinode_tests::shared::{NodeTest, NodeTestResult, TENSOR_RANK}; +use burn_std::rand::{SeedableRng, StdRng}; +use tokio::process::{Child, Command}; + +#[derive(Clone)] +struct AllReduceTest { + shape: [usize; TENSOR_RANK], + op: ReduceOperation, + local_strategy: AllReduceStrategy, + global_strategy: AllReduceStrategy, +} + +impl Display for AllReduceTest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let op_str = match self.op { + ReduceOperation::Sum => "sum", + ReduceOperation::Mean => "mean", + }; + let local_strategy_str = match self.local_strategy { + AllReduceStrategy::Centralized => "local_centralized", + AllReduceStrategy::Tree(n) => &format!("local_tree_{n}"), + AllReduceStrategy::Ring => "local_ring", + }; + let global_strategy_str = match self.global_strategy { + AllReduceStrategy::Centralized => "global_centralized", + AllReduceStrategy::Tree(n) => &format!("global_tree_{n}"), + AllReduceStrategy::Ring => "global_ring", + }; + + write!(f, "{op_str}_{local_strategy_str}_{global_strategy_str}") + } +} + +/// Framed TCP connection for sending tests and receiving results +type TestChannel = tokio_serde::Framed< + tokio_util::codec::Framed, + NodeTestResult, + NodeTest, + MessagePack, +>; + +/// Handle for each node process +struct NodeProcessHandle { + process: Child, + channel: TestChannel, +} + +/// Main function to run the multi-node all-reduce test. +/// Launches a orchestrator and multiple nodes based on the provided topology. +#[tokio::main(flavor = "multi_thread", worker_threads = 10)] +async fn main() { + let all_reduce_tests = vec![ + AllReduceTest { + shape: [4, 64, 512], + op: ReduceOperation::Mean, + local_strategy: AllReduceStrategy::Tree(2), + global_strategy: AllReduceStrategy::Tree(2), + }, + AllReduceTest { + shape: [4, 64, 512], + op: ReduceOperation::Mean, + local_strategy: AllReduceStrategy::Tree(2), + global_strategy: AllReduceStrategy::Ring, + }, + AllReduceTest { + shape: [4, 64, 512], + op: ReduceOperation::Mean, + local_strategy: AllReduceStrategy::Centralized, + global_strategy: AllReduceStrategy::Centralized, + }, + ]; + + let test_files_dir = "target/test_files"; + fs::create_dir_all(test_files_dir).expect("Couldn't create test_files directory"); + + let topology: Vec = vec![4; 4]; + + let mut orchestrator = launch_orchestrator(test_files_dir); + + let launcher_endpoint = "127.0.0.1:4000"; + + // Build and run node processes + let mut all_tests_durations = vec![]; + if let Ok(mut nodes) = launch_nodes(&topology, launcher_endpoint).await { + // Run one test + for test in all_reduce_tests.clone() { + let test_name = test.to_string(); + + let time = + test_all_reduce_centralized_no_collective::(&topology, test.clone()); + println!( + "{test_name}: Benchmark (no collective, centralized, single-threaded): {} secs", + time.as_secs_f32() + ); + + match test_all_reduce(&topology, test, &mut nodes).await { + Err(node_idx) => { + println!("{test_name}: Node with index {node_idx} failed!"); + // Kill other node processes + for mut node in nodes.drain(..) { + node.process.kill().await.unwrap(); + node.process.wait().await.unwrap(); + } + break; + } + Ok(durations) => { + all_tests_durations.append(&mut durations.clone()); + let avg = durations.iter().map(|dur| dur.as_secs_f32()).sum::() + / durations.len() as f32; + println!("{test_name}: Success in {avg} secs"); + } + } + } + } + + if !all_tests_durations.is_empty() { + let avg = all_tests_durations + .iter() + .map(|dur| dur.as_secs_f32()) + .sum::() + / all_tests_durations.len() as f32; + println!("Average for all tests: {avg} secs"); + } + + // Shutdown orchestrator + orchestrator.kill().await.unwrap(); + orchestrator.wait().await.unwrap(); +} + +/// Launch a global orchestrator with an output file in the given directory. +/// Necessary for global collective operations +/// +/// Server listens on localhost port 3000 +fn launch_orchestrator(test_files_dir: &str) -> Child { + let out_path = format!("{test_files_dir}/orchestrator_out.txt"); + let out = File::create(out_path).expect("Could't create orchestrator output file"); + + Command::new("cargo") + .args(["run", "--bin", "global", "--", "3000"]) + .stdout(out.try_clone().unwrap()) + .stderr(out) + .spawn() + .expect("failed to launch orchestrator") +} + +/// Launch nodes for a all_reduce test +/// Each node will connect to the global orchestrator and run an all-reduce operation. +/// The topology is a vector where each element represents the number of devices in that node. +async fn launch_nodes( + topology: &[usize], + launcher_endpoint: &str, +) -> Result, ()> { + println!( + "Launching {} nodes with topology: {:?}", + topology.len(), + topology + ); + + // Listen for node connections + let listener = TcpListener::bind(launcher_endpoint).await.unwrap(); + println!("Server listening on {launcher_endpoint}"); + + let mut nodes = vec![]; + + for node_idx in 0..topology.len() { + // Create log file + let output_filename = format!("target/test_files/node_{}_log.txt", node_idx + 1); + let out = File::create(output_filename).expect("Could't open node log file"); + + // Start a process for each node. Pass on our feature flags + let node_process: Child = Command::new("cargo") + .args([ + "run", + "--release", + "--features", + #[cfg(feature = "ndarray")] + "ndarray", + "--bin", + "node", + "--", + launcher_endpoint, + &node_idx.to_string(), + ]) + .stdout(out.try_clone().unwrap()) + .stderr(out) + .spawn() + .expect("node failed"); + + // Wait for child to connect for io + let (socket, _peer_addr) = listener.accept().await.unwrap(); + let length_delimited = tokio_util::codec::Framed::new(socket, LengthDelimitedCodec::new()); + let channel: TestChannel = tokio_serde::Framed::new( + length_delimited, + MessagePack::::default(), + ); + + nodes.push(NodeProcessHandle { + process: node_process, + channel, + }); + } + + Ok(nodes) +} + +async fn test_all_reduce( + topology: &[usize], + test: AllReduceTest, + nodes: &mut [NodeProcessHandle], +) -> Result, usize> { + dispatch_all_reduce_test(topology, test, nodes).await; + + let mut all_durations = vec![]; + for (idx, handle) in nodes.iter_mut().enumerate() { + match handle.channel.next().await { + Some(Ok(mut result)) => { + if !result.success { + return Err(idx); + } + all_durations.append(&mut result.durations); + } + _ => { + return Err(idx); + } + } + } + + Ok(all_durations) +} + +async fn dispatch_all_reduce_test( + topology: &[usize], + test: AllReduceTest, + nodes: &mut [NodeProcessHandle], +) { + let total_device_count: usize = topology.iter().sum(); + let (mut all_inputs, expected) = + generate_random_input(test.shape, test.op, total_device_count, 42); + + // URL for the global orchestrator on port 3000 + let global_url = "ws://localhost:3000"; + let global_address = Address::from_str(global_url).unwrap(); + + for (node_idx, &device_count) in topology.iter().enumerate() { + // Construct URL for node + // Ports 3001... are for each node + let data_service_port = node_idx as u16 + 3001; + let node_url = format!("ws://localhost:{data_service_port}"); + let node_address = Address::from_str(&node_url).unwrap(); + + // take input tensors for each device + let inputs = all_inputs[0..device_count].to_vec(); + all_inputs = all_inputs[device_count..].to_vec(); + + let test = NodeTest { + device_count, + node_id: node_idx.into(), + node_count: topology.len() as u32, + global_address: global_address.clone(), + node_address, + data_service_port, + all_reduce_op: test.op, + global_strategy: test.global_strategy, + local_strategy: test.local_strategy, + inputs, + expected: expected.clone(), + }; + let handle = &mut nodes[node_idx]; + + handle.channel.send(test).await.unwrap(); + } + + assert!( + all_inputs.is_empty(), + "Not all inputs have been sent to tests" + ); +} + +/// Run the test sequentially with no collective operations to get the optimal single-threaded speed +fn test_all_reduce_centralized_no_collective( + topology: &[usize], + test: AllReduceTest, +) -> Duration { + let total_device_count: usize = topology.iter().sum(); + let (all_inputs, _expected) = + generate_random_input(test.shape, test.op, total_device_count, 42); + + let mut all_inputs = all_inputs + .into_iter() + .map(|data| Tensor::::from_data(data, &B::Device::default())); + + let start = Instant::now(); + + // Sequential test + let mut result = all_inputs.next().unwrap(); + for other in all_inputs { + result = result.add(other); + } + if test.op == ReduceOperation::Mean { + result.div_scalar(total_device_count as u32); + } + + start.elapsed() +} + +/// Generates random input tensors and expected output based on the provided shape and reduce kind. +fn generate_random_input( + shape: [usize; 3], + reduce_kind: ReduceOperation, + input_count: usize, + seed: u64, +) -> (Vec, TensorData) { + let mut rng = StdRng::seed_from_u64(seed); + + // A random tensor for each device + let input: Vec = (0..input_count) + .map(|_| { + TensorData::random::(shape, burn::tensor::Distribution::Default, &mut rng) + }) + .collect(); + + // Sum up the inputs + let device = ::Device::default(); + let mut expected_tensor = Tensor::::zeros(shape, &device); + for item in input.iter().take(input_count) { + let input_tensor = Tensor::::from_data(item.clone(), &device); + expected_tensor = expected_tensor.add(input_tensor); + } + + if reduce_kind == ReduceOperation::Mean { + expected_tensor = expected_tensor.div_scalar(input_count as u32); + } + + // All-Reduce results should have this value + let expected = expected_tensor.to_data(); + + (input, expected) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/lib.rs new file mode 100644 index 0000000..eec3c89 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/lib.rs @@ -0,0 +1 @@ +pub mod shared; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/shared.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/shared.rs new file mode 100644 index 0000000..533a8ec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/multinode-tests/src/shared.rs @@ -0,0 +1,43 @@ +use std::time::Duration; + +use burn::tensor::TensorData; +use burn_collective::{AllReduceStrategy, NodeId, ReduceOperation}; +use burn_communication::Address; +use serde::{Deserialize, Serialize}; + +/// Ranks of inputs and outputs for all testing +pub const TENSOR_RANK: usize = 3; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeTest { + /// How many threads to start on this node + pub device_count: usize, + /// ID for this node + pub node_id: NodeId, + /// How many nodes in the cluster + pub node_count: u32, + /// Global server address + pub global_address: Address, + /// Node address + pub node_address: Address, + /// Node's data service port, for initializing the p2p tensor data service + pub data_service_port: u16, + /// What kind of all-reduce + pub all_reduce_op: ReduceOperation, + /// Node's data service port, for initializing the p2p tensor data service + pub global_strategy: AllReduceStrategy, + /// What kind of aggregation + pub local_strategy: AllReduceStrategy, + + /// Input data for test: all tensors are D=3 + pub inputs: Vec, + /// Expected output for test + pub expected: TensorData, +} + +/// Result sent back from each node for each test +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeTestResult { + pub success: bool, + pub durations: Vec, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/api.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/api.rs new file mode 100644 index 0000000..ed0a2de --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/api.rs @@ -0,0 +1,121 @@ +use burn_tensor::backend::Backend; + +use crate::{ + CollectiveConfig, PeerId, ReduceOperation, global::shared::GlobalCollectiveError, + local::server::get_collective_client, +}; + +/// Errors from collective operations +#[allow(unused)] +#[derive(Debug, Clone)] +pub enum CollectiveError { + /// The [config](CollectiveConfig) was invalid. + /// Usually happens if only some global parameters have been defined + InvalidConfig, + /// Cannot un-register a node twice + MultipleUnregister, + /// Cannot register a node twice + MultipleRegister, + /// Trying to register a different way than is currently being done + RegisterParamsMismatch, + /// Trying to all-reduce tensors of different shapes: shape must match + AllReduceShapeMismatch, + /// Trying to all-reduce a different way than is currently being done: op must match + AllReduceOperationMismatch, + /// Trying to reduce tensors of different shapes: shape must match + ReduceShapeMismatch, + /// Trying to reduce a different way than is currently being done: op must match + ReduceOperationMismatch, + /// Trying to reduce with different roots + ReduceRootMismatch, + /// Trying to broadcast with different roots + BroadcastRootMismatch, + /// Trying to broadcast but no peer sent a tensor + BroadcastNoTensor, + /// Trying to broadcast but multiple peers sent a tensor + BroadcastMultipleTensors, + /// Local collective server couldn't respond + LocalServerMissing, + /// Another operation was called before Register + RegisterNotFirstOperation, + /// The global orchestrator had an error + Global(GlobalCollectiveError), + + #[allow(unused)] + Other(String), +} + +/// Registers a device. `num_devices` must be the same for every register, +/// and `device_id` must be unique. +/// +/// * `id` - The peer id of the caller +/// +/// With auto-diff backends, make sure to use the inner backend. +pub fn register( + id: PeerId, + device: B::Device, + config: CollectiveConfig, +) -> Result<(), CollectiveError> { + log::info!("Registering peer {id} with config: {config}"); + let mut client = get_collective_client::(); + client.register(id, device, config) +} + +/// Calls for an all-reduce operation with the given parameters, and returns the result. +/// The `params` must be the same as the parameters passed by the other nodes. +/// +/// * `id` - The peer id of the caller +/// * `tensor` - The input tensor to reduce with the peers' tensors +/// * `config` - Config of the collective operation, must be coherent with the other calls +pub fn all_reduce( + id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, +) -> Result { + let client = get_collective_client::(); + client.all_reduce(id, tensor, op) +} + +/// Broadcasts, or receives a broadcasted tensor. +/// +/// * `id` - The peer id of the caller +/// * `tensor` - If defined, this tensor will be broadcasted. Otherwise, this call will receive +/// the broadcasted tensor. +/// +/// Returns the broadcasted tensor. +pub fn broadcast( + id: PeerId, + tensor: Option, +) -> Result { + let client = get_collective_client::(); + client.broadcast(id, tensor) +} + +/// Reduces a tensor onto one device. +/// +/// * `id` - The peer id of the caller +/// * `tensor` - The tensor to send as input +/// * `root` - The ID of the peer that will receive the result. +/// +/// Returns Ok(None) if the root tensor is not the caller. Otherwise, returns the reduced tensor. +pub fn reduce( + id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + root: PeerId, +) -> Result, CollectiveError> { + let client = get_collective_client::(); + client.reduce(id, tensor, op, root) +} + +/// Closes the collective session, unregistering the device +pub fn finish_collective(id: PeerId) -> Result<(), CollectiveError> { + let client = get_collective_client::(); + client.finish(id) +} + +/// Resets the local collective server. All registered callers and ongoing operations are forgotten +pub fn reset_collective() { + let client = get_collective_client::(); + client.reset(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/config.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/config.rs new file mode 100644 index 0000000..5deed4b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/config.rs @@ -0,0 +1,337 @@ +use std::fmt::Display; + +use burn_communication::Address; +use serde::{Deserialize, Serialize}; + +/// Parameter struct for setting up and getting parameters for collective operations. +/// Used in most collective api calls. +/// This config is per-node. It is passed to [reduce](crate::register). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CollectiveConfig { + pub(crate) num_devices: usize, + pub(crate) local_all_reduce_strategy: AllReduceStrategy, + pub(crate) local_reduce_strategy: ReduceStrategy, + pub(crate) local_broadcast_strategy: BroadcastStrategy, + + // Global parameters (all are optional, but if one is defined they should all be) + pub(crate) num_nodes: Option, + pub(crate) global_address: Option
, + pub(crate) node_address: Option
, + pub(crate) data_service_port: Option, + + // These strategies may be defined when no other global params are defined + pub(crate) global_all_reduce_strategy: Option, + pub(crate) global_reduce_strategy: Option, + pub(crate) global_broadcast_strategy: Option, +} + +impl Default for CollectiveConfig { + fn default() -> Self { + Self::new() + } +} + +impl Display for CollectiveConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let num_devices = self.num_devices; + let local_all_reduce_strategy = self.local_all_reduce_strategy; + let local_reduce_strategy = self.local_reduce_strategy; + let local_broadcast_strategy = self.local_broadcast_strategy; + let num_nodes = self.num_nodes; + let global_address = &self.global_address; + let node_address = &self.node_address; + let data_service_port = self.data_service_port; + let global_all_reduce_strategy = self.global_all_reduce_strategy; + let global_reduce_strategy = self.global_reduce_strategy; + let global_broadcast_strategy = self.global_broadcast_strategy; + + write!( + f, + r#" +CollectiveConfig {{ + num_devices: {num_devices:?}, + local_all_reduce_strategy: {local_all_reduce_strategy:?}, + local_reduce_strategy: {local_reduce_strategy:?}, + local_broadcast_strategy: {local_broadcast_strategy:?}, + num_nodes: {num_nodes:?}, + global_address: {global_address:?}, + node_address: {node_address:?}, + data_service_port: {data_service_port:?}, + global_all_reduce_strategy: {global_all_reduce_strategy:?}, + global_reduce_strategy: {global_reduce_strategy:?}, + global_broadcast_strategy: {global_broadcast_strategy:?}, +}} +"# + ) + } +} + +impl CollectiveConfig { + fn new() -> Self { + Self { + num_devices: 1, + local_all_reduce_strategy: AllReduceStrategy::Tree(2), + local_reduce_strategy: ReduceStrategy::Tree(2), + local_broadcast_strategy: BroadcastStrategy::Tree(2), + + num_nodes: None, + global_address: None, + node_address: None, + data_service_port: None, + global_all_reduce_strategy: Some(AllReduceStrategy::Ring), + global_reduce_strategy: Some(ReduceStrategy::Tree(2)), + global_broadcast_strategy: Some(BroadcastStrategy::Tree(2)), + } + } + + /// Selects the number of devices (local peers) on the current node + pub fn with_num_devices(mut self, num: usize) -> Self { + self.num_devices = num; + self + } + + /// Selects an all-reduce strategy to use on the local level. + /// + /// In multi-node contexts, use of the Ring strategy in the local level may be less + /// advantageous. With multiple nodes, the global all-reduce step is enabled, and its result + /// is redistributed to all devices. + /// The Ring strategy inherently distributes the result, which in this context would not be + /// necessary. + /// + /// It is recommended to use a tree strategy locally, and a ring strategy globally. + pub fn with_local_all_reduce_strategy(mut self, strategy: AllReduceStrategy) -> Self { + self.local_all_reduce_strategy = strategy; + self + } + + /// Selects a reduce strategy to use on the local level. + pub fn with_local_reduce_strategy(mut self, strategy: ReduceStrategy) -> Self { + self.local_reduce_strategy = strategy; + self + } + + /// Selects a broadcast strategy to use on the local level. + pub fn with_local_broadcast_strategy(mut self, strategy: BroadcastStrategy) -> Self { + self.local_broadcast_strategy = strategy; + self + } + + /// Set the number of nodes in the collective + /// + /// This parameter is a global parameter and should only be set in multi-node contexts + pub fn with_num_nodes(mut self, n: u32) -> Self { + self.num_nodes = Some(n); + self + } + + /// Set the network address of the Global Collective Orchestrator + /// + /// This parameter is a global parameter and should only be set in multi-node contexts + pub fn with_global_address(mut self, addr: Address) -> Self { + self.global_address = Some(addr); + self + } + + /// Define the address for this node + /// + /// This parameter is a global parameter and should only be set in multi-node contexts + pub fn with_node_address(mut self, addr: Address) -> Self { + self.node_address = Some(addr); + self + } + + /// Selects the network port on which to expose the tensor data service + /// used for peer-to-peer tensor downloading. + /// + /// This parameter is a global parameter and should only be set in multi-node contexts + pub fn with_data_service_port(mut self, port: u16) -> Self { + self.data_service_port = Some(port); + self + } + + /// Selects an all-reduce strategy to use on the global level. + /// + /// This parameter is a global parameter and should only be set in multi-node contexts. + /// See [the local strategy](Self::with_local_all_reduce_strategy) + pub fn with_global_all_reduce_strategy(mut self, strategy: AllReduceStrategy) -> Self { + self.global_all_reduce_strategy = Some(strategy); + self + } + + /// Selects an reduce strategy to use on the global level. + /// + /// This parameter is a global parameter and should only be set in multi-node contexts. + /// See [the local strategy](Self::with_local_reduce_strategy) + pub fn with_global_reduce_strategy(mut self, strategy: ReduceStrategy) -> Self { + self.global_reduce_strategy = Some(strategy); + self + } + + /// Selects an broadcst strategy to use on the global level. + /// + /// This parameter is a global parameter and should only be set in multi-node contexts. + /// See [the local strategy](Self::with_local_broadcast_strategy) + pub fn with_global_broadcast_strategy(mut self, strategy: BroadcastStrategy) -> Self { + self.global_broadcast_strategy = Some(strategy); + self + } + + /// Returns whether the config is valid. If only some required global-level parameters are + /// defined and others are not, the config is invalid. + pub fn is_valid(&self) -> bool { + match ( + self.num_nodes, + &self.global_address, + &self.node_address, + self.data_service_port, + ) { + (None, None, None, None) => true, + (Some(_), Some(_), Some(_), Some(_)) => true, + // Global parameters have only been partially defined! + _ => false, + } + } + + /// Return the global parameters for registering in a multi-node context. + /// + /// If only some global parameters are defined, returns None. Use [is_valid](Self::is_valid) to check for + /// validity in this case. + pub(crate) fn global_register_params(&self) -> Option { + match ( + self.num_nodes, + &self.global_address, + &self.node_address, + self.data_service_port, + ) { + // Only local collective + (None, None, None, None) => None, + // Local + global collective + (Some(num_nodes), Some(global_addr), Some(node_addr), Some(data_service_port)) => { + Some(GlobalRegisterParams { + num_nodes, + global_address: global_addr.clone(), + node_address: node_addr.clone(), + data_service_port, + }) + } + // Config is invalid! + _ => None, + } + } +} + +/// Helper struct for parameters in a multi-node register operation. Either they are all defined, +/// or all not defined. Passed to the global client for registering on the global level and +/// opening the p2p tensor service. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GlobalRegisterParams { + /// The address for the connection to the global orchestrator. + pub global_address: Address, + /// The address for the connection to this node. + pub node_address: Address, + /// The port on which to open the tensor data service for peer-to-peer tensor transfers with + /// other nodes. Should match the port given in the node url. + pub data_service_port: u16, + + /// The number of nodes globally. Should be the same between different nodes + pub num_nodes: u32, +} + +/// Parameters for an all-reduce that should be the same between all devices +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct SharedAllReduceParams { + pub op: ReduceOperation, + pub local_strategy: AllReduceStrategy, + pub global_strategy: Option, +} + +/// Parameters for a reduce that should be the same between all devices +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct SharedReduceParams {} + +/// Parameters for a broadcast that should be the same between all devices +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct SharedBroadcastParams { + pub op: ReduceOperation, + pub local_strategy: BroadcastStrategy, + pub global_strategy: Option, +} + +/// Reduce can be done different ways +#[derive(Debug, PartialEq, Clone, Copy, Serialize, Deserialize)] +pub enum ReduceOperation { + Sum, + Mean, +} + +/// All reduce can be implemented with different algorithms, which all have the same result. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize)] +pub enum AllReduceStrategy { + /// One device is the "central". The other devices, "peripherals", send their tensors to the + /// central. The central does the reduction, and sends the result back to each peripheral. + Centralized, + + /// Devices are organized in a tree structure (with a given arity). Each node reduces its + /// children's tensors with its own, and sends the result to its parent. Leaf nodes will + /// simply send their tensors to their parents. + /// When the root node calculates the result, it is propagated down the tree. + Tree(u32), + + /// Devices are organized in a ring. The tensors are split into N slices, where N is the + /// number of devices participating. The slices are progressively sent around the ring until + /// every device has one fully reduced slice of the tensor. Then, the resulting slices are sent + /// around until every device has the full result. + /// See `ring.rs` for details. + Ring, +} + +/// Reduce can be implemented with different algorithms, which all have the same result. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize)] +pub enum ReduceStrategy { + /// See [all-reduce](AllReduceStrategy::Centralized) + Centralized, + + /// See [all-reduce](AllReduceStrategy::Tree) + Tree(u32), +} + +/// Broadcast can be implemented with different algorithms, which all have the same result. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize)] +pub enum BroadcastStrategy { + /// See [all-reduce](AllReduceStrategy::Centralized) + Centralized, + + /// See [all-reduce](AllReduceStrategy::Tree) + Tree(u32), +} + +/// A unique identifier for a peer in the context of collective operations. +/// They must be unique, even in multi-node contexts. +/// +/// This is like the rank in NCCL +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PeerId(u32); + +impl Display for PeerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PeerId({})", self.0) + } +} + +impl From for PeerId { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for PeerId { + fn from(value: i32) -> Self { + Self(value as u32) + } +} + +impl From for PeerId { + fn from(value: usize) -> Self { + Self(value as u32) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/base.rs new file mode 100644 index 0000000..4f58a11 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/base.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; + +/// Unique identifier for any node in the global collective. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +pub struct NodeId(u32); + +impl From for NodeId { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for NodeId { + fn from(value: usize) -> Self { + Self(value as u32) + } +} + +impl From for NodeId { + fn from(value: i32) -> Self { + Self(value as u32) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/mod.rs new file mode 100644 index 0000000..3147a86 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod node; +pub(crate) mod shared; + +#[cfg(feature = "orchestrator")] +pub mod orchestrator; +#[cfg(feature = "orchestrator")] +pub use orchestrator::*; + +mod base; +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/base.rs new file mode 100644 index 0000000..6e1470b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/base.rs @@ -0,0 +1,203 @@ +use burn_communication::Protocol; +use burn_communication::data_service::TensorDataServer; +use burn_communication::{Address, ProtocolServer, data_service::TensorDataService}; +use burn_tensor::backend::Backend; +use std::collections::HashMap; +use std::{marker::PhantomData, sync::Arc}; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; + +use crate::node::sync::SyncService; +use crate::{ + AllReduceStrategy, BroadcastStrategy, GlobalRegisterParams, NodeId, PeerId, ReduceStrategy, +}; +use crate::{ + ReduceOperation, + global::{ + node::{ + centralized::centralized_all_reduce_sum, ring::ring_all_reduce_sum, + tree::tree_all_reduce_sum, worker::GlobalClientWorker, + }, + shared::{GlobalCollectiveError, RemoteRequest, RemoteResponse}, + }, + local::server::get_collective_server_runtime, +}; + +/// Must be synchronized between all nodes for collective operations to work +pub(crate) struct NodeState { + pub node_id: NodeId, + pub nodes: HashMap, + pub num_global_devices: u32, +} + +/// A node talks to the global orchestrator as well as other nodes with a peer-to-peer service +pub(crate) struct Node +where + B: Backend, + P: Protocol, +{ + // State is written during `register` and read during other operations, + // sometimes by multiple threads (ex. syncing during an all-reduce) + state: Arc>>, + data_service: Arc>, + sync_service: Arc>, + worker: GlobalClientWorker, + _n: PhantomData

, +} + +impl Node +where + B: Backend, + P: Protocol, +{ + pub fn new(global_address: &Address, comms_server: P::Server) -> Self { + let state = Arc::new(tokio::sync::RwLock::new(None)); + let cancel_token = CancellationToken::new(); + let data_service = Arc::new(TensorDataService::new(cancel_token.clone())); + let sync_service = Arc::new(SyncService::new(state.clone())); + + let runtime = get_collective_server_runtime(); + let server = comms_server + .route_tensor_data_service(data_service.clone()) + .route("/sync", { + let sync_service = sync_service.clone(); + async move |channel: ::Channel| { + sync_service.handle_sync_connection(channel).await; + } + }) + .serve({ + let cancel_token = cancel_token.clone(); + async move { cancel_token.cancelled().await } + }); + + runtime.spawn(server); + + let worker = GlobalClientWorker::new(&runtime, cancel_token.clone(), global_address); + + Self { + state, + data_service, + sync_service, + worker, + _n: PhantomData, + } + } + + pub async fn register( + &mut self, + peers: Vec, + global_params: GlobalRegisterParams, + ) -> Result<(), GlobalCollectiveError> { + let req = RemoteRequest::Register { + node_addr: global_params.node_address, + num_nodes: global_params.num_nodes, + peers, + }; + match self.worker.request(req).await { + RemoteResponse::Register { + node_id, + nodes, + num_global_devices, + } => { + let mut state = self.state.write().await; + *state = Some(NodeState { + node_id, + nodes, + num_global_devices, + }); + } + RemoteResponse::Error(err) => { + return Err(err); + } + resp => { + log::error!("Response to a register request should be an ack, not {resp:?}"); + return Err(GlobalCollectiveError::WrongOrchestratorResponse); + } + } + + Ok(()) + } + + /// Performs an all-reduce + /// + /// Reads the NodeState + pub async fn all_reduce( + &self, + tensor: B::FloatTensorPrimitive, + strategy: AllReduceStrategy, + op: ReduceOperation, + ) -> Result { + let state = self.state.read().await; + let Some(ref state) = *state else { + return Err(GlobalCollectiveError::AllReduceBeforeRegister); + }; + let node = state.node_id; + let nodes = &state.nodes; + + let mut result = match strategy { + AllReduceStrategy::Centralized => { + centralized_all_reduce_sum( + node, + nodes, + &self.data_service, + self.sync_service.clone(), + tensor, + ) + .await? + } + AllReduceStrategy::Tree(arity) => { + tree_all_reduce_sum( + node, + nodes, + self.data_service.clone(), + self.sync_service.clone(), + tensor, + arity, + ) + .await? + } + AllReduceStrategy::Ring => { + ring_all_reduce_sum( + node, + nodes, + self.data_service.clone(), + self.sync_service.clone(), + tensor, + ) + .await? + } + }; + + if op == ReduceOperation::Mean { + result = B::float_div_scalar(result, (state.num_global_devices as f32).into()); + } + + Ok(result) + } + + pub async fn reduce( + &self, + _tensor: B::FloatTensorPrimitive, + _strategy: ReduceStrategy, + _root: PeerId, + _op: ReduceOperation, + ) -> Result, GlobalCollectiveError> { + unimplemented!("Global reduce unimplemented"); + } + + pub async fn broadcast( + &self, + _tensor: Option, + _strategy: BroadcastStrategy, + ) -> Result { + unimplemented!("Global broadcast unimplemented"); + } + + pub async fn finish(&mut self) { + let res = self.worker.close_connection().await; + if let Err(err) = res { + log::error!("Global collective client error: {err:?}"); + } + self.data_service.close().await; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/centralized.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/centralized.rs new file mode 100644 index 0000000..362e88c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/centralized.rs @@ -0,0 +1,96 @@ +use std::{collections::HashMap, sync::Arc}; + +use crate::{NodeId, global::shared::GlobalCollectiveError, node::sync::SyncService}; +use burn_communication::data_service::TensorDataService; +use burn_communication::{Address, Protocol}; +use burn_tensor::TensorMetadata; +use burn_tensor::backend::Backend; +use futures::StreamExt; +use futures::stream::FuturesUnordered; + +/// Global all-reduce, using a centralized strategy. +/// +/// Returns the resulting tensor on the same device as the input tensor +pub(crate) async fn centralized_all_reduce_sum( + node: NodeId, + nodes: &HashMap, + data_service: &Arc>, + sync_service: Arc>, + tensor: B::FloatTensorPrimitive, +) -> Result +where + B: Backend, + P: Protocol, +{ + let ids = nodes.keys().cloned().collect::>(); + let central = get_central_node(ids.clone()); + + let shape = tensor.shape(); + let device = &B::float_device(&tensor); + + let res = if central == node { + // Transfer 1: download tensors from other nodes + let mut futures = ids + .iter() + .filter(|id| **id != central) // Only non-central nodes + .map(|id| { + let address = nodes.get(id).unwrap(); + let device = device.clone(); + let data_service = data_service.clone(); + async move { + let data = data_service + .download_tensor((*address).clone(), 0.into()) + .await + .expect("Couldn't find the tensor for transfer id 0"); + B::float_from_data(data, &device) + } + }) + .collect::>(); + + // Sum all downloads async + let mut sum = tensor; + while let Some(res) = futures.next().await { + if shape != res.shape() { + return Err(GlobalCollectiveError::PeerSentIncoherentTensor); + } + sum = B::float_add(sum, res); + } + + // Transfer 2: Expose result + let other_nodes_count = ids.len() as u32 - 1; + data_service + .expose(sum.clone(), other_nodes_count, 1.into()) + .await; + + sum + } else { + // Transfer 1: Expose input + data_service.expose(tensor, 1, 0.into()).await; + + // Transfer 2: Download result + let central_addr = nodes.get(¢ral).unwrap().clone(); + let data = data_service + .download_tensor(central_addr, 1.into()) + .await + .expect("Couldn't find the tensor for transfer id 1"); + + let res = B::float_from_data(data, device); + if shape != res.shape() { + return Err(GlobalCollectiveError::PeerSentIncoherentTensor); + } + + res + }; + + // Wait for all nodes to finish + sync_service.sync().await; + + Ok(res) +} + +/// Get the central node for a centralized all-reduce +pub(crate) fn get_central_node(mut nodes: Vec) -> NodeId { + nodes.sort(); + + *nodes.first().unwrap() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/mod.rs new file mode 100644 index 0000000..38714c6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/mod.rs @@ -0,0 +1,6 @@ +pub mod base; +pub mod centralized; +pub mod ring; +pub mod sync; +pub mod tree; +pub mod worker; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/ring.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/ring.rs new file mode 100644 index 0000000..4ba806e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/ring.rs @@ -0,0 +1,216 @@ +//! Implements the collective ring all-reduce algorithm on the global level + +use core::ops::Range; +use std::{collections::HashMap, sync::Arc}; + +use crate::{ + NodeId, + global::shared::GlobalCollectiveError, + local::{get_ring_reduce_slice_ranges, get_slice_dim}, + node::sync::SyncService, +}; +use burn_communication::{Address, Protocol, data_service::TensorDataService}; +use burn_tensor::{Slice, TensorMetadata, backend::Backend}; + +// https://blog.dailydoseofds.com/p/all-reduce-and-ring-reduce-for-model + +// Example: tensors=3, slices=3 + +// phase 1 +// o->o o +// o o->o +//>o o o-> + +// o 1->o +//>o o 1-> +// 1->o o + +// o 1 2 +// 2 o 1 +// 1 2 o + +// phase 2 +//>o 1 2-> +// 2->o 1 +// 1 2->o + +// 2->1 2 +// 2 2->1 +//>1 2 2-> + +// 2 2 2 +// 2 2 2 +// 2 2 2 + +/// Ring all-reduce algorithm with summation +/// +/// * `node` - The id of the current node +/// * `nodes` - Map of all nodes in the operation +/// * `data_service` - The data service handles peer-to-peer tensor transfers +/// * `sync_service` - The sync service handles syncing with peers +/// * `tensor` - The tensor to reduce. At least one dimension size must be greater than the number +/// of nodes +pub(crate) async fn ring_all_reduce_sum( + node: NodeId, + nodes: &HashMap, + data_service: Arc>, + sync_service: Arc>, + tensor: B::FloatTensorPrimitive, +) -> Result +where + B: Backend, + P: Protocol, +{ + let shape = tensor.shape(); + + let device = &B::float_device(&tensor); + // Slice tensors in N parts, N is node count + let slice_dim = get_slice_dim(&shape); + if shape[slice_dim] < nodes.len() { + return Err(GlobalCollectiveError::RingReduceImpossible); + } + + let ring = get_ring_topology(nodes.keys().cloned().collect::>()); + let slice_ranges = get_ring_reduce_slice_ranges(shape[slice_dim], ring.len()); + let mut slices = slice_tensor::(tensor, slice_dim, slice_ranges); + + let mut send_slice_idx = ring + .iter() + .position(|id| *id == node) + .expect("Node is in ring"); + let prev_node_idx = (send_slice_idx + ring.len() - 1) % ring.len(); // +ring.len for overflow + let prev_node = nodes.get(&ring[prev_node_idx]).unwrap(); + let mut transfer_counter: u64 = 0; + + // Phase 1: add + do_cycles::( + &mut slices, + &mut transfer_counter, + &mut send_slice_idx, + true, + prev_node.clone(), + &data_service, + device, + ) + .await?; + + // Phase 2: replace + do_cycles::( + &mut slices, + &mut transfer_counter, + &mut send_slice_idx, + false, + prev_node.clone(), + &data_service, + device, + ) + .await?; + + // Wait for all nodes to finish + sync_service.sync().await; + + // merge slices + Ok(B::float_cat(slices, slice_dim)) +} + +/// Do N-1 cycles of ring-reduce +/// +/// * `slices` - Slices of the original tensor, len equal to node count +/// * `transfer_counter` - counter for each step (one send one receive) +/// * `send_slice_idx` - counter for the index of each slice to send +/// * `is_phase_one` - In phase 1, the tensors are aggregated. Otherwise, they are overridden +/// * `data_service` - TensorDataService for peer-to-peer tensor transfers +/// * `device` - The device on which all local tensors are stored. Should match `slices` +async fn do_cycles( + slices: &mut [B::FloatTensorPrimitive], + transfer_counter: &mut u64, + send_slice_idx: &mut usize, + is_phase_one: bool, + prev_node: Address, + data_service: &Arc>, + device: &B::Device, +) -> Result<(), GlobalCollectiveError> +where + B: Backend, + P: Protocol, +{ + let slice_count = slices.len(); + for _ in 0..(slice_count - 1) { + let transfer_id = (*transfer_counter).into(); + // +slice_count to avoid overflow + let recv_slice_idx = (*send_slice_idx + slice_count - 1) % slice_count; + let slice_send = slices[*send_slice_idx].clone(); + + let upload = { + let data_service = data_service.clone(); + tokio::spawn(async move { + data_service + .expose(slice_send.clone(), 1, transfer_id) + .await + }) + }; + let download = { + let data_client = data_service.clone(); + let next_node = prev_node.clone(); + tokio::spawn(async move { data_client.download_tensor(next_node, transfer_id).await }) + }; + + upload.await.unwrap(); + let download = download.await.unwrap(); + if is_phase_one { + let download = download.expect("Peer closed download connection"); + let tensor = B::float_from_data(download, device); + slices[recv_slice_idx] = B::float_add(slices[recv_slice_idx].clone(), tensor); + } else { + let tensor = B::float_from_data(download.unwrap(), device); + let old_shape = slices[recv_slice_idx].shape(); + if old_shape != tensor.shape() { + return Err(GlobalCollectiveError::PeerSentIncoherentTensor); + } + slices[recv_slice_idx] = tensor; + } + + // Move slice index + *send_slice_idx = recv_slice_idx; + *transfer_counter += 1; + } + + Ok(()) +} + +/// But a tensor into even slices across a dimension +/// +/// * `tensor` - the tensor to slice +/// * `slice_dim` - the dimension to slice across +/// * `slice_ranges` - The ranges of indices on `slice_dim` to use when slicing the tensor +fn slice_tensor( + tensor: B::FloatTensorPrimitive, + slice_dim: usize, + slice_ranges: Vec>, +) -> Vec { + let shape = tensor.shape(); + // full range across all dims as Slice + let full_range = shape + .iter() + .map(|dim| Slice::from(0..*dim)) + .collect::>(); + + // Slice tensors + let mut slices = vec![]; + for range in &slice_ranges { + let mut all_ranges = full_range.clone(); + all_ranges[slice_dim] = Slice::from(range.clone()); + let slice = B::float_slice(tensor.clone(), &all_ranges); + slices.push(slice); + } + + slices +} + +/// Get the ring topology +fn get_ring_topology(mut nodes: Vec) -> Vec { + // This ordering could be more sophisticated, using node proximities etc + nodes.sort(); + + nodes +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/sync.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/sync.rs new file mode 100644 index 0000000..abbcf65 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/sync.rs @@ -0,0 +1,100 @@ +use std::{ + marker::PhantomData, + sync::{Arc, Mutex}, + vec, +}; + +use burn_communication::{CommunicationChannel, Message, Protocol, ProtocolClient}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{Notify, RwLock}; + +use crate::{NodeId, node::base::NodeState}; + +/// Handles the status of sync requests from other nodes +pub(crate) struct SyncService { + /// Current node's state, shared with the thread that does aggregations + node_state: Arc>>, + /// The number of peers that have requested to sync with us since the last successful sync. + syncing_peers: Mutex>, + /// Notification on each incoming sync request + sync_notif: Notify, + + _p: PhantomData

, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SyncRequest(NodeId); + +impl SyncService

{ + pub fn new(node_state: Arc>>) -> Self { + Self { + node_state, + syncing_peers: Mutex::new(vec![]), + sync_notif: Notify::new(), + _p: PhantomData, + } + } + + fn add_syncing_peer(&self, peer: NodeId) { + let mut syncing_peers = self.syncing_peers.lock().unwrap(); + syncing_peers.push(peer); + } + + /// Sync with all peers. + pub async fn sync(&self) { + // we can't sync while we register + let node_state = self.node_state.read().await; + let node_state = node_state + .as_ref() + .expect("Trying to sync a node before having registered to the orchestrator"); + + // this peer is syncing + self.add_syncing_peer(node_state.node_id); + for (id, addr) in &node_state.nodes { + if *id == node_state.node_id { + continue; + } + + let mut connection = P::Client::connect(addr.clone(), "sync") + .await + .expect("Couldn't connect to peer for sync"); + let msg = SyncRequest(node_state.node_id); + let sync_bytes = rmp_serde::to_vec(&msg).unwrap(); + connection + .send(Message::new(sync_bytes.into())) + .await + .expect("Peer closed connection unexpectedly"); + } + loop { + { + // compare currently synced peers with list of all nodes + let mut syncing_peers = self.syncing_peers.lock().unwrap().to_vec(); + syncing_peers.sort(); + + let mut all_node_ids = node_state.nodes.keys().cloned().collect::>(); + all_node_ids.sort(); + + if syncing_peers == all_node_ids { + // all nodes have synced + syncing_peers.clear(); + return; + } + } + // Wait for the next sync to come in + self.sync_notif.notified().await + } + } + + pub async fn handle_sync_connection(&self, mut channel: C) { + let msg = channel.recv().await.unwrap(); + let Some(msg) = msg else { + return; + }; + + let msg = rmp_serde::from_slice::(&msg.data).unwrap(); + + self.add_syncing_peer(msg.0); + + self.sync_notif.notify_waiters(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/tree.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/tree.rs new file mode 100644 index 0000000..a4c4488 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/tree.rs @@ -0,0 +1,198 @@ +use std::{collections::HashMap, sync::Arc}; + +use crate::{NodeId, global::shared::GlobalCollectiveError, node::sync::SyncService}; +use burn_communication::{Address, Protocol, data_service::TensorDataService}; +use burn_tensor::{TensorMetadata, backend::Backend}; +use futures::{StreamExt, stream::FuturesUnordered}; + +struct TreeTopology { + parents: HashMap, + children: HashMap>, +} + +/// Global all-reduce, using a b-tree strategy. +/// +/// Returns the resulting tensor on the same device as the input tensor +pub(crate) async fn tree_all_reduce_sum( + node: NodeId, + nodes: &HashMap, + data_service: Arc>, + sync_service: Arc>, + tensor: B::FloatTensorPrimitive, + arity: u32, +) -> Result +where + B: Backend, + P: Protocol, +{ + let shape = tensor.shape(); + let device = &B::float_device(&tensor); + + // Topology could be cached based on (nodes.keys().cloned(), arity) + let strategy = get_tree_topology(nodes.keys().cloned().collect::>(), arity); + + // Transfer 1: Download and sum tensors from children + let mut result = tensor; + + if let Some(children) = strategy.children.get(&node) { + let mut downloads = children + .iter() + .map(|child| { + let child_addr = nodes.get(child).unwrap().clone(); + let data_service = data_service.clone(); + async move { + let data = data_service + .download_tensor(child_addr.clone(), 0.into()) + .await + .ok_or(GlobalCollectiveError::PeerLost(*child))?; + Ok::(B::float_from_data( + data, device, + )) + } + }) + .collect::>(); + + for _ in children { + let res = downloads.next().await.unwrap().unwrap(); + if res.shape() != shape { + return Err(GlobalCollectiveError::PeerSentIncoherentTensor); + } + result = B::float_add(result, res); + } + } + + // Transfer 2: Expose result to parent and download final result if not root + if let Some(parent) = strategy.parents.get(&node) { + data_service.expose(result.clone(), 1, 0.into()).await; + + let parent_addr = nodes.get(parent).unwrap().clone(); + + let data = data_service + .download_tensor(parent_addr.clone(), 1.into()) + .await + .ok_or(GlobalCollectiveError::PeerLost(*parent))?; + + let parent_tensor = B::float_from_data(data, device); + if parent_tensor.shape() != shape { + return Err(GlobalCollectiveError::PeerSentIncoherentTensor); + } + result = parent_tensor; + } + + // Transfer 3: Expose final result to children (if any) + if let Some(children) = strategy.children.get(&node) + && !children.is_empty() + { + data_service + .expose(result.clone(), children.len() as u32, 1.into()) + .await; + } + + // Final barrier + sync_service.sync().await; + + Ok(result) +} + +/// Get the tree topology. +/// +/// * `nodes` - List of node ids. Order doesn't matter. Nodes must be unique. +fn get_tree_topology(mut nodes: Vec, arity: u32) -> TreeTopology { + assert!(arity >= 1, "Arity must be ≥ 1"); + + nodes.sort(); // Sort + + let n = nodes.len(); + let k = arity as usize; + + let mut parents: HashMap<_, _> = HashMap::with_capacity(n); + let mut children: HashMap<_, _> = HashMap::with_capacity(n); + + for (i, &parent_id) in nodes.iter().enumerate() { + // compute the window [first_child, last_child) + let first = i * k + 1; + if first < n { + let last = usize::min(first + k, n); + let mut ch = Vec::with_capacity(last - first); + for &child_id in &nodes[first..last] { + parents.insert(child_id, parent_id); + ch.push(child_id); + } + children.insert(parent_id, ch); + } else { + // leaf‐node: no children + children.insert(parent_id, Vec::new()); + } + } + + TreeTopology { parents, children } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Test the tree topology algorithm with arity 2 and 7 nodes + #[test] + fn test_get_tree_topology_arity2_size7() { + let mut nodes = vec![]; + for i in 0..7 { + nodes.push(i.into()); + } + + let topology = get_tree_topology(nodes, 2); + + // Root is 0, so it should have no parent + assert!(!topology.parents.contains_key(&0.into())); + + // Parents: + // Node 1 and 2 → parent 0 + // Node 3 and 4 → parent 1 + // Node 5 and 6 → parent 2 + let expected_parents = [ + (1.into(), 0.into()), + (2.into(), 0.into()), + (3.into(), 1.into()), + (4.into(), 1.into()), + (5.into(), 2.into()), + (6.into(), 2.into()), + ]; + for (child, parent) in &expected_parents { + assert_eq!( + topology.parents.get(child), + Some(parent), + "wrong parent for {child:?}" + ); + } + // There should be exactly 6 entries in parents + assert_eq!(topology.parents.len(), expected_parents.len()); + + // Children: + // 0 → [1, 2] + // 1 → [3, 4] + // 2 → [5, 6] + // 3,4,5,6 → [] + assert_eq!( + topology.children.get(&0.into()), + Some(&vec![1.into(), 2.into()]) + ); + assert_eq!( + topology.children.get(&1.into()), + Some(&vec![3.into(), 4.into()]) + ); + assert_eq!( + topology.children.get(&2.into()), + Some(&vec![5.into(), 6.into()]) + ); + // Leaves + for leaf in 3..7 { + assert_eq!( + topology.children.get(&leaf.into()), + Some(&Vec::new()), + "leaf {leaf:?} should have no children" + ); + } + // Ensure we have exactly 7 entries in children + assert_eq!(topology.children.len(), 7); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/worker.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/worker.rs new file mode 100644 index 0000000..84ed67e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/node/worker.rs @@ -0,0 +1,297 @@ +use std::{collections::HashMap, marker::PhantomData, sync::Arc, time::Duration}; + +use burn_communication::{Address, CommunicationChannel, Message, ProtocolClient}; +use tokio::{ + runtime::Runtime, + sync::{ + Mutex, + mpsc::{Receiver, Sender}, + }, + task::JoinHandle, +}; +use tokio_util::sync::CancellationToken; + +use crate::global::shared::{ + CollectiveMessage, CollectiveMessageResponse, GlobalCollectiveError, RemoteRequest, + RemoteResponse, RequestId, SessionId, +}; + +/// Worker that handles communication with the orchestrator for global collective operations. +pub(crate) struct GlobalClientWorker { + handle: Option>>, + cancel_token: CancellationToken, + request_sender: Sender, + _phantom_data: PhantomData

, +} + +// Rename +struct GlobalClientWorkerState { + requests: HashMap>, +} + +impl GlobalClientWorkerState { + fn new() -> Self { + Self { + requests: HashMap::new(), + } + } +} + +#[derive(Debug)] +pub(crate) struct ClientRequest { + pub request: RemoteRequest, + pub callback: Sender, +} + +impl ClientRequest { + pub(crate) fn new(request: RemoteRequest, callback: Sender) -> Self { + Self { request, callback } + } +} + +impl GlobalClientWorker { + /// Create a new global client worker and start the tasks. + pub(crate) fn new( + runtime: &Runtime, + cancel_token: CancellationToken, + global_address: &Address, + ) -> Self { + let (request_sender, request_recv) = tokio::sync::mpsc::channel::(10); + + let state = Arc::new(Mutex::new(GlobalClientWorkerState::new())); + + let handle = runtime.spawn(Self::start( + state, + cancel_token.clone(), + global_address.clone(), + request_recv, + )); + + Self { + handle: Some(handle), + cancel_token, + request_sender, + _phantom_data: PhantomData, + } + } + + /// Start the global client tasks + async fn start( + state: Arc>, + cancel_token: CancellationToken, + global_address: Address, + request_recv: Receiver, + ) -> Result<(), GlobalCollectiveError> { + // Init the connection. + let (request, response) = Self::init_connection(&global_address).await?; + + // Websocket async worker loading responses from the server. + let response_handle = tokio::spawn(Self::response_loader( + state.clone(), + response, + cancel_token.clone(), + )); + + // Channel async worker sending operations to the server. + let request_handle = tokio::spawn(Self::request_sender( + request_recv, + state, + request, + cancel_token.clone(), + )); + + if let Err(e) = response_handle.await { + log::error!("Response handler failed: {e:?}"); + } + if let Err(e) = request_handle.await { + log::error!("Request handler failed: {e:?}"); + } + + Ok(()) + } + + async fn init_connection( + address: &Address, + ) -> Result<(C::Channel, C::Channel), GlobalCollectiveError> { + let session_id = SessionId::new(); + + let stream_request = tokio::spawn(Self::connect_with_retry( + address.clone(), + "request", + std::time::Duration::from_secs(1), + None, + session_id, + )); + let stream_response = tokio::spawn(Self::connect_with_retry( + address.clone(), + "response", + std::time::Duration::from_secs(1), + None, + session_id, + )); + + let Ok(Some(request)) = stream_request.await else { + return Err(GlobalCollectiveError::OrchestratorUnreachable); + }; + let Ok(Some(response)) = stream_response.await else { + return Err(GlobalCollectiveError::OrchestratorUnreachable); + }; + + Ok((request, response)) + } + + /// Connect with websocket with retries. + async fn connect_with_retry( + address: Address, + route: &str, + retry_pause: Duration, + retry_max: Option, + session_id: SessionId, + ) -> Option { + let mut retries = 0; + loop { + if let Some(max) = retry_max + && retries >= max + { + log::warn!("Failed to connect to {address} after {max} retries."); + return None; + } + + // Try to connect to the request address. + println!("Connecting to {address} ..."); + let result = C::connect(address.clone(), route).await; + + if let Some(mut stream) = result { + let init_msg = CollectiveMessage::Init(session_id); + let bytes: bytes::Bytes = rmp_serde::to_vec(&init_msg).unwrap().into(); + stream + .send(Message::new(bytes)) + .await + .expect("Can send the init message on the websocket."); + return Some(stream); + } + + println!("Failed to connect to {address}, retrying... Attempt #{retries}"); + tokio::time::sleep(retry_pause).await; + retries += 1; + } + } + + /// Unregister the worker and close the connection. + pub(crate) async fn close_connection(&mut self) -> Result<(), GlobalCollectiveError> { + if let Some(handle) = self.handle.take() { + // Un-register from server + let req = RemoteRequest::Finish; + let resp = self.request(req).await; + if resp != RemoteResponse::FinishAck { + log::error!("Requested to finish, did not get FinishAck; got {resp:?}"); + return Err(GlobalCollectiveError::WrongOrchestratorResponse); + } + + self.cancel_token.cancel(); + + if let Err(e) = handle.await.unwrap() { + log::error!("Connection error {e:?}"); + } + } + + Ok(()) + } + + async fn response_loader( + state: Arc>, + mut stream_response: C::Channel, + cancel_token: CancellationToken, + ) { + loop { + tokio::select! { + // Check if the cancel token is cancelled + _ = cancel_token.cancelled() => { + break; + } + // .. Or get a message from the websocket + response = stream_response.recv() => { + match response { + Err(err) => { + log::error!("Error receiving message from websocket: {err:?}"); + break; + } + Ok(response) => { + let Some(response) = response else { + log::warn!("Closed connection"); + break; + }; + + let response: CollectiveMessageResponse = rmp_serde::from_slice(&response.data) + .expect("Can deserialize messages from the websocket."); + let state_resp = state.lock().await; + let response_callback = state_resp + .requests + .get(&response.request_id) + .expect("Got a response to an unknown request"); + response_callback.send(response.content).await.unwrap(); + } + } + } + } + } + + log::info!("Worker closing connection"); + stream_response + .close() + .await + .expect("Can close the websocket stream."); + } + + async fn request_sender( + mut request_recv: Receiver, + worker: Arc>, + mut stream_request: C::Channel, + cancel_token: CancellationToken, + ) { + loop { + tokio::select! { + _ = cancel_token.cancelled() => { + break; + }, + request = request_recv.recv() => { + let Some(request) = request else { + continue; + }; + + let id = RequestId::new(); + + // Register the callback if there is one + { + let mut state = worker.lock().await; + state.requests.insert(id, request.callback); + } + + let request = CollectiveMessage::Request(id, request.request); + + let bytes = rmp_serde::to_vec::(&request) + .expect("Can serialize tasks to bytes.") + .into(); + stream_request + .send(Message::new(bytes)) + .await + .expect("Can send the message on the websocket."); + } + } + } + + log::info!("Worker closing connection"); + stream_request + .close() + .await + .expect("Can send the close message on the websocket."); + } + + pub(crate) async fn request(&self, req: RemoteRequest) -> RemoteResponse { + let (callback, mut response_recv) = tokio::sync::mpsc::channel::(10); + let client_req = ClientRequest::new(req, callback); + self.request_sender.send(client_req).await.unwrap(); + + response_recv.recv().await.unwrap() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/base.rs new file mode 100644 index 0000000..d6a7c24 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/base.rs @@ -0,0 +1,138 @@ +use std::fmt::Debug; +use std::sync::Arc; +use tokio::sync::Mutex; + +use crate::global::{ + orchestrator::state::GlobalCollectiveState, + shared::{CollectiveMessage, GlobalCollectiveError}, +}; +use burn_communication::{ + CommunicationChannel, Message, ProtocolServer, util::os_shutdown_signal, websocket::WsServer, +}; + +/// The global collective state manages collective operations on the global level +#[derive(Clone)] +pub(crate) struct GlobalOrchestrator { + state: Arc>, +} + +impl GlobalOrchestrator { + /// Starts the comms server with two routes: "/request" and "/response" + pub(crate) async fn start( + shutdown_signal: F, + comms_server: S, + ) -> Result<(), GlobalCollectiveError> + where + F: Future + Send + 'static, + { + let state = GlobalCollectiveState::new(); + let server = Self { + state: Arc::new(tokio::sync::Mutex::new(state)), + }; + + comms_server + .route("/response", { + let server = server.clone(); + async move |socket| { + if let Err(err) = server.handle_socket_response::(socket).await { + log::error!("[Response Handler] Error: {err:?}") + } + } + }) + .route("/request", { + let server = server.clone(); + async move |socket| { + if let Err(err) = server.handle_socket_request::(socket).await { + log::error!("[Request Handler] Error: {err:?}") + } + } + }) + .serve(shutdown_signal) + .await + .map_err(|err| GlobalCollectiveError::Server(format!("{err:?}")))?; + + Ok(()) + } + + async fn handle_socket_response( + self, + mut stream: S::Channel, + ) -> Result<(), GlobalCollectiveError> { + log::info!("[Response Handler] On new connection."); + + let msg = stream + .recv() + .await + .map_err(|err| GlobalCollectiveError::Server(format!("{err:?}")))?; + let Some(msg) = msg else { + log::warn!("Response socket closed early!"); + return Ok(()); + }; + + let msg = rmp_serde::from_slice::(&msg.data) + .map_err(|_| GlobalCollectiveError::InvalidMessage)?; + + let CollectiveMessage::Init(id) = msg else { + return Err(GlobalCollectiveError::FirstMsgNotInit); + }; + + let mut receiver = { + let mut state = self.state.lock().await; + state.get_session_responder(id) + }; + + while let Some(response) = receiver.recv().await { + let bytes = rmp_serde::to_vec(&response).unwrap(); + + stream.send(Message::new(bytes.into())).await?; + } + + log::info!("[Response Handler] Closing connection."); + Ok(()) + } + + async fn handle_socket_request( + self, + mut stream: S::Channel, + ) -> Result<(), GlobalCollectiveError> { + log::info!("[Request Handler] On new connection."); + + let mut session_id = None; + + loop { + let packet = stream.recv().await?; + let Some(msg) = packet else { + log::info!("Peer closed the connection"); + break; + }; + + let mut state = self.state.lock().await; + + let msg = rmp_serde::from_slice::(&msg.data) + .map_err(|_| GlobalCollectiveError::InvalidMessage)?; + match msg { + CollectiveMessage::Init(id) => { + state.init_session(id); + session_id = Some(id); + } + CollectiveMessage::Request(request_id, remote_request) => { + let session_id = session_id.ok_or(GlobalCollectiveError::FirstMsgNotInit)?; + state + .process_request(session_id, request_id, remote_request) + .await; + } + } + } + + Ok(()) + } +} + +/// Start a global orchestrator with WebSocket on the given port +pub async fn start_global_orchestrator(port: u16) { + let server = WsServer::new(port); + let res = GlobalOrchestrator::start(os_shutdown_signal(), server).await; + if let Err(err) = res { + log::error!("Global Collective Orchestrator error: {err:?}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/mod.rs new file mode 100644 index 0000000..bf619fe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod base; +pub(crate) mod state; + +pub use base::start_global_orchestrator; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/state.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/state.rs new file mode 100644 index 0000000..5d98b93 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/orchestrator/state.rs @@ -0,0 +1,219 @@ +use crate::{ + PeerId, + global::{ + NodeId, + shared::{ + CollectiveMessageResponse, GlobalCollectiveError, RemoteRequest, RemoteResponse, + RequestId, SessionId, + }, + }, +}; +use burn_communication::Address; +use std::collections::HashMap; +use tokio::sync::mpsc::{Receiver, Sender}; + +pub(crate) struct Session { + response_sender: Sender, + response_receiver: Option>, +} + +impl Session { + fn new() -> Self { + let (response_sender, recv) = tokio::sync::mpsc::channel::(1); + Self { + response_sender, + response_receiver: Some(recv), + } + } + + async fn respond(&mut self, response: CollectiveMessageResponse) { + self.response_sender.send(response).await.unwrap(); + } +} + +pub(crate) struct GlobalCollectiveState { + /// The ids passed to each register so far, and their addresses + registered_nodes: HashMap, + /// Address for each node + node_addresses: HashMap, + /// Peer on each node + node_peers: HashMap>, + + /// How many total nodes for the current register operation, as defined by the first caller + cur_num_nodes: Option, + /// How many peers have registered total + num_global_peers: u32, + + register_requests: Vec<(SessionId, RequestId, NodeId)>, + + sessions: HashMap, +} + +impl GlobalCollectiveState { + pub fn new() -> Self { + Self { + registered_nodes: HashMap::new(), + node_addresses: HashMap::new(), + node_peers: HashMap::new(), + cur_num_nodes: None, + num_global_peers: 0, + register_requests: Vec::new(), + sessions: HashMap::new(), + } + } + + pub(crate) fn init_session(&mut self, id: SessionId) { + if self.sessions.contains_key(&id) { + return; + } + self.sessions.insert(id, Session::new()); + } + + /// Create the session with given id if necessary, and get the response receiver + pub(crate) fn get_session_responder( + &mut self, + id: SessionId, + ) -> Receiver { + self.init_session(id); + let session = self.sessions.get_mut(&id).unwrap(); + let response_recv = session.response_receiver.take(); + + response_recv.unwrap() + } + + pub(crate) async fn respond( + &mut self, + session_id: SessionId, + response: CollectiveMessageResponse, + ) { + let session = self.sessions.get_mut(&session_id).unwrap(); + session.respond(response).await; + } + + /// Process an incoming node's request + pub(crate) async fn process_request( + &mut self, + session_id: SessionId, + request_id: RequestId, + request: RemoteRequest, + ) { + if let Err(err) = match request { + RemoteRequest::Register { + node_addr, + num_nodes, + peers, + } => { + self.register(session_id, request_id, node_addr, num_nodes, peers) + .await + } + RemoteRequest::Finish => self.finish(session_id, request_id).await, + } { + // Error occurred, send it as response + let content = RemoteResponse::Error(err); + self.respond( + session_id, + CollectiveMessageResponse { + request_id, + content, + }, + ) + .await; + } + } + + /// Un-register a node. Any pending requests will be cancelled, returning error responses. + async fn finish( + &mut self, + session_id: SessionId, + request_id: RequestId, + ) -> Result<(), GlobalCollectiveError> { + let node_id = self + .registered_nodes + .remove(&session_id) + .ok_or(GlobalCollectiveError::NotRegisteredOnFinish)?; + self.node_addresses.remove(&node_id); + self.node_peers.remove(&node_id); + self.num_global_peers = 0; + + let mut register_requests = vec![]; + core::mem::swap(&mut register_requests, &mut self.register_requests); + for (session, req, node_id) in register_requests { + if session == session_id { + // Send a response if we are finishing a session with a pending register request + let content = RemoteResponse::Error(GlobalCollectiveError::PendingRegisterOnFinish); + let response = CollectiveMessageResponse { + request_id: req, + content, + }; + self.respond(session_id, response).await; + } else { + // keep the register request + self.register_requests.push((session, req, node_id)); + } + } + + self.respond( + session_id, + CollectiveMessageResponse { + request_id, + content: RemoteResponse::FinishAck, + }, + ) + .await; + + Ok(()) + } + + async fn register( + &mut self, + session_id: SessionId, + request_id: RequestId, + node_addr: Address, + num_nodes: u32, + peers: Vec, + ) -> Result<(), GlobalCollectiveError> { + match &self.cur_num_nodes { + Some(cur_num_nodes) => { + if *cur_num_nodes != num_nodes { + return Err(GlobalCollectiveError::RegisterParamsMismatch); + } + } + None => { + self.cur_num_nodes = Some(num_nodes); + } + } + + self.num_global_peers += peers.len() as u32; + + let node_id: NodeId = self.registered_nodes.len().into(); + self.registered_nodes.insert(session_id, node_id); + if self.node_addresses.values().any(|addr| node_addr == *addr) { + return Err(GlobalCollectiveError::DoubleRegister); + } + self.node_addresses.insert(node_id, node_addr); + self.node_peers.insert(node_id, peers); + + self.register_requests + .push((session_id, request_id, node_id)); + + if self.registered_nodes.len() == num_nodes as usize { + let mut callbacks = vec![]; + core::mem::swap(&mut callbacks, &mut self.register_requests); + + for (session, request, node_id) in callbacks { + let content = RemoteResponse::Register { + node_id, + nodes: self.node_addresses.clone(), + num_global_devices: self.num_global_peers, + }; + let resp = CollectiveMessageResponse { + request_id: request, + content, + }; + self.respond(session, resp).await; + } + } + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/shared.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/shared.rs new file mode 100644 index 0000000..8dcc8b2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/global/shared.rs @@ -0,0 +1,132 @@ +use std::{collections::HashMap, sync::atomic::AtomicU32}; + +use crate::{NodeId, PeerId}; +use burn_communication::{Address, CommunicationError}; +use burn_std::id::IdGenerator; +use serde::{Deserialize, Serialize}; + +/// A unique identifier for each request made to a global orchestrator +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub(crate) struct RequestId(u32); + +static REQ_ID_COUNTER: AtomicU32 = AtomicU32::new(0); +impl RequestId { + pub(crate) fn new() -> Self { + let id = REQ_ID_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Self(id) + } +} + +impl Default for RequestId { + fn default() -> Self { + Self::new() + } +} + +/// Unique identifier that can represent a session between a node and the orchestrator. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize, PartialOrd, Ord)] +pub(crate) struct SessionId { + id: u64, +} + +impl SessionId { + /// Create a new [session id](SessionId). + pub(crate) fn new() -> Self { + Self { + id: IdGenerator::generate(), + } + } +} + +/// Requests sent from the client +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) enum CollectiveMessage { + Init(SessionId), + Request(RequestId, RemoteRequest), +} + +/// Responses sent to the client +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct CollectiveMessageResponse { + pub request_id: RequestId, + pub content: RemoteResponse, +} + +/// Requests made from a client to a server. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) enum RemoteRequest { + // Register a node + Register { + /// Endpoint for this node + node_addr: Address, + /// Number of total nodes + num_nodes: u32, + /// List of peers on this node + peers: Vec, + }, + + /// Unregister node + Finish, +} + +/// Responses for each server request +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) enum RemoteResponse { + /// Response to a register request + Register { + /// The orchestrator gives the node its id + node_id: NodeId, + /// All the nodes in the collective: including self + nodes: HashMap, + /// How many devices exist globally? For averaging values + num_global_devices: u32, + }, + + // Finish + FinishAck, + + // There was a server-side error + Error(GlobalCollectiveError), +} + +/// Errors that occur during collective operations on the global level +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum GlobalCollectiveError { + /// Operations that can't be done before registering + AllReduceBeforeRegister, + /// Ring all-reduce can't be done if all tensor dimensions are smaller than the number of nodes. + RingReduceImpossible, + + /// Either a node has unregistered twice, or a Finish has been called before a Register + NotRegisteredOnFinish, + /// Finish has been called before a Register operation was finished + PendingRegisterOnFinish, + /// Trying to register a different way than is currently being done + RegisterParamsMismatch, + /// Trying to register while already registered + DoubleRegister, + /// Trying to aggregate a different way than is currently being done + AllReduceParamsMismatch, + + /// First message on socket should be Message::Init + FirstMsgNotInit, + /// Messages should be rmp_serde serialized `Message` types + InvalidMessage, + /// A peer behaved unexpectedly + PeerSentIncoherentTensor, + /// Tried to download from a peer, but the peer closed or lost the connection + PeerLost(NodeId), + /// Error from the coordinator + Server(String), + + /// The node received an invalid response + WrongOrchestratorResponse, + /// Node couldn't connect to coordinator + OrchestratorUnreachable, +} + +impl From for GlobalCollectiveError { + fn from(err: E) -> Self { + Self::Server(format!("{err:?}")) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/lib.rs new file mode 100644 index 0000000..a299cf6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/lib.rs @@ -0,0 +1,21 @@ +mod global; +pub use global::*; + +mod config; +pub use config::*; + +mod api; +pub use api::*; + +mod local; + +#[cfg(all( + test, + any( + feature = "test-ndarray", + feature = "test-wgpu", + feature = "test-cuda", + feature = "test-metal" + ) +))] +mod tests; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/base.rs new file mode 100644 index 0000000..53ac957 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/base.rs @@ -0,0 +1,118 @@ +use crate::local::tensor_map::{CollectiveTensorMap, get_peer_devices}; +use crate::{ + AllReduceStrategy, CollectiveConfig, CollectiveError, ReduceOperation, + local::{ + all_reduce_sum_centralized, all_reduce_sum_ring, all_reduce_sum_tree, + broadcast_centralized, broadcast_tree, reduce_sum_centralized, reduce_sum_tree, + }, + node::base::Node, +}; +use burn_communication::Protocol; +use burn_tensor::backend::Backend; + +#[cfg(feature = "tracing")] +use tracing::Instrument; + +/// Perform an all-reduce with no multi-node operations (global ops) +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors, config)) +)] +pub(crate) async fn all_reduce_local_only( + tensors: CollectiveTensorMap, + op: ReduceOperation, + config: &CollectiveConfig, +) -> Result, CollectiveError> { + let local_strategy = &config.local_all_reduce_strategy; + + let mut reduced_tensors = match local_strategy { + AllReduceStrategy::Centralized => all_reduce_sum_centralized::(tensors), + AllReduceStrategy::Tree(arity) => all_reduce_sum_tree::(tensors, *arity), + AllReduceStrategy::Ring => all_reduce_sum_ring::(tensors), + }; + + if op == ReduceOperation::Mean { + #[cfg(feature = "tracing")] + let _span = tracing::info_span!("mean_reduction").entered(); + + // Apply mean division + let div = (reduced_tensors.len() as f32).into(); + + reduced_tensors = reduced_tensors + .into_iter() + .map(|(id, t)| (id, B::float_div_scalar(t, div))) + .collect(); + } + Ok(reduced_tensors) +} + +/// Do an all-reduce in a multi-node context +/// +/// With Tree and Centralized strategies, the all-reduce is split between a +/// reduce (all tensors are reduced to one device), and a broadcast (the result is sent to all +/// other devices). The all-reduce on the global level is done between both steps. +/// Due to the nature of the Ring strategy, this separation can't be done. +/// +/// For the Ring strategy, this isn't possible, because it is more like a +/// reduce-scatter plus an all-gather, so using a Ring strategy locally in a multi-node +/// setup may be unadvantageous. +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors, config, global_client)) +)] +pub(crate) async fn all_reduce_with_global( + tensors: CollectiveTensorMap, + op: ReduceOperation, + config: &CollectiveConfig, + global_client: &mut Node, +) -> Result, CollectiveError> { + let peer_devices = get_peer_devices::(&tensors); + + // For Centralized and Tree, we only need to do a reduce here, we'll do a broadcast later + let main_device = *tensors.keys().next().unwrap(); + + let mut main_tensor = match config.local_all_reduce_strategy { + AllReduceStrategy::Centralized => reduce_sum_centralized::(tensors, &main_device), + AllReduceStrategy::Tree(arity) => reduce_sum_tree::(tensors, &main_device, arity), + AllReduceStrategy::Ring => all_reduce_sum_ring::(tensors) + .remove(&main_device) + .unwrap(), + }; + + // Do aggregation on global level with the main tensor + main_tensor = { + let fut = async { + let global_strategy = config + .global_all_reduce_strategy + .expect("global_all_reduce_strategy must be set"); + + global_client + .all_reduce(main_tensor, global_strategy, op) + .await + }; + #[cfg(feature = "tracing")] + { + fut.instrument(tracing::info_span!("global_all_reduce")) + } + #[cfg(not(feature = "tracing"))] + { + fut + } + } + .await + .map_err(CollectiveError::Global)?; + + // Broadcast result to all devices + let tensors = match config.local_all_reduce_strategy { + AllReduceStrategy::Tree(arity) => { + broadcast_tree::(peer_devices, main_device, main_tensor, arity) + } + // If we chose the ring strategy and we must still broadcast the global result, + // we use the centralized strategy for broadcasting, but the tree may be better. + AllReduceStrategy::Centralized | AllReduceStrategy::Ring => { + broadcast_centralized::(peer_devices, main_device, main_tensor) + } + }; + + Ok(tensors) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/centralized.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/centralized.rs new file mode 100644 index 0000000..d3d9168 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/centralized.rs @@ -0,0 +1,26 @@ +use burn_tensor::backend::Backend; + +use crate::local::tensor_map::{CollectiveTensorMap, get_peer_devices}; +use crate::local::{broadcast_centralized, reduce_sum_centralized}; + +/// Perform an all-reduce operation by reducing all tensors on one device, and broadcasting the +/// result to all other devices +/// +/// Internally, this is just a call to `reduce` followed by a `broadcast` +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors)) +)] +pub(crate) fn all_reduce_sum_centralized( + tensors: CollectiveTensorMap, +) -> CollectiveTensorMap { + // Get corresponding devices for each peer + let peer_devices = get_peer_devices::(&tensors); + let central_device = *tensors.keys().next().unwrap(); + + // Reduce to central device + let central_tensor = reduce_sum_centralized::(tensors, ¢ral_device); + + // Broadcast result to all + broadcast_centralized::(peer_devices, central_device, central_tensor) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/mod.rs new file mode 100644 index 0000000..19947b9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/mod.rs @@ -0,0 +1,11 @@ +mod base; +mod centralized; +mod op; +mod ring; +mod tree; + +pub(crate) use base::*; +pub(crate) use centralized::*; +pub(crate) use op::*; +pub(crate) use ring::*; +pub(crate) use tree::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/op.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/op.rs new file mode 100644 index 0000000..04b7e6a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/op.rs @@ -0,0 +1,141 @@ +use crate::global::node::base::Node; +use crate::local::tensor_map::CollectiveTensorMap; +use crate::{CollectiveConfig, CollectiveError, PeerId, ReduceOperation, local}; +use burn_communication::Protocol; +use burn_std::Shape; +use burn_tensor::TensorMetadata; +use burn_tensor::backend::Backend; +use std::sync::mpsc::SyncSender; + +/// An on-going all-reduce operation +#[derive(Debug)] +pub struct AllReduceOp { + /// all-reduce calls, one for each calling device + calls: Vec>, + /// The reduce operation of the current all-reduce, as defined by the first caller + op: ReduceOperation, + /// The shape of the current all-reduce, as defined by the first caller + shape: Shape, +} + +/// Struct for each device that calls an all-reduce operation +#[derive(Debug)] +pub struct AllReduceOpCall { + /// Id of the caller for this operation + caller: PeerId, + /// The tensor primitive passed as input + input: B::FloatTensorPrimitive, + /// Callback for the result of the all-reduce + result_sender: SyncSender>, +} + +/// Type sent to the collective client upon completion of a all-reduce aggregation +pub(crate) type AllReduceResult = Result; + +impl AllReduceOp { + pub fn new(shape: Shape, reduce_op: ReduceOperation) -> Self { + Self { + calls: vec![], + op: reduce_op, + shape, + } + } + + /// Get a list of the peers. + fn peers(&self) -> Vec { + self.calls.iter().map(|c| c.caller).collect() + } + + /// Register a call to all-reduce in this operation. + /// + /// # Returns + /// + /// `true` if enough peers have registered, and the all-reduce is ready + pub fn register_call( + &mut self, + caller: PeerId, + input: B::FloatTensorPrimitive, + result_sender: SyncSender>, + op: ReduceOperation, + peer_count: usize, + ) -> Result { + if self.shape != input.shape() { + return Err(CollectiveError::AllReduceShapeMismatch); + } + if self.op != op { + return Err(CollectiveError::AllReduceOperationMismatch); + } + + self.calls.push(AllReduceOpCall { + caller, + input, + result_sender, + }); + + Ok(self.calls.len() == peer_count) + } + + /// Runs the all-reduce if the operation is ready. Otherwise, do nothing + #[cfg_attr(feature = "tracing", tracing::instrument( + level = "trace", + skip(self, config, global_client), + fields( + ?self.op, + ?self.shape, + self.peers = ?self.peers(), + ) + ))] + pub async fn execute( + mut self, + config: &CollectiveConfig, + global_client: &mut Option>, + ) { + // all registered callers have sent a tensor to aggregate + match self.all_reduce(config, global_client).await { + Ok(mut tensors) => { + // Return resulting tensors + self.calls.iter().for_each(|call| { + let result = tensors + .remove(&call.caller) + .expect("tensor/peer internal mismatch."); + call.result_sender.send(Ok(result)).unwrap(); + }); + assert_eq!(tensors.len(), 0, "tensor/peer internal mismatch."); + } + Err(err) => { + // Send error to all subscribers + self.fail(err); + } + } + } + + /// Perform an all-reduce operation. + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, config, global_client)) + )] + async fn all_reduce( + &mut self, + config: &CollectiveConfig, + global_client: &mut Option>, + ) -> Result, CollectiveError> { + let tensors = self + .calls + .iter() + .map(|call| (call.caller, call.input.clone())) + .collect(); + + if let Some(global_client) = global_client.as_mut() { + local::all_reduce_with_global(tensors, self.op, config, global_client).await + } else { + local::all_reduce_local_only::(tensors, self.op, config).await + } + } + + /// Send a collective error as result to operation caller + pub fn fail(self, err: CollectiveError) { + self.calls.iter().for_each(|op| { + op.result_sender.send(Err(err.clone())).unwrap(); + }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/ring.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/ring.rs new file mode 100644 index 0000000..621b116 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/ring.rs @@ -0,0 +1,194 @@ +use super::tree::all_reduce_sum_tree; +use crate::PeerId; +use crate::local::tensor_map; +use crate::local::tensor_map::CollectiveTensorMap; +use burn_tensor::{Shape, Slice, TensorMetadata, backend::Backend}; +use std::{collections::HashMap, ops::Range}; + +/// Ring implementation of All-Reduce (Ring-Reduce) +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors)) +)] +pub(crate) fn all_reduce_sum_ring( + tensors: CollectiveTensorMap, +) -> CollectiveTensorMap { + // https://blog.dailydoseofds.com/p/all-reduce-and-ring-reduce-for-model + + // Example: tensors=3, slices=3 + + // phase 1 + // o->o o + // o o->oå + // o o o-> + + // o 1->o + // o o 1-> + // 1->o o + + // o 1 2 + // 2 o 1 + // 1 2 o + + // phase 2 + // o 1 2-> + // 2->o 1 + // 1 2->o + + // 2->1 2 + // 2 2->1 + // 1 2 2-> + + // 2 2 2 + // 2 2 2 + // 2 2 2 + + // Verify all shapes are the same + let shape = tensor_map::get_common_shape::(&tensors) + .expect("Cannot aggregate tensors with different sizes"); + + // Chose an axis + let slice_dim = get_slice_dim(&shape); + + let slice_dim_size = shape[slice_dim]; + let tensor_count = tensors.len(); + if slice_dim_size < tensor_count { + // Tensor cannot be split into N slices! Use a fallback algorithm: binary tree + return all_reduce_sum_tree::(tensors, 2); + } + + // Split tensors into slices + let mut sliced_tensors = slice_tensors::(tensors, shape, slice_dim); + + // phase 1: aggregate in ring N-1 times (Reduce-Scatter) + ring_cycles::(&mut sliced_tensors, true); + + // phase 2: share (overwrite) in a ring N-1 times (All-Gather) + ring_cycles::(&mut sliced_tensors, false); + + // merge slices and put back in result + sliced_tensors + .into_iter() + .map(|(id, slices)| (id, B::float_cat(slices, slice_dim))) + .collect() +} + +/// Get the dimension to slice across: the largest dimension of the shape +pub(crate) fn get_slice_dim(shape: &Shape) -> usize { + // get dimension with the greatest size. + shape + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.cmp(b)) + .map(|(index, _)| index) + .unwrap() +} + +/// With a ring of N tensors, send the tensors N-1 times, either for the first of second phase. +/// During the first phase, the tensor slices are summed. +/// During the second, the slices are replaced. +fn ring_cycles( + sliced_tensors: &mut [(PeerId, Vec)], + is_phase_one: bool, +) { + let tensor_count = sliced_tensors.len(); + for cycle in 0..(tensor_count - 1) { + for i in 0..tensor_count { + let src_tensor_idx = i; + let dest_tensor_idx = (i + 1) % tensor_count; + + let slice_idx = if is_phase_one { + (i + (tensor_count - 1) * cycle) % tensor_count + } else { + // in phase 2, the starting slice is different (see diagrams) + (i + 1 + (tensor_count - 1) * cycle) % tensor_count + }; + + let src_slice = sliced_tensors[src_tensor_idx].1.remove(slice_idx); + let mut dest_slice = sliced_tensors[dest_tensor_idx].1.remove(slice_idx); + + let dest_device = B::float_device(&dest_slice); + let src_slice_on_dest = B::float_to_device(src_slice.clone(), &dest_device); + if is_phase_one { + dest_slice = B::float_add(dest_slice, src_slice_on_dest); + } else { + let slices: Vec = dest_slice + .shape() + .iter() + .map(|&d| Slice::new(0, Some(d as isize), 1)) + .collect(); + + // in phase 2, we don't sum the two slices, we replace with the new one. + dest_slice = + B::float_slice_assign(dest_slice, slices.as_slice(), src_slice_on_dest); + } + + sliced_tensors[src_tensor_idx] + .1 + .insert(slice_idx, src_slice); + sliced_tensors[dest_tensor_idx] + .1 + .insert(slice_idx, dest_slice); + } + } +} + +/// Slice a list of tensors the same way, evenly across a given dimension. +/// The given `shape` should be the same for every tensor. +fn slice_tensors( + mut tensors: HashMap, + shape: Shape, + slice_dim: usize, +) -> Vec<(PeerId, Vec<::FloatTensorPrimitive>)> { + // Get slice index ranges + let ranges = get_ring_reduce_slice_ranges(shape[slice_dim], tensors.len()); + + // Slice tensors + let mut sliced_tensors = vec![]; + for (id, tensor) in tensors.drain() { + let mut slices = vec![]; + for range in &ranges { + let full_range = shape + .iter() + .enumerate() + .map(|(dim_idx, dim)| { + if dim_idx == slice_dim { + Slice::from(range.clone()) + } else { + Slice::from(0..*dim) + } + }) + .collect::>(); + let slice = B::float_slice(tensor.clone(), &full_range); + slices.push(slice); + } + sliced_tensors.push((id, slices)); + } + + sliced_tensors +} + +/// Get the index ranges for the slices to split a tensor evently across a given axis. +/// +/// * `slice_dim_size` - The size of the dim to slice on +/// * `slice_count` - The number of slices +/// +/// Returns a vector of index ranges for each slice. +pub(crate) fn get_ring_reduce_slice_ranges( + slice_dim_size: usize, + slice_count: usize, +) -> Vec> { + let mut ranges: Vec> = vec![]; + + let slice_size = slice_dim_size.div_ceil(slice_count); + + for i in 0..slice_count { + let start = i * slice_size; + let end = start + slice_size; + + ranges.push(Range { start, end }); + } + ranges.last_mut().unwrap().end = slice_dim_size; + + ranges +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/tree.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/tree.rs new file mode 100644 index 0000000..ba4cdf8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/all_reduce/tree.rs @@ -0,0 +1,89 @@ +use crate::PeerId; +use crate::local::tensor_map::CollectiveTensorMap; +use burn_tensor::backend::{Backend, DeviceOps}; +use std::collections::HashMap; + +/// Performs an all-reduce on the provided tensors in a b-tree structure with `arity`. +/// Similar to [reduce_sum_tree](reduce_sum_tree), but this function broadcasts the result with +/// the same tree algorithm. +/// The returned tensors are on the same devices as the corresponding inputs +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors)) +)] +pub(crate) fn all_reduce_sum_tree( + tensors: CollectiveTensorMap, + arity: u32, +) -> CollectiveTensorMap { + let mut input = tensors.into_iter().collect::>(); + + // Sort to put devices of the same type together + input.sort_by(|a, b| { + let dev_a = B::float_device(&a.1); + let dev_b = B::float_device(&b.1); + dev_a.id().cmp(&dev_b.id()) + }); + // Recursive all-reduce + let out = all_reduce_sum_tree_inner::(input, arity); + + let mut tensors = HashMap::new(); + for (id, tensor) in out { + tensors.insert(id, tensor); + } + tensors +} + +/// Recursive function that sums `tensors` and redistributes the result to the host devices +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors)) +)] +fn all_reduce_sum_tree_inner( + mut tensors: Vec<(PeerId, B::FloatTensorPrimitive)>, + arity: u32, +) -> Vec<(PeerId, B::FloatTensorPrimitive)> { + let mut parent_tensors = vec![]; + let mut children_groups = vec![]; + + // Phase 1: Sum tensors in groups of `arity` + 1 + while !tensors.is_empty() { + // Maps ids to devices for each child of this parent + let mut children = vec![]; + let (parent, mut parent_tensor) = tensors.remove(0); + let parent_device = B::float_device(&parent_tensor); + + for _ in 0..arity { + if tensors.is_empty() { + break; + } + let (child, mut child_tensor) = tensors.remove(0); + let child_device = B::float_device(&child_tensor); + children.push((child, child_device)); + child_tensor = B::float_to_device(child_tensor, &parent_device); + parent_tensor = B::float_add(parent_tensor, child_tensor); + } + + parent_tensors.push((parent, parent_tensor)); + children_groups.push(children); + } + + if parent_tensors.len() > 1 { + // Parents are not yet at the root, do the upper part of the tree + parent_tensors = all_reduce_sum_tree_inner::(parent_tensors, arity); + } + + // Phase 2: Redistribute result from each parent to the respective devices + for (parent, parent_tensor) in parent_tensors { + let children = children_groups.remove(0); + for (child, child_device) in children { + // replace child tensors with result + tensors.push(( + child, + B::float_to_device(parent_tensor.clone(), &child_device), + )); + } + tensors.push((parent, parent_tensor)); + } + + tensors +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/centralized.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/centralized.rs new file mode 100644 index 0000000..aec3da9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/centralized.rs @@ -0,0 +1,29 @@ +use std::collections::HashMap; + +use crate::PeerId; +use crate::local::tensor_map::{CollectiveTensorMap, PeerDeviceMap}; +use burn_tensor::backend::Backend; + +/// Broadcasts the tensor from one device in a map to all the others +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(devices, tensor)) +)] +pub(crate) fn broadcast_centralized( + mut devices: PeerDeviceMap, + central: PeerId, + tensor: B::FloatTensorPrimitive, +) -> CollectiveTensorMap { + let mut output = HashMap::new(); + + devices + .remove(¢ral) + .expect("Central device id is in `devices`"); + for (dest, dest_device) in devices { + let tensor = B::float_to_device(tensor.clone(), &dest_device); + output.insert(dest, tensor); + } + output.insert(central, tensor); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/mod.rs new file mode 100644 index 0000000..a96b320 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/mod.rs @@ -0,0 +1,7 @@ +mod centralized; +mod op; +mod tree; + +pub(crate) use centralized::*; +pub(crate) use op::*; +pub(crate) use tree::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/op.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/op.rs new file mode 100644 index 0000000..ee5e2c7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/op.rs @@ -0,0 +1,172 @@ +use crate::local::tensor_map::{CollectiveTensorMap, PeerDeviceMap}; +use crate::{ + BroadcastStrategy, CollectiveConfig, CollectiveError, PeerId, + local::{broadcast_centralized, broadcast_tree}, + node::base::Node, +}; +use burn_communication::Protocol; +#[allow(unused_imports)] // TensorMetadata is used by tracing::instrument. +use burn_tensor::TensorMetadata; +use burn_tensor::backend::Backend; +use std::sync::mpsc::SyncSender; + +/// An on-going broadcast operation +pub struct BroadcastOp { + /// broadcast calls, one for each calling device + calls: Vec>, + /// The tensor to broadcast, as defined by the root. Should be defined before all + /// peers call the operation. + tensor: Option, + + /// ID of the root (or use the first call's peer). + root: Option, +} + +/// Struct for each device that calls an broadcast operation +pub struct BroadcastOpCall { + /// Id of the caller of the operation + caller: PeerId, + /// Device of the calling peer + device: B::Device, + /// Callback for the result of the broadcast + result_sender: SyncSender>, +} + +/// Type sent to the collective client upon completion of a broadcast op +pub(crate) type BroadcastResult = Result; + +impl BroadcastOp { + pub fn new() -> Self { + Self { + calls: vec![], + tensor: None, + root: None, + } + } + + /// Get the effective root of the broadcast operation. + /// If the root is set, return it. Otherwise, return the first caller's peer. + pub fn effective_root(&self) -> PeerId { + self.root.unwrap_or(self.calls.first().unwrap().caller) + } + + pub fn peers(&self) -> Vec { + self.calls.iter().map(|c| c.caller).collect() + } + + fn peer_devices(&self) -> PeerDeviceMap { + self.calls + .iter() + .map(|call| (call.caller, call.device.clone())) + .collect() + } + + /// Register a call to reduce in this operation. + /// When the last caller registers a reduce, the operation is executed. + pub fn register_call( + &mut self, + caller: PeerId, + input: Option, + result_sender: SyncSender>, + device: B::Device, + peer_count: usize, + ) -> Result { + if input.is_some() { + if self.tensor.is_some() { + return Err(CollectiveError::BroadcastMultipleTensors); + } + self.tensor = input; + } + + self.calls.push(BroadcastOpCall { + caller, + device, + result_sender, + }); + + Ok(self.calls.len() == peer_count) + } + + /// Runs the broadcast if the operation is ready. Otherwise, do nothing + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(self, config, global_client), + fields( + self.peers = ?self.peers(), + self.shape = ?self.tensor.as_ref().map(|t| t.shape()), + self.dtype = ?self.tensor.as_ref().map(|t| t.dtype()), + ) + ))] + pub async fn execute( + mut self, + config: &CollectiveConfig, + global_client: &mut Option>, + ) { + // all registered callers have sent a tensor to aggregate + match self.broadcast(config, global_client).await { + Ok(mut tensors) => { + // Return resulting tensors + self.calls.iter().for_each(|call| { + let result = tensors + .remove(&call.caller) + .expect("tensor/peer internal mismatch."); + call.result_sender.send(Ok(result)).unwrap(); + }); + assert_eq!(tensors.len(), 0, "tensor/peer internal mismatch."); + } + Err(err) => { + // Send error to all subscribers + self.fail(err); + } + } + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, config, global_client)) + )] + async fn broadcast( + &mut self, + config: &CollectiveConfig, + global_client: &mut Option>, + ) -> Result, CollectiveError> { + // Do broadcast on global level with the main tensor + if let Some(global_client) = &global_client { + let strategy = config + .global_broadcast_strategy + .expect("global_broadcast_strategy not defined"); + + self.tensor = Some( + global_client + .broadcast(self.tensor.clone(), strategy) + .await + .map_err(CollectiveError::Global)?, + ) + } + + // At this point tensor must be defined + let Some(tensor) = self.tensor.take() else { + return Err(CollectiveError::BroadcastNoTensor); + }; + + let root = self.effective_root(); + let peer_devices = self.peer_devices(); + + // Broadcast locally + Ok(match config.local_broadcast_strategy { + BroadcastStrategy::Tree(arity) => { + broadcast_tree::(peer_devices, root, tensor, arity) + } + BroadcastStrategy::Centralized => { + broadcast_centralized::(peer_devices, root, tensor) + } + }) + } + + /// Send a collective error as result to operation caller + pub fn fail(self, err: CollectiveError) { + self.calls.iter().for_each(|call| { + call.result_sender.send(Err(err.clone())).unwrap(); + }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/tree.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/tree.rs new file mode 100644 index 0000000..e79dd76 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/broadcast/tree.rs @@ -0,0 +1,98 @@ +use burn_tensor::backend::{Backend, DeviceOps}; +use std::collections::HashMap; + +use crate::PeerId; +use crate::local::tensor_map::{CollectiveTensorMap, PeerDeviceMap}; + +/// Performs a broadcast on the provided tensors in a b-tree structure with `arity`. +/// +/// Tensor must be on the device in the `devices` map corresponding to the `root` key. +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(devices, tensor)) +)] +pub(crate) fn broadcast_tree( + mut devices: PeerDeviceMap, + root: PeerId, + tensor: B::FloatTensorPrimitive, + arity: u32, +) -> CollectiveTensorMap { + // Convert hash map to vector of key-value pairs because order matters + let mut devices_vec = vec![]; + let root_device = devices.remove(&root).unwrap(); + for (id, tensor) in devices.drain() { + devices_vec.push((id, tensor)); + } + + // Sort to put devices of the same type together + devices_vec.sort_by(|a, b| { + let dev_a = &a.1; + let dev_b = &b.1; + dev_a.id().cmp(&dev_b.id()) + }); + + // put the root first + devices_vec.insert(0, (root, root_device)); + + // Recursive broadcast + let out = broadcast_tree_inner::(tensor, devices_vec, arity); + + // put results in a hash map + let mut tensors = HashMap::new(); + for (id, tensor) in out { + tensors.insert(id, tensor); + } + + tensors +} + +/// Recursive function that broadcasts tensor across the other devices. Tensor should be on the +/// first device of the list +/// +/// Broadcasts the tensor across the devices in the tree in a pre-order traversal. +fn broadcast_tree_inner( + tensor: B::FloatTensorPrimitive, + mut all_devices: Vec<(PeerId, B::Device)>, + arity: u32, +) -> Vec<(PeerId, B::FloatTensorPrimitive)> { + let mut parents = vec![]; + let mut children_groups = vec![]; + + // Put devices in groups of `arity` + the parent + while !all_devices.is_empty() { + let mut children = vec![]; + let parent = all_devices.remove(0); + + for _ in 0..arity { + if all_devices.is_empty() { + break; + } + children.push(all_devices.remove(0)); + } + + parents.push(parent); + children_groups.push(children); + } + + let mut parents = if parents.len() > 1 { + broadcast_tree_inner::(tensor, parents, arity) + } else { + let root = parents.first().unwrap(); + // `tensor` should already be on the root's device, no need to call B::float_to_device + vec![(root.0, tensor)] + }; + + // Redistribute result from each parent to the respective devices + let mut tensors = vec![]; + for children in children_groups { + let parent = parents.remove(0); + for (child_id, child_device) in children { + // replace child's tensor with parent's + let child_tensor = B::float_to_device(parent.1.clone(), &child_device); + tensors.push((child_id, child_tensor)); + } + tensors.push(parent); + } + + tensors +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/client.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/client.rs new file mode 100644 index 0000000..9770f1f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/client.rs @@ -0,0 +1,271 @@ +use crate::local::all_reduce::AllReduceResult; +use crate::{ + CollectiveConfig, CollectiveError, PeerId, ReduceOperation, + local::{ + BroadcastResult, ReduceResult, + server::{FinishResult, Message, RegisterResult}, + }, +}; +use burn_tensor::backend::Backend; +use std::sync::mpsc::{Receiver, SyncSender}; + +/// Local client to communicate with the local server. Each thread has a client. +#[derive(Clone)] +pub(crate) struct LocalCollectiveClient { + pub channel: SyncSender>, +} + +/// A pending operation that can be waited on. +pub(crate) struct PendingCollectiveOperation { + rx: Receiver>, +} + +impl From> for Receiver> { + fn from(value: PendingCollectiveOperation) -> Self { + value.rx + } +} + +impl PendingCollectiveOperation { + /// Wait on the operation. + /// + /// Given a `Receiver>`, this function will wait: + /// - Unwraps `Ok(Result)` into `Result`; + /// - maps `Err(RecvError)` to `Err(CollectiveError::LocalServerMissing)`. + pub(crate) fn wait(self) -> Result { + let tensor = self + .rx + .recv() + .unwrap_or(Err(CollectiveError::LocalServerMissing))?; + + Ok(tensor) + } +} + +impl LocalCollectiveClient { + /// Common logic for starting a collective operation. + /// + /// - Allocates `(callback, recv)` channels, + /// - Passes the `callback` to the `Message` builder, + /// - Sends the message through the collective channel, + /// - Returns the `recv`. + pub(crate) fn start_operation(&self, builder: F) -> PendingCollectiveOperation + where + F: FnOnce(SyncSender>) -> Message, + { + let (tx, rx) = std::sync::mpsc::sync_channel(1); + self.channel.send((builder)(tx)).unwrap(); + PendingCollectiveOperation { rx } + } + + /// Common logic for starting a collective operation, with validation. + /// + /// When `valid` is `Err`, this function returns a `Receiver>` that + /// immediately returns `Err(valid)`; + /// otherwise, it behaves like [`LocalCollectiveClient::start_operation`]. + pub(crate) fn start_valid_operation( + &self, + valid: Result<(), CollectiveError>, + builder: F, + ) -> PendingCollectiveOperation + where + F: FnOnce(SyncSender>) -> Message, + { + match valid { + Err(e) => { + let (tx, rx) = std::sync::mpsc::sync_channel(1); + tx.send(Err(e)).unwrap(); + PendingCollectiveOperation { rx } + } + _ => self.start_operation(builder), + } + } + + pub(crate) fn reset(&self) { + self.channel.send(Message::Reset).unwrap(); + } + + pub(crate) fn register( + &mut self, + id: PeerId, + device: B::Device, + config: CollectiveConfig, + ) -> RegisterResult { + self.register_start(id, device, config).wait() + } + + pub(crate) fn register_start( + &mut self, + id: PeerId, + device: B::Device, + config: CollectiveConfig, + ) -> PendingCollectiveOperation<()> { + self.start_valid_operation( + match config.is_valid() { + true => Ok(()), + false => Err(CollectiveError::InvalidConfig), + }, + |callback| Message::Register { + device_id: id, + device, + config, + callback, + }, + ) + } + + /// Calls for an all-reduce operation with the given parameters and returns the result. + /// The `params` must be the same as the parameters passed by the other nodes. + /// + /// # Arguments + /// * `id` - The peer id of the caller + /// * `tensor` - The input tensor to reduce with the peers' tensors + /// * `config` - Config of the collective operation. Must be coherent with the other calls. + /// + /// # Result + /// - `Ok(tensor)` if the operation was successful + /// - `Err(CollectiveError)` on error. + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, tensor)) + )] + pub fn all_reduce( + &self, + id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + ) -> AllReduceResult { + self.all_reduce_start(id, tensor, op).wait() + } + + /// Starts an all-reduce operation with the given parameters. + /// + /// The `params` must be the same as the parameters passed by the other nodes. + /// + /// This receiver can be waited on using [`LocalCollectiveClient::operation_wait`]. + /// + /// # Arguments + /// * `id` - The peer id of the caller + /// * `tensor` - The input tensor to reduce with the peers' tensors + /// * `config` - Config of the collective operation. Must be coherent with the other calls. + /// + /// # Result + /// + /// A `Receiver<>` that will yield: + /// - `Ok(AllReduceResult)` if the operation was successful + /// - `Err(SendError)` if the channel was dropped. + pub(crate) fn all_reduce_start( + &self, + id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + ) -> PendingCollectiveOperation { + self.start_operation(|callback| Message::AllReduce { + device_id: id, + tensor, + op, + callback, + }) + } + + /// Reduces a tensor onto one device. + /// + /// # Arguments + /// - `id` - The peer id of the caller. + /// - `tensor` - The tensor to send as input. + /// - `op` - The reduce operation to apply. + /// - `root` - The ID of the peer that will receive the result. + /// + /// Returns Ok(None) if the root tensor is not the caller. Otherwise, returns the reduced tensor. + pub fn reduce( + &self, + id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + root: PeerId, + ) -> ReduceResult { + self.reduce_start(id, tensor, op, root).wait() + } + + /// Starts a reduce operation on a tensor onto one device. + /// + /// This receiver can be waited on using [`LocalCollectiveClient::operation_wait`]. + /// + /// # Arguments + /// - `id` - The peer id of the caller. + /// - `tensor` - The tensor to send as input. + /// - `op` - The reduce operation to apply. + /// - `root` - The ID of the peer that will receive the result. + /// + /// # Result + /// + /// A `Receiver<>` that will yield: + /// - `Ok(ReduceResult)` if the operation was successful + /// - `Err(SendError)` if the channel was dropped. + pub(crate) fn reduce_start( + &self, + id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + root: PeerId, + ) -> PendingCollectiveOperation> { + self.start_operation(|callback| Message::Reduce { + device_id: id, + tensor, + op, + root, + callback, + }) + } + + /// Broadcasts, or receives a broadcasted tensor. + /// + /// # Arguments + /// - `id` - The peer id of the caller + /// - `tensor` - If defined, this tensor will be broadcasted. + /// Otherwise, this call will receive the broadcasted tensor. + /// + /// # Result + /// Synchronously waits on the broadcasted tensor. + pub fn broadcast( + &self, + id: PeerId, + tensor: Option, + ) -> BroadcastResult { + self.broadcast_start(id, tensor).wait() + } + + /// Starts a Broadcast, or receives a broadcasted tensor. + /// + /// This receiver can be waited on using [`LocalCollectiveClient::operation_wait`]. + /// + /// # Arguments + /// - `id` - The peer id of the caller + /// - `tensor` - If defined, this tensor will be broadcasted. Otherwise, this call will receive + /// the broadcasted tensor. + /// + /// # Result + /// + /// A `Receiver<>` that will yield: + /// - `Ok(BroadcastResult)` if the operation was successful + /// - `Err(SendError)` if the channel was dropped. + pub(crate) fn broadcast_start( + &self, + id: PeerId, + tensor: Option, + ) -> PendingCollectiveOperation { + self.start_operation(|callback| Message::Broadcast { + device_id: id, + tensor, + callback, + }) + } + + pub(crate) fn finish(&self, id: PeerId) -> FinishResult { + self.finish_start(id).wait() + } + + pub(crate) fn finish_start(&self, id: PeerId) -> PendingCollectiveOperation<()> { + self.start_operation(|callback| Message::Finish { id, callback }) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/mod.rs new file mode 100644 index 0000000..83ed168 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/mod.rs @@ -0,0 +1,12 @@ +mod all_reduce; +mod broadcast; +mod reduce; + +pub(crate) mod tensor_map; + +pub(crate) use all_reduce::*; +pub(crate) use broadcast::*; +pub(crate) use reduce::*; + +pub(crate) mod client; +pub(crate) mod server; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/centralized.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/centralized.rs new file mode 100644 index 0000000..9cb6540 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/centralized.rs @@ -0,0 +1,30 @@ +use burn_tensor::backend::Backend; + +use crate::PeerId; +use crate::local::tensor_map::CollectiveTensorMap; + +#[cfg(feature = "tracing")] +use crate::local::tensor_map::get_common_shape; + +/// Sums the tensors on one device and returns the result +#[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensors), + fields(shape = ?get_common_shape::(&tensors).unwrap()) +))] +pub(crate) fn reduce_sum_centralized( + mut tensors: CollectiveTensorMap, + central: &PeerId, +) -> B::FloatTensorPrimitive { + let mut central_tensor = tensors + .remove(central) + .expect("Source device id is in the map"); + let central_device = B::float_device(¢ral_tensor); + + for (_, tensor) in tensors { + let rhs = B::float_to_device(tensor.clone(), ¢ral_device); + central_tensor = B::float_add(central_tensor, rhs); + } + + central_tensor +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/mod.rs new file mode 100644 index 0000000..a96b320 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/mod.rs @@ -0,0 +1,7 @@ +mod centralized; +mod op; +mod tree; + +pub(crate) use centralized::*; +pub(crate) use op::*; +pub(crate) use tree::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/op.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/op.rs new file mode 100644 index 0000000..747c03c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/op.rs @@ -0,0 +1,163 @@ +use burn_communication::Protocol; +use burn_tensor::{Shape, TensorMetadata, backend::Backend}; +use std::sync::mpsc::SyncSender; + +use crate::{ + CollectiveConfig, CollectiveError, PeerId, ReduceOperation, ReduceStrategy, + local::{reduce_sum_centralized, reduce_sum_tree}, + node::base::Node, +}; + +/// An on-going reduce operation +pub struct ReduceOp { + /// reduce calls, one for each calling device + calls: Vec>, + /// The reduce operation, as defined by the first caller + op: ReduceOperation, + /// The peer that receives the reduce result, as defined by the first caller + root: PeerId, + /// The shape of the tensor to reduce, as defined by the first caller + shape: Shape, +} + +/// Struct for each device that calls an reduce operation +pub struct ReduceOpCall { + /// Id of the caller of the operation + caller: PeerId, + /// The tensor primitive passed as input + input: B::FloatTensorPrimitive, + /// Callback for the result of the reduce + result_sender: SyncSender>, +} + +/// Type sent to the collective client upon completion of a reduce aggregation +pub(crate) type ReduceResult = Result, CollectiveError>; + +impl ReduceOp { + pub fn new(shape: Shape, reduce_op: ReduceOperation, root: PeerId) -> Self { + Self { + calls: vec![], + op: reduce_op, + root, + shape, + } + } + + fn peers(&self) -> Vec { + self.calls.iter().map(|c| c.caller).collect() + } + + /// Register a call to reduce in this operation. + /// When the last caller registers a reduce, the operation is executed. + pub fn register_call( + &mut self, + caller: PeerId, + input: B::FloatTensorPrimitive, + result_sender: SyncSender>, + op: ReduceOperation, + root: PeerId, + peer_count: usize, + ) -> Result { + if self.shape != input.shape() { + return Err(CollectiveError::ReduceShapeMismatch); + } + if self.op != op { + return Err(CollectiveError::ReduceOperationMismatch); + } + if self.root != root { + return Err(CollectiveError::ReduceRootMismatch); + } + + self.calls.push(ReduceOpCall { + caller, + input, + result_sender, + }); + + Ok(self.calls.len() == peer_count) + } + + /// Runs the all-reduce if the operation is ready. Otherwise, do nothing + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(self, config, global_client), + fields( + ?self.op, + ?self.shape, + self.peers = ?self.peers(), + ) + ))] + pub async fn execute( + mut self, + root: PeerId, + config: &CollectiveConfig, + global_client: &mut Option>, + ) { + match self.reduce(config, global_client).await { + Ok(mut result) => { + // Return resulting tensor to root, None to others + self.calls.iter().for_each(|op| { + let msg = if op.caller == root { + Ok(result.take()) + } else { + Ok(None) + }; + op.result_sender.send(msg).unwrap(); + }); + } + Err(err) => { + self.fail(err); + } + } + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, config, global_client)) + )] + async fn reduce( + &mut self, + config: &CollectiveConfig, + global_client: &mut Option>, + ) -> Result, CollectiveError> { + let tensors = self + .calls + .iter() + .map(|call| (call.caller, call.input.clone())) + .collect(); + + // For Centralized and Tree, we only need to do a reduce here, we'll do a broadcast later + let mut local_sum = match config.local_reduce_strategy { + ReduceStrategy::Centralized => reduce_sum_centralized::(tensors, &self.root), + ReduceStrategy::Tree(arity) => reduce_sum_tree::(tensors, &self.root, arity), + }; + + // Do aggregation on a global level with the main tensor + let result = if let Some(global_client) = global_client { + let strategy = config + .global_reduce_strategy + .expect("global_reduce_strategy not defined"); + + global_client + .reduce(local_sum, strategy, self.root, self.op) + .await + .map_err(CollectiveError::Global)? + } else { + // Mean division locally + if self.op == ReduceOperation::Mean { + let local_tensor_count = self.calls.len() as f32; + local_sum = B::float_div_scalar(local_sum, local_tensor_count.into()) + } + Some(local_sum) + }; + + Ok(result) + } + + /// Send a collective error as result to operation caller + pub fn fail(self, err: CollectiveError) { + self.calls.iter().for_each(|op| { + op.result_sender.send(Err(err.clone())).unwrap(); + }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/tree.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/tree.rs new file mode 100644 index 0000000..021f787 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/reduce/tree.rs @@ -0,0 +1,77 @@ +use crate::PeerId; +use crate::local::tensor_map::CollectiveTensorMap; +use burn_tensor::backend::{Backend, DeviceOps}; + +/// Performs a reduce on the provided tensors in a b-tree structure with `arity`. +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors)) +)] +pub(crate) fn reduce_sum_tree( + mut tensors: CollectiveTensorMap, + root: &PeerId, + arity: u32, +) -> B::FloatTensorPrimitive { + // Convert hash map to vector of key-value pairs because order matters + let mut input = vec![]; + let root_tensor = tensors.remove(root).unwrap(); + for (_, tensor) in tensors.drain() { + input.push(tensor); + } + + // Sort to put devices of the same type together + input.sort_by(|a, b| { + let dev_a = B::float_device(a); + let dev_b = B::float_device(b); + dev_a.id().cmp(&dev_b.id()) + }); + + // put the root first + input.insert(0, root_tensor); + + reduce_sum_tree_inner::(input, arity) +} + +/// Recursive function that sums `tensors` +/// +/// Traverses `tensors` and reduces in a post-order traversal. The first tensor in the list is +/// chosen as the root +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensors)) +)] +fn reduce_sum_tree_inner( + mut tensors: Vec, + arity: u32, +) -> B::FloatTensorPrimitive { + let mut parents = vec![]; + let mut children_groups = vec![]; + + // Sum tensors in groups of `arity` + 1 + while !tensors.is_empty() { + let mut children = vec![]; + let mut parent_tensor = tensors.remove(0); + let parent_device = B::float_device(&parent_tensor); + + for _ in 0..arity { + if tensors.is_empty() { + break; + } + let child_tensor = tensors.remove(0); + children.push(B::float_device(&child_tensor)); + let rhs = B::float_to_device(child_tensor, &parent_device); + parent_tensor = B::float_add(parent_tensor, rhs); + } + + parents.push(parent_tensor); + children_groups.push(children); + } + + if parents.len() > 1 { + // Parents are not yet at the root, do the upper part of the tree + reduce_sum_tree_inner::(parents, arity) + } else { + // Root of tree + parents.remove(0) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/server.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/server.rs new file mode 100644 index 0000000..fed5ba3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/server.rs @@ -0,0 +1,495 @@ +use crate::{ + CollectiveConfig, CollectiveError, PeerId, ReduceOperation, + global::node::base::Node, + local::{ + AllReduceOp, AllReduceResult, BroadcastOp, BroadcastResult, ReduceOp, ReduceResult, + client::LocalCollectiveClient, + }, +}; +use burn_communication::websocket::{WebSocket, WsServer}; +use burn_tensor::{TensorMetadata, backend::Backend}; +use std::sync::{MutexGuard, OnceLock}; +use std::{ + any::{Any, TypeId}, + collections::HashMap, + fmt::Debug, + sync::{ + Arc, Mutex, + mpsc::{Receiver, SyncSender}, + }, +}; +use tokio::runtime::{Builder, Runtime}; + +/// Define the client/server communication on the network +type Network = WebSocket; +/// Type sent to the collective client upon completion of a register request +pub(crate) type RegisterResult = Result<(), CollectiveError>; +/// Type sent to the collective client upon completion of a finish request +pub(crate) type FinishResult = Result<(), CollectiveError>; + +/// The local collective server that manages all the collective aggregation operations +/// (like all-reduce) between local threads. +/// This thread takes in messages from different clients. The clients must register, than they can +/// send an aggregate message. They must all use the same parameters for the same aggregate +/// operation. +pub(crate) struct LocalCollectiveServer { + /// Channel receiver for messages from clients + message_rec: Receiver>, + + /// The collective configuration. Must be the same by every peer when calling register + config: Option, + + /// The ids passed to each register so far + peers: Vec, + + /// Callbacks for when all registers are done + callbacks_register: Vec>, + + /// Map of each peer's id and its device + devices: HashMap, + + /// Current uncompleted all-reduce operation + all_reduce_op: Option>, + + /// Current uncompleted reduce call + reduce_op: Option>, + + /// Uncompleted broadcast calls, one for each calling device. + broadcast_op: Option>, + + /// Client for global collective operations + global_client: Option>, +} + +#[derive(Debug)] +pub(crate) enum Message { + /// Register a new peer with the collective. + Register { + device_id: PeerId, + device: B::Device, + config: CollectiveConfig, + callback: SyncSender, + }, + /// Perform an all-reduce operation. + AllReduce { + device_id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + callback: SyncSender>, + }, + /// Perform a reduce operation. + Reduce { + device_id: PeerId, + tensor: B::FloatTensorPrimitive, + op: ReduceOperation, + root: PeerId, + callback: SyncSender>, + }, + /// Perform a broadcast operation (one-sender, many-receiver). + Broadcast { + device_id: PeerId, + tensor: Option, + callback: SyncSender>, + }, + /// Reset the collective server. + Reset, + Finish { + id: PeerId, + callback: SyncSender, + }, +} + +/// The type-erased box type for [`LocalCollectiveClient`]. +type LocalClientBox = Box; + +/// Global state map from [`Backend`] to boxed [`LocalCollectiveClient`]. +static BACKEND_CLIENT_MAP: OnceLock>> = OnceLock::new(); + +/// Gets a locked mutable view of the `STATE_MAP`. +pub(crate) fn get_backend_client_map() -> MutexGuard<'static, HashMap> { + BACKEND_CLIENT_MAP + .get_or_init(Default::default) + .lock() + .unwrap() +} + +/// Get a [`LocalCollectiveClient`] for the given [`Backend`]. +/// +/// Will start the local collective client/server pair if necessary. +pub(crate) fn get_collective_client() -> LocalCollectiveClient { + let typeid = TypeId::of::(); + let mut state_map = get_backend_client_map(); + match state_map.get(&typeid) { + Some(val) => val.downcast_ref().cloned().unwrap(), + None => { + let client = LocalCollectiveServer::::setup(LocalCollectiveClientConfig::default()); + state_map.insert(typeid, Box::new(client.clone())); + client + } + } +} + +/// Global runtime. +static SERVER_RUNTIME: OnceLock> = OnceLock::new(); + +/// Get the global [`Runtime`]. +pub(crate) fn get_collective_server_runtime() -> Arc { + SERVER_RUNTIME + .get_or_init(|| { + Builder::new_multi_thread() + .enable_all() + .build() + .expect("Unable to initialize runtime") + .into() + }) + .clone() +} + +/// Configuration for the local collective client/server pair. +pub struct LocalCollectiveClientConfig { + /// Channel capacity for the messaging queue from client to server. + pub channel_capacity: usize, +} + +impl Default for LocalCollectiveClientConfig { + fn default() -> Self { + Self { + channel_capacity: 50, + } + } +} + +impl From for LocalCollectiveClientConfig { + fn from(capacity: usize) -> Self { + Self { + channel_capacity: capacity, + } + } +} + +impl LocalCollectiveServer { + fn new(rec: Receiver>) -> Self { + Self { + message_rec: rec, + config: None, + peers: vec![], + devices: HashMap::new(), + all_reduce_op: None, + reduce_op: None, + broadcast_op: None, + callbacks_register: vec![], + global_client: None, + } + } + + /// Setup a client/server pair with the given config. + pub(crate) fn setup(cfg: C) -> LocalCollectiveClient + where + C: Into, + { + let cfg = cfg.into(); + let (tx, rx) = std::sync::mpsc::sync_channel(cfg.channel_capacity); + + get_collective_server_runtime().spawn(async { + let typeid = TypeId::of::(); + log::info!("Starting server for backend: {typeid:?}"); + let mut server = LocalCollectiveServer::new(rx); + + loop { + match server.message_rec.recv() { + Ok(message) => server.process_message(message).await, + Err(err) => { + log::error!( + "Error receiving message from local collective server: {err:?}" + ); + break; + } + } + } + }); + + LocalCollectiveClient { channel: tx } + } + + async fn process_message(&mut self, message: Message) { + match message { + Message::Register { + device_id, + device, + config, + callback, + } => { + self.process_register_message(device_id, device, config, &callback) + .await + } + Message::AllReduce { + device_id, + tensor, + op, + callback, + } => { + self.process_all_reduce_message(device_id, tensor, op, callback) + .await + } + Message::Reduce { + device_id, + tensor, + op, + root, + callback, + } => { + self.process_reduce_message(device_id, tensor, op, root, callback) + .await + } + Message::Broadcast { + device_id, + tensor, + callback, + } => { + self.process_broadcast_message(device_id, tensor, callback) + .await + } + Message::Reset => self.reset(), + Message::Finish { id, callback } => self.process_finish_message(id, callback).await, + } + } + + async fn process_register_message( + &mut self, + device_id: PeerId, + device: B::Device, + config: CollectiveConfig, + callback: &SyncSender, + ) { + if !config.is_valid() { + callback.send(Err(CollectiveError::InvalidConfig)).unwrap(); + return; + } + if self.peers.contains(&device_id) { + callback + .send(Err(CollectiveError::MultipleRegister)) + .unwrap(); + return; + } + if self.peers.is_empty() || self.config.is_none() { + self.config = Some(config); + } else if let Some(cfg) = &self.config + && *cfg != config + { + callback + .send(Err(CollectiveError::RegisterParamsMismatch)) + .unwrap(); + return; + } + + self.peers.push(device_id); + self.callbacks_register.push(callback.clone()); + self.devices.insert(device_id, device); + + let config = self.config.as_ref().unwrap(); + let global_params = config.global_register_params(); + if let Some(global_params) = &global_params + && self.global_client.is_none() + { + let server = WsServer::new(global_params.data_service_port); + let client = Node::new(&global_params.global_address, server); + self.global_client = Some(client) + } + + // All have registered, callback + if self.peers.len() == config.num_devices { + let mut register_result = Ok(()); + + // if an error occurs on the global register, it must be passed back to every local peer + if let Some(global_params) = global_params { + let client = self + .global_client + .as_mut() + .expect("Global client should be initialized"); + + register_result = client + .register(self.peers.clone(), global_params) + .await + .map_err(CollectiveError::Global); + }; + + // Send results to all callbacks. + self.callbacks_register + .drain(..) + .for_each(|tx| tx.send(register_result.clone()).unwrap()); + } + } + + /// Processes an Message::AllReduce. + async fn process_all_reduce_message( + &mut self, + peer_id: PeerId, + tensor: ::FloatTensorPrimitive, + op: ReduceOperation, + callback: SyncSender>, + ) { + if !self.peers.contains(&peer_id) { + callback + .send(Err(CollectiveError::RegisterNotFirstOperation)) + .unwrap(); + return; + } + + if self.all_reduce_op.is_none() { + // First call to all-reduce + self.all_reduce_op = Some(AllReduceOp::new(tensor.shape(), op)); + } + // Take the operation, we'll put it back if we're not done + let mut all_reduce_op = self.all_reduce_op.take().unwrap(); + + // On the last caller, the all-reduce is done here + let res = + all_reduce_op.register_call(peer_id, tensor, callback.clone(), op, self.peers.len()); + + // Upon an error or the last call, the all_reduce_op is dropped + match res { + Ok(is_ready) => { + if is_ready { + all_reduce_op + .execute(self.config.as_ref().unwrap(), &mut self.global_client) + .await; + } else { + // Put operation back, we're waiting for more calls + self.all_reduce_op = Some(all_reduce_op) + } + } + Err(err) => all_reduce_op.fail(err), + } + } + + /// Processes a Message::Reduce. + async fn process_reduce_message( + &mut self, + peer_id: PeerId, + tensor: ::FloatTensorPrimitive, + op: ReduceOperation, + root: PeerId, + callback: SyncSender>, + ) { + if !self.peers.contains(&root) { + callback + .send(Err(CollectiveError::RegisterNotFirstOperation)) + .unwrap(); + return; + } + + if self.reduce_op.is_none() { + // First call to reduce + self.reduce_op = Some(ReduceOp::new(tensor.shape(), op, root)); + } + let mut reduce_op = self.reduce_op.take().unwrap(); + + // On the last caller, the all-reduce is done here + let res = reduce_op.register_call( + peer_id, + tensor, + callback.clone(), + op, + root, + self.peers.len(), + ); + + // Upon an error or the last call, the all_reduce_op is dropped + match res { + Ok(is_ready) => { + if is_ready { + reduce_op + .execute(root, self.config.as_ref().unwrap(), &mut self.global_client) + .await; + } else { + // Put operation back, we're waiting for more calls + self.reduce_op = Some(reduce_op) + } + } + Err(err) => reduce_op.fail(err), + } + } + + /// Processes a Message::Broadcast. + async fn process_broadcast_message( + &mut self, + caller: PeerId, + tensor: Option<::FloatTensorPrimitive>, + callback: SyncSender>, + ) { + if self.config.is_none() { + callback + .send(Err(CollectiveError::RegisterNotFirstOperation)) + .unwrap(); + return; + } + if !self.peers.contains(&caller) { + callback + .send(Err(CollectiveError::RegisterNotFirstOperation)) + .unwrap(); + return; + } + + if self.broadcast_op.is_none() { + // First call to broadcast + self.broadcast_op = Some(BroadcastOp::new()); + } + let device = self.devices.get(&caller).unwrap().clone(); + + let mut broadcast_op = self.broadcast_op.take().unwrap(); + + // On the last caller, the all-reduce is done here + let res = + broadcast_op.register_call(caller, tensor, callback.clone(), device, self.peers.len()); + + // Upon an error or the last call, the all_reduce_op is dropped + match res { + Ok(is_ready) => { + if is_ready { + broadcast_op + .execute(self.config.as_ref().unwrap(), &mut self.global_client) + .await; + } else { + // Put operation back, we're waiting for more calls + self.broadcast_op = Some(broadcast_op) + } + } + Err(err) => broadcast_op.fail(err), + } + } + + /// Reinitializes the collective server + fn reset(&mut self) { + self.peers.clear(); + self.all_reduce_op = None; + self.reduce_op = None; + self.broadcast_op = None; + } + + /// Processes a Message::Finish. + async fn process_finish_message(&mut self, id: PeerId, callback: SyncSender) { + if self.config.is_none() { + callback + .send(Err(CollectiveError::RegisterNotFirstOperation)) + .unwrap(); + return; + } + if !self.peers.contains(&id) { + callback + .send(Err(CollectiveError::MultipleUnregister)) + .unwrap(); + return; + } + + // Remove registered with id + self.peers.retain(|x| *x != id); + + if self.peers.is_empty() + && let Some(mut global_client) = self.global_client.take() + { + global_client.finish().await; + } + + callback.send(Ok(())).unwrap(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/tensor_map.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/tensor_map.rs new file mode 100644 index 0000000..4c32d88 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/local/tensor_map.rs @@ -0,0 +1,33 @@ +//! # Common Tensor Map for Local Collective Operations +use crate::PeerId; +use burn_std::Shape; +use burn_tensor::TensorMetadata; +use burn_tensor::backend::Backend; +use std::collections::HashMap; + +pub type CollectiveTensorMap = HashMap::FloatTensorPrimitive>; + +pub type PeerDeviceMap = HashMap::Device>; + +/// Get the shape of the tensors. They should all have the same shape, otherwise None is returned. +pub fn get_common_shape(tensors: &CollectiveTensorMap) -> Option { + let mut it = tensors.values(); + if let Some(first) = it.next() { + let shape = first.shape(); + for tensor in it { + if tensor.shape() != shape { + return None; + } + } + return Some(shape); + } + None +} + +/// Get the `{ peer_id -> device }` mapping for the given tensors. +pub fn get_peer_devices(tensors: &CollectiveTensorMap) -> PeerDeviceMap { + tensors + .iter() + .map(|(id, tensor)| (*id, B::float_device(tensor))) + .collect() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/all_reduce.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/all_reduce.rs new file mode 100644 index 0000000..9823507 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/all_reduce.rs @@ -0,0 +1,174 @@ +mod tests { + use std::sync::mpsc::SyncSender; + + use burn_std::rand::get_seeded_rng; + use burn_tensor::{Shape, Tensor, TensorData, TensorPrimitive, Tolerance, backend::Backend}; + + use serial_test::serial; + + #[cfg(feature = "test-ndarray")] + pub type TestBackend = burn_ndarray::NdArray; + + #[cfg(feature = "test-cuda")] + pub type TestBackend = burn_cuda::Cuda; + + #[cfg(feature = "test-wgpu")] + pub type TestBackend = burn_wgpu::Wgpu; + + #[cfg(feature = "test-metal")] + pub type TestBackend = burn_wgpu::Wgpu; + + #[cfg(feature = "test-vulkan")] + pub type TestBackend = burn_wgpu::Wgpu; + + use crate::{ + AllReduceStrategy, CollectiveConfig, PeerId, ReduceOperation, all_reduce, register, + reset_collective, + }; + + pub fn run_peer( + id: PeerId, + config: CollectiveConfig, + input: TensorData, + op: ReduceOperation, + output: SyncSender>, + ) { + let device = B::Device::default(); + + register::(id, device.clone(), config).unwrap(); + + let tensor = Tensor::::from_data(input, &device); + + let tensor = Tensor::from_primitive(TensorPrimitive::Float( + all_reduce::(id, tensor.into_primitive().tensor(), op).unwrap(), + )); + + output.send(tensor).unwrap(); + } + + fn generate_random_input( + shape: Shape, + op: ReduceOperation, + thread_count: usize, + ) -> (Vec, TensorData) { + let input: Vec = (0..thread_count) + .map(|_| { + TensorData::random::( + shape.clone(), + burn_tensor::Distribution::Default, + &mut get_seeded_rng(), + ) + }) + .collect(); + + let device = ::Device::default(); + + let mut expected_tensor = Tensor::::zeros(shape, &device); + for item in input.iter().take(thread_count as usize) { + let input_tensor = Tensor::::from_data(item.clone(), &device); + expected_tensor = expected_tensor.add(input_tensor); + } + if op == ReduceOperation::Mean { + expected_tensor = expected_tensor.div_scalar(thread_count as u32); + } + + let expected = expected_tensor.to_data(); + + (input, expected) + } + + fn test_all_reduce( + device_count: usize, + op: ReduceOperation, + strategy: AllReduceStrategy, + tensor_size: usize, + ) { + reset_collective::(); + + let (send, recv) = std::sync::mpsc::sync_channel(32); + + let shape = Shape { + dims: vec![tensor_size], + }; + + let (input, expected) = generate_random_input(shape, op, device_count); + + let config = CollectiveConfig::default() + .with_num_devices(device_count) + .with_local_all_reduce_strategy(strategy); + + for id in 0..device_count { + let send = send.clone(); + let input = input[id as usize].clone(); + + std::thread::spawn({ + let config = config.clone(); + move || run_peer::(id.into(), config, input, op, send) + }); + } + + let first = recv.recv().unwrap().to_data(); + for _ in 1..device_count { + let tensor = recv.recv().unwrap(); + tensor.to_data().assert_eq(&first, true); + } + + let tol: Tolerance = Tolerance::balanced(); + expected.assert_approx_eq(&first, tol); + } + + #[test] + #[serial] + pub fn test_all_reduce_centralized_sum() { + test_all_reduce::(4, ReduceOperation::Sum, AllReduceStrategy::Centralized, 4); + } + + #[test] + #[serial] + pub fn test_all_reduce_centralized_mean() { + test_all_reduce::(4, ReduceOperation::Mean, AllReduceStrategy::Centralized, 4); + } + + #[test] + #[serial] + pub fn test_all_reduce_binary_tree_sum() { + test_all_reduce::(4, ReduceOperation::Sum, AllReduceStrategy::Tree(2), 4); + } + + #[test] + #[serial] + pub fn test_all_reduce_binary_tree_mean() { + test_all_reduce::(4, ReduceOperation::Mean, AllReduceStrategy::Tree(2), 4); + } + + #[test] + #[serial] + pub fn test_all_reduce_5_tree_sum() { + test_all_reduce::(4, ReduceOperation::Sum, AllReduceStrategy::Tree(5), 4); + } + + #[test] + #[serial] + pub fn test_all_reduce_5_tree_mean() { + test_all_reduce::(4, ReduceOperation::Mean, AllReduceStrategy::Tree(5), 4); + } + + #[test] + #[serial] + pub fn test_all_reduce_ring_sum() { + test_all_reduce::(3, ReduceOperation::Sum, AllReduceStrategy::Ring, 3); + } + + #[test] + #[serial] + pub fn test_all_reduce_ring_mean() { + test_all_reduce::(3, ReduceOperation::Mean, AllReduceStrategy::Ring, 3); + } + + #[test] + #[serial] + pub fn test_all_reduce_ring_irregular_sum() { + // this should trigger the fallback algorithm when the tensor is too small. + test_all_reduce::(4, ReduceOperation::Sum, AllReduceStrategy::Ring, 3); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/broadcast.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/broadcast.rs new file mode 100644 index 0000000..c39720e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/broadcast.rs @@ -0,0 +1,126 @@ +mod tests { + use std::sync::mpsc::SyncSender; + + use burn_std::rand::get_seeded_rng; + use burn_tensor::{Shape, Tensor, TensorData, TensorPrimitive, Tolerance, backend::Backend}; + + use serial_test::serial; + + #[cfg(feature = "test-ndarray")] + pub type TestBackend = burn_ndarray::NdArray; + + #[cfg(feature = "test-cuda")] + pub type TestBackend = burn_cuda::Cuda; + + #[cfg(feature = "test-wgpu")] + pub type TestBackend = burn_wgpu::Wgpu; + + #[cfg(feature = "test-metal")] + pub type TestBackend = burn_wgpu::Wgpu; + + #[cfg(feature = "test-vulkan")] + pub type TestBackend = burn_wgpu::Wgpu; + + use crate::{ + BroadcastStrategy, CollectiveConfig, PeerId, broadcast, register, reset_collective, + }; + + pub fn run_peer( + id: PeerId, + config: CollectiveConfig, + input: Option, + output: SyncSender>, + ) { + let device = B::Device::default(); + + register::(id, device.clone(), config).unwrap(); + + let tensor = input.map(|data| B::float_from_data(data, &device)); + let tensor = broadcast::(id, tensor).unwrap(); + let tensor = Tensor::::from_primitive(TensorPrimitive::Float(tensor)); + + output.send(tensor).unwrap(); + } + + fn generate_random_input(shape: Shape) -> TensorData { + TensorData::random::( + shape.clone(), + burn_tensor::Distribution::Default, + &mut get_seeded_rng(), + ) + } + + fn test_broadcast( + device_count: usize, + strategy: BroadcastStrategy, + tensor_size: usize, + ) { + reset_collective::(); + + let (send, recv) = std::sync::mpsc::sync_channel(32); + + let shape = Shape { + dims: vec![tensor_size], + }; + + let input = generate_random_input(shape); + + let config = CollectiveConfig::default() + .with_num_devices(device_count) + .with_local_broadcast_strategy(strategy); + + for id in 0..device_count { + // The peer #0 is the root: it sends the tensor + let input = if id == 0 { Some(input.clone()) } else { None }; + + std::thread::spawn({ + let config = config.clone(); + let send = send.clone(); + move || run_peer::(id.into(), config, input, send) + }); + } + + // Expect all peers to receive the input tensor + let tol: Tolerance = Tolerance::balanced(); + for _ in 0..device_count { + let tensor = recv.recv().unwrap().to_data(); + input.assert_approx_eq(&tensor, tol); + } + } + + #[test] + #[serial] + pub fn test_broadcast_centralized_sum() { + test_broadcast::(4, BroadcastStrategy::Centralized, 4); + } + + #[test] + #[serial] + pub fn test_broadcast_centralized_mean() { + test_broadcast::(4, BroadcastStrategy::Centralized, 4); + } + + #[test] + #[serial] + pub fn test_broadcast_binary_tree_sum() { + test_broadcast::(4, BroadcastStrategy::Tree(2), 4); + } + + #[test] + #[serial] + pub fn test_broadcast_binary_tree_mean() { + test_broadcast::(4, BroadcastStrategy::Tree(2), 4); + } + + #[test] + #[serial] + pub fn test_broadcast_5_tree_sum() { + test_broadcast::(4, BroadcastStrategy::Tree(5), 4); + } + + #[test] + #[serial] + pub fn test_broadcast_5_tree_mean() { + test_broadcast::(4, BroadcastStrategy::Tree(5), 4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/mod.rs new file mode 100644 index 0000000..df65fe2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/mod.rs @@ -0,0 +1,3 @@ +mod all_reduce; +mod broadcast; +mod reduce; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/reduce.rs b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/reduce.rs new file mode 100644 index 0000000..8adeeab --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-collective/src/tests/reduce.rs @@ -0,0 +1,162 @@ +mod tests { + use std::sync::mpsc::SyncSender; + + use burn_std::rand::get_seeded_rng; + use burn_tensor::{Shape, Tensor, TensorData, TensorPrimitive, Tolerance, backend::Backend}; + + use serial_test::serial; + + #[cfg(feature = "test-ndarray")] + pub type TestBackend = burn_ndarray::NdArray; + + #[cfg(feature = "test-cuda")] + pub type TestBackend = burn_cuda::Cuda; + + #[cfg(feature = "test-wgpu")] + pub type TestBackend = burn_wgpu::Wgpu; + + #[cfg(feature = "test-metal")] + pub type TestBackend = burn_wgpu::Wgpu; + + #[cfg(feature = "test-vulkan")] + pub type TestBackend = burn_wgpu::Wgpu; + + use crate::{ + CollectiveConfig, PeerId, ReduceOperation, ReduceStrategy, reduce, register, + reset_collective, + }; + + pub fn run_peer( + id: PeerId, + config: CollectiveConfig, + input: TensorData, + op: ReduceOperation, + root: PeerId, + output: SyncSender>>, + ) { + let device = B::Device::default(); + + register::(id, device.clone(), config).unwrap(); + + let tensor = Tensor::::from_data(input, &device); + + let tensor = tensor.into_primitive().tensor(); + let tensor = reduce::(id, tensor, op, root).unwrap(); + let tensor = tensor.map(|t| Tensor::::from_primitive(TensorPrimitive::Float(t))); + + output.send(tensor).unwrap(); + } + + fn generate_random_input( + shape: Shape, + op: ReduceOperation, + thread_count: usize, + ) -> (Vec, TensorData) { + let input: Vec = (0..thread_count) + .map(|_| { + TensorData::random::( + shape.clone(), + burn_tensor::Distribution::Default, + &mut get_seeded_rng(), + ) + }) + .collect(); + + let device = ::Device::default(); + + let mut expected_tensor = Tensor::::zeros(shape, &device); + for item in input.iter().take(thread_count) { + let input_tensor = Tensor::::from_data(item.clone(), &device); + expected_tensor = expected_tensor.add(input_tensor); + } + if op == ReduceOperation::Mean { + expected_tensor = expected_tensor.div_scalar(thread_count as u32); + } + + let expected = expected_tensor.to_data(); + + (input, expected) + } + + fn test_reduce( + device_count: usize, + op: ReduceOperation, + strategy: ReduceStrategy, + tensor_size: usize, + ) { + reset_collective::(); + + let (send, recv) = std::sync::mpsc::sync_channel(32); + + let shape = Shape { + dims: vec![tensor_size], + }; + + let (input, expected) = generate_random_input(shape, op, device_count); + + let config = CollectiveConfig::default() + .with_num_devices(device_count) + .with_local_reduce_strategy(strategy); + + let root: PeerId = 0.into(); + for id in 0..device_count { + let send = send.clone(); + let input = input[id as usize].clone(); + + std::thread::spawn({ + let config = config.clone(); + move || run_peer::(id.into(), config, input, op, root, send) + }); + } + + let mut result = None; + for _ in 0..device_count { + let tensor = recv.recv().unwrap(); + if tensor.is_some() { + if result.is_some() { + panic!("Two peers received the result of an reduce!"); + } + result = tensor.map(|t| t.to_data()); + } + } + + let tol: Tolerance = Tolerance::balanced(); + expected.assert_approx_eq(&result.expect("One peer has received the result"), tol); + } + + #[test] + #[serial] + pub fn test_reduce_centralized_sum() { + test_reduce::(4, ReduceOperation::Sum, ReduceStrategy::Centralized, 4); + } + + #[test] + #[serial] + pub fn test_reduce_centralized_mean() { + test_reduce::(4, ReduceOperation::Mean, ReduceStrategy::Centralized, 4); + } + + #[test] + #[serial] + pub fn test_reduce_binary_tree_sum() { + test_reduce::(4, ReduceOperation::Sum, ReduceStrategy::Tree(2), 4); + } + + #[test] + #[serial] + pub fn test_reduce_binary_tree_mean() { + test_reduce::(4, ReduceOperation::Mean, ReduceStrategy::Tree(2), 4); + } + + #[test] + #[serial] + pub fn test_reduce_5_tree_sum() { + test_reduce::(4, ReduceOperation::Sum, ReduceStrategy::Tree(5), 4); + } + + #[test] + #[serial] + pub fn test_reduce_5_tree_mean() { + test_reduce::(4, ReduceOperation::Mean, ReduceStrategy::Tree(5), 4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-communication/Cargo.toml new file mode 100644 index 0000000..a2884f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/Cargo.toml @@ -0,0 +1,44 @@ +[package] +authors = ["Guilhem Ané (@Cielbird)", "Nathaniel Simard (@nathanielsimard)"] +description = "Abstractions for network communication for Burn" +edition.workspace = true +license.workspace = true +name = "burn-communication" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-communication" +version.workspace = true + +[lints] +workspace = true + +[features] +tracing = [ + "burn-std/tracing", + "burn-tensor?/tracing", +] + +data-service = ["burn-tensor"] +websocket = ["axum", "tokio-tungstenite", "futures"] + +[dependencies] +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = true } +bytes = { workspace = true } +derive-new = { workspace = true } +futures-util = { workspace = true } +log = { workspace = true } +rmp-serde = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_bytes = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "signal", "tracing"] } +tokio-util = { workspace = true } +tracing = { workspace = true, features = ["default"] } +tracing-core = { workspace = true, features = ["default"] } +tracing-subscriber = { workspace = true, features = ["default", "fmt", "env-filter"] } + +# Tensor Data Service +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", optional = true } + +# Websocket +axum = { workspace = true, features = ["ws"], optional = true } +tokio-tungstenite = { workspace = true, optional = true } +futures = { workspace = true, optional = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/README.md b/crates/stable-diffusion-burn/burn-crates/burn-communication/README.md new file mode 100644 index 0000000..d5f1490 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/README.md @@ -0,0 +1,15 @@ +# Burn Communication + +Abstractions for network communication + +The Protocol trait defines how to communicate in a server/client style. +The server can set up routes with callbacks upon connection. + +## WebSocket + +Communication with WebSockets is implemented with the `websocket` feature. + +## Tensor Data Service + +The tensor data service provides easy utilities to share tensors peer-to-peer. +One peer can expose a tensor, and another can download it. Each peer is both a client and a server. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/base.rs new file mode 100644 index 0000000..65e3c44 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/base.rs @@ -0,0 +1,104 @@ +use burn_std::future::DynFut; +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Display}; +use std::hash::Hash; +use std::str::FromStr; + +/// Allows nodes to find each other +#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Debug)] +pub struct Address { + pub(crate) inner: String, +} + +impl FromStr for Address { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(Self { + inner: s.to_string(), + }) + } +} + +impl Display for Address { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.inner) + } +} + +/// The protocol used for the communications. +pub trait Protocol: Clone + Send + Sync + 'static { + /// The client implementation for the current protocol. + type Client: ProtocolClient; + /// The server implementation for the current protocol. + type Server: ProtocolServer; +} + +/// Error that happens during a communication. +pub trait CommunicationError: Debug + Send + 'static {} + +/// The client is only used to create a [channel](CommunicationChannel), which should be use to +/// transmit information with the [server](ProtocolServer). +pub trait ProtocolClient: Send + Sync + 'static { + /// Channel used by this protocol. + type Channel: CommunicationChannel; + /// The error type. + type Error: CommunicationError; + + /// Opens a new [channel](CommunicationChannel) with the current protocol at the given + /// [address](Address) and route. + /// + /// * `address` - Address to connect to + /// * `route` - The name of the route (no slashes) + /// + /// Returns None if the connection can't be done. + fn connect(address: Address, route: &str) -> DynFut>; +} + +/// Data sent and received by the client and server. +#[derive(new)] +pub struct Message { + /// The data is always encoded as bytes. + pub data: bytes::Bytes, +} + +/// Defines how to create a server that respond to a [channel](CommunicationChannel). +pub trait ProtocolServer: Sized + Send + Sync + 'static { + /// Channel used by this protocol. + type Channel: CommunicationChannel; + /// The error type. + type Error: CommunicationError; + + /// Defines an endpoint with the function that responds. + /// TODO Docs: does it need a slash? + fn route(self, path: &str, callback: C) -> Self + where + C: FnOnce(Self::Channel) -> Fut + Clone + Send + Sync + 'static, + Fut: Future + Send + 'static; + + /// Start the server. + fn serve( + self, + shutdown: F, + ) -> impl Future> + Send + 'static + where + F: Future + Send + 'static; +} + +/// Handles communications. +pub trait CommunicationChannel: Send + 'static { + type Error: CommunicationError; + + /// Send a [message](Message) on the channel. + fn send( + &mut self, + message: Message, + ) -> impl std::future::Future> + Send; + + /// Receive a [message](Message) on the channel and returns a new [response message](Message). + fn recv( + &mut self, + ) -> impl std::future::Future, Self::Error>> + Send; + + fn close(&mut self) -> impl std::future::Future> + Send; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/data_service.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/data_service.rs new file mode 100644 index 0000000..94b54fe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/data_service.rs @@ -0,0 +1,258 @@ +//! This module enables direct data transfer between servers without blocking the client or any server. +//! +//! It eliminates the need for intermediate data transfer through the client, avoiding the process of downloading data from one server and reuploading it to another. +//! +//! The module provides an optimized mechanism for servers to communicate directly, streamlining data movement between them without involving the client. + +use crate::Message; +use crate::base::Protocol; +use crate::base::{Address, CommunicationChannel, ProtocolClient, ProtocolServer}; +use burn_tensor::{TensorData, backend::Backend}; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, marker::PhantomData, sync::Arc}; +use tokio::sync::Mutex; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct TensorTransferId(u64); + +impl From for TensorTransferId { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl TensorTransferId { + pub fn next(&mut self) { + self.0 += 1; + } +} + +#[derive(Debug, Serialize, Deserialize)] +enum DataServiceMessage { + TensorRequest(TensorTransferId), + Tensor(TensorData), +} + +type ClientChannelRef = Arc::Channel>>; + +pub struct TensorDataService> { + /// Maps tensor transfer IDs to their exposed state. + pub exposed_tensors: Mutex>, + /// Maps node addresses to their channels. + pub channels: Mutex>>, + /// Notify when a new tensor is exposed. + pub new_tensor_notify: Arc, + + cancel_token: CancellationToken, + + _phantom_data: PhantomData, +} + +pub struct TensorExposeState { + /// The bytes of the tensor data message. Message::Data(...) serialized with rmp_serde + pub bytes: bytes::Bytes, + /// How many times the tensor will be downloaded + pub max_downloads: u32, + /// How man times the tensor has been downloaded + pub cur_download_count: u32, +} + +/// Provides a routing function for a tensor data service for a communications server +pub trait TensorDataServer { + /// Routes the tensor data service to the "/data" route + fn route_tensor_data_service(self, state: Arc>) -> Self; +} + +impl + 'static> + TensorDataServer for S +{ + fn route_tensor_data_service(self, state: Arc>) -> Self { + self.route("/data", async move |stream: S::Channel| { + state.handle_data_channel(stream).await; + }) + } +} + +impl TensorDataService { + pub fn new(cancel_token: CancellationToken) -> Self { + Self { + exposed_tensors: Mutex::new(HashMap::new()), + channels: Mutex::new(HashMap::new()), + new_tensor_notify: Arc::new(Notify::new()), + cancel_token, + _phantom_data: PhantomData::, + } + } + + /// Exposes a tensor to the data server, allowing it to be downloaded by other nodes. + pub async fn expose( + &self, + tensor: B::FloatTensorPrimitive, + max_downloads: u32, + transfer_id: TensorTransferId, + ) { + let data = B::float_into_data(tensor).await.unwrap(); + self.expose_data(data, max_downloads, transfer_id).await + } + + /// Exposes a tensor data to the data server, allowing it to be downloaded by other nodes. + pub async fn expose_data( + &self, + tensor_data: TensorData, + max_downloads: u32, + transfer_id: TensorTransferId, + ) { + let bytes: bytes::Bytes = rmp_serde::to_vec(&DataServiceMessage::Tensor(tensor_data)) + .unwrap() + .into(); + let mut exposed_tensors = self.exposed_tensors.lock().await; + exposed_tensors.insert( + transfer_id, + TensorExposeState { + bytes, + max_downloads, + cur_download_count: 0, + }, + ); + core::mem::drop(exposed_tensors); + self.new_tensor_notify.notify_waiters(); + } + + pub async fn close(&self) { + // Send a closing message to every open WebSocket stream + + let mut streams = self.channels.lock().await; + for (_, stream) in streams.drain() { + let mut stream = stream.lock().await; + + stream + .close() + .await + .expect("Failed to close WebSocket stream"); + } + } + + /// Downloads a tensor that is exposed on another server. Requires a Tokio 1.x runtime + /// + /// Returns None if the peer closes the connection + pub async fn download_tensor( + &self, + remote: Address, + transfer_id: TensorTransferId, + ) -> Option { + log::info!("Downloading tensor from {remote:?}"); + + let stream = self.get_data_stream(remote).await; + let mut stream = stream.lock().await; + + // Send the download request with the download id + let bytes: bytes::Bytes = + rmp_serde::to_vec(&DataServiceMessage::TensorRequest(transfer_id)) + .unwrap() + .into(); + stream + .send(Message::new(bytes)) + .await + .expect("Failed to send download id"); + + if let Ok(msg) = stream.recv().await { + let Some(msg) = msg else { + log::warn!("Received None message from the websocket, closing connection."); + return None; + }; + + let DataServiceMessage::Tensor(data) = rmp_serde::from_slice(&msg.data) + .expect("Can deserialize messages from the websocket.") + else { + panic!("Message should have been TensorData") + }; + return Some(data); + } + log::warn!("Closed connection"); + None + } + + /// Get the WebSocket stream for the given address, or create a new one if it doesn't exist. + async fn get_data_stream( + &self, + address: Address, + ) -> Arc::Channel>> { + let mut streams = self.channels.lock().await; + match streams.get(&address) { + Some(stream) => stream.clone(), + None => { + // Open a new WebSocket connection to the address + let stream = P::Client::connect(address.clone(), "data").await; + + let Some(stream) = stream else { + panic!("Failed to connect to data server at {address:?}"); + }; + + let stream = Arc::new(Mutex::new(stream)); + streams.insert(address.clone(), stream.clone()); + + stream + } + } + } + + /// Get the requested exposed tensor data, and update download counter + async fn get_exposed_tensor_bytes( + &self, + transfer_id: TensorTransferId, + ) -> Option { + loop { + { + let mut exposed_tensors = self.exposed_tensors.lock().await; + // take the tensor out of the hashmap while we download + if let Some(mut exposed_state) = exposed_tensors.remove(&transfer_id) { + exposed_state.cur_download_count += 1; + let bytes = if exposed_state.cur_download_count == exposed_state.max_downloads { + exposed_state.bytes + } else { + let bytes = exposed_state.bytes.clone(); + exposed_tensors.insert(transfer_id, exposed_state); + bytes + }; + return Some(bytes); + } + } + // No matching tensor, wait for a new one to come in. + self.new_tensor_notify.notified().await; + } + } + + /// Handle incoming connections for downloading tensors. + pub(crate) async fn handle_data_channel( + &self, + mut channel: ::Channel, + ) { + log::info!("[Data Handler] New connection for download."); + + while !self.cancel_token.is_cancelled() { + match channel.recv().await { + Ok(message) => { + if let Some(msg) = message { + let bytes = msg.data; + let msg: DataServiceMessage = rmp_serde::from_slice(&bytes) + .expect("Can deserialize messages from the websocket."); + let DataServiceMessage::TensorRequest(transfer_id) = msg else { + panic!("Received a message that wasn't a tensor request! {msg:?}"); + }; + + let bytes = self.get_exposed_tensor_bytes(transfer_id).await.unwrap(); + + channel.send(Message::new(bytes)).await.unwrap(); + } else { + log::info!("Closed connection"); + return; + } + } + Err(err) => panic!("Failed to receive message from websocket: {err:?}"), + }; + } + log::info!("[Data Service] Closing connection for download."); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/lib.rs new file mode 100644 index 0000000..afa8c39 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/lib.rs @@ -0,0 +1,13 @@ +#[macro_use] +extern crate derive_new; + +mod base; +pub use base::*; + +pub mod util; + +#[cfg(feature = "websocket")] +pub mod websocket; + +#[cfg(feature = "data-service")] +pub mod data_service; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/util.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/util.rs new file mode 100644 index 0000000..7b2b909 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/util.rs @@ -0,0 +1,46 @@ +use tracing_core::{Level, LevelFilter}; +use tracing_subscriber::{ + Layer, filter::filter_fn, layer::SubscriberExt, registry, util::SubscriberInitExt, +}; + +/// Utilities to help handle communication termination. +pub async fn os_shutdown_signal() { + let ctrl_c = async { + tokio::signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } +} + +pub(crate) fn init_logging() { + let layer = tracing_subscriber::fmt::layer() + .with_filter(LevelFilter::INFO) + .with_filter(filter_fn(|m| { + if let Some(path) = m.module_path() { + // The wgpu crate is logging too much, so we skip `info` level. + if path.starts_with("wgpu") && *m.level() >= Level::INFO { + return false; + } + } + true + })); + + // If we start multiple servers in the same process, this will fail, it's ok + let _ = registry().with(layer).try_init(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/base.rs new file mode 100644 index 0000000..ffe919b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/base.rs @@ -0,0 +1,35 @@ +use crate::{ + base::{Address, Protocol}, + websocket::{client::WsClient, server::WsServer}, +}; + +#[derive(Clone)] +/// A websocket implements a [communication protocol](Protocol) that can be used to communicate +/// over the internet. +pub struct WebSocket {} + +impl Protocol for WebSocket { + type Client = WsClient; + type Server = WsServer; +} + +/// Parse an address, add the ws:// prefix if needed, and return an error if the address is invalid +pub(crate) fn parse_ws_address(mut address: Address) -> Result { + let s = &address.inner; + let parts = s.split("://").collect::>(); + let num_parts = parts.len(); + let url = if num_parts == 2 { + if parts[0] == "ws" { + s.to_owned() + } else { + return Err(format!("Invalid prefix: {}", parts[0])); + } + } else if num_parts == 1 { + return Err(format!("ws://{s}")); + } else { + return Err(format!("Invalid url: {s}")); + }; + + address.inner = url; + Ok(address) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/client.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/client.rs new file mode 100644 index 0000000..feaa4d1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/client.rs @@ -0,0 +1,109 @@ +use crate::{ + base::{Address, CommunicationChannel, CommunicationError, Message, ProtocolClient}, + websocket::base::parse_ws_address, +}; +use burn_std::future::DynFut; +use futures::{SinkExt, StreamExt}; +use tokio::net::TcpStream; +use tokio_tungstenite::{ + MaybeTlsStream, WebSocketStream, connect_async_with_config, + tungstenite::{self, protocol::WebSocketConfig}, +}; + +#[derive(Clone)] +pub struct WsClient; + +impl ProtocolClient for WsClient { + type Channel = WsClientChannel; + type Error = WsClientError; + + fn connect(address: Address, route: &str) -> DynFut> { + Box::pin(connect_ws(address, route.to_owned())) + } +} + +/// Open a new WebSocket connection to the address +async fn connect_ws(address: Address, route: String) -> Option { + let address = parse_ws_address(address).ok()?; + let address = format!("{address}/{route}"); + const MB: usize = 1024 * 1024; + let (stream, _) = connect_async_with_config( + address.clone(), + Some( + WebSocketConfig::default() + .write_buffer_size(0) + .max_message_size(None) + .max_frame_size(Some(MB * 512)) + .accept_unmasked_frames(true) + .read_buffer_size(64 * 1024), // 64 KiB (previous default) + ), + true, + ) + .await + .ok()?; + + Some(WsClientChannel { inner: stream }) +} +pub struct WsClientChannel { + inner: WebSocketStream>, +} + +impl CommunicationChannel for WsClientChannel { + type Error = WsClientError; + + async fn send(&mut self, msg: Message) -> Result<(), WsClientError> { + self.inner + .send(tungstenite::Message::Binary(msg.data)) + .await?; + + Ok(()) + } + + async fn recv(&mut self) -> Result, WsClientError> { + match self.inner.next().await { + Some(next) => match next { + Ok(tungstenite::Message::Binary(data)) => Ok(Some(Message { data })), + Ok(tungstenite::Message::Close(_close_frame)) => Ok(None), + Err(err) => Err(WsClientError::Tungstenite(err)), + msg => Err(WsClientError::UnknownMessage(format!("{msg:?}"))), + }, + None => todo!(), + } + } + + async fn close(&mut self) -> Result<(), WsClientError> { + let reason = "Peer is closing".to_string(); + + self.inner + .send(tungstenite::Message::Close(Some( + tungstenite::protocol::CloseFrame { + code: tungstenite::protocol::frame::coding::CloseCode::Normal, + reason: reason.clone().into(), + }, + ))) + .await?; + + Ok(()) + } +} + +#[derive(Debug)] +pub enum WsClientError { + Io(std::io::Error), + Tungstenite(tungstenite::Error), + UnknownMessage(String), + Other(String), +} +impl CommunicationError for WsClientError {} + +impl From for WsClientError { + fn from(err: std::io::Error) -> Self { + Self::Io(err) + } +} + +impl From for WsClientError { + fn from(err: tungstenite::Error) -> Self { + Self::Tungstenite(err) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/mod.rs new file mode 100644 index 0000000..ca64da1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/mod.rs @@ -0,0 +1,7 @@ +mod base; +mod client; +mod server; + +pub use base::*; +pub use client::*; +pub use server::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/server.rs b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/server.rs new file mode 100644 index 0000000..1f8b5fa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-communication/src/websocket/server.rs @@ -0,0 +1,141 @@ +use std::net::SocketAddr; + +use crate::{ + base::{CommunicationChannel, CommunicationError, Message, ProtocolServer}, + util::init_logging, +}; +use axum::{ + Router, + extract::{ + State, WebSocketUpgrade, + ws::{self, WebSocket}, + }, + routing::get, +}; +use futures::StreamExt; + +#[derive(Clone, Debug)] +pub struct WsServer { + port: u16, + router: Router<()>, +} + +pub struct WsServerChannel { + inner: WebSocket, +} + +impl WsServer { + pub fn new(port: u16) -> Self { + Self { + port, + router: Router::new(), + } + } +} + +impl ProtocolServer for WsServer { + type Channel = WsServerChannel; + type Error = WsServerError; + + async fn serve(self, shutdown: F) -> Result<(), Self::Error> + where + F: Future + Send + 'static, + { + init_logging(); + + let address = format!("0.0.0.0:{}", self.port); + log::info!("Starting server {address}"); + + let listener = tokio::net::TcpListener::bind(address).await?; + + axum::serve( + listener, + self.router + .into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown) + .await?; + + Ok(()) + } + + fn route(mut self, path: &str, callback: C) -> Self + where + C: FnOnce(WsServerChannel) -> Fut + Clone + Send + Sync + 'static, + Fut: Future + Send + 'static, + { + // Format path: should start with a / + let path = if path.starts_with("/") { + path.to_owned() + } else { + format!("/{path}") + }; + + let method = get(|ws: WebSocketUpgrade, _: State<()>| async { + ws.on_upgrade(async move |socket| { + callback(WsServerChannel { inner: socket }).await; + }) + }); + + self.router = self.router.route(&path, method); + + self + } +} + +impl CommunicationChannel for WsServerChannel { + type Error = WsServerError; + + async fn send(&mut self, message: Message) -> Result<(), WsServerError> { + self.inner.send(ws::Message::Binary(message.data)).await?; + + Ok(()) + } + + async fn recv(&mut self) -> Result, WsServerError> { + match self.inner.next().await { + Some(next) => match next { + Ok(ws::Message::Binary(data)) => Ok(Some(Message { data })), + Ok(ws::Message::Close(_close_frame)) => Ok(None), + Err(err) => Err(WsServerError::Axum(err)), + msg => Err(WsServerError::UnknownMessage(format!("{msg:?}"))), + }, + None => todo!(), + } + } + + async fn close(&mut self) -> Result<(), WsServerError> { + let reason = "Peer is closing".to_string(); + + self.inner + .send(ws::Message::Close(Some(ws::CloseFrame { + code: 1000, // code: Normal + reason: reason.clone().into(), + }))) + .await?; + + Ok(()) + } +} + +#[derive(Debug)] +pub enum WsServerError { + Io(std::io::Error), + Axum(axum::Error), + UnknownMessage(String), + Other(String), +} + +impl CommunicationError for WsServerError {} + +impl From for WsServerError { + fn from(err: std::io::Error) -> Self { + Self::Io(err) + } +} + +impl From for WsServerError { + fn from(err: axum::Error) -> Self { + Self::Axum(err) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-core/Cargo.toml new file mode 100644 index 0000000..3c34ac0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/Cargo.toml @@ -0,0 +1,151 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Flexible and Comprehensive Deep Learning Framework in Rust" +documentation = "https://docs.rs/burn-core" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-core" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-core" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [ + "std", + "burn-std/default", + "burn-dataset?/default", + "burn-tensor/default", +] +doc = [ + "std", + "dataset", + "audio", + # Doc features + "burn-std/doc", + "burn-dataset/doc", + "burn-tensor/doc", +] +tracing = [ + "burn-std/tracing", + "burn-tensor/tracing", + "burn-dataset?/tracing", + "burn-vision?/tracing", +] + + +dataset = ["burn-dataset"] + +network = ["burn-std/network"] +sqlite = ["burn-dataset?/sqlite"] +sqlite-bundled = ["burn-dataset?/sqlite-bundled"] +std = [ + "bincode/std", + "burn-std/std", + "burn-tensor/std", + "flate2", + "half/std", + "log", + "rand/std", + "rmp-serde", + "serde/std", + "serde_json/std", + "num-traits/std", +] +vision = ["burn-vision", "burn-dataset?/vision"] +audio = ["burn-dataset?/audio"] + +# Custom deserializer for Record that is helpful for importing data, such as PyTorch pt files. +record-item-custom-serde = ["thiserror"] + +# Serialization formats +experimental-named-tensor = ["burn-tensor/experimental-named-tensor"] + +test-cuda = [ + "burn-cuda/default", +] # To use cuda during testing, default uses ndarray. +test-rocm = [ + "burn-rocm/default", +] # To use hip during testing, default uses ndarray. +test-tch = [ + "burn-tch/default", +] # To use tch during testing, default uses ndarray. +test-wgpu = [ + "burn-wgpu/default", +] # To use wgpu during testing, default uses ndarray. +test-vulkan = [ + "test-wgpu", + "burn-wgpu/vulkan", +] # To use wgpu-spirv during testing, default uses ndarray. +test-metal = [ + "test-wgpu", + "burn-wgpu/metal", +] # To use wgpu-spirv during testing, default uses ndarray. + +# Memory checks are disabled by default +test-memory-checks = ["burn-fusion/memory-checks"] + +[dependencies] + +# ** Please make sure all dependencies support no_std when std is disabled ** + +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } +burn-dataset = { path = "../burn-dataset", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-derive = { path = "../burn-derive", version = "=0.21.0-pre.2" } +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", default-features = false } +burn-vision = { path = "../burn-vision", version = "=0.21.0-pre.2", optional = true, default-features = false } + +data-encoding = { workspace = true } +uuid = { workspace = true } + +derive-new = { workspace = true } +log = { workspace = true, optional = true } +rand = { workspace = true } + +# The same implementation of HashMap in std but with no_std support (only alloc crate is needed) +hashbrown = { workspace = true, features = ["serde"] } # no_std compatible + +# Serialize Deserialize +flate2 = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"] } + +ahash = { workspace = true } +bincode = { workspace = true } +half = { workspace = true } +num-traits = { workspace = true } +rmp-serde = { workspace = true, optional = true } +serde_json = { workspace = true, features = ["alloc"] } #Default enables std +spin = { workspace = true } # Using in place of use std::sync::Mutex when std is disabled +thiserror = { workspace = true, optional = true } + +[target.'cfg(target_has_atomic = "ptr")'.dependencies] +regex = { workspace = true } + +# FOR TESTING +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-rocm = { path = "../burn-rocm", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-remote = { path = "../burn-remote", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-router = { path = "../burn-router", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +portable-atomic-util = { workspace = true } +portable-atomic = { workspace = true } + +[dev-dependencies] +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2" } +burn-dataset = { path = "../burn-dataset", version = "=0.21.0-pre.2", features = [ + "fake", +] } +rstest = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/README.md b/crates/stable-diffusion-burn/burn-crates/burn-core/README.md new file mode 100644 index 0000000..166396f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/README.md @@ -0,0 +1,15 @@ +# Burn Core + +This crate should be used with [burn](https://github.com/tracel-ai/burn). It contains the core +traits and components for building and training deep learning models with Burn. + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-core.svg)](https://crates.io/crates/burn-core) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-core/blob/master/README.md) + +## Feature Flags + +This crate can be used without the standard library (`#![no_std]`) with `alloc` by disabling the +default `std` feature. + +- `std` - enables the standard library. Enabled by default. +- `experimental-named-tensor` - enables experimental named tensor. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/config.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/config.rs new file mode 100644 index 0000000..0b9d5c9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/config.rs @@ -0,0 +1,98 @@ +use alloc::{format, string::String, string::ToString}; +pub use burn_derive::Config; +use core::fmt::Debug; + +/// Configuration IO error. +#[derive(Debug)] +pub enum ConfigError { + /// Invalid format. + InvalidFormat(String), + + /// File not found. + FileNotFound(String), +} + +impl core::fmt::Display for ConfigError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut message = "Config error => ".to_string(); + + match self { + Self::InvalidFormat(err) => { + message += format!("Invalid format: {err}").as_str(); + } + Self::FileNotFound(err) => { + message += format!("File not found: {err}").as_str(); + } + }; + + f.write_str(message.as_str()) + } +} + +impl core::error::Error for ConfigError {} + +/// Configuration trait. +pub trait Config: Debug + serde::Serialize + serde::de::DeserializeOwned { + /// Saves the configuration to a file. + /// + /// # Arguments + /// + /// * `file` - File to save the configuration to. + /// + /// # Returns + /// + /// The output of the save operation. + #[cfg(feature = "std")] + fn save>(&self, file: P) -> std::io::Result<()> { + std::fs::write(file, config_to_json(self)) + } + + /// Loads the configuration from a file. + /// + /// # Arguments + /// + /// * `file` - File to load the configuration from. + /// + /// # Returns + /// + /// The loaded configuration. + #[cfg(feature = "std")] + fn load>(file: P) -> Result { + let content = std::fs::read_to_string(file.as_ref()) + .map_err(|_| ConfigError::FileNotFound(file.as_ref().to_string_lossy().to_string()))?; + config_from_str(&content) + } + + /// Loads the configuration from a binary buffer. + /// + /// # Arguments + /// + /// * `data` - Binary buffer to load the configuration from. + /// + /// # Returns + /// + /// The loaded configuration. + fn load_binary(data: &[u8]) -> Result { + let content = core::str::from_utf8(data).map_err(|_| { + ConfigError::InvalidFormat("Could not parse data as utf-8.".to_string()) + })?; + config_from_str(content) + } +} + +/// Converts a configuration to a JSON string. +/// +/// # Arguments +/// +/// * `config` - Configuration to convert. +/// +/// # Returns +/// +/// The JSON string. +pub fn config_to_json(config: &C) -> String { + serde_json::to_string_pretty(config).unwrap() +} + +fn config_from_str(content: &str) -> Result { + serde_json::from_str(content).map_err(|err| ConfigError::InvalidFormat(format!("{err}"))) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/base.rs new file mode 100644 index 0000000..5db4c18 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/base.rs @@ -0,0 +1,49 @@ +use burn_tensor::backend::Backend; + +pub use crate::data::dataset::{Dataset, DatasetIterator}; +use core::iter::Iterator; +use std::sync::Arc; + +/// A progress struct that can be used to track the progress of a data loader. +#[derive(new, Clone, Debug)] +pub struct Progress { + /// The number of items that have been processed. + pub items_processed: usize, + + /// The total number of items that need to be processed. + pub items_total: usize, +} + +/// A data loader iterator that can be used to iterate over a data loader. +pub trait DataLoaderIterator: Iterator { + /// Returns the progress of the data loader. + fn progress(&self) -> Progress; +} + +/// A data loader that can be used to iterate over a dataset. +pub trait DataLoader: Send + Sync { + /// Returns a boxed [iterator](DataLoaderIterator) to iterate over the data loader. + fn iter<'a>(&'a self) -> Box + 'a>; + + /// The number of items (not the number of batches nor the number of iterations), + /// corresponding to the items_total of the progress returned by the iterator. + fn num_items(&self) -> usize; + + /// Move the data loader to the given device, ensuring the batches are assigned to the correct device. + fn to_device(&self, device: &B::Device) -> Arc>; + + /// Returns a new data loader containing a subset of the data. + /// + /// The subset includes items from `start` (inclusive) to `end` (exclusive), + /// preserving the batch size and ordering of the original data loader. + /// + /// # Arguments + /// + /// * `start` - The starting index of the subset (inclusive). + /// * `end` - The ending index of the subset (exclusive). + /// + /// # Returns + /// + /// A boxed [`DataLoader`] instance containing only the specified range. + fn slice(&self, start: usize, end: usize) -> Arc>; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batch.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batch.rs new file mode 100644 index 0000000..29ca7cc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batch.rs @@ -0,0 +1,259 @@ +use super::{BatchStrategy, DataLoader, DataLoaderIterator, Progress, batcher::Batcher}; +use burn_dataset::{ + Dataset, + transform::{PartialDataset, ShuffledDataset}, +}; +use burn_tensor::backend::Backend; +use rand::SeedableRng; +use std::ops::DerefMut; +use std::sync::Arc; + +/// A data loader that can be used to iterate over a dataset in batches. +pub struct BatchDataLoader { + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + device: B::Device, + rng: Option>>, +} + +impl Clone for BatchDataLoader { + fn clone(&self) -> Self { + Self { + strategy: self.strategy.clone_dyn(), + dataset: self.dataset.clone(), + batcher: self.batcher.clone(), + device: self.device.clone(), + rng: self.rng.clone(), + } + } +} + +impl BatchDataLoader { + /// Creates a new batch data loader. + /// + /// # Arguments + /// + /// * `strategy` - The batch strategy. + /// * `dataset` - The dataset. + /// * `batcher` - The batcher. + /// * `device` - The device to use when loading a batch. + /// * `rng` - The rng determining if the dataset is shuffled each time a dataloader + /// iterator is created. + /// + /// # Returns + /// + /// The batch data loader. + pub fn new( + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + device: B::Device, + rng: Option, + ) -> Self { + Self { + strategy, + dataset, + batcher, + device, + rng: rng.map(|rng| Arc::new(spin::Mutex::new(rng))), + } + } +} + +/// A data loader iterator that can be used to iterate over a data loader. +struct BatchDataloaderIterator { + current_index: usize, + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + device: B::Device, +} + +impl DataLoader for BatchDataLoader +where + B: Backend, + I: Send + Sync + Clone + 'static, + O: Send + 'static, +{ + fn iter<'a>(&'a self) -> Box + 'a> { + // When starting a new iteration, we first check if the dataloader was created with an rng, + // implying that we should shuffle the dataset beforehand, while advancing the current + // rng to ensure that each new iteration shuffles the dataset differently. + let dataset = match &self.rng { + Some(rng) => Arc::new(ShuffledDataset::new( + self.dataset.clone(), + rng.lock().deref_mut(), + )), + None => self.dataset.clone(), + }; + Box::new(BatchDataloaderIterator::new( + self.strategy.clone_dyn(), + dataset, + self.batcher.clone(), + self.device.clone(), + )) + } + + fn num_items(&self) -> usize { + self.dataset.len() + } + + fn to_device(&self, device: &B::Device) -> Arc> { + let rng = self.rng.as_ref().map(|rng| { + let mut rng = rng.lock(); + rng.fork() + }); + Arc::new(Self::new( + self.strategy.clone_dyn(), + self.dataset.clone(), + self.batcher.clone(), + device.clone(), + rng, + )) + } + + fn slice(&self, start: usize, end: usize) -> Arc> { + let rng = self.rng.as_ref().map(|rng| { + let mut rng = rng.lock(); + rng.fork() + }); + let dataloader = Self::new( + self.strategy.clone_dyn(), + Arc::new(PartialDataset::new(self.dataset.clone(), start, end)), + self.batcher.clone(), + self.device.clone(), + rng, + ); + Arc::new(dataloader) + } +} + +impl BatchDataloaderIterator { + /// Creates a new batch data loader iterator. + /// + /// # Arguments + /// + /// * `strategy` - The batch strategy. + /// * `dataset` - The dataset. + /// * `batcher` - The batcher. + /// * `device` - The device to use when loading a batch. + /// + /// # Returns + /// + /// The batch data loader iterator. + pub fn new( + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + device: B::Device, + ) -> Self { + BatchDataloaderIterator { + current_index: 0, + strategy, + dataset, + batcher, + device, + } + } +} + +impl Iterator for BatchDataloaderIterator { + type Item = O; + + fn next(&mut self) -> Option { + while let Some(item) = self.dataset.get(self.current_index) { + self.current_index += 1; + self.strategy.add(item); + + if let Some(items) = self.strategy.batch(false) { + return Some(self.batcher.batch(items, &self.device)); + } + } + + if let Some(items) = self.strategy.batch(true) { + return Some(self.batcher.batch(items, &self.device)); + } + + None + } +} + +impl DataLoaderIterator for BatchDataloaderIterator { + fn progress(&self) -> Progress { + Progress::new(self.current_index, self.dataset.len()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + use crate::data::dataloader::FixBatchStrategy; + use crate::data::dataloader::batcher::TestBatcher; + use crate::data::dataset::FakeDataset; + + #[test] + fn test_batch_dataloader() { + let batcher = Arc::new(TestBatcher::new()); + let dataset = Arc::new(FakeDataset::::new(27)); + let dataloader = BatchDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset.clone(), + batcher, + Default::default(), + None, + ); + + let mut items_dataset = HashSet::new(); + let mut items_dataloader = HashSet::new(); + + for item in dataset.iter() { + items_dataset.insert(item); + } + + for items in dataloader.iter() { + for item in items { + items_dataloader.insert(item); + } + } + + assert_eq!(items_dataset, items_dataloader); + } + + #[test] + fn test_batch_dataloader_slice() { + let batcher = Arc::new(TestBatcher::new()); + let dataset = Arc::new(FakeDataset::::new(27)); + let dataloader = BatchDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset.clone(), + batcher, + Default::default(), + None, + ); + let dataloader_slice = dataloader.slice(5, 15); + + let mut items_dataloader = HashSet::new(); + let mut items_dataloader_slice = HashSet::new(); + + let mut idx = 0; + for items in dataloader.iter() { + for item in items { + if (5..15).contains(&idx) { + items_dataloader.insert(item); + } + idx += 1; + } + } + + for items in dataloader_slice.iter() { + for item in items { + items_dataloader_slice.insert(item); + } + } + + assert_eq!(items_dataloader, items_dataloader_slice); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batcher.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batcher.rs new file mode 100644 index 0000000..bf8009e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/batcher.rs @@ -0,0 +1,31 @@ +use burn_tensor::backend::Backend; + +#[cfg(test)] +use crate::TestBackend; + +/// A trait for batching items of type `I` into items of type `O`. +pub trait Batcher: Send + Sync { + /// Batches the given items on the specified device. + /// + /// # Arguments + /// + /// * `items` - The items to batch. + /// * `device` - The backend device to use. + /// + /// # Returns + /// + /// The batched items. + fn batch(&self, items: Vec, device: &B::Device) -> O; +} + +/// Test batcher +#[cfg(test)] +#[derive(new, Clone)] +pub struct TestBatcher; + +#[cfg(test)] +impl Batcher> for TestBatcher { + fn batch(&self, items: Vec, _device: &::Device) -> Vec { + items + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/builder.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/builder.rs new file mode 100644 index 0000000..4619450 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/builder.rs @@ -0,0 +1,260 @@ +use super::{ + BatchDataLoader, BatchStrategy, DataLoader, FixBatchStrategy, MultiThreadDataLoader, + batcher::Batcher, +}; +use burn_dataset::Dataset; +use burn_tensor::backend::Backend; +use rand::{SeedableRng, rngs::StdRng}; +use std::sync::Arc; + +/// A builder for data loaders. +pub struct DataLoaderBuilder { + strategy: Option>>, + batcher: Arc>, + num_threads: Option, + shuffle: Option, + device: Option, +} + +impl DataLoaderBuilder +where + B: Backend, + I: Send + Sync + Clone + std::fmt::Debug + 'static, + O: Send + Clone + std::fmt::Debug + 'static, +{ + /// Creates a new data loader builder. + /// + /// # Arguments + /// + /// * `batcher` - The batcher. + /// + /// # Returns + /// + /// The data loader builder. + pub fn new(batcher: Bt) -> Self + where + Bt: Batcher + 'static, + { + Self { + batcher: Arc::new(batcher), + strategy: None, + num_threads: None, + shuffle: None, + device: None, + } + } + + /// Sets the batch size to a fix number. + /// + /// The [fix batch strategy](FixBatchStrategy) will be used. + /// + /// # Arguments + /// + /// * `batch_size` - The batch size. + /// + /// # Returns + /// + /// The data loader builder. + pub fn batch_size(mut self, batch_size: usize) -> Self { + self.strategy = Some(Box::new(FixBatchStrategy::new(batch_size))); + self + } + + /// Sets the seed for shuffling. + /// + /// Each time the dataloader starts a new iteration, the dataset will be shuffled. + /// + /// # Arguments + /// + /// * `seed` - The seed. + /// + /// # Returns + /// + /// The data loader builder. + pub fn shuffle(mut self, seed: u64) -> Self { + self.shuffle = Some(seed); + self + } + + /// Sets the number of workers. + /// + /// - `Some(0)` or `None`: the dataloader will run without work threads. + /// - `Some(n); n > 0`: the dataloader will run with `n` background threads. + /// + /// A 1-worker threaded dataloader will run loads in a background thread, + /// while a 0-worker threaded dataloader will run loads in the main thread. + /// + /// # Arguments + /// + /// * `num_workers` - The number of workers. + /// + /// # Returns + /// + /// The data loader builder. + pub fn num_workers(mut self, num_workers: usize) -> Self { + self.num_threads = Some(num_workers); + self + } + + /// Sets the data loader device. + /// + /// # Arguments + /// + /// * `device` - The device to use when loading a batch. + /// + /// # Returns + /// + /// The data loader builder. + pub fn set_device(mut self, device: B::Device) -> Self { + self.device = Some(device); + self + } + + /// Builds the data loader. + /// + /// # Arguments + /// + /// * `dataset` - The dataset. + /// + /// # Returns + /// + /// The data loader. + pub fn build(self, dataset: D) -> Arc> + where + D: Dataset + 'static, + { + let dataset = Arc::new(dataset); + + let device = self.device.unwrap_or_default(); + let rng = self.shuffle.map(StdRng::seed_from_u64); + let strategy = match self.strategy { + Some(strategy) => strategy, + None => Box::new(FixBatchStrategy::new(1)), + }; + + if let Some(num_threads) = self.num_threads + && num_threads > 0 + { + return Arc::new(MultiThreadDataLoader::new( + strategy, + dataset, + self.batcher, + num_threads, + device, + rng, + )); + } + + Arc::new(BatchDataLoader::new( + strategy, + dataset, + self.batcher, + device, + rng, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use crate::data::dataset::FakeDataset; + + #[derive(new, Clone)] + struct TestBatcherDevice; + + #[cfg(test)] + impl Batcher for TestBatcherDevice { + fn batch(&self, _items: Vec, device: &TestDevice) -> TestDevice { + *device + } + } + + type TestDevice = ::Device; + + #[test] + fn test_dataloader_no_workers() { + type TestDevice = ::Device; + + let default_device = TestDevice::default(); + let dataloader = DataLoaderBuilder::new(TestBatcherDevice::new()) + .batch_size(1) + .build(FakeDataset::::new(9)); + + assert_eq!(dataloader.num_items(), 9); + + for device in dataloader.iter() { + assert_eq!(device, default_device) + } + } + + #[test] + fn test_dataloader_default_device() { + let default_device = TestDevice::default(); + let dataloader = DataLoaderBuilder::new(TestBatcherDevice::new()) + .batch_size(1) + .num_workers(1) + .build(FakeDataset::::new(9)); + + assert_eq!(dataloader.num_items(), 9); + + for device in dataloader.iter() { + assert_eq!(device, default_device) + } + } + + #[test] + fn test_dataloader_slice_multi_device() { + let dataloader = DataLoaderBuilder::new(TestBatcherDevice::new()) + .batch_size(1) + .num_workers(1) + .build(FakeDataset::::new(11)); + + #[cfg(all( + test, + not(feature = "test-tch"), + not(feature = "test-wgpu"), + not(feature = "test-cuda") + ))] + // Only one device exists... + let (device1, device2) = ( + burn_ndarray::NdArrayDevice::Cpu, + burn_ndarray::NdArrayDevice::Cpu, + ); + + #[cfg(all(test, feature = "test-tch"))] + let (device1, device2) = ( + burn_tch::LibTorchDevice::Cuda(0), + burn_tch::LibTorchDevice::Cuda(1), + ); + + #[cfg(all(test, feature = "test-wgpu"))] + let (device1, device2) = ( + burn_wgpu::WgpuDevice::DiscreteGpu(0), + burn_wgpu::WgpuDevice::DiscreteGpu(1), + ); + + #[cfg(all(test, feature = "test-cuda"))] + let (device1, device2) = (burn_cuda::CudaDevice::new(0), burn_cuda::CudaDevice::new(1)); + + assert_eq!(dataloader.num_items(), 11); + let dataloader_1 = dataloader.slice(0, 5).to_device(&device1); + let dataloader_2 = dataloader.slice(5, 11).to_device(&device2); + + assert_eq!(dataloader_1.num_items(), 5); + assert_eq!(dataloader_2.num_items(), 6); + + let (mut iterator_1, mut iterator_2) = (dataloader_1.iter(), dataloader_2.iter()); + + for _ in 0..5 { + assert_eq!(iterator_1.next(), Some(device1)); + assert_eq!(iterator_2.next(), Some(device2)); + } + + assert_eq!(iterator_1.next(), None); + // For uneven split, the last dataloader (partial dataset) will have the remaining item + assert_eq!(iterator_2.next(), Some(device2)); + assert_eq!(iterator_2.next(), None); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/mod.rs new file mode 100644 index 0000000..52556af --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/mod.rs @@ -0,0 +1,16 @@ +mod base; +mod batch; +mod builder; +mod multithread; +mod strategy; + +/// Module for batching items. +pub mod batcher; +/// Module to split a dataloader. +pub mod split; + +pub use base::*; +pub use batch::*; +pub use builder::*; +pub use multithread::*; +pub use strategy::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/multithread.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/multithread.rs new file mode 100644 index 0000000..2277be8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/multithread.rs @@ -0,0 +1,441 @@ +use burn_dataset::Dataset; +use burn_dataset::transform::PartialDataset; +use burn_tensor::backend::Backend; +use rand::distr::{Distribution, StandardUniform}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; + +use super::batcher::Batcher; +use super::{BatchDataLoader, BatchStrategy, DataLoader, DataLoaderIterator, Progress}; +use std::sync::{Arc, OnceLock, mpsc}; +use std::thread; + +const MAX_QUEUED_ITEMS: usize = 100; + +type RngSeed = ::Seed; + +/// A multi-threaded data loader that can be used to iterate over a dataset. +pub struct MultiThreadDataLoader { + // Configuration parameters needed for initialization + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + device: B::Device, + seed: Option, + num_threads: usize, + + // The lazily initialized data loaders + dataloaders: OnceLock>>, +} + +/// A message that can be sent between threads. +#[derive(Debug)] +pub enum Message { + /// A batch of items. + Batch(usize, O, Progress), + + /// The thread is done. + Done, +} + +struct MultiThreadsDataloaderIterator { + num_done: usize, + workers: Vec>, + receiver: mpsc::Receiver>, + progresses: Vec, +} + +impl MultiThreadDataLoader +where + I: Send + Sync + Clone + 'static, + O: Send + 'static, +{ + /// Creates a new multi-threaded batch data loader. + /// + /// # Arguments + /// + /// * `strategy` - The batch strategy. + /// * `dataset` - The dataset. + /// * `batcher` - The batcher. + /// * `num_threads` - The number of threads. + /// * `device` - The device to use when loading a batch. + /// * `rng` - The rng determining if the dataset is shuffled each time a dataloader + /// iterator is created. + /// + /// # Returns + /// + /// The multi-threaded batch data loader. + pub fn new( + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + num_threads: usize, + device: B::Device, + rng: Option, + ) -> Self { + let mut seed = None; + if let Some(mut rng) = rng { + // RNG stream splitting (not state cloning): derive a new seed from the RNG's output. + // This is exactly what `rng.fork()` does. + let mut s = RngSeed::default(); + rng.fill_bytes(&mut s); + + seed = Some(s); + } + Self::from_seed(strategy, dataset, batcher, num_threads, device, seed) + } + + fn from_seed( + strategy: Box>, + dataset: Arc>, + batcher: Arc>, + num_threads: usize, + device: B::Device, + seed: Option, + ) -> Self { + Self { + strategy, + dataset, + batcher, + num_threads, + device, + seed, + dataloaders: OnceLock::new(), + } + } + + /// Force initialization if needed. + fn initialize(&self) -> &[BatchDataLoader] { + self.dataloaders + .get_or_init(|| { + let mut dataset = self.dataset.clone(); + if let Some(seed) = self.seed.as_ref() { + // Pre-shuffle the dataset before split if shuffle is enabled. + // This ensures that each thread gets a uniform random sample of the dataset. + let mut rng = StdRng::from_seed(*seed); + dataset = Arc::new(burn_dataset::transform::ShuffledDataset::new( + dataset, &mut rng, + )); + } + + let datasets = match self.strategy.batch_size() { + Some(batch_size) => { + PartialDataset::split_chunks(dataset, self.num_threads, batch_size) + } + None => PartialDataset::split(dataset, self.num_threads), + }; + + // Create more rngs from the first one, one for each new dataloader. + let mut rng = self.seed.map(StdRng::from_seed); + let rngs = (0..self.num_threads).map(|_| { + rng.as_mut().map(|rng| { + StdRng::seed_from_u64(Distribution::sample(&StandardUniform, rng)) + }) + }); + + datasets + .into_iter() + .zip(rngs) + .map(|(dataset, rng)| { + let strategy = self.strategy.clone_dyn(); + BatchDataLoader::new( + strategy, + Arc::new(dataset), + self.batcher.clone(), + self.device.clone(), + rng, + ) + }) + .collect() + }) + .as_ref() + } +} + +impl DataLoader for MultiThreadDataLoader +where + I: Send + Sync + Clone + 'static, + O: Send + 'static + std::fmt::Debug, +{ + fn iter<'a>(&'a self) -> Box + 'a> { + // This will initialize the loader if it hasn't been initialized yet + let dataloaders = self.initialize(); + + let (sender, receiver) = mpsc::sync_channel::>(MAX_QUEUED_ITEMS); + + let mut progresses = Vec::with_capacity(dataloaders.len()); + + let handlers: Vec<_> = dataloaders + .iter() + .enumerate() + .map(|(index, dataloader)| { + let dataloader_cloned = dataloader.clone(); + let sender_cloned = sender.clone(); + progresses.push(Progress::new(0, dataloader_cloned.num_items())); + + thread::spawn(move || { + let mut iterator = dataloader_cloned.iter(); + while let Some(item) = iterator.next() { + let progress = iterator.progress(); + + match sender_cloned.send(Message::Batch(index, item, progress)) { + Ok(_) => {} + // The receiver is probably gone, no need to panic, just need to stop + // iterating. + Err(_) => return, + }; + } + // Same thing. + sender_cloned.send(Message::Done).ok(); + }) + }) + .collect(); + + Box::new(MultiThreadsDataloaderIterator::new( + receiver, handlers, progresses, + )) + } + + fn num_items(&self) -> usize { + // For num_items, we can directly use the dataset size without + // necessarily initializing the full loader + self.dataset.len() + } + + fn to_device(&self, device: &B::Device) -> Arc> { + Arc::new(Self::from_seed( + self.strategy.clone_dyn(), + self.dataset.clone(), + self.batcher.clone(), + self.num_threads, + device.clone(), + self.seed, + )) + } + + fn slice(&self, start: usize, end: usize) -> Arc> { + let dataloader = Self::from_seed( + self.strategy.clone_dyn(), + Arc::new(PartialDataset::new(self.dataset.clone(), start, end)), + self.batcher.clone(), + self.num_threads, + self.device.clone(), + self.seed, + ); + Arc::new(dataloader) + } +} + +impl MultiThreadsDataloaderIterator { + pub fn new( + receiver: mpsc::Receiver>, + workers: Vec>, + progresses: Vec, + ) -> Self { + MultiThreadsDataloaderIterator { + num_done: 0, + workers, + receiver, + progresses, + } + } +} +impl DataLoaderIterator for MultiThreadsDataloaderIterator { + fn progress(&self) -> Progress { + let mut items_total = 0; + let mut items_processed = 0; + + for progress in self.progresses.iter() { + items_total += progress.items_total; + items_processed += progress.items_processed; + } + + Progress::new(items_processed, items_total) + } +} + +impl Iterator for MultiThreadsDataloaderIterator { + type Item = O; + + fn next(&mut self) -> Option { + if self.workers.is_empty() { + return None; + } + + loop { + let item = self.receiver.recv(); + let item = item.unwrap(); + + match item { + Message::Batch(index, item, progress) => { + if let Some(current) = self.progresses.get_mut(index) { + *current = progress; + } + return Some(item); + } + Message::Done => { + self.num_done += 1; + } + }; + + if self.num_done == self.workers.len() { + while let Some(worker) = self.workers.pop() { + worker.join().unwrap(); + } + return None; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::data::dataloader::FixBatchStrategy; + use crate::data::dataloader::batcher::TestBatcher; + use crate::data::dataset::FakeDataset; + use burn_dataset::InMemDataset; + use std::collections::HashSet; + + #[test] + fn test_multi_thread_batch_dataloader() { + let batcher = Arc::new(TestBatcher::new()); + let dataset = Arc::new(FakeDataset::::new(27)); + let dataloader_single_thread = BatchDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset.clone(), + batcher.clone(), + Default::default(), + None, + ); + let dataloader_multi_thread = MultiThreadDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset, + batcher, + 4, + Default::default(), + None, + ); + + let mut items_single_thread = HashSet::new(); + let mut items_multi_thread = HashSet::new(); + + for items in dataloader_single_thread.iter() { + for item in items { + items_single_thread.insert(item); + } + } + + for items in dataloader_multi_thread.iter() { + for item in items { + items_multi_thread.insert(item); + } + } + + assert_eq!(items_single_thread, items_multi_thread); + } + + #[test] + fn test_multi_thread_batch_dataloader_shuffle() { + let num_classes = 2; + let class_size = 100; + let batch_size = 10; + + // Items is a deliberately ordered dataset. + let mut items = Vec::new(); + for class in 0..num_classes { + items.extend(vec![class; class_size]); + } + + { + // Unshuffled multithreaded loader + let dataset = Arc::new(InMemDataset::new(items.clone())); + let batcher = Arc::new(TestBatcher::new()); + + let loader = MultiThreadDataLoader::new( + Box::new(FixBatchStrategy::new(batch_size)), + dataset, + batcher, + num_classes, + Default::default(), + // No rng means no shuffling. + None, + ); + + for batch in loader.iter() { + let mut batch_items = HashSet::new(); + for item in batch { + batch_items.insert(item); + } + + // Since the dataset is not shuffled, we expect each batch to contain the same item. + assert_eq!(batch_items.len(), 1); + } + } + + { + // Shuffled multithreaded loader + let dataset = Arc::new(InMemDataset::new(items.clone())); + let batcher = Arc::new(TestBatcher::new()); + + let loader = MultiThreadDataLoader::new( + Box::new(FixBatchStrategy::new(batch_size)), + dataset.clone(), + batcher.clone(), + num_classes, + Default::default(), + // The rng enables shuffling. + Some(StdRng::seed_from_u64(42)), + ); + + for batch in loader.iter() { + let mut batch_items = HashSet::new(); + for item in batch { + batch_items.insert(item); + } + + // Since the dataset is shuffled, we expect to see all items. + assert_eq!(batch_items.len(), num_classes); + } + } + } + + #[test] + fn test_multi_thread_batch_dataloader_incomplete_batches() { + let batcher = Arc::new(TestBatcher::new()); + let dataset = Arc::new(FakeDataset::::new(27)); + let dataloader_single_thread = BatchDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset.clone(), + batcher.clone(), + Default::default(), + None, + ); + let dataloader_multi_thread = MultiThreadDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset, + batcher, + 4, + Default::default(), + None, + ); + + let mut items_single_thread = HashSet::new(); + let mut items_multi_thread = HashSet::new(); + + let mut single_thread_cnt = 0; + let mut multi_thread_cnt = 0; + for items in dataloader_single_thread.iter() { + items_single_thread.insert(items); + single_thread_cnt += 1; + } + + for items in dataloader_multi_thread.iter() { + items_multi_thread.insert(items); + multi_thread_cnt += 1; + } + + assert_eq!(single_thread_cnt, multi_thread_cnt); + assert_eq!(items_single_thread, items_multi_thread); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/split.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/split.rs new file mode 100644 index 0000000..6aff7e3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/split.rs @@ -0,0 +1,134 @@ +use std::sync::Arc; + +use burn_tensor::backend::Backend; + +use super::DataLoader; + +/// Splits a dataloader into multiple partial dataloaders (one per device). +pub fn split_dataloader( + dataloader: Arc>, + devices: &[B::Device], +) -> Vec>> { + let num_splits = devices.len(); + if num_splits > 1 { + let num_items = dataloader.num_items(); + let mut dataloaders = Vec::with_capacity(num_splits); + + let mut start = 0; + let step = num_items / num_splits; + for (i, device) in devices.iter().enumerate() { + let end = if i == (num_splits - 1) { + num_items + } else { + start + step + }; + let dataloader = dataloader.slice(start, end).to_device(device); + dataloaders.push(dataloader); + start = end; + } + dataloaders + } else { + vec![dataloader] + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + use crate::TestBackend; + use crate::data::dataloader::batcher::Batcher; + use crate::data::dataloader::{BatchDataLoader, FixBatchStrategy}; + use crate::data::dataset::FakeDataset; + + #[test] + fn test_split_batch_dataloader() { + type TestDevice = ::Device; + + #[derive(new, Clone)] + pub struct TestBatcher; + + #[cfg(test)] + impl Batcher, TestDevice)> for TestBatcher { + fn batch(&self, items: Vec, device: &TestDevice) -> (Vec, TestDevice) { + (items, *device) + } + } + + let batcher = Arc::new(TestBatcher::new()); + let dataset = Arc::new(FakeDataset::::new(11)); + + #[allow(clippy::arc_with_non_send_sync)] + let dataloader = Arc::new(BatchDataLoader::new( + Box::new(FixBatchStrategy::new(5)), + dataset.clone(), + batcher, + Default::default(), + None, + )); + + #[cfg(all( + test, + not(feature = "test-tch"), + not(feature = "test-wgpu"), + not(feature = "test-cuda") + ))] + // Only one device exists... + let (device1, device2) = ( + burn_ndarray::NdArrayDevice::Cpu, + burn_ndarray::NdArrayDevice::Cpu, + ); + + #[cfg(all(test, feature = "test-tch"))] + let (device1, device2) = ( + burn_tch::LibTorchDevice::Cuda(0), + burn_tch::LibTorchDevice::Cuda(1), + ); + + #[cfg(all(test, feature = "test-wgpu"))] + let (device1, device2) = ( + burn_wgpu::WgpuDevice::DiscreteGpu(0), + burn_wgpu::WgpuDevice::DiscreteGpu(1), + ); + + #[cfg(all(test, feature = "test-cuda"))] + let (device1, device2) = (burn_cuda::CudaDevice::new(0), burn_cuda::CudaDevice::new(1)); + + let dataloaders = split_dataloader(dataloader.clone(), &[device1, device2]); + + assert_eq!(dataloaders.len(), 2); + + let [dataloader_1, dataloader_2] = match dataloaders.try_into() { + Ok(arr) => arr, + Err(_) => unreachable!(), + }; + assert_eq!(dataloader_1.num_items(), 5); + assert_eq!(dataloader_2.num_items(), 6); + + let mut items_dataloader = HashSet::new(); + let mut items_dataloader_split = HashSet::new(); + + for (items, _device) in dataloader.iter() { + for item in items { + items_dataloader.insert(item); + } + } + + for (items, device) in dataloader_1.iter() { + assert_eq!(device, device1); + for item in items { + items_dataloader_split.insert(item); + } + } + + for (items, device) in dataloader_2.iter() { + assert_eq!(device, device2); + for item in items { + items_dataloader_split.insert(item); + } + } + + assert_eq!(items_dataloader, items_dataloader_split); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/strategy.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/strategy.rs new file mode 100644 index 0000000..d6e3413 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/dataloader/strategy.rs @@ -0,0 +1,87 @@ +/// A strategy to batch items. +pub trait BatchStrategy: Send + Sync { + /// Adds an item to the strategy. + /// + /// # Arguments + /// + /// * `item` - The item to add. + fn add(&mut self, item: I); + + /// Batches the items. + /// + /// # Arguments + /// + /// * `force` - Whether to force batching. + /// + /// # Returns + /// + /// The batched items. + fn batch(&mut self, force: bool) -> Option>; + + /// Creates a new strategy of the same type. + /// + /// # Returns + /// + /// The new strategy. + fn clone_dyn(&self) -> Box>; + + /// Returns the expected batch size for this strategy. + /// + /// # Returns + /// + /// The batch size, or None if the strategy doesn't have a fixed batch size. + fn batch_size(&self) -> Option; +} + +/// A strategy to batch items with a fixed batch size. +pub struct FixBatchStrategy { + items: Vec, + batch_size: usize, +} + +impl FixBatchStrategy { + /// Creates a new strategy to batch items with a fixed batch size. + /// + /// # Arguments + /// + /// * `batch_size` - The batch size. + /// + /// # Returns + /// + /// The strategy. + pub fn new(batch_size: usize) -> Self { + FixBatchStrategy { + items: Vec::with_capacity(batch_size), + batch_size, + } + } +} + +impl BatchStrategy for FixBatchStrategy { + fn add(&mut self, item: I) { + self.items.push(item); + } + + fn batch(&mut self, force: bool) -> Option> { + if self.items.len() < self.batch_size && !force { + return None; + } + + let mut items = Vec::with_capacity(self.batch_size); + std::mem::swap(&mut items, &mut self.items); + + if items.is_empty() { + return None; + } + + Some(items) + } + + fn clone_dyn(&self) -> Box> { + Box::new(Self::new(self.batch_size)) + } + + fn batch_size(&self) -> Option { + Some(self.batch_size) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/mod.rs new file mode 100644 index 0000000..8881032 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/data/mod.rs @@ -0,0 +1,15 @@ +/// Dataloader module. +#[cfg(feature = "dataset")] +pub mod dataloader; + +/// Dataset module. +#[cfg(feature = "dataset")] +pub mod dataset { + pub use burn_dataset::*; +} + +/// Network module. +#[cfg(feature = "network")] +pub mod network { + pub use burn_std::network::*; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/lib.rs new file mode 100644 index 0000000..fbab361 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/lib.rs @@ -0,0 +1,118 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![recursion_limit = "135"] + +//! The core crate of Burn. + +#[macro_use] +extern crate derive_new; + +/// Re-export serde for proc macros. +pub use serde; + +/// The configuration module. +pub mod config; + +/// Data module. +#[cfg(feature = "std")] +pub mod data; + +/// Module for the neural network module. +pub mod module; + +/// Module for the recorder. +pub mod record; + +/// Module for the tensor. +pub mod tensor; +// Tensor at root: `burn::Tensor` +pub use tensor::Tensor; + +/// Module for visual operations +#[cfg(feature = "vision")] +pub mod vision; + +extern crate alloc; + +/// Backend for test cases +#[cfg(all( + test, + not(feature = "test-tch"), + not(feature = "test-wgpu"), + not(feature = "test-cuda"), + not(feature = "test-rocm") +))] +pub type TestBackend = burn_ndarray::NdArray; + +#[cfg(all(test, feature = "test-tch"))] +/// Backend for test cases +pub type TestBackend = burn_tch::LibTorch; + +#[cfg(all(test, feature = "test-wgpu"))] +/// Backend for test cases +pub type TestBackend = burn_wgpu::Wgpu; + +#[cfg(all(test, feature = "test-cuda"))] +/// Backend for test cases +pub type TestBackend = burn_cuda::Cuda; + +#[cfg(all(test, feature = "test-rocm"))] +/// Backend for test cases +pub type TestBackend = burn_rocm::Rocm; + +/// Backend for autodiff test cases +#[cfg(test)] +pub type TestAutodiffBackend = burn_autodiff::Autodiff; + +#[cfg(all(test, feature = "test-memory-checks"))] +mod tests { + burn_fusion::memory_checks!(); +} + +#[cfg(test)] +mod test_utils { + use crate as burn; + use crate::module::Module; + use crate::module::Param; + use burn_tensor::Tensor; + use burn_tensor::backend::Backend; + + /// Simple linear module. + #[derive(Module, Debug)] + pub struct SimpleLinear { + pub weight: Param>, + pub bias: Option>>, + } + + impl SimpleLinear { + pub fn new(in_features: usize, out_features: usize, device: &B::Device) -> Self { + let weight = Tensor::random( + [out_features, in_features], + burn_tensor::Distribution::Default, + device, + ); + let bias = Tensor::random([out_features], burn_tensor::Distribution::Default, device); + + Self { + weight: Param::from_tensor(weight), + bias: Some(Param::from_tensor(bias)), + } + } + } +} + +pub mod prelude { + //! Structs and macros used by most projects. Add `use + //! burn::prelude::*` to your code to quickly get started with + //! Burn. + pub use crate::{ + config::Config, + module::Module, + tensor::{ + Bool, Device, ElementConversion, Float, Int, Shape, SliceArg, Tensor, TensorData, + backend::Backend, cast::ToElement, s, + }, + }; + pub use burn_std::device::Device as DeviceOps; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/base.rs new file mode 100644 index 0000000..f1853af --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/base.rs @@ -0,0 +1,470 @@ +use super::{Param, ParamId, Quantizer}; +use crate::{ + record::Record, + tensor::backend::{AutodiffBackend, Backend}, +}; +use alloc::{string::String, vec::Vec}; +pub use burn_derive::Module; +use burn_tensor::{Bool, Int, Tensor, ops::Device}; + +/// Type alias to `Vec` which supports `no_std` environments, but automatically using +/// the `alloc` crate. +pub type Devices = Vec>; + +// At the moment, our plan is to continue experimenting with the macro internally and monitor its development. +// We may consider making it public in the future. +macro_rules! module { + (map=$module:ident, ops=$item:expr) => {{ + struct Mapper; + impl ModuleMapper for Mapper { + fn map_float( + &mut self, + param: Param>, + ) -> Param> { + let (id, tensor, mapper) = param.consume(); + let func = $item; + let tensor = func(tensor); + Param::from_mapped_value(id, tensor, mapper) + } + } + let mut mapper = Mapper; + $module.map(&mut mapper) + }}; + (visit_float=$module:ident, ops=$item:expr, state=$state_ty:ty, init=$init:expr) => {{ + struct Visitor<'a, B: Backend> { + state: &'a mut $state_ty, + backend: core::marker::PhantomData, + } + impl<'a, B: Backend> ModuleVisitor for Visitor<'a, B> { + fn visit_float(&mut self, param: &Param>) { + let func = $item; + func(¶m.val(), &mut self.state) + } + } + #[allow(clippy::redundant_closure_call)] + let mut state = $init(); + let mut visitor = Visitor { + state: &mut state, + backend: core::marker::PhantomData, + }; + $module.visit(&mut visitor); + state + }}; +} + +/// Trait for all neural network modules. +/// +/// Modules should be created using the [derive](burn_derive::Module) attribute. +/// This will make your module trainable, savable and loadable via +/// `state` and `load`. +/// +/// # Example +/// +/// A module should have a [backend](crate::tensor::backend::Backend) defined as a generic +/// parameter B. This will be used by the [derive](burn_derive::Module) attribute to generate the code +/// necessary to optimize and train the module on any backend. +/// +/// ```rust, ignore +/// // Not necessary when using the burn crate directly. +/// use burn_core as burn; +/// +/// use burn::{ +/// module::Module, +/// nn::Linear, +/// tensor::Tensor, +/// tensor::backend::Backend, +/// }; +/// +/// #[derive(Module, Debug)] +/// struct MyModule { +/// my_param: Linear, +/// my_other_field: usize, +/// } +/// ``` +pub trait Module: Clone + Send + core::fmt::Debug { + /// Type to save and load the module. + type Record: Record; + + /// Return all the devices found in the underneath module tree added to the given vector + /// without duplicates. + fn collect_devices(&self, devices: Devices) -> Devices; + + /// Return all the devices found in the underneath module tree without duplicates. + fn devices(&self) -> Devices { + self.collect_devices(Devices::::new()) + } + + /// Fork the module and all of its sub-modules to the given device. + /// + /// # Notes + /// + /// This is similar to [to_device](Module::to_device), but it ensures the output module on the + /// new device will have its own autodiff graph. + fn fork(self, device: &B::Device) -> Self; + + /// Move the module and all of its sub-modules to the given device. + /// + /// # Warnings + /// + /// The operation supports autodiff and it will be registered when activated. However, this may + /// not be what you want. The output model will be an intermediary model, meaning that you + /// can't optimize it with gradient descent. If you want to optimize the output network on the + /// target device, use [fork](Module::fork) instead. + fn to_device(self, device: &B::Device) -> Self; + + /// Each tensor in the module tree will not require grad. + /// + /// # Warnings + /// + /// This should not be used for inference, use [valid](AutodiffModule::valid) when using + /// AD modules. This is mostly useful when performing partial finetuning, which is updating only + /// a small fraction of the parameters instead of finetuning all of them. + fn no_grad(self) -> Self { + module!( + map = self, + ops = |tensor: Tensor| tensor.set_require_grad(false) + ) + } + + /// Move the module and all of its sub-modules to the autodiff backend. + /// + /// # Notes + /// + /// * Only plain modules (not already on an autodiff backend) can be moved. + /// * Calling `train()` on a module that is already on an autodiff backend + /// will result in a type error, because the module's inner backend does not match. + fn train(self) -> >::TrainModule + where + AB: AutodiffBackend, + Self: HasAutodiffModule, + { + >::TrainModule::from_inner(self) + } + + /// Get the number of parameters the module has, including all of its sub-modules. + fn num_params(&self) -> usize { + module!( + visit_float = self, + ops = |tensor: &Tensor, state: &mut usize| { + *state += tensor.shape().num_elements(); + }, + state = usize, + init = || 0 + ) + } + /// Visit each tensor parameter in the module with a [visitor](ModuleVisitor). + fn visit>(&self, visitor: &mut Visitor); + + /// Map each tensor parameter in the module with a [mapper](ModuleMapper). + fn map>(self, mapper: &mut Mapper) -> Self; + + /// Load the module state from a record. + fn load_record(self, record: Self::Record) -> Self; + + /// Convert the module into a record containing the state. + fn into_record(self) -> Self::Record; + + #[cfg(feature = "std")] + /// Save the module to a file using the provided [file recorder](crate::record::FileRecorder). + /// + /// List of supported file recorders: + /// + /// * [default](crate::record::DefaultFileRecorder) + /// * [bincode](crate::record::BinFileRecorder) + /// * [bincode compressed with gzip](crate::record::BinGzFileRecorder) + /// * [json pretty](crate::record::PrettyJsonFileRecorder) + /// * [json compressed with gzip](crate::record::JsonGzFileRecorder) + /// * [named mpk](crate::record::NamedMpkFileRecorder) + /// * [named mpk compressed with gzip](crate::record::NamedMpkGzFileRecorder) + /// + /// ## Notes + /// + /// The file extension is automatically added depending on the file recorder provided, you + /// don't have to specify it. + fn save_file( + self, + file_path: PB, + recorder: &FR, + ) -> Result<(), crate::record::RecorderError> + where + FR: crate::record::FileRecorder, + PB: Into, + { + let record = Self::into_record(self); + recorder.record(record, file_path.into()) + } + + #[cfg(feature = "std")] + /// Load the module from a file using the provided [file recorder](crate::record::FileRecorder). + /// + /// The recorder should be the same as the one used to save the module, see + /// [save_file](Self::save_file). + /// + /// ## Notes + /// + /// The file extension is automatically added depending on the file recorder provided, you + /// don't have to specify it. + fn load_file( + self, + file_path: PB, + recorder: &FR, + device: &B::Device, + ) -> Result + where + FR: crate::record::FileRecorder, + PB: Into, + { + let record = recorder.load(file_path.into(), device)?; + + Ok(self.load_record(record)) + } + + /// Quantize the weights of the module. + fn quantize_weights(self, quantizer: &mut Quantizer) -> Self { + self.map(quantizer) + } +} + +/// Module visitor trait for traversing and inspecting module parameters. +pub trait ModuleVisitor { + /// Visit a float parameter in the module. + /// + /// # Parameters + /// - `param`: The float parameter to visit + #[allow(unused_variables)] + fn visit_float(&mut self, param: &Param>) {} + + /// Visit an int parameter in the module. + /// + /// # Parameters + /// - `param`: The integer parameter to visit + #[allow(unused_variables)] + fn visit_int(&mut self, param: &Param>) {} + + /// Visit a bool parameter in the module. + /// + /// # Parameters + /// - `param`: The boolean parameter to visit + #[allow(unused_variables)] + fn visit_bool(&mut self, param: &Param>) {} + + /// Called when entering a submodule. + /// + /// # Parameters + /// - `name`: The name of the submodule being entered + /// - `container_type`: The type of the container with format: + /// - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear") + /// - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum") + /// - For Vec containers: "Vec" (name is the index) + /// - For Tuple containers: "Tuple" (name is the index) + /// - For Array containers: "Array" (name is the index) + /// + /// Note: Option containers do not call enter_module/exit_module to preserve + /// the field name in the path (e.g., "bias" instead of "bias.Some") + #[allow(unused_variables)] + fn enter_module(&mut self, name: &str, container_type: &str) {} + + /// Called when exiting a submodule. + /// + /// # Parameters + /// - `name`: The name of the submodule being exited + /// - `container_type`: The type of the container with format: + /// - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear") + /// - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum") + /// - For Vec containers: "Vec" (name is the index) + /// - For Tuple containers: "Tuple" (name is the index) + /// - For Array containers: "Array" (name is the index) + /// + /// Note: Option containers do not call enter_module/exit_module to preserve + /// the field name in the path (e.g., "bias" instead of "bias.Some") + #[allow(unused_variables)] + fn exit_module(&mut self, name: &str, container_type: &str) {} + + /// Visit a float tensor with its full module path. + /// + /// # Parameters + /// - `path`: The path components to the tensor as a slice (e.g., &["encoder", "layer1", "weight"]). + /// Each element represents a module name in the hierarchy, with the final element + /// being the parameter name. This allows efficient reuse of the path stack. + /// - `id`: The unique identifier of the parameter + /// - `tensor`: The float tensor to visit + #[allow(unused_variables)] + fn visit_float_with_path( + &mut self, + path: &[String], + id: ParamId, + tensor: &Tensor, + ) { + } + + /// Visit an int tensor with its full module path. + /// + /// # Parameters + /// - `path`: The path components to the tensor as a slice (e.g., &["encoder", "layer1", "weight"]). + /// Each element represents a module name in the hierarchy, with the final element + /// being the parameter name. This allows efficient reuse of the path stack. + /// - `id`: The unique identifier of the parameter + /// - `tensor`: The integer tensor to visit + #[allow(unused_variables)] + fn visit_int_with_path( + &mut self, + path: &[String], + id: ParamId, + tensor: &Tensor, + ) { + } + + /// Visit a bool tensor with its full module path. + /// + /// # Parameters + /// - `path`: The path components to the tensor as a slice (e.g., &["encoder", "layer1", "weight"]). + /// Each element represents a module name in the hierarchy, with the final element + /// being the parameter name. This allows efficient reuse of the path stack. + /// - `id`: The unique identifier of the parameter + /// - `tensor`: The boolean tensor to visit + #[allow(unused_variables)] + fn visit_bool_with_path( + &mut self, + path: &[String], + id: ParamId, + tensor: &Tensor, + ) { + } +} + +/// Module mapper trait for transforming module parameters. +pub trait ModuleMapper { + /// Called when entering a submodule. + /// + /// # Parameters + /// - `name`: The name of the submodule being entered + /// - `container_type`: The type of the container with format: + /// - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear") + /// - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum") + /// - For Vec containers: "Vec" (name is the index) + /// - For Tuple containers: "Tuple" (name is the index) + /// - For Array containers: "Array" (name is the index) + /// + /// Note: Option containers do not call enter_module/exit_module to preserve + /// the field name in the path (e.g., "bias" instead of "bias.Some") + #[allow(unused_variables)] + fn enter_module(&mut self, name: &str, container_type: &str) {} + + /// Called when exiting a submodule. + /// + /// # Parameters + /// - `name`: The name of the submodule being exited + /// - `container_type`: The type of the container with format: + /// - For user-defined structs: "Struct:TypeName" (e.g., "Struct:Linear") + /// - For user-defined enums: "Enum:TypeName" (e.g., "Enum:MyEnum") + /// - For Vec containers: "Vec" (name is the index) + /// - For Tuple containers: "Tuple" (name is the index) + /// - For Array containers: "Array" (name is the index) + /// + /// Note: Option containers do not call enter_module/exit_module to preserve + /// the field name in the path (e.g., "bias" instead of "bias.Some") + #[allow(unused_variables)] + fn exit_module(&mut self, name: &str, container_type: &str) {} + + /// Map a float parameter in the module. + /// + /// # Parameters + /// - `param`: The float parameter to transform + /// + /// # Returns + /// The transformed parameter + #[allow(unused_variables)] + fn map_float(&mut self, param: Param>) -> Param> { + let (id, tensor, mapper) = param.consume(); + Param::from_mapped_value(id, tensor, mapper) + } + + /// Map an int parameter in the module. + /// + /// # Parameters + /// - `param`: The integer parameter to transform + /// + /// # Returns + /// The transformed parameter + #[allow(unused_variables)] + fn map_int( + &mut self, + param: Param>, + ) -> Param> { + let (id, tensor, mapper) = param.consume(); + Param::from_mapped_value(id, tensor, mapper) + } + + /// Map a bool parameter in the module. + /// + /// # Parameters + /// - `param`: The boolean parameter to transform + /// + /// # Returns + /// The transformed parameter + #[allow(unused_variables)] + fn map_bool( + &mut self, + param: Param>, + ) -> Param> { + let (id, tensor, mapper) = param.consume(); + Param::from_mapped_value(id, tensor, mapper) + } +} + +/// Module with auto-differentiation backend. +pub trait AutodiffModule: Module + Send + core::fmt::Debug { + /// Inner module without auto-differentiation. + type InnerModule: Module; + + /// Returns the same module, but on the inner backend without auto-differentiation. + fn valid(&self) -> Self::InnerModule; + + /// Wraps an inner module back into an auto-diff module. + fn from_inner(module: Self::InnerModule) -> Self; +} + +/// Helper trait to associate a module with its autodiff version. +pub trait HasAutodiffModule { + /// The module with auto-differentiation. + type TrainModule: AutodiffModule; +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::TestAutodiffBackend; + use crate::test_utils::SimpleLinear; + + #[test] + fn test_module_val_train_stateful() { + let device = Default::default(); + let module = SimpleLinear::::new(4, 4, &device); + + assert!(module.weight.is_require_grad()); + assert!(module.weight.require_grad); + + let module = module.valid(); + assert!(!module.weight.is_require_grad()); + assert!(module.weight.require_grad); // stateful + + // Without `HasAutodiffModule`, we would need to specify the module type as well, which would be annoying + // let module: SimpleLinear = module.train(); + let module = module.train::(); + assert!(module.weight.is_require_grad()); + assert!(module.weight.require_grad); // stateful + + let module = module.no_grad(); + assert!(!module.weight.is_require_grad()); + assert!(!module.weight.require_grad); // stateful + + let module = module.valid(); + assert!(!module.weight.is_require_grad()); // always + assert!(!module.weight.require_grad); // stateful + + let module = module.train::(); + assert!(!module.weight.is_require_grad()); + assert!(!module.weight.require_grad); // stateful + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/display.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/display.rs new file mode 100644 index 0000000..ede441a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/display.rs @@ -0,0 +1,543 @@ +use alloc::{ + borrow::ToOwned, + format, + string::{String, ToString}, + vec::Vec, +}; +use core::any; +use core::fmt::{Display, Write}; + +/// Default display settings for a module. +pub trait ModuleDisplayDefault { + /// Attributes of the module used for display purposes. + /// + /// # Arguments + /// + /// * `_content` - The content object that contains display settings and attributes. + /// + /// # Returns + /// + /// An optional content object containing the display attributes. + fn content(&self, _content: Content) -> Option; + + /// Gets the number of the parameters of the module. + fn num_params(&self) -> usize { + 0 + } +} + +/// Trait to implement custom display settings for a module. +/// +/// In order to implement custom display settings for a module, +/// 1. Add #[module(custom_display)] attribute to the module struct after #[derive(Module)] +/// 2. Implement ModuleDisplay trait for the module +pub trait ModuleDisplay: ModuleDisplayDefault { + /// Formats the module with provided display settings. + /// + /// # Arguments + /// + /// * `passed_settings` - Display settings passed to the module. + /// + /// # Returns + /// + /// A string representation of the formatted module. + fn format(&self, passed_settings: DisplaySettings) -> String { + let settings = if let Some(custom_settings) = self.custom_settings() { + custom_settings.inherit(passed_settings) + } else { + passed_settings + }; + + let indent = " ".repeat(settings.level * settings.indentation_size()); + let indent_close_braces = " ".repeat((settings.level - 1) * settings.indentation_size()); + + let settings = settings.level_up(); + + let self_type = extract_type_name::(); + + // Use custom content if it is implemented and show_all_attributes is false, + // otherwise use default content + let content = if !settings.show_all_attributes() { + self.custom_content(Content::new(settings.clone())) + .unwrap_or_else(|| { + self.content(Content::new(settings.clone())) + .unwrap_or_else(|| { + panic!("Default content should be implemented for {self_type}.") + }) + }) + } else { + self.content(Content::new(settings.clone())) + .unwrap_or_else(|| panic!("Default content should be implemented for {self_type}.")) + }; + + let top_level_type = if let Some(top_level_type) = content.top_level_type { + top_level_type.to_owned() + } else { + self_type.to_owned() + }; + + // If there is only one item in the content, return it or no attributes + if let Some(item) = content.single_item { + return item; + } else if content.attributes.is_empty() { + return top_level_type.to_string(); + } + + let mut result = String::new(); + + // Print the struct name + if settings.new_line_after_attribute() { + writeln!(result, "{top_level_type} {{").unwrap(); + } else { + write!(result, "{top_level_type} {{").unwrap(); + } + + for (i, attribute) in content.attributes.iter().enumerate() { + if settings.new_line_after_attribute() { + writeln!(result, "{indent}{}: {}", attribute.name, attribute.value).unwrap(); + } else if i == 0 { + write!(result, "{}: {}", attribute.name, attribute.value).unwrap(); + } else { + write!(result, ", {}: {}", attribute.name, attribute.value).unwrap(); + } + } + + if settings.show_num_parameters() { + let num_params = self.num_params(); + if num_params > 0 { + if settings.new_line_after_attribute() { + writeln!(result, "{indent}params: {num_params}").unwrap(); + } else { + write!(result, ", params: {num_params}").unwrap(); + } + } + } + + if settings.new_line_after_attribute() { + write!(result, "{indent_close_braces}}}").unwrap(); + } else { + write!(result, "}}").unwrap(); + } + + result + } + + /// Custom display settings for the module. + /// + /// # Returns + /// + /// An optional display settings object. + fn custom_settings(&self) -> Option { + None + } + + /// Custom attributes for the module. + /// + /// # Arguments + /// + /// * `_content` - The content object that contains display settings and attributes. + /// + /// # Returns + /// + /// An optional content object containing the custom attributes. + fn custom_content(&self, _content: Content) -> Option { + None + } +} + +/// Custom module display settings. +#[derive(Debug, Clone)] +pub struct DisplaySettings { + /// Whether to print the module parameter ids. + show_param_id: Option, + + /// Whether to print the module attributes. + show_all_attributes: Option, + + /// Whether to print the module number of parameters. + show_num_parameters: Option, + + /// Print new line after an attribute. + new_line_after_attribute: Option, + + /// Indentation size. + indentation_size: Option, + + /// Level of indentation. + level: usize, +} + +impl Default for DisplaySettings { + fn default() -> Self { + DisplaySettings { + show_param_id: None, + show_all_attributes: None, + show_num_parameters: None, + new_line_after_attribute: None, + indentation_size: None, + level: 1, + } + } +} + +impl DisplaySettings { + /// Create a new format settings. + /// + /// # Returns + /// + /// A new instance of `DisplaySettings`. + pub fn new() -> Self { + Default::default() + } + + /// Sets a flag to show module parameters. + /// + /// # Arguments + /// + /// * `flag` - Boolean flag to show module parameters. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance. + pub fn with_show_param_id(mut self, flag: bool) -> Self { + self.show_param_id = Some(flag); + self + } + + /// Sets a flag to show module attributes. + /// + /// # Arguments + /// + /// * `flag` - Boolean flag to show all module attributes. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance. + pub fn with_show_all_attributes(mut self, flag: bool) -> Self { + self.show_all_attributes = Some(flag); + self + } + + /// Sets a flag to show the number of module parameters. + /// + /// # Arguments + /// + /// * `flag` - Boolean flag to show the number of module parameters. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance. + pub fn with_show_num_parameters(mut self, flag: bool) -> Self { + self.show_num_parameters = Some(flag); + self + } + + /// Sets a flag to print a new line after an attribute. + /// + /// # Arguments + /// + /// * `flag` - Boolean flag to print a new line after an attribute. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance. + pub fn with_new_line_after_attribute(mut self, flag: bool) -> Self { + self.new_line_after_attribute = Some(flag); + self + } + + /// Sets the indentation size. + /// + /// # Arguments + /// + /// * `size` - The size of the indentation. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance. + pub fn with_indentation_size(mut self, size: usize) -> Self { + self.indentation_size = Some(size); + self + } + + /// Inherits settings from the provided settings and return a new settings object. + /// + /// # Arguments + /// + /// * `top` - The top level `DisplaySettings` to inherit from. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance. + pub fn inherit(self, top: Self) -> Self { + let mut updated = self.clone(); + + if let Some(show_param_id) = top.show_param_id { + updated.show_param_id = Some(show_param_id); + }; + + if let Some(show_all_attributes) = top.show_all_attributes { + updated.show_all_attributes = Some(show_all_attributes); + } + + if let Some(show_num_parameters) = top.show_num_parameters { + updated.show_num_parameters = Some(show_num_parameters); + } + + if let Some(new_line_after_attribute) = top.new_line_after_attribute { + updated.new_line_after_attribute = Some(new_line_after_attribute); + } + + if let Some(indentation_size) = top.indentation_size { + updated.indentation_size = Some(indentation_size); + } + + updated.level = top.level; + + updated + } + + /// A convenience method to wrap the DisplaySettings struct in an option. + /// + /// # Returns + /// + /// An optional `DisplaySettings`. + pub fn optional(self) -> Option { + Some(self) + } + + /// Increases the level of indentation. + /// + /// # Returns + /// + /// Updated `DisplaySettings` instance with increased indentation level. + pub fn level_up(mut self) -> Self { + self.level += 1; + self + } + + /// Gets `show_param_id` flag, substitutes false if not set. + /// + /// This flag is used to print the module parameter ids. + /// + /// # Returns + /// + /// A boolean value indicating whether to show parameter ids. + pub fn show_param_id(&self) -> bool { + self.show_param_id.unwrap_or(false) + } + + /// Gets `show_all_attributes`, substitutes false if not set. + /// + /// This flag is used to force to print all module attributes, overriding custom attributes. + /// + /// # Returns + /// + /// A boolean value indicating whether to show all attributes. + pub fn show_all_attributes(&self) -> bool { + self.show_all_attributes.unwrap_or(false) + } + + /// Gets `show_num_parameters`, substitutes true if not set. + /// + /// This flag is used to print the number of module parameters. + /// + /// # Returns + /// + /// A boolean value indicating whether to show the number of parameters. + pub fn show_num_parameters(&self) -> bool { + self.show_num_parameters.unwrap_or(true) + } + + /// Gets `new_line_after_attribute`, substitutes true if not set. + /// + /// This flag is used to print a new line after an attribute. + /// + /// # Returns + /// + /// A boolean value indicating whether to print a new line after an attribute. + pub fn new_line_after_attribute(&self) -> bool { + self.new_line_after_attribute.unwrap_or(true) + } + + /// Gets `indentation_size`, substitutes 2 if not set. + /// + /// This flag is used to set the size of indentation. + /// + /// # Returns + /// + /// An integer value indicating the size of indentation. + pub fn indentation_size(&self) -> usize { + self.indentation_size.unwrap_or(2) + } +} + +/// Struct to store the attributes of a module for formatting. +#[derive(Clone, Debug)] +pub struct Content { + /// List of attributes. + pub attributes: Vec, + + /// Single item content. + pub single_item: Option, + + /// Display settings. + pub display_settings: DisplaySettings, + + /// Top level type name. + pub top_level_type: Option, +} + +impl Content { + /// Creates a new attributes struct. + /// + /// # Arguments + /// + /// * `display_settings` - Display settings for the content. + /// + /// # Returns + /// + /// A new instance of `Content`. + pub fn new(display_settings: DisplaySettings) -> Self { + Content { + attributes: Vec::new(), + single_item: None, + display_settings, + top_level_type: None, + } + } + + /// Adds an attribute to the format settings. The value will be formatted and stored as a string. + /// + /// # Arguments + /// + /// * `name` - Name of the attribute. + /// * `value` - Value of the attribute. + /// + /// # Returns + /// + /// Updated `Content` instance with the new attribute added. + pub fn add(mut self, name: &str, value: &T) -> Self { + if self.single_item.is_some() { + panic!("Cannot add multiple attributes when single item is set."); + } + + let attribute = Attribute { + name: name.to_owned(), + value: value.format(self.display_settings.clone()), // TODO level + 1 + ty: any::type_name::().to_string(), + }; + self.attributes.push(attribute); + self + } + + /// Adds a single item. + /// + /// # Arguments + /// + /// * `value` - Rendered string of the single item. + /// + /// # Returns + /// + /// Updated `Content` instance with the single item added. + pub fn add_single(mut self, value: &T) -> Self { + if !self.attributes.is_empty() { + panic!("Cannot add single item when attributes are set."); + } + + self.single_item = Some(value.format(self.display_settings.clone())); + + self + } + + /// Adds a single item. + /// + /// # Arguments + /// + /// * `value` - Formatted display value. + /// + /// # Returns + /// + /// Updated `Content` instance with the formatted single item added. + pub fn add_formatted(mut self, value: &T) -> Self { + if !self.attributes.is_empty() { + panic!("Cannot add single item when attributes are set."); + } + + self.single_item = Some(format!("{value}")); + self + } + + /// A convenience method to wrap the Attributes struct in an option + /// because it is often used as an optional field. + /// + /// # Returns + /// + /// An optional `Content`. + pub fn optional(self) -> Option { + if self.attributes.is_empty() && self.single_item.is_none() && self.top_level_type.is_none() + { + None + } else { + Some(self) + } + } + + /// Sets the top level type name. + /// + /// # Arguments + /// + /// * `ty` - The type name to set. + /// + /// # Returns + /// + /// Updated `Content` instance with the top level type name set. + pub fn set_top_level_type(mut self, ty: &str) -> Self { + self.top_level_type = Some(ty.to_owned()); + self + } +} + +/// Attribute to print in the display method. +#[derive(Clone, Debug)] +pub struct Attribute { + /// Name of the attribute. + pub name: String, + + /// Value of the attribute. + pub value: String, + + /// Type of the attribute. + pub ty: String, +} + +/// Extracts the short name of a type T +/// +/// # Returns +/// +/// A string slice representing the short name of the type. +pub fn extract_type_name() -> &'static str { + // Get the full type name of T, including module path and generic parameters + let ty = any::type_name::(); + + // Find the first occurrence of '<' in the full type name + // If not found, use the length of the type name + let end = ty.find('<').unwrap_or(ty.len()); + + // Slice the type name up to the first '<' or the end + let ty = &ty[0..end]; + + // Find the last occurrence of "::" in the sliced type name + // If found, add 2 to skip the "::" itself + // If not found, start from the beginning of the type name + let start = ty.rfind("::").map(|i| i + 2).unwrap_or(0); + + // Find the last occurrence of '<' in the sliced type name + // If not found, use the length of the type name + let end = ty.rfind('<').unwrap_or(ty.len()); + + // If the start index is less than the end index, + // return the slice of the type name from start to end + // Otherwise, return the entire sliced type name + if start < end { &ty[start..end] } else { ty } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/initializer.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/initializer.rs new file mode 100644 index 0000000..cc25294 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/initializer.rs @@ -0,0 +1,627 @@ +use crate::tensor::Shape; + +use crate::config::Config; +use crate::module::{Param, ParamId}; +use crate::tensor::backend::Backend; +use crate::tensor::{Distribution, Tensor, s}; + +use crate as burn; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Enum specifying with what values a tensor should be initialized +#[derive(Config, Debug, PartialEq)] +pub enum Initializer { + /// Fills tensor with specified value everywhere + Constant { + /// The value to fill the tensor with + value: f64, + }, + /// Fills tensor with 1s everywhere + Ones, + /// Fills tensor with 0s everywhere + Zeros, + /// Fills tensor with values drawn uniformly between specified values + Uniform { + /// The minimum value to draw from + min: f64, + + /// The maximum value to draw from + max: f64, + }, + /// Fills tensor with values drawn from normal distribution with specified mean and std + Normal { + /// The mean of the normal distribution + mean: f64, + + /// The standard deviation of the normal distribution + std: f64, + }, + /// Fills tensor with values according to the uniform version of Kaiming initialization + KaimingUniform { + /// The gain to use in initialization formula + gain: f64, + + /// Whether to use fan out only in initialization formula + fan_out_only: bool, + }, + /// Fills tensor with values according to the uniform version of Kaiming initialization + KaimingNormal { + /// The gain to use in initialization formula + gain: f64, + + /// Whether to use fan out only in initialization formula + fan_out_only: bool, + }, + /// Fills tensor with values according to the uniform version of Xavier Glorot initialization + /// described in [Understanding the difficulty of training deep feedforward neural networks + /// ](https://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf) + XavierUniform { + /// The gain to use in initialization formula + gain: f64, + }, + /// Fills tensor with values according to the normal version of Xavier Glorot initialization + /// described in [Understanding the difficulty of training deep feedforward neural networks + /// ](https://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf) + XavierNormal { + /// The gain to use in initialization formula + gain: f64, + }, + /// Fills tensor with values according to the (semi) orthogonal initialization + /// described in [Exact solutions to the nonlinear dynamics of learning in deep linear neural networks` + /// - [Saxe, A. et al. (2013)](https://arxiv.org/abs/1312.6120) + Orthogonal { + /// The gain to use in initialization formula + gain: f64, + }, +} + +impl Initializer { + /// Inits a tensor parameter of given shape with values depending on initializer kind. + /// + /// # Params + /// + /// - shape: Shape of the initiated tensor. + pub fn init>( + &self, + shape: S, + device: &B::Device, + ) -> Param> { + self.init_with(shape, None, None, device) + } + + /// Inits a tensor parameter of given shape with values depending on initializer kind. + /// + /// # Params + /// + /// - shape: Shape of the initiated tensor. + pub fn init_with>( + &self, + shape: S, + fan_in: Option, + fan_out: Option, + device: &B::Device, + ) -> Param> { + let device = device.clone(); + let shape: Shape = shape.into(); + let config = self.clone(); + let shape_for_closure = shape.clone(); + + Param::uninitialized( + ParamId::new(), + move |device, require_grad| { + B::memory_persistent_allocations(device, (), move |_| { + let mut tensor = config.init_tensor(shape.clone(), fan_in, fan_out, device); + + if require_grad { + tensor = tensor.require_grad(); + } + + tensor + }) + }, + device, + true, + shape_for_closure, + ) + } + + fn init_tensor>( + &self, + shape: S, + fan_in: Option, + fan_out: Option, + device: &B::Device, + ) -> Tensor { + let shape = shape.into(); + match self { + Initializer::Constant { value } => Tensor::::full(shape, *value, device), + Initializer::Ones => Tensor::::ones(shape, device), + Initializer::Zeros => Tensor::::zeros(shape, device), + Initializer::Uniform { min, max } => uniform_draw(shape, *min, *max, device), + Initializer::Normal { mean, std } => normal_draw(shape, *mean, *std, device), + Initializer::KaimingUniform { gain, fan_out_only } => { + let a = 3.0f64.sqrt() * *gain * self.kaiming_std(*fan_out_only, fan_in, fan_out); + uniform_draw(shape, -a, a, device) + } + Initializer::KaimingNormal { gain, fan_out_only } => { + let std = *gain * self.kaiming_std(*fan_out_only, fan_in, fan_out); + normal_draw(shape, 0.0, std, device) + } + Initializer::XavierUniform { gain } => { + let a = 3.0f64.sqrt() * *gain * self.xavier_std(fan_in, fan_out); + uniform_draw(shape, -a, a, device) + } + Initializer::XavierNormal { gain } => { + let std = *gain * self.xavier_std(fan_in, fan_out); + normal_draw(shape, 0.0, std, device) + } + Initializer::Orthogonal { gain } => { + // following the implementation in pytorch: + // https://github.com/pytorch/pytorch/blob/v2.7.0/torch/nn/init.py#L574 + + assert!( + D >= 2, + "Expected D (in Tensor) to be greater or equal 2; (D >= 2)" + ); + + let rows: usize = shape.dims::()[0]; + let cols: usize = shape.num_elements() / rows; + + let mut t: Tensor = normal_draw([rows, cols], 0.0, 1.0, device); + + if rows < cols { + t = t.transpose(); + } + + let (q, r) = qr_decomposition(t, device); + let [r_rows, r_cols] = r.clone().dims(); + + let diag_r = Tensor::::ones([1, r_rows], device) + .matmul(Tensor::::eye(r_cols, device).mul(r.clone())); + + let ph = diag_r.clone().sign(); + + let mut q = q.mul(ph); + + if rows < cols { + q = q.transpose(); + } + + q.reshape(shape).mul_scalar(*gain) + } + } + } + + fn kaiming_std( + &self, + fan_out_only: bool, + fan_in: Option, + fan_out: Option, + ) -> f64 { + let fan = if fan_out_only { fan_out } else { fan_in }; + let fan = fan.expect( + "Can't use Kaiming initialization without specifying fan. Use init_with method.", + ); + + 1.0 / (fan as f64).sqrt() + } + + fn xavier_std(&self, fan_in: Option, fan_out: Option) -> f64 { + let fan_in = fan_in.expect( + "Can't use Xavier initialization without specifying fan in. Use init_with method and \ + provide fan_in.", + ); + let fan_out = fan_out.expect( + "Can't use Xavier initialization without specifying fan out. Use init_with method and \ + provide fan_out.", + ); + (2.0 / (fan_in + fan_out) as f64).sqrt() + } +} + +fn uniform_draw>( + shape: S, + low: f64, + high: f64, + device: &B::Device, +) -> Tensor { + let distribution = Distribution::Uniform(low, high); + Tensor::::random(shape, distribution, device) +} + +fn normal_draw>( + shape: S, + mean: f64, + std: f64, + device: &B::Device, +) -> Tensor { + let distribution = Distribution::Normal(mean, std); + Tensor::::random(shape, distribution, device) +} + +fn qr_decomposition( + a: Tensor, + device: &B::Device, +) -> (Tensor, Tensor) { + // Calculate the QR decomposition using Gram-Schmidt-process: https://en.wikipedia.org/wiki/Gram%E2%80%93Schmidt_process + + let [m, n] = a.clone().dims(); + let mut q = Tensor::::zeros([m, n], device); + let mut r = Tensor::::zeros([n, n], device); + + for j in 0..n { + let mut v: Tensor = a.clone().slice(s![.., j..=j]).squeeze_dim(1); + + for i in 0..j { + let q_i: Tensor = q.clone().slice(s![.., i..=i]).squeeze_dim(1); + let r_ij = q_i.clone().mul(v.clone()).sum(); + + r = r + .clone() + .slice_assign([i..i + 1, j..j + 1], r_ij.clone().unsqueeze()); + + v = v - q_i.mul(r_ij); + } + + // norm of v + let r_jj = v + .clone() + .powf(Tensor::from_floats([2.0], device)) + .sum() + .sqrt(); + + r = r + .clone() + .slice_assign([j..j + 1, j..j + 1], r_jj.clone().unsqueeze()); + + let q_j = v / r_jj; + + q = q + .clone() + .slice_assign([0..m, j..j + 1], q_j.unsqueeze_dim(1)); + } + + (q, r) +} + +#[cfg(test)] +mod tests { + use super::*; + + use burn_tensor::{ElementConversion, TensorData}; + use num_traits::Pow; + + pub type TB = burn_ndarray::NdArray; + use burn_tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + fn assert_normal_init(expected_mean: f64, expected_var: f64, tensor: &Tensor) { + let (actual_vars, actual_means) = tensor.clone().var_mean(0); + let actual_vars = actual_vars.to_data(); + let actual_vars = actual_vars.as_slice::().unwrap(); + let actual_means = actual_means.to_data(); + let actual_means = actual_means.as_slice::().unwrap(); + + for i in 0..tensor.shape()[0] { + let actual_var = actual_vars[i] as f64; + let actual_mean = actual_means[i] as f64; + + assert!( + (expected_var - actual_var).abs() <= 0.1, + "Expected variance to be between {expected_var} += 0.1, but got {actual_var}" + ); + assert!( + (expected_mean - actual_mean).abs() <= 0.1, + "Expected mean to be between {expected_mean} += 0.1, but got {actual_mean}" + ); + } + } + + #[test] + fn initializer_uniform_init() { + let device = Default::default(); + TB::seed(&device, 0); + + let (min, max) = (0.0, 1.0); + let uniform = Initializer::Uniform { min, max }; + let tensor: Tensor = uniform.init([2, 2, 2, 2], &Default::default()).into_value(); + + tensor + .into_data() + .assert_within_range::(min.elem()..max.elem()); + } + + #[test] + fn initializer_normal_init() { + // seed random generator + let device = Default::default(); + TB::seed(&device, 0); + + let (mean, std) = (0.0, 1.0); + let normal: Tensor = Initializer::Normal { mean, std } + .init([1000], &Default::default()) + .into_value(); + let (var_act, mean_act) = normal.var_mean(0); + + let var_act: f32 = var_act.into_scalar().elem(); + let mean_act: f32 = mean_act.into_scalar().elem(); + + assert!( + var_act > 0.9 && var_act < 1.1, + "Expected variance to be between 1.0 += 0.1, but got {var_act}" + ); + assert!( + mean_act > -0.1 && mean_act < 0.1, + "Expected mean to be between 0.0 += 0.1, but got {mean_act}" + ); + } + + #[test] + fn initializer_constant_init() { + let value = 5.0; + let constants: Tensor = Initializer::Constant { value } + .init([2, 2, 2, 2], &Default::default()) + .into_value(); + constants.sum().to_data().assert_approx_eq::( + &TensorData::from([value as f32 * 16.0]), + Tolerance::default(), + ); + } + + #[test] + fn initializer_zeros_init() { + let zeros: Tensor = Initializer::Zeros + .init([2, 2, 2, 2], &Default::default()) + .into_value(); + zeros + .sum() + .to_data() + .assert_approx_eq::(&TensorData::from([0.0]), Tolerance::default()); + } + + #[test] + fn initializer_ones_init() { + let ones: Tensor = Initializer::Ones + .init([2, 2, 2, 2], &Default::default()) + .into_value(); + ones.sum() + .to_data() + .assert_approx_eq::(&TensorData::from([16.0]), Tolerance::default()); + } + + #[test] + fn initializer_kaiming_uniform_init() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2_f64; + let (fan_in, fan_out) = (5, 6); + let k = (gain * (3.0 / fan_in as f64).sqrt()).elem::(); + + let tensor: Tensor = Initializer::KaimingUniform { + gain, + fan_out_only: false, + } + .init_with([fan_out, fan_in], Some(fan_in), None, &Default::default()) + .into_value(); + tensor.into_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_kaiming_normal_init() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2.; + let (fan_in, fan_out) = (1000, 10); + let expected_mean = 0_f64; + + let expected_var = (gain * (1. / (fan_in as f64)).sqrt()).pow(2.); + let tensor: Tensor = Initializer::KaimingNormal { + gain, + fan_out_only: false, + } + .init_with([fan_out, fan_in], Some(fan_in), None, &Default::default()) + .into_value(); + assert_normal_init(expected_mean, expected_var, &tensor) + } + + #[test] + fn initializer_kaiming_uniform_init_bias() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2_f64; + let shape = [3]; + let fan_in = 5; + let k = (gain * (3.0 / fan_in as f64).sqrt()).elem::(); + + let tensor: Tensor = Initializer::KaimingUniform { + gain, + fan_out_only: false, + } + .init_with(shape, Some(fan_in), None, &Default::default()) + .into_value(); + tensor.into_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_kaiming_uniform_init_fan_out() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2_f64; + let (fan_in, fan_out) = (5, 6); + let k = (gain * (3.0 / fan_out as f64).sqrt()).elem::(); + + let tensor: Tensor = Initializer::KaimingUniform { + gain, + fan_out_only: true, + } + .init_with([fan_out, fan_in], None, Some(fan_out), &Default::default()) + .into_value(); + tensor.into_data().assert_within_range(-k..k); + } + + #[test] + #[should_panic] + fn initializer_kaiming_uniform_no_fan() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2_f64; + let (fan_in, fan_out) = (5, 6); + + let _: Tensor = Initializer::KaimingUniform { + gain, + fan_out_only: false, + } + .init([fan_out, fan_in], &Default::default()) + .into_value(); + } + + #[test] + fn initializer_xavier_uniform_init() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2.; + let (fan_in, fan_out) = (5, 6); + let bound = (gain * (6. / (fan_in + fan_out) as f64).sqrt()).elem::(); + let tensor: Tensor = Initializer::XavierUniform { gain } + .init_with( + [fan_out, fan_in], + Some(fan_in), + Some(fan_out), + &Default::default(), + ) + .into_value(); + + tensor.into_data().assert_within_range(-bound..bound); + } + + #[test] + fn initializer_xavier_normal_init() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2.; + let (fan_in, fan_out) = (1000, 10); + let expected_mean = 0_f64; + + let expected_var = (gain * (2. / (fan_in as f64 + fan_out as f64)).sqrt()).powf(2.); + let tensor: Tensor = Initializer::XavierNormal { gain } + .init_with( + [fan_out, fan_in], + Some(fan_in), + Some(fan_out), + &Default::default(), + ) + .into_value(); + assert_normal_init(expected_mean, expected_var, &tensor) + } + + #[test] + #[should_panic] + fn initializer_xavier_uniform_no_fan() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 2.; + let (fan_in, fan_out) = (5, 6); + let _: Tensor = Initializer::XavierUniform { gain } + .init([fan_out, fan_in], &Default::default()) + .into_value(); + } + + #[test] + fn test_qr_decomposition() { + let device = Default::default(); + TB::seed(&device, 0); + + // test values follow the example from https://pytorch.org/docs/stable/generated/torch.linalg.qr.html#torch.linalg.qr + let a = Tensor::::from_floats( + [[12., -51., 4.], [6., 167., -68.], [-4., 24., -41.]], + &Default::default(), + ); + let qr = qr_decomposition(a.clone(), &Default::default()); + + // Q @ R should reconstruct input `a` + let q_matmul_r = qr.0.clone().matmul(qr.1.clone()); + + // assert that the difference between input (`a`) and Q @ R is (almost) zero + q_matmul_r + .into_data() + .assert_approx_eq::(&a.into_data(), Tolerance::rel_abs(0.1, 0.1)); + } + + #[test] + fn initializer_orthogonal_correct() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 1.; + + // test 2D tensor + let size = 10; + let q: Tensor = Initializer::Orthogonal { gain } + .init([size, size], &Default::default()) + .into_value(); + let eye = Tensor::::eye(size, &Default::default()); + + // Q.T @ Q should be close to identity matrix + q.clone() + .transpose() + .matmul(q) + .into_data() + .assert_approx_eq::(&eye.into_data(), Tolerance::rel_abs(0.1, 0.1)); + } + + #[test] + fn initializer_orthogonal_init() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 1.; + + // test 2D tensor + let shape = [25, 30]; + let t: Tensor = Initializer::Orthogonal { gain } + .init(shape, &Default::default()) + .into_value(); + let dims = t.dims(); + assert_eq!( + shape, dims, + "Expected the shape of the input tensor to match the shape of the output. ({shape:?}, {dims:?})" + ); + + // test 3D tensor + let shape = [24, 6, 85]; + let t: Tensor = Initializer::Orthogonal { gain } + .init(shape, &Default::default()) + .into_value(); + let dims = t.dims(); + assert_eq!( + shape, dims, + "Expected the shape of the input tensor to match the shape of the output. ({shape:?}, {dims:?})" + ); + } + + #[test] + #[should_panic] + fn initializer_orthogonal_init_1d() { + let device = Default::default(); + TB::seed(&device, 0); + + let gain = 1.; + + // test 1D tensor + let shape = [3]; + let _: Tensor = Initializer::Orthogonal { gain } + .init(shape, &Default::default()) + .into_value(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/mod.rs new file mode 100644 index 0000000..7bf82b2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/mod.rs @@ -0,0 +1,16 @@ +mod base; +mod display; +mod initializer; +mod param; +mod quantize; +#[cfg(feature = "std")] +mod reinit; + +pub use base::*; +pub use display::*; +pub use initializer::*; +pub use param::*; +pub use quantize::*; + +#[cfg(feature = "std")] +pub use reinit::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/base.rs new file mode 100644 index 0000000..387903d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/base.rs @@ -0,0 +1,424 @@ +use super::ParamId; +use alloc::{boxed::Box, format}; +use burn_std::stub::RwLock; +use burn_tensor::Shape; +use core::cell::OnceCell; +use core::ops::Deref; + +#[cfg(target_has_atomic = "ptr")] +use alloc::sync::Arc; + +#[cfg(not(target_has_atomic = "ptr"))] +use portable_atomic_util::Arc; + +#[cfg(target_has_atomic = "ptr")] +type Mapper = Arc T + Send + Sync>; + +#[cfg(not(target_has_atomic = "ptr"))] +type Mapper = Arc T + Send + Sync>>; + +#[cfg(target_has_atomic = "ptr")] +fn new_mapper T + Send + Sync + 'static>(func: F) -> Mapper { + Arc::new(func) +} + +#[cfg(not(target_has_atomic = "ptr"))] +fn new_mapper T + Send + Sync + 'static>(func: F) -> Mapper { + Arc::new(Box::new(func)) +} + +/// Parameters are the fundamental building blocks of [modules](crate::module::Module) where they +/// serve as containers for [tensors](crate::tensor::Tensor) that can be updated during +/// training, and loaded during inference. If you don't want to save the tensors +/// and/or don't want to update it during training, you don't need this type to wrap your tensor. +/// +/// # Core Lazy Initialization Architecture +/// +/// `Param` has a dual-state design using `OnceCell`: +/// +/// ## State Management +/// +/// **Two possible states:** +/// +/// 1. **Initialized**: `state: OnceCell` contains value, `initialization: None` +/// 2. **Uninitialized (Lazy)**: `state` is empty, `initialization: Some(RwLock>>)` +pub struct Param { + /// The unique ID of this parameter. This is used by eg. optimizers to associate a gradient with a specific parameter. + pub id: ParamId, + /// The OnceCell holding the initialized parameter value. + /// Empty for uninitialized parameters, populated after first access or explicit initialization. + pub(crate) state: OnceCell, + /// The deferred initialization state for lazy parameters. + /// + /// **State Transitions:** + /// - Initialized params: `None` + /// - Uninitialized params: `Some(RwLock)>)` + /// - After lazy init triggers: `Some(RwLock)` (inner Option is taken) + pub(crate) initialization: Option>>>, + pub(crate) param_mapper: ParamMapper, + // For stateful `module.valid()` <> `module.train()` + pub(crate) require_grad: bool, +} + +#[derive(Clone)] +/// Applies transformations when loading and saving parameters. +/// +/// # Mapper System +/// +/// `ParamMapper` allows applying transformations during serialization and deserialization: +/// - `load: Option>` - transformation during deserialization (applied in `transform_for_load()`) +/// - `save: Option>` - transformation during serialization (applied in `transform_for_save()`) +/// +/// These are commonly used for: +/// - Quantization/dequantization +/// - Precision conversion (e.g., FP32 ↔ FP16) +/// - Custom parameter transformations +pub struct ParamMapper { + load: Option>, + save: Option>, +} + +impl core::fmt::Debug for ParamMapper { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!( + "ParamMapper {{ load: {}, save: {} }}", + self.load.is_some(), + self.save.is_some() + )) + } +} + +impl ParamMapper { + /// Applies the transformation when loading the given parameter. + pub fn on_load(&self, param: T) -> T { + match &self.load { + Some(mapper) => mapper(param), + None => param, + } + } + /// Applies the transformation when saving the given parameter. + pub fn on_save(&self, param: T) -> T { + match &self.save { + Some(mapper) => mapper(param), + None => param, + } + } +} + +impl Default for ParamMapper { + fn default() -> Self { + Self { + load: None, + save: None, + } + } +} + +impl core::fmt::Display for Param { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(format!("Param: {}", self.id).as_str()) + } +} + +impl core::fmt::Debug for Param { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(format!("Param: {} - {:?}", self.id, self.param_mapper).as_str()) + } +} + +/// Trait that defines what is necessary for a type to be a parameter. +pub trait Parameter: Clone + core::fmt::Debug + Send { + /// The device type to be used. + type Device: Clone; + + /// Fetch the device. + fn device(&self) -> Self::Device; + + /// Fetch the gradient requirement. + fn is_require_grad(&self) -> bool; + + /// Set the gradient requirement. + fn set_require_grad(self, require_grad: bool) -> Self; +} + +/// The deferred initialization state for lazy parameters. +#[allow(clippy::type_complexity)] +pub(crate) struct Uninitialized { + /// The initialization function. Called with `(device, is_require_grad) -> Parameter`. + /// This function is consumed during initialization via `FnOnce`. + init: Box P + Send>, + /// The target device on which the parameter should be initialized. + /// Used by `lazy_device()` to provide device information without triggering initialization. + pub(crate) device: P::Device, + /// The gradient requirement for the parameter. + /// Used by `lazy_is_require_grad()` to provide gradient settings without triggering initialization. + pub(crate) is_require_grad: bool, + /// The shape of the tensor parameter. + /// Used by `lazy_shape()` to provide shape information without triggering initialization. + pub(crate) shape: Shape, +} + +impl Uninitialized

{ + /// Consumes the uninitialized state and runs the initialization function. + /// + /// This is called by [Param::val] when accessing an uninitialized parameter for the first time. + /// The function is given the stored device and gradient requirement, and returns the initialized parameter. + fn initialize(self) -> P { + let init = self.init; + init(&self.device, self.is_require_grad) + } +} + +impl Param { + /// Create a new parameter that is already initialized. + pub fn initialized(id: ParamId, value: T) -> Self { + let require_grad = value.is_require_grad(); + Self { + id, + state: OnceCell::from(value), + initialization: None, + param_mapper: Default::default(), + require_grad, + } + } + + /// Create a new parameter that is not already initialized. + pub fn uninitialized( + id: ParamId, + init: F, + device: T::Device, + is_require_grad: bool, + shape: Shape, + ) -> Self + where + F: FnOnce(&T::Device, bool) -> T + Send + 'static, + { + Self { + id, + state: OnceCell::new(), + initialization: Some(RwLock::new(Some(Uninitialized { + init: Box::new(init), + device, + is_require_grad, + shape, + }))), + param_mapper: Default::default(), + require_grad: is_require_grad, + } + } + + /// Gets the parameter value, initializing it lazily if needed. + /// + /// For initialized parameters, this returns a clone of the cached value. + /// For uninitialized parameters, this triggers initialization: + pub fn val(&self) -> T { + self.state + .get_or_init(|| { + let mut result = self + .initialization + .as_ref() + .expect("Should have an initialization when no state provided.") + .write() + .unwrap(); + let state = result.take().expect("Should exist when not initialized"); + state.initialize() + }) + .clone() + } + + /// Check if the parameter has been initialized. + /// + /// Returns `true` if the parameter's value has been computed and cached, + /// `false` if it's still lazy and will be initialized on first access. + pub fn is_initialized(&self) -> bool { + self.state.get().is_some() + } + + /// Gets the parameter's value while consuming the parameter. + pub fn into_value(self) -> T { + self.consume().1 + } + + /// Gets the parameter id and value while consuming the parameter. + pub fn consume(self) -> (ParamId, T, ParamMapper) { + let tensor = self.val(); + + core::mem::drop(self.state); + + (self.id, tensor, self.param_mapper) + } + + /// Execute the given function on the inner value. + pub fn map T>(self, func: F) -> Self { + let (id, tensor, param_mapper) = self.consume(); + let tensor = func(tensor); + let require_grad = tensor.is_require_grad(); + + Self { + id, + state: OnceCell::from(tensor), + initialization: None, + param_mapper, + require_grad, + } + } + + /// Create an initialized parameter with the given id, value, and param mapper. + /// + /// This is a helper method for creating parameters while preserving the param mapper, + /// typically used in ModuleMapper implementations. + pub fn from_mapped_value(id: ParamId, value: T, param_mapper: ParamMapper) -> Self { + let require_grad = value.is_require_grad(); + Self { + id, + state: OnceCell::from(value), + initialization: None, + param_mapper, + require_grad, + } + } + + /// Runs a transformation on the parameter when loading. + pub fn load_mapper T + Send + Sync + 'static>(mut self, func: F) -> Self { + self.param_mapper.load = Some(new_mapper(func)); + + self + } + + /// Runs a transformation on the parameter when saving. + pub fn save_mapper T + Send + Sync + 'static>(mut self, func: F) -> Self { + self.param_mapper.save = Some(new_mapper(func)); + + self + } + + /// Execute the given function on the inner value. + pub fn init_mapper T + Send + 'static>(self, func: F) -> Self + where + T: 'static, + { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.map(func), + }; + + let mut init = initialization.write().unwrap(); + + match init.as_mut() { + Some(value) => { + #[allow(clippy::type_complexity)] + let mut prev: Box T + Send> = + Box::new(|_, _| panic!("Fake func to not have null ref.")); + core::mem::swap(&mut prev, &mut value.init); + + value.init = Box::new(|a, b| { + let tensor = prev(a, b); + func(tensor) + }); + core::mem::drop(init); + self + } + None => { + core::mem::drop(init); + self.map(func) + } + } + } + + /// The device on which the parameter is or will be initialized, **without triggering initialization**. + /// + /// This is critical for the load optimization: when loading tensors into an uninitialized parameter, + /// we need to know the target device to move the loaded tensor appropriately, but we don't want to + /// trigger the initialization function (which would allocate an unnecessary tensor). + /// + /// Use this instead of [crate::tensor::Tensor::device] when you need the device but want to + /// preserve lazy initialization. + pub fn lazy_device(&self) -> T::Device { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.device(), + }; + + let init = initialization.read().unwrap(); + + match init.as_ref() { + Some(value) => value.device.clone(), + None => self.device(), + } + } + + /// The gradient requirement on which the parameter is or will be initialized, **without triggering initialization**. + /// + /// Similar to [lazy_device](Self::lazy_device), this is critical for the load optimization. + /// When loading tensors into an uninitialized parameter, we need to apply the correct gradient + /// setting to the loaded tensor without triggering the initialization function. + /// + /// # Notes + /// + /// This is a crate-private function, since users are not expected to use `is_require_grad` of an + /// uninitialized module to then override its value. All low-level functions should be provided + /// by `burn` and should handle those details. + pub(crate) fn lazy_is_require_grad(&self) -> bool { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.is_require_grad(), + }; + + let init = initialization.read().unwrap(); + + match init.as_ref() { + Some(value) => value.is_require_grad, + None => self.is_require_grad(), + } + } + + /// Override the gradient requirement for the current parameter. + pub fn set_require_grad(self, require_grad: bool) -> Self { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.map(|tensor| tensor.set_require_grad(require_grad)), + }; + + let mut init = initialization.write().unwrap(); + let mut is_lazy = false; + + if let Some(value) = init.as_mut() { + is_lazy = true; + value.is_require_grad = require_grad; + }; + + core::mem::drop(init); + + if is_lazy { + return self; + } + + self.map(|tensor| tensor.set_require_grad(require_grad)) + } +} + +impl Clone for Param { + fn clone(&self) -> Self { + let mut param = Param::initialized(self.id, self.val()); + param.param_mapper = self.param_mapper.clone(); + param + } +} + +impl Deref for Param { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.state.get_or_init(|| { + let mut result = self + .initialization + .as_ref() + .expect("Should have an initialization when no state provided.") + .write() + .unwrap(); + + let state = result.take().expect("Should exist when not initialized"); + state.initialize() + }) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/constant.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/constant.rs new file mode 100644 index 0000000..8dd4d1f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/constant.rs @@ -0,0 +1,408 @@ +use alloc::{format, string::ToString}; +use core::{fmt::Display, marker::PhantomData}; + +use crate as burn; +use crate::{ + module::{ + AutodiffModule, Content, Devices, Module, ModuleDisplay, ModuleDisplayDefault, + ModuleMapper, ModuleVisitor, + }, + record::{PrecisionSettings, Record}, +}; +use burn_tensor::{ + BasicAutodiffOps, BasicOps, Tensor, + backend::{AutodiffBackend, Backend}, + ops::Device, +}; + +/// Record used for constant type implementing the [module](crate::module::Module) trait. +#[derive(Debug, Clone, Copy, new, Default)] +pub struct ConstantRecord; + +impl serde::Serialize for ConstantRecord { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // nothing to serialize + S::serialize_none(serializer) + } +} + +impl<'de> serde::Deserialize<'de> for ConstantRecord { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_option(serde::de::IgnoredAny).ok(); + Ok(ConstantRecord::new()) + } +} + +impl Record for ConstantRecord { + type Item = ConstantRecord; + + fn into_item(self) -> Self::Item { + self + } + + fn from_item(item: Self::Item, _device: &B::Device) -> Self { + item + } +} +/// Constant macro. +#[macro_export] +macro_rules! constant { + (module) => { + type Record = burn::module::ConstantRecord; + + fn visit>(&self, _visitor: &mut V) { + // Nothing to do + } + + fn map>(self, _mapper: &mut M) -> Self { + self + } + + fn load_record(self, _record: Self::Record) -> Self { + self + } + + fn into_record(self) -> Self::Record { + burn::module::ConstantRecord::new() + } + + fn to_device(self, _: &B::Device) -> Self { + self + } + + fn fork(self, _: &B::Device) -> Self { + self + } + + fn collect_devices(&self, devices: burn::module::Devices) -> burn::module::Devices { + devices + } + }; + + (ad_module, $type:ty) => { + type InnerModule = $type; + + fn valid(&self) -> Self::InnerModule { + self.clone() + } + + fn from_inner(module: Self::InnerModule) -> Self { + module + } + }; + + ($type:ty) => { + impl burn::module::Module for $type { + constant!(module); + } + + impl burn::module::AutodiffModule for $type { + constant!(ad_module, $type); + } + + impl burn::module::ModuleDisplayDefault for $type { + fn content(&self, content: burn::module::Content) -> Option { + let string = format!("{}", self); + content.add_formatted(&string).optional() + } + } + + impl burn::module::ModuleDisplay for $type {} + }; +} + +// General Types +constant!(alloc::string::String); +constant!(bool); + +// Float Types +constant!(f64); +constant!(f32); +constant!(half::bf16); +constant!(half::f16); + +// Unsigned Integer Types +constant!(usize); +constant!(u64); +constant!(u32); +constant!(u16); +constant!(u8); + +// Signed Integer Types +constant!(isize); +constant!(i64); +constant!(i32); +constant!(i16); +constant!(i8); + +impl burn::module::ModuleDisplay for str {} +impl burn::module::ModuleDisplayDefault for str { + fn content(&self, content: burn::module::Content) -> Option { + content.add_formatted(&self).optional() + } +} + +impl> Module for Tensor { + type Record = ConstantRecord; + + fn visit>(&self, _visitor: &mut V) {} + + fn map>(self, _mapper: &mut M) -> Self { + self + } + + fn into_record(self) -> Self::Record { + ConstantRecord + } + + fn load_record(self, _record: Self::Record) -> Self { + self + } + + fn to_device(self, device: &B::Device) -> Self { + self.to_device(device) + } + + fn fork(self, device: &B::Device) -> Self { + self.to_device(device) + } + + fn collect_devices(&self, mut devices: Devices) -> Devices { + let device = self.device(); + + if !devices.contains(&device) { + devices.push(device) + } + + devices + } +} + +impl> ModuleDisplayDefault for Tensor { + fn content(&self, content: Content) -> Option { + let string = format!("Tensor {{rank: {D}, shape: {:?}}}", self.shape().as_slice()); + content.add_single(&string).optional() + } +} + +impl> ModuleDisplay for Tensor {} + +impl> AutodiffModule + for Tensor +{ + type InnerModule = Tensor; + + fn valid(&self) -> Self::InnerModule { + self.clone().inner() + } + + fn from_inner(tensor: Self::InnerModule) -> Self { + Tensor::from_inner(tensor) + } +} + +impl Module for PhantomData { + type Record = ConstantRecord; + + fn visit>(&self, _visitor: &mut V) { + // Nothing to do + } + + fn map>(self, _mapper: &mut M) -> Self { + self + } + + fn load_record(self, _record: Self::Record) -> Self { + self + } + + fn into_record(self) -> Self::Record { + ConstantRecord::new() + } + + fn to_device(self, _: &Device) -> Self { + self + } + + fn fork(self, _: &Device) -> Self { + self + } + + fn collect_devices(&self, devices: Devices) -> Devices { + devices + } +} + +impl ModuleDisplayDefault for PhantomData { + fn content(&self, content: Content) -> Option { + content.add_single(&"PhantomData".to_string()).optional() + } +} + +impl ModuleDisplay for PhantomData {} + +impl AutodiffModule for PhantomData { + type InnerModule = PhantomData; + + fn valid(&self) -> Self::InnerModule { + PhantomData + } + + fn from_inner(_module: Self::InnerModule) -> Self { + PhantomData + } +} + +/// Container to satisfy the Module trait for types that are not modules. +#[derive(Clone, Debug)] +pub struct Ignored(pub T); + +impl Module for Ignored +where + B: Backend, + T: Sync + Send + core::fmt::Debug + Clone, +{ + type Record = ConstantRecord; + + fn visit>(&self, _visitor: &mut V) { + // Nothing to do + } + + fn map>(self, _mapper: &mut M) -> Self { + self + } + + fn load_record(self, _record: Self::Record) -> Self { + self + } + + fn into_record(self) -> Self::Record { + ConstantRecord::new() + } + + fn to_device(self, _: &Device) -> Self { + self + } + + fn fork(self, _: &Device) -> Self { + self + } + + fn collect_devices(&self, devices: Devices) -> Devices { + devices + } +} + +impl ModuleDisplayDefault for Ignored +where + T: Sync + Send + core::fmt::Debug + Clone, +{ + fn content(&self, content: Content) -> Option { + // For now, just print the debug representation of the ignored value + content.add_single(&format!("{:?}", self.0)).optional() + } +} + +impl ModuleDisplay for Ignored where T: Sync + Send + core::fmt::Debug + Clone {} + +impl Display for Ignored +where + T: Sync + Send + core::fmt::Debug + Clone, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self.0) + } +} + +impl AutodiffModule for Ignored +where + B: AutodiffBackend, + T: Sync + Send + core::fmt::Debug + Clone, +{ + type InnerModule = Ignored; + + fn valid(&self) -> Self::InnerModule { + self.clone() + } + + fn from_inner(module: Self::InnerModule) -> Self { + module + } +} + +// Implement deref for Ignored +impl core::ops::Deref for Ignored { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use core::marker::PhantomData; + + use burn_tensor::backend::Backend; + use burn_tensor::{Device, Tensor}; + + use crate::TestBackend; + use crate::{ + TestAutodiffBackend, + record::{BinBytesRecorder, FullPrecisionSettings, Recorder}, + }; + use burn::module::Module; + + use crate as burn; + + #[test] + fn tensor_load_record_setting() { + let device: &Device = &Default::default(); + let tensor = Tensor::::ones([3, 3], device); + + let byte_recorder = BinBytesRecorder::::default(); + let bytes = Recorder::::record( + &byte_recorder, + tensor.clone().into_record(), + (), + ) + .unwrap(); + + let no_grad_is_require_grad = tensor + .clone() + .no_grad() + .load_record( + Recorder::::load(&byte_recorder, bytes.clone(), device) + .unwrap(), + ) + .is_require_grad(); + + let with_default_is_require_grad = tensor + .load_record( + Recorder::::load(&byte_recorder, bytes.clone(), device) + .unwrap(), + ) + .is_require_grad(); + + assert!(!no_grad_is_require_grad); + assert!(!with_default_is_require_grad); + } + + #[test] + fn empty_module_with_phantom() { + #[derive(Module, Debug, new)] + struct EmptyModule { + _phantom: PhantomData, + } + + let _module = EmptyModule::::new(); + + assert_eq!(core::mem::size_of::>(), 0); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/id.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/id.rs new file mode 100644 index 0000000..6522c62 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/id.rs @@ -0,0 +1,116 @@ +use core::hash::{BuildHasher, Hasher}; + +use alloc::string::String; +use burn_std::id::IdGenerator; +use data_encoding::BASE32_DNSSEC; + +// Hashbrown changed its default hasher in 0.15, but there are some issues +// https://github.com/rust-lang/hashbrown/issues/577 +// Also, `param_serde_deserialize_legacy_uuid` doesn't pass with the default hasher. +type DefaultHashBuilder = core::hash::BuildHasherDefault; + +/// Parameter ID. +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)] +pub struct ParamId { + value: u64, +} + +impl From for ParamId { + fn from(value: u64) -> Self { + Self { value } + } +} + +impl Default for ParamId { + fn default() -> Self { + Self::new() + } +} + +impl ParamId { + /// Create a new parameter ID. + pub fn new() -> Self { + Self { + value: IdGenerator::generate(), + } + } + + /// Gets the internal value of the id. + pub fn val(&self) -> u64 { + self.value + } + + /// Convert the parameter ID into a string. + pub fn serialize(self) -> String { + BASE32_DNSSEC.encode(&self.value.to_le_bytes()) + } + + /// Deserialize a param id. + /// + /// Preserves compatibility with previous formats (6 bytes, 16-byte uuid). + pub fn deserialize(encoded: &str) -> ParamId { + let u64_id = match BASE32_DNSSEC.decode(encoded.as_bytes()) { + Ok(bytes) => { + let mut buffer = [0u8; 8]; + buffer[..bytes.len()].copy_from_slice(&bytes); + u64::from_le_bytes(buffer) + } + Err(err) => match uuid::Uuid::try_parse(encoded) { + // Backward compatibility with uuid parameter identifiers + Ok(id) => { + // Hash the 128-bit uuid to 64-bit + // Though not *theoretically* unique, the probability of a collision should be extremely low + let mut hasher = DefaultHashBuilder::default().build_hasher(); + // let mut hasher = DefaultHasher::new(); + hasher.write(id.as_bytes()); + hasher.finish() + } + Err(_) => panic!("Invalid id. {err}"), + }, + }; + + ParamId::from(u64_id) + } +} + +impl core::fmt::Display for ParamId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(&self.serialize()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn param_serde_deserialize() { + let val = ParamId::from(123456u64); + let deserialized = ParamId::deserialize(&val.serialize()); + assert_eq!(val, deserialized); + } + + #[test] + fn param_serde_deserialize_legacy() { + let legacy_val = [45u8; 6]; + let param_id = ParamId::deserialize(&BASE32_DNSSEC.encode(&legacy_val)); + assert_eq!(param_id.val().to_le_bytes()[0..6], legacy_val); + assert_eq!(param_id.val().to_le_bytes()[6..], [0, 0]); + } + + #[test] + fn param_serde_deserialize_legacy_uuid() { + // Ensure support for legacy uuid deserialization and make sure it results in the same output + let legacy_id = "30b82c23-788d-4d63-a743-ada258d5f13c"; + let param_id1 = ParamId::deserialize(legacy_id); + let param_id2 = ParamId::deserialize(legacy_id); + assert_eq!(param_id1, param_id2); + } + + #[test] + #[should_panic = "Invalid id."] + fn param_serde_deserialize_invalid_id() { + let invalid_uuid = "30b82c23-788d-4d63-ada258d5f13c"; + let _ = ParamId::deserialize(invalid_uuid); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/mod.rs new file mode 100644 index 0000000..d569d25 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/mod.rs @@ -0,0 +1,13 @@ +mod base; +mod constant; +mod id; +mod primitive; +mod running; +mod tensor; +mod visitor; + +pub use base::*; +pub use constant::*; +pub use id::*; +pub use running::*; +pub use visitor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/primitive.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/primitive.rs new file mode 100644 index 0000000..293da84 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/primitive.rs @@ -0,0 +1,426 @@ +use crate::module::{ + AutodiffModule, Content, Module, ModuleDisplay, ModuleDisplayDefault, ModuleMapper, + ModuleVisitor, +}; + +use alloc::{format, string::ToString, vec::Vec}; + +use burn_tensor::{ + backend::{AutodiffBackend, Backend}, + ops::Device, +}; +use core::fmt::Debug; + +impl Module for Option +where + T: Module + Debug + Send + Clone, + B: Backend, +{ + type Record = Option; + + fn visit>(&self, visitor: &mut V) { + if let Some(module) = self { + module.visit(visitor) + } + } + + fn map>(self, mapper: &mut M) -> Self { + self.map(|module| module.map(mapper)) + } + + fn load_record(self, record: Self::Record) -> Self { + let is_constant = self.num_params() == 0; + + if is_constant { + return self; + } + + self.zip(record) + .map(|(module, record)| module.load_record(record)) + } + + fn into_record(self) -> Self::Record { + self.map(Module::into_record) + } + + fn to_device(self, device: &Device) -> Self { + self.map(|module| module.to_device(device)) + } + + fn fork(self, device: &Device) -> Self { + self.map(|module| module.fork(device)) + } + + fn collect_devices(&self, mut devices: Vec) -> Vec { + if let Some(module) = self.as_ref() { + devices = module.collect_devices(devices); + } + + devices + } +} + +impl ModuleDisplayDefault for Option { + fn content(&self, content: Content) -> Option { + match self { + Some(module) => content.add_single(module).optional(), + None => content.add_single("None").optional(), + } + } +} + +impl ModuleDisplay for Option {} + +impl AutodiffModule for Option +where + T: AutodiffModule + Debug + Send + Clone, + B: AutodiffBackend, +{ + type InnerModule = Option; + + fn valid(&self) -> Self::InnerModule { + self.as_ref().map(|module| module.valid()) + } + + fn from_inner(module: Self::InnerModule) -> Self { + module.map(|module| T::from_inner(module)) + } +} + +impl Module for Vec +where + T: Module + Debug + Send + Clone, + B: Backend, +{ + type Record = Vec; + + fn num_params(&self) -> usize { + let mut num_params = 0; + for module in self.iter() { + num_params += module.num_params(); + } + + num_params + } + + fn visit>(&self, visitor: &mut V) { + for (i, module) in self.iter().enumerate() { + let index_str = alloc::format!("{}", i); + visitor.enter_module(&index_str, "Vec"); + module.visit(visitor); + visitor.exit_module(&index_str, "Vec"); + } + } + + fn map>(self, mapper: &mut M) -> Self { + self.into_iter() + .enumerate() + .map(|(i, module)| { + let index_str = alloc::format!("{}", i); + mapper.enter_module(&index_str, "Vec"); + let mapped = module.map(mapper); + mapper.exit_module(&index_str, "Vec"); + mapped + }) + .collect() + } + + fn into_record(self) -> Self::Record { + self.into_iter().map(Module::into_record).collect() + } + + fn load_record(self, record: Self::Record) -> Self { + assert_eq!( + self.len(), + record.len(), + r#"[Load Record Error] The vec record does not the same length as the module. + Make sure you module initialization is compatible with the record being loaded. + "#, + ); + + self.into_iter() + .zip(record) + .map(|(module, record)| module.load_record(record)) + .collect() + } + + fn to_device(self, device: &Device) -> Self { + self.into_iter() + .map(|module| module.to_device(device)) + .collect() + } + + fn fork(self, device: &Device) -> Self { + self.into_iter().map(|module| module.fork(device)).collect() + } + + fn collect_devices(&self, mut devices: Vec) -> Vec { + for module in self.iter() { + devices = module.collect_devices(devices); + } + + devices + } +} + +impl ModuleDisplayDefault for Vec { + fn content(&self, content: Content) -> Option { + self.iter() + .enumerate() + .fold(content, |acc, (i, module)| { + let index = format!("{i}"); + acc.add(&index, module) + }) + .set_top_level_type(format!("Vec<0..{}>", self.len()).as_str()) + .optional() + } +} + +impl ModuleDisplay for Vec {} + +impl AutodiffModule for Vec +where + T: AutodiffModule + Debug + Send + Clone, + B: AutodiffBackend, +{ + type InnerModule = Vec; + + fn valid(&self) -> Self::InnerModule { + self.iter().map(|module| module.valid()).collect() + } + + fn from_inner(module: Self::InnerModule) -> Self { + module + .into_iter() + .map(|module| T::from_inner(module)) + .collect() + } +} + +impl Module for [T; N] +where + T: Module + Debug + Send + Clone, + B: Backend, +{ + type Record = [T::Record; N]; + + fn collect_devices(&self, mut devices: Vec) -> Vec { + for module in self.iter() { + devices = module.collect_devices(devices); + } + + devices + } + + fn num_params(&self) -> usize { + let mut num_params = 0; + for module in self.iter() { + num_params += module.num_params(); + } + + num_params + } + + fn visit>(&self, visitor: &mut V) { + for (i, module) in self.iter().enumerate() { + let index_str = alloc::format!("{}", i); + visitor.enter_module(&index_str, "Array"); + module.visit(visitor); + visitor.exit_module(&index_str, "Array"); + } + } + + fn map>(self, mapper: &mut M) -> Self { + let mut result = Vec::with_capacity(N); + for (i, module) in IntoIterator::into_iter(self).enumerate() { + let index_str = alloc::format!("{}", i); + mapper.enter_module(&index_str, "Array"); + let mapped = module.map(mapper); + mapper.exit_module(&index_str, "Array"); + result.push(mapped); + } + result + .try_into() + .unwrap_or_else(|v: Vec| panic!("Expected array of length {}, got {}", N, v.len())) + } + + fn load_record(self, record: Self::Record) -> Self { + self.into_iter() + .zip(record) + .map(|(module, record)| module.load_record(record)) + .collect::>() + .try_into() + .unwrap() + } + + fn into_record(self) -> Self::Record { + self.map(Module::into_record) + } + + fn to_device(self, device: &Device) -> Self { + self.map(|module| module.to_device(device)) + } + + fn fork(self, device: &Device) -> Self { + self.map(|module| module.fork(device)) + } +} + +impl ModuleDisplayDefault for [T; N] { + fn content(&self, content: Content) -> Option { + self.iter() + .enumerate() + .fold(content, |acc, (i, module)| { + let index = format!("{i}"); + acc.add(&index, module) + }) + .set_top_level_type(format!("[0..{}]", self.len()).as_str()) + .optional() + } +} + +impl ModuleDisplay for [T; N] {} + +impl AutodiffModule for [T; N] +where + T: AutodiffModule + Debug + Send + Clone, + T::InnerModule: Debug, + B: AutodiffBackend, +{ + type InnerModule = [T::InnerModule; N]; + + fn valid(&self) -> Self::InnerModule { + self.clone().map(|module| module.valid()) + } + + fn from_inner(module: Self::InnerModule) -> Self { + module.map(|module| T::from_inner(module)) + } +} + +/// A macro for generating implementations for tuple modules of different sizes. +/// For example: `impl_module_tuple!([L0, L1][0, 1])`. +/// Would generate an implementation for a tuple of size 2. +/// For this macro to work properly, please adhere to the convention: +/// `impl_module_tuple!([L0, L1, ..., Ln][0, 1, ..., n])`. +macro_rules! impl_module_tuple { + // `$l` represents the generic modules. + // `$i` represents the indices of the modules in the tuple. + ([$($l:ident),*][$($i:tt),*]) => { + impl Module for ($($l,)*) + where + B: Backend, + $($l: Module + Debug + Send + Clone,)* + { + type Record = ($($l::Record),*); + + fn collect_devices(&self, mut devices: Vec) -> Vec { + $(devices = self.$i.collect_devices(devices);)* + devices + } + + fn fork(self, device: &Device) -> Self { + ($(self.$i.fork(device),)*) + } + + fn to_device(self, device: &Device) -> Self { + ($(self.$i.to_device(device),)*) + } + + fn visit>(&self, visitor: &mut V) { + $( + let index_str = $i.to_string(); + visitor.enter_module(&index_str, "Tuple"); + self.$i.visit(visitor); + visitor.exit_module(&index_str, "Tuple"); + )* + } + + fn map>(self, mapper: &mut M) -> Self { + ($( + { + let index_str = $i.to_string(); + mapper.enter_module(&index_str, "Tuple"); + let mapped = self.$i.map(mapper); + mapper.exit_module(&index_str, "Tuple"); + mapped + } + ,)*) + } + + fn load_record(self, record: Self::Record) -> Self { + ($(self.$i.load_record(record.$i),)*) + } + + fn into_record(self) -> Self::Record { + ($(self.$i.into_record(),)*) + } + } + + impl AutodiffModule for ($($l,)*) + where + B: AutodiffBackend, + $($l: AutodiffModule + Debug + Send + Clone,)* + { + type InnerModule = ($($l::InnerModule,)*); + + fn valid(&self) -> Self::InnerModule { + ($(self.$i.valid(),)*) + } + + fn from_inner(module: Self::InnerModule) -> Self { + ($($l::from_inner(module.$i),)*) + } + } + + impl<$($l,)*> ModuleDisplayDefault for ($($l,)*) + where + $($l: ModuleDisplay,)* + { + fn content(&self, content: Content) -> Option { + let content = content + $(.add(&format!("{}", $i), &self.$i))* + .set_top_level_type(format!("({})", stringify!($($l),*)).as_str()); + content.optional() + } + } + + impl<$($l,)*> ModuleDisplay for ($($l,)*) where $($l: ModuleDisplay,)* {} + + }; +} + +impl_module_tuple!([L0, L1][0, 1]); +impl_module_tuple!([L0, L1, L2][0, 1, 2]); +impl_module_tuple!([L0, L1, L2, L3][0, 1, 2, 3]); +impl_module_tuple!([L0, L1, L2, L3, L4][0, 1, 2, 3, 4]); +impl_module_tuple!([L0, L1, L2, L3, L4, L5][0, 1, 2, 3, 4, 5]); +impl_module_tuple!([L0, L1, L2, L3, L4, L5, L6][0, 1, 2, 3, 4, 5, 6]); +impl_module_tuple!([L0, L1, L2, L3, L4, L5, L6, L7][0, 1, 2, 3, 4, 5, 6, 7]); +impl_module_tuple!([L0, L1, L2, L3, L4, L5, L6, L7, L8][0, 1, 2, 3, 4, 5, 6, 7, 8]); +impl_module_tuple!([L0, L1, L2, L3, L4, L5, L6, L7, L8, L9][0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn dont_override_constant_module_when_loading_record() { + let module = Some(42); + + let record = Module::::into_record(module); + let loaded = Module::::load_record(module, record); + + assert_eq!(loaded, module); + } + #[test] + fn dont_override_constant_module_when_loading_none_record() { + let module = Some(42); + + let record = None; + let loaded = Module::::load_record(module, record); + + assert_eq!(loaded, module); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/running.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/running.rs new file mode 100644 index 0000000..77b2f93 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/running.rs @@ -0,0 +1,258 @@ +use super::ParamId; +use crate::module::{ + AutodiffModule, Content, Module, ModuleDisplay, ModuleDisplayDefault, ModuleMapper, + ModuleVisitor, Param, +}; + +use alloc::string::ToString; +use alloc::vec::Vec; + +#[cfg(target_has_atomic = "ptr")] +use alloc::sync::Arc; + +#[cfg(not(target_has_atomic = "ptr"))] +use portable_atomic_util::Arc; + +use burn_std::stub::Mutex; +use burn_tensor::{ + Tensor, + backend::{AutodiffBackend, Backend}, + ops::Device, +}; + +#[cfg(feature = "std")] +mod threading { + pub(super) use std::collections::HashMap; + pub(super) use std::thread::ThreadId; + + #[inline(always)] + pub(super) fn get_thread_current_id() -> ThreadId { + std::thread::current().id() + } +} + +#[cfg(not(feature = "std"))] +mod threading { + pub(super) use burn_std::stub::ThreadId; + pub(super) use hashbrown::HashMap; + + #[inline(always)] + pub(super) fn get_thread_current_id() -> ThreadId { + panic!("Current thread id is not available") + } +} + +// Re-export items from the disabled/enabled blocks +use threading::*; + +/// A state that can be updated during the forward pass while being thread safe. +/// +/// # Note +/// +/// The state value is the average of all updates on all threads. +#[derive(Clone, Debug)] +pub struct RunningState { + id: ParamId, + values: Arc>>, + value: Arc>, +} + +// Implement display for the module + +impl core::fmt::Display for RunningState { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "RunningState(id={})", self.id) + } +} + +impl ModuleDisplayDefault for RunningState { + fn content(&self, content: Content) -> Option { + content + .add_formatted(&"RunningState".to_string()) + .optional() + } +} + +impl ModuleDisplay for RunningState {} + +impl Module for RunningState> { + type Record = Param>; + + fn visit>(&self, visitor: &mut V) { + let tensor = self.value.lock().unwrap(); + let param = Param::initialized(self.id, tensor.clone()); + visitor.visit_float(¶m) + } + + fn map>(self, mapper: &mut M) -> Self { + let mut tensor = self.value.lock().unwrap(); + let param = Param::initialized(self.id, tensor.clone()); + let param_out = mapper.map_float(param); + let (_, tensor_out, _) = param_out.consume(); + + *tensor = tensor_out; + core::mem::drop(tensor); + + self + } + + fn into_record(self) -> Self::Record { + self.sync(); + let tensor = self.value.lock().unwrap(); + + Param::initialized(self.id, tensor.clone()) + } + + fn load_record(mut self, record: Self::Record) -> Self { + let mut tensor = self.value.lock().unwrap(); + *tensor = record.val().to_device(&tensor.device()); + self.id = record.id; + + core::mem::drop(tensor); + + self + } + + fn to_device(self, device: &Device) -> Self { + let mut tensor = self.value.lock().unwrap(); + let tensor_out = tensor.clone().to_device(device); + + *tensor = tensor_out; + core::mem::drop(tensor); + + self + } + + fn fork(self, device: &Device) -> Self { + self.to_device(device) // Same thing here since no grad. + } + + fn collect_devices(&self, mut devices: Vec>) -> Vec> { + let device = self.value.lock().unwrap().device(); + + if !devices.contains(&device) { + devices.push(device) + } + + devices + } +} + +impl RunningState> { + /// Create a new running state. + pub fn new(value: Tensor) -> Self { + Self { + id: ParamId::new(), + values: Arc::new(Mutex::new(HashMap::new())), + value: Arc::new(Mutex::new(value)), + } + } + + /// Create a new running state. + pub fn with_id(id: ParamId, value: Tensor) -> Self { + Self { + id, + values: Arc::new(Mutex::new(HashMap::new())), + value: Arc::new(Mutex::new(value)), + } + } + + /// Create a new running state from a record. + pub fn from_record(record: Param>) -> Self { + let tensor = record.val(); + Self { + id: record.id, + values: Arc::new(Mutex::new(HashMap::new())), + value: Arc::new(Mutex::new(tensor)), + } + } + + /// Update the value on the current thread. + pub fn update(&self, value: Tensor) { + let thread_id = get_thread_current_id(); + let mut map = self.values.lock().unwrap(); + + if map.contains_key(&thread_id) { + self.update_value(&mut map); + } + + map.insert(thread_id, value); + } + + /// Get the current value, + /// + /// # Note + /// + /// The current value might be outdated by one update. + pub fn value(&self) -> Tensor { + let value = self.value.lock().unwrap(); + value.clone() + } + + /// Get the current value and make sure it is sync. + /// + /// # Note + /// + /// Don't use this function after an update on the same thread where other threads might have to + /// register their update before the actual synchronization needs to happen. + pub fn value_sync(&self) -> Tensor { + let thread_id = get_thread_current_id(); + let mut map = self.values.lock().unwrap(); + + if map.contains_key(&thread_id) { + self.update_value(&mut map); + } + + let value = self.value.lock().unwrap(); + value.clone() + } + + fn sync(&self) { + let mut map = self.values.lock().unwrap(); + + if !map.is_empty() { + self.update_value(&mut map); + } + } + + fn update_value(&self, map: &mut HashMap>) { + let mut value_updated: Option> = None; + let mut counter = 0; + + for (_key, tensor) in map.drain() { + counter += 1; + + value_updated = match value_updated { + Some(current) => { + let device = current.device(); + Some(tensor.to_device(&device).add(current)) + } + None => Some(tensor), + }; + } + + if let Some(value) = value_updated { + let value = value.div_scalar(counter); + let mut value_old = self.value.lock().unwrap(); + *value_old = value; + } + } +} + +impl AutodiffModule for RunningState> { + type InnerModule = RunningState>; + + fn valid(&self) -> Self::InnerModule { + self.sync(); + let value = self.value(); + + RunningState::with_id(self.id, value.inner()) + } + + fn from_inner(module: Self::InnerModule) -> Self { + module.sync(); + let value = module.value(); + + RunningState::with_id(module.id, Tensor::from_inner(value)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/tensor.rs new file mode 100644 index 0000000..b1bdb9f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/tensor.rs @@ -0,0 +1,571 @@ +use super::{Param, ParamId, Parameter}; +use crate::module::{ + AutodiffModule, Content, HasAutodiffModule, Module, ModuleDisplay, ModuleDisplayDefault, + ModuleMapper, ModuleVisitor, +}; +use crate::tensor::{ + Tensor, + backend::{AutodiffBackend, Backend}, +}; +use alloc::{format, string::ToString, vec::Vec}; +use burn_tensor::{Bool, Float, Int, TensorData, ops::Device}; + +impl Parameter for Tensor { + type Device = B::Device; + + fn device(&self) -> Self::Device { + Tensor::device(self) + } + + fn is_require_grad(&self) -> bool { + Tensor::is_require_grad(self) + } + + fn set_require_grad(self, require_grad: bool) -> Self { + Tensor::set_require_grad(self, require_grad) + } +} + +impl Parameter for Tensor { + type Device = B::Device; + + fn device(&self) -> Self::Device { + Tensor::device(self) + } + + fn is_require_grad(&self) -> bool { + false + } + + fn set_require_grad(self, _require_grad: bool) -> Self { + self + } +} + +impl Parameter for Tensor { + type Device = B::Device; + + fn device(&self) -> Self::Device { + Tensor::device(self) + } + + fn is_require_grad(&self) -> bool { + false + } + + fn set_require_grad(self, _require_grad: bool) -> Self { + self + } +} + +impl Param> { + /// Create a new parameter from a float tensor. + /// + /// # Warnings + /// + /// We strongly recommend using [Param::uninitialized] if you are using this method to + /// initialize parameters inside a module, since the tensor initialization will be lazy, + /// making the loading of weights more performant. + pub fn from_tensor(value: Tensor) -> Self { + // When creating a parameter from a float tensor, we automatically mark it as requiring + // gradients, so that it can be updated by an optimizer. + Param::initialized(ParamId::new(), value.require_grad()) + } + + /// The shape of the parameter, **without triggering initialization**. + /// + /// This is critical for shape validation during loading: when applying tensors to an + /// uninitialized parameter, we need to validate the shape without triggering the + /// initialization function (which would allocate an unnecessary tensor). + /// + /// Use this instead of [crate::tensor::Tensor::shape] when you need the shape but want to + /// preserve lazy initialization. + pub fn lazy_shape(&self) -> burn_tensor::Shape { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.shape(), + }; + + let init = initialization.read().unwrap(); + + match init.as_ref() { + Some(value) => value.shape.clone(), + None => self.shape(), + } + } + + /// Create a new parameter from data. + pub fn from_data(data: T, device: &B::Device) -> Self + where + T: Into, + { + // When creating a parameter from a float tensor, we automatically mark it as requiring + // gradients, so that it can be updated by an optimizer. + B::memory_persistent_allocations(device, data, |data| { + let value = Tensor::from_data(data, device); + Param::initialized(ParamId::new(), value.require_grad()) + }) + } + + /// Transform a parameter for loading by applying load transformations. + /// + /// This method is used to restore a parameter from a tensor (typically during deserialization). + /// It ensures the tensor is moved to the expected device, applies the param mapper's + /// `on_load` transformation, and preserves the autodiff settings (require_grad). + pub fn transform_for_load(self, tensor: Tensor, param_id: ParamId) -> Self { + let mut new_tensor = tensor; + + let mapper = self.param_mapper.clone(); + + let expected_device = self.lazy_device(); + let expected_require_grad = self.lazy_is_require_grad(); + + // Make sure we load the tensor into the same module device. + if new_tensor.device() != expected_device { + new_tensor = new_tensor.to_device(&expected_device).detach(); + } + + new_tensor = mapper.on_load(new_tensor); + + // Make sure we load the tensor with the same autodiff setting. + new_tensor = new_tensor.set_require_grad(expected_require_grad); + + let mut loaded = Self::initialized(param_id, new_tensor); + loaded.param_mapper = mapper; + loaded + } + + /// Transform a parameter for saving by applying save transformations. + /// + /// This method is used to prepare a parameter for saving (typically during serialization). + /// It applies the param mapper's `on_save` transformation, which can be used + /// to modify the tensor before serialization (e.g., quantization, precision conversion). + pub fn transform_for_save(&self) -> Self { + let mut tensor = self.val(); + let mapper = self.param_mapper.clone(); + + tensor = mapper.on_save(tensor); + + Self::initialized(self.id, tensor) + } +} + +impl Param> { + /// The shape of the parameter, **without triggering initialization**. + /// + /// This is critical for shape validation during loading: when applying tensors to an + /// uninitialized parameter, we need to validate the shape without triggering the + /// initialization function (which would allocate an unnecessary tensor). + /// + /// Use this instead of [crate::tensor::Tensor::shape] when you need the shape but want to + /// preserve lazy initialization. + pub fn lazy_shape(&self) -> burn_tensor::Shape { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.shape(), + }; + + let init = initialization.read().unwrap(); + + match init.as_ref() { + Some(value) => value.shape.clone(), + None => self.shape(), + } + } + + /// Transform a parameter for loading by applying load transformations. + /// + /// This method is used to restore a parameter from a tensor (typically during deserialization). + /// It ensures the tensor is moved to the expected device and applies the param mapper's + /// `on_load` transformation. + pub fn transform_for_load(self, tensor: Tensor, param_id: ParamId) -> Self { + let mut new_tensor = tensor; + + let mapper = self.param_mapper.clone(); + + let expected_device = self.lazy_device(); + + // Make sure we load the tensor into the same module device. + if new_tensor.device() != expected_device { + new_tensor = new_tensor.to_device(&expected_device); + } + + new_tensor = mapper.on_load(new_tensor); + + let mut loaded = Self::initialized(param_id, new_tensor); + loaded.param_mapper = mapper; + loaded + } + + /// Transform a parameter for saving by applying save transformations. + /// + /// This method is used to prepare a parameter for saving (typically during serialization). + /// It applies the param mapper's `on_save` transformation, which can be used + /// to modify the tensor before serialization (e.g., quantization, precision conversion). + pub fn transform_for_save(&self) -> Self { + let mut tensor = self.val(); + let mapper = self.param_mapper.clone(); + + tensor = mapper.on_save(tensor); + + Self::initialized(self.id, tensor) + } +} + +impl Param> { + /// The shape of the parameter, **without triggering initialization**. + /// + /// This is critical for shape validation during loading: when applying tensors to an + /// uninitialized parameter, we need to validate the shape without triggering the + /// initialization function (which would allocate an unnecessary tensor). + /// + /// **Returns:** + /// - For uninitialized params: the shape from the `Uninitialized` struct + /// - For initialized params: the actual shape from the tensor + /// + /// Use this instead of [crate::tensor::Tensor::shape] when you need the shape but want to + /// preserve lazy initialization. + pub fn lazy_shape(&self) -> burn_tensor::Shape { + let initialization = match &self.initialization { + Some(init) => init, + None => return self.shape(), + }; + + let init = initialization.read().unwrap(); + + match init.as_ref() { + Some(value) => value.shape.clone(), + None => self.shape(), + } + } + + /// Transform a parameter for loading by applying load transformations. + /// + /// This method is used to restore a parameter from a tensor (typically during deserialization). + /// It ensures the tensor is moved to the expected device and applies the param mapper's + /// `on_load` transformation. + pub fn transform_for_load(self, tensor: Tensor, param_id: ParamId) -> Self { + let mut new_tensor = tensor; + + let mapper = self.param_mapper.clone(); + + let expected_device = self.lazy_device(); + + // Make sure we load the tensor into the same module device. + if new_tensor.device() != expected_device { + new_tensor = new_tensor.to_device(&expected_device); + } + + new_tensor = mapper.on_load(new_tensor); + + let mut loaded = Self::initialized(param_id, new_tensor); + loaded.param_mapper = mapper; + loaded + } + + /// Transform a parameter for saving by applying save transformations. + /// + /// This method is used to prepare a parameter for saving (typically during serialization). + /// It applies the param mapper's `on_save` transformation, which can be used + /// to modify the tensor before serialization (e.g., quantization, precision conversion). + pub fn transform_for_save(&self) -> Self { + let mut tensor = self.val(); + let mapper = self.param_mapper.clone(); + + tensor = mapper.on_save(tensor); + + Self::initialized(self.id, tensor) + } +} + +impl Module for Param> { + type Record = Param>; + + fn visit>(&self, visitor: &mut V) { + visitor.visit_float(self) + } + + fn map>(self, mapper: &mut M) -> Self { + mapper.map_float(self) + } + + fn into_record(self) -> Self::Record { + self.transform_for_save() + } + + fn load_record(self, record: Self::Record) -> Self { + let (record_param_id, record_tensor, _) = record.consume(); + self.transform_for_load(record_tensor, record_param_id) + } + + fn to_device(self, device: &Device) -> Self { + self.map(|tensor| tensor.to_device(device)) + } + + fn fork(self, device: &Device) -> Self { + self.map(|tensor| { + let is_require_grad = tensor.is_require_grad(); + let mut tensor = tensor.to_device(device).detach(); + + if is_require_grad { + tensor = tensor.require_grad(); + } + + tensor + }) + } + + fn collect_devices(&self, mut devices: Vec>) -> Vec> { + let device = self.val().device(); + + if !devices.contains(&device) { + devices.push(device) + } + + devices + } +} + +impl ModuleDisplayDefault for Param> { + fn content(&self, content: Content) -> Option { + let id = if content.display_settings.show_param_id() { + format!(", id: {}", self.id) + } else { + "".to_string() + }; + let string = format!( + "ParamTensor {{rank: {D}, shape: {:?}, kind: float{id}}}", + self.shape().as_slice() + ); + content.add_formatted(&string).optional() + } +} +impl ModuleDisplay for Param> {} + +impl Module for Param> { + type Record = Param>; + + fn visit>(&self, visitor: &mut V) { + visitor.visit_int(self) + } + + fn map>(self, mapper: &mut M) -> Self { + mapper.map_int(self) + } + + fn into_record(self) -> Self::Record { + self.transform_for_save() + } + + fn load_record(self, record: Self::Record) -> Self { + let (record_param_id, record_tensor, _) = record.consume(); + self.transform_for_load(record_tensor, record_param_id) + } + + fn to_device(self, device: &Device) -> Self { + self.map(|tensor| tensor.to_device(device)) + } + + fn fork(self, device: &Device) -> Self { + self.to_device(device) // Don't support autodiff. + } + + fn collect_devices(&self, mut devices: Vec>) -> Vec> { + let device = self.val().device(); + + if !devices.contains(&device) { + devices.push(device) + } + + devices + } +} + +impl ModuleDisplayDefault for Param> { + fn content(&self, content: Content) -> Option { + let id = if content.display_settings.show_param_id() { + format!(", id: {}", self.id) + } else { + "".to_string() + }; + let string = format!( + "ParamTensor {{rank: {D}, shape: {:?}, kind: int{id}}}", + self.shape().as_slice() + ); + content.add_formatted(&string).optional() + } +} +impl ModuleDisplay for Param> {} + +impl Module for Param> { + type Record = Param>; + + fn visit>(&self, visitor: &mut V) { + visitor.visit_bool(self) + } + + fn map>(self, mapper: &mut M) -> Self { + mapper.map_bool(self) + } + + fn into_record(self) -> Self::Record { + self.transform_for_save() + } + + fn load_record(self, record: Self::Record) -> Self { + let (record_param_id, record_tensor, _) = record.consume(); + self.transform_for_load(record_tensor, record_param_id) + } + + fn to_device(self, device: &Device) -> Self { + self.map(|tensor| tensor.to_device(device)) + } + + fn fork(self, device: &Device) -> Self { + self.to_device(device) // Don't support autodiff. + } + + fn collect_devices(&self, mut devices: Vec>) -> Vec> { + let device = self.val().device(); + + if !devices.contains(&device) { + devices.push(device) + } + + devices + } +} + +impl ModuleDisplayDefault for Param> { + fn content(&self, content: Content) -> Option { + let id = if content.display_settings.show_param_id() { + format!(", id: {}", self.id) + } else { + "".to_string() + }; + + let string = format!( + "ParamTensor {{rank: {D}, shape: {:?}, kind: bool{id}}}", + self.shape().as_slice() + ); + content.add_formatted(&string).optional() + } +} + +impl ModuleDisplay for Param> {} + +impl AutodiffModule for Param> { + type InnerModule = Param>; + + fn valid(&self) -> Self::InnerModule { + // Preserve initialized param `require_grad` state, but reset the inner value's + let require_grad = self.require_grad; + let mut param = Param::initialized(self.id, self.val().inner().set_require_grad(false)); + param.require_grad = require_grad; + param + } + + fn from_inner(module: Self::InnerModule) -> Self { + // Reinstate the param's `require_grad` state + let tensor = Tensor::from_inner(module.val()).set_require_grad(module.require_grad); + Param::initialized(module.id, tensor) + } +} + +impl HasAutodiffModule + for Param> +{ + type TrainModule = Param>; +} + +impl AutodiffModule for Param> { + type InnerModule = Param>; + + fn valid(&self) -> Self::InnerModule { + Param::initialized(self.id, self.val().inner()) + } + + fn from_inner(module: Self::InnerModule) -> Self { + Param::initialized(module.id, Tensor::from_inner(module.val())) + } +} + +impl AutodiffModule for Param> { + type InnerModule = Param>; + + fn valid(&self) -> Self::InnerModule { + Param::initialized(self.id, self.val().inner()) + } + + fn from_inner(module: Self::InnerModule) -> Self { + Param::initialized(module.id, Tensor::from_inner(module.val())) + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + use crate::{ + TestAutodiffBackend, + module::Module, + record::{BinBytesRecorder, FullPrecisionSettings, Recorder}, + }; + + #[test] + fn test_load_record_setting() { + let device = Default::default(); + let tensor = Tensor::::ones([3, 3], &device).require_grad(); + + let byte_recorder = BinBytesRecorder::::default(); + let bytes = byte_recorder + .record( + Param::initialized(ParamId::new(), tensor.clone()).into_record(), + (), + ) + .unwrap(); + + let no_grad_is_require_grad = Param::initialized(ParamId::new(), tensor.clone()) + .no_grad() + .load_record(byte_recorder.load(bytes.clone(), &device).unwrap()) + .is_require_grad(); + + let with_default_is_require_grad = Param::initialized(ParamId::new(), tensor) + .load_record(byte_recorder.load(bytes, &device).unwrap()) + .is_require_grad(); + + assert!(!no_grad_is_require_grad); + assert!(with_default_is_require_grad); + } + + #[test] + fn test_param_require_grad_stateful() { + let device = Default::default(); + let tensor = Tensor::::ones([3, 3], &device).require_grad(); + + let param = Param::initialized(ParamId::new(), tensor); + assert!(param.is_require_grad()); + assert!(param.require_grad); + + let param = param.valid(); + assert!(!param.is_require_grad()); + assert!(param.require_grad); // stateful + + // Without `HasAutodiffModule`, we would need to specify the param type as well, which would be annoying: + // let param: Param> = param.train(); + let param = param.train::(); + assert!(param.is_require_grad()); + assert!(param.require_grad); // stateful + + let param = param.no_grad(); + assert!(!param.is_require_grad()); + assert!(!param.require_grad); // stateful + + let param = param.valid(); + assert!(!param.is_require_grad()); // always + assert!(!param.require_grad); // stateful + + let param = param.train::(); + assert!(!param.is_require_grad()); + assert!(!param.require_grad); // stateful + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/visitor.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/visitor.rs new file mode 100644 index 0000000..169f260 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/param/visitor.rs @@ -0,0 +1,38 @@ +use super::{Param, ParamId}; +use crate::module::{Module, ModuleVisitor}; +use alloc::vec::Vec; +use burn_tensor::{Bool, Int, Tensor, backend::Backend}; +use core::marker::PhantomData; + +struct ParamIdCollector<'a, M> { + param_ids: &'a mut Vec, + phantom: PhantomData, +} + +impl ModuleVisitor for ParamIdCollector<'_, M> +where + B: Backend, + M: Module, +{ + fn visit_float(&mut self, param: &Param>) { + self.param_ids.push(param.id); + } + fn visit_int(&mut self, param: &Param>) { + self.param_ids.push(param.id); + } + fn visit_bool(&mut self, param: &Param>) { + self.param_ids.push(param.id); + } +} + +/// List all the parameter ids in a module. +pub fn list_param_ids, B: Backend>(module: &M) -> Vec { + let mut params_ids = Vec::new(); + let mut visitor = ParamIdCollector { + param_ids: &mut params_ids, + phantom: PhantomData::, + }; + module.visit(&mut visitor); + + params_ids +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/quantize.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/quantize.rs new file mode 100644 index 0000000..6d86418 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/quantize.rs @@ -0,0 +1,65 @@ +use burn_tensor::{ + Tensor, + backend::Backend, + quantization::{Calibration, QuantScheme, compute_q_params, compute_range}, +}; + +use crate::module::{ModuleMapper, Param}; + +/// Describes how to quantize a module. +pub struct Quantizer { + /// The calibration method used in quantization. + pub calibration: Calibration, + /// The quantization scheme. + pub scheme: QuantScheme, +} + +impl ModuleMapper for Quantizer { + fn map_float(&mut self, param: Param>) -> Param> { + let (id, tensor, mapper) = param.consume(); + let range = compute_range(&self.scheme, &tensor, &self.calibration); + let qparams = compute_q_params(&self.scheme, range); + let tensor = tensor.quantize(&self.scheme, qparams); + Param::from_mapped_value(id, tensor, mapper) + } +} + +#[cfg(all(test, not(feature = "test-tch")))] +mod tests { + use crate::test_utils::SimpleLinear; + use crate::{ + TestBackend, + module::{Module, Quantizer}, + }; + use burn_tensor::{ + Device, Tolerance, + ops::QuantizedTensor, + quantization::{Calibration, QTensorPrimitive, QuantLevel, QuantParam, QuantValue}, + }; + + type B = TestBackend; + + #[test] + fn should_quantize_module() { + let device: Device = Default::default(); + let module = SimpleLinear::::new(32, 32, &device); + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::Tensor) + .with_param(QuantParam::F32); + + let result = module.weight.val(); + + let calibration = Calibration::MinMax; + let mut quantizer = Quantizer { + calibration, + scheme, + }; + let q_module = module.quantize_weights(&mut quantizer); + let q_result = q_module.weight.val().dequantize(); + + result + .into_data() + .assert_approx_eq::(&q_result.into_data(), Tolerance::permissive()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/reinit.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/reinit.rs new file mode 100644 index 0000000..10d6e9b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/module/reinit.rs @@ -0,0 +1,203 @@ +use super::{Module, ModuleMapper}; +use burn_tensor::{ + Element, ElementConversion, Tensor, TensorData, + backend::Backend, + ops::{FloatElem, IntElem}, +}; +use rand::{RngExt, SeedableRng}; + +#[derive(Debug)] +/// Overrides float and int tensors of [burn modules](super::Module). +/// +/// This is useful for testing. +pub struct Reinitializer { + float: ReinitStrategy>, + int: ReinitStrategy>, +} + +#[derive(Debug)] +#[allow(missing_docs)] +enum ReinitStrategy { + Range { min: E, max: E }, + Constant { value: E }, + Random { seed: u64, min: E, max: E }, +} + +impl Default for Reinitializer { + fn default() -> Self { + Self::new() + } +} + +impl Reinitializer { + /// Create a new [reinitializer](Reinitializer). + pub fn new() -> Self { + Self { + float: ReinitStrategy::Constant { + value: 0.elem::>(), + }, + int: ReinitStrategy::Constant { + value: 0.elem::>(), + }, + } + } + + /// Apply the reinitialization to the given [module](Module). + pub fn apply>(mut self, module: M) -> M { + module.map(&mut self) + } + + /// Set the reinitialization strategy to constant for all tensors. + pub fn constant(self, constant: f64) -> Self { + self.constant_float(constant).constant_int(constant as i64) + } + + /// Set the reinitialization strategy to constant for float tensors. + pub fn constant_float(mut self, constant: f64) -> Self { + self.float = ReinitStrategy::Constant { + value: constant.elem(), + }; + self + } + + /// Set the reinitialization strategy to constant for int tensors. + pub fn constant_int(mut self, constant: i64) -> Self { + self.int = ReinitStrategy::Constant { + value: constant.elem(), + }; + self + } + /// Set the reinitialization strategy to random for all tensors. + pub fn random(self, seed: u64, min: f64, max: f64) -> Self { + self.random_float(seed, min, max) + .random_int(seed, min as i64, max as i64) + } + + /// Set the reinitialization strategy to random for float tensors. + pub fn random_float(mut self, seed: u64, min: f64, max: f64) -> Self { + self.float = ReinitStrategy::Random { + seed, + min: min.elem(), + max: max.elem(), + }; + self + } + + /// Set the reinitialization strategy to random for int tensors. + pub fn random_int(mut self, seed: u64, min: i64, max: i64) -> Self { + self.int = ReinitStrategy::Random { + seed, + min: min.elem(), + max: max.elem(), + }; + self + } + + /// Set the reinitialization strategy to range for all tensors. + pub fn range(self, min: f64, max: f64) -> Self { + self.range_float(min, max).range_int(min as i64, max as i64) + } + + /// Set the reinitialization strategy to range for float tensors. + pub fn range_float(mut self, min: f64, max: f64) -> Self { + self.float = ReinitStrategy::Range { + min: min.elem(), + max: max.elem(), + }; + self + } + + /// Set the reinitialization strategy to range for int tensors. + pub fn range_int(mut self, min: i64, max: i64) -> Self { + self.int = ReinitStrategy::Range { + min: min.elem(), + max: max.elem(), + }; + self + } +} + +impl ModuleMapper for Reinitializer { + fn map_float( + &mut self, + param: super::Param>, + ) -> super::Param> { + let (id, tensor, mapper) = param.consume(); + let device = tensor.device(); + let shape = tensor.shape(); + let num_elements = shape.num_elements(); + + let tensor = match &self.float { + ReinitStrategy::Range { min, max } => { + let tensor = Tensor::arange(0..num_elements as i64, &device) + .reshape(shape) + .float(); + let (factor, bias) = resolve::>(*min, *max, num_elements); + tensor * factor + bias + } + ReinitStrategy::Constant { value } => Tensor::full(shape, *value, &device), + ReinitStrategy::Random { seed, min, max } => { + let data = TensorData::new( + random_vector::>(*seed, min.elem(), max.elem(), num_elements), + shape, + ); + Tensor::from_data(data, &device) + } + }; + + super::Param::from_mapped_value(id, tensor, mapper) + } + + fn map_int( + &mut self, + param: super::Param>, + ) -> super::Param> { + let (id, tensor, mapper) = param.consume(); + let device = tensor.device(); + let shape = tensor.shape(); + let num_elements = shape.num_elements(); + + let tensor = match &self.int { + ReinitStrategy::Range { min, max } => { + let tensor = Tensor::arange(0..num_elements as i64, &device).reshape(shape); + let (factor, bias) = resolve::>(*min, *max, num_elements); + tensor * factor + bias + } + ReinitStrategy::Constant { value } => Tensor::full(shape, *value, &device), + ReinitStrategy::Random { seed, min, max } => { + let data = TensorData::new( + random_vector::>(*seed, min.elem(), max.elem(), num_elements), + shape, + ); + Tensor::from_data(data, &device) + } + }; + + super::Param::from_mapped_value(id, tensor, mapper) + } + + fn map_bool( + &mut self, + param: super::Param>, + ) -> super::Param> { + let (id, tensor, mapper) = param.consume(); + super::Param::from_mapped_value(id, tensor, mapper) + } +} + +fn resolve(min: E, max: E, num_elements: usize) -> (E, E) { + let range = max.elem::() - min.elem::(); + let factor = range / num_elements as f64; + let bias = min.elem::(); + + (factor.elem(), bias.elem()) +} + +fn random_vector(seed: u64, min: f64, max: f64, num_elements: usize) -> Vec { + let mut rng = rand::rngs::StdRng::seed_from_u64(seed); + let dist = rand::distr::Uniform::new(min, max).unwrap(); + (0..num_elements) + .map(|_| rng.sample(dist)) + .map(|e| e.elem::()) + .collect() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/base.rs new file mode 100644 index 0000000..a213384 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/base.rs @@ -0,0 +1,17 @@ +pub use burn_derive::Record; +use burn_tensor::backend::Backend; + +use super::PrecisionSettings; +use serde::{Serialize, de::DeserializeOwned}; + +/// Trait to define a family of types which can be recorded using any [settings](PrecisionSettings). +pub trait Record: Send { + /// Type of the item that can be serialized and deserialized. + type Item: Serialize + DeserializeOwned + Clone; + + /// Convert the current record into the corresponding item that follows the given [settings](PrecisionSettings). + fn into_item(self) -> Self::Item; + + /// Convert the given item into a record. + fn from_item(item: Self::Item, device: &B::Device) -> Self; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/file.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/file.rs new file mode 100644 index 0000000..1cf5ec3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/file.rs @@ -0,0 +1,421 @@ +use super::{PrecisionSettings, Recorder, RecorderError, bin_config}; +use burn_tensor::backend::Backend; +use core::marker::PhantomData; +use flate2::{Compression, read::GzDecoder, write::GzEncoder}; +use serde::{Serialize, de::DeserializeOwned}; +use std::io::{BufReader, BufWriter}; +use std::{fs::File, path::PathBuf}; + +/// Recorder trait specialized to save and load data to and from files. +pub trait FileRecorder: + Recorder +{ + /// File extension of the format used by the recorder. + fn file_extension() -> &'static str; +} + +/// Default [file recorder](FileRecorder). +pub type DefaultFileRecorder = NamedMpkFileRecorder; + +/// File recorder using the [bincode format](bincode). +#[derive(new, Debug, Default, Clone)] +pub struct BinFileRecorder { + _settings: PhantomData, +} + +/// File recorder using the [bincode format](bincode) compressed with gzip. +#[derive(new, Debug, Default, Clone)] +pub struct BinGzFileRecorder { + _settings: PhantomData, +} + +/// File recorder using the [json format](serde_json) compressed with gzip. +#[derive(new, Debug, Default, Clone)] +pub struct JsonGzFileRecorder { + _settings: PhantomData, +} + +/// File recorder using [pretty json format](serde_json) for easy readability. +#[derive(new, Debug, Default, Clone)] +pub struct PrettyJsonFileRecorder { + _settings: PhantomData, +} + +/// File recorder using the [named msgpack](rmp_serde) format compressed with gzip. +#[derive(new, Debug, Default, Clone)] +pub struct NamedMpkGzFileRecorder { + _settings: PhantomData, +} + +/// File recorder using the [named msgpack](rmp_serde) format. +#[derive(new, Debug, Default, Clone)] +pub struct NamedMpkFileRecorder { + _settings: PhantomData, +} + +impl FileRecorder for BinGzFileRecorder { + fn file_extension() -> &'static str { + "bin.gz" + } +} +impl FileRecorder for BinFileRecorder { + fn file_extension() -> &'static str { + "bin" + } +} +impl FileRecorder for JsonGzFileRecorder { + fn file_extension() -> &'static str { + "json.gz" + } +} +impl FileRecorder for PrettyJsonFileRecorder { + fn file_extension() -> &'static str { + "json" + } +} + +impl FileRecorder for NamedMpkGzFileRecorder { + fn file_extension() -> &'static str { + "mpk.gz" + } +} + +impl FileRecorder for NamedMpkFileRecorder { + fn file_extension() -> &'static str { + "mpk" + } +} + +macro_rules! str2reader { + ( + $file:expr + ) => {{ + $file.set_extension(>::file_extension()); + let path = $file.as_path(); + + File::open(path) + .map_err(|err| match err.kind() { + std::io::ErrorKind::NotFound => RecorderError::FileNotFound(err.to_string()), + _ => RecorderError::Unknown(err.to_string()), + }) + .map(|file| BufReader::new(file)) + }}; +} + +macro_rules! str2writer { + ( + $file:expr + ) => {{ + $file.set_extension(>::file_extension()); + let path = $file.as_path(); + + log::debug!("Writing to file: {:?}", path); + + // Add parent directories if they don't exist + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).ok(); + } + + if path.exists() { + log::warn!("File exists, replacing"); + std::fs::remove_file(path).map_err(|err| RecorderError::Unknown(err.to_string()))?; + } + + File::create(path) + .map_err(|err| match err.kind() { + std::io::ErrorKind::NotFound => RecorderError::FileNotFound(err.to_string()), + _ => RecorderError::Unknown(err.to_string()), + }) + .map(|file| BufWriter::new(file)) + }}; +} + +impl Recorder for BinGzFileRecorder { + type Settings = S; + type RecordArgs = PathBuf; + type RecordOutput = (); + type LoadArgs = PathBuf; + + fn save_item( + &self, + item: I, + mut file: Self::RecordArgs, + ) -> Result<(), RecorderError> { + let config = bin_config(); + let writer = str2writer!(file)?; + let mut writer = GzEncoder::new(writer, Compression::default()); + + bincode::serde::encode_into_std_write(&item, &mut writer, config) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(()) + } + + fn load_item( + &self, + file: &mut Self::LoadArgs, + ) -> Result { + let reader = str2reader!(file)?; + let mut reader = GzDecoder::new(reader); + let state = bincode::serde::decode_from_std_read(&mut reader, bin_config()) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(state) + } +} + +impl Recorder for BinFileRecorder { + type Settings = S; + type RecordArgs = PathBuf; + type RecordOutput = (); + type LoadArgs = PathBuf; + + fn save_item( + &self, + item: I, + mut file: Self::RecordArgs, + ) -> Result<(), RecorderError> { + let config = bin_config(); + let mut writer = str2writer!(file)?; + bincode::serde::encode_into_std_write(&item, &mut writer, config) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + Ok(()) + } + + fn load_item( + &self, + file: &mut Self::LoadArgs, + ) -> Result { + let mut reader = str2reader!(file)?; + let state = bincode::serde::decode_from_std_read(&mut reader, bin_config()) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + Ok(state) + } +} + +impl Recorder for JsonGzFileRecorder { + type Settings = S; + type RecordArgs = PathBuf; + type RecordOutput = (); + type LoadArgs = PathBuf; + + fn save_item( + &self, + item: I, + mut file: Self::RecordArgs, + ) -> Result<(), RecorderError> { + let writer = str2writer!(file)?; + let writer = GzEncoder::new(writer, Compression::default()); + serde_json::to_writer(writer, &item) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(()) + } + + fn load_item( + &self, + file: &mut Self::LoadArgs, + ) -> Result { + let reader = str2reader!(file)?; + let reader = GzDecoder::new(reader); + let state = serde_json::from_reader(reader) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(state) + } +} + +impl Recorder for PrettyJsonFileRecorder { + type Settings = S; + type RecordArgs = PathBuf; + type RecordOutput = (); + type LoadArgs = PathBuf; + + fn save_item( + &self, + item: I, + mut file: Self::RecordArgs, + ) -> Result<(), RecorderError> { + let writer = str2writer!(file)?; + serde_json::to_writer_pretty(writer, &item) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + Ok(()) + } + + fn load_item( + &self, + file: &mut Self::LoadArgs, + ) -> Result { + let reader = str2reader!(file)?; + let state = serde_json::from_reader(reader) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(state) + } +} + +impl Recorder for NamedMpkGzFileRecorder { + type Settings = S; + type RecordArgs = PathBuf; + type RecordOutput = (); + type LoadArgs = PathBuf; + + fn save_item( + &self, + item: I, + mut file: Self::RecordArgs, + ) -> Result<(), RecorderError> { + let writer = str2writer!(file)?; + let mut writer = GzEncoder::new(writer, Compression::default()); + rmp_serde::encode::write_named(&mut writer, &item) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(()) + } + + fn load_item( + &self, + file: &mut Self::LoadArgs, + ) -> Result { + let reader = str2reader!(file)?; + let reader = GzDecoder::new(reader); + let state = rmp_serde::decode::from_read(reader) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(state) + } +} + +impl Recorder for NamedMpkFileRecorder { + type Settings = S; + type RecordArgs = PathBuf; + type RecordOutput = (); + type LoadArgs = PathBuf; + + fn save_item( + &self, + item: I, + mut file: Self::RecordArgs, + ) -> Result<(), RecorderError> { + let mut writer = str2writer!(file)?; + + rmp_serde::encode::write_named(&mut writer, &item) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(()) + } + + fn load_item( + &self, + file: &mut Self::LoadArgs, + ) -> Result { + let reader = str2reader!(file)?; + let state = rmp_serde::decode::from_read(reader) + .map_err(|err| RecorderError::Unknown(err.to_string()))?; + + Ok(state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate as burn; + use crate::config::Config; + use crate::module::Ignored; + use crate::test_utils::SimpleLinear; + use crate::{ + TestBackend, + module::Module, + record::{BinBytesRecorder, FullPrecisionSettings}, + }; + use burn_tensor::backend::Backend; + + #[inline(always)] + fn file_path() -> PathBuf { + std::env::temp_dir() + .as_path() + .join("burn_test_file_recorder") + } + + #[test] + fn test_can_save_and_load_jsongz_format() { + test_can_save_and_load(JsonGzFileRecorder::::default()) + } + + #[test] + fn test_can_save_and_load_bin_format() { + test_can_save_and_load(BinFileRecorder::::default()) + } + + #[test] + fn test_can_save_and_load_bingz_format() { + test_can_save_and_load(BinGzFileRecorder::::default()) + } + + #[test] + fn test_can_save_and_load_pretty_json_format() { + test_can_save_and_load(PrettyJsonFileRecorder::::default()) + } + + #[test] + fn test_can_save_and_load_mpkgz_format() { + test_can_save_and_load(NamedMpkGzFileRecorder::::default()) + } + + #[test] + fn test_can_save_and_load_mpk_format() { + test_can_save_and_load(NamedMpkFileRecorder::::default()) + } + + fn test_can_save_and_load(recorder: Recorder) + where + Recorder: FileRecorder, + { + let device = Default::default(); + let model_before = create_model(&device); + recorder + .record(model_before.clone().into_record(), file_path()) + .unwrap(); + + let model_after = + create_model(&device).load_record(recorder.load(file_path(), &device).unwrap()); + + let byte_recorder = BinBytesRecorder::::default(); + let model_bytes_before = byte_recorder + .record(model_before.into_record(), ()) + .unwrap(); + let model_bytes_after = byte_recorder.record(model_after.into_record(), ()).unwrap(); + + assert_eq!(model_bytes_after, model_bytes_before); + } + + #[derive(Config, Debug)] + pub enum PaddingConfig2d { + Same, + Valid, + Explicit(usize, usize), + } + + // Dummy model with different record types + #[derive(Module, Debug)] + pub struct Model { + linear1: SimpleLinear, + phantom: PhantomData, + arr: [usize; 2], + int: usize, + ignore: Ignored, + } + + pub fn create_model(device: &::Device) -> Model { + let linear1 = SimpleLinear::new(32, 32, device); + + Model { + linear1, + phantom: PhantomData, + arr: [2, 2], + int: 0, + ignore: Ignored(PaddingConfig2d::Same), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/memory.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/memory.rs new file mode 100644 index 0000000..327b0e7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/memory.rs @@ -0,0 +1,141 @@ +use super::{PrecisionSettings, Recorder, RecorderError, bin_config}; +use alloc::vec::Vec; +use burn_tensor::backend::Backend; +use serde::{Serialize, de::DeserializeOwned}; + +/// Recorder trait specialized to save and load data to and from bytes. +/// +/// # Notes +/// +/// This is especially useful in no_std environment where weights are stored directly in +/// compiled binaries. +pub trait BytesRecorder< + B: Backend, + L: AsRef<[u8]> + Send + Sync + core::fmt::Debug + Clone + core::default::Default, +>: Recorder, LoadArgs = L> +{ +} + +/// In memory recorder using the [bincode format](bincode). +#[derive(new, Debug, Default, Clone)] +pub struct BinBytesRecorder< + S: PrecisionSettings, + L: AsRef<[u8]> + Send + Sync + core::fmt::Debug + Clone + core::default::Default = Vec, +> { + _settings: core::marker::PhantomData, + _loadargs: core::marker::PhantomData, +} + +impl< + S: PrecisionSettings, + B: Backend, + L: AsRef<[u8]> + Send + Sync + core::fmt::Debug + Clone + core::default::Default, +> BytesRecorder for BinBytesRecorder +{ +} + +impl< + S: PrecisionSettings, + B: Backend, + L: AsRef<[u8]> + Send + Sync + core::fmt::Debug + Clone + core::default::Default, +> Recorder for BinBytesRecorder +{ + type Settings = S; + type RecordArgs = (); + type RecordOutput = Vec; + type LoadArgs = L; + + fn save_item( + &self, + item: I, + _args: Self::RecordArgs, + ) -> Result { + Ok(bincode::serde::encode_to_vec(item, bin_config()).unwrap()) + } + + fn load_item( + &self, + args: &mut Self::LoadArgs, + ) -> Result { + let state = bincode::borrow_decode_from_slice::<'_, bincode::serde::BorrowCompat, _>( + args.as_ref(), + bin_config(), + ) + .unwrap() + .0; + Ok(state.0) + } +} + +#[cfg(feature = "std")] +/// In memory recorder using the [Named MessagePack](rmp_serde). +#[derive(new, Debug, Default, Clone)] +pub struct NamedMpkBytesRecorder { + _settings: core::marker::PhantomData, +} + +#[cfg(feature = "std")] +impl BytesRecorder> for NamedMpkBytesRecorder {} + +#[cfg(feature = "std")] +impl Recorder for NamedMpkBytesRecorder { + type Settings = S; + type RecordArgs = (); + type RecordOutput = Vec; + type LoadArgs = Vec; + + fn save_item( + &self, + item: I, + _args: Self::RecordArgs, + ) -> Result { + rmp_serde::encode::to_vec_named(&item).map_err(|e| RecorderError::Unknown(e.to_string())) + } + fn load_item( + &self, + args: &mut Self::LoadArgs, + ) -> Result { + rmp_serde::decode::from_slice(args).map_err(|e| RecorderError::Unknown(e.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::SimpleLinear; + use crate::{ + TestBackend, module::Module, record::FullPrecisionSettings, tensor::backend::Backend, + }; + + #[test] + fn test_can_save_and_load_bin_format() { + test_can_save_and_load(BinBytesRecorder::::default()) + } + + #[cfg(feature = "std")] + #[test] + fn test_can_save_and_load_named_mpk_format() { + test_can_save_and_load(NamedMpkBytesRecorder::::default()) + } + + fn test_can_save_and_load(recorder: Recorder) + where + Recorder: BytesRecorder>, + { + let device = Default::default(); + let model1 = create_model::(&device); + let model2 = create_model::(&device); + let bytes1 = recorder.record(model1.into_record(), ()).unwrap(); + let bytes2 = recorder.record(model2.clone().into_record(), ()).unwrap(); + + let model2_after = model2.load_record(recorder.load(bytes1.clone(), &device).unwrap()); + let bytes2_after = recorder.record(model2_after.into_record(), ()).unwrap(); + + assert_ne!(bytes1, bytes2); + assert_eq!(bytes1, bytes2_after); + } + + pub fn create_model(device: &B::Device) -> SimpleLinear { + SimpleLinear::new(32, 32, device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/mod.rs new file mode 100644 index 0000000..198a1f3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/mod.rs @@ -0,0 +1,22 @@ +mod primitive; +mod tensor; + +mod base; +mod memory; +mod recorder; +mod settings; + +pub use base::*; +pub use memory::*; +pub use recorder::*; +pub use settings::*; + +#[cfg(feature = "std")] +mod file; +#[cfg(feature = "std")] +pub use file::*; + +pub use primitive::ParamSerde; + +#[cfg(feature = "record-item-custom-serde")] +pub mod serde; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/primitive.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/primitive.rs new file mode 100644 index 0000000..0a58c91 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/primitive.rs @@ -0,0 +1,336 @@ +use alloc::{string::String, vec, vec::Vec}; +use core::{fmt, marker::PhantomData}; + +use super::tensor::{BoolTensorSerde, FloatTensorSerde, IntTensorSerde}; +use super::{PrecisionSettings, Record}; +use crate::module::{Param, ParamId}; + +use burn_tensor::{Bool, Int, Tensor, backend::Backend}; + +use hashbrown::HashMap; +use serde::{ + Deserialize, Serialize, + de::{Error, SeqAccess, Visitor}, + ser::SerializeTuple, +}; + +impl Record for () +where + B: Backend, +{ + type Item = (); + + fn into_item(self) -> Self::Item {} + + fn from_item(_item: Self::Item, _device: &B::Device) -> Self {} +} + +impl Record for Vec +where + T: Record, + B: Backend, +{ + type Item = Vec>; + + fn into_item(self) -> Self::Item { + self.into_iter().map(Record::into_item).collect() + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + item.into_iter() + .map(|i| Record::from_item(i, device)) + .collect() + } +} + +impl Record for Option +where + T: Record, + B: Backend, +{ + type Item = Option>; + + fn into_item(self) -> Self::Item { + self.map(Record::into_item) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + item.map(|i| Record::from_item(i, device)) + } +} + +impl Record for [T; N] +where + T: Record, + B: Backend, +{ + /// The record item is an array of the record item of the elements. + /// The reason why we wrap the array in a struct is because serde does not support + /// deserializing arrays of variable size, + /// see [serde/issues/1937](https://github.com/serde-rs/serde/issues/1937). + /// for backward compatibility reasons. Serde APIs were created before const generics. + type Item = Array>; + + fn into_item(self) -> Self::Item { + Array(self.map(Record::into_item)) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + item.0.map(|i| Record::from_item(i, device)) + } +} + +/// A macro for generating implementations for tuple records of different sizes. +/// For example: `impl_record_tuple!([R0, R1][0, 1])`. +/// Would generate an implementation for a tuple of size 2. +/// For this macro to work properly, please adhere to the convention: +/// `impl_record_tuple!([R0, R1, ..., Rn][0, 1, ..., n])`. +macro_rules! impl_record_tuple { + // `$r` represents the generic records. + // `$i` represents the indices of the records in the tuple. + ([$($r:ident),*][$($i:tt),*]) => { + impl Record for ($($r,)*) + where + B: Backend, + $($r: Record),* + { + type Item = ($($r::Item,)*); + + fn into_item(self) -> Self::Item { + ($(self.$i.into_item(),)*) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + ($(Record::from_item(item.$i, device),)*) + } + } + }; +} + +impl_record_tuple!([R0, R1][0, 1]); +impl_record_tuple!([R0, R1, R2][0, 1, 2]); +impl_record_tuple!([R0, R1, R2, R3][0, 1, 2, 3]); +impl_record_tuple!([R0, R1, R2, R3, R4][0, 1, 2, 3, 4]); +impl_record_tuple!([R0, R1, R2, R3, R4, R5][0, 1, 2, 3, 4, 5]); +impl_record_tuple!([R0, R1, R2, R3, R4, R5, R6][0, 1, 2, 3, 4, 5, 6]); +impl_record_tuple!([R0, R1, R2, R3, R4, R5, R6, R7][0, 1, 2, 3, 4, 5, 6, 7]); +impl_record_tuple!([R0, R1, R2, R3, R4, R5, R6, R7, R8][0, 1, 2, 3, 4, 5, 6, 7, 8]); +impl_record_tuple!([R0, R1, R2, R3, R4, R5, R6, R7, R8, R9][0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + +impl Record for HashMap +where + T: Record, + B: Backend, +{ + type Item = HashMap>; + + fn into_item(self) -> Self::Item { + let mut items = HashMap::with_capacity(self.len()); + self.into_iter().for_each(|(id, record)| { + items.insert(id.serialize(), record.into_item()); + }); + items + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + let mut record = HashMap::with_capacity(item.len()); + item.into_iter().for_each(|(id, item)| { + record.insert(ParamId::deserialize(&id), T::from_item(item, device)); + }); + record + } +} + +/// (De)serialize parameters into a clean format. +#[derive(new, Debug, Clone, Serialize, Deserialize)] +pub struct ParamSerde { + id: String, + param: T, +} + +impl Record for Param> +where + B: Backend, +{ + type Item = ParamSerde>; + + fn into_item(self) -> Self::Item { + let (id, tensor, mapper) = self.consume(); + let tensor = mapper.on_save(tensor); + ParamSerde::new(id.serialize(), tensor.into_item()) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + B::memory_persistent_allocations(device, item, |item| { + Param::initialized( + ParamId::deserialize(&item.id), + Tensor::from_item(item.param, device).require_grad(), // Same behavior as when we create a new + // Param from a tensor. + ) + }) + } +} + +impl Record for Param> +where + B: Backend, +{ + type Item = ParamSerde>; + + fn into_item(self) -> Self::Item { + let (id, tensor, mapper) = self.consume(); + let tensor = mapper.on_save(tensor); + ParamSerde::new(id.serialize(), tensor.into_item()) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + B::memory_persistent_allocations(device, item, |item| { + Param::initialized( + ParamId::deserialize(&item.id), + Tensor::from_item(item.param, device), + ) + }) + } +} + +impl Record for Param> +where + B: Backend, +{ + type Item = ParamSerde; + + fn into_item(self) -> Self::Item { + let (id, tensor, mapper) = self.consume(); + let tensor = mapper.on_save(tensor); + ParamSerde::new(id.serialize(), tensor.into_item::()) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + B::memory_persistent_allocations(device, item, |item| { + Param::initialized( + ParamId::deserialize(&item.id), + Tensor::from_item::(item.param, device), + ) + }) + } +} + +// Type that can be serialized as is without any conversion. +macro_rules! primitive { + ($type:ty) => { + impl Record for $type { + type Item = $type; + + fn into_item(self) -> Self::Item { + self + } + + fn from_item(item: Self::Item, _device: &B::Device) -> Self { + item + } + } + }; +} + +// General Types +primitive!(alloc::string::String); +primitive!(bool); + +// Float Types +primitive!(f64); +primitive!(f32); + +primitive!(half::bf16); +primitive!(half::f16); + +// Unsigned Integer Types +primitive!(usize); +primitive!(u64); +primitive!(u32); +primitive!(u16); +primitive!(u8); + +// Signed Integer Types +primitive!(isize); +primitive!(i64); +primitive!(i32); +primitive!(i16); +primitive!(i8); + +/// A wrapper around an array of size N, so that it can be serialized and deserialized +/// using serde. +/// +/// The reason why we wrap the array in a struct is because serde does not support +/// deserializing arrays of variable size, +/// see [serde/issues/1937](https://github.com/serde-rs/serde/issues/1937) +/// for backward compatibility reasons. Serde APIs were created before const generics. +#[derive(Clone)] +pub struct Array([T; N]); + +impl Serialize for Array { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_tuple(self.0.len())?; + for element in &self.0 { + seq.serialize_element(element)?; + } + seq.end() + } +} + +impl<'de, T, const N: usize> Deserialize<'de> for Array +where + T: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct ArrayVisitor { + marker: PhantomData, + } + + impl<'de, T, const N: usize> Visitor<'de> for ArrayVisitor + where + T: Deserialize<'de>, + { + type Value = Array; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a fixed size array") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut items = vec![]; + + for i in 0..N { + let item = seq + .next_element()? + .ok_or_else(|| Error::invalid_length(i, &self))?; + items.push(item); + } + + let array: [T; N] = items + .into_iter() + .collect::>() + .try_into() + .map_err(|_| "An array of size {N}") + .unwrap(); + + Ok(Array(array)) + } + } + + deserializer.deserialize_tuple( + N, + ArrayVisitor { + marker: PhantomData, + }, + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/recorder.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/recorder.rs new file mode 100644 index 0000000..66dd49f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/recorder.rs @@ -0,0 +1,329 @@ +use core::any::type_name; +use core::marker::PhantomData; + +use alloc::format; +use alloc::string::{String, ToString}; +use burn_tensor::backend::Backend; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; + +use super::{BinBytesRecorder, FullPrecisionSettings, PrecisionSettings, Record}; + +#[cfg(feature = "std")] +use super::{ + BinFileRecorder, BinGzFileRecorder, DefaultFileRecorder, HalfPrecisionSettings, + PrettyJsonFileRecorder, +}; + +/// Record any item implementing [Serialize](Serialize) and [DeserializeOwned](DeserializeOwned). +pub trait Recorder: + Send + Sync + core::default::Default + core::fmt::Debug + Clone +{ + /// Type of the settings used by the recorder. + type Settings: PrecisionSettings; + + /// Arguments used to record objects. + type RecordArgs: Clone; + + /// Record output type. + type RecordOutput; + + /// Arguments used to load recorded objects. + type LoadArgs; + + /// Records an item. + /// + /// # Arguments + /// + /// * `record` - The item to record. + /// * `args` - Arguments used to record the item. + /// + /// # Returns + /// + /// The output of the recording. + fn record( + &self, + record: R, + args: Self::RecordArgs, + ) -> Result + where + R: Record, + { + let item = record.into_item::(); + let item = BurnRecord::new::(item); + + self.save_item(item, args) + } + + /// Load an item from the given arguments. + fn load(&self, mut args: Self::LoadArgs, device: &B::Device) -> Result + where + R: Record, + { + let item: BurnRecord, B> = + self.load_item(&mut args).map_err(|err| { + if let Ok(record) = self.load_item::(&mut args) { + let mut message = "Unable to load record.".to_string(); + let metadata = recorder_metadata::(); + if metadata.float != record.metadata.float { + message += format!( + "\nMetadata has a different float type: Actual {:?}, Expected {:?}", + record.metadata.float, metadata.float + ) + .as_str(); + } + if metadata.int != record.metadata.int { + message += format!( + "\nMetadata has a different int type: Actual {:?}, Expected {:?}", + record.metadata.int, metadata.int + ) + .as_str(); + } + if metadata.format != record.metadata.format { + message += format!( + "\nMetadata has a different format: Actual {:?}, Expected {:?}", + record.metadata.format, metadata.format + ) + .as_str(); + } + if metadata.version != record.metadata.version { + message += format!( + "\nMetadata has a different Burn version: Actual {:?}, Expected {:?}", + record.metadata.version, metadata.version + ) + .as_str(); + } + + message += format!("\nError: {err:?}").as_str(); + + return RecorderError::Unknown(message); + } + + err + })?; + + Ok(R::from_item(item.item, device)) + } + + /// Saves an item. + /// + /// This method is used by [record](Recorder::record) to save the item. + /// + /// # Arguments + /// + /// * `item` - Item to save. + /// * `args` - Arguments to use to save the item. + /// + /// # Returns + /// + /// The output of the save operation. + fn save_item( + &self, + item: I, + args: Self::RecordArgs, + ) -> Result; + + /// Loads an item. + /// + /// This method is used by [load](Recorder::load) to load the item. + /// + /// # Arguments + /// + /// * `args` - Arguments to use to load the item. + /// + /// # Returns + /// + /// The loaded item. + fn load_item(&self, args: &mut Self::LoadArgs) -> Result + where + I: DeserializeOwned; +} + +fn recorder_metadata() -> BurnMetadata +where + R: Recorder, + B: Backend, +{ + BurnMetadata::new( + type_name::<::FloatElem>().to_string(), + type_name::<::IntElem>().to_string(), + type_name::().to_string(), + env!("CARGO_PKG_VERSION").to_string(), + format!("{:?}", R::Settings::default()), + ) +} + +/// Error that can occur when using a [Recorder](Recorder). +#[derive(Debug)] +pub enum RecorderError { + /// File not found. + FileNotFound(String), + + /// Failed to read file. + DeserializeError(String), + + /// Other error. + Unknown(String), +} + +impl core::fmt::Display for RecorderError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(format!("{self:?}").as_str()) + } +} + +impl core::error::Error for RecorderError {} + +pub(crate) fn bin_config() -> bincode::config::Configuration { + bincode::config::standard() +} + +/// Metadata of a record. +#[derive(new, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct BurnMetadata { + /// Float type used to record the item. + pub float: String, + + /// Int type used to record the item. + pub int: String, + + /// Format used to record the item. + pub format: String, + + /// Burn record version used to record the item. + pub version: String, + + /// Settings used to record the item. + pub settings: String, +} + +/// Record that can be saved by a [Recorder](Recorder). +#[derive(Serialize, Deserialize, Debug)] +pub struct BurnRecord { + /// Metadata of the record. + pub metadata: BurnMetadata, + + /// Item to record. + pub item: I, + + _b: PhantomData, +} + +impl BurnRecord { + /// Creates a new record. + /// + /// # Arguments + /// + /// * `item` - Item to record. + /// + /// # Returns + /// + /// The new record. + pub fn new>(item: I) -> Self { + let metadata = recorder_metadata::(); + + Self { + metadata, + item, + _b: PhantomData, + } + } +} + +/// Record that can be saved by a [Recorder](Recorder) without the item. +#[derive(new, Debug, Serialize, Deserialize)] +pub struct BurnRecordNoItem { + /// Metadata of the record. + pub metadata: BurnMetadata, +} + +/// Default recorder. +/// +/// It uses the [named msgpack](rmp_serde) format for serialization with full precision. +#[cfg(feature = "std")] +pub type DefaultRecorder = DefaultFileRecorder; + +/// Recorder optimized for compactness. +/// +/// It uses the [named msgpack](rmp_serde) format for serialization with half precision. +/// If you are looking for the recorder that offers the smallest file size, have a look at +/// [sensitive compact recorder](SensitiveCompactRecorder). +#[cfg(feature = "std")] +pub type CompactRecorder = DefaultFileRecorder; + +/// Recorder optimized for compactness making it a good choice for model deployment. +/// +/// It uses the [bincode](bincode) format for serialization and half precision. +/// This format is not resilient to type changes since no metadata is encoded. +/// Favor [default recorder](DefaultRecorder) or [compact recorder](CompactRecorder) +/// for long term data storage. +#[cfg(feature = "std")] +pub type SensitiveCompactRecorder = BinGzFileRecorder; + +/// Training recorder compatible with no-std inference. +#[cfg(feature = "std")] +pub type NoStdTrainingRecorder = BinFileRecorder; + +/// Inference recorder compatible with no-std. +pub type NoStdInferenceRecorder = BinBytesRecorder; + +/// Debug recorder. +/// +/// It uses the [pretty json](serde_json) format for serialization with full precision making it +/// human readable. +#[cfg(feature = "std")] +pub type DebugRecordSettings = PrettyJsonFileRecorder; + +#[cfg(all(test, feature = "std"))] +mod tests { + static FILE_PATH: &str = "/tmp/burn_test_record"; + + use crate::TestBackend; + + use super::*; + use burn_tensor::{Device, ElementConversion}; + + #[test] + #[should_panic] + fn err_when_invalid_item() { + #[derive(new, Serialize, Deserialize, Clone)] + struct Item { + value: S::FloatElem, + } + + impl Record for Item + where + D: PrecisionSettings, + B: Backend, + { + type Item = Item; + + fn into_item(self) -> Self::Item { + Item { + value: self.value.elem(), + } + } + + fn from_item(item: Self::Item, _device: &B::Device) -> Self { + Item { + value: item.value.elem(), + } + } + } + + let item = Item::::new(16.elem()); + let device: Device = Default::default(); + + // Serialize in f32. + let recorder = DefaultFileRecorder::::new(); + Recorder::::record(&recorder, item, FILE_PATH.into()).unwrap(); + + // Can't deserialize f32 into f16. + let recorder = DefaultFileRecorder::::new(); + Recorder::::load::>( + &recorder, + FILE_PATH.into(), + &device, + ) + .unwrap(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/adapter.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/adapter.rs new file mode 100644 index 0000000..b7b3e4f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/adapter.rs @@ -0,0 +1,83 @@ +use super::data::NestedValue; + +/// A trait that defines the adapter for a Burn module. +/// +/// This is used to adapt an incoming module to a Burn module. +pub trait BurnModuleAdapter: Sized { + /// Adapts a module. + fn adapt(name: &str, data: NestedValue) -> NestedValue { + match name { + "BatchNorm" => Self::adapt_batch_norm(data), + "Conv1d" => Self::adapt_conv1d(data), + "Conv2d" => Self::adapt_conv2d(data), + "Conv3d" => Self::adapt_conv3d(data), + "ConvTranspose1d" => Self::adapt_conv_transpose_1d(data), + "ConvTranspose2d" => Self::adapt_conv_transpose_2d(data), + "ConvTranspose3d" => Self::adapt_conv_transpose_3d(data), + "Embedding" => Self::adapt_embedding(data), + "GroupNorm" => Self::adapt_group_norm(data), + "LayerNorm" => Self::adapt_layer_norm(data), + "Linear" => Self::adapt_linear(data), + _ => data, + } + } + + /// Adapts a linear module. + fn adapt_linear(data: NestedValue) -> NestedValue { + data + } + + /// Adapts a Convolution 1D module. + fn adapt_conv1d(data: NestedValue) -> NestedValue { + data + } + + /// Adapts a Convolution 2D module. + fn adapt_conv2d(data: NestedValue) -> NestedValue { + data + } + + /// Adapts a Convolution 3D module. + fn adapt_conv3d(data: NestedValue) -> NestedValue { + data + } + + /// Adapts convolution transpose 1D module. + fn adapt_conv_transpose_1d(data: NestedValue) -> NestedValue { + data + } + + /// Adapts convolution transpose 2D module. + fn adapt_conv_transpose_2d(data: NestedValue) -> NestedValue { + data + } + + /// Adapts convolution transpose 2D module. + fn adapt_conv_transpose_3d(data: NestedValue) -> NestedValue { + data + } + + /// Adapts embedding module. + fn adapt_embedding(data: NestedValue) -> NestedValue { + data + } + + /// Adapts group normalization module. + fn adapt_group_norm(data: NestedValue) -> NestedValue { + data + } + + /// Adapts layer normalization module. + fn adapt_layer_norm(data: NestedValue) -> NestedValue { + data + } + + /// Adapts batch normalization module. + fn adapt_batch_norm(data: NestedValue) -> NestedValue { + data + } +} + +/// Default adapter that takes no action. +pub struct DefaultAdapter; +impl BurnModuleAdapter for DefaultAdapter {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/data.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/data.rs new file mode 100644 index 0000000..1db82e2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/data.rs @@ -0,0 +1,399 @@ +use std::collections::HashMap; + +use super::adapter::BurnModuleAdapter; +use super::de::Deserializer; +use super::error::Error; +use super::ser::Serializer; +use crate::record::{PrecisionSettings, Record}; +use crate::tensor::backend::Backend; + +use alloc::fmt; +use burn_tensor::Bytes; +use num_traits::cast::ToPrimitive; +use regex::Regex; +use serde::Deserialize; + +/// The main data structure used for deserialization. +/// +/// It can hold tree-like structures of nested maps and vectors. +#[derive(Clone)] +pub enum NestedValue { + /// The default value, which actually does not hold any value and it is used to indicate that + /// the value should be populated with the default value. It contains an optional string with + /// the originator field name. + Default(Option), + + /// A boolean value. + Bool(bool), + + /// A string value. + String(String), + + /// Floating point 32-bit value. + F32(f32), + + /// Floating point 64-bit value. + F64(f64), + + /// Signed 16-bit integer value. + I16(i16), + + /// Signed 32-bit integer value. + I32(i32), + + /// Signed 64-bit integer value. + I64(i64), + + /// Unsigned 8-bit integer value. + U8(u8), + + /// Unsigned 16-bit integer value used for bf16 and f16 serialization + U16(u16), + + /// Unsigned 64-bit integer value. + U64(u64), + + /// A map of nested values (typically used for structs) + Map(HashMap), + + /// A vector of nested values (typically used for vector of structs or numbers) + Vec(Vec), + + /// A vector of 8-bit unsigned integer values. + U8s(Vec), + + /// A vector of 16-bit unsigned integer values. + U16s(Vec), + + /// A vector of 32-bit floating point values. + F32s(Vec), + + /// An opaque vector of bytes, with alignment. + Bytes(Bytes), +} + +impl NestedValue { + /// Get the nested value as a map. + pub fn as_map(self) -> Option> { + match self { + NestedValue::Map(map) => Some(map), + _ => None, + } + } + + /// Get the nested value as a boolean. + pub fn as_bool(self) -> Option { + match self { + NestedValue::Bool(bool) => Some(bool), + _ => None, + } + } + + /// Get the nested value as a string. + pub fn as_string(self) -> Option { + match self { + NestedValue::String(string) => Some(string), + _ => None, + } + } + + /// Get the nested value as a f32. + pub fn as_f32(self) -> Option { + match self { + NestedValue::F32(f32) => Some(f32), + NestedValue::F64(f) => f.to_f32(), + _ => None, + } + } + + /// Get the nested value as a f64. + pub fn as_f64(self) -> Option { + match self { + NestedValue::F64(f64) => Some(f64), + NestedValue::F32(f) => f.to_f64(), + _ => None, + } + } + + /// Get the nested value as an i16. + pub fn as_i16(self) -> Option { + match self { + NestedValue::I16(i16) => Some(i16), + NestedValue::I32(i) => i.to_i16(), + NestedValue::I64(i) => i.to_i16(), + NestedValue::U16(u) => u.to_i16(), + NestedValue::U64(u) => u.to_i16(), + _ => None, + } + } + + /// Get the nested value as an i32. + pub fn as_i32(self) -> Option { + match self { + NestedValue::I32(i32) => Some(i32), + NestedValue::I16(i) => i.to_i32(), + NestedValue::I64(i) => i.to_i32(), + NestedValue::U16(u) => u.to_i32(), + NestedValue::U64(u) => u.to_i32(), + _ => None, + } + } + + /// Get the nested value as an i64. + pub fn as_i64(self) -> Option { + match self { + NestedValue::I64(i64) => Some(i64), + NestedValue::I16(i) => i.to_i64(), + NestedValue::I32(i) => i.to_i64(), + NestedValue::U16(u) => u.to_i64(), + NestedValue::U64(u) => u.to_i64(), + _ => None, + } + } + + /// Get the nested value as a u8. + pub fn as_u8(self) -> Option { + match self { + NestedValue::U8(u8) => Some(u8), + NestedValue::I16(i) => i.to_u8(), + NestedValue::I32(i) => i.to_u8(), + NestedValue::I64(i) => i.to_u8(), + NestedValue::U16(u) => u.to_u8(), + NestedValue::U64(u) => u.to_u8(), + _ => None, + } + } + + /// Get the nested value as a u16. + pub fn as_u16(self) -> Option { + match self { + NestedValue::U16(u16) => Some(u16), + NestedValue::I16(i) => i.to_u16(), + NestedValue::I32(i) => i.to_u16(), + NestedValue::I64(i) => i.to_u16(), + NestedValue::U64(u) => u.to_u16(), + _ => None, + } + } + + /// Get the nested value as a u64. + pub fn as_u64(self) -> Option { + match self { + NestedValue::U64(u64) => Some(u64), + NestedValue::I16(i) => i.to_u64(), + NestedValue::I32(i) => i.to_u64(), + NestedValue::I64(i) => i.to_u64(), + NestedValue::U16(u) => u.to_u64(), + _ => None, + } + } + + /// Get the nested value as a vector of bytes. + pub fn as_bytes(self) -> Option { + match self { + NestedValue::Bytes(u) => Some(u), + NestedValue::U8s(u) => Some(Bytes::from_elems(u)), + _ => None, + } + } + + /// Deserialize a nested value into a record type. + pub fn try_into_record(self, device: &B::Device) -> Result + where + B: Backend, + T: Record, + PS: PrecisionSettings, + A: BurnModuleAdapter, + { + let deserializer = Deserializer::::new(self, false); + + let item = T::Item::deserialize(deserializer)?; + + // Convert the deserialized item into a Record instance + Ok(T::from_item::(item, device)) + } +} + +/// Remap the tensor locations according to the key remapping. +/// +/// # Arguments +/// +/// * `tensors` - A map of tensors. +/// * `key_remap` - A vector of tuples containing a regular expression and a replacement string. +/// See [regex::Regex::replace](https://docs.rs/regex/latest/regex/struct.Regex.html#method.replace) +/// for more information. +/// +/// # Returns +/// +/// A map of tensors with the remapped keys and +/// a vector of tuples containing the remapped and original. +pub fn remap( + mut tensors: HashMap, + key_remap: Vec<(Regex, String)>, +) -> (HashMap, Vec<(String, String)>) { + if key_remap.is_empty() { + let remapped_names = tensors + .keys() + .cloned() + .map(|s| (s.clone(), s)) // Name is the same as the remapped name + .collect(); + return (tensors, remapped_names); + } + + let mut remapped = HashMap::new(); + let mut remapped_names = Vec::new(); + + for (name, tensor) in tensors.drain() { + let mut new_name = name.clone(); + for (pattern, replacement) in &key_remap { + if pattern.is_match(&new_name) { + new_name = pattern + .replace_all(&new_name, replacement.as_str()) + .to_string(); + } + } + + remapped_names.push((new_name.clone(), name)); + remapped.insert(new_name, tensor); + } + + (remapped, remapped_names) +} + +/// Helper function to insert a value into a nested map/vector of tensors. +fn insert_nested_value(current: &mut NestedValue, keys: &[&str], value: NestedValue) { + if keys.is_empty() { + *current = value; + return; + } + + match current { + NestedValue::Map(map) => { + if !map.contains_key(keys[0]) { + let next = if keys[1..] + .first() + .and_then(|k| k.parse::().ok()) + .is_some() + { + NestedValue::Vec(Vec::new()) + } else { + NestedValue::Map(HashMap::new()) + }; + map.insert(keys[0].to_string(), next); + } + insert_nested_value(map.get_mut(keys[0]).unwrap(), &keys[1..], value); + } + NestedValue::Vec(vec) => { + let index = keys[0].parse::().unwrap(); + if index >= vec.len() { + vec.resize_with(index + 1, || NestedValue::Map(HashMap::new())); + } + insert_nested_value(&mut vec[index], &keys[1..], value); + } + _ => panic!("Invalid structure encountered"), + } +} + +/// A trait for encapsulating the serialization logic. +pub trait Serializable { + /// Serializes the object into a `NestedValue` using the provided `Serializer`. + /// This method is generic over the precision settings `PS`. + /// + /// # Parameters + /// - `serializer`: The `Serializer` to use for serializing the object. + /// + /// # Returns + /// - `Result`: The result of serialization. + /// Returns a `NestedValue` on success, + /// or an `Error` on failure. + /// + /// # Type Parameters + /// - `PS`: The precision settings to use during serialization. + /// This is a generic parameter and can be any type + /// that implements the `PrecisionSettings` trait. + fn serialize(&self, serializer: Serializer) -> Result + where + PS: PrecisionSettings; +} + +/// Convert a vector of tensors to a nested value. +pub fn unflatten(input: HashMap) -> Result +where + PS: PrecisionSettings, + T: Serializable, +{ + let mut result = NestedValue::Map(HashMap::new()); + + for (key, value) in input { + let parts: Vec<&str> = key.split('.').collect(); + let st = value.serialize::(Serializer::new())?; + + insert_nested_value(&mut result, &parts, st); + } + + cleanup_empty_maps(&mut result); + + Ok(result) +} + +/// Removes empty maps from the nested value. +/// +/// We need to clean up empty maps from the nested value +/// in some cases when there is non-contiguous indices in keys. +fn cleanup_empty_maps(current: &mut NestedValue) { + match current { + NestedValue::Map(map) => { + map.values_mut().for_each(cleanup_empty_maps); + } + NestedValue::Vec(vec) => { + vec.iter_mut().for_each(cleanup_empty_maps); + vec.retain(|v| !matches!(v, NestedValue::Map(m) if m.is_empty())); + } + _ => {} + } +} + +fn write_vec_truncated( + vec: &[T], + f: &mut core::fmt::Formatter, +) -> fmt::Result { + write!(f, "Vec([")?; + for (i, v) in vec.iter().take(3).enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{v:?}")?; + } + write!(f, ", ...] len={})", vec.len()) +} + +impl fmt::Debug for NestedValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + // Truncate values for vector + NestedValue::Vec(vec) if vec.len() > 3 => write_vec_truncated(vec, f), + NestedValue::U8s(vec) if vec.len() > 3 => write_vec_truncated(vec, f), + NestedValue::U16s(vec) if vec.len() > 3 => write_vec_truncated(vec, f), + NestedValue::F32s(vec) if vec.len() > 3 => write_vec_truncated(vec, f), + NestedValue::Bytes(bytes) if bytes.len() > 3 => write_vec_truncated(bytes, f), + // Handle other variants as usual + NestedValue::Default(origin) => f.debug_tuple("Default").field(origin).finish(), + NestedValue::Bool(b) => f.debug_tuple("Bool").field(b).finish(), + NestedValue::String(s) => f.debug_tuple("String").field(s).finish(), + NestedValue::F32(val) => f.debug_tuple("F32").field(val).finish(), + NestedValue::F64(val) => f.debug_tuple("F64").field(val).finish(), + NestedValue::I16(val) => f.debug_tuple("I16").field(val).finish(), + NestedValue::I32(val) => f.debug_tuple("I32").field(val).finish(), + NestedValue::I64(val) => f.debug_tuple("I64").field(val).finish(), + NestedValue::U8(val) => f.debug_tuple("U8").field(val).finish(), + NestedValue::U16(val) => f.debug_tuple("U16").field(val).finish(), + NestedValue::U64(val) => f.debug_tuple("U64").field(val).finish(), + NestedValue::Map(map) => f.debug_map().entries(map.iter()).finish(), + NestedValue::Vec(vec) => f.debug_list().entries(vec.iter()).finish(), + NestedValue::U8s(vec) => f.debug_list().entries(vec.iter()).finish(), + NestedValue::U16s(vec) => f.debug_list().entries(vec.iter()).finish(), + NestedValue::F32s(vec) => f.debug_list().entries(vec.iter()).finish(), + NestedValue::Bytes(bytes) => f.debug_list().entries(bytes.iter()).finish(), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/de.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/de.rs new file mode 100644 index 0000000..383b1f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/de.rs @@ -0,0 +1,1006 @@ +use core::ptr; +use std::collections::HashMap; + +use super::data::NestedValue; +use super::{adapter::BurnModuleAdapter, error::Error}; + +use serde::de::{EnumAccess, VariantAccess}; +use serde::{ + de::{self, DeserializeSeed, IntoDeserializer, MapAccess, SeqAccess, Visitor}, + forward_to_deserialize_any, +}; + +const RECORD_ITEM_SUFFIX: &str = "RecordItem"; + +/// A deserializer for the nested value data structure. +pub struct Deserializer { + // This string starts with the input data and characters are truncated off + // the beginning as data is parsed. + value: Option, + default_for_missing_fields: bool, + phantom: std::marker::PhantomData, +} + +impl Deserializer { + /// Creates a new deserializer with the given nested value. + /// + /// # Arguments + /// + /// * `value` - A nested value. + /// * `default_for_missing_fields` - A boolean indicating whether to add missing fields with default value. + pub fn new(value: NestedValue, default_for_missing_fields: bool) -> Self { + Self { + value: Some(value), + default_for_missing_fields, + phantom: std::marker::PhantomData, + } + } +} + +impl<'de, A: BurnModuleAdapter> serde::Deserializer<'de> for Deserializer { + type Error = Error; + + fn deserialize_any(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_any is not implemented") + } + + fn deserialize_struct( + self, + name: &'static str, + fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + let value = match self.value { + Some(value) => { + // Adapt modules + if let Some(name) = name.strip_suffix(RECORD_ITEM_SUFFIX) { + A::adapt(name, value) + } else { + value + } + } + None => { + return Err(de::Error::custom(format!( + "Expected some value but got {:?}", + self.value + ))); + } + }; + + match value { + NestedValue::Map(map) => { + // Add missing fields into the map with default value if needed. + let map = if self.default_for_missing_fields { + let mut map = map; + for field in fields.iter().map(|s| s.to_string()) { + map.entry(field.clone()) + .or_insert(NestedValue::Default(Some(field))); + } + map + } else { + map + }; + + visitor.visit_map(HashMapAccess::::new( + map, + self.default_for_missing_fields, + )) + } + + _ => Err(de::Error::custom(format!( + "Expected struct but got {value:?}" + ))), + } + } + + fn deserialize_string(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_string(self.value.unwrap().as_string().unwrap().to_string()) + } + + fn deserialize_ignored_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_unit() + } + + fn deserialize_map(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match self.value { + Some(NestedValue::Map(map)) => visitor.visit_map(HashMapAccess::::new( + map, + self.default_for_missing_fields, + )), + + _ => Err(de::Error::custom(format!( + "Expected map value but got {:?}", + self.value + ))), + } + } + + fn deserialize_bool(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_bool(self.value.unwrap().as_bool().unwrap()) + } + + fn deserialize_i8(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_i8 is not implemented") + } + + fn deserialize_i16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i16(self.value.unwrap().as_i16().unwrap().to_owned()) + } + + fn deserialize_i32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i32(self.value.unwrap().as_i32().unwrap().to_owned()) + } + + fn deserialize_i64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i64(self.value.unwrap().as_i64().unwrap().to_owned()) + } + + fn deserialize_u8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u8(self.value.unwrap().as_u8().unwrap().to_owned()) + } + + fn deserialize_u16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u16(self.value.unwrap().as_u16().unwrap().to_owned()) + } + + fn deserialize_u32(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_u32 is not implemented") + } + + fn deserialize_u64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u64(self.value.unwrap().as_u64().unwrap().to_owned()) + } + + fn deserialize_f32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_f32(self.value.unwrap().as_f32().unwrap().to_owned()) + } + + fn deserialize_f64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_f64(self.value.unwrap().as_f64().unwrap().to_owned()) + } + + fn deserialize_char(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_char is not implemented") + } + + fn deserialize_str(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_str(self.value.unwrap().as_string().unwrap().as_ref()) + } + + fn deserialize_bytes(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_bytes is not implemented") + } + + fn deserialize_byte_buf(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + let bytes = self.value.unwrap().as_bytes().unwrap(); + match bytes.try_into_vec::() { + Ok(bytes) => visitor.visit_byte_buf(bytes), + Err(bytes) => visitor.visit_bytes(&bytes), + } + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if let Some(value) = self.value { + visitor.visit_some(Deserializer::::new( + value, + self.default_for_missing_fields, + )) + } else { + visitor.visit_none() + } + } + + fn deserialize_unit(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_unit is not implemented") + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_unit_struct is not implemented") + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(Deserializer::::new( + self.value.unwrap(), + self.default_for_missing_fields, + )) + } + + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if let Some(value) = self.value { + match value { + NestedValue::Vec(_) => visitor.visit_seq(VecSeqAccess::::new( + value, + self.default_for_missing_fields, + )), + NestedValue::U8s(_) => visitor.visit_seq(VecSeqAccess::::new( + value, + self.default_for_missing_fields, + )), + NestedValue::U16s(_) => visitor.visit_seq(VecSeqAccess::::new( + value, + self.default_for_missing_fields, + )), + NestedValue::F32s(_) => visitor.visit_seq(VecSeqAccess::::new( + value, + self.default_for_missing_fields, + )), + _ => Err(de::Error::custom(format!("Expected Vec but got {value:?}"))), + } + } else { + Err(de::Error::custom("Expected Vec but got None")) + } + } + + fn deserialize_tuple(self, _len: usize, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_tuple is not implemented") + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_tuple_struct is not implemented") + } + + /// Deserializes an enum by attempting to match its variants against the provided data. + /// + /// This function attempts to deserialize an enum by iterating over its possible variants + /// and trying to deserialize the data into each until one succeeds. We need to do this + /// because we don't have a way to know which variant to deserialize from the data. + /// + /// This is similar to Serde's + /// [untagged enum deserialization](https://serde.rs/enum-representations.html#untagged), + /// but it's on the deserializer side. Using `#[serde(untagged)]` on the enum will force + /// using `deserialize_any`, which is not what we want because we want to use methods, such + /// as `visit_struct`. Also we do not wish to use auto generate code for Deserialize just + /// for enums because it will affect other serialization and deserialization, such + /// as JSON and Bincode. + /// + /// # Safety + /// The function uses an unsafe block to clone the `visitor`. This is necessary because + /// the `Visitor` trait does not have a `Clone` implementation, and we need to clone it + /// as we are going to use it multiple times. The Visitor is a code generated unit struct + /// with no states or mutations, so it is safe to clone it in this case. We mainly care + /// about the `visit_enum` method, which is the only method that will be called on the + /// cloned visitor. + fn deserialize_enum( + self, + _name: &'static str, + variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + fn clone_unsafely(thing: &T) -> T { + unsafe { + // Allocate memory for the clone. + let mut clone = std::mem::MaybeUninit::::uninit(); + // Get a mutable pointer to the allocated memory. + let clone_ptr = clone.as_mut_ptr(); + // Copy the memory + ptr::copy_nonoverlapping(thing as *const T, clone_ptr, 1); + // Assume the cloned data is initialized and convert it to an owned instance of T. + clone.assume_init() + } + } + + // Try each variant in order + for &variant in variants { + // clone visitor to avoid moving it + let cloned_visitor = clone_unsafely(&visitor); + let result = cloned_visitor.visit_enum(ProbeEnumAccess::::new( + self.value.clone().unwrap(), + variant.to_owned(), + self.default_for_missing_fields, + )); + + if result.is_ok() { + return result; + } + } + + Err(de::Error::custom("No variant match")) + } + + fn deserialize_identifier(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize_identifier is not implemented") + } +} + +/// A sequence access for a vector in the nested value data structure. +struct VecSeqAccess { + iter: Box>, + default_for_missing_fields: bool, + phantom: std::marker::PhantomData, +} + +// Concrete implementation for `Vec` +impl VecSeqAccess { + fn new(vec: NestedValue, default_for_missing_fields: bool) -> Self { + match vec { + NestedValue::Vec(v) => VecSeqAccess { + iter: Box::new(v.into_iter()), + default_for_missing_fields, + phantom: std::marker::PhantomData, + }, + _ => panic!("Invalid vec sequence"), + } + } +} + +// Concrete implementation for `Vec` +impl VecSeqAccess { + fn new(vec: NestedValue, default_for_missing_fields: bool) -> Self { + match vec { + NestedValue::U8s(v) => VecSeqAccess { + iter: Box::new(v.into_iter()), + default_for_missing_fields, + phantom: std::marker::PhantomData, + }, + _ => panic!("Invalid vec sequence"), + } + } +} + +// Concrete implementation for `Vec` +impl VecSeqAccess { + fn new(vec: NestedValue, default_for_missing_fields: bool) -> Self { + match vec { + NestedValue::U16s(v) => VecSeqAccess { + iter: Box::new(v.into_iter()), + default_for_missing_fields, + phantom: std::marker::PhantomData, + }, + _ => panic!("Invalid vec sequence"), + } + } +} + +// Concrete implementation for `Vec` +impl VecSeqAccess { + fn new(vec: NestedValue, default_for_missing_fields: bool) -> Self { + match vec { + NestedValue::F32s(v) => VecSeqAccess { + iter: Box::new(v.into_iter()), + default_for_missing_fields, + phantom: std::marker::PhantomData, + }, + _ => panic!("Invalid vec sequence"), + } + } +} + +// Concrete implementation for `Vec` +impl<'de, A> SeqAccess<'de> for VecSeqAccess +where + NestedValueWrapper: IntoDeserializer<'de, Error>, + A: BurnModuleAdapter, +{ + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + let item = match self.iter.next() { + Some(v) => v, + None => return Ok(None), + }; + + seed.deserialize( + NestedValueWrapper::::new(item, self.default_for_missing_fields).into_deserializer(), + ) + .map(Some) + } +} + +// Concrete implementation for `Vec` +impl<'de, A> SeqAccess<'de> for VecSeqAccess +where + NestedValueWrapper: IntoDeserializer<'de, Error>, + A: BurnModuleAdapter, +{ + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + let item = match self.iter.next() { + Some(v) => v, + None => return Ok(None), + }; + + seed.deserialize( + NestedValueWrapper::::new(NestedValue::U8(item), self.default_for_missing_fields) + .into_deserializer(), + ) + .map(Some) + } +} + +// Concrete implementation for `Vec` +impl<'de, A> SeqAccess<'de> for VecSeqAccess +where + NestedValueWrapper: IntoDeserializer<'de, Error>, + A: BurnModuleAdapter, +{ + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + let item = match self.iter.next() { + Some(v) => v, + None => return Ok(None), + }; + + seed.deserialize( + NestedValueWrapper::::new(NestedValue::U16(item), self.default_for_missing_fields) + .into_deserializer(), + ) + .map(Some) + } +} + +// Concrete implementation for `Vec` +impl<'de, A> SeqAccess<'de> for VecSeqAccess +where + NestedValueWrapper: IntoDeserializer<'de, Error>, + A: BurnModuleAdapter, +{ + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + let item = match self.iter.next() { + Some(v) => v, + None => return Ok(None), + }; + + seed.deserialize( + NestedValueWrapper::::new(NestedValue::F32(item), self.default_for_missing_fields) + .into_deserializer(), + ) + .map(Some) + } +} + +/// A map access for a map in the nested value data structure. +struct HashMapAccess { + iter: std::collections::hash_map::IntoIter, + next_value: Option, + default_for_missing_fields: bool, + phantom: std::marker::PhantomData, +} + +impl HashMapAccess { + fn new(map: HashMap, default_for_missing_fields: bool) -> Self { + HashMapAccess { + iter: map.into_iter(), + next_value: None, + default_for_missing_fields, + phantom: std::marker::PhantomData, + } + } +} + +impl<'de, A> MapAccess<'de> for HashMapAccess +where + String: IntoDeserializer<'de, Error>, + NestedValueWrapper: IntoDeserializer<'de, Error>, + A: BurnModuleAdapter, +{ + type Error = Error; + + fn next_key_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + match self.iter.next() { + Some((k, v)) => { + // Keep the value for the next call to next_value_seed. + self.next_value = Some(v); + // Deserialize the key. + seed.deserialize(k.into_deserializer()).map(Some) + } + None => Ok(None), + } + } + + fn next_value_seed(&mut self, seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + match self.next_value.take() { + Some(NestedValue::Default(originator)) => { + seed.deserialize(DefaultDeserializer::new(originator)) + } + Some(v) => seed.deserialize( + NestedValueWrapper::new(v, self.default_for_missing_fields).into_deserializer(), + ), + None => seed.deserialize(DefaultDeserializer::new(None)), + } + } +} + +struct ProbeEnumAccess { + value: NestedValue, + current_variant: String, + default_for_missing_fields: bool, + phantom: std::marker::PhantomData, +} + +impl ProbeEnumAccess { + fn new(value: NestedValue, current_variant: String, default_for_missing_fields: bool) -> Self { + ProbeEnumAccess { + value, + current_variant, + default_for_missing_fields, + phantom: std::marker::PhantomData, + } + } +} + +impl<'de, A> EnumAccess<'de> for ProbeEnumAccess +where + A: BurnModuleAdapter, +{ + type Error = Error; + type Variant = Self; + + fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> + where + V: DeserializeSeed<'de>, + { + seed.deserialize(self.current_variant.clone().into_deserializer()) + .map(|v| (v, self)) + } +} + +impl<'de, A> VariantAccess<'de> for ProbeEnumAccess +where + A: BurnModuleAdapter, +{ + type Error = Error; + + fn newtype_variant_seed(self, seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + let value = seed.deserialize( + NestedValueWrapper::::new(self.value, self.default_for_missing_fields) + .into_deserializer(), + )?; + Ok(value) + } + + fn unit_variant(self) -> Result<(), Self::Error> { + // Support tensor `DType` deserialization + match self.value { + NestedValue::Map(value) if value.contains_key("DType") => { + match value.get("DType") { + Some(NestedValue::String(variant)) => { + if *variant == self.current_variant { + Ok(()) + } else { + Err(Error::Other("Wrong variant".to_string())) // wrong match + } + } + _ => panic!("expected DType variant as string"), + } + } + _ => unimplemented!( + "unit variant is not implemented because it is not used in the burn module" + ), + } + } + + fn tuple_variant(self, _len: usize, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("tuple variant is not implemented because it is not used in the burn module") + } + + fn struct_variant( + self, + _fields: &'static [&'static str], + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + unimplemented!( + "struct variant is not implemented because it is not used in the burn module" + ) + } +} + +/// A wrapper for the nested value data structure with a burn module adapter. +struct NestedValueWrapper { + value: NestedValue, + default_for_missing_fields: bool, + phantom: std::marker::PhantomData, +} + +impl NestedValueWrapper { + fn new(value: NestedValue, default_for_missing_fields: bool) -> Self { + Self { + value, + default_for_missing_fields, + phantom: std::marker::PhantomData, + } + } +} + +impl IntoDeserializer<'_, Error> for NestedValueWrapper { + type Deserializer = Deserializer; + + fn into_deserializer(self) -> Self::Deserializer { + Deserializer::::new(self.value, self.default_for_missing_fields) + } +} + +/// A default deserializer that always returns the default value. +struct DefaultDeserializer { + /// The originator field name (the top-level missing field name) + originator_field_name: Option, +} + +impl DefaultDeserializer { + fn new(originator_field_name: Option) -> Self { + Self { + originator_field_name, + } + } +} + +impl<'de> serde::Deserializer<'de> for DefaultDeserializer { + type Error = Error; + + fn deserialize_any(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!() + } + + fn deserialize_i32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i32(Default::default()) + } + + fn deserialize_f32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_f32(Default::default()) + } + + fn deserialize_i16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i16(Default::default()) + } + + fn deserialize_i64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i64(Default::default()) + } + + fn deserialize_u16(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u16(Default::default()) + } + + fn deserialize_u64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u64(Default::default()) + } + + fn deserialize_f64(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_f64(Default::default()) + } + + fn deserialize_bool(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_bool(Default::default()) + } + + fn deserialize_char(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_char(Default::default()) + } + + fn deserialize_str(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_str(Default::default()) + } + + fn deserialize_i8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_i8(Default::default()) + } + + fn deserialize_u8(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u8(Default::default()) + } + + fn deserialize_u32(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_u32(Default::default()) + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_none() + } + + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(DefaultSeqAccess::new(None)) + } + + fn deserialize_string(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_string(Default::default()) + } + + fn deserialize_struct( + self, + name: &'static str, + _fields: &'static [&'static str], + _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + // Return an error if the originator field name is not set + Err(Error::Other(format!( + "Missing source values for the '{}' field of type '{}'. Please verify the source data and ensure the field name is correct", + self.originator_field_name.unwrap_or("UNKNOWN".to_string()), + name, + ))) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + len: usize, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(DefaultSeqAccess::new(Some(len))) + } + + fn deserialize_tuple(self, len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(DefaultSeqAccess::new(Some(len))) + } + + fn deserialize_map(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_map(DefaultMapAccess::new()) + } + + forward_to_deserialize_any! { + u128 bytes byte_buf unit unit_struct newtype_struct + enum identifier ignored_any + } +} + +/// A default sequence access that always returns None (empty sequence). +pub struct DefaultSeqAccess { + size: Option, +} + +impl Default for DefaultSeqAccess { + fn default() -> Self { + Self::new(None) + } +} + +impl DefaultSeqAccess { + /// Creates a new default sequence access with the given size hint. + pub fn new(size: Option) -> Self { + DefaultSeqAccess { size } + } +} + +impl<'de> SeqAccess<'de> for DefaultSeqAccess { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + match self.size { + Some(0) => Ok(None), + Some(ref mut size) => { + *size -= 1; + seed.deserialize(DefaultDeserializer::new(None)).map(Some) + } + None => Ok(None), + } + } + + fn size_hint(&self) -> Option { + self.size + } +} + +/// A default map access that always returns None (empty map). +pub struct DefaultMapAccess; + +impl Default for DefaultMapAccess { + fn default() -> Self { + Self::new() + } +} + +impl DefaultMapAccess { + /// Creates a new default map access. + pub fn new() -> Self { + DefaultMapAccess + } +} + +impl<'de> MapAccess<'de> for DefaultMapAccess { + type Error = Error; + + fn next_key_seed(&mut self, _seed: T) -> Result, Self::Error> + where + T: DeserializeSeed<'de>, + { + // Since this is a default implementation, we'll just return None. + Ok(None) + } + + fn next_value_seed(&mut self, _seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + unimplemented!("This should never be called since next_key_seed always returns None") + } + + fn size_hint(&self) -> Option { + // Since this is a default implementation, we'll just return None. + None + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/error.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/error.rs new file mode 100644 index 0000000..4d67578 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/error.rs @@ -0,0 +1,40 @@ +use crate::record::RecorderError; + +/// The error type for Record serde. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// Failed to deserialize. + #[error("failed to deserialize: {0}")] + Deserialize(#[from] serde::de::value::Error), + + /// Failed to serialize. + #[error("failed to serialize")] + Serialize(String), + + /// Encountered an invalid state. + #[error("invalid state")] + InvalidState, + + /// Other error. + #[error("other error: {0}")] + Other(String), +} + +impl serde::de::Error for Error { + fn custom(msg: T) -> Self { + Error::Deserialize(serde::de::value::Error::custom(msg.to_string())) + } +} + +impl serde::ser::Error for Error { + fn custom(msg: T) -> Self { + Error::Serialize(msg.to_string()) + } +} + +// Implement From trait for Error to RecorderError +impl From for RecorderError { + fn from(error: Error) -> Self { + RecorderError::DeserializeError(error.to_string()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/mod.rs new file mode 100644 index 0000000..0d31821 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/mod.rs @@ -0,0 +1,17 @@ +//! Module contains the serde implementation for the record module +//! useful for custom importing model weights, such as PyTorch's pt file format. + +/// The adapter trait that is used to convert the nested value to the module type. +pub mod adapter; + +/// The main data structure used for deserialization. +pub mod data; + +/// The deserializer that is used to convert the nested value to the record. +pub mod ser; + +/// The deserializer that is used to convert the nested value to the record. +pub mod de; + +/// Error types. +pub mod error; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/ser.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/ser.rs new file mode 100644 index 0000000..a712e74 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/serde/ser.rs @@ -0,0 +1,387 @@ +use std::collections::HashMap; + +use super::{ + data::NestedValue, + error::{self, Error}, +}; + +use serde::{ + Serialize, + ser::{self, SerializeSeq, SerializeStruct, Serializer as SerializerTrait}, +}; + +/// Simple struct serializer that converts a struct into NestedValues. +/// +/// NOTE: This is used to serialize Param structs into NestedValues and not so much for +/// the actual serialization of modules (although it could be used for that as well if all +/// primitive types are implemented). +#[derive(Clone)] +pub struct Serializer { + /// The state of the serialization process + state: Option, +} + +impl Serializer { + /// Creates a new serializer. + pub fn new() -> Self { + Serializer { state: None } + } +} + +impl Default for Serializer { + fn default() -> Self { + Self::new() + } +} + +impl SerializerTrait for Serializer { + type Ok = NestedValue; + type Error = Error; + type SerializeSeq = Self; + type SerializeTuple = ser::Impossible; + type SerializeTupleStruct = ser::Impossible; + type SerializeTupleVariant = ser::Impossible; + type SerializeMap = ser::Impossible; + type SerializeStruct = Self; + type SerializeStructVariant = ser::Impossible; + + fn serialize_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + Ok(self) + } + + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result + where + T: Serialize + ?Sized, + { + value.serialize(self) + } + + fn serialize_seq(self, _len: Option) -> Result { + Ok(self) + } + + fn serialize_i32(self, v: i32) -> Result { + Ok(NestedValue::I32(v)) + } + + fn serialize_str(self, v: &str) -> Result { + Ok(NestedValue::String(v.to_string())) + } + + fn serialize_i16(self, v: i16) -> Result { + Ok(NestedValue::I16(v)) + } + + fn serialize_i64(self, v: i64) -> Result { + Ok(NestedValue::I64(v)) + } + + fn serialize_u16(self, v: u16) -> Result { + Ok(NestedValue::U16(v)) + } + + fn serialize_u64(self, v: u64) -> Result { + Ok(NestedValue::U64(v)) + } + + fn serialize_f32(self, v: f32) -> Result { + Ok(NestedValue::F32(v)) + } + + fn serialize_f64(self, v: f64) -> Result { + Ok(NestedValue::F64(v)) + } + + // The following methods are not implemented because they are not needed for the + // serialization of Param structs. + + fn serialize_char(self, _v: char) -> Result { + unimplemented!() + } + + fn serialize_bytes(self, v: &[u8]) -> Result { + Ok(NestedValue::U8s(v.to_vec())) + } + + fn serialize_none(self) -> Result { + Ok(NestedValue::Default(None)) + } + fn serialize_u32(self, _v: u32) -> Result { + unimplemented!() + } + fn serialize_bool(self, _v: bool) -> Result { + unimplemented!() + } + + fn serialize_i8(self, _v: i8) -> Result { + unimplemented!() + } + + fn serialize_u8(self, v: u8) -> Result { + Ok(NestedValue::U8(v)) + } + + fn serialize_some(self, value: &T) -> Result + where + T: Serialize + ?Sized, + { + value.serialize(self) + } + + fn serialize_unit(self) -> Result { + unimplemented!() + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result { + unimplemented!() + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + ) -> Result { + Ok(NestedValue::Map(HashMap::from([( + _name.to_string(), + NestedValue::String(_variant.to_string()), + )]))) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, + ) -> Result + where + T: Serialize + ?Sized, + { + unimplemented!() + } + + fn serialize_tuple(self, _len: usize) -> Result { + unimplemented!() + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + unimplemented!() + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + unimplemented!() + } + + fn serialize_map(self, _len: Option) -> Result { + unimplemented!() + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + unimplemented!() + } +} + +// Implementing the SerializeStruct trait for Serializer +impl SerializeStruct for Serializer { + type Ok = NestedValue; + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + let serialized_value = value.serialize(Serializer::new())?; + + match self.state { + Some(NestedValue::Map(ref mut map)) => { + map.insert(key.to_string(), serialized_value); // Inserting into the state + } + Some(_) => { + panic!("Invalid state encountered"); + } + None => { + let mut map = HashMap::new(); + map.insert(key.to_string(), serialized_value); // Inserting into the state + self.state = Some(NestedValue::Map(map)); + } + } + + Ok(()) + } + + fn end(self) -> Result { + if self.state.is_none() { + // If the state is empty, return an empty map + Ok(NestedValue::Map(HashMap::new())) + } else { + self.state.ok_or(error::Error::InvalidState) + } + } +} + +impl SerializeSeq for Serializer { + type Ok = NestedValue; + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + let serialized_value = value.serialize(Serializer::new())?; + + match self.state { + Some(NestedValue::Vec(ref mut vec)) => { + vec.push(serialized_value); // Inserting into the state + } + Some(NestedValue::U8s(ref mut vec)) => { + if let NestedValue::U8(val) = serialized_value { + vec.push(val); + } else { + panic!("Invalid value type encountered"); + } + } + Some(NestedValue::U16s(ref mut vec)) => { + if let NestedValue::U16(val) = serialized_value { + vec.push(val); + } else { + panic!("Invalid value type encountered"); + } + } + Some(NestedValue::F32s(ref mut vec)) => { + if let NestedValue::F32(val) = serialized_value { + vec.push(val); + } else { + panic!("Invalid value type encountered"); + } + } + Some(_) => { + panic!("Invalid state encountered"); + } + None => { + let val = match serialized_value { + NestedValue::U8(val) => NestedValue::U8s(vec![val]), + NestedValue::U16(val) => NestedValue::U16s(vec![val]), + NestedValue::F32(val) => NestedValue::F32s(vec![val]), + _ => NestedValue::Vec(vec![serialized_value]), + }; + self.state = Some(val); + } + } + + Ok(()) + } + + fn end(self) -> Result { + if self.state.is_none() { + // If the state is empty, return an empty vector + Ok(NestedValue::Vec(Vec::new())) + } else { + self.state.ok_or(error::Error::InvalidState) + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + TestBackend, + module::{Param, ParamId}, + record::{FullPrecisionSettings, Record}, + tensor::Tensor, + }; + use serde::Deserialize; + + use super::*; + + #[derive(Serialize, Deserialize, Debug, Clone)] + struct MyStruct1 { + a: MyStruct3, + b: MyStruct2, + } + + #[derive(Serialize, Deserialize, Debug, Clone)] + struct MyStruct2 { + a: i32, + b: Option, + c: String, + d: Option, + } + + #[derive(Serialize, Deserialize, Debug, Clone)] + struct MyStruct3 { + x: String, + y: String, + } + + #[test] + fn test_serialize() { + let my_struct = MyStruct1 { + a: MyStruct3 { + x: "Hello".to_owned(), + y: "World".to_owned(), + }, + b: MyStruct2 { + a: 1, + b: None, + c: "Hello".to_owned(), + d: Some("World".to_owned()), + }, + }; + + let serialized = my_struct + .serialize(Serializer::new()) + .expect("Should serialize item successfully"); + + let serialized_str = format!("{serialized:?}"); + + // Compare the lengths of expected and actual serialized strings because + // the order of the fields is not guaranteed for HashMaps. + assert_eq!(serialized_str.len(), 135); + } + + #[test] + fn test_param_serde() { + let device = Default::default(); + let tensor: Tensor = Tensor::ones([2, 2], &device); + let param = Param::initialized(ParamId::new(), tensor); + let param_item = param.into_item::(); + + let serialized = param_item + .serialize(Serializer::new()) + .expect("Should serialize item successfully"); + + let bytes = serialized.as_map().expect("is a map")["param"] + .clone() + .as_map() + .expect("param is a map")["bytes"] + .clone() + .as_bytes() + .expect("has bytes vec"); + assert_eq!(&*bytes, [1.0f32; 4].map(|f| f.to_le_bytes()).as_flattened()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/settings.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/settings.rs new file mode 100644 index 0000000..e99919c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/settings.rs @@ -0,0 +1,40 @@ +use burn_tensor::Element; +use serde::{Serialize, de::DeserializeOwned}; + +/// Settings allowing to control the precision when (de)serializing items. +pub trait PrecisionSettings: + Send + Sync + core::fmt::Debug + core::default::Default + Clone +{ + /// Float element type. + type FloatElem: Element + Serialize + DeserializeOwned; + + /// Integer element type. + type IntElem: Element + Serialize + DeserializeOwned; +} + +/// Default precision settings. +#[derive(Debug, Default, Clone)] +pub struct FullPrecisionSettings; + +/// Precision settings optimized for compactness. +#[derive(Debug, Default, Clone)] +pub struct HalfPrecisionSettings; + +/// Precision settings optimized for precision. +#[derive(Debug, Default, Clone)] +pub struct DoublePrecisionSettings; + +impl PrecisionSettings for FullPrecisionSettings { + type FloatElem = f32; + type IntElem = i32; +} + +impl PrecisionSettings for DoublePrecisionSettings { + type FloatElem = f64; + type IntElem = i64; +} + +impl PrecisionSettings for HalfPrecisionSettings { + type FloatElem = half::f16; + type IntElem = i16; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/tensor.rs new file mode 100644 index 0000000..cb5f3d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/record/tensor.rs @@ -0,0 +1,159 @@ +use core::marker::PhantomData; + +use super::{PrecisionSettings, Record}; +use burn_tensor::{Bool, DType, Element, Int, Tensor, TensorData, backend::Backend}; +use serde::{Deserialize, Serialize}; + +use alloc::format; + +/// Deserialize the value into [`TensorData`]. +fn deserialize_data<'de, E, De>(deserializer: De) -> Result +where + E: Element + Deserialize<'de>, + De: serde::Deserializer<'de>, +{ + let data = TensorData::deserialize(deserializer).map_err(|e| { + serde::de::Error::custom(format!( + "{e:?}\nThe internal data format has changed since version 0.14.0. If you are trying to load a record saved in a previous version, use the `record-backward-compat` feature flag with a previous version (<=0.16.0). Once you have saved the record in the new format, you can upgrade back to the current version.\n" + )) + })?; + let data = if let DType::QFloat(_) = data.dtype { + data // do not convert quantized tensors + } else { + data.convert::() + }; + Ok(data) +} + +/// This struct implements serde to lazily serialize and deserialize a float tensor +/// using the given [record settings](RecordSettings). +#[derive(new, Clone, Debug)] +pub struct FloatTensorSerde { + data: TensorData, + _e: PhantomData, +} + +/// This struct implements serde to lazily serialize and deserialize an int tensor +/// using the given [record settings](RecordSettings). +#[derive(new, Clone, Debug)] +pub struct IntTensorSerde { + data: TensorData, + _e: PhantomData, +} + +/// This struct implements serde to lazily serialize and deserialize an bool tensor. +#[derive(new, Clone, Debug)] +pub struct BoolTensorSerde { + data: TensorData, +} + +// --- SERDE IMPLEMENTATIONS --- // + +impl Serialize for FloatTensorSerde { + fn serialize(&self, serializer: Se) -> Result + where + Se: serde::Serializer, + { + self.data.serialize(serializer) + } +} + +impl<'de, S: PrecisionSettings> Deserialize<'de> for FloatTensorSerde { + fn deserialize(deserializer: De) -> Result + where + De: serde::Deserializer<'de>, + { + let data = deserialize_data::(deserializer)?; + + Ok(Self::new(data)) + } +} + +impl Serialize for IntTensorSerde { + fn serialize(&self, serializer: Se) -> Result + where + Se: serde::Serializer, + { + self.data.serialize(serializer) + } +} + +impl<'de, S: PrecisionSettings> Deserialize<'de> for IntTensorSerde { + fn deserialize(deserializer: De) -> Result + where + De: serde::Deserializer<'de>, + { + let data = deserialize_data::(deserializer)?; + + Ok(Self::new(data)) + } +} + +impl Serialize for BoolTensorSerde { + fn serialize(&self, serializer: Se) -> Result + where + Se: serde::Serializer, + { + self.data.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for BoolTensorSerde { + fn deserialize(deserializer: De) -> Result + where + De: serde::Deserializer<'de>, + { + let data = deserialize_data::(deserializer)?; + + Ok(Self::new(data)) + } +} + +// --- RECORD IMPLEMENTATIONS --- // + +impl Record for Tensor { + type Item = FloatTensorSerde; + + fn into_item(self) -> Self::Item { + let data = self.into_data(); + let data = if let DType::QFloat(_) = data.dtype { + data // do not convert quantized tensors + } else { + data.convert::() + }; + FloatTensorSerde::new(data) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + let data = if let DType::QFloat(_) = item.data.dtype { + item.data // do not convert quantized tensors + } else { + item.data.convert::() + }; + Tensor::from_data(data, device) + } +} + +impl Record for Tensor { + type Item = IntTensorSerde; + + fn into_item(self) -> Self::Item { + IntTensorSerde::new(self.into_data().convert::()) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + Tensor::from_data(item.data.convert::(), device) + } +} + +impl Record for Tensor { + type Item = BoolTensorSerde; + + fn into_item(self) -> Self::Item { + BoolTensorSerde::new(self.into_data()) + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + Tensor::from_data(item.data, device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/tensor.rs new file mode 100644 index 0000000..074606b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/tensor.rs @@ -0,0 +1 @@ +pub use burn_tensor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/src/vision.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/src/vision.rs new file mode 100644 index 0000000..adcd3f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/src/vision.rs @@ -0,0 +1 @@ +pub use burn_vision::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_config.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_config.rs new file mode 100644 index 0000000..65c4376 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_config.rs @@ -0,0 +1,113 @@ +use burn::config::{Config, config_to_json}; +use burn_core as burn; + +#[derive(Config, Debug, PartialEq, Eq)] +pub struct TestEmptyStructConfig {} + +#[derive(Config, Debug, PartialEq)] +pub struct TestStructConfig { + int: i32, + #[config(default = 2)] + int_default: i32, + float: f32, + #[config(default = 2.0)] + float_default: f32, + string: String, + other_config: TestEmptyStructConfig, +} + +#[derive(Config, Debug, PartialEq)] +pub enum TestEnumConfig { + None, + Single(f32), + Multiple(f32, String), + Named { first: f32, second: String }, +} + +#[cfg(feature = "std")] +#[inline(always)] +fn file_path(file_name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(file_name) +} + +#[cfg(feature = "std")] +#[test] +fn struct_config_should_impl_serde() { + let config = TestStructConfig::new(2, 3.0, "Allow".to_string(), TestEmptyStructConfig::new()); + let file_path = file_path("test_struct_config.json"); + + config.save(&file_path).unwrap(); + + let config_loaded = TestStructConfig::load(&file_path).unwrap(); + assert_eq!(config, config_loaded); +} + +#[test] +fn struct_config_should_impl_clone() { + let config = TestStructConfig::new(2, 3.0, "Allow".to_string(), TestEmptyStructConfig::new()); + assert_eq!(config, config.clone()); +} + +#[test] +fn struct_config_should_impl_display() { + let config = TestStructConfig::new(2, 3.0, "Allow".to_string(), TestEmptyStructConfig::new()); + assert_eq!(burn::config::config_to_json(&config), config.to_string()); +} + +#[cfg(feature = "std")] +#[test] +fn enum_config_no_value_should_impl_serde() { + let config = TestEnumConfig::None; + let file_path = file_path("test_enum_no_value_config.json"); + + config.save(&file_path).unwrap(); + + let config_loaded = TestEnumConfig::load(&file_path).unwrap(); + assert_eq!(config, config_loaded); +} + +#[cfg(feature = "std")] +#[test] +fn enum_config_one_value_should_impl_serde() { + let config = TestEnumConfig::Single(42.0); + let file_path = file_path("test_enum_one_value_config.json"); + + config.save(&file_path).unwrap(); + + let config_loaded = TestEnumConfig::load(&file_path).unwrap(); + assert_eq!(config, config_loaded); +} + +#[cfg(feature = "std")] +#[test] +fn enum_config_multiple_values_should_impl_serde() { + let config = TestEnumConfig::Multiple(42.0, "Allow".to_string()); + let file_path = file_path("test_enum_multiple_values_config.json"); + + config.save(&file_path).unwrap(); + + let config_loaded = TestEnumConfig::load(&file_path).unwrap(); + assert_eq!(config, config_loaded); +} + +#[test] +fn enum_config_should_impl_clone() { + let config = TestEnumConfig::Multiple(42.0, "Allow".to_string()); + assert_eq!(config, config.clone()); +} + +#[test] +fn enum_config_should_impl_display() { + let config = TestEnumConfig::Multiple(42.0, "Allow".to_string()); + assert_eq!(burn::config::config_to_json(&config), config.to_string()); +} + +#[test] +fn struct_config_can_load_binary() { + let config = TestStructConfig::new(2, 3.0, "Allow".to_string(), TestEmptyStructConfig::new()); + + let binary = config_to_json(&config).as_bytes().to_vec(); + + let config_loaded = TestStructConfig::load_binary(&binary).unwrap(); + assert_eq!(config, config_loaded); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_module.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_module.rs new file mode 100644 index 0000000..4195db4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_module.rs @@ -0,0 +1,323 @@ +use std::marker::PhantomData; + +use burn::module::Initializer; +use burn::module::{Module, Param}; +use burn::tensor::backend::Backend; +use burn::tensor::{Int, Tensor}; +use burn_core as burn; + +pub type TestBackend = burn_ndarray::NdArray; +#[cfg(feature = "std")] +pub type TestAutodiffBackend = burn_autodiff::Autodiff; + +#[derive(Module, Debug)] +pub struct ModuleBasic { + weight_basic: Param>, +} + +#[derive(Module, Debug)] +#[allow(unused)] +struct ModuleTensorConstInt { + weight_basic: Tensor, +} + +impl ModuleBasic { + fn new(device: &B::Device) -> Self { + Self { + weight_basic: Initializer::Normal { + std: 1.0, + mean: 0.0, + } + .init([20, 20], device), + } + } +} + +#[derive(Module, Debug)] +struct ModuleWithConstGeneric { + modules: [ModuleBasic; N], +} + +#[derive(Module, Debug)] +struct ModuleWithGenericModule { + module: M, + _backend: PhantomData, +} + +#[derive(Module, Debug)] +#[allow(clippy::large_enum_variant)] +enum ModuleEnum { + Basic(ModuleBasic), + Composed(ModuleComposed), +} + +#[derive(Module, Debug)] +#[allow(unused)] +enum ModuleEnumNested { + AnotherEnum(ModuleEnum), +} + +#[derive(Module, Debug)] +enum ModuleEnumWithGenericModule> { + Basic(ModuleBasic), + Generic(ModuleWithGenericModule), +} + +#[derive(Module, Debug)] +pub struct ModuleComposed { + weight: Param>, + basic: ModuleBasic, + tuple: (ModuleBasic, ModuleBasic), +} + +impl ModuleComposed { + fn new(device: &B::Device) -> Self { + let weight = Initializer::Normal { + std: 1.0, + mean: 0.0, + } + .init([20, 20], device); + + Self { + weight, + basic: ModuleBasic::new(device), + tuple: (ModuleBasic::new(device), ModuleBasic::new(device)), + } + } +} + +#[allow(dead_code)] +mod compiletime_clone_impl_check { + use burn_core::{ + module::{Module, ModuleDisplay}, + prelude::Backend, + record::{PrecisionSettings, Record}, + }; + + use super::*; + + type RecordItem = <>::Record as Record>::Item; + + fn implements_clone() {} + + fn basic_implements_clone() { + implements_clone::, B, S>>(); + implements_clone::, B, S>>(); + } + + fn generic_implements_clone() + where + B: Backend, + S: PrecisionSettings, + M: Module + ModuleDisplay, + RecordItem: Clone, + { + implements_clone::, B, S>>(); + implements_clone::, B, S>>(); + } +} + +mod state { + use super::*; + + #[test] + fn should_load_from_record_basic() { + let device = ::Device::default(); + let module_1 = ModuleBasic::::new(&device); + let mut module_2 = ModuleBasic::::new(&device); + let state_1 = module_1.clone().into_record(); + + assert_ne!( + module_1.weight_basic.to_data(), + module_2.weight_basic.to_data() + ); + + module_2 = module_2.load_record(state_1); + + assert_eq!( + module_1.weight_basic.to_data(), + module_2.weight_basic.to_data() + ); + } + + #[test] + fn should_load_from_record_compose() { + let device = ::Device::default(); + let module_1 = ModuleComposed::::new(&device); + let mut module_2 = ModuleComposed::::new(&device); + assert_ne!(module_1.weight.to_data(), module_2.weight.to_data()); + assert_ne!( + module_1.basic.weight_basic.to_data(), + module_2.basic.weight_basic.to_data() + ); + + let state_1 = module_1.clone().into_record(); + module_2 = module_2.load_record(state_1); + + assert_eq!(module_1.weight.to_data(), module_2.weight.to_data()); + assert_eq!( + module_1.basic.weight_basic.to_data(), + module_2.basic.weight_basic.to_data() + ); + } + + #[test] + fn should_load_from_record_enum() { + let device = ::Device::default(); + let module_1 = ModuleEnum::Basic(ModuleBasic::::new(&device)); + let mut module_2 = ModuleEnum::Basic(ModuleBasic::::new(&device)); + let state_1 = module_1.clone().into_record(); + + let ModuleEnum::Basic(module_1_basic) = module_1 else { + panic!("Invalid module type") + }; + let ModuleEnum::Basic(module_2_basic) = module_2.clone() else { + panic!("Invalid module type") + }; + assert_ne!( + module_1_basic.weight_basic.to_data(), + module_2_basic.weight_basic.to_data() + ); + + module_2 = module_2.load_record(state_1); + + let ModuleEnum::Basic(module_2_basic) = module_2 else { + panic!("Invalid module type") + }; + assert_eq!( + module_1_basic.weight_basic.to_data(), + module_2_basic.weight_basic.to_data() + ); + } + + #[test] + fn should_load_from_record_const_generic() { + let device = ::Device::default(); + let module_1 = ModuleWithConstGeneric { + modules: [ + ModuleBasic::::new(&device), + ModuleBasic::::new(&device), + ], + }; + let mut module_2 = ModuleWithConstGeneric { + modules: [ + ModuleBasic::::new(&device), + ModuleBasic::::new(&device), + ], + }; + let state_1 = module_1.clone().into_record(); + + assert_ne!( + module_1.modules[0].weight_basic.to_data(), + module_2.modules[0].weight_basic.to_data(), + ); + assert_ne!( + module_1.modules[1].weight_basic.to_data(), + module_2.modules[1].weight_basic.to_data(), + ); + + module_2 = module_2.load_record(state_1); + + assert_eq!( + module_1.modules[0].weight_basic.to_data(), + module_2.modules[0].weight_basic.to_data(), + ); + assert_eq!( + module_1.modules[1].weight_basic.to_data(), + module_2.modules[1].weight_basic.to_data(), + ); + } + + #[test] + #[should_panic(expected = "Can't parse record from a different variant")] + fn should_panic_load_from_incorrect_enum_variant() { + let device = ::Device::default(); + let module_1 = ModuleEnum::Basic(ModuleBasic::::new(&device)); + let module_2 = ModuleEnum::Composed(ModuleComposed::::new(&device)); + let state_1 = module_1.clone().into_record(); + + module_2.load_record(state_1); + } +} + +mod num_params { + use super::*; + + #[test] + fn should_calculate_num_params_basic() { + let device = ::Device::default(); + let module = ModuleBasic::::new(&device); + assert_eq!(20 * 20, module.num_params()); + } + + #[test] + fn should_output_state_composed() { + let device = ::Device::default(); + let module = ModuleComposed::::new(&device); + assert_eq!(4 * 20 * 20, module.num_params()); + } + + #[test] + fn should_calculate_num_params_enum() { + let device = ::Device::default(); + let module = ModuleEnum::Basic(ModuleBasic::::new(&device)); + assert_eq!(20 * 20, module.num_params()); + + let module = ModuleEnum::Composed(ModuleComposed::::new(&device)); + assert_eq!(4 * 20 * 20, module.num_params()); + } +} + +#[cfg(feature = "std")] +mod require_grad { + use burn_tensor::backend::AutodiffBackend; + + use super::*; + + #[test] + fn should_have_grad_by_default() { + let device = ::Device::default(); + let module = ModuleBasic::::new(&device); + let mut grads = calculate_grads(&module); + + let grad_x = module.weight_basic.grad_remove(&mut grads); + + assert!(grad_x.is_some()); + } + + #[test] + fn should_have_no_grad_after_no_grad() { + let device = ::Device::default(); + let module = ModuleBasic::::new(&device).no_grad(); + let mut grads = calculate_grads(&module); + + let grad_x = module.weight_basic.grad_remove(&mut grads); + + assert!(grad_x.is_none()); + } + + #[test] + fn should_have_grad_when_from_record() { + let device = ::Device::default(); + let module = ModuleBasic::::new(&device); + let record = ModuleBasicRecord { + weight_basic: module.weight_basic.clone(), // Even when param is no_grad, + }; + let module = module.load_record(record); + let mut grads = calculate_grads(&module); + + let grad_x = module.weight_basic.grad_remove(&mut grads); + + assert!(grad_x.is_some()); + } + + fn calculate_grads( + module: &ModuleBasic, + ) -> ::Gradients { + let device = module.weight_basic.device(); + let x = Tensor::ones([20, 20], &device).require_grad(); + let y = module.weight_basic.val().matmul(x); + + y.backward() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_record.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_record.rs new file mode 100644 index 0000000..43baa13 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_derive_record.rs @@ -0,0 +1,17 @@ +use burn_core as burn; +use burn_core::record::Record; + +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +// It compiles +#[derive(Record)] +pub struct TestWithBackendRecord { + tensor: Tensor, +} + +// It compiles +#[derive(Record)] +pub struct TestWithoutBackendRecord { + _tensor: usize, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_record_resilience.rs b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_record_resilience.rs new file mode 100644 index 0000000..0f1d82d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-core/tests/test_record_resilience.rs @@ -0,0 +1,344 @@ +#[cfg(feature = "std")] +mod tests { + use burn::{ + module::{Module, Param}, + record::{ + BinFileRecorder, DefaultFileRecorder, FileRecorder, FullPrecisionSettings, + PrettyJsonFileRecorder, RecorderError, + }, + }; + use burn_core as burn; + use burn_ndarray::NdArrayDevice; + use burn_tensor::{Tensor, backend::Backend}; + use std::path::PathBuf; + + type TestBackend = burn_ndarray::NdArray; + + /// Simple linear module. + #[derive(Module, Debug)] + pub struct Linear { + pub weight: Param>, + pub bias: Option>>, + } + + impl Linear { + pub fn new(in_features: usize, out_features: usize, device: &B::Device) -> Self { + let weight = Tensor::random( + [out_features, in_features], + burn_tensor::Distribution::Default, + device, + ); + let bias = Tensor::random([out_features], burn_tensor::Distribution::Default, device); + + Self { + weight: Param::from_tensor(weight), + bias: Some(Param::from_tensor(bias)), + } + } + } + + #[derive(Module, Debug)] + pub struct Model { + single_const: f32, + linear1: Linear, + array_const: [usize; 2], + linear2: Linear, + } + + #[derive(Module, Debug)] + pub struct ModelNewOptionalField { + single_const: f32, + linear1: Linear, + array_const: [usize; 2], + linear2: Linear, + new_field: Option, + } + + #[derive(Module, Debug)] + pub struct ModelNewConstantField { + single_const: f32, + linear1: Linear, + array_const: [usize; 2], + linear2: Linear, + new_field: usize, + } + + #[derive(Module, Debug)] + #[allow(unused)] + pub struct ModelNewFieldOrders { + array_const: [usize; 2], + linear2: Linear, + single_const: f32, + linear1: Linear, + } + + #[test] + fn deserialize_with_new_optional_field_works_with_default_file_recorder() { + deserialize_with_new_optional_field( + "default", + DefaultFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_removed_optional_field_works_with_default_file_recorder() { + deserialize_with_removed_optional_field( + "default", + DefaultFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_new_constant_field_works_with_default_file_recorder() { + deserialize_with_new_constant_field( + "default", + DefaultFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_removed_constant_field_works_with_default_file_recorder() { + deserialize_with_removed_constant_field( + "default", + DefaultFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_new_field_order_works_with_default_file_recorder() { + deserialize_with_new_field_order( + "default", + DefaultFileRecorder::::new(), + ) + .unwrap(); + } + #[test] + fn deserialize_with_new_optional_field_works_with_pretty_json() { + deserialize_with_new_optional_field( + "pretty-json", + PrettyJsonFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_removed_optional_field_works_with_pretty_json() { + deserialize_with_removed_optional_field( + "pretty-json", + PrettyJsonFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_new_constant_field_works_with_pretty_json() { + deserialize_with_new_constant_field( + "pretty-json", + PrettyJsonFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_removed_constant_field_works_with_pretty_json() { + deserialize_with_removed_constant_field( + "pretty-json", + PrettyJsonFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_new_field_order_works_with_pretty_json() { + deserialize_with_new_field_order( + "pretty-json", + PrettyJsonFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + #[should_panic] + fn deserialize_with_new_optional_field_doesnt_works_with_bin_file_recorder() { + deserialize_with_new_optional_field("bin", BinFileRecorder::::new()) + .unwrap(); + } + + #[test] + fn deserialize_with_removed_optional_field_works_with_bin_file_recorder() { + deserialize_with_removed_optional_field( + "bin", + BinFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + fn deserialize_with_new_constant_field_works_with_bin_file_recorder() { + deserialize_with_new_constant_field("bin", BinFileRecorder::::new()) + .unwrap(); + } + + #[test] + fn deserialize_with_removed_constant_field_works_with_bin_file_recorder() { + deserialize_with_removed_constant_field( + "bin", + BinFileRecorder::::new(), + ) + .unwrap(); + } + + #[test] + #[should_panic] + fn deserialize_with_new_field_order_works_with_bin_file_recorder() { + deserialize_with_new_field_order("bin", BinFileRecorder::::new()) + .unwrap(); + } + + #[inline(always)] + fn file_path(filename: String) -> PathBuf { + std::env::temp_dir().join(filename) + } + + #[test] + fn test_tensor_serde() { + let tensor: burn_tensor::Tensor = + burn_tensor::Tensor::ones([1], &NdArrayDevice::default()); + let encoded = serde_json::to_string(&tensor).unwrap(); + let decoded: burn_tensor::Tensor = serde_json::from_str(&encoded).unwrap(); + assert_eq!(tensor.into_data(), decoded.into_data()); + } + + fn deserialize_with_new_optional_field(name: &str, recorder: R) -> Result<(), RecorderError> + where + R: FileRecorder, + { + let device = Default::default(); + let file_path: PathBuf = file_path(format!("deserialize_with_new_optional_field-{name}")); + let model = Model { + single_const: 32.0, + linear1: Linear::::new(20, 20, &device), + array_const: [2, 2], + linear2: Linear::::new(20, 20, &device), + }; + + recorder + .record(model.into_record(), file_path.clone()) + .unwrap(); + let result = + recorder.load::>(file_path.clone(), &device); + std::fs::remove_file(file_path).ok(); + + result?; + Ok(()) + } + + fn deserialize_with_removed_optional_field( + name: &str, + recorder: R, + ) -> Result<(), RecorderError> + where + R: FileRecorder, + { + let device = Default::default(); + let file_path: PathBuf = + file_path(format!("deserialize_with_removed_optional_field-{name}")); + let model = ModelNewOptionalField { + single_const: 32.0, + linear1: Linear::::new(20, 20, &device), + array_const: [2, 2], + linear2: Linear::::new(20, 20, &device), + new_field: None, + }; + + recorder + .record(model.into_record(), file_path.clone()) + .unwrap(); + let result = recorder.load::>(file_path.clone(), &device); + std::fs::remove_file(file_path).ok(); + + result?; + Ok(()) + } + + fn deserialize_with_new_constant_field(name: &str, recorder: R) -> Result<(), RecorderError> + where + R: FileRecorder, + { + let device = Default::default(); + let file_path: PathBuf = file_path(format!("deserialize_with_new_constant_field-{name}")); + let model = Model { + single_const: 32.0, + array_const: [2, 2], + linear1: Linear::::new(20, 20, &device), + linear2: Linear::::new(20, 20, &device), + }; + + recorder + .record(model.into_record(), file_path.clone()) + .unwrap(); + let result = + recorder.load::>(file_path.clone(), &device); + std::fs::remove_file(file_path).ok(); + + result?; + Ok(()) + } + + fn deserialize_with_removed_constant_field( + name: &str, + recorder: R, + ) -> Result<(), RecorderError> + where + R: FileRecorder, + { + let device = Default::default(); + let file_path: PathBuf = + file_path(format!("deserialize_with_removed_constant_field-{name}")); + let model = ModelNewConstantField { + single_const: 32.0, + array_const: [2, 2], + linear1: Linear::::new(20, 20, &device), + linear2: Linear::::new(20, 20, &device), + new_field: 0, + }; + + recorder + .record(model.into_record(), file_path.clone()) + .unwrap(); + let result = recorder.load::>(file_path.clone(), &device); + std::fs::remove_file(file_path).ok(); + + result?; + Ok(()) + } + + fn deserialize_with_new_field_order(name: &str, recorder: R) -> Result<(), RecorderError> + where + R: FileRecorder, + { + let device = Default::default(); + let file_path: PathBuf = file_path(format!("deserialize_with_new_field_order-{name}")); + let model = Model { + array_const: [2, 2], + single_const: 32.0, + linear1: Linear::::new(20, 20, &device), + linear2: Linear::::new(20, 20, &device), + }; + + recorder + .record(model.into_record(), file_path.clone()) + .unwrap(); + + let result = + recorder.load::>(file_path.clone(), &device); + std::fs::remove_file(file_path).ok(); + + result?; + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cpu/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-cpu/Cargo.toml new file mode 100644 index 0000000..0546fad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cpu/Cargo.toml @@ -0,0 +1,42 @@ +[package] +authors = ["marcantoinem "] +categories = ["science"] +description = "MLIR based CPU backend for the Burn framework" +documentation = "https://docs.rs/burn-cpu" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "cpu"] +license.workspace = true +name = "burn-cpu" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-cpu" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std", "fusion", "autotune", "burn-cubecl/default", "cubecl/default"] +doc = ["burn-cubecl/doc"] +fusion = ["burn-fusion", "burn-cubecl/fusion"] +std = ["burn-cubecl/std", "cubecl/std"] +tracing = [ + "burn-backend/tracing", + "burn-cubecl/tracing", + "burn-fusion?/tracing", + "cubecl/tracing", +] + +autotune = ["burn-cubecl/autotune"] +autotune-checks = ["burn-cubecl/autotune-checks"] + +[dependencies] +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } +burn-cubecl = { path = "../burn-cubecl", version = "=0.21.0-pre.2", default-features = false } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", features = [ + "cubecl-cpu", +] } +cubecl = { workspace = true, features = ["cpu"] } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cpu/README.md b/crates/stable-diffusion-burn/burn-crates/burn-cpu/README.md new file mode 100644 index 0000000..e1f1639 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cpu/README.md @@ -0,0 +1,12 @@ +# Burn CPU Backend + +[Burn](https://github.com/tracel-ai/burn) CubeCL CPU backend + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-cuda.svg)](https://crates.io/crates/burn-cuda) + +This crate provides a MLIR based CPU backend for [Burn](https://github.com/tracel-ai/burn) using the +[cubecl](https://github.com/tracel-ai/cubecl.git) crates. + +## Usage Example + +Example coming soon diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cpu/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-cpu/src/lib.rs new file mode 100644 index 0000000..1fb94de --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cpu/src/lib.rs @@ -0,0 +1,47 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] + +extern crate alloc; + +use burn_cubecl::CubeBackend; +pub use cubecl::cpu::CpuDevice; +use cubecl::cpu::CpuRuntime; + +#[cfg(not(feature = "fusion"))] +pub type Cpu = CubeBackend; + +#[cfg(feature = "fusion")] +pub type Cpu = burn_fusion::Fusion>; + +#[cfg(test)] +mod tests { + use super::*; + use burn_backend::{Backend, DType, QTensorPrimitive}; + use burn_cubecl::tensor::CubeTensor; + + #[test] + fn should_support_dtypes() { + type B = Cpu; + let device = Default::default(); + + assert!(B::supports_dtype(&device, DType::F64)); + assert!(B::supports_dtype(&device, DType::F32)); + assert!(B::supports_dtype(&device, DType::F16)); + assert!(B::supports_dtype(&device, DType::BF16)); + assert!(B::supports_dtype(&device, DType::I64)); + assert!(B::supports_dtype(&device, DType::I32)); + assert!(B::supports_dtype(&device, DType::I16)); + assert!(B::supports_dtype(&device, DType::I8)); + assert!(B::supports_dtype(&device, DType::U64)); + assert!(B::supports_dtype(&device, DType::U32)); + assert!(B::supports_dtype(&device, DType::U16)); + assert!(B::supports_dtype(&device, DType::U8)); + assert!(B::supports_dtype( + &device, + DType::QFloat(CubeTensor::::default_scheme()) + )); + + // Currently not registered in supported types + assert!(!B::supports_dtype(&device, DType::Flex32)); + assert!(!B::supports_dtype(&device, DType::Bool)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/Cargo.toml new file mode 100644 index 0000000..53ff232 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/Cargo.toml @@ -0,0 +1,52 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Provide optimizations that can be used with cubecl based backends." +documentation = "https://docs.rs/burn-cubecl-fusion" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "gpu"] +license.workspace = true +name = "burn-cubecl-fusion" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-cubecl-fusion" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["autotune", "std", "cubecl/default", "burn-fusion/default"] + +autotune = [] +autotune-checks = ["cubecl/autotune-checks", "burn-backend", "half"] +doc = ["default"] +std = ["cubecl/std", "burn-backend?/std", "burn-fusion/std"] +tracing = [ + "cubecl/tracing", + "burn-std/tracing", + "burn-backend/tracing", + "burn-fusion/tracing", +] + +[dependencies] +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", default-features = false } +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2", default-features = false } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", features = [ + "cubecl", +] } +cubecl = { workspace = true } +cubek = { workspace = true, features = ["matmul", "reduce", "quantization"] } +half = { workspace = true, optional = true } + +# Only for `TensorData` with autotune-checks +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false, optional = true } + +derive-new = { workspace = true } +serde = { workspace = true } + +[dev-dependencies] +cubecl = { workspace = true, features = ["test-runtime"] } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/README.md b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/README.md new file mode 100644 index 0000000..88c5797 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/README.md @@ -0,0 +1,3 @@ +# Burn CubeCl Fusion + +Provide optimizations that can be used with [cubecl](../burn-cubecl) based backends. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/base.rs new file mode 100644 index 0000000..9dd8103 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/base.rs @@ -0,0 +1,165 @@ +use burn_fusion::stream::Context; +use burn_std::{DType, Strides, quantization::QParamTensor, strides}; +use cubecl::{ + CubeElement, Runtime, + client::ComputeClient, + ir::{AddressType, ElemType}, + prelude::{TensorArg, TensorHandleRef}, +}; +use cubecl::{ + ir::LineSize, + quant::scheme::{QuantParam, QuantScheme}, +}; +use std::marker::PhantomData; + +/// Defines a fallback operation when fusion isn't possible. +pub trait FallbackOperation: Send + Sync { + /// Executes the fallback procedure. + fn run(&self, context: &mut Context<'_, CubeFusionHandle>); +} + +/// Runtime parameters for quantization. Can be used to construct a scales handle from the base +/// tensor handle. +pub type QParams = burn_std::quantization::QParams; + +/// Handle to be used when fusing operations. +pub struct CubeFusionHandle { + /// Compute client for jit. + pub client: ComputeClient, + /// The buffer where the data are stored. + pub handle: cubecl::server::Handle, + /// The device of the current tensor. + pub device: R::Device, + /// The element type of the tensor. + pub dtype: DType, + /// The strides of the tensor. + pub strides: Strides, + /// Quantization runtime parameters, if applicable + pub qparams: Option, +} + +impl core::fmt::Debug for CubeFusionHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "CubeFusionHandle {{ device: {:?}, runtime: {}}}", + self.device, + R::name(&self.client), + )) + } +} + +impl Clone for CubeFusionHandle { + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + handle: self.handle.clone(), + device: self.device.clone(), + strides: self.strides.clone(), + dtype: self.dtype, + qparams: self.qparams.clone(), + } + } +} + +unsafe impl Send for CubeFusionHandle {} +unsafe impl Sync for CubeFusionHandle {} + +impl CubeFusionHandle { + /// Return the reference to a tensor handle. + pub fn as_handle_ref<'a>(&'a self, shape: &'a [usize]) -> TensorHandleRef<'a, R> { + TensorHandleRef { + handle: &self.handle, + strides: &self.strides, + shape, + runtime: PhantomData, + elem_size: self.dtype.size(), + } + } + + pub fn required_address_type(&self) -> AddressType { + match self.dtype { + DType::QFloat(scheme) => { + let len = self.handle.size() as usize * 8 / scheme.size_bits_value(); + AddressType::from_len(len) + } + _ => AddressType::from_len(self.handle.size() as usize / self.dtype.size()), + } + } + + /// Return the reference to a tensor argument. + pub fn as_tensor_arg<'a>( + &'a self, + shape: &'a [usize], + line_size: LineSize, + ) -> TensorArg<'a, R> { + let handle: TensorHandleRef<'a, R> = self.as_handle_ref(shape); + + unsafe { + TensorArg::from_raw_parts_and_size( + handle.handle, + handle.strides, + handle.shape, + line_size, + self.dtype.size(), + ) + } + } + /// Construct a separate tensor for the quantization scales, if present + pub fn params(&self, scheme: QuantScheme) -> Option { + let qparams = self.qparams.as_ref()?; + let mut handle = self.handle.clone(); + handle.offset_start = Some(qparams.scales.offset_start as u64); + handle.offset_end = Some(qparams.scales.offset_end as u64); + + Some(Self { + client: self.client.clone(), + handle, + device: self.device.clone(), + dtype: match scheme.param { + QuantParam::F32 => DType::F32, + QuantParam::F16 => DType::F16, + QuantParam::BF16 => DType::BF16, + QuantParam::UE8M0 | QuantParam::UE4M3 => unimplemented!("Not yet supported"), + }, + strides: qparams.scales.metadata.strides().clone(), + qparams: None, + }) + } +} + +pub(crate) fn strides_dyn_rank(shape: &[usize]) -> Strides { + let mut strides = strides![0; shape.len()]; + + let mut current = 1; + shape.iter().enumerate().rev().for_each(|(index, val)| { + strides[index] = current; + current *= val; + }); + + strides +} + +pub(crate) fn elem_dtype() -> DType { + match E::cube_type().elem_type() { + ElemType::Float(kind) => match kind { + cubecl::ir::FloatKind::F64 => DType::F64, + cubecl::ir::FloatKind::F16 => DType::F16, + cubecl::ir::FloatKind::BF16 => DType::BF16, + cubecl::ir::FloatKind::F32 => DType::F32, + _ => todo!(), + }, + ElemType::Int(kind) => match kind { + cubecl::ir::IntKind::I64 => DType::I64, + cubecl::ir::IntKind::I32 => DType::I32, + cubecl::ir::IntKind::I16 => DType::I16, + cubecl::ir::IntKind::I8 => DType::I8, + }, + ElemType::UInt(kind) => match kind { + cubecl::ir::UIntKind::U64 => DType::U64, + cubecl::ir::UIntKind::U32 => DType::U32, + cubecl::ir::UIntKind::U16 => DType::U16, + cubecl::ir::UIntKind::U8 => DType::U8, + }, + ElemType::Bool => DType::Bool, + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/base.rs new file mode 100644 index 0000000..48480ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/base.rs @@ -0,0 +1,6 @@ +/// The element type ID to be used for dynamic element type while expanding a fused kernel. +pub(crate) const DYN_ELEM_ID: u8 = u8::MAX; +/// The element type ID to be used for the quantization store element type while expanding a fused kernel. +pub(crate) const Q_STORE_DYN_ELEM_ID: u8 = u8::MAX - 1; +/// The element type ID to be used for the quantization param element type while expanding a fused kernel. +pub(crate) const Q_PARAM_DYN_ELEM_ID: u8 = u8::MAX - 2; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/io.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/io.rs new file mode 100644 index 0000000..7129d82 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/io.rs @@ -0,0 +1,792 @@ +//! This module declares input-output primitives to read and write values during kernel expansion. +use super::{DYN_ELEM_ID, ir::*, tensor::GlobalTensor}; +use burn_std::quantization::QuantScheme; +use cubecl::quant::scheme::QuantLevel; +use cubecl::{ + intrinsic, + ir::{ExpandElement, Variable}, + prelude::*, + std::{FastDivmod, tensor::View}, +}; +use cubek::quantization::layout::{BlockScaledLayout, PerTensorLayout, ScalesLayout}; +use serde::{Deserialize, Serialize}; + +/// Define how a tensor might be transformed at runtime. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] +pub enum Transform { + /// A reshape operation has been registered on a tensor. + /// + /// This enum entry contains a sequence of [arguments](FuseArg) that points to global scalars representing the + /// new shape for the current tensor. + Reshape(Vec), + /// Two axes have been swapped on a tensor. + /// + /// The enum entry contains those two axes. + SwapDims(usize, usize), +} + +/// Reads the value from the [arg](FuseArg) and cast it to the generic cube primitive. +/// +/// # Notes +/// +/// The [global arguments](GlobalArgs) for both inputs and outputs as well as the +/// [local arguments](LocalArgs) need to be passed to this function. +/// +/// This is because the [argument](FuseArg) might point to a global input, output or local variable +/// created during kernel expansion. +#[cube] +pub fn read( + inputs: &GlobalArgs, + outputs: &GlobalArgs, + locals: &LocalArgs, + ref_pos: usize, + #[comptime] arg: FuseArg, + #[comptime] config: &FuseBlockConfig, +) -> Line { + match arg { + FuseArg::Input(pos, _precision, layout) => { + let global = inputs.tensors.index(pos); + let line_size = global.tensor.line_size(); + + if comptime![!global.broadcasted && line_size != config.width] { + read_input_aligned(inputs, locals, pos, ref_pos, layout, config, None) + } else { + read_input(inputs, locals, pos, ref_pos, layout, config, None) + } + } + FuseArg::MultiBlockLocal(key, _) | FuseArg::MultiBlockGlobal(key, _) => { + Line::cast_from(outputs.variables.read(key)) + } + FuseArg::Output(pos, _precision, layout) => { + read_output(inputs, outputs, locals, pos, ref_pos, layout, config) + } + FuseArg::BlockLocal { pos, ty } => match comptime![ty] { + FuseType::F64 => Line::cast_from(locals.l_f64.find(pos)), + FuseType::F32 | FuseType::Flex32 => Line::cast_from(locals.l_f32.find(pos)), + FuseType::F16 => Line::cast_from(locals.l_f16.find(pos)), + FuseType::BF16 => Line::cast_from(locals.l_bf16.find(pos)), + FuseType::U64 => Line::cast_from(locals.l_u64.find(pos)), + FuseType::U32 => Line::cast_from(locals.l_u32.find(pos)), + FuseType::U16 => Line::cast_from(locals.l_u16.find(pos)), + FuseType::U8 => Line::cast_from(locals.l_u8.find(pos)), + FuseType::I64 => Line::cast_from(locals.l_i64.find(pos)), + FuseType::I32 => Line::cast_from(locals.l_i32.find(pos)), + FuseType::I16 => Line::cast_from(locals.l_i16.find(pos)), + FuseType::I8 => Line::cast_from(locals.l_i8.find(pos)), + FuseType::Bool => Line::cast_from(locals.l_bool.find(pos)), + }, + FuseArg::Scalar(..) => { + let scalar = read_scalar::(inputs, arg); + Line::new(scalar) + } + FuseArg::ScalarShape(_) => { + let scalar = read_scalar_shape(inputs, arg); + Line::cast_from(scalar) + } + FuseArg::Literal(val, _precision) => Line::new(from_const_int::(val)), + FuseArg::InputReshaped { + original, + shape, + broadcasted, + } => match comptime![original.as_ref().clone()] { + FuseArg::Input(pos, _precision, layout) => { + let global = inputs.tensors.index(pos); + let line_size = global.tensor.line_size(); + + if comptime![!broadcasted && line_size != config.width] { + read_input_aligned( + inputs, + locals, + pos, + ref_pos, + layout, + config, + comptime![Some(Transform::Reshape(shape))], + ) + } else { + read_input( + inputs, + locals, + pos, + ref_pos, + layout, + config, + comptime![Some(Transform::Reshape(shape))], + ) + } + } + _ => comptime![panic!("Only input can be reshaped")], + }, + FuseArg::InputSwapDims { + original, + dims, + broadcasted, + } => match comptime![original.as_ref().clone()] { + FuseArg::Input(pos, _precision, layout) => { + let global = inputs.tensors.index(pos); + let line_size = global.tensor.line_size(); + + if comptime![!broadcasted && line_size != config.width] { + read_input_aligned( + inputs, + locals, + pos, + ref_pos, + layout, + config, + comptime![Some(Transform::SwapDims(dims.0, dims.1))], + ) + } else { + read_input( + inputs, + locals, + pos, + ref_pos, + layout, + config, + comptime![Some(Transform::SwapDims(dims.0, dims.1))], + ) + } + } + _ => comptime![panic!("Only input can be swapped dims")], + }, + } +} + +/// Computes the offset for the current global tensor with a quantized layout. +/// +/// The offset can be used to fetch the correct data from the quantized tensor as if it was in a +/// linear contiguous format. +#[cube] +fn index_offset_with_quant_layout( + tensor: &GlobalTensor, + locals: &LocalArgs, + index: usize, + #[comptime] rank: usize, + #[comptime] scheme: QuantScheme, +) -> usize { + let (start, end) = (0, rank - 1); + let num_quants = scheme.num_quants(); + + let offset_ref = index * locals.ref_line_size; + let mut offset = 0; + + #[unroll] + for i in start..end { + let ogwl = offset_ref / locals.ref_strides[i]; + offset += ogwl % tensor.tensor.shape(i) * tensor.tensor.stride(i); + } + + // Handle packed representation in last dim + let ogwl = offset_ref / locals.ref_strides[end]; + let shape_last = tensor.tensor.shape(end).div_ceil(num_quants); + let stride_last = tensor.tensor.stride(end); + offset += (ogwl.div_ceil(num_quants)) % shape_last * stride_last; + + offset / tensor.tensor.line_size() +} + +/// Reads a global quantized tensor at the given position. +/// +/// # Notes +/// +/// The values returned in the [Line] are not dequantized. +#[cube] +pub fn read_quantized( + inputs: &GlobalArgs, + locals: &LocalArgs, + ref_pos: usize, + #[comptime] arg: FuseArg, + #[comptime] config: &FuseBlockConfig, + #[comptime] scheme: QuantScheme, +) -> Line { + match arg { + FuseArg::Input(pos, _precision, _layout) => { + let global = inputs.tensors.index(pos); + + let offset = + index_offset_with_quant_layout(global, locals, ref_pos, config.rank, scheme); + let val = global.tensor[offset]; + Line::cast_from(val) + } + _ => panic!("Not supported"), + } +} + +/// Reads a global scalar. +#[cube] +pub fn read_scalar(inputs: &GlobalArgs, #[comptime] arg: FuseArg) -> C { + match arg { + FuseArg::Scalar(pos, _precision) => { + let scalar = inputs.scalars.index(pos); + scalar.get::() + } + _ => comptime![panic!("Not a scalar")], + } +} + +/// Reads a global scalar that is used as a reshape position. +#[cube] +pub fn read_scalar_shape(inputs: &GlobalArgs, #[comptime] arg: FuseArg) -> usize { + match arg { + FuseArg::ScalarShape(pos) => inputs.reshapes[pos], + _ => comptime![panic!("Not a scalar shape")], + } +} + +/// Reads an input tensor. +#[cube] +pub fn read_input( + inputs: &GlobalArgs, + locals: &LocalArgs, + #[comptime] pos: usize, + ref_pos: usize, + #[comptime] layout: LayoutInfo, + #[comptime] config: &FuseBlockConfig, + #[comptime] transform: Option, +) -> Line { + let tensor = inputs.tensors.index(pos); + let offset = match layout { + LayoutInfo::SameAsRef => ref_pos, + LayoutInfo::IsRef => ref_pos, + LayoutInfo::Unknown => get_offset(inputs, locals, tensor, ref_pos, None, config, transform), + }; + Line::cast_from(tensor.tensor[offset]) +} + +/// Returns a slice of data in the asked precision of the input tensor at the given position. +#[cube] +pub fn read_input_window( + inputs: &GlobalArgs, + #[comptime] pos: usize, + start: usize, + end: usize, +) -> Slice { + let tensor = inputs.tensors.index(pos); + let slice = tensor.tensor.slice(start, end); + slice.downcast() +} + +/// Returns the input as a slice. +#[cube] +pub fn input_as_slice(inputs: &GlobalArgs, #[comptime] pos: usize) -> Slice { + let tensor = inputs.tensors.index(pos); + let slice = tensor.tensor.to_slice(); + slice.downcast() +} + +/// Returns the input tensor as a quantized scale view. +#[cube] +pub fn input_as_scales_view( + inputs: &GlobalArgs, + #[comptime] pos: usize, + #[comptime] tensor_pos: usize, + #[comptime] level: QuantLevel, + #[comptime] config: &FuseBlockConfig, +) -> View { + set_polyfill_typed::>(); + let tensor = inputs.tensors.index(tensor_pos); + let scales = inputs.tensors.index(pos); + let tensor_len = tensor.tensor.len(); + let rank = config.rank; + let layout = match level { + QuantLevel::Tensor => ScalesLayout::new_PerTensor(PerTensorLayout::new(tensor_len)), + QuantLevel::Block(block_size) => { + let block_size = comptime![block_size.to_dim_vec(rank)]; + let mut tensor_shape = Sequence::new(); + let mut scales_strides = Sequence::new(); + #[unroll] + for i in 0..rank { + tensor_shape.push(FastDivmod::new_Fallback(tensor.tensor.shape(i))); + scales_strides.push(scales.tensor.stride(i)); + } + let line_size = scales.tensor.line_size(); + let layout = BlockScaledLayout::new( + tensor_shape, + tensor_len, + scales_strides, + block_size, + line_size, + ); + ScalesLayout::new_BlockScaled(layout) + } + }; + View::new::, usize>(&scales.tensor.to_slice().downcast(), layout) +} + +/// Reads the input tensor aligned. +#[cube] +pub fn read_input_aligned( + inputs: &GlobalArgs, + locals: &LocalArgs, + #[comptime] pos: usize, + ref_pos: usize, + #[comptime] layout: LayoutInfo, + #[comptime] config: &FuseBlockConfig, + #[comptime] transform: Option, +) -> Line { + let mut result: Line = Line::::empty(config.width); + let tensor = inputs.tensors.index(pos); + + match transform.clone() { + Some(Transform::Reshape(shape)) => { + // Very brute force, not really efficient, but not easy to optimize and not a very + // frequent workflow. + let ref_pos = ref_pos * config.width; + #[unroll] + for i in 0..config.width { + let index = reshaped_index( + inputs, + locals, + ref_pos + i, + config.rank, + comptime![shape.clone()], + ); + let index = reshaped_index_to_original_index(&tensor.tensor, index, config.rank); + result[i] = C::cast_from(tensor.tensor[index][0]) + } + } + Some(Transform::SwapDims(dim1, dim2)) => { + let offset = + get_offset_aligned(inputs, locals, tensor, ref_pos, layout, config, transform); + let i = comptime![swap_dims_transform(config.rank - 1, (dim1, dim2))]; + let stride = tensor.tensor.stride(i); + + #[unroll] + for i in 0..config.width { + let index = offset + i * stride; + result[i] = C::cast_from(tensor.tensor[index][0]) + } + } + None => { + let offset = + get_offset_aligned(inputs, locals, tensor, ref_pos, layout, config, transform); + let stride = tensor.tensor.stride(config.rank - 1); + #[unroll] + for i in 0..config.width { + let index = offset + i * stride; + result[i] = C::cast_from(tensor.tensor[index][0]) + } + } + } + + result +} + +/// Computes the offset of the given [GlobalTensor] at on the reference position with a linear +/// layout. +#[cube] +pub fn get_offset_aligned( + inputs: &GlobalArgs, + locals: &LocalArgs, + tensor: &GlobalTensor, + ref_pos: usize, + #[comptime] layout: LayoutInfo, + #[comptime] config: &FuseBlockConfig, + #[comptime] transform: Option, +) -> usize { + match layout { + LayoutInfo::SameAsRef | LayoutInfo::IsRef => { + (ref_pos * locals.ref_line_size) / tensor.tensor.line_size() + } + LayoutInfo::Unknown => get_offset( + inputs, + locals, + tensor, + ref_pos, + None, + config, + comptime!(transform.clone()), + ), + } +} + +/// Reads an output tensor. +#[cube] +pub fn read_output( + inputs: &GlobalArgs, + outputs: &GlobalArgs, + locals: &LocalArgs, + #[comptime] pos: usize, + ref_pos: usize, + #[comptime] layout: LayoutInfo, + #[comptime] config: &FuseBlockConfig, +) -> Line { + let tensor = outputs.tensors.index(pos); + let offset = match layout { + LayoutInfo::SameAsRef => ref_pos, + LayoutInfo::IsRef => ref_pos, + LayoutInfo::Unknown => get_offset(inputs, locals, tensor, ref_pos, None, config, None), + }; + Line::cast_from(tensor.tensor[offset]) +} + +#[cube] +/// Write the given value at the [arg](Arg) position. +pub fn write( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + ref_pos: usize, + value: Line, + #[comptime] arg: FuseArg, + #[comptime] config: &FuseBlockConfig, +) { + match arg { + FuseArg::Output(pos, precision, layout) => { + let tensor = outputs.tensors.index(pos); + let offset = match layout { + LayoutInfo::SameAsRef => ref_pos, + LayoutInfo::IsRef => ref_pos, + LayoutInfo::Unknown => { + get_offset(inputs, locals, tensor, ref_pos, None, config, None) + } + }; + let tensor = outputs.tensors.index_mut(pos); + set_polyfill::>(comptime![precision.into_type()]); + + tensor.tensor[offset] = Line::cast_from(value); + } + FuseArg::BlockLocal { .. } => write_scalar::(locals, value, arg), + FuseArg::MultiBlockLocal(key, _) | FuseArg::MultiBlockGlobal(key, _) => { + outputs.variables.write(key, Line::cast_from(value)) + } + _ => comptime![panic!("Can't write into inputs and scalars")], + } +} + +#[cube] +/// Write the given value at the [arg](Arg) position. +pub fn write_scalar( + locals: &mut LocalArgs, + value: Line, + #[comptime] arg: FuseArg, +) { + match arg { + FuseArg::BlockLocal { pos, ty } => match comptime![ty] { + FuseType::F64 => locals.l_f64.insert(pos, Line::cast_from(value)), + FuseType::F32 | FuseType::Flex32 => locals.l_f32.insert(pos, Line::cast_from(value)), + FuseType::F16 => locals.l_f16.insert(pos, Line::cast_from(value)), + FuseType::BF16 => locals.l_bf16.insert(pos, Line::cast_from(value)), + FuseType::U64 => locals.l_u64.insert(pos, Line::cast_from(value)), + FuseType::U32 => locals.l_u32.insert(pos, Line::cast_from(value)), + FuseType::U16 => locals.l_u16.insert(pos, Line::cast_from(value)), + FuseType::U8 => locals.l_u8.insert(pos, Line::cast_from(value)), + FuseType::I64 => locals.l_i64.insert(pos, Line::cast_from(value)), + FuseType::I32 => locals.l_i32.insert(pos, Line::cast_from(value)), + FuseType::I16 => locals.l_i16.insert(pos, Line::cast_from(value)), + FuseType::I8 => locals.l_i8.insert(pos, Line::cast_from(value)), + FuseType::Bool => locals.l_bool.insert(pos, Line::cast_from(value)), + }, + _ => comptime![panic!("Can't write into something else than scalars")], + } +} + +#[cube] +pub(crate) fn global_offset( + inputs: &GlobalArgs, + outputs: &GlobalArgs, + locals: &LocalArgs, + index: usize, + #[comptime] arg: FuseArg, + #[comptime] range: Option<(usize, usize)>, + #[comptime] config: &FuseBlockConfig, +) -> usize { + match arg { + FuseArg::Input(pos, _precision, _layout) => { + let tensor = inputs.tensors.index(pos); + get_offset(inputs, locals, tensor, index, range, config, None) + } + FuseArg::Output(pos, _precision, _layout) => { + let tensor = outputs.tensors.index(pos); + get_offset(inputs, locals, tensor, index, range, config, None) + } + _ => panic!("Only input and output tensors have global offset."), + } +} + +#[cube] +fn get_offset( + inputs: &GlobalArgs, + locals: &LocalArgs, + tensor: &GlobalTensor, + ref_pos: usize, + #[comptime] range: Option<(usize, usize)>, + #[comptime] config: &FuseBlockConfig, + #[comptime] transform: Option, +) -> usize { + index_offset_with_layout( + inputs, + tensor, + locals, + ref_pos, + range, + config.rank, + transform, + ) +} + +#[cube] +/// Gets the line size for a global tensor. +pub fn global_line_size(global: &GlobalArgs, #[comptime] pos: usize) -> comptime_type!(LineSize) { + let tensor = global.tensors.index(pos); + tensor.tensor.line_size() +} + +#[cube] +/// Gets the rank for a global tensor. +pub fn global_rank(global: &GlobalArgs, #[comptime] pos: usize) -> usize { + let tensor = global.tensors.index(pos); + tensor.tensor.rank() +} + +#[cube] +/// Gets the length for a global tensor. +pub fn global_len(global: &GlobalArgs, #[comptime] pos: usize) -> usize { + let tensor = global.tensors.index(pos); + tensor.tensor.len() +} + +#[cube] +/// Gets the buffer length for a global tensor. +pub fn global_buffer_len(global: &GlobalArgs, #[comptime] pos: usize) -> usize { + let tensor = global.tensors.index(pos); + tensor.tensor.buffer_len() +} + +#[cube] +/// Gets the reference tensor length. +pub fn ref_len( + inputs: &GlobalArgs, + outputs: &GlobalArgs, + locals: &LocalArgs, + #[comptime] config: &FuseBlockConfig, +) -> usize { + match config.ref_layout.clone() { + RefLayout::Concrete(arg) => match comptime![arg] { + FuseArg::Input(index, _, _) => global_len(inputs, index), + FuseArg::Output(index, _, _) => global_len(outputs, index), + _ => panic!("Invalid concrete ref layout."), + }, + RefLayout::Virtual(..) => num_elements(locals, config), + } +} + +#[cube] +/// Gets the reference buffer tensor length. +pub fn ref_buffer_len( + inputs: &GlobalArgs, + outputs: &GlobalArgs, + locals: &LocalArgs, + #[comptime] config: &FuseBlockConfig, +) -> usize { + match config.ref_layout.clone() { + RefLayout::Concrete(arg) => match comptime![arg] { + FuseArg::Input(index, _, _) => global_buffer_len(inputs, index), + FuseArg::Output(index, _, _) => global_buffer_len(outputs, index), + _ => panic!("Invalid concrete ref layout."), + }, + RefLayout::Virtual(VirtualLayout::SwapDims(arg, ..)) => match arg { + FuseArg::Input(index, _, _) => global_buffer_len(inputs, index), + FuseArg::Output(index, _, _) => global_buffer_len(outputs, index), + _ => panic!("Invalid concrete ref layout."), + }, + RefLayout::Virtual(VirtualLayout::Reshaped { .. }) => num_elements(locals, config), + RefLayout::Virtual(VirtualLayout::Shape(..)) => num_elements(locals, config), + RefLayout::Virtual(VirtualLayout::Runtime { .. }) => num_elements(locals, config), + } +} + +#[cube] +/// Gets the reference number of elements. +pub fn num_elements(locals: &LocalArgs, #[comptime] config: &FuseBlockConfig) -> usize { + let mut length = 1; + + for i in 0..config.rank { + length *= locals.ref_shape[i]; + } + + length +} + +#[cube] +/// Gets the reference axis shape. +pub fn ref_shape(locals: &LocalArgs, axis: usize) -> usize { + locals.ref_shape[axis] +} + +#[cube] +/// Gets the reference axis stride. +pub fn ref_stride(locals: &LocalArgs, axis: usize) -> usize { + locals.ref_strides[axis] +} + +#[cube] +/// Gets the reference line size. +pub fn ref_line_size(locals: &LocalArgs) -> comptime_type!(LineSize) { + comptime![locals.ref_line_size] +} + +#[cube] +/// Gets the given tensor axis shape. +pub fn global_shape(global: &GlobalArgs, axis: usize, #[comptime] pos: usize) -> usize { + let tensor = global.tensors.index(pos); + tensor.tensor.shape(axis) +} + +#[cube] +/// Gets the given tensor axis stride. +pub fn global_stride(global: &GlobalArgs, dim: usize, #[comptime] pos: usize) -> usize { + let tensor = global.tensors.index(pos); + tensor.tensor.stride(dim) +} + +#[cube] +fn index_offset_with_layout( + inputs: &GlobalArgs, + tensor: &GlobalTensor, + locals: &LocalArgs, + index: usize, + #[comptime] range: Option<(usize, usize)>, + #[comptime] rank: usize, + #[comptime] transform: Option, +) -> usize { + match comptime![transform.clone()] { + Some(Transform::Reshape(shape)) => { + comptime![assert!( + range.is_none(), + "Can't get a range on a reshaped tensor." + )]; + + let index = index * locals.ref_line_size; + let index = reshaped_index(inputs, locals, index, rank, shape); + reshaped_index_to_original_index(&tensor.tensor, index, rank) + } + Some(Transform::SwapDims(dim1, dim2)) => { + let (start, end) = comptime! {match range { + Some(range) => range, + None => (0, rank), + }}; + + let offset_ref = index * locals.ref_line_size; + let mut offset = 0; + + #[unroll] + for i in start..end { + let index = comptime![swap_dims_transform(i, (dim1, dim2))]; + let ogwl = offset_ref / locals.ref_strides[i]; + offset += ogwl % tensor.tensor.shape(index) * tensor.tensor.stride(index); + } + + offset / tensor.tensor.line_size() + } + None => { + let (start, end) = comptime! {match range { + Some(range) => range, + None => (0, rank), + }}; + + let offset_ref = index * locals.ref_line_size; + let mut offset = 0; + + #[unroll] + for i in start..end { + let ogwl = offset_ref / locals.ref_strides[i]; + offset += ogwl % tensor.tensor.shape(i) * tensor.tensor.stride(i); + } + + offset / tensor.tensor.line_size() + } + } +} + +pub(crate) fn swap_dims_transform(i: usize, dims: (usize, usize)) -> usize { + if i == dims.0 { + dims.1 + } else if i == dims.1 { + dims.0 + } else { + i + } +} + +#[cube] +#[allow(clippy::clone_on_copy)] +/// The index the input tensor would be at if it was contiguous. +fn reshaped_index( + inputs: &GlobalArgs, + locals: &LocalArgs, + index: usize, + #[comptime] rank: usize, + #[comptime] shape: Vec, +) -> usize { + let mut offset = 0; + let mut stride_curr = 1; + + #[unroll] + for r in 0..rank { + let i = reverse_index(rank, r).comptime(); + let arg = shape[i].clone(); + let shape_i = read_scalar_shape(inputs, arg); + let ogwl = index / locals.ref_strides[i]; + + offset += ogwl % shape_i * stride_curr; + + stride_curr *= shape_i; + } + + offset +} + +#[allow(unreachable_code)] +#[cube] +#[allow(clippy::clone_on_copy)] +fn reshaped_index_to_original_index( + original: &Tensor>, + index_reshaped: usize, + #[comptime] rank: usize, +) -> usize { + let mut remaining = index_reshaped; + let mut offset = 0; + + #[unroll] + for r in 0..rank { + let i = reverse_index(rank, r); + let shape = original.shape(i); + let stride = original.stride(i); + + let coordinate = remaining % shape; + + remaining /= shape; + offset += coordinate * stride; + } + + offset / original.line_size() +} + +#[cube] +#[allow(unused_variables)] +pub(crate) fn reverse_index( + #[comptime] rank: usize, + #[comptime] iter: usize, +) -> comptime_type!(usize) { + rank - iter - 1 +} + +/// Generic way to construct any [`CubePrimitive`] from an int. Used for fusion. +#[allow(unused_variables)] +#[cube] +fn from_const_int(#[comptime] value: usize) -> C { + intrinsic!(|scope| { + ExpandElement::Plain(Variable::constant(value.into(), C::as_type(scope))).into() + }) +} + +#[cube] +#[allow(clippy::extra_unused_type_parameters)] +fn set_polyfill_typed() { + intrinsic!(|scope| { + let elem_type = C::as_type(scope); + set_polyfill::expand::(scope, elem_type); + }) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/ir.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/ir.rs new file mode 100644 index 0000000..240efac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/ir.rs @@ -0,0 +1,917 @@ +use super::tensor::GlobalTensor; +use crate::engine::codegen::DYN_ELEM_ID; +use burn_std::{ + DType, Shape, Strides, bf16, f16, + quantization::{QuantScheme, QuantStore, QuantValue}, + strides, +}; +use core::fmt::Display; +use cubecl::{ + ir::{ElemType, FloatKind, IntKind, StorageType, UIntKind}, + prelude::*, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] +/// Argument to a [fuse operation](FuseOp). +pub enum FuseArg { + /// A readonly input tensor. + Input(usize, FuseType, LayoutInfo), + /// A readwrite output tensor. + Output(usize, FuseType, LayoutInfo), + /// A temporary local variable within a single [block](FuseBlockConfig). + BlockLocal { + /// The position of the current variable relative to all local variables within a single block. + pos: usize, + /// The type of the current variable. + ty: FuseType, + }, + /// A variable shared between multiple [block](FuseBlockConfig) that must have a compatible + /// scope. + MultiBlockLocal(MultiBlockPos, FuseType), + /// A variable shared between multiple [blocks](FuseBlockConfig) within a global accessible + /// scope. + MultiBlockGlobal(MultiBlockPos, FuseType), + /// A global scalar. + Scalar(usize, FuseType), + /// A global scalar used in a reshape operation. + /// + /// This is not a scalar defined by a user for computation, but a scalar defined as part of + /// a reshape operation. + ScalarShape(usize), + /// Only constant that can be encoded into an u32 can be used as literal. + Literal(usize, FuseType), + /// A readonly input tensor that is reshaped. + InputReshaped { + original: Box, + shape: Vec, + broadcasted: bool, + }, + /// A readonly input tensor with swapped dimensions. + InputSwapDims { + original: Box, + dims: (usize, usize), + broadcasted: bool, + }, +} + +/// Metadata of a variable shared between blocks. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] +pub struct MultiBlockPos { + /// The block position in all blocks included in a fused trace. + pub block_pos: usize, + /// The [FuseArg::BlockLocal] position in the block where the variable is first initialized. + pub block_local_pos: usize, +} + +#[derive( + CubeType, Clone, Copy, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord, +)] +/// Layout information. +pub enum LayoutInfo { + /// The layout if the same as the reference. + SameAsRef, + /// The reference layout. + IsRef, + /// The layout if unknown. + Unknown, +} + +impl FuseArg { + pub fn precision(&self) -> FuseType { + *match self { + FuseArg::Input(_, p, _) => p, + FuseArg::BlockLocal { ty, .. } => ty, + FuseArg::MultiBlockLocal(_, p) => p, + FuseArg::MultiBlockGlobal(_, p) => p, + FuseArg::Output(_, p, _) => p, + FuseArg::Scalar(_, p) => p, + FuseArg::Literal(_, p) => p, + FuseArg::ScalarShape(_) => return FuseType::U32, + FuseArg::InputReshaped { original, .. } => return original.precision(), + FuseArg::InputSwapDims { original, .. } => return original.precision(), + } + } +} + +impl CubeType for FuseArg { + type ExpandType = Self; +} + +impl IntoMut for FuseArg { + fn into_mut(self, _context: &mut Scope) -> Self { + self + } +} + +impl IntoRuntime for FuseArg { + fn __expand_runtime_method(self, _context: &mut Scope) -> Self::ExpandType { + self + } +} + +impl CubeDebug for FuseArg {} + +#[derive(CubeType, Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +/// Operations that can be executed and fused automatically using a fuse-on-read and/or +/// fuse-on-write strategy. +pub enum FuseOp { + Add(BinaryFuseArgs), + Sub(BinaryFuseArgs), + Mul(BinaryFuseArgs), + Div(BinaryFuseArgs), + Powf(BinaryFuseArgs), + Abs(UnaryFuseArgs), + Exp(UnaryFuseArgs), + Log(UnaryFuseArgs), + Log1p(UnaryFuseArgs), + Cos(UnaryFuseArgs), + Sin(UnaryFuseArgs), + Tanh(UnaryFuseArgs), + Erf(UnaryFuseArgs), + Sqrt(UnaryFuseArgs), + Recip(UnaryFuseArgs), + Assign(UnaryFuseArgs), + Equal(BinaryFuseArgs), + Lower(BinaryFuseArgs), + Greater(BinaryFuseArgs), + LowerEqual(BinaryFuseArgs), + Rem(BinaryFuseArgs), + GreaterEqual(BinaryFuseArgs), + Clamp { + input: FuseArg, + min: FuseArg, + max: FuseArg, + out: FuseArg, + }, + ConditionalAssign { + cond: FuseArg, + lhs: FuseArg, + rhs: FuseArg, + out: FuseArg, + }, + Gather { + input: FuseArg, + indices: FuseArg, + output: FuseArg, + dim: usize, + }, + Select { + input: FuseArg, + indices: FuseArg, + output: FuseArg, + dim: usize, + }, + Dequantize { + values: FuseArg, + params: FuseArg, + output: FuseArg, + scheme: QuantSchemeFuse, + }, +} + +impl Display for FuseOp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FuseOp::Add(args) => write!(f, "{} = {} + {}", args.out, args.lhs, args.rhs), + FuseOp::Sub(args) => write!(f, "{} = {} - {}", args.out, args.lhs, args.rhs), + FuseOp::Mul(args) => write!(f, "{} = {} * {}", args.out, args.lhs, args.rhs), + FuseOp::Div(args) => write!(f, "{} = {} / {}", args.out, args.lhs, args.rhs), + FuseOp::Powf(args) => write!(f, "{} = powf({}, {})", args.out, args.lhs, args.rhs), + FuseOp::Abs(args) => write!(f, "{} = abs({})", args.out, args.input), + FuseOp::Exp(args) => write!(f, "{} = exp({})", args.out, args.input), + FuseOp::Log(args) => write!(f, "{} = log({})", args.out, args.input), + FuseOp::Log1p(args) => write!(f, "{} = log1p({})", args.out, args.input), + FuseOp::Cos(args) => write!(f, "{} = cos({})", args.out, args.input), + FuseOp::Sin(args) => write!(f, "{} = sin({})", args.out, args.input), + FuseOp::Tanh(args) => write!(f, "{} = tanh({})", args.out, args.input), + FuseOp::Erf(args) => write!(f, "{} = erf({})", args.out, args.input), + FuseOp::Sqrt(args) => write!(f, "{} = sqrt({})", args.out, args.input), + FuseOp::Recip(args) => write!(f, "{} = recip({})", args.out, args.input), + FuseOp::Assign(args) => write!(f, "{} = {}", args.out, args.input), + FuseOp::Equal(args) => write!(f, "{} = {} == {}", args.out, args.lhs, args.rhs), + FuseOp::Lower(args) => write!(f, "{} = {} < {}", args.out, args.lhs, args.rhs), + FuseOp::Greater(args) => write!(f, "{} = {} > {}", args.out, args.lhs, args.rhs), + FuseOp::LowerEqual(args) => write!(f, "{} = {} <= {}", args.out, args.lhs, args.rhs), + FuseOp::Rem(args) => write!(f, "{} = {} % {}", args.out, args.lhs, args.rhs), + FuseOp::GreaterEqual(args) => write!(f, "{} = {} >= {}", args.out, args.lhs, args.rhs), + FuseOp::Clamp { + input, + min, + max, + out, + } => write!(f, "{} = clamp({}, min={}, max={})", out, input, min, max), + FuseOp::ConditionalAssign { + cond, + lhs, + rhs, + out, + } => write!( + f, + "{} = select(cond={}, lhs={}, rhs={})", + out, cond, lhs, rhs + ), + FuseOp::Gather { + input, + indices, + output, + dim, + } => write!( + f, + "{} = gather(input={}, indices={}, dim={})", + output, input, indices, dim + ), + FuseOp::Select { + input, + indices, + output, + dim, + } => write!( + f, + "{} = select(input={}, indices={}, dim={})", + output, input, indices, dim + ), + FuseOp::Dequantize { + values, + params, + output, + scheme: _, + } => write!( + f, + "{} = dequantize(values={}, params={})", + output, values, params + ), + } + } +} + +#[derive( + CubeType, CubeLaunch, Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord, +)] +pub struct QuantSchemeFuse { + #[cube(comptime)] + pub(crate) scheme: QuantScheme, +} + +impl FuseOp { + /// Element type used for the computation. + pub(crate) fn cmp_elem(&self) -> ElemType { + match self { + FuseOp::Add(op) => op.lhs.precision().into_elem(), + FuseOp::Sub(op) => op.lhs.precision().into_elem(), + FuseOp::Mul(op) => op.lhs.precision().into_elem(), + FuseOp::Div(op) => op.lhs.precision().into_elem(), + FuseOp::Powf(op) => op.lhs.precision().into_elem(), + FuseOp::Abs(op) => op.out.precision().into_elem(), + FuseOp::Exp(op) => op.out.precision().into_elem(), + FuseOp::Log(op) => op.out.precision().into_elem(), + FuseOp::Log1p(op) => op.out.precision().into_elem(), + FuseOp::Cos(op) => op.out.precision().into_elem(), + FuseOp::Sin(op) => op.out.precision().into_elem(), + FuseOp::Tanh(op) => op.out.precision().into_elem(), + FuseOp::Erf(op) => op.out.precision().into_elem(), + FuseOp::Recip(op) => op.out.precision().into_elem(), + FuseOp::Sqrt(op) => op.out.precision().into_elem(), + FuseOp::Assign(op) => op.out.precision().into_elem(), + FuseOp::Equal(op) => op.lhs.precision().into_elem(), + FuseOp::Lower(op) => op.lhs.precision().into_elem(), + FuseOp::Greater(op) => op.lhs.precision().into_elem(), + FuseOp::LowerEqual(op) => op.lhs.precision().into_elem(), + FuseOp::GreaterEqual(op) => op.lhs.precision().into_elem(), + FuseOp::ConditionalAssign { out, .. } => out.precision().into_elem(), + FuseOp::Gather { output, .. } => output.precision().into_elem(), + FuseOp::Select { output, .. } => output.precision().into_elem(), + FuseOp::Dequantize { output, .. } => output.precision().into_elem(), + FuseOp::Rem(op) => op.out.precision().into_elem(), + FuseOp::Clamp { out, .. } => out.precision().into_elem(), + } + } + + /// Element type used for the computation. + pub(crate) fn cmp_type(&self) -> StorageType { + self.cmp_elem().into() + } +} + +#[derive(CubeType, CubeLaunch, Default, Clone)] +/// Global arguments that are used for fusing [element wise operations](ElemTypewiseOp). +pub struct GlobalArgs { + /// Tensors that are stored in global memory. + pub tensors: Sequence, + /// Scalars that are stored in global memory. + pub scalars: Sequence, + /// To be used to perform reshape inside a fused kernel. + pub reshapes: Sequence, + /// When there are no metadata as a reference layout, we provide runtime shape/strides in this + /// sequence instead. + pub runtime_layouts: Sequence, + /// Variables shared between blocks. + pub variables: MultiBlockVariables, +} + +impl<'a, R: Runtime> GlobalArgsLaunch<'a, R> { + pub fn required_address_type(&self) -> AddressType { + self.tensors + .values + .iter() + .map(|it| it.address_type) + .max() + .unwrap_or_default() + } +} + +/// Variables shared between blocks. +#[derive(CubeType, Default, Clone)] +pub struct MultiBlockVariables { + variables: Registry>>>>, +} + +#[cube] +impl MultiBlockVariables { + /// Initializes the variable with the given key and line size. + /// + /// # Notes + /// + /// The type of [`NumericExpand`] must be set before calling this function. + pub fn init(&mut self, #[comptime] key: MultiBlockPos, #[comptime] line_size: usize) { + let mut registers = Registry::< + usize, + Registry>>>, + >::find_or_default::(&mut self.variables, key.block_pos); + let cell = RuntimeCell::new(Line::empty(line_size)); + registers.insert(key.block_local_pos, cell); + } + + /// Read the variable using the provided key. + /// + /// # Notes + /// + /// The variable must be initialized. + pub fn read(&self, #[comptime] key: MultiBlockPos) -> Line> { + let registers = self.variables.find(key.block_pos); + let cell = registers.find(key.block_local_pos); + cell.read() + } + + /// Write to the variable using the provided key and value. + /// + /// # Notes + /// + /// The variable must be initialized. + pub fn write( + &mut self, + #[comptime] key: MultiBlockPos, + value: Line>, + ) { + let registers = self.variables.find(key.block_pos); + // Try find for local(visibility) registers. + let cell = registers.find(key.block_local_pos); + cell.store(value); + } +} + +// Because we only create it DURING compilation, not as a real launch arg. +unsafe impl Send for MultiBlockVariables {} +unsafe impl Sync for MultiBlockVariables {} + +impl LaunchArg for MultiBlockVariables { + type RuntimeArg<'a, R: Runtime> = (); + type CompilationArg = (); + + fn compilation_arg(_runtime_arg: &Self::RuntimeArg<'_, R>) -> Self::CompilationArg { + } + + fn expand( + _arg: &Self::CompilationArg, + _builder: &mut KernelBuilder, + ) -> ::ExpandType { + MultiBlockVariablesExpand { + variables: Default::default(), + } + } +} + +impl Default for GlobalArgsLaunch<'_, R> { + fn default() -> Self { + Self { + tensors: Default::default(), + scalars: Default::default(), + reshapes: Default::default(), + variables: Default::default(), + runtime_layouts: Default::default(), + _phantom_runtime: std::marker::PhantomData, + _phantom_a: std::marker::PhantomData, + } + } +} + +impl core::fmt::Debug for GlobalArgsLaunch<'_, R> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({:?})", self.tensors.values) + } +} + +impl GlobalArgsLaunch<'_, R> { + /// Get the shape of the given [argument](Arg). + /// + /// # Panics + /// + /// If the argument doesn't have an handle. + pub fn shape(&self, arg: &FuseArg) -> Shape { + match self.resolve_arg(arg) { + TensorArg::Handle { handle, .. } => handle.shape.into(), + TensorArg::Alias { .. } => panic!("Unsupported yet"), + } + } + + /// Shape used by the reference tensor. + pub fn shape_ref(&self, ref_layout: &RefLayout, rank: usize) -> Shape { + match ref_layout { + RefLayout::Concrete(arg) => self.shape(arg), + RefLayout::Virtual(layout) => match layout { + VirtualLayout::SwapDims(original, dims) => { + let mut shape = self.shape(original); + shape.swap(dims.0, dims.1); + shape + } + VirtualLayout::Reshaped { reshape_pos, .. } => { + let start = *reshape_pos * rank; + let end = start + rank; + self.reshapes.values[start..end] + .iter() + .map(|s| s.elem) + .collect() + } + VirtualLayout::Shape(original, _) => self.shape(original), + VirtualLayout::Runtime { pos } => { + let start = (*pos * 2) * rank; + let end = start + rank; + self.runtime_layouts.values[start..end] + .iter() + .map(|s| s.elem) + .collect() + } + }, + } + } + + /// Get the strides of the given [argument](Arg). + /// + /// # Panics + /// + /// If the argument doesn't have an handle. + pub fn strides(&self, arg: &FuseArg) -> Strides { + match self.resolve_arg(arg) { + TensorArg::Handle { handle, .. } => handle.strides.into(), + TensorArg::Alias { .. } => panic!("Unsupported yet"), + } + } + + pub fn strides_ref(&self, ref_layout: &RefLayout, rank: usize) -> Strides { + match ref_layout { + RefLayout::Concrete(arg) => self.strides(arg), + // When not concrete, we operate on the contiguous layout. + _ => { + let shape = self.shape_ref(ref_layout, rank); + let mut strides = strides![0; shape.len()]; + + let mut current = 1; + shape.iter().enumerate().rev().for_each(|(index, val)| { + strides[index] = current; + current *= val; + }); + + strides + } + } + } + + /// Get the line size of the given [argument](Arg). + /// + /// # Panics + /// + /// If the argument doesn't have an handle. + pub fn line_size(&self, arg: &FuseArg) -> LineSize { + match self.resolve_arg(arg) { + TensorArg::Handle { line_size, .. } => *line_size, + TensorArg::Alias { .. } => panic!("Unsupported yet"), + } + } + + /// Resolve the [argument](Arg) to a [tensor argument](TensorArg). + /// + /// # Panics + /// + /// If the argument isn't a global input or output tensor. + pub fn resolve_arg(&self, arg: &FuseArg) -> &TensorArg<'_, R> { + match arg { + FuseArg::Input(pos, _, _) => &self.tensors.values[*pos].tensor, + FuseArg::Output(pos, _, _) => &self.tensors.values[*pos].tensor, + other => panic!("Arg not found: {other:?}"), + } + } +} + +#[derive(CubeType, Clone)] +/// Keep track of all local variables that are used as argument in fused +/// [element wise operations](ElemwiseOp). +pub struct LocalArgs { + pub l_f64: Registry>, + pub l_f32: Registry>, + pub l_f16: Registry>, + pub l_bf16: Registry>, + pub l_i64: Registry>, + pub l_i32: Registry>, + pub l_i16: Registry>, + pub l_i8: Registry>, + pub l_u64: Registry>, + pub l_u32: Registry>, + pub l_u16: Registry>, + pub l_u8: Registry>, + pub l_bool: Registry>, + pub ref_shape: Slice, + pub ref_strides: Slice, + #[cube(comptime)] + pub ref_line_size: LineSize, +} + +#[cube] +impl LocalArgs { + /// Creates a new [LocalArgs] container. + pub fn new( + ref_shape: Slice, + ref_strides: Slice, + #[comptime] ref_line_size: LineSize, + ) -> LocalArgs { + LocalArgs { + l_f64: Registry::>::new(), + l_f32: Registry::>::new(), + l_f16: Registry::>::new(), + l_bf16: Registry::>::new(), + l_i64: Registry::>::new(), + l_i32: Registry::>::new(), + l_i16: Registry::>::new(), + l_i8: Registry::>::new(), + l_u64: Registry::>::new(), + l_u32: Registry::>::new(), + l_u16: Registry::>::new(), + l_u8: Registry::>::new(), + l_bool: Registry::>::new(), + ref_shape, + ref_strides, + ref_line_size, + } + } +} + +#[derive(CubeType, Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +/// Unary [element wise operation](ElemwiseOp) arguments. +pub struct UnaryFuseArgs { + pub input: FuseArg, + pub out: FuseArg, +} + +#[derive(CubeType, Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +/// Binary [element wise operation](ElemwiseOp) arguments. +pub struct BinaryFuseArgs { + pub lhs: FuseArg, + pub rhs: FuseArg, + pub out: FuseArg, +} + +#[derive( + CubeType, Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] +/// Precisions supported by [element wise operations](ElemwiseOp). +/// +/// This is a custom type instead of [ElemType] so it can implement [CubeType] +/// and restricts the supported types for fusion. +pub enum FuseType { + F64, + F32, + Flex32, + F16, + BF16, + I64, + I32, + I16, + I8, + U64, + U32, + U16, + U8, + Bool, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +/// Configuration that encapsulates all comptime information necessary for element wise fusion. +pub struct FuseBlockConfig { + pub rank: usize, + pub ref_layout: RefLayout, + pub ops: Vec, + pub width: LineSize, +} + +impl FuseBlockConfig { + pub fn multi_block_variables(&self, registers: &mut Vec<(MultiBlockPos, StorageType)>) { + for op in self.ops.iter() { + op.multi_block_variables(registers); + } + } +} + +impl FuseArg { + pub fn multi_block_variable(&self, registers: &mut Vec<(MultiBlockPos, StorageType)>) { + match self { + FuseArg::MultiBlockGlobal(arg, fuse_type) + // TODO: we need to init the multi-block local, but at some point we could avoid + // that for performance (easier for the underlying compiler). + | FuseArg::MultiBlockLocal(arg, fuse_type) => { + registers.push((arg.clone(), fuse_type.into_type())) + } + _ => {} + }; + } +} + +impl FuseOp { + pub fn multi_block_variables(&self, registers: &mut Vec<(MultiBlockPos, StorageType)>) { + match self { + FuseOp::Add(binary_fuse_args) + | FuseOp::Sub(binary_fuse_args) + | FuseOp::Mul(binary_fuse_args) + | FuseOp::Div(binary_fuse_args) + | FuseOp::Powf(binary_fuse_args) + | FuseOp::Equal(binary_fuse_args) + | FuseOp::Lower(binary_fuse_args) + | FuseOp::Greater(binary_fuse_args) + | FuseOp::LowerEqual(binary_fuse_args) + | FuseOp::Rem(binary_fuse_args) + | FuseOp::GreaterEqual(binary_fuse_args) => { + binary_fuse_args.lhs.multi_block_variable(registers); + binary_fuse_args.rhs.multi_block_variable(registers); + binary_fuse_args.out.multi_block_variable(registers); + } + FuseOp::Abs(unary_fuse_args) + | FuseOp::Exp(unary_fuse_args) + | FuseOp::Log(unary_fuse_args) + | FuseOp::Log1p(unary_fuse_args) + | FuseOp::Cos(unary_fuse_args) + | FuseOp::Sin(unary_fuse_args) + | FuseOp::Tanh(unary_fuse_args) + | FuseOp::Erf(unary_fuse_args) + | FuseOp::Sqrt(unary_fuse_args) + | FuseOp::Recip(unary_fuse_args) + | FuseOp::Assign(unary_fuse_args) => { + unary_fuse_args.input.multi_block_variable(registers); + unary_fuse_args.out.multi_block_variable(registers); + } + FuseOp::Clamp { + input, + min, + max, + out, + } => { + input.multi_block_variable(registers); + min.multi_block_variable(registers); + max.multi_block_variable(registers); + out.multi_block_variable(registers); + } + FuseOp::ConditionalAssign { + cond, + lhs, + rhs, + out, + } => { + cond.multi_block_variable(registers); + lhs.multi_block_variable(registers); + rhs.multi_block_variable(registers); + out.multi_block_variable(registers); + } + FuseOp::Gather { + input, + indices, + output, + dim: _, + } => { + input.multi_block_variable(registers); + indices.multi_block_variable(registers); + output.multi_block_variable(registers); + } + FuseOp::Select { + input, + indices, + output, + dim: _, + } => { + input.multi_block_variable(registers); + indices.multi_block_variable(registers); + output.multi_block_variable(registers); + } + FuseOp::Dequantize { + values, + params, + output, + scheme: _, + } => { + values.multi_block_variable(registers); + params.multi_block_variable(registers); + output.multi_block_variable(registers); + } + } + } +} + +#[cube] +/// Initializes block variables, both globals and locals. +pub fn multi_block_variables_init( + #[comptime] block: &FuseBlockConfig, + variables: &mut MultiBlockVariables, +) { + let output = comptime! { + let mut output = Vec::<(MultiBlockPos, StorageType)>::new(); + block.multi_block_variables(&mut output); + output + }; + + #[unroll] + for i in 0..comptime!(output.len()) { + let (key, dtype) = comptime!(output.get(i).unwrap().clone()); + set_polyfill::>(dtype); + variables.init(key, block.width); + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +/// A reference layout determines how a fuse execution will access elements in tensors. +/// +/// It can either follow the same layout as a concrete tensor, or follow a virtual layout. +pub enum RefLayout { + Concrete(FuseArg), + Virtual(VirtualLayout), +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +/// A virtual layout is always contiguous and retrieves its shape from either a reshaped tensor or a +/// tensor with swap dimensions. +pub enum VirtualLayout { + /// Virtual tensor with the provided shape id and contiguous strides. + Reshaped { + reshape_pos: usize, + line_size: LineSize, + }, + /// Virtual tensor with the same shape as the given input, but with swap dims and contiguous + /// strides. + SwapDims(FuseArg, (usize, usize)), + /// Virtual tensor with the same shape as the given input, but with contiguous strides. + Shape(FuseArg, usize), + /// We don't have access to global metadata, they are passed as runtime values. + Runtime { pos: usize }, +} + +impl FuseArg { + /// Adds layout information. + /// + /// It's going to impact how the input or output is read and written to. + pub fn add_layout_info(&mut self, layout: LayoutInfo) { + match self { + FuseArg::Input(_, _, old) => { + *old = layout; + } + FuseArg::Output(_, _, old) => { + *old = layout; + } + _ => {} + } + } +} + +impl RegistryQuery for FuseArg {} + +impl From for FuseType { + fn from(value: ElemType) -> Self { + match value { + ElemType::Float(kind) => match kind { + FloatKind::F16 => Self::F16, + FloatKind::BF16 => Self::BF16, + FloatKind::F32 => Self::F32, + FloatKind::Flex32 => Self::Flex32, + _ => panic!("Unsupported precision for fusion: {value}"), + }, + ElemType::Int(kind) => match kind { + IntKind::I64 => Self::I64, + IntKind::I32 => Self::I32, + IntKind::I16 => Self::I16, + IntKind::I8 => Self::I8, + }, + ElemType::UInt(kind) => match kind { + UIntKind::U64 => Self::U64, + UIntKind::U32 => Self::U32, + UIntKind::U16 => Self::U16, + UIntKind::U8 => Self::U8, + }, + ElemType::Bool => Self::Bool, + } + } +} + +impl From for FuseType { + fn from(value: StorageType) -> Self { + value.elem_type().into() + } +} + +impl FuseType { + /// Converts the [fused element type](FuseType) into the [cubecl element type](ElemType). + pub fn into_elem(self) -> ElemType { + match self { + FuseType::F32 => ElemType::Float(FloatKind::F32), + FuseType::Flex32 => ElemType::Float(FloatKind::Flex32), + FuseType::F16 => ElemType::Float(FloatKind::F16), + FuseType::BF16 => ElemType::Float(FloatKind::BF16), + FuseType::I64 => ElemType::Int(IntKind::I64), + FuseType::I32 => ElemType::Int(IntKind::I32), + FuseType::I16 => ElemType::Int(IntKind::I16), + FuseType::I8 => ElemType::Int(IntKind::I8), + FuseType::U64 => ElemType::UInt(UIntKind::U64), + FuseType::U32 => ElemType::UInt(UIntKind::U32), + FuseType::U16 => ElemType::UInt(UIntKind::U16), + FuseType::U8 => ElemType::UInt(UIntKind::U8), + FuseType::Bool => ElemType::Bool, + FuseType::F64 => ElemType::Float(FloatKind::F64), + } + } + + /// Convert the [fused element type](FuseType) into the [cubecl storage type](StorageType). + pub fn into_type(self) -> StorageType { + self.into_elem().into() + } +} + +impl From for FuseType { + fn from(value: DType) -> Self { + match value { + DType::F32 => Self::F32, + DType::Flex32 => Self::Flex32, + DType::F16 => Self::F16, + DType::BF16 => Self::BF16, + DType::I64 => Self::I64, + DType::I32 => Self::I32, + DType::I16 => Self::I16, + DType::I8 => Self::I8, + DType::U64 => Self::U64, + DType::U32 => Self::U32, + DType::U16 => Self::U16, + DType::U8 => Self::U8, + DType::Bool => Self::Bool, + DType::F64 => Self::F64, + DType::QFloat(scheme) => match scheme.store { + QuantStore::Native => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S => Self::I8, + QuantValue::E4M3 | QuantValue::E5M2 => { + unimplemented!("Unsupported precision for fusion") + } + QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E2M1 => { + panic!("Can't store native sub-byte values") + } + }, + QuantStore::PackedU32(_) => Self::U32, + QuantStore::PackedNative(_) => match scheme.value { + QuantValue::E2M1 => unimplemented!("Unsupported precision for fusion"), + other => panic!("{other:?} doesn't support native packing"), + }, + }, + } + } +} + +impl Display for FuseArg { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FuseArg::Input(pos, ..) => write!(f, "input({pos})"), + FuseArg::Output(pos, ..) => write!(f, "output({pos})"), + FuseArg::BlockLocal { pos, ty } => write!(f, "local({pos}, {ty:?})"), + FuseArg::MultiBlockLocal(mbp, ..) => write!(f, "{mbp}"), + FuseArg::MultiBlockGlobal(mbp, ..) => write!(f, "global_{mbp}"), + FuseArg::Scalar(pos, ..) => write!(f, "scalar({pos})"), + FuseArg::ScalarShape(pos) => write!(f, "scalar_shape({pos})"), + FuseArg::Literal(val, ..) => write!(f, "literal_{val}"), + FuseArg::InputReshaped { original, .. } => write!(f, "input_reshaped_{original}"), + FuseArg::InputSwapDims { original, .. } => write!(f, "input_swap_dims_{original}"), + } + } +} + +impl Display for MultiBlockPos { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "block_local({}-{})", + self.block_pos, self.block_local_pos + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/kernel.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/kernel.rs new file mode 100644 index 0000000..0ca435a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/kernel.rs @@ -0,0 +1,927 @@ +use super::{DYN_ELEM_ID, Q_PARAM_DYN_ELEM_ID, Q_STORE_DYN_ELEM_ID, io::*, ir::*}; +use burn_std::quantization::{QuantScheme, QuantStore, QuantValue}; +use cubecl::{ + ir::{ElemType, FloatKind, StorageType, UIntKind}, + prelude::*, +}; +use cubek::quantization::{dequantize::dequantize_symmetric_packed_value_at, scheme::QuantMode}; + +#[cube] +/// Fuse element-wise operations at the given write position. +/// +/// # Arguments +/// +/// - `inputs`: Contains all readonly global kernel arguments. +/// - `outputs`: Contains all readwrite global kernel arguments. +/// - `locals`: Contains all local variables defined during kernel expansion. +/// - `write_pos`: The logical position the values are written to. +/// - `write_values`: The explicit values to write at the given position. +/// - `write_args`: The arguments associated to the `writes_values`. +/// - `config`: The current [fuse block configuration](FuseBlockConfig). +/// +/// # Notes +/// +/// The function will start by writing `write_values`. +pub fn fuse_on_write( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + write_values: Registry>, + #[comptime] write_args: Vec, + #[comptime] config: &FuseBlockConfig, +) { + comment!("Fuse on write begin"); + // Write the values given as arguments. + #[unroll] + for i in 0..write_args.len() { + let arg = comptime![write_args.get(i).unwrap().clone()]; + let val = write_values.find(arg.clone()); + + write::(inputs, outputs, locals, write_pos, val, arg, config); + } + + fuse(inputs, outputs, locals, write_pos, config); + comment!("Fuse on write end"); +} + +#[cube] +/// Fuse element-wise operations at the given read position. +/// +/// # Arguments +/// +/// - `inputs`: Contains all readonly global kernel arguments. +/// - `outputs`: Contains all readwrite global kernel arguments. +/// - `locals`: Contains all local variables defined during kernel expansion. +/// - `read_pos`: The logical position the values are read from. +/// - `read_args`: The arguments associated to the `read_pos`. +/// - `config`: The current [fuse block configuration](FuseBlockConfig). +/// +/// # Returns +/// +/// - A sequence of values associated to the given `read_args`. +pub fn fuse_on_read( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + read_pos: usize, + #[comptime] read_args: Sequence, + #[comptime] config: &FuseBlockConfig, +) -> Sequence> { + comment!("Fuse on read begin"); + fuse(inputs, outputs, locals, read_pos, config); + + let mut output = Sequence::new(); + + #[unroll] + for i in 0..read_args.len() { + let arg = comptime![read_args.index(i).clone()]; + let value = read::(inputs, outputs, locals, read_pos, arg, config); + + let value_line_size = value.line_size(); + let output_line_size = config.width; + + // We currently don't support broadcasting __across__ blocks. + if comptime!(value_line_size != output_line_size) { + let mut tmp = Line::::empty(config.width); + comptime!( + assert_eq!(value_line_size, 1, "The input line_size must be 1 or the same as the config width."); + ); + + let val = value[0]; + + #[unroll] + for i in 0..config.width { + tmp[i] = val; + } + + output.push(tmp); + } else { + output.push(value); + } + } + + comment!("Fuse on read end"); + output +} + +#[cube] +/// Initializes [LocalArgs] given the input and output [arguments](GlobalArgs) with the [FuseBlockConfig]. +/// +/// # Notes +/// +/// The goal is to resolve and cache the reference shape and strides, as it is used in many +/// different function during kernel expansion. +pub fn init_locals( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + #[comptime] config: &FuseBlockConfig, +) -> LocalArgs { + comment!("Init locals begin"); + let mut ref_shape = Array::new(config.rank); + let mut ref_strides = Array::new(config.rank); + + let locals = match config.ref_layout.clone() { + RefLayout::Concrete(arg) => match comptime![arg] { + FuseArg::Input(index, ..) => { + let layout = inputs.tensors.index(index); + + #[unroll] + for i in 0..config.rank { + ref_shape[i] = layout.tensor.shape(i); + ref_strides[i] = layout.tensor.stride(i); + } + + LocalArgs::new( + ref_shape.to_slice(), + ref_strides.to_slice(), + layout.tensor.line_size(), + ) + } + FuseArg::Output(index, ..) => { + let layout = outputs.tensors.index(index); + + #[unroll] + for i in 0..config.rank { + ref_shape[i] = layout.tensor.shape(i); + ref_strides[i] = layout.tensor.stride(i); + } + + LocalArgs::new( + ref_shape.to_slice(), + ref_strides.to_slice(), + layout.tensor.line_size(), + ) + } + _ => comptime![panic!("Invalid concrete ref layout.")], + }, + RefLayout::Virtual(layout) => match layout { + VirtualLayout::SwapDims(original, dims) => { + let layout = match original.clone() { + FuseArg::Input(pos, ..) => inputs.tensors.index(pos), + FuseArg::Output(pos, ..) => outputs.tensors.index(pos), + _ => comptime![panic!("Unsupported")], + }; + + let mut stride_curr = 1; + + #[unroll] + #[allow(clippy::clone_on_copy)] + for i in 0..config.rank { + let reverse = reverse_index(config.rank, i); + let swap = comptime![swap_dims_transform(reverse, dims)]; + let shape = layout.tensor.shape(swap.clone()); + + ref_shape[reverse] = shape; + ref_strides[reverse] = stride_curr; + + stride_curr *= ref_shape[comptime![reverse]]; + } + + LocalArgs::new( + ref_shape.to_slice(), + ref_strides.to_slice(), + layout.tensor.line_size(), + ) + } + VirtualLayout::Reshaped { + reshape_pos, + line_size, + } => { + let mut stride_curr = 1; + let start = reshape_pos * config.rank; + + #[unroll] + #[allow(clippy::clone_on_copy)] + for i in 0..config.rank { + let reverse = reverse_index(config.rank, i); + let arg = comptime![FuseArg::ScalarShape(start + reverse)]; + let shape = read_scalar_shape(inputs, arg.clone()); + + ref_shape[comptime![reverse]] = shape; + ref_strides[comptime![reverse]] = stride_curr; + + stride_curr *= ref_shape[comptime![reverse]]; + } + + LocalArgs::new(ref_shape.to_slice(), ref_strides.to_slice(), line_size) + } + VirtualLayout::Runtime { pos } => { + let start_shape = (pos * 2) * config.rank; + let start_strides = start_shape + config.rank; + + #[unroll] + for i in 0..config.rank { + let shape_index = start_shape + i; + let strides_index = start_strides + i; + + ref_shape[i] = *inputs.runtime_layouts.index(shape_index); + ref_strides[i] = *inputs.runtime_layouts.index(strides_index); + } + + LocalArgs::new(ref_shape.to_slice(), ref_strides.to_slice(), config.width) + } + VirtualLayout::Shape(original, line_size) => { + let layout = match original.clone() { + FuseArg::Input(pos, ..) => inputs.tensors.index(pos), + FuseArg::Output(pos, ..) => outputs.tensors.index(pos), + _ => comptime![panic!("Unsupported")], + }; + let mut stride_curr = 1; + + #[unroll] + #[allow(clippy::clone_on_copy)] + for i in 0..config.rank { + let reverse = reverse_index(config.rank, i); + let shape = layout.tensor.shape(reverse); + + ref_shape[comptime![reverse]] = shape; + ref_strides[comptime![reverse]] = stride_curr; + + stride_curr *= ref_shape[comptime![reverse]]; + } + + LocalArgs::new(ref_shape.to_slice(), ref_strides.to_slice(), line_size) + } + }, + }; + comment!("Init locals end"); + locals +} + +#[cube] +/// Expands all [operations](FuseOp) registered in the [block config](FuseBlockConfig]. +fn fuse( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + pos: usize, + #[comptime] config: &FuseBlockConfig, +) { + #[unroll] + for index in 0..config.ops.len() { + let op = config.ops[index].clone(); + set_polyfill::>(op.cmp_type()); + + match op { + FuseOp::Add(op) => { + add::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Div(op) => { + div::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Sub(op) => { + sub::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Mul(op) => { + mul::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Powf(op) => { + powf::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Erf(op) => { + erf::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Sqrt(op) => { + sqrt::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Abs(op) => { + abs::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Log(op) => { + log::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Log1p(op) => { + log1p::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Recip(op) => { + recip::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Assign(op) => { + assign::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Exp(op) => { + exp::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Cos(op) => { + cos::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Sin(op) => { + sin::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Tanh(op) => { + tanh::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Equal(op) => { + equal::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Greater(op) => { + greater::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::GreaterEqual(op) => greater_equal::>( + inputs, outputs, locals, pos, op, config, + ), + FuseOp::Lower(op) => { + lower::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::LowerEqual(op) => { + lower_equal::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::ConditionalAssign { + cond, + lhs, + rhs, + out, + } => conditional_assign::>( + inputs, outputs, locals, pos, cond, lhs, rhs, out, config, + ), + FuseOp::Gather { + input, + indices, + output, + dim, + } => gather::>( + inputs, outputs, locals, pos, dim, input, indices, output, config, + ), + FuseOp::Select { + input, + indices, + output, + dim, + } => select_indices::>( + inputs, outputs, locals, pos, dim, input, indices, output, config, + ), + FuseOp::Dequantize { + values, + params, + output, + scheme, + } => dequantize::>( + inputs, + outputs, + locals, + pos, + values, + params, + output, + scheme.scheme, + config, + ), + FuseOp::Rem(op) => { + rem::>(inputs, outputs, locals, pos, op, config) + } + FuseOp::Clamp { + input, + min, + max, + out, + } => clamp::>( + inputs, outputs, locals, pos, input, min, max, out, config, + ), + } + } +} + +macro_rules! binary_op { + ($ident:ident, $op:tt) => { + #[cube] + fn $ident( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] op: BinaryFuseArgs, + #[comptime] config: &FuseBlockConfig, + ) { + let lhs = read::(inputs, outputs, &locals, write_pos, op.lhs, config); + let rhs = read::(inputs, outputs, &locals, write_pos, op.rhs, config); + let result = lhs $op rhs; + + write::(inputs, outputs, locals, write_pos, result, op.out, config); + } + }; +} + +macro_rules! binary_func { + ($ident:ident, $func:expr, $c:tt) => { + #[cube] + fn $ident( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] op: BinaryFuseArgs, + #[comptime] config: &FuseBlockConfig, + ) { + let lhs = read::(inputs, outputs, &locals, write_pos, op.lhs, config); + let rhs = read::(inputs, outputs, &locals, write_pos, op.rhs, config); + let result = $func(lhs, rhs); + + write::(inputs, outputs, locals, write_pos, result, op.out, config); + } + }; +} + +macro_rules! comparison_op { + ($ident:ident, $op:tt) => { + #[cube] + fn $ident( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] op: BinaryFuseArgs, + #[comptime] config: &FuseBlockConfig, + ) { + let lhs = read::(inputs, outputs, &locals, write_pos, op.lhs, config); + let rhs = read::(inputs, outputs, &locals, write_pos, op.rhs, config); + let result = Line::new(lhs $op rhs); + + write::(inputs, outputs, locals, write_pos, result, op.out, config); + } + }; +} + +macro_rules! unary_func { + ($ident:ident, $func:expr, $c:tt) => { + #[cube] + fn $ident( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] op: UnaryFuseArgs, + #[comptime] config: &FuseBlockConfig, + ) { + let input = read::(inputs, outputs, &locals, write_pos, op.input, config); + let result = $func(input); + + write::(inputs, outputs, locals, write_pos, result, op.out, config); + } + }; +} + +#[cube] +fn assign( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] op: UnaryFuseArgs, + #[comptime] config: &FuseBlockConfig, +) { + let input = read::(inputs, outputs, locals, write_pos, op.input, config); + + write::(inputs, outputs, locals, write_pos, input, op.out, config); +} + +#[cube] +fn gather( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] dim: usize, + #[comptime] input: FuseArg, + #[comptime] indices: FuseArg, + #[comptime] output: FuseArg, + #[comptime] config: &FuseBlockConfig, +) { + let line_size = locals.ref_line_size; + + let pos_input = comptime! { + match input { + FuseArg::Input(pos, ..) => pos, + _ => panic!("Input tensor isn't an input"), + } + }; + let pos_indices = comptime! { + match indices { + FuseArg::Input(pos, ..) => pos, + _ => panic!("Indices tensor isn't an input"), + } + }; + + let stride_input_dim = global_stride(inputs, dim, pos_input); + + let mut index = 0; + let mut result = Line::empty(line_size); + + if comptime![dim > 0] { + let index_before = global_offset( + inputs, + outputs, + locals, + write_pos, + input.clone(), + comptime![Some((0, dim))], + config, + ); + index += index_before; + } + + if comptime![dim + 1 < config.rank] { + let index_after = global_offset( + inputs, + outputs, + locals, + write_pos, + input, + comptime![Some((dim + 1, config.rank))], + config, + ); + index += index_after; + } + + let index_offset = global_offset( + inputs, + outputs, + locals, + write_pos, + indices, + comptime![Some((0, config.rank))], + config, + ); + + if comptime![dim == config.rank - 1] { + // Per-element indexing (along the dimension) + #[unroll] + for i in 0..line_size { + let offset = read_input::( + inputs, + locals, + pos_indices, + index_offset + i, + LayoutInfo::IsRef, + config, + None, + ); + + let input = read_input::( + inputs, + locals, + pos_input, + index + (offset[0] as usize * stride_input_dim), + LayoutInfo::IsRef, + config, + None, + ); + + result[i] = input[0]; + } + } else { + // Shared index for whole line + let stride_input_line = global_stride(inputs, config.rank - 1, pos_input); + + let offset = read_input::( + inputs, + locals, + pos_indices, + index_offset, + LayoutInfo::IsRef, + config, + None, + ); + + index += offset[0] as usize * stride_input_dim; + + #[unroll] + for i in 0..line_size { + let input = read_input::( + inputs, + locals, + pos_input, + index + i * stride_input_line, + LayoutInfo::IsRef, + config, + None, + ); + + result[i] = input[0]; + } + } + + write::(inputs, outputs, locals, write_pos, result, output, config); +} + +#[cube] +fn select_indices( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] dim: usize, + #[comptime] input: FuseArg, + #[comptime] indices: FuseArg, + #[comptime] output: FuseArg, + #[comptime] config: &FuseBlockConfig, +) { + let (line_size_ref, stride_dim_ref, shape_dim_ref) = ( + locals.ref_line_size, + locals.ref_strides[dim], + locals.ref_shape[dim], + ); + + let pos_input = comptime! { + match input { + FuseArg::Input(pos, ..) => pos, + _ => panic!("Input tensor isn't an input"), + } + }; + let pos_indices = match indices { + FuseArg::Input(pos, ..) => pos, + _ => panic!("Indices tensor isn't an input"), + }; + + let stride_input_dim = global_stride(inputs, dim, pos_input); + + let mut index = 0; + let mut result = Line::empty(line_size_ref); + + if comptime![dim != config.rank - 1] { + // In this scenario the select is actually broadcasted along the axis we're working on. + // + // Therefore the same indices are used to fetch multiple entries in the input tensor. + + if comptime![dim > 0] { + let index_before = global_offset( + inputs, + outputs, + locals, + write_pos, + input.clone(), + comptime![Some((0, dim))], + config, + ); + index += index_before; + } + + if comptime![dim + 1 < config.rank] { + let index_after = global_offset( + inputs, + outputs, + locals, + write_pos, + input.clone(), + comptime![Some((dim + 1, config.rank))], + config, + ); + index += index_after; + } + + let stride_input_line = global_stride(inputs, comptime![config.rank - 1], pos_input); + let write_pos_input = write_pos * line_size_ref; + let coordinate_dim = write_pos_input / stride_dim_ref % shape_dim_ref; + let offset_dim = read_input::( + inputs, + locals, + pos_indices, + coordinate_dim, + LayoutInfo::IsRef, + config, + None, + ); + + index += offset_dim[0] as usize * stride_input_dim; + + #[unroll] + for i in 0..line_size_ref { + let input = read_input::( + inputs, + locals, + pos_input, + index + i * stride_input_line, + LayoutInfo::IsRef, + config, + None, + ); + result[i] = input[0]; + } + } else { + // In this scenario the select is actually performed on the last dimension we're working on. + // + // Therefore we need to fetch multiple indices that correspond to different entries in the + // input tensor. + + if comptime![dim > 0] { + let index_before = global_offset( + inputs, + outputs, + locals, + write_pos, + input.clone(), + comptime![Some((0, dim))], + config, + ); + index += index_before; + } + + if comptime![dim + 1 < config.rank] { + let index_after = global_offset( + inputs, + outputs, + locals, + write_pos, + input, + comptime![Some((dim + 1, config.rank))], + config, + ); + index += index_after; + } + + let write_pos_indices = write_pos * line_size_ref; + + #[unroll] + for i in 0..line_size_ref { + let coordinate_dim = (write_pos_indices + i) / stride_dim_ref % shape_dim_ref; + let offset_dim = read_input::( + inputs, + locals, + pos_indices, + coordinate_dim, + LayoutInfo::IsRef, + config, + None, + ); + + let input = read_input::( + inputs, + locals, + pos_input, + index + (offset_dim[0] as usize * stride_input_dim), + LayoutInfo::IsRef, + config, + None, + ); + result[i] = input[0]; + } + } + + write::(inputs, outputs, locals, write_pos, result, output, config); +} + +#[cube] +fn conditional_assign( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] cond: FuseArg, + #[comptime] lhs: FuseArg, + #[comptime] rhs: FuseArg, + #[comptime] out: FuseArg, + #[comptime] config: &FuseBlockConfig, +) { + let cond = read::(inputs, outputs, locals, write_pos, cond, config); + let lhs = read::(inputs, outputs, locals, write_pos, lhs, config); + let rhs = read::(inputs, outputs, locals, write_pos, rhs, config); + let result = select_many(cond, lhs, rhs); + + write::(inputs, outputs, locals, write_pos, result, out, config); +} + +#[cube] +fn clamp( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] input: FuseArg, + #[comptime] min: FuseArg, + #[comptime] max: FuseArg, + #[comptime] out: FuseArg, + #[comptime] config: &FuseBlockConfig, +) { + let input = read::(inputs, outputs, locals, write_pos, input, config); + let min = read::(inputs, outputs, locals, write_pos, min, config); + let max = read::(inputs, outputs, locals, write_pos, max, config); + let result = cubecl::prelude::clamp(input, min, max); + + write::(inputs, outputs, locals, write_pos, result, out, config); +} + +#[cube] +#[allow(clippy::explicit_counter_loop)] +fn dequantize( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + write_pos: usize, + #[comptime] input: FuseArg, + #[comptime] scales: FuseArg, + #[comptime] output: FuseArg, + #[comptime] scheme: QuantScheme, + #[comptime] config: &FuseBlockConfig, +) { + comptime!(assert_eq!( + scheme.mode, + QuantMode::Symmetric, + "Only symmetric quantization mode is supported." + )); + + set_polyfill::>(comptime![match scheme.store { + QuantStore::Native => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S => StorageType::Scalar(ElemType::UInt(UIntKind::U8)), + QuantValue::E4M3 => StorageType::Scalar(ElemType::Float(FloatKind::E4M3)), + QuantValue::E5M2 => StorageType::Scalar(ElemType::Float(FloatKind::E5M2)), + QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E2M1 => unreachable!("Can't store native sub-byte values"), + }, + QuantStore::PackedU32(_) => ElemType::UInt(UIntKind::U32).into(), + QuantStore::PackedNative(_) => match scheme.value { + QuantValue::E2M1 => StorageType::Packed(ElemType::Float(FloatKind::E4M3), 2), + other => panic!("{other:?} doesn't support native packing"), + }, + }]); + set_polyfill::>(comptime![match scheme.param { + cubecl::quant::scheme::QuantParam::F32 => + StorageType::Scalar(ElemType::Float(FloatKind::F32)), + cubecl::quant::scheme::QuantParam::F16 => + StorageType::Scalar(ElemType::Float(FloatKind::F16)), + cubecl::quant::scheme::QuantParam::BF16 => + StorageType::Scalar(ElemType::Float(FloatKind::BF16)), + cubecl::quant::scheme::QuantParam::UE8M0 => + StorageType::Scalar(ElemType::Float(FloatKind::UE8M0)), + cubecl::quant::scheme::QuantParam::UE4M3 => + StorageType::Scalar(ElemType::Float(FloatKind::E4M3)), + }]); + + let tensor_pos = comptime!(match input { + FuseArg::Input(pos, _, _) => pos, + _ => panic!("Not supported"), + }); + let pos = comptime!(match scales { + FuseArg::Input(pos, ..) => pos, + _ => unreachable!(""), + }); + let input = read_quantized::>( + inputs, locals, write_pos, input, config, scheme, + ); + + let line_size = input.line_size(); + let num_quants = scheme.num_quants(); + + let scales = input_as_scales_view::>( + inputs, + pos, + tensor_pos, + scheme.level, + config, + ); + let result = dequantize_symmetric_packed_value_at::< + C, + ElemExpand, + ElemExpand, + >(write_pos * num_quants, input, &scales, scheme); + + let line_size_result = comptime!(num_quants * line_size); + + let line = if comptime!(line_size == 1) { + result[0] + } else { + let mut line = Line::empty(line_size_result); + + #[unroll] + for i in 0..line_size { + let value = result[i]; + + #[unroll] + for j in 0..num_quants { + let index = i * num_quants + j; + line[index] = value[j]; + } + } + + line + }; + + write::(inputs, outputs, locals, write_pos, line, output, config); +} + +binary_op!(add, +); +binary_op!(mul, *); +binary_op!(div, /); +binary_op!(sub, -); + +comparison_op!(equal, ==); +comparison_op!(greater, >); +comparison_op!(greater_equal, >=); +comparison_op!(lower, <); +comparison_op!(lower_equal, <=); + +binary_func!(powf, Line::::powf, Float); +binary_func!(rem, Line::::rem, Float); + +unary_func!(exp, Line::::exp, Float); +unary_func!(log, Line::::ln, Float); +unary_func!(log1p, Line::::log1p, Float); +unary_func!(sqrt, Line::::sqrt, Float); +unary_func!(cos, Line::::cos, Float); +unary_func!(sin, Line::::sin, Float); +unary_func!(tanh, Line::::tanh, Float); +unary_func!(erf, Line::::erf, Float); +unary_func!(recip, Line::::recip, Float); +unary_func!(abs, Line::::abs, Numeric); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/mod.rs new file mode 100644 index 0000000..00951a5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod io; +pub(crate) mod ir; +pub(crate) mod kernel; +pub(crate) mod tensor; +pub(crate) mod view; + +mod base; +pub(crate) use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/tensor.rs new file mode 100644 index 0000000..2197e46 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/tensor.rs @@ -0,0 +1,90 @@ +use super::DYN_ELEM_ID; +use cubecl::{ + ir::{ElemType, Type}, + prelude::*, +}; +use serde::{Deserialize, Serialize}; +use std::hash::Hash; + +/// Represents a global tensor with the given [element type](ElemType). +/// +/// # Warning +/// +/// The `tensor` field type [Line>] must be set using polyfill before +/// use. +#[derive(CubeType, Clone)] +pub struct GlobalTensor { + /// The global tensor type. + pub tensor: Tensor>>, + /// The element type of the tensor. + #[cube(comptime)] + pub elem: ElemType, + /// Whether the current tensor is logically broadcasted. + #[cube(comptime)] + pub broadcasted: bool, +} + +// Everything below is to implement [LaunchArg]. + +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Hash, Debug)] +pub struct GlobalTensorCompilationArg { + tensor: TensorCompilationArg, + elem: ElemType, + broadcasted: bool, +} + +#[derive(new, Debug)] +pub struct GlobalTensorArg<'a, R: Runtime> { + pub tensor: >> as LaunchArg>::RuntimeArg<'a, R>, + pub elem: ElemType, + pub broadcasted: bool, + pub address_type: AddressType, +} + +impl CompilationArg for GlobalTensorCompilationArg {} + +impl LaunchArg for GlobalTensor { + type RuntimeArg<'a, R: Runtime> = GlobalTensorArg<'a, R>; + type CompilationArg = GlobalTensorCompilationArg; + + fn compilation_arg(runtime_arg: &Self::RuntimeArg<'_, R>) -> Self::CompilationArg { + let tensor = >> as LaunchArg>::compilation_arg( + &runtime_arg.tensor, + ); + GlobalTensorCompilationArg { + tensor, + elem: runtime_arg.elem, + broadcasted: runtime_arg.broadcasted, + } + } + + fn expand(arg: &Self::CompilationArg, builder: &mut KernelBuilder) -> GlobalTensorExpand { + let tensor = builder.input_tensor(Type::scalar(arg.elem).line(arg.tensor.line_size)); + + GlobalTensorExpand { + tensor: tensor.into(), + elem: arg.elem, + broadcasted: arg.broadcasted, + } + } + fn expand_output( + arg: &Self::CompilationArg, + builder: &mut KernelBuilder, + ) -> GlobalTensorExpand { + let tensor = match arg.tensor.inplace { + Some(id) => builder.inplace_output(id), + None => builder.output_tensor(Type::scalar(arg.elem).line(arg.tensor.line_size)), + }; + GlobalTensorExpand { + tensor: tensor.into(), + elem: arg.elem, + broadcasted: arg.broadcasted, + } + } +} + +impl ArgSettings for GlobalTensorArg<'_, R> { + fn register(&self, launcher: &mut KernelLauncher) { + launcher.register_tensor(&self.tensor) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/view.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/view.rs new file mode 100644 index 0000000..4d6da2c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/codegen/view.rs @@ -0,0 +1,358 @@ +use super::{ + DYN_ELEM_ID, + io::{ + Transform, global_buffer_len, global_line_size, input_as_slice, read_input, + read_input_window, ref_buffer_len, ref_len, + }, + ir::{FuseArg, FuseBlockConfig, GlobalArgs, LayoutInfo, LocalArgs}, + kernel::fuse_on_write, +}; +use cubecl::{ + CubeType, + io::read_masked, + ir::StorageType, + prelude::{barrier::BarrierExpand, *}, + std::tensor::{ + ViewOperations, ViewOperationsExpand, ViewOperationsMut, ViewOperationsMutExpand, + layout::Coords1d, + }, +}; + +#[allow(dead_code, reason = "only used in expand")] +#[derive(CubeType)] +pub struct GlobalInput { + inputs: GlobalArgs, + locals: LocalArgs, + #[cube(comptime)] + pos: usize, + #[cube(comptime)] + ty: StorageType, + #[cube(comptime)] + layout: LayoutInfo, + #[cube(comptime)] + config: FuseBlockConfig, + #[cube(comptime)] + transform: Option, +} + +#[cube] +impl GlobalInput { + pub fn new( + inputs: &GlobalArgs, + locals: &LocalArgs, + #[comptime] arg: FuseArg, + #[comptime] config: FuseBlockConfig, + #[comptime] transform: Option, + ) -> GlobalInput { + let (pos, ty, layout) = comptime![match arg { + FuseArg::Input(pos, prec, layout) => (pos, prec.into_type(), layout), + _ => unreachable!("Must be concrete input"), + }]; + + GlobalInput { + inputs: inputs.clone(), + locals: locals.clone(), + pos, + ty, + layout, + config, + transform, + } + } +} + +impl ViewOperations for GlobalInput {} +impl ViewOperationsExpand for GlobalInputExpand { + #[allow(clippy::too_many_arguments)] + fn __expand_read_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + ) -> ::ExpandType { + ViewOperationsExpand::::__expand_read_unchecked_method(self, scope, pos) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_read_checked_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + ) -> ::ExpandType { + let zero = E::__expand_cast_from(scope, 0.into()); + ViewOperationsExpand::::__expand_read_masked_method(self, scope, pos, zero) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_read_masked_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + value: ::ExpandType, + ) -> ::ExpandType { + let in_bounds = ViewOperationsExpand::::__expand_is_in_bounds_method( + self, + scope, + pos.clone(), + ); + scope.register_type::>(self.ty); + let slice = input_as_slice::expand(scope, self.inputs.clone(), self.pos); + read_masked::expand::(scope, in_bounds, slice, pos, value) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_read_unchecked_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + ) -> ::ExpandType { + let value = read_input::expand::( + scope, + self.inputs.clone(), + self.locals.clone(), + self.pos, + pos, + self.layout, + self.config.clone(), + self.transform.clone(), + ); + E::__expand_cast_from(scope, value) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_to_linear_slice_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + end: ExpandElementTyped, + ) -> SliceExpand { + scope.register_type::>(self.ty); + let end = add::expand(scope, end.clone(), 1.into()); + read_input_window::expand(scope, self.inputs.clone(), self.pos, pos, end) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_tensor_map_load_method( + &self, + _scope: &mut Scope, + _barrier: BarrierExpand, + _shared_memory: SliceExpand, + _pos: ExpandElementTyped, + ) { + panic!("Not a tensor map") + } + + #[allow(clippy::too_many_arguments)] + fn __expand_shape_method(&self, scope: &mut Scope) -> ExpandElementTyped { + global_buffer_len::expand(scope, self.inputs.clone(), self.pos) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_is_in_bounds_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + ) -> ExpandElementTyped { + let buffer_len = global_buffer_len::expand(scope, self.inputs.clone(), self.pos); + lt::expand(scope, pos, buffer_len) + } +} + +impl Lined for GlobalInput {} +impl LinedExpand for GlobalInputExpand { + fn line_size(&self) -> LineSize { + let mut temp_scope = Scope::root(false); + global_line_size::expand(&mut temp_scope, self.inputs.clone(), self.pos) + } +} + +#[allow(dead_code, reason = "only used in expand")] +#[derive(CubeType)] +pub struct FusedOutput { + inputs: GlobalArgs, + outputs: GlobalArgs, + locals: LocalArgs, + arg: FuseArg, + #[cube(comptime)] + config: FuseBlockConfig, +} + +#[cube] +impl FusedOutput { + pub fn new( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + arg: FuseArg, + #[comptime] config: FuseBlockConfig, + ) -> Self { + FusedOutput { + inputs: inputs.clone(), + outputs: outputs.clone(), + locals: locals.clone(), + arg, + config, + } + } +} + +impl ViewOperations, Coords1d> for FusedOutput {} +impl ViewOperationsExpand, Coords1d> for FusedOutputExpand { + #[allow(clippy::too_many_arguments)] + fn __expand_read_method( + &self, + _scope: &mut Scope, + _pos: ExpandElementTyped, + ) -> as CubeType>::ExpandType { + todo!() + } + + #[allow(clippy::too_many_arguments)] + fn __expand_read_checked_method( + &self, + _scope: &mut Scope, + _pos: ExpandElementTyped, + ) -> as CubeType>::ExpandType { + todo!() + } + + #[allow(clippy::too_many_arguments)] + fn __expand_read_masked_method( + &self, + _scope: &mut Scope, + _pos: ExpandElementTyped, + _value: as CubeType>::ExpandType, + ) -> as CubeType>::ExpandType { + todo!() + } + + #[allow(clippy::too_many_arguments)] + fn __expand_read_unchecked_method( + &self, + _scope: &mut Scope, + _pos: ExpandElementTyped, + ) -> as CubeType>::ExpandType { + todo!() + } + + #[allow(clippy::too_many_arguments)] + fn __expand_to_linear_slice_method( + &self, + _scope: &mut Scope, + _pos: ExpandElementTyped, + _size: ExpandElementTyped, + ) -> SliceExpand, ReadOnly> { + todo!() + } + + #[allow(clippy::too_many_arguments)] + fn __expand_tensor_map_load_method( + &self, + _scope: &mut Scope, + _barrier: BarrierExpand, + _shared_memory: SliceExpand, ReadWrite>, + _pos: ExpandElementTyped, + ) { + panic!("Not a tensor map") + } + + #[allow(clippy::too_many_arguments)] + fn __expand_shape_method(&self, scope: &mut Scope) -> ExpandElementTyped { + ref_len::expand( + scope, + self.inputs.clone(), + self.outputs.clone(), + self.locals.clone(), + self.config.clone(), + ) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_is_in_bounds_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + ) -> ExpandElementTyped { + let buffer_len = ref_buffer_len::expand( + scope, + self.inputs.clone(), + self.outputs.clone(), + self.locals.clone(), + self.config.clone(), + ); + lt::expand(scope, pos, buffer_len) + } +} + +impl ViewOperationsMut, Coords1d> for FusedOutput {} +impl ViewOperationsMutExpand, Coords1d> for FusedOutputExpand { + #[allow(clippy::too_many_arguments)] + fn __expand_write_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + value: as CubeType>::ExpandType, + ) { + let values = Registry::>::__expand_new(scope); + let mut args = comptime![Vec::::new()]; + + values + .clone() + .__expand_insert_method(scope, comptime![self.arg.clone()], value); + comptime![args.push(self.arg.clone())]; + + fuse_on_write::expand( + scope, + self.inputs.clone(), + self.outputs.clone(), + self.locals.clone(), + pos, + values, + args, + self.config.clone(), + ); + } + + #[allow(clippy::too_many_arguments)] + fn __expand_write_checked_method( + &self, + scope: &mut Scope, + pos: ExpandElementTyped, + value: as CubeType>::ExpandType, + ) { + let in_bounds = ViewOperationsExpand::, Coords1d>::__expand_is_in_bounds_method( + self, + scope, + pos.clone(), + ); + if_expand(scope, in_bounds.into(), |scope| { + self.__expand_write_method(scope, pos, value); + }) + } + + #[allow(clippy::too_many_arguments)] + fn __expand_to_linear_slice_mut_method( + &self, + _scope: &mut Scope, + _pos: ExpandElementTyped, + _size: ExpandElementTyped, + ) -> SliceExpand, ReadWrite> { + todo!("Not yet supported") + } + + #[allow(clippy::too_many_arguments)] + fn __expand_tensor_map_store_method( + &self, + _scope: &mut Scope, + _shared_memory: SliceExpand, ReadOnly>, + _pos: ExpandElementTyped, + ) { + panic!("Not a tensor map") + } +} + +impl Lined for FusedOutput {} +impl LinedExpand for FusedOutputExpand { + fn line_size(&self) -> LineSize { + self.locals.ref_line_size + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/fuser.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/fuser.rs new file mode 100644 index 0000000..010d9dd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/fuser.rs @@ -0,0 +1,769 @@ +use super::{ + codegen::ir::{BinaryFuseArgs, FuseArg, FuseOp, FuseType, UnaryFuseArgs}, + settings::FuseSettings, + trace::{FuseTrace, TraceFuser, block::QuantInput}, +}; +use crate::engine::codegen::ir::QuantSchemeFuse; +use burn_fusion::{FuserProperties, FuserStatus, OperationFuser}; +use burn_ir::{ + BaseOperationIr, BinaryOpIr, FloatOperationIr, NumericOperationIr, OperationIr, ScalarOpIr, + TensorIr, UnaryOpIr, +}; +use burn_std::{DType, Shape}; +use cubecl::ir::ElemType; + +/// The base operation fuser that can be used to fuse [all supported fuse operations](FuseOp). +/// +/// +/// This fuser doesn't create a ready-to-execute kernel, but rather generates a +/// [trace](FuseTrace) that be used with a [runner](super::trace::TraceRunner). +/// +/// Since this fuser supports fusing multiple blocks, you can fuse any compute-bound operations +/// with the combination of fuse-on-read and fuse-on-write strategy. +/// +/// # Notes +/// +/// It is responsible to translate [OperationIr] into [FuseOp] and it uses the [TraceFuser] +/// to actually fuse the [FuseOp] when possible. +#[derive(Debug, Clone)] +pub(crate) struct TraceOperationFuser { + fuser: TryTraceFuser, + pub(crate) settings: FuseSettings, + pub(crate) current_output_shape: Shape, + status: FuserStatus, + pub(crate) num_ops: usize, + pub(crate) num_views: usize, + pub(crate) max_bindings: u32, +} + +impl TraceOperationFuser { + /// Checks if the [operation](OperationIr) can be fused with the current fuser. + pub(crate) fn can_fuse(&self, op: &OperationIr) -> bool { + let len_previous = self.len(); + let mut fuser_cloned = self.clone(); + + fuser_cloned.fuse(op); + let len_after = fuser_cloned.len(); + + len_after > len_previous + } +} + +impl OperationFuser for TraceOperationFuser { + fn fuse(&mut self, op: &OperationIr) { + if let FuserStatus::Closed = self.status { + return; + } + + match op { + OperationIr::Drop(tensor) => { + if self.num_ops == 0 { + self.status = FuserStatus::Closed; + return; + } + + self.fuser.fuser.fuse_dropped(tensor); + } + OperationIr::BaseFloat(ops) => { + if !self.fuse_base(ops) { + self.status = FuserStatus::Closed; + return; + } + } + OperationIr::BaseInt(ops) => { + if !self.fuse_base(ops) { + self.status = FuserStatus::Closed; + return; + } + } + OperationIr::Float(_dtype, ops) => { + if !self.fuse_float(ops) { + self.status = FuserStatus::Closed; + return; + } + } + OperationIr::NumericFloat(_dtype, ops) => { + if !self.fuse_numeric(ops) { + self.status = FuserStatus::Closed; + return; + } + } + OperationIr::NumericInt(_dtype, ops) => { + if !self.fuse_numeric(ops) { + self.status = FuserStatus::Closed; + return; + } + } + OperationIr::BaseBool(ops) => { + if !self.fuse_base(ops) { + self.status = FuserStatus::Closed; + return; + } + } + _ => { + self.status = FuserStatus::Closed; + return; + } + }; + + self.status = FuserStatus::Open; + self.num_ops += 1; + } + + fn finish(&mut self) -> FuseTrace { + self.fuser.finish(self.current_output_shape.clone()) + } + + fn len(&self) -> usize { + self.num_ops + } + + fn reset(&mut self) { + self.num_ops = 0; + self.num_views = 0; + self.status = FuserStatus::Open; + self.fuser = TryTraceFuser::new( + self.max_bindings, + self.fuser.fuser.bool_precision, + self.settings, + ); + self.current_output_shape = Shape::new([]); + } + + fn status(&self) -> FuserStatus { + self.status + } + + fn properties(&self) -> FuserProperties { + let ready = self.num_ops > 0; + + FuserProperties { + ready, + score: self.num_ops as u64, + } + } + + fn clone_dyn(&self) -> Box> { + Box::new(self.clone()) + } +} + +impl TraceOperationFuser { + /// Creates a new fuser. + pub fn new(max_bindings: u32, bool_precision: FuseType, settings: FuseSettings) -> Self { + Self { + fuser: TryTraceFuser::new(max_bindings, bool_precision, settings), + settings, + num_ops: 0, + num_views: 0, + max_bindings, + current_output_shape: Shape::new([]), + status: FuserStatus::Open, + } + } + + /// Closes the fuser. + pub fn close(&mut self) { + self.status = FuserStatus::Closed; + } + + /// Declares an input tensor argument where the kernel is responsible to load. + /// + /// # Returns + /// + /// - The argument that maps to the tensor to be used during kernel expansion. + pub fn input_unhandled(&mut self, tensor: &TensorIr) -> FuseArg { + self.fuser.fuser.input_unhandled(tensor) + } + + /// Declares an input quantized tensor argument where the kernel is responsible to load. + /// + /// # Returns + /// + /// None if it's not possible to fuse a quantized tensor. Otherwise: + /// + /// - The argument that maps to the tensor values to be used during kernel expansion. + /// - The argument that maps to the tensor params to be used during kernel expansion. + pub fn input_quantized_unhandled(&mut self, tensor: &TensorIr) -> Option<(FuseArg, FuseArg)> { + self.fuser.fuser.input_quantized_unhandled(tensor) + } + + /// Declares an output tensor argument where the kernel is responsible to write values. + /// + /// # Notes + /// + /// Normally you don't have to declare outputs explicitly before they are going to be + /// fused based on the operations [fused](Self::fuse). + /// + /// # Returns + /// + /// - The argument that maps to the tensor to be used during kernel expansion. + pub fn output_unhandled(&mut self, tensor: &TensorIr) -> FuseArg { + if self.current_output_shape.is_empty() { + self.current_output_shape = tensor.shape.clone(); + } else if self.current_output_shape.iter().sum::() < tensor.shape.iter().sum() { + // The larguest shape win. + self.current_output_shape = tensor.shape.clone(); + } + + self.fuser.fuser.output_unhandled(tensor) + } + + /// Closes the previous block and declares a new one. + /// + /// # Arguments + /// + /// - arguments: Tensors that are logical outputs of the current block and inputs of the following blocks. + /// - settings: [FuseSettings] to be used by the next block. + /// + /// # Returns + /// + /// None if it's impossible to create a next block with the given arguments. Otherwise, the + /// corresponding [arguments](Arg) to the given tensors are returned. + pub fn next_block( + &mut self, + arguments: [&TensorIr; N], + settings: FuseSettings, + global: bool, + ) -> [FuseArg; N] { + let block_pos = self.fuser.fuser.num_previous_blocks(); + let current_output_shape = + core::mem::replace(&mut self.current_output_shape, Shape::new([])); + + self.fuser.fuser.next_block(current_output_shape, settings); + + self.settings = settings; + self.status = FuserStatus::Open; + + arguments.map(|arg| self.fuser.fuser.block_local_input(arg, block_pos, global)) + } + + /// Tag the [tensor](TensorIr) as received from a previous block. + /// + /// This will avoid reading the input again and instead use le local version when possible. + pub fn block_local_input(&mut self, tensor: &TensorIr, block_pos: usize, global: bool) { + self.fuser + .fuser + .block_local_input(tensor, block_pos, global); + } + + fn fuse_base(&mut self, ops: &BaseOperationIr) -> bool { + match ops { + BaseOperationIr::Equal(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Equal(BinaryFuseArgs { lhs, rhs, out }) + }), + BaseOperationIr::EqualElem(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Equal(BinaryFuseArgs { lhs, rhs, out }) + }), + BaseOperationIr::Cast(desc) => { + self.fuse_unary_op(&desc.input, &desc.out, |input, out| { + FuseOp::Assign(UnaryFuseArgs { input, out }) + }) + } + BaseOperationIr::SwapDims(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + if self.fuser.fuse(|fuser| { + fuser.input_swap_dims(&desc.input, &desc.out, (desc.dim1, desc.dim2))?; + + Some(()) + }) { + self.num_views += 1; + true + } else { + false + } + } + BaseOperationIr::Reshape(desc) => { + if desc.input.shape == desc.out.shape { + return self.fuse_unary_op(&desc.input, &desc.out, |input, out| { + FuseOp::Assign(UnaryFuseArgs { input, out }) + }); + } + + if desc.input.shape.rank() > desc.out.shape.rank() { + // Not yet supported. + return false; + } + + if !self.output_is_compatible(&desc.out) { + return false; + } + + if self.fuser.fuse(|fuser| { + fuser.input_reshaped(&desc.input, &desc.out)?; + Some(()) + }) { + self.num_views += 1; + true + } else { + false + } + } + BaseOperationIr::Ones(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + let elem: ElemType = desc.out.dtype.into(); + let precision = elem.into(); + let input = FuseArg::Literal(1, precision); + + self.fuser.fuse(|fuser| { + let out = fuser.output(&desc.out)?; + + fuser.fuse_operation(FuseOp::Assign(UnaryFuseArgs { input, out })); + + Some(()) + }) + } + BaseOperationIr::Zeros(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + let elem: ElemType = desc.out.dtype.into(); + let precision = elem.into(); + let input = FuseArg::Literal(0, precision); + + self.fuser.fuse(|fuser| { + let out = fuser.output(&desc.out)?; + + fuser.fuse_operation(FuseOp::Assign(UnaryFuseArgs { input, out })); + + Some(()) + }) + } + BaseOperationIr::Gather(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let input = build.input_indexed(&desc.tensor)?; + let indices = build.input_indexed(&desc.indices)?; + let output = build.output(&desc.out)?; + + build.fuse_operation(FuseOp::Gather { + input, + indices, + output, + dim: desc.dim, + }); + + Some(()) + }) + } + BaseOperationIr::Select(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let input = build.input_indexed(&desc.tensor)?; + let indices = build.input_indexed(&desc.indices)?; + let output = build.output(&desc.out)?; + + build.fuse_operation(FuseOp::Select { + input, + indices, + output, + dim: desc.dim, + }); + + Some(()) + }) + } + BaseOperationIr::MaskWhere(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let cond = build.input(&desc.mask)?; + let rhs = build.input(&desc.tensor)?; + let lhs = build.input(&desc.value)?; + let out = build.output(&desc.out)?; + + build.fuse_operation(FuseOp::ConditionalAssign { + cond, + lhs, + rhs, + out, + }); + + Some(()) + }) + } + BaseOperationIr::MaskFill(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let cond = build.input(&desc.mask)?; + let lhs = build.scalar(&desc.value, desc.out.dtype); + let rhs = build.input(&desc.tensor)?; + let out = build.output(&desc.out)?; + + build.fuse_operation(FuseOp::ConditionalAssign { + cond, + lhs, + rhs, + out, + }); + + Some(()) + }) + } + _ => false, + } + } + + fn fuse_float(&mut self, ops: &FloatOperationIr) -> bool { + match ops { + FloatOperationIr::Exp(desc) => { + self.fuse_unary_ops(desc, |input, out| FuseOp::Exp(UnaryFuseArgs { input, out })) + } + FloatOperationIr::Log(desc) => { + self.fuse_unary_ops(desc, |input, out| FuseOp::Log(UnaryFuseArgs { input, out })) + } + FloatOperationIr::Log1p(desc) => self.fuse_unary_ops(desc, |input, out| { + FuseOp::Log1p(UnaryFuseArgs { input, out }) + }), + FloatOperationIr::Cos(desc) => { + self.fuse_unary_ops(desc, |input, out| FuseOp::Cos(UnaryFuseArgs { input, out })) + } + FloatOperationIr::Sin(desc) => { + self.fuse_unary_ops(desc, |input, out| FuseOp::Sin(UnaryFuseArgs { input, out })) + } + FloatOperationIr::PowfScalar(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Powf(BinaryFuseArgs { lhs, rhs, out }) + }), + FloatOperationIr::Tanh(desc) => self.fuse_unary_ops(desc, |input, out| { + FuseOp::Tanh(UnaryFuseArgs { input, out }) + }), + FloatOperationIr::Erf(desc) => { + self.fuse_unary_ops(desc, |input, out| FuseOp::Erf(UnaryFuseArgs { input, out })) + } + FloatOperationIr::Sqrt(desc) => self.fuse_unary_ops(desc, |input, out| { + FuseOp::Sqrt(UnaryFuseArgs { input, out }) + }), + FloatOperationIr::Recip(desc) => self.fuse_unary_ops(desc, |input, out| { + FuseOp::Recip(UnaryFuseArgs { input, out }) + }), + FloatOperationIr::Dequantize(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let qinput = build.input_quantized(&desc.input)?; + let out = build.output(&desc.out)?; + + match qinput { + QuantInput::AlreadyDequantized { local } => { + build.fuse_operation(FuseOp::Assign(UnaryFuseArgs { + input: local, + out, + })); + } + QuantInput::Quantized { values, params } => { + build.fuse_operation(FuseOp::Dequantize { + values, + params, + output: out, + scheme: match desc.input.dtype { + DType::QFloat(scheme) => QuantSchemeFuse { scheme }, + _ => unreachable!("Should be a quant tensor."), + }, + }); + } + } + + Some(()) + }) + } + _ => false, + } + } + + fn fuse_numeric(&mut self, op: &NumericOperationIr) -> bool { + match op { + NumericOperationIr::Add(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Add(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::AddScalar(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Add(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Sub(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Sub(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::SubScalar(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Sub(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Mul(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Mul(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::MulScalar(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Mul(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Div(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Div(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::DivScalar(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Div(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Abs(desc) => { + self.fuse_unary_ops(desc, |input, out| FuseOp::Abs(UnaryFuseArgs { input, out })) + } + NumericOperationIr::Lower(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Lower(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::LowerElem(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Lower(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Greater(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Greater(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::GreaterElem(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Greater(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::LowerEqual(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::LowerEqual(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::LowerEqualElem(desc) => self + .fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::LowerEqual(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::GreaterEqual(desc) => self + .fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::GreaterEqual(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::GreaterEqualElem(desc) => self + .fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::GreaterEqual(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Full(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let input = build.scalar(&desc.value, desc.out.dtype); + let out = build.output(&desc.out)?; + + build.fuse_operation(FuseOp::Assign(UnaryFuseArgs { input, out })); + + Some(()) + }) + } + NumericOperationIr::Rem(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Rem(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::RemScalar(desc) => self.fuse_scalar_ops(desc, |lhs, rhs, out| { + FuseOp::Rem(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Powf(desc) => self.fuse_binary_ops(desc, |lhs, rhs, out| { + FuseOp::Powf(BinaryFuseArgs { lhs, rhs, out }) + }), + NumericOperationIr::Clamp(desc) => { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let input = build.input(&desc.tensor)?; + let min = build.scalar(&desc.min, desc.out.dtype); + let max = build.scalar(&desc.max, desc.out.dtype); + let out = build.output(&desc.out)?; + + build.fuse_operation(FuseOp::Clamp { + input, + min, + max, + out, + }); + + Some(()) + }) + } + _ => false, + } + } + + fn fuse_binary_ops(&mut self, desc: &BinaryOpIr, func: Func) -> bool + where + Func: Fn(FuseArg, FuseArg, FuseArg) -> FuseOp, + { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let lhs = build.input(&desc.lhs)?; + let rhs = build.input(&desc.rhs)?; + let out = build.output(&desc.out)?; + + build.fuse_operation(func(lhs, rhs, out)); + + Some(()) + }) + } + + fn fuse_unary_ops(&mut self, desc: &UnaryOpIr, func: Func) -> bool + where + Func: Fn(FuseArg, FuseArg) -> FuseOp, + { + self.fuse_unary_op(&desc.input, &desc.out, func) + } + + fn fuse_unary_op(&mut self, input: &TensorIr, out: &TensorIr, func: Func) -> bool + where + Func: Fn(FuseArg, FuseArg) -> FuseOp, + { + if !self.output_is_compatible(out) { + return false; + } + + self.fuser.fuse(|build| { + let input = build.input(input)?; + let out = build.output(out)?; + build.fuse_operation(func(input, out)); + Some(()) + }) + } + + fn fuse_scalar_ops(&mut self, desc: &ScalarOpIr, func: Func) -> bool + where + Func: Fn(FuseArg, FuseArg, FuseArg) -> FuseOp, + { + if !self.output_is_compatible(&desc.out) { + return false; + } + + self.fuser.fuse(|build| { + let elem = desc.lhs.dtype; + let lhs = build.input(&desc.lhs)?; + let rhs = build.scalar(&desc.rhs, elem); + let out = build.output(&desc.out)?; + + build.fuse_operation(func(lhs, rhs, out)); + + Some(()) + }) + } + + fn output_is_compatible(&mut self, out: &TensorIr) -> bool { + if self.current_output_shape.is_empty() { + self.current_output_shape.clone_from(&out.shape); + return true; + } + + let rank = self.current_output_shape.len(); + + // Rank should be equal. + if rank != out.shape.num_dims() { + return false; + } + + let mut updated = self.current_output_shape.clone(); + let mut should_update = false; + + #[allow(clippy::needless_range_loop)] + for i in 0..rank { + let curr = self.current_output_shape[i]; + let new = out.shape[i]; + + if curr == new { + continue; + } + + // Broadcast not enabled. + if !self.settings.broadcast { + return false; + } + + // Broadcasted on new dim. + if new == 0 { + continue; + } + + // Broadcasted on curr dim - update reference output shape. + if curr == 0 && self.settings.output_shape_updates { + should_update = true; + updated[i] = new; + continue; + } + + return false; + } + + if should_update { + // For now forced to have exact shape. + if updated != out.shape { + return false; + } + + self.current_output_shape.clone_from_slice(&out.shape); + } + + true + } +} + +#[derive(Debug, Clone)] +/// Builder wrapper to limit the number of bindings in generated kernels. +struct TryTraceFuser { + fuser: TraceFuser, + max_bindings: u32, + max_ops: u32, + added_ops: bool, +} + +impl TryTraceFuser { + fn new(max_bindings: u32, bool_precision: FuseType, settings: FuseSettings) -> Self { + Self { + fuser: TraceFuser::new(bool_precision, settings), + max_bindings, + // A good default, avoid errors with for loops over only memory + // bound operations. + max_ops: 64, + added_ops: false, + } + } + + fn fuse(&mut self, add_ops: impl FnOnce(&mut TraceFuser) -> Option<()>) -> bool { + if self.fuser.num_ops_fused() > self.max_ops { + return false; + } + + // Always allow the first operation to be added. + if !self.added_ops { + self.added_ops = true; + + if add_ops(&mut self.fuser).is_none() { + return false; + } + return true; + } + + let mut cloned = self.fuser.clone(); + if add_ops(&mut cloned).is_none() { + return false; + } + + if cloned.estimate_bindings() > self.max_bindings { + return false; + } + + self.fuser = cloned; + true + } + + fn finish(&mut self, shape: Shape) -> FuseTrace { + self.fuser.finish(shape) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/base.rs new file mode 100644 index 0000000..723de12 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/base.rs @@ -0,0 +1,99 @@ +use crate::{ + CubeFusionHandle, + engine::{ + launch::{ + HandleInput, HandleOutput, LaunchPlan, executor::LaunchPlanExecutor, + input::InputPlanner, output::OutputPlanner, runner::TraceRunner, + vectorization::VectorizationPlanner, + }, + trace::{FuseTrace, TraceError, TuneOutput}, + }, +}; +use burn_fusion::stream::Context; +use cubecl::{CubeElement, Runtime, client::ComputeClient}; +use std::marker::PhantomData; + +/// The launcher is responsible to launch a fused kernel using the [TraceRunner] and a [FuseTrace]. +pub struct FuseTraceLauncher<'a, R: Runtime, Runner: TraceRunner> { + trace: &'a FuseTrace, + runner: &'a Runner, + _runtime: PhantomData, +} + +impl<'a, R: Runtime, Runner: TraceRunner> FuseTraceLauncher<'a, R, Runner> { + /// Creates a new launcher. + pub fn new(trace: &'a FuseTrace, runner: &'a Runner) -> Self { + Self { + trace, + runner, + _runtime: PhantomData, + } + } + /// Launches the fuse kernel on the given device modifying the context. + pub fn launch( + &self, + client: &ComputeClient, + device: &R::Device, + context: &mut Context<'_, CubeFusionHandle>, + ) -> Result, TraceError> { + let mut plan = LaunchPlan::new(&self.trace.blocks); + + InputPlanner::new(&self.trace.resources, &self.trace.blocks).run(context, &mut plan); + + OutputPlanner::new(&self.trace.resources, &self.trace.blocks) + .run::(client, device, context, &mut plan); + + VectorizationPlanner::new(&self.trace.resources, &self.trace.blocks).run( + client, + self.runner, + context, + &mut plan, + ); + + match LaunchPlanExecutor::new(&self.trace.resources, &self.trace.blocks).execute::<_, BT>( + client, + self.runner, + context, + plan, + ) { + Err(err) => { + self.rollback(context, err.handles_input, err.handles_output); + Err(err.error) + } + Ok(val) => Ok(val), + } + } + + fn rollback( + &self, + context: &mut Context<'_, CubeFusionHandle>, + handle_inputs: Vec>, + handle_outputs: Vec>, + ) { + for input in handle_inputs { + match input { + HandleInput::Normal(input) => { + context + .handles + .register_handle(input.global_ir.id, input.handle_rollback()); + } + HandleInput::QuantValues(input) => { + context + .handles + .register_handle(input.global_ir.id, input.handle); + } + HandleInput::QuantParams(_) => { + // The scales are part of the quant data handle. + } + }; + } + for output in handle_outputs { + if let HandleOutput::Owned { + global_id, handle, .. + } = output + { + context.handles.register_handle(global_id, handle); + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/executor.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/executor.rs new file mode 100644 index 0000000..acfe5ee --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/executor.rs @@ -0,0 +1,292 @@ +use super::{HandleInput, HandleOutput, LaunchPlan, ReferenceSelection}; +use crate::engine::launch::runner::TraceRunner; +use crate::engine::trace::{FuseResources, TensorView, TraceError, TuneOutput, block::FuseBlock}; +use crate::{ + CubeFusionHandle, elem_dtype, + engine::{ + codegen::ir::{ + FuseBlockConfig, FuseOp, FuseType, GlobalArgsLaunch, RefLayout, VirtualLayout, + }, + codegen::tensor::GlobalTensorArg, + }, +}; +use burn_fusion::stream::{Context, ScalarId}; +use burn_ir::ScalarIr; +use burn_std::DType; +use cubecl::{ + CubeElement, Runtime, + client::ComputeClient, + ir::AddressType, + prelude::{InputScalar, ScalarArg, TensorArg}, +}; +use std::marker::PhantomData; + +/// Execute a [plan](LaunchPlan) using a [runner](TraceRunner) modifying the [context](Context). +pub struct LaunchPlanExecutor<'a, R: Runtime> { + resources: &'a FuseResources, + blocks: &'a Vec, + _r: PhantomData, +} + +#[derive(new, Debug)] +pub struct ExecutionError> { + pub error: TraceError, + pub handles_input: Vec>, + pub handles_output: Vec>, +} + +impl<'a, R: Runtime> LaunchPlanExecutor<'a, R> { + pub fn new(resources: &'a FuseResources, blocks: &'a Vec) -> Self { + Self { + resources, + blocks, + _r: PhantomData, + } + } + + pub fn execute, BT: CubeElement>( + self, + client: &ComputeClient, + runner: &Runner, + context: &mut Context<'_, CubeFusionHandle>, + plan: LaunchPlan<'a, R>, + ) -> Result, ExecutionError> { + let mut num_writes = 0; + for b in plan.blocks.iter() { + for writes in b.writes.values() { + num_writes += writes.len(); + } + } + + #[cfg(feature = "autotune-checks")] + let mut tune_output = TuneOutput::Checked { + handles: std::collections::HashMap::new(), + }; + + #[cfg(not(feature = "autotune-checks"))] + let mut tune_output = TuneOutput::UnChecked(PhantomData); + + if num_writes == 0 { + // Nothing to write, can skip execution. + return Ok(tune_output); + } + + let mut inputs = GlobalArgsLaunch::default(); + let mut outputs = GlobalArgsLaunch::default(); + + register_inputs(&plan.handle_inputs, &mut inputs); + register_scalars( + self.resources.scalars.iter(), + self.resources.views.iter(), + context, + &mut inputs, + ); + register_outputs::(&plan.handle_outputs, &mut outputs, &mut tune_output); + + for layout in plan.runtime_layouts { + for s in layout.shape.iter() { + inputs.runtime_layouts.push(ScalarArg::new(*s)); + } + for s in layout.strides.iter() { + inputs.runtime_layouts.push(ScalarArg::new(*s)); + } + } + + let mut configs = Vec::with_capacity(plan.blocks.len()); + + for (block_plan, block) in plan.blocks.into_iter().zip(self.blocks) { + let reference = match block_plan.reference { + ReferenceSelection::Concrete { layout, .. } => RefLayout::Concrete(layout), + ReferenceSelection::VirtualShape { original, .. } => { + RefLayout::Virtual(VirtualLayout::Shape(original, block_plan.width)) + } + ReferenceSelection::SwapDims { original, dims } => { + RefLayout::Virtual(VirtualLayout::SwapDims(original, dims)) + } + ReferenceSelection::Reshaped { reshape_pos } => { + RefLayout::Virtual(VirtualLayout::Reshaped { + reshape_pos, + line_size: block_plan.width, + }) + } + ReferenceSelection::Runtime { pos } => { + RefLayout::Virtual(VirtualLayout::Runtime { pos }) + } + ReferenceSelection::Searching => { + return Err(ExecutionError::new( + TraceError::ReferenceNotFound, + plan.handle_inputs, + plan.handle_outputs, + )); + } + }; + + let mut ops = Vec::::new(); + + for read_ops in block_plan.reads.into_values() { + for op in read_ops { + ops.push(op); + } + } + + for op in block.ops.iter() { + ops.push(op.clone()); + } + + for opsw in block_plan.writes.into_values() { + for op in opsw { + ops.push(op); + } + } + + let config = FuseBlockConfig { + rank: plan.rank, + ref_layout: reference, + ops, + width: block_plan.width, + }; + configs.push(config); + } + + Runner::run(runner, client, inputs, outputs, &configs).map_err(|err| { + ExecutionError::new( + TraceError::RunnerError(err), + plan.handle_inputs, + plan.handle_outputs, + ) + })?; + + Ok(tune_output) + } +} + +fn register_inputs<'h, R: Runtime>( + handle_inputs: &'h [HandleInput], + inputs: &mut GlobalArgsLaunch<'h, R>, +) { + for hi in handle_inputs.iter() { + match hi { + HandleInput::Normal(hi) => { + let arg = hi.handle.as_tensor_arg(&hi.global_ir.shape, hi.line_size); + inputs.tensors.push(GlobalTensorArg::new( + arg, + hi.precision.into_elem(), + hi.broadcated, + hi.handle.required_address_type(), + )); + } + HandleInput::QuantValues(hi) => { + let arg = hi.handle.as_tensor_arg(&hi.global_ir.shape, hi.line_size); + inputs.tensors.push(GlobalTensorArg::new( + arg, + hi.precision.into_elem(), + false, + hi.handle.required_address_type(), + )); + } + HandleInput::QuantParams(hi) => { + let arg = hi.handle.as_tensor_arg(&hi.shape, 1); + inputs.tensors.push(GlobalTensorArg::new( + arg, + hi.precision.into_elem(), + false, + hi.handle.required_address_type(), + )); + } + } + } +} + +fn register_outputs<'s, BT: CubeElement, R: Runtime>( + handle_outputs: &'s [HandleOutput], + outputs: &mut GlobalArgsLaunch<'s, R>, + #[allow(unused_variables)] tune_output: &mut TuneOutput, +) { + for item in handle_outputs.iter() { + match item { + HandleOutput::Alias { + input_pos, + precision, + #[cfg(feature = "autotune-checks")] + debug_info, + } => { + outputs.tensors.push(GlobalTensorArg::new( + TensorArg::alias(*input_pos), + precision.into_elem(), + false, + AddressType::default(), + )); + + #[cfg(feature = "autotune-checks")] + if let TuneOutput::Checked { handles, .. } = tune_output { + handles.insert( + debug_info.relative_id, + (debug_info.global_shape.clone(), debug_info.handle.clone()), + ); + } + } + HandleOutput::Owned { + precision, + handle, + global_shape, + vectorization: line_size, + #[cfg(feature = "autotune-checks")] + relative_id, + .. + } => { + let arg = handle.as_tensor_arg(global_shape, *line_size); + + let elem = match precision { + FuseType::Bool => match elem_dtype::() { + DType::U32 => FuseType::U32.into_elem(), + DType::U8 => FuseType::U8.into_elem(), + _ => todo!(), + }, + _ => precision.into_elem(), + }; + + #[cfg(feature = "autotune-checks")] + if let TuneOutput::Checked { handles, .. } = tune_output { + handles.insert(*relative_id, (global_shape.clone(), handle.clone())); + } + + outputs.tensors.push(GlobalTensorArg::new( + arg, + elem, + false, + handle.required_address_type(), + )); + } + } + } +} + +fn register_scalars<'h, R: Runtime>( + scalars: impl Iterator, + views: impl DoubleEndedIterator, + context: &mut Context<'_, CubeFusionHandle>, + inputs: &mut GlobalArgsLaunch<'h, R>, +) { + for (precision, id) in scalars { + let dtype = precision.into_type(); + match context.scalars.get(&ScalarId { value: *id }) { + Some(scalar) => match scalar { + ScalarIr::Float(val) => inputs.scalars.push(InputScalar::new(*val, dtype)), + ScalarIr::Int(val) => inputs.scalars.push(InputScalar::new(*val, dtype)), + ScalarIr::UInt(val) => inputs.scalars.push(InputScalar::new(*val, dtype)), + ScalarIr::Bool(val) => inputs.scalars.push(InputScalar::new(*val as u8, dtype)), + }, + None => panic!("Scalar ID not found"), + } + } + + for relative in views { + if let TensorView::Reshape { reshaped, .. } = relative { + let global = context.tensors.get(reshaped).unwrap(); + + for shape in global.shape.iter() { + inputs.reshapes.push(ScalarArg::new(*shape)); + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/input.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/input.rs new file mode 100644 index 0000000..9792b87 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/input.rs @@ -0,0 +1,247 @@ +use super::{BlockPlan, HandleInput, InputReference}; +use super::{LaunchPlan, NormalHandleInput, PotentialInplace}; +use crate::CubeFusionHandle; +use crate::engine::launch::{QuantParamsHandleInput, QuantValuesHandleInput}; +use crate::engine::trace::block::FuseBlock; +use crate::engine::trace::{FuseResources, RegisterTensor, TensorView}; +use burn_fusion::stream::Context; +use burn_ir::{TensorIr, TensorStatus}; +use burn_std::quantization::params_shape; +use cubecl::Runtime; +use std::marker::PhantomData; + +/// Fetch and register [input handles](HandleInput). Also identifies potential inputs that +/// can be used inplace and/or as the [reference layout](super::super::ir::RefLayout). +pub struct InputPlanner<'a, R: Runtime> { + resources: &'a FuseResources, + blocks: &'a Vec, + _r: PhantomData, +} + +impl<'a, R: Runtime> InputPlanner<'a, R> { + pub fn new(resources: &'a FuseResources, blocks: &'a Vec) -> Self { + Self { + resources, + blocks, + _r: PhantomData, + } + } + + pub fn run(self, context: &mut Context<'_, CubeFusionHandle>, plan: &mut LaunchPlan<'a, R>) { + for (pos, input) in self.resources.inputs.iter().enumerate() { + match input { + RegisterTensor::Normal(tensor_relative, precision) => { + let mut tensor_global = + context.tensors.get(&tensor_relative.id).unwrap().clone(); + let handle = context + .handles + .get_handle(&tensor_global.id, &TensorStatus::ReadOnly); + + if let TensorStatus::ReadWrite = tensor_relative.status { + plan.cleared.push(tensor_global.id); + } + + let mut new_strides = handle.strides.clone(); + + self.analyze(plan, pos, tensor_relative, &handle); + + if tensor_global.shape.rank() < plan.rank { + let num_elem: usize = tensor_global.shape.iter().product(); + for _ in 0..(plan.rank - tensor_global.shape.rank()) { + tensor_global.shape.insert(0, 1); + new_strides.insert(0, num_elem); + } + } + + plan.handle_inputs + .push(HandleInput::Normal(NormalHandleInput::new( + tensor_global, + tensor_relative, + *precision, + handle, + new_strides, + ))); + } + RegisterTensor::QuantValues(tensor_relative) => { + let tensor_global = context.tensors.get(&tensor_relative.id).unwrap().clone(); + let handle = context + .handles + .get_handle(&tensor_global.id, &TensorStatus::ReadOnly); + + let scheme = match tensor_relative.dtype { + burn_std::DType::QFloat(scheme) => scheme, + _ => unreachable!("Can't have quant data without QFloat"), + }; + let params = handle.params(scheme).unwrap(); + let precision = tensor_relative.dtype.into(); + let precision_scales = params.dtype.into(); + + let global_shape = tensor_global.shape.clone(); + let shape_params = params_shape(&global_shape, scheme.level); + plan.handle_inputs + .push(HandleInput::QuantValues(QuantValuesHandleInput { + relative_id: tensor_relative.id, + global_ir: tensor_global, + precision, + handle, + line_size: 1, + })); + + plan.handle_inputs + .push(HandleInput::QuantParams(QuantParamsHandleInput { + precision: precision_scales, + handle: params, + shape: shape_params, + })); + } + RegisterTensor::QuantParams(_) => { + // It is registered at the same time as quant data. + // The order is important and the index in the vector as well, so that's why we + // have QuantParams. + } + } + } + } + + fn analyze( + &self, + plan: &mut LaunchPlan<'a, R>, + pos: usize, + tensor_relative: &'a TensorIr, + handle: &CubeFusionHandle, + ) { + if !self + .resources + .inputs_unhandled + .contains(&tensor_relative.id) + { + let mut is_a_view = false; + // For each view we try to see if it's not possible to set it as a reference input. + for view in self.resources.views.iter() { + for (block_plan, block) in plan.blocks.iter_mut().zip(self.blocks) { + is_a_view = is_a_view + || Self::analyze_view(pos, tensor_relative, block, block_plan, view); + } + } + + if !is_a_view { + self.analyze_normal(plan, pos, tensor_relative, handle); + } + } + } + + /// Analyzes if the given tensor can be used inplace in one of the block. + fn analyze_normal( + &self, + plan: &mut LaunchPlan<'a, R>, + pos: usize, + tensor_relative: &'a TensorIr, + handle: &CubeFusionHandle, + ) { + enum BlockInplaceSelection { + Notinit, + /// The block reads the input, and therefore can use it for inplace. + Selected(usize), + /// The same input is used in multiple blocks. + Unavailable, + } + + let mut block_inplace_selection = BlockInplaceSelection::Notinit; + + for (idx, block) in plan.blocks.iter().enumerate() { + if block.reads.contains_key(&tensor_relative.id) { + match block_inplace_selection { + BlockInplaceSelection::Notinit => { + block_inplace_selection = BlockInplaceSelection::Selected(idx); + } + BlockInplaceSelection::Selected(_) => { + block_inplace_selection = BlockInplaceSelection::Unavailable; + } + BlockInplaceSelection::Unavailable => {} + } + } + } + + if let BlockInplaceSelection::Selected(idx) = block_inplace_selection { + if self.blocks[idx].shape_ref != tensor_relative.shape { + return; + } + + let block_plan = &mut plan.blocks[idx]; + if tensor_relative.status == TensorStatus::ReadWrite { + if self.blocks[idx].settings.inplace && handle.handle.can_mut() { + block_plan.potential_inplaces.push(PotentialInplace { + input_pos: pos, + tensor_relative, + strides: handle.strides.clone(), + }); + } + // Inplace tensors are normally really good as the reference layout, since + // it's normally better to be based on writes rather than on reads. + block_plan.potential_reference_input = + Some(InputReference::Normal { input_pos: pos }); + } else { + block_plan.potential_reference_input = + Some(InputReference::Normal { input_pos: pos }); + } + } + } + + /// Analyzes if the given tensor is also the view provided, and check if it can be used as the reference layout + /// for the given block. + fn analyze_view( + pos: usize, + tensor_relative: &'a TensorIr, + block: &FuseBlock, + block_plan: &mut BlockPlan<'a>, + view: &TensorView, + ) -> bool { + match view { + TensorView::Reshape { + reshaped, + original, + reshape_pos, + shape_relative, + } => { + if original == &tensor_relative.id || reshaped == &tensor_relative.id { + if block_plan.potential_reference_input.is_none() + && shape_relative == &block.shape_ref + { + block_plan.potential_reference_input = Some(InputReference::Reshaped { + reshape_pos: *reshape_pos, + }); + } + return true; + } + } + TensorView::SwapDims { + swapped, + original, + dims, + .. + } => { + if swapped == &tensor_relative.id { + return true; + } + + if original == &tensor_relative.id { + let shape = tensor_relative + .shape + .clone() + .swapped(dims.0, dims.1) + .unwrap(); + + if block_plan.potential_reference_input.is_none() && shape == block.shape_ref { + block_plan.potential_reference_input = Some(InputReference::SwapDims { + original_pos: pos, + dims: *dims, + }); + } + return true; + } + } + }; + + false + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/mod.rs new file mode 100644 index 0000000..94c9405 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/mod.rs @@ -0,0 +1,11 @@ +pub(crate) mod executor; +pub(crate) mod input; +pub(crate) mod output; +pub(crate) mod runner; +pub(crate) mod vectorization; + +pub(crate) mod plan; +pub use plan::*; + +mod base; +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/output.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/output.rs new file mode 100644 index 0000000..e425ff4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/output.rs @@ -0,0 +1,696 @@ +use super::{ + super::codegen::ir::FuseType, BlockPlan, HandleOutput, InputReference, LaunchPlan, + NormalHandleInput, ReferenceSelection, +}; +use crate::{ + CubeFusionHandle, elem_dtype, + engine::{ + codegen::ir::{FuseArg, FuseOp, LayoutInfo}, + launch::HandleInput, + settings::RefLayoutSetting, + trace::{FuseResources, RegisterTensor, RuntimeLayout, TensorView, block::FuseBlock}, + }, + strides_dyn_rank, +}; +use burn_fusion::stream::Context; +use burn_ir::{TensorId, TensorIr}; +use burn_std::{DType, Shape}; +use burn_std::{ + Strides, + tensor::{ReshapeAction, contiguous_strides, is_contiguous, reshape_action}, +}; +use cubecl::{CubeElement, Runtime, client::ComputeClient, ir::StorageType}; + +/// Create or reuse handles for the outputs. +/// +/// It is also responsible to select the reference tensor. +pub struct OutputPlanner<'a, R: Runtime> { + resources: &'a FuseResources, + outputs_sorted: Vec>, + handles: Vec>>, + globals: Vec>, + blocks: &'a Vec, +} + +#[derive(Debug)] +struct OutputSorted<'a> { + pos_original: usize, + precision: FuseType, + tensor_relative: &'a TensorIr, +} + +#[derive(Debug)] +enum OutputKind { + Normal, + Inplace { + /// The position in the potential inplace vector + input_pos: usize, + }, + Transform(TensorView), +} + +impl<'a, R: Runtime> OutputPlanner<'a, R> { + pub fn new(resources: &'a FuseResources, blocks: &'a Vec) -> Self { + let mut outputs_sorted: Vec<_> = resources + .outputs + .iter() + .enumerate() + .filter_map(|(pos, entry)| match entry { + RegisterTensor::Normal(ir, p) => Some((pos, ir, p)), + RegisterTensor::QuantValues(_) => None, + RegisterTensor::QuantParams(_) => None, + }) + .map(|(pos, tensor, precision)| OutputSorted { + pos_original: pos, + precision: *precision, + tensor_relative: tensor, + }) + .collect(); + + outputs_sorted.sort_by(|a, b| { + let a_val: usize = a.tensor_relative.shape.iter().sum(); + let b_val: usize = b.tensor_relative.shape.iter().sum(); + + b_val.cmp(&a_val) + }); + + let mut handles = Vec::with_capacity(resources.outputs.len()); + let mut globals = Vec::with_capacity(resources.outputs.len()); + + for _ in 0..resources.outputs.len() { + handles.push(None); + globals.push(None); + } + + Self { + resources, + outputs_sorted, + handles, + globals, + blocks, + } + } + + pub fn run( + mut self, + client: &ComputeClient, + device: &R::Device, + context: &mut Context<'_, CubeFusionHandle>, + plan: &mut LaunchPlan<'a, R>, + ) { + // So that we can borrow self during the iteration. + let mut outputs = Vec::new(); + core::mem::swap(&mut outputs, &mut self.outputs_sorted); + + for output in outputs.into_iter() { + let tensor_global = context + .tensors + .get(&output.tensor_relative.id) + .unwrap() + .clone(); + let strides = strides_dyn_rank(&tensor_global.shape); + let (kind, block_idx) = self.output_kind(plan, &tensor_global, &output, &strides); + + match kind { + OutputKind::Inplace { input_pos } => { + self.inplace_output(context, plan, output, tensor_global, input_pos, block_idx); + } + OutputKind::Normal => { + self.normal_output::( + client, + device, + context, + plan, + output, + tensor_global, + strides, + block_idx, + ); + } + OutputKind::Transform(TensorView::Reshape { original, .. }) => { + self.reshaped_output::( + client, + device, + context, + plan, + output, + tensor_global, + strides, + original, + block_idx, + ); + } + OutputKind::Transform(TensorView::SwapDims { original, dims, .. }) => { + self.swapped_dims_output::( + client, + device, + context, + plan, + output, + tensor_global, + original, + dims, + block_idx, + ); + } + } + } + + for (handle, global) in self.handles.into_iter().zip(self.globals.into_iter()) { + plan.handle_outputs.push(handle.unwrap()); + plan.global_outputs.push(global.unwrap()); + } + + for i in 0..plan.blocks.len() { + if !plan.blocks[i].reference.is_found() { + match self.blocks[i].settings.ref_layout { + RefLayoutSetting::SameAsBlock { block_pos } => { + plan.blocks[i].reference = + plan.blocks[block_pos as usize].reference.clone(); + } + _ => { + let new_runtime = Self::select_reference_from_inputs( + &self.blocks[i], + &mut plan.blocks[i], + &plan.handle_inputs, + ); + + if let Some(shape) = new_runtime { + let pos = plan.runtime_layouts.len(); + let mut shape_global = shape.clone(); + for (i, s) in shape.iter().enumerate() { + shape_global[i] = *context.shapes_relative2global.get(s).unwrap(); + } + + let strides = strides_dyn_rank(&shape_global); + + plan.blocks[i].reference = ReferenceSelection::Runtime { pos }; + plan.runtime_layouts.push(RuntimeLayout { + shape: shape_global, + strides, + }); + } + } + }; + } else { + Self::add_layout_info_inputs(&mut plan.blocks[i], &plan.handle_inputs); + } + } + + // Make sure dropped are correctly executed. + for id in self.resources.dropped.iter() { + if let Some(tensor_global) = context.tensors.get(id) { + context.handles.remove_handle(tensor_global.id); + } + } + for id in plan.cleared.drain(..) { + context.handles.remove_handle(id); + } + } + + fn select_reference_from_inputs( + block: &FuseBlock, + block_plan: &mut BlockPlan<'_>, + handle_inputs: &[HandleInput], + ) -> Option { + if let Some(input_ref) = block_plan.potential_reference_input.take() { + match input_ref { + InputReference::Normal { input_pos } => { + let reference = handle_inputs + .get(input_pos) + .unwrap() + .as_normal() + .expect("Quant can't be used as inplace"); + + let set_ref_as_concrete = |block: &mut BlockPlan<'_>| { + block.reference = ReferenceSelection::Concrete { + layout: FuseArg::Input( + input_pos, + reference.precision, + LayoutInfo::IsRef, + ), + shape: reference.global_ir.shape.clone(), + strides: reference.handle.strides.clone(), + }; + }; + + let set_ref_as_virtual = |block: &mut BlockPlan<'_>| { + block.reference = ReferenceSelection::VirtualShape { + original: FuseArg::Input( + input_pos, + reference.precision, + LayoutInfo::Unknown, + ), + shape: reference.global_ir.shape.clone(), + strides: contiguous_strides(&reference.global_ir.shape), + }; + }; + + match block.settings.ref_layout { + RefLayoutSetting::Any => set_ref_as_concrete(block_plan), + RefLayoutSetting::SameAsBlock { .. } => { + // Skip set ref. + } + RefLayoutSetting::OnlyContiguous => { + if is_contiguous(&reference.global_ir.shape, &reference.handle.strides) + { + set_ref_as_concrete(block_plan) + } else { + set_ref_as_virtual(block_plan) + } + } + } + + Self::add_layout_info_inputs(block_plan, handle_inputs); + } + InputReference::SwapDims { original_pos, dims } => { + let reference = handle_inputs + .get(original_pos) + .unwrap() + .as_normal() + .expect("Quant can't be used in swap dims operation"); + block_plan.reference = ReferenceSelection::SwapDims { + original: FuseArg::Input( + original_pos, + reference.precision, + LayoutInfo::Unknown, + ), + dims, + }; + } + InputReference::Reshaped { reshape_pos } => { + block_plan.reference = ReferenceSelection::Reshaped { reshape_pos }; + } + }; + None + } else { + Some(block.shape_ref.clone()) + } + } + + fn add_layout_info_inputs(block: &mut BlockPlan<'_>, handle_inputs: &[HandleInput]) { + for hi in handle_inputs.iter().filter_map(|h| match h { + HandleInput::Normal(input) => Some(input), + _ => None, + }) { + let (strides, shape) = match &block.reference { + ReferenceSelection::Concrete { strides, shape, .. } + | ReferenceSelection::VirtualShape { strides, shape, .. } => (strides, shape), + _ => continue, + }; + + if strides == &hi.handle.strides + && shape == &hi.global_ir.shape + && let Some(ops) = block.reads.get_mut(&hi.relative_id) + { + for op in ops.iter_mut() { + if let FuseOp::Assign(op) = op { + op.input.add_layout_info(LayoutInfo::SameAsRef); + } + } + } + } + } + + fn output_kind( + &self, + plan: &mut LaunchPlan<'a, R>, + tensor_global: &TensorIr, + output: &OutputSorted, + strides: &[usize], + ) -> (OutputKind, usize) { + let mut block_idx = None; + for (i, block) in plan.blocks.iter().enumerate() { + if block.writes.contains_key(&output.tensor_relative.id) { + block_idx = Some(i); + break; + } + } + let block_idx = block_idx.unwrap(); + + if let Some(transform) = self.resources.views.iter().find(|v| match v { + TensorView::Reshape { reshaped, .. } => reshaped == &output.tensor_relative.id, + TensorView::SwapDims { swapped, .. } => swapped == &output.tensor_relative.id, + }) { + return (OutputKind::Transform(transform.clone()), block_idx); + } + + let block = &plan.blocks[block_idx]; + let kind = block + .potential_inplaces + .iter() + .enumerate() + .find(|(_pos, pi)| { + pi.tensor_relative.dtype == tensor_global.dtype + && pi.tensor_relative.shape == output.tensor_relative.shape + && &*pi.strides == strides + && block.reference.compatible_strides_for_inplace(strides) + }) + .map(|(pos, _)| OutputKind::Inplace { input_pos: pos }) + .unwrap_or(OutputKind::Normal); + + (kind, block_idx) + } + + fn inplace_output( + &mut self, + context: &mut Context<'_, CubeFusionHandle>, + plan: &mut LaunchPlan<'a, R>, + output: OutputSorted, + tensor_global: TensorIr, + input_index: usize, + block_idx: usize, + ) { + let block = &mut plan.blocks[block_idx]; + let potential_inplace = block.potential_inplaces.remove(input_index); + let handle_input = match plan.handle_inputs.get(potential_inplace.input_pos).unwrap() { + HandleInput::Normal(handle) => handle, + _ => { + unreachable!("Quant tensor handle can't be used inplace yet.") + } + }; + + if !block.reference.is_found() + && !matches!( + self.blocks[block_idx].settings.ref_layout, + RefLayoutSetting::SameAsBlock { .. } + ) + { + let index_input = self + .resources + .inputs + .get_index(potential_inplace.tensor_relative.id) + .unwrap(); + + block.reference = ReferenceSelection::Concrete { + layout: FuseArg::Input(index_input, output.precision, LayoutInfo::IsRef), + shape: tensor_global.shape.clone(), + strides: handle_input.handle.strides.clone(), + }; + + if let Some(ops) = block.reads.get_mut(&handle_input.relative_id) { + for op in ops.iter_mut() { + if let FuseOp::Assign(op) = op { + op.input.add_layout_info(LayoutInfo::IsRef); + break; + }; + } + } + + if let Some(ops) = block.writes.get_mut(&output.tensor_relative.id) { + for op in ops { + if let FuseOp::Assign(op) = op { + op.out.add_layout_info(LayoutInfo::IsRef); + break; + } + } + }; + } else { + // Already validated, necessary for correctness. + if let Some(ops) = block.writes.get_mut(&output.tensor_relative.id) { + for op in ops { + if let FuseOp::Assign(op) = op { + op.out.add_layout_info(LayoutInfo::SameAsRef); + break; + } + } + }; + } + + context + .handles + .register_handle(tensor_global.id, handle_input.handle.clone()); + + self.handles[output.pos_original] = Some(HandleOutput::Alias { + input_pos: potential_inplace.input_pos, + precision: output.precision, + #[cfg(feature = "autotune-checks")] + debug_info: super::HandleOutputAliasDebugInfo { + relative_id: output.tensor_relative.id, + handle: handle_input.handle.clone(), + global_shape: tensor_global.shape.dims.clone(), + }, + }); + self.globals[output.pos_original] = Some(tensor_global); + } + + #[allow(clippy::too_many_arguments)] + fn normal_output( + &mut self, + client: &ComputeClient, + device: &R::Device, + context: &mut Context<'_, CubeFusionHandle>, + plan: &mut LaunchPlan<'a, R>, + output: OutputSorted, + tensor_global: TensorIr, + strides: Strides, + block_idx: usize, + ) { + let block = &mut plan.blocks[block_idx]; + + if !block.reference.is_found() + && self.blocks[block_idx].shape_ref == output.tensor_relative.shape + && !matches!( + self.blocks[block_idx].settings.ref_layout, + RefLayoutSetting::SameAsBlock { .. } + ) + { + block.reference = ReferenceSelection::Concrete { + layout: FuseArg::Output(output.pos_original, output.precision, LayoutInfo::IsRef), + shape: tensor_global.shape.clone(), + strides: strides.clone(), + }; + + // Sometimes outputs that are manually handled don't have any write registered. + if let Some(ops) = block.writes.get_mut(&output.tensor_relative.id) { + for op in ops { + if let FuseOp::Assign(op) = op { + op.out.add_layout_info(LayoutInfo::IsRef); + break; + } + } + }; + } else if let ReferenceSelection::Concrete { + shape: ref_shape, + strides: ref_strides, + .. + } = &block.reference + && ref_strides == &strides + && ref_shape == &tensor_global.shape + && let Some(ops) = block.writes.get_mut(&output.tensor_relative.id) + { + for op in ops { + if let FuseOp::Assign(op) = op { + op.out.add_layout_info(LayoutInfo::SameAsRef); + break; + } + } + }; + + // We encode bool tensors as `B`. + let dtype = match tensor_global.dtype { + DType::Bool => elem_dtype::(), + _ => tensor_global.dtype, + }; + let size = tensor_global.shape.iter().product::() * StorageType::from(dtype).size(); + + let handle = CubeFusionHandle { + client: client.clone(), + handle: client.empty(size), + device: device.clone(), + strides, + dtype, + qparams: None, + }; + + plan.rank = usize::max(tensor_global.shape.rank(), plan.rank); + context + .handles + .register_handle(tensor_global.id, handle.clone()); + + self.handles[output.pos_original] = Some(HandleOutput::Owned { + precision: output.precision, + handle, + global_shape: tensor_global.shape.clone(), + global_id: tensor_global.id, + relative_id: output.tensor_relative.id, + vectorization: 1, + }); + self.globals[output.pos_original] = Some(tensor_global); + } + + #[allow(clippy::too_many_arguments)] + fn reshaped_output( + &mut self, + client: &ComputeClient, + device: &R::Device, + context: &mut Context<'_, CubeFusionHandle>, + plan: &mut LaunchPlan<'a, R>, + output: OutputSorted, + tensor_global: TensorIr, + strides: Strides, + original: TensorId, + block_idx: usize, + ) { + let block = &mut plan.blocks[block_idx]; + + let (pos_input, original_handle) = Self::find_child_input(&plan.handle_inputs, original); + + // We encode bool tensors as `B`. + let dtype = match tensor_global.dtype { + DType::Bool => elem_dtype::(), + _ => tensor_global.dtype, + }; + + let action = reshape_action( + &original_handle.global_ir.shape, + &original_handle.handle.strides, + &tensor_global.shape, + ); + + let update = match action { + ReshapeAction::UpdateStrides { strides } => Some(strides), + ReshapeAction::NoChange => Some(original_handle.handle.strides.clone()), + ReshapeAction::Recompute => None, + }; + + match update { + Some(strides) => { + // We modify the metadata instead. + remove_concrete_write(block, output.tensor_relative.id, output.pos_original); + + let handle = CubeFusionHandle { + client: client.clone(), + handle: original_handle.handle.handle.clone(), + device: device.clone(), + strides, + dtype, + qparams: original_handle.handle.qparams.clone(), + }; + context + .handles + .register_handle(tensor_global.id, handle.clone()); + + // IT will never be access, just a way to keep the original position working. + self.handles[output.pos_original] = Some(HandleOutput::Alias { + input_pos: pos_input, + precision: output.precision, + #[cfg(feature = "autotune-checks")] + debug_info: super::HandleOutputAliasDebugInfo { + relative_id: output.tensor_relative.id, + handle: handle.clone(), + global_shape: tensor_global.shape.dims.clone(), + }, + }); + self.globals[output.pos_original] = Some(tensor_global); + } + None => { + self.normal_output::( + client, + device, + context, + plan, + output, + tensor_global, + strides, + block_idx, + ); + } + } + } + + #[allow(clippy::too_many_arguments)] + fn swapped_dims_output( + &mut self, + client: &ComputeClient, + device: &R::Device, + context: &mut Context<'_, CubeFusionHandle>, + plan: &mut LaunchPlan<'a, R>, + output: OutputSorted, + tensor_global: TensorIr, + original: TensorId, + dims: (usize, usize), + block_idx: usize, + ) { + let block = &mut plan.blocks[block_idx]; + let (pos_input, original_handle) = Self::find_child_input(&plan.handle_inputs, original); + + // We encode bool tensors as `B`. + let dtype = match tensor_global.dtype { + DType::Bool => elem_dtype::(), + _ => tensor_global.dtype, + }; + + // TODO: Check if we can also remove the read, if we have a dead partial graph. + // + // We modify the metadata instead. + remove_concrete_write(block, output.tensor_relative.id, output.pos_original); + + let strides = original_handle.handle.strides.clone(); + + let mut handle = CubeFusionHandle { + client: client.clone(), + handle: original_handle.handle.handle.clone(), + device: device.clone(), + strides, + dtype, + qparams: original_handle.handle.qparams.clone(), + }; + handle.strides.swap(dims.0, dims.1); + + context + .handles + .register_handle(tensor_global.id, handle.clone()); + + // IT will never be access, just a way to keep the original position working. + self.handles[output.pos_original] = Some(HandleOutput::Alias { + input_pos: pos_input, + precision: output.precision, + #[cfg(feature = "autotune-checks")] + debug_info: super::HandleOutputAliasDebugInfo { + relative_id: output.tensor_relative.id, + handle: handle.clone(), + global_shape: tensor_global.shape.dims.clone(), + }, + }); + self.globals[output.pos_original] = Some(tensor_global); + } + + fn find_child_input( + handle_inputs: &[HandleInput], + original: TensorId, + ) -> (usize, &NormalHandleInput) { + handle_inputs + .iter() + .enumerate() + .find_map(|(pi, handle)| match handle { + HandleInput::Normal(handle) => match handle.relative_id == original { + true => Some((pi, handle)), + false => None, + }, + _ => None, // Quant tensor can't be reshaped. + }) + .unwrap() + } +} + +fn remove_concrete_write(block: &mut BlockPlan, id: TensorId, output_pos: usize) { + let ops = block.writes.remove(&id); + + if let Some(ops) = ops { + let mut keep = Vec::with_capacity(ops.len()); + + for op in ops { + if let FuseOp::Assign(args) = &op { + if let FuseArg::Output(pos, ..) = args.out { + if pos != output_pos { + keep.push(op); + } + } else { + keep.push(op); + } + } + } + block.writes.insert(id, keep); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/plan.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/plan.rs new file mode 100644 index 0000000..1ba8c23 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/plan.rs @@ -0,0 +1,273 @@ +use crate::{ + CubeFusionHandle, + engine::{ + codegen::ir::{FuseArg, FuseOp, FuseType}, + launch::vectorization::Vect, + trace::{RuntimeLayout, block::FuseBlock}, + }, +}; +use burn_ir::{TensorId, TensorIr}; +use burn_std::{Shape, Strides}; +use cubecl::{Runtime, ir::LineSize}; +use std::collections::BTreeMap; + +/// The `LaunchPlan` is responsible for aggregating all runtime information required +/// to dispatch a fused kernel. +/// +/// It maps abstract IR tensors to memory handles, manages vectorization +/// strategies, and tracks layout transformations. +#[derive(Debug)] +pub struct LaunchPlan<'a, R: Runtime> { + /// The IR representation of tensors that are results of the fusion. + pub global_outputs: Vec, + /// Memory handles and metadata for all input tensors. + pub handle_inputs: Vec>, + /// Memory handles and metadata for all output tensors, including aliased inputs. + pub handle_outputs: Vec>, + /// The rank across all tensors in the plan. + /// + /// Smaller tensors are unsqueezed during launch. + pub rank: usize, + /// Detailed planning for each individual computation block within the fusion. + pub blocks: Vec>, + /// Mapping of tensor IDs to their specific vectorization factors. + pub vectorizations: BTreeMap, + /// Tensors that can be cleared or deallocated after this plan executes. + pub cleared: Vec, + /// Metadata for shapes and strides passed from the host when they cannot be + /// inferred from input tensors (e.g., complex deep fusions). + pub runtime_layouts: Vec, +} + +/// Information regarding the execution of a specific block of operations within a fusion. +#[derive(Debug)] +pub struct BlockPlan<'a> { + /// List of inputs that are candidates for in-place memory reuse within this block. + pub potential_inplaces: Vec>, + /// The input tensor chosen to define the iteration space, if any. + pub potential_reference_input: Option, + /// How the master layout is determined for this block. + pub reference: ReferenceSelection, + /// Mapping of tensor IDs to the read operations performed on them. + pub reads: BTreeMap>, + /// Mapping of tensor IDs to the write operations performed on them. + pub writes: BTreeMap>, + /// The width for the operations in this block. + pub width: LineSize, +} + +/// Metadata for an input tensor being used as a reference for a block's layout. +#[derive(Debug)] +pub enum InputReference { + /// Standard input at the specified position. + Normal { input_pos: usize }, + /// Input that has an axis swapped. + SwapDims { + original_pos: usize, + dims: (usize, usize), + }, + /// Input that has been reshaped. + Reshaped { reshape_pos: usize }, +} + +/// Strategies for selecting the reference layout of a fused block. +/// +/// The reference layout determines how global indices are mapped to tensor coordinates. +#[derive(Clone, Debug)] +pub enum ReferenceSelection { + /// The engine is still calculating the optimal reference. + Searching, + /// Layout from a normal tensor. + Concrete { + layout: FuseArg, + shape: Shape, + strides: Strides, + }, + /// Layout from a swapped dim tensor. + SwapDims { + original: FuseArg, + dims: (usize, usize), + }, + /// Layout from a reshaped tensor. + Reshaped { reshape_pos: usize }, + /// Layout that has the shape of an input, but not its strides. + VirtualShape { + original: FuseArg, + shape: Shape, + strides: Strides, + }, + /// The layout is provided dynamically by the host at runtime. + Runtime { pos: usize }, +} + +impl LaunchPlan<'_, R> { + /// Creates a new `LaunchPlan` from a slice of fusion blocks. + /// + /// Initializes blocks with default "Searching" references and calculates + /// the initial max rank. + pub fn new(fuse_blocks: &[FuseBlock]) -> Self { + let mut rank = 0; + let mut blocks = Vec::with_capacity(fuse_blocks.len()); + + for b in fuse_blocks.iter() { + rank = usize::max(b.shape_ref.len(), rank); + let block = BlockPlan { + reference: ReferenceSelection::Searching, + reads: b.reads.clone(), + writes: b.writes.clone(), + width: 0, + potential_inplaces: Vec::new(), + potential_reference_input: None, + }; + blocks.push(block); + } + + LaunchPlan { + global_outputs: Vec::new(), + handle_inputs: Vec::new(), + handle_outputs: Vec::new(), + rank, + blocks, + vectorizations: Default::default(), + cleared: Default::default(), + runtime_layouts: Default::default(), + } + } +} + +/// Debugging information for aliased handles when `autotune-checks` is enabled. +#[cfg(feature = "autotune-checks")] +#[derive(Debug)] +pub struct HandleOutputAliasDebugInfo { + pub handle: CubeFusionHandle, + pub relative_id: TensorId, + pub global_shape: Shape, +} + +/// Represents the output of a fused kernel execution. +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum HandleOutput { + /// An output that reuses the memory of an input tensor (In-place). + Alias { + /// Index of the input handle being aliased. + input_pos: usize, + /// Data type precision. + precision: FuseType, + #[cfg(feature = "autotune-checks")] + debug_info: HandleOutputAliasDebugInfo, + }, + /// An output that requires a newly allocated memory buffer. + Owned { + global_id: TensorId, + relative_id: TensorId, + precision: FuseType, + handle: CubeFusionHandle, + global_shape: Shape, + vectorization: LineSize, + }, +} + +/// A standard input handle with associated layout and vectorization metadata. +#[derive(Debug)] +pub struct NormalHandleInput { + pub relative_id: TensorId, + pub global_ir: TensorIr, + pub precision: FuseType, + pub handle: CubeFusionHandle, + pub line_size: LineSize, + pub broadcated: bool, + /// Stores the original strides of the handle for restoration during plan rollback. + pub orig_strides: Strides, +} + +/// An input handle containing values for a quantized tensor. +#[derive(Debug)] +pub struct QuantValuesHandleInput { + pub relative_id: TensorId, + pub global_ir: TensorIr, + pub precision: FuseType, + pub handle: CubeFusionHandle, + pub line_size: LineSize, +} + +/// An input handle containing parameters (scales/offsets) for quantization. +#[derive(Debug)] +pub struct QuantParamsHandleInput { + pub precision: FuseType, + pub handle: CubeFusionHandle, + pub shape: Shape, +} + +/// Different types of inputs that can be passed to a fused kernel. +#[derive(Debug)] +pub enum HandleInput { + Normal(NormalHandleInput), + QuantValues(QuantValuesHandleInput), + QuantParams(QuantParamsHandleInput), +} + +impl HandleInput { + /// Returns a reference to the inner `NormalHandleInput` if the variant matches. + pub fn as_normal(&self) -> Option<&NormalHandleInput> { + match self { + HandleInput::Normal(normal) => Some(normal), + _ => None, + } + } +} + +impl NormalHandleInput { + /// Creates a new `NormalHandleInput` tracking original strides. + pub fn new( + tensor_global: TensorIr, + tensor_relative: &TensorIr, + precision: FuseType, + mut handle: CubeFusionHandle, + mut strides: Strides, + ) -> Self { + // Swap current handle strides with provided strides to track the original state for rollback. + core::mem::swap(&mut handle.strides, &mut strides); + Self { + precision, + handle, + relative_id: tensor_relative.id, + global_ir: tensor_global, + line_size: 1, + broadcated: false, + orig_strides: strides, + } + } + + /// Restores the handle's original strides and returns the handle. + /// + /// Used when a plan is invalidated or needs to be rolled back. + pub fn handle_rollback(mut self) -> CubeFusionHandle { + core::mem::swap(&mut self.handle.strides, &mut self.orig_strides); + self.handle + } +} + +/// A candidate for in-place optimization. +#[derive(Debug)] +pub struct PotentialInplace<'a> { + /// Position of the input handle in the `handle_inputs` vector. + pub input_pos: usize, + /// Reference to the IR of the relative tensor. + pub tensor_relative: &'a TensorIr, + /// Current strides of the potential in-place candidate. + pub strides: Strides, +} + +impl ReferenceSelection { + pub fn is_found(&self) -> bool { + !matches!(self, Self::Searching) + } + + pub fn compatible_strides_for_inplace(&self, strides_inplace: &[usize]) -> bool { + match self { + ReferenceSelection::Concrete { strides, .. } => &**strides == strides_inplace, + _ => false, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/runner.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/runner.rs new file mode 100644 index 0000000..e08d243 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/runner.rs @@ -0,0 +1,96 @@ +use super::super::codegen::ir::{FuseBlockConfig, GlobalArgsLaunch}; +use crate::{ + CubeFusionHandle, + engine::launch::{ + LaunchPlan, + vectorization::{Vect, vectorization_default}, + }, +}; +use burn_fusion::stream::Context; +use burn_ir::{TensorId, TensorIr}; +use cubecl::prelude::*; +use std::collections::{BTreeMap, HashMap}; + +/// A trace runner is responsible for determining the vectorization factor as well as launching +/// a kernel based on global [inputs](GlobalArgsLaunch) and [outputs](GlobalArgsLaunch) +/// with provided [fuse block configs](FuseBlockConfig). +pub trait TraceRunner: Vectorization { + /// The error that might happen while running the trace. + type Error; + + /// Run the trace with the given inputs and outputs. + /// + /// There is one [fuse config](FuseBlockConfig) for each [block](super::block::FuseBlock) registered + /// in the [optimization builder](burn_fusion::OptimizationBuilder). + fn run<'a>( + &'a self, + client: &'a ComputeClient, + inputs: GlobalArgsLaunch<'a, R>, + outputs: GlobalArgsLaunch<'a, R>, + configs: &'a [FuseBlockConfig], + ) -> Result<(), Self::Error>; +} + +pub enum VectorizationHandle<'a, R: Runtime> { + NormalInput(&'a CubeFusionHandle, &'a TensorIr), + QuantValues(&'a CubeFusionHandle, &'a TensorIr), + QuantParams, +} + +impl<'a, R: Runtime> VectorizationHandle<'a, R> { + /// Returns if the current vectorization handle is from the given tensor id. + pub fn is_from_tensor(&self, id: TensorId) -> bool { + match self { + VectorizationHandle::NormalInput(_, tensor_ir) => tensor_ir.id == id, + VectorizationHandle::QuantValues(_, tensor_ir) => tensor_ir.id == id, + VectorizationHandle::QuantParams => false, + } + } +} + +#[derive(Default)] +pub struct VectorizationAxis { + axis: HashMap, +} + +impl VectorizationAxis { + pub fn get usize>(&self, id: TensorId, default: F) -> usize { + self.axis.get(&id).copied().unwrap_or_else(default) + } + pub fn insert(&mut self, id: TensorId, axis: usize) { + self.axis.insert(id, axis); + } +} + +pub trait Vectorization { + /// Returns the vectorization options. + fn axis(&self, _plan: &LaunchPlan<'_, R>) -> VectorizationAxis { + VectorizationAxis::default() + } + /// The vectorization factor for all inputs and outputs. + #[allow(clippy::too_many_arguments)] + fn vectorization<'a>( + &self, + _context: &Context<'_, CubeFusionHandle>, + vectorizations: &mut BTreeMap, + inputs: impl Iterator>, + outputs: impl Iterator, + reshaped: impl Iterator, + swapped: impl Iterator, + line_sizes: &[LineSize], + max: LineSize, + axis: VectorizationAxis, + ) { + vectorization_default( + vectorizations, + inputs, + outputs, + reshaped, + swapped, + line_sizes, + &Default::default(), + max, + &axis, + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/base.rs new file mode 100644 index 0000000..c0a45d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/base.rs @@ -0,0 +1,439 @@ +use crate::{ + CubeFusionHandle, + engine::launch::runner::{VectorizationAxis, VectorizationHandle}, +}; +use burn_fusion::stream::Context; +use burn_ir::{TensorId, TensorIr}; +use cubecl::{Runtime, ir::LineSize}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Copy)] +pub enum Vect { + Broadcasted, + Aligned(LineSize), +} + +impl Vect { + pub fn line_size(&self) -> LineSize { + match self { + Vect::Broadcasted => 1, + Vect::Aligned(val) => *val, + } + } + + pub fn is_broadcast(&self) -> bool { + matches!(self, Vect::Broadcasted) + } +} + +#[derive(Default, Clone, Serialize, Deserialize, Debug)] +pub struct LineSizeOverrides { + state: Option>>, + default: Option>, +} + +#[allow(unused)] +impl LineSizeOverrides { + pub fn overrides(&mut self, tensor_id: &TensorId, line_sizes: Vec) { + let map = match &mut self.state { + Some(val) => val, + None => { + self.state = Some(BTreeMap::new()); + self.state.as_mut().unwrap() + } + }; + + map.insert(*tensor_id, line_sizes); + } + pub fn overrides_default(&mut self, line_sizes: Vec) { + self.default = Some(line_sizes); + } + + pub fn mapping(&self, context: &Context<'_, CubeFusionHandle>) -> Self { + match &self.state { + Some(state) => { + let mut state_new = BTreeMap::new(); + + for (k, v) in state.iter() { + let global = context.tensors.get(k).unwrap(); + state_new.insert(global.id, v.clone()); + } + + Self { + state: Some(state_new), + default: self.default.clone(), + } + } + None => Self { + state: None, + default: self.default.clone(), + }, + } + } + + pub fn tensor(&self, tensor_id: &TensorId) -> Option<&Vec> { + let map = match &self.state { + Some(val) => val, + None => match &self.default { + Some(val) => return Some(val), + None => return None, + }, + }; + + match map.get(tensor_id) { + Some(val) => Some(val), + None => match &self.default { + Some(val) => Some(val), + None => None, + }, + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn vectorization_default<'a, R: Runtime>( + vectorizations: &mut BTreeMap, + inputs: impl Iterator>, + outputs: impl Iterator, + reshaped: impl Iterator, + swapped: impl Iterator, + line_sizes: &[LineSize], + overrides: &LineSizeOverrides, + max: LineSize, + axis: &VectorizationAxis, +) { + let swapped: Vec<_> = swapped.collect(); + + for input in inputs { + if let Some((s, o, mr, dims)) = swapped + .iter() + .find(|(_s, o, _mr, _dims)| input.is_from_tensor(o.id)) + { + let (handle, id) = match input { + VectorizationHandle::NormalInput(handle, tensor_ir) => (handle, &tensor_ir.id), + VectorizationHandle::QuantValues(..) => panic!("Can't be swapped"), + VectorizationHandle::QuantParams => panic!("Can't be swapped"), + }; + let val = vectorization_swapped( + handle, + s, + o, + *mr, + dims, + max, + axis, + line_sizes, + overrides.tensor(id), + ); + multi_reads_vectorization_update(vectorizations, o.id, val); + } else { + match input { + VectorizationHandle::NormalInput(handle, tensor_ir) => { + let val = vectorization_input( + handle, + tensor_ir, + axis, + line_sizes, + overrides.tensor(&tensor_ir.id), + ); + vectorizations.insert(tensor_ir.id, val); + } + VectorizationHandle::QuantValues(handle, tensor_ir) => { + let val = vectorization_input( + handle, + tensor_ir, + axis, + line_sizes, + overrides.tensor(&tensor_ir.id), + ); + let num_quants = match tensor_ir.dtype { + burn_std::DType::QFloat(quant_scheme) => quant_scheme.num_quants(), + _ => panic!(""), + }; + let val = match val { + Vect::Broadcasted => Vect::Aligned(1), + Vect::Aligned(val) => Vect::Aligned(val.div_ceil(num_quants)), + }; + vectorizations.insert(tensor_ir.id, val); + } + VectorizationHandle::QuantParams => { + // Doesn't have vectorization for now. + } + }; + } + } + + for (reshaped, original, multi_reads) in reshaped { + let val = vectorization_reshape( + reshaped, + original, + multi_reads, + axis, + line_sizes, + max, + overrides.tensor(&original.id), + ); + multi_reads_vectorization_update(vectorizations, original.id, val); + } + + for tensor in outputs { + let val = vectorization_output(tensor, axis, line_sizes, max, overrides.tensor(&tensor.id)); + vectorizations.insert(tensor.id, val); + } +} + +fn multi_reads_vectorization_update( + vectorizations: &mut BTreeMap, + original: TensorId, + vect: Vect, +) { + if let Some(ori_vect) = vectorizations.get(&original).cloned() { + match ori_vect { + Vect::Broadcasted => { + // keep the original as is. + } + Vect::Aligned(ori) => match vect { + Vect::Broadcasted => { + vectorizations.insert(original, Vect::Aligned(1)); + } + Vect::Aligned(new) => { + let val = if new != ori { 1 } else { new }; + vectorizations.insert(original, Vect::Aligned(val)); + } + }, + }; + } else { + vectorizations.insert(original, vect); + } +} + +// The default version uses the last dimension as vectorization axis and assumes a +// perpendicular contiguous line. +fn vectorization_input( + handle: &CubeFusionHandle, + desc: &TensorIr, + axis: &VectorizationAxis, + line_sizes: &[LineSize], + overrides: Option<&Vec>, +) -> Vect { + let axis = axis.get(desc.id, || handle.strides.len() - 1); + let shape_axis = desc.shape[axis]; + + if shape_axis == 1 { + return Vect::Broadcasted; + } + + // Last dimension strides should be 1, otherwise vecX won't be contiguous. + if handle.strides[axis] != 1 { + return Vect::Aligned(1); + } + + let inner = |s: LineSize| { + // The last dimension should be a multiple of the vector size or broadcated. + if shape_axis.is_multiple_of(s) { + return Some(Vect::Aligned(s)); + } + None + }; + + match overrides { + Some(vals) => { + for s in vals { + if let Some(val) = inner(*s) { + return val; + } + } + } + None => { + for s in line_sizes { + if let Some(val) = inner(*s) { + return val; + } + } + } + } + + Vect::Aligned(1) +} + +fn vectorization_output( + desc: &TensorIr, + axis: &VectorizationAxis, + line_sizes: &[LineSize], + max: LineSize, + overrides: Option<&Vec>, +) -> Vect { + let axis = axis.get(desc.id, || desc.shape.rank() - 1); + + let inner = |s: LineSize| { + // The dimension should be a multiple of the vector size. + if desc.shape[axis].is_multiple_of(s) && s <= max { + return Some(Vect::Aligned(s)); + } + + None + }; + match overrides { + Some(val) => { + for s in val { + if let Some(val) = inner(*s) { + return val; + } + } + } + None => { + for s in line_sizes { + if let Some(val) = inner(*s) { + return val; + } + } + } + } + + Vect::Aligned(1) +} + +fn vectorization_reshape( + reshaped: &TensorIr, + original: &TensorIr, + multi_reads: bool, + axis: &VectorizationAxis, + line_sizes: &[LineSize], + max: LineSize, + overrides: Option<&Vec>, +) -> Vect { + let axis = axis.get(reshaped.id, || reshaped.shape.rank() - 1); + let reshape_shape_axis = reshaped.shape[axis]; + + if !multi_reads && reshape_shape_axis == 1 { + return Vect::Broadcasted; + } + + // If the axis is not the last dim, didn't think of it, return Aligned(1) to be sure. + if axis != reshaped.shape.rank() - 1 { + return Vect::Aligned(1); + } + + let original_shape_axis = original.shape[original.shape.rank() - 1]; + + if original_shape_axis != reshape_shape_axis { + return Vect::Aligned(1); + } + + let inner = |s: LineSize| { + if !multi_reads { + // The last dimension should be a multiple of the vector size or broadcated. + if reshape_shape_axis.is_multiple_of(s) && s <= max { + Some(Vect::Aligned(s)) + } else { + None + } + } else { + // Since the original tensor must share the same vectorization factor as the + // reshaped tensor, they must have compatible shapes when both are access + // independently. + if reshape_shape_axis.is_multiple_of(s) + && original_shape_axis.is_multiple_of(s) + && s <= max + { + Some(Vect::Aligned(s)) + } else { + None + } + } + }; + + match overrides { + Some(val) => { + for i in val { + if let Some(vect) = inner(*i) { + return vect; + } + } + } + None => { + for s in line_sizes { + if let Some(vect) = inner(*s) { + return vect; + } + } + } + } + + Vect::Aligned(1) +} + +#[allow(clippy::too_many_arguments)] +fn vectorization_swapped( + handle: &CubeFusionHandle, + swapped: &TensorIr, + original: &TensorIr, + multi_reads: bool, + dims: &(usize, usize), + max: LineSize, + axis: &VectorizationAxis, + line_sizes: &[LineSize], + overrides: Option<&Vec>, +) -> Vect { + let axis = axis.get(swapped.id, || swapped.shape.rank() - 1); + + let swapped_axis = swapped.shape[axis]; + let shape_axis = original.shape[axis]; + + let axis_index = axis; + let dim_index = if dims.0 == axis_index { + dims.1 + } else if dims.1 == axis_index { + dims.0 + } else { + axis_index + }; + + // Last dimension strides should be 1, otherwise vecX won't be contiguous. + if multi_reads { + if handle.strides[axis_index] != 1 { + return Vect::Aligned(1); + } + if handle.strides[dim_index] != 1 { + return Vect::Aligned(1); + } + } else if handle.strides[dim_index] != 1 { + return Vect::Aligned(1); + } + + if !multi_reads && swapped_axis == 1 { + return Vect::Broadcasted; + } + + let inner = |s: LineSize| { + // The last dimension should be a multiple of the vector size or broadcated. + if multi_reads { + if swapped_axis.is_multiple_of(s) && s <= max { + return Some(Vect::Aligned(s)); + } + } else if swapped_axis.is_multiple_of(s) && shape_axis.is_multiple_of(s) && s <= max { + return Some(Vect::Aligned(s)); + } + None + }; + + match overrides { + Some(val) => { + for s in val { + if let Some(val) = inner(*s) { + return val; + } + } + } + None => { + for s in line_sizes { + if let Some(val) = inner(*s) { + return val; + } + } + } + } + + Vect::Aligned(1) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/mod.rs new file mode 100644 index 0000000..e7de0b2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod planner; + +pub use base::*; +pub use planner::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/planner.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/planner.rs new file mode 100644 index 0000000..8697ad0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/launch/vectorization/planner.rs @@ -0,0 +1,438 @@ +use super::{ + super::{BlockPlan, HandleOutput, LaunchPlan}, + Vect, +}; +use crate::{ + CubeFusionHandle, + engine::{ + launch::{ + HandleInput, + runner::{Vectorization, VectorizationHandle}, + }, + settings::VectorizationSetting, + trace::{FuseResources, TensorView, block::FuseBlock}, + }, +}; +use burn_fusion::stream::Context; +use burn_ir::TensorId; +use cubecl::{ + Runtime, + client::ComputeClient, + ir::{ElemType, StorageType, UIntKind}, +}; +use cubecl::{ + ir::LineSize, + quant::scheme::{QuantScheme, QuantStore, QuantValue}, +}; +use std::marker::PhantomData; + +/// Select the best vectorization factor for each tensor handle. +pub struct VectorizationPlanner<'a, R: Runtime> { + resources: &'a FuseResources, + blocks: &'a Vec, + _r: PhantomData, +} + +impl<'a, R: Runtime> VectorizationPlanner<'a, R> { + pub fn new(resources: &'a FuseResources, blocks: &'a Vec) -> Self { + Self { + resources, + blocks, + _r: PhantomData, + } + } + pub fn run>( + self, + client: &ComputeClient, + runner: &Runner, + context: &Context<'_, CubeFusionHandle>, + plan: &mut LaunchPlan<'a, R>, + ) { + let has_multiple_read = |tensor: &TensorId| { + let mut read_count = 0; + for block in plan.blocks.iter() { + read_count += block.reads.get(tensor).map(|a| a.len()).unwrap_or(0); + } + read_count > 1 + }; + let tensors_reshaped = self.resources.views.iter().filter_map(|view| match view { + TensorView::Reshape { + reshaped, original, .. + } => Some(( + context.tensors.get(reshaped).unwrap(), + context.tensors.get(original).unwrap(), + has_multiple_read(original), + )), + TensorView::SwapDims { .. } => None, + }); + let tensors_swapped = self.resources.views.iter().filter_map(|view| match view { + TensorView::SwapDims { + swapped, + original, + dims, + .. + } => Some(( + context.tensors.get(swapped).unwrap(), + context.tensors.get(original).unwrap(), + has_multiple_read(original), + dims, + )), + TensorView::Reshape { .. } => None, + }); + + let mut ref_elem = (ElemType::UInt(UIntKind::U64).into(), 8); + let mut quants_line_sizes: Option> = None; + + for input in plan.handle_inputs.iter() { + let elem: StorageType = match input { + HandleInput::Normal(h) => h.global_ir.dtype.into(), + HandleInput::QuantValues(handle) => match handle.global_ir.dtype { + burn_std::DType::QFloat(scheme) => { + line_sizes_quants(client, &mut quants_line_sizes, scheme); + continue; + } + _ => panic!("Unable to retrieve the scheme for quantized values."), + }, + HandleInput::QuantParams(..) => continue, + }; + let elem_size = elem.size(); + + if ref_elem.1 >= elem_size { + ref_elem = (elem, elem_size); + } + } + for r in plan.global_outputs.iter() { + let elem: StorageType = r.dtype.into(); + let elem_size = elem.size(); + + if ref_elem.1 >= elem_size { + ref_elem = (elem, elem_size); + } + } + + let filtered = plan + .handle_inputs + .iter() + .map(|item| { + item.as_normal() + // Filter out indexed resources. + .map(|item| !self.resources.indexed.contains_key(&item.relative_id)) + .unwrap_or(true) + }) + .collect::>(); + + let line_sizes = match quants_line_sizes { + // Quantization normally triggers higher vectorization than anything else, no need to + // compare to ref elem. + Some(line_sizes) => line_sizes, + None => client + .io_optimized_line_sizes(ref_elem.0.size()) + .collect::>(), + }; + + let vectorization_axis = runner.axis(plan); + + runner.vectorization( + context, + &mut plan.vectorizations, + plan.handle_inputs + .iter() + .enumerate() + .filter_map(|(i, item)| { + if filtered[i] { + Some(match item { + HandleInput::Normal(h) => { + VectorizationHandle::NormalInput(&h.handle, &h.global_ir) + } + HandleInput::QuantValues(h) => { + VectorizationHandle::QuantValues(&h.handle, &h.global_ir) + } + HandleInput::QuantParams(_) => VectorizationHandle::QuantParams, + }) + } else { + None + } + }), + plan.global_outputs.iter(), + tensors_reshaped, + tensors_swapped, + &line_sizes, + u8::MAX as usize, + vectorization_axis, + ); + + for tensor in self.resources.indexed.keys() { + let global = context.tensors.get(tensor).unwrap(); + plan.vectorizations.insert(global.id, Vect::Aligned(1)); + } + + let mut block_vectorization = Vec::with_capacity(self.blocks.len()); + for _ in 0..self.blocks.len() { + block_vectorization.push(Vec::new()); + } + + for (input_pos, handle) in plan.handle_inputs.iter_mut().enumerate() { + let (global_ir, relative_id) = match handle { + HandleInput::Normal(h) => (&h.global_ir, &h.relative_id), + HandleInput::QuantValues(h) => (&h.global_ir, &h.relative_id), + HandleInput::QuantParams(_) => continue, + }; + let (vect, br) = match plan.vectorizations.get(&global_ir.id) { + Some(v) => (v.line_size(), v.is_broadcast()), + None => panic!("No vectorization factor found for {:?}", global_ir.id), + }; + + for (block_pos, block_plan) in plan.blocks.iter().enumerate() { + if block_plan.reads.contains_key(relative_id) { + block_vectorization[block_pos].push(BlockVectorization { + action: VectorizationAction::Input(input_pos), + potential: vect, + broadcasted: br, + }); + } + } + } + + for (output_pos, handle) in plan.handle_outputs.iter().enumerate() { + if let HandleOutput::Owned { + global_id, + relative_id, + .. + } = handle + { + for (block_pos, block_plan) in plan.blocks.iter().enumerate() { + if block_plan.writes.contains_key(relative_id) { + let vectorization = plan.vectorizations.get(global_id).unwrap().line_size(); + block_vectorization[block_pos].push(BlockVectorization { + action: VectorizationAction::Output(output_pos), + potential: vectorization, + broadcasted: false, + }); + } + } + } + } + + let mut previous_widths = Vec::with_capacity(block_vectorization.len()); + + // Unhandled inputs might not get included in any fused blocks for now. + // + // So we ensure they are vectorized by setting their vectorization before we set the + // vectorizations in blocks. + // + // Unhandled Outputs are correctly vectorized, so this is only necessary for inputs. + for input in self.resources.inputs_unhandled.iter() { + let pos = self + .resources + .inputs + .get_index(*input) + .unwrap_or_else(|| self.resources.inputs.get_index_quant(*input).unwrap()); + let input_global = context.tensors.get(input).unwrap(); + + match plan.vectorizations.get(&input_global.id).unwrap() { + Vect::Aligned(vect) => { + let handle = &mut plan.handle_inputs[pos]; + match handle { + HandleInput::Normal(handle) => { + handle.line_size = *vect; + } + HandleInput::QuantValues(handle) => { + handle.line_size = *vect; + } + HandleInput::QuantParams(_) => {} + } + } + Vect::Broadcasted => {} + } + } + + for ((tmp, block_plan), block) in block_vectorization + .into_iter() + .zip(plan.blocks.iter_mut()) + .zip(self.blocks) + { + match block.settings.vectorization { + VectorizationSetting::Activated => { + apply_vectorization_block( + tmp, + &mut plan.handle_inputs, + &mut plan.handle_outputs, + block_plan, + u8::MAX as usize, + ); + } + VectorizationSetting::SmallerOrEqualThanPreviousBlock { block_pos } => { + apply_vectorization_block( + tmp, + &mut plan.handle_inputs, + &mut plan.handle_outputs, + block_plan, + previous_widths[block_pos], + ); + if block_plan.width == 0 { + block_plan.width = previous_widths[block_pos]; + } + } + VectorizationSetting::EqualThanPreviousBlock { block_pos } => { + apply_vectorization_block( + tmp, + &mut plan.handle_inputs, + &mut plan.handle_outputs, + block_plan, + previous_widths[block_pos], + ); + // Enforces the width. + block_plan.width = previous_widths[block_pos]; + } + VectorizationSetting::Deactivated => { + apply_vectorization_block( + tmp, + &mut plan.handle_inputs, + &mut plan.handle_outputs, + block_plan, + 1, + ); + block_plan.width = 1; + } + } + + // When only virtual inputs/outputs are present for a block, we need to set a width. + if block_plan.width == 0 { + if let Some(w) = previous_widths.last() { + block_plan.width = *w; + } else { + block_plan.width = 1; + } + } + + previous_widths.push(block_plan.width); + } + } +} + +#[derive(Debug)] +enum VectorizationAction { + Input(usize), + Output(usize), +} + +#[derive(Debug)] +struct BlockVectorization { + action: VectorizationAction, + potential: LineSize, + broadcasted: bool, +} + +fn apply_vectorization_block( + block_vectorization: Vec, + inputs: &mut [HandleInput], + outputs: &mut [HandleOutput], + block_plan: &mut BlockPlan, + max: LineSize, +) { + for item in block_vectorization { + match item.action { + VectorizationAction::Input(pos) => { + let (vect, br) = if item.potential <= max { + (item.potential, item.broadcasted) + } else { + (1, false) + }; + + match &mut inputs[pos] { + HandleInput::Normal(input) => { + input.line_size = vect; + input.broadcated = br; + } + HandleInput::QuantValues(input) => { + input.line_size = vect; + } + HandleInput::QuantParams(_) => { + // Not vectorized + } + } + + if block_plan.width < vect { + block_plan.width = vect; + } + } + VectorizationAction::Output(pos) => { + if let HandleOutput::Owned { vectorization, .. } = &mut outputs[pos] { + let vect = if item.potential <= max { + item.potential + } else { + 1 + }; + *vectorization = vect; + + if block_plan.width < vect { + block_plan.width = vect; + } + } + } + } + } +} + +fn line_sizes_quants( + client: &ComputeClient, + quants_line_sizes: &mut Option>, + scheme: QuantScheme, +) { + match scheme.store { + QuantStore::Native => match scheme.value { + // Type sizes are the same so just treat fp8/fp4x2 as i8 + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::E4M3 + | QuantValue::E5M2 + | QuantValue::E2M1 => { + let line_sizes = client + .io_optimized_line_sizes(size_of::()) + .collect::>(); + + match &quants_line_sizes { + Some(sizes) => { + if sizes[0] < line_sizes[0] { + *quants_line_sizes = Some(line_sizes); + } + } + None => { + *quants_line_sizes = Some(line_sizes); + } + } + } + QuantValue::Q4F | QuantValue::Q4S | QuantValue::Q2F | QuantValue::Q2S => { + unreachable!("Can't store native sub-byte values") + } + }, + QuantStore::PackedU32(_) => { + let mut line_sizes = client + .io_optimized_line_sizes(size_of::()) + .collect::>(); + for val in line_sizes.iter_mut() { + *val *= scheme.num_quants(); + } + + match &quants_line_sizes { + Some(sizes) => { + if sizes[0] < line_sizes[0] { + let mut min = *line_sizes.last().unwrap(); + + while min > 1 { + min /= 2; + line_sizes.push(min); + } + *quants_line_sizes = Some(line_sizes); + } + } + None => { + *quants_line_sizes = Some(line_sizes); + } + } + } + QuantStore::PackedNative(_) => { + panic!("Not yet supported") + } + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/mod.rs new file mode 100644 index 0000000..8f7d49b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod codegen; +pub(crate) mod fuser; +pub(crate) mod launch; +pub(crate) mod settings; + +pub mod trace; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/settings.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/settings.rs new file mode 100644 index 0000000..bcfcd47 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/settings.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; + +/// Controls which operations can be fused. +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct FuseSettings { + /// Enables broadcasting of shapes. + pub broadcast: bool, + /// Enables output shape updates. + /// + /// When broadcast is enabled, the output shape can become bigger after a fusion, + /// therefore an update is needed. + pub output_shape_updates: bool, + /// Enables the reuse of input buffers. + pub inplace: bool, + /// Whether vectorization is enabled. + pub vectorization: VectorizationSetting, + /// How [reference layout](super::ir::RefLayout) selection is done. + pub ref_layout: RefLayoutSetting, +} + +impl Default for FuseSettings { + fn default() -> Self { + Self { + broadcast: true, + output_shape_updates: true, + inplace: true, + vectorization: VectorizationSetting::Activated, + ref_layout: RefLayoutSetting::Any, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +/// How vectorization is handled during fusion. +pub enum VectorizationSetting { + /// The biggest line_size possible will be used. + Activated, + /// Equivalent to using line_size of one. + Deactivated, + /// This is a good setting when a block processes values calculated from a previous block. + SmallerOrEqualThanPreviousBlock { block_pos: usize }, + /// This is a good setting when a block processes values calculated from a previous block. + EqualThanPreviousBlock { block_pos: usize }, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +/// Influence how the [reference layout](super::ir::RefLayout) selection is done. +pub enum RefLayoutSetting { + /// Any reference layout is allowed. + Any, + /// Only contiguous reference layout is allowed. + /// + /// Note that forcing a contiguous reference layout might reduce the opportunity of inplace + /// fusion. + OnlyContiguous, + SameAsBlock { + block_pos: u32, + }, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/base.rs new file mode 100644 index 0000000..7281a85 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/base.rs @@ -0,0 +1,377 @@ +use crate::engine::{ + codegen::ir::{FuseArg, FuseType}, + trace::block::FuseBlock, +}; +use burn_ir::{TensorId, TensorIr}; +use burn_std::{Shape, Strides}; +use cubecl::prelude::*; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, HashSet}, + marker::PhantomData, +}; + +#[cfg(feature = "autotune-checks")] +use crate::CubeFusionHandle; +#[cfg(feature = "autotune-checks")] +use burn_backend::TensorData; +#[cfg(feature = "autotune-checks")] +use std::collections::HashMap; + +#[derive(Clone, Serialize, Deserialize, Debug)] +/// A trace contains all [blocks](FuseBlock) and the [resources](FuseResources) used by the +/// kernel. +pub struct FuseTrace { + pub blocks: Vec, + pub resources: FuseResources, +} + +impl core::fmt::Display for FuseTrace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "FuseTrace")?; + for b in self.blocks.iter() { + writeln!(f, " - Block shape={:?}", b.shape_ref)?; + for (tensor, ops) in b.reads.iter() { + for op in ops.iter() { + writeln!(f, " - {op} <== {tensor}")?; + } + } + for op in b.ops.iter() { + writeln!(f, " - {op}")?; + } + for (tensor, ops) in b.writes.iter() { + for op in ops.iter() { + writeln!(f, " - {op} <== {tensor}")?; + } + } + } + + Ok(()) + } +} + +pub enum TuneOutput { + UnChecked(PhantomData), + #[cfg(feature = "autotune-checks")] + Checked { + handles: HashMap, CubeFusionHandle)>, + }, +} + +impl TuneOutput { + #[allow(unused_variables)] + pub fn merge(self, other: Self) -> Self { + let mut result = self; + + match &mut result { + TuneOutput::UnChecked(..) => {} + #[cfg(feature = "autotune-checks")] + TuneOutput::Checked { handles } => match other { + TuneOutput::UnChecked(..) => {} + TuneOutput::Checked { handles: o } => { + for (k, v) in o.into_iter() { + handles.insert(k, v); + } + } + }, + } + + result + } +} + +impl cubecl::tune::AutotuneOutput for TuneOutput { + #[cfg(feature = "autotune-checks")] + fn check_equivalence(&self, other: Self) { + use burn_backend::Tolerance; + use burn_std::DType; + + if let ( + TuneOutput::Checked { + handles: handles_ref, + }, + TuneOutput::Checked { handles }, + ) = (self, &other) + { + let mut num_checked = 0; + let mut num_handles = 0; + for (id, (shape, handle)) in handles_ref.iter() { + num_handles += 1; + if let Some((shape_other, other)) = handles.get(id) { + use burn_std::is_contiguous; + use cubecl::std::tensor::into_contiguous_ref; + + let current_handle = if !is_contiguous(&shape, &handle.strides) { + into_contiguous_ref::( + &handle.client, + &handle.as_handle_ref(&shape), + handle.dtype.into(), + ) + .unwrap() + .handle + } else { + handle.handle.clone() + }; + let other_handle = if !is_contiguous(&shape, &other.strides) { + into_contiguous_ref::( + &other.client, + &other.as_handle_ref(&shape), + other.dtype.into(), + ) + .unwrap() + .handle + } else { + other.handle.clone() + }; + + let data_ref = handle.client.read_one(current_handle); + let data_other = other.client.read_one(other_handle); + let data_ref = TensorData::from_bytes(data_ref, shape.clone(), handle.dtype); + let data_other = + TensorData::from_bytes(data_other, shape_other.clone(), handle.dtype); + + match handle.dtype { + DType::F64 => { + data_ref.assert_approx_eq::(&data_other, Tolerance::permissive()) + } + DType::F32 => { + data_ref.assert_approx_eq::(&data_other, Tolerance::permissive()) + } + DType::F16 => data_ref + .assert_approx_eq::(&data_other, Tolerance::permissive()), + DType::BF16 => data_ref + .assert_approx_eq::(&data_other, Tolerance::permissive()), + _ => data_ref.assert_eq(&data_other, true), + } + num_checked += 1; + } else { + // Debug info for the tests. + println!("No tensor found for {id:?}=>{shape:?}"); + } + } + + // At least one check is needed per output when there is an output. + // + // Some optimizations might write more outputs than needed, so it might be fined if + // the number of handles is different, but at least one is required. + // + // An optimization might not create outputs if its dead code detection is triggered, + // therefore avoiding useless computation. + if num_handles > 0 { + assert!(num_checked >= 1); + } + } + } +} + +#[derive(Clone, Serialize, Deserialize, Debug, Default)] +/// Declare all resources used by the kernel, and potentially multiple [blocks](FuseBlock). +/// +/// # Notes +/// +/// Each block can't contain their own resources, since they are shared between blocks. The +/// vectorization factor of one input tensor must be the same for all blocks. +pub struct FuseResources { + pub outputs: RegisteredTensors, + pub inputs: RegisteredTensors, + pub scalars: Vec<(FuseType, u64)>, + // TODO: Making put a map of global registers. + pub views: Vec, + pub indexed: BTreeMap, + pub inputs_unhandled: Vec, + pub outputs_unhandled: Vec, + pub num_reshaped: usize, + /// Necessary to remove some entries from the context. + pub dropped: HashSet, + /// We know during fusion that we have to have those buffers has global. + /// The pos here can be interpreted as GLOBAL pos where the output pos are locals. + pub buffers: RegisteredTensors, + /// Global registers available everywhere. + /// + /// TODO: Not all registers should be globals. + pub registers: BTreeMap, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct RuntimeLayout { + pub shape: Shape, + pub strides: Strides, +} + +impl Default for RuntimeLayout { + fn default() -> Self { + Self { + shape: Shape::new([]), + strides: Strides::new(&[]), + } + } +} + +#[derive(Debug)] +pub enum TraceError { + ReferenceNotFound, + RunnerError(Err), +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub enum TensorView { + Reshape { + reshaped: TensorId, + original: TensorId, + reshape_pos: usize, + shape_relative: Shape, + }, + SwapDims { + swapped: TensorId, + original: TensorId, + dims: (usize, usize), + }, +} + +#[derive(Default, Clone, Serialize, Deserialize, Debug)] +pub struct RegisteredTensors { + tensors: Vec, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub enum RegisterTensor { + Normal(TensorIr, FuseType), + QuantValues(TensorIr), + QuantParams(TensorId), +} + +impl RegisterTensor { + pub fn as_normal_tensor(&self) -> Option<(&TensorIr, &FuseType)> { + match self { + RegisterTensor::Normal(tensor_ir, precision) => Some((tensor_ir, precision)), + RegisterTensor::QuantValues(_) => None, + RegisterTensor::QuantParams(_) => None, + } + } +} + +impl RegisteredTensors { + /// Iterate over all the registered tensors. + pub fn iter(&self) -> impl Iterator { + self.tensors.iter() + } + + /// Consumes and iterate over all the registered tensors. + pub fn into_iter(self) -> impl Iterator { + self.tensors.into_iter() + } + + /// Returns the number of tensors registered. + pub fn len(&self) -> usize { + self.tensors.len() + } + + /// Retrieve the [tensor id](TensorId) at the given index. + pub fn get_id(&self, index: usize) -> Option { + self.tensors.get(index).map(|entry| match entry { + RegisterTensor::Normal(tensor_ir, _) => tensor_ir.id, + RegisterTensor::QuantValues(tensor_ir) => tensor_ir.id, + RegisterTensor::QuantParams(tensor_id) => *tensor_id, + }) + } + + /// Doesn't return quantized tensor. + pub fn get_index(&self, tensor_id: TensorId) -> Option { + self.tensors + .iter() + .enumerate() + .find(|(_pos, entry)| match entry { + RegisterTensor::Normal(tensor_ir, _) => tensor_ir.id == tensor_id, + RegisterTensor::QuantValues(_) => false, + RegisterTensor::QuantParams(_) => false, + }) + .map(|(pos, _)| pos) + } + + /// Get the index of a quantized tensor. + pub fn get_index_quant(&self, tensor_id: TensorId) -> Option { + self.tensors + .iter() + .enumerate() + .find(|(_pos, entry)| match entry { + RegisterTensor::Normal(..) => false, + RegisterTensor::QuantValues(tensor_ir) => tensor_ir.id == tensor_id, + RegisterTensor::QuantParams(_) => false, + }) + .map(|(pos, _)| pos) + } + + /// Doesn't return quantized tensor. + pub fn get(&self, tensor_id: TensorId) -> Option<(&TensorIr, &FuseType)> { + self.tensors + .iter() + .find(|entry| match entry { + RegisterTensor::Normal(tensor_ir, _) => tensor_ir.id == tensor_id, + RegisterTensor::QuantValues(_) => false, + RegisterTensor::QuantParams(_) => false, + }) + .and_then(|entry| match entry { + RegisterTensor::Normal(tensor_ir, fuse_precision) => { + Some((tensor_ir, fuse_precision)) + } + RegisterTensor::QuantValues(_) => None, + RegisterTensor::QuantParams(_) => None, + }) + } + + /// Insert a quantized tensor. + /// + /// It will return the positions for both the value tensor and param tensor. + pub fn insert_quant(&mut self, tensor: TensorIr) -> (usize, usize) { + if let Some(old) = self.tensors.iter().enumerate().find(|(_, val)| match &val { + RegisterTensor::QuantValues(tensor_ir) => tensor_ir == &tensor, + _ => false, + }) { + let values = old.0; + let params = values + 1; + return (values, params); + } + + let params = RegisterTensor::QuantParams(tensor.id); + let values = RegisterTensor::QuantValues(tensor); + let pos_values = self.len(); + self.tensors.push(values); + + let pos_params = self.len(); + self.tensors.push(params); + + (pos_values, pos_params) + } + + /// Insert a normal tensor with the given [precision](FusePrecision) in the current block. + pub fn insert(&mut self, precision: FuseType, tensor: TensorIr) -> usize { + if let Some(old) = self.tensors.iter().enumerate().find(|(_, val)| match &val { + RegisterTensor::Normal(tensor_ir, _) => tensor_ir.id == tensor.id, + _ => false, + }) { + return old.0; + } + + let value = RegisterTensor::Normal(tensor, precision); + let pos = self.len(); + + self.tensors.push(value); + + pos + } + + /// Update the already registered tensor with the given [tensor ir](TensorIr). + /// + /// # Notes + /// + /// This function only works with normal tensors, not quantized tensors. + pub fn update(&mut self, tensor: &TensorIr) { + if let Some(entry) = self.tensors.iter_mut().find(|entry| match entry { + RegisterTensor::Normal(tensor_ir, _) => tensor_ir.id == tensor.id, + _ => false, + }) && let RegisterTensor::Normal(tensor_ir, _) = entry + { + tensor_ir.status = tensor.status + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/block.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/block.rs new file mode 100644 index 0000000..d145898 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/block.rs @@ -0,0 +1,555 @@ +use super::{FuseResources, RegisteredTensors, TensorView}; +use crate::engine::{ + codegen::ir::{FuseArg, FuseOp, FuseType, LayoutInfo, MultiBlockPos, UnaryFuseArgs}, + settings::FuseSettings, +}; +use burn_ir::{TensorId, TensorIr, TensorStatus}; +use burn_std::{DType, Shape, quantization::QuantParam}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, btree_map::Entry}; + +#[derive(Clone, Serialize, Deserialize, Debug)] +/// A block containing all [operations](FuseOp) as well as reads and writes for each tensor along +/// with the [fusion settings](FuseSettings). +pub struct FuseBlock { + /// Contains the [fusion settings](FuseSettings) associated to the current block. + pub settings: FuseSettings, + /// Contains all the [operations](FuseOp) registered in the current block. + pub ops: Vec, + /// The reference shape of the current block. + pub shape_ref: Shape, + /// Contains all tensor inputs of the current block except for manually handled tensors. + /// + /// # Notes + /// + /// Some reads might not have read operations registered, such as dequantization, but it's + /// important to be registered here for vectorization. Input tensors that are not + /// registered here must be vectorized manually. + pub reads: BTreeMap>, + /// Contains all tensor outputs of the current block except for manually handled tensors. + /// We can have multiple writes when the same variable is reused after in another block. + pub writes: BTreeMap>, +} + +#[derive(Clone, Debug)] +/// It is responsible to build a [trace](FuseBlock). +pub struct FuseBlockBuilder { + pub settings: FuseSettings, + locals: LocalVariablePool, + pub ops: Vec, + reads: BTreeMap>, + // Only for global registers. + writes: BTreeMap>, + bool_precision: FuseType, + // Output declared in this block alone. + outputs: RegisteredTensors, + pub outputs_unhandled: Vec, + pub local_inputs: BTreeMap, + /// The reference shape used by this block. + pub shape_ref: Shape, +} + +#[derive(Debug)] +/// How a quantized input can be read. +pub enum QuantInput { + /// If already dequantized, we cache the dequantization and returns the local variable + /// corresponding to the float value. + AlreadyDequantized { local: FuseArg }, + /// Otherwise we return the information necessary to dequantize the tensor. + Quantized { values: FuseArg, params: FuseArg }, +} + +impl FuseBlockBuilder { + pub fn new(bool_precision: FuseType, settings: FuseSettings) -> Self { + Self { + bool_precision, + settings, + locals: Default::default(), + ops: Default::default(), + reads: Default::default(), + writes: Default::default(), + outputs: Default::default(), + outputs_unhandled: Default::default(), + local_inputs: Default::default(), + shape_ref: Shape::new([]), + } + } + + /// Register an output tensor. + pub fn output(&mut self, tensor: &TensorIr, resources: &mut FuseResources) -> Option { + if resources.indexed.contains_key(&tensor.id) { + return None; + } + if matches!(tensor.dtype, DType::QFloat(..)) { + return None; + } + let precision = tensor.dtype.into(); + + // Bool tensors are encoded as bool_precision. + let precision_output = match precision { + FuseType::Bool => self.bool_precision, + _ => precision, + }; + + let out = match self.locals.get(precision, tensor.id) { + Some(local) => local, + None => { + let out = self.locals.create(precision, tensor.id); + + self.outputs.insert(precision_output, tensor.clone()); + resources.outputs.insert(precision_output, tensor.clone()); + + out + } + }; + + Some(out) + } + + /// Register an input tensor. + pub fn multi_block_variable( + &mut self, + block_pos: usize, + tensor: &TensorIr, + global: bool, + ) -> Option { + let precision = tensor.dtype.into(); + + if let Some(val) = self.local_inputs.get(&tensor.id) { + return Some(val.clone()); + } + + let val = match self.locals.get(precision, tensor.id) { + Some(val) => val, + None => { + return None; + } + }; + + let arg = if global { + FuseArg::MultiBlockGlobal( + MultiBlockPos { + block_pos, + block_local_pos: self.writes.len(), + }, + val.precision(), + ) + } else { + FuseArg::MultiBlockLocal( + MultiBlockPos { + block_pos, + block_local_pos: self.writes.len(), + }, + val.precision(), + ) + }; + + let ops = match self.writes.get_mut(&tensor.id) { + Some(ops) => ops, + None => { + self.writes.insert(tensor.id, Vec::new()); + self.writes.get_mut(&tensor.id).unwrap() + } + }; + ops.push(FuseOp::Assign(UnaryFuseArgs { + input: val, + out: arg.clone(), + })); + + Some(arg) + } + + /// Register an input tensor. + pub fn input(&mut self, tensor: &TensorIr, resources: &mut FuseResources) -> Option { + if resources.indexed.contains_key(&tensor.id) { + return None; + } + + if matches!(tensor.dtype, DType::QFloat(..)) { + return None; + } + let precision = tensor.dtype.into(); + + // Bool tensors are encoded as bool_precision. + let precision_input = match precision { + FuseType::Bool => self.bool_precision, + _ => precision, + }; + + if let Some(val) = self.local_inputs.get(&tensor.id) { + return Some(val.clone()); + } + + let arg = match self.locals.get(precision, tensor.id) { + Some(local) => { + resources.inputs.update(tensor); + + local + } + None => { + let input = if resources.outputs.get_index(tensor.id).is_some() { + if let Some(val) = resources.registers.get(&tensor.id) { + return Some(val.clone()); + }; + + let pos = resources.buffers.insert(precision, tensor.clone()); + FuseArg::Output(pos, precision_input, LayoutInfo::Unknown) + } else { + let pos = resources.inputs.insert(precision_input, tensor.clone()); + FuseArg::Input(pos, precision_input, LayoutInfo::Unknown) + }; + + let out = self.locals.create(precision, tensor.id); + + let reads = if let Entry::Vacant(e) = self.reads.entry(tensor.id) { + e.insert(Vec::with_capacity(1)); + self.reads.get_mut(&tensor.id).unwrap() + } else { + self.reads.get_mut(&tensor.id).unwrap() + }; + + reads.push(FuseOp::Assign(UnaryFuseArgs { + input, + out: out.clone(), + })); + + out + } + }; + + Some(arg) + } + + /// Register an input quantized tensor. + pub fn input_quant( + &mut self, + tensor: &TensorIr, + resources: &mut FuseResources, + ) -> Option { + if resources.indexed.contains_key(&tensor.id) { + return None; + } + + let precision = tensor.dtype.into(); + let precision_scales = match tensor.dtype { + DType::QFloat(scheme) => match scheme.param { + QuantParam::F32 => FuseType::F32, + QuantParam::F16 => FuseType::F16, + QuantParam::BF16 => FuseType::BF16, + QuantParam::UE8M0 | QuantParam::UE4M3 => { + unimplemented!("Unsupported fuse precision"); + } + }, + _ => return None, + }; + + let arg = match self.locals.get(precision, tensor.id) { + Some(local) => { + resources.inputs.update(tensor); + QuantInput::AlreadyDequantized { local } + } + None => { + let (new_input, q_index) = resources.inputs.insert_quant(tensor.clone()); + let input = FuseArg::Input(new_input, precision, LayoutInfo::Unknown); + let scales = FuseArg::Input(q_index, precision_scales, LayoutInfo::Unknown); + + // Important to flag that there is a read, even if no operation is registered. + if let Entry::Vacant(e) = self.reads.entry(tensor.id) { + e.insert(Vec::new()); + }; + + QuantInput::Quantized { + values: input, + params: scales, + } + } + }; + + Some(arg) + } + + /// Register an input with swapped dims. + pub fn input_swap_dims( + &mut self, + tensor: &TensorIr, + output: &TensorIr, + dims: (usize, usize), + resources: &mut FuseResources, + ) -> Option { + if matches!(tensor.dtype, DType::QFloat(..)) { + return None; + } + let precision = tensor.dtype.into(); + + // Bool tensors are encoded as bool_precision. + let precision_input = match precision { + FuseType::Bool => self.bool_precision, + _ => precision, + }; + + let input_index = match self.locals.get(precision, tensor.id) { + Some(_) => { + // Can't fused an already fused input. + if resources.outputs.get(tensor.id).is_some() { + return None; + } + + match resources.inputs.get_index(tensor.id) { + Some(index) => { + resources.inputs.update(tensor); + index + } + None => { + return None; + } + } + } + None => resources.inputs.insert(precision_input, tensor.clone()), + }; + + let out = self.output(output, resources)?; + let original = FuseArg::Input(input_index, precision_input, LayoutInfo::Unknown); + + let broadcasted = output.shape[output.shape.rank() - 1] == 0; + + resources.views.push(TensorView::SwapDims { + swapped: output.id, + original: tensor.id, + dims, + }); + + let input = FuseArg::InputSwapDims { + original: Box::new(original), + dims, + broadcasted, + }; + + let reads = if let Entry::Vacant(e) = self.reads.entry(tensor.id) { + e.insert(Vec::with_capacity(1)); + self.reads.get_mut(&tensor.id).unwrap() + } else { + self.reads.get_mut(&tensor.id).unwrap() + }; + + reads.push(FuseOp::Assign(UnaryFuseArgs { + input, + out: out.clone(), + })); + + Some(out) + } + + /// Register an input that is reshaped. + pub fn input_reshaped( + &mut self, + tensor: &TensorIr, + output: &TensorIr, + resources: &mut FuseResources, + ) -> Option { + if matches!(tensor.dtype, DType::QFloat(..)) { + return None; + } + let precision = tensor.dtype.into(); + + // Bool tensors are encoded as bool_precision. + let precision_input = match precision { + FuseType::Bool => self.bool_precision, + _ => precision, + }; + + let input_index = match self.locals.get(precision, tensor.id) { + Some(_) => { + // Can't fused an already fused input. + if resources.outputs.get(tensor.id).is_some() { + return None; + } + + match resources.inputs.get_index(tensor.id) { + Some(index) => { + resources.inputs.update(tensor); + index + } + None => { + return None; + } + } + } + None => resources.inputs.insert(precision_input, tensor.clone()), + }; + + let out = self.output(output, resources)?; + let original = FuseArg::Input(input_index, precision_input, LayoutInfo::Unknown); + + let mut shape = Vec::new(); + + let index = resources.num_reshaped; + resources.num_reshaped += 1; + + let rank = output.shape.rank(); + + for i in 0..output.shape.rank() { + let id = index * rank + i; + shape.push(FuseArg::ScalarShape(id)); + } + + resources.views.push(TensorView::Reshape { + reshaped: output.id, + original: tensor.id, + reshape_pos: index, + shape_relative: output.shape.clone(), + }); + + let input = FuseArg::InputReshaped { + original: Box::new(original), + shape, + broadcasted: output.shape[rank - 1] == 0, + }; + + let reads = if let Entry::Vacant(e) = self.reads.entry(tensor.id) { + e.insert(Vec::with_capacity(1)); + self.reads.get_mut(&tensor.id).unwrap() + } else { + self.reads.get_mut(&tensor.id).unwrap() + }; + + reads.push(FuseOp::Assign(UnaryFuseArgs { + input, + out: out.clone(), + })); + + Some(out) + } + + /// Build into a fuse block. + pub fn build( + &self, + resources: &FuseResources, + outputs: &mut RegisteredTensors, + buffers: &mut Vec, + ) -> FuseBlock { + let ops = self.ops.clone(); + let reads = self.reads.clone(); + let tensor_writes = self.tensor_writes(resources, buffers); + + let mut writes = self.writes.clone(); + + for (tensor, precision) in tensor_writes + .iter() + .filter_map(|entry| entry.as_normal_tensor()) + { + if let Some(local) = self.locals.get_any_precision(tensor.id) { + let out_index = outputs.insert(*precision, tensor.clone()); + + let ops = match writes.get_mut(&tensor.id) { + Some(ops) => ops, + None => { + writes.insert(tensor.id, Vec::new()); + writes.get_mut(&tensor.id).unwrap() + } + }; + + ops.push(FuseOp::Assign(UnaryFuseArgs { + input: local, + out: FuseArg::Output(out_index, *precision, LayoutInfo::Unknown), + })); + } + } + + FuseBlock { + settings: self.settings, + ops, + shape_ref: self.shape_ref.clone(), + reads, + writes, + } + } + + /// Return the tensor that needs to be written to. + /// + /// # Notes + /// + /// The buffers vector passed as input is only to track the intermediary buffer writes needed + /// during execution. + pub fn tensor_writes( + &self, + resources: &FuseResources, + buffers: &mut Vec, + ) -> RegisteredTensors { + let mut result = RegisteredTensors::default(); + + // All tensors where their latest representation is not read write should be written to since they + // are going to be used after the fused kernel by other operations. + for output in self.outputs.iter() { + if let Some((tensor, _precision)) = output.as_normal_tensor() { + // We get the latest representation from the resources, not just this block. + if let Some((tensor, precision)) = resources.outputs.get(tensor.id) { + if !matches!(tensor.status, TensorStatus::ReadWrite) { + result.insert(*precision, tensor.clone()); + } else if resources.buffers.get(tensor.id).is_some() + && !buffers.contains(&tensor.id) + { + result.insert(*precision, tensor.clone()); + // We make sure we don't write multiple time in the same buffer, only the + // earliest possible. + buffers.push(tensor.id); + } + } + } + } + + result + } +} + +#[derive(Default, Clone, Debug)] +pub struct LocalVariablePool { + values: BTreeMap>, +} + +impl LocalVariablePool { + fn get(&self, precision: FuseType, tensor_id: TensorId) -> Option { + if let Some(indexes) = self.values.get(&precision) + && let Some(index) = indexes.get(&tensor_id) + { + return Some(FuseArg::BlockLocal { + pos: *index, + ty: precision, + }); + } + + None + } + + fn get_any_precision(&self, tensor_id: TensorId) -> Option { + for (precision, indexes) in self.values.iter() { + if let Some(index) = indexes.get(&tensor_id) { + return Some(FuseArg::BlockLocal { + pos: *index, + ty: *precision, + }); + } + } + + None + } + + fn create(&mut self, precision: FuseType, tensor_id: TensorId) -> FuseArg { + if let Some(indexes) = self.values.get_mut(&precision) { + let new_index = indexes.len(); + indexes.insert(tensor_id, new_index); + return FuseArg::BlockLocal { + pos: new_index, + ty: precision, + }; + } + + let new_index = 0; + self.values + .insert(precision, BTreeMap::from_iter([(tensor_id, new_index)])); + + FuseArg::BlockLocal { + pos: new_index, + ty: precision, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/fuser.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/fuser.rs new file mode 100644 index 0000000..40b4c15 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/fuser.rs @@ -0,0 +1,330 @@ +use super::{ + super::{ + codegen::ir::{FuseArg, FuseOp, FuseType, LayoutInfo}, + settings::FuseSettings, + }, + FuseResources, + block::FuseBlockBuilder, +}; +use super::{FuseTrace, RegisteredTensors}; +use crate::engine::trace::block::QuantInput; +use burn_fusion::stream::ScalarId; +use burn_ir::{ScalarIr, TensorIr}; +use burn_std::{DType, Shape}; +use cubecl::quant::scheme::QuantParam; + +#[derive(Clone, Debug)] +/// It is responsible to create a [trace](FuseTrace) composed of multiple [blocks](super::block::FuseBlock). +/// +/// It mostly handles the [resources](KernelResources) needed by the generated fused kernel, and +/// delegates most of the work to the [block builder](FuseBlockBuilder). +pub struct TraceFuser { + settings: FuseSettings, + pub bool_precision: FuseType, + // The tensors returned by the block that don't need to be written to global memory. + block_current: FuseBlockBuilder, + blocks_previous: Vec, + resources: FuseResources, +} + +impl TraceFuser { + /// Create a new trace builder with the given bool precision and [fuse settings](FuseSettings). + pub fn new(bool_precision: FuseType, settings: FuseSettings) -> Self { + Self { + settings, + bool_precision, + block_current: FuseBlockBuilder::new(bool_precision, settings), + blocks_previous: Default::default(), + resources: Default::default(), + } + } + + /// Get the number of blocks that are closed. + pub fn num_previous_blocks(&self) -> usize { + self.blocks_previous.len() + } + + /// Tag a tensor as dropped. + pub fn fuse_dropped(&mut self, tensor: &TensorIr) { + self.resources.outputs.update(tensor); + self.resources.inputs.update(tensor); + self.resources.dropped.insert(tensor.id); + } + + /// Register an operation. + pub fn fuse_operation(&mut self, op: FuseOp) { + self.block_current.ops.push(op); + } + + /// The number of operations fused. + pub fn num_ops_fused(&self) -> u32 { + let mut num_ops_fused = 0; + + for block in self.blocks_previous.iter() { + num_ops_fused += block.ops.len(); + } + + num_ops_fused += self.block_current.ops.len(); + num_ops_fused as u32 + } + + /// Close the current block with the given reference shape and creates a new one with new [fusion settings](FuseSettings). + pub fn next_block(&mut self, shape_ref: Shape, settings: FuseSettings) { + let mut block_new = FuseBlockBuilder::new(self.bool_precision, settings); + core::mem::swap(&mut self.block_current, &mut block_new); + block_new.shape_ref = shape_ref; + self.blocks_previous.push(block_new); + self.settings = settings; + } + + // Estimate how many bindings are in use right now. This can return more than the actual number + // but should never return less. + pub fn estimate_bindings(&self) -> u32 { + let mut buffers = Vec::new(); + let mut estimation = 1; // Metadata takes one. + + // We assume we are not going to write multiple times in the same output buffer. + for b in self.blocks_previous.iter() { + estimation += b.tensor_writes(&self.resources, &mut buffers).len() as u32; + } + + estimation += self + .block_current + .tensor_writes(&self.resources, &mut buffers) + .len() as u32; + estimation += self.resources.inputs.len() as u32; + // One buffer per scalar type for now. + estimation += self.resources.scalars.len() as u32; + + estimation + } + + /// Tag the [tensor](TensorIr) as received from a previous block. + /// + /// This will avoid reading the input again and instead use le local version when possible. + pub fn block_local_input( + &mut self, + tensor: &TensorIr, + block_pos: usize, + global: bool, + ) -> FuseArg { + let block = &mut self.blocks_previous[block_pos]; + + let src_arg = match block.multi_block_variable(block_pos, tensor, global) { + Some(val) => val, + None => { + // We try to read the input if not present. + block.input(tensor, &mut self.resources); + block + .multi_block_variable(block_pos, tensor, global) + .unwrap() + } + }; + + self.resources.outputs.update(tensor); + + if global { + self.resources.registers.insert(tensor.id, src_arg.clone()); + } + + self.block_current + .local_inputs + .insert(tensor.id, src_arg.clone()); + src_arg + } + + /// Register an output tensor that won't be automatically synced into global memory. + /// + /// It is therefore the responsibility of the operation to write the result to given tensor. + pub fn output_unhandled(&mut self, tensor: &TensorIr) -> FuseArg { + let arg = self + .output(tensor) + .expect("Can't add a new output that is already used in an index operation"); + + self.resources.outputs_unhandled.push(arg.clone()); + self.block_current.outputs_unhandled.push(arg.clone()); + arg + } + + /// Register an input tensor that won't be automatically read into a local variable. + /// + /// It is therefore the responsibility of the operation to read the given tensor. + pub fn input_unhandled(&mut self, tensor: &TensorIr) -> FuseArg { + if self.resources.indexed.contains_key(&tensor.id) { + panic!("Can't add a new input that is already used in an index operation"); + } + + self.resources.outputs.update(tensor); + + let precision = tensor.dtype.into(); + // Bool tensors are encoded as bool_precision. + let precision_input = match precision { + FuseType::Bool => self.bool_precision, + _ => precision, + }; + let new_input = self + .resources + .inputs + .insert(precision_input, tensor.clone()); + let arg = FuseArg::Input(new_input, precision_input, LayoutInfo::Unknown); + + self.resources.inputs_unhandled.push(tensor.id); + arg + } + + /// Register an input tensor. + pub fn input_quantized_unhandled(&mut self, tensor: &TensorIr) -> Option<(FuseArg, FuseArg)> { + if self.resources.indexed.contains_key(&tensor.id) { + panic!("Can't add a new input that is already used in an index operation"); + } + self.resources.outputs.update(tensor); + + let precision = tensor.dtype.into(); + let precision_scales = match tensor.dtype { + DType::QFloat(scheme) => match scheme.param { + QuantParam::F32 => FuseType::F32, + QuantParam::F16 => FuseType::F16, + QuantParam::BF16 => FuseType::BF16, + QuantParam::UE8M0 | QuantParam::UE4M3 => { + unimplemented!("Unsupported fuse precision"); + } + }, + _ => return None, + }; + + let (new_input, q_index) = self.resources.inputs.insert_quant(tensor.clone()); + let input = FuseArg::Input(new_input, precision, LayoutInfo::Unknown); + let scales = FuseArg::Input(q_index, precision_scales, LayoutInfo::Unknown); + + self.resources.inputs_unhandled.push(tensor.id); + Some((input, scales)) + } + + /// Register an input tensor. + pub fn input(&mut self, tensor: &TensorIr) -> Option { + if matches!(tensor.dtype, DType::QFloat(_)) { + return None; + } + + self.resources.outputs.update(tensor); + + self.block_current.input(tensor, &mut self.resources) + } + + /// Register an input tensor. + pub fn input_quantized(&mut self, tensor: &TensorIr) -> Option { + self.resources.outputs.update(tensor); + self.block_current.input_quant(tensor, &mut self.resources) + } + + /// Register an output tensor. + pub fn output(&mut self, tensor: &TensorIr) -> Option { + if matches!(tensor.dtype, DType::QFloat(_)) { + return None; + } + self.block_current.output(tensor, &mut self.resources) + } + + /// Register an input that will be accessed using custom indexing with no vectorization. + pub fn input_indexed(&mut self, tensor: &TensorIr) -> Option { + if matches!(tensor.dtype, DType::QFloat(_)) { + return None; + } + + if let Some(val) = self.resources.indexed.get(&tensor.id) { + self.resources.outputs.update(tensor); + return Some(val.clone()); + }; + + if self.resources.inputs.get(tensor.id).is_some() { + return None; + } + + if self.resources.outputs.get(tensor.id).is_some() { + return None; + } + + let input = self.input_unhandled(tensor); + self.resources.indexed.insert(tensor.id, input.clone()); + + Some(input) + } + + /// Register an input with swapped dims. + pub fn input_swap_dims( + &mut self, + tensor: &TensorIr, + output: &TensorIr, + dims: (usize, usize), + ) -> Option { + if matches!(tensor.dtype, DType::QFloat(_)) { + return None; + } + + self.resources.outputs.update(tensor); + self.block_current + .input_swap_dims(tensor, output, dims, &mut self.resources) + } + + /// Register an input that is reshaped. + pub fn input_reshaped(&mut self, tensor: &TensorIr, output: &TensorIr) -> Option { + if matches!(tensor.dtype, DType::QFloat(_)) { + return None; + } + + self.resources.outputs.update(tensor); + self.block_current + .input_reshaped(tensor, output, &mut self.resources) + } + + /// Register a scalar value. + pub fn scalar(&mut self, elem: &ScalarIr, dtype: DType) -> FuseArg { + let precision = dtype.into(); + let id = if let ScalarIr::UInt(value) = elem { + ScalarId { value: *value } + } else { + unreachable!() // should always be u64 + }; + + // Bool scalars are encoded as bool_precision. + let precision = match precision { + FuseType::Bool => self.bool_precision, + _ => precision, + }; + let new_index = self.resources.scalars.len(); + + self.resources.scalars.push((precision, id.value)); + FuseArg::Scalar(new_index, precision) + } + + /// Finish fusing and returns the created trace. + pub fn finish(&mut self, shape_ref: Shape) -> FuseTrace { + let mut resources = self.resources.clone(); + let mut outputs = RegisteredTensors::default(); + let mut buffers = Vec::new(); + + for tensor in resources.buffers.iter() { + let (tensor, ty) = tensor.as_normal_tensor().unwrap(); + outputs.insert(*ty, tensor.clone()); + } + + let mut blocks = Vec::new(); + + let mut register_block = |block: &FuseBlockBuilder| { + let block = block.build(&self.resources, &mut outputs, &mut buffers); + blocks.push(block); + }; + + for block in self.blocks_previous.iter() { + register_block(block); + } + self.block_current.shape_ref = shape_ref; + register_block(&self.block_current); + + // We update the output tensors registered to be the ones that are written to in global + // memory. + resources.outputs = outputs; + + FuseTrace { blocks, resources } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/mod.rs new file mode 100644 index 0000000..286c232 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/engine/trace/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod block; + +mod base; +mod fuser; + +pub use base::*; +pub use fuser::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/lib.rs new file mode 100644 index 0000000..4612836 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/lib.rs @@ -0,0 +1,11 @@ +#[macro_use] +extern crate derive_new; + +pub mod optim; + +mod base; + +pub(crate) mod engine; +pub(crate) mod tune; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/base.rs new file mode 100644 index 0000000..b2e25bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/base.rs @@ -0,0 +1,63 @@ +use crate::optim::{ + elemwise::{ElemwiseOptimization, ElemwiseOptimizationState}, + matmul::{MatmulOptimization, MatmulOptimizationState}, + reduce::{ReduceOptimization, ReduceOptimizationState}, + reduce_broadcasted::{ReduceBroadcastedOptimization, ReduceBroadcastedOptimizationState}, +}; +use cubecl::Runtime; +use serde::{Deserialize, Serialize}; + +/// Fusion optimization type for cubecl. +/// +/// More optimization variants should be added here. +#[allow(clippy::large_enum_variant)] +pub enum CubeOptimization { + ElementWise(ElemwiseOptimization), + Matmul(MatmulOptimization), + Reduce(ReduceOptimization), + ReduceBroadcasted(ReduceBroadcastedOptimization), +} + +impl core::fmt::Debug for CubeOptimization { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let value = self.to_opt_state(); + f.write_fmt(format_args!("{value:?}")) + } +} + +impl CubeOptimization { + /// Serializes the current optimization to its state. + pub fn to_opt_state(&self) -> CubeOptimizationState { + match self { + Self::ElementWise(value) => CubeOptimizationState::ElementWise(value.to_state()), + Self::Matmul(value) => CubeOptimizationState::Matmul(value.to_state()), + Self::Reduce(value) => CubeOptimizationState::Reduce(value.to_state()), + Self::ReduceBroadcasted(value) => { + CubeOptimizationState::ReduceBroadcasted(value.to_state()) + } + } + } +} + +impl burn_fusion::NumOperations for CubeOptimization { + fn len(&self) -> usize { + match self { + Self::ElementWise(op) => op.num_ops_fused(), + Self::Matmul(op) => op.num_ops_fused(), + Self::Reduce(op) => op.num_ops_fused(), + Self::ReduceBroadcasted(op) => op.num_ops_fused(), + } + } +} + +/// Fusion optimization state type for cubecl. +/// +/// More optimization variants should be added here. +#[allow(clippy::large_enum_variant)] +#[derive(Serialize, Deserialize, Debug)] +pub enum CubeOptimizationState { + ElementWise(ElemwiseOptimizationState), + Matmul(MatmulOptimizationState), + Reduce(ReduceOptimizationState), + ReduceBroadcasted(ReduceBroadcastedOptimizationState), +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/fuser.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/fuser.rs new file mode 100644 index 0000000..94c4285 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/fuser.rs @@ -0,0 +1,87 @@ +use super::optimization::ElemwiseOptimization; +use crate::{ + engine::{ + codegen::ir::FuseType, + fuser::TraceOperationFuser, + settings::{FuseSettings, RefLayoutSetting, VectorizationSetting}, + }, + optim::CubeOptimization, +}; +use burn_fusion::OperationFuser; +use burn_std::Shape; +use cubecl::Runtime; + +/// Fuses element wise operations. +pub struct ElementWiseFuser { + fuser: TraceOperationFuser, + device: R::Device, +} + +impl Clone for ElementWiseFuser { + fn clone(&self) -> Self { + Self { + fuser: self.fuser.clone(), + device: self.device.clone(), + } + } +} + +impl ElementWiseFuser { + pub fn shape_id(&self) -> Shape { + self.fuser.current_output_shape.clone() + } + pub fn new(device: R::Device, bool_precision: FuseType) -> Self { + let client = R::client(&device); + let props = client.properties(); + let max_bindings = props.hardware.max_bindings; + + Self { + fuser: TraceOperationFuser::new( + max_bindings, + bool_precision, + FuseSettings { + broadcast: true, + output_shape_updates: true, + inplace: true, + vectorization: VectorizationSetting::Activated, + ref_layout: RefLayoutSetting::Any, + }, + ), + device, + } + } +} + +impl OperationFuser> for ElementWiseFuser { + fn fuse(&mut self, operation: &burn_ir::OperationIr) { + self.fuser.fuse(operation); + } + + fn finish(&mut self) -> CubeOptimization { + let client = R::client(&self.device); + let trace = self.fuser.finish(); + let elementwise = ElemwiseOptimization::new(trace, client, self.device.clone(), self.len()); + + CubeOptimization::ElementWise(elementwise) + } + + fn reset(&mut self) { + self.fuser.reset() + } + + fn status(&self) -> burn_fusion::FuserStatus { + self.fuser.status() + } + + fn properties(&self) -> burn_fusion::FuserProperties { + self.fuser.properties() + } + + fn len(&self) -> usize { + self.fuser.len() + } + + fn clone_dyn(&self) -> Box>> { + Box::new(self.clone()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/mod.rs new file mode 100644 index 0000000..7c4308c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/mod.rs @@ -0,0 +1,5 @@ +mod fuser; +mod optimization; + +pub use fuser::*; +pub use optimization::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/optimization.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/optimization.rs new file mode 100644 index 0000000..f2eeb05 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/elemwise/optimization.rs @@ -0,0 +1,140 @@ +use crate::{ + CubeFusionHandle, + engine::{ + codegen::{ + io::ref_len, + ir::{ + FuseArg, FuseBlockConfig, GlobalArgs, GlobalArgsLaunch, RefLayout, + multi_block_variables_init, + }, + kernel::{fuse_on_write, init_locals}, + }, + launch::{ + FuseTraceLauncher, + runner::{TraceRunner, Vectorization}, + }, + trace::FuseTrace, + }, +}; +use burn_fusion::stream::Context; +use cubecl::{CubeDim, calculate_cube_count_elemwise, client::ComputeClient, prelude::*}; +use serde::{Deserialize, Serialize}; + +#[derive(new)] +/// Fuse element wise operations into a single kernel. +pub struct ElemwiseOptimization { + pub(crate) trace: FuseTrace, + client: ComputeClient, + device: R::Device, + len: usize, +} + +#[derive(Serialize, Deserialize, Debug)] +/// State for the [elemwise optimization](ElemwiseOptimization). +pub struct ElemwiseOptimizationState { + trace: FuseTrace, + len: usize, +} + +impl ElemwiseOptimization { + /// Execute the optimization. + pub fn execute(&self, context: &mut Context<'_, CubeFusionHandle>) { + let launcher = FuseTraceLauncher::new(&self.trace, &ElemwiseRunner); + + match launcher.launch::(&self.client, &self.device, context) { + Ok(_) => (), + Err(err) => { + panic!("{err:?} - {:?}", self.trace); + } + } + } + + /// Number of element wise operations fused. + pub fn num_ops_fused(&self) -> usize { + self.len + } + + /// Create an optimization from its [state](ElemwiseOptimizationState). + pub fn from_state(device: &R::Device, state: ElemwiseOptimizationState) -> Self { + Self { + trace: state.trace, + len: state.len, + client: R::client(device), + device: device.clone(), + } + } + + /// Convert the optimization to its [state](ElemwiseOptimizationState). + pub fn to_state(&self) -> ElemwiseOptimizationState { + ElemwiseOptimizationState { + trace: self.trace.clone(), + len: self.len, + } + } +} + +pub struct ElemwiseRunner; + +impl Vectorization for ElemwiseRunner {} +impl TraceRunner for ElemwiseRunner { + type Error = LaunchError; // No error possible + + fn run<'a>( + &'a self, + client: &'a ComputeClient, + inputs: GlobalArgsLaunch<'a, R>, + outputs: GlobalArgsLaunch<'a, R>, + configs: &[FuseBlockConfig], + ) -> Result<(), Self::Error> { + let config = &configs[0]; + let shape = match &config.ref_layout { + RefLayout::Concrete(arg) => match arg { + FuseArg::Input(..) => inputs.shape_ref(&config.ref_layout, config.rank), + FuseArg::Output(..) => outputs.shape_ref(&config.ref_layout, config.rank), + _ => panic!("Invalid concreate ref layout"), + }, + RefLayout::Virtual(_) => inputs.shape_ref(&config.ref_layout, config.rank), + }; + let working_units = shape.iter().product::() / config.width; + let cube_dim = CubeDim::new(client, working_units); + let cube_count = calculate_cube_count_elemwise(client, working_units, cube_dim); + let address_type = inputs + .required_address_type() + .max(outputs.required_address_type()); + + unsafe { + elemwise_fuse::launch_unchecked( + client, + cube_count, + cube_dim, + address_type, + inputs, + outputs, + config.clone(), + )?; + }; + + Ok(()) + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +fn elemwise_fuse( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + #[comptime] config: &FuseBlockConfig, +) { + // We write no values for this fusion. + let values = Registry::>::new(); + let args = comptime![Vec::::new()]; + let pos = ABSOLUTE_POS; + + multi_block_variables_init(config, &mut outputs.variables); + + let mut locals = init_locals(inputs, outputs, config); + let length = ref_len(inputs, outputs, &locals, config); + + if pos < length { + fuse_on_write::(inputs, outputs, &mut locals, pos, values, args, config) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/args.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/args.rs new file mode 100644 index 0000000..69e5827 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/args.rs @@ -0,0 +1,548 @@ +use crate::engine::codegen::{ + io::ref_line_size, + ir::{FuseArg, FuseBlockConfig, FuseType, GlobalArgs, LocalArgs, multi_block_variables_init}, + kernel::init_locals, + view::{FusedOutput, GlobalInput, GlobalInputExpand}, +}; +use cubecl::{ + intrinsic, + prelude::*, + quant::scheme::{QuantLevel, QuantScheme}, + std::{ + FastDivmod, + quant::{ + RunWithQuantType, + view::{QuantizedView, run_with_quant_type}, + }, + tensor::{ + View, ViewExpand, + layout::{Coords1d, Coords2d, VirtualLayout}, + }, + }, +}; +use cubek::matmul::{ + components::global::memory::{ + BatchLayout, BlockScaledLayout, GlobalLayout, GlobalLayoutConfig, GlobalLayoutExpand, + GlobalScaleLayout, GlobalScaleLayoutExpand, NoopLayout, + }, + definition::MatrixLayout, + launch::{BatchedCoords, MatmulArgs}, +}; +use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; + +#[derive(Clone)] +pub struct FusedMatmulArgs; + +#[derive(CubeLaunch, CubeType)] +pub struct FusedMatmulInput { + global: GlobalArgs, + #[cube(comptime)] + config: FuseBlockConfig, + #[cube(comptime)] + a: MatmulArg, + #[cube(comptime)] + b: MatmulArg, + #[cube(comptime)] + c: Option, + #[cube(comptime)] + out: FuseArg, +} + +#[cube] +impl MatmulArgs for FusedMatmulArgs { + type Output = GlobalArgs; + type Input = FusedMatmulInput; + type State = FusedMatmulState; + type Config = (); + + fn init_state( + inputs: &Self::Input, + outputs: &mut Self::Output, + _config: (), + #[comptime] lhs_layout_config: GlobalLayoutConfig, + #[comptime] rhs_layout_config: GlobalLayoutConfig, + #[comptime] out_layout_config: GlobalLayoutConfig, + ) -> Self::State { + multi_block_variables_init(&inputs.config, &mut outputs.variables); + + let mut locals = init_locals(&inputs.global, outputs, &inputs.config); + let rank = comptime![inputs.config.rank]; + + let mut batch_shape = Sequence::new(); + let mut batch_strides_out = Sequence::new(); + + #[unroll] + for i in 0..rank - 2 { + batch_shape.push(FastDivmod::new_Fallback(locals.ref_shape[i] as u32)); + batch_strides_out.push(locals.ref_strides[i]); + } + + let batch_lhs = input_batch_layout( + &inputs.global, + &batch_shape, + comptime![inputs.a.clone()], + comptime![inputs.config.clone()], + ); + let batch_rhs = input_batch_layout( + &inputs.global, + &batch_shape, + comptime![inputs.b.clone()], + comptime![inputs.config.clone()], + ); + let batch_acc = match comptime![inputs.c.clone()] { + Some(c) => Option::Some(input_batch_layout( + &inputs.global, + &batch_shape, + comptime![c], + comptime![inputs.config.clone()], + )), + None => Option::new_None(), + }; + let batch_out = BatchLayout::new(batch_strides_out, batch_shape.clone()); + + FusedMatmulState::new( + inputs, + outputs, + &mut locals, + batch_lhs, + batch_rhs, + batch_acc, + VirtualLayout::new::(batch_out), + batch_shape, + &inputs.config, + lhs_layout_config, + rhs_layout_config, + out_layout_config, + ) + } + + fn view_lhs( + state: &Self::State, + ) -> View, BatchedCoords> { + global_view( + &state.inputs, + &state.locals, + &state.batch_shape, + comptime![state.a.clone()], + comptime![state.config.clone()], + state.lhs_layout_config, + ) + } + + fn batch_lhs( + state: &Self::State, + batch: usize, + ) -> usize { + state.a_batch.to_source_pos(batch) + } + + fn view_rhs( + state: &Self::State, + ) -> View, BatchedCoords> { + global_view( + &state.inputs, + &state.locals, + &state.batch_shape, + comptime![state.b.clone()], + comptime![state.config.clone()], + comptime![state.rhs_layout_config], + ) + } + + fn batch_rhs( + state: &Self::State, + batch: usize, + ) -> usize { + state.b_batch.to_source_pos(batch) + } + + fn view_acc( + state: &Self::State, + ) -> Option, BatchedCoords>> { + match comptime![state.c.clone()] { + Some(c) => { + let view = global_view( + &state.inputs, + &state.locals, + &state.batch_shape, + c, + comptime![state.config.clone()], + comptime![state.out_layout_config], + ); + Option::Some(view) + } + None => Option::new_None(), + } + } + + fn batch_acc( + state: &Self::State, + batch: usize, + ) -> usize { + match state.c_batch { + Some(c_batch) => c_batch.to_source_pos(batch), + None => batch, + } + } + + fn view_out( + state: &mut Self::State, + ) -> View, BatchedCoords, ReadWrite> { + let rank = comptime![state.config.rank]; + + let shape_row = state.locals.ref_shape[rank - 2] as u32; + let shape_col = state.locals.ref_shape[rank - 1] as u32; + + let stride_row = state.locals.ref_strides[rank - 2]; + let stride_col = state.locals.ref_strides[rank - 1]; + + let layout = GlobalLayout::new( + VirtualLayout::new::(NoopLayout::new()), + shape_row, + shape_col, + stride_row, + stride_col, + ref_line_size(&state.locals), + 1u32, + state.out_layout_config, + ); + let mut buffer = FusedOutput::new( + &state.inputs, + &mut state.outputs, + &mut state.locals, + comptime![state.out.clone()], + comptime![state.config.clone()], + ); + View::new_mut::(&mut buffer, layout) + } + + fn batch_out( + state: &Self::State, + batch: usize, + ) -> usize { + state.out_batch.to_source_pos(batch) + } + + fn runtime_config(_state: &Self::State) { + } +} + +#[cube] +fn global_view( + inputs: &GlobalArgs, + locals: &LocalArgs, + batch_shape: &Sequence>, + #[comptime] arg: MatmulArg, + #[comptime] config: FuseBlockConfig, + #[comptime] layout_config: GlobalLayoutConfig, +) -> View, BatchedCoords> { + let rank = comptime![config.rank]; + let data = comptime![arg.data().clone()]; + let data_tensor = match comptime![data.clone()] { + FuseArg::Input(pos, ..) => inputs.tensors.index(pos), + _ => panic!("Input must be concrete"), + }; + + let mut shape_row = data_tensor.tensor.shape(rank - 2) as u32; + let mut shape_col = data_tensor.tensor.shape(rank - 1) as u32; + let mut packing = comptime![1]; + + if arg.scheme().is_some() { + let scheme = arg.scheme().unwrap(); + let num_quants = scheme.num_quants() as u32; + comptime![packing = num_quants]; + match comptime![layout_config.matrix_layout] { + MatrixLayout::RowMajor => shape_col *= num_quants, + MatrixLayout::ColMajor => shape_row *= num_quants, + }; + } + + let shape = (shape_row, shape_col); + + // Noop for normal inputs because batch offset is cached, quantized uses logical batches + let batch_layout = match comptime![arg.clone()] { + MatmulArg::Normal(_) => VirtualLayout::new::(NoopLayout::new()), + MatmulArg::Quantized { data, .. } => { + let data_arg = comptime![MatmulArg::Normal(data)]; + input_batch_layout(inputs, batch_shape, data_arg, comptime![config.clone()]) + } + }; + + let data_layout = global_layout( + inputs, + shape, + batch_layout, + arg.data().clone(), + config.clone(), + data_tensor.tensor.line_size(), + layout_config, + packing, + ); + let data_buf = GlobalInput::new(inputs, locals, data, comptime![config.clone()], None); + + match comptime![arg.clone()] { + MatmulArg::Normal(_) => View::new::(&data_buf, data_layout), + MatmulArg::Quantized { scales, scheme, .. } => { + let scales_layout = match comptime![scheme.level] { + QuantLevel::Tensor => GlobalScaleLayout::new_PerTensor(shape), + QuantLevel::Block(block_size) => { + let block_size = comptime![block_size.as_dim::<2>()]; + + let scales_arg = comptime![MatmulArg::Normal(scales.clone())]; + let batch_layout = input_batch_layout( + inputs, + batch_shape, + scales_arg, + comptime![config.clone()], + ); + + let scales_layout = global_layout( + inputs, + shape, + batch_layout, + comptime![scales.clone()], + comptime![config.clone()], + 1usize, + layout_config, + 1u32, + ); + GlobalScaleLayout::new_BlockScaled(BlockScaledLayout::new( + shape, + scales_layout, + comptime![(block_size[0] as u32, block_size[1] as u32)], + )) + } + }; + let scales_buf = GlobalInput::new(inputs, locals, scales, config, None); + create_quant_view_dynamic(data_buf, data_layout, scales_buf, scales_layout, scheme) + } + } +} + +#[cube] +fn input_batch_layout( + inputs: &GlobalArgs, + batch_shape: &Sequence>, + #[comptime] arg: MatmulArg, + #[comptime] config: FuseBlockConfig, +) -> VirtualLayout { + let rank = comptime![config.rank]; + match comptime![arg.clone()] { + MatmulArg::Normal(arg) => { + let data_tensor = match comptime![arg.clone()] { + FuseArg::Input(pos, ..) => inputs.tensors.index(pos), + _ => panic!("Input must be concrete"), + }; + + let mut batch_strides = Sequence::new(); + #[unroll] + for i in 0..rank - 2 { + let shape = data_tensor.tensor.shape(i); + let stride = select(shape == 1, 0, data_tensor.tensor.stride(i)); + batch_strides.push(stride); + } + + VirtualLayout::new::(BatchLayout::new(batch_strides, batch_shape.clone())) + } + MatmulArg::Quantized { .. } => VirtualLayout::new::(NoopLayout::new()), + } +} + +#[cube] +fn global_layout( + inputs: &GlobalArgs, + shape: Coords2d, + batch_layout: VirtualLayout, + #[comptime] arg: FuseArg, + #[comptime] config: FuseBlockConfig, + #[comptime] line_size: LineSize, + #[comptime] layout_config: GlobalLayoutConfig, + #[comptime] packing: u32, +) -> GlobalLayout { + let rank = comptime![config.rank]; + let data_tensor = match comptime![arg.clone()] { + FuseArg::Input(pos, ..) => inputs.tensors.index(pos), + _ => panic!("Input must be concrete"), + }; + + let (shape_row, shape_col) = shape; + + let stride_row = data_tensor.tensor.stride(rank - 2); + let stride_col = data_tensor.tensor.stride(rank - 1); + + GlobalLayout::new( + batch_layout, + shape_row, + shape_col, + stride_row, + stride_col, + line_size, + packing, + layout_config, + ) +} + +struct CreateQuantView<'a, E: Numeric> { + scope: &'a mut Scope, + data_buf: GlobalInputExpand, + data_layout: GlobalLayoutExpand, + scales_buf: GlobalInputExpand, + scales_layout: GlobalScaleLayoutExpand, + scheme: QuantScheme, + _ty: PhantomData, +} + +impl<'a, E: Numeric> RunWithQuantType for CreateQuantView<'a, E> { + type Output = ViewExpand, BatchedCoords>; + + fn execute(self) -> Self::Output { + create_quant_view::expand::( + self.scope, + self.data_buf, + self.data_layout, + self.scales_buf, + self.scales_layout, + self.scheme, + ) + } +} + +#[cube] +#[allow(unused)] +fn create_quant_view_dynamic( + data_buf: GlobalInput, + data_layout: GlobalLayout, + scales_buf: GlobalInput, + scales_layout: GlobalScaleLayout, + #[comptime] scheme: QuantScheme, +) -> View, BatchedCoords> { + intrinsic!(|scope| { + let func = CreateQuantView { + scope, + data_buf, + data_layout, + scales_buf, + scales_layout, + scheme, + _ty: PhantomData, + }; + run_with_quant_type(func, scheme) + }) +} + +#[cube] +fn create_quant_view( + data_buf: GlobalInput, + data_layout: GlobalLayout, + scales_buf: GlobalInput, + scales_layout: GlobalScaleLayout, + #[comptime] scheme: QuantScheme, +) -> View, BatchedCoords> { + let data_view: View, BatchedCoords> = + View::new::(&data_buf, data_layout); + let scales_view: View = + View::new::(&scales_buf, scales_layout); + QuantizedView::new(data_view, scales_view, scheme).view() +} + +#[derive(CubeType)] +pub struct FusedMatmulState { + inputs: GlobalArgs, + outputs: GlobalArgs, + locals: LocalArgs, + a_batch: VirtualLayout, + b_batch: VirtualLayout, + c_batch: Option>, + out_batch: VirtualLayout, + #[cube(comptime)] + config: FuseBlockConfig, + #[cube(comptime)] + a: MatmulArg, + #[cube(comptime)] + b: MatmulArg, + #[cube(comptime)] + c: Option, + #[cube(comptime)] + out: FuseArg, + #[cube(comptime)] + lhs_layout_config: GlobalLayoutConfig, + #[cube(comptime)] + rhs_layout_config: GlobalLayoutConfig, + #[cube(comptime)] + out_layout_config: GlobalLayoutConfig, + batch_shape: Sequence>, +} + +#[cube] +impl FusedMatmulState { + #[allow(clippy::too_many_arguments)] + pub fn new( + inputs: &FusedMatmulInput, + outputs: &mut GlobalArgs, + locals: &mut LocalArgs, + a_batch: VirtualLayout, + b_batch: VirtualLayout, + c_batch: Option>, + out_batch: VirtualLayout, + batch_shape: Sequence>, + #[comptime] config: &FuseBlockConfig, + #[comptime] lhs_layout_config: GlobalLayoutConfig, + #[comptime] rhs_layout_config: GlobalLayoutConfig, + #[comptime] out_layout_config: GlobalLayoutConfig, + ) -> FusedMatmulState { + FusedMatmulState { + inputs: inputs.global.clone(), + outputs: outputs.clone(), + config: comptime![config.clone()], + locals: locals.clone(), + a_batch, + b_batch, + c_batch, + out_batch, + a: comptime![inputs.a.clone()], + b: comptime![inputs.b.clone()], + c: comptime![inputs.c.clone()], + out: comptime![inputs.out.clone()], + lhs_layout_config, + rhs_layout_config, + out_layout_config, + batch_shape, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] +/// Argument to a matmul operation. +pub enum MatmulArg { + Normal(FuseArg), + Quantized { + data: FuseArg, + scales: FuseArg, + precision: FuseType, + scheme: QuantScheme, + }, +} + +impl MatmulArg { + pub fn data(&self) -> &FuseArg { + match self { + MatmulArg::Normal(arg) => arg, + MatmulArg::Quantized { data, .. } => data, + } + } + + pub fn scheme(&self) -> Option<&QuantScheme> { + match self { + MatmulArg::Normal(_) => None, + MatmulArg::Quantized { scheme, .. } => Some(scheme), + } + } + + pub fn precision(&self) -> FuseType { + match self { + MatmulArg::Normal(arg) => arg.precision(), + MatmulArg::Quantized { precision, .. } => *precision, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/fuser.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/fuser.rs new file mode 100644 index 0000000..3a550b6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/fuser.rs @@ -0,0 +1,160 @@ +use super::optimization::{FusedMatmul, MatmulOptimization}; +use crate::{ + engine::{codegen::ir::FuseType, fuser::TraceOperationFuser, settings::FuseSettings}, + optim::CubeOptimization, + optim::matmul::args::MatmulArg, +}; +use burn_fusion::{FuserStatus, OperationFuser}; +use burn_ir::{FloatOperationIr, OperationIr}; +use burn_std::DType; +use cubecl::Runtime; + +/// Fused element wise operations that are normally memory bound. +pub struct MatmulFuser { + fuser: TraceOperationFuser, + fuser_fallback: TraceOperationFuser, + device: R::Device, + matmul: Option, +} + +impl Clone for MatmulFuser { + fn clone(&self) -> Self { + Self { + fuser: self.fuser.clone(), + fuser_fallback: self.fuser_fallback.clone(), + device: self.device.clone(), + matmul: self.matmul.clone(), + } + } +} + +impl MatmulFuser { + pub fn new(device: R::Device, bool_precision: FuseType) -> Self { + let client = R::client(&device); + let props = client.properties(); + let max_bindings = props.hardware.max_bindings; + let settings_matmul = FuseSettings { + output_shape_updates: false, + ..Default::default() + }; + let settings_fallback = FuseSettings::default(); + + Self { + fuser: TraceOperationFuser::new(max_bindings, bool_precision, settings_matmul), + fuser_fallback: TraceOperationFuser::new( + max_bindings, + bool_precision, + settings_fallback, + ), + device, + matmul: None, + } + } +} + +impl OperationFuser> for MatmulFuser { + fn fuse(&mut self, operation: &OperationIr) { + if let FuserStatus::Closed = self.fuser.status() { + return; + } + + if self.matmul.is_none() { + if let OperationIr::Float(_, FloatOperationIr::Matmul(op)) = operation { + // Precision shouldn't be hardcoded but I don't know how to get float precision of the backend + let lhs = match op.lhs.dtype { + DType::QFloat(scheme) => { + let (data, scales) = self.fuser.input_quantized_unhandled(&op.lhs).unwrap(); + MatmulArg::Quantized { + data, + scales, + precision: op.out.dtype.into(), + scheme, + } + } + _ => MatmulArg::Normal(self.fuser.input_unhandled(&op.lhs)), + }; + let rhs = match op.rhs.dtype { + DType::QFloat(scheme) => { + let (data, scales) = self.fuser.input_quantized_unhandled(&op.rhs).unwrap(); + MatmulArg::Quantized { + data, + scales, + precision: op.out.dtype.into(), + scheme, + } + } + _ => MatmulArg::Normal(self.fuser.input_unhandled(&op.rhs)), + }; + + let out = self.fuser.output_unhandled(&op.out); + + self.matmul = Some(FusedMatmul::new( + lhs, + rhs, + out, + op.clone().into(), + Default::default(), + )); + } else { + self.fuser.close(); + self.fuser_fallback.close(); + } + } else { + let can_register = + self.fuser.can_fuse(operation) && self.fuser_fallback.can_fuse(operation); + + match can_register { + true => { + self.fuser.fuse(operation); + self.fuser_fallback.fuse(operation); + } + false => { + self.fuser.close(); + self.fuser_fallback.close(); + } + }; + } + } + + fn finish(&mut self) -> CubeOptimization { + let client = R::client(&self.device); + let trace = self.fuser.finish(); + let trace_fallback = self.fuser_fallback.finish(); + + let matmul = MatmulOptimization::new( + trace, + trace_fallback, + client, + self.device.clone(), + self.len(), + self.matmul.as_ref().unwrap().clone(), + ); + + CubeOptimization::Matmul(matmul) + } + + fn reset(&mut self) { + self.fuser.reset(); + self.fuser_fallback.reset(); + self.matmul = None; + } + + fn status(&self) -> burn_fusion::FuserStatus { + self.fuser.status() + } + + fn properties(&self) -> burn_fusion::FuserProperties { + let mut properties = self.fuser.properties(); + properties.score += 1; + properties + } + + fn len(&self) -> usize { + // Matmul operation isn't registered in the fuser + self.fuser.len() + 1 + } + + fn clone_dyn(&self) -> Box>> { + Box::new(self.clone()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/mod.rs new file mode 100644 index 0000000..75a082a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/mod.rs @@ -0,0 +1,8 @@ +mod fuser; +mod optimization; + +pub(crate) mod args; +pub(crate) mod tune; + +pub use fuser::*; +pub use optimization::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/optimization.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/optimization.rs new file mode 100644 index 0000000..88332a8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/optimization.rs @@ -0,0 +1,649 @@ +use super::args::FusedMatmulInputLaunch; +#[cfg(feature = "autotune")] +use super::tune::fused_matmul_autotune; +use crate::{ + CubeFusionHandle, FallbackOperation, + engine::{ + codegen::ir::{FuseArg, FuseBlockConfig, FuseType, GlobalArgsLaunch, RefLayout}, + launch::{ + FuseTraceLauncher, HandleInput, LaunchPlan, + runner::{TraceRunner, Vectorization, VectorizationAxis}, + }, + trace::{FuseTrace, TraceError, TuneOutput}, + }, + optim::{ + elemwise::ElemwiseRunner, + matmul::args::{FusedMatmulArgs, MatmulArg}, + }, +}; +use burn_fusion::stream::Context; +use burn_ir::BinaryOpIr; +use cubecl::{ + client::ComputeClient, + prelude::*, + std::tensor::{MatrixBatchLayout, matrix_batch_layout}, +}; +use cubek::matmul::{ + components::tile::{cmma::CmmaMatmul, mma::MmaMatmul}, + definition::{ + MatmulElems, MatmulGlobalElems, MatmulLineSizes, MatmulProblem, MatmulSetupError, + MatrixLayout, + }, + launch::launch_kernel_virtual, + routines::{ + BlueprintStrategy, Routine, + double_buffering::{CyclicDoubleBufferingAlgorithm, DoubleBufferingArgs}, + double_unit::DoubleUnitAlgorithm, + ordered_double_buffering::{OrderedDoubleBufferingAlgorithm, OrderedSelectionArgs}, + simple::{SimpleAlgorithm, SimpleArgs}, + simple_unit::SimpleUnitAlgorithm, + vecmat::{DoubleVecMatAlgorithm, SimpleVecMatAlgorithm}, + }, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +/// Fuse matmul operation followed by elemwise operations into a single kernel. +pub struct MatmulOptimization { + pub(crate) info: Arc>, +} + +pub struct MatmulOptimizationTuneArg { + pub(crate) info: Arc>, + pub(crate) fallback: Box>, +} + +pub(crate) struct MatmulOptimizationInfo { + trace: FuseTrace, + trace_fallback: FuseTrace, + pub(crate) client: ComputeClient, + pub(crate) device: R::Device, + pub(crate) len: usize, + pub(crate) matmul: FusedMatmul, +} + +#[derive(Serialize, Deserialize, Debug)] +/// State for the [matrix optimization](MatmulOptimizationState). +pub struct MatmulOptimizationState { + trace: FuseTrace, + trace_fallback: FuseTrace, + matmul: FusedMatmul, + len: usize, +} + +impl MatmulOptimizationInfo { + /// Returns the number of output buffers added by fusion. + pub fn num_output_buffers(&self) -> usize { + self.trace_fallback.resources.outputs.len() + } + + /// Number of operations fused. + pub fn num_ops_fused(&self) -> usize { + self.len + } +} + +impl MatmulOptimizationTuneArg { + pub(crate) fn execute_fused( + &self, + context: &mut Context<'_, CubeFusionHandle>, + selector: FusedMatmulSelector, + ) -> Result, TraceError> { + let launch = FusedMatmulLaunch::new(&self.info.matmul, selector); + let launcher = FuseTraceLauncher::new(&self.info.trace, &launch); + + launcher.launch::(&self.info.client, &self.info.device, context) + } + + pub fn execute_fallback( + &self, + context: &mut Context<'_, CubeFusionHandle>, + ) -> TuneOutput { + self.fallback.run(context); + + #[cfg(feature = "autotune-checks")] + let mut output = TuneOutput::Checked { + handles: Default::default(), + }; + #[cfg(not(feature = "autotune-checks"))] + let output = TuneOutput::UnChecked(core::marker::PhantomData); + + #[cfg(feature = "autotune-checks")] + if let TuneOutput::Checked { handles } = &mut output { + let out_desc = context.tensors.get(&self.info.matmul.op.out.id).unwrap(); + let handle_out = context + .handles + .get_handle(&out_desc.id, &burn_ir::TensorStatus::ReadOnly); + + handles.insert( + self.info.matmul.op.out.id, + (out_desc.shape.dims.clone(), handle_out.clone()), + ); + } + + let launcher = FuseTraceLauncher::new(&self.info.trace_fallback, &ElemwiseRunner); + let output_write = launcher + .launch::(&self.info.client, &self.info.device, context) + .unwrap(); + + output.merge(output_write) + } +} + +impl MatmulOptimization { + pub fn new( + trace: FuseTrace, + trace_fallback: FuseTrace, + client: ComputeClient, + device: R::Device, + len: usize, + matmul: FusedMatmul, + ) -> Self { + let info = MatmulOptimizationInfo { + trace, + trace_fallback, + client, + device, + len, + matmul, + }; + + Self { + info: Arc::new(info), + } + } + /// Execute the optimization. + pub fn execute( + &mut self, + context: &mut Context<'_, CubeFusionHandle>, + fallback: impl FnOnce(usize) -> Box>, + ) { + // The index of the fallback matmul is always 0. + let fallback = fallback(0); + let arg = MatmulOptimizationTuneArg { + info: self.info.clone(), + fallback, + }; + + #[cfg(feature = "autotune")] + fused_matmul_autotune::(arg, context); + + #[cfg(not(feature = "autotune"))] + if arg + .execute_fused::(context, FusedMatmulSelector::default()) + .is_err() + { + arg.execute_fallback::(context); + } + } + + /// Number of operations fused. + pub fn num_ops_fused(&self) -> usize { + self.info.num_ops_fused() + } + + /// Create an optimization from its [state](MatmulOptimizationState). + pub fn from_state(device: &R::Device, state: MatmulOptimizationState) -> Self { + let info = MatmulOptimizationInfo { + trace: state.trace, + trace_fallback: state.trace_fallback, + len: state.len, + client: R::client(device), + device: device.clone(), + matmul: state.matmul.clone(), + }; + + Self { + info: Arc::new(info), + } + } + + /// Convert the optimization to its [state](MatmulOptimizationState). + pub fn to_state(&self) -> MatmulOptimizationState { + MatmulOptimizationState { + trace: self.info.trace.clone(), + trace_fallback: self.info.trace_fallback.clone(), + matmul: self.info.matmul.clone(), + len: self.info.len, + } + } +} + +#[derive(Clone, Copy, Serialize, Deserialize, Debug)] +pub enum FusedMatmulSelector { + Simple { + multi_rows: bool, + tile_matmul: AcceleratedTileKind, + }, + DoubleBuffering { + specialized: bool, + tile_matmul: AcceleratedTileKind, + }, + OrderedDoubleBuffering { + tile_matmul: AcceleratedTileKind, + }, + SimpleVecMat, + DoubleVecMat, + SimpleUnit, + DoubleUnit, +} + +impl FusedMatmulSelector { + /// Not efficient, but only called once when initializing the tunables. + pub fn name(&self) -> String { + let name = match self { + FusedMatmulSelector::Simple { + multi_rows, + tile_matmul, + } => match multi_rows { + false => format!("simple_{tile_matmul:?}"), + true => format!("simple_multirows_{tile_matmul:?}"), + }, + FusedMatmulSelector::DoubleBuffering { + specialized, + tile_matmul, + } => match specialized { + false => format!("double_buffering_{tile_matmul:?}"), + true => format!("double_buffering_specialized_{tile_matmul:?}"), + }, + FusedMatmulSelector::OrderedDoubleBuffering { tile_matmul } => { + format!("double_buffering_ordered_{tile_matmul:?}").to_lowercase() + } + FusedMatmulSelector::SimpleVecMat => "simple_vec_mat".into(), + FusedMatmulSelector::DoubleVecMat => "double_buffering_vec_mat".into(), + FusedMatmulSelector::SimpleUnit => "simple_unit".into(), + FusedMatmulSelector::DoubleUnit => "double_buffering_unit".into(), + }; + + format!("fused_{name}") + } +} + +impl Default for FusedMatmulSelector { + fn default() -> Self { + FusedMatmulSelector::Simple { + multi_rows: false, + tile_matmul: AcceleratedTileKind::Cmma, + } + } +} + +#[derive(new, Clone, Serialize, Deserialize, Debug)] +pub struct FusedMatmul { + pub(crate) lhs: MatmulArg, + pub(crate) rhs: MatmulArg, + out: FuseArg, + pub(crate) op: BinaryOpIr, + pub(crate) selector: FusedMatmulSelector, +} + +#[derive(new)] +pub struct FusedMatmulLaunch<'a> { + pub(crate) matmul: &'a FusedMatmul, + pub(crate) selector: FusedMatmulSelector, +} + +#[derive(Debug)] +pub enum FusedMatmulError { + LaunchError(MatmulSetupError), + InvalidInput(&'static str), +} + +impl From for FusedMatmulError { + fn from(value: MatmulSetupError) -> Self { + Self::LaunchError(value) + } +} + +impl<'a, R: Runtime> Vectorization for FusedMatmulLaunch<'a> { + fn axis(&self, plan: &LaunchPlan<'_, R>) -> VectorizationAxis { + let lhs_id = self.matmul.op.lhs.id; + let rhs_id = self.matmul.op.rhs.id; + + let mut tensor_lhs = None; + let mut tensor_rhs = None; + + for input in plan.handle_inputs.iter() { + match input { + HandleInput::Normal(input) => { + if input.relative_id == lhs_id { + tensor_lhs = Some((input.global_ir.id, &input.handle.strides)); + } + if input.relative_id == rhs_id { + tensor_rhs = Some((input.global_ir.id, &input.handle.strides)); + } + } + HandleInput::QuantValues(input) => { + if input.relative_id == lhs_id { + tensor_lhs = Some((input.global_ir.id, &input.handle.strides)); + } + if input.relative_id == rhs_id { + tensor_rhs = Some((input.global_ir.id, &input.handle.strides)); + } + } + HandleInput::QuantParams(_) => {} + } + } + + let (lhs_id_global, lhs_strides) = tensor_lhs.unwrap(); + let (rhs_id_global, rhs_strides) = tensor_rhs.unwrap(); + + let mut axis = VectorizationAxis::default(); + + if let MatrixBatchLayout::MildlyPermuted { transposed, .. } = + matrix_batch_layout(lhs_strides, self.matmul.lhs.scheme()) + && transposed + { + axis.insert(lhs_id_global, lhs_strides.len() - 2); + } + + if let MatrixBatchLayout::MildlyPermuted { transposed, .. } = + matrix_batch_layout(rhs_strides, self.matmul.rhs.scheme()) + && transposed + { + axis.insert(rhs_id_global, rhs_strides.len() - 2); + } + + axis + } +} + +impl TraceRunner for FusedMatmulLaunch<'_> { + type Error = FusedMatmulError; + + fn run<'a>( + &'a self, + client: &'a ComputeClient, + inputs: GlobalArgsLaunch<'a, R>, + outputs: GlobalArgsLaunch<'a, R>, + configs: &'a [FuseBlockConfig], + ) -> Result<(), FusedMatmulError> { + let global_elems = MatmulGlobalElems { + lhs: self.matmul.lhs.precision().into_type(), + rhs: self.matmul.rhs.precision().into_type(), + out: self.matmul.out.precision().into_type(), + }; + let dtypes = MatmulElems::from_globals(&global_elems); + self.matmul_fused(client, inputs, outputs, &configs[0], dtypes) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +/// Which tile matmul to use for accelerated algorithms +pub enum AcceleratedTileKind { + #[default] + Cmma, + Mma, +} + +macro_rules! with_tile_kind { + ($kind: expr, $T: ident, $launch: expr) => { + match $kind { + AcceleratedTileKind::Cmma => { + type $T = CmmaMatmul; + ($launch)() + } + AcceleratedTileKind::Mma => { + type $T = MmaMatmul; + ($launch)() + } + } + }; +} + +impl FusedMatmulLaunch<'_> { + fn matmul_fused<'a, R: Runtime>( + &'a self, + client: &'a ComputeClient, + inputs: GlobalArgsLaunch<'a, R>, + outputs: GlobalArgsLaunch<'a, R>, + config: &'a FuseBlockConfig, + dtypes: MatmulElems, + ) -> Result<(), FusedMatmulError> { + let lhs_shape = inputs.shape(self.matmul.lhs.data()); + let rhs_shape = inputs.shape(self.matmul.rhs.data()); + let out_shape = outputs.shape_ref(&config.ref_layout, config.rank); + + let lhs_strides = inputs.strides(self.matmul.lhs.data()); + let lhs_scheme = self.matmul.lhs.scheme(); + let rhs_strides = inputs.strides(self.matmul.rhs.data()); + let rhs_scheme = self.matmul.rhs.scheme(); + + if matrix_batch_layout(&lhs_strides, lhs_scheme) == MatrixBatchLayout::HighlyPermuted { + return Err(FusedMatmulError::InvalidInput( + "Lhs needs to be contiguous, but can't when fusing.", + )); + } + if matrix_batch_layout(&rhs_strides, rhs_scheme) == MatrixBatchLayout::HighlyPermuted { + return Err(FusedMatmulError::InvalidInput( + "Rhs needs to be contiguous, but can't when fusing.", + )); + } + + let mut line_sizes = MatmulLineSizes { + lhs: inputs.line_size(self.matmul.lhs.data()), + rhs: inputs.line_size(self.matmul.rhs.data()), + out: match &config.ref_layout { + RefLayout::Concrete(arg) => match arg { + FuseArg::Input(..) => inputs.line_size(arg), + FuseArg::Output(..) => outputs.line_size(arg), + _ => panic!("Invalid ref layout"), + }, + RefLayout::Virtual(_) => 1, + }, + }; + + let address_type = inputs + .required_address_type() + .max(outputs.required_address_type()); + + if line_sizes.out == 1 && (line_sizes.lhs > 1 || line_sizes.rhs > 1) { + return Err(FusedMatmulError::InvalidInput( + "Output line size of 1 removes the gain from fusion", + )); + } + + if let MatmulArg::Quantized { scheme, .. } = self.matmul.lhs { + line_sizes.lhs *= scheme.num_quants(); + } + if let MatmulArg::Quantized { scheme, .. } = self.matmul.rhs { + line_sizes.rhs *= scheme.num_quants(); + } + + let out_strides = MatrixLayout::RowMajor.to_strides(&out_shape); + let problem = MatmulProblem::from_shapes_and_strides( + lhs_shape, + rhs_shape, + out_shape, + lhs_strides, + rhs_strides, + out_strides, + dtypes.as_global_elems(), + address_type, + self.matmul.lhs.scheme(), + self.matmul.rhs.scheme(), + ); + + match self.selector { + FusedMatmulSelector::Simple { + multi_rows, + tile_matmul, + } => with_tile_kind!(tile_matmul, Accelerated, || match launch_inner_fix_dtype::< + R, + SimpleAlgorithm, + >( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &BlueprintStrategy::Inferred(SimpleArgs { multi_rows }), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + }), + FusedMatmulSelector::DoubleBuffering { + specialized, + tile_matmul, + } => with_tile_kind!(tile_matmul, Accelerated, || match launch_inner_fix_dtype::< + R, + CyclicDoubleBufferingAlgorithm, + >( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &BlueprintStrategy::Inferred(DoubleBufferingArgs { specialized }), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + }), + FusedMatmulSelector::OrderedDoubleBuffering { tile_matmul } => { + let row_count = match self.matmul.lhs.precision() { + FuseType::F16 | FuseType::BF16 => 8, + _ => 4, + }; + + with_tile_kind!(tile_matmul, Accelerated, || match launch_inner_fix_dtype::< + R, + OrderedDoubleBufferingAlgorithm, + >( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &BlueprintStrategy::Inferred(OrderedSelectionArgs { + row_count: Some(row_count), + rows_per_plane: Some(2), + partition_k: Some(2), + }), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + }) + } + FusedMatmulSelector::SimpleUnit => { + match launch_inner_fix_dtype::( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &Default::default(), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + } + } + FusedMatmulSelector::DoubleUnit => { + match launch_inner_fix_dtype::( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &Default::default(), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + } + } + FusedMatmulSelector::SimpleVecMat => { + match launch_inner_fix_dtype::( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &Default::default(), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + } + } + FusedMatmulSelector::DoubleVecMat => { + match launch_inner_fix_dtype::( + client, + FusedMatmulInputLaunch::new( + inputs, + config.clone(), + self.matmul.lhs.clone(), + self.matmul.rhs.clone(), + None, + self.matmul.out.clone(), + ), + outputs, + problem, + line_sizes, + &Default::default(), + ) { + Ok(_) => Ok(()), + Err(err) => Err(FusedMatmulError::LaunchError(err)), + } + } + } + } +} + +fn launch_inner_fix_dtype<'a, R: Runtime, A: Routine<()>>( + client: &ComputeClient, + input: FusedMatmulInputLaunch<'a, R>, + output: GlobalArgsLaunch<'a, R>, + problem: MatmulProblem, + line_sizes: MatmulLineSizes, + blueprint_strategy: &BlueprintStrategy<(), A>, +) -> Result<(), MatmulSetupError> { + launch_kernel_virtual::( + client, + input, + output, + (), + problem, + line_sizes, + blueprint_strategy, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/tune.rs new file mode 100644 index 0000000..cd60c0a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/matmul/tune.rs @@ -0,0 +1,269 @@ +use super::optimization::MatmulOptimizationTuneArg; +use crate::{ + CubeFusionHandle, + engine::trace::TuneOutput, + optim::matmul::{AcceleratedTileKind, FusedMatmulSelector}, + tune::{TuneContext, TuneInput}, +}; +use burn_fusion::stream::Context; +use cubecl::{ + AutotuneKey, CubeElement, CubeTuneId, Runtime, + tune::{LocalTuner, Tunable, TunableSet, TuneGroup, local_tuner}, +}; +use cubek::matmul::{ + definition::MatmulKind, + launch::{MatmulAutotuneKey, MatmulGlobalScale, should_tune_double_buffering}, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, AutotuneKey)] +pub struct FusedMatmulAutotuneKey { + matmul_key: MatmulAutotuneKey, + #[autotune(anchor)] + num_out_buffers: usize, + #[autotune(anchor)] + num_ops: usize, +} + +/// Executes autotune on matmul operations +pub fn fused_matmul_autotune( + optimization: MatmulOptimizationTuneArg, + context: &mut Context>, +) { + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + const PRIORITY_MAX: i8 = 3; + const PRIORITY_HIGH: i8 = 2; + const PRIORITY_MEDIUM: i8 = 1; + const PRIORITY_MIN: i8 = 0; + + let cmma = TuneGroup::::new("cmma", |key| { + if matches!( + key.matmul_key.analysis.kind, + MatmulKind::General + // Those variants are just because the unit alternatives aren't very good yet. + | MatmulKind::VecMat | MatmulKind::MatVec + ) { + PRIORITY_MAX + } else { + PRIORITY_MEDIUM + } + }); + + let mma = TuneGroup::::new("mma", |key| { + if matches!( + key.matmul_key.analysis.kind, + // General is usually bad, but I think shapes like 16x8196 would be classed as + // general and are very good with MMA + // Should highly degenerated matrices that aren't VecMat have their own class? + MatmulKind::General | MatmulKind::VecMat | MatmulKind::MatVec + ) { + PRIORITY_MAX + } else { + PRIORITY_MEDIUM + } + }); + + let odd = TuneGroup::::new("odd", |key| { + if key.matmul_key.definition.lhs_pow2_factor == 0 + || key.matmul_key.definition.rhs_pow2_factor == 0 + { + PRIORITY_MAX + } else { + PRIORITY_MIN + } + }); + + let unit = TuneGroup::::new("unit", |key| { + if !matches!(key.matmul_key.analysis.kind, MatmulKind::General) + || matches!( + key.matmul_key.analysis.scale_global, + MatmulGlobalScale::Small + ) + { + PRIORITY_MAX + } else { + PRIORITY_MIN + } + }); + + fn double_buffering_priority(key: &FusedMatmulAutotuneKey, max: i8, min: i8) -> i8 { + if should_tune_double_buffering(key.num_out_buffers > 1, &key.matmul_key) { + max + } else { + min + } + } + + let mut set = TunableSet::new(create_key::, input_gen::).with(Tunable::new( + "fused_matmul_fallback", + tune_fallback::, + )); // First one should always work. + + // Unit matmuls + for (selector, double_buf) in [ + (FusedMatmulSelector::SimpleUnit, false), + (FusedMatmulSelector::DoubleUnit, true), + (FusedMatmulSelector::SimpleVecMat, false), + (FusedMatmulSelector::DoubleVecMat, true), + ] { + set = set.with( + Tunable::new(selector.name(), move |input| { + tune_fused::(input, selector) + }) + .group(&unit, move |key| match double_buf { + true => double_buffering_priority(key, PRIORITY_MAX, PRIORITY_HIGH), + false => PRIORITY_MAX, + }), + ); + } + + // Accelerated matmuls + for (tile_matmul, group) in [ + (AcceleratedTileKind::Cmma, &cmma), + (AcceleratedTileKind::Mma, &mma), + ] { + for (selector, double_buf, extra_group) in [ + ( + FusedMatmulSelector::Simple { + multi_rows: false, + tile_matmul, + }, + false, + None, + ), + ( + FusedMatmulSelector::Simple { + multi_rows: true, + tile_matmul, + }, + false, + None, + ), + ( + FusedMatmulSelector::OrderedDoubleBuffering { tile_matmul }, + true, + None, + ), + ( + FusedMatmulSelector::DoubleBuffering { + specialized: false, + tile_matmul, + }, + true, + None, + ), + ( + FusedMatmulSelector::DoubleBuffering { + specialized: true, + tile_matmul, + }, + true, + Some(&odd), + ), + ] { + let mut tunable = Tunable::new(selector.name(), move |input| { + tune_fused::(input, selector) + }) + .group(group, move |key| match double_buf { + true => double_buffering_priority(key, PRIORITY_MAX, PRIORITY_HIGH), + false => PRIORITY_MAX, + }); + if let Some(group) = extra_group { + tunable = tunable.group(group, |_| PRIORITY_MAX); + } + set = set.with(tunable); + } + } + + set + }); + + TUNER.execute( + &CubeTuneId::new(&optimization.info.client, &optimization.info.device), + &optimization.info.client.clone(), + tunables, + TuneInput::new(context, optimization), + ); +} + +pub(crate) fn create_key( + input: &TuneInput>, +) -> FusedMatmulAutotuneKey { + let opt = input.optimization(); + let context = match input.context() { + TuneContext::Original(context) => context, + TuneContext::Fork(_) => panic!("Not supported when generating key"), + }; + + let lhs = context.tensors.get(&opt.info.matmul.op.lhs.id).unwrap(); + let rhs = context.tensors.get(&opt.info.matmul.op.rhs.id).unwrap(); + let out = context.tensors.get(&opt.info.matmul.op.out.id).unwrap(); + + let lhs_strides = context + .handles + .get_handle(&lhs.id, &burn_ir::TensorStatus::ReadOnly) + .strides; + let rhs_strides = context + .handles + .get_handle(&rhs.id, &burn_ir::TensorStatus::ReadOnly) + .strides; + + let key = MatmulAutotuneKey::generate( + &opt.info.client, + &lhs.shape, + &rhs.shape, + &lhs_strides, + &rhs_strides, + lhs.dtype.into(), + rhs.dtype.into(), + out.dtype.into(), + opt.info.matmul.lhs.scheme(), + opt.info.matmul.rhs.scheme(), + ); + FusedMatmulAutotuneKey::new(key, opt.info.num_output_buffers(), opt.info.num_ops_fused()) +} + +fn input_gen( + _key: &FusedMatmulAutotuneKey, + input: &TuneInput>, +) -> TuneInput> { + input.clone() +} + +fn tune_fused( + input: TuneInput>, + selector: FusedMatmulSelector, +) -> Result, String> { + let optimization = input.optimization(); + let context = input.context(); + + match context { + TuneContext::Original(context) => match optimization.execute_fused::(context, selector) + { + Ok(out) => Ok(out), + Err(_) => { + return tune_fallback::(input); + } + }, + TuneContext::Fork(mut context_owned) => { + optimization.execute_fused::(&mut context_owned.as_context(), selector) + } + } + .map_err(|e| format!("{e:?}")) +} + +fn tune_fallback( + input: TuneInput>, +) -> Result, String> { + let optimization = input.optimization(); + let context = input.context(); + + Ok(match context { + TuneContext::Original(context) => optimization.execute_fallback::(context), + TuneContext::Fork(mut context_owned) => { + optimization.execute_fallback::(&mut context_owned.as_context()) + } + }) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/mod.rs new file mode 100644 index 0000000..ff3f4d8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/mod.rs @@ -0,0 +1,8 @@ +pub mod elemwise; +pub mod matmul; +pub mod reduce; +pub mod reduce_broadcasted; + +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/args.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/args.rs new file mode 100644 index 0000000..7b8ff77 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/args.rs @@ -0,0 +1,208 @@ +use crate::engine::codegen::{ + io::{ref_buffer_len, ref_len, ref_line_size, ref_shape, ref_stride}, + ir::{FuseArg, FuseBlockConfig, GlobalArgs, GlobalArgsExpand, LocalArgs, LocalArgsExpand}, + kernel::{fuse_on_read, fuse_on_write, init_locals}, +}; +use cubecl::prelude::*; +use cubek::reduce::components::args::{ReduceArgs, ReduceDType}; + +#[derive(Clone)] +pub struct FusedReduceArgs; + +#[derive(CubeType, CubeLaunch)] +pub struct FusedReduceInput { + pub global: GlobalArgs, + #[cube(comptime)] + pub config: FuseBlockConfig, + #[cube(comptime)] + pub arg: FuseArg, +} + +#[derive(CubeType, CubeLaunch)] +pub struct FusedReduceOutput { + pub global: GlobalArgs, + #[cube(comptime)] + pub config: FuseBlockConfig, + #[cube(comptime)] + pub arg: FuseArg, +} + +pub struct FusedReduceState { + inputs: *const GlobalArgs, + outputs: *mut GlobalArgs, + locals_on_read: *mut LocalArgs, + locals_on_write: *mut LocalArgs, + config_on_read: FuseBlockConfig, + config_on_write: FuseBlockConfig, + // TODO: Should be a list when multiple blocks are there. + input: FuseArg, + out: FuseArg, +} + +#[derive(Clone)] +pub struct FusedReduceStateExpand { + inputs: GlobalArgsExpand, + outputs: GlobalArgsExpand, + locals_on_read: LocalArgsExpand, + locals_on_write: LocalArgsExpand, + config_on_read: FuseBlockConfig, + config_on_write: FuseBlockConfig, + input: FuseArg, + out: FuseArg, +} + +#[cube] +impl ReduceArgs for FusedReduceArgs { + type Input = FusedReduceInput; + type Output = FusedReduceOutput; + type State = FusedReduceState; + + fn init_state( + input: &Self::Input, + output: &mut Self::Output, + ) -> Self::State

{ + let mut locals_read = init_locals(&input.global, &mut output.global, &input.config); + let mut locals_write = init_locals(&input.global, &mut output.global, &output.config); + // TODO Add stuff from previous blocks to the local of each block. + FusedReduceState::new(input, output, &mut locals_read, &mut locals_write) + } + + fn read_input(state: &Self::State

, index: usize) -> Line { + let value = fuse_on_read::( + unsafe { &(*state.inputs) }, + unsafe { &mut (*state.outputs) }, + unsafe { &mut (*state.locals_on_read) }, + index, + comptime! { + let mut sequence = Sequence::new(); + // TODO: Register local arguments from previous blocks. + sequence.push(state.input.clone()); + sequence + }, + &state.config_on_read, + )[0]; + value + } + + fn read_output(_state: &Self::State

, _index: usize) -> Line { + Line::empty(1usize) + } + + fn write_output(state: &mut Self::State

, index: usize, value: Line) { + let mut values = Registry::>::new(); + let mut args = comptime![Vec::::new()]; + + values.insert(comptime![state.out.clone()], value); + comptime![args.push(state.out.clone())]; + fuse_on_write( + unsafe { &(*state.inputs) }, + unsafe { &mut (*state.outputs) }, + unsafe { &mut (*state.locals_on_write) }, + index, + values, + args, + &state.config_on_write, + ); + } + + fn len_input(state: &Self::State

) -> usize { + ref_len( + unsafe { &(*state.inputs) }, + unsafe { &(*state.outputs) }, + unsafe { &(*state.locals_on_read) }, + &state.config_on_read, + ) + } + + fn len_output(state: &Self::State

) -> usize { + ref_len( + unsafe { &(*state.inputs) }, + unsafe { &(*state.outputs) }, + unsafe { &(*state.locals_on_write) }, + &state.config_on_write, + ) + } + + fn buffer_len_input(state: &Self::State

) -> usize { + ref_buffer_len( + unsafe { &(*state.inputs) }, + unsafe { &(*state.outputs) }, + unsafe { &(*state.locals_on_read) }, + &state.config_on_read, + ) + } + + fn buffer_len_output(state: &Self::State

) -> usize { + ref_buffer_len( + unsafe { &(*state.inputs) }, + unsafe { &(*state.outputs) }, + unsafe { &(*state.locals_on_write) }, + &state.config_on_write, + ) + } + + fn rank_input(state: &Self::State

) -> usize { + state.config_on_read.rank.runtime() + } + + fn rank_output(state: &Self::State

) -> usize { + state.config_on_write.rank.runtime() + } + + fn shape_input(state: &Self::State

, dim: usize) -> usize { + ref_shape(unsafe { &(*state.locals_on_read) }, dim) + } + + fn shape_output(state: &Self::State

, dim: usize) -> usize { + ref_shape(unsafe { &(*state.locals_on_write) }, dim) + } + + fn stride_input(state: &Self::State

, dim: usize) -> usize { + ref_stride(unsafe { &(*state.locals_on_read) }, dim) + } + + fn stride_output(state: &Self::State

, dim: usize) -> usize { + ref_stride(unsafe { &(*state.locals_on_write) }, dim) + } + + fn line_size_input(state: &Self::State

) -> comptime_type!(LineSize) { + ref_line_size(unsafe { &(*state.locals_on_read) }) + } + + fn line_size_output(state: &Self::State

) -> comptime_type!(LineSize) { + ref_line_size(unsafe { &(*state.locals_on_write) }) + } +} + +#[cube] +impl FusedReduceState { + pub fn new( + inputs: &FusedReduceInput, + outputs: &mut FusedReduceOutput, + locals_on_read: &mut LocalArgs, + locals_on_write: &mut LocalArgs, + ) -> FusedReduceState { + FusedReduceState { + inputs: &inputs.global, + outputs: &mut outputs.global, + locals_on_read, + locals_on_write, + config_on_read: comptime![inputs.config.clone()], + config_on_write: comptime![outputs.config.clone()], + input: comptime![inputs.arg.clone()], + out: comptime![outputs.arg.clone()], + } + } +} + +impl CubeType for FusedReduceState { + type ExpandType = FusedReduceStateExpand; +} + +impl IntoMut for FusedReduceStateExpand { + fn into_mut(self, _context: &mut Scope) -> Self { + self + } +} + +impl CubeDebug for FusedReduceStateExpand {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/fuser.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/fuser.rs new file mode 100644 index 0000000..add86fd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/fuser.rs @@ -0,0 +1,328 @@ +use super::{ + ReduceSettings, + optimization::{FusedReduce, ReduceInstruction, ReduceOptimization}, +}; +use crate::{ + engine::{ + codegen::ir::FuseType, + fuser::TraceOperationFuser, + settings::{FuseSettings, RefLayoutSetting, VectorizationSetting}, + }, + optim::CubeOptimization, +}; +use burn_fusion::{FuserStatus, OperationFuser}; +use burn_ir::{NumericOperationIr, OperationIr, ReduceDimOpIr}; +use burn_std::Shape; +use cubecl::Runtime; + +/// Fuses element wise operations around a reduce operation. +pub struct ReduceFuser { + pub(crate) fuser: TraceOperationFuser, + pub(crate) fuser_read_fallback: TraceOperationFuser, + fuser_write_fallback: TraceOperationFuser, + settings_write: FuseSettings, + pub(crate) device: R::Device, + pub(crate) reduce: Option, + settings: ReduceSettings, +} + +impl Clone for ReduceFuser { + fn clone(&self) -> Self { + Self { + fuser: self.fuser.clone(), + fuser_read_fallback: self.fuser_read_fallback.clone(), + fuser_write_fallback: self.fuser_write_fallback.clone(), + settings_write: self.settings_write, + device: self.device.clone(), + reduce: self.reduce.clone(), + settings: self.settings, + } + } +} + +#[derive(Debug)] +pub enum ReduceFuserInfo { + FusedReduce { shape_input_id: Shape, axis: usize }, + FusedElemwise { shape_id: Shape }, +} + +impl ReduceFuser { + pub fn new(device: R::Device, bool_precision: FuseType, settings: ReduceSettings) -> Self { + let client = R::client(&device); + let props = client.properties(); + let max_bindings = props.hardware.max_bindings; + let settings_read = FuseSettings { + // Inplace would work, but not when we have a concrete output to write too. + inplace: true, + ref_layout: RefLayoutSetting::OnlyContiguous, + broadcast: false, + output_shape_updates: true, + vectorization: VectorizationSetting::Activated, + }; + let settings_write = FuseSettings { + inplace: false, + output_shape_updates: false, + vectorization: VectorizationSetting::SmallerOrEqualThanPreviousBlock { block_pos: 0 }, + broadcast: false, + ref_layout: RefLayoutSetting::OnlyContiguous, + }; + let settings_fallback = FuseSettings::default(); + + Self { + fuser: TraceOperationFuser::new(max_bindings, bool_precision, settings_read), + fuser_read_fallback: TraceOperationFuser::new( + max_bindings, + bool_precision, + settings_fallback, + ), + fuser_write_fallback: TraceOperationFuser::new( + max_bindings, + bool_precision, + settings_fallback, + ), + settings_write, + device, + reduce: None, + settings, + } + } + + pub fn reduce_info(&self) -> ReduceFuserInfo { + match &self.reduce { + Some(reduce) => { + let shape_input_id = reduce.op.input.shape.clone(); + let axis = reduce.axis; + + ReduceFuserInfo::FusedReduce { + shape_input_id, + axis, + } + } + None => { + let shape_id = self.fuser_read_fallback.current_output_shape.clone(); + ReduceFuserInfo::FusedElemwise { shape_id } + } + } + } + fn on_reduce(&mut self, op: &ReduceDimOpIr, inst: ReduceInstruction) { + // TODO: Fix: we need to have fuse-on-read with an identity block. + // + // if self.fuser.num_ops == 0 && false { + // self.fuser.current_output_shape = op.input.shape.dims.clone(); + // } else if self.fuser.current_output_shape != op.input.shape.dims { + + if self.fuser.current_output_shape != op.input.shape { + self.fuser.close(); + self.fuser_read_fallback.close(); + return; + } + + let [input] = self + .fuser + .next_block([&op.input], self.settings_write, false); + + let output = self.fuser.output_unhandled(&op.out); + let axis = op.axis; + + let fuse_on_write_activated = match self.settings { + ReduceSettings::Always => true, + // We only activate fuse-on-write when the reduction isn't on the last dimension, otherwise + // vectorization is impossible. Only [LineMode::Perpendicular] supports vectorization. + // + // We could still fuse some output operations, but it would probably lead to worse performance. + ReduceSettings::OnlyParallel => axis != op.input.shape.rank() - 1, + ReduceSettings::Never => false, + }; + + if !fuse_on_write_activated { + self.fuser.close(); + } + + let acc = match inst { + ReduceInstruction::Mean | ReduceInstruction::Prod | ReduceInstruction::Sum => { + match input.precision() { + FuseType::F16 | FuseType::BF16 => FuseType::F32, + FuseType::I16 | FuseType::I8 => FuseType::I32, + FuseType::U16 | FuseType::U8 => FuseType::U32, + _ => input.precision(), + } + } + _ => input.precision(), + }; + + self.reduce = Some(FusedReduce { + input, + output, + acc, + axis, + op: op.clone(), + use_planes: false, + shared: false, + inst, + }); + + self.fuser_read_fallback.close(); + } + + fn on_elemwise_read(&mut self, operation: &OperationIr) { + let can_register = + self.fuser.can_fuse(operation) && self.fuser_read_fallback.can_fuse(operation); + + match can_register { + true => { + self.fuser.fuse(operation); + self.fuser_read_fallback.fuse(operation); + } + false => { + self.fuser.close(); + self.fuser_read_fallback.close(); + } + }; + } + + fn on_elemwise_write(&mut self, operation: &OperationIr) { + let can_register = + self.fuser.can_fuse(operation) && self.fuser_write_fallback.can_fuse(operation); + + match can_register { + true => { + self.fuser.fuse(operation); + self.fuser_write_fallback.fuse(operation); + } + false => { + self.fuser.close(); + self.fuser_write_fallback.close(); + } + }; + } +} + +impl OperationFuser> for ReduceFuser { + fn fuse(&mut self, operation: &OperationIr) { + if let FuserStatus::Closed = self.fuser.status() { + return; + } + + if self.reduce.is_none() { + if let OperationIr::NumericFloat(_, op) = operation { + match op { + NumericOperationIr::SumDim(op) => { + self.on_reduce(op, ReduceInstruction::Sum); + } + NumericOperationIr::MeanDim(op) => { + self.on_reduce(op, ReduceInstruction::Mean); + } + NumericOperationIr::ProdDim(op) => { + self.on_reduce(op, ReduceInstruction::Prod); + } + NumericOperationIr::ArgMax(op) => { + self.on_reduce(op, ReduceInstruction::ArgMax); + } + NumericOperationIr::ArgMin(op) => { + self.on_reduce(op, ReduceInstruction::ArgMin); + } + NumericOperationIr::MinDim(op) => { + self.on_reduce(op, ReduceInstruction::Min); + } + NumericOperationIr::MaxDim(op) => { + self.on_reduce(op, ReduceInstruction::Max); + } + NumericOperationIr::MaxAbsDim(op) => { + self.on_reduce(op, ReduceInstruction::MaxAbs); + } + _ => { + self.on_elemwise_read(operation); + } + }; + } else if let OperationIr::NumericInt(_, op) = operation { + match op { + NumericOperationIr::SumDim(op) => { + self.on_reduce(op, ReduceInstruction::Sum); + } + NumericOperationIr::MeanDim(op) => { + self.on_reduce(op, ReduceInstruction::Mean); + } + NumericOperationIr::ProdDim(op) => { + self.on_reduce(op, ReduceInstruction::Prod); + } + NumericOperationIr::ArgMax(op) => { + self.on_reduce(op, ReduceInstruction::ArgMax); + } + NumericOperationIr::ArgMin(op) => { + self.on_reduce(op, ReduceInstruction::ArgMin); + } + NumericOperationIr::MinDim(op) => { + self.on_reduce(op, ReduceInstruction::Min); + } + NumericOperationIr::MaxDim(op) => { + self.on_reduce(op, ReduceInstruction::Max); + } + NumericOperationIr::MaxAbsDim(op) => { + self.on_reduce(op, ReduceInstruction::MaxAbs); + } + _ => { + self.on_elemwise_read(operation); + } + }; + } else { + self.on_elemwise_read(operation); + } + } else { + self.on_elemwise_write(operation); + } + } + + fn finish(&mut self) -> CubeOptimization { + let client = R::client(&self.device); + let trace = self.fuser.finish(); + let trace_read_fallback = self.fuser_read_fallback.finish(); + let trace_write_fallback = self.fuser_write_fallback.finish(); + let fuse_reduce = self.reduce.as_ref().unwrap(); + + let reduce = ReduceOptimization::new( + trace, + trace_read_fallback, + trace_write_fallback, + client, + self.device.clone(), + self.len(), + self.fuser_read_fallback.len(), + fuse_reduce.clone(), + self.settings, + ); + + CubeOptimization::Reduce(reduce) + } + + fn reset(&mut self) { + self.fuser.reset(); + self.fuser_read_fallback.reset(); + self.fuser_write_fallback.reset(); + self.reduce = None; + } + + fn status(&self) -> burn_fusion::FuserStatus { + self.fuser.status() + } + + fn properties(&self) -> burn_fusion::FuserProperties { + let mut properties = self.fuser.properties(); + + if self.reduce.is_some() { + properties.ready = true; + properties.score += 1; + } else { + properties.ready = false; + }; + + properties + } + + fn len(&self) -> usize { + self.fuser.len() + if self.reduce.is_some() { 1 } else { 0 } + } + + fn clone_dyn(&self) -> Box>> { + Box::new(self.clone()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/mod.rs new file mode 100644 index 0000000..75a082a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/mod.rs @@ -0,0 +1,8 @@ +mod fuser; +mod optimization; + +pub(crate) mod args; +pub(crate) mod tune; + +pub use fuser::*; +pub use optimization::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/optimization.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/optimization.rs new file mode 100644 index 0000000..fb5d95b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/optimization.rs @@ -0,0 +1,492 @@ +use super::args::{ + FusedReduceInput, FusedReduceInputLaunch, FusedReduceOutput, FusedReduceOutputLaunch, +}; +#[cfg(feature = "autotune")] +use super::tune::fused_reduce_autotune; +use crate::{ + CubeFusionHandle, FallbackOperation, + engine::{ + codegen::ir::{ + FuseArg, FuseBlockConfig, FuseType, GlobalArgsLaunch, RefLayout, + multi_block_variables_init, + }, + launch::{ + FuseTraceLauncher, + runner::{TraceRunner, Vectorization}, + }, + trace::{FuseTrace, TraceError, TuneOutput}, + }, + optim::{elemwise::ElemwiseRunner, reduce::args::FusedReduceArgs}, +}; +use burn_fusion::stream::Context; +use burn_ir::ReduceDimOpIr; +use burn_std::DType; +use cubecl::{Runtime, client::ComputeClient, ir::StorageType, prelude::*}; +use cubek::reduce::{ + LineMode, ReduceDtypes, ReduceError, + components::instructions::ReduceOperationConfig, + init_tensors, + launch::{RoutineStrategy, reduce_kernel_virtual}, + routines::{ + ReduceBlueprint, ReduceLaunchSettings, ReduceLineSettings, ReduceProblem, Routine, + cube::CubeRoutine, plane::PlaneRoutine, unit::UnitRoutine, + }, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[cfg(not(feature = "autotune"))] +use cubek::reduce::routines::{BlueprintStrategy, unit::UnitStrategy}; + +pub struct ReduceOptimization { + pub(crate) info: Arc>, +} + +pub(crate) struct ReduceOptimizationInfo { + pub(crate) trace: FuseTrace, + trace_read_fallback: FuseTrace, + trace_write_fallback: FuseTrace, + pub(crate) client: ComputeClient, + pub(crate) device: R::Device, + pub(crate) len: usize, + pub(crate) len_read: usize, + pub(crate) reduce: FusedReduce, + settings: ReduceSettings, +} + +impl ReduceOptimizationInfo { + pub fn from_state(device: &R::Device, state: ReduceOptimizationState) -> Self { + let client = R::client(device); + + Self { + trace: state.trace, + trace_read_fallback: state.trace_read_fallback, + trace_write_fallback: state.trace_write_fallback, + client, + device: device.clone(), + len: state.len, + len_read: state.len_read, + reduce: state.reduce, + settings: state.settings, + } + } + pub fn to_state(&self) -> ReduceOptimizationState { + ReduceOptimizationState { + trace: self.trace.clone(), + trace_read_fallback: self.trace_read_fallback.clone(), + trace_write_fallback: self.trace_write_fallback.clone(), + len: self.len, + len_read: self.len_read, + reduce: self.reduce.clone(), + settings: self.settings, + } + } +} + +#[derive(Serialize, Deserialize, Copy, Clone)] +pub enum ReduceSettings { + Always, + /// We only activate fuse-on-write when the reduction isn't on the last dimension, otherwise + /// vectorization is impossible. Only [LineMode::Perpendicular] supports vectorization. + /// + /// We could still fuse some output operations, but it would probably lead to worse performance. + OnlyParallel, + Never, +} + +pub(crate) struct ReduceOptimizationTuneArg { + pub(crate) info: Arc>, + pub(crate) fallback: Arc>>, +} + +impl Clone for ReduceOptimizationTuneArg { + fn clone(&self) -> Self { + Self { + info: self.info.clone(), + fallback: self.fallback.clone(), + } + } +} + +#[derive(Clone, Copy, Serialize, Deserialize, Debug)] +pub enum ReduceInstruction { + ArgMax, + ArgMin, + Mean, + Prod, + Sum, + Max, + Min, + MaxAbs, +} + +pub trait ReduceFallbackFn: Send + Sync { + fn run(&self, context: &mut Context<'_, CubeFusionHandle>); +} + +#[derive(Serialize, Deserialize)] +pub struct ReduceOptimizationState { + pub(crate) trace: FuseTrace, + pub(crate) trace_read_fallback: FuseTrace, + pub(crate) trace_write_fallback: FuseTrace, + pub(crate) reduce: FusedReduce, + pub(crate) len: usize, + pub(crate) len_read: usize, + pub(crate) settings: ReduceSettings, +} + +impl core::fmt::Debug for ReduceOptimizationState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "{{ len_read: {}, len_total: {} }}", + self.len_read, self.len + )) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FusedReduce { + pub(crate) input: FuseArg, + pub(crate) output: FuseArg, + pub(crate) acc: FuseType, + pub(crate) axis: usize, + pub(crate) op: ReduceDimOpIr, + pub(crate) use_planes: bool, + pub(crate) shared: bool, + pub(crate) inst: ReduceInstruction, +} + +#[derive(new)] +pub struct FusedReduceLaunch<'a> { + reduce: &'a FusedReduce, + strategy: RoutineStrategy, +} + +#[derive(Debug)] +pub enum FusedReduceError { + Reduce(ReduceError), + InvalidSelection(Box<&'static str>), + InvalidInput, +} + +impl From for FusedReduceError { + fn from(value: ReduceError) -> Self { + Self::Reduce(value) + } +} + +impl ReduceOptimizationTuneArg { + pub fn execute_fused( + &self, + context: &mut Context<'_, CubeFusionHandle>, + strategy: RoutineStrategy, + ) -> Result, TraceError> { + let launch = FusedReduceLaunch::new(&self.info.reduce, strategy); + let launcher = FuseTraceLauncher::new(&self.info.trace, &launch); + launcher.launch::(&self.info.client, &self.info.device, context) + } + + pub fn execute_fallback( + &self, + context: &mut Context<'_, CubeFusionHandle>, + ) -> TuneOutput { + let launcher = FuseTraceLauncher::new(&self.info.trace_read_fallback, &ElemwiseRunner); + + #[allow(unused_mut)] // It is used when `autotune-checks` is activated. + let mut output_read = launcher + .launch::(&self.info.client, &self.info.device, context) + .unwrap(); + + self.fallback.run(context); + + #[cfg(feature = "autotune-checks")] + if let TuneOutput::Checked { handles } = &mut output_read { + let out_desc = context.tensors.get(&self.info.reduce.op.out.id).unwrap(); + let handle_out = context + .handles + .get_handle(&out_desc.id, &burn_ir::TensorStatus::ReadOnly); + + handles.insert( + self.info.reduce.op.out.id, + (out_desc.shape.dims.clone(), handle_out.clone()), + ); + } + + let launcher = FuseTraceLauncher::new(&self.info.trace_write_fallback, &ElemwiseRunner); + + let output_write = launcher + .launch::(&self.info.client, &self.info.device, context) + .unwrap(); + + output_read.merge(output_write) + } +} + +#[allow(clippy::too_many_arguments)] +impl ReduceOptimization { + pub fn new( + trace: FuseTrace, + trace_read_fallback: FuseTrace, + trace_write_fallback: FuseTrace, + client: ComputeClient, + device: R::Device, + len: usize, + len_read: usize, + reduce: FusedReduce, + settings: ReduceSettings, + ) -> Self { + let info = ReduceOptimizationInfo { + trace, + trace_read_fallback, + trace_write_fallback, + client, + device, + len, + len_read, + reduce, + settings, + }; + + Self { + info: Arc::new(info), + } + } + /// Execute the optimization. + pub fn execute( + &mut self, + context: &mut Context<'_, CubeFusionHandle>, + fallback: impl FnOnce(usize) -> Box>, + ) { + // The index of the fallback reduce is the number of ops fused as read. + let fallback = fallback(self.info.len_read); + let arg = ReduceOptimizationTuneArg { + info: self.info.clone(), + fallback: Arc::new(fallback), + }; + + #[cfg(feature = "autotune")] + fused_reduce_autotune::(arg, context); + + #[cfg(not(feature = "autotune"))] + if arg + .execute_fused::( + context, + RoutineStrategy::Unit(BlueprintStrategy::Inferred(UnitStrategy)), + ) + .is_err() + { + arg.execute_fallback::(context); + } + } + + pub fn num_output_buffers(&self) -> usize { + self.info.trace_read_fallback.resources.outputs.len() + } + + pub fn to_state(&self) -> ReduceOptimizationState { + ReduceOptimizationState { + trace: self.info.trace.clone(), + trace_read_fallback: self.info.trace_read_fallback.clone(), + trace_write_fallback: self.info.trace_write_fallback.clone(), + reduce: self.info.reduce.clone(), + len: self.info.len, + len_read: self.info.len_read, + settings: self.info.settings, + } + } + + pub fn from_state(device: &R::Device, state: ReduceOptimizationState) -> Self { + let client = R::client(device); + + let info = ReduceOptimizationInfo { + trace: state.trace, + trace_read_fallback: state.trace_read_fallback, + trace_write_fallback: state.trace_write_fallback, + reduce: state.reduce, + len: state.len, + len_read: state.len_read, + client, + device: device.clone(), + settings: state.settings, + }; + + Self { + info: Arc::new(info), + } + } + + /// Returns the number of output buffers added by fusion. + pub fn num_ops_fused(&self) -> usize { + self.info.len + } +} + +// TODO: Implement better vectorization here. +impl Vectorization for FusedReduceLaunch<'_> {} + +impl TraceRunner for FusedReduceLaunch<'_> { + type Error = FusedReduceError; + + fn run<'a>( + &'a self, + client: &'a ComputeClient, + inputs: GlobalArgsLaunch<'a, R>, + outputs: GlobalArgsLaunch<'a, R>, + configs: &'a [FuseBlockConfig], + ) -> Result<(), FusedReduceError> { + let [config_read, config_write] = [&configs[0], &configs[1]]; + let shape = match &config_read.ref_layout { + RefLayout::Concrete(FuseArg::Output(..)) => { + outputs.shape_ref(&config_read.ref_layout, config_read.rank) + } + _ => inputs.shape_ref(&config_read.ref_layout, config_read.rank), + }; + let reduce_count: usize = shape + .iter() + .enumerate() + .map(|(i, s)| if i == self.reduce.axis { 1 } else { *s }) + .product(); + + let line_mode = match self.reduce.axis == config_read.rank - 1 { + true => LineMode::Parallel, + false => LineMode::Perpendicular, + }; + let address_type = inputs + .required_address_type() + .max(outputs.required_address_type()); + + let settings = ReduceLineSettings { + line_mode, + line_size_input: config_read.width, + line_size_output: config_write.width, + }; + let problem = ReduceProblem { + vector_size: shape[self.reduce.axis], + vector_count: reduce_count, + axis: self.reduce.axis, + dtypes: ReduceDtypes { + input: self.reduce.op.input.dtype.into(), + output: self.reduce.op.out.dtype.into(), + accumulation: self.reduce.acc.into_elem().into(), + }, + address_type, + }; + + let (blueprint, settings) = match self.strategy.clone() { + RoutineStrategy::Unit(strategy) => { + let routine = UnitRoutine; + routine.prepare(client, problem, settings, strategy)? + } + RoutineStrategy::Plane(strategy) => { + let routine = PlaneRoutine; + routine.prepare(client, problem, settings, strategy)? + } + RoutineStrategy::Cube(strategy) => { + let routine = CubeRoutine; + routine.prepare(client, problem, settings, strategy)? + } + }; + + let kwargs = ReduceKwArgs { + client, + inputs, + outputs, + axis: self.reduce.axis, + config_fuse_read: config_read.clone(), + config_fuse_write: config_write.clone(), + input: self.reduce.input.clone(), + output: self.reduce.output.clone(), + blueprint, + settings, + }; + let result = launch_reduce_mixed_precision( + kwargs, + self.reduce.inst, + self.reduce.op.input.dtype, + self.reduce.op.out.dtype, + DType::from(self.reduce.acc.into_elem()), + ); + + match result { + Ok(_) => Ok(()), + Err(err) => Err(FusedReduceError::Reduce(ReduceError::Launch(err))), + } + } +} + +struct ReduceKwArgs<'a, 'b, Run: Runtime> { + client: &'b ComputeClient, + inputs: GlobalArgsLaunch<'a, Run>, + outputs: GlobalArgsLaunch<'a, Run>, + axis: usize, + blueprint: ReduceBlueprint, + settings: ReduceLaunchSettings, + config_fuse_read: FuseBlockConfig, + config_fuse_write: FuseBlockConfig, + input: FuseArg, + output: FuseArg, +} + +fn launch_reduce_mixed_precision( + kwargs: ReduceKwArgs<'_, '_, Run>, + instruction: ReduceInstruction, + dtype_input: DType, + dtype_output: DType, + dtype_acc: DType, +) -> Result<(), LaunchError> { + let config = match instruction { + ReduceInstruction::ArgMax => ReduceOperationConfig::ArgMax, + ReduceInstruction::ArgMin => ReduceOperationConfig::ArgMin, + ReduceInstruction::Prod => ReduceOperationConfig::Prod, + ReduceInstruction::Mean => ReduceOperationConfig::Mean, + ReduceInstruction::Sum => ReduceOperationConfig::Sum, + ReduceInstruction::Max => ReduceOperationConfig::Max, + ReduceInstruction::Min => ReduceOperationConfig::Min, + ReduceInstruction::MaxAbs => ReduceOperationConfig::MaxAbs, + }; + launch_reduce::(kwargs, config, dtype_input, dtype_output, dtype_acc) +} + +fn launch_reduce( + kwargs: ReduceKwArgs<'_, '_, Run>, + inst: ReduceOperationConfig, + dtype_input: DType, + dtype_output: DType, + dtype_acc: DType, +) -> Result<(), LaunchError> { + unsafe { + reduce_kernel_fused::launch_unchecked::( + kwargs.client, + kwargs.settings.cube_count, + kwargs.settings.cube_dim, + kwargs.settings.address_type, + FusedReduceInputLaunch::new(kwargs.inputs, kwargs.config_fuse_read, kwargs.input), + FusedReduceOutputLaunch::new(kwargs.outputs, kwargs.config_fuse_write, kwargs.output), + ScalarArg::new(kwargs.axis), + kwargs.blueprint, + inst, + dtype_input.into(), + dtype_output.into(), + dtype_acc.into(), + ) + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub fn reduce_kernel_fused( + input: &FusedReduceInput, + output: &mut FusedReduceOutput, + axis_reduce: usize, + #[comptime] blueprint: ReduceBlueprint, + #[comptime] config: ReduceOperationConfig, + #[define(In)] _input_dtype: StorageType, + #[define(Out)] _output_dtype: StorageType, + #[define(Acc)] _acc_dtype: StorageType, +) { + multi_block_variables_init(&input.config, &mut output.global.variables); + multi_block_variables_init(&output.config, &mut output.global.variables); + + let (input, mut output) = init_tensors::(input, output); + + reduce_kernel_virtual::(&input, &mut output, axis_reduce, blueprint, config); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/tune.rs new file mode 100644 index 0000000..c1909f9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce/tune.rs @@ -0,0 +1,196 @@ +use super::optimization::ReduceOptimizationTuneArg; +use crate::{ + CubeFusionHandle, + engine::trace::TuneOutput, + tune::{TuneContext, TuneInput}, +}; +use burn_fusion::stream::Context; +use cubecl::{ + AutotuneKey, CubeElement, CubeTuneId, Runtime, + tune::{LocalTuner, Tunable, TunableSet, TuneGroup, local_tuner}, +}; +use cubek::reduce::{ + launch::{RoutineStrategy, tune_key::ReduceAutotuneKey}, + routines::{BlueprintStrategy, cube::CubeStrategy, plane::PlaneStrategy, unit::UnitStrategy}, +}; +use serde::{Deserialize, Serialize}; + +/// Autotune key for standard fused reduction operations. +/// +/// Records metadata about the fusion graph (IO and ops) alongside +/// the core reduction parameters to ensure stable kernel selection. +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, AutotuneKey)] +pub struct FusedReduceAutotuneKey { + reduce_key: ReduceAutotuneKey, + #[autotune(anchor)] + fuse_num_reads: usize, + #[autotune(anchor)] + fuse_num_writes: usize, + #[autotune(anchor)] + fuse_num_ops: usize, +} + +/// Executes autotuning for fused reduction operations. +/// +/// This tuner evaluates different hardware-specific strategies (Plane, Cube, Unit) +/// and assigns priorities based on the `vector_count` of the reduction. +pub fn fused_reduce_autotune( + arg: ReduceOptimizationTuneArg, + context: &mut Context>, +) { + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + const PRIORITY_MAX: i8 = 2; + const PRIORITY_MIN: i8 = 1; + + let mut set = TunableSet::new(create_key::, input_gen::); + let group = TuneGroup::::new("fused_reduce", |_key| PRIORITY_MAX); + + // Fallback implementation for robustness. + set = set.with(Tunable::new( + "fused_reduce_fallback", + tune_fallback::, + )); + + // Define properties to categorize hardware strategies. + enum ReduceProps { + GreatWithLowReduceCount, + GreatWithHighReduceCount, + Balanced, + } + + let strategies = [ + ( + "fused_unit", + RoutineStrategy::Unit(BlueprintStrategy::Inferred(UnitStrategy)), + ReduceProps::GreatWithHighReduceCount, + ), + ( + "fused_plane", + RoutineStrategy::Plane(BlueprintStrategy::Inferred(PlaneStrategy { + independent: true, + })), + ReduceProps::Balanced, + ), + ( + "fused_cube", + RoutineStrategy::Cube(BlueprintStrategy::Inferred(CubeStrategy { + // Two steps reduction doesn't work with fuse-on-write, we can't activate plane + // when using the cube algo. + use_planes: false, + })), + ReduceProps::GreatWithLowReduceCount, + ), + ]; + + for (name, strategy, props) in strategies { + let tunable = Tunable::new(name, move |input| tune_reduce::(input, &strategy)) + .group(&group, move |key| match props { + ReduceProps::GreatWithLowReduceCount => { + if key.reduce_key.vector_count < 128 { + PRIORITY_MAX + } else { + PRIORITY_MIN + } + } + ReduceProps::GreatWithHighReduceCount => { + if key.reduce_key.vector_count > 64 { + PRIORITY_MAX + } else { + PRIORITY_MIN + } + } + ReduceProps::Balanced => PRIORITY_MAX, + }); + + set = set.with(tunable); + } + + set + }); + + TUNER.execute( + &CubeTuneId::new(&arg.info.client, &arg.info.device), + &arg.info.client.clone(), + tunables, + TuneInput::new(context, arg), + ); +} + +/// Creates the autotune key by extracting tensor metadata and fusion block statistics. +pub(crate) fn create_key( + input: &TuneInput>, +) -> FusedReduceAutotuneKey { + let opt = input.optimization(); + let context = match input.context() { + TuneContext::Original(context) => context, + TuneContext::Fork(_) => panic!("Forked context not supported for key generation"), + }; + + let input_tensor = context.tensors.get(&opt.info.reduce.op.input.id).unwrap(); + let out_tensor = context.tensors.get(&opt.info.reduce.op.out.id).unwrap(); + let acc = opt.info.reduce.acc.into_elem(); + + let key = ReduceAutotuneKey::generate( + input_tensor.dtype.into(), + out_tensor.dtype.into(), + acc, + &input_tensor.shape, + opt.info.reduce.axis == input_tensor.shape.rank() - 1, + opt.info.reduce.axis, + ); + + // Assume the fusion contains at least a read and a write block. + let read_block = &opt.info.trace.blocks[0]; + let write_block = &opt.info.trace.blocks[1]; + + FusedReduceAutotuneKey::new( + key, + read_block.reads.len() + write_block.reads.len(), + read_block.writes.len() + write_block.writes.len(), + read_block.ops.len() + write_block.ops.len(), + ) +} + +/// Identity generator for tuning inputs. +fn input_gen( + _key: &FusedReduceAutotuneKey, + input: &TuneInput>, +) -> TuneInput> { + input.clone() +} + +/// Executes a fused reduction optimization. +fn tune_reduce( + input: TuneInput>, + strategy: &RoutineStrategy, +) -> Result, String> { + let optimization = input.optimization(); + + match input.context() { + TuneContext::Original(context) => { + optimization.execute_fused::(context, strategy.clone()) + } + TuneContext::Fork(mut context_owned) => { + optimization.execute_fused::(&mut context_owned.as_context(), strategy.clone()) + } + } + .map_err(|e| format!("{e:?}")) +} + +/// Executes the fallback path for a reduction optimization. +fn tune_fallback( + input: TuneInput>, +) -> Result, String> { + let optimization = input.optimization(); + + match input.context() { + TuneContext::Original(context) => optimization.execute_fallback::(context), + TuneContext::Fork(mut context_owned) => { + optimization.execute_fallback::(&mut context_owned.as_context()) + } + }; + + Ok(TuneOutput::UnChecked(std::marker::PhantomData)) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/base.rs new file mode 100644 index 0000000..023c9cb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/base.rs @@ -0,0 +1,375 @@ +use crate::{ + engine::codegen::ir::FuseType, + optim::{ + CubeOptimization, + reduce::{ReduceFuser, ReduceFuserInfo, ReduceSettings}, + reduce_broadcasted::{ + ReduceBroadcastedOptimization, ReduceBroadcastedOptimizationInfo, + fuser::{ + block::{ReduceBlockFuser, ReduceBlockFusionAnalysis, ReduceBroadcastedStatus}, + full::ReduceBroadcastedFullFuser, + full_analyzer::FullFuserAnalyzer, + }, + }, + }, +}; +use burn_fusion::{FuserProperties, FuserStatus, OperationFuser}; +use burn_ir::OperationIr; +use cubecl::Runtime; +use std::sync::Arc; + +/// Fuses element wise operations around a reduce operation. +pub struct ReduceBroadcastedFuser { + blocks: Vec>, + fuser_default: ReduceFuser, + num_ops: usize, + state: ReduceBroadcastedStatus, + max_bindings: u32, + bool_precision: FuseType, +} + +impl Clone for ReduceBroadcastedFuser { + fn clone(&self) -> Self { + Self { + blocks: self.blocks.clone(), + fuser_default: self.fuser_default.clone(), + num_ops: self.num_ops, + state: self.state.clone(), + max_bindings: self.max_bindings, + bool_precision: self.bool_precision, + } + } +} + +impl ReduceBroadcastedFuser { + pub fn new(device: R::Device, bool_precision: FuseType) -> Self { + let fuser = ReduceFuser::new(device, bool_precision, ReduceSettings::Always); + let max_bindings = fuser.fuser.max_bindings; + let block = ReduceBlockFuser::new(fuser.clone()); + + Self { + blocks: vec![block], + fuser_default: fuser, + num_ops: 0, + state: ReduceBroadcastedStatus::Starting, + max_bindings, + bool_precision, + } + } +} + +impl OperationFuser> for ReduceBroadcastedFuser { + fn fuse(&mut self, operation: &OperationIr) { + if matches!( + &self.state, + ReduceBroadcastedStatus::Closed | ReduceBroadcastedStatus::Abort + ) { + return; + } + + let block = self.blocks.last_mut().unwrap(); + let analyze = block.analyze(operation, &self.state, &self.fuser_default); + + let info = match analyze { + ReduceBlockFusionAnalysis::Accept => { + block.fuse(operation); + self.num_ops += 1; + block.fuser.reduce_info() + } + ReduceBlockFusionAnalysis::Refuse => { + self.state = ReduceBroadcastedStatus::Closed; + return; + } + ReduceBlockFusionAnalysis::NewBlockRequired => { + let info = block.fuser.reduce_info(); + let mut block = ReduceBlockFuser::new(self.fuser_default.clone()); + block.fuse(operation); + self.num_ops += 1; + self.blocks.push(block); + info + } + }; + + match info { + ReduceFuserInfo::FusedReduce { + shape_input_id, + axis, + } => { + // Only support last axis for now. + if axis != shape_input_id.len() - 1 { + self.state = ReduceBroadcastedStatus::Abort; + } else { + self.state = ReduceBroadcastedStatus::Init { + shape_id: shape_input_id, + axis, + }; + } + } + ReduceFuserInfo::FusedElemwise { .. } => {} + } + } + + fn finish(&mut self) -> CubeOptimization { + let analyzer = FullFuserAnalyzer::new(&self.blocks); + let mut full = + ReduceBroadcastedFullFuser::new(self.max_bindings, self.bool_precision, analyzer); + let mut num_ops = 0; + let fallbacks = self + .blocks + .iter_mut() + .map(|block| block.finish(&mut num_ops, &mut full)) + .collect::>(); + + let broadcasted = Arc::new(full.finish()); + let info = Arc::new(ReduceBroadcastedOptimizationInfo { + fallbacks, + broadcasted, + }); + CubeOptimization::ReduceBroadcasted(ReduceBroadcastedOptimization { info, num_ops }) + } + + fn reset(&mut self) { + let block = ReduceBlockFuser::new(self.fuser_default.clone()); + self.blocks = vec![block]; + self.num_ops = 0; + self.state = ReduceBroadcastedStatus::Starting; + } + + fn status(&self) -> FuserStatus { + match self.state { + ReduceBroadcastedStatus::Closed | ReduceBroadcastedStatus::Abort => { + return FuserStatus::Closed; + } + _ => {} + }; + + let fuser = self.blocks.last().unwrap(); + fuser.fuser.status() + } + + fn properties(&self) -> FuserProperties { + let ready = match self.state { + ReduceBroadcastedStatus::Starting | ReduceBroadcastedStatus::Abort => false, + ReduceBroadcastedStatus::Closed => { + if self.blocks.len() == 1 { + !self.blocks[0].is_elemwise() + } else { + true + } + } + _ => true, + }; + let mut props = FuserProperties { score: 0, ready }; + for block in self.blocks.iter() { + let p = block.properties(); + props.score += p.score; + props.ready = p.ready && props.ready; + } + props + } + + fn len(&self) -> usize { + self.num_ops + } + + fn clone_dyn(&self) -> Box>> { + Box::new(self.clone()) + } +} + +#[cfg(test)] +mod tests { + use burn_ir::{ + BaseOperationIr, BinaryOpIr, CreationOpIr, ReduceDimOpIr, TensorId, TensorIr, TensorStatus, + }; + use burn_std::{DType, Shape}; + + use super::*; + + type Run = cubecl::TestRuntime; + + #[test] + fn reduce_broadcast_workflow_1() { + let device: ::Device = Default::default(); + let mut fuser = ReduceBroadcastedFuser::::new(device, FuseType::I32); + let (tensor1_out, tensor1) = tensor(0, &[1, 2], TensorStatus::ReadWrite); + let (tensor2_out, tensor2) = tensor(1, &[1, 0], TensorStatus::ReadWrite); + + fuser.fuse(&OperationIr::BaseFloat(BaseOperationIr::Ones( + CreationOpIr { out: tensor1_out }, + ))); + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::SumDim(ReduceDimOpIr { + input: tensor1, + out: tensor2_out, + axis: 1, + }), + )); + + let status = fuser.status(); + assert_eq!(2, fuser.len()); + assert_eq!(status, FuserStatus::Open); + assert_eq!( + fuser.properties(), + FuserProperties { + score: 2, + ready: true + } + ); + + // An existing tensor + let (_tensor3_out, tensor3) = tensor(2, &[1, 0], TensorStatus::ReadWrite); + // A new tensor + let (tensor4_out, tensor4) = tensor(3, &[1, 0], TensorStatus::ReadWrite); + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::Add(BinaryOpIr { + lhs: tensor2, + rhs: tensor3, + out: tensor4_out, + }), + )); + + let status = fuser.status(); + assert_eq!(3, fuser.len()); + assert_eq!(status, FuserStatus::Open); + assert_eq!( + fuser.properties(), + FuserProperties { + score: 3, + ready: true + } + ); + + // An existing tensor + let (_tensor5_out, tensor5) = tensor(4, &[1, 2], TensorStatus::ReadWrite); + // A new tensor + let (tensor6_out, tensor6) = tensor(5, &[1, 2], TensorStatus::ReadWrite); + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::Add(BinaryOpIr { + lhs: tensor4, + rhs: tensor5, + out: tensor6_out, + }), + )); + + let status = fuser.status(); + assert_eq!(4, fuser.len()); + assert_eq!(status, FuserStatus::Open); + assert_eq!( + fuser.properties(), + FuserProperties { + score: 4, + ready: true + } + ); + + let (tensor7_out, _tensor7) = tensor(6, &[1, 0], TensorStatus::ReadWrite); + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::SumDim(ReduceDimOpIr { + input: tensor6, + out: tensor7_out, + axis: 1, + }), + )); + assert_eq!(5, fuser.len()); + assert_eq!(status, FuserStatus::Open); + assert_eq!( + fuser.properties(), + FuserProperties { + score: 5, + ready: true + } + ); + + let _optimization = fuser.finish(); + } + + #[test] + fn reduce_broadcast_workflow_2() { + let device: ::Device = Default::default(); + let mut fuser = ReduceBroadcastedFuser::::new(device, FuseType::I32); + let (tensor1_out, tensor1) = tensor(0, &[1, 2], TensorStatus::ReadWrite); + // An existing tensor + let (_tensor2_out, mut tensor2) = tensor(2, &[1, 2], TensorStatus::ReadOnly); + let (tensor3_out, tensor3) = tensor(3, &[1, 2], TensorStatus::ReadWrite); + + // First reduce output + let (tensor4_out, tensor4) = tensor(1, &[1, 0], TensorStatus::ReadWrite); + + fuser.fuse(&OperationIr::BaseFloat(BaseOperationIr::Ones( + CreationOpIr { out: tensor1_out }, + ))); + + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::Add(BinaryOpIr { + lhs: tensor1, + rhs: tensor2.clone(), + out: tensor3_out, + }), + )); + + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::SumDim(ReduceDimOpIr { + input: tensor3, + out: tensor4_out, + axis: 1, + }), + )); + + let status = fuser.status(); + assert_eq!(3, fuser.len()); + assert_eq!(status, FuserStatus::Open); + assert_eq!( + fuser.properties(), + FuserProperties { + score: 3, + ready: true + } + ); + + // A new tensor + let (tensor5_out, _tensor5) = tensor(5, &[1, 2], TensorStatus::ReadWrite); + // Last time we use tensor2. + tensor2.status = TensorStatus::ReadWrite; + fuser.fuse(&OperationIr::NumericFloat( + DType::F32, + burn_ir::NumericOperationIr::Add(BinaryOpIr { + lhs: tensor4, + rhs: tensor2, + out: tensor5_out, + }), + )); + + let status = fuser.status(); + assert_eq!(4, fuser.len()); + assert_eq!(status, FuserStatus::Open); + assert_eq!( + fuser.properties(), + FuserProperties { + score: 4, + ready: true + } + ); + + let _optimization = fuser.finish(); + } + + fn tensor(id: u64, shape: &[usize], status: TensorStatus) -> (TensorIr, TensorIr) { + let tensor = TensorIr { + id: TensorId::new(id), + shape: Shape::from(shape), + status: TensorStatus::NotInit, + dtype: DType::F32, + }; + let mut tensor_init = tensor.clone(); + tensor_init.status = status; + + (tensor, tensor_init) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/block.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/block.rs new file mode 100644 index 0000000..c3153f6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/block.rs @@ -0,0 +1,214 @@ +use crate::optim::{ + CubeOptimization, + elemwise::ElemwiseOptimization, + reduce::{FusedReduce, ReduceFuser, ReduceFuserInfo}, + reduce_broadcasted::{ReduceBlockOptimInfo, fuser::full::ReduceBroadcastedFullFuser}, +}; +use burn_fusion::{FuserProperties, OperationFuser}; +use burn_ir::OperationIr; +use burn_std::Shape; +use cubecl::Runtime; +use std::sync::Arc; + +/// Responsible for fusing a single reduce block or elementwise block. +/// +/// When the block kind is reduce, it supports fuse-on-read and fuse-on-write fusion. +/// Broadcasting isn't supported; another block should handle it instead. +pub struct ReduceBlockFuser { + /// We use [ReduceFuser] for both elementwise and reduce blocks, keeping only the + /// fuse-on-read trace if the block is tagged as elementwise. + /// + /// # Notes + /// + /// A single elementwise block can only exist at the end of a full [ReduceBlockFuser], + /// otherwise the optimization will be included in the reduce fusion block. + pub fuser: ReduceFuser, + pub(crate) ops: Vec, + pub(crate) kind: ReduceBlockKind, +} + +/// The current state of the fusion process. +#[derive(Debug, Clone)] +pub enum ReduceBroadcastedStatus { + /// Fusion is starting; no reduction has been fused yet. + Starting, + /// Fusion is initialized with at least one reduce operation. + /// + /// # Notes + /// + /// Subsequent reduce operations must be compatible with the previous reduction to fuse. + Init { shape_id: Shape, axis: usize }, + /// No more operations can be fused. + Closed, + /// Invalid axis. + Abort, +} + +/// The [ReduceBlockFuser] capacity to accept an [OperationIr]. +#[derive(Clone, Copy, Debug)] +pub enum ReduceBlockFusionAnalysis { + /// The operation can be fused; call [ReduceBlockFuser::fuse()]. + Accept, + /// The operation cannot be fused; the optimization should close. + Refuse, + /// The operation can be fused, but requires a new block. + NewBlockRequired, +} + +impl ReduceBlockFuser { + /// Creates a new block. + pub fn new(fuser: ReduceFuser) -> Self { + Self { + fuser: fuser.clone(), + ops: Vec::new(), + kind: ReduceBlockKind::Elemwise, + } + } + + /// Returns true if this is an elementwise fuser. + pub fn is_elemwise(&self) -> bool { + matches!(self.kind, ReduceBlockKind::Elemwise) + } + + /// Analyzes if fusion is possible within this block. + pub fn analyze( + &self, + op: &OperationIr, + status: &ReduceBroadcastedStatus, + default_node: &ReduceFuser, + ) -> ReduceBlockFusionAnalysis { + let mut fuser_try = self.fuser.clone(); + let before = fuser_try.len(); + fuser_try.fuse(op); + let after = fuser_try.len(); + + if after > before { + return ReduceBlockFusionAnalysis::Accept; + } + + // Can't create a new block if the previous one was not a reduction. + if self.fuser.reduce.is_none() { + return ReduceBlockFusionAnalysis::Refuse; + } + + let mut fuser_try = default_node.clone(); + let before = fuser_try.len(); + fuser_try.fuse(op); + let after = fuser_try.len(); + + if after > before { + let info = fuser_try.reduce_info(); + + return match (info, status) { + ( + ReduceFuserInfo::FusedReduce { + shape_input_id, + axis, + }, + ReduceBroadcastedStatus::Init { + shape_id, + axis: axis_init, + }, + ) => { + if shape_id == &shape_input_id && axis_init == &axis { + ReduceBlockFusionAnalysis::NewBlockRequired + } else { + ReduceBlockFusionAnalysis::Refuse + } + } + ( + ReduceFuserInfo::FusedElemwise { shape_id }, + ReduceBroadcastedStatus::Init { + shape_id: shape_init, + .. + }, + ) => { + if &shape_id == shape_init { + ReduceBlockFusionAnalysis::NewBlockRequired + } else { + ReduceBlockFusionAnalysis::Refuse + } + } + _ => ReduceBlockFusionAnalysis::Refuse, + }; + } + + ReduceBlockFusionAnalysis::Refuse + } + + /// Fuses an operation within this block. + /// + /// # Warning + /// + /// Ensure [Self::analyze()] is called before this function to confirm the operation is accepted. + pub fn fuse(&mut self, op: &OperationIr) { + self.fuser.fuse(op); + self.ops.push(op.clone()); + + // Update the kind if a reduction is introduced to an elementwise block. + if let (Some(reduce), ReduceBlockKind::Elemwise) = (&self.fuser.reduce, &self.kind) { + self.kind = ReduceBlockKind::Reduce { + ops_index: self.ops.len() - 1, + reduce: Box::new(reduce.clone()), + }; + } + } + + /// Computes the fuser properties. + pub fn properties(&self) -> FuserProperties { + let mut properties = self.fuser.properties(); + if let ReduceBlockKind::Elemwise = &self.kind { + // Elementwise traces are always ready to run. + properties.ready = true; + } + properties + } + + pub fn finish( + &mut self, + num_ops: &mut usize, + full: &mut ReduceBroadcastedFullFuser, + ) -> ReduceBlockOptimInfo { + full.register(self); + + match &self.kind { + ReduceBlockKind::Elemwise => { + let len = self.fuser.fuser_read_fallback.len(); + let device = self.fuser.device.clone(); + *num_ops += len; + let trace = self.fuser.fuser_read_fallback.finish(); + let client = R::client(&device); + let elementwise = ElemwiseOptimization::new(trace, client, device, len); + ReduceBlockOptimInfo::Elemwise(Arc::new(elementwise)) + } + ReduceBlockKind::Reduce { .. } => { + *num_ops += self.fuser.len(); + let optim = self.fuser.finish(); + let info = match optim { + CubeOptimization::Reduce(optim) => optim.info, + _ => unreachable!("Expected Reduce optimization"), + }; + ReduceBlockOptimInfo::Reduce(info) + } + } + } +} + +#[derive(Clone, Debug)] +pub enum ReduceBlockKind { + Elemwise, + Reduce { + ops_index: usize, + reduce: Box, + }, +} + +impl Clone for ReduceBlockFuser { + fn clone(&self) -> Self { + Self { + fuser: self.fuser.clone(), + ops: self.ops.clone(), + kind: self.kind.clone(), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full.rs new file mode 100644 index 0000000..9f43c5f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full.rs @@ -0,0 +1,163 @@ +use crate::{ + engine::{ + codegen::ir::FuseType, + fuser::TraceOperationFuser, + settings::{FuseSettings, RefLayoutSetting, VectorizationSetting}, + }, + optim::{ + reduce::{FusedReduce, ReduceInstruction}, + reduce_broadcasted::{ + ReduceBroadcastedInfo, + fuser::{ + block::{ReduceBlockFuser, ReduceBlockKind}, + full_analyzer::FullFuserAnalyzer, + }, + launch::ReduceBroadcastedFuseBlock, + }, + }, +}; +use burn_fusion::OperationFuser; +use cubecl::Runtime; +use cubek::reduce::components::instructions::ReduceOperationConfig; + +/// Responsible for fusing a single trace for all operations involved in this optimization. +pub struct ReduceBroadcastedFullFuser { + pub(crate) fuser: TraceOperationFuser, + analyzer: FullFuserAnalyzer, + blocks: Vec, + settings_read: FuseSettings, + settings_write: FuseSettings, +} + +impl ReduceBroadcastedFullFuser { + /// Creates a new fuser with the given settings. + pub fn new(max_bindings: u32, bool_precision: FuseType, analyzer: FullFuserAnalyzer) -> Self { + let settings_read = FuseSettings { + output_shape_updates: true, + broadcast: true, + inplace: false, + ref_layout: RefLayoutSetting::OnlyContiguous, + vectorization: VectorizationSetting::Activated, + }; + let settings_write = FuseSettings { + output_shape_updates: false, + inplace: false, + broadcast: false, + ref_layout: RefLayoutSetting::OnlyContiguous, + // Deactivated for now, but would be cool to support vectorization of the output. + vectorization: VectorizationSetting::Deactivated, + }; + let fuser = TraceOperationFuser::new(max_bindings, bool_precision, settings_read); + + Self { + fuser, + blocks: Vec::new(), + settings_write, + settings_read, + analyzer, + } + } + + /// Finishes fusing all blocks. + pub fn finish(mut self) -> ReduceBroadcastedInfo { + let mut reduce_axis = 0; + let mut blocks = Vec::new(); + + for block in self.blocks.iter() { + match block { + ReduceBlockKind::Elemwise => {} + ReduceBlockKind::Reduce { reduce, .. } => { + let config = match reduce.inst { + ReduceInstruction::ArgMax => ReduceOperationConfig::ArgMax, + ReduceInstruction::ArgMin => ReduceOperationConfig::ArgMin, + ReduceInstruction::Prod => ReduceOperationConfig::Prod, + ReduceInstruction::Mean => ReduceOperationConfig::Mean, + ReduceInstruction::Sum => ReduceOperationConfig::Sum, + ReduceInstruction::Max => ReduceOperationConfig::Max, + ReduceInstruction::Min => ReduceOperationConfig::Min, + ReduceInstruction::MaxAbs => ReduceOperationConfig::MaxAbs, + }; + + let block = ReduceBroadcastedFuseBlock { + op: config, + input: reduce.input.clone(), + output: reduce.output.clone(), + }; + reduce_axis = reduce.axis; + blocks.push(block); + } + } + } + + let trace = self.fuser.finish(); + + ReduceBroadcastedInfo { + blocks, + trace, + reduce_axis, + } + } + + /// Registers a [ReduceBlockFuser] to build the trace. + pub fn register(&mut self, block: &ReduceBlockFuser) { + // Helper to close previous blocks if necessary + if !self.fuser.is_empty() { + let mut settings = self.settings_read; + settings.vectorization = VectorizationSetting::EqualThanPreviousBlock { block_pos: 0 }; + settings.ref_layout = RefLayoutSetting::SameAsBlock { block_pos: 0 }; + self.fuser.next_block([], settings, false); + + let analysis = self.analyzer.retrieve_next(); + + for (tensor, block_pos) in analysis.inputs { + self.fuser.block_local_input(&tensor, block_pos, false); + } + } + + match &block.kind { + ReduceBlockKind::Elemwise => { + for op in &block.ops { + self.fuser.fuse(op); + } + self.blocks.push(ReduceBlockKind::Elemwise); + } + ReduceBlockKind::Reduce { ops_index, reduce } => { + for op in &block.ops[0..*ops_index] { + self.fuser.fuse(op); + } + + let [input] = self + .fuser + .next_block([&reduce.op.input], self.settings_write, false); + + let output = self.fuser.output_unhandled(&reduce.op.out); + let analysis = self.analyzer.retrieve_next(); + + // Can be broadcasted so the generated buffer can be global. + for (tensor, block_pos) in analysis.inputs { + self.fuser.block_local_input(&tensor, block_pos, false); + } + + let fused_reduce = FusedReduce { + input, + output, + acc: reduce.acc, + axis: reduce.axis, + op: reduce.op.clone(), + use_planes: reduce.use_planes, + shared: reduce.shared, + inst: reduce.inst, + }; + + self.blocks.push(ReduceBlockKind::Reduce { + ops_index: *ops_index, + reduce: Box::new(fused_reduce), + }); + + for op in &block.ops[*ops_index + 1..block.ops.len()] { + self.fuser.fuse(op); + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full_analyzer.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full_analyzer.rs new file mode 100644 index 0000000..b845c2e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/full_analyzer.rs @@ -0,0 +1,159 @@ +use super::block::ReduceBlockKind; +use crate::optim::reduce_broadcasted::fuser::block::ReduceBlockFuser; +use burn_ir::{TensorId, TensorIr}; +use cubecl::Runtime; +use std::collections::BTreeMap; + +#[derive(Debug)] +pub struct FullFuserAnalyzer { + // We need to know the block id of which we can reuse the read local input. + analyses: Vec>, +} + +impl FullFuserAnalyzer { + pub fn new(blocks: &[ReduceBlockFuser]) -> Self { + let mut state = AnalysisState::default(); + + for block in blocks.iter() { + for (pos, op) in block.ops.iter().enumerate() { + let potential_from_previous_blocks = op.inputs(); + let potential_to_next_blocks = op.outputs(); + + match &block.kind { + ReduceBlockKind::Elemwise => { + state.register( + potential_from_previous_blocks, + potential_to_next_blocks, + BlockKind::Full, + ); + } + ReduceBlockKind::Reduce { ops_index, .. } => { + if pos < *ops_index { + state.register( + potential_from_previous_blocks, + potential_to_next_blocks, + BlockKind::Full, + ); + } else if pos > *ops_index { + state.register( + potential_from_previous_blocks, + potential_to_next_blocks, + BlockKind::Single, + ); + } else { + state.next_block(); + } + } + } + } + state.next_block(); + } + + // First one is never called. + state.analyses.remove(0); + + Self { + analyses: state.analyses, + } + } + + pub fn retrieve_next(&mut self) -> FullFuserAnalysis { + let inputs = self.analyses.remove(0); + FullFuserAnalysis { inputs } + } +} + +#[derive(Debug)] +pub struct FullFuserAnalysis { + /// The tensor received from a previous block. + pub inputs: Vec<(TensorIr, usize)>, +} + +#[derive(Default)] +struct AnalysisState { + /// That pool contains tensors that are available in the fuse-on-write part of a reduce, not + /// broadcasted. + available_from_previous_single: BTreeMap, + /// That pool contains tensors that are available in the fuse-on-read of a reduce and the + /// element-wise broadcasted part + available_from_previous_full: BTreeMap, + block_data: Vec<(TensorIr, usize)>, + analyses: Vec>, + current_full: Vec, + current_single: Vec, +} + +enum BlockKind { + Full, + Single, +} + +impl AnalysisState { + fn next_block(&mut self) { + let block_pos = self.analyses.len(); + let data = core::mem::take(&mut self.block_data); + self.analyses.push(data); + + // Makes the current tensor reads available for the next block. + for p in self.current_single.drain(..) { + // We need to keep the earliest block position. + self.available_from_previous_single + .entry(p.id) + .or_insert(block_pos); + } + for p in self.current_full.drain(..) { + // We need to keep the earliest block position. + self.available_from_previous_full + .entry(p.id) + .or_insert(block_pos); + } + } + + fn register<'a>( + &mut self, + potential_from_previous_blocks: impl Iterator, + potential_to_next_blocks: impl Iterator, + kind: BlockKind, + ) { + match kind { + BlockKind::Full => { + for potential in potential_from_previous_blocks { + // We can't since it's not in the same scope. + // + // TODO: Find a way to merge multiple reduce loops. + // + // if let Some(block_pos) = self.available_from_previous_full.get(&potential.id) { + // self.block_data.push((potential.clone(), *block_pos)); + // } + + // We can since it's a broadcast. + if let Some(block_pos) = self.available_from_previous_single.get(&potential.id) + { + self.block_data.push((potential.clone(), *block_pos)); + } + + // Can reuse the read. + self.current_full.push(potential.clone()); + } + + for p in potential_to_next_blocks { + self.current_full.push(p.clone()); + } + } + BlockKind::Single => { + for potential in potential_from_previous_blocks { + if let Some(block_pos) = self.available_from_previous_single.get(&potential.id) + { + self.block_data.push((potential.clone(), *block_pos)); + } + // Can reuse the read. + self.current_single.push(potential.clone()); + } + + for p in potential_to_next_blocks { + self.current_single.push(p.clone()); + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/mod.rs new file mode 100644 index 0000000..22bb521 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/fuser/mod.rs @@ -0,0 +1,6 @@ +mod base; +mod block; +mod full; +mod full_analyzer; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/launch.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/launch.rs new file mode 100644 index 0000000..320f429 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/launch.rs @@ -0,0 +1,139 @@ +use crate::{ + engine::{ + codegen::ir::{FuseArg, FuseBlockConfig, GlobalArgsLaunch, RefLayout}, + launch::runner::{TraceRunner, Vectorization}, + }, + optim::reduce_broadcasted::unit::{ + ElemwiseFuseBlockLaunch, ReduceFuseBlockLaunch, reduce_kernel_broadcasted, + }, +}; +use cubecl::{ + Runtime, + ir::{ElemType, FloatKind, StorageType}, + prelude::*, + server::LaunchError, +}; +use cubek::reduce::{ + LineMode, ReduceDtypes, + components::instructions::ReduceOperationConfig, + launch::RoutineStrategy, + routines::{ + BlueprintStrategy, GlobalReduceBlueprint, ReduceLineSettings, ReduceProblem, Routine, + unit::{UnitRoutine, UnitStrategy}, + }, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ReduceBroadcastedFuseBlock { + pub(crate) op: ReduceOperationConfig, + pub(crate) input: FuseArg, + pub(crate) output: FuseArg, +} + +#[derive(new)] +pub struct FusedReduceBroadcastedLaunch<'a> { + blocks: &'a Vec, + reduce_axis: usize, + // TODO: Support multiple strategies. + _strategy: RoutineStrategy, +} + +impl Vectorization for FusedReduceBroadcastedLaunch<'_> {} + +impl TraceRunner for FusedReduceBroadcastedLaunch<'_> { + type Error = LaunchError; + + fn run<'a>( + &'a self, + client: &'a ComputeClient, + inputs: GlobalArgsLaunch<'a, R>, + outputs: GlobalArgsLaunch<'a, R>, + configs: &'a [FuseBlockConfig], + ) -> Result<(), Self::Error> { + let routine = UnitRoutine; + let first_config = &configs[0]; + + let shape = match &first_config.ref_layout { + RefLayout::Concrete(FuseArg::Output(..)) => { + outputs.shape_ref(&first_config.ref_layout, first_config.rank) + } + _ => inputs.shape_ref(&first_config.ref_layout, first_config.rank), + }; + + let vector_size = shape[self.reduce_axis]; + let vector_count = shape.iter().product::() / vector_size; + let address_type = inputs + .required_address_type() + .max(outputs.required_address_type()); + + let (blueprint, settings) = routine + .prepare::( + client, + ReduceProblem { + vector_size, + vector_count, + axis: self.reduce_axis, + dtypes: ReduceDtypes { + input: StorageType::Scalar(ElemType::Float(FloatKind::F32)), + output: StorageType::Scalar(ElemType::Float(FloatKind::F32)), + accumulation: StorageType::Scalar(ElemType::Float(FloatKind::F32)), + }, + address_type, + }, + ReduceLineSettings { + line_mode: LineMode::Parallel, + line_size_input: first_config.width, + line_size_output: 1, + }, + BlueprintStrategy::Inferred(UnitStrategy), + ) + .unwrap(); + + assert_eq!(blueprint.line_mode, LineMode::Parallel); + + let mut blocks = SequenceArg::new(); + let mut index = 0; + + for block in self.blocks { + let arg = ReduceFuseBlockLaunch::new( + block.op, + configs[index].clone(), + configs[index + 1].clone(), + block.input.clone(), + block.output.clone(), + match blueprint.global { + GlobalReduceBlueprint::Unit(bpt) => bpt, + _ => panic!(), + }, + ); + index += 2; + blocks.push(arg); + } + + let block_end = match configs.len() > index { + true => OptionArgs::Some(ElemwiseFuseBlockLaunch::new( + configs.last().cloned().unwrap(), + )), + false => OptionArgs::None, + }; + + // TODO: Ensure parallel is selected. + + unsafe { + reduce_kernel_broadcasted::launch_unchecked::( + client, + settings.cube_count, + settings.cube_dim, + settings.address_type, + inputs, + outputs, + ScalarArg::new(self.reduce_axis), + blocks, + block_end, + )?; + } + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/mod.rs new file mode 100644 index 0000000..b29f224 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/mod.rs @@ -0,0 +1,9 @@ +mod fuser; +mod optimization; + +pub(crate) mod launch; +pub(crate) mod tune; +pub(crate) mod unit; + +pub use fuser::*; +pub use optimization::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/optimization.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/optimization.rs new file mode 100644 index 0000000..a6c28c4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/optimization.rs @@ -0,0 +1,222 @@ +#[cfg(feature = "autotune")] +use crate::optim::reduce::tune::fused_reduce_autotune; +use crate::{ + CubeFusionHandle, FallbackOperation, + engine::{ + launch::FuseTraceLauncher, + trace::{FuseTrace, TraceError, TuneOutput}, + }, + optim::{ + elemwise::{ElemwiseOptimization, ElemwiseOptimizationState}, + reduce::{ReduceOptimizationInfo, ReduceOptimizationState, ReduceOptimizationTuneArg}, + reduce_broadcasted::{ + launch::{FusedReduceBroadcastedLaunch, ReduceBroadcastedFuseBlock}, + tune::fused_broadcasted_reduce_autotune, + }, + }, +}; +use burn_fusion::stream::Context; +use cubecl::{Runtime, prelude::*}; +use cubek::reduce::launch::RoutineStrategy; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +pub struct ReduceBroadcastedOptimization { + pub(crate) info: Arc>, + pub(crate) num_ops: usize, +} + +pub(crate) struct ReduceBroadcastedOptimizationInfo { + pub(crate) fallbacks: Vec>, + pub(crate) broadcasted: Arc, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct ReduceBroadcastedInfo { + pub(crate) blocks: Vec, + pub(crate) trace: FuseTrace, + pub(crate) reduce_axis: usize, +} + +pub(crate) enum ReduceBlockOptimInfo { + Reduce(Arc>), + Elemwise(Arc>), +} + +impl ReduceBlockOptimInfo { + pub fn from_state(device: &R::Device, state: ReduceBlockState) -> Self { + match state { + ReduceBlockState::Reduce(state) => { + Self::Reduce(Arc::new(ReduceOptimizationInfo::from_state(device, state))) + } + ReduceBlockState::Elemwise(state) => { + Self::Elemwise(Arc::new(ElemwiseOptimization::from_state(device, state))) + } + } + } + pub fn to_state(&self) -> ReduceBlockState { + match self { + Self::Reduce(info) => ReduceBlockState::Reduce(info.to_state()), + Self::Elemwise(info) => ReduceBlockState::Elemwise(info.to_state()), + } + } +} + +pub(crate) struct ReduceBroadcastedOptimizationTuneArg { + pub(crate) fallbacks: Vec>, + pub(crate) broadcasted: Arc, + pub(crate) client: ComputeClient, + pub(crate) device: R::Device, +} + +pub(crate) enum ReduceBlockOptimArg { + Reduce(ReduceOptimizationTuneArg), + Elemwise(Arc>), +} + +impl ReduceBlockOptimArg { + pub fn execute_fallback( + &self, + context: &mut Context<'_, CubeFusionHandle>, + ) -> Option> { + match self { + ReduceBlockOptimArg::Reduce(reduce) => { + #[cfg(feature = "autotune")] + { + fused_reduce_autotune::(reduce.clone(), context); + None + } + #[cfg(not(feature = "autotune"))] + Some(reduce.execute_fallback::(context)) + } + ReduceBlockOptimArg::Elemwise(elem) => { + elem.execute::(context); + None + } + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ReduceBroadcastedOptimizationState { + fallbacks: Vec, + broadcasted: ReduceBroadcastedInfo, + num_ops: usize, +} + +#[derive(Serialize, Deserialize, Debug)] +#[allow(clippy::large_enum_variant)] // Only for serialization. +pub enum ReduceBlockState { + Reduce(ReduceOptimizationState), + Elemwise(ElemwiseOptimizationState), +} + +impl ReduceBroadcastedOptimizationTuneArg { + pub fn execute_fused( + &self, + context: &mut Context<'_, CubeFusionHandle>, + strategy: RoutineStrategy, + ) -> Result, TraceError> { + let launch = FusedReduceBroadcastedLaunch::new( + &self.broadcasted.blocks, + self.broadcasted.reduce_axis, + strategy, + ); + let launcher = FuseTraceLauncher::new(&self.broadcasted.trace, &launch); + + launcher + .launch::(&self.client, &self.device, context) + .map_err(|err| TraceError::RunnerError(format!("{:?}", err))) + } + + pub fn execute_fallback( + &self, + context: &mut Context<'_, CubeFusionHandle>, + ) { + for fallback in self.fallbacks.iter() { + fallback.execute_fallback::(context); + } + } +} + +#[allow(clippy::too_many_arguments)] +impl ReduceBroadcastedOptimization { + /// Execute the optimization. + pub fn execute( + &mut self, + context: &mut Context<'_, CubeFusionHandle>, + fallback: impl Fn(usize) -> Box>, + ) { + let mut current_index = 0; + let mut client = None; + let mut device = None; + + let fallbacks = self + .info + .fallbacks + .iter() + .map(|info| { + match info { + ReduceBlockOptimInfo::Reduce(info) => { + // The index of the fallback reduce is the number of ops fused as read. + let fallback = fallback(current_index + info.len_read); + client = Some(info.client.clone()); + device = Some(info.device.clone()); + let arg = ReduceOptimizationTuneArg { + info: info.clone(), + fallback: Arc::new(fallback), + }; + current_index += info.len; + ReduceBlockOptimArg::Reduce(arg) + } + ReduceBlockOptimInfo::Elemwise(op) => ReduceBlockOptimArg::Elemwise(op.clone()), + } + }) + .collect(); + + let arg = ReduceBroadcastedOptimizationTuneArg { + fallbacks, + client: client.unwrap(), + device: device.unwrap(), + broadcasted: self.info.broadcasted.clone(), + }; + + #[cfg(feature = "autotune")] + fused_broadcasted_reduce_autotune::(arg, context); + + #[cfg(not(feature = "autotune"))] + arg.execute_fallback::(context); + } + + pub fn to_state(&self) -> ReduceBroadcastedOptimizationState { + ReduceBroadcastedOptimizationState { + fallbacks: self + .info + .fallbacks + .iter() + .map(|info| info.to_state()) + .collect(), + broadcasted: self.info.broadcasted.as_ref().clone(), + num_ops: self.num_ops, + } + } + + pub fn from_state(device: &R::Device, state: ReduceBroadcastedOptimizationState) -> Self { + Self { + info: Arc::new(ReduceBroadcastedOptimizationInfo { + fallbacks: state + .fallbacks + .into_iter() + .map(|state| ReduceBlockOptimInfo::from_state(device, state)) + .collect(), + broadcasted: Arc::new(state.broadcasted), + }), + num_ops: state.num_ops, + } + } + + /// Returns the number of output buffers added by fusion. + pub fn num_ops_fused(&self) -> usize { + self.num_ops + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/tune.rs new file mode 100644 index 0000000..509f28f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/tune.rs @@ -0,0 +1,180 @@ +use super::optimization::ReduceBroadcastedOptimizationTuneArg; +use crate::{ + CubeFusionHandle, + engine::trace::TuneOutput, + optim::{reduce::ReduceOptimizationInfo, reduce_broadcasted::ReduceBlockOptimArg}, + tune::{TuneContext, TuneInput}, +}; +use burn_fusion::stream::Context; +use cubecl::{ + AutotuneKey, CubeElement, CubeTuneId, Runtime, + tune::{LocalTuner, Tunable, TunableSet, TuneGroup, local_tuner}, +}; +use cubek::reduce::{ + launch::{RoutineStrategy, tune_key::ReduceAutotuneKey}, + routines::{BlueprintStrategy, unit::UnitStrategy}, +}; +use serde::{Deserialize, Serialize}; + +/// Autotune key for fused broadcasted reduction operations. +/// +/// Captures the characteristics of the fusion (reads, writes, ops) to ensure +/// the best kernel is selected for specific fused graph shapes. +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, AutotuneKey)] +pub struct FusedBroadcastedReduceAutotuneKey { + reduce_key: ReduceAutotuneKey, + #[autotune(anchor)] + fuse_num_reads: usize, + #[autotune(anchor)] + fuse_num_writes: usize, + #[autotune(anchor)] + fuse_num_ops: usize, + fuse_num_blocks: usize, +} + +/// Executes the autotuning process for fused reduction operations. +/// +/// This function initializes a local tuner and attempts multiple strategies +/// (fallback vs. unit strategy) to find the most efficient execution path. +pub fn fused_broadcasted_reduce_autotune( + arg: ReduceBroadcastedOptimizationTuneArg, + context: &mut Context>, +) { + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + const PRIORITY_MAX: i8 = 2; + let mut set = TunableSet::new(create_key::, input_gen::); + + let group = TuneGroup::::new( + "fused_reduce_broadcasted", + |_key| PRIORITY_MAX, + ); + + // Standard fallback implementation - guaranteed to work. + set = set.with(Tunable::new( + "fused_reduce_broadcasted_fallback", + tune_fallback::, + )); + + // Specialized unit strategy for fused reductions. + set = set.with( + Tunable::new("fused_reduce_broadcasted_unit", move |input| { + tune_reduce::( + input, + &RoutineStrategy::Unit(BlueprintStrategy::Inferred(UnitStrategy)), + ) + }) + .group(&group, |_| PRIORITY_MAX), + ); + + set + }); + + TUNER.execute( + &CubeTuneId::new(&arg.client, &arg.device), + &arg.client.clone(), + tunables, + TuneInput::new(context, arg), + ); +} + +/// Generates the autotune key based on the current optimization context and trace blocks. +pub(crate) fn create_key( + input: &TuneInput>, +) -> FusedBroadcastedReduceAutotuneKey { + let opt = input.optimization(); + let context = match input.context() { + TuneContext::Original(context) => context, + TuneContext::Fork(_) => unreachable!("Forked context not supported for key generation"), + }; + + // The fusion must start with a reduction block to be valid here. + let info = match &opt.fallbacks[0] { + ReduceBlockOptimArg::Reduce(reduce) => &reduce.info, + ReduceBlockOptimArg::Elemwise(_) => { + unreachable!("Fusion must start with a reduction block") + } + }; + + let key = generate_reduce_autotune_key(info, context); + + // Sum up complexity metrics across all blocks in the fused trace. + let (mut num_reads, mut num_writes, mut num_ops) = (0, 0, 0); + + for block in opt.broadcasted.trace.blocks.iter() { + num_reads += block.reads.len(); + num_writes += block.writes.len(); + num_ops += block.ops.len(); + } + + FusedBroadcastedReduceAutotuneKey::new( + key, + num_reads, + num_writes, + num_ops, + info.trace.blocks.len(), + ) +} + +/// Helper to generate the base reduction key (shapes, types, axes). +fn generate_reduce_autotune_key( + info: &ReduceOptimizationInfo, + context: &Context>, +) -> ReduceAutotuneKey { + let input = context.tensors.get(&info.reduce.op.input.id).unwrap(); + let out = context.tensors.get(&info.reduce.op.out.id).unwrap(); + let acc = info.reduce.acc.into_elem(); + + ReduceAutotuneKey::generate( + input.dtype.into(), + out.dtype.into(), + acc, + &input.shape, + info.reduce.axis == input.shape.rank() - 1, // Is it the last dimension? + info.reduce.axis, + ) +} + +/// Simple input generator that clones the input for the tuner. +fn input_gen( + _key: &FusedBroadcastedReduceAutotuneKey, + input: &TuneInput>, +) -> TuneInput> { + input.clone() +} + +/// Executes a fused reduction using a specific routine strategy. +fn tune_reduce( + input: TuneInput>, + strategy: &RoutineStrategy, +) -> Result, String> { + let optimization = input.optimization(); + + match input.context() { + TuneContext::Original(context) => { + optimization.execute_fused::(context, strategy.clone()) + } + TuneContext::Fork(mut context_owned) => { + optimization.execute_fused::(&mut context_owned.as_context(), strategy.clone()) + } + } + .map_err(|e| format!("{e:?}")) +} + +/// Executes the fallback implementation for the reduction. +fn tune_fallback( + input: TuneInput>, +) -> Result, String> { + let optimization = input.optimization(); + + match input.context() { + TuneContext::Original(context) => optimization.execute_fallback::(context), + TuneContext::Fork(mut context_owned) => { + optimization.execute_fallback::(&mut context_owned.as_context()) + } + }; + + // Fallback is often used as a baseline, returning unchecked output. + Ok(TuneOutput::UnChecked(std::marker::PhantomData)) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/unit.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/unit.rs new file mode 100644 index 0000000..5e9e12d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/optim/reduce_broadcasted/unit.rs @@ -0,0 +1,202 @@ +use crate::{ + engine::codegen::{ + ir::{FuseArg, FuseBlockConfig, FuseType, GlobalArgs, multi_block_variables_init}, + kernel::{fuse_on_write, init_locals}, + }, + optim::reduce::args::{FusedReduceArgs, FusedReduceInput, FusedReduceOutput}, +}; +use cubecl::{Runtime, prelude::*, std::tensor::r#virtual::VirtualTensor}; +use cubek::reduce::{ + LineMode, ReduceInstruction, ReducePrecision, + components::{ + global::unit::GlobalFullUnitReduce, + instructions::{ReduceOperation, ReduceOperationConfig}, + }, + init_tensors, + routines::UnitReduceBlueprint, +}; + +/// A configuration block for a reduction operation within a fused kernel. +/// +/// This struct holds all the compile-time information needed to perform a +/// reduction, including the operation type (Sum, Max, etc.) and the layout +/// configuration for both input and output. +#[derive(CubeType, CubeLaunch, Clone)] +pub struct ReduceFuseBlock { + #[cube(comptime)] + op: ReduceOperationConfig, + #[cube(comptime)] + config_input: FuseBlockConfig, + #[cube(comptime)] + config_output: FuseBlockConfig, + #[cube(comptime)] + input: FuseArg, + #[cube(comptime)] + output: FuseArg, + #[cube(comptime)] + blueprint: UnitReduceBlueprint, +} + +/// A configuration block for an elementwise operation that follows a reduction. +#[derive(CubeType, CubeLaunch, Clone)] +pub struct ElemwiseFuseBlock { + #[cube(comptime)] + config: FuseBlockConfig, +} + +/// The entry point for a broadcasted reduction kernel. +/// +/// This kernel initializes local variables for multiple reduction blocks and then +/// executes the reduction sequence. +/// +/// # Arguments +/// +/// * `inputs` - Global arguments containing input tensor handles. +/// * `outputs` - Global arguments containing output tensor handles. +/// * `reduce_axis` - The dimension along which the reduction is performed. +/// * `blocks` - A sequence of reduction operations to execute. +/// * `block_end` - An optional elementwise block to execute after reductions are complete. +#[cube(launch_unchecked, address_type = "dynamic")] +pub fn reduce_kernel_broadcasted( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + reduce_axis: usize, + blocks: Sequence, + block_end: Option, +) { + #[unroll] + for i in 0..blocks.len() { + let block = blocks.index(i); + multi_block_variables_init(&block.config_input, &mut outputs.variables); + multi_block_variables_init(&block.config_output, &mut outputs.variables); + } + + reduce_many(inputs, outputs, reduce_axis, blocks, block_end); +} + +const REDUCE_INPUT: u8 = 0; +const REDUCE_ACC: u8 = 1; +const REDUCE_OUT: u8 = 2; + +type In = NumericExpand; +type Acc = NumericExpand; +type Out = NumericExpand; + +/// Configures the precision polyfills for the reduction based on the block's `FuseType`. +#[cube] +fn set_polyfill_block(block: &ReduceFuseBlock) { + let input_precision = comptime!(block.input.precision()); + let output_precision = comptime!(block.output.precision()); + let acc_precision = comptime!(match input_precision { + FuseType::F64 => FuseType::F64, + FuseType::F32 => FuseType::F32, + FuseType::Flex32 => FuseType::F32, + FuseType::F16 => FuseType::F32, + FuseType::BF16 => FuseType::F32, + FuseType::I64 => FuseType::I64, + FuseType::I32 => FuseType::I32, + FuseType::I16 => FuseType::I32, + FuseType::I8 => FuseType::I32, + FuseType::U64 => FuseType::U64, + FuseType::U32 => FuseType::U32, + FuseType::U16 => FuseType::U32, + FuseType::U8 => FuseType::U32, + FuseType::Bool => FuseType::I32, + }); + + set_polyfill::(comptime!(input_precision.into_type())); + set_polyfill::(comptime!(output_precision.into_type())); + set_polyfill::(comptime!(acc_precision.into_type())); +} + +/// Internal logic for executing a sequence of reduction blocks followed by an optional +/// trailing elementwise block. +#[cube] +#[allow(clippy::clone_on_copy)] +fn reduce_many( + inputs: &GlobalArgs, + outputs: &mut GlobalArgs, + reduce_axis: usize, + blocks: Sequence, + block_end: Option, +) { + let mut axis_size = 0; + + #[unroll] + for i in 0..blocks.len() { + let block = blocks.index(i); + let input = FusedReduceInput { + global: inputs.clone(), + config: comptime!(block.config_input.clone()), + arg: comptime!(block.input.clone()), + }; + let global = outputs.clone(); + let config = comptime!(block.config_output.clone()); + let arg = comptime!(block.output.clone()); + let mut output = FusedReduceOutput { + global, + config, + arg, + }; + + set_polyfill_block(block); + let (input, mut output) = init_tensors::(&input, &mut output); + + axis_size = reduce_step::<(In, Acc), Out, ReduceOperation>( + &input, + &mut output, + reduce_axis, + block.op, + comptime!(block.blueprint.clone()), + ); + } + + if let Some(block) = block_end { + let global_index = ABSOLUTE_POS; + let width = comptime!(block.config.width as u32); + let num_iter = axis_size / usize::cast_from(width); + + for i in 0..num_iter { + // Register block local inputs. + let values = Registry::>::new(); + let args = comptime![Vec::::new()]; + let index = global_index * num_iter + i; + let mut locals = init_locals(inputs, outputs, &block.config); + + fuse_on_write::( + inputs, + outputs, + &mut locals, + index, + values, + args, + &block.config.clone(), + ) + } + } +} + +#[cube] +/// Executes a single reduction step using a specified instruction and blueprint. +/// +/// Returns the size of the axis that was reduced. +fn reduce_step>( + input: &VirtualTensor, + output: &mut VirtualTensor, + reduce_axis: usize, + #[comptime] config: I::Config, + #[comptime] blueprint: UnitReduceBlueprint, +) -> usize { + let inst = I::from_config(config); + let axis_size = input.shape(reduce_axis); + + GlobalFullUnitReduce::execute::( + input, + output, + reduce_axis, + &inst, + LineMode::Parallel, + comptime!(blueprint), + ); + axis_size +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/tune.rs new file mode 100644 index 0000000..03df4b5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl-fusion/src/tune.rs @@ -0,0 +1,107 @@ +use crate::CubeFusionHandle; +use burn_fusion::stream::{Context, ContextOwned}; +use cubecl::Runtime; +use std::sync::Arc; + +/// Fusion context used when tuning kernels. +/// +/// Either the original context is returned or a fork of the original. +/// The fork is only given when performing autotuning, and not when actually performing the +/// operation. +pub enum TuneContext<'a, R: Runtime> { + Original(&'a mut Context<'a, CubeFusionHandle>), + Fork(Box>>), +} + +/// Fusion input wrapper containing the context and the optimization. +/// +/// # Safety +/// +/// This should only be used with the [tuner](cubecl::tune::LocalTuner), since safety assumptions +/// are made based on its behavior. +pub struct TuneInput { + context: UnsafeTuneContext, + optimization: Arc, +} + +/// Unsafe wrapper around the context. +/// +/// # Safety +/// +/// The wrapper removes the context lifetime. +/// +/// For it to be correct, the context must not be used after the invocation of the +/// [cubecl::tune::LocalTuner::execute] function. This is the case, since autotune functions are +/// tuned using a cloned version of the input; therefore, a fork of the context will be used to find +/// the best kernel to use, which can be async. +enum UnsafeTuneContext { + Original(*mut Context<'static, CubeFusionHandle>), + Fork(Box>>), +} + +unsafe impl Send for UnsafeTuneContext {} +unsafe impl Send for TuneInput {} + +impl TuneInput { + /// Create a new autotune input from the [context](Context) and an optimization. + pub fn new(context: &mut Context>, optimization: O) -> Self { + let context = UnsafeTuneContext::new(context); + + Self { + context, + optimization: Arc::new(optimization), + } + } + + /// Retrieve the [autotune context](TuneContext) for the current input. + pub fn context(&self) -> TuneContext<'static, R> { + self.context.get() + } + + /// Retrieve the optimization for the current input. + pub fn optimization(&self) -> &O { + &self.optimization + } +} + +impl UnsafeTuneContext { + fn new(context: &mut Context<'_, CubeFusionHandle>) -> Self { + let ptr = core::ptr::from_mut(context); + + // It is necessary for the lifetime. + #[allow(clippy::unnecessary_cast)] + Self::Original(ptr as *mut Context<'static, _>) + } + + fn get(&self) -> TuneContext<'static, R> { + match self { + UnsafeTuneContext::Original(ptr) => { + TuneContext::Original(unsafe { ptr.as_mut().unwrap() }) + } + UnsafeTuneContext::Fork(context) => TuneContext::Fork(Box::new(context.fork())), + } + } +} + +impl Clone for TuneInput { + fn clone(&self) -> Self { + Self { + context: self.context.clone(), + optimization: self.optimization.clone(), + } + } +} + +impl Clone for UnsafeTuneContext { + fn clone(&self) -> Self { + let context = match self { + UnsafeTuneContext::Original(ptr) => { + let context: &mut Context<'static, CubeFusionHandle> = + unsafe { ptr.as_mut().unwrap() }; + context.fork() + } + UnsafeTuneContext::Fork(context) => context.fork(), + }; + UnsafeTuneContext::Fork(Box::new(context)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/Cargo.toml new file mode 100644 index 0000000..40ee069 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/Cargo.toml @@ -0,0 +1,88 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Generic backend that can be compiled just-in-time to any shader language target" +documentation = "https://docs.rs/burn-cubecl" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "gpu"] +license.workspace = true +name = "burn-cubecl" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-cubecl" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [ + "autotune", + "std", + "fusion", + "cubecl/default", + "burn-fusion?/default", + "burn-cubecl-fusion?/default", +] +std = [ + "cubecl/std", + "burn-backend/std", + "burn-fusion?/std", + "burn-cubecl-fusion?/std", +] +doc = ["default"] +memory-checks = ["burn-fusion?/memory-checks"] +tracing = [ + "dep:tracing", + "cubecl/tracing", + "burn-std/tracing", + "burn-backend/tracing", + "burn-fusion?/tracing", + "burn-cubecl-fusion?/tracing", +] + +autotune = ["burn-cubecl-fusion?/autotune"] +autotune-checks = [ + "autotune", + "cubecl/autotune-checks", + "burn-cubecl-fusion?/autotune-checks", +] + +fusion = ["burn-fusion", "burn-cubecl-fusion"] +fusion-experimental = ["fusion"] + +template = [] + +[dependencies] +burn-cubecl-fusion = { path = "../burn-cubecl-fusion", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2", default-features = false } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false, features = [ + "cubecl", +] } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false, features = [ + "cubecl", +] } +cubecl = { workspace = true, features = ["stdlib"] } +cubek = { workspace = true, features = [ + "attention", + "matmul", + "convolution", + "reduce", + "random", + "quantization", +] } +tracing = { workspace = true, features = ["attributes"], optional = true } + +derive-new = { workspace = true } +log = { workspace = true } + +# Async +futures-lite = { workspace = true, features = ["std"] } + +# Template +serde = { workspace = true } +text_placeholder = { workspace = true, features = ["struct_context"] } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/README.md b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/README.md new file mode 100644 index 0000000..d1811f3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/README.md @@ -0,0 +1,3 @@ +# Burn CubeCL Backend + +Generic backend that can be compiled just-in-time (JIT) to any shader language target. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/backend.rs new file mode 100644 index 0000000..0c6756c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/backend.rs @@ -0,0 +1,196 @@ +use crate::{CubeRuntime, FloatElement, IntElement, element::BoolElement, tensor::CubeTensor}; +use burn_backend::{Backend, DTypeUsage, DTypeUsageSet, DeviceOps, ExecutionError, TensorData}; +use burn_std::DType; +use cubecl::{ + features::{MmaConfig, TypeUsage}, + server::ComputeServer, +}; +use std::marker::PhantomData; + +#[cfg(not(feature = "fusion"))] +use burn_backend::tensor::{BoolTensor, FloatTensor, IntTensor, QuantizedTensor}; +#[cfg(not(feature = "fusion"))] +use burn_ir::{BackendIr, TensorHandle}; + +/// Generic tensor backend that can be compiled just-in-time to any shader runtime +#[derive(new)] +pub struct CubeBackend { + _runtime: PhantomData, + _float_elem: PhantomData, + _int_elem: PhantomData, + _bool_elem: PhantomData, +} + +impl Backend for CubeBackend +where + R: CubeRuntime, + R::Server: ComputeServer, + R::Device: DeviceOps, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + type Device = R::Device; + + type FloatElem = F; + type IntElem = I; + type BoolElem = BT; + + type FloatTensorPrimitive = CubeTensor; + type IntTensorPrimitive = CubeTensor; + type BoolTensorPrimitive = CubeTensor; + type QuantizedTensorPrimitive = CubeTensor; + + fn name(device: &Self::Device) -> String { + let client = R::client(device); + format!("cubecl<{}>", R::name(&client)) + } + + fn seed(_device: &Self::Device, seed: u64) { + cubek::random::seed(seed); + } + + fn ad_enabled(_device: &Self::Device) -> bool { + false + } + + fn sync(device: &Self::Device) -> Result<(), ExecutionError> { + let client = R::client(device); + futures_lite::future::block_on(client.sync()).map_err(|err| ExecutionError::WithContext { + reason: format!("{err}"), + }) + } + + fn memory_persistent_allocations Output>( + device: &Self::Device, + input: Input, + func: Func, + ) -> Output { + let client = R::client(device); + client.memory_persistent_allocation(input, func) + } + + fn memory_cleanup(device: &Self::Device) { + let client = R::client(device); + client.memory_cleanup(); + } + + fn staging<'a, Iter>(data: Iter, device: &Self::Device) + where + Iter: Iterator, + { + let client = R::client(device); + client.staging(data.map(|td| &mut td.bytes), false); + } + + fn supports_dtype(device: &Self::Device, dtype: DType) -> bool { + let client = R::client(device); + + let type_usage = client.properties().type_usage(dtype.into()); + // Same as `TypeUsage::all_scalar()`, but we make the usage explicit here + type_usage.is_superset( + TypeUsage::Buffer + | TypeUsage::Conversion + | TypeUsage::Arithmetic + | TypeUsage::DotProduct, + ) + } + + fn dtype_usage(device: &Self::Device, dtype: DType) -> DTypeUsageSet { + let client = R::client(device); + + let props = client.properties(); + let storage = dtype.into(); + let usage = props.type_usage(storage); + + let mut out = DTypeUsageSet::new(); + + if usage.is_superset(TypeUsage::Buffer | TypeUsage::Conversion) { + out |= DTypeUsage::Storage; + } + + if usage.contains(TypeUsage::Arithmetic) { + out |= DTypeUsage::Arithmetic; + } + + let has_mma = |cfg: &MmaConfig| { + cfg.a_type == storage || cfg.b_type == storage || cfg.cd_type == storage + }; + if props.features.cmma.iter().any(has_mma) || props.features.mma.iter().any(has_mma) { + out |= DTypeUsage::Accelerated; + } + + out + } +} + +impl core::fmt::Debug + for CubeBackend +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("CubeCLBackend") + } +} + +impl Clone + for CubeBackend +{ + fn clone(&self) -> Self { + Self::new() + } +} + +impl Default + for CubeBackend +{ + fn default() -> Self { + Self::new() + } +} + +impl CubeRuntime for R +where + R::Device: DeviceOps, +{ + type CubeDevice = R::Device; + type CubeServer = R::Server; +} + +#[cfg(not(feature = "fusion"))] +impl BackendIr + for CubeBackend +{ + type Handle = CubeTensor; + + fn float_tensor(handle: TensorHandle) -> FloatTensor { + handle.handle + } + + fn int_tensor(handle: TensorHandle) -> IntTensor { + handle.handle + } + + fn bool_tensor(handle: TensorHandle) -> BoolTensor { + handle.handle + } + + fn quantized_tensor(handle: TensorHandle) -> QuantizedTensor { + handle.handle + } + + fn float_tensor_handle(tensor: FloatTensor) -> Self::Handle { + tensor + } + + fn int_tensor_handle(tensor: IntTensor) -> Self::Handle { + tensor + } + + fn bool_tensor_handle(tensor: BoolTensor) -> Self::Handle { + tensor + } + + fn quantized_tensor_handle(tensor: QuantizedTensor) -> Self::Handle { + tensor + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/element.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/element.rs new file mode 100644 index 0000000..c9a02b7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/element.rs @@ -0,0 +1,94 @@ +use burn_backend::{Element, bf16, f16}; +use cubecl::{ + CubeElement as CubeElem, flex32, + prelude::{Float, Int, Numeric}, +}; +use cubek::{ + matmul::definition::{MatmulPrecision, MatrixPrecision}, + reduce::ReducePrecision, +}; + +/// The base element trait for the jit backend. +pub trait CubeElement: Element + CubeElem + PartialEq + Numeric {} + +/// Element that can be used for matrix multiplication. Includes ints and floats. +pub trait MatmulElement: + CubeElement + MatmulPrecision> +{ +} + +/// The float element type for the jit backend. +pub trait FloatElement: MatmulElement + Float {} + +/// The int element type for the jit backend. +pub trait IntElement: + MatmulElement + Int + ReducePrecision +{ +} + +/// The element type for booleans for the jit backend. +pub trait BoolElement: CubeElement + Int { + /// The true value for the boolean element. + fn true_val() -> Self { + Self::from_int(1) + } + + /// The false value for the boolean element. + fn false_val() -> Self { + Self::from_int(0) + } + + /// New bool element from Rust bool. + fn new_bool(val: bool) -> Self { + match val { + true => Self::true_val(), + false => Self::false_val(), + } + } +} + +impl CubeElement for u64 {} +impl CubeElement for u32 {} +impl CubeElement for u16 {} +impl CubeElement for u8 {} +impl CubeElement for i64 {} +impl CubeElement for i32 {} +impl CubeElement for i16 {} +impl CubeElement for i8 {} +impl CubeElement for f64 {} +impl CubeElement for f32 {} +impl CubeElement for flex32 {} +impl CubeElement for f16 {} +impl CubeElement for bf16 {} + +impl FloatElement for f64 {} +impl FloatElement for f32 {} +impl FloatElement for flex32 {} +impl FloatElement for bf16 {} +impl FloatElement for f16 {} +impl IntElement for i64 {} +impl IntElement for i32 {} +impl IntElement for i16 {} +impl IntElement for i8 {} +impl IntElement for u64 {} +impl IntElement for u32 {} +impl IntElement for u16 {} +impl IntElement for u8 {} + +impl BoolElement for u8 {} +impl BoolElement for u32 {} + +impl MatmulElement for f64 {} +impl MatmulElement for f32 {} +impl MatmulElement for flex32 {} +impl MatmulElement for bf16 {} +impl MatmulElement for f16 {} + +impl MatmulElement for i64 {} +impl MatmulElement for i32 {} +impl MatmulElement for i16 {} +impl MatmulElement for i8 {} +impl MatmulElement for u64 {} +impl MatmulElement for u32 {} +impl MatmulElement for u16 {} +impl MatmulElement for u8 {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/fusion.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/fusion.rs new file mode 100644 index 0000000..6404c7e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/fusion.rs @@ -0,0 +1,205 @@ +use crate::BoolElement; +use crate::{CubeBackend, CubeRuntime, FloatElement, IntElement, kernel, tensor::CubeTensor}; +use burn_backend::tensor::{BoolTensor, FloatTensor, IntTensor, QuantizedTensor}; +use burn_backend::{DType, Shape}; +use burn_cubecl_fusion::optim::reduce::ReduceSettings; +use burn_cubecl_fusion::optim::reduce_broadcasted::ReduceBroadcastedFuser; +use burn_cubecl_fusion::{ + CubeFusionHandle, FallbackOperation, + optim::{ + CubeOptimization, CubeOptimizationState, + elemwise::{ElementWiseFuser, ElemwiseOptimization}, + matmul::{MatmulFuser, MatmulOptimization}, + reduce::{ReduceFuser, ReduceOptimization}, + reduce_broadcasted::ReduceBroadcastedOptimization, + }, +}; +use burn_fusion::{ + FusionBackend, FusionRuntime, + stream::{Operation, OrderedExecution}, +}; +use burn_ir::{BackendIr, TensorHandle}; +use burn_std::Metadata; +use core::marker::PhantomData; +use std::sync::Arc; + +impl burn_fusion::Optimization> for CubeOptimization +where + R: CubeRuntime, + BT: BoolElement, +{ + fn execute( + &mut self, + context: &mut burn_fusion::stream::Context< + '_, + as FusionRuntime>::FusionHandle, + >, + execution: &OrderedExecution>, + ) { + match self { + Self::ElementWise(op) => op.execute::(context), + Self::Matmul(op) => op.execute::(context, |index| { + let operation = execution.operation_within_optimization(index); + Box::new(FallbackOperationWrapper::new(operation)) + }), + Self::Reduce(op) => op.execute::(context, |index| { + let operation = execution.operation_within_optimization(index); + Box::new(FallbackOperationWrapper::new(operation)) + }), + Self::ReduceBroadcasted(op) => op.execute::(context, |index| { + let operation = execution.operation_within_optimization(index); + Box::new(FallbackOperationWrapper::new(operation)) + }), + } + } + + fn to_state(&self) -> CubeOptimizationState { + self.to_opt_state() + } + + fn from_state(device: &R::Device, state: CubeOptimizationState) -> Self { + match state { + CubeOptimizationState::ElementWise(state) => { + Self::ElementWise(ElemwiseOptimization::from_state(device, state)) + } + CubeOptimizationState::Matmul(state) => { + Self::Matmul(MatmulOptimization::from_state(device, state)) + } + CubeOptimizationState::Reduce(state) => { + Self::Reduce(ReduceOptimization::from_state(device, state)) + } + CubeOptimizationState::ReduceBroadcasted(state) => { + Self::ReduceBroadcasted(ReduceBroadcastedOptimization::from_state(device, state)) + } + } + } +} + +struct FallbackOperationWrapper { + operation: O, +} + +impl FallbackOperationWrapper { + fn new(op: O) -> Self { + Self { operation: op } + } +} + +impl FallbackOperation + for FallbackOperationWrapper>>> +{ + fn run(&self, context: &mut burn_fusion::stream::Context<'_, CubeFusionHandle>) { + self.operation.as_ref().execute(context.handles); + } +} + +impl BackendIr + for CubeBackend +{ + type Handle = CubeFusionHandle; + + fn float_tensor(handle: TensorHandle) -> FloatTensor { + into_tensor(handle.handle, handle.shape) + } + + fn int_tensor(handle: TensorHandle) -> IntTensor { + into_tensor(handle.handle, handle.shape) + } + + fn bool_tensor(handle: TensorHandle) -> BoolTensor { + into_tensor(handle.handle, handle.shape) + } + + fn quantized_tensor(handle: TensorHandle) -> QuantizedTensor { + into_tensor(handle.handle, handle.shape) + } + + fn float_tensor_handle(tensor: FloatTensor) -> Self::Handle { + tensor.into() + } + + fn int_tensor_handle(tensor: IntTensor) -> Self::Handle { + tensor.into() + } + + fn bool_tensor_handle(tensor: BoolTensor) -> Self::Handle { + tensor.into() + } + + fn quantized_tensor_handle(tensor: QuantizedTensor) -> Self::Handle { + tensor.into() + } +} + +impl FusionRuntime for FusionCubeRuntime { + type OptimizationState = CubeOptimizationState; + type Optimization = CubeOptimization; + type FusionHandle = CubeFusionHandle; + type FusionDevice = R::CubeDevice; + type BoolRepr = BT; + + fn fusers(device: R::Device) -> Vec>> { + vec![ + Box::new(ElementWiseFuser::new( + device.clone(), + BT::as_type_native_unchecked().into(), + )), + Box::new(MatmulFuser::new( + device.clone(), + BT::as_type_native_unchecked().into(), + )), + Box::new(ReduceFuser::new( + device.clone(), + BT::as_type_native_unchecked().into(), + ReduceSettings::Always, + )), + Box::new(ReduceBroadcastedFuser::new( + device.clone(), + BT::as_type_native_unchecked().into(), + )), + ] + } +} + +/// Fusion runtime for JIT runtimes. +#[derive(Debug)] +pub struct FusionCubeRuntime { + _b: PhantomData, + _bool: PhantomData, +} + +impl FusionBackend + for CubeBackend +{ + type FusionRuntime = FusionCubeRuntime; + + type FullPrecisionBackend = CubeBackend; + + fn cast_float(tensor: FloatTensor, dtype: DType) -> Self::Handle { + kernel::cast(tensor, dtype).into() + } +} + +fn into_tensor(handle: CubeFusionHandle, shape: Shape) -> CubeTensor { + CubeTensor { + client: handle.client, + handle: handle.handle, + device: handle.device, + meta: Box::new(Metadata::new(shape, handle.strides)), + dtype: handle.dtype, + qparams: handle.qparams, + } +} + +impl From> for CubeFusionHandle { + fn from(value: CubeTensor) -> Self { + Self { + client: value.client, + handle: value.handle, + device: value.device, + strides: value.meta.strides, + dtype: value.dtype, + qparams: value.qparams, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/base.rs new file mode 100644 index 0000000..0656ce5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/base.rs @@ -0,0 +1,150 @@ +use crate::{ + CubeBackend, CubeRuntime, kernel::attention::attention_autotune, + ops::numeric::empty_device_dtype, tensor::CubeTensor, +}; +use burn_backend::{ + DType, Shape, + ops::{AttentionModuleOptions, attention::attention_fallback}, +}; +use cubek::attention::definition::{ + AccumulatorPrecision, AttentionGlobalTypes, AttentionOptions, AttentionSetupError, +}; +use cubek::attention::launch; + +#[derive(Debug)] +/// Strategy used to select which attention implementation to run. +pub enum AttentionStrategy { + /// Flash Attention using accelerated inner matmuls. + FlashBlackboxAccelerated, + + /// Flash Attention using unit inner matmuls. + FlashUnit, + + /// Fallback implementation using multiple separate kernels. + Fallback, + + /// Automatically benchmark and select the best strategy at runtime. + #[cfg(feature = "autotune")] + Autotune, +} + +impl Default for AttentionStrategy { + fn default() -> Self { + // if autotune is enabled, default to autotune + #[cfg(feature = "autotune")] + return AttentionStrategy::Autotune; + + // if autotune is disabled, default to fallback to make sure it runs + #[cfg(not(feature = "autotune"))] + AttentionStrategy::Fallback + } +} + +#[allow(clippy::too_many_arguments)] +/// Launch an attention kernel with given strategy +pub fn attention( + query: CubeTensor, + key: CubeTensor, + value: CubeTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, + strategy: &AttentionStrategy, + out: Option>, +) -> Result, AttentionSetupError> { + let mut out = out.unwrap_or_else(|| init_attention_output(&query, &value)); + match strategy { + AttentionStrategy::FlashBlackboxAccelerated => flash_attention( + query, + key, + value, + mask, + attn_bias, + options, + out, + launch::Strategy::BlackboxAccelerated( + cubek::attention::launch::BlueprintStrategy::Inferred(()), + ), + ), + AttentionStrategy::FlashUnit => flash_attention( + query, + key, + value, + mask, + attn_bias, + options, + out, + launch::Strategy::Unit(cubek::attention::launch::BlueprintStrategy::Inferred(())), + ), + AttentionStrategy::Fallback => { + out = attention_fallback::>( + query, key, value, mask, attn_bias, options, + ); + Ok(out) + } + #[cfg(feature = "autotune")] + AttentionStrategy::Autotune => { + attention_autotune(query, key, value, mask, attn_bias, options, out) + } + } +} + +#[allow(clippy::too_many_arguments)] +/// Launch a flash attention kernel +pub fn flash_attention( + query: CubeTensor, + key: CubeTensor, + value: CubeTensor, + mask: Option>, + _attn_bias: Option>, + options: AttentionModuleOptions, + out: CubeTensor, + strategy: launch::Strategy, +) -> Result, AttentionSetupError> { + let client = &query.client; + + let dtypes = AttentionGlobalTypes { + query: query.dtype.into(), + key: key.dtype.into(), + value: value.dtype.into(), + mask: mask.as_ref().map(|m| m.dtype).unwrap_or(DType::U8).into(), + out: out.dtype.into(), + }; + + cubek::attention::launch::launch_ref::( + strategy, + client, + &query.as_handle_ref(), + &key.as_handle_ref(), + &value.as_handle_ref(), + &mask.as_ref().map(|mask| mask.as_handle_ref()), + &out.as_handle_ref(), + &dtypes, + AttentionOptions { + causal: options.is_causal, + accumulator_precision: AccumulatorPrecision::Strict(cubecl::ir::StorageType::Scalar( + cubecl::ir::ElemType::Float(cubecl::ir::FloatKind::F32), + )), + }, + )?; + + Ok(out) +} + +pub(crate) fn init_attention_output( + query: &CubeTensor, + value: &CubeTensor, +) -> CubeTensor { + let num_batches = query.meta.shape[0]; + let num_heads = query.meta.shape[1]; + let seq_q = query.meta.shape[2]; + let val_dim = value.meta.shape[3]; + let out_shape = Shape::new([num_batches, num_heads, seq_q, val_dim]); + + empty_device_dtype::( + query.client.clone(), + query.device.clone(), + out_shape, + query.dtype, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/mod.rs new file mode 100644 index 0000000..8ff38a9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod tune; + +pub use base::*; +pub use tune::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/tune.rs new file mode 100644 index 0000000..875ad0e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/attention/tune.rs @@ -0,0 +1,166 @@ +use crate::{ + CubeRuntime, CubeTuneId, + kernel::attention::{AttentionStrategy, attention}, + tensor::CubeTensor, +}; +use burn_backend::ops::AttentionModuleOptions; +use cubecl::tune::{LocalTuner, Tunable, TunableSet, TuneGroup, local_tuner}; +use cubek::attention::{definition::AttentionSetupError, launch::AttentionAutotuneKey}; + +/// Executes autotune on attention operations +pub fn attention_autotune( + query: CubeTensor, + key: CubeTensor, + value: CubeTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, + out: CubeTensor, +) -> Result, AttentionSetupError> { + let client = query.client.clone(); + + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + const PRIORITY_MAX: i8 = 3; + const PRIORITY_MIN: i8 = 0; + + let flash_attention = + TuneGroup::::new("flash_attention", |_key| PRIORITY_MAX); + let fallback = TuneGroup::::new("fallback", |_key| PRIORITY_MIN); + + let mut set = TunableSet::new(create_key::, input_gen::); + + // First entry should always work, since it is considered the fallback. + set = set.with( + Tunable::new( + "fallback", + |query, key, value, mask, attn_bias, out, options| { + attention::( + query, + key, + value, + mask, + attn_bias, + options, + &AttentionStrategy::Fallback, + Some(out), + ) + .map_err(|err| std::format!("{err:?}")) + }, + ) + .group(&fallback, |_key| PRIORITY_MAX), + ); + + set = set.with( + Tunable::new( + "blackbox_accelerated", + |query, key, value, mask, attn_bias, out, options| { + attention::( + query, + key, + value, + mask, + attn_bias, + options, + &AttentionStrategy::FlashBlackboxAccelerated, + Some(out), + ) + .map_err(|err| std::format!("{err:?}")) + }, + ) + .group(&flash_attention, |_key| PRIORITY_MAX), + ); + + set = set.with( + Tunable::new( + "unit", + |query, key, value, mask, attn_bias, out, options| { + attention::( + query, + key, + value, + mask, + attn_bias, + options, + &AttentionStrategy::FlashUnit, + Some(out), + ) + .map_err(|err| std::format!("{err:?}")) + }, + ) + .group(&flash_attention, |_key| PRIORITY_MIN), + ); + + set + }); + + TUNER.execute( + &CubeTuneId::new(&client, &query.device), + &client, + tunables, + (query, key, value, mask, attn_bias, out.clone(), options), + ); + + Ok(out) +} + +fn create_key( + query: &CubeTensor, + key: &CubeTensor, + value: &CubeTensor, + mask: &Option>, + _attn_bias: &Option>, + out: &CubeTensor, + _options: &AttentionModuleOptions, +) -> AttentionAutotuneKey { + let total_batches = query.meta.shape[0] * query.meta.shape[1]; + let seq_q = query.meta.shape[2]; + let head_dim = query.meta.shape[3]; + let seq_kv = value.meta.shape[2]; + let val_dim = value.meta.shape[3]; + + AttentionAutotuneKey::generate( + query.dtype.into(), + key.dtype.into(), + value.dtype.into(), + out.dtype.into(), + total_batches, + seq_q, + head_dim, + seq_kv, + val_dim, + mask.is_some(), + ) +} + +#[allow(clippy::type_complexity)] +#[allow(clippy::too_many_arguments)] +fn input_gen( + _key: &AttentionAutotuneKey, + query: &CubeTensor, + key: &CubeTensor, + value: &CubeTensor, + mask: &Option>, + attn_bias: &Option>, + out: &CubeTensor, + options: &AttentionModuleOptions, +) -> ( + CubeTensor, + CubeTensor, + CubeTensor, + Option>, + Option>, + CubeTensor, + AttentionModuleOptions, +) { + ( + query.clone(), + key.clone(), + value.clone(), + mask.clone(), + attn_bias.clone(), + out.copy(), + *options, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary.rs new file mode 100644 index 0000000..7b137a8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary.rs @@ -0,0 +1,302 @@ +use crate::{ + CubeRuntime, + kernel::utils::{ + address_type, broadcast_shape, linear_view, linear_view_alias, linear_view_ref, + }, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::{TensorMetadata, bf16, f16}; +use cubecl::{ + calculate_cube_count_elemwise, intrinsic, prelude::*, std::tensor::layout::linear::LinearView, +}; + +pub(crate) trait BinaryOpFamily: Send + Sync + 'static { + type BinaryOp: BinaryOp; +} + +#[cube] +pub(crate) trait BinaryOp: 'static + Send + Sync { + /// Execute a binary operation. + fn execute(lhs: Line, rhs: Line) -> Line; +} + +pub(crate) struct AddOp; +pub(crate) struct SubOp; +pub(crate) struct MulOp; +pub(crate) struct DivOp; +pub(crate) struct RemainderOp; +pub(crate) struct AndOp; +pub(crate) struct OrOp; +pub(crate) struct PowOp; + +impl BinaryOpFamily for AddOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for SubOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for MulOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for DivOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for RemainderOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for PowOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for AndOp { + type BinaryOp = Self; +} + +impl BinaryOpFamily for OrOp { + type BinaryOp = Self; +} + +#[cube] +impl BinaryOp for AddOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs + rhs + } +} + +#[cube] +impl BinaryOp for SubOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs - rhs + } +} + +#[cube] +impl BinaryOp for MulOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs * rhs + } +} + +#[cube] +impl BinaryOp for DivOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs / rhs + } +} + +#[cube] +impl BinaryOp for RemainderOp { + fn execute(lhs: Line, rhs: Line) -> Line { + Line::rem(lhs, rhs) + } +} + +#[cube] +impl BinaryOp for PowOp { + #[allow(unused)] + fn execute(lhs: Line, rhs: Line) -> Line { + intrinsic!(|scope| { + let elem = N::as_type(scope).elem_type(); + + if let cubecl::ir::ElemType::Float(kind) = elem { + match kind { + cubecl::ir::FloatKind::F16 => { + let lhs = as Cast>::__expand_cast_from(scope, lhs); + let rhs = as Cast>::__expand_cast_from(scope, rhs); + let out = Line::__expand_powf(scope, lhs, rhs); + return as Cast>::__expand_cast_from(scope, out); + } + cubecl::ir::FloatKind::BF16 => { + let lhs = as Cast>::__expand_cast_from(scope, lhs); + let rhs = as Cast>::__expand_cast_from(scope, rhs); + let out = Line::__expand_powf(scope, lhs, rhs); + return as Cast>::__expand_cast_from(scope, out); + } + cubecl::ir::FloatKind::F64 => { + let lhs = as Cast>::__expand_cast_from(scope, lhs); + let rhs = as Cast>::__expand_cast_from(scope, rhs); + let out = Line::__expand_powf(scope, lhs, rhs); + return as Cast>::__expand_cast_from(scope, out); + } + _ => {} + } + }; + + let lhs = as Cast>::__expand_cast_from(scope, lhs); + let rhs = as Cast>::__expand_cast_from(scope, rhs); + let out = Line::__expand_powf(scope, lhs, rhs); + return as Cast>::__expand_cast_from(scope, out); + }) + } +} + +#[cube] +impl BinaryOp for AndOp { + fn execute(lhs: Line, rhs: Line) -> Line { + Line::cast_from(Line::::cast_from(lhs).and(Line::::cast_from(rhs))) + } +} + +#[cube] +impl BinaryOp for OrOp { + fn execute(lhs: Line, rhs: Line) -> Line { + Line::cast_from(Line::::cast_from(lhs).or(Line::::cast_from(rhs))) + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_scalar_binop( + input: &LinearView>, + scalar: InputScalar, + output: &mut LinearView, ReadWrite>, + #[define(C)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = + O::BinaryOp::::execute(input[ABSOLUTE_POS], Line::new(scalar.get::())); +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_binop( + lhs: &LinearView>, + rhs: &LinearView>, + out: &mut LinearView, ReadWrite>, + #[define(C)] _dtype: StorageType, +) { + if !out.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + out[ABSOLUTE_POS] = O::BinaryOp::::execute(lhs[ABSOLUTE_POS], rhs[ABSOLUTE_POS]); +} + +pub(crate) fn launch_binop( + lhs: CubeTensor, + rhs: CubeTensor, +) -> CubeTensor { + let line_size_lhs = max_line_size(&lhs); + let line_size_rhs = max_line_size(&rhs); + let line_size = Ord::min(line_size_lhs, line_size_rhs); + + let shape_out = broadcast_shape(&[&lhs, &rhs]); + let dtype = lhs.dtype; + + let client = lhs.client.clone(); + let num_elems = shape_out.num_elements(); + let working_units = num_elems / line_size as usize; + + let cube_dim = CubeDim::new(&lhs.client, working_units); + let cube_count = calculate_cube_count_elemwise(&lhs.client, working_units, cube_dim); + + unsafe { + if lhs.can_mut_broadcast(&rhs) { + kernel_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view(&lhs, line_size), + linear_view_ref(&rhs, &lhs, line_size), + linear_view_alias(&lhs, line_size, 0), + dtype.into(), + ) + .expect("Kernel to never fail"); + + lhs + } else if rhs.can_mut_broadcast(&lhs) { + kernel_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view_ref(&lhs, &rhs, line_size), + linear_view(&rhs, line_size), + linear_view_alias(&rhs, line_size, 1), + dtype.into(), + ) + .expect("Kernel to never fail"); + + rhs + } else { + let output = + empty_device_dtype(lhs.client.clone(), lhs.device.clone(), shape_out, dtype); + + kernel_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs, output), + linear_view_ref(&lhs, &output, line_size), + linear_view_ref(&rhs, &output, line_size), + linear_view(&output, line_size), + dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} + +pub(crate) fn launch_scalar_binop( + tensor: CubeTensor, + scalar: InputScalar, +) -> CubeTensor { + // Vectorization is only enabled when the last dimension is contiguous. + let line_size = max_line_size(&tensor); + let client = tensor.client.clone(); + let num_elems = tensor.meta.num_elements(); + let dtype = tensor.dtype; + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + if tensor.can_mut() && tensor.is_nonoverlapping() { + kernel_scalar_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor), + linear_view(&tensor, line_size), + scalar, + linear_view_alias(&tensor, line_size, 0), + dtype.into(), + ) + .expect("Kernel to never fail"); + + tensor + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + dtype, + ); + + kernel_scalar_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + scalar, + linear_view(&output, line_size), + dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_float.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_float.rs new file mode 100644 index 0000000..2a3537d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_float.rs @@ -0,0 +1,119 @@ +use crate::{ + CubeRuntime, + kernel::utils::{ + address_type, broadcast_shape, linear_view, linear_view_alias, linear_view_ref, + }, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +pub(crate) trait BinaryOpFloatFamily: Send + Sync + 'static { + type BinaryOp: BinaryOpFloat; +} + +#[cube] +pub(crate) trait BinaryOpFloat: 'static + Send + Sync { + /// Execute a binary operation. + fn execute(lhs: Line, rhs: Line) -> Line; +} + +pub(crate) struct ArcTan2Op; + +impl BinaryOpFloatFamily for ArcTan2Op { + type BinaryOp = Self; +} + +#[cube] +impl BinaryOpFloat for ArcTan2Op { + fn execute(lhs: Line, rhs: Line) -> Line { + Line::atan2(lhs, rhs) + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_binop( + lhs: &LinearView>, + rhs: &LinearView>, + out: &mut LinearView, ReadWrite>, + #[define(C)] _dtype: StorageType, +) { + if !out.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + out[ABSOLUTE_POS] = O::BinaryOp::::execute(lhs[ABSOLUTE_POS], rhs[ABSOLUTE_POS]); +} + +pub(crate) fn launch_binop_float( + lhs: CubeTensor, + rhs: CubeTensor, +) -> CubeTensor { + let line_size_lhs = max_line_size(&lhs); + let line_size_rhs = max_line_size(&rhs); + let line_size = Ord::min(line_size_lhs, line_size_rhs); + + let shape_out = broadcast_shape(&[&lhs, &rhs]); + let dtype = lhs.dtype; + + let client = lhs.client.clone(); + let num_elems = shape_out.num_elements(); + let working_units = num_elems / line_size as usize; + + let cube_dim = CubeDim::new(&lhs.client, working_units); + let cube_count = calculate_cube_count_elemwise(&lhs.client, working_units, cube_dim); + + unsafe { + if lhs.can_mut_broadcast(&rhs) { + kernel_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view(&lhs, line_size), + linear_view_ref(&rhs, &lhs, line_size), + linear_view_alias(&lhs, line_size, 0), + dtype.into(), + ) + .expect("Kernel to never fail"); + + lhs + } else if rhs.can_mut_broadcast(&lhs) { + kernel_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view_ref(&lhs, &rhs, line_size), + linear_view(&rhs, line_size), + linear_view_alias(&rhs, line_size, 1), + dtype.into(), + ) + .expect("Kernel to never fail"); + + rhs + } else { + let output = + empty_device_dtype(lhs.client.clone(), lhs.device.clone(), shape_out, dtype); + + kernel_binop::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs, output), + linear_view_ref(&lhs, &output, line_size), + linear_view_ref(&rhs, &output, line_size), + linear_view(&output, line_size), + dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} + +/// Calculate the four-quadrant inverse tangent of `lhs / rhs`. +pub fn atan2(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop_float::(lhs, rhs) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_int.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_int.rs new file mode 100644 index 0000000..89a50c0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/binary_int.rs @@ -0,0 +1,229 @@ +use crate::{ + CubeRuntime, + kernel::utils::{ + address_type, broadcast_shape, linear_view, linear_view_alias, linear_view_ref, + }, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::TensorMetadata; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +pub(crate) trait BinaryOpIntFamily: Send + Sync + 'static { + type BinaryOp: BinaryOpInt; +} + +#[cube] +pub(crate) trait BinaryOpInt: 'static + Send + Sync { + /// Execute a binary operation. + fn execute(lhs: Line, rhs: Line) -> Line; +} + +pub(crate) struct BitwiseAndOp; +pub(crate) struct BitwiseOrOp; +pub(crate) struct BitwiseXorOp; +pub(crate) struct BitwiseShrOp; +pub(crate) struct BitwiseShlOp; + +impl BinaryOpIntFamily for BitwiseAndOp { + type BinaryOp = Self; +} + +impl BinaryOpIntFamily for BitwiseOrOp { + type BinaryOp = Self; +} + +impl BinaryOpIntFamily for BitwiseXorOp { + type BinaryOp = Self; +} + +impl BinaryOpIntFamily for BitwiseShrOp { + type BinaryOp = Self; +} + +impl BinaryOpIntFamily for BitwiseShlOp { + type BinaryOp = Self; +} + +#[cube] +impl BinaryOpInt for BitwiseAndOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs & rhs + } +} + +#[cube] +impl BinaryOpInt for BitwiseOrOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs | rhs + } +} + +#[cube] +impl BinaryOpInt for BitwiseXorOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs ^ rhs + } +} + +#[cube] +impl BinaryOpInt for BitwiseShrOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs >> rhs + } +} + +#[cube] +impl BinaryOpInt for BitwiseShlOp { + fn execute(lhs: Line, rhs: Line) -> Line { + lhs << rhs + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_scalar_binop_int( + input: &LinearView>, + scalar: InputScalar, + output: &mut LinearView, ReadWrite>, + #[define(C)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = + O::BinaryOp::::execute(input[ABSOLUTE_POS], Line::new(scalar.get::())); +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_binop_int( + lhs: &LinearView>, + rhs: &LinearView>, + out: &mut LinearView, ReadWrite>, + #[define(C)] _dtype: StorageType, +) { + if !out.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + out[ABSOLUTE_POS] = O::BinaryOp::::execute(lhs[ABSOLUTE_POS], rhs[ABSOLUTE_POS]); +} + +pub(crate) fn launch_binop_int( + lhs: CubeTensor, + rhs: CubeTensor, +) -> CubeTensor { + let line_size_lhs = max_line_size(&lhs); + let line_size_rhs = max_line_size(&rhs); + let line_size = Ord::min(line_size_lhs, line_size_rhs); + + let shape_out = broadcast_shape(&[&lhs, &rhs]); + + let client = lhs.client.clone(); + let num_elems = shape_out.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&lhs.client, working_units); + let cube_count = calculate_cube_count_elemwise(&lhs.client, working_units, cube_dim); + + unsafe { + if lhs.can_mut_broadcast(&rhs) { + kernel_binop_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view(&lhs, line_size), + linear_view_ref(&rhs, &lhs, line_size), + linear_view_alias(&lhs, line_size, 0), + lhs.dtype.into(), + ) + .expect("Kernel to never fail"); + + lhs + } else if rhs.can_mut_broadcast(&lhs) { + kernel_binop_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view_ref(&lhs, &rhs, line_size), + linear_view(&rhs, line_size), + linear_view_alias(&rhs, line_size, 1), + lhs.dtype.into(), + ) + .expect("Kernel to never fail"); + + rhs + } else { + let output = + empty_device_dtype(lhs.client.clone(), lhs.device.clone(), shape_out, lhs.dtype); + + kernel_binop_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs, output), + linear_view_ref(&lhs, &output, line_size), + linear_view_ref(&rhs, &output, line_size), + linear_view(&output, line_size), + lhs.dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} + +pub(crate) fn launch_scalar_binop_int( + tensor: CubeTensor, + scalar: InputScalar, +) -> CubeTensor { + let line_size = max_line_size(&tensor); + let client = tensor.client.clone(); + let num_elems = tensor.meta.shape.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + if tensor.can_mut() && tensor.is_nonoverlapping() { + kernel_scalar_binop_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor), + linear_view(&tensor, line_size), + scalar, + linear_view_alias(&tensor, line_size, 0), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + tensor + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + tensor.dtype, + ); + + kernel_scalar_binop_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + scalar, + linear_view(&output, line_size), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/base.rs new file mode 100644 index 0000000..0c9931f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/base.rs @@ -0,0 +1,70 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view}, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::{DType, TensorMetadata}; +use cubecl::std::tensor::layout::linear::LinearView; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +#[cube(launch, address_type = "dynamic")] +pub(crate) fn cast_element( + input: &LinearView>, + output: &mut LinearView, ReadWrite>, + #[define(I, O)] _dtypes: [StorageType; 2], +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = Line::cast_from(input[ABSOLUTE_POS]); +} + +/// Cast a tensor to the given element type. +/// +/// Note: When input element is semantically a boolean, prefer bool_cast function. +pub fn cast(input: CubeTensor, dtype: DType) -> CubeTensor { + let dtype_output = match dtype { + DType::Flex32 => DType::F32, + _ => dtype, + }; + let dtype_input = match input.dtype { + DType::Flex32 => DType::F32, + _ => input.dtype, + }; + + if dtype_input == dtype_output { + return input; + } + + let client = input.client.clone(); + + let line_size = max_line_size(&input); + + let num_elems: usize = input.meta.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&client, working_units); + let cube_count = calculate_cube_count_elemwise(&client, working_units, cube_dim); + + let output = empty_device_dtype( + client.clone(), + input.device.clone(), + input.shape(), + dtype, // We take the same dtype as passed as input (Flex32 not F32) + ); + + cast_element::launch( + &client, + cube_count, + cube_dim, + address_type!(input, output), + linear_view(&input, line_size), + linear_view(&output, line_size), + [dtype_input.into(), dtype_output.into()], + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/bool_cast.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/bool_cast.rs new file mode 100644 index 0000000..f024f96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/bool_cast.rs @@ -0,0 +1,55 @@ +use crate::{ + CubeElement, CubeRuntime, + kernel::utils::{address_type, linear_view}, + ops::{max_line_size, numeric::empty_device}, + tensor::CubeTensor, +}; +use burn_backend::TensorMetadata; +use cubecl::{ + CubeDim, calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn bool_cast_kernel( + input: &LinearView>, + output: &mut LinearView, ReadWrite>, + #[define(B)] _input_ty: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = Line::cast_from(input[ABSOLUTE_POS] & Line::cast_from(1u32)); +} + +/// Cast a bool tensor to the given element type. +/// +/// This alternative to cast is necessary because bool are represented as u32 or u8 +/// where any non-zero value means true. Depending how it was created +/// it may hold an uncanny bit combination. Naively casting it would not +/// necessarily yield 0 or 1. +pub fn bool_cast(tensor: CubeTensor) -> CubeTensor { + let output = + empty_device::(tensor.client.clone(), tensor.device.clone(), tensor.shape()); + + let line_size = max_line_size(&tensor); + let num_elems = tensor.meta.num_elements(); + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + bool_cast_kernel::launch_unchecked::( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + linear_view(&output, line_size), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + } + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/mod.rs new file mode 100644 index 0000000..52d9d7b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cast/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod bool_cast; + +pub use base::*; +pub use bool_cast::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/clamp.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/clamp.rs new file mode 100644 index 0000000..45bd59d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/clamp.rs @@ -0,0 +1,42 @@ +use cubecl::prelude::*; + +use crate::{ + CubeRuntime, + kernel::{NumericUnaryOp, NumericUnaryOpFamily, launch_unary_numeric}, + tensor::CubeTensor, +}; + +#[derive(CubeLaunch, CubeType)] +struct Options { + min_value: InputScalar, + max_value: InputScalar, +} + +pub(crate) fn clamp( + input: CubeTensor, + min_value: InputScalar, + max_value: InputScalar, +) -> CubeTensor { + struct ClampOp; + + #[cube] + impl NumericUnaryOp for ClampOp { + type Options = Options; + + fn execute(input: Line, options: &Self::Options) -> Line { + let line_size = input.size(); + cubecl::prelude::clamp( + input, + Line::empty(line_size).fill(options.min_value.get::()), + Line::empty(line_size).fill(options.max_value.get::()), + ) + } + } + + impl NumericUnaryOpFamily for ClampOp { + type Options = Options; + type Unary = Self; + } + + launch_unary_numeric::(input, |_| OptionsLaunch::new(min_value, max_value)) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/comparison.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/comparison.rs new file mode 100644 index 0000000..de1851c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/comparison.rs @@ -0,0 +1,432 @@ +use crate::{ + CubeRuntime, + kernel::utils::{ + address_type, broadcast_shape, linear_view, linear_view_alias, linear_view_ref, + }, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::{DType, TensorMetadata}; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +#[cube] +pub(crate) trait ComparisonOpFamily: 'static + Send + Sync { + type Operation: ComparisonOp; +} + +#[cube] +pub(crate) trait ComparisonOp: 'static + Send + Sync { + /// Execute a comparison operation. + fn execute(lhs: Line, rhs: Line) -> bool; +} + +struct EqualOp; +struct GreaterEqualOp; +struct LowerEqualOp; +struct GreaterOp; +struct LowerOp; + +impl ComparisonOpFamily for EqualOp { + type Operation = Self; +} + +#[cube] +impl ComparisonOp for EqualOp { + fn execute(lhs: Line, rhs: Line) -> bool { + lhs == rhs + } +} + +impl ComparisonOpFamily for GreaterEqualOp { + type Operation = Self; +} + +#[cube] +impl ComparisonOp for GreaterEqualOp { + fn execute(lhs: Line, rhs: Line) -> bool { + lhs >= rhs + } +} + +impl ComparisonOpFamily for LowerEqualOp { + type Operation = Self; +} + +#[cube] +impl ComparisonOp for LowerEqualOp { + fn execute(lhs: Line, rhs: Line) -> bool { + lhs <= rhs + } +} + +impl ComparisonOpFamily for GreaterOp { + type Operation = Self; +} + +#[cube] +impl ComparisonOp for GreaterOp { + fn execute(lhs: Line, rhs: Line) -> bool { + lhs > rhs + } +} + +impl ComparisonOpFamily for LowerOp { + type Operation = Self; +} + +#[cube] +impl ComparisonOp for LowerOp { + fn execute(lhs: Line, rhs: Line) -> bool { + lhs < rhs + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_scalar_cmp( + input: &LinearView>, + scalar: InputScalar, + output: &mut LinearView, ReadWrite>, + #[define(N, Bool)] _dtypes: [StorageType; 2], +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = Line::cast_from(O::Operation::::execute( + input[ABSOLUTE_POS], + Line::new(scalar.get::()), + )); +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_cmp( + lhs: &LinearView>, + rhs: &LinearView>, + out: &mut LinearView, ReadWrite>, + #[define(N, Bool)] _dtype: [StorageType; 2], +) { + if !out.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + out[ABSOLUTE_POS] = Line::cast_from(O::Operation::::execute( + lhs[ABSOLUTE_POS], + rhs[ABSOLUTE_POS], + )); +} + +pub(crate) fn launch_cmp( + lhs: CubeTensor, + rhs: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + let line_size_lhs = max_line_size(&lhs); + let line_size_rhs = max_line_size(&rhs); + + let line_size = Ord::min(line_size_lhs, line_size_rhs); + + let shape_out = broadcast_shape(&[&lhs, &rhs]); + let client = lhs.client.clone(); + let num_elems = shape_out.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&lhs.client, working_units); + let cube_count = calculate_cube_count_elemwise(&lhs.client, working_units, cube_dim); + + let dtypes = [lhs.dtype.into(), dtype_bool.into()]; + let same_tensor_type = dtypes[0] == dtypes[1]; + if same_tensor_type && lhs.can_mut_broadcast(&rhs) { + unsafe { + kernel_cmp::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view(&lhs, line_size), + linear_view_ref(&rhs, &lhs, line_size), + linear_view_alias(&lhs, line_size, 0), + dtypes, + ) + .expect("Kernel to never fail"); + } + + CubeTensor::new(lhs.client, lhs.handle, *lhs.meta, lhs.device, dtype_bool) + } else if same_tensor_type && rhs.can_mut_broadcast(&lhs) { + unsafe { + kernel_cmp::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs), + linear_view_ref(&lhs, &rhs, line_size), + linear_view(&rhs, line_size), + linear_view_alias(&rhs, line_size, 1), + dtypes, + ) + .expect("Kernel to never fail"); + }; + + CubeTensor::new(rhs.client, rhs.handle, *rhs.meta, rhs.device, dtype_bool) + } else { + let output = empty_device_dtype( + lhs.client.clone(), + lhs.device.clone(), + shape_out, + dtype_bool, + ); + + unsafe { + kernel_cmp::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(lhs, rhs, output), + linear_view_ref(&lhs, &output, line_size), + linear_view_ref(&rhs, &output, line_size), + linear_view(&output, line_size), + dtypes, + ) + .expect("Kernel to never fail"); + }; + + output + } +} + +pub(crate) fn launch_scalar_cmp( + tensor: CubeTensor, + scalar: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + let line_size = max_line_size(&tensor); + let client = tensor.client.clone(); + let num_elems = tensor.meta.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + let dtypes = [tensor.dtype.into(), dtype_bool.into()]; + let same_tensor_type = dtypes[0] == dtypes[1]; + + if same_tensor_type && tensor.can_mut() && tensor.is_nonoverlapping() { + unsafe { + kernel_scalar_cmp::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor), + linear_view(&tensor, line_size), + scalar, + linear_view_alias(&tensor, line_size, 0), + dtypes, + ) + .expect("Kernel to never fail"); + } + + CubeTensor::new( + tensor.client, + tensor.handle, + *tensor.meta, + tensor.device, + dtype_bool, + ) + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + dtype_bool, + ); + + unsafe { + kernel_scalar_cmp::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + scalar, + linear_view(&output, line_size), + dtypes, + ) + .expect("Kernel to never fail"); + } + + output + } +} + +pub fn equal( + lhs: CubeTensor, + rhs: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + launch_cmp::(lhs, rhs, dtype_bool) +} + +pub fn greater( + lhs: CubeTensor, + rhs: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + launch_cmp::(lhs, rhs, dtype_bool) +} + +pub fn greater_equal( + lhs: CubeTensor, + rhs: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + launch_cmp::(lhs, rhs, dtype_bool) +} + +pub fn lower( + lhs: CubeTensor, + rhs: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + launch_cmp::(lhs, rhs, dtype_bool) +} + +pub fn lower_equal( + lhs: CubeTensor, + rhs: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + launch_cmp::(lhs, rhs, dtype_bool) +} + +pub fn equal_elem( + lhs: CubeTensor, + rhs: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + launch_scalar_cmp::(lhs, rhs, dtype_bool) +} + +pub fn greater_elem( + lhs: CubeTensor, + rhs: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + launch_scalar_cmp::(lhs, rhs, dtype_bool) +} + +pub fn lower_elem( + lhs: CubeTensor, + rhs: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + launch_scalar_cmp::(lhs, rhs, dtype_bool) +} + +pub fn greater_equal_elem( + lhs: CubeTensor, + rhs: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + launch_scalar_cmp::(lhs, rhs, dtype_bool) +} + +pub fn lower_equal_elem( + lhs: CubeTensor, + rhs: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + launch_scalar_cmp::(lhs, rhs, dtype_bool) +} + +// Unary comparison / predicate / relational ops + +#[cube] +pub(crate) trait PredicateOp: 'static + Send + Sync { + /// Execute a predicate operation. + fn execute(input: Line) -> bool; +} + +pub(crate) trait PredicateOpFamily: 'static + Send + Sync { + type Operation: PredicateOp; +} + +struct IsNanOp; +struct IsInfOp; + +impl PredicateOpFamily for IsNanOp { + type Operation = Self; +} + +#[cube] +impl PredicateOp for IsNanOp { + fn execute(input: Line) -> bool { + Line::is_nan(input) + } +} + +impl PredicateOpFamily for IsInfOp { + type Operation = Self; +} +#[cube] +impl PredicateOp for IsInfOp { + fn execute(input: Line) -> bool { + Line::is_inf(input) + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn kernel_predicate( + input: &LinearView>, + output: &mut LinearView, ReadWrite>, + #[define(F, Bool)] _dtypes: [StorageType; 2], +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = Line::cast_from(O::Operation::::execute(input[ABSOLUTE_POS])); +} + +pub(crate) fn launch_predicate( + tensor: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + let line_size = max_line_size(&tensor); + + let client = tensor.client.clone(); + let num_elems = tensor.meta.num_elements(); + + let dtypes = [tensor.dtype.into(), dtype_bool.into()]; + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + dtype_bool, + ); + + unsafe { + kernel_predicate::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view_ref(&tensor, &output, line_size), + linear_view(&output, line_size), + dtypes, + ) + .expect("Kernel to never fail"); + } + + output +} + +pub fn is_nan(tensor: CubeTensor, dtype_bool: DType) -> CubeTensor { + launch_predicate::(tensor, dtype_bool) +} + +pub fn is_inf(tensor: CubeTensor, dtype_bool: DType) -> CubeTensor { + launch_predicate::(tensor, dtype_bool) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/contiguous.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/contiguous.rs new file mode 100644 index 0000000..c2cf693 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/contiguous.rs @@ -0,0 +1,124 @@ +use burn_backend::{DType, QTensorPrimitive, TensorMetadata}; +use cubecl::quant::scheme::{QuantStore, QuantValue}; +use cubecl::server::AllocationKind; + +use crate::{CubeRuntime, ops::empty_qtensor, tensor::CubeTensor}; + +/// Make a jit tensor contiguous. +pub fn into_contiguous(tensor: CubeTensor) -> CubeTensor { + if tensor.is_contiguous() { + return tensor; + } + + if tensor.qparams.is_some() { + return into_contiguous_quantized(tensor, AllocationKind::Contiguous); + } + + let output = cubecl::std::tensor::into_contiguous_ref( + &tensor.client, + &tensor.as_handle_ref(), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + CubeTensor::new( + tensor.client, + output.handle, + *output.metadata, + tensor.device, + tensor.dtype, + ) +} + +/// Make a jit tensor contiguous with an aligned last stride. Tensor is considered already contiguous +/// if runtime can read it as is. This is equivalent in practice. +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensor)) +)] +pub fn into_contiguous_aligned(tensor: CubeTensor) -> CubeTensor { + if R::can_read_tensor(tensor.meta.shape(), tensor.meta.strides()) { + return tensor; + } + + if tensor.qparams.is_some() { + return into_contiguous_quantized(tensor, AllocationKind::Optimized); + } + + let output = cubecl::std::tensor::into_contiguous_pitched_ref( + &tensor.client, + &tensor.as_handle_ref(), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + CubeTensor::new( + tensor.client, + output.handle, + *output.metadata, + tensor.device, + tensor.dtype, + ) +} + +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensor)) +)] +fn into_contiguous_quantized( + tensor: CubeTensor, + kind: AllocationKind, +) -> CubeTensor { + let scheme = tensor.scheme(); + let output = empty_qtensor(tensor.shape(), *tensor.scheme(), &tensor.device, kind); + let (values, scales) = tensor.quantized_handles().unwrap(); + let (out_values, out_scales) = output.quantized_handles().unwrap(); + + match scheme.store { + QuantStore::PackedU32(packed_dim) => { + cubecl::std::tensor::into_contiguous_packed_ref( + &values.client, + &values.as_handle_ref(), + &out_values.as_handle_ref(), + packed_dim, + tensor.meta.shape(), + scheme.num_quants(), + DType::U32.into(), + ) + .expect("Kernel to never fail"); + } + // e2m1 is special because it has a native packed representation, `e2m1x2`. + // It's internally stored as `u8` with a packing factor of 2. + QuantStore::PackedNative(packed_dim) if scheme.value == QuantValue::E2M1 => { + cubecl::std::tensor::into_contiguous_packed_ref( + &values.client, + &values.as_handle_ref(), + &out_values.as_handle_ref(), + packed_dim, + tensor.meta.shape(), + scheme.num_quants(), + DType::U8.into(), + ) + .expect("Kernel to never fail"); + } + _ => { + cubecl::std::tensor::copy_into( + &values.client, + &values.as_handle_ref(), + &out_values.as_handle_ref(), + values.dtype.into(), + ) + .expect("Kernel to never fail"); + } + } + + cubecl::std::tensor::copy_into( + &scales.client, + &scales.as_handle_ref(), + &out_scales.as_handle_ref(), + scales.dtype.into(), + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/fallback.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/fallback.rs new file mode 100644 index 0000000..b289c1c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/fallback.rs @@ -0,0 +1,142 @@ +use burn_backend::{ + TensorMetadata, + ops::{ConvOptions, ConvTransposeOptions}, +}; +use burn_std::Shape; +use cubek::convolution::components::ConvSetupError; + +use crate::{ + CubeRuntime, + kernel::conv::{conv_transpose2d, conv_transpose3d}, + ops::{permute_nchw_to_nhwc, permute_nhwc_to_nchw, reshape}, + tensor::CubeTensor, +}; + +pub(crate) fn conv_data_backward_fallback( + out_grad: CubeTensor, + weights: CubeTensor, + in_shape: Shape, + options: ConvOptions, +) -> Result, ConvSetupError> { + let dim_c = out_grad.rank(); + + let kernel_size = &weights.meta.shape()[1..dim_c]; + let in_shape = &in_shape[1..dim_c]; + let out_shape = &out_grad.meta.shape()[1..dim_c]; + + let mut padding_out = [0; N_DIM]; + + for i in 0..N_DIM { + padding_out[i] = calculate_padding_out( + kernel_size[i], + options.stride[i], + options.padding[i], + options.dilation[i], + in_shape[i], + out_shape[i], + ); + } + + // We don't yet have NHWC kernels for conv_transpose so need to do this. + // Should eventually use NHWC kernels instead + let out_grad = permute_nhwc_to_nchw(out_grad); + let weights = permute_nhwc_to_nchw(weights); + + let in_grad = match N_DIM { + 1 => conv_transpose1d_from_conv_transpose2d( + out_grad, + weights, + ConvTransposeOptions::new( + [options.stride[0]], + [options.padding[0]], + [padding_out[0]], + [options.dilation[0]], + options.groups, + ), + ), + 2 => conv_transpose2d( + out_grad, + weights, + None, + ConvTransposeOptions::new( + [options.stride[0], options.stride[1]], + [options.padding[0], options.padding[1]], + [padding_out[0], padding_out[1]], + [options.dilation[0], options.dilation[1]], + options.groups, + ), + Default::default(), + ), + 3 => Ok(conv_transpose3d( + out_grad, + weights, + None, + ConvTransposeOptions::new( + [options.stride[0], options.stride[1], options.stride[2]], + [options.padding[0], options.padding[1], options.padding[2]], + [padding_out[0], padding_out[1], padding_out[2]], + [ + options.dilation[0], + options.dilation[1], + options.dilation[2], + ], + options.groups, + ), + ) + .unwrap()), + _ => unimplemented!("Invalid dimensionality"), + }?; + Ok(permute_nchw_to_nhwc(in_grad)) +} + +fn calculate_padding_out( + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + size_in: usize, + size_out: usize, +) -> usize { + if stride <= 1 { + return 0; + } + + let out = 1 + + ((size_in + 2 * padding - dilation * (kernel_size - 1) - 1) as f64 / stride as f64).ceil() + as usize; + i64::max(0, out as i64 - size_out as i64) as usize +} + +fn conv_transpose1d_from_conv_transpose2d( + x: CubeTensor, + weight: CubeTensor, + options: ConvTransposeOptions<1>, +) -> Result, ConvSetupError> { + let [channels_in, channels_out, kernel_size] = weight.shape().dims(); + let [batch_size, _channels_in, length_in] = x.shape().dims(); + + let weight = reshape( + weight, + Shape::new([channels_in, channels_out, kernel_size, 1]), + ); + let x = reshape(x, Shape::new([batch_size, channels_in, length_in, 1])); + + let tensor = conv_transpose2d( + x, + weight, + None, + ConvTransposeOptions::new( + [options.stride[0], 1], + [options.padding[0], 0], + [options.padding_out[0], 0], + [options.dilation[0], 1], + options.groups, + ), + Default::default(), + )?; + let [batch_size, channels_out, height_out, _weight_out] = tensor.shape().dims(); + Ok(reshape( + tensor, + Shape::from([batch_size, channels_out, height_out]), + )) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/launch.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/launch.rs new file mode 100644 index 0000000..82c9dfb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/launch.rs @@ -0,0 +1,132 @@ +use burn_backend::ops::ConvOptions; +use burn_std::Shape; +use cubek::{ + convolution::{ + AcceleratedTileKind, ConvolutionArgs, ReadingStrategy, Strategy, backward_data, + components::ConvSetupError, + }, + matmul::{ + definition::{MatmulElems, MatmulGlobalElems}, + launch::MatmulInputHandleRef, + }, +}; + +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; + +pub fn dgrad_gemm_simple_sync( + out_grad: CubeTensor, + weights: CubeTensor, + input_shape: Shape, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + let read_strategy = match tile_kind { + AcceleratedTileKind::Cmma => ReadingStrategy::Cyclic, + AcceleratedTileKind::Mma => ReadingStrategy::Strided, + }; + launch_backwards_data::( + &Strategy::Simple { + read_strategy, + tile_kind, + }, + out_grad, + weights, + input_shape, + options, + ) +} + +pub fn dgrad_gemm_simple_async( + out_grad: CubeTensor, + weights: CubeTensor, + input_shape: Shape, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + let read_strategy = match tile_kind { + AcceleratedTileKind::Cmma => ReadingStrategy::AsyncCyclic, + AcceleratedTileKind::Mma => ReadingStrategy::AsyncStrided, + }; + launch_backwards_data::( + &Strategy::Simple { + read_strategy, + tile_kind, + }, + out_grad, + weights, + input_shape, + options, + ) +} + +pub fn dgrad_gemm_simple_tma( + out_grad: CubeTensor, + weights: CubeTensor, + input_shape: Shape, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + launch_backwards_data::( + &Strategy::Simple { + read_strategy: ReadingStrategy::Tma, + tile_kind, + }, + out_grad, + weights, + input_shape, + options, + ) +} + +/// Perform a convolution backwards data pass using the implicit GEMM (im2col) algorithm, using +/// cubecl tiling matmul components. +/// +/// * `input` - The input feature map +/// * `out_grad` - The output gradients +/// * `weight_shape` - The shape of the weights/weight gradients +/// * `options` - The options to use for the convolution +pub fn launch_backwards_data( + strategy: &Strategy, + out_grad: CubeTensor, + weights: CubeTensor, + input_shape: Shape, + options: ConvOptions, +) -> Result, ConvSetupError> { + if options.groups != 1 || options.stride.iter().any(|&s| s != 1) { + return Err(ConvSetupError::Groups(options.groups)); + } + + let out_dtype = out_grad.dtype; + + let in_grad = empty_device_dtype( + out_grad.client.clone(), + out_grad.device.clone(), + input_shape, + out_dtype, + ); + + let client = out_grad.client.clone(); + let dtypes = MatmulElems::from_globals(&MatmulGlobalElems { + lhs: out_grad.dtype.into(), + rhs: weights.dtype.into(), + out: out_dtype.into(), + }); + let out_grad = MatmulInputHandleRef::new(out_grad.as_handle_ref(), out_grad.dtype.into()); + let weights = MatmulInputHandleRef::new(weights.as_handle_ref(), weights.dtype.into()); + + backward_data::launch_ref::( + strategy, + &client, + &out_grad, + &weights, + &in_grad.as_handle_ref(), + ConvolutionArgs { + stride: options.stride, + padding: options.padding, + dilation: options.dilation, + }, + dtypes, + )?; + + Ok(in_grad) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/mod.rs new file mode 100644 index 0000000..0df8c51 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/implicit_gemm/mod.rs @@ -0,0 +1,2 @@ +pub mod launch; +pub use launch::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/mod.rs new file mode 100644 index 0000000..80e94c4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/mod.rs @@ -0,0 +1,8 @@ +pub mod fallback; +pub mod implicit_gemm; + +#[cfg(feature = "autotune")] +pub mod tune; + +#[cfg(feature = "autotune")] +pub(crate) use tune::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/tune.rs new file mode 100644 index 0000000..7805bf6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_data/tune.rs @@ -0,0 +1,172 @@ +use burn_backend::ops::ConvOptions; +use burn_std::Shape; +use cubecl::{ + ir::StorageType, + tune::{LocalTuner, Tunable, TunableSet, anchor, local_tuner}, +}; +use cubek::convolution::AcceleratedTileKind; + +use crate::{ + CubeAutotuneKey, CubeRuntime, CubeTuneId, + kernel::conv::{ + ConvAutotuneKey, + backward_data::{fallback::conv_data_backward_fallback, implicit_gemm::*}, + }, + tensor::CubeTensor, +}; + +/// Executes autotune on conv2d operations +pub fn dgrad_autotune( + out_grad: CubeTensor, + weights: CubeTensor, + input_shape: Shape, + options: ConvOptions, +) -> CubeTensor { + let client = out_grad.client.clone(); + + static TUNER: LocalTuner = local_tuner!(); + + // Note: TMA isn't currently implemented properly, and will always error. + // It's kept here so it gets automatically enabled as soon as cubek updates. + // No CMMA for TMA because swizzling will be mandatory for good performance on dgrad. + let tunables = TUNER.init(|| { + TunableSet::new(create_key::, create_wgrad_input::) + .with(Tunable::new( + "wgrad_fallback", + conv_data_backward_fallback::, + )) + .with(Tunable::new( + "simple_sync_cmma", + |input, grad, shape, options| { + dgrad_gemm_simple_sync(input, grad, shape, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_sync_mma", + |input, grad, shape, options| { + dgrad_gemm_simple_sync(input, grad, shape, options, AcceleratedTileKind::Mma) + }, + )) + .with(Tunable::new( + "simple_async_cmma", + |input, grad, shape, options| { + dgrad_gemm_simple_async(input, grad, shape, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_async_mma", + |input, grad, shape, options| { + dgrad_gemm_simple_async(input, grad, shape, options, AcceleratedTileKind::Mma) + }, + )) + .with(Tunable::new( + "simple_tma_mma", + |input, grad, shape, options| { + dgrad_gemm_simple_tma(input, grad, shape, options, AcceleratedTileKind::Mma) + }, + )) + }); + + TUNER.execute( + &CubeTuneId::new(&out_grad.client, &out_grad.device), + &client, + tunables, + (out_grad, weights, input_shape, options), + ) +} + +pub fn create_wgrad_input( + _key: &CubeAutotuneKey, + out_grad: &CubeTensor, + weights: &CubeTensor, + input_shape: &Shape, + options: &ConvOptions, +) -> (CubeTensor, CubeTensor, Shape, ConvOptions) { + ( + out_grad.clone(), + weights.clone(), + input_shape.clone(), + options.clone(), + ) +} + +fn create_key( + out_grad: &CubeTensor, + weights: &CubeTensor, + input_shape: &Shape, + options: &ConvOptions, +) -> CubeAutotuneKey { + let dtype = out_grad.dtype; + let rank = out_grad.meta.num_dims(); + let dim_c = rank - 1; + + let batch_size = out_grad.meta.shape()[0]; + let in_channels = input_shape[dim_c]; + let out_channels = out_grad.meta.shape()[dim_c]; + + let kernel_size = weights.meta.shape()[1..dim_c].to_vec(); + let in_shape = input_shape[1..dim_c] + .iter() + .map(|shape| anchor(*shape, None, None, None)) + .collect(); + + let ConvOptions { + stride, + padding, + dilation, + groups, + } = options.clone(); + + let lhs_stride_align = if out_grad.meta.strides()[dim_c] == 1 { + stride_align(out_grad.meta.strides(), out_grad.dtype.into()) + } else { + 0 + }; + let lhs_shape_align = pow2_factor(out_channels).min(lhs_stride_align); + let rhs_stride_align = if weights.meta.strides()[dim_c] == 1 { + stride_align(weights.meta.strides(), weights.dtype.into()) + } else { + 0 + }; + let rhs_shape_align = pow2_factor(in_channels).min(rhs_stride_align); + + CubeAutotuneKey::Conv(ConvAutotuneKey::new( + kernel_size, + stride.to_vec(), + padding.to_vec(), + dilation.to_vec(), + groups, + in_channels, + out_channels, + in_shape, + batch_size, + false, + dtype, + lhs_shape_align, + lhs_stride_align, + rhs_shape_align, + rhs_stride_align, + )) +} + +/// Maximum factor relevant for strides. Currently set to 2^10 because that's 128-byte swizzle's +/// repeat number, so it's the largest align that can have performance impacts. +const MAX_STRIDE_FACTOR: u32 = 10; + +/// Defines the non-contiguous stride alignment in terms of powers of two +fn stride_align(strides: &[usize], elem: StorageType) -> u8 { + let max = MAX_STRIDE_FACTOR; + let dim_c = strides.len() - 1; + let factor = strides[..dim_c] + .iter() + .map(|it| (*it * elem.size_bits()) / 8) + .map(|it| it.trailing_zeros()) + .min() + .unwrap_or(max); + factor.min(max) as u8 +} + +/// Defines the potential vectorization. +fn pow2_factor(axis: usize) -> u8 { + axis.trailing_zeros().min(4) as u8 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/fallback.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/fallback.rs new file mode 100644 index 0000000..6ddd9f2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/fallback.rs @@ -0,0 +1,112 @@ +use burn_backend::{TensorMetadata, ops::ConvOptions}; +use burn_std::{Shape, Slice}; +use cubek::convolution::components::ConvSetupError; + +use crate::{ + CubeRuntime, + kernel::{conv::base::conv_forward_nhwc, slice, slice_assign}, + ops::{numeric::empty_device_dtype, swap_dims}, + tensor::CubeTensor, +}; + +/// Calculate the convolution backward pass with regard to the weight gradients. +pub fn conv_weight_backward_fallback( + input: CubeTensor, + output_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, +) -> Result, ConvSetupError> { + match options.groups == 1 { + true => conv_weight_grad_no_groups::(input, output_grad, weight_shape, options), + false => conv_weight_grad_groups::(input, output_grad, weight_shape, options), + } +} + +fn conv_weight_grad_no_groups( + input: CubeTensor, + output_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, +) -> Result, ConvSetupError> { + let dim_c = input.rank() - 1; + + let input_swapped = swap_dims(input, 0, dim_c); + let out_grad_swapped = swap_dims(output_grad, 0, dim_c); + let weight_grad_swapped = conv_forward_nhwc( + input_swapped, + out_grad_swapped, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + Default::default(), + )?; + let mut weight_grad = swap_dims(weight_grad_swapped, 0, dim_c); + if weight_grad.shape() != weight_shape { + let ranges = weight_shape.iter().map(|&s| 0..s).collect::>(); + weight_grad = slice(weight_grad, &ranges); + } + + Ok(weight_grad) +} + +#[allow(clippy::single_range_in_vec_init, reason = "False positive")] +fn conv_weight_grad_groups( + input: CubeTensor, + output_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, +) -> Result, ConvSetupError> { + let mut weight_grad = empty_device_dtype( + input.client.clone(), + input.device.clone(), + weight_shape.clone(), + input.dtype, + ); + + let dim_c = input.rank() - 1; + + let channels_out = weight_shape[0]; + let increment_co = channels_out / options.groups; + + let input_swapped = swap_dims(input, 0, dim_c); + let output_grad_swapped = swap_dims(output_grad, 0, dim_c); + + let kernel_size = &weight_shape[1..dim_c]; + let kernel_size_slice = kernel_size.iter().map(|&s| 0..s).collect::>(); + let increment_ci = weight_grad.meta.shape()[dim_c]; + + for g in 0..options.groups { + let start_idx_ci = g * increment_ci; + let end_idx_ci = (g + 1) * increment_ci; + let start_idx_co = g * increment_co; + let end_idx_co = (g + 1) * increment_co; + + let input = slice(input_swapped.clone(), &[start_idx_ci..end_idx_ci]); + let grad = slice(output_grad_swapped.clone(), &[start_idx_co..end_idx_co]); + + let weight_grad_tmp = conv_forward_nhwc( + input, + grad, + None, + ConvOptions::new(options.dilation, options.padding, options.stride, 1), + Default::default(), + )?; + let mut weight_grad_tmp = swap_dims(weight_grad_tmp, 0, dim_c); + let kernel_size_tmp = &weight_grad_tmp.meta.shape()[1..dim_c]; + + if kernel_size != kernel_size_tmp { + let mut slices = vec![0..increment_co]; + slices.extend(kernel_size_slice.clone()); + slices.push(0..increment_ci); + weight_grad_tmp = slice(weight_grad_tmp, &slices); + } + + let mut slices = vec![start_idx_co..end_idx_co]; + slices.extend(kernel_size_slice.clone()); + slices.push(0..increment_ci); + let slices = slices.into_iter().map(Slice::from).collect::>(); + + weight_grad = slice_assign(weight_grad, &slices, weight_grad_tmp); + } + + Ok(weight_grad) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/launch.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/launch.rs new file mode 100644 index 0000000..9514e21 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/launch.rs @@ -0,0 +1,132 @@ +use burn_backend::ops::ConvOptions; +use burn_std::Shape; +use cubek::{ + convolution::{ + AcceleratedTileKind, ConvolutionArgs, ReadingStrategy, Strategy, backward_weight, + components::ConvSetupError, + }, + matmul::{ + definition::{MatmulElems, MatmulGlobalElems}, + launch::MatmulInputHandleRef, + }, +}; + +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; + +pub(crate) fn wgrad_gemm_simple_sync( + input: CubeTensor, + out_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + let read_strategy = match tile_kind { + AcceleratedTileKind::Cmma => ReadingStrategy::Cyclic, + AcceleratedTileKind::Mma => ReadingStrategy::Strided, + }; + launch_backwards_weight::( + &Strategy::Simple { + read_strategy, + tile_kind, + }, + input, + out_grad, + weight_shape, + options, + ) +} + +pub(crate) fn wgrad_gemm_simple_async( + input: CubeTensor, + out_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + let read_strategy = match tile_kind { + AcceleratedTileKind::Cmma => ReadingStrategy::AsyncCyclic, + AcceleratedTileKind::Mma => ReadingStrategy::AsyncStrided, + }; + launch_backwards_weight::( + &Strategy::Simple { + read_strategy, + tile_kind, + }, + input, + out_grad, + weight_shape, + options, + ) +} + +pub(crate) fn wgrad_gemm_simple_tma( + input: CubeTensor, + out_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + launch_backwards_weight::( + &Strategy::Simple { + read_strategy: ReadingStrategy::Tma, + tile_kind, + }, + input, + out_grad, + weight_shape, + options, + ) +} + +/// Perform a convolution backwards weight pass using the implicit GEMM (im2col) algorithm, using +/// cubecl tiling matmul components. +/// +/// * `input` - The input feature map +/// * `out_grad` - The output gradients +/// * `weight_shape` - The shape of the weights/weight gradients +/// * `options` - The options to use for the convolution +pub fn launch_backwards_weight( + strategy: &Strategy, + input: CubeTensor, + out_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, +) -> Result, ConvSetupError> { + if options.groups != 1 { + return Err(ConvSetupError::Groups(options.groups)); + } + + let out_dtype = out_grad.dtype; + + let weight_grad = empty_device_dtype( + input.client.clone(), + input.device.clone(), + weight_shape, + out_dtype, + ); + + let client = input.client.clone(); + let dtypes = MatmulElems::from_globals(&MatmulGlobalElems { + lhs: input.dtype.into(), + rhs: out_grad.dtype.into(), + out: out_dtype.into(), + }); + let input = MatmulInputHandleRef::new(input.as_handle_ref(), input.dtype.into()); + let out_grad = MatmulInputHandleRef::new(out_grad.as_handle_ref(), out_grad.dtype.into()); + + backward_weight::launch_ref::( + strategy, + &client, + &input, + &out_grad, + &weight_grad.as_handle_ref(), + ConvolutionArgs { + stride: options.stride, + padding: options.padding, + dilation: options.dilation, + }, + dtypes, + )?; + + Ok(weight_grad) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/mod.rs new file mode 100644 index 0000000..0df8c51 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/implicit_gemm/mod.rs @@ -0,0 +1,2 @@ +pub mod launch; +pub use launch::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/mod.rs new file mode 100644 index 0000000..80e94c4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/mod.rs @@ -0,0 +1,8 @@ +pub mod fallback; +pub mod implicit_gemm; + +#[cfg(feature = "autotune")] +pub mod tune; + +#[cfg(feature = "autotune")] +pub(crate) use tune::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/tune.rs new file mode 100644 index 0000000..f82f308 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/backward_weight/tune.rs @@ -0,0 +1,175 @@ +use burn_backend::ops::ConvOptions; +use burn_std::Shape; +use cubecl::{ + ir::StorageType, + tune::{LocalTuner, Tunable, TunableSet, anchor, local_tuner}, +}; +use cubek::convolution::AcceleratedTileKind; + +use crate::{ + CubeAutotuneKey, CubeRuntime, CubeTuneId, + kernel::conv::{ + ConvAutotuneKey, + backward_weight::{fallback::conv_weight_backward_fallback, implicit_gemm::*}, + }, + tensor::CubeTensor, +}; + +/// Executes autotune on the weight gradients pass for convolution +pub fn wgrad_autotune( + input: CubeTensor, + out_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, +) -> CubeTensor { + let client = input.client.clone(); + + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + TunableSet::new(create_key::, create_wgrad_input::) + .with(Tunable::new( + "wgrad_fallback", + conv_weight_backward_fallback::, + )) + .with(Tunable::new( + "simple_sync_cmma", + |input, grad, shape, options| { + wgrad_gemm_simple_sync(input, grad, shape, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_sync_mma", + |input, grad, shape, options| { + wgrad_gemm_simple_sync(input, grad, shape, options, AcceleratedTileKind::Mma) + }, + )) + .with(Tunable::new( + "simple_async_cmma", + |input, grad, shape, options| { + wgrad_gemm_simple_async(input, grad, shape, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_async_mma", + |input, grad, shape, options| { + wgrad_gemm_simple_async(input, grad, shape, options, AcceleratedTileKind::Mma) + }, + )) + .with(Tunable::new( + "simple_tma_cmma", + |input, grad, shape, options| { + wgrad_gemm_simple_tma(input, grad, shape, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_tma_mma", + |input, grad, shape, options| { + wgrad_gemm_simple_tma(input, grad, shape, options, AcceleratedTileKind::Mma) + }, + )) + }); + + TUNER.execute( + &CubeTuneId::new(&input.client, &input.device), + &client, + tunables, + (input, out_grad, weight_shape, options), + ) +} + +pub fn create_wgrad_input( + _key: &CubeAutotuneKey, + input: &CubeTensor, + out_grad: &CubeTensor, + weight_shape: &Shape, + options: &ConvOptions, +) -> (CubeTensor, CubeTensor, Shape, ConvOptions) { + ( + input.clone(), + out_grad.clone(), + weight_shape.clone(), + options.clone(), + ) +} + +fn create_key( + input: &CubeTensor, + out_grad: &CubeTensor, + weight_shape: &Shape, + options: &ConvOptions, +) -> CubeAutotuneKey { + let dtype = input.dtype; + let rank = input.meta.num_dims(); + let dim_c = rank - 1; + + let batch_size = input.meta.shape()[0]; + let in_channels = input.meta.shape()[dim_c]; + let out_channels = weight_shape[0]; + + let kernel_size = weight_shape[1..dim_c].to_vec(); + let in_shape = input.meta.shape()[1..dim_c] + .iter() + .map(|shape| anchor(*shape, None, None, None)) + .collect(); + + let ConvOptions { + stride, + padding, + dilation, + groups, + } = options.clone(); + + let lhs_stride_align = if out_grad.meta.strides()[dim_c] == 1 { + stride_align(out_grad.meta.strides(), out_grad.dtype.into()) + } else { + 0 + }; + let lhs_shape_align = pow2_factor(out_channels).min(lhs_stride_align); + let rhs_stride_align = if input.meta.strides()[dim_c] == 1 { + stride_align(input.meta.strides(), input.dtype.into()) + } else { + 0 + }; + let rhs_shape_align = pow2_factor(in_channels).min(rhs_stride_align); + + CubeAutotuneKey::Conv(ConvAutotuneKey::new( + kernel_size, + stride.to_vec(), + padding.to_vec(), + dilation.to_vec(), + groups, + in_channels, + out_channels, + in_shape, + batch_size, + false, + dtype, + lhs_shape_align, + lhs_stride_align, + rhs_shape_align, + rhs_stride_align, + )) +} + +/// Maximum factor relevant for strides. Currently set to 2^10 because that's 128-byte swizzle's +/// repeat number, so it's the largest align that can have performance impacts. +const MAX_STRIDE_FACTOR: u32 = 10; + +/// Defines the non-contiguous stride alignment in terms of powers of two +fn stride_align(strides: &[usize], elem: StorageType) -> u8 { + let max = MAX_STRIDE_FACTOR; + let dim_c = strides.len() - 1; + let factor = strides[..dim_c] + .iter() + .map(|it| (*it * elem.size_bits()) / 8) + .map(|it| it.trailing_zeros()) + .min() + .unwrap_or(max); + factor.min(max) as u8 +} + +/// Defines the potential vectorization. +fn pow2_factor(axis: usize) -> u8 { + axis.trailing_zeros().min(4) as u8 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/base.rs new file mode 100644 index 0000000..7d81082 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/base.rs @@ -0,0 +1,189 @@ +use burn_backend::ops::ConvOptions; +use burn_std::Shape; +use cubek::convolution::{AcceleratedTileKind, components::ConvSetupError}; + +#[cfg(feature = "autotune")] +use crate::kernel::conv::{backward_weight::wgrad_autotune, dgrad_autotune}; +use crate::{ + CubeRuntime, + kernel::conv::{ + backward_data::{fallback::conv_data_backward_fallback, implicit_gemm::*}, + backward_weight::{fallback::conv_weight_backward_fallback, implicit_gemm::*}, + forward::implicit_gemm::conv_gemm_simple_sync, + }, + ops::{permute_nchw_to_nhwc, permute_nchw_to_nhwc_shape, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; + +use super::conv_direct; +#[cfg(feature = "autotune")] +use super::forward::conv_autotune; + +/// The strategy to be used when launching a convolution kernel. +pub enum ConvStrategy { + /// A simple direct convolution. + Direct, + #[cfg(feature = "autotune")] + /// Using autotune to choose the best kernel based on runtime information. + Autotune, + /// Implicit GEMM implementation of convolution. Lower memory usage but requires CMMA and + /// has constraints on tensor shape. + ImplicitGemm, +} + +impl Default for ConvStrategy { + fn default() -> Self { + // if autotune is enabled, default to autotune + #[cfg(feature = "autotune")] + return ConvStrategy::Autotune; + + // if autotune is disabled, default to the more memory-conservative algorithm + #[cfg(not(feature = "autotune"))] + ConvStrategy::Direct + } +} + +/// Performs an N-dimensional convolution with the given strategy +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +/// * `strategy` - The convolution algorithm to use. Autotune will pick the fastest available option. +pub fn conv_forward( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, + strategy: ConvStrategy, +) -> Result, ConvSetupError> { + let input = permute_nchw_to_nhwc(input); + let weight = permute_nchw_to_nhwc(weight); + + let out = conv_forward_nhwc(input, weight, bias, options, strategy)?; + + Ok(permute_nhwc_to_nchw(out)) +} + +/// Performs an N-dimensional convolution with the given strategy on NHWC inputs/outputs +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +/// * `strategy` - The convolution algorithm to use. Autotune will pick the fastest available option. +pub fn conv_forward_nhwc( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, + strategy: ConvStrategy, +) -> Result, ConvSetupError> { + match strategy { + ConvStrategy::Direct => conv_direct::(input, weight, bias, options), + #[cfg(feature = "autotune")] + ConvStrategy::Autotune => Ok(conv_autotune::(input, weight, bias, options)), + ConvStrategy::ImplicitGemm => { + if options.groups != 1 { + conv_direct::(input, weight, bias, options) + } else { + conv_gemm_simple_sync::( + input, + weight, + bias, + options, + AcceleratedTileKind::Cmma, + ) + } + } + } +} + +/// Performs an N-dimensional convolution backwards pass with regard to weight, with the given strategy +/// +/// * `input` - The input feature map +/// * `out_grad` - The output gradients +/// * `weight_shape` - The shape of the weights/weight gradients +/// * `options` - The options used for the convolution +/// * `strategy` - The convolution algorithm to use. Autotune will pick the fastest available option. +pub fn conv_weight_backward( + input: CubeTensor, + out_grad: CubeTensor, + weight_shape: Shape, + options: ConvOptions, + strategy: ConvStrategy, +) -> Result, ConvSetupError> { + let input = permute_nchw_to_nhwc(input); + let out_grad = permute_nchw_to_nhwc(out_grad); + let weight_shape = permute_nchw_to_nhwc_shape(weight_shape); + + let weight_grad = match strategy { + ConvStrategy::Direct => { + conv_weight_backward_fallback::(input, out_grad, weight_shape, options) + } + #[cfg(feature = "autotune")] + ConvStrategy::Autotune => Ok(wgrad_autotune::( + input, + out_grad, + weight_shape, + options, + )), + ConvStrategy::ImplicitGemm => { + if options.groups != 1 { + conv_weight_backward_fallback::(input, out_grad, weight_shape, options) + } else { + wgrad_gemm_simple_sync::( + input, + out_grad, + weight_shape, + options, + AcceleratedTileKind::Cmma, + ) + } + } + }?; + + Ok(permute_nhwc_to_nchw(weight_grad)) +} + +/// Performs an N-dimensional convolution backwards data pass with the given strategy +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `in_shape` - The shape of the input to the layer +/// * `options` - The options to use for the convolution +/// * `strategy` - The convolution algorithm to use. Autotune will pick the fastest available option. +pub fn conv_data_backward( + out_grad: CubeTensor, + weights: CubeTensor, + in_shape: Shape, + options: ConvOptions, + strategy: ConvStrategy, +) -> Result, ConvSetupError> { + let out_grad = permute_nchw_to_nhwc(out_grad); + let weights = permute_nchw_to_nhwc(weights); + let in_shape = permute_nchw_to_nhwc_shape(in_shape); + + let weight_grad = match strategy { + ConvStrategy::Direct => { + conv_data_backward_fallback::(out_grad, weights, in_shape, options)? + } + #[cfg(feature = "autotune")] + ConvStrategy::Autotune => dgrad_autotune::(out_grad, weights, in_shape, options), + ConvStrategy::ImplicitGemm => { + if options.groups != 1 || options.stride.iter().any(|&s| s != 1) { + conv_data_backward_fallback::(out_grad, weights, in_shape, options)? + } else { + dgrad_gemm_simple_sync::( + out_grad, + weights, + in_shape, + options, + AcceleratedTileKind::Cmma, + )? + } + } + }; + + Ok(permute_nhwc_to_nchw(weight_grad)) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/base.rs new file mode 100644 index 0000000..5fa9ed2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/base.rs @@ -0,0 +1,54 @@ +use crate::{CubeRuntime, tensor::CubeTensor}; +use burn_backend::ops::ConvTransposeOptions; +use cubek::convolution::components::ConvSetupError; + +#[cfg(feature = "autotune")] +use super::conv_transpose2d_autotune; +use super::{conv_transpose2d_col2im, conv_transpose2d_direct}; + +/// The strategy to be used when launching a conv_transpose kernel. +pub enum ConvTranspose2dStrategy { + /// A simple direct convolution. + Direct, + #[cfg(feature = "autotune")] + /// Using autotune to choose the best kernel based on runtime information. + Autotune, + /// GEMM (im2col) based implementation of convolution. Significantly increased memory usage. + Gemm, +} + +impl Default for ConvTranspose2dStrategy { + fn default() -> Self { + // if autotune is enabled, default to autotune + #[cfg(feature = "autotune")] + return ConvTranspose2dStrategy::Autotune; + + // if autotune is disabled, default to the more memory-conservative algorithm + #[cfg(not(feature = "autotune"))] + ConvTranspose2dStrategy::Direct + } +} + +/// Performs a 2D convolution with the given strategy +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +/// * `strategy` - The convolution algorithm to use. Autotune will pick the fastest available option. +pub fn conv_transpose2d( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + strategy: ConvTranspose2dStrategy, +) -> Result, ConvSetupError> { + match strategy { + ConvTranspose2dStrategy::Direct => conv_transpose2d_direct(input, weight, bias, options), + #[cfg(feature = "autotune")] + ConvTranspose2dStrategy::Autotune => { + Ok(conv_transpose2d_autotune(input, weight, bias, options)) + } + ConvTranspose2dStrategy::Gemm => conv_transpose2d_col2im(input, weight, bias, options), + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/col2im.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/col2im.rs new file mode 100644 index 0000000..99b3b47 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/col2im.rs @@ -0,0 +1,302 @@ +use crate::{ + CubeRuntime, + kernel::{ + conv::batches_per_run, + into_contiguous_aligned, + matmul::{MatmulStrategy, matmul}, + slice, + utils::{address_type, decompose_linear, linear_view, shape_divmod}, + }, + ops::{numeric::empty_device_dtype, reshape, swap_dims}, + tensor::CubeTensor, +}; +use burn_backend::{ + Shape, + ops::{ConvTransposeOptions, conv::calculate_conv_transpose_output_size}, +}; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::layout::linear::LinearView}, +}; +use cubek::convolution::components::ConvSetupError; + +/// Perform a 2D convolution transposition using the GEMM (col2im) algorithm. +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +pub fn conv_transpose2d_col2im( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvTransposeOptions<2>, +) -> Result, ConvSetupError> { + let [input_channels, im_ch_per_group, kernel_h, kernel_w] = weight.meta.shape().dims(); + let [batch_size, _, input_h, input_w] = input.meta.shape().dims(); + let groups = options.groups; + let input_ch_per_group = input_channels / groups; + let ConvTransposeOptions { + padding: [padding_h, padding_w], + padding_out: [padding_out_h, padding_out_w], + dilation: [dilation_h, dilation_w], + stride: [stride_h, stride_w], + .. + } = options.clone(); + + let im_h = calculate_conv_transpose_output_size( + kernel_h, + stride_h, + padding_h, + padding_out_h, + dilation_h, + input_h, + ); + let im_w = calculate_conv_transpose_output_size( + kernel_w, + stride_w, + padding_w, + padding_out_w, + dilation_w, + input_w, + ); + let im_channels = im_ch_per_group * groups; + + let batches_per_run = batches_per_run( + batch_size, + input_h * input_w, + input.client.properties().hardware.plane_size_max as usize, + )?; + let col_shape_0 = im_ch_per_group * kernel_h * kernel_w; + + let weight = reshape( + weight.clone(), + Shape::new([groups, input_ch_per_group, col_shape_0]), + ); + let weight = into_contiguous_aligned(swap_dims(weight, 1, 2)); + + if batches_per_run != batch_size { + let runs = batch_size / batches_per_run; + + let im_shape = Shape::new([runs, batches_per_run, im_channels, im_h, im_w]); + let image = empty_device_dtype( + input.client.clone(), + input.device.clone(), + im_shape, + input.dtype, + ); + + let input_shape = Shape::new([runs, batches_per_run, input_channels, input_h, input_w]); + let input = reshape(input, input_shape); + let input_shape_run = Shape::new([batches_per_run, input_channels, input_h, input_w]); + + for run in 0..runs { + let input = index(input.clone(), run); + let input = reshape(input, input_shape_run.clone()); + let im_shape = Shape::new([batches_per_run, im_channels, im_h, im_w]); + let image_slice = index(image.clone(), run); + let image_slice = reshape(image_slice, im_shape); + execute( + input, + weight.clone(), + bias.clone(), + image_slice, + options.clone(), + kernel_h, + kernel_w, + )?; + } + Ok(reshape( + image, + Shape::new([batch_size, im_channels, im_h, im_w]), + )) + } else { + let im_shape = Shape::new([batches_per_run, im_channels, im_h, im_w]); + let image = empty_device_dtype( + input.client.clone(), + input.device.clone(), + im_shape, + input.dtype, + ); + execute( + input, + weight, + bias, + image.clone(), + options, + kernel_h, + kernel_w, + )?; + Ok(image) + } +} + +pub(crate) fn index(tensor: CubeTensor, i: usize) -> CubeTensor { + #[allow(clippy::single_range_in_vec_init)] + let mut indices = vec![i..i + 1]; + for dim in tensor.meta.shape()[1..].iter() { + indices.push(0..*dim); + } + let mut tensor = slice(tensor, &indices); + tensor.meta.remove(0); + tensor +} + +#[allow(clippy::too_many_arguments)] +fn execute( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + image: CubeTensor, + options: ConvTransposeOptions<2>, + kernel_h: usize, + kernel_w: usize, +) -> Result<(), ConvSetupError> { + let [batch_size, _, input_h, input_w] = input.meta.shape().dims(); + let [groups, col_shape_0, input_ch_per_group] = weight.meta.shape().dims(); + + let col_shape_1 = batch_size * input_h * input_w; + + let input = swap_dims(input, 0, 1); + let input_shape = Shape::new([groups, input_ch_per_group, col_shape_1]); + let input = reshape(input, input_shape); + + let dtype = input.dtype; + let columns = matmul(weight, input, None, MatmulStrategy::default(), dtype)?; + let columns = reshape(columns, Shape::new([col_shape_0 * groups, col_shape_1])); + + col2im( + columns, bias, image, kernel_h, kernel_w, input_h, input_w, options, + )?; + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn col2im( + columns: CubeTensor, + bias: Option>, + out: CubeTensor, + kernel_h: usize, + kernel_w: usize, + out_h: usize, + out_w: usize, + options: ConvTransposeOptions<2>, +) -> Result<(), LaunchError> { + let dtype = columns.dtype; + + let columns = into_contiguous_aligned(columns); + let bias = bias.map(into_contiguous_aligned); + + let num_elems = out.meta.num_elements(); + + let cube_dim = CubeDim::new(&columns.client, num_elems); + let cube_count = calculate_cube_count_elemwise(&columns.client, num_elems, cube_dim); + + unsafe { + col2im_kernel::launch_unchecked( + &columns.client, + cube_count, + cube_dim, + address_type!(columns, bias, out), + columns.as_tensor_arg(1), + bias.as_ref().map(|bias| bias.as_tensor_arg(1)).into(), + linear_view(&out, 1), + shape_divmod(&out), + Col2ImArgsLaunch::new( + ScalarArg::new(out_h), + ScalarArg::new(out_w), + ScalarArg::new(kernel_h), + ScalarArg::new(kernel_w), + ScalarArg::new(options.padding[0]), + ScalarArg::new(options.padding[1]), + ScalarArg::new(options.dilation[0]), + ScalarArg::new(options.dilation[1]), + ScalarArg::new(options.stride[0]), + ScalarArg::new(options.stride[1]), + ), + dtype.into(), + ) + } +} + +#[derive(CubeLaunch, CubeType)] +struct Col2ImArgs { + out_h: usize, + out_w: usize, + + kernel_h: usize, + kernel_w: usize, + + pad_h: usize, + pad_w: usize, + dilation_h: usize, + dilation_w: usize, + stride_h: usize, + stride_w: usize, +} + +#[cube(launch_unchecked, address_type = "dynamic")] +fn col2im_kernel( + columns: &Tensor, + bias: &Option>, + image: &mut LinearView, + image_shape: Sequence>, + args: &Col2ImArgs, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= image.shape() { + terminate!(); + } + + let (_, pos) = decompose_linear(ABSOLUTE_POS, &image_shape); + let [batch, ch_im, im_y, im_x] = *pos else { + unreachable!() + }; + + let im_x = im_x + args.pad_w; + let im_y = im_y + args.pad_h; + + let kernel_extent_w = (args.kernel_w - 1) * args.dilation_w + 1; + let kernel_extent_h = (args.kernel_h - 1) * args.dilation_h + 1; + + let mut val = E::from_int(0); + + let x_col_start = if im_x >= kernel_extent_w { + (im_x - kernel_extent_w) / args.stride_w + 1 + } else { + 0usize.runtime() + }; + let x_col_end = clamp_max(im_x / args.stride_w + 1, args.out_w); + let y_col_start = if im_y >= kernel_extent_h { + (im_y - kernel_extent_h) / args.stride_h + 1 + } else { + 0usize.runtime() + }; + let y_col_end = clamp_max(im_y / args.stride_h + 1, args.out_h); + + for col_y in y_col_start..y_col_end { + let kernel_y = im_y - col_y * args.stride_h; + for col_x in x_col_start..x_col_end { + let kernel_x = im_x - col_x * args.stride_w; + + if kernel_y.is_multiple_of(args.dilation_h) && kernel_x.is_multiple_of(args.dilation_w) + { + let kernel_y = kernel_y / args.dilation_h; + let kernel_x = kernel_x / args.dilation_w; + + let col_k = + ch_im * args.kernel_h * args.kernel_w + kernel_y * args.kernel_w + kernel_x; + let col_n = batch * args.out_h * args.out_w + col_y * args.out_w + col_x; + let col_pos = col_k * columns.stride(0) + col_n * columns.stride(1); + val += columns[col_pos]; + } + } + } + + match bias { + Some(bias) => image[ABSOLUTE_POS] = val + bias[ch_im], + None => image[ABSOLUTE_POS] = val, + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/mod.rs new file mode 100644 index 0000000..214333f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/mod.rs @@ -0,0 +1,15 @@ +mod base; +mod col2im; + +mod transpose_direct; + +#[cfg(feature = "autotune")] +mod tune; + +pub use base::*; +pub use col2im::*; + +pub use transpose_direct::*; + +#[cfg(feature = "autotune")] +pub use tune::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/transpose_direct.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/transpose_direct.rs new file mode 100644 index 0000000..eb1bf4c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/transpose_direct.rs @@ -0,0 +1,185 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, decompose_linear, linear_view, shape_divmod}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use burn_backend::{Shape, ops::ConvTransposeOptions}; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::layout::linear::LinearView}, +}; +use cubek::convolution::components::ConvSetupError; + +#[derive(CubeLaunch, CubeType)] +struct ConvArgs { + conv_stride_0: usize, + conv_stride_1: usize, + dilation_0: usize, + dilation_1: usize, + padding_0: usize, + padding_1: usize, + groups: usize, +} + +#[cube(launch, address_type = "dynamic")] +fn conv_transpose2d_direct_kernel( + input: &Tensor, + weight: &Tensor, + bias: &Option>, + output: &mut LinearView, + out_shape: Sequence>, + args: ConvArgs, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= output.shape() { + terminate!(); + } + + let in_c_per_group = weight.shape(0) / args.groups; + let out_c_per_group = weight.shape(1); + let kernel_h = weight.shape(2); + let kernel_w = weight.shape(3); + + let (_, pos) = decompose_linear(ABSOLUTE_POS, &out_shape); + let [batch, oc_out, out_y, out_x] = *pos else { + unreachable!() + }; + + let k = oc_out / out_c_per_group; + let group = k % args.groups; + let out_c = oc_out - out_c_per_group * group; + + let in_c_start = group * in_c_per_group; + let in_c_end = in_c_start + in_c_per_group; + + let stride_0_i = args.conv_stride_0 as i32; + let stride_1_i = args.conv_stride_1 as i32; + + let kms_h = (kernel_h * args.dilation_0) as i32 - stride_0_i; + let kms_w = (kernel_w * args.dilation_1) as i32 - stride_1_i; + + let y_start = ((out_y + args.padding_0) as i32 - kms_h) / stride_0_i; + let x_start = ((out_x + args.padding_1) as i32 - kms_w) / stride_1_i; + + let y_end = clamp(kms_h + y_start + 1, 0, input.shape(2) as i32) as usize; + let x_end = clamp(kms_w + x_start + 1, 0, input.shape(3) as i32) as usize; + let y_start = clamp_min(y_start, 0) as usize; + let x_start = clamp_min(x_start, 0) as usize; + + let idx_input_batch = batch * input.stride(0); + let idx_weight_oc = out_c * weight.stride(1); + + let bias: Option = bias.map(|bias| bias[oc_out]); + let mut sum = bias.unwrap_or_default(); + + let numerator_h_base = out_y + args.padding_0; + let numerator_w_base = out_x + args.padding_1; + + for in_c in in_c_start..in_c_end { + let idx_input_ic = in_c * input.stride(1); + let idx_weight_ic = in_c * weight.stride(0); + + for in_y in y_start..y_end { + let numerator_tmp = in_y * args.conv_stride_0; + let numerator_h = numerator_h_base - numerator_tmp; + + if numerator_h_base >= numerator_tmp && numerator_h.is_multiple_of(args.dilation_0) { + let kernel_y = numerator_h / args.dilation_0; + let idx_input_y = in_y * input.stride(2); + let idx_weight_ky = kernel_y * weight.stride(2); + + for in_x in x_start..x_end { + let numerator_tmp = in_x * args.conv_stride_1; + let numerator_w = numerator_w_base - numerator_tmp; + + if numerator_w_base >= numerator_tmp + && numerator_w.is_multiple_of(args.dilation_1) + { + let kernel_x = numerator_w / args.dilation_1; + let idx_input_x = in_x * input.stride(3); + let idx_weight_kx = kernel_x * weight.stride(3); + + let index_input = + idx_input_batch + idx_input_ic + idx_input_y + idx_input_x; + let index_weight = + idx_weight_ic + idx_weight_oc + idx_weight_ky + idx_weight_kx; + + let value = input[index_input]; + let weight = weight[index_weight]; + + sum += value * weight; + } + } + } + } + } + + output[ABSOLUTE_POS] = sum; +} + +/// Perform a 2D convolution transposition using the direct algorithm. +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +/// +pub fn conv_transpose2d_direct( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvTransposeOptions<2>, +) -> Result, ConvSetupError> { + let [batch_size, _, in_height, in_width] = input.meta.shape().dims(); + let [_, out_channels, kernel_0, kernel_1] = weight.meta.shape().dims(); + + let out_0 = (in_height - 1) * options.stride[0] + + options.dilation[0] * (kernel_0 - 1) + + options.padding_out[0] + - 2 * options.padding[0] + + 1; + let out_1 = (in_width - 1) * options.stride[1] + + options.dilation[1] * (kernel_1 - 1) + + options.padding_out[1] + - 2 * options.padding[1] + + 1; + + let shape_out = Shape::new([batch_size, out_channels * options.groups, out_0, out_1]); + + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + shape_out.clone(), + input.dtype, + ); + + let num_elems = output.meta.num_elements(); + let cube_dim = CubeDim::new(&input.client, num_elems); + let cube_count = calculate_cube_count_elemwise(&input.client, num_elems, cube_dim); + + conv_transpose2d_direct_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, weight, bias, output), + input.as_tensor_arg(1), + weight.as_tensor_arg(1), + bias.as_ref().map(|bias| bias.as_tensor_arg(1)).into(), + linear_view(&output, 1), + shape_divmod(&output), + ConvArgsLaunch::new( + ScalarArg::new(options.stride[0]), + ScalarArg::new(options.stride[1]), + ScalarArg::new(options.dilation[0]), + ScalarArg::new(options.dilation[1]), + ScalarArg::new(options.padding[0]), + ScalarArg::new(options.padding[1]), + ScalarArg::new(options.groups), + ), + input.dtype.into(), + )?; + + Ok(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/tune.rs new file mode 100644 index 0000000..ef832ea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose2d/tune.rs @@ -0,0 +1,91 @@ +use burn_backend::ops::ConvTransposeOptions; +use cubecl::tune::{LocalTuner, Tunable, TunableSet, local_tuner}; + +use crate::{ + CubeAutotuneKey, CubeRuntime, CubeTuneId, + kernel::conv::{ConvTranspose2dAutotuneKey, conv_transpose2d_col2im, conv_transpose2d_direct}, + tensor::CubeTensor, +}; + +/// Executes autotune on conv2d operations +pub fn conv_transpose2d_autotune( + input: CubeTensor, + weights: CubeTensor, + bias: Option>, + options: ConvTransposeOptions<2>, +) -> CubeTensor { + let client = input.client.clone(); + + static TUNER: LocalTuner = local_tuner!(); + + let tune_set = TUNER.init(|| { + TunableSet::new(create_key::, create_transpose2d_input::) + .with(Tunable::new( + "conv_transpose2d_direct", + conv_transpose2d_direct::, + )) + .with(Tunable::new( + "conv_transpose2d_col2im", + conv_transpose2d_col2im::, + )) + }); + + TUNER.execute( + &CubeTuneId::new(&input.client, &input.device), + &client, + tune_set, + (input, weights, bias, options), + ) +} + +pub fn create_transpose2d_input( + _key: &CubeAutotuneKey, + input: &CubeTensor, + weights: &CubeTensor, + bias: &Option>, + options: &ConvTransposeOptions<2>, +) -> ( + CubeTensor, + CubeTensor, + Option>, + ConvTransposeOptions<2>, +) { + ( + input.clone(), + weights.clone(), + bias.clone(), + options.clone(), + ) +} + +fn create_key( + input: &CubeTensor, + weights: &CubeTensor, + bias: &Option>, + options: &ConvTransposeOptions<2>, +) -> CubeAutotuneKey { + let [batch_size, in_channels, height, width] = input.meta.shape().dims(); + let [out_channels, _, kernel_h, kernel_w] = weights.meta.shape().dims(); + let ConvTransposeOptions { + stride, + padding, + dilation, + groups, + padding_out, + } = options.clone(); + CubeAutotuneKey::ConvTranspose(ConvTranspose2dAutotuneKey::new( + [kernel_h, kernel_w], + stride, + padding, + padding_out, + dilation, + groups, + in_channels, + out_channels, + height, + width, + batch_size, + bias.is_some(), + input.dtype, + )) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose3d.rs new file mode 100644 index 0000000..6df5864 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/conv_transpose3d.rs @@ -0,0 +1,222 @@ +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::layout::linear::LinearView}, +}; + +use crate::{ + CubeRuntime, + kernel::utils::{address_type, decompose_linear, linear_view, shape_divmod}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use burn_backend::{Shape, ops::ConvTransposeOptions}; + +#[derive(CubeLaunch, CubeType)] +struct ConvArgs { + conv_stride_0: usize, + conv_stride_1: usize, + conv_stride_2: usize, + dilation_0: usize, + dilation_1: usize, + dilation_2: usize, + padding_0: usize, + padding_1: usize, + padding_2: usize, + groups: usize, +} + +#[cube(launch, address_type = "dynamic")] +fn conv_transpose3d_kernel( + input: &Tensor, + weight: &Tensor, + bias: &Option>, + output: &mut LinearView, + out_shape: Sequence>, + args: ConvArgs, + #[define(E)] _dtype: StorageType, +) { + let in_channels = weight.shape(0); + let out_c_per_group = weight.shape(1); + let kernel_size_0 = weight.shape(2); + let kernel_size_1 = weight.shape(3); + let kernel_size_2 = weight.shape(4); + + let stride_0_i = args.conv_stride_0 as i32; + let stride_1_i = args.conv_stride_1 as i32; + let stride_2_i = args.conv_stride_2 as i32; + + let (_, pos) = decompose_linear(ABSOLUTE_POS, &out_shape); + let [batch, out_c_out, out_z, out_y, out_x] = *pos else { + unreachable!() + }; + + let groups = args.groups; + let in_c_per_group = in_channels / groups; + + let k = out_c_out / out_c_per_group; + let group = k % groups; + let out_channel = out_c_out - out_c_per_group * group; + + let in_c_start = group * in_c_per_group; + let in_c_end = in_c_start + in_c_per_group; + + let kernel_d = (kernel_size_0 * args.dilation_0 - args.conv_stride_0) as i32; + let kernel_h = (kernel_size_1 * args.dilation_1 - args.conv_stride_1) as i32; + let kernel_w = (kernel_size_2 * args.dilation_2 - args.conv_stride_2) as i32; + + let z_start = ((out_z + args.padding_0) as i32 - kernel_d) / stride_0_i; + let y_start = ((out_y + args.padding_1) as i32 - kernel_h) / stride_1_i; + let x_start = ((out_x + args.padding_2) as i32 - kernel_w) / stride_2_i; + + let z_end = clamp(kernel_d + z_start + 1, 0, input.shape(2) as i32) as usize; + let y_end = clamp(kernel_h + y_start + 1, 0, input.shape(3) as i32) as usize; + let x_end = clamp(kernel_w + x_start + 1, 0, input.shape(4) as i32) as usize; + + let z_start = clamp_min(z_start, 0) as usize; + let y_start = clamp_min(y_start, 0) as usize; + let x_start = clamp_min(x_start, 0) as usize; + + let index_input_batch = batch * input.stride(0); + let index_weight_out_c = out_channel * weight.stride(1); + + let bias: Option = bias.map(|bias| bias[out_c_out]); + let mut sum = bias.unwrap_or_default(); + + let numerator_d_base = out_z + args.padding_0; + let numerator_h_base = out_y + args.padding_1; + let numerator_w_base = out_x + args.padding_2; + + for in_c in in_c_start..in_c_end { + let index_input_in_c = in_c * input.stride(1); + let index_weight_in_c = in_c * weight.stride(0); + + for in_z in z_start..z_end { + let numerator_tmp = in_z * args.conv_stride_0; + let numerator_d = numerator_d_base - numerator_tmp; + + if numerator_d_base >= numerator_tmp && numerator_d.is_multiple_of(args.dilation_0) { + let kernel_z = numerator_d / args.dilation_0; + let index_input_z = in_z * input.stride(2); + let index_weight_kz = kernel_z * weight.stride(2); + + for in_y in y_start..y_end { + let numerator_tmp = in_y * args.conv_stride_1; + let numerator_h = numerator_h_base - numerator_tmp; + + if numerator_h_base >= numerator_tmp + && numerator_h.is_multiple_of(args.dilation_1) + { + let kernel_y = numerator_h / args.dilation_1; + let index_input_y = in_y * input.stride(3); + let index_weight_ky = kernel_y * weight.stride(3); + + for in_x in x_start..x_end { + let numerator_tmp = in_x * args.conv_stride_2; + let numerator_w = numerator_w_base - numerator_tmp; + + if numerator_w_base >= numerator_tmp + && numerator_w.is_multiple_of(args.dilation_2) + { + let kernel_x = numerator_w / args.dilation_2; + let index_input_x = in_x * input.stride(4); + let index_weight_kx = kernel_x * weight.stride(4); + + let index_input = index_input_batch + + index_input_in_c + + index_input_z + + index_input_y + + index_input_x; + + let index_weight = index_weight_in_c + + index_weight_out_c + + index_weight_kz + + index_weight_ky + + index_weight_kx; + + let value = input[index_input]; + let weight = weight[index_weight]; + + sum += value * weight; + } + } + } + } + } + } + } + + output[ABSOLUTE_POS] = sum; +} + +pub(crate) fn conv_transpose3d( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvTransposeOptions<3>, +) -> Result, LaunchError> { + let [batch_size, _, in_depth, in_height, in_width] = input.meta.shape().dims(); + let [_, out_channels, kernel_0, kernel_1, kernel_2] = weight.meta.shape().dims(); + + let out_0 = (in_depth - 1) * options.stride[0] + + options.dilation[0] * (kernel_0 - 1) + + options.padding_out[0] + - 2 * options.padding[0] + + 1; + let out_1 = (in_height - 1) * options.stride[1] + + options.dilation[1] * (kernel_1 - 1) + + options.padding_out[1] + - 2 * options.padding[1] + + 1; + let out_2 = (in_width - 1) * options.stride[2] + + options.dilation[2] * (kernel_2 - 1) + + options.padding_out[2] + - 2 * options.padding[2] + + 1; + + let shape_out = Shape::new([ + batch_size, + out_channels * options.groups, + out_0, + out_1, + out_2, + ]); + + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + shape_out.clone(), + input.dtype, + ); + + let num_elems = output.meta.num_elements(); + let cube_dim = CubeDim::new(&input.client, num_elems); + let cube_count = calculate_cube_count_elemwise(&input.client, num_elems, cube_dim); + + conv_transpose3d_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, weight, bias, output), + input.as_tensor_arg(1), + weight.as_tensor_arg(1), + bias.as_ref().map(|bias| bias.as_tensor_arg(1)).into(), + linear_view(&output, 1), + shape_divmod(&output), + ConvArgsLaunch::new( + ScalarArg::new(options.stride[0]), + ScalarArg::new(options.stride[1]), + ScalarArg::new(options.stride[2]), + ScalarArg::new(options.dilation[0]), + ScalarArg::new(options.dilation[1]), + ScalarArg::new(options.dilation[2]), + ScalarArg::new(options.padding[0]), + ScalarArg::new(options.padding[1]), + ScalarArg::new(options.padding[2]), + ScalarArg::new(options.groups), + ), + input.dtype.into(), + )?; + + Ok(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv2d.rs new file mode 100644 index 0000000..2f9f07d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv2d.rs @@ -0,0 +1,314 @@ +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, FastDivmodArgs}, +}; +use cubek::convolution::components::ConvSetupError; + +use burn_backend::{ + Shape, + ops::{DeformConvOptions, conv::calculate_conv_output_size}, +}; + +use crate::{ + CubeRuntime, + kernel::{ + AddOp, into_contiguous_aligned, launch_binop, + matmul::{MatmulStrategy, matmul}, + utils::address_type, + }, + ops::{numeric::zeros_client, reshape, swap_dims}, + tensor::CubeTensor, +}; + +#[derive(CubeLaunch, CubeType)] +struct DeformConv2dArgs { + conv_stride_h: usize, + conv_stride_w: usize, + dilation_h: usize, + dilation_w: usize, + padding_h: InputScalar, + padding_w: InputScalar, + offset_groups: usize, + + kernel_height: usize, + kernel_width: usize, + out_h: usize, + out_w: usize, +} + +#[cube(launch, address_type = "dynamic")] +fn deform_im2col_kernel( + input: &Tensor, + offset: &Tensor, + mask: &Option>, + columns: &mut Tensor, + pos_shape: Sequence>, + args: &DeformConv2dArgs, + #[comptime] kernel_h_unroll: Option, + #[comptime] kernel_w_unroll: Option, + #[define(F)] _dtype: StorageType, +) { + // position shape: [in_channels, batch_size, out_h, out_w] + // columns shape: [[in_channels, kernel_h, kernel_w], [batch_size, out_h, out_w]] + + let kernel_height = kernel_h_unroll.unwrap_or(args.kernel_height); + let unroll_h = kernel_h_unroll.is_some(); + let kernel_width = kernel_w_unroll.unwrap_or(args.kernel_width); + let unroll_w = kernel_w_unroll.is_some(); + + let out_h = args.out_h; + let out_w = args.out_w; + let in_channels = input.shape(1); + let height = input.shape(2); + let width = input.shape(3); + let col_stride_0 = columns.stride(0); + + let (rem, out_x) = pos_shape[3].div_mod(ABSOLUTE_POS); + let (rem, out_y) = pos_shape[2].div_mod(rem); + let (in_channel, batch) = pos_shape[1].div_mod(rem); + + if in_channel >= in_channels { + terminate!() + } + + let out_k_base = in_channel * kernel_height * kernel_width; + let out_n = batch * out_h * out_w + out_y * out_w + out_x; + + let channels_per_offset_group = in_channels / args.offset_groups; + let group_index = in_channel / channels_per_offset_group; + + let mut col_base_idx = out_k_base * columns.stride(0) + out_n * columns.stride(1); + + let input_base_idx = batch * input.stride(0) + in_channel * input.stride(1); + + let offset_base_idx = batch * offset.stride(0) + + group_index * kernel_height * kernel_width * 2 * offset.stride(1); + + let mask_base_idx = mask.as_ref().map(|mask| { + batch * mask.stride(0) + group_index * kernel_height * kernel_width * mask.stride(1) + }); + + #[unroll(unroll_h)] + for kernel_y in 0..kernel_height { + #[unroll(unroll_w)] + for kernel_x in 0..kernel_width { + let mask_index = kernel_y * kernel_width + kernel_x; + let offset_index = mask_index * 2; + + let offset_y = offset[offset_base_idx + + offset_index * offset.stride(1) + + out_y * offset.stride(2) + + out_x * offset.stride(3)]; + let offset_x = offset[offset_base_idx + + (offset_index + 1) * offset.stride(1) + + out_y * offset.stride(2) + + out_x * offset.stride(3)]; + let y = F::cast_from(out_y * args.conv_stride_h + kernel_y * args.dilation_h) + - args.padding_h.get::() + + offset_y; + let x = F::cast_from(out_x * args.conv_stride_w + kernel_x * args.dilation_w) + - args.padding_w.get::() + + offset_x; + + let interpolated = bilinear_interpolate(input, height, width, y, x, input_base_idx); + let value = match mask.zip::(mask_base_idx) { + Some((mask, base_idx)) => { + let mask_value = mask[base_idx + + mask_index * mask.stride(1) + + out_y * mask.stride(2) + + out_x * mask.stride(3)]; + mask_value * interpolated + } + None => interpolated, + }; + + columns[col_base_idx] = value; + col_base_idx += col_stride_0; + } + } +} + +#[cube] +pub(crate) fn bilinear_interpolate( + input: &Tensor, + height: usize, + width: usize, + y: F, + x: F, + offset: usize, +) -> F { + // To simplify code + let y = f32::cast_from(y); + let x = f32::cast_from(x); + let stride_y = input.stride(2); + let stride_x = input.stride(3); + + let mut result = F::new(0.0); + if y > -1.0 && height as f32 > y && x > -1.0 && width as f32 > x { + let y_low = y.floor(); + let x_low = x.floor(); + let y_high = (y_low + 1.) as usize; + let x_high = (x_low + 1.) as usize; + + let zero = F::new(0.0); + let v1: F = if y_low >= 0. && x_low >= 0. { + input[offset + y_low as usize * stride_y + x_low as usize * stride_x] + } else { + zero + }; + let v2: F = if y_low >= 0. && x_high < width { + input[offset + y_low as usize * stride_y + x_high * stride_x] + } else { + zero + }; + let v3: F = if y_high < height && x_low >= 0. { + input[offset + y_high * stride_y + x_low as usize * stride_x] + } else { + zero + }; + let v4: F = if y_high < height && x_high < width { + input[offset + y_high * stride_y + x_high * stride_x] + } else { + zero + }; + + let l_y = y - y_low; + let l_x = x - x_low; + let h_y = 1.0 - l_y; + let h_x = 1.0 - l_x; + + let w1 = F::cast_from(h_y * h_x); + let w2 = F::cast_from(h_y * l_x); + let w3 = F::cast_from(l_y * h_x); + let w4 = F::cast_from(l_y * l_x); + + result = w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4; + } + result +} + +pub(crate) fn deform_im2col( + input: CubeTensor, + offset: CubeTensor, + mask: Option>, + options: DeformConvOptions<2>, + out_dims: (usize, usize), + kernel_dims: (usize, usize), +) -> Result, LaunchError> { + let client = input.client.clone(); + let device = input.device.clone(); + let dtype = input.dtype; + + let [batch_size, in_channels, _, _] = input.meta.shape().dims(); + let (out_height, out_width) = out_dims; + let (kernel_height, kernel_width) = kernel_dims; + + let shape_out = Shape::new([ + in_channels * kernel_height * kernel_width, + batch_size * out_height * out_width, + ]); + + let pos_shape = [in_channels, batch_size, out_height, out_width] + .into_iter() + .map(|s| FastDivmodArgs::new(&client, s)) + .collect(); + + let output = zeros_client(client.clone(), device.clone(), shape_out.clone(), dtype); + + let num_kernels = in_channels * batch_size * out_height * out_width; + let cube_dim = CubeDim::new(&input.client, num_kernels); + let cube_count = calculate_cube_count_elemwise(&input.client, num_kernels, cube_dim); + + deform_im2col_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, offset, mask, output), + input.as_tensor_arg(1), + offset.as_tensor_arg(1), + mask.as_ref().map(|mask| mask.as_tensor_arg(1)).into(), + output.as_handle_ref().as_tensor_arg(1), + pos_shape, + DeformConv2dArgsLaunch::new( + ScalarArg::new(options.stride[0]), + ScalarArg::new(options.stride[1]), + ScalarArg::new(options.dilation[0]), + ScalarArg::new(options.dilation[1]), + { + let val = options.padding[0] as f32; + InputScalar::new(val, dtype) + }, + { + let val = options.padding[1] as f32; + InputScalar::new(val, dtype) + }, + ScalarArg::new(options.offset_groups), + ScalarArg::new(kernel_height), + ScalarArg::new(kernel_width), + ScalarArg::new(out_height), + ScalarArg::new(out_width), + ), + Some(kernel_height), + Some(kernel_width), + dtype.into(), + )?; + + Ok(output) +} + +pub(crate) fn deform_conv2d( + input: CubeTensor, + offset: CubeTensor, + weight: CubeTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, +) -> Result, ConvSetupError> { + let input = into_contiguous_aligned(input); + let offset = into_contiguous_aligned(offset); + let weight = into_contiguous_aligned(weight); + let mask = mask.map(|it| into_contiguous_aligned(it)); + let bias = bias.map(|it| into_contiguous_aligned(it)); + + let [batch_size, _, in_height, in_width] = input.meta.shape().dims(); + let [out_channels, _, kernel_h, kernel_w] = weight.meta.shape().dims(); + let groups = options.weight_groups; + + let out_h = calculate_conv_output_size( + kernel_h, + options.stride[0], + options.padding[0], + options.dilation[0], + in_height, + ); + let out_w = calculate_conv_output_size( + kernel_w, + options.stride[1], + options.padding[1], + options.dilation[1], + in_width, + ); + let out_dims = (out_h, out_w); + + let columns = deform_im2col(input, offset, mask, options, out_dims, (kernel_h, kernel_w))?; + + let [col_size_0, col_size_1] = columns.meta.shape().dims(); + let col_size_0 = col_size_0 / groups; + let out_c_per_group = out_channels / groups; + + let dtype = weight.dtype; + let weight = reshape(weight, Shape::new([groups, out_c_per_group, col_size_0])); + let columns = reshape(columns, Shape::new([groups, col_size_0, col_size_1])); + let out = matmul(weight, columns, None, MatmulStrategy::default(), dtype)?; + + let out = reshape(out, Shape::new([out_channels, batch_size, out_h, out_w])); + let out = swap_dims(out, 0, 1); + + if let Some(bias) = bias { + let bias = reshape(bias, Shape::new([1, out_channels, 1, 1])); + Ok(launch_binop::(out, bias)) + } else { + Ok(out) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv_transpose2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv_transpose2d.rs new file mode 100644 index 0000000..610dfe1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/deform_conv_transpose2d.rs @@ -0,0 +1,720 @@ +use super::{bilinear_interpolate, deform_im2col, index}; +use crate::{ + CubeRuntime, + kernel::{ + cast, into_contiguous_aligned, + matmul::{MatmulStrategy, matmul}, + reduce::reduce_dim, + slice_assign, + utils::{address_type, decompose_linear, linear_view}, + }, + ops::{ + numeric::{empty_device_dtype, zeros_client}, + reshape, swap_dims, + }, + tensor::CubeTensor, +}; +use burn_backend::{DType, Shape, TensorMetadata, ops::DeformConvOptions}; +use cubecl::{ + CubeDim, CubeLaunch, calculate_cube_count_elemwise, cube, + features::TypeUsage, + ir::FloatKind, + prelude::*, + std::{FastDivmod, FastDivmodArgs, tensor::layout::linear::LinearView}, +}; +use cubek::{ + convolution::components::ConvSetupError, + reduce::components::instructions::ReduceOperationConfig, +}; +use std::marker::PhantomData; + +/// Calculate the [deformable 2D convolution](crate::ops::ModuleOps::deform_conv2d) backward pass using convolutions. +#[allow( + clippy::single_range_in_vec_init, + clippy::type_complexity, + clippy::too_many_arguments +)] +pub(crate) fn deform_conv2d_backward( + input: CubeTensor, + offset: CubeTensor, + weight: CubeTensor, + mask: Option>, + bias: Option>, + out_grad: CubeTensor, + options: DeformConvOptions<2>, +) -> Result< + ( + CubeTensor, + CubeTensor, + CubeTensor, + Option>, + Option>, + ), + ConvSetupError, +> { + let [_, _, out_h, out_w] = out_grad.meta.shape().dims(); + let [_, _, kernel_h, kernel_w] = weight.meta.shape().dims(); + + let gradient_bias = bias.map(|bias| { + let grad = reduce_dim( + out_grad.clone(), + None, + 0, + Default::default(), + ReduceOperationConfig::Sum, + ) + .unwrap(); + let grad = reduce_dim( + grad, + None, + 2, + Default::default(), + ReduceOperationConfig::Sum, + ) + .unwrap(); + let grad = reduce_dim( + grad, + None, + 3, + Default::default(), + ReduceOperationConfig::Sum, + ) + .unwrap(); + + reshape(grad, bias.meta.shape) + }); + + let input = into_contiguous_aligned(input); + let offset = into_contiguous_aligned(offset); + let weight = into_contiguous_aligned(weight); + let mask = mask.map(|it| into_contiguous_aligned(it)); + + let (input_gradient, offset_gradient, mask_gradient) = backward_gradient_inputs( + input.clone(), + weight.clone(), + offset.clone(), + mask.clone(), + out_grad.clone(), + &options, + (kernel_h, kernel_w), + )?; + + let weight_grad = compute_weight_grad( + input, + offset, + mask, + out_grad, + options, + (kernel_h, kernel_w), + (out_h, out_w), + )?; + + Ok(( + input_gradient, + offset_gradient, + weight_grad, + mask_gradient, + gradient_bias, + )) +} + +fn compute_weight_grad( + input: CubeTensor, + offset: CubeTensor, + mask: Option>, + out_grad: CubeTensor, + options: DeformConvOptions<2>, + kernel_dims: (usize, usize), + out_dims: (usize, usize), +) -> Result, ConvSetupError> { + let [_, in_channels, _, _] = input.meta.shape().dims(); + let [_, out_channels, _, _] = out_grad.meta.shape().dims(); + let (kernel_h, kernel_w) = kernel_dims; + let groups = options.weight_groups; + let dtype = input.dtype; + + let in_c_per_group = in_channels / groups; + let out_c_per_group = out_channels / groups; + + let columns = deform_im2col(input, offset, mask, options, out_dims, kernel_dims)?; + let [col_size_0, col_size_1] = columns.meta.shape().dims(); + let col_size_0 = col_size_0 / groups; + + let out_grad = swap_dims(out_grad, 0, 1); + let out_grad = reshape(out_grad, Shape::new([groups, out_c_per_group, col_size_1])); + + let columns = reshape(columns, Shape::new([groups, col_size_0, col_size_1])); + let columns = swap_dims(columns, 1, 2); + + let grad_weight = matmul(out_grad, columns, None, MatmulStrategy::default(), dtype)?; + + Ok(reshape( + grad_weight, + Shape::new([out_channels, in_c_per_group, kernel_h, kernel_w]), + )) +} + +type InputGradients = (CubeTensor, CubeTensor, Option>); + +fn backward_gradient_inputs( + image: CubeTensor, + weight: CubeTensor, + offset: CubeTensor, + mask: Option>, + out_grad: CubeTensor, + options: &DeformConvOptions<2>, + kernel_dims: (usize, usize), +) -> Result, ConvSetupError> { + let client = out_grad.client.clone(); + let device = out_grad.device.clone(); + + let [out_channels, in_c_per_group, kernel_h, kernel_w] = weight.meta.shape().dims(); + let [batch_size, _, out_h, out_w] = out_grad.meta.shape().dims(); + + let groups = options.weight_groups; + let out_c_per_group = out_channels / groups; + + let col_shape_0 = in_c_per_group * kernel_h * kernel_w; + let col_shape_1 = batch_size * out_h * out_w; + let col_shape = Shape::new([groups, col_shape_0, col_shape_1]); + let mut columns = empty_device_dtype(client, device, col_shape, weight.dtype); + + let weight = reshape(weight, Shape::new([groups, out_c_per_group, col_shape_0])); + + let out_grad = swap_dims(out_grad, 0, 1); + let out_grad_shape = Shape::new([groups, out_c_per_group, col_shape_1]); + let out_grad = reshape(out_grad, out_grad_shape); + + for group in 0..groups { + let dtype = weight.dtype; + let weight = swap_dims(index(weight.clone(), group), 0, 1); + let out_grad = index(out_grad.clone(), group); + let values = matmul(weight, out_grad, None, MatmulStrategy::default(), dtype)?; + let values = reshape(values, Shape::new([1, col_shape_0, col_shape_1])); + columns = slice_assign( + columns, + &[ + burn_backend::Slice::from(group..group + 1), + burn_backend::Slice::from(0..col_shape_0), + burn_backend::Slice::from(0..col_shape_1), + ], + values, + ); + } + + let columns = reshape(columns, Shape::new([col_shape_0 * groups, col_shape_1])); + + let input_shape = image.shape(); + let (offset_gradient, mask_gradient) = compute_offset_and_mask_gradient( + columns.clone(), + image, + offset.clone(), + mask.clone(), + options, + kernel_dims, + )?; + + let input_gradient = + compute_input_grad(columns, offset, mask, options, kernel_dims, input_shape)?; + + Ok((input_gradient, offset_gradient, mask_gradient)) +} + +fn compute_offset_and_mask_gradient( + columns: CubeTensor, + image: CubeTensor, + offset: CubeTensor, + mask: Option>, + options: &DeformConvOptions<2>, + kernel_dims: (usize, usize), +) -> Result<(CubeTensor, Option>), ConvSetupError> { + let client = offset.client.clone(); + let device = offset.device.clone(); + let (kernel_h, kernel_w) = kernel_dims; + + let [batches, _, out_h, out_w] = offset.meta.shape().dims(); + let offset_groups = options.offset_groups; + + let pos_shape = [batches, offset_groups, kernel_h, kernel_w, 2, out_h, out_w]; + let pos_shape = pos_shape + .into_iter() + .map(|s| FastDivmodArgs::new(&client, s)) + .collect(); + + let grad_offset = + empty_device_dtype(client.clone(), device.clone(), offset.shape(), offset.dtype); + let grad_mask = mask + .as_ref() + .map(|mask| empty_device_dtype(client.clone(), device.clone(), mask.shape(), mask.dtype)); + + let num_elements_offset = offset.meta.num_elements(); + let cube_dim = CubeDim::new(&image.client, num_elements_offset); + let cube_count = calculate_cube_count_elemwise(&image.client, num_elements_offset, cube_dim); + + let dtype: StorageType = image.dtype.into(); + unsafe { + deform_col2img_coord_kernel::launch_unchecked( + &image.client, + cube_count, + cube_dim, + address_type!(image, offset, mask, grad_offset, grad_mask), + image.as_tensor_arg(1), + offset.as_tensor_arg(1), + mask.as_ref().map(|mask| mask.as_tensor_arg(1)).into(), + columns.as_tensor_arg(1), + linear_view(&grad_offset, 1), + grad_mask + .as_ref() + .map(|grad_mask| grad_mask.as_tensor_arg(1)) + .into(), + pos_shape, + DeformConv2dCol2ImgCoordArgsLaunch::new( + ScalarArg::new(options.stride[0]), + ScalarArg::new(options.stride[1]), + ScalarArg::new(options.dilation[0]), + ScalarArg::new(options.dilation[1]), + InputScalar::new(options.padding[0] as f32, dtype.elem_type()), + InputScalar::new(options.padding[1] as f32, dtype.elem_type()), + ScalarArg::new(offset_groups), + ScalarArg::new(kernel_h), + ScalarArg::new(kernel_w), + ), + dtype, + ) + }?; + + Ok((grad_offset, grad_mask)) +} + +#[derive(CubeLaunch, CubeType)] +struct DeformConv2dCol2ImgCoordArgs { + stride_h: usize, + stride_w: usize, + dilation_h: usize, + dilation_w: usize, + pad_h: InputScalar, + pad_w: InputScalar, + offset_groups: usize, + kernel_height: usize, + kernel_width: usize, +} + +#[allow(clippy::collapsible_if)] +#[cube(launch_unchecked, address_type = "dynamic")] +fn deform_col2img_coord_kernel( + image: &Tensor, + offset: &Tensor, + mask: &Option>, + columns: &Tensor, + grad_offset: &mut LinearView, + grad_mask: &mut Option>, + pos_shape: Sequence>, + args: &DeformConv2dCol2ImgCoordArgs, + #[define(F)] _dtype: StorageType, +) { + // Position format: [batch, [offset_groups, kernel_h, kernel_w, 2], out_h, out_w] + // Columns format: [[in_channel, kernel_h, kernel_w], [batch, out_h, out_w]] + // Alternatively : [batch, offset_channels, out_h, out_w] + + if ABSOLUTE_POS >= grad_offset.shape() { + terminate!(); + } + + let out_h = offset.shape(2); + let out_w = offset.shape(3); + let in_channels = image.shape(1); + let height = image.shape(2); + let width = image.shape(3); + let kernel_w = args.kernel_width; + let kernel_h = args.kernel_height; + + let mut grad_offset_val = F::new(0.0); + let mut grad_mask_val = F::new(0.0); + + let (_, pos) = decompose_linear(ABSOLUTE_POS, &pos_shape); + let [batch, offset_group, kernel_y, kernel_x, dir, out_y, out_x] = *pos else { + unreachable!() + }; + + let channels_per_offset_group = in_channels / args.offset_groups; + + let col_n = batch * out_h * out_w + out_y * out_w + out_x; + + let col_base_idx = + offset_group * channels_per_offset_group * kernel_h * kernel_w * columns.stride(0) + + col_n * columns.stride(1); + let mut image_base_idx = + batch * image.stride(0) + offset_group * channels_per_offset_group * image.stride(1); + + let offset_pos_1 = + offset_group * kernel_h * kernel_w * 2 + kernel_y * kernel_w * 2 + kernel_x * 2; + let offset_base_idx = batch * offset.stride(0) + + offset_pos_1 * offset.stride(1) + + out_y * offset.stride(2) + + out_x * offset.stride(3); + + let offset_y_idx = offset_base_idx; + let offset_x_idx = offset_base_idx + offset.stride(1); + + let offset_y = offset[offset_y_idx]; + let offset_x = offset[offset_x_idx]; + + let mask_pos_1 = offset_group * kernel_h * kernel_w + kernel_y * kernel_w + kernel_x; + let mask_value = match &mask { + Some(mask) => { + let mask_idx = batch * mask.stride(0) + + mask_pos_1 * mask.stride(1) + + out_y * mask.stride(2) + + out_x * mask.stride(3); + mask[mask_idx] + } + None => F::new(1.0), + }; + + let is_y_direction = dir == 0; + + for col_c in 0..channels_per_offset_group { + let col_pos = col_base_idx + col_c * kernel_h * kernel_w * columns.stride(0); + + let y = F::cast_from(out_y * args.stride_h + kernel_y * args.dilation_h) + - args.pad_h.get::() + + offset_y; + let x = F::cast_from(out_x * args.stride_w + kernel_x * args.dilation_w) + - args.pad_w.get::() + + offset_x; + + let weight = + get_coordinate_weight(image, image_base_idx, height, width, y, x, is_y_direction); + let columns_value = columns[col_pos]; + + grad_offset_val += mask_value * weight * columns_value; + + if grad_mask.is_some() && is_y_direction { + grad_mask_val += + columns_value * bilinear_interpolate(image, height, width, y, x, image_base_idx); + } + + image_base_idx += image.stride(1); + } + + grad_offset[ABSOLUTE_POS] = grad_offset_val; + + if let Some(grad_mask) = grad_mask { + if is_y_direction { + let idx = batch * grad_mask.stride(0) + + mask_pos_1 * grad_mask.stride(1) + + out_y * grad_mask.stride(2) + + out_x * grad_mask.stride(3); + + grad_mask[idx] = grad_mask_val + } + } +} + +#[cube] +fn get_coordinate_weight( + input: &Tensor, + offset: usize, + height: usize, + width: usize, + y: F, + x: F, + is_y_direction: bool, +) -> F { + let stride_y = input.stride(2); + let stride_x = input.stride(3); + + let y = f32::cast_from(y); + let x = f32::cast_from(x); + + let y_low = f32::floor(y); + let x_low = f32::floor(x); + let y_high = y_low + 1.; + let x_high = x_low + 1.; + + let valid_y_low = y_low >= 0. && y_low < height as f32; + let valid_y_high = y_high >= 0. && y_high < height as f32; + let valid_x_low = x_low >= 0. && x_low < width as f32; + let valid_x_high = x_high >= 0. && x_high < width as f32; + + let bottom_left = if valid_y_low && valid_x_low { + input[offset + y_low as usize * stride_y + x_low as usize * stride_x] + } else { + F::new(0.0) + }; + let bottom_right = if valid_y_low && valid_x_high { + input[offset + y_low as usize * stride_y + x_high as usize * stride_x] + } else { + F::new(0.0) + }; + let top_left = if valid_y_high && valid_x_low { + input[offset + y_high as usize * stride_y + x_low as usize * stride_x] + } else { + F::new(0.0) + }; + let top_right = if valid_y_high && valid_x_high { + input[offset + y_high as usize * stride_y + x_high as usize * stride_x] + } else { + F::new(0.0) + }; + + if is_y_direction { + let delta_x = F::cast_from(x - x_low); + delta_x * (top_right - bottom_right) + (F::new(1.0) - delta_x) * (top_left - bottom_left) + } else { + let delta_y = F::cast_from(y - y_low); + delta_y * (top_right - top_left) + (F::new(1.0) - delta_y) * (bottom_right - bottom_left) + } +} + +fn compute_input_grad( + columns: CubeTensor, + offset: CubeTensor, + mask: Option>, + options: &DeformConvOptions<2>, + kernel_dims: (usize, usize), + input_shape: Shape, +) -> Result, LaunchError> { + let client = offset.client.clone(); + let device = offset.device.clone(); + + let supports_fadd = client + .properties() + .type_usage(StorageType::Atomic(FloatKind::F32.into())) + .contains(TypeUsage::AtomicAdd); + let supports_same_type = client + .properties() + .type_usage(StorageType::Atomic(columns.dtype.into())) + .contains(TypeUsage::AtomicAdd); + + let [batches, in_channels, height, width] = input_shape.dims(); + let [_, _, out_h, out_w] = offset.meta.shape().dims(); + let (kernel_h, kernel_w) = kernel_dims; + + let pos_shape = [in_channels, kernel_h, kernel_w, batches, out_h, out_w]; + let pos_shape = pos_shape + .into_iter() + .map(|s| FastDivmodArgs::new(&client, s)) + .collect(); + + let shape = Shape::new([batches, in_channels, height, width]); + let grad_in = match supports_fadd && supports_same_type { + // Use type as is to save a cast + true => zeros_client(client.clone(), device.clone(), shape, columns.dtype), + // Force `f32` to enable bitcasting as `u32`, or use intrinsic when supported + false => zeros_client(client.clone(), device.clone(), shape, DType::F32), + }; + let grad_arg = grad_in.as_tensor_arg(1); + + let num_elements = columns.meta.num_elements(); + let cube_dim = CubeDim::new(&offset.client, num_elements); + let cube_count = calculate_cube_count_elemwise(&offset.client, num_elements, cube_dim); + + let launch = match supports_fadd { + true => deform_col2img_kernel::launch_unchecked::, + false => deform_col2img_kernel::launch_unchecked::, + }; + let dtype = offset.dtype; + let dtypes: [StorageType; 2] = match supports_same_type { + true => [dtype.into(), dtype.into()], + false => [dtype.into(), DType::F32.into()], + }; + + unsafe { + launch( + &offset.client, + cube_count, + cube_dim, + address_type!(offset, mask, columns, grad_in), + offset.as_tensor_arg(1), + mask.as_ref().map(|mask| mask.as_tensor_arg(1)).into(), + linear_view(&columns, 1), + grad_arg, + pos_shape, + DeformConv2dCol2ImgArgsLaunch::new( + ScalarArg::new(options.stride[0]), + ScalarArg::new(options.stride[1]), + ScalarArg::new(options.dilation[0]), + ScalarArg::new(options.dilation[1]), + InputScalar::new(options.padding[0] as f32, dtypes[0].elem_type()), + InputScalar::new(options.padding[1] as f32, dtypes[0].elem_type()), + ScalarArg::new(options.offset_groups), + ScalarArg::new(kernel_h), + ScalarArg::new(kernel_w), + ), + dtypes, + ) + }?; + + Ok(if !supports_same_type || !supports_fadd { + cast(grad_in, dtype) + } else { + grad_in + }) +} + +#[derive(CubeLaunch, CubeType)] +struct DeformConv2dCol2ImgArgs { + stride_h: usize, + stride_w: usize, + dilation_h: usize, + dilation_w: usize, + pad_h: InputScalar, + pad_w: InputScalar, + offset_groups: usize, + kernel_height: usize, + kernel_width: usize, +} + +#[cube(launch_unchecked, address_type = "dynamic")] +fn deform_col2img_kernel( + offset: &Tensor, + mask: &Option>, + columns: &LinearView, + grad_input: &mut Tensor>>, + pos_shape: Sequence>, + args: &DeformConv2dCol2ImgArgs, + #[define(F, FP)] _dtype: [StorageType; 2], +) { + // Position format: [[in_channels, kernel_h, kernel_w], [batch_size, out_h, out_w]] + if ABSOLUTE_POS >= columns.shape() { + terminate!(); + } + + let n_in_channels = grad_input.shape(1); + let height = grad_input.shape(2); + let width = grad_input.shape(3); + let kernel_h = args.kernel_height; + let kernel_w = args.kernel_width; + let n_offset_groups = args.offset_groups; + + let (_, pos) = decompose_linear(ABSOLUTE_POS, &pos_shape); + let [in_channel, kernel_y, kernel_x, batch, out_y, out_x] = *pos else { + unreachable!() + }; + + let channels_per_offset_group = n_in_channels / n_offset_groups; + let offset_group = in_channel / channels_per_offset_group; + + let offset_pos_1 = + offset_group * kernel_h * kernel_w * 2 + kernel_y * kernel_w * 2 + kernel_x * 2; + let offset_base_idx = batch * offset.stride(0) + + offset_pos_1 * offset.stride(1) + + out_y * offset.stride(2) + + out_x * offset.stride(3); + + let offset_y_idx = offset_base_idx; + let offset_x_idx = offset_base_idx + offset.stride(1); + + let offset_y = offset[offset_y_idx]; + let offset_x = offset[offset_x_idx]; + + let mask_value = match mask { + Some(mask) => { + let mask_pos_1 = offset_group * kernel_h * kernel_w + kernel_y * kernel_w + kernel_x; + mask[batch * mask.stride(0) + + mask_pos_1 * mask.stride(1) + + out_y * mask.stride(2) + + out_x * mask.stride(3)] + } + None => F::new(1.0), + }; + + let y = F::cast_from(out_y * args.stride_h + kernel_y * args.dilation_h) + - args.pad_h.get::() + + offset_y; + let x = F::cast_from(out_x * args.stride_w + kernel_x * args.dilation_w) + - args.pad_w.get::() + + offset_x; + + for dy in -1..=1i32 { + #[unroll] + for dx in -1..=1i32 { + let yp = y.floor() + F::cast_from(dy); + let xp = x.floor() + F::cast_from(dx); + + if yp >= F::new(0.0) + && yp < F::cast_from(height) + && xp >= F::new(0.0) + && xp < F::cast_from(width) + && F::abs(y - yp) < F::new(1.0) + && F::abs(x - xp) < F::new(1.0) + { + let gradient_pos = batch * grad_input.stride(0) + + in_channel * grad_input.stride(1) + + usize::cast_from(yp) * grad_input.stride(2) + + usize::cast_from(xp) * grad_input.stride(3); + + let weight = (F::new(1.0) - F::abs(y - yp)) * (F::new(1.0) - F::abs(x - xp)); + + let value = mask_value * F::cast_from(weight) * columns[ABSOLUTE_POS]; + + FAdd::Op::::float_atomic_add::(&mut grad_input[gradient_pos], value); + } + } + } +} + +type ProxyType = <::Op as FloatAtomicAdd>::ProxyType; + +#[cube] +trait FloatAtomicAddFamily: Send + Sync + 'static { + type Op: FloatAtomicAdd; +} + +#[cube] +trait FloatAtomicAdd: Send + Sync + 'static { + type ProxyType: Numeric; + + fn float_atomic_add(ptr: &mut Atomic, value: F); +} + +#[derive(CubeType)] +struct IntrinsicFloatAtomicAdd { + #[cube(comptime)] + _ty: PhantomData, +} + +#[derive(CubeType)] +struct CASFloatAtomicAdd; + +struct IntrinsicFloatAtomicAddFamily; + +impl FloatAtomicAddFamily for IntrinsicFloatAtomicAddFamily { + type Op = IntrinsicFloatAtomicAdd; +} + +impl FloatAtomicAddFamily for CASFloatAtomicAdd { + type Op = Self; +} + +#[cube] +impl FloatAtomicAdd for IntrinsicFloatAtomicAdd { + type ProxyType = FAdd; + + fn float_atomic_add(ptr: &mut Atomic, value: F) { + let value = FAdd::cast_from(value); + ptr.fetch_add(value); + } +} + +#[cube] +impl FloatAtomicAdd for CASFloatAtomicAdd { + type ProxyType = u32; + + fn float_atomic_add(ptr: &mut Atomic, value: F) { + let value = f32::cast_from(value); + if value != 0.0 { + let mut v = ptr.load(); + loop { + let prev = v; + let v_float = f32::from_bits(v); + let new = (v_float + value).to_bits(); + v = ptr.compare_exchange_weak(v, new); + if prev == v { + break; + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/direct.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/direct.rs new file mode 100644 index 0000000..3e19f65 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/direct.rs @@ -0,0 +1,320 @@ +use crate::{ + CubeRuntime, + kernel::{ + into_contiguous_aligned, + utils::{address_type, linear_view}, + }, + ops::max_line_size, + tensor::CubeTensor, +}; +use crate::{kernel::utils::decompose_linear, ops::numeric::empty_device_dtype}; +use burn_backend::{ + TensorMetadata, + ops::{ConvOptions, conv::calculate_conv_output_sizes}, +}; +use cubecl::std::{FastDivmod, FastDivmodArgs}; +use cubecl::{ + calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView, + tensor_line_size_parallel, +}; +use cubek::convolution::components::ConvSetupError; + +#[derive(CubeLaunch, CubeType, Clone)] +pub(crate) struct ConvParam { + pub stride: u32, + pub dilation: u32, + pub padding: i32, +} + +#[derive(CubeLaunch, CubeType)] +struct Conv2dArgs { + conv_params: Sequence, + channels_per_group: u32, +} + +#[cube(launch_unchecked, address_type = "dynamic")] +fn direct_conv2d_kernel( + input: &Tensor>, + weight: &Tensor>, + bias: Option>>, + output: &mut LinearView, ReadWrite>, + args: Conv2dArgs, + shape_out: Sequence>, + shape_out_c: FastDivmod, + #[comptime] has_padding: bool, + #[define(E)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let n_spatial = comptime![shape_out.len()]; + + let line_size_out = output.line_size(); + let pos = ABSOLUTE_POS * line_size_out; + + let in_c_per_group = weight.shape(weight.rank() - 1) as u32; + + let (rem, out_c) = shape_out_c.div_mod(pos as u32); + let (b, spatial_pos) = decompose_linear(rem, &shape_out); + + let g = out_c / args.channels_per_group; + let ic_start = in_c_per_group * g; + + let bias: Option> = bias.map(|bias| bias[out_c as usize / line_size_out]); + let mut sum = bias.unwrap_or_else(|| Line::empty(line_size_out).fill(E::from_int(0))); + + let in_offs = b as usize * input.stride(0) + ic_start as usize; + + let stride_oc = weight.stride(0); + + let mut in_shape = Sequence::new(); + let mut in_strides = Sequence::new(); + let mut kernel_shape = Sequence::new(); + let mut kernel_strides = Sequence::new(); + + #[unroll] + for i in 0..n_spatial { + in_shape.push(input.shape(i + 1) as u32); + in_strides.push(input.stride(i + 1)); + kernel_shape.push(weight.shape(i + 1) as u32); + kernel_strides.push(weight.stride(i + 1)); + } + + let weight_offs = out_c as usize * stride_oc; + + let loop_params = LoopParams { + out_pos: spatial_pos, + in_shape, + in_strides, + kernel_shape, + kernel_strides, + conv_params: args.conv_params, + in_c_per_group, + stride_oc, + }; + + kernel_loop( + input, + weight, + &mut sum, + in_offs, + true, + weight_offs, + &loop_params, + 0usize, + has_padding, + ); + + output[ABSOLUTE_POS] = sum; +} + +#[derive(CubeType, Clone)] +struct LoopParams { + out_pos: Sequence, + in_shape: Sequence, + in_strides: Sequence, + kernel_shape: Sequence, + kernel_strides: Sequence, + conv_params: Sequence, + + in_c_per_group: u32, + stride_oc: usize, +} + +#[cube] +fn kernel_loop( + input: &Tensor>, + weight: &Tensor>, + sum: &mut Line, + in_offs: usize, + in_bounds: bool, + weight_offs: usize, + params: &LoopParams, + #[comptime] kernel_dim: usize, + #[comptime] has_padding: bool, +) { + if comptime![kernel_dim < params.kernel_shape.len()] { + let out_idx = *params.out_pos.index(kernel_dim); + let conv = params.conv_params.index(kernel_dim); + let shape = *params.in_shape.index(kernel_dim); + let stride = *params.in_strides.index(kernel_dim); + let k_stride = *params.kernel_strides.index(kernel_dim); + + for pos in 0..*params.kernel_shape.index(kernel_dim) { + let in_pos = (out_idx * conv.stride + pos * conv.dilation) as i32 - conv.padding; + let in_offs = in_offs + in_pos as usize * stride; + let weight_offs = weight_offs + pos as usize * k_stride; + let mut in_bounds = in_bounds; + + if has_padding { + in_bounds &= in_pos >= 0 && (in_pos as u32) < shape; + } + + kernel_loop( + input, + weight, + sum, + in_offs, + in_bounds, + weight_offs, + params, + comptime![kernel_dim + 1], + has_padding, + ); + } + } else { + kernel_loop_inner( + input, + weight, + sum, + in_offs, + in_bounds, + weight_offs, + params.in_c_per_group, + params.stride_oc, + ); + } +} + +#[cube] +fn kernel_loop_inner( + input: &Tensor>, + weight: &Tensor>, + sum: &mut Line, + in_offs: usize, + in_bounds: bool, + weight_offs: usize, + in_c_per_group: u32, + stride_oc: usize, +) { + let line_size_in = input.line_size(); + let line_size_out = sum.size(); + + if in_bounds { + for in_c in range_stepped(0, in_c_per_group, line_size_in as u32) { + let in_pos = in_offs + in_c as usize; + let mut weight_pos = weight_offs + in_c as usize; + + let val = input[in_pos / line_size_in]; + + #[unroll] + for v in 0..line_size_out { + let weight = weight[weight_pos / line_size_in]; + let val = val * weight; + + #[unroll] + for i in 0..line_size_in { + sum[v] += val[i]; + } + weight_pos += stride_oc; + } + } + } +} + +/// Perform a 2D convolution using the direct convolution algorithm. +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +/// +pub fn conv_direct( + mut input: CubeTensor, + mut weight: CubeTensor, + bias: Option>, + options: ConvOptions, +) -> Result, ConvSetupError> { + let client = input.client.clone(); + let out_dtype = input.dtype; + let rank = input.meta.shape().num_dims(); + let dim_c = rank - 1; + + // We only care about the channels here, everything else can be permuted + if input.meta.strides()[dim_c] != 1 { + input = into_contiguous_aligned(input); + } + if weight.meta.strides()[dim_c] != 1 { + weight = into_contiguous_aligned(weight); + } + + let batch_size = input.meta.shape()[0]; + let in_shape = &input.meta.shape()[1..dim_c]; + let out_channels = weight.meta.shape()[0]; + let kernel_shape = &weight.meta.shape()[1..dim_c]; + + let channels_per_group = out_channels / options.groups; + + let out_size = calculate_conv_output_sizes( + kernel_shape, + &options.stride, + &options.padding, + &options.dilation, + in_shape, + ); + + let mut shape_out = vec![batch_size]; + shape_out.extend(out_size.iter().copied()); + shape_out.push(out_channels); + + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + shape_out.into(), + out_dtype, + ); + + // Need custom line size calculation here to account for the groups division. Need to vectorize + // over `channels_per_group` instead. + let mut grouped_out_shape = output.shape(); + grouped_out_shape[dim_c] = channels_per_group; + let line_size_out = tensor_line_size_parallel( + input.client.io_optimized_line_sizes(input.dtype.size()), + &grouped_out_shape, + output.meta.strides(), + dim_c, + ); + // Use channels_per_group instead of in_channels to avoid issues here + let line_size_in = max_line_size(&weight); + + let shape_out = output.meta.shape()[1..dim_c] + .iter() + .map(|s| FastDivmodArgs::::new(&client, *s as u32)) + .collect(); + let shape_out_c = FastDivmodArgs::::new(&client, out_channels as u32); + + let mut conv_params = SequenceArg::new(); + + for i in 0..kernel_shape.len() { + conv_params.push(ConvParamLaunch::new( + ScalarArg::new(options.stride[i] as u32), + ScalarArg::new(options.dilation[i] as u32), + ScalarArg::new(options.padding[i] as i32), + )); + } + + let working_units = output.meta.num_elements() / line_size_out; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + unsafe { + direct_conv2d_kernel::launch_unchecked( + &input.client, + cube_count, + cube_dim, + address_type!(input, weight, bias, output), + input.as_tensor_arg(line_size_in), + weight.as_tensor_arg(line_size_in), + bias.as_ref().map(|b| b.as_tensor_arg(line_size_out)).into(), + linear_view(&output, line_size_out), + Conv2dArgsLaunch::new(conv_params, ScalarArg::new(channels_per_group as u32)), + shape_out, + shape_out_c, + options.padding.iter().any(|it| *it != 0), + out_dtype.into(), + ) + }?; + + Ok(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/launch.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/launch.rs new file mode 100644 index 0000000..4637fcd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/launch.rs @@ -0,0 +1,167 @@ +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; +use burn_backend::ops::{ConvOptions, conv::calculate_conv_output_sizes}; +use cubek::{ + convolution::{ + AcceleratedTileKind, ConvolutionArgs, ReadingStrategy, Strategy, + components::ConvSetupError, forward, + }, + matmul::{ + definition::{MatmulElems, MatmulGlobalElems}, + launch::MatmulInputHandleRef, + }, +}; + +/// Perform a 2D convolution using the implicit GEMM (im2col) algorithm, using cubecl tiling matmul +/// components. Uses [`CmmaLargeMAlgorithm`] for the stage size +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +pub fn conv_gemm_simple_sync( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + let read_strategy = match tile_kind { + AcceleratedTileKind::Cmma => ReadingStrategy::Cyclic, + AcceleratedTileKind::Mma => ReadingStrategy::Strided, + }; + launch_convolution_forward::( + &Strategy::Simple { + read_strategy, + tile_kind, + }, + input, + weight, + bias, + options, + ) +} + +pub fn conv_gemm_simple_async( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + let read_strategy = match tile_kind { + AcceleratedTileKind::Cmma => ReadingStrategy::AsyncCyclic, + AcceleratedTileKind::Mma => ReadingStrategy::AsyncStrided, + }; + launch_convolution_forward::( + &Strategy::Simple { + read_strategy, + tile_kind, + }, + input, + weight, + bias, + options, + ) +} + +/// Perform a 2D convolution using the implicit GEMM (im2col) algorithm, using cubecl tiling matmul +/// components. Uses [`CmmaLargeMAlgorithm`] for the stage size +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +pub fn conv_gemm_simple_tma( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, + tile_kind: AcceleratedTileKind, +) -> Result, ConvSetupError> { + launch_convolution_forward::( + &Strategy::Simple { + read_strategy: ReadingStrategy::Tma, + tile_kind, + }, + input, + weight, + bias, + options, + ) +} + +/// Perform a 2D convolution using the implicit GEMM (im2col) algorithm, using cubecl tiling matmul +/// components, using the specified algorithm. +/// +/// * `input` - The input feature map +/// * `weight` - The weights (filter) applied to each kernel +/// * `bias` - The bias added to each channel +/// * `options` - The options to use for the convolution +pub fn launch_convolution_forward( + strategy: &Strategy, + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, +) -> Result, ConvSetupError> { + if options.groups != 1 { + return Err(ConvSetupError::Groups(options.groups)); + } + + let out_dtype = input.dtype; + let rank = input.meta.shape().num_dims(); + let batch_size = input.meta.shape()[0]; + let dim_c = rank - 1; + let shape = &input.meta.shape()[1..dim_c]; + + let out_channels = weight.meta.shape()[0]; + let weight_shape = &weight.meta.shape()[1..dim_c]; + + let mut out_shape = calculate_conv_output_sizes( + weight_shape, + &options.stride, + &options.padding, + &options.dilation, + shape, + ); + + out_shape.insert(0, batch_size); + out_shape.push(out_channels); + + let out = empty_device_dtype( + input.client.clone(), + input.device.clone(), + out_shape.into(), + out_dtype, + ); + + let bias = bias + .as_ref() + .map(|bias| MatmulInputHandleRef::Normal(bias.as_handle_ref(), bias.dtype.into())); + + let client = input.client.clone(); + let dtypes = MatmulElems::from_globals(&MatmulGlobalElems { + lhs: input.dtype.into(), + rhs: weight.dtype.into(), + out: out_dtype.into(), + }); + let input = MatmulInputHandleRef::new(input.as_handle_ref(), input.dtype.into()); + let weight = MatmulInputHandleRef::new(weight.as_handle_ref(), weight.dtype.into()); + + forward::launch_ref::( + strategy, + &client, + &input, + &weight, + &bias, + &out.as_handle_ref(), + ConvolutionArgs { + stride: options.stride, + padding: options.padding, + dilation: options.dilation, + }, + dtypes, + )?; + + Ok(out) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/mod.rs new file mode 100644 index 0000000..0df8c51 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/implicit_gemm/mod.rs @@ -0,0 +1,2 @@ +pub mod launch; +pub use launch::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/mod.rs new file mode 100644 index 0000000..d5091ae --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/mod.rs @@ -0,0 +1,7 @@ +pub mod implicit_gemm; + +#[cfg(feature = "autotune")] +pub mod tune; + +#[cfg(feature = "autotune")] +pub(crate) use tune::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/tune.rs new file mode 100644 index 0000000..e7d4e70 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/forward/tune.rs @@ -0,0 +1,174 @@ +use burn_backend::ops::ConvOptions; +use cubecl::{ + ir::StorageType, + tune::{LocalTuner, Tunable, TunableSet, anchor, local_tuner}, +}; +use cubek::convolution::AcceleratedTileKind; + +use crate::{ + CubeAutotuneKey, CubeRuntime, CubeTuneId, + kernel::conv::{ConvAutotuneKey, conv_direct, conv_im2col_1x1, forward::implicit_gemm::*}, + tensor::CubeTensor, +}; + +/// Executes autotune on convolution operations +pub fn conv_autotune( + input: CubeTensor, + weight: CubeTensor, + bias: Option>, + options: ConvOptions, +) -> CubeTensor { + let client = input.client.clone(); + + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + TunableSet::new(create_key::, create_conv_input::) + .with(Tunable::new("conv_direct", conv_direct::)) + .with(Tunable::new("conv_im2col_1x1", conv_im2col_1x1::)) + .with(Tunable::new( + "simple_sync_cmma", + |input, weight, bias, options| { + conv_gemm_simple_sync(input, weight, bias, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_sync_mma", + |input, weight, bias, options| { + conv_gemm_simple_sync(input, weight, bias, options, AcceleratedTileKind::Mma) + }, + )) + .with(Tunable::new( + "simple_async_cmma", + |input, weight, bias, options| { + conv_gemm_simple_async(input, weight, bias, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_async_mma", + |input, weight, bias, options| { + conv_gemm_simple_async(input, weight, bias, options, AcceleratedTileKind::Mma) + }, + )) + .with(Tunable::new( + "simple_tma_cmma", + |input, weight, bias, options| { + conv_gemm_simple_tma(input, weight, bias, options, AcceleratedTileKind::Cmma) + }, + )) + .with(Tunable::new( + "simple_tma_mma", + |input, weight, bias, options| { + conv_gemm_simple_tma(input, weight, bias, options, AcceleratedTileKind::Mma) + }, + )) + }); + + TUNER.execute( + &CubeTuneId::new(&input.client, &input.device), + &client, + tunables, + (input, weight, bias, options), + ) +} + +pub fn create_conv_input( + _key: &CubeAutotuneKey, + input: &CubeTensor, + weights: &CubeTensor, + bias: &Option>, + options: &ConvOptions, +) -> ( + CubeTensor, + CubeTensor, + Option>, + ConvOptions, +) { + ( + input.clone(), + weights.clone(), + bias.clone(), + options.clone(), + ) +} + +fn create_key( + input: &CubeTensor, + weights: &CubeTensor, + bias: &Option>, + options: &ConvOptions, +) -> CubeAutotuneKey { + let dtype = input.dtype; + let rank = input.meta.shape().num_dims(); + let dim_c = rank - 1; + + let batch_size = input.meta.shape()[0]; + let in_channels = input.meta.shape()[dim_c]; + let out_channels = weights.meta.shape()[0]; + + let kernel_size = weights.meta.shape()[1..dim_c].to_vec(); + let in_shape = input.meta.shape()[1..dim_c] + .iter() + .map(|shape| anchor(*shape, None, None, None)) + .collect(); + + let ConvOptions { + stride, + padding, + dilation, + groups, + } = options.clone(); + + let lhs_stride_align = if input.meta.strides()[dim_c] == 1 { + stride_align(input.meta.strides(), input.dtype.into()) + } else { + 0 + }; + let lhs_shape_align = pow2_factor(in_channels).min(lhs_stride_align); + let rhs_stride_align = if weights.meta.strides()[dim_c] == 1 { + stride_align(weights.meta.strides(), weights.dtype.into()) + } else { + 0 + }; + let rhs_shape_align = pow2_factor(in_channels).min(rhs_stride_align); + + CubeAutotuneKey::Conv(ConvAutotuneKey::new( + kernel_size, + stride.to_vec(), + padding.to_vec(), + dilation.to_vec(), + groups, + in_channels, + out_channels, + in_shape, + batch_size, + bias.is_some(), + dtype, + lhs_shape_align, + lhs_stride_align, + rhs_shape_align, + rhs_stride_align, + )) +} + +/// Maximum factor relevant for strides. Currently set to 2^10 because that's 128-byte swizzle's +/// repeat number, so it's the largest align that can have performance impacts. +const MAX_STRIDE_FACTOR: u32 = 10; + +/// Defines the non-contiguous stride alignment in terms of powers of two +fn stride_align(strides: &[usize], elem: StorageType) -> u8 { + let max = MAX_STRIDE_FACTOR; + let dim_c = strides.len() - 1; + let factor = strides[..dim_c] + .iter() + .map(|it| (*it * elem.size_bits()) / 8) + .map(|it| it.trailing_zeros()) + .min() + .unwrap_or(max); + factor.min(max) as u8 +} + +/// Defines the potential vectorization. +fn pow2_factor(axis: usize) -> u8 { + axis.trailing_zeros().min(4) as u8 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/im2col.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/im2col.rs new file mode 100644 index 0000000..ef11084 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/im2col.rs @@ -0,0 +1,187 @@ +use burn_backend::{ + DType, + ops::{ConvOptions, conv::calculate_conv_output_sizes}, +}; +use burn_std::{Metadata, Shape}; +use core::iter; +use cubecl::{ + prelude::*, + std::tensor::{TensorHandle, into_contiguous_pitched_ref}, +}; +use cubek::convolution::components::ConvSetupError; + +use crate::{ + CubeRuntime, + kernel::{ + AddOp, into_contiguous_aligned, launch_binop, + matmul::{MatmulStrategy, matmul}, + utils::split_dim, + }, + ops::{reshape, swap_dims}, + tensor::CubeTensor, +}; + +#[cfg(not(test))] +pub(crate) fn batches_per_run( + batch_size: usize, + out_shape: usize, + plane_size: usize, +) -> Result { + use cubek::matmul::definition::MatmulAvailabilityError; + + let cube_count_per_batch = out_shape.div_ceil(plane_size); + let max_cube_count = u16::MAX as usize; + let max_simultaneous = Ord::min(max_cube_count / cube_count_per_batch, batch_size); + if max_simultaneous == 0 { + return Err(MatmulAvailabilityError::CubeCountTooBig(CubeCount::Static( + cube_count_per_batch as u32, + 1, + 1, + )) + .into()); + } + Ok((0..=max_simultaneous) + .rev() + .find(|per_run| batch_size.is_multiple_of(*per_run)) + .expect("Logically not possible")) +} + +#[cfg(test)] +#[allow(unused)] +pub(crate) fn batches_per_run( + batch_size: usize, + out_shape: usize, + plane_size: usize, +) -> Result { + Ok(1) +} + +pub fn conv_im2col_1x1( + input: CubeTensor, + mut weight: CubeTensor, + bias: Option>, + options: ConvOptions, +) -> Result, ConvSetupError> { + if options.groups != 1 { + return Err(ConvSetupError::Groups(options.groups)); + } + + let rank = input.meta.num_dims(); + let dim_c = rank - 1; + + let batch_size = input.meta.shape()[0]; + let in_channels = input.meta.shape()[dim_c]; + let in_shape = &input.meta.shape()[1..dim_c]; + let out_channels = weight.meta.shape()[0]; + let kernel_shape = &weight.meta.shape()[1..dim_c]; + + if kernel_shape.iter().any(|s| *s != 1) { + return Err(ConvSetupError::Unknown); + } + + let out_shape = calculate_conv_output_sizes( + kernel_shape, + &options.stride, + &options.padding, + &options.dilation, + in_shape, + ); + + let mut split_m = vec![batch_size]; + split_m.extend(out_shape.iter().copied()); + + if kernel_shape.iter().any(|it| *it != 1) || in_shape != out_shape { + return Err(ConvSetupError::Unknown); + } + + let input = reshape_input(input); // [(NHW), C] : [M, K] + let dtype = input.dtype; + + // Efficient permutation that takes the stride required for TMA into account + let weight = if weight.meta.strides()[dim_c] != 1 { + // Remove kernel dims so padded dim is channels + *weight.meta = Metadata::new( + [out_channels, in_channels], // [N, K] + [weight.meta.strides()[0], weight.meta.strides()[dim_c]], + ); + // Pitched contiguous to skip running another kernel for TMA + into_contiguous_aligned(weight) + } else { + // Already compatible, skip initial reshape + *weight.meta = Metadata::new([out_channels, in_channels], [weight.meta.strides()[0], 1]); + weight + }; + + // Permute to N-major, while keeping memory layout K-major. K-major for both sides is the most + // efficient for matmul, and allows skipping a contiguous kernel + let weight = swap_dims(weight, 0, 1); // [K, N] + + let out = matmul(input, weight, None, MatmulStrategy::default(), dtype)?; // [M, N] + + // Skip reshape to avoid potential `into_contiguous`. We're only splitting dims so it's safe. + let mut out = split_dim(out, 0, &split_m); // [N, H, W, C] + + if let Some(bias) = bias { + let mut bias_shape = iter::repeat_n(1, rank - 1).collect::>(); + bias_shape.push(out_channels); + let bias = reshape(bias, bias_shape.into()); + out = launch_binop::(out, bias); + } + + Ok(out) +} + +/// Reshapes NHWC input to [(N, H, W), C] +fn reshape_input(mut input: CubeTensor) -> CubeTensor { + let rank = input.meta.num_dims(); + let dim_c = rank - 1; + let dtype = input.dtype; + + let batch_size = input.meta.shape()[0]; + let in_c: usize = input.meta.shape()[dim_c]; + let in_shape: Shape = input.meta.shape()[1..dim_c].into(); + + if !is_spatial_contiguous(input.meta.shape(), input.meta.strides()) { + let contiguous = + into_contiguous_pitched_ref(&input.client, &input.as_handle_ref(), dtype.into()) + .expect("Kernel to never fail"); + input = from_handle(&input.client, &input.device, contiguous, dtype); + } + *input.meta = Metadata::new( + [batch_size * in_shape.num_elements(), in_c], // [M, K] + [input.meta.strides()[dim_c - 1], input.meta.strides()[dim_c]], + ); + input +} + +fn is_spatial_contiguous(shape: &[usize], strides: &[usize]) -> bool { + let rank = shape.len(); + + let mut ordered = strides.to_vec(); + ordered.sort(); + if ordered != strides { + return false; + } + + for i in (1..rank - 2).rev() { + if strides[i + 1] * shape[i + 1] != strides[i] { + return false; + } + } + true +} + +fn from_handle( + client: &ComputeClient, + device: &R::Device, + handle: TensorHandle, + dtype: DType, +) -> CubeTensor { + CubeTensor::new( + client.clone(), + handle.handle, + *handle.metadata, + device.clone(), + dtype, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/mod.rs new file mode 100644 index 0000000..03dde01 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/mod.rs @@ -0,0 +1,25 @@ +mod backward_data; +mod backward_weight; +mod base; +mod conv_transpose2d; +mod conv_transpose3d; +mod deform_conv2d; +mod deform_conv_transpose2d; +mod direct; +mod forward; +mod im2col; + +mod tune_key; + +pub(crate) use backward_data::*; +pub(crate) use conv_transpose2d::*; +pub(crate) use conv_transpose3d::*; +pub(crate) use deform_conv_transpose2d::*; +pub(crate) use deform_conv2d::*; +pub(crate) use direct::*; +pub(crate) use im2col::*; + +pub use base::*; +pub use conv_transpose2d::{ConvTranspose2dStrategy, conv_transpose2d}; + +pub(crate) use tune_key::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/tune_key.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/tune_key.rs new file mode 100644 index 0000000..1df268d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/conv/tune_key.rs @@ -0,0 +1,50 @@ +use burn_backend::DType; +use cubecl::AutotuneKey; +use serde::{Deserialize, Serialize}; + +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, AutotuneKey)] +/// Autotune key representative of matmul versions +pub struct ConvAutotuneKey { + pub kernel_size: Vec, + pub stride: Vec, + pub padding: Vec, + pub dilation: Vec, + pub groups: usize, + #[autotune(anchor)] + pub in_channels: usize, + #[autotune(anchor)] + pub out_channels: usize, + pub shape: Vec, + #[autotune(anchor)] + pub batch_size: usize, + pub has_bias: bool, + pub dtype: DType, + + pub lhs_shape_align: u8, + pub lhs_stride_align: u8, + pub rhs_shape_align: u8, + pub rhs_stride_align: u8, +} + +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, AutotuneKey)] +/// Autotune key representative of matmul versions +pub struct ConvTranspose2dAutotuneKey { + pub kernel_size: [usize; 2], + pub stride: [usize; 2], + pub padding: [usize; 2], + pub padding_out: [usize; 2], + pub dilation: [usize; 2], + pub groups: usize, + #[autotune(anchor)] + pub in_channels: usize, + #[autotune(anchor)] + pub out_channels: usize, + #[autotune(anchor)] + pub height: usize, + #[autotune(anchor)] + pub width: usize, + #[autotune(anchor)] + pub batch_size: usize, + pub has_bias: bool, + pub dtype: DType, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cross.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cross.rs new file mode 100644 index 0000000..21071b6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/cross.rs @@ -0,0 +1,101 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, broadcast_shape, linear_view, linear_view_ref}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use cubecl::std::tensor::layout::linear::LinearView; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn cross_kernel( + lhs: &LinearView>, + rhs: &LinearView>, + output: &mut LinearView, ReadWrite>, + #[define(E)] _dtype: StorageType, +) { + // Each thread processes one 3-element vector + let vector_idx = ABSOLUTE_POS; + let base_pos = vector_idx * 3; + + if !output.is_in_bounds(base_pos) { + terminate!(); + } + + // Extract vectors + let a0 = lhs[base_pos]; + let a1 = lhs[base_pos + 1]; + let a2 = lhs[base_pos + 2]; + let b0 = rhs[base_pos]; + let b1 = rhs[base_pos + 1]; + let b2 = rhs[base_pos + 2]; + + // Compute cross product: a × b + let x = a1 * b2 - a2 * b1; + let y = a2 * b0 - a0 * b2; + let z = a0 * b1 - a1 * b0; + + // Store result + output[base_pos] = x; + output[base_pos + 1] = y; + output[base_pos + 2] = z; +} + +pub(crate) fn cross( + lhs: CubeTensor, + rhs: CubeTensor, + dim: usize, +) -> CubeTensor { + let ndims = lhs.meta.num_dims(); + + // Validate that the cross dimension has size 3 + if lhs.meta.shape()[dim] != 3 || rhs.meta.shape()[dim] != 3 { + panic!( + "Cross product requires dimension {} to have size 3, but got {} and {}", + dim, + lhs.meta.shape()[dim], + rhs.meta.shape()[dim] + ); + } + + // For now, only support cross on the last dimension + if dim != ndims - 1 { + unimplemented!( + "Cross product on non-last dimension not yet implemented for CubeCL backend" + ); + } + + let output_shape = broadcast_shape(&[&lhs, &rhs]); + + // Since the cross dimension is forced to be size 3, line size would be restricted to 1 anyway + let line_size = 1; + + let output = empty_device_dtype( + lhs.client.clone(), + lhs.device.clone(), + output_shape.clone(), + lhs.dtype, + ); + + // Number of vectors to process + let num_vectors = output_shape.num_elements() / 3; + + let cube_dim = CubeDim::new(&lhs.client, num_vectors); + let cube_count = calculate_cube_count_elemwise(&lhs.client, num_vectors, cube_dim); + + unsafe { + cross_kernel::launch_unchecked( + &lhs.client, + cube_count, + cube_dim, + address_type!(lhs, rhs, output), + linear_view_ref(&lhs, &output, line_size), + linear_view_ref(&rhs, &output, line_size), + linear_view(&output, line_size), + lhs.dtype.into(), + ) + .expect("Kernel to never fail"); + }; + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/base.rs new file mode 100644 index 0000000..17630f5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/base.rs @@ -0,0 +1,163 @@ +use cubecl::prelude::*; + +use crate::{CubeRuntime, tensor::CubeTensor}; +use burn_backend::ops::{GridSampleOptions, GridSamplePaddingMode, InterpolateMode}; + +use super::bilinear::grid_sample_bilinear_launch; + +/// Grid sample operation supporting bilinear interpolation +pub fn grid_sample( + input: CubeTensor, + grid: CubeTensor, + options: GridSampleOptions, +) -> CubeTensor { + match options.mode { + InterpolateMode::Bilinear => grid_sample_bilinear_launch(input, grid, options), + _ => panic!( + "Unsupported grid_sample interpolation mode: {:?}", + options.mode + ), + } +} + +/// Compile-time padding mode for kernel specialization +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum PaddingMode { + /// Fill with zeros for out-of-bounds coordinates. + Zeros, + /// Clamp coordinates to the border (use nearest edge value). + Border, + /// Reflect coordinates at the boundary. + Reflection, +} + +impl From for PaddingMode { + fn from(mode: GridSamplePaddingMode) -> Self { + match mode { + GridSamplePaddingMode::Zeros => PaddingMode::Zeros, + GridSamplePaddingMode::Border => PaddingMode::Border, + GridSamplePaddingMode::Reflection => PaddingMode::Reflection, + } + } +} + +/// Fetch value based on padding mode (dispatch to appropriate handler) +#[cube] +pub(crate) fn fetch_value( + input: &Tensor, + base: usize, + stride_h: usize, + stride_w: usize, + y: i32, + x: i32, + h: i32, + w: i32, + #[comptime] padding_mode: PaddingMode, +) -> F { + match padding_mode { + PaddingMode::Zeros => fetch_with_zeros(input, base, stride_h, stride_w, y, x, h, w), + PaddingMode::Border => fetch_with_border(input, base, stride_h, stride_w, y, x, h, w), + PaddingMode::Reflection => { + fetch_with_reflection(input, base, stride_h, stride_w, y, x, h, w) + } + } +} + +/// Fetch value with zeros padding (return 0 for out-of-bounds). +#[cube] +pub(crate) fn fetch_with_zeros( + input: &Tensor, + base: usize, + stride_h: usize, + stride_w: usize, + y: i32, + x: i32, + h: i32, + w: i32, +) -> F { + let in_bounds = x >= 0 && x < w && y >= 0 && y < h; + let x_clamped = clamp(x, 0, w - 1) as usize; + let y_clamped = clamp(y, 0, h - 1) as usize; + let idx = base + y_clamped * stride_h + x_clamped * stride_w; + select(in_bounds, input[idx], F::new(0.0)) +} + +/// Fetch value with border padding (clamp to edge). +#[cube] +pub(crate) fn fetch_with_border( + input: &Tensor, + base: usize, + stride_h: usize, + stride_w: usize, + y: i32, + x: i32, + h: i32, + w: i32, +) -> F { + let x_clamped = clamp(x, 0, w - 1) as usize; + let y_clamped = clamp(y, 0, h - 1) as usize; + let idx = base + y_clamped * stride_h + x_clamped * stride_w; + input[idx] +} + +/// Fetch value with reflection padding. +/// Assumes float reflection was applied to center, so indices are at most 2 steps out of bounds. +#[cube] +pub(crate) fn fetch_with_reflection( + input: &Tensor, + base: usize, + stride_h: usize, + stride_w: usize, + y: i32, + x: i32, + h: i32, + w: i32, +) -> F { + let x_reflected = reflect_coord_bounded(x, w); + let y_reflected = reflect_coord_bounded(y, h); + let idx = base + y_reflected * stride_h + x_reflected * stride_w; + input[idx] +} + +/// Reflect an integer index that may be out of bounds. +/// After float reflection, indices can be up to 2 steps out for bicubic (1 step for bilinear). +#[cube] +fn reflect_coord_bounded(idx: i32, size: i32) -> usize { + let max_idx = size - 1; + let neg_reflected = -idx - 1; + let pos_reflected = 2 * max_idx + 1 - idx; + let result = select( + idx < 0, + neg_reflected, + select(idx > max_idx, pos_reflected, idx), + ); + clamp(result, 0, max_idx) as usize +} + +/// Reflect a float coordinate into the valid sampling range. +#[cube] +pub(crate) fn reflect_coord(coord: F, size: u32, #[comptime] align_corners: bool) -> F { + let size_f = F::cast_from(size); + if align_corners { + reflect_float_impl::(coord, F::new(0.0), size_f - F::new(1.0)) + } else { + reflect_float_impl::(coord, F::new(-0.5), size_f - F::new(0.5)) + } +} + +/// Reflect a float coordinate into [min_val, max_val] using a triangle wave pattern. +#[cube] +fn reflect_float_impl(coord: F, min_val: F, max_val: F) -> F { + let span = max_val - min_val; + + let is_valid = span > F::new(0.0); + let safe_span = select(is_valid, span, F::new(1.0)); + + // Triangle wave formula: span - |((x mod 2*span) - span)| + min_val + let period = safe_span * F::new(2.0); + let x = (coord - min_val).abs(); + let x_mod = x - (x / period).floor() * period; + let reflected = safe_span - (x_mod - safe_span).abs() + min_val; + + select(is_valid, reflected, min_val) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/bilinear.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/bilinear.rs new file mode 100644 index 0000000..fa7edc9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/bilinear.rs @@ -0,0 +1,177 @@ +use cubecl::std::{FastDivmod, FastDivmodArgs}; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +use crate::{ + CubeRuntime, kernel::utils::address_type, ops::numeric::empty_device_dtype, tensor::CubeTensor, +}; +use burn_backend::{Shape, ops::GridSampleOptions}; + +use super::base::{PaddingMode, fetch_value, reflect_coord}; + +/// Grid sample with bilinear interpolation. +/// +/// Each thread processes all channels for one spatial output position: +/// 1. Reading (x, y) coordinates from the grid tensor (once per spatial position) +/// 2. Converting normalized [-1, 1] coords to pixel coordinates (once) +/// 3. For each channel: fetch 4 corner values, interpolate, and write output +#[cube(launch, address_type = "dynamic")] +fn grid_sample_bilinear_kernel( + input: &Tensor, // [N, C, H_in, W_in] + grid: &Tensor, // [N, H_out, W_out, 2] + output: &mut Tensor, // [N, C, H_out, W_out] + shape_spatial: Sequence>, // [N, H_out, W_out] for thread decomposition + #[comptime] align_corners: bool, + #[comptime] pad_mode: PaddingMode, + #[define(F)] _dtype: StorageType, +) { + // Thread index maps to spatial position (n, h_out, w_out) only + let spatial_idx = ABSOLUTE_POS; + let num_spatial = output.shape(0) * output.shape(2) * output.shape(3); + if spatial_idx >= num_spatial { + terminate!(); + } + + // Decompose spatial index into (n, h_out, w_out) + let (rem, w_out) = shape_spatial[2].div_mod(spatial_idx); + let (n, h_out) = shape_spatial[1].div_mod(rem); + + let channels = input.shape(1) as u32; + let h_in = input.shape(2) as u32; + let w_in = input.shape(3) as u32; + + // Read grid coordinates once per spatial position + let grid_offset = n * grid.stride(0) + h_out * grid.stride(1) + w_out * grid.stride(2); + let gx = grid[grid_offset]; // x coordinate in [-1, 1] + let gy = grid[grid_offset + 1]; // y coordinate in [-1, 1] + + // Convert normalized coordinates to pixel coordinates + let (px, py) = if align_corners { + let px = (gx + F::new(1.0)) * F::cast_from((w_in - 1) as f32) / F::new(2.0); + let py = (gy + F::new(1.0)) * F::cast_from((h_in - 1) as f32) / F::new(2.0); + (px, py) + } else { + let px = (gx + F::new(1.0)) * F::cast_from(w_in as f32) / F::new(2.0) - F::new(0.5); + let py = (gy + F::new(1.0)) * F::cast_from(h_in as f32) / F::new(2.0) - F::new(0.5); + (px, py) + }; + + // For reflection padding, reflect the coordinate into the valid sampling range. + // This ensures integer indices are at most 1 step out of bounds. + let (px, py) = if comptime!(pad_mode == PaddingMode::Reflection) { + let px = reflect_coord::(px, w_in, align_corners); + let py = reflect_coord::(py, h_in, align_corners); + (px, py) + } else { + (px, py) + }; + + // Compute floor and ceil indices + let x0_f = px.floor(); + let y0_f = py.floor(); + let x1_f = x0_f + F::new(1.0); + let y1_f = y0_f + F::new(1.0); + + // Compute interpolation weights + let wx = px - x0_f; + let wy = py - y0_f; + let wx_ = F::new(1.0) - wx; + let wy_ = F::new(1.0) - wy; + + // Convert to integers for indexing + let x0 = i32::cast_from(x0_f); + let y0 = i32::cast_from(y0_f); + let x1 = i32::cast_from(x1_f); + let y1 = i32::cast_from(y1_f); + + let w_in = w_in as i32; + let h_in = h_in as i32; + + // Pre-compute strides + let stride_n = input.stride(0); + let stride_c = input.stride(1); + let stride_h = input.stride(2); + let stride_w = input.stride(3); + let out_stride_n = output.stride(0); + let out_stride_c = output.stride(1); + let out_stride_h = output.stride(2); + let out_stride_w = output.stride(3); + + // Base offsets for this spatial position + let in_base_n = n * stride_n; + let out_base_spatial = n * out_stride_n + h_out * out_stride_h + w_out * out_stride_w; + + // Loop over all channels - grid coords and weights are reused + for c in 0..channels { + let in_base = in_base_n + c as usize * stride_c; + + let v00 = fetch_value( + input, in_base, stride_h, stride_w, y0, x0, h_in, w_in, pad_mode, + ); + let v01 = fetch_value( + input, in_base, stride_h, stride_w, y1, x0, h_in, w_in, pad_mode, + ); + let v10 = fetch_value( + input, in_base, stride_h, stride_w, y0, x1, h_in, w_in, pad_mode, + ); + let v11 = fetch_value( + input, in_base, stride_h, stride_w, y1, x1, h_in, w_in, pad_mode, + ); + + // Bilinear interpolation + let result = wx_ * wy_ * v00 + wx_ * wy * v01 + wx * wy_ * v10 + wx * wy * v11; + + let out_idx = out_base_spatial + c as usize * out_stride_c; + output[out_idx] = result; + } +} + +/// Launch the grid sample bilinear kernel +pub(crate) fn grid_sample_bilinear_launch( + input: CubeTensor, + grid: CubeTensor, + options: GridSampleOptions, +) -> CubeTensor { + let [batch_size, channels, _h_in, _w_in] = input.meta.shape().dims(); + let [_n, h_out, w_out, two] = grid.meta.shape().dims(); + assert_eq!(two, 2, "Grid last dimension must be 2"); + + // Create output tensor [N, C, H_out, W_out] + let output_shape = Shape::new([batch_size, channels, h_out, w_out]); + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + output_shape, + input.dtype, + ); + + // Spatial threading: one thread per (n, h_out, w_out) + let spatial_shape = Shape::new([batch_size, h_out, w_out]); + let num_spatial = spatial_shape.num_elements(); + + let mut shape_spatial = SequenceArg::new(); + for dim in spatial_shape.iter() { + shape_spatial.push(FastDivmodArgs::new(&input.client, *dim)); + } + + let cube_dim = CubeDim::new(&input.client, num_spatial); + let cube_count = calculate_cube_count_elemwise(&input.client, num_spatial, cube_dim); + + let padding_mode: PaddingMode = options.padding_mode.into(); + + grid_sample_bilinear_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, grid, output), + input.as_tensor_arg(1), + grid.as_tensor_arg(1), + output.as_tensor_arg(1), + shape_spatial, + options.align_corners, + padding_mode, + input.dtype.into(), + ) + .expect("Grid sample kernel failed"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/mod.rs new file mode 100644 index 0000000..7bd74df --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/grid_sample/mod.rs @@ -0,0 +1,4 @@ +mod base; +mod bilinear; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/flip.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/flip.rs new file mode 100644 index 0000000..5bbfb0a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/flip.rs @@ -0,0 +1,99 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, shape_divmod}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use burn_backend::{DType, TensorMetadata}; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::layout::linear::LinearView}, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn flip_kernel( + input: &Tensor, + output: &mut LinearView, + in_shape: Sequence>, + indices: Sequence, + #[define(E, Bool)] _dtypes: [StorageType; 2], +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let rank = in_shape.len().comptime(); + + let mut offset = ABSOLUTE_POS; + let mut offset_input = 0; + + #[unroll] + for i in 0..rank { + let dim = rank - i - 1; + let shape = input.shape(dim); + + let (rem, offset_local) = in_shape[dim].div_mod(offset); + offset = rem; + + let flip = indices.index(dim).get::() == Bool::from_int(1); + let offset_local = select(flip, shape - offset_local - 1, offset_local); + + offset_input += offset_local * input.stride(dim); + } + + output[ABSOLUTE_POS] = input[offset_input]; +} + +pub(crate) fn flip( + tensor: CubeTensor, + indices: &[usize], + dtype_bool: DType, +) -> CubeTensor { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + tensor.dtype, + ); + flip_on_output(tensor, output, indices, dtype_bool) +} + +pub(crate) fn flip_on_output( + tensor: CubeTensor, + output: CubeTensor, + indices: &[usize], + dtype_bool: DType, +) -> CubeTensor { + let dtype_input = tensor.dtype; + let ndims = tensor.meta.num_dims(); + let mut indices_sequence = SequenceArg::<'_, R, InputScalar>::new(); + + for i in 0..ndims { + indices_sequence.push({ + let val = indices.contains(&i) as u8; + InputScalar::new(val, dtype_bool) + }); + } + + let num_elements = output.meta.num_elements(); + let cube_dim = CubeDim::new(&tensor.client, num_elements); + let cube_count = calculate_cube_count_elemwise(&tensor.client, num_elements, cube_dim); + + unsafe { + flip_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, output), + tensor.as_tensor_arg(1), + linear_view(&output, 1), + shape_divmod(&tensor), + indices_sequence, + [dtype_input.into(), dtype_bool.into()], + ) + .expect("Kernel to never fail"); + } + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/gather.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/gather.rs new file mode 100644 index 0000000..163f6e9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/gather.rs @@ -0,0 +1,76 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, broadcast_strides, linear_view, shape_divmod}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use burn_backend::TensorMetadata; +use cubecl::frontend::{ABSOLUTE_POS, Numeric, Tensor}; +use cubecl::std::{FastDivmod, tensor::index_offset_contiguous_fastdivmod}; +use cubecl::{CubeDim, std::tensor::layout::linear::LinearView}; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn gather_kernel( + input: &Tensor>, + indices: &LinearView>, + output: &mut LinearView, ReadWrite>, + in_strides: Sequence, // zeroed out for broadcast dims and `dim` + out_shape: Sequence>, + dim: usize, + #[define(T, I)] _dtypes: [StorageType; 2], +) { + if !indices.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let mut offset = index_offset_contiguous_fastdivmod( + ABSOLUTE_POS, + &out_shape, + &in_strides, + input.line_size(), + ); + + offset += usize::cast_from(indices[ABSOLUTE_POS]) * input.stride(dim); + + output[ABSOLUTE_POS] = input[offset]; +} + +pub(crate) fn gather( + dim: usize, + tensor: CubeTensor, + indices: CubeTensor, +) -> CubeTensor { + let shape_output = indices.shape(); + let total_elem = shape_output.num_elements(); + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + shape_output, + tensor.dtype, + ); + + let cube_dim = CubeDim::new(&tensor.client, total_elem); + let cube_count = calculate_cube_count_elemwise(&tensor.client, total_elem, cube_dim); + let mut in_strides = broadcast_strides(&output, &tensor); + in_strides.values[dim] = ScalarArg::new(0); // Zero `dim` to exclude it from the indexing + + unsafe { + gather_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, indices, output), + tensor.as_tensor_arg(1), + linear_view(&indices, 1), + linear_view(&output, 1), + in_strides, + shape_divmod(&output), + ScalarArg::new(dim), + [tensor.dtype.into(), indices.dtype.into()], + ) + .expect("Kernel to never fail"); + } + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/mod.rs new file mode 100644 index 0000000..83ce64a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/mod.rs @@ -0,0 +1,18 @@ +mod flip; +mod gather; +mod repeat_dim; +mod scatter; +mod select; +mod select_assign; +mod slice; +mod slice_assign; + +pub(crate) use flip::*; +pub(crate) use repeat_dim::*; +pub(crate) use select::*; +pub(crate) use select_assign::*; +pub use slice::*; +pub(crate) use slice_assign::*; + +pub(crate) use gather::*; +pub(crate) use scatter::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/repeat_dim.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/repeat_dim.rs new file mode 100644 index 0000000..452b46d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/repeat_dim.rs @@ -0,0 +1,93 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, shape_divmod}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, FastDivmodArgs}, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn repeat_dim_kernel( + input: &Tensor, + output: &mut Tensor, + out_shape: Sequence>, + in_shape: FastDivmod, + #[comptime] dim: usize, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= output.len() { + terminate!(); + } + + let rank = out_shape.len().comptime(); + + let mut pos = ABSOLUTE_POS; + let mut offset_input = 0; + let mut offset_output = 0; + + #[unroll] + for i in 0..rank { + let i = rank - i - 1; + + let (rem, mut local_pos) = out_shape[i].div_mod(pos); + pos = rem; + + offset_output += local_pos * output.stride(i); + + if i == dim { + local_pos = in_shape.modulo(local_pos); + } + + offset_input += local_pos * input.stride(i); + } + + output[offset_output] = input[offset_input]; +} + +pub(crate) fn repeat_dim( + mut input: CubeTensor, + dim: usize, + times: usize, +) -> CubeTensor { + if input.meta.shape()[dim] == 1 { + input.meta.strides[dim] = 0; + input.meta.shape = input.meta.shape.repeat(dim, times).unwrap(); + return input; + } + + let shape = input.meta.shape.clone().repeat(dim, times).unwrap(); + + // Create output handle + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + shape, + input.dtype, + ); + + let working_units = output.meta.num_elements(); + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + unsafe { + repeat_dim_kernel::launch_unchecked( + &input.client, + cube_count, + cube_dim, + address_type!(input, output), + input.as_tensor_arg(1), + output.as_tensor_arg(1), + shape_divmod(&output), + FastDivmodArgs::new(&input.client, input.meta.shape()[dim]), + dim, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + }; + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/scatter.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/scatter.rs new file mode 100644 index 0000000..e80dedc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/scatter.rs @@ -0,0 +1,109 @@ +use crate::{ + CubeRuntime, + kernel::{ + AddOp, BinaryOp, BinaryOpFamily, OrOp, + utils::{address_type, shape_divmod}, + }, + tensor::CubeTensor, +}; +use cubecl::{CubeDim, calculate_cube_count_elemwise}; +use cubecl::{prelude::*, std::FastDivmod}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn scatter_kernel( + input: &mut Tensor, + indices: &Tensor, + value: &Tensor, + in_shape: Sequence>, + #[comptime] dim: usize, + #[define(T, I)] _dtypes: [StorageType; 2], +) { + let rank = in_shape.len().comptime(); + let stride_input = input.stride(dim); + let stride_value = value.stride(dim); + let stride_indices = indices.stride(dim); + let shape_value = value.shape(dim); + + let mut offset = ABSOLUTE_POS; + let mut offset_input = 0; + let mut offset_indices = 0; + let mut offset_value = 0; + let mut num_elems = 1; + + #[unroll] + for i in 0..rank { + let i = rank - i - 1; + if i != dim { + let shape_input_loop = input.shape(i); + + let (rem, local_pos) = in_shape[i].div_mod(offset); + offset = rem; + + offset_input += local_pos * input.stride(i); + offset_indices += local_pos * indices.stride(i); + offset_value += local_pos * value.stride(i); + + num_elems *= shape_input_loop; + } + } + + let should_stop = ABSOLUTE_POS >= num_elems; + if should_stop { + terminate!(); + } + + for i in 0..shape_value { + let value_idx = (stride_value * i) + offset_value; + let index_idx = (stride_indices * i) + offset_indices; + + let value = value[value_idx]; + let index = usize::cast_from(indices[index_idx]); + + let input_idx = (stride_input * index) + offset_input; + + let value = + Op::BinaryOp::::execute(Line::cast_from(input[input_idx]), Line::cast_from(value)); + input[input_idx] = value[0]; + } +} + +pub(crate) fn scatter( + dim: usize, + tensor: CubeTensor, + indices: CubeTensor, + value: CubeTensor, + is_bool: bool, +) -> CubeTensor { + let tensor = match tensor.can_mut() && tensor.is_nonoverlapping() { + true => tensor, + false => tensor.copy(), + }; + + let num_elems = tensor.meta.num_elements() / tensor.meta.shape()[dim]; + + let working_units = num_elems; + let cube_dim = CubeDim::new(&indices.client, working_units); + let cube_count = calculate_cube_count_elemwise(&indices.client, working_units, cube_dim); + + let launch = match is_bool { + true => scatter_kernel::launch_unchecked::, + false => scatter_kernel::launch_unchecked::, + }; + + unsafe { + launch( + &indices.client.clone(), + cube_count, + cube_dim, + address_type!(tensor, indices, value), + tensor.as_tensor_arg(1), + indices.as_tensor_arg(1), + value.as_tensor_arg(1), + shape_divmod(&tensor), + dim, + [tensor.dtype.into(), indices.dtype.into()], + ) + .expect("Kernel to never fail"); + } + tensor +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select.rs new file mode 100644 index 0000000..14b40a8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select.rs @@ -0,0 +1,82 @@ +use crate::{CubeRuntime, kernel::utils::address_type, tensor::CubeTensor}; +use crate::{ + kernel::utils::{linear_view, shape_divmod}, + ops::numeric::empty_device_dtype, +}; +use burn_backend::TensorMetadata; +use cubecl::{CubeDim, calculate_cube_count_elemwise, std::tensor::layout::linear::LinearView}; +use cubecl::{prelude::*, std::FastDivmod}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn select_kernel( + input: &Tensor, + indices: &LinearView, + output: &mut LinearView, + out_shape: Sequence>, + dim: usize, + #[define(T, I)] _dtypes: [StorageType; 2], +) { + if ABSOLUTE_POS >= output.shape() { + terminate!(); + } + + let rank = out_shape.len().comptime(); + + let mut offset = ABSOLUTE_POS; + let mut offset_input = 0; + + #[unroll] + for i in 0..rank { + let i = rank - i - 1; + let (rem, offset_local) = out_shape[i].div_mod(offset); + offset = rem; + + let offset_local = cubecl::prelude::select( + i == dim, + usize::cast_from(indices[offset_local]), + offset_local, + ); + + offset_input += offset_local * input.stride(i); + } + + output[ABSOLUTE_POS] = input[offset_input]; +} + +pub(crate) fn select( + tensor: CubeTensor, + dim: usize, + indices: CubeTensor, +) -> CubeTensor { + let mut shape_output = tensor.shape(); + shape_output[dim] = indices.meta.shape()[0]; + let total_elem = shape_output.num_elements(); + + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + shape_output, + tensor.dtype, + ); + + let working_units = total_elem; + let cube_dim = CubeDim::new(&indices.client, working_units); + let cube_count = calculate_cube_count_elemwise(&indices.client, working_units, cube_dim); + + unsafe { + select_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, indices, output), + tensor.as_tensor_arg(1), + linear_view(&indices, 1), + linear_view(&output, 1), + shape_divmod(&output), + ScalarArg::new(dim), + [tensor.dtype.into(), indices.dtype.into()], + ) + .expect("Kernel to never fail"); + }; + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select_assign.rs new file mode 100644 index 0000000..821bcb2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/select_assign.rs @@ -0,0 +1,98 @@ +use crate::kernel::{ + AddOp, BinaryOp, BinaryOpFamily, OrOp, + utils::{address_type, linear_view, shape_divmod}, +}; +use crate::{CubeRuntime, tensor::CubeTensor}; +use cubecl::{CubeDim, calculate_cube_count_elemwise, std::tensor::layout::linear::LinearView}; +use cubecl::{prelude::*, std::FastDivmod}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn select_assign_kernel( + tensor: &mut Tensor, + indices: &LinearView, + value: &Tensor, + value_shape: Sequence>, + num_elems: usize, + #[comptime] dim: usize, + #[define(F, I)] _dtypes: [StorageType; 2], +) { + if ABSOLUTE_POS >= num_elems { + terminate!(); + } + + let rank = value_shape.len().comptime(); + + let mut offset = ABSOLUTE_POS; + let mut offset_tensor = 0; + let mut offset_value = 0; + + // Calculate offsets and num_elems + #[unroll] + for i in 0..rank { + let i = rank - i - 1; + if i != dim { + let (rem, local_pos) = value_shape[i].div_mod(offset); + offset = rem; + + offset_tensor += local_pos * tensor.stride(i); + offset_value += local_pos * value.stride(i); + } + } + + let strides_tensor_dim = tensor.stride(dim); + let strides_value_dim = value.stride(dim); + + // Main operation + for i in 0..value.shape(dim) { + let index_tensor = usize::cast_from(indices[i]) * strides_tensor_dim + offset_tensor; + let index_value = i * strides_value_dim + offset_value; + + let value = Op::BinaryOp::::execute( + Line::cast_from(tensor[index_tensor]), + Line::cast_from(value[index_value]), + ); + tensor[index_tensor] = F::cast_from(value); + } +} + +pub(crate) fn select_assign( + tensor: CubeTensor, + dim: usize, + indices: CubeTensor, + value: CubeTensor, + is_bool: bool, +) -> CubeTensor { + let tensor = match tensor.can_mut() && tensor.is_nonoverlapping() { + true => tensor, + false => tensor.copy(), + }; + + let num_elems = tensor.meta.num_elements() / tensor.meta.shape()[dim]; + let working_units = num_elems; + let cube_dim = CubeDim::new(&indices.client, working_units); + let cube_count = calculate_cube_count_elemwise(&indices.client, working_units, cube_dim); + + let launch = match is_bool { + true => select_assign_kernel::launch_unchecked::, + false => select_assign_kernel::launch_unchecked::, + }; + + unsafe { + launch( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, indices, value), + tensor.as_tensor_arg(1), + linear_view(&indices, 1), + value.as_tensor_arg(1), + shape_divmod(&value), + ScalarArg::new(num_elems), + dim, + [tensor.dtype.into(), indices.dtype.into()], + ) + .expect("Kernel to never fail"); + }; + + tensor +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice.rs new file mode 100644 index 0000000..f371e39 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice.rs @@ -0,0 +1,247 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, shape_divmod}, + ops::numeric::empty_device_dtype, + tensor::CubeTensor, +}; +use burn_backend::{Slice, TensorMetadata}; +use burn_std::{Metadata, SliceOps}; +use cubecl::{ + calculate_cube_count_elemwise, intrinsic, + prelude::*, + std::{FastDivmod, tensor::layout::linear::LinearView}, +}; +use std::ops::Range; + +/// Slice a jit tensor with a set of ranges +pub fn slice(tensor: CubeTensor, indices: &[Range]) -> CubeTensor { + let mut dims = tensor.shape(); + let mut offset_start = 0u64; + let mut offset_end = 0u64; + + for i in 0..indices.len() { + offset_start += (tensor.meta.strides()[i] * indices[i].start) as u64; + offset_end += (tensor.meta.strides()[i] * (dims[i] - indices[i].end)) as u64; + dims[i] = indices[i].end - indices[i].start; + } + + let offset_start = offset_start * tensor.dtype.size() as u64; + let offset_end = offset_end * tensor.dtype.size() as u64; + + let memory_offset_alignment = tensor.client.properties().memory.alignment; + + if offset_start.is_multiple_of(memory_offset_alignment) + && offset_end.is_multiple_of(memory_offset_alignment) + { + CubeTensor::new( + tensor.client, + tensor + .handle + .offset_start(offset_start) + .offset_end(offset_end), + Metadata::new(dims, tensor.meta.strides), + tensor.device, + tensor.dtype, + ) + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + dims, + tensor.dtype, + ); + slice_on_output(tensor, output, indices) + } +} + +#[cube(launch_unchecked, address_type = "dynamic")] +fn slice_kernel( + input: &Tensor, + output: &mut LinearView, + out_shape: Sequence>, + indices: Sequence, + #[define(E)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let rank = comptime![out_shape.len()]; + let mut offset_output = ABSOLUTE_POS; + let mut offset_input = 0; + + #[unroll] + for i in 0..rank { + // Iterate in reverse to use divmod + let dim = rank - i - 1; + + let range_start = indices[dim]; + let (rem, offset_local) = out_shape[dim].div_mod(offset_output); + offset_output = rem; + + let offset_local = offset_local + range_start; + + offset_input += offset_local * input.stride(dim); + } + + output[ABSOLUTE_POS] = input[offset_input]; +} + +pub(crate) fn slice_on_output( + tensor: CubeTensor, + output: CubeTensor, + indices: &[Range], +) -> CubeTensor { + let ndims = tensor.meta.num_dims(); + let mut indices_sequence = SequenceArg::::new(); + + for i in 0..ndims { + let start = indices.get(i).map(|index| index.start).unwrap_or(0); + indices_sequence.push(ScalarArg::new(start)); + } + + let working_units = output.meta.num_elements(); + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + slice_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, output), + tensor.as_tensor_arg(1), + linear_view(&output, 1), + shape_divmod(&output), + indices_sequence, + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + }; + + output +} + +/// Kernel for slicing with steps +#[cube(launch_unchecked, address_type = "dynamic")] +fn slice_with_steps_kernel( + input: &Tensor, + output: &mut LinearView, + out_shape: Sequence>, + starts: Sequence, + ends: Sequence, + steps: Sequence, + #[define(E)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let rank = comptime![out_shape.len()]; + let mut output_offset = ABSOLUTE_POS; + let mut input_offset = 0; + + // Calculate the input offset based on output position and slice info + #[unroll] + for i in 0..rank { + // Iterate in reverse to use divmod + let dim = rank - i - 1; + let start = starts[dim]; + let end = ends[dim]; + let step = steps[dim]; + + let (rem, output_idx) = out_shape[dim].div_mod(output_offset); + output_offset = rem; + + let input_idx = if step > 0 { + // Forward stepping + start + output_idx * (step as usize) + } else { + // Backward stepping - start from end-1 + let abs_step = (-step) as usize; + let end_minus_1 = end - 1; + end_minus_1 - output_idx * abs_step + }; + + input_offset += input_idx * input.stride(dim); + } + + output[ABSOLUTE_POS] = input[input_offset]; +} + +/// Slice a tensor with steps +pub fn slice_with_steps(tensor: CubeTensor, slices: &[Slice]) -> CubeTensor { + // Check if all steps are 1 - if so, use the optimized regular slice + let all_steps_one = slices.iter().all(|info| info.step == 1); + + if all_steps_one { + // Convert Slice to Range for step=1 + let simple_ranges: Vec> = slices + .iter() + .enumerate() + .map(|(i, slice)| slice.to_range(tensor.meta.shape()[i])) + .collect(); + return slice(tensor, &simple_ranges); + } + + // Calculate output shape + let shape_output = tensor.shape().slice(slices).unwrap(); + + // Create output tensor + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + shape_output.clone(), + tensor.dtype, + ); + + // Prepare three separate sequences for kernel + let mut starts = SequenceArg::::new(); + let mut ends = SequenceArg::::new(); + let mut steps = SequenceArg::::new(); + + for (dim, slice) in slices.iter().enumerate() { + let range = slice.to_range(tensor.meta.shape()[dim]); + starts.push(ScalarArg::new(range.start)); + ends.push(ScalarArg::new(range.end)); + steps.push(ScalarArg::new(slice.step as i32)); + } + + // Pad with default values if needed to match tensor dimensions + for dim in slices.len()..tensor.meta.num_dims() { + starts.push(ScalarArg::new(0)); + ends.push(ScalarArg::new(tensor.meta.shape()[dim])); + steps.push(ScalarArg::new(1)); + } + + // Launch kernel + let working_units = shape_output.num_elements(); + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + slice_with_steps_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, output), + tensor.as_tensor_arg(1), + linear_view(&output, 1), + shape_divmod(&output), + starts, + ends, + steps, + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + } + + output +} + +/// This is annoying and we need to find a way to do this automatically at some point +#[allow(unused)] +#[cube] +fn unwrap(value: u32) -> comptime_type!(u32) { + intrinsic!(|_| value.constant().unwrap().as_u32()) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice_assign.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice_assign.rs new file mode 100644 index 0000000..e0851b3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/index/slice_assign.rs @@ -0,0 +1,258 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, shape_divmod}, + tensor::CubeTensor, +}; +use cubecl::{ + calculate_cube_count_elemwise, intrinsic, + prelude::*, + std::{FastDivmod, FastDivmodArgs, tensor::layout::linear::LinearView}, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn slice_assign_kernel( + input: &mut Tensor>, + value: &LinearView>, + slice_shape: Sequence>, + slice_offsets: Sequence, + #[define(E)] _dtype: StorageType, +) { + if !value.is_in_bounds(ABSOLUTE_POS) { + terminate!() + } + + let rank = comptime!(slice_shape.len()); + + let line_size = input.line_size(); + let mut offset_remainder = ABSOLUTE_POS * line_size; + let mut offset_input = 0; + + #[allow(clippy::explicit_counter_loop)] + #[unroll] + for i in 0..rank { + let dim = rank - i - 1; + let (rem, offset_local) = slice_shape[dim].div_mod(offset_remainder); + + let range_start = slice_offsets[dim]; + let offset_local_input = offset_local + range_start; + + offset_input += offset_local_input * input.stride(dim); + offset_remainder = rem; + } + + // Value tensor is accessed linearly since it's a LinearView + input[offset_input / line_size] = value[ABSOLUTE_POS]; +} + +/// Kernel for slice assign with steps +#[cube(launch_unchecked, address_type = "dynamic")] +fn slice_assign_with_steps_kernel( + input: &mut Tensor, + value: &LinearView, + value_shape: Sequence>, + starts: Sequence, + ends: Sequence, + steps: Sequence, + #[define(E)] _dtype: StorageType, +) { + if !value.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let rank = comptime![value_shape.len()]; + let mut value_offset = ABSOLUTE_POS; + let mut input_offset = 0; + + // Calculate the input offset based on value position and slice info + #[unroll] + for i in 0..rank { + // Iterate in reverse to use divmod + let dim = rank - i - 1; + let start = starts[dim]; + let end = ends[dim]; + let step = steps[dim]; + + let (rem, value_idx) = value_shape[dim].div_mod(value_offset); + value_offset = rem; + + let input_idx = if step > 0 { + // Forward stepping + start + value_idx * (step as usize) + } else if step < 0 { + // Backward stepping - start from end-1 + // For negative steps, we iterate backwards through the selected indices + let abs_step = (-step) as usize; + let end_minus_1 = end - 1; + end_minus_1 - value_idx * abs_step + } else { + // step == 0, shouldn't happen + value_idx + }; + + input_offset += input_idx * input.stride(dim); + } + + input[input_offset] = value[ABSOLUTE_POS]; +} + +pub(crate) fn slice_assign( + tensor: CubeTensor, + indices: &[burn_backend::Slice], + value: CubeTensor, +) -> CubeTensor { + // Check if any slice has non-unit step + let has_non_unit_step = indices.iter().any(|s| s.step != 1 && s.step != 0); + + if has_non_unit_step { + // Use slice_assign_with_steps + return slice_assign_with_steps(tensor, indices, value); + } + + let client = tensor.client.clone(); + let tensor = match tensor.can_mut() && tensor.is_nonoverlapping() { + true => tensor, + false => tensor.copy(), + }; + let ndims = tensor.meta.num_dims(); + + let line_size = if tensor.meta.strides()[ndims - 1] == 1 && value.meta.strides()[ndims - 1] == 1 + { + let last = indices + .get(ndims - 1) + .cloned() + .unwrap_or(burn_backend::Slice { + start: 0, + end: Some(tensor.meta.shape()[ndims - 1] as isize), + step: 1, + }); + let end = last.end.unwrap_or(tensor.meta.shape()[ndims - 1] as isize); + let shape = (end - last.start) as usize; + let offset = last.start as usize; + client + .io_optimized_line_sizes(tensor.dtype.size()) + .filter(|&it| { + shape.is_multiple_of(it) + && strides_compatible(tensor.meta.strides(), it) + && strides_compatible(value.meta.strides(), it) + && offset.is_multiple_of(it) + }) + .max() + .unwrap_or(1) + } else { + 1 + }; + + let mut shape = SequenceArg::>::new(); + let mut offsets = SequenceArg::::new(); + + for i in 0..ndims { + let slice = indices.get(i).cloned().unwrap_or(burn_backend::Slice { + start: 0, + end: Some(tensor.meta.shape()[i] as isize), + step: 1, + }); + let start = slice.start as usize; + let end = slice.end.unwrap_or(tensor.meta.shape()[i] as isize); + let length = (end - slice.start) as usize; + + shape.push(FastDivmodArgs::::new(&client, length)); + offsets.push(ScalarArg::new(start)); + } + + let working_units = value.meta.num_elements() / line_size; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + slice_assign_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, value), + tensor.as_tensor_arg(line_size), + linear_view(&value, line_size), + shape, + offsets, + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + } + + tensor +} + +/// Slice assign with steps support +/// +/// This function handles slice assignment with arbitrary step values, including negative steps. +/// It follows NumPy/PyTorch semantics where values[i] is assigned to selected_indices[i]. +/// +/// For example, with s![0..6;-1] which selects indices [5,4,3,2,1,0]: +/// - values[0] goes to index 5 +/// - values[1] goes to index 4 +/// - etc. +pub(crate) fn slice_assign_with_steps( + tensor: CubeTensor, + slices: &[burn_backend::Slice], + value: CubeTensor, +) -> CubeTensor { + let tensor = match tensor.can_mut() && tensor.is_nonoverlapping() { + true => tensor, + false => tensor.copy(), + }; + + // Prepare sequences for kernel + let mut starts = SequenceArg::::new(); + let mut ends = SequenceArg::::new(); + let mut steps = SequenceArg::::new(); + + for (dim, slice) in slices.iter().enumerate() { + let range = slice.to_range(tensor.meta.shape()[dim]); + starts.push(ScalarArg::new(range.start)); + ends.push(ScalarArg::new(range.end)); + steps.push(ScalarArg::new(slice.step as i32)); + } + + // Pad with default values if needed to match tensor dimensions + for dim in slices.len()..tensor.meta.num_dims() { + starts.push(ScalarArg::new(0)); + ends.push(ScalarArg::new(tensor.meta.shape()[dim])); + steps.push(ScalarArg::new(1)); + } + + // Launch kernel + let working_units = value.meta.num_elements(); + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + slice_assign_with_steps_kernel::launch_unchecked( + &tensor.client, + cube_count, + cube_dim, + address_type!(tensor, value), + tensor.as_tensor_arg(1), + linear_view(&value, 1), + shape_divmod(&value), + starts, + ends, + steps, + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + } + + tensor +} + +fn strides_compatible(strides: &[usize], vec: usize) -> bool { + strides + .iter() + .all(|stride| *stride % vec == 0 || *stride == 1) +} + +/// Helper function for unwrap +#[allow(unused)] +#[cube] +fn unwrap(value: u32) -> comptime_type!(u32) { + intrinsic!(|_| value.constant().unwrap().as_u32()) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/base.rs new file mode 100644 index 0000000..b89feec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/base.rs @@ -0,0 +1,79 @@ +use crate::{ + CubeRuntime, + kernel::into_contiguous, + ops::{numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::{ + Shape, TensorMetadata, + ops::{InterpolateMode, InterpolateOptions}, +}; + +use super::{ + bicubic::interpolate_bicubic_launch, bilinear::interpolate_bilinear_launch, + nearest::interpolate_nearest_launch, nearest_backward::interpolate_nearest_backward_launch, +}; + +/// Interpolate operation +/// +/// Supports nearest, bilinear and bicubic modes +pub fn interpolate( + input: CubeTensor, + output_size: [usize; 2], + options: InterpolateOptions, +) -> CubeTensor { + let [batch_size, channels, _, _] = input.meta.shape().dims(); + let [out_height, out_width] = output_size; + + let input = into_contiguous(permute_nchw_to_nhwc(input)); + + let shape_out = Shape::new([batch_size, out_height, out_width, channels]); + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + shape_out, + input.dtype, + ); + + let align_corners = options.align_corners; + let output = match options.mode { + InterpolateMode::Nearest => interpolate_nearest_launch(input, output), + InterpolateMode::Bilinear => interpolate_bilinear_launch(input, output, align_corners), + InterpolateMode::Bicubic => interpolate_bicubic_launch(input, output, align_corners), + }; + + permute_nhwc_to_nchw(output) +} + +/// Backward interpolate operation +/// +/// Note: only nearest mode is supported +pub fn interpolate_backward( + input: CubeTensor, + out_grad: CubeTensor, + _output_size: [usize; 2], + options: InterpolateOptions, +) -> CubeTensor { + let input = permute_nchw_to_nhwc(input); + let out_grad = permute_nchw_to_nhwc(out_grad); + + let output_shape = input.shape(); + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + output_shape, + input.dtype, + ); + + let output = match options.mode { + InterpolateMode::Nearest => interpolate_nearest_backward_launch(out_grad, output), + InterpolateMode::Bilinear => { + panic!("bilinear interpolation backward is not supported by JIT backend") + } + InterpolateMode::Bicubic => { + panic!("bicubic interpolation backward is not supported by JIT backend") + } + }; + + permute_nhwc_to_nchw(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bicubic.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bicubic.rs new file mode 100644 index 0000000..3373de3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bicubic.rs @@ -0,0 +1,194 @@ +use cubecl::std::{ + FastDivmod, + tensor::layout::{linear::LinearLayout, *}, +}; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_layout, shape_divmod}, + ops::max_line_size, + tensor::CubeTensor, +}; + +#[cube(launch, address_type = "dynamic")] +fn interpolate_bicubic_kernel( + input: &Tensor>, + output: &mut Tensor>, + shape_out: Sequence>, + out_layout: LinearLayout, + #[comptime] align_corners: bool, + #[define(F)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= output.len() { + terminate!(); + } + + let line_size = input.line_size(); + let out_idx = out_layout.to_source_pos(ABSOLUTE_POS); + + let (rem, c) = shape_out[3].div_mod(ABSOLUTE_POS * line_size); + let (rem, x) = shape_out[2].div_mod(rem); + let (b, y) = shape_out[1].div_mod(rem); + + let input_height = input.shape(1) - 1; + let input_height_f = input_height as f32; + + let frac = if align_corners { + let output_height = clamp_min(output.shape(1) - 1, 1) as f32; + (y * input_height) as f32 / output_height + } else { + let in_size = (input_height + 1) as f32; + let out_size = output.shape(1) as f32; + (y as f32 + 0.5) * (in_size / out_size) - 0.5 + }; + let y_in_f = frac.floor(); + let yw = Line::empty(line_size).fill(F::cast_from(frac - y_in_f)); + + // Clamp indices in float space to handle negative coordinates from half_pixel + let y0 = clamp(y_in_f - 1.0, 0.0, input_height_f) as usize; + let y1 = clamp(y_in_f, 0.0, input_height_f) as usize; + let y2 = clamp(y_in_f + 1.0, 0.0, input_height_f) as usize; + let y3 = clamp(y_in_f + 2.0, 0.0, input_height_f) as usize; + + let input_width = input.shape(2) - 1; + let input_width_f = input_width as f32; + + let frac = if align_corners { + let output_width = clamp_min(output.shape(2) - 1, 1) as f32; + (x * input_width) as f32 / output_width + } else { + let in_size = (input_width + 1) as f32; + let out_size = output.shape(2) as f32; + (x as f32 + 0.5) * (in_size / out_size) - 0.5 + }; + let x_in_f = frac.floor(); + let xw = Line::empty(line_size).fill(F::cast_from(frac - x_in_f)); + + // Clamp indices in float space to handle negative coordinates from half_pixel + let x0 = clamp(x_in_f - 1.0, 0.0, input_width_f) as usize; + let x1 = clamp(x_in_f, 0.0, input_width_f) as usize; + let x2 = clamp(x_in_f + 1.0, 0.0, input_width_f) as usize; + let x3 = clamp(x_in_f + 2.0, 0.0, input_width_f) as usize; + + let index_base = b * input.stride(0) + c * input.stride(3); + let in_stride_y = input.stride(1); + let in_stride_x = input.stride(2); + + let y0_stride = y0 * in_stride_y; + let y1_stride = y1 * in_stride_y; + let y2_stride = y2 * in_stride_y; + let y3_stride = y3 * in_stride_y; + let x0_stride = x0 * in_stride_x; + let x1_stride = x1 * in_stride_x; + let x2_stride = x2 * in_stride_x; + let x3_stride = x3 * in_stride_x; + + let inp_0 = input[(index_base + y0_stride + x0_stride) / line_size]; + let inp_1 = input[(index_base + y0_stride + x1_stride) / line_size]; + let inp_2 = input[(index_base + y0_stride + x2_stride) / line_size]; + let inp_3 = input[(index_base + y0_stride + x3_stride) / line_size]; + + let coefficients0 = cubic_interp_1d::(inp_0, inp_1, inp_2, inp_3, xw); + + let inp_0 = input[(index_base + y1_stride + x0_stride) / line_size]; + let inp_1 = input[(index_base + y1_stride + x1_stride) / line_size]; + let inp_2 = input[(index_base + y1_stride + x2_stride) / line_size]; + let inp_3 = input[(index_base + y1_stride + x3_stride) / line_size]; + + let coefficients1 = cubic_interp_1d::(inp_0, inp_1, inp_2, inp_3, xw); + + let inp_0 = input[(index_base + y2_stride + x0_stride) / line_size]; + let inp_1 = input[(index_base + y2_stride + x1_stride) / line_size]; + let inp_2 = input[(index_base + y2_stride + x2_stride) / line_size]; + let inp_3 = input[(index_base + y2_stride + x3_stride) / line_size]; + + let coefficients2 = cubic_interp_1d::(inp_0, inp_1, inp_2, inp_3, xw); + + let inp_0 = input[(index_base + y3_stride + x0_stride) / line_size]; + let inp_1 = input[(index_base + y3_stride + x1_stride) / line_size]; + let inp_2 = input[(index_base + y3_stride + x2_stride) / line_size]; + let inp_3 = input[(index_base + y3_stride + x3_stride) / line_size]; + + let coefficients3 = cubic_interp_1d::(inp_0, inp_1, inp_2, inp_3, xw); + + let val = cubic_interp_1d::( + coefficients0, + coefficients1, + coefficients2, + coefficients3, + yw, + ); + + output[out_idx] = val; +} + +#[cube] +fn cubic_interp_1d( + x0: Line, + x1: Line, + x2: Line, + x3: Line, + t: Line, +) -> Line { + let a = lined(&x0, -0.75); + + let coeffs0 = cubic_convolution_2::(t + lined(&x0, 1.0), a); + let coeffs1 = cubic_convolution_1::(t, a); + let coeffs2 = cubic_convolution_1::(lined(&x0, 1.0) - t, a); + let coeffs3 = cubic_convolution_2::(lined(&x0, 2.0) - t, a); + + x0 * coeffs0 + x1 * coeffs1 + x2 * coeffs2 + x3 * coeffs3 +} + +#[cube] +fn cubic_convolution_1(x: Line, a: Line) -> Line { + let conv = (a + lined(&x, 2.0)) * x; + let tmp = a + lined(&x, 3.0); + (conv - tmp) * x * x + lined(&x, 1.0) +} + +#[cube] +fn cubic_convolution_2(x: Line, a: Line) -> Line { + let conv = a * x; + let conv = (conv - lined(&x, 5.0) * a) * x; + let tmp = lined(&x, 8.0) * a; + let conv = (conv + tmp) * x; + + conv - lined(&x, 4.0) * a +} + +#[cube] +fn lined(x: &Line, #[comptime] v: f32) -> Line { + Line::empty(x.size()).fill(F::new(v)) +} + +pub(crate) fn interpolate_bicubic_launch( + input: CubeTensor, + output: CubeTensor, + align_corners: bool, +) -> CubeTensor { + let line_size = max_line_size(&input); + let out_shape = shape_divmod(&output); + let out_layout = linear_layout(&output, line_size); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + interpolate_bicubic_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, output), + input.as_tensor_arg(line_size), + output.as_tensor_arg(line_size), + out_shape, + out_layout, + align_corners, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bilinear.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bilinear.rs new file mode 100644 index 0000000..ca7c65f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/bilinear.rs @@ -0,0 +1,149 @@ +use cubecl::std::{ + FastDivmod, + tensor::layout::{linear::LinearLayout, *}, +}; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_layout, shape_divmod}, + ops::max_line_size, + tensor::CubeTensor, +}; + +#[cube(launch, address_type = "dynamic")] +fn interpolate_bilinear_kernel( + input: &Tensor>, + output: &mut Tensor>, + shape_out: Sequence>, + out_layout: LinearLayout, + #[comptime] align_corners: bool, + #[define(F)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= output.len() { + terminate!(); + } + + let line_size = input.line_size(); + let out_idx = out_layout.to_source_pos(ABSOLUTE_POS); + + let (rem, c) = shape_out[3].div_mod(ABSOLUTE_POS * line_size); + let (rem, x) = shape_out[2].div_mod(rem); + let (b, y) = shape_out[1].div_mod(rem); + + let frac = if align_corners { + let numerator = (input.shape(1) - 1) as f32; + let denominator = clamp_min(output.shape(1) - 1, 1) as f32; + y as f32 * (numerator / denominator) + } else { + let in_size = input.shape(1) as f32; + let out_size = output.shape(1) as f32; + clamp( + (y as f32 + 0.5) * (in_size / out_size) - 0.5, + 0.0, + in_size - 1.0, + ) + }; + + let v0 = frac.floor(); + let v1 = frac.ceil(); + let yw = F::cast_from(frac - v0); + let yw_ = Line::empty(line_size).fill(F::new(1.0) - yw); + let yw = Line::empty(line_size).fill(yw); + let y0_ok = v0 >= 0.0; + let y0 = v0 as usize; + let y1 = v1 as usize; + + let frac = if align_corners { + let numerator = (input.shape(2) - 1) as f32; + let denominator = clamp_min(output.shape(2) - 1, 1) as f32; + x as f32 * (numerator / denominator) + } else { + let in_size = input.shape(2) as f32; + let out_size = output.shape(2) as f32; + clamp( + (x as f32 + 0.5) * (in_size / out_size) - 0.5, + 0.0, + in_size - 1.0, + ) + }; + let v0 = frac.floor(); + let v1 = frac.ceil(); + let xw = F::cast_from(frac - v0); + let xw_ = Line::empty(line_size).fill(F::new(1.0) - xw); + let xw = Line::empty(line_size).fill(xw); + let x0_ok = v0 >= 0.0; + let x0 = v0 as usize; + let x1 = v1 as usize; + + let index_base = b * input.stride(0) + c * input.stride(3); + + let in_stride_y = input.stride(1); + let in_stride_x = input.stride(2); + + let y0_stride = y0 * in_stride_y; + let y1_stride = y1 * in_stride_y; + let x0_stride = x0 * in_stride_x; + let x1_stride = x1 * in_stride_x; + + let height = input.shape(1); + let width = input.shape(2); + + let y1_ok = y1 < height; + let x1_ok = x1 < width; + + let zero = Line::empty(line_size).fill(F::new(0.0)); + + let p_a = select( + x0_ok && y0_ok, + input[(index_base + y0_stride + x0_stride) / line_size] * xw_ * yw_, + zero, + ); + let p_b = select( + x1_ok && y0_ok, + input[(index_base + y0_stride + x1_stride) / line_size] * xw * yw_, + zero, + ); + let p_c = select( + x0_ok && y1_ok, + input[(index_base + y1_stride + x0_stride) / line_size] * xw_ * yw, + zero, + ); + let p_d = select( + x1_ok && y1_ok, + input[(index_base + y1_stride + x1_stride) / line_size] * xw * yw, + zero, + ); + + output[out_idx] = p_a + p_b + p_c + p_d; +} + +pub(crate) fn interpolate_bilinear_launch( + input: CubeTensor, + output: CubeTensor, + align_corners: bool, +) -> CubeTensor { + let line_size = max_line_size(&input); + let out_shape = shape_divmod(&output); + let out_layout = linear_layout(&output, line_size); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + interpolate_bilinear_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, output), + input.as_tensor_arg(line_size), + output.as_tensor_arg(line_size), + out_shape, + out_layout, + align_corners, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/mod.rs new file mode 100644 index 0000000..19e90af --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/mod.rs @@ -0,0 +1,7 @@ +mod base; +mod bicubic; +mod bilinear; +mod nearest; +mod nearest_backward; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest.rs new file mode 100644 index 0000000..50398f9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest.rs @@ -0,0 +1,80 @@ +use cubecl::std::{ + FastDivmod, + tensor::layout::{linear::LinearLayout, *}, +}; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_layout, shape_divmod}, + ops::max_line_size, + tensor::CubeTensor, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn interpolate_nearest_kernel( + input: &Tensor>, + output: &mut Tensor>, + shape_out: Sequence>, + out_layout: LinearLayout, + #[define(F)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= output.len() { + terminate!(); + } + + let line_size = input.line_size(); + let out_idx = out_layout.to_source_pos(ABSOLUTE_POS); + + let out_pos = ABSOLUTE_POS * line_size; + + let (h_in, w_in) = (input.shape(1) as f32, input.shape(2) as f32); + let (h_out, w_out) = (output.shape(1) as f32, output.shape(2) as f32); + + let (rem, c) = shape_out[3].div_mod(out_pos); + let (rem, x) = shape_out[2].div_mod(rem); + let (b, y) = shape_out[1].div_mod(rem); + + let y = y as f32 * (h_in / h_out); + let x = x as f32 * (w_in / w_out); + + let in_idx = b * input.stride(0) + + y as usize * input.stride(1) + + x as usize * input.stride(2) + + c * input.stride(3); + + output[out_idx] = input[in_idx / line_size]; +} + +pub(crate) fn interpolate_nearest_launch( + input: CubeTensor, + output: CubeTensor, +) -> CubeTensor { + let client = input.client.clone(); + + let line_size = max_line_size(&input); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + let shape_out = shape_divmod(&output); + let out_layout = linear_layout(&output, line_size); + + unsafe { + interpolate_nearest_kernel::launch_unchecked( + &client, + cube_count, + cube_dim, + address_type!(input, output), + input.as_tensor_arg(line_size), + output.as_tensor_arg(line_size), + shape_out, + out_layout, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + }; + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest_backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest_backward.rs new file mode 100644 index 0000000..b7b8cdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/interpolate/nearest_backward.rs @@ -0,0 +1,103 @@ +use cubecl::std::{ + FastDivmod, + tensor::layout::{linear::LinearLayout, *}, +}; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; + +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_layout, shape_divmod}, + ops::max_line_size, + tensor::CubeTensor, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn interpolate_nearest_backward_kernel( + grad: &Tensor>, + output: &mut Tensor>, + shape_out: Sequence>, + out_layout: LinearLayout, + #[define(F)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= output.len() { + terminate!(); + } + + let line_size = grad.line_size(); + let out_idx = out_layout.to_source_pos(ABSOLUTE_POS); + + let out_h = output.shape(1); + let out_w = output.shape(2); + let grad_h = grad.shape(1); + let grad_w = grad.shape(2); + + let (rem, c) = shape_out[3].div_mod(ABSOLUTE_POS * line_size); + let (rem, out_x) = shape_out[2].div_mod(rem); + let (b, out_y) = shape_out[1].div_mod(rem); + + let grad_y_start = start_index::(out_y, grad_h, out_h); + let grad_y_end = end_index::(out_y, grad_h, out_h); + let grad_x_start = start_index::(out_x, grad_w, out_w); + let grad_x_end = end_index::(out_x, grad_w, out_w); + + let index_grad_base = b * grad.stride(0) + c * grad.stride(3); + + let mut sum = Line::empty(line_size).fill(F::new(0.0)); + + for grad_y in grad_y_start..grad_y_end { + for grad_x in grad_x_start..grad_x_end { + let index_grad = index_grad_base + grad_y * grad.stride(1) + grad_x * grad.stride(2); + + sum += grad[index_grad]; + } + } + + output[out_idx] = sum; +} + +#[cube] +fn start_index(input_index: usize, output_size: usize, input_size: usize) -> usize { + let numerator = F::cast_from(input_index * output_size); + let div = (numerator / F::cast_from(input_size)).ceil(); + + usize::cast_from(div) +} + +#[cube] +fn end_index(input_index: usize, output_size: usize, input_size: usize) -> usize { + let numerator = F::cast_from((input_index + 1) * output_size); + let div = (numerator / F::cast_from(input_size)).ceil(); + let index = usize::cast_from(div); + + clamp_max(index, output_size) +} + +pub(crate) fn interpolate_nearest_backward_launch( + out_grad: CubeTensor, + output: CubeTensor, +) -> CubeTensor { + let line_size = max_line_size(&out_grad); + let out_shape = shape_divmod(&output); + let out_layout = linear_layout(&output, line_size); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&out_grad.client, working_units); + let cube_count = calculate_cube_count_elemwise(&out_grad.client, working_units, cube_dim); + + unsafe { + interpolate_nearest_backward_kernel::launch_unchecked( + &out_grad.client, + cube_count, + cube_dim, + address_type!(out_grad, output), + out_grad.as_tensor_arg(line_size), + output.as_tensor_arg(line_size), + out_shape, + out_layout, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + }; + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/base.rs new file mode 100644 index 0000000..9c7057e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/base.rs @@ -0,0 +1,39 @@ +use burn_backend::DType; +use cubecl::prelude::InputScalar; + +use super::{MaskFillStrategy, mask_where::MaskWhereStrategy}; +use crate::{CubeRuntime, tensor::CubeTensor}; + +/// Execute the mask fill kernel. +pub(crate) fn mask_fill_auto( + tensor: CubeTensor, + mask: CubeTensor, + value: InputScalar, + dtype_bool: DType, +) -> CubeTensor { + let strategy = if tensor.can_mut() && tensor.is_nonoverlapping() { + MaskFillStrategy::Inplace + } else { + MaskFillStrategy::Readonly + }; + + super::mask_fill(tensor, mask, value, strategy, dtype_bool) +} + +/// Execute the mask where kernel. +pub(crate) fn mask_where_auto( + tensor: CubeTensor, + mask: CubeTensor, + value: CubeTensor, + dtype_bool: DType, +) -> CubeTensor { + let strategy = if tensor.can_mut_broadcast(&value) { + MaskWhereStrategy::InplaceLhs + } else if value.can_mut_broadcast(&tensor) { + MaskWhereStrategy::InplaceRhs + } else { + MaskWhereStrategy::Readonly + }; + + super::mask_where(tensor, mask, value, strategy, dtype_bool) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_fill.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_fill.rs new file mode 100644 index 0000000..b65df81 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_fill.rs @@ -0,0 +1,88 @@ +use burn_backend::{DType, TensorMetadata}; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, linear_view_alias, linear_view_ref}, + ops::{max_line_size_many, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn mask_fill_kernel( + input: &LinearView>, + mask: &LinearView>, + output: &mut LinearView, ReadWrite>, + value: InputScalar, + #[define(T, B)] _dtypes: [StorageType; 2], +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let mask = Line::cast_from(mask[ABSOLUTE_POS]); + let input = input[ABSOLUTE_POS]; + let value = Line::new(value.get::()); + + output[ABSOLUTE_POS] = select_many(mask, value, input); +} + +#[derive(Clone, Copy, Debug)] +/// Define how to run the mask fill kernel. +/// +/// # Notes +/// +/// All assertions should be done before choosing the strategy. +pub enum MaskFillStrategy { + /// Don't mutate any input. + Readonly, + /// Reuse the input tensor inplace. + Inplace, +} + +/// Execute the mask fill kernel with the given strategy. +pub fn mask_fill( + input: CubeTensor, + mask: CubeTensor, + value: InputScalar, + strategy: MaskFillStrategy, + dtype_bool: DType, +) -> CubeTensor { + let ndims = input.meta.num_dims(); + let output = match strategy { + MaskFillStrategy::Readonly => empty_device_dtype( + input.client.clone(), + input.device.clone(), + input.shape(), + input.dtype, + ), + MaskFillStrategy::Inplace => input.clone(), + }; + + let line_size = max_line_size_many(&[&input, &mask], ndims - 1); + let working_units = input.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + let out_arg = match strategy { + MaskFillStrategy::Readonly => linear_view(&output, line_size), + MaskFillStrategy::Inplace => linear_view_alias(&output, line_size, 0), + }; + + unsafe { + mask_fill_kernel::launch_unchecked( + &input.client, + cube_count, + cube_dim, + address_type!(input, mask, output), + linear_view(&input, line_size), + linear_view_ref(&mask, &input, line_size), + out_arg, + value, + [output.dtype.into(), dtype_bool.into()], + ) + .expect("Kernel to never fail"); + } + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_where.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_where.rs new file mode 100644 index 0000000..1a6cb5c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mask_where.rs @@ -0,0 +1,91 @@ +use burn_backend::DType; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +use crate::{ + CubeRuntime, + kernel::utils::{ + address_type, broadcast_shape, linear_view, linear_view_alias, linear_view_ref, + }, + ops::{max_line_size_many, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; + +#[cube(launch, address_type = "dynamic")] +fn mask_where_kernel( + input: &LinearView>, + value: &LinearView>, + mask: &LinearView>, + output: &mut LinearView, ReadWrite>, + #[define(T, B)] _dtypes: [StorageType; 2], +) { + let pos = ABSOLUTE_POS; + if !output.is_in_bounds(pos) { + terminate!(); + } + + output[pos] = select_many(Line::cast_from(mask[pos]), value[pos], input[pos]); +} + +#[derive(Clone, Copy, Debug)] +/// Define how to run the mask where kernel. +/// +/// # Notes +/// +/// All assertions should be done before choosing the strategy. +pub enum MaskWhereStrategy { + /// Don't mutate any input. + Readonly, + /// Reuse the lhs tensor inplace. + InplaceLhs, + /// Reuse the rhs tensor inplace. + InplaceRhs, +} + +/// Execute the mask where kernel with the given strategy. +pub fn mask_where( + input: CubeTensor, + mask: CubeTensor, + value: CubeTensor, + strategy: MaskWhereStrategy, + dtype_bool: DType, +) -> CubeTensor { + let line_size = max_line_size_many(&[&input, &mask, &value], input.meta.num_dims() - 1); + + let working_units = input.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + let out_shape = broadcast_shape(&[&input, &mask, &value]); + + let output = match strategy { + MaskWhereStrategy::Readonly => empty_device_dtype( + input.client.clone(), + input.device.clone(), + out_shape, + input.dtype, + ), + MaskWhereStrategy::InplaceLhs => input.clone(), + MaskWhereStrategy::InplaceRhs => value.clone(), + }; + + let out = match strategy { + MaskWhereStrategy::Readonly => linear_view(&output, line_size), + MaskWhereStrategy::InplaceLhs => linear_view_alias(&output, line_size, 0), + MaskWhereStrategy::InplaceRhs => linear_view_alias(&output, line_size, 1), + }; + + mask_where_kernel::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, value, mask, output), + linear_view_ref(&input, &output, line_size), + linear_view_ref(&value, &output, line_size), + linear_view_ref(&mask, &output, line_size), + out, + [output.dtype.into(), dtype_bool.into()], + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mod.rs new file mode 100644 index 0000000..0044101 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mask/mod.rs @@ -0,0 +1,8 @@ +mod base; +mod mask_fill; +mod mask_where; + +pub(crate) use base::*; + +pub use mask_fill::*; +pub use mask_where::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/base.rs new file mode 100644 index 0000000..2f9759c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/base.rs @@ -0,0 +1,155 @@ +use super::init_matmul_output; +use crate::{CubeRuntime, kernel::quantization::dequantize, tensor::CubeTensor}; +use burn_backend::{DType, QTensorPrimitive}; +use burn_std::QuantLevel; +use cubek::matmul::{ + definition::{MatmulElems, MatmulGlobalElems, MatmulSetupError}, + launch::{MatmulInputHandleRef, Strategy}, +}; + +#[cfg(feature = "autotune")] +use super::matmul_autotune; + +/// The strategy to be used when launching a matmul kernel. +pub enum MatmulStrategy { + #[cfg(feature = "autotune")] + /// Using autotune to choose the best kernel based on runtime information. + Autotune, + /// Cube implementation of matmul. + Cube, +} + +impl Default for MatmulStrategy { + fn default() -> Self { + // if autotune is enabled, default to autotune + #[cfg(feature = "autotune")] + return MatmulStrategy::Autotune; + + #[cfg(not(feature = "autotune"))] + MatmulStrategy::Cube + } +} + +/// Launch a matmul kernel using the given strategy. +pub fn matmul( + lhs: CubeTensor, + rhs: CubeTensor, + out: Option>, + strategy: MatmulStrategy, + out_dtype: DType, +) -> Result, MatmulSetupError> { + match strategy { + MatmulStrategy::Cube => { + let out = out.unwrap_or_else(|| init_matmul_output(&lhs, &rhs, out_dtype)); + launch_matmul(&Default::default(), lhs, rhs, out.clone())?; + Ok(out) + } + #[cfg(feature = "autotune")] + MatmulStrategy::Autotune => Ok(matmul_autotune(lhs, rhs, out, out_dtype)), + } +} + +pub(crate) fn launch_matmul_naive( + strategy: &Strategy, + mut lhs: CubeTensor, + mut rhs: CubeTensor, + out: CubeTensor, +) -> Result<(), MatmulSetupError> { + // Naive has very specific layout requirements for block scaled tensors, so we need to manually + // dequantize if it fails to launch normally. This is because naive is assumed to always work. + if lhs.qparams.is_some() || rhs.qparams.is_some() { + match launch_matmul(strategy, lhs.clone(), rhs.clone(), out.clone()) { + Err(_) => { + if lhs.qparams.is_some() { + lhs = dequantize(lhs, out.dtype); + } + if rhs.qparams.is_some() { + rhs = dequantize(rhs, out.dtype); + } + launch_matmul(strategy, lhs, rhs, out) + } + Ok(_) => Ok(()), + } + } else { + launch_matmul(strategy, lhs, rhs, out) + } +} + +pub(crate) fn launch_matmul( + strategy: &Strategy, + lhs: CubeTensor, + mut rhs: CubeTensor, + out: CubeTensor, +) -> Result<(), MatmulSetupError> { + let client = &lhs.client; + + let lhs_quant_handles = lhs.quantized_handles(); + let out_dtype: DType = out.dtype; + + let (lhs_dtype, lhs_handle) = match &lhs_quant_handles { + None => ( + lhs.dtype, + MatmulInputHandleRef::new(lhs.as_handle_ref(), lhs.dtype.into()), + ), + Some((data, scale)) => ( + out_dtype, + MatmulInputHandleRef::quantized( + data.as_handle_ref(), + scale.as_handle_ref(), + lhs.meta.shape(), + lhs.scheme(), + data.dtype.into(), + scale.dtype.into(), + ), + ), + }; + + let rhs_quant_handles = rhs.quantized_handles(); + + let (rhs_dtype, rhs_handle) = match &rhs_quant_handles { + None => ( + lhs.dtype, + MatmulInputHandleRef::new(rhs.as_handle_ref(), lhs.dtype.into()), + ), + Some((data, scale)) => { + // Extremely hacky fix to ensure naive can run in every case + if matches!(strategy, Strategy::Naive) + && matches!(rhs.scheme().level, QuantLevel::Block(_)) + { + rhs = dequantize(rhs.clone(), lhs.dtype); + ( + lhs.dtype, + MatmulInputHandleRef::new(rhs.as_handle_ref(), rhs.dtype.into()), + ) + } else { + ( + out_dtype, + MatmulInputHandleRef::quantized( + data.as_handle_ref(), + scale.as_handle_ref(), + rhs.meta.shape(), + rhs.scheme(), + data.dtype.into(), + scale.dtype.into(), + ), + ) + } + } + }; + + let mut dtypes = MatmulElems::from_globals(&MatmulGlobalElems { + lhs: lhs_dtype.into(), + rhs: rhs_dtype.into(), + out: out_dtype.into(), + }); + cubek::matmul::launch::launch_ref( + strategy, + client, + &lhs_handle, + &rhs_handle, + &out.as_handle_ref(), + &mut dtypes, + )?; + + Ok(()) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/mod.rs new file mode 100644 index 0000000..86df664 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/mod.rs @@ -0,0 +1,10 @@ +mod base; +mod tune; + +/// Contains utilities for matmul operation +pub mod utils; + +pub use base::*; +#[cfg(feature = "autotune")] +pub use tune::*; +pub use utils::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/base.rs new file mode 100644 index 0000000..ae58638 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/base.rs @@ -0,0 +1,409 @@ +use crate::{ + CubeRuntime, CubeTuneId, + kernel::matmul::{launch_matmul, launch_matmul_naive, utils::init_matmul_output}, + tensor::CubeTensor, +}; +use burn_backend::DType; +use cubecl::tune::{LocalTuner, Tunable, TunableSet, TuneGroup, local_tuner}; +use cubek::matmul::{ + definition::MatmulKind, + launch::{MatmulAutotuneKey, MatmulGlobalScale, Strategy, should_tune_double_buffering}, + routines::{ + BlueprintStrategy, TileSizeSelection, double_buffering::DoubleBufferingArgs, + double_unit::DoubleUnitSelectionArgs, ordered_double_buffering::OrderedSelectionArgs, + simple::SimpleArgs, simple_unit::SimpleUnitSelectionArgs, + }, +}; + +fn matmul_input_gen( + _key: &MatmulAutotuneKey, + lhs: &CubeTensor, + rhs: &CubeTensor, + out: &CubeTensor, +) -> (CubeTensor, CubeTensor, CubeTensor) { + (lhs.clone(), rhs.clone(), out.copy()) +} + +/// Executes autotune on matmul operations +pub fn matmul_autotune( + lhs: CubeTensor, + rhs: CubeTensor, + out: Option>, + out_dtype: DType, +) -> CubeTensor { + let output = out.unwrap_or_else(|| init_matmul_output(&lhs, &rhs, out_dtype)); + + let client = lhs.client.clone(); + + static TUNER: LocalTuner = local_tuner!(); + + let tunables = TUNER.init(|| { + const PRIORITY_MAX: i8 = 3; + const PRIORITY_HIGH: i8 = 2; + const PRIORITY_MEDIUM: i8 = 1; + const PRIORITY_MIN: i8 = 0; + const PRIORITY_NEVER: i8 = -1; + + let cmma = TuneGroup::::new("cmma", |key| { + if matches!( + key.analysis.kind, + MatmulKind::General + // Those variants are just because the unit alternatives aren't very good yet. + | MatmulKind::VecMat | MatmulKind::MatVec + ) { + PRIORITY_HIGH + } else { + PRIORITY_MEDIUM + } + }); + + let mma = TuneGroup::::new("mma", |key| { + if matches!( + key.analysis.kind, + // General is usually bad, but I think shapes like 16x8196 would be classed as + // general and are very good with MMA + // Should highly degenerated matrices that aren't VecMat have their own class? + MatmulKind::General | MatmulKind::VecMat | MatmulKind::MatVec + ) { + PRIORITY_HIGH + } else { + PRIORITY_MEDIUM + } + }); + + let unit = TuneGroup::::new("unit", |key| { + if !matches!(key.analysis.kind, MatmulKind::General) + || matches!(key.analysis.scale_global, MatmulGlobalScale::Small) + { + PRIORITY_HIGH + } else { + PRIORITY_MIN + } + }); + + let tma = TuneGroup::::new("tma", |key| { + // For large matmul, we set the max priority to TMA kernels, higher than any other + // matmuls, since they are the best kernels no matter what. + // + // But only when all axis are large. + let max_axis = usize::max(key.definition.m, key.definition.n); + let max_axis = usize::max(key.definition.k, max_axis); + + let min_axis = usize::min(key.definition.m, key.definition.n); + let min_axis = usize::min(key.definition.k, min_axis); + + let skewed_factor = max_axis / min_axis; + + let priority_max = if matches!(key.analysis.kind, MatmulKind::General) + && matches!(key.analysis.scale_global, MatmulGlobalScale::Large) + && skewed_factor < 4 + { + PRIORITY_MAX + } else { + PRIORITY_HIGH + }; + + if key.definition.lhs_stride_factor >= 4 && key.definition.rhs_stride_factor >= 4 { + priority_max + } else { + PRIORITY_NEVER + } + }); + + fn double_buffering_priority(key: &MatmulAutotuneKey, max: i8, min: i8) -> i8 { + if should_tune_double_buffering(false, key) { + max + } else { + min + } + } + + let mut set = TunableSet::new(create_key::, matmul_input_gen::); + + // First entry should always work, since it is considered the fallback. + set = set.with( + Tunable::new("matmul_naive", |lhs, rhs, out| { + launch_matmul_naive::(&Strategy::Naive, lhs, rhs, out) + .map_err(|err| std::format!("{err:?}")) + }) + .group(&unit, |key| { + if matches!(key.analysis.scale_global, MatmulGlobalScale::Small) + || matches!(key.analysis.kind, MatmulKind::InnerProduct) + { + PRIORITY_MAX + } else { + PRIORITY_MIN + } + }), + ); + + // Unit VecMat + for (strategy, double_buf) in [ + ( + Strategy::SimpleVecMat(BlueprintStrategy::Inferred(().into())), + false, + ), + ( + Strategy::DoubleVecMat(BlueprintStrategy::Inferred(().into())), + true, + ), + ] { + set = set.with( + Tunable::new(strategy.to_string(), move |lhs, rhs, out| { + launch_matmul::(&strategy, lhs, rhs, out) + .map_err(|err| std::format!("{err:?}")) + }) + .group(&unit, move |key| match double_buf { + false => PRIORITY_MAX, + true => double_buffering_priority(key, PRIORITY_MAX, PRIORITY_HIGH), + }), + ); + } + + // Unit matmuls + for tile_size in [ + TileSizeSelection::MaxTileSize, + TileSizeSelection::MinTileSize, + ] { + for (strategy, double_buf) in [ + ( + Strategy::SimpleUnit(BlueprintStrategy::Inferred(SimpleUnitSelectionArgs { + tile_size, + })), + false, + ), + ( + Strategy::DoubleUnit(BlueprintStrategy::Inferred(DoubleUnitSelectionArgs { + tile_size, + })), + true, + ), + ] { + set = set.with( + Tunable::new(strategy.to_string(), move |lhs, rhs, out| { + launch_matmul::(&strategy, lhs, rhs, out) + .map_err(|err| format!("{err:?}")) + }) + .group(&unit, move |key| match double_buf { + false => PRIORITY_MAX, + true => double_buffering_priority(key, PRIORITY_MAX, PRIORITY_HIGH), + }), + ) + } + } + + // Accelerated matmuls + for (strategy, double_buf, group_extra, tile_group) in [ + ( + Strategy::SimpleCyclicCmma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: false, + })), + false, + None, + &cmma, + ), + ( + Strategy::SimpleCyclicMma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: false, + })), + false, + None, + &mma, + ), + ( + Strategy::SimpleCyclicCmma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: true, + })), + false, + None, + &cmma, + ), + ( + Strategy::SimpleCyclicMma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: true, + })), + false, + None, + &mma, + ), + ( + Strategy::OrderedDoubleCmma(BlueprintStrategy::Inferred(OrderedSelectionArgs { + partition_k: Some(2), + row_count: Some(4), + rows_per_plane: Some(2), + })), + true, + None, + &cmma, + ), + ( + Strategy::OrderedDoubleMma(BlueprintStrategy::Inferred(OrderedSelectionArgs { + partition_k: Some(2), + row_count: Some(4), + rows_per_plane: Some(2), + })), + true, + None, + &mma, + ), + ( + Strategy::OrderedDoubleCmma(BlueprintStrategy::Inferred(OrderedSelectionArgs { + partition_k: Some(2), + row_count: Some(8), + rows_per_plane: Some(2), + })), + true, + None, + &cmma, + ), + ( + Strategy::OrderedDoubleMma(BlueprintStrategy::Inferred(OrderedSelectionArgs { + partition_k: Some(2), + row_count: Some(8), + rows_per_plane: Some(2), + })), + true, + None, + &mma, + ), + ( + Strategy::DoubleCyclicCmma(BlueprintStrategy::Inferred(DoubleBufferingArgs { + specialized: false, + })), + true, + None, + &cmma, + ), + ( + Strategy::DoubleCyclicMma(BlueprintStrategy::Inferred(DoubleBufferingArgs { + specialized: false, + })), + true, + None, + &mma, + ), + ( + Strategy::DoubleCyclicCmma(BlueprintStrategy::Inferred(DoubleBufferingArgs { + specialized: true, + })), + true, + None, + &cmma, + ), + ( + Strategy::DoubleCyclicMma(BlueprintStrategy::Inferred(DoubleBufferingArgs { + specialized: true, + })), + true, + None, + &mma, + ), + ( + Strategy::SpecializedCyclicCmma(BlueprintStrategy::Inferred(().into())), + true, + None, + &cmma, + ), + ( + Strategy::SpecializedCyclicMma(BlueprintStrategy::Inferred(().into())), + true, + None, + &mma, + ), + ( + Strategy::SimpleTmaCmma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: false, + })), + false, + Some(&tma), + &cmma, + ), + ( + Strategy::SimpleTmaMma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: false, + })), + false, + Some(&tma), + &mma, + ), + ( + Strategy::SimpleTmaCmma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: true, + })), + false, + Some(&tma), + &cmma, + ), + ( + Strategy::SimpleTmaMma(BlueprintStrategy::Inferred(SimpleArgs { + multi_rows: true, + })), + false, + Some(&tma), + &mma, + ), + ( + Strategy::SpecializedTmaCmma(BlueprintStrategy::Inferred(().into())), + true, + Some(&tma), + &cmma, + ), + ( + Strategy::SpecializedTmaMma(BlueprintStrategy::Inferred(().into())), + true, + Some(&tma), + &mma, + ), + ] { + let priority_within_group = |key: &MatmulAutotuneKey, double_buf: bool| match double_buf + { + false => PRIORITY_MAX, + true => double_buffering_priority(key, PRIORITY_MAX, PRIORITY_HIGH), + }; + let mut tunable = Tunable::new(strategy.to_string(), move |lhs, rhs, out| { + launch_matmul::(&strategy, lhs, rhs, out).map_err(|err| format!("{err:?}")) + }); + + // tile group + tunable = tunable.group(tile_group, move |key| { + priority_within_group(key, double_buf) + }); + + // extra group + if let Some(group) = group_extra { + tunable = tunable.group(group, move |key| priority_within_group(key, double_buf)); + } + set = set.with(tunable); + } + + set + }); + + TUNER.execute( + &CubeTuneId::new(&lhs.client, &lhs.device), + &client, + tunables, + (lhs, rhs, output.clone()), + ); + + output +} + +fn create_key( + lhs: &CubeTensor, + rhs: &CubeTensor, + out: &CubeTensor, +) -> MatmulAutotuneKey { + MatmulAutotuneKey::generate( + &lhs.client, + lhs.meta.shape(), + rhs.meta.shape(), + lhs.meta.strides(), + rhs.meta.strides(), + lhs.dtype.into(), + rhs.dtype.into(), + out.dtype.into(), + lhs.try_scheme(), + rhs.try_scheme(), + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/mod.rs new file mode 100644 index 0000000..87ee928 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/tune/mod.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "autotune")] +mod base; + +#[cfg(feature = "autotune")] +pub use base::matmul_autotune; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/utils.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/utils.rs new file mode 100644 index 0000000..fbbf69f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/matmul/utils.rs @@ -0,0 +1,16 @@ +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; +use burn_backend::{DType, calculate_matmul_output}; + +/// Creates an empty output tensor with matmul output shape +pub fn init_matmul_output( + lhs: &CubeTensor, + rhs: &CubeTensor, + dtype: DType, +) -> CubeTensor { + empty_device_dtype( + lhs.client.clone(), + lhs.device.clone(), + calculate_matmul_output(lhs.meta.shape(), rhs.meta.shape()).unwrap(), + dtype, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mod.rs new file mode 100644 index 0000000..1ebfd02 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/mod.rs @@ -0,0 +1,51 @@ +mod binary; +mod binary_float; +mod binary_int; +mod cast; +mod clamp; +mod comparison; +mod contiguous; +mod cross; +mod index; +mod mask; +mod unary_float; +mod unary_int; +mod unary_numeric; + +pub(crate) use binary::*; +pub(crate) use binary_float::*; +pub(crate) use binary_int::*; +pub use cast::*; +pub use contiguous::*; +pub(crate) use cross::*; +pub use mask::*; +pub(crate) use unary_float::*; +pub(crate) use unary_int::*; +pub(crate) use unary_numeric::*; + +pub use crate::cubecl::prelude::KernelMetadata; + +/// Attention kernels +pub mod attention; +/// Convolution kernels +pub mod conv; +/// Grid sampling kernels +pub mod grid_sample; +/// Interpolation kernels +pub mod interpolate; +/// Matmul kernels +pub mod matmul; +/// Pooling kernels +pub mod pool; +/// Pseudo-random number generator kernels +pub mod prng; +/// Quantization operations +pub mod quantization; +/// Reduction algorithms +pub mod reduce; + +pub(crate) use clamp::*; +pub(crate) use comparison::*; +pub use index::*; + +pub(crate) mod utils; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d.rs new file mode 100644 index 0000000..1a182cf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d.rs @@ -0,0 +1,117 @@ +use crate::{ + CubeRuntime, + kernel::{ + into_contiguous_aligned, + pool::pool2d::{Position, view4d}, + utils::{address_type, decompose_linear, shape_divmod}, + }, + ops::{max_line_size, numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::Shape; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::View}, +}; + +#[cube(launch, address_type = "dynamic")] +fn adaptive_avg_pool2d_direct( + input: &Tensor>, + output: &mut View, Position, ReadWrite>, + out_shape: Sequence>, + working_units: usize, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= working_units { + terminate!(); + } + + let (_, pos) = decompose_linear(ABSOLUTE_POS * output.line_size(), &out_shape); + let [b, oh, ow, c] = *pos else { unreachable!() }; + + let (_, out_h, out_w, _) = output.shape(); + let (in_stride_h, in_stride_w) = (input.stride(1), input.stride(2)); + let (in_h, in_w) = (input.shape(1), input.shape(2)); + + let ih_start = start_index(oh, out_h, in_h); + let ih_end = end_index(oh, out_h, in_h); + + let iw_start = start_index(ow, out_w, in_w); + let iw_end = end_index(ow, out_w, in_w); + + let mut sum = Line::empty(input.line_size()).fill(E::from_int(0)); + + let index_input_base = b * input.stride(0) + c * input.stride(3); + + for ih in ih_start..ih_end { + let index_input_2 = ih * in_stride_h; + + for iw in iw_start..iw_end { + let index_input_3 = iw * in_stride_w; + + let index_input = index_input_base + index_input_2 + index_input_3; + sum += input[index_input / input.line_size()]; + } + } + + let num_ih = ih_end - ih_start; + let num_iw = iw_end - iw_start; + + output[(b, oh, ow, c)] = sum / Line::cast_from(num_ih * num_iw); +} + +#[cube] +fn start_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + (output_size_index * input_size) / output_size +} + +#[cube] +fn end_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + let index = (output_size_index + 1) * input_size; + let index = index.div_ceil(output_size); + + if input_size < index { + input_size + } else { + index + } +} + +pub(crate) fn adaptive_avg_pool2d( + input: CubeTensor, + output_size: [usize; 2], +) -> CubeTensor { + let [batch_size, channels, _, _] = input.meta.shape().dims(); + + let input = into_contiguous_aligned(permute_nchw_to_nhwc(input)); + let line_size = max_line_size(&input); + + let output_shape = Shape::new([batch_size, output_size[0], output_size[1], channels]); + let num_elems: usize = output_shape.num_elements(); + let output = empty_device_dtype( + input.client.clone(), + input.device.clone(), + output_shape, + input.dtype, + ); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&input.client, working_units); + let cube_count = calculate_cube_count_elemwise(&input.client, working_units, cube_dim); + + adaptive_avg_pool2d_direct::launch( + &input.client, + cube_count, + cube_dim, + address_type!(input, output), + input.as_tensor_arg(line_size), + view4d(&output, line_size), + shape_divmod(&output), + ScalarArg::new(working_units), + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + permute_nhwc_to_nchw(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d_backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d_backward.rs new file mode 100644 index 0000000..da309dd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/adaptive_avg_pool2d_backward.rs @@ -0,0 +1,119 @@ +use crate::{ + CubeRuntime, + kernel::{ + into_contiguous_aligned, + pool::pool2d::{Position, view4d}, + utils::{address_type, decompose_linear, shape_divmod}, + }, + ops::{max_line_size, numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::Shape; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::View}, +}; + +#[cube(launch, address_type = "dynamic")] +fn adaptive_avg_pool2d_backward_direct( + grad: &Tensor>, + output: &mut View, Position, ReadWrite>, + out_shape: Sequence>, + working_units: usize, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= working_units { + terminate!(); + } + + let (_, out_h, out_w, _) = output.shape(); + let (grad_stride_h, grad_stride_w) = (grad.stride(1), grad.stride(2)); + let (grad_h, grad_w) = (grad.shape(1), grad.shape(2)); + + let (_, pos) = decompose_linear(ABSOLUTE_POS * output.line_size(), &out_shape); + let [b, ih, iw, c] = *pos else { unreachable!() }; + + let oh_start = start_index(ih, out_h, grad_h); + let oh_end = end_index(ih, out_h, grad_h); + + let ow_start = start_index(iw, out_w, grad_w); + let ow_end = end_index(iw, out_w, grad_w); + + let mut grad_acc = Line::empty(grad.line_size()).fill(E::from_int(0)); + + let index_base = b * grad.stride(0) + (c * grad.stride(3)); + + for oh in oh_start..oh_end { + let ih_start = start_index(oh, grad_h, out_h); + let ih_end = end_index(oh, grad_h, out_h); + + if ih >= ih_start && ih < ih_end { + for ow in ow_start..ow_end { + let iw_start = start_index(ow, grad_w, out_w); + let iw_end = end_index(ow, grad_w, out_w); + + if iw >= iw_start && iw < iw_end { + let num_ih = ih_end - ih_start; + let num_iw = iw_end - iw_start; + + let index = index_base + (oh * grad_stride_h) + (ow * grad_stride_w); + grad_acc += grad[index / grad.line_size()] / Line::cast_from(num_iw * num_ih); + } + } + } + } + + output[(b, ih, iw, c)] = grad_acc; +} + +#[cube] +fn start_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + (output_size_index * input_size) / output_size +} + +#[cube] +fn end_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + let index = (output_size_index + 1) * input_size; + let index = index.div_ceil(output_size); + + if input_size < index { + input_size + } else { + index + } +} + +pub(crate) fn adaptive_avg_pool2d_backward( + x: CubeTensor, + out_grad: CubeTensor, +) -> CubeTensor { + let [batches, channels, height, width] = x.meta.shape().dims(); + + let out_grad = into_contiguous_aligned(permute_nchw_to_nhwc(out_grad)); + let line_size = max_line_size(&out_grad); + + let out_shape = Shape::new([batches, height, width, channels]); + let output = empty_device_dtype(x.client.clone(), x.device.clone(), out_shape, x.dtype); + + let num_elems = output.meta.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&x.client, working_units); + let cube_count = calculate_cube_count_elemwise(&x.client, working_units, cube_dim); + + adaptive_avg_pool2d_backward_direct::launch( + &x.client, + cube_count, + cube_dim, + address_type!(out_grad, output), + out_grad.as_tensor_arg(line_size), + view4d(&output, line_size), + shape_divmod(&output), + ScalarArg::new(working_units), + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + permute_nhwc_to_nchw(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d.rs new file mode 100644 index 0000000..3146fff --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d.rs @@ -0,0 +1,166 @@ +use super::pool2d::{ + Pool2dDirectArgsLaunch, Pool2dDirectStrategy, Pool2dDirectStrategyFamily, pool2d_direct, +}; +use crate::{ + CubeRuntime, + kernel::{ + into_contiguous_aligned, + pool::pool2d::{Position, view4d}, + utils::{address_type, shape_divmod}, + }, + ops::{max_line_size, numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::{Shape, ops::conv::calculate_pool_output_size}; +use cubecl::{CubeDim, calculate_cube_count_elemwise, prelude::ScalarArg}; +use cubecl::{prelude::*, std::tensor::View}; + +struct AvgPoolStrategy; + +impl Pool2dDirectStrategyFamily for AvgPoolStrategy { + type Indices = (); + type Config = AvgPoolStrategyConfig; + type Pool2d = Self; +} + +#[derive(CubeType, Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct AvgPoolStrategyConfig { + count_include_pad: bool, + /// Total padded height (input_height + 2 * padding_0) + padded_h: u32, + /// Total padded width (input_width + 2 * padding_1) + padded_w: u32, +} + +#[cube] +impl Pool2dDirectStrategy for AvgPoolStrategy { + type Accumulator = (Line, u32); + type Config = AvgPoolStrategyConfig; + type Indices = (); + + fn initialize( + #[comptime] _config: &Self::Config, + #[comptime] line_size: LineSize, + ) -> Self::Accumulator { + let sum = Line::empty(line_size).fill(N::from_int(0)); + // Count will be set dynamically: either by accumulate (count_include_pad=false) + // or by set_padded_count (count_include_pad=true) + let count = 0u32; + + (sum, count) + } + + fn accumulate( + #[comptime] config: &Self::Config, + accumulator: &mut Self::Accumulator, + _index: usize, + result: Line, + ) { + let (sum, count) = accumulator; + + // Only count valid positions when count_include_pad=false + if comptime![!config.count_include_pad] { + *count += 1; + } + + *sum += result; + } + + fn count_position( + #[comptime] config: &Self::Config, + accumulator: &mut Self::Accumulator, + ih: u32, + iw: u32, + ) { + // When count_include_pad=true, count positions within padded bounds + // (excludes ceil_mode extensions beyond the padded input) + if comptime![config.count_include_pad] && ih < config.padded_h && iw < config.padded_w { + let (_sum, count) = accumulator; + *count += 1; + } + } + + fn store( + #[comptime] _config: &Self::Config, + position: Position, + output: &mut View, Position, ReadWrite>, + _output_indices: &mut (), + accumulator: Self::Accumulator, + ) { + let (sum, count) = accumulator; + output[position] = sum / Line::cast_from(count); + } +} + +pub(crate) fn avg_pool2d( + x: CubeTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, +) -> CubeTensor { + let [batch_size, channels, in_h, in_w] = x.meta.shape().dims(); + let dilation = 1; + + let size_0 = calculate_pool_output_size( + kernel_size[0], + stride[0], + padding[0], + dilation, + in_h, + ceil_mode, + ); + let size_1 = calculate_pool_output_size( + kernel_size[1], + stride[1], + padding[1], + dilation, + in_w, + ceil_mode, + ); + + // Padded dimensions (for count_include_pad with ceil_mode) + let padded_0 = in_h + 2 * padding[0]; + let padded_1 = in_w + 2 * padding[1]; + + let x = into_contiguous_aligned(permute_nchw_to_nhwc(x)); + let line_size = max_line_size(&x); + + let shape_out = Shape::new([batch_size, size_0, size_1, channels]); + let output = empty_device_dtype(x.client.clone(), x.device.clone(), shape_out, x.dtype); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&x.client, working_units); + let cube_count = calculate_cube_count_elemwise(&x.client, working_units, cube_dim); + + pool2d_direct::launch::( + &x.client, + cube_count, + cube_dim, + address_type!(x, output), + x.as_tensor_arg(line_size), + view4d(&output, line_size), + (), + shape_divmod(&output), + ScalarArg::new(working_units), + Pool2dDirectArgsLaunch::new( + ScalarArg::new(stride[0] as u32), + ScalarArg::new(stride[1] as u32), + ScalarArg::new(dilation as u32), + ScalarArg::new(dilation as u32), + ScalarArg::new(padding[0] as u32), + ScalarArg::new(padding[1] as u32), + ), + (kernel_size[0] as u32, kernel_size[1] as u32), + AvgPoolStrategyConfig { + count_include_pad, + padded_h: padded_0 as u32, + padded_w: padded_1 as u32, + }, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + permute_nhwc_to_nchw(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d_backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d_backward.rs new file mode 100644 index 0000000..59b462c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/avg_pool2d_backward.rs @@ -0,0 +1,183 @@ +use crate::{ + CubeRuntime, + kernel::{ + pool::pool2d::{Position, view4d}, + utils::{address_type, decompose_linear, shape_divmod}, + }, + ops::{max_line_size, numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::Shape; +use cubecl::{ + calculate_cube_count_elemwise, + prelude::*, + std::{FastDivmod, tensor::View}, +}; + +#[derive(CubeLaunch, CubeType)] +pub(crate) struct PoolBackwardArgs { + pub stride_0: i32, + pub stride_1: i32, + pub dilation_0: i32, + pub dilation_1: i32, + pub padding_0: i32, + pub padding_1: i32, +} + +#[cube(launch_unchecked, address_type = "dynamic")] +fn avg_pool2d_backward_kernel( + grad: &Tensor>, + output: &mut View, Position, ReadWrite>, + out_shape: Sequence>, + working_units: usize, + args: &PoolBackwardArgs, + #[comptime] kernel_size_0: i32, + #[comptime] kernel_size_1: i32, + #[comptime] count_include_pad: bool, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= working_units { + terminate!(); + } + + let line_size = grad.line_size(); + + let (_, pos) = decompose_linear(ABSOLUTE_POS * output.line_size(), &out_shape); + let [batch, ih, iw, channel] = *pos else { + unreachable!() + }; + + let mut grad_acc = Line::empty(grad.line_size()).fill(E::from_int(0)); + + let (oh_start, oh_end, ow_start, ow_end) = loop_ranges( + ih as i32, + iw as i32, + grad.shape(1) as u32, + grad.shape(2) as u32, + args, + kernel_size_0, + kernel_size_1, + ); + + let padding_0 = args.padding_0 as u32; + let padding_1 = args.padding_1 as u32; + let stride_0 = args.stride_0 as u32; + let stride_1 = args.stride_1 as u32; + let kernel_size_0 = comptime![kernel_size_0 as u32]; + let kernel_size_1 = comptime![kernel_size_1 as u32]; + + let index_base = batch * grad.stride(0) + channel * grad.stride(3); + let border_bottom = output.shape().1 as u32 + padding_0; + let border_right = output.shape().2 as u32 + padding_1; + let begin_h = ih as u32 + padding_0; + let begin_w = iw as u32 + padding_1; + + for oh in oh_start..oh_end { + let ih_start = oh * stride_0; + let ih_end = clamp_max(ih_start + kernel_size_0, border_bottom); + let ih_start = clamp_min(ih_start, padding_0); + + if begin_h >= ih_start && (ih as u32) < ih_end { + for ow in ow_start..ow_end { + let index = + index_base + oh as usize * grad.stride(1) + ow as usize * grad.stride(2); + + let iw_start = ow * stride_1; + let iw_end = clamp_max(iw_start + kernel_size_1, border_right); + let iw_start = clamp_min(iw_start, padding_1); + + if begin_w >= iw_start && (iw as u32) < iw_end { + if count_include_pad { + grad_acc += grad[index / line_size] + / Line::cast_from(kernel_size_0 * kernel_size_1); + } else { + let ih_diff = ih_end - ih_start; + let iw_diff = iw_end - iw_start; + let count = Line::cast_from(ih_diff * iw_diff); + grad_acc += grad[index / line_size] / count; + } + } + } + } + } + + output[(batch, ih, iw, channel)] = grad_acc; +} + +#[cube] +fn loop_ranges( + ih: i32, + iw: i32, + grad_h: u32, + grad_w: u32, + args: &PoolBackwardArgs, + #[comptime] kernel_size_0: i32, + #[comptime] kernel_size_1: i32, +) -> (u32, u32, u32, u32) { + let kms_0 = args.dilation_0 * kernel_size_0 - args.stride_0; + let kms_1 = args.dilation_1 * kernel_size_1 - args.stride_1; + + let oh_start = clamp_min((ih + args.padding_0 - kms_0) / args.stride_0, 0) as u32; + let ow_start = clamp_min((iw + args.padding_1 - kms_1) / args.stride_1, 0) as u32; + let oh_end = clamp_max(clamp_min(kms_0, 0) as u32 + oh_start, grad_h - 1) + 1; + let ow_end = clamp_max(clamp_min(kms_1, 0) as u32 + ow_start, grad_w - 1) + 1; + + (oh_start, oh_end, ow_start, ow_end) +} + +pub(crate) fn avg_pool2d_backward( + x: CubeTensor, + grad: CubeTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + _ceil_mode: bool, +) -> CubeTensor { + let [batches, channels, height, width] = x.meta.shape().dims(); + + let grad = permute_nchw_to_nhwc(grad); + + let line_size = if x.meta.strides()[3] == grad.meta.strides()[3] { + max_line_size(&x) + } else { + 1 + }; + + let dilation = 1; + + let out_shape = Shape::new([batches, height, width, channels]); + let output = empty_device_dtype(x.client.clone(), x.device.clone(), out_shape, x.dtype); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&x.client, working_units); + let cube_count = calculate_cube_count_elemwise(&x.client, working_units, cube_dim); + + unsafe { + avg_pool2d_backward_kernel::launch_unchecked( + &grad.client, + cube_count, + cube_dim, + address_type!(grad, output), + grad.as_tensor_arg(line_size), + view4d(&output, line_size), + shape_divmod(&output), + ScalarArg::new(working_units), + PoolBackwardArgsLaunch::new( + ScalarArg::new(stride[0] as i32), + ScalarArg::new(stride[1] as i32), + ScalarArg::new(dilation), + ScalarArg::new(dilation), + ScalarArg::new(padding[0] as i32), + ScalarArg::new(padding[1] as i32), + ), + kernel_size[0] as i32, + kernel_size[1] as i32, + count_include_pad, + output.dtype.into(), + ) + } + .expect("Kernel to never fail"); + + permute_nhwc_to_nchw(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d.rs new file mode 100644 index 0000000..5826727 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d.rs @@ -0,0 +1,255 @@ +use super::pool2d::{ + Pool2dDirectArgsLaunch, Pool2dDirectStrategy, Pool2dDirectStrategyFamily, pool2d_direct, +}; +use crate::{ + CubeRuntime, + kernel::{ + into_contiguous_aligned, + pool::pool2d::{Position, view4d}, + utils::{address_type, shape_divmod}, + }, + ops::{max_line_size, numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::{DType, Shape, ops::conv::calculate_pool_output_size}; +use cubecl::{CubeDim, calculate_cube_count_elemwise, prelude::*, std::tensor::View}; + +struct MaxPoolStrategy; +struct MaxPoolWithIndicesStrategy; + +impl Pool2dDirectStrategyFamily for MaxPoolStrategy { + type Indices = (); + type Config = (); + type Pool2d = Self; +} + +impl Pool2dDirectStrategyFamily for MaxPoolWithIndicesStrategy { + type Indices = View, Position, ReadWrite>; + type Config = (); + type Pool2d = Self; +} + +#[cube] +impl Pool2dDirectStrategy for MaxPoolStrategy { + type Accumulator = Line; + type Config = (); + type Indices = (); + + fn initialize( + #[comptime] _config: &Self::Config, + #[comptime] line_size: LineSize, + ) -> Self::Accumulator { + Line::empty(line_size).fill(N::min_value()) + } + + fn accumulate( + #[comptime] _config: &Self::Config, + accumulator: &mut Self::Accumulator, + _index: LineSize, + result: Line, + ) { + *accumulator = max(*accumulator, result); + } + + fn count_position( + #[comptime] _config: &Self::Config, + _accumulator: &mut Self::Accumulator, + _ih: u32, + _iw: u32, + ) { + } + + fn store( + #[comptime] _config: &Self::Config, + position: Position, + output: &mut View, Position, ReadWrite>, + _output_indices: &mut (), + accumulator: Self::Accumulator, + ) { + output[position] = accumulator; + } +} + +#[cube] +impl Pool2dDirectStrategy for MaxPoolWithIndicesStrategy { + type Accumulator = (Line, Line); + type Config = (); + type Indices = View, Position, ReadWrite>; + + fn initialize( + #[comptime] _config: &Self::Config, + #[comptime] line_size: LineSize, + ) -> Self::Accumulator { + let val = Line::empty(line_size).fill(N::min_value()); + let idx = Line::empty(line_size).fill(0i32); + (val, idx) + } + + fn accumulate( + #[comptime] _config: &Self::Config, + accumulator: &mut Self::Accumulator, + index: usize, + result: Line, + ) { + let indices = Line::cast_from(index); + accumulator.1 = select_many(result.greater_than(accumulator.0), indices, accumulator.1); + accumulator.0 = max(result, accumulator.0); + } + + fn count_position( + #[comptime] _config: &Self::Config, + _accumulator: &mut Self::Accumulator, + _ih: u32, + _iw: u32, + ) { + } + + fn store( + #[comptime] _config: &Self::Config, + position: Position, + output: &mut View, Position, ReadWrite>, + output_indices: &mut View, Position, ReadWrite>, + accumulator: Self::Accumulator, + ) { + output[position] = accumulator.0; + output_indices[position] = accumulator.1; + } +} + +pub(crate) fn max_pool2d( + x: CubeTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, +) -> CubeTensor { + let [batch_size, channels, height, width] = x.meta.shape().dims(); + + let size_0 = calculate_pool_output_size( + kernel_size[0], + stride[0], + padding[0], + dilation[0], + height, + ceil_mode, + ); + let size_1 = calculate_pool_output_size( + kernel_size[1], + stride[1], + padding[1], + dilation[1], + width, + ceil_mode, + ); + + let x = into_contiguous_aligned(permute_nchw_to_nhwc(x)); + + let line_size = max_line_size(&x); + + let shape_out = Shape::new([batch_size, size_0, size_1, channels]); + let output = empty_device_dtype(x.client.clone(), x.device.clone(), shape_out, x.dtype); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&x.client, working_units); + let cube_count = calculate_cube_count_elemwise(&x.client, working_units, cube_dim); + + pool2d_direct::launch::( + &x.client, + cube_count, + cube_dim, + address_type!(x, output), + x.as_tensor_arg(line_size), + view4d(&output, line_size), + (), + shape_divmod(&output), + ScalarArg::new(working_units), + Pool2dDirectArgsLaunch::new( + ScalarArg::new(stride[0] as u32), + ScalarArg::new(stride[1] as u32), + ScalarArg::new(dilation[0] as u32), + ScalarArg::new(dilation[1] as u32), + ScalarArg::new(padding[0] as u32), + ScalarArg::new(padding[1] as u32), + ), + (kernel_size[0] as u32, kernel_size[1] as u32), + (), + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + permute_nhwc_to_nchw(output) +} + +pub(crate) fn max_pool2d_with_indices( + x: CubeTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + dtype_indices: DType, +) -> (CubeTensor, CubeTensor) { + let [batch_size, channels, size_0, size_1] = x.meta.shape().dims(); + + let size_0 = calculate_pool_output_size( + kernel_size[0], + stride[0], + padding[0], + dilation[0], + size_0, + ceil_mode, + ); + let size_1 = calculate_pool_output_size( + kernel_size[1], + stride[1], + padding[1], + dilation[1], + size_1, + ceil_mode, + ); + + let x = into_contiguous_aligned(permute_nchw_to_nhwc(x)); + let line_size = max_line_size(&x); + + let shape_out = Shape::new([batch_size, size_0, size_1, channels]); + let output = empty_device_dtype( + x.client.clone(), + x.device.clone(), + shape_out.clone(), + x.dtype, + ); + let indices = empty_device_dtype(x.client.clone(), x.device.clone(), shape_out, dtype_indices); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&x.client, working_units); + let cube_count = calculate_cube_count_elemwise(&x.client, working_units, cube_dim); + + pool2d_direct::launch::( + &x.client, + cube_count, + cube_dim, + address_type!(x, output, indices), + x.as_tensor_arg(line_size), + view4d(&output, line_size), + view4d(&indices, line_size), + shape_divmod(&output), + ScalarArg::new(working_units), + Pool2dDirectArgsLaunch::new( + ScalarArg::new(stride[0] as u32), + ScalarArg::new(stride[1] as u32), + ScalarArg::new(dilation[0] as u32), + ScalarArg::new(dilation[1] as u32), + ScalarArg::new(padding[0] as u32), + ScalarArg::new(padding[1] as u32), + ), + (kernel_size[0] as u32, kernel_size[1] as u32), + (), + output.dtype.into(), + ) + .expect("Kernel to never fail"); + + let output = permute_nhwc_to_nchw(output); + let indices = permute_nhwc_to_nchw(indices); + (output, indices) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d_backward.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d_backward.rs new file mode 100644 index 0000000..22c2239 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/max_pool2d_backward.rs @@ -0,0 +1,156 @@ +use crate::{ + CubeRuntime, + kernel::{ + into_contiguous_aligned, + utils::{address_type, decompose_linear, shape_divmod}, + }, + ops::{max_line_size, numeric::empty_device_dtype, permute_nchw_to_nhwc, permute_nhwc_to_nchw}, + tensor::CubeTensor, +}; +use burn_backend::Shape; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::FastDivmod}; + +use super::{PoolBackwardArgs, PoolBackwardArgsLaunch}; + +#[cube(launch_unchecked, address_type = "dynamic")] +fn max_pool2d_with_indices_backward_kernel( + grad: &Tensor>, + indices: &Tensor>, + output: &mut Tensor>, + out_shape: Sequence>, + working_units: usize, + args: &PoolBackwardArgs, + #[comptime] kernel_size_0: i32, + #[comptime] kernel_size_1: i32, + #[define(E, I)] _dtypes: [StorageType; 2], +) { + if ABSOLUTE_POS >= working_units { + terminate!(); + } + + let (_, pos) = decompose_linear(ABSOLUTE_POS * output.line_size(), &out_shape); + let [batch, ih, iw, channel] = *pos else { + unreachable!() + }; + + let line_size = grad.line_size(); + + let index_current = ih * output.shape(2) + iw; + + let (oh_start, oh_end, ow_start, ow_end) = loop_ranges( + ih as i32, + iw as i32, + grad.shape(1) as u32, + grad.shape(2) as u32, + args, + kernel_size_0, + kernel_size_1, + ); + + let mut grad_acc = Line::empty(grad.line_size()).fill(E::from_int(0)); + + let grad_idx_base = batch * grad.stride(0) + channel * grad.stride(3); + let ind_idx_base = batch * indices.stride(0) + channel * indices.stride(3); + + for oh in oh_start..oh_end { + for ow in ow_start..ow_end { + let grad_index = + grad_idx_base + oh as usize * grad.stride(1) + ow as usize * grad.stride(2); + let indices_index = + ind_idx_base + oh as usize * indices.stride(1) + ow as usize * indices.stride(2); + let index_max = Line::::cast_from(indices[indices_index / line_size]); + + grad_acc += select_many( + index_max.equal(Line::cast_from(index_current)), + grad[grad_index / line_size], + Line::new(E::from_int(0)), + ); + } + } + + let index_output = batch * output.stride(0) + + ih * output.stride(1) + + iw * output.stride(2) + + channel * output.stride(3); + + output[index_output / output.line_size()] = grad_acc; +} + +#[cube] +fn loop_ranges( + ih: i32, + iw: i32, + grad_h: u32, + grad_w: u32, + args: &PoolBackwardArgs, + #[comptime] kernel_size_0: i32, + #[comptime] kernel_size_1: i32, +) -> (u32, u32, u32, u32) { + let kms_0 = args.dilation_0 * kernel_size_0 - args.stride_0; + let kms_1 = args.dilation_1 * kernel_size_1 - args.stride_1; + + let oh_start = clamp_min((ih + args.padding_0 - kms_0) / args.stride_0, 0) as u32; + let ow_start = clamp_min((iw + args.padding_1 - kms_1) / args.stride_1, 0) as u32; + let oh_end = clamp_max(clamp_min(kms_0, 0) as u32 + oh_start, grad_h - 1) + 1; + let ow_end = clamp_max(clamp_min(kms_1, 0) as u32 + ow_start, grad_w - 1) + 1; + + (oh_start, oh_end, ow_start, ow_end) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn max_pool2d_with_indices_backward( + x: CubeTensor, + grad: CubeTensor, + indices: CubeTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + _ceil_mode: bool, +) -> CubeTensor { + let [batches, channels, height, width] = x.meta.shape().dims(); + + let grad = into_contiguous_aligned(permute_nchw_to_nhwc(grad)); + let indices = into_contiguous_aligned(permute_nchw_to_nhwc(indices)); + + let line_size = if grad.meta.strides()[3] == indices.meta.strides()[3] { + max_line_size(&grad) + } else { + 1 + }; + + let out_shape = Shape::new([batches, height, width, channels]); + let output = empty_device_dtype(x.client.clone(), x.device.clone(), out_shape, x.dtype); + + let working_units = output.meta.num_elements() / line_size as usize; + let cube_dim = CubeDim::new(&x.client, working_units); + let cube_count = calculate_cube_count_elemwise(&x.client, working_units, cube_dim); + + unsafe { + max_pool2d_with_indices_backward_kernel::launch_unchecked( + &x.client, + cube_count, + cube_dim, + address_type!(grad, indices, output), + grad.as_tensor_arg(line_size), + indices.as_tensor_arg(line_size), + output.as_tensor_arg(line_size), + shape_divmod(&output), + ScalarArg::new(working_units), + PoolBackwardArgsLaunch::new( + ScalarArg::new(stride[0] as i32), + ScalarArg::new(stride[1] as i32), + ScalarArg::new(dilation[0] as i32), + ScalarArg::new(dilation[1] as i32), + ScalarArg::new(padding[0] as i32), + ScalarArg::new(padding[1] as i32), + ), + kernel_size[0] as i32, + kernel_size[1] as i32, + [x.dtype.into(), indices.dtype.into()], + ) + .expect("Kernel to never fail") + }; + + permute_nhwc_to_nchw(output) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/mod.rs new file mode 100644 index 0000000..73b4249 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/mod.rs @@ -0,0 +1,15 @@ +mod adaptive_avg_pool2d; +mod adaptive_avg_pool2d_backward; +mod avg_pool2d; +mod avg_pool2d_backward; +mod max_pool2d; +mod max_pool2d_backward; + +pub(super) mod pool2d; + +pub(crate) use adaptive_avg_pool2d::*; +pub(crate) use adaptive_avg_pool2d_backward::*; +pub(crate) use avg_pool2d::*; +pub(crate) use avg_pool2d_backward::*; +pub(crate) use max_pool2d::*; +pub(crate) use max_pool2d_backward::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/pool2d.rs new file mode 100644 index 0000000..7dde8fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/pool/pool2d.rs @@ -0,0 +1,155 @@ +use core::hash::Hash; +use cubecl::{ + prelude::*, + std::{ + FastDivmod, + tensor::{ + View, + launch::ViewArg, + layout::fixed_dim::{FixedDimLayout, FixedDimLayoutLaunch}, + }, + }, +}; + +use crate::{CubeRuntime, kernel::utils::decompose_linear, tensor::CubeTensor}; + +pub trait Pool2dDirectStrategyFamily: Send + Sync + 'static { + type Indices: LaunchArg; + type Config: CubeType + Clone + Send + Sync + core::fmt::Debug + Hash + core::cmp::Eq; + type Pool2d: Pool2dDirectStrategy; +} + +pub(super) type Position = (usize, usize, usize, usize); + +#[cube] +pub(crate) trait Pool2dDirectStrategy: Send + Sync + 'static { + type Accumulator: CubeType; + type Config: CubeType + Clone + Send + Sync + core::fmt::Debug + Hash + core::cmp::Eq; + + type Indices: LaunchArg; + + fn initialize( + #[comptime] config: &Self::Config, + #[comptime] line_size: LineSize, + ) -> Self::Accumulator; + + fn accumulate( + #[comptime] config: &Self::Config, + accumulator: &mut Self::Accumulator, + index: usize, + result: Line, + ); + + /// Count a position within the kernel window (for avg_pool count_include_pad). + /// Called for each position in the kernel window with the current ih/iw coordinates. + /// Only avg_pool uses this; max_pool implements as no-op. + fn count_position( + #[comptime] config: &Self::Config, + accumulator: &mut Self::Accumulator, + ih: u32, + iw: u32, + ); + + fn store( + #[comptime] config: &Self::Config, + position: Position, + output: &mut View, Position, ReadWrite>, + output_indices: &mut Self::Indices, + accumulator: Self::Accumulator, + ); +} + +#[derive(CubeLaunch, CubeType)] +pub struct Pool2dDirectArgs { + pub strides_0: u32, + pub strides_1: u32, + pub dilation_0: u32, + pub dilation_1: u32, + pub padding_0: u32, + pub padding_1: u32, +} + +#[cube(launch, address_type = "dynamic")] +pub fn pool2d_direct( + input: &Tensor>, + output: &mut View, Position, ReadWrite>, + indices: &mut S::Indices, + out_shape: Sequence>, + working_units: usize, + args: &Pool2dDirectArgs, + #[comptime] kernel_size: (u32, u32), + #[comptime] config: &S::Config, + #[define(E)] _dtype: StorageType, +) { + if ABSOLUTE_POS >= working_units { + terminate!(); + } + + let (_, pos) = decompose_linear(ABSOLUTE_POS * output.line_size(), &out_shape); + let [b, oh, ow, c] = *pos else { unreachable!() }; + + let (in_stride_h, in_stride_w) = (input.stride(1), input.stride(2)); + let (in_h, in_w) = (input.shape(1) as u32, input.shape(2) as u32); + + let mut accumulator = S::Pool2d::::initialize(config, input.line_size()); + + let in_b_off = b * input.stride(0); + let in_c_off = c * input.stride(3); + + let border_bottom = in_h + args.padding_0; + let border_right = in_w + args.padding_1; + + for kh in 0..kernel_size.0 { + let ih = oh as u32 * args.strides_0 + kh * args.dilation_0; + let within_padding_h = ih >= args.padding_0 && ih < border_bottom; + + for kw in 0..kernel_size.1 { + let iw = ow as u32 * args.strides_1 + kw * args.dilation_1; + let within_padding_w = iw >= args.padding_1 && iw < border_right; + + // Let strategy handle position counting (only used by avg_pool) + S::Pool2d::::count_position(config, &mut accumulator, ih, iw); + + // Only accumulate values from valid input positions + if within_padding_h && within_padding_w { + let ih_pad = ih - args.padding_0; + let iw_pad = iw - args.padding_1; + + let in_h_off = ih_pad as usize * in_stride_h; + let in_w_off = iw_pad as usize * in_stride_w; + + let index_input = in_b_off + in_c_off + in_h_off + in_w_off; + + S::Pool2d::::accumulate( + config, + &mut accumulator, + ih_pad as usize * in_w as usize + iw_pad as usize, + input[index_input / input.line_size()], + ); + } + } + } + + S::Pool2d::::store(config, (b, oh, ow, c), output, indices, accumulator); +} + +pub(super) fn view4d( + tensor: &CubeTensor, + line_size: LineSize, +) -> ViewArg<'_, Position, R> { + let shape = tensor.meta.shape(); + let shape = ( + ScalarArg::new(shape[0]), + ScalarArg::new(shape[1]), + ScalarArg::new(shape[2]), + ScalarArg::new(shape[3]), + ); + let handle = tensor.as_handle_ref(); + let len = handle.shape.iter().product::(); + let layout = + FixedDimLayoutLaunch::::from_shape_handle_unchecked(&handle, shape, line_size); + let buffer = unsafe { + ArrayArg::from_raw_parts_and_size(handle.handle, len, line_size, handle.elem_size) + }; + ViewArg::new::>(buffer, layout) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/bernoulli.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/bernoulli.rs new file mode 100644 index 0000000..e83c5d6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/bernoulli.rs @@ -0,0 +1,18 @@ +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; +use burn_backend::{DType, Shape}; + +/// Pseudo-random generator with bernoulli distribution +pub fn random_bernoulli( + shape: Shape, + device: &R::Device, + probability: f32, + dtype: DType, +) -> CubeTensor { + let client = R::client(device); + let output = empty_device_dtype(client.clone(), device.clone(), shape, dtype); + + cubek::random::random_bernoulli(&client, probability, output.as_handle_ref(), dtype.into()) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/mod.rs new file mode 100644 index 0000000..10f2cfe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/mod.rs @@ -0,0 +1,7 @@ +mod bernoulli; +mod normal; +mod uniform; + +pub use bernoulli::*; +pub use normal::*; +pub use uniform::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/normal.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/normal.rs new file mode 100644 index 0000000..06c483c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/normal.rs @@ -0,0 +1,20 @@ +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; +use burn_backend::{DType, Shape}; + +/// Pseudo-random generator with uniform distribution +pub fn random_normal( + shape: Shape, + device: &R::Device, + mean: f32, + std: f32, + dtype: DType, +) -> CubeTensor { + let client = R::client(device); + let output = empty_device_dtype(client.clone(), device.clone(), shape, dtype); + let output_handle = output.as_handle_ref(); + + cubek::random::random_normal(&client, mean, std, output_handle, dtype.into()) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/uniform.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/uniform.rs new file mode 100644 index 0000000..06cf097 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/prng/uniform.rs @@ -0,0 +1,43 @@ +use crate::{CubeRuntime, ops::numeric::empty_device_dtype, tensor::CubeTensor}; +use burn_backend::{DType, Shape, TensorMetadata}; + +/// Pseudo-random generator with uniform distribution +pub fn random_uniform( + shape: Shape, + device: &R::Device, + lower_bound: f32, + upper_bound: f32, + dtype: DType, +) -> CubeTensor { + let client = R::client(device); + let output = empty_device_dtype(client.clone(), device.clone(), shape, dtype); + let output_handle = output.as_handle_ref(); + + cubek::random::random_uniform( + &client, + lower_bound, + upper_bound, + output_handle, + dtype.into(), + ) + .expect("Kernel to never fail"); + + output +} + +/// Pseudo-random generator for uniform distribution, based on +/// another tensor. +pub fn random_like_uniform( + tensor: &CubeTensor, + lower_bound: f32, + upper_bound: f32, + dtype: DType, +) -> CubeTensor { + random_uniform( + tensor.shape(), + &tensor.device, + lower_bound, + upper_bound, + dtype, + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/dequantize.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/dequantize.rs new file mode 100644 index 0000000..747d3a2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/dequantize.rs @@ -0,0 +1,34 @@ +use crate::tensor::CubeTensor; +use crate::{CubeRuntime, ops::numeric::empty_device_dtype}; +use burn_backend::{DType, TensorMetadata}; + +/// Convert the tensor back to a higher precision data type. +pub fn dequantize(tensor: CubeTensor, dtype: DType) -> CubeTensor +where + R: CubeRuntime, +{ + let scheme = match tensor.dtype { + DType::QFloat(scheme) => scheme, + _ => return tensor, + }; + + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + dtype, + ); + let (values, params) = tensor.quantized_handles().unwrap(); + + cubek::quantization::dequantize::launch_ref( + &values.client, + &values.as_handle_ref(), + &output.as_handle_ref(), + ¶ms.as_handle_ref(), + &scheme, + dtype.into(), + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/mod.rs new file mode 100644 index 0000000..a0244df --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/mod.rs @@ -0,0 +1,5 @@ +mod dequantize; +mod quantize; + +pub use dequantize::*; +pub use quantize::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/quantize.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/quantize.rs new file mode 100644 index 0000000..1c2d8bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/quantization/quantize.rs @@ -0,0 +1,29 @@ +use crate::CubeRuntime; +use crate::{ops::empty_qtensor_optimized, tensor::CubeTensor}; +use burn_backend::{TensorMetadata, quantization::QuantScheme}; + +/// Convert the tensor to a lower precision data type based on the quantization scheme and parameters. +pub fn quantize( + tensor: CubeTensor, + scheme: &QuantScheme, + scale: CubeTensor, +) -> CubeTensor +where + R: CubeRuntime, +{ + let output = empty_qtensor_optimized(tensor.shape(), *scheme, &tensor.device); + let (out_values, out_params) = output.clone().quantized_handles().unwrap(); + + cubek::quantization::quantize::launch_ref( + &tensor.client, + &tensor.as_handle_ref(), + &out_values.as_handle_ref(), + &scale.as_handle_ref(), + &out_params.as_handle_ref(), + scheme, + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/base.rs new file mode 100644 index 0000000..7d7b283 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/base.rs @@ -0,0 +1,242 @@ +#[cfg(feature = "autotune")] +use super::{autotune_reduce, autotune_sum}; +use crate::{ + CubeRuntime, + ops::numeric::{empty_device_contiguous_dtype, zeros_client}, + tensor::CubeTensor, +}; +use burn_backend::{DType, TensorMetadata}; +use burn_std::Metadata; +use cubecl::{AutotuneKey, client::ComputeClient, features::TypeUsage, ir::StorageType}; +use cubek::reduce::{ + ReduceDtypes, ReduceError, ReduceStrategy, + components::instructions::ReduceOperationConfig, + launch::{LineSizeStrategy, RoutineStrategy}, + routines::{BlueprintStrategy, unit::UnitStrategy}, + shared_sum, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize, AutotuneKey)] +/// Autotune key representative of sum versions +pub struct SumAutotuneKey { + /// The type of the tensor + dtype: burn_backend::DType, + /// The anchored length of the tensor + #[autotune(anchor)] + length: usize, +} + +/// Check if the client supports atomic add for the given element type. +fn supports_atomic_add(client: &ComputeClient, dtype: DType) -> bool { + client + .properties() + .type_usage(StorageType::Atomic(dtype.into())) + .contains(TypeUsage::AtomicAdd) +} + +/// [Sum](sum) with fallback when `client` doesn't support atomic add for the type `E`. +pub fn sum_fallback( + tensor: CubeTensor, + mut strategy: SumStrategy, +) -> Result, ReduceError> { + // Early check before creating output and fallback + if matches!(strategy, SumStrategy::OneShot(_)) + && !supports_atomic_add(&tensor.client, tensor.dtype) + { + strategy = SumStrategy::Chained(Default::default()); + } + sum(tensor, strategy) +} + +/// Specialize reduce function to compute the sum of all elements of the `input` tensor and return +/// the value into a single-element tensor of shape `1 x 1 x 1 x ...` with the same rank as `input`. +/// +/// This is expected to be faster for larger tensors than calling [reduce] with the `Sum` instruction. +/// +/// Return an error if the `client` doesn't support atomic add for the type `E`. +pub fn sum( + tensor: CubeTensor, + strategy: SumStrategy, +) -> Result, ReduceError> { + let client = tensor.client.clone(); + let device = tensor.device.clone(); + + match strategy { + SumStrategy::OneShot(cube_count) => { + let output = zeros_client(client.clone(), device, [1].into(), tensor.dtype); + shared_sum::( + &client, + tensor.as_handle_ref(), + output.as_handle_ref(), + cube_count, + tensor.dtype.into(), + )?; + + Ok(output) + } + SumStrategy::Chained(strategy) => { + reduce::(tensor, None, strategy, ReduceOperationConfig::Sum) + } + #[cfg(feature = "autotune")] + SumStrategy::Autotune => Ok(autotune_sum::(&client, tensor)), + } +} + +/// Select a strategy to perform a sum. +pub enum SumStrategy { + /// Run a single kernel with many cubes working in parallel to sum all elements. + /// The provided value is the number of elements summed per unit (up-to-rounding ) + OneShot(u32), + /// Use multiple kernels + Chained(KernelReduceStrategy), + /// Use autotune to find the best cube count given the hardware and the input. + #[cfg(feature = "autotune")] + Autotune, +} + +impl Default for SumStrategy { + fn default() -> Self { + #[cfg(feature = "autotune")] + return Self::Autotune; + + #[cfg(not(feature = "autotune"))] + return Self::OneShot(4); + } +} + +/// Reduce all elements of the `input` tensor using the instruction `Rd` and the given [Strategy](ReduceStrategy). +/// +/// Return an error if `strategy` is `Specific(strategy)` and the specified strategy is not supported by the `client`. +/// +/// If there is no error, the output is a tensor with decreasing strides +/// where the shape of reduced dim is set to 1 but all shape are similar to the input. +pub fn reduce( + mut tensor: CubeTensor, + output_dtype: Option, + strategy: KernelReduceStrategy, + config: ReduceOperationConfig, +) -> Result, cubek::reduce::ReduceError> { + // In practice, it looks like starting by the axis with the smallest shape + // and going in increasing order lead to the fastest calculation. + let sorted_axis = argsort(tensor.meta.shape()); + for axis in sorted_axis { + tensor = reduce_dim::(tensor, output_dtype, axis, strategy.clone(), config)?; + } + // reshape to scalar tensor + *tensor.meta = Metadata::new([1], [1]); + Ok(tensor) +} + +fn argsort(shape: &[usize]) -> Vec { + let mut indices = (0..shape.len()).collect::>(); + indices.sort_by_key(|&i| &shape[i]); + indices +} + +/// Reduce the given `axis` of the `input` tensor using the instruction `Rd` and the given [Strategy](ReduceStrategy). +/// +/// Return an error if `strategy` is `Specific(strategy)` and the specified strategy is not supported by the `client`. +/// Also returns an error if the `axis` is larger than the `input` rank or if the shape of `output` is invalid. +/// +/// If there is no error, the output is a tensor with decreasing strides +/// where the shape of reduced dim is set to 1 but all shape are similar to the input. +pub fn reduce_dim( + input: CubeTensor, + output_dtype: Option, + dim: usize, + strategy: KernelReduceStrategy, + config: ReduceOperationConfig, +) -> Result, cubek::reduce::ReduceError> { + debug_assert!( + !matches!( + config, + ReduceOperationConfig::ArgMax | ReduceOperationConfig::ArgMin + ) || output_dtype.is_some(), + "The `output_dtype` has to be `Some` only when the `config` is `ArgMax` or `ArgMin`. + " + ); + + let dtypes = config.precision(input.dtype.into(), output_dtype.map(Into::into)); + let client = input.client.clone(); + let output = init_reduce_output::(&input, dim, &dtypes).ok_or( + cubek::reduce::ReduceError::InvalidAxis { + axis: dim, + rank: input.meta.num_dims(), + }, + )?; + + let result = match strategy { + KernelReduceStrategy::Unspecified => cubek::reduce::reduce::( + &client, + input.as_handle_ref(), + output.as_handle_ref(), + dim, + ReduceStrategy { + routine: RoutineStrategy::Unit(BlueprintStrategy::Inferred(UnitStrategy)), + line_size: LineSizeStrategy { + parallel_output_vectorization: false, + }, + }, + config, + dtypes, + ), + KernelReduceStrategy::Specific(strategy) => cubek::reduce::reduce::( + &client, + input.as_handle_ref(), + output.as_handle_ref(), + dim, + strategy, + config, + dtypes, + ), + #[cfg(feature = "autotune")] + KernelReduceStrategy::Autotune => { + autotune_reduce::(&client, input, output.clone(), dim, config, dtypes); + Ok(()) + } + }; + result.map(|_| output) +} + +/// Creates an empty output tensor with the proper shape and decreasing strides to reduce the given `axis` of `input` +/// or return `None` if `axis` is out-of-bound. +pub fn init_reduce_output( + input: &CubeTensor, + dim: usize, + dtypes: &ReduceDtypes, +) -> Option> { + (dim < input.meta.num_dims()).then(|| { + let mut shape_out = input.shape(); + shape_out[dim] = 1; + empty_device_contiguous_dtype( + input.client.clone(), + input.device.clone(), + shape_out, + dtypes.output.elem_type().into(), + ) + }) +} + +/// Select a strategy to perform a reduction. +#[derive(Clone, Debug)] +pub enum KernelReduceStrategy { + /// Use a best-effort strategy based on the hardware capacity. + /// This differs from Autotune as it doesn't try and compare many strategies to select the best. + Unspecified, + /// Fix the exact strategy for the reduction. + Specific(cubek::reduce::launch::ReduceStrategy), + /// Use autotune to find the best strategy given the hardware and the inputs. + #[cfg(feature = "autotune")] + Autotune, +} + +impl Default for KernelReduceStrategy { + fn default() -> Self { + #[cfg(feature = "autotune")] + return Self::Autotune; + + #[cfg(not(feature = "autotune"))] + return Self::Unspecified; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/mod.rs new file mode 100644 index 0000000..13a6c23 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/mod.rs @@ -0,0 +1,7 @@ +mod base; +#[cfg(feature = "autotune")] +mod tune; + +pub use base::*; +#[cfg(feature = "autotune")] +pub use tune::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/tune.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/tune.rs new file mode 100644 index 0000000..25c3c23 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/reduce/tune.rs @@ -0,0 +1,286 @@ +#![allow(missing_docs)] + +use super::SumAutotuneKey; +use crate::{CubeAutotuneKey, CubeRuntime, CubeTuneId, tensor::CubeTensor}; +use cubecl::{ + client::ComputeClient, + tune::{LocalTuner, Tunable, TunableSet, TuneGroup, local_tuner}, +}; +use cubek::reduce::{ + ReduceDtypes, ReduceStrategy, + components::instructions::ReduceOperationConfig, + launch::{LineSizeStrategy, RoutineStrategy, tune_key::ReduceAutotuneKey}, + routines::{BlueprintStrategy, cube::CubeStrategy, plane::PlaneStrategy, unit::UnitStrategy}, +}; + +/// Executes autotune on reduce operations. +pub fn autotune_reduce( + client: &ComputeClient, + input: CubeTensor, + output: CubeTensor, + axis: usize, + config: ReduceOperationConfig, + dtypes: ReduceDtypes, +) { + use reduce_ops::*; + + static TUNER: LocalTuner = local_tuner!("reduce-dim"); + + let tunables = TUNER.init(|| { + const PRIORITY_MAX: i8 = 2; + const PRIORITY_MIN: i8 = 1; + const PRIORITY_SKIP: i8 = -1; + + let mut set = TunableSet::new(create_key::, reduce_input_gen::); + + let default_group = + TuneGroup::::new("default_reduce", |_key| PRIORITY_MAX); + let vectorized_parallel_group = + TuneGroup::::new("vectorized_parallel_reduce", |key| { + if key.axis_is_contiguous { + PRIORITY_MAX + } else { + // We disable the tunable with the setting [line_size.parallel_output_vectorization] + // when the reduce isn't parallel, since it would duplicate tunables. + PRIORITY_SKIP + } + }); + + enum ReduceProps { + GreatWithLowReduceCount, + GreatWithHighReduceCount, + Balanced, + } + + for (line_size, line_size_ident) in [ + ( + LineSizeStrategy { + parallel_output_vectorization: true, + }, + "_vectorized_parallel_reduce", + ), + ( + LineSizeStrategy { + parallel_output_vectorization: false, + }, + "", + ), + ] { + for (name, routine, props) in [ + ( + "unit", + RoutineStrategy::Unit(BlueprintStrategy::Inferred(UnitStrategy)), + ReduceProps::GreatWithHighReduceCount, + ), + ( + "plane", + RoutineStrategy::Plane(BlueprintStrategy::Inferred(PlaneStrategy { + independent: true, + })), + ReduceProps::Balanced, + ), + ( + "cube", + RoutineStrategy::Cube(BlueprintStrategy::Inferred(CubeStrategy { + use_planes: true, + })), + ReduceProps::GreatWithLowReduceCount, + ), + ] { + let name = format!("{name}{line_size_ident}"); + let mut tunable = Tunable::new( + name, + move |(input, output, axis, config, dtypes): ( + CubeTensor, + CubeTensor, + usize, + ReduceOperationConfig, + ReduceDtypes, + )| { + let strategy = ReduceStrategy { + routine: routine.clone(), + line_size, + }; + cubek::reduce::reduce::( + &input.client, + input.as_handle_ref(), + output.as_handle_ref(), + axis, + strategy, + config, + dtypes, + ) + .map_err(|e| format!("{e}")) + }, + ); + if line_size.parallel_output_vectorization { + tunable = tunable.group(&vectorized_parallel_group, |_| PRIORITY_MAX); + } + + tunable = tunable.group(&default_group, move |key| match props { + ReduceProps::GreatWithLowReduceCount => { + if key.vector_count < 128 { + PRIORITY_MAX + } else { + // When you have a high level of vector to reduce, it is normally + // better to use another routine. + PRIORITY_MIN + } + } + ReduceProps::GreatWithHighReduceCount => { + if key.vector_count > 64 { + PRIORITY_MAX + } else { + // Bellow 64 it is normally better to use another routine + PRIORITY_MIN + } + } + ReduceProps::Balanced => PRIORITY_MAX, + }); + set = set.with(tunable); + } + } + + set + }); + + TUNER.execute( + &CubeTuneId::new(&input.client, &input.device), + client, + tunables, + (input, output, axis, config, dtypes), + ); +} + +pub(crate) fn create_key( + input: &CubeTensor, + output: &CubeTensor, + axis: &usize, + _config: &ReduceOperationConfig, + dtypes: &ReduceDtypes, +) -> ReduceAutotuneKey { + let elem_input = input.dtype.into(); + let elem_output = output.dtype.into(); + let elem_acc = dtypes.accumulation.elem_type(); + + ReduceAutotuneKey::generate( + elem_input, + elem_output, + elem_acc, + input.meta.shape(), + input.meta.strides()[*axis] == 1, + *axis, + ) +} + +mod reduce_ops { + #![allow(missing_docs)] + + use cubek::reduce::ReduceDtypes; + + use super::*; + + pub(crate) fn reduce_input_gen( + _key: &ReduceAutotuneKey, + input: &CubeTensor, + output: &CubeTensor, + dim: &usize, + config: &ReduceOperationConfig, + dtypes: &ReduceDtypes, + ) -> ( + CubeTensor, + CubeTensor, + usize, + ReduceOperationConfig, + ReduceDtypes, + ) { + (input.clone(), output.copy(), *dim, *config, *dtypes) + } +} + +/// Executes autotune on reduce operations. +#[cfg(feature = "autotune")] +pub fn autotune_sum( + client: &ComputeClient, + input: CubeTensor, +) -> CubeTensor { + use sum_ops::*; + + static TUNER: LocalTuner = local_tuner!("autotune-sum"); + + let tunables = TUNER.init(|| { + TunableSet::new(create_key_sum::, sum_input_gen::) + .with(Tunable::new("sum_chained", sum_chained::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + .with(Tunable::new("sum_one_shot", sum_one_shot::)) + }); + + TUNER.execute( + &CubeTuneId::new(&input.client, &input.device), + client, + tunables, + input, + ) +} + +pub(crate) fn create_key_sum(input: &CubeTensor) -> CubeAutotuneKey { + CubeAutotuneKey::Sum(SumAutotuneKey::generate(input)) +} + +impl SumAutotuneKey { + #[allow(unused)] + pub(crate) fn generate(input: &CubeTensor) -> Self { + let dtype = input.dtype; + let length = input.meta.num_elements(); + Self::new(dtype, length) + } +} +mod sum_ops { + #![allow(missing_docs)] + use crate::ops::numeric::zeros_client; + + use super::*; + + pub(crate) fn sum_input_gen( + _key: &CubeAutotuneKey, + input: &CubeTensor, + ) -> CubeTensor { + input.clone() + } + + pub(crate) fn sum_one_shot( + input: CubeTensor, + ) -> Result, String> { + let client = input.client.clone(); + let device = input.device.clone(); + let output = zeros_client(client.clone(), device, [1].into(), input.dtype); + + cubek::reduce::shared_sum::( + &input.client, + input.as_handle_ref(), + output.as_handle_ref(), + C, + input.dtype.into(), + ) + .map_err(|e| e.to_string()) + .map(|_| output) + } + + #[cfg(feature = "autotune")] + pub(crate) fn sum_chained( + input: CubeTensor, + ) -> Result, String> { + crate::kernel::reduce::reduce::( + input, + None, + crate::kernel::reduce::KernelReduceStrategy::Autotune, + cubek::reduce::components::instructions::ReduceOperationConfig::Sum, + ) + .map_err(|e| e.to_string()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_float.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_float.rs new file mode 100644 index 0000000..7f67d93 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_float.rs @@ -0,0 +1,191 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, linear_view_alias}, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::TensorMetadata; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +pub(crate) trait FloatUnaryOpFamily: 'static + Send + Sync { + type Options: LaunchArg; + type Unary: FloatUnaryOp; +} + +#[cube] +pub(crate) trait FloatUnaryOp: 'static + Send + Sync { + type Options: LaunchArg; + + fn execute(input: Line, options: &Self::Options) -> Line; +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn unary_float( + input: &LinearView>, + output: &mut LinearView, ReadWrite>, + options: &O::Options, + #[define(F)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = O::Unary::::execute(input[ABSOLUTE_POS], options); +} + +pub(crate) fn launch_unary_float(tensor: CubeTensor, args: Args) -> CubeTensor +where + // Magic fix for lifetime, the closure is supposed to capture everything required to create the + // argument. + for<'a> Args: FnOnce(&'a ()) -> RuntimeArg<'a, O::Options, R>, + R: CubeRuntime, + O: FloatUnaryOpFamily, +{ + let line_size = max_line_size(&tensor); + + let client = tensor.client.clone(); + let num_elems = tensor.meta.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + if tensor.can_mut() && tensor.is_nonoverlapping() { + unary_float::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor), + linear_view(&tensor, line_size), + linear_view_alias(&tensor, line_size, 0), + args(&()), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + tensor + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + tensor.dtype, + ); + + unary_float::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + linear_view(&output, line_size), + args(&()), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} + +/// Use comptime enum to implement all unary operations that don't have any input argument in the +/// kernel definition. +pub(crate) mod unary_basic { + use super::*; + + pub(crate) fn launch(tensor: CubeTensor, args: Args) -> CubeTensor + where + R: CubeRuntime, + for<'a> Args: FnOnce(&'a ()) -> BasicFloatUnaryKind, + { + launch_unary_float::(tensor, |input| { + BasicFloatUnaryOptionsLaunch::new(args(input)) + }) + } + + #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)] + pub enum BasicFloatUnaryKind { + Exp, + Log, + Log1p, + Sqrt, + Abs, + Sign, + ArcCos, + ArcCosh, + ArcSin, + ArcSinh, + ArcTan, + ArcTanh, + Cos, + Cosh, + Sin, + Sinh, + Tan, + Tanh, + Round, + Floor, + Ceil, + Trunc, + Erf, + Recip, + } + + #[derive(CubeLaunch, CubeType)] + struct BasicFloatUnaryOptions { + #[cube(comptime)] + kind: BasicFloatUnaryKind, + } + struct BasicFloatUnary; + + #[cube] + impl FloatUnaryOp for BasicFloatUnary { + type Options = BasicFloatUnaryOptions; + + fn execute(input: Line, options: &Self::Options) -> Line { + match comptime![options.kind] { + BasicFloatUnaryKind::Exp => Line::exp(input), + BasicFloatUnaryKind::Log => Line::ln(input), + BasicFloatUnaryKind::Log1p => Line::log1p(input), + BasicFloatUnaryKind::Sqrt => Line::sqrt(input), + BasicFloatUnaryKind::Abs => Line::abs(input), + BasicFloatUnaryKind::Sign => { + let zero = Line::new(F::new(0.0)); + let one = Line::new(F::new(1.0)); + let minus_one = Line::new(F::new(-1.0)); + + let is_positive = input.greater_than(zero); + let is_negative = input.less_than(zero); + let sign = select_many(is_negative, minus_one, zero); + + select_many(is_positive, one, sign) + } + BasicFloatUnaryKind::Cos => Line::cos(input), + BasicFloatUnaryKind::Sin => Line::sin(input), + BasicFloatUnaryKind::Tan => Line::tan(input), + BasicFloatUnaryKind::Cosh => Line::cosh(input), + BasicFloatUnaryKind::Sinh => Line::sinh(input), + BasicFloatUnaryKind::Tanh => Line::tanh(input), + BasicFloatUnaryKind::Round => Line::round(input), + BasicFloatUnaryKind::Floor => Line::floor(input), + BasicFloatUnaryKind::Ceil => Line::ceil(input), + BasicFloatUnaryKind::Trunc => Line::trunc(input), + BasicFloatUnaryKind::Erf => Line::erf(input), + BasicFloatUnaryKind::Recip => Line::recip(input), + BasicFloatUnaryKind::ArcCos => Line::acos(input), + BasicFloatUnaryKind::ArcCosh => Line::acosh(input), + BasicFloatUnaryKind::ArcSin => Line::asin(input), + BasicFloatUnaryKind::ArcSinh => Line::asinh(input), + BasicFloatUnaryKind::ArcTan => Line::atan(input), + BasicFloatUnaryKind::ArcTanh => Line::atanh(input), + } + } + } + + impl FloatUnaryOpFamily for BasicFloatUnary { + type Options = BasicFloatUnaryOptions; + type Unary = Self; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_int.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_int.rs new file mode 100644 index 0000000..c941e74 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_int.rs @@ -0,0 +1,142 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, linear_view_alias}, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::TensorMetadata; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +pub(crate) trait IntUnaryOpFamily: 'static + Send + Sync { + type Options: LaunchArg; + type Unary: IntUnaryOp; +} + +#[cube] +pub(crate) trait IntUnaryOp: 'static + Send + Sync { + type Options: LaunchArg; + + fn execute(input: Line, options: &Self::Options) -> Line; +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn unary_int( + input: &LinearView>, + output: &mut LinearView, ReadWrite>, + options: &O::Options, + #[define(I)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = O::Unary::::execute(input[ABSOLUTE_POS], options); +} + +pub(crate) fn launch_unary_int(tensor: CubeTensor, args: Args) -> CubeTensor +where + for<'a> Args: FnOnce(&'a ()) -> RuntimeArg<'a, O::Options, R>, + R: CubeRuntime, + O: IntUnaryOpFamily, +{ + let line_size = max_line_size(&tensor); + let client = tensor.client.clone(); + let num_elems = tensor.meta.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + if tensor.can_mut() && tensor.is_nonoverlapping() { + unary_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor), + linear_view(&tensor, line_size), + linear_view_alias(&tensor, line_size, 0), + args(&()), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + tensor + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + tensor.dtype, + ); + + unary_int::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + linear_view(&output, line_size), + args(&()), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + output + } + } +} + +pub(crate) mod unary_basic_int { + + use super::*; + + pub(crate) fn launch(tensor: CubeTensor, args: Args) -> CubeTensor + where + R: CubeRuntime, + for<'a> Args: FnOnce(&'a ()) -> BasicIntUnaryKind, + { + launch_unary_int::(tensor, |input| { + BasicIntUnaryOptionsLaunch::new(args(input)) + }) + } + + #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)] + pub enum BasicIntUnaryKind { + BitwiseNot, + Sign, + } + + #[derive(CubeLaunch, CubeType)] + struct BasicIntUnaryOptions { + #[cube(comptime)] + kind: BasicIntUnaryKind, + } + struct BasicIntUnary; + + #[cube] + impl IntUnaryOp for BasicIntUnary { + type Options = BasicIntUnaryOptions; + + fn execute(input: Line, options: &Self::Options) -> Line { + match comptime![options.kind] { + BasicIntUnaryKind::BitwiseNot => !input, + BasicIntUnaryKind::Sign => { + let zero = Line::new(I::new(0)); + let one = Line::new(I::new(1)); + let minus_one = Line::new(I::new(-1)); + + let is_positive = input.greater_than(zero); + let is_negative = input.less_than(zero); + let sign = select_many(is_negative, minus_one, zero); + + select_many(is_positive, one, sign) + } + } + } + } + + impl IntUnaryOpFamily for BasicIntUnary { + type Options = BasicIntUnaryOptions; + type Unary = Self; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_numeric.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_numeric.rs new file mode 100644 index 0000000..af58fa7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/unary_numeric.rs @@ -0,0 +1,90 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, linear_view_alias}, + ops::{max_line_size, numeric::empty_device_dtype}, + tensor::CubeTensor, +}; +use burn_backend::TensorMetadata; +use cubecl::{calculate_cube_count_elemwise, prelude::*, std::tensor::layout::linear::LinearView}; + +pub(crate) trait NumericUnaryOpFamily: 'static + Send + Sync { + type Options: LaunchArg; + type Unary: NumericUnaryOp; +} + +#[cube] +pub(crate) trait NumericUnaryOp: 'static + Send + Sync { + type Options: LaunchArg; + + fn execute(input: Line, options: &Self::Options) -> Line; +} + +#[cube(launch_unchecked, address_type = "dynamic")] +pub(crate) fn unary_numeric( + input: &LinearView>, + output: &mut LinearView, ReadWrite>, + options: &O::Options, + #[define(N)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + output[ABSOLUTE_POS] = O::Unary::::execute(input[ABSOLUTE_POS], options); +} + +pub(crate) fn launch_unary_numeric(tensor: CubeTensor, args: Args) -> CubeTensor +where + // Magic fix for lifetime, the closure is supposed to capture everything required to create the + // argument. + for<'a> Args: FnOnce(&'a ()) -> RuntimeArg<'a, O::Options, R>, + R: CubeRuntime, + O: NumericUnaryOpFamily, +{ + let line_size = max_line_size(&tensor); + let client = tensor.client.clone(); + let num_elems = tensor.meta.num_elements(); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&tensor.client, working_units); + let cube_count = calculate_cube_count_elemwise(&tensor.client, working_units, cube_dim); + + unsafe { + if tensor.can_mut() && tensor.is_nonoverlapping() { + unary_numeric::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor), + linear_view(&tensor, line_size), + linear_view_alias(&tensor, line_size, 0), + args(&()), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + tensor + } else { + let output = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + tensor.shape(), + tensor.dtype, + ); + + unary_numeric::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(tensor, output), + linear_view(&tensor, line_size), + linear_view(&output, line_size), + args(&()), + tensor.dtype.into(), + ) + .expect("Kernel to never fail"); + + output + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/utils.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/utils.rs new file mode 100644 index 0000000..fbe4c4e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/kernel/utils.rs @@ -0,0 +1,198 @@ +use burn_backend::Shape; +use cubecl::{ + ir::LineSize, + prelude::*, + std::{ + FastDivmod, FastDivmodArgs, FastDivmodInt, + tensor::layout::linear::{LinearLayoutArgs, LinearViewLaunch}, + }, +}; +use cubecl::{prelude::SequenceArg, std::tensor::layout::linear::LinearLayout}; + +use crate::{CubeRuntime, tensor::CubeTensor}; + +pub fn shape_divmod<'a, R: CubeRuntime>( + tensor: &CubeTensor, +) -> SequenceArg<'a, R, FastDivmod> { + let mut arg = SequenceArg::new(); + for dim in tensor.meta.shape().iter() { + arg.push(FastDivmodArgs::::new(&tensor.client, *dim)); + } + arg +} + +pub fn linear_layout<'a, R: CubeRuntime>( + tensor: &'a CubeTensor, + line_size: LineSize, +) -> LinearLayoutArgs<'a, R> { + LinearLayoutArgs::from_shape_strides( + &tensor.client, + tensor.meta.shape(), + tensor.meta.strides(), + line_size, + ) +} + +pub fn linear_layout_ref<'a, R: CubeRuntime>( + tensor: &'a CubeTensor, + reference: &'a CubeTensor, + line_size: LineSize, +) -> LinearLayoutArgs<'a, R> { + LinearLayoutArgs::from_shape_strides_with_reference( + &tensor.client, + tensor.meta.shape(), + reference.meta.shape(), + tensor.meta.strides(), + line_size, + ) +} + +pub fn linear_view<'a, R: CubeRuntime>( + tensor: &'a CubeTensor, + line_size: LineSize, +) -> LinearViewLaunch<'a, R> { + let len = tensor.meta.num_elements(); + let layout = linear_layout(tensor, line_size); + let buffer = unsafe { + ArrayArg::from_raw_parts_and_size(&tensor.handle, len, line_size, tensor.elem_size()) + }; + LinearViewLaunch::new::(buffer, layout) +} + +pub fn linear_view_ref<'a, R: CubeRuntime>( + tensor: &'a CubeTensor, + reference: &'a CubeTensor, + line_size: LineSize, +) -> LinearViewLaunch<'a, R> { + let len = tensor.meta.num_elements(); + let layout = linear_layout_ref(tensor, reference, line_size); + let buffer = unsafe { + ArrayArg::from_raw_parts_and_size(&tensor.handle, len, line_size, tensor.elem_size()) + }; + LinearViewLaunch::new::(buffer, layout) +} + +pub fn linear_view_alias<'a, R: CubeRuntime>( + tensor: &'a CubeTensor, + line_size: LineSize, + pos: usize, +) -> LinearViewLaunch<'a, R> { + let layout = linear_layout(tensor, line_size); + let buffer = ArrayArg::Alias { input_pos: pos }; + LinearViewLaunch::new::(buffer, layout) +} + +pub fn split_dim( + mut tensor: CubeTensor, + dim: usize, + shape: &[usize], +) -> CubeTensor { + let mut stride = tensor.meta.strides()[dim]; + tensor.meta.remove(dim); + + for size in shape.iter().rev() { + tensor.meta.insert(dim, *size, stride); + stride *= size; + } + + tensor +} + +pub fn broadcast_shape(tensors: &[&CubeTensor]) -> Shape { + let rank = tensors[0].meta.num_dims(); + debug_assert!( + tensors.iter().all(|it| it.meta.num_dims() == rank), + "Broadcast tensors must have the same rank" + ); + + let dims = (0..rank).map(|dim| { + let max = tensors.iter().map(|it| it.meta.shape()[dim]).max(); + let max = max.unwrap_or(1); + debug_assert!( + tensors + .iter() + .all(|it| it.meta.shape()[dim] == max || it.meta.shape()[dim] == 1), + "Broadcast dims must be size 1" + ); + max + }); + + Shape::from(dims) +} + +pub fn broadcast_strides<'a, R: CubeRuntime>( + reference: &CubeTensor, + tensor: &'a CubeTensor, +) -> SequenceArg<'a, R, usize> { + if reference.meta.shape() != tensor.meta.shape() { + tensor + .meta + .strides() + .iter() + .zip( + tensor + .meta + .shape() + .iter() + .zip(reference.meta.shape().iter()), + ) + .map(|(stride, (shape, ref_shape))| if *shape == *ref_shape { *stride } else { 0 }) + .map(ScalarArg::new) + .collect() + } else { + tensor + .meta + .strides() + .iter() + .copied() + .map(ScalarArg::new) + .collect() + } +} + +#[cube] +pub(crate) fn decompose_linear( + pos: I, + shape: &Sequence>, +) -> (I, Sequence) { + let rank = comptime![shape.len()]; + let mut offs = pos; + let mut out = Sequence::new(); + + #[unroll] + for i in 0..rank { + let dim = comptime![rank - i - 1]; + let (rem, offs_local) = shape.index(dim).div_mod(offs); + out.push(offs_local); + offs = rem; + } + + (offs, out.rev()) +} + +pub(crate) trait RequiredAddrType { + fn required_address_type(&self) -> AddressType; +} + +impl RequiredAddrType for CubeTensor { + fn required_address_type(&self) -> AddressType { + self.required_address_type() + } +} +impl RequiredAddrType for Option> { + fn required_address_type(&self) -> AddressType { + self.as_ref() + .map(|it| it.required_address_type()) + .unwrap_or_default() + } +} + +macro_rules! address_type { + ($($tensor: tt),*) => { + [$($crate::kernel::utils::RequiredAddrType::required_address_type(&$tensor)),*] + .into_iter() + .max() + .unwrap_or_default() + }; +} +pub(crate) use address_type; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/lib.rs new file mode 100644 index 0000000..108b02d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/lib.rs @@ -0,0 +1,50 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! Burn JIT Backend + +#[macro_use] +extern crate derive_new; +extern crate alloc; + +/// Utilities for implementing JIT kernels +pub mod ops; + +/// Kernel module +pub mod kernel; +/// Tensor module. +pub mod tensor; + +/// Elements for JIT backend +pub mod element; + +use cubecl::{CubeTask, Runtime}; +pub use element::{BoolElement, CubeElement, FloatElement, IntElement}; + +mod backend; + +pub use backend::*; + +// Re-export cubecl. +pub use cubecl; + +mod tune_key; +pub use tune_key::CubeAutotuneKey; + +#[cfg(any(feature = "fusion", test))] +/// Module for interacting with fusion +pub mod fusion; + +#[cfg(feature = "template")] +/// Module for compiling custom non-jit kernels +pub mod template; + +/// Just-in-Time runtime extending the [cube runtime](Runtime). +pub trait CubeRuntime: Runtime { + /// The device that should also implement [burn_backend::backend::DeviceOps]. + type CubeDevice: burn_backend::DeviceOps; + /// The cube server with the [CubeAutotuneKey]. + type CubeServer: cubecl::server::ComputeServer>>; +} + +pub use cubecl::CubeTuneId; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/activation.rs new file mode 100644 index 0000000..1c8d17e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/activation.rs @@ -0,0 +1,11 @@ +use crate::{CubeBackend, CubeRuntime, FloatElement, IntElement, element::BoolElement}; +use burn_backend::ops::ActivationOps; + +impl ActivationOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/base.rs new file mode 100644 index 0000000..c99fb2d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/base.rs @@ -0,0 +1,432 @@ +use crate::{CubeRuntime, kernel, ops::numeric::empty_device_dtype, tensor::CubeTensor}; +use burn_backend::{ + DType, ExecutionError, QTensorPrimitive, Shape, TensorData, + quantization::{QuantLevel, QuantStore, params_shape}, +}; +use burn_backend::{TensorMetadata, ops::unfold::calculate_unfold_shape}; +use burn_std::{ + Metadata, strides, + tensor::{ReshapeAction, contiguous_strides, reshape_action}, +}; +use cubecl::{ir::LineSize, server::CopyDescriptor}; +use cubecl::{quant::scheme::BlockSize, tensor_line_size_parallel}; + +pub(crate) fn from_data(data: TensorData, device: &R::Device) -> CubeTensor { + let client = R::client(device); + let alloc = client.create_tensor(data.bytes, &data.shape, data.dtype.size()); + let shape: Shape = (&data.shape).into(); + CubeTensor::new( + client, + alloc.handle, + Metadata::new(shape, alloc.strides), + device.clone(), + data.dtype, + ) +} + +pub(crate) async fn into_data( + tensor: CubeTensor, +) -> Result { + let tensor = kernel::into_contiguous_aligned(tensor); + + let elem_size = tensor.elem_size(); + let shape = tensor.meta.shape(); + let strides = tensor.meta.strides(); + let binding = CopyDescriptor::new(tensor.handle.binding(), shape, strides, elem_size); + let bytes = tensor + .client + .read_one_tensor_async(binding) + .await + .map_err(|err| ExecutionError::WithContext { + reason: format!("{err}"), + })?; + + Ok(TensorData::from_bytes( + bytes, + tensor.meta.shape, + tensor.dtype, + )) +} + +/// Read data from a `CubeTensor` synchronously +#[allow(unused, reason = "useful for debugging kernels")] +pub fn into_data_sync(tensor: CubeTensor) -> TensorData { + burn_std::future::block_on(into_data(tensor)).unwrap() +} + +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(tensor, device)) +)] +pub(crate) fn to_device( + tensor: CubeTensor, + device: &R::Device, +) -> CubeTensor { + if &tensor.device == device { + return tensor; + } + + let tensor = kernel::into_contiguous_aligned(tensor); + let client = R::client(device); + tensor.to_client(client, device.clone()) +} + +pub(crate) fn empty( + shape: Shape, + device: &R::Device, + dtype: DType, +) -> CubeTensor { + let client = R::client(device); + let alloc = client.empty_tensor(&shape, dtype.size()); + + CubeTensor::new( + client, + alloc.handle, + Metadata::new(shape, alloc.strides), + device.clone(), + dtype, + ) +} + +pub(crate) fn swap_dims( + mut tensor: CubeTensor, + dim1: usize, + dim2: usize, +) -> CubeTensor { + tensor.meta.swap(dim1, dim2); + + if let DType::QFloat(scheme) = tensor.dtype + && let QuantLevel::Block(block_size) = scheme.level + { + let rank = tensor.rank(); + let qparams = tensor.qparams.as_mut().unwrap(); + let mut block_size = block_size.to_dim_vec(rank); + block_size.swap(dim1, dim2); + + // Truncate unit dims from the start + let block_size = BlockSize::new_trim(block_size); + if block_size.len() > BlockSize::MAX_DIMS { + panic!("Swapped block size would exceed max dims"); + } + + qparams.scales.metadata.swap(dim1, dim2); + + tensor.dtype = DType::QFloat(scheme.with_level(QuantLevel::Block(block_size))) + } + + if let DType::QFloat(scheme) = &mut tensor.dtype + && let QuantStore::PackedU32(packed_dim) | QuantStore::PackedNative(packed_dim) = + &mut scheme.store + { + let rank = tensor.meta.num_dims(); + + if *packed_dim == rank - dim1 - 1 { + *packed_dim = rank - dim2 - 1; + } else if *packed_dim == rank - dim2 - 1 { + *packed_dim = rank - dim1 - 1; + } + } + + tensor +} + +/// Permute a tensor's dimensions +pub fn permute(mut tensor: CubeTensor, axes: &[usize]) -> CubeTensor { + tensor.meta.permute(axes).unwrap(); + + if let DType::QFloat(scheme) = tensor.dtype + && let QuantLevel::Block(block_size) = scheme.level + { + let rank = tensor.rank(); + let qparams = tensor.qparams.as_mut().unwrap(); + + let mut block_size = block_size.to_dim_vec(rank); + block_size = axes.iter().map(|i| block_size[*i]).collect(); + + // Truncate unit dims from the start + let block_size = block_size + .into_iter() + .skip_while(|it| *it == 1) + .collect::>(); + if block_size.len() > BlockSize::MAX_DIMS { + panic!("Swapped block size would exceed max dims"); + } + + qparams.scales.metadata.permute(axes).unwrap(); + + tensor.dtype = DType::QFloat(scheme.with_level(QuantLevel::block(&block_size))) + } + + if let DType::QFloat(scheme) = &mut tensor.dtype + && let QuantStore::PackedU32(packed_dim) = &mut scheme.store + { + let rank = tensor.meta.num_dims(); + let new_pos = axes + .iter() + .position(|axis| *axis == rank - *packed_dim - 1) + .unwrap_or(0); + *packed_dim = rank - new_pos - 1; + } + + tensor +} + +/// Permute a tensor's dimensions from NCHW to NHWC, or the N-dimensional equivalent +pub fn permute_nchw_to_nhwc(tensor: CubeTensor) -> CubeTensor { + let rank = tensor.meta.num_dims(); + let c_dim = 1; + + let mut dims = vec![0]; + dims.extend(2..rank); + dims.push(c_dim); + + permute(tensor, &dims) +} + +/// Permute a shape's dimensions from NCHW to NHWC, or the N-dimensional equivalent +pub fn permute_nchw_to_nhwc_shape(shape: Shape) -> Shape { + let rank = shape.num_dims(); + let c_dim = 1; + + let mut dims = vec![0]; + dims.extend(2..rank); + dims.push(c_dim); + + shape.permuted(&dims).expect("Shape permute should succeed") +} + +/// Permute a tensor's dimensions from NHWC to NCHW, or the N-dimensional equivalent +pub fn permute_nhwc_to_nchw(tensor: CubeTensor) -> CubeTensor { + let rank = tensor.meta.num_dims(); + let c_dim = rank - 1; + + let mut dims = vec![0]; + dims.push(c_dim); + dims.extend(1..c_dim); + + permute(tensor, &dims) +} + +/// Permute a shape's dimensions from NHWC to NCHW, or the N-dimensional equivalent +pub fn permute_nhwc_to_nchw_shape(shape: Shape) -> Shape { + let rank = shape.num_dims(); + let c_dim = rank - 1; + + let mut dims = vec![0]; + dims.push(c_dim); + dims.extend(1..c_dim); + + shape.permuted(&dims).expect("Shape permute should succeed") +} + +pub(crate) fn expand(tensor: CubeTensor, target_shape: Shape) -> CubeTensor { + let ndims_in = tensor.meta.shape().num_dims(); + let ndims_out = target_shape.num_dims(); + + // Initialize new strides with zeros + let mut new_strides = strides![0usize; ndims_out]; + + // Calculate the difference in dimensions + let dim_diff = ndims_out.saturating_sub(ndims_in); + + // Compare dimensions from the end, setting strides for matching dimensions or broadcasted ones + let mut tensor_dim_iter = tensor.meta.shape().iter().rev(); + for i in (0..ndims_out).rev() { + if i >= dim_diff { + if let Some(&tensor_dim) = tensor_dim_iter.next() { + if tensor_dim == target_shape[i] || tensor_dim == 1 { + // Copy stride for non-broadcast dimensions or set to 0 for broadcast ones + new_strides[i] = if tensor_dim == target_shape[i] { + tensor.meta.strides()[i - dim_diff] + } else { + 0 + }; + } else { + // Error handling: Dimension mismatch for broadcasting + panic!( + "Dimension mismatch: cannot broadcast dimension {tensor_dim} of tensor to target shape" + ); + } + } else { + // If the input tensor has fewer dimensions, treat missing dimensions as 1 + // and set stride to 0 (broadcasting) + new_strides[i] = 0; + } + } else { + // For extra dimensions in the target shape, set stride to 0 (broadcasting) + new_strides[i] = 0; + } + } + + // Extra check to ensure block scales must be properly handled once they're added + if tensor.qparams.is_some() { + match tensor.scheme().level { + QuantLevel::Tensor => {} + QuantLevel::Block(_) => todo!(), + } + } + + CubeTensor { + client: tensor.client, + device: tensor.device, + meta: Box::new(Metadata::new(target_shape, new_strides)), + handle: tensor.handle, + dtype: tensor.dtype, + qparams: tensor.qparams, + } +} + +/// Reshape a jit tensor to a new shape +pub fn reshape(mut tensor: CubeTensor, shape: Shape) -> CubeTensor { + let analysis = reshape_action(tensor.meta.shape(), tensor.meta.strides(), &shape); + + match analysis { + ReshapeAction::UpdateStrides { strides } => { + *tensor.meta = Metadata::new(shape, strides); + return tensor; + } + ReshapeAction::NoChange => return tensor, + ReshapeAction::Recompute => (), + } + + let out = empty_device_dtype( + tensor.client.clone(), + tensor.device.clone(), + shape, + tensor.dtype, + ); + + cubecl::std::tensor::copy_into( + &tensor.client, + &tensor.as_handle_ref(), + &out.as_handle_ref(), + tensor.dtype.into(), + ) + .expect("Kernel should not fail"); + + out +} + +/// Reshape a jit tensor to a new shape +pub fn q_reshape(mut tensor: CubeTensor, shape: Shape) -> CubeTensor { + let scheme = *tensor.scheme(); + + let shape_values = { + let rank = shape.num_dims(); + let mut shape = shape.clone(); + shape[rank - 1] = shape[rank - 1].div_ceil(scheme.num_quants()); + shape + }; + let shape_scales = params_shape(&shape, scheme.level); + let (values, scales) = tensor.quantized_handles().unwrap(); + + let analysis_values = reshape_action(values.meta.shape(), values.meta.strides(), &shape_values); + let analysis_scales = reshape_action(scales.meta.shape(), scales.meta.strides(), &shape_scales); + + match (analysis_values, analysis_scales) { + ( + ReshapeAction::UpdateStrides { strides }, + ReshapeAction::UpdateStrides { + strides: scales_strides, + }, + ) => { + let qparams = tensor.qparams.as_mut().unwrap(); + + *tensor.meta = Metadata::new(shape, strides); + qparams.scales.metadata = Metadata::new(shape_scales, scales_strides); + } + (ReshapeAction::UpdateStrides { strides }, ReshapeAction::NoChange) => { + *tensor.meta = Metadata::new(shape, strides); + } + ( + ReshapeAction::NoChange, + ReshapeAction::UpdateStrides { + strides: scales_strides, + }, + ) => { + let qparams = tensor.qparams.as_mut().unwrap(); + + qparams.scales.metadata = Metadata::new(shape_scales, scales_strides); + } + (ReshapeAction::NoChange, ReshapeAction::NoChange) => {} + _ => { + tensor = kernel::into_contiguous(tensor); + *tensor.meta = Metadata::new(shape, contiguous_strides(&shape_values)); + + let qparams = tensor.qparams.as_mut().unwrap(); + + let strides = contiguous_strides(&shape_scales); + qparams.scales.metadata = Metadata::new(shape_scales, strides); + } + } + + tensor +} + +pub(crate) fn max_line_size(tensor: &CubeTensor) -> LineSize { + tensor_line_size_parallel( + tensor.client.io_optimized_line_sizes(tensor.dtype.size()), + tensor.meta.shape(), + tensor.meta.strides(), + tensor.meta.num_dims() - 1, + ) +} + +pub(crate) fn max_line_size_many( + tensors: &[&CubeTensor], + axis: usize, +) -> LineSize { + let vec = tensors + .iter() + .map(|tensor| { + tensor_line_size_parallel( + tensor.client.io_optimized_line_sizes(tensor.dtype.size()), + tensor.meta.shape(), + tensor.meta.strides(), + axis, + ) + }) + .min(); + + vec.unwrap_or(0) +} + +/// Unfold windows along a dimension. +/// +/// Returns a view of the tensor with all complete windows of size `size` in dimension `dim`; +/// where windows are advanced by `step` at each index. +/// +/// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. +/// +/// The new view will have the unfolded dimension replaced by two dimensions; +/// one in the position of the original dimension, with size equal to the number of windows, +/// and one appended to the right-most position, with size equal to `size`. +/// +/// # Arguments +/// +/// * `tensor` - The input tensor to unfold; of shape ``[pre=..., dim shape, post=...]`` +/// * `dim` - the dimension to unfold. +/// * `size` - the size of each unfolded window. +/// * `step` - the step between each window. +/// +/// # Returns +/// +/// A tensor view with the shape ``[pre=..., windows, post=..., size]``. +pub fn unfold( + tensor: CubeTensor, + dim: usize, + size: usize, + step: usize, +) -> CubeTensor { + let shape = calculate_unfold_shape(tensor.shape(), dim, size, step); + + let d_stride = tensor.meta.strides()[dim]; + let mut strides = tensor.meta.strides.clone(); + strides[dim] = step * d_stride; + strides.push(d_stride); + + CubeTensor { + meta: Box::new(Metadata::new(shape, strides)), + ..tensor + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/bool_tensor.rs new file mode 100644 index 0000000..6edcf9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/bool_tensor.rs @@ -0,0 +1,200 @@ +use crate::{ + CubeBackend, CubeRuntime, FloatElement, IntElement, + element::BoolElement, + kernel::{self, AndOp, OrOp}, +}; +use burn_backend::{ + ExecutionError, Slice, + ops::BoolTensorOps, + tensor::{BoolTensor, Device, FloatTensor, IntTensor}, +}; +use burn_backend::{Scalar, Shape, TensorData}; +use cubecl::prelude::InputScalar; +use std::ops::Range; + +use super::{expand, numeric, permute, unfold}; + +impl BoolTensorOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + fn bool_empty(shape: Shape, device: &Device) -> BoolTensor { + super::empty(shape, device, BT::dtype()) + } + + fn bool_zeros(shape: Shape, device: &Device) -> BoolTensor { + numeric::zeros(device.clone(), shape, BT::dtype()) + } + + fn bool_ones(shape: Shape, device: &Device) -> BoolTensor { + numeric::ones(device.clone(), shape, BT::dtype()) + } + + async fn bool_into_data(tensor: BoolTensor) -> Result { + super::into_data(tensor).await + } + + fn bool_from_data(data: TensorData, device: &Device) -> BoolTensor { + if data.dtype != BT::dtype() { + unimplemented!("Unsupported dtype for `bool_from_data`") + } + super::from_data(data, device) + } + + fn bool_into_int(tensor: BoolTensor) -> IntTensor { + kernel::bool_cast::(tensor) + } + + fn bool_device(tensor: &BoolTensor) -> Device { + tensor.device.clone() + } + + fn bool_to_device(tensor: BoolTensor, device: &Device) -> BoolTensor { + super::to_device(tensor, device) + } + + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor { + super::reshape(tensor, shape) + } + + fn bool_slice(tensor: BoolTensor, slices: &[Slice]) -> BoolTensor { + // Check if all steps are 1 + let all_steps_one = slices.iter().all(|info| info.step == 1); + + if all_steps_one { + // Use optimized slice for step=1 + let simple_ranges: Vec> = slices + .iter() + .enumerate() + .map(|(i, slice)| slice.to_range(tensor.meta.shape()[i])) + .collect(); + + kernel::slice(tensor, &simple_ranges) + } else { + // Use slice with steps kernel + kernel::slice_with_steps(tensor, slices) + } + } + + fn bool_slice_assign( + tensor: BoolTensor, + ranges: &[Slice], + value: BoolTensor, + ) -> BoolTensor { + kernel::slice_assign(tensor, ranges, value) + } + + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + kernel::equal(lhs, rhs, BT::dtype()) + } + + fn bool_not(tensor: BoolTensor) -> BoolTensor { + kernel::equal_elem( + tensor, + InputScalar::new(BT::false_val(), BT::dtype()), + BT::dtype(), + ) + } + + fn bool_and(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + kernel::launch_binop::(lhs, rhs) + } + + fn bool_or(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + kernel::launch_binop::(lhs, rhs) + } + + fn bool_into_float(tensor: BoolTensor) -> FloatTensor { + kernel::bool_cast::(tensor) + } + + fn bool_swap_dims(mut tensor: BoolTensor, dim1: usize, dim2: usize) -> BoolTensor { + tensor.meta.swap(dim1, dim2); + + tensor + } + + fn bool_repeat_dim(tensor: BoolTensor, dim: usize, times: usize) -> BoolTensor { + kernel::repeat_dim(tensor, dim, times) + } + + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + permute(tensor, axes) + } + + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor { + expand(tensor, shape) + } + + fn bool_select( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + ) -> BoolTensor { + kernel::select(tensor, dim, indices) + } + + fn bool_select_or( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + kernel::select_assign(tensor, dim, indices, value, true) + } + + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + kernel::flip(tensor, axes, BT::dtype()) + } + + fn bool_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + unfold(tensor, dim, size, step) + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + kernel::mask_where_auto(tensor, mask, value, BT::dtype()) + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + let dtype = tensor.dtype; + kernel::mask_fill_auto(tensor, mask, InputScalar::new(value, dtype), dtype) + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + kernel::gather(dim, tensor, indices) + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + kernel::scatter(dim, tensor, indices, value, true) + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::equal_elem(lhs, InputScalar::new(rhs, dtype), dtype) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/int_tensor.rs new file mode 100644 index 0000000..531182e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/int_tensor.rs @@ -0,0 +1,546 @@ +use self::unary_basic_int::BasicIntUnaryKind; + +use super::{expand, numeric, permute, unfold}; +use crate::kernel::{ + BitwiseShlOp, BitwiseShrOp, NumericUnaryOp, NumericUnaryOpFamily, launch_binop_int, + launch_scalar_binop_int, launch_unary_numeric, reduce, unary_basic_int, +}; +use crate::{ + CubeBackend, CubeRuntime, FloatElement, IntElement, + kernel::{ + self, + matmul::{MatmulStrategy, matmul}, + }, +}; +use crate::{ + element::BoolElement, + kernel::prng::{random_bernoulli, random_normal, random_uniform}, +}; +use burn_backend::tensor::{BoolTensor, Device, FloatTensor, IntElem, IntTensor}; +use burn_backend::{DType, IntDType, Slice, ops::IntTensorOps}; +use burn_backend::{Distribution, ElementConversion, Shape, TensorData}; +use burn_backend::{ExecutionError, Scalar}; +use cubecl::frontend::Numeric; +use cubecl::prelude::*; +use cubek::reduce::components::instructions::ReduceOperationConfig; +use std::ops::Range; + +impl IntTensorOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + fn int_empty(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + let dtype = dtype.into(); + super::empty(shape, device, dtype) + } + + async fn int_into_data(tensor: IntTensor) -> Result { + super::into_data(tensor).await + } + + fn int_from_data(data: TensorData, device: &Device) -> IntTensor { + match data.dtype { + DType::I64 + | DType::I32 + | DType::I16 + | DType::I8 + | DType::U64 + | DType::U32 + | DType::U16 + | DType::U8 => super::from_data(data, device), + _ => unimplemented!("Unsupported dtype for `int_from_data`"), + } + } + + fn int_device(tensor: &IntTensor) -> Device { + tensor.device.clone() + } + + fn int_to_device(tensor: IntTensor, device: &Device) -> IntTensor { + super::to_device(tensor, device) + } + + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor { + super::reshape(tensor, shape) + } + + fn int_slice(tensor: IntTensor, slices: &[Slice]) -> IntTensor { + // Check if all steps are 1 + let all_steps_one = slices.iter().all(|info| info.step == 1); + + if all_steps_one { + // Use optimized slice for step=1 + let simple_ranges: Vec> = slices + .iter() + .enumerate() + .map(|(i, slice)| slice.to_range(tensor.meta.shape()[i])) + .collect(); + + kernel::slice(tensor, &simple_ranges) + } else { + // Use slice with steps kernel + kernel::slice_with_steps(tensor, slices) + } + } + + fn int_slice_assign( + tensor: IntTensor, + ranges: &[Slice], + value: IntTensor, + ) -> IntTensor { + kernel::slice_assign(tensor, ranges, value) + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let dtype = lhs.dtype; + matmul(lhs, rhs, None, MatmulStrategy::default(), dtype).unwrap() + } + + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + value: IntTensor, + ) -> IntTensor { + kernel::mask_where_auto(tensor, mask, value, BT::dtype()) + } + + fn int_mask_fill( + tensor: IntTensor, + mask: BoolTensor, + value: Scalar, + ) -> IntTensor { + let dtype = tensor.dtype; + kernel::mask_fill_auto(tensor, mask, InputScalar::new(value, dtype), BT::dtype()) + } + + fn int_gather( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + ) -> IntTensor { + kernel::gather(dim, tensor, indices) + } + + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + kernel::scatter(dim, tensor, indices, value, false) + } + + fn int_select( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + ) -> IntTensor { + kernel::select(tensor, dim, indices) + } + + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + kernel::select_assign(tensor, dim, indices, value, false) + } + + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + kernel::equal(lhs, rhs, BT::dtype()) + } + + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::equal_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + kernel::greater(lhs, rhs, BT::dtype()) + } + + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::greater_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + kernel::greater_equal(lhs, rhs, BT::dtype()) + } + + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::greater_equal_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + kernel::lower(lhs, rhs, BT::dtype()) + } + + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::lower_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + kernel::lower_equal(lhs, rhs, BT::dtype()) + } + + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::lower_equal_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::add(lhs, rhs) + } + + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::add_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::sub(lhs, rhs) + } + + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::sub_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::mul(lhs, rhs) + } + + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::mul_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::div(lhs, rhs) + } + + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::div_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::remainder(lhs, rhs) + } + + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::remainder_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn int_zeros(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + let dtype = dtype.into(); + numeric::zeros(device.clone(), shape, dtype) + } + + fn int_ones(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + let dtype = dtype.into(); + numeric::ones(device.clone(), shape, dtype) + } + + fn int_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: IntDType, + ) -> IntTensor { + let dtype: DType = dtype.into(); + let client = R::client(device); + numeric::full_device_dtype( + client, + shape, + device.clone(), + InputScalar::new(fill_value, dtype), + dtype, + ) + } + + fn int_sum(tensor: IntTensor) -> IntTensor { + reduce::sum_fallback(tensor, Default::default()).unwrap() + } + + fn int_sum_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Sum, + ) + .unwrap() + } + + fn int_prod(tensor: IntTensor) -> IntTensor { + reduce::reduce( + tensor, + None, + Default::default(), + ReduceOperationConfig::Prod, + ) + .unwrap() + } + + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Prod, + ) + .unwrap() + } + + fn int_max(tensor: IntTensor) -> IntTensor { + reduce::reduce(tensor, None, Default::default(), ReduceOperationConfig::Max).unwrap() + } + + fn int_max_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Max, + ) + .unwrap() + } + + fn int_max_abs(tensor: IntTensor) -> IntTensor { + reduce::reduce( + tensor, + None, + Default::default(), + ReduceOperationConfig::MaxAbs, + ) + .unwrap() + } + + fn int_max_abs_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::MaxAbs, + ) + .unwrap() + } + + fn int_min(tensor: IntTensor) -> IntTensor { + reduce::reduce(tensor, None, Default::default(), ReduceOperationConfig::Min).unwrap() + } + + fn int_min_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Min, + ) + .unwrap() + } + + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Mean, + ) + .unwrap() + } + + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor { + numeric::cumsum(tensor, dim) + } + + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor { + numeric::cumprod(tensor, dim) + } + + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor { + numeric::cummin(tensor, dim) + } + + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor { + numeric::cummax(tensor, dim) + } + + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor { + let dtype = tensor.dtype; + reduce::reduce_dim( + tensor, + Some(dtype), + dim, + Default::default(), + ReduceOperationConfig::ArgMax, + ) + .unwrap() + } + + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor { + let dtype = tensor.dtype; + reduce::reduce_dim( + tensor, + Some(dtype), + dim, + Default::default(), + ReduceOperationConfig::ArgMin, + ) + .unwrap() + } + + fn int_clamp(tensor: IntTensor, min: Scalar, max: Scalar) -> IntTensor { + let dtype = tensor.dtype; + kernel::clamp( + tensor, + InputScalar::new(min, dtype), + InputScalar::new(max, dtype), + ) + } + + fn int_abs(tensor: IntTensor) -> IntTensor { + struct Abs; + + #[cube] + impl NumericUnaryOp for Abs { + type Options = (); + + fn execute(input: Line, _options: &Self::Options) -> Line { + Line::abs(input) + } + } + + impl NumericUnaryOpFamily for Abs { + type Options = (); + type Unary = Self; + } + + launch_unary_numeric::(tensor, |_| ()) + } + + fn int_sign(tensor: IntTensor) -> IntTensor { + unary_basic_int::launch::(tensor, |_| BasicIntUnaryKind::Sign) + } + + fn int_into_float(tensor: IntTensor) -> FloatTensor { + kernel::cast(tensor, F::dtype()) + } + + fn int_swap_dims(mut tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor { + tensor.meta.swap(dim1, dim2); + + tensor + } + + fn int_repeat_dim(tensor: IntTensor, dim: usize, times: usize) -> IntTensor { + kernel::repeat_dim(tensor, dim, times) + } + + fn int_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> IntTensor { + let dtype = IntElem::::dtype(); + match distribution { + Distribution::Default => random_uniform(shape, device, 0., 255., dtype), + Distribution::Uniform(low, high) => { + random_uniform(shape, device, low.elem(), high.elem(), dtype) + } + Distribution::Bernoulli(prob) => random_bernoulli(shape, device, prob as f32, dtype), + Distribution::Normal(mean, std) => { + random_normal(shape, device, mean.elem(), std.elem(), dtype) + } + } + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + permute(tensor, axes) + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + expand(tensor, shape) + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + kernel::flip(tensor, axes, BT::dtype()) + } + + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::bitwise_and(lhs, rhs) + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::bitwise_and_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::bitwise_or(lhs, rhs) + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::bitwise_or_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + numeric::bitwise_xor(lhs, rhs) + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + numeric::bitwise_xor_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + unary_basic_int::launch::(tensor, |_| BasicIntUnaryKind::BitwiseNot) + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + launch_binop_int::(lhs, rhs) + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + launch_scalar_binop_int::(lhs, InputScalar::new(rhs, dtype)) + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + launch_binop_int::(lhs, rhs) + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let dtype = lhs.dtype; + launch_scalar_binop_int::(lhs, InputScalar::new(rhs, dtype)) + } + + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor { + kernel::cast(tensor, dtype.into()) + } + + fn int_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + unfold(tensor, dim, size, step) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/mod.rs new file mode 100644 index 0000000..6d55e80 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/mod.rs @@ -0,0 +1,14 @@ +mod activation; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; + +pub(crate) mod base; +pub use base::*; +pub use qtensor::*; + +/// Numeric utility functions for jit backends +pub mod numeric; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/module.rs new file mode 100644 index 0000000..279df90 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/module.rs @@ -0,0 +1,344 @@ +use crate::{ + CubeBackend, CubeRuntime, FloatElement, IntElement, + element::BoolElement, + kernel::{self, conv::ConvTranspose2dStrategy}, +}; +use burn_backend::tensor::{BoolTensor, FloatTensor, IntTensor}; +use burn_backend::{ + TensorMetadata, + ops::{ + AttentionModuleOptions, ConvOptions, ConvTransposeOptions, DeformConv2dBackward, + DeformConvOptions, InterpolateOptions, MaxPool2dBackward, MaxPool2dWithIndices, ModuleOps, + }, +}; + +impl ModuleOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + fn conv1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<1>, + ) -> FloatTensor { + kernel::conv::conv_forward::(x, weight, bias, options, Default::default()).unwrap() + } + + fn conv1d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, + ) -> FloatTensor { + kernel::conv::conv_data_backward( + output_grad, + weight, + x.shape(), + options, + Default::default(), + ) + .unwrap() + } + + fn conv1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, + ) -> FloatTensor { + kernel::conv::conv_weight_backward::( + x, + output_grad, + weight.shape(), + options, + Default::default(), + ) + .unwrap() + } + + fn conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<2>, + ) -> FloatTensor { + kernel::conv::conv_forward::(x, weight, bias, options, Default::default()).unwrap() + } + + fn conv2d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, + ) -> FloatTensor { + kernel::conv::conv_data_backward( + output_grad, + weight, + x.shape(), + options, + Default::default(), + ) + .unwrap() + } + + fn conv2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, + ) -> FloatTensor { + kernel::conv::conv_weight_backward::( + x, + output_grad, + weight.shape(), + options, + Default::default(), + ) + .unwrap() + } + + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> FloatTensor { + kernel::conv::deform_conv2d(x, offset, weight, mask, bias, options).unwrap() + } + + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + let (x, o, w, m, b) = kernel::conv::deform_conv2d_backward( + x, + offset, + weight, + mask, + bias, + output_grad, + options, + ) + .unwrap(); + DeformConv2dBackward::new(x, o, w, m, b) + } + + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> FloatTensor { + kernel::conv::conv_forward::(x, weight, bias, options, Default::default()).unwrap() + } + + fn conv3d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, + ) -> FloatTensor { + kernel::conv::conv_data_backward( + output_grad, + weight, + x.shape(), + options, + Default::default(), + ) + .unwrap() + } + + fn conv3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, + ) -> FloatTensor { + kernel::conv::conv_weight_backward::( + x, + output_grad, + weight.shape(), + options, + Default::default(), + ) + .unwrap() + } + + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + kernel::conv::conv_transpose2d(x, weight, bias, options, ConvTranspose2dStrategy::default()) + .unwrap() + } + + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + kernel::conv::conv_transpose3d(x, weight, bias, options).expect("Kernel to never fail") + } + + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + kernel::pool::avg_pool2d( + x, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ) + } + + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + kernel::pool::avg_pool2d_backward( + x, + grad, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ) + } + + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor { + kernel::pool::max_pool2d(x, kernel_size, stride, padding, dilation, ceil_mode) + } + + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices { + let (output, indices) = kernel::pool::max_pool2d_with_indices( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + I::dtype(), + ); + + MaxPool2dWithIndices::new(output, indices) + } + + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool2dBackward { + MaxPool2dBackward::new(kernel::pool::max_pool2d_with_indices_backward( + x, + output_grad, + indices, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + )) + } + + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor { + kernel::pool::adaptive_avg_pool2d(x, output_size) + } + + fn adaptive_avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + kernel::pool::adaptive_avg_pool2d_backward(x, grad) + } + + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + kernel::interpolate::interpolate(x, output_size, options) + } + + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + kernel::interpolate::interpolate_backward(x, grad, output_size, options) + } + + fn attention( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, + ) -> FloatTensor { + // Fall back to naive attention for features the flash kernel doesn't support. + if attn_bias.is_some() || options.softcap.is_some() || options.scale.is_some() { + return burn_backend::ops::attention::attention_fallback::( + query, key, value, mask, attn_bias, options, + ); + } + + kernel::attention::attention( + query, + key, + value, + mask, + attn_bias, + options, + &Default::default(), + None, + ) + .expect("Kernel to never fail") + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/numeric.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/numeric.rs new file mode 100644 index 0000000..b10a32d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/numeric.rs @@ -0,0 +1,442 @@ +use crate::{ + CubeRuntime, + kernel::utils::{address_type, linear_view, shape_divmod}, +}; +use crate::{element::CubeElement, tensor::CubeTensor}; +use crate::{ + kernel::{ + AddOp, BitwiseAndOp, BitwiseOrOp, BitwiseXorOp, DivOp, MulOp, PowOp, RemainderOp, SubOp, + launch_binop, launch_binop_int, launch_scalar_binop, launch_scalar_binop_int, + }, + ops::max_line_size, +}; +use burn_backend::{DType, Shape, TensorMetadata}; +use burn_std::Metadata; +use cubecl::{calculate_cube_count_elemwise, prelude::*}; +use cubecl::{client::ComputeClient, server::Allocation}; +use cubecl::{ + server::AllocationDescriptor, + std::{FastDivmod, tensor::layout::linear::LinearView}, +}; + +/// Creates a tensor filled with `value` +pub fn full( + shape: Shape, + device: &R::Device, + value: E, +) -> CubeTensor { + let client = R::client(device); + + full_client::(client, shape, device.clone(), value) +} + +/// Creates a tensor filled with `value` +pub fn full_client( + client: ComputeClient, + shape: Shape, + device: R::Device, + value: E, +) -> CubeTensor { + let dtype = E::dtype(); + full_device_dtype(client, shape, device, InputScalar::new(value, dtype), dtype) +} + +/// Creates a tensor filled with `value` +pub fn full_device_dtype( + client: ComputeClient, + shape: Shape, + device: R::Device, + value: InputScalar, + dtype: DType, +) -> CubeTensor { + let empty = empty_device_dtype(client, device, shape, dtype); + + #[cube(launch_unchecked, address_type = "dynamic")] + pub fn full_kernel( + tensor: &mut LinearView, + value: InputScalar, + #[define(C)] _dtype: StorageType, + ) { + if !tensor.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + tensor[ABSOLUTE_POS] = value.get::(); + } + + let num_elems = empty.meta.num_elements(); + let line_size = max_line_size(&empty); + + let working_units = num_elems / line_size as usize; + let cube_dim = CubeDim::new(&empty.client, working_units); + let cube_count = calculate_cube_count_elemwise(&empty.client, working_units, cube_dim); + + unsafe { + full_kernel::launch_unchecked( + &empty.client, + cube_count, + cube_dim, + address_type!(empty), + linear_view(&empty, line_size), + value, + empty.dtype.into(), + ) + .expect("Kernel to never fail"); + } + + empty +} + +/// Creates a tensor filled with zeros +pub fn zeros(device: R::Device, shape: Shape, dtype: DType) -> CubeTensor { + let client = R::client(&device); + full_device_dtype(client, shape, device, InputScalar::new(0u32, dtype), dtype) +} + +/// Creates a tensor filled with ones +pub fn ones(device: R::Device, shape: Shape, dtype: DType) -> CubeTensor { + let client = R::client(&device); + full_device_dtype(client, shape, device, InputScalar::new(1u32, dtype), dtype) +} + +/// Creates a tensor filled with zeros +pub fn zeros_client( + client: ComputeClient, + device: R::Device, + shape: Shape, + dtype: DType, +) -> CubeTensor { + full_device_dtype(client, shape, device, InputScalar::new(0u32, dtype), dtype) +} + +/// Creates a tensor filled with ones +pub fn ones_client( + client: ComputeClient, + device: R::Device, + shape: Shape, + dtype: DType, +) -> CubeTensor { + full_device_dtype(client, shape, device, InputScalar::new(1u32, dtype), dtype) +} + +/// Create a tensor with uninitialized memory +pub fn empty_device( + client: ComputeClient, + device: R::Device, + shape: Shape, +) -> CubeTensor { + let Allocation { handle, strides } = client.empty_tensor(&shape, size_of::()); + + CubeTensor::new( + client, + handle, + Metadata::new(shape, strides), + device, + E::dtype(), + ) +} + +/// Create a tensor with uninitialized memory +pub fn empty_device_dtype( + client: ComputeClient, + device: R::Device, + shape: Shape, + dtype: DType, +) -> CubeTensor { + let Allocation { handle, strides } = client.empty_tensor(&shape, dtype.size()); + + CubeTensor::new(client, handle, Metadata::new(shape, strides), device, dtype) +} + +/// Create a contiguous tensor with uninitialized memory +pub fn empty_device_contiguous_dtype( + client: ComputeClient, + device: R::Device, + shape: Shape, + dtype: DType, +) -> CubeTensor { + let descriptor = AllocationDescriptor::contiguous(&shape, dtype.size()); + let Allocation { handle, strides } = client.empty_tensors(vec![descriptor]).remove(0); + + CubeTensor::new(client, handle, Metadata::new(shape, strides), device, dtype) +} + +/// Add two tensors +pub fn add(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop::(lhs, rhs) +} + +/// Add a tensor and a scalar +pub fn add_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop::(lhs, rhs) +} + +/// Subtract two tensors +pub fn sub(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop::(lhs, rhs) +} + +/// Subtract a tensor and a scalar +pub fn sub_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop::(lhs, rhs) +} + +/// Multiply two tensors +pub fn mul(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop::(lhs, rhs) +} + +/// Multiply a tensor and a scalar +pub fn mul_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop::(lhs, rhs) +} + +/// Divide two tensors +pub fn div(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop::(lhs, rhs) +} + +/// Divide a tensor by a scalar +pub fn div_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop::(lhs, rhs) +} + +/// Calculate remainder of two tensors +pub fn remainder(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop::(lhs, rhs) +} + +/// Calculate the remainder of a tensor with a scalar +pub fn remainder_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop::(lhs, rhs) +} + +/// Calculate the power of two tensors +pub fn pow(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop::(lhs, rhs) +} + +/// Bitwise and two tensors +pub fn bitwise_and(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop_int::(lhs, rhs) +} + +/// Bitwise and with a scalar +pub fn bitwise_and_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop_int::(lhs, rhs) +} + +/// Bitwise or two tensors +pub fn bitwise_or(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop_int::(lhs, rhs) +} + +/// Bitwise or with a scalar +pub fn bitwise_or_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop_int::(lhs, rhs) +} + +/// Bitwise xor two tensors +pub fn bitwise_xor(lhs: CubeTensor, rhs: CubeTensor) -> CubeTensor { + launch_binop_int::(lhs, rhs) +} + +/// Bitwise xor with a scalar +pub fn bitwise_xor_scalar(lhs: CubeTensor, rhs: InputScalar) -> CubeTensor { + launch_scalar_binop_int::(lhs, rhs) +} + +/// Operation family trait for cumulative operations +pub(crate) trait CumulativeOpFamily: Send + Sync + 'static { + type CumulativeOp: CumulativeOp; +} + +/// Trait for cumulative operations +#[cube] +pub(crate) trait CumulativeOp: 'static + Send + Sync { + /// Execute a cumulative operation + fn execute(lhs: C, rhs: C) -> C; + + /// Get the initial value for the accumulator + fn init_value(first_element: C) -> C; +} + +// Operation types +struct SumOp; +struct ProdOp; +struct MaxOp; +struct MinOp; + +// Implement CumulativeOpFamily for each operation +impl CumulativeOpFamily for SumOp { + type CumulativeOp = Self; +} + +impl CumulativeOpFamily for ProdOp { + type CumulativeOp = Self; +} + +impl CumulativeOpFamily for MaxOp { + type CumulativeOp = Self; +} + +impl CumulativeOpFamily for MinOp { + type CumulativeOp = Self; +} + +// Implement CumulativeOp for each operation type +#[cube] +impl CumulativeOp for SumOp { + fn execute(lhs: N, rhs: N) -> N { + lhs + rhs + } + + fn init_value(_first_element: N) -> N { + N::from_int(0) + } +} + +#[cube] +impl CumulativeOp for ProdOp { + fn execute(lhs: N, rhs: N) -> N { + lhs * rhs + } + + fn init_value(_first_element: N) -> N { + N::from_int(1) + } +} + +#[cube] +impl CumulativeOp for MaxOp { + fn execute(lhs: N, rhs: N) -> N { + max(lhs, rhs) + } + + fn init_value(first_element: N) -> N { + first_element + } +} + +#[cube] +impl CumulativeOp for MinOp { + fn execute(lhs: N, rhs: N) -> N { + min(lhs, rhs) + } + + fn init_value(first_element: N) -> N { + first_element + } +} + +/// Generic cumulative operation kernel +/// +/// # Limitations +/// +/// This is a **naive sequential implementation** along the cumulative dimension: +/// - Each output element sequentially reads all previous elements along the dimension +/// - Computational complexity: O(n^2) memory reads where n is the size of the cumulative dimension +/// - **Performance:** Suitable for small tensors or small dimensions. For large tensors, +/// performance will degrade significantly compared to an optimized parallel scan algorithm. +/// +/// # TODO +/// +/// Implement an efficient GPU-optimized parallel scan algorithm. +#[cube(launch_unchecked, address_type = "dynamic")] +fn cumulative_kernel( + input: &Tensor, + output: &mut LinearView, + shape: Sequence>, + #[comptime] dim: usize, + #[define(C)] _dtype: StorageType, +) { + if !output.is_in_bounds(ABSOLUTE_POS) { + terminate!(); + } + + let rank = comptime![shape.len()]; + let dim_stride = input.stride(dim); + + let mut remainder = ABSOLUTE_POS; + let mut offset = 0; + let mut dim_idx = 0; + + #[unroll] + for i in 0..shape.len() { + let i = comptime![rank - i - 1]; + let (rem, local_idx) = shape.index(i).div_mod(remainder); + remainder = rem; + if i == dim { + dim_idx = local_idx; + } else { + offset += local_idx * input.stride(i); + } + } + + // Read first element + let first_read_idx = offset + dim_idx * dim_stride; + let first_elem = input[first_read_idx]; + + // Initialize accumulator + let mut result = O::CumulativeOp::::init_value(first_elem); + + // Accumulate values + for i in 0..=dim_idx { + let read_idx = offset + i * dim_stride; + result = O::CumulativeOp::::execute(result, input[read_idx]); + } + output[ABSOLUTE_POS] = result; +} + +/// Compute the cumulative sum along a dimension +pub fn cumsum(input: CubeTensor, dim: usize) -> CubeTensor { + cumulative_op::(input, dim) +} + +/// Compute the cumulative product along a dimension +pub fn cumprod(input: CubeTensor, dim: usize) -> CubeTensor { + cumulative_op::(input, dim) +} + +/// Compute the cumulative minimum along a dimension +pub fn cummin(input: CubeTensor, dim: usize) -> CubeTensor { + cumulative_op::(input, dim) +} + +/// Compute the cumulative maximum along a dimension +pub fn cummax(input: CubeTensor, dim: usize) -> CubeTensor { + cumulative_op::(input, dim) +} + +/// Generic cumulative operation function +fn cumulative_op( + input: CubeTensor, + dim: usize, +) -> CubeTensor { + let client = input.client.clone(); + let device = input.device.clone(); + + let output = empty_device_dtype(client.clone(), device, input.shape(), input.dtype); + + let num_elems = output.meta.num_elements(); + let working_units = num_elems; + let cube_dim = CubeDim::new(&client, working_units); + let cube_count = calculate_cube_count_elemwise(&client, working_units, cube_dim); + + unsafe { + cumulative_kernel::launch_unchecked::( + &client, + cube_count, + cube_dim, + address_type!(input, output), + input.as_tensor_arg(1), + linear_view(&output, 1), + shape_divmod(&input), + dim, + output.dtype.into(), + ) + .expect("Kernel to never fail"); + } + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/qtensor.rs new file mode 100644 index 0000000..70e5a7c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/qtensor.rs @@ -0,0 +1,313 @@ +use burn_backend::{ + Bytes, DType, ExecutionError, QTensorPrimitive, Shape, Slice, TensorData, TensorMetadata, + TensorPrimitive, + ops::QTensorOps, + quantization::{ + QParamTensor, QuantLevel, QuantMode, QuantParam, QuantPropagation, QuantScheme, QuantValue, + QuantizationParametersPrimitive, params_shape, + }, + tensor::{Device, FloatElem, FloatTensor, IntTensor, QuantizedTensor}, +}; +use burn_std::Metadata; +use cubecl::server::{Allocation, AllocationDescriptor, AllocationKind}; +use cubecl::{e2m1x2, quant::scheme::QuantStore}; + +use crate::{ + CubeBackend, CubeRuntime, FloatElement, IntElement, + element::BoolElement, + kernel::{self, matmul::MatmulStrategy}, + tensor::{CubeTensor, QParams}, +}; + +use super::{into_data, permute, swap_dims}; + +/// Create a quantized tensor with packed values (u32). +fn new_qtensor_optimized( + data: Bytes, + shape: impl Into, + scheme: QuantScheme, + device: &R::Device, +) -> CubeTensor { + new_qtensor(data, shape, scheme, device, AllocationKind::Optimized) +} + +/// Create a quantized tensor with packed values (u32). +fn new_qtensor( + data: Bytes, + shape: impl Into, + scheme: QuantScheme, + device: &R::Device, + kind: AllocationKind, +) -> CubeTensor { + new_quantized(shape, scheme, device, Some(data), kind) +} + +/// Create an empty quantized tensor. +pub fn empty_qtensor_optimized( + shape: impl Into, + scheme: QuantScheme, + device: &R::Device, +) -> CubeTensor { + empty_qtensor(shape, scheme, device, AllocationKind::Optimized) +} + +/// Create an empty quantized tensor. +pub fn empty_qtensor( + shape: impl Into, + scheme: QuantScheme, + device: &R::Device, + kind: AllocationKind, +) -> CubeTensor { + new_quantized(shape, scheme, device, None, kind) +} + +fn new_quantized( + shape: impl Into, + scheme: QuantScheme, + device: &R::Device, + data: Option, + alloc_kind: AllocationKind, +) -> CubeTensor { + let client = R::client(device); + let shape: Shape = shape.into(); + let mut shape_value: Shape = shape.clone(); + + let rank = shape.rank(); + let shape_last = shape[rank - 1]; + let num_quants = scheme.num_quants(); + + let data_size = match scheme.store { + QuantStore::PackedU32(_) => { + if !shape_last.is_multiple_of(num_quants) { + panic!("Can't store in u32") + } + shape_value[rank - 1] = shape_last.div_ceil(num_quants); + size_of::() + } + QuantStore::Native => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S | QuantValue::E4M3 | QuantValue::E5M2 => { + size_of::() + } + QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E2M1 => { + panic!("Can't store native sub-byte values") + } + }, + QuantStore::PackedNative(_) => match scheme.value { + QuantValue::E2M1 => size_of::(), + other => panic!("{other:?} doesn't support native packing"), + }, + }; + + let scales_dtype = match scheme.param { + QuantParam::F32 => DType::F32, + QuantParam::F16 => DType::F16, + QuantParam::BF16 => DType::BF16, + // Represented by U8 and reinterpreted in the kernel + QuantParam::UE8M0 | QuantParam::UE4M3 => DType::U8, + }; + + let scales_shape = params_shape(&shape, scheme.level); + let data_desc = AllocationDescriptor::new(alloc_kind, &shape_value, data_size); + let scales_desc = AllocationDescriptor::new(alloc_kind, &scales_shape, scales_dtype.size()); + + let mut tensors = match data { + Some(data) => { + let num_bytes = shape_value.num_elements() * data_size; + + match data.split(num_bytes) { + Ok((bytes_data, bytes_scales)) => client + .create_tensors(vec![(data_desc, bytes_data), (scales_desc, bytes_scales)]), + Err((data, _)) => client.create_tensors_from_slices(vec![ + (data_desc, &data[..num_bytes]), + (scales_desc, &data[num_bytes..]), + ]), + } + } + None => client.empty_tensors(vec![data_desc, scales_desc]), + }; + let Allocation { + handle: scales_handle, + strides: scales_strides, + } = tensors.remove(1); + let Allocation { handle, strides } = tensors.remove(0); + + let scales = QParamTensor { + offset_start: scales_handle.offset_start.unwrap_or(0) as usize, + offset_end: scales_handle.offset_end.unwrap_or(0) as usize, + metadata: Metadata::new(scales_shape, scales_strides), + dtype: scales_dtype, + }; + let qparams = QParams { scales }; + + CubeTensor::new_quantized( + client, + handle, + shape, + device.clone(), + strides, + DType::QFloat(scheme), + qparams, + ) +} + +impl QTensorOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + fn q_from_data(data: TensorData, device: &Device) -> QuantizedTensor { + match data.dtype { + DType::QFloat(scheme) => match scheme { + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + value: + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E4M3 + | QuantValue::E5M2 + | QuantValue::E2M1, + .. + } => { + // TensorData quantized representation is the same, with multiple quantized values + // packed into u32 and quantization parameters appended to the bytes + new_qtensor_optimized(data.bytes, data.shape.clone(), scheme, device) + } + }, + _ => panic!( + "Invalid dtype (expected DType::QFloat, got {:?})", + data.dtype + ), + } + } + + // TODO: quantize_dynamic (we can compute min-max on the fly and scale, especially when not per-tensor) + + fn quantize( + tensor: FloatTensor, + scheme: &QuantScheme, + qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + kernel::quantization::quantize(tensor, scheme, qparams.scales) + } + + fn dequantize(tensor: QuantizedTensor) -> FloatTensor { + kernel::quantization::dequantize(tensor, FloatElem::::dtype()) + } + + fn q_device(tensor: &QuantizedTensor) -> Device { + tensor.device.clone() + } + + fn q_to_device(tensor: QuantizedTensor, device: &Device) -> QuantizedTensor { + super::to_device(tensor, device) + } + + fn q_reshape(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + super::q_reshape(tensor, shape) + } + + async fn q_into_data(tensor: QuantizedTensor) -> Result { + if tensor.qparams.is_none() { + return into_data(tensor).await; + } + + let (shape, dtype) = (tensor.shape(), tensor.dtype); + let (values, params) = tensor.quantized_handles().unwrap(); + + let mut data_values = into_data(values).await?; + let data_params = into_data(params).await?; + + data_values.bytes.extend_from_byte_slice(&data_params.bytes); + + Ok(TensorData { + bytes: data_values.bytes, + shape: shape.to_vec(), + dtype, + }) + } + + fn q_swap_dims( + tensor: QuantizedTensor, + dim1: usize, + dim2: usize, + ) -> QuantizedTensor { + swap_dims(tensor, dim1, dim2) + } + + fn q_permute(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + permute(tensor, axes) + } + + fn q_flip(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_gather( + _dim: usize, + _tensor: QuantizedTensor, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_select( + _tensor: QuantizedTensor, + _dim: usize, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_slice(_tensor: QuantizedTensor, _slices: &[Slice]) -> QuantizedTensor { + unimplemented!() + } + + fn q_expand(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } + + fn q_matmul(lhs: TensorPrimitive, rhs: TensorPrimitive) -> TensorPrimitive { + let (propagation, scheme) = match (&lhs, &rhs) { + (TensorPrimitive::QFloat(lhs), _) => (lhs.propagation(), *lhs.scheme()), + (_, TensorPrimitive::QFloat(rhs)) => (rhs.propagation(), *rhs.scheme()), + _ => unreachable!(), + }; + + // Inherit precision for mixed inputs, default to `FloatElem` for fully quantized. + let out_dtype = match (&lhs, &rhs) { + (TensorPrimitive::Float(lhs), _) => lhs.dtype, + (_, TensorPrimitive::Float(rhs)) => rhs.dtype, + _ => F::dtype(), + }; + + let (_lhs_dtype, lhs) = match lhs { + TensorPrimitive::Float(lhs) => (lhs.dtype, lhs), + TensorPrimitive::QFloat(lhs) => (out_dtype, lhs), + }; + let (_rhs_dtype, rhs) = match rhs { + TensorPrimitive::Float(rhs) => (rhs.dtype, rhs), + TensorPrimitive::QFloat(rhs) => (out_dtype, rhs), + }; + + let out = + kernel::matmul::matmul(lhs, rhs, None, MatmulStrategy::default(), out_dtype).unwrap(); + + match propagation { + QuantPropagation::Propagate => { + TensorPrimitive::QFloat(Self::quantize_dynamic(out, &scheme)) + } + QuantPropagation::Inhibit => TensorPrimitive::Float(out), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/tensor.rs new file mode 100644 index 0000000..c99d066 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/tensor.rs @@ -0,0 +1,620 @@ +use super::{expand, numeric, permute, unfold}; +use crate::CubeBackend; +use crate::kernel::prng::{random_bernoulli, random_normal, random_uniform}; +use crate::kernel::unary_basic::BasicFloatUnaryKind; +use crate::kernel::{ + self, FloatUnaryOp, FloatUnaryOpFamily, launch_unary_float, reduce, unary_basic, +}; +use crate::{CubeRuntime, FloatElement, IntElement}; +use crate::{ + element::BoolElement, + kernel::matmul::{MatmulStrategy, matmul}, +}; +use burn_backend::ops::GridSampleOptions; +use burn_backend::tensor::{BoolTensor, Device, FloatElem, FloatTensor, IntTensor}; +use burn_backend::{Backend, ExecutionError, Scalar}; +use burn_backend::{DType, ElementConversion, FloatDType, Slice}; +use burn_backend::{Distribution, Shape, TensorData, ops::FloatTensorOps}; +use cubecl::prelude::*; +use cubek::reduce::components::instructions::ReduceOperationConfig; +use std::ops::Range; + +impl FloatTensorOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(data), + fields(?data.shape, ?data.dtype) + ))] + fn float_from_data(data: TensorData, device: &Device) -> FloatTensor { + match data.dtype { + DType::F64 | DType::F32 | DType::F16 | DType::BF16 => super::from_data(data, device), + _ => unimplemented!("Unsupported dtype for `float_from_data`"), + } + } + + fn float_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> FloatTensor { + let dtype = FloatElem::::dtype(); + match distribution { + Distribution::Default => random_uniform(shape, device, 0., 1., dtype), + Distribution::Uniform(low, high) => { + random_uniform(shape, device, low.elem(), high.elem(), dtype) + } + Distribution::Bernoulli(prob) => random_bernoulli(shape, device, prob as f32, dtype), + Distribution::Normal(mean, std) => { + random_normal(shape, device, mean.elem(), std.elem(), dtype) + } + } + } + + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensor), + fields(from = ?tensor.device, meta = ?tensor.meta, dtype = ?tensor.dtype) + ))] + async fn float_into_data(tensor: FloatTensor) -> Result { + super::into_data(tensor).await + } + + fn float_device(tensor: &FloatTensor) -> Device { + tensor.device.clone() + } + + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensor), + fields(from = ?tensor.device, meta = ?tensor.meta, dtype = ?tensor.dtype) + ))] + fn float_to_device(tensor: FloatTensor, device: &Device) -> FloatTensor { + super::to_device(tensor, device) + } + + fn float_empty(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + let dtype = dtype.into(); + super::empty(shape, device, dtype) + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + numeric::add(lhs, rhs) + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let dtype = lhs.dtype; + numeric::add_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn float_zeros(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + let dtype = dtype.into(); + numeric::zeros(device.clone(), shape, dtype) + } + + fn float_full( + shape: Shape, + fill_value: Scalar, + device: &R::Device, + dtype: FloatDType, + ) -> FloatTensor { + let dtype: DType = dtype.into(); + let client = R::client(device); + numeric::full_device_dtype( + client, + shape, + device.clone(), + InputScalar::new(fill_value, dtype), + dtype, + ) + } + + fn float_ones(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + let dtype = dtype.into(); + numeric::ones(device.clone(), shape, dtype) + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + numeric::sub(lhs, rhs) + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let dtype = lhs.dtype; + numeric::sub_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + numeric::mul(lhs, rhs) + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let dtype = lhs.dtype; + numeric::mul_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + numeric::div(lhs, rhs) + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let dtype = lhs.dtype; + numeric::div_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + numeric::remainder(lhs, rhs) + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let dtype = lhs.dtype; + numeric::remainder_scalar(lhs, InputScalar::new(rhs, dtype)) + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let dtype = lhs.dtype; + matmul(lhs, rhs, None, MatmulStrategy::default(), dtype).unwrap() + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + kernel::cross(lhs, rhs, dim) + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + super::swap_dims(tensor, dim1, dim2) + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + super::reshape(tensor, shape) + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + kernel::gather(dim, tensor, indices) + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + kernel::scatter(dim, tensor, indices, value, false) + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + ) -> FloatTensor { + kernel::select(tensor, dim, indices) + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + kernel::select_assign(tensor, dim, indices, value, false) + } + + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor { + // Check if all steps are 1 + let all_steps_one = slices.iter().all(|info| info.step == 1); + + if all_steps_one { + // Use optimized slice for step=1 + let simple_ranges: Vec> = slices + .iter() + .enumerate() + .map(|(i, slice)| slice.to_range(tensor.meta.shape()[i])) + .collect(); + + kernel::slice(tensor, &simple_ranges) + } else { + // Use slice with steps kernel + kernel::slice_with_steps(tensor, slices) + } + } + + fn float_slice_assign( + tensor: FloatTensor, + ranges: &[Slice], + value: FloatTensor, + ) -> FloatTensor { + kernel::slice_assign(tensor, ranges, value) + } + + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + value: FloatTensor, + ) -> FloatTensor { + kernel::mask_where_auto(tensor, mask, value, BT::dtype()) + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor { + let dtype = tensor.dtype; + kernel::mask_fill_auto(tensor, mask, InputScalar::new(value, dtype), BT::dtype()) + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + kernel::equal(lhs, rhs, BT::dtype()) + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::equal_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + kernel::greater(lhs, rhs, BT::dtype()) + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::greater_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + kernel::greater_equal(lhs, rhs, BT::dtype()) + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::greater_equal_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + kernel::lower(lhs, rhs, BT::dtype()) + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::lower_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + kernel::lower_equal(lhs, rhs, BT::dtype()) + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let dtype = lhs.dtype; + kernel::lower_equal_elem(lhs, InputScalar::new(rhs, dtype), BT::dtype()) + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + reduce::sum_fallback(tensor, Default::default()).unwrap() + } + + fn float_max(tensor: FloatTensor) -> FloatTensor { + reduce::reduce(tensor, None, Default::default(), ReduceOperationConfig::Max).unwrap() + } + + fn float_max_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Max, + ) + .unwrap() + } + + fn float_min(tensor: FloatTensor) -> FloatTensor { + reduce::reduce(tensor, None, Default::default(), ReduceOperationConfig::Min).unwrap() + } + + fn float_min_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Min, + ) + .unwrap() + } + + fn float_max_abs(tensor: FloatTensor) -> FloatTensor { + reduce::reduce( + tensor, + None, + Default::default(), + ReduceOperationConfig::MaxAbs, + ) + .unwrap() + } + + fn float_max_abs_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::MaxAbs, + ) + .unwrap() + } + + fn float_sum_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Sum, + ) + .unwrap() + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Mean, + ) + .unwrap() + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + numeric::cumsum(tensor, dim) + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + numeric::cumprod(tensor, dim) + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + numeric::cummin(tensor, dim) + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + numeric::cummax(tensor, dim) + } + + fn float_prod(tensor: FloatTensor) -> FloatTensor { + reduce::reduce( + tensor, + None, + Default::default(), + ReduceOperationConfig::Prod, + ) + .unwrap() + } + + fn float_prod_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce::reduce_dim( + tensor, + None, + dim, + Default::default(), + ReduceOperationConfig::Prod, + ) + .unwrap() + } + + fn float_exp(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Exp) + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Log) + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Log1p) + } + + fn float_powf_scalar_impl(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + struct Powf; + + #[cube] + impl FloatUnaryOp for Powf { + type Options = InputScalar; + + fn execute(input: Line, options: &Self::Options) -> Line { + Line::powf(input, Line::new(options.get::())) + } + } + + impl FloatUnaryOpFamily for Powf { + type Options = InputScalar; + type Unary = Self; + } + + let dtype = lhs.dtype; + launch_unary_float::(lhs, |_| InputScalar::new(rhs, dtype)) + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Sqrt) + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Abs) + } + + fn float_sign(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Sign) + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Cos) + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Sin) + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Tan) + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Cosh) + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Sinh) + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Tanh) + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::ArcCos) + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::ArcCosh) + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::ArcSin) + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::ArcSinh) + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::ArcTan) + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::ArcTanh) + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + crate::kernel::atan2::(lhs, rhs) + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Round) + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Floor) + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Ceil) + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Trunc) + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Erf) + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + Some(::IntElem::dtype()), + dim, + Default::default(), + ReduceOperationConfig::ArgMax, + ) + .unwrap() + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor { + reduce::reduce_dim( + tensor, + Some(::IntElem::dtype()), + dim, + Default::default(), + ReduceOperationConfig::ArgMin, + ) + .unwrap() + } + + fn float_into_int(tensor: FloatTensor) -> IntTensor { + kernel::cast(tensor, I::dtype()) + } + + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + let dtype = tensor.dtype; + kernel::clamp( + tensor, + InputScalar::new(min, dtype), + InputScalar::new(max, dtype), + ) + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + unary_basic::launch::(tensor, |_| BasicFloatUnaryKind::Recip) + } + + fn float_repeat_dim(tensor: FloatTensor, dim: usize, times: usize) -> FloatTensor { + kernel::repeat_dim(tensor, dim, times) + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + numeric::pow(lhs, rhs) + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + permute(tensor, axes) + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + expand(tensor, shape) + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + kernel::flip(tensor, axes, BT::dtype()) + } + + fn float_cast(tensor: FloatTensor, dtype: FloatDType) -> FloatTensor { + kernel::cast(tensor, dtype.into()) + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + unfold(tensor, dim, size, step) + } + + fn float_is_nan(tensor: FloatTensor) -> BoolTensor { + kernel::is_nan(tensor, BT::dtype()) + } + + fn float_is_inf(tensor: FloatTensor) -> BoolTensor { + kernel::is_inf(tensor, BT::dtype()) + } + + fn float_grid_sample_2d( + tensor: FloatTensor, + grid: FloatTensor, + options: GridSampleOptions, + ) -> FloatTensor { + kernel::grid_sample::grid_sample(tensor, grid, options) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/transaction.rs new file mode 100644 index 0000000..6326802 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/ops/transaction.rs @@ -0,0 +1,143 @@ +use burn_backend::{ + DType, TensorData, + backend::ExecutionError, + ops::{TransactionOps, TransactionPrimitive, TransactionPrimitiveData}, +}; +use burn_std::{Shape, Strides}; +use cubecl::server::{Binding, CopyDescriptor}; + +use crate::{CubeBackend, CubeRuntime, FloatElement, IntElement, element::BoolElement}; + +impl TransactionOps for CubeBackend +where + R: CubeRuntime, + F: FloatElement, + I: IntElement, + BT: BoolElement, +{ + async fn tr_execute( + transaction: TransactionPrimitive, + ) -> Result { + let mut client = None; + + enum Kind { + Float, + Int, + Bool, + } + + #[derive(new)] + struct BindingData { + index: usize, + kind: Kind, + handle: Option, + shape: Shape, + strides: Strides, + dtype: DType, + } + + let mut num_bindings = 0; + + let mut kinds = Vec::new(); + + for t in transaction.read_floats.into_iter() { + if client.is_none() { + client = Some(t.client.clone()); + } + + let t = crate::kernel::into_contiguous_aligned(t); + let binding = BindingData::new( + num_bindings, + Kind::Float, + Some(t.handle.binding()), + t.meta.shape, + t.meta.strides, + t.dtype, + ); + + kinds.push(binding); + num_bindings += 1; + } + for t in transaction.read_ints.into_iter() { + if client.is_none() { + client = Some(t.client.clone()); + } + + let t = crate::kernel::into_contiguous_aligned(t); + let binding = BindingData::new( + num_bindings, + Kind::Int, + Some(t.handle.binding()), + t.meta.shape, + t.meta.strides, + t.dtype, + ); + + kinds.push(binding); + num_bindings += 1; + } + for t in transaction.read_bools.into_iter() { + if client.is_none() { + client = Some(t.client.clone()); + } + + let t = crate::kernel::into_contiguous_aligned(t); + let binding = BindingData::new( + num_bindings, + Kind::Bool, + Some(t.handle.binding()), + t.meta.shape, + t.meta.strides, + t.dtype, + ); + + kinds.push(binding); + num_bindings += 1; + } + + let client = client.unwrap(); + + let bindings = kinds + .iter_mut() + .map(|b| { + CopyDescriptor::new( + b.handle.take().unwrap(), + &b.shape, + &b.strides, + b.dtype.size(), + ) + }) + .collect(); + + let mut data: Vec> = client + .read_tensor_async(bindings) + .await + .map_err(|err| ExecutionError::WithContext { + reason: format!("{err:?}"), + })? + .into_iter() + .map(Some) + .collect::>>(); + + let mut result = TransactionPrimitiveData::default(); + + for binding in kinds { + let bytes = data.get_mut(binding.index).unwrap().take().unwrap(); + let t_data = TensorData::from_bytes(bytes, binding.shape, binding.dtype); + + match binding.kind { + Kind::Float => { + result.read_floats.push(t_data); + } + Kind::Int => { + result.read_ints.push(t_data); + } + Kind::Bool => { + result.read_bools.push(t_data); + } + } + } + + Ok(result) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/base.rs new file mode 100644 index 0000000..1ee2f9b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/base.rs @@ -0,0 +1,103 @@ +use super::SourceTemplate; +use crate::{CubeRuntime, element::CubeElement, tensor::CubeTensor}; +use cubecl::{CompilationError, Compiler, CubeTask, prelude::*}; + +/// Kernel source to create a [source](SourceTemplate) +pub trait KernelSource: Send + 'static + Sync { + /// Convert to [source](SourceTemplate) + fn source(&self) -> SourceTemplate; + /// Identifier for the kernel, used for caching kernel compilation. + fn id(&self) -> KernelId; +} + +#[derive(new)] +/// Wraps a [kernel source](KernelSource) into a [cube task](CubeTask). +pub struct SourceKernel { + kernel_source: K, + cube_dim: CubeDim, +} + +impl CubeTask for SourceKernel { + fn compile( + &self, + _compiler: &mut C, + _options: &C::CompilationOptions, + _mode: ExecutionMode, + _address_type: StorageType, + ) -> Result, CompilationError> { + let source_template = self.kernel_source.source(); + let source = source_template.complete(); + + Ok(CompiledKernel { + entrypoint_name: "main".to_string(), + debug_name: Some(core::any::type_name::()), + source, + cube_dim: self.cube_dim, + debug_info: None, + repr: None, + }) + } +} + +impl KernelMetadata for SourceKernel { + fn id(&self) -> KernelId { + self.kernel_source.id() + } + + fn address_type(&self) -> StorageType { + u32::as_type_native_unchecked() + } +} + +/// Generates kernel source code by replacing some information using templating. +#[macro_export] +macro_rules! kernel_source { + ( + $struct:ident, + $file:expr + ) => { + /// Generated kernel from a source file. + #[derive(new)] + pub struct $struct; + + impl $struct { + fn source(&self) -> $crate::template::SourceTemplate { + $crate::template::SourceTemplate::new(include_str!($file)) + } + } + }; +} + +/// Create a vector containing the dimension, strides and shape of tensors. +/// +/// # Example +/// +/// With two tensors (lhs, rhs) +/// +/// | Indexes | Value | +/// |:------------------------:|:-----------:| +/// | 0..1 | D | +/// | 1..D + 1 | lhs strides | +/// | (D + 1)..(2 * D + 1) | rhs strides | +/// | (2 * D + 1)..(3 * D + 1) | lhs shape | +/// | (3 * D + 1)..(4 * D + 1) | rhs shape | +pub fn build_info(tensors: &[&CubeTensor]) -> Vec { + let ndims = tensors[0].meta.num_dims(); + let mut info: Vec = vec![0; tensors.len() * 2 * ndims + 1]; + info[0] = ndims as u32; + + let mut current = 1; + for tensor in tensors.iter() { + for d in 0..ndims { + info[current] = tensor.meta.strides()[d] as u32; + current += 1; + } + } + for tensor in tensors.iter() { + for d in 0..ndims { + info[current] = tensor.meta.shape()[d] as u32; + current += 1; + } + } + info +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/mod.rs new file mode 100644 index 0000000..8c34090 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/mod.rs @@ -0,0 +1,5 @@ +mod base; +pub use base::*; + +mod source; +pub use source::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/source.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/source.rs new file mode 100644 index 0000000..b13c2f6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/template/source.rs @@ -0,0 +1,69 @@ +use std::collections::HashMap; + +/// Kernel source code abstraction allowing for templating. +/// +/// The templates can have text placeholders in the form {{ label }}. +/// They will be updated with their proper value when `generate` is called. +#[derive(Debug)] +pub struct SourceTemplate { + items: HashMap, + templates: Vec, +} + +impl SourceTemplate { + /// Create a new source template. + pub fn new(template: S) -> Self + where + S: Into, + { + Self { + items: HashMap::new(), + templates: vec![template.into()], + } + } + + /// Register the value for a placeholder item. + /// + /// # Notes + /// + /// The value can't have placeholders, since it would require recursive templating with + /// possibly circular dependencies. If you want to add a value that has some + /// placeholders, consider adding a new template to the source using + /// [add_template](SourceTemplate::add_template). The added template can be a function, and you can + /// register the function call instead. + pub fn register(mut self, name: Name, value: Value) -> Self + where + Name: Into, + Value: Into, + { + self.items.insert(name.into(), value.into()); + self + } + + /// Add a new template. + pub fn add_template(mut self, template: S) -> Self + where + S: Into, + { + self.templates.push(template.into()); + self + } + + /// Complete the template and returns the source code. + pub fn complete(mut self) -> String { + let mut source = self.templates.remove(0); + + for s in self.templates.into_iter() { + source.push_str(&s); + } + + let template = text_placeholder::Template::new(&source); + let mut context = HashMap::new(); + + for (key, value) in self.items.iter() { + context.insert(key.as_str(), value.as_str()); + } + + template.fill_with_hashmap(&context) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/base.rs new file mode 100644 index 0000000..118b5de --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/base.rs @@ -0,0 +1,379 @@ +use crate::CubeRuntime; +use crate::element::CubeElement; +use crate::kernel::{NumericUnaryOp, NumericUnaryOpFamily, launch_unary_numeric}; +use burn_backend::quantization::QuantScheme; +use burn_backend::{DType, QTensorPrimitive, Shape, TensorMetadata}; +use burn_std::{Metadata, strides, tensor::is_contiguous}; +use cubecl::client::ComputeClient; +use cubecl::frontend::Numeric; +use cubecl::prelude::{TensorHandleRef, *}; +use cubecl::server::Handle; +use cubecl::std::tensor::TensorHandle; +use std::marker::PhantomData; + +use super::QParams; + +/// The basic tensor primitive struct. +pub struct CubeTensor { + /// Compute client for the [runtime](CubeRuntime). + pub client: ComputeClient, + /// The buffer where the data are stored. + pub handle: Handle, + /// The metadata of the tensor. + pub meta: Box, + /// The device of the tensor. + pub device: R::Device, + /// The datatype of the tensor. + pub dtype: DType, + /// Runtime quantization parameters, if applicable + pub qparams: Option, +} + +impl From> for TensorHandle { + fn from(val: CubeTensor) -> Self { + TensorHandle::new( + val.handle, + val.meta.shape().clone(), + val.meta.strides().clone(), + val.dtype.into(), + ) + } +} + +impl cubecl::tune::AutotuneOutput for CubeTensor { + #[cfg(feature = "autotune-checks")] + fn check_equivalence(&self, other: Self) { + use crate::ops::into_data_sync; + use burn_backend::Tolerance; + + let expected = into_data_sync::(self.clone()); + let actual = into_data_sync::(other); + expected.assert_approx_eq::(&actual, Tolerance::permissive()); + } +} + +impl core::fmt::Debug for CubeTensor +where + R: CubeRuntime, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "CubeTensor {{ shape: {:?}, device: {:?}, strides: {:?}, elem: {}, runtime: {}}}", + self.meta.shape(), + self.device, + self.meta.strides(), + self.dtype.name(), + R::name(&self.client), + )) + } +} + +impl Clone for CubeTensor +where + R: CubeRuntime, +{ + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + handle: self.handle.clone(), + meta: self.meta.clone(), + device: self.device.clone(), + dtype: self.dtype, + qparams: self.qparams.clone(), + } + } +} + +impl TensorMetadata for CubeTensor { + fn dtype(&self) -> DType { + self.dtype + } + + fn shape(&self) -> Shape { + self.meta.shape().clone() + } + + fn rank(&self) -> usize { + self.meta.rank() + } +} + +impl QTensorPrimitive for CubeTensor { + fn scheme(&self) -> &QuantScheme { + if let DType::QFloat(scheme) = &self.dtype { + scheme + } else { + panic!( + "Quantization scheme is not valid for dtype {:?}", + self.dtype, + ) + } + } +} + +impl CubeTensor +where + R: CubeRuntime, +{ + /// Create a new standard tensor + pub fn new( + client: ComputeClient, + handle: Handle, + metadata: Metadata, + device: R::Device, + dtype: DType, + ) -> Self { + CubeTensor { + client, + handle, + meta: Box::new(metadata), + device, + dtype, + qparams: None, + } + } + + /// Create a new tensor with a contiguous memory layout. + pub fn new_contiguous( + client: ComputeClient, + device: R::Device, + shape: Shape, + handle: Handle, + dtype: DType, + ) -> Self { + let ndims = shape.num_dims(); + let mut strides = strides![0; ndims]; + let mut current = 1; + + shape.iter().enumerate().rev().for_each(|(index, val)| { + strides[index] = current; + current *= val; + }); + + Self { + client, + handle, + meta: Box::new(Metadata::new(shape, strides)), + device, + dtype, + qparams: None, + } + } + + /// Change the context of the current tensor and return the newly transferred tensor. + pub fn to_client(&self, client: ComputeClient, device: R::Device) -> Self { + let desc = + self.handle + .copy_descriptor(self.meta.shape(), self.meta.strides(), self.elem_size()); + let alloc = self.client.to_client_tensor(desc, &client); + + Self { + client, + handle: alloc.handle, + meta: Box::new(Metadata::new(self.shape(), alloc.strides)), + device, + dtype: self.dtype, + qparams: self.qparams.clone(), + } + } + + /// Return the reference to a tensor handle. + pub fn as_handle_ref(&self) -> TensorHandleRef<'_, R> { + TensorHandleRef { + handle: &self.handle, + strides: self.meta.strides(), + shape: self.meta.shape(), + runtime: PhantomData, + elem_size: self.elem_size(), + } + } + + /// Returns the element size of this tensor + pub fn elem_size(&self) -> usize { + self.dtype.size() + } + + /// Return the reference to a tensor argument. + pub fn as_tensor_arg<'a>(&'a self, line_size: LineSize) -> TensorArg<'a, R> { + let size = self.dtype.size(); + let handle: TensorHandleRef<'a, R> = self.as_handle_ref(); + + unsafe { + TensorArg::from_raw_parts_and_size( + handle.handle, + handle.strides, + handle.shape, + line_size, + size, + ) + } + } + + /// Return the reference to an array argument. + pub fn as_array_arg(&self, line_size: LineSize) -> ArrayArg<'_, R> { + unsafe { + ArrayArg::from_raw_parts::( + &self.handle, + self.handle.size() as usize / core::mem::size_of::(), + line_size, + ) + } + } + + /// Returns the address type required to index this tensor + pub fn required_address_type(&self) -> AddressType { + match self.try_scheme() { + Some(scheme) => { + let len = self.handle.size() as usize * 8 / scheme.size_bits_value(); + AddressType::from_len(len) + } + None => AddressType::from_len(self.handle.size() as usize / self.dtype.size()), + } + } + + /// Return the `QuantScheme` if present + pub fn try_scheme(&self) -> Option<&QuantScheme> { + match &self.dtype { + DType::QFloat(scheme) => Some(scheme), + _ => None, + } + } + + pub(crate) fn can_mut_broadcast(&self, rhs: &Self) -> bool { + if !self.handle.can_mut() || !self.is_nonoverlapping() { + return false; + } + let ndims = self.meta.num_dims(); + + for i in 0..ndims { + let shape_lhs = self.meta.shape()[i]; + let shape_rhs = rhs.meta.shape()[i]; + + // Output tensor will be different from the mutable tensor. + if shape_lhs < shape_rhs { + return false; + } + } + + true + } + + /// Copy the current tensor. + pub fn copy(&self) -> Self { + struct Copy; + + #[cube] + impl NumericUnaryOp for Copy { + type Options = (); + + fn execute(input: Line, _options: &Self::Options) -> Line { + input + } + } + + impl NumericUnaryOpFamily for Copy { + type Options = (); + type Unary = Self; + } + + let tensor = self.clone(); + launch_unary_numeric::(tensor, |_| ()) + } + + /// Check if the tensor is safe to mutate. + pub fn can_mut(&self) -> bool { + self.handle.can_mut() + } + + /// Assert that both tensors are on the same device. + pub fn assert_is_on_same_device(&self, other: &Self) { + if self.device != other.device { + panic!( + "Both tensors should be on the same device {:?} != {:?}", + self.device, other.device + ); + } + } + + /// Check if the current tensor is contiguous. + /// + /// A tensor is contiguous if the elements are stored in memory + /// if the strides in non-increasing order and the + /// strides at position k is equal to the product of the shapes + /// at all positions greater than k. However, all axes with a shape of 1 are ignored. + pub fn is_contiguous(&self) -> bool { + is_contiguous(self.meta.shape(), self.meta.strides()) + } + + /// Check if the current tensor has a contiguous backing buffer (no overlap and no empty memory + /// regions within the shape). + pub fn is_contiguous_buffer(&self) -> bool { + self.meta.shape().num_elements() * self.dtype.size() == self.handle.size() as usize + } + + /// Checks if the tensor is non-overlapping (can be safely written to). + pub fn is_nonoverlapping(&self) -> bool { + let shape = self.meta.shape(); + let strides = self.meta.strides(); + + if strides.contains(&0) { + return false; + } + let rank = self.rank(); + if rank > 1 { + let mut dims = shape.iter().zip(strides.iter()).collect::>(); + dims.sort_by_key(|(_, stride)| **stride); + + let mut max_offset = 0; + for (shape, stride) in dims.into_iter() { + if *stride <= max_offset && *shape != 1 { + return false; + } + + max_offset += (*shape - 1) * *stride; + } + } + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_contiguous_non_increasing() { + assert!(is_contiguous(&[3, 1], &[1, 1])); + } + + #[test] + fn is_contiguous_basic() { + assert!(is_contiguous(&[32, 32], &[32, 1])); + } + + #[test] + fn is_contiguous_permuted() { + assert!(!is_contiguous(&[32, 32], &[1, 32])); + } + + #[test] + fn is_contiguous_slice() { + assert!(!is_contiguous(&[32, 1, 64], &[32, 64, 1])); + } + + #[test] + fn is_contiguous_4d_positive() { + assert!(is_contiguous(&[8, 256, 32, 32], &[262144, 1024, 32, 1])); + } + + #[test] + fn is_contiguous_4d_negative() { + assert!(!is_contiguous(&[256, 8, 32, 32], &[1024, 262144, 32, 1])); + } + + /// Based on a bug encountered in interpolate_1d + #[test] + fn is_contiguous_4d_unit_shape() { + assert!(!is_contiguous(&[1, 1, 1, 9], &[72, 1, 72, 8])); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/mod.rs new file mode 100644 index 0000000..013604d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod quantization; + +pub use base::*; +pub use quantization::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/quantization.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/quantization.rs new file mode 100644 index 0000000..c77b3ba --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tensor/quantization.rs @@ -0,0 +1,122 @@ +use burn_backend::{DType, Shape, TensorMetadata as _, quantization::QParamTensor}; +use burn_std::{Metadata, Strides}; +use cubecl::quant::scheme::{QuantStore, QuantValue}; +use cubecl::{client::ComputeClient, server::Handle}; + +use crate::CubeRuntime; + +use super::CubeTensor; + +/// Runtime parameters for quantization. Can be used to construct a scales handle from the base +/// tensor handle. +pub type QParams = burn_backend::quantization::QParams; + +impl CubeTensor { + /// Create a new quantized tensor + pub fn new_quantized( + client: ComputeClient, + handle: Handle, + shape: Shape, + device: R::Device, + strides: Strides, + dtype: DType, + qparams: QParams, + ) -> Self { + CubeTensor { + client, + handle, + meta: Box::new(Metadata::new(shape, strides)), + device, + dtype, + qparams: Some(qparams), + } + } + + /// Returns the two tensors: (values, params) for a quantized tensor. + /// For the values, native types that aren't supported as a normal `DType` will be returned + /// as an unsigned integer tensor representing the bits. Should be reconstructed using `from_bits` + /// in kernels. + pub fn quantized_handles(&self) -> Option<(CubeTensor, CubeTensor)> { + let params = self.scales()?; + let scheme = match self.dtype { + DType::QFloat(sc) => sc, + _ => return None, + }; + let values = match scheme.store { + QuantStore::Native => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S => CubeTensor { + client: self.client.clone(), + handle: self.handle.clone(), + meta: self.meta.clone(), + device: self.device.clone(), + dtype: DType::I8, + qparams: None, + }, + QuantValue::E4M3 | QuantValue::E5M2 => CubeTensor { + client: self.client.clone(), + handle: self.handle.clone(), + meta: self.meta.clone(), + device: self.device.clone(), + dtype: DType::U8, + qparams: None, + }, + QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E2M1 => { + panic!("Can't store native sub-byte values") + } + }, + QuantStore::PackedU32(packed_dim) => { + let packed_dim = self.rank() - packed_dim - 1; + let mut shape = self.shape(); + shape[packed_dim] = shape[packed_dim].div_ceil(scheme.num_quants()); + + CubeTensor { + client: self.client.clone(), + handle: self.handle.clone(), + meta: Box::new(Metadata::new(shape, self.meta.strides.clone())), + device: self.device.clone(), + dtype: DType::U32, + qparams: None, + } + } + QuantStore::PackedNative(packed_dim) => match scheme.value { + QuantValue::E2M1 => { + let packed_dim = self.rank() - packed_dim - 1; + let mut shape = self.shape(); + shape[packed_dim] = shape[packed_dim].div_ceil(scheme.num_quants()); + + CubeTensor { + client: self.client.clone(), + handle: self.handle.clone(), + meta: Box::new(Metadata::new(shape, self.meta.strides.clone())), + device: self.device.clone(), + dtype: DType::U8, + qparams: None, + } + } + other => panic!("{other:?} doesn't support native packing"), + }, + }; + + Some((values, params)) + } + + /// Construct a separate tensor for the quantization scales, if present + pub fn scales(&self) -> Option> { + let qparams = self.qparams.as_ref()?; + let mut handle = self.handle.clone(); + handle.offset_start = Some(qparams.scales.offset_start as u64); + handle.offset_end = Some(qparams.scales.offset_end as u64); + + Some(CubeTensor::new( + self.client.clone(), + handle, + qparams.scales.metadata.clone(), + self.device.clone(), + qparams.scales.dtype, + )) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tune_key.rs b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tune_key.rs new file mode 100644 index 0000000..d6892c7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cubecl/src/tune_key.rs @@ -0,0 +1,30 @@ +use crate::kernel::{ + conv::{ConvAutotuneKey, ConvTranspose2dAutotuneKey}, + reduce::SumAutotuneKey, +}; +use cubecl::tune::AutotuneKey; +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +#[derive(Hash, Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] +/// Key for all autotune-enabled operations +pub enum CubeAutotuneKey { + /// Key for sum operations + Sum(SumAutotuneKey), + /// Key for convolution operations + Conv(ConvAutotuneKey), + /// Key for transpose convolution operations + ConvTranspose(ConvTranspose2dAutotuneKey), +} + +impl Display for CubeAutotuneKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CubeAutotuneKey::Sum(reduce_key) => std::fmt::Debug::fmt(&reduce_key, f), + CubeAutotuneKey::Conv(conv_key) => std::fmt::Debug::fmt(&conv_key, f), + CubeAutotuneKey::ConvTranspose(conv_key) => std::fmt::Debug::fmt(&conv_key, f), + } + } +} + +impl AutotuneKey for CubeAutotuneKey {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cuda/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-cuda/Cargo.toml new file mode 100644 index 0000000..4be0644 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cuda/Cargo.toml @@ -0,0 +1,41 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "CUDA backend for the Burn framework" +documentation = "https://docs.rs/burn-cuda" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "gpu", "cuda"] +license.workspace = true +name = "burn-cuda" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-cuda" +version.workspace = true + +[lints] +workspace = true + +[features] +autotune = ["burn-cubecl/autotune"] +autotune-checks = ["burn-cubecl/autotune-checks"] +default = ["std", "fusion", "autotune", "burn-cubecl/default", "cubecl/default"] +doc = ["burn-cubecl/doc"] +fusion = ["burn-fusion", "burn-cubecl/fusion"] +std = ["burn-cubecl/std", "cubecl/std"] +tracing = [ + "burn-backend/tracing", + "burn-cubecl/tracing", + "burn-fusion?/tracing", + "cubecl/tracing", +] + +[dependencies] +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } +burn-cubecl = { path = "../burn-cubecl", version = "=0.21.0-pre.2", default-features = false } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false, features = [ + "cubecl-cuda", +] } +cubecl = { workspace = true, features = ["cuda"] } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cuda/README.md b/crates/stable-diffusion-burn/burn-crates/burn-cuda/README.md new file mode 100644 index 0000000..bf23978 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cuda/README.md @@ -0,0 +1,30 @@ +# Burn CUDA Backend + +[Burn](https://github.com/tracel-ai/burn) CUDA backend + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-cuda.svg)](https://crates.io/crates/burn-cuda) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-cuda/blob/master/README.md) + +This crate provides a CUDA backend for [Burn](https://github.com/tracel-ai/burn) using the +[cubecl](https://github.com/tracel-ai/cubecl.git) and [cudarc](https://github.com/coreylowman/cudarc.git) +crates. + +## Usage Example + +```rust +#[cfg(feature = "cuda")] +mod cuda { + use burn_autodiff::Autodiff; + use burn_cuda::{Cuda, CudaDevice}; + use mnist::training; + + pub fn run() { + let device = CudaDevice::default(); + training::run::>>(device); + } +} +``` + +## Dependencies + +Requires CUDA 12.x to be installed and on the `PATH`. \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-cuda/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-cuda/src/lib.rs new file mode 100644 index 0000000..43d8805 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-cuda/src/lib.rs @@ -0,0 +1,47 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] + +extern crate alloc; + +use burn_cubecl::CubeBackend; +pub use cubecl::cuda::CudaDevice; +use cubecl::cuda::CudaRuntime; + +#[cfg(not(feature = "fusion"))] +pub type Cuda = CubeBackend; + +#[cfg(feature = "fusion")] +pub type Cuda = burn_fusion::Fusion>; + +#[cfg(all(test, not(target_os = "macos")))] +mod tests { + use super::*; + use burn_backend::{Backend, DType, QTensorPrimitive}; + use burn_cubecl::tensor::CubeTensor; + + #[test] + fn should_support_dtypes() { + type B = Cuda; + let device = Default::default(); + + assert!(B::supports_dtype(&device, DType::F32)); + assert!(B::supports_dtype(&device, DType::Flex32)); + assert!(B::supports_dtype(&device, DType::F16)); + assert!(B::supports_dtype(&device, DType::BF16)); + assert!(B::supports_dtype(&device, DType::I64)); + assert!(B::supports_dtype(&device, DType::I32)); + assert!(B::supports_dtype(&device, DType::I16)); + assert!(B::supports_dtype(&device, DType::I8)); + assert!(B::supports_dtype(&device, DType::U64)); + assert!(B::supports_dtype(&device, DType::U32)); + assert!(B::supports_dtype(&device, DType::U16)); + assert!(B::supports_dtype(&device, DType::U8)); + assert!(B::supports_dtype(&device, DType::Bool)); + assert!(B::supports_dtype( + &device, + DType::QFloat(CubeTensor::::default_scheme()) + )); + + // Currently not registered in supported types + assert!(!B::supports_dtype(&device, DType::F64)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-dataset/Cargo.toml new file mode 100644 index 0000000..1f016dd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/Cargo.toml @@ -0,0 +1,84 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Library with simple dataset APIs for creating ML data pipelines" +documentation = "https://docs.rs/burn-dataset" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-dataset" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-dataset" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["sqlite-bundled"] +doc = ["default"] +tracing = [ + "burn-std/tracing", +] + +audio = ["hound"] +builtin-sources = ["vision", "dep:tar", "nlp"] +fake = ["dep:fake"] +network = ["dep:burn-std"] +sqlite = ["__sqlite-shared", "dep:rusqlite"] +sqlite-bundled = ["__sqlite-shared", "rusqlite/bundled"] +vision = ["dep:flate2", "dep:globwalk", "dep:image", "network"] +nlp = ["dep:zip", "dep:encoding_rs"] +# internal +__sqlite-shared = [ + "dep:r2d2", + "dep:r2d2_sqlite", + "dep:serde_rusqlite", + "dep:image", + "dep:gix-tempfile", +] +dataframe = ["dep:polars", "dep:planus"] + +[dependencies] +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", optional = true, features = [ + "network", +] } +csv = { workspace = true } +derive-new = { workspace = true } +dirs = { workspace = true } +fake = { workspace = true, optional = true } +flate2 = { workspace = true, optional = true } +gix-tempfile = { workspace = true, optional = true } +globwalk = { workspace = true, optional = true } +hound = { workspace = true, optional = true } +image = { workspace = true, optional = true } +planus = { workspace = true, optional = true } +encoding_rs = { workspace = true, optional = true } +polars = { workspace = true, optional = true } +r2d2 = { workspace = true, optional = true } +r2d2_sqlite = { workspace = true, optional = true } +rand = { workspace = true, features = ["std", "sys_rng"] } +zip = { workspace = true, optional = true } +rmp-serde = { workspace = true } +rusqlite = { workspace = true, optional = true } +sanitize-filename = { workspace = true } +serde = { workspace = true, features = ["std", "derive"] } +serde_json = { workspace = true, features = ["std"] } +serde_rusqlite = { workspace = true, optional = true } +strum = { workspace = true } +tar = { workspace = true, optional = true } +tempfile = { workspace = true } +thiserror = { workspace = true } + + +[dev-dependencies] +fake = { workspace = true } +rayon = { workspace = true } +rstest = { workspace = true } + +[package.metadata.cargo-udeps.ignore] +normal = ["strum", "strum_macros"] + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/README.md b/crates/stable-diffusion-burn/burn-crates/burn-dataset/README.md new file mode 100644 index 0000000..742aa23 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/README.md @@ -0,0 +1,17 @@ +# Burn Dataset + +> [Burn](https://github.com/tracel-ai/burn) dataset library + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-dataset.svg)](https://crates.io/crates/burn-dataset) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-dataset/blob/master/README.md) + +The Burn Dataset library is designed to streamline your machine learning (ML) data pipeline creation +process. It offers a variety of dataset implementations, transformation functions, and data sources. + +## Feature Flags + +- `audio` - enables audio dataset (SpeechCommandsDataset). Run the following example to try it out: + + ```shell + cargo run --example speech_commands --features audio + ``` diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/hf_dataset.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/hf_dataset.rs new file mode 100644 index 0000000..69748d5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/hf_dataset.rs @@ -0,0 +1,22 @@ +use burn_dataset::HuggingfaceDatasetLoader; +use burn_dataset::SqliteDataset; +use serde::Deserialize; + +#[derive(Deserialize, Debug, Clone)] +struct MnistItemRaw { + pub _image_bytes: Vec, + pub _label: usize, +} +fn main() { + // There are some datasets, such as https://huggingface.co/datasets/ylecun/mnist/tree/main that contains a script, + // In this cases you must enable trusting remote code execution if you want to use it. + let _train_ds: SqliteDataset = HuggingfaceDatasetLoader::new("mnist") + .with_trust_remote_code(true) + .dataset("train") + .unwrap(); + + // However not all dataset requires it https://huggingface.co/datasets/Anthropic/hh-rlhf/tree/main + let _train_ds: SqliteDataset = HuggingfaceDatasetLoader::new("Anthropic/hh-rlhf") + .dataset("train") + .unwrap(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/speech_commands.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/speech_commands.rs new file mode 100644 index 0000000..7efec10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/examples/speech_commands.rs @@ -0,0 +1,23 @@ +#[cfg(feature = "audio")] +use burn_dataset::{Dataset, audio::SpeechCommandsDataset}; + +#[cfg(feature = "audio")] +fn speech_command() { + let index: usize = 4835; + let test = SpeechCommandsDataset::test(); + let item = test.get(index).unwrap(); + + println!("Item: {:?}", item); + println!("Item Length: {:?}", item.audio_samples.len()); + println!("Label: {}", item.label); + + assert_eq!(test.len(), 4890); + assert_eq!(item.label.to_string(), "Yes"); + assert_eq!(item.sample_rate, 16000); + assert_eq!(item.audio_samples.len(), 16000); +} + +fn main() { + #[cfg(feature = "audio")] + speech_command() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/mod.rs new file mode 100644 index 0000000..5d357e9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/mod.rs @@ -0,0 +1,3 @@ +mod speech_commands; + +pub use speech_commands::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/speech_commands.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/speech_commands.rs new file mode 100644 index 0000000..8e6cdaf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/audio/speech_commands.rs @@ -0,0 +1,208 @@ +use crate::{ + Dataset, HuggingfaceDatasetLoader, SqliteDataset, + transform::{Mapper, MapperDataset}, +}; + +use hound::WavReader; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumCount, FromRepr}; + +type MappedDataset = MapperDataset, ConvertSamples, SpeechItemRaw>; + +/// Enum representing speech command classes in the Speech Commands dataset. +/// Class names are based on the Speech Commands dataset from Huggingface. +/// See [speech_commands](https://huggingface.co/datasets/speech_commands) +/// for more information. +#[allow(missing_docs)] +#[derive(Debug, Display, Clone, Copy, FromRepr, Serialize, Deserialize, EnumCount)] +pub enum SpeechCommandClass { + // Target command words + Yes = 0, + No = 1, + Up = 2, + Down = 3, + Left = 4, + Right = 5, + On = 6, + Off = 7, + Stop = 8, + Go = 9, + Zero = 10, + One = 11, + Two = 12, + Three = 13, + Four = 14, + Five = 15, + Six = 16, + Seven = 17, + Eight = 18, + Nine = 19, + + // Non-target words that can be grouped into "Other" + Bed = 20, + Bird = 21, + Cat = 22, + Dog = 23, + Happy = 24, + House = 25, + Marvin = 26, + Sheila = 27, + Tree = 28, + Wow = 29, + + // Commands from v2 dataset, that can be grouped into "Other" + Backward = 30, + Forward = 31, + Follow = 32, + Learn = 33, + Visual = 34, + + // Background noise + Silence = 35, + + // Other miscellaneous words + Other = 36, +} + +/// Struct containing raw speech data returned from a database. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SpeechItemRaw { + /// Audio file bytes. + pub audio_bytes: Vec, + + /// Label index. + pub label: usize, + + /// Indicates if the label is unknown. + pub is_unknown: bool, +} + +/// Speech item with audio samples and label. +/// +/// The audio samples are floats in the range [-1.0, 1.0]. +/// The sample rate is in Hz. +/// The label is the class index (see [SpeechCommandClass]). +/// To convert to usize simply use `as usize`. To convert label to string use `.to_string()`. +/// +/// The original label is also stored in the `label_original` field for debugging and remapping if needed. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SpeechItem { + /// Audio samples in the range [-1.0, 1.0]. + pub audio_samples: Vec, + + /// The sample rate of the audio. + pub sample_rate: usize, + + /// The label of the audio. + pub label: SpeechCommandClass, +} + +/// Speech Commands dataset from Huggingface v0.02. +/// See [Speech Commands dataset](https://huggingface.co/datasets/speech_commands). +/// +/// The data is downloaded from Huggingface and stored in a SQLite database (3.0 GB). +/// The dataset contains 99,720 audio samples of 2,607 people saying 35 different words. +/// +/// NOTE: The most samples are under 1 second long but there are some with pure background noise that +/// need splitting into shorter segmants. +/// +/// The labels are 20 target words, silence and other words. +/// +/// The dataset is split into 3 parts: +/// - train: 84,848 audio files +/// - test: 4,890 audio files +/// - validation: 9,982 audio files +pub struct SpeechCommandsDataset { + dataset: MappedDataset, +} + +impl SpeechCommandsDataset { + /// Create a new dataset with the given split. + pub fn new(split: &str) -> Self { + let dataset: SqliteDataset = + HuggingfaceDatasetLoader::new("speech_commands") + .with_subset("v0.02") + .dataset(split) + .unwrap(); + let dataset = MapperDataset::new(dataset, ConvertSamples); + Self { dataset } + } + + /// Create a new dataset with the train split. + pub fn train() -> Self { + Self::new("train") + } + + /// Create a new dataset with the test split. + pub fn test() -> Self { + Self::new("test") + } + + /// Create a new dataset with the validation split. + pub fn validation() -> Self { + Self::new("validation") + } + + /// Returns the number of classes in the dataset + pub fn num_classes() -> usize { + SpeechCommandClass::COUNT + } +} + +impl Dataset for SpeechCommandsDataset { + fn get(&self, index: usize) -> Option { + self.dataset.get(index) + } + + fn len(&self) -> usize { + self.dataset.len() + } +} + +/// Mapper converting audio bytes into audio samples and the label to enum class. +struct ConvertSamples; + +impl ConvertSamples { + /// Convert label to enum class. + fn to_speechcommandclass(label: usize) -> SpeechCommandClass { + SpeechCommandClass::from_repr(label).unwrap() + } + + /// Convert audio bytes into samples of floats [-1.0, 1.0]. + fn to_audiosamples(bytes: &Vec) -> (Vec, usize) { + let reader = WavReader::new(bytes.as_slice()).unwrap(); + let spec = reader.spec(); + + // Maximum value of the audio samples (using bit shift to raise 2 to the power of bits per sample). + let max_value = (1 << (spec.bits_per_sample - 1)) as f32; + + // The sample rate of the audio. + let sample_rate = spec.sample_rate as usize; + + // Convert the audio samples to floats [-1.0, 1.0]. + let audio_samples: Vec = reader + .into_samples::() + .filter_map(Result::ok) + .map(|sample| sample as f32 / max_value) + .collect(); + + (audio_samples, sample_rate) + } +} + +impl Mapper for ConvertSamples { + /// Convert audio bytes into samples of floats [-1.0, 1.0] + /// and the label to enum class with the target word, other and silence classes. + fn map(&self, item: &SpeechItemRaw) -> SpeechItem { + let (audio_samples, sample_rate) = Self::to_audiosamples(&item.audio_bytes); + + // Convert the label to enum class, with the target words, other and silence classes. + let label = Self::to_speechcommandclass(item.label); + + SpeechItem { + audio_samples, + sample_rate, + label, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/base.rs new file mode 100644 index 0000000..eb53980 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/base.rs @@ -0,0 +1,71 @@ +use std::sync::Arc; + +use crate::DatasetIterator; + +/// The dataset trait defines a basic collection of items with a predefined size. +pub trait Dataset: Send + Sync { + /// Gets the item at the given index. + fn get(&self, index: usize) -> Option; + + /// Gets the number of items in the dataset. + fn len(&self) -> usize; + + /// Checks if the dataset is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns an iterator over the dataset. + fn iter(&self) -> DatasetIterator<'_, I> + where + Self: Sized, + { + DatasetIterator::new(self) + } +} + +impl Dataset for Arc +where + D: Dataset, +{ + fn get(&self, index: usize) -> Option { + self.as_ref().get(index) + } + + fn len(&self) -> usize { + self.as_ref().len() + } +} + +impl Dataset for Arc> { + fn get(&self, index: usize) -> Option { + self.as_ref().get(index) + } + + fn len(&self) -> usize { + self.as_ref().len() + } +} + +impl Dataset for Box +where + D: Dataset, +{ + fn get(&self, index: usize) -> Option { + self.as_ref().get(index) + } + + fn len(&self) -> usize { + self.as_ref().len() + } +} + +impl Dataset for Box> { + fn get(&self, index: usize) -> Option { + self.as_ref().get(index) + } + + fn len(&self) -> usize { + self.as_ref().len() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/dataframe.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/dataframe.rs new file mode 100644 index 0000000..16ae43b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/dataframe.rs @@ -0,0 +1,465 @@ +use std::marker::PhantomData; + +use crate::Dataset; + +use polars::frame::row::Row; +use polars::prelude::*; +use serde::de::DeserializeSeed; +use serde::{ + Deserialize, + de::{self, DeserializeOwned, Deserializer, SeqAccess, Visitor}, + forward_to_deserialize_any, +}; + +/// Error type for DataframeDataset +#[derive(thiserror::Error, Debug)] +pub enum DataframeDatasetError { + /// Error occurred during deserialization or other operations + #[error("{0}")] + Other(String), +} + +impl de::Error for DataframeDatasetError { + fn custom(msg: T) -> Self { + DataframeDatasetError::Other(msg.to_string()) + } +} + +/// Dataset implementation for Polars DataFrame +/// +/// This struct provides a way to access data from a Polars DataFrame +/// as if it were a Dataset of type I. +pub struct DataframeDataset { + df: DataFrame, + len: usize, + column_name_mapping: Vec, + phantom: PhantomData, +} + +impl DataframeDataset +where + I: Clone + Send + Sync + DeserializeOwned, +{ + /// Create a new DataframeDataset from a Polars DataFrame + /// + /// # Arguments + /// + /// * `df` - A Polars DataFrame + /// + /// # Returns + /// + /// A Result containing the new DataframeDataset or a DataframeDatasetError + pub fn new(df: DataFrame) -> Result { + let len = df.height(); + let field_names = extract_field_names::(); + + let column_name_mapping = field_names + .iter() + .map(|name| { + df.schema() + .try_get_full(name) + .expect("Corresponding column should exist in the DataFrame") + .0 + }) + .collect::>(); + + Ok(DataframeDataset { + df, + len, + column_name_mapping, + phantom: PhantomData, + }) + } +} + +impl Dataset for DataframeDataset +where + I: Clone + Send + Sync + DeserializeOwned, +{ + /// Get an item from the dataset at the specified index + /// + /// # Arguments + /// + /// * `index` - The index of the item to retrieve + /// + /// # Returns + /// + /// An Option containing the item if it exists, or None if it doesn't + fn get(&self, index: usize) -> Option { + let row = self.df.get_row(index).ok()?; + + let mut deserializer = RowDeserializer::new(&row, &self.column_name_mapping); + I::deserialize(&mut deserializer).ok() + } + + /// Get the length of the dataset + fn len(&self) -> usize { + self.len + } + + /// Check if the dataset is empty + fn is_empty(&self) -> bool { + self.len == 0 + } +} + +/// A deserializer for Polars DataFrame rows +struct RowDeserializer<'a> { + row: &'a Row<'a>, + column_name_mapping: &'a Vec, + index: usize, +} + +impl<'a> RowDeserializer<'a> { + /// Create a new RowDeserializer + /// + /// # Arguments + /// + /// * `row` - A reference to a Polars DataFrame row + /// * `column_name_mapping` - A reference to a vector mapping field names to column indices + fn new(row: &'a Row, column_name_mapping: &'a Vec) -> RowDeserializer<'a> { + RowDeserializer { + row, + column_name_mapping, + index: 0, + } + } +} + +impl<'de, 'a> Deserializer<'de> for &'a mut RowDeserializer<'a> { + type Error = DataframeDatasetError; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + let i = self.column_name_mapping[self.index]; + + let value = &self.row.0[i]; + match value { + AnyValue::Null => visitor.visit_none(), + AnyValue::Boolean(b) => visitor.visit_bool(*b), + AnyValue::Int8(i) => visitor.visit_i8(*i), + AnyValue::Int16(i) => visitor.visit_i16(*i), + AnyValue::Int32(i) => visitor.visit_i32(*i), + AnyValue::Int64(i) => visitor.visit_i64(*i), + AnyValue::UInt8(i) => visitor.visit_u8(*i), + AnyValue::UInt16(i) => visitor.visit_u16(*i), + AnyValue::UInt32(i) => visitor.visit_u32(*i), + AnyValue::UInt64(i) => visitor.visit_u64(*i), + AnyValue::Float32(f) => visitor.visit_f32(*f), + AnyValue::Float64(f) => visitor.visit_f64(*f), + AnyValue::Date(i) => visitor.visit_i32(*i), + AnyValue::String(s) => visitor.visit_string(s.to_string()), + AnyValue::Binary(b) => { + visitor.visit_seq(de::value::SeqDeserializer::new(b.iter().copied())) + } + AnyValue::Time(t) => visitor.visit_i64(*t), + ty => Err(DataframeDatasetError::Other( + format!("Unsupported type: {ty:?}").to_string(), + )), + } + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(self) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string + bytes byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map enum identifier ignored_any + } +} + +impl<'de, 'a> SeqAccess<'de> for RowDeserializer<'a> { + type Error = DataframeDatasetError; + + fn next_element_seed(&mut self, seed: T) -> Result, DataframeDatasetError> + where + T: DeserializeSeed<'de>, + { + if self.index >= self.row.0.len() { + return Ok(None); + } + let mut deserializer = RowDeserializer { + row: self.row, + column_name_mapping: self.column_name_mapping, + index: self.index, + }; + self.index += 1; + seed.deserialize(&mut deserializer).map(Some) + } +} + +struct FieldExtractor { + fields: Vec<&'static str>, +} + +impl<'de> Deserializer<'de> for &mut FieldExtractor { + type Error = de::value::Error; + + fn deserialize_any(self, _visitor: V) -> core::result::Result + where + V: Visitor<'de>, + { + Err(de::Error::custom("Field extractor")) + } + + fn deserialize_struct( + self, + _name: &'static str, + fields: &'static [&'static str], + _visitor: V, + ) -> core::result::Result + where + V: Visitor<'de>, + { + self.fields.extend_from_slice(fields); + Err(de::Error::custom("Field extractor")) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string bytes + byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map enum identifier ignored_any + } +} + +/// Extract field names from a type T that implements Deserialize +/// +/// # Returns +/// +/// A vector of field names as static string slices +fn extract_field_names<'de, T>() -> Vec<&'static str> +where + T: Deserialize<'de>, +{ + let mut extractor = FieldExtractor { fields: Vec::new() }; + let _ = T::deserialize(&mut extractor); + extractor.fields +} + +#[cfg(test)] +mod tests { + use polars::prelude::*; + use serde::Deserialize; + + use super::*; + #[derive(Clone, Debug, Deserialize, PartialEq)] + struct TestData { + int32: i32, + bool: bool, + float64: f64, + string: String, + int16: i16, + uint32: u32, + uint64: u64, + float32: f32, + int64: i64, + int8: i8, + binary: Vec, + } + + fn create_test_dataframe() -> DataFrame { + let s0 = Column::new("int32".into(), &[1i32, 2i32, 3i32]); + let s1 = Column::new("bool".into(), &[true, false, true]); + let s2 = Column::new("float64".into(), &[1.1f64, 2.2f64, 3.3f64]); + let s3 = Column::new("string".into(), &["Boo", "Boo2", "Boo3"]); + let s6 = Column::new("int16".into(), &[1i16, 2i16, 3i16]); + let s8 = Column::new("uint32".into(), &[1u32, 2u32, 3u32]); + let s9 = Column::new("uint64".into(), &[1u64, 2u64, 3u64]); + let s10 = Column::new("float32".into(), &[1.1f32, 2.2f32, 3.3f32]); + let s11 = Column::new("int64".into(), &[1i64, 2i64, 3i64]); + let s12 = Column::new("int8".into(), &[1i8, 2i8, 3i8]); + + let binary_data: Vec<&[u8]> = vec![&[1, 2, 3], &[4, 5, 6], &[7, 8, 9]]; + + let s13 = Column::new("binary".into(), binary_data); + DataFrame::new_infer_height(vec![s0, s1, s2, s3, s6, s8, s9, s10, s11, s12, s13]).unwrap() + } + + #[test] + fn test_dataframe_dataset_creation() { + let df = create_test_dataframe(); + let dataset = DataframeDataset::::new(df); + assert!(dataset.is_ok()); + } + + #[test] + fn test_dataframe_dataset_length() { + let df = create_test_dataframe(); + let dataset = DataframeDataset::::new(df).unwrap(); + assert_eq!(dataset.len(), 3); + assert!(!dataset.is_empty()); + } + + #[test] + fn test_dataframe_dataset_get() { + let df = create_test_dataframe(); + let dataset = DataframeDataset::::new(df).unwrap(); + + let expected_items = vec![ + TestData { + int32: 1, + bool: true, + float64: 1.1, + string: "Boo".to_string(), + int16: 1, + uint32: 1, + uint64: 1, + float32: 1.1, + int64: 1, + int8: 1, + binary: vec![1, 2, 3], + }, + TestData { + int32: 2, + bool: false, + float64: 2.2, + string: "Boo2".to_string(), + int16: 2, + uint32: 2, + uint64: 2, + float32: 2.2, + int64: 2, + int8: 2, + binary: vec![4, 5, 6], + }, + TestData { + int32: 3, + bool: true, + float64: 3.3, + string: "Boo3".to_string(), + int16: 3, + uint32: 3, + uint64: 3, + float32: 3.3, + int64: 3, + int8: 3, + binary: vec![7, 8, 9], + }, + ]; + + for (index, expected_item) in expected_items.iter().enumerate() { + let item = dataset.get(index).unwrap(); + assert_eq!(&item, expected_item); + } + } + + #[test] + fn test_dataframe_dataset_out_of_bounds() { + let df = create_test_dataframe(); + let dataset = DataframeDataset::::new(df).unwrap(); + assert!(dataset.get(3).is_none()); + } + + #[test] + fn test_dataframe_dataset() { + let df = create_test_dataframe(); + let dataset: DataframeDataset = DataframeDataset::new(df).unwrap(); + + assert_eq!(dataset.len(), 3); + assert!(!dataset.is_empty()); + + let item = dataset.get(1).unwrap(); + assert_eq!( + item, + TestData { + int32: 2, + bool: false, + float64: 2.2, + string: "Boo2".to_string(), + int16: 2, + uint32: 2, + uint64: 2, + float32: 2.2, + int64: 2, + int8: 2, + binary: vec![4, 5, 6], + } + ); + + let item = dataset.get(2).unwrap(); + + assert_eq!( + item, + TestData { + int32: 3, + bool: true, + float64: 3.3, + string: "Boo3".to_string(), + int16: 3, + uint32: 3, + uint64: 3, + float32: 3.3, + int64: 3, + int8: 3, + binary: vec![7, 8, 9], + } + ); + } + + #[test] + #[should_panic = "Corresponding column should exist in the DataFrame: SchemaFieldNotFound(ErrString(\"non_existent\"))"] + fn test_non_existing_struct_fields() { + #[derive(Clone, Debug, Deserialize, PartialEq)] + struct PartialTestData { + int32: i32, + bool: bool, + non_existent: String, + } + + let df = create_test_dataframe(); + let dataset = DataframeDataset::::new(df); + + assert!(dataset.is_err()); + if let Err(e) = dataset { + assert!(matches!(e, DataframeDatasetError::Other(_))); + } + } + + #[test] + fn test_partial_table() { + #[derive(Clone, Debug, Deserialize, PartialEq)] + struct PartialTestData { + int32: i32, + bool: bool, + string: String, + } + + let df = create_test_dataframe(); + let dataset = DataframeDataset::::new(df).unwrap(); + + assert_eq!(dataset.len(), 3); + assert!(!dataset.is_empty()); + + let item = dataset.get(1).unwrap(); + assert_eq!( + item, + PartialTestData { + int32: 2, + bool: false, + string: "Boo2".to_string(), + } + ); + + let item = dataset.get(2).unwrap(); + assert_eq!( + item, + PartialTestData { + int32: 3, + bool: true, + string: "Boo3".to_string(), + } + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/fake.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/fake.rs new file mode 100644 index 0000000..c27f8cf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/fake.rs @@ -0,0 +1,38 @@ +use crate::{Dataset, DatasetIterator, InMemDataset}; +use fake::{Dummy, Fake, Faker}; + +/// Dataset filled with fake items generated from the [fake](fake) crate. +pub struct FakeDataset { + dataset: InMemDataset, +} + +impl> FakeDataset { + /// Create a new fake dataset with the given size. + pub fn new(size: usize) -> Self { + let mut items = Vec::with_capacity(size); + for _ in 0..size { + items.push(Faker.fake()); + } + let dataset = InMemDataset::new(items); + + Self { dataset } + } +} + +impl Dataset for FakeDataset { + fn iter(&self) -> DatasetIterator<'_, I> { + DatasetIterator::new(self) + } + + fn get(&self, index: usize) -> Option { + self.dataset.get(index) + } + + fn len(&self) -> usize { + self.dataset.len() + } + + fn is_empty(&self) -> bool { + self.dataset.is_empty() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/in_memory.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/in_memory.rs new file mode 100644 index 0000000..13854cd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/in_memory.rs @@ -0,0 +1,192 @@ +use std::{ + fs::File, + io::{BufRead, BufReader}, + path::Path, +}; + +use serde::de::DeserializeOwned; + +use crate::Dataset; + +/// Dataset where all items are stored in ram. +pub struct InMemDataset { + items: Vec, +} + +impl InMemDataset { + /// Creates a new in memory dataset from the given items. + pub fn new(items: Vec) -> Self { + InMemDataset { items } + } +} + +impl Dataset for InMemDataset +where + I: Clone + Send + Sync, +{ + fn get(&self, index: usize) -> Option { + self.items.get(index).cloned() + } + fn len(&self) -> usize { + self.items.len() + } +} + +impl InMemDataset +where + I: Clone + DeserializeOwned, +{ + /// Create from a dataset. All items are loaded in memory. + pub fn from_dataset(dataset: &impl Dataset) -> Self { + let items: Vec = dataset.iter().collect(); + Self::new(items) + } + + /// Create from a json rows file (one json per line). + /// + /// [Supported field types](https://docs.rs/serde_json/latest/serde_json/value/enum.Value.html) + pub fn from_json_rows>(path: P) -> Result { + let file = File::open(path)?; + let reader = BufReader::new(file); + let mut items = Vec::new(); + + for line in reader.lines() { + let item = serde_json::from_str(line.unwrap().as_str()).unwrap(); + items.push(item); + } + + let dataset = Self::new(items); + + Ok(dataset) + } + + /// Create from a csv file. + /// + /// The provided `csv::ReaderBuilder` can be configured to fit your csv format. + /// + /// The supported field types are: String, integer, float, and bool. + /// + /// See: + /// - [Reading with Serde](https://docs.rs/csv/latest/csv/tutorial/index.html#reading-with-serde) + /// - [Delimiters, quotes and variable length records](https://docs.rs/csv/latest/csv/tutorial/index.html#delimiters-quotes-and-variable-length-records) + pub fn from_csv>( + path: P, + builder: &csv::ReaderBuilder, + ) -> Result { + let mut rdr = builder.from_path(path)?; + + let mut items = Vec::new(); + + for result in rdr.deserialize() { + let item: I = result?; + items.push(item); + } + + let dataset = Self::new(items); + + Ok(dataset) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::{SqliteDataset, test_data}; + + use rstest::{fixture, rstest}; + use serde::{Deserialize, Serialize}; + + const DB_FILE: &str = "tests/data/sqlite-dataset.db"; + const JSON_FILE: &str = "tests/data/dataset.json"; + const CSV_FILE: &str = "tests/data/dataset.csv"; + const CSV_FMT_FILE: &str = "tests/data/dataset-fmt.csv"; + + type SqlDs = SqliteDataset; + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] + pub struct Sample { + column_str: String, + column_bytes: Vec, + column_int: i64, + column_bool: bool, + column_float: f64, + } + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] + pub struct SampleCsv { + column_str: String, + column_int: i64, + column_bool: bool, + column_float: f64, + } + + #[fixture] + fn train_dataset() -> SqlDs { + SqliteDataset::from_db_file(DB_FILE, "train").unwrap() + } + + #[rstest] + pub fn from_dataset(train_dataset: SqlDs) { + let dataset = InMemDataset::from_dataset(&train_dataset); + + let non_existing_record_index: usize = 10; + let record_index: usize = 0; + + assert_eq!(train_dataset.get(non_existing_record_index), None); + assert_eq!(dataset.get(record_index).unwrap().column_str, "HI1"); + } + + #[test] + pub fn from_json_rows() { + let dataset = InMemDataset::::from_json_rows(JSON_FILE).unwrap(); + + let non_existing_record_index: usize = 10; + let record_index: usize = 1; + + assert_eq!(dataset.get(non_existing_record_index), None); + assert_eq!(dataset.get(record_index).unwrap().column_str, "HI2"); + assert!(!dataset.get(record_index).unwrap().column_bool); + } + + #[test] + pub fn from_csv_rows() { + let rdr = csv::ReaderBuilder::new(); + let dataset = InMemDataset::::from_csv(CSV_FILE, &rdr).unwrap(); + + let non_existing_record_index: usize = 10; + let record_index: usize = 1; + + assert_eq!(dataset.get(non_existing_record_index), None); + assert_eq!(dataset.get(record_index).unwrap().column_str, "HI2"); + assert_eq!(dataset.get(record_index).unwrap().column_int, 1); + assert!(!dataset.get(record_index).unwrap().column_bool); + assert_eq!(dataset.get(record_index).unwrap().column_float, 1.0); + } + + #[test] + pub fn from_csv_rows_fmt() { + let mut rdr = csv::ReaderBuilder::new(); + let rdr = rdr.delimiter(b' ').has_headers(false); + let dataset = InMemDataset::::from_csv(CSV_FMT_FILE, rdr).unwrap(); + + let non_existing_record_index: usize = 10; + let record_index: usize = 1; + + assert_eq!(dataset.get(non_existing_record_index), None); + assert_eq!(dataset.get(record_index).unwrap().column_str, "HI2"); + assert_eq!(dataset.get(record_index).unwrap().column_int, 1); + assert!(!dataset.get(record_index).unwrap().column_bool); + assert_eq!(dataset.get(record_index).unwrap().column_float, 1.0); + } + + #[test] + pub fn given_in_memory_dataset_when_iterate_should_iterate_though_all_items() { + let items_original = test_data::string_items(); + let dataset = InMemDataset::new(items_original.clone()); + + let items: Vec = dataset.iter().collect(); + + assert_eq!(items_original, items); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/iterator.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/iterator.rs new file mode 100644 index 0000000..cbea476 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/iterator.rs @@ -0,0 +1,31 @@ +use crate::dataset::Dataset; +use std::iter::Iterator; + +/// Dataset iterator. +pub struct DatasetIterator<'a, I> { + current: usize, + dataset: &'a dyn Dataset, +} + +impl<'a, I> DatasetIterator<'a, I> { + /// Creates a new dataset iterator. + pub fn new(dataset: &'a D) -> Self + where + D: Dataset, + { + DatasetIterator { + current: 0, + dataset, + } + } +} + +impl Iterator for DatasetIterator<'_, I> { + type Item = I; + + fn next(&mut self) -> Option { + let item = self.dataset.get(self.current); + self.current += 1; + item + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/mod.rs new file mode 100644 index 0000000..9d9061e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/mod.rs @@ -0,0 +1,25 @@ +mod base; +mod in_memory; +mod iterator; + +pub use base::*; +pub use in_memory::*; +pub use iterator::*; + +#[cfg(any(test, feature = "fake"))] +mod fake; + +#[cfg(any(test, feature = "fake"))] +pub use self::fake::*; + +#[cfg(feature = "dataframe")] +mod dataframe; + +#[cfg(feature = "dataframe")] +pub use dataframe::*; + +#[cfg(any(feature = "sqlite", feature = "sqlite-bundled"))] +pub use sqlite::*; + +#[cfg(any(feature = "sqlite", feature = "sqlite-bundled"))] +mod sqlite; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/sqlite.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/sqlite.rs new file mode 100644 index 0000000..a3cca83 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/dataset/sqlite.rs @@ -0,0 +1,851 @@ +use std::{ + collections::HashSet, + fs, io, + marker::PhantomData, + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; + +use crate::Dataset; + +use gix_tempfile::{ + AutoRemove, ContainingDirectory, Handle, + handle::{Writable, persist}, +}; +use r2d2::{Pool, PooledConnection}; +use r2d2_sqlite::{ + SqliteConnectionManager, + rusqlite::{OpenFlags, OptionalExtension}, +}; +use sanitize_filename::sanitize; +use serde::{Serialize, de::DeserializeOwned}; +use serde_rusqlite::{columns_from_statement, from_row_with_columns}; + +/// Result type for the sqlite dataset. +pub type Result = core::result::Result; + +/// Sqlite dataset error. +#[derive(thiserror::Error, Debug)] +pub enum SqliteDatasetError { + /// IO related error. + #[error("IO error: {0}")] + Io(#[from] io::Error), + + /// Sql related error. + #[error("Sql error: {0}")] + Sql(#[from] serde_rusqlite::rusqlite::Error), + + /// Serde related error. + #[error("Serde error: {0}")] + Serde(#[from] rmp_serde::encode::Error), + + /// The database file already exists error. + #[error("Overwrite flag is set to false and the database file already exists: {0}")] + FileExists(PathBuf), + + /// Error when creating the connection pool. + #[error("Failed to create connection pool: {0}")] + ConnectionPool(#[from] r2d2::Error), + + /// Error when persisting the temporary database file. + #[error("Could not persist the temporary database file: {0}")] + PersistDbFile(#[from] persist::Error), + + /// Any other error. + #[error("{0}")] + Other(&'static str), +} + +impl From<&'static str> for SqliteDatasetError { + fn from(s: &'static str) -> Self { + SqliteDatasetError::Other(s) + } +} + +/// This struct represents a dataset where all items are stored in an SQLite database. +/// Each instance of this struct corresponds to a specific table within the SQLite database, +/// and allows for interaction with the data stored in the table in a structured and typed manner. +/// +/// The SQLite database must contain a table with the same name as the `split` field. This table should +/// have a primary key column named `row_id`, which is used to index the rows in the table. The `row_id` +/// should start at 1, while the corresponding dataset `index` should start at 0, i.e., `row_id` = `index` + 1. +/// +/// Table columns can be represented in two ways: +/// +/// 1. The table can have a column for each field in the `I` struct. In this case, the column names in the table +/// should match the field names of the `I` struct. The field names can be a subset of column names and +/// can be in any order. +/// +/// For the supported field types, refer to: +/// - [Serialization field types](https://docs.rs/serde_rusqlite/latest/serde_rusqlite) +/// - [SQLite data types](https://www.sqlite.org/datatype3.html) +/// +/// 2. The fields in the `I` struct can be serialized into a single column `item` in the table. In this case, the table +/// should have a single column named `item` of type `BLOB`. This is useful when the `I` struct contains complex fields +/// that cannot be mapped to a SQLite type, such as nested structs, vectors, etc. The serialization is done using +/// [MessagePack](https://msgpack.org/). +/// +/// Note: The code automatically figures out which of the above two cases is applicable, and uses the appropriate +/// method to read the data from the table. +#[derive(Debug)] +pub struct SqliteDataset { + db_file: PathBuf, + split: String, + conn_pool: Pool, + columns: Vec, + len: usize, + select_statement: String, + row_serialized: bool, + phantom: PhantomData, +} + +impl SqliteDataset { + /// Initializes a `SqliteDataset` from a SQLite database file and a split name. + pub fn from_db_file>(db_file: P, split: &str) -> Result { + // Create a connection pool + let conn_pool = create_conn_pool(&db_file, false)?; + + // Determine how the table is stored + let row_serialized = Self::check_if_row_serialized(&conn_pool, split)?; + + // Create a select statement and save it + let select_statement = if row_serialized { + format!("select item from {split} where row_id = ?") + } else { + format!("select * from {split} where row_id = ?") + }; + + // Save the column names and the number of rows + let (columns, len) = fetch_columns_and_len(&conn_pool, &select_statement, split)?; + + Ok(SqliteDataset { + db_file: db_file.as_ref().to_path_buf(), + split: split.to_string(), + conn_pool, + columns, + len, + select_statement, + row_serialized, + phantom: PhantomData, + }) + } + + /// Returns true if table has two columns: row_id (integer) and item (blob). + /// + /// This is used to determine if the table is row serialized or not. + fn check_if_row_serialized( + conn_pool: &Pool, + split: &str, + ) -> Result { + // This struct is used to store the column name and type + struct Column { + name: String, + ty: String, + } + + const COLUMN_NAME: usize = 1; + const COLUMN_TYPE: usize = 2; + + let sql_statement = format!("PRAGMA table_info({split})"); + + let conn = conn_pool.get()?; + + let mut stmt = conn.prepare(sql_statement.as_str())?; + let column_iter = stmt.query_map([], |row| { + Ok(Column { + name: row + .get::(COLUMN_NAME) + .unwrap() + .to_lowercase(), + ty: row + .get::(COLUMN_TYPE) + .unwrap() + .to_lowercase(), + }) + })?; + + let mut columns: Vec = vec![]; + + for column in column_iter { + columns.push(column?); + } + + if columns.len() != 2 { + Ok(false) + } else { + // Check if the column names and types match the expected values + Ok(columns[0].name == "row_id" + && columns[0].ty == "integer" + && columns[1].name == "item" + && columns[1].ty == "blob") + } + } + + /// Get the database file name. + pub fn db_file(&self) -> PathBuf { + self.db_file.clone() + } + + /// Get the split name. + pub fn split(&self) -> &str { + self.split.as_str() + } +} + +impl Dataset for SqliteDataset +where + I: Clone + Send + Sync + DeserializeOwned, +{ + /// Get an item from the dataset. + fn get(&self, index: usize) -> Option { + // Row ids start with 1 (one) and index starts with 0 (zero) + let row_id = index + 1; + + // Get a connection from the pool + let connection = self.conn_pool.get().unwrap(); + let mut statement = connection.prepare(self.select_statement.as_str()).unwrap(); + + if self.row_serialized { + // Fetch with a single column `item` and deserialize it with MessagePack + statement + .query_row([row_id], |row| { + // Deserialize item (blob) with MessagePack (rmp-serde) + Ok( + rmp_serde::from_slice::(row.get_ref(0).unwrap().as_blob().unwrap()) + .unwrap(), + ) + }) + .optional() //Converts Error (not found) to None + .unwrap() + } else { + // Fetch a row with multiple columns and deserialize it serde_rusqlite + statement + .query_row([row_id], |row| { + // Deserialize the row with serde_rusqlite + Ok(from_row_with_columns::(row, &self.columns).unwrap()) + }) + .optional() //Converts Error (not found) to None + .unwrap() + } + } + + /// Return the number of rows in the dataset. + fn len(&self) -> usize { + self.len + } +} + +/// Fetch the column names and the number of rows from the database. +fn fetch_columns_and_len( + conn_pool: &Pool, + select_statement: &str, + split: &str, +) -> Result<(Vec, usize)> { + // Save the column names + let connection = conn_pool.get()?; + let statement = connection.prepare(select_statement)?; + let columns = columns_from_statement(&statement); + + // Count the number of rows and save it as len + // + // NOTE: Using coalesce(max(row_id), 0) instead of count(*) because count(*) is super slow for large tables. + // The coalesce(max(row_id), 0) returns 0 if the table is empty, otherwise it returns the max row_id, + // which corresponds to the number of rows in the table. + // The main assumption, which always holds true, is that the row_id is always increasing and there are no gaps. + // This is true for all the datasets that we are using, otherwise row_id will not correspond to the index. + let mut statement = + connection.prepare(format!("select coalesce(max(row_id), 0) from {split}").as_str())?; + + let len = statement.query_row([], |row| { + let len: usize = row.get(0)?; + Ok(len) + })?; + Ok((columns, len)) +} + +/// Helper function to create a connection pool +fn create_conn_pool>( + db_file: P, + write: bool, +) -> Result> { + let sqlite_flags = if write { + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE + } else { + OpenFlags::SQLITE_OPEN_READ_ONLY + }; + + let manager = SqliteConnectionManager::file(db_file).with_flags(sqlite_flags); + Pool::new(manager).map_err(SqliteDatasetError::ConnectionPool) +} + +/// The `SqliteDatasetStorage` struct represents a SQLite database for storing datasets. +/// It consists of an optional name, a database file path, and a base directory for storage. +#[derive(Clone, Debug)] +pub struct SqliteDatasetStorage { + name: Option, + db_file: Option, + base_dir: Option, +} + +impl SqliteDatasetStorage { + /// Creates a new instance of `SqliteDatasetStorage` using a dataset name. + /// + /// # Arguments + /// + /// * `name` - A string slice that holds the name of the dataset. + pub fn from_name(name: &str) -> Self { + SqliteDatasetStorage { + name: Some(name.to_string()), + db_file: None, + base_dir: None, + } + } + + /// Creates a new instance of `SqliteDatasetStorage` using a database file path. + /// + /// # Arguments + /// + /// * `db_file` - A reference to the Path that represents the database file path. + pub fn from_file>(db_file: P) -> Self { + SqliteDatasetStorage { + name: None, + db_file: Some(db_file.as_ref().to_path_buf()), + base_dir: None, + } + } + + /// Sets the base directory for storing the dataset. + /// + /// # Arguments + /// + /// * `base_dir` - A string slice that represents the base directory. + pub fn with_base_dir>(mut self, base_dir: P) -> Self { + self.base_dir = Some(base_dir.as_ref().to_path_buf()); + self + } + + /// Checks if the database file exists in the given path. + /// + /// # Returns + /// + /// * A boolean value indicating whether the file exists or not. + pub fn exists(&self) -> bool { + self.db_file().exists() + } + + /// Fetches the database file path. + /// + /// # Returns + /// + /// * A `PathBuf` instance representing the file path. + pub fn db_file(&self) -> PathBuf { + match &self.db_file { + Some(db_file) => db_file.clone(), + None => { + let name = sanitize(self.name.as_ref().expect("Name is not set")); + Self::base_dir(self.base_dir.to_owned()).join(format!("{name}.db")) + } + } + } + + /// Determines the base directory for storing the dataset. + /// + /// # Arguments + /// + /// * `base_dir` - An `Option` that may contain a `PathBuf` instance representing the base directory. + /// + /// # Returns + /// + /// * A `PathBuf` instance representing the base directory. + pub fn base_dir(base_dir: Option) -> PathBuf { + match base_dir { + Some(base_dir) => base_dir, + None => dirs::cache_dir() + .expect("Could not get cache directory") + .join("burn-dataset"), + } + } + + /// Provides a writer instance for the SQLite dataset. + /// + /// # Arguments + /// + /// * `overwrite` - A boolean indicating if the existing database file should be overwritten. + /// + /// # Returns + /// + /// * A `Result` which is `Ok` if the writer could be created, `Err` otherwise. + pub fn writer(&self, overwrite: bool) -> Result> + where + I: Clone + Send + Sync + Serialize + DeserializeOwned, + { + SqliteDatasetWriter::new(self.db_file(), overwrite) + } + + /// Provides a reader instance for the SQLite dataset. + /// + /// # Arguments + /// + /// * `split` - A string slice that defines the data split for reading (e.g., "train", "test"). + /// + /// # Returns + /// + /// * A `Result` which is `Ok` if the reader could be created, `Err` otherwise. + pub fn reader(&self, split: &str) -> Result> + where + I: Clone + Send + Sync + Serialize + DeserializeOwned, + { + if !self.exists() { + panic!("The database file does not exist"); + } + + SqliteDataset::from_db_file(self.db_file(), split) + } +} + +/// This `SqliteDatasetWriter` struct is a SQLite database writer dedicated to storing datasets. +/// It retains the current writer's state and its database connection. +/// +/// Being thread-safe, this writer can be concurrently used across multiple threads. +/// +/// Typical applications include: +/// +/// - Generation of a new dataset +/// - Storage of preprocessed data or metadata +/// - Enlargement of a dataset's item count post preprocessing +#[derive(Debug)] +pub struct SqliteDatasetWriter { + db_file: PathBuf, + db_file_tmp: Option>, + splits: Arc>>, + overwrite: bool, + conn_pool: Option>, + is_completed: Arc>, + phantom: PhantomData, +} + +impl SqliteDatasetWriter +where + I: Clone + Send + Sync + Serialize + DeserializeOwned, +{ + /// Creates a new instance of `SqliteDatasetWriter`. + /// + /// # Arguments + /// + /// * `db_file` - A reference to the Path that represents the database file path. + /// * `overwrite` - A boolean indicating if the existing database file should be overwritten. + /// + /// # Returns + /// + /// * A `Result` which is `Ok` if the writer could be created, `Err` otherwise. + pub fn new>(db_file: P, overwrite: bool) -> Result { + let writer = Self { + db_file: db_file.as_ref().to_path_buf(), + db_file_tmp: None, + splits: Arc::new(RwLock::new(HashSet::new())), + overwrite, + conn_pool: None, + is_completed: Arc::new(RwLock::new(false)), + phantom: PhantomData, + }; + + writer.init() + } + + /// Initializes the dataset writer by creating the database file, tables, and connection pool. + /// + /// # Returns + /// + /// * A `Result` which is `Ok` if the writer could be initialized, `Err` otherwise. + fn init(mut self) -> Result { + // Remove the db file if it already exists + if self.db_file.exists() { + if self.overwrite { + fs::remove_file(&self.db_file)?; + } else { + return Err(SqliteDatasetError::FileExists(self.db_file)); + } + } + + // Create the database file directory if it does not exist + let db_file_dir = self + .db_file + .parent() + .ok_or("Unable to get parent directory")?; + + if !db_file_dir.exists() { + fs::create_dir_all(db_file_dir)?; + } + + // Create a temp database file name as {base_dir}/{name}.db.tmp + let mut db_file_tmp = self.db_file.clone(); + db_file_tmp.set_extension("db.tmp"); + if db_file_tmp.exists() { + fs::remove_file(&db_file_tmp)?; + } + + // Create the temp database file and wrap it with a gix_tempfile::Handle + // This will ensure that the temp file is deleted when the writer is dropped + // or when process exits with SIGINT or SIGTERM (tempfile crate does not do this) + gix_tempfile::signal::setup(Default::default()); + self.db_file_tmp = Some(gix_tempfile::writable_at( + &db_file_tmp, + ContainingDirectory::Exists, + AutoRemove::Tempfile, + )?); + + let conn_pool = create_conn_pool(db_file_tmp, true)?; + self.conn_pool = Some(conn_pool); + + Ok(self) + } + + /// Serializes and writes an item to the database. The item is written to the table for the + /// specified split. If the table does not exist, it is created. If the table exists, the item + /// is appended to the table. The serialization is done using the [MessagePack](https://msgpack.org/) + /// + /// # Arguments + /// + /// * `split` - A string slice that defines the data split for writing (e.g., "train", "test"). + /// * `item` - A reference to the item to be written to the database. + /// + /// # Returns + /// + /// * A `Result` containing the index of the inserted row if successful, an error otherwise. + pub fn write(&self, split: &str, item: &I) -> Result { + // Acquire the read lock (wont't block other reads) + let is_completed = self.is_completed.read().unwrap(); + + // If the writer is completed, return an error + if *is_completed { + return Err(SqliteDatasetError::Other( + "Cannot save to a completed dataset writer", + )); + } + + // create the table for the split if it does not exist + if !self.splits.read().unwrap().contains(split) { + self.create_table(split)?; + } + + // Get a connection from the pool + let conn_pool = self.conn_pool.as_ref().unwrap(); + let conn = conn_pool.get()?; + + // Serialize the item using MessagePack + let serialized_item = rmp_serde::to_vec(item)?; + + // Turn off the synchronous and journal mode for speed up + // We are sacrificing durability for speed but it's okay because + // we always recreate the dataset if it is not completed. + pragma_update_with_error_handling(&conn, "synchronous", "OFF")?; + pragma_update_with_error_handling(&conn, "journal_mode", "OFF")?; + + // Insert the serialized item into the database + let insert_statement = format!("insert into {split} (item) values (?)"); + conn.execute(insert_statement.as_str(), [serialized_item])?; + + // Get the primary key of the last inserted row and convert to index (row_id-1) + let index = (conn.last_insert_rowid() - 1) as usize; + + Ok(index) + } + + /// Marks the dataset as completed and persists the temporary database file. + pub fn set_completed(&mut self) -> Result<()> { + let mut is_completed = self.is_completed.write().unwrap(); + + // Force close the connection pool + // This is required on Windows platform where the connection pool prevents + // from persisting the db by renaming the temp file. + if let Some(pool) = self.conn_pool.take() { + std::mem::drop(pool); + } + + // Rename the database file from tmp to db + let _file_result = self + .db_file_tmp + .take() // take ownership of the temporary file and set to None + .unwrap() // unwrap the temporary file + .persist(&self.db_file)? + .ok_or("Unable to persist the database file")?; + + *is_completed = true; + Ok(()) + } + + /// Creates table for the data split. + /// + /// Note: call is idempotent and thread-safe. + /// + /// # Arguments + /// + /// * `split` - A string slice that defines the data split for the table (e.g., "train", "test"). + /// + /// # Returns + /// + /// * A `Result` which is `Ok` if the table could be created, `Err` otherwise. + /// + /// TODO (@antimora): add support creating a table with columns corresponding to the item fields + fn create_table(&self, split: &str) -> Result<()> { + // Check if the split already exists + if self.splits.read().unwrap().contains(split) { + return Ok(()); + } + + let conn_pool = self.conn_pool.as_ref().unwrap(); + let connection = conn_pool.get()?; + let create_table_statement = format!( + "create table if not exists {split} (row_id integer primary key autoincrement not \ + null, item blob not null)" + ); + + connection.execute(create_table_statement.as_str(), [])?; + + // Add the split to the splits + self.splits.write().unwrap().insert(split.to_string()); + + Ok(()) + } +} + +/// Runs a pragma update and ignores the `ExecuteReturnedResults` error. +/// +/// Sometimes ExecuteReturnedResults is returned when running a pragma update. This is not an error +/// and can be ignored. This function runs the pragma update and ignores the error if it is +/// `ExecuteReturnedResults`. +fn pragma_update_with_error_handling( + conn: &PooledConnection, + setting: &str, + value: &str, +) -> Result<()> { + let result = conn.pragma_update(None, setting, value); + if let Err(error) = result + && error != rusqlite::Error::ExecuteReturnedResults + { + return Err(SqliteDatasetError::Sql(error)); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use rayon::prelude::*; + use rstest::{fixture, rstest}; + use serde::{Deserialize, Serialize}; + use tempfile::{NamedTempFile, TempDir, tempdir}; + + use super::*; + + type SqlDs = SqliteDataset; + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] + pub struct Sample { + column_str: String, + column_bytes: Vec, + column_int: i64, + column_bool: bool, + column_float: f64, + } + + #[fixture] + fn train_dataset() -> SqlDs { + SqliteDataset::::from_db_file("tests/data/sqlite-dataset.db", "train").unwrap() + } + + #[rstest] + pub fn len(train_dataset: SqlDs) { + assert_eq!(train_dataset.len(), 2); + } + + #[rstest] + pub fn get_some(train_dataset: SqlDs) { + let item = train_dataset.get(0).unwrap(); + assert_eq!(item.column_str, "HI1"); + assert_eq!(item.column_bytes, vec![55, 231, 159]); + assert_eq!(item.column_int, 1); + assert!(item.column_bool); + assert_eq!(item.column_float, 1.0); + } + + #[rstest] + pub fn get_none(train_dataset: SqlDs) { + assert_eq!(train_dataset.get(10), None); + } + + #[rstest] + pub fn multi_thread(train_dataset: SqlDs) { + let indices: Vec = vec![0, 1, 1, 3, 4, 5, 6, 0, 8, 1]; + let results: Vec> = + indices.par_iter().map(|&i| train_dataset.get(i)).collect(); + + let mut match_count = 0; + for (_index, result) in indices.iter().zip(results.iter()) { + if let Some(_val) = result { + match_count += 1 + } + } + + assert_eq!(match_count, 5); + } + + #[test] + fn sqlite_dataset_storage() { + // Test with non-existing file + let storage = SqliteDatasetStorage::from_file("non-existing.db"); + assert!(!storage.exists()); + + // Test with non-existing name + let storage = SqliteDatasetStorage::from_name("non-existing.db"); + assert!(!storage.exists()); + + // Test with existing file + let storage = SqliteDatasetStorage::from_file("tests/data/sqlite-dataset.db"); + assert!(storage.exists()); + let result = storage.reader::("train"); + assert!(result.is_ok()); + let train = result.unwrap(); + assert_eq!(train.len(), 2); + + // Test get writer + let temp_file = NamedTempFile::new().unwrap(); + let storage = SqliteDatasetStorage::from_file(temp_file.path()); + assert!(storage.exists()); + let result = storage.writer::(true); + assert!(result.is_ok()); + } + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] + pub struct Complex { + column_str: String, + column_bytes: Vec, + column_int: i64, + column_bool: bool, + column_float: f64, + column_complex: Vec>>, + } + + /// Create a temporary directory. + #[fixture] + fn tmp_dir() -> TempDir { + // Create a TempDir. This object will be automatically + // deleted when it goes out of scope. + tempdir().unwrap() + } + type Writer = SqliteDatasetWriter; + + /// Create a SqliteDatasetWriter with a temporary directory. + /// Make sure to return the temporary directory so that it is not deleted. + #[fixture] + fn writer_fixture(tmp_dir: TempDir) -> (Writer, TempDir) { + let temp_dir_str = tmp_dir.path(); + let storage = SqliteDatasetStorage::from_name("preprocessed").with_base_dir(temp_dir_str); + let overwrite = true; + let result = storage.writer::(overwrite); + assert!(result.is_ok()); + let writer = result.unwrap(); + (writer, tmp_dir) + } + + #[test] + fn test_new() { + // Test that the constructor works with overwrite = true + let test_path = NamedTempFile::new().unwrap(); + let _writer = SqliteDatasetWriter::::new(&test_path, true).unwrap(); + assert!(!test_path.path().exists()); + + // Test that the constructor works with overwrite = false + let test_path = NamedTempFile::new().unwrap(); + let result = SqliteDatasetWriter::::new(&test_path, false); + assert!(result.is_err()); + + // Test that the constructor works with no existing file + let temp = NamedTempFile::new().unwrap(); + let test_path = temp.path().to_path_buf(); + assert!(temp.close().is_ok()); + assert!(!test_path.exists()); + let _writer = SqliteDatasetWriter::::new(&test_path, true).unwrap(); + assert!(!test_path.exists()); + } + + #[rstest] + pub fn sqlite_writer_write(writer_fixture: (Writer, TempDir)) { + // Get the dataset_saver from the fixture and tmp_dir (will be deleted after scope) + let (writer, _tmp_dir) = writer_fixture; + + assert!(writer.overwrite); + assert!(!writer.db_file.exists()); + + let new_item = Complex { + column_str: "HI1".to_string(), + column_bytes: vec![1_u8, 2, 3], + column_int: 0, + column_bool: true, + column_float: 1.0, + column_complex: vec![vec![vec![[1, 23_u8, 3]]]], + }; + + let index = writer.write("train", &new_item).unwrap(); + assert_eq!(index, 0); + + let mut writer = writer; + + writer.set_completed().expect("Failed to set completed"); + + assert!(writer.db_file.exists()); + assert!(writer.db_file_tmp.is_none()); + + let result = writer.write("train", &new_item); + + // Should fail because the writer is completed + assert!(result.is_err()); + + let dataset = SqliteDataset::::from_db_file(writer.db_file, "train").unwrap(); + + let fetched_item = dataset.get(0).unwrap(); + assert_eq!(fetched_item, new_item); + assert_eq!(dataset.len(), 1); + } + + #[rstest] + pub fn sqlite_writer_write_multi_thread(writer_fixture: (Writer, TempDir)) { + // Get the dataset_saver from the fixture and tmp_dir (will be deleted after scope) + let (writer, _tmp_dir) = writer_fixture; + + let writer = Arc::new(writer); + let record_count = 20; + + let splits = ["train", "test"]; + + (0..record_count).into_par_iter().for_each(|index: i64| { + let thread_id: std::thread::ThreadId = std::thread::current().id(); + let sample = Complex { + column_str: format!("test_{thread_id:?}_{index}"), + column_bytes: vec![index as u8, 2, 3], + column_int: index, + column_bool: true, + column_float: 1.0, + column_complex: vec![vec![vec![[1, index as u8, 3]]]], + }; + + // half for train and half for test + let split = splits[index as usize % 2]; + + let _index = writer.write(split, &sample).unwrap(); + }); + + let mut writer = Arc::try_unwrap(writer).unwrap(); + + writer + .set_completed() + .expect("Should set completed successfully"); + + let train = + SqliteDataset::::from_db_file(writer.db_file.clone(), "train").unwrap(); + let test = SqliteDataset::::from_db_file(writer.db_file, "test").unwrap(); + + assert_eq!(train.len(), record_count as usize / 2); + assert_eq!(test.len(), record_count as usize / 2); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/lib.rs new file mode 100644 index 0000000..822a19d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/lib.rs @@ -0,0 +1,52 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! # Burn Dataset +//! +//! Burn Dataset is a library for creating and loading datasets. + +#[macro_use] +extern crate derive_new; + +extern crate alloc; +extern crate dirs; + +/// Sources for datasets. +pub mod source; + +pub mod transform; + +/// Audio datasets. +#[cfg(feature = "audio")] +pub mod audio; + +/// Vision datasets. +#[cfg(feature = "vision")] +pub mod vision; + +/// Natural language processing datasets. +#[cfg(feature = "nlp")] +pub mod nlp; + +/// Network dataset utilities. +#[cfg(feature = "network")] +pub mod network { + pub use burn_std::network::*; +} + +mod dataset; +pub use dataset::*; +#[cfg(any(feature = "sqlite", feature = "sqlite-bundled"))] +pub use source::huggingface::downloader::*; + +#[cfg(test)] +mod test_data { + pub fn string_items() -> Vec { + vec![ + "1 Item".to_string(), + "2 Items".to_string(), + "3 Items".to_string(), + "4 Items".to_string(), + ] + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/ag_news.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/ag_news.rs new file mode 100644 index 0000000..dc151f1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/ag_news.rs @@ -0,0 +1,211 @@ +//! AG NEWS Dataset Module +//! +//! This module provides functionality for loading the AG NEWS text classification dataset. +//! AG NEWS is a collection of news articles categorized into different topics. +//! The dataset is split into training (120,000 articles) and test (7,600 articles) sets. +//! +//! ## Dataset Details +//! - **Classes**: 4 categories (World, Sports, Business, Sci/Tech) +//! - **AG NEWS mirror**: [fastai](https://github.com/fastai/fastai/blob/master/fastai/data/external.py#L83) +//! - **License**: [Apache License](https://github.com/fastai/fastai/blob/master/LICENSE) +//! +//! ## Usage Example +//! ```rust +//! use burn_dataset::nlp::AgNewsDataset; +//! +//! // Create an AG NEWS dataset accessor +//! let dataset = AgNewsDataset::new(); +//! +//! // Access training and test sets +//! let train_dataset = dataset.train(); +//! let test_dataset = dataset.test(); +//! ``` + +use std::{path::PathBuf, sync::Mutex}; + +use flate2::read::GzDecoder; +use serde::{Deserialize, Serialize}; +use tar::Archive; + +use crate::InMemDataset; +use crate::network::downloader; + +/// AG NEWS mirror from [fastai](https://github.com/fastai/fastai/blob/master/fastai/data/external.py#L83). +/// Licensed under the [Apache License](https://github.com/fastai/fastai/blob/master/LICENSE). +const AG_NEWS_URL: &str = "https://s3.amazonaws.com/fast-ai-nlp/ag_news_csv.tgz"; + +/// Represents an item in the AG NEWS dataset. +/// +/// Each item contains a label, title, and content of a news article. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct AgNewsItem { + /// The category label of the news article. + pub label: String, + /// The title of the news article. + pub title: String, + /// The content/body of the news article. + pub content: String, +} + +/// AG NEWS dataset accessor. +/// +/// This struct provides convenient access to the AG NEWS text classification dataset. +/// It automatically downloads (if not already downloaded), extracts, and loads the datasets. +/// +/// The dataset is split into training (120,000 articles) and test (7,600 articles) sets. +pub struct AgNewsDataset { + agnews_dir: PathBuf, +} + +/// AG NEWS dataset download lock. +/// +/// This lock ensures that only one thread downloads the AG NEWS dataset at a time. +static DOWNLOAD_LOCK: Mutex<()> = Mutex::new(()); + +impl AgNewsDataset { + /// Creates a new AG NEWS dataset accessor. + /// + /// This will download and extract the dataset if it's not already present. + pub fn new() -> Self { + Self { + agnews_dir: Self::download(), + } + } + + /// Downloads and extracts the AG NEWS dataset. + /// + /// # Returns + /// Path to the directory containing the extracted dataset. + fn download() -> PathBuf { + // Acquire the lock. This will block if another thread already holds the lock. + let _lock = DOWNLOAD_LOCK.lock().unwrap(); + + // Dataset files are stored in the burn-dataset cache directory + let cache_dir = dirs::cache_dir() + .expect("Could not get cache directory") + .join("burn-dataset"); + + // AG NEWS dataset directory + let agnews_dir = cache_dir.join("ag_news_csv"); + + // AG NEWS dataset url + let url = AG_NEWS_URL; + + // AG NEWS dataset archive filename + let filename = "ag_news_csv.tgz"; + + // Check for already downloaded content + if !agnews_dir.exists() { + // Download gzip file + let bytes = downloader::download_file_as_bytes(url, filename); + + // Decode gzip file content and unpack archive + let gz_buffer = GzDecoder::new(&bytes[..]); + let mut archive = Archive::new(gz_buffer); + archive.unpack(cache_dir).unwrap(); + } + + agnews_dir + } + + /// Parses a CSV file into an in-memory dataset. + /// + /// # Arguments + /// * `file_path` - Path to the CSV file to parse. + /// + /// # Returns + /// An `InMemDataset` containing the parsed data. + fn parse_csv(file_path: &str) -> InMemDataset { + let mut rdr = csv::ReaderBuilder::new(); + let rdr = rdr.has_headers(false); + + InMemDataset::from_csv(file_path, &rdr).expect("Failed to parse CSV file") + } + + /// Gets the training dataset. + /// + /// # Returns + /// An `InMemDataset` instance containing 120,000 training articles. + pub fn train(&self) -> InMemDataset { + let file_path = self.agnews_dir.join("train.csv"); + Self::parse_csv(file_path.to_str().unwrap()) + } + + /// Gets the test dataset. + /// + /// # Returns + /// An `InMemDataset` instance containing 7,600 test articles. + pub fn test(&self) -> InMemDataset { + let file_path = self.agnews_dir.join("test.csv"); + Self::parse_csv(file_path.to_str().unwrap()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Dataset; + + // AG NEWS dataset train and test dataset lengths + const TRAIN_DATASET_LEN: usize = 120000; + const TEST_DATASET_LEN: usize = 7600; + + #[test] + fn test_agnews_download() { + let agnews_dir = AgNewsDataset::download(); + assert!(agnews_dir.exists()); + } + + #[test] + fn test_agnews_len() { + let agnews = AgNewsDataset::new(); + let train_dataset = agnews.train(); + let test_dataset = agnews.test(); + assert_eq!(train_dataset.len(), TRAIN_DATASET_LEN); + assert_eq!(test_dataset.len(), TEST_DATASET_LEN); + } + + #[test] + fn test_agnews_first_and_last_item() { + let agnews = AgNewsDataset::new(); + + // Test the first and the last item in training dataset + let train_dataset = agnews.train(); + let first_item = train_dataset.get(0).unwrap(); + let last_item = train_dataset.get(train_dataset.len() - 1).unwrap(); + assert!(compare_item(&first_item, &("3".to_string(), "Wall St. Bears Claw Back Into the Black (Reuters)".to_string(), "Reuters - Short-sellers, Wall Street's dwindling\\band of ultra-cynics, are seeing green again.".to_string()))); + assert!(compare_item( + &last_item, + &( + "2".to_string(), + "Nets get Carter from Raptors".to_string(), + "INDIANAPOLIS -- All-Star Vince Carter was traded by the Toronto Raptors to the New Jersey Nets for Alonzo Mourning, Eric Williams, Aaron Williams, and a pair of first-round draft picks yesterday.".to_string() + ) + )); + + // Test the first and the last item in test dataset + let test_dataset = agnews.test(); + let first_item = test_dataset.get(0).unwrap(); + let last_item = test_dataset.get(test_dataset.len() - 1).unwrap(); + assert!(compare_item( + &first_item, + &( + "3".to_string(), + "Fears for T N pension after talks".to_string(), + "Unions representing workers at Turner Newall say they are 'disappointed' after talks with stricken parent firm Federal Mogul.".to_string() + ) + )); + assert!(compare_item( + &last_item, + &( + "3".to_string(), + "EBay gets into rentals".to_string(), + "EBay plans to buy the apartment and home rental service Rent.com for \\$415 million, adding to its already exhaustive breadth of offerings.".to_string() + ) + )); + } + + fn compare_item(item: &AgNewsItem, target: &(String, String, String)) -> bool { + item.label == target.0 && item.title == target.1 && item.content == target.2 + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/mod.rs new file mode 100644 index 0000000..52574b3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/mod.rs @@ -0,0 +1,7 @@ +#[cfg(feature = "builtin-sources")] +mod ag_news; +mod text_folder; + +#[cfg(feature = "builtin-sources")] +pub use ag_news::*; +pub use text_folder::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/text_folder.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/text_folder.rs new file mode 100644 index 0000000..9df8850 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/nlp/text_folder.rs @@ -0,0 +1,421 @@ +use crate::transform::{Mapper, MapperDataset}; +use crate::{Dataset, InMemDataset}; + +use encoding_rs::{GB18030, GBK, UTF_8, UTF_16BE, UTF_16LE}; +use globwalk::{self, DirEntry}; +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; +use thiserror::Error; + +const SUPPORTED_FILES: [&str; 1] = ["txt"]; + +/// Text data type. +#[derive(Debug, Clone, PartialEq)] +pub struct TextData { + /// The text content. + pub text: String, + + /// Original text source. + pub text_path: String, +} + +/// Text dataset item. +#[derive(Debug, Clone, PartialEq)] +pub struct TextDatasetItem { + /// Text content. + pub text: TextData, + + /// Label for the text. + pub label: usize, +} + +/// Raw text dataset item. +#[derive(Debug, Clone)] +struct TextDatasetItemRaw { + /// Text path. + text_path: PathBuf, + + /// Text label. + label: String, +} + +impl TextDatasetItemRaw { + fn new>(text_path: P, label: String) -> TextDatasetItemRaw { + TextDatasetItemRaw { + text_path: text_path.as_ref().to_path_buf(), + label, + } + } +} + +struct PathToTextDatasetItem { + classes: HashMap, +} + +/// Parse the text content from file with auto-detection of encoding. +fn parse_text_content(text_path: &PathBuf) -> String { + // Read raw bytes from disk + let mut file = fs::File::open(text_path).unwrap(); + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).unwrap(); + + // Try to detect encoding and decode text + // First try UTF-8 with BOM + if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) && bytes.len() >= 3 { + let (result, _, had_errors) = UTF_8.decode(&bytes[3..]); + if !had_errors { + return result.into_owned(); + } + } + + // Try UTF-8 without BOM + let (result, _, had_errors) = UTF_8.decode(&bytes); + if !had_errors { + return result.into_owned(); + } + + // Try UTF-16LE with BOM + if bytes.starts_with(&[0xFF, 0xFE]) && bytes.len() >= 2 { + let (result, had_errors) = UTF_16LE.decode_with_bom_removal(&bytes[2..]); + if !had_errors { + return result.into_owned(); + } + } + + // Try UTF-16BE with BOM + if bytes.starts_with(&[0xFE, 0xFF]) && bytes.len() >= 2 { + let (result, had_errors) = UTF_16BE.decode_with_bom_removal(&bytes[2..]); + if !had_errors { + return result.into_owned(); + } + } + + // Try GB18030 encoding + let (result, _, had_errors) = GB18030.decode(&bytes); + if !had_errors { + return result.into_owned(); + } + + // Try GBK encoding + let (result, _, had_errors) = GBK.decode(&bytes); + if !had_errors { + return result.into_owned(); + } + + // Default fallback - use from_utf8_lossy for any remaining cases + String::from_utf8_lossy(&bytes).to_string() +} + +impl Mapper for PathToTextDatasetItem { + /// Convert a raw text dataset item (path-like) to text content with a target label. + fn map(&self, item: &TextDatasetItemRaw) -> TextDatasetItem { + let label = *self.classes.get(&item.label).unwrap(); + + // Load text from disk + let text_content = parse_text_content(&item.text_path); + + let text_data = TextData { + text: text_content, + text_path: item.text_path.display().to_string(), + }; + + TextDatasetItem { + text: text_data, + label, + } + } +} + +/// Error type for [TextFolderDataset](TextFolderDataset). +#[derive(Error, Debug)] +pub enum TextLoaderError { + /// Unknown error. + #[error("unknown: `{0}`")] + Unknown(String), + + /// I/O operation error. + #[error("I/O error: `{0}`")] + IOError(String), + + /// Invalid file error. + #[error("Invalid file extension: `{0}`")] + InvalidFileExtensionError(String), + + /// Encoding error. + #[error("Encoding error: `{0}`")] + EncodingError(String), +} + +type TextDatasetMapper = + MapperDataset, PathToTextDatasetItem, TextDatasetItemRaw>; + +/// A generic dataset to load texts from disk. +pub struct TextFolderDataset { + dataset: TextDatasetMapper, +} + +impl Dataset for TextFolderDataset { + fn get(&self, index: usize) -> Option { + self.dataset.get(index) + } + + fn len(&self) -> usize { + self.dataset.len() + } +} + +impl TextFolderDataset { + /// Create a text classification dataset from the root folder. + /// + /// # Arguments + /// + /// * `root` - Dataset root folder. + /// + /// # Returns + /// A new dataset instance. + pub fn new_classification>(root: P) -> Result { + // New dataset containing any of the supported file types + TextFolderDataset::new_classification_with(root, &SUPPORTED_FILES) + } + + /// Create a text classification dataset from the root folder. + /// The included texts are filtered based on the provided extensions. + /// + /// # Arguments + /// + /// * `root` - Dataset root folder. + /// * `extensions` - List of allowed extensions. + /// + /// # Returns + /// A new dataset instance. + pub fn new_classification_with(root: P, extensions: &[S]) -> Result + where + P: AsRef, + S: AsRef, + { + // Glob all texts with extensions + let walker = globwalk::GlobWalkerBuilder::from_patterns( + root.as_ref(), + &[format!( + "*.{{{}}}", // "*.{ext1,ext2,ext3} + extensions + .iter() + .map(Self::check_extension) + .collect::, _>>()? + .join(",") + )], + ) + .follow_links(true) + .sort_by(|p1: &DirEntry, p2: &DirEntry| p1.path().cmp(p2.path())) // order by path + .build() + .map_err(|err| TextLoaderError::Unknown(format!("{err:?}")))? + .filter_map(Result::ok); + + // Get all dataset items + let mut items = Vec::new(); + let mut classes = HashSet::new(); + for text in walker { + let text_path = text.path(); + + // Label name is represented by the parent folder name + let label = text_path + .parent() + .ok_or_else(|| { + TextLoaderError::IOError("Could not resolve text parent folder".to_string()) + })? + .file_name() + .ok_or_else(|| { + TextLoaderError::IOError( + "Could not resolve text parent folder name".to_string(), + ) + })? + .to_string_lossy() + .into_owned(); + + classes.insert(label.clone()); + + items.push(TextDatasetItemRaw::new(text_path, label)) + } + + // Sort class names + let mut classes = classes.into_iter().collect::>(); + classes.sort(); + + Self::with_items(items, &classes) + } + + /// Create a text classification dataset with the specified items. + /// + /// # Arguments + /// + /// * `items` - List of dataset items, each item represented by a tuple `(text path, label)`. + /// * `classes` - Dataset class names. + /// + /// # Returns + /// A new dataset instance. + pub fn new_classification_with_items, S: AsRef>( + items: Vec<(P, String)>, + classes: &[S], + ) -> Result { + // Parse items and check valid text extension types + let items = items + .into_iter() + .map(|(path, label)| { + // Map text path and label + let path = path.as_ref(); + let label = label; + + Self::check_extension(&path.extension().unwrap().to_str().unwrap())?; + + Ok(TextDatasetItemRaw::new(path, label)) + }) + .collect::, _>>()?; + + Self::with_items(items, classes) + } + + /// Create a text dataset with the specified items. + /// + /// # Arguments + /// + /// * `items` - Raw dataset items. + /// * `classes` - Dataset class names. + /// + /// # Returns + /// A new dataset instance. + fn with_items>( + items: Vec, + classes: &[S], + ) -> Result { + // NOTE: right now we don't need to validate the supported text files since + // the method is private. We assume it's already validated. + let dataset = InMemDataset::new(items); + + // Class names to index map + let classes = classes.iter().map(|c| c.as_ref()).collect::>(); + let classes_map: HashMap<_, _> = classes + .into_iter() + .enumerate() + .map(|(idx, cls)| (cls.to_string(), idx)) + .collect(); + + let mapper = PathToTextDatasetItem { + classes: classes_map, + }; + let dataset = MapperDataset::new(dataset, mapper); + + Ok(Self { dataset }) + } + + /// Check if extension is supported. + fn check_extension>(extension: &S) -> Result { + let extension = extension.as_ref(); + if !SUPPORTED_FILES.contains(&extension) { + Err(TextLoaderError::InvalidFileExtensionError( + extension.to_string(), + )) + } else { + Ok(extension.to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + const TEXT_ROOT: &str = "tests/data/text_folder"; + + #[test] + fn test_text_folder_dataset() { + let dataset = TextFolderDataset::new_classification(TEXT_ROOT).unwrap(); + + // Dataset should have 4 elements (2 positive + 2 negative) + assert_eq!(dataset.len(), 4); + assert_eq!(dataset.get(4), None); + + // Check that we have items from both classes + let mut found_positive = false; + let mut found_negative = false; + + for i in 0..dataset.len() { + let item = dataset.get(i).unwrap(); + if item.label == 0 { + found_negative = true; + // Check that the text content is loaded correctly + assert!(!item.text.text.is_empty()); + assert!(item.text.text_path.contains("negative")); + } else if item.label == 1 { + found_positive = true; + // Check that the text content is loaded correctly + assert!(!item.text.text.is_empty()); + assert!(item.text.text_path.contains("positive")); + } + } + + // Verify we found items from both classes + assert!(found_positive); + assert!(found_negative); + } + + #[test] + fn test_text_folder_dataset_with_invalid_extension() { + // Try to create a dataset with an unsupported extension + let result = TextFolderDataset::new_classification_with(TEXT_ROOT, &["invalid"]); + assert!(result.is_err()); + } + + #[test] + fn test_text_folder_dataset_with_items() { + // Create the dataset + let root = Path::new(TEXT_ROOT); + let items = vec![ + ( + root.join("positive").join("sample1.txt"), + "positive".to_string(), + ), + ( + root.join("negative").join("sample2.txt"), + "negative".to_string(), + ), + ]; + let classes = vec!["positive", "negative"]; + let dataset = TextFolderDataset::new_classification_with_items(items, &classes).unwrap(); + + // Dataset should have 2 elements + assert_eq!(dataset.len(), 2); + assert_eq!(dataset.get(2), None); + + // Get items + let item0 = dataset.get(0).unwrap(); + let item1 = dataset.get(1).unwrap(); + + // Check item0 + assert!(compare_item( + &item0, + &( + "This is a positive text sample for testing the text folder dataset functionality." + .to_string(), + 0 + ) + )); + + // Check item1 + assert_eq!(item1.label, 1); + assert!(item1.text.text_path.contains("negative")); + assert!(compare_item( + &item1, + &( + "另一个负面文本样本，用以确保数据集能够处理同一类别中的多个文件。".to_string(), + 1 + ) + )); + } + + fn compare_item(item: &TextDatasetItem, target: &(String, usize)) -> bool { + item.text.text == target.0 && item.label == target.1 + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/downloader.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/downloader.rs new file mode 100644 index 0000000..a97bd46 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/downloader.rs @@ -0,0 +1,367 @@ +use std::fs::{self, create_dir_all}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::{SqliteDataset, SqliteDatasetError, SqliteDatasetStorage}; + +use sanitize_filename::sanitize; +use serde::de::DeserializeOwned; +use thiserror::Error; + +const PYTHON_SOURCE: &str = include_str!("importer.py"); +#[cfg(not(target_os = "windows"))] +const VENV_BIN_PYTHON: &str = "bin/python3"; +#[cfg(target_os = "windows")] +const VENV_BIN_PYTHON: &str = "Scripts\\python"; + +/// Error type for [HuggingfaceDatasetLoader](HuggingfaceDatasetLoader). +#[derive(Error, Debug)] +pub enum ImporterError { + /// Unknown error. + #[error("unknown: `{0}`")] + Unknown(String), + + /// Fail to download python dependencies. + #[error("fail to download python dependencies: `{0}`")] + FailToDownloadPythonDependencies(String), + + /// Fail to create sqlite dataset. + #[error("sqlite dataset: `{0}`")] + SqliteDataset(#[from] SqliteDatasetError), + + /// python3 is not installed. + #[error("python3 is not installed")] + PythonNotInstalled, + + /// venv environment is not initialized. + #[error("venv environment is not initialized")] + VenvNotInitialized, +} + +/// Load a dataset from [huggingface datasets](https://huggingface.co/datasets). +/// +/// The dataset with all splits is stored in a single sqlite database (see [SqliteDataset](SqliteDataset)). +/// +/// # Example +/// ```no_run +/// use burn_dataset::HuggingfaceDatasetLoader; +/// use burn_dataset::SqliteDataset; +/// use serde::{Deserialize, Serialize}; +/// +/// #[derive(Deserialize, Debug, Clone)] +/// struct MnistItemRaw { +/// pub image_bytes: Vec, +/// pub label: usize, +/// } +/// +/// let train_ds:SqliteDataset = HuggingfaceDatasetLoader::new("mnist") +/// .dataset("train") +/// .unwrap(); +/// ``` +/// +/// # Note +/// This loader relies on the [`datasets` library by HuggingFace](https://huggingface.co/docs/datasets/index) +/// to download datasets. This is a Python library, so you must have an existing Python installation. +pub struct HuggingfaceDatasetLoader { + name: String, + subset: Option, + base_dir: Option, + huggingface_token: Option, + huggingface_cache_dir: Option, + huggingface_data_dir: Option, + trust_remote_code: bool, + use_python_venv: bool, +} + +impl HuggingfaceDatasetLoader { + /// Create a huggingface dataset loader. + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + subset: None, + base_dir: None, + huggingface_token: None, + huggingface_cache_dir: None, + huggingface_data_dir: None, + trust_remote_code: false, + use_python_venv: true, + } + } + + /// Create a huggingface dataset loader for a subset of the dataset. + /// + /// The subset name must be one of the subsets listed in the dataset page. + /// + /// If no subset names are listed, then do not use this method. + pub fn with_subset(mut self, subset: &str) -> Self { + self.subset = Some(subset.to_string()); + self + } + + /// Specify a base directory to store the dataset. + /// + /// If not specified, the dataset will be stored in the system cache directory under `burn-dataset`. + pub fn with_base_dir(mut self, base_dir: &str) -> Self { + self.base_dir = Some(base_dir.into()); + self + } + + /// Specify a huggingface token to download datasets behind authentication. + /// + /// You can get a token from [tokens settings](https://huggingface.co/settings/tokens) + pub fn with_huggingface_token(mut self, huggingface_token: &str) -> Self { + self.huggingface_token = Some(huggingface_token.to_string()); + self + } + + /// Specify a huggingface cache directory to store the downloaded datasets. + /// + /// If not specified, the dataset will be stored in the system cache directory under `huggingface/datasets`. + pub fn with_huggingface_cache_dir(mut self, huggingface_cache_dir: &str) -> Self { + self.huggingface_cache_dir = Some(huggingface_cache_dir.to_string()); + self + } + + /// Specify a relative path to a subset of a dataset. This is used in some datasets for the + /// manual steps of dataset download process. + /// + /// Unless you've encountered a ManualDownloadError + /// when loading your dataset you probably don't have to worry about this setting. + pub fn with_huggingface_data_dir(mut self, huggingface_data_dir: &str) -> Self { + self.huggingface_data_dir = Some(huggingface_data_dir.to_string()); + self + } + + /// Specify whether or not to trust remote code. + /// + /// If not specified, trust remote code is set to true. + pub fn with_trust_remote_code(mut self, trust_remote_code: bool) -> Self { + self.trust_remote_code = trust_remote_code; + self + } + + /// Specify whether or not to use the burn-dataset Python + /// virtualenv for running the importer script. If false, local + /// `python3`'s environment is used. + /// + /// If not specified, the virtualenv is used. + pub fn with_use_python_venv(mut self, use_python_venv: bool) -> Self { + self.use_python_venv = use_python_venv; + self + } + + /// Load the dataset. + pub fn dataset( + self, + split: &str, + ) -> Result, ImporterError> { + let db_file = self.db_file()?; + let dataset = SqliteDataset::from_db_file(db_file, split)?; + Ok(dataset) + } + + /// Get the path to the sqlite database file. + /// + /// If the database file does not exist, it will be downloaded and imported. + pub fn db_file(self) -> Result { + // determine (and create if needed) the base directory + let base_dir = SqliteDatasetStorage::base_dir(self.base_dir); + + if !base_dir.exists() { + create_dir_all(&base_dir).expect("Failed to create base directory"); + } + + //sanitize the name and subset + let name = sanitize(self.name.as_str()); + + // create the db file path + let db_file_name = if let Some(subset) = self.subset.clone() { + format!("{name}-{}.db", sanitize(subset.as_str())) + } else { + format!("{name}.db") + }; + + let db_file = base_dir.join(db_file_name); + + // import the dataset if needed + if !Path::new(&db_file).exists() { + import( + self.name, + self.subset, + db_file.clone(), + base_dir, + self.huggingface_token, + self.huggingface_cache_dir, + self.huggingface_data_dir, + self.trust_remote_code, + self.use_python_venv, + )?; + } + + Ok(db_file) + } +} + +/// Import a dataset from huggingface. The transformed dataset is stored as sqlite database. +#[allow(clippy::too_many_arguments)] +fn import( + name: String, + subset: Option, + base_file: PathBuf, + base_dir: PathBuf, + huggingface_token: Option, + huggingface_cache_dir: Option, + huggingface_data_dir: Option, + trust_remote_code: bool, + use_python_venv: bool, +) -> Result<(), ImporterError> { + let python_path = if use_python_venv { + install_python_deps(&base_dir)? + } else { + get_python_name()?.into() + }; + + let mut command = Command::new(python_path); + + command.arg(importer_script_path(&base_dir)); + + command.arg("--name"); + command.arg(name); + + command.arg("--file"); + command.arg(base_file); + + if let Some(subset) = subset { + command.arg("--subset"); + command.arg(subset); + } + + if let Some(huggingface_token) = huggingface_token { + command.arg("--token"); + command.arg(huggingface_token); + } + + if let Some(huggingface_cache_dir) = huggingface_cache_dir { + command.arg("--cache_dir"); + command.arg(huggingface_cache_dir); + } + if let Some(huggingface_data_dir) = huggingface_data_dir { + command.arg("--data_dir"); + command.arg(huggingface_data_dir); + } + if trust_remote_code { + command.arg("--trust_remote_code"); + command.arg("True"); + } + let mut handle = command.spawn().unwrap(); + + let exit_status = handle + .wait() + .map_err(|err| ImporterError::Unknown(format!("{err:?}")))?; + + if !exit_status.success() { + return Err(ImporterError::Unknown(format!("{exit_status}"))); + } + + Ok(()) +} + +/// check python --version output is `Python 3.x.x` +fn check_python_version_is_3(python: &str) -> bool { + let output = Command::new(python).arg("--version").output(); + match output { + Ok(output) => { + if output.status.success() { + let version_string = String::from_utf8_lossy(&output.stdout); + if let Some(index) = version_string.find(' ') { + let version = &version_string[index + 1..]; + version.starts_with("3.") + } else { + false + } + } else { + false + } + } + Err(_error) => false, + } +} + +/// get python3 name `python` `python3` or `py` +fn get_python_name() -> Result<&'static str, ImporterError> { + let python_name_list = ["python3", "python", "py"]; + for python_name in python_name_list.iter() { + if check_python_version_is_3(python_name) { + return Ok(python_name); + } + } + Err(ImporterError::PythonNotInstalled) +} + +fn importer_script_path(base_dir: &Path) -> PathBuf { + let path_file = base_dir.join("importer.py"); + + fs::write(&path_file, PYTHON_SOURCE).expect("Write python dataset downloader"); + path_file +} + +fn install_python_deps(base_dir: &Path) -> Result { + let venv_dir = base_dir.join("venv"); + let venv_python_path = venv_dir.join(VENV_BIN_PYTHON); + // If the venv environment is already initialized, skip the initialization. + if !check_python_version_is_3(venv_python_path.to_str().unwrap()) { + let python_name = get_python_name()?; + let mut command = Command::new(python_name); + command.args([ + "-m", + "venv", + venv_dir + .as_os_str() + .to_str() + .expect("Path utf8 conversion should not fail"), + ]); + + // Spawn the venv creation process and wait for it to complete. + let mut handle = command.spawn().unwrap(); + + handle.wait().map_err(|err| { + ImporterError::FailToDownloadPythonDependencies(format!(" error: {err}")) + })?; + // Check if the venv environment can be used successfully." + if !check_python_version_is_3(venv_python_path.to_str().unwrap()) { + return Err(ImporterError::VenvNotInitialized); + } + } + + let mut ensurepip_cmd = Command::new(&venv_python_path); + ensurepip_cmd.args(["-m", "ensurepip", "--upgrade"]); + let status = ensurepip_cmd.status().map_err(|err| { + ImporterError::FailToDownloadPythonDependencies(format!("failed to run ensurepip: {err}")) + })?; + if !status.success() { + return Err(ImporterError::FailToDownloadPythonDependencies( + "ensurepip failed to initialize pip".to_string(), + )); + } + + let mut command = Command::new(&venv_python_path); + command.args([ + "-m", + "pip", + "--quiet", + "install", + "pyarrow", + "sqlalchemy", + "Pillow", + "soundfile", + "datasets", + ]); + + // Spawn the pip install process and wait for it to complete. + let mut handle = command.spawn().unwrap(); + handle + .wait() + .map_err(|err| ImporterError::FailToDownloadPythonDependencies(format!(" error: {err}")))?; + + Ok(venv_python_path) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/importer.py b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/importer.py new file mode 100644 index 0000000..c3ce923 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/importer.py @@ -0,0 +1,207 @@ +import argparse + +import pyarrow as pa +from datasets import Audio, Image, load_dataset +from sqlalchemy import Column, Integer, Table, create_engine, event, inspect +from sqlalchemy.types import LargeBinary + + +def download_and_export( + name: str, + subset: str, + db_file: str, + token: str, + cache_dir: str, + data_dir: str | None, + trust_remote_code: bool, +): + """ + Download a dataset from using HuggingFace dataset and export it to a sqlite database. + """ + + # TODO For media columns (Image and Audio) sometimes when decode=False, + # bytes can be none {'bytes': None, 'path': 'healthy_train.265.jpg'} + # We should handle this case, but unfortunately we did not come across this case yet to test it. + + print("*" * 80) + print("Starting huggingface dataset download and export") + print(f"Dataset Name: {name}") + print(f"Subset Name: {subset}") + print(f"Sqlite database file: {db_file}") + print(f"Trust remote code: {trust_remote_code}") + if cache_dir is None: + print(f"Custom cache dir: {cache_dir}") + print("*" * 80) + + # Load the dataset + dataset_all = load_dataset( + name, + subset, + cache_dir=cache_dir, + data_dir=data_dir, + use_auth_token=token, + trust_remote_code=trust_remote_code, + ) + + print(f"Dataset: {dataset_all}") + + # Create the database connection descriptor (sqlite) + engine = create_engine(f"sqlite:///{db_file}") + + # Set some sqlite pragmas to speed up the database + event.listen(engine, "connect", set_sqlite_pragma) + + # Add an row_id column to each table as primary key (datasets does not have API for this) + event.listen(Table, "before_create", add_pk_column) + + # Export each split in the dataset + for key in dataset_all.keys(): + dataset = dataset_all[key] + + # Disable decoding for audio and image fields + dataset = disable_decoding(dataset) + + # Flatten the dataset + dataset = dataset.flatten() + + # Rename columns to remove dots from the names + dataset = rename_columns(dataset) + + print(f"Saving dataset: {name} - {key}") + print(f"Dataset features: {dataset.features}") + + # Save the dataset to a sqlite database + dataset.to_sql( + key, # table name + engine, + # don't save the index, use row_id instead (index is not unique) + index=False, + dtype=blob_columns(dataset), # save binary columns as blob + ) + + # Print the schema of the database so we can reference the columns in the rust code + print_table_info(engine) + + +def disable_decoding(dataset): + """ + Disable decoding for audio and image fields. The fields will be saved as raw file bytes. + """ + for k, v in dataset.features.items(): + if isinstance(v, Audio): + dataset = dataset.cast_column(k, Audio(decode=False)) + elif isinstance(v, Image): + dataset = dataset.cast_column(k, Image(decode=False)) + + return dataset + + +def rename_columns(dataset): + """ + Rename columns to remove dots from the names. Dots appear in the column names because of the flattening. + Dots are not allowed in column names in rust and sql (unless quoted). So we replace them with underscores. + This way there is an easy name mapping between the rust and sql columns. + """ + + for name in dataset.features.keys(): + if "." in name: + dataset = dataset.rename_column(name, name.replace(".", "_")) + + return dataset + + +def blob_columns(dataset): + """ + Make sure all binary columns are blob columns in the database because + `to_sql` exports binary values as TEXT instead of BLOB. + """ + type_mapping = {} + for name, value in dataset.features.items(): + if value.pa_type is not None and pa.types.is_binary(value.pa_type): + type_mapping[name] = LargeBinary + return type_mapping + + +def set_sqlite_pragma(dbapi_connection, connection_record): + """ + Set some sqlite pragmas to speed up the database + """ + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA synchronous = OFF") + cursor.execute("PRAGMA journal_mode = OFF") + cursor.close() + + +def add_pk_column(target, connection, **kw): + """ + Add an id column to each table. + """ + target.append_column(Column("row_id", Integer, primary_key=True)) + + +def print_table_info(engine): + """ + Print the schema of the database so we can reference the columns in the rust code + """ + print(f"Printing table schema for sqlite3 db ({engine})") + inspector = inspect(engine) + for table_name in inspector.get_table_names(): + print(f"Table: {table_name}") + for column in inspector.get_columns(table_name): + print(f"Column: {column['name']} - {column['type']}") + print("") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Huggingface datasets downloader to use with burn-dataset" + ) + parser.add_argument( + "--name", type=str, help="Name of the dataset to download", required=True + ) + parser.add_argument( + "--file", type=str, help="Base file name where the data is saved", required=True + ) + parser.add_argument( + "--subset", type=str, help="Subset name", required=False, default=None + ) + parser.add_argument( + "--token", + type=str, + help="HuggingFace authentication token", + required=False, + default=None, + ) + parser.add_argument( + "--cache_dir", type=str, help="Cache directory", required=False, default=None + ) + parser.add_argument( + "--data_dir", type=str, help="Relative path to a specific subset of your dataset", required=False, default=None + ) + parser.add_argument( + "--trust_remote_code", + type=bool, + help="Trust remote code", + required=False, + default=None, + ) + + return parser.parse_args() + + +def run(): + args = parse_args() + + download_and_export( + args.name, + args.subset, + args.file, + args.token, + args.data_dir, + args.cache_dir, + args.trust_remote_code, + ) + + +if __name__ == "__main__": + run() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/mod.rs new file mode 100644 index 0000000..a6ccec9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/huggingface/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod downloader; + +pub use downloader::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/mod.rs new file mode 100644 index 0000000..b9ed762 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/source/mod.rs @@ -0,0 +1,3 @@ +/// Huggingface source +#[cfg(any(feature = "sqlite", feature = "sqlite-bundled"))] +pub mod huggingface; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/composed.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/composed.rs new file mode 100644 index 0000000..dfe0a9e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/composed.rs @@ -0,0 +1,56 @@ +use crate::Dataset; + +/// Compose multiple datasets together to create a bigger one. +#[derive(new)] +pub struct ComposedDataset { + datasets: Vec, +} + +impl Dataset for ComposedDataset +where + D: Dataset, + I: Clone, +{ + fn get(&self, index: usize) -> Option { + let mut current_index = 0; + for dataset in self.datasets.iter() { + if index < dataset.len() + current_index { + return dataset.get(index - current_index); + } + current_index += dataset.len(); + } + None + } + fn len(&self) -> usize { + let mut total = 0; + for dataset in self.datasets.iter() { + total += dataset.len(); + } + total + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FakeDataset; + + #[test] + fn test_composed_dataset() { + let dataset1 = FakeDataset::::new(10); + let dataset2 = FakeDataset::::new(5); + + let items1 = dataset1.iter().collect::>(); + let items2 = dataset2.iter().collect::>(); + + let composed = ComposedDataset::new(vec![dataset1, dataset2]); + + assert_eq!(composed.len(), 15); + + let expected_items: Vec = items1.iter().chain(items2.iter()).cloned().collect(); + + let items = composed.iter().collect::>(); + + assert_eq!(items, expected_items); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mapper.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mapper.rs new file mode 100644 index 0000000..f73eec8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mapper.rs @@ -0,0 +1,60 @@ +use crate::Dataset; +use std::marker::PhantomData; + +/// Basic mapper trait to be used with the [mapper dataset](MapperDataset). +pub trait Mapper: Send + Sync { + /// Maps an item of type I to an item of type O. + fn map(&self, item: &I) -> O; +} + +/// Dataset mapping each element in an inner dataset to another element type lazily. +#[derive(new)] +pub struct MapperDataset { + dataset: D, + mapper: M, + input: PhantomData, +} + +impl Dataset for MapperDataset +where + D: Dataset, + M: Mapper + Send + Sync, + I: Send + Sync, + O: Send + Sync, +{ + fn get(&self, index: usize) -> Option { + let item = self.dataset.get(index); + item.map(|item| self.mapper.map(&item)) + } + + fn len(&self) -> usize { + self.dataset.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{InMemDataset, test_data}; + + #[test] + pub fn given_mapper_dataset_when_iterate_should_iterate_though_all_map_items() { + struct StringToFirstChar; + + impl Mapper for StringToFirstChar { + fn map(&self, item: &String) -> String { + let mut item = item.clone(); + item.truncate(1); + item + } + } + + let items_original = test_data::string_items(); + let dataset = InMemDataset::new(items_original); + let dataset = MapperDataset::new(dataset, StringToFirstChar); + + let items: Vec = dataset.iter().collect(); + + assert_eq!(vec!["1", "2", "3", "4"], items); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mod.rs new file mode 100644 index 0000000..a35f57d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/mod.rs @@ -0,0 +1,30 @@ +//! # Dataset Transformations +//! +//! This module provides a collection of [`crate::Dataset`] composition wrappers; +//! providing composition, subset selection, sampling, random shuffling, and windowing. +//! +//! * [`ComposedDataset`] - composes a list of datasets. +//! * [`PartialDataset`] - selects a contiguous index range subset of a dataset. +//! * [`ShuffledDataset`] - a randomly shuffled / mutably shuffle-able dataset; +//! a thin wrapper around [`SelectionDataset`]. +//! * [`SamplerDataset`] - samples a dataset; support for with/without replacement, +//! and under/oversampling. +//! * [`SelectionDataset`] - selects a subset of a dataset via indices; support for shuffling. +//! * [`WindowsDataset`] - creates a sliding window over a dataset. +mod composed; +mod mapper; +mod options; +mod partial; +mod sampler; +mod selection; +mod shuffle; +mod window; + +pub use composed::*; +pub use mapper::*; +pub use options::*; +pub use partial::*; +pub use sampler::*; +pub use selection::*; +pub use shuffle::*; +pub use window::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/options.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/options.rs new file mode 100644 index 0000000..5da2c5e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/options.rs @@ -0,0 +1,199 @@ +use rand::SeedableRng; +use rand::prelude::StdRng; +use rand::rngs::SysRng; + +/// Defines a source for a `StdRng`. +/// +/// # Examples +/// +/// ```rust,no_run +/// use rand::rngs::StdRng; +/// use rand::SeedableRng; +/// use burn_dataset::transform::RngSource; +/// +/// // Default via `StdRng::from_os_rng()` (`RngSource::Default`) +/// let system: RngSource = RngSource::default(); +/// +/// // From a fixed seed (`RngSource::Seed`) +/// let seeded: RngSource = 42.into(); +/// +/// // From an existing rng (`RngSource::Rng`) +/// let rng = StdRng::seed_from_u64(123); +/// let with_rng: RngSource = rng.into(); +/// +/// // Forks the parent RNG to derive an independent, deterministic child RNG. +/// // The original `rng` is modified, and the resulting `RngSource` contains +/// // a new RNG starting from a unique state. +/// let mut rng = StdRng::seed_from_u64(123); +/// let forked: RngSource = (&mut rng).into(); +/// ``` +#[derive(Debug, Default, PartialEq, Eq)] +#[allow(clippy::large_enum_variant)] +pub enum RngSource { + /// Build a new rng from the system. + #[default] + Default, + + /// The rng is passed as a seed. + Seed(u64), + + /// The rng is passed as an option. + Rng(StdRng), +} + +impl From for StdRng { + fn from(source: RngSource) -> Self { + match source { + RngSource::Default => StdRng::try_from_rng(&mut SysRng).unwrap(), + RngSource::Rng(rng) => rng, + RngSource::Seed(seed) => StdRng::seed_from_u64(seed), + } + } +} + +impl From for RngSource { + fn from(seed: u64) -> Self { + Self::Seed(seed) + } +} + +impl From for RngSource { + fn from(rng: StdRng) -> Self { + Self::Rng(rng) + } +} + +/// Derive an independent RNG from a mutable parent RNG. +/// +/// This advances the parent RNG and creates a new RNG seeded from its output. +/// The derived RNG is *not* a clone of the parent's state, but an independent +/// stream (equivalent to `SeedableRng::fork`). +impl From<&mut StdRng> for RngSource { + fn from(rng: &mut StdRng) -> Self { + Self::Rng(rng.fork()) + } +} + +/// Helper option to describe the size of a wrapper, relative to a wrapped object. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub enum SizeConfig { + /// Use the size of the source dataset. + #[default] + Default, + + /// Use the size as a ratio of the source dataset size. + /// + /// Must be >= 0. + Ratio(f64), + + /// Use a fixed size. + Fixed(usize), +} + +impl SizeConfig { + /// Construct a source which will have the same size as the source dataset. + pub fn source() -> Self { + Self::Default + } + + /// Resolve the effective size. + /// + /// ## Arguments + /// + /// - `source_size`: the size of the source dataset. + /// + /// ## Returns + /// + /// The resolved size of the wrapper dataset. + pub fn resolve(self, source_size: usize) -> usize { + match self { + SizeConfig::Default => source_size, + SizeConfig::Ratio(ratio) => { + assert!(ratio >= 0.0, "Ratio must be positive: {ratio}"); + ((source_size as f64) * ratio) as usize + } + SizeConfig::Fixed(size) => size, + } + } +} + +impl From for SizeConfig { + fn from(size: usize) -> Self { + Self::Fixed(size) + } +} + +impl From for SizeConfig { + fn from(ratio: f64) -> Self { + Self::Ratio(ratio) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::SeedableRng; + + #[test] + fn test_rng_source_default() { + let rng_source: RngSource = Default::default(); + assert_eq!(&rng_source, &RngSource::Default); + assert_eq!(&rng_source, &RngSource::default()); + + // Exercise the from_os_rng() call; but we don't know its seed; + let _rng: StdRng = rng_source.into(); + } + + #[test] + fn test_rng_source_seed() { + let rng_source = RngSource::from(42); + assert_eq!(&rng_source, &RngSource::Seed(42)); + + let rng: StdRng = rng_source.into(); + let expected = StdRng::seed_from_u64(42); + + assert_eq!(rng, expected); + } + + #[test] + fn test_rng_source_rng() { + // From StdRng (owned). + { + let original = StdRng::seed_from_u64(42); + + let rng_source = RngSource::from(original); + let rng: StdRng = rng_source.into(); + // No longer clone, but from <> into should not have advanced the state + let original = StdRng::seed_from_u64(42); + assert_eq!(rng, original); + } + + // From &mut StdRng (forks parent) + { + let mut original = StdRng::seed_from_u64(42); + let mut rng = StdRng::seed_from_u64(42); + let rng_forked = rng.fork(); + + let rng_source = RngSource::from(&mut original); + + // Ensure the original was advanced + assert_eq!(original, rng); + + // Ensure the sourced RNG matches the fork + let rng: StdRng = rng_source.into(); + assert_eq!(rng, rng_forked); + } + } + + #[test] + fn test_size_config() { + assert_eq!(SizeConfig::default(), SizeConfig::Default); + + assert_eq!(SizeConfig::from(42), SizeConfig::Fixed(42)); + + assert_eq!(SizeConfig::from(1.5), SizeConfig::Ratio(1.5)); + + assert_eq!(SizeConfig::source(), SizeConfig::Default); + assert_eq!(SizeConfig::source().resolve(50), 50); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/partial.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/partial.rs new file mode 100644 index 0000000..520aa8d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/partial.rs @@ -0,0 +1,206 @@ +use crate::Dataset; +use std::{marker::PhantomData, sync::Arc}; + +/// Only use a fraction of an existing dataset lazily. +#[derive(new, Clone)] +pub struct PartialDataset { + dataset: D, + start_index: usize, + end_index: usize, + input: PhantomData, +} + +impl PartialDataset +where + D: Dataset, +{ + /// Splits a dataset into multiple partial datasets. + pub fn split(dataset: D, num: usize) -> Vec, I>> { + let dataset = Arc::new(dataset); // cheap cloning. + + let mut current = 0; + let mut datasets = Vec::with_capacity(num); + + let batch_size = dataset.len() / num; + + for i in 0..num { + let start = current; + let mut end = current + batch_size; + + if i == (num - 1) { + end = dataset.len(); + } + + let dataset = PartialDataset::new(dataset.clone(), start, end); + + current += batch_size; + datasets.push(dataset); + } + + datasets + } + + /// Splits a dataset by distributing complete chunks/batches across multiple partial datasets. + pub fn split_chunks( + dataset: D, + num: usize, + batch_size: usize, + ) -> Vec, I>> { + let dataset = Arc::new(dataset); // cheap cloning. + let total_items = dataset.len(); + + // Total number of complete batches + let total_batches = total_items.div_ceil(batch_size); + let batches_per_split = total_batches / num; + let extra_batches = total_batches % num; + + let mut datasets = Vec::with_capacity(num); + let mut current_batch = 0; + + for i in 0..num { + // Extra batches distributed across first splits + let split_batches = if i < extra_batches { + batches_per_split + 1 + } else { + batches_per_split + }; + + let start_batch = current_batch; + let end_batch = start_batch + split_batches; + + let start_index = start_batch * batch_size; + let end_index = core::cmp::min(end_batch * batch_size, total_items); + + if start_index < total_items { + datasets.push(PartialDataset::new(dataset.clone(), start_index, end_index)); + } + + current_batch = end_batch; + } + + datasets + } +} + +impl Dataset for PartialDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + fn get(&self, index: usize) -> Option { + let index = index + self.start_index; + if index < self.start_index || index >= self.end_index { + return None; + } + self.dataset.get(index) + } + + fn len(&self) -> usize { + usize::min(self.end_index - self.start_index, self.dataset.len()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FakeDataset; + use std::collections::HashSet; + + #[test] + fn test_start_from_beginning() { + let dataset_original = FakeDataset::::new(27); + let mut items_original_1 = HashSet::new(); + let mut items_original_2 = HashSet::new(); + let mut items_partial = HashSet::new(); + dataset_original.iter().enumerate().for_each(|(i, item)| { + match i >= 10 { + true => items_original_2.insert(item), + false => items_original_1.insert(item), + }; + }); + + let dataset_partial = PartialDataset::new(dataset_original, 0, 10); + + for item in dataset_partial.iter() { + items_partial.insert(item); + } + + assert_eq!(dataset_partial.len(), 10); + assert_eq!(items_original_1, items_partial); + for item in items_original_2 { + assert!(!items_partial.contains(&item)); + } + } + + #[test] + fn test_start_inside() { + let dataset_original = FakeDataset::::new(27); + let mut items_original_1 = HashSet::new(); + let mut items_original_2 = HashSet::new(); + let mut items_partial = HashSet::new(); + + dataset_original.iter().enumerate().for_each(|(i, item)| { + match !(10..20).contains(&i) { + true => items_original_2.insert(item), + false => items_original_1.insert(item), + }; + }); + + let dataset_partial = PartialDataset::new(dataset_original, 10, 20); + for item in dataset_partial.iter() { + items_partial.insert(item); + } + + assert_eq!(dataset_partial.len(), 10); + assert_eq!(items_original_1, items_partial); + for item in items_original_2 { + assert!(!items_partial.contains(&item)); + } + } + + #[test] + fn test_split_contains_all_items_without_duplicates() { + let dataset_original = FakeDataset::::new(27); + let mut items_original = Vec::new(); + let mut items_partial = Vec::new(); + for item in dataset_original.iter() { + items_original.push(item); + } + + let dataset_partials = PartialDataset::split(dataset_original, 4); + let expected_len = [6, 6, 6, 9]; + + for (i, dataset) in dataset_partials.iter().enumerate() { + assert_eq!(dataset.len(), expected_len[i]); + for item in dataset.iter() { + items_partial.push(item); + } + } + + assert_eq!(items_original, items_partial); + } + + #[test] + fn test_split_chunks_contains_all_items_without_duplicates() { + let dataset_original = FakeDataset::::new(27); + let mut items_original = Vec::new(); + let mut items_partial = Vec::new(); + for item in dataset_original.iter() { + items_original.push(item); + } + + let dataset_partials = PartialDataset::split_chunks(dataset_original, 4, 5); + // [(2 * 5), (2 * 5), 5, 2] -> 5 complete chunks + 1 incomplete with 2 remaining items + // OTOH, `split(dataset, 4)` would yield [6, 6, 6, 9] -> 4 incomplete chunks + 4 incomplete with [1, 1, 1, 4] + let expected_len = [10, 10, 5, 2]; + + for (i, dataset) in dataset_partials.iter().enumerate() { + assert_eq!(dataset.len(), expected_len[i]); + for item in dataset.iter() { + items_partial.push(item); + } + } + + assert_eq!(items_original, items_partial); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/sampler.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/sampler.rs new file mode 100644 index 0000000..a4f0268 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/sampler.rs @@ -0,0 +1,438 @@ +use crate::Dataset; +use crate::transform::{RngSource, SizeConfig}; +use rand::prelude::SliceRandom; +use rand::{RngExt, distr::Uniform, rngs::StdRng, seq::IteratorRandom}; +use std::{marker::PhantomData, ops::DerefMut, sync::Mutex}; + +/// Options to configure a [SamplerDataset]. +#[derive(Debug, PartialEq)] +pub struct SamplerDatasetOptions { + /// The sampling mode. + pub replace_samples: bool, + + /// The size source of the wrapper relative to the dataset. + pub size_config: SizeConfig, + + /// The source of the random number generator. + pub rng_source: RngSource, +} + +impl Default for SamplerDatasetOptions { + fn default() -> Self { + Self { + replace_samples: true, + size_config: SizeConfig::Default, + rng_source: RngSource::Default, + } + } +} + +impl From> for SamplerDatasetOptions +where + T: Into, +{ + fn from(option: Option) -> Self { + match option { + Some(option) => option.into(), + None => Self::default(), + } + } +} + +impl From for SamplerDatasetOptions { + fn from(size: usize) -> Self { + Self::default().with_replacement().with_fixed_size(size) + } +} + +impl SamplerDatasetOptions { + /// Set the replacement mode. + pub fn with_replace_samples(self, replace_samples: bool) -> Self { + Self { + replace_samples, + ..self + } + } + + /// Set the replacement mode to WithReplacement. + pub fn with_replacement(self) -> Self { + self.with_replace_samples(true) + } + + /// Set the replacement mode to WithoutReplacement. + pub fn without_replacement(self) -> Self { + self.with_replace_samples(false) + } + + /// Set the size source. + pub fn with_size(self, source: S) -> Self + where + S: Into, + { + Self { + size_config: source.into(), + ..self + } + } + + /// Set the size to the size of the source. + pub fn with_source_size(self) -> Self { + self.with_size(SizeConfig::Default) + } + + /// Set the size to a fixed size. + pub fn with_fixed_size(self, size: usize) -> Self { + self.with_size(size) + } + + /// Set the size to be a multiple of the ration and the source size. + pub fn with_size_ratio(self, size_ratio: f64) -> Self { + self.with_size(size_ratio) + } + + /// Set the `RngSource`. + pub fn with_rng(self, rng: R) -> Self + where + R: Into, + { + Self { + rng_source: rng.into(), + ..self + } + } + + /// Use the system rng. + pub fn with_system_rng(self) -> Self { + self.with_rng(RngSource::Default) + } + + /// Use a rng, built from a seed. + pub fn with_seed(self, seed: u64) -> Self { + self.with_rng(seed) + } +} + +/// Sample items from a dataset. +/// +/// This is a convenient way of modeling a dataset as a probability distribution of a fixed size. +/// You have multiple options to instantiate the dataset sampler. +/// +/// * With replacement (Default): This is the most efficient way of using the sampler because no state is +/// required to keep indices that have been selected. +/// +/// * Without replacement: This has a similar effect to using a +/// [shuffled dataset](crate::transform::ShuffledDataset), but with more flexibility since you can +/// set the dataset to an arbitrary size. Once every item has been used, a new cycle is +/// created with a new random suffle. +pub struct SamplerDataset { + dataset: D, + size: usize, + state: Mutex, + input: PhantomData, +} +enum SamplerState { + WithReplacement(StdRng), + WithoutReplacement(StdRng, Vec), +} + +impl SamplerDataset +where + D: Dataset, + I: Send + Sync, +{ + /// Creates a new sampler dataset with replacement. + /// + /// When the sample size is less than or equal to the source dataset size, + /// data will be sampled without replacement from the source dataset in + /// a uniformly shuffled order. + /// + /// When the sample size is greater than the source dataset size, + /// the entire source dataset will be sampled once for every multiple + /// of the size ratios; with the remaining samples taken without replacement + /// uniformly from the source. All samples will be returned uniformly shuffled. + /// + /// ## Arguments + /// + /// * `dataset`: the dataset to wrap. + /// * `options`: the options to configure the sampler dataset. + /// + /// ## Examples + /// ```rust,ignore + /// use burn_dataset::transform::{ + /// SamplerDataset, + /// SamplerDatasetOptions, + /// }; + /// + /// // Examples below assuming `dataset.len()` = `10`. + /// + /// // sample size: 5 + /// // WithReplacement + /// // rng: StdRng::from_os_rng() + /// SamplerDataset::new(dataset, 5); + /// + /// // sample size: 10 (source) + /// // WithReplacement + /// // rng: StdRng::from_os_rng() + /// SamplerDataset::new(dataset, SamplerDatasetOptions::default()); + /// + /// // sample size: 15 + /// // WithoutReplacement + /// // rng: StdRng::seed_from_u64(42) + /// SamplerDataset::new( + /// dataset, + /// SamplerDatasetOptions::default() + /// .with_size(1.5) + /// .without_replacement() + /// .with_rng(42), + /// ); + /// ``` + pub fn new(dataset: D, options: O) -> Self + where + O: Into, + { + let options = options.into(); + let size = options.size_config.resolve(dataset.len()); + let rng = options.rng_source.into(); + Self { + dataset, + size, + state: Mutex::new(match options.replace_samples { + true => SamplerState::WithReplacement(rng), + false => SamplerState::WithoutReplacement(rng, Vec::with_capacity(size)), + }), + input: PhantomData, + } + } + + /// Creates a new sampler dataset with replacement. + /// + /// # Arguments + /// + /// - `dataset`: the dataset to wrap. + /// - `size`: the effective size of the sampled dataset. + pub fn with_replacement(dataset: D, size: usize) -> Self { + Self::new( + dataset, + SamplerDatasetOptions::default() + .with_replacement() + .with_fixed_size(size), + ) + } + + /// Creates a new sampler dataset without replacement. + /// + /// When the sample size is less than or equal to the source dataset size, + /// data will be sampled without replacement from the source dataset in + /// a uniformly shuffled order. + /// + /// When the sample size is greater than the source dataset size, + /// the entire source dataset will be sampled once for every multiple + /// of the size ratios; with the remaining samples taken without replacement + /// uniformly from the source. All samples will be returned uniformly shuffled. + /// + /// # Arguments + /// - `dataset`: the dataset to wrap. + /// - `size`: the effective size of the sampled dataset. + pub fn without_replacement(dataset: D, size: usize) -> Self { + Self::new( + dataset, + SamplerDatasetOptions::default() + .without_replacement() + .with_fixed_size(size), + ) + } + + /// Determines if the sampler is using the "with replacement" strategy. + /// + /// # Returns + /// - `true`: If the sampler is configured to sample with replacement. + /// - `false`: If the sampler is configured to sample without replacement. + pub fn is_with_replacement(&self) -> bool { + match self.state.lock().unwrap().deref_mut() { + SamplerState::WithReplacement(_) => true, + SamplerState::WithoutReplacement(_, _) => false, + } + } + + fn index(&self) -> usize { + match self.state.lock().unwrap().deref_mut() { + SamplerState::WithReplacement(rng) => { + rng.sample(Uniform::new(0, self.dataset.len()).unwrap()) + } + SamplerState::WithoutReplacement(rng, indices) => { + if indices.is_empty() { + // Refill the state. + let idx_range = 0..self.dataset.len(); + for _ in 0..(self.size / self.dataset.len()) { + // No need to `.choose_multiple` here because we're using + // the entire source range; and `.choose_multiple` will + // not return a random sample anyway. + indices.extend(idx_range.clone()) + } + + // From `choose_multiple` documentation: + // > Although the elements are selected randomly, the order of elements in + // > the buffer is neither stable nor fully random. If random ordering is + // > desired, shuffle the result. + indices.extend(idx_range.sample(rng, self.size - indices.len())); + + // The real shuffling is done here. + indices.shuffle(rng); + } + + indices.pop().expect("Indices are refilled when empty.") + } + } + } +} + +impl Dataset for SamplerDataset +where + D: Dataset, + I: Send + Sync, +{ + fn get(&self, index: usize) -> Option { + if index >= self.size { + return None; + } + + self.dataset.get(self.index()) + } + + fn len(&self) -> usize { + self.size + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::bool_assert_comparison)] + + use super::*; + use crate::FakeDataset; + use rand::SeedableRng; + use std::collections::HashMap; + + #[test] + fn test_samplerdataset_options() { + let options = SamplerDatasetOptions::default(); + assert_eq!(options.replace_samples, true); + assert_eq!(options.size_config, SizeConfig::Default); + assert_eq!(options.rng_source, RngSource::Default); + + // ReplacementMode + let options = options.with_replace_samples(false); + assert_eq!(options.replace_samples, false); + let options = options.with_replacement(); + assert_eq!(options.replace_samples, true); + let options = options.without_replacement(); + assert_eq!(options.replace_samples, false); + + // SourceSize + let options = options.with_size(SizeConfig::Default); + assert_eq!(options.size_config, SizeConfig::Default); + let options = options.with_source_size(); + assert_eq!(options.size_config, SizeConfig::Default); + let options = options.with_fixed_size(10); + assert_eq!(options.size_config, SizeConfig::Fixed(10)); + let options = options.with_size_ratio(1.5); + assert_eq!(options.size_config, SizeConfig::Ratio(1.5)); + + // RngSource + let options = options.with_system_rng(); + assert_eq!(options.rng_source, RngSource::Default); + let options = options.with_seed(42); + assert_eq!(options.rng_source, RngSource::Seed(42)); + let rng = StdRng::seed_from_u64(9); + let options = options.with_rng(rng); + assert!(matches!(options.rng_source, RngSource::Rng(_))); + } + + #[test] + fn sampler_dataset_constructors_test() { + let ds = SamplerDataset::new(FakeDataset::::new(10), 15); + assert_eq!(ds.len(), 15); + assert_eq!(ds.dataset.len(), 10); + assert!(ds.is_with_replacement()); + + let ds = SamplerDataset::with_replacement(FakeDataset::::new(10), 15); + assert_eq!(ds.len(), 15); + assert_eq!(ds.dataset.len(), 10); + assert!(ds.is_with_replacement()); + + let ds = SamplerDataset::without_replacement(FakeDataset::::new(10), 15); + assert_eq!(ds.len(), 15); + assert_eq!(ds.dataset.len(), 10); + assert!(!ds.is_with_replacement()); + } + + #[test] + fn sampler_dataset_with_replacement_iter() { + let factor = 3; + let len_original = 10; + let dataset_sampler = SamplerDataset::with_replacement( + FakeDataset::::new(len_original), + len_original * factor, + ); + let mut total = 0; + + for _item in dataset_sampler.iter() { + total += 1; + } + + assert_eq!(total, factor * len_original); + } + + #[test] + fn sampler_dataset_without_replacement_bucket_test() { + let factor = 3; + let len_original = 10; + + let dataset_sampler = SamplerDataset::new( + FakeDataset::::new(len_original), + SamplerDatasetOptions::default() + .without_replacement() + .with_size_ratio(factor as f64), + ); + + let mut buckets = HashMap::new(); + + for item in dataset_sampler.iter() { + let count = match buckets.get(&item) { + Some(count) => count + 1, + None => 1, + }; + + buckets.insert(item, count); + } + + let mut total = 0; + for count in buckets.into_values() { + assert_eq!(count, factor); + total += count; + } + assert_eq!(total, factor * len_original); + } + + #[test] + fn sampler_dataset_without_replacement_uniform_order_test() { + // This is a reversion test on the indices.shuffle(rng) call in SamplerDataset::index(). + let size = 1000; + let dataset_sampler = + SamplerDataset::without_replacement(FakeDataset::::new(size), size); + + let indices: Vec<_> = (0..size).map(|_| dataset_sampler.index()).collect(); + let mean_delta = indices + .windows(2) + .map(|pair| pair[1].abs_diff(pair[0])) + .sum::() as f64 + / (size - 1) as f64; + + let expected = (size + 2) as f64 / 3.0; + + assert!( + (mean_delta - expected).abs() <= 0.25 * expected, + "Sampled indices are not uniformly distributed: mean_delta: {mean_delta}, expected: {expected}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/selection.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/selection.rs new file mode 100644 index 0000000..283be57 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/selection.rs @@ -0,0 +1,374 @@ +use crate::Dataset; +use crate::transform::RngSource; +use rand::prelude::SliceRandom; +use rand::rngs::StdRng; +use std::marker::PhantomData; +use std::sync::Arc; + +/// Generates a vector of indices from 0 to size - 1. +/// +/// # Arguments +/// +/// * `size` - The size of the dataset. +/// +/// # Returns +/// +/// A vector containing indices from 0 to size - 1. +#[inline(always)] +pub fn iota(size: usize) -> Vec { + (0..size).collect() +} + +/// Generates a shuffled vector of indices up to a size. +/// +/// # Arguments +/// +/// * `size` - The size of the dataset to shuffle. +/// +/// # Returns +/// +/// A vector of shuffled indices. +#[inline(always)] +pub fn shuffled_indices(size: usize, rng: &mut StdRng) -> Vec { + let mut indices = iota(size); + indices.shuffle(rng); + indices +} + +/// A dataset that selects a subset of indices from an existing dataset. +/// +/// Indices may appear multiple times, but they must be within the bounds of the original dataset. +#[derive(Clone)] +pub struct SelectionDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + /// The wrapped dataset from which to select indices. + pub wrapped: Arc, + + /// The indices to select from the wrapped dataset. + pub indices: Vec, + + input: PhantomData, +} + +impl SelectionDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + /// Creates a new selection dataset with the given dataset and indices. + /// + /// Checks that all indices are within the bounds of the dataset. + /// + /// # Arguments + /// + /// * `dataset` - The original dataset to select from. + /// * `indices` - A slice of indices to select from the dataset. + /// These indices must be within the bounds of the dataset. + /// + /// # Panics + /// + /// Panics if any index is out of bounds for the dataset. + pub fn from_indices_checked(dataset: S, indices: Vec) -> Self + where + S: Into>, + { + let dataset = dataset.into(); + + let size = dataset.len(); + if let Some(idx) = indices.iter().find(|&i| *i >= size) { + panic!("Index out of bounds for wrapped dataset size: {idx} >= {size}"); + } + + Self::from_indices_unchecked(dataset, indices) + } + + /// Creates a new selection dataset with the given dataset and indices without checking bounds. + /// + /// # Arguments + /// + /// * `dataset` - The original dataset to select from. + /// * `indices` - A vector of indices to select from the dataset. + /// + /// # Safety + /// + /// This function does not check if the indices are within the bounds of the dataset. + pub fn from_indices_unchecked(dataset: S, indices: Vec) -> Self + where + S: Into>, + { + Self { + wrapped: dataset.into(), + indices, + input: PhantomData, + } + } + + /// Creates a new selection dataset that selects all indices from the dataset. + /// + /// This allocates a 1-to-1 mapping of indices to the dataset size, + /// essentially functioning as a no-op selection. This is only useful + /// when the dataset will later be shuffled or transformed in place. + /// + /// # Arguments + /// + /// * `dataset` - The original dataset to select from. + /// + /// # Returns + /// + /// A new `SelectionDataset` that selects all indices from the dataset. + pub fn new_select_all(dataset: S) -> Self + where + S: Into>, + { + let dataset = dataset.into(); + let size = dataset.len(); + Self::from_indices_unchecked(dataset, iota(size)) + } + + /// Creates a new selection dataset with shuffled indices. + /// + /// Selects every index of the dataset and shuffles them + /// with randomness from the provided random number generator. + /// + /// # Arguments + /// + /// * `dataset` - The original dataset to select from. + /// * `rng` - A mutable reference to a random number generator. + /// + /// # Returns + /// + /// A new `SelectionDataset` with shuffled indices. + pub fn new_shuffled(dataset: S, rng_source: R) -> Self + where + S: Into>, + R: Into, + { + let mut this = Self::new_select_all(dataset); + this.shuffle(rng_source); + this + } + + /// Shuffles the indices of the dataset using a mutable random number generator. + /// + /// This method modifies the dataset in place, shuffling the indices. + /// + /// # Arguments + /// + /// * `rng` - A mutable reference to a random number generator. + pub fn shuffle(&mut self, rng_source: R) + where + R: Into, + { + let mut rng: StdRng = rng_source.into().into(); + self.indices.shuffle(&mut rng) + } + + /// Creates a new dataset that is a slice of the current selection dataset. + /// + /// Slices the *selection indices* from ``[start..end]``. + /// + /// Independent of future shuffles on the parent, but shares the same wrapped dataset. + /// + /// + /// # Arguments + /// + /// * `start` - The start of the range. + /// * `end` - The end of the range (exclusive). + // TODO: SliceArg in burn-tensor should be lifted to burn-std; this should use SliceArg. + pub fn slice(&self, start: usize, end: usize) -> Self { + Self::from_indices_unchecked(self.wrapped.clone(), self.indices[start..end].to_vec()) + } + + /// Split into `num` datasets by slicing the selection indices evenly. + /// + /// Split is done via `slice`, so the datasets share the same wrapped dataset. + /// + /// Independent of future shuffles on the parent, but shares the same wrapped dataset. + /// + /// # Arguments + /// + /// * `num` - The number of datasets to split into. + /// + /// # Returns + /// + /// A vector of `SelectionDataset` instances, each containing a subset of the indices. + pub fn split(&self, num: usize) -> Vec { + let n = self.indices.len(); + + let mut current = 0; + let mut datasets = Vec::with_capacity(num); + + let batch_size = n / num; + for i in 0..num { + let start = current; + let mut end = current + batch_size; + + if i == (num - 1) { + end = n; + } + + let dataset = self.slice(start, end); + + current += batch_size; + datasets.push(dataset); + } + + datasets + } +} + +impl Dataset for SelectionDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + fn get(&self, index: usize) -> Option { + let index = self.indices.get(index)?; + self.wrapped.get(*index) + } + + fn len(&self) -> usize { + self.indices.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FakeDataset; + use rand::SeedableRng; + + #[test] + fn test_iota() { + let size = 10; + let indices = iota(size); + assert_eq!(indices.len(), size); + assert_eq!(indices, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + } + + #[test] + fn test_shuffled_indices_same_seed_is_deterministic() { + let size = 10; + + let mut rng1 = StdRng::seed_from_u64(10); + // `StdRng` is no longer `Clone`, so its internal state cannot be duplicated. + // To test determinism, we must explicitly create a second RNG from the same seed. + let mut rng2 = StdRng::seed_from_u64(10); + + let mut expected = iota(size); + expected.shuffle(&mut rng1); + + let indices = shuffled_indices(size, &mut rng2); + + assert_eq!(indices, expected); + } + + #[test] + fn test_shuffled_indices_forked_rngs_differ() { + let size = 10; + + let mut rng1 = StdRng::seed_from_u64(10); + let mut rng2 = rng1.fork(); + + let mut a = iota(size); + let mut b = iota(size); + + a.shuffle(&mut rng1); + b.shuffle(&mut rng2); + + assert_ne!(a, b); + } + + #[should_panic(expected = "Index out of bounds for wrapped dataset size: 300 >= 27")] + #[test] + fn test_from_indices_checked_panics() { + let source_dataset = FakeDataset::::new(27); + let indices: Vec = vec![15, 1, 12, 300]; + SelectionDataset::from_indices_checked(source_dataset, indices); + } + + #[test] + fn test_checked_selection_dataset() { + let source_dataset = FakeDataset::::new(27); + + let indices: Vec = vec![15, 1, 12, 12]; + let expected: Vec = indices + .iter() + .map(|i| source_dataset.get(*i).unwrap()) + .collect(); + + let selection = SelectionDataset::from_indices_checked(source_dataset, indices.clone()); + + assert_eq!(&selection.indices, &indices); + + let items = selection.iter().collect::>(); + + assert_eq!(items, expected); + } + + #[test] + fn test_shuffled_dataset() { + let dataset = FakeDataset::::new(27); + let source_items = dataset.iter().collect::>(); + + let selection = SelectionDataset::new_shuffled(dataset, 42); + + let indices = shuffled_indices(source_items.len(), &mut StdRng::seed_from_u64(42)); + + assert_eq!(&selection.indices, &indices); + assert_eq!(selection.len(), source_items.len()); + + let expected_items: Vec<_> = indices + .iter() + .map(|&i| source_items[i].to_string()) + .collect(); + assert_eq!(&selection.iter().collect::>(), &expected_items); + } + + #[test] + fn test_slice() { + let dataset = FakeDataset::::new(27); + let source_items = dataset.iter().collect::>(); + + let selection = SelectionDataset::new_select_all(dataset); + + let start = 5; + let end = 15; + let sliced_selection = selection.slice(start, end); + + assert_eq!(sliced_selection.len(), end - start); + + #[allow(clippy::needless_range_loop)] + for i in start..end { + assert_eq!( + sliced_selection.get(i - start), + Some(source_items[i].to_string()) + ); + } + } + + #[test] + fn test_split() { + let dataset = FakeDataset::::new(28); + let source_items = dataset.iter().collect::>(); + + let selection = SelectionDataset::new_select_all(dataset); + + let split_contents: Vec> = selection + .split(3) + .iter() + .map(|d| d.iter().collect::>()) + .collect(); + assert_eq!( + split_contents, + vec![ + source_items[0..9].to_vec(), + source_items[9..18].to_vec(), + source_items[18..28].to_vec(), + ] + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/shuffle.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/shuffle.rs new file mode 100644 index 0000000..8e93e49 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/shuffle.rs @@ -0,0 +1,109 @@ +use crate::Dataset; +use crate::transform::{RngSource, SelectionDataset}; + +/// A Shuffled a dataset. +/// +/// This is a thin wrapper around a [SelectionDataset] which selects and shuffles +/// the full indices of the original dataset. +/// +/// Consider using [SelectionDataset] if you are only interested in +/// shuffling mechanisms. +/// +/// Consider using [sampler dataset](crate::transform::SamplerDataset) if you +/// want a probability distribution which is computed lazily. +pub struct ShuffledDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + wrapped: SelectionDataset, +} + +impl ShuffledDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + /// Creates a new selection dataset with shuffled indices. + /// + /// This is a thin wrapper around `SelectionDataset::new_shuffled`. + /// + /// # Arguments + /// + /// * `dataset` - The original dataset to select from. + /// * `rng_source` - The source of the random number generator. + /// + /// # Returns + /// + /// A new `ShuffledDataset`. + pub fn new(dataset: D, rng_source: R) -> Self + where + R: Into, + { + Self { + wrapped: SelectionDataset::new_shuffled(dataset, rng_source), + } + } + + /// Creates a new selection dataset with shuffled indices using a fixed seed. + /// + /// This is a thin wrapper around `SelectionDataset::new_shuffled_with_seed`. + /// + /// # Arguments + /// + /// * `dataset` - The original dataset to select from. + /// * `seed` - A fixed seed for the random number generator. + /// + /// # Returns + /// + /// A new `ShuffledDataset`. + #[deprecated(since = "0.19.0", note = "Use `new(dataset, seed)` instead`")] + pub fn with_seed(dataset: D, seed: u64) -> Self { + Self::new(dataset, seed) + } +} + +impl Dataset for ShuffledDataset +where + D: Dataset, + I: Clone + Send + Sync, +{ + fn get(&self, index: usize) -> Option { + self.wrapped.get(index) + } + + fn len(&self) -> usize { + self.wrapped.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FakeDataset; + use crate::transform::selection::shuffled_indices; + use rand::SeedableRng; + use rand::prelude::StdRng; + + #[test] + fn test_shuffled_dataset() { + let dataset = FakeDataset::::new(27); + let source_items = dataset.iter().collect::>(); + + let seed = 42; + + #[allow(deprecated)] + let shuffled = ShuffledDataset::with_seed(dataset, seed); + + let mut rng = StdRng::seed_from_u64(seed); + let indices = shuffled_indices(source_items.len(), &mut rng); + + assert_eq!(shuffled.len(), source_items.len()); + + let expected_items: Vec<_> = indices + .iter() + .map(|&i| source_items[i].to_string()) + .collect(); + assert_eq!(&shuffled.iter().collect::>(), &expected_items); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/window.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/window.rs new file mode 100644 index 0000000..e6bb8b9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/transform/window.rs @@ -0,0 +1,290 @@ +use std::{cmp::max, marker::PhantomData, num::NonZeroUsize}; + +use crate::Dataset; + +/// Functionality to create a window. +pub trait Window { + /// Creates a window of a collection. + /// + /// # Returns + /// + /// A `Vec` representing the window. + fn window(&self, current: usize, size: NonZeroUsize) -> Option>; +} + +impl + ?Sized> Window for T { + fn window(&self, current: usize, size: NonZeroUsize) -> Option> { + (current..current + size.get()) + .map(|x| self.get(x)) + .collect() + } +} + +/// Functionality to create a `WindowsIterator`. +pub trait Windows { + /// Creates and returns an iterator over all the windows of length `size`. + fn windows(&self, size: usize) -> WindowsIterator<'_, I>; +} + +impl> Windows for T { + /// Is empty if the `Dataset` is shorter than `size`. + /// + /// # Panics + /// + /// Panics if `size` is 0. + /// + /// # Examples + /// + /// ``` + /// use crate::burn_dataset::{ + /// transform::{Windows, WindowsDataset}, + /// Dataset, InMemDataset, + /// }; + /// + /// let items = [1, 2, 3, 4].to_vec(); + /// let dataset = InMemDataset::new(items.clone()); + /// + /// for window in dataset.windows(2) { + /// // do sth with window + /// } + /// ``` + fn windows(&self, size: usize) -> WindowsIterator<'_, I> { + let size = NonZeroUsize::new(size).expect("window size must be non-zero"); + WindowsIterator::new(self, size) + } +} + +/// Overlapping windows iterator. +pub struct WindowsIterator<'a, I> { + /// The size of the windows. + pub size: NonZeroUsize, + current: usize, + dataset: &'a dyn Dataset, +} + +impl<'a, I> WindowsIterator<'a, I> { + /// Creates a new `WindowsIterator` instance. The windows overlap. + /// Is empty if the input `Dataset` is shorter than `size`. + /// + /// # Parameters + /// + /// - `dataset`: The dataset over which windows will be created. + /// - `size`: The size of the windows. + pub fn new(dataset: &'a dyn Dataset, size: NonZeroUsize) -> Self { + WindowsIterator { + current: 0, + dataset, + size, + } + } +} + +impl Iterator for WindowsIterator<'_, I> { + type Item = Vec; + + fn next(&mut self) -> Option> { + self.current += 1; + self.dataset.window(self.current - 1, self.size) + } +} + +impl Clone for WindowsIterator<'_, I> { + fn clone(&self) -> Self { + WindowsIterator { + size: self.size, + dataset: self.dataset, + current: self.current, + } + } +} + +/// Dataset designed to work with overlapping windows of data. +pub struct WindowsDataset { + /// The size of the windows. + pub size: NonZeroUsize, + dataset: D, + input: PhantomData, +} + +impl WindowsDataset +where + D: Dataset, +{ + /// Creates a new `WindowsDataset` instance. The windows overlap. + /// Is empty if the input `Dataset` is shorter than `size`. + /// + /// # Parameters + /// + /// - `dataset`: The dataset over which windows will be created. + /// - `size`: The size of the windows. + pub fn new(dataset: D, size: usize) -> Self + where + D:, + { + let size = NonZeroUsize::new(size).expect("window size must be non-zero"); + WindowsDataset:: { + size, + dataset, + input: PhantomData, + } + } +} + +impl Dataset> for WindowsDataset +where + D: Dataset, + I: Send + Sync, +{ + /// Retrieves a window of items from the dataset. + /// + /// # Parameters + /// + /// - `index`: The index of the window. + /// + /// # Returns + /// + /// A vector representing the window. + fn get(&self, index: usize) -> Option> { + self.dataset.window(index, self.size) + } + + /// Retrieves the number of windows in the dataset. + /// + /// # Returns + /// + /// A size representing the number of windows. + fn len(&self) -> usize { + let len = self.dataset.len() as isize - self.size.get() as isize + 1; + max(len, 0) as usize + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::{ + Dataset, InMemDataset, + transform::{Windows, WindowsDataset}, + }; + + #[rstest] + pub fn windows_should_be_equal_to_vec_windows() { + let items = [1, 2, 3, 4, 5].to_vec(); + let dataset = InMemDataset::new(items.clone()); + let expected = items + .windows(3) + .map(|x| x.to_vec()) + .collect::>>(); + + let result = dataset.windows(3).collect::>>(); + + assert_eq!(result, expected); + } + + #[rstest] + pub fn windows_dataset_should_be_equal_to_vec_windows() { + let items = [1, 2, 3, 4, 5].to_vec(); + let dataset = InMemDataset::new(items.clone()); + let expected = items + .windows(3) + .map(|x| x.to_vec()) + .collect::>>(); + + let result = WindowsDataset::new(dataset, 3) + .iter() + .collect::>>(); + + assert_eq!(result, expected); + } + + #[rstest] + pub fn cloned_iterator_should_be_equal() { + let items = [1, 2, 3, 4, 5].to_vec(); + let dataset = InMemDataset::new(items.clone()); + let original = dataset.windows(4); + + let cloned = original.clone(); + + assert!(std::ptr::eq(cloned.dataset, original.dataset)); + assert_eq!(cloned.size, original.size); + assert_eq!(cloned.current, original.current); + } + + #[rstest] + pub fn cloned_iterator_should_be_unaffected() { + let items = [1, 2, 3, 4, 5].to_vec(); + let dataset = InMemDataset::new(items.clone()); + let mut original = dataset.windows(4); + + let cloned = original.clone(); + original.current = 2; + + assert_ne!(cloned.current, original.current); + } + + #[rstest] + #[should_panic(expected = "window size must be non-zero")] + pub fn windows_should_panic() { + let items = [1, 2].to_vec(); + let dataset = InMemDataset::new(items.clone()); + + dataset.windows(0); + } + + #[rstest] + #[should_panic(expected = "window size must be non-zero")] + pub fn new_window_dataset_should_panic() { + let items = [1, 2].to_vec(); + let dataset = InMemDataset::new(items.clone()); + + WindowsDataset::new(dataset, 0); + } + + #[rstest] + pub fn window_dataset_len_should_be_equal() { + let dataset = InMemDataset::new([1, 2, 3, 4].to_vec()); + + let result = WindowsDataset::new(dataset, 2).len(); + + assert_eq!(result, 3); + } + + #[rstest] + pub fn window_iterator_should_be_empty() { + let dataset = InMemDataset::new([1, 2].to_vec()); + let mut peekable = dataset.windows(4).peekable(); + + let result = peekable.peek(); + + assert_eq!(result, None); + } + + #[rstest] + pub fn window_dataset_len_should_be_zero() { + let dataset = InMemDataset::new([1, 2].to_vec()); + + let result = WindowsDataset::new(dataset, 4).len(); + + assert_eq!(result, 0); + } + + #[rstest] + pub fn window_dataset_get_should_be_equal() { + let dataset = InMemDataset::new([1, 2, 3, 4].to_vec()); + let expected = Some([1, 2, 3].to_vec()); + + let result = WindowsDataset::new(dataset, 3).get(0); + + assert_eq!(result, expected); + } + + #[rstest] + pub fn window_dataset_get_should_be_none() { + let dataset = InMemDataset::new([1, 2].to_vec()); + + let result = WindowsDataset::new(dataset, 4).get(0); + + assert_eq!(result, None); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/cifar.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/cifar.rs new file mode 100644 index 0000000..1598db2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/cifar.rs @@ -0,0 +1,241 @@ +//! CIFAR Dataset Module +//! +//! This module provides functionality for loading the CIFAR-10 and CIFAR-100 image classification datasets. +//! CIFAR (Canadian Institute For Advanced Research) datasets are widely used benchmarks in computer vision, +//! consisting of 32×32 pixel color images split into training (50,000 images) and test (10,000 images) sets. +//! +//! ## Dataset Variants +//! - **CIFAR-10**: Contains 10 distinct classes (e.g., airplane, automobile, bird, cat) +//! - CIFAR-10 mirror from [fastai](https://github.com/fastai/fastai/blob/master/fastai/data/external.py#L44). +//! - Licensed under the [Apache License](https://github.com/fastai/fastai/blob/master/LICENSE). +//! - **CIFAR-100**: Contains 100 fine-grained classes (e.g., beaver, dolphin, oak tree) +//! - CIFAR-100 mirror from [fastai](https://github.com/fastai/fastai/blob/master/fastai/data/external.py#L75). +//! - Licensed under the [Apache License](https://github.com/fastai/fastai/blob/master/LICENSE). +//! +//! ## Usage Example +//! ```rust +//! use burn_dataset::vision::CifarDataset; +//! use burn_dataset::vision::CifarType; +//! +//! // Create a CIFAR-10 dataset accessor +//! let dataset = CifarDataset::new(CifarType::Cifar10); +//! +//! // Access training and test sets +//! let train_dataset = dataset.train(); +//! let test_dataset = dataset.test(); +//! ``` +//! ```rust +//! use burn_dataset::vision::CifarDataset; +//! use burn_dataset::vision::CifarType; +//! +//! // Create a CIFAR-100 dataset accessor +//! let dataset = CifarDataset::new(CifarType::Cifar100); +//! +//! // Access training and test sets +//! let train_dataset = dataset.train(); +//! let test_dataset = dataset.test(); +//! ``` + +use std::{path::PathBuf, sync::Mutex}; + +use flate2::read::GzDecoder; +use tar::Archive; + +use crate::network::downloader; +use crate::vision::ImageFolderDataset; + +/// CIFAR-10 mirror from [fastai](https://github.com/fastai/fastai/blob/master/fastai/data/external.py#L44). +/// Licensed under the [Apache License](https://github.com/fastai/fastai/blob/master/LICENSE). +const CIFAR10_URL: &str = "https://s3.amazonaws.com/fast-ai-sample/cifar10.tgz"; + +/// CIFAR-100 mirror from [fastai](https://github.com/fastai/fastai/blob/master/fastai/data/external.py#L75). +/// Licensed under the [Apache License](https://github.com/fastai/fastai/blob/master/LICENSE). +const CIFAR100_URL: &str = "https://s3.amazonaws.com/fast-ai-imageclas/cifar100.tgz"; + +/// Enum representing the types of CIFAR datasets available. +/// +/// CIFAR (Canadian Institute For Advanced Research) datasets are widely used benchmarks for image classification. +/// This enum provides support for the two main CIFAR datasets. +#[derive(Debug, Clone, Copy)] +#[allow(dead_code)] +pub enum CifarType { + /// CIFAR-10 dataset containing 10 classes with 60,000 images in total. + Cifar10, + /// CIFAR-100 dataset containing 100 classes with 60,000 images in total. + Cifar100, +} + +/// CIFAR dataset accessor. +/// +/// This struct provides convenient access to the CIFAR-10 and CIFAR-100 image classification datasets. +/// It automatically downloads (if not already downloaded), extracts, and loads the datasets. +/// +/// All images in CIFAR datasets are 32×32 pixel color images, with 50,000 images in the training set +/// and 10,000 images in the test set. +/// +/// ## Differences between datasets +/// - **CIFAR-10**: Contains 10 mutually exclusive classes such as airplane, automobile, bird, cat, etc. +/// - **CIFAR-100**: Contains 100 fine-grained classes such as beaver, dolphin, etc. +pub struct CifarDataset { + cifar_dir: PathBuf, +} + +impl CifarDataset { + /// Creates a new CIFAR dataset accessor. + /// + /// # Arguments + /// * `cifar_type` - Specifies whether to use CIFAR-10 or CIFAR-100 dataset + pub fn new(cifar_type: CifarType) -> Self { + Self { + cifar_dir: download(&cifar_type), + } + } + + /// Gets the training dataset. + /// + /// # Returns + /// An `ImageFolderDataset` instance containing 50,000 training images + pub fn train(&self) -> ImageFolderDataset { + ImageFolderDataset::new_classification(self.cifar_dir.join("train")).unwrap() + } + + /// Gets the test dataset. + /// + /// # Returns + /// An `ImageFolderDataset` instance containing 10,000 test images + pub fn test(&self) -> ImageFolderDataset { + ImageFolderDataset::new_classification(self.cifar_dir.join("test")).unwrap() + } +} + +/// CIFAR dataset download lock. +/// +/// This lock ensures that only one thread downloads the CIFAR dataset at a time. +static DOWNLOAD_LOCK: Mutex<()> = Mutex::new(()); + +fn download(cifar_type: &CifarType) -> PathBuf { + // Acquire the lock. This will block if another thread already holds the lock. + let _lock = DOWNLOAD_LOCK.lock().unwrap(); + + // Dataset files are stored in the burn-dataset cache directory + let cache_dir = dirs::cache_dir() + .expect("Could not get cache directory") + .join("burn-dataset"); + + // Cifar store directory + let cifar_dir = match cifar_type { + CifarType::Cifar10 => cache_dir.join("cifar10"), + CifarType::Cifar100 => cache_dir.join("cifar100"), + }; + + // Cifar dataset url + let url = match cifar_type { + CifarType::Cifar10 => CIFAR10_URL, + CifarType::Cifar100 => CIFAR100_URL, + }; + + // Cifar dataset archive filename + let filename = match cifar_type { + CifarType::Cifar10 => "cifar10.tgz", + CifarType::Cifar100 => "cifar100.tgz", + }; + + // Check for already downloaded content + if !cifar_dir.exists() { + // Download gzip file + let bytes = downloader::download_file_as_bytes(url, filename); + + // Decode gzip file content and unpack archive + let gz_buffer = GzDecoder::new(&bytes[..]); + let mut archive = Archive::new(gz_buffer); + archive.unpack(cache_dir).unwrap(); + } + + cifar_dir +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Dataset, vision::Annotation}; + + /// CIFAR dataset length + const TRAINDATASET_LEN: usize = 50000; + const TESTDATASET_LEN: usize = 10000; + + /// CIFAR-10 label range + const CIFAR10_LABEL_MIN: usize = 0; + const CIFAR10_LABEL_MAX: usize = 9; + + /// CIFAR-100 label range + const CIFAR100_LABEL_MIN: usize = 0; + const CIFAR100_LABEL_MAX: usize = 99; + + #[test] + fn test_cifar10_download() { + let cifar_dir = download(&CifarType::Cifar10); + assert!(cifar_dir.exists()); + } + + #[test] + fn test_cifar100_download() { + let cifar_dir = download(&CifarType::Cifar100); + assert!(cifar_dir.exists()); + } + + #[test] + fn test_cifar10_len() { + let dataset = CifarDataset::new(CifarType::Cifar10); + let train_dataset = dataset.train(); + let test_dataset = dataset.test(); + assert_eq!(train_dataset.len(), TRAINDATASET_LEN); + assert_eq!(test_dataset.len(), TESTDATASET_LEN); + } + + #[test] + fn test_cifar100_len() { + let dataset = CifarDataset::new(CifarType::Cifar100); + let train_dataset = dataset.train(); + let test_dataset = dataset.test(); + assert_eq!(train_dataset.len(), TRAINDATASET_LEN); + assert_eq!(test_dataset.len(), TESTDATASET_LEN); + } + + #[test] + fn test_cifar10_label_range() { + let dataset = CifarDataset::new(CifarType::Cifar10); + let test_dataset = dataset.test(); + let (min, max) = get_label_range(&test_dataset); + assert_eq!(min, CIFAR10_LABEL_MIN); + assert_eq!(max, CIFAR10_LABEL_MAX); + } + + #[test] + fn test_cifar100_label_range() { + let dataset = CifarDataset::new(CifarType::Cifar100); + let test_dataset = dataset.test(); + let (min, max) = get_label_range(&test_dataset); + assert_eq!(min, CIFAR100_LABEL_MIN); + assert_eq!(max, CIFAR100_LABEL_MAX); + } + + fn get_label_range(dataset: &ImageFolderDataset) -> (usize, usize) { + let labels: Vec<_> = dataset.iter().map(|item| item.annotation).collect(); + let mut min = 128; + let mut max = 0; + for label in labels { + let index = match label { + Annotation::Label(index) => index, + _ => 0, + }; + if index < min { + min = index; + } + if index > max { + max = index; + } + } + + (min, max) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/image_folder.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/image_folder.rs new file mode 100644 index 0000000..63d0ad5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/image_folder.rs @@ -0,0 +1,1124 @@ +use crate::transform::{Mapper, MapperDataset}; +use crate::{Dataset, InMemDataset}; + +use globwalk::{self, DirEntry}; +use image::{self, ColorType}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use thiserror::Error; + +const SUPPORTED_FILES: [&str; 4] = ["bmp", "jpg", "jpeg", "png"]; +const BBOX_MIN_NUM_VALUES: usize = 4; + +/// Image data type. +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum PixelDepth { + /// 8-bit unsigned. + U8(u8), + /// 16-bit unsigned. + U16(u16), + /// 32-bit floating point. + F32(f32), +} + +impl TryFrom for u8 { + type Error = &'static str; + + fn try_from(value: PixelDepth) -> Result { + if let PixelDepth::U8(v) = value { + Ok(v) + } else { + Err("Value is not u8") + } + } +} + +impl TryFrom for u16 { + type Error = &'static str; + + fn try_from(value: PixelDepth) -> Result { + if let PixelDepth::U16(v) = value { + Ok(v) + } else { + Err("Value is not u16") + } + } +} + +impl TryFrom for f32 { + type Error = &'static str; + + fn try_from(value: PixelDepth) -> Result { + if let PixelDepth::F32(v) = value { + Ok(v) + } else { + Err("Value is not f32") + } + } +} + +/// Annotation type for different tasks. +#[derive(Debug, Clone, PartialEq)] +pub enum Annotation { + /// Image-level label. + Label(usize), + /// Multiple image-level labels. + MultiLabel(Vec), + /// Object bounding boxes. + BoundingBoxes(Vec), + /// Segmentation mask. + SegmentationMask(SegmentationMask), +} + +/// Segmentation mask annotation. +/// For semantic segmentation, a mask has a single channel (C = 1). +/// For instance segmentation, there may be multiple masks per image (C >= 1). +#[derive(Debug, Clone, PartialEq)] +pub struct SegmentationMask { + /// Segmentation mask. + pub mask: Vec, +} + +/// Object detection bounding box annotation. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct BoundingBox { + /// Coordinates in [x_min, y_min, width, height] format. + pub coords: [f32; 4], + + /// Box class label. + pub label: usize, +} + +/// Image dataset item. +#[derive(Debug, Clone, PartialEq)] +pub struct ImageDatasetItem { + /// Image as a vector with a valid image type. + pub image: Vec, + + /// Original source image width. + pub image_width: usize, + + /// Original source image height. + pub image_height: usize, + + /// Annotation for the image. + pub annotation: Annotation, + + /// Original image source. + pub image_path: String, +} + +/// Raw annotation types. +#[derive(Deserialize, Serialize, Debug, Clone)] +enum AnnotationRaw { + Label(String), + MultiLabel(Vec), + BoundingBoxes(Vec), + SegmentationMask(PathBuf), +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +struct ImageDatasetItemRaw { + /// Image path. + image_path: PathBuf, + + /// Image annotation. + annotation: AnnotationRaw, +} + +impl ImageDatasetItemRaw { + fn new>(image_path: P, annotation: AnnotationRaw) -> ImageDatasetItemRaw { + ImageDatasetItemRaw { + image_path: image_path.as_ref().to_path_buf(), + annotation, + } + } +} + +struct PathToImageDatasetItem { + classes: HashMap, +} + +fn segmentation_mask_to_vec_usize(mask_path: &PathBuf) -> Vec { + // Load image from disk + let image = image::open(mask_path).unwrap(); + + // Image as Vec + // if rgb8 or rgb16, keep only the first channel assuming all channels are the same + + match image.color() { + ColorType::L8 => image.into_luma8().iter().map(|&x| x as usize).collect(), + ColorType::L16 => image.into_luma16().iter().map(|&x| x as usize).collect(), + ColorType::Rgb8 => image + .into_rgb8() + .iter() + .step_by(3) + .map(|&x| x as usize) + .collect(), + ColorType::Rgb16 => image + .into_rgb16() + .iter() + .step_by(3) + .map(|&x| x as usize) + .collect(), + _ => panic!("Unrecognized image color type"), + } +} + +/// Parse the image annotation to the corresponding type. +fn parse_image_annotation( + annotation: &AnnotationRaw, + classes: &HashMap, +) -> Annotation { + // TODO: add support for other annotations + // - [ ] Object bounding boxes + // - [x] Segmentation mask + // For now, only image classification labels and segmentation are supported. + + // Map class string to label id + match annotation { + AnnotationRaw::Label(name) => Annotation::Label(*classes.get(name).unwrap()), + AnnotationRaw::MultiLabel(names) => Annotation::MultiLabel( + names + .iter() + .map(|name| *classes.get(name).unwrap()) + .collect(), + ), + AnnotationRaw::SegmentationMask(mask_path) => { + Annotation::SegmentationMask(SegmentationMask { + mask: segmentation_mask_to_vec_usize(mask_path), + }) + } + AnnotationRaw::BoundingBoxes(v) => Annotation::BoundingBoxes(v.clone()), + } +} + +/// Retrieve all available classes from the COCO JSON +fn parse_coco_classes( + json: &serde_json::Value, +) -> Result, ImageLoaderError> { + let mut classes = HashMap::new(); + + if let Some(json_classes) = json["categories"].as_array() { + for class in json_classes { + let id = class["id"] + .as_u64() + .ok_or_else(|| ImageLoaderError::ParsingError("Invalid class ID".to_string())) + .and_then(|v| { + usize::try_from(v).map_err(|_| { + ImageLoaderError::ParsingError("Class ID out of usize range".to_string()) + }) + })?; + + let name = class["name"] + .as_str() + .filter(|&s| !s.is_empty()) + .ok_or_else(|| ImageLoaderError::ParsingError("Invalid class name".to_string()))? + .to_string(); + + classes.insert(name, id); + } + } + + if classes.is_empty() { + return Err(ImageLoaderError::ParsingError( + "No classes found in annotations".to_string(), + )); + } + + Ok(classes) +} + +/// Retrieve annotations from COCO JSON +fn parse_coco_bbox_annotations( + json: &serde_json::Value, +) -> Result, ImageLoaderError> { + let mut annotations = HashMap::new(); + + if let Some(json_annotations) = json["annotations"].as_array() { + for annotation in json_annotations { + let image_id = annotation["image_id"].as_u64().ok_or_else(|| { + ImageLoaderError::ParsingError("Invalid image ID in annotation".into()) + })?; + + let class_id = annotation["category_id"] + .as_u64() + .ok_or_else(|| { + ImageLoaderError::ParsingError("Invalid class ID in annotations".to_string()) + }) + .and_then(|v| { + usize::try_from(v).map_err(|_| { + ImageLoaderError::ParsingError( + "Class ID in annotations out of usize range".to_string(), + ) + }) + })?; + + let bbox_coords = annotation["bbox"] + .as_array() + .ok_or_else(|| ImageLoaderError::ParsingError("missing bbox array".to_string()))? + .iter() + .map(|v| { + v.as_f64() + .ok_or_else(|| { + ImageLoaderError::ParsingError("invalid bbox value".to_string()) + }) + .map(|val| val as f32) + }) + .collect::, _>>()?; + + if bbox_coords.len() < BBOX_MIN_NUM_VALUES { + return Err(ImageLoaderError::ParsingError(format!( + "not enough bounding box coordinates in annotation for image {image_id}", + ))); + } + + let bbox = BoundingBox { + coords: [ + bbox_coords[0], + bbox_coords[1], + bbox_coords[2], + bbox_coords[3], + ], + label: class_id, + }; + + annotations + .entry(image_id) + .and_modify(|entry| { + if let AnnotationRaw::BoundingBoxes(bboxes) = entry { + bboxes.push(bbox.clone()); + } + }) + .or_insert_with(|| AnnotationRaw::BoundingBoxes(vec![bbox])); + } + } + + if annotations.is_empty() { + return Err(ImageLoaderError::ParsingError( + "no annotations found".to_string(), + )); + } + + Ok(annotations) +} + +/// Retrieve all available images from the COCO JSON +fn parse_coco_images>( + images_path: &P, + mut annotations: HashMap, + json: &serde_json::Value, +) -> Result, ImageLoaderError> { + let mut images = Vec::new(); + if let Some(json_images) = json["images"].as_array() { + for image in json_images { + let image_id = image["id"].as_u64().ok_or_else(|| { + ImageLoaderError::ParsingError("Invalid image ID in image list".to_string()) + })?; + + let file_name = image["file_name"] + .as_str() + .ok_or_else(|| ImageLoaderError::ParsingError("Invalid image ID".to_string()))? + .to_string(); + + let mut image_path = images_path.as_ref().to_path_buf(); + image_path.push(file_name); + + if !image_path.exists() { + return Err(ImageLoaderError::IOError(format!( + "Image {} not found", + image_path.display() + ))); + } + + let annotation = annotations + .remove(&image_id) + .unwrap_or_else(|| AnnotationRaw::BoundingBoxes(Vec::new())); + + images.push(ImageDatasetItemRaw { + annotation, + image_path, + }); + } + } + + if images.is_empty() { + return Err(ImageLoaderError::ParsingError( + "No images found in annotations".to_string(), + )); + } + + Ok(images) +} + +impl Mapper for PathToImageDatasetItem { + /// Convert a raw image dataset item (path-like) to a 3D image array with a target label. + fn map(&self, item: &ImageDatasetItemRaw) -> ImageDatasetItem { + let annotation = parse_image_annotation(&item.annotation, &self.classes); + + // Load image from disk + let image = image::open(&item.image_path).unwrap(); + + // Save image dimensions for manipulation + let img_width = image.width() as usize; + let img_height = image.height() as usize; + + // Image as Vec + let img_vec = match image.color() { + ColorType::L8 => image + .into_luma8() + .iter() + .map(|&x| PixelDepth::U8(x)) + .collect(), + ColorType::La8 => image + .into_luma_alpha8() + .iter() + .map(|&x| PixelDepth::U8(x)) + .collect(), + ColorType::L16 => image + .into_luma16() + .iter() + .map(|&x| PixelDepth::U16(x)) + .collect(), + ColorType::La16 => image + .into_luma_alpha16() + .iter() + .map(|&x| PixelDepth::U16(x)) + .collect(), + ColorType::Rgb8 => image + .into_rgb8() + .iter() + .map(|&x| PixelDepth::U8(x)) + .collect(), + ColorType::Rgba8 => image + .into_rgba8() + .iter() + .map(|&x| PixelDepth::U8(x)) + .collect(), + ColorType::Rgb16 => image + .into_rgb16() + .iter() + .map(|&x| PixelDepth::U16(x)) + .collect(), + ColorType::Rgba16 => image + .into_rgba16() + .iter() + .map(|&x| PixelDepth::U16(x)) + .collect(), + ColorType::Rgb32F => image + .into_rgb32f() + .iter() + .map(|&x| PixelDepth::F32(x)) + .collect(), + ColorType::Rgba32F => image + .into_rgba32f() + .iter() + .map(|&x| PixelDepth::F32(x)) + .collect(), + _ => panic!("Unrecognized image color type"), + }; + + ImageDatasetItem { + image: img_vec, + image_width: img_width, + image_height: img_height, + annotation, + image_path: item.image_path.display().to_string(), + } + } +} + +/// Error type for [ImageFolderDataset](ImageFolderDataset). +#[derive(Error, Debug)] +pub enum ImageLoaderError { + /// Unknown error. + #[error("unknown: `{0}`")] + Unknown(String), + + /// I/O operation error. + #[error("I/O error: `{0}`")] + IOError(String), + + /// Invalid file error. + #[error("Invalid file extension: `{0}`")] + InvalidFileExtensionError(String), + + /// Parsing error. + #[error("Parsing error: `{0}`")] + ParsingError(String), +} + +type ImageDatasetMapper = + MapperDataset, PathToImageDatasetItem, ImageDatasetItemRaw>; + +/// A generic dataset to load images from disk. +pub struct ImageFolderDataset { + dataset: ImageDatasetMapper, +} + +impl Dataset for ImageFolderDataset { + fn get(&self, index: usize) -> Option { + self.dataset.get(index) + } + + fn len(&self) -> usize { + self.dataset.len() + } +} + +impl ImageFolderDataset { + /// Create an image classification dataset from the root folder. + /// + /// # Arguments + /// + /// * `root` - Dataset root folder. + /// + /// # Returns + /// A new dataset instance. + pub fn new_classification>(root: P) -> Result { + // New dataset containing any of the supported file types + ImageFolderDataset::new_classification_with(root, &SUPPORTED_FILES) + } + + /// Create an image classification dataset from the root folder. + /// The included images are filtered based on the provided extensions. + /// + /// # Arguments + /// + /// * `root` - Dataset root folder. + /// * `extensions` - List of allowed extensions. + /// + /// # Returns + /// A new dataset instance. + pub fn new_classification_with( + root: P, + extensions: &[S], + ) -> Result + where + P: AsRef, + S: AsRef, + { + // Glob all images with extensions + let walker = globwalk::GlobWalkerBuilder::from_patterns( + root.as_ref(), + &[format!( + "*.{{{}}}", // "*.{ext1,ext2,ext3} + extensions + .iter() + .map(Self::check_extension) + .collect::, _>>()? + .join(",") + )], + ) + .follow_links(true) + .sort_by(|p1: &DirEntry, p2: &DirEntry| p1.path().cmp(p2.path())) // order by path + .build() + .map_err(|err| ImageLoaderError::Unknown(format!("{err:?}")))? + .filter_map(Result::ok); + + // Get all dataset items + let mut items = Vec::new(); + let mut classes = HashSet::new(); + for img in walker { + let image_path = img.path(); + + // Label name is represented by the parent folder name + let label = image_path + .parent() + .ok_or_else(|| { + ImageLoaderError::IOError("Could not resolve image parent folder".to_string()) + })? + .file_name() + .ok_or_else(|| { + ImageLoaderError::IOError( + "Could not resolve image parent folder name".to_string(), + ) + })? + .to_string_lossy() + .into_owned(); + + classes.insert(label.clone()); + + items.push(ImageDatasetItemRaw::new( + image_path, + AnnotationRaw::Label(label), + )) + } + + // Sort class names + let mut classes = classes.into_iter().collect::>(); + classes.sort(); + + Self::with_items(items, &classes) + } + + /// Create an image classification dataset with the specified items. + /// + /// # Arguments + /// + /// * `items` - List of dataset items, each item represented by a tuple `(image path, label)`. + /// * `classes` - Dataset class names. + /// + /// # Returns + /// A new dataset instance. + pub fn new_classification_with_items, S: AsRef>( + items: Vec<(P, String)>, + classes: &[S], + ) -> Result { + // Parse items and check valid image extension types + let items = items + .into_iter() + .map(|(path, label)| { + // Map image path and label + let path = path.as_ref(); + let label = AnnotationRaw::Label(label); + + Self::check_extension(&path.extension().unwrap().to_str().unwrap())?; + + Ok(ImageDatasetItemRaw::new(path, label)) + }) + .collect::, _>>()?; + + Self::with_items(items, classes) + } + + /// Create a multi-label image classification dataset with the specified items. + /// + /// # Arguments + /// + /// * `items` - List of dataset items, each item represented by a tuple `(image path, labels)`. + /// * `classes` - Dataset class names. + /// + /// # Returns + /// A new dataset instance. + pub fn new_multilabel_classification_with_items, S: AsRef>( + items: Vec<(P, Vec)>, + classes: &[S], + ) -> Result { + // Parse items and check valid image extension types + let items = items + .into_iter() + .map(|(path, labels)| { + // Map image path and multi-label + let path = path.as_ref(); + let labels = AnnotationRaw::MultiLabel(labels); + + Self::check_extension(&path.extension().unwrap().to_str().unwrap())?; + + Ok(ImageDatasetItemRaw::new(path, labels)) + }) + .collect::, _>>()?; + + Self::with_items(items, classes) + } + + /// Create an image segmentation dataset with the specified items. + /// + /// # Arguments + /// + /// * `items` - List of dataset items, each item represented by a tuple `(image path, annotation path)`. + /// * `classes` - Dataset class names. + /// + /// # Returns + /// A new dataset instance. + pub fn new_segmentation_with_items, S: AsRef>( + items: Vec<(P, P)>, + classes: &[S], + ) -> Result { + // Parse items and check valid image extension types + let items = items + .into_iter() + .map(|(image_path, mask_path)| { + // Map image path and segmentation mask path + let image_path = image_path.as_ref(); + let annotation = AnnotationRaw::SegmentationMask(mask_path.as_ref().to_path_buf()); + + Self::check_extension(&image_path.extension().unwrap().to_str().unwrap())?; + + Ok(ImageDatasetItemRaw::new(image_path, annotation)) + }) + .collect::, _>>()?; + + Self::with_items(items, classes) + } + + /// Create a COCO detection dataset based on the annotations JSON and image directory. + /// + /// # Arguments + /// + /// * `annotations_json` - Path to the JSON file containing annotations in COCO format (for + /// example instances_train2017.json). + /// + /// * `images_path` - Path containing the images matching the annotations JSON. + /// + /// # Returns + /// A new dataset instance. + pub fn new_coco_detection, I: AsRef>( + annotations_json: A, + images_path: I, + ) -> Result { + let file = fs::File::open(annotations_json) + .map_err(|e| ImageLoaderError::IOError(format!("Failed to open annotations: {e}")))?; + let json: Value = serde_json::from_reader(file).map_err(|e| { + ImageLoaderError::ParsingError(format!("Failed to parse annotations: {e}")) + })?; + + let classes = parse_coco_classes(&json)?; + let annotations = parse_coco_bbox_annotations(&json)?; + let items = parse_coco_images(&images_path, annotations, &json)?; + let dataset = InMemDataset::new(items); + let mapper = PathToImageDatasetItem { classes }; + let dataset = MapperDataset::new(dataset, mapper); + + Ok(Self { dataset }) + } + + /// Create an image dataset with the specified items. + /// + /// # Arguments + /// + /// * `items` - Raw dataset items. + /// * `classes` - Dataset class names. + /// + /// # Returns + /// A new dataset instance. + fn with_items>( + items: Vec, + classes: &[S], + ) -> Result { + // NOTE: right now we don't need to validate the supported image files since + // the method is private. We assume it's already validated. + let dataset = InMemDataset::new(items); + + // Class names to index map + let classes = classes.iter().map(|c| c.as_ref()).collect::>(); + let classes_map: HashMap<_, _> = classes + .into_iter() + .enumerate() + .map(|(idx, cls)| (cls.to_string(), idx)) + .collect(); + + let mapper = PathToImageDatasetItem { + classes: classes_map, + }; + let dataset = MapperDataset::new(dataset, mapper); + + Ok(Self { dataset }) + } + + /// Check if extension is supported. + fn check_extension>(extension: &S) -> Result { + let extension = extension.as_ref(); + if !SUPPORTED_FILES.contains(&extension) { + Err(ImageLoaderError::InvalidFileExtensionError( + extension.to_string(), + )) + } else { + Ok(extension.to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + const DATASET_ROOT: &str = "tests/data/image_folder"; + const SEGMASK_ROOT: &str = "tests/data/segmask_folder"; + const COCO_JSON: &str = "tests/data/dataset_coco.json"; + const COCO_IMAGES: &str = "tests/data/image_folder_coco"; + + #[test] + pub fn image_folder_dataset() { + let dataset = ImageFolderDataset::new_classification(DATASET_ROOT).unwrap(); + + // Dataset has 3 elements + assert_eq!(dataset.len(), 3); + assert_eq!(dataset.get(3), None); + + // Dataset elements should be: orange (0), red (1), red (1) + assert_eq!(dataset.get(0).unwrap().annotation, Annotation::Label(0)); + assert_eq!(dataset.get(1).unwrap().annotation, Annotation::Label(1)); + assert_eq!(dataset.get(2).unwrap().annotation, Annotation::Label(1)); + } + + #[test] + pub fn image_folder_dataset_filtered() { + let dataset = ImageFolderDataset::new_classification_with(DATASET_ROOT, &["jpg"]).unwrap(); + + // Filtered dataset has 2 elements + assert_eq!(dataset.len(), 2); + assert_eq!(dataset.get(2), None); + + // Dataset elements should be: orange (0), red (1) + assert_eq!(dataset.get(0).unwrap().annotation, Annotation::Label(0)); + assert_eq!(dataset.get(1).unwrap().annotation, Annotation::Label(1)); + } + + #[test] + pub fn image_folder_dataset_with_items_sizes() { + let root = Path::new(DATASET_ROOT); + let items = vec![ + (root.join("orange").join("dot.jpg"), "orange".to_string()), + (root.join("red").join("dot.jpg"), "red".to_string()), + (root.join("red").join("dot.png"), "red".to_string()), + ]; + let dataset = + ImageFolderDataset::new_classification_with_items(items, &["orange", "red"]).unwrap(); + + // Dataset has 3 elements + assert_eq!(dataset.len(), 3); + assert_eq!(dataset.get(3), None); + + // Test item sizes + + assert_eq!( + ( + dataset.get(0).unwrap().image_width, + dataset.get(0).unwrap().image_height + ), + (1, 1) + ); + assert_eq!( + ( + dataset.get(1).unwrap().image_width, + dataset.get(1).unwrap().image_height + ), + (1, 1) + ); + assert_eq!( + ( + dataset.get(2).unwrap().image_width, + dataset.get(2).unwrap().image_height + ), + (1, 1) + ); + } + + #[test] + pub fn image_folder_dataset_with_items() { + let root = Path::new(DATASET_ROOT); + let items = vec![ + (root.join("orange").join("dot.jpg"), "orange".to_string()), + (root.join("red").join("dot.jpg"), "red".to_string()), + (root.join("red").join("dot.png"), "red".to_string()), + ]; + let dataset = + ImageFolderDataset::new_classification_with_items(items, &["orange", "red"]).unwrap(); + + // Dataset has 3 elements + assert_eq!(dataset.len(), 3); + assert_eq!(dataset.get(3), None); + + // Dataset elements should be: orange (0), red (1), red (1) + assert_eq!(dataset.get(0).unwrap().annotation, Annotation::Label(0)); + assert_eq!(dataset.get(1).unwrap().annotation, Annotation::Label(1)); + assert_eq!(dataset.get(2).unwrap().annotation, Annotation::Label(1)); + } + + #[test] + pub fn image_folder_dataset_multilabel() { + let root = Path::new(DATASET_ROOT); + let items = vec![ + ( + root.join("orange").join("dot.jpg"), + vec!["dot".to_string(), "orange".to_string()], + ), + ( + root.join("red").join("dot.jpg"), + vec!["dot".to_string(), "red".to_string()], + ), + ( + root.join("red").join("dot.png"), + vec!["dot".to_string(), "red".to_string()], + ), + ]; + let dataset = ImageFolderDataset::new_multilabel_classification_with_items( + items, + &["dot", "orange", "red"], + ) + .unwrap(); + + // Dataset has 3 elements + assert_eq!(dataset.len(), 3); + assert_eq!(dataset.get(3), None); + + // Dataset elements should be: [dot, orange] (0, 1), [dot, red] (0, 2), [dot, red] (0, 2) + assert_eq!( + dataset.get(0).unwrap().annotation, + Annotation::MultiLabel(vec![0, 1]) + ); + assert_eq!( + dataset.get(1).unwrap().annotation, + Annotation::MultiLabel(vec![0, 2]) + ); + assert_eq!( + dataset.get(2).unwrap().annotation, + Annotation::MultiLabel(vec![0, 2]) + ); + } + + #[test] + #[should_panic] + pub fn image_folder_dataset_invalid_extension() { + // Some invalid file extension + let _ = ImageFolderDataset::new_classification_with(DATASET_ROOT, &["ico"]).unwrap(); + } + + #[test] + pub fn pixel_depth_try_into_u8() { + let val = u8::MAX; + let pix: u8 = PixelDepth::U8(val).try_into().unwrap(); + assert_eq!(pix, val); + } + + #[test] + #[should_panic] + pub fn pixel_depth_try_into_u8_invalid() { + let _: u8 = PixelDepth::U16(u8::MAX as u16 + 1).try_into().unwrap(); + } + + #[test] + pub fn pixel_depth_try_into_u16() { + let val = u16::MAX; + let pix: u16 = PixelDepth::U16(val).try_into().unwrap(); + assert_eq!(pix, val); + } + + #[test] + #[should_panic] + pub fn pixel_depth_try_into_u16_invalid() { + let _: u16 = PixelDepth::F32(u16::MAX as f32).try_into().unwrap(); + } + + #[test] + pub fn pixel_depth_try_into_f32() { + let val = f32::MAX; + let pix: f32 = PixelDepth::F32(val).try_into().unwrap(); + assert_eq!(pix, val); + } + + #[test] + #[should_panic] + pub fn pixel_depth_try_into_f32_invalid() { + let _: f32 = PixelDepth::U16(u16::MAX).try_into().unwrap(); + } + + #[test] + pub fn parse_image_annotation_label_string() { + let classes = HashMap::from([("0".to_string(), 0_usize), ("1".to_string(), 1_usize)]); + let anno = AnnotationRaw::Label("0".to_string()); + assert_eq!( + parse_image_annotation(&anno, &classes), + Annotation::Label(0) + ); + } + + #[test] + pub fn parse_image_annotation_multilabel_string() { + let classes = HashMap::from([ + ("0".to_string(), 0_usize), + ("1".to_string(), 1_usize), + ("2".to_string(), 2_usize), + ]); + let anno = AnnotationRaw::MultiLabel(vec!["0".to_string(), "2".to_string()]); + assert_eq!( + parse_image_annotation(&anno, &classes), + Annotation::MultiLabel(vec![0, 2]) + ); + } + + #[test] + pub fn segmask_image_path_to_vec_usize() { + let root = Path::new(SEGMASK_ROOT); + + // checkerboard mask + const TEST_CHECKERBOARD_MASK_PATTERN: [u8; 64] = [ + 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, + 1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, + 2, 1, 2, 1, 2, 1, + ]; + assert_eq!( + TEST_CHECKERBOARD_MASK_PATTERN + .iter() + .map(|&x| x as usize) + .collect::>(), + segmentation_mask_to_vec_usize(&root.join("annotations").join("mask_checkerboard.png")), + ); + + // random 2 colors mask + const TEST_RANDOM2COLORS_MASK_PATTERN: [u8; 64] = [ + 1, 2, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, + 2, 1, 1, 2, 2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 1, + 1, 1, 1, 1, 1, 1, + ]; + assert_eq!( + TEST_RANDOM2COLORS_MASK_PATTERN + .iter() + .map(|&x| x as usize) + .collect::>(), + segmentation_mask_to_vec_usize( + &root.join("annotations").join("mask_random_2colors.png") + ), + ); + // random 3 colors mask + const TEST_RANDOM3COLORS_MASK_PATTERN: [u8; 64] = [ + 3, 1, 3, 3, 1, 1, 3, 2, 3, 3, 3, 3, 1, 3, 2, 1, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 3, 3, + 3, 2, 3, 2, 2, 3, 2, 3, 3, 1, 3, 1, 3, 3, 1, 1, 3, 2, 1, 2, 2, 2, 1, 2, 1, 2, 3, 3, 1, + 3, 3, 2, 1, 2, 2, + ]; + assert_eq!( + TEST_RANDOM3COLORS_MASK_PATTERN + .iter() + .map(|&x| x as usize) + .collect::>(), + segmentation_mask_to_vec_usize( + &root.join("annotations").join("mask_random_3colors.png") + ), + ); + } + + #[test] + pub fn segmask_folder_dataset() { + let root = Path::new(SEGMASK_ROOT); + + let items = vec![ + ( + root.join("images").join("image_checkerboard.png"), + root.join("annotations").join("mask_checkerboard.png"), + ), + ( + root.join("images").join("image_random_2colors.png"), + root.join("annotations").join("mask_random_2colors.png"), + ), + ( + root.join("images").join("image_random_3colors.png"), + root.join("annotations").join("mask_random_3colors.png"), + ), + ]; + let dataset = ImageFolderDataset::new_segmentation_with_items( + items, + &[ + "foo", // 0 + "bar", // 1 + "baz", // 2 + "qux", // 3 + ], + ) + .unwrap(); + + // Dataset has 3 elements; each (image, annotation) is a single item + assert_eq!(dataset.len(), 3); + assert_eq!(dataset.get(3), None); + + // checkerboard mask + const TEST_CHECKERBOARD_MASK_PATTERN: [u8; 64] = [ + 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, + 1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, + 2, 1, 2, 1, 2, 1, + ]; + assert_eq!( + dataset.get(0).unwrap().annotation, + Annotation::SegmentationMask(SegmentationMask { + mask: TEST_CHECKERBOARD_MASK_PATTERN + .iter() + .map(|&x| x as usize) + .collect() + }) + ); + // random 2 colors mask + const TEST_RANDOM2COLORS_MASK_PATTERN: [u8; 64] = [ + 1, 2, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, + 2, 1, 1, 2, 2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 1, + 1, 1, 1, 1, 1, 1, + ]; + assert_eq!( + dataset.get(1).unwrap().annotation, + Annotation::SegmentationMask(SegmentationMask { + mask: TEST_RANDOM2COLORS_MASK_PATTERN + .iter() + .map(|&x| x as usize) + .collect() + }) + ); + // random 3 colors mask + const TEST_RANDOM3COLORS_MASK_PATTERN: [u8; 64] = [ + 3, 1, 3, 3, 1, 1, 3, 2, 3, 3, 3, 3, 1, 3, 2, 1, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 3, 3, + 3, 2, 3, 2, 2, 3, 2, 3, 3, 1, 3, 1, 3, 3, 1, 1, 3, 2, 1, 2, 2, 2, 1, 2, 1, 2, 3, 3, 1, + 3, 3, 2, 1, 2, 2, + ]; + assert_eq!( + dataset.get(2).unwrap().annotation, + Annotation::SegmentationMask(SegmentationMask { + mask: TEST_RANDOM3COLORS_MASK_PATTERN + .iter() + .map(|&x| x as usize) + .collect() + }) + ); + } + + #[test] + pub fn coco_detection_dataset() { + let dataset = ImageFolderDataset::new_coco_detection(COCO_JSON, COCO_IMAGES).unwrap(); + assert_eq!(dataset.len(), 3); // we have only three images defined + assert_eq!(dataset.get(3), None); + + const TWO_DOTS_AND_TRIANGLE_B1: BoundingBox = BoundingBox { + coords: [3.125_172, 18.090_784, 10.960_11, 10.740_027], + label: 0, + }; + + const TWO_DOTS_AND_TRIANGLE_B2: BoundingBox = BoundingBox { + coords: [3.257_221_5, 3.037_139, 10.563_961, 10.828_06], + label: 0, + }; + + const TWO_DOTS_AND_TRIANGLE_B3: BoundingBox = BoundingBox { + coords: [15.097_662, 3.389_271, 12.632_737, 11.180_193], + label: 1, + }; + + const DOTS_TRIANGLE_B1: BoundingBox = BoundingBox { + coords: [3.125_172, 17.914_719, 10.828_06, 11.004_127], + label: 0, + }; + + const DOTS_TRIANGLE_B2: BoundingBox = BoundingBox { + coords: [15.273_727, 3.301_238, 12.192_573, 11.708_39], + label: 1, + }; + + const ONE_DOT_B1: BoundingBox = BoundingBox { + coords: [10.079_78, 9.595_598, 10.960_11, 11.356_258], + label: 0, + }; + + for item in dataset.iter() { + let file_name = Path::new(&item.image_path).file_name().unwrap(); + match item.annotation { + // check if the number of bounding boxes is correct + Annotation::BoundingBoxes(v) => { + if file_name == "two_dots_and_triangle.jpg" { + assert_eq!(v.len(), 3); + assert!(v.contains(&TWO_DOTS_AND_TRIANGLE_B1)); + assert!(v.contains(&TWO_DOTS_AND_TRIANGLE_B2)); + assert!(v.contains(&TWO_DOTS_AND_TRIANGLE_B3)); + } else if file_name == "dot_triangle.jpg" { + assert_eq!(v.len(), 2); + assert!(v.contains(&DOTS_TRIANGLE_B1)); + assert!(v.contains(&DOTS_TRIANGLE_B2)); + } else if file_name == "one_dot.jpg" { + assert_eq!(v.len(), 1); + assert!(v.contains(&ONE_DOT_B1)); + } else { + panic!("{}", format!("unexpected image name: {}", item.image_path)); + } + } + _ => panic!("unexpected annotation"), + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mnist.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mnist.rs new file mode 100644 index 0000000..909c779 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mnist.rs @@ -0,0 +1,221 @@ +use std::fs::{File, create_dir_all}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use flate2::read::GzDecoder; +use serde::{Deserialize, Serialize}; + +use crate::{ + Dataset, InMemDataset, + transform::{Mapper, MapperDataset}, +}; + +use crate::network::downloader::download_file_as_bytes; + +// CVDF mirror of http://yann.lecun.com/exdb/mnist/ +const URL: &str = "https://storage.googleapis.com/cvdf-datasets/mnist/"; +const TRAIN_IMAGES: &str = "train-images-idx3-ubyte"; +const TRAIN_LABELS: &str = "train-labels-idx1-ubyte"; +const TEST_IMAGES: &str = "t10k-images-idx3-ubyte"; +const TEST_LABELS: &str = "t10k-labels-idx1-ubyte"; + +const WIDTH: usize = 28; +const HEIGHT: usize = 28; + +/// MNIST item. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct MnistItem { + /// Image as a 2D array of floats. + pub image: [[f32; WIDTH]; HEIGHT], + + /// Label of the image. + pub label: u8, +} + +#[derive(Deserialize, Debug, Clone)] +struct MnistItemRaw { + pub image_bytes: Vec, + pub label: u8, +} + +struct BytesToImage; + +impl Mapper for BytesToImage { + /// Convert a raw MNIST item (image bytes) to a MNIST item (2D array image). + fn map(&self, item: &MnistItemRaw) -> MnistItem { + // Ensure the image dimensions are correct. + debug_assert_eq!(item.image_bytes.len(), WIDTH * HEIGHT); + + // Convert the image to a 2D array of floats. + let mut image_array = [[0f32; WIDTH]; HEIGHT]; + for (i, pixel) in item.image_bytes.iter().enumerate() { + let x = i % WIDTH; + let y = i / HEIGHT; + image_array[y][x] = *pixel as f32; + } + + MnistItem { + image: image_array, + label: item.label, + } + } +} + +type MappedDataset = MapperDataset, BytesToImage, MnistItemRaw>; + +/// The MNIST dataset consists of 70,000 28x28 black-and-white images in 10 classes (one for each digits), with 7,000 +/// images per class. There are 60,000 training images and 10,000 test images. +/// +/// The data is downloaded from the web from the [CVDF mirror](https://github.com/cvdfoundation/mnist). +pub struct MnistDataset { + dataset: MappedDataset, +} + +impl Dataset for MnistDataset { + fn get(&self, index: usize) -> Option { + self.dataset.get(index) + } + + fn len(&self) -> usize { + self.dataset.len() + } +} + +impl MnistDataset { + /// Creates a new train dataset. + pub fn train() -> Self { + Self::new("train") + } + + /// Creates a new test dataset. + pub fn test() -> Self { + Self::new("test") + } + + fn new(split: &str) -> Self { + // Download dataset + let root = MnistDataset::download(split); + + // MNIST is tiny so we can load it in-memory + // Train images (u8): 28 * 28 * 60000 = 47.04Mb + // Test images (u8): 28 * 28 * 10000 = 7.84Mb + let images = MnistDataset::read_images(&root, split); + let labels = MnistDataset::read_labels(&root, split); + + // Collect as vector of MnistItemRaw + let items: Vec<_> = images + .into_iter() + .zip(labels) + .map(|(image_bytes, label)| MnistItemRaw { image_bytes, label }) + .collect(); + + let dataset = InMemDataset::new(items); + let dataset = MapperDataset::new(dataset, BytesToImage); + + Self { dataset } + } + + /// Download the MNIST dataset files from the web. + /// Panics if the download cannot be completed or the content of the file cannot be written to disk. + fn download(split: &str) -> PathBuf { + // Dataset files are stored in the burn-dataset cache directory + let cache_dir = dirs::cache_dir() + .expect("Could not get cache directory") + .join("burn-dataset"); + let split_dir = cache_dir.join("mnist").join(split); + + if !split_dir.exists() { + create_dir_all(&split_dir).expect("Failed to create base directory"); + } + + // Download split files + match split { + "train" => { + MnistDataset::download_file(TRAIN_IMAGES, &split_dir); + MnistDataset::download_file(TRAIN_LABELS, &split_dir); + } + "test" => { + MnistDataset::download_file(TEST_IMAGES, &split_dir); + MnistDataset::download_file(TEST_LABELS, &split_dir); + } + _ => panic!("Invalid split specified {split}"), + }; + + split_dir + } + + /// Download a file from the MNIST dataset URL to the destination directory. + /// File download progress is reported with the help of a [progress bar](indicatif). + fn download_file>(name: &str, dest_dir: &P) -> PathBuf { + // Output file name + let file_name = dest_dir.as_ref().join(name); + + if !file_name.exists() { + // Download gzip file + let bytes = download_file_as_bytes(&format!("{URL}{name}.gz"), name); + + // Create file to write the downloaded content to + let mut output_file = File::create(&file_name).unwrap(); + + // Decode gzip file content and write to disk + let mut gz_buffer = GzDecoder::new(&bytes[..]); + std::io::copy(&mut gz_buffer, &mut output_file).unwrap(); + } + + file_name + } + + /// Read images at the provided path for the specified split. + /// Each image is a vector of bytes. + fn read_images>(root: &P, split: &str) -> Vec> { + let file_name = if split == "train" { + TRAIN_IMAGES + } else { + TEST_IMAGES + }; + let file_name = root.as_ref().join(file_name); + + // Read number of images from 16-byte header metadata + let mut f = File::open(file_name).unwrap(); + let mut buf = [0u8; 4]; + let _ = f.seek(SeekFrom::Start(4)).unwrap(); + f.read_exact(&mut buf) + .expect("Should be able to read image file header"); + let size = u32::from_be_bytes(buf); + + let mut buf_images: Vec = vec![0u8; WIDTH * HEIGHT * (size as usize)]; + let _ = f.seek(SeekFrom::Start(16)).unwrap(); + f.read_exact(&mut buf_images) + .expect("Should be able to read image file header"); + + buf_images + .chunks(WIDTH * HEIGHT) + .map(|chunk| chunk.to_vec()) + .collect() + } + + /// Read labels at the provided path for the specified split. + fn read_labels>(root: &P, split: &str) -> Vec { + let file_name = if split == "train" { + TRAIN_LABELS + } else { + TEST_LABELS + }; + let file_name = root.as_ref().join(file_name); + + // Read number of labels from 8-byte header metadata + let mut f = File::open(file_name).unwrap(); + let mut buf = [0u8; 4]; + let _ = f.seek(SeekFrom::Start(4)).unwrap(); + f.read_exact(&mut buf) + .expect("Should be able to read label file header"); + let size = u32::from_be_bytes(buf); + + let mut buf_labels: Vec = vec![0u8; size as usize]; + let _ = f.seek(SeekFrom::Start(8)).unwrap(); + f.read_exact(&mut buf_labels) + .expect("Should be able to read labels from file"); + + buf_labels + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mod.rs new file mode 100644 index 0000000..948eb0b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/src/vision/mod.rs @@ -0,0 +1,9 @@ +#[cfg(feature = "builtin-sources")] +mod cifar; +mod image_folder; +mod mnist; + +#[cfg(feature = "builtin-sources")] +pub use cifar::*; +pub use image_folder::*; +pub use mnist::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset-fmt.csv b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset-fmt.csv new file mode 100644 index 0000000..74a155b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset-fmt.csv @@ -0,0 +1,2 @@ +HI1 1 true 1.0 +HI2 1 false 1.0 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.csv b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.csv new file mode 100644 index 0000000..2ec5fe8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.csv @@ -0,0 +1,3 @@ +column_str,column_int,column_bool,column_float +HI1,1,true,1.0 +HI2,1,false,1.0 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.json b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.json new file mode 100644 index 0000000..a04eaa2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset.json @@ -0,0 +1,2 @@ +{"column_str":"HI1","column_bytes":[1,2,3,3],"column_int":1,"column_bool":true,"column_float":1.0} +{"column_str":"HI2","column_bytes":[1,2,3,3],"column_int":1,"column_bool":false,"column_float":1.0} \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset_coco.json b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset_coco.json new file mode 100644 index 0000000..6a75bf9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/dataset_coco.json @@ -0,0 +1,132 @@ +{ + "images": [ + { + "width": 32, + "height": 32, + "id": 0, + "file_name": "two_dots_and_triangle.jpg" + }, + { + "width": 32, + "height": 32, + "id": 1, + "file_name": "dot_triangle.jpg" + }, + { + "width": 32, + "height": 32, + "id": 2, + "file_name": "one_dot.jpg" + } + ], + "categories": [ + { + "id": 0, + "name": "dot" + }, + { + "id": 1, + "name": "triangle" + } + ], + "annotations": [ + { + "id": 0, + "image_id": 0, + "category_id": 0, + "segmentation": [], + "bbox": [ + 3.1251719394773056, + 18.0907840440165, + 10.96011004126548, + 10.740027510316379 + ], + "ignore": 0, + "iscrowd": 0, + "area": 117.71188335928603 + }, + { + "id": 1, + "image_id": 0, + "category_id": 0, + "segmentation": [], + "bbox": [ + 3.2572214580467658, + 3.0371389270976605, + 10.563961485557085, + 10.828060522696012 + ], + "ignore": 0, + "iscrowd": 0, + "area": 114.38721432504178 + }, + { + "id": 2, + "image_id": 0, + "category_id": 1, + "segmentation": [], + "bbox": [ + 15.097661623108666, + 3.3892709766162312, + 12.632737276478679, + 11.18019257221458 + ], + "ignore": 0, + "iscrowd": 0, + "area": 141.23643546522516 + }, + { + "id": 3, + "image_id": 1, + "category_id": 0, + "segmentation": [], + "bbox": [ + 3.125171939477304, + 17.914718019257222, + 10.82806052269601, + 11.004126547455297 + ], + "ignore": 0, + "iscrowd": 0, + "area": 119.15334825525184 + }, + { + "id": 4, + "image_id": 1, + "category_id": 1, + "segmentation": [], + "bbox": [ + 15.27372764786794, + 3.301237964236589, + 12.192572214580478, + 11.708390646492433 + ], + "ignore": 0, + "iscrowd": 0, + "area": 142.7553984738776 + }, + { + "id": 5, + "image_id": 2, + "category_id": 0, + "segmentation": [], + "bbox": [ + 10.07977991746905, + 9.59559834938102, + 10.960110041265464, + 11.356258596973863 + ], + "ignore": 0, + "iscrowd": 0, + "area": 124.46584387990049 + } + ], + "info": { + "year": 2024, + "version": "1.0", + "description": "", + "contributor": "", + "url": "", + "date_created": "2024-12-11 22:16:31.823494" + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/orange/dot.jpg b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/orange/dot.jpg new file mode 100755 index 0000000000000000000000000000000000000000..3629f7fd4b40fdc5451390dc885715c4cae1f855 GIT binary patch literal 727 zcmex=}T+5Ri6|E+FFJVCMj-APxLK zz#zy0bSxt?qY?v?AS1INnAuRebI{N?Mn?>~P20{M%Pff?d0xX;l1B?$Bv6EF@~*g^hcWGV+@WL##!?#l!i}s%{{wrhu|0V$3_|6dk literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/red/dot.jpg b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/red/dot.jpg new file mode 100755 index 0000000000000000000000000000000000000000..be19ad3337777765edd5d303537de35c0342fe09 GIT binary patch literal 634 zcmex= zRY=j$kxe)-kzJ`!#HexNLJno8jR!@8E`CrkPAY2R|V^&07y2J$~}^+4C1KUw!=a z`ODXD-+%o41@ado12e>1aG#<1OAzQUCSV+}u!H=?$W#u*%z`YeiiT`Lj)Clng~Cck zjT|CQ6Blkg$f;}`^g%SK=pvVxipfLOk07sseMX$en#l4Q++zrT-D2QjW&}navmk># R!_ReH8tV_7|JLyTCIFvl#0&rc literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/red/dot.png b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder/red/dot.png new file mode 100755 index 0000000000000000000000000000000000000000..93d9dd10f4adb5ad36411ce240abbde6766b1d54 GIT binary patch literal 120 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx1|;Q0k92}K#X;^)4C~IxyaaMs(j9#r85lP9 zbN@+X1@buyJR*x382Ao@Fyrz36)8YLUQZXt5Q*^Q3hlL@iZ9ae4Pzd+~kx+GMu~Jj}lp;*P3=@*TBj4i-2aBA}N-S#Eegb6ze$3W#ElT_M zfTg}6>;o8u!iBzIC_?xd!ueg1poX^-mVM2hdcyk&+d@LCB%C z)pqJlCag3qz}^Me*BtT_CyzLp_5J|mXKQA0y@sFDu)nQ?YST^Y(A*GMw}zgX^h-x6 z99RV>6nN2r7PKLPY3S!ryrgP@j}$Ah9)5UqJ+vJVhirbf;l>AZ zbr)kU2Kb*<^^fI%@G3CftEzuRRdxCufV%(;2IlqacLNVkk-vA|m$?gA(hVFrG4J!f z4h(DtQqBiGA&>q)9NpQ*M&RnZfTb2l7zX0%X6HsREsi~BfweTN%G1D(H-NjFD7R~o zzKtZ9o7;bKZp=>|gB={pY8RP-F5bxV1_K`_2r));oY`!OGnp*$R?!kK#hXlGk|-s} ziHV8kCCMpCa*9<>l(i%bM;Zek%k#0a#blBHx2eY=8EH)l$4Fo$hLafe1X5@U+FP{1 zVvM{%5&0i(VyL4qa|~FK6Y2c+<5{WDWZNci&ErZz+R*(RsnSJIrwgTgj6q~LAu);6 zMhY-I8!b<$*mUseSeMxOK*921>&(|(#lIaLiTtp!`?c%ey>ivn?$!#BZmo;nEKIEds5=_!fBV8%UPJ5U$q%=bk5A>*_VgVI)LrhEOSblxu5fN^{bt7_DyT6RzH4v zX^fOn1Tm+|)2Fm=x;dwUjM2K^?>aoOt9xehz-M2c={WU%=Uay+He5*)D^G3hy!X{)HI38{KNT_(odgoc;aR0QQzgAavtm5R@FE6KBY|+hM nweK7-Kb9R`_UVvgyIXpXJ!&p9UdaF0R<-U6`h>bV)boD>$RbUE literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder_coco/one_dot.jpg b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/image_folder_coco/one_dot.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b719f53061cd3b2cfc5b71a7a2244a91df02aefd GIT binary patch literal 1434 zcmb7DeQXnT7=G?|y;qoL6NO)ZLN9OQd9k&^n5rmR# z3i)X_g|OPT2uBv-XnQhFIh!b_G!{=#ebzjOV>*6W$MMc?n$0kMLt9H?=MH+8(p`dj z1ki{eR7B8?4s;@o3VJ8VlR}8%zGe8u20uuw2(da4CC1a(k2t~xANm&3n1UCOp{(-Eo^aPOK4~&jweUbNo zvu^-J6J6nC*yzL4leM-2zYPQSU4Zi&!1DZ@ZWPnw82S!qBCQ%Sz_H`Nnl`GvXPLdt zL|D*m#hiKlv>Wh)XH37yOmqomK`@yFizw!pZ5Er&YOz}Fc@D{*C+AtMQh_8p6_?9p z%P%Y{P>LLiOVNvPJaJ4yt{~(pcB@_a-=MdWIX~37%v&^|tl7 zdd^&F9=rN@z*OO#dhxG6XOlEci9{mMstc{a$+d7x>W1wPR;_X0Q6p)26e<%iRhZ0KoxhtiRN>9s)V3cKoJNdF~OU$2bDZOVU6Q330|@fmL8x&}D-{v2b5JW?+G3iL{(Z zErSw;RMcV=)Paa*=mAdFq@s;A6AcInBsEUdSVYiZDQS2+cM-vvpy|wgd~?oszW4Vx3SsT+|E zK%Oi#7nOAUKwr3^wIJ=^ z0}6|^%nU%NFdSztu{#iNM;u+@C=u{c#GEv#KAAOxEdqW~z(s}SSew&?4&_!`=9Bmi#(OCw zffmw04=ON0ITS!4I3OI~HuRK%0pke^oLoOSp{Imj15o>MPPL1gRlq4L7SUx zS?-!hr;Cs$0MOs_{H+iG@tXkC2YLQpBhODa0Z_vL7i@F+#U}w`{zU)5IbZMz0QbEB zotNi)rh0(0`v81~N{n`+a~%qwWNt3NWHW$b3xG!tfNS>5+%P7@;nV;?2F@zA8{pV+ zfMt28y={TMC7AGVZi{jr%#Xhg8lXs0xX?s+(Gr>#iD*}`*hM0DmCI$WGMU0n$tc`d zH<^s_VptE()6-M#?(O5n`6xM0P7om|%n{L2nwD}3nS%S@#&-cL!EI7e1PdffP%Ob; z1|OV)@D`myP(_DDq}j7P@o^y^CgtK0ue^OYQ-D)y0Lv8XtwD6Xd_#ETHfS;BG#NP;576g}HBL6IU> z2{hvousSk5zO*(lD8XQSP^*iYG#S2VOz}2bRvoC>s?&`;oznMOeob`!?k93Ol(FBe z8nKm4zc;paeMd-t2e?QoWR9R8f8joAIbPr$8?bWx(7By2jGoAQrzRpJY_n)OseiO7 zd!^wp_qASG6T?H)?*rV`zRE1)aB+|Op3|9z&unKbQxhoQf zp1*mzr77xA>pEI$-LWI|C2ZB-IqCVMCfERoXI7^hYfE3AowYh3g02ba88S}&bYahW z4`a{7CfzR0J+<^k`kFxPVDrU^6~3*3fpri*5qc-B{JXpFHdNOaT=%C2S}uydstPq7 z2yJWMbtWQ=47~O+N(Bt|hTIjT*tn~|7OQa?=b3PROiX2*N|_wI$TkT3#QfJjISES{ zJSp>=1MCi)GeYCT)|k|;sr|6A=dle2Ye((&>b6mlrpZdamCLk#;;rB!H;IeuqQ;oI zx`wJ_)=Kt{Rr<`bB;8u2-+=hT(fggd$Fq~q>n4AjOnl?jBl=6HO5Z*!j}31O>bN33 an!~n`=j6$f;ph*-(lbBB-}$yue&`==a*EXe literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.png b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.png new file mode 100644 index 0000000000000000000000000000000000000000..3c87252ebe032f7f28544c96150a30651e7f55f4 GIT binary patch literal 117 zcmeAS@N?(olHy`uVBq!ia0vp^93afW1SGw4HSYi^8&4O<5Dr<@gN>XF1_I2AdcWW2 z$giqh5wLJGuc%hY3+Gm@xw$}|lRS`<1LTSR0&>!&wL(^WV7O!(qN_G@GcV8_22WQ% Jmvv4FO#oZUApZaW literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.txt new file mode 100644 index 0000000..2e01635 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_checkerboard.txt @@ -0,0 +1,8 @@ +1 2 1 2 1 2 1 2 +2 1 2 1 2 1 2 1 +1 2 1 2 1 2 1 2 +2 1 2 1 2 1 2 1 +1 2 1 2 1 2 1 2 +2 1 2 1 2 1 2 1 +1 2 1 2 1 2 1 2 +2 1 2 1 2 1 2 1 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.png b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.png new file mode 100644 index 0000000000000000000000000000000000000000..f0a129ab263b2438c63a5a860f1752d12e999e67 GIT binary patch literal 123 zcmeAS@N?(olHy`uVBq!ia0vp^93afW1SGw4HSYi^Cr=m05DwYYgN>Xm4h%;Qr2Kww zsjuGj$@rcY-&UqQp*B}e<;qNrTG?>>$0~;Hg&|o-AF>8iuDrt*lC?A}Zd>S|pN!kG WwbVCupA!L^#^CAd=d#Wzp$PzXXeti? literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.txt new file mode 100644 index 0000000..4fa2b7c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_2colors.txt @@ -0,0 +1,8 @@ +1 2 1 1 1 2 1 1 +1 2 1 1 1 1 2 1 +2 2 2 1 2 1 2 2 +2 2 2 2 2 2 1 1 +2 2 2 1 2 1 1 1 +1 1 2 2 2 2 2 1 +2 2 1 2 1 2 1 2 +2 1 1 1 1 1 1 1 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.png b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.png new file mode 100644 index 0000000000000000000000000000000000000000..38984e71cad9b1508bf8659605a18091146c8fa9 GIT binary patch literal 137 zcmeAS@N?(olHy`uVBq!ia0vp^93afW1SGw4HSYka08bak5Dr<>gN?Z?jsk}bA}`mU zF5{E7nYlZAHB&BQY4p3k`P*L2Tk$7{c@o3V4=ZeLe+q0=xjl&?_i9bZ0^J4gI!|rZ lJ-O}A?%mt8HmGxcV7G}fv#~WQQU;pM;OXk;vd$@?2>>74Fth*w literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.txt new file mode 100644 index 0000000..08f222b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/annotations/mask_random_3colors.txt @@ -0,0 +1,8 @@ +3 1 3 3 1 1 3 2 +3 3 3 3 1 3 2 1 +2 2 2 2 1 1 2 2 +1 1 1 3 3 3 2 3 +2 2 3 2 3 3 1 3 +1 3 3 1 1 3 2 1 +2 2 2 1 2 1 2 3 +3 1 3 3 2 1 2 2 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_checkerboard.png b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_checkerboard.png new file mode 100644 index 0000000000000000000000000000000000000000..2087501e41b6f2663152e5a335d1161f2e4c1aab GIT binary patch literal 165 zcmeAS@N?(olHy`uVBq!ia0vp^93afW1SGw4HSYka98VX=5Dr<^gN%#~1`JG&*Bzw< zoG%4GmKI35CwuR_c2~H_dX2dA#rhCNAm97vqDR4^&v%ML7#i*0KX*L>iC+JFryI-& ppK|zjrRztK=Dv2?zfoWPX-X^jhPHh)a3G-5mQxc}yY137zc hR&4&*bneq*W}ZgFzjxQ>_W;dh@O1TaS?83{1ORC6F(&{3 literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_random_3colors.png b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/segmask_folder/images/image_random_3colors.png new file mode 100644 index 0000000000000000000000000000000000000000..880bac466f179eccc75613d429a4993c69510ae6 GIT binary patch literal 204 zcmeAS@N?(olHy`uVBq!ia0vp^93afW1SGw4HSYka$(}BbAsn)%2N&`lRS#qG&nfqQ`0$<%JF6?cA*VDeVjGtWP)}Oz`AgCQbJ@M}gPkWO z=I#8UBKdQJr<>iIO`)G($j#N?%Lo*o$j|+UVX}|gf*@<5oj~U?c)I$ztaD0e0ssYW BOJV>3 literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/sqlite-dataset.db b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/sqlite-dataset.db new file mode 100644 index 0000000000000000000000000000000000000000..a53bf20693a9875ce0ba4b2a6add2a63eac2ade4 GIT binary patch literal 12288 zcmeI#ze~eF6bJD95JH17MJd5Scw-A9s6$s%IkgxQn?|rxX*I<{+CUm9x|RMHF8)QG z`UeOu{t2A~FRhi>>MBLP2bbKvmrL$GWO>z!)sB5?g;A#w)1iC)tii_LXI&mS4K%3)8L9 zIjd2A_IN=p3?4t@U0~m>HFD}(7!u;`bV4A=K5IHU-fZ1jxhuv009U< z00Izz00bZa0SG_<0xK<`Ny^q{CX*4)$_K^ClO}#m_ls}O3tOMn|07v{(MJm@Od$XP q2tWV=5P$##AOHafKmY;|SVe)Xq-;+s|2sbj!ld}mis{G?_5TOlBBfaX literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample1.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample1.txt new file mode 100644 index 0000000..7e3d1f2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample1.txt @@ -0,0 +1 @@ +This is a negative text sample for testing the text folder dataset functionality. \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample2.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample2.txt new file mode 100644 index 0000000..a4abb75 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/negative/sample2.txt @@ -0,0 +1 @@ +另一个负面文本样本，用以确保数据集能够处理同一类别中的多个文件。 \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample1.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample1.txt new file mode 100644 index 0000000..6e18f0b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample1.txt @@ -0,0 +1 @@ +This is a positive text sample for testing the text folder dataset functionality. \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample2.txt b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample2.txt new file mode 100644 index 0000000..d025616 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dataset/tests/data/text_folder/positive/sample2.txt @@ -0,0 +1 @@ +另一个正面文本样本，以确保数据集能够处理同一类别中的多个文件。 \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-derive/Cargo.toml new file mode 100644 index 0000000..8b67e20 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/Cargo.toml @@ -0,0 +1,23 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Derive crate for the Burn framework" +edition.workspace = true +keywords = [] +license.workspace = true +name = "burn-derive" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-derive" +version.workspace = true + +[lints] +workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true } +derive-new = { workspace = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/README.md b/crates/stable-diffusion-burn/burn-crates/burn-derive/README.md new file mode 100644 index 0000000..bc34709 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/README.md @@ -0,0 +1,6 @@ +# Burn Derive + +This crate should only be used with [burn](https://github.com/tracel-ai/burn). + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-derive.svg)](https://crates.io/crates/burn-derive) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-derive/blob/master/README.md) diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer.rs new file mode 100644 index 0000000..e5e6285 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer.rs @@ -0,0 +1,87 @@ +use super::ConfigEnumAnalyzer; +use crate::config::ConfigStructAnalyzer; +use crate::shared::{attribute::AttributeItem, field::FieldTypeAnalyzer}; +use proc_macro2::TokenStream; +use quote::quote; +use syn::{Field, Ident}; + +pub struct ConfigAnalyzerFactory {} + +pub trait ConfigAnalyzer { + fn gen_new_fn(&self) -> TokenStream { + quote! {} + } + fn gen_builder_fns(&self) -> TokenStream { + quote! {} + } + fn gen_serde_impl(&self) -> TokenStream; + fn gen_clone_impl(&self) -> TokenStream; + fn gen_display_impl(&self) -> TokenStream; + fn gen_config_impl(&self) -> TokenStream; +} + +impl ConfigAnalyzerFactory { + pub fn new() -> Self { + Self {} + } + + pub fn create_analyzer(&self, item: &syn::DeriveInput) -> Box { + let name = item.ident.clone(); + let config_type = parse_asm(item); + + match config_type { + ConfigType::Struct(data) => Box::new(self.create_struct_analyzer(name, data)), + ConfigType::Enum(data) => Box::new(self.create_enum_analyzer(name, data)), + } + } + + fn create_struct_analyzer(&self, name: Ident, fields: Vec) -> ConfigStructAnalyzer { + let fields = fields.into_iter().map(FieldTypeAnalyzer::new); + + let mut fields_required = Vec::new(); + let mut fields_option = Vec::new(); + let mut fields_default = Vec::new(); + + for field in fields { + let attributes: Vec = field + .attributes() + .filter(|attr| attr.has_name("config")) + .map(|attr| attr.item()) + .collect(); + + if !attributes.is_empty() { + let item = attributes.first().unwrap().clone(); + fields_default.push((field.clone(), item)); + continue; + } + + if field.is_of_type(&["Option"]) { + fields_option.push(field.clone()); + continue; + } + + fields_required.push(field.clone()); + } + + ConfigStructAnalyzer::new(name, fields_required, fields_option, fields_default) + } + + fn create_enum_analyzer(&self, name: Ident, data: syn::DataEnum) -> ConfigEnumAnalyzer { + ConfigEnumAnalyzer::new(name, data) + } +} + +enum ConfigType { + Struct(Vec), + Enum(syn::DataEnum), +} + +fn parse_asm(ast: &syn::DeriveInput) -> ConfigType { + match &ast.data { + syn::Data::Struct(struct_data) => { + ConfigType::Struct(struct_data.fields.clone().into_iter().collect()) + } + syn::Data::Enum(enum_data) => ConfigType::Enum(enum_data.clone()), + syn::Data::Union(_) => panic!("Only struct and enum can be derived"), + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_enum.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_enum.rs new file mode 100644 index 0000000..84ddd9b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_enum.rs @@ -0,0 +1,141 @@ +use crate::shared::enum_variant::map_enum_variant; + +use super::ConfigAnalyzer; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; + +pub struct ConfigEnumAnalyzer { + name: Ident, + data: syn::DataEnum, +} + +impl ConfigEnumAnalyzer { + pub fn new(name: Ident, data: syn::DataEnum) -> Self { + Self { name, data } + } + + fn serde_enum_ident(&self) -> Ident { + Ident::new(&format!("{}Serde", self.name), self.name.span()) + } + + fn gen_serde_enum(&self) -> TokenStream { + let enum_name = self.serde_enum_ident(); + let data = &self.data.variants; + + quote! { + #[derive(burn::serde::Serialize, burn::serde::Deserialize)] + #[serde(crate = "burn::serde")] + enum #enum_name { + #data + } + + } + } + + fn gen_serialize_fn(&self) -> TokenStream { + let enum_name = self.serde_enum_ident(); + let variants = self.data.variants.iter().map(|variant| { + let variant_name = &variant.ident; + let (inputs, outputs) = map_enum_variant(variant, |ident| quote! { #ident.clone() }); + + quote! { Self::#variant_name #inputs => #enum_name::#variant_name #outputs } + }); + + let name = &self.name; + + quote! { + impl burn::serde::Serialize for #name { + fn serialize(&self, serializer: S) -> Result + where + S: burn::serde::Serializer { + let serde_state = match self { + #(#variants),* + }; + serde_state.serialize(serializer) + } + } + + } + } + + fn gen_deserialize_fn(&self) -> TokenStream { + let enum_name = self.serde_enum_ident(); + let variants = self.data.variants.iter().map(|variant| { + let variant_name = &variant.ident; + let (inputs, outputs) = map_enum_variant(variant, |ident| quote! { #ident.clone() }); + + quote! { #enum_name::#variant_name #inputs => Self::#variant_name #outputs } + }); + let name = &self.name; + + quote! { + impl<'de> burn::serde::Deserialize<'de> for #name { + fn deserialize(deserializer: D) -> Result + where + D: burn::serde::Deserializer<'de> { + let serde_state = #enum_name::deserialize(deserializer)?; + Ok(match serde_state { + #(#variants),* + }) + } + } + + } + } +} + +impl ConfigAnalyzer for ConfigEnumAnalyzer { + fn gen_serde_impl(&self) -> TokenStream { + let struct_gen = self.gen_serde_enum(); + let serialize_gen = self.gen_serialize_fn(); + let deserialize_gen = self.gen_deserialize_fn(); + + quote! { + #struct_gen + #serialize_gen + #deserialize_gen + } + } + + fn gen_clone_impl(&self) -> TokenStream { + let variants = self.data.variants.iter().map(|variant| { + let variant_name = &variant.ident; + let (inputs, outputs) = map_enum_variant(variant, |ident| quote! { #ident.clone() }); + + quote! { Self::#variant_name #inputs => Self::#variant_name #outputs } + }); + let name = &self.name; + + quote! { + impl Clone for #name { + fn clone(&self) -> Self { + match self { + #(#variants),* + } + } + } + + } + } + + fn gen_display_impl(&self) -> TokenStream { + let name = &self.name; + + quote! { + impl core::fmt::Display for #name { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(&burn::config::config_to_json(self)) + } + } + } + } + + fn gen_config_impl(&self) -> TokenStream { + let name = &self.name; + + quote! { + impl burn::config::Config for #name { + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_struct.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_struct.rs new file mode 100644 index 0000000..d347f71 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/analyzer_struct.rs @@ -0,0 +1,380 @@ +use super::ConfigAnalyzer; +use crate::shared::{attribute::AttributeItem, field::FieldTypeAnalyzer}; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; + +pub struct ConfigStructAnalyzer { + name: Ident, + fields_required: Vec, + fields_option: Vec, + fields_default: Vec<(FieldTypeAnalyzer, AttributeItem)>, +} + +impl ConfigStructAnalyzer { + pub fn new( + name: Ident, + fields_required: Vec, + fields_option: Vec, + fields_default: Vec<(FieldTypeAnalyzer, AttributeItem)>, + ) -> Self { + Self { + name, + fields_required, + fields_option, + fields_default, + } + } + + fn wrap_impl_block(&self, tokens: TokenStream) -> TokenStream { + let name = &self.name; + + quote! { + impl #name { + #tokens + } + } + } + + fn names(&self) -> Vec { + let mut names = Vec::new(); + + for field in self.fields_required.iter() { + names.push(field.clone()); + } + + for field in self.fields_option.iter() { + names.push(field.clone()); + } + + for (field, _) in self.fields_default.iter() { + names.push(field.clone()); + } + + names + } + + fn name_types(&self, names: &[FieldTypeAnalyzer]) -> Vec { + let mut name_types = Vec::new(); + + for field in names.iter() { + let name = field.ident(); + let ty = &field.field.ty; + + name_types.push(quote! { + #name: #ty + }); + } + + name_types + } + + fn serde_struct_ident(&self) -> Ident { + Ident::new(&format!("{}Serde", self.name), self.name.span()) + } + + fn gen_serialize_fn( + &self, + struct_name: &Ident, + struct_gen: &TokenStream, + names: &[FieldTypeAnalyzer], + ) -> TokenStream { + let name = &self.name; + let names = names.iter().map(|name| { + let name = name.ident(); + quote! { #name: self.#name.clone() } + }); + + quote! { + impl burn::serde::Serialize for #name { + + fn serialize(&self, serializer: S) -> Result + where + S: burn::serde::Serializer { + #[derive(burn::serde::Serialize)] + #[serde(crate = "burn::serde")] + #struct_gen + + let serde_state = #struct_name { + #(#names),* + }; + serde_state.serialize(serializer) + } + } + + } + } + + fn gen_deserialize_fn( + &self, + struct_name: &Ident, + struct_gen: &TokenStream, + names: &[FieldTypeAnalyzer], + ) -> TokenStream { + let name = &self.name; + let names = names.iter().map(|name| { + let name = name.ident(); + quote! { #name: serde_state.#name } + }); + + quote! { + impl<'de> burn::serde::Deserialize<'de> for #name { + fn deserialize(deserializer: D) -> Result + where + D: burn::serde::Deserializer<'de> { + #[derive(burn::serde::Deserialize)] + #[serde(crate = "burn::serde")] + #struct_gen + + let serde_state = #struct_name::deserialize(deserializer)?; + Ok(#name { + #(#names),* + }) + } + } + + } + } + + fn gen_serde_struct(&self, names: &[TokenStream]) -> TokenStream { + let struct_name = self.serde_struct_ident(); + + quote! { + struct #struct_name { + #(#names),* + } + + } + } +} + +impl ConfigAnalyzer for ConfigStructAnalyzer { + fn gen_new_fn(&self) -> TokenStream { + let mut body = quote! {}; + let mut args = Vec::new(); + + let mut fn_docs = quote! {}; + let mut has_field_docs = false; + let mut has_required_docs = false; + let mut has_option_docs = false; + let mut has_default_docs = false; + let mut docs_header = |fn_docs: &mut TokenStream, + required_docs: bool, + option_docs: bool, + default_docs: bool| { + if !has_field_docs { + has_field_docs = true; + fn_docs.extend(quote! { + #[doc = "# Arguments"] + }); + } + if !has_required_docs && required_docs { + fn_docs.extend(quote! { + #[doc = "###### Required Arguments"] + }); + has_required_docs = true; + } + if !has_option_docs && option_docs { + fn_docs.extend(quote! { + #[doc = "###### Optional Arguments"] + }); + has_option_docs = true; + } + if !has_default_docs && default_docs { + fn_docs.extend(quote! { + #[doc = "###### Default Arguments"] + }); + has_default_docs = true; + } + }; + + for field in self.fields_required.iter() { + let name = field.ident(); + let ty = &field.field.ty; + let docs = field.docs(); + + body.extend(quote! { + #name: #name, + }); + args.push(quote! { + #name: #ty + }); + docs_header(&mut fn_docs, true, false, false); + let doc_str = format!("###### `{}`\n\n", quote!(#name)); + fn_docs.extend(quote! { + #[doc = #doc_str] + #(#docs)* + }); + } + + for field in self.fields_option.iter() { + let name = field.ident(); + let docs = field.docs(); + + body.extend(quote! { + #name: None, + }); + docs_header(&mut fn_docs, false, true, false); + let default_doc = "- Defaults to `None`"; + let doc_str = format!("###### `{}`\n", quote!(#name)); + fn_docs.extend(quote! { + #[doc = #doc_str] + #(#docs)* + #[doc = #default_doc] + }); + } + + for (field, attribute) in self.fields_default.iter() { + let name = field.ident(); + let value = &attribute.value; + let docs = field.docs(); + + match value { + syn::Lit::Str(value) => { + let stream: proc_macro2::TokenStream = value.value().parse().unwrap(); + + body.extend(quote! { + #name: #stream, + }); + } + _ => { + body.extend(quote! { + #name: #value, + }); + } + }; + docs_header(&mut fn_docs, false, false, true); + let default_doc = format!("- Defaults to `{}`", quote!(#value)); + let doc_str = format!("###### `{}`\n", quote!(#name)); + fn_docs.extend(quote! { + #[doc = #doc_str] + #(#docs)* + #[doc = #default_doc] + }); + } + + let body = quote! { + #[doc = "Create a new instance of the config."] + #fn_docs + #[allow(clippy::too_many_arguments)] + pub fn new( + #(#args),* + ) -> Self { + Self { #body } + } + }; + self.wrap_impl_block(body) + } + + fn gen_builder_fns(&self) -> TokenStream { + let mut body = quote! {}; + + for (field, attribute) in self.fields_default.iter() { + let name = field.ident(); + let ty = &field.field.ty; + let value = &attribute.value; + let docs = field.docs(); + let default_doc = format!("- Defaults to `{}`", quote!(#value)); + let doc_str = format!( + "Sets the value for the field [`{}`](Self::{0}).\n\n", + quote!(#name) + ); + let fn_docs = quote! { + #[doc = #doc_str] + #(#docs)* + #[doc = #default_doc] + }; + let fn_name = Ident::new(&format!("with_{name}"), name.span()); + + body.extend(quote! { + #fn_docs + pub fn #fn_name(mut self, #name: #ty) -> Self { + self.#name = #name; + self + } + }); + } + + for field in self.fields_option.iter() { + let name = field.ident(); + let ty = &field.field.ty; + let docs = field.docs(); + let default_doc = "- Defaults to `None`"; + let doc_str = format!( + "Sets the value for the field [`{}`](Self::{0}).\n\n", + quote!(#name) + ); + let fn_docs = quote! { + #[doc = #doc_str] + #(#docs)* + #[doc = #default_doc] + }; + let fn_name = Ident::new(&format!("with_{name}"), name.span()); + + body.extend(quote! { + #fn_docs + pub fn #fn_name(mut self, #name: #ty) -> Self { + self.#name = #name; + self + } + }); + } + + self.wrap_impl_block(body) + } + + fn gen_serde_impl(&self) -> TokenStream { + let names = self.names(); + + let struct_name = self.serde_struct_ident(); + let name_types = self.name_types(&names); + let struct_gen = self.gen_serde_struct(&name_types); + + let serialize_gen = self.gen_serialize_fn(&struct_name, &struct_gen, &names); + let deserialize_gen = self.gen_deserialize_fn(&struct_name, &struct_gen, &names); + + quote! { + #serialize_gen + #deserialize_gen + } + } + + fn gen_clone_impl(&self) -> TokenStream { + let name = &self.name; + let names = self.names().into_iter().map(|name| { + let name = name.ident(); + quote! { #name: self.#name.clone() } + }); + + quote! { + impl Clone for #name { + fn clone(&self) -> Self { + Self { + #(#names),* + } + } + } + + } + } + + fn gen_display_impl(&self) -> TokenStream { + let name = &self.name; + + quote! { + impl core::fmt::Display for #name { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(&burn::config::config_to_json(self)) + } + } + } + } + + fn gen_config_impl(&self) -> TokenStream { + let name = &self.name; + + quote! { + impl burn::config::Config for #name { + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/base.rs new file mode 100644 index 0000000..cca3f04 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/base.rs @@ -0,0 +1,24 @@ +use super::ConfigAnalyzerFactory; +use quote::quote; + +pub(crate) fn derive_impl(item: &syn::DeriveInput) -> proc_macro::TokenStream { + let factory = ConfigAnalyzerFactory::new(); + let analyzer = factory.create_analyzer(item); + + let constructor = analyzer.gen_new_fn(); + let builders = analyzer.gen_builder_fns(); + let serde = analyzer.gen_serde_impl(); + let clone = analyzer.gen_clone_impl(); + let display = analyzer.gen_display_impl(); + let config_impl = analyzer.gen_config_impl(); + + quote! { + #config_impl + #constructor + #builders + #serde + #clone + #display + } + .into() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/mod.rs new file mode 100644 index 0000000..36a5c0e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/config/mod.rs @@ -0,0 +1,9 @@ +mod analyzer; +mod analyzer_enum; +mod analyzer_struct; +mod base; + +pub(crate) use analyzer::*; +pub(crate) use analyzer_enum::*; +pub(crate) use analyzer_struct::*; +pub(crate) use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/lib.rs new file mode 100644 index 0000000..a9af743 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/lib.rs @@ -0,0 +1,34 @@ +#![warn(missing_docs)] + +//! The derive crate of Burn. + +#[macro_use] +extern crate derive_new; + +use proc_macro::TokenStream; + +pub(crate) mod config; +pub(crate) mod module; +pub(crate) mod record; +pub(crate) mod shared; + +/// Derive macro for the module. +#[proc_macro_derive(Module, attributes(module))] +pub fn module_derive(input: TokenStream) -> TokenStream { + let input = syn::parse(input).unwrap(); + module::derive_impl(&input) +} + +/// Derive macro for the record. +#[proc_macro_derive(Record)] +pub fn record_derive(input: TokenStream) -> TokenStream { + let input = syn::parse(input).unwrap(); + record::derive_impl(&input) +} + +/// Derive macro for the config. +#[proc_macro_derive(Config, attributes(config))] +pub fn config_derive(input: TokenStream) -> TokenStream { + let item = syn::parse(input).unwrap(); + config::derive_impl(&item) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/base.rs new file mode 100644 index 0000000..1099d5d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/base.rs @@ -0,0 +1,39 @@ +use super::{ + codegen::{generate_module_const, generate_module_standard}, + codegen_enum::EnumModuleCodegen, + codegen_struct::StructModuleCodegen, +}; +use proc_macro::TokenStream; + +pub(crate) fn derive_impl(ast: &syn::DeriveInput) -> TokenStream { + let has_backend = ast + .generics + .type_params() + .map(|param| param.ident == "B") + .reduce(|accum, is_backend| is_backend || accum) + .unwrap_or(false); + + match &ast.data { + syn::Data::Struct(_) => { + if has_backend { + generate_module_standard(ast, StructModuleCodegen::from_ast(ast)) + } else { + generate_module_const(ast) + } + } + syn::Data::Enum(_data) => match EnumModuleCodegen::from_ast(ast) { + Ok(enum_codegen) => { + if has_backend { + generate_module_standard(ast, enum_codegen) + } else { + generate_module_const(ast) + } + } + Err(err) => err.to_compile_error(), + }, + syn::Data::Union(_) => { + syn::Error::new_spanned(ast, "Union modules aren't supported").to_compile_error() + } + } + .into() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen.rs new file mode 100644 index 0000000..268118f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen.rs @@ -0,0 +1,319 @@ +use super::{display, record::ModuleRecordCodegen}; +use crate::shared::generics::GenericsHelper; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::{Attribute, Generics, parse_quote}; + +/// Basic trait to be implemented for Module generation. +pub(crate) trait ModuleCodegen { + type RecordCodegen: ModuleRecordCodegen; + + fn gen_num_params(&self) -> TokenStream; + fn gen_visit(&self) -> TokenStream; + fn gen_collect_devices(&self) -> TokenStream; + fn gen_to_device(&self) -> TokenStream; + fn gen_fork(&self) -> TokenStream; + fn gen_map(&self) -> TokenStream; + fn gen_valid(&self) -> TokenStream; + fn gen_from_inner(&self) -> TokenStream; + fn gen_into_record(&self) -> TokenStream; + fn gen_load_record(&self) -> TokenStream; + fn gen_clone(&self) -> TokenStream; + + fn record_codegen(self) -> Self::RecordCodegen; +} + +pub(crate) fn generate_module_standard( + ast: &syn::DeriveInput, + codegen: Codegen, +) -> TokenStream { + let name = &ast.ident; + + let generics = GenericsParser::from_ast(&ast.generics); + + let display_fn = display::display_fn(ast); + let attributes_fn = display::attributes_fn(ast); + let num_params_fn = codegen.gen_num_params(); + let visit = codegen.gen_visit(); + let map_mut = codegen.gen_map(); + let collect_devices = codegen.gen_collect_devices(); + let to_device = codegen.gen_to_device(); + let fork = codegen.gen_fork(); + let valid_fn = codegen.gen_valid(); + let from_inner_fn = codegen.gen_from_inner(); + let into_record_fn = codegen.gen_into_record(); + let load_record_fn = codegen.gen_load_record(); + let clone_fn = codegen.gen_clone(); + + let record = codegen.record_codegen(); + let record_name = Ident::new(format!("{name}Record").as_str(), name.span()); + let record_type = record.gen_record_type(&record_name, &generics.module); + + let (generics_module, generics_ty_module, generics_where_module) = + generics.module.split_for_impl(); + let (generics_module_autodiff, generics_ty_module_autodiff, generics_where_module_autodiff) = + generics.module_autodiff.split_for_impl(); + let (generics_module_has_autodiff, _generics_ty, generics_where_module_has_autodiff) = + generics.module_has_autodiff.split_for_impl(); + + let generics_ty_inner_module = generics.inner_module_ty; + let generics_ty_train_module = generics.train_module_ty; + let generics_ty_train_inner_module = generics.train_inner_ty; + + let mut codegen = quote! { + impl #generics_module burn::module::Module for #name #generics_ty_module #generics_where_module { + type Record = #record_name #generics_ty_module; + + #load_record_fn + #into_record_fn + + #num_params_fn + + #visit + #map_mut + + #collect_devices + #to_device + #fork + + } + + impl #generics_module_autodiff burn::module::AutodiffModule for #name #generics_ty_module_autodiff #generics_where_module_autodiff + { + type InnerModule=#name; + + #valid_fn + + #from_inner_fn + } + + impl #generics_module_has_autodiff burn::module::HasAutodiffModule for #name #generics_where_module_has_autodiff + { + type TrainModule=#name; + } + + impl #generics_module core::fmt::Display for #name #generics_ty_module #generics_where_module { + #display_fn + } + + + impl #generics_module burn::module::ModuleDisplayDefault for #name #generics_ty_module #generics_where_module { + #attributes_fn + + fn num_params(&self) -> usize { + burn::module::Module::num_params(self) + } + } + + impl #generics_module Clone for #name #generics_ty_module #generics_where_module { + #clone_fn + } + + #record_type + }; + + if !has_custom_display(&ast.attrs) { + codegen.extend(quote! { + impl #generics_module burn::module::ModuleDisplay for #name #generics_ty_module #generics_where_module { + + } + }); + } + + codegen +} + +// When there is no backend in the generic parameter, the type is considered as a constant. +pub(crate) fn generate_module_const(ast: &syn::DeriveInput) -> TokenStream { + let name = &ast.ident; + let (generics, generics_ty, generics_where) = ast.generics.split_for_impl(); + + let backend: syn::Generics = parse_quote! { }; + let backend_ad: syn::Generics = parse_quote! { }; + + let mut generics_module = ast.generics.clone(); + let mut generics_module_autodiff = ast.generics.clone(); + + for param in backend.params.into_iter() { + generics_module.params.push(param); + } + for param in backend_ad.params.into_iter() { + generics_module_autodiff.params.push(param); + } + let (generics_module, _, _) = generics_module.split_for_impl(); + let (generics_module_ad, _, _) = generics_module_autodiff.split_for_impl(); + + let display_fn = display::display_fn(ast); + let attributes_fn = display::attributes_fn(ast); + + let mut codegen = quote! { + impl #generics_module burn::module::Module for #name #generics_ty #generics_where { + burn::constant!(module); + } + + impl #generics_module_ad burn::module::AutodiffModule + for #name #generics_ty #generics_where { + burn::constant!(ad_module, #name #generics_ty); + } + + impl #generics core::fmt::Display for #name #generics_ty #generics_where { + #display_fn + } + + + impl #generics burn::module::ModuleDisplayDefault for #name #generics_ty #generics_where { + #attributes_fn + } + + }; + + if !has_custom_display(&ast.attrs) { + codegen.extend(quote! { + impl #generics burn::module::ModuleDisplay for #name #generics_ty #generics_where { + + } + }); + } + + codegen +} + +struct GenericsParser { + module: Generics, + module_autodiff: Generics, + module_has_autodiff: Generics, + inner_module_ty: TokenStream, + train_module_ty: TokenStream, + train_inner_ty: TokenStream, +} + +impl GenericsParser { + fn from_ast(generics: &Generics) -> Self { + let mut module = GenericsHelper::new(generics.clone()); + let mut module_autodiff = GenericsHelper::new(generics.clone()); + let mut module_has_autodiff = GenericsHelper::new(generics.clone()); + + let backend_trait = module.fetch_backend_trait(); + + module_autodiff.add_predicate(parse_quote! { + B: burn::tensor::backend::AutodiffBackend + }); + + module_autodiff.add_predicate(parse_quote! { + ::InnerBackend: #backend_trait + }); + + module_has_autodiff.add_predicate(parse_quote! { + B: burn::tensor::backend::AutodiffBackend + }); + + module_has_autodiff.add_predicate(parse_quote! { + ::InnerBackend: #backend_trait + }); + + let mut generics_names_except_backend = quote! {}; + let mut train_generics_names_except_backend = quote! {}; + let mut train_inner_generics_names_except_backend = quote! {}; + + module + .types() + .into_iter() + .filter(|ident| ident != "B") + .for_each(|ident| { + module.add_predicate( + parse_quote! { + #ident: burn::module::Module + } + ); + + module.add_predicate( + parse_quote! { + #ident: burn::module::ModuleDisplay + } + ); + + module_autodiff.add_predicate( + parse_quote! { + #ident: burn::module::AutodiffModule + } + ); + + module_autodiff.add_predicate( + parse_quote! { + <#ident as burn::module::AutodiffModule>::InnerModule: burn::module::Module + } + ); + + module_autodiff.add_predicate( + parse_quote! { + <#ident as burn::module::AutodiffModule>::InnerModule: burn::module::ModuleDisplay + } + ); + + generics_names_except_backend.extend(quote! { <#ident as burn::module::AutodiffModule>::InnerModule, }); + + module_autodiff.add_predicate( + parse_quote! { + #ident: burn::module::ModuleDisplay + } + ); + + module_has_autodiff.add_predicate( + parse_quote! { + #ident: burn::module::Module + } + ); + + module_has_autodiff.add_predicate( + parse_quote! { + #ident: burn::module::ModuleDisplay + } + ); + + module_has_autodiff.add_predicate( + parse_quote! { + #ident: burn::module::HasAutodiffModule + } + ); + + module_has_autodiff.add_predicate( + parse_quote! { + #ident::TrainModule: burn::module::ModuleDisplay + } + ); + train_generics_names_except_backend.extend(quote! { #ident, }); + train_inner_generics_names_except_backend.extend(quote! { #ident::TrainModule, }); + + }); + + module.consts().into_iter().for_each(|ident| { + generics_names_except_backend.extend(quote! { #ident, }); + train_generics_names_except_backend.extend(quote! { #ident, }); + train_inner_generics_names_except_backend.extend(quote! { #ident, }); + }); + + Self { + module: module.generics, + module_autodiff: module_autodiff.generics, + module_has_autodiff: module_has_autodiff.generics, + inner_module_ty: generics_names_except_backend, + train_module_ty: train_generics_names_except_backend, + train_inner_ty: train_inner_generics_names_except_backend, + } + } +} + +fn has_custom_display(attrs: &[Attribute]) -> bool { + attrs.iter().any(|attr| { + attr.path().is_ident("module") + && attr + .parse_nested_meta(|meta| { + if meta.path.is_ident("custom_display") { + Ok(()) + } else { + Err(meta.error("unsupported attribute")) + } + }) + .is_ok() + }) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_enum.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_enum.rs new file mode 100644 index 0000000..b0d45db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_enum.rs @@ -0,0 +1,236 @@ +use super::{codegen::ModuleCodegen, record_enum::EnumModuleRecordCodegen}; +use crate::shared::enum_variant::{EnumVariant, parse_variants}; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::Visibility; + +pub(crate) struct EnumModuleCodegen { + pub name: Ident, + pub variants: Vec, + pub vis: Visibility, +} + +impl ModuleCodegen for EnumModuleCodegen { + type RecordCodegen = EnumModuleRecordCodegen; + + fn gen_num_params(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|_| { + quote! { + burn::module::Module::::num_params(module) + } + }); + + quote! { + fn num_params(&self) -> usize { + #match_body + } + } + } + + fn gen_visit(&self) -> TokenStream { + let enum_name = self.name.to_string(); + let container_type = format!("Enum:{}", enum_name); + let match_body = self.gen_variants_match_fn(|variant_name| { + let variant_str = variant_name.to_string(); + quote! { + { + visitor.enter_module(#variant_str, #container_type); + burn::module::Module::visit(module, visitor); + visitor.exit_module(#variant_str, #container_type); + } + } + }); + + quote! { + fn visit>(&self, visitor: &mut Visitor) { + #match_body + } + } + } + + fn gen_collect_devices(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|_| { + quote! { + burn::module::Module::::collect_devices(module, devices) + } + }); + + quote! { + fn collect_devices( + &self, + devices: burn::module::Devices + ) -> burn::module::Devices { + #match_body + } + } + } + + fn gen_to_device(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|variant| { + quote! { + Self::#variant(burn::module::Module::::to_device(module, device)) + } + }); + + quote! { + fn to_device(self, device: &B::Device) -> Self { + #match_body + } + } + } + + fn gen_fork(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|variant| { + quote! { + Self::#variant(burn::module::Module::::fork(module, device)) + } + }); + + quote! { + fn fork(self, device: &B::Device) -> Self { + #match_body + } + } + } + + fn gen_map(&self) -> TokenStream { + let enum_name = self.name.to_string(); + let container_type = format!("Enum:{}", enum_name); + let match_body = self.gen_variants_match_fn(|variant| { + let variant_str = variant.to_string(); + quote! { + { + mapper.enter_module(#variant_str, #container_type); + let result = burn::module::Module::::map(module, mapper); + mapper.exit_module(#variant_str, #container_type); + Self::#variant(result) + } + } + }); + + quote! { + fn map>(self, mapper: &mut Mapper) -> Self { + #match_body + } + } + } + + fn gen_valid(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|variant| { + quote! { + Self::InnerModule::#variant(burn::module::AutodiffModule::::valid(module)) + } + }); + + quote! { + fn valid(&self) -> Self::InnerModule { + #match_body + } + } + } + + fn gen_from_inner(&self) -> TokenStream { + let match_body = + self.gen_variants_match_fn_param("module", "Self::InnerModule::", |variant| { + quote! { + Self::#variant(burn::module::AutodiffModule::::from_inner(module)) + } + }); + + quote! { + fn from_inner(module: Self::InnerModule) -> Self { + #match_body + } + } + } + + fn gen_into_record(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|variant| { + quote! { + Self::Record::#variant(burn::module::Module::::into_record(module)) + } + }); + + quote! { + fn into_record(self) -> Self::Record { + #match_body + } + } + } + + fn gen_load_record(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|variant| { + quote! { + { + let Self::Record::#variant(r) = record else {panic!("Can't parse record from a different variant");}; + Self::#variant(burn::module::Module::::load_record(module, r)) + } + } + }); + + quote! { + fn load_record(self, record: Self::Record) -> Self { + #match_body + } + } + } + + fn gen_clone(&self) -> TokenStream { + let match_body = self.gen_variants_match_fn(|variant| { + quote! { + Self::#variant(module.clone()) + } + }); + + quote! { + fn clone(&self) -> Self { + #match_body + } + } + } + + fn record_codegen(self) -> Self::RecordCodegen { + EnumModuleRecordCodegen::new(self.variants, self.vis) + } +} + +impl EnumModuleCodegen { + pub fn from_ast(ast: &syn::DeriveInput) -> syn::Result { + Ok(Self { + name: ast.ident.clone(), + variants: parse_variants(ast)?, + vis: ast.vis.clone(), + }) + } + + /// Generate the enum variants' match arms with the provided function + fn gen_variants_match_fn(&self, func: F) -> TokenStream + where + F: Fn(Ident) -> TokenStream, + { + self.gen_variants_match_fn_param("self", "Self::", func) + } + + /// Generate a match expression over the given argument (e.g., `self`) + /// and using the provided prefix for variants (e.g., `Self::` or `Self::InnerModule::`) + fn gen_variants_match_fn_param(&self, arg: &str, prefix: &str, func: F) -> TokenStream + where + F: Fn(Ident) -> TokenStream, + { + let match_arms = self.variants.iter().map(|variant| { + let name = &variant.ident; + let full_variant = syn::parse_str::(&format!("{prefix}{name}")).unwrap(); + let arm_pattern = quote! { #full_variant(module) }; + let arm_code = func(name.clone()); + quote! { #arm_pattern => #arm_code, } + }); + + let arg = Ident::new(arg, Span::call_site()); + + quote! { + match #arg { + #(#match_arms)* + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_struct.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_struct.rs new file mode 100644 index 0000000..251c253 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/codegen_struct.rs @@ -0,0 +1,267 @@ +use super::{codegen::ModuleCodegen, record_struct::StructModuleRecordCodegen}; +use crate::shared::field::{FieldTypeAnalyzer, parse_fields}; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::Visibility; + +pub(crate) struct StructModuleCodegen { + pub name: Ident, + pub fields: Vec, + pub vis: Visibility, +} + +impl ModuleCodegen for StructModuleCodegen { + type RecordCodegen = StructModuleRecordCodegen; + + fn gen_num_params(&self) -> TokenStream { + let body = self.gen_fields_fn(|name| { + quote! { + num_params += burn::module::Module::::num_params(&self.#name); + } + }); + + quote! { + fn num_params(&self) -> usize { + let mut num_params = 0; + #body + num_params + } + } + } + + fn gen_visit(&self) -> TokenStream { + let struct_name = self.name.to_string(); + let container_type = format!("Struct:{}", struct_name); + let body = self.gen_fields_fn(|name| { + let name_str = name.to_string(); + quote! { + visitor.enter_module(#name_str, #container_type); + burn::module::Module::visit(&self.#name, visitor); + visitor.exit_module(#name_str, #container_type); + } + }); + + quote! { + fn visit>(&self, visitor: &mut Visitor) { + #body + } + } + } + + fn gen_collect_devices(&self) -> TokenStream { + let body = self.gen_fields_fn(|name| { + quote! { + let devices = burn::module::Module::::collect_devices(&self.#name, devices); + } + }); + + quote! { + fn collect_devices( + &self, + devices: burn::module::Devices + ) -> burn::module::Devices { + #body + + devices + } + } + } + + fn gen_to_device(&self) -> TokenStream { + let (names, body) = self.gen_fields_fn_names(|name| { + quote! { + let #name = burn::module::Module::::to_device(self.#name, device); + } + }); + + quote! { + fn to_device(self, device: &B::Device) -> Self { + #body + + Self { + #(#names),* + } + } + } + } + + fn gen_fork(&self) -> TokenStream { + let (names, body) = self.gen_fields_fn_names(|name| { + quote! { + let #name = burn::module::Module::::fork(self.#name, device); + } + }); + + quote! { + fn fork(self, device: &B::Device) -> Self { + #body + + Self { + #(#names),* + } + } + } + } + + fn gen_map(&self) -> TokenStream { + let struct_name = self.name.to_string(); + let container_type = format!("Struct:{}", struct_name); + let (names, body) = self.gen_fields_fn_names(|name| { + let name_str = name.to_string(); + quote! { + mapper.enter_module(#name_str, #container_type); + let #name = burn::module::Module::::map(self.#name, mapper); + mapper.exit_module(#name_str, #container_type); + } + }); + + quote! { + fn map>(self, mapper: &mut Mapper) -> Self { + #body + + Self { + #(#names),* + } + } + } + } + + fn gen_valid(&self) -> TokenStream { + let (names, body) = self.gen_fields_fn_names(|name| { + quote! { + let #name = burn::module::AutodiffModule::::valid(&self.#name); + } + }); + + quote! { + fn valid(&self) -> Self::InnerModule { + #body + + Self::InnerModule { + #(#names),* + } + } + } + } + + fn gen_from_inner(&self) -> TokenStream { + let (names, body) = self.gen_fields_fn_names(|name| { + quote! { + let #name = burn::module::AutodiffModule::::from_inner(#name); + } + }); + + // Destructure inner module to move all fields + let destructure = quote! { + let Self::InnerModule { #(#names),* } = module; + }; + + quote! { + fn from_inner(module: Self::InnerModule) -> Self { + #destructure + #body + + Self { + #(#names),* + } + } + } + } + + fn gen_into_record(&self) -> TokenStream { + let body = self.gen_fields_fn(|name| { + quote! { + #name: burn::module::Module::::into_record(self.#name), + } + }); + + quote! { + fn into_record(self) -> Self::Record { + Self::Record { + #body + } + } + } + } + + fn gen_load_record(&self) -> TokenStream { + let body = self.gen_fields_fn(|name| { + quote! { + #name: burn::module::Module::::load_record(self.#name, record.#name), + } + }); + + quote! { + fn load_record(self, record: Self::Record) -> Self { + Self { + #body + } + } + } + } + + fn gen_clone(&self) -> TokenStream { + let (names, body) = self.gen_fields_fn_names(|name| { + quote! { + let #name = self.#name.clone(); + } + }); + + quote! { + fn clone(&self) -> Self { + #body + + Self { + #(#names),* + } + } + } + } + + fn record_codegen(self) -> Self::RecordCodegen { + StructModuleRecordCodegen::new(self.fields, self.vis) + } +} + +impl StructModuleCodegen { + pub fn from_ast(ast: &syn::DeriveInput) -> Self { + Self { + name: ast.ident.clone(), + fields: parse_fields(ast) + .into_iter() + .map(FieldTypeAnalyzer::new) + .collect(), + vis: ast.vis.clone(), + } + } + + fn gen_fields_fn_names(&self, func: F) -> (Vec, TokenStream) + where + F: Fn(Ident) -> TokenStream, + { + let mut body = quote! {}; + let mut names = Vec::new(); + + for field in self.fields.iter() { + let name = field.ident(); + + names.push(name.clone()); + body.extend(func(field.ident())); + } + + (names, body) + } + + fn gen_fields_fn(&self, func: F) -> TokenStream + where + F: Fn(Ident) -> TokenStream, + { + let mut body = quote! {}; + + for field in self.fields.iter() { + body.extend(func(field.ident())); + } + + body + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/display.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/display.rs new file mode 100644 index 0000000..bc8a8d0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/display.rs @@ -0,0 +1,94 @@ +use quote::quote; + +pub fn attributes_fn(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { + match &ast.data { + syn::Data::Struct(data_struct) => { + let fields = match &data_struct.fields { + syn::Fields::Named(named_fields) => named_fields.named.iter().collect::>(), + syn::Fields::Unit => Vec::new(), + _ => panic!("attributes_fn only supports structs with named or unit fields"), + }; + let field_prints = fields.iter().map(|field| { + let field_name = &field.ident; + quote! { .add(stringify!(#field_name), &self.#field_name) } + }); + let struct_name = &ast.ident; + quote! { + fn content(&self, mut content: burn::module::Content) -> Option { + content + .set_top_level_type(&stringify!(#struct_name)) + #(#field_prints)* + .optional() + } + } + } + syn::Data::Enum(data_enum) => { + let variant_prints = data_enum.variants.iter().map(|variant| { + let variant_name = &variant.ident; + match &variant.fields { + syn::Fields::Unit => { + quote! { + Self::#variant_name => { + content.add_formatted(&stringify!(#variant_name).to_string()) + .optional() + + } + } + } + syn::Fields::Named(named_fields) => { + let field_prints = named_fields.named.iter().map(|field| { + let field_name = &field.ident; + quote! { .add(stringify!(#field_name), &self.#field_name) } + }); + + let field_names = named_fields.named.iter().map(|field| { + let field_name = &field.ident; + quote! { #field_name } + }); + + quote! { + Self::#variant_name { #(#field_names),* } => { + content.set_top_level_type(&stringify!(#variant_name)) + #(#field_prints)* + .optional() + } + } + } + syn::Fields::Unnamed(unnamed_fields) => { + let field_names = (0..unnamed_fields.unnamed.len()).map(|i| { + syn::Ident::new(&format!("_{i}"), proc_macro2::Span::call_site()) + }); + + let field_prints = field_names.clone().map(|field_name| { + quote! { .add(stringify!(#field_name), #field_name) } + }); + quote! { + Self::#variant_name(#(#field_names),*) => { + content.set_top_level_type(&stringify!(#variant_name)) + #(#field_prints)* + .optional() + } + } + } + } + }); + quote! { + fn content(&self, mut content: burn::module::Content) -> Option { + match self { + #(#variant_prints)* + } + } + } + } + _ => panic!("attributes_fn only supports structs and enums"), + } +} + +pub fn display_fn(_ast: &syn::DeriveInput) -> proc_macro2::TokenStream { + quote! { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let formatted = burn::module::ModuleDisplay::format(self, Default::default()); + write!(f, "{}", formatted) + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/mod.rs new file mode 100644 index 0000000..95642fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/mod.rs @@ -0,0 +1,11 @@ +pub(crate) mod codegen; +pub(crate) mod codegen_enum; +pub(crate) mod codegen_struct; +pub(crate) mod display; +pub(crate) mod record; +pub(crate) mod record_enum; +pub(crate) mod record_struct; + +mod base; + +pub(crate) use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record.rs new file mode 100644 index 0000000..72e3cd2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record.rs @@ -0,0 +1,8 @@ +use proc_macro2::{Ident, TokenStream}; +use syn::Generics; + +/// Basic trait to generate a record type based on the Module struct. +pub(crate) trait ModuleRecordCodegen { + /// Generate the record type (i.e a struct) + fn gen_record_type(&self, record_name: &Ident, generics: &Generics) -> TokenStream; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_enum.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_enum.rs new file mode 100644 index 0000000..dda1911 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_enum.rs @@ -0,0 +1,41 @@ +use crate::shared::enum_variant::EnumVariant; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::{Generics, Visibility}; + +use super::record::ModuleRecordCodegen; + +#[derive(new)] +pub(crate) struct EnumModuleRecordCodegen { + variants: Vec, + vis: Visibility, +} + +impl ModuleRecordCodegen for EnumModuleRecordCodegen { + fn gen_record_type(&self, record_name: &Ident, generics: &Generics) -> TokenStream { + let mut variants = quote! {}; + let vis = &self.vis; + + // Capture the Record enum variant types + for variant in self.variants.iter() { + let ty = &variant.ty; + let name = &variant.ident; + + variants.extend(quote! { + /// The module record associative type. + #name(<#ty as burn::module::Module>::Record), + }); + } + + let (generics, _generics_ty, generics_where) = generics.split_for_impl(); + + quote! { + + /// The record type for the module. + #[derive(burn::record::Record)] + #vis enum #record_name #generics #generics_where { + #variants + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_struct.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_struct.rs new file mode 100644 index 0000000..0f4af32 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/module/record_struct.rs @@ -0,0 +1,40 @@ +use crate::shared::field::FieldTypeAnalyzer; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::{Generics, Visibility}; + +use super::record::ModuleRecordCodegen; + +#[derive(new)] +pub(crate) struct StructModuleRecordCodegen { + fields: Vec, + vis: Visibility, +} + +impl ModuleRecordCodegen for StructModuleRecordCodegen { + fn gen_record_type(&self, record_name: &Ident, generics: &Generics) -> TokenStream { + let mut fields = quote! {}; + let vis = &self.vis; + + for field in self.fields.iter() { + let ty = &field.field.ty; + let name = &field.field.ident; + + fields.extend(quote! { + /// The module record associative type. + #vis #name: <#ty as burn::module::Module>::Record, + }); + } + + let (generics, _generics_ty, generics_where) = generics.split_for_impl(); + + quote! { + + /// The record type for the module. + #[derive(burn::record::Record)] + #vis struct #record_name #generics #generics_where { + #fields + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/base.rs new file mode 100644 index 0000000..e68191b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/base.rs @@ -0,0 +1,13 @@ +use super::{ + codegen::generate_record, + item::{codegen_enum::EnumRecordItemCodegen, codegen_struct::StructRecordItemCodegen}, +}; + +pub(crate) fn derive_impl(ast: &syn::DeriveInput) -> proc_macro::TokenStream { + match &ast.data { + syn::Data::Struct(_) => generate_record::(ast), + syn::Data::Enum(_) => generate_record::(ast), + syn::Data::Union(_) => panic!("Union modules aren't supported yet."), + } + .into() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/codegen.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/codegen.rs new file mode 100644 index 0000000..f9a3ea8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/codegen.rs @@ -0,0 +1,145 @@ +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::{Generics, parse_quote}; + +use crate::record::item::codegen::RecordItemCodegen; + +pub(crate) fn generate_record(ast: &syn::DeriveInput) -> TokenStream { + let record_gen: syn::Result> = RecordCodegen::from_ast(ast); + match record_gen { + Ok(record_gen) => { + let item_type = record_gen.gen_record_type(); + let record_impl = record_gen.gen_impl_record(); + + quote! { + #item_type + #record_impl + } + } + Err(err) => err.to_compile_error(), + } +} + +pub(crate) struct RecordCodegen { + /// Record type info. + ty: RecordType, + /// Record item code gen. + codegen: G, +} + +impl RecordCodegen { + /// Generate the record type with the correct generics. + pub(crate) fn gen_record_type(&self) -> TokenStream { + // Add precision settings type bound + let param: syn::Generics = parse_quote! { }; + let mut generics = self.ty.generics.clone(); + + for param in param.params.into_iter() { + generics.params.push(param); + } + + // Generate the record item definition + self.codegen + .gen_item_type(&self.ty.item, &generics, self.ty.has_backend) + } + + /// Generate the implementation for the Record trait. + pub(crate) fn gen_impl_record(&self) -> TokenStream { + // Capture the record type's generics and bounds in where clauses + let item_generics = self.record_item_generics(); + let (_, ty_generics_item, _) = item_generics.split_for_impl(); + let (impl_generics, ty_generics, where_clause) = self.ty.generics.split_for_impl(); + + let impl_generics = if let Some(impl_generic) = self.impl_generics() { + impl_generic + } else { + quote! { #impl_generics } + }; + + let name_item = &self.ty.item; + let into_item_fn = self.codegen.gen_into_item(name_item); + let from_item_fn = self.codegen.gen_from_item(); + + // Return the generated stream of token trees (i.e., code to be generated) + let name = &self.ty.name; + quote! { + impl #impl_generics burn::record::Record for #name #ty_generics #where_clause { + type Item = #name_item #ty_generics_item; + + #into_item_fn + #from_item_fn + + } + } + } + + /// Add backend generic type to the implementation block. + fn impl_generics(&self) -> Option { + if self.ty.has_backend { + return None; + } + + let param: syn::TypeParam = parse_quote! { B: burn::tensor::backend::Backend }; + let mut generics = self.ty.generics.clone(); + generics.params.push(syn::GenericParam::Type(param)); + + let (impl_generics, _ty_generics, _where_clause) = generics.split_for_impl(); + + Some(quote! {#impl_generics}) + } + + /// Get the generics attached to the record item type. + fn record_item_generics(&self) -> Generics { + let param: syn::Generics = parse_quote! { }; + let mut generics = self.ty.generics.clone(); + for param in param.params.into_iter() { + generics.params.push(param); + } + + if !self.ty.has_backend { + let param: syn::TypeParam = parse_quote! { B: burn::tensor::backend::Backend }; + generics.params.push(syn::GenericParam::Type(param)); + } + + generics + } + + pub(crate) fn from_ast(ast: &syn::DeriveInput) -> syn::Result { + Ok(Self { + ty: RecordType::from_ast(ast), + codegen: G::from_ast(ast)?, + }) + } +} + +/// Information about a record type. +struct RecordType { + /// Record type name. + name: Ident, + /// Record item type name. + item: Ident, + /// Lifetimes and type parameters attached to the record type declaration. + generics: Generics, + /// Whether or not the record type should specify a backend generic. + has_backend: bool, +} + +impl RecordType { + fn from_ast(ast: &syn::DeriveInput) -> Self { + let name = ast.ident.clone(); + let item = Ident::new(format!("{name}Item").as_str(), name.span()); + let has_backend = ast + .generics + .type_params() + .map(|param| param.ident == "B") + .reduce(|accum, is_backend| is_backend || accum) + .unwrap_or(false); + + Self { + name, + item, + generics: ast.generics.clone(), + has_backend, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen.rs new file mode 100644 index 0000000..5e3638d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen.rs @@ -0,0 +1,21 @@ +use proc_macro2::{Ident, TokenStream}; +use syn::Generics; + +/// Basic trait to be implemented for record generation. +pub(crate) trait RecordItemCodegen { + /// Initialize the record item. + fn from_ast(ast: &syn::DeriveInput) -> syn::Result + where + Self: Sized; + /// Generate the record item type. + fn gen_item_type( + &self, + item_name: &Ident, + generics: &Generics, + has_backend: bool, + ) -> TokenStream; + /// Generate the into_item function. + fn gen_into_item(&self, item_name: &Ident) -> TokenStream; + /// Generate the from item function. + fn gen_from_item(&self) -> TokenStream; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_enum.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_enum.rs new file mode 100644 index 0000000..934a1f2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_enum.rs @@ -0,0 +1,137 @@ +use crate::shared::enum_variant::{EnumVariant, parse_variants}; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::{Generics, Visibility, parse_quote}; + +use super::codegen::RecordItemCodegen; + +pub(crate) struct EnumRecordItemCodegen { + /// Enum variants. + variants: Vec, + vis: Visibility, +} + +impl RecordItemCodegen for EnumRecordItemCodegen { + fn from_ast(ast: &syn::DeriveInput) -> syn::Result { + Ok(Self { + variants: parse_variants(ast)?, + vis: ast.vis.clone(), + }) + } + + fn gen_item_type( + &self, + item_name: &Ident, + generics: &Generics, + has_backend: bool, + ) -> TokenStream { + let mut variants = quote! {}; + let mut serde_bounds = quote! {}; + let mut clone_bounds = vec![]; + let mut clone_match_arms = quote! {}; + let vis = &self.vis; + + // Capture the Record enum variant types and names to transpose them in RecordItem + for variant in self.variants.iter() { + let ty = &variant.ty; + let name = &variant.ident; + + variants.extend(quote! { + /// Variant to be serialized. + #name(<#ty as burn::record::Record>::Item), + }); + + // Item types must implement serialization/deserialization + serde_bounds.extend(quote! { + <#ty as burn::record::Record>::Item: burn::serde::Serialize + burn::serde::de::DeserializeOwned, + }); + clone_bounds.push(parse_quote! { + <#ty as burn::record::Record>::Item: Clone + }); + + clone_match_arms.extend(quote! { + Self::#name(inner) => Self::#name(inner.clone()), + }); + } + let serde_bound = serde_bounds.to_string(); + + // Capture the type's generics and bounds in where clauses + let mut generics = generics.clone(); + if !has_backend { + let param: syn::TypeParam = parse_quote! { B: burn::tensor::backend::Backend }; + generics.params.push(syn::GenericParam::Type(param)); + } + let (generics, type_generics, generics_where) = generics.split_for_impl(); + + let clone_bounds = generics_where.cloned().map(|mut where_clause| { + for predicate in clone_bounds { + where_clause.predicates.push(predicate); + } + where_clause + }); + + let clone_impl = quote! { + impl #generics Clone for #item_name #type_generics #clone_bounds { + fn clone(&self) -> Self { + match self { + #clone_match_arms + } + } + } + }; + + // Return the generated stream of token trees (i.e., code to be generated) + quote! { + + /// The record item type for the module. + #[derive(burn::serde::Serialize, burn::serde::Deserialize)] + #[serde(crate = "burn::serde")] + #[serde(bound = #serde_bound)] + #vis enum #item_name #generics #generics_where { + #variants + } + + #clone_impl + } + } + + fn gen_into_item(&self, _item_name: &Ident) -> TokenStream { + let mut into_item_match_arms = quote! {}; + + for variant in self.variants.iter() { + let name = &variant.ident; + + into_item_match_arms.extend(quote! { + Self::#name(record) => Self::Item::#name(burn::record::Record::::into_item::(record)), + }); + } + + quote! { + fn into_item(self) -> Self::Item { + match self { + #into_item_match_arms + } + } + } + } + + fn gen_from_item(&self) -> TokenStream { + let mut from_item_match_arms = quote! {}; + + for variant in self.variants.iter() { + let name = &variant.ident; + + from_item_match_arms.extend(quote! { + Self::Item::#name(item) => Self::#name(burn::record::Record::::from_item::(item, device)), + }); + } + + quote! { + fn from_item(item: Self::Item, device: &B::Device) -> Self { + match item { + #from_item_match_arms + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_struct.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_struct.rs new file mode 100644 index 0000000..00bf268 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/codegen_struct.rs @@ -0,0 +1,136 @@ +use crate::shared::field::{FieldTypeAnalyzer, parse_fields}; +use proc_macro2::{Ident, TokenStream}; +use quote::quote; +use syn::{Generics, Visibility, parse_quote}; + +use super::codegen::RecordItemCodegen; + +pub(crate) struct StructRecordItemCodegen { + fields: Vec, + vis: Visibility, +} + +impl RecordItemCodegen for StructRecordItemCodegen { + fn from_ast(ast: &syn::DeriveInput) -> syn::Result { + Ok(Self { + fields: parse_fields(ast) + .into_iter() + .map(FieldTypeAnalyzer::new) + .collect(), + vis: ast.vis.clone(), + }) + } + + fn gen_item_type( + &self, + item_name: &Ident, + generics: &Generics, + has_backend: bool, + ) -> TokenStream { + let mut fields = quote! {}; + let mut serde_bounds = quote! {}; + let mut clone_bounds = vec![]; + let mut clone_delegate = quote! {}; + let vis = &self.vis; + + for field in self.fields.iter() { + let ty = &field.field.ty; + let name = &field.field.ident; + + fields.extend(quote! { + /// Field to be serialized. + pub #name: <#ty as burn::record::Record>::Item, + }); + + serde_bounds.extend(quote! { + <#ty as burn::record::Record>::Item: burn::serde::Serialize + burn::serde::de::DeserializeOwned, + }); + + clone_bounds.push(parse_quote! { + <#ty as burn::record::Record>::Item: Clone + }); + + clone_delegate.extend(quote! { + #name: self.#name.clone(), + }); + } + let serde_bound = serde_bounds.to_string(); + + let mut generics = generics.clone(); + if !has_backend { + let param: syn::TypeParam = parse_quote! { B: burn::tensor::backend::Backend }; + generics.params.push(syn::GenericParam::Type(param)); + } + let (generics, type_generics, generics_where) = generics.split_for_impl(); + + let clone_bounds = generics_where.cloned().map(|mut where_clause| { + for predicate in clone_bounds { + where_clause.predicates.push(predicate); + } + where_clause + }); + + let clone_impl = quote! { + impl #generics Clone for #item_name #type_generics #clone_bounds { + fn clone(&self) -> Self { + Self { + #clone_delegate + } + } + } + }; + + quote! { + + /// The record item type for the module. + #[derive(burn::serde::Serialize, burn::serde::Deserialize)] + #[serde(crate = "burn::serde")] + #[serde(bound = #serde_bound)] + #vis struct #item_name #generics #generics_where { + #fields + } + + #clone_impl + } + } + + fn gen_into_item(&self, item_name: &Ident) -> TokenStream { + let mut body_into_item = quote! {}; + + for field in self.fields.iter() { + let name = &field.field.ident; + + body_into_item.extend(quote! { + #name: burn::record::Record::::into_item::(self.#name), + }); + } + + quote! { + fn into_item(self) -> Self::Item { + #item_name { + #body_into_item + } + } + } + } + + fn gen_from_item(&self) -> TokenStream { + let mut body_from_item = quote! {}; + + for field in self.fields.iter() { + let name = &field.field.ident; + + body_from_item.extend(quote! { + #name: burn::record::Record::::from_item::(item.#name, device), + }); + } + + quote! { + fn from_item(item: Self::Item, device: &B::Device) -> Self { + Self { + #body_from_item + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/mod.rs new file mode 100644 index 0000000..6b2b096 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/item/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod codegen; +pub(crate) mod codegen_enum; +pub(crate) mod codegen_struct; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/mod.rs new file mode 100644 index 0000000..31321f7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/record/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod codegen; +pub(crate) mod item; + +mod base; +pub(crate) use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/attribute.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/attribute.rs new file mode 100644 index 0000000..bc1f037 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/attribute.rs @@ -0,0 +1,49 @@ +use syn::{Attribute, Meta}; + +pub struct AttributeAnalyzer { + attr: Attribute, +} + +#[derive(Clone)] +pub struct AttributeItem { + pub value: syn::Lit, +} + +impl AttributeAnalyzer { + pub fn new(attr: Attribute) -> Self { + Self { attr } + } + + pub fn item(&self) -> AttributeItem { + let value = match &self.attr.meta { + Meta::List(val) => val.parse_args::().unwrap(), + Meta::NameValue(meta) => meta.clone(), + Meta::Path(_) => panic!("Path meta unsupported"), + }; + + let lit = match value.value { + syn::Expr::Lit(lit) => lit.lit, + _ => panic!("Only literal is supported"), + }; + + AttributeItem { value: lit } + } + + pub fn has_name(&self, name: &str) -> bool { + Self::path_syn_name(self.attr.path()) == name + } + + fn path_syn_name(path: &syn::Path) -> String { + let length = path.segments.len(); + let mut name = String::new(); + for (i, segment) in path.segments.iter().enumerate() { + if i == length - 1 { + name += segment.ident.to_string().as_str(); + } else { + let tmp = segment.ident.to_string() + "::"; + name += tmp.as_str(); + } + } + name + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/enum_variant.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/enum_variant.rs new file mode 100644 index 0000000..4dc98c4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/enum_variant.rs @@ -0,0 +1,103 @@ +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::{FieldsNamed, Variant}; + +/// Process a variant of an enum where the output is the result of the given mapper. +pub(crate) fn map_enum_variant( + variant: &Variant, + mapper: Mapper, +) -> (TokenStream, TokenStream) +where + Mapper: Fn(&Ident) -> TokenStream, +{ + let gen_fields_unnamed = |num: usize| { + let mut inputs = Vec::new(); + let mut outputs = Vec::new(); + + for i in 0..num { + let arg_name = Ident::new(&format!("arg_{i}"), Span::call_site()); + let input = quote! { #arg_name }; + let output = mapper(&arg_name); + + inputs.push(input); + outputs.push(output); + } + + (quote! (( #(#inputs),* )), quote! (( #(#outputs),* ))) + }; + let gen_fields_named = |fields: &FieldsNamed| { + let mut inputs = Vec::new(); + let mut outputs = Vec::new(); + + fields.named.iter().for_each(|field| { + let ident = field.ident.as_ref().expect("Named field to have a name."); + let input = quote! { #ident }; + let output = mapper(ident); + + inputs.push(input); + outputs.push(quote! { + #ident: #output + }); + }); + + (quote! {{ #(#inputs),* }}, quote! {{ #(#outputs),* }}) + }; + + match &variant.fields { + syn::Fields::Named(fields) => gen_fields_named(fields), + syn::Fields::Unnamed(_) => gen_fields_unnamed(variant.fields.len()), + syn::Fields::Unit => (quote! {}, quote! {}), + } +} + +/// An enum variant (simplified). +pub(crate) struct EnumVariant { + pub ident: syn::Ident, + pub ty: syn::Type, +} +pub(crate) fn parse_variants(ast: &syn::DeriveInput) -> syn::Result> { + let enum_data = match &ast.data { + syn::Data::Enum(data) => data, + _ => { + return Err(syn::Error::new_spanned( + ast, + "Module can only be derived for enums.", + )); + } + }; + + let mut variants = Vec::new(); + + for variant in enum_data.variants.iter() { + match &variant.fields { + syn::Fields::Unnamed(fields) if fields.unnamed.len() == 1 => { + let field = &fields.unnamed[0]; + + variants.push(EnumVariant { + ident: variant.ident.clone(), + ty: field.ty.clone(), + }); + } + syn::Fields::Unnamed(_) => { + return Err(syn::Error::new_spanned( + variant, + "Module derive only supports tuple enum variants with exactly one field.", + )); + } + syn::Fields::Named(_) => { + return Err(syn::Error::new_spanned( + variant, + "Module derive does not support struct enum variants.", + )); + } + syn::Fields::Unit => { + return Err(syn::Error::new_spanned( + variant, + "Module derive does not support unit enum variants.", + )); + } + } + } + + Ok(variants) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/field.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/field.rs new file mode 100644 index 0000000..aa64bcc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/field.rs @@ -0,0 +1,99 @@ +use super::attribute::AttributeAnalyzer; +use proc_macro2::Ident; +use syn::{Field, Type, TypePath}; + +#[derive(Clone)] +pub struct FieldTypeAnalyzer { + pub field: Field, +} + +impl FieldTypeAnalyzer { + pub fn new(field: Field) -> Self { + FieldTypeAnalyzer { field } + } + + pub fn ident(&self) -> Ident { + self.field.ident.clone().unwrap() + } + + pub fn is_of_type(&self, paths: &[&str]) -> bool { + match &self.field.ty { + syn::Type::Path(path) => { + let name = Self::path_name(path); + paths.contains(&name.as_str()) + } + _ => false, + } + } + + #[allow(dead_code)] + pub fn first_generic_field(&self) -> TypePath { + let err = || panic!("Field {} as no generic", self.field.ident.clone().unwrap()); + match &self.field.ty { + syn::Type::Path(path) => Self::path_generic_argument(path), + _ => err(), + } + } + pub fn path_generic_argument(path: &TypePath) -> TypePath { + let segment = path.path.segments.last().unwrap(); + let err = || panic!("Path segment {} has no generic", segment.ident.clone(),); + match &segment.arguments { + syn::PathArguments::None => err(), + syn::PathArguments::AngleBracketed(param) => { + let first_param = param.args.first().unwrap(); + + if let syn::GenericArgument::Type(Type::Path(path)) = first_param { + path.clone() + } else { + err() + } + } + syn::PathArguments::Parenthesized(_) => err(), + } + } + + fn path_name(path: &TypePath) -> String { + let length = path.path.segments.len(); + let mut name = String::new(); + for (i, segment) in path.path.segments.iter().enumerate() { + if i == length - 1 { + name += segment.ident.to_string().as_str(); + } else { + let tmp = segment.ident.to_string() + "::"; + name += tmp.as_str(); + } + } + name + } + + /// Returns the docs of the field. + pub fn docs(&self) -> impl Iterator { + self.field + .attrs + .iter() + .filter(|attr| attr.path().is_ident("doc")) + } + + pub fn attributes(&self) -> impl Iterator { + self.field + .attrs + .clone() + .into_iter() + .map(AttributeAnalyzer::new) + } +} + +pub(crate) fn parse_fields(ast: &syn::DeriveInput) -> Vec { + let mut fields = Vec::new(); + + match &ast.data { + syn::Data::Struct(struct_data) => { + for field in struct_data.fields.iter() { + fields.push(field.clone()); + } + } + syn::Data::Enum(_) => panic!("Only struct can be derived"), + syn::Data::Union(_) => panic!("Only struct can be derived"), + }; + fields +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/generics.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/generics.rs new file mode 100644 index 0000000..ce753d0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/generics.rs @@ -0,0 +1,63 @@ +use proc_macro2::Ident; +use quote::quote; +use syn::{Generics, WhereClause, WherePredicate, parse_quote}; + +#[derive(new)] +pub struct GenericsHelper { + pub(crate) generics: Generics, +} + +impl GenericsHelper { + pub fn add_predicate(&mut self, predicate: WherePredicate) { + let where_clause: WhereClause = match &self.generics.where_clause { + Some(val) => parse_quote! { + #val + #predicate, + }, + None => parse_quote! { + where + #predicate, + }, + }; + self.generics.where_clause = Some(where_clause); + } + + pub fn consts(&self) -> Vec { + self.generics + .const_params() + .map(|c| c.ident.clone()) + .collect() + } + + pub fn types(&self) -> Vec { + self.generics + .type_params() + .map(|tp| tp.ident.clone()) + .collect() + } + + pub fn fetch_backend_trait(&self) -> proc_macro2::TokenStream { + static BACKEND_TRAIT_COMPILATION_ERROR_MSG: &str = + "Modules should be generic over a backend. + - The generic argument named `B` should have its first trait bound being a backend trait. + - The default backend trait is `burn::tensor::backend::Backend`. + - Any backend trait is supported."; + + for param in self.generics.params.iter() { + if let syn::GenericParam::Type(ty) = ¶m + && ty.ident == "B" + { + let bound = ty + .bounds + .first() + .expect(BACKEND_TRAIT_COMPILATION_ERROR_MSG); + + return quote! { + #bound + }; + } + } + + panic!("{BACKEND_TRAIT_COMPILATION_ERROR_MSG}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/mod.rs new file mode 100644 index 0000000..009851f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-derive/src/shared/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod attribute; +pub(crate) mod enum_variant; +pub(crate) mod field; +pub(crate) mod generics; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/Cargo.toml new file mode 100644 index 0000000..706f4d8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/Cargo.toml @@ -0,0 +1,84 @@ +[package] +authors = [ + "laggui ", + "nathanielsimard ", +] +categories = ["science"] +description = "Backend dispatch for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-dispatch" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-dispatch" +documentation = "https://docs.rs/burn-dispatch" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [ + "std", + "ndarray", + "burn-autodiff?/default", + "burn-cpu?/default", + "burn-cuda?/default", + "burn-ndarray?/default", + "burn-rocm?/default", + "burn-tch?/default", + "burn-wgpu?/default", +] +doc = ["default"] +std = [ + "burn-backend/std", + "burn-std/std", + "burn-autodiff?/std", + "burn-cpu?/std", + "burn-cuda?/std", + "burn-ndarray?/std", + "burn-rocm?/std", + "burn-tch?/std", + "burn-wgpu?/std", +] +tracing = [ + "burn-autodiff?/tracing", + "burn-cpu?/tracing", + "burn-cuda?/tracing", + "burn-ndarray?/tracing", + "burn-rocm?/tracing", + "burn-tch?/tracing", + "burn-wgpu?/tracing", +] + +# Backends +cuda = ["burn-cuda"] +rocm = ["burn-rocm"] +ndarray = ["burn-ndarray"] +tch = ["burn-tch"] +vulkan = ["wgpu", "burn-wgpu/vulkan"] +webgpu = ["wgpu", "burn-wgpu/webgpu"] +metal = ["wgpu", "burn-wgpu/metal"] +wgpu = ["burn-wgpu"] +cpu = ["burn-cpu"] +autodiff = ["burn-autodiff"] + +[dependencies] +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } + +# Backends +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-cpu = { path = "../burn-cpu", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-rocm = { path = "../burn-rocm", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true, default-features = false } + +# Op macros with `.as_$inner_kind()` +paste = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/README.md b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/README.md new file mode 100644 index 0000000..7b81db5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/README.md @@ -0,0 +1,3 @@ +# Burn Backend Dispatch + +A multi-backend dispatch that forwards the tensor operations to the appropriate backend. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/build.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/build.rs new file mode 100644 index 0000000..8ee0352 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/build.rs @@ -0,0 +1,35 @@ +fn main() { + println!("cargo::rustc-check-cfg=cfg(wgpu_metal)"); + println!("cargo::rustc-check-cfg=cfg(wgpu_vulkan)"); + println!("cargo::rustc-check-cfg=cfg(wgpu_webgpu)"); + + // Detect which single wgpu backend is enabled + let metal = cfg!(feature = "metal"); + let vulkan = cfg!(feature = "vulkan"); + let webgpu = cfg!(feature = "webgpu"); + let enabled = [(metal, "metal"), (vulkan, "vulkan"), (webgpu, "webgpu")] + .iter() + .filter(|x| x.0) + .map(|x| x.1) + .collect::>(); + + // WGPU features are mutually exclusive, but we don't want to workspace to throw a compile error. + // In workspace builds with multiple features, we emit a warning and disable all WGPU backends. + if enabled.len() > 1 { + println!( + "cargo:warning=Only one WGPU backend can be enabled at once. Detected: [{}]. No WGPU backend will be available in this build. This is expected in workspace builds. For production, enable only one of: metal, vulkan, or webgpu.", + enabled.join(", ") + ); + return; + } + + if metal { + println!("cargo:rustc-cfg=wgpu_metal"); + } + if vulkan { + println!("cargo:rustc-cfg=wgpu_vulkan"); + } + if webgpu { + println!("cargo:rustc-cfg=wgpu_webgpu"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/backend.rs new file mode 100644 index 0000000..be4c55f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/backend.rs @@ -0,0 +1,392 @@ +use alloc::format; +use alloc::string::String; + +use burn_backend::Backend; +use burn_backend::ExecutionError; +use burn_std::DType; + +#[cfg(feature = "autodiff")] +use burn_autodiff::grads::Gradients; +#[cfg(feature = "autodiff")] +use burn_backend::AutodiffBackend; + +use crate::backends::*; +use crate::{DispatchDevice, DispatchTensor}; + +/// The main execution backend in Burn. +/// +/// [`Dispatch`] acts as a global backend that can manage multiple underlying +/// backends (e.g., `Cpu`, `Cuda`, `Wgpu`, `Metal`, etc.). +/// It is responsible for: +/// - Dispatching tensor operations to the appropriate backend. +/// - Managing cross-backend tensor transfers. +/// +/// Essentially, [`Dispatch`] is the single entry point for executing tensor operations +/// in a backend-agnostic way. It allows Burn to provide a unified, global backend +/// for users while still leveraging multiple specialized backends under the hood. +/// +/// # Example +/// +/// ```ignore +/// use burn::Dispatch; +/// use burn::DispatchDevice; +/// +/// // Select the device to execute operations on +/// let device = DispatchDevice::Cuda(Default::default()); +/// +/// // Create a tensor using the global backend +/// let t = Tensor::::zeros([128, 128], &device); +/// ``` +#[derive(Debug, Default, Clone)] +pub struct Dispatch; + +impl Backend for Dispatch { + type Device = DispatchDevice; + + type FloatTensorPrimitive = DispatchTensor; + + // TODO: either allow default dtype generic or remove associated types entirely? + type FloatElem = f32; + + type IntTensorPrimitive = DispatchTensor; + + type IntElem = i32; + + type BoolTensorPrimitive = DispatchTensor; + + type BoolElem = u8; + + type QuantizedTensorPrimitive = DispatchTensor; + + fn name(device: &Self::Device) -> String { + let inner = dispatch_device!(device, |device| B::name(device)); + format!("dispatch<{inner}>") + } + + fn seed(device: &Self::Device, seed: u64) { + dispatch_device!(device, |device| B::seed(device, seed)) + } + + fn sync(device: &Self::Device) -> Result<(), ExecutionError> { + dispatch_device!(device, |device| B::sync(device)) + } + + fn dtype_usage(device: &Self::Device, dtype: DType) -> burn_backend::DTypeUsageSet { + dispatch_device!(device, |device| B::dtype_usage(device, dtype)) + } + + fn ad_enabled(device: &Self::Device) -> bool { + match device { + #[cfg(feature = "autodiff")] + DispatchDevice::Autodiff(_) => true, + _ => false, + } + } +} + +#[cfg(feature = "autodiff")] +impl AutodiffBackend for Dispatch { + type InnerBackend = Dispatch; + + type Gradients = Gradients; + + fn backward(tensor: DispatchTensor) -> Self::Gradients { + match tensor { + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => match *tensor { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => tensor.autodiff().backward(), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => tensor.autodiff().backward(), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => tensor.autodiff().backward(), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => tensor.autodiff().backward(), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => tensor.autodiff().backward(), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => tensor.autodiff().backward(), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => tensor.autodiff().backward(), + DispatchTensor::Autodiff(_) => { + panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + _ => panic!("Requires autodiff tensor."), + } + } + + fn grad(tensor: &DispatchTensor, grads: &Self::Gradients) -> Option { + match &tensor { + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => match &**tensor { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::Cpu(crate::BackendTensor::Float(t))), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::Cuda(crate::BackendTensor::Float(t))), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::Metal(crate::BackendTensor::Float(t))), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::Rocm(crate::BackendTensor::Float(t))), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::Vulkan(crate::BackendTensor::Float(t))), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::WebGpu(crate::BackendTensor::Float(t))), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => tensor + .as_autodiff() + .grad(grads) + .map(|t| DispatchTensor::NdArray(crate::BackendTensor::Float(t))), + DispatchTensor::Autodiff(_) => { + panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + _ => panic!("Requires autodiff tensor."), + } + } + + fn grad_remove(tensor: &DispatchTensor, grads: &mut Self::Gradients) -> Option { + match &tensor { + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => match &**tensor { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::Cpu(crate::BackendTensor::Float(t))), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::Cuda(crate::BackendTensor::Float(t))), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::Metal(crate::BackendTensor::Float(t))), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::Rocm(crate::BackendTensor::Float(t))), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::Vulkan(crate::BackendTensor::Float(t))), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::WebGpu(crate::BackendTensor::Float(t))), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => tensor + .as_autodiff() + .grad_remove(grads) + .map(|t| DispatchTensor::NdArray(crate::BackendTensor::Float(t))), + DispatchTensor::Autodiff(_) => { + panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + _ => panic!("Requires autodiff tensor."), + } + } + + fn grad_replace(tensor: &DispatchTensor, grads: &mut Self::Gradients, grad: DispatchTensor) { + match &tensor { + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => match (&**tensor, grad) { + #[cfg(feature = "cpu")] + (DispatchTensor::Cpu(tensor), DispatchTensor::Cpu(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + #[cfg(feature = "cuda")] + (DispatchTensor::Cuda(tensor), DispatchTensor::Cuda(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + #[cfg(wgpu_metal)] + (DispatchTensor::Metal(tensor), DispatchTensor::Metal(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + #[cfg(feature = "rocm")] + (DispatchTensor::Rocm(tensor), DispatchTensor::Rocm(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + #[cfg(wgpu_vulkan)] + (DispatchTensor::Vulkan(tensor), DispatchTensor::Vulkan(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + #[cfg(wgpu_webgpu)] + (DispatchTensor::WebGpu(tensor), DispatchTensor::WebGpu(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + #[cfg(feature = "ndarray")] + (DispatchTensor::NdArray(tensor), DispatchTensor::NdArray(grad)) => { + tensor.as_autodiff().grad_replace(grads, grad.float()) + } + (DispatchTensor::Autodiff(_), _) => { + panic!("Autodiff should not wrap an autodiff tensor.") + } + (t, g) => panic!( + "The provided tensors are not on the same backend. Got backends {t:?} and {g:?}." + ), + }, + _ => panic!("Requires autodiff tensor."), + } + } + + fn inner(tensor: DispatchTensor) -> DispatchTensor { + match tensor { + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => match *tensor { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => { + DispatchTensor::Cpu(crate::BackendTensor::Float(tensor.autodiff().primitive)) + } + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => { + DispatchTensor::Cuda(crate::BackendTensor::Float(tensor.autodiff().primitive)) + } + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => { + DispatchTensor::Metal(crate::BackendTensor::Float(tensor.autodiff().primitive)) + } + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => { + DispatchTensor::Rocm(crate::BackendTensor::Float(tensor.autodiff().primitive)) + } + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => { + DispatchTensor::Vulkan(crate::BackendTensor::Float(tensor.autodiff().primitive)) + } + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => { + DispatchTensor::WebGpu(crate::BackendTensor::Float(tensor.autodiff().primitive)) + } + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => DispatchTensor::NdArray( + crate::BackendTensor::Float(tensor.autodiff().primitive), + ), + DispatchTensor::Autodiff(_) => { + panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + _ => panic!("Requires autodiff tensor."), + } + } + + fn int_inner(tensor: DispatchTensor) -> DispatchTensor { + tensor + } + + fn bool_inner(tensor: DispatchTensor) -> DispatchTensor { + tensor + } + + fn q_inner(tensor: DispatchTensor) -> DispatchTensor { + tensor + } + + fn from_inner(tensor: DispatchTensor) -> DispatchTensor { + match tensor { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => DispatchTensor::Autodiff(Box::new(DispatchTensor::Cpu( + crate::BackendTensor::Autodiff(Autodiff::>::from_inner(tensor.float())), + ))), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => DispatchTensor::Autodiff(Box::new( + DispatchTensor::Cuda(crate::BackendTensor::Autodiff( + Autodiff::>::from_inner(tensor.float()), + )), + )), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => DispatchTensor::Autodiff(Box::new( + DispatchTensor::Metal(crate::BackendTensor::Autodiff( + Autodiff::>::from_inner(tensor.float()), + )), + )), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => DispatchTensor::Autodiff(Box::new( + DispatchTensor::Rocm(crate::BackendTensor::Autodiff( + Autodiff::>::from_inner(tensor.float()), + )), + )), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => DispatchTensor::Autodiff(Box::new( + DispatchTensor::Vulkan(crate::BackendTensor::Autodiff( + Autodiff::>::from_inner(tensor.float()), + )), + )), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => DispatchTensor::Autodiff(Box::new( + DispatchTensor::WebGpu(crate::BackendTensor::Autodiff( + Autodiff::>::from_inner(tensor.float()), + )), + )), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => DispatchTensor::Autodiff(Box::new( + DispatchTensor::NdArray(crate::BackendTensor::Autodiff( + Autodiff::>::from_inner(tensor.float()), + )), + )), + DispatchTensor::Autodiff(_) => { + panic!("Autodiff should not wrap an autodiff tensor.") + } + } + } + + fn int_from_inner(tensor: DispatchTensor) -> DispatchTensor { + tensor + } + + fn bool_from_inner(tensor: DispatchTensor) -> DispatchTensor { + tensor + } + + fn q_from_inner(tensor: DispatchTensor) -> DispatchTensor { + tensor + } +} + +impl DispatchTensor { + pub(crate) fn device(&self) -> DispatchDevice { + match self { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => DispatchDevice::Cpu(tensor.device()), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => DispatchDevice::Cuda(tensor.device()), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => DispatchDevice::Metal(tensor.device()), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => DispatchDevice::Rocm(tensor.device()), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => DispatchDevice::Vulkan(tensor.device()), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => DispatchDevice::WebGpu(tensor.device()), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => DispatchDevice::NdArray(tensor.device()), + #[cfg(feature = "tch")] + DispatchTensor::LibTorch(tensor) => DispatchDevice::LibTorch(tensor.device()), + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => DispatchDevice::autodiff(tensor.device()), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/device.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/device.rs new file mode 100644 index 0000000..54de519 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/device.rs @@ -0,0 +1,415 @@ +use burn_backend::{DeviceId, DeviceOps}; + +use crate::backends::*; + +/// Represents a device for the [`Dispatch`](crate::Dispatch). +/// +/// Each variant corresponds to a backend that the [`Dispatch`](crate::Dispatch) can dispatch operations to. +/// +/// # Example +/// +/// ```ignore +/// use burn::DispatchDevice; +/// +/// #[cfg(feature = "cpu")] +/// let cpu_device = DispatchDevice::Cpu(Default::default()); +/// +/// #[cfg(feature = "cuda")] +/// let cuda_device = DispatchDevice::Cuda(Default::default()); +/// ``` +#[derive(Clone, Eq)] +pub enum DispatchDevice { + /// The [CPU backend](Cpu) device. + #[cfg(feature = "cpu")] + Cpu(CpuDevice), + + /// The [CUDA backend](Cuda) device. + #[cfg(feature = "cuda")] + Cuda(CudaDevice), + + /// The [Metal backend](Metal) device (via WGPU runtime). + #[cfg(wgpu_metal)] + Metal(WgpuDevice), + + /// The [ROCm backend](Rocm) device. + #[cfg(feature = "rocm")] + Rocm(RocmDevice), + + /// The [Vulkan backend](Vulkan) device. + #[cfg(wgpu_vulkan)] + Vulkan(WgpuDevice), + + /// The [WebGPU backend](WebGpu) device (via WGPU runtime). + #[cfg(wgpu_webgpu)] + WebGpu(WgpuDevice), + + /// The [NdArray backend](NdArray) device (CPU-only). + #[cfg(feature = "ndarray")] + NdArray(NdArrayDevice), + + /// The [LibTorch backend](LibTorch) device. + #[cfg(feature = "tch")] + LibTorch(LibTorchDevice), + + /// The [autodiff enabled backend](Autodiff) device. + #[cfg(feature = "autodiff")] + Autodiff(AutodiffDevice), +} + +#[cfg(feature = "autodiff")] +// This tuple struct mainly restricts users from creating Autodiff(Autodiff) devices. +/// A wrapper that enables automatic differentiation for a [`DispatchDevice`]. +/// +/// Use [`DispatchDevice::autodiff`] to construct this type. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AutodiffDevice(pub(crate) Box); + +// Useful for match in dispatch macros +impl core::ops::Deref for AutodiffDevice { + type Target = DispatchDevice; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl core::fmt::Debug for DispatchDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + #[cfg(feature = "cpu")] + Self::Cpu(device) => f.debug_tuple("Cpu").field(device).finish(), + #[cfg(feature = "cuda")] + Self::Cuda(device) => f.debug_tuple("Cuda").field(device).finish(), + #[cfg(wgpu_metal)] + Self::Metal(device) => f.debug_tuple("Metal").field(device).finish(), + #[cfg(feature = "rocm")] + Self::Rocm(device) => f.debug_tuple("Rocm").field(device).finish(), + #[cfg(wgpu_vulkan)] + Self::Vulkan(device) => f.debug_tuple("Vulkan").field(device).finish(), + #[cfg(wgpu_webgpu)] + Self::WebGpu(device) => f.debug_tuple("WebGpu").field(device).finish(), + #[cfg(feature = "ndarray")] + Self::NdArray(device) => f.debug_tuple("NdArray").field(device).finish(), + #[cfg(feature = "tch")] + Self::LibTorch(device) => f.debug_tuple("LibTorch").field(device).finish(), + #[cfg(feature = "autodiff")] + // Format without `AutodiffDevice` wrapper + Self::Autodiff(device) => f.debug_tuple("Autodiff").field(&device.0).finish(), + } + } +} + +impl Default for DispatchDevice { + #[allow(unreachable_code)] + fn default() -> Self { + // TODO: which priority? + + #[cfg(feature = "cpu")] + return Self::Cpu(CpuDevice); + + #[cfg(feature = "cuda")] + return Self::Cuda(CudaDevice::default()); + + #[cfg(wgpu_metal)] + return Self::Metal(burn_wgpu::WgpuDevice::default()); + + #[cfg(feature = "rocm")] + return Self::Rocm(RocmDevice::default()); + + #[cfg(wgpu_vulkan)] + return Self::Vulkan(burn_wgpu::WgpuDevice::default()); + + #[cfg(wgpu_webgpu)] + return Self::WebGpu(burn_wgpu::WgpuDevice::default()); + + #[cfg(feature = "ndarray")] + return Self::NdArray(NdArrayDevice::default()); + + #[cfg(feature = "tch")] + return Self::LibTorch(LibTorchDevice::default()); + } +} + +impl PartialEq for DispatchDevice { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + // If both are Autodiff, compare the inner devices + #[cfg(feature = "autodiff")] + (DispatchDevice::Autodiff(a), DispatchDevice::Autodiff(b)) => a == b, + // If one is Autodiff, compare it to the raw device + #[cfg(feature = "autodiff")] + (DispatchDevice::Autodiff(a), b) => a.0.as_ref() == b, + #[cfg(feature = "autodiff")] + (a, DispatchDevice::Autodiff(b)) => a == b.0.as_ref(), + #[cfg(feature = "cpu")] + (Self::Cpu(a), Self::Cpu(b)) => a == b, + #[cfg(feature = "cuda")] + (Self::Cuda(a), Self::Cuda(b)) => a == b, + #[cfg(wgpu_metal)] + (Self::Metal(a), Self::Metal(b)) => a == b, + #[cfg(feature = "rocm")] + (Self::Rocm(a), Self::Rocm(b)) => a == b, + #[cfg(wgpu_vulkan)] + (Self::Vulkan(a), Self::Vulkan(b)) => a == b, + #[cfg(wgpu_webgpu)] + (Self::WebGpu(a), Self::WebGpu(b)) => a == b, + #[cfg(feature = "ndarray")] + (Self::NdArray(a), Self::NdArray(b)) => a == b, + #[cfg(feature = "tch")] + (Self::LibTorch(a), Self::LibTorch(b)) => a == b, + #[allow(unreachable_patterns)] + (_, _) => false, + } + } +} + +/// Base multiplier to avoid type_id clashes between backends. +/// Limits the number of device types per backend, but this is a sensible limit. +const TYPE_ID_BASE: u16 = 10; + +impl DispatchDevice { + #[cfg(feature = "autodiff")] + /// Creates a new [`DispatchDevice`] with [automatic differentiation](Autodiff) enabled. + pub fn autodiff(device: impl Into) -> DispatchDevice { + let device = device.into(); + DispatchDevice::Autodiff(AutodiffDevice(Box::new(device))) + } + + /// Returns a unique number per variant to encode into type_id. + fn backend_id(&self) -> BackendId { + match self { + #[cfg(feature = "cpu")] + Self::Cpu(_) => BackendId::Cpu, + #[cfg(feature = "cuda")] + Self::Cuda(_) => BackendId::Cuda, + #[cfg(wgpu_metal)] + Self::Metal(_) => BackendId::Metal, + #[cfg(feature = "rocm")] + Self::Rocm(_) => BackendId::Rocm, + #[cfg(wgpu_vulkan)] + Self::Vulkan(_) => BackendId::Vulkan, + #[cfg(wgpu_webgpu)] + Self::WebGpu(_) => BackendId::WebGpu, + #[cfg(feature = "ndarray")] + Self::NdArray(_) => BackendId::NdArray, + #[cfg(feature = "tch")] + Self::LibTorch(_) => BackendId::LibTorch, + #[cfg(feature = "autodiff")] + Self::Autodiff(device) => device.0.backend_id(), + } + } + + /// Encode variant ID and backend type ID into a unique `type_id`. + fn encode_type_id(&self, backend_type_id: u16) -> u16 { + u16::from(self.backend_id()) * TYPE_ID_BASE + backend_type_id + } + + /// Decode an encoded `type_id` into variant ID and backend type ID. + fn decode_type_id(type_id: u16) -> (BackendId, u16) { + let variant = type_id / TYPE_ID_BASE; + let backend_type_id = type_id % TYPE_ID_BASE; + ( + BackendId::try_from(variant).expect("Unknown DispatchDevice variant"), + backend_type_id, + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u16)] +enum BackendId { + #[cfg(feature = "cpu")] + Cpu = 0, + #[cfg(feature = "cuda")] + Cuda = 1, + #[cfg(wgpu_metal)] + Metal = 2, + #[cfg(feature = "rocm")] + Rocm = 3, + #[cfg(wgpu_vulkan)] + Vulkan = 4, + #[cfg(wgpu_webgpu)] + WebGpu = 5, + #[cfg(feature = "ndarray")] + NdArray = 6, + #[cfg(feature = "tch")] + LibTorch = 7, +} + +impl From for u16 { + fn from(variant: BackendId) -> Self { + variant as u16 + } +} + +impl TryFrom for BackendId { + type Error = (); + + fn try_from(value: u16) -> Result { + match value { + #[cfg(feature = "cpu")] + 0 => Ok(Self::Cpu), + #[cfg(feature = "cuda")] + 1 => Ok(Self::Cuda), + #[cfg(wgpu_metal)] + 2 => Ok(Self::Metal), + #[cfg(feature = "rocm")] + 3 => Ok(Self::Rocm), + #[cfg(wgpu_vulkan)] + 4 => Ok(Self::Vulkan), + #[cfg(wgpu_webgpu)] + 5 => Ok(Self::WebGpu), + #[cfg(feature = "ndarray")] + 6 => Ok(Self::NdArray), + #[cfg(feature = "tch")] + 7 => Ok(Self::LibTorch), + _ => Err(()), + } + } +} + +impl DeviceOps for DispatchDevice { + fn inner(&self) -> &Self { + match self { + #[cfg(feature = "autodiff")] + DispatchDevice::Autodiff(device) => &device.0, + device => device, + } + } +} + +impl burn_std::device::Device for DispatchDevice { + fn from_id(mut device_id: DeviceId) -> Self { + let (dispatch_id, backend_type_id) = Self::decode_type_id(device_id.type_id); + device_id.type_id = backend_type_id; + + match dispatch_id { + #[cfg(feature = "cpu")] + BackendId::Cpu => Self::Cpu(CpuDevice::from_id(device_id)), + #[cfg(feature = "cuda")] + BackendId::Cuda => Self::Cuda(CudaDevice::from_id(device_id)), + #[cfg(wgpu_metal)] + BackendId::Metal => Self::Metal(WgpuDevice::from_id(device_id)), + #[cfg(feature = "rocm")] + BackendId::Rocm => Self::Rocm(RocmDevice::from_id(device_id)), + #[cfg(wgpu_vulkan)] + BackendId::Vulkan => Self::Vulkan(WgpuDevice::from_id(device_id)), + #[cfg(wgpu_webgpu)] + BackendId::WebGpu => Self::WebGpu(WgpuDevice::from_id(device_id)), + #[cfg(feature = "ndarray")] + BackendId::NdArray => Self::NdArray(NdArrayDevice::from_id(device_id)), + #[cfg(feature = "tch")] + BackendId::LibTorch => Self::LibTorch(LibTorchDevice::from_id(device_id)), + } + } + + fn to_id(&self) -> DeviceId { + let mut device_id = match self { + #[cfg(feature = "cpu")] + Self::Cpu(device) => device.to_id(), + #[cfg(feature = "cuda")] + Self::Cuda(device) => device.to_id(), + #[cfg(wgpu_metal)] + Self::Metal(device) => device.to_id(), + #[cfg(feature = "rocm")] + Self::Rocm(device) => device.to_id(), + #[cfg(wgpu_vulkan)] + Self::Vulkan(device) => device.to_id(), + #[cfg(wgpu_webgpu)] + Self::WebGpu(device) => device.to_id(), + #[cfg(feature = "ndarray")] + Self::NdArray(device) => device.to_id(), + #[cfg(feature = "tch")] + Self::LibTorch(device) => device.to_id(), + #[cfg(feature = "autodiff")] + Self::Autodiff(device) => device.0.to_id(), + }; + device_id.type_id = self.encode_type_id(device_id.type_id); + device_id + } + + fn device_count(type_id: u16) -> usize { + let (dispatch_id, backend_type_id) = Self::decode_type_id(type_id); + match dispatch_id { + #[cfg(feature = "cpu")] + BackendId::Cpu => CpuDevice::device_count(backend_type_id), + #[cfg(feature = "cuda")] + BackendId::Cuda => CudaDevice::device_count(backend_type_id), + #[cfg(wgpu_metal)] + BackendId::Metal => WgpuDevice::device_count(backend_type_id), + #[cfg(feature = "rocm")] + BackendId::Rocm => RocmDevice::device_count(backend_type_id), + #[cfg(wgpu_vulkan)] + BackendId::Vulkan => WgpuDevice::device_count(backend_type_id), + #[cfg(wgpu_webgpu)] + BackendId::WebGpu => WgpuDevice::device_count(backend_type_id), + #[cfg(feature = "ndarray")] + BackendId::NdArray => NdArrayDevice::device_count(backend_type_id), + #[cfg(feature = "tch")] + BackendId::LibTorch => LibTorchDevice::device_count(backend_type_id), + } + } +} + +#[cfg(feature = "cpu")] +impl From for DispatchDevice { + fn from(device: CpuDevice) -> Self { + DispatchDevice::Cpu(device) + } +} + +#[cfg(feature = "cuda")] +impl From for DispatchDevice { + fn from(device: CudaDevice) -> Self { + DispatchDevice::Cuda(device) + } +} + +#[cfg(wgpu_metal)] +impl From for DispatchDevice { + fn from(device: WgpuDevice) -> Self { + DispatchDevice::Metal(device) + } +} + +#[cfg(feature = "rocm")] +impl From for DispatchDevice { + fn from(device: RocmDevice) -> Self { + DispatchDevice::Rocm(device) + } +} + +#[cfg(wgpu_vulkan)] +impl From for DispatchDevice { + fn from(device: WgpuDevice) -> Self { + DispatchDevice::Vulkan(device) + } +} + +#[cfg(wgpu_webgpu)] +impl From for DispatchDevice { + fn from(device: WgpuDevice) -> Self { + DispatchDevice::WebGpu(device) + } +} + +#[cfg(feature = "ndarray")] +impl From for DispatchDevice { + fn from(device: NdArrayDevice) -> Self { + DispatchDevice::NdArray(device) + } +} + +#[cfg(feature = "tch")] +impl From for DispatchDevice { + fn from(device: LibTorchDevice) -> Self { + DispatchDevice::LibTorch(device) + } +} + +#[cfg(feature = "tch")] +impl From for DispatchDevice { + fn from(device: LibTorchDevice) -> Self { + DispatchDevice::LibTorch(device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/lib.rs new file mode 100644 index 0000000..c0141ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/lib.rs @@ -0,0 +1,90 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![recursion_limit = "138"] + +//! Burn multi-backend dispatch. +//! +//! # Available Backends +//! +//! The dispatch backend supports the following variants, each enabled via cargo features: +//! +//! | Backend | Feature | Description | +//! |------------|------------|-------------| +//! | `Cpu` | `cpu` | Rust CPU backend (MLIR + LLVM) | +//! | `Cuda` | `cuda` | NVIDIA CUDA backend | +//! | `Metal` | `metal` | Apple Metal backend via `wgpu` (MSL) | +//! | `Rocm` | `rocm` | AMD ROCm backend | +//! | `Vulkan` | `vulkan` | Vulkan backend via `wgpu` (SPIR-V) | +//! | `WebGpu` | `webgpu` | WebGPU backend via `wgpu` (WGSL) | +//! | `NdArray` | `ndarray` | Pure Rust CPU backend using `ndarray` | +//! | `LibTorch` | `tch` | Libtorch backend via `tch` | +//! | `Autodiff` | `autodiff` | Autodiff-enabled backend (used in combination with any of the backends above) | +//! +//! **Note:** WGPU-based backends (`metal`, `vulkan`, `webgpu`) are mutually exclusive. +//! All other backends can be combined freely. +//! +//! ## WGPU Backend Exclusivity +//! +//! The WGPU-based backends (`metal`, `vulkan`, `webgpu`) are **mutually exclusive** due to +//! the current automatic compile, which can only select one target at a time. +//! +//! Enable only **one** of these features in your `Cargo.toml`: +//! - `metal` +//! - `vulkan` +//! - `webgpu` +//! +//! If multiple WGPU features are enabled, the build script will emit a warning and **disable all WGPU +//! backends** to prevent unintended behavior. + +#[cfg(not(any( + feature = "cpu", + feature = "cuda", + wgpu_metal, + feature = "rocm", + wgpu_vulkan, + wgpu_webgpu, + feature = "ndarray", + feature = "tch", +)))] +compile_error!("At least one backend feature must be enabled."); + +#[macro_use] +mod macros; + +mod backend; +mod device; +mod ops; +mod tensor; + +pub use backend::*; +pub use device::*; +pub use tensor::*; + +extern crate alloc; + +/// Backends and devices used. +pub(crate) mod backends { + #[cfg(feature = "autodiff")] + pub use burn_autodiff::Autodiff; + + #[cfg(feature = "cpu")] + pub use burn_cpu::{Cpu, CpuDevice}; + #[cfg(feature = "cuda")] + pub use burn_cuda::{Cuda, CudaDevice}; + #[cfg(feature = "rocm")] + pub use burn_rocm::{Rocm, RocmDevice}; + #[cfg(wgpu_metal)] + pub use burn_wgpu::Metal; + #[cfg(wgpu_vulkan)] + pub use burn_wgpu::Vulkan; + #[cfg(wgpu_webgpu)] + pub use burn_wgpu::WebGpu; + #[cfg(any(wgpu_metal, wgpu_vulkan, wgpu_webgpu))] + pub use burn_wgpu::WgpuDevice; + + #[cfg(feature = "ndarray")] + pub use burn_ndarray::{NdArray, NdArrayDevice}; + #[cfg(feature = "tch")] + pub use burn_tch::{LibTorch, LibTorchDevice}; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/macros.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/macros.rs new file mode 100644 index 0000000..187cada --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/macros.rs @@ -0,0 +1,1197 @@ +/// Supplies a list of all supported backends and their corresponding feature flags +/// to a callback macro. This centralizes the backend registry. +macro_rules! backend_list { + ($callback:ident, $($extra:tt)*) => { + $callback! { + $($extra)*; + [Cpu, feature = "cpu"], + [Cuda, feature = "cuda"], + [Metal, wgpu_metal], + [Rocm, feature = "rocm"], + [Vulkan, wgpu_vulkan], + [WebGpu, wgpu_webgpu], + [NdArray, feature = "ndarray"], + [LibTorch, feature = "tch"] + } + }; +} + +/// Supplies a matrix of cross-backend combinations. Used for operations where the source and destination backends may differ. +macro_rules! backend_matrix { + ($callback:ident, $($extra:tt)*) => { + $callback! { + $($extra)*; + [Cpu, feature = "cpu"] => [[Cuda, feature = "cuda"], [Metal, wgpu_metal], [Rocm, feature = "rocm"], [Vulkan, wgpu_vulkan], [WebGpu, wgpu_webgpu], [NdArray, feature = "ndarray"], [LibTorch, feature = "tch"]]; + [Cuda, feature = "cuda"] => [[Cpu, feature = "cpu"], [Metal, wgpu_metal], [Rocm, feature = "rocm"], [Vulkan, wgpu_vulkan], [WebGpu, wgpu_webgpu], [NdArray, feature = "ndarray"], [LibTorch, feature = "tch"]]; + [Metal, wgpu_metal] => [[Cpu, feature = "cpu"], [Cuda, feature = "cuda"], [Rocm, feature = "rocm"], [NdArray, feature = "ndarray"], [LibTorch, feature = "tch"]]; + [Rocm, feature = "rocm"] => [[Cpu, feature = "cpu"], [Cuda, feature = "cuda"], [Metal, wgpu_metal], [Vulkan, wgpu_vulkan], [WebGpu, wgpu_webgpu], [NdArray, feature = "ndarray"], [LibTorch, feature = "tch"]]; + [Vulkan, wgpu_vulkan] => [[Cpu, feature = "cpu"], [Cuda, feature = "cuda"], [Rocm, feature = "rocm"], [NdArray, feature = "ndarray"], [LibTorch, feature = "tch"]]; + [WebGpu, wgpu_webgpu] => [[Cpu, feature = "cpu"], [Cuda, feature = "cuda"], [Rocm, feature = "rocm"], [NdArray, feature = "ndarray"], [LibTorch, feature = "tch"]]; + [NdArray, feature = "ndarray"] => [[Cpu, feature = "cpu"], [Cuda, feature = "cuda"], [Metal, wgpu_metal], [Rocm, feature = "rocm"], [Vulkan, wgpu_vulkan], [WebGpu, wgpu_webgpu], [LibTorch, feature = "tch"]]; + [LibTorch, feature = "tch"] => [[Cpu, feature = "cpu"], [Cuda, feature = "cuda"], [Metal, wgpu_metal], [Rocm, feature = "rocm"], [Vulkan, wgpu_vulkan], [WebGpu, wgpu_webgpu], [NdArray, feature = "ndarray"]] + } + }; +} + +/// Match arm generator for `dispatch_device`. +/// Maps each backend variant to a block where the specific backend type is bound to `B`. +macro_rules! dispatch_device_arms { + ( + $device:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => { + match $device { + // Autodiff arm first + #[cfg(feature = "autodiff")] + $crate::DispatchDevice::Autodiff(inner) => { + // Recursively dispatch on inner + dispatch_device_arms!( + @autodiff + &**inner, + |$inner| $body; + $([$Backend, $cfg]),* + ) + }, + $( + #[cfg($cfg)] + $crate::DispatchDevice::$Backend($inner) => { + type B = $Backend; + $body + } + )* + } + }; + ( + @autodiff + $device:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => { + match $device { + $( + #[cfg($cfg)] + $crate::DispatchDevice::$Backend($inner) => { + type B = Autodiff<$Backend>; + $body + } + )* + $crate::DispatchDevice::Autodiff(_) => panic!("Autodiff should not wrap an autodiff device.") + } + }; +} + +/// Dispatches an operation body based on the provided device. +macro_rules! dispatch_device { + ($device:expr, |$inner:ident| $body:expr) => { + backend_list!(dispatch_device_arms, $device, |$inner| $body) + }; +} + +/// Match arm generator for `to_device`. +/// Handles the logic for same-backend transfers (fast path) and cross-backend +/// transfers by generating a grid of all device combinations provided via `backend_matrix`. +macro_rules! to_device_arms { + ( + $kind:ident, $inner_fn:ident, $tensor:expr, $device:expr, $to_device:ident, |$inner:ident, $device_ident:ident| $body:expr; + $( [$B1:ident, $src_cfg:meta] => [ $( [$B2:ident, $dst_cfg:meta] ),+ ] );* + ) => { + match ($tensor, $device) { + // --- Same backend to_device --- + $( + #[cfg($src_cfg)] + ($crate::DispatchTensor::$B1(tensor), $crate::DispatchDevice::$B1(d)) => { + $crate::DispatchTensor::$B1($crate::BackendTensor::$kind( + $B1::::$to_device(tensor.$inner_fn(), d) + )) + } + )* + + // --- Cross backend arms --- + // This loop generates the grid of combinations + $( + $( + #[cfg(all($src_cfg, $dst_cfg))] + ($crate::DispatchTensor::$B1(tensor), $crate::DispatchDevice::$B2($device_ident)) => { + type B1 = $B1; + type B2 = $B2; + let $inner = tensor.$inner_fn(); + + $crate::DispatchTensor::$B2( + $crate::BackendTensor::$kind($body) + ) + } + )+ + )* + #[cfg(feature = "autodiff")] + (_, $crate::DispatchDevice::Autodiff(_)) | ($crate::DispatchTensor::Autodiff(_), _) => panic!("Operation not marked for autodiff.") + } + }; +} + +/// Handles tensor movement between devices, supporting both same-backend transfers +/// and cross-backend dispatches. +macro_rules! to_device { + ($kind:ident, $inner_fn:ident, $tensor:expr, $device:expr, $to_device:ident, |$inner:ident, $device_ident:ident| $body:expr) => { + backend_matrix!( + to_device_arms, + $kind, + $inner_fn, + $tensor, + $device, + $to_device, + |$inner, $device_ident| $body + ) + }; +} + +/// Match arm generator for `float_to_device`. +/// +/// Similar to `to_device_arms`, but float tensors are checked for autodiff support. +macro_rules! float_to_device_arms { + ( + $tensor:expr, $device:expr, $to_device:ident, |$inner:ident, $device_ident:ident| $body:expr; + $( [$B1:ident, $src_cfg:meta] => [ $( [$B2:ident, $dst_cfg:meta] ),+ ] );* + ) => { + match ($tensor, $device) { + #[cfg(feature = "autodiff")] + ($crate::DispatchTensor::Autodiff(tensor), $crate::DispatchDevice::Autodiff(device)) => { + float_to_device_arms!( + @autodiff + *tensor, &**device, $to_device; + $([$B1, $src_cfg]);* + ) + + } + // --- Same backend to_device --- + $( + #[cfg($src_cfg)] + ($crate::DispatchTensor::$B1(tensor), $crate::DispatchDevice::$B1(d)) => { + $crate::DispatchTensor::$B1($crate::BackendTensor::Float( + $B1::::$to_device(tensor.float(), d) + )) + } + )* + + // --- Cross backend arms --- + // This loop generates the grid of combinations + $( + $( + #[cfg(all($src_cfg, $dst_cfg))] + ($crate::DispatchTensor::$B1(tensor), $crate::DispatchDevice::$B2($device_ident)) => { + type B1 = $B1; + type B2 = $B2; + let $inner = tensor.float(); + + $crate::DispatchTensor::$B2( + $crate::BackendTensor::Float($body) + ) + } + )+ + )* + #[cfg(feature = "autodiff")] + ($crate::DispatchTensor::Autodiff(_), _) | (_, $crate::DispatchDevice::Autodiff(_)) => panic!("Cannot move between autodiff and non-autodiff instances.") + } + }; + + // Autodiff(DispatchTensor) + ( + @autodiff + $tensor:expr, $device:expr, $to_device:ident; + $( [$B1:ident, $src_cfg:meta] );* + ) => {{ + match ($tensor, $device) { + // --- Same backend to_device --- + $( + #[cfg($src_cfg)] + ($crate::DispatchTensor::$B1(tensor), $crate::DispatchDevice::$B1(d)) => { + $crate::DispatchTensor::Autodiff(Box::new($crate::DispatchTensor::$B1($crate::BackendTensor::Autodiff( + Autodiff::<$B1>::$to_device(tensor.autodiff(), d) + )))) + } + )* + (_, _) => unimplemented!("Autodiff tensor cannot be moved between backends.") + } + }}; +} + +/// Handles float tensor movement between devices (that might support autodiff). +macro_rules! float_to_device { + ($kind:ident, $inner_fn:ident, $tensor:expr, $device:expr, $to_device:ident, |$inner:ident, $device_ident:ident| $body:expr) => { + backend_matrix!( + float_to_device_arms, + $tensor, + $device, + $to_device, + |$inner, $device_ident| $body + ) + }; +} + +/// Dispatches a tensor creation operation (e.g., zeros, ones) to the correct backend +/// based on the provided device. +macro_rules! creation_op { + ($kind:ident, $device:expr, |$inner:ident| $body:expr) => { + backend_list!(creation_op_arms, $kind, $device, |$inner| $body) + }; +} + +/// Match arm generator for `creation_float`. +/// +/// Similar to `creation_op_arms`, but float tensors are checked for autodiff support. +macro_rules! creation_op_arms { + ( + $kind:ident, + $device:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $device { + // Autodiff arm first + #[cfg(feature = "autodiff")] + $crate::DispatchDevice::Autodiff(inner) => { + // Recursively dispatch on inner + creation_op_arms!( + @autodiff + $kind, + &**inner, + |$inner| $body; + $([$Backend, $cfg]),* + ) + }, + $( + #[cfg($cfg)] + $crate::DispatchDevice::$Backend($inner) => { + type B = $Backend; + $crate::DispatchTensor::$Backend( + $crate::BackendTensor::$kind($body) + ) + } + )* + } + }}; + + ( + @autodiff + $kind:ident, + $device:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $device { + $( + #[cfg($cfg)] + $crate::DispatchDevice::$Backend($inner) => { + type B = Autodiff<$Backend>; + wrap_float!( + @wrap_autodiff + $kind, + $Backend, + { $body } + ) + } + )* + $crate::DispatchDevice::Autodiff(_) => panic!("Autodiff should not wrap an autodiff device.") + } + }}; +} + +/// Wrap the result in the backend tensor kind, handling float -> autodiff. +#[cfg(feature = "autodiff")] +macro_rules! wrap_float { + ( + @wrap_autodiff Float, + $Backend:ident, + $expr:expr + ) => { + $crate::DispatchTensor::Autodiff(Box::new($crate::DispatchTensor::$Backend( + $crate::BackendTensor::Autodiff($expr), + ))) + }; + + ( + @wrap_autodiff $other:ident, + $Backend:ident, + $expr:expr + ) => { + $crate::DispatchTensor::$Backend($crate::BackendTensor::$other($expr)) + }; +} + +/// Match arm generator for `unary_op`. +/// Unwraps the inner tensor primitive (e.g., `inner.float()`) and provides the backend type `B` +/// for the operation. +/// +/// When the return kind is provided, the result is wrapped in the corresponding `DispatchTensor` variant. +macro_rules! unary_op_arms { + ( + $kind:ident, + $inner_kind:ident, + $tensor:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $tensor { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend($inner) => { + type B = $Backend; + let $inner = $inner.$inner_kind(); + $crate::DispatchTensor::$Backend($crate::BackendTensor::$kind($body)) + } + )* + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(_) => panic!("Operation not marked for autodiff.") + } + }}; + + // Operations that do not return a tensor kind + ( + $inner_kind:ident, + $tensor:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $tensor { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend($inner) => { + type B = $Backend; + let $inner = $inner.$inner_kind(); + $body + } + )* + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(_) => panic!("Operation not marked for autodiff.") + } + }}; +} + +/// Backend dispatch for unary operations. +/// +/// When the return `=> Kind` is not provided, the operation output is not wrapped in a dispatch tensor (e.g., `into_data(..)`) +macro_rules! unary_op { + ($tensor:expr, $inner_kind:ident, |$inner:ident| $body:expr => $kind:ident) => { + backend_list!(unary_op_arms, $kind, $inner_kind, $tensor, |$inner| { + $body + }) + }; + ($tensor:expr, $inner_kind:ident, |$inner:ident| $body:expr) => { + backend_list!(unary_op_arms, $inner_kind, $tensor, |$inner| { $body }) + }; +} + +/// Match arm generator for `unary_float`. +/// +/// Similar to `unary_op_arms`, but float tensors are checked for autodiff support. +macro_rules! unary_float_arms { + ( + $mode:ident, // `owned` or `ref` + $kind:ident, + $inner_kind:ident, + $tensor:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $tensor { + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(inner) => { + unary_float_arms!( + @autodiff $mode, + $kind, + { if_mode!($mode, &**inner, *inner) }, + |$inner| $body; + $([$Backend, $cfg]),* + ) + }, + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend($inner) => { + type B = $Backend; + let $inner = unary_float_arms!(@unwrap $mode, $inner, $inner_kind); + $crate::DispatchTensor::$Backend( + $crate::BackendTensor::$kind($body) + ) + } + )* + } + }}; + + // --- Autodiff recursive arm --- + ( + @autodiff $mode:ident, + $kind:ident, + $tensor:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $tensor { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend($inner) => { + type B = Autodiff<$Backend>; + let $inner = unary_float_arms!(@unwrap_ad $mode, $inner); + wrap_float!( @wrap_autodiff $kind, $Backend, { $body } ) + } + )* + $crate::DispatchTensor::Autodiff(_) => panic!("Autodiff should not wrap an autodiff tensor.") + } + }}; + + // --- Non-wrapping arms (operations not returning a tensor) --- + ( + $mode:ident, + $inner_kind:ident, + $tensor:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $tensor { + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(inner) => { + unary_float_arms!( + @autodiff $mode, + { if_mode!($mode, &**inner, *inner) }, + |$inner| $body; + $([$Backend, $cfg]),* + ) + }, + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend($inner) => { + type B = $Backend; + let $inner = unary_float_arms!(@unwrap $mode, $inner, $inner_kind); + $body + } + )* + } + }}; + ( + @autodiff $mode:ident, + $tensor:expr, + |$inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match $tensor { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend($inner) => { + type B = Autodiff<$Backend>; + let $inner = unary_float_arms!(@unwrap_ad $mode, $inner); + $body + } + )* + $crate::DispatchTensor::Autodiff(_) => panic!("Autodiff should not wrap an autodiff tensor.") + } + }}; + + // --- Helpers to unwarp the tensor based on owned/ref --- + (@unwrap owned, $inner:ident, $inner_kind:ident) => { $inner.$inner_kind() }; + (@unwrap ref, $inner:ident, $inner_kind:ident) => { + paste::paste! { $inner.[< as_ $inner_kind >]() } + }; + + (@unwrap_ad owned, $inner:ident) => { $inner.autodiff() }; + (@unwrap_ad ref, $inner:ident) => { $inner.as_autodiff() }; + +} + +#[cfg(feature = "autodiff")] +/// Utility to pick a token based on mode +macro_rules! if_mode { + (ref, $if_ref:expr, $if_owned:expr) => { + $if_ref + }; + (owned, $if_ref:expr, $if_owned:expr) => { + $if_owned + }; +} + +/// Backend dispatch for float unary operations (that might support autodiff). +/// +/// When the return `=> Kind` is not provided, the operation output is not wrapped in a dispatch tensor (e.g., `into_data(..)`) +macro_rules! unary_float { + // Owned with return kind + ($tensor:expr, $inner_kind:ident, |$inner:ident| $body:expr => $kind:ident) => { + backend_list!( + unary_float_arms, + owned, + $kind, + $inner_kind, + $tensor, + |$inner| { $body } + ) + }; + // Owned without return kind + ($tensor:expr, $inner_kind:ident, |$inner:ident| $body:expr) => { + backend_list!(unary_float_arms, owned, $inner_kind, $tensor, |$inner| { + $body + }) + }; + // Reference without return kind + (ref $tensor:expr, $inner_kind:ident, |$inner:ident| $body:expr) => { + backend_list!(unary_float_arms, ref, $inner_kind, $tensor, |$inner| { + $body + }) + }; +} +/// Match arm generator for `binary_op`. +/// Matches two tensors to ensure they share the same backend before unwrapping them for the operation. +macro_rules! binary_op_arms { + ( + $kind:ident, + ($lhs:expr, $lhs_kind:ident), + ($rhs:expr, $rhs_kind:ident), + |$lhs_inner:ident, $rhs_inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match ($lhs, $rhs) { + $( + #[cfg($cfg)] + ($crate::DispatchTensor::$Backend($lhs_inner), $crate::DispatchTensor::$Backend($rhs_inner)) => { + type B = $Backend; + let $lhs_inner = $lhs_inner.$lhs_kind(); + let $rhs_inner = $rhs_inner.$rhs_kind(); + $crate::DispatchTensor::$Backend($crate::BackendTensor::$kind($body)) + } + )* + #[allow(unreachable_patterns)] + (lhs, rhs) => { + panic!( + "The provided tensors are not on the same backend. Got backends {:?} and {:?}.", lhs, rhs + ); + } + } + }}; +} + +/// Backend dispatch for binary operations. +/// Automatically verifies that both tensors reside on the same backend. +macro_rules! binary_op { + (($lhs:expr, $lhs_kind:ident), ($rhs:expr, $rhs_kind:ident), |$lhs_inner:ident, $rhs_inner:ident| $body:expr => $kind:ident) => { + backend_list!( + binary_op_arms, + $kind, + ($lhs, $lhs_kind), + ($rhs, $rhs_kind), + |$lhs_inner, $rhs_inner| { $body } + ) + }; +} + +/// Match arm generator for `binary_float`. +/// Matches two tensors to ensure they share the same backend before unwrapping them for the operation. +macro_rules! binary_float_arms { + // (float, float) binary op + ( + $kind:ident, + ($lhs:expr, float), + ($rhs:expr, float), + |$lhs_inner:ident, $rhs_inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match ($lhs, $rhs) { + // Autodiff arms first + #[cfg(feature = "autodiff")] + ($crate::DispatchTensor::Autodiff(lhs_inner), $crate::DispatchTensor::Autodiff(rhs_inner)) => { + // Recursively dispatch on inner + binary_float_arms!( + @autodiff + $kind, + (*lhs_inner, autodiff), + (*rhs_inner, autodiff), + |$lhs_inner, $rhs_inner| $body; + $([$Backend, $cfg]),* + ) + }, + $( + #[cfg($cfg)] + ($crate::DispatchTensor::$Backend($lhs_inner), $crate::DispatchTensor::$Backend($rhs_inner)) => { + type B = $Backend; + let $lhs_inner = $lhs_inner.float(); + let $rhs_inner = $rhs_inner.float(); + $crate::DispatchTensor::$Backend($crate::BackendTensor::$kind($body)) + } + )* + #[allow(unreachable_patterns)] + (lhs, rhs) => { + panic!( + "The provided tensors are not on the same backend. Got backends {:?} and {:?}.", lhs, rhs + ); + } + } + }}; + // (float, any) binary op + ( + $kind:ident, + ($lhs:expr, float), + ($rhs:expr, $rhs_kind:ident), + |$lhs_inner:ident, $rhs_inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match ($lhs, $rhs) { + $( + // Autodiff arms first + #[cfg(all(feature = "autodiff", $cfg))] + ($crate::DispatchTensor::Autodiff(lhs_inner), $crate::DispatchTensor::$Backend($rhs_inner)) => { + // Match on inner + match *lhs_inner { + $crate::DispatchTensor::$Backend($lhs_inner) => { + type B = Autodiff<$Backend>; + let $lhs_inner = $lhs_inner.autodiff(); + let $rhs_inner = $rhs_inner.$rhs_kind(); + wrap_float!( + @wrap_autodiff + $kind, + $Backend, + { $body } + ) + } + $crate::DispatchTensor::Autodiff(_) => panic!("Autodiff should not wrap an autodiff tensor."), + _ => panic!("The provided tensors are not on the same backend.") + } + }, + + #[cfg($cfg)] + ($crate::DispatchTensor::$Backend($lhs_inner), $crate::DispatchTensor::$Backend($rhs_inner)) => { + type B = $Backend; + let $lhs_inner = $lhs_inner.float(); + let $rhs_inner = $rhs_inner.$rhs_kind(); + $crate::DispatchTensor::$Backend($crate::BackendTensor::$kind($body)) + } + )* + #[allow(unreachable_patterns)] + (lhs, rhs) => { + panic!( + "The provided tensors are not on the same backend. Got backends {:?} and {:?}.", lhs, rhs + ); + } + } + }}; + ( + $kind:ident, + ($lhs:expr, $lhs_kind:ident), + ($rhs:expr, $rhs_kind:ident), + |$lhs_inner:ident, $rhs_inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match ($lhs, $rhs) { + $( + #[cfg($cfg)] + ($crate::DispatchTensor::$Backend($lhs_inner), $crate::DispatchTensor::$Backend($rhs_inner)) => { + type B = $Backend; + let $lhs_inner = $lhs_inner.$lhs_kind(); + let $rhs_inner = $rhs_inner.$rhs_kind(); + $crate::DispatchTensor::$Backend($crate::BackendTensor::$kind($body)) + } + )* + (lhs, rhs) => { + panic!( + "The provided tensors are not on the same backend. Got backends {:?} and {:?}.", lhs, rhs + ); + } + } + }}; + // Autodiff (lhs, rhs) tensors + ( + @autodiff + $kind:ident, + ($lhs:expr, $lhs_kind:ident), + ($rhs:expr, $rhs_kind:ident), + |$lhs_inner:ident, $rhs_inner:ident| $body:expr; + $([$Backend:ident, $cfg:meta]),* + ) => {{ + match ($lhs, $rhs) { + $( + #[cfg($cfg)] + ($crate::DispatchTensor::$Backend($lhs_inner), $crate::DispatchTensor::$Backend($rhs_inner)) => { + type B = Autodiff<$Backend>; + let $lhs_inner = $lhs_inner.$lhs_kind(); + let $rhs_inner = $rhs_inner.$rhs_kind(); + wrap_float!( + @wrap_autodiff + $kind, + $Backend, + { $body } + ) + } + )* + #[cfg(feature = "autodiff")] + ($crate::DispatchTensor::Autodiff(_), _) | (_, $crate::DispatchTensor::Autodiff(_)) => panic!("Autodiff should not wrap an autodiff tensor."), + (lhs, rhs) => { + panic!( + "The provided tensors are not on the same backend. Got backends {:?} and {:?}.", lhs, rhs + ); + } + } + }}; + +} + +/// Backend dispatch for binary operations. +/// Automatically verifies that both tensors reside on the same backend. +macro_rules! binary_float { + (($lhs:expr, $lhs_kind:ident), ($rhs:expr, $rhs_kind:ident), |$lhs_inner:ident, $rhs_inner:ident| $body:expr => $kind:ident) => { + backend_list!( + binary_float_arms, + $kind, + ($lhs, $lhs_kind), + ($rhs, $rhs_kind), + |$lhs_inner, $rhs_inner| { $body } + ) + }; +} + +/// The core logic for a single backend in a `multi_op`. +/// Handles the manual unwrapping of required/optional inputs and the +/// re-wrapping of multiple required/optional output tensors. +macro_rules! multi_op_arm { + ( + $Backend:ident, + [ $( ($x:ident, $x_kind:ident) ),+ ], + [ $( ($opt_in:ident, $opt_kind:ident) ),* ], + [ $( ($out:ident, $out_kind:ident) ),+ ], + [ $( $opt_out:ident ),* ], + $body:expr + ) => {{ + type B = $Backend; + + // Required inputs + $( + let $x = match $x { + $crate::DispatchTensor::$Backend(inner) => inner.$x_kind(), + #[allow(unreachable_patterns)] + _ => panic!("Input tensor {} is on the wrong device", stringify!($x)), + }; + )+ + + // Optional inputs + $( + let $opt_in = $opt_in.map(|o| match o { + $crate::DispatchTensor::$Backend(inner) => inner.$opt_kind(), + #[allow(unreachable_patterns)] + _ => panic!("Optional tensor {} is on the wrong device", stringify!($opt_in)), + }); + )* + + let ($($out),+, $($opt_out),*) = $body; + + // Outputs and optional outputs + ( + $( $crate::DispatchTensor::$Backend($crate::BackendTensor::$out_kind($out)) ),+, + $( $opt_out.map(|t| $crate::DispatchTensor::$Backend($crate::BackendTensor::Float(t))) ),* + ) + }}; +} + +#[cfg(feature = "autodiff")] +macro_rules! wrap_input_autodiff { + ($Backend:ident, $inner:expr, int) => { + $inner.int() + }; + ($Backend:ident, $inner:expr, bool) => { + $inner.bool() + }; + // Float tensors: wrap with autodiff + ($Backend:ident, $inner:expr, float) => { + $inner.autodiff() + }; +} + +#[cfg(feature = "autodiff")] +// DispatchTensor::Autodiff(DispatchTensor::$Backend(BackendTensor::Autodiff())) +macro_rules! multi_op_arm_autodiff { + ( + $Backend:ident, + [ $( ($x:ident, $x_kind:ident) ),+ ], + [ $( ($opt_in:ident, $opt_kind:ident) ),* ], + [ $( ($out:ident, $out_kind:ident) ),+ ], + [ $( $opt_out:ident ),* ], + $body:expr + ) => {{ + type B = Autodiff<$Backend>; + + // Required inputs + $( + let $x = match $x { + $crate::DispatchTensor::Autodiff(inner) => { + match *inner { + $crate::DispatchTensor::$Backend(inner) => wrap_input_autodiff!($Backend, inner, $x_kind), + _ => panic!("Input tensor {} is on the wrong device", stringify!($x)), + } + }, + // Unreachable, except when input is int + $crate::DispatchTensor::$Backend(inner) => wrap_input_autodiff!($Backend, inner, $x_kind), + _ => panic!("Input tensor {} is on the wrong device", stringify!($x)), + }; + )+ + + // Optional inputs (always assumed to be float / autodiff) + $( + let $opt_in = $opt_in.map(|o| match o { + $crate::DispatchTensor::Autodiff(inner) => { + match *inner { + $crate::DispatchTensor::$Backend(inner) => wrap_input_autodiff!($Backend, inner, $opt_kind), + _ => panic!("Input tensor {} is on the wrong device", stringify!($opt_in)), + } + }, + _ => panic!("Optional tensor {} is on the wrong device", stringify!($opt_in)), + }); + )* + + let ($($out),+, $($opt_out),*) = $body; + + // Outputs and optional outputs + ( + $( wrap_float!(@wrap_autodiff $out_kind, $Backend, $out) ),+, + $( $opt_out.map(|t| wrap_float!(@wrap_autodiff Float, $Backend, t)) ),* + ) + }}; +} + +/// Helper to extract the first identifier from an input list. +/// Used to determine the device/backend for dispatching multi-tensor operations. +macro_rules! first_input { + ([ ($x:ident, $kind:ident) $(, $rest:tt)* ]) => { + $x + }; +} + +/// Match arm generator for `multi_op`. +/// Determines the backend based on the first input and delegates to `multi_op_arm` +/// to handle the repetition-heavy unwrapping and wrapping logic. +macro_rules! multi_op_arms_autodiff { + ( + $inputs:tt, + $opt_inputs:tt, + $outputs:tt, + $opt_outputs:tt, + $body:expr; + $( [$Backend:ident, $cfg:meta] ),* + ) => {{ + match &first_input!($inputs) { + // Autodiff first + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(inner) => { + match **inner { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + multi_op_arm_autodiff!( + $Backend, + $inputs, + $opt_inputs, + $outputs, + $opt_outputs, + $body + ) + } + )* + $crate::DispatchTensor::Autodiff(_) => panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + multi_op_arm!( + $Backend, + $inputs, + $opt_inputs, + $outputs, + $opt_outputs, + $body + ) + } + )* + } + }}; +} + +/// Match arm generator for `multi_op`. +/// +/// Similar to `multi_op_arms`, but skips autodiff checks. +macro_rules! multi_op_arms { + ( + $inputs:tt, + $opt_inputs:tt, + $outputs:tt, + $opt_outputs:tt, + $body:expr; + $( [$Backend:ident, $cfg:meta] ),* + ) => {{ + match &first_input!($inputs) { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + multi_op_arm!( + $Backend, + $inputs, + $opt_inputs, + $outputs, + $opt_outputs, + $body + ) + } + )* + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(_) => panic!("Operation not marked for autodiff.") + } + }}; +} + +/// High-level macro for complex module operations (e.g., conv2d) and multi-tensor operations. +/// Handles variable numbers of required/optional inputs and wraps multiple outputs. +/// +/// Usage: +/// ```ignore +/// multi_op!( +/// inputs[(x, float), (weight, float)], +/// opt_inputs[(bias, float)], +/// => Float, +/// B::conv2d(x, weight, bias, options) +/// ) +/// ``` +macro_rules! multi_op { + // --- Single output shorthands --- + // Automatically wraps body in tuple and extracts .0 + ( + inputs[$( ($x:ident, $kind:ident) ),+], + => Float, + $body:expr + ) => { + multi_op!( + inputs[$( ($x, $kind) ),+], + opt_inputs[], + outputs[(out, Float)], + opt_outputs[], + { ($body,) } + ) + .0 + }; + ( + inputs[$( ($x:ident, $kind:ident) ),+], + opt_inputs[ $(($opt_in:ident, $opt_kind:ident)),* ], + => $out_kind:ident, + $body:expr + ) => { + multi_op!( + inputs[$( ($x, $kind) ),+], + opt_inputs[ $(($opt_in, $opt_kind)),* ], + outputs[(out, $out_kind)], + opt_outputs[], + { ($body,) } + ) + .0 + }; + // Int/Bool op specialization (not marked for autodiff) + ( + inputs[$( ($x:ident, $kind:ident) ),+], + => $out_kind:ident, + $body:expr + ) => { + backend_list!( + multi_op_arms, + [ $(($x, $kind)),+ ], + [], + [ (out, $out_kind) ], + [], + { ($body,) } + ).0 + }; + + // --- Required + optional for both inputs and outputs --- + ( + inputs[ $(($x:ident, $kind:ident)),+ ], + opt_inputs[ $(($opt_in:ident, $opt_kind:ident)),* ], + outputs[ $( ($out:ident, $out_kind:ident) ),+ ], + opt_outputs[ $($opt_out:ident),* ], + $body:expr + ) => { + backend_list!( + multi_op_arms_autodiff, + [ $(($x, $kind)),+ ], + [ $(($opt_in, $opt_kind)),* ], + [ $(($out, $out_kind)),+ ], + [ $($opt_out),* ], + $body + ) + }; + + ( + inputs[ $(($x:ident, $kind:ident)),+ ], + opt_inputs[ $(($opt_in:ident, $opt_kind:ident)),* ], + outputs[ $($out:ident),+ ], + $body:expr + ) => { + multi_op!( + inputs[ $(($x, $kind)),+ ], + opt_inputs[ $(($opt_in, $opt_kind)),* ], + outputs[ $(($out, Float)),+ ], + opt_outputs[], + $body + ) + }; + + ( + inputs[ $(($x:ident, $kind:ident)),+ ], + outputs[ $( ($out:ident, $out_kind:ident) ),+ ], + $body:expr + ) => { + multi_op!( + inputs[ $(($x, $kind)),+ ], + opt_inputs[], + outputs[ $(($out, $out_kind)),+ ], + opt_outputs[], + $body + ) + }; +} + +/// Unwraps a `Vec` for a known backend. +macro_rules! unwrap_vec { + ($Backend:ident, $vec:expr, $kind:ident) => { + $vec.into_iter() + .map(|t| match t { + $crate::DispatchTensor::$Backend(inner) => inner.$kind(), + #[allow(unreachable_patterns)] + _ => panic!( + "Tensor is on the wrong backend (expected {}).", + stringify!($Backend) + ), + }) + .collect::>() + }; + + // Autodiff-wrapped backend + (@autodiff $Backend:ident, $vec:expr, $kind:ident) => { + $vec.into_iter() + .map(|t| match t { + $crate::DispatchTensor::Autodiff(inner) => match *inner { + $crate::DispatchTensor::$Backend(inner) => inner.$kind(), + _ => panic!( + "Autodiff float tensor is on the wrong backend (expected {}).", + stringify!($Backend) + ), + }, + _ => panic!( + "Expected autodiff-wrapped float tensor for backend {}.", + stringify!($Backend) + ), + }) + .collect::>() + }; +} + +/// Match arm generator for `vec_op`. +macro_rules! vec_op_arms { + (Float, $inner_kind:ident, $tensors:expr, |$inner:ident| $body:expr; $([$Backend:ident, $cfg:meta]),*) => { + match &$tensors[0] { + // Autodiff arm first + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(inner) => { + // Recursively dispatch on inner + match **inner { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + type B = Autodiff<$Backend>; + + let $inner = unwrap_vec!(@autodiff $Backend, $tensors, autodiff); + wrap_float!( @wrap_autodiff Float, $Backend, { $body } ) + } + )* + $crate::DispatchTensor::Autodiff(_) => panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + type B = $Backend; + + let $inner = unwrap_vec!($Backend, $tensors, $inner_kind); + $crate::DispatchTensor::$Backend($crate::BackendTensor::Float($body)) + } + )* + } + }; + ($kind:ident, $inner_kind:ident, $tensors:expr, |$inner:ident| $body:expr; $([$Backend:ident, $cfg:meta]),*) => { + match &$tensors[0] { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + type B = $Backend; + + let $inner = unwrap_vec!($Backend, $tensors, $inner_kind); + $crate::DispatchTensor::$Backend($crate::BackendTensor::$kind($body)) + } + )* + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(_) => panic!("Operation not marked for autodiff.") + } + }; +} + +/// Backend dispatch for operations on multiple inputs (vec). +/// Automatically verifies that tensors reside on the first backend. +macro_rules! vec_op { + ($tensors:expr, $inner_kind:ident, |$inner:ident| $body:expr => $kind:ident) => { + backend_list!(vec_op_arms, $kind, $inner_kind, $tensors, |$inner| { + $body + }) + }; +} + +/// Match arm generator for `transaction_op`. +macro_rules! transaction_op_arms { + ($tx:ident, $first:expr; $([$Backend:ident, $cfg:meta]),*) => { + match $first { + // Autodiff arm first + #[cfg(feature = "autodiff")] + $crate::DispatchTensor::Autodiff(inner) => { + // Recursively dispatch on inner + match **inner { + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + type B = $Backend; + + // Unwrap vec + let floats = unwrap_vec!(@autodiff $Backend, $tx.read_floats, autodiff_inner); + let ints = unwrap_vec!($Backend, $tx.read_ints, int); + let bools = unwrap_vec!($Backend, $tx.read_bools, bool); + // Not supported + let qfloats = $tx.read_qfloats.into_iter().map(|_t| todo!("Quantization not supported yet")).collect(); + + B::tr_execute(TransactionPrimitive::new(floats, qfloats, ints, bools)).await + } + )* + $crate::DispatchTensor::Autodiff(_) => panic!("Autodiff should not wrap an autodiff tensor.") + } + }, + + $( + #[cfg($cfg)] + $crate::DispatchTensor::$Backend(_) => { + type B = $Backend; + + // Unwrap vec + let floats = unwrap_vec!($Backend, $tx.read_floats, float); + let ints = unwrap_vec!($Backend, $tx.read_ints, int); + let bools = unwrap_vec!($Backend, $tx.read_bools, bool); + // Not supported + let qfloats = $tx.read_qfloats.into_iter().map(|_t| todo!("Quantization not supported yet")).collect(); + + B::tr_execute(TransactionPrimitive::new(floats, qfloats, ints, bools)).await + } + )* + } + }; +} + +/// Helper to dispatch a transaction based on the first available tensor. +macro_rules! transaction_op { + ($tx:ident, $first:expr) => { + backend_list!(transaction_op_arms, $tx, $first) + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/activation.rs new file mode 100644 index 0000000..6f58039 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/activation.rs @@ -0,0 +1,50 @@ +use burn_backend::{Scalar, ops::ActivationOps, tensor::FloatTensor}; + +use crate::Dispatch; +use crate::backends::*; + +impl ActivationOps for Dispatch { + fn leaky_relu(tensor: FloatTensor, negative_slope: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::leaky_relu(tensor, negative_slope) => Float) + } + + fn relu(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::relu(tensor) => Float) + } + + fn relu_backward(output: FloatTensor, grad: FloatTensor) -> FloatTensor { + binary_float!((output, float), (grad, float), |output, grad| B::relu_backward(output, grad) => Float) + } + + fn gelu(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::gelu(tensor) => Float) + } + + fn prelu(tensor: FloatTensor, alpha: FloatTensor) -> FloatTensor { + binary_float!((tensor, float), (alpha, float), |tensor, alpha| B::prelu(tensor, alpha) => Float) + } + + fn gelu_backward(x: FloatTensor, grad: FloatTensor) -> FloatTensor { + binary_float!((x, float), (grad, float), |x, grad| B::gelu_backward(x, grad) => Float) + } + + fn sigmoid(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::sigmoid(tensor) => Float) + } + + fn sigmoid_backward(output: FloatTensor, grad: FloatTensor) -> FloatTensor { + binary_float!((output, float), (grad, float), |output, grad| B::sigmoid_backward(output, grad) => Float) + } + + fn hard_sigmoid(tensor: FloatTensor, alpha: Scalar, beta: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::hard_sigmoid(tensor, alpha, beta) => Float) + } + + fn log_sigmoid(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::log_sigmoid(tensor) => Float) + } + + fn log_sigmoid_backward(x: FloatTensor, grad: FloatTensor) -> FloatTensor { + binary_float!((x, float), (grad, float), |x, grad| B::log_sigmoid_backward(x, grad) => Float) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/bool_tensor.rs new file mode 100644 index 0000000..1831e04 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/bool_tensor.rs @@ -0,0 +1,222 @@ +use burn_backend::{ + ExecutionError, Scalar, TensorData, + ops::BoolTensorOps, + tensor::{BoolTensor, FloatTensor, IntTensor}, +}; +use burn_std::{Shape, Slice}; + +use crate::backends::*; +use crate::{Dispatch, DispatchDevice}; + +impl BoolTensorOps for Dispatch { + fn bool_empty(shape: Shape, device: &DispatchDevice) -> BoolTensor { + creation_op!(Bool, device, |device| B::bool_empty(shape, device)) + } + + fn bool_zeros(shape: Shape, device: &DispatchDevice) -> BoolTensor { + creation_op!(Bool, device, |device| B::bool_zeros(shape, device)) + } + + fn bool_ones(shape: Shape, device: &DispatchDevice) -> BoolTensor { + creation_op!(Bool, device, |device| B::bool_ones(shape, device)) + } + + async fn bool_into_data(tensor: BoolTensor) -> Result { + unary_op!(tensor, bool, |tensor| B::bool_into_data(tensor).await) + } + + fn bool_from_data(data: TensorData, device: &DispatchDevice) -> BoolTensor { + creation_op!(Bool, device, |device| B::bool_from_data(data, device)) + } + + fn bool_into_int(tensor: BoolTensor) -> IntTensor { + unary_op!(tensor, bool, |tensor| B::bool_into_int(tensor) => Int) + } + + fn bool_into_float(tensor: BoolTensor) -> FloatTensor { + unary_op!(tensor, bool, |tensor| B::bool_into_float(tensor) => Float) + } + + fn bool_device(tensor: &BoolTensor) -> DispatchDevice { + tensor.device() + } + + fn bool_to_device(tensor: BoolTensor, device: &DispatchDevice) -> BoolTensor { + to_device!( + Bool, + bool, + tensor, + device, + bool_to_device, + |inner, device| { + let data = + burn_backend::read_sync(B1::bool_into_data(inner)).expect("Should read data"); + B2::bool_from_data(data, device) + } + ) + } + + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_reshape(tensor, shape) => Bool) + } + + fn bool_slice(tensor: BoolTensor, slices: &[Slice]) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_slice(tensor, slices) => Bool) + } + + fn bool_slice_assign( + tensor: BoolTensor, + slices: &[Slice], + value: BoolTensor, + ) -> BoolTensor { + binary_op!((tensor, bool), (value, bool), |tensor, value| B::bool_slice_assign(tensor, slices, value) => Bool) + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + multi_op!( + inputs[(tensor, bool), (mask, bool), (value, bool)], => Bool, + B::bool_mask_where(tensor, mask, value) + ) + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + binary_op!((tensor, bool), (mask, bool), |tensor, mask| B::bool_mask_fill(tensor, mask, value) => Bool) + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + binary_op!((tensor, bool), (indices, int), |tensor, indices| B::bool_gather(dim, tensor, indices) => Bool) + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + multi_op!( + inputs[(tensor, bool), (indices, int), (value, bool)], => Bool, + B::bool_scatter_or(dim, tensor, indices, value) + ) + } + + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + binary_op!((lhs, bool), (rhs, bool), |lhs, rhs| B::bool_equal(lhs, rhs) => Bool) + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, bool, |lhs| B::bool_equal_elem(lhs, rhs) => Bool) + } + + fn bool_not(tensor: BoolTensor) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_not(tensor) => Bool) + } + + fn bool_and(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + binary_op!((lhs, bool), (rhs, bool), |lhs, rhs| B::bool_and(lhs, rhs) => Bool) + } + + fn bool_or(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + binary_op!((lhs, bool), (rhs, bool), |lhs, rhs| B::bool_or(lhs, rhs) => Bool) + } + + fn bool_swap_dims(tensor: BoolTensor, dim1: usize, dim2: usize) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_swap_dims(tensor, dim1, dim2) => Bool) + } + + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_permute(tensor, axes) => Bool) + } + + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_flip(tensor, axes) => Bool) + } + + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_expand(tensor, shape) => Bool) + } + + fn bool_unfold( + tensor: BoolTensor, + dim: usize, + size: usize, + step: usize, + ) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_unfold(tensor, dim, size, step) => Bool) + } + + fn bool_select( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + ) -> BoolTensor { + binary_op!((tensor, bool), (indices, int), |tensor, indices| B::bool_select(tensor, dim, indices) => Bool) + } + + fn bool_select_or( + tensor: BoolTensor, + dim: usize, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + multi_op!( + inputs[(tensor, bool), (indices, int), (value, bool)], => Bool, + B::bool_select_or(tensor, dim, indices, value) + ) + } + + fn bool_repeat_dim(tensor: BoolTensor, dim: usize, times: usize) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_repeat_dim(tensor, dim, times) => Bool) + } + + fn bool_cat(tensors: Vec>, dim: usize) -> BoolTensor { + vec_op!(tensors, bool, |tensors| B::bool_cat(tensors, dim) => Bool) + } + + fn bool_not_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + binary_op!((lhs, bool), (rhs, bool), |lhs, rhs| B::bool_not_equal(lhs, rhs) => Bool) + } + + fn bool_not_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, bool, |lhs| B::bool_not_equal_elem(lhs, rhs) => Bool) + } + + fn bool_xor(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + binary_op!((lhs, bool), (rhs, bool), |lhs, rhs| B::bool_xor(lhs, rhs) => Bool) + } + + fn bool_transpose(tensor: BoolTensor) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_transpose(tensor) => Bool) + } + + fn bool_any(tensor: BoolTensor) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_any(tensor) => Bool) + } + + fn bool_any_dim(tensor: BoolTensor, dim: usize) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_any_dim(tensor, dim) => Bool) + } + + fn bool_all(tensor: BoolTensor) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_all(tensor) => Bool) + } + + fn bool_all_dim(tensor: BoolTensor, dim: usize) -> BoolTensor { + unary_op!(tensor, bool, |tensor| B::bool_all_dim(tensor, dim) => Bool) + } + + async fn bool_argwhere(tensor: BoolTensor) -> IntTensor { + unary_op!(tensor, bool, |tensor| B::bool_argwhere(tensor).await => Int) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/int_tensor.rs new file mode 100644 index 0000000..d4d648a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/int_tensor.rs @@ -0,0 +1,503 @@ +use burn_backend::{ + ExecutionError, Scalar, TensorData, + ops::IntTensorOps, + tensor::{BoolTensor, FloatTensor, IntTensor}, +}; +use burn_std::{IntDType, Shape, Slice}; + +use crate::backends::*; +use crate::{Dispatch, DispatchDevice}; + +impl IntTensorOps for Dispatch { + fn int_empty(shape: Shape, device: &DispatchDevice, dtype: IntDType) -> IntTensor { + creation_op!(Int, device, |device| B::int_empty(shape, device, dtype)) + } + + async fn int_into_data(tensor: IntTensor) -> Result { + unary_op!(tensor, int, |tensor| B::int_into_data(tensor).await) + } + + fn int_from_data(data: TensorData, device: &DispatchDevice) -> IntTensor { + creation_op!(Int, device, |device| B::int_from_data(data, device)) + } + + fn int_device(tensor: &IntTensor) -> DispatchDevice { + tensor.device() + } + + fn int_to_device(tensor: IntTensor, device: &DispatchDevice) -> IntTensor { + to_device!(Int, int, tensor, device, int_to_device, |inner, device| { + let data = burn_backend::read_sync(B1::int_into_data(inner)).expect("Should read data"); + B2::int_from_data(data, device) + }) + } + + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_reshape(tensor, shape) => Int) + } + + fn int_slice(tensor: IntTensor, slices: &[Slice]) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_slice(tensor, slices) => Int) + } + + fn int_slice_assign( + tensor: IntTensor, + slices: &[Slice], + value: IntTensor, + ) -> IntTensor { + binary_op!((tensor, int), (value, int), |tensor, value| B::int_slice_assign(tensor, slices, value) => Int) + } + + fn int_into_float(tensor: IntTensor) -> FloatTensor { + unary_op!(tensor, int, |tensor| B::int_into_float(tensor) => Float) + } + + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + value: IntTensor, + ) -> IntTensor { + multi_op!( + inputs[(tensor, int), (mask, bool), (value, int)], => Int, + B::int_mask_where(tensor, mask, value) + ) + } + + fn int_mask_fill( + tensor: IntTensor, + mask: BoolTensor, + value: Scalar, + ) -> IntTensor { + binary_op!((tensor, int), (mask, bool), |tensor, mask| B::int_mask_fill(tensor, mask, value) => Int) + } + + fn int_gather( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + ) -> IntTensor { + binary_op!((tensor, int), (indices, int), |tensor, indices| B::int_gather(dim, tensor, indices) => Int) + } + + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + multi_op!( + inputs[(tensor, int), (indices, int), (value, int)], => Int, + B::int_scatter_add(dim, tensor, indices, value) + ) + } + + fn int_select( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + ) -> IntTensor { + binary_op!((tensor, int), (indices, int), |tensor, indices| B::int_select(tensor, dim, indices) => Int) + } + + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + multi_op!( + inputs[(tensor, int), (indices, int), (value, int)], => Int, + B::int_select_add(tensor, dim, indices, value) + ) + } + + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_equal(lhs, rhs) => Bool) + } + + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, int, |lhs| B::int_equal_elem(lhs, rhs) => Bool) + } + + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_greater(lhs, rhs) => Bool) + } + + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, int, |lhs| B::int_greater_elem(lhs, rhs) => Bool) + } + + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_greater_equal(lhs, rhs) => Bool) + } + + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, int, |lhs| B::int_greater_equal_elem(lhs, rhs) => Bool) + } + + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_lower(lhs, rhs) => Bool) + } + + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, int, |lhs| B::int_lower_elem(lhs, rhs) => Bool) + } + + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_lower_equal(lhs, rhs) => Bool) + } + + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, int, |lhs| B::int_lower_equal_elem(lhs, rhs) => Bool) + } + + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_add(lhs, rhs) => Int) + } + + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_add_scalar(lhs, rhs) => Int) + } + + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_sub(lhs, rhs) => Int) + } + + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_sub_scalar(lhs, rhs) => Int) + } + + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_mul(lhs, rhs) => Int) + } + + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_mul_scalar(lhs, rhs) => Int) + } + + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_div(lhs, rhs) => Int) + } + + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_div_scalar(lhs, rhs) => Int) + } + + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_remainder(lhs, rhs) => Int) + } + + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_remainder_scalar(lhs, rhs) => Int) + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_matmul(lhs, rhs) => Int) + } + + fn int_sum(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_sum(tensor) => Int) + } + + fn int_sum_dim(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_sum_dim(tensor, dim) => Int) + } + + fn int_prod(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_prod(tensor) => Int) + } + + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_prod_dim(tensor, dim) => Int) + } + + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_mean_dim(tensor, dim) => Int) + } + + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_cumsum(tensor, dim) => Int) + } + + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_cumprod(tensor, dim) => Int) + } + + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_cummin(tensor, dim) => Int) + } + + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_cummax(tensor, dim) => Int) + } + + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_argmax(tensor, dim) => Int) + } + + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_argmin(tensor, dim) => Int) + } + + fn int_abs(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_abs(tensor) => Int) + } + + fn int_swap_dims(tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_swap_dims(tensor, dim1, dim2) => Int) + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_permute(tensor, axes) => Int) + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_flip(tensor, axes) => Int) + } + + fn int_random( + shape: Shape, + distribution: burn_backend::Distribution, + device: &DispatchDevice, + ) -> IntTensor { + creation_op!(Int, device, |device| { + B::int_random(shape, distribution, device) + }) + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_expand(tensor, shape) => Int) + } + + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::bitwise_and(lhs, rhs) => Int) + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::bitwise_and_scalar(lhs, rhs) => Int) + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::bitwise_or(lhs, rhs) => Int) + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::bitwise_or_scalar(lhs, rhs) => Int) + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::bitwise_xor(lhs, rhs) => Int) + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::bitwise_xor_scalar(lhs, rhs) => Int) + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::bitwise_not(tensor) => Int) + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::bitwise_left_shift(lhs, rhs) => Int) + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::bitwise_left_shift_scalar(lhs, rhs) => Int) + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::bitwise_right_shift(lhs, rhs) => Int) + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::bitwise_right_shift_scalar(lhs, rhs) => Int) + } + + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_cast(tensor, dtype) => Int) + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_unfold(tensor, dim, size, step) => Int) + } + + fn int_repeat_dim(tensor: IntTensor, dim: usize, times: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_repeat_dim(tensor, dim, times) => Int) + } + + fn int_cat(tensors: Vec>, dim: usize) -> IntTensor { + vec_op!(tensors, int, |tensors| B::int_cat(tensors, dim) => Int) + } + + fn int_not_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_not_equal(lhs, rhs) => Bool) + } + + fn int_not_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + unary_op!(lhs, int, |lhs| B::int_not_equal_elem(lhs, rhs) => Bool) + } + + fn int_powi(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_powi(lhs, rhs) => Int) + } + + fn int_powf(lhs: IntTensor, rhs: FloatTensor) -> IntTensor { + binary_op!((lhs, int), (rhs, int), |lhs, rhs| B::int_powf(lhs, rhs) => Int) + } + + fn int_powi_scalar_impl(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_powi_scalar_impl(lhs, rhs) => Int) + } + + fn int_powf_scalar_impl(lhs: IntTensor, rhs: Scalar) -> IntTensor { + unary_op!(lhs, int, |lhs| B::int_powf_scalar_impl(lhs, rhs) => Int) + } + + fn int_clamp_min(tensor: IntTensor, min: Scalar) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_clamp_min(tensor, min) => Int) + } + + fn int_clamp_max(tensor: IntTensor, max: Scalar) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_clamp_max(tensor, max) => Int) + } + + fn int_clamp(tensor: IntTensor, min: Scalar, max: Scalar) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_clamp(tensor, min, max) => Int) + } + + fn int_neg(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_neg(tensor) => Int) + } + + fn int_zeros(shape: Shape, device: &DispatchDevice, dtype: IntDType) -> IntTensor { + creation_op!(Int, device, |device| B::int_zeros(shape, device, dtype)) + } + + fn int_ones(shape: Shape, device: &DispatchDevice, dtype: IntDType) -> IntTensor { + creation_op!(Int, device, |device| B::int_ones(shape, device, dtype)) + } + + fn int_full( + shape: Shape, + fill_value: Scalar, + device: &DispatchDevice, + dtype: IntDType, + ) -> IntTensor { + creation_op!(Int, device, |device| B::int_full( + shape, fill_value, device, dtype + )) + } + + fn int_mean(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_mean(tensor) => Int) + } + + fn int_max(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_max(tensor) => Int) + } + + fn int_max_dim(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_max_dim(tensor, dim) => Int) + } + + fn int_max_dim_with_indices( + tensor: IntTensor, + dim: usize, + ) -> (IntTensor, IntTensor) { + multi_op!( + inputs[(tensor, int)], + outputs[(out, Int), (indices, Int)], + B::int_max_dim_with_indices(tensor, dim) + ) + } + + fn int_max_abs(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_max_abs(tensor) => Int) + } + + fn int_max_abs_dim(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_max_abs_dim(tensor, dim) => Int) + } + + fn int_min(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_min(tensor) => Int) + } + + fn int_min_dim(tensor: IntTensor, dim: usize) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_min_dim(tensor, dim) => Int) + } + + fn int_min_dim_with_indices( + tensor: IntTensor, + dim: usize, + ) -> (IntTensor, IntTensor) { + multi_op!( + inputs[(tensor, int)], + outputs[(out, Int), (indices, Int)], + B::int_min_dim_with_indices(tensor, dim) + ) + } + + fn int_transpose(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_transpose(tensor) => Int) + } + + fn int_arange_step( + range: std::ops::Range, + step: usize, + device: &DispatchDevice, + ) -> IntTensor { + creation_op!(Int, device, |device| B::int_arange_step( + range, step, device + )) + } + + fn int_arange(range: std::ops::Range, device: &DispatchDevice) -> IntTensor { + creation_op!(Int, device, |device| B::int_arange(range, device)) + } + + fn int_any(tensor: IntTensor) -> BoolTensor { + unary_op!(tensor, int, |tensor| B::int_any(tensor) => Bool) + } + + fn int_any_dim(tensor: IntTensor, dim: usize) -> BoolTensor { + unary_op!(tensor, int, |tensor| B::int_any_dim(tensor, dim) => Bool) + } + + fn int_all(tensor: IntTensor) -> BoolTensor { + unary_op!(tensor, int, |tensor| B::int_all(tensor) => Bool) + } + + fn int_all_dim(tensor: IntTensor, dim: usize) -> BoolTensor { + unary_op!(tensor, int, |tensor| B::int_all_dim(tensor, dim) => Bool) + } + + fn int_sign(tensor: IntTensor) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_sign(tensor) => Int) + } + + fn int_sort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_sort(tensor, dim, descending) => Int) + } + + fn int_sort_with_indices( + tensor: IntTensor, + dim: usize, + descending: bool, + ) -> (IntTensor, IntTensor) { + multi_op!( + inputs[(tensor, int)], + outputs[(out, Int), (indices, Int)], + B::int_sort_with_indices(tensor, dim, descending) + ) + } + + fn int_argsort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + unary_op!(tensor, int, |tensor| B::int_argsort(tensor, dim, descending) => Int) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/mod.rs new file mode 100644 index 0000000..df092af --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/mod.rs @@ -0,0 +1,7 @@ +mod activation; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/module.rs new file mode 100644 index 0000000..1992e50 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/module.rs @@ -0,0 +1,628 @@ +use burn_backend::{ + ops::{ + DeformConv2dBackward, MaxPool1dBackward, MaxPool1dWithIndices, MaxPool2dBackward, + MaxPool2dWithIndices, ModuleOps, + }, + tensor::{FloatTensor, IntTensor}, +}; + +use crate::Dispatch; +use crate::backends::*; + +impl ModuleOps for Dispatch { + fn conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: burn_backend::ops::ConvOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float)], + opt_inputs[(bias, float)], + => Float, + B::conv2d(x, weight, bias, options) + ) + } + + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: burn_backend::ops::DeformConvOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (offset, float), (weight, float)], + opt_inputs[(mask, float), (bias, float)], + => Float, + B::deform_conv2d(x, offset, weight, mask, bias, options) + ) + } + + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: burn_backend::ops::DeformConvOptions<2>, + ) -> DeformConv2dBackward { + let (x_grad, offset_grad, weight_grad, mask_grad, bias_grad) = multi_op!( + inputs[(x, float), (offset, float), (weight, float), (output_grad, float)], + opt_inputs[(mask, float), (bias, float)], + outputs[(x_grad, Float), (offset_grad, Float), (weight_grad, Float)], + opt_outputs[mask_grad, bias_grad], + { + let res = B::deform_conv2d_backward(x, offset, weight, mask, bias, output_grad, options); + (res.x_grad, res.offset_grad, res.weight_grad, res.mask_grad, res.bias_grad) + } + ); + DeformConv2dBackward::new(x_grad, offset_grad, weight_grad, mask_grad, bias_grad) + } + + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: burn_backend::ops::ConvOptions<3>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float)], + opt_inputs[(bias, float)], + => Float, + B::conv3d(x, weight, bias, options) + ) + } + + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: burn_backend::ops::ConvTransposeOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float)], + opt_inputs[(bias, float)], + => Float, + B::conv_transpose2d(x, weight, bias, options) + ) + } + + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: burn_backend::ops::ConvTransposeOptions<3>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float)], + opt_inputs[(bias, float)], + => Float, + B::conv_transpose3d(x, weight, bias, options) + ) + } + + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + multi_op!(inputs[(x, float)], + => Float, + B::avg_pool2d(x, kernel_size, stride, padding, count_include_pad, ceil_mode) + ) + } + + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (grad, float)], + => Float, + B::avg_pool2d_backward(x, grad, kernel_size, stride, padding, count_include_pad, ceil_mode) + ) + } + + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor { + multi_op!( + inputs[(x, float)], + => Float, + B::adaptive_avg_pool2d(x, output_size) + ) + } + + fn adaptive_avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (grad, float)], + => Float, + B::adaptive_avg_pool2d_backward(x, grad) + ) + } + + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor { + multi_op!( + inputs[(x, float)], + => Float, + B::max_pool2d(x, kernel_size, stride, padding, dilation, ceil_mode) + ) + } + + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices { + let (out, indices) = multi_op!( + inputs[(x, float)], + outputs[(out, Float), (indices, Int)], + { + let res = B::max_pool2d_with_indices(x, kernel_size, stride, padding, dilation, ceil_mode); + (res.output, res.indices) + } + ); + MaxPool2dWithIndices::new(out, indices) + } + + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool2dBackward { + let x_grad = multi_op!( + inputs[(x, float), (output_grad, float), (indices, int)], + => Float, + { + let res = B::max_pool2d_with_indices_backward(x, kernel_size, stride, padding, dilation, ceil_mode, output_grad, indices); + res.x_grad + } + ); + MaxPool2dBackward::new(x_grad) + } + + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: burn_backend::ops::InterpolateOptions, + ) -> FloatTensor { + multi_op!( + inputs[(x, float)], + => Float, + B::interpolate(x, output_size, options) + ) + } + + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: burn_backend::ops::InterpolateOptions, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (grad, float)], + => Float, + B::interpolate_backward(x, grad, output_size, options) + ) + } + + fn embedding(weights: FloatTensor, indices: IntTensor) -> FloatTensor { + multi_op!( + inputs[(weights, float), (indices, int)], + => Float, + B::embedding(weights, indices) + ) + } + + fn embedding_backward( + weights: FloatTensor, + output_grad: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + multi_op!( + inputs[(weights, float), (output_grad, float), (indices, int)], + => Float, + B::embedding_backward(weights, output_grad, indices) + ) + } + + fn conv1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: burn_backend::ops::ConvOptions<1>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float)], + opt_inputs[(bias, float)], + => Float, + B::conv1d(x, weight, bias, options) + ) + } + + fn conv1d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvOptions<1>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv1d_x_backward(x, weight, output_grad, options) + ) + } + + fn conv1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvOptions<1>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv1d_weight_backward(x, weight, output_grad, options) + ) + } + + fn conv1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (bias, float), (output_grad, float)], + => Float, + B::conv1d_bias_backward(x, bias, output_grad) + ) + } + + fn conv2d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv2d_x_backward(x, weight, output_grad, options) + ) + } + + fn conv2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv2d_weight_backward(x, weight, output_grad, options) + ) + } + + fn conv2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (bias, float), (output_grad, float)], + => Float, + B::conv2d_bias_backward(x, bias, output_grad) + ) + } + + fn conv3d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvOptions<3>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv3d_x_backward(x, weight, output_grad, options) + ) + } + + fn conv3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvOptions<3>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv3d_weight_backward(x, weight, output_grad, options) + ) + } + + fn conv3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (bias, float), (output_grad, float)], + => Float, + B::conv3d_bias_backward(x, bias, output_grad) + ) + } + + fn conv_transpose1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: burn_backend::ops::ConvTransposeOptions<1>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float)], + opt_inputs[(bias, float)], + => Float, + B::conv_transpose1d(x, weight, bias, options) + ) + } + + fn conv_transpose1d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvTransposeOptions<1>, + ) -> FloatTensor { + multi_op!( + inputs[(weight, float), (output_grad, float)], + => Float, + B::conv_transpose1d_x_backward(weight, output_grad, options) + ) + } + + fn conv_transpose1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvTransposeOptions<1>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv_transpose1d_weight_backward(x, weight, output_grad, options) + ) + } + + fn conv_transpose1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (bias, float), (output_grad, float)], + => Float, + B::conv_transpose1d_bias_backward(x, bias, output_grad) + ) + } + + fn conv_transpose2d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvTransposeOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(weight, float), (output_grad, float)], + => Float, + B::conv_transpose2d_x_backward(weight, output_grad, options) + ) + } + + fn conv_transpose2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvTransposeOptions<2>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv_transpose2d_weight_backward(x, weight, output_grad, options) + ) + } + + fn conv_transpose2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (bias, float), (output_grad, float)], + => Float, + B::conv_transpose2d_bias_backward(x, bias, output_grad) + ) + } + + fn conv_transpose3d_x_backward( + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvTransposeOptions<3>, + ) -> FloatTensor { + multi_op!( + inputs[(weight, float), (output_grad, float)], + => Float, + B::conv_transpose3d_x_backward(weight, output_grad, options) + ) + } + + fn conv_transpose3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: burn_backend::ops::ConvTransposeOptions<3>, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (weight, float), (output_grad, float)], + => Float, + B::conv_transpose3d_weight_backward(x, weight, output_grad, options) + ) + } + + fn conv_transpose3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (bias, float), (output_grad, float)], + => Float, + B::conv_transpose3d_bias_backward(x, bias, output_grad) + ) + } + + fn unfold4d( + x: FloatTensor, + kernel_size: [usize; 2], + options: burn_backend::ops::UnfoldOptions, + ) -> FloatTensor { + multi_op!(inputs[(x, float)], => Float, B::unfold4d(x, kernel_size, options)) + } + + fn avg_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + multi_op!(inputs[(x, float)], => Float, + B::avg_pool1d(x, kernel_size, stride, padding, count_include_pad, ceil_mode) + ) + } + + fn avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (grad, float)], + => Float, + B::avg_pool1d_backward(x, grad, kernel_size, stride, padding, count_include_pad, ceil_mode) + ) + } + + fn adaptive_avg_pool1d(x: FloatTensor, output_size: usize) -> FloatTensor { + multi_op!(inputs[(x, float)], => Float, B::adaptive_avg_pool1d(x, output_size)) + } + + fn adaptive_avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(x, float), (grad, float)], + => Float, + B::adaptive_avg_pool1d_backward(x, grad) + ) + } + + fn max_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> FloatTensor { + multi_op!(inputs[(x, float)], => Float, + B::max_pool1d(x, kernel_size, stride, padding, dilation, ceil_mode)) + } + + fn max_pool1d_with_indices( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> MaxPool1dWithIndices { + let (out, indices) = multi_op!( + inputs[(x, float)], + outputs[(out, Float), (indices, Int)], + { + let res = B::max_pool1d_with_indices(x, kernel_size, stride, padding, dilation, ceil_mode); + (res.output, res.indices) + } + ); + MaxPool1dWithIndices::new(out, indices) + } + + fn max_pool1d_with_indices_backward( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool1dBackward { + let x_grad = multi_op!( + inputs[(x, float), (output_grad, float), (indices, int)], + => Float, + { + let res = B::max_pool1d_with_indices_backward(x, kernel_size, stride, padding, dilation, ceil_mode, output_grad, indices); + res.x_grad + } + ); + MaxPool1dBackward::new(x_grad) + } + + fn attention( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: burn_backend::ops::AttentionModuleOptions, + ) -> FloatTensor { + multi_op!( + inputs[(query, float), (key, float), (value, float)], + opt_inputs[(mask, bool), (attn_bias, float)], + => Float, + B::attention(query, key, value, mask, attn_bias, options) + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/qtensor.rs new file mode 100644 index 0000000..e928765 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/qtensor.rs @@ -0,0 +1,212 @@ +use burn_backend::{ + ExecutionError, QTensorPrimitive, TensorData, TensorPrimitive, + ops::QTensorOps, + quantization::QuantizationParametersPrimitive, + tensor::{FloatTensor, IntTensor, QuantizedTensor}, +}; +use burn_std::{QuantPropagation, Shape, Slice}; + +use crate::backends::*; +use crate::{Dispatch, DispatchDevice}; + +impl QTensorOps for Dispatch { + fn q_from_data(data: TensorData, device: &DispatchDevice) -> QuantizedTensor { + creation_op!(Quantized, device, |device| B::q_from_data(data, device)) + } + + fn quantize( + tensor: FloatTensor, + scheme: &burn_std::QuantScheme, + qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + binary_op!( + (tensor, float), + (qparams.scales, float), + |tensor, scales| { + B::quantize(tensor, scheme, QuantizationParametersPrimitive { scales }) + } => Quantized + ) + } + + fn dequantize(tensor: QuantizedTensor) -> FloatTensor { + unary_op!(tensor, quantized, |tensor| B::dequantize(tensor) => Float) + } + + fn q_device(tensor: &QuantizedTensor) -> DispatchDevice { + tensor.device() + } + + fn q_to_device( + tensor: QuantizedTensor, + device: &DispatchDevice, + ) -> QuantizedTensor { + to_device!( + Quantized, + quantized, + tensor, + device, + q_to_device, + |inner, device| { + let data = + burn_backend::read_sync(B1::q_into_data(inner)).expect("Should read data"); + B2::q_from_data(data, device) + } + ) + } + + fn q_reshape(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + unary_op!(tensor, quantized, |tensor| B::q_reshape(tensor, shape) => Quantized) + } + + async fn q_into_data(tensor: QuantizedTensor) -> Result { + unary_op!(tensor, quantized, |tensor| B::q_into_data(tensor).await) + } + + fn q_expand(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + unary_op!(tensor, quantized, |tensor| B::q_expand(tensor, shape) => Quantized) + } + + fn q_swap_dims( + tensor: QuantizedTensor, + dim1: usize, + dim2: usize, + ) -> QuantizedTensor { + unary_op!(tensor, quantized, |tensor| B::q_swap_dims(tensor, dim1, dim2) => Quantized) + } + + fn q_permute(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + unary_op!(tensor, quantized, |tensor| B::q_permute(tensor, axes) => Quantized) + } + + fn q_flip(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + unary_op!(tensor, quantized, |tensor| B::q_flip(tensor, axes) => Quantized) + } + + fn q_select( + tensor: QuantizedTensor, + dim: usize, + indices: IntTensor, + ) -> QuantizedTensor { + binary_op!( + (tensor, quantized), + (indices, int), + |tensor, indices| B::q_select(tensor, dim, indices) => Quantized + ) + } + + fn q_slice(tensor: QuantizedTensor, slices: &[Slice]) -> QuantizedTensor { + unary_op!(tensor, quantized, |tensor| B::q_slice(tensor, slices) => Quantized) + } + + fn q_matmul(lhs: TensorPrimitive, rhs: TensorPrimitive) -> TensorPrimitive { + // TODO: this would be much cleaner if we consolidated tensor primitive types + match (lhs, rhs) { + (TensorPrimitive::QFloat(lhs), TensorPrimitive::QFloat(rhs)) => { + if matches!(lhs.propagation(), QuantPropagation::Propagate) { + let out = binary_op!( + (lhs, quantized), + (rhs, quantized), + |lhs, rhs| { + if let TensorPrimitive::QFloat(out) = B::q_matmul( + TensorPrimitive::QFloat(lhs), + TensorPrimitive::QFloat(rhs), + ) { + out + } else { + unreachable!() + } + } => Quantized + ); + TensorPrimitive::QFloat(out) + } else { + let out = binary_op!( + (lhs, quantized), + (rhs, quantized), + |lhs, rhs| { + if let TensorPrimitive::Float(out) = B::q_matmul( + TensorPrimitive::QFloat(lhs), + TensorPrimitive::QFloat(rhs), + ) { + out + } else { + unreachable!() + } + } => Float + ); + TensorPrimitive::Float(out) + } + } + (TensorPrimitive::Float(lhs), TensorPrimitive::QFloat(rhs)) => { + if matches!(rhs.propagation(), QuantPropagation::Propagate) { + let out = binary_op!( + (lhs, float), + (rhs, quantized), + |lhs, rhs| { + if let TensorPrimitive::QFloat(out) = B::q_matmul( + TensorPrimitive::Float(lhs), + TensorPrimitive::QFloat(rhs), + ) { + out + } else { + unreachable!() + } + } => Quantized + ); + TensorPrimitive::QFloat(out) + } else { + let out = binary_op!( + (lhs, float), + (rhs, quantized), + |lhs, rhs| { + if let TensorPrimitive::Float(out) = B::q_matmul( + TensorPrimitive::Float(lhs), + TensorPrimitive::QFloat(rhs), + ) { + out + } else { + unreachable!() + } + } => Float + ); + TensorPrimitive::Float(out) + } + } + (TensorPrimitive::QFloat(lhs), TensorPrimitive::Float(rhs)) => { + if matches!(lhs.propagation(), QuantPropagation::Propagate) { + let out = binary_op!( + (lhs, quantized), + (rhs, float), + |lhs, rhs| { + if let TensorPrimitive::QFloat(out) = B::q_matmul( + TensorPrimitive::QFloat(lhs), + TensorPrimitive::Float(rhs), + ) { + out + } else { + unreachable!() + } + } => Quantized + ); + TensorPrimitive::QFloat(out) + } else { + let out = binary_op!( + (lhs, quantized), + (rhs, float), + |lhs, rhs| { + if let TensorPrimitive::Float(out) = B::q_matmul( + TensorPrimitive::QFloat(lhs), + TensorPrimitive::Float(rhs), + ) { + out + } else { + unreachable!() + } + } => Float + ); + TensorPrimitive::Float(out) + } + } + _ => unreachable!(), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/tensor.rs new file mode 100644 index 0000000..d4f2c49 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/tensor.rs @@ -0,0 +1,594 @@ +use burn_backend::{ + ExecutionError, Scalar, TensorData, + ops::FloatTensorOps, + tensor::{BoolTensor, FloatTensor, IntTensor}, +}; +use burn_std::{FloatDType, Shape, Slice}; + +use crate::backends::*; +use crate::{Dispatch, DispatchDevice}; + +// TODO: remove backend default elem type genericsnow that we have per-device defaults +// https://github.com/tracel-ai/burn/issues/3642 + +impl FloatTensorOps for Dispatch { + fn float_from_data( + data: burn_backend::TensorData, + device: &DispatchDevice, + ) -> FloatTensor { + creation_op!(Float, device, |device| B::float_from_data(data, device)) + } + + fn float_random( + shape: Shape, + distribution: burn_backend::Distribution, + device: &DispatchDevice, + ) -> FloatTensor { + creation_op!(Float, device, |device| { + B::float_random(shape, distribution, device) + }) + } + + async fn float_into_data(tensor: FloatTensor) -> Result { + unary_float!(tensor, float, |tensor| B::float_into_data(tensor).await) + } + + fn float_device(tensor: &FloatTensor) -> DispatchDevice { + tensor.device() + } + + fn float_to_device(tensor: FloatTensor, device: &DispatchDevice) -> FloatTensor { + float_to_device!( + Float, + float, + tensor, + device, + float_to_device, + |inner, device| { + let data = + burn_backend::read_sync(B1::float_into_data(inner)).expect("Should read data"); + B2::float_from_data(data, device) + } + ) + } + + fn float_into_int(tensor: FloatTensor) -> IntTensor { + unary_float!(tensor, float, |tensor| B::float_into_int(tensor) => Int) + } + + fn float_empty(shape: Shape, device: &DispatchDevice, dtype: FloatDType) -> FloatTensor { + creation_op!(Float, device, |device| B::float_empty(shape, device, dtype)) + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_add(lhs, rhs) => Float) + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + unary_float!(lhs, float, |lhs| B::float_add_scalar(lhs, rhs) => Float) + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_sub(lhs, rhs) => Float) + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + unary_float!(lhs, float, |lhs| B::float_sub_scalar(lhs, rhs) => Float) + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_mul(lhs, rhs) => Float) + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + unary_float!(lhs, float, |lhs| B::float_mul_scalar(lhs, rhs) => Float) + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_div(lhs, rhs) => Float) + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + unary_float!(lhs, float, |lhs| B::float_div_scalar(lhs, rhs) => Float) + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_remainder(lhs, rhs) => Float) + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + unary_float!(lhs, float, |lhs| B::float_remainder_scalar(lhs, rhs) => Float) + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_matmul(lhs, rhs) => Float) + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_cross(lhs, rhs, dim) => Float) + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_recip(tensor) => Float) + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_swap_dims(tensor, dim1, dim2) => Float) + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_permute(tensor, axes) => Float) + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_flip(tensor, axes) => Float) + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_reshape(tensor, shape) => Float) + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + binary_float!((tensor, float), (indices, int), |tensor, indices| B::float_gather(dim, tensor, indices) => Float) + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(tensor, float), (indices, int), (value, float)], => Float, + B::float_scatter_add(dim, tensor, indices, value) + ) + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + ) -> FloatTensor { + binary_float!((tensor, float), (indices, int), |tensor, indices| B::float_select(tensor, dim, indices) => Float) + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(tensor, float), (indices, int), (value, float)], => Float, + B::float_select_add(tensor, dim, indices, value) + ) + } + + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_slice(tensor, slices) => Float) + } + + fn float_slice_assign( + tensor: FloatTensor, + slices: &[Slice], + value: FloatTensor, + ) -> FloatTensor { + binary_float!((tensor, float), (value, float), |tensor, value| B::float_slice_assign(tensor, slices, value) => Float) + } + + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + value: FloatTensor, + ) -> FloatTensor { + multi_op!( + inputs[(tensor, float), (mask, bool), (value, float)], => Float, + B::float_mask_where(tensor, mask, value) + ) + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor { + binary_float!((tensor, float), (mask, bool), |tensor, mask| B::float_mask_fill(tensor, mask, value) => Float) + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_equal(lhs, rhs) => Bool) + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + unary_float!(lhs, float, |lhs| B::float_equal_elem(lhs, rhs) => Bool) + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_greater(lhs, rhs) => Bool) + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + unary_float!(lhs, float, |lhs| B::float_greater_elem(lhs, rhs) => Bool) + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_greater_equal(lhs, rhs) => Bool) + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + unary_float!(lhs, float, |lhs| B::float_greater_equal_elem(lhs, rhs) => Bool) + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_lower(lhs, rhs) => Bool) + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + unary_float!(lhs, float, |lhs| B::float_lower_elem(lhs, rhs) => Bool) + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_lower_equal(lhs, rhs) => Bool) + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + unary_float!(lhs, float, |lhs| B::float_lower_equal_elem(lhs, rhs) => Bool) + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sum(tensor) => Float) + } + + fn float_sum_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sum_dim(tensor, dim) => Float) + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_mean_dim(tensor, dim) => Float) + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cumsum(tensor, dim) => Float) + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cumprod(tensor, dim) => Float) + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cummin(tensor, dim) => Float) + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cummax(tensor, dim) => Float) + } + + fn float_cast(tensor: FloatTensor, dtype: FloatDType) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cast(tensor, dtype) => Float) + } + + fn float_exp(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_exp(tensor) => Float) + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_log(tensor) => Float) + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_log1p(tensor) => Float) + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_powf(lhs, rhs) => Float) + } + + fn float_powf_scalar_impl(tensor: FloatTensor, value: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_powf_scalar_impl(tensor, value) => Float) + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sqrt(tensor) => Float) + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_abs(tensor) => Float) + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cos(tensor) => Float) + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sin(tensor) => Float) + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_tan(tensor) => Float) + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_cosh(tensor) => Float) + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sinh(tensor) => Float) + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_tanh(tensor) => Float) + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_acos(tensor) => Float) + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_acosh(tensor) => Float) + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_asin(tensor) => Float) + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_asinh(tensor) => Float) + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_atan(tensor) => Float) + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_atanh(tensor) => Float) + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_atan2(lhs, rhs) => Float) + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_round(tensor) => Float) + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_floor(tensor) => Float) + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_ceil(tensor) => Float) + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_trunc(tensor) => Float) + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_erf(tensor) => Float) + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor { + unary_float!(tensor, float, |tensor| B::float_argmax(tensor, dim) => Int) + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor { + unary_float!(tensor, float, |tensor| B::float_argmin(tensor, dim) => Int) + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_expand(tensor, shape) => Float) + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + unary_float!(tensor, float, |tensor| { + B::float_unfold(tensor, dim, size, step) + } => Float) + } + + fn float_detach(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_detach(tensor) => Float) + } + + fn float_set_require_grad(tensor: FloatTensor, require_grad: bool) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_set_require_grad(tensor, require_grad) => Float) + } + + fn float_is_require_grad(tensor: &FloatTensor) -> bool { + unary_float!(ref tensor, float, |tensor| B::float_is_require_grad(tensor)) + } + + // Default implementation + fn float_zeros(shape: Shape, device: &DispatchDevice, dtype: FloatDType) -> FloatTensor { + creation_op!(Float, device, |device| B::float_zeros(shape, device, dtype)) + } + + fn float_ones(shape: Shape, device: &DispatchDevice, dtype: FloatDType) -> FloatTensor { + creation_op!(Float, device, |device| B::float_ones(shape, device, dtype)) + } + + fn float_full( + shape: Shape, + fill_value: Scalar, + device: &DispatchDevice, + dtype: FloatDType, + ) -> FloatTensor { + creation_op!(Float, device, |device| B::float_full( + shape, fill_value, device, dtype + )) + } + + fn float_repeat_dim(tensor: FloatTensor, dim: usize, times: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_repeat_dim(tensor, dim, times) => Float) + } + + fn float_clamp_min(tensor: FloatTensor, min: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_clamp_min(tensor, min) => Float) + } + + fn float_clamp_max(tensor: FloatTensor, max: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_clamp_max(tensor, max) => Float) + } + + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_clamp(tensor, min, max) => Float) + } + + fn float_neg(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_neg(tensor) => Float) + } + + fn float_transpose(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_transpose(tensor) => Float) + } + + fn float_not_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float!((lhs, float), (rhs, float), |lhs, rhs| B::float_not_equal(lhs, rhs) => Bool) + } + + fn float_not_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + unary_float!(lhs, float, |lhs| B::float_not_equal_elem(lhs, rhs) => Bool) + } + + fn float_prod(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_prod(tensor) => Float) + } + + fn float_prod_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_prod_dim(tensor, dim) => Float) + } + + fn float_mean(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_mean(tensor) => Float) + } + + fn float_powi(lhs: FloatTensor, rhs: IntTensor) -> FloatTensor { + binary_float!((lhs, float), (rhs, int), |lhs, rhs| B::float_powi(lhs, rhs) => Float) + } + + fn float_powi_scalar_impl(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + unary_float!(lhs, float, |lhs| B::float_powi_scalar_impl(lhs, rhs) => Float) + } + + fn float_powf_scalar(tensor: FloatTensor, value: Scalar) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_powf_scalar(tensor, value) => Float) + } + + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + vec_op!(tensors, float, |tensors| B::float_cat(tensors, dim) => Float) + } + + fn float_max(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_max(tensor) => Float) + } + + fn float_max_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_max_dim(tensor, dim) => Float) + } + + fn float_max_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + multi_op!( + inputs[(tensor, float)], + outputs[(out, Float), (indices, Int)], + B::float_max_dim_with_indices(tensor, dim) + ) + } + + fn float_min(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_min(tensor) => Float) + } + + fn float_min_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_min_dim(tensor, dim) => Float) + } + + fn float_min_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + multi_op!( + inputs[(tensor, float)], + outputs[(out, Float), (indices, Int)], + B::float_min_dim_with_indices(tensor, dim) + ) + } + + fn float_max_abs(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_max_abs(tensor) => Float) + } + + fn float_max_abs_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_max_abs_dim(tensor, dim) => Float) + } + + fn float_any(tensor: FloatTensor) -> BoolTensor { + unary_float!(tensor, float, |tensor| B::float_any(tensor) => Bool) + } + + fn float_any_dim(tensor: FloatTensor, dim: usize) -> BoolTensor { + unary_float!(tensor, float, |tensor| B::float_any_dim(tensor, dim) => Bool) + } + + fn float_all(tensor: FloatTensor) -> BoolTensor { + unary_float!(tensor, float, |tensor| B::float_all(tensor) => Bool) + } + + fn float_all_dim(tensor: FloatTensor, dim: usize) -> BoolTensor { + unary_float!(tensor, float, |tensor| B::float_all_dim(tensor, dim) => Bool) + } + + fn float_sign(tensor: FloatTensor) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sign(tensor) => Float) + } + + fn float_sort(tensor: FloatTensor, dim: usize, descending: bool) -> FloatTensor { + unary_float!(tensor, float, |tensor| B::float_sort(tensor, dim, descending) => Float) + } + + fn float_sort_with_indices( + tensor: FloatTensor, + dim: usize, + descending: bool, + ) -> (FloatTensor, IntTensor) { + multi_op!( + inputs[(tensor, float)], + outputs[(out, Float), (indices, Int)], + B::float_sort_with_indices(tensor, dim, descending) + ) + } + + fn float_argsort(tensor: FloatTensor, dim: usize, descending: bool) -> IntTensor { + unary_float!(tensor, float, |tensor| B::float_argsort(tensor, dim, descending) => Int) + } + + fn float_grid_sample_2d( + tensor: FloatTensor, + grid: FloatTensor, + options: burn_backend::ops::GridSampleOptions, + ) -> FloatTensor { + binary_float!((tensor, float), (grid, float), |tensor, grid| B::float_grid_sample_2d(tensor, grid, options) => Float) + } + + fn float_is_nan(tensor: FloatTensor) -> BoolTensor { + unary_float!(tensor, float, |tensor| B::float_is_nan(tensor) => Bool) + } + + fn float_is_inf(tensor: FloatTensor) -> BoolTensor { + unary_float!(tensor, float, |tensor| B::float_is_inf(tensor) => Bool) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/transaction.rs new file mode 100644 index 0000000..fd357db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/ops/transaction.rs @@ -0,0 +1,26 @@ +use burn_backend::{ + ExecutionError, + ops::{TransactionOps, TransactionPrimitive, TransactionPrimitiveData}, +}; + +use crate::Dispatch; +use crate::backends::*; + +impl TransactionOps for Dispatch { + async fn tr_execute( + transaction: TransactionPrimitive, + ) -> Result { + let first_tensor = transaction + .read_floats + .first() + .or(transaction.read_ints.first()) + .or(transaction.read_bools.first()); + + match first_tensor { + Some(tensor) => { + transaction_op!(transaction, tensor) + } + None => Ok(TransactionPrimitiveData::default()), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/tensor.rs new file mode 100644 index 0000000..94448e5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-dispatch/src/tensor.rs @@ -0,0 +1,274 @@ +use burn_backend::{Backend, QTensorPrimitive, TensorMetadata}; + +use crate::backends::*; + +#[cfg(feature = "autodiff")] +use burn_backend::tensor::FloatTensor; + +// TODO: if we reduce the different associated types for float/int/bool/quantized tensor primitives down to a single +// `B::TensorPrimitive` we can simplify this. + +/// Tensor which points to a backend tensor primitive kind. +#[derive(Clone, Debug)] +pub enum BackendTensor { + /// Float tensor handle. + Float(B::FloatTensorPrimitive), + /// Int tensor handle. + Int(B::IntTensorPrimitive), + /// Bool tensor handle. + Bool(B::BoolTensorPrimitive), + /// Quantized tensor handle. + Quantized(B::QuantizedTensorPrimitive), + #[cfg(feature = "autodiff")] + /// Autodiff float tensor handle. + Autodiff(FloatTensor>), +} + +impl BackendTensor { + /// Returns the inner float tensor primitive. + pub(crate) fn float(self) -> B::FloatTensorPrimitive { + match self { + BackendTensor::Float(tensor) => tensor, + BackendTensor::Int(_) => panic!("Should be float, got int"), + BackendTensor::Bool(_) => panic!("Should be float, got bool"), + BackendTensor::Quantized(_) => panic!("Should be float, got quantized"), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(_) => panic!("Should be float, got autodiff"), + } + } + /// Returns the inner float tensor primitive. + pub(crate) fn as_float(&self) -> &B::FloatTensorPrimitive { + match self { + BackendTensor::Float(tensor) => tensor, + BackendTensor::Int(_) => panic!("Should be float, got int"), + BackendTensor::Bool(_) => panic!("Should be float, got bool"), + BackendTensor::Quantized(_) => panic!("Should be float, got quantized"), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(_) => panic!("Should be float, got autodiff"), + } + } + + /// Returns the inner int tensor primitive. + pub(crate) fn int(self) -> B::IntTensorPrimitive { + match self { + BackendTensor::Int(tensor) => tensor, + BackendTensor::Float(_) => panic!("Should be int, got float"), + BackendTensor::Bool(_) => panic!("Should be int, got bool"), + BackendTensor::Quantized(_) => panic!("Should be int, got quantized"), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(_) => panic!("Should be int, got autodiff"), + } + } + + /// Returns the inner bool tensor primitive. + pub(crate) fn bool(self) -> B::BoolTensorPrimitive { + match self { + BackendTensor::Bool(tensor) => tensor, + BackendTensor::Float(_) => panic!("Should be bool, got float"), + BackendTensor::Int(_) => panic!("Should be bool, got int"), + BackendTensor::Quantized(_) => panic!("Should be bool, got quantized"), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(_) => panic!("Should be bool, got autodiff"), + } + } + + /// Returns the inner quantized tensor primitive. + pub(crate) fn quantized(self) -> B::QuantizedTensorPrimitive { + match self { + BackendTensor::Quantized(tensor) => tensor, + _ => unreachable!(), + } + } + + #[cfg(feature = "autodiff")] + /// Returns the inner autodiff tensor primitive. + pub(crate) fn autodiff(self) -> FloatTensor> { + match self { + BackendTensor::Autodiff(tensor) => tensor, + // NOTE: this is the panicking code reached in tensor.rs:74:18: + _ => unreachable!(), + } + } + + #[cfg(feature = "autodiff")] + /// Returns the inner autodiff tensor primitive. + pub(crate) fn as_autodiff(&self) -> &FloatTensor> { + match self { + BackendTensor::Autodiff(tensor) => tensor, + _ => unreachable!(), + } + } + + #[cfg(feature = "autodiff")] + /// Returns the inner autodiff tensor primitive. + pub(crate) fn autodiff_inner(self) -> B::FloatTensorPrimitive { + match self { + BackendTensor::Autodiff(tensor) => tensor.primitive, + _ => unreachable!(), + } + } + + /// Returns the backend device. + pub(crate) fn device(&self) -> B::Device { + match self { + BackendTensor::Float(tensor) => B::float_device(tensor), + BackendTensor::Int(tensor) => B::int_device(tensor), + BackendTensor::Bool(tensor) => B::bool_device(tensor), + BackendTensor::Quantized(tensor) => B::q_device(tensor), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(tensor) => B::float_device(&tensor.primitive), + } + } +} + +impl TensorMetadata for BackendTensor { + fn dtype(&self) -> burn_std::DType { + match self { + BackendTensor::Float(tensor) => tensor.dtype(), + BackendTensor::Int(tensor) => tensor.dtype(), + BackendTensor::Bool(tensor) => tensor.dtype(), + BackendTensor::Quantized(tensor) => tensor.dtype(), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(tensor) => tensor.dtype(), + } + } + + fn shape(&self) -> burn_std::Shape { + match self { + BackendTensor::Float(tensor) => tensor.shape(), + BackendTensor::Int(tensor) => tensor.shape(), + BackendTensor::Bool(tensor) => tensor.shape(), + BackendTensor::Quantized(tensor) => tensor.shape(), + #[cfg(feature = "autodiff")] + BackendTensor::Autodiff(tensor) => tensor.shape(), + } + } +} + +impl QTensorPrimitive for BackendTensor { + fn scheme(&self) -> &burn_std::QuantScheme { + match self { + BackendTensor::Quantized(tensor) => tensor.scheme(), + _ => panic!( + "Quantization scheme is not valid for dtype {:?}", + self.dtype(), + ), + } + } +} + +/// Dispatch tensor that can hold tensors from any enabled backend. +/// +/// This enum wraps backend-specific tensor types, allowing runtime selection +/// of the backend to execute operations on. +#[derive(Clone, Debug)] +pub enum DispatchTensor { + /// The [CPU backend](Cpu) tensor. + #[cfg(feature = "cpu")] + Cpu(BackendTensor), + + /// The [CUDA backend](Cuda) tensor. + #[cfg(feature = "cuda")] + Cuda(BackendTensor), + + /// The [Metal backend](Metal) tensor. + #[cfg(wgpu_metal)] + Metal(BackendTensor), + + /// The [ROCm backend](Rocm) tensor. + #[cfg(feature = "rocm")] + Rocm(BackendTensor), + + /// The [Vulkan backend](Vulkan) tensor. + #[cfg(wgpu_vulkan)] + Vulkan(BackendTensor), + + /// The [WebGPU backend](WebGpu) tensor. + #[cfg(wgpu_webgpu)] + WebGpu(BackendTensor), + + /// The [NdArray backend](NdArray) tensor. + #[cfg(feature = "ndarray")] + NdArray(BackendTensor), + + /// The [LibTorch backend](LibTorch) tensor. + #[cfg(feature = "tch")] + LibTorch(BackendTensor), + + /// The [autodiff enabled backend](Autodiff) tensor. + #[cfg(feature = "autodiff")] + Autodiff(Box), +} + +impl TensorMetadata for DispatchTensor { + fn dtype(&self) -> burn_std::DType { + match self { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => tensor.dtype(), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => tensor.dtype(), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => tensor.dtype(), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => tensor.dtype(), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => tensor.dtype(), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => tensor.dtype(), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => tensor.dtype(), + #[cfg(feature = "tch")] + DispatchTensor::LibTorch(tensor) => tensor.dtype(), + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => tensor.dtype(), + } + } + + fn shape(&self) -> burn_std::Shape { + match self { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => tensor.shape(), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => tensor.shape(), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => tensor.shape(), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => tensor.shape(), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => tensor.shape(), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => tensor.shape(), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => tensor.shape(), + #[cfg(feature = "tch")] + DispatchTensor::LibTorch(tensor) => tensor.shape(), + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => tensor.shape(), + } + } +} + +impl QTensorPrimitive for DispatchTensor { + fn scheme(&self) -> &burn_std::QuantScheme { + match self { + #[cfg(feature = "cpu")] + DispatchTensor::Cpu(tensor) => tensor.scheme(), + #[cfg(feature = "cuda")] + DispatchTensor::Cuda(tensor) => tensor.scheme(), + #[cfg(wgpu_metal)] + DispatchTensor::Metal(tensor) => tensor.scheme(), + #[cfg(feature = "rocm")] + DispatchTensor::Rocm(tensor) => tensor.scheme(), + #[cfg(wgpu_vulkan)] + DispatchTensor::Vulkan(tensor) => tensor.scheme(), + #[cfg(wgpu_webgpu)] + DispatchTensor::WebGpu(tensor) => tensor.scheme(), + #[cfg(feature = "ndarray")] + DispatchTensor::NdArray(tensor) => tensor.scheme(), + #[cfg(feature = "tch")] + DispatchTensor::LibTorch(tensor) => tensor.scheme(), + #[cfg(feature = "autodiff")] + DispatchTensor::Autodiff(tensor) => tensor.scheme(), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-fusion/Cargo.toml new file mode 100644 index 0000000..c8256f7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/Cargo.toml @@ -0,0 +1,42 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Kernel fusion backend decorator for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-fusion" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-fusion" +documentation = "https://docs.rs/burn-fusion" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std", "tracing"] +std = ["serde/std", "tracing?/std"] +doc = ["default"] +memory-checks = ["std"] + +tracing = [ + "dep:tracing", + "burn-backend/tracing", + "burn-ir/tracing", +] + +[dependencies] +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2" } +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2" } +tracing = { workspace = true, optional = true, features = ["attributes"] } + +hashbrown = { workspace = true } +derive-new = { workspace = true } +spin = { workspace = true } +log = { workspace = true } +serde = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/README.md b/crates/stable-diffusion-burn/burn-crates/burn-fusion/README.md new file mode 100644 index 0000000..8c508f5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/README.md @@ -0,0 +1,3 @@ +# Burn Fusion + +A kernel fusion backend decorator for Burn. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/backend.rs new file mode 100644 index 0000000..318f885 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/backend.rs @@ -0,0 +1,240 @@ +use crate::{ + FusionTensor, + client::GlobalFusionClient, + stream::{Context, OrderedExecution}, +}; +use burn_backend::{ + Backend, DType, DeviceOps, Element, ExecutionError, + tensor::{BoolTensor, Device, FloatTensor, IntTensor, QuantizedTensor}, +}; +use burn_ir::{BackendIr, OperationIr, TensorHandle}; +use serde::{Serialize, de::DeserializeOwned}; +use std::marker::PhantomData; + +/// Get the client for the given device. +pub fn get_client(device: &Device) -> Client { + GlobalFusionClient::load(device) +} + +/// Enable dynamic operation fusion on a backend that implements [fusion backend](crate::FusionBackend). +#[derive(Clone, Debug, Default)] +pub struct Fusion { + _backend: PhantomData, +} + +impl Backend for Fusion { + type Device = B::Device; + + type FloatTensorPrimitive = FusionTensor; + + type FloatElem = B::FloatElem; + + type IntTensorPrimitive = FusionTensor; + + type IntElem = B::IntElem; + + type BoolTensorPrimitive = FusionTensor; + + type BoolElem = B::BoolElem; + + type QuantizedTensorPrimitive = FusionTensor; + + fn name(device: &Self::Device) -> String { + format!("fusion<{}>", B::name(device)) + } + + fn seed(device: &B::Device, seed: u64) { + let client = GlobalFusionClient::::load(device); + client.drain(); + B::seed(device, seed); + } + + fn sync(device: &Self::Device) -> Result<(), ExecutionError> { + let client = GlobalFusionClient::::load(device); + client.drain(); + B::sync(device) + } + + fn ad_enabled(_device: &Self::Device) -> bool { + false + } + + fn memory_persistent_allocations Output>( + device: &Self::Device, + input: Input, + func: Func, + ) -> Output { + B::memory_persistent_allocations(device, input, func) + } + + fn memory_cleanup(device: &Self::Device) { + B::memory_cleanup(device) + } + + fn staging<'a, Iter>(data: Iter, device: &Self::Device) + where + Iter: Iterator, + { + B::staging(data, device); + } + + fn supports_dtype(device: &Self::Device, dtype: DType) -> bool { + B::supports_dtype(device, dtype) + } + + fn dtype_usage(device: &Self::Device, dtype: DType) -> burn_backend::DTypeUsageSet { + B::dtype_usage(device, dtype) + } +} + +/// The status of a [fuser](OperationFuser). +#[derive(Clone, Debug, Copy, PartialEq, Eq)] +pub enum FuserStatus { + /// No more operations can be fused. + Closed, + /// More operations can be fused. + Open, +} + +/// The properties of a [fuser](OperationFuser). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FuserProperties { + /// The score of the optimization, higher is better. + pub score: u64, + /// If the operation is ready to be executed. + pub ready: bool, +} + +/// The fusion operation abstraction allows implementations to fuse many +/// [tensor operations](OperationIr) into one, improving the performance of the backend. +/// +/// +/// # Notes +/// +/// The implementations are free to execute the registered operations the way they want to improve +/// the speed and efficiency of the computational graph. It doesn't mean that all registered +/// operations should be fused, but that another way of executing them is more efficient. +/// +/// Also, it is important to return (FuserStatus::Closed) when no more registered operation can +/// improve the performance. +pub trait OperationFuser: Send { + /// Register a new [tensor operation](OperationIr). + fn fuse(&mut self, operation: &OperationIr); + /// Finish the optimization and create a fusion operation. + fn finish(&mut self) -> O; + /// Reset the state. + fn reset(&mut self); + /// Return the builder [status](FuserStatus). + fn status(&self) -> FuserStatus; + /// Return the builder [properties](FuserProperties). + fn properties(&self) -> FuserProperties; + /// The number of operation fused. + fn len(&self) -> usize; + /// If no operations are fused. + fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Clone the optimization builder. + fn clone_dyn(&self) -> Box>; +} + +/// The number of operations contained in the data structure. +pub trait NumOperations: core::fmt::Debug { + /// The number of registered operations. + fn len(&self) -> usize; + /// If the current optimization is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// The optimization created from a [fuser](OperationFuser). +pub trait Optimization: Send + NumOperations { + /// Execute the optimization. + fn execute( + &mut self, + context: &mut Context<'_, R::FusionHandle>, + execution: &OrderedExecution, + ); + + /// Returns the state that can be serialized. + fn to_state(&self) -> R::OptimizationState; + /// Create the optimization from the state. + fn from_state(device: &R::FusionDevice, state: R::OptimizationState) -> Self; +} + +/// Type alias for `::FusionDevice`. +pub type FusionDevice = ::FusionDevice; +/// Type alias for `::FusionHandle`. +pub type FusionHandle = ::FusionHandle; +/// Client alias. +pub type Client = GlobalFusionClient; + +/// Trait that defines a runtime that will benefits from fused operations. +pub trait FusionRuntime: Send + Sync + Sized + core::fmt::Debug + 'static { + /// The state that can be serialized for an optimization. + type OptimizationState: Serialize + DeserializeOwned; + /// Optimization type for the backend. + type Optimization: Optimization; + /// Handle used to store tensor dynamically. + type FusionHandle: Clone + Send; + /// Device used by the runtime. + type FusionDevice: DeviceOps; + /// The type that represents booleans on the backend. + type BoolRepr: Element; + + /// The list of fusers that will be used to optimize the computational graph. + fn fusers(device: Self::FusionDevice) -> Vec>>; +} + +/// Trait that allows an existing [backend](Backend) to specify graph optimizations using +/// [operation fuser](crate::OperationFuser). +pub trait FusionBackend: + BackendIr, Device = FusionDevice> +{ + /// The runtime used for this backend. + type FusionRuntime: FusionRuntime; + + /// Cast a float tensor and returns the resulting handle. + fn cast_float(tensor: FloatTensor, dtype: DType) -> Self::Handle; + + /// Pointer to the full precision fusion backend. + type FullPrecisionBackend: FusionBackend; +} + +// Fusion implements `BackendIr` to enable router backend usage. +impl BackendIr for Fusion { + type Handle = FusionTensor; + + fn float_tensor(handle: TensorHandle) -> FloatTensor { + handle.handle + } + + fn int_tensor(handle: TensorHandle) -> IntTensor { + handle.handle + } + + fn bool_tensor(handle: TensorHandle) -> BoolTensor { + handle.handle + } + + fn quantized_tensor(handle: TensorHandle) -> QuantizedTensor { + handle.handle + } + + fn float_tensor_handle(tensor: FloatTensor) -> Self::Handle { + tensor + } + + fn int_tensor_handle(tensor: IntTensor) -> Self::Handle { + tensor + } + + fn bool_tensor_handle(tensor: BoolTensor) -> Self::Handle { + tensor + } + + fn quantized_tensor_handle(tensor: QuantizedTensor) -> Self::Handle { + tensor + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/client.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/client.rs new file mode 100644 index 0000000..70c893a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/client.rs @@ -0,0 +1,307 @@ +use crate::{ + FusionBackend, FusionDevice, FusionHandle, FusionRuntime, FusionServer, FusionTensor, + stream::{OperationStreams, StreamId, execution::Operation}, +}; +use burn_backend::{Device, DeviceContext, DeviceId, DeviceState}; +use burn_backend::{TensorData, backend::ExecutionError}; +use burn_ir::{OperationIr, TensorId, TensorIr}; +use std::sync::Arc; + +/// Use a mutex to communicate with the fusion server. +pub struct GlobalFusionClient { + server: DeviceContext>, + device: FusionDevice, +} + +impl DeviceState for FusionServer { + fn init(device_id: DeviceId) -> Self { + let device = FusionDevice::::from_id(device_id); + FusionServer::new(device) + } +} + +impl Clone for GlobalFusionClient +where + R: FusionRuntime, +{ + fn clone(&self) -> Self { + Self { + server: self.server.clone(), + device: self.device.clone(), + } + } +} +impl GlobalFusionClient +where + R: FusionRuntime + 'static, +{ + /// Loads the client from the given device. + pub fn load(device: &FusionDevice) -> Self { + Self { + device: device.clone(), + server: DeviceContext::locate(device), + } + } +} + +impl GlobalFusionClient +where + R: FusionRuntime + 'static, +{ + /// Create a new client for the given [device](FusionRuntime::FusionDevice). + pub fn new(device: FusionDevice) -> Self { + Self { + device: device.clone(), + server: DeviceContext::locate(&device), + } + } + + /// Register a new [tensor operation intermediate representation](OperationIr). + /// + /// Returns the new (uninitialized) output tensor(s) generated by the registered operation. + pub fn register( + &self, + streams: OperationStreams, + repr: OperationIr, + operation: O, + ) -> Vec> + where + O: Operation + 'static, + { + // Create output tensors returned by this operation + let outputs = repr + .outputs() + .map(|output| { + FusionTensor::new( + output.id, + output.shape.clone(), + output.dtype, + self.clone(), + StreamId::current(), + ) + }) + .collect(); + + self.server + .lock() + .register(streams, repr, Arc::new(operation)); + + outputs + } + + /// Register all lazy computation. + pub fn drain(&self) { + let id = StreamId::current(); + self.server.lock().drain_stream(id); + } + + /// Create a new (uninitialized) empty tensor handle and returns its corresponding [tensor id](TensorId). + pub fn create_empty_handle(&self) -> TensorId { + self.server.lock().create_empty_handle() + } + + /// Get the current device used by all operations handled by this client. + pub fn device(&self) -> &FusionDevice { + &self.device + } + + /// Create a tensor with the given handle and returns its corresponding [tensor id](TensorId). + pub fn register_tensor_handle(&self, handle: FusionHandle) -> TensorId { + let mut server = self.server.lock(); + let id = server.create_empty_handle(); + server.handles.register_handle(id, handle); + core::mem::drop(server); + + id + } + + /// Read the values contained by a float tensor. + pub fn read_tensor_float( + self, + tensor: TensorIr, + stream: StreamId, + ) -> impl Future> + Send + where + B: FusionBackend, + { + self.server.lock().read_float::(tensor, stream) + } + + /// Read the values contained by an int tensor. + pub fn read_tensor_int( + self, + tensor: TensorIr, + id: StreamId, + ) -> impl Future> + Send + where + B: FusionBackend, + { + self.server.lock().read_int::(tensor, id) + } + + /// Read the values contained by a bool tensor. + pub fn read_tensor_bool( + self, + tensor: TensorIr, + stream: StreamId, + ) -> impl Future> + Send + where + B: FusionBackend, + { + self.server.lock().read_bool::(tensor, stream) + } + + /// Read the values contained by a quantized tensor. + pub fn read_tensor_quantized( + self, + tensor: TensorIr, + stream: StreamId, + ) -> impl Future> + Send + where + B: FusionBackend, + { + self.server.lock().read_quantized::(tensor, stream) + } + + /// Change the client of the given float tensor. + pub fn change_client_float( + &self, + tensor: TensorIr, + client: Self, + stream: StreamId, + ) -> FusionTensor + where + B: FusionBackend, + { + let guard = self.server.lock_device_kind(); + let mut server_current = self.server.lock(); + server_current.drain_stream(stream); + + let mut server_other = client.server.lock(); + let id = server_current.change_server_float::( + &tensor, + stream, + &client.device, + &mut server_other, + ); + + core::mem::drop(server_current); + core::mem::drop(server_other); + core::mem::drop(guard); + + FusionTensor::new(id, tensor.shape, tensor.dtype, client, StreamId::current()) + } + + /// Change the client of the given int tensor. + pub fn change_client_int( + &self, + tensor: TensorIr, + client: Self, + stream: StreamId, + ) -> FusionTensor + where + B: FusionBackend, + { + let guard = self.server.lock_device_kind(); + let mut server_current = self.server.lock(); + server_current.drain_stream(stream); + + let mut server_other = client.server.lock(); + let id = server_current.change_server_int::( + &tensor, + stream, + &client.device, + &mut server_other, + ); + + core::mem::drop(server_other); + core::mem::drop(server_current); + core::mem::drop(guard); + + FusionTensor::new(id, tensor.shape, tensor.dtype, client, StreamId::current()) + } + + /// Change the client of the given bool tensor. + pub fn change_client_bool( + &self, + tensor: TensorIr, + client: Self, + stream: StreamId, + ) -> FusionTensor + where + B: FusionBackend, + { + let guard = self.server.lock_device_kind(); + let mut server_current = self.server.lock(); + server_current.drain_stream(stream); + + let mut server_other = client.server.lock(); + let id = server_current.change_server_bool::( + &tensor, + stream, + &client.device, + &mut server_other, + ); + + core::mem::drop(server_other); + core::mem::drop(server_current); + core::mem::drop(guard); + + FusionTensor::new(id, tensor.shape, tensor.dtype, client, StreamId::current()) + } + + /// Change the client of the given quantized tensor. + pub fn change_client_quantized( + &self, + tensor: TensorIr, + client: Self, + stream: StreamId, + ) -> FusionTensor + where + B: FusionBackend, + { + let guard = self.server.lock_device_kind(); + let mut server_current = self.server.lock(); + server_current.drain_stream(stream); + + let mut server_other = client.server.lock(); + let id = + server_current.change_server_quantized::(&tensor, &client.device, &mut server_other); + + core::mem::drop(server_other); + core::mem::drop(server_current); + core::mem::drop(guard); + + FusionTensor::new(id, tensor.shape, tensor.dtype, client, StreamId::current()) + } + + /// Resolve the given float tensor to a primitive tensor. + pub fn resolve_tensor_float(&self, tensor: FusionTensor) -> B::FloatTensorPrimitive + where + B: FusionBackend, + { + let mut server = self.server.lock(); + server.drain_stream(tensor.stream); + server.resolve_server_float::(&tensor.into_ir()) + } + + /// Resolve the given int tensor to a primitive tensor. + pub fn resolve_tensor_int(&self, tensor: FusionTensor) -> B::IntTensorPrimitive + where + B: FusionBackend, + { + let mut server = self.server.lock(); + server.drain_stream(tensor.stream); + server.resolve_server_int::(&tensor.into_ir()) + } + + /// Resolve the given bool tensor to a primitive tensor. + pub fn resolve_tensor_bool(&self, tensor: FusionTensor) -> B::BoolTensorPrimitive + where + B: FusionBackend, + { + let mut server = self.server.lock(); + server.drain_stream(tensor.stream); + server.resolve_server_bool::(&tensor.into_ir()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/lib.rs new file mode 100644 index 0000000..d3c32e1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/lib.rs @@ -0,0 +1,29 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! # Burn Fusion +//! +//! This library is a part of the Burn project. It is a standalone crate that +//! can be used to perform automatic operation fusion on backends that support it. + +#[macro_use] +extern crate derive_new; + +/// Client module exposing types to communicate with the fusion server. +pub mod client; +/// Stream module exposing all tensor operations that can be optimized. +pub mod stream; + +/// Search module for stream optimizations. +pub(crate) mod search; + +mod backend; +mod ops; +mod server; +mod tensor; + +pub(crate) use server::*; + +pub use backend::*; +pub use ops::NoOp; +pub use tensor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/activation.rs new file mode 100644 index 0000000..ec95a9a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/activation.rs @@ -0,0 +1,4 @@ +use crate::{Fusion, FusionBackend}; +use burn_backend::ops::ActivationOps; + +impl ActivationOps for Fusion {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/base.rs new file mode 100644 index 0000000..90ccc78 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/base.rs @@ -0,0 +1,15 @@ +use crate::{FusionBackend, stream::Operation}; +use burn_ir::HandleContainer; +use std::marker::PhantomData; + +/// A no-operation placeholder for the fusion backend. +/// +/// `NoOp` is an implementation of [`Operation`] that doesn't execute anything. +#[derive(new, Clone, Debug)] +pub struct NoOp { + _b: PhantomData, +} + +impl Operation for NoOp { + fn execute(&self, _handles: &mut HandleContainer) {} +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/binary.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/binary.rs new file mode 100644 index 0000000..e60165c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/binary.rs @@ -0,0 +1,117 @@ +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_float_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(Debug)] + struct $name { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl $name { + fn new(desc: BinaryOpIr) -> Self { + Self { + desc, + _b: PhantomData, + } + } + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let rhs = handles.get_float_tensor::(&self.desc.rhs); + let output = $ops(lhs, rhs); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_float_cmp_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let rhs = handles.get_float_tensor::(&self.desc.rhs); + let output = $ops(lhs, rhs); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_int_cmp_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(Debug)] + struct $name { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl $name { + fn new(desc: BinaryOpIr) -> Self { + Self { + desc, + _b: PhantomData, + } + } + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_int_tensor::(&self.desc.lhs); + let rhs = handles.get_int_tensor::(&self.desc.rhs); + let output = $ops(lhs, rhs); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_int_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_int_tensor::(&self.desc.lhs); + let rhs = handles.get_int_tensor::(&self.desc.rhs); + let output = $ops(lhs, rhs); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/bool_tensor.rs new file mode 100644 index 0000000..fc473a9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/bool_tensor.rs @@ -0,0 +1,856 @@ +use crate::{ + Fusion, FusionBackend, get_client, + stream::{OperationStreams, execution::Operation}, +}; +use burn_backend::{ + Element, ExecutionError, Scalar, Shape, Slice, TensorData, + ops::BoolTensorOps, + tensor::{BoolTensor, Device, FloatTensor, IndexingUpdateOp, IntTensor}, +}; +use burn_ir::{ + BaseOperationIr, BinaryOpIr, BoolOperationIr, CastOpIr, CatOpIr, CreationOpIr, FlipOpIr, + GatherOpIr, HandleContainer, InitOperationIr, MaskFillOpIr, MaskWhereOpIr, OperationIr, + OperationOutput, PermuteOpIr, RepeatDimOpIr, ScalarOpIr, ScatterOpIr, ShapeOpIr, + SliceAssignOpIr, SliceOpIr, SwapDimsOpIr, TensorIr, UnaryOpIr, UnfoldOpIr, +}; +use std::marker::PhantomData; + +use super::NoOp; + +impl BoolTensorOps for Fusion { + fn bool_empty(shape: Shape, device: &Device) -> BoolTensor { + #[derive(new, Debug)] + struct EmptyOps { + desc: TensorIr, + device: Device, + } + + impl Operation for EmptyOps { + fn execute(&self, handles: &mut HandleContainer) { + let output = B::bool_empty(self.desc.shape.clone(), &self.device); + handles.register_bool_tensor::(&self.desc.id, output); + } + } + + let client = get_client::(device); + let desc = + CreationOpIr::create(shape, B::BoolElem::dtype(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseBool(BaseOperationIr::Empty(desc.clone())), + EmptyOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn bool_zeros(shape: Shape, device: &Device) -> BoolTensor { + #[derive(new, Debug)] + struct ZerosOps { + desc: TensorIr, + device: Device, + } + + impl Operation for ZerosOps { + fn execute(&self, handles: &mut HandleContainer) { + let output = B::bool_zeros(self.desc.shape.clone(), &self.device); + handles.register_bool_tensor::(&self.desc.id, output); + } + } + + let client = get_client::(device); + let desc = + CreationOpIr::create(shape, B::BoolElem::dtype(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseBool(BaseOperationIr::Zeros(desc.clone())), + ZerosOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn bool_ones(shape: Shape, device: &Device) -> BoolTensor { + #[derive(new, Debug)] + struct OnesOps { + desc: TensorIr, + device: Device, + } + + impl Operation for OnesOps { + fn execute(&self, handles: &mut HandleContainer) { + let output = B::bool_ones(self.desc.shape.clone(), &self.device); + handles.register_bool_tensor::(&self.desc.id, output); + } + } + + let client = get_client::(device); + let desc = + CreationOpIr::create(shape, B::BoolElem::dtype(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseBool(BaseOperationIr::Ones(desc.clone())), + OnesOps::::new(desc.out, device.clone()), + ) + .output() + } + + async fn bool_into_data(tensor: BoolTensor) -> Result { + tensor.bool_into_data::().await + } + + fn bool_from_data(data: burn_backend::TensorData, device: &Device) -> BoolTensor { + let client = get_client::(device); + let tensor = B::bool_from_data(data, device); + let shape = burn_backend::TensorMetadata::shape(&tensor); + + let handle = B::bool_tensor_handle(tensor); + let desc = InitOperationIr::create(shape, B::BoolElem::dtype(), || { + client.register_tensor_handle(handle) + }); + + client + .register( + OperationStreams::default(), + OperationIr::Init(desc), + NoOp::::new(), + ) + .output() + } + + fn bool_into_int(tensor: BoolTensor) -> IntTensor { + #[derive(new, Debug)] + struct IntoIntOps { + desc: CastOpIr, + _b: PhantomData, + } + + impl Operation for IntoIntOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_into_int(input); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), B::IntElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Bool(BoolOperationIr::IntoInt(desc.clone())), + IntoIntOps::::new(desc), + ) + .output() + } + + fn bool_into_float(tensor: BoolTensor) -> FloatTensor { + #[derive(new, Debug)] + struct IntoFloatOps { + desc: CastOpIr, + _b: PhantomData, + } + + impl Operation for IntoFloatOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_into_float(input); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), B::FloatElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Bool(BoolOperationIr::IntoFloat(desc.clone())), + IntoFloatOps::::new(desc), + ) + .output() + } + + fn bool_device(tensor: &BoolTensor) -> Device { + tensor.client.device().clone() + } + + fn bool_to_device(tensor: BoolTensor, device: &Device) -> BoolTensor { + let device_original: &B::Device = tensor.client.device(); + + if device_original == device { + return tensor; + } + + let id = tensor.stream; + let client_target = get_client::(device); + let client_original = tensor.client.clone(); + + client_original + .clone() + .change_client_bool::(tensor.into_ir(), client_target, id) + } + + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor { + if tensor.shape == shape { + return tensor; + } + + #[derive(new, Debug)] + struct ReshapeDimsOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ReshapeDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_reshape(input, self.desc.out.shape.clone()); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Reshape(desc.clone())), + ReshapeDimsOps::::new(desc), + ) + .output() + } + + fn bool_slice(tensor: BoolTensor, slices: &[Slice]) -> BoolTensor { + #[derive(new, Debug)] + struct SliceOps { + desc: SliceOpIr, + _b: PhantomData, + } + + impl Operation for SliceOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + + let output = B::bool_slice(tensor, self.desc.ranges.as_slice()); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Slice(desc.clone())), + SliceOps::::new(desc), + ) + .output() + } + + fn bool_slice_assign( + tensor: BoolTensor, + slices: &[Slice], + value: BoolTensor, + ) -> BoolTensor { + #[derive(new, Debug)] + struct SliceAssignOps { + desc: SliceAssignOpIr, + _b: PhantomData, + } + + impl Operation for SliceAssignOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + let value = handles.get_bool_tensor::(&self.desc.value); + + let output = B::bool_slice_assign(tensor, self.desc.ranges.as_slice(), value); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &value]); + + let client = tensor.client.clone(); + let desc = + SliceAssignOpIr::create(tensor.into_ir(), slices.into(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::SliceAssign(desc.clone())), + SliceAssignOps::::new(desc), + ) + .output() + } + + fn bool_cat(tensors: Vec>, dim: usize) -> BoolTensor { + #[derive(new, Debug)] + struct CatOps { + desc: CatOpIr, + _b: PhantomData, + } + + impl Operation for CatOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensors = self + .desc + .tensors + .iter() + .map(|tensor| handles.get_bool_tensor::(tensor)) + .collect(); + + let output = B::bool_cat(tensors, self.desc.dim); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs(&tensors); + + let client = tensors.first().unwrap().client.clone(); + let tensors = tensors.into_iter().map(|t| t.into_ir()).collect(); + let desc = CatOpIr::create(tensors, dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Cat(desc.clone())), + CatOps::::new(desc), + ) + .output() + } + + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + #[derive(new, Debug)] + struct EqualOps { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl Operation for EqualOps { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_bool_tensor::(&self.desc.lhs); + let rhs = handles.get_bool_tensor::(&self.desc.rhs); + let output = B::bool_equal(lhs, rhs); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Equal(desc.clone())), + EqualOps::::new(desc), + ) + .output() + } + + fn bool_not(tensor: BoolTensor) -> BoolTensor { + #[derive(new, Debug)] + struct NotOps { + desc: UnaryOpIr, + _b: PhantomData, + } + + impl Operation for NotOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_not(input); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Bool(BoolOperationIr::Not(desc.clone())), + NotOps::::new(desc), + ) + .output() + } + + fn bool_and(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + #[derive(new, Debug)] + struct AndOps { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl Operation for AndOps { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_bool_tensor::(&self.desc.lhs); + let rhs = handles.get_bool_tensor::(&self.desc.rhs); + let output = B::bool_and(lhs, rhs); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Bool(BoolOperationIr::And(desc.clone())), + AndOps::::new(desc), + ) + .output() + } + + fn bool_or(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + #[derive(new, Debug)] + struct OrOps { + desc: BinaryOpIr, + _b: PhantomData, + } + + impl Operation for OrOps { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_bool_tensor::(&self.desc.lhs); + let rhs = handles.get_bool_tensor::(&self.desc.rhs); + let output = B::bool_or(lhs, rhs); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + client + .register( + streams, + OperationIr::Bool(BoolOperationIr::Or(desc.clone())), + OrOps::::new(desc), + ) + .output() + } + + fn bool_swap_dims(tensor: BoolTensor, dim1: usize, dim2: usize) -> BoolTensor { + #[derive(new, Debug)] + struct SwapDimsOps { + desc: SwapDimsOpIr, + _b: PhantomData, + } + + impl Operation for SwapDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_swap_dims(input, self.desc.dim1, self.desc.dim2); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::SwapDims(desc.clone())), + SwapDimsOps::::new(desc), + ) + .output() + } + + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + #[derive(new, Debug)] + struct PermuteDimsOps { + desc: PermuteOpIr, + _b: PhantomData, + } + + impl Operation for PermuteDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_permute(input, self.desc.axes.as_slice()); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Permute(desc.clone())), + PermuteDimsOps::::new(desc), + ) + .output() + } + + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor { + #[derive(new, Debug)] + struct ExpandOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ExpandOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_expand(input, self.desc.out.shape.clone()); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Expand(desc.clone())), + ExpandOps::::new(desc), + ) + .output() + } + + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + #[derive(new, Debug)] + struct FlipOps { + desc: FlipOpIr, + _b: PhantomData, + } + + impl Operation for FlipOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_flip(input, self.desc.axes.as_slice()); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Flip(desc.clone())), + FlipOps::::new(desc), + ) + .output() + } + + fn bool_repeat_dim(tensor: BoolTensor, dim: usize, times: usize) -> BoolTensor { + #[derive(new, Debug)] + struct RepeatDimOps { + desc: RepeatDimOpIr, + _b: PhantomData, + } + + impl Operation for RepeatDimOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + + let output = B::bool_repeat_dim(tensor, self.desc.dim, self.desc.times); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = RepeatDimOpIr::create(tensor.into_ir(), dim, times, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::RepeatDim(desc.clone())), + RepeatDimOps::::new(desc), + ) + .output() + } + + fn bool_unfold( + tensor: BoolTensor, + dim: usize, + size: usize, + step: usize, + ) -> BoolTensor { + #[derive(new, Debug)] + struct UnfoldOps { + desc: UnfoldOpIr, + _b: PhantomData, + } + + impl Operation for UnfoldOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_bool_tensor::(&self.desc.input); + let output = B::bool_unfold(input, self.desc.dim, self.desc.size, self.desc.step); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnfoldOpIr::create(tensor.into_ir(), dim, size, step, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Unfold(desc.clone())), + UnfoldOps::::new(desc), + ) + .output() + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + #[derive(new, Debug)] + struct MaskWhereOps { + desc: MaskWhereOpIr, + _b: PhantomData, + } + + impl Operation for MaskWhereOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + let value = handles.get_bool_tensor::(&self.desc.value); + let mask = handles.get_bool_tensor::(&self.desc.mask); + + let output = B::bool_mask_where(tensor, mask, value); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &mask, &value]); + + let client = tensor.client.clone(); + let desc = MaskWhereOpIr::create(tensor.into_ir(), mask.into_ir(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::MaskWhere(desc.clone())), + MaskWhereOps::::new(desc), + ) + .output() + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + #[derive(new, Debug)] + struct MaskFillOps { + desc: MaskFillOpIr, + _b: PhantomData, + } + + impl Operation for MaskFillOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + let mask = handles.get_bool_tensor::(&self.desc.mask); + + let output = B::bool_mask_fill(tensor, mask, self.desc.value.into()); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &mask]); + + let client = tensor.client.clone(); + let value = value.into(); + let desc = MaskFillOpIr::create(tensor.into_ir(), mask.into_ir(), value, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::MaskFill(desc.clone())), + MaskFillOps::::new(desc), + ) + .output() + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + #[derive(new, Debug)] + struct GatherOps { + desc: GatherOpIr, + _b: PhantomData, + } + + impl Operation for GatherOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::bool_gather(self.desc.dim, tensor, indices); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices]); + + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Gather(desc.clone())), + GatherOps::::new(desc), + ) + .output() + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + #[derive(new, Debug)] + struct ScatterOps { + desc: ScatterOpIr, + _b: PhantomData, + } + + impl Operation for ScatterOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_bool_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + let value = handles.get_bool_tensor::(&self.desc.value); + + let output = B::bool_scatter_or(self.desc.dim, tensor, indices, value); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices, &value]); + + let client = tensor.client.clone(); + let desc = ScatterOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::Scatter(desc.clone())), + ScatterOps::::new(desc), + ) + .output() + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + #[derive(new, Debug)] + struct EqualElemOps { + desc: ScalarOpIr, + _b: PhantomData, + } + impl Operation for EqualElemOps { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_bool_tensor::(&self.desc.lhs); + let output = B::bool_equal_elem(lhs, self.desc.rhs.into()); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseBool(BaseOperationIr::EqualElem(desc.clone())), + EqualElemOps::::new(desc), + ) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/int_tensor.rs new file mode 100644 index 0000000..5af2a2c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/int_tensor.rs @@ -0,0 +1,1980 @@ +use super::NoOp; +use crate::{ + Fusion, FusionBackend, binary_int_cmp_ops, binary_int_ops, get_client, reduce_int_ops, + scalar_int_cmp_ops, scalar_int_ops, + stream::{OperationStreams, execution::Operation}, + unary_int_ops, +}; +use burn_backend::{ + Distribution, Element, ExecutionError, IntDType, Scalar, Shape, Slice, TensorData, + ops::IntTensorOps, + tensor::{BoolTensor, Device, FloatTensor, IndexingUpdateOp, IntElem, IntTensor}, +}; +use burn_ir::*; +use std::marker::PhantomData; + +impl IntTensorOps for Fusion { + fn int_empty(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + #[derive(new, Debug)] + struct EmptyOps { + desc: TensorIr, + device: Device, + } + + impl Operation for EmptyOps { + fn execute(&self, handles: &mut HandleContainer) { + let output = B::int_empty( + self.desc.shape.clone(), + &self.device, + self.desc.dtype.into(), + ); + handles.register_int_tensor::(&self.desc.id, output); + } + } + + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseInt(BaseOperationIr::Empty(desc.clone())), + EmptyOps::::new(desc.out, device.clone()), + ) + .output() + } + + async fn int_into_data(tensor: IntTensor) -> Result { + tensor.int_into_data::().await + } + + fn int_from_data(data: TensorData, device: &Device) -> IntTensor { + let client = get_client::(device); + let dtype = data.dtype; + let tensor = B::int_from_data(data, device); + let shape = burn_backend::TensorMetadata::shape(&tensor); + + let handle = B::int_tensor_handle(tensor); + let desc = InitOperationIr::create(shape, dtype, || client.register_tensor_handle(handle)); + + client + .register( + OperationStreams::default(), + OperationIr::Init(desc), + NoOp::::new(), + ) + .output() + } + + fn int_device(tensor: &IntTensor) -> Device { + tensor.client.device().clone() + } + + fn int_to_device(tensor: IntTensor, device: &Device) -> IntTensor { + let device_original: &B::Device = tensor.client.device(); + + if device_original == device { + return tensor; + } + + let id = tensor.stream; + let client_target = get_client::(device); + let client_original = tensor.client.clone(); + + client_original + .clone() + .change_client_int::(tensor.into_ir(), client_target, id) + } + + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor { + if tensor.shape == shape { + return tensor; + } + + #[derive(new, Debug)] + struct ReshapeDimsOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ReshapeDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_reshape(input, self.desc.out.shape.clone()); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Reshape(desc.clone())), + ReshapeDimsOps::::new(desc), + ) + .output() + } + + fn int_slice(tensor: IntTensor, slices: &[Slice]) -> IntTensor { + #[derive(new, Debug)] + struct SliceOps { + desc: SliceOpIr, + _b: PhantomData, + } + + impl Operation for SliceOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + + let output = B::int_slice(tensor, self.desc.ranges.as_slice()); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Slice(desc.clone())), + SliceOps::::new(desc), + ) + .output() + } + + fn int_slice_assign( + tensor: IntTensor, + slices: &[burn_backend::Slice], + value: IntTensor, + ) -> IntTensor { + #[derive(new, Debug)] + struct SliceAssignOps { + desc: SliceAssignOpIr, + _b: PhantomData, + } + + impl Operation for SliceAssignOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let value = handles.get_int_tensor::(&self.desc.value); + + let output = B::int_slice_assign(tensor, self.desc.ranges.as_slice(), value); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &value]); + + let client = tensor.client.clone(); + let desc = + SliceAssignOpIr::create(tensor.into_ir(), slices.into(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::SliceAssign(desc.clone())), + SliceAssignOps::::new(desc), + ) + .output() + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(MatmulOps, B::int_matmul); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = MatmulOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Matmul(desc.clone())), + MatmulOps::::new(desc.into()), + ) + .output() + } + + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + value: IntTensor, + ) -> IntTensor { + #[derive(new, Debug)] + struct MaskWhereOps { + desc: MaskWhereOpIr, + _b: PhantomData, + } + + impl Operation for MaskWhereOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let value = handles.get_int_tensor::(&self.desc.value); + let mask = handles.get_bool_tensor::(&self.desc.mask); + + let output = B::int_mask_where(tensor, mask, value); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &mask, &value]); + + let client = tensor.client.clone(); + let desc = MaskWhereOpIr::create(tensor.into_ir(), mask.into_ir(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::MaskWhere(desc.clone())), + MaskWhereOps::::new(desc), + ) + .output() + } + + fn int_mask_fill( + tensor: IntTensor, + mask: BoolTensor, + value: Scalar, + ) -> IntTensor { + #[derive(new, Debug)] + struct MaskFillOps { + desc: MaskFillOpIr, + _b: PhantomData, + } + + impl Operation for MaskFillOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let mask = handles.get_bool_tensor::(&self.desc.mask); + + let output = B::int_mask_fill(tensor, mask, self.desc.value.into()); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &mask]); + + let client = tensor.client.clone(); + let value = value.into(); + let desc = MaskFillOpIr::create(tensor.into_ir(), mask.into_ir(), value, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::MaskFill(desc.clone())), + MaskFillOps::::new(desc), + ) + .output() + } + + fn int_gather( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + ) -> IntTensor { + #[derive(new, Debug)] + struct GatherOps { + desc: GatherOpIr, + _b: PhantomData, + } + + impl Operation for GatherOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::int_gather(self.desc.dim, tensor, indices); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices]); + + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Gather(desc.clone())), + GatherOps::::new(desc), + ) + .output() + } + + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + #[derive(new, Debug)] + struct ScatterOps { + desc: ScatterOpIr, + _b: PhantomData, + } + + impl Operation for ScatterOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + let value = handles.get_int_tensor::(&self.desc.value); + + let output = B::int_scatter_add(self.desc.dim, tensor, indices, value); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices, &value]); + + let client = tensor.client.clone(); + let desc = ScatterOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Scatter(desc.clone())), + ScatterOps::::new(desc), + ) + .output() + } + + fn int_select( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + ) -> IntTensor { + #[derive(new, Debug)] + struct SelectOps { + desc: SelectOpIr, + _b: PhantomData, + } + + impl Operation for SelectOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::int_select(tensor, self.desc.dim, indices); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices]); + + let client = tensor.client.clone(); + let desc = SelectOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Select(desc.clone())), + SelectOps::::new(desc), + ) + .output() + } + + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + #[derive(new, Debug)] + struct SelectAssignOps { + desc: SelectAssignOpIr, + _b: PhantomData, + } + + impl Operation for SelectAssignOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + let value = handles.get_int_tensor::(&self.desc.value); + + let output = B::int_select_add(tensor, self.desc.dim, indices, value); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices, &value]); + + let client = tensor.client.clone(); + let desc = SelectAssignOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::SelectAssign(desc.clone())), + SelectAssignOps::::new(desc), + ) + .output() + } + + fn int_cat(tensors: Vec>, dim: usize) -> IntTensor { + #[derive(new, Debug)] + struct CatOps { + desc: CatOpIr, + _b: PhantomData, + } + + impl Operation for CatOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensors = self + .desc + .tensors + .iter() + .map(|tensor| handles.get_int_tensor::(tensor)) + .collect(); + + let output = B::int_cat(tensors, self.desc.dim); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs(&tensors); + + let client = tensors.first().unwrap().client.clone(); + let tensors = tensors.into_iter().map(|t| t.into_ir()).collect(); + let desc = CatOpIr::create(tensors, dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Cat(desc.clone())), + CatOps::::new(desc), + ) + .output() + } + + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_int_cmp_ops!(EqualOps, B::int_equal); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Equal(desc.clone())), + EqualOps::::new(desc), + ) + .output() + } + + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + scalar_int_cmp_ops!(EqualElemOps, B::int_equal_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::EqualElem(desc.clone())), + EqualElemOps::::new(desc), + ) + .output() + } + + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_int_cmp_ops!(GreaterOps, B::int_greater); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericInt(desc.lhs.dtype, NumericOperationIr::Greater(desc.clone())), + GreaterOps::::new(desc), + ) + .output() + } + + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + scalar_int_cmp_ops!(GreaterElemOps, B::int_greater_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::GreaterElem(desc.clone()), + ), + GreaterElemOps::::new(desc), + ) + .output() + } + + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_int_cmp_ops!(GreaterEqualOps, B::int_greater_equal); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::GreaterEqual(desc.clone()), + ), + GreaterEqualOps::::new(desc), + ) + .output() + } + + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + scalar_int_cmp_ops!(GreaterEqualElemOps, B::int_greater_equal_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::GreaterEqualElem(desc.clone()), + ), + GreaterEqualElemOps::::new(desc), + ) + .output() + } + + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_int_cmp_ops!(LowerOps, B::int_lower); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericInt(desc.lhs.dtype, NumericOperationIr::Lower(desc.clone())), + LowerOps::::new(desc), + ) + .output() + } + + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + scalar_int_cmp_ops!(LowerElemOps, B::int_lower_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::LowerElem(desc.clone()), + ), + LowerElemOps::::new(desc), + ) + .output() + } + + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + binary_int_cmp_ops!(LowerEqualOps, B::int_lower_equal); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::LowerEqual(desc.clone()), + ), + LowerEqualOps::::new(desc), + ) + .output() + } + + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + scalar_int_cmp_ops!(LowerEqualElemOps, B::int_lower_equal_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::LowerEqualElem(desc.clone()), + ), + LowerEqualElemOps::::new(desc), + ) + .output() + } + + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(AddOps, B::int_add); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Add(desc.clone())), + AddOps::::new(desc), + ) + .output() + } + + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(AddOps, B::int_add_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::AddScalar(desc.clone()), + ), + AddOps::::new(desc), + ) + .output() + } + + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(SubOps, B::int_sub); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Sub(desc.clone())), + SubOps::::new(desc), + ) + .output() + } + + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(SubOps, B::int_sub_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::SubScalar(desc.clone()), + ), + SubOps::::new(desc), + ) + .output() + } + + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(MulOps, B::int_mul); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Mul(desc.clone())), + MulOps::::new(desc), + ) + .output() + } + + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(MulOps, B::int_mul_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MulScalar(desc.clone()), + ), + MulOps::::new(desc), + ) + .output() + } + + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(DivOps, B::int_div); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Div(desc.clone())), + DivOps::::new(desc), + ) + .output() + } + + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(DivOps, B::int_div_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::DivScalar(desc.clone()), + ), + DivOps::::new(desc), + ) + .output() + } + + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(ModOps, B::int_remainder); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Rem(desc.clone())), + ModOps::::new(desc), + ) + .output() + } + + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(ModOps, B::int_remainder_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::RemScalar(desc.clone()), + ), + ModOps::::new(desc), + ) + .output() + } + + fn int_zeros(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + #[derive(new, Debug)] + struct ZerosOps { + desc: TensorIr, + device: Device, + } + + impl Operation for ZerosOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.desc.shape.clone(); + let output = B::int_zeros(shape, &self.device, self.desc.dtype.into()); + handles.register_int_tensor::(&self.desc.id, output); + } + } + + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseInt(BaseOperationIr::Zeros(desc.clone())), + ZerosOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn int_ones(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + #[derive(new, Debug)] + struct OnesOps { + desc: TensorIr, + device: Device, + } + + impl Operation for OnesOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.desc.shape.clone(); + let output = B::int_ones(shape, &self.device, self.desc.dtype.into()); + handles.register_int_tensor::(&self.desc.id, output); + } + } + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseInt(BaseOperationIr::Ones(desc.clone())), + OnesOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn int_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: IntDType, + ) -> IntTensor { + #[derive(new, Debug)] + struct FullOps { + out: TensorIr, + elem: ScalarIr, + device: Device, + } + + impl Operation for FullOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.out.shape.clone(); + let output = + B::int_full(shape, self.elem.into(), &self.device, self.out.dtype.into()); + handles.register_int_tensor::(&self.out.id, output); + } + } + + let client = get_client::(device); + let dtype = dtype.into(); + let value = fill_value.into(); + let desc = FullOpIr::create(shape, dtype, value, || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Full(desc.clone())), + FullOps::::new(desc.out, desc.value, device.clone()), + ) + .output() + } + + fn int_sum(tensor: IntTensor) -> IntTensor { + unary_int_ops!(SumOps, B::int_sum, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Sum(desc.clone())), + SumOps::::new(desc.into()), + ) + .output() + } + + fn int_sum_dim(tensor: IntTensor, axis: usize) -> IntTensor { + reduce_int_ops!(SumDimOps, B::int_sum_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), axis, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::SumDim(desc.clone())), + SumDimOps::::new(desc), + ) + .output() + } + + fn int_prod(tensor: IntTensor) -> IntTensor { + unary_int_ops!(ProdOps, B::int_prod, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Prod(desc.clone())), + ProdOps::::new(desc.into()), + ) + .output() + } + + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(ProdDimOps, B::int_prod_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::ProdDim(desc.clone())), + ProdDimOps::::new(desc), + ) + .output() + } + + fn int_mean(tensor: IntTensor) -> IntTensor { + unary_int_ops!(MeanOps, B::int_mean, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Mean(desc.clone())), + MeanOps::::new(desc.into()), + ) + .output() + } + + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(MeanDimOps, B::int_mean_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::MeanDim(desc.clone())), + MeanDimOps::::new(desc), + ) + .output() + } + + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor { + #[derive(new, Debug)] + struct CumsumOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CumsumOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_cumsum(input, self.desc.axis); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::CumSum(desc.clone())), + CumsumOps::::new(desc), + ) + .output() + } + + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor { + #[derive(new, Debug)] + struct CumprodOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CumprodOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_cumprod(input, self.desc.axis); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::CumProd(desc.clone())), + CumprodOps::::new(desc), + ) + .output() + } + + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor { + #[derive(new, Debug)] + struct CumminOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CumminOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_cummin(input, self.desc.axis); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::CumMin(desc.clone())), + CumminOps::::new(desc), + ) + .output() + } + + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor { + #[derive(new, Debug)] + struct CummaxOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CummaxOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_cummax(input, self.desc.axis); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::CumMax(desc.clone())), + CummaxOps::::new(desc), + ) + .output() + } + + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(ArgMaxOps, B::int_argmax); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::ArgMax(desc.clone())), + ArgMaxOps::::new(desc), + ) + .output() + } + + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(ArgMinOps, B::int_argmin); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::ArgMin(desc.clone())), + ArgMinOps::::new(desc), + ) + .output() + } + + fn int_clamp(tensor: IntTensor, min: Scalar, max: Scalar) -> IntTensor { + #[derive(new, Debug)] + struct ClampOps { + desc: ClampOpIr, + _b: PhantomData, + } + + impl Operation for ClampOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.tensor); + let output = B::int_clamp(input, self.desc.min.into(), self.desc.max.into()); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let min = min.into(); + let max = max.into(); + let desc = ClampOpIr::create(tensor.into_ir(), min, max, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Clamp(desc.clone())), + ClampOps::::new(desc), + ) + .output() + } + + fn int_abs(tensor: IntTensor) -> IntTensor { + unary_int_ops!(AbsOps, B::int_abs); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Abs(desc.clone())), + AbsOps::::new(desc), + ) + .output() + } + + fn int_into_float(tensor: IntTensor) -> FloatTensor { + #[derive(new, Debug)] + struct IntoFloatOps { + desc: CastOpIr, + _b: PhantomData, + } + + impl Operation for IntoFloatOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_into_float(input); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), B::FloatElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::IntoFloat(desc.clone())), + IntoFloatOps::::new(desc), + ) + .output() + } + + fn int_swap_dims(tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor { + #[derive(new, Debug)] + struct SwapDimsOps { + desc: SwapDimsOpIr, + _b: PhantomData, + } + + impl Operation for SwapDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_swap_dims(input, self.desc.dim1, self.desc.dim2); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::SwapDims(desc.clone())), + SwapDimsOps::::new(desc), + ) + .output() + } + + fn int_max(tensor: IntTensor) -> IntTensor { + unary_int_ops!(MaxOps, B::int_max, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Max(desc.clone())), + MaxOps::::new(desc.into()), + ) + .output() + } + + fn int_max_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(MaxDimOps, B::int_max_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::MaxDim(desc.clone())), + MaxDimOps::::new(desc), + ) + .output() + } + + fn int_max_dim_with_indices( + tensor: IntTensor, + dim: usize, + ) -> (IntTensor, IntTensor) { + #[derive(new, Debug)] + struct MaxDimWithIndicesOps { + desc: ReduceDimWithIndicesOpIr, + _b: PhantomData, + } + + impl Operation for MaxDimWithIndicesOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let (output, indices) = B::int_max_dim_with_indices(tensor, self.desc.dim); + + handles.register_int_tensor::(&self.desc.out.id, output); + handles.register_int_tensor::(&self.desc.out_indices.id, indices); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let dtype = tensor.dtype; + let desc = ReduceDimWithIndicesOpIr::create(tensor.into_ir(), dim, dtype, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(dtype, NumericOperationIr::MaxDimWithIndices(desc.clone())), + MaxDimWithIndicesOps::::new(desc), + ) + .outputs() + .into() + } + + fn int_min(tensor: IntTensor) -> IntTensor { + unary_int_ops!(MinOps, B::int_min, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::Min(desc.clone())), + MinOps::::new(desc.into()), + ) + .output() + } + + fn int_max_abs(tensor: IntTensor) -> IntTensor { + unary_int_ops!(MaxAbsOps, B::int_max_abs, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::MaxAbs(desc.clone())), + MaxAbsOps::::new(desc.into()), + ) + .output() + } + + fn int_max_abs_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(MaxAbsDimOps, B::int_max_abs_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MaxAbsDim(desc.clone()), + ), + MaxAbsDimOps::::new(desc), + ) + .output() + } + + fn int_min_dim(tensor: IntTensor, dim: usize) -> IntTensor { + reduce_int_ops!(MinDimOps, B::int_min_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericInt(desc.out.dtype, NumericOperationIr::MinDim(desc.clone())), + MinDimOps::::new(desc), + ) + .output() + } + + fn int_min_dim_with_indices( + tensor: IntTensor, + dim: usize, + ) -> (IntTensor, IntTensor) { + #[derive(new, Debug)] + struct MinDimWithIndicesOps { + desc: ReduceDimWithIndicesOpIr, + _b: PhantomData, + } + + impl Operation for MinDimWithIndicesOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + let (output, indices) = B::int_min_dim_with_indices(tensor, self.desc.dim); + + handles.register_int_tensor::(&self.desc.out.id, output); + handles.register_int_tensor::(&self.desc.out_indices.id, indices); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let dtype = tensor.dtype; + let desc = ReduceDimWithIndicesOpIr::create(tensor.into_ir(), dim, dtype, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericInt(dtype, NumericOperationIr::MinDimWithIndices(desc.clone())), + MinDimWithIndicesOps::::new(desc), + ) + .outputs() + .into() + } + + fn int_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> IntTensor { + #[derive(new, Debug)] + struct IntRandomOps { + desc: RandomOpIr, + device: Device, + } + + impl Operation for IntRandomOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.desc.out.shape.clone(); + let output = B::int_random(shape, self.desc.distribution, &self.device); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let dtype = IntElem::::dtype(); + let client = get_client::(device); + let desc = RandomOpIr::create(shape, dtype, distribution, || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::NumericInt(dtype, NumericOperationIr::IntRandom(desc.clone())), + IntRandomOps::::new(desc, device.clone()), + ) + .output() + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + #[derive(new, Debug)] + struct PermuteDimsOps { + desc: PermuteOpIr, + _b: PhantomData, + } + + impl Operation for PermuteDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_permute(input, self.desc.axes.as_slice()); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Permute(desc.clone())), + PermuteDimsOps::::new(desc), + ) + .output() + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + #[derive(new, Debug)] + struct ExpandOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ExpandOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_expand(input, self.desc.out.shape.clone()); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Expand(desc.clone())), + ExpandOps::::new(desc), + ) + .output() + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + #[derive(new, Debug)] + struct FlipDimsOps { + desc: FlipOpIr, + _b: PhantomData, + } + + impl Operation for FlipDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let axes = &self.desc.axes; + let output = B::int_flip(input, axes); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Flip(desc.clone())), + FlipDimsOps::::new(desc), + ) + .output() + } + + fn int_repeat_dim(tensor: IntTensor, dim: usize, times: usize) -> IntTensor { + #[derive(new, Debug)] + struct RepeatDimOps { + desc: RepeatDimOpIr, + _b: PhantomData, + } + + impl Operation for RepeatDimOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_int_tensor::(&self.desc.tensor); + + let output = B::int_repeat_dim(tensor, self.desc.dim, self.desc.times); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = RepeatDimOpIr::create(tensor.into_ir(), dim, times, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::RepeatDim(desc.clone())), + RepeatDimOps::::new(desc), + ) + .output() + } + + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(BitwiseAndOps, B::bitwise_and); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseAnd(desc.clone())), + BitwiseAndOps::::new(desc), + ) + .output() + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(BitwiseAndOps, B::bitwise_and_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseAndScalar(desc.clone())), + BitwiseAndOps::::new(desc), + ) + .output() + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(BitwiseOrOps, B::bitwise_or); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseOr(desc.clone())), + BitwiseOrOps::::new(desc), + ) + .output() + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(BitwiseOrOps, B::bitwise_or_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseOrScalar(desc.clone())), + BitwiseOrOps::::new(desc), + ) + .output() + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(BitwiseXorOps, B::bitwise_xor); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseXor(desc.clone())), + BitwiseXorOps::::new(desc), + ) + .output() + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(BitwiseXorOps, B::bitwise_xor_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseXorScalar(desc.clone())), + BitwiseXorOps::::new(desc), + ) + .output() + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + unary_int_ops!(BitwiseNotOps, B::bitwise_not); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseNot(desc.clone())), + BitwiseNotOps::::new(desc), + ) + .output() + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(BitwiseLeftShiftOps, B::bitwise_left_shift); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseLeftShift(desc.clone())), + BitwiseLeftShiftOps::::new(desc), + ) + .output() + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(BitwiseLeftShiftOps, B::bitwise_left_shift_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseLeftShiftScalar(desc.clone())), + BitwiseLeftShiftOps::::new(desc), + ) + .output() + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + binary_int_ops!(BitwiseRightShiftOps, B::bitwise_right_shift); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseRightShift(desc.clone())), + BitwiseRightShiftOps::::new(desc), + ) + .output() + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + scalar_int_ops!(BitwiseRightShiftOps, B::bitwise_right_shift_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Int(IntOperationIr::BitwiseRightShiftScalar(desc.clone())), + BitwiseRightShiftOps::::new(desc), + ) + .output() + } + + fn int_cast(tensor: IntTensor, dtype: burn_backend::IntDType) -> IntTensor { + #[derive(new, Debug)] + struct CastOps { + desc: CastOpIr, + dtype: burn_backend::IntDType, + _b: PhantomData, + } + + impl Operation for CastOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output: B::IntTensorPrimitive = B::int_cast(input, self.dtype); + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), dtype.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Cast(desc.clone())), + CastOps::::new(desc, dtype), + ) + .output() + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + #[derive(new, Debug)] + struct UnfoldOps { + desc: UnfoldOpIr, + _b: PhantomData, + } + + impl Operation for UnfoldOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = B::int_unfold(input, self.desc.dim, self.desc.size, self.desc.step); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnfoldOpIr::create(tensor.into_ir(), dim, size, step, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Unfold(desc.clone())), + UnfoldOps::::new(desc), + ) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/mod.rs new file mode 100644 index 0000000..e9bc3b5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/mod.rs @@ -0,0 +1,12 @@ +mod activation; +mod binary; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; +mod unary; + +mod base; +pub use base::NoOp; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/module.rs new file mode 100644 index 0000000..01e687b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/module.rs @@ -0,0 +1,1563 @@ +use crate::{ + Fusion, FusionBackend, + stream::{OperationStreams, execution::Operation}, +}; +use burn_backend::{ + Element, + ops::{ + ConvOptions, ConvTransposeOptions, DeformConv2dBackward, DeformConvOptions, + InterpolateOptions, MaxPool1dBackward, MaxPool1dWithIndices, MaxPool2dBackward, + MaxPool2dWithIndices, ModuleOps, + }, + tensor::{FloatTensor, IntTensor}, +}; +use burn_ir::*; +use std::marker::PhantomData; + +macro_rules! make_ops { + ($name:ident, $desc:ty, $fn:expr) => { + #[derive(new, Debug)] + struct $name { + desc: $desc, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + #[allow(clippy::redundant_closure_call)] + $fn(&self.desc, handles) + } + } + }; +} + +impl ModuleOps> for Fusion { + fn conv1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<1>, + ) -> FloatTensor { + make_ops!(Conv1dOps, Conv1dOpIr, |desc: &Conv1dOpIr, + handles: &mut HandleContainer< + B::Handle, + >| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + let output = B::conv1d(x, weight, bias, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + }); + + let mut streams = OperationStreams::with_inputs([&x, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + + let client = x.client.clone(); + let desc = Conv1dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv1d(desc.clone())), + Conv1dOps::::new(desc), + ) + .output() + } + + fn conv1d_x_backward( + x: FloatTensor>, + weight: FloatTensor>, + output_grad: FloatTensor>, + options: ConvOptions<1>, + ) -> FloatTensor> { + make_ops!( + Conv1dXBackwardOps, + Conv1dXBackwardOpIr, + |desc: &Conv1dXBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = + B::conv1d_x_backward(x, weight, output_grad, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &weight, &output_grad]); + + let client = x.client.clone(); + let desc = Conv1dXBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv1dXBackward(desc.clone())), + Conv1dXBackwardOps::::new(desc), + ) + .output() + } + + fn conv1d_weight_backward( + x: FloatTensor>, + weight: FloatTensor>, + output_grad: FloatTensor>, + options: ConvOptions<1>, + ) -> FloatTensor> { + make_ops!( + Conv1dWeightBackwardOps, + Conv1dWeightBackwardOpIr, + |desc: &Conv1dWeightBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = + B::conv1d_weight_backward(x, weight, output_grad, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &weight, &output_grad]); + + let client = x.client.clone(); + let desc = Conv1dWeightBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv1dWeightBackward(desc.clone())), + Conv1dWeightBackwardOps::::new(desc), + ) + .output() + } + + fn conv1d_bias_backward( + x: FloatTensor>, + bias: FloatTensor>, + output_grad: FloatTensor>, + ) -> FloatTensor> { + make_ops!( + Conv1dBiasBackwardOps, + Conv1dBiasBackwardOpIr, + |desc: &Conv1dBiasBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let bias = handles.get_float_tensor::(&desc.bias); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = B::conv1d_bias_backward(x, bias, output_grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &bias, &output_grad]); + + let client = x.client.clone(); + let desc = Conv1dBiasBackwardOpIr::create( + x.into_ir(), + bias.into_ir(), + output_grad.into_ir(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv1dBiasBackward(desc.clone())), + Conv1dBiasBackwardOps::::new(desc), + ) + .output() + } + + fn conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<2>, + ) -> FloatTensor { + make_ops!(Conv2dOps, Conv2dOpIr, |args: &Conv2dOpIr, + handles: &mut HandleContainer< + B::Handle, + >| { + let x = handles.get_float_tensor::(&args.x); + let weight = handles.get_float_tensor::(&args.weight); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv2d(x, weight, bias, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + }); + + let mut streams = OperationStreams::with_inputs([&x, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + + let client = x.client.clone(); + let desc = Conv2dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv2d(desc.clone())), + Conv2dOps::::new(desc), + ) + .output() + } + + fn conv2d_x_backward( + x: FloatTensor>, + weight: FloatTensor>, + output_grad: FloatTensor>, + options: ConvOptions<2>, + ) -> FloatTensor> { + make_ops!( + Conv2dXBackwardOps, + Conv2dXBackwardOpIr, + |desc: &Conv2dXBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = + B::conv2d_x_backward(x, weight, output_grad, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &weight, &output_grad]); + + let client = x.client.clone(); + let desc = Conv2dXBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv2dXBackward(desc.clone())), + Conv2dXBackwardOps::::new(desc), + ) + .output() + } + + fn conv2d_weight_backward( + x: FloatTensor>, + weight: FloatTensor>, + output_grad: FloatTensor>, + options: ConvOptions<2>, + ) -> FloatTensor> { + make_ops!( + Conv2dWeightBackwardOps, + Conv2dWeightBackwardOpIr, + |desc: &Conv2dWeightBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = + B::conv2d_weight_backward(x, weight, output_grad, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &weight, &output_grad]); + + let client = x.client.clone(); + let desc = Conv2dWeightBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv2dWeightBackward(desc.clone())), + Conv2dWeightBackwardOps::::new(desc), + ) + .output() + } + + fn conv2d_bias_backward( + x: FloatTensor>, + bias: FloatTensor>, + output_grad: FloatTensor>, + ) -> FloatTensor> { + make_ops!( + Conv2dBiasBackwardOps, + Conv2dBiasBackwardOpIr, + |desc: &Conv2dBiasBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let bias = handles.get_float_tensor::(&desc.bias); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = B::conv2d_bias_backward(x, bias, output_grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &bias, &output_grad]); + + let client = x.client.clone(); + let desc = Conv2dBiasBackwardOpIr::create( + x.into_ir(), + bias.into_ir(), + output_grad.into_ir(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv2dBiasBackward(desc.clone())), + Conv2dBiasBackwardOps::::new(desc), + ) + .output() + } + + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> FloatTensor { + make_ops!( + DeformConv2dOps, + DeformConv2dOpIr, + |args: &DeformConv2dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let offset = handles.get_float_tensor::(&args.offset); + let weight = handles.get_float_tensor::(&args.weight); + let mask = args + .mask + .as_ref() + .map(|mask| handles.get_float_tensor::(mask)); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = + B::deform_conv2d(x, offset, weight, mask, bias, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + let mut streams = OperationStreams::with_inputs([&x, &offset, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + if let Some(mask) = mask.as_ref() { + streams.tensor(mask) + } + + let client = x.client.clone(); + let desc = DeformConv2dOpIr::create( + x.into_ir(), + offset.into_ir(), + weight.into_ir(), + mask.map(|mask| mask.into_ir()), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::DeformableConv2d(Box::new(desc.clone()))), + DeformConv2dOps::::new(desc), + ) + .output() + } + + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + make_ops!( + DeformConv2dBackwardOps, + DeformConv2dBackwardOpIr, + |args: &DeformConv2dBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let offset = handles.get_float_tensor::(&args.offset); + let weight = handles.get_float_tensor::(&args.weight); + let mask = args + .mask + .as_ref() + .map(|mask| handles.get_float_tensor::(mask)); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + let output_grad = handles.get_float_tensor::(&args.out_grad); + + let output = B::deform_conv2d_backward( + x, + offset, + weight, + mask, + bias, + output_grad, + args.options.clone().into(), + ); + + handles.register_float_tensor::(&args.input_grad.id, output.x_grad); + handles.register_float_tensor::(&args.offset_grad.id, output.offset_grad); + handles.register_float_tensor::(&args.weight_grad.id, output.weight_grad); + if let Some((mask_grad, field)) = output.mask_grad.zip(args.mask_grad.as_ref()) { + handles.register_float_tensor::(&field.id, mask_grad); + } + if let Some((bias_grad, field)) = output.bias_grad.zip(args.bias_grad.as_ref()) { + handles.register_float_tensor::(&field.id, bias_grad); + } + } + ); + + let has_bias = bias.is_some(); + let has_mask = mask.is_some(); + + let mut streams = OperationStreams::with_inputs([&x, &offset, &weight, &output_grad]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias); + } + if let Some(mask) = mask.as_ref() { + streams.tensor(mask); + } + + let client = x.client.clone(); + let desc = DeformConv2dBackwardOpIr::create( + x.into_ir(), + offset.into_ir(), + weight.into_ir(), + mask.map(|mask| mask.into_ir()), + bias.map(|bias| bias.into_ir()), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + let mut outputs = client + .register( + streams, + OperationIr::Module(ModuleOperationIr::DeformableConv2dBackward(Box::new( + desc.clone(), + ))), + DeformConv2dBackwardOps::::new(desc), + ) + .into_iter(); + + // When the number of outputs is variable, the order is important + let input_grad = outputs.next().unwrap(); + let offset_grad = outputs.next().unwrap(); + let weight_grad = outputs.next().unwrap(); + let mask_grad = has_mask.then(|| outputs.next().unwrap()); + let bias_grad = has_bias.then(|| outputs.next().unwrap()); + + DeformConv2dBackward::new(input_grad, offset_grad, weight_grad, mask_grad, bias_grad) + } + + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> FloatTensor { + make_ops!(Conv3dOps, Conv3dOpIr, |args: &Conv3dOpIr, + handles: &mut HandleContainer< + B::Handle, + >| { + let x = handles.get_float_tensor::(&args.x); + let weight = handles.get_float_tensor::(&args.weight); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv3d(x, weight, bias, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + }); + + let mut streams = OperationStreams::with_inputs([&x, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + + let client = x.client.clone(); + let desc = Conv3dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv3d(desc.clone())), + Conv3dOps::::new(desc), + ) + .output() + } + + fn conv3d_x_backward( + x: FloatTensor>, + weight: FloatTensor>, + output_grad: FloatTensor>, + options: ConvOptions<3>, + ) -> FloatTensor> { + make_ops!( + Conv3dXBackwardOps, + Conv3dXBackwardOpIr, + |desc: &Conv3dXBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = + B::conv3d_x_backward(x, weight, output_grad, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &weight, &output_grad]); + + let client = x.client.clone(); + let desc = Conv3dXBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv3dXBackward(desc.clone())), + Conv3dXBackwardOps::::new(desc), + ) + .output() + } + + fn conv3d_weight_backward( + x: FloatTensor>, + weight: FloatTensor>, + output_grad: FloatTensor>, + options: ConvOptions<3>, + ) -> FloatTensor> { + make_ops!( + Conv3dWeightBackwardOps, + Conv3dWeightBackwardOpIr, + |desc: &Conv3dWeightBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = + B::conv3d_weight_backward(x, weight, output_grad, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &weight, &output_grad]); + + let client = x.client.clone(); + let desc = Conv3dWeightBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv3dWeightBackward(desc.clone())), + Conv3dWeightBackwardOps::::new(desc), + ) + .output() + } + + fn conv3d_bias_backward( + x: FloatTensor>, + bias: FloatTensor>, + output_grad: FloatTensor>, + ) -> FloatTensor> { + make_ops!( + Conv3dBiasBackwardOps, + Conv3dBiasBackwardOpIr, + |desc: &Conv3dBiasBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&desc.x); + let bias = handles.get_float_tensor::(&desc.bias); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + let output = B::conv3d_bias_backward(x, bias, output_grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &bias, &output_grad]); + + let client = x.client.clone(); + let desc = Conv3dBiasBackwardOpIr::create( + x.into_ir(), + bias.into_ir(), + output_grad.into_ir(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Conv3dBiasBackward(desc.clone())), + Conv3dBiasBackwardOps::::new(desc), + ) + .output() + } + + fn conv_transpose1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<1>, + ) -> FloatTensor { + make_ops!( + ConvTranspose1dOps, + ConvTranspose1dOpIr, + |args: &ConvTranspose1dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let weight = handles.get_float_tensor::(&args.weight); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv_transpose1d(x, weight, bias, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + let mut streams = OperationStreams::with_inputs([&x, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + + let client = x.client.clone(); + let desc = ConvTranspose1dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::ConvTranspose1d(desc.clone())), + ConvTranspose1dOps::::new(desc), + ) + .output() + } + + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + make_ops!( + ConvTranspose2dOps, + ConvTranspose2dOpIr, + |args: &ConvTranspose2dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let weight = handles.get_float_tensor::(&args.weight); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv_transpose2d(x, weight, bias, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + let mut streams = OperationStreams::with_inputs([&x, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + + let client = x.client.clone(); + let desc = ConvTranspose2dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::ConvTranspose2d(desc.clone())), + ConvTranspose2dOps::::new(desc), + ) + .output() + } + + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + make_ops!( + ConvTranspose3dOps, + ConvTranspose3dOpIr, + |args: &ConvTranspose3dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let weight = handles.get_float_tensor::(&args.weight); + let bias = args + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv_transpose3d(x, weight, bias, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + let mut streams = OperationStreams::with_inputs([&x, &weight]); + if let Some(bias) = bias.as_ref() { + streams.tensor(bias) + } + + let client = x.client.clone(); + let desc = ConvTranspose3dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::ConvTranspose3d(desc.clone())), + ConvTranspose3dOps::::new(desc), + ) + .output() + } + + fn avg_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + make_ops!( + AvgPool1dOps, + AvgPool1dOpIr, + |args: &AvgPool1dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::avg_pool1d( + x, + args.kernel_size, + args.stride, + args.padding, + args.count_include_pad, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = AvgPool1dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AvgPool1d(desc.clone())), + AvgPool1dOps::::new(desc), + ) + .output() + } + + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + make_ops!( + AvgPool2dOps, + AvgPool2dOpIr, + |args: &AvgPool2dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::avg_pool2d( + x, + args.kernel_size, + args.stride, + args.padding, + args.count_include_pad, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = AvgPool2dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AvgPool2d(desc.clone())), + AvgPool2dOps::::new(desc), + ) + .output() + } + + fn avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + make_ops!( + AvgPool1dBackwardOps, + AvgPool1dBackwardOpIr, + |args: &AvgPool1dBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let output = B::avg_pool1d_backward( + x, + grad, + args.kernel_size, + args.stride, + args.padding, + args.count_include_pad, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &grad]); + + let client = x.client.clone(); + let desc = AvgPool1dBackwardOpIr::create( + x.into_ir(), + grad.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AvgPool1dBackward(desc.clone())), + AvgPool1dBackwardOps::::new(desc), + ) + .output() + } + + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + make_ops!( + AvgPool2dBackwardOps, + AvgPool2dBackwardOpIr, + |args: &AvgPool2dBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let output = B::avg_pool2d_backward( + x, + grad, + args.kernel_size, + args.stride, + args.padding, + args.count_include_pad, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &grad]); + + let client = x.client.clone(); + let desc = AvgPool2dBackwardOpIr::create( + x.into_ir(), + grad.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AvgPool2dBackward(desc.clone())), + AvgPool2dBackwardOps::::new(desc), + ) + .output() + } + + fn max_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> FloatTensor { + make_ops!( + MaxPool1dOps, + MaxPool1dOpIr, + |args: &MaxPool1dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::max_pool1d( + x, + args.kernel_size, + args.stride, + args.padding, + args.dilation, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = MaxPool1dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::MaxPool1d(desc.clone())), + MaxPool1dOps::::new(desc), + ) + .output() + } + + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor { + make_ops!( + MaxPool2dOps, + MaxPool2dOpIr, + |args: &MaxPool2dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::max_pool2d( + x, + args.kernel_size, + args.stride, + args.padding, + args.dilation, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = MaxPool2dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::MaxPool2d(desc.clone())), + MaxPool2dOps::::new(desc), + ) + .output() + } + + fn max_pool1d_with_indices( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> MaxPool1dWithIndices { + make_ops!( + MaxPool1dWithIndicesOps, + MaxPool1dWithIndicesOpIr, + |args: &MaxPool1dWithIndicesOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::max_pool1d_with_indices( + x, + args.kernel_size, + args.stride, + args.padding, + args.dilation, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output.output); + handles.register_int_tensor::(&args.out_indices.id, output.indices); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = MaxPool1dWithIndicesOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + B::IntElem::dtype(), + || client.create_empty_handle(), + ); + + let [out, out_indices] = client + .register( + streams, + OperationIr::Module(ModuleOperationIr::MaxPool1dWithIndices(desc.clone())), + MaxPool1dWithIndicesOps::::new(desc), + ) + .outputs(); + + MaxPool1dWithIndices::new(out, out_indices) + } + + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices { + make_ops!( + MaxPool2dWithIndicesOps, + MaxPool2dWithIndicesOpIr, + |args: &MaxPool2dWithIndicesOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::max_pool2d_with_indices( + x, + args.kernel_size, + args.stride, + args.padding, + args.dilation, + args.ceil_mode, + ); + + handles.register_float_tensor::(&args.out.id, output.output); + handles.register_int_tensor::(&args.out_indices.id, output.indices); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = MaxPool2dWithIndicesOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + B::IntElem::dtype(), + || client.create_empty_handle(), + ); + + let [out, out_indices] = client + .register( + streams, + OperationIr::Module(ModuleOperationIr::MaxPool2dWithIndices(desc.clone())), + MaxPool2dWithIndicesOps::::new(desc), + ) + .outputs(); + + MaxPool2dWithIndices::new(out, out_indices) + } + + fn max_pool1d_with_indices_backward( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool1dBackward { + make_ops!( + MaxPool1dWithIndicesBackwardOps, + MaxPool1dWithIndicesBackwardOpIr, + |args: &MaxPool1dWithIndicesBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let indices = handles.get_int_tensor::(&args.indices); + let output = B::max_pool1d_with_indices_backward( + x, + args.kernel_size, + args.stride, + args.padding, + args.dilation, + args.ceil_mode, + grad, + indices, + ); + + handles.register_float_tensor::(&args.out.id, output.x_grad); + } + ); + + let streams = OperationStreams::with_inputs([&x, &output_grad, &indices]); + + let client = x.client.clone(); + let desc = MaxPool1dWithIndicesBackwardOpIr::create( + x.into_ir(), + output_grad.into_ir(), + indices.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + let out = client + .register( + streams, + OperationIr::Module(ModuleOperationIr::MaxPool1dWithIndicesBackward( + desc.clone(), + )), + MaxPool1dWithIndicesBackwardOps::::new(desc), + ) + .output(); + + MaxPool1dBackward::new(out) + } + + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool2dBackward { + make_ops!( + MaxPool2dWithIndicesBackwardOps, + MaxPool2dWithIndicesBackwardOpIr, + |args: &MaxPool2dWithIndicesBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let indices = handles.get_int_tensor::(&args.indices); + let output = B::max_pool2d_with_indices_backward( + x, + args.kernel_size, + args.stride, + args.padding, + args.dilation, + args.ceil_mode, + grad, + indices, + ); + + handles.register_float_tensor::(&args.out.id, output.x_grad); + } + ); + + let streams = OperationStreams::with_inputs([&x, &output_grad, &indices]); + + let client = x.client.clone(); + let desc = MaxPool2dWithIndicesBackwardOpIr::create( + x.into_ir(), + output_grad.into_ir(), + indices.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + let out = client + .register( + streams, + OperationIr::Module(ModuleOperationIr::MaxPool2dWithIndicesBackward( + desc.clone(), + )), + MaxPool2dWithIndicesBackwardOps::::new(desc), + ) + .output(); + + MaxPool2dBackward::new(out) + } + + fn adaptive_avg_pool1d(x: FloatTensor, output_size: usize) -> FloatTensor { + make_ops!( + AdaptiveAvgPool1dOps, + AdaptiveAvgPool1dOpIr, + |args: &AdaptiveAvgPool1dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::adaptive_avg_pool1d(x, args.output_size); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = AdaptiveAvgPool1dOpIr::create(x.into_ir(), output_size, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AdaptiveAvgPool1d(desc.clone())), + AdaptiveAvgPool1dOps::::new(desc), + ) + .output() + } + + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor { + make_ops!( + AdaptiveAvgPool2dOps, + AdaptiveAvgPool2dOpIr, + |args: &AdaptiveAvgPool2dOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::adaptive_avg_pool2d(x, args.output_size); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = AdaptiveAvgPool2dOpIr::create(x.into_ir(), output_size, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AdaptiveAvgPool2d(desc.clone())), + AdaptiveAvgPool2dOps::::new(desc), + ) + .output() + } + + fn adaptive_avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + make_ops!( + AdaptiveAvgPool1dBackwardOps, + AdaptiveAvgPool1dBackwardOpIr, + |args: &AdaptiveAvgPool1dBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let output = B::adaptive_avg_pool1d_backward(x, grad); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &grad]); + + let client = x.client.clone(); + let desc = AdaptiveAvgPool1dBackwardOpIr::create(x.into_ir(), grad.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AdaptiveAvgPool1dBackward(desc.clone())), + AdaptiveAvgPool1dBackwardOps::::new(desc), + ) + .output() + } + + fn adaptive_avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + make_ops!( + AdaptiveAvgPool2dBackwardOps, + AdaptiveAvgPool2dBackwardOpIr, + |args: &AdaptiveAvgPool2dBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let output = B::adaptive_avg_pool2d_backward(x, grad); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + let streams = OperationStreams::with_inputs([&x, &grad]); + + let client = x.client.clone(); + let desc = AdaptiveAvgPool2dBackwardOpIr::create(x.into_ir(), grad.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::AdaptiveAvgPool2dBackward(desc.clone())), + AdaptiveAvgPool2dBackwardOps::::new(desc), + ) + .output() + } + + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + make_ops!( + InterpolateOps, + InterpolateOpIr, + |args: &InterpolateOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let output = B::interpolate(x, args.output_size, args.options.clone().into()); + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x]); + + let client = x.client.clone(); + let desc = InterpolateOpIr::create(x.into_ir(), output_size, options.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Interpolate(desc.clone())), + InterpolateOps::::new(desc), + ) + .output() + } + + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + make_ops!( + InterpolateBackwardOps, + InterpolateBackwardOpIr, + |args: &InterpolateBackwardOpIr, handles: &mut HandleContainer| { + let x = handles.get_float_tensor::(&args.x); + let grad = handles.get_float_tensor::(&args.grad); + let output = + B::interpolate_backward(x, grad, args.output_size, args.options.clone().into()); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let streams = OperationStreams::with_inputs([&x, &grad]); + + let client = x.client.clone(); + let desc = InterpolateBackwardOpIr::create( + x.into_ir(), + grad.into_ir(), + output_size, + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::InterpolateBackward(desc.clone())), + InterpolateBackwardOps::::new(desc), + ) + .output() + } + + fn attention( + query: FloatTensor>, + key: FloatTensor>, + value: FloatTensor>, + mask: Option>>, + attn_bias: Option>>, + options: burn_backend::ops::AttentionModuleOptions, + ) -> FloatTensor> { + make_ops!( + AttentionOps, + AttentionOpIr, + |args: &AttentionOpIr, handles: &mut HandleContainer| { + let query = handles.get_float_tensor::(&args.query); + let key = handles.get_float_tensor::(&args.key); + let value = handles.get_float_tensor::(&args.value); + let mask = args.mask.as_ref().map(|m| handles.get_bool_tensor::(m)); + let attn_bias = args + .attn_bias + .as_ref() + .map(|ab| handles.get_float_tensor::(ab)); + + let output = B::attention( + query, + key, + value, + mask, + attn_bias, + args.options.clone().into(), + ); + + handles.register_float_tensor::(&args.out.id, output); + } + ); + + let mut streams = OperationStreams::with_inputs([&query, &key, &value]); + if let Some(mask) = &mask { + streams.tensor(mask); + } + if let Some(attn_bias) = &attn_bias { + streams.tensor(attn_bias); + } + + let client = query.client.clone(); + let desc = AttentionOpIr::create( + query.into_ir(), + key.into_ir(), + value.into_ir(), + mask.map(|m| m.into_ir()), + attn_bias.map(|ab| ab.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::Module(ModuleOperationIr::Attention(desc.clone())), + AttentionOps::::new(desc), + ) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/qtensor.rs new file mode 100644 index 0000000..fb9aec7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/qtensor.rs @@ -0,0 +1,501 @@ +use std::marker::PhantomData; + +use burn_backend::{ + DType, Element, ExecutionError, QTensorPrimitive, Shape, Slice, TensorData, TensorPrimitive, + ops::QTensorOps, + quantization::{QuantPropagation, QuantScheme, QuantizationParametersPrimitive}, + tensor::{Device, FloatTensor, IntTensor, QuantizedTensor}, +}; +use burn_ir::{ + BaseOperationIr, DequantizeOpIr, FlipOpIr, FloatOperationIr, GatherOpIr, HandleContainer, + InitOperationIr, MatmulOpIr, OperationIr, OperationOutput, PermuteOpIr, + QuantizationParametersIr, QuantizeOpIr, SelectOpIr, ShapeOpIr, SliceOpIr, SwapDimsOpIr, +}; + +use crate::{ + Fusion, FusionBackend, get_client, + stream::{OperationStreams, execution::Operation}, +}; + +use super::NoOp; + +impl QTensorOps for Fusion { + fn q_from_data(data: TensorData, device: &Device) -> QuantizedTensor { + let client = get_client::(device); + let dtype = data.dtype; + let tensor = B::q_from_data(data, device); + let shape = burn_backend::TensorMetadata::shape(&tensor); + + let handle = B::quantized_tensor_handle(tensor); + let desc = InitOperationIr::create(shape, dtype, || client.register_tensor_handle(handle)); + + client + .register( + OperationStreams::default(), + OperationIr::Init(desc), + NoOp::::new(), + ) + .output() + } + + fn quantize( + tensor: FloatTensor, + scheme: &QuantScheme, + qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + #[derive(new, Debug)] + struct QuantizeOp { + desc: QuantizeOpIr, + _b: PhantomData, + } + + impl Operation for QuantizeOp { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let scales = handles.get_float_tensor::(&self.desc.qparams.scales); + + let qparams = QuantizationParametersPrimitive { scales }; + let output = B::quantize(tensor, &self.desc.scheme, qparams); + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &qparams.scales]); + + let client = tensor.client.clone(); + let qparams = QuantizationParametersIr { + scales: qparams.scales.into_ir(), + }; + let desc = QuantizeOpIr::create(tensor.into_ir(), qparams, *scheme, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.tensor.dtype, FloatOperationIr::Quantize(desc.clone())), + QuantizeOp::::new(desc), + ) + .output() + } + + fn dequantize(tensor: QuantizedTensor) -> FloatTensor { + #[derive(new, Debug)] + struct DequantizeOp { + desc: DequantizeOpIr, + _b: PhantomData, + } + + impl Operation for DequantizeOp { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_quantized_tensor::(&self.desc.input); + + let output = B::dequantize(tensor); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let dtype = B::FloatElem::dtype(); + let desc = DequantizeOpIr::create(tensor.into_ir(), dtype, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(dtype, FloatOperationIr::Dequantize(desc.clone())), + DequantizeOp::::new(desc), + ) + .output() + } + + fn q_device(tensor: &QuantizedTensor) -> Device { + tensor.client.device().clone() + } + + fn q_to_device(tensor: QuantizedTensor, device: &Device) -> QuantizedTensor { + let device_original: &B::Device = tensor.client.device(); + let device_target: B::Device = device.clone(); + + if device_original == &device_target { + return tensor; + } + + let id = tensor.stream; + let client_target = get_client::(&device_target); + let client_original = tensor.client.clone(); + + client_original.change_client_quantized::(tensor.into_ir(), client_target, id) + } + + fn q_reshape(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + if tensor.shape == shape { + return tensor; + } + + #[derive(new, Debug)] + struct ReshapeDimsOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ReshapeDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_quantized_tensor::(&self.desc.input); + let output = B::q_reshape(input, self.desc.out.shape.clone()); + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Reshape(desc.clone())), + ReshapeDimsOps::::new(desc), + ) + .output() + } + + async fn q_into_data(tensor: QuantizedTensor) -> Result { + tensor.q_into_data::().await + } + + fn q_swap_dims( + tensor: QuantizedTensor, + dim1: usize, + dim2: usize, + ) -> QuantizedTensor { + #[derive(new, Debug)] + struct SwapDimsOps { + desc: SwapDimsOpIr, + _b: PhantomData, + } + + impl Operation for SwapDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_quantized_tensor::(&self.desc.input); + let output = B::q_swap_dims(input, self.desc.dim1, self.desc.dim2); + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::SwapDims(desc.clone())), + SwapDimsOps::::new(desc), + ) + .output() + } + + fn q_permute(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + #[derive(new, Debug)] + struct PermuteDimsOps { + desc: PermuteOpIr, + _b: PhantomData, + } + + impl Operation for PermuteDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_quantized_tensor::(&self.desc.input); + let output = B::q_permute(input, self.desc.axes.as_slice()); + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Permute(desc.clone())), + PermuteDimsOps::::new(desc), + ) + .output() + } + + fn q_flip(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + #[derive(new, Debug)] + struct FlipOps { + desc: FlipOpIr, + _b: PhantomData, + } + + impl Operation for FlipOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_quantized_tensor::(&self.desc.input); + let output = B::q_flip(input, &self.desc.axes); + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Flip(desc.clone())), + FlipOps::::new(desc), + ) + .output() + } + + fn q_gather( + dim: usize, + tensor: QuantizedTensor, + indices: IntTensor, + ) -> QuantizedTensor { + #[derive(new, Debug)] + struct GatherOps { + desc: GatherOpIr, + _b: PhantomData, + } + + impl Operation for GatherOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_quantized_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::q_gather(self.desc.dim, tensor, indices); + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Gather(desc.clone())), + GatherOps::::new(desc), + ) + .output() + } + + fn q_select( + tensor: QuantizedTensor, + dim: usize, + indices: IntTensor, + ) -> QuantizedTensor { + #[derive(new, Debug)] + struct SelectOps { + desc: SelectOpIr, + _b: PhantomData, + } + + impl Operation for SelectOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_quantized_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::q_select(tensor, self.desc.dim, indices); + + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SelectOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Select(desc.clone())), + SelectOps::::new(desc), + ) + .output() + } + + fn q_slice(tensor: QuantizedTensor, slices: &[Slice]) -> QuantizedTensor { + #[derive(new, Debug)] + struct SliceOps { + desc: SliceOpIr, + _b: PhantomData, + } + + impl Operation for SliceOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_quantized_tensor::(&self.desc.tensor); + + let output = B::q_slice(tensor, self.desc.ranges.as_slice()); + + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Slice(desc.clone())), + SliceOps::::new(desc), + ) + .output() + } + + fn q_expand(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + #[derive(new, Debug)] + struct ExpandOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ExpandOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_quantized_tensor::(&self.desc.input); + let output = B::q_expand(input, self.desc.out.shape.clone()); + + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Expand(desc.clone())), + ExpandOps::::new(desc), + ) + .output() + } + + fn q_matmul(lhs: TensorPrimitive, rhs: TensorPrimitive) -> TensorPrimitive { + #[derive(new, Debug)] + struct MatmulOps { + desc: MatmulOpIr, + lhs_quantized: bool, + rhs_quantized: bool, + _b: PhantomData, + } + + impl Operation for MatmulOps { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = match self.lhs_quantized { + true => { + TensorPrimitive::QFloat(handles.get_quantized_tensor::(&self.desc.lhs)) + } + false => TensorPrimitive::Float(handles.get_float_tensor::(&self.desc.lhs)), + }; + let rhs = match self.rhs_quantized { + true => { + TensorPrimitive::QFloat(handles.get_quantized_tensor::(&self.desc.rhs)) + } + false => TensorPrimitive::Float(handles.get_float_tensor::(&self.desc.rhs)), + }; + let output = B::q_matmul(lhs, rhs); + match output { + TensorPrimitive::Float(output) => { + handles.register_float_tensor::(&self.desc.out.id, output); + } + TensorPrimitive::QFloat(output) => { + handles.register_quantized_tensor::(&self.desc.out.id, output); + } + } + } + } + + let mut propagation = QuantPropagation::Inhibit; + let mut scheme = QuantScheme::default(); + let mut streams = OperationStreams::default(); + let mut lhs_quantized = false; + let mut rhs_quantized = false; + match &lhs { + TensorPrimitive::QFloat(lhs) => { + propagation = lhs.propagation(); + scheme = *lhs.scheme(); + lhs_quantized = true; + streams.tensor(lhs); + } + TensorPrimitive::Float(lhs) => { + streams.tensor(lhs); + } + } + match &rhs { + TensorPrimitive::QFloat(rhs) => { + propagation = rhs.propagation(); + scheme = *rhs.scheme(); + rhs_quantized = true; + streams.tensor(rhs); + } + TensorPrimitive::Float(rhs) => { + streams.tensor(rhs); + } + } + + let dtype = match propagation { + QuantPropagation::Propagate => DType::QFloat(scheme), + QuantPropagation::Inhibit => B::FloatElem::dtype(), + }; + + let client = match &lhs { + TensorPrimitive::Float(lhs) => lhs.client.clone(), + TensorPrimitive::QFloat(lhs) => lhs.client.clone(), + }; + + let lhs = match lhs { + TensorPrimitive::Float(lhs) => lhs.into_ir(), + TensorPrimitive::QFloat(lhs) => lhs.into_ir(), + }; + let rhs = match rhs { + TensorPrimitive::Float(rhs) => rhs.into_ir(), + TensorPrimitive::QFloat(rhs) => rhs.into_ir(), + }; + + let desc = MatmulOpIr::create_mixed(lhs, rhs, dtype, || client.create_empty_handle()); + + let out = client + .register( + streams, + OperationIr::Float(dtype, FloatOperationIr::Matmul(desc.clone())), + MatmulOps::::new(desc, lhs_quantized, rhs_quantized), + ) + .output(); + + match propagation { + QuantPropagation::Propagate => TensorPrimitive::QFloat(out), + QuantPropagation::Inhibit => TensorPrimitive::Float(out), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/tensor.rs new file mode 100644 index 0000000..4c998e0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/tensor.rs @@ -0,0 +1,2403 @@ +use super::NoOp; +use crate::{ + Fusion, FusionBackend, binary_float_cmp_ops, binary_float_ops, get_client, reduce_float_ops, + reduce_float2int_ops, scalar_float_cmp_ops, scalar_float_ops, + stream::{OperationStreams, execution::Operation}, + unary_float_ops, +}; +use burn_backend::{ + Distribution, Element, ExecutionError, FloatDType, Scalar, Shape, Slice, TensorData, + ops::{FloatTensorOps, GridSampleOptions}, + tensor::{BoolTensor, Device, FloatElem, FloatTensor, IndexingUpdateOp, IntTensor}, +}; +use burn_ir::*; +use std::marker::PhantomData; + +impl FloatTensorOps for Fusion { + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(data), + fields(?data.shape, ?data.dtype) + ))] + fn float_from_data(data: TensorData, device: &Device) -> FloatTensor { + let client = get_client::(device); + let dtype = data.dtype; + let tensor = B::float_from_data(data, device); + let shape = burn_backend::TensorMetadata::shape(&tensor); + + let handle = B::float_tensor_handle(tensor); + let desc = InitOperationIr::create(shape, dtype, || client.register_tensor_handle(handle)); + + client + .register( + OperationStreams::default(), + OperationIr::Init(desc), + NoOp::::new(), + ) + .output() + } + + fn float_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> FloatTensor { + #[derive(new, Debug)] + struct RandomOps { + desc: RandomOpIr, + device: Device, + } + + impl Operation for RandomOps { + fn execute(&self, handles: &mut HandleContainer) { + let output: B::FloatTensorPrimitive = B::float_random( + self.desc.out.shape.clone(), + self.desc.distribution, + &self.device, + ); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let dtype = FloatElem::::dtype(); + let client = get_client::(device); + let desc = RandomOpIr::create(shape, dtype, distribution, || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::Float(dtype, FloatOperationIr::Random(desc.clone())), + RandomOps::::new(desc, device.clone()), + ) + .output() + } + + fn float_zeros(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + #[derive(new, Debug)] + struct ZerosOps { + out: TensorIr, + device: Device, + } + + impl Operation for ZerosOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.out.shape.clone(); + let output = B::float_zeros(shape, &self.device, self.out.dtype.into()); + handles.register_float_tensor::(&self.out.id, output); + } + } + + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseFloat(BaseOperationIr::Zeros(desc.clone())), + ZerosOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn float_ones(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + #[derive(new, Debug)] + struct OnesOps { + out: TensorIr, + device: Device, + } + + impl Operation for OnesOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.out.shape.clone(); + let output = B::float_ones(shape, &self.device, self.out.dtype.into()); + handles.register_float_tensor::(&self.out.id, output); + } + } + + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseFloat(BaseOperationIr::Ones(desc.clone())), + OnesOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn float_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: FloatDType, + ) -> FloatTensor { + #[derive(new, Debug)] + struct FullOps { + out: TensorIr, + elem: ScalarIr, + device: Device, + } + + impl Operation for FullOps { + fn execute(&self, handles: &mut HandleContainer) { + let shape = self.out.shape.clone(); + let dtype = self.out.dtype.into(); + let output: B::FloatTensorPrimitive = + B::float_full(shape, self.elem.into(), &self.device, dtype); + handles.register_float_tensor::(&self.out.id, output); + } + } + + let dtype = dtype.into(); + let client = get_client::(device); + let value = fill_value.into(); + let desc = FullOpIr::create(shape, dtype, value, || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::NumericFloat(dtype, NumericOperationIr::Full(desc.clone())), + FullOps::::new(desc.out, desc.value, device.clone()), + ) + .output() + } + + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensor), + fields( + from = ?tensor.client.device(), + shape = ?tensor.shape, + dtype = ?tensor.dtype + ) + ))] + async fn float_into_data(tensor: FloatTensor) -> Result { + tensor.into_data::().await + } + + fn float_device(tensor: &FloatTensor) -> Device { + tensor.client.device().clone() + } + + #[cfg_attr(feature = "tracing", tracing::instrument( + level="trace", + skip(tensor), + fields( + from = ?tensor.client.device(), + shape = ?tensor.shape, + dtype = ?tensor.dtype, + ) + ))] + fn float_to_device(tensor: FloatTensor, device: &Device) -> FloatTensor { + let device_original: &B::Device = tensor.client.device(); + + if device_original == device { + return tensor; + } + + let id = tensor.stream; + let client_target = get_client::(device); + let client_original = tensor.client.clone(); + + client_original + .clone() + .change_client_float::(tensor.into_ir(), client_target, id) + } + + fn float_into_int(tensor: FloatTensor) -> IntTensor { + #[derive(new, Debug)] + struct IntoIntOps { + desc: CastOpIr, + _b: PhantomData, + } + + impl Operation for IntoIntOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_into_int(input); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), B::IntElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.input.dtype, FloatOperationIr::IntoInt(desc.clone())), + IntoIntOps::::new(desc), + ) + .output() + } + + fn float_empty(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + #[derive(new, Debug)] + struct EmptyOps { + desc: TensorIr, + device: Device, + } + + impl Operation for EmptyOps { + fn execute(&self, handles: &mut HandleContainer) { + let output = B::float_empty( + self.desc.shape.clone(), + &self.device, + self.desc.dtype.into(), + ); + handles.register_float_tensor::(&self.desc.id, output); + } + } + + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register( + OperationStreams::default(), + OperationIr::BaseFloat(BaseOperationIr::Empty(desc.clone())), + EmptyOps::::new(desc.out, device.clone()), + ) + .output() + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(AddOps, B::float_add); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Add(desc.clone())), + AddOps::::new(desc), + ) + .output() + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + scalar_float_ops!(AddOps, B::float_add_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::AddScalar(desc.clone()), + ), + AddOps::::new(desc), + ) + .output() + } + + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + #[derive(new, Debug)] + struct ClampOps { + desc: ClampOpIr, + _b: PhantomData, + } + + impl Operation for ClampOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.tensor); + let output = B::float_clamp(input, self.desc.min.into(), self.desc.max.into()); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let min = min.into(); + let max = max.into(); + let desc = ClampOpIr::create(tensor.into_ir(), min, max, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.tensor.dtype, + NumericOperationIr::Clamp(desc.clone()), + ), + ClampOps::::new(desc), + ) + .output() + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(SubOps, B::float_sub); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Sub(desc.clone())), + SubOps::::new(desc), + ) + .output() + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + scalar_float_ops!(SubOps, B::float_sub_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::SubScalar(desc.clone()), + ), + SubOps::::new(desc), + ) + .output() + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(MulOps, B::float_mul); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Mul(desc.clone())), + MulOps::::new(desc), + ) + .output() + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + scalar_float_ops!(MulOps, B::float_mul_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MulScalar(desc.clone()), + ), + MulOps::::new(desc), + ) + .output() + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(DivOps, B::float_div); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Div(desc.clone())), + DivOps::::new(desc), + ) + .output() + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + scalar_float_ops!(DivOps, B::float_div_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::DivScalar(desc.clone()), + ), + DivOps::::new(desc), + ) + .output() + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(ModOps, B::float_remainder); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Rem(desc.clone())), + ModOps::::new(desc), + ) + .output() + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + scalar_float_ops!(ModOps, B::float_remainder_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::RemScalar(desc.clone()), + ), + ModOps::::new(desc), + ) + .output() + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(MatmulOps, B::float_matmul); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = MatmulOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Matmul(desc.clone())), + MatmulOps::::new(desc.into()), + ) + .output() + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + #[derive(new, Debug)] + struct CrossOps { + desc: CrossOpIr, + _b: PhantomData, + } + + impl Operation for CrossOps { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let rhs = handles.get_float_tensor::(&self.desc.rhs); + let output = B::float_cross(lhs, rhs, self.desc.dim); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = CrossOpIr::create(lhs.into_ir(), rhs.into_ir(), dim, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Cross(desc.clone())), + CrossOps::::new(desc), + ) + .output() + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + #[derive(new, Debug)] + struct SwapDimsOps { + desc: SwapDimsOpIr, + _b: PhantomData, + } + + impl Operation for SwapDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_swap_dims(input, self.desc.dim1, self.desc.dim2); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::SwapDims(desc.clone())), + SwapDimsOps::::new(desc), + ) + .output() + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + if tensor.shape == shape { + return tensor; + } + + #[derive(new, Debug)] + struct ReshapeDimsOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ReshapeDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_reshape(input, self.desc.out.shape.clone()); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Reshape(desc.clone())), + ReshapeDimsOps::::new(desc), + ) + .output() + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + #[derive(new, Debug)] + struct GatherOps { + desc: GatherOpIr, + _b: PhantomData, + } + + impl Operation for GatherOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::float_gather(self.desc.dim, tensor, indices); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices]); + + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Gather(desc.clone())), + GatherOps::::new(desc), + ) + .output() + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + #[derive(new, Debug)] + struct ScatterOps { + desc: ScatterOpIr, + _b: PhantomData, + } + + impl Operation for ScatterOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + let value = handles.get_float_tensor::(&self.desc.value); + + let output = B::float_scatter_add(self.desc.dim, tensor, indices, value); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices, &value]); + + let client = tensor.client.clone(); + let desc = ScatterOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Scatter(desc.clone())), + ScatterOps::::new(desc), + ) + .output() + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + ) -> FloatTensor { + #[derive(new, Debug)] + struct SelectOps { + desc: SelectOpIr, + _b: PhantomData, + } + + impl Operation for SelectOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + + let output = B::float_select(tensor, self.desc.dim, indices); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices]); + + let client = tensor.client.clone(); + let desc = SelectOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Select(desc.clone())), + SelectOps::::new(desc), + ) + .output() + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + #[derive(new, Debug)] + struct SelectAssignOps { + desc: SelectAssignOpIr, + _b: PhantomData, + } + + impl Operation for SelectAssignOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let indices = handles.get_int_tensor::(&self.desc.indices); + let value = handles.get_float_tensor::(&self.desc.value); + + let output = B::float_select_add(tensor, self.desc.dim, indices, value); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &indices, &value]); + + let client = tensor.client.clone(); + let desc = SelectAssignOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::SelectAssign(desc.clone())), + SelectAssignOps::::new(desc), + ) + .output() + } + + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor { + #[derive(new, Debug)] + struct SliceOps { + desc: SliceOpIr, + _b: PhantomData, + } + + impl Operation for SliceOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + + let output = B::float_slice(tensor, self.desc.ranges.as_slice()); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Slice(desc.clone())), + SliceOps::::new(desc), + ) + .output() + } + + fn float_slice_assign( + tensor: FloatTensor, + slices: &[burn_backend::Slice], + value: FloatTensor, + ) -> FloatTensor { + #[derive(new, Debug)] + struct SliceAssignOps { + desc: SliceAssignOpIr, + _b: PhantomData, + } + + impl Operation for SliceAssignOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let value = handles.get_float_tensor::(&self.desc.value); + + let output = B::float_slice_assign(tensor, self.desc.ranges.as_slice(), value); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &value]); + + let client = tensor.client.clone(); + let desc = + SliceAssignOpIr::create(tensor.into_ir(), slices.into(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::SliceAssign(desc.clone())), + SliceAssignOps::::new(desc), + ) + .output() + } + + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + value: FloatTensor, + ) -> FloatTensor { + #[derive(new, Debug)] + struct MaskWhereOps { + desc: MaskWhereOpIr, + _b: PhantomData, + } + + impl Operation for MaskWhereOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let value = handles.get_float_tensor::(&self.desc.value); + let mask = handles.get_bool_tensor::(&self.desc.mask); + + let output = B::float_mask_where(tensor, mask, value); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &mask, &value]); + + let client = tensor.client.clone(); + let desc = MaskWhereOpIr::create(tensor.into_ir(), mask.into_ir(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::MaskWhere(desc.clone())), + MaskWhereOps::::new(desc), + ) + .output() + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor { + #[derive(new, Debug)] + struct MaskFillOps { + desc: MaskFillOpIr, + _b: PhantomData, + } + + impl Operation for MaskFillOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let mask = handles.get_bool_tensor::(&self.desc.mask); + + let output = B::float_mask_fill(tensor, mask, self.desc.value.into()); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &mask]); + + let client = tensor.client.clone(); + let value = value.into(); + let desc = MaskFillOpIr::create(tensor.into_ir(), mask.into_ir(), value, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::MaskFill(desc.clone())), + MaskFillOps::::new(desc), + ) + .output() + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float_cmp_ops!(EqualOps, B::float_equal); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Equal(desc.clone())), + EqualOps::::new(desc), + ) + .output() + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + scalar_float_cmp_ops!(EqualElemOps, B::float_equal_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::EqualElem(desc.clone())), + EqualElemOps::::new(desc), + ) + .output() + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float_cmp_ops!(GreaterOps, B::float_greater); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::Greater(desc.clone()), + ), + GreaterOps::::new(desc), + ) + .output() + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + scalar_float_cmp_ops!(GreaterElemOps, B::float_greater_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::GreaterElem(desc.clone()), + ), + GreaterElemOps::::new(desc), + ) + .output() + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float_cmp_ops!(GreaterEqualOps, B::float_greater_equal); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::GreaterEqual(desc.clone()), + ), + GreaterEqualOps::::new(desc), + ) + .output() + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + scalar_float_cmp_ops!(GreaterEqualElemOps, B::float_greater_equal_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::GreaterEqualElem(desc.clone()), + ), + GreaterEqualElemOps::::new(desc), + ) + .output() + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float_cmp_ops!(LowerOps, B::float_lower); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericFloat(desc.lhs.dtype, NumericOperationIr::Lower(desc.clone())), + LowerOps::::new(desc), + ) + .output() + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + scalar_float_cmp_ops!(LowerElemOps, B::float_lower_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::LowerElem(desc.clone()), + ), + LowerElemOps::::new(desc), + ) + .output() + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + binary_float_cmp_ops!(LowerEqualOps, B::float_lower_equal); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + B::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::LowerEqual(desc.clone()), + ), + LowerEqualOps::::new(desc), + ) + .output() + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + scalar_float_cmp_ops!(LowerEqualElemOps, B::float_lower_equal_elem); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::LowerEqualElem(desc.clone()), + ), + LowerEqualElemOps::::new(desc), + ) + .output() + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(SumOps, B::float_sum, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Sum(desc.clone())), + SumOps::::new(desc.into()), + ) + .output() + } + + fn float_sum_dim(tensor: FloatTensor, axis: usize) -> FloatTensor { + reduce_float_ops!(SumDimOps, B::float_sum_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), axis, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::SumDim(desc.clone())), + SumDimOps::::new(desc), + ) + .output() + } + + fn float_prod(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ProdOps, B::float_prod, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Prod(desc.clone())), + ProdOps::::new(desc.into()), + ) + .output() + } + + fn float_prod_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce_float_ops!(ProdDimOps, B::float_prod_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::ProdDim(desc.clone()), + ), + ProdDimOps::::new(desc), + ) + .output() + } + + fn float_mean(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(MeanOps, B::float_mean, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Mean(desc.clone())), + MeanOps::::new(desc.into()), + ) + .output() + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce_float_ops!(MeanDimOps, B::float_mean_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MeanDim(desc.clone()), + ), + MeanDimOps::::new(desc), + ) + .output() + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(new, Debug)] + struct CumsumOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CumsumOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_cumsum(input, self.desc.axis); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::CumSum(desc.clone())), + CumsumOps::::new(desc), + ) + .output() + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(new, Debug)] + struct CumprodOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CumprodOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_cumprod(input, self.desc.axis); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::CumProd(desc.clone()), + ), + CumprodOps::::new(desc), + ) + .output() + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(new, Debug)] + struct CumminOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CumminOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_cummin(input, self.desc.axis); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::CumMin(desc.clone())), + CumminOps::::new(desc), + ) + .output() + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + #[derive(new, Debug)] + struct CummaxOps { + desc: DimOpIr, + _b: PhantomData, + } + + impl Operation for CummaxOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_cummax(input, self.desc.axis); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::CumMax(desc.clone())), + CummaxOps::::new(desc), + ) + .output() + } + + fn float_exp(lhs: FloatTensor) -> FloatTensor { + unary_float_ops!(ExpOps, B::float_exp); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let desc = UnaryOpIr::create(lhs.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Exp(desc.clone())), + ExpOps::::new(desc), + ) + .output() + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(LogOps, B::float_log); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Log(desc.clone())), + LogOps::::new(desc), + ) + .output() + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(Log1pOps, B::float_log1p); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Log1p(desc.clone())), + Log1pOps::::new(desc), + ) + .output() + } + + fn float_powf_scalar_impl(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + scalar_float_ops!(PowfOps, B::float_powf_scalar); + + let streams = OperationStreams::with_inputs([&lhs]); + + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::PowfScalar(desc.clone())), + PowfOps::::new(desc), + ) + .output() + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(SqrtOps, B::float_sqrt); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Sqrt(desc.clone())), + SqrtOps::::new(desc), + ) + .output() + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(AbsOps, B::float_abs); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Abs(desc.clone())), + AbsOps::::new(desc), + ) + .output() + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(CosOps, B::float_cos); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Cos(desc.clone())), + CosOps::::new(desc), + ) + .output() + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(SinOps, B::float_sin); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Sin(desc.clone())), + SinOps::::new(desc), + ) + .output() + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(TanOps, B::float_tan); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Tan(desc.clone())), + TanOps::::new(desc), + ) + .output() + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(CoshOps, B::float_cosh); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Cosh(desc.clone())), + CoshOps::::new(desc), + ) + .output() + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(SinhOps, B::float_sinh); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Sinh(desc.clone())), + SinhOps::::new(desc), + ) + .output() + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(TanhOps, B::float_tanh); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Tanh(desc.clone())), + TanhOps::::new(desc), + ) + .output() + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ArcCosOps, B::float_acos); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcCos(desc.clone())), + ArcCosOps::::new(desc), + ) + .output() + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ArcCoshOps, B::float_acosh); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcCosh(desc.clone())), + ArcCoshOps::::new(desc), + ) + .output() + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ArcSinOps, B::float_asin); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcSin(desc.clone())), + ArcSinOps::::new(desc), + ) + .output() + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ArcSinhOps, B::float_asinh); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcSinh(desc.clone())), + ArcSinhOps::::new(desc), + ) + .output() + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ArcTanOps, B::float_atan); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcTan(desc.clone())), + ArcTanOps::::new(desc), + ) + .output() + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(ArcTanhOps, B::float_atanh); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcTanh(desc.clone())), + ArcTanhOps::::new(desc), + ) + .output() + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(ArcTan2Ops, B::float_atan2); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::ArcTan2(desc.clone())), + ArcTan2Ops::::new(desc), + ) + .output() + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(Recip, B::float_recip); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Recip(desc.clone())), + Recip::::new(desc), + ) + .output() + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(TanhOps, B::float_erf); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Erf(desc.clone())), + TanhOps::::new(desc), + ) + .output() + } + + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + #[derive(new, Debug)] + struct CatOps { + desc: CatOpIr, + _b: PhantomData, + } + + impl Operation for CatOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensors = self + .desc + .tensors + .iter() + .map(|tensor| handles.get_float_tensor::(tensor)) + .collect(); + + let output = B::float_cat(tensors, self.desc.dim); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs(&tensors); + + let client = tensors.first().unwrap().client.clone(); + let tensors = tensors.into_iter().map(|t| t.into_ir()).collect(); + let desc = CatOpIr::create(tensors, dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Cat(desc.clone())), + CatOps::::new(desc), + ) + .output() + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor { + reduce_float2int_ops!(ArgMaxOps, B::float_argmax); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + // TODO: rename `create_with_dtype` specifically for ARG / indices + let desc = ReduceDimOpIr::create_arg(tensor.into_ir(), dim, B::IntElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.input.dtype, + NumericOperationIr::ArgMax(desc.clone()), + ), + ArgMaxOps::::new(desc), + ) + .output() + } + + fn float_repeat_dim(tensor: FloatTensor, dim: usize, times: usize) -> FloatTensor { + #[derive(new, Debug)] + struct RepeatDimOps { + desc: RepeatDimOpIr, + _b: PhantomData, + } + + impl Operation for RepeatDimOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + + let output = B::float_repeat_dim(tensor, self.desc.dim, self.desc.times); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = RepeatDimOpIr::create(tensor.into_ir(), dim, times, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::RepeatDim(desc.clone())), + RepeatDimOps::::new(desc), + ) + .output() + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor { + reduce_float2int_ops!(ArgMinOps, B::float_argmin); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create_arg(tensor.into_ir(), dim, B::IntElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.input.dtype, + NumericOperationIr::ArgMin(desc.clone()), + ), + ArgMinOps::::new(desc), + ) + .output() + } + + fn float_max(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(MaxOps, B::float_max, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Max(desc.clone())), + MaxOps::::new(desc.into()), + ) + .output() + } + + fn float_max_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce_float_ops!(MaxDimOps, B::float_max_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::MaxDim(desc.clone())), + MaxDimOps::::new(desc), + ) + .output() + } + + fn float_max_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + #[derive(new, Debug)] + struct MaxDimWithIndicesOps { + desc: ReduceDimWithIndicesOpIr, + _b: PhantomData, + } + + impl Operation for MaxDimWithIndicesOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let (output, indices) = B::float_max_dim_with_indices(tensor, self.desc.dim); + + handles.register_float_tensor::(&self.desc.out.id, output); + handles.register_int_tensor::(&self.desc.out_indices.id, indices); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = + ReduceDimWithIndicesOpIr::create(tensor.into_ir(), dim, B::IntElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.tensor.dtype, + NumericOperationIr::MaxDimWithIndices(desc.clone()), + ), + MaxDimWithIndicesOps::::new(desc), + ) + .outputs() + .into() + } + + fn float_min(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(MinOps, B::float_min, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Min(desc.clone())), + MinOps::::new(desc.into()), + ) + .output() + } + + fn float_min_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce_float_ops!(MinDimOps, B::float_min_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::MinDim(desc.clone())), + MinDimOps::::new(desc), + ) + .output() + } + + fn float_min_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + #[derive(new, Debug)] + struct MinDimWithIndicesOps { + desc: ReduceDimWithIndicesOpIr, + _b: PhantomData, + } + + impl Operation for MinDimWithIndicesOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let (output, indices) = B::float_min_dim_with_indices(tensor, self.desc.dim); + + handles.register_float_tensor::(&self.desc.out.id, output); + handles.register_int_tensor::(&self.desc.out_indices.id, indices); + } + } + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = + ReduceDimWithIndicesOpIr::create(tensor.into_ir(), dim, B::IntElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.tensor.dtype, + NumericOperationIr::MinDimWithIndices(desc.clone()), + ), + MinDimWithIndicesOps::::new(desc), + ) + .outputs() + .into() + } + + fn float_max_abs(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(MaxAbsOps, B::float_max_abs, reduce); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::MaxAbs(desc.clone())), + MaxAbsOps::::new(desc.into()), + ) + .output() + } + + fn float_max_abs_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + reduce_float_ops!(MaxAbsDimOps, B::float_max_abs_dim); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MaxAbsDim(desc.clone()), + ), + MaxAbsDimOps::::new(desc), + ) + .output() + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + binary_float_ops!(PowOps, B::float_powf); + + let streams = OperationStreams::with_inputs([&lhs, &rhs]); + + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::NumericFloat(desc.out.dtype, NumericOperationIr::Powf(desc.clone())), + PowOps::::new(desc), + ) + .output() + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + #[derive(new, Debug)] + struct PermuteDimsOps { + desc: PermuteOpIr, + _b: PhantomData, + } + + impl Operation for PermuteDimsOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_permute(input, self.desc.axes.as_slice()); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Permute(desc.clone())), + PermuteDimsOps::::new(desc), + ) + .output() + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + #[derive(new, Debug)] + struct ExpandOps { + desc: ShapeOpIr, + _b: PhantomData, + } + + impl Operation for ExpandOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_expand(input, self.desc.out.shape.clone()); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Expand(desc.clone())), + ExpandOps::::new(desc), + ) + .output() + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + #[derive(new, Debug)] + struct FlipOps { + desc: FlipOpIr, + _b: PhantomData, + } + + impl Operation for FlipOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_flip(input, &self.desc.axes); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseInt(BaseOperationIr::Flip(desc.clone())), + FlipOps::::new(desc), + ) + .output() + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(RoundOps, B::float_round); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Round(desc.clone())), + RoundOps::::new(desc), + ) + .output() + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(FloorOps, B::float_floor); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Floor(desc.clone())), + FloorOps::::new(desc), + ) + .output() + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(CeilOps, B::float_ceil); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Ceil(desc.clone())), + CeilOps::::new(desc), + ) + .output() + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + unary_float_ops!(TruncOps, B::float_trunc); + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::Trunc(desc.clone())), + TruncOps::::new(desc), + ) + .output() + } + + fn float_cast(tensor: FloatTensor, dtype: burn_backend::FloatDType) -> FloatTensor { + #[derive(new, Debug)] + struct CastOps { + desc: CastOpIr, + dtype: burn_backend::FloatDType, + _b: PhantomData, + } + + impl Operation for CastOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output: B::FloatTensorPrimitive = B::float_cast(input, self.dtype); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), dtype.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Cast(desc.clone())), + CastOps::::new(desc, dtype), + ) + .output() + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + #[derive(new, Debug)] + struct UnfoldOps { + desc: UnfoldOpIr, + _b: PhantomData, + } + + impl Operation for UnfoldOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_unfold(input, self.desc.dim, self.desc.size, self.desc.step); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnfoldOpIr::create(tensor.into_ir(), dim, size, step, || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::BaseFloat(BaseOperationIr::Unfold(desc.clone())), + UnfoldOps::::new(desc), + ) + .output() + } + + fn float_is_nan(tensor: FloatTensor) -> BoolTensor { + #[derive(new, Debug)] + struct IsNanOps { + desc: UnaryOpIr, + _b: PhantomData, + } + impl Operation for IsNanOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_is_nan(input); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create_comparison(tensor.into_ir(), B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.input.dtype, FloatOperationIr::IsNan(desc.clone())), + IsNanOps::::new(desc), + ) + .output() + } + + fn float_is_inf(tensor: FloatTensor) -> BoolTensor { + #[derive(new, Debug)] + struct IsInfOps { + desc: UnaryOpIr, + _b: PhantomData, + } + impl Operation for IsInfOps { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = B::float_is_inf(input); + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor]); + + let client = tensor.client.clone(); + let desc = UnaryOpIr::create_comparison(tensor.into_ir(), B::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.input.dtype, FloatOperationIr::IsInf(desc.clone())), + IsInfOps::::new(desc), + ) + .output() + } + + fn float_grid_sample_2d( + tensor: FloatTensor, + grid: FloatTensor, + options: GridSampleOptions, + ) -> FloatTensor { + #[derive(new, Debug)] + struct GridSample2dOps { + desc: GridSample2dOpIr, + _b: PhantomData, + } + + impl Operation for GridSample2dOps { + fn execute(&self, handles: &mut HandleContainer) { + let tensor = handles.get_float_tensor::(&self.desc.tensor); + let grid = handles.get_float_tensor::(&self.desc.grid); + let output = + B::float_grid_sample_2d(tensor, grid, self.desc.options.clone().into()); + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + + let streams = OperationStreams::with_inputs([&tensor, &grid]); + + let client = tensor.client.clone(); + let desc = + GridSample2dOpIr::create(tensor.into_ir(), grid.into_ir(), options.into(), || { + client.create_empty_handle() + }); + + client + .register( + streams, + OperationIr::Float(desc.out.dtype, FloatOperationIr::GridSample2d(desc.clone())), + GridSample2dOps::::new(desc), + ) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/transaction.rs new file mode 100644 index 0000000..186a1d8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/transaction.rs @@ -0,0 +1,36 @@ +use burn_backend::{ + backend::ExecutionError, + ops::{TransactionOps, TransactionPrimitive}, +}; + +use crate::{Fusion, FusionBackend}; + +impl TransactionOps> for Fusion { + async fn tr_execute( + transaction: TransactionPrimitive, + ) -> Result { + B::tr_execute(TransactionPrimitive::new( + transaction + .read_floats + .into_iter() + .map(|t| t.client.clone().resolve_tensor_float::(t)) + .collect(), + transaction + .read_qfloats + .into_iter() + .map(|_t| todo!("Quantization not supported yet")) + .collect(), + transaction + .read_ints + .into_iter() + .map(|t| t.client.clone().resolve_tensor_int::(t)) + .collect(), + transaction + .read_bools + .into_iter() + .map(|t| t.client.clone().resolve_tensor_bool::(t)) + .collect(), + )) + .await + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/unary.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/unary.rs new file mode 100644 index 0000000..b5017ea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/ops/unary.rs @@ -0,0 +1,319 @@ +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs.into()); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + }; + ( + $name:ident, + $ops:expr, + noconvert + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! reduce_float_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ReduceDimOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = $ops(input, self.desc.axis); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! reduce_float2int_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ReduceDimOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = $ops(input, self.desc.axis); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! reduce_int_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ReduceDimOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = $ops(input, self.desc.axis); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float2int_ops { + ( + $name:ident, + $ops:expr, + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs.clone()); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! unary_float_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: UnaryOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = $ops(input); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + }; + ( + $name:ident, + $ops:expr, + reduce + ) => { + #[derive(new, Debug)] + struct $name { + desc: UnaryOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_float_tensor::(&self.desc.input); + let output = $ops(input); + + handles.register_float_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! unary_int_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: UnaryOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = $ops(input); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; + ( + $name:ident, + $ops:expr, + reduce + ) => { + #[derive(new, Debug)] + struct $name { + desc: UnaryOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let input = handles.get_int_tensor::(&self.desc.input); + let output = $ops(input); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float_cmp_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_float_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs.into()); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_int_cmp_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_int_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs.into()); + + handles.register_bool_tensor::(&self.desc.out.id, output); + } + } + }; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_int_ops { + ( + $name:ident, + $ops:expr + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_int_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs.into()); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; + ( + $name:ident, + $ops:expr, + noconvert + ) => { + #[derive(new, Debug)] + struct $name { + desc: ScalarOpIr, + _b: PhantomData, + } + + impl Operation for $name { + fn execute(&self, handles: &mut HandleContainer) { + let lhs = handles.get_int_tensor::(&self.desc.lhs); + let output = $ops(lhs, self.desc.rhs); + + handles.register_int_tensor::(&self.desc.out.id, output); + } + } + }; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/block.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/block.rs new file mode 100644 index 0000000..c729d5d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/block.rs @@ -0,0 +1,271 @@ +use crate::{FuserStatus, NumOperations, OperationFuser, stream::store::ExecutionStrategy}; +use burn_ir::{OperationIr, TensorId, TensorIr}; +use std::{collections::HashSet, sync::Arc}; + +/// A block represents a list of operations, not necessarily in the same order as the execution +/// stream. +/// +/// The start and end position of the relative execution stream are tracked in the block alongside +/// the ordering. +pub struct Block { + builders: Vec>>, + operations: Vec, + ids: HashSet, + ordering: Vec, + /// The start position in the relative execution stream. + pub start_pos: usize, + /// The end position in the relative execution stream. + pub end_pos: usize, +} + +/// The result of [registering](Block::register) an [operation](OperationIr). +pub enum RegistrationResult { + /// If the [operation](OperationIr) is correctly registered. + Accepted, + /// If the [operation](OperationIr) isn't part of the graph. + /// + /// In this case the operation isn't registered. + NotPartOfTheGraph, +} + +/// The optimization found for a [block](Block). +#[derive(Debug, new)] +pub struct BlockOptimization { + /// The [execution strategy](ExecutionStrategy) to be used to execute the [block](Block). + pub strategy: ExecutionStrategy, + /// The ordering of each operation in the relative execution stream. + pub ordering: Vec, +} + +impl Block { + /// Create a new block that will be optimized with the provided [optimization builders](OptimizationBuilder). + pub fn new(builders: &[Box>]) -> Self { + Self { + builders: builders.iter().map(|o| o.clone_dyn()).collect(), + operations: Vec::new(), + ids: HashSet::new(), + ordering: Vec::new(), + start_pos: usize::MAX, + end_pos: usize::MIN, + } + } + + /// Sort the [blocks](Block) based on the start position. + pub fn sort(blocks: &mut [Self]) { + blocks.sort_by(|a, b| a.start_pos.cmp(&b.start_pos)); + } + + /// Optimize the block. + pub fn optimize(mut self) -> BlockOptimization { + match find_best_optimization_index(&mut self.builders) { + Some(index) => { + let opt = self.builders[index].finish(); + let opt_len = opt.len(); + if opt_len < self.operations.len() { + self.ordering.drain(opt_len..); + } + + let strategy = ExecutionStrategy::Optimization { + ordering: Arc::new(self.ordering.clone()), + opt, + }; + BlockOptimization::new(strategy, self.ordering) + } + None => { + let strategy = ExecutionStrategy::Operations { + ordering: Arc::new(self.ordering.clone()), + }; + BlockOptimization::new(strategy, self.ordering) + } + } + } + + /// Returns if the block contains any of the provided [tensors](TensorIr). + pub fn contains_tensors(&self, tensors: &[&TensorIr]) -> bool { + for node in tensors { + if self.ids.contains(&node.id) { + return true; + } + } + + false + } + + /// Merge the current block with the other one and returns if the operation is successful. + /// + /// # Warning + /// + /// This will modify the current block even if the other block isn't correctly merged. + pub fn merge(&mut self, other: &Block) -> bool { + for (op, pos) in other.operations.iter().zip(&other.ordering) { + self.register(op, *pos, true); + } + + // The operation is successful if the current block can still be optimized. + self.still_optimizing() + } + + /// Register an [operation](OperationIr) in the current block. + /// + /// You need to provide the order of the operation as well as a force flag. + /// + /// When the force flag is true, the builder will always accept the operation, otherwise it + /// might refuse it if the operation [isn't part of the graph](RegistrationResult::NotPartOfTheGraph). + /// + /// Forcing is useful to fuse operations that are part of different graphs, but included + /// in the same optimization. + pub fn register( + &mut self, + operation: &OperationIr, + order: usize, + force: bool, + ) -> RegistrationResult { + if self.ids.is_empty() { + self.register_op(operation, order); + return RegistrationResult::Accepted; + } + let mut contains = false; + for node in operation.nodes() { + contains = self.ids.contains(&node.id); + + if contains { + break; + } + } + + if !contains && !force { + return RegistrationResult::NotPartOfTheGraph; + } + + self.register_op(operation, order); + RegistrationResult::Accepted + } + + /// If the block can still be optimized further. + pub fn still_optimizing(&self) -> bool { + let mut num_stopped = 0; + + for optimization in self.builders.iter() { + if let FuserStatus::Closed = optimization.status() { + num_stopped += 1 + } + } + + num_stopped < self.builders.len() + } + + fn register_op(&mut self, operation: &OperationIr, pos: usize) { + self.operations.push(operation.clone()); + self.ordering.push(pos); + + if pos < self.start_pos { + self.start_pos = pos; + } + if pos + 1 > self.end_pos { + self.end_pos = pos + 1; + } + + for builder in self.builders.iter_mut() { + builder.fuse(operation); + } + + for node in operation.nodes() { + self.ids.insert(node.id); + } + } +} + +impl BlockOptimization { + /// Maps the ordering of the current block optimization using the given mapping. + pub fn map_ordering(&mut self, mapping: &[usize]) { + for i in self.ordering.iter_mut() { + *i = mapping[*i]; + } + self.strategy.map_ordering(mapping); + } +} + +impl ExecutionStrategy { + /// Maps the ordering of the current execution strategy using the given mapping. + pub fn map_ordering(&mut self, mapping: &[usize]) { + match self { + ExecutionStrategy::Optimization { ordering, .. } => { + let mut ordering_mapped = ordering.to_vec(); + + for o in ordering_mapped.iter_mut() { + *o = mapping[*o]; + } + *ordering = Arc::new(ordering_mapped); + } + ExecutionStrategy::Operations { ordering } => { + let mut ordering_mapped = ordering.to_vec(); + + for o in ordering_mapped.iter_mut() { + *o = mapping[*o]; + } + + *ordering = Arc::new(ordering_mapped); + } + ExecutionStrategy::Composed(items) => { + for item in items.iter_mut() { + item.map_ordering(mapping); + } + } + } + } +} + +fn find_best_optimization_index( + optimizations: &mut [Box>], +) -> Option { + let mut best_index = None; + let mut best_score = 0; + + for (i, optimization) in optimizations.iter().enumerate() { + let properties = optimization.properties(); + + if properties.ready && properties.score >= best_score { + best_index = Some(i); + best_score = properties.score; + } + } + + best_index +} + +impl PartialEq for Block { + fn eq(&self, other: &Self) -> bool { + // Since the ordering can be seen as operation ids, we can use it to compare + // blocks. + let mut sorted_a = self.ordering.clone(); + let mut sorted_b = other.ordering.clone(); + sorted_a.sort(); + sorted_b.sort(); + + sorted_a == sorted_b + } +} + +impl core::fmt::Debug for Block { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "Block {{ pos: [{:?}, {:?}; {:?}] }}", + self.start_pos, + self.end_pos, + self.ordering.len(), + )) + } +} + +impl Clone for Block { + fn clone(&self) -> Self { + Self { + builders: self.builders.iter().map(|b| b.clone_dyn()).collect(), + operations: self.operations.clone(), + ids: self.ids.clone(), + ordering: self.ordering.clone(), + start_pos: self.start_pos, + end_pos: self.end_pos, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/merging.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/merging.rs new file mode 100644 index 0000000..251ceb3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/merging.rs @@ -0,0 +1,441 @@ +use super::Block; +use crate::NumOperations; + +#[derive(Debug, PartialEq)] +/// The result of [merging](merge_blocks) [blocks](Block). +pub enum MergeBlocksResult { + /// All [blocks](Block) merged into one. + Full(Block), + /// Some [blocks](Block) merged and some failed. + Partial { + merged: Vec>, + failed: Vec>, + }, + /// All [blocks](Block) failed to merge. + Fail, +} + +/// Merge multiple [block](Block) together. +/// +/// The resulting [blocks](Block) might be sorted if the flag is true, otherwise the order isn't +/// guarantee. This is mostly useful for testing. +/// +/// # Strategy +/// +/// The merging strategy is in two steps: +/// +/// 1. The first step is to recursively try to merge adjacent blocks. This has the advantage of +/// trying multiple blocks ordering, therefore trying multiple permutation of the blocks. +/// However, it has the downside of not trying to merge blocks that are further away in the list +/// of blocks. Since trying all combinations possible is exponential, therefore not possible, we +/// fallback on the second strategy. +/// 2. The second step is to reduce blocks by setting an accumulator block, then sequentially +/// trying to merge the remaining blocks. We try some permutations based on the result from +/// step1. +pub fn merge_blocks(blocks: &[&Block], sorted: bool) -> MergeBlocksResult { + if blocks.is_empty() { + return MergeBlocksResult::Fail; + } + + if blocks.len() == 1 { + return MergeBlocksResult::Full(blocks[0].clone()); + } + + if blocks.len() == 2 { + let block0 = blocks[0]; + let block1 = blocks[1]; + + return match merge_two(block0, block1) { + Some(result) => MergeBlocksResult::Full(result), + None => MergeBlocksResult::Fail, + }; + } + + let mut step1 = merge_blocks_step1(blocks); + + if step1.full.len() == 1 && step1.failed.is_empty() && step1.partial.is_empty() { + MergeBlocksResult::Full(step1.full.remove(0)) + } else if step1.partial.len() == 1 && step1.failed.is_empty() && step1.full.is_empty() { + MergeBlocksResult::Full(step1.partial.remove(0)) + } else { + let result = merge_blocks_step2(step1); + + if !sorted { + return result; + } + + match result { + MergeBlocksResult::Full(block) => MergeBlocksResult::Full(block), + MergeBlocksResult::Partial { + mut merged, + mut failed, + } => { + Block::sort(&mut merged); + Block::sort(&mut failed); + + MergeBlocksResult::Partial { merged, failed } + } + MergeBlocksResult::Fail => MergeBlocksResult::Fail, + } + } +} + +struct MergeBlockStep1 { + full: Vec>, + partial: Vec>, + failed: Vec>, +} + +impl Default for MergeBlockStep1 { + fn default() -> Self { + Self { + full: Default::default(), + partial: Default::default(), + failed: Default::default(), + } + } +} + +fn merge_blocks_step1(blocks: &[&Block]) -> MergeBlockStep1 { + let step_size = blocks.len() / 2; + let num_steps = f32::ceil(blocks.len() as f32 / step_size as f32) as usize; + + let mut result = MergeBlockStep1::default(); + + for i in 0..num_steps { + let start = i * step_size; + let end = usize::min(start + step_size, blocks.len()); + + match merge_blocks(&blocks[start..end], false) { + MergeBlocksResult::Full(block) => { + result.full.push(block); + } + MergeBlocksResult::Partial { + mut merged, + mut failed, + } => { + result.partial.append(&mut merged); + result.failed.append(&mut failed); + } + MergeBlocksResult::Fail => { + for b in &blocks[start..end] { + result.failed.push((*b).clone()); + } + } + } + } + + result +} + +fn merge_blocks_step2(mut step1: MergeBlockStep1) -> MergeBlocksResult { + // First let's try to merge partial graphs. + if step1.partial.len() > 1 { + match merge_accumulator(&step1.partial[0], &step1.partial[1..]) { + MergeBlocksResult::Full(block) => { + step1.partial = vec![block]; + } + MergeBlocksResult::Partial { merged, mut failed } => { + step1.partial = merged; + step1.failed.append(&mut failed); + } + MergeBlocksResult::Fail => {} + } + } + + // Then let's try to merge partial graphs with failed merges. + if !step1.failed.is_empty() { + step1.partial.append(&mut step1.failed); + match merge_accumulator(&step1.partial[0], &step1.partial[1..]) { + MergeBlocksResult::Full(block) => { + step1.partial = vec![block]; + } + MergeBlocksResult::Partial { merged, mut failed } => { + step1.partial = merged; + step1.failed.append(&mut failed); + } + MergeBlocksResult::Fail => {} + } + } + + // Then let's try to merge full graphs. + if step1.full.len() > 1 { + match merge_accumulator(&step1.full[0], &step1.full[1..]) { + MergeBlocksResult::Full(block) => { + step1.full = vec![block]; + } + MergeBlocksResult::Partial { merged, mut failed } => { + step1.full = merged; + step1.failed.append(&mut failed); + } + MergeBlocksResult::Fail => {} + } + } + + // Then let's try to merge full graphs with failed graphs. + if !step1.full.is_empty() { + step1.full.append(&mut step1.failed); + match merge_accumulator(&step1.full[0], &step1.full[1..]) { + MergeBlocksResult::Full(block) => { + step1.full = vec![block]; + } + MergeBlocksResult::Partial { merged, mut failed } => { + step1.full = merged; + step1.failed.append(&mut failed); + } + MergeBlocksResult::Fail => {} + } + } + + // Then let's try to merge full graphs with partial graphs. + if !step1.full.is_empty() || !step1.partial.is_empty() { + step1.full.append(&mut step1.partial); + match merge_accumulator(&step1.full[0], &step1.full[1..]) { + MergeBlocksResult::Full(block) => { + step1.full = vec![block]; + } + MergeBlocksResult::Partial { merged, mut failed } => { + step1.full = merged; + step1.failed.append(&mut failed); + } + MergeBlocksResult::Fail => { + // We do nothing. + } + } + } + + if step1.full.is_empty() { + MergeBlocksResult::Fail + } else if step1.failed.is_empty() { + if step1.full.len() == 1 { + MergeBlocksResult::Full(step1.full.remove(0)) + } else { + MergeBlocksResult::Partial { + merged: step1.full, + failed: vec![], + } + } + } else { + MergeBlocksResult::Partial { + merged: step1.full, + failed: step1.failed, + } + } +} + +fn merge_accumulator( + base: &Block, + blocks: &[Block], +) -> MergeBlocksResult { + let mut base = base.clone(); + let mut merged_failed = Vec::>::new(); + let mut merged_success = false; + + for block in blocks { + let mut base_current = base.clone(); + match base_current.merge(block) { + false => { + merged_failed.push((*block).clone()); + } + true => { + merged_success = true; + base = base_current; + } + } + } + + if merged_success { + if merged_failed.is_empty() { + MergeBlocksResult::Full(base) + } else { + MergeBlocksResult::Partial { + merged: vec![base], + failed: merged_failed, + } + } + } else { + MergeBlocksResult::Fail + } +} + +fn merge_two(a: &Block, b: &Block) -> Option> { + let mut base = a.clone(); + + if base.merge(b) { + return Some(base); + } + + let mut base = b.clone(); + + match base.merge(a) { + true => Some(base), + false => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + pub use crate::stream::execution::tests::{TestOptimization, TestOptimizationBuilder}; + use crate::{ + OperationFuser, + stream::tests::{operation_1, operation_2, operation_3}, + }; + + #[test] + fn test_merge_blocks_no_block() { + let actual = merge_blocks::(&[], true); + + assert_eq!(actual, MergeBlocksResult::Fail); + } + + #[test] + fn test_merge_blocks_single() { + let builders = builders(); + let block = Block::new(&builders); + let actual = merge_blocks::(&[&block], true); + + assert_eq!(actual, MergeBlocksResult::Full(block)); + } + + #[test] + fn test_merge_blocks_two_blocks() { + let builders = builders(); + let mut block1 = Block::new(&builders); + let mut block2 = Block::new(&builders); + block1.register(&operation_1(), 0, false); + block1.register(&operation_1(), 1, false); + block2.register(&operation_1(), 2, false); + block2.register(&operation_1(), 3, false); + + let actual = merge_blocks::(&[&block1, &block2], true); + + let mut expected = Block::new(&builders); + expected.register(&operation_1(), 0, false); + expected.register(&operation_1(), 1, false); + expected.register(&operation_1(), 2, false); + expected.register(&operation_1(), 3, false); + + assert_eq!(actual, MergeBlocksResult::Full(expected)); + } + + #[test] + fn test_merge_blocks_three_blocks() { + let builders = builders(); + let mut block1 = Block::new(&builders); + let mut block2 = Block::new(&builders); + let mut block3 = Block::new(&builders); + block1.register(&operation_1(), 0, false); + block2.register(&operation_1(), 1, false); + block3.register(&operation_1(), 2, false); + + let actual = merge_blocks::(&[&block1, &block2, &block3], true); + + let mut expected = Block::new(&builders); + expected.register(&operation_1(), 0, false); + expected.register(&operation_1(), 1, false); + expected.register(&operation_1(), 2, false); + + assert_eq!(actual, MergeBlocksResult::Full(expected)); + } + + #[test] + fn test_merge_blocks_three_blocks_partial() { + let builders = builders(); + let mut block1 = Block::new(&builders); + let mut block2 = Block::new(&builders); + let mut block3 = Block::new(&builders); + block1.register(&operation_1(), 0, false); + block2.register(&operation_2(), 1, false); + block3.register(&operation_1(), 2, false); + + let actual = merge_blocks::(&[&block1, &block2, &block3], true); + + let mut expected1 = Block::new(&builders); + let mut expected2 = Block::new(&builders); + expected1.register(&operation_1(), 0, false); + expected1.register(&operation_1(), 2, false); + expected2.register(&operation_2(), 1, false); + + assert_eq!( + actual, + MergeBlocksResult::Partial { + merged: vec![expected1, expected2], + failed: vec![] + } + ); + } + + #[test] + fn test_merge_blocks_four_blocks_partial_with_failure() { + let builders = builders(); + let mut block1 = Block::new(&builders); + let mut block2 = Block::new(&builders); + let mut block3 = Block::new(&builders); + let mut block4 = Block::new(&builders); + block1.register(&operation_1(), 0, false); + block2.register(&operation_2(), 1, false); + block3.register(&operation_1(), 2, false); + block4.register(&operation_3(), 3, false); + + let actual = merge_blocks::(&[&block1, &block2, &block3, &block4], true); + + let mut expected1 = Block::new(&builders); + let mut expected2 = Block::new(&builders); + let mut failed = Block::new(&builders); + expected1.register(&operation_1(), 0, false); + expected1.register(&operation_1(), 2, false); + expected2.register(&operation_2(), 1, false); + failed.register(&operation_3(), 3, false); + + assert_eq!( + actual, + MergeBlocksResult::Partial { + merged: vec![expected1], + failed: vec![expected2, failed] + } + ); + } + + #[test] + fn test_merge_blocks_five_blocks_partial_with_failure() { + let builders = builders(); + let mut block1 = Block::new(&builders); + let mut block2 = Block::new(&builders); + let mut block3 = Block::new(&builders); + let mut block4 = Block::new(&builders); + let mut block5 = Block::new(&builders); + block1.register(&operation_1(), 0, false); + block2.register(&operation_2(), 1, false); + block3.register(&operation_1(), 2, false); + block4.register(&operation_3(), 3, false); + block5.register(&operation_2(), 4, false); + + let actual = + merge_blocks::(&[&block1, &block2, &block3, &block4, &block5], true); + + let mut expected1 = Block::new(&builders); + let mut expected2 = Block::new(&builders); + let mut failed = Block::new(&builders); + expected1.register(&operation_1(), 0, false); + expected1.register(&operation_1(), 2, false); + expected2.register(&operation_2(), 1, false); + expected2.register(&operation_2(), 4, false); + failed.register(&operation_3(), 3, false); + + assert_eq!( + actual, + MergeBlocksResult::Partial { + merged: vec![expected1, expected2], + failed: vec![failed] + } + ); + } + + fn builders() -> Vec>> { + let builder_1 = TestOptimizationBuilder::new(0, vec![operation_1(); 10]); + let builder_2 = TestOptimizationBuilder::new(1, vec![operation_2(); 10]); + + vec![Box::new(builder_1), Box::new(builder_2)] + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/mod.rs new file mode 100644 index 0000000..827eaeb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/mod.rs @@ -0,0 +1,7 @@ +mod block; +mod optimization; + +pub(super) mod merging; +pub(super) use block::*; + +pub use optimization::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/blocks.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/blocks.rs new file mode 100644 index 0000000..d7c91fe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/blocks.rs @@ -0,0 +1,232 @@ +use std::sync::Arc; + +use crate::{ + NumOperations, + search::{ + Block, BlockOptimization, + merging::{MergeBlocksResult, merge_blocks}, + }, + stream::store::ExecutionStrategy, +}; + +/// Try to optimize a list of [blocks](Block) into a [block optimization](BlockOptimization). +/// +/// # Notes +/// +/// What we know here is that every block is independent at that time and can be executed +/// in any order. +/// +/// The contract is that the length of operations executed must include all operations. If we don't +/// find an optimization that can be executed with that constraint, we return a +/// [BlocksOptimizerResult::WithHoles]. +pub struct BlocksOptimizer { + blocks: Vec>, + resolved: Vec, + last_checked: usize, +} + +/// When we can't find a proper optimization for the provided list of [blocks](Block). +pub enum BlocksOptimizerResult { + /// When an optimization fill the hole stream. + Full(BlockOptimization), + /// The optimization found with the holes indices. + WithHoles { + strategies: Vec>>, + ordering: Vec, + holes: Vec, + }, +} + +enum BlockOptimizationStep { + Contiguous { + strategy: ExecutionStrategy, + }, + /// Only happen when we fallback on executing a single operation. + Operation { + strategy: ExecutionStrategy, + }, + WithHoles { + strategy: ExecutionStrategy, + holes: Vec, + }, + Stop, +} + +impl BlocksOptimizer { + /// Create a new optimizer with the given blocks. + pub fn new(blocks: Vec>) -> Self { + let num_ops: usize = blocks.iter().map(|g| g.end_pos).max().unwrap(); + + Self { + blocks, + resolved: vec![false; num_ops], + last_checked: 0, + } + } + + /// Optimizes the blocks. + /// + /// The strategy is quite simple. We try to merge as much [blocks](Block) together as we can, + /// then we iterate over them in order composing optimizations with the remaining blocks, all + /// while minimizing fallbacks operations to avoid having holes in the optimization stream. + pub fn optimize(mut self) -> BlocksOptimizerResult { + self = self.merging_pass(); + + let mut strategies = Vec::with_capacity(self.blocks.len()); + let mut ordering = Vec::new(); + let mut blocks = Vec::new(); + core::mem::swap(&mut blocks, &mut self.blocks); + + for block in blocks { + match self.optimize_block(block, &mut ordering) { + BlockOptimizationStep::Contiguous { strategy } => { + strategies.push(Box::new(strategy)); + } + BlockOptimizationStep::Operation { strategy } => { + strategies.push(Box::new(strategy)); + break; + } + BlockOptimizationStep::WithHoles { strategy, holes } => { + strategies.push(Box::new(strategy)); + + return BlocksOptimizerResult::WithHoles { + strategies, + ordering, + holes, + }; + } + BlockOptimizationStep::Stop => { + break; + } + } + } + + let optimization = match strategies.len() > 1 { + true => BlockOptimization { + strategy: ExecutionStrategy::Composed(strategies), + ordering, + }, + false => BlockOptimization { + strategy: *strategies.remove(0), + ordering, + }, + }; + + BlocksOptimizerResult::Full(optimization) + } + + /// Optimize a single block. + fn optimize_block( + &mut self, + block: Block, + ordering: &mut Vec, + ) -> BlockOptimizationStep { + let last_index = block.end_pos; + let mut block_optimization = block.optimize(); + let opt_size = block_optimization.ordering.len(); + + for pos in block_optimization.ordering.iter() { + self.update_check(*pos); + } + + if self.last_checked != ordering.len() + opt_size { + if !ordering.is_empty() { + // Don't include that block and need further exploring. + return BlockOptimizationStep::Stop; + } + + return self.optimize_holes(block_optimization, last_index, ordering); + } + + ordering.append(&mut block_optimization.ordering); + BlockOptimizationStep::Contiguous { + strategy: block_optimization.strategy, + } + } + + /// The provided optimization has holes. + fn optimize_holes( + &mut self, + mut optimization: BlockOptimization, + last_index: usize, + ordering_global: &mut Vec, + ) -> BlockOptimizationStep { + match optimization.strategy { + ExecutionStrategy::Optimization { opt, ordering } => { + ordering_global.append(&mut optimization.ordering); + let holes = self.find_holes(last_index); + + if holes.is_empty() { + let strategy = ExecutionStrategy::Optimization { opt, ordering }; + BlockOptimizationStep::Contiguous { strategy } + } else { + let strategy = ExecutionStrategy::Optimization { opt, ordering }; + BlockOptimizationStep::WithHoles { strategy, holes } + } + } + ExecutionStrategy::Operations { ordering } => { + let min = ordering.iter().min().unwrap(); + ordering_global.push(*min); + + let strategy = ExecutionStrategy::Operations { + ordering: Arc::new(vec![*min]), + }; + BlockOptimizationStep::Operation { strategy } + } + _ => unreachable!(), + } + } + + fn update_check(&mut self, pos: usize) { + self.resolved[pos] = true; + + for i in self.last_checked..self.resolved.len() { + if self.resolved[i] { + self.last_checked += 1; + } else { + break; + } + } + } + + fn find_holes(&mut self, last: usize) -> Vec { + let mut fallbacks = Vec::new(); + + for i in self.last_checked..last { + if !self.resolved[i] { + fallbacks.push(i); + self.resolved[i] = true; + } + self.last_checked += 1; + } + + fallbacks + } + + /// Try to merge blocks together. + fn merging_pass(mut self) -> Self { + if self.blocks.len() == 1 { + return self; + } + + Block::sort(&mut self.blocks); + let blocks = self.blocks.iter().collect::>(); + + match merge_blocks(&blocks, false) { + MergeBlocksResult::Full(block) => { + self.blocks = vec![block]; + } + MergeBlocksResult::Partial { + mut merged, + mut failed, + } => { + merged.append(&mut failed); + self.blocks = merged; + Block::sort(&mut self.blocks); + } + MergeBlocksResult::Fail => {} + } + + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/mod.rs new file mode 100644 index 0000000..339914a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/mod.rs @@ -0,0 +1,4 @@ +mod blocks; +mod stream; + +pub use stream::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/stream.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/stream.rs new file mode 100644 index 0000000..617cb46 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/search/optimization/stream.rs @@ -0,0 +1,277 @@ +use super::blocks::BlocksOptimizer; +use crate::{ + NumOperations, OperationFuser, + search::{ + Block, BlockOptimization, RegistrationResult, + merging::{MergeBlocksResult, merge_blocks}, + optimization::blocks::BlocksOptimizerResult, + }, + stream::store::ExecutionStrategy, +}; +use burn_ir::OperationIr; + +/// Optimize a stream of [operations](OperationIr) using a list of [builders](OptimizationBuilder). +pub struct StreamOptimizer { + builders: Vec>>, + blocks: Vec>, + length: usize, + stopped: bool, + max_blocks: Option, +} + +impl StreamOptimizer { + /// Create a new stream optimizer. + pub fn new(builders: Vec>>) -> Self { + Self { + builders, + blocks: Vec::new(), + length: 0, + stopped: false, + // Too high and it may breaks the fusion cache always retriggering explorations. + max_blocks: Some(5), + } + } + + /// Register a new [operation](OperationIr) in the optimizer. + /// + /// You can use the function [Self::still_optimizing] to know if the operations are actually + /// being registered. + pub fn register(&mut self, operation: &OperationIr) { + if self.stopped { + return; + } + + if self.blocks.is_empty() { + self.on_new_block(operation); + self.length += 1; + return; + } + + match self.merge_blocks(operation, false) { + MergeBlockStep::Full | MergeBlockStep::NoNeed => {} + MergeBlockStep::Fail | MergeBlockStep::Partial => { + // With the given operation, blocks are no longer independent. + self.stopped = true; + return; + } + } + + if let Some(max_blocks) = self.max_blocks { + if self.register_max_block(operation, max_blocks) { + self.length += 1; + } else { + self.stopped = true; + } + return; + } + + let added_count = self.register_inner(operation, false); + if added_count == 0 { + self.on_new_block(operation); + } + + self.length += 1; + } + + /// Optimize the current stream on the given [operations](OperationIr). + /// + /// # Notes + /// + /// The operations provided are the same as the ones used in the [register](Self::register) + /// method, this simply remove the need for the current type to also keep track of the list of + /// operations. + pub fn optimize(&self, operations: &[OperationIr]) -> BlockOptimization { + let result = BlocksOptimizer::new(self.blocks.clone()).optimize(); + + match result { + BlocksOptimizerResult::Full(block_optimization) => block_optimization, + BlocksOptimizerResult::WithHoles { + mut strategies, + mut ordering, + mut holes, + } => { + loop { + let mut search = self.new_empty_search(); + + let mut operations_holes = Vec::with_capacity(holes.len()); + + for index in holes.iter() { + let op = &operations[*index]; + operations_holes.push(op.clone()); + search.register(op); + } + + let mut optimization_of_holes = search.optimize(&operations_holes); + + optimization_of_holes.map_ordering(&holes); + + strategies.push(Box::new(optimization_of_holes.strategy)); + holes.drain(0..optimization_of_holes.ordering.len()); + ordering.append(&mut optimization_of_holes.ordering); + + if holes.is_empty() { + break; + } + } + + BlockOptimization::new(ExecutionStrategy::Composed(strategies), ordering) + } + } + } + + /// Reset the state of the optimizer. + pub fn reset(&mut self) { + self.builders.iter_mut().for_each(|b| b.reset()); + self.length = 0; + self.blocks.clear(); + self.stopped = false; + } + + /// Returns if some optimizations are still possible within the stream. + pub fn still_optimizing(&self) -> bool { + if self.stopped { + return false; + } + if self.blocks.is_empty() { + return true; + } + + let mut num_stopped = 0; + + for block in self.blocks.iter() { + if !block.still_optimizing() { + num_stopped += 1 + } + } + + num_stopped < self.blocks.len() + } + + fn register_max_block(&mut self, operation: &OperationIr, max_blocks: usize) -> bool { + if max_blocks == 1 { + // Register in the single block with a force. + self.register_inner(operation, true); + return true; + } + let added_count = self.register_inner(operation, false); + + if added_count > 0 { + return true; + } + + if added_count == 0 && self.blocks.len() < max_blocks { + self.on_new_block(operation); + return true; + } + + self.merge_blocks(operation, true); + + if self.blocks.len() >= max_blocks { + self.stopped = true; + return false; + } + + let added_count = self.register_inner(operation, false); + + if added_count == 0 { + self.on_new_block(operation); + } + + true + } + + fn register_inner(&mut self, operation: &OperationIr, force: bool) -> usize { + let mut added_count = 0; + for block in self.blocks.iter_mut() { + match block.register(operation, self.length, force) { + RegistrationResult::Accepted => { + added_count += 1; + } + RegistrationResult::NotPartOfTheGraph => {} + } + } + added_count + } + + fn new_empty_search(&self) -> Self { + Self::new( + self.builders + .iter() + .map(|b| { + let mut b = b.clone_dyn(); + b.reset(); + b + }) + .collect(), + ) + } + + fn merge_blocks(&mut self, operation: &OperationIr, all: bool) -> MergeBlockStep { + let nodes = operation.nodes(); + let mut block_merges = Vec::new(); + + for (i, block) in self.blocks.iter().enumerate() { + if all || block.contains_tensors(&nodes) { + block_merges.push(i); + } + } + + if block_merges.len() <= 1 { + return MergeBlockStep::NoNeed; + } + + let blocks_to_merge = self + .blocks + .iter() + .enumerate() + .filter_map(|(i, g)| match block_merges.contains(&i) { + true => Some(g), + false => None, + }) + .collect::>(); + + let merged = merge_blocks(&blocks_to_merge, false); + + let mut clear_blocks = || { + let mut indices = block_merges.to_vec(); + indices.sort(); + + for g in indices.into_iter().rev() { + self.blocks.remove(g); + } + }; + + match merged { + MergeBlocksResult::Full(block) => { + clear_blocks(); + self.blocks.push(block); + Block::sort(&mut self.blocks); + MergeBlockStep::Full + } + MergeBlocksResult::Partial { + mut merged, + mut failed, + } => { + clear_blocks(); + self.blocks.append(&mut merged); + self.blocks.append(&mut failed); + Block::sort(&mut self.blocks); + MergeBlockStep::Partial + } + MergeBlocksResult::Fail => MergeBlockStep::Fail, + } + } + + fn on_new_block(&mut self, operation: &OperationIr) { + let mut block = Block::new(&self.builders); + block.register(operation, self.length, true); + self.blocks.push(block); + } +} + +enum MergeBlockStep { + Full, + Partial, + Fail, + NoNeed, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/server.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/server.rs new file mode 100644 index 0000000..a63f36b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/server.rs @@ -0,0 +1,215 @@ +use std::sync::Arc; + +use crate::{ + FusionBackend, FusionRuntime, + stream::{MultiStream, OperationStreams, StreamId, execution::Operation}, +}; +use burn_backend::{TensorData, backend::ExecutionError}; +use burn_ir::{HandleContainer, OperationIr, TensorId, TensorIr}; + +pub struct FusionServer { + streams: MultiStream, + pub(crate) handles: HandleContainer, +} + +impl FusionServer +where + R: FusionRuntime, +{ + pub fn new(device: R::FusionDevice) -> Self { + Self { + streams: MultiStream::new(device.clone()), + handles: HandleContainer::new(), + } + } + + pub fn register( + &mut self, + streams: OperationStreams, + repr: OperationIr, + operation: Arc>, + ) { + self.streams + .register(streams, repr, operation, &mut self.handles) + } + + pub fn drain_stream(&mut self, id: StreamId) { + self.streams.drain(&mut self.handles, id) + } + + pub fn create_empty_handle(&mut self) -> TensorId { + self.handles.create_tensor_uninit() + } + + pub fn read_float( + &mut self, + tensor: TensorIr, + id: StreamId, + ) -> impl Future> + Send + use + where + B: FusionBackend, + { + // Make sure all registered operations are executed. + // The underlying backend can still be async. + self.drain_stream(id); + let tensor_float = self.handles.get_float_tensor::(&tensor); + self.streams.mark_read(id, &tensor, &self.handles); + B::float_into_data(tensor_float) + } + + pub fn read_int( + &mut self, + tensor: TensorIr, + id: StreamId, + ) -> impl Future> + Send + use + where + B: FusionBackend, + { + // Make sure all registered operations are executed. + // The underlying backend can still be async. + self.drain_stream(id); + let tensor_int = self.handles.get_int_tensor::(&tensor); + self.streams.mark_read(id, &tensor, &self.handles); + B::int_into_data(tensor_int) + } + + pub fn read_bool( + &mut self, + tensor: TensorIr, + id: StreamId, + ) -> impl Future> + Send + use + where + B: FusionBackend, + { + // Make sure all registered operations are executed. + // The underlying backend can still be async. + self.drain_stream(id); + let tensor_bool = self.handles.get_bool_tensor::(&tensor); + self.streams.mark_read(id, &tensor, &self.handles); + B::bool_into_data(tensor_bool) + } + + pub fn read_quantized( + &mut self, + tensor: TensorIr, + id: StreamId, + ) -> impl Future> + Send + use + where + B: FusionBackend, + { + // Make sure all registered operations are executed. + // The underlying backend can still be async. + self.drain_stream(id); + let tensor_q = self.handles.get_quantized_tensor::(&tensor); + self.streams.mark_read(id, &tensor, &self.handles); + B::q_into_data(tensor_q) + } + + pub fn change_server_float( + &mut self, + tensor: &TensorIr, + stream_tensor: StreamId, + device: &R::FusionDevice, + server_device: &mut Self, + ) -> TensorId + where + B: FusionBackend, + { + let tensor_float = self.handles.get_float_tensor::(tensor); + self.streams.mark_read(stream_tensor, tensor, &self.handles); + + let tensor = B::float_to_device(tensor_float, device); + let id = server_device.create_empty_handle(); + + server_device + .handles + .register_float_tensor::(&id, tensor.clone()); + + id + } + + pub fn resolve_server_float(&mut self, tensor: &TensorIr) -> B::FloatTensorPrimitive + where + B: FusionBackend, + { + self.handles.get_float_tensor::(tensor) + } + + pub fn resolve_server_int(&mut self, tensor: &TensorIr) -> B::IntTensorPrimitive + where + B: FusionBackend, + { + self.handles.get_int_tensor::(tensor) + } + + pub fn resolve_server_bool(&mut self, tensor: &TensorIr) -> B::BoolTensorPrimitive + where + B: FusionBackend, + { + self.handles.get_bool_tensor::(tensor) + } + + pub fn change_server_int( + &mut self, + tensor: &TensorIr, + stream_tensor: StreamId, + device: &R::FusionDevice, + server_device: &mut Self, + ) -> TensorId + where + B: FusionBackend, + { + let tensor_int = self.handles.get_int_tensor::(tensor); + self.streams.mark_read(stream_tensor, tensor, &self.handles); + let tensor = B::int_to_device(tensor_int, device); + let id = server_device.create_empty_handle(); + + server_device + .handles + .register_int_tensor::(&id, tensor.clone()); + + id + } + + pub fn change_server_bool( + &mut self, + tensor: &TensorIr, + stream_tensor: StreamId, + device: &R::FusionDevice, + server_device: &mut Self, + ) -> TensorId + where + B: FusionBackend, + { + let tensor_bool = self.handles.get_bool_tensor::(tensor); + self.streams.mark_read(stream_tensor, tensor, &self.handles); + let tensor = B::bool_to_device(tensor_bool, device); + let id = server_device.create_empty_handle(); + + server_device + .handles + .register_bool_tensor::(&id, tensor.clone()); + + id + } + + pub fn change_server_quantized( + &mut self, + tensor: &TensorIr, + device: &R::FusionDevice, + server_device: &mut Self, + ) -> TensorId + where + B: FusionBackend, + { + let tensor = self.handles.get_quantized_tensor::(tensor); + let tensor = B::q_to_device(tensor, device); + let id = server_device.create_empty_handle(); + + server_device + .handles + .register_quantized_tensor::(&id, tensor); + + id + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/base.rs new file mode 100644 index 0000000..39336fd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/base.rs @@ -0,0 +1 @@ +pub use burn_backend::StreamId; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/context.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/context.rs new file mode 100644 index 0000000..bf5f717 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/context.rs @@ -0,0 +1,1274 @@ +use burn_backend::{Shape, Slice}; +use burn_ir::*; +use hashbrown::HashMap; + +/// The context contains the relative graph tensor mapping so that a relative tensor id can be +/// mapped to an existing tensor that can be fetched and updated with the +/// [handle container](HandleContainer). +/// +/// It also contains all scalar values, which can change even for the same graph. They are sorted +/// in the order in which they appear in the graph. +#[allow(clippy::too_many_arguments)] +#[derive(new)] +pub struct Context<'a, H> { + /// The tensor mapping where local tensor id points to the updated tensor representation. + pub tensors: &'a mut HashMap, + /// Handle container to retrieve tensors based on their representation. + pub handles: &'a mut HandleContainer, + /// Scalars found in the graph in the order they appeared. + pub scalars: &'a mut HashMap, + /// Shape mapping from relative shape ids to global (real) shape ids. + pub shapes_relative2global: &'a HashMap, +} + +#[derive(Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Debug)] +/// Scalar unique identifier. +pub struct ScalarId { + /// The value. + pub value: u64, +} + +pub(crate) struct OperationConverter { + tensors_relative2global: HashMap, + tensors_global2relative: HashMap, + shapes_global2relative: HashMap, + shapes_relative2global: HashMap, + scalars: HashMap, +} + +impl Default for OperationConverter { + fn default() -> Self { + let mut val = Self { + tensors_relative2global: Default::default(), + tensors_global2relative: Default::default(), + shapes_global2relative: Default::default(), + shapes_relative2global: Default::default(), + scalars: Default::default(), + }; + + // global 1 is always shape id 0. + val.shapes_global2relative.insert(1, 0); + val.shapes_relative2global.insert(0, 1); + + val + } +} + +/// Fork of a [context](Context) which owns its data. +pub struct ContextOwned { + tensors: HashMap, + handles: HandleContainer, + scalars: HashMap, + shapes_relative2global: HashMap, +} + +impl ContextOwned { + /// Convert into [context](Context). + pub fn as_context(&mut self) -> Context<'_, H> { + Context { + tensors: &mut self.tensors, + handles: &mut self.handles, + scalars: &mut self.scalars, + shapes_relative2global: &self.shapes_relative2global, + } + } + + /// Fork the context again. + pub fn fork(&self) -> ContextOwned { + ContextOwned { + tensors: self.tensors.clone(), + handles: self.handles.fork(), + scalars: self.scalars.clone(), + shapes_relative2global: self.shapes_relative2global.clone(), + } + } +} + +impl Context<'_, H> { + /// Fork the context into an [owned context](ContextOwned). + pub fn fork(&self) -> ContextOwned { + ContextOwned { + tensors: self.tensors.clone(), + handles: self.handles.fork(), + scalars: self.scalars.clone(), + shapes_relative2global: self.shapes_relative2global.clone(), + } + } +} + +pub(crate) trait RelativeOps { + /// Convert (usually an [`OperationIr`]) to a relative form. + /// + /// The id and the shape of tensors will be computed relative to existing + /// operations in the queue. We do this because we want to fuse operations + /// that have similar shapes, but we do not care about the exact values. + /// + /// Similar we do not care about the exact ids of the tensor, but about their + /// relative ids (how close they are in the operation queue) + fn to_relative(&self, converter: &mut OperationConverter) -> Self; +} + +impl OperationConverter { + pub(crate) fn context<'a, H>( + &'a mut self, + handles: &'a mut HandleContainer, + ) -> Context<'a, H> { + Context { + handles, + tensors: &mut self.tensors_relative2global, + scalars: &mut self.scalars, + shapes_relative2global: &self.shapes_relative2global, + } + } + + pub(crate) fn clear(&mut self) { + self.tensors_relative2global.clear(); + self.tensors_global2relative.clear(); + + self.shapes_global2relative.clear(); + self.shapes_relative2global.clear(); + + // global 1 is always shape id 0. + self.shapes_global2relative.insert(1, 0); + self.shapes_relative2global.insert(0, 1); + + self.scalars.clear(); + } +} + +impl RelativeOps for OperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + OperationIr::BaseFloat(ops) => OperationIr::BaseFloat(ops.to_relative(converter)), + OperationIr::BaseInt(ops) => OperationIr::BaseInt(ops.to_relative(converter)), + OperationIr::BaseBool(ops) => OperationIr::BaseBool(ops.to_relative(converter)), + OperationIr::NumericFloat(dtype, ops) => { + OperationIr::NumericFloat(*dtype, ops.to_relative(converter)) + } + OperationIr::NumericInt(dtype, ops) => { + OperationIr::NumericInt(*dtype, ops.to_relative(converter)) + } + OperationIr::Bool(ops) => OperationIr::Bool(ops.to_relative(converter)), + OperationIr::Int(ops) => OperationIr::Int(ops.to_relative(converter)), + OperationIr::Float(dtype, ops) => { + OperationIr::Float(*dtype, ops.to_relative(converter)) + } + OperationIr::Module(ops) => OperationIr::Module(ops.to_relative(converter)), + OperationIr::Custom(ops) => OperationIr::Custom(ops.to_relative(converter)), + OperationIr::Init(ops) => OperationIr::Init(ops.to_relative(converter)), + OperationIr::Drop(tensor) => OperationIr::Drop(tensor.to_relative(converter)), + } + } +} + +impl RelativeOps for ModuleOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + ModuleOperationIr::Embedding(desc) => ModuleOperationIr::Embedding(EmbeddingOpIr { + weights: desc.weights.to_relative(converter), + indices: desc.indices.to_relative(converter), + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::EmbeddingBackward(desc) => { + ModuleOperationIr::EmbeddingBackward(EmbeddingBackwardOpIr { + weights: desc.weights.to_relative(converter), + out_grad: desc.out_grad.to_relative(converter), + indices: desc.indices.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv1d(desc) => ModuleOperationIr::Conv1d(Conv1dOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::Conv1dXBackward(desc) => { + ModuleOperationIr::Conv1dXBackward(Conv1dXBackwardOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv1dWeightBackward(desc) => { + ModuleOperationIr::Conv1dWeightBackward(Conv1dWeightBackwardOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv1dBiasBackward(desc) => { + ModuleOperationIr::Conv1dBiasBackward(Conv1dBiasBackwardOpIr { + x: desc.x.to_relative(converter), + bias: desc.bias.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv2d(desc) => ModuleOperationIr::Conv2d(Conv2dOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::Conv2dXBackward(desc) => { + ModuleOperationIr::Conv2dXBackward(Conv2dXBackwardOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv2dWeightBackward(desc) => { + ModuleOperationIr::Conv2dWeightBackward(Conv2dWeightBackwardOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv2dBiasBackward(desc) => { + ModuleOperationIr::Conv2dBiasBackward(Conv2dBiasBackwardOpIr { + x: desc.x.to_relative(converter), + bias: desc.bias.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv3d(desc) => ModuleOperationIr::Conv3d(Conv3dOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::Conv3dXBackward(desc) => { + ModuleOperationIr::Conv3dXBackward(Conv3dXBackwardOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv3dWeightBackward(desc) => { + ModuleOperationIr::Conv3dWeightBackward(Conv3dWeightBackwardOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Conv3dBiasBackward(desc) => { + ModuleOperationIr::Conv3dBiasBackward(Conv3dBiasBackwardOpIr { + x: desc.x.to_relative(converter), + bias: desc.bias.to_relative(converter), + output_grad: desc.output_grad.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::DeformableConv2d(desc) => { + ModuleOperationIr::DeformableConv2d(Box::new(DeformConv2dOpIr { + x: desc.x.to_relative(converter), + offset: desc.offset.to_relative(converter), + weight: desc.weight.to_relative(converter), + mask: desc.mask.as_ref().map(|t| t.to_relative(converter)), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + })) + } + ModuleOperationIr::DeformableConv2dBackward(desc) => { + ModuleOperationIr::DeformableConv2dBackward(Box::new(DeformConv2dBackwardOpIr { + x: desc.x.to_relative(converter), + offset: desc.offset.to_relative(converter), + weight: desc.weight.to_relative(converter), + mask: desc.mask.as_ref().map(|t| t.to_relative(converter)), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + out_grad: desc.out_grad.to_relative(converter), + options: desc.options.clone(), + input_grad: desc.input_grad.to_relative(converter), + offset_grad: desc.offset_grad.to_relative(converter), + weight_grad: desc.weight_grad.to_relative(converter), + mask_grad: desc.mask_grad.as_ref().map(|t| t.to_relative(converter)), + bias_grad: desc.bias_grad.as_ref().map(|t| t.to_relative(converter)), + })) + } + ModuleOperationIr::ConvTranspose1d(desc) => { + ModuleOperationIr::ConvTranspose1d(ConvTranspose1dOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::ConvTranspose2d(desc) => { + ModuleOperationIr::ConvTranspose2d(ConvTranspose2dOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::ConvTranspose3d(desc) => { + ModuleOperationIr::ConvTranspose3d(ConvTranspose3dOpIr { + x: desc.x.to_relative(converter), + weight: desc.weight.to_relative(converter), + bias: desc.bias.as_ref().map(|t| t.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::AvgPool1d(desc) => ModuleOperationIr::AvgPool1d(AvgPool1dOpIr { + x: desc.x.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + count_include_pad: desc.count_include_pad, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::AvgPool2d(desc) => ModuleOperationIr::AvgPool2d(AvgPool2dOpIr { + x: desc.x.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + count_include_pad: desc.count_include_pad, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::AvgPool1dBackward(desc) => { + ModuleOperationIr::AvgPool1dBackward(AvgPool1dBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + count_include_pad: desc.count_include_pad, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::AvgPool2dBackward(desc) => { + ModuleOperationIr::AvgPool2dBackward(AvgPool2dBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + count_include_pad: desc.count_include_pad, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::AdaptiveAvgPool1d(desc) => { + ModuleOperationIr::AdaptiveAvgPool1d(AdaptiveAvgPool1dOpIr { + x: desc.x.to_relative(converter), + output_size: desc.output_size, + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::AdaptiveAvgPool2d(desc) => { + ModuleOperationIr::AdaptiveAvgPool2d(AdaptiveAvgPool2dOpIr { + x: desc.x.to_relative(converter), + output_size: desc.output_size, + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::AdaptiveAvgPool1dBackward(desc) => { + ModuleOperationIr::AdaptiveAvgPool1dBackward(AdaptiveAvgPool1dBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::AdaptiveAvgPool2dBackward(desc) => { + ModuleOperationIr::AdaptiveAvgPool2dBackward(AdaptiveAvgPool2dBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::MaxPool1d(desc) => ModuleOperationIr::MaxPool1d(MaxPool1dOpIr { + x: desc.x.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + dilation: desc.dilation, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::MaxPool1dWithIndices(desc) => { + ModuleOperationIr::MaxPool1dWithIndices(MaxPool1dWithIndicesOpIr { + x: desc.x.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + dilation: desc.dilation, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + out_indices: desc.out_indices.to_relative(converter), + }) + } + ModuleOperationIr::MaxPool1dWithIndicesBackward(desc) => { + ModuleOperationIr::MaxPool1dWithIndicesBackward(MaxPool1dWithIndicesBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + indices: desc.indices.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + dilation: desc.dilation, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::MaxPool2d(desc) => ModuleOperationIr::MaxPool2d(MaxPool2dOpIr { + x: desc.x.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + dilation: desc.dilation, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }), + ModuleOperationIr::MaxPool2dWithIndices(desc) => { + ModuleOperationIr::MaxPool2dWithIndices(MaxPool2dWithIndicesOpIr { + x: desc.x.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + dilation: desc.dilation, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + out_indices: desc.out_indices.to_relative(converter), + }) + } + ModuleOperationIr::MaxPool2dWithIndicesBackward(desc) => { + ModuleOperationIr::MaxPool2dWithIndicesBackward(MaxPool2dWithIndicesBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + indices: desc.indices.to_relative(converter), + kernel_size: desc.kernel_size, + stride: desc.stride, + padding: desc.padding, + dilation: desc.dilation, + ceil_mode: desc.ceil_mode, + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Interpolate(desc) => { + ModuleOperationIr::Interpolate(InterpolateOpIr { + x: desc.x.to_relative(converter), + output_size: desc.output_size, + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::InterpolateBackward(desc) => { + ModuleOperationIr::InterpolateBackward(InterpolateBackwardOpIr { + x: desc.x.to_relative(converter), + grad: desc.grad.to_relative(converter), + output_size: desc.output_size, + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + ModuleOperationIr::Attention(desc) => ModuleOperationIr::Attention(AttentionOpIr { + query: desc.query.to_relative(converter), + key: desc.key.to_relative(converter), + value: desc.value.to_relative(converter), + mask: desc.mask.as_ref().map(|m| m.to_relative(converter)), + attn_bias: desc.attn_bias.as_ref().map(|ab| ab.to_relative(converter)), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }), + } + } +} + +impl RelativeOps for FloatOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + FloatOperationIr::Exp(desc) => FloatOperationIr::Exp(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Log(desc) => FloatOperationIr::Log(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Log1p(desc) => FloatOperationIr::Log1p(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Erf(desc) => FloatOperationIr::Erf(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::PowfScalar(desc) => FloatOperationIr::PowfScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Sqrt(desc) => FloatOperationIr::Sqrt(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Cos(desc) => FloatOperationIr::Cos(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Sin(desc) => FloatOperationIr::Sin(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Tanh(desc) => FloatOperationIr::Tanh(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Tan(desc) => FloatOperationIr::Tan(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Cosh(desc) => FloatOperationIr::Cosh(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Sinh(desc) => FloatOperationIr::Sinh(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcCos(desc) => FloatOperationIr::ArcCos(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcCosh(desc) => FloatOperationIr::ArcCosh(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcSin(desc) => FloatOperationIr::ArcSin(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcSinh(desc) => FloatOperationIr::ArcSinh(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcTan(desc) => FloatOperationIr::ArcTan(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcTanh(desc) => FloatOperationIr::ArcTanh(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::ArcTan2(desc) => FloatOperationIr::ArcTan2(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::IntoInt(desc) => FloatOperationIr::IntoInt(CastOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Matmul(desc) => FloatOperationIr::Matmul(MatmulOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Cross(desc) => FloatOperationIr::Cross(CrossOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + dim: desc.dim, + }), + FloatOperationIr::Random(desc) => FloatOperationIr::Random(RandomOpIr { + out: desc.out.to_relative(converter), + distribution: desc.distribution, + }), + FloatOperationIr::Recip(desc) => FloatOperationIr::Recip(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Quantize(desc) => FloatOperationIr::Quantize(QuantizeOpIr { + tensor: desc.tensor.to_relative(converter), + qparams: QuantizationParametersIr { + scales: desc.qparams.scales.to_relative(converter), + }, + scheme: desc.scheme, + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Dequantize(desc) => FloatOperationIr::Dequantize(DequantizeOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Round(desc) => FloatOperationIr::Round(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Floor(desc) => FloatOperationIr::Floor(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Ceil(desc) => FloatOperationIr::Ceil(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::Trunc(desc) => FloatOperationIr::Ceil(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::IsNan(desc) => FloatOperationIr::IsNan(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::IsInf(desc) => FloatOperationIr::IsInf(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + FloatOperationIr::GridSample2d(desc) => { + FloatOperationIr::GridSample2d(GridSample2dOpIr { + tensor: desc.tensor.to_relative(converter), + grid: desc.grid.to_relative(converter), + options: desc.options.clone(), + out: desc.out.to_relative(converter), + }) + } + } + } +} + +impl RelativeOps for BoolOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + BoolOperationIr::IntoFloat(desc) => BoolOperationIr::IntoFloat(CastOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BoolOperationIr::IntoInt(desc) => BoolOperationIr::IntoInt(CastOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BoolOperationIr::Not(desc) => BoolOperationIr::Not(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BoolOperationIr::And(desc) => BoolOperationIr::And(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BoolOperationIr::Or(desc) => BoolOperationIr::Or(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + } + } +} + +impl RelativeOps for IntOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + IntOperationIr::IntoFloat(desc) => IntOperationIr::IntoFloat(CastOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + IntOperationIr::Matmul(desc) => IntOperationIr::Matmul(MatmulOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + IntOperationIr::BitwiseAnd(desc) => IntOperationIr::BitwiseAnd(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + IntOperationIr::BitwiseAndScalar(desc) => { + IntOperationIr::BitwiseAndScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs, + out: desc.out.to_relative(converter), + }) + } + IntOperationIr::BitwiseOr(desc) => IntOperationIr::BitwiseOr(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + IntOperationIr::BitwiseOrScalar(desc) => IntOperationIr::BitwiseOrScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs, + out: desc.out.to_relative(converter), + }), + IntOperationIr::BitwiseXor(desc) => IntOperationIr::BitwiseXor(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + IntOperationIr::BitwiseXorScalar(desc) => { + IntOperationIr::BitwiseXorScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs, + out: desc.out.to_relative(converter), + }) + } + IntOperationIr::BitwiseNot(desc) => IntOperationIr::BitwiseNot(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + IntOperationIr::BitwiseLeftShift(desc) => { + IntOperationIr::BitwiseLeftShift(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + IntOperationIr::BitwiseLeftShiftScalar(desc) => { + IntOperationIr::BitwiseLeftShiftScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs, + out: desc.out.to_relative(converter), + }) + } + IntOperationIr::BitwiseRightShift(desc) => { + IntOperationIr::BitwiseRightShift(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + IntOperationIr::BitwiseRightShiftScalar(desc) => { + IntOperationIr::BitwiseRightShiftScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs, + out: desc.out.to_relative(converter), + }) + } + } + } +} + +impl RelativeOps for CustomOpIr { + fn to_relative(&self, converter: &mut OperationConverter) -> CustomOpIr { + let id = self.id.clone(); + + CustomOpIr { + id, + inputs: self + .inputs + .iter() + .map(|x| x.to_relative(converter)) + .collect(), + outputs: self + .outputs + .iter() + .map(|x| x.to_relative(converter)) + .collect(), + } + } +} + +impl RelativeOps for NumericOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + NumericOperationIr::Add(desc) => NumericOperationIr::Add(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::AddScalar(desc) => NumericOperationIr::AddScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Sub(desc) => NumericOperationIr::Sub(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::SubScalar(desc) => NumericOperationIr::SubScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Div(desc) => NumericOperationIr::Div(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::DivScalar(desc) => NumericOperationIr::DivScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Rem(desc) => NumericOperationIr::Rem(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::RemScalar(desc) => NumericOperationIr::RemScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Mul(desc) => NumericOperationIr::Mul(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::MulScalar(desc) => NumericOperationIr::MulScalar(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Abs(desc) => NumericOperationIr::Abs(UnaryOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Full(desc) => NumericOperationIr::Full(FullOpIr { + out: desc.out.to_relative(converter), + value: desc.value.to_relative(converter), + }), + NumericOperationIr::MeanDim(desc) => NumericOperationIr::MeanDim(ReduceDimOpIr { + input: desc.input.to_relative(converter), + axis: desc.axis, + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Mean(desc) => NumericOperationIr::Mean(ReduceOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Sum(desc) => NumericOperationIr::Sum(ReduceOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::SumDim(desc) => { + NumericOperationIr::SumDim(ReduceDimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, // Axis should stay the same. + }) + } + NumericOperationIr::Prod(desc) => NumericOperationIr::Prod(ReduceOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::ProdDim(desc) => NumericOperationIr::ProdDim(ReduceDimOpIr { + input: desc.input.to_relative(converter), + axis: desc.axis, + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Greater(desc) => NumericOperationIr::Greater(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::GreaterElem(desc) => NumericOperationIr::GreaterElem(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::GreaterEqual(desc) => { + NumericOperationIr::GreaterEqual(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + NumericOperationIr::GreaterEqualElem(desc) => { + NumericOperationIr::GreaterEqualElem(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + NumericOperationIr::Lower(desc) => NumericOperationIr::Lower(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::LowerElem(desc) => NumericOperationIr::LowerElem(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::LowerEqual(desc) => NumericOperationIr::LowerEqual(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::LowerEqualElem(desc) => { + NumericOperationIr::LowerEqualElem(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }) + } + NumericOperationIr::ArgMax(desc) => NumericOperationIr::ArgMax(ReduceDimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, // Axis should stay the same. + }), + NumericOperationIr::ArgMin(desc) => NumericOperationIr::ArgMin(ReduceDimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, // Axis should stay the same. + }), + NumericOperationIr::Max(desc) => NumericOperationIr::Max(ReduceOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::MaxDimWithIndices(desc) => { + NumericOperationIr::MaxDimWithIndices(ReduceDimWithIndicesOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + out: desc.out.to_relative(converter), + out_indices: desc.out_indices.to_relative(converter), + }) + } + NumericOperationIr::MinDimWithIndices(desc) => { + NumericOperationIr::MinDimWithIndices(ReduceDimWithIndicesOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + out: desc.out.to_relative(converter), + out_indices: desc.out_indices.to_relative(converter), + }) + } + NumericOperationIr::Min(desc) => NumericOperationIr::Min(ReduceOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::MaxDim(desc) => NumericOperationIr::MaxDim(ReduceDimOpIr { + input: desc.input.to_relative(converter), + axis: desc.axis, + out: desc.out.to_relative(converter), + }), + NumericOperationIr::MinDim(desc) => NumericOperationIr::MinDim(ReduceDimOpIr { + input: desc.input.to_relative(converter), + axis: desc.axis, + out: desc.out.to_relative(converter), + }), + NumericOperationIr::MaxAbs(desc) => NumericOperationIr::MaxAbs(ReduceOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::MaxAbsDim(desc) => NumericOperationIr::MaxAbsDim(ReduceDimOpIr { + input: desc.input.to_relative(converter), + axis: desc.axis, + out: desc.out.to_relative(converter), + }), + NumericOperationIr::Clamp(desc) => NumericOperationIr::Clamp(ClampOpIr { + tensor: desc.tensor.to_relative(converter), + min: desc.min.to_relative(converter), + max: desc.max.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::IntRandom(desc) => NumericOperationIr::IntRandom(RandomOpIr { + out: desc.out.to_relative(converter), + distribution: desc.distribution, + }), + NumericOperationIr::Powf(desc) => NumericOperationIr::Powf(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + NumericOperationIr::CumSum(desc) => NumericOperationIr::CumSum(DimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, + }), + NumericOperationIr::CumProd(desc) => NumericOperationIr::CumProd(DimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, + }), + NumericOperationIr::CumMin(desc) => NumericOperationIr::CumMin(DimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, + }), + NumericOperationIr::CumMax(desc) => NumericOperationIr::CumMax(DimOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axis: desc.axis, + }), + } + } +} + +impl RelativeOps for BaseOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + match self { + BaseOperationIr::Reshape(desc) => BaseOperationIr::Reshape(ShapeOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::SwapDims(desc) => BaseOperationIr::SwapDims(SwapDimsOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + dim1: desc.dim1, + dim2: desc.dim2, + }), + BaseOperationIr::Permute(desc) => BaseOperationIr::Permute(PermuteOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axes: desc.axes.clone(), + }), + BaseOperationIr::Expand(desc) => BaseOperationIr::Expand(ShapeOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Unfold(desc) => BaseOperationIr::Unfold(UnfoldOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + dim: desc.dim, + size: desc.size, + step: desc.step, + }), + BaseOperationIr::Flip(desc) => BaseOperationIr::Flip(FlipOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + axes: desc.axes.clone(), + }), + BaseOperationIr::Slice(desc) => BaseOperationIr::Slice(SliceOpIr { + tensor: desc.tensor.to_relative(converter), + ranges: desc.ranges.iter().map(|_info| Slice::from(0..1)).collect(), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::SliceAssign(desc) => BaseOperationIr::SliceAssign(SliceAssignOpIr { + tensor: desc.tensor.to_relative(converter), + ranges: desc.ranges.iter().map(|_range| Slice::from(0..1)).collect(), + value: desc.value.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Gather(desc) => BaseOperationIr::Gather(GatherOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + indices: desc.indices.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Scatter(desc) => BaseOperationIr::Scatter(ScatterOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + indices: desc.indices.to_relative(converter), + value: desc.value.to_relative(converter), + update: desc.update, + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Select(desc) => BaseOperationIr::Select(SelectOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + indices: desc.indices.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::SelectAssign(desc) => { + BaseOperationIr::SelectAssign(SelectAssignOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + indices: desc.indices.to_relative(converter), + value: desc.value.to_relative(converter), + update: desc.update, + out: desc.out.to_relative(converter), + }) + } + BaseOperationIr::MaskWhere(desc) => BaseOperationIr::MaskWhere(MaskWhereOpIr { + tensor: desc.tensor.to_relative(converter), + mask: desc.mask.to_relative(converter), + value: desc.value.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::MaskFill(desc) => BaseOperationIr::MaskFill(MaskFillOpIr { + tensor: desc.tensor.to_relative(converter), + mask: desc.mask.to_relative(converter), + value: desc.value.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Equal(desc) => BaseOperationIr::Equal(BinaryOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::EqualElem(desc) => BaseOperationIr::EqualElem(ScalarOpIr { + lhs: desc.lhs.to_relative(converter), + rhs: desc.rhs.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::RepeatDim(desc) => BaseOperationIr::RepeatDim(RepeatDimOpIr { + tensor: desc.tensor.to_relative(converter), + dim: desc.dim, + times: desc.times, + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Cat(desc) => BaseOperationIr::Cat(CatOpIr { + tensors: desc + .tensors + .iter() + .map(|tensor| tensor.to_relative(converter)) + .collect(), + dim: desc.dim, + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Cast(desc) => BaseOperationIr::Cast(CastOpIr { + input: desc.input.to_relative(converter), + out: desc.out.to_relative(converter), + }), + BaseOperationIr::Empty(desc) => BaseOperationIr::Empty(desc.to_relative(converter)), + BaseOperationIr::Ones(desc) => BaseOperationIr::Ones(desc.to_relative(converter)), + BaseOperationIr::Zeros(desc) => BaseOperationIr::Zeros(desc.to_relative(converter)), + } + } +} + +impl RelativeOps for InitOperationIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + Self { + out: self.out.to_relative(converter), + } + } +} + +impl RelativeOps for CreationOpIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + Self { + out: self.out.to_relative(converter), + } + } +} + +impl RelativeOps for TensorIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + let relative_id = self.id.to_relative(converter); + + // We can create relative shapes by mapping each shape found to an ID, which is a `usize`. + let mut relative_shape = Vec::with_capacity(self.shape.rank()); + for dim in self.shape.iter() { + if let Some(dim_id) = converter.shapes_global2relative.get(dim) { + // We already saw that dim value before, so we retrieve its ID. + relative_shape.push(*dim_id); + } else { + // We never saw this dim value before, therefore we create a new ID. + let dim_id = converter.shapes_global2relative.len(); + relative_shape.push(dim_id); + + converter.shapes_global2relative.insert(*dim, dim_id); + converter.shapes_relative2global.insert(dim_id, *dim); + } + } + + // We create the relative tensor. + let relative_tensor = TensorIr { + id: relative_id, + shape: Shape::from(relative_shape), + status: self.status, + dtype: self.dtype, + }; + + // We update both mappings. + converter + .tensors_relative2global + .insert(relative_id, self.clone()); + converter + .tensors_global2relative + .insert(self.id, relative_tensor.clone()); + + relative_tensor + } +} + +impl RelativeOps for TensorId { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + if let Some(value) = converter.tensors_global2relative.get(self) { + // If we already have the same tensor registered, we have to update its value, but not + // its id. + value.id + } else { + // We create a new relative id since we never seen this tensor in the graph before. + TensorId::new(converter.tensors_relative2global.len() as u64) + } + } +} + +impl RelativeOps for ScalarIr { + fn to_relative(&self, converter: &mut OperationConverter) -> Self { + if matches!(self, ScalarIr::Bool(_)) { + todo!("Unsupported dtype ({self:?}) for scalar") + } + + let id = ScalarId { + value: converter.scalars.len() as u64, + }; + + converter.scalars.insert(id, *self); + ScalarIr::UInt(id.value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn_backend::DType; + use burn_ir::{TensorId, TensorIr, TensorStatus}; + + #[test] + fn tensor_description_to_relative() { + let tensor1 = TensorIr { + id: TensorId::new(500), + shape: Shape::new([512, 32, 2048]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }; + let tensor2 = TensorIr { + id: TensorId::new(501), + shape: Shape::new([512, 128, 2048]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }; + let mut converter = OperationConverter::default(); + let tensor1_local = tensor1.to_relative(&mut converter); + let tensor2_local = tensor2.to_relative(&mut converter); + + assert_eq!( + tensor1_local, + TensorIr { + id: TensorId::new(0), + shape: Shape::new([1, 2, 3]), + status: TensorStatus::ReadOnly, + dtype: DType::F32 + } + ); + assert_eq!( + tensor2_local, + TensorIr { + id: TensorId::new(1), + shape: Shape::new([1, 4, 3]), + status: TensorStatus::ReadOnly, + dtype: DType::F32 + } + ); + } + + #[test] + fn scalar_ir_to_relative() { + let scalar1 = ScalarIr::Float(1.0); + let scalar2 = ScalarIr::UInt(1); + let mut converter = OperationConverter::default(); + let scalar1_local = scalar1.to_relative(&mut converter); + let scalar2_local = scalar2.to_relative(&mut converter); + + assert_eq!(scalar1_local, ScalarIr::UInt(0)); + assert_eq!(scalar2_local, ScalarIr::UInt(1)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/base.rs new file mode 100644 index 0000000..d76231e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/base.rs @@ -0,0 +1,16 @@ +use burn_ir::HandleContainer; + +use crate::FusionRuntime; + +/// The mode in which the execution is done. +#[derive(Clone, Copy, Debug)] +pub(crate) enum ExecutionMode { + Lazy, + Sync, +} + +/// General trait to abstract how a single operation is executed. +pub trait Operation: Send + Sync + core::fmt::Debug { + /// Execute the operation. + fn execute(&self, handles: &mut HandleContainer); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/explorer.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/explorer.rs new file mode 100644 index 0000000..b90f878 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/explorer.rs @@ -0,0 +1,91 @@ +use burn_ir::OperationIr; + +use super::ExecutionMode; +use crate::{ + NumOperations, OperationFuser, + search::{BlockOptimization, StreamOptimizer}, +}; + +/// Explore and create new optimization. +pub struct Explorer { + optimizer: StreamOptimizer, + num_deferred: usize, + num_explored: usize, + is_still_optimizing: bool, +} + +/// The result of an exploration done by the [explorer](Explorer). +pub enum ExplorationAction { + /// Found a new optimization. + Completed(BlockOptimization), + /// We should continue exploring before arriving at a conclusion. + Continue, +} + +impl Explorer { + /// Create a new explorer. + pub(crate) fn new(optimizations: Vec>>) -> Self { + Self { + optimizer: StreamOptimizer::new(optimizations), + num_deferred: 0, + num_explored: 0, + is_still_optimizing: true, + } + } + + /// Indicate that a new operation is added. + pub(crate) fn on_new_operation(&mut self) { + self.num_deferred += 1; + } + + /// If the explorer is up to date. + pub(crate) fn is_up_to_date(&self) -> bool { + self.num_deferred == 0 + } + + /// Explore the provided operations. + pub(crate) fn explore( + &mut self, + operations: &[OperationIr], + mode: ExecutionMode, + ) -> ExplorationAction { + self.update(operations); + + // Can only continue exploration when not sync. + if let ExecutionMode::Lazy = mode + && self.is_still_optimizing + { + return ExplorationAction::Continue; + } + + let optimization = self.optimizer.optimize(operations); + + ExplorationAction::Completed(optimization) + } + + /// Reset the state of the explorer to the provided list of operations. + pub(crate) fn reset(&mut self, operations: &[OperationIr]) { + self.optimizer.reset(); + self.num_explored = 0; + self.num_deferred = operations.len(); + self.is_still_optimizing = true; + } + + /// Register any operations that we had deferred + fn update(&mut self, operations: &[OperationIr]) { + for i in (0..self.num_deferred).rev() { + if !self.is_still_optimizing { + break; + } + let index = operations.len() - 1 - i; + let relative = &operations[index]; + + self.optimizer.register(relative); + self.num_explored += 1; + + self.is_still_optimizing = self.optimizer.still_optimizing(); + } + + self.num_deferred = 0; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/mod.rs new file mode 100644 index 0000000..736ddf7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/mod.rs @@ -0,0 +1,17 @@ +pub(crate) mod validator; + +mod base; +mod explorer; +mod ordering; +mod policy; +mod processor; + +pub use base::*; +pub use ordering::*; + +pub(crate) use explorer::*; +pub(crate) use policy::*; +pub(crate) use processor::*; + +#[cfg(test)] +pub(crate) mod tests; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/ordering.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/ordering.rs new file mode 100644 index 0000000..b870c4a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/ordering.rs @@ -0,0 +1,71 @@ +use std::sync::Arc; + +use burn_ir::HandleContainer; + +use crate::{FusionRuntime, NumOperations, Optimization, stream::Context}; + +use super::Operation; + +/// Manage the execution of potentially multiple optimizations and operations out of order. +pub struct OrderedExecution { + operations: Vec>>, + num_executed: usize, + ordering: Option>>, +} + +impl OrderedExecution { + /// Returns the operation that can be executed without impacting the state of the execution. + /// + /// This is useful to implement fallback for optimizations. + #[allow(clippy::borrowed_box)] + pub fn operation_within_optimization(&self, index: usize) -> Arc> { + match &self.ordering { + Some(val) => { + let index = val[index]; + self.operations[index].clone() + } + None => panic!("No ordering provided"), + } + } + + pub(crate) fn new(operations: Vec>>) -> Self { + Self { + operations, + num_executed: 0, + ordering: None, + } + } + + pub(crate) fn finish(mut self) -> (Vec>>, usize) { + self.operations.drain(0..self.num_executed); + (self.operations, self.num_executed) + } + + pub(crate) fn execute_optimization( + &mut self, + optimization: &mut R::Optimization, + context: &mut Context<'_, R::FusionHandle>, + ordering: Arc>, + ) { + if ordering.len() > self.operations.len() { + panic!("Ordering is bigger than operations"); + } + self.ordering = Some(ordering); + let num_drained = optimization.len(); + optimization.execute(context, self); + self.num_executed += num_drained; + } + + pub(crate) fn execute_operations( + &mut self, + handles: &mut HandleContainer, + ordering: &[usize], + ) { + self.num_executed += ordering.len(); + + for id in ordering { + let op = &self.operations[*id]; + op.execute(handles); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/policy.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/policy.rs new file mode 100644 index 0000000..14c21b8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/policy.rs @@ -0,0 +1,572 @@ +use burn_ir::OperationIr; + +use super::ExecutionMode; +use super::validator::{ + ExecutionPlanOperationsStore, TriggerOperationsStore, TriggerProgress, TriggerValidator, + ValidatorState, +}; +use crate::stream::execution::validator::OperationsValidator; +use crate::stream::store::{ExecutionPlanId, ExecutionPlanStore, ExecutionTrigger, SearchQuery}; +use std::marker::PhantomData; + +/// The policy keeps track of all possible execution plans for the current operations. +/// +/// # Details +/// +/// We keep track of each new operation added and invalidate potential execution plans +/// when we see a different operation is added. +/// +/// Therefore, the overhead is very minimal, since the time-complexity of checking for existing +/// execution plans scales with the number of concurrent potential plans for the current operations, +/// which isn't supposed to be big at any time. +pub(crate) struct Policy { + /// List of potential execution plans that are compatible with current stream segment + candidates: Vec>, + /// List of candidate execution plans that have been found; we can still keep searching + /// to potentially find a better one. + availables: Vec, + /// The found execution plan that should be executed, along with the number of operations + /// in the plan. + found: Option<(ExecutionPlanId, usize)>, + /// The number of operations that have been analyzed + num_operations: usize, + _item_type: PhantomData, +} + +#[derive(new)] +struct AvailableItem { + id: ExecutionPlanId, + size: usize, + triggers: Vec, +} + +/// Action to be made depending on the stream. +#[derive(PartialEq, Eq, Debug)] +pub enum Action { + /// Continue exploring using the [builder](crate::OptimizationBuilder). + Explore, + /// The current policy indicates that an exploration may be possible in the future, so the + /// best action is to defer any execution. + /// + /// Sometimes, it can be a false positive and a new exploration should be built from scratch. + /// Therefore it's important to keep the previous operations to rebuild the state if it + /// happens. + Defer, + /// An exploration has been found, and the best action is to execute it! + Execute(ExecutionPlanId), +} + +impl Policy { + /// Create a new policy. + pub(crate) fn new() -> Self { + Self { + candidates: Vec::new(), + availables: Vec::new(), + found: None, + num_operations: 0, + _item_type: PhantomData, + } + } + + /// Returns the [action](Action) that should be taken given the state of the policy. + pub fn action( + &self, + store: &ExecutionPlanStore, + operations: &[OperationIr], + mode: ExecutionMode, + ) -> Action { + if self.num_operations < operations.len() { + panic!( + "Internal Error: Can't retrieve the policy action on a list of operations bigger than what is analyzed." + ); + } + + if let Some((id, _length)) = self.found { + return Action::Execute(id); + } + + match mode { + ExecutionMode::Lazy => self.action_lazy(operations), + ExecutionMode::Sync => self.action_sync(operations, store), + } + } + + /// Update the policy state. + pub fn update(&mut self, store: &ExecutionPlanStore, operation: &OperationIr) { + // reset the candidates to contain all execution plans starting with the operation. + if self.num_operations == 0 { + self.candidates = store + .find(SearchQuery::PlansStartingWith(operation)) + .into_iter() + .map(OperationsValidator::new) + .collect(); + } + + self.update_candidates(store, operation); + self.check_candidates(store); + + self.update_availables(store, operation); + self.check_availables(); + self.num_operations += 1; + } + + // Reset the state of the policy. + pub fn reset(&mut self) { + self.candidates.clear(); + self.availables.clear(); + + self.num_operations = 0; + self.found = None; + } + + /// Check which candidates can be removed, and which one can go from + /// 'candidate' to 'available' + fn check_candidates(&mut self, store: &ExecutionPlanStore) { + let mut candidates_to_remove = Vec::new(); + + for candidate in self.candidates.iter() { + match candidate.state { + ValidatorState::Found { size } => { + let item = store.get_unchecked(candidate.id); + let mut triggers = Vec::with_capacity(item.triggers.len()); + + for (index, trigger) in item.triggers.iter().enumerate() { + triggers.push(match trigger { + ExecutionTrigger::OnOperations(_) => TriggerValidator::OnOperations { + matching: OperationsValidator::new(index), + progress: TriggerProgress::NotInit, + }, + ExecutionTrigger::OnSync => TriggerValidator::OnSync, + ExecutionTrigger::Always => TriggerValidator::Always, + }); + } + + self.availables + .push(AvailableItem::new(candidate.id, size, triggers)); + candidates_to_remove.push(candidate.id); + } + ValidatorState::Invalidated => { + candidates_to_remove.push(candidate.id); + } + ValidatorState::Validating => {} + }; + } + + let mut updated_candidates = Vec::new(); + core::mem::swap(&mut updated_candidates, &mut self.candidates); + + self.candidates = updated_candidates + .into_iter() + .filter(|candidate| !candidates_to_remove.iter().any(|id| id == &candidate.id)) + .collect(); + } + + fn check_availables(&mut self) { + for available in self.availables.iter() { + for trigger in available.triggers.iter() { + match trigger { + TriggerValidator::OnOperations { + matching, + progress: _, + } => { + if let ValidatorState::Found { + size: _size_of_trigger, + } = matching.state + { + self.found = Some((available.id, available.size)); + return; + } + } + TriggerValidator::Always => { + self.found = Some((available.id, available.size)); + return; + } + TriggerValidator::OnSync => { + // Does nothing during an update. + } + } + } + } + } + + fn update_candidates(&mut self, store: &ExecutionPlanStore, operation: &OperationIr) { + let main_store = ExecutionPlanOperationsStore::new(store); + + self.candidates + .iter_mut() + .for_each(|candidate| candidate.update(operation, self.num_operations, &main_store)); + } + + fn update_availables(&mut self, store: &ExecutionPlanStore, operation: &OperationIr) { + self.availables.iter_mut().for_each(|available| { + let store_trigger = TriggerOperationsStore::new(available.id, store); + + available.triggers.iter_mut().for_each(|trigger| { + if let TriggerValidator::OnOperations { matching, progress } = trigger { + match progress { + TriggerProgress::NotInit => { + *progress = TriggerProgress::NumChecked(0); + } + TriggerProgress::NumChecked(num_check) => { + matching.update(operation, *num_check, &store_trigger); + *num_check += 1; + } + } + } + }); + }); + } + + fn action_lazy(&self, operations: &[OperationIr]) -> Action { + if !self.candidates.is_empty() { + return Action::Defer; + } + + for available in self.availables.iter() { + if available.size == operations.len() { + return Action::Defer; + } + + for trigger in available.triggers.iter() { + if let TriggerValidator::OnOperations { + matching, + progress: _, + } = trigger + && let ValidatorState::Validating = matching.state + { + return Action::Defer; + } + } + } + + Action::Explore + } + + fn action_sync(&self, operations: &[OperationIr], store: &ExecutionPlanStore) -> Action { + for available in self.availables.iter() { + if available.size == operations.len() { + return Action::Execute(available.id); + } + } + + for candidate in self.candidates.iter() { + let item = store.get_unchecked(candidate.id); + + if item.operations.len() == operations.len() { + return Action::Execute(candidate.id); + } + } + + Action::Explore + } +} + +#[cfg(test)] +mod tests { + use burn_backend::{DType, Shape}; + use burn_ir::{FloatOperationIr, TensorId, TensorIr, TensorStatus, UnaryOpIr}; + + use super::*; + use crate::{ + search::BlockOptimization, + stream::store::{ExecutionPlan, ExecutionStrategy, ExecutionTrigger}, + }; + use std::ops::Range; + + #[test] + fn given_no_optimization_should_explore() { + let store = ExecutionPlanStore::default(); + let mut policy = Policy::new(); + let stream = TestStream::new(3); + + stream.assert_updates( + &store, + &mut policy, + AssertUpdatesOptions::OperationsIndex(0..3), + Action::Explore, + ); + } + + #[test] + fn given_existing_optimizations_when_sync_should_execute_one_when_available() { + let mut store = ExecutionPlanStore::default(); + let mut policy = Policy::new(); + let stream = TestStream::new(3); + + let id_1 = store.add(ExecutionPlan { + operations: stream.operations[0..2].to_vec(), + triggers: Vec::new(), + optimization: BlockOptimization::new(ExecutionStrategy::operations(2), Vec::new()), + }); + let _id_2 = store.add(ExecutionPlan { + operations: stream.operations[0..3].to_vec(), + triggers: Vec::new(), + optimization: BlockOptimization::new(ExecutionStrategy::operations(3), Vec::new()), + }); + + stream.assert_updates( + &store, + &mut policy, + AssertUpdatesOptions::OperationsIndex(0..2), + Action::Defer, + ); + + let action = policy.action(&store, &stream.operations[0..2], ExecutionMode::Sync); + assert_eq!(action, Action::Execute(id_1)); + } + + #[test] + fn given_existing_plan_when_found_trigger_should_execute_plan() { + let mut store = ExecutionPlanStore::default(); + let mut policy = Policy::new(); + + let stream = TestStream::new(3); + let id = store.add(ExecutionPlan { + operations: stream.operations[0..2].to_vec(), + triggers: stream.operations[2..3] + .iter() + .map(|desc| ExecutionTrigger::OnOperations(vec![desc.clone()])) + .collect(), + optimization: BlockOptimization::new(ExecutionStrategy::operations(2), Vec::new()), + }); + + stream.assert_updates( + &store, + &mut policy, + AssertUpdatesOptions::OperationsIndex(0..2), + Action::Defer, + ); + stream.assert_updates( + &store, + &mut policy, + AssertUpdatesOptions::OperationsIndex(2..3), + Action::Execute(id), + ); + } + + #[test] + fn should_support_multiple_triggers() { + let mut store = ExecutionPlanStore::default(); + let mut policy_1 = Policy::new(); + let mut policy_2 = Policy::new(); + + let mut stream_1 = TestStream::new(2); + let mut stream_2 = TestStream::new(2); + + // Create different end operation for each stream. + let trigger_id_1 = 5; + let trigger_id_2 = 6; + stream_1.new_ops(trigger_id_1); + stream_2.new_ops(trigger_id_2); + + let id = store.add(ExecutionPlan { + operations: stream_1.operations[0..2].to_vec(), + triggers: vec![ + ExecutionTrigger::OnOperations(vec![stream_1.operations[2].clone()]), + ExecutionTrigger::OnOperations(vec![stream_2.operations[2].clone()]), + ], + optimization: BlockOptimization::new(ExecutionStrategy::operations(2), Vec::new()), + }); + + stream_1.assert_updates( + &store, + &mut policy_1, + AssertUpdatesOptions::OperationsIndex(0..2), + Action::Defer, + ); + stream_2.assert_updates( + &store, + &mut policy_2, + AssertUpdatesOptions::OperationsIndex(0..2), + Action::Defer, + ); + + stream_1.assert_updates( + &store, + &mut policy_1, + AssertUpdatesOptions::OperationsIndex(2..3), // First trigger. + Action::Execute(id), + ); + stream_2.assert_updates( + &store, + &mut policy_2, + AssertUpdatesOptions::OperationsIndex(2..3), // Second trigger. + Action::Execute(id), + ); + } + + #[test] + fn should_select_right_optimization() { + let mut store = ExecutionPlanStore::default(); + let mut policy_1 = Policy::new(); + let mut policy_2 = Policy::new(); + + let mut stream_1 = TestStream::new(2); + let mut stream_2 = TestStream::new(2); + + // Create different streams after op 2. + stream_1.new_ops(4); + stream_1.new_ops(5); + + stream_2.new_ops(5); + stream_2.new_ops(6); + + let optimization_stream_1 = store.add(ExecutionPlan { + operations: stream_1.operations[0..3].to_vec(), + triggers: stream_1.operations[3..4] + .iter() + .map(|desc| ExecutionTrigger::OnOperations(vec![desc.clone()])) + .collect(), + optimization: BlockOptimization::new(ExecutionStrategy::operations(3), Vec::new()), + }); + let optimization_stream_2 = store.add(ExecutionPlan { + operations: stream_2.operations[0..3].to_vec(), + triggers: stream_2.operations[3..4] + .iter() + .map(|desc| ExecutionTrigger::OnOperations(vec![desc.clone()])) + .collect(), + optimization: BlockOptimization::new(ExecutionStrategy::operations(3), Vec::new()), + }); + assert_ne!(optimization_stream_1, optimization_stream_2); + + stream_1.assert_updates( + &store, + &mut policy_1, + AssertUpdatesOptions::OperationsIndex(0..3), + Action::Defer, + ); + stream_2.assert_updates( + &store, + &mut policy_2, + AssertUpdatesOptions::OperationsIndex(0..3), + Action::Defer, + ); + + stream_1.assert_updates( + &store, + &mut policy_1, + AssertUpdatesOptions::OperationsIndex(3..4), + Action::Execute(optimization_stream_1), + ); + stream_2.assert_updates( + &store, + &mut policy_2, + AssertUpdatesOptions::OperationsIndex(3..4), + Action::Execute(optimization_stream_2), + ); + } + + #[test] + fn should_invalidate_wrong_optimizations() { + let mut store = ExecutionPlanStore::default(); + let stream_1 = TestStream::new(4); + let mut stream_2 = TestStream::new(2); + stream_2.new_ops(6); + stream_2.new_ops(7); + + store.add(ExecutionPlan { + operations: stream_1.operations[0..3].to_vec(), + triggers: stream_1.operations[3..4] + .iter() + .map(|desc| ExecutionTrigger::OnOperations(vec![desc.clone()])) + .collect(), + optimization: BlockOptimization::new(ExecutionStrategy::operations(3), Vec::new()), + }); + + let mut policy = Policy::new(); + // Same path as stream 1 + stream_2.assert_updates( + &store, + &mut policy, + AssertUpdatesOptions::OperationsIndex(0..2), + Action::Defer, + ); + + // But is different. + stream_2.assert_updates( + &store, + &mut policy, + AssertUpdatesOptions::OperationsIndex(2..4), + Action::Explore, + ); + } + + #[derive(Default, Debug)] + struct TestStream { + tensors: Vec, + operations: Vec, + } + + #[derive(Debug)] + enum AssertUpdatesOptions { + OperationsIndex(Range), + } + + impl TestStream { + /// Create a new test stream with `num_ops` operations registered. + pub fn new(num_ops: usize) -> Self { + let mut stream = Self::default(); + for id in 0..num_ops { + stream.new_ops(id as u64 + 1); + } + + stream + } + + /// The first follow should only be cache miss. + pub fn assert_updates( + &self, + optimizations: &ExecutionPlanStore<()>, + policy: &mut Policy<()>, + options: AssertUpdatesOptions, + action: Action, + ) { + match options { + AssertUpdatesOptions::OperationsIndex(range) => { + for i in range { + let stream = &self.operations[0..i]; + let next_ops = &self.operations[i]; + policy.update(optimizations, next_ops); + let result = policy.action(optimizations, stream, ExecutionMode::Lazy); + + assert_eq!(result, action); + } + } + } + } + + /// Add a simple operation to the stream. + pub fn new_ops(&mut self, out_id: u64) { + if self.tensors.is_empty() { + // Root node. + self.new_empty_node(0); + } + + // Out node. + self.new_empty_node(out_id); + + self.operations.push(OperationIr::Float( + DType::F32, + FloatOperationIr::Log(self.unary_description()), + )); + } + + fn new_empty_node(&mut self, id: u64) { + self.tensors.push(TensorIr { + id: TensorId::new(id), + shape: Shape::new([32, 32, 1]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }); + } + + fn unary_description(&self) -> UnaryOpIr { + let size = self.tensors.len(); + + UnaryOpIr { + input: self.tensors[size - 2].clone(), + out: self.tensors[size - 1].clone(), + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/processor.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/processor.rs new file mode 100644 index 0000000..a363e66 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/processor.rs @@ -0,0 +1,184 @@ +use burn_ir::OperationIr; + +use super::{ExecutionMode, ExplorationAction, Explorer}; +use crate::search::BlockOptimization; +use crate::stream::execution::{Action, Policy}; +use crate::stream::store::{ExecutionPlan, ExecutionPlanId, ExecutionPlanStore, ExecutionTrigger}; +use crate::{NumOperations, OperationFuser}; + +/// Process a [stream segment](StreamSegment) following a [policy](Policy). +pub(crate) struct Processor { + policy: Policy, + explorer: Explorer, +} + +/// A part of a stream that can be executed partially using [execution plan](ExecutionPlan). +pub(crate) trait StreamSegment { + /// The operations in the segment. + fn operations(&self) -> &[OperationIr]; + /// Execute part of the segment using the given plan id. + fn execute(&mut self, id: ExecutionPlanId, store: &mut ExecutionPlanStore); +} + +impl Processor { + /// Create a new stream processor. + pub fn new(optimizations: Vec>>) -> Self { + Self { + policy: Policy::new(), + explorer: Explorer::new(optimizations), + } + } + + /// Process the [stream segment](StreamSegment) with the provided [mode](ExecutionMode). + pub fn process( + &mut self, + mut segment: Segment, + store: &mut ExecutionPlanStore, + mode: ExecutionMode, + ) where + Segment: StreamSegment, + { + // We assume that we always register a new operation in lazy mode. + if let ExecutionMode::Lazy = mode { + self.on_new_operation(&segment, store); + } + + loop { + if segment.operations().is_empty() { + break; + } + + let action = self.policy.action(store, segment.operations(), mode); + + match action { + Action::Explore => { + self.explore(&mut segment, store, mode); + + if self.explorer.is_up_to_date() { + break; + } + } + Action::Defer => { + match mode { + ExecutionMode::Lazy => break, + ExecutionMode::Sync => panic!("Can't defer while sync"), + }; + } + Action::Execute(id) => { + if let ExecutionMode::Sync = mode { + store.add_trigger(id, ExecutionTrigger::OnSync); + } + + segment.execute(id, store); + self.reset(store, segment.operations()); + } + }; + } + } + + fn on_new_operation(&mut self, segment: &Segment, store: &mut ExecutionPlanStore) + where + Segment: StreamSegment, + { + self.policy.update( + store, + segment + .operations() + .last() + .expect("At least one operation in the operation list."), + ); + self.explorer.on_new_operation(); + } + + fn explore>( + &mut self, + item: &mut Item, + store: &mut ExecutionPlanStore, + mode: ExecutionMode, + ) { + match self.explorer.explore(item.operations(), mode) { + ExplorationAction::Completed(optim) => { + let id = Self::on_exploration_completed( + &self.policy, + item.operations(), + store, + optim, + mode, + ); + item.execute(id, store); + self.reset(store, item.operations()); + } + ExplorationAction::Continue => { + if let ExecutionMode::Sync = mode { + panic!("Can't continue exploring when sync.") + } + } + } + } + + fn reset(&mut self, store: &mut ExecutionPlanStore, operations: &[OperationIr]) { + self.explorer.reset(operations); + self.policy.reset(); + + // Reset the policy state with the remaining operations + for operation in operations.iter() { + self.policy.update(store, operation); + } + } + + /// We found an optimization (i.e. a new execution plan). + /// Cache it in the store. + fn on_exploration_completed( + policy: &Policy, + operations: &[OperationIr], + store: &mut ExecutionPlanStore, + optimization: BlockOptimization, + mode: ExecutionMode, + ) -> ExecutionPlanId { + let num_optimized = optimization.ordering.len(); + let relative = &operations[0..num_optimized]; + + match mode { + ExecutionMode::Lazy => { + let next_ops = &operations[num_optimized..operations.len()]; + + let trigger = if next_ops.is_empty() { + // Happens if the next ops is included in the fused operation, and there is no + // way the builder can still continue fusing. + ExecutionTrigger::Always + } else { + ExecutionTrigger::OnOperations(next_ops.to_vec()) + }; + + match policy.action(store, relative, ExecutionMode::Sync) { + Action::Execute(id) => { + store.add_trigger(id, trigger); + id + } + _ => { + let plan = ExecutionPlan { + operations: relative.to_vec(), + triggers: vec![trigger], + optimization, + }; + store.add(plan) + } + } + } + ExecutionMode::Sync => match policy.action(store, relative, ExecutionMode::Sync) { + Action::Execute(id) => { + store.add_trigger(id, ExecutionTrigger::OnSync); + id + } + _ => { + let plan = ExecutionPlan { + operations: relative.to_vec(), + triggers: vec![ExecutionTrigger::OnSync], + optimization, + }; + store.add(plan) + } + }, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/tests.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/tests.rs new file mode 100644 index 0000000..b945481 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/tests.rs @@ -0,0 +1,671 @@ +//! A testing module that ensures the correctness of the explorer, policy, and processor. +//! +//! The primary focus is on validating the seamless interaction between these three components to +//! execute and optimize a stream of operations accurately. +//! +//! To test these components effectively, we create mock types for the stream, optimization, +//! optimization builder, and stream segment. These mock types aid in comprehensively +//! understanding the process of optimizing streams. +use std::sync::Arc; + +use burn_backend::{DType, Shape}; +use burn_ir::{ + BinaryOpIr, FloatOperationIr, NumericOperationIr, OperationIr, ScalarIr, ScalarOpIr, TensorId, + TensorIr, TensorStatus, UnaryOpIr, +}; + +use crate::{ + FuserProperties, FuserStatus, NumOperations, OperationFuser, + search::BlockOptimization, + stream::store::{ + ExecutionPlan, ExecutionPlanId, ExecutionPlanStore, ExecutionStrategy, ExecutionTrigger, + }, +}; + +use super::*; + +/// A fake stream of operations for testing purpose. +pub struct TestStream { + processor: Processor, + store: ExecutionPlanStore, + executed: Vec, + operations: Vec, +} + +/// A fake [optimization builder](OptimizationBuilder) for testing purpose. +/// +/// The optimizer tries to fuse only the `expected_operations` if they appear +/// in the operations queue +#[derive(Clone)] +pub struct TestOptimizationBuilder { + builder_id: usize, + expected_operations: Vec, + actual: Vec, +} + +/// A fake optimization for testing purpose. +#[derive(new, Debug, PartialEq)] +pub struct TestOptimization { + builder_id: usize, + size: usize, +} + +impl NumOperations for TestOptimization { + fn len(&self) -> usize { + self.size + } +} + +/// A fake [stream segment](StreamSegment) for testing purpose. +#[derive(new)] +pub struct TestSegment<'i> { + operations: &'i mut Vec, + executed: &'i mut Vec, +} + +impl ExecutionStrategy { + /// Create an ordered execution strategy with the given size. + pub fn operations(size: usize) -> Self { + Self::Operations { + ordering: Arc::new((0..size).collect()), + } + } +} + +impl ExecutionStrategy { + /// Only use it for testing, to easily create ordered strategies. + pub fn optimization(opt: TestOptimization) -> Self { + let ordering = Arc::new((0..opt.size).collect()); + Self::Optimization { opt, ordering } + } +} + +/// This is a substantial test case that examines a lengthy scenario with a diverse set of conditions. +/// +/// While it's usually preferable to split tests into multiple independent scenarios, in this case, it is +/// crucial to verify that the stream's state is correctly updated when various cases occur consecutively. +#[test] +fn should_support_complex_stream() { + // We have 2 different optimization builders in this test case. + let builder_id_1 = 0; + let builder_id_2 = 1; + + // We will have a total of 3 execution plans to execute. + let plan_id_1 = 0; + let plan_id_2 = 1; + let plan_id_3 = 2; + + let builder_1 = TestOptimizationBuilder::new(builder_id_1, vec![operation_1(), operation_2()]); + let builder_2 = TestOptimizationBuilder::new(builder_id_2, vec![operation_2(), operation_2()]); + let mut stream = TestStream::new(vec![Box::new(builder_1), Box::new(builder_2)]); + + // builder_1 is still waiting to see next op is operation_2 + // builder_2 is closed because it's not the right operation + stream.add(operation_1()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(0); + + // No optimization found for the first two operations. + stream.add(operation_1()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(1); + stream.assert_last_executed(plan_id_1); + stream.assert_plan( + plan_id_1, + ExecutionPlan { + operations: vec![operation_1(), operation_1()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization::new(ExecutionStrategy::operations(2), Vec::new()), + }, + ); + + // Nothing to execute. + stream.add(operation_1()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(1); + + // Now we should trigger the first optimization builder. + stream.add(operation_2()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(2); + stream.assert_last_executed(plan_id_2); + stream.assert_plan( + plan_id_2, + ExecutionPlan { + operations: vec![operation_1(), operation_2()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization::new( + ExecutionStrategy::optimization(TestOptimization::new(builder_id_1, 2)), + vec![0, 1], + ), + }, + ); + + // Nothing to execute. + stream.add(operation_2()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(2); + + // Now we should trigger the second optimization builder. + stream.add(operation_2()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(3); + stream.assert_last_executed(plan_id_3); + stream.assert_plan( + plan_id_3, + ExecutionPlan { + operations: vec![operation_2(), operation_2()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::optimization(TestOptimization::new(builder_id_2, 2)), + ordering: vec![0, 1], + }, + }, + ); + + // Nothing to execute. + stream.add(operation_1()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(3); + + // Now we should trigger the first optimization builder (second plan). + stream.add(operation_2()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(4); + stream.assert_last_executed(plan_id_2); + stream.assert_plan( + plan_id_2, + ExecutionPlan { + operations: vec![operation_1(), operation_2()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::optimization(TestOptimization::new(builder_id_1, 2)), + ordering: vec![0, 1], + }, + }, + ); + + // Nothing to execute. + stream.add(operation_2()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(4); + + // Now we should trigger the first optimization builder (third plan). + stream.add(operation_2()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(5); + stream.assert_last_executed(plan_id_3); +} + +/// In this scenario we will never use an optimization, but we check that we reuse the execution plan stored. +#[test] +fn should_reuse_basic_operations() { + let builder_id_1 = 0; + let plan_id_1 = 0; + let plan_id_2 = 1; + + let builder_1 = TestOptimizationBuilder::new(builder_id_1, vec![operation_1(), operation_2()]); + let mut stream = TestStream::new(vec![Box::new(builder_1)]); + + stream.add(operation_3()); + stream.assert_last_executed(plan_id_1); + stream.assert_number_of_operations(0); + stream.assert_plan( + plan_id_1, + ExecutionPlan { + operations: vec![operation_3()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::operations(1), + ordering: vec![0], + }, + }, + ); + + stream.add(operation_3()); + stream.assert_last_executed(plan_id_1); + stream.assert_number_of_operations(0); + stream.assert_plan( + plan_id_1, + ExecutionPlan { + operations: vec![operation_3()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::operations(1), + ordering: vec![0], + }, + }, + ); + + // Lazy try to build optimization 1. + stream.add(operation_1()); + // But not possible. + stream.add(operation_3()); + + // Creates a new plan with both operations. + stream.assert_plan( + plan_id_2, + ExecutionPlan { + operations: vec![operation_1(), operation_3()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::operations(2), + ordering: vec![0], + }, + }, + ); + stream.assert_number_of_operations(0); + stream.assert_last_executed(plan_id_2); +} + +// In this scenario we validate that we support multiple optimization builders with overlapping +// operations. +// +// This is a very long scenario that validates a lot of things. +#[test] +fn should_support_overlapping_optimizations() { + // We have 2 different optimization builders in this test case. + let builder_id_1 = 0; + let builder_id_2 = 0; + + // We will have a total of 5 execution plans to execute. + let plan_id_1 = 0; + let plan_id_2 = 1; + let plan_id_3 = 2; + let plan_id_4 = 3; + let plan_id_5 = 4; + + let builder_1 = TestOptimizationBuilder::new(builder_id_1, vec![operation_1(), operation_2()]); + let builder_2 = TestOptimizationBuilder::new( + builder_id_2, + vec![operation_1(), operation_2(), operation_1(), operation_1()], + ); + let mut stream = TestStream::new(vec![Box::new(builder_1), Box::new(builder_2)]); + + stream.add(operation_1()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(0); + + stream.add(operation_2()); + stream.assert_number_of_operations(2); + stream.assert_number_of_executions(0); + + stream.add(operation_1()); + stream.assert_number_of_operations(3); + stream.assert_number_of_executions(0); + + stream.add(operation_2()); + stream.assert_number_of_operations(2); + stream.assert_number_of_executions(1); + stream.assert_last_executed(plan_id_1); + stream.assert_plan( + plan_id_1, + ExecutionPlan { + operations: vec![operation_1(), operation_2()], + triggers: vec![ExecutionTrigger::OnOperations(vec![ + operation_1(), + operation_2(), + ])], + optimization: BlockOptimization { + strategy: ExecutionStrategy::optimization(TestOptimization::new(builder_id_1, 2)), + ordering: vec![0, 1], + }, + }, + ); + + stream.add(operation_2()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(3); + stream.assert_plan( + plan_id_1, + ExecutionPlan { + operations: vec![operation_1(), operation_2()], + triggers: vec![ + ExecutionTrigger::OnOperations(vec![operation_1(), operation_2()]), + ExecutionTrigger::OnOperations(vec![operation_2()]), + ], + optimization: BlockOptimization { + strategy: ExecutionStrategy::optimization(TestOptimization::new(builder_id_1, 2)), + ordering: vec![0, 1], + }, + }, + ); + stream.assert_plan( + plan_id_2, + ExecutionPlan { + operations: vec![operation_2()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::operations(1), + ordering: vec![0], + }, + }, + ); + + stream.add(operation_1()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(3); + + stream.add(operation_2()); + stream.assert_number_of_operations(2); + stream.assert_number_of_executions(3); + + stream.add(operation_1()); + stream.assert_number_of_operations(3); + stream.assert_number_of_executions(3); + + stream.add(operation_1()); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(4); + + stream.assert_plan( + plan_id_3, + ExecutionPlan { + operations: vec![operation_1(), operation_2(), operation_1(), operation_1()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::optimization(TestOptimization::new(builder_id_1, 4)), + ordering: vec![0], + }, + }, + ); + + stream.add(operation_1()); + stream.assert_number_of_operations(1); + stream.assert_number_of_executions(4); + + stream.add(operation_2()); + stream.assert_number_of_operations(2); + stream.assert_number_of_executions(4); + + stream.add(operation_1()); + stream.assert_number_of_operations(3); + stream.assert_number_of_executions(4); + + stream.sync(); + stream.assert_number_of_operations(0); + stream.assert_number_of_executions(6); + stream.assert_plan( + plan_id_1, + ExecutionPlan { + operations: vec![operation_1(), operation_2()], + triggers: vec![ + ExecutionTrigger::OnOperations(vec![operation_1(), operation_2()]), + ExecutionTrigger::OnOperations(vec![operation_2()]), + ExecutionTrigger::OnSync, + ], + optimization: BlockOptimization { + strategy: ExecutionStrategy::optimization(TestOptimization::new(builder_id_1, 2)), + ordering: vec![0, 1], + }, + }, + ); + stream.assert_plan( + plan_id_4, + ExecutionPlan { + operations: vec![operation_1()], + triggers: vec![ExecutionTrigger::OnSync], + optimization: BlockOptimization { + strategy: ExecutionStrategy::operations(1), + ordering: vec![0], + }, + }, + ); + + stream.add(operation_3()); + stream.assert_last_executed(plan_id_5); + stream.assert_plan( + plan_id_5, + ExecutionPlan { + operations: vec![operation_3()], + triggers: vec![ExecutionTrigger::Always], + optimization: BlockOptimization { + strategy: ExecutionStrategy::operations(1), + ordering: vec![0], + }, + }, + ); + + stream.add(operation_3()); + stream.assert_last_executed(plan_id_5); +} + +impl TestStream { + /// Create a new stream with the given optimization builders. + fn new(optimizations: Vec>>) -> Self { + Self { + processor: Processor::::new(optimizations), + store: ExecutionPlanStore::::new(), + executed: Vec::new(), + operations: Vec::new(), + } + } + + /// Add an operation to the stream. + fn add(&mut self, operation: OperationIr) { + self.operations.push(operation); + self.processor.process( + TestSegment::new(&mut self.operations, &mut self.executed), + &mut self.store, + ExecutionMode::Lazy, + ); + } + + /// Sync the stream. + fn sync(&mut self) { + self.processor.process( + TestSegment::new(&mut self.operations, &mut self.executed), + &mut self.store, + ExecutionMode::Sync, + ); + } + + /// Assert that the plan has been executed as provided. + fn assert_plan(&self, id: ExecutionPlanId, expected: ExecutionPlan) { + let actual = self.store.get_unchecked(id); + assert_eq!(actual.operations, expected.operations, "Same operations"); + assert_eq!(actual.triggers, expected.triggers, "Same triggers"); + } + + /// Assert that the given plan id has been the last executed. + fn assert_last_executed(&self, id: ExecutionPlanId) { + match self.executed.last() { + Some(last_id) => assert_eq!(*last_id, id), + None => panic!("No plan has been executed"), + } + } + + /// Assert the number of executions since the start of the stream. + fn assert_number_of_executions(&self, number: usize) { + assert_eq!(self.executed.len(), number); + } + + /// Assert the number of operations queued. + fn assert_number_of_operations(&self, number: usize) { + assert_eq!(self.operations.len(), number); + } +} + +impl TestOptimizationBuilder { + /// Create a new optimization builder that follows a pattern with a trigger. + pub fn new(builder_id: usize, operations: Vec) -> Self { + Self { + builder_id, + expected_operations: operations, + actual: Vec::new(), + } + } +} + +impl OperationFuser for TestOptimizationBuilder { + /// Register a new operation. + fn fuse(&mut self, operation: &OperationIr) { + self.actual.push(operation.clone()); + } + + /// Build the optimization. + fn finish(&mut self) -> TestOptimization { + TestOptimization::new(self.builder_id, self.len()) + } + + /// Reset the state. + fn reset(&mut self) { + self.actual.clear(); + } + + /// Return the optimization status. + fn status(&self) -> FuserStatus { + if self.actual.len() < self.expected_operations.len() { + let operations = &self.expected_operations[0..self.actual.len()]; + + return match self.actual == operations { + // Still optimizing. + true => FuserStatus::Open, + // Never gonna be possible on that stream. + false => FuserStatus::Closed, + }; + } + + FuserStatus::Closed + } + + /// Return the properties of this optimization. + fn properties(&self) -> FuserProperties { + if self.actual.len() < self.expected_operations.len() { + // Optimization not possible. + return FuserProperties { + score: 0, + ready: false, + }; + } + + let stream_is_ok = + self.actual[0..self.expected_operations.len()] == self.expected_operations; + + if !stream_is_ok { + // Optimization not possible. + return FuserProperties { + score: 0, + ready: false, + }; + } + + // Optimization possible. + FuserProperties { + score: 1, + ready: true, + } + } + + // The number of operations that should be handle by the optimization. + fn len(&self) -> usize { + self.expected_operations.len() + } + fn clone_dyn(&self) -> Box> { + Box::new(self.clone()) + } +} + +impl StreamSegment for TestSegment<'_> { + // The operations in the process. + fn operations(&self) -> &[OperationIr] { + self.operations + } + + // Execute the process. + fn execute(&mut self, id: ExecutionPlanId, store: &mut ExecutionPlanStore) { + let execution_plan = store.get_unchecked(id); + + self.execute_strategy(&execution_plan.optimization.strategy); + + self.executed.push(id); + } +} + +impl TestSegment<'_> { + fn execute_strategy(&mut self, strategy: &ExecutionStrategy) { + match strategy { + ExecutionStrategy::Optimization { opt, .. } => { + self.operations.drain(0..opt.size); + } + ExecutionStrategy::Operations { ordering } => { + self.operations.drain(0..ordering.len()); + } + ExecutionStrategy::Composed(strategies) => { + for strategy in strategies { + self.execute_strategy(strategy); + } + } + } + } +} + +/// Just a simple operation. +pub fn operation_1() -> OperationIr { + OperationIr::NumericFloat( + DType::F32, + NumericOperationIr::Add(BinaryOpIr { + lhs: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + rhs: TensorIr { + id: TensorId::new(1), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + out: TensorIr { + id: TensorId::new(2), + shape: Shape::new([32, 32]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }, + }), + ) +} + +/// Just a simple operation. +pub fn operation_2() -> OperationIr { + OperationIr::NumericFloat( + DType::F32, + NumericOperationIr::AddScalar(ScalarOpIr { + lhs: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + rhs: ScalarIr::Float(5.0), + out: TensorIr { + id: TensorId::new(2), + shape: Shape::new([32, 32]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }, + }), + ) +} + +/// Just a simple operation. +pub fn operation_3() -> OperationIr { + OperationIr::Float( + DType::F32, + FloatOperationIr::Log(UnaryOpIr { + input: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + out: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }, + }), + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/validator.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/validator.rs new file mode 100644 index 0000000..2095a9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/execution/validator.rs @@ -0,0 +1,136 @@ +use burn_ir::OperationIr; + +use crate::stream::store::{ExecutionPlanId, ExecutionPlanStore, ExecutionTrigger}; + +/// Compare each operation in the list of operations provided by the [store](OperationsStore) +/// to verify if the newly added operations match the original list. +/// +/// It is used by the [policy](crate::stream::execution::Policy) to check each candidate as well +/// as to verify if a list of operations is optimal to execute based on their triggers. +#[derive(Debug)] +pub(crate) struct OperationsValidator { + /// The ID used to retrieve the operation list. + pub(crate) id: ID, + /// The current [state](MatchingState). + pub(crate) state: ValidatorState, +} + +/// The state of the validator. +#[derive(Debug)] +pub(crate) enum ValidatorState { + /// A matching operation list has been found. + Found { size: usize }, + /// No matching operation list has been found. + Invalidated, + /// Potentially going to find a matching operation list when more operations are added. + Validating, +} + +/// Provides a list of operations based on an Id. +pub(crate) trait OperationsStore { + /// The type used for the identifier. + type Id: Copy; + + /// retrieve the list of operations corresponding on the provided id. + fn get(&self, id: Self::Id) -> &[OperationIr]; +} + +impl OperationsValidator { + /// Create a new validator. + pub(crate) fn new(id: ID) -> Self { + Self { + id, + state: ValidatorState::Validating, + } + } + + /// Update the state of the validator based on the newly added operation. + pub(crate) fn update(&mut self, added: &OperationIr, added_position: usize, store: &S) + where + S: OperationsStore, + ID: PartialEq + Copy, + { + match &self.state { + ValidatorState::Found { size: _ } => return, + ValidatorState::Invalidated => return, + ValidatorState::Validating => {} + }; + + let item = store.get(self.id); + let operation_candidate = match item.get(added_position) { + Some(val) => val, + None => { + self.state = ValidatorState::Invalidated; + return; + } + }; + + if operation_candidate != added { + self.state = ValidatorState::Invalidated; + return; + } + + // Finished + if item.len() == added_position + 1 { + self.state = ValidatorState::Found { size: item.len() }; + } + } +} + +/// [Operations store](OperationsStore) used to retrieve the list of operations for a trigger. +#[derive(new)] +pub(crate) struct TriggerOperationsStore<'a, O> { + id: ExecutionPlanId, + store: &'a ExecutionPlanStore, +} + +/// Validates when operations match a trigger. +#[derive(Debug)] +pub(crate) enum TriggerValidator { + OnOperations { + matching: OperationsValidator, + progress: TriggerProgress, + }, + Always, + OnSync, +} + +/// The progress made into the trigger validation process. +#[derive(Debug)] +pub(crate) enum TriggerProgress { + /// When the validation hasn't started. + NotInit, + /// The number of operations that have been checked. + NumChecked(usize), +} + +/// An execution plan can have many triggers, so we use the position in the list to identify a +/// trigger. +pub(crate) type TriggerId = usize; + +impl OperationsStore for TriggerOperationsStore<'_, O> { + type Id = TriggerId; + + fn get(&self, id: Self::Id) -> &[OperationIr] { + match &self.store.get_unchecked(self.id).triggers[id] { + ExecutionTrigger::OnOperations(operations) => operations, + ExecutionTrigger::OnSync => &[], + ExecutionTrigger::Always => &[], + } + } +} + +/// [Operations store](OperationsStore) used to retrieve the list of operations for an +/// [execution plan](crate::stream::store::ExecutionPlan). +#[derive(new)] +pub(crate) struct ExecutionPlanOperationsStore<'a, O> { + store: &'a ExecutionPlanStore, +} + +impl OperationsStore for ExecutionPlanOperationsStore<'_, O> { + type Id = ExecutionPlanId; + + fn get(&self, id: Self::Id) -> &[OperationIr] { + &self.store.get_unchecked(id).operations + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/memory_checks.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/memory_checks.rs new file mode 100644 index 0000000..b61f7a5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/memory_checks.rs @@ -0,0 +1,249 @@ +use hashbrown::HashMap; +use std::{ + fmt::Display, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + mpsc::SyncSender, + }, + thread::JoinHandle, + time::Duration, +}; + +use burn_ir::{HandleContainer, TensorId, TensorStatus}; +use burn_std::id::StreamId; + +use crate::FusionRuntime; + +use super::Stream; + +/// Memory checks struct to validate there is no memory leak with the fusion runtime. +#[derive(Clone)] +pub(crate) struct MemoryChecks { + sender: SyncSender, + num_queued: Arc, + // Keeps track of its thread. + _handle: Arc>, +} + +enum Message { + Register(StreamAnalyses), + Check(SyncSender), +} + +enum MemoryReport { + Success, + NotReady, + NotStarted, + Fail(String), +} + +#[derive(Default)] +struct StreamAnalyses { + streams: HashMap, + num_handles: usize, +} + +impl Display for StreamAnalyses { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("\n==== Fusion Memory Report ====\n")?; + f.write_fmt(format_args!(" - Handles: {}\n", self.num_handles))?; + f.write_fmt(format_args!(" - Streams: {}\n", self.streams.len()))?; + + for (id, analysis) in self.streams.iter() { + f.write_fmt(format_args!( + " - {} => operations: {} cursor: {}\n", + id, analysis.num_operations, analysis.cursor + ))?; + for (tid, (origin, status)) in analysis.variables.iter() { + f.write_fmt(format_args!( + " - {tid} => origin: {origin} status: {status:?}\n", + ))?; + } + } + + f.write_str("==============================\n") + } +} + +#[derive(Default, Debug)] +struct Analysis { + variables: HashMap, + num_operations: usize, + cursor: u64, +} + +#[macro_export] +/// Export memory checks tests. +macro_rules! memory_checks { + () => { + #[cfg(test)] + mod memory_checks { + #[test] + fn test_memory_leaks() { + burn_fusion::stream::memory_checks::check_memory_leaks(); + } + } + }; +} + +static INSTANCE: spin::Mutex> = spin::Mutex::new(None); + +/// Performs memory checks and panics if a leak is discovered. +pub fn check_memory_leaks() { + let mut num_try_uninit = 0; + let max_try = 25; + + loop { + let report = fetch_memory_report(); + match report { + MemoryReport::Success => return, + MemoryReport::NotReady => { + num_try_uninit = 0; + std::thread::sleep(Duration::from_millis(100)) + } + MemoryReport::NotStarted => { + if num_try_uninit >= max_try { + // Nothing is running on the fusion runtime. + return; + } + num_try_uninit += 1; + std::thread::sleep(Duration::from_millis(100)) + } + MemoryReport::Fail(msg) => panic!("{msg}"), + } + } +} + +fn fetch_memory_report() -> MemoryReport { + let report = INSTANCE.lock(); + + let report = match report.as_ref() { + Some(client) => client, + None => return MemoryReport::NotStarted, + }; + + let (sender, rec) = std::sync::mpsc::sync_channel(1); + match report.sender.send(Message::Check(sender)) { + Ok(_) => {} + Err(err) => { + panic!("Channel closed can't send the check call: {err:?}") + } + }; + + match rec.recv() { + Ok(report) => report, + Err(err) => panic!("Received an error from fetching check results: {err}"), + } +} + +impl Default for MemoryChecks { + fn default() -> Self { + let mut instance = INSTANCE.lock(); + let result = match instance.as_mut() { + Some(client) => client.clone(), + None => { + let this = Self::spawn_new(); + *instance = Some(this.clone()); + this + } + }; + core::mem::drop(instance); + result + } +} + +impl MemoryChecks { + pub(crate) fn check( + &mut self, + streams: &HashMap>, + handles: &HandleContainer, + ) { + let mut analyses = StreamAnalyses { + num_handles: handles.num_handles(), + streams: Default::default(), + }; + + for (id, s) in streams.iter() { + let analysis = Analysis { + variables: s.queue.variables.clone(), + num_operations: s.queue.global.len(), + cursor: s.cursor, + }; + analyses.streams.insert(*id, analysis); + } + + self.num_queued.fetch_add(1, Ordering::Relaxed); + match self.sender.send(Message::Register(analyses)) { + Ok(..) => {} + Err(err) => { + panic!("Can't register memory checks analysis: {err:?}") + } + } + } + + fn spawn_new() -> Self { + let (sender, rec) = std::sync::mpsc::sync_channel(100); + let num_queued = Arc::new(AtomicU64::new(0)); + let num_queued_moved = num_queued.clone(); + + let handle = std::thread::spawn(move || { + let mut last_analyses = None; + + loop { + let payload = match rec.recv() { + Err(_err) => { + // A client has panic, safe to skip as it may be normal. + continue; + } + Ok(payload) => payload, + }; + match payload { + Message::Register(payload) => { + last_analyses = Some(payload); + num_queued_moved.fetch_sub(1, Ordering::Relaxed); + } + Message::Check(callback) => { + if num_queued_moved.load(Ordering::Relaxed) > 1 { + callback.send(MemoryReport::NotReady).unwrap(); + continue; + } + + // We assume that if nothing has been registered in the last second + // while being at a count of 1, it's the end. + std::thread::sleep(Duration::from_secs(5)); + + if num_queued_moved.load(Ordering::Relaxed) <= 1 { + match last_analyses.take() { + Some(val) => { + callback.send(Self::final_check(val)).unwrap(); + } + None => { + callback + .send(MemoryReport::Fail("No analyses".into())) + .unwrap(); + } + } + } else { + callback.send(MemoryReport::NotReady).unwrap(); + } + } + } + } + }); + + Self { + sender, + num_queued, + _handle: Arc::new(handle), + } + } + + fn final_check(analyses: StreamAnalyses) -> MemoryReport { + if !analyses.streams.is_empty() || analyses.num_handles > 0 { + return MemoryReport::Fail(format!("{analyses}")); + } + + MemoryReport::Success + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/mod.rs new file mode 100644 index 0000000..1e8cf1b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/mod.rs @@ -0,0 +1,33 @@ +pub(crate) mod execution; +pub(crate) mod queue; +pub(crate) mod shared_tensors; +pub(crate) mod store; + +#[cfg(feature = "memory-checks")] +/// Memory checks module. +pub mod memory_checks; + +#[cfg(not(feature = "memory-checks"))] +#[macro_export] +/// Export memory checks tests. +macro_rules! memory_checks { + () => { + #[cfg(test)] + mod memory_checks { + #[ignore = "'memory-checks' disabled"] + #[test] + fn test_memory_leaks() { + // + } + } + }; +} + +mod base; +mod context; +mod multi; + +pub use base::*; +pub use context::*; +pub use execution::*; +pub use multi::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/multi.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/multi.rs new file mode 100644 index 0000000..a017f4c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/multi.rs @@ -0,0 +1,472 @@ +use std::sync::Arc; + +use burn_ir::{HandleContainer, OperationIr, TensorId, TensorIr, TensorStatus}; +use hashbrown::{HashMap, HashSet}; + +use super::{ + StreamId, + execution::{ExecutionMode, Operation, Processor, StreamSegment}, + queue::OperationQueue, + shared_tensors::SharedTensors, + store::{ExecutionPlanId, ExecutionPlanStore}, +}; +use crate::{ + DropOp, FusionRuntime, + stream::shared_tensors::{SharedTensorAnalysis, SharedTensorDropAction}, +}; + +/// Keep track of multiple concurrent lazy streams of operations. +pub struct MultiStream { + streams: HashMap>, + optimizations: ExecutionPlanStore, + shared_tensors: SharedTensors, + device: R::FusionDevice, + #[cfg(feature = "memory-checks")] + memory_checks: super::memory_checks::MemoryChecks, +} + +#[derive(Debug)] +enum DropAction { + SkipSharedTensor, + ForceSharedTensor(Vec, TensorId), + ContinueDrop, +} + +impl MultiStream { + pub(crate) fn new(device: R::FusionDevice) -> Self { + Self { + streams: HashMap::new(), + optimizations: ExecutionPlanStore::new(), + shared_tensors: SharedTensors::default(), + device, + #[cfg(feature = "memory-checks")] + memory_checks: super::memory_checks::MemoryChecks::default(), + } + } + + /// Register a new tensor operation. + pub(crate) fn register( + &mut self, + streams: OperationStreams, + mut repr: OperationIr, + operation: Arc>, + handles: &mut HandleContainer, + ) { + let id = self.resolve_streams(&streams, handles, &mut repr); + + let drop_action = match &mut repr { + OperationIr::Drop(tensor_ir) => Some(self.handle_drop_op(id, tensor_ir)), + _ => None, + }; + + let sync = match drop_action { + Some(DropAction::SkipSharedTensor) => return, + Some(DropAction::ContinueDrop) => true, + Some(DropAction::ForceSharedTensor(stream_ids, tid)) => { + for stream_id in stream_ids { + if let Some(stream) = self.streams.get_mut(&stream_id) { + stream.queue.variables.remove(&tid); + if stream.queue.variables.is_empty() { + self.streams.remove(&stream_id); + } + } + } + true + } + None => false, + }; + + let num_executed = self.enqueue_operation(id, repr, &streams, operation, handles); + + if num_executed > 0 + && let Some(stream) = self.streams.get_mut(&id) + { + let cleared = self.shared_tensors.on_executed_ops(id, stream); + self.clear_shared_tensors(&cleared, id); + let to_drop = self.shared_tensors.clear_tensors(cleared); + self.drop_shared_tensors(to_drop, handles, id); + } + + let stream = match self.streams.get(&id) { + Some(val) => val, + None => { + #[cfg(feature = "memory-checks")] + self.memory_checks.check(&self.streams, handles); + return; + } + }; + + if !stream.queue.variables.is_empty() && sync { + // Not draining the queue can cause a memory leak when a stream is closing. + self.drain(handles, id); + } + + #[cfg(feature = "memory-checks")] + self.memory_checks.check(&self.streams, handles); + } + + /// Checks if the current operation is a drop. + /// + /// When a tensor is shared across multiple concurrent streams, dropping a tensor might cause a + /// problem when the same tensor is registered lazily on another stream, but not yet executed. + fn handle_drop_op(&mut self, id: StreamId, tensor_ir: &mut TensorIr) -> DropAction { + match !matches!(tensor_ir.status, TensorStatus::ReadWrite) { + true => { + let stream = self.streams.get(&id); + let on_drop = self + .shared_tensors + .on_drop(id, tensor_ir.id, stream.is_none()); + + match on_drop { + SharedTensorDropAction::ForceDrop(streams) => { + tensor_ir.status = TensorStatus::ReadWrite; + DropAction::ForceSharedTensor(streams, tensor_ir.id) + } + SharedTensorDropAction::Skip => DropAction::SkipSharedTensor, + } + } + false => DropAction::ContinueDrop, + } + } + + /// Enqueue an operation on the queue. + fn enqueue_operation( + &mut self, + id: StreamId, + repr: OperationIr, + streams: &OperationStreams, + operation: Arc>, + handles: &mut HandleContainer, + ) -> usize { + let stream = match self.streams.get_mut(&id) { + Some(stream) => stream, + None => { + let stream = Stream::new(self.device.clone()); + self.streams.insert(id, stream); + self.streams + .get_mut(&id) + .expect("Just added, so should be included in the hashmap.") + } + }; + + stream.queue.add(repr, operation, streams, id); + + let len_before = stream.queue.global.len(); + stream.processor.process( + Segment::new(&mut stream.queue, handles), + &mut self.optimizations, + ExecutionMode::Lazy, + ); + let len_after = stream.queue.global.len(); + let num_executed = len_before - len_after; + + stream.cursor += num_executed as u64; + + num_executed + } + + /// Mark a tensor as read. + #[allow(unused_variables)] + pub fn mark_read( + &mut self, + id: StreamId, + ir: &TensorIr, + handles: &HandleContainer, + ) { + if !matches!(ir.status, TensorStatus::ReadWrite) { + return; + }; + + let stream = match self.streams.get_mut(&id) { + Some(val) => val, + None => return, + }; + + stream.queue.variables.remove(&ir.id); + + if stream.queue.variables.is_empty() { + self.streams.remove(&id); + } + + #[cfg(feature = "memory-checks")] + self.memory_checks.check(&self.streams, handles); + } + + /// Drain a stream + pub fn drain(&mut self, handles: &mut HandleContainer, id: StreamId) { + if let Some(stream) = self.streams.get_mut(&id) { + let old = unsafe { StreamId::swap(id) }; + let num_executed = stream.queue.global.len(); + stream.processor.process( + Segment::new(&mut stream.queue, handles), + &mut self.optimizations, + ExecutionMode::Sync, + ); + stream.cursor += num_executed as u64; + + let cleared = self.shared_tensors.on_executed_ops(id, stream); + self.clear_shared_tensors(&cleared, id); + let to_drop = self.shared_tensors.clear_tensors(cleared); + + self.drop_shared_tensors(to_drop, handles, id); + unsafe { + StreamId::swap(old); + }; + } + } + + /// When one of the provided streams is different from the current stream, we drain them. + /// + /// Returns the selected stream id. + fn resolve_streams( + &mut self, + streams: &OperationStreams, + handles: &mut HandleContainer, + op: &mut OperationIr, + ) -> StreamId { + let current = streams.current; + let nodes = op.nodes(); + + let analysis = self.analyse_shared_tensors(&nodes, streams, current); + + self.merge_streams_timelines(handles, &analysis, current, &nodes); + self.register_shared_tensors_drop(&analysis, op); + + current + } + + /// Drain the stream only if one of the tensor in the given nodes is also included in the + /// stream queue. + fn resolve_stream( + &mut self, + handles: &mut HandleContainer, + id: StreamId, + nodes: &[&TensorIr], + ) { + if let Some(stream) = self.streams.get(&id) { + for node in nodes { + if stream.queue.variables.contains_key(&node.id) { + self.drain(handles, id); + return; + } + } + } + } + + fn analyse_shared_tensors( + &mut self, + nodes: &[&TensorIr], + streams: &OperationStreams, + current: StreamId, + ) -> MultiSharedTensorAnalysis { + let mut shared_analysis = MultiSharedTensorAnalysis::default(); + + for node in nodes.iter() { + let analysis = self + .shared_tensors + .analyse(current, node, streams, &self.streams); + match analysis { + SharedTensorAnalysis::SharedFromCurrentStream => { + shared_analysis.current.push(node.id); + } + SharedTensorAnalysis::NotShared => {} + SharedTensorAnalysis::SharedFromExistingStream { + stream_id, + original_cursor, + } => { + shared_analysis + .existing + .push((node.id, stream_id, original_cursor)); + } + SharedTensorAnalysis::SharedFromNewStream { stream_id } => { + shared_analysis.new.push((node.id, stream_id)); + } + } + } + + shared_analysis + } + + fn merge_streams_timelines( + &mut self, + handles: &mut HandleContainer, + analysis: &MultiSharedTensorAnalysis, + current: StreamId, + nodes: &[&TensorIr], + ) { + // If we only have current tensors that are shared, we're safe to not sync the timelines. + if analysis.new.is_empty() && analysis.existing.is_empty() { + return; + } + + let mut streams_to_sync = HashSet::new(); + for (_tensor_id, stream_id) in analysis.new.iter() { + streams_to_sync.insert(*stream_id); + } + + for (_tensor_id, stream_id, original_cursor) in analysis.existing.iter() { + if let Some(stream) = self.streams.get(stream_id) { + // We only have to sync a stream when the stream isn't up to date with + // the original cursor of the current operation. + if stream.cursor <= *original_cursor && *stream_id != current { + streams_to_sync.insert(*stream_id); + } + } + } + + for id in streams_to_sync.drain() { + log::trace!("Drain stream {id} for use in current {current}"); + self.resolve_stream(handles, id, nodes); + } + } + + fn register_shared_tensors_drop( + &mut self, + analysis: &MultiSharedTensorAnalysis, + op: &mut OperationIr, + ) { + let mut readonly_tensors = Vec::new(); + + for (tensor_id, _stream_id) in analysis.new.iter() { + readonly_tensors.push(*tensor_id); + } + for (tensor_id, _stream_id, _cursor) in analysis.existing.iter() { + readonly_tensors.push(*tensor_id); + } + for tensor_id in analysis.current.iter() { + readonly_tensors.push(*tensor_id); + } + + self.shared_tensors + .tag_manual_drop(op.mark_read_only(&readonly_tensors)); + } + + fn drop_shared_tensors( + &mut self, + tensors: Vec, + handles: &mut HandleContainer, + current: StreamId, + ) { + for (stream_id, s) in self.streams.iter_mut() { + for tensor in tensors.iter() { + if let Some((original, _status)) = s.queue.variables.get(&tensor.id) + && original != stream_id + { + s.queue.variables.remove(&tensor.id); + } + } + } + for tensor in tensors { + let streams = OperationStreams { + streams: HashMap::new(), + current, + }; + + let op = Arc::new(DropOp { id: tensor.id }); + self.register(streams, OperationIr::Drop(tensor), op, handles); + } + } + fn clear_shared_tensors(&mut self, tensors: &[TensorId], current: StreamId) { + let mut to_remove = Vec::new(); + for (stream_id, s) in self.streams.iter_mut() { + for tensor in tensors.iter() { + s.queue.variables.remove(tensor); + } + + if s.queue.variables.is_empty() && current != *stream_id { + to_remove.push(*stream_id); + } + } + + for s in to_remove { + self.streams.remove(&s); + } + } +} + +pub(crate) struct Stream { + pub(crate) queue: OperationQueue, + processor: Processor, + pub(crate) cursor: u64, +} + +#[derive(new)] +struct Segment<'a, R: FusionRuntime> { + queue: &'a mut OperationQueue, + handles: &'a mut HandleContainer, +} + +impl StreamSegment for Segment<'_, R> { + fn operations(&self) -> &[OperationIr] { + &self.queue.relative + } + + fn execute(&mut self, id: ExecutionPlanId, store: &mut ExecutionPlanStore) { + self.queue.execute(id, self.handles, store) + } +} + +impl Stream { + fn new(device: R::FusionDevice) -> Self { + Self { + processor: Processor::new(R::fusers(device)), + queue: OperationQueue::new(), + cursor: 0, + } + } +} + +#[derive(Debug)] +/// Manage the streams used for the current [operation](OperationIr). +pub struct OperationStreams { + pub(crate) streams: HashMap, + pub(crate) current: StreamId, +} + +impl Default for OperationStreams { + fn default() -> Self { + Self { + streams: HashMap::new(), + current: StreamId::current(), + } + } +} + +impl OperationStreams { + /// Register a tensor in the list of streams used for the current [operation](OperationIr). + /// + /// You only need to register input tensors, not the outputs. + /// So init tensor operations should have no streams registered. + pub fn tensor(&mut self, tensor: &crate::FusionTensor) { + self.streams.insert(tensor.id, tensor.stream); + } + + pub(crate) fn get(&self, id: TensorId) -> Option { + self.streams.get(&id).cloned() + } + + /// Create new operation streams with the given inputs. + /// + /// The inputs are automatically registered. + pub fn with_inputs<'a, R: FusionRuntime + 'a, I>(tensors: I) -> Self + where + I: IntoIterator>, + { + let mut streams = OperationStreams::default(); + for tensor in tensors.into_iter() { + streams.tensor(tensor) + } + streams + } +} + +#[derive(Default, Debug)] +struct MultiSharedTensorAnalysis { + /// Tensors that are shared with other streams, but we're currently executing on the same stream + /// the tensor was originally created. + current: Vec, + /// Tensors that are shared with new streams. + new: Vec<(TensorId, StreamId)>, + /// Tensors that are shared with existing streams. + existing: Vec<(TensorId, StreamId, u64)>, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/base.rs new file mode 100644 index 0000000..a297c75 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/base.rs @@ -0,0 +1,95 @@ +use std::sync::Arc; + +use crate::FusionRuntime; +use crate::stream::{OperationConverter, OperationStreams, RelativeOps, execution::Operation}; +use burn_backend::StreamId; +use burn_ir::{OperationIr, TensorId, TensorStatus}; + +use hashbrown::HashMap; + +/// A growing list of [tensor operation descriptions](OperationIr). +pub struct OperationQueue { + /// List of operation descriptions. These contain the exact tensor IDs + /// and shapes so that kernels can be run correctly. + /// + /// The length of this list is the same as the length of the `operations` list. + pub(crate) global: Vec, + /// List of operation descriptions. The tensor IDs and shapes are relative + /// because we don't need to know the exact values, but they are sufficient to + /// determine which operations can be fused. + pub(crate) relative: Vec, + pub(crate) converter: OperationConverter, + pub(crate) operations: Vec>>, + pub(crate) variables: HashMap, +} + +impl Default for OperationQueue { + fn default() -> Self { + Self::new() + } +} + +impl OperationQueue { + /// Create a new empty queue. + pub fn new() -> Self { + Self { + global: Vec::new(), + relative: Vec::new(), + converter: OperationConverter::default(), + operations: Vec::new(), + variables: HashMap::new(), + } + } + + /// Add a new tensor operation to the queue. + /// + /// The new [operation intermediate representation](OperationIr) will be converted to a local + /// representation that can be reused when the same pattern emerge in different but similar + /// scenario, so that the same optimization can be used. + pub fn add( + &mut self, + global: OperationIr, + operation: Arc>, + streams: &OperationStreams, + current: StreamId, + ) { + for node in global.nodes() { + if let Some(stream_id) = streams.get(node.id) { + self.variables.insert(node.id, (stream_id, node.status)); + } else { + self.variables.insert(node.id, (current, node.status)); + } + } + let relative = global.to_relative(&mut self.converter); + self.relative.push(relative); + self.global.push(global); + self.operations.push(operation); + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + + #[test] + fn stream_id_from_different_threads() { + let current = StreamId::current(); + + let thread1 = std::thread::spawn(|| (StreamId::current(), StreamId::current())); + let thread2 = std::thread::spawn(StreamId::current); + + let (stream_1, stream_11) = thread1.join().unwrap(); + let stream_2 = thread2.join().unwrap(); + + assert_ne!(current, stream_1, "Should be different from thread 1"); + assert_ne!(current, stream_2, "Should be different from thread 2"); + assert_ne!( + stream_1, stream_2, + "Should be different from different threads" + ); + assert_eq!( + stream_1, stream_11, + "Should be the same, since same thread." + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/execution.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/execution.rs new file mode 100644 index 0000000..f32bdc0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/execution.rs @@ -0,0 +1,153 @@ +use std::sync::Arc; + +use burn_ir::{HandleContainer, TensorStatus}; + +use crate::{ + FusionRuntime, + search::BlockOptimization, + stream::{ + Context, Operation, OperationConverter, OrderedExecution, RelativeOps, + store::{ExecutionPlanId, ExecutionPlanStore, ExecutionStrategy}, + }, +}; + +use super::OperationQueue; + +impl OperationQueue { + /// Execute the queue partially following the execution strategy from the plan. + pub(crate) fn execute( + &mut self, + id: ExecutionPlanId, + handles: &mut HandleContainer, + store: &mut ExecutionPlanStore, + ) { + let plan = store.get_mut_unchecked(id); + self.execute_block_optimization(&mut plan.optimization, handles); + } + + fn execute_block_optimization( + &mut self, + step: &mut BlockOptimization, + handles: &mut HandleContainer, + ) { + let mut operations = Vec::new(); + core::mem::swap(&mut operations, &mut self.operations); + let (operations, num_drained) = + QueueExecution::run(step, &mut self.converter, handles, operations); + + self.operations = operations; + self.drain_queue(num_drained, handles); + } + + /// Bookkeeping after executing `num_drained` operations from the queue. + fn drain_queue(&mut self, num_drained: usize, handles: &mut HandleContainer) { + self.global[0..num_drained] + .iter() + .flat_map(|desc| desc.nodes()) + .for_each(|tensor| { + if tensor.status == TensorStatus::ReadWrite { + self.variables.remove(&tensor.id); + }; + handles.free(tensor) + }); + + self.global.drain(0..num_drained); + + self.reset_relative(); + } + + fn reset_relative(&mut self) { + self.relative.clear(); + self.converter.clear(); + + for node in self.global.iter() { + let relative = node.to_relative(&mut self.converter); + self.relative.push(relative); + } + } +} + +/// A queue execution has the responsibility to run the provided +/// [optimization](FusionRuntime::Optimization) without holes. +enum QueueExecution<'a, R: FusionRuntime> { + Single { + handles: &'a mut HandleContainer, + converter: &'a mut OperationConverter, + execution: OrderedExecution, + }, + Multiple { + context: &'a mut Context<'a, R::FusionHandle>, + execution: OrderedExecution, + }, +} + +impl<'a, R: FusionRuntime> QueueExecution<'a, R> { + fn run( + optimization: &mut BlockOptimization, + converter: &'a mut OperationConverter, + handles: &'a mut HandleContainer, + operations: Vec>>, + ) -> (Vec>>, usize) { + let execution = OrderedExecution::new(operations); + + if matches!(&optimization.strategy, ExecutionStrategy::Composed(..)) { + let mut context = converter.context(handles); + let mut this = QueueExecution::Multiple { + context: &mut context, + execution, + }; + + this = this.execute_strategy(&mut optimization.strategy); + + match this { + QueueExecution::Multiple { execution, .. } => execution.finish(), + _ => unreachable!(), + } + } else { + let mut this = QueueExecution::Single { + handles, + converter, + execution, + }; + this = this.execute_strategy(&mut optimization.strategy); + + match this { + QueueExecution::Single { execution, .. } => execution.finish(), + _ => unreachable!(), + } + } + } + + fn execute_strategy(mut self, strategy: &mut ExecutionStrategy) -> Self { + match &mut self { + QueueExecution::Single { + handles, + converter, + execution, + } => match strategy { + ExecutionStrategy::Optimization { ordering, opt } => { + let mut context = converter.context(handles); + execution.execute_optimization(opt, &mut context, ordering.clone()) + } + ExecutionStrategy::Operations { ordering } => { + execution.execute_operations(handles, ordering) + } + ExecutionStrategy::Composed(_) => unreachable!(), + }, + QueueExecution::Multiple { context, execution } => match strategy { + ExecutionStrategy::Optimization { opt, ordering } => { + execution.execute_optimization(opt, context, ordering.clone()); + } + ExecutionStrategy::Operations { ordering } => { + execution.execute_operations(context.handles, ordering); + } + ExecutionStrategy::Composed(items) => { + for item in items.iter_mut() { + self = self.execute_strategy(item); + } + } + }, + }; + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/mod.rs new file mode 100644 index 0000000..450cba0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/queue/mod.rs @@ -0,0 +1,4 @@ +mod base; +mod execution; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/shared_tensors.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/shared_tensors.rs new file mode 100644 index 0000000..dc0063e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/shared_tensors.rs @@ -0,0 +1,306 @@ +use burn_backend::StreamId; +use burn_ir::{TensorId, TensorIr}; +use hashbrown::HashMap; + +use super::{OperationStreams, Stream}; +use crate::FusionRuntime; + +#[derive(Default)] +/// Manages tensors that are shared between multiple streams. +pub struct SharedTensors { + shared_tensors: HashMap, + shared_tensors_manual_drop: HashMap, +} + +#[derive(Default, Debug)] +/// A tensor that is shared between multiple streams. +struct SharedTensor { + streams: HashMap, +} + +#[derive(Debug)] +struct SharedTensorState { + cursor_current: u64, + cursor_origin: u64, +} + +#[derive(Debug)] +/// What do to when a tensor is dropped. +pub enum SharedTensorDropAction { + /// Performs the drop and removes the shared tensor from the provided list of + /// stream ids. + ForceDrop(Vec), + /// Skip the drop. + Skip, +} + +#[derive(Debug)] +/// Information about a shared tensor. +pub enum SharedTensorAnalysis { + /// The tensor is not shared. + NotShared, + /// The tensor is shared, but its original stream is the current one. + SharedFromCurrentStream, + /// The tensor is shared, and its original stream is an existing stream. + SharedFromExistingStream { + /// The stream id of the existing stream. + stream_id: StreamId, + /// The position of execution in the existing stream where the tensor was created. + original_cursor: u64, + }, + /// The tensor is shared, and its original stream is a new one without any operation + /// executed. + SharedFromNewStream { + /// The stream id of the new stream. + stream_id: StreamId, + }, +} + +impl SharedTensors { + /// Function to call when a drop operation is registered on the given stream and tensor. + pub fn on_drop( + &mut self, + stream_id: StreamId, + tensor_id: TensorId, + stream_completed: bool, + ) -> SharedTensorDropAction { + let mut execute_still = false; + + if let Some(shared) = self.shared_tensors.get_mut(&tensor_id) { + if stream_completed { + shared.drop(stream_id); + execute_still = shared.streams.is_empty(); + } + } else { + execute_still = true; + } + + if execute_still { + let state = self.shared_tensors.remove(&tensor_id); + self.shared_tensors_manual_drop.remove(&tensor_id); + + return match state { + Some(val) => { + let streams = val.streams.keys().copied().collect(); + SharedTensorDropAction::ForceDrop(streams) + } + None => SharedTensorDropAction::ForceDrop(Vec::new()), + }; + } + + SharedTensorDropAction::Skip + } + + /// Function to call when one or many operations were executed on the stream. + /// + /// Returns the tensor id that can be cleared with [Self::clear_tensors] + pub fn on_executed_ops( + &mut self, + id: StreamId, + stream: &mut Stream, + ) -> Vec { + let mut cleared = Vec::new(); + for (tensor_id, state) in self.shared_tensors.iter_mut() { + match state.update(id, stream) { + SharedTensorUpdate::RemovedFromStream(no_more_stream) => { + stream.queue.variables.remove(tensor_id); + + if no_more_stream { + cleared.push(*tensor_id); + } + } + SharedTensorUpdate::ReadyForCleanup => { + cleared.push(*tensor_id); + } + SharedTensorUpdate::NoChange => {} + } + } + cleared + } + + /// Clear the provided tensors and returns the list of tensors that can be manually dropped. + pub fn clear_tensors(&mut self, tensors: Vec) -> Vec { + let mut to_drop = Vec::new(); + for id in tensors { + self.shared_tensors.remove(&id); + + if let Some(tensor) = self.shared_tensors_manual_drop.remove(&id) { + to_drop.push(tensor); + } + } + + self.register_manual_drop(to_drop) + } + + /// Analyses the current tensor and updates its state. + pub fn analyse( + &mut self, + id: StreamId, + node: &TensorIr, + streams_op: &OperationStreams, + streams: &HashMap>, + ) -> SharedTensorAnalysis { + let stream_id = match streams_op.streams.get(&node.id) { + Some(val) => val, + None => { + return match self.shared_tensors.contains_key(&node.id) { + true => SharedTensorAnalysis::SharedFromCurrentStream, + false => SharedTensorAnalysis::NotShared, + }; + } + }; + + if stream_id == &id { + return match self.shared_tensors.contains_key(&node.id) { + true => SharedTensorAnalysis::SharedFromCurrentStream, + false => SharedTensorAnalysis::NotShared, + }; + } + + // Here the node is tagged as newly shared. + let stream_current = streams.get(&id); + let stream = streams.get(stream_id); + + let state = match self.shared_tensors.get_mut(&node.id) { + Some(state) => state, + None => { + self.shared_tensors.insert(node.id, SharedTensor::default()); + self.shared_tensors.get_mut(&node.id).unwrap() + } + }; + + state.register_new_stream(id, stream_current); + match state.register_new_stream(*stream_id, stream) { + Some(origin) => SharedTensorAnalysis::SharedFromExistingStream { + stream_id: *stream_id, + original_cursor: origin, + }, + None => SharedTensorAnalysis::SharedFromNewStream { + stream_id: *stream_id, + }, + } + } + + /// Tag the provided tensors as manually dropped. + pub fn tag_manual_drop(&mut self, dropped: Vec) { + for tensor in dropped { + self.shared_tensors_manual_drop.insert(tensor.id, tensor); + } + } + + fn register_manual_drop(&mut self, mut tensors: Vec) -> Vec { + if self.shared_tensors_manual_drop.is_empty() { + return tensors; + } + + let mut to_drop = Vec::new(); + for id in self.shared_tensors_manual_drop.keys() { + if !self.shared_tensors.contains_key(id) { + to_drop.push(*id); + } + } + + for id in to_drop { + let entry = self.shared_tensors_manual_drop.remove(&id).unwrap(); + tensors.push(entry); + } + + tensors + } +} + +/// The result from a [SharedTensor::update]. +pub enum SharedTensorUpdate { + /// The tensor is removed from the current stream. + /// + /// Also contains if the current stream is empty. + RemovedFromStream(bool), + /// If the tensor is shared across zero streams. + ReadyForCleanup, + /// If nothing has been done from the update. + NoChange, +} + +impl SharedTensor { + /// Register the tensor as also part of the given stream. + /// + /// The stream might not exist yet when the current tensor is part of the first operation in + /// the newly created stream. + fn register_new_stream( + &mut self, + id: StreamId, + stream: Option<&Stream>, + ) -> Option { + let cursor_current = match stream { + Some(stream) => stream.cursor + stream.queue.global.len() as u64, + None => 1, + }; + + match self.streams.get_mut(&id) { + Some(s) => { + s.cursor_current = cursor_current; + Some(s.cursor_origin) + } + None => { + let state = SharedTensorState { + cursor_current, + cursor_origin: cursor_current, + }; + self.streams.insert(id, state); + None + } + } + } + + /// Update the current shared tensor state on the given stream. + /// + /// If the shared tensor is no longer needed on the stream, we will remove it from the list of + /// shared streams. + fn update(&mut self, id: StreamId, stream: &Stream) -> SharedTensorUpdate { + let entry = match self.streams.remove(&id) { + Some(val) => val, + None => { + return if self.streams.is_empty() { + SharedTensorUpdate::ReadyForCleanup + } else { + SharedTensorUpdate::NoChange + }; + } + }; + + // We can only free the shared tensor if the latest cursor is executed. + if entry.cursor_current <= stream.cursor { + SharedTensorUpdate::RemovedFromStream(self.streams.is_empty()) + } else { + self.streams.insert(id, entry); + SharedTensorUpdate::NoChange + } + } + + fn drop(&mut self, id: StreamId) { + self.streams.remove(&id); + } +} + +impl core::fmt::Debug for SharedTensors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("\n==== Shared Tensors ====\n")?; + + for sh in self.shared_tensors.iter() { + f.write_fmt(format_args!(" - Shared {}", sh.0))?; + for (id, state) in sh.1.streams.iter() { + f.write_fmt(format_args!( + " [{}, cursor={}..{}] ", + id, state.cursor_origin, state.cursor_current + ))?; + } + f.write_str("\n")?; + } + for sh in self.shared_tensors_manual_drop.iter() { + f.write_fmt(format_args!(" - Manual Drop {}", sh.0))?; + f.write_str("\n")?; + } + + f.write_str("========================\n") + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/base.rs new file mode 100644 index 0000000..ffae95e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/base.rs @@ -0,0 +1,94 @@ +use std::sync::Arc; + +use crate::search::BlockOptimization; + +use super::{ExecutionPlanIndex, InsertQuery, SearchQuery}; +use burn_ir::OperationIr; +use serde::{Deserialize, Serialize}; + +/// The store that contains all explorations done on a device. +#[derive(Default)] +pub(crate) struct ExecutionPlanStore { + plans: Vec>, + index: ExecutionPlanIndex, +} + +/// How a list of operations should be executed. +#[derive(PartialEq, Debug, Clone)] +pub(crate) enum ExecutionStrategy { + /// An optimization was found, and therefore should be executed. + Optimization { opt: O, ordering: Arc> }, + /// No optimization was found, each operation should be executed individually. + Operations { ordering: Arc> }, + /// A composition of multiple execution strategies. + Composed(Vec>), +} + +/// The trigger that indicates when to stop exploring. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub(crate) enum ExecutionTrigger { + OnOperations(Vec), + OnSync, + Always, +} + +/// The unique identifier for an exploration that was executed. +pub(crate) type ExecutionPlanId = usize; + +/// The outcome of an exploration that can be stored. +#[derive(Debug)] +pub(crate) struct ExecutionPlan { + /// The operations on which the exploration is related to. + pub(crate) operations: Vec, + /// The criteria that signal when this plan should be executed. Only one trigger is necessary. + pub(crate) triggers: Vec, + /// The optimization that should be used when executing this plan. + pub(crate) optimization: BlockOptimization, +} + +impl ExecutionPlanStore { + pub fn new() -> Self { + Self { + plans: Vec::new(), + index: ExecutionPlanIndex::default(), + } + } + + pub fn find(&self, query: SearchQuery<'_>) -> Vec { + self.index.find(query) + } + + pub fn add(&mut self, exploration: ExecutionPlan) -> ExecutionPlanId { + if exploration.operations.is_empty() { + panic!("Can't add an empty optimization."); + } + + let id = self.plans.len(); + + self.index.insert(InsertQuery::NewPlan { + operations: &exploration.operations, + id, + }); + + self.plans.push(exploration); + + id + } + + pub fn get_mut_unchecked(&mut self, id: ExecutionPlanId) -> &mut ExecutionPlan { + &mut self.plans[id] + } + + pub fn get_unchecked(&self, id: ExecutionPlanId) -> &ExecutionPlan { + &self.plans[id] + } + + /// Add a new end condition for an optimization. + pub fn add_trigger(&mut self, id: ExecutionPlanId, trigger: ExecutionTrigger) { + let criteria = &mut self.plans[id].triggers; + + if !criteria.contains(&trigger) { + criteria.push(trigger); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/index.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/index.rs new file mode 100644 index 0000000..9651b8a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/index.rs @@ -0,0 +1,293 @@ +use crate::stream::store::ExecutionPlanId; +use burn_ir::OperationIr; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, hash_map::DefaultHasher}, + hash::{Hash, Hasher}, +}; + +/// Index used to search optimizations. +#[derive(Default, Serialize, Deserialize, Clone)] +pub struct ExecutionPlanIndex { + /// We can't use `HashMap>` since `OperationIr` + /// doesn't implement [`Eq`](core::cmp::Eq). + /// + /// `OperationIr` can't implement `Eq` since float types don't implement it. + /// + /// We rely instead on [`PartialEq`](core::cmp::PartialEq) to manually handle hash collisions. + /// This is OK because we use `relative` operations where any scalar values are set to zeros, + /// see [`RelativeStreamConverter`](crate::stream::RelativeStreamConverter). + /// + /// Map from the hash of the `OperationIr` to a list of `(OperationIr, index)` pairs, + /// where `index` is the index of all the execution plans that start with the `OperationIr` + /// in the `starters` list. + mapping: HashMap>, + starters: Vec>, +} + +pub enum SearchQuery<'a> { + PlansStartingWith(&'a OperationIr), +} + +pub enum InsertQuery<'a> { + NewPlan { + operations: &'a [OperationIr], + id: ExecutionPlanId, + }, +} + +impl ExecutionPlanIndex { + /// Search optimizations with the given [query](SearchQuery). + pub fn find(&self, query: SearchQuery<'_>) -> Vec { + match query { + SearchQuery::PlansStartingWith(ops) => self.find_starting_with(ops), + } + } + + /// Register a new optimization with the given [query](InsertQuery). + pub fn insert(&mut self, query: InsertQuery<'_>) { + match query { + InsertQuery::NewPlan { operations, id } => { + if let Some(operation) = operations.first() { + self.insert_new_operation(operation, id) + } + } + } + } + + /// Find execution plans starting with the `OperationIr` + fn find_starting_with(&self, operation: &OperationIr) -> Vec { + let key = self.operation_key(operation); + let values = match self.mapping.get(&key) { + Some(val) => val, + None => return Vec::new(), + }; + + if values.is_empty() { + return Vec::new(); + } + + let (_, index) = match values.iter().find(|value| &value.0 == operation) { + Some(val) => val, + None => return Vec::new(), + }; + + match self.starters.get(*index) { + Some(value) => value.clone(), + None => Vec::new(), + } + } + + /// Update the index for an execution plan starting with operation `ops` + fn insert_new_operation(&mut self, ops: &OperationIr, new_id: ExecutionPlanId) { + let key = self.operation_key(ops); + let values = match self.mapping.get_mut(&key) { + Some(val) => val, + None => { + // New starter ops. + let index = self.starters.len(); + self.starters.push(vec![new_id]); + self.mapping.insert(key, vec![(ops.clone(), index)]); + + return; + } + }; + let (_, index) = match values.iter_mut().find(|value| &value.0 == ops) { + Some(val) => val, + None => { + // New with hash collision. + let index = self.starters.len(); + self.starters.push(vec![new_id]); + values.push((ops.clone(), index)); + return; + } + }; + + // New optimization for an existing starter. + self.starters + .get_mut(*index) + .expect("Should exist") + .push(new_id); + } + + // Hash the value of the first operation in a list. + fn operation_key(&self, ops: &OperationIr) -> u64 { + let mut hasher = DefaultHasher::new(); + ops.hash(&mut hasher); + hasher.finish() + } +} + +#[cfg(test)] +mod tests { + use burn_backend::{DType, Shape}; + use burn_ir::{ + BinaryOpIr, NumericOperationIr, ScalarIr, ScalarOpIr, TensorId, TensorIr, TensorStatus, + }; + + use super::*; + + #[test] + fn should_find_optimization_id_based_on_tensor_ops() { + let mut index = ExecutionPlanIndex::default(); + let stream_1 = [ops_1()]; + let optimization_id_1 = 0; + + index.insert(InsertQuery::NewPlan { + operations: &stream_1, + id: optimization_id_1, + }); + + let found = index.find(SearchQuery::PlansStartingWith(&stream_1[0])); + + assert_eq!(found, vec![optimization_id_1]); + } + + #[test] + fn should_support_multiple_optimization_ids_with_same_starting_ops() { + let mut index = ExecutionPlanIndex::default(); + let stream_1 = [ops_1(), ops_2(), ops_1()]; + let stream_2 = [ops_1(), ops_1(), ops_2()]; + let optimization_id_1 = 0; + let optimization_id_2 = 1; + + index.insert(InsertQuery::NewPlan { + operations: &stream_1, + id: optimization_id_1, + }); + index.insert(InsertQuery::NewPlan { + operations: &stream_2, + id: optimization_id_2, + }); + + let found = index.find(SearchQuery::PlansStartingWith(&stream_1[0])); + + assert_eq!(found, vec![optimization_id_1, optimization_id_2]); + } + + #[test] + fn should_only_find_optimization_with_correct_starting_ops() { + let mut index = ExecutionPlanIndex::default(); + let stream_1 = [ops_1(), ops_1()]; + let stream_2 = [ops_2(), ops_1()]; + let optimization_id_1 = 0; + let optimization_id_2 = 1; + + index.insert(InsertQuery::NewPlan { + operations: &stream_1, + id: optimization_id_1, + }); + index.insert(InsertQuery::NewPlan { + operations: &stream_2, + id: optimization_id_2, + }); + + let found = index.find(SearchQuery::PlansStartingWith(&stream_1[0])); + + assert_eq!(found, vec![optimization_id_1]); + } + + #[test] + fn should_handle_hash_collisions() { + let mut index = ExecutionPlanIndex::default(); + let stream_1 = [ops_1(), ops_1()]; + let stream_2 = [ops_3(), ops_1()]; + let optimization_id_1 = 0; + let optimization_id_2 = 1; + + let stream_1_key = index.operation_key(&stream_1[0]); + let stream_2_key = index.operation_key(&stream_2[0]); + + assert_ne!( + stream_1_key, stream_2_key, + "Ops 1 and Ops 3 should not have the same hash" + ); // ops 1 and 3 have different variants, so the hash differs + assert_ne!(stream_1[0], stream_2[0], "Ops 1 and Ops 3 are different."); + + index.insert(InsertQuery::NewPlan { + operations: &stream_1, + id: optimization_id_1, + }); + index.insert(InsertQuery::NewPlan { + operations: &stream_2, + id: optimization_id_2, + }); + + let found = index.find(SearchQuery::PlansStartingWith(&stream_1[0])); + + assert_eq!(found, vec![optimization_id_1]); + } + + fn ops_1() -> OperationIr { + OperationIr::NumericFloat( + DType::F32, + NumericOperationIr::Add(BinaryOpIr { + lhs: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + rhs: TensorIr { + id: TensorId::new(1), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + out: TensorIr { + id: TensorId::new(2), + shape: Shape::new([32, 32]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }, + }), + ) + } + + fn ops_2() -> OperationIr { + OperationIr::NumericFloat( + DType::F32, + NumericOperationIr::AddScalar(ScalarOpIr { + lhs: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + rhs: ScalarIr::Float(5.0), + out: TensorIr { + id: TensorId::new(2), + shape: Shape::new([32, 32]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }, + }), + ) + } + + fn ops_3() -> OperationIr { + OperationIr::NumericFloat( + DType::F32, + NumericOperationIr::Sub(BinaryOpIr { + lhs: TensorIr { + id: TensorId::new(0), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + rhs: TensorIr { + id: TensorId::new(1), + shape: Shape::new([32, 32]), + status: TensorStatus::ReadOnly, + dtype: DType::F32, + }, + out: TensorIr { + id: TensorId::new(2), + shape: Shape::new([32, 32]), + status: TensorStatus::NotInit, + dtype: DType::F32, + }, + }), + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/mod.rs new file mode 100644 index 0000000..f14aff0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/stream/store/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod index; + +pub(crate) use base::*; +pub(super) use index::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/tensor.rs new file mode 100644 index 0000000..ddd1fde --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-fusion/src/tensor.rs @@ -0,0 +1,232 @@ +use crate::{ + Client, FusionBackend, FusionRuntime, + stream::{Operation, OperationStreams, StreamId}, +}; +use burn_backend::{ + DType, ExecutionError, QTensorPrimitive, Shape, TensorData, TensorMetadata, + quantization::QuantScheme, +}; +use burn_ir::{OperationIr, TensorId, TensorIr, TensorStatus}; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; + +/// Tensor primitive for the [fusion backend](crate::FusionBackend) for all kind. +pub struct FusionTensor { + /// Tensor id. + pub id: TensorId, + /// The shape of the tensor. + pub shape: Shape, + /// The fusion client. + pub client: Client, + /// The datatype of the tensor. + pub dtype: DType, + /// The current stream id this tensor is on. + pub stream: StreamId, + pub(crate) count: Arc, +} + +impl Clone for FusionTensor { + fn clone(&self) -> Self { + self.count.fetch_add(1, Ordering::Acquire); + + Self { + id: self.id, + shape: self.shape.clone(), + client: self.client.clone(), + dtype: self.dtype, + stream: self.stream, + count: self.count.clone(), + } + } +} + +impl core::fmt::Debug for FusionTensor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str( + format!( + "{{ id: {:?}, shape: {:?}, device: {:?} }}", + self.id, + self.shape, + self.client.device().clone(), + ) + .as_str(), + ) + } +} + +impl TensorMetadata for FusionTensor { + fn dtype(&self) -> DType { + self.dtype + } + + fn shape(&self) -> Shape { + self.shape.clone() + } + + fn rank(&self) -> usize { + self.shape.num_dims() + } +} + +impl FusionTensor { + pub(crate) fn new( + id: TensorId, + shape: Shape, + dtype: DType, + client: Client, + stream: StreamId, + ) -> Self { + Self { + id, + shape, + client, + dtype, + stream, + count: Arc::new(AtomicU32::new(1)), + } + } + + fn status(&self, count: u32) -> TensorStatus { + if count <= 1 { + TensorStatus::ReadWrite + } else { + TensorStatus::ReadOnly + } + } + + /// Intermediate representation to be used when using an uninitialized tensor as output. + pub fn to_ir_out(&self) -> TensorIr { + TensorIr { + status: TensorStatus::NotInit, + shape: self.shape.clone(), + id: self.id, + dtype: self.dtype, + } + } + + /// Intermediate representation to be used when using an initialized tensor used as input. + pub fn into_ir(mut self) -> TensorIr { + let count = self.count.load(Ordering::Acquire); + let status = self.status(count); + + let mut shape_out = Shape::from(Vec::::new()); + core::mem::swap(&mut self.shape, &mut shape_out); + + if let TensorStatus::ReadWrite = status { + // Avoids an unwanted drop on the same thread. + // + // Since `drop` is called after `into_ir`, we must not register a drop if the tensor + // was consumed with a `ReadWrite` status. + self.count.fetch_add(1, Ordering::Acquire); + } + + TensorIr { + status, + shape: shape_out, + id: self.id, + dtype: self.dtype, + } + } + + pub(crate) async fn into_data(self) -> Result + where + B: FusionBackend, + { + let id = self.stream; + let client = self.client.clone(); + let desc = self.into_ir(); + client.read_tensor_float::(desc, id).await + } + + pub(crate) async fn q_into_data(self) -> Result + where + B: FusionBackend, + { + if let DType::QFloat(_scheme) = self.dtype { + let id = self.stream; + let client = self.client.clone(); + let desc = self.into_ir(); + client.read_tensor_quantized::(desc, id).await + } else { + panic!("Expected quantized float dtype, got {:?}", self.dtype) + } + } + + pub(crate) async fn int_into_data(self) -> Result + where + B: FusionBackend, + { + let id = self.stream; + let client = self.client.clone(); + let desc = self.into_ir(); + client.read_tensor_int::(desc, id).await + } + + pub(crate) async fn bool_into_data(self) -> Result + where + B: FusionBackend, + { + let id = self.stream; + let client = self.client.clone(); + let desc = self.into_ir(); + client.read_tensor_bool::(desc, id).await + } +} + +#[derive(new, Debug)] +pub(crate) struct DropOp { + pub(crate) id: TensorId, +} + +impl Operation for DropOp { + fn execute(&self, handles: &mut burn_ir::HandleContainer) { + handles.remove_handle(self.id); + } +} + +impl Drop for FusionTensor { + fn drop(&mut self) { + let count = self.count.fetch_sub(1, Ordering::Acquire); + + // Workaround to prevent segfaults when an operation panics + if std::thread::panicking() { + return; + } + + match self.status(count) { + TensorStatus::ReadWrite => { + let mut shape = Shape::from(Vec::::new()); + core::mem::swap(&mut shape, &mut self.shape); + + let ir = TensorIr { + id: self.id, + shape, + status: TensorStatus::ReadWrite, + dtype: self.dtype, + }; + let mut streams = OperationStreams::default(); + streams.tensor(self); + + self.client + .register(streams, OperationIr::Drop(ir), DropOp { id: self.id }); + } + TensorStatus::ReadOnly => {} + TensorStatus::NotInit => {} + } + } +} + +impl QTensorPrimitive for FusionTensor { + fn scheme(&self) -> &QuantScheme { + if let DType::QFloat(scheme) = &self.dtype { + scheme + } else { + panic!( + "Quantization scheme is not valid for dtype {:?}", + self.dtype, + ) + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-ir/Cargo.toml new file mode 100644 index 0000000..a850f3e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/Cargo.toml @@ -0,0 +1,33 @@ +[package] +authors = ["laggui ", "nathanielsimard "] +categories = ["science"] +description = "Intermediate representation for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor"] +license.workspace = true +name = "burn-ir" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-ir" +documentation = "https://docs.rs/burn-ir" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std"] +std = ["burn-backend/std"] +doc = ["default"] +tracing = [ + "burn-backend/tracing", +] + +[dependencies] +serde = { workspace = true } +hashbrown = { workspace = true } # no_std compatible + +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/README.md b/crates/stable-diffusion-burn/burn-crates/burn-ir/README.md new file mode 100644 index 0000000..3e26709 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/README.md @@ -0,0 +1,7 @@ +# Burn Intermediate Representation + +Defines an Intermediate Representation (IR) used to represent tensors and operations. + +The abstraction over computation allows execution across different targets (e.g., remote backend). +It also enables optimization and transformation of tensor computations before execution (e.g., +operator fusion). diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/backend.rs new file mode 100644 index 0000000..b9ac29c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/backend.rs @@ -0,0 +1,63 @@ +use burn_backend::{ + Backend, Shape, + tensor::{BoolTensor, FloatTensor, IntTensor, QuantizedTensor}, +}; + +/// A tensor representation containing a reference to a tensor resource with a given shape. +#[derive(Clone)] +pub struct TensorHandle { + /// The type that can be used to point to a tensor of any kind. + pub handle: H, + /// The shape associated to the tensor. + pub shape: Shape, +} + +/// Backend extension trait that allows an existing [backend](Backend) to use the Burn tensor +/// intermediate representation for compilation purpose or other... +pub trait BackendIr: Backend { + /// The type that can be used to point to a tensor of any kind. + type Handle: Sync + Send + Clone; + + /// Convert a [handle](BackendIr::Handle) to a [float tensor](Backend::FloatTensorPrimitive). + fn float_tensor(handle: TensorHandle) -> FloatTensor; + /// Convert a [handle](BackendIr::Handle) to an [int tensor](Backend::IntTensorPrimitive). + fn int_tensor(handle: TensorHandle) -> IntTensor; + /// Convert a [handle](BackendIr::Handle) to a [bool tensor](Backend::BoolTensorPrimitive). + fn bool_tensor(handle: TensorHandle) -> BoolTensor; + /// Convert a [handle](BackendIr::Handle) to a [quantized tensor](Backend::QuantizedTensorPrimitive). + fn quantized_tensor(handle: TensorHandle) -> QuantizedTensor; + + /// Convert a [float tensor](Backend::FloatTensorPrimitive) to a [handle](BackendIr::Handle). + fn float_tensor_handle(tensor: FloatTensor) -> Self::Handle; + /// Convert an [int tensor](Backend::IntTensorPrimitive) to a [handle](BackendIr::Handle). + fn int_tensor_handle(tensor: IntTensor) -> Self::Handle; + /// Convert a [bool tensor](Backend::BoolTensorPrimitive) to a [handle](BackendIr::Handle). + fn bool_tensor_handle(tensor: BoolTensor) -> Self::Handle; + /// Convert a [quantized tensor](Backend::QuantizedTensorPrimitive) to a [handle](BackendIr::Handle). + fn quantized_tensor_handle(tensor: QuantizedTensor) -> Self::Handle; +} + +/// Handle which points to a backend tensor primitive kind. +#[derive(Clone, Debug)] +pub enum HandleKind { + /// Float tensor handle. + Float(B::FloatTensorPrimitive), + /// Int tensor handle. + Int(B::IntTensorPrimitive), + /// Bool tensor handle. + Bool(B::BoolTensorPrimitive), + /// Quantized tensor handle. + Quantized(B::QuantizedTensorPrimitive), +} + +impl HandleKind { + /// Returns the handle kind name. + pub fn name(&self) -> &str { + match self { + HandleKind::Float(_) => "float", + HandleKind::Int(_) => "int", + HandleKind::Bool(_) => "bool", + HandleKind::Quantized(_) => "quantized", + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/builder.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/builder.rs new file mode 100644 index 0000000..7bd2a4a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/builder.rs @@ -0,0 +1,1113 @@ +#![allow(missing_docs)] + +use alloc::vec::Vec; +use burn_backend::{ + DType, Distribution, Shape, Slice, SliceOps, calculate_matmul_output, + ops::{ + conv::{ + calculate_conv_output_shape, calculate_conv_transpose_output_shape, + calculate_pool_output_shape, + }, + unfold::calculate_unfold_shape, + }, + quantization::QuantScheme, + tensor::IndexingUpdateOp, +}; + +use crate::{ScalarIr, TensorId, TensorIr}; + +use super::operation::*; + +impl CreationOpIr { + pub fn create(shape: Shape, dtype: DType, new_id: impl FnOnce() -> TensorId) -> Self { + let out = TensorIr::uninit(new_id(), shape, dtype); + + CreationOpIr { out } + } +} + +impl InitOperationIr { + pub fn create(shape: Shape, dtype: DType, new_id: impl FnOnce() -> TensorId) -> Self { + let out = TensorIr::uninit(new_id(), shape, dtype); + + InitOperationIr { out } + } +} + +impl RandomOpIr { + pub fn create( + shape: Shape, + dtype: DType, + distribution: Distribution, + new_id: impl FnOnce() -> TensorId, + ) -> Self { + let out = TensorIr::uninit(new_id(), shape, dtype); + + RandomOpIr { out, distribution } + } +} + +impl FullOpIr { + pub fn create( + shape: Shape, + dtype: DType, + value: ScalarIr, + new_id: impl FnOnce() -> TensorId, + ) -> Self { + // TODO: check that ScalarIr dtype matches dtype? + let out = TensorIr::uninit(new_id(), shape, dtype); + + FullOpIr { out, value } + } +} + +impl CastOpIr { + pub fn create(input: TensorIr, dtype: DType, new_id: impl FnOnce() -> TensorId) -> Self { + let out = TensorIr::uninit(new_id(), input.shape.clone(), dtype); + CastOpIr { input, out } + } +} + +impl ShapeOpIr { + pub fn expand(input: TensorIr, shape: Shape, new_id: impl FnOnce() -> TensorId) -> Self { + let shape = input.shape.expand(shape).unwrap(); + Self::create(input, shape, new_id) + } + + pub fn reshape(input: TensorIr, shape: Shape, new_id: impl FnOnce() -> TensorId) -> Self { + let shape = input.shape.reshape(shape).unwrap(); + Self::create(input, shape, new_id) + } + + fn create(input: TensorIr, shape: Shape, new_id: impl FnOnce() -> TensorId) -> Self { + let out = TensorIr::uninit(new_id(), shape, input.dtype); + ShapeOpIr { input, out } + } +} + +// "Lower" specific operations into a binary or unary op representation. +// Useful when collecting inputs and outputs and don't care about the other semantics. +impl From for BinaryOpIr { + fn from(value: MatmulOpIr) -> Self { + Self { + lhs: value.lhs, + rhs: value.rhs, + out: value.out, + } + } +} + +impl From for UnaryOpIr { + fn from(value: ReduceOpIr) -> Self { + Self { + input: value.input, + out: value.out, + } + } +} + +#[derive(Debug)] +#[allow(missing_docs)] +pub enum IrError { + DTypeMismatch, +} + +fn dtype_compat(lhs: &DType, rhs: &DType) -> bool { + let lhs_qfloat = matches!(lhs, DType::QFloat(_)); + let rhs_qfloat = matches!(rhs, DType::QFloat(_)); + if lhs_qfloat && (rhs_qfloat || rhs.is_float()) + || lhs.is_float() && (rhs_qfloat || rhs.is_float()) + { + true + } else { + lhs == rhs + } +} + +fn output_check<'a, I>(inputs: I, compat: impl Fn(&DType, &DType) -> bool) -> Result +where + I: IntoIterator, +{ + let mut iter = inputs.into_iter(); + let first = iter.next().unwrap(); + for d in iter { + if !compat(first, d) { + return Err(IrError::DTypeMismatch); + } + } + Ok(*first) +} + +fn output_dtype<'a, I: IntoIterator>(inputs: I) -> Result { + output_check(inputs, |a, b| a == b) +} + +fn output_dtype_mixed<'a, I: IntoIterator>(inputs: I) -> Result { + output_check(inputs, dtype_compat) +} + +/// Macro to implement `create` constructors for operations with a single output. +/// +/// Supports shape and dtype validation. +macro_rules! impl_ir_create { + (@create_fn $op:ident { $( $field:ident : $ty:ty ),* $(,)? } , $shape:expr, $dtype:expr) => { + #[doc = "Create a new operation IR from the given inputs."] + #[doc = "`new_id` should generate a unique `TensorId` for the uninitialized output tensor."] + #[allow(clippy::too_many_arguments)] + pub fn create($( $field : $ty ),*, new_id: impl FnOnce() -> crate::TensorId) -> $op { + let shape = $shape; + let dtype = $dtype; + let out = TensorIr::uninit(new_id(), shape, dtype); + $op { $( $field ),*, out } + } + }; + + // Case: simple op, single `create` + ( + $op:ident { $( $field:ident : $ty:ty ),* $(,)? }, + shape = $shape:expr, + dtype = $dtype:expr + ) => { + impl $op { + impl_ir_create!(@create_fn $op { $( $field : $ty ),* }, $shape, $dtype); + } + }; + + // Case: op with one additional constructor that accepts an explicit output dtype + ( + $op:ident { $( $field:ident : $ty:ty ),* $(,)? }, + shape = $shape:expr, + dtype = $dtype:expr, + $fn_name:ident ( $extra:ident : $extra_ty:ty ) + ) => { + impl $op { + impl_ir_create!(@create_fn $op { $( $field : $ty ),* }, $shape, $dtype); + + #[doc = "Create a new operation IR from the given inputs and the given output dtype."] + #[allow(clippy::too_many_arguments)] + pub fn $fn_name($( $field : $ty ),*, $extra: $extra_ty, new_id: impl FnOnce() -> crate::TensorId) -> Self { + let shape = $shape; + let _ = $dtype; // still validates dtype if needed + let out = TensorIr::uninit(new_id(), shape, $extra); + $op { $( $field ),*, out } + } + } + }; +} + +impl_ir_create!( + UnaryOpIr { input: TensorIr }, + shape = input.shape.clone(), + dtype = input.dtype, + // Additional constructor for unary comparisons + create_comparison(bool_dtype: DType) +); + +impl_ir_create!( + BinaryOpIr { + lhs: TensorIr, + rhs: TensorIr + }, + shape = lhs.shape.broadcast(&rhs.shape).unwrap(), + dtype = output_dtype([&lhs.dtype, &rhs.dtype]).unwrap(), + // Additional constructor for binary comparisons + create_comparison(bool_dtype: DType) +); + +impl_ir_create!( + ScalarOpIr { + lhs: TensorIr, + rhs: ScalarIr + }, + shape = lhs.shape.clone(), + dtype = lhs.dtype, + // Additional constructor for scalar comparisons + create_comparison(bool_dtype: DType) +); + +impl_ir_create!( + MatmulOpIr { + lhs: TensorIr, + rhs: TensorIr + }, + shape = calculate_matmul_output(&lhs.shape, &rhs.shape).unwrap(), + dtype = output_dtype_mixed([&lhs.dtype, &rhs.dtype]).unwrap(), + // Additional constructor for mixed dtypes + create_mixed(out_dtype: DType) +); + +impl_ir_create!( + SwapDimsOpIr { + input: TensorIr, + dim1: usize, + dim2: usize + }, + shape = input.shape.clone().swapped(dim1, dim2).unwrap(), + dtype = input.dtype +); + +impl_ir_create!( + PermuteOpIr { input: TensorIr, axes: Vec }, + shape = input.shape.clone().permuted(&axes).unwrap(), + dtype = input.dtype +); + +impl_ir_create!( + RepeatDimOpIr { + tensor: TensorIr, + dim: usize, + times: usize + }, + shape = tensor.shape.clone().repeat(dim, times).unwrap(), + dtype = tensor.dtype +); + +impl_ir_create!( + FlipOpIr { input: TensorIr, axes: Vec }, + shape = input.shape.clone(), // TODO: check if axes are within the tensor dimensions + dtype = input.dtype +); + +impl_ir_create!( + CatOpIr { tensors: Vec, dim: usize }, + shape = Shape::cat(tensors.iter().map(|t| &t.shape), dim).unwrap(), + dtype = output_dtype(tensors.iter().map(|t| &t.dtype)).unwrap() +); + +impl_ir_create!( + GatherOpIr { + tensor: TensorIr, + dim: usize, + indices: TensorIr + }, + shape = indices.shape.clone(), // TODO: check dims compat between tensor and indices + dtype = tensor.dtype +); + +impl_ir_create!( + ScatterOpIr { + tensor: TensorIr, + dim: usize, + indices: TensorIr, + value: TensorIr, + update: IndexingUpdateOp + }, + shape = tensor.shape.clone(), // TODO: check dims compat between tensor and indices + dtype = output_dtype([&tensor.dtype, &value.dtype]).unwrap() +); + +impl_ir_create!( + ReduceOpIr { input: TensorIr }, + shape = [1].into(), + dtype = input.dtype +); + +impl_ir_create!( + ReduceDimOpIr { + input: TensorIr, + axis: usize + }, + shape = input.shape.clone().reduce(axis).unwrap(), + dtype = input.dtype, + // Additional constructor for argument reduction + create_arg(ind_dtype: DType) +); + +impl_ir_create!( + DimOpIr { + input: TensorIr, + axis: usize + }, + shape = input.shape.clone(), // TODO: check dims within rank + dtype = input.dtype +); + +impl_ir_create!( + SelectOpIr { + tensor: TensorIr, + dim: usize, + indices: TensorIr + }, + // TODO: shape.select? + shape = { + let mut s = tensor.shape.clone(); + s[dim] = indices.shape[0]; + s + }, + dtype = tensor.dtype +); + +impl_ir_create!( + SelectAssignOpIr { + tensor: TensorIr, + dim: usize, + indices: TensorIr, + value: TensorIr, + update: IndexingUpdateOp + }, + // TODO: check value and indices shape match for dim + shape = tensor.shape.clone(), + dtype = output_dtype([&tensor.dtype, &value.dtype]).unwrap() +); + +impl_ir_create!( + SliceOpIr { + tensor: TensorIr, + ranges: Vec, + }, + shape = tensor.shape.clone().slice(&ranges).unwrap(), + dtype = tensor.dtype +); + +impl_ir_create!( + SliceAssignOpIr { + tensor: TensorIr, + ranges: Vec, + value: TensorIr + }, + // TODO: check slice and value number of elements match + shape = tensor.shape.clone(), + dtype = output_dtype([&tensor.dtype, &value.dtype]).unwrap() +); + +impl_ir_create!( + MaskWhereOpIr { + tensor: TensorIr, + mask: TensorIr, + value: TensorIr + }, + shape = Shape::broadcast_many([&tensor.shape, &mask.shape, &value.shape]).unwrap(), + dtype = output_dtype([&tensor.dtype, &value.dtype]).unwrap() +); + +impl_ir_create!( + MaskFillOpIr { + tensor: TensorIr, + mask: TensorIr, + value: ScalarIr + }, + shape = tensor.shape.broadcast(&mask.shape).unwrap(), + dtype = tensor.dtype +); + +impl_ir_create!( + ClampOpIr { + tensor: TensorIr, + min: ScalarIr, + max: ScalarIr + }, + shape = tensor.shape.clone(), + dtype = tensor.dtype +); + +impl_ir_create!( + AvgPool1dOpIr { + x: TensorIr, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool + }, + shape = calculate_pool_output_shape( + &x.shape, + &[kernel_size], + &[stride], + &[padding], + &[1], + ceil_mode + ) + .unwrap(), + dtype = x.dtype +); + +impl_ir_create!( + AvgPool1dBackwardOpIr { + x: TensorIr, + grad: TensorIr, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + AvgPool2dOpIr { + x: TensorIr, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool + }, + shape = calculate_pool_output_shape( + &x.shape, + &kernel_size, + &stride, + &padding, + &[1, 1], + ceil_mode + ) + .unwrap(), + dtype = x.dtype +); + +impl_ir_create!( + AvgPool2dBackwardOpIr { + x: TensorIr, + grad: TensorIr, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + MaxPool1dOpIr { + x: TensorIr, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool + }, + shape = calculate_pool_output_shape( + &x.shape, + &[kernel_size], + &[stride], + &[padding], + &[dilation], + ceil_mode + ) + .unwrap(), + dtype = x.dtype +); + +impl_ir_create!( + MaxPool2dOpIr { + x: TensorIr, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool + }, + shape = calculate_pool_output_shape( + &x.shape, + &kernel_size, + &stride, + &padding, + &dilation, + ceil_mode + ) + .unwrap(), + dtype = x.dtype +); + +impl_ir_create!( + MaxPool1dWithIndicesBackwardOpIr { + x: TensorIr, + grad: TensorIr, + indices: TensorIr, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + MaxPool2dWithIndicesBackwardOpIr { + x: TensorIr, + grad: TensorIr, + indices: TensorIr, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + AdaptiveAvgPool1dOpIr { + x: TensorIr, + output_size: usize + }, + shape = Shape::new([x.shape[0], x.shape[1], output_size]), + dtype = x.dtype +); + +impl_ir_create!( + AdaptiveAvgPool2dOpIr { + x: TensorIr, + output_size: [usize; 2] + }, + shape = Shape::new([x.shape[0], x.shape[1], output_size[0], output_size[1]]), + dtype = x.dtype +); + +impl_ir_create!( + AdaptiveAvgPool1dBackwardOpIr { + x: TensorIr, + grad: TensorIr, + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + AdaptiveAvgPool2dBackwardOpIr { + x: TensorIr, + grad: TensorIr, + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + InterpolateOpIr { + x: TensorIr, + output_size: [usize; 2], + options: InterpolateOptionsIr + }, + shape = Shape::new([x.shape[0], x.shape[1], output_size[0], output_size[1]]), + dtype = x.dtype +); + +impl_ir_create!( + InterpolateBackwardOpIr { + x: TensorIr, + grad: TensorIr, + output_size: [usize; 2], + options: InterpolateOptionsIr + }, + shape = x.shape.clone(), + dtype = x.dtype +); + +impl_ir_create!( + GridSample2dOpIr { + tensor: TensorIr, + grid: TensorIr, + options: GridSampleOptionsIr + }, + // Input tensor: [N, C, H_in, W_in] + // Grid: [N, H_out, W_out, 2] + // Output: [N, C, H_out, W_out] + shape = Shape::new([ + tensor.shape[0], + tensor.shape[1], + grid.shape[1], + grid.shape[2] + ]), + dtype = tensor.dtype +); + +impl_ir_create!( + Conv1dOpIr { + x: TensorIr, + weight: TensorIr, + bias: Option, + options: Conv1dOptionsIr + }, + shape = calculate_conv_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.dilation, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + Conv1dXBackwardOpIr { + x: TensorIr, + weight: TensorIr, + output_grad: TensorIr, + options: Conv1dOptionsIr + }, + shape = x.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv1dWeightBackwardOpIr { + x: TensorIr, + weight: TensorIr, + output_grad: TensorIr, + options: Conv1dOptionsIr + }, + shape = weight.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv1dBiasBackwardOpIr { + x: TensorIr, + bias: TensorIr, + output_grad: TensorIr, + }, + shape = bias.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv2dOpIr { + x: TensorIr, + weight: TensorIr, + bias: Option, + options: Conv2dOptionsIr + }, + shape = calculate_conv_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.dilation, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + Conv2dXBackwardOpIr { + x: TensorIr, + weight: TensorIr, + output_grad: TensorIr, + options: Conv2dOptionsIr + }, + shape = x.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv2dWeightBackwardOpIr { + x: TensorIr, + weight: TensorIr, + output_grad: TensorIr, + options: Conv2dOptionsIr + }, + shape = weight.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv2dBiasBackwardOpIr { + x: TensorIr, + bias: TensorIr, + output_grad: TensorIr, + }, + shape = bias.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv3dOpIr { + x: TensorIr, + weight: TensorIr, + bias: Option, + options: Conv3dOptionsIr + }, + shape = calculate_conv_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.dilation, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + Conv3dXBackwardOpIr { + x: TensorIr, + weight: TensorIr, + output_grad: TensorIr, + options: Conv3dOptionsIr + }, + shape = x.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv3dWeightBackwardOpIr { + x: TensorIr, + weight: TensorIr, + output_grad: TensorIr, + options: Conv3dOptionsIr + }, + shape = weight.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + Conv3dBiasBackwardOpIr { + x: TensorIr, + bias: TensorIr, + output_grad: TensorIr, + }, + shape = bias.shape.clone(), + dtype = output_grad.dtype +); + +impl_ir_create!( + DeformConv2dOpIr { + x: TensorIr, + offset: TensorIr, + weight: TensorIr, + mask: Option, + bias: Option, + options: DeformableConv2dOptionsIr + }, + shape = calculate_conv_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.dilation, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&offset.dtype), + Some(&weight.dtype), + mask.as_ref().map(|m| &m.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + ConvTranspose1dOpIr { + x: TensorIr, + weight: TensorIr, + bias: Option, + options: ConvTranspose1dOptionsIr + }, + shape = calculate_conv_transpose_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.padding_out, + &options.dilation, + options.groups, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + ConvTranspose2dOpIr { + x: TensorIr, + weight: TensorIr, + bias: Option, + options: ConvTranspose2dOptionsIr + }, + shape = calculate_conv_transpose_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.padding_out, + &options.dilation, + options.groups, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + ConvTranspose3dOpIr { + x: TensorIr, + weight: TensorIr, + bias: Option, + options: ConvTranspose3dOptionsIr + }, + shape = calculate_conv_transpose_output_shape( + &x.shape, + &weight.shape, + &options.stride, + &options.padding, + &options.padding_out, + &options.dilation, + options.groups, + ) + .unwrap(), + dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap() +); + +impl_ir_create!( + UnfoldOpIr { + input: TensorIr, + dim: usize, + size: usize, + step: usize + }, + shape = calculate_unfold_shape(input.shape.clone(), dim, size, step), + dtype = input.dtype +); + +impl_ir_create!( + CrossOpIr { + lhs: TensorIr, + rhs: TensorIr, + dim: usize + }, + shape = lhs.shape.broadcast(&rhs.shape).unwrap(), + dtype = output_dtype([&lhs.dtype, &rhs.dtype]).unwrap() +); + +impl_ir_create!( + QuantizeOpIr { + tensor: TensorIr, + qparams: QuantizationParametersIr, + scheme: QuantScheme + }, + shape = tensor.shape.clone(), + dtype = DType::QFloat(scheme) +); + +impl_ir_create!( + AttentionOpIr { + query: TensorIr, + key: TensorIr, + value: TensorIr, + mask: Option, + attn_bias: Option, + options: AttentionOptionsIr, + }, + shape = Shape::new([query.shape[0], query.shape[1], query.shape[2], value.shape[3]]), + dtype = query.dtype +); + +impl DequantizeOpIr { + pub fn create(input: TensorIr, dtype: DType, new_id: impl FnOnce() -> TensorId) -> Self { + let out = TensorIr::uninit(new_id(), input.shape.clone(), dtype); + + DequantizeOpIr { input, out } + } +} + +// Operations with multiple outputs + +impl ReduceDimWithIndicesOpIr { + pub fn create( + tensor: TensorIr, + dim: usize, + dtype_indices: DType, + mut new_id: impl FnMut() -> TensorId, + ) -> Self { + let mut shape = tensor.shape.clone(); + shape[dim] = 1; + let out = TensorIr::uninit(new_id(), shape.clone(), tensor.dtype); + let out_indices = TensorIr::uninit(new_id(), shape.clone(), dtype_indices); + + ReduceDimWithIndicesOpIr { + tensor, + dim, + out, + out_indices, + } + } +} + +impl DeformConv2dBackwardOpIr { + #[allow(clippy::too_many_arguments)] + pub fn create( + x: TensorIr, + offset: TensorIr, + weight: TensorIr, + mask: Option, + bias: Option, + out_grad: TensorIr, + options: DeformableConv2dOptionsIr, + mut new_id: impl FnMut() -> TensorId, + ) -> Self { + let dtype = output_dtype( + [ + Some(&x.dtype), + Some(&weight.dtype), + mask.as_ref().map(|m| &m.dtype), + bias.as_ref().map(|b| &b.dtype), + ] + .iter() + .filter_map(|&d| d), + ) + .unwrap(); + + let input_grad = TensorIr::uninit(new_id(), x.shape.clone(), dtype); + let offset_grad = TensorIr::uninit(new_id(), offset.shape.clone(), dtype); + let weight_grad = TensorIr::uninit(new_id(), weight.shape.clone(), dtype); + let mask_grad = mask + .as_ref() + .map(|t| TensorIr::uninit(new_id(), t.shape.clone(), dtype)); + let bias_grad = bias + .as_ref() + .map(|t| TensorIr::uninit(new_id(), t.shape.clone(), dtype)); + + DeformConv2dBackwardOpIr { + x, + offset, + weight, + mask, + bias, + out_grad, + options, + input_grad, + offset_grad, + weight_grad, + mask_grad, + bias_grad, + } + } +} + +impl MaxPool1dWithIndicesOpIr { + #[allow(clippy::too_many_arguments)] + pub fn create( + x: TensorIr, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + dtype_indices: DType, + mut new_id: impl FnMut() -> TensorId, + ) -> Self { + let shape = calculate_pool_output_shape( + &x.shape, + &[kernel_size], + &[stride], + &[padding], + &[dilation], + ceil_mode, + ) + .unwrap(); + let out = TensorIr::uninit(new_id(), shape.clone(), x.dtype); + let out_indices = TensorIr::uninit(new_id(), shape, dtype_indices); + + MaxPool1dWithIndicesOpIr { + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + out, + out_indices, + } + } +} + +impl MaxPool2dWithIndicesOpIr { + #[allow(clippy::too_many_arguments)] + pub fn create( + x: TensorIr, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + dtype_indices: DType, + mut new_id: impl FnMut() -> TensorId, + ) -> Self { + let shape = calculate_pool_output_shape( + &x.shape, + &kernel_size, + &stride, + &padding, + &dilation, + ceil_mode, + ) + .unwrap(); + let out = TensorIr::uninit(new_id(), shape.clone(), x.dtype); + let out_indices = TensorIr::uninit(new_id(), shape, dtype_indices); + + MaxPool2dWithIndicesOpIr { + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + out, + out_indices, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/handle.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/handle.rs new file mode 100644 index 0000000..1b0b202 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/handle.rs @@ -0,0 +1,216 @@ +use hashbrown::HashMap; + +use crate::{BackendIr, TensorHandle, TensorId, TensorIr, TensorStatus}; + +/// Keep all [tensor handles](BackendIr::Handle) in one place and ensure that all resources +/// are used optimally. +#[derive(Default)] +pub struct HandleContainer { + handles: HashMap>, + counter: u64, +} + +impl HandleContainer { + /// Fork the container, useful for autotune. + pub fn fork(&self) -> Self { + let mut handles = HashMap::with_capacity(self.handles.len()); + + for (id, handle) in self.handles.iter() { + handles.insert(*id, handle.clone()); + } + + Self { + handles, + counter: self.counter, + } + } +} + +impl core::fmt::Debug for HandleContainer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("HandleContainer") + .field("handles", &self.handles.keys()) // only care about the IDs when debugging + .field("counter", &self.counter) + .finish() + } +} + +/// Backend [tensor handle](BackendIr::Handle) wrapper tracking their creation state +#[derive(Clone)] +pub enum Handle { + /// No [tensor handle](BackendIr::Handle) has been created yet + NotInit, + /// A [tensor handle](BackendIr::Handle) has been created + Existing(H), +} + +impl HandleContainer { + /// Create a new HandleContainer + pub fn new() -> Self { + Self { + handles: HashMap::new(), + counter: 0, + } + } + + /// Register a handle for the given [tensor id](TensorId). + pub fn register_handle(&mut self, id: TensorId, handle: H) { + self.handles.insert(id, Handle::Existing(handle)); + } + + /// Whether an handle exists. + pub fn has_handle(&mut self, id: &TensorId) -> bool { + self.handles.contains_key(id) + } + + /// Get the reference to a handle. + pub fn get_handle_ref(&self, id: &TensorId) -> Option<&H> { + self.handles + .get(id) + .filter(|h| !matches!(h, Handle::NotInit)) + .map(|h| match h { + Handle::Existing(handle) => handle, + Handle::NotInit => unreachable!(), + }) + } + + /// Get the handle for the given [tensor id](TensorId). The status is used to determine if the + /// tensor should be popped out of the current tensor map, necessary for inplace operations. + /// + /// # Warnings + /// + /// Make sure the status corresponds to the operation you want to execute the handle on, + /// otherwise you might remove a tensor handle that will be required in the future. + pub fn get_handle(&mut self, id: &TensorId, status: &TensorStatus) -> H { + let (id, handle) = self + .handles + .remove_entry(id) + .unwrap_or_else(|| panic!("Should have handle for tensor {id:?}")); + + match handle { + Handle::Existing(handle) => match status { + TensorStatus::ReadOnly => { + self.handles.insert(id, Handle::Existing(handle.clone())); + handle + } + TensorStatus::ReadWrite => handle, + TensorStatus::NotInit => panic!( + "Cannot get uninitialized tensor {id:?}. Tensor exist but with wrong status" + ), + }, + Handle::NotInit => panic!("Cannot get uninitialized handle {id:?}."), + } + } + + /// Get the tensor handle for the given [tensor intermediate representation](TensorIr). + pub fn get_tensor_handle(&mut self, tensor: &TensorIr) -> TensorHandle { + TensorHandle { + handle: self.get_handle(&tensor.id, &tensor.status), + shape: tensor.shape.clone(), + } + } + + /// Get the [float tensor](burn_backend::backend::Backend::FloatTensorPrimitive) corresponding to the + /// given [tensor intermediate representation](TensorIr). + pub fn get_float_tensor(&mut self, tensor: &TensorIr) -> B::FloatTensorPrimitive + where + B: BackendIr, + { + B::float_tensor(self.get_tensor_handle(tensor)) + } + + /// Get the [int tensor](burn_backend::backend::Backend::IntTensorPrimitive) corresponding to the + /// given [tensor intermediate representation](TensorIr). + pub fn get_int_tensor(&mut self, tensor: &TensorIr) -> B::IntTensorPrimitive + where + B: BackendIr, + { + B::int_tensor(self.get_tensor_handle(tensor)) + } + + /// Get the [bool tensor](burn_backend::backend::Backend::BoolTensorPrimitive) corresponding to the + /// given [tensor intermediate representation](TensorIr). + pub fn get_bool_tensor(&mut self, tensor: &TensorIr) -> B::BoolTensorPrimitive + where + B: BackendIr, + { + B::bool_tensor(self.get_tensor_handle(tensor)) + } + + /// Get the [quantized tensor](burn_backend::backend::Backend::QuantizedTensorPrimitive) corresponding to the + /// given [tensor intermediate representation](TensorIr). + pub fn get_quantized_tensor(&mut self, tensor: &TensorIr) -> B::QuantizedTensorPrimitive + where + B: BackendIr, + { + B::quantized_tensor(self.get_tensor_handle(tensor)) + } + + /// Register a new [float tensor](burn_backend::backend::Backend::FloatTensorPrimitive) with the corresponding [tensor id](TensorId). + pub fn register_float_tensor(&mut self, id: &TensorId, tensor: B::FloatTensorPrimitive) + where + B: BackendIr, + { + let handle = B::float_tensor_handle(tensor); + self.handles.insert(*id, Handle::Existing(handle)); + } + + /// Register a new [quantized tensor](burn_backend::backend::Backend::QuantizedTensorPrimitive) with the corresponding [tensor ids](TensorId). + pub fn register_quantized_tensor( + &mut self, + id: &TensorId, + tensor: B::QuantizedTensorPrimitive, + ) where + B: BackendIr, + { + let handle = B::quantized_tensor_handle(tensor); + self.handles.insert(*id, Handle::Existing(handle)); + } + + /// Register a new [int tensor](burn_backend::backend::Backend::IntTensorPrimitive) with the corresponding [tensor id](TensorId). + pub fn register_int_tensor(&mut self, id: &TensorId, tensor: B::IntTensorPrimitive) + where + B: BackendIr, + { + let handle = B::int_tensor_handle(tensor); + self.handles.insert(*id, Handle::Existing(handle)); + } + + /// Register a new [bool tensor](burn_backend::backend::Backend::BoolTensorPrimitive) with the corresponding [tensor id](TensorId). + pub fn register_bool_tensor(&mut self, id: &TensorId, tensor: B::BoolTensorPrimitive) + where + B: BackendIr, + { + let handle = B::bool_tensor_handle(tensor); + self.handles.insert(*id, Handle::Existing(handle)); + } + + /// Lazily create a new empty tensor and return its corresponding [tensor id](TensorId). + pub fn create_tensor_uninit(&mut self) -> TensorId { + let id = TensorId::new(self.counter); + self.counter += 1; + self.handles.insert(id, Handle::NotInit); + id + } + + /// Remove tensor handle from container. + pub fn remove_handle(&mut self, id: TensorId) -> Option> { + self.handles.remove(&id) + } + + /// Remove tensor handle from container if writable + pub fn free(&mut self, tensor: &TensorIr) { + match tensor.status { + TensorStatus::ReadOnly => (), + TensorStatus::NotInit => (), + TensorStatus::ReadWrite => { + self.handles.remove(&tensor.id); + } + }; + } + + /// Returns the number of handles. + pub fn num_handles(&self) -> usize { + self.handles.len() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/lib.rs new file mode 100644 index 0000000..a60e3db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/lib.rs @@ -0,0 +1,21 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! Burn intermediate representation. + +extern crate alloc; + +mod backend; +mod builder; +mod handle; +mod operation; +mod scalar; +mod tensor; + +pub use backend::*; +pub use builder::*; +pub use handle::*; +pub use operation::*; +pub use scalar::*; +pub use tensor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/operation.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/operation.rs new file mode 100644 index 0000000..9f00881 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/operation.rs @@ -0,0 +1,3011 @@ +use burn_backend::ops::AttentionModuleOptions; +use burn_backend::tensor::IndexingUpdateOp; +use core::hash::Hash; +use serde::{Deserialize, Serialize}; + +use alloc::borrow::ToOwned; +use alloc::boxed::Box; +use alloc::{string::String, vec::Vec}; + +use burn_backend::{ + DType, Distribution, Slice, + ops::{ + ConvOptions, ConvTransposeOptions, DeformConvOptions, GridSampleOptions, + GridSamplePaddingMode, InterpolateMode, InterpolateOptions, + }, + quantization::QuantScheme, +}; + +use crate::{ScalarIr, TensorId, TensorIr, TensorStatus}; + +/// Custom operation in fusion stream, declaring its inputs and outputs. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct CustomOpIr { + /// Unique identifier of the operation. + pub id: String, + /// Input tensors used in the custom operation. + pub inputs: Vec, + /// Output tensors used in the custom operation. + pub outputs: Vec, +} + +impl CustomOpIr { + /// Create a new custom operation intermediate representation. + pub fn new(id: &'static str, inputs: &[TensorIr], outputs: &[TensorIr]) -> Self { + Self { + id: id.to_owned(), + inputs: inputs.to_vec(), + outputs: outputs.to_vec(), + } + } + + /// Cast the intermediate representation, and get the in and output tensors. + pub fn as_fixed( + &self, + ) -> (&[TensorIr; N_IN], &[TensorIr; N_OUT]) { + ( + self.inputs.as_slice().try_into().expect( + "Wrong number of inputs expected (expected {D}, is {}), check your implementation", + ), + self.outputs.as_slice().try_into().expect( + "Wrong number of outputs expected (expected {D}, is {}), check your implementation", + ), + ) + } + + fn inputs(&self) -> Box + '_> { + Box::new(self.inputs.iter()) + } + + fn outputs(&self) -> Box + '_> { + Box::new(self.outputs.iter()) + } +} + +/// Describe all tensor operations possible. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(clippy::large_enum_variant)] +pub enum OperationIr { + /// Basic operation on a float tensor. + BaseFloat(BaseOperationIr), + /// Basic operation on an int tensor. + BaseInt(BaseOperationIr), + /// Basic operation on a bool tensor. + BaseBool(BaseOperationIr), + /// Numeric operation on a float tensor. + NumericFloat(DType, NumericOperationIr), + /// Numeric operation on an int tensor. + NumericInt(DType, NumericOperationIr), + /// Operation specific to a bool tensor. + Bool(BoolOperationIr), + /// Operation specific to an int tensor. + Int(IntOperationIr), + /// Operation specific to a float tensor. + Float(DType, FloatOperationIr), + /// Module operation. + Module(ModuleOperationIr), + /// Initialize operation. + Init(InitOperationIr), + /// A custom operation. + Custom(CustomOpIr), + /// A tensor is dropped. + Drop(TensorIr), +} + +/// Operation intermediate representation specific to a float tensor. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub enum FloatOperationIr { + /// Operation corresponding to [exp](burn_backend::ops::FloatTensorOps::float_exp). + Exp(UnaryOpIr), + /// Operation corresponding to [log](burn_backend::ops::FloatTensorOps::float_log). + Log(UnaryOpIr), + /// Operation corresponding to [log1p](burn_backend::ops::FloatTensorOps::float_log1p). + Log1p(UnaryOpIr), + /// Operation corresponding to [erf](burn_backend::ops::FloatTensorOps::float_erf). + Erf(UnaryOpIr), + /// Operation corresponding to [powf_scalar](burn_backend::ops::FloatTensorOps::float_powf_scalar). + PowfScalar(ScalarOpIr), + /// Operation corresponding to [sqrt](burn_backend::ops::FloatTensorOps::float_sqrt). + Sqrt(UnaryOpIr), + /// Operation corresponding to [cos](burn_backend::ops::FloatTensorOps::float_cos). + Cos(UnaryOpIr), + /// Operation corresponding to [cosh](burn_backend::ops::FloatTensorOps::float_cosh). + Cosh(UnaryOpIr), + /// Operation corresponding to [sin](burn_backend::ops::FloatTensorOps::float_sin). + Sin(UnaryOpIr), + /// Operation corresponding to [sin](burn_backend::ops::FloatTensorOps::float_sinh). + Sinh(UnaryOpIr), + /// Operation corresponding to [tan](burn_backend::ops::FloatTensorOps::float_tan). + Tan(UnaryOpIr), + /// Operation corresponding to [tanh](burn_backend::ops::FloatTensorOps::float_tanh). + Tanh(UnaryOpIr), + /// Operation corresponding to [acos](burn_backend::ops::FloatTensorOps::float_acos). + ArcCos(UnaryOpIr), + /// Operation corresponding to [acosh](burn_backend::ops::FloatTensorOps::float_acosh). + ArcCosh(UnaryOpIr), + /// Operation corresponding to [asin](burn_backend::ops::FloatTensorOps::float_asin). + ArcSin(UnaryOpIr), + /// Operation corresponding to [asinh](burn_backend::ops::FloatTensorOps::float_asinh). + ArcSinh(UnaryOpIr), + /// Operation corresponding to [atan](burn_backend::ops::FloatTensorOps::float_atan). + ArcTan(UnaryOpIr), + /// Operation corresponding to [atanh](burn_backend::ops::FloatTensorOps::float_atanh). + ArcTanh(UnaryOpIr), + /// Operation corresponding to [atan2](burn_backend::ops::FloatTensorOps::float_atan2). + ArcTan2(BinaryOpIr), + /// Operation corresponding to [round](burn_backend::ops::FloatTensorOps::float_round). + Round(UnaryOpIr), + /// Operation corresponding to [floor](burn_backend::ops::FloatTensorOps::float_floor). + Floor(UnaryOpIr), + /// Operation corresponding to [ceil](burn_backend::ops::FloatTensorOps::float_ceil). + Ceil(UnaryOpIr), + /// Operation corresponding to [trunc](burn_backend::ops::FloatTensorOps::float_trunc). + Trunc(UnaryOpIr), + /// Operation corresponding to [into_int](burn_backend::ops::FloatTensorOps::float_into_int). + IntoInt(CastOpIr), + /// Operation corresponding to [matmul](burn_backend::ops::FloatTensorOps::float_matmul). + Matmul(MatmulOpIr), + /// Operation corresponding to [cross](burn_backend::ops::FloatTensorOps::float_cross). + Cross(CrossOpIr), + /// Operation corresponding to [random](burn_backend::ops::FloatTensorOps::float_random). + Random(RandomOpIr), + /// Operation corresponding to [recip](burn_backend::ops::FloatTensorOps::float_recip). + Recip(UnaryOpIr), + /// Operation corresponding to [is_nan](burn_backend::ops::FloatTensorOps::float_is_nan). + IsNan(UnaryOpIr), + /// Operation corresponding to [is_nan](burn_backend::ops::FloatTensorOps::float_is_inf). + IsInf(UnaryOpIr), + /// Operation corresponding to [quantize](burn_backend::ops::QTensorOps::quantize). + Quantize(QuantizeOpIr), + /// Operation corresponding to [dequantize](burn_backend::ops::QTensorOps::dequantize). + Dequantize(DequantizeOpIr), + /// Operation corresponding to [grid_sample_2d](burn_backend::ops::FloatTensorOps::float_grid_sample_2d). + GridSample2d(GridSample2dOpIr), +} + +/// Operation intermediate representation specific to module. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub enum ModuleOperationIr { + /// Operation corresponding to [embedding](burn_backend::ops::ModuleOps::embedding). + Embedding(EmbeddingOpIr), + /// Operation corresponding to [embedding_backward](burn_backend::ops::ModuleOps::embedding_backward). + EmbeddingBackward(EmbeddingBackwardOpIr), + /// Operation corresponding to [conv1d](burn_backend::ops::ModuleOps::conv1d). + Conv1d(Conv1dOpIr), + /// Operation corresponding to [conv1d_x_backward](burn_backend::ops::ModuleOps::conv1d_x_backward). + Conv1dXBackward(Conv1dXBackwardOpIr), + /// Operation corresponding to [conv1d_weight_backward](burn_backend::ops::ModuleOps::conv1d_weight_backward). + Conv1dWeightBackward(Conv1dWeightBackwardOpIr), + /// Operation corresponding to [conv1d_bias_backward](burn_backend::ops::ModuleOps::conv1d_bias_backward). + Conv1dBiasBackward(Conv1dBiasBackwardOpIr), + /// Operation corresponding to [conv2d](burn_backend::ops::ModuleOps::conv2d). + Conv2d(Conv2dOpIr), + /// Operation corresponding to [conv2d_x_backward](burn_backend::ops::ModuleOps::conv2d_x_backward). + Conv2dXBackward(Conv2dXBackwardOpIr), + /// Operation corresponding to [conv2d_weight_backward](burn_backend::ops::ModuleOps::conv2d_weight_backward). + Conv2dWeightBackward(Conv2dWeightBackwardOpIr), + /// Operation corresponding to [conv2d_bias_backward](burn_backend::ops::ModuleOps::conv2d_bias_backward). + Conv2dBiasBackward(Conv2dBiasBackwardOpIr), + /// Operation corresponding to [conv3d](burn_backend::ops::ModuleOps::conv3d). + Conv3d(Conv3dOpIr), + /// Operation corresponding to [conv3d_x_backward](burn_backend::ops::ModuleOps::conv3d_x_backward). + Conv3dXBackward(Conv3dXBackwardOpIr), + /// Operation corresponding to [conv3d_weight_backward](burn_backend::ops::ModuleOps::conv3d_weight_backward). + Conv3dWeightBackward(Conv3dWeightBackwardOpIr), + /// Operation corresponding to [conv3d_bias_backward](burn_backend::ops::ModuleOps::conv3d_bias_backward). + Conv3dBiasBackward(Conv3dBiasBackwardOpIr), + /// Operation corresponding to [deform_conv2d](burn_backend::ops::ModuleOps::deform_conv2d) + DeformableConv2d(Box), + /// Operation corresponding to [deform_conv2d_backward](burn_backend::ops::ModuleOps::deform_conv2d_backward) + DeformableConv2dBackward(Box), + /// Operation corresponding to [conv transpose 1d](burn_backend::ops::ModuleOps::conv_transpose1d). + ConvTranspose1d(ConvTranspose1dOpIr), + /// Operation corresponding to [conv transpose 2d](burn_backend::ops::ModuleOps::conv_transpose2d). + ConvTranspose2d(ConvTranspose2dOpIr), + /// Operation corresponding to [conv transpose 3d](burn_backend::ops::ModuleOps::conv_transpose3d). + ConvTranspose3d(ConvTranspose3dOpIr), + /// Operation corresponding to [avg pool 1d](burn_backend::ops::ModuleOps::avg_pool1d). + AvgPool1d(AvgPool1dOpIr), + /// Operation corresponding to [avg pool 2d](burn_backend::ops::ModuleOps::avg_pool2d). + AvgPool2d(AvgPool2dOpIr), + /// Operation corresponding to + /// [avg pool 1d backward](burn_backend::ops::ModuleOps::avg_pool1d_backward). + AvgPool1dBackward(AvgPool1dBackwardOpIr), + /// Operation corresponding to + /// [avg pool 2d backward](burn_backend::ops::ModuleOps::avg_pool2d_backward). + AvgPool2dBackward(AvgPool2dBackwardOpIr), + /// Operation corresponding to + /// [adaptive avg pool 1d](burn_backend::ops::ModuleOps::adaptive_avg_pool1d). + AdaptiveAvgPool1d(AdaptiveAvgPool1dOpIr), + /// Operation corresponding to + /// [adaptive avg pool 2d](burn_backend::ops::ModuleOps::adaptive_avg_pool2d). + AdaptiveAvgPool2d(AdaptiveAvgPool2dOpIr), + /// Operation corresponding to + /// [adaptive avg pool 1d backward](burn_backend::ops::ModuleOps::adaptive_avg_pool1d_backward). + AdaptiveAvgPool1dBackward(AdaptiveAvgPool1dBackwardOpIr), + /// Operation corresponding to + /// [adaptive avg pool 2d backward](burn_backend::ops::ModuleOps::adaptive_avg_pool2d_backward). + AdaptiveAvgPool2dBackward(AdaptiveAvgPool2dBackwardOpIr), + /// Operation corresponding to + /// [max pool 1d](burn_backend::ops::ModuleOps::max_pool1d). + MaxPool1d(MaxPool1dOpIr), + /// Operation corresponding to + /// [max pool 1d with indices](burn_backend::ops::ModuleOps::max_pool1d_with_indices). + MaxPool1dWithIndices(MaxPool1dWithIndicesOpIr), + /// Operation corresponding to + /// [max pool 1d with indices backward](burn_backend::ops::ModuleOps::max_pool1d_with_indices_backward). + MaxPool1dWithIndicesBackward(MaxPool1dWithIndicesBackwardOpIr), + /// Operation corresponding to + /// [max pool 2d](burn_backend::ops::ModuleOps::max_pool1d). + MaxPool2d(MaxPool2dOpIr), + /// Operation corresponding to + /// [max pool 2d with indices](burn_backend::ops::ModuleOps::max_pool2d_with_indices). + MaxPool2dWithIndices(MaxPool2dWithIndicesOpIr), + /// Operation corresponding to + /// [max pool 2d with indices backward](burn_backend::ops::ModuleOps::max_pool2d_with_indices_backward). + MaxPool2dWithIndicesBackward(MaxPool2dWithIndicesBackwardOpIr), + /// Operation corresponding to [interpolate](burn_backend::ops::ModuleOps::interpolate). + Interpolate(InterpolateOpIr), + /// Operation corresponding to [interpolate backward](burn_backend::ops::ModuleOps::interpolate_backward). + InterpolateBackward(InterpolateBackwardOpIr), + /// Operation corresponding to [attention](burn_backend::ops::ModuleOps::attention). + Attention(AttentionOpIr), +} + +/// Basic operations that can be done on any tensor type. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub enum BaseOperationIr { + /// Operation corresponding to: + /// + /// Float => [reshape](burn_backend::ops::FloatTensorOps::float_reshape). + /// Int => [reshape](burn_backend::ops::IntTensorOps::int_reshape). + /// Bool => [reshape](burn_backend::ops::BoolTensorOps::bool_reshape). + Reshape(ShapeOpIr), + + /// Operation corresponding to: + /// + /// Float => [swap_dims](burn_backend::ops::FloatTensorOps::float_swap_dims). + /// Int => [swap_dims](burn_backend::ops::IntTensorOps::int_swap_dims). + /// Bool => [swap_dims](burn_backend::ops::BoolTensorOps::bool_swap_dims). + SwapDims(SwapDimsOpIr), + + /// Operation corresponding to: + /// + /// Float => [permute](burn_backend::ops::FloatTensorOps::float_permute). + /// Int => [permute](burn_backend::ops::IntTensorOps::int_permute). + /// Bool => [permute](burn_backend::ops::BoolTensorOps::bool_permute). + Permute(PermuteOpIr), + + /// Operation corresponding to: + /// Float => [flip](burn_backend::ops::FloatTensorOps::float_flip). + /// Int => [flip](burn_backend::ops::IntTensorOps::int_flip). + /// Bool => [flip](burn_backend::ops::BoolTensorOps::bool_flip). + Flip(FlipOpIr), + + /// Operation corresponding to: + /// + /// Float => [expand](burn_backend::ops::FloatTensorOps::float_expand). + /// Int => [expand](burn_backend::ops::IntTensorOps::int_expand). + /// Bool => [expand](burn_backend::ops::BoolTensorOps::bool_expand). + Expand(ShapeOpIr), + + /// Unfold windows along an axis. + /// + Unfold(UnfoldOpIr), + + /// Operation corresponding to: + /// + /// Float => [slice](burn_backend::ops::FloatTensorOps::float_slice). + /// Int => [slice](burn_backend::ops::IntTensorOps::int_slice). + /// Bool => [slice](burn_backend::ops::BoolTensorOps::bool_slice). + Slice(SliceOpIr), + /// Operation corresponding to: + /// + /// Float => [slice assign](burn_backend::ops::FloatTensorOps::float_slice_assign). + /// Int => [slice assign](burn_backend::ops::IntTensorOps::int_slice_assign). + /// Bool => [slice assign](burn_backend::ops::BoolTensorOps::bool_slice_assign). + SliceAssign(SliceAssignOpIr), + /// Operation corresponding to: + /// + /// Float => [select](burn_backend::ops::FloatTensorOps::float_select). + /// Int => [select](burn_backend::ops::IntTensorOps::int_select). + /// Bool => [select](burn_backend::ops::BoolTensorOps::bool_select). + Select(SelectOpIr), + /// Operation corresponding to: + /// + /// Float => [select assign](burn_backend::ops::FloatTensorOps::float_select_add). + /// Int => [select assign](burn_backend::ops::IntTensorOps::int_select_add). + /// Bool => [select assign](burn_backend::ops::BoolTensorOps::bool_select_or). + SelectAssign(SelectAssignOpIr), + /// Operation corresponding to: + /// + /// Float => [mask where](burn_backend::ops::FloatTensorOps::float_mask_where). + /// Int => [mask where](burn_backend::ops::IntTensorOps::int_mask_where). + /// Bool => [mask where](burn_backend::ops::BoolTensorOps::bool_mask_where). + MaskWhere(MaskWhereOpIr), + /// Operation corresponding to: + /// + /// Float => [mask fill](burn_backend::ops::FloatTensorOps::float_mask_fill). + /// Int => [mask fill](burn_backend::ops::IntTensorOps::int_mask_fill). + /// Bool => [mask fill](burn_backend::ops::BoolTensorOps::bool_mask_fill). + MaskFill(MaskFillOpIr), + /// Operation corresponding to: + /// + /// Float => [gather](burn_backend::ops::FloatTensorOps::float_gather). + /// Int => [gather](burn_backend::ops::IntTensorOps::int_gather). + /// Bool => [gather](burn_backend::ops::BoolTensorOps::bool_gather). + Gather(GatherOpIr), + /// Operation corresponding to: + /// + /// Float => [scatter](burn_backend::ops::FloatTensorOps::float_scatter_add). + /// Int => [scatter](burn_backend::ops::IntTensorOps::int_scatter_add). + /// Bool => [scatter](burn_backend::ops::BoolTensorOps::bool_scatter_or). + Scatter(ScatterOpIr), + /// Operation corresponding to: + /// + /// Float => [equal](burn_backend::ops::FloatTensorOps::float_equal). + /// Int => [equal](burn_backend::ops::IntTensorOps::int_equal). + /// Bool => [equal](burn_backend::ops::BoolTensorOps::bool_equal). + Equal(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [equal elem](burn_backend::ops::FloatTensorOps::float_equal_elem). + /// Int => [equal elem](burn_backend::ops::IntTensorOps::int_equal_elem). + /// Bool => [equal elem](burn_backend::ops::BoolTensorOps::bool_equal_elem). + EqualElem(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [repeat dim](burn_backend::ops::FloatTensorOps::float_repeat_dim). + /// Int => [repeat dim](burn_backend::ops::IntTensorOps::int_repeat_dim). + /// Bool => [repeat dim](burn_backend::ops::BoolTensorOps::bool_repeat_dim). + RepeatDim(RepeatDimOpIr), + /// Operation corresponding to: + /// + /// Float => [cat](burn_backend::ops::FloatTensorOps::float_cat). + /// Int => [cat](burn_backend::ops::IntTensorOps::int_cat). + /// Bool => [cat](burn_backend::ops::BoolTensorOps::bool_cat). + Cat(CatOpIr), + /// Cast operation, no direct operation and should be supported by fusion backend. + Cast(CastOpIr), + /// Operation corresponding to: + /// + /// Float => [empty](burn_backend::ops::FloatTensorOps::float_empty). + /// Int => [empty](burn_backend::ops::IntTensorOps::int_empty). + /// Bool => [empty](burn_backend::ops::BoolTensorOps::bool_empty). + Empty(CreationOpIr), + /// Operation corresponding to: + /// + /// Float => [ones](burn_backend::ops::FloatTensorOps::float_ones). + /// Int => [ones](burn_backend::ops::IntTensorOps::int_ones). + /// Bool => [ones](burn_backend::ops::BoolTensorOps::bool_ones). + Ones(CreationOpIr), + /// Operation corresponding to: + /// + /// Float => [zeros](burn_backend::ops::FloatTensorOps::float_zeros). + /// Int => [zeros](burn_backend::ops::IntTensorOps::int_zeros). + /// Bool => [zeros](burn_backend::ops::BoolTensorOps::bool_zeros). + Zeros(CreationOpIr), +} + +/// Numeric operations on int and float tensors. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub enum NumericOperationIr { + /// Operation corresponding to: + /// + /// Float => [add](burn_backend::ops::FloatTensorOps::float_add). + /// Int => [add](burn_backend::ops::IntTensorOps::int_add). + Add(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [add scalar](burn_backend::ops::FloatTensorOps::float_add_scalar). + /// Int => [add scalar](burn_backend::ops::IntTensorOps::int_add_scalar). + AddScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [sub](burn_backend::ops::FloatTensorOps::float_sub). + /// Int => [sub](burn_backend::ops::IntTensorOps::int_sub). + Sub(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [sub scalar](burn_backend::ops::FloatTensorOps::float_sub_scalar). + /// Int => [sub scalar](burn_backend::ops::IntTensorOps::int_sub_scalar). + SubScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [div](burn_backend::ops::FloatTensorOps::float_div). + /// Int => [div](burn_backend::ops::IntTensorOps::int_div). + Div(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [div scalar](burn_backend::ops::FloatTensorOps::float_div_scalar). + /// Int => [div scalar](burn_backend::ops::IntTensorOps::int_div_scalar). + DivScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [rem](burn_backend::ops::FloatTensorOps::float_remainder). + /// Int => [rem](burn_backend::ops::IntTensorOps::int_remainder). + Rem(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [rem scalar](burn_backend::ops::FloatTensorOps::float_remainder_scalar). + /// Int => [rem scalar](burn_backend::ops::IntTensorOps::int_remainder_scalar). + RemScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [mul](burn_backend::ops::FloatTensorOps::float_mul). + /// Int => [mul](burn_backend::ops::IntTensorOps::int_mul). + Mul(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [mul scalar](burn_backend::ops::FloatTensorOps::float_mul_scalar). + /// Int => [mul scalar](burn_backend::ops::IntTensorOps::int_mul_scalar). + MulScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [abs](burn_backend::ops::FloatTensorOps::float_abs). + /// Int => [abs](burn_backend::ops::IntTensorOps::int_abs). + Abs(UnaryOpIr), + /// Operation corresponding to: + /// + /// Float => [full](burn_backend::ops::FloatTensorOps::float_full). + /// Int => [full](burn_backend::ops::IntTensorOps::int_full). + Full(FullOpIr), + /// Operation corresponding to: + /// + /// Float => [mean dim](burn_backend::ops::FloatTensorOps::float_mean_dim). + /// Int => [mean dim](burn_backend::ops::IntTensorOps::int_mean_dim). + MeanDim(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [mean](burn_backend::ops::FloatTensorOps::float_mean). + /// Int => [mean](burn_backend::ops::IntTensorOps::int_mean). + Mean(ReduceOpIr), + /// Operation corresponding to: + /// + /// Float => [sum](burn_backend::ops::FloatTensorOps::float_sum). + /// Int => [sum](burn_backend::ops::IntTensorOps::int_sum). + Sum(ReduceOpIr), + /// Operation corresponding to: + /// + /// Float => [sum dim](burn_backend::ops::FloatTensorOps::float_sum_dim). + /// Int => [sum dim](burn_backend::ops::IntTensorOps::int_sum_dim). + SumDim(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [prod](burn_backend::ops::FloatTensorOps::float_prod). + /// Int => [prod](burn_backend::ops::IntTensorOps::int_prod). + Prod(ReduceOpIr), + /// Operation corresponding to: + /// + /// Float => [prod dim](burn_backend::ops::FloatTensorOps::float_prod_dim). + /// Int => [prod dim](burn_backend::ops::IntTensorOps::int_prod_dim). + ProdDim(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [greater](burn_backend::ops::FloatTensorOps::float_greater). + /// Int => [greater](burn_backend::ops::IntTensorOps::int_greater). + Greater(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [greater elem](burn_backend::ops::FloatTensorOps::float_greater_elem). + /// Int => [greater elem](burn_backend::ops::IntTensorOps::int_greater_elem). + GreaterElem(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [greater equal](burn_backend::ops::FloatTensorOps::float_greater_elem). + /// Int => [greater elem](burn_backend::ops::IntTensorOps::int_greater_elem). + GreaterEqual(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [greater equal elem](burn_backend::ops::FloatTensorOps::float_greater_equal_elem). + /// Int => [greater equal elem](burn_backend::ops::IntTensorOps::int_greater_equal_elem). + GreaterEqualElem(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [lower](burn_backend::ops::FloatTensorOps::float_lower). + /// Int => [lower](burn_backend::ops::IntTensorOps::int_lower). + Lower(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [lower elem](burn_backend::ops::FloatTensorOps::float_lower_elem). + /// Int => [lower elem](burn_backend::ops::IntTensorOps::int_lower_elem). + LowerElem(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [lower equal](burn_backend::ops::FloatTensorOps::float_lower_equal). + /// Int => [lower equal](burn_backend::ops::IntTensorOps::int_lower_equal). + LowerEqual(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [lower equal elem](burn_backend::ops::FloatTensorOps::float_lower_equal_elem). + /// Int => [lower equal elem](burn_backend::ops::IntTensorOps::int_lower_equal_elem). + LowerEqualElem(ScalarOpIr), + /// Operation corresponding to: + /// + /// Float => [argmax](burn_backend::ops::FloatTensorOps::float_argmax). + /// Int => [argmax](burn_backend::ops::IntTensorOps::int_argmax). + ArgMax(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [argmin](burn_backend::ops::FloatTensorOps::float_argmin). + /// Int => [argmin](burn_backend::ops::IntTensorOps::int_argmin). + ArgMin(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [max](burn_backend::ops::FloatTensorOps::float_max). + /// Int => [max](burn_backend::ops::IntTensorOps::int_max). + Max(ReduceOpIr), + /// Operation corresponding to: + /// + /// Float => [max dim with indices](burn_backend::ops::FloatTensorOps::float_max_dim_with_indices). + /// Int => [max dim with indices](burn_backend::ops::IntTensorOps::int_max_dim_with_indices). + MaxDimWithIndices(ReduceDimWithIndicesOpIr), + /// Operation corresponding to: + /// + /// Float => [min dim with indices](burn_backend::ops::FloatTensorOps::float_min_dim_with_indices). + /// Int => [min dim with indices](burn_backend::ops::IntTensorOps::int_min_dim_with_indices). + MinDimWithIndices(ReduceDimWithIndicesOpIr), + /// Operation corresponding to: + /// + /// Float => [min](burn_backend::ops::FloatTensorOps::float_min). + /// Int => [min](burn_backend::ops::IntTensorOps::int_min). + Min(ReduceOpIr), + /// Operation corresponding to: + /// + /// Float => [max dim](burn_backend::ops::FloatTensorOps::float_max_dim). + /// Int => [max dim](burn_backend::ops::IntTensorOps::int_max_dim). + MaxDim(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [min dim](burn_backend::ops::FloatTensorOps::float_min_dim). + /// Int => [min dim](burn_backend::ops::IntTensorOps::int_min_dim). + MinDim(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [max_abs](burn_backend::ops::FloatTensorOps::float_max_abs). + /// Int => [max_abs](burn_backend::ops::IntTensorOps::int_max_abs). + MaxAbs(ReduceOpIr), + /// Operation corresponding to: + /// + /// Float => [max_abs dim](burn_backend::ops::FloatTensorOps::float_max_abs_dim). + /// Int => [max_abs dim](burn_backend::ops::IntTensorOps::int_max_abs_dim). + MaxAbsDim(ReduceDimOpIr), + /// Operation corresponding to: + /// + /// Float => [clamp](burn_backend::ops::FloatTensorOps::float_clamp). + /// Int => [clamp](burn_backend::ops::IntTensorOps::int_clamp). + Clamp(ClampOpIr), + /// Operation corresponding to: + /// + /// Int => [random](burn_backend::ops::IntTensorOps::int_random). + IntRandom(RandomOpIr), + /// Operation corresponding to: + /// + /// Float => [powf](burn_backend::ops::FloatTensorOps::float_powf). + /// Int => [powf](burn_backend::ops::IntTensorOps::int_powf). + Powf(BinaryOpIr), + /// Operation corresponding to: + /// + /// Float => [cumsum](burn_backend::ops::FloatTensorOps::float_cumsum). + /// Int => [cumsum](burn_backend::ops::IntTensorOps::int_cumsum). + CumSum(DimOpIr), + /// Operation corresponding to: + /// + /// Float => [cumprod](burn_backend::ops::FloatTensorOps::float_cumprod). + /// Int => [cumprod](burn_backend::ops::IntTensorOps::int_cumprod). + CumProd(DimOpIr), + /// Operation corresponding to: + /// + /// Float => [cummin](burn_backend::ops::FloatTensorOps::float_cummin). + /// Int => [cummin](burn_backend::ops::IntTensorOps::int_cummin). + CumMin(DimOpIr), + /// Operation corresponding to: + /// + /// Float => [cummax](burn_backend::ops::FloatTensorOps::float_cummax). + /// Int => [cummax](burn_backend::ops::IntTensorOps::int_cummax). + CumMax(DimOpIr), +} + +/// Operation intermediate representation specific to an int tensor. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub enum IntOperationIr { + /// Operation corresponding to [into float](burn_backend::ops::IntTensorOps::int_into_float). + IntoFloat(CastOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise and](burn_backend::ops::IntTensorOps::bitwise_and). + BitwiseAnd(BinaryOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise and scalar](burn_backend::ops::IntTensorOps::bitwise_and_scalar). + BitwiseAndScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise or](burn_backend::ops::IntTensorOps::bitwise_or). + BitwiseOr(BinaryOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise or scalar](burn_backend::ops::IntTensorOps::bitwise_or_scalar). + BitwiseOrScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise xor](burn_backend::ops::IntTensorOps::bitwise_xor). + BitwiseXor(BinaryOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise xor scalar](burn_backend::ops::IntTensorOps::bitwise_xor_scalar). + BitwiseXorScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise not](burn_backend::ops::IntTensorOps::bitwise_not). + BitwiseNot(UnaryOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise left shift](burn_backend::ops::IntTensorOps::bitwise_left_shift). + BitwiseLeftShift(BinaryOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise left shift scalar](burn_backend::ops::IntTensorOps::bitwise_left_shift_scalar). + BitwiseLeftShiftScalar(ScalarOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise right shift](burn_backend::ops::IntTensorOps::bitwise_right_shift). + BitwiseRightShift(BinaryOpIr), + /// Operation corresponding to: + /// + /// Int => [bitwise right shift scalar](burn_backend::ops::IntTensorOps::bitwise_right_shift_scalar). + BitwiseRightShiftScalar(ScalarOpIr), + /// Operation corresponding to [matmul](burn_backend::ops::IntTensorOps::int_matmul). + Matmul(MatmulOpIr), +} + +/// Operation intermediate representation specific to a bool tensor. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub enum BoolOperationIr { + /// Operation corresponding to [into float](burn_backend::ops::BoolTensorOps::bool_into_float). + IntoFloat(CastOpIr), + /// Operation corresponding to [into int](burn_backend::ops::BoolTensorOps::bool_into_int). + IntoInt(CastOpIr), + /// Operation corresponding to [not](burn_backend::ops::BoolTensorOps::bool_not). + Not(UnaryOpIr), + /// Operation corresponding to [and](burn_backend::ops::BoolTensorOps::bool_and). + And(BinaryOpIr), + /// Operation corresponding to [or](burn_backend::ops::BoolTensorOps::bool_or). + Or(BinaryOpIr), +} + +/// Swap dim operation intermediate representation. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct SwapDimsOpIr { + /// Input tensor intermediate representation. + pub input: TensorIr, + /// Output tensor intermediate representation. + pub out: TensorIr, + /// The first dim to swap. + pub dim1: usize, + /// The second dim to swap. + pub dim2: usize, +} + +/// Permute operation intermediate representation. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct PermuteOpIr { + /// Input tensor intermediate representation. + pub input: TensorIr, + /// Output tensor intermediate representation. + pub out: TensorIr, + /// The new order of the dimensions. + pub axes: Vec, +} + +/// Shape operation intermediate representation. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct ShapeOpIr { + /// Input tensor intermediate representation. + pub input: TensorIr, + /// Output tensor intermediate representation with the new shape. + pub out: TensorIr, +} + +/// Unfold operation intermediate representation. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct UnfoldOpIr { + /// Input tensor intermediate representation. + pub input: TensorIr, + /// Output tensor intermediate representation. + pub out: TensorIr, + + /// The selected dim. + pub dim: usize, + /// The window size. + pub size: usize, + /// The window step along dim. + pub step: usize, +} + +/// Flip operation intermediate representation. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct FlipOpIr { + /// Input tensor intermediate representation. + pub input: TensorIr, + /// Output tensor intermediate representation. + pub out: TensorIr, + /// The dimensions to flip. + pub axes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct RandomOpIr { + pub out: TensorIr, + pub distribution: Distribution, +} + +/// Creation operation intermediate representation. +/// As opposed to [InitOperationIr], creation operations are lazy initialized. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct CreationOpIr { + /// Output tensor intermediate representation. + pub out: TensorIr, +} + +/// Full operation intermediate representation. +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct FullOpIr { + /// Output tensor intermediate representation. + pub out: TensorIr, + /// Fill value. + pub value: ScalarIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +/// Declares a tensor has been initialized. +/// +/// It is necessary to register for proper orphan detection and avoid memory leak. +pub struct InitOperationIr { + /// The initialized tensor. + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct BinaryOpIr { + pub lhs: TensorIr, + pub rhs: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MatmulOpIr { + pub lhs: TensorIr, + pub rhs: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct CrossOpIr { + pub lhs: TensorIr, + pub rhs: TensorIr, + pub out: TensorIr, + pub dim: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct UnaryOpIr { + pub input: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ScalarOpIr { + pub lhs: TensorIr, + // TODO: Make that an enum with `Value` and `Id` variants for relative/global + // conversion. + pub rhs: ScalarIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Hash)] +#[allow(missing_docs)] +pub struct ReduceOpIr { + pub input: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Hash)] +#[allow(missing_docs)] +pub struct ReduceDimOpIr { + pub input: TensorIr, + pub out: TensorIr, + pub axis: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct CastOpIr { + pub input: TensorIr, + pub out: TensorIr, +} + +/// IR for operations that operate along a dimension without reducing it. +/// Unlike `ReduceDimOpIr`, the output shape is the same as the input shape. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Hash)] +#[allow(missing_docs)] +pub struct DimOpIr { + pub input: TensorIr, + pub out: TensorIr, + pub axis: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct GatherOpIr { + pub tensor: TensorIr, + pub dim: usize, + pub indices: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ScatterOpIr { + pub tensor: TensorIr, + pub dim: usize, + pub indices: TensorIr, + pub value: TensorIr, + pub update: IndexingUpdateOp, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct SelectOpIr { + pub tensor: TensorIr, + pub dim: usize, + pub indices: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct SelectAssignOpIr { + pub tensor: TensorIr, + pub dim: usize, + pub indices: TensorIr, + pub value: TensorIr, + pub update: IndexingUpdateOp, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct SliceOpIr { + pub tensor: TensorIr, + pub ranges: Vec, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct SliceAssignOpIr { + pub tensor: TensorIr, + pub ranges: Vec, + pub value: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaskWhereOpIr { + pub tensor: TensorIr, + pub mask: TensorIr, + pub value: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaskFillOpIr { + pub tensor: TensorIr, + pub mask: TensorIr, + pub value: ScalarIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ClampOpIr { + pub tensor: TensorIr, + pub min: ScalarIr, + pub max: ScalarIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct RepeatDimOpIr { + pub tensor: TensorIr, + pub dim: usize, + pub times: usize, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct CatOpIr { + pub tensors: Vec, + pub dim: usize, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ReduceDimWithIndicesOpIr { + pub tensor: TensorIr, + pub dim: usize, + pub out: TensorIr, + pub out_indices: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct EmbeddingOpIr { + pub weights: TensorIr, + pub indices: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct EmbeddingBackwardOpIr { + pub weights: TensorIr, + pub out_grad: TensorIr, + pub indices: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv1dOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub bias: Option, + pub options: Conv1dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv1dXBackwardOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub output_grad: TensorIr, + pub options: Conv1dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv1dWeightBackwardOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub output_grad: TensorIr, + pub options: Conv1dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv1dBiasBackwardOpIr { + pub x: TensorIr, + pub bias: TensorIr, + pub output_grad: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv2dOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub bias: Option, + pub options: Conv2dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv2dXBackwardOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub output_grad: TensorIr, + pub options: Conv2dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv2dWeightBackwardOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub output_grad: TensorIr, + pub options: Conv2dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv2dBiasBackwardOpIr { + pub x: TensorIr, + pub bias: TensorIr, + pub output_grad: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct DeformConv2dOpIr { + pub x: TensorIr, + pub offset: TensorIr, + pub weight: TensorIr, + pub mask: Option, + pub bias: Option, + pub options: DeformableConv2dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct DeformConv2dBackwardOpIr { + pub x: TensorIr, + pub offset: TensorIr, + pub weight: TensorIr, + pub mask: Option, + pub bias: Option, + pub out_grad: TensorIr, + pub options: DeformableConv2dOptionsIr, + pub input_grad: TensorIr, + pub offset_grad: TensorIr, + pub weight_grad: TensorIr, + pub mask_grad: Option, + pub bias_grad: Option, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv3dOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub bias: Option, + pub options: Conv3dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv3dXBackwardOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub output_grad: TensorIr, + pub options: Conv3dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv3dWeightBackwardOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub output_grad: TensorIr, + pub options: Conv3dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv3dBiasBackwardOpIr { + pub x: TensorIr, + pub bias: TensorIr, + pub output_grad: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ConvTranspose1dOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub bias: Option, + pub options: ConvTranspose1dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ConvTranspose2dOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub bias: Option, + pub options: ConvTranspose2dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ConvTranspose3dOpIr { + pub x: TensorIr, + pub weight: TensorIr, + pub bias: Option, + pub options: ConvTranspose3dOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv1dOptionsIr { + pub stride: [usize; 1], + pub padding: [usize; 1], + pub dilation: [usize; 1], + pub groups: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv2dOptionsIr { + pub stride: [usize; 2], + pub padding: [usize; 2], + pub dilation: [usize; 2], + pub groups: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct DeformableConv2dOptionsIr { + pub stride: [usize; 2], + pub padding: [usize; 2], + pub dilation: [usize; 2], + pub weight_groups: usize, + pub offset_groups: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct Conv3dOptionsIr { + pub stride: [usize; 3], + pub padding: [usize; 3], + pub dilation: [usize; 3], + pub groups: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ConvTranspose1dOptionsIr { + pub stride: [usize; 1], + pub padding: [usize; 1], + pub padding_out: [usize; 1], + pub dilation: [usize; 1], + pub groups: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ConvTranspose2dOptionsIr { + pub stride: [usize; 2], + pub padding: [usize; 2], + pub padding_out: [usize; 2], + pub dilation: [usize; 2], + pub groups: usize, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ConvTranspose3dOptionsIr { + pub stride: [usize; 3], + pub padding: [usize; 3], + pub padding_out: [usize; 3], + pub dilation: [usize; 3], + pub groups: usize, +} + +/// Quantization parameters intermediate representation. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct QuantizationParametersIr { + /// The scaling factor. + pub scales: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct QuantizeOpIr { + pub tensor: TensorIr, + pub qparams: QuantizationParametersIr, + pub scheme: QuantScheme, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct DequantizeOpIr { + pub input: TensorIr, + pub out: TensorIr, +} + +impl From> for Conv1dOptionsIr { + fn from(value: ConvOptions<1>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + dilation: value.dilation, + groups: value.groups, + } + } +} + +impl From> for Conv2dOptionsIr { + fn from(value: ConvOptions<2>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + dilation: value.dilation, + groups: value.groups, + } + } +} + +impl From> for Conv3dOptionsIr { + fn from(value: ConvOptions<3>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + dilation: value.dilation, + groups: value.groups, + } + } +} + +impl From> for DeformableConv2dOptionsIr { + fn from(value: DeformConvOptions<2>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + dilation: value.dilation, + weight_groups: value.weight_groups, + offset_groups: value.offset_groups, + } + } +} + +impl From> for ConvTranspose1dOptionsIr { + fn from(value: ConvTransposeOptions<1>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + padding_out: value.padding_out, + dilation: value.dilation, + groups: value.groups, + } + } +} + +impl From> for ConvTranspose2dOptionsIr { + fn from(value: ConvTransposeOptions<2>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + padding_out: value.padding_out, + dilation: value.dilation, + groups: value.groups, + } + } +} + +impl From> for ConvTranspose3dOptionsIr { + fn from(value: ConvTransposeOptions<3>) -> Self { + Self { + stride: value.stride, + padding: value.padding, + padding_out: value.padding_out, + dilation: value.dilation, + groups: value.groups, + } + } +} + +impl From for ConvOptions<1> { + fn from(val: Conv1dOptionsIr) -> Self { + ConvOptions { + stride: val.stride, + padding: val.padding, + dilation: val.dilation, + groups: val.groups, + } + } +} + +impl From for ConvOptions<2> { + fn from(val: Conv2dOptionsIr) -> Self { + ConvOptions { + stride: val.stride, + padding: val.padding, + dilation: val.dilation, + groups: val.groups, + } + } +} + +impl From for ConvOptions<3> { + fn from(val: Conv3dOptionsIr) -> Self { + ConvOptions { + stride: val.stride, + padding: val.padding, + dilation: val.dilation, + groups: val.groups, + } + } +} + +impl From for DeformConvOptions<2> { + fn from(value: DeformableConv2dOptionsIr) -> Self { + DeformConvOptions { + stride: value.stride, + padding: value.padding, + dilation: value.dilation, + weight_groups: value.weight_groups, + offset_groups: value.offset_groups, + } + } +} + +impl From for ConvTransposeOptions<1> { + fn from(val: ConvTranspose1dOptionsIr) -> Self { + ConvTransposeOptions { + stride: val.stride, + padding: val.padding, + padding_out: val.padding_out, + dilation: val.dilation, + groups: val.groups, + } + } +} + +impl From for ConvTransposeOptions<2> { + fn from(val: ConvTranspose2dOptionsIr) -> Self { + ConvTransposeOptions { + stride: val.stride, + padding: val.padding, + padding_out: val.padding_out, + dilation: val.dilation, + groups: val.groups, + } + } +} + +impl From for ConvTransposeOptions<3> { + fn from(val: ConvTranspose3dOptionsIr) -> Self { + ConvTransposeOptions { + stride: val.stride, + padding: val.padding, + padding_out: val.padding_out, + dilation: val.dilation, + groups: val.groups, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AvgPool1dOpIr { + pub x: TensorIr, + pub kernel_size: usize, + pub stride: usize, + pub padding: usize, + pub count_include_pad: bool, + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AvgPool2dOpIr { + pub x: TensorIr, + pub kernel_size: [usize; 2], + pub stride: [usize; 2], + pub padding: [usize; 2], + pub count_include_pad: bool, + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AvgPool1dBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub kernel_size: usize, + pub stride: usize, + pub padding: usize, + pub count_include_pad: bool, + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AvgPool2dBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub kernel_size: [usize; 2], + pub stride: [usize; 2], + pub padding: [usize; 2], + pub count_include_pad: bool, + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AdaptiveAvgPool1dOpIr { + pub x: TensorIr, + pub output_size: usize, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AdaptiveAvgPool2dOpIr { + pub x: TensorIr, + pub output_size: [usize; 2], + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AdaptiveAvgPool1dBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AdaptiveAvgPool2dBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaxPool1dOpIr { + pub x: TensorIr, + pub kernel_size: usize, + pub stride: usize, + pub padding: usize, + pub dilation: usize, + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaxPool1dWithIndicesOpIr { + pub x: TensorIr, + pub kernel_size: usize, + pub stride: usize, + pub padding: usize, + pub dilation: usize, + pub ceil_mode: bool, + pub out: TensorIr, + pub out_indices: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaxPool1dWithIndicesBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub indices: TensorIr, + pub kernel_size: usize, + pub stride: usize, + pub padding: usize, + pub dilation: usize, + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaxPool2dOpIr { + pub x: TensorIr, + pub kernel_size: [usize; 2], + pub stride: [usize; 2], + pub padding: [usize; 2], + pub dilation: [usize; 2], + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[allow(missing_docs)] +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +pub struct MaxPool2dWithIndicesOpIr { + pub x: TensorIr, + pub kernel_size: [usize; 2], + pub stride: [usize; 2], + pub padding: [usize; 2], + pub dilation: [usize; 2], + pub ceil_mode: bool, + pub out: TensorIr, + pub out_indices: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct MaxPool2dWithIndicesBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub indices: TensorIr, + pub kernel_size: [usize; 2], + pub stride: [usize; 2], + pub padding: [usize; 2], + pub dilation: [usize; 2], + pub ceil_mode: bool, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub enum InterpolateModeIr { + Nearest, + Bilinear, + Bicubic, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct InterpolateOptionsIr { + pub mode: InterpolateModeIr, + pub align_corners: bool, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct InterpolateOpIr { + pub x: TensorIr, + pub output_size: [usize; 2], + pub options: InterpolateOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AttentionOptionsIr { + pub scale: Option, + pub softcap: Option, + pub is_causal: bool, +} + +impl From for AttentionModuleOptions { + fn from(ir: AttentionOptionsIr) -> Self { + AttentionModuleOptions { + scale: ir.scale.map(|s| s.elem()), + softcap: ir.softcap.map(|s| s.elem()), + is_causal: ir.is_causal, + } + } +} + +impl From for AttentionOptionsIr { + fn from(ir: AttentionModuleOptions) -> Self { + AttentionOptionsIr { + scale: ir.scale.map(ScalarIr::Float), + softcap: ir.softcap.map(ScalarIr::Float), + is_causal: ir.is_causal, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct AttentionOpIr { + pub query: TensorIr, + pub key: TensorIr, + pub value: TensorIr, + pub mask: Option, + pub attn_bias: Option, + pub options: AttentionOptionsIr, + pub out: TensorIr, +} + +impl From for InterpolateMode { + fn from(val: InterpolateModeIr) -> Self { + match val { + InterpolateModeIr::Nearest => Self::Nearest, + InterpolateModeIr::Bilinear => Self::Bilinear, + InterpolateModeIr::Bicubic => Self::Bicubic, + } + } +} + +impl From for InterpolateOptions { + fn from(val: InterpolateOptionsIr) -> Self { + Self::new(val.mode.into()).with_align_corners(val.align_corners) + } +} + +impl From for InterpolateModeIr { + fn from(val: InterpolateMode) -> Self { + match val { + InterpolateMode::Nearest => Self::Nearest, + InterpolateMode::Bilinear => Self::Bilinear, + InterpolateMode::Bicubic => Self::Bicubic, + } + } +} + +impl From for InterpolateOptionsIr { + fn from(val: InterpolateOptions) -> Self { + Self { + mode: val.mode.into(), + align_corners: val.align_corners, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct InterpolateBackwardOpIr { + pub x: TensorIr, + pub grad: TensorIr, + pub output_size: [usize; 2], + pub options: InterpolateOptionsIr, + pub out: TensorIr, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub enum GridSamplePaddingModeIr { + Zeros, + Border, + Reflection, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct GridSampleOptionsIr { + pub mode: InterpolateModeIr, + pub padding_mode: GridSamplePaddingModeIr, + pub align_corners: bool, +} + +#[derive(Clone, Debug, Hash, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct GridSample2dOpIr { + pub tensor: TensorIr, + pub grid: TensorIr, + pub options: GridSampleOptionsIr, + pub out: TensorIr, +} + +impl From for GridSamplePaddingMode { + fn from(val: GridSamplePaddingModeIr) -> Self { + match val { + GridSamplePaddingModeIr::Zeros => Self::Zeros, + GridSamplePaddingModeIr::Border => Self::Border, + GridSamplePaddingModeIr::Reflection => Self::Reflection, + } + } +} + +impl From for GridSamplePaddingModeIr { + fn from(val: GridSamplePaddingMode) -> Self { + match val { + GridSamplePaddingMode::Zeros => Self::Zeros, + GridSamplePaddingMode::Border => Self::Border, + GridSamplePaddingMode::Reflection => Self::Reflection, + } + } +} + +impl From for GridSampleOptions { + fn from(val: GridSampleOptionsIr) -> Self { + Self { + mode: val.mode.into(), + padding_mode: val.padding_mode.into(), + align_corners: val.align_corners, + } + } +} + +impl From for GridSampleOptionsIr { + fn from(val: GridSampleOptions) -> Self { + Self { + mode: val.mode.into(), + padding_mode: val.padding_mode.into(), + align_corners: val.align_corners, + } + } +} + +impl OperationIr { + /// Get all input [tensors](TensorIr) involved with the current operation. + pub fn inputs(&self) -> impl Iterator { + match self { + OperationIr::BaseFloat(repr) => repr.inputs(), + OperationIr::BaseInt(repr) => repr.inputs(), + OperationIr::BaseBool(repr) => repr.inputs(), + OperationIr::NumericFloat(_dtype, repr) => repr.inputs(), + OperationIr::NumericInt(_dtype, repr) => repr.inputs(), + OperationIr::Bool(repr) => repr.inputs(), + OperationIr::Int(repr) => repr.inputs(), + OperationIr::Float(_dtype, repr) => repr.inputs(), + OperationIr::Module(repr) => repr.inputs(), + OperationIr::Init(repr) => repr.inputs(), + OperationIr::Custom(repr) => repr.inputs(), + OperationIr::Drop(repr) => Box::new([repr].into_iter()), + } + } + + /// Get all output [tensors](TensorIr) involved with the current operation. + pub fn outputs(&self) -> impl Iterator { + match self { + OperationIr::BaseFloat(repr) => repr.outputs(), + OperationIr::BaseInt(repr) => repr.outputs(), + OperationIr::BaseBool(repr) => repr.outputs(), + OperationIr::NumericFloat(_dtype, repr) => repr.outputs(), + OperationIr::NumericInt(_dtype, repr) => repr.outputs(), + OperationIr::Bool(repr) => repr.outputs(), + OperationIr::Int(repr) => repr.outputs(), + OperationIr::Float(_dtype, repr) => repr.outputs(), + OperationIr::Module(repr) => repr.outputs(), + OperationIr::Init(repr) => repr.outputs(), + OperationIr::Custom(repr) => repr.outputs(), + OperationIr::Drop(_repr) => Box::new([].into_iter()), + } + } + + /// Get all [tensor](TensorIr) involved with the current operation. + pub fn nodes(&self) -> Vec<&TensorIr> { + self.inputs().chain(self.outputs()).collect() + } + + /// Set the given nodes that are [read write](super::TensorStatus::ReadWrite) to + /// [read only](super::TensorStatus::ReadOnly) in the current operation. + /// + /// Returns the tensor that were updated with their original representation. + pub fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + match self { + OperationIr::BaseFloat(repr) => repr.mark_read_only(nodes), + OperationIr::BaseInt(repr) => repr.mark_read_only(nodes), + OperationIr::BaseBool(repr) => repr.mark_read_only(nodes), + OperationIr::NumericFloat(_dtype, repr) => repr.mark_read_only(nodes), + OperationIr::NumericInt(_dtype, repr) => repr.mark_read_only(nodes), + OperationIr::Bool(repr) => repr.mark_read_only(nodes), + OperationIr::Int(repr) => repr.mark_read_only(nodes), + OperationIr::Float(_dtype, repr) => repr.mark_read_only(nodes), + OperationIr::Module(repr) => repr.mark_read_only(nodes), + OperationIr::Init(_) => Vec::new(), + OperationIr::Drop(repr) => { + let mut output = Vec::new(); + repr.mark_read_only(nodes, &mut output); + output + } + OperationIr::Custom(repr) => { + let mut output = Vec::new(); + + for input in repr.inputs.iter_mut() { + input.mark_read_only(nodes, &mut output); + } + + output + } + } + } +} + +impl BaseOperationIr { + fn inputs(&self) -> Box + '_> { + match self { + BaseOperationIr::Reshape(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::SwapDims(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::Permute(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::Expand(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::Flip(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::Slice(repr) => Box::new([&repr.tensor].into_iter()), + BaseOperationIr::SliceAssign(repr) => Box::new([&repr.tensor, &repr.value].into_iter()), + BaseOperationIr::Gather(repr) => Box::new([&repr.tensor, &repr.indices].into_iter()), + BaseOperationIr::Scatter(repr) => { + Box::new([&repr.tensor, &repr.indices, &repr.value].into_iter()) + } + BaseOperationIr::Select(repr) => Box::new([&repr.tensor, &repr.indices].into_iter()), + BaseOperationIr::SelectAssign(repr) => { + Box::new([&repr.tensor, &repr.indices, &repr.value].into_iter()) + } + BaseOperationIr::MaskWhere(repr) => { + Box::new([&repr.tensor, &repr.mask, &repr.value].into_iter()) + } + BaseOperationIr::MaskFill(repr) => Box::new([&repr.tensor, &repr.mask].into_iter()), + BaseOperationIr::Equal(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + BaseOperationIr::EqualElem(repr) => Box::new([&repr.lhs].into_iter()), + BaseOperationIr::RepeatDim(repr) => Box::new([&repr.tensor].into_iter()), + BaseOperationIr::Cat(repr) => Box::new(repr.tensors.iter()), + BaseOperationIr::Cast(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::Unfold(repr) => Box::new([&repr.input].into_iter()), + BaseOperationIr::Empty(_repr) => Box::new([].into_iter()), + BaseOperationIr::Ones(_repr) => Box::new([].into_iter()), + BaseOperationIr::Zeros(_repr) => Box::new([].into_iter()), + } + } + + fn outputs(&self) -> Box + '_> { + match self { + BaseOperationIr::Reshape(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::SwapDims(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Permute(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Expand(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Flip(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Slice(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::SliceAssign(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Gather(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Scatter(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Select(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::SelectAssign(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::MaskWhere(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::MaskFill(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Equal(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::EqualElem(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::RepeatDim(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Cat(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Cast(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Unfold(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Empty(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Ones(repr) => Box::new([&repr.out].into_iter()), + BaseOperationIr::Zeros(repr) => Box::new([&repr.out].into_iter()), + } + } + + fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + let mut output = Vec::new(); + + match self { + BaseOperationIr::Reshape(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BaseOperationIr::SwapDims(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Permute(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + + BaseOperationIr::Expand(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + + BaseOperationIr::Flip(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Slice(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + } + BaseOperationIr::SliceAssign(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.value.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Gather(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.indices.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Scatter(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.indices.mark_read_only(nodes, &mut output); + repr.value.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Select(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.indices.mark_read_only(nodes, &mut output); + } + BaseOperationIr::SelectAssign(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.indices.mark_read_only(nodes, &mut output); + repr.value.mark_read_only(nodes, &mut output); + } + BaseOperationIr::MaskWhere(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.mask.mark_read_only(nodes, &mut output); + repr.value.mark_read_only(nodes, &mut output); + } + BaseOperationIr::MaskFill(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.mask.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Equal(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + BaseOperationIr::EqualElem(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + BaseOperationIr::RepeatDim(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Cat(repr) => { + for t in repr.tensors.iter_mut() { + t.mark_read_only(nodes, &mut output); + } + } + BaseOperationIr::Cast(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Unfold(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BaseOperationIr::Empty(_) => {} + BaseOperationIr::Zeros(_) => {} + BaseOperationIr::Ones(_) => {} + }; + + output + } +} + +impl NumericOperationIr { + fn inputs(&self) -> Box + '_> { + match self { + NumericOperationIr::Add(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::AddScalar(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::Sub(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::SubScalar(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::Mul(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::MulScalar(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::Div(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::DivScalar(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::Rem(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::RemScalar(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::GreaterElem(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::GreaterEqualElem(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::LowerElem(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::LowerEqualElem(repr) => Box::new([&repr.lhs].into_iter()), + NumericOperationIr::Greater(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::GreaterEqual(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::Lower(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::LowerEqual(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::ArgMax(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::ArgMin(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::Clamp(repr) => Box::new([&repr.tensor].into_iter()), + NumericOperationIr::Abs(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::Full(_repr) => Box::new([].into_iter()), + NumericOperationIr::MeanDim(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::Mean(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::Sum(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::SumDim(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::Prod(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::ProdDim(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::Max(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::MaxDimWithIndices(repr) => Box::new([&repr.tensor].into_iter()), + NumericOperationIr::MinDimWithIndices(repr) => Box::new([&repr.tensor].into_iter()), + NumericOperationIr::Min(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::MaxDim(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::MinDim(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::MaxAbs(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::MaxAbsDim(repr) => Box::new([&repr.input].into_iter()), + NumericOperationIr::IntRandom(_repr) => Box::new([].into_iter()), + NumericOperationIr::Powf(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + NumericOperationIr::CumMin(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumMax(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumProd(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumSum(repr) => Box::new([&repr.out].into_iter()), + } + } + + fn outputs(&self) -> Box + '_> { + match self { + NumericOperationIr::Add(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::AddScalar(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Sub(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::SubScalar(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Mul(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MulScalar(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Div(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::DivScalar(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Rem(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::RemScalar(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::GreaterElem(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::GreaterEqualElem(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::LowerElem(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::LowerEqualElem(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Greater(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::GreaterEqual(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Lower(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::LowerEqual(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::ArgMax(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::ArgMin(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Clamp(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Abs(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Full(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MeanDim(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Mean(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Sum(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::SumDim(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Prod(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::ProdDim(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Max(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MaxDimWithIndices(repr) => { + Box::new([&repr.out, &repr.out_indices].into_iter()) + } + NumericOperationIr::MinDimWithIndices(repr) => { + Box::new([&repr.out, &repr.out_indices].into_iter()) + } + NumericOperationIr::Min(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MaxDim(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MinDim(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MaxAbs(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::MaxAbsDim(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::IntRandom(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::Powf(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumMin(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumMax(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumProd(repr) => Box::new([&repr.out].into_iter()), + NumericOperationIr::CumSum(repr) => Box::new([&repr.out].into_iter()), + } + } + fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + let mut output = Vec::new(); + + match self { + NumericOperationIr::Add(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::AddScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Sub(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::SubScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Mul(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MulScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Div(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::DivScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Rem(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::RemScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::GreaterElem(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::GreaterEqualElem(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::LowerElem(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::LowerEqualElem(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Greater(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::GreaterEqual(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Lower(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::LowerEqual(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::ArgMax(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::ArgMin(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Clamp(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Abs(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Full(_) => {} + NumericOperationIr::MeanDim(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Mean(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Sum(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::SumDim(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Prod(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::ProdDim(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Max(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MaxDimWithIndices(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MinDimWithIndices(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + } + NumericOperationIr::Min(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MaxDim(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MinDim(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MaxAbs(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::MaxAbsDim(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::IntRandom(_) => {} + NumericOperationIr::Powf(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + NumericOperationIr::CumSum(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::CumProd(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::CumMin(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + NumericOperationIr::CumMax(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + }; + + output + } +} + +impl FloatOperationIr { + fn inputs(&self) -> Box + '_> { + match self { + FloatOperationIr::Matmul(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + FloatOperationIr::Cross(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + FloatOperationIr::Random(_repr) => Box::new([].into_iter()), + FloatOperationIr::Exp(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Log(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Log1p(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Erf(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Recip(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::PowfScalar(repr) => Box::new([&repr.lhs].into_iter()), + FloatOperationIr::Sqrt(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Cos(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Sin(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Tanh(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Round(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Floor(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Ceil(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Trunc(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::IntoInt(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Quantize(repr) => { + Box::new([&repr.tensor, &repr.qparams.scales].into_iter()) + } + FloatOperationIr::Dequantize(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::IsNan(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::IsInf(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::GridSample2d(repr) => { + Box::new([&repr.tensor, &repr.grid].into_iter()) + } + FloatOperationIr::Tan(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Cosh(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::Sinh(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcCos(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcCosh(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcSin(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcSinh(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcTan(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcTanh(repr) => Box::new([&repr.input].into_iter()), + FloatOperationIr::ArcTan2(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + } + } + fn outputs(&self) -> Box + '_> { + match self { + FloatOperationIr::Matmul(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Cross(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Random(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Exp(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Log(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Log1p(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Erf(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Recip(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::PowfScalar(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Sqrt(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Cos(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Sin(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Tanh(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Round(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Floor(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Ceil(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Trunc(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::IntoInt(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Quantize(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Dequantize(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::IsNan(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::IsInf(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::GridSample2d(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Tan(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Cosh(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::Sinh(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcCos(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcCosh(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcSin(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcSinh(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcTan(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcTanh(repr) => Box::new([&repr.out].into_iter()), + FloatOperationIr::ArcTan2(repr) => Box::new([&repr.out].into_iter()), + } + } + + fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + let mut output = Vec::new(); + + match self { + FloatOperationIr::Matmul(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Cross(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Random(_) => {} + FloatOperationIr::Exp(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Log(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Log1p(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Erf(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Recip(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::PowfScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Sqrt(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Cos(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Sin(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Tanh(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Round(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Floor(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Ceil(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Trunc(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Quantize(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.qparams.scales.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Dequantize(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::IntoInt(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::IsNan(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::IsInf(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + FloatOperationIr::GridSample2d(repr) => { + repr.tensor.mark_read_only(nodes, &mut output); + repr.grid.mark_read_only(nodes, &mut output); + } + FloatOperationIr::Tan(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::Cosh(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::Sinh(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcCos(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcCosh(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcSin(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcSinh(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcTan(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcTanh(repr) => repr.input.mark_read_only(nodes, &mut output), + FloatOperationIr::ArcTan2(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + }; + + output + } +} + +impl IntOperationIr { + fn inputs(&self) -> Box + '_> { + match self { + IntOperationIr::Matmul(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + IntOperationIr::IntoFloat(repr) => Box::new([&repr.input].into_iter()), + IntOperationIr::BitwiseAnd(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + IntOperationIr::BitwiseAndScalar(repr) => Box::new([&repr.lhs].into_iter()), + IntOperationIr::BitwiseOr(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + IntOperationIr::BitwiseOrScalar(repr) => Box::new([&repr.lhs].into_iter()), + IntOperationIr::BitwiseXor(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + IntOperationIr::BitwiseXorScalar(repr) => Box::new([&repr.lhs].into_iter()), + IntOperationIr::BitwiseNot(repr) => Box::new([&repr.input].into_iter()), + IntOperationIr::BitwiseLeftShift(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + IntOperationIr::BitwiseLeftShiftScalar(repr) => Box::new([&repr.lhs].into_iter()), + IntOperationIr::BitwiseRightShift(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + IntOperationIr::BitwiseRightShiftScalar(repr) => Box::new([&repr.lhs].into_iter()), + } + } + + fn outputs(&self) -> Box + '_> { + match self { + IntOperationIr::Matmul(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::IntoFloat(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseAnd(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseAndScalar(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseOr(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseOrScalar(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseXor(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseXorScalar(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseNot(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseLeftShift(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseLeftShiftScalar(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseRightShift(repr) => Box::new([&repr.out].into_iter()), + IntOperationIr::BitwiseRightShiftScalar(repr) => Box::new([&repr.out].into_iter()), + } + } + + fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + let mut output = Vec::new(); + + match self { + IntOperationIr::Matmul(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::IntoFloat(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseAnd(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseAndScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseOr(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseOrScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseXor(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseXorScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseNot(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseLeftShift(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseLeftShiftScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseRightShift(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + IntOperationIr::BitwiseRightShiftScalar(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + } + }; + + output + } +} + +impl BoolOperationIr { + fn inputs(&self) -> Box + '_> { + match self { + BoolOperationIr::IntoFloat(repr) => Box::new([&repr.input].into_iter()), + BoolOperationIr::IntoInt(repr) => Box::new([&repr.input].into_iter()), + BoolOperationIr::Not(repr) => Box::new([&repr.input].into_iter()), + BoolOperationIr::And(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + BoolOperationIr::Or(repr) => Box::new([&repr.lhs, &repr.rhs].into_iter()), + } + } + fn outputs(&self) -> Box + '_> { + match self { + BoolOperationIr::IntoFloat(repr) => Box::new([&repr.out].into_iter()), + BoolOperationIr::IntoInt(repr) => Box::new([&repr.out].into_iter()), + BoolOperationIr::Not(repr) => Box::new([&repr.out].into_iter()), + BoolOperationIr::And(repr) => Box::new([&repr.out].into_iter()), + BoolOperationIr::Or(repr) => Box::new([&repr.out].into_iter()), + } + } + fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + let mut output = Vec::new(); + + match self { + BoolOperationIr::IntoFloat(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BoolOperationIr::IntoInt(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BoolOperationIr::Not(repr) => { + repr.input.mark_read_only(nodes, &mut output); + } + BoolOperationIr::And(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + BoolOperationIr::Or(repr) => { + repr.lhs.mark_read_only(nodes, &mut output); + repr.rhs.mark_read_only(nodes, &mut output); + } + }; + + output + } +} + +impl ModuleOperationIr { + fn inputs(&self) -> Box + '_> { + match self { + ModuleOperationIr::Embedding(repr) => { + Box::new([&repr.weights, &repr.indices].into_iter()) + } + ModuleOperationIr::EmbeddingBackward(repr) => { + Box::new([&repr.weights, &repr.out_grad, &repr.indices].into_iter()) + } + ModuleOperationIr::Conv1d(repr) => { + if let Some(bias) = &repr.bias { + Box::new([&repr.x, &repr.weight, bias].into_iter()) + } else { + Box::new([&repr.x, &repr.weight].into_iter()) + } + } + ModuleOperationIr::Conv1dXBackward(repr) => { + Box::new([&repr.x, &repr.weight, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv1dWeightBackward(repr) => { + Box::new([&repr.x, &repr.weight, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv1dBiasBackward(repr) => { + Box::new([&repr.x, &repr.bias, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv2d(repr) => { + if let Some(bias) = &repr.bias { + Box::new([&repr.x, &repr.weight, bias].into_iter()) + } else { + Box::new([&repr.x, &repr.weight].into_iter()) + } + } + ModuleOperationIr::Conv2dXBackward(repr) => { + Box::new([&repr.x, &repr.weight, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv2dWeightBackward(repr) => { + Box::new([&repr.x, &repr.weight, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv2dBiasBackward(repr) => { + Box::new([&repr.x, &repr.bias, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv3d(repr) => { + if let Some(bias) = &repr.bias { + Box::new([&repr.x, &repr.weight, bias].into_iter()) + } else { + Box::new([&repr.x, &repr.weight].into_iter()) + } + } + ModuleOperationIr::Conv3dXBackward(repr) => { + Box::new([&repr.x, &repr.weight, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv3dWeightBackward(repr) => { + Box::new([&repr.x, &repr.weight, &repr.output_grad].into_iter()) + } + ModuleOperationIr::Conv3dBiasBackward(repr) => { + Box::new([&repr.x, &repr.bias, &repr.output_grad].into_iter()) + } + ModuleOperationIr::DeformableConv2d(repr) => match (&repr.mask, &repr.bias) { + (Some(mask), Some(bias)) => { + Box::new([&repr.x, &repr.offset, &repr.weight, mask, bias].into_iter()) + } + (Some(mask), None) => { + Box::new([&repr.x, &repr.offset, &repr.weight, mask].into_iter()) + } + (None, Some(bias)) => { + Box::new([&repr.x, &repr.offset, &repr.weight, bias].into_iter()) + } + (None, None) => Box::new([&repr.x, &repr.offset, &repr.weight].into_iter()), + }, + ModuleOperationIr::DeformableConv2dBackward(repr) => match (&repr.mask, &repr.bias) { + (Some(mask), Some(bias)) => Box::new( + [ + &repr.x, + &repr.offset, + &repr.weight, + &repr.out_grad, + mask, + bias, + ] + .into_iter(), + ), + (Some(mask), None) => Box::new( + [&repr.x, &repr.offset, &repr.weight, &repr.out_grad, mask].into_iter(), + ), + (None, Some(bias)) => Box::new( + [&repr.x, &repr.offset, &repr.weight, &repr.out_grad, bias].into_iter(), + ), + (None, None) => { + Box::new([&repr.x, &repr.offset, &repr.weight, &repr.out_grad].into_iter()) + } + }, + ModuleOperationIr::ConvTranspose1d(repr) => { + if let Some(bias) = &repr.bias { + Box::new([&repr.x, &repr.weight, bias].into_iter()) + } else { + Box::new([&repr.x, &repr.weight].into_iter()) + } + } + ModuleOperationIr::ConvTranspose2d(repr) => { + if let Some(bias) = &repr.bias { + Box::new([&repr.x, &repr.weight, bias].into_iter()) + } else { + Box::new([&repr.x, &repr.weight].into_iter()) + } + } + ModuleOperationIr::ConvTranspose3d(repr) => { + if let Some(bias) = &repr.bias { + Box::new([&repr.x, &repr.weight, bias].into_iter()) + } else { + Box::new([&repr.x, &repr.weight].into_iter()) + } + } + ModuleOperationIr::AvgPool1d(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::AvgPool2d(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::AvgPool1dBackward(repr) => { + Box::new([&repr.x, &repr.grad].into_iter()) + } + ModuleOperationIr::AvgPool2dBackward(repr) => { + Box::new([&repr.x, &repr.grad].into_iter()) + } + ModuleOperationIr::AdaptiveAvgPool1d(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::AdaptiveAvgPool2d(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::AdaptiveAvgPool1dBackward(repr) => { + Box::new([&repr.x, &repr.grad].into_iter()) + } + ModuleOperationIr::AdaptiveAvgPool2dBackward(repr) => { + Box::new([&repr.x, &repr.grad].into_iter()) + } + ModuleOperationIr::MaxPool1d(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::MaxPool1dWithIndices(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::MaxPool1dWithIndicesBackward(repr) => { + Box::new([&repr.x, &repr.indices, &repr.grad].into_iter()) + } + ModuleOperationIr::MaxPool2d(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::MaxPool2dWithIndices(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::MaxPool2dWithIndicesBackward(repr) => { + Box::new([&repr.x, &repr.indices, &repr.grad].into_iter()) + } + ModuleOperationIr::Interpolate(repr) => Box::new([&repr.x].into_iter()), + ModuleOperationIr::InterpolateBackward(repr) => { + Box::new([&repr.x, &repr.grad].into_iter()) + } + ModuleOperationIr::Attention(repr) => { + if let Some(mask) = &repr.mask { + if let Some(attn_bias) = &repr.attn_bias { + Box::new([&repr.query, &repr.key, &repr.value, mask, attn_bias].into_iter()) + } else { + Box::new([&repr.query, &repr.key, &repr.value, mask].into_iter()) + } + } else if let Some(attn_bias) = &repr.attn_bias { + Box::new([&repr.query, &repr.key, &repr.value, attn_bias].into_iter()) + } else { + Box::new([&repr.query, &repr.key, &repr.value].into_iter()) + } + } + } + } + fn outputs(&self) -> Box + '_> { + match self { + ModuleOperationIr::Embedding(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::EmbeddingBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv1d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv1dXBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv1dWeightBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv1dBiasBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv2d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv2dXBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv2dWeightBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv2dBiasBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv3d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv3dXBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv3dWeightBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Conv3dBiasBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::DeformableConv2d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::DeformableConv2dBackward(repr) => { + match (&repr.mask_grad, &repr.bias_grad) { + (Some(mask_grad), Some(bias_grad)) => Box::new( + [ + &repr.input_grad, + &repr.offset_grad, + &repr.weight_grad, + mask_grad, + bias_grad, + ] + .into_iter(), + ), + (Some(mask_grad), None) => Box::new( + [ + &repr.input_grad, + &repr.offset_grad, + &repr.weight_grad, + mask_grad, + ] + .into_iter(), + ), + (None, Some(bias_grad)) => Box::new( + [ + &repr.input_grad, + &repr.offset_grad, + &repr.weight_grad, + bias_grad, + ] + .into_iter(), + ), + (None, None) => Box::new( + [&repr.input_grad, &repr.offset_grad, &repr.weight_grad].into_iter(), + ), + } + } + ModuleOperationIr::ConvTranspose1d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::ConvTranspose2d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::ConvTranspose3d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AvgPool1d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AvgPool2d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AvgPool1dBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AvgPool2dBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AdaptiveAvgPool1d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AdaptiveAvgPool2d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AdaptiveAvgPool1dBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::AdaptiveAvgPool2dBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::MaxPool1d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::MaxPool1dWithIndices(repr) => { + Box::new([&repr.out, &repr.out_indices].into_iter()) + } + ModuleOperationIr::MaxPool1dWithIndicesBackward(repr) => { + Box::new([&repr.out].into_iter()) + } + ModuleOperationIr::MaxPool2d(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::MaxPool2dWithIndices(repr) => { + Box::new([&repr.out, &repr.out_indices].into_iter()) + } + ModuleOperationIr::MaxPool2dWithIndicesBackward(repr) => { + Box::new([&repr.out].into_iter()) + } + ModuleOperationIr::Interpolate(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::InterpolateBackward(repr) => Box::new([&repr.out].into_iter()), + ModuleOperationIr::Attention(repr) => Box::new([&repr.out].into_iter()), + } + } + + fn mark_read_only(&mut self, nodes: &[TensorId]) -> Vec { + let mut output = Vec::new(); + + match self { + ModuleOperationIr::Embedding(repr) => { + repr.weights.mark_read_only(nodes, &mut output); + repr.indices.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::EmbeddingBackward(repr) => { + repr.weights.mark_read_only(nodes, &mut output); + repr.out_grad.mark_read_only(nodes, &mut output); + repr.indices.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv1d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + + if let Some(bias) = &mut repr.bias { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::Conv1dXBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv1dWeightBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv1dBiasBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.bias.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv2d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + + if let Some(bias) = &mut repr.bias { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::Conv2dXBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv2dWeightBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv2dBiasBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.bias.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv3d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + + if let Some(bias) = &mut repr.bias { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::Conv3dXBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv3dWeightBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Conv3dBiasBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.bias.mark_read_only(nodes, &mut output); + repr.output_grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::DeformableConv2d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.offset.mark_read_only(nodes, &mut output); + + match (&mut repr.mask, &mut repr.bias) { + (Some(mask), Some(bias)) => { + mask.mark_read_only(nodes, &mut output); + bias.mark_read_only(nodes, &mut output); + } + (Some(mask), None) => { + mask.mark_read_only(nodes, &mut output); + } + (None, Some(bias)) => { + bias.mark_read_only(nodes, &mut output); + } + (None, None) => {} + }; + } + ModuleOperationIr::DeformableConv2dBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + repr.offset.mark_read_only(nodes, &mut output); + repr.out_grad.mark_read_only(nodes, &mut output); + + if let Some(mask) = repr.mask.as_mut() { + mask.mark_read_only(nodes, &mut output); + } + if let Some(bias) = repr.bias.as_mut() { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::ConvTranspose1d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + + if let Some(bias) = &mut repr.bias { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::ConvTranspose2d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + + if let Some(bias) = &mut repr.bias { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::ConvTranspose3d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.weight.mark_read_only(nodes, &mut output); + + if let Some(bias) = &mut repr.bias { + bias.mark_read_only(nodes, &mut output); + } + } + ModuleOperationIr::AvgPool1d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AvgPool2d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AvgPool1dBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AvgPool2dBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AdaptiveAvgPool1d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AdaptiveAvgPool2d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AdaptiveAvgPool1dBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::AdaptiveAvgPool2dBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::MaxPool1d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::MaxPool1dWithIndices(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::MaxPool1dWithIndicesBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::MaxPool2d(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::MaxPool2dWithIndices(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::MaxPool2dWithIndicesBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Interpolate(repr) => { + repr.x.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::InterpolateBackward(repr) => { + repr.x.mark_read_only(nodes, &mut output); + repr.grad.mark_read_only(nodes, &mut output); + } + ModuleOperationIr::Attention(repr) => { + repr.query.mark_read_only(nodes, &mut output); + repr.key.mark_read_only(nodes, &mut output); + repr.value.mark_read_only(nodes, &mut output); + if let Some(mask) = &mut repr.mask { + mask.mark_read_only(nodes, &mut output); + } + if let Some(attn_bias) = &mut repr.attn_bias { + attn_bias.mark_read_only(nodes, &mut output); + } + } + }; + + output + } +} + +impl InitOperationIr { + fn inputs(&self) -> Box + '_> { + Box::new([].into_iter()) + } + fn outputs(&self) -> Box + '_> { + Box::new([&self.out].into_iter()) + } +} + +impl TensorIr { + fn mark_read_only(&mut self, nodes: &[TensorId], output: &mut Vec) { + if self.status == TensorStatus::ReadWrite && nodes.contains(&self.id) { + output.push(self.clone()); + self.status = TensorStatus::ReadOnly; + } + } +} + +impl core::hash::Hash for RandomOpIr { + fn hash(&self, state: &mut H) { + self.out.hash(state); + + match self.distribution { + Distribution::Default => 1u8.hash(state), + Distribution::Bernoulli(_) => 2u8.hash(state), + Distribution::Uniform(_, _) => 3u8.hash(state), + Distribution::Normal(_, _) => 4u8.hash(state), + } + } +} + +/// Extension trait to extract outputs when registering an operation. +pub trait OperationOutput { + /// Extract a single output. + fn output(self) -> O; + + /// Extract a fixed number of outputs. + fn outputs(self) -> [O; N]; +} + +impl OperationOutput for Vec { + fn output(self) -> O { + let [tensor] = self.outputs(); + tensor + } + + fn outputs(self) -> [O; N] { + self.try_into().unwrap() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/scalar.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/scalar.rs new file mode 100644 index 0000000..3434776 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/scalar.rs @@ -0,0 +1,77 @@ +use burn_backend::{DType, Scalar}; +use burn_backend::{Element, ElementConversion}; +use core::hash::Hash; +use serde::{Deserialize, Serialize}; + +/// A scalar representation. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub enum ScalarIr { + Float(f64), + Int(i64), + UInt(u64), + Bool(bool), +} + +impl Hash for ScalarIr { + fn hash(&self, state: &mut H) { + match self { + ScalarIr::Float(x) => x.to_bits().hash(state), + ScalarIr::Int(x) => x.hash(state), + ScalarIr::UInt(x) => x.hash(state), + ScalarIr::Bool(x) => x.hash(state), + } + } +} + +impl ScalarIr { + /// Creates a scalar with the specified data type. + pub fn new(value: E, dtype: &DType) -> Self { + if dtype.is_float() { + Self::Float(value.elem()) + } else if dtype.is_int() { + Self::Int(value.elem()) + } else if dtype.is_uint() { + Self::UInt(value.elem()) + } else if dtype.is_bool() { + Self::Bool(value.elem()) + } else { + unimplemented!("Scalar not supported for {dtype:?}") + } + } + + /// Converts and returns the converted element. + pub fn elem(self) -> E { + match self { + ScalarIr::Float(x) => x.elem(), + ScalarIr::Int(x) => x.elem(), + ScalarIr::UInt(x) => x.elem(), + ScalarIr::Bool(x) => x.elem(), + } + } +} + +// The enums are similar, but both types have different roles: +// - `Scalar`: runtime literal value +// - `ScalarIr`: serializable literal representation (used for IR) +impl From for ScalarIr { + fn from(value: Scalar) -> Self { + match value { + Scalar::Float(x) => Self::Float(x), + Scalar::Int(x) => Self::Int(x), + Scalar::UInt(x) => Self::UInt(x), + Scalar::Bool(x) => Self::Bool(x), + } + } +} + +impl From for Scalar { + fn from(value: ScalarIr) -> Self { + match value { + ScalarIr::Float(x) => Self::Float(x), + ScalarIr::Int(x) => Self::Int(x), + ScalarIr::UInt(x) => Self::UInt(x), + ScalarIr::Bool(x) => Self::Bool(x), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ir/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/tensor.rs new file mode 100644 index 0000000..a2eea66 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ir/src/tensor.rs @@ -0,0 +1,67 @@ +use serde::{Deserialize, Serialize}; + +use burn_backend::{DType, Shape}; + +/// The tensor unique identifier. +#[derive(Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)] +pub struct TensorId { + value: u64, +} + +impl core::fmt::Display for TensorId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("TensorId({:?})", self.value)) + } +} + +/// The status of the current tensor. +#[derive(Hash, Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum TensorStatus { + /// The tensor can be read, but not written. + ReadOnly, + /// The tensor can be mutated inplace. + ReadWrite, + /// No handle exists for that tensor. + NotInit, +} + +/// A tensor definition represents a snapshot of a tensor when it was used. +/// +/// # Example +/// +/// A tensor that is used multiple times has its status updated for each operation. +/// +/// 1. Status::NotInit +/// 2. Status::ReadOnly +/// 3. Status::ReadOnly +/// 4. Status::ReadWrite +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct TensorIr { + /// The [tensor id](TensorId). + pub id: TensorId, + /// The shape of the tensor. + pub shape: Shape, + /// The [status](TensorStatus) of the tensor when it was used. + pub status: TensorStatus, + /// The [type](DType) of the tensor. + pub dtype: DType, +} + +impl TensorId { + /// Create a new tensor id. + pub fn new(value: u64) -> Self { + Self { value } + } +} + +impl TensorIr { + /// Create a new tensor that is not already initialized. + pub fn uninit(id: TensorId, shape: Shape, dtype: DType) -> Self { + Self { + id, + status: TensorStatus::NotInit, + shape, + dtype, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/Cargo.toml new file mode 100644 index 0000000..70891e2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/Cargo.toml @@ -0,0 +1,97 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Ndarray backend for the Burn framework" +documentation = "https://docs.rs/burn-ndarray" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-ndarray" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-ndarray" +version.workspace = true + +[lints] +workspace = true + +[features] +blas-accelerate = [ + "blas-src/accelerate", # Accelerate framework (macOS only) + "ndarray/blas", +] +blas-netlib = ["blas-src/netlib", "ndarray/blas"] +blas-openblas = ["blas-src/openblas", "ndarray/blas", "openblas-src"] +blas-openblas-system = [ + "blas-src/openblas", + "ndarray/blas", + "openblas-src/system", +] +default = ["std", "simd", "multi-threads"] +doc = ["default"] +multi-threads = [ + "rayon", + "ndarray/rayon", + "matrixmultiply/threading", +] +simd = ["macerator", "bytemuck", "seq-macro", "itertools"] +std = [ + "burn-autodiff", + "burn-std/std", + "burn-backend/std", + "burn-ir/std", + "ndarray/std", + "matrixmultiply/std", + "rand/std", + "rand/std_rng", + "num-traits/std", + "macerator/std", +] +tracing = [ + "burn-autodiff?/tracing", + "burn-std/tracing", + "burn-backend/tracing", + "burn-ir/tracing", +] + +# Serves as a ref impl for some burn-cubecl kernels +export_tests = [] + +[dependencies] + +# ** Please make sure all dependencies support no_std when std is disabled ** + +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2", default-features = false } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } + +atomic_float = { workspace = true } +blas-src = { workspace = true, default-features = false, optional = true } # no-std compatible +const-random = { workspace = true } +libm = { workspace = true } +matrixmultiply = { workspace = true, default-features = false } +ndarray = { workspace = true } +num-traits = { workspace = true } +openblas-src = { workspace = true, optional = true } +paste = { workspace = true } +rand = { workspace = true, default-features = false } + +# SIMD +bytemuck = { workspace = true, optional = true } +itertools = { version = "0.14", optional = true } +macerator = { workspace = true, optional = true } +seq-macro = { version = "0.3", optional = true } + +# Parallel +rayon = { workspace = true, optional = true } + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +portable-atomic = { workspace = true } +portable-atomic-util = { workspace = true } + +[dev-dependencies] +bytes = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/README.md b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/README.md new file mode 100644 index 0000000..b2c4d10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/README.md @@ -0,0 +1,30 @@ +# Burn NdArray + +> [Burn](https://github.com/tracel-ai/burn) ndarray backend + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-ndarray.svg)](https://crates.io/crates/burn-ndarray) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-ndarray/blob/master/README.md) + +## Feature Flags + +This crate can be used without the standard library (`#![no_std]`) with `alloc` by disabling the +default `std` feature. + +The following flags support various BLAS options: + +- `blas-accelerate` - Accelerate framework (macOS only) +- `blas-netlib` - Netlib +- `blas-openblas` - OpenBLAS static linked +- `blas-openblas-system` - OpenBLAS from the system + +Note: under the `no_std` mode, the seed is fixed if the seed is not +initialized by `Backend::seed` method. + +### Platform Support + +| Option | CPU | GPU | Linux | MacOS | Windows | Android | iOS | WASM | +| :--------- | :-: | :-: | :---: | :---: | :-----: | :-----: | :-: | :--: | +| Pure Rust | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | +| Accelerate | Yes | No | No | Yes | No | No | Yes | No | +| Netlib | Yes | No | Yes | Yes | Yes | No | No | No | +| Openblas | Yes | No | Yes | Yes | Yes | Yes | Yes | No | diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/build.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/build.rs new file mode 100644 index 0000000..dcb4354 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/build.rs @@ -0,0 +1,6 @@ +fn main() { + // https://github.com/rust-ndarray/ndarray/issues/1197 + if cfg!(feature = "blas-accelerate") { + println!("cargo:rustc-link-lib=framework=Accelerate"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/backend.rs new file mode 100644 index 0000000..31d39b0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/backend.rs @@ -0,0 +1,221 @@ +use crate::rand::NdArrayRng; +use crate::{NdArrayQTensor, NdArrayTensor}; +use crate::{ + SharedArray, + element::{FloatNdArrayElement, IntNdArrayElement, QuantElement}, +}; +use alloc::string::String; +use burn_backend::quantization::{QuantLevel, QuantMode, QuantScheme, QuantStore, QuantValue}; +use burn_backend::tensor::{BoolTensor, FloatTensor, IntTensor, QuantizedTensor}; +use burn_backend::{Backend, DType, DeviceId, DeviceOps}; +use burn_ir::{BackendIr, HandleKind, TensorHandle}; +use burn_std::stub::Mutex; +use core::marker::PhantomData; +use rand::SeedableRng; + +pub(crate) static SEED: Mutex> = Mutex::new(None); + +/// The device type for the ndarray backend. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum NdArrayDevice { + /// The CPU device. + #[default] + Cpu, +} + +impl DeviceOps for NdArrayDevice {} + +impl burn_backend::Device for NdArrayDevice { + fn from_id(_device_id: DeviceId) -> Self { + Self::Cpu + } + + fn to_id(&self) -> DeviceId { + DeviceId { + type_id: 0, + index_id: 0, + } + } + + fn device_count(_type_id: u16) -> usize { + 1 + } +} + +/// Tensor backend that uses the [ndarray](ndarray) crate for executing tensor operations. +/// +/// This backend is compatible with CPUs and can be compiled for almost any platform, including +/// `wasm`, `arm`, and `x86`. +#[derive(Clone, Copy, Default, Debug)] +pub struct NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + _e: PhantomData, + _i: PhantomData, + _q: PhantomData, +} + +impl Backend for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + type Device = NdArrayDevice; + + type FloatTensorPrimitive = NdArrayTensor; + type FloatElem = E; + + type IntTensorPrimitive = NdArrayTensor; + type IntElem = I; + + type BoolTensorPrimitive = NdArrayTensor; + type BoolElem = bool; + + type QuantizedTensorPrimitive = NdArrayQTensor; + + fn ad_enabled(_device: &Self::Device) -> bool { + false + } + + fn name(_device: &Self::Device) -> String { + String::from("ndarray") + } + + fn seed(_device: &Self::Device, seed: u64) { + let rng = NdArrayRng::seed_from_u64(seed); + let mut seed = SEED.lock().unwrap(); + *seed = Some(rng); + } + + fn dtype_usage(_device: &Self::Device, dtype: DType) -> burn_backend::DTypeUsageSet { + match dtype { + DType::F64 + | DType::F32 + | DType::Flex32 + | DType::I64 + | DType::I32 + | DType::I16 + | DType::I8 + | DType::U64 + | DType::U32 + | DType::U16 + | DType::U8 + | DType::Bool => burn_backend::DTypeUsage::general(), + DType::F16 | DType::BF16 => burn_backend::DTypeUsageSet::empty(), + DType::QFloat(scheme) => { + match scheme { + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + #[cfg(not(feature = "export_tests"))] + value: QuantValue::Q8F | QuantValue::Q8S, + // For tests, "native" sub-byte quant serves as a reference for value equality. + // Values are stored as i8 regardless. + #[cfg(feature = "export_tests")] + value: + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S, + store: QuantStore::Native, + .. + } => burn_backend::DTypeUsage::general(), + _scheme => burn_backend::DTypeUsageSet::empty(), + } + } + } + } +} + +impl BackendIr for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + type Handle = HandleKind; + + fn float_tensor(handle: TensorHandle) -> FloatTensor { + match handle.handle { + HandleKind::Float(handle) => handle, + _ => panic!("Expected float handle, got {}", handle.handle.name()), + } + } + + fn int_tensor(handle: TensorHandle) -> IntTensor { + match handle.handle { + HandleKind::Int(handle) => handle, + _ => panic!("Expected int handle, got {}", handle.handle.name()), + } + } + + fn bool_tensor(handle: TensorHandle) -> BoolTensor { + match handle.handle { + HandleKind::Bool(handle) => handle, + _ => panic!("Expected bool handle, got {}", handle.handle.name()), + } + } + + fn quantized_tensor(handle: TensorHandle) -> QuantizedTensor { + match handle.handle { + HandleKind::Quantized(handle) => handle, + _ => panic!("Expected quantized handle, got {}", handle.handle.name()), + } + } + + fn float_tensor_handle(tensor: FloatTensor) -> Self::Handle { + HandleKind::Float(tensor) + } + + fn int_tensor_handle(tensor: IntTensor) -> Self::Handle { + HandleKind::Int(tensor) + } + + fn bool_tensor_handle(tensor: BoolTensor) -> Self::Handle { + HandleKind::Bool(tensor) + } + + fn quantized_tensor_handle(tensor: QuantizedTensor) -> Self::Handle { + HandleKind::Quantized(tensor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn_backend::QTensorPrimitive; + + #[test] + fn should_support_dtypes() { + type B = NdArray; + let device = Default::default(); + + assert!(B::supports_dtype(&device, DType::F64)); + assert!(B::supports_dtype(&device, DType::F32)); + assert!(B::supports_dtype(&device, DType::Flex32)); + assert!(B::supports_dtype(&device, DType::I64)); + assert!(B::supports_dtype(&device, DType::I32)); + assert!(B::supports_dtype(&device, DType::I16)); + assert!(B::supports_dtype(&device, DType::I8)); + assert!(B::supports_dtype(&device, DType::U64)); + assert!(B::supports_dtype(&device, DType::U32)); + assert!(B::supports_dtype(&device, DType::U16)); + assert!(B::supports_dtype(&device, DType::U8)); + assert!(B::supports_dtype(&device, DType::Bool)); + assert!(B::supports_dtype( + &device, + DType::QFloat(NdArrayQTensor::default_scheme()) + )); + + assert!(!B::supports_dtype(&device, DType::F16)); + assert!(!B::supports_dtype(&device, DType::BF16)); + // QuantStore::U32 not supported + assert!(!B::supports_dtype( + &device, + DType::QFloat(QuantScheme::default()) + )); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/element.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/element.rs new file mode 100644 index 0000000..8485352 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/element.rs @@ -0,0 +1,207 @@ +use burn_backend::Element; +use num_traits::Signed; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +use num_traits::Pow; + +use libm::{log1p, log1pf}; + +/// A float element for ndarray backend. +pub trait FloatNdArrayElement: NdArrayElement + Signed + core::cmp::PartialOrd +where + Self: Sized, +{ +} + +/// An int element for ndarray backend. +pub trait IntNdArrayElement: NdArrayElement + core::cmp::PartialOrd {} + +/// A general element for ndarray backend. +pub trait NdArrayElement: + Element + + ndarray::LinalgScalar + + ndarray::ScalarOperand + + ExpElement + + AddAssignElement + + num_traits::FromPrimitive + + core::ops::AddAssign + + core::cmp::PartialEq + + core::ops::Rem +{ +} + +/// A element for ndarray backend that supports exp ops. +pub trait ExpElement { + /// Exponent + fn exp_elem(self) -> Self; + /// Log + fn log_elem(self) -> Self; + /// Log1p + fn log1p_elem(self) -> Self; + /// Powf + fn powf_elem(self, value: f32) -> Self; + /// Powi + fn powi_elem(self, value: i32) -> Self; + /// Sqrt + fn sqrt_elem(self) -> Self; + /// Abs + fn abs_elem(self) -> Self; +} + +/// The addition assignment operator implemented for ndarray elements. +pub trait AddAssignElement { + /// Performs the addition assignment operation. + /// + /// For `bool`, this corresponds to logical OR assignment. + fn add_assign(&mut self, rhs: Rhs); +} + +impl AddAssignElement for E { + fn add_assign(&mut self, rhs: Self) { + *self += rhs; + } +} + +impl AddAssignElement for bool { + fn add_assign(&mut self, rhs: Self) { + *self = *self || rhs; // logical OR for bool + } +} + +/// A quantized element for the ndarray backend. +pub trait QuantElement: NdArrayElement {} + +impl QuantElement for i8 {} + +impl FloatNdArrayElement for f64 {} +impl FloatNdArrayElement for f32 {} + +impl IntNdArrayElement for i64 {} +impl IntNdArrayElement for i32 {} +impl IntNdArrayElement for i16 {} +impl IntNdArrayElement for i8 {} + +impl IntNdArrayElement for u64 {} +impl IntNdArrayElement for u32 {} +impl IntNdArrayElement for u16 {} +impl IntNdArrayElement for u8 {} + +macro_rules! make_float { + ( + $ty:ty, + $log1p:expr + ) => { + impl NdArrayElement for $ty {} + + #[allow(clippy::cast_abs_to_unsigned)] + impl ExpElement for $ty { + #[inline(always)] + fn exp_elem(self) -> Self { + self.exp() + } + + #[inline(always)] + fn log_elem(self) -> Self { + self.ln() + } + + #[inline(always)] + fn log1p_elem(self) -> Self { + $log1p(self) + } + + #[inline(always)] + fn powf_elem(self, value: f32) -> Self { + self.pow(value) + } + + #[inline(always)] + fn powi_elem(self, value: i32) -> Self { + #[cfg(feature = "std")] + let val = self.powi(value); + + #[cfg(not(feature = "std"))] + let val = Self::powf_elem(self, value as f32); + + val + } + + #[inline(always)] + fn sqrt_elem(self) -> Self { + self.sqrt() + } + + #[inline(always)] + fn abs_elem(self) -> Self { + self.abs() + } + } + }; +} +macro_rules! make_int { + ( + $ty:ty, + $abs:expr + ) => { + impl NdArrayElement for $ty {} + + #[allow(clippy::cast_abs_to_unsigned)] + impl ExpElement for $ty { + #[inline(always)] + fn exp_elem(self) -> Self { + (self as f32).exp() as $ty + } + + #[inline(always)] + fn log_elem(self) -> Self { + (self as f32).ln() as $ty + } + + #[inline(always)] + fn log1p_elem(self) -> Self { + log1pf(self as f32) as $ty + } + + #[inline(always)] + fn powf_elem(self, value: f32) -> Self { + (self as f32).pow(value) as $ty + } + + #[inline(always)] + fn powi_elem(self, value: i32) -> Self { + #[cfg(feature = "std")] + let val = f32::powi(self as f32, value) as $ty; + + #[cfg(not(feature = "std"))] + let val = Self::powf_elem(self, value as f32); + + val + } + + #[inline(always)] + fn sqrt_elem(self) -> Self { + (self as f32).sqrt() as $ty + } + + #[inline(always)] + fn abs_elem(self) -> Self { + $abs(self) + } + } + }; +} + +make_float!(f64, log1p); +make_float!(f32, log1pf); + +make_int!(i64, i64::wrapping_abs); +make_int!(i32, i32::wrapping_abs); +make_int!(i16, i16::wrapping_abs); +make_int!(i8, i8::wrapping_abs); +make_int!(u64, |x| x); +make_int!(u32, |x| x); +make_int!(u16, |x| x); +make_int!(u8, |x| x); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/lib.rs new file mode 100644 index 0000000..34a4625 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/lib.rs @@ -0,0 +1,29 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! Burn ndarray backend. + +#[cfg(any( + feature = "blas-netlib", + feature = "blas-openblas", + feature = "blas-openblas-system", +))] +extern crate blas_src; + +mod backend; +mod element; +mod ops; +mod parallel; +mod rand; +mod sharing; +mod storage; +mod tensor; + +pub use backend::*; +pub use element::*; +pub(crate) use sharing::*; +pub(crate) use storage::*; +pub use tensor::*; + +extern crate alloc; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/activation.rs new file mode 100644 index 0000000..9a872b5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/activation.rs @@ -0,0 +1,18 @@ +use crate::{ + NdArray, NdArrayTensor, SharedArray, + element::{FloatNdArrayElement, IntNdArrayElement, QuantElement}, + execute_with_numeric_dtype, + ops::NdArrayMathOps, +}; +use burn_backend::{ElementConversion, TensorMetadata, ops::ActivationOps, tensor::FloatTensor}; + +impl ActivationOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + fn relu(tensor: FloatTensor) -> FloatTensor { + execute_with_numeric_dtype!(tensor, |array| NdArrayMathOps::clamp_min(array, 0.elem())) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/adaptive_avgpool.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/adaptive_avgpool.rs new file mode 100644 index 0000000..baaee09 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/adaptive_avgpool.rs @@ -0,0 +1,103 @@ +use crate::{ + SharedArray, element::FloatNdArrayElement, iter_range_par, run_par, sharing::UnsafeSharedRef, +}; +use burn_backend::ElementConversion; +use ndarray::Array4; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +pub(crate) fn adaptive_avg_pool2d( + x: SharedArray, + output_size: [usize; 2], +) -> SharedArray { + let [batch_size, channels, input_height, input_width] = x.shape().try_into().unwrap(); + + let mut output = Array4::from_elem( + (batch_size, channels, output_size[0], output_size[1]), + 0.elem(), + ); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output = unsafe_shared_out.get(); + for h in 0..output_size[0] { + for w in 0..output_size[1] { + let ih_start = start_index(h, output_size[0], input_height); + let ih_end = end_index(h, output_size[0], input_height); + let iw_start = start_index(w, output_size[1], input_width); + let iw_end = end_index(w, output_size[1], input_width); + + let mut sum_val: E = 0.elem(); + + for ih in ih_start..ih_end { + for iw in iw_start..iw_end { + sum_val += x[[b, c, ih, iw]]; + } + } + + let count: E = (((ih_end - ih_start) * (iw_end - iw_start)) as i32).elem(); + output[[b, c, h, w]] = sum_val / count.elem(); + } + } + }) + }); + + output.into_dyn().into_shared() +} + +pub(crate) fn adaptive_avg_pool2d_backward( + x: SharedArray, + grad: SharedArray, +) -> SharedArray { + let [_, _, input_height, input_width] = x.shape().try_into().unwrap(); + let [batch_size, channels, output_height, output_width] = grad.shape().try_into().unwrap(); + + let mut output_grad = + Array4::from_elem((batch_size, channels, input_height, input_width), 0.elem()); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output_grad); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output_grad = unsafe_shared_out.get(); + for oh in 0..output_height { + for ow in 0..output_width { + let ih_start = start_index(oh, output_height, input_height); + let ih_end = end_index(oh, output_height, input_height); + + let iw_start = start_index(ow, output_width, input_width); + let iw_end = end_index(ow, output_width, input_width); + + let count: E = (((ih_end - ih_start) * (iw_end - iw_start)) as i32).elem(); + + for ih in ih_start..ih_end { + for iw in iw_start..iw_end { + output_grad[[b, c, ih, iw]] += grad[[b, c, oh, ow]] / count.elem(); + } + } + } + } + }) + }); + + output_grad.into_dyn().into_shared() +} + +fn start_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + ((output_size_index as f32 * input_size as f32) / output_size as f32).floor() as usize +} + +fn end_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + let index = + (((output_size_index + 1) as f32 * input_size as f32) / output_size as f32).ceil() as usize; + + usize::min(index, input_size) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/avgpool.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/avgpool.rs new file mode 100644 index 0000000..4d015dd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/avgpool.rs @@ -0,0 +1,172 @@ +use crate::{ + SharedArray, element::FloatNdArrayElement, iter_range_par, run_par, sharing::UnsafeSharedRef, +}; + +use burn_backend::ElementConversion; +use burn_backend::ops::conv::calculate_pool_output_size; +use ndarray::Array4; + +pub(crate) fn avg_pool2d( + x: SharedArray, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, +) -> SharedArray { + let [kernel_height, kernel_width] = kernel_size; + let [padding_height, padding_width] = padding; + let [stride_height, stride_width] = stride; + let [batch_size, channels, x_height, x_width] = x.shape().try_into().unwrap(); + + let out_height = calculate_pool_output_size( + kernel_height, + stride_height, + padding_height, + 1, + x_height, + ceil_mode, + ); + let out_width = calculate_pool_output_size( + kernel_width, + stride_width, + padding_width, + 1, + x_width, + ceil_mode, + ); + + // Padded input bounds (for count_include_pad calculation) + let padded_height = x_height + 2 * padding_height; + let padded_width = x_width + 2 * padding_width; + + let mut output = Array4::from_elem((batch_size, channels, out_height, out_width), 0.elem()); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output = unsafe_shared_out.get(); + + for oh in 0..out_height { + for ow in 0..out_width { + let mut sum_val: E = 0.elem(); + let mut valid_count = 0usize; + let mut padded_count = 0usize; + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw; + + // Check if within padded bounds (excludes ceil_mode extensions) + if ih < padded_height && iw < padded_width { + padded_count += 1; + + // Check if within valid (non-padding) input bounds + if ih >= padding_height + && ih < x_height + padding_height + && iw >= padding_width + && iw < x_width + padding_width + { + let ih_valid = ih - padding_height; + let iw_valid = iw - padding_width; + sum_val += x[[b, c, ih_valid, iw_valid]]; + valid_count += 1; + } + } + } + } + + // count_include_pad: count positions within padded bounds (not ceil_mode extensions) + // !count_include_pad: count only valid (non-padding) positions + let count: E = if count_include_pad { + (padded_count as i32).elem() + } else { + (valid_count as i32).elem() + }; + + output[[b, c, oh, ow]] = sum_val / count; + } + } + }) + }); + + output.into_dyn().into_shared() +} + +pub(crate) fn avg_pool2d_backward( + x: SharedArray, + grad: SharedArray, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + _ceil_mode: bool, +) -> SharedArray { + let [kernel_height, kernel_width] = kernel_size; + let [stride_height, stride_width] = stride; + let [padding_height, padding_width] = padding; + let [batch_size, channels, x_height, x_width] = x.shape().try_into().unwrap(); + let [_batch_size, _channels, out_height, out_width] = grad.shape().try_into().unwrap(); + + // Padded input bounds (for count_include_pad calculation) + let padded_height = x_height + 2 * padding_height; + let padded_width = x_width + 2 * padding_width; + + let mut output_grad = Array4::from_elem((batch_size, channels, x_height, x_width), 0.elem()); + let unsafe_shared_grad = UnsafeSharedRef::new(&mut output_grad); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output_grad = unsafe_shared_grad.get(); + + for oh in 0..out_height { + for ow in 0..out_width { + let ih_start_kernel = oh * stride_height; + let iw_start_kernel = ow * stride_width; + + let ih_end_kernel = ih_start_kernel + kernel_height; + let iw_end_kernel = iw_start_kernel + kernel_width; + + // Clip to valid input bounds (for gradient distribution) + let ih_start = usize::max(ih_start_kernel, padding_height); + let iw_start = usize::max(iw_start_kernel, padding_width); + let ih_end = usize::min(ih_end_kernel, x_height + padding_height); + let iw_end = usize::min(iw_end_kernel, x_width + padding_width); + + // Calculate count based on count_include_pad + let count = if count_include_pad { + // Count positions within padded bounds (not ceil_mode extensions) + let ih_start_padded = ih_start_kernel; + let iw_start_padded = iw_start_kernel; + let ih_end_padded = usize::min(ih_end_kernel, padded_height); + let iw_end_padded = usize::min(iw_end_kernel, padded_width); + (ih_end_padded - ih_start_padded) * (iw_end_padded - iw_start_padded) + } else { + // Count only valid (non-padding) positions + (ih_end - ih_start) * (iw_end - iw_start) + }; + + for ih in ih_start..ih_end { + for iw in iw_start..iw_end { + let ih = ih - padding_height; + let iw = iw - padding_width; + + output_grad[[b, c, ih, iw]] += + grad[[b, c, oh, ow]] / (count as i32).elem(); + } + } + } + } + }) + }); + + output_grad.into_dyn().into_shared() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/base.rs new file mode 100644 index 0000000..5d2ce42 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/base.rs @@ -0,0 +1,1448 @@ +use alloc::{vec, vec::Vec}; +use burn_backend::element::{Element, ElementConversion}; +#[cfg(feature = "simd")] +use burn_backend::{DType, quantization::QuantValue}; +use core::fmt::Debug; +use core::marker::PhantomData; +use ndarray::IntoDimension; +use ndarray::SliceInfo; +use ndarray::Zip; +use ndarray::s; +use ndarray::{Array2, ArrayD}; +use num_traits::Signed; +#[cfg(feature = "simd")] +use paste::paste; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +#[cfg(feature = "simd")] +use crate::ops::simd::{ + binary::try_binary_simd, + binary_elemwise::{ + VecAdd, VecBitAnd, VecBitOr, VecBitXor, VecClamp, VecDiv, VecMax, VecMin, VecMul, VecSub, + try_binary_scalar_simd, + }, + cmp::{ + VecEquals, VecGreater, VecGreaterEq, VecLower, VecLowerEq, try_cmp_scalar_simd, + try_cmp_simd, + }, + unary::{RecipVec, VecAbs, VecBitNot, try_unary_simd}, +}; +use crate::reshape; +use crate::{ + IntNdArrayElement, ShapeOps, + ops::macros::{ + cummax_dim, cummin_dim, cumprod_dim, cumsum_dim, keepdim, mean_dim, prod_dim, sum_dim, + }, +}; +use crate::{SharedArray, element::NdArrayElement}; +use burn_backend::ops::unfold::calculate_unfold_shape; +use burn_backend::{Shape, Slice}; +use ndarray::ArrayView; +use ndarray::Axis; +use ndarray::Dim; +use ndarray::IxDyn; +use ndarray::SliceInfoElem; + +pub struct NdArrayOps { + e: PhantomData, +} + +pub(crate) struct NdArrayMathOps { + e: PhantomData, +} + +impl NdArrayOps +where + E: Copy + Debug + Element + crate::AddAssignElement, +{ + pub fn slice(tensor: ArrayView, slices: &[Slice]) -> SharedArray { + let slices = Self::to_slice_args_with_steps(slices, tensor.shape().num_dims()); + tensor.slice_move(slices.as_slice()).to_shared() + } + + pub fn slice_assign( + tensor: SharedArray, + slices: &[Slice], + value: SharedArray, + ) -> SharedArray { + let slices = Self::to_slice_args_with_steps(slices, tensor.shape().num_dims()); + let mut array = tensor.into_owned(); + array.slice_mut(slices.as_slice()).assign(&value); + array.into_shared() + } + + pub fn mask_where( + tensor: SharedArray, + mask: SharedArray, + source: SharedArray, + ) -> SharedArray { + let tensor = tensor.broadcast(mask.dim()).unwrap(); + let source = source.broadcast(mask.dim()).unwrap(); + Zip::from(&tensor) + .and(&mask) + .and(&source) + .map_collect(|&x, &mask_val, &y| if mask_val { y } else { x }) + .into_shared() + } + + pub fn mask_fill(tensor: SharedArray, mask: SharedArray, value: E) -> SharedArray { + // Use into_owned() instead of clone() - only copies if shared, avoids copy if unique + let mut output = tensor.into_owned(); + let broadcast_mask = mask.broadcast(output.dim()).unwrap(); + Zip::from(&mut output) + .and(&broadcast_mask) + .for_each(|out, &mask_val| { + if mask_val { + *out = value; + } + }); + output.into_shared() + } + + pub fn gather( + dim: usize, + mut tensor: SharedArray, + mut indices: SharedArray, + ) -> SharedArray { + let ndims = tensor.shape().num_dims(); + if dim != ndims - 1 { + tensor.swap_axes(ndims - 1, dim); + indices.swap_axes(ndims - 1, dim); + } + let (shape_tensor, shape_indices) = (tensor.shape(), indices.shape().into_shape()); + let (size_tensor, size_index) = (shape_tensor[ndims - 1], shape_indices[ndims - 1]); + let batch_size = Self::gather_batch_size(shape_tensor, &shape_indices); + + let indices = NdArrayOps::reshape(indices, Shape::new([batch_size, size_index])); + let tensor = NdArrayOps::reshape(tensor, Shape::new([batch_size, size_tensor])); + let mut output = Array2::from_elem((batch_size, size_index), 0.elem::()); + + for b in 0..batch_size { + let indices = indices.slice(s!(b, ..)); + for (i, index) in indices.iter().enumerate() { + output[[b, i]] = tensor[[b, index.elem::() as usize]]; + } + } + + let mut output = NdArrayOps::reshape(output.into_shared().into_dyn(), shape_indices); + + if dim != ndims - 1 { + output.swap_axes(ndims - 1, dim); + } + + output + } + + pub fn scatter( + dim: usize, + mut tensor: SharedArray, + mut indices: SharedArray, + mut value: SharedArray, + ) -> SharedArray { + let ndims = tensor.shape().num_dims(); + if dim != ndims - 1 { + tensor.swap_axes(ndims - 1, dim); + indices.swap_axes(ndims - 1, dim); + value.swap_axes(ndims - 1, dim); + } + + let (shape_tensor, shape_indices, shape_value) = + (tensor.shape().into_shape(), indices.shape(), value.shape()); + let (size_tensor, size_index, size_value) = ( + shape_tensor[ndims - 1], + shape_indices[ndims - 1], + shape_value[ndims - 1], + ); + let batch_size = Self::gather_batch_size(&shape_tensor, shape_indices); + + if shape_value != shape_indices { + panic!( + "Invalid dimension: the shape of the index tensor should be the same as the value \ + tensor: Index {:?} value {:?}", + shape_indices, shape_value + ); + } + + let indices = NdArrayOps::reshape(indices, Shape::new([batch_size, size_index])); + let value = NdArrayOps::reshape(value, Shape::new([batch_size, size_value])); + let mut tensor = NdArrayOps::reshape(tensor, Shape::new([batch_size, size_tensor])); + + for b in 0..batch_size { + let indices = indices.slice(s!(b, ..)); + + for (i, index) in indices.iter().enumerate() { + let index = index.elem::() as usize; + tensor[[b, index]].add_assign(value[[b, i]]); + } + } + + let mut output = NdArrayOps::reshape(tensor.into_shared().into_dyn(), shape_tensor); + if dim != ndims - 1 { + output.swap_axes(ndims - 1, dim); + } + output + } + + fn gather_batch_size(shape_tensor: &[usize], shape_indices: &[usize]) -> usize { + let ndims = shape_tensor.num_dims(); + let mut batch_size = 1; + + for i in 0..ndims - 1 { + if shape_tensor[i] != shape_indices[i] { + panic!( + "Unsupported dimension, only the last dimension can differ: Tensor {:?} Index \ + {:?}", + shape_tensor, shape_indices + ); + } + batch_size *= shape_indices[i]; + } + + batch_size + } + + pub fn reshape(tensor: SharedArray, shape: Shape) -> SharedArray { + reshape!( + ty E, + shape shape, + array tensor, + d shape.num_dims() + ) + } + + pub(crate) fn concatenate( + arrays: &[ndarray::ArrayView], + dim: usize, + ) -> SharedArray { + let array = ndarray::concatenate(Axis(dim), arrays) + .unwrap() + .into_shared(); + + // Transform column-major layout into row-major (standard) layout. (fix #1053) + // Get shape first (via reference), then pass ownership to avoid clone + let shape = array.shape().into_shape(); + Self::reshape(array, shape) + } + + pub fn cat(tensors: Vec>, dim: usize) -> SharedArray { + let arrays: Vec<_> = tensors.iter().map(|t| t.view()).collect(); + Self::concatenate(&arrays, dim) + } + + #[allow(clippy::wrong_self_convention)] + fn to_slice_args_with_steps( + burn_slices: &[burn_backend::Slice], + ndims: usize, + ) -> Vec { + let mut slices = vec![SliceInfoElem::NewAxis; ndims]; + + for i in 0..ndims { + slices[i] = if i < burn_slices.len() { + let slice = &burn_slices[i]; + + // Check for empty range (would result in no elements) + if let Some(end) = slice.end + && slice.start == end + { + SliceInfoElem::Slice { + start: 0, + end: Some(0), + step: 1, + } + } else { + // Pass slice parameters directly to ndarray + // ndarray handles both positive and negative steps correctly: + // - Positive step: iterates forward from start + // - Negative step: iterates backward from the last element in range + SliceInfoElem::Slice { + start: slice.start, + end: slice.end, + step: slice.step, + } + } + } else { + // Dimension not specified in slices - use full range + SliceInfoElem::Slice { + start: 0, + end: None, + step: 1, + } + } + } + + slices + } + + pub fn swap_dims(mut tensor: SharedArray, dim1: usize, dim2: usize) -> SharedArray { + tensor.swap_axes(dim1, dim2); + + tensor + } + + pub fn permute(tensor: SharedArray, axes: &[usize]) -> SharedArray { + tensor.permuted_axes(axes.into_dimension()) + } + + /// Broadcasts the tensor to the given shape + pub(crate) fn expand(tensor: SharedArray, shape: Shape) -> SharedArray { + tensor + .broadcast(shape.into_dimension()) + .expect("The shapes should be broadcastable") + // need to convert view to owned array because NdArrayTensor expects owned array + // and try_into_owned_nocopy() panics for broadcasted arrays (zero strides) + .into_owned() + .into_shared() + } + + pub fn flip(tensor: SharedArray, axes: &[usize]) -> SharedArray { + let slice_items: Vec<_> = (0..tensor.shape().num_dims()) + .map(|i| { + if axes.contains(&i) { + SliceInfoElem::Slice { + start: 0, + end: None, + step: -1, + } + } else { + SliceInfoElem::Slice { + start: 0, + end: None, + step: 1, + } + } + }) + .collect(); + let slice_info = + SliceInfo::, IxDyn, IxDyn>::try_from(slice_items).unwrap(); + tensor.slice(slice_info).into_owned().into_shared() + } + + /// Unfold windows along a dimension. + /// + /// # Warning + /// + /// This is a copy impl; `ndarray` doesn't expose the layout machinery + /// necessary to build the stride view. + /// + /// Returns a copy of the tensor with all complete windows of size `size` in dimension `dim`; + /// where windows are advanced by `step` at each index. + /// + /// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. + /// + /// # Arguments + /// + /// * `tensor` - The input tensor to unfold; of shape ``[pre=..., dim shape, post=...]`` + /// * `dim` - the dimension to unfold. + /// * `size` - the size of each unfolded window. + /// * `step` - the step between each window. + /// + /// # Returns + /// + /// A tensor view with shape ``[pre=..., windows, post=..., size]``. + #[allow(unused)] + pub(crate) fn unfold( + tensor: SharedArray, + dim: usize, + size: usize, + step: usize, + ) -> SharedArray { + let result_shape = calculate_unfold_shape(tensor.shape(), dim, size, step); + let windows = result_shape[dim]; + + let mut slices = vec![Slice::new(0, None, 1); tensor.shape().len()]; + let new_axis = slices.len(); + + let mut stack = Vec::with_capacity(windows); + for widx in 0..windows { + let start = widx * step; + let end = start + size; + slices[dim] = Slice::new(start as isize, Some(end as isize), 1); + + let mut window_slice = + tensor.slice(Self::to_slice_args_with_steps(&slices, slices.len()).as_slice()); + window_slice.insert_axis_inplace(Axis(new_axis)); + window_slice.swap_axes(dim, new_axis); + + stack.push(window_slice); + } + Self::concatenate(&stack, dim) + } +} + +#[cfg(feature = "simd")] +macro_rules! dispatch_binary_simd { + (noq, $elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_binary_simd::<$elem, $elem, $ty, $ty, $op>($lhs, $rhs),)* + _ => Err(($lhs, $rhs)), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_binary_simd::<$elem, $elem, $ty, $ty, $op>($lhs, $rhs),)* + DType::QFloat(strategy) => match strategy.value { + QuantValue::Q8F | QuantValue::Q8S => try_binary_simd::<$elem, $elem, i8, i8, $op>($lhs, $rhs), + _ => Err(($lhs, $rhs)), + }, + _ => Err(($lhs, $rhs)), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; +} + +#[cfg(not(feature = "simd"))] +macro_rules! dispatch_binary_simd { + (noq, $elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ ($lhs, $rhs) }}; + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ ($lhs, $rhs) }}; +} + +#[cfg(feature = "simd")] +macro_rules! dispatch_binary_scalar_simd { + (noq, $elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_binary_scalar_simd::<$elem, $elem, $ty, $ty, $op>($lhs, $rhs),)* + _ => Err($lhs), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_binary_scalar_simd::<$elem, $elem, $ty, $ty, $op>($lhs, $rhs),)* + DType::QFloat(strategy) => match strategy.value { + QuantValue::Q8F | QuantValue::Q8S => try_binary_scalar_simd::<$elem, $elem, i8, i8, $op>($lhs, $rhs), + QuantValue::Q4F | QuantValue::Q4S | QuantValue::Q2F | QuantValue::Q2S | QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1 => Err($lhs) + }, + _ => Err($lhs), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; +} + +#[cfg(not(feature = "simd"))] +macro_rules! dispatch_binary_scalar_simd { + (noq, $elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ $lhs }}; + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ $lhs }}; +} + +#[cfg(feature = "simd")] +macro_rules! dispatch_cmp_simd { + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_cmp_simd::<$elem, $ty, $op>($lhs, $rhs),)* + DType::QFloat(strategy) => match strategy.value { + QuantValue::Q8F | QuantValue::Q8S => try_cmp_simd::<$elem, i8, $op>($lhs, $rhs), + QuantValue::Q4F | QuantValue::Q4S | QuantValue::Q2F | QuantValue::Q2S | QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1 => Err(($lhs, $rhs)) + }, + _ => Err(($lhs, $rhs)), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; +} + +#[cfg(not(feature = "simd"))] +macro_rules! dispatch_cmp_simd { + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ ($lhs, $rhs) }}; +} + +#[cfg(feature = "simd")] +macro_rules! dispatch_cmp_scalar_simd { + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_cmp_scalar_simd::<$elem, $ty, $op>($lhs, $rhs),)* + DType::QFloat(strategy) => match strategy.value { + QuantValue::Q8F | QuantValue::Q8S => try_cmp_scalar_simd::<$elem, i8, $op>($lhs, $rhs), + QuantValue::Q4F | QuantValue::Q4S | QuantValue::Q2F | QuantValue::Q2S | QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1 => Err($lhs) + }, + _ => Err($lhs), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; +} + +#[cfg(not(feature = "simd"))] +macro_rules! dispatch_cmp_scalar_simd { + ($elem: ty, $op: ty, $lhs: expr, $rhs: expr, $($ty: ty),*) => {{ $lhs }}; +} + +#[cfg(feature = "simd")] +macro_rules! dispatch_unary_simd { + ($elem: ty, $op: ty, $lhs: expr, $($ty: ty),*) => {{ + paste! { + let simd = match $elem::dtype() { + $(DType::[<$ty:upper>] => try_unary_simd::<$elem, $elem, $ty, $ty, $op>($lhs),)* + _ => Err($lhs), + }; + match simd { + Ok(out) => return out, + Err(args) => args, + } + } + }}; +} + +#[cfg(not(feature = "simd"))] +macro_rules! dispatch_unary_simd { + ($elem: ty, $op: ty, $lhs: expr, $($ty: ty),*) => {{ $lhs }}; +} + +// Helper function to broadcast two tensors to a common shape for comparison operations +// Returns broadcasted views that can be safely zipped +fn broadcast_for_comparison<'a, E: Copy, S1, S2>( + lhs: &'a ndarray::ArrayBase, + rhs: &'a ndarray::ArrayBase, +) -> ( + ndarray::ArrayView<'a, E, ndarray::IxDyn>, + ndarray::ArrayView<'a, E, ndarray::IxDyn>, +) +where + S1: ndarray::Data, + S2: ndarray::Data, +{ + // Get shapes + let lhs_shape = lhs.shape(); + let rhs_shape = rhs.shape(); + + // Compute broadcast shape using ndarray's broadcast compatibility rules + let ndims = lhs_shape.len().max(rhs_shape.len()); + let mut broadcast_shape = vec![1; ndims]; + + for i in 0..ndims { + let lhs_dim = if i < lhs_shape.len() { + lhs_shape[lhs_shape.len() - 1 - i] + } else { + 1 + }; + let rhs_dim = if i < rhs_shape.len() { + rhs_shape[rhs_shape.len() - 1 - i] + } else { + 1 + }; + + if lhs_dim == rhs_dim { + broadcast_shape[ndims - 1 - i] = lhs_dim; + } else if lhs_dim == 1 { + broadcast_shape[ndims - 1 - i] = rhs_dim; + } else if rhs_dim == 1 { + broadcast_shape[ndims - 1 - i] = lhs_dim; + } else { + panic!( + "Incompatible shapes for broadcasting: {:?} and {:?}", + lhs_shape, rhs_shape + ); + } + } + + // Create IxDyn from broadcast shape + let broadcast_dim = ndarray::IxDyn(&broadcast_shape); + + // Broadcast both arrays + let lhs_broadcast = lhs + .broadcast(broadcast_dim.clone()) + .expect("Failed to broadcast lhs"); + let rhs_broadcast = rhs + .broadcast(broadcast_dim) + .expect("Failed to broadcast rhs"); + + (lhs_broadcast, rhs_broadcast) +} + +impl NdArrayMathOps +where + E: Copy + NdArrayElement, +{ + pub fn add(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_binary_simd!( + E, VecAdd, lhs, rhs, u8, i8, u16, i16, u32, i32, f32, u64, i64, f64 + ); + + let array = &lhs + &rhs; + array.into_shared() + } + + pub fn add_scalar(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!( + E, + VecAdd, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + i32, + f32, + u64, + i64, + f64 + ); + + let array = lhs + rhs; + array.into_shared() + } + + pub fn sub(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_binary_simd!( + E, VecSub, lhs, rhs, u8, i8, u16, i16, u32, i32, f32, u64, i64, f64 + ); + + let array = lhs - rhs; + array.into_shared() + } + + pub fn sub_scalar(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!( + E, + VecSub, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + i32, + f32, + u64, + i64, + f64 + ); + + let array = lhs - rhs; + array.into_shared() + } + + pub fn mul(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = + dispatch_binary_simd!(noq, E, VecMul, lhs, rhs, u16, i16, u32, i32, f32, f64); + + let array = lhs * rhs; + array.into_shared() + } + + pub fn mul_scalar(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!( + noq, + E, + VecMul, + lhs, + rhs.elem(), + u16, + i16, + u32, + i32, + f32, + f64 + ); + + let array = lhs * rhs; + array.into_shared() + } + + pub fn div(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_binary_simd!(noq, E, VecDiv, lhs, rhs, f32, f64); + + let array = lhs / rhs; + array.into_shared() + } + + pub fn div_scalar(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!(noq, E, VecDiv, lhs, rhs.elem(), f32, f64); + + let array = lhs / rhs; + array.into_shared() + } + + pub fn remainder(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + // Use into_owned() instead of clone() - only copies if shared, avoids copy if unique + let mut out = lhs.into_owned(); + Zip::from(&mut out).and(&rhs).for_each(|out_elem, &b| { + // out_elem holds lhs value; read it before overwriting with remainder + let a_f = (*out_elem).to_f64(); + let b_f = b.to_f64(); + let r = a_f - b_f * (a_f / b_f).floor(); + *out_elem = r.elem(); + }); + out.into_shared() + } + + pub fn remainder_scalar(lhs: SharedArray, rhs: E) -> SharedArray + where + E: core::ops::Rem, + { + let array = lhs.mapv(|x| ((x % rhs) + rhs) % rhs); + array.into_shared() + } + + pub fn recip(tensor: SharedArray) -> SharedArray { + let tensor = dispatch_unary_simd!(E, RecipVec, tensor, f32); + + let array = tensor.map(|x| 1.elem::() / *x); + array.into_shared() + } + + /// Sum all elements - zero-copy for borrowed storage. + pub fn sum_view(view: ArrayView<'_, E, IxDyn>) -> SharedArray { + let sum = view.sum(); + ArrayD::from_elem(IxDyn(&[1]), sum).into_shared() + } + + /// Mean of all elements - zero-copy for borrowed storage. + pub fn mean_view(view: ArrayView<'_, E, IxDyn>) -> SharedArray { + let mean = view.mean().unwrap(); + ArrayD::from_elem(IxDyn(&[1]), mean).into_shared() + } + + /// Product of all elements - zero-copy for borrowed storage. + pub fn prod_view(view: ArrayView<'_, E, IxDyn>) -> SharedArray { + let prod = view.iter().fold(E::one(), |acc, &x| acc * x); + ArrayD::from_elem(IxDyn(&[1]), prod).into_shared() + } + + pub fn mean_dim(tensor: SharedArray, dim: usize) -> SharedArray { + let ndims = tensor.shape().num_dims(); + match ndims { + d if (1..=6).contains(&d) => keepdim!(dim, tensor, mean), + _ => panic!("Dim not supported {ndims}"), + } + } + + pub fn sum_dim(tensor: SharedArray, dim: usize) -> SharedArray { + let ndims = tensor.shape().num_dims(); + match ndims { + d if (1..=6).contains(&d) => keepdim!(dim, tensor, sum), + _ => panic!("Dim not supported {ndims}"), + } + } + + pub fn prod_dim(tensor: SharedArray, dim: usize) -> SharedArray { + let ndims = tensor.shape().num_dims(); + match ndims { + d if (1..=6).contains(&d) => keepdim!(dim, tensor, prod), + _ => panic!("Dim not supported {ndims}"), + } + } + + pub fn cumsum(tensor: SharedArray, dim: usize) -> SharedArray { + cumsum_dim(tensor, dim) + } + + pub fn cumprod(tensor: SharedArray, dim: usize) -> SharedArray { + cumprod_dim(tensor, dim) + } + + pub fn select( + tensor: SharedArray, + dim: usize, + indices: SharedArray, + ) -> SharedArray { + let array = tensor.select( + Axis(dim), + &indices + .into_iter() + .map(|i| i.elem::() as usize) + .collect::>(), + ); + + array.into_shared() + } + + pub fn select_assign( + tensor: SharedArray, + dim: usize, + indices: SharedArray, + value: SharedArray, + ) -> SharedArray { + let mut output_array = tensor.into_owned(); + + for (index_value, index) in indices.into_iter().enumerate() { + let mut view = output_array.index_axis_mut(Axis(dim), index.elem::() as usize); + let value = value.index_axis(Axis(dim), index_value); + + view.zip_mut_with(&value, |a, b| *a += *b); + } + + output_array.into_shared() + } + + pub(crate) fn elementwise_op( + lhs: SharedArray, + rhs: SharedArray, + var_name: impl FnMut(&E, &OtherE) -> E, + ) -> SharedArray { + let lhs = lhs.broadcast(rhs.dim()).unwrap_or(lhs.view()); + let rhs = rhs.broadcast(lhs.dim()).unwrap_or(rhs.view()); + + Zip::from(lhs).and(rhs).map_collect(var_name).into_shared() + } + + pub(crate) fn elementwise_op_scalar( + lhs: SharedArray, + var_name: impl FnMut(E) -> E, + ) -> SharedArray { + lhs.mapv(var_name).into_shared() + } + + pub(crate) fn abs(tensor: SharedArray) -> SharedArray { + let tensor = dispatch_unary_simd!(E, VecAbs, tensor, i8, i16, i32, f32, f64); + + tensor.mapv_into(|a| a.abs_elem()).into_shared() + } + + pub(crate) fn equal(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_cmp_simd!( + E, VecEquals, lhs, rhs, u8, i8, u16, i16, u32, f32, i32, u64, i64, f64 + ); + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs == rhs) + .into_shared() + } + + pub(crate) fn equal_elem(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_cmp_scalar_simd!( + E, + VecEquals, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + f32, + i32, + u64, + i64, + f64 + ); + + lhs.mapv(|a| a == rhs).into_shared() + } + + pub(crate) fn sign_op(tensor: SharedArray) -> SharedArray + where + E: Signed, + { + let zero = 0.elem(); + let one = 1.elem::(); + + tensor + .mapv(|x| { + if x == zero { + zero + } else { + match x.is_positive() { + true => one, + false => -one, + } + } + }) + .into_shared() + } +} + +impl NdArrayMathOps +where + E: Copy + NdArrayElement + PartialOrd, +{ + /// Max of all elements - zero-copy for borrowed storage. + pub fn max_view(view: ArrayView<'_, E, IxDyn>) -> SharedArray { + let max = view + .iter() + .copied() + .reduce(|a, b| if a > b { a } else { b }) + .expect("Cannot compute max of empty tensor"); + ArrayD::from_elem(IxDyn(&[1]), max).into_shared() + } + + /// Min of all elements - zero-copy for borrowed storage. + pub fn min_view(view: ArrayView<'_, E, IxDyn>) -> SharedArray { + let min = view + .iter() + .copied() + .reduce(|a, b| if a < b { a } else { b }) + .expect("Cannot compute min of empty tensor"); + ArrayD::from_elem(IxDyn(&[1]), min).into_shared() + } + + /// Argmax along dimension - zero-copy for borrowed storage. + pub fn argmax_view( + view: ArrayView<'_, E, IxDyn>, + dim: usize, + ) -> SharedArray { + arg_view(view, dim, CmpType::Max) + } + + /// Argmin along dimension - zero-copy for borrowed storage. + pub fn argmin_view( + view: ArrayView<'_, E, IxDyn>, + dim: usize, + ) -> SharedArray { + arg_view(view, dim, CmpType::Min) + } + + pub fn cummin(tensor: SharedArray, dim: usize) -> SharedArray { + cummin_dim(tensor, dim) + } + + pub fn cummax(tensor: SharedArray, dim: usize) -> SharedArray { + cummax_dim(tensor, dim) + } + + pub fn argmax( + tensor: SharedArray, + dim: usize, + ) -> SharedArray { + arg(tensor, dim, CmpType::Max) + } + + pub fn argmin( + tensor: SharedArray, + dim: usize, + ) -> SharedArray { + arg(tensor, dim, CmpType::Min) + } + + pub fn clamp_min(tensor: SharedArray, min: E) -> SharedArray { + let mut tensor = dispatch_binary_scalar_simd!( + E, + VecMax, + tensor, + min.elem(), + u8, + i8, + u16, + i16, + u32, + i32, + f32, + u64, + i64, + f64 + ); + + tensor.mapv_inplace(|x| match x < min { + true => min, + false => x, + }); + + tensor + } + + pub fn clamp_max(tensor: SharedArray, max: E) -> SharedArray { + let mut tensor = dispatch_binary_scalar_simd!( + E, + VecMin, + tensor, + max.elem(), + u8, + i8, + u16, + i16, + u32, + i32, + f32, + u64, + i64, + f64 + ); + + tensor.mapv_inplace(|x| match x > max { + true => max, + false => x, + }); + + tensor + } + + pub fn clamp(tensor: SharedArray, min: E, max: E) -> SharedArray { + let mut tensor = dispatch_binary_scalar_simd!( + E, + VecClamp, + tensor, + (min.elem(), max.elem()), + u8, + i8, + u16, + i16, + u32, + i32, + f32, + u64, + i64, + f64 + ); + + tensor.mapv_inplace(|x| match x < min { + true => min, + false => match x > max { + true => max, + false => x, + }, + }); + + tensor + } + + pub(crate) fn greater(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_cmp_simd!( + E, VecGreater, lhs, rhs, u8, i8, u16, i16, u32, f32, i32, u64, i64, f64 + ); + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs > rhs) + .into_shared() + } + + pub(crate) fn greater_elem(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_cmp_scalar_simd!( + E, + VecGreater, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + f32, + i32, + u64, + i64, + f64 + ); + + lhs.mapv(|a| a > rhs).into_shared() + } + + pub(crate) fn greater_equal(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_cmp_simd!( + E, + VecGreaterEq, + lhs, + rhs, + u8, + i8, + u16, + i16, + u32, + f32, + i32, + u64, + i64, + f64 + ); + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs >= rhs) + .into_shared() + } + + pub(crate) fn greater_equal_elem(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_cmp_scalar_simd!( + E, + VecGreaterEq, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + f32, + i32, + u64, + i64, + f64 + ); + + lhs.mapv(|a| a >= rhs).into_shared() + } + + pub(crate) fn lower_equal(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_cmp_simd!( + E, VecLowerEq, lhs, rhs, u8, i8, u16, i16, u32, f32, i32, u64, i64, f64 + ); + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs <= rhs) + .into_shared() + } + + pub(crate) fn lower_equal_elem(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_cmp_scalar_simd!( + E, + VecLowerEq, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + f32, + i32, + u64, + i64, + f64 + ); + + lhs.mapv(|a| a <= rhs).into_shared() + } + + pub(crate) fn lower(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = dispatch_cmp_simd!( + E, VecLower, lhs, rhs, u8, i8, u16, i16, u32, f32, i32, u64, i64, f64 + ); + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs < rhs) + .into_shared() + } + + pub(crate) fn lower_elem(lhs: SharedArray, rhs: E) -> SharedArray { + let lhs = dispatch_cmp_scalar_simd!( + E, + VecLower, + lhs, + rhs.elem(), + u8, + i8, + u16, + i16, + u32, + f32, + i32, + u64, + i64, + f64 + ); + + lhs.mapv(|a| a < rhs).into_shared() + } +} + +pub struct NdArrayBitOps(PhantomData); + +impl NdArrayBitOps { + pub(crate) fn bitand(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = + dispatch_binary_simd!(I, VecBitAnd, lhs, rhs, i8, u8, i16, u16, i32, u32, i64, u64); + + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &I, b: &I| { + (a.elem::() & (b.elem::())).elem() + }) + } + + pub(crate) fn bitand_scalar(lhs: SharedArray, rhs: I) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!( + I, + VecBitAnd, + lhs, + rhs.elem(), + i8, + u8, + i16, + u16, + i32, + u32, + i64, + u64 + ); + + NdArrayMathOps::elementwise_op_scalar(lhs, |a: I| { + (a.elem::() & rhs.elem::()).elem() + }) + } + + pub(crate) fn bitor(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = + dispatch_binary_simd!(I, VecBitOr, lhs, rhs, i8, u8, i16, u16, i32, u32, i64, u64); + + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &I, b: &I| { + (a.elem::() | (b.elem::())).elem() + }) + } + + pub(crate) fn bitor_scalar(lhs: SharedArray, rhs: I) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!( + I, + VecBitOr, + lhs, + rhs.elem(), + i8, + u8, + i16, + u16, + i32, + u32, + i64, + u64 + ); + + NdArrayMathOps::elementwise_op_scalar(lhs, |a: I| { + (a.elem::() | rhs.elem::()).elem() + }) + } + + pub(crate) fn bitxor(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + let (lhs, rhs) = + dispatch_binary_simd!(I, VecBitXor, lhs, rhs, i8, u8, i16, u16, i32, u32, i64, u64); + + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &I, b: &I| { + (a.elem::() ^ (b.elem::())).elem() + }) + } + + pub(crate) fn bitxor_scalar(lhs: SharedArray, rhs: I) -> SharedArray { + let lhs = dispatch_binary_scalar_simd!( + I, + VecBitXor, + lhs, + rhs.elem(), + i8, + u8, + i16, + u16, + i32, + u32, + i64, + u64 + ); + + NdArrayMathOps::elementwise_op_scalar(lhs, |a: I| { + (a.elem::() ^ rhs.elem::()).elem() + }) + } + + pub(crate) fn bitnot(tensor: SharedArray) -> SharedArray { + let tensor = + dispatch_unary_simd!(I, VecBitNot, tensor, i8, u8, i16, u16, i32, u32, i64, u64); + + NdArrayMathOps::elementwise_op_scalar(tensor, |a: I| (!a.elem::()).elem()) + } +} + +pub struct NdArrayBoolOps; + +// Rust booleans are either `00000000` or `00000001`, so bitwise and/or is fine, but bitwise not would +// produce invalid values. +impl NdArrayBoolOps { + pub(crate) fn equal(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + #[cfg(feature = "simd")] + let (lhs, rhs) = match try_cmp_simd::(lhs, rhs) { + Ok(out) => return out, + Err(args) => args, + }; + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs == rhs) + .into_shared() + } + + pub(crate) fn equal_elem(lhs: SharedArray, rhs: bool) -> SharedArray { + #[cfg(feature = "simd")] + let lhs = match try_cmp_scalar_simd::(lhs, rhs.elem()) { + Ok(out) => return out, + Err(args) => args, + }; + + lhs.mapv(|a| a == rhs).into_shared() + } + + pub(crate) fn and(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + #[cfg(feature = "simd")] + let (lhs, rhs) = match try_binary_simd::(lhs, rhs) { + Ok(out) => return out, + Err(args) => args, + }; + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs && rhs) + .into_shared() + } + + pub(crate) fn or(lhs: SharedArray, rhs: SharedArray) -> SharedArray { + #[cfg(feature = "simd")] + let (lhs, rhs) = match try_binary_simd::(lhs, rhs) { + Ok(out) => return out, + Err(args) => args, + }; + + // Use the helper to broadcast both arrays to a common shape + let (lhs_broadcast, rhs_broadcast) = broadcast_for_comparison(&lhs, &rhs); + // Now we can safely zip and compare + Zip::from(&lhs_broadcast) + .and(&rhs_broadcast) + .map_collect(|&lhs, &rhs| lhs || rhs) + .into_shared() + } + + /// Any element is true - zero-copy for borrowed storage. + pub fn any_view(view: ArrayView<'_, bool, IxDyn>) -> bool { + view.iter().any(|&x| x) + } + + /// All elements are true - zero-copy for borrowed storage. + pub fn all_view(view: ArrayView<'_, bool, IxDyn>) -> bool { + view.iter().all(|&x| x) + } +} + +enum CmpType { + Min, + Max, +} + +fn arg( + tensor: SharedArray, + dim: usize, + cmp: CmpType, +) -> SharedArray { + arg_view(tensor.view(), dim, cmp) +} + +/// View-based argmax/argmin - zero-copy for borrowed storage. +fn arg_view( + view: ArrayView<'_, E, IxDyn>, + dim: usize, + cmp: CmpType, +) -> SharedArray { + let mut reshape = view.shape().to_vec(); + reshape[dim] = 1; + + let output = view.map_axis(Axis(dim), |arr| { + // Find the min/max value in the array, and return its index. + let (_e, idx) = arr.indexed_iter().fold((arr[0], 0usize), |acc, (idx, e)| { + let cmp = match cmp { + CmpType::Min => e < &acc.0, + CmpType::Max => e > &acc.0, + }; + + if cmp { (*e, idx) } else { acc } + }); + + (idx as i64).elem() + }); + + let output = output.to_shape(Dim(reshape.as_slice())).unwrap(); + + output.into_shared() +} + +#[cfg(test)] +mod tests { + use burn_backend::TensorData; + + use crate::NdArrayTensor; + + use super::*; + + #[test] + fn should_generate_row_major_layout_for_cat() { + let expected_shape: &[usize] = &[4, 6, 2]; + let expected_strides: &[isize] = &[12, 2, 1]; + let NdArrayTensor::I32(expected_storage) = NdArrayTensor::from_data(TensorData::from([ + [[1, 0], [2, 0], [3, 0], [4, 0], [5, 0], [6, 0]], + [[7, 0], [8, 0], [9, 0], [10, 0], [11, 0], [12, 0]], + [[13, 0], [14, 0], [15, 0], [16, 0], [17, 0], [18, 0]], + [[19, 0], [20, 0], [21, 0], [22, 0], [23, 0], [24, 0]], + ])) else { + panic!() + }; + let expected_array = expected_storage.into_shared(); + + let NdArrayTensor::I32(tensor_storage) = NdArrayTensor::from_data(TensorData::from([ + [1, 2, 3, 4, 5, 6], + [7, 8, 9, 10, 11, 12], + [13, 14, 15, 16, 17, 18], + [19, 20, 21, 22, 23, 24], + ])) else { + panic!() + }; + let tensor = tensor_storage.into_shared(); + + // unsqueeze dim on the outermost axis + let array = NdArrayOps::reshape(tensor, Shape::from([4, 6, 1])); + let NdArrayTensor::I32(zeros_storage) = + NdArrayTensor::from_data(TensorData::zeros::([4, 6, 1])) + else { + panic!() + }; + let zeros = zeros_storage.into_shared(); + // make `ndarray` concatenates array on the outermost axis + let array = NdArrayOps::cat([array, zeros].to_vec(), 2); + + assert!(array.is_standard_layout()); + assert_eq!(array.shape(), expected_shape); + assert_eq!(array.strides(), expected_strides); + assert_eq!( + array.into_iter().collect::>(), + expected_array.into_iter().collect::>(), + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/bool_tensor.rs new file mode 100644 index 0000000..7cca959 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/bool_tensor.rs @@ -0,0 +1,220 @@ +// Language +use alloc::vec; +use alloc::vec::Vec; +use burn_backend::Scalar; +use burn_backend::{ElementConversion, TensorMetadata, tensor::FloatTensor}; +use burn_backend::{ + backend::ExecutionError, + ops::BoolTensorOps, + tensor::{BoolTensor, IntTensor}, +}; +use ndarray::IntoDimension; + +// Current crate +use crate::element::{FloatNdArrayElement, IntNdArrayElement, QuantElement}; +use crate::{NdArray, execute_with_int_dtype, tensor::NdArrayTensor}; +use crate::{NdArrayDevice, SharedArray, slice}; + +// Workspace crates +use burn_backend::{Shape, TensorData, backend::Backend}; + +use super::{NdArrayBoolOps, NdArrayOps}; + +impl BoolTensorOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + fn bool_from_data(data: TensorData, _device: &NdArrayDevice) -> NdArrayTensor { + if !data.dtype.is_bool() { + unimplemented!("Unsupported dtype for `bool_from_data`") + } + NdArrayTensor::from_data(data) + } + + async fn bool_into_data(tensor: NdArrayTensor) -> Result { + Ok(tensor.into_data()) + } + + fn bool_to_device(tensor: NdArrayTensor, _device: &NdArrayDevice) -> NdArrayTensor { + tensor + } + + fn bool_reshape(tensor: NdArrayTensor, shape: Shape) -> NdArrayTensor { + NdArrayOps::reshape(tensor.bool(), shape).into() + } + + fn bool_slice(tensor: NdArrayTensor, slices: &[burn_backend::Slice]) -> NdArrayTensor { + slice!(tensor, slices) + } + + fn bool_into_int(tensor: NdArrayTensor) -> NdArrayTensor { + // Use mapv directly instead of collecting to Vec and going through TensorData + let int_array: SharedArray = tensor.bool().mapv(|b| b.elem()).into_shared(); + int_array.into() + } + + fn bool_device(_tensor: &NdArrayTensor) -> as Backend>::Device { + NdArrayDevice::Cpu + } + + fn bool_empty(shape: Shape, _device: & as Backend>::Device) -> NdArrayTensor { + Self::bool_zeros(shape, _device) + } + + fn bool_zeros(shape: Shape, _device: & as Backend>::Device) -> NdArrayTensor { + let values = vec![false; shape.num_elements()]; + NdArrayTensor::from_data(TensorData::new(values, shape)) + } + + fn bool_ones(shape: Shape, _device: & as Backend>::Device) -> NdArrayTensor { + let values = vec![true; shape.num_elements()]; + NdArrayTensor::from_data(TensorData::new(values, shape)) + } + + fn bool_slice_assign( + tensor: NdArrayTensor, + slices: &[burn_backend::Slice], + value: NdArrayTensor, + ) -> NdArrayTensor { + NdArrayOps::slice_assign(tensor.bool(), slices, value.bool()).into() + } + + fn bool_cat(tensors: Vec, dim: usize) -> NdArrayTensor { + NdArrayOps::cat(tensors.into_iter().map(|it| it.bool()).collect(), dim).into() + } + + fn bool_equal(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + NdArrayBoolOps::equal(lhs.bool(), rhs.bool()).into() + } + + fn bool_not(tensor: NdArrayTensor) -> NdArrayTensor { + tensor.bool().mapv(|a| !a).into_shared().into() + } + + fn bool_and(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + NdArrayBoolOps::and(lhs.bool(), rhs.bool()).into() + } + + fn bool_or(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + NdArrayBoolOps::or(lhs.bool(), rhs.bool()).into() + } + + fn bool_into_float(tensor: NdArrayTensor) -> FloatTensor { + let arr: SharedArray = tensor.bool().mapv(|b| b.elem()).into_shared(); + arr.into() + } + + fn bool_swap_dims(tensor: NdArrayTensor, dim1: usize, dim2: usize) -> NdArrayTensor { + NdArrayOps::swap_dims(tensor.bool(), dim1, dim2).into() + } + + fn bool_permute(tensor: NdArrayTensor, axes: &[usize]) -> NdArrayTensor { + tensor.bool().permuted_axes(axes.into_dimension()).into() + } + + fn bool_expand(tensor: NdArrayTensor, shape: Shape) -> NdArrayTensor { + NdArrayOps::expand(tensor.bool(), shape).into() + } + + fn bool_select(tensor: NdArrayTensor, dim: usize, indices: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!(indices, I, |indices: SharedArray| -> NdArrayTensor { + let tensor_bool = tensor.bool(); + let indices_vec: Vec = indices + .into_iter() + .map(|i| i.elem::() as usize) + .collect(); + + let selected = tensor_bool.select(ndarray::Axis(dim), &indices_vec); + selected.into_shared().into() + }) + } + + fn bool_select_or( + tensor: NdArrayTensor, + dim: usize, + indices: NdArrayTensor, + value: NdArrayTensor, + ) -> NdArrayTensor { + execute_with_int_dtype!(indices, I, |indices: SharedArray| -> NdArrayTensor { + let mut output_array = tensor.bool().into_owned(); + let value_bool = value.bool(); + + for (index_value, index) in indices.into_iter().enumerate() { + let index_usize = index.elem::() as usize; + let mut view = output_array.index_axis_mut(ndarray::Axis(dim), index_usize); + let value_slice = value_bool.index_axis(ndarray::Axis(dim), index_value); + // For boolean tensors, select_assign should use logical OR operation + view.zip_mut_with(&value_slice, |a, b| *a = *a || *b); + } + output_array.into_shared().into() + }) + } + + fn bool_flip(tensor: NdArrayTensor, axes: &[usize]) -> NdArrayTensor { + NdArrayOps::flip(tensor.bool(), axes).into() + } + + fn bool_unfold(tensor: NdArrayTensor, dim: usize, size: usize, step: usize) -> NdArrayTensor { + NdArrayOps::unfold(tensor.bool(), dim, size, step).into() + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + NdArrayOps::mask_where(tensor.bool(), mask.bool(), value.bool()).into() + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + NdArrayOps::mask_fill(tensor.bool(), mask.bool(), value.elem()).into() + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + execute_with_int_dtype!(indices, |indices| NdArrayOps::gather( + dim, + tensor.bool(), + indices + )) + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + execute_with_int_dtype!(indices, |indices| NdArrayOps::scatter( + dim, + tensor.bool(), + indices, + value.bool() + )) + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + NdArrayBoolOps::equal_elem(lhs.bool(), rhs.elem()).into() + } + + fn bool_any(tensor: BoolTensor) -> BoolTensor { + // Use view() for zero-copy on borrowed storage with short-circuit evaluation + let result = NdArrayBoolOps::any_view(tensor.bool().view()); + NdArrayTensor::from_data(TensorData::new(vec![result], Shape::new([1]))) + } + + fn bool_all(tensor: BoolTensor) -> BoolTensor { + // Use view() for zero-copy on borrowed storage with short-circuit evaluation + let result = NdArrayBoolOps::all_view(tensor.bool().view()); + NdArrayTensor::from_data(TensorData::new(vec![result], Shape::new([1]))) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/conv.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/conv.rs new file mode 100644 index 0000000..5fb2cad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/conv.rs @@ -0,0 +1,574 @@ +use burn_backend::{ + ElementConversion, + ops::{ + ConvOptions, ConvTransposeOptions, + conv::{calculate_conv_output_size, calculate_conv_transpose_output_size}, + }, +}; +use ndarray::{ + Array3, Array4, Array5, ArrayView2, ArrayView3, ArrayViewMut2, ArrayViewMut3, Axis, Dim, s, +}; + +use crate::{ + NdArrayElement, SharedArray, iter_par, iter_range_par, + ops::padding::{apply_padding_4d, apply_padding_5d}, + run_par, + sharing::UnsafeSharedRef, + tensor::NdArrayTensor, +}; + +#[inline(always)] +fn conv2d_mad_inner( + mut output: ArrayViewMut2, + x: ArrayView2, + k: E, + k_xy: (usize, usize), + out_xy: (usize, usize), + stride: (usize, usize), + dilation: (usize, usize), +) { + let (kh, kw) = k_xy; + let (out_width, out_height) = out_xy; + let (stride_width, stride_height) = stride; + let (dilation_width, dilation_height) = dilation; + + for oh in 0..out_height { + // Construct a sub-slice view of the input row. + // This is done upfront so that rustc does not have to emit bounds checks + // in the hot loop below. + let ir = x + .row(oh * stride_height + kh * dilation_height) + .to_slice() + .unwrap(); + + // Ditto. Construct a sub-slice view of the output row, and explicitly specify + // the bounds upfront as 0..out_width so that rustc can make the assumption + // that all accesses are in-bounds in the below loop. + let mut or = output.row_mut(oh); + let or = &mut or.as_slice_mut().unwrap()[0..out_width]; + + #[allow(clippy::needless_range_loop)] + for ow in 0..out_width { + let iw = ow * stride_width + kw * dilation_width; + or[ow] += ir[iw] * k; + } + } +} + +#[inline(always)] +fn conv3d_mad_inner( + mut output: ArrayViewMut3, + x: ArrayView3, + k: E, + k_xyz: (usize, usize, usize), + out_xyz: (usize, usize, usize), + stride: (usize, usize, usize), + dilation: (usize, usize, usize), +) { + let (kd, kh, kw) = k_xyz; + let (out_width, out_height, out_depth) = out_xyz; + let (stride_width, stride_height, stride_depth) = stride; + let (dilation_width, dilation_height, dilation_depth) = dilation; + + for od in 0..out_depth { + let id = od * stride_depth + kd * dilation_depth; + + for oh in 0..out_height { + let ih = oh * stride_height + kh * dilation_height; + + // Construct a sub-slice view of the input row. + // This is done upfront so that rustc does not have to emit bounds checks + // in the hot loop below. + let ir = x.slice(s![id, ih, ..]).to_slice().unwrap(); + + // Ditto. Construct a sub-slice view of the output row, and explicitly specify + // the bounds upfront as 0..out_width so that rustc can make the assumption + // that all accesses are in-bounds in the below loop. + let or = &mut output + .slice_mut(s![od, oh, 0..out_width]) + .into_slice() + .unwrap()[0..out_width]; + + #[allow(clippy::needless_range_loop)] + for ow in 0..out_width { + let iw = ow * stride_width + kw * dilation_width; + or[ow] += ir[iw] * k; + } + } + } +} + +pub(crate) fn conv2d( + x: SharedArray, + weight: SharedArray, + bias: Option>, + options: ConvOptions<2>, +) -> SharedArray +where + NdArrayTensor: From>, +{ + let [dilation_height, dilation_width] = options.dilation; + let [padding_height, padding_width] = options.padding; + let [stride_height, stride_width] = options.stride; + let [batch_size, _in_channels, in_height, in_width] = x.shape().try_into().unwrap(); + let [out_channels, in_channels, kernel_height, kernel_width] = + weight.shape().try_into().unwrap(); + let channels_per_group = out_channels / options.groups; + + let out_height = calculate_conv_output_size( + kernel_height, + stride_height, + padding_height, + dilation_height, + in_height, + ); + let out_width = calculate_conv_output_size( + kernel_width, + stride_width, + padding_width, + dilation_width, + in_width, + ); + + let x = apply_padding_4d::(x, options.padding, 0i32.elem()); + + // Convert inputs from dynamic indexes to static to improve perf. + let x = x.into_dimensionality::().unwrap(); + let weights = weight.into_dimensionality::().unwrap(); + + let mut output = Array3::zeros(Dim([batch_size * out_channels, out_height, out_width])); + + run_par!(|| { + iter_par!(output.axis_iter_mut(Axis(0))) + .enumerate() + .for_each( + #[inline(never)] + |(k, mut output)| { + let b = k / out_channels; + let oc = k % out_channels; + let g = oc / channels_per_group; + + for ic in (in_channels * g)..(in_channels * (g + 1)) { + let weight_ic = ic - (g * in_channels); + + let x = x.slice(s![b, ic, .., ..]); + let k = weights.slice(s![oc, weight_ic, .., ..]); + + for kh in 0..kernel_height { + for kw in 0..kernel_width { + let k = k[[kh, kw]]; + + // NOTE: This function call is duplicated twice so that the compiler can perform auto-vectorization + // in the case that the stride/dilation is 1. + #[allow(clippy::if_same_then_else)] + if (1, 1, 1, 1) + == ( + stride_width, + stride_height, + dilation_width, + dilation_height, + ) + { + conv2d_mad_inner( + output.view_mut(), + x.view(), + k, + (kh, kw), + (out_width, out_height), + (stride_width, stride_height), + (dilation_width, dilation_height), + ); + } else { + conv2d_mad_inner( + output.view_mut(), + x.view(), + k, + (kh, kw), + (out_width, out_height), + (stride_width, stride_height), + (dilation_width, dilation_height), + ); + } + } + } + } + + if let Some(bias) = &bias { + let bias = bias[oc]; + + for oh in 0..out_height { + // Get a mutable slice reference to the row we're looping over. + // We explicitly define the bounds to 0..out_width so that rustc can make + // the assumption that all accesses are in-bounds. + let mut or = output.row_mut(oh); + let or = &mut or.as_slice_mut().unwrap()[0..out_width]; + + #[allow(clippy::needless_range_loop)] + for ow in 0..out_width { + or[ow] += bias; + } + } + } + }, + ); + }); + + output + .to_shape([batch_size, out_channels, out_height, out_width]) + .unwrap() + .into_dyn() + .into_shared() +} + +pub(crate) fn conv_transpose2d( + x: SharedArray, + weight: SharedArray, + bias: Option>, + options: ConvTransposeOptions<2>, +) -> SharedArray { + let [dilation_height, dilation_width] = options.dilation; + let [padding_height, padding_width] = options.padding; + let [stride_height, stride_width] = options.stride; + let [out_padding_height, out_padding_width] = options.padding_out; + let [batch_size, _in_channels, in_height, in_width] = x.shape().try_into().unwrap(); + let [in_channels, out_channels, kernel_height, kernel_width] = + weight.shape().try_into().unwrap(); + + let out_height = calculate_conv_transpose_output_size( + kernel_height, + stride_height, + padding_height, + out_padding_height, + dilation_height, + in_height, + ); + let out_width = calculate_conv_transpose_output_size( + kernel_width, + stride_width, + padding_width, + out_padding_width, + dilation_width, + in_width, + ); + + let x = x; + let mut output = Array4::zeros(Dim([ + batch_size, + out_channels * options.groups, + out_height, + out_width, + ])); + + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, batch_size * out_channels * options.groups).for_each(|k| unsafe { + let b = k / (out_channels * options.groups); + let oc = k % out_channels; + let g = (k / out_channels) % options.groups; + + let output = unsafe_shared_out.get(); + + let oc_out = oc + (out_channels * g); + let ic_start = g * (in_channels / options.groups); + let ic_end = ic_start + in_channels / options.groups; + + for ic in ic_start..ic_end { + for ih in 0..in_height { + for iw in 0..in_width { + for kh in 0..kernel_height { + for kw in 0..kernel_width { + let oh = ih * stride_height + kh * dilation_height; + let ow = iw * stride_width + kw * dilation_width; + + if oh >= out_height + padding_height + || ow >= out_width + padding_width + || oh < padding_height + || ow < padding_width + { + continue; + } + + let oh = oh - padding_height; + let ow = ow - padding_width; + + output[[b, oc_out, oh, ow]] += + x[[b, ic, ih, iw]] * weight[[ic, oc, kh, kw]]; + } + } + } + } + } + + if let Some(bias) = &bias { + for oh in 0..out_height { + for ow in 0..out_width { + output[[b, oc_out, oh, ow]] += bias[oc_out]; + } + } + } + }); + }); + + output.into_dyn().into_shared() +} + +pub(crate) fn conv3d( + x: SharedArray, + weight: SharedArray, + bias: Option>, + options: ConvOptions<3>, +) -> SharedArray +where + NdArrayTensor: From>, +{ + let [dilation_depth, dilation_height, dilation_width] = options.dilation; + let [padding_depth, padding_height, padding_width] = options.padding; + let [stride_depth, stride_height, stride_width] = options.stride; + let [batch_size, _in_channels, in_depth, in_height, in_width] = x.shape().try_into().unwrap(); + let [ + out_channels, + in_channels, + kernel_depth, + kernel_height, + kernel_width, + ] = weight.shape().try_into().unwrap(); + let out_c_per_group = out_channels / options.groups; + + let out_depth = calculate_conv_output_size( + kernel_depth, + stride_depth, + padding_depth, + dilation_depth, + in_depth, + ); + let out_height = calculate_conv_output_size( + kernel_height, + stride_height, + padding_height, + dilation_height, + in_height, + ); + let out_width = calculate_conv_output_size( + kernel_width, + stride_width, + padding_width, + dilation_width, + in_width, + ); + + let x = apply_padding_5d::(x, options.padding, 0i32.elem()); + + // Convert inputs from dynamic indexes to static to improve perf. + let x = x.into_dimensionality::().unwrap(); + let weights = weight.into_dimensionality::().unwrap(); + + let mut output = Array4::zeros(Dim([ + batch_size * out_channels, + out_depth, + out_height, + out_width, + ])); + + run_par!(|| { + iter_par!(output.axis_iter_mut(Axis(0))) + .enumerate() + .for_each( + #[inline(never)] + |(k, mut output)| { + let b = k / out_channels; + let oc = k % out_channels; + let g = oc / out_c_per_group; + + for ic in (in_channels * g)..(in_channels * (g + 1)) { + let weight_ic = ic - (g * in_channels); + + let x = x.slice(s![b, ic, .., .., ..]); + let k = weights.slice(s![oc, weight_ic, .., .., ..]); + + for kd in 0..kernel_depth { + for kh in 0..kernel_height { + for kw in 0..kernel_width { + let k = k[[kd, kh, kw]]; + + // NOTE: This function call is duplicated twice so that the compiler can perform auto-vectorization + // in the case that the stride/dilation is 1. + #[allow(clippy::if_same_then_else)] + if (1, 1, 1, 1, 1, 1) + == ( + stride_width, + stride_height, + stride_depth, + dilation_width, + dilation_height, + dilation_depth, + ) + { + conv3d_mad_inner( + output.view_mut(), + x.view(), + k, + (kd, kh, kw), + (out_width, out_height, out_depth), + (stride_width, stride_height, stride_depth), + (dilation_width, dilation_height, dilation_depth), + ); + } else { + conv3d_mad_inner( + output.view_mut(), + x.view(), + k, + (kd, kh, kw), + (out_width, out_height, out_depth), + (stride_width, stride_height, stride_depth), + (dilation_width, dilation_height, dilation_depth), + ); + } + } + } + } + } + + if let Some(bias) = &bias { + let bias = bias[oc]; + + // Get a mutable iterator to the row we're looping over. + let orows = output.rows_mut(); + for mut or in orows { + // We explicitly define the bounds to 0..out_width so that rustc can make + // the assumption that all accesses are in-bounds. + let or = &mut or.as_slice_mut().unwrap()[0..out_width]; + + #[allow(clippy::needless_range_loop)] + for ow in 0..out_width { + or[ow] += bias; + } + } + } + }, + ); + }); + + output + .to_shape([batch_size, out_channels, out_depth, out_height, out_width]) + .unwrap() + .into_dyn() + .into_shared() +} + +pub(crate) fn conv_transpose3d( + x: SharedArray, + weight: SharedArray, + bias: Option>, + options: ConvTransposeOptions<3>, +) -> SharedArray { + let [dilation_depth, dilation_height, dilation_width] = options.dilation; + let [padding_depth, padding_height, padding_width] = options.padding; + let [stride_depth, stride_height, stride_width] = options.stride; + let [out_padding_depth, out_padding_height, out_padding_width] = options.padding_out; + let [batch_size, _in_channels, in_depth, in_height, in_width] = x.shape().try_into().unwrap(); + let [ + in_channels, + out_channels, + kernel_depth, + kernel_height, + kernel_width, + ] = weight.shape().try_into().unwrap(); + + let out_depth = calculate_conv_transpose_output_size( + kernel_depth, + stride_depth, + padding_depth, + out_padding_depth, + dilation_depth, + in_depth, + ); + let out_height = calculate_conv_transpose_output_size( + kernel_height, + stride_height, + padding_height, + out_padding_height, + dilation_height, + in_height, + ); + let out_width = calculate_conv_transpose_output_size( + kernel_width, + stride_width, + padding_width, + out_padding_width, + dilation_width, + in_width, + ); + + let x = x; + let mut output = Array5::zeros(Dim([ + batch_size, + out_channels * options.groups, + out_depth, + out_height, + out_width, + ])); + + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, batch_size * out_channels * options.groups).for_each(|k| unsafe { + let b = k / (out_channels * options.groups); + let oc = k % out_channels; + let g = (k / out_channels) % options.groups; + + let output = unsafe_shared_out.get(); + + let oc_out = oc + (out_channels * g); + let ic_start = g * (in_channels / options.groups); + let ic_end = ic_start + in_channels / options.groups; + + for ic in ic_start..ic_end { + for id in 0..in_depth { + for ih in 0..in_height { + for iw in 0..in_width { + for kd in 0..kernel_depth { + for kh in 0..kernel_height { + for kw in 0..kernel_width { + let od = id * stride_depth + kd * dilation_depth; + let oh = ih * stride_height + kh * dilation_height; + let ow = iw * stride_width + kw * dilation_width; + + if od >= out_depth + padding_depth + || oh >= out_height + padding_height + || ow >= out_width + padding_width + || od < padding_depth + || oh < padding_height + || ow < padding_width + { + continue; + } + + let od = od - padding_depth; + let oh = oh - padding_height; + let ow = ow - padding_width; + + output[[b, oc_out, od, oh, ow]] += + x[[b, ic, id, ih, iw]] * weight[[ic, oc, kd, kh, kw]]; + } + } + } + } + } + } + } + + if let Some(bias) = &bias { + for od in 0..out_depth { + for oh in 0..out_height { + for ow in 0..out_width { + output[[b, oc_out, od, oh, ow]] += bias[oc_out]; + } + } + } + } + }); + }); + + output.into_dyn().into_shared() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/deform_conv.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/deform_conv.rs new file mode 100644 index 0000000..390010b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/deform_conv.rs @@ -0,0 +1,662 @@ +use burn_backend::ops::{DeformConvOptions, conv::calculate_conv_output_size}; +use core::ops::AddAssign; +use ndarray::{ + Array2, Array4, ArrayView2, ArrayView3, ArrayView4, ArrayView6, ArrayViewMut2, Axis, Dim, Ix4, + Zip, s, +}; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +use crate::{FloatNdArrayElement, NdArrayTensor, ShapeOps, SharedArray, iter_par, run_par}; + +use super::matmul::matmul; + +#[inline(always)] +#[allow(clippy::too_many_arguments)] +fn deform_im2col_kernel( + out_y: usize, + out_x: usize, + input: ArrayView2, + offset: ArrayView3, + mask: Option>, + mut columns: ArrayViewMut2, + args: DeformConvOptions<2>, + (kernel_h, kernel_w): (usize, usize), +) { + // position shape: [in_channels, batch_size, out_h, out_w] + // columns shape: [[in_channels, kernel_h, kernel_w], [batch_size, out_h, out_w]] + + let (height, width) = input.dim(); + + for kernel_y in 0..kernel_h { + for kernel_x in 0..kernel_w { + let mask_value = mask + .map(|it| it[[kernel_y, kernel_x]]) + .unwrap_or_else(|| F::from_elem(1.0)); + + let offset = offset.slice(s![kernel_y, kernel_x, ..]); + let y = F::from_elem(out_y * args.stride[0] + kernel_y * args.dilation[0]) + - F::from_elem(args.padding[0]) + + offset[0]; + let x = F::from_elem(out_x * args.stride[1] + kernel_x * args.dilation[1]) + - F::from_elem(args.padding[1]) + + offset[1]; + + let interpolated = bilinear_interpolate(input, height, width, y, x); + + columns[[kernel_y, kernel_x]] = mask_value * interpolated; + } + } +} + +fn bilinear_interpolate( + input: ArrayView2, + height: usize, + width: usize, + y: F, + x: F, +) -> F { + // To simplify code + let y = y.to_f32(); + let x = x.to_f32(); + + let mut result = F::from_elem(0.0); + if y > -1.0 && height as f32 > y && x > -1.0 && width as f32 > x { + let y_low = f32::floor(y); + let x_low = f32::floor(x); + let y_high = (y_low + 1.) as usize; + let x_high = (x_low + 1.) as usize; + + let zero = F::from_elem(0.0); + let v1: F = if y_low >= 0. && x_low >= 0. { + input[[y_low as usize, x_low as usize]] + } else { + zero + }; + let v2: F = if y_low >= 0. && x_high < width { + input[[y_low as usize, x_high]] + } else { + zero + }; + let v3: F = if y_high < height && x_low >= 0. { + input[[y_high, x_low as usize]] + } else { + zero + }; + let v4: F = if y_high < height && x_high < width { + input[[y_high, x_high]] + } else { + zero + }; + + let l_y = y - y_low; + let l_x = x - x_low; + let h_y = 1.0 - l_y; + let h_x = 1.0 - l_x; + + let w1 = F::from_elem(h_y * h_x); + let w2 = F::from_elem(h_y * l_x); + let w3 = F::from_elem(l_y * h_x); + let w4 = F::from_elem(l_y * l_x); + + result = w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4; + } + result +} + +pub(crate) fn deform_conv2d( + input: SharedArray, + offset: SharedArray, + weight: SharedArray, + mask: Option>, + bias: Option>, + args: DeformConvOptions<2>, +) -> SharedArray +where + NdArrayTensor: From>, +{ + let [batch_size, _, in_height, in_width] = input.shape().dims(); + let [out_channels, _, kernel_h, kernel_w] = weight.shape().dims(); + let groups = args.weight_groups; + + let weight = weight.as_standard_layout(); + + let out_h = calculate_conv_output_size( + kernel_h, + args.stride[0], + args.padding[0], + args.dilation[0], + in_height, + ); + let out_w = calculate_conv_output_size( + kernel_w, + args.stride[1], + args.padding[1], + args.dilation[1], + in_width, + ); + let out_dims = (out_h, out_w); + + let input = input.into_dimensionality::().unwrap(); + let offset = offset.into_dimensionality::().unwrap(); + let mask = mask.as_ref().map(|it| { + it.to_shape(( + batch_size, + args.offset_groups, + kernel_h, + kernel_w, + out_h, + out_w, + )) + .unwrap() + }); + + let columns = deform_im2col( + input.view(), + offset.view(), + mask.as_ref().map(|it| it.view()), + args, + out_dims, + (kernel_h, kernel_w), + ); + + let (col_size_0, col_size_1) = columns.dim(); + let col_size_0 = col_size_0 / groups; + let out_c_per_group = out_channels / groups; + + let weight = weight + .to_shape((groups, out_c_per_group, col_size_0)) + .unwrap(); + let columns = columns.to_shape((groups, col_size_0, col_size_1)).unwrap(); + let out = matmul( + weight.to_owned().into_dyn().into_shared(), + columns.to_owned().into_dyn().into_shared(), + ); + + let mut out = out + .into_shape_with_order((out_channels, batch_size, out_h, out_w)) + .unwrap(); + out.swap_axes(0, 1); + + if let Some(bias) = bias { + let bias = bias.to_shape((1, out_channels, 1, 1)).unwrap(); + out.add_assign(&bias); + } + + out.into_dyn().into_shared() +} + +pub(crate) fn deform_im2col( + input: ArrayView4, + offset: ArrayView4, + mask: Option>, + args: DeformConvOptions<2>, + out_dims: (usize, usize), + kernel_dims: (usize, usize), +) -> Array2 { + let (batch_size, in_channels, _, _) = input.dim(); + let (kernel_h, kernel_w) = kernel_dims; + let (out_h, out_w) = out_dims; + let channels_per_offset_group = in_channels / args.offset_groups; + + let mut columns = Array4::zeros(Dim([ + in_channels, + kernel_h, + kernel_w, + batch_size * out_h * out_w, + ])); + + let groups = args.offset_groups; + + run_par!(|| { + iter_par!(columns.axis_iter_mut(Axis(3))) + .enumerate() + .for_each(|(index, mut columns)| { + let out_x = index % out_w; + let out_y = (index / out_w) % out_h; + let batch = (index / (out_w * out_h)) % batch_size; + let offset = offset.slice(s![batch, .., out_y, out_x]); + let offset = offset.to_shape((groups, kernel_h, kernel_w, 2)).unwrap(); + let mask = mask + .as_ref() + .map(|it| it.slice(s![batch, .., .., .., out_y, out_x])); + columns + .axis_iter_mut(Axis(0)) + .enumerate() + .for_each(|(in_channel, mut columns)| { + let group_index = in_channel / channels_per_offset_group; + deform_im2col_kernel( + out_y, + out_x, + input.slice(s![batch, in_channel, .., ..]), + offset.slice(s![group_index, .., .., ..]), + mask.as_ref().map(|it| it.slice(s![group_index, .., ..])), + columns.view_mut(), + args.clone(), + kernel_dims, + ); + }); + }); + }); + + columns + // Columns is created here, so we know it's contiguous + .into_shape_with_order(( + in_channels * kernel_h * kernel_w, + batch_size * out_h * out_w, + )) + .unwrap() +} + +pub mod backward { + #[cfg(target_has_atomic = "32")] + use core::sync::atomic::Ordering; + + use atomic_float::AtomicF32; + use ndarray::{Array1, Array5, ArrayView4, ArrayView6, Ix4}; + + use super::*; + + pub(crate) type DeformConv2dBackward = ( + SharedArray, + SharedArray, + SharedArray, + Option>, + Option>, + ); + + /// Calculate the [deformable 2D convolution](crate::ops::ModuleOps::deform_conv2d) backward pass using convolutions. + pub(crate) fn deform_conv2d_backward( + input: SharedArray, + offset: SharedArray, + weight: SharedArray, + mask: Option>, + bias: Option>, + out_grad: SharedArray, + args: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + let [batch_size, out_channels, out_h, out_w] = out_grad.shape().dims(); + let [_, _, kernel_h, kernel_w] = weight.shape().dims(); + let groups = args.weight_groups; + let out_c_per_group = out_channels / groups; + let col_shape_1 = batch_size * out_h * out_w; + let mut out_grad = out_grad.into_dimensionality::().unwrap(); + + let gradient_bias = bias.map(|_| { + let out_grad = out_grad + .clone() + .sum_axis(Axis(0)) + .sum_axis(Axis(1)) + .sum_axis(Axis(1)); + + out_grad.into_dyn().into_shared() + }); + + out_grad.swap_axes(0, 1); + let out_grad = out_grad + .to_shape((groups, out_c_per_group, col_shape_1)) + .unwrap(); + + let input = input.into_dimensionality::().unwrap(); + let offset = offset.into_dimensionality::().unwrap(); + let mask = mask.map(|it| { + it.into_shape_with_order(( + batch_size, + args.offset_groups, + kernel_h, + kernel_w, + out_h, + out_w, + )) + .unwrap() + }); + + let (input_gradient, offset_gradient, mask_gradient) = backward_gradient_inputs( + input.view(), + weight, + offset.view(), + mask.as_ref().map(|it| it.view()), + out_grad.view(), + &args, + (kernel_h, kernel_w), + ); + + let weight_grad = compute_weight_grad( + input.view(), + offset.view(), + mask.as_ref().map(|it| it.view()), + out_grad.view(), + args, + (kernel_h, kernel_w), + (out_h, out_w), + ); + + ( + input_gradient, + offset_gradient, + weight_grad, + mask_gradient, + gradient_bias, + ) + } + + fn compute_weight_grad( + input: ArrayView4, + offset: ArrayView4, + mask: Option>, + out_grad: ArrayView3, + options: DeformConvOptions<2>, + kernel_dims: (usize, usize), + out_dims: (usize, usize), + ) -> SharedArray { + let in_channels = input.dim().1; + let (groups, out_c_per_group, _) = out_grad.dim(); + let (kernel_h, kernel_w) = kernel_dims; + + let in_c_per_group = in_channels / groups; + + let columns = deform_im2col(input, offset, mask, options, out_dims, kernel_dims); + let (col_size_0, col_size_1) = columns.dim(); + let col_size_0 = col_size_0 / groups; + + let mut columns = columns.to_shape((groups, col_size_0, col_size_1)).unwrap(); + columns.swap_axes(1, 2); + + let grad_weight = matmul( + out_grad.to_owned().into_dyn().into_shared(), + columns.to_owned().into_dyn().into_shared(), + ); + + let grad_weight = grad_weight + .into_shape_with_order((out_c_per_group * groups, in_c_per_group, kernel_h, kernel_w)) + .unwrap(); + grad_weight.into_dyn().into_shared() + } + + type InputGradients = (SharedArray, SharedArray, Option>); + + fn backward_gradient_inputs( + image: ArrayView4, + weight: SharedArray, + offset: ArrayView4, + mask: Option>, + out_grad: ArrayView3, + args: &DeformConvOptions<2>, + kernel_dims: (usize, usize), + ) -> InputGradients { + let input_shape = image.dim(); + let in_channels = input_shape.1; + let [out_channels, in_c_per_group, kernel_h, kernel_w] = weight.shape().dims(); + let (batch_size, _, out_h, out_w) = offset.dim(); + + let groups = args.weight_groups; + let out_c_per_group = out_channels / groups; + + let col_shape_0 = in_c_per_group * kernel_h * kernel_w; + + let mut weight = weight + .to_shape((groups, out_c_per_group, col_shape_0)) + .unwrap(); + weight.swap_axes(1, 2); + let columns = matmul( + weight.to_owned().into_dyn().into_shared(), + out_grad.to_owned().into_dyn().into_shared(), + ); + + let columns = columns + .to_shape((in_channels, kernel_h, kernel_w, batch_size, out_h, out_w)) + .unwrap(); + + let (offset_gradient, mask_gradient) = compute_offset_and_mask_gradient( + columns.view(), + image.view(), + offset, + mask, + args, + kernel_dims, + ); + + let input_gradient = + compute_input_grad(columns.view(), offset, mask, args, kernel_dims, input_shape); + + (input_gradient, offset_gradient, mask_gradient) + } + + fn compute_offset_and_mask_gradient( + columns: ArrayView6, + image: ArrayView4, + offset: ArrayView4, + mask: Option>, + args: &DeformConvOptions<2>, + kernel_dims: (usize, usize), + ) -> (SharedArray, Option>) { + let (kernel_h, kernel_w) = kernel_dims; + let (_, in_channels, height, width) = image.dim(); + let (batch_size, offset_channels, out_h, out_w) = offset.dim(); + let offs_groups = args.offset_groups; + let channels_per_offset_group = in_channels / args.offset_groups; + + let mut grad_offset = Array5::zeros(( + offs_groups, + kernel_h, + kernel_w, + 2, + batch_size * out_h * out_w, + )); + let mut grad_mask = + Array4::zeros((offs_groups, kernel_h, kernel_w, batch_size * out_h * out_w)); + + grad_mask + .axis_iter_mut(Axis(3)) + .zip(grad_offset.axis_iter_mut(Axis(4))) + .enumerate() + .for_each(|(index, (mut grad_mask, mut grad_offset))| { + let out_x = index % out_w; + let out_y = (index / out_w) % out_h; + let batch = index / (out_w * out_h); + let offset = offset.slice(s![batch, .., out_y, out_x]); + let offset = offset + .to_shape((offs_groups, kernel_h, kernel_w, 2)) + .unwrap(); + let mask: Option> = mask + .as_ref() + .map(|mask| mask.slice(s![batch, .., .., .., out_y, out_x])); + let columns = columns.slice(s![.., .., .., batch, out_y, out_x]); + let image = image.slice(s![batch, .., .., ..]); + + for ((group, kernel_y, kernel_x), grad_mask) in grad_mask.indexed_iter_mut() { + let grad_mask: &mut F = grad_mask; + let mut grad_offset = grad_offset.slice_mut(s![group, kernel_y, kernel_x, ..]); + let offset = offset.slice(s![group, kernel_y, kernel_x, ..]); + let mask = mask.map(|it| it[[group, kernel_y, kernel_x]]); + let columns = columns.slice(s![.., kernel_y, kernel_x]); + let group_offset = group * channels_per_offset_group; + let image = image.slice(s![group_offset.., .., ..]); + let y = F::from_elem(out_y * args.stride[0] + kernel_y * args.dilation[0]) + - F::from_elem(args.padding[0]) + + offset[0]; + let x = F::from_elem(out_x * args.stride[1] + kernel_x * args.dilation[1]) + - F::from_elem(args.padding[1]) + + offset[1]; + for (i, grad_offset) in grad_offset.iter_mut().enumerate() { + let is_y_direction = i % 2 == 0; + let use_mask = mask.is_some(); + + for channel in 0..channels_per_offset_group { + let mask = mask.unwrap_or_else(|| F::one()); + let image = image.index_axis(Axis(0), channel); + let weight = + get_coordinate_weight(image, height, width, y, x, is_y_direction); + *grad_offset += mask * weight * columns[channel]; + if use_mask && is_y_direction { + *grad_mask += columns[channel] + * bilinear_interpolate(image, height, width, y, x); + } + } + } + } + }); + + let mask_gradient = mask.map(|_| { + let mut grad_mask = grad_mask + .into_shape_with_order((offset_channels / 2, batch_size, out_h, out_w)) + .unwrap(); + grad_mask.swap_axes(0, 1); + grad_mask.into_dyn().into_shared() + }); + let mut grad_offset = grad_offset + .into_shape_with_order((offset_channels, batch_size, out_h, out_w)) + .unwrap(); + grad_offset.swap_axes(0, 1); + let offset_gradient = grad_offset.into_dyn().into_shared(); + (offset_gradient, mask_gradient) + } + + fn get_coordinate_weight( + input: ArrayView2, + height: usize, + width: usize, + y: F, + x: F, + is_y_direction: bool, + ) -> F { + let y = y.to_f32(); + let x = x.to_f32(); + + let y_low = f32::floor(y); + let x_low = f32::floor(x); + let y_high = y_low + 1.; + let x_high = x_low + 1.; + + let valid_y_low = y_low >= 0. && y_low < height as f32; + let valid_y_high = y_high >= 0. && y_high < height as f32; + let valid_x_low = x_low >= 0. && x_low < width as f32; + let valid_x_high = x_high >= 0. && x_high < width as f32; + + let bottom_left = if valid_y_low && valid_x_low { + input[[y_low as usize, x_low as usize]] + } else { + F::zero() + }; + let bottom_right = if valid_y_low && valid_x_high { + input[[y_low as usize, x_high as usize]] + } else { + F::zero() + }; + let top_left = if valid_y_high && valid_x_low { + input[[y_high as usize, x_low as usize]] + } else { + F::zero() + }; + let top_right = if valid_y_high && valid_x_high { + input[[y_high as usize, x_high as usize]] + } else { + F::zero() + }; + + if is_y_direction { + let delta_x = F::from_elem(x - x_low); + delta_x * (top_right - bottom_right) + (F::one() - delta_x) * (top_left - bottom_left) + } else { + let delta_y = F::from_elem(y - y_low); + delta_y * (top_right - top_left) + (F::one() - delta_y) * (bottom_right - bottom_left) + } + } + + fn compute_input_grad( + columns: ArrayView6, + offset: ArrayView4, + mask: Option>, + args: &DeformConvOptions<2>, + kernel_dims: (usize, usize), + input_shape: (usize, usize, usize, usize), + ) -> SharedArray { + let (batch_size, in_channels, height, width) = input_shape; + let (kernel_h, kernel_w) = kernel_dims; + let offs_groups = args.offset_groups; + let channels_per_offset_group = in_channels / offs_groups; + + let grad_in = + Array4::from_shape_simple_fn((batch_size, in_channels, height, width), || { + AtomicF32::new(0.0) + }); + + let compute_for_each = |(in_channel, kernel_y, kernel_x, batch, out_y, out_x), col: &F| { + let group = in_channel / channels_per_offset_group; + let offset = offset.slice(s![batch, .., out_y, out_x]); + let offset = offset + .to_shape((offs_groups, kernel_h, kernel_w, 2)) + .unwrap(); + let offset = offset.slice(s![group, kernel_y, kernel_x, ..]); + let offset = [offset[0], offset[1]]; + let mask = mask + .as_ref() + .map(|it| it[[batch, group, kernel_y, kernel_x, out_y, out_x]].to_f32()); + let y = F::from_elem(out_y * args.stride[0] + kernel_y * args.dilation[0]) + - F::from_elem(args.padding[0]) + + offset[0]; + let x = F::from_elem(out_x * args.stride[1] + kernel_x * args.dilation[1]) + - F::from_elem(args.padding[1]) + + offset[1]; + let grad_in = grad_in.slice(s![batch, in_channel, .., ..]); + deform_col2img_kernel(y.to_f32(), x.to_f32(), mask, col.to_f32(), grad_in); + }; + + // `for_each` expects a 2-tuple argument with `.into_par_iter()`, but 2 separate arguments otherwise + #[cfg(feature = "multi-threads")] + run_par!(|| { + iter_par!(Zip::indexed(columns)) + .for_each(|(args0, args1)| compute_for_each(args0, args1)) + }); + + #[cfg(not(feature = "multi-threads"))] + run_par!(|| { iter_par!(Zip::indexed(columns)).for_each(&compute_for_each) }); + + let grad_in: Array1 = grad_in + .into_iter() + .map(|it| F::from_elem(it.into_inner())) + .collect(); + let grad_in = grad_in + .into_shape_with_order((batch_size, in_channels, height, width)) + .unwrap(); + grad_in.into_dyn().into_shared() + } + + fn deform_col2img_kernel( + y: f32, + x: f32, + mask: Option, + col: f32, + grad_input: ArrayView2, + ) { + let (height, width) = grad_input.dim(); + let mask_value = mask.unwrap_or(1.0); + + for dy in -1..=1 { + for dx in -1..=1 { + let yp = f32::floor(y) + dy as f32; + let xp = f32::floor(x) + dx as f32; + + if yp >= 0.0 + && yp < height as f32 + && xp >= 0.0 + && xp < width as f32 + && f32::abs(y - yp) < 1.0 + && f32::abs(x - xp) < 1.0 + { + let weight = (1.0 - f32::abs(y - yp)) * (1.0 - f32::abs(x - xp)); + + #[cfg_attr(not(target_has_atomic = "32"), allow(unused))] + let value = mask_value * weight * col; + + #[cfg(target_has_atomic = "32")] + grad_input[[yp as usize, xp as usize]].fetch_add(value, Ordering::AcqRel); + #[cfg(not(target_has_atomic = "32"))] + panic!("Can't use deformable convolution backwards pass without atomics"); + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/grid_sample.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/grid_sample.rs new file mode 100644 index 0000000..256c2fd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/grid_sample.rs @@ -0,0 +1,214 @@ +use burn_backend::ElementConversion; +use burn_backend::ops::{GridSampleOptions, GridSamplePaddingMode, InterpolateMode}; +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +use ndarray::Array4; + +use crate::SharedArray; +use crate::{FloatNdArrayElement, UnsafeSharedRef, iter_range_par, run_par}; + +/// Sample a tensor using grid-based sampling. +/// +/// # Arguments +/// +/// * `tensor` - The tensor being sampled from, must be contiguous with shape (N, C, H_in, W_in) +/// * `grid` - A tensor of locations, with shape (N, H_out, W_out, 2). Values are [-1, 1]. +/// A [x = -1, y = -1] means top-left, and [x = 1, y = 1] means bottom-right +/// * `options` - Grid sampling options (mode, padding_mode, align_corners) +/// +/// # Returns +/// +/// A tensor with shape (N, C, H_out, W_out) +pub(crate) fn grid_sample_2d( + tensor: SharedArray, + grid: SharedArray, + options: GridSampleOptions, +) -> SharedArray { + match options.mode { + InterpolateMode::Bilinear => (), + _ => todo!( + "grid_sample_2d with {:?} mode is not implemented", + options.mode + ), + } + + let tensor = tensor.into_dimensionality::().unwrap(); + let grid = grid.into_dimensionality::().unwrap(); + + let (batch_size, channels, height_in, width_in) = tensor.dim(); + let (b, height_out, width_out, d) = grid.dim(); + assert!(batch_size == b); + assert!(2 == d); + + let mut output = Array4::zeros((batch_size, channels, height_out, width_out)); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + let sample_count = batch_size * channels * height_out * width_out; + let strides = ( + channels * height_out * width_out, + height_out * width_out, + width_out, + ); + + let align = options.align_corners; + let pad_mode = options.padding_mode; + + run_par!(|| { + iter_range_par!(0, sample_count).for_each(|id| { + let (b, c, y, x) = ( + id / strides.0, + id % strides.0 / strides.1, + id % strides.1 / strides.2, + id % strides.2, + ); + + let sample_x = grid[(b, y, x, 0)].elem::(); + let sample_y = grid[(b, y, x, 1)].elem::(); + + // Convert normalized grid coordinates [-1, 1] to pixel coordinates + let (px, py) = if align { + // align_corners=true: x_pixel = (x_norm + 1) * (width - 1) / 2 + // Maps -1 to 0 and 1 to width - 1 + let px = (sample_x + 1.0) * ((width_in - 1) as f64) / 2.0; + let py = (sample_y + 1.0) * ((height_in - 1) as f64) / 2.0; + (px, py) + } else { + // align_corners=false: x_pixel = (x_norm + 1) * width / 2 - 0.5 + // Maps -1 to -0.5 and 1 to width - 0.5 + let px = (sample_x + 1.0) * (width_in as f64) / 2.0 - 0.5; + let py = (sample_y + 1.0) * (height_in as f64) / 2.0 - 0.5; + (px, py) + }; + + // Bilinear interpolation with the specified padding mode + let val = + bilinear_interpolate(&tensor, b, c, px, py, width_in, height_in, pad_mode, align); + + unsafe { + let output = unsafe_shared_out.get(); + output[(b, c, y, x)] = val.elem(); + } + }); + }); + + output.into_dyn().into_shared() +} + +/// Bilinear interpolation at a point with configurable padding mode. +#[allow(clippy::too_many_arguments)] +fn bilinear_interpolate( + source: &ndarray::ArrayBase>, + b: usize, + c: usize, + x: f64, + y: f64, + width: usize, + height: usize, + padding_mode: GridSamplePaddingMode, + align_corners: bool, +) -> f64 +where + E: FloatNdArrayElement, + S: ndarray::Data, +{ + // Handle inf/nan coordinates + if !x.is_finite() || !y.is_finite() { + return match padding_mode { + GridSamplePaddingMode::Zeros => 0.0, + GridSamplePaddingMode::Border => { + // Clamp to center of image for inf/nan + let cx = ((width - 1) as f64 / 2.0).clamp(0.0, (width - 1) as f64); + let cy = ((height - 1) as f64 / 2.0).clamp(0.0, (height - 1) as f64); + source[(b, c, cy as usize, cx as usize)].elem::() + } + GridSamplePaddingMode::Reflection => 0.0, // Simplified: treat as zeros for inf/nan + }; + } + + // Apply padding mode to get actual sampling coordinates + let (x, y) = match padding_mode { + GridSamplePaddingMode::Border => { + // Clamp coordinates to valid range [0, size-1] + let x = x.clamp(0.0, (width - 1) as f64); + let y = y.clamp(0.0, (height - 1) as f64); + (x, y) + } + GridSamplePaddingMode::Reflection => { + // Reflect coordinates at boundaries + let x = reflect_coordinate(x, width, align_corners); + let y = reflect_coordinate(y, height, align_corners); + (x, y) + } + GridSamplePaddingMode::Zeros => (x, y), // Keep as-is, handle out-of-bounds in read + }; + + // Get the four corner indices + let x0 = x.floor() as i64; + let y0 = y.floor() as i64; + let x1 = x0.saturating_add(1); + let y1 = y0.saturating_add(1); + + // Compute interpolation weights (fractional part) + let x_frac = x - x.floor(); + let y_frac = y - y.floor(); + + // Helper to read a value based on padding mode + let read_value = |xi: i64, yi: i64| -> f64 { + match padding_mode { + GridSamplePaddingMode::Zeros => { + // Return 0 for out-of-bounds + if xi >= 0 && xi < width as i64 && yi >= 0 && yi < height as i64 { + source[(b, c, yi as usize, xi as usize)].elem::() + } else { + 0.0 + } + } + GridSamplePaddingMode::Border | GridSamplePaddingMode::Reflection => { + // Coordinates should already be in valid range after clamping/reflection + let xi = xi.clamp(0, (width - 1) as i64) as usize; + let yi = yi.clamp(0, (height - 1) as i64) as usize; + source[(b, c, yi, xi)].elem::() + } + } + }; + + // Read the four corners + let v00 = read_value(x0, y0); + let v01 = read_value(x0, y1); + let v10 = read_value(x1, y0); + let v11 = read_value(x1, y1); + + // Bilinear interpolation weights + let w00 = (1.0 - x_frac) * (1.0 - y_frac); + let w01 = (1.0 - x_frac) * y_frac; + let w10 = x_frac * (1.0 - y_frac); + let w11 = x_frac * y_frac; + + v00 * w00 + v01 * w01 + v10 * w10 + v11 * w11 +} + +/// Reflect a coordinate at the boundaries using a triangle wave pattern. +/// +/// For align_corners=true: reflects within [0, size-1] +/// For align_corners=false: reflects within [-0.5, size-0.5] +fn reflect_coordinate(coord: f64, size: usize, align_corners: bool) -> f64 { + let size_f = size as f64; + let (min_val, max_val) = if align_corners { + (0.0, size_f - 1.0) + } else { + (-0.5, size_f - 0.5) + }; + + let span = max_val - min_val; + if span <= 0.0 { + return min_val; + } + + // Triangle wave formula: span - |((x mod 2*span) - span)| + let period = 2.0 * span; + let x = (coord - min_val).abs(); + let x_mod = x - (x / period).floor() * period; + span - (x_mod - span).abs() + min_val +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/int_tensor.rs new file mode 100644 index 0000000..8c41a66 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/int_tensor.rs @@ -0,0 +1,497 @@ +// Language +use crate::rand::get_seeded_rng; +use alloc::vec::Vec; +use burn_backend::backend::ExecutionError; +use burn_backend::ops::IntTensorOps; +use burn_backend::tensor::{FloatTensor, IntTensor}; +use burn_backend::{Distribution, IntDType, Scalar, TensorMetadata}; + +use burn_backend::ElementConversion; + +// Current crate +use crate::{NdArray, cast_to_dtype, execute_with_dtype, tensor::NdArrayTensor}; +use crate::{NdArrayDevice, SEED, slice}; +use crate::{SharedArray, element::QuantElement}; +use crate::{cat_with_dtype, execute_with_float_dtype}; +use crate::{element::FloatNdArrayElement, ops::matmul::matmul}; +use crate::{element::IntNdArrayElement, execute_with_int_dtype}; + +// Workspace crates +use super::{NdArrayBitOps, NdArrayMathOps, NdArrayOps}; +use burn_backend::{DType, Shape, TensorData, backend::Backend}; + +impl IntTensorOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + fn int_from_data(data: TensorData, _device: &NdArrayDevice) -> NdArrayTensor { + if data.dtype.is_int() || data.dtype.is_uint() { + NdArrayTensor::from_data(data) + } else { + unimplemented!("Unsupported dtype for `int_from_data`: {:?}", data.dtype) + } + } + + async fn int_into_data(tensor: NdArrayTensor) -> Result { + Ok(tensor.into_data()) + } + + fn int_to_device(tensor: NdArrayTensor, _device: &NdArrayDevice) -> NdArrayTensor { + tensor + } + + fn int_reshape(tensor: NdArrayTensor, shape: Shape) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::reshape(array, shape)) + } + + fn int_slice(tensor: NdArrayTensor, slices: &[burn_backend::Slice]) -> NdArrayTensor { + slice!(tensor, slices) + } + + fn int_device(_tensor: &NdArrayTensor) -> as Backend>::Device { + NdArrayDevice::Cpu + } + + fn int_empty( + shape: Shape, + device: & as Backend>::Device, + dtype: IntDType, + ) -> NdArrayTensor { + Self::int_zeros(shape, device, dtype) + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + execute_with_int_dtype!((lhs, rhs), matmul) + } + + fn int_mask_where( + tensor: NdArrayTensor, + mask: NdArrayTensor, + source: NdArrayTensor, + ) -> NdArrayTensor { + execute_with_int_dtype!((tensor, source), |tensor, source| { + NdArrayOps::mask_where(tensor, mask.bool(), source) + }) + } + + fn int_mask_fill(tensor: NdArrayTensor, mask: NdArrayTensor, value: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::mask_fill( + array, + mask.bool(), + value.elem() + )) + } + + fn int_slice_assign( + tensor: NdArrayTensor, + slices: &[burn_backend::Slice], + value: NdArrayTensor, + ) -> NdArrayTensor { + execute_with_int_dtype!((tensor, value), |tensor, value| NdArrayOps::slice_assign( + tensor, slices, value + )) + } + + fn int_cat(tensors: Vec, dim: usize) -> NdArrayTensor { + cat_with_dtype!(tensors, dim, [I64, I32, I16, I8, U64, U32, U16, U8]) + } + + fn int_equal(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::equal) + } + + fn int_equal_elem(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::equal_elem(array, rhs.elem())) + } + + fn int_greater(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::greater) + } + + fn int_greater_elem(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::greater_elem(array, rhs.elem())) + } + + fn int_greater_equal(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::greater_equal) + } + + fn int_greater_equal_elem(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::greater_equal_elem( + array, + rhs.elem() + )) + } + + fn int_lower(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::lower) + } + + fn int_lower_elem(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::lower_elem(array, rhs.elem())) + } + + fn int_lower_equal(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::lower_equal) + } + + fn int_lower_equal_elem(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::lower_equal_elem( + array, + rhs.elem() + )) + } + + fn int_add(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::add) + } + + fn int_add_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::add_scalar(array, rhs.elem())) + } + + fn int_sub(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::sub) + } + + fn int_sub_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::sub_scalar(array, rhs.elem())) + } + + fn int_mul(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::mul) + } + + fn int_mul_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::mul_scalar(array, rhs.elem())) + } + + fn int_div(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::div) + } + + fn int_div_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::div_scalar(array, rhs.elem())) + } + + fn int_remainder(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayMathOps::remainder) + } + + fn int_remainder_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayMathOps::remainder_scalar( + array, + rhs.elem() + )) + } + + fn int_sum(tensor: NdArrayTensor) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!(tensor, E, |array: SharedArray| NdArrayMathOps::sum_view( + array.view() + )) + } + + fn int_sum_dim(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::sum_dim(array, dim)) + } + + fn int_prod(tensor: NdArrayTensor) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!( + tensor, + E, + |array: SharedArray| NdArrayMathOps::prod_view(array.view()) + ) + } + + fn int_prod_dim(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::prod_dim(array, dim)) + } + + fn int_mean(tensor: NdArrayTensor) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!( + tensor, + E, + |array: SharedArray| NdArrayMathOps::mean_view(array.view()) + ) + } + + fn int_mean_dim(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::mean_dim(array, dim)) + } + + fn int_max(tensor: NdArrayTensor) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!(tensor, E, |array: SharedArray| NdArrayMathOps::max_view( + array.view() + )) + } + + fn int_min(tensor: NdArrayTensor) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!(tensor, E, |array: SharedArray| NdArrayMathOps::min_view( + array.view() + )) + } + + fn int_cumsum(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::cumsum(array, dim)) + } + + fn int_cumprod(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::cumprod(array, dim)) + } + + fn int_cummin(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::cummin(array, dim)) + } + + fn int_cummax(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::cummax(array, dim)) + } + + fn int_gather(dim: usize, tensor: NdArrayTensor, indices: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!(tensor, E, |array| -> NdArrayTensor { + execute_with_int_dtype!(indices, |idx_array| NdArrayOps::gather( + dim, array, idx_array + )) + }) + } + + fn int_scatter_add( + dim: usize, + tensor: NdArrayTensor, + indices: NdArrayTensor, + value: NdArrayTensor, + ) -> NdArrayTensor { + execute_with_int_dtype!((tensor, value), I, |tensor, value| -> NdArrayTensor { + execute_with_int_dtype!(indices, |idx_array| NdArrayOps::::scatter( + dim, tensor, idx_array, value + )) + }) + } + + fn int_select(tensor: NdArrayTensor, dim: usize, indices: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!(tensor, E, |array| -> NdArrayTensor { + execute_with_int_dtype!(indices, |idx_array| NdArrayMathOps::select( + array, dim, idx_array + )) + }) + } + + fn int_select_add( + tensor: NdArrayTensor, + dim: usize, + indices: NdArrayTensor, + value: NdArrayTensor, + ) -> NdArrayTensor { + execute_with_int_dtype!((tensor, value), I, |tensor, value| -> NdArrayTensor { + execute_with_int_dtype!(indices, |idx_array| NdArrayMathOps::::select_assign( + tensor, dim, idx_array, value + )) + }) + } + fn int_argmax(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!(tensor, E, |array: SharedArray| { + NdArrayMathOps::argmax_view::(array.view(), dim) + }) + } + + fn int_argmin(tensor: NdArrayTensor, dim: usize) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_int_dtype!(tensor, E, |array: SharedArray| { + NdArrayMathOps::argmin_view::(array.view(), dim) + }) + } + + fn int_clamp_min(tensor: NdArrayTensor, min: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::clamp_min(array, min.elem())) + } + + fn int_clamp_max(tensor: NdArrayTensor, max: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::clamp_max(array, max.elem())) + } + + fn int_clamp(tensor: NdArrayTensor, min: Scalar, max: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayMathOps::clamp( + array, + min.elem(), + max.elem() + )) + } + + fn int_abs(tensor: NdArrayTensor) -> NdArrayTensor { + match tensor.dtype() { + DType::I64 | DType::I32 | DType::I16 | DType::I8 => { + execute_with_dtype!(tensor, I, NdArrayMathOps::abs, [ + I64 => i64, I32 => i32, I16 => i16, I8 => i8 + ]) + } + // Already unsigned + DType::U64 | DType::U32 | DType::U16 | DType::U8 => tensor, + other => panic!("Unsupported dtype: {other:?}"), + } + } + + fn int_into_float(tensor: NdArrayTensor) -> FloatTensor { + execute_with_int_dtype!(tensor, IntElem, |array: SharedArray| array + .mapv(|a: IntElem| a.elem::()) + .into_shared()) + } + + fn int_swap_dims(tensor: NdArrayTensor, dim1: usize, dim2: usize) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::swap_dims(array, dim1, dim2)) + } + + fn int_random( + shape: Shape, + distribution: Distribution, + device: &NdArrayDevice, + ) -> NdArrayTensor { + let mut seed = SEED.lock().unwrap(); + let mut rng = seed.take().unwrap_or_else(get_seeded_rng); + + let effective_distribution = if distribution == Distribution::Default { + Distribution::Uniform(0.0, 255.0) // Assuming UniformInt is the integer variant + } else { + distribution + }; + + let tensor = Self::int_from_data( + TensorData::random::(shape, effective_distribution, &mut rng), + device, + ); + *seed = Some(rng); + tensor + } + + fn int_powi(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), I, |lhs, rhs| NdArrayMathOps::elementwise_op( + lhs, + rhs, + |a: &I, b: &I| { (a.elem::().pow(b.elem::())).elem() } + )) + } + + fn int_powf(lhs: NdArrayTensor, rhs: FloatTensor) -> NdArrayTensor { + execute_with_int_dtype!(lhs, I, |array| -> NdArrayTensor { + execute_with_float_dtype!(rhs, E, |rhs_array| { + NdArrayMathOps::elementwise_op(array, rhs_array, |a: &I, b: &E| { + (a.elem::().pow(*b as u32)).elem() + }) + }) + }) + } + + fn int_powf_scalar_impl(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, I, |array| { + NdArrayMathOps::elementwise_op_scalar(array, |a: I| { + (a.elem::().pow(rhs.elem())).elem() + }) + }) + } + + fn int_permute(tensor: NdArrayTensor, axes: &[usize]) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::permute(array, axes)) + } + + fn int_flip(tensor: NdArrayTensor, axes: &[usize]) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::flip(array, axes)) + } + + fn int_sign(tensor: NdArrayTensor) -> NdArrayTensor { + match tensor.dtype() { + DType::I64 | DType::I32 | DType::I16 | DType::I8 => { + execute_with_dtype!(tensor, I, NdArrayMathOps::sign_op, [ + I64 => i64, I32 => i32, I16 => i16, I8 => i8 + ]) + } + DType::U64 | DType::U32 | DType::U16 | DType::U8 => { + Self::int_greater_elem(tensor, 0.into()) + } + other => panic!("Unsupported dtype: {other:?}"), + } + } + + fn int_expand(tensor: NdArrayTensor, shape: Shape) -> NdArrayTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::expand(array, shape)) + } + + fn bitwise_and(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayBitOps::bitand) + } + + fn bitwise_and_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayBitOps::bitand_scalar(array, rhs.elem())) + } + + fn bitwise_or(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayBitOps::bitor) + } + + fn bitwise_or_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayBitOps::bitor_scalar(array, rhs.elem())) + } + + fn bitwise_xor(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), NdArrayBitOps::bitxor) + } + + fn bitwise_xor_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, |array| NdArrayBitOps::bitxor_scalar(array, rhs.elem())) + } + + fn bitwise_not(tensor: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!(tensor, NdArrayBitOps::bitnot) + } + + fn bitwise_left_shift(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), I, |lhs, rhs| { + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &I, b: &I| { + (a.elem::() << (b.elem::())).elem() + }) + }) + } + + fn bitwise_left_shift_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, I, |array| { + NdArrayMathOps::elementwise_op_scalar(array, |a: I| { + (a.elem::() << rhs.elem::()).elem() + }) + }) + } + + fn bitwise_right_shift(lhs: NdArrayTensor, rhs: NdArrayTensor) -> NdArrayTensor { + execute_with_int_dtype!((lhs, rhs), I, |lhs, rhs| { + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &I, b: &I| { + (a.elem::() >> (b.elem::())).elem() + }) + }) + } + + fn bitwise_right_shift_scalar(lhs: NdArrayTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_int_dtype!(lhs, I, |array| { + NdArrayMathOps::elementwise_op_scalar(array, |a: I| { + (a.elem::() >> rhs.elem::()).elem() + }) + }) + } + + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor { + execute_with_int_dtype!(tensor, |array| cast_to_dtype(array, dtype.into())) + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + execute_with_int_dtype!(tensor, |array| NdArrayOps::unfold(array, dim, size, step)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/interpolate.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/interpolate.rs new file mode 100644 index 0000000..66da329 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/interpolate.rs @@ -0,0 +1,302 @@ +use burn_backend::ElementConversion; +use ndarray::{Array4, ArrayBase, DataOwned}; +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +use crate::{FloatNdArrayElement, ShapeOps, SharedArray, UnsafeSharedRef, iter_range_par, run_par}; + +pub(crate) fn nearest_interpolate( + x: SharedArray, + output_size: [usize; 2], +) -> SharedArray { + let x = x.into_dimensionality::().unwrap(); + + let (batch_size, channels, in_height, in_width) = x.dim(); + let [out_height, out_width] = output_size; + + let y_ratio = (in_height as f64) / (out_height as f64); + let x_ratio = (in_width as f64) / (out_width as f64); + + let out_element_num = batch_size * channels * out_height * out_width; + let strides = ( + channels * out_height * out_width, + out_height * out_width, + out_width, + ); + + let mut output = Array4::zeros((batch_size, channels, out_height, out_width)); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, out_element_num).for_each(|id| { + let (b, c, h, w) = ( + id / strides.0, + id % strides.0 / strides.1, + id % strides.1 / strides.2, + id % strides.2, + ); + + let y_in = (y_ratio * h as f64).floor() as usize; + let x_in = (x_ratio * w as f64).floor() as usize; + + unsafe { + let output = unsafe_shared_out.get(); + output[(b, c, h, w)] = x[(b, c, y_in, x_in)]; + } + }); + }); + + output.into_dyn().into_shared() +} + +pub(crate) fn nearest_interpolate_backward( + x: SharedArray, + grad: SharedArray, + output_size: [usize; 2], +) -> SharedArray { + let [batch_size, channels, input_height, input_width] = x.shape().dims(); + let [output_height, output_width] = output_size; + + let mut output_grad = + Array4::from_elem((batch_size, channels, input_height, input_width), 0.elem()); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output_grad); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output_grad = unsafe_shared_out.get(); + + for oh in 0..output_height { + for ow in 0..output_width { + let ih = start_index(oh, output_height, input_height); + let iw = start_index(ow, output_width, input_width); + + output_grad[[b, c, ih, iw]] += grad[[b, c, oh, ow]] + } + } + }) + }); + + output_grad.into_dyn().into_shared() +} + +fn start_index(output_size_index: usize, output_size: usize, input_size: usize) -> usize { + ((output_size_index as f32 * input_size as f32) / output_size as f32).floor() as usize +} + +// clamp ceil(frac) to stay within bounds in case of floating-point imprecision +pub(crate) fn ceil_clamp(frac: f64, max: usize) -> f64 { + frac.ceil().min(max as f64) +} + +pub(crate) fn bilinear_interpolate( + x: SharedArray, + output_size: [usize; 2], + align_corners: bool, +) -> SharedArray { + let x = x.into_dimensionality::().unwrap(); + + let (batch_size, channels, in_height, in_width) = x.dim(); + let [out_height, out_width] = output_size; + + let out_element_num = batch_size * channels * out_height * out_width; + let strides = ( + channels * out_height * out_width, + out_height * out_width, + out_width, + ); + + let mut output = Array4::zeros((batch_size, channels, out_height, out_width)); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, out_element_num).for_each(|id| { + let (b, c, h, w) = ( + id / strides.0, + id % strides.0 / strides.1, + id % strides.1 / strides.2, + id % strides.2, + ); + + let (y_frac, x_frac) = if align_corners { + let y_ratio = ((in_height - 1) as f64) / (core::cmp::max(out_height - 1, 1) as f64); + let x_ratio = ((in_width - 1) as f64) / (core::cmp::max(out_width - 1, 1) as f64); + (y_ratio * h as f64, x_ratio * w as f64) + } else { + let y_frac = (h as f64 + 0.5) * (in_height as f64 / out_height as f64) - 0.5; + let x_frac = (w as f64 + 0.5) * (in_width as f64 / out_width as f64) - 0.5; + ( + y_frac.clamp(0.0, (in_height - 1) as f64), + x_frac.clamp(0.0, (in_width - 1) as f64), + ) + }; + let val = + bilinear_interpolate_single(&x, b, c, x_frac, y_frac, in_width - 1, in_height - 1); + + unsafe { + let output = unsafe_shared_out.get(); + output[(b, c, h, w)] = val.elem(); + } + }); + }); + + output.into_dyn().into_shared() +} + +pub(crate) fn bicubic_interpolate( + x: SharedArray, + output_size: [usize; 2], + align_corners: bool, +) -> SharedArray { + fn cubic_interp1d(x0: f64, x1: f64, x2: f64, x3: f64, t: f64) -> f64 { + fn cubic_convolution1(x: f64, a: f64) -> f64 { + ((a + 2.0) * x - (a + 3.0)) * x * x + 1.0 + } + + fn cubic_convolution2(x: f64, a: f64) -> f64 { + ((a * x - 5.0 * a) * x + 8.0 * a) * x - 4.0 * a + } + + let coeffs = [ + cubic_convolution2(t + 1.0, -0.75), + cubic_convolution1(t, -0.75), + cubic_convolution1(1.0 - t, -0.75), + cubic_convolution2(2.0 - t, -0.75), + ]; + + x0 * coeffs[0] + x1 * coeffs[1] + x2 * coeffs[2] + x3 * coeffs[3] + } + + let x = x.into_dimensionality::().unwrap(); + + let (batch_size, channels, in_height, in_width) = x.dim(); + let [out_height, out_width] = output_size; + + let out_element_num = batch_size * channels * out_height * out_width; + let strides = ( + channels * out_height * out_width, + out_height * out_width, + out_width, + ); + + let mut output = Array4::zeros((batch_size, channels, out_height, out_width)); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, out_element_num).for_each(|id| { + let (b, c, h, w) = ( + id / strides.0, + id % strides.0 / strides.1, + id % strides.1 / strides.2, + id % strides.2, + ); + + let (y_frac, x_frac) = if align_corners { + let y_ratio = ((in_height - 1) as f64) / (core::cmp::max(out_height - 1, 1) as f64); + let x_ratio = ((in_width - 1) as f64) / (core::cmp::max(out_width - 1, 1) as f64); + (y_ratio * h as f64, x_ratio * w as f64) + } else { + let y_frac = (h as f64 + 0.5) * (in_height as f64 / out_height as f64) - 0.5; + let x_frac = (w as f64 + 0.5) * (in_width as f64 / out_width as f64) - 0.5; + (y_frac, x_frac) + }; + let y0 = y_frac.floor(); + let yw = y_frac - y0; + let y_in = y0 as isize; + + let x0 = x_frac.floor(); + let xw = x_frac - x0; + let x_in = x0 as isize; + + let max_h = (in_height - 1) as isize; + let max_w = (in_width - 1) as isize; + + let ys_in = [ + (y_in - 1).clamp(0, max_h) as usize, + y_in.clamp(0, max_h) as usize, + (y_in + 1).clamp(0, max_h) as usize, + (y_in + 2).clamp(0, max_h) as usize, + ]; + + let xs_in = [ + (x_in - 1).clamp(0, max_w) as usize, + x_in.clamp(0, max_w) as usize, + (x_in + 1).clamp(0, max_w) as usize, + (x_in + 2).clamp(0, max_w) as usize, + ]; + + let coefficients = ys_in.map(|y| { + cubic_interp1d( + x[(b, c, y, xs_in[0])].elem(), + x[(b, c, y, xs_in[1])].elem(), + x[(b, c, y, xs_in[2])].elem(), + x[(b, c, y, xs_in[3])].elem(), + xw, + ) + }); + + let result = cubic_interp1d( + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + yw, + ) + .elem(); + + unsafe { + let output = unsafe_shared_out.get(); + output[(b, c, h, w)] = result; + } + }); + }); + + output.into_dyn().into_shared() +} + +/// Sample an element of the source array with bilinear interpolation +/// +/// * `source` - The tensor to read from. Has shape (batch_size, channels, height, width) +/// * `b` - The batch to read from +/// * `c` - The channel to read from +/// * `x` - The x position to read in the array +/// * `y` - The y position to read in the array +/// * `x_max` - The max x position (inclusive) +/// * `y_max` - The max y position (inclusive) +/// +/// # Returns +/// +/// The interpolated value read from the array +pub(crate) fn bilinear_interpolate_single( + source: &ArrayBase>, + b: usize, + c: usize, + x: f64, + y: f64, + x_max: usize, + y_max: usize, +) -> f64 +where + E: FloatNdArrayElement, + S: DataOwned, +{ + let y0 = y.floor(); + let y1 = ceil_clamp(y, y_max); + let yw = y - y0; + + let x0 = x.floor(); + let x1 = ceil_clamp(x, x_max); + let xw = x - x0; + + let (x0, x1, y0, y1) = (x0 as usize, x1 as usize, y0 as usize, y1 as usize); + + let p_a = source[(b, c, y0, x0)].elem::() * (1.0 - xw) * (1.0 - yw); + let p_b = source[(b, c, y0, x1)].elem::() * xw * (1.0 - yw); + let p_c = source[(b, c, y1, x0)].elem::() * (1.0 - xw) * yw; + let p_d = source[(b, c, y1, x1)].elem::() * xw * yw; + + p_a + p_b + p_c + p_d +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/macros.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/macros.rs new file mode 100644 index 0000000..b3ac4f9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/macros.rs @@ -0,0 +1,107 @@ +macro_rules! keepdim { + ( + $dim:expr, + $self:expr, + mean + ) => {{ + // Get shape first (via reference), then pass ownership to avoid clone + let mut shape = $self.shape().into_shape(); + shape[$dim] = 1; + let tensor: SharedArray = mean_dim($self, $dim); + NdArrayOps::reshape(tensor, shape) + }}; + ( + $dim:expr, + $self:expr, + sum + ) => {{ + // Get shape first (via reference), then pass ownership to avoid clone + let mut shape = $self.shape().into_shape(); + shape[$dim] = 1; + let tensor: SharedArray = sum_dim($self, $dim); + NdArrayOps::reshape(tensor, shape) + }}; + ( + $dim:expr, + $self:expr, + prod + ) => {{ + // Get shape first (via reference), then pass ownership to avoid clone + let mut shape = $self.shape().into_shape(); + shape[$dim] = 1; + let tensor: SharedArray = prod_dim($self, $dim); + NdArrayOps::reshape(tensor, shape) + }}; +} + +use burn_backend::ElementConversion; +pub(crate) use keepdim; +use ndarray::{Axis, Zip}; + +use crate::{SharedArray, element::NdArrayElement}; + +pub(crate) fn mean_dim(tensor: SharedArray, dim: usize) -> SharedArray { + tensor.mean_axis(Axis(dim)).unwrap().into_shared() +} + +pub(crate) fn sum_dim(tensor: SharedArray, dim: usize) -> SharedArray { + tensor.sum_axis(Axis(dim)).into_shared() +} + +pub(crate) fn prod_dim(tensor: SharedArray, dim: usize) -> SharedArray { + tensor + .fold_axis(Axis(dim), 1.elem::(), |acc, &x| acc.mul(x.elem())) + .into_shared() +} + +/// Generic cumulative operation function with closure-based operation. +pub(crate) fn cumulative_with_op(tensor: SharedArray, dim: usize, op: F) -> SharedArray +where + E: NdArrayElement, + F: Fn(&mut E, &E), +{ + let axis = Axis(dim); + let shape = tensor.shape().to_vec(); + // Use into_owned() instead of to_owned() - only copies if shared, avoids copy if unique + let mut result = tensor.into_owned(); + let dim_size = shape[dim]; + + for i in 1..dim_size { + let prev = result.index_axis(axis, i - 1).to_owned(); + let mut current = result.index_axis_mut(axis, i); + Zip::from(&mut current).and(&prev).for_each(&op); + } + + result.into_shared() +} + +// Define all cumulative operation functions using the generic function +pub(crate) fn cumsum_dim(tensor: SharedArray, dim: usize) -> SharedArray { + cumulative_with_op(tensor, dim, |c, &p| *c = c.add(p.elem())) +} + +pub(crate) fn cumprod_dim(tensor: SharedArray, dim: usize) -> SharedArray { + cumulative_with_op(tensor, dim, |c, &p| *c = c.mul(p.elem())) +} + +pub(crate) fn cummin_dim>( + tensor: SharedArray, + dim: usize, +) -> SharedArray { + cumulative_with_op(tensor, dim, |c, &p| { + if p < *c { + *c = p; + } + }) +} + +pub(crate) fn cummax_dim>( + tensor: SharedArray, + dim: usize, +) -> SharedArray { + cumulative_with_op(tensor, dim, |c, &p| { + if p > *c { + *c = p; + } + }) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/matmul.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/matmul.rs new file mode 100644 index 0000000..3fb7b46 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/matmul.rs @@ -0,0 +1,362 @@ +use crate::UnsafeSharedRef; +use crate::{NdArrayElement, ShapeOps, SharedArray, iter_range_par, ops::NdArrayOps, run_par}; + +use alloc::{vec, vec::Vec}; +use burn_backend::ElementConversion; +use burn_backend::Shape; +use ndarray::{IxDyn, s}; + +pub(crate) fn matmul( + lhs: SharedArray, + rhs: SharedArray, +) -> SharedArray { + let shape_lhs = lhs.shape(); + let shape_rhs = rhs.shape(); + let ndims = shape_lhs.num_dims(); + let m = shape_lhs[ndims - 2]; // # of left rows + let k = shape_rhs[ndims - 2]; // # of left cols and right rows + let n = shape_rhs[ndims - 1]; // # of right cols + + let (out_shape, strides_lhs, strides_rhs, strides_out) = output_shape(shape_lhs, shape_rhs); + let l_mat_size = m * k; // size of matrix component of left array + let r_mat_size = k * n; // size of matrix component of right array + let out_mat_size = m * n; // size of matrix component of output array + + let num_l_batches = shape_lhs.num_elements() / l_mat_size; + let num_r_batches = shape_rhs.num_elements() / r_mat_size; + let num_out_batches = out_shape.num_elements() / out_mat_size; + + let lhs_array = NdArrayOps::reshape(lhs, Shape::new([num_l_batches, m, k])); + let rhs_array = NdArrayOps::reshape(rhs, Shape::new([num_r_batches, k, n])); + + let alpha: E = 1.0.elem(); + let beta: E = 0.0.elem(); + + let out = run_par!(|| { + let mut out_array = ndarray::Array3::::zeros((num_out_batches, m, n)); + let unsafe_shared_out_array = UnsafeSharedRef::new(&mut out_array); + + iter_range_par!(0, num_out_batches).for_each(|out_batch| { + // Here, we: + // 1. Un-flatten the output batch into a component-based batch index. + // 2. Use the strides for left and right batch indices to convert it to a flattened + // batch for left and right. + let out_index = strides_out.unflatten(out_batch); + let l_batch = strides_lhs.flatten(&out_index); + let r_batch = strides_rhs.flatten(&out_index); + + let lhs_slice = lhs_array.slice(s!(l_batch, .., ..)); + let rhs_slice = rhs_array.slice(s!(r_batch, .., ..)); + + unsafe { + let mut out_slice = unsafe_shared_out_array + .get() + .slice_mut(s!(out_batch, .., ..)); + + ndarray::linalg::general_mat_mul( + alpha, + &lhs_slice, + &rhs_slice, + beta, + &mut out_slice, + ) + } + }); + + out_array.into_shared().into_dyn() + }); + + NdArrayOps::reshape(out, out_shape) +} + +#[derive(Debug, PartialEq)] +struct Strides { + strides: Vec, +} +impl Strides { + fn new(strides: Vec) -> Self { + Strides { strides } + } + + fn unflatten(&self, linear_index: usize) -> Vec { + let mut coord = Vec::with_capacity(self.strides.len()); + let mut rem = linear_index; + for stride in self.strides.iter() { + coord.push(rem / stride); + rem %= stride; + } + coord + } + + fn flatten(&self, index: &Vec) -> usize { + assert_eq!(self.strides.len(), index.len()); + self.strides + .iter() + .zip(index) + .map(|(stride, index)| stride * index) + .sum() + } +} + +/// Compute the (broadcasted) output shape of matrix multiplication, along with strides for +/// the non-matrix dimensions of all arrays. +/// +/// # Arguments +/// * `lsh`: Shape of the first (left-hand) matrix multiplication argument. +/// * `rsh`: Shape of the second (right-hand) matrix multiplication argument. +/// +/// # Panics +/// * If `D` is not at least 2. +/// * If the matrix multiplication dimensions (last 2) are incompatible. +/// * If any other dimension is not the same for both tensors, or equal to 1. (Any dimension where +/// one dim is equal to 1 is broadcast.) +fn output_shape(lsh: &[usize], rsh: &[usize]) -> (Shape, Strides, Strides, Strides) { + let ndims = lsh.num_dims(); + if ndims < 2 { + panic!("Matrix multiplication requires an array with at least 2 dimensions."); + } + + // Fetch matrix dimensions and check compatibility. + let l_rows = lsh[ndims - 2]; + let l_cols = lsh[ndims - 1]; + let r_rows = rsh[ndims - 2]; + let r_cols = rsh[ndims - 1]; + if l_cols != r_rows { + panic!("Dimensions are incompatible for matrix multiplication."); + } + // Set matrix dimensions of the output shape. + let mut osh = vec![0; ndims]; + osh[ndims - 2] = l_rows; + osh[ndims - 1] = r_cols; + + // Set other array dimensions, broadcasting as necessary. + // Compute the strides inline. + let mut cur_l_stride: usize = 1; + let mut cur_r_stride: usize = 1; + let mut cur_o_stride: usize = 1; + let mut l_strides = Vec::with_capacity(ndims - 2); + let mut r_strides = Vec::with_capacity(ndims - 2); + let mut o_strides = Vec::with_capacity(ndims - 2); + for i in (0..ndims - 2).rev() { + let l_dim = lsh[i]; + let r_dim = rsh[i]; + + // Compatible dimensions are: + // 1. Both dimensions are equal. + // 2. One of the dimensions is equal to 1. + let o_dim: usize; + if l_dim == r_dim { + o_dim = l_dim; // both dimensions are equal + l_strides.push(cur_l_stride); + r_strides.push(cur_r_stride); + } else if l_dim == 1 { + o_dim = r_dim; // broadcast the left + l_strides.push(0); + r_strides.push(cur_r_stride); + } else if r_dim == 1 { + o_dim = l_dim; // broadcast the right + l_strides.push(cur_l_stride); + r_strides.push(0); + } else { + panic!("Dimensions differ and cannot be broadcasted."); + } + osh[i] = o_dim; + o_strides.push(cur_o_stride); + cur_o_stride *= o_dim; + + cur_l_stride *= l_dim; + cur_r_stride *= r_dim; + } + l_strides.reverse(); + r_strides.reverse(); + o_strides.reverse(); + + ( + Shape::from(osh), + Strides::new(l_strides), + Strides::new(r_strides), + Strides::new(o_strides), + ) +} + +pub(crate) fn cross( + lhs: SharedArray, + rhs: SharedArray, + dim: usize, +) -> SharedArray { + let shape_lhs = lhs.shape(); + let shape_rhs = rhs.shape(); + let ndims = shape_lhs.num_dims(); + + // Broadcast the shapes except along dim + let mut broadcast_shape = vec![0; ndims]; + for i in 0..ndims { + if i == dim { + broadcast_shape[i] = shape_lhs[i]; // already checked to be 3 + } else { + let l = shape_lhs[i]; + let r = shape_rhs[i]; + if l == r { + broadcast_shape[i] = l; + } else if l == 1 { + broadcast_shape[i] = r; + } else if r == 1 { + broadcast_shape[i] = l; + } else { + panic!("Tensors are not broadcastable along dimension {}", i); + } + } + } + + // Broadcast lhs and rhs + let lhs_broadcast = if shape_lhs == broadcast_shape.as_slice() { + lhs + } else { + NdArrayOps::expand(lhs, Shape::from(broadcast_shape.clone())) + }; + let rhs_broadcast = if shape_rhs == broadcast_shape.as_slice() { + rhs + } else { + NdArrayOps::expand(rhs, Shape::from(broadcast_shape.clone())) + }; + + // Now, move dim to the last dimension + let mut perm = (0..ndims).collect::>(); + perm.remove(dim); + perm.push(dim); + + let lhs_permuted = NdArrayOps::permute(lhs_broadcast, &perm); + let rhs_permuted = NdArrayOps::permute(rhs_broadcast, &perm); + + // Reshape to (*, 3) + let total_elements = lhs_permuted.shape().num_elements(); + let batch_size = total_elements / 3; + let lhs_reshaped = NdArrayOps::reshape(lhs_permuted, Shape::new([batch_size, 3])); + let rhs_reshaped = NdArrayOps::reshape(rhs_permuted, Shape::new([batch_size, 3])); + + // Compute cross product + let mut result = ndarray::ArrayD::::zeros(IxDyn(&[batch_size, 3])); + for i in 0..batch_size { + let a1 = lhs_reshaped[IxDyn(&[i, 0])]; + let a2 = lhs_reshaped[IxDyn(&[i, 1])]; + let a3 = lhs_reshaped[IxDyn(&[i, 2])]; + let b1 = rhs_reshaped[IxDyn(&[i, 0])]; + let b2 = rhs_reshaped[IxDyn(&[i, 1])]; + let b3 = rhs_reshaped[IxDyn(&[i, 2])]; + result[IxDyn(&[i, 0])] = a2.mul(b3).sub(a3.mul(b2)); + result[IxDyn(&[i, 1])] = a3.mul(b1).sub(a1.mul(b3)); + result[IxDyn(&[i, 2])] = a1.mul(b2).sub(a2.mul(b1)); + } + + let result_shared = result.into_shared(); + + // Reshape back to the broadcast shape with dim at the end + let mut result_shape = broadcast_shape; + result_shape.remove(dim); + result_shape.push(3); + let result_reshaped = NdArrayOps::reshape(result_shared, Shape::from(result_shape)); + + // Permute back + let mut inv_perm = vec![0; ndims]; + for (i, &p) in perm.iter().enumerate() { + inv_perm[p] = i; + } + NdArrayOps::permute(result_reshaped, &inv_perm) +} + +#[cfg(test)] +mod tests { + use super::*; + + impl Strides { + fn empty() -> Self { + Strides { + strides: Vec::with_capacity(0), + } + } + } + + #[test] + fn test_output_shape() { + // plain matrix multiply + assert_eq!( + output_shape(&[5, 3], &[3, 7]), + ( + Shape::from([5, 7]), + Strides::empty(), + Strides::empty(), + Strides::empty() + ) + ); + // matrix multiply with one extra stack dimension + assert_eq!( + output_shape(&[4, 5, 3], &[4, 3, 7]), + ( + Shape::from([4, 5, 7]), + Strides::new(vec![1]), + Strides::new(vec![1]), + Strides::new(vec![1]) + ) + ); + // rank 3, broadcast left + assert_eq!( + output_shape(&[1, 5, 3], &[4, 3, 7]), + ( + Shape::from([4, 5, 7]), + Strides::new(vec![0]), + Strides::new(vec![1]), + Strides::new(vec![1]) + ) + ); + // rank 3, broadcast right + assert_eq!( + output_shape(&[4, 5, 3], &[1, 3, 7]), + ( + Shape::from([4, 5, 7]), + Strides::new(vec![1]), + Strides::new(vec![0]), + Strides::new(vec![1]) + ) + ); + // rank 4, multi broadcast + assert_eq!( + output_shape(&[1, 4, 5, 3], &[8, 1, 3, 7]), + ( + Shape::from([8, 4, 5, 7]), + Strides::new(vec![0, 1]), + Strides::new(vec![1, 0]), + Strides::new(vec![4, 1]) + ) + ); + // rank 5, multi-broadcast + assert_eq!( + output_shape(&[1, 3, 4, 5, 3], &[8, 3, 1, 3, 7]), + ( + Shape::from([8, 3, 4, 5, 7]), + Strides::new(vec![0, 4, 1]), + Strides::new(vec![3, 1, 0]), + Strides::new(vec![12, 4, 1]) + ) + ) + } + + #[test] + #[should_panic( + expected = "Matrix multiplication requires an array with at least 2 dimensions." + )] + fn test_output_shape_too_small() { + output_shape(&[4], &[4]); + } + + #[test] + #[should_panic(expected = "Dimensions are incompatible for matrix multiplication.")] + fn test_output_shape_bad_matrix_dims() { + output_shape(&[5, 3], &[4, 7]); + } + + #[test] + #[should_panic(expected = "Dimensions differ and cannot be broadcasted.")] + fn test_output_shape_non_broadcast() { + output_shape(&[4, 5, 3], &[2, 3, 7]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/maxpool.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/maxpool.rs new file mode 100644 index 0000000..2a162cf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/maxpool.rs @@ -0,0 +1,247 @@ +use crate::{ + ShapeOps, SharedArray, + element::{FloatNdArrayElement, IntNdArrayElement}, + iter_range_par, + ops::padding::apply_padding_4d, + run_par, + sharing::UnsafeSharedRef, +}; + +use burn_backend::ElementConversion; +use burn_backend::ops::conv::calculate_pool_output_size; +use ndarray::Array4; + +pub(crate) fn max_pool2d( + x: SharedArray, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, +) -> SharedArray { + let [kernel_height, kernel_width] = kernel_size; + let [padding_height, padding_width] = padding; + let [stride_height, stride_width] = stride; + let [dilation_height, dilation_width] = dilation; + let [batch_size, channels, x_height, x_width] = x.shape().dims(); + let inf = (-f32::INFINITY).elem::(); + + let out_height = calculate_pool_output_size( + kernel_height, + stride_height, + padding_height, + dilation_height, + x_height, + ceil_mode, + ); + let out_width = calculate_pool_output_size( + kernel_width, + stride_width, + padding_width, + dilation_width, + x_width, + ceil_mode, + ); + + // Calculate extra padding needed for ceil_mode + // The maximum input position accessed is: (out_size - 1) * stride + (kernel_size - 1) * dilation + // This must be < input_size + 2 * total_padding + let max_ih = + (out_height.saturating_sub(1)) * stride_height + (kernel_height - 1) * dilation_height; + let max_iw = (out_width.saturating_sub(1)) * stride_width + (kernel_width - 1) * dilation_width; + let padded_height = x_height + 2 * padding_height; + let padded_width = x_width + 2 * padding_width; + let extra_pad_h = max_ih.saturating_sub(padded_height.saturating_sub(1)); + let extra_pad_w = max_iw.saturating_sub(padded_width.saturating_sub(1)); + let total_padding = [padding_height + extra_pad_h, padding_width + extra_pad_w]; + + let x = apply_padding_4d::(x, total_padding, inf); + + // Offset to account for extra padding (extra_pad is added on both sides by apply_padding_4d) + let offset_h = extra_pad_h; + let offset_w = extra_pad_w; + + let mut output = Array4::from_elem((batch_size, channels, out_height, out_width), inf); + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output = unsafe_shared_out.get(); + + for oh in 0..out_height { + for ow in 0..out_width { + let mut max_val = inf; + + for kh in 0..kernel_height { + let ih = offset_h + oh * stride_height + kh * dilation_height; + + for kw in 0..kernel_width { + let iw = offset_w + ow * stride_width + kw * dilation_width; + + let val = x[[b, c, ih, iw]]; + + if val > max_val { + max_val = val; + } + } + } + + output[[b, c, oh, ow]] = max_val; + } + } + }) + }); + + output.into_dyn().into_shared() +} + +pub(crate) fn max_pool2d_with_indices( + x: SharedArray, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, +) -> (SharedArray, SharedArray) { + let [kernel_height, kernel_width] = kernel_size; + let [padding_height, padding_width] = padding; + let [stride_height, stride_width] = stride; + let [dilation_height, dilation_width] = dilation; + let [batch_size, channels, x_height, x_width] = x.shape().dims(); + let inf = (-f32::INFINITY).elem::(); + + let out_height = calculate_pool_output_size( + kernel_height, + stride_height, + padding_height, + dilation_height, + x_height, + ceil_mode, + ); + let out_width = calculate_pool_output_size( + kernel_width, + stride_width, + padding_width, + dilation_width, + x_width, + ceil_mode, + ); + + // Calculate extra padding needed for ceil_mode + let max_ih = + (out_height.saturating_sub(1)) * stride_height + (kernel_height - 1) * dilation_height; + let max_iw = (out_width.saturating_sub(1)) * stride_width + (kernel_width - 1) * dilation_width; + let padded_height = x_height + 2 * padding_height; + let padded_width = x_width + 2 * padding_width; + let extra_pad_h = max_ih.saturating_sub(padded_height.saturating_sub(1)); + let extra_pad_w = max_iw.saturating_sub(padded_width.saturating_sub(1)); + let total_padding = [padding_height + extra_pad_h, padding_width + extra_pad_w]; + + let x = apply_padding_4d::(x, total_padding, inf); + + // Offset to account for extra padding + let offset_h = extra_pad_h; + let offset_w = extra_pad_w; + + let mut output = Array4::from_elem((batch_size, channels, out_height, out_width), inf); + let mut indices = Array4::::zeros((batch_size, channels, out_height, out_width)); + + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + let unsafe_shared_indices = UnsafeSharedRef::new(&mut indices); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output = unsafe_shared_out.get(); + let indices = unsafe_shared_indices.get(); + + for oh in 0..out_height { + for ow in 0..out_width { + let mut max_val = inf; + let mut index = 0; + + for kh in 0..kernel_height { + let ih = offset_h + oh * stride_height + kh * dilation_height; + + for kw in 0..kernel_width { + let iw = offset_w + ow * stride_width + kw * dilation_width; + let val = x[[b, c, ih, iw]]; + + if val > max_val { + max_val = val; + + // Calculate index in original (unpadded) input + let ih_orig = ih as i64 - (total_padding[0]) as i64; + let iw_orig = iw as i64 - (total_padding[1]) as i64; + + // Clamp to valid range for index calculation + let ih_clamped = ih_orig.max(0).min(x_height as i64 - 1); + let iw_clamped = iw_orig.max(0).min(x_width as i64 - 1); + + index = ih_clamped * x_width as i64 + iw_clamped; + } + } + } + + output[[b, c, oh, ow]] = max_val; + indices[[b, c, oh, ow]] = index.elem(); + } + } + }) + }); + + let output = output.into_dyn().into_shared(); + let indices = indices.into_dyn().into_shared(); + + (output, indices) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn max_pool2d_backward( + x: SharedArray, + _kernel_size: [usize; 2], + _stride: [usize; 2], + _padding: [usize; 2], + _dilation: [usize; 2], + _ceil_mode: bool, + output_grad: SharedArray, + indices: SharedArray, +) -> SharedArray { + let [_batch_size, _channels, height, width] = output_grad.shape().dims(); + let [batch_size, channels, height_x, width_x] = x.shape().dims(); + + let output_grad = output_grad; + let indices = indices; + + let mut output = Array4::zeros((batch_size, channels, height_x, width_x)); + + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + run_par!(|| { + iter_range_par!(0, batch_size * channels).for_each(|k| unsafe { + let b = k / channels; + let c = k % channels; + + let output = unsafe_shared_out.get(); + + for h in 0..height { + for w in 0..width { + let index = indices[[b, c, h, w]].elem::(); + let grad = output_grad[[b, c, h, w]]; + + let index_h = index as usize / width_x; + let index_w = index as usize % width_x; + + output[[b, c, index_h, index_w]] += grad; + } + } + }); + }); + + output.into_dyn().into_shared() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/mod.rs new file mode 100644 index 0000000..f4f215e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/mod.rs @@ -0,0 +1,24 @@ +mod activation; +mod base; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +#[cfg(feature = "simd")] +mod simd; +mod tensor; +mod transaction; + +pub(crate) mod adaptive_avgpool; +pub(crate) mod avgpool; +pub(crate) mod conv; +pub(crate) mod deform_conv; +pub(crate) mod grid_sample; +pub(crate) mod interpolate; +pub(crate) mod macros; +pub(crate) mod matmul; +pub(crate) mod maxpool; +pub(crate) mod padding; +pub(crate) mod quantization; + +pub(crate) use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/module.rs new file mode 100644 index 0000000..67daa39 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/module.rs @@ -0,0 +1,367 @@ +use super::{ + adaptive_avgpool::{adaptive_avg_pool2d, adaptive_avg_pool2d_backward}, + avgpool::{avg_pool2d, avg_pool2d_backward}, + conv::{conv_transpose2d, conv_transpose3d, conv2d, conv3d}, + deform_conv::{backward::deform_conv2d_backward, deform_conv2d}, + interpolate::{bicubic_interpolate, bilinear_interpolate, nearest_interpolate}, + maxpool::{max_pool2d, max_pool2d_backward, max_pool2d_with_indices}, +}; +#[cfg(feature = "simd")] +use crate::ops::simd::{ + avgpool::try_avg_pool2d_simd, conv::try_conv2d_simd, maxpool::try_max_pool2d_simd, +}; +use crate::{ + NdArray, SharedArray, element::FloatNdArrayElement, execute_with_int_dtype, + tensor::NdArrayTensor, +}; +use crate::{ + element::{IntNdArrayElement, QuantElement}, + ops::interpolate::nearest_interpolate_backward, +}; +use burn_backend::{ + ElementConversion, TensorMetadata, + ops::{attention::attention_fallback, *}, + tensor::FloatTensor, +}; + +macro_rules! module_op { + // Module op with inputs (inp), optional (opt) and arguments (args). + // Converts NdArrayStorage to SharedArray for compatibility with existing operations. + (inp($($x:tt),+), opt($($opt:tt),*), $element:ident, $op:expr) => {{ + #[allow(unused_parens, unreachable_patterns)] + match ($($x),+) { + ($(NdArrayTensor::F32($x)),+) => { + type $element = f32; + $op( + $($x.into_shared()),+ + $(, $opt.map(|o| match o { NdArrayTensor::F32(val) => val.into_shared(), _ => panic!("Optional argument type mismatch") }))* + ) + } + ($(NdArrayTensor::F64($x)),+) => { + type $element = f64; + $op( + $($x.into_shared()),+ + $(, $opt.map(|o| match o { NdArrayTensor::F64(val) => val.into_shared(), _ => panic!("Optional argument type mismatch") }))* + ) + } + _ => panic!("Data type mismatch"), + } + }}; +} + +impl ModuleOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + fn conv2d( + x: NdArrayTensor, + weight: NdArrayTensor, + bias: Option, + options: ConvOptions<2>, + ) -> NdArrayTensor { + module_op!(inp(x, weight), opt(bias), E, |x, weight, bias| { + #[cfg(feature = "simd")] + let (x, weight, bias) = match try_conv2d_simd(x, weight, bias, options.clone()) { + Ok(out) => return out.into(), + Err(args) => args, + }; + conv2d::(x, weight, bias, options).into() + }) + } + + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> FloatTensor { + module_op!( + inp(x, offset, weight), + opt(mask, bias), + E, + |x, offset, weight, mask, bias| deform_conv2d::( + x, offset, weight, mask, bias, options + ) + .into() + ) + } + + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + module_op!( + inp(x, offset, weight, output_grad), + opt(mask, bias), + E, + |x, offset, weight, output_grad, mask, bias| { + let (x, offset, weight, mask, bias) = deform_conv2d_backward::( + x, + offset, + weight, + mask, + bias, + output_grad, + options, + ); + DeformConv2dBackward::new( + x.into(), + offset.into(), + weight.into(), + mask.map(|m| m.into()), + bias.map(|b| b.into()), + ) + } + ) + } + + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + module_op!(inp(x, weight), opt(bias), E, |x, weight, bias| { + conv_transpose2d::(x, weight, bias, options).into() + }) + } + + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + module_op!(inp(x), opt(), E, |x| { + #[cfg(feature = "simd")] + let x = match if ceil_mode { + // SIMD path doesn't support ceil_mode yet, skip it + Err(x) + } else { + try_avg_pool2d_simd(x, kernel_size, stride, padding, count_include_pad) + } { + Ok(out) => return out.into(), + Err(x) => x, + }; + avg_pool2d::( + x, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ) + .into() + }) + } + + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + module_op!(inp(x, grad), opt(), E, |x, grad| avg_pool2d_backward::( + x, + grad, + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode + ) + .into()) + } + + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor { + module_op!(inp(x), opt(), E, |x| { + #[cfg(feature = "simd")] + let x = match if ceil_mode { + // SIMD path doesn't support ceil_mode yet, skip it + Err(x) + } else { + try_max_pool2d_simd(x, kernel_size, stride, padding, dilation) + } { + Ok(out) => return out.into(), + Err(x) => x, + }; + max_pool2d::(x, kernel_size, stride, padding, dilation, ceil_mode).into() + }) + } + + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices> { + module_op!(inp(x), opt(), E, |x| { + let (output, indices) = max_pool2d_with_indices::( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + MaxPool2dWithIndices::new(output.into(), indices.into()) + }) + } + + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: FloatTensor, + indices: NdArrayTensor, + ) -> MaxPool2dBackward> { + execute_with_int_dtype!(indices, IntElem, |idx_s: SharedArray| { + // Convert indices from runtime dtype to the expected I type + // (pool indices are bounded by tensor dimensions, so conversion is safe) + let indices: SharedArray = idx_s.mapv(|x| x.elem()).into_shared(); + module_op!(inp(x, output_grad), opt(), E, |x, output_grad| { + let output = max_pool2d_backward::( + x, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + output_grad, + indices, + ); + MaxPool2dBackward::new(output.into()) + }) + }) + } + + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor { + module_op!(inp(x), opt(), E, |x| adaptive_avg_pool2d::( + x, + output_size + ) + .into()) + } + + fn adaptive_avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + module_op!(inp(x, grad), opt(), E, |x, grad| { + adaptive_avg_pool2d_backward::(x, grad).into() + }) + } + + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + match options.mode { + InterpolateMode::Nearest => { + module_op!(inp(x), opt(), E, |x| nearest_interpolate::( + x, + output_size + ) + .into()) + } + InterpolateMode::Bilinear => { + let align_corners = options.align_corners; + module_op!(inp(x), opt(), E, |x| bilinear_interpolate::( + x, + output_size, + align_corners + ) + .into()) + } + InterpolateMode::Bicubic => { + let align_corners = options.align_corners; + module_op!(inp(x), opt(), E, |x| bicubic_interpolate::( + x, + output_size, + align_corners + ) + .into()) + } + } + } + + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + match options.mode { + InterpolateMode::Nearest => module_op!(inp(x, grad), opt(), E, |x, grad| { + nearest_interpolate_backward::(x, grad, output_size).into() + }), + InterpolateMode::Bilinear => { + panic!("bilinear interpolation backward is not supported for ndarray backend") + } + InterpolateMode::Bicubic => { + panic!("bicubic interpolation backward is not supported for ndarray backend") + } + } + } + + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> FloatTensor { + module_op!(inp(x, weight), opt(bias), E, |x, weight, bias| conv3d::( + x, weight, bias, options + ) + .into()) + } + + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + module_op!(inp(x, weight), opt(bias), E, |x, weight, bias| { + conv_transpose3d::(x, weight, bias, options).into() + }) + } + + fn attention( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, + ) -> FloatTensor { + attention_fallback::(query, key, value, mask, attn_bias, options) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/padding.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/padding.rs new file mode 100644 index 0000000..d9c6fd3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/padding.rs @@ -0,0 +1,72 @@ +use crate::{NdArrayElement, SharedArray}; +use ndarray::{Array4, Array5}; + +use super::NdArrayOps; + +pub(crate) fn apply_padding_4d( + x: SharedArray, + padding: [usize; 2], + elem: E, +) -> SharedArray { + let [batch_size, input_channels, height, width] = x.shape().try_into().unwrap(); + let [padding_height, padding_width] = padding; + let padded_height = height + 2 * padding_height; + let padded_width = width + 2 * padding_width; + + let x_new = Array4::from_elem( + (batch_size, input_channels, padded_height, padded_width), + elem, + ); + let mut x_new = x_new.into_shared().into_dyn(); + + x_new = NdArrayOps::slice_assign( + x_new, + &[ + burn_backend::Slice::from(0..batch_size), + burn_backend::Slice::from(0..input_channels), + burn_backend::Slice::from(padding_height..height + padding_height), + burn_backend::Slice::from(padding_width..width + padding_width), + ], + x, + ); + + x_new +} + +pub(crate) fn apply_padding_5d( + x: SharedArray, + padding: [usize; 3], + elem: E, +) -> SharedArray { + let [batch_size, input_channels, depth, height, width] = x.shape().try_into().unwrap(); + let [padding_depth, padding_height, padding_width] = padding; + let padded_depth = depth + 2 * padding_depth; + let padded_height = height + 2 * padding_height; + let padded_width = width + 2 * padding_width; + + let x_new = Array5::from_elem( + ( + batch_size, + input_channels, + padded_depth, + padded_height, + padded_width, + ), + elem, + ); + let mut x_new = x_new.into_shared().into_dyn(); + + x_new = NdArrayOps::slice_assign( + x_new, + &[ + burn_backend::Slice::from(0..batch_size), + burn_backend::Slice::from(0..input_channels), + burn_backend::Slice::from(padding_depth..depth + padding_depth), + burn_backend::Slice::from(padding_height..height + padding_height), + burn_backend::Slice::from(padding_width..width + padding_width), + ], + x, + ); + + x_new +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/qtensor.rs new file mode 100644 index 0000000..12a606f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/qtensor.rs @@ -0,0 +1,346 @@ +use alloc::{vec, vec::Vec}; + +use burn_backend::{ + DType, ExecutionError, Shape, TensorData, TensorMetadata, + ops::QTensorOps, + quantization::{ + QParams, QuantLevel, QuantMode, QuantScheme, QuantStore, QuantValue, + QuantizationParametersPrimitive, QuantizedBytes, + }, + tensor::{FloatTensor, IntTensor, QuantizedTensor}, +}; + +use crate::{ + FloatNdArrayElement, NdArray, NdArrayDevice, NdArrayQTensor, NdArrayTensor, SharedArray, + element::{IntNdArrayElement, QuantElement}, + execute_with_dtype, execute_with_int_dtype, execute_with_numeric_dtype, slice, +}; + +use super::quantization::{QuantizationStrategy, SymmetricQuantization}; +use super::{NdArrayMathOps, NdArrayOps}; + +impl QTensorOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + fn q_from_data(data: TensorData, _device: &NdArrayDevice) -> QuantizedTensor { + match data.dtype { + DType::QFloat(scheme) => { + let shape = data.shape.clone(); + let num_elements = data.num_elements(); + let q_bytes = QuantizedBytes { + bytes: data.into_bytes(), + scheme, + num_elements, + }; + + match scheme { + QuantScheme { + level: QuantLevel::Tensor | QuantLevel::Block(_), + mode: QuantMode::Symmetric, + value: QuantValue::Q8F | QuantValue::Q8S, + .. + } => { + // We can load QuantStore::U32 w/ QuantizedBytes impl + let (values, qparams) = q_bytes.into_vec_i8(); + let data = TensorData::new(values, shape); + // Overwrite storage + let scheme = scheme.with_store(QuantStore::Native); + + let qparams = qparams + .scales + .into_iter() + .map(|scales| QParams { scales }) + .collect(); + + NdArrayQTensor { + qtensor: NdArrayTensor::from_data(data), + scheme, + qparams, + } + } + QuantScheme { + value: + QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E2M1 + | QuantValue::E4M3 + | QuantValue::E5M2, + .. + } => unimplemented!("from_data not supported for scheme {scheme:?}"), + } + } + _ => panic!( + "Invalid dtype (expected DType::QFloat, got {:?})", + data.dtype + ), + } + } + + fn quantize( + tensor: FloatTensor, + scheme: &QuantScheme, + qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + let shape = tensor.shape(); + let data_f = tensor.into_data(); + let scales = qparams.scales.into_data().convert::(); + + // Implement with ndarray instead of QuantizationStrategy? + let (data, qparams) = match scheme { + QuantScheme { + level: QuantLevel::Tensor, + mode: QuantMode::Symmetric, + #[cfg(not(feature = "export_tests"))] + value: QuantValue::Q8F | QuantValue::Q8S, + // For tests, "native" sub-byte quant serves as a reference for value equality. + // Values are stored as i8 regardless. + #[cfg(feature = "export_tests")] + value: + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S, + store: QuantStore::Native, + .. + } => { + let scales = scales.iter().next().unwrap(); + let strategy = QuantizationStrategy::PerTensorSymmetric( + SymmetricQuantization::init(scales, scheme.value), + ); + let values = strategy.quantize(data_f.as_slice().unwrap()); + ( + TensorData::quantized(values, shape.clone(), *scheme, &[scales]), + vec![QParams { scales }], + ) + } + QuantScheme { + level: QuantLevel::Block(block_size), + mode: QuantMode::Symmetric, + #[cfg(not(feature = "export_tests"))] + value: QuantValue::Q8F | QuantValue::Q8S, + #[cfg(feature = "export_tests")] + value: + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S, + store: QuantStore::Native, + .. + } => { + let scales = scales.as_slice().unwrap(); + let (strategy, qparams) = scales + .iter() + .map(|&s| { + ( + SymmetricQuantization::init(s, scheme.value), + QParams { scales: s }, + ) + }) + .unzip(); + let strategy = QuantizationStrategy::PerBlockSymmetric(strategy, *block_size); + let values = strategy.quantize(data_f.as_slice().unwrap()); + ( + TensorData::quantized(values, shape.clone(), *scheme, scales), + qparams, + ) + } + scheme => unimplemented!("Quantization not supported for scheme {scheme:?}"), + }; + + let num_elements = data.num_elements(); + let q_bytes = QuantizedBytes { + bytes: data.into_bytes(), + scheme: *scheme, + num_elements, + }; + let (values, _) = q_bytes.into_vec_i8(); + let data = TensorData::new(values, shape).convert::(); + + NdArrayQTensor { + qtensor: NdArrayTensor::from_data(data), + scheme: *scheme, + qparams, + } + } + + fn dequantize(tensor: QuantizedTensor) -> FloatTensor { + let strategy = tensor.strategy(); + let scheme = tensor.scheme; + let shape = tensor.shape(); + let data = match tensor.qtensor { + NdArrayTensor::I8(storage) => { + let data = storage.into_shared().into_iter().collect(); + dequantize(data, shape, scheme, &strategy) + } + _ => unreachable!(), + }; + NdArrayTensor::from_data(data) + } + + fn q_device(_tensor: &QuantizedTensor) -> NdArrayDevice { + NdArrayDevice::Cpu + } + + fn q_to_device( + tensor: QuantizedTensor, + _device: &NdArrayDevice, + ) -> QuantizedTensor { + tensor + } + + fn q_reshape(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + NdArrayQTensor { + qtensor: execute_with_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayOps::reshape(array, shape) + }), + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + async fn q_into_data(tensor: QuantizedTensor) -> Result { + let shape = tensor.qtensor.shape(); + let scales = tensor.qparams.iter().map(|q| q.scales).collect::>(); + Ok(execute_with_numeric_dtype!( + tensor.qtensor, + E, + |array: SharedArray| { + let values = array.into_iter().collect(); + TensorData::quantized(values, shape, tensor.scheme, &scales) + } + )) + } + + fn q_swap_dims( + tensor: QuantizedTensor, + dim1: usize, + dim2: usize, + ) -> QuantizedTensor { + NdArrayQTensor { + qtensor: execute_with_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayOps::swap_dims(array, dim1, dim2) + }), + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + fn q_permute(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + NdArrayQTensor { + qtensor: execute_with_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayOps::permute(array, axes) + }), + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + fn q_flip(tensor: QuantizedTensor, axes: &[usize]) -> QuantizedTensor { + NdArrayQTensor { + qtensor: execute_with_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayOps::flip(array, axes) + }), + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + fn q_gather( + dim: usize, + tensor: QuantizedTensor, + indices: IntTensor, + ) -> QuantizedTensor { + let qtensor = execute_with_int_dtype!(indices, IntElem, |idx_array: SharedArray< + IntElem, + >| + -> NdArrayTensor { + execute_with_numeric_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayOps::gather(dim, array, idx_array) + }) + }); + NdArrayQTensor { + qtensor, + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + fn q_select( + tensor: QuantizedTensor, + dim: usize, + indices: IntTensor, + ) -> QuantizedTensor { + let qtensor = execute_with_int_dtype!(indices, IntElem, |idx_array: SharedArray< + IntElem, + >| + -> NdArrayTensor { + execute_with_numeric_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayMathOps::select(array, dim, idx_array) + }) + }); + NdArrayQTensor { + qtensor, + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + fn q_slice( + tensor: QuantizedTensor, + slices: &[burn_backend::Slice], + ) -> QuantizedTensor { + NdArrayQTensor { + qtensor: slice!(tensor.qtensor, slices), + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } + + fn q_argmax(tensor: QuantizedTensor, dim: usize) -> IntTensor { + execute_with_numeric_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayMathOps::argmax::(array, dim) + }) + } + + fn q_argmin(tensor: QuantizedTensor, dim: usize) -> IntTensor { + execute_with_numeric_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayMathOps::argmin::(array, dim) + }) + } + + fn q_expand(tensor: QuantizedTensor, shape: Shape) -> QuantizedTensor { + NdArrayQTensor { + qtensor: execute_with_dtype!(tensor.qtensor, E, |array: SharedArray| { + NdArrayOps::expand(array, shape) + }), + scheme: tensor.scheme, + qparams: tensor.qparams, + } + } +} + +fn dequantize( + data: Vec, + shape: Shape, + scheme: QuantScheme, + strategy: &QuantizationStrategy, +) -> TensorData { + let qparams = match strategy { + QuantizationStrategy::PerTensorSymmetric(quant) => vec![quant.scale], + QuantizationStrategy::PerBlockSymmetric(quant, _block_size) => { + quant.iter().map(|q| q.scale).collect() + } + }; + let q_bytes = QuantizedBytes::new(data, scheme, &qparams); + let (values, _qparams) = q_bytes.into_vec_i8(); + TensorData::new(strategy.dequantize(&values), shape) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/quantization.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/quantization.rs new file mode 100644 index 0000000..adaf1b1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/quantization.rs @@ -0,0 +1,218 @@ +use alloc::vec::Vec; +use num_traits::{Float, PrimInt}; + +use burn_backend::quantization::{BlockSize, QuantValue}; + +// NOTE: this mainly serves as a simple reference implementation. +// The de/quantization ops should be refactored to use ndarray. + +/// Quantization strategy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum QuantizationStrategy { + /// Per-tensor symmetric quantization. + PerTensorSymmetric(SymmetricQuantization), + /// Per-block symmetric quantization. + PerBlockSymmetric(Vec>, BlockSize), +} + +impl QuantizationStrategy { + /// Quantize the values to a lower precision data type. + pub fn quantize(&self, values: &[f32]) -> Vec { + match self { + QuantizationStrategy::PerTensorSymmetric(strategy) => strategy.quantize(values), + QuantizationStrategy::PerBlockSymmetric(strategy, block_size) => { + let block_elems = block_size.num_elements(); + let num_blocks = strategy.len(); + let numel = values.len(); + assert_eq!( + numel / block_elems, + num_blocks, + "Invalid per-block quantization with num blocks {num_blocks} and {numel} values" + ); + values + .chunks(block_elems) + .enumerate() + .flat_map(|(block_id, block)| strategy[block_id].quantize(block)) + .collect() + } + } + } + + /// Dequantize the values to a higher precision data type. + pub fn dequantize(&self, values: &[i8]) -> Vec { + match self { + QuantizationStrategy::PerTensorSymmetric(strategy) => strategy.dequantize(values), + QuantizationStrategy::PerBlockSymmetric(strategy, block_size) => { + let block_elems = block_size.num_elements(); + let num_blocks = strategy.len(); + let numel = values.len(); + assert_eq!( + numel / block_elems, + num_blocks, + "Invalid per-block quantization with block size {block_elems}, num blocks {num_blocks} and {numel} values" + ); + values + .chunks(block_elems) + .enumerate() + .flat_map(|(block_id, block)| strategy[block_id].dequantize(block)) + .collect() + } + } + } +} + +/// Quantization scheme to convert elements of a higher precision data type `E` to a lower precision +/// data type `Q` and vice-versa. +pub trait Quantization { + /// Returns the quantization range `[a, b]`. + fn range(&self) -> (E, E); + /// Convert the values to a lower precision data type. + fn quantize(&self, values: &[E]) -> Vec; + /// Convert a single value to a lower precision data type. + fn quantize_one(&self, value: E) -> Q; + /// Convert the values back to a higher precision data type. + fn dequantize(&self, values: &[Q]) -> Vec; + /// Convert a single value back to a higher precision data type. + fn dequantize_one(&self, value: Q) -> E; +} + +fn valid_scale(mut scale: E) -> E { + // If scale is 0 (most likely due to a tensor full of zeros), we arbitrarily adjust the + // scale to 0.1 to avoid division by zero. + if scale.eq(&E::zero()) { + scale = E::from(0.1).unwrap(); + } + scale +} + +/// Symmetric quantization scheme. +#[derive(Debug, Clone, Copy)] +pub struct SymmetricQuantization { + /// The scaling factor. + pub scale: E, + // The quantization value data type. + value: QuantValue, +} + +impl SymmetricQuantization { + /// Initialize a symmetric quantization scheme with the given parameters. + pub fn init(scale: E, value: QuantValue) -> Self { + Self { + scale: valid_scale(scale), + value, + } + } + + #[allow(dead_code)] + /// Create a new quantization scheme for an input range `[alpha, beta]`. + fn new(alpha: E, beta: E, value: QuantValue) -> Self { + let (a, b) = value.range(); + let a = E::from(a).unwrap(); + let b = E::from(b).unwrap(); + + // Compute scale to convert a floating point value in range `[-alpha, alpha]` to the quantized range + let alpha = alpha.abs().max(beta.abs()); + let scale = valid_scale((alpha + alpha) / (b - a)); + Self { scale, value } + } +} + +impl Quantization for SymmetricQuantization { + fn quantize(&self, values: &[E]) -> Vec { + values.iter().map(|x| self.quantize_one(*x)).collect() + } + + fn dequantize(&self, values: &[Q]) -> Vec { + values.iter().map(|x_q| self.dequantize_one(*x_q)).collect() + } + + fn quantize_one(&self, value: E) -> Q { + let (a, b) = self.range(); + + // x_q = clamp(round(x / scale), a, b) + Q::from(value.div(self.scale).round().clamp(a, b)).unwrap() + } + + fn dequantize_one(&self, value: Q) -> E { + // x = scale * x_q + self.scale * E::from(value).unwrap() + } + + fn range(&self) -> (E, E) { + let (a, b) = self.value.range(); + let a = E::from(a).unwrap(); + let b = E::from(b).unwrap(); + (a, b) + } +} + +impl PartialEq for SymmetricQuantization { + fn eq(&self, other: &Self) -> bool { + self.scale == other.scale + } +} + +impl Eq for SymmetricQuantization {} + +#[cfg(test)] +mod tests { + use burn_backend::TensorData; + + use super::*; + use alloc::vec; + + #[test] + fn test_int8_symmetric_quantization() { + let x: [f32; 4] = [-1.8, -1.0, 0.0, 0.5]; + let expected_q = vec![-127, -71, 0, 35]; + let expected_d = vec![-1.8, -1.0062993, 0.0, 0.496063]; + + let symmetric = SymmetricQuantization::::new(-1.8, 0.5, QuantValue::Q8S); + + let q: Vec = symmetric.quantize(&x); + assert_eq!(q, expected_q); + + let d = symmetric.dequantize(&expected_q); + + assert_eq!(d, expected_d); + } + + #[test] + fn test_int8_symmetric_quantization_per_block() { + let x: [f32; 8] = [-1.8, -1.0, 0.0, 0.5, -1.8, -1.0, 0.0, 0.5]; + let expected_q = vec![-127, -71, 0, 35, -127, -71, 0, 35]; + let expected_d = vec![ + -1.8, -1.0062993, 0.0, 0.496063, -1.8, -1.0062993, 0.0, 0.496063, + ]; + + let symmetric = SymmetricQuantization::::new(-1.8, 0.5, QuantValue::Q8S); + let strategy = QuantizationStrategy::PerBlockSymmetric( + vec![symmetric, symmetric], + BlockSize::new([4]), + ); + + let q: Vec = strategy.quantize(&x); + assert_eq!(q, expected_q); + + let d = symmetric.dequantize(&expected_q); + + assert_eq!(d, expected_d); + } + + #[test] + fn should_support_dequantize() { + let strategy = QuantizationStrategy::PerTensorSymmetric(SymmetricQuantization { + scale: 0.1, + value: QuantValue::Q8S, + }); + + let output = strategy.dequantize(&[-127i8, -77, -26, 25, 76, 127]); + + let output = TensorData::new(output, [2, 3]); + + output.assert_approx_eq::( + &TensorData::from([[-12.7, -7.7, -2.6], [2.5, 7.6, 12.7]]), + Default::default(), + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/avgpool.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/avgpool.rs new file mode 100644 index 0000000..41d5ba6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/avgpool.rs @@ -0,0 +1,443 @@ +use core::{marker::PhantomData, mem::transmute}; + +use crate::{SharedArray, iter_range_par, run_par, sharing::UnsafeSharedRef}; + +use burn_backend::DType; +use burn_backend::{Element, ElementConversion}; +use bytemuck::Zeroable; +use macerator::{Simd, VAdd, VDiv}; +use ndarray::{Array4, s}; +use nhwc::avg_pool_nhwc; + +use super::should_use_simd; + +#[macerator::with_simd] +fn is_accelerated(_x: PhantomData) -> bool { + ::is_accelerated::() && ::is_accelerated::() +} + +pub(crate) fn try_avg_pool2d_simd( + x: SharedArray, + ksize: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + with_pad: bool, +) -> Result, SharedArray> { + // Strides must be unit, dilation isn't supported, rows must be contiguous + if x.strides()[1] != 1 || !should_use_simd(x.shape()[1]) { + return Err(x); + } + + match E::dtype() { + DType::F64 if is_accelerated::(PhantomData) => Ok(cast(avg_pool_nhwc::( + cast(x), + ksize, + stride, + padding, + with_pad, + ))), + DType::F32 if is_accelerated::(PhantomData) => Ok(cast(avg_pool_nhwc::( + cast(x), + ksize, + stride, + padding, + with_pad, + ))), + _ => Err(x), + } +} + +fn cast(tensor: SharedArray) -> SharedArray { + unsafe { transmute::, SharedArray>(tensor) } +} + +mod nhwc { + use itertools::Itertools; + use macerator::{Simd, Vector, vload_unaligned, vstore_unaligned}; + use ndarray::{ArrayView3, ArrayViewMut3}; + use seq_macro::seq; + + use crate::ops::simd::lanes; + + use super::*; + + // Until you can use associated constants as array size, we need to hardcode this. + // The most common config (x86-v3) has 16 registers, so use half of them for accumulators. + const BLOCK_REGISTERS: usize = 8; + + pub(crate) fn avg_pool_nhwc( + x: SharedArray, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + with_pad: bool, + ) -> SharedArray { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + let [batch_size, channels, x_height, x_width] = x.shape().try_into().unwrap(); + let lanes = lanes::(); + + let ch_block = lanes * BLOCK_REGISTERS; + + let out_height = ((x_height + 2 * pad_h - (kernel_height - 1) - 1) / stride_height) + 1; + let out_width = ((x_width + 2 * pad_w - (kernel_width - 1) - 1) / stride_width) + 1; + + let mut output = unsafe { + Array4::::uninit((batch_size, out_height, out_width, channels)).assume_init() + }; + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + let x = x.view(); + let x = x.permuted_axes(vec![0, 2, 3, 1]); + + // Floor division ensures `blocks * lanes * blocking factor` is always `<= out_channels`. + // An exclusive loop will always have `lanes * blocking factor` elements in bounds. + let blocks = channels / ch_block; + let blocks_end = blocks * ch_block; + // Floor division means simd_end is always divisible by `lanes` and `<= out_channels`. An + // exclusive loop will always have `lanes` elements in bounds. + let simd_end = channels / lanes * lanes; + let num_simd_unblocked = (simd_end - blocks_end) / lanes; + let remainder = channels - simd_end; + + run_par!(|| { + // SAFETY: Loop ranges are non-overlapping, so the unsafe shared reference is safe. + iter_range_par!(0, batch_size * blocks).for_each(|k| unsafe { + let block = k % blocks; + let b = k / blocks; + + let output = unsafe_shared_out.get(); + + let x = x.slice(s![b, .., .., ..]); + let out = output.slice_mut(s![b, .., .., ..]); + + loop_blocked(x, out, kernel_size, stride, padding, with_pad, block); + }); + // SAFETY: See `loop_unblocked` + iter_range_par!(0, batch_size * num_simd_unblocked).for_each(|k| unsafe { + let ch = (k % num_simd_unblocked) * lanes + blocks_end; + let b = k / num_simd_unblocked; + + let output = unsafe_shared_out.get(); + + let x = x.slice(s![b, .., .., ..]); + let out = output.slice_mut(s![b, .., .., ..]); + + loop_unblocked(x, out, kernel_size, stride, padding, with_pad, ch); + }); + // SAFETY: Loop ranges are non-overlapping, so the unsafe shared reference is safe. + iter_range_par!(0, batch_size * remainder).for_each(|k| unsafe { + let ch = (k % remainder) + simd_end; + let b = k / remainder; + + let output = unsafe_shared_out.get(); + + let x = x.slice(s![b, .., .., ..]); + let out = output.slice_mut(s![b, .., .., ..]); + + loop_scalar(x, out, kernel_size, stride, padding, with_pad, ch); + }); + }); + + output = output.permuted_axes([0, 3, 1, 2]); + + output.into_dyn().into_shared() + } + + /// Execute the blocked (unrolled) portion of the pool. + #[allow( + clippy::too_many_arguments, + clippy::erasing_op, + clippy::identity_op, + unused_mut + )] + #[macerator::with_simd] + fn loop_blocked<'a, S: Simd, E: Element + VAdd + VDiv>( + x: ArrayView3<'a, E>, + mut out: ArrayViewMut3<'a, E>, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + with_pad: bool, + block: usize, + ) where + 'a: 'a, + { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + + let (x_height, x_width, _) = x.dim(); + let (out_height, out_width, _) = out.dim(); + let lanes = E::lanes::(); + + let ch_block = lanes * BLOCK_REGISTERS; + + // If pixels are more than `padding` from the edges, the in pixel cannot be out of bounds + for oh in pad_h..out_height.saturating_sub(pad_h) { + for ow in pad_w..out_width.saturating_sub(pad_w) { + seq!(N in 0..8 { + let mut sum~N: Vector = Zeroable::zeroed(); + }); + let ch = block * ch_block; + let ch_end = ch + ch_block; + let mut out = out.slice_mut(s![oh, ow, ch..ch_end]); + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw - pad_w; + let x = x.slice(s![ih, iw, ch..ch_end]); + + seq!(N in 0..8 { + // SAFETY: + // Load a full vector from x[N * lanes]. This is bounds checked by the + // slice above. + sum~N += unsafe { vload_unaligned(&x[N * lanes]) }; + }); + } + } + + let count = kernel_height * kernel_width; + let count = (count as u64).elem::(); + let count_v = count.splat(); + seq!(N in 0..8 { + let s~N = sum~N / count_v; + // SAFETY: + // Store a full vector to out[N * lanes]. This is bounds checked by the + // slice above. + unsafe { vstore_unaligned(&mut out[N * lanes], s~N) }; + }); + } + } + + // Border pixels need bounds checks + if (pad_h, pad_w) != (0, 0) { + let v_borders = (0..pad_h) + .chain(out_height.saturating_sub(pad_h)..out_height) + .cartesian_product(0..out_width); + let h_borders = (0..out_height) + .cartesian_product((0..pad_w).chain(out_width.saturating_sub(pad_w)..out_width)); + + for (oh, ow) in v_borders.chain(h_borders) { + seq!(N in 0..8 { + let mut sum~N: Vector = Zeroable::zeroed(); + }); + let mut count: usize = 0; + let ch = block * ch_block; + let ch_end = ch + ch_block; + let mut out = out.slice_mut(s![oh, ow, ch..ch_end]); + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh; + if ih < pad_h || ih >= x_height + pad_h { + continue; + } + let ih = ih - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw; + if iw < pad_w || iw >= x_width + pad_w { + continue; + } + let iw = iw - pad_w; + count += 1; + + let x = x.slice(s![ih, iw, ch..ch_end]); + + seq!(N in 0..8 { + // SAFETY: + // Load a full vector from x[N * lanes]. This is bounds checked by the + // slice above. + sum~N += unsafe { vload_unaligned(&x[N * lanes]) }; + }); + } + } + + if with_pad { + count = kernel_height * kernel_width; + } + + let count = (count as u64).elem::(); + let count_v = count.splat(); + seq!(N in 0..8 { + let s~N = sum~N / count_v; + // SAFETY: + // Store a full vector to out[N * lanes]. This is bounds checked by the + // slice above. + unsafe { vstore_unaligned(&mut out[N * lanes], s~N) }; + }); + } + } + } + + /// Execute the unblocked (not unrolled) portion of the pool. + /// + /// SAFETY: Safe as long as `ch + simd_lanes <= out_channels`. + #[allow(clippy::too_many_arguments, unused_mut)] + #[macerator::with_simd] + unsafe fn loop_unblocked<'a, S: Simd, E: Element + VAdd + VDiv>( + x: ArrayView3<'a, E>, + mut out: ArrayViewMut3<'a, E>, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + with_pad: bool, + ch: usize, + ) where + 'a: 'a, + { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + + let (x_height, x_width, _) = x.dim(); + let (out_height, out_width, _) = out.dim(); + + // If pixels are not within padding range, bounds checks are always true + for oh in pad_h..out_height - pad_h { + for ow in pad_w..out_width - pad_w { + let mut sum: Vector = Zeroable::zeroed(); + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw - pad_w; + // Load a full vector from `x`. In bounds as long as `out_channels >= ch + lanes` + let s0 = unsafe { vload_unaligned(&x[[ih, iw, ch]]) }; + sum += s0; + } + } + + let count = kernel_height * kernel_width; + let count: E = (count as u64).elem(); + let count_v = count.splat(); + let s0 = sum / count_v; + // Store a full vector to `out`. In bounds as long as `out_channels >= ch + lanes`. + unsafe { vstore_unaligned(&mut out[[oh, ow, ch]], s0) }; + } + } + + // Border pixels need bounds checks + if (pad_h, pad_w) != (0, 0) { + let v_borders = (0..pad_h) + .chain(out_height.saturating_sub(pad_h)..out_height) + .cartesian_product(0..out_width); + let h_borders = (0..out_height) + .cartesian_product((0..pad_w).chain(out_width.saturating_sub(pad_w)..out_width)); + + for (oh, ow) in v_borders.chain(h_borders) { + let mut sum: Vector = Zeroable::zeroed(); + let mut count: usize = 0; + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh; + if ih < pad_h || ih >= x_height + pad_h { + continue; + } + let ih = ih - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw; + if iw < pad_w || iw >= x_width + pad_w { + continue; + } + let iw = iw - pad_w; + count += 1; + + // Load a full vector from `x`. In bounds as long as `out_channels >= ch + lanes` + sum += unsafe { vload_unaligned(&x[[ih, iw, ch]]) }; + } + } + + if with_pad { + count = kernel_height * kernel_width; + } + + let count = (count as u64).elem::(); + let count_v = count.splat(); + let s0 = sum / count_v; + // Store a full vector to `out`. In bounds as long as `out_channels >= ch + lanes`. + unsafe { vstore_unaligned(&mut out[[oh, ow, ch]], s0) }; + } + } + } + + /// Execute scalar portion of the pooling + #[allow(clippy::too_many_arguments)] + fn loop_scalar( + x: ArrayView3<'_, E>, + mut out: ArrayViewMut3<'_, E>, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + with_pad: bool, + ch: usize, + ) { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + + let (x_height, x_width, _) = x.dim(); + let (out_height, out_width, _) = out.dim(); + + // If pixels are not within padding range, bounds checks are always true + for oh in pad_h..out_height.saturating_sub(pad_h) { + for ow in pad_w..out_width.saturating_sub(pad_w) { + let mut sum: E = Zeroable::zeroed(); + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw - pad_w; + sum = sum + x[[ih, iw, ch]]; + } + } + + let count = (kernel_height * kernel_width) as u64; + out[[oh, ow, ch]] = sum / count.elem(); + } + } + + // Border pixels need bounds checks + if (pad_h, pad_w) != (0, 0) { + let v_borders = (0..pad_h) + .chain(out_height.saturating_sub(pad_h)..out_height) + .cartesian_product(0..out_width); + let h_borders = (0..out_height) + .cartesian_product((0..pad_w).chain(out_width.saturating_sub(pad_w)..out_width)); + + for (oh, ow) in v_borders.chain(h_borders) { + let mut sum: E = Zeroable::zeroed(); + let mut count: usize = 0; + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh; + if ih < pad_h || ih >= x_height + pad_h { + continue; + } + let ih = ih - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw; + if iw < pad_w || iw >= x_width + pad_w { + continue; + } + let iw = iw - pad_w; + count += 1; + sum = sum + x[[ih, iw, ch]]; + } + } + + if with_pad { + count = kernel_height * kernel_width; + } + + out[[oh, ow, ch]] = sum / (count as u64).elem(); + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/base.rs new file mode 100644 index 0000000..005316f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/base.rs @@ -0,0 +1,115 @@ +use core::{marker::PhantomData, mem::MaybeUninit}; + +use macerator::{Arch, Scalar, Simd}; +use ndarray::{ArcArray, ArrayD, IxDyn, ShapeBuilder}; + +/// Whether SIMD instructions are worth using +#[cfg(all( + any( + target_arch = "x86", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "wasm32", + target_arch = "loongarch64" + ), + not(test) +))] +pub fn should_use_simd(len: usize) -> bool { + len >= 32 +} + +/// Whether SIMD instructions are worth using +#[cfg(all( + not(any( + target_arch = "x86", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "wasm32", + target_arch = "loongarch64" + )), + not(test) +))] +pub fn should_use_simd(_len: usize) -> bool { + false +} + +#[cfg(test)] +pub fn should_use_simd(_len: usize) -> bool { + true +} + +pub(crate) fn lanes() -> usize { + #[allow(non_camel_case_types)] + struct lanes<__T0>(__T0); + + impl ::macerator::WithSimd for lanes> { + type Output = usize; + #[inline(always)] + fn with_simd<__S: ::macerator::Simd>(self) -> ::Output { + let Self(__ty) = self; + #[allow(unused_unsafe)] + unsafe { + lanes_simd::<__S, E>(__ty) + } + } + } + (Arch::new()).dispatch(lanes(PhantomData::)) +} + +fn lanes_simd(_ty: PhantomData) -> usize { + E::lanes::() +} + +pub(crate) fn uninit_array_like(reference: &ArcArray) -> ArrayD { + let shape = reference.raw_dim(); + let strides = reference.strides(); + let strides = strides.iter().map(|it| *it as usize).collect::>(); + let shape_strides = shape.strides(IxDyn(&strides)); + let size = reference.len(); + let mut out_data: Vec> = Vec::with_capacity(size); + unsafe { out_data.set_len(size) }; + unsafe { ArrayD::from_shape_vec_unchecked(shape_strides, out_data).assume_init() } +} + +pub trait MinMax { + fn min(self, other: Self) -> Self; + fn max(self, other: Self) -> Self; +} + +macro_rules! impl_minmax { + ($ty: ty) => { + impl MinMax for $ty { + fn min(self, other: Self) -> Self { + Ord::min(self, other) + } + fn max(self, other: Self) -> Self { + Ord::max(self, other) + } + } + }; + ($($ty: ty),*) => { + $(impl_minmax!($ty);)* + } +} + +impl_minmax!(u8, i8, u16, i16, u32, i32, u64, i64); + +impl MinMax for f32 { + fn min(self, other: Self) -> Self { + self.min(other) + } + + fn max(self, other: Self) -> Self { + self.max(other) + } +} + +impl MinMax for f64 { + fn min(self, other: Self) -> Self { + self.min(other) + } + + fn max(self, other: Self) -> Self { + self.max(other) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary.rs new file mode 100644 index 0000000..dae3ed5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary.rs @@ -0,0 +1,299 @@ +use core::{marker::PhantomData, slice}; + +use burn_backend::Element; +use macerator::{ + Scalar, Simd, VAdd, VBitAnd, VBitOr, VBitXor, VDiv, VMul, VOrd, VSub, Vector, vload_unaligned, + vstore_unaligned, +}; +use ndarray::ArrayD; +use seq_macro::seq; + +use crate::{NdArrayElement, SharedArray, ops::simd::uninit_array_like}; + +use super::{ + MinMax, + binary_elemwise::{ + VecAdd, VecBitAnd, VecBitOr, VecBitXor, VecDiv, VecMax, VecMin, VecMul, VecSub, + }, + should_use_simd, +}; + +pub trait SimdBinop { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector; + fn apply(lhs: T, rhs: T) -> Out; + fn is_accelerated() -> bool; +} + +impl SimdBinop for VecAdd { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs + rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs + rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl SimdBinop for VecDiv { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs / rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs / rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl SimdBinop for VecMul { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs * rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs * rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl SimdBinop for VecSub { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs - rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs - rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl SimdBinop for VecMin { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs.min(rhs) + } + + fn apply(lhs: T, rhs: T) -> T { + MinMax::min(lhs, rhs) + } + + fn is_accelerated() -> bool { + ::is_min_max_accelerated::() + } +} + +impl SimdBinop for VecMax { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs.max(rhs) + } + + fn apply(lhs: T, rhs: T) -> T { + MinMax::max(lhs, rhs) + } + + fn is_accelerated() -> bool { + ::is_min_max_accelerated::() + } +} + +impl SimdBinop for VecBitAnd { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs & rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs.bitand(rhs) + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl SimdBinop for VecBitOr { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs | rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs.bitor(rhs) + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl SimdBinop for VecBitXor { + fn apply_vec(lhs: Vector, rhs: Vector) -> Vector { + lhs ^ rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs.bitxor(rhs) + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +#[macerator::with_simd] +fn is_accelerated>( + _x: PhantomData<(T, Out, Op)>, +) -> bool { + Op::is_accelerated::() +} + +#[allow(clippy::result_large_err)] +pub fn try_binary_simd< + E: Element, + EOut: Element, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdBinop, +>( + lhs: SharedArray, + rhs: SharedArray, +) -> Result, (SharedArray, SharedArray)> { + let lhs_len = lhs.len(); + let rhs_len = rhs.len(); + if !should_use_simd(lhs_len.max(rhs_len)) + || !lhs.is_standard_layout() + || !rhs.is_standard_layout() + || lhs.shape() != rhs.shape() + || !is_accelerated::(PhantomData) + { + return Err((lhs, rhs)); + } + // Used to assert traits based on the dynamic `DType`. + let lhs = unsafe { core::mem::transmute::, SharedArray>(lhs) }; + let rhs = unsafe { core::mem::transmute::, SharedArray>(rhs) }; + let out = binary_simd_same::(lhs, rhs); + + // Used to assert traits based on the dynamic `DType`. + let out = unsafe { core::mem::transmute::, SharedArray>(out) }; + Ok(out) +} + +fn binary_simd_same< + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdBinop, +>( + lhs: SharedArray, + rhs: SharedArray, +) -> SharedArray { + let out = if lhs.is_unique() { + let mut buf = lhs.into_owned(); + let lhs = buf.as_slice_mut().unwrap(); + let rhs = rhs.as_slice().unwrap(); + let out = + unsafe { core::mem::transmute::<&mut [T], &mut [Out]>(unsafe_alias_slice_mut(lhs)) }; + binary(lhs, rhs, out, PhantomData::); + unsafe { core::mem::transmute::, ArrayD>(buf) } + } else if rhs.is_unique() { + let mut buf = rhs.into_owned(); + let lhs = lhs.as_slice().unwrap(); + let rhs = buf.as_slice_mut().unwrap(); + let out = + unsafe { core::mem::transmute::<&mut [T], &mut [Out]>(unsafe_alias_slice_mut(rhs)) }; + binary(lhs, rhs, out, PhantomData::); + unsafe { core::mem::transmute::, ArrayD>(buf) } + } else { + let mut out = uninit_array_like(&lhs); + let lhs = lhs.as_slice().unwrap(); + let rhs = rhs.as_slice().unwrap(); + let out_slice = out.as_slice_mut().unwrap(); + binary(lhs, rhs, out_slice, PhantomData::); + out + }; + out.into_shared() +} + +#[allow(clippy::erasing_op, clippy::identity_op)] +#[macerator::with_simd] +fn binary< + 'a, + S: Simd, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdBinop, +>( + lhs: &'a [T], + rhs: &'a [T], + out: &'a mut [Out], + _op: PhantomData, +) where + 'a: 'a, +{ + let lanes = T::lanes::(); + let mut chunks_lhs = lhs.chunks_exact(8 * lanes); + let mut chunks_rhs = rhs.chunks_exact(8 * lanes); + let mut chunks_out = out.chunks_exact_mut(8 * lanes); + while let Some(((lhs, rhs), out)) = chunks_lhs + .next() + .zip(chunks_rhs.next()) + .zip(chunks_out.next()) + { + seq!(N in 0..8 { + // Load one full vector from `lhs`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let lhs~N = unsafe { vload_unaligned::(&lhs[N * lanes]) }; + // Load one full vector from `rhs`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let rhs~N = unsafe { vload_unaligned(&rhs[N * lanes]) }; + let s~N = Op::apply_vec(lhs~N, rhs~N); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + unsafe { vstore_unaligned(&mut out[N * lanes], s~N) }; + }); + } + let mut chunks_lhs = chunks_lhs.remainder().chunks_exact(lanes); + let mut chunks_rhs = chunks_rhs.remainder().chunks_exact(lanes); + let mut chunks_out = chunks_out.into_remainder().chunks_exact_mut(lanes); + while let Some(((lhs, rhs), out)) = chunks_lhs + .next() + .zip(chunks_rhs.next()) + .zip(chunks_out.next()) + { + // Load one full vector from `lhs`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let lhs0 = unsafe { vload_unaligned::(lhs.as_ptr()) }; + // Load one full vector from `rhs`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let rhs0 = unsafe { vload_unaligned(rhs.as_ptr()) }; + let s0 = Op::apply_vec(lhs0, rhs0); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + unsafe { vstore_unaligned(out.as_mut_ptr(), s0) }; + } + + for ((lhs, rhs), out) in chunks_lhs + .remainder() + .iter() + .zip(chunks_rhs.remainder()) + .zip(chunks_out.into_remainder()) + { + *out = Op::apply(*lhs, *rhs) + } +} + +/// Unsafely alias a slice to use as an inline argument +fn unsafe_alias_slice_mut<'a, T>(slice: &mut [T]) -> &'a mut [T] { + let ptr = slice.as_mut_ptr(); + let len = slice.len(); + unsafe { slice::from_raw_parts_mut(ptr, len) } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary_elemwise.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary_elemwise.rs new file mode 100644 index 0000000..7534da5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/binary_elemwise.rs @@ -0,0 +1,419 @@ +use core::marker::PhantomData; + +use bytemuck::cast; +use macerator::{ + Scalar, Simd, VAdd, VBitAnd, VBitOr, VBitXor, VDiv, VMul, VOrd, VSub, Vector, vload, + vload_unaligned, vstore, vstore_unaligned, +}; +use ndarray::ArrayD; +use seq_macro::seq; + +use crate::{NdArrayElement, SharedArray, ops::simd::uninit_array_like}; + +use super::{MinMax, should_use_simd}; + +pub trait ScalarSimdBinop { + type Rhs: Copy; + type RhsVec: Copy; + fn splat(rhs: Self::Rhs) -> Self::RhsVec; + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector; + fn apply(lhs: T, rhs: Self::Rhs) -> Out; + fn is_accelerated() -> bool; +} + +pub struct VecAdd; +pub struct VecDiv; +pub struct VecMul; +pub struct VecSub; +pub struct VecMin; +pub struct VecMax; +pub struct VecClamp; +pub struct VecBitAnd; +pub struct VecBitOr; +pub struct VecBitXor; + +impl ScalarSimdBinop for VecAdd { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs + rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs + rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl ScalarSimdBinop for VecDiv { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs / rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs / rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl ScalarSimdBinop for VecMul { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs * rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs * rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl ScalarSimdBinop for VecSub { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs - rhs + } + + fn apply(lhs: T, rhs: T) -> T { + lhs - rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl ScalarSimdBinop for VecMin { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs.min(rhs) + } + + fn apply(lhs: T, rhs: T) -> T { + lhs.min(rhs) + } + + fn is_accelerated() -> bool { + ::is_min_max_accelerated::() + } +} + +impl ScalarSimdBinop for VecMax { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs.max(rhs) + } + + fn apply(lhs: T, rhs: T) -> T { + lhs.max(rhs) + } + + fn is_accelerated() -> bool { + ::is_min_max_accelerated::() + } +} + +impl ScalarSimdBinop for VecClamp { + type Rhs = (T, T); + type RhsVec = (Vector, Vector); + + fn splat((min, max): Self::Rhs) -> Self::RhsVec { + (min.splat(), max.splat()) + } + + fn apply_vec(lhs: Vector, (min, max): Self::RhsVec) -> Vector { + lhs.min(max).max(min) + } + + fn apply(lhs: T, (min, max): Self::Rhs) -> T { + lhs.min(max).max(min) + } + + fn is_accelerated() -> bool { + ::is_min_max_accelerated::() + } +} + +impl ScalarSimdBinop for VecBitAnd { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs & rhs + } + + fn apply(lhs: T, rhs: Self::Rhs) -> T { + lhs & rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl ScalarSimdBinop for VecBitOr { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs | rhs + } + + fn apply(lhs: T, rhs: Self::Rhs) -> T { + lhs | rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +impl ScalarSimdBinop for VecBitXor { + type Rhs = T; + type RhsVec = Vector; + + fn splat(rhs: Self::Rhs) -> Self::RhsVec { + rhs.splat() + } + + fn apply_vec(lhs: Vector, rhs: Self::RhsVec) -> Vector { + lhs ^ rhs + } + + fn apply(lhs: T, rhs: Self::Rhs) -> T { + lhs ^ rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +#[macerator::with_simd] +fn is_accelerated>( + _x: PhantomData<(T, Out, Op)>, +) -> bool { + Op::is_accelerated::() +} + +pub fn try_binary_scalar_simd< + E: NdArrayElement, + EOut: NdArrayElement, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: ScalarSimdBinop, +>( + input: SharedArray, + elem: Op::Rhs, +) -> Result, SharedArray> { + if !should_use_simd(input.len()) + || input.as_slice_memory_order().is_none() + || !is_accelerated::(PhantomData) + { + return Err(input); + } + // Used to assert traits based on the dynamic `DType`. + let input = unsafe { core::mem::transmute::, SharedArray>(input) }; + let out = if size_of::() == size_of::() + && align_of::() >= align_of::() + && input.is_unique() + { + unsafe { binary_scalar_simd_inplace::(input, elem) } + } else { + binary_scalar_simd_owned::(input, elem) + }; + // Used to assert traits based on the dynamic `DType`. + let out = unsafe { core::mem::transmute::, SharedArray>(out) }; + Ok(out) +} + +/// Execute operation in place on an owned tensor +/// SAFETY: +/// Must ensure `size_of:: == size_of::` and `align_of:: >= align_of::`. +unsafe fn binary_scalar_simd_inplace< + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: ScalarSimdBinop, +>( + input: SharedArray, + elem: Op::Rhs, +) -> SharedArray { + let mut buffer = input.into_owned(); + let slice = buffer.as_slice_memory_order_mut().unwrap(); + unsafe { binary_scalar_slice_inplace::(slice, elem, PhantomData) }; + // Buffer has the same elem size and is filled with the operation output, so this is safe + let out = unsafe { core::mem::transmute::, ArrayD>(buffer) }; + out.into_shared() +} + +/// Create a new copy of the tensor as the output +fn binary_scalar_simd_owned< + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: ScalarSimdBinop, +>( + input: SharedArray, + elem: Op::Rhs, +) -> SharedArray { + let mut out = uninit_array_like(&input); + let input = input.as_slice_memory_order().unwrap(); + let out_slice = out.as_slice_memory_order_mut().unwrap(); + binary_scalar_slice::(input, out_slice, elem, PhantomData); + out.into_shared() +} + +#[inline(always)] +#[allow(clippy::erasing_op, clippy::identity_op)] +#[macerator::with_simd] +fn binary_scalar_slice< + 'a, + S: Simd, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: ScalarSimdBinop, +>( + input: &'a [T], + out: &'a mut [Out], + rhs: Op::Rhs, + _op: PhantomData, +) where + 'a: 'a, +{ + let lanes = T::lanes::(); + let mut chunks_input = input.chunks_exact(8 * lanes); + let mut chunks_out = out.chunks_exact_mut(8 * lanes); + let rhs_vec = Op::splat::(rhs); + while let Some((input, out)) = chunks_input.next().zip(chunks_out.next()) { + seq!(N in 0..8 { + // Load one full vector from `input`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let s~N = unsafe { vload_unaligned(&input[N * lanes]) }; + let s~N = Op::apply_vec(s~N, rhs_vec); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + unsafe { vstore_unaligned(&mut out[N * lanes], s~N) }; + }); + } + let mut chunks_input = chunks_input.remainder().chunks_exact(lanes); + let mut chunks_out = chunks_out.into_remainder().chunks_exact_mut(lanes); + while let Some((input, out)) = chunks_input.next().zip(chunks_out.next()) { + // Load one full vector from `input`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let s0 = unsafe { vload_unaligned(input.as_ptr()) }; + let s0 = Op::apply_vec(s0, rhs_vec); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + unsafe { vstore_unaligned(out.as_mut_ptr(), s0) }; + } + + for (input, out) in chunks_input + .remainder() + .iter() + .zip(chunks_out.into_remainder()) + { + *out = Op::apply(*input, rhs) + } +} + +/// Execute operation in line. +/// SAFETY: +/// Must ensure `size_of:: == size_of::` and `align_of:: >= align_of::`. +#[inline(always)] +#[macerator::with_simd] +unsafe fn binary_scalar_slice_inplace< + 'a, + S: Simd, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: ScalarSimdBinop, +>( + buf: &'a mut [T], + rhs: Op::Rhs, + _op: PhantomData<(Out, Op)>, +) where + 'a: 'a, +{ + let (head, main, tail) = unsafe { buf.align_to_mut::>() }; + for elem in head.iter_mut().chain(tail) { + *elem = cast(Op::apply(*elem, rhs)); + } + let mut chunks = main.chunks_exact_mut(8); + let rhs = Op::splat::(rhs); + for elem in chunks.by_ref() { + seq!(N in 0..8 { + // Load a full vector from the aligned portion of the buffer. + // SAFETY: `align_to_mut` guarantees we're aligned to `T::Vector`'s size, and there is + // always a full vector in bounds. + let s~N = unsafe { vload(&elem[N] as *const _ as *const T) }; + let s~N = Op::apply_vec(s~N, rhs); + // Store a full vector at the same position as the input. Cast is safe because `Out` is + // size and align compatible + unsafe { vstore_unaligned(&mut elem[N] as *mut _ as *mut Out, s~N) }; + }); + } + + for elem in chunks.into_remainder() { + // Load a full vector from the aligned portion of the buffer. + // SAFETY: `align_to_mut` guarantees we're aligned to `T::Vector`'s size, and there is + // always a full vector in bounds. + let s0 = unsafe { vload(elem as *const _ as *const T) }; + + let s0 = Op::apply_vec(s0, rhs); + // Store a full vector at the same position as the input. Cast is safe because `Out` is + // size and align compatible + unsafe { vstore(elem as *mut _ as *mut Out, s0) }; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/cmp.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/cmp.rs new file mode 100644 index 0000000..c9f8c0e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/cmp.rs @@ -0,0 +1,374 @@ +use core::{marker::PhantomData, slice}; + +use burn_backend::Element; +use macerator::{Mask, Scalar, Simd, VEq, VOrd, Vector, vload_unaligned}; +use ndarray::ArrayD; +use seq_macro::seq; + +use crate::{NdArrayElement, SharedArray, ops::simd::uninit_array_like}; + +use super::should_use_simd; + +pub trait SimdCmpOp { + fn apply_vec(lhs: Vector, rhs: Vector) -> Mask; + fn apply(lhs: T, rhs: T) -> bool; + fn is_accelerated() -> bool; +} + +pub struct VecEquals; + +impl SimdCmpOp for VecEquals { + fn apply_vec(lhs: Vector, rhs: Vector) -> Mask { + lhs.eq(rhs) + } + + fn apply(lhs: T, rhs: T) -> bool { + lhs == rhs + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +pub struct VecGreater; + +impl SimdCmpOp for VecGreater { + fn apply_vec(lhs: Vector, rhs: Vector) -> Mask { + lhs.gt(rhs) + } + + fn apply(lhs: T, rhs: T) -> bool { + lhs > rhs + } + + fn is_accelerated() -> bool { + ::is_cmp_accelerated::() + } +} + +pub struct VecGreaterEq; + +impl SimdCmpOp for VecGreaterEq { + fn apply_vec(lhs: Vector, rhs: Vector) -> Mask { + lhs.ge(rhs) + } + + fn apply(lhs: T, rhs: T) -> bool { + lhs >= rhs + } + + fn is_accelerated() -> bool { + ::is_cmp_accelerated::() + } +} + +pub struct VecLowerEq; + +impl SimdCmpOp for VecLowerEq { + fn apply_vec(lhs: Vector, rhs: Vector) -> Mask { + lhs.le(rhs) + } + + fn apply(lhs: T, rhs: T) -> bool { + lhs <= rhs + } + + fn is_accelerated() -> bool { + ::is_cmp_accelerated::() + } +} + +pub struct VecLower; + +impl SimdCmpOp for VecLower { + fn apply_vec(lhs: Vector, rhs: Vector) -> Mask { + lhs.lt(rhs) + } + + fn apply(lhs: T, rhs: T) -> bool { + lhs < rhs + } + + fn is_accelerated() -> bool { + ::is_cmp_accelerated::() + } +} + +#[macerator::with_simd] +fn is_accelerated>(_x: PhantomData<(T, Op)>) -> bool { + Op::is_accelerated::() +} + +#[allow(clippy::result_large_err)] +pub fn try_cmp_simd>( + lhs: SharedArray, + rhs: SharedArray, +) -> Result, (SharedArray, SharedArray)> { + let lhs_len = lhs.len(); + let rhs_len = rhs.len(); + if !should_use_simd(lhs_len.max(rhs_len)) + || !lhs.is_standard_layout() + || !rhs.is_standard_layout() + || lhs.shape() != rhs.shape() + || !is_accelerated::(PhantomData) + { + return Err((lhs, rhs)); + } + // Used to assert traits based on the dynamic `DType`. + let lhs = unsafe { core::mem::transmute::, SharedArray>(lhs) }; + let rhs = unsafe { core::mem::transmute::, SharedArray>(rhs) }; + let out = cmp_simd_same::(lhs, rhs); + + Ok(out) +} + +fn cmp_simd_same>( + lhs: SharedArray, + rhs: SharedArray, +) -> SharedArray { + let out = if lhs.is_unique() && size_of::() == size_of::() { + let mut buf = lhs.into_owned(); + let lhs = buf.as_slice_mut().unwrap(); + let rhs = rhs.as_slice().unwrap(); + let out = + unsafe { core::mem::transmute::<&mut [T], &mut [bool]>(unsafe_alias_slice_mut(lhs)) }; + cmp(lhs, rhs, out, PhantomData::); + unsafe { core::mem::transmute::, ArrayD>(buf) } + } else if rhs.is_unique() && size_of::() == size_of::() { + let mut buf = rhs.into_owned(); + let lhs = lhs.as_slice().unwrap(); + let rhs = buf.as_slice_mut().unwrap(); + let out = + unsafe { core::mem::transmute::<&mut [T], &mut [bool]>(unsafe_alias_slice_mut(rhs)) }; + cmp(lhs, rhs, out, PhantomData::); + unsafe { core::mem::transmute::, ArrayD>(buf) } + } else { + let mut out = uninit_array_like(&lhs); + let lhs = lhs.as_slice().unwrap(); + let rhs = rhs.as_slice().unwrap(); + let out_slice = out.as_slice_mut().unwrap(); + cmp(lhs, rhs, out_slice, PhantomData::); + out + }; + out.into_shared() +} + +#[allow(clippy::erasing_op, clippy::identity_op)] +#[macerator::with_simd] +fn cmp<'a, S: Simd, T: NdArrayElement + Scalar, Op: SimdCmpOp>( + lhs: &'a [T], + rhs: &'a [T], + out: &'a mut [bool], + _op: PhantomData, +) where + 'a: 'a, +{ + let lanes = T::lanes::(); + let mut chunks_lhs = lhs.chunks_exact(8 * lanes); + let mut chunks_rhs = rhs.chunks_exact(8 * lanes); + let mut chunks_out = out.chunks_exact_mut(8 * lanes); + while let Some(((lhs, rhs), out)) = chunks_lhs + .next() + .zip(chunks_rhs.next()) + .zip(chunks_out.next()) + { + seq!(N in 0..8 { + // Load one full vector from `lhs`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let lhs~N = unsafe { vload_unaligned::(&lhs[N * lanes]) }; + // Load one full vector from `rhs`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let rhs~N = unsafe { vload_unaligned(&rhs[N * lanes]) }; + let s~N = Op::apply_vec(lhs~N, rhs~N); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + unsafe { T::mask_store_as_bool(&mut out[N * lanes], s~N) }; + }); + } + let mut chunks_lhs = chunks_lhs.remainder().chunks_exact(lanes); + let mut chunks_rhs = chunks_rhs.remainder().chunks_exact(lanes); + let mut chunks_out = chunks_out.into_remainder().chunks_exact_mut(lanes); + while let Some(((lhs, rhs), out)) = chunks_lhs + .next() + .zip(chunks_rhs.next()) + .zip(chunks_out.next()) + { + // Load one full vector from `lhs`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let lhs0 = unsafe { vload_unaligned::(lhs.as_ptr()) }; + // Load one full vector from `rhs`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let rhs0 = unsafe { vload_unaligned(rhs.as_ptr()) }; + let s0 = Op::apply_vec(lhs0, rhs0); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + unsafe { T::mask_store_as_bool(out.as_mut_ptr(), s0) }; + } + + for ((lhs, rhs), out) in chunks_lhs + .remainder() + .iter() + .zip(chunks_rhs.remainder()) + .zip(chunks_out.into_remainder()) + { + *out = Op::apply(*lhs, *rhs) + } +} + +/// Unsafely alias a slice to use as an inline argument +fn unsafe_alias_slice_mut<'a, T>(slice: &mut [T]) -> &'a mut [T] { + let ptr = slice.as_mut_ptr(); + let len = slice.len(); + unsafe { slice::from_raw_parts_mut(ptr, len) } +} + +pub use elemwise::try_cmp_scalar_simd; + +mod elemwise { + use bytemuck::cast; + use macerator::vload; + + use super::*; + + pub fn try_cmp_scalar_simd>( + input: SharedArray, + elem: T, + ) -> Result, SharedArray> { + if !should_use_simd(input.len()) + || input.as_slice_memory_order().is_none() + || !is_accelerated::(PhantomData) + { + return Err(input); + } + // Used to assert traits based on the dynamic `DType`. + let input = unsafe { core::mem::transmute::, SharedArray>(input) }; + let out = if size_of::() == size_of::() + && align_of::() >= align_of::() + && input.is_unique() + { + unsafe { cmp_scalar_simd_inplace::(input, elem) } + } else { + cmp_scalar_simd_owned::(input, elem) + }; + Ok(out) + } + + /// Execute operation in place on an owned tensor + /// SAFETY: + /// Must ensure `size_of:: == size_of::` and `align_of:: >= align_of::`. + unsafe fn cmp_scalar_simd_inplace>( + input: SharedArray, + elem: T, + ) -> SharedArray { + let mut buffer = input.into_owned(); + let slice = buffer.as_slice_memory_order_mut().unwrap(); + unsafe { cmp_scalar_slice_inplace::(slice, elem, PhantomData) }; + // Buffer has the same elem size and is filled with the operation output, so this is safe + let out = unsafe { core::mem::transmute::, ArrayD>(buffer) }; + out.into_shared() + } + + /// Create a new copy of the tensor as the output + fn cmp_scalar_simd_owned>( + input: SharedArray, + elem: T, + ) -> SharedArray { + let mut out = uninit_array_like(&input); + let input = input.as_slice_memory_order().unwrap(); + let out_slice = out.as_slice_memory_order_mut().unwrap(); + cmp_scalar_slice::(input, out_slice, elem, PhantomData); + out.into_shared() + } + + #[inline(always)] + #[allow(clippy::erasing_op, clippy::identity_op)] + #[macerator::with_simd] + fn cmp_scalar_slice<'a, S: Simd, T: NdArrayElement + Scalar, Op: SimdCmpOp>( + input: &'a [T], + out: &'a mut [bool], + rhs: T, + _op: PhantomData, + ) where + 'a: 'a, + { + let lanes = T::lanes::(); + let mut chunks_input = input.chunks_exact(8 * lanes); + let mut chunks_out = out.chunks_exact_mut(8 * lanes); + let rhs_vec = rhs.splat::(); + while let Some((input, out)) = chunks_input.next().zip(chunks_out.next()) { + seq!(N in 0..8 { + // Load one full vector from `input`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let s~N = unsafe { vload_unaligned(&input[N * lanes]) }; + let s~N = Op::apply_vec(s~N, rhs_vec); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + unsafe { T::mask_store_as_bool(&mut out[N * lanes], s~N) }; + }); + } + let mut chunks_input = chunks_input.remainder().chunks_exact(lanes); + let mut chunks_out = chunks_out.into_remainder().chunks_exact_mut(lanes); + while let Some((input, out)) = chunks_input.next().zip(chunks_out.next()) { + // Load one full vector from `input`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let s0 = unsafe { vload_unaligned(input.as_ptr()) }; + let s0 = Op::apply_vec(s0, rhs_vec); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + unsafe { T::mask_store_as_bool(out.as_mut_ptr(), s0) }; + } + + for (input, out) in chunks_input + .remainder() + .iter() + .zip(chunks_out.into_remainder()) + { + *out = Op::apply(*input, rhs) + } + } + + /// Execute operation in line. + /// SAFETY: + /// Must ensure `size_of:: == size_of::` and `align_of:: >= align_of::`. + #[inline(always)] + #[macerator::with_simd] + unsafe fn cmp_scalar_slice_inplace<'a, S: Simd, T: NdArrayElement + Scalar, Op: SimdCmpOp>( + buf: &'a mut [T], + rhs: T, + _op: PhantomData, + ) where + 'a: 'a, + { + let (head, main, tail) = unsafe { buf.align_to_mut::>() }; + for elem in head.iter_mut().chain(tail) { + *elem = cast(Op::apply(*elem, rhs)); + } + let mut chunks = main.chunks_exact_mut(8); + let rhs = rhs.splat::(); + for elem in chunks.by_ref() { + seq!(N in 0..8 { + // Load a full vector from the aligned portion of the buffer. + // SAFETY: `align_to_mut` guarantees we're aligned to `T::Vector`'s size, and there is + // always a full vector in bounds. + let s~N = unsafe { vload(&elem[N] as *const _ as *const T) }; + let s~N = Op::apply_vec(s~N, rhs); + // Store a full vector at the same position as the input. Cast is safe because `Out` is + // size and align compatible + unsafe { T::mask_store_as_bool(&mut elem[N] as *mut _ as *mut bool, s~N) }; + }); + } + + for elem in chunks.into_remainder() { + // Load a full vector from the aligned portion of the buffer. + // SAFETY: `align_to_mut` guarantees we're aligned to `T::Vector`'s size, and there is + // always a full vector in bounds. + let s0 = unsafe { vload(elem as *const _ as *const T) }; + + let s0 = Op::apply_vec(s0, rhs); + // Store a full vector at the same position as the input. Cast is safe because `Out` is + // size and align compatible + unsafe { T::mask_store_as_bool(elem as *mut _ as *mut bool, s0) }; + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/conv.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/conv.rs new file mode 100644 index 0000000..5bbd463 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/conv.rs @@ -0,0 +1,494 @@ +use core::{marker::PhantomData, mem::transmute}; + +use burn_backend::{ + DType, Element, + ops::{ConvOptions, conv::calculate_conv_output_size}, +}; +use bytemuck::Zeroable; +use macerator::{Simd, VMulAdd, Vector, vload_unaligned, vstore_unaligned}; +use ndarray::{ + ArcArray1, Array4, ArrayView3, ArrayView4, ArrayViewMut2, ArrayViewMut3, Dim, Ix1, Ix4, s, +}; +use seq_macro::seq; + +use crate::{FloatNdArrayElement, SharedArray, UnsafeSharedRef, iter_range_par, run_par}; + +type Args = (SharedArray, SharedArray, Option>); + +#[allow(clippy::result_large_err)] +pub fn try_conv2d_simd( + x: SharedArray, + weight: SharedArray, + bias: Option>, + options: ConvOptions<2>, +) -> Result, Args> { + match E::dtype() { + DType::F64 => conv2d::(x, weight, bias, options, PhantomData), + DType::F32 => conv2d::(x, weight, bias, options, PhantomData), + DType::I64 => conv2d::(x, weight, bias, options, PhantomData), + DType::I32 => conv2d::(x, weight, bias, options, PhantomData), + DType::I16 => conv2d::(x, weight, bias, options, PhantomData), + DType::U64 => conv2d::(x, weight, bias, options, PhantomData), + DType::U32 => conv2d::(x, weight, bias, options, PhantomData), + DType::U16 => conv2d::(x, weight, bias, options, PhantomData), + _ => Err((x, weight, bias)), + } +} + +fn cast(tensor: SharedArray) -> SharedArray { + unsafe { transmute::, SharedArray>(tensor) } +} + +/// Out-channel last SIMD accelerated direct convolution. Loop order and register blocking based on +/// E. Georganas, S. Avancha, K. Banerjee, D. Kalamkar, G. Henry, H. Pabst, A. Heinecke (2018). +/// Anatomy Of High-Performance Deep Learning Convolutions On SIMD Architectures. +/// SC '18, Article 6, pp. 1-12. arXiv:1808.05567. . +#[allow(clippy::result_large_err)] +fn conv2d( + x: SharedArray, + weight: SharedArray, + bias: Option>, + options: ConvOptions<2>, + _ty: PhantomData, +) -> Result, Args> { + let [out_channels, _, k_height, k_width] = weight.shape().try_into().unwrap(); + let channels_per_group = out_channels / options.groups; + + #[macerator::with_simd] + fn precheck(_ty: PhantomData) -> (usize, bool) { + (E::lanes::(), E::is_accelerated::()) + } + + let (lanes, accelerated) = precheck::(PhantomData); + + if !accelerated || !channels_per_group.is_multiple_of(lanes) { + return Err((x, weight, bias)); + } + + let x = cast::<_, E>(x); + let weight = cast::<_, E>(weight); + let bias = bias.map(|bias| cast::<_, E>(bias)); + + let [batch_size, _in_channels, in_height, in_width] = x.shape().try_into().unwrap(); + let [dilate_h, dilate_w] = options.dilation; + let [stride_h, stride_w] = options.stride; + let [pad_h, pad_w] = options.padding; + let padded = options.padding != [0, 0]; + let strided = options.stride != [1, 1] || options.dilation != [1, 1]; + let grouped = options.groups != 1; + + let out_height = calculate_conv_output_size(k_height, stride_h, pad_h, dilate_h, in_height); + let out_width = calculate_conv_output_size(k_width, stride_w, pad_w, dilate_w, in_width); + + let x = x.into_dimensionality::().unwrap(); + let weights = weight.into_dimensionality::().unwrap(); + let weights = weights.permuted_axes([1, 2, 3, 0]); + let weights = weights.as_standard_layout(); + let bias = bias.map(|bias| bias.into_dimensionality::().unwrap()); + // floor division means `(oc_blocks - 1) * lanes` can never be greater than `out_channels - lanes`. + let oc_blocks = out_channels / lanes; + + let mut out = unsafe { + Array4::::uninit(Dim([batch_size, out_height, out_width, out_channels])).assume_init() + }; + let unsafe_shared_out = UnsafeSharedRef::new(&mut out); + + run_par!(|| { + // SAFETY: Slices are guaranteed to be non-overlapping, so having an unsafe shared reference + // is safe. `oc_blocks * lanes` must be `<= out_channels` to satisfy safety of inner function. + iter_range_par!(0, batch_size * oc_blocks).for_each(|k| unsafe { + let b = k / oc_blocks; + let ob = k % oc_blocks; + let x = x.slice(s![b, .., .., ..]); + let out = unsafe_shared_out.get(); + let mut out = out.slice_mut(s![b, .., .., ..]); + let w = weights.view(); + + match (padded, strided, grouped) { + (true, true, true) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (true, false, true) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (false, true, true) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (false, false, true) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (true, true, false) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (true, false, false) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (false, true, false) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + (false, false, false) => { + conv2d_launch::(x, w, &bias, &mut out, &options, ob) + } + } + }); + }); + + let output = out.permuted_axes([0, 3, 1, 2]); + Ok(cast(output.into_dyn().into_shared())) +} + +/// Size of register blocks, we need to hardcode this because Rust and the `seq` macro don't support +/// using associated constants as constant parameters. 8 works for all semi-modern CPUs but might +/// not be perfectly optimized for AVX-512 capable CPUs (which probably should use 16). +/// This should always be conservative, since oversizing it will cause register spills and that's +/// **much** worse than the performance lost with lower values. +const REGISTER_BLOCK: usize = 8; +inner_with_register_blocking_size!(8); + +/// Run a loop of conv2d. +/// # SAFETY +/// See `conv2d_inner_nopad`, `conv2d_inner_nopad_nostride`, `conv2d_remainder`. +/// Required preconditions: `ob * simd_lanes` must be `<= out_channels - simd_lanes`, `weights` and +/// `out` must have unit stride for the out channels. +#[inline(always)] +#[macerator::with_simd] +unsafe fn conv2d_launch< + 'a, + S: Simd, + E: VMulAdd, + const PAD: bool, + const STRIDE: bool, + const GROUPS: bool, +>( + x: ArrayView3<'a, E>, + weights: ArrayView4<'a, E>, + bias: &'a Option>, + out: &'a mut ArrayViewMut3<'a, E>, + options: &'a ConvOptions<2>, + ob: usize, +) where + 'a: 'a, +{ + let (in_channels, k_height, k_width, out_channels) = weights.dim(); + let (out_height, out_width, _) = out.dim(); + let channels_per_group = out_channels / options.groups; + let lanes = E::lanes::(); + + let [mut pad_h, mut pad_w] = options.padding; + let [stride_h, stride_w] = options.stride; + let [dilate_h, dilate_w] = options.dilation; + + // Trick compiler into inlining 0 to padding + if !PAD { + pad_h = 0; + pad_w = 0; + } + + let oc_b = channels_per_group.min(lanes); + let ow_b = REGISTER_BLOCK; + + let ow_start = pad_w; + let ow_width = out_width.saturating_sub(2 * pad_w); + let oh_start = pad_h; + let oh_end = out_height.saturating_sub(pad_h); + + let ow_blocks = ow_width / ow_b; + let oc = ob * oc_b; + let group = oc / channels_per_group; + let mut ic_off = group * in_channels; + if !GROUPS { + ic_off = 0; + } + + unsafe { + let bias = if let Some(bias) = &bias { + vload_unaligned::(&bias[oc]) + } else { + Zeroable::zeroed() + }; + + for oh in oh_start..oh_end { + let mut out = out.slice_mut(s![oh, .., ..]); + for ow_block in 0..ow_blocks { + let ow = ow_block * ow_b + ow_start; + + #[allow(clippy::if_same_then_else)] + if STRIDE { + conv2d_inner_nopad( + &x, &weights, &mut out, bias, oh, ow, oc, ic_off, stride_h, stride_w, + dilate_h, dilate_w, k_height, k_width, pad_h, pad_w, + ); + } else { + conv2d_inner_nopad_nostride( + &x, &weights, &mut out, bias, oh, ow, oc, ic_off, k_height, k_width, pad_h, + pad_w, + ); + } + } + } + conv2d_remainder( + x, + weights, + out, + bias, + oc, + ic_off, + ow_blocks * ow_b, + stride_h, + stride_w, + dilate_h, + dilate_w, + pad_h, + pad_w, + k_height, + k_width, + ); + } +} + +/// Execute the non-unrolled and/or padded portion of the convolution. This has more checks and is +/// much slower, so we want to minimize the amount of pixels that need to be processed by this +/// +/// SAFETY: `oc` must be an index that's at most `out_channels - simd_lanes`, so the full vector +/// is in bounds. Weights and `out` must be channels last (with `stride == 1`). +#[allow(clippy::too_many_arguments)] +#[inline(always)] +unsafe fn conv2d_remainder( + x: ArrayView3, + weights: ArrayView4, + out: &mut ArrayViewMut3, + bias: Vector, + oc: usize, + ic_off: usize, + owb_end: usize, + stride_h: usize, + stride_w: usize, + dilate_h: usize, + dilate_w: usize, + pad_h: usize, + pad_w: usize, + k_height: usize, + k_width: usize, +) { + let in_channels = weights.shape()[0]; + let (_, in_height, in_width) = x.dim(); + let (out_height, out_width, _) = out.dim(); + let oh_start = pad_h; + let oh_end = out_height.saturating_sub(pad_h); + let ow_start = pad_w; + + let height1 = in_height + pad_h; + let width1 = in_width + pad_w; + + for oh in (0..oh_start).chain(oh_end..out_height) { + for ow in 0..out_width { + let mut acc = bias; + + for ic in 0..in_channels { + for kh in 0..k_height { + let ih = oh * stride_h + kh * dilate_h; + if (ih < pad_h) | (ih >= height1) { + continue; + } + let ih = ih - pad_h; + + for kw in 0..k_width { + let iw = ow * stride_w + kw * dilate_w; + if (iw < pad_w) | (iw >= width1) { + continue; + } + let iw = iw - pad_w; + + // Load a full vector from the weights. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and out channels are last. + // We need to ensure the weights are reshaped appropriately. + let f0 = unsafe { vload_unaligned(&weights[[ic, kh, kw, oc]]) }; + + // The loop bounds ensure `ic`, `ih` and `iw` are always in bounds, but the + // compiler can't prove this. We can't use `as_slice` with fixed bounds + // because we want to support arbitrary input layouts. So an unchecked load + // is used. + let i0 = unsafe { x.uget([ic, ih, iw]) }.splat::(); + acc = i0.mul_add(f0, acc); + } + } + } + + // Store a full vector from the output. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and oc stride is 1. We create `out` with + // channels last, so this always holds. + unsafe { vstore_unaligned(&mut out[[oh, ow, oc]], acc) }; + } + } + for ow in (0..ow_start).chain(owb_end..out_width) { + for oh in 0..out_height { + let mut acc = bias; + + for ic in 0..in_channels { + for kh in 0..k_height { + let ih = oh * stride_h + kh * dilate_h; + if (ih < pad_h) | (ih >= height1) { + continue; + } + let ih = ih - pad_h; + + for kw in 0..k_width { + let iw = ow * stride_w + kw * dilate_w; + if (iw < pad_w) | (iw >= width1) { + continue; + } + let iw = iw - pad_w; + + // Load a full vector from the weights. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and out channels are last. + // We need to ensure the weights are reshaped appropriately. + let f0 = unsafe { vload_unaligned(&weights[[ic, kh, kw, oc]]) }; + + // The loop bounds ensure `ic`, `ih` and `iw` are always in bounds, but the + // compiler can't prove this. We can't use `as_slice` with fixed bounds + // because we want to support arbitrary input layouts. So an unchecked load + // is used. + let i0 = unsafe { x.uget([ic_off + ic, ih, iw]) }.splat::(); + acc = i0.mul_add(f0, acc); + } + } + } + + // Store a full vector from the output. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and oc stride is 1. We create `out` with + // channels last, so this always holds. + unsafe { vstore_unaligned(&mut out[[oh, ow, oc]], acc) }; + } + } +} + +macro_rules! inner_with_register_blocking_size { + ($rb: literal) => { + /// Execute the unrolled and unpadded portion of the convolution. Any pixel that is more than + /// `pad_h` away from the horizontal border, and `pad_w` away from the vertical border is + /// guaranteed to always be in bounds (because of the way out size is calculated). + /// + /// SAFETY: `oc` must be an index that's at most `out_channels - simd_lanes`, so the full vector + /// is in bounds. Weights and `out` must be channels last (with `stride == 1`). + #[allow(clippy::erasing_op, clippy::identity_op, clippy::too_many_arguments)] + #[inline(always)] + unsafe fn conv2d_inner_nopad( + x: &ArrayView3, + weights: &ArrayView4, + out: &mut ArrayViewMut2, + bias: Vector, + oh: usize, + ow: usize, + oc: usize, + ic_off: usize, + stride_h: usize, + stride_w: usize, + dilate_h: usize, + dilate_w: usize, + k_height: usize, + k_width: usize, + pad_h: usize, + pad_w: usize, + ) { + let in_channels = weights.shape()[0]; + + seq!(N in 0..$rb { + let mut acc~N = bias; + }); + + for ic in 0..in_channels { + for kh in 0..k_height { + let ih = oh * stride_h + kh * dilate_h - pad_h; + + for kw in 0..k_width { + // Load a full vector from the weights. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and out channels are last. + // We need to ensure the weights are reshaped appropriately. + let f0 = unsafe { vload_unaligned(&weights[[ic, kh, kw, oc]]) }; + let iw = ow * stride_w + kw * dilate_w - pad_w; + + seq!(N in 0..$rb { + // The loop bounds ensure `ic`, `ih` and `iw` are always in bounds, but the + // compiler can't prove this. We can't use `as_slice` with fixed bounds + // because we want to support arbitrary input layouts. So an unchecked load + // is used. + let i~N = unsafe { x.uget([ic + ic_off, ih, iw + N * stride_w]) }.splat::(); + }); + seq!(N in 0..$rb { + acc~N = i~N.mul_add(f0, acc~N); + }); + } + } + } + + seq!(N in 0..$rb { + // Store a full vector from the output. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and oc stride is 1. We create `out` with + // channels last, so this always holds. + unsafe { vstore_unaligned(&mut out[[ow + N, oc]], acc~N) }; + }); + } + + /// Execute the unrolled and unpadded portion of the convolution. Any pixel that is more than + /// `pad_h` away from the horizontal border, and `pad_w` away from the vertical border is + /// guaranteed to always be in bounds (because of the way out size is calculated). + /// + /// SAFETY: `oc` must be an index that's at most `out_channels - simd_lanes`, so the full vector + /// is in bounds. Weights and `out` must be channels last (with `stride == 1`). + #[allow(clippy::erasing_op, clippy::identity_op, clippy::too_many_arguments)] + #[inline(always)] + unsafe fn conv2d_inner_nopad_nostride( + x: &ArrayView3, + weights: &ArrayView4, + out: &mut ArrayViewMut2, + bias: Vector, + oh: usize, + ow: usize, + oc: usize, + ic_off: usize, + k_height: usize, + k_width: usize, + pad_h: usize, + pad_w: usize, + ) { + let in_channels = weights.shape()[0]; + + seq!(N in 0..$rb { + let mut acc~N = bias; + }); + + for ic in 0..in_channels { + for kh in 0..k_height { + let ih = oh + kh - pad_h; + + for kw in 0..k_width { + // Load a full vector from the weights. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and out channels are last. + // We need to ensure the weights are reshaped appropriately. + let f0 = unsafe { vload_unaligned(&weights[[ic, kh, kw, oc]]) }; + let iw = ow + kw - pad_w; + + seq!(N in 0..$rb { + // The loop bounds ensure `ic`, `ih` and `iw` are always in bounds, but the + // compiler can't prove this. We can't use `as_slice` with fixed bounds + // because we want to support arbitrary input layouts. So an unchecked load + // is used. + let i~N = unsafe { x.uget([ic + ic_off, ih, iw + N]) }.splat::(); + }); + seq!(N in 0..$rb { + acc~N = i~N.mul_add(f0, acc~N); + }); + } + } + } + + seq!(N in 0..$rb { + // Store a full vector from the output. This is guaranteed to be in bounds + // as long as `oc <= out_channels - simd_lanes` and oc stride is 1. We create `out` with + // channels last, so this always holds. + unsafe { vstore_unaligned(&mut out[[ow + N, oc]], acc~N) }; + }); + } + }; +} +pub(crate) use inner_with_register_blocking_size; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/maxpool.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/maxpool.rs new file mode 100644 index 0000000..9f4b206 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/maxpool.rs @@ -0,0 +1,394 @@ +use core::{marker::PhantomData, mem::transmute}; + +use crate::{SharedArray, iter_range_par, run_par, sharing::UnsafeSharedRef}; + +use burn_backend::{DType, Element, quantization::QuantValue}; +use macerator::{Simd, VOrd}; +use ndarray::{Array4, s}; +use nhwc::max_pool2d_nhwc; + +use super::{MinMax, should_use_simd}; + +#[macerator::with_simd] +fn is_accelerated_impl(_x: PhantomData) -> bool { + ::is_min_max_accelerated::() +} + +fn is_accelerated() -> bool { + is_accelerated_impl::(PhantomData) +} + +macro_rules! launch_kernel { + ($ty: ty, $func: ident, $x: expr, $($arg: expr),*) => { + match <$ty as Element>::dtype() { + DType::F64 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::F32 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::I64 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::I32 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::I16 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::I8 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::U64 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::U32 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::U16 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::U8 if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::Bool if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + DType::QFloat(scheme) => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S if is_accelerated::() => Ok(cast($func::(cast($x), $($arg),*))), + _ => Err($x) + }, + _ => Err($x), + } + }; +} + +pub(crate) fn try_max_pool2d_simd( + x: SharedArray, + ksize: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], +) -> Result, SharedArray> { + let [_, c, _, _] = x.shape().try_into().unwrap(); + if !should_use_simd(c) || x.strides()[1] != 1 { + return Err(x); + } + + launch_kernel!(E, max_pool2d_nhwc, x, ksize, stride, padding, dilation) +} + +fn cast(tensor: SharedArray) -> SharedArray { + unsafe { transmute::, SharedArray>(tensor) } +} + +mod nhwc { + use itertools::Itertools; + use macerator::{Simd, vload_unaligned, vstore_unaligned}; + use ndarray::{ArrayView3, ArrayViewMut3, Ix4}; + use seq_macro::seq; + + use crate::ops::simd::lanes; + + use super::*; + + // Until you can use associated constants as array size, we need to hardcode this. + // The most common config (x86-v3) has 16 registers, so use half of them for accumulators. + const BLOCK_REGISTERS: usize = 8; + + pub(crate) fn max_pool2d_nhwc( + x: SharedArray, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ) -> SharedArray { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + let [dilation_height, dilation_width] = dilation; + let [batch_size, channels, x_height, x_width] = x.shape().try_into().unwrap(); + let lanes = lanes::(); + + let ch_block = lanes * BLOCK_REGISTERS; + + let out_height = ((x_height + 2 * pad_h - dilation_height * (kernel_height - 1) - 1) + / stride_height) + + 1; + let out_width = + ((x_width + 2 * pad_w - dilation_width * (kernel_width - 1) - 1) / stride_width) + 1; + + let mut output = unsafe { + Array4::::uninit((batch_size, out_height, out_width, channels)).assume_init() + }; + let unsafe_shared_out = UnsafeSharedRef::new(&mut output); + + let x = x.into_dimensionality::().unwrap(); + let x = x.view(); + let x = x.permuted_axes([0, 2, 3, 1]); + + // Floor division ensures `blocks * lanes * blocking factor` is always `<= out_channels`. + // An exclusive loop will always have `lanes * blocking factor` elements in bounds. + let blocks = channels / ch_block; + let blocks_end = blocks * ch_block; + // Floor division means simd_end is always divisible by `lanes` and `<= out_channels`. An + // exclusive loop will always have `lanes` elements in bounds. + let simd_end = channels / lanes * lanes; + let simd_unblocked = (simd_end - blocks_end) / lanes; + let remainder = channels - simd_end; + + run_par!(|| { + // SAFETY: Loop ranges are non-overlapping, so the unsafe shared reference is safe. + iter_range_par!(0, batch_size * blocks).for_each(|k| unsafe { + let block = k % blocks; + let b = k / blocks; + + let output = unsafe_shared_out.get(); + let x = x.slice(s![b, .., .., ..]); + let out = output.slice_mut(s![b, .., .., ..]); + loop_blocked(x, out, kernel_size, stride, padding, dilation, block); + }); + // SAFETY: See `loop_unblocked` + iter_range_par!(0, batch_size * simd_unblocked).for_each(|k| unsafe { + let ch = (k % simd_unblocked) * lanes + blocks_end; + let b = k / simd_unblocked; + + let output = unsafe_shared_out.get(); + let x = x.slice(s![b, .., .., ..]); + let out = output.slice_mut(s![b, .., .., ..]); + loop_unblocked(x, out, kernel_size, stride, padding, dilation, ch); + }); + // SAFETY: Loop ranges are non-overlapping, so the unsafe shared reference is safe. + iter_range_par!(0, batch_size * remainder).for_each(|k| unsafe { + let ch = (k % remainder) + simd_end; + let b = k / remainder; + + let output = unsafe_shared_out.get(); + let x = x.slice(s![b, .., .., ..]); + let out = output.slice_mut(s![b, .., .., ..]); + loop_scalar(x, out, kernel_size, stride, padding, dilation, ch); + }); + }); + + output = output.permuted_axes([0, 3, 1, 2]); + + output.into_dyn().into_shared() + } + + /// Execute the blocked (unrolled) portion of the pool. + #[allow( + clippy::too_many_arguments, + clippy::erasing_op, + clippy::identity_op, + unused_mut + )] + #[inline(always)] + #[macerator::with_simd] + fn loop_blocked<'a, S: Simd, E: Element + VOrd + MinMax>( + x: ArrayView3<'a, E>, + mut out: ArrayViewMut3<'a, E>, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + block: usize, + ) where + 'a: 'a, + { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + let [dilation_height, dilation_width] = dilation; + + let (x_height, x_width, _) = x.dim(); + let (out_height, out_width, _) = out.dim(); + let lanes = E::lanes::(); + let ch_block = lanes * BLOCK_REGISTERS; + + let min = E::MIN.splat::(); + // If outside padding area, kernels are guaranteed to be in bounds + for oh in pad_h..out_height.saturating_sub(pad_h) { + for ow in pad_w..out_width.saturating_sub(pad_w) { + seq!(N in 0..8 { + let mut acc~N = min; + }); + let ch = block * ch_block; + let ch_end = ch + ch_block; + let mut out = out.slice_mut(s![oh, ow, ch..ch_end]); + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh * dilation_height - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw * dilation_width - pad_w; + let x = x.slice(s![ih, iw, ch..ch_end]); + + seq!(N in 0..8 { + // SAFETY: + // Load a full vector from x[N * lanes]. This is bounds checked by the + // slice above. + acc~N = acc~N.max(unsafe { vload_unaligned(&x[N * lanes]) }); + }); + } + } + + seq!(N in 0..8 { + // SAFETY: + // Store a full vector to out[N * lanes]. This is bounds checked by the + // slice above. + unsafe { vstore_unaligned(&mut out[N * lanes], acc~N) }; + }); + } + } + + // Border pixels need bounds checks + if (pad_h, pad_w) != (0, 0) { + let v_borders = (0..pad_h) + .chain(out_height.saturating_sub(pad_h)..out_height) + .cartesian_product(0..out_width); + let h_borders = (0..out_height) + .cartesian_product((0..pad_w).chain(out_width.saturating_sub(pad_w)..out_width)); + + for (oh, ow) in v_borders.chain(h_borders) { + seq!(N in 0..8 { + let mut acc~N = min; + }); + let ch = block * ch_block; + let ch_end = ch + ch_block; + let mut out = out.slice_mut(s![oh, ow, ch..ch_end]); + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh * dilation_height; + if ih < pad_h || ih >= x_height + pad_h { + continue; + } + let ih = ih - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw * dilation_width; + if iw < pad_w || iw >= x_width + pad_w { + continue; + } + let iw = iw - pad_w; + + let x = x.slice(s![ih, iw, ch..ch_end]); + + seq!(N in 0..8 { + // SAFETY: + // Load a full vector from x[N * lanes]. This is bounds checked by the + // slice above. + acc~N = acc~N.max(unsafe { vload_unaligned(&x[N * lanes]) }); + }); + } + } + + seq!(N in 0..8 { + // SAFETY: + // Store a full vector to out[N * lanes]. This is bounds checked by the + // slice above. + unsafe { vstore_unaligned(&mut out[N * lanes], acc~N) }; + }); + } + } + } + + /// Execute the unblocked (not unrolled) portion of the pool. + /// + /// SAFETY: Safe as long as `ch + simd_lanes <= out_channels`. + #[allow(clippy::too_many_arguments, unused_mut)] + #[inline(always)] + #[macerator::with_simd] + unsafe fn loop_unblocked<'a, S: Simd, E: Element + VOrd + MinMax>( + x: ArrayView3<'a, E>, + mut out: ArrayViewMut3<'a, E>, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ch: usize, + ) where + 'a: 'a, + { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + let [dilation_height, dilation_width] = dilation; + + let (x_height, x_width, _) = x.dim(); + let (out_height, out_width, _) = out.dim(); + + for oh in pad_h..out_height.saturating_sub(pad_h) { + for ow in pad_w..out_width.saturating_sub(pad_w) { + let mut acc = E::MIN.splat::(); + let out = &mut out[[oh, ow, ch]]; + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh * dilation_height - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw * dilation_width - pad_w; + // Load a full vector from `x`. In bounds as long as `out_channels >= ch + lanes` + acc = acc.max(unsafe { vload_unaligned(&x[[ih, iw, ch]]) }); + } + } + // Store a full vector to `out`. In bounds as long as `out_channels >= ch + lanes`. + unsafe { vstore_unaligned(out, acc) }; + } + } + + // Border pixels need bounds checks + if (pad_h, pad_w) != (0, 0) { + let v_borders = (0..pad_h) + .chain(out_height.saturating_sub(pad_h)..out_height) + .cartesian_product(0..out_width); + let h_borders = (0..out_height) + .cartesian_product((0..pad_w).chain(out_width.saturating_sub(pad_w)..out_width)); + + for (oh, ow) in v_borders.chain(h_borders) { + let mut acc = E::MIN.splat::(); + let out = &mut out[[oh, ow, ch]]; + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh * dilation_height; + if ih < pad_h || ih >= x_height + pad_h { + continue; + } + let ih = ih - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw * dilation_width; + if iw < pad_w || iw >= x_width + pad_w { + continue; + } + let iw = iw - pad_w; + // Load a full vector from `x`. In bounds as long as `out_channels >= ch + lanes` + acc = acc.max(unsafe { vload_unaligned(&x[[ih, iw, ch]]) }); + } + } + // Store a full vector to `out`. In bounds as long as `out_channels >= ch + lanes`. + unsafe { vstore_unaligned(out, acc) }; + } + } + } + + fn loop_scalar( + x: ArrayView3<'_, E>, + mut out: ArrayViewMut3<'_, E>, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ch: usize, + ) { + let [kernel_height, kernel_width] = kernel_size; + let [pad_h, pad_w] = padding; + let [stride_height, stride_width] = stride; + let [dilation_height, dilation_width] = dilation; + + let (x_height, x_width, _) = x.dim(); + let (out_height, out_width, _) = out.dim(); + + for oh in 0..out_height { + for ow in 0..out_width { + let mut acc = E::MIN; + + for kh in 0..kernel_height { + let ih = oh * stride_height + kh * dilation_height; + if ih < pad_h || ih >= x_height + pad_h { + continue; + } + let ih = ih - pad_h; + + for kw in 0..kernel_width { + let iw = ow * stride_width + kw * dilation_width; + if iw < pad_w || iw >= x_width + pad_w { + continue; + } + let iw = iw - pad_w; + acc = acc.max(x[[ih, iw, ch]]); + } + } + + out[[oh, ow, ch]] = acc; + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/mod.rs new file mode 100644 index 0000000..2032f30 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod avgpool; +mod base; +pub(crate) mod binary; +pub(crate) mod binary_elemwise; +pub(crate) mod cmp; +pub(crate) mod conv; +pub(crate) mod maxpool; +pub(crate) mod unary; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/unary.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/unary.rs new file mode 100644 index 0000000..68d2626 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/simd/unary.rs @@ -0,0 +1,234 @@ +use core::marker::PhantomData; + +use bytemuck::cast; +use macerator::{ + Scalar, Simd, VAbs, VBitNot, VRecip, Vector, vload, vload_unaligned, vstore, vstore_unaligned, +}; +use ndarray::ArrayD; +use num_traits::Signed; +use seq_macro::seq; + +use crate::{NdArrayElement, SharedArray}; + +use super::should_use_simd; + +pub trait SimdUnop { + fn apply_vec(input: Vector) -> Vector; + fn apply(input: T) -> Out; + fn is_accelerated() -> bool; +} + +pub struct RecipVec; + +impl SimdUnop for RecipVec { + fn apply_vec(input: Vector) -> Vector { + input.recip() + } + + fn apply(input: f32) -> f32 { + input.recip() + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +pub struct VecAbs; + +impl SimdUnop for VecAbs { + fn apply_vec(input: Vector) -> Vector { + input.abs() + } + + fn apply(input: T) -> T { + input.abs() + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +pub struct VecBitNot; + +impl SimdUnop for VecBitNot { + fn apply_vec(input: Vector) -> Vector { + !input + } + + fn apply(input: T) -> T { + input.not() + } + + fn is_accelerated() -> bool { + ::is_accelerated::() + } +} + +#[macerator::with_simd] +fn is_accelerated>( + _x: PhantomData<(T, Out, Op)>, +) -> bool { + Op::is_accelerated::() +} + +pub fn try_unary_simd< + E: NdArrayElement, + EOut: NdArrayElement, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdUnop, +>( + input: SharedArray, +) -> Result, SharedArray> { + if !should_use_simd(input.len()) + || input.as_slice_memory_order().is_none() + || !is_accelerated::(PhantomData) + { + return Err(input); + } + // Used to assert traits based on the dynamic `DType`. + let input = unsafe { core::mem::transmute::, SharedArray>(input) }; + let out = if size_of::() == size_of::() + && align_of::() >= align_of::() + && input.is_unique() + { + unsafe { unary_scalar_simd_inplace::(input) } + } else { + unary_scalar_simd_owned::(input) + }; + // Used to assert traits based on the dynamic `DType`. + let out = unsafe { core::mem::transmute::, SharedArray>(out) }; + Ok(out) +} + +/// Execute operation in line. +/// SAFETY: +/// Must ensure `size_of:: == size_of::` and `align_of:: >= align_of::`. +unsafe fn unary_scalar_simd_inplace< + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdUnop, +>( + input: SharedArray, +) -> SharedArray { + let mut buffer = input.into_owned(); + let slice = buffer.as_slice_memory_order_mut().unwrap(); + // This is only called when in and out have the same size, so it's safe + unsafe { unary_slice_inplace::(slice, PhantomData) }; + // Buffer has the same elem size and is filled with the operation output, so this is safe + let out = unsafe { core::mem::transmute::, ArrayD>(buffer) }; + out.into_shared() +} + +fn unary_scalar_simd_owned< + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdUnop, +>( + input: SharedArray, +) -> SharedArray { + let mut out = unsafe { ArrayD::uninit(input.shape()).assume_init() }; + let input = input.as_slice_memory_order().unwrap(); + let out_slice = out.as_slice_memory_order_mut().unwrap(); + unary_slice::(input, out_slice, PhantomData); + out.into_shared() +} + +#[allow(clippy::erasing_op, clippy::identity_op)] +#[macerator::with_simd] +fn unary_slice< + 'a, + S: Simd, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdUnop, +>( + input: &'a [T], + out: &'a mut [Out], + _op: PhantomData, +) where + 'a: 'a, +{ + let lanes = T::lanes::(); + let mut chunks_input = input.chunks_exact(8 * lanes); + let mut chunks_out = out.chunks_exact_mut(8 * lanes); + while let Some((input, out)) = chunks_input.next().zip(chunks_out.next()) { + seq!(N in 0..8 { + // Load one full vector from `input`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + let s~N = unsafe { vload_unaligned(&input[N * lanes]) }; + let s~N = Op::apply_vec::(s~N); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == 8 * lanes` + unsafe { vstore_unaligned(&mut out[N * lanes], s~N) }; + }); + } + let mut chunks_input = chunks_input.remainder().chunks_exact(lanes); + let mut chunks_out = chunks_out.into_remainder().chunks_exact_mut(lanes); + while let Some((input, out)) = chunks_input.next().zip(chunks_out.next()) { + // Load one full vector from `input`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + let s0 = unsafe { vload_unaligned(input.as_ptr()) }; + let s0 = Op::apply_vec::(s0); + // Store one full vector to `out`. + // SAFETY: Guaranteed to be in bounds because `len == lanes` + unsafe { vstore_unaligned(out.as_mut_ptr(), s0) }; + } + + for (input, out) in chunks_input + .remainder() + .iter() + .zip(chunks_out.into_remainder()) + { + *out = Op::apply(*input) + } +} + +/// Execute operation in line. +/// SAFETY: +/// Must ensure `size_of:: == size_of::` and `align_of:: >= align_of::`. +#[macerator::with_simd] +unsafe fn unary_slice_inplace< + 'a, + S: Simd, + T: NdArrayElement + Scalar, + Out: NdArrayElement + Scalar, + Op: SimdUnop, +>( + buf: &'a mut [T], + _op: PhantomData<(Out, Op)>, +) where + 'a: 'a, +{ + let (head, main, tail) = unsafe { buf.align_to_mut::>() }; + for elem in head.iter_mut().chain(tail) { + *elem = cast(Op::apply(*elem)); + } + let mut chunks = main.chunks_exact_mut(8); + for elem in chunks.by_ref() { + seq!(N in 0..8 { + // Load a full vector from the aligned portion of the buffer. + // SAFETY: `align_to_mut` guarantees we're aligned to `T::Vector`'s size, and there is + // always a full vector in bounds. + let s~N = unsafe { vload(&elem[N] as *const _ as *const T) }; + let s~N = Op::apply_vec::(s~N); + // Store a full vector at the same position as the input. Cast is safe because `Out` is + // size and align compatible + unsafe { vstore(&mut elem[N] as *mut _ as *mut Out, s~N) }; + }); + } + + for elem in chunks.into_remainder() { + // Load a full vector from the aligned portion of the buffer. + // SAFETY: `align_to_mut` guarantees we're aligned to `T::Vector`'s size, and there is + // always a full vector in bounds. + let s0 = unsafe { vload(elem as *const _ as *const T) }; + + let s0 = Op::apply_vec::(s0); + // Store a full vector at the same position as the input. Cast is safe because `Out` is + // size and align compatible + unsafe { vstore(elem as *mut _ as *mut Out, s0) }; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/tensor.rs new file mode 100644 index 0000000..9f7383d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/tensor.rs @@ -0,0 +1,688 @@ +// Language +use alloc::vec::Vec; +use burn_backend::backend::ExecutionError; +use burn_backend::ops::GridSampleOptions; +use burn_backend::tensor::FloatTensor; +use burn_backend::{TensorMetadata, element::cast::ToElement}; + +// Current crate +use super::{ + NdArrayMathOps, NdArrayOps, + matmul::{cross, matmul}, +}; +use crate::{ + NdArray, cast_to_dtype, cat_with_dtype, execute_with_int_dtype, tensor::NdArrayTensor, +}; +use crate::{NdArrayDevice, SEED, slice}; +use crate::{ + SharedArray, + element::{ExpElement, FloatNdArrayElement, IntNdArrayElement, QuantElement}, +}; +use crate::{execute_with_float_dtype, ops::grid_sample::grid_sample_2d}; + +// Workspace crates +use crate::rand::get_seeded_rng; +use burn_backend::{Distribution, FloatDType, Scalar}; +use burn_backend::{ElementConversion, Shape, TensorData, backend::Backend, ops::FloatTensorOps}; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float; + +use libm::erf; + +#[cfg(feature = "std")] +#[allow(dead_code)] +fn round_ties_even_wrapper(x: f64) -> f64 { + x.round_ties_even() +} + +#[cfg(not(feature = "std"))] +#[allow(dead_code)] +fn round_ties_even_wrapper(x: f64) -> f64 { + if (x - x.floor()) == 0.5 { + (x * 0.5).round() * 2.0 + } else { + x.round() + } +} + +impl FloatTensorOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ + fn float_from_data(data: TensorData, _device: &NdArrayDevice) -> FloatTensor { + NdArrayTensor::from_data(data) + } + + fn float_random( + shape: Shape, + distribution: Distribution, + device: &NdArrayDevice, + ) -> FloatTensor { + let mut seed = SEED.lock().unwrap(); + let mut rng = seed.take().unwrap_or_else(get_seeded_rng); + let tensor = Self::float_from_data( + TensorData::random::(shape, distribution, &mut rng), + device, + ); + *seed = Some(rng); + tensor + } + + async fn float_into_data(tensor: FloatTensor) -> Result { + Ok(tensor.into_data()) + } + + fn float_device(_tensor: &FloatTensor) -> NdArrayDevice { + NdArrayDevice::Cpu + } + + fn float_to_device(tensor: FloatTensor, _device: &NdArrayDevice) -> FloatTensor { + tensor + } + + fn float_empty( + shape: Shape, + device: & as Backend>::Device, + dtype: FloatDType, + ) -> FloatTensor { + Self::float_zeros(shape, device, dtype) + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), NdArrayMathOps::add) + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::add_scalar(array, rhs.elem()) + }) + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), NdArrayMathOps::sub) + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::sub_scalar(array, rhs.elem()) + }) + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), NdArrayMathOps::mul) + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::mul_scalar(array, rhs.elem()) + }) + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), NdArrayMathOps::div) + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::div_scalar(array, rhs.elem()) + }) + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), NdArrayMathOps::remainder) + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::remainder_scalar(array, rhs.elem()) + }) + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), matmul) + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), |lhs, rhs| cross(lhs, rhs, dim)) + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::recip(array) + }) + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::swap_dims(array, dim1, dim2) + }) + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::reshape(array, shape) + }) + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: NdArrayTensor, + ) -> FloatTensor { + execute_with_int_dtype!( + indices, + IntElem, + |idx_array: SharedArray| -> NdArrayTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::gather(dim, array, idx_array) + }) + } + ) + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: NdArrayTensor, + value: FloatTensor, + ) -> FloatTensor { + execute_with_int_dtype!( + indices, + IntElem, + |idx_array: SharedArray| -> NdArrayTensor { + execute_with_float_dtype!((tensor, value), |tensor, value| NdArrayOps::scatter( + dim, tensor, idx_array, value + )) + } + ) + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: NdArrayTensor, + ) -> FloatTensor { + execute_with_int_dtype!( + indices, + IntElem, + |idx_array: SharedArray| -> NdArrayTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::select(array, dim, idx_array) + }) + } + ) + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: NdArrayTensor, + value: FloatTensor, + ) -> FloatTensor { + execute_with_int_dtype!( + indices, + IntElem, + |idx_array: SharedArray| -> NdArrayTensor { + execute_with_float_dtype!((tensor, value), |tensor, value| { + NdArrayMathOps::select_assign(tensor, dim, idx_array, value) + }) + } + ) + } + + fn float_slice(tensor: FloatTensor, slices: &[burn_backend::Slice]) -> FloatTensor { + slice!(tensor, slices) + } + + fn float_slice_assign( + tensor: FloatTensor, + slices: &[burn_backend::Slice], + value: FloatTensor, + ) -> FloatTensor { + execute_with_float_dtype!((tensor, value), |tensor, value| { + NdArrayOps::slice_assign(tensor, slices, value) + }) + } + + fn float_mask_where( + tensor: FloatTensor, + mask: NdArrayTensor, + value: FloatTensor, + ) -> FloatTensor { + execute_with_float_dtype!((tensor, value), |tensor, value| { + NdArrayOps::mask_where(tensor, mask.bool(), value) + }) + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: NdArrayTensor, + value: Scalar, + ) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::mask_fill(array, mask.bool(), value.elem()) + }) + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> NdArrayTensor { + execute_with_float_dtype!((lhs, rhs), |lhs, rhs| { NdArrayMathOps::equal(lhs, rhs) }) + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::equal_elem(array, rhs.elem()) + }) + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> NdArrayTensor { + execute_with_float_dtype!((lhs, rhs), |lhs, rhs| { NdArrayMathOps::greater(lhs, rhs) }) + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::greater_elem(array, rhs.elem()) + }) + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> NdArrayTensor { + execute_with_float_dtype!((lhs, rhs), |lhs, rhs| { + NdArrayMathOps::greater_equal(lhs, rhs) + }) + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::greater_equal_elem(array, rhs.elem()) + }) + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> NdArrayTensor { + execute_with_float_dtype!((lhs, rhs), |lhs, rhs| { NdArrayMathOps::lower(lhs, rhs) }) + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::lower_elem(array, rhs.elem()) + }) + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> NdArrayTensor { + execute_with_float_dtype!((lhs, rhs), |lhs, rhs| { + NdArrayMathOps::lower_equal(lhs, rhs) + }) + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> NdArrayTensor { + execute_with_float_dtype!(lhs, FloatElem, |array: SharedArray| { + NdArrayMathOps::lower_equal_elem(array, rhs.elem()) + }) + } + + fn float_detach(tensor: FloatTensor) -> FloatTensor { + tensor + } + + fn float_mean(tensor: FloatTensor) -> FloatTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::mean_view(array.view()) + }) + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::sum_view(array.view()) + }) + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::mean_dim(array, dim) + }) + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::cumsum(array, dim) + }) + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::cumprod(array, dim) + }) + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::cummin(array, dim) + }) + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::cummax(array, dim) + }) + } + + fn float_sum_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::sum_dim(array, dim) + }) + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::argmax_view::(array.view(), dim) + }) + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> NdArrayTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::argmin_view::(array.view(), dim) + }) + } + + fn float_exp(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array.mapv_into(|a: FloatElem| a.exp_elem()).into_shared() + }) + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array.mapv_into(|a: FloatElem| a.log_elem()).into_shared() + }) + } + + fn float_prod(tensor: FloatTensor) -> FloatTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::prod_view(array.view()) + }) + } + + fn float_prod_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::prod_dim(array, dim) + }) + } + + fn float_max(tensor: FloatTensor) -> FloatTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::max_view(array.view()) + }) + } + + fn float_min(tensor: FloatTensor) -> FloatTensor { + // Use view() for zero-copy on borrowed storage + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::min_view(array.view()) + }) + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array.mapv_into(|a: FloatElem| a.log1p_elem()).into_shared() + }) + } + + fn float_powf_scalar_impl(tensor: FloatTensor, value: Scalar) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| a.powf_elem(value.elem())) + .into_shared() + }) + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array.mapv_into(|a: FloatElem| a.sqrt_elem()).into_shared() + }) + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::abs(array) + }) + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).cos().elem()) + .into_shared() + }) + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).cosh().elem()) + .into_shared() + }) + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).sin().elem()) + .into_shared() + }) + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).sinh().elem()) + .into_shared() + }) + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).tan().elem()) + .into_shared() + }) + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).tanh().elem()) + .into_shared() + }) + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).acos().elem()) + .into_shared() + }) + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).acosh().elem()) + .into_shared() + }) + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).asin().elem()) + .into_shared() + }) + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).asinh().elem()) + .into_shared() + }) + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).atan().elem()) + .into_shared() + }) + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).atanh().elem()) + .into_shared() + }) + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), FloatElem, |lhs, rhs| { + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &FloatElem, b: &FloatElem| a.atan2(*b)) + }) + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| round_ties_even_wrapper(a.to_f64()).elem()) + .into_shared() + }) + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).floor().elem()) + .into_shared() + }) + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).ceil().elem()) + .into_shared() + }) + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| (a.to_f64()).trunc().elem()) + .into_shared() + }) + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array + .mapv_into(|a: FloatElem| erf(a.to_f64()).elem()) + .into_shared() + }) + } + + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + cat_with_dtype!(tensors, dim, [F64, F32]) + } + + fn float_clamp_min(tensor: FloatTensor, min: Scalar) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::clamp_min(array, min.elem()) + }) + } + + fn float_clamp_max(tensor: FloatTensor, max: Scalar) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::clamp_max(array, max.elem()) + }) + } + + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::clamp(array, min.elem(), max.elem()) + }) + } + + fn float_into_int(tensor: FloatTensor) -> NdArrayTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + array.mapv(|a: FloatElem| a.elem::()).into_shared() + }) + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + execute_with_float_dtype!((lhs, rhs), FloatElem, |lhs, rhs| { + NdArrayMathOps::elementwise_op(lhs, rhs, |a: &FloatElem, b: &FloatElem| a.powf(*b)) + }) + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::permute(array, axes) + }) + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::flip(array, axes) + }) + } + + fn float_sign(tensor: FloatTensor) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayMathOps::sign_op(array) + }) + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::expand(array, shape) + }) + } + + fn float_cast(tensor: FloatTensor, dtype: FloatDType) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + cast_to_dtype(array, dtype.into()) + }) + } + + fn float_grid_sample_2d( + tensor: FloatTensor, + grid: FloatTensor, + options: GridSampleOptions, + ) -> FloatTensor { + execute_with_float_dtype!((tensor, grid), |tensor, grid| grid_sample_2d( + tensor, grid, options + )) + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + execute_with_float_dtype!(tensor, FloatElem, |array: SharedArray| { + NdArrayOps::unfold(array, dim, size, step) + }) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/transaction.rs new file mode 100644 index 0000000..b308c0f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/ops/transaction.rs @@ -0,0 +1,13 @@ +use crate::{ + FloatNdArrayElement, NdArray, NdArrayTensor, SharedArray, + element::{IntNdArrayElement, QuantElement}, +}; +use burn_backend::ops::TransactionOps; + +impl TransactionOps + for NdArray +where + NdArrayTensor: From>, + NdArrayTensor: From>, +{ +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/parallel.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/parallel.rs new file mode 100644 index 0000000..a665761 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/parallel.rs @@ -0,0 +1,76 @@ +/// Macro for running a function in parallel. +#[cfg(feature = "multi-threads")] +#[macro_export(local_inner_macros)] +macro_rules! run_par { + ( + $func:expr + ) => {{ + use rayon::prelude::*; + + #[allow(clippy::redundant_closure_call)] + rayon::scope(|_| $func()) + }}; +} + +/// Macro for running a function in parallel. +#[cfg(not(feature = "multi-threads"))] +#[macro_export(local_inner_macros)] +macro_rules! run_par { + ( + $func:expr + ) => {{ $func() }}; +} + +/// Macro for iterating in parallel. +#[cfg(not(feature = "multi-threads"))] +#[macro_export(local_inner_macros)] +macro_rules! iter_par { + ( + $iter:expr + ) => {{ $iter }}; +} + +/// Macro for iterating in parallel. +#[cfg(feature = "multi-threads")] +#[macro_export(local_inner_macros)] +macro_rules! iter_par { + ( + $iter:expr + ) => {{ $iter.into_par_iter() }}; +} + +/// Macro for iterating in parallel. +#[cfg(feature = "multi-threads")] +#[macro_export(local_inner_macros)] +macro_rules! iter_slice_par { + ( + $slice:expr + ) => {{ $slice.into_par_iter() }}; +} + +/// Macro for iterating in parallel. +#[cfg(not(feature = "multi-threads"))] +#[macro_export(local_inner_macros)] +macro_rules! iter_slice_par { + ( + $slice:expr + ) => {{ $slice.iter() }}; +} + +/// Macro for iterating over a range in parallel. +#[cfg(feature = "multi-threads")] +#[macro_export(local_inner_macros)] +macro_rules! iter_range_par { + ( + $start:expr, $end:expr + ) => {{ ($start..$end).into_par_iter() }}; +} + +/// Macro for iterating over a range in parallel. +#[cfg(not(feature = "multi-threads"))] +#[macro_export(local_inner_macros)] +macro_rules! iter_range_par { + ( + $start:expr, $end:expr + ) => {{ ($start..$end) }}; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/rand.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/rand.rs new file mode 100644 index 0000000..94b9bcd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/rand.rs @@ -0,0 +1,36 @@ +//! Random number generation utilities for burn-ndarray + +#[cfg(not(feature = "std"))] +use rand::rngs::SmallRng; +#[cfg(feature = "std")] +use rand::rngs::StdRng; + +/// Type alias for the RNG used by burn-ndarray +#[cfg(feature = "std")] +pub type NdArrayRng = StdRng; +#[cfg(not(feature = "std"))] +pub type NdArrayRng = SmallRng; + +#[cfg(not(feature = "std"))] +use rand::SeedableRng; + +/// Get a seeded random number generator +/// +/// For std builds, uses OS entropy. +/// For no_std builds, uses a compile-time random seed. +#[cfg(feature = "std")] +pub fn get_seeded_rng() -> NdArrayRng { + // Use the standard implementation from burn-std + burn_std::rand::get_seeded_rng() +} + +/// Get a seeded random number generator +/// +/// For std builds, uses OS entropy. +/// For no_std builds, uses a compile-time random seed. +#[cfg(not(feature = "std"))] +pub fn get_seeded_rng() -> NdArrayRng { + // Use compile-time random seed for no_std + const SEED: u64 = const_random::const_random!(u64); + SmallRng::seed_from_u64(SEED) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/sharing.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/sharing.rs new file mode 100644 index 0000000..75d5142 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/sharing.rs @@ -0,0 +1,19 @@ +use core::cell::UnsafeCell; + +/// Similar to `SyncUnsafeCell` see [Rust issues](https://github.com/rust-lang/rust/issues/95439). +pub(crate) struct UnsafeSharedRef<'a, T> { + cell: UnsafeCell<&'a mut T>, +} + +unsafe impl Sync for UnsafeSharedRef<'_, T> {} + +impl<'a, T> UnsafeSharedRef<'a, T> { + pub fn new(data: &'a mut T) -> Self { + Self { + cell: UnsafeCell::new(data), + } + } + pub unsafe fn get(&self) -> &'a mut T { + unsafe { core::ptr::read(self.cell.get()) } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/storage.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/storage.rs new file mode 100644 index 0000000..0ddaafd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/storage.rs @@ -0,0 +1,514 @@ +//! Copy-on-write storage for zero-copy tensor loading. +//! +//! This module provides `NdArrayStorage`, which enables true zero-copy loading +//! from burnpack files. When data is borrowed from external memory (like mmap'd files +//! or static data), it remains zero-copy until a mutating operation is performed, +//! at which point it's copied (copy-on-write semantics). +//! +//! This integrates with ndarray's existing COW patterns - operations that check +//! `is_unique()` will see borrowed data as non-unique, triggering the allocation path. + +use alloc::vec::Vec; +use burn_backend::Element; +use burn_std::Bytes; +use core::mem; +use ndarray::{ArcArray, ArrayView, IxDyn}; + +/// Storage that supports both owned data and borrowed (zero-copy) data. +/// +/// # Copy-on-Write Semantics +/// +/// - **Borrowed**: Data from external source (burnpack, mmap, static). +/// Reports `is_unique() == false` to trigger copy on mutation. +/// - **Owned**: Standard `ArcArray` with built-in COW via Arc refcount. +/// +/// # Example +/// +/// ```ignore +/// // Zero-copy load +/// let storage = NdArrayStorage::from_borrowed(bytes, shape); +/// storage.is_unique(); // false - will copy on mutation +/// +/// // Read operations use view() - zero-copy +/// let view = storage.view(); +/// +/// // Mutation converts to owned +/// let owned = storage.into_owned(); // Copies here +/// ``` +#[derive(Debug)] +pub enum NdArrayStorage { + /// Borrowed from external source (e.g., burnpack zero-copy load). + /// Keeps `Bytes` alive to ensure the referenced memory is valid. + Borrowed { + /// Source bytes - keeps external memory alive via reference counting + bytes: Bytes, + /// Shape of the tensor + shape: Vec, + }, + + /// Standard owned storage with ArcArray COW semantics. + Owned(ArcArray), +} + +impl Clone for NdArrayStorage { + fn clone(&self) -> Self { + match self { + // For borrowed data, clone the Bytes (cheap Arc clone) and shape + Self::Borrowed { bytes, shape } => Self::Borrowed { + bytes: bytes.clone(), + shape: shape.clone(), + }, + // For owned data, clone the ArcArray (cheap Arc clone) + Self::Owned(arr) => Self::Owned(arr.clone()), + } + } +} + +impl NdArrayStorage { + /// Create borrowed storage from external bytes. + /// + /// Returns the bytes and shape back on failure (misaligned or too small), + /// enabling zero-copy even for native allocations by avoiding defensive cloning. + /// + /// # Requirements + /// + /// The caller must ensure that: + /// - The `Bytes` contain valid data for the element type `E` + /// - The data is contiguous in row-major (C) order matching the provided shape + /// + /// These requirements are upheld when loading from `TensorData` (burnpack, etc.) + /// which always stores data contiguously in row-major order. + pub fn from_borrowed(bytes: Bytes, shape: Vec) -> Result)> { + // Validate alignment + let ptr = bytes.as_ptr(); + if !(ptr as usize).is_multiple_of(mem::align_of::()) { + return Err((bytes, shape)); + } + + // Validate size (using checked arithmetic to prevent overflow) + let num_elements = match shape + .iter() + .try_fold(1usize, |acc, &dim| acc.checked_mul(dim)) + { + Some(n) => n, + None => return Err((bytes, shape)), + }; + let expected_size = match num_elements.checked_mul(mem::size_of::()) { + Some(s) => s, + None => return Err((bytes, shape)), + }; + if bytes.len() < expected_size { + return Err((bytes, shape)); + } + + Ok(Self::Borrowed { bytes, shape }) + } + + /// Create owned storage from an ArcArray. + #[inline] + pub fn from_owned(array: ArcArray) -> Self { + Self::Owned(array) + } + + /// Returns whether this storage is uniquely owned and can be mutated in-place. + /// + /// - **Borrowed**: Always returns `false` to trigger copy-on-write. + /// - **Owned**: Delegates to `ArcArray::is_unique()`. + /// + /// This integrates with existing SIMD code patterns like: + /// ```ignore + /// if tensor.is_unique() { + /// // mutate in place + /// } else { + /// // allocate new + /// } + /// ``` + #[inline] + pub fn is_unique(&self) -> bool { + match self { + Self::Borrowed { .. } => false, // Force copy path + Self::Owned(arr) => arr.is_unique(), + } + } + + /// Get a read-only view of the data. + /// + /// This is zero-copy for both borrowed and owned variants. + #[inline] + pub fn view(&self) -> ArrayView<'_, E, IxDyn> { + match self { + Self::Borrowed { bytes, shape } => { + let ptr = bytes.as_ptr() as *const E; + let dim = IxDyn(shape); + // SAFETY: + // - `bytes` is kept alive for the lifetime of `self` + // - Alignment was validated in `from_borrowed` + // - Size was validated in `from_borrowed` + unsafe { ArrayView::from_shape_ptr(dim, ptr) } + } + Self::Owned(arr) => arr.view(), + } + } + + /// Convert to owned ArcArray. + /// + /// - **Borrowed**: Copies the data into a new ArcArray. + /// - **Owned + unique**: Returns the array without copying. + /// - **Owned + shared**: Clones the data. + pub fn into_owned(self) -> ArcArray { + match self { + Self::Borrowed { bytes, shape } => { + let ptr = bytes.as_ptr() as *const E; + let dim = IxDyn(&shape); + // SAFETY: Same as view() - bytes is valid for this scope + let view = unsafe { ArrayView::from_shape_ptr(dim, ptr) }; + view.to_owned().into_shared() + } + Self::Owned(arr) => arr, + } + } + + /// Convert to shared ArcArray, suitable for returning from operations. + /// + /// This is equivalent to `into_owned()` but named for clarity. + #[inline] + pub fn into_shared(self) -> ArcArray { + self.into_owned() + } + + /// Get the shape of the tensor. + pub fn shape(&self) -> &[usize] { + match self { + Self::Borrowed { shape, .. } => shape, + Self::Owned(arr) => arr.shape(), + } + } + + /// Get the number of dimensions. + #[inline] + pub fn ndim(&self) -> usize { + self.shape().len() + } + + /// Get the total number of elements. + #[inline] + pub fn len(&self) -> usize { + self.shape().iter().product() + } + + /// Check if the tensor is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns `true` if this is borrowed (zero-copy) storage. + #[inline] + pub fn is_borrowed(&self) -> bool { + matches!(self, Self::Borrowed { .. }) + } + + /// Returns `true` if this is owned storage. + #[inline] + pub fn is_owned(&self) -> bool { + matches!(self, Self::Owned(_)) + } + + /// Ensure owned and return mutable reference to the ArcArray. + /// + /// Converts borrowed to owned if necessary. + pub fn ensure_owned(&mut self) -> &mut ArcArray { + if let Self::Borrowed { bytes, shape } = self { + let ptr = bytes.as_ptr() as *const E; + let dim = IxDyn(shape); + // SAFETY: Same as view() + let view = unsafe { ArrayView::from_shape_ptr(dim, ptr) }; + *self = Self::Owned(view.to_owned().into_shared()); + } + match self { + Self::Owned(arr) => arr, + Self::Borrowed { .. } => unreachable!(), + } + } +} + +/// Convert from ArcArray to NdArrayStorage. +impl From> for NdArrayStorage { + fn from(array: ArcArray) -> Self { + Self::Owned(array) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use burn_std::Bytes; + + #[test] + fn test_borrowed_is_not_unique() { + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + assert!(!storage.is_unique()); + assert!(storage.is_borrowed()); + } + + #[test] + fn test_owned_unique_when_single_ref() { + let array = ndarray::ArrayD::from_elem(IxDyn(&[2, 2]), 1.0f32).into_shared(); + let storage = NdArrayStorage::from_owned(array); + + assert!(storage.is_unique()); + assert!(storage.is_owned()); + } + + #[test] + fn test_owned_not_unique_when_cloned() { + let array = ndarray::ArrayD::from_elem(IxDyn(&[2, 2]), 1.0f32).into_shared(); + let storage = NdArrayStorage::from_owned(array); + let _clone = storage.clone(); + + assert!(!storage.is_unique()); + } + + #[test] + fn test_view_zero_copy() { + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + let view = storage.view(); + assert_eq!(view[[0, 0]], 1.0); + assert_eq!(view[[1, 1]], 4.0); + } + + #[test] + fn test_into_owned_copies_borrowed() { + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + let owned = storage.into_owned(); + assert_eq!(owned[[0, 0]], 1.0); + assert_eq!(owned[[1, 1]], 4.0); + } + + #[test] + fn test_from_borrowed_validates_alignment() { + use burn_std::AllocationProperty; + + // Test 1: Properly aligned data should succeed + let aligned_data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let aligned_bytes = Bytes::from_elems(aligned_data); + + // Verify test setup - should be 4-byte aligned for f32 + assert_eq!( + (aligned_bytes.as_ptr() as usize) % core::mem::align_of::(), + 0, + "Test setup: f32 data should be properly aligned" + ); + + let result = NdArrayStorage::::from_borrowed(aligned_bytes, vec![2, 2]); + assert!( + result.is_ok(), + "from_borrowed should succeed for properly aligned data" + ); + + // Test 2: Misaligned data should fail + // Create a buffer large enough to find a misaligned offset + // (static data placement varies by platform, so we find an offset dynamically) + let buffer: &[u8] = &[0u8; 32]; + let shared = bytes::Bytes::from_static(buffer); + let base = shared.as_ptr() as usize; + let align = core::mem::align_of::(); + + // Find an offset in 1..align that produces misalignment (at least one must exist) + let misalign_offset = (1..align) + .find(|&off| !(base + off).is_multiple_of(align)) + .expect("Should find a misaligned offset"); + + let sliced = shared.slice(misalign_offset..(misalign_offset + 16)); + let misaligned_bytes = Bytes::from_shared(sliced, AllocationProperty::Other); + + // Verify test setup - should NOT be 4-byte aligned + assert_ne!( + (misaligned_bytes.as_ptr() as usize) % align, + 0, + "Test setup: sliced data should be misaligned for f32" + ); + + let result = NdArrayStorage::::from_borrowed(misaligned_bytes, vec![4]); + assert!( + result.is_err(), + "from_borrowed should return Err for misaligned data" + ); + } + + #[test] + fn test_insufficient_size_returns_err() { + // Create bytes that are too small for the requested shape + let data: Vec = vec![1.0, 2.0]; // 8 bytes + let bytes = Bytes::from_elems(data); + + // Try to create storage for 4 elements (needs 16 bytes) + let result = NdArrayStorage::::from_borrowed(bytes, vec![4]); + assert!( + result.is_err(), + "from_borrowed should return Err when bytes are too small" + ); + } + + // ========================================================================== + // Zero-copy hardening tests + // These tests verify the zero-copy guarantee is maintained. If any of these + // fail, it indicates a regression in zero-copy functionality. + // ========================================================================== + + #[test] + fn test_zero_copy_native_allocation() { + // CRITICAL: Verify that native allocations (Bytes::from_elems) are zero-copy + // on initial load. The view() must return a pointer to the SAME memory. + // + // Note: Native allocations copy on clone (this is expected), but the initial + // load is still zero-copy, avoiding an extra copy in the common case where + // the tensor is used without cloning. + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let original_ptr = bytes.as_ptr(); + + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + // Initial load must be zero-copy + let view = storage.view(); + let view_ptr = view.as_ptr() as *const u8; + + assert_eq!( + original_ptr, view_ptr, + "ZERO-COPY REGRESSION: native allocation view() must return pointer to original bytes" + ); + + // Verify data integrity + assert_eq!(view[[0, 0]], 1.0); + assert_eq!(view[[0, 1]], 2.0); + assert_eq!(view[[1, 0]], 3.0); + assert_eq!(view[[1, 1]], 4.0); + } + + #[test] + fn test_zero_copy_shared_bytes_pointer_identity() { + // CRITICAL: Test with SharedBytesAllocationController for true zero-copy. + // This simulates the actual burnpack/mmap loading path. + use burn_std::AllocationProperty; + + // Create static-like data using bytes::Bytes + let data: &[u8] = &[ + 0, 0, 128, 63, // 1.0f32 in little-endian + 0, 0, 0, 64, // 2.0f32 + 0, 0, 64, 64, // 3.0f32 + 0, 0, 128, 64, // 4.0f32 + ]; + let shared = bytes::Bytes::from_static(data); + let original_ptr = shared.as_ptr(); + + // Create Bytes with SharedBytesAllocationController + let bytes = Bytes::from_shared(shared, AllocationProperty::Other); + + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + // Verify pointer identity + let view_ptr = storage.view().as_ptr() as *const u8; + assert_eq!( + original_ptr, view_ptr, + "ZERO-COPY REGRESSION: SharedBytes view must point to original static data" + ); + + // Clone should also share the same memory + let cloned = storage.clone(); + let cloned_ptr = cloned.view().as_ptr() as *const u8; + assert_eq!( + original_ptr, cloned_ptr, + "ZERO-COPY REGRESSION: SharedBytes clone must share memory" + ); + } + + #[test] + fn test_clone_borrowed_stays_borrowed() { + // Verify that cloning borrowed storage produces another borrowed storage. + // Note: The underlying Bytes may or may not share memory depending on + // the allocation controller (native allocations copy, file-backed may share). + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + let cloned = storage.clone(); + + // Both should still be borrowed (the storage type is preserved) + assert!( + storage.is_borrowed(), + "ZERO-COPY REGRESSION: original should remain borrowed after clone" + ); + assert!( + cloned.is_borrowed(), + "ZERO-COPY REGRESSION: clone should be borrowed type" + ); + + // Both should report not unique (important for COW behavior) + assert!( + !storage.is_unique(), + "ZERO-COPY REGRESSION: original should not be unique after clone" + ); + assert!( + !cloned.is_unique(), + "ZERO-COPY REGRESSION: clone should not be unique" + ); + + // Data should be identical + assert_eq!(storage.view(), cloned.view(), "Clone should have same data"); + } + + #[test] + fn test_zero_copy_triggers_copy_on_mutation() { + // Verify that into_owned() on borrowed data creates a NEW allocation + // (this is the "copy" in copy-on-write) + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let original_ptr = bytes.as_ptr(); + + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + assert!(storage.is_borrowed(), "should start as borrowed"); + + let owned = storage.into_owned(); + let owned_ptr = owned.as_ptr() as *const u8; + + assert_ne!( + original_ptr, owned_ptr, + "into_owned() on borrowed data MUST allocate new memory (copy-on-write)" + ); + } + + #[test] + fn test_borrowed_reports_not_unique() { + // CRITICAL: Borrowed storage must report is_unique() == false + // This is what triggers copy-on-write in mutation operations + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let storage = + NdArrayStorage::::from_borrowed(bytes, vec![2, 2]).expect("should create"); + + assert!( + !storage.is_unique(), + "ZERO-COPY REGRESSION: borrowed storage MUST report is_unique() == false \ + to trigger copy-on-write. If this is true, mutations will corrupt shared data!" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/tensor.rs new file mode 100644 index 0000000..1f8b5c5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-ndarray/src/tensor.rs @@ -0,0 +1,864 @@ +use core::mem; + +use burn_backend::{ + DType, Element, QTensorPrimitive, Shape, TensorData, TensorMetadata, + quantization::{QParams, QuantLevel, QuantMode, QuantScheme, QuantValue}, +}; + +use crate::NdArrayStorage; +use crate::ops::quantization::{QuantizationStrategy, SymmetricQuantization}; +use alloc::vec::Vec; +use ndarray::{ArcArray, ArrayD, IxDyn}; + +/// Concrete storage type for ndarray (owned with COW semantics via Arc) +pub type SharedArray = ArcArray; + +/// Tensor primitive used by the [ndarray backend](crate::NdArray). +/// +/// Supports both owned and borrowed (zero-copy) data via `NdArrayStorage`. +/// When data is borrowed from external sources (like burnpack files), +/// it remains zero-copy until a mutating operation is performed. +#[derive(Debug, Clone)] +#[allow(missing_docs)] +pub enum NdArrayTensor { + F64(NdArrayStorage), + F32(NdArrayStorage), + I64(NdArrayStorage), + I32(NdArrayStorage), + I16(NdArrayStorage), + I8(NdArrayStorage), + U64(NdArrayStorage), + U32(NdArrayStorage), + U16(NdArrayStorage), + U8(NdArrayStorage), + Bool(NdArrayStorage), +} + +impl NdArrayTensor { + /// Extract bool array, converting to owned if necessary. + pub(crate) fn bool(self) -> SharedArray { + match self { + NdArrayTensor::Bool(storage) => storage.into_shared(), + _ => unimplemented!("Expected bool tensor, got {:?}", self.dtype()), + } + } + + /// Returns true if this tensor uses borrowed (zero-copy) storage. + #[inline] + pub fn is_borrowed(&self) -> bool { + macro_rules! check { + ($($variant:ident),*) => { + match self { + $(NdArrayTensor::$variant(s) => s.is_borrowed(),)* + } + }; + } + check!(F64, F32, I64, I32, I16, I8, U64, U32, U16, U8, Bool) + } +} + +pub(crate) fn cast_to_dtype(array: SharedArray, dtype: DType) -> NdArrayTensor +where + NdArrayTensor: From>, +{ + fn cast(array: SharedArray) -> SharedArray { + array.mapv(|a| a.elem()).into_shared() + } + + if E1::dtype() == dtype { + return array.into(); + } + + match dtype { + DType::F64 => cast::(array).into(), + DType::F32 => cast::(array).into(), + DType::Flex32 => cast::(array).into(), + DType::I64 => cast::(array).into(), + DType::I32 => cast::(array).into(), + DType::I16 => cast::(array).into(), + DType::I8 => cast::(array).into(), + DType::U64 => cast::(array).into(), + DType::U32 => cast::(array).into(), + DType::U16 => cast::(array).into(), + DType::U8 => cast::(array).into(), + DType::Bool => cast::(array).into(), + dtype => panic!("Unsupported dtype: {dtype:?}"), + } +} + +macro_rules! impl_from { + ($($ty: ty => $dtype: ident),*) => { + // From SharedArray (owned) -> NdArrayTensor + $(impl From> for NdArrayTensor { + fn from(value: SharedArray<$ty>) -> NdArrayTensor { + NdArrayTensor::$dtype(NdArrayStorage::from_owned(value)) + } + })* + + // From NdArrayStorage -> NdArrayTensor + $(impl From> for NdArrayTensor { + fn from(value: NdArrayStorage<$ty>) -> NdArrayTensor { + NdArrayTensor::$dtype(value) + } + })* + }; +} + +impl_from!( + f64 => F64, f32 => F32, + i64 => I64, i32 => I32, i16 => I16, i8 => I8, + u64 => U64, u32 => U32, u16 => U16, u8 => U8, + bool => Bool +); + +/// Macro to execute an operation on a given element type. +/// +/// Extracts the storage from NdArrayTensor, converts to SharedArray, and passes to operation. +/// +/// # Panics +/// Since there is no automatic type cast at this time, binary operations for different +/// floating point precision data types will panic with a data type mismatch. +#[macro_export] +macro_rules! execute_with_dtype { + (($lhs:expr, $rhs:expr),$element:ident, $op:expr, [$($dtype: ident => $ty: ty),*]) => {{ + let lhs_dtype = burn_backend::TensorMetadata::dtype(&$lhs); + let rhs_dtype = burn_backend::TensorMetadata::dtype(&$rhs); + match ($lhs, $rhs) { + $( + ($crate::NdArrayTensor::$dtype(lhs), $crate::NdArrayTensor::$dtype(rhs)) => { + #[allow(unused)] + type $element = $ty; + // Convert storage to SharedArray for compatibility with existing operations + $op(lhs.into_shared(), rhs.into_shared()).into() + } + )* + _ => panic!( + "Data type mismatch (lhs: {:?}, rhs: {:?})", + lhs_dtype, rhs_dtype + ), + } + }}; + // Binary op: type automatically inferred by the compiler + (($lhs:expr, $rhs:expr), $op:expr) => {{ + $crate::execute_with_dtype!(($lhs, $rhs), E, $op) + }}; + + // Binary op: generic type cannot be inferred for an operation + (($lhs:expr, $rhs:expr), $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!(($lhs, $rhs), $element, $op, [ + F64 => f64, F32 => f32, + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8, + Bool => bool + ]) + }}; + + ($tensor:expr, $element:ident, $op:expr, [$($dtype: ident => $ty: ty),*]) => {{ + match $tensor { + $( + $crate::NdArrayTensor::$dtype(storage) => { + #[allow(unused)] + type $element = $ty; + // Convert to SharedArray for compatibility with most operations + $op(storage.into_shared()).into() + } + )* + #[allow(unreachable_patterns)] + other => unimplemented!("unsupported dtype: {:?}", other.dtype()) + } + }}; + // Unary op: type automatically inferred by the compiler + ($tensor:expr, $op:expr) => {{ + $crate::execute_with_dtype!($tensor, E, $op) + }}; + + // Unary op: generic type cannot be inferred for an operation + ($tensor:expr, $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!($tensor, $element, $op, [ + F64 => f64, F32 => f32, + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8, + Bool => bool + ]) + }}; +} + +/// Macro to execute an operation a given element type. +/// Only handles float types. +/// +/// # Panics +/// Since there is no automatic type cast at this time, binary operations for different +/// floating point precision data types will panic with a data type mismatch. +#[macro_export] +macro_rules! execute_with_float_dtype { + // Binary op: type automatically inferred by the compiler + (($lhs:expr, $rhs:expr), $op:expr) => {{ + $crate::execute_with_float_dtype!(($lhs, $rhs), E, $op) + }}; + + // Binary op: generic type cannot be inferred for an operation + (($lhs:expr, $rhs:expr), $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!(($lhs, $rhs), $element, $op, [ + F64 => f64, F32 => f32 + ]) + }}; + + // Unary op: type automatically inferred by the compiler + ($tensor:expr, $op:expr) => {{ + $crate::execute_with_float_dtype!($tensor, E, $op) + }}; + + // Unary op: generic type cannot be inferred for an operation + ($tensor:expr, $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!($tensor, $element, $op, [ + F64 => f64, F32 => f32 + ]) + }}; +} + +/// Macro to execute an operation a given element type. +/// Only handles int types. +/// +/// # Panics +/// Since there is no automatic type cast at this time, binary operations for different +/// floating point precision data types will panic with a data type mismatch. +#[macro_export] +macro_rules! execute_with_int_dtype { + // Binary op: type automatically inferred by the compiler + (($lhs:expr, $rhs:expr), $op:expr) => {{ + $crate::execute_with_int_dtype!(($lhs, $rhs), E, $op) + }}; + + // Binary op: generic type cannot be inferred for an operation + (($lhs:expr, $rhs:expr), $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!(($lhs, $rhs), $element, $op, [ + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8 + ]) + }}; + + // Unary op: type automatically inferred by the compiler + ($tensor:expr, $op:expr) => {{ + $crate::execute_with_int_dtype!($tensor, E, $op) + }}; + + // Unary op: generic type cannot be inferred for an operation + ($tensor:expr, $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!($tensor, $element, $op, [ + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8 + ]) + }}; +} + +/// Macro to execute an operation a given element type. +/// Only handles numeric types +/// +/// # Panics +/// Since there is no automatic type cast at this time, binary operations for different +/// floating point precision data types will panic with a data type mismatch. +#[macro_export] +macro_rules! execute_with_numeric_dtype { + // Binary op: type automatically inferred by the compiler + (($lhs:expr, $rhs:expr), $op:expr) => {{ + $crate::execute_with_numeric_dtype!(($lhs, $rhs), E, $op) + }}; + + // Binary op: generic type cannot be inferred for an operation + (($lhs:expr, $rhs:expr), $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!(($lhs, $rhs), $element, $op, [ + F64 => f64, F32 => f32, + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8 + ]) + }}; + + // Unary op: type automatically inferred by the compiler + ($tensor:expr, $op:expr) => {{ + $crate::execute_with_numeric_dtype!($tensor, E, $op) + }}; + + // Unary op: generic type cannot be inferred for an operation + ($tensor:expr, $element:ident, $op:expr) => {{ + $crate::execute_with_dtype!($tensor, $element, $op, [ + F64 => f64, F32 => f32, + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8 + ]) + }}; +} + +/// Macro to execute a cat operation on a given set of element types. +/// +/// Uses zero-copy views from storage for concatenation. +/// +/// # Panics +/// Since there is no automatic type cast at this time, binary operations for different +/// floating point precision data types will panic with a data type mismatch. +#[macro_export] +macro_rules! cat_with_dtype { + ($tensors: expr, $dim: expr, [$($dtype: ident),*]) => { + match &$tensors[0] { + $(NdArrayTensor::$dtype(_) => { + let tensors = $tensors + .iter() + .map(|t| { + if let NdArrayTensor::$dtype(storage) = t { + // Use storage.view() for zero-copy access + storage.view() + } else { + panic!("Concatenate data type mismatch (expected {:?}, got {:?})", $tensors[0].dtype(), t.dtype()) + } + }) + .collect::>(); + NdArrayOps::concatenate(&tensors, $dim).into() + })* + _ => panic!("Unsupported dtype: {:?}", $tensors[0].dtype()) + } + }; +} + +impl TensorMetadata for NdArrayTensor { + fn dtype(&self) -> DType { + match self { + NdArrayTensor::F64(_) => DType::F64, + NdArrayTensor::F32(_) => DType::F32, + NdArrayTensor::I64(_) => DType::I64, + NdArrayTensor::I32(_) => DType::I32, + NdArrayTensor::I16(_) => DType::I16, + NdArrayTensor::I8(_) => DType::I8, + NdArrayTensor::U64(_) => DType::U64, + NdArrayTensor::U32(_) => DType::U32, + NdArrayTensor::U16(_) => DType::U16, + NdArrayTensor::U8(_) => DType::U8, + NdArrayTensor::Bool(_) => DType::Bool, + } + } + + fn shape(&self) -> Shape { + // Use storage's shape method (works for both borrowed and owned) + macro_rules! get_shape { + ($($variant:ident),*) => { + match self { + $(NdArrayTensor::$variant(storage) => Shape::from(storage.shape().to_vec()),)* + } + }; + } + get_shape!(F64, F32, I64, I32, I16, I8, U64, U32, U16, U8, Bool) + } + + fn rank(&self) -> usize { + self.shape().num_dims() + } +} + +pub(crate) trait ShapeOps { + fn num_dims(self) -> usize; + fn num_elements(self) -> usize; + fn dims(self) -> [usize; N]; + fn into_shape(self) -> Shape; +} + +impl ShapeOps for &[usize] { + fn num_dims(self) -> usize { + self.len() + } + + fn num_elements(self) -> usize { + self.iter().product() + } + + fn dims(self) -> [usize; N] { + self.try_into().unwrap() + } + + fn into_shape(self) -> Shape { + Shape::from(self) + } +} + +mod utils { + use burn_std::tensor::is_contiguous; + + use super::*; + + impl NdArrayTensor { + pub(crate) fn into_data(self) -> TensorData { + let shape = self.shape(); + let contiguous = self.is_contiguous(); + + fn inner( + shape: Shape, + is_contiguous: bool, + array: ArcArray, + ) -> TensorData { + let vec = if is_contiguous { + match array.try_into_owned_nocopy() { + Ok(owned) => { + let (mut vec, offset) = owned.into_raw_vec_and_offset(); + if let Some(offset) = offset { + vec.drain(..offset); + } + if vec.len() > shape.num_elements() { + vec.drain(shape.num_elements()..vec.len()); + } + vec + } + Err(array) => array.into_iter().collect(), + } + } else { + array.into_iter().collect() + }; + + TensorData::new(vec, shape) + } + + // Convert storage to owned array before extracting data + execute_with_dtype!(self, |arr| inner(shape, contiguous, arr)) + } + + pub(crate) fn is_contiguous(&self) -> bool { + // For borrowed data, we assume it's contiguous (it came from TensorData which is contiguous) + // For owned data, we check the strides + macro_rules! check_contiguous { + ($($variant:ident),*) => { + match self { + $(NdArrayTensor::$variant(storage) => { + match storage { + NdArrayStorage::Borrowed { .. } => { + // Borrowed storage requires contiguous row-major data + // (see NdArrayStorage::from_borrowed documentation) + true + } + NdArrayStorage::Owned(array) => { + let shape = array.shape(); + let mut strides = Vec::with_capacity(array.strides().len()); + for &stride in array.strides() { + if stride <= 0 { + return false; + } + strides.push(stride as usize); + } + is_contiguous(shape, &strides) + } + } + })* + } + }; + } + check_contiguous!(F64, F32, I64, I32, I16, I8, U64, U32, U16, U8, Bool) + } + } +} + +/// Converts a slice of usize to a typed dimension. +#[macro_export(local_inner_macros)] +macro_rules! to_typed_dims { + ( + $n:expr, + $dims:expr, + justdim + ) => {{ + let mut dims = [0; $n]; + for i in 0..$n { + dims[i] = $dims[i]; + } + let dim: Dim<[usize; $n]> = Dim(dims); + dim + }}; +} + +/// Reshapes an array into a tensor. +#[macro_export(local_inner_macros)] +macro_rules! reshape { + ( + ty $ty:ty, + n $n:expr, + shape $shape:expr, + array $array:expr + ) => {{ + let dim = $crate::to_typed_dims!($n, $shape, justdim); + let array = match $array.is_standard_layout() { + true => { + match $array.to_shape(dim) { + Ok(val) => val.into_shared(), + Err(err) => { + core::panic!("Shape should be compatible shape={dim:?}: {err:?}"); + } + } + }, + false => $array.to_shape(dim).unwrap().as_standard_layout().into_shared(), + }; + array.into_dyn() + }}; + ( + ty $ty:ty, + shape $shape:expr, + array $array:expr, + d $D:expr + ) => {{ + match $D { + 1 => reshape!(ty $ty, n 1, shape $shape, array $array), + 2 => reshape!(ty $ty, n 2, shape $shape, array $array), + 3 => reshape!(ty $ty, n 3, shape $shape, array $array), + 4 => reshape!(ty $ty, n 4, shape $shape, array $array), + 5 => reshape!(ty $ty, n 5, shape $shape, array $array), + 6 => reshape!(ty $ty, n 6, shape $shape, array $array), + _ => core::panic!("NdArray supports arrays up to 6 dimensions, received: {}", $D), + } + }}; +} + +/// Slice a tensor +#[macro_export] +macro_rules! slice { + ($tensor:expr, $slices:expr) => { + slice!($tensor, $slices, F64, F32, I64, I32, I16, I8, U64, U32, U16, U8, Bool) + }; + ($tensor:expr, $slices:expr, $($variant:ident),*) => { + match $tensor { + $(NdArrayTensor::$variant(s) => { NdArrayOps::slice(s.view(), $slices).into() })* + } + }; +} + +impl NdArrayTensor { + /// Create a new [ndarray tensor](NdArrayTensor) from [data](TensorData). + /// + /// This method attempts zero-copy loading when possible. If the data has properly + /// aligned bytes that can be borrowed, it creates a borrowed tensor. Otherwise, + /// it falls back to copying the data. + /// + /// Zero-copy loading works when: + /// - The data's bytes are properly aligned for the element type + /// - The bytes can be borrowed (e.g., from mmap'd file or static data) + pub fn from_data(data: TensorData) -> NdArrayTensor { + // Try borrowed storage first, fall back to owned if not possible + match Self::try_from_data_borrowed(data) { + Ok(tensor) => tensor, + Err(data) => Self::from_data_owned(data), + } + } + + /// Try to create a tensor with borrowed storage (zero-copy). + /// + /// Takes ownership of TensorData and returns it back on failure. + /// No cloning occurs - bytes are moved into storage or returned on failure. + /// + /// Returns `Err(data)` if borrowing is not possible (e.g., misaligned data). + fn try_from_data_borrowed(data: TensorData) -> Result { + let TensorData { + bytes, + shape, + dtype, + } = data; + + macro_rules! try_borrow { + ($ty:ty, $variant:ident, $bytes:expr, $shape:expr) => { + match NdArrayStorage::<$ty>::from_borrowed($bytes, $shape) { + Ok(storage) => return Ok(NdArrayTensor::$variant(storage)), + Err((bytes, shape)) => (bytes, shape), + } + }; + } + + // Try to create borrowed storage; get bytes back on failure + let (bytes, shape) = match dtype { + DType::F64 => try_borrow!(f64, F64, bytes, shape), + DType::F32 => try_borrow!(f32, F32, bytes, shape), + DType::I64 => try_borrow!(i64, I64, bytes, shape), + DType::I32 => try_borrow!(i32, I32, bytes, shape), + DType::I16 => try_borrow!(i16, I16, bytes, shape), + DType::I8 => try_borrow!(i8, I8, bytes, shape), + DType::U64 => try_borrow!(u64, U64, bytes, shape), + DType::U32 => try_borrow!(u32, U32, bytes, shape), + DType::U16 => try_borrow!(u16, U16, bytes, shape), + DType::U8 => try_borrow!(u8, U8, bytes, shape), + DType::Bool => try_borrow!(bool, Bool, bytes, shape), + _ => (bytes, shape), // QFloat not supported for zero-copy + }; + + Err(TensorData { + bytes, + shape, + dtype, + }) + } + + /// Create a tensor with owned storage. + /// + /// This may or may not copy data depending on whether the underlying bytes + /// can be reclaimed (via `try_into_vec`). If bytes are uniquely owned, + /// no copy occurs; otherwise data is copied to a new allocation. + fn from_data_owned(mut data: TensorData) -> NdArrayTensor { + let shape = mem::take(&mut data.shape); + + macro_rules! execute { + ($data: expr, [$($dtype: ident => $ty: ty),*]) => { + match $data.dtype { + $(DType::$dtype => { + match data.into_vec::<$ty>() { + // Safety: TensorData checks shape validity on creation + Ok(vec) => unsafe { ArrayD::from_shape_vec_unchecked(shape, vec) }.into_shared(), + Err(err) => panic!("Data should have the same element type as the tensor {err:?}"), + }.into() + },)* + other => unimplemented!("Unsupported dtype {other:?}"), + } + }; + } + + execute!(data, [ + F64 => f64, F32 => f32, + I64 => i64, I32 => i32, I16 => i16, I8 => i8, + U64 => u64, U32 => u32, U16 => u16, U8 => u8, + Bool => bool + ]) + } +} + +/// A quantized tensor for the ndarray backend. +#[derive(Clone, Debug)] +pub struct NdArrayQTensor { + /// The quantized tensor. + pub qtensor: NdArrayTensor, + /// The quantization scheme. + pub scheme: QuantScheme, + /// The quantization parameters. + pub qparams: Vec>, +} + +impl NdArrayQTensor { + /// Returns the quantization strategy, including quantization parameters, for the given tensor. + pub fn strategy(&self) -> QuantizationStrategy { + match self.scheme { + QuantScheme { + level: QuantLevel::Tensor, + mode: QuantMode::Symmetric, + value: + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::E4M3 + | QuantValue::E5M2 + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::E2M1 + | QuantValue::Q2F + | QuantValue::Q2S, + .. + } => QuantizationStrategy::PerTensorSymmetric(SymmetricQuantization::init( + self.qparams[0].scales, + self.scheme.value, + )), + QuantScheme { + level: QuantLevel::Block(block_size), + mode: QuantMode::Symmetric, + value: + QuantValue::Q8F + | QuantValue::Q8S + | QuantValue::E4M3 + | QuantValue::E5M2 + | QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::E2M1 + | QuantValue::Q2F + | QuantValue::Q2S, + .. + } => QuantizationStrategy::PerBlockSymmetric( + self.qparams + .iter() + .map(|q| SymmetricQuantization::init(q.scales, self.scheme.value)) + .collect(), + block_size, + ), + } + } +} + +impl QTensorPrimitive for NdArrayQTensor { + fn scheme(&self) -> &QuantScheme { + &self.scheme + } + + fn default_scheme() -> QuantScheme { + QuantScheme::default().with_store(burn_backend::quantization::QuantStore::Native) + } +} + +impl TensorMetadata for NdArrayQTensor { + fn dtype(&self) -> DType { + DType::QFloat(self.scheme) + } + + fn shape(&self) -> Shape { + self.qtensor.shape() + } + + fn rank(&self) -> usize { + self.shape().num_dims() + } +} + +#[cfg(test)] +mod tests { + use crate::NdArray; + use alloc::vec; + + use super::*; + use burn_backend::{ + Distribution, + ops::{FloatTensorOps, QTensorOps}, + quantization::{QuantStore, QuantizationParametersPrimitive}, + }; + use burn_std::rand::get_seeded_rng; + + #[test] + fn should_support_into_and_from_data_1d() { + let data_expected = TensorData::random::( + Shape::new([3]), + Distribution::Default, + &mut get_seeded_rng(), + ); + let tensor = NdArrayTensor::from_data(data_expected.clone()); + + let data_actual = tensor.into_data(); + + assert_eq!(data_expected, data_actual); + } + + #[test] + fn should_support_into_and_from_data_2d() { + let data_expected = TensorData::random::( + Shape::new([2, 3]), + Distribution::Default, + &mut get_seeded_rng(), + ); + let tensor = NdArrayTensor::from_data(data_expected.clone()); + + let data_actual = tensor.into_data(); + + assert_eq!(data_expected, data_actual); + } + + #[test] + fn should_support_into_and_from_data_3d() { + let data_expected = TensorData::random::( + Shape::new([2, 3, 4]), + Distribution::Default, + &mut get_seeded_rng(), + ); + let tensor = NdArrayTensor::from_data(data_expected.clone()); + + let data_actual = tensor.into_data(); + + assert_eq!(data_expected, data_actual); + } + + #[test] + fn should_support_into_and_from_data_4d() { + let data_expected = TensorData::random::( + Shape::new([2, 3, 4, 2]), + Distribution::Default, + &mut get_seeded_rng(), + ); + let tensor = NdArrayTensor::from_data(data_expected.clone()); + + let data_actual = tensor.into_data(); + + assert_eq!(data_expected, data_actual); + } + + #[test] + fn should_support_qtensor_strategy() { + type B = NdArray; + let scale: f32 = 0.009_019_608; + let device = Default::default(); + + let tensor = B::float_from_data(TensorData::from([-1.8f32, -1.0, 0.0, 0.5]), &device); + let scheme = QuantScheme::default() + .with_value(QuantValue::Q8S) + .with_store(QuantStore::Native); + let qparams = QuantizationParametersPrimitive { + scales: B::float_from_data(TensorData::from([scale]), &device), + }; + let qtensor: NdArrayQTensor = B::quantize(tensor, &scheme, qparams); + + assert_eq!(qtensor.scheme(), &scheme); + assert_eq!( + qtensor.strategy(), + QuantizationStrategy::PerTensorSymmetric(SymmetricQuantization::init( + scale, + QuantValue::Q8S + )) + ); + } + + // ========================================================================== + // Zero-copy integration tests + // These tests verify end-to-end zero-copy behavior through NdArrayTensor. + // ========================================================================== + + #[test] + fn zero_copy_creates_borrowed_storage() { + // Verify that from_data creates borrowed storage when possible. + // Note: For native allocations, Bytes::clone() copies data internally, + // but the storage type (Borrowed) is preserved, which is important for + // the is_unique() behavior that triggers copy-on-write. + use burn_std::Bytes; + + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let tensor_data = TensorData::from_bytes(bytes, Shape::new([2, 2]), DType::F32); + + let tensor = NdArrayTensor::from_data(tensor_data); + + match &tensor { + NdArrayTensor::F32(storage) => { + assert!( + storage.is_borrowed(), + "ZERO-COPY REGRESSION: from_data should create borrowed storage \ + for properly aligned TensorData with Bytes" + ); + assert!( + !storage.is_unique(), + "ZERO-COPY REGRESSION: borrowed storage must report is_unique() == false" + ); + } + _ => panic!("Expected F32 tensor"), + } + } + + #[test] + fn zero_copy_data_integrity() { + // Verify data is correctly accessible through borrowed storage + use burn_std::Bytes; + + let data: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let bytes = Bytes::from_elems(data); + let tensor_data = TensorData::from_bytes(bytes, Shape::new([2, 2]), DType::F32); + + let tensor = NdArrayTensor::from_data(tensor_data); + + match &tensor { + NdArrayTensor::F32(storage) => { + let view = storage.view(); + assert_eq!(view[[0, 0]], 1.0); + assert_eq!(view[[0, 1]], 2.0); + assert_eq!(view[[1, 0]], 3.0); + assert_eq!(view[[1, 1]], 4.0); + } + _ => panic!("Expected F32 tensor"), + } + } + + #[test] + fn zero_copy_fallback_when_bytes_owned() { + // When TensorData owns bytes exclusively, it may use the copy path + // This is expected behavior - verify it still works correctly + let data = TensorData::from([1.0f32, 2.0, 3.0, 4.0]); + let tensor = NdArrayTensor::from_data(data.clone()); + let result = tensor.into_data(); + + assert_eq!(data, result, "Data should round-trip correctly"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-nn/Cargo.toml new file mode 100644 index 0000000..abad124 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/Cargo.toml @@ -0,0 +1,87 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Neural network building blocks for the Burn deep learning framework" +documentation = "https://docs.rs/burn-nn" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-nn" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-nn" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [ + "std", + "burn-core/default", +] +doc = [ + "std", + # Doc features + "burn-core/doc", +] +std = [ + "burn-core/std", + "num-traits/std", +] +tracing = [ + "burn-core/tracing", + "burn-cuda?/tracing", + "burn-rocm?/tracing", + "burn-tch?/tracing", + "burn-wgpu?/tracing", + "burn-fusion?/tracing", +] + +test-cuda = [ + "burn-cuda/default", +] # To use cuda during testing, default uses ndarray. +test-rocm = [ + "burn-rocm/default", +] # To use hip during testing, default uses ndarray. +test-tch = [ + "burn-tch/default", +] # To use tch during testing, default uses ndarray. +test-wgpu = [ + "burn-wgpu/default", +] # To use wgpu during testing, default uses ndarray. +test-vulkan = [ + "test-wgpu", + "burn-wgpu/vulkan", +] # To use wgpu-spirv during testing, default uses ndarray. +test-metal = [ + "test-wgpu", + "burn-wgpu/metal", +] # To use wgpu-spirv during testing, default uses ndarray. + +# Memory checks are disabled by default +test-memory-checks = ["burn-fusion/memory-checks"] + +[dependencies] + +# ** Please make sure all dependencies support no_std when std is disabled ** +burn-core = { path = "../burn-core", version = "=0.21.0-pre.2", default-features = false } + +num-traits = { workspace = true } + +# FOR TESTING +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-rocm = { path = "../burn-rocm", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-remote = { path = "../burn-remote", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-router = { path = "../burn-router", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } + +[dev-dependencies] +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2" } +rstest = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/README.md b/crates/stable-diffusion-burn/burn-crates/burn-nn/README.md new file mode 100644 index 0000000..daef9bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/README.md @@ -0,0 +1,3 @@ +# Burn Neural Networks + +Core building blocks for Burn neural networks. \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/activation_wrapper.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/activation_wrapper.rs new file mode 100644 index 0000000..eeff82c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/activation_wrapper.rs @@ -0,0 +1,598 @@ +use burn_core as burn; + +use crate::activation::{ + Celu, CeluConfig, Elu, EluConfig, Gelu, HardShrink, HardShrinkConfig, HardSigmoid, + HardSigmoidConfig, HardSwish, LeakyRelu, LeakyReluConfig, PRelu, PReluConfig, Relu, Selu, + Shrink, ShrinkConfig, Sigmoid, SoftShrink, SoftShrinkConfig, Softplus, SoftplusConfig, + Softsign, SwiGlu, SwiGluConfig, Tanh, ThresholdedRelu, ThresholdedReluConfig, +}; +use burn::config::Config; +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// [`Activation`] Configuration. +#[derive(Config, Debug)] +#[non_exhaustive] +pub enum ActivationConfig { + /// [`Gelu`] activation layer. + Gelu, + + /// [`Gelu`] activation layer with tanh approximation. + GeluApproximate, + + /// [`PRelu`] activation layer. + PRelu(PReluConfig), + + /// [`Relu`] activation layer. + Relu, + + /// [`LeakyRelu`] activation layer. + LeakyRelu(LeakyReluConfig), + + /// [`SwiGlu`] activation layer. + SwiGlu(SwiGluConfig), + + /// [`Selu`] activation layer. + Selu, + + /// [`Sigmoid`] activation layer. + Sigmoid, + + /// [`Tanh`] activation layer. + Tanh, + + /// [`HardSigmoid`] activation layer. + HardSigmoid(HardSigmoidConfig), + + /// [`HardSwish`] activation layer. + HardSwish, + + /// [`Softplus`] activation layer. + Softplus(SoftplusConfig), + + /// [`Softsign`] activation layer. + Softsign, + + /// [`Elu`] activation layer. + Elu(EluConfig), + + /// [`Celu`] activation layer. + Celu(CeluConfig), + + /// [`ThresholdedRelu`] activation layer. + ThresholdedRelu(ThresholdedReluConfig), + + /// [`HardShrink`] activation layer. + HardShrink(HardShrinkConfig), + + /// [`SoftShrink`] activation layer. + SoftShrink(SoftShrinkConfig), + + /// [`Shrink`] activation layer. + Shrink(ShrinkConfig), +} + +impl From for ActivationConfig { + fn from(config: PReluConfig) -> Self { + Self::PRelu(config) + } +} + +impl From for ActivationConfig { + fn from(config: LeakyReluConfig) -> Self { + Self::LeakyRelu(config) + } +} + +impl From for ActivationConfig { + fn from(config: SwiGluConfig) -> Self { + Self::SwiGlu(config) + } +} + +impl From for ActivationConfig { + fn from(config: HardSigmoidConfig) -> Self { + Self::HardSigmoid(config) + } +} + +impl From for ActivationConfig { + fn from(config: SoftplusConfig) -> Self { + Self::Softplus(config) + } +} + +impl From for ActivationConfig { + fn from(config: EluConfig) -> Self { + Self::Elu(config) + } +} + +impl From for ActivationConfig { + fn from(config: CeluConfig) -> Self { + Self::Celu(config) + } +} + +impl From for ActivationConfig { + fn from(config: ThresholdedReluConfig) -> Self { + Self::ThresholdedRelu(config) + } +} + +impl From for ActivationConfig { + fn from(config: HardShrinkConfig) -> Self { + Self::HardShrink(config) + } +} + +impl From for ActivationConfig { + fn from(config: SoftShrinkConfig) -> Self { + Self::SoftShrink(config) + } +} + +impl From for ActivationConfig { + fn from(config: ShrinkConfig) -> Self { + Self::Shrink(config) + } +} + +impl ActivationConfig { + /// Initialize a wrapped activation layer. + pub fn init(&self, device: &B::Device) -> Activation { + match self { + ActivationConfig::Relu => Relu.into(), + ActivationConfig::LeakyRelu(conf) => conf.init().into(), + ActivationConfig::Gelu => Gelu::new().into(), + ActivationConfig::GeluApproximate => Gelu::new_approximate().into(), + ActivationConfig::PRelu(conf) => conf.init(device).into(), + ActivationConfig::SwiGlu(conf) => conf.init(device).into(), + ActivationConfig::HardSigmoid(conf) => conf.init().into(), + ActivationConfig::HardSwish => HardSwish.into(), + ActivationConfig::Softplus(conf) => conf.init().into(), + ActivationConfig::Selu => Selu.into(), + ActivationConfig::Sigmoid => Sigmoid.into(), + ActivationConfig::Tanh => Tanh.into(), + ActivationConfig::Softsign => Softsign.into(), + ActivationConfig::Elu(conf) => conf.init().into(), + ActivationConfig::Celu(conf) => conf.init().into(), + ActivationConfig::HardShrink(conf) => conf.init().into(), + ActivationConfig::SoftShrink(conf) => conf.init().into(), + ActivationConfig::Shrink(conf) => conf.init().into(), + ActivationConfig::ThresholdedRelu(conf) => conf.init().into(), + } + } +} + +/// Activation Layer Wrapper. +/// +/// Provides support for many in-built `burn::nn` activations. +#[derive(Module, Debug)] +#[non_exhaustive] +#[allow(clippy::large_enum_variant)] +pub enum Activation { + /// [`Gelu`] activation layer. + Gelu(Gelu), + + /// [`PRelu`] activation layer. + PRelu(PRelu), + + /// [`Relu`] activation layer. + Relu(Relu), + + /// [`LeakyRelu`] activation layer. + LeakyRelu(LeakyRelu), + + /// [`SwiGlu`] activation layer. + SwiGlu(SwiGlu), + + /// [`Selu`] activation layer. + Selu(Selu), + + /// [`Sigmoid`] activation layer. + Sigmoid(Sigmoid), + + /// [`Tanh`] activation layer. + Tanh(Tanh), + + /// [`HardSigmoid`] activation layer. + HardSigmoid(HardSigmoid), + + /// [`HardSwish`] activation layer. + HardSwish(HardSwish), + + /// [`Softplus`] activation layer. + Softplus(Softplus), + + /// [`Softsign`] activation layer. + Softsign(Softsign), + + /// [`Elu`] activation layer. + Elu(Elu), + + /// [`Celu`] activation layer. + Celu(Celu), + + /// [`ThresholdedRelu`] activation layer. + ThresholdedRelu(ThresholdedRelu), + + /// [`HardShrink`] activation layer. + HardShrink(HardShrink), + + /// [`SoftShrink`] activation layer. + SoftShrink(SoftShrink), + + /// [`Shrink`] activation layer. + Shrink(Shrink), +} + +impl From for Activation { + fn from(layer: Gelu) -> Self { + Self::Gelu(layer) + } +} + +impl From> for Activation { + fn from(layer: PRelu) -> Self { + Self::PRelu(layer) + } +} + +impl From for Activation { + fn from(layer: Relu) -> Self { + Self::Relu(layer) + } +} + +impl From for Activation { + fn from(layer: LeakyRelu) -> Self { + Self::LeakyRelu(layer) + } +} + +impl From> for Activation { + fn from(layer: SwiGlu) -> Self { + Self::SwiGlu(layer) + } +} + +impl From for Activation { + fn from(layer: Selu) -> Self { + Self::Selu(layer) + } +} + +impl From for Activation { + fn from(layer: Sigmoid) -> Self { + Self::Sigmoid(layer) + } +} + +impl From for Activation { + fn from(layer: Tanh) -> Self { + Self::Tanh(layer) + } +} + +impl From for Activation { + fn from(layer: HardSigmoid) -> Self { + Self::HardSigmoid(layer) + } +} + +impl From for Activation { + fn from(layer: HardSwish) -> Self { + Self::HardSwish(layer) + } +} + +impl From for Activation { + fn from(layer: Softplus) -> Self { + Self::Softplus(layer) + } +} + +impl From for Activation { + fn from(layer: Softsign) -> Self { + Self::Softsign(layer) + } +} + +impl From for Activation { + fn from(layer: Elu) -> Self { + Self::Elu(layer) + } +} + +impl From for Activation { + fn from(layer: Celu) -> Self { + Self::Celu(layer) + } +} + +impl From for Activation { + fn from(layer: ThresholdedRelu) -> Self { + Self::ThresholdedRelu(layer) + } +} + +impl From for Activation { + fn from(layer: HardShrink) -> Self { + Self::HardShrink(layer) + } +} + +impl From for Activation { + fn from(layer: SoftShrink) -> Self { + Self::SoftShrink(layer) + } +} + +impl From for Activation { + fn from(layer: Shrink) -> Self { + Self::Shrink(layer) + } +} + +impl Activation { + /// Forward pass. + pub fn forward(&self, input: Tensor) -> Tensor { + match self { + Activation::Relu(layer) => layer.forward(input), + Activation::LeakyRelu(layer) => layer.forward(input), + Activation::Gelu(layer) => layer.forward(input), + Activation::PRelu(layer) => layer.forward(input), + Activation::SwiGlu(layer) => layer.forward(input), + Activation::HardSigmoid(layer) => layer.forward(input), + Activation::HardSwish(layer) => layer.forward(input), + Activation::Softplus(layer) => layer.forward(input), + Activation::Selu(layer) => layer.forward(input), + Activation::Sigmoid(layer) => layer.forward(input), + Activation::Tanh(layer) => layer.forward(input), + Activation::Softsign(layer) => layer.forward(input), + Activation::Elu(layer) => layer.forward(input), + Activation::Celu(layer) => layer.forward(input), + Activation::ThresholdedRelu(layer) => layer.forward(input), + Activation::HardShrink(layer) => layer.forward(input), + Activation::SoftShrink(layer) => layer.forward(input), + Activation::Shrink(layer) => layer.forward(input), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::module::Module; + + fn make_input(device: &B::Device) -> Tensor { + Tensor::from_data([[-1.0, -0.5, 0.0], [1.0, 0.5, 0.0]], device) + } + + fn expect_tensor(actual: Tensor, expected: Tensor) { + actual.to_data().assert_eq(&expected.to_data(), true); + } + + fn check_stateless_config_output( + config: ActivationConfig, + input: Tensor, + expected: Tensor, + device: &B::Device, + ) { + let act = config.init(device); + let output = act.forward(input); + expect_tensor(output, expected); + } + + #[test] + fn test_gelu() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Gelu::new().forward(input.clone()); + + check_stateless_config_output(ActivationConfig::Gelu, input, expected, &device) + } + + #[test] + fn test_gelu_approximate() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Gelu::new_approximate().forward(input.clone()); + + check_stateless_config_output(ActivationConfig::GeluApproximate, input, expected, &device) + } + + #[test] + fn test_prelu() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = PReluConfig::new(); + let expected = inner_config.init(&device).forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_relu() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Relu.forward(input.clone()); + + check_stateless_config_output(ActivationConfig::Relu, input, expected, &device) + } + + #[test] + fn test_leaky_relu() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = LeakyReluConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_swi_glu() { + let device = Default::default(); + let input = make_input::(&device); + + let d_input = input.shape()[1]; + let d_output = 2 * d_input; + + let inner_config = SwiGluConfig::new(d_input, d_output); + let mut reference: SwiGlu = inner_config.init(&device); + + let config: ActivationConfig = inner_config.into(); + let layer = config.init(&device); + + match &layer { + Activation::SwiGlu(inner) => { + // Clone the initialized weights. + let state = inner.clone().into_record(); + reference = reference.load_record(state); + } + _ => unreachable!(), + }; + + expect_tensor( + layer.forward(input.clone()), + reference.forward(input.clone()), + ) + } + + #[test] + fn test_selu() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Selu.forward(input.clone()); + + check_stateless_config_output(ActivationConfig::Selu, input, expected, &device) + } + + #[test] + fn test_sigmoid() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Sigmoid.forward(input.clone()); + + check_stateless_config_output(ActivationConfig::Sigmoid, input, expected, &device) + } + + #[test] + fn test_tanh() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Tanh.forward(input.clone()); + + check_stateless_config_output(ActivationConfig::Tanh, input, expected, &device) + } + + #[test] + fn test_hard_sigmoid() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = HardSigmoidConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_softsign() { + let device = Default::default(); + let input = make_input::(&device); + + let expected = Softsign.forward(input.clone()); + + check_stateless_config_output(ActivationConfig::Softsign, input, expected, &device) + } + + #[test] + fn test_elu() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = EluConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_softplus() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = SoftplusConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_celu() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = CeluConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_thresholded_relu() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = ThresholdedReluConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_hard_shrink() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = HardShrinkConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_soft_shrink() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = SoftShrinkConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } + + #[test] + fn test_shrink() { + let device = Default::default(); + let input = make_input::(&device); + + let inner_config = ShrinkConfig::new(); + let expected = inner_config.init().forward(input.clone()); + + check_stateless_config_output(inner_config.into(), input, expected, &device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/celu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/celu.rs new file mode 100644 index 0000000..0c44ced --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/celu.rs @@ -0,0 +1,104 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::activation::celu; +use burn::tensor::backend::Backend; + +/// CELU (Continuously Differentiable Exponential Linear Unit) layer. +/// +/// Applies the CELU function element-wise: +/// `celu(x) = max(0, x) + min(0, alpha * (exp(x / alpha) - 1))` +/// +/// Should be created with [CeluConfig](CeluConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Celu { + /// The alpha value for the CELU formulation. + pub alpha: f64, +} + +/// Configuration to create a [Celu](Celu) layer using the [init function](CeluConfig::init). +#[derive(Config, Debug)] +pub struct CeluConfig { + /// The alpha value for the CELU formulation. Default is 1.0 + #[config(default = "1.0")] + pub alpha: f64, +} + +impl CeluConfig { + /// Initialize a new [Celu](Celu) Layer + pub fn init(&self) -> Celu { + Celu { alpha: self.alpha } + } +} + +impl ModuleDisplay for Celu { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("alpha", &self.alpha).optional() + } +} + +impl Celu { + /// Forward pass for the Celu layer. + /// + /// See [celu](burn::tensor::activation::celu) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + celu(input, self.alpha) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_celu_forward() { + let device = ::Device::default(); + let model: Celu = CeluConfig::new().init(); + let input = + Tensor::::from_data(TensorData::from([[0.5, -0.5, -1.0]]), &device); + let out = model.forward(input); + // celu(0.5, 1) = 0.5 + // celu(-0.5, 1) = 1 * (exp(-0.5) - 1) = -0.393469 + // celu(-1.0, 1) = 1 * (exp(-1) - 1) = -0.632121 + let expected = TensorData::from([[0.5, -0.393469, -0.632121]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_celu_with_alpha() { + let device = ::Device::default(); + let model: Celu = CeluConfig::new().with_alpha(2.0).init(); + let input = Tensor::::from_data(TensorData::from([[0.0, -2.0]]), &device); + let out = model.forward(input); + // celu(0, 2) = 0 + // celu(-2, 2) = 2 * (exp(-1) - 1) = -1.264241 + let expected = TensorData::from([[0.0, -1.264241]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = CeluConfig::new().init(); + assert_eq!(alloc::format!("{config}"), "Celu {alpha: 1}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/elu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/elu.rs new file mode 100644 index 0000000..d7b1f38 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/elu.rs @@ -0,0 +1,85 @@ +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_core as burn; + +use burn::tensor::activation::elu; + +/// ELU (Exponential Linear Unit) layer. +/// +/// Should be created with [EluConfig](EluConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Elu { + /// The alpha value. + pub alpha: f64, +} +/// Configuration to create an [Elu](Elu) layer using the [init function](EluConfig::init). +#[derive(Config, Debug)] +pub struct EluConfig { + /// The alpha value. Default is 1.0 + #[config(default = "1.0")] + pub alpha: f64, +} +impl EluConfig { + /// Initialize a new [Elu](Elu) Layer + pub fn init(&self) -> Elu { + Elu { alpha: self.alpha } + } +} + +impl ModuleDisplay for Elu { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("alpha", &self.alpha).optional() + } +} + +impl Elu { + /// Forward pass for the ELU layer. + /// + /// See [elu](burn::tensor::activation::elu) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + elu(input, self.alpha) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_elu_forward() { + let device = ::Device::default(); + let model: Elu = EluConfig::new().init(); + let input = + Tensor::::from_data(TensorData::from([[0.4410, -0.2507]]), &device); + let out = model.forward(input); + // elu(0.4410, 1.0) = 0.4410 + // elu(-0.2507, 1.0) = 1.0 * (exp(-0.2507) - 1) = -0.22186 + let expected = TensorData::from([[0.4410, -0.22186]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = EluConfig::new().init(); + assert_eq!(alloc::format!("{config}"), "Elu {alpha: 1}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/gelu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/gelu.rs new file mode 100644 index 0000000..e1588aa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/gelu.rs @@ -0,0 +1,82 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the Gaussian Error Linear Units function element-wise. +/// +/// See also [gelu](burn::tensor::activation::gelu) +/// +/// When `approximate` is true, uses the tanh approximation: +/// `0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))` +#[derive(Module, Clone, Debug, Default)] +pub struct Gelu { + /// Whether to use tanh approximation. + pub approximate: bool, +} + +impl Gelu { + /// Create the module with exact GELU. + pub fn new() -> Self { + Self::default() + } + + /// Create the module with tanh approximation. + pub fn new_approximate() -> Self { + Self { approximate: true } + } + + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + if self.approximate { + burn::tensor::activation::gelu_approximate(input) + } else { + burn::tensor::activation::gelu(input) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::Tolerance; + use burn::tensor::ops::FloatElem; + + type FT = FloatElem; + + #[test] + fn display() { + let layer = Gelu::new(); + + assert_eq!(alloc::format!("{layer}"), "Gelu {\n approximate: false\n}"); + } + + #[test] + fn forward_approximate() { + let device = Default::default(); + let input = + Tensor::::from_data([[-1.0, 0.0, 1.0], [0.5, -0.5, 2.0]], &device); + + let output = Gelu::new_approximate().forward(input); + + // PyTorch: torch.nn.functional.gelu(x, approximate="tanh") + let expected = Tensor::::from_data( + [ + [-0.1588079929, 0.0000000000, 0.8411920071], + [0.3457140028, -0.1542859972, 1.9545977116], + ], + &device, + ); + + output + .into_data() + .assert_approx_eq::(&expected.into_data(), Tolerance::rel_abs(1e-5, 1e-5)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/glu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/glu.rs new file mode 100644 index 0000000..2d0201f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/glu.rs @@ -0,0 +1,51 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the gated linear unit function. +/// +/// See also [glu](burn::tensor::activation::glu) +#[derive(Module, Clone, Debug, Default)] +pub struct GLU { + dim: usize, +} + +impl GLU { + /// Create the module. + /// + /// # Arguments + /// * `dim` - The dimension on which to split the input. + pub fn new(dim: usize) -> Self { + Self { dim } + } + + /// Applies the gated linear unit function. + /// + /// GLU(a,b)=a⊗σ(b) where `a` is the first half of the input matrices and `b` is the second half. + /// + /// **Note**: + /// * The size of the input tensor along `dim` must be divisible by 2. + /// + /// ### Arguments + /// * `tensor` - The input tensor. + /// + /// ### Returns + /// * A tensor with the same shape as the input, except the size along `dim` is halved. + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::glu(input, self.dim) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let layer = GLU::new(1); + + assert_eq!(alloc::format!("{layer}"), "GLU {\n dim: 1\n}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_shrink.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_shrink.rs new file mode 100644 index 0000000..b94dd85 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_shrink.rs @@ -0,0 +1,98 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::activation::hard_shrink; +use burn::tensor::backend::Backend; + +/// Hard Shrink layer. +/// +/// Applies the Hard Shrink function element-wise: +/// `hard_shrink(x) = x if |x| > lambda else 0` +/// +/// Should be created with [HardShrinkConfig](HardShrinkConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct HardShrink { + /// The lambda value for the Hard Shrink formulation. + pub lambda: f64, +} + +/// Configuration to create a [HardShrink](HardShrink) layer using the [init function](HardShrinkConfig::init). +#[derive(Config, Debug)] +pub struct HardShrinkConfig { + /// The lambda value for the Hard Shrink formulation. Default is 0.5 + #[config(default = "0.5")] + pub lambda: f64, +} + +impl HardShrinkConfig { + /// Initialize a new [HardShrink](HardShrink) Layer + pub fn init(&self) -> HardShrink { + HardShrink { + lambda: self.lambda, + } + } +} + +impl ModuleDisplay for HardShrink { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("lambda", &self.lambda).optional() + } +} + +impl HardShrink { + /// Forward pass for the Hard Shrink layer. + /// + /// See [hard_shrink](burn::tensor::activation::hard_shrink) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + hard_shrink(input, self.lambda) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn test_hard_shrink_forward() { + let device = ::Device::default(); + let model: HardShrink = HardShrinkConfig::new().init(); + let input = + Tensor::::from_data([[0.5, -0.5, -1.0], [8.0, 0.3, 0.0]], &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0_f32, 0.0, -1.0], [8.0, 0.0, 0.0]]); + assert_eq!(out.into_data(), expected); + } + + #[test] + fn test_hard_shrink_with_lambda() { + let device = ::Device::default(); + let model: HardShrink = HardShrinkConfig::new().with_lambda(0.2).init(); + let input = + Tensor::::from_data([[0.1, -0.1, -0.3], [0.5, 0.1, 0.0]], &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0_f32, 0.0, -0.3], [0.5, 0.0, 0.0]]); + assert_eq!(out.into_data(), expected); + } + + #[test] + fn display() { + let config = HardShrinkConfig::new().init(); + assert_eq!(alloc::format!("{config}"), "HardShrink {lambda: 0.5}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_sigmoid.rs new file mode 100644 index 0000000..d1dd60a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_sigmoid.rs @@ -0,0 +1,97 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::activation::hard_sigmoid; +use burn::tensor::backend::Backend; + +/// Hard Sigmoid layer. +/// +/// Should be created with [HardSigmoidConfig](HardSigmoidConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct HardSigmoid { + /// The alpha value. + pub alpha: f64, + /// The beta value. + pub beta: f64, +} +/// Configuration to create a [Hard Sigmoid](HardSigmoid) layer using the [init function](HardSigmoidConfig::init). +#[derive(Config, Debug)] +pub struct HardSigmoidConfig { + /// The alpha value. Default is 0.2 + #[config(default = "0.2")] + pub alpha: f64, + /// The beta value. Default is 0.5 + #[config(default = "0.5")] + pub beta: f64, +} +impl HardSigmoidConfig { + /// Initialize a new [Hard Sigmoid](HardSigmoid) Layer + pub fn init(&self) -> HardSigmoid { + HardSigmoid { + alpha: self.alpha, + beta: self.beta, + } + } +} + +impl ModuleDisplay for HardSigmoid { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("alpha", &self.alpha) + .add("beta", &self.beta) + .optional() + } +} + +impl HardSigmoid { + /// Forward pass for the Hard Sigmoid layer. + /// + /// See [hard_sigmoid](burn::tensor::activation::hard_sigmoid) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + hard_sigmoid(input, self.alpha, self.beta) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_hard_sigmoid_forward() { + let device = ::Device::default(); + let model: HardSigmoid = HardSigmoidConfig::new().init(); + let input = + Tensor::::from_data(TensorData::from([[0.4410, -0.2507]]), &device); + let out = model.forward(input); + let expected = TensorData::from([[0.5882, 0.44986]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = HardSigmoidConfig::new().init(); + assert_eq!( + alloc::format!("{config}"), + "HardSigmoid {alpha: 0.2, beta: 0.5}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_swish.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_swish.rs new file mode 100644 index 0000000..af97b13 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/hard_swish.rs @@ -0,0 +1,58 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::activation::hard_swish; +use burn::tensor::backend::Backend; + +/// Hard Swish layer. +#[derive(Module, Clone, Debug, Default)] +pub struct HardSwish; + +impl HardSwish { + /// Create the module. + pub fn new() -> Self { + Self + } + + /// Forward pass for the Hard Swish layer. + /// + /// See [hard_swish](burn::tensor::activation::hard_swish) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + hard_swish(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_hard_swish_forward() { + let device = ::Device::default(); + let model = HardSwish::new(); + + let input = Tensor::::from_data( + TensorData::from([[3.0f32, -3.0], [0.0, 1.0]]), + &device, + ); + let out = model.forward(input); + let expected = TensorData::from([[3.0f32, 0.0], [0.0, 0.6666667]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let layer = HardSwish::new(); + assert_eq!(alloc::format!("{layer}"), "HardSwish"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/leaky_relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/leaky_relu.rs new file mode 100644 index 0000000..bfed047 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/leaky_relu.rs @@ -0,0 +1,124 @@ +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_core as burn; + +use burn::tensor::activation::leaky_relu; + +/// Leaky ReLu layer. +/// +/// Should be created with [LeakyReluConfig](LeakyReluConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct LeakyRelu { + /// The negative slope. + pub negative_slope: f64, +} +/// Configuration to create a [Leaky Relu](LeakyRelu) layer using the [init function](LeakyReluConfig::init). +#[derive(Config, Debug)] +pub struct LeakyReluConfig { + /// The negative slope. Default is 0.01 + #[config(default = "0.01")] + pub negative_slope: f64, +} +impl LeakyReluConfig { + /// Initialize a new [Leaky Relu](LeakyRelu) Layer + pub fn init(&self) -> LeakyRelu { + LeakyRelu { + negative_slope: self.negative_slope, + } + } +} + +impl ModuleDisplay for LeakyRelu { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("negative_slope", &self.negative_slope) + .optional() + } +} + +impl LeakyRelu { + /// Forward pass for the Leaky ReLu layer. + /// + /// See [leaky_relu](burn::tensor::activation::leaky_relu) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + leaky_relu(input, self.negative_slope) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_leaky_relu_forward() { + let device = ::Device::default(); + let model: LeakyRelu = LeakyReluConfig::new().init(); + let input = + Tensor::::from_data(TensorData::from([[0.4410, -0.2507]]), &device); + let out = model.forward(input); + let expected = TensorData::from([[0.4410, -0.002507]]); + out.to_data().assert_eq(&expected, false); + } + #[test] + fn test_leaky_relu_forward_multi_dim() { + let input = [ + [ + [-1.0222, 1.5810, 0.3457, -1.3530], + [0.0231, 0.8681, 0.2473, -0.0377], + [0.3520, -1.1199, 1.2219, 0.2804], + ], + [ + [1.0002, 0.7259, 0.8779, 0.2084], + [1.5615, -0.1057, -0.4886, -1.5184], + [-0.5523, -0.2741, -0.0210, -1.1352], + ], + ]; + let expected = TensorData::from([ + [ + [-1.0222e-02, 1.5810e+00, 3.457e-01, -1.3530e-02], + [2.31e-02, 8.681e-01, 2.473e-01, -3.77e-04], + [3.52e-01, -1.1199e-02, 1.2219e+00, 2.804e-01], + ], + [ + [1.0002e+00, 7.259e-01, 8.779e-01, 2.084e-01], + [1.5615e+00, -1.057e-03, -4.886e-03, -1.5184e-02], + [-5.523e-03, -2.741e-03, -2.1e-04, -1.1352e-02], + ], + ]); + + let device = ::Device::default(); + let model: LeakyRelu = LeakyReluConfig::new().init(); + let input_data = Tensor::::from_data(TensorData::from(input), &device); + let actual_output = model.forward(input_data); + actual_output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()) + } + + #[test] + fn display() { + let config = LeakyReluConfig::new().init(); + assert_eq!( + alloc::format!("{config}"), + "LeakyRelu {negative_slope: 0.01}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/mod.rs new file mode 100644 index 0000000..ced6eef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/mod.rs @@ -0,0 +1,68 @@ +//! # Activation Layers +//! +//! Users who desire a selectable activation function should +//! consider [`Activation`], which provides an abstraction over: +//! * [`Relu`] - the default, +//! * ['PRelu'] +//! * [`Gelu`] +//! * [`LeakyRelu`] +//! * [`SwiGlu`] +//! * [`Selu`] +//! * [`Sigmoid`] +//! * [`HardSigmoid`] +//! * [`HardSwish`] +//! * [`Softplus`] +//! * [`Softsign`] +//! * [`Tanh`] +//! * [`Elu`] +//! * [`Celu`] +//! * [`ThresholdedRelu`] +//! +//! The activation layer [`GLU`] has shape-changing behaviors +//! not compatible with the common API, and is not included +//! in the abstraction wrappers. + +mod activation_wrapper; + +// These are pub(crate) for dual-export in `nn` without re-exporting +// all of `nn.activation`, or manually listing each symbol. +pub(crate) mod celu; +pub(crate) mod elu; +pub(crate) mod gelu; +pub(crate) mod glu; +pub(crate) mod hard_shrink; +pub(crate) mod hard_sigmoid; +pub(crate) mod hard_swish; +pub(crate) mod leaky_relu; +pub(crate) mod prelu; +pub(crate) mod relu; +pub(crate) mod selu; +pub(crate) mod shrink; +pub(crate) mod sigmoid; +pub(crate) mod soft_shrink; +pub(crate) mod softplus; +pub(crate) mod softsign; +pub(crate) mod swiglu; +pub(crate) mod tanh; +pub(crate) mod thresholded_relu; + +pub use activation_wrapper::*; +pub use celu::*; +pub use elu::*; +pub use gelu::*; +pub use glu::*; +pub use hard_shrink::*; +pub use hard_sigmoid::*; +pub use hard_swish::*; +pub use leaky_relu::*; +pub use prelu::*; +pub use relu::*; +pub use selu::*; +pub use shrink::*; +pub use sigmoid::*; +pub use soft_shrink::*; +pub use softplus::*; +pub use softsign::*; +pub use swiglu::*; +pub use tanh::*; +pub use thresholded_relu::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/prelu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/prelu.rs new file mode 100644 index 0000000..1c170d1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/prelu.rs @@ -0,0 +1,87 @@ +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay, Param}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_core as burn; +/// Parametric Relu layer. +/// +/// Should be created using [PReluConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct PRelu { + /// the weights learnt for PReLu. can be of shape \[1\] or \[num_parameters\] in which case it must + /// be the same as number of channels in the input tensor + pub alpha: Param>, + + /// Alpha value for the PRelu layer + pub alpha_value: f64, +} + +impl ModuleDisplay for PRelu { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [num_parameters] = self.alpha.shape().dims(); + + content + .add("num_parameters", &num_parameters) + .add("alpha_value", &self.alpha_value) + .optional() + } +} + +/// Configuration to create a [Parametric Relu](PRelu) layer using the [init function](PReluConfig::init). +#[derive(Config, Debug)] +pub struct PReluConfig { + /// The number of parameters. + #[config(default = "1")] + pub num_parameters: usize, + /// The learnable weight alpha. Default is 0.25 + #[config(default = "0.25")] + pub alpha: f64, +} + +impl PReluConfig { + /// Initialize a new [Parametric Relu](PRelu) Layer + pub fn init(&self, device: &B::Device) -> PRelu { + PRelu { + // alpha is a tensor of length num_parameters + alpha: Initializer::Constant { value: self.alpha }.init([self.num_parameters], device), + alpha_value: self.alpha, + } + } +} + +impl PRelu { + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + /// + /// See also [prelu](burn::tensor::activation::prelu) for more information. + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::prelu(input, self.alpha.val()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn display() { + let layer = PReluConfig::new().init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "PRelu {num_parameters: 1, alpha_value: 0.25, params: 1}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/relu.rs new file mode 100644 index 0000000..a87068f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/relu.rs @@ -0,0 +1,39 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the rectified linear unit function element-wise +/// See also [relu](burn::tensor::activation::relu) +/// +#[derive(Module, Clone, Debug, Default)] +pub struct Relu; + +impl Relu { + /// Create the module. + pub fn new() -> Self { + Self {} + } + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::relu(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let layer = Relu::new(); + + assert_eq!(alloc::format!("{layer}"), "Relu"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/selu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/selu.rs new file mode 100644 index 0000000..50597b2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/selu.rs @@ -0,0 +1,38 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the Scaled Exponential Linear Unit function element-wise. +/// See also [selu](burn::tensor::activation::selu) +#[derive(Module, Clone, Debug, Default)] +pub struct Selu; + +impl Selu { + /// Create the module. + pub fn new() -> Self { + Self {} + } + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::selu(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let layer = Selu::new(); + + assert_eq!(alloc::format!("{layer}"), "Selu"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/shrink.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/shrink.rs new file mode 100644 index 0000000..a1fb412 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/shrink.rs @@ -0,0 +1,114 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::activation::shrink; +use burn::tensor::backend::Backend; + +/// Shrink layer. +/// +/// Applies the Shrink function element-wise: +/// `shrink(x) = x - bias if x > lambda, x + bias if x < -lambda, 0 otherwise` +/// +/// Should be created with [ShrinkConfig](ShrinkConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Shrink { + /// The lambda value for the Shrink formulation. + pub lambda: f64, + /// The bias value for the Shrink formulation. + // Usually bias = lambda, but need this to handle onnx spec https://onnx.ai/onnx/operators/onnx__Shrink.html + pub bias: f64, +} + +/// Configuration to create a [Shrink](Shrink) layer using the [init function](ShrinkConfig::init). +#[derive(Config, Debug)] +pub struct ShrinkConfig { + /// The lambda value for the Shrink formulation. Default is 0.5 + #[config(default = "0.5")] + pub lambda: f64, + /// The bias value for the Shrink formulation. Default is 0.5. + #[config(default = "0.5")] + pub bias: f64, +} + +impl ShrinkConfig { + /// Initialize a new [Shrink](Shrink) Layer + pub fn init(&self) -> Shrink { + Shrink { + lambda: self.lambda, + bias: self.bias, + } + } +} + +impl ModuleDisplay for Shrink { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("lambda", &self.lambda) + .add("bias", &self.bias) + .optional() + } +} + +impl Shrink { + /// Forward pass for the Shrink layer. + /// + /// See [shrink](burn::tensor::activation::shrink) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + shrink(input, self.lambda, self.bias) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn test_shrink_forward() { + let device = ::Device::default(); + let model: Shrink = ShrinkConfig::new().init(); + let input = + Tensor::::from_data([[0.5, -0.5, -1.0], [8.0, 0.3, 0.0]], &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0_f32, 0.0, -0.5], [7.5, 0.0, 0.0]]); + assert_eq!(out.into_data(), expected); + } + + #[test] + fn test_shrink_with_lambda_and_bias() { + let device = ::Device::default(); + let model: Shrink = ShrinkConfig::new() + .with_lambda(0.25) + .with_bias(0.125) + .init(); + let input = + Tensor::::from_data([[0.125, -0.125, -0.5], [0.75, 0.1, 0.0]], &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0_f32, 0.0, -0.375], [0.625, 0.0, 0.0]]); + assert_eq!(out.into_data(), expected); + } + + #[test] + fn display() { + let config = ShrinkConfig::new().init(); + assert_eq!( + alloc::format!("{config}"), + "Shrink {lambda: 0.5, bias: 0.5}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/sigmoid.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/sigmoid.rs new file mode 100644 index 0000000..d6774b5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/sigmoid.rs @@ -0,0 +1,38 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the sigmoid function element-wise +/// See also [sigmoid](burn::tensor::activation::sigmoid) +#[derive(Module, Clone, Debug, Default)] +pub struct Sigmoid; + +impl Sigmoid { + /// Create the module. + pub fn new() -> Self { + Self {} + } + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::sigmoid(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let layer = Sigmoid::new(); + + assert_eq!(alloc::format!("{layer}"), "Sigmoid"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/soft_shrink.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/soft_shrink.rs new file mode 100644 index 0000000..2db41f5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/soft_shrink.rs @@ -0,0 +1,98 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::activation::soft_shrink; +use burn::tensor::backend::Backend; + +/// Soft Shrink layer. +/// +/// Applies the Soft Shrink function element-wise: +/// `soft_shrink(x) = x - lambda if x > lambda, x + lambda if x < -lambda, 0 otherwise` +/// +/// Should be created with [SoftShrinkConfig](SoftShrinkConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct SoftShrink { + /// The lambda value for the Soft Shrink formulation. + pub lambda: f64, +} + +/// Configuration to create a [SoftShrink](SoftShrink) layer using the [init function](SoftShrinkConfig::init). +#[derive(Config, Debug)] +pub struct SoftShrinkConfig { + /// The lambda value for the Soft Shrink formulation. Default is 0.5 + #[config(default = "0.5")] + pub lambda: f64, +} + +impl SoftShrinkConfig { + /// Initialize a new [SoftShrink](SoftShrink) Layer + pub fn init(&self) -> SoftShrink { + SoftShrink { + lambda: self.lambda, + } + } +} + +impl ModuleDisplay for SoftShrink { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("lambda", &self.lambda).optional() + } +} + +impl SoftShrink { + /// Forward pass for the Soft Shrink layer. + /// + /// See [soft_shrink](burn::tensor::activation::soft_shrink) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + soft_shrink(input, self.lambda) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn test_soft_shrink_forward() { + let device = ::Device::default(); + let model: SoftShrink = SoftShrinkConfig::new().init(); + let input = + Tensor::::from_data([[0.5, -0.5, -1.0], [8.0, 0.3, 0.0]], &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0_f32, 0.0, -0.5], [7.5, 0.0, 0.0]]); + assert_eq!(out.into_data(), expected); + } + + #[test] + fn test_soft_shrink_with_lambda() { + let device = ::Device::default(); + let model: SoftShrink = SoftShrinkConfig::new().with_lambda(0.25).init(); + let input = + Tensor::::from_data([[0.125, -0.125, -0.5], [0.75, 0.1, 0.0]], &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0_f32, 0.0, -0.25], [0.5, 0.0, 0.0]]); + assert_eq!(out.into_data(), expected); + } + + #[test] + fn display() { + let config = SoftShrinkConfig::new().init(); + assert_eq!(alloc::format!("{config}"), "SoftShrink {lambda: 0.5}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softplus.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softplus.rs new file mode 100644 index 0000000..ad89d20 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softplus.rs @@ -0,0 +1,105 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::activation::softplus; +use burn::tensor::backend::Backend; + +/// Softplus layer. +/// +/// Applies the softplus function element-wise: +/// `softplus(x) = (1/beta) * log(1 + exp(beta * x))` +/// +/// Should be created with [SoftplusConfig](SoftplusConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Softplus { + /// The beta value. + pub beta: f64, +} + +/// Configuration to create a [Softplus](Softplus) layer using the [init function](SoftplusConfig::init). +#[derive(Config, Debug)] +pub struct SoftplusConfig { + /// The beta value. Default is 1.0 + #[config(default = "1.0")] + pub beta: f64, +} + +impl SoftplusConfig { + /// Initialize a new [Softplus](Softplus) Layer + pub fn init(&self) -> Softplus { + Softplus { beta: self.beta } + } +} + +impl ModuleDisplay for Softplus { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("beta", &self.beta).optional() + } +} + +impl Softplus { + /// Forward pass for the Softplus layer. + /// + /// See [softplus](burn::tensor::activation::softplus) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + softplus(input, self.beta) + } +} + +#[cfg(test)] +#[allow(clippy::approx_constant)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_softplus_forward() { + let device = ::Device::default(); + let model: Softplus = SoftplusConfig::new().init(); + let input = + Tensor::::from_data(TensorData::from([[0.0, 1.0, -1.0]]), &device); + let out = model.forward(input); + // softplus(0) = log(2) ≈ 0.6931 + // softplus(1) = log(1 + e) ≈ 1.3133 + // softplus(-1) = log(1 + e^-1) ≈ 0.3133 + let expected = TensorData::from([[0.6931, 1.3133, 0.3133]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_softplus_with_beta() { + let device = ::Device::default(); + let model: Softplus = SoftplusConfig::new().with_beta(2.0).init(); + let input = Tensor::::from_data(TensorData::from([[0.0, 1.0]]), &device); + let out = model.forward(input); + // softplus(0, beta=2) = (1/2) * log(1 + exp(0)) = 0.5 * log(2) ≈ 0.3466 + // softplus(1, beta=2) = (1/2) * log(1 + exp(2)) = 0.5 * log(8.389) ≈ 1.0635 + let expected = TensorData::from([[0.3466, 1.0635]]); + out.to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = SoftplusConfig::new().init(); + assert_eq!(alloc::format!("{config}"), "Softplus {beta: 1}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softsign.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softsign.rs new file mode 100644 index 0000000..be55539 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/softsign.rs @@ -0,0 +1,38 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the softsign function element-wise +/// See also [softsign](burn::tensor::activation::softsign) +#[derive(Module, Clone, Debug, Default)] +pub struct Softsign; + +impl Softsign { + /// Create the module. + pub fn new() -> Self { + Self {} + } + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::softsign(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let layer = Softsign::new(); + + assert_eq!(alloc::format!("{layer}"), "Softsign"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/swiglu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/swiglu.rs new file mode 100644 index 0000000..8385494 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/swiglu.rs @@ -0,0 +1,153 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::activation::silu; +use burn::tensor::{Tensor, backend::Backend}; + +use crate::{Linear, LinearConfig, LinearLayout}; + +/// Configuration to create a [SwiGlu](SwiGlu) activation layer using the [init function](SwiGluConfig::init). +#[derive(Config, Debug)] +pub struct SwiGluConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the output features. + pub d_output: usize, + /// If a bias should be applied during the linear transformation. Default behaviour is False + /// for SwiGLU activation implementations. + #[config(default = false)] + pub bias: bool, + /// The type of function used to initialize the linear layer parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}" + )] + pub initializer: Initializer, + /// The layout in which the linear parameters are stored. + #[config(default = "LinearLayout::Row")] + pub layout: LinearLayout, +} + +/// Applies the SwiGLU or Swish Gated Linear Unit to the input tensor. +/// The SwiGLU activation function is defined as: +/// `SwiGLU(x) = Swish(W_inner * x + b_inner) * (W_outer * x + b_outer)` +/// +/// Should be created with [SwiGluConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct SwiGlu { + /// The inner linear layer for Swish activation function + /// with `d_input` input features and `d_output` output features. + pub linear_inner: Linear, + /// The outer linear layer for element wise multiplication + /// with `d_input` input features and `d_output` output features. + pub linear_outer: Linear, +} + +impl ModuleDisplay for SwiGlu { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, d_output] = self.linear_inner.weight.shape().dims(); + content + .add("d_input", &d_input) + .add("d_output", &d_output) + .add("bias", &self.linear_inner.bias.is_some()) + .optional() + } +} + +impl SwiGluConfig { + /// Initialize a new [SwiGLU](SwiGlu) activation layer. + pub fn init(&self, device: &B::Device) -> SwiGlu { + SwiGlu { + linear_inner: LinearConfig::new(self.d_input, self.d_output) + .with_bias(self.bias) + .with_initializer(self.initializer.clone()) + .with_layout(self.layout) + .init(device), + linear_outer: LinearConfig::new(self.d_input, self.d_output) + .with_bias(self.bias) + .with_initializer(self.initializer.clone()) + .with_layout(self.layout) + .init(device), + } + } +} + +impl SwiGlu { + /// Applies the Swish Gated Linear Unit to the input tensor. + /// + /// # Shapes + /// + /// - input: `[batch_size, seq_length, d_input]` + /// - output: `[batch_size, seq_length, d_output]` + pub fn forward(&self, input: Tensor) -> Tensor { + let x = self.linear_inner.forward(input.clone()); + let x = silu(x); + x.mul(self.linear_outer.forward(input)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_swiglu_forward_no_bias() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = SwiGluConfig::new(3, 3).with_initializer(Initializer::Constant { value: 0.5 }); + let swiglu = config.init(&device); + let input = + Tensor::::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let output = swiglu.forward(input); + let expected_output = Tensor::::from_data( + [[8.5732, 8.5732, 8.5732], [56.2189, 56.2189, 56.2189]], + &device, + ); + output + .to_data() + .assert_approx_eq::(&expected_output.to_data(), Tolerance::default()); + } + + #[test] + fn test_swiglu_forward_with_bias() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = SwiGluConfig::new(3, 3) + .with_bias(true) + .with_initializer(Initializer::Constant { value: 0.5 }); + let swiglu = config.init(&device); + let input = + Tensor::::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + let output = swiglu.forward(input); + let expected_output = Tensor::::from_data( + [[11.8909, 11.8909, 11.8909], [63.9785, 63.9785, 63.9785]], + &device, + ); + output + .to_data() + .assert_approx_eq::(&expected_output.to_data(), Tolerance::default()); + } + + #[test] + fn display() { + let config = SwiGluConfig::new(3, 5); + let swiglu = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{swiglu}"), + "SwiGlu {d_input: 3, d_output: 5, bias: false, params: 30}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/tanh.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/tanh.rs new file mode 100644 index 0000000..96df816 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/tanh.rs @@ -0,0 +1,38 @@ +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Applies the tanh activation function element-wise +/// See also [tanh](burn::tensor::activation::tanh) +#[derive(Module, Clone, Debug, Default)] +pub struct Tanh; + +impl Tanh { + /// Create the module. + pub fn new() -> Self { + Self {} + } + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + burn::tensor::activation::tanh(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let layer = Tanh::new(); + + assert_eq!(alloc::format!("{layer}"), "Tanh"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/thresholded_relu.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/thresholded_relu.rs new file mode 100644 index 0000000..3495c6a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/activation/thresholded_relu.rs @@ -0,0 +1,82 @@ +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_core as burn; + +use burn::tensor::activation::thresholded_relu; + +/// Thresholded ReLU layer. +/// +/// Should be created with [ThresholdedReluConfig](ThresholdedReluConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct ThresholdedRelu { + /// The alpha threshold. + pub alpha: f64, +} + +/// Configuration to create a [ThresholdedRelu](ThresholdedRelu) layer using the [init function](ThresholdedReluConfig::init). +#[derive(Config, Debug)] +pub struct ThresholdedReluConfig { + /// The alpha threshold. Default is 1.0 + #[config(default = "1.0")] + pub alpha: f64, +} + +impl ThresholdedReluConfig { + /// Initialize a new [ThresholdedRelu](ThresholdedRelu) layer. + pub fn init(&self) -> ThresholdedRelu { + ThresholdedRelu { alpha: self.alpha } + } +} + +impl ModuleDisplay for ThresholdedRelu { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("alpha", &self.alpha).optional() + } +} + +impl ThresholdedRelu { + /// Forward pass for the Thresholded ReLU layer. + /// + /// See [thresholded_relu](burn::tensor::activation::thresholded_relu) for more information. + /// + /// # Shapes + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + thresholded_relu(input, self.alpha) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn test_thresholded_relu_forward() { + let device = ::Device::default(); + let model: ThresholdedRelu = ThresholdedReluConfig::new().init(); + let input = + Tensor::::from_data(TensorData::from([[0.5, 1.5, -0.2]]), &device); + let out = model.forward(input); + let expected = TensorData::from([[0.0, 1.5, 0.0]]); + out.to_data().assert_eq(&expected, false); + } + + #[test] + fn display() { + let config = ThresholdedReluConfig::new().init(); + assert_eq!(alloc::format!("{config}"), "ThresholdedRelu {alpha: 1}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/lib.rs new file mode 100644 index 0000000..97163ef --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/lib.rs @@ -0,0 +1,63 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![recursion_limit = "256"] + +//! Burn neural network module. + +/// Loss module +pub mod loss; + +/// Neural network modules implementations. +pub mod modules; +pub use modules::*; + +pub mod activation; +pub use activation::{ + celu::*, elu::*, gelu::*, glu::*, hard_shrink::*, hard_sigmoid::*, leaky_relu::*, prelu::*, + relu::*, selu::*, shrink::*, sigmoid::*, soft_shrink::*, softplus::*, softsign::*, swiglu::*, + tanh::*, thresholded_relu::*, +}; + +mod padding; +pub use padding::*; + +// For backward compat, `burn::nn::Initializer` +pub use burn_core::module::Initializer; + +extern crate alloc; + +/// Backend for test cases +#[cfg(all( + test, + not(feature = "test-tch"), + not(feature = "test-wgpu"), + not(feature = "test-cuda"), + not(feature = "test-rocm") +))] +pub type TestBackend = burn_ndarray::NdArray; + +#[cfg(all(test, feature = "test-tch"))] +/// Backend for test cases +pub type TestBackend = burn_tch::LibTorch; + +#[cfg(all(test, feature = "test-wgpu"))] +/// Backend for test cases +pub type TestBackend = burn_wgpu::Wgpu; + +#[cfg(all(test, feature = "test-cuda"))] +/// Backend for test cases +pub type TestBackend = burn_cuda::Cuda; + +#[cfg(all(test, feature = "test-rocm"))] +/// Backend for test cases +pub type TestBackend = burn_rocm::Rocm; + +/// Backend for autodiff test cases +#[cfg(test)] +pub type TestAutodiffBackend = burn_autodiff::Autodiff; + +#[cfg(all(test, feature = "test-memory-checks"))] +mod tests { + burn_fusion::memory_checks!(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/binary_cross_entropy.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/binary_cross_entropy.rs new file mode 100644 index 0000000..b3ea03d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/binary_cross_entropy.rs @@ -0,0 +1,432 @@ +use burn_core as burn; + +use alloc::vec::Vec; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::activation::log_sigmoid; +use burn::tensor::{Int, Tensor, backend::Backend}; +use burn::{config::Config, module::Module}; + +/// Configuration to create a [Binary Cross-entropy loss](BinaryCrossEntropyLoss) using the [init function](BinaryCrossEntropyLossConfig::init). +#[derive(Config, Debug)] +pub struct BinaryCrossEntropyLossConfig { + /// Create weighted binary cross-entropy with a weight for each class. + /// + /// The loss of a specific sample will simply be multiplied by its label weight. + pub weights: Option>, + + /// Create binary cross-entropy with label smoothing according to [When Does Label Smoothing Help?](https://arxiv.org/abs/1906.02629). + /// + /// Hard labels {0, 1} will be changed to `y_smoothed = y(1 - a) + a / num_classes`. + /// Alpha = 0 would be the same as default. + pub smoothing: Option, + + /// Treat the inputs as logits, applying a sigmoid activation when computing the loss. + #[config(default = false)] + pub logits: bool, +} + +impl BinaryCrossEntropyLossConfig { + /// Initialize [Binary Cross-entropy loss](BinaryCrossEntropyLoss). + pub fn init(&self, device: &B::Device) -> BinaryCrossEntropyLoss { + self.assertions(); + BinaryCrossEntropyLoss { + weights: self + .weights + .as_ref() + .map(|e| Tensor::::from_floats(e.as_slice(), device)), + smoothing: self.smoothing, + logits: self.logits, + } + } + + fn assertions(&self) { + if let Some(alpha) = self.smoothing { + assert!( + (0.0..=1.).contains(&alpha), + "Alpha of Cross-entropy loss with smoothed labels should be in interval [0, 1]. Got {alpha}" + ); + }; + if let Some(weights) = self.weights.as_ref() { + assert!( + weights.iter().all(|e| e > &0.), + "Weights of cross-entropy have to be positive." + ); + } + } +} + +/// Calculate the binary cross entropy loss from the input logits and the targets. +/// +/// Should be created using [BinaryCrossEntropyLossConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct BinaryCrossEntropyLoss { + /// Weights for cross-entropy. + pub weights: Option>, + /// Label smoothing alpha. + pub smoothing: Option, + /// Treat the inputs as logits + pub logits: bool, +} + +impl ModuleDisplay for BinaryCrossEntropyLoss { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("weights", &self.weights) + .add("smoothing", &self.smoothing) + .add("logits", &self.logits) + .optional() + } +} + +impl BinaryCrossEntropyLoss { + /// Compute the criterion on the input tensor. + /// + /// # Shapes + /// + /// Binary: + /// - logits: `[batch_size]` + /// - targets: `[batch_size]` + /// + /// Multi-label: + /// - logits: `[batch_size, num_classes]` + /// - targets: `[batch_size, num_classes]` + pub fn forward( + &self, + logits: Tensor, + targets: Tensor, + ) -> Tensor { + self.assertions(&logits, &targets); + + let mut targets_float = targets.clone().float(); + let shape = targets.dims(); + + if let Some(alpha) = self.smoothing { + let num_classes = if D > 1 { shape[D - 1] } else { 2 }; + targets_float = targets_float * (1. - alpha) + alpha / num_classes as f32; + } + + let mut loss = if self.logits { + // Numerically stable by combining `log(sigmoid(x))` with `log_sigmoid(x)` + (targets_float.neg() + 1.) * logits.clone() - log_sigmoid(logits) + } else { + // - (target * log(input) + (1 - target) * log(1 - input)) + // https://github.com/tracel-ai/burn/issues/2739: clamp at -100.0 to avoid undefined values + (targets_float.clone() - 1) * logits.clone().neg().log1p().clamp_min(-100.0) + - targets_float * logits.log().clamp_min(-100.0) + }; + + if let Some(weights) = &self.weights { + let weights = if D > 1 { + weights.clone().expand(shape) + } else { + // Flatten targets and expand resulting weights to make it compatible with + // Tensor for binary 1-D case + weights + .clone() + .gather(0, targets.flatten(0, 0)) + .expand(shape) + }; + loss = loss * weights; + } + + loss.mean() + } + + fn assertions(&self, logits: &Tensor, targets: &Tensor) { + let logits_dims = logits.dims(); + let targets_dims = targets.dims(); + assert!( + logits_dims == targets_dims, + "Shape of targets ({targets_dims:?}) should correspond to outer shape of logits ({logits_dims:?})." + ); + + if let Some(weights) = &self.weights + && D > 1 + { + let targets_classes = targets_dims[D - 1]; + let weights_classes = weights.dims()[0]; + assert!( + weights_classes == targets_classes, + "The number of classes ({weights_classes}) does not match the weights provided ({targets_classes})." + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::{TensorData, activation::sigmoid}; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_binary_cross_entropy_preds_all_correct() { + let device = Default::default(); + let preds = Tensor::::from_floats([1.0, 0.0, 1.0, 0.0], &device); + let targets = + Tensor::::from_data(TensorData::from([1, 0, 1, 0]), &device); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .init(&device) + .forward(preds, targets) + .into_data(); + + let loss_expected = TensorData::from([0.000]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::default()); + } + + #[test] + fn test_binary_cross_entropy_preds_all_incorrect() { + let device = Default::default(); + let preds = Tensor::::from_floats([0.0, 1.0, 0.0, 1.0], &device); + let targets = + Tensor::::from_data(TensorData::from([1, 0, 1, 0]), &device); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .init(&device) + .forward(preds, targets) + .into_data(); + + let loss_expected = TensorData::from([100.000]); // clamped value + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::default()); + } + + #[test] + fn test_binary_cross_entropy() { + // import torch + // from torch import nn + // input = torch.tensor([0.8271, 0.9626, 0.3796, 0.2355]) + // target = torch.tensor([0., 1., 0., 1.]) + // loss = nn.BCELoss() + // sigmoid = nn.Sigmoid() + // out = loss(sigmoid(input), target) # tensor(0.7491) + + let device = Default::default(); + let logits = + Tensor::::from_floats([0.8271, 0.9626, 0.3796, 0.2355], &device); + let targets = + Tensor::::from_data(TensorData::from([0, 1, 0, 1]), &device); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .init(&device) + .forward(sigmoid(logits), targets) + .into_data(); + + let loss_expected = TensorData::from([0.7491]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::relative(1e-4)); + } + + #[test] + fn test_binary_cross_entropy_with_logits() { + let device = Default::default(); + let logits = + Tensor::::from_floats([0.8271, 0.9626, 0.3796, 0.2355], &device); + let targets = + Tensor::::from_data(TensorData::from([0, 1, 0, 1]), &device); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .with_logits(true) + .init(&device) + .forward(logits, targets) + .into_data(); + + let loss_expected = TensorData::from([0.7491]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::relative(1e-4)); + } + + #[test] + fn test_binary_cross_entropy_with_weights() { + // import torch + // from torch import nn + // input = torch.tensor([0.8271, 0.9626, 0.3796, 0.2355]) + // target = torch.tensor([0, 1, 0, 1]) + // weights = torch.tensor([3., 7.]).gather(0, target) + // loss = nn.BCELoss(weights) + // sigmoid = nn.Sigmoid() + // out = loss(sigmoid(input), target.float()) # tensor(3.1531) + + let device = Default::default(); + let logits = + Tensor::::from_floats([0.8271, 0.9626, 0.3796, 0.2355], &device); + let targets = + Tensor::::from_data(TensorData::from([0, 1, 0, 1]), &device); + let weights = [3., 7.]; + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .with_weights(Some(weights.to_vec())) + .init(&device) + .forward(sigmoid(logits), targets) + .into_data(); + + let loss_expected = TensorData::from([3.1531]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::relative(1e-4)); + } + + #[test] + fn test_binary_cross_entropy_with_smoothing() { + // import torch + // from torch import nn + // input = torch.tensor([0.8271, 0.9626, 0.3796, 0.2355]) + // target = torch.tensor([0., 1., 0., 1.]) + // target_smooth = target * (1 - 0.1) + (0.1 / 2) + // loss = nn.BCELoss() + // sigmoid = nn.Sigmoid() + // out = loss(sigmoid(input), target_smooth) # tensor(0.7490) + + let device = Default::default(); + let logits = + Tensor::::from_floats([0.8271, 0.9626, 0.3796, 0.2355], &device); + let targets = + Tensor::::from_data(TensorData::from([0, 1, 0, 1]), &device); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .with_smoothing(Some(0.1)) + .init(&device) + .forward(sigmoid(logits), targets) + .into_data(); + + let loss_expected = TensorData::from([0.7490]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::relative(1e-4)); + } + + #[test] + fn test_binary_cross_entropy_multilabel() { + // import torch + // from torch import nn + // input = torch.tensor([[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]]) + // target = torch.tensor([[1., 0., 1.], [1., 0., 0.]]) + // weights = torch.tensor([3., 7., 0.9]) + // loss = nn.BCEWithLogitsLoss() + // out = loss(input, target) # tensor(0.7112) + + let device = Default::default(); + let logits = Tensor::::from_floats( + [[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]], + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[1, 0, 1], [1, 0, 0]]), + &device, + ); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .with_logits(true) + .init(&device) + .forward(logits, targets) + .into_data(); + + let loss_expected = TensorData::from([0.7112]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::relative(1e-4)); + } + + #[test] + fn test_binary_cross_entropy_multilabel_with_weights() { + // import torch + // from torch import nn + // input = torch.tensor([[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]]) + // target = torch.tensor([[1., 0., 1.], [1., 0., 0.]]) + // loss = nn.BCEWithLogitsLoss() + // out = loss(input, target) # tensor(3.1708) + + let device = Default::default(); + let logits = Tensor::::from_floats( + [[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]], + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[1, 0, 1], [1, 0, 0]]), + &device, + ); + let weights = [3., 7., 0.9]; + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .with_logits(true) + .with_weights(Some(weights.to_vec())) + .init(&device) + .forward(logits, targets) + .into_data(); + + let loss_expected = TensorData::from([3.1708]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::default()); + } + + #[test] + fn test_binary_cross_entropy_multilabel_with_smoothing() { + // import torch + // from torch import nn + // input = torch.tensor([[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]]) + // target = torch.tensor([[1., 0., 1.], [1., 0., 0.]]) + // target_smooth = target * (1 - 0.1) + (0.1 / 3) + // loss = nn.BCELoss() + // sigmoid = nn.Sigmoid() + // out = loss(sigmoid(input), target_smooth) # tensor(0.7228) + + let device = Default::default(); + let logits = Tensor::::from_floats( + [[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]], + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[1, 0, 1], [1, 0, 0]]), + &device, + ); + + let loss_actual = BinaryCrossEntropyLossConfig::new() + .with_smoothing(Some(0.1)) + .init(&device) + .forward(sigmoid(logits), targets) + .into_data(); + + let loss_expected = TensorData::from([0.7228]); + loss_actual.assert_approx_eq::(&loss_expected, Tolerance::default()); + } + + #[test] + #[should_panic = "The number of classes"] + fn multilabel_weights_should_match_target() { + // import torch + // from torch import nn + // input = torch.tensor([[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]]) + // target = torch.tensor([[1., 0., 1.], [1., 0., 0.]]) + // loss = nn.BCEWithLogitsLoss() + // out = loss(input, target) # tensor(3.1708) + + let device = Default::default(); + let logits = Tensor::::from_floats( + [[0.5150, 0.3097, 0.7556], [0.4974, 0.9879, 0.1564]], + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[1, 0, 1], [1, 0, 0]]), + &device, + ); + let weights = [3., 7.]; + + let _loss = BinaryCrossEntropyLossConfig::new() + .with_logits(true) + .with_weights(Some(weights.to_vec())) + .init(&device) + .forward(logits, targets); + } + + #[test] + fn display() { + let config = + BinaryCrossEntropyLossConfig::new().with_weights(Some(alloc::vec![3., 7., 0.9])); + let loss = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{loss}"), + "BinaryCrossEntropyLoss {weights: Tensor {rank: 1, shape: [3]}, smoothing: None, logits: false}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cosine_embedding.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cosine_embedding.rs new file mode 100644 index 0000000..48c89bc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cosine_embedding.rs @@ -0,0 +1,317 @@ +use alloc::format; + +use burn::tensor::linalg::cosine_similarity; + +use burn_core as burn; + +use crate::loss::reduction::Reduction; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::module::{Ignored, Module}; +use burn::tensor::{Int, Tensor, activation::relu, backend::Backend}; + +/// Configuration for CosineEmbeddingLoss. +#[derive(Config, Debug)] +pub struct CosineEmbeddingLossConfig { + /// Margin for negative samples. + #[config(default = 0.0)] + pub margin: f32, + + /// Specifies the reduction to apply to the output. + #[config(default = "Reduction::Mean")] + pub reduction: Reduction, +} + +impl CosineEmbeddingLossConfig { + /// Initialize CosineEmbeddingLoss. + pub fn init(&self) -> CosineEmbeddingLoss { + CosineEmbeddingLoss { + margin: self.margin, + reduction: Ignored(self.reduction.clone()), + } + } +} + +/// Cosine embedding loss between two tensors. +/// +/// Measures cosine distance between tensors. +/// Used for learning embeddings or similarity. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct CosineEmbeddingLoss { + /// Margin value. Default: 0.0 + pub margin: f32, + + /// Reduction method + pub reduction: Ignored, +} + +impl Default for CosineEmbeddingLoss { + fn default() -> Self { + CosineEmbeddingLossConfig::new().init() + } +} + +impl ModuleDisplay for CosineEmbeddingLoss { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("margin", &self.margin) + .add("reduction", format!("{:?}", &self.reduction.0).as_str()) + .optional() + } +} + +impl CosineEmbeddingLoss { + /// Creates a new instance + pub fn new() -> Self { + CosineEmbeddingLossConfig::new().init() + } + + /// Compute loss with reduction. + /// + /// # Shapes + /// + /// - input1: ``[batch_size, embedding_dim]`` + /// - input2: ``[batch_size, embedding_dim]`` + /// - target: ``[batch_size]`` with values 1 or -1 + /// + /// # Returns + /// + /// Loss tensor of shape ``[1]`` + pub fn forward( + &self, + input1: Tensor, + input2: Tensor, + target: Tensor, + ) -> Tensor { + let tensor = self.forward_no_reduction(input1, input2, target); + match &self.reduction.0 { + Reduction::Mean | Reduction::Auto => tensor.mean(), + Reduction::Sum => tensor.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + + /// Compute loss without applying reduction. + /// + /// # Arguments + /// + /// * `input1` - First input tensor of shape ``[batch_size, embedding_dim]`` + /// * `input2` - Second input tensor of shape ``[batch_size, embedding_dim]`` + /// * `target` - Target tensor of shape ``[batch_size]`` with values 1 or -1 + /// + /// # Returns + /// + /// Tensor of per-element losses with shape ``[batch_size]`` + pub fn forward_no_reduction( + &self, + input1: Tensor, + input2: Tensor, + target: Tensor, + ) -> Tensor { + self.assertions(&input1, &input2, &target); + + // cos_sim shape: [batch_size, 1] + let cos_sim = cosine_similarity(input1, input2, 1, None); + // cos_sim shape: [batch_size] + let cos_sim: Tensor = cos_sim.squeeze_dim(1); + + let mut loss = cos_sim.zeros_like(); + + // Similar pairs (target == 1) - Formula: L = 1 - cos_sim + let similar_mask = target.clone().equal_elem(1); + let similar_loss = cos_sim.clone().neg().add_scalar(1); + loss = loss.mask_where(similar_mask, similar_loss); + + // Dissimilar pairs (target == -1) - Formula: L = max(0, cos_sim - margin) + let dissimilar_mask = target.equal_elem(-1); + let dissimilar_loss = relu(cos_sim.clone().sub_scalar(self.margin)); + loss = loss.mask_where(dissimilar_mask, dissimilar_loss); + + // return loss shape: [batch_size] + loss + } + + fn assertions( + &self, + input1: &Tensor, + input2: &Tensor, + target: &Tensor, + ) { + let [batch_size1, dim1] = input1.dims(); + let [batch_size2, dim2] = input2.dims(); + let [batch_size_target] = target.dims(); + + assert_eq!( + batch_size1, batch_size2, + "Batch size of input1 ({batch_size1}) must match batch size of input2 ({batch_size2})" + ); + + assert_eq!( + dim1, dim2, + "Embedding dimension of input1 ({dim1}) must match embedding dimension of input2 ({dim2})" + ); + + assert_eq!( + batch_size1, batch_size_target, + "Batch size of inputs ({batch_size1}) must match batch size of target ({batch_size_target})" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn cosine_embedding_loss_positive_target() { + let device = Default::default(); + + // Two identical vectors should have cosine similarity of 1 + let input1 = Tensor::::from_data( + TensorData::from([[1.0, 0.0], [0.0, 1.0]]), + &device, + ); + + let input2 = Tensor::::from_data( + TensorData::from([[1.0, 0.0], [0.0, 1.0]]), + &device, + ); + + // Target 1 means that inputs should be similar + let target = Tensor::::from_data(TensorData::from([1, 1]), &device); + + let loss = CosineEmbeddingLossConfig::new().init(); + let loss_no_reduction = + loss.forward_no_reduction(input1.clone(), input2.clone(), target.clone()); + let loss_mean = loss.forward(input1.clone(), input2.clone(), target.clone()); + + let loss_sum = loss.forward(input1, input2, target); + + // For identical vectors, 1 - cos_sim = 1 - 1 = 0 + let expected_no_reduction = TensorData::from([0.0, 0.0]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected_no_reduction, Tolerance::default()); + + let expected_mean = TensorData::from([0.0]); + loss_mean + .into_data() + .assert_approx_eq::(&expected_mean, Tolerance::default()); + + let expected_sum = TensorData::from([0.0]); + loss_sum + .into_data() + .assert_approx_eq::(&expected_sum, Tolerance::default()); + } + + #[test] + fn cosine_embedding_loss_negative_target() { + let device = Default::default(); + + // Two identical vectors should have cosine similarity of 1 + let input1 = Tensor::::from_data( + TensorData::from([[1.0, 0.0], [0.0, 1.0]]), + &device, + ); + + let input2 = Tensor::::from_data( + TensorData::from([[1.0, 0.0], [0.0, 1.0]]), + &device, + ); + + // Target -1 means that inputs should be dissimilar + let target = Tensor::::from_data(TensorData::from([-1, -1]), &device); + + // With margin 0.0, max(0, cos_sim - margin) = max(0, 1 - 0) = 1 + let loss = CosineEmbeddingLossConfig::new().init(); + let loss_no_reduction = + loss.forward_no_reduction(input1.clone(), input2.clone(), target.clone()); + let loss_mean = loss.forward(input1.clone(), input2.clone(), target.clone()); + + // Create a loss with Sum reduction for testing + let loss_sum_config = CosineEmbeddingLossConfig::new().with_reduction(Reduction::Sum); + let loss_sum = + loss_sum_config + .init() + .forward(input1.clone(), input2.clone(), target.clone()); + + let expected_no_reduction = TensorData::from([1.0, 1.0]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected_no_reduction, Tolerance::default()); + + let expected_mean = TensorData::from([1.0]); + loss_mean + .into_data() + .assert_approx_eq::(&expected_mean, Tolerance::default()); + + let expected_sum = TensorData::from([2.0]); + loss_sum + .into_data() + .assert_approx_eq::(&expected_sum, Tolerance::default()); + + // With margin 0.5, max(0, cos_sim - margin) = max(0, 1 - 0.5) = 0.5 + let loss_with_margin = CosineEmbeddingLossConfig::new().with_margin(0.5).init(); + let loss_with_margin = loss_with_margin.forward(input1, input2, target); + + let expected = TensorData::from([0.5]); + loss_with_margin + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn cosine_embedding_loss_mixed_targets() { + let device = Default::default(); + + let input1 = Tensor::::from_data( + TensorData::from([[1.0, 0.0], [0.0, 1.0]]), + &device, + ); + + let input2 = Tensor::::from_data( + TensorData::from([[1.0, 0.0], [0.0, 1.0]]), + &device, + ); + + // Mixed targets + let target = Tensor::::from_data(TensorData::from([1, -1]), &device); + + let loss = CosineEmbeddingLossConfig::new().init(); + let loss_no_reduction = + loss.forward_no_reduction(input1.clone(), input2.clone(), target.clone()); + let loss_mean = loss.forward(input1, input2, target); + + let expected_no_reduction = TensorData::from([0.0, 1.0]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected_no_reduction, Tolerance::default()); + + let expected_mean = TensorData::from([0.5]); + loss_mean + .into_data() + .assert_approx_eq::(&expected_mean, Tolerance::default()); + } + + #[test] + fn display() { + let config = CosineEmbeddingLossConfig::new().with_margin(0.5); + let loss = config.init(); + + assert_eq!( + alloc::format!("{loss}"), + "CosineEmbeddingLoss {margin: 0.5, reduction: Mean}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cross_entropy.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cross_entropy.rs new file mode 100644 index 0000000..27e9fc4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/cross_entropy.rs @@ -0,0 +1,466 @@ +use burn_core as burn; +use burn_core::tensor::IndexingUpdateOp; + +use alloc::string::ToString; +use alloc::vec; +use alloc::vec::Vec; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::activation::log_softmax; +use burn::tensor::{Bool, Int, Tensor, backend::Backend}; +use burn::{config::Config, module::Module}; + +/// Configuration to create a [Cross-entropy loss](CrossEntropyLoss) using the [init function](CrossEntropyLossConfig::init). +#[derive(Config, Debug)] +pub struct CrossEntropyLossConfig { + /// Create padded cross entropy. + /// + /// Prevents pad tokens from impacting loss calculation. + pub pad_tokens: Option>, + + /// Create weighted cross-entropy. + /// + /// The loss of a specific sample will simply be given by: weight * log(p(x)) * 1, + /// + /// # Pre-conditions + /// - The order of the weight vector should correspond to the label integer assignment. + /// - Targets assigned negative Int's will not be allowed. + pub weights: Option>, + + /// Create cross-entropy with label smoothing. + /// + /// Hard labels {0, 1} will be changed to y_smoothed = y(1 - a) + a / nr_classes. + /// Alpha = 0 would be the same as default. + pub smoothing: Option, + + /// Create cross-entropy with probabilities as input instead of logits. + /// + #[config(default = true)] + pub logits: bool, +} + +impl CrossEntropyLossConfig { + /// Initialize [Cross-entropy loss](CrossEntropyLoss). + pub fn init(&self, device: &B::Device) -> CrossEntropyLoss { + self.assertions(); + CrossEntropyLoss { + pad_tokens: self.pad_tokens.clone(), + weights: self + .weights + .as_ref() + .map(|e| Tensor::::from_floats(e.as_slice(), device)), + smoothing: self.smoothing, + logits: self.logits, + } + } + + fn assertions(&self) { + if let Some(alpha) = self.smoothing { + assert!( + (0.0..=1.).contains(&alpha), + "Alpha of Cross-entropy loss with smoothed labels should be in interval [0, 1]. Got {alpha}" + ); + }; + if let Some(weights) = self.weights.as_ref() { + assert!( + weights.iter().all(|e| e > &0.), + "Weights of cross-entropy have to be positive." + ); + } + } +} + +/// Calculate the cross entropy loss from the input logits and the targets. +/// +/// Should be created using [CrossEntropyLossConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct CrossEntropyLoss { + /// Pad tokens to ignore in the loss calculation. + pub pad_tokens: Option>, + /// Weights for cross-entropy. + pub weights: Option>, + /// Label smoothing factor. + pub smoothing: Option, + /// Use logits as input. + pub logits: bool, +} + +impl ModuleDisplay for CrossEntropyLoss { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let pad_tokens = if let Some(pad_tokens) = &self.pad_tokens { + alloc::format!("Vec<0..{}>", pad_tokens.len()) + } else { + "None".to_string() + }; + + content + .add("pad_tokens", &pad_tokens) + .add("weights", &self.weights) + .add("smoothing", &self.smoothing) + .add("logits", &self.logits) + .optional() + } +} + +impl CrossEntropyLoss { + /// For backward compatibility. + pub fn new(pad_index: Option, device: &B::Device) -> Self { + CrossEntropyLossConfig::new() + .with_pad_tokens(pad_index.map(|e| vec![e])) + .init(device) + } + + /// Compute the criterion on the input tensor. + /// + /// # Shapes + /// + /// - logits: `[batch_size, num_targets]` + /// - targets: `[batch_size]` + pub fn forward(&self, logits: Tensor, targets: Tensor) -> Tensor { + Self::assertions(logits.clone(), targets.clone()); + match self.smoothing { + Some(alpha) => self.forward_smoothed(logits, targets, alpha), + _ => self.forward_default(logits, targets), + } + } + + fn forward_smoothed( + &self, + logits: Tensor, + targets: Tensor, + alpha: f32, + ) -> Tensor { + let mask = self.padding_mask(&targets); + let tensor = if self.logits { + log_softmax(logits, 1) + } else { + logits.log() + }; + let [batch_size, nr_classes] = tensor.dims(); + let tensor = tensor + * Self::compute_smoothed_targets([batch_size, nr_classes], targets.clone(), alpha); + + match &self.weights { + Some(weights) => { + let tensor = tensor + * weights + .clone() + .reshape([1, nr_classes]) + .repeat_dim(0, batch_size); + let weights = weights.clone().gather(0, targets); + let tensor = Self::apply_mask_2d(tensor, mask); + tensor.sum().neg() / weights.sum() + } + None => { + let tensor = Self::apply_mask_2d(tensor, mask); + tensor.sum_dim(1).mean().neg() + } + } + } + + fn forward_default(&self, logits: Tensor, targets: Tensor) -> Tensor { + let [batch_size] = targets.dims(); + + let mask = self.padding_mask(&targets); + let tensor = log_softmax(logits, 1); + let tensor = tensor.gather(1, targets.clone().reshape([batch_size, 1])); + + match &self.weights { + Some(weights) => { + let weights = weights.clone().gather(0, targets); + let tensor = tensor.reshape([batch_size]) * weights.clone(); + let tensor = Self::apply_mask_1d(tensor, mask); + tensor.sum().neg() / weights.sum() + } + None => { + let tensor = Self::apply_mask_1d(tensor.reshape([batch_size]), mask); + tensor.mean().neg() + } + } + } + + fn compute_smoothed_targets( + shape: [usize; 2], + targets: Tensor, + alpha: f32, + ) -> Tensor { + let [batch_size, nr_classes] = shape; + let device = &targets.device(); + let targets_matrix = Tensor::::zeros(shape, device).scatter( + 1, + targets.reshape([batch_size, 1]), + Tensor::ones([batch_size, 1], device), + IndexingUpdateOp::Add, + ); + targets_matrix * (1. - alpha) + alpha / nr_classes as f32 + } + + fn padding_mask(&self, targets: &Tensor) -> Option> { + let mut mask = None; + if let Some(pad_tokens) = &self.pad_tokens { + let mut res = targets.clone().equal_elem(pad_tokens[0] as i64).int(); + for x in pad_tokens { + res = res + targets.clone().equal_elem(*x as i64).int(); + } + mask = Some(res.greater_elem(0)); + } + + mask + } + + fn apply_mask_1d(mut tensor: Tensor, mask: Option>) -> Tensor { + if let Some(mask) = mask { + tensor = tensor.mask_fill(mask, 0); + } + + tensor + } + + fn apply_mask_2d(mut tensor: Tensor, mask: Option>) -> Tensor { + if let Some(mask) = mask { + let [batch_size, nr_classes] = tensor.dims(); + tensor = tensor.mask_fill(mask.reshape([batch_size, 1]).repeat_dim(1, nr_classes), 0); + } + + tensor + } + + fn assertions(logits: Tensor, targets: Tensor) { + let [logits_height, _] = logits.dims(); + let [targets_height] = targets.dims(); + assert!( + logits_height == targets_height, + "Shape of targets ({targets_height}) should correspond to outer shape of logits ({logits_height})." + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::{Distribution, TensorData, loss::cross_entropy_with_logits, ops::IntElem}; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + macro_rules! setup { + () => {{ + let [batch_size, num_targets] = [4, 5]; + let device = Default::default(); + let logits = Tensor::::random( + [batch_size, num_targets], + Distribution::Normal(0., 1.0), + &device, + ); + let targets = + Tensor::::from_data(TensorData::from([2, 0, 4, 1]), &device); + let targets_logits = Tensor::::from_data( + TensorData::from([ + [0.0, 0.0, 1.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + ]), + &device, + ); + (logits, targets, targets_logits) + }}; + } + + macro_rules! setup_padded { + () => {{ + let [batch_size, num_targets, pad_index] = [4, 5, 1]; + let device = Default::default(); + let logits = Tensor::::random( + [batch_size, num_targets], + Distribution::Normal(0., 1.0), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([2, 0, 4, pad_index as i64]).convert::>(), + &device, + ); + let targets_logits = Tensor::::from_data( + TensorData::from([ + [0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ]), + &device, + ); + (logits, targets, targets_logits) + }}; + } + + #[test] + fn test_cross_entropy_loss_with_weights() { + let (logits, targets, targets_logits) = setup!(); + let weights = vec![1.0, 2., 3., 4., 5.]; + let device = Default::default(); + let loss_1 = CrossEntropyLossConfig::new() + .with_weights(Some(weights.clone())) + .init(&device) + .forward(logits.clone(), targets); + let tensor = log_softmax(logits, 1); + let loss_2 = tensor + * targets_logits + * Tensor::::from_floats(weights.as_slice(), &device) + .unsqueeze() + .repeat_dim(0, 4); + let loss_2 = loss_2.sum().neg() / (1. + 2. + 3. + 5.); + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn test_label_smoothing_with_weights_and_alpha_zero() { + let (logits, targets, _) = setup!(); + let device = Default::default(); + let weights = vec![1.0, 2., 3., 4., 5.]; + let loss_1 = CrossEntropyLossConfig::new() + .with_weights(Some(weights.clone())) + .init(&device) + .forward(logits.clone(), targets.clone()); + let loss_2 = CrossEntropyLossConfig::new() + .with_weights(Some(weights.clone())) + .with_smoothing(Some(0.)) + .init(&device) + .forward(logits.clone(), targets); + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn test_cross_entropy_loss() { + let (logits, targets, targets_logits) = setup!(); + let device = Default::default(); + let loss_1 = CrossEntropyLossConfig::new() + .init(&device) + .forward(logits.clone(), targets); + let loss_2 = cross_entropy_with_logits(logits, targets_logits); + + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn test_label_smoothing_alpha_equal_zero() { + let (logits, targets, _) = setup!(); + let device = Default::default(); + let loss_1 = CrossEntropyLossConfig::new() + .init(&device) + .forward(logits.clone(), targets.clone()); + let loss_2 = CrossEntropyLossConfig::new() + .with_smoothing(Some(0.)) + .init(&device) + .forward(logits, targets); + + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn test_cross_entropy_loss_with_pad_token() { + let (logits, targets, targets_logits) = setup_padded!(); + let pad_index = 1; + + let loss_1 = CrossEntropyLossConfig::new() + .with_pad_tokens(Some(vec![pad_index, 2])) + .init(&logits.device()) + .forward(logits.clone(), targets); + let loss_2 = cross_entropy_with_logits(logits, targets_logits); + + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn test_label_smoothing_with_zero_alpha_and_pad_token() { + let (logits, targets, _) = setup_padded!(); + let pad_index = 1; + + let loss_1 = CrossEntropyLossConfig::new() + .with_pad_tokens(Some(vec![pad_index, 2])) + .init(&logits.device()) + .forward(logits.clone(), targets.clone()); + let loss_2 = CrossEntropyLossConfig::new() + .with_pad_tokens(Some(vec![pad_index, 2])) + .with_smoothing(Some(0.)) + .init(&logits.device()) + .forward(logits.clone(), targets); + + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn test_label_smoothing_target_conversion() { + let (logits, targets, _) = setup!(); + let smoothed_targets = + CrossEntropyLoss::compute_smoothed_targets(logits.dims(), targets, 0.05); + let targets_logits = Tensor::::from_data( + TensorData::from([ + [0.01, 0.01, 0.96, 0.01, 0.01], + [0.96, 0.01, 0.01, 0.01, 0.01], + [0.01, 0.01, 0.01, 0.01, 0.96], + [0.01, 0.96, 0.01, 0.01, 0.01], + ]), + &Default::default(), + ); + smoothed_targets + .into_data() + .assert_approx_eq::(&targets_logits.into_data(), Tolerance::default()); + } + + #[test] + fn test_label_smoothing() { + let (logits, targets, _) = setup!(); + let device = Default::default(); + let loss_1 = CrossEntropyLossConfig::new() + .with_smoothing(Some(0.05)) + .init(&device) + .forward(logits.clone(), targets); + let targets_logits = Tensor::::from_data( + TensorData::from([ + [0.01, 0.01, 0.96, 0.01, 0.01], + [0.96, 0.01, 0.01, 0.01, 0.01], + [0.01, 0.01, 0.01, 0.01, 0.96], + [0.01, 0.96, 0.01, 0.01, 0.01], + ]), + &device, + ); + + let x = log_softmax(logits, 1); + let loss_2 = (x * targets_logits).sum_dim(1).mean().neg(); + + loss_1 + .into_data() + .assert_approx_eq::(&loss_2.into_data(), Tolerance::default()); + } + + #[test] + fn display() { + let config = CrossEntropyLossConfig::new() + .with_weights(Some(alloc::vec![3., 7., 0.9])) + .with_smoothing(Some(0.5)); + let loss = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{loss}"), + "CrossEntropyLoss {pad_tokens: None, weights: Tensor {rank: 1, shape: [3]}, smoothing: 0.5, logits: true}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/ctc.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/ctc.rs new file mode 100644 index 0000000..47a554d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/ctc.rs @@ -0,0 +1,1730 @@ +#![allow(clippy::excessive_precision)] + +use super::Reduction; +use alloc::vec; +use burn::config::Config; +use burn::module::Module; +use burn::tensor::{Bool, Element, Int, Tensor, backend::Backend, s}; +use burn_core as burn; +use burn_core::tensor::Numeric; +use core::f32; + +/// Configuration for the [CTC Loss](CTCLoss) module. +#[derive(Config, Debug)] +pub struct CTCLossConfig { + /// The index number used to represent the blank label. Default value is `0`. + #[config(default = 0)] + pub blank: usize, + /// Whether to zero infinite losses and the associated gradients. Default value is `false`. + #[config(default = false)] + pub zero_infinity: bool, +} + +impl CTCLossConfig { + /// Initialize a new [CTC Loss](CTCLoss) module + pub fn init(&self) -> CTCLoss { + CTCLoss { + blank: self.blank, + zero_infinity: self.zero_infinity, + } + } +} + +/// Computes the Connectionist Temporal Classification (CTC) loss. +/// +/// Calculates the loss between a continuous (unsegmented) time series and a target sequence. +/// CTC sums over the probability of all possible alignments of the input to the target, +/// producing a loss value that is differentiable with respect to each input node. +/// +/// The input to this loss is expected to be **log-probabilities** (e.g,, via `log_softmax`), +/// not raw logits. +/// +/// # References +/// +/// - [Connectionist Temporal Classification: Labelling Unsegmented Sequence Data with Recurrent Neural Networks](https://www.cs.toronto.edu/~graves/icml_2006.pdf) +/// +/// # Example +/// +/// ```rust,ignore +/// use burn::tensor::{Tensor, Int}; +/// use burn::tensor::activation::log_softmax; +/// use burn::nn::loss::{CTCLossConfig, CTCLoss}; +/// +/// let device = Default::default(); +/// +/// // Initialize CTC Loss with default configuration +/// let ctc_loss = CTCLossConfig::new().init(); +/// +/// // Initialize CTC Loss with custom configuration +/// let ctc_loss = CTCLossConfig::new() +/// .with_blank(1) +/// .with_zero_infinity(true) +/// .init(); +/// +/// // Prepare inputs (Logits shape: [Time, Batch, Class]) +/// // In your actual code, the logits would be the output of your model +/// let logits = Tensor::::ones([10, 2, 5], &device); +/// let log_probs = log_softmax(logits, 2); +/// +/// // Targets shape: [Batch, Max_Target_Len] +/// // Note: Targets should not contain the blank index (1). +/// let targets = Tensor::::from_data([[0, 2], [3, 4]], &device); +/// +/// // Lengths shape: [Batch] +/// let input_lengths = Tensor::::from_data([10, 8], &device); +/// let target_lengths = Tensor::::from_data([2, 2], &device); +/// +/// // Compute loss +/// let loss = ctc_loss.forward(log_probs, targets, input_lengths, target_lengths); +/// ``` +#[derive(Module, Clone, Debug)] +pub struct CTCLoss { + blank: usize, + zero_infinity: bool, +} + +impl CTCLoss { + /// Computes the CTC loss for the input log-probabilities and targets with no reduction applied. + /// + /// # Arguments + /// + /// - `log_probs`: The log-probabilities of the outputs (e.g., from `log_softmax`). + /// - `targets`: A 2D tensor containing the target class indices. These indices should not + /// include the blank index used in CTC loss. The targets are padded to the length of the longest sequence. + /// - `input_lengths`: A 1D tensor containing the actual length of the input sequence for each batch. This + /// allows retrieving the actual sequence of log-probabilities from `log_probs` if the batch contains + /// sequences of varying lengths. + /// - `target_lengths`: A 1D tensor containing the actual length of the target sequence for each target + /// sequence in `targets`. + /// + /// # Returns + /// + /// - A 1D tensor of shape `[batch_size]` containing the loss for each sample. + /// + /// # Shapes + /// + /// - `log_probs`: `[time_steps, batch_size, num_classes]` where `num_classes` includes blank. + /// - `targets`: `[batch_size, max_target_length]` + /// - `input_lengths`: `[batch_size]` + /// - `target_lengths`: `[batch_size]` + pub fn forward( + &self, + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + ) -> Tensor { + let device = log_probs.device(); + let [max_input_length, batch_size, num_classes] = log_probs.dims(); // [T, N, C] + let max_target_len = targets.dims()[1]; + let input_lengths_len = input_lengths.dims()[0]; + let target_lengths_len = target_lengths.dims()[0]; + self.assertions( + batch_size, + num_classes, + targets.clone(), + input_lengths_len, + target_lengths_len, + ); + + // Build the modified label sequence l' by inserting blanks around every label + let blank_inserted_targets = + self.insert_blanks::(&targets, batch_size, max_target_len, &device); + + // Initialize the forward variable alpha + let max_l_prime_len = 2 * max_target_len + 1; + let mut log_alpha_t_s = + Tensor::::full([batch_size, max_l_prime_len], f32::NEG_INFINITY, &device); + log_alpha_t_s = self.initialize_log_alpha( + log_probs.clone(), + blank_inserted_targets.clone(), + log_alpha_t_s, + ); + + let l_prime_combined_mask = self.create_l_prime_mask( + blank_inserted_targets.clone(), + batch_size, + max_l_prime_len, + &device, + ); + let s_mask = + self.create_s_mask(max_l_prime_len, batch_size, target_lengths.clone(), &device); + + // Loop over time steps since an arbitrary time step t depends on t - 1 + for t in 1..max_input_length { + let combined_s_t_mask = self.create_combined_s_t_mask( + input_lengths.clone(), + t, + batch_size, + max_l_prime_len, + s_mask.clone(), + ); + log_alpha_t_s = self.compute_log_alpha_t_s( + t, + combined_s_t_mask, + log_alpha_t_s, + l_prime_combined_mask.clone(), + log_probs.clone(), + blank_inserted_targets.clone(), + ); + } + + let last_blank_indices = target_lengths.mul_scalar(2).reshape([batch_size, 1]); + let last_label_indices = last_blank_indices.clone().sub_scalar(1); + let log_alpha_last_blank = log_alpha_t_s + .clone() + .gather(1, last_blank_indices) + .squeeze_dim::<1>(1); + let log_alpha_last_label = log_alpha_t_s + .clone() + .gather(1, last_label_indices) + .squeeze_dim::<1>(1); + let log_likelihood = self.log_sum_exp(log_alpha_last_blank, log_alpha_last_label, &device); + let mut ctc_loss_tensor = log_likelihood.neg(); + + if self.zero_infinity { + let inf_mask = ctc_loss_tensor.clone().is_inf(); + ctc_loss_tensor = ctc_loss_tensor + .clone() + .mask_where(inf_mask, ctc_loss_tensor.clone().zeros_like()); + } + + ctc_loss_tensor + } + + /// Computes the CTC loss for the input log-probabilities and targets with reduction. + /// + /// # Arguments + /// + /// - `log_probs`: The log-probabilities of the outputs (e.g., from `log_softmax`). + /// - `targets`: A 2D tensor containing the target class indices. These indices should not + /// include the blank index used in CTC loss. The targets are padded to the length of the longest sequence. + /// - `input_lengths`: A 1D tensor containing the actual length of the input sequence for each batch. This + /// allows retrieving the actual sequence of log-probabilities from `log_probs` if the batch contains + /// sequences of varying lengths. + /// - `target_lengths`: A 1D tensor containing the actual length of the target sequence for each target + /// sequence in `targets`. + /// - `reduction`: The reduction stratey to apply to the loss tensor containing the CTC loss values for + /// each sample (e.g., mean, sum). For the mean reduction strategy, the output losses will be divided + /// by the target lengths and then the mean over the batch is taken. This follows PyTorch's behavior. + /// + /// # Returns + /// + /// - A 1D tensor of shape `[1]` containing the reduced loss value. + /// + /// # Shapes + /// + /// - `log_probs`: `[time_steps, batch_size, num_classes]` where `num_classes` includes blank. + /// - `targets`: `[batch_size, max_target_length]` + /// - `input_lengths`: `[batch_size]` + /// - `target_lengths`: `[batch_size]` + /// + /// # Panics + /// - If `reduction` is not one of `Reduction::Auto`, `Reduction::Mean`, and `Reduction::Sum`. + /// - If `blank` index is greater than or equal to `num_classes`. + /// - If the batch dimension of `log_probs`, `targets`, `input_lengths`, and `target_lengths` do not match. + pub fn forward_with_reduction( + &self, + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + reduction: Reduction, + ) -> Tensor { + let ctc_loss_tensor = + self.forward(log_probs, targets, input_lengths, target_lengths.clone()); + + match reduction { + Reduction::Auto | Reduction::Mean => { + // Following PyTorch's behavior where the output losses are divided + // by the target lengths and then the mean over the batch is taken + let target_lengths_float = target_lengths.float(); + ctc_loss_tensor.div(target_lengths_float).mean() + } + Reduction::Sum => ctc_loss_tensor.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + + fn assertions( + &self, + batch_size: usize, + num_classes: usize, + targets: Tensor, + input_lengths_len: usize, + target_lengths_len: usize, + ) { + assert!( + self.blank < num_classes, + "blank index {} must be less than num_classes {}", + self.blank, + num_classes + ); + assert_eq!( + targets.dims()[0], + batch_size, + "targets batch dimension {} must equal batch_size {}", + targets.dims()[0], + batch_size + ); + assert_eq!( + input_lengths_len, batch_size, + "input_lengths length {} must equal batch_size {}", + input_lengths_len, batch_size + ); + assert_eq!( + target_lengths_len, batch_size, + "target_lengths length {} must equal batch_size {}", + target_lengths_len, batch_size + ); + } + + fn insert_blanks( + &self, + targets: &Tensor, + batch_size: usize, + max_target_len: usize, + device: &B::Device, + ) -> Tensor { + // The modified label sequences have (max_target_len + 1) blank labels + let blank_tensor = Tensor::::full( + [batch_size, 2 * max_target_len + 1], + self.blank as i64, + device, + ); + + blank_tensor.slice_assign(s![.., 1..;2], targets.clone()) + } + + fn initialize_log_alpha( + &self, + log_probs: Tensor, + blank_inserted_targets: Tensor, + log_alpha_t_s: Tensor, + ) -> Tensor { + // Given alpha_t(s), we have: + // alpha_1(1) = (y_blank)^1 => log_alpha_1(1) = ln(y_blank)^1 + // alpha_1(2) = (y_l1)^1 => log_alpha_1(2) = ln(y_l1)^1 + // alpha_1(s) = 0 (for every s > 2) => log_alpha_1(s) = -neg_inf + let log_probs_t0 = log_probs + .clone() + .slice(s![0..1, .., ..]) + .squeeze_dim::<2>(0); // shape: [N, C] + + // log_alpha shape: [N, 2*S+1] + // log_probs shape: [T, N, C] + // log_alpha[:, 0] = log_probs[0, :, blank] + let first_blank = blank_inserted_targets.clone().slice(s![.., 0..1]); // [N, 1] + // log_probs_t0 have C columns where each represents a unique class (includes blank) + let log_prob_blank = log_probs_t0.clone().gather(1, first_blank); // [N, 1] + let temp_log_alpha_t_s = log_alpha_t_s.slice_assign(s![.., 0..1], log_prob_blank); + + // log_alpha[:, 1] = log_probs[0, :, targets[:, 0]] + let first_label = blank_inserted_targets.clone().slice(s![.., 1..2]); // [N, 1] + let log_prob_first_label = log_probs_t0.gather(1, first_label); // [N, 1] + temp_log_alpha_t_s.slice_assign(s![.., 1..2], log_prob_first_label) + } + + fn right_shift_2d_tensor( + &self, + org_2d_tensor: Tensor, + shift_by: usize, + device: &B::Device, + ) -> Tensor + where + K: Numeric, + K::Elem: Element, + { + assert!( + shift_by == 1 || shift_by == 2, + "The parameter shift_by must 1 or 2" + ); + + let [rows, cols] = org_2d_tensor.dims(); + let padding_shape = [rows, shift_by]; + let padding_tensor = if org_2d_tensor.dtype().is_float() { + Tensor::::full(padding_shape, f32::NEG_INFINITY, device) + } else { + Tensor::::full(padding_shape, 0, device) + }; + let org_tensor_shortened = org_2d_tensor.slice(s![.., ..cols - shift_by]); + + Tensor::cat(vec![padding_tensor, org_tensor_shortened], 1) + } + + fn create_l_prime_mask( + &self, + blank_inserted_targets: Tensor, + batch_size: usize, + max_l_prime_len: usize, + device: &B::Device, + ) -> Tensor { + let l_prime_s = blank_inserted_targets.clone(); + let l_prime_s_minus_2 = self.right_shift_2d_tensor(blank_inserted_targets, 2, device); + + // Create a single mask that is true for entries where alpha_{t-1}(s - 2) should also + // be added to compute alpha_{t}(s) + let s_is_not_blank_mask = l_prime_s.clone().not_equal_elem(self.blank as i64); + let s_not_equal_s_minus_2_mask = l_prime_s.not_equal(l_prime_s_minus_2); + + // The 2 leftmost columns of the returned mask should only contain false. + // These are invalid positions since s - 2 is a valid index only when s >= 2. + let col_indices = Tensor::::arange(0..(max_l_prime_len as i64), device) + .reshape([1, max_l_prime_len]) + .expand([batch_size, max_l_prime_len]); + let s_greater_than_1_mask = col_indices.greater_equal_elem(2); + + s_is_not_blank_mask + .bool_and(s_not_equal_s_minus_2_mask) + .bool_and(s_greater_than_1_mask) + } + + fn create_s_mask( + &self, + max_l_prime_len: usize, + batch_size: usize, + target_lengths: Tensor, + device: &B::Device, + ) -> Tensor { + let col_indices = Tensor::::arange(0..max_l_prime_len as i64, device) + .reshape([1, max_l_prime_len]); + let col_indices_expanded = col_indices.expand([batch_size, max_l_prime_len]); + let blank_inserted_target_lengths = target_lengths + .mul_scalar(2) + .add_scalar(1) + .reshape([batch_size, 1]); + let target_lengths_expanded = + blank_inserted_target_lengths.expand([batch_size, max_l_prime_len]); + + col_indices_expanded.lower(target_lengths_expanded) + } + + fn log_sum_exp( + &self, + log_tensor1: Tensor, + log_tensor2: Tensor, + device: &B::Device, + ) -> Tensor { + let shape = log_tensor1.dims(); + let ones_tensor = Tensor::::ones(shape, device); + + // Let A and B represent parameters tensor1 and tensor2 respectively. + // Let C be the tensor this method returns. + // If an entry in both A and B are neg_inf, then the same entry + // in C should also contain neg_inf. + // If an entry in only one of A or B is neg_inf, then the same entry in + // C should contain the value of the other tensor entry which is not neg_inf. + let tensor1_is_neg_inf = log_tensor1.clone().equal_elem(f32::NEG_INFINITY); + let tensor2_is_neg_inf = log_tensor2.clone().equal_elem(f32::NEG_INFINITY); + let temp_tensor1 = ones_tensor + .clone() + .mask_where(tensor1_is_neg_inf.clone(), log_tensor2.clone()); + let neg_inf_lse_tensor = + temp_tensor1.mask_where(tensor2_is_neg_inf.clone(), log_tensor1.clone()); + + // Create sanitized tensors for math operations to prevent NaN. Replace neg_inf + // with 0.0. The tensor neg_inf_lse_tensor contains correct values for entries + // where at least one of the corresponding entries in log_tensor1 or log_tensor2 + // is neg_inf. Hence, the math operations below is computing the values for entries + // that are not already filled with their actual/correct values. Thus, result for + // these positions (where we sanitize) are not used anyway since the + // unfilled_entries_mask is applied at the end. + let tensor1_safe = log_tensor1 + .clone() + .mask_fill(tensor1_is_neg_inf.clone(), 0.0); + let tensor2_safe = log_tensor2 + .clone() + .mask_fill(tensor2_is_neg_inf.clone(), 0.0); + + // Create a mask which contains true for entries whose values were not + // set by operations above + let filled_entries_mask = tensor1_is_neg_inf.bool_or(tensor2_is_neg_inf); + let unfilled_entries_mask = filled_entries_mask.bool_not(); + + let max_tensor = tensor1_safe.clone().max_pair(tensor2_safe.clone()); + let diff_tensor = tensor1_safe.sub(tensor2_safe); + let exp_tensor = diff_tensor.abs().neg().exp(); + let ln_tensor = ones_tensor.add(exp_tensor).log(); + let lse_tensor = max_tensor.add(ln_tensor); + neg_inf_lse_tensor.mask_where(unfilled_entries_mask, lse_tensor) + } + + fn create_combined_s_t_mask( + &self, + input_lengths: Tensor, + t: usize, + batch_size: usize, + max_l_prime_len: usize, + s_mask: Tensor, + ) -> Tensor { + // Create masks for valid t and s + let t_mask_1d = input_lengths + .clone() + .greater_elem(t as i64) + .reshape([batch_size, 1]); + let t_mask = t_mask_1d.expand([batch_size, max_l_prime_len]); + + t_mask.bool_and(s_mask.clone()) + } + + fn compute_log_alpha_t_s( + &self, + t: usize, + combined_s_t_mask: Tensor, + log_alpha_t_s: Tensor, + l_prime_combined_mask: Tensor, + log_probs: Tensor, + blank_inserted_targets: Tensor, + ) -> Tensor { + let device = log_probs.device(); + let log_alpha_t_minus_1 = log_alpha_t_s.clone(); + + // No move from last time step: alpha_{t-1}(s) + let log_alpha_s = log_alpha_t_minus_1.clone(); + + // Single move from last time step: alpha_{t-1}(s - 1) + let log_alpha_s_minus_1 = + self.right_shift_2d_tensor(log_alpha_t_minus_1.clone(), 1, &device); + + // A skip move (moving 2 positions) from last time step: alpha_{t-1}(s - 2) + let log_alpha_s_minus_2 = + self.right_shift_2d_tensor(log_alpha_t_minus_1.clone(), 2, &device); + + // Compute alpha_{t}(s) using recursion, corresponding to equation 6 of the paper. + let log_alpha_bar = self.log_sum_exp(log_alpha_s, log_alpha_s_minus_1, &device); + let log_alpha_bar_plus_log_alpha_s_minus_2 = + self.log_sum_exp(log_alpha_bar.clone(), log_alpha_s_minus_2, &device); + let log_alpha_s_to_s_minus_2 = log_alpha_bar.mask_where( + l_prime_combined_mask.clone(), + log_alpha_bar_plus_log_alpha_s_minus_2, + ); // [N, 2 * U + 1] + let log_probs_t = log_probs.clone().slice(s![t, .., ..]).squeeze_dim::<2>(0); // [N, C] + let log_probs_l_prime_s = log_probs_t.gather(1, blank_inserted_targets.clone()); + let temp_log_alpha_t_s = log_alpha_s_to_s_minus_2.add(log_probs_l_prime_s); + log_alpha_t_s.mask_where(combined_s_t_mask, temp_log_alpha_t_s) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn_ndarray::{NdArray, NdArrayDevice}; + + type TestBackend = NdArray; + + fn assert_approx_equal(actual: &[f32], expected: &[f32], tol: f32) { + assert_eq!( + actual.len(), + expected.len(), + "Length mismatch: actual {} vs expected {}", + actual.len(), + expected.len() + ); + for (i, (a, e)) in actual.iter().zip(expected.iter()).enumerate() { + assert!( + (a - e).abs() < tol, + "Mismatch at index {}: expected {:.6}, got {:.6} (diff: {:.6})", + i, + e, + a, + (a - e).abs() + ); + } + } + + // --------------------------------------------------------------- + // insert_blanks tests + // --------------------------------------------------------------- + + #[test] + fn test_insert_blanks_single_sample() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + let targets = Tensor::::from_data([[1_i64, 2, 3]], &device); + let result = ctc.insert_blanks::(&targets, 1, 3, &device); + let result_data = result.into_data().to_vec::().unwrap(); + assert_eq!(result_data, vec![0, 1, 0, 2, 0, 3, 0]); + } + + #[test] + fn test_insert_blanks_batch() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + let targets = Tensor::::from_data([[1_i64, 2], [3, 4]], &device); + let result = ctc.insert_blanks::(&targets, 2, 2, &device); + let result_data = result.into_data().to_vec::().unwrap(); + assert_eq!(result_data, vec![0, 1, 0, 2, 0, 0, 3, 0, 4, 0]); + } + + #[test] + fn test_insert_blanks_custom_blank() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().with_blank(2).init(); + + let targets = Tensor::::from_data([[0_i64, 1]], &device); + let result = ctc.insert_blanks::(&targets, 1, 2, &device); + let result_data = result.into_data().to_vec::().unwrap(); + // l' = [blank=2, 0, blank=2, 1, blank=2] + assert_eq!(result_data, vec![2, 0, 2, 1, 2]); + } + + // --------------------------------------------------------------- + // Assertions + // --------------------------------------------------------------- + + #[test] + #[should_panic(expected = "blank index")] + fn test_ctc_loss_panics_invalid_blank_index() { + let device = NdArrayDevice::Cpu; + // blank=5 is out of bounds for num_classes=3 + let ctc = CTCLossConfig::new().with_blank(5).init(); + + let log_probs = Tensor::::zeros([2, 1, 3], &device); + let targets = Tensor::::from_data([[1]], &device); + let input_lengths = Tensor::::from_data([2], &device); + let target_lengths = Tensor::::from_data([1], &device); + + ctc.forward(log_probs, targets, input_lengths, target_lengths); + } + + #[test] + #[should_panic(expected = "must equal batch_size")] + fn test_ctc_loss_panics_mismatched_batch_size() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + // Logits batch size = 2 + let log_probs = Tensor::::zeros([2, 2, 3], &device); + // Targets batch size = 1 (Mismatch) + let targets = Tensor::::from_data([[1]], &device); + let input_lengths = Tensor::::from_data([2, 2], &device); + let target_lengths = Tensor::::from_data([1, 1], &device); + + ctc.forward(log_probs, targets, input_lengths, target_lengths); + } + + #[test] + #[should_panic(expected = "input_lengths length")] + fn test_ctc_loss_panics_input_lengths_mismatch() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + // Logits batch size = 2 + let log_probs = Tensor::::zeros([2, 2, 3], &device); + let targets = Tensor::::from_data([[1], [2]], &device); + + // Input lengths size = 1 (Mismatch) + let input_lengths = Tensor::::from_data([2], &device); + let target_lengths = Tensor::::from_data([1, 1], &device); + + ctc.forward(log_probs, targets, input_lengths, target_lengths); + } + + #[test] + #[should_panic(expected = "target_lengths length")] + fn test_ctc_loss_panics_target_lengths_mismatch() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + // Logits batch size = 2 + let log_probs = Tensor::::zeros([2, 2, 3], &device); + let targets = Tensor::::from_data([[1], [2]], &device); + let input_lengths = Tensor::::from_data([2, 2], &device); + + // Target lengths size = 1 (Mismatch) + let target_lengths = Tensor::::from_data([1], &device); + + ctc.forward(log_probs, targets, input_lengths, target_lengths); + } + + // --------------------------------------------------------------- + // Edge Case & Config Tests + // --------------------------------------------------------------- + + #[test] + fn test_ctc_loss_repeated_labels_minimum_input_length() { + // T=3, N=1, C=2, blank=0, target=[1, 1], uniform P = 1/2. + // + // The minimum T for target [1, 1] is 3: the only valid path is (1, 0, 1). + // prob = (1/2)^3 = 1/8 + // Loss = -ln(1/8) = 3 * ln(2) + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + let log_probs = Tensor::::full([3, 1, 2], 0.5_f32.ln(), &device); + let targets = Tensor::::from_data([[1_i64, 1]], &device); + let input_lengths = Tensor::::from_data([3_i64], &device); + let target_lengths = Tensor::::from_data([2_i64], &device); + + let loss = ctc.forward(log_probs, targets, input_lengths, target_lengths); + let loss_data = loss.into_data().to_vec::().unwrap(); + let expected = 3.0 * 2.0_f32.ln(); + assert_approx_equal(&loss_data, &[expected], 1e-3); + } + + #[test] + fn test_ctc_loss_custom_blank_uniform() { + // T=3, N=1, C=3, blank=2, target=[0, 1], uniform P = 1/3. + // + // Two distinct labels, 3 classes, 3 time steps, just with + // blank=2 instead of 0. + // 5 valid paths → total = 5/27 + // Loss = -ln(5/27) + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().with_blank(2).init(); + + let log_probs = Tensor::::full([3, 1, 3], (1.0_f32 / 3.0).ln(), &device); + let targets = Tensor::::from_data([[0_i64, 1]], &device); + let input_lengths = Tensor::::from_data([3_i64], &device); + let target_lengths = Tensor::::from_data([2_i64], &device); + + let loss = ctc.forward(log_probs, targets, input_lengths, target_lengths); + let loss_data = loss.into_data().to_vec::().unwrap(); + let expected = -(5.0_f32 / 27.0).ln(); + assert_approx_equal(&loss_data, &[expected], 1e-3); + } + + // --------------------------------------------------------------- + // zero_infinity tests + // --------------------------------------------------------------- + + #[test] + fn test_ctc_loss_zero_infinity_produces_inf_when_disabled() { + // T=2, N=1, C=3, blank=0, target=[1, 1], input_length=2 + // Target [1, 1] requires at least 3 time steps → no valid paths → loss = +inf + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().with_zero_infinity(false).init(); + + let log_probs = Tensor::::full([2, 1, 3], (1.0_f32 / 3.0).ln(), &device); + let targets = Tensor::::from_data([[1_i64, 1]], &device); + let input_lengths = Tensor::::from_data([2_i64], &device); + let target_lengths = Tensor::::from_data([2_i64], &device); + + let loss = ctc.forward(log_probs, targets, input_lengths, target_lengths); + let loss_data = loss.into_data().to_vec::().unwrap(); + assert!( + loss_data[0].is_infinite() && loss_data[0] > 0.0, + "Expected +inf, got {}", + loss_data[0] + ); + } + + #[test] + fn test_ctc_loss_zero_infinity_masks_inf_when_enabled() { + // Same inputs as above, but zero_infinity=true → loss should be 0.0 + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().with_zero_infinity(true).init(); + + let log_probs = Tensor::::full([2, 1, 3], (1.0_f32 / 3.0).ln(), &device); + let targets = Tensor::::from_data([[1_i64, 1]], &device); + let input_lengths = Tensor::::from_data([2_i64], &device); + let target_lengths = Tensor::::from_data([2_i64], &device); + + let loss = ctc.forward(log_probs, targets, input_lengths, target_lengths); + let loss_data = loss.into_data().to_vec::().unwrap(); + assert_approx_equal(&loss_data, &[0.0], 1e-6); + } + + #[test] + fn test_ctc_loss_zero_infinity_does_not_affect_finite_loss() { + // Verify that zero_infinity=true does not change a finite loss value. + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().with_zero_infinity(true).init(); + + let log_probs = Tensor::::full([2, 1, 2], 0.5_f32.ln(), &device); + let targets = Tensor::::from_data([[1_i64]], &device); + let input_lengths = Tensor::::from_data([2_i64], &device); + let target_lengths = Tensor::::from_data([1_i64], &device); + + let loss = ctc.forward(log_probs, targets, input_lengths, target_lengths); + let loss_data = loss.into_data().to_vec::().unwrap(); + let expected = -(0.75_f32).ln(); + assert_approx_equal(&loss_data, &[expected], 1e-3); + } +} + +#[cfg(test)] +mod pytorch_comparison_tests { + use super::*; + use burn::tensor::activation::log_softmax; + use burn_autodiff::Autodiff; + use burn_core::tensor::TensorData; + use burn_ndarray::{NdArray, NdArrayDevice}; + + type InnerBackend = NdArray; + type TestBackend = Autodiff; + + fn assert_approx_equal(actual: &[f32], expected: &[f32], tol: f32) { + assert_eq!( + actual.len(), + expected.len(), + "Length mismatch: actual {} vs expected {}", + actual.len(), + expected.len() + ); + for (i, (a, e)) in actual.iter().zip(expected.iter()).enumerate() { + assert!( + (a - e).abs() < tol, + "Mismatch at index {}: expected {:.6}, got {:.6} (diff: {:.6})", + i, + e, + a, + (a - e).abs() + ); + } + } + + /// Deterministic logits: sin((t*7 + n*13 + c*3) * 0.1). + fn generate_logits( + t_size: usize, + n_size: usize, + c_size: usize, + device: &NdArrayDevice, + ) -> Tensor { + let mut data = Vec::with_capacity(t_size * n_size * c_size); + for t in 0..t_size { + for n in 0..n_size { + for c in 0..c_size { + data.push(((t * 7 + n * 13 + c * 3) as f32 * 0.1).sin()); + } + } + } + Tensor::::from_data(TensorData::new(data, [t_size, n_size, c_size]), device) + } + + /// Runs a CTC forward + backward test and asserts against expected values from PyTorch. + /// + /// This helper performs the following steps: + /// 1. Generates deterministic logits using a sine-wave formula. + /// 2. Computes the CTC loss (forward pass). + /// 3. Asserts the computed loss matches `expected_losses`. + /// 4. Backpropagates the sum of the loss. + /// 5. Asserts the resulting gradients w.r.t. logits match `expected_grad_flat`. + /// + /// # Arguments + /// + /// - `expected_losses`: per-sample loss values from PyTorch (reduction='none'). + /// - `expected_grad_flat`: flattened gradient of sum(loss) w.r.t. logits. + #[allow(clippy::too_many_arguments)] + fn run_comparison( + label: &str, + t_size: usize, + n_size: usize, + c_size: usize, + targets_flat: Vec, + target_shape: [usize; 2], + input_lengths: Vec, + target_lengths: Vec, + blank: usize, + expected_losses: &[f32], + expected_grad_flat: &[f32], + loss_tol: f32, + grad_tol: f32, + ) { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().with_blank(blank).init(); + + let logits = generate_logits(t_size, n_size, c_size, &device).require_grad(); + let log_probs = log_softmax(logits.clone(), 2); + + let targets = Tensor::::from_data( + TensorData::new(targets_flat, target_shape), + &device, + ); + let input_lengths = Tensor::::from_data( + TensorData::new(input_lengths, [n_size]), + &device, + ); + let target_lengths = Tensor::::from_data( + TensorData::new(target_lengths, [n_size]), + &device, + ); + + let loss = ctc.forward(log_probs, targets, input_lengths, target_lengths); + let loss_data = loss.clone().into_data().to_vec::().unwrap(); + + println!("=== {} ===", label); + println!(" Loss: {:?}", loss_data); + assert_approx_equal(&loss_data, expected_losses, loss_tol); + + let loss_sum = loss.sum(); + let grads = loss_sum.backward(); + let logits_grad = logits.grad(&grads).unwrap(); + let grad_data = logits_grad.into_data().to_vec::().unwrap(); + assert_approx_equal(&grad_data, expected_grad_flat, grad_tol); + } + + #[test] + fn test_ctc_loss_uniform_input_lengths() { + // T=5, N=3, C=4, all input_lengths = 5 + // Expected losses and gradient from PyTorch + let expected_losses = [3.5236570835113525_f32, 3.495313882827759, 4.262677192687988]; + let expected_grad_flat = [ + -0.1679008007_f32, + -0.4595540464, + 0.2795598209, + 0.3478950262, + -0.3913056254, + -0.0832268298, + 0.2535884976, + 0.2209439576, + -0.0502742566, + 0.2766197622, + 0.2054125518, + -0.4317580462, + -0.0544800088, + -0.3144550920, + 0.0847885981, + 0.2841464877, + -0.1844545156, + -0.2063435912, + 0.2222184092, + 0.1685796976, + 0.0278018005, + 0.2657383382, + -0.0336986706, + -0.2598414719, + -0.0482986756, + -0.0098767160, + -0.1533526182, + 0.2115280181, + -0.1380317956, + -0.2198686600, + 0.2042596638, + 0.1536407918, + 0.0534787849, + 0.1819230020, + -0.2805589139, + 0.0451571345, + -0.0895631388, + 0.1996460557, + -0.2741115987, + 0.1640286744, + -0.2200077325, + -0.1693530381, + 0.2101601064, + 0.1792006642, + 0.0398471877, + -0.1131042913, + -0.2363226712, + 0.3095797896, + -0.2163617164, + 0.2740726173, + -0.2124865055, + 0.1547756046, + -0.4312027395, + -0.0446923785, + 0.2330704331, + 0.2428246588, + -0.0050083841, + -0.6256869435, + 0.2689785957, + 0.3617166877, + ]; + run_comparison( + "T=5, N=3, C=4 (uniform input lengths)", + 5, + 3, + 4, + vec![1, 2, 0, 1, 0, 0, 3, 2, 1], + [3, 3], + vec![5, 5, 5], + vec![2, 1, 3], + 0, + &expected_losses, + &expected_grad_flat, + 1e-3, + 1e-3, + ); + } + + #[test] + fn test_ctc_loss_repeated_labels() { + // T=8, N=4, C=6, includes consecutive repeated label [1,1,2] + // Expected losses and gradient from PyTorch + let expected_losses = [ + 8.84203052520752_f32, + 9.023029327392578, + 9.398024559020996, + 9.008068084716797, + ]; + let expected_grad_flat = [ + -0.2766432464, + -0.5202965736, + 0.1523768753, + 0.1896236390, + 0.2200277001, + 0.2349116206, + -0.1854365915, + 0.2031330466, + -0.4260218740, + 0.1678018719, + 0.1360142529, + 0.1045092493, + -0.6603536606, + 0.2278252542, + 0.1691786796, + 0.1262856424, + 0.0972681716, + 0.0397959016, + -0.0894432291, + -0.5457318425, + 0.1490373611, + 0.1462858170, + 0.1569476575, + 0.1829041988, + -0.2842915654, + -0.4220107496, + 0.1822281033, + 0.1889107376, + 0.1791101843, + 0.1560532600, + -0.1155678406, + 0.2295538932, + -0.2645366490, + -0.0288553704, + 0.1027252972, + 0.0766806602, + -0.5448347330, + 0.2031028718, + 0.1589304954, + 0.1322451383, + 0.1189499870, + -0.0683937520, + -0.0873993114, + -0.3051757514, + -0.2355299890, + 0.1586059481, + 0.2018169016, + 0.2676822543, + -0.3225219846, + -0.2611543834, + 0.1922984123, + 0.1632783115, + 0.1297036558, + 0.0983960181, + -0.1507159024, + 0.2256962359, + -0.1040333956, + -0.1514528394, + 0.0985243544, + 0.0819815546, + -0.2940836251, + 0.1586865336, + 0.1468491107, + 0.1485087872, + 0.1639631987, + -0.3239239752, + -0.0767390430, + -0.0434846729, + -0.4023587406, + -0.0052628326, + 0.2273432612, + 0.3005020022, + -0.2598774135, + -0.2188862711, + 0.1678501070, + 0.1352078766, + 0.1002781317, + 0.0754275694, + -0.1502914876, + 0.1930875033, + -0.0709601715, + -0.2219523191, + 0.1243555173, + 0.1257609427, + -0.0574148744, + 0.1152269915, + 0.1307857931, + 0.1599020809, + 0.2068412602, + -0.5553412437, + -0.0536844917, + 0.0758557543, + -0.2106334567, + -0.2509877980, + 0.1757438034, + 0.2637061775, + -0.1759711355, + -0.2431350052, + 0.1071053818, + 0.1259848624, + 0.1004033238, + 0.0856125653, + -0.1173698306, + 0.1213828772, + -0.1768893301, + -0.2070008069, + 0.1709136516, + 0.2089634240, + 0.0153109450, + 0.0967332721, + 0.1268781722, + 0.1706230640, + 0.2291058898, + -0.6386513710, + -0.0536664203, + 0.1378114969, + 0.0360041447, + -0.2989685237, + -0.0084722806, + 0.1872915775, + -0.1523490399, + -0.2111770809, + -0.0390694551, + 0.1366800815, + 0.1302325875, + 0.1356829405, + -0.0982905105, + -0.0127884001, + -0.3586881459, + -0.0259541404, + 0.2114149332, + 0.2843062580, + -0.0324133746, + 0.1084750593, + 0.1447229236, + 0.1862253845, + 0.2259712219, + -0.6329812407, + -0.1173689738, + 0.1914442331, + 0.1654772907, + -0.1376858056, + -0.2194855511, + 0.1176188141, + -0.1529908478, + -0.0606661662, + -0.3384291232, + 0.1524862647, + 0.1777049750, + 0.2218948901, + -0.0923086405, + -0.2855934799, + -0.3215619624, + 0.1726681292, + 0.2303666323, + 0.2964293361, + -0.2508065701, + 0.1479703039, + 0.1753441393, + 0.1917535067, + 0.1919818372, + -0.4562432170, + -0.2350299209, + 0.2257601619, + 0.1863904297, + 0.0388212129, + -0.2966264784, + 0.0806845874, + -0.1992894858, + 0.1068909168, + -0.5761897564, + 0.1624972969, + 0.2155302167, + 0.2905607820, + -0.1168124676, + -0.6870660186, + 0.1488010883, + 0.1881926507, + 0.2230074406, + 0.2438773215, + -0.5771554708, + 0.1980127096, + 0.1924194694, + 0.1714663208, + 0.1415647417, + -0.1263078004, + -0.3408652246, + 0.2292248607, + 0.1707807332, + 0.1269564927, + -0.2634142637, + 0.0773174241, + ]; + run_comparison( + "T=8, N=4, C=6 (repeated labels)", + 8, + 4, + 6, + vec![1, 1, 2, 0, 2, 3, 2, 1, 5, 0, 0, 0, 1, 2, 3, 4], + [4, 4], + vec![8, 8, 8, 8], + vec![3, 4, 1, 4], + 0, + &expected_losses, + &expected_grad_flat, + 1e-3, + 1e-3, + ); + } + + #[test] + fn test_ctc_loss_long_sequence() { + // T=10, N=2, C=8 + // Expected losses and gradient from PyTorch + let expected_losses = [12.629399299621582, 12.298524856567383]; + let expected_grad_flat = [ + -0.2570972741, + -0.6013792753, + 0.1061997041, + 0.1321590245, + 0.1533492655, + 0.1637226790, + 0.1598964781, + 0.1431493312, + -0.2540431321, + 0.1788398325, + -0.4038805366, + 0.1477340311, + 0.1197479516, + 0.0920107216, + 0.0686140805, + 0.0509770736, + -0.1364373565, + -0.3724762201, + 0.1489177048, + -0.0966964588, + 0.1463697106, + 0.1275274903, + 0.1033692732, + 0.0794258416, + -0.1771971881, + 0.2073454857, + -0.3109439015, + 0.1249521226, + -0.0101635465, + 0.0692621097, + 0.0533472970, + 0.0433975980, + -0.1398337185, + -0.0874802172, + 0.1705365479, + -0.2174201906, + 0.1150254831, + 0.0460043959, + 0.0647982135, + 0.0483694859, + -0.2332949787, + 0.1969220787, + -0.1270586401, + 0.1098557115, + -0.1364655048, + 0.0715296715, + 0.0553609394, + 0.0631506816, + -0.2169117928, + 0.0929956511, + 0.1624538749, + -0.2009791434, + 0.0904926360, + -0.0248185843, + 0.0532633252, + 0.0435040221, + -0.2313277274, + 0.1497355998, + -0.0024202778, + 0.1029939279, + -0.2776987851, + 0.0963881761, + 0.0351882279, + 0.1271408647, + -0.2590557337, + 0.1577988416, + 0.1429322213, + -0.1401246637, + 0.0866033062, + -0.1151762009, + 0.0683368817, + 0.0586853735, + -0.1322475076, + 0.0806737095, + 0.0528722852, + 0.0920089707, + -0.3037962914, + 0.1280544847, + -0.1391123086, + 0.2215466499, + -0.1918463260, + 0.1376975775, + 0.1160097718, + -0.0549413785, + 0.0970225409, + -0.2708687484, + 0.1147320047, + 0.0521945432, + -0.0504456684, + -0.0012221609, + 0.0644332916, + 0.0818370953, + -0.1036835983, + 0.1512031406, + -0.4072600305, + 0.2651379406, + -0.0681083873, + 0.0860663429, + 0.0810486302, + 0.0434282124, + 0.1056238264, + -0.2994530201, + 0.1729898751, + -0.1215954795, + -0.0481944978, + -0.1697723418, + 0.0725984722, + 0.0692019314, + 0.0859903544, + 0.1680216491, + -0.4071443677, + 0.2292988002, + -0.0205532499, + 0.0566616580, + 0.0326749459, + 0.0861379728, + 0.1142501161, + -0.0448331088, + 0.2054910213, + -0.4298293889, + -0.0647637174, + -0.4240962267, + 0.1013666242, + -0.0110451467, + 0.1519176364, + 0.1661346704, + -0.0719586164, + 0.1524447650, + -0.0496110357, + 0.0562372655, + -0.1889088154, + 0.1013496071, + 0.1339637935, + 0.1694275290, + 0.2007708699, + -0.4232292175, + -0.0401752405, + -0.2951072752, + 0.1443216652, + -0.2857291698, + 0.1489982456, + 0.1327733696, + 0.1096193567, + 0.0852990299, + -0.0413062274, + 0.0820900649, + -0.7903561592, + 0.1329460591, + 0.1535883099, + 0.1631743014, + 0.1585651338, + 0.1412984729, + -0.1033771932, + 0.1799504310, + 0.1697744429, + -0.5749052763, + 0.1189445183, + 0.0911802500, + 0.0679325759, + 0.0505003072, + ]; + run_comparison( + "T=10, N=2, C=8", + 10, + 2, + 8, + vec![1, 3, 5, 7, 2, 2, 4, 6, 1, 3], + [2, 5], + vec![10, 10], + vec![5, 5], + 0, + &expected_losses, + &expected_grad_flat, + 1e-3, + 1e-3, + ); + } + + #[test] + fn test_ctc_loss_mixed_input_lengths() { + // T=12, N=3, C=5, input_lengths=[12, 7, 10] + // Expected losses and gradient from PyTorch + let expected_losses = [10.595505714416504, 6.8078508377075195, 7.705057144165039]; + let expected_grad_flat = [ + -0.4790987670, + -0.2554937005, + 0.1991624236, + 0.2478453964, + 0.2875846624, + -0.3495813310, + 0.2268397957, + 0.2150714993, + -0.2442178279, + 0.1518878639, + -0.2764556706, + 0.2474014312, + -0.2137086987, + 0.1371368915, + 0.1056260392, + -0.2729502618, + -0.3609606028, + 0.2159237266, + 0.2238420397, + 0.1941450834, + -0.2953839302, + 0.1920599341, + 0.1974952668, + -0.2054278404, + 0.1112565696, + -0.1719199270, + 0.2299505472, + -0.2864859998, + 0.1497263014, + 0.0787290633, + -0.2035763413, + -0.3042884767, + 0.2126964629, + 0.1810975969, + 0.1140707731, + -0.2759391963, + 0.0975771844, + 0.1823379993, + -0.1112988219, + 0.1073228419, + -0.1336459517, + 0.1869296581, + -0.1996247321, + 0.1846873760, + -0.0383463502, + -0.2254105806, + -0.1834360659, + 0.1925925612, + 0.1462381780, + 0.0700158924, + -0.2259973884, + -0.0393539183, + 0.1802661419, + -0.0571591072, + 0.1422442794, + -0.0609069727, + 0.1089282706, + -0.0313654318, + 0.2186669111, + -0.2353227735, + -0.2840364873, + -0.0632198900, + 0.1755636632, + 0.1377806067, + 0.0339120962, + -0.1904856712, + -0.2139032930, + 0.1827126741, + 0.0056131603, + 0.2160631120, + -0.0243270602, + -0.0070458520, + 0.1070247591, + 0.2239368409, + -0.2995886803, + -0.2955487072, + 0.0309870224, + 0.1654911339, + 0.1581364125, + -0.0590658709, + -0.2191396207, + -0.3791662455, + 0.1803640425, + 0.1225430891, + 0.2953987718, + -0.0436352938, + -0.1575258970, + 0.1785279512, + 0.1756918877, + -0.1530586481, + -0.1834939867, + 0.0909025446, + 0.1423641294, + 0.1959712654, + -0.2457439601, + -0.3619639874, + -0.3929221630, + 0.1820438206, + 0.2454170734, + 0.3274252713, + -0.0628800318, + -0.2567180395, + 0.2112283260, + 0.0507859327, + 0.0575838275, + -0.0587697029, + 0.1174769849, + 0.0783569664, + 0.2290501744, + -0.3661144078, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + -0.0725664943, + -0.1532069892, + 0.2162397504, + -0.1248963475, + 0.1344300956, + -0.0362483934, + 0.1295878887, + -0.0502482466, + 0.2470482886, + -0.2901395261, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + -0.1349253207, + 0.0867646411, + 0.1998746395, + -0.2658679783, + 0.1141540110, + -0.0705668628, + 0.1519546807, + -0.2509805560, + 0.2475892603, + -0.0779965296, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + -0.2338010073, + 0.2471641302, + 0.1834627241, + -0.3026831448, + 0.1058573127, + -0.1155209392, + 0.1921830922, + -0.4129956067, + 0.2229512781, + 0.1133821756, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + -0.2636392713, + 0.2323469073, + -0.2913427949, + 0.1800564528, + 0.1425786912, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + 0.0000000000, + ]; + run_comparison( + "T=12, N=3, C=5 (mixed input lengths)", + 12, + 3, + 5, + vec![1, 4, 2, 0, 3, 1, 0, 0, 2, 4, 1, 3], + [3, 4], + vec![12, 7, 10], + vec![3, 2, 4], + 0, + &expected_losses, + &expected_grad_flat, + 1e-3, + 1e-3, + ); + } + + #[test] + fn test_ctc_loss_sum_reduction() { + // Same inputs as comparison_uniform_input_lengths, sum reduction + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + let logits = generate_logits(5, 3, 4, &device).require_grad(); + let log_probs = log_softmax(logits.clone(), 2); + let targets = Tensor::::from_data( + TensorData::new(vec![1_i64, 2, 0, 1, 0, 0, 3, 2, 1], [3, 3]), + &device, + ); + let il = Tensor::::from_data([5_i64, 5, 5], &device); + let tl = Tensor::::from_data([2_i64, 1, 3], &device); + + let loss = ctc.forward_with_reduction(log_probs, targets, il, tl, Reduction::Sum); + let loss_data = loss.clone().into_data().to_vec::().unwrap(); + + let expected_sum = 11.2816486359_f32; // Expected value from PyTorch + assert_approx_equal(&loss_data, &[expected_sum], 1e-3); + + let grads = loss.backward(); + let logits_grad = logits.grad(&grads).unwrap(); + let grad_data = logits_grad.into_data().to_vec::().unwrap(); + // Expected gradient from PyTorch + let expected_grad = [ + -0.1679008007_f32, + -0.4595540464, + 0.2795598209, + 0.3478950262, + -0.3913056254, + -0.0832268298, + 0.2535884976, + 0.2209439576, + -0.0502742566, + 0.2766197622, + 0.2054125518, + -0.4317580462, + -0.0544800088, + -0.3144550920, + 0.0847885981, + 0.2841464877, + -0.1844545156, + -0.2063435912, + 0.2222184092, + 0.1685796976, + 0.0278018005, + 0.2657383382, + -0.0336986706, + -0.2598414719, + -0.0482986756, + -0.0098767160, + -0.1533526182, + 0.2115280181, + -0.1380317956, + -0.2198686600, + 0.2042596638, + 0.1536407918, + 0.0534787849, + 0.1819230020, + -0.2805589139, + 0.0451571345, + -0.0895631388, + 0.1996460557, + -0.2741115987, + 0.1640286744, + -0.2200077325, + -0.1693530381, + 0.2101601064, + 0.1792006642, + 0.0398471877, + -0.1131042913, + -0.2363226712, + 0.3095797896, + -0.2163617164, + 0.2740726173, + -0.2124865055, + 0.1547756046, + -0.4312027395, + -0.0446923785, + 0.2330704331, + 0.2428246588, + -0.0050083841, + -0.6256869435, + 0.2689785957, + 0.3617166877, + ]; + assert_approx_equal(&grad_data, &expected_grad, 1e-3); + } + + #[test] + fn test_ctc_loss_mean_reduction() { + let device = NdArrayDevice::Cpu; + let ctc = CTCLossConfig::new().init(); + + let logits = generate_logits(5, 3, 4, &device).require_grad(); + let log_probs = log_softmax(logits.clone(), 2); + let targets = Tensor::::from_data( + TensorData::new(vec![1_i64, 2, 0, 1, 0, 0, 3, 2, 1], [3, 3]), + &device, + ); + let il = Tensor::::from_data([5_i64, 5, 5], &device); + let tl = Tensor::::from_data([2_i64, 1, 3], &device); + + let loss = ctc.forward_with_reduction(log_probs, targets, il, tl, Reduction::Mean); + let loss_data = loss.clone().into_data().to_vec::().unwrap(); + + let expected_mean = 2.2260115147_f32; // Expected value from PyTorch + assert_approx_equal(&loss_data, &[expected_mean], 1e-3); + + let grads = loss.backward(); + let logits_grad = logits.grad(&grads).unwrap(); + let grad_data = logits_grad.into_data().to_vec::().unwrap(); + // Expected gradient from PyTorch + let expected_grad = [ + -0.0279834662_f32, + -0.0765923411, + 0.0465933047, + 0.0579825081, + -0.1304352134, + -0.0277422778, + 0.0845294967, + 0.0736479908, + -0.0055860290, + 0.0307355281, + 0.0228236169, + -0.0479731150, + -0.0090800021, + -0.0524091832, + 0.0141314333, + 0.0473577492, + -0.0614848398, + -0.0687812045, + 0.0740728080, + 0.0561932363, + 0.0030890885, + 0.0295264814, + -0.0037442972, + -0.0288712755, + -0.0080497796, + -0.0016461194, + -0.0255587716, + 0.0352546684, + -0.0460105985, + -0.0732895583, + 0.0680865571, + 0.0512135960, + 0.0059420872, + 0.0202136654, + -0.0311732125, + 0.0050174589, + -0.0149271907, + 0.0332743451, + -0.0456852652, + 0.0273381118, + -0.0733359158, + -0.0564510152, + 0.0700533763, + 0.0597335547, + 0.0044274656, + -0.0125671430, + -0.0262580756, + 0.0343977548, + -0.0360602848, + 0.0456787720, + -0.0354144201, + 0.0257959347, + -0.1437342465, + -0.0148974592, + 0.0776901469, + 0.0809415579, + -0.0005564869, + -0.0695207715, + 0.0298865121, + 0.0401907414, + ]; + assert_approx_equal(&grad_data, &expected_grad, 1e-3); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/huber.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/huber.rs new file mode 100644 index 0000000..fb132f7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/huber.rs @@ -0,0 +1,215 @@ +use burn_core as burn; + +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::{config::Config, module::Module}; + +use super::Reduction; + +/// Configuration to create a [Huber loss](HuberLoss). +#[derive(Config, Debug)] +pub struct HuberLossConfig { + /// The bound where the Huber loss function changes from quadratic to linear behaviour. + pub delta: f32, +} + +impl HuberLossConfig { + /// Initialize [Huber loss](HuberLoss). + pub fn init(&self) -> HuberLoss { + self.assertions(); + HuberLoss { + delta: self.delta, + lin_bias: self.delta * self.delta * 0.5, + } + } + + fn assertions(&self) { + assert!( + self.delta >= 0., // This also tests for normality + "Delta for Huber loss must be a non-negative number." + ); + } +} + +/// Calculate the Huber loss between the inputs and the target. +/// +/// The loss for each element of the residuals `r = targets - predictions` is given by +/// +/// ```text +/// L(r) = 0.5 * r^2 if |r| <= d +/// L(r) = 0.5 * d^2 + d * (|r| - d) if |r| > d +/// ``` +/// +/// where `d` is the configured `delta`. In particular, this is equal to the +/// [L2 Loss](super::MseLoss) for residuals with magnitude smaller than `delta`, +/// but behaves linearly instead of quadratically for large residuals. +/// +/// This loss function is less sensitive to outliers than the mean squared error loss. +/// +/// See also: +#[derive(Module, Debug, Clone)] +#[module(custom_display)] +pub struct HuberLoss { + /// The bound where the Huber loss function changes from quadratic to linear behaviour. + pub delta: f32, + /// Precomputed value for the linear bias. + pub lin_bias: f32, // delta * delta * 0.5 precomputed +} + +impl ModuleDisplay for HuberLoss { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("delta", &self.delta) + .add("lin_bias", &self.lin_bias) + .optional() + } +} + +impl HuberLoss { + /// Compute the loss element-wise for the predictions and targets, then reduce + /// to a single loss value. + /// + /// `Reduction::Auto` behaves as `Reduction::Mean`. + /// + /// # Shapes + /// + /// - predictions: \[...dims\] + /// - targets: \[...dims\] + /// - output: \[1\] + pub fn forward( + &self, + predictions: Tensor, + targets: Tensor, + reduction: Reduction, + ) -> Tensor { + let loss = self.forward_no_reduction(predictions, targets); + match reduction { + Reduction::Mean | Reduction::Auto => loss.mean(), + Reduction::Sum => loss.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + /// Compute the loss element-wise for the predictions and targets. + /// + /// # Shapes + /// + /// - predictions: [...dims] + /// - targets: [...dims] + /// - output: [...dims] + pub fn forward_no_reduction( + &self, + predictions: Tensor, + targets: Tensor, + ) -> Tensor { + let residuals = targets - predictions; + self.forward_residuals(residuals) + } + /// Compute the loss element-wise for the given residuals. + /// + /// # Shapes + /// + /// - residuals: [...dims] + /// - output: [...dims] + pub fn forward_residuals( + &self, + residuals: Tensor, + ) -> Tensor { + let is_large = residuals.clone().abs().greater_elem(self.delta); + // We are interested in `sign(r)` when `abs(r) > self.delta`. Note that the + // `sign()` function, in general, suffers from a jump at 0. + // Instead the following tensor implements `delta * sign(r)` for values outside + // the bound: + let softsign = residuals.clone().clamp(-self.delta, self.delta); + + // 0.5 * d^2 + d * (|r| - d) = + // d * |r| - 0.5 * d^2 + // Moreover |r| = sign(r) * r + let outside = softsign.mul(residuals.clone()).sub_scalar(self.lin_bias); + + let inside = residuals.square().mul_scalar(0.5); + inside.mask_where(is_large, outside) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + type TestTensor = Tensor; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_huber_loss() { + let predict = TensorData::from([-2., -0.5, 0., 0.3, 1.]); + let targets = TensorData::from([0., 0., 0., 0., 0.]); + + let device = Default::default(); + + let predict = TestTensor::<1>::from_data(predict, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let huber = HuberLossConfig::new(0.5).init(); + + let loss_sum = huber.forward(predict.clone(), targets.clone(), Reduction::Sum); + let loss = huber.forward(predict.clone(), targets.clone(), Reduction::Auto); + let loss_no_reduction = huber.forward_no_reduction(predict, targets); + + let expected = TensorData::from([0.875, 0.125, 0., 0.045, 0.375]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([0.284]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([1.42]); + loss_sum + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[cfg(feature = "std")] + #[test] + fn test_huber_ad_loss() { + type TestAutodiffTensor = Tensor; + + let predict = TensorData::from([-2., -0.5, 0., 0.3, 1.]); + let targets = TensorData::from([0., 0., 0., 0., 0.]); + + let device = Default::default(); + let predict = TestAutodiffTensor::from_data(predict, &device).require_grad(); + let targets = TestAutodiffTensor::from_data(targets, &device); + + let loss = HuberLossConfig::new(0.5).init(); + let loss = loss.forward_no_reduction(predict.clone(), targets); + + let grads = loss.backward(); + let grads_predict = predict.grad(&grads).unwrap(); + + let expected = TensorData::from([-0.5, -0.5, 0., 0.3, 0.5]); + grads_predict + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = HuberLossConfig::new(0.5); + let loss = config.init(); + + assert_eq!( + alloc::format!("{loss}"), + "HuberLoss {delta: 0.5, lin_bias: 0.125}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/kldiv.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/kldiv.rs new file mode 100644 index 0000000..d4b84c3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/kldiv.rs @@ -0,0 +1,200 @@ +use burn_core as burn; + +use super::Reduction; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::{config::Config, module::Module}; + +/// Configuration to create a [KLDiv loss](KLDivLoss). +#[derive(Config, Debug)] +pub struct KLDivLossConfig { + /// Specifies whether target is the log space. Default: False. + #[config(default = false)] + pub log_target: bool, +} + +impl KLDivLossConfig { + /// Initialize [KLDiv Loss](KLDivLoss). + pub fn init(&self) -> KLDivLoss { + KLDivLoss { + log_target: self.log_target, + } + } +} + +/// Kullback-Leibler Divergence Loss +/// +/// KL Divergence shows the difference between two probability distributions by measuring information loss +/// +/// KLDivLoss = +/// ```tex +/// y_{true} \cdot (\log{y_{true}} - \log{y_{pred}}) +/// ``` +/// By default, the loss expects the input in the log-space. +/// The targets may also be provided in the log-space if `log_target` is true. +/// +/// See +/// - [Kullback–Leibler divergence](https://en.wikipedia.org/wiki/Kullback-Leibler_divergence) +#[derive(Module, Debug, Clone)] +#[module(custom_display)] +pub struct KLDivLoss { + /// Specifies whether target is the log space. Default: False. + pub log_target: bool, +} + +impl ModuleDisplay for KLDivLoss { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("log_target", &self.log_target).optional() + } +} + +impl KLDivLoss { + /// Compute the criterion on the input tensor. + /// + /// `Reduction::Auto` behaves as `Reduction::BatchMean`,`Reduction::Mean` dose not align with the math definition. + /// + /// # Shapes + /// + /// - predictions: \[batch_size,num_targets\] + /// - targets: \[batch_size,num_targets\] + /// - output: \[1\] + pub fn forward( + &self, + predictions: Tensor, + targets: Tensor, + reduction: Reduction, + ) -> Tensor { + let loss = self.forward_no_reduction(predictions, targets); + match reduction { + Reduction::BatchMean | Reduction::Auto => { + let batch_size = loss.dims()[0] as f32; + loss.sum().div_scalar(batch_size) + } + Reduction::Mean => loss.mean(), + Reduction::Sum => loss.sum(), + } + } + /// Compute the criterion on the input tensor without reducing. + pub fn forward_no_reduction( + &self, + predictions: Tensor, + targets: Tensor, + ) -> Tensor { + match self.log_target { + true => targets.clone().exp().mul(targets.sub(predictions)), + false => { + let epsilon = 1e-8; + let log_target = targets.clone().clamp(epsilon, 1.0).log(); + targets.mul(log_target.sub(predictions)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + type TestTensor = Tensor; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_kl_div_loss() { + let predict = TensorData::from([[-1.0, -0.5], [-2.0, -0.2]]); + let targets = TensorData::from([[0.4, 0.6], [0.1, 0.9]]); + + let device = Default::default(); + let predict = TestTensor::<2>::from_data(predict, &device); + let targets = TestTensor::<2>::from_data(targets, &device); + + let kl_loss = KLDivLossConfig { log_target: false }.init(); + + let loss_sum = kl_loss.forward(predict.clone(), targets.clone(), Reduction::Sum); + let loss_batch_mean = + kl_loss.forward(predict.clone(), targets.clone(), Reduction::BatchMean); + let loss_no_reduction = kl_loss.forward_no_reduction(predict, targets); + + let expected_no_reduction = + TensorData::from([[0.0334837139, -0.0064953566], [-0.0302585065, 0.0851755068]]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected_no_reduction, Tolerance::absolute(1e-5)); + + let expected_sum = TensorData::from([0.08191]); + loss_sum + .into_data() + .assert_approx_eq::(&expected_sum, Tolerance::absolute(1e-5)); + + let expected_batch_mean = TensorData::from([0.04095]); + loss_batch_mean + .into_data() + .assert_approx_eq::(&expected_batch_mean, Tolerance::absolute(1e-5)); + } + + #[test] + fn test_kl_div_loss_log_target() { + let device = Default::default(); + let predict = TestTensor::<1>::from_data([-1.0, -2.0], &device); + let targets = TestTensor::<1>::from_data([-0.5, -1.5], &device); + + let kl_loss = KLDivLossConfig { log_target: true }.init(); + + let loss_no_reduction = kl_loss.forward_no_reduction(predict.clone(), targets.clone()); + let expected_none = TensorData::from([0.3032653299, 0.1115650801]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected_none, Tolerance::absolute(1e-5)); + + let loss_batch_mean = + kl_loss.forward(predict.clone(), targets.clone(), Reduction::BatchMean); + let expected_bm = TensorData::from([0.207415204965]); + loss_batch_mean + .into_data() + .assert_approx_eq::(&expected_bm, Tolerance::absolute(1e-5)); + + let loss_sum = kl_loss.forward(predict, targets, Reduction::Sum); + let expected_sum = TensorData::from([0.414830409931]); + loss_sum + .into_data() + .assert_approx_eq::(&expected_sum, Tolerance::absolute(1e-5)); + } + + #[cfg(feature = "std")] + #[test] + fn test_kl_div_ad_loss() { + type TestAutodiffTensor = Tensor; + + let device = Default::default(); + let predict = TestAutodiffTensor::from_data([[-1.0, -0.5]], &device).require_grad(); + let targets = TestAutodiffTensor::from_data([[0.4, 0.6]], &device); + + let kl_loss = KLDivLossConfig { log_target: false }.init(); + let loss = kl_loss.forward(predict.clone(), targets, Reduction::Sum); + + let grads = loss.backward(); + let grads_predict = predict.grad(&grads).unwrap(); + + // d/d_pred [target * (log_target - pred)] = -target + let expected = TensorData::from([[-0.4, -0.6]]); + grads_predict + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = KLDivLossConfig { log_target: true }; + let loss = config.init(); + + assert_eq!(alloc::format!("{loss}"), "KLDivLoss {log_target: true}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/lp_loss.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/lp_loss.rs new file mode 100644 index 0000000..4aaa866 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/lp_loss.rs @@ -0,0 +1,672 @@ +use super::Reduction; +use burn::config::Config; +use burn::module::Module; +use burn::tensor::{Tensor, backend::Backend}; +use burn_core as burn; + +/// Configuration for the [Lp Loss](LpLoss) module. +/// +/// # Example +/// +/// ```ignore +/// use burn_nn::loss::{LpLossConfig, Reduction}; +/// +/// // Create L1 loss (MAE when using mean reduction) +/// let l1_loss = LpLossConfig::l1(); +/// +/// // Create L2 loss (MSE when using mean reduction) +/// let l2_loss = LpLossConfig::l2(); +/// +/// // Create custom Lp loss with p=3 +/// let l3_loss = LpLossConfig::new(3.0).init(); +/// ``` +#[derive(Config, Debug)] +pub struct LpLossConfig { + /// The exponent `p` determining the type of error measurement. + /// + /// Common values: + /// - `p = 1.0`: L1 loss (MAE with mean reduction) - robust to outliers + /// - `p = 2.0`: L2 loss (MSE with mean reduction) - standard choice, differentiable everywhere + /// - `p > 2.0`: Increasingly sensitive to large errors (outliers) + /// - `0 < p < 1`: More robust to outliers than L1 (quasi-norm) + pub p: f64, +} + +impl LpLossConfig { + /// Initializes a [Lp Loss](LpLoss) module. + /// + /// # Panics + /// + /// Panics if `p <= 0`. + pub fn init(&self) -> LpLoss { + self.assertions(); + LpLoss { p: self.p } + } + + /// Creates L1 loss (p=1). + /// + /// When used with `Reduction::Mean`, this computes Mean Absolute Error (MAE). + /// When used with `Reduction::Sum`, this computes Sum of Absolute Errors (SAE). + pub fn l1() -> LpLoss { + LpLoss { p: 1.0 } + } + + /// Creates L2 loss (p=2). + /// + /// When used with `Reduction::Mean`, this computes Mean Squared Error (MSE). + /// When used with `Reduction::Sum`, this computes Sum of Squared Errors (SSE). + pub fn l2() -> LpLoss { + LpLoss { p: 2.0 } + } + + fn assertions(&self) { + assert!(self.p > 0.0, "The order of the norm p must be positive.") + } +} + +/// Computes the Lp Loss between predictions and targets. +/// +/// This loss function computes the element-wise p-th power of absolute errors, +/// then reduces them via mean or sum. +/// +/// # Mathematical Definition +/// +/// For predictions `ŷ` and targets `y`, the element-wise loss is: +/// +/// ```text +/// Lᵢ = |ŷᵢ - yᵢ|ᵖ +/// ``` +/// +/// With mean reduction (default), the final loss is: +/// +/// ```text +/// L = (1/n) × Σᵢ |ŷᵢ - yᵢ|ᵖ +/// ``` +/// +/// # Notes +/// +/// - This implementation computes `|error|^p`, **not** the Lp norm `(Σ|error|^p)^(1/p)`. +/// - The `p = 1` case uses an optimized `abs()` operation. +/// - The `p = 2` case uses an optimized computation `error * error` instead of `powf`. +/// +/// # Example +/// +/// ```ignore +/// use burn_nn::loss::{LpLossConfig, Reduction}; +/// use burn::tensor::Tensor; +/// +/// // Create L2 loss +/// let l2_loss = LpLossConfig::l2(); +/// +/// let predictions: Tensor = /* model output */; +/// let targets: Tensor = /* ground truth */; +/// +/// // Compute loss with mean reduction (MSE) +/// let mse = l2_loss.forward(predictions.clone(), targets.clone(), Reduction::Mean); +/// +/// // Compute loss with sum reduction (SSE) +/// let sse = l2_loss.forward(predictions.clone(), targets.clone(), Reduction::Sum); +/// +/// // Compute loss with no reduction +/// let unreduced_l2_loss = l2_loss.forward_no_reduction(predictions, targets); +/// ``` +#[derive(Module, Clone, Debug)] +pub struct LpLoss { + /// The order of the norm (e.g., 1 for L1, 2 for L2). + /// Equivalently, the exponent `p` for computing `|error|^p`. + pub p: f64, +} + +impl LpLoss { + /// Computes the element-wise loss `|error|^p` with reduction. + /// + /// # Arguments + /// + /// * `predictions` - The model's predicted values. + /// * `targets` - The ground truth target values. + /// * `reduction` - Specifies how to reduce the element-wise losses: + /// - `Reduction::Mean` or `Reduction::Auto`: Returns the mean of all element-wise losses. + /// - `Reduction::Sum`: Returns the sum of all element-wise losses. + /// + /// # Returns + /// + /// A scalar tensor containing the reduced loss value. + /// + /// # Shapes + /// + /// - predictions: `[...dims]` - Any shape + /// - targets: `[...dims]` - Must match predictions shape + /// - output: `[1]` - Scalar loss value + pub fn forward( + &self, + predictions: Tensor, + targets: Tensor, + reduction: Reduction, + ) -> Tensor { + let unreduced_loss = self.forward_no_reduction(predictions, targets); + + match reduction { + Reduction::Mean | Reduction::Auto => unreduced_loss.mean(), + Reduction::Sum => unreduced_loss.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + + /// Computes the element-wise loss `|error|^p` without reduction. + /// + /// # Arguments + /// + /// * `predictions` - The model's predicted values. + /// * `targets` - The ground truth target values. + /// + /// # Returns + /// + /// A tensor of the same shape as the inputs, containing `|prediction - target|^p` + /// for each element. + /// + /// # Shapes + /// + /// - predictions: `[...dims]` - Any shape + /// - targets: `[...dims]` - Must match predictions shape + /// - output: `[...dims]` - Same shape as inputs + pub fn forward_no_reduction( + &self, + predictions: Tensor, + targets: Tensor, + ) -> Tensor { + let error = predictions.sub(targets); + + // Use simplified/optimized expressions for common cases (p = 1, p = 2) + if self.p == 1.0 { + // L1 loss + error.abs() + } else if self.p == 2.0 { + // L2 loss + error.clone().mul(error) + } else { + error.abs().powf_scalar(self.p) + } + } + + /// Computes the element-wise loss `|error|^p` with reduction over specified dimensions. + /// + /// Calculates element-wise `|predictions - targets|^p`, then takes the mean + /// over the specified dimensions. Useful for per-sample or per-channel losses (e.g., when + /// working with images). + /// + /// Dimensions can be provided in any order. They are sorted internally and + /// reduced from highest to lowest to ensure indices remain valid. + /// + /// # Arguments + /// + /// * `predictions` - The model's predicted values. + /// * `targets` - The ground truth target values. + /// * `dims` - Dimensions to reduce over. + /// + /// # Returns + /// + /// A tensor with the specified dimensions reduced to size 1. + /// + /// # Example + /// + /// ```ignore + /// // Image tensor: [batch, C, H, W] + /// let l2_loss = LpLossConfig::l2(); + /// + /// // Per-image MSE for PSNR: reduce over C, H, W → [batch, 1, 1, 1] + /// let mse_per_image = l2_loss.forward_reduce_dims(predictions, targets, &[1, 2, 3]); + /// ``` + pub fn forward_reduce_dims( + &self, + predictions: Tensor, + targets: Tensor, + dims: &[usize], + ) -> Tensor { + let error = self.forward_no_reduction(predictions, targets); + + // Sort the dimensions to ascending order + let mut sorted_dims = dims.to_vec(); + sorted_dims.sort(); + + // Reduce over specified dimensions + error.mean_dims(sorted_dims.as_slice()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_lp_loss_l1_constructor() { + let loss_func_l1 = LpLossConfig::l1(); + let loss_func_p1 = LpLossConfig::new(1.0).init(); + assert_eq!(loss_func_l1.p, 1.0); + assert_eq!(loss_func_l1.p, loss_func_p1.p); + } + + #[test] + fn test_lp_loss_l2_constructor() { + let loss_func_l2 = LpLossConfig::l2(); + let loss_func_p2 = LpLossConfig::new(2.0).init(); + assert_eq!(loss_func_l2.p, 2.0); + assert_eq!(loss_func_l2.p, loss_func_p2.p); + } + + #[test] + fn test_lp_loss_l1() { + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + + let targets = Tensor::::from_data( + TensorData::from([[2.0, 1.0], [3.0, 2.0]]), + &device, + ); + + let loss_func = LpLossConfig::l1(); + let loss_no_reduction = + loss_func.forward_no_reduction(predictions.clone(), targets.clone()); + let loss_auto = loss_func.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_sum = loss_func.forward(predictions, targets, Reduction::Sum); + + let expected = TensorData::from([[1.0, 1.0], [0.0, 2.0]]); + loss_no_reduction.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([1.0]); + loss_auto.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([4.0]); + loss_sum.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_lp_loss_l2() { + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + + let targets = Tensor::::from_data( + TensorData::from([[2.0, 1.0], [3.0, 2.0]]), + &device, + ); + + let loss_func = LpLossConfig::l2(); + let loss_no_reduction = + loss_func.forward_no_reduction(predictions.clone(), targets.clone()); + let loss_auto = loss_func.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_sum = loss_func.forward(predictions, targets, Reduction::Sum); + + let expected = TensorData::from([[1.0, 1.0], [0.0, 4.0]]); + loss_no_reduction.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([1.5]); + loss_auto.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([6.0]); + loss_sum.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_lp_loss_p_half() { + // L0.5 quasi-norm: more robust to outliers than L1 + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + + let targets = Tensor::::from_data( + TensorData::from([[2.0, 1.0], [3.0, 0.0]]), + &device, + ); + + let loss_func = LpLossConfig::new(0.5).init(); + let loss_no_reduction = + loss_func.forward_no_reduction(predictions.clone(), targets.clone()); + let loss_auto = loss_func.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_sum = loss_func.forward(predictions, targets, Reduction::Sum); + + // |1-2|^0.5 = 1, |2-1|^0.5 = 1, |3-3|^0.5 = 0, |4-0|^0.5 = 2 + let expected = TensorData::from([[1.0, 1.0], [0.0, 2.0]]); + loss_no_reduction.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([1.0]); + loss_auto.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([4.0]); + loss_sum.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_lp_loss_p3() { + // L3 norm: more sensitive to outliers than L2 + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + + let targets = Tensor::::from_data( + TensorData::from([[2.0, 1.0], [3.0, 2.0]]), + &device, + ); + + let loss_func = LpLossConfig::new(3.0).init(); + let loss_no_reduction = + loss_func.forward_no_reduction(predictions.clone(), targets.clone()); + let loss_auto = loss_func.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_sum = loss_func.forward(predictions, targets, Reduction::Sum); + + // |1-2|^3 = 1, |2-1|^3 = 1, |3-3|^3 = 0, |4-2|^3 = 8 + let expected = TensorData::from([[1.0, 1.0], [0.0, 8.0]]); + loss_no_reduction.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([2.5]); + loss_auto.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([10.0]); + loss_sum.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_lp_loss_zero_error() { + // Test when predictions exactly match targets + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + + let targets = predictions.clone(); + + let loss_func_l1 = LpLossConfig::l1(); + let loss_func_l2 = LpLossConfig::l2(); + + let l1_loss = loss_func_l1.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let l2_loss = loss_func_l2.forward(predictions, targets, Reduction::Auto); + + let expected = TensorData::from([0.0]); + l1_loss.into_data().assert_eq(&expected, false); + l2_loss.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_lp_loss_negative_errors() { + // Test that negative errors are handled correctly (absolute value) + let device = Default::default(); + let predictions = + Tensor::::from_data(TensorData::from([1.0, 2.0, 3.0]), &device); + let targets = + Tensor::::from_data(TensorData::from([3.0, 4.0, 5.0]), &device); + let loss_func_l1 = LpLossConfig::l1(); + let loss_func_p1 = LpLossConfig::new(1.0).init(); + + let loss_no_reduction_l1 = + loss_func_l1.forward_no_reduction(predictions.clone(), targets.clone()); + let loss_no_reduction_p1 = loss_func_p1.forward_no_reduction(predictions, targets); + + // All errors are negative: 1-3=-2, 2-4=-2, 3-5=-2, but |error| = 2 + let expected = TensorData::from([2.0, 2.0, 2.0]); + loss_no_reduction_l1.into_data().assert_eq(&expected, false); + loss_no_reduction_p1.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_lp_loss_3d_tensor() { + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[[0.0, 2.0], [3.0, 5.0]], [[4.0, 6.0], [7.0, 10.0]]]), + &device, + ); + let loss_func_l2 = LpLossConfig::l2(); + let loss_func_p2 = LpLossConfig::new(2.0).init(); + + let loss_l2 = loss_func_l2.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_p2 = loss_func_p2.forward(predictions, targets, Reduction::Auto); + + // Errors: 1, 0, 0, -1, 1, 0, 0, -2 + // Squared: 1, 0, 0, 1, 1, 0, 0, 4 + // Mean: 7/8 = 0.875 + let expected = TensorData::from([0.875]); + loss_l2.into_data().assert_eq(&expected, false); + loss_p2.into_data().assert_eq(&expected, false); + } + + #[test] + #[should_panic(expected = "The order of the norm p must be positive.")] + fn test_lp_loss_negative_p_panics() { + let _ = LpLossConfig::new(-1.0).init(); + } + + #[test] + #[should_panic(expected = "The order of the norm p must be positive.")] + fn test_lp_loss_zero_p_panics() { + let _ = LpLossConfig::new(0.0).init(); + } + + #[test] + fn test_lp_loss_fractional_p() { + // Test p = 1.5 + let device = Default::default(); + let predictions = + Tensor::::from_data(TensorData::from([0.0, 4.0]), &device); + + let targets = Tensor::::from_data(TensorData::from([1.0, 0.0]), &device); + + let loss_func = LpLossConfig::new(1.5).init(); + let loss_no_reduction = loss_func.forward_no_reduction(predictions, targets); + + // |0-1|^1.5 = 1, |4-0|^1.5 = 8 + let expected = TensorData::from([1.0, 8.0]); + loss_no_reduction.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_forward_reduce_dims_single_dim() { + let device = Default::default(); + // Shape: [2, 3] + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[0.0, 2.0, 6.0], [1.0, 5.0, 6.0]]), + &device, + ); + let loss_func_l2 = LpLossConfig::l2(); + let loss_func_p2 = LpLossConfig::new(2.0).init(); + + // Reduce over dim 1 -> should give [2, 1] shape + let loss_l2 = loss_func_l2.forward_reduce_dims(predictions.clone(), targets.clone(), &[1]); + let loss_p2 = loss_func_p2.forward_reduce_dims(predictions, targets, &[1]); + + // Errors row 0: [1, 0, -3] -> squared: [1, 0, 9] -> mean: 10/3 + // Errors row 1: [3, 0, 0] -> squared: [9, 0, 0] -> mean: 3 + let expected = TensorData::from([[10.0 / 3.0], [3.0]]); + loss_l2 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + loss_p2 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_forward_reduce_dims_first_dim() { + let device = Default::default(); + // Shape: [2, 3] + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[0.0, 2.0, 6.0], [1.0, 5.0, 6.0]]), + &device, + ); + let loss_func = LpLossConfig::l2(); + + // Reduce over dim 0 -> should give [1, 3] shape + let loss = loss_func.forward_reduce_dims(predictions, targets, &[0]); + + // Squared errors: [[1, 0, 9], [9, 0, 0]] + // Mean over dim 0: [5, 0, 4.5] + let expected = TensorData::from([[5.0, 0.0, 4.5]]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_forward_reduce_dims_multiple_dims() { + let device = Default::default(); + // Shape: [2, 2, 2] + let predictions = Tensor::::from_data( + TensorData::from([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[[0.0, 2.0], [3.0, 6.0]], [[4.0, 6.0], [7.0, 10.0]]]), + &device, + ); + let loss_func = LpLossConfig::l2(); + + // Reduce over dims 1 and 2 -> should give [2, 1, 1] shape + let loss = loss_func.forward_reduce_dims(predictions, targets, &[1, 2]); + + // Batch 0 errors: [[1, 0], [0, -2]] -> squared: [[1, 0], [0, 4]] -> mean: 5/4 = 1.25 + // Batch 1 errors: [[1, 0], [0, -2]] -> squared: [[1, 0], [0, 4]] -> mean: 5/4 = 1.25 + let expected = TensorData::from([[[1.25]], [[1.25]]]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_forward_reduce_dims_all_dims() { + let device = Default::default(); + // Shape: [2, 2] + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[2.0, 1.0], [3.0, 2.0]]), + &device, + ); + let loss_func = LpLossConfig::l2(); + + // Reduce over all dims -> should give [1, 1] shape + let loss = loss_func.forward_reduce_dims(predictions, targets, &[0, 1]); + + // Errors: [[-1, 1], [0, 2]] -> squared: [[1, 1], [0, 4]] -> mean: 1.5 + let expected = TensorData::from([[1.5]]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_forward_reduce_dims_image_batch() { + // Simulate per-image loss for [batch, C, H, W] tensor (common use case for PSNR) + let device = Default::default(); + // Shape: [2, 1, 2, 2] (batch=2, C=1, H=2, W=2) + let predictions = Tensor::::from_data( + TensorData::from([ + [[[1.0, 2.0], [3.0, 4.0]]], // Image 1 + [[[5.0, 6.0], [7.0, 8.0]]], // Image 2 + ]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([ + [[[0.0, 2.0], [3.0, 6.0]]], // Target 1 + [[[5.0, 5.0], [7.0, 7.0]]], // Target 2 + ]), + &device, + ); + let loss_func = LpLossConfig::l2(); + + // Reduce over C, H, W (dims 1, 2, 3) to get per-image MSE + let loss = loss_func.forward_reduce_dims(predictions, targets, &[1, 2, 3]); + + // Image 1 errors: [[1, 0], [0, -2]] -> squared: [[1, 0], [0, 4]] -> mean: 1.25 + // Image 2 errors: [[0, 1], [0, 1]] -> squared: [[0, 1], [0, 1]] -> mean: 0.5 + let expected = TensorData::from([[[[1.25]]], [[[0.5]]]]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_forward_reduce_dims_with_p1() { + let device = Default::default(); + // Shape: [2, 3] + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[0.0, 5.0, 3.0], [1.0, 5.0, 9.0]]), + &device, + ); + let loss_func = LpLossConfig::l1(); + + // Reduce over dim 1 -> should give [2, 1] shape + let loss = loss_func.forward_reduce_dims(predictions, targets, &[1]); + + // Abs errors row 0: [1, 3, 0] -> mean: 4/3 + // Abs errors row 1: [3, 0, 3] -> mean: 2 + let expected = TensorData::from([[4.0 / 3.0], [2.0]]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_forward_reduce_dims_empty_dims() { + // Reducing over no dimensions should return the unreduced loss + let device = Default::default(); + let predictions = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[0.0, 2.0], [3.0, 6.0]]), + &device, + ); + let loss_func = LpLossConfig::l2(); + let loss_reduce_dims = + loss_func.forward_reduce_dims(predictions.clone(), targets.clone(), &[]); + let loss_no_reduction = loss_func.forward_no_reduction(predictions, targets); + + // Should be equivalent + loss_reduce_dims + .into_data() + .assert_eq(&loss_no_reduction.into_data(), true); + } + + #[test] + fn test_forward_reduce_dims_zero_error() { + let device = Default::default(); + // Shape: [2, 2, 2] + let predictions = Tensor::::from_data( + TensorData::from([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]), + &device, + ); + let targets = predictions.clone(); + let loss_func = LpLossConfig::l2(); + let loss = loss_func.forward_reduce_dims(predictions, targets, &[1, 2]); + + // All zeros, reduced to shape: [2, 1, 1] + let expected = TensorData::from([[[0.0]], [[0.0]]]); + loss.into_data().assert_eq(&expected, false); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mod.rs new file mode 100644 index 0000000..e40aa36 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mod.rs @@ -0,0 +1,23 @@ +mod binary_cross_entropy; +mod cosine_embedding; +mod cross_entropy; +mod ctc; +mod huber; +mod kldiv; +mod lp_loss; +mod mse; +mod poisson; +mod reduction; +mod smooth_l1; + +pub use binary_cross_entropy::*; +pub use cosine_embedding::*; +pub use cross_entropy::*; +pub use ctc::*; +pub use huber::*; +pub use kldiv::*; +pub use lp_loss::*; +pub use mse::*; +pub use poisson::*; +pub use reduction::*; +pub use smooth_l1::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mse.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mse.rs new file mode 100644 index 0000000..a4b8006 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/mse.rs @@ -0,0 +1,93 @@ +use burn_core as burn; + +use crate::loss::reduction::Reduction; + +use burn::module::Module; +use burn::tensor::{Tensor, backend::Backend}; + +/// Calculate the mean squared error loss from the input logits and the targets. +#[derive(Module, Clone, Debug)] +pub struct MseLoss; + +impl Default for MseLoss { + fn default() -> Self { + Self::new() + } +} + +impl MseLoss { + /// Create the criterion. + pub fn new() -> Self { + Self + } + + /// Compute the criterion on the input tensor. + /// + /// # Shapes + /// + /// - logits: [batch_size, num_targets] + /// - targets: [batch_size, num_targets] + pub fn forward( + &self, + logits: Tensor, + targets: Tensor, + reduction: Reduction, + ) -> Tensor { + let tensor = self.forward_no_reduction(logits, targets); + match reduction { + Reduction::Mean | Reduction::Auto => tensor.mean(), + Reduction::Sum => tensor.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + + /// Compute the criterion on the input tensor without reducing. + pub fn forward_no_reduction( + &self, + logits: Tensor, + targets: Tensor, + ) -> Tensor { + logits.sub(targets).square() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn test_mse_loss() { + let device = Default::default(); + let logits = Tensor::::from_data( + TensorData::from([[1.0, 2.0], [3.0, 4.0]]), + &device, + ); + + let targets = Tensor::::from_data( + TensorData::from([[2.0, 1.0], [3.0, 2.0]]), + &device, + ); + + let mse = MseLoss::new(); + let loss_no_reduction = mse.forward_no_reduction(logits.clone(), targets.clone()); + let loss = mse.forward(logits.clone(), targets.clone(), Reduction::Auto); + let loss_sum = mse.forward(logits, targets, Reduction::Sum); + + let expected = TensorData::from([[1.0, 1.0], [0.0, 4.0]]); + loss_no_reduction.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([1.5]); + loss.into_data().assert_eq(&expected, false); + + let expected = TensorData::from([6.0]); + loss_sum.into_data().assert_eq(&expected, false); + } + + #[test] + fn display() { + let loss = MseLoss::new(); + assert_eq!(alloc::format!("{loss}"), "MseLoss"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/poisson.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/poisson.rs new file mode 100644 index 0000000..6d26d33 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/poisson.rs @@ -0,0 +1,417 @@ +use burn_core as burn; +use core::f32::consts::PI; + +use burn::tensor::cast::ToElement; + +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::{config::Config, module::Module}; + +use super::Reduction; + +/// Configuration for creating a [PoissonNllLoss](PoissonNllLoss) instance. +/// +/// This configuration allows customization of the Poisson Negative Log Likelihood (NLL) loss +/// behavior, such as whether the input is in log-space, whether to include the Stirling +/// approximation term, and a small epsilon value to avoid numerical instability. +#[derive(Config, Debug)] +pub struct PoissonNllLossConfig { + /// If `true`, the predictions are expected to be in log-space. + /// + /// When `log_input` is `true`, the loss is computed as: + /// ```text + /// L(predictions, target) = exp(predictions) - target * predictions + /// ``` + /// When `log_input` is `false`, the loss is computed as: + /// ```text + /// L(predictions, target) = predictions - target * log(predictions + eps) + /// ``` + #[config(default = true)] + pub log_input: bool, + /// Whether to compute the full loss, including the Stirling approximation term. + /// + /// When `full` is `true`, the Stirling approximation term is added to the loss: + /// ```text + /// target * log(target) - target + 0.5 * log(2 * PI * target) + /// ``` + #[config(default = false)] + pub full: bool, + /// A small value to avoid evaluation of `log(0)` when `log_input` is `false`. + /// + /// This epsilon value is added to the predictions to ensure numerical stability + /// when computing the logarithm. + #[config(default = 1e-8)] + pub eps: f64, +} + +impl PoissonNllLossConfig { + /// Initializes a [PoissonNllLoss](PoissonNllLoss) instance with the current configuration. + /// + /// # Panics + /// - Panics if `eps` is not a positive number. + pub fn init(&self) -> PoissonNllLoss { + self.assertions(); + PoissonNllLoss { + log_input: self.log_input, + full: self.full, + eps: self.eps, + } + } + + /// Validates the configuration parameters. + /// + /// # Panics + /// - Panics if `eps` is not a positive number. + fn assertions(&self) { + assert!( + self.eps > 0., + "eps for PoissonNllLoss must be a positive number." + ); + } +} + +/// Negative Log Likelihood (NLL) loss with a Poisson distribution assumption for the target. +/// +/// This loss function is used when the target values are assumed to follow a Poisson distribution. +/// The loss is defined as: +/// ```text +/// target ~ Poisson(input) +/// L(predictions, target) = predictions - target * log(predictions) + log(target!) +/// ``` +/// The last term (`log(target!)`) can be omitted or approximated using Stirling's formula. +/// The approximation is applied for `target > 1`, while for `target <= 1`, zeros are added to the loss. +/// +/// For more details, see: +/// +#[derive(Module, Debug, Clone)] +#[module(custom_display)] +pub struct PoissonNllLoss { + /// If `true`, the predictions are expected to be in log-space. + pub log_input: bool, + /// Whether to compute the full loss, including the Stirling approximation term. + pub full: bool, + /// A small value to avoid evaluation of `log(0)` when `log_input` is `false`. + pub eps: f64, +} + +impl ModuleDisplay for PoissonNllLoss { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("log_input", &self.log_input) + .add("full", &self.full) + .add("eps", &self.eps) + .optional() + } +} + +impl PoissonNllLoss { + /// Computes the loss element-wise for the given predictions and targets, then reduces + /// the result to a single loss value. + /// + /// # Arguments + /// - `predictions`: The predicted values. + /// - `targets`: The target values. + /// - `reduction`: The reduction method to apply. `Reduction::Auto` behaves as `Reduction::Mean`. + /// + /// # Shapes + /// - `predictions`: `[...dims]` + /// - `targets`: `[...dims]` + /// - `output`: `[1]` + /// + /// # Panics + /// - Panics if the shapes of `predictions` and `targets` do not match. + /// - Panics if any target value is negative. + /// - Panics if `log_input` is `false` and any prediction value is negative. + pub fn forward( + &self, + predictions: Tensor, + targets: Tensor, + reduction: Reduction, + ) -> Tensor { + let loss = self.forward_no_reduction(predictions, targets); + match reduction { + Reduction::Mean | Reduction::Auto => loss.mean(), + Reduction::Sum => loss.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + + /// Computes the loss element-wise for the given predictions and targets without reduction. + /// + /// # Arguments + /// - `predictions`: The predicted values. + /// - `targets`: The target values. + /// + /// # Shapes + /// - `predictions`: `[...dims]` + /// - `targets`: `[...dims]` + /// - `output`: `[...dims]` + /// + /// # Panics + /// - Panics if the shapes of `predictions` and `targets` do not match. + /// - Panics if any target value is negative. + /// - Panics if `log_input` is `false` and any prediction value is negative. + pub fn forward_no_reduction( + &self, + predictions: Tensor, + targets: Tensor, + ) -> Tensor { + self.assertions(&predictions, &targets); + let mut loss; + if self.log_input { + loss = predictions.clone().exp() - targets.clone() * predictions; + } else { + loss = predictions.clone() - targets.clone() * (predictions + self.eps).log(); + } + if self.full { + let log_stirling_term = targets.clone() * targets.clone().log() - targets.clone() + + (targets.clone() * 2. * PI).log() * 0.5; + loss = loss + + log_stirling_term + .mask_where(targets.clone().lower_equal_elem(1), targets.zeros_like()); + } + loss + } + + /// Validates the input tensors for the loss computation. + /// + /// # Panics + /// - Panics if the shapes of `predictions` and `targets` do not match. + /// - Panics if any target value is negative. + /// - Panics if `log_input` is `false` and any prediction value is negative. + fn assertions( + &self, + predictions: &Tensor, + targets: &Tensor, + ) { + let predictions_dims = predictions.dims(); + let targets_dims = targets.dims(); + assert!( + predictions_dims == targets_dims, + "Shape of targets ({targets_dims:?}) should correspond to outer shape of predictions ({predictions_dims:?})." + ); + assert!( + targets + .clone() + .greater_equal_elem(0.) + .all() + .into_scalar() + .to_bool(), + "All the values of `targets` must be non-negative." + ); + if !self.log_input { + assert!( + predictions + .clone() + .greater_equal_elem(0.) + .all() + .into_scalar() + .to_bool(), + "When `log_input` is `false`, all the values of `predictions` must be non-negative." + ); + } + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::approx_constant)] + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + type TestTensor = Tensor; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_poisson_nll_loss() { + let predictions = TensorData::from([0., 0., -40., 1., 2., 3.]); + let targets = TensorData::from([1., 4.5, 2.5, 0., 0., 2.]); + + let device = Default::default(); + + let predictions = TestTensor::<1>::from_data(predictions, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().init(); + + let loss_sum = poisson.forward(predictions.clone(), targets.clone(), Reduction::Sum); + let loss = poisson.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_no_reduction = poisson.forward_no_reduction(predictions, targets); + + let expected = TensorData::from([1.0000, 1.0000, 100.0000, 2.7183, 7.3891, 14.0855]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([21.0321]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([126.1929]); + loss_sum + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_poisson_nll_loss_no_log_input() { + let predictions = TensorData::from([0.0, 0.5, 1.0, 1.0, 2.71828, 7.38905, 20.0855]); + let targets = TensorData::from([2., 3., 1., 4.5, 0., 0., 2.]); + + let device = Default::default(); + + let predictions = TestTensor::<1>::from_data(predictions, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().with_log_input(false).init(); + + let loss_no_reduction = poisson.forward_no_reduction(predictions.clone(), targets.clone()); + + let expected = TensorData::from([36.84136, 2.579441, 1.0, 1.0, 2.71828, 7.38905, 14.0855]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_poisson_nll_loss_full() { + let predictions = TensorData::from([0., 0., -40., 1., 2., 3.]); + let targets = TensorData::from([1., 4.5, 2.5, 0., 0., 2.]); + + let device = Default::default(); + + let predictions = TestTensor::<1>::from_data(predictions, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().with_full(true).init(); + + let loss_sum = poisson.forward(predictions.clone(), targets.clone(), Reduction::Sum); + let loss = poisson.forward(predictions.clone(), targets.clone(), Reduction::Auto); + let loss_no_reduction = poisson.forward_no_reduction(predictions, targets); + + let expected = TensorData::from([1.0000, 4.9393, 101.1678, 2.7183, 7.3891, 14.7373]); + loss_no_reduction + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([21.9920]); + loss.into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([131.9518]); + loss_sum + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[cfg(feature = "std")] + #[test] + fn test_poisson_nll_loss_gradients() { + type TestAutodiffTensor = Tensor; + + let predictions = TensorData::from([0., 0., -40., 1., 2., 3.]); + let targets = TensorData::from([1., 4.5, 2.5, 0., 0., 2.]); + + let device = Default::default(); + + let predictions1 = TestAutodiffTensor::from_data(predictions, &device).require_grad(); + let predictions2 = predictions1.clone(); + let targets = TestAutodiffTensor::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().with_full(false).init(); + let poisson_full = PoissonNllLossConfig::new().with_full(true).init(); + + let loss_sum = poisson.forward(predictions1.clone(), targets.clone(), Reduction::Sum); + let loss_full_sum = + poisson_full.forward(predictions2.clone(), targets.clone(), Reduction::Sum); + + let grads = loss_sum.backward(); + let grads_full = loss_full_sum.backward(); + + let grads_predictions1 = predictions1.grad(&grads).unwrap(); + let grads_predictions2 = predictions2.grad(&grads_full).unwrap(); + + let expected = TensorData::from([0.0000, -3.5000, -2.5000, 2.7183, 7.3891, 18.0855]); + + grads_predictions1 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + grads_predictions2 + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + #[should_panic = "eps for PoissonNllLoss must be a positive number."] + fn test_negative_eps() { + let _poisson = PoissonNllLossConfig::new().with_eps(0.).init(); + } + + #[test] + #[should_panic = "All the values of `targets` must be non-negative."] + fn test_targets_with_negative_values() { + let predictions = TensorData::from([0., 0., -40., 1., 2., 3., 4.]); + let targets = TensorData::from([1., 4.5, 2.5, 0., 0., 2., -0.42]); + + let device = Default::default(); + + let predictions = TestTensor::<1>::from_data(predictions, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().init(); + + let _loss = poisson.forward(predictions.clone(), targets.clone(), Reduction::Auto); + } + + #[test] + #[should_panic = "Shape of targets"] + fn test_shape_tensors() { + let predictions = TensorData::from([0., 1., 2.]); + let targets = TensorData::from([0., 1.]); + + let device = Default::default(); + + let predictions = TestTensor::<1>::from_data(predictions, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().init(); + + let _loss = poisson.forward_no_reduction(predictions.clone(), targets.clone()); + } + + #[test] + #[should_panic = "When `log_input` is `false`, all the values of `predictions` must be non-negative."] + fn test_exp_predictions_non_negative() { + let predictions = TensorData::from([0.3, -0.1, 0.4]); + let targets = TensorData::from([0., 1., 0.]); + + let device = Default::default(); + + let predictions = TestTensor::<1>::from_data(predictions, &device); + let targets = TestTensor::<1>::from_data(targets, &device); + + let poisson = PoissonNllLossConfig::new().with_log_input(false).init(); + + let _loss = poisson.forward_no_reduction(predictions.clone(), targets.clone()); + } + + #[test] + fn display() { + let config = PoissonNllLossConfig::new(); + let loss = config.init(); + + assert_eq!( + alloc::format!("{loss}"), + "PoissonNllLoss {log_input: true, full: false, eps: 0.00000001}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/reduction.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/reduction.rs new file mode 100644 index 0000000..ddb15fe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/reduction.rs @@ -0,0 +1,19 @@ +use burn_core as burn; + +use burn::config::Config; + +/// The reduction type for the loss. +#[derive(Config, Debug)] +pub enum Reduction { + /// The mean of the losses will be returned. + Mean, + + /// The sum of the losses will be returned. + Sum, + + /// The sum of the losses divided by the batch_size will be returned. + BatchMean, + + /// The mean of the losses will be returned. + Auto, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/smooth_l1.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/smooth_l1.rs new file mode 100644 index 0000000..cd2fc24 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/loss/smooth_l1.rs @@ -0,0 +1,520 @@ +use super::Reduction; +use burn::config::Config; +use burn::module::Module; +use burn::tensor::{Tensor, backend::Backend}; +use burn_core as burn; + +/// Configuration for the [SmoothL1Loss](SmoothL1Loss) module. +/// +/// Smooth L1 loss combines L1 and L2 loss, using L2 loss for small errors (below beta) +/// and L1 loss for large errors (above beta). This makes it less sensitive to outliers +/// than MSE while maintaining smooth gradients near zero. +/// +/// # Example +/// +/// ```ignore +/// use burn_nn::loss::{SmoothL1LossConfig, Reduction}; +/// +/// // Create Smooth L1 loss with default beta=1.0 +/// let smooth_l1 = SmoothL1LossConfig::new().init(); +/// +/// // Create with custom beta +/// let smooth_l1_custom = SmoothL1LossConfig::new().with_beta(0.5).init(); +/// ``` +#[derive(Config, Debug)] +pub struct SmoothL1LossConfig { + /// Specifies the threshold at which to change between L1 and L2 loss. + /// The value must be positive. Default: 1.0 + #[config(default = 1.0)] + pub beta: f32, +} + +impl SmoothL1LossConfig { + /// Initializes a [Smooth L1 Loss](SmoothL1Loss) module. + /// + /// # Panics + /// + /// Panics if `beta <= 0`. + pub fn init(&self) -> SmoothL1Loss { + self.assertions(); + SmoothL1Loss { beta: self.beta } + } + + fn assertions(&self) { + assert!(self.beta > 0.0, "The parameter beta must be positive.") + } +} + +/// Computes the Smooth L1 Loss between predictions and targets. +/// +/// This loss function uses L2 loss for small errors (below beta) and L1 loss for +/// large errors (above beta), providing robustness to outliers while maintaining +/// smooth gradients near |x - y| = 0. +/// +/// # Mathematical Definition +/// +/// For predictions `x` and targets `y`, the element-wise loss is: +/// +/// - L_i = 0.5 * (x_i - y_i)² / beta , if |x_i - y_i| < beta +/// - L_i = |x_i - y_i| - 0.5 * beta , otherwise +/// +/// # Notes +/// +/// Smooth L1 loss is closely related to HuberLoss since it is equivalent to HuberLoss +/// scaled by `1/beta`: +/// `SmoothL1(x, y, beta) = Huber(x, y, beta) / beta` +/// +/// This leads to the following differences: +/// +/// - As beta approaches 0, Smooth L1 loss converges to L1Loss, while HuberLoss converges to 0. +/// When beta = 0, Smooth L1 loss is equivalent to L1 loss. Thus, the `beta` +/// parameter in Burn must be positive. L1Loss should be used for beta = 0. +/// - As beta approaches positive infinity, Smooth L1 loss converges to a constant 0 loss, while +/// HuberLoss converges to L2Loss. +/// +/// # Example +/// +/// ```rust,ignore +/// use burn_nn::loss::{SmoothL1LossConfig, Reduction}; +/// use burn::tensor::Tensor; +/// +/// // Create Smooth L1 loss with the default beta=1.0 +/// let smooth_l1 = SmoothL1LossConfig::new().init(); +/// +/// let predictions: Tensor = /* model output */; +/// let targets: Tensor = /* ground truth */; +/// +/// // Compute element-wise loss without reduction +/// let element_wise = smooth_l1.forward(predictions.clone(), targets.clone()); +/// +/// // Compute loss with mean reduction +/// let loss = smooth_l1.forward_with_reduction(predictions.clone(), targets.clone(), Reduction::Mean); +/// +/// // Per-image loss: reduce over C, H, W → [batch, 1, 1, 1] +/// let loss_per_image = smooth_l1.forward_reduce_dims(predictions, targets, &[1, 2, 3]); +/// ``` +#[derive(Module, Clone, Debug)] +pub struct SmoothL1Loss { + /// Specifies the threshold at which to change between L1 and L2 loss. + /// The value must be positive. Default: 1.0 + pub beta: f32, +} + +impl SmoothL1Loss { + /// Computes the element-wise smooth L1 loss without reduction. + /// + /// # Arguments + /// + /// - `predictions` - The model's predicted values. + /// - `targets` - The ground truth target values. + /// + /// # Returns + /// + /// A tensor of the same shape as the inputs, containing the smooth L1 loss + /// for each element. + /// + /// # Shapes + /// + /// - predictions: `[...dims]` - Any shape + /// - targets: `[...dims]` - Must match predictions shape + /// - output: `[...dims]` - Same shape as inputs + pub fn forward( + &self, + predictions: Tensor, + targets: Tensor, + ) -> Tensor { + let error = predictions.sub(targets); + let abs_error = error.clone().abs(); + + // The L1 case: |error| - 0.5 * beta (when |error| >= beta) + let l1_loss = abs_error.clone().sub_scalar(0.5 * self.beta); + + // The L2 case: 0.5 * (error)^2 / beta (when |error| < beta) + let l2_loss = error.square().mul_scalar(0.5).div_scalar(self.beta); + + let l2_mask = abs_error.lower_elem(self.beta); + l1_loss.mask_where(l2_mask, l2_loss) + } + + /// Computes the smooth L1 loss with reduction. + /// + /// # Arguments + /// + /// - `predictions` - The model's predicted values. + /// - `targets` - The ground truth target values. + /// - `reduction` - Specifies how to reduce the element-wise losses: + /// - `Reduction::Mean` or `Reduction::Auto`: Returns the mean of all element-wise losses. + /// - `Reduction::Sum`: Returns the sum of all element-wise losses. + /// + /// # Returns + /// + /// A scalar tensor containing the reduced loss value. + /// + /// # Shapes + /// + /// - predictions: `[...dims]` - Any shape + /// - targets: `[...dims]` - Must match predictions shape + /// - output: `[1]` - Scalar loss value + pub fn forward_with_reduction( + &self, + predictions: Tensor, + targets: Tensor, + reduction: Reduction, + ) -> Tensor { + let unreduced_loss = self.forward(predictions, targets); + + match reduction { + Reduction::Mean | Reduction::Auto => unreduced_loss.mean(), + Reduction::Sum => unreduced_loss.sum(), + other => panic!("{other:?} reduction is not supported"), + } + } + + /// Computes the smooth L1 loss with reduction over specified dimensions. + /// + /// Calculates element-wise smooth L1 loss, then takes the mean + /// over the specified dimensions. Useful for per-sample or per-channel losses. + /// + /// Dimensions can be provided in any order. They are sorted internally and + /// reduced from highest to lowest to ensure indices remain valid. + /// + /// # Arguments + /// + /// - `predictions` - The model's predicted values. + /// - `targets` - The ground truth target values. + /// - `dims` - Dimensions to reduce over. + /// + /// # Returns + /// + /// A tensor with the specified dimensions reduced to size 1. + /// + /// # Example + /// + /// ```ignore + /// // Consider image tensor with shape [batch, C, H, W] + /// let smooth_l1 = SmoothL1LossConfig::new().init(); + /// + /// // Per-image loss: reduce over C, H, W → [batch, 1, 1, 1] + /// let loss_per_image = smooth_l1.forward_reduce_dims(predictions, targets, &[1, 2, 3]); + /// ``` + pub fn forward_reduce_dims( + &self, + predictions: Tensor, + targets: Tensor, + dims: &[usize], + ) -> Tensor { + let error = self.forward(predictions, targets); + + // Sort the dimensions to ascending order + let mut sorted_dims = dims.to_vec(); + sorted_dims.sort(); + + // Reduce over specified dimensions + error.mean_dims(sorted_dims.as_slice()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + + type FT = FloatElem; + + // ========================================================================= + // Configuration Tests + // ========================================================================= + + #[test] + fn test_smooth_l1_config_default_beta() { + let loss = SmoothL1LossConfig::new().init(); + assert_eq!(loss.beta, 1.0); + } + + #[test] + fn test_smooth_l1_config_custom_beta() { + let loss = SmoothL1LossConfig::new().with_beta(2.5).init(); + assert_eq!(loss.beta, 2.5); + } + + #[test] + #[should_panic(expected = "The parameter beta must be positive")] + fn test_smooth_l1_config_beta_zero_panics() { + SmoothL1LossConfig::new().with_beta(0.0).init(); + } + + #[test] + #[should_panic(expected = "The parameter beta must be positive")] + fn test_smooth_l1_config_beta_negative_panics() { + SmoothL1LossConfig::new().with_beta(-1.0).init(); + } + + // ========================================================================= + // Forward Pass (Element-wise) Tests + // ========================================================================= + + #[test] + fn test_smooth_l1_forward_l2_region() { + // Beta = 1.0, errors = 0.0 and 0.5 (both < beta, use L2 formula) + // L2 formula: 0.5 * error^2 / beta + // error = 0.0 -> loss = 0.5 * 0.0 / 1.0 = 0.0 + // error = 0.5 -> loss = 0.5 * 0.25 / 1.0 = 0.125 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([[0.0_f32, 0.5]]), &device); + let targets = + Tensor::::from_data(TensorData::from([[0.0_f32, 0.0]]), &device); + + let output = loss.forward(predictions, targets); + let expected = TensorData::from([[0.0_f32, 0.125]]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_forward_l1_region() { + // Beta = 1.0, errors = 0.0 and 2.0 (2.0 >= beta, use L1 formula) + // L1 formula: |error| - 0.5 * beta + // L2 formula: 0.5 * (error)^2 / beta + // error = 0.0 -> loss = 0.0 + // error = 2.0 -> loss = 2.0 - 0.5 = 1.5 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([[0.0_f32, 2.0]]), &device); + let targets = + Tensor::::from_data(TensorData::from([[0.0_f32, 0.0]]), &device); + + let output = loss.forward(predictions, targets); + let expected = TensorData::from([[0.0_f32, 1.5]]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_forward_zero_error() { + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([[1.0_f32, 2.0, 3.0]]), &device); + let targets = predictions.clone(); + + let output = loss.forward(predictions, targets); + let expected = TensorData::from([[0.0_f32, 0.0, 0.0]]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_forward_negative_errors() { + // Ensure absolute value is used correctly + // L1 formula: |error| - 0.5 * beta + // L2 formula: 0.5 * (error)^2 / beta + // Beta = 1.0, error = -3.0 (L1: 3.0 - 0.5 = 2.5) + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([-3.0_f32]), &device); + let targets = Tensor::::zeros([1], &device); + + let output = loss.forward(predictions, targets); + let expected = TensorData::from([2.5_f32]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_forward_mixed_regions() { + // Test with errors in both L1 and L2 regions + // Beta = 1.0 + // L1 formula: |error| - 0.5 * beta + // L2 formula: 0.5 * (error)^2 / beta + // error = 0.5 -> L2: 0.5 * 0.25 / 1 = 0.125 + // error = 1.5 -> L1: 1.5 - 0.5 = 1.0 + // error = 3.0 -> L1: 3.0 - 0.5 = 2.5 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([0.5_f32, 1.5, 3.0]), &device); + let targets = Tensor::::zeros([3], &device); + + let output = loss.forward(predictions, targets); + let expected = TensorData::from([0.125_f32, 1.0, 2.5]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_custom_beta_values() { + // Test with beta = 0.5 + // error = 0.25 (< beta): L2 = 0.5 * 0.0625 / 0.5 = 0.0625 + // error = 1.0 (>= beta): L1 = 1.0 - 0.25 = 0.75 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().with_beta(0.5).init(); + + let predictions = + Tensor::::from_data(TensorData::from([0.25_f32, 1.0]), &device); + let targets = Tensor::::zeros([2], &device); + + let output = loss.forward(predictions, targets); + let expected = TensorData::from([0.0625_f32, 0.75]); + output.into_data().assert_eq(&expected, false); + } + + // ========================================================================= + // forward_with_reduction Tests + // ========================================================================= + + #[test] + fn test_smooth_l1_reduction_mean() { + // Errors: 0.5 (L2: 0.125), 2.0 (L1: 1.5) + // Mean: (0.125 + 1.5) / 2 = 0.8125 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([[0.5_f32, 2.0]]), &device); + let targets = + Tensor::::from_data(TensorData::from([[0.0_f32, 0.0]]), &device); + + let output = loss.forward_with_reduction(predictions, targets, Reduction::Mean); + let expected = TensorData::from([0.8125_f32]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_reduction_sum() { + // Errors: 0.5 (L2: 0.125), 2.0 (L1: 1.5) + // Sum: 1.625 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = + Tensor::::from_data(TensorData::from([[0.5_f32, 2.0]]), &device); + let targets = + Tensor::::from_data(TensorData::from([[0.0_f32, 0.0]]), &device); + + let output = loss.forward_with_reduction(predictions, targets, Reduction::Sum); + let expected = TensorData::from([1.625_f32]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_reduction_auto_equals_mean() { + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = Tensor::::from_data(TensorData::from([2.0_f32]), &device); + let targets = Tensor::::zeros([1], &device); + + let mean_out = + loss.forward_with_reduction(predictions.clone(), targets.clone(), Reduction::Mean); + let auto_out = loss.forward_with_reduction(predictions, targets, Reduction::Auto); + + mean_out.into_data().assert_eq(&auto_out.into_data(), false); + } + + // ========================================================================= + // Dimension Reduction Tests + // ========================================================================= + + #[test] + fn test_smooth_l1_forward_reduce_dims_single_dim() { + // Beta = 2.0 + // L1 formula: |error| - 0.5 * beta + // L2 formula: 0.5 * (error)^2 / beta + // Row 0: errors [0.0, 1.0, 4.0] + // error = 0.0 -> L2: 0.0 + // error = 1.0 -> L2: 0.5 * 1.0 / 2.0 = 0.25 + // error = 4.0 -> L1: 4.0 - 1.0 = 3.0 + // Mean = 3.25 / 3 = 1.083333... + // Row 1: errors [0.0, 0.0, 0.0] -> Mean = 0.0 + let device = Default::default(); + let loss = SmoothL1LossConfig::new().with_beta(2.0).init(); + + let predictions = Tensor::::from_data( + TensorData::from([[0.0_f32, 1.0, 4.0], [5.0_f32, 5.0, 5.0]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[0.0_f32, 0.0, 0.0], [5.0_f32, 5.0, 5.0]]), + &device, + ); + + let output = loss.forward_reduce_dims(predictions, targets, &[1]); + let expected = TensorData::from([[3.25_f32 / 3.0], [0.0]]); // 3.25/3 = 1.0833... + output + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_smooth_l1_forward_reduce_dims_image_batch() { + // Simulate per-image Smooth L1 loss for [batch, C, H, W] tensor + // (common in object detection like Fast R-CNN) + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); // beta = 1.0 + + // Shape: [2, 1, 2, 2] (batch=2, C=1, H=2, W=2) + let predictions = Tensor::::from_data( + TensorData::from([ + [[[0.5_f32, 2.0], [0.0, 3.0]]], // Image 1 + [[[1.0_f32, 0.0], [0.5, 1.5]]], // Image 2 + ]), + &device, + ); + let targets = Tensor::::zeros([2, 1, 2, 2], &device); + + // Reduce over C, H, W (dims 1, 2, 3) to get per-image loss + let output = loss.forward_reduce_dims(predictions, targets, &[1, 2, 3]); + + // Image 1: losses [[0.125, 1.5], [0.0, 2.5]] -> mean: 4.125 / 4 = 1.03125 + // Image 2: losses [[0.5, 0.0], [0.125, 1.0]] -> mean: 1.625 / 4 = 0.40625 + let expected = TensorData::from([[[[1.03125_f32]]], [[[0.40625_f32]]]]); + output.into_data().assert_eq(&expected, false); + } + + #[test] + fn test_smooth_l1_forward_reduce_dims_unsorted() { + // Test that unsorted dimensions are handled correctly (sorted internally) + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = Tensor::::from_data( + TensorData::from([[[1.0_f32, 2.0], [3.0, 4.0]], [[5.0_f32, 6.0], [7.0, 8.0]]]), + &device, + ); + let targets = Tensor::::zeros([2, 2, 2], &device); + + // Pass dims in reverse order + let output = loss.forward_reduce_dims(predictions.clone(), targets.clone(), &[2, 1]); + let expected_output = loss.forward_reduce_dims(predictions, targets, &[1, 2]); + + output + .into_data() + .assert_eq(&expected_output.into_data(), false); + } + + #[test] + fn test_smooth_l1_forward_reduce_dims_empty_dims() { + // Reducing over no dimensions should return the unreduced loss + let device = Default::default(); + let loss = SmoothL1LossConfig::new().init(); + + let predictions = Tensor::::from_data( + TensorData::from([[0.5_f32, 2.0], [0.0, 3.0]]), + &device, + ); + let targets = Tensor::::zeros([2, 2], &device); + + let loss_reduce_dims = loss.forward_reduce_dims(predictions.clone(), targets.clone(), &[]); + let loss_no_reduction = loss.forward(predictions, targets); + + loss_reduce_dims + .into_data() + .assert_eq(&loss_no_reduction.into_data(), false); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/cross_attention.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/cross_attention.rs new file mode 100644 index 0000000..d3c9358 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/cross_attention.rs @@ -0,0 +1,621 @@ +//! Cross-Attention Module for Burn +//! +//! Features: +//! - Asymmetric Input Shapes (Query vs Context) +//! - Grouped Query Attention (GQA) & Multi-Query Attention (MQA) support +//! - Quantization-Safe Masking (min_float) +//! - Sparse-Ready (quiet_softmax) +//! - KV Caching for Streaming Inference + +use crate::cache::TensorCache; +use crate::modules::{Linear, LinearConfig}; +use crate::{Dropout, DropoutConfig}; +use burn_core as burn; + +use burn::{ + config::Config, + module::{Initializer, Module}, + tensor::{ + Bool, Tensor, + activation::{quiet_softmax, softmax}, + backend::Backend, + }, +}; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +#[derive(Config, Debug)] +/// Configuration to create a [CrossAttention](CrossAttention) layer using the [init function](CrossAttentionConfig::init). +pub struct CrossAttentionConfig { + /// Dimension of the Query (e.g., Decoder state). + pub d_model: usize, + /// Dimension of the Context (e.g., Encoder audio embeddings). + pub d_context: usize, + /// Number of heads for the Query. + pub n_heads: usize, + /// Number of heads for Key/Value (Set to 1 for MQA, set to n_heads for MHA). + pub n_heads_kv: usize, + /// Dimension of a single head. + pub d_head: usize, + /// Dropout rate. + #[config(default = 0.1)] + pub dropout: f64, + /// Masking value. Use -1.0e4 for f16/bf16 safety. + #[config(default = -1.0e4)] + pub min_float: f64, + /// Use quiet_softmax to allow zero-attention (good for sparse/quantized models). + #[config(default = false)] + pub quiet_softmax: bool, +} + +#[derive(Module, Debug)] +/// The Cross attention module +/// +/// # Params +/// +/// - `query`: [`Linear`] layer with `d_model` input and output features. +/// - `key`: [`Linear`] layer with `d_model` input and output features. +/// - `value`: [`Linear`] layer with `d_model` input and output features. +/// - `output`: [`Linear`] layer with `d_model` input and output features. +/// +/// Should be created with [CrossAttentionConfig]. +pub struct CrossAttention { + query: Linear, + key: Linear, + value: Linear, + output: Linear, + dropout: Dropout, + + n_heads: usize, + n_heads_kv: usize, + d_head: usize, + scale: f64, + min_float: f64, + quiet_softmax: bool, +} + +/// Cache for the [Cross Attention](CrossAttention) layer. +/// +/// To be used during inference when context is constant. +pub struct CrossAttentionCache { + /// Cached key tensor. + pub k: TensorCache, + /// Cached value tensor. + pub v: TensorCache, +} + +impl CrossAttentionCache { + /// Create a new empty cache. + pub fn new() -> Self { + Self { + k: TensorCache::empty(), + v: TensorCache::empty(), + } + } +} + +impl Default for CrossAttentionCache { + fn default() -> Self { + Self::new() + } +} + +impl CrossAttentionConfig { + /// Initializes a new cross-attention module. + /// + /// # Arguments + /// + /// * `device` - The device on which to initialize the module. + /// + /// # Returns + /// + /// A new [CrossAttention] module. + pub fn init(&self, device: &B::Device) -> CrossAttention { + // Safety Rail for GQA + assert_eq!( + self.n_heads % self.n_heads_kv, + 0, + "Query heads must be divisible by KV heads" + ); + + let init_linear = |in_dim, out_dim| { + LinearConfig::new(in_dim, out_dim) + .with_initializer(Initializer::KaimingUniform { + gain: 1.0 / (self.d_head as f64).sqrt(), + fan_out_only: false, + }) + .init(device) + }; + + CrossAttention { + // ADVICE: Asymmetric Projections + query: init_linear(self.d_model, self.n_heads * self.d_head), + key: init_linear(self.d_context, self.n_heads_kv * self.d_head), + value: init_linear(self.d_context, self.n_heads_kv * self.d_head), + output: init_linear(self.n_heads * self.d_head, self.d_model), + + dropout: DropoutConfig::new(self.dropout).init(), + n_heads: self.n_heads, + n_heads_kv: self.n_heads_kv, + d_head: self.d_head, + scale: (self.d_head as f64).sqrt().recip(), + min_float: self.min_float, + quiet_softmax: self.quiet_softmax, + } + } +} + +impl CrossAttention { + /// Applies cross-attention to query using context as key and value. + /// + /// # Arguments + /// + /// * `query` - Query tensor of shape `[batch, seq_len_query, d_model]`. + /// * `context` - Context tensor of shape `[batch, seq_len_context, d_context]`. + /// * `mask` - Optional attention mask of shape `[batch, seq_len_context]` where `true` indicates positions to mask. + /// + /// # Returns + /// + /// Output tensor of shape `[batch, seq_len_query, d_model]`. + pub fn forward( + &self, + query: Tensor, + context: Tensor, + mask: Option>, + ) -> Tensor { + let [batch, l_q, _] = query.dims(); + let [_, l_k, _] = context.dims(); + + // 1. Projections + let q = self.query.forward(query); + let k = self.key.forward(context.clone()); + let v = self.value.forward(context); + + // 2. Reshape Heads + // Q: [Batch, Heads, L_q, D_head] + let q = q + .reshape([batch, l_q, self.n_heads, self.d_head]) + .swap_dims(1, 2); + + // K, V: [Batch, Heads_KV, L_k, D_head] + let k = k + .reshape([batch, l_k, self.n_heads_kv, self.d_head]) + .swap_dims(1, 2); + let v = v + .reshape([batch, l_k, self.n_heads_kv, self.d_head]) + .swap_dims(1, 2); + + // 3. GQA Expansion + // ADVICE: Handle GQA by repeating KV heads to match Query heads + let (k, v) = if self.n_heads != self.n_heads_kv { + let n_rep = self.n_heads / self.n_heads_kv; + (self.repeat_kv(k, n_rep), self.repeat_kv(v, n_rep)) + } else { + (k, v) + }; + + // 4. Score Calculation + let scores = q.matmul(k.transpose()) * self.scale; + + // 5. Masking + // ADVICE: Use min_float for F16/FP8 safety + let scores = if let Some(mask) = mask { + let mask = mask.reshape([batch, 1, 1, l_k]); + scores.mask_fill(mask, self.min_float) + } else { + scores + }; + + // 6. Softmax + // ADVICE: Optional Quiet Softmax for sparse networks + let weights = if self.quiet_softmax { + quiet_softmax(scores, 3) + } else { + softmax(scores, 3) + }; + + let weights = self.dropout.forward(weights); + + // 7. Aggregate & Output + let output = weights.matmul(v); + let output = output + .swap_dims(1, 2) + .reshape([batch, l_q, self.n_heads * self.d_head]); + + self.output.forward(output) + } + + /// Applies cross-attention to query using context as key and value. + /// + /// This method uses a cache to avoid recomputing key and value tensors when the context is the same. + /// + /// # Arguments + /// + /// * `query` - Query tensor of shape `[batch, seq_len_query, d_model]`. + /// * `context` - Context tensor of shape `[batch, seq_len_context, d_context]`. + /// * `mask` - Optional attention mask of shape `[batch, seq_len_context]` where `true` indicates positions to mask. + /// * `cache` - The cache to use. + /// + /// # Returns + /// + /// Output tensor of shape `[batch, seq_len_query, d_model]`. + pub fn forward_cache( + &self, + query: Tensor, + context: Tensor, + mask: Option>, + cache: &mut CrossAttentionCache, + ) -> Tensor { + let [batch, l_q, _] = query.dims(); + + // 1. Projections + let q = self.query.forward(query); + + let k_compute = |context: Tensor| { + let [batch, l_k, _] = context.dims(); + self.key + .forward(context) + .reshape([batch, l_k, self.n_heads_kv, self.d_head]) + .swap_dims(1, 2) + }; + let v_compute = |context: Tensor| { + let [batch, l_k, _] = context.dims(); + self.value + .forward(context) + .reshape([batch, l_k, self.n_heads_kv, self.d_head]) + .swap_dims(1, 2) + }; + + let k = cache.k.forward_full(context.clone(), k_compute); + let v = cache.v.forward_full(context, v_compute); + + let [_, _, l_k, _] = k.dims(); + + // 2. Reshape Heads + // Q: [Batch, Heads, L_q, D_head] + let q = q + .reshape([batch, l_q, self.n_heads, self.d_head]) + .swap_dims(1, 2); + + // K, V are already in their correct shape from k_compute and v_compute + + // 3. GQA Expansion + // ADVICE: Handle GQA by repeating KV heads to match Query heads + let (k, v) = if self.n_heads != self.n_heads_kv { + let n_rep = self.n_heads / self.n_heads_kv; + (self.repeat_kv(k, n_rep), self.repeat_kv(v, n_rep)) + } else { + (k, v) + }; + + // 4. Score Calculation + let scores = q.matmul(k.transpose()) * self.scale; + + // 5. Masking + // ADVICE: Use min_float for F16/FP8 safety + let scores = if let Some(mask) = mask { + let mask = mask.reshape([batch, 1, 1, l_k]); + scores.mask_fill(mask, self.min_float) + } else { + scores + }; + + // 6. Softmax + // ADVICE: Optional Quiet Softmax for sparse networks + let weights = if self.quiet_softmax { + quiet_softmax(scores, 3) + } else { + softmax(scores, 3) + }; + + let weights = self.dropout.forward(weights); + + // 7. Aggregate & Output + let output = weights.matmul(v); + let output = output + .swap_dims(1, 2) + .reshape([batch, l_q, self.n_heads * self.d_head]); + + self.output.forward(output) + } + + /// Helper for Grouped Query Attention + fn repeat_kv(&self, x: Tensor, n_rep: usize) -> Tensor { + let [b, h, l, d] = x.dims(); + x.reshape([b, h, 1, l, d]) + .expand([b, h, n_rep, l, d]) + .reshape([b, h * n_rep, l, d]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::{Distribution, Int, Shape, Tensor, Tolerance}; + + #[test] + fn test_cross_attention_mha_shapes() { + let [ + batch_size, + seq_len_query, + seq_len_context, + d_model, + d_context, + n_heads, + d_head, + ] = [7, 13, 15, 32, 40, 4, 8]; + let device = Default::default(); + let config = CrossAttentionConfig { + d_model, + d_context, + n_heads, + n_heads_kv: n_heads, // MHA case + d_head, + dropout: 0.1, + min_float: -1.0e4, + quiet_softmax: false, + }; + let cross_attn = config.init::(&device); + + let query = Tensor::random( + [batch_size, seq_len_query, d_model], + Distribution::Default, + &device, + ); + let context = Tensor::random( + [batch_size, seq_len_context, d_context], + Distribution::Default, + &device, + ); + + let output = cross_attn.forward(query, context, None); + + assert_eq!( + output.shape(), + Shape::new([batch_size, seq_len_query, d_model]), + "Output should have the correct shape", + ); + } + + #[test] + fn test_cross_attention_gqa_shapes() { + let [ + batch_size, + seq_len_query, + seq_len_context, + d_model, + d_context, + n_heads, + n_heads_kv, + d_head, + ] = [7, 13, 15, 32, 40, 4, 2, 8]; + let device = Default::default(); + let config = CrossAttentionConfig { + d_model, + d_context, + n_heads, + n_heads_kv, // GQA case + d_head, + dropout: 0.1, + min_float: -1.0e4, + quiet_softmax: false, + }; + let cross_attn = config.init::(&device); + + let query = Tensor::random( + [batch_size, seq_len_query, d_model], + Distribution::Default, + &device, + ); + let context = Tensor::random( + [batch_size, seq_len_context, d_context], + Distribution::Default, + &device, + ); + + let output = cross_attn.forward(query, context, None); + + assert_eq!( + output.shape(), + Shape::new([batch_size, seq_len_query, d_model]), + "Output should have the correct shape", + ); + } + + #[test] + fn test_cross_attention_mqa_shapes() { + let [ + batch_size, + seq_len_query, + seq_len_context, + d_model, + d_context, + n_heads, + d_head, + ] = [7, 13, 15, 32, 40, 4, 8]; + let device = Default::default(); + let config = CrossAttentionConfig { + d_model, + d_context, + n_heads, + n_heads_kv: 1, // MQA case + d_head, + dropout: 0.1, + min_float: -1.0e4, + quiet_softmax: false, + }; + let cross_attn = config.init::(&device); + + let query = Tensor::random( + [batch_size, seq_len_query, d_model], + Distribution::Default, + &device, + ); + let context = Tensor::random( + [batch_size, seq_len_context, d_context], + Distribution::Default, + &device, + ); + + let output = cross_attn.forward(query, context, None); + + assert_eq!( + output.shape(), + Shape::new([batch_size, seq_len_query, d_model]), + "Output should have the correct shape", + ); + } + + #[test] + fn test_cross_attention_mask() { + let [ + batch_size, + seq_len_query, + seq_len_context, + d_model, + d_context, + n_heads, + d_head, + ] = [3, 6, 8, 12, 16, 4, 3]; + let num_padded = 2; + let device = Default::default(); + let config = CrossAttentionConfig { + d_model, + d_context, + n_heads, + n_heads_kv: n_heads, + d_head, + dropout: 0.0, // No dropout for deterministic test + min_float: -1.0e4, + quiet_softmax: false, + }; + let cross_attn = config.init::(&device); + + // Create a padding mask for the context + let mut mask: Tensor = + Tensor::zeros([batch_size, seq_len_context], &device); + mask = mask.slice_assign( + [0..batch_size, seq_len_context - num_padded..seq_len_context], + Tensor::ones([batch_size, num_padded], &device), + ); + let mask_bool = mask.equal_elem(1); + + let query = Tensor::::random( + [batch_size, seq_len_query, d_model], + Distribution::Default, + &device, + ); + + let context_1 = Tensor::::random( + [batch_size, seq_len_context, d_context], + Distribution::Default, + &device, + ); + + // Change the padded part of the context tensor + let context_2 = context_1.clone().slice_assign( + [ + 0..batch_size, + seq_len_context - num_padded..seq_len_context, + 0..d_context, + ], + Tensor::random( + [batch_size, num_padded, d_context], + Distribution::Default, + &device, + ), + ); + + // The outputs should be the same since the changed part is masked. + let output_1 = cross_attn.forward(query.clone(), context_1, Some(mask_bool.clone())); + let output_2 = cross_attn.forward(query, context_2, Some(mask_bool)); + + output_1 + .into_data() + .assert_approx_eq(&output_2.into_data(), Tolerance::::default()); + } + + #[test] + #[should_panic] + fn test_gqa_panic_if_n_heads_not_divisible_by_n_heads_kv() { + let device = Default::default(); + let config = CrossAttentionConfig { + d_model: 32, + d_context: 32, + n_heads: 5, + n_heads_kv: 2, + d_head: 8, + dropout: 0.1, + min_float: -1.0e4, + quiet_softmax: false, + }; + config.init::(&device); + } + + #[test] + fn test_cross_attention_cache() { + let [ + batch_size, + seq_len_query, + seq_len_context, + d_model, + d_context, + n_heads, + d_head, + ] = [3, 6, 8, 12, 16, 4, 3]; + let device = Default::default(); + let config = CrossAttentionConfig { + d_model, + d_context, + n_heads, + n_heads_kv: n_heads, + d_head, + dropout: 0.0, // No dropout for deterministic test + min_float: -1.0e4, + quiet_softmax: false, + }; + let cross_attn = config.init::(&device); + + let query1 = Tensor::::random( + [batch_size, seq_len_query, d_model], + Distribution::Default, + &device, + ); + let context = Tensor::::random( + [batch_size, seq_len_context, d_context], + Distribution::Default, + &device, + ); + + // First forward pass, no cache + let output1 = cross_attn.forward(query1.clone(), context.clone(), None); + + // Second forward pass with cache + let mut cache = CrossAttentionCache::new(); + let output2 = cross_attn.forward_cache(query1.clone(), context.clone(), None, &mut cache); + + // The two outputs should be identical + output1 + .into_data() + .assert_approx_eq(&output2.into_data(), Tolerance::::default()); + + // Third forward pass with different query, but same context and cache + let query2 = Tensor::::random( + [batch_size, seq_len_query, d_model], + Distribution::Default, + &device, + ); + let output3 = cross_attn.forward_cache(query2.clone(), context.clone(), None, &mut cache); + + // For control, do a forward pass without cache with query2 + let output4 = cross_attn.forward(query2.clone(), context.clone(), None); + + // output3 and output4 should be identical + output3 + .into_data() + .assert_approx_eq(&output4.into_data(), Tolerance::::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mask.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mask.rs new file mode 100644 index 0000000..0a12714 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mask.rs @@ -0,0 +1,161 @@ +use burn_core as burn; +use burn_core::config::Config; + +use alloc::vec::Vec; +use burn::tensor::ops::IntElem; + +use burn::tensor::{Bool, ElementConversion, Int, Shape, Tensor, TensorData, backend::Backend}; + +/// Generate an autoregressive attention mask. +/// +/// The mask can be used in Transformer modules to train models to generate tensors sequentially. +pub fn generate_autoregressive_mask( + batch_size: usize, + seq_length: usize, + device: &B::Device, +) -> Tensor { + let mask = Tensor::::tril_mask([seq_length, seq_length], 0, device); + mask.expand([batch_size, seq_length, seq_length]) +} + +/// Generate a padding attention mask. +pub struct GeneratePaddingMask { + /// The generated tensor. + pub tensor: Tensor, + + /// The generated mask. + pub mask: Tensor, +} + +/// Defines an enumeration to specify sequence length options for padding +#[derive(Config, Debug, Copy)] +pub enum SeqLengthOption { + /// No maximum length; use the longest sequence + NoMax, + /// Maximum length specified, truncate if necessary + Max(usize), + /// Fixed length, pad or truncate to this exact length + Fixed(usize), +} + +impl From> for SeqLengthOption { + fn from(val: Option) -> Self { + match val { + Some(max) => SeqLengthOption::Max(max), + None => SeqLengthOption::NoMax, + } + } +} + +/// Generates a padding attention mask for a batch of token sequences. +/// +/// # Arguments +/// +/// * `pad_token` - The token ID used for padding +/// * `tokens_list` - Vector of token sequences (each sequence is a vector of token IDs) +/// * `seq_length` - Sequence length option (NoMax, Max, or Fixed) +/// * `device` - The device for tensor operations +/// +/// # Returns +/// +/// A `GeneratePaddingMask` containing the padded tensor and corresponding mask +pub fn generate_padding_mask( + pad_token: usize, + tokens_list: Vec>, + seq_length: impl Into, + device: &B::Device, +) -> GeneratePaddingMask { + let tokens_max = || { + tokens_list + .iter() + .map(|tokens| tokens.len()) + .max() + .unwrap_or(1) + }; + + let size = match seq_length.into() { + SeqLengthOption::NoMax => tokens_max(), + SeqLengthOption::Max(max) => usize::min(tokens_max(), max), + SeqLengthOption::Fixed(limit) => limit, + }; + let batch_size = tokens_list.len(); + + let mut tensor = Tensor::zeros([batch_size, size], device); + tensor = tensor.add_scalar(pad_token as i64); + + for (index, tokens) in tokens_list.into_iter().enumerate() { + let seq_length = tokens.len().min(size); + tensor = tensor.slice_assign( + [index..index + 1, 0..seq_length], + Tensor::from_data( + TensorData::new( + tokens + .into_iter() + .take(size) + .map(|e| (e as i64).elem::>()) + .collect(), + Shape::new([1, seq_length]), + ), + device, + ), + ); + } + + let mask = tensor.clone().equal_elem(pad_token as i64); + + GeneratePaddingMask { tensor, mask } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use alloc::vec; + use burn::tensor::TensorData; + + #[test] + fn test_generate_autoregressive_mask() { + let device = ::Device::default(); + + let mask = generate_autoregressive_mask::(2, 3, &device); + + mask.into_data().assert_eq( + &TensorData::from([ + [ + [false, true, true], + [false, false, true], + [false, false, false], + ], + [ + [false, true, true], + [false, false, true], + [false, false, false], + ], + ]), + false, + ); + } + + #[test] + fn test_generate_padding_mask() { + let device = ::Device::default(); + let tokens = vec![ + vec![3, 3, 3], + vec![3, 3, 3], + vec![3, 3, 3, 4], + vec![3, 3, 3, 4, 10, 15], + ]; + + let mask = generate_padding_mask::(0, tokens, None, &device); + + mask.mask.into_data().assert_eq( + &TensorData::from([ + [false, false, false, true, true, true], + [false, false, false, true, true, true], + [false, false, false, false, true, true], + [false, false, false, false, false, false], + ]), + false, + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mha.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mha.rs new file mode 100644 index 0000000..abe77a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mha.rs @@ -0,0 +1,531 @@ +use burn_core as burn; + +use crate::activation::Gelu; +use crate::cache::TensorCache; +use crate::{Dropout, DropoutConfig, Linear, LinearConfig}; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::{Bool, Tensor, backend::Backend}; + +use burn::tensor::activation::{quiet_softmax, softmax}; +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Configuration to create a [Multi Head Attention](MultiHeadAttention) layer using the [init function](MultiHeadAttentionConfig::init). +#[derive(Config, Debug)] +pub struct MultiHeadAttentionConfig { + /// The size of each linear layer. + pub d_model: usize, + /// The number of heads. + pub n_heads: usize, + /// The dropout rate. Default: 0.1 + #[config(default = 0.1)] + pub dropout: f64, + /// The minimum value a float can take. Default: -1.0e4 + /// This is used to mask attention scores before calculating attention weights. + /// A value too low might result in NaN. + #[config(default = -1.0e4)] + pub min_float: f64, + /// Use "quiet softmax" instead of regular softmax. + /// + /// - Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head). + /// - Usage may reduce the entropy of weights in the model, enhancing quantization and compression. + /// + /// Reference: + #[config(default = false)] + pub quiet_softmax: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// The multihead attention module as describe in the paper [Attention Is All You Need](https://arxiv.org/abs/1706.03762). +/// +/// # Params +/// +/// - `query`: [`Linear`] layer with `d_model` input and output features. +/// - `key`: [`Linear`] layer with `d_model` input and output features. +/// - `value`: [`Linear`] layer with `d_model` input and output features. +/// - `output`: [`Linear`] layer with `d_model` input and output features. +/// +/// Should be created with [MultiHeadAttentionConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct MultiHeadAttention { + /// Linear layer to transform the input features into the query space. + pub query: Linear, + /// Linear layer to transform the input features into the key space. + pub key: Linear, + /// Linear layer to transform the input features into the value space. + pub value: Linear, + /// Linear layer to transform the output features back to the original space. + pub output: Linear, + /// Dropout layer. + pub dropout: Dropout, + /// Activation function. + pub activation: Gelu, + /// The size of each linear layer. + pub d_model: usize, + /// The number of heads. + pub n_heads: usize, + /// Size of the key and query vectors. + pub d_k: usize, + /// Minimum value a float can take. + pub min_float: f64, + /// Use "quiet softmax" instead of regular softmax. + pub quiet_softmax: bool, +} + +impl ModuleDisplay for MultiHeadAttention { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("d_model", &self.d_model) + .add("n_heads", &self.n_heads) + .add("d_k", &self.d_k) + .add("dropout", &self.dropout.prob) + .add("min_float", &self.min_float) + .add("quiet_softmax", &self.quiet_softmax) + .optional() + } +} + +/// [Multihead attention](MultiHeadAttention) forward pass input argument. +#[derive(Debug, Clone)] +pub struct MhaInput { + /// Shape `[batch_size, seq_length_1, d_model]` + query: Tensor, + /// Shape `[batch_size, seq_length_2, d_model]` + key: Tensor, + /// Shape `[batch_size, seq_length_2, d_model]` + value: Tensor, + mask_pad: Option>, + mask_attn: Option>, +} + +impl MultiHeadAttentionConfig { + /// Initialize a new [multihead attention](MultiHeadAttention) module. + pub fn init(&self, device: &B::Device) -> MultiHeadAttention { + let linear = |config: &Self| { + LinearConfig::new(config.d_model, config.d_model) + .with_initializer(self.initializer.clone()) + .init(device) + }; + + MultiHeadAttention { + query: linear(self), + key: linear(self), + value: linear(self), + output: linear(self), + dropout: DropoutConfig::new(self.dropout).init(), + activation: Gelu::new(), + n_heads: self.n_heads, + d_k: self.d_model / self.n_heads, + min_float: self.min_float, + quiet_softmax: self.quiet_softmax, + d_model: self.d_model, + } + } +} + +impl MhaInput { + /// Create a [multihead attention](MultiHeadAttention) input argument + /// by setting the query, key and value to the given tensor. + /// + /// # Shape + /// - tensor: `[batch_size, seq_length, d_model]` + pub fn self_attn(tensor: Tensor) -> Self { + Self { + query: tensor.clone(), + key: tensor.clone(), + value: tensor, + mask_pad: None, + mask_attn: None, + } + } + + /// Create a [multihead attention](MultiHeadAttention) input argument. + pub fn new(query: Tensor, key: Tensor, value: Tensor) -> Self { + Self { + query, + key, + value, + mask_pad: None, + mask_attn: None, + } + } + + /// Register the padding mask. + pub fn mask_pad(mut self, mask_pad: Tensor) -> Self { + self.mask_pad = Some(mask_pad); + self + } + + /// Register the attention mask. + pub fn mask_attn(mut self, mask_attn: Tensor) -> Self { + self.mask_attn = Some(mask_attn); + self + } +} + +/// [Multihead attention](MultiHeadAttention) outputs. +#[derive(Debug, Clone)] +pub struct MhaOutput { + /// The attention weights `[batch_size, n_heads, seq_length_1, seq_length_2]`. + pub weights: Tensor, + /// The context tensor `[batch_size, seq_length_1, d_model]`. + pub context: Tensor, +} + +impl MultiHeadAttention { + /// Applies the forward pass on the input tensors. + /// + /// See [MultiHeadAttention](MultiHeadAttention) for more information. + /// + /// # Shapes + /// + /// - query: `[batch_size, seq_length_1, d_model]` + /// - key: `[batch_size, seq_length_2, d_model]` + /// - value: `[batch_size, seq_length_2, d_model]` + /// - output: `[batch_size, seq_length_1, d_model]` + pub fn forward(&self, input: MhaInput) -> MhaOutput { + let [batch_size, seq_length_1, d_model] = input.query.dims(); + + let query = self.attention_linear(input.query, &self.query); + let key = self.attention_linear(input.key, &self.key); + let value = self.attention_linear(input.value, &self.value); + + let attn_scores = self.attn_scores(query, key); + let weights = self.attn_weights(attn_scores, input.mask_pad, input.mask_attn); + + let context = weights.clone().matmul(value); + let context = context + .swap_dims(1, 2) + .reshape([batch_size, seq_length_1, d_model]); + let context = self.output.forward(context); + + MhaOutput { weights, context } + } + + /// Applies the forward pass using a cache. + /// + /// # Shapes + /// + /// - query: `[batch_size, seq_length_1, d_model]` + /// - key: `[batch_size, seq_length_2, d_model]` + /// - value: `[batch_size, seq_length_2, d_model]` + /// - output: `[batch_size, seq_length_1, d_model]` + pub fn forward_cache(&self, input: MhaInput, cache: &mut MhaCache) -> MhaOutput { + let [batch_size, seq_length_1, d_model] = input.query.dims(); + + let query = cache + .query + .forward(input.query, |t| self.attention_linear(t, &self.query)); + let key = cache + .key + .forward(input.key, |t| self.attention_linear(t, &self.key)); + let value = cache + .value + .forward(input.value, |t| self.attention_linear(t, &self.value)); + + let attn_scores = self.attn_scores(query, key); + let weights = self.attn_weights(attn_scores, input.mask_pad, input.mask_attn); + + let context = weights.clone().matmul(value); + let context = context + .swap_dims(1, 2) + .reshape([batch_size, seq_length_1, d_model]); + + let context = cache.output.forward(context, |t| self.output.forward(t)); + + MhaOutput { weights, context } + } + + fn attn_scores(&self, query: Tensor, key: Tensor) -> Tensor { + let attn_scores = query + .matmul(key.transpose()) + .div_scalar((self.d_k as f32).sqrt()); + + self.dropout.forward(attn_scores) + } + + fn attn_weights( + &self, + mut attn_scores: Tensor, + mask_pad: Option>, + mask_attn: Option>, + ) -> Tensor { + if let Some(mask_pad) = mask_pad { + let [batch_size, seq_length] = mask_pad.dims(); + + attn_scores = attn_scores.mask_fill( + mask_pad.reshape([batch_size, 1, 1, seq_length]), + self.min_float, + ); + } + + if let Some(mask_attn) = mask_attn { + let [batch_size, seq_length_1, seq_length_2] = mask_attn.dims(); + + attn_scores = attn_scores.mask_fill( + mask_attn.reshape([batch_size, 1, seq_length_1, seq_length_2]), + self.min_float, + ); + } + + if self.quiet_softmax { + quiet_softmax(attn_scores, 3) + } else { + softmax(attn_scores, 3) + } + } + + fn attention_linear(&self, x: Tensor, linear: &Linear) -> Tensor { + let [batch_size, seq_length, _d_model] = x.dims(); + linear + .forward(x) + .reshape([batch_size, seq_length, self.n_heads, self.d_k]) + .swap_dims(1, 2) + } +} + +/// Cache for the [Multi Head Attention](MultiHeadAttention) layer. +/// +/// To be used during inference when decoding tokens. +pub struct MhaCache { + query: MhaLinearCache, + key: MhaLinearCache, + value: MhaLinearCache, + output: MhaLinearCache, +} + +enum MhaLinearCache { + Autoregressive(TensorCache, usize), + Full(TensorCache), +} + +impl MhaCache { + /// Initialize a cache for autoregressive inference. + pub fn autoregressive() -> Self { + Self { + query: MhaLinearCache::Autoregressive(TensorCache::empty(), 2), + key: MhaLinearCache::Autoregressive(TensorCache::empty(), 2), + value: MhaLinearCache::Autoregressive(TensorCache::empty(), 2), + output: MhaLinearCache::Autoregressive(TensorCache::empty(), 1), + } + } + + /// Initialize a cache for autoregressive inference, but with a fixed memory used for keys and + /// values (cross-attention). + pub fn autoregressive_cross_attention() -> Self { + Self { + query: MhaLinearCache::Autoregressive(TensorCache::empty(), 2), + key: MhaLinearCache::Full(TensorCache::empty()), + value: MhaLinearCache::Full(TensorCache::empty()), + output: MhaLinearCache::Autoregressive(TensorCache::empty(), 1), + } + } +} + +impl MhaLinearCache { + pub fn forward) -> Tensor>( + &mut self, + tensor: Tensor, + func: F, + ) -> Tensor { + match self { + MhaLinearCache::Autoregressive(cache, dim) => { + cache.forward_autoregressive(tensor, *dim, func) + } + MhaLinearCache::Full(cache) => cache.forward_full(tensor, func), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TestBackend, attention::generate_autoregressive_mask}; + use alloc::vec::Vec; + use burn::tensor::Int; + use burn::tensor::Tolerance; + use burn::tensor::ops::FloatElem; + use burn::tensor::{Distribution, Shape}; + + #[test] + fn test_self_attention_shapes() { + let [batch_size, seq_length, d_model, n_heads] = [7, 13, 32, 4]; + let device = Default::default(); + let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init::(&device); + let input = MhaInput::self_attn(Tensor::random( + [batch_size, seq_length, d_model], + Distribution::Default, + &device, + )); + + let output = mha.forward(input); + + assert_eq!( + output.context.shape(), + Shape::new([batch_size, seq_length, d_model]), + "Context should have the correct shape", + ); + assert_eq!( + output.weights.shape(), + Shape::new([batch_size, n_heads, seq_length, seq_length]), + "Weights should have the correct shape", + ); + } + + #[test] + fn test_generic_mha_shapes() { + let [batch_size, seq_length_1, seq_length_2, d_model, n_heads] = [7, 13, 15, 32, 4]; + let mha = MultiHeadAttentionConfig::new(d_model, n_heads) + .init::(&Default::default()); + let device = Default::default(); + let input = MhaInput::new( + Tensor::random( + [batch_size, seq_length_1, d_model], + Distribution::Default, + &device, + ), + Tensor::random( + [batch_size, seq_length_2, d_model], + Distribution::Default, + &device, + ), + Tensor::random( + [batch_size, seq_length_2, d_model], + Distribution::Default, + &device, + ), + ); + + let output = mha.forward(input); + + assert_eq!( + output.context.shape(), + Shape::new([batch_size, seq_length_1, d_model]), + "Context should have the correct shape", + ); + assert_eq!( + output.weights.shape(), + Shape::new([batch_size, n_heads, seq_length_1, seq_length_2]), + "Weights should have the correct shape", + ); + } + + #[test] + fn test_self_attention_mask_pad() { + let [batch_size, seq_length, d_model, n_heads, num_padded] = [3, 6, 32, 2, 2]; + let device = Default::default(); + let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init::(&device); + + // Create a padding mask + let mask_pad: Tensor = + Tensor::zeros([batch_size, seq_length], &device); + let mask_pad = mask_pad.slice_assign( + [0..batch_size, seq_length - num_padded..seq_length], + Tensor::ones([batch_size, num_padded], &device), + ); + let mask_pad = mask_pad.equal_elem(1).to_device(&device); + + let tensor_1 = Tensor::::random( + [batch_size, seq_length, d_model], + Distribution::Default, + &device, + ); + // Change the end of the tensor + let tensor_2 = tensor_1.clone().slice_assign( + [ + 0..batch_size, + seq_length - num_padded..seq_length, + 0..d_model, + ], + Tensor::random( + [batch_size, num_padded, d_model], + Distribution::Default, + &device, + ), + ); + + let input_1 = MhaInput::self_attn(tensor_1).mask_pad(mask_pad.clone()); + let input_2 = MhaInput::self_attn(tensor_2).mask_pad(mask_pad); + + let output_1 = mha.forward(input_1); + let output_2 = mha.forward(input_2); + + // Check that the beginning of each tensor is the same + output_1 + .context + .slice([0..batch_size, 0..seq_length - num_padded, 0..d_model]) + .into_data() + .assert_approx_eq( + &output_2 + .context + .slice([0..batch_size, 0..seq_length - num_padded, 0..d_model]) + .into_data(), + Tolerance::::default(), + ); + } + + #[test] + fn test_autoregressive_mask_should_have_same_output_as_autoregressive_decoding() { + let [batch_size, seq_length, d_model, n_heads] = [3, 4, 12, 2]; + let device = Default::default(); + let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init::(&device); + + let tensor = Tensor::::random( + [batch_size, seq_length, d_model], + Distribution::Default, + &device, + ); + let mask_attn = generate_autoregressive_mask(batch_size, seq_length, &tensor.device()); + let input = MhaInput::self_attn(tensor.clone()).mask_attn(mask_attn); + + let output_1 = mha.forward(input); + let mut output_2 = Vec::new(); + let mut cache = MhaCache::autoregressive(); + + for i in 1..seq_length + 1 { + let tensor = tensor.clone().slice([0..batch_size, 0..i, 0..d_model]); + let input = MhaInput::self_attn(tensor); + let next_tok = mha.forward_cache(input, &mut cache).context.slice([ + 0..batch_size, + i - 1..i, + 0..d_model, + ]); + output_2.push(next_tok); + } + + let output_2 = Tensor::cat(output_2, 1); + + output_1 + .context + .into_data() + .assert_approx_eq::>( + &output_2.into_data(), + Tolerance::default(), + ); + } + + #[test] + fn display() { + let config = MultiHeadAttentionConfig::new(2, 4); + let mha = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{mha}"), + "MultiHeadAttention {d_model: 2, n_heads: 4, d_k: 0, \ + dropout: 0.1, min_float: -10000, quiet_softmax: false, params: 24}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mod.rs new file mode 100644 index 0000000..460e473 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/attention/mod.rs @@ -0,0 +1,7 @@ +mod cross_attention; +mod mask; +mod mha; + +pub use cross_attention::*; +pub use mask::*; +pub use mha::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/autoregressive.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/autoregressive.rs new file mode 100644 index 0000000..2025e14 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/autoregressive.rs @@ -0,0 +1,52 @@ +use alloc::vec; +use burn_core as burn; + +use super::{CacheState, TensorCache}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +impl TensorCache { + pub(crate) fn forward_autoregressive( + &mut self, + tensor: Tensor, + dim_cat: usize, + func: F, + ) -> Tensor + where + F: Fn(Tensor) -> Tensor, + { + let mut tensor_old = CacheState::Empty; + core::mem::swap(&mut self.state, &mut tensor_old); + + let tensor_new = match tensor_old { + CacheState::Value(tensor_old) => { + let [batch_size, seq_length, d_model] = tensor.dims(); + let next_seq_token = + tensor.slice([0..batch_size, (seq_length - 1)..seq_length, 0..d_model]); + let next_seq_token = func(next_seq_token); + + Tensor::cat(vec![tensor_old, next_seq_token], dim_cat) + } + _ => func(tensor), + }; + + self.state = CacheState::Value(tensor_new.clone()); + tensor_new + } + + pub(crate) fn forward_full(&mut self, tensor: Tensor, func: F) -> Tensor + where + F: Fn(Tensor) -> Tensor, + { + let mut tensor_old = CacheState::Empty; + core::mem::swap(&mut self.state, &mut tensor_old); + + let tensor_new = match tensor_old { + CacheState::Value(tensor_old) => tensor_old, + _ => func(tensor), + }; + + self.state = CacheState::Value(tensor_new.clone()); + tensor_new + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/base.rs new file mode 100644 index 0000000..4be0c41 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/base.rs @@ -0,0 +1,27 @@ +use burn_core as burn; + +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +pub(crate) enum CacheState { + Value(T), + Empty, +} + +/// A cache for a tensor. +pub struct TensorCache { + pub(crate) state: CacheState>, +} + +impl TensorCache { + /// Creates a new empty cache. + /// + /// # Returns + /// + /// The empty cache. + pub fn empty() -> Self { + Self { + state: CacheState::Empty, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/mod.rs new file mode 100644 index 0000000..8050cb4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/cache/mod.rs @@ -0,0 +1,4 @@ +mod autoregressive; +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/checks.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/checks.rs new file mode 100644 index 0000000..0e24c27 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/checks.rs @@ -0,0 +1,22 @@ +pub(crate) fn checks_channels_div_groups(channels_in: usize, channels_out: usize, groups: usize) { + let channels_in_div_by_group = channels_in.is_multiple_of(groups); + let channels_out_div_by_group = channels_out.is_multiple_of(groups); + + if !channels_in_div_by_group || !channels_out_div_by_group { + panic!( + "Both channels must be divisible by the number of groups. Got \ + channels_in={channels_in}, channels_out={channels_out}, groups={groups}" + ); + } +} + +// https://github.com/tracel-ai/burn/issues/2676 +/// Only symmetric padding is currently supported. As such, using `Same` padding with an even kernel +/// size is not supported as it will not produce the same output size. +pub(crate) fn check_same_padding_support(kernel_size: &[usize]) { + for k in kernel_size.iter() { + if k % 2 == 0 { + unimplemented!("Same padding with an even kernel size is not supported"); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv1d.rs new file mode 100644 index 0000000..ea9be64 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv1d.rs @@ -0,0 +1,283 @@ +use alloc::format; + +use burn_core as burn; + +use crate::{PaddingConfig1d, conv::checks}; +use burn::tensor::{Tensor, backend::Backend, module::conv1d, ops::PaddedConvOptions}; +use burn::{ + config::Config, + module::{Content, DisplaySettings, Ignored, Initializer, Module, ModuleDisplay, Param}, +}; + +/// Configuration to create a [1D convolution](Conv1d) layer using the [init function](Conv1dConfig::init). +#[derive(Config, Debug)] +pub struct Conv1dConfig { + /// The number of input channels. + pub channels_in: usize, + /// The number of output channels. + pub channels_out: usize, + /// The size of the kernel. + pub kernel_size: usize, + /// The stride of the convolution. + #[config(default = "1")] + pub stride: usize, + /// Spacing between kernel elements. + #[config(default = "1")] + pub dilation: usize, + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub groups: usize, + /// The padding configuration. + /// + /// Supports symmetric and asymmetric padding. `Same` padding with even kernel sizes + /// will automatically use asymmetric padding to preserve input dimensions. + #[config(default = "PaddingConfig1d::Valid")] + pub padding: PaddingConfig1d, + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a 1D convolution over input tensors. +/// +/// Should be created with [Conv1dConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Conv1d { + /// Tensor of shape `[channels_out, channels_in / groups, kernel_size]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: usize, + /// Size of the kernel. + pub kernel_size: usize, + /// Spacing between kernel elements. + pub dilation: usize, + /// Controls the connections between input and output channels. + pub groups: usize, + /// Padding configuration. + pub padding: Ignored, +} + +impl ModuleDisplay for Conv1d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + // Format padding + let padding_formatted = format!("{}", &self.padding); + + // Format stride/dilation as strings + let stride = format!("{:?}", self.stride); + let kernel_size = format!("{:?}", self.kernel_size); + let dilation = format!("{:?}", self.dilation); + + // Extract channels in/out from weight dims + let [channels_out, group_channels_in, _] = self.weight.dims(); + let channels_in = group_channels_in * self.groups; + let ch_out = format!("{:?}", channels_out); + let ch_in = format!("{:?}", channels_in); + + content + .add("ch_in", &ch_in) + .add("ch_out", &ch_out) + .add("stride", &stride) + .add("kernel_size", &kernel_size) + .add("dilation", &dilation) + .add("groups", &self.groups) + .add("padding", &padding_formatted) + .optional() + } +} +impl Conv1dConfig { + /// Initialize a new [conv1d](Conv1d) module. + pub fn init(&self, device: &B::Device) -> Conv1d { + checks::checks_channels_div_groups(self.channels_in, self.channels_out, self.groups); + + let shape = [ + self.channels_out, + self.channels_in / self.groups, + self.kernel_size, + ]; + + let fan_in: usize = self.channels_in / self.groups * self.kernel_size; + let weight = self + .initializer + .init_with(shape, Some(fan_in), None, device); + let mut bias = None; + + if self.bias { + bias = + Some( + self.initializer + .init_with([self.channels_out], Some(fan_in), None, device), + ); + } + + Conv1d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + padding: Ignored(self.padding.clone()), + dilation: self.dilation, + groups: self.groups, + } + } +} + +impl Conv1d { + /// Applies the forward pass on the input tensor. + /// + /// See [conv1d](burn::tensor::module::conv1d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels_in, length_in]` + /// - output: `[batch_size, channels_out, length_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + let length = input.dims()[2]; + + // Calculate padding as pair - handles Same, Valid, and Explicit uniformly + let (left, right) = + self.padding + .calculate_padding_1d_pair(length, self.kernel_size, self.stride); + + let options = PaddedConvOptions::asymmetric( + [self.stride], + [left], + [right], + [self.dilation], + self.groups, + ); + + conv1d( + input, + self.weight.val(), + self.bias.as_ref().map(|bias| bias.val()), + options, + ) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::{ElementConversion, ops::FloatElem}; + type FT = FloatElem; + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = Conv1dConfig::new(5, 5, 5); + let k = (config.channels_in * config.kernel_size) as f64; + let k = (config.groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&device); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = Conv1dConfig::new(5, 5, 5).with_initializer(Initializer::Zeros); + let conv = config.init::(&Default::default()); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight + .to_data() + .assert_eq(&TensorData::zeros::(conv.weight.shape()), false); + } + + #[test] + fn same_with_even_kernel_uses_asymmetric_padding() { + let device = Default::default(); + let config = Conv1dConfig::new(4, 4, 2) + .with_padding(PaddingConfig1d::Same) + .with_initializer(Initializer::Constant { value: 1.0 }) + .with_bias(false); + let conv = config.init::(&device); + + // Input: [batch=1, channels=4, length=5] + let input = Tensor::::ones([1, 4, 5], &device); + let output = conv.forward(input); + + // Same padding should preserve spatial dimensions + assert_eq!(output.dims(), [1, 4, 5]); + } + + #[test] + fn display() { + let config = Conv1dConfig::new(5, 5, 5); + let conv = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{conv}"), + "Conv1d {ch_in: 5, ch_out: 5, stride: 1, kernel_size: 5, dilation: 1, groups: 1, padding: Valid, params: 130}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = Conv1dConfig::new(5, 3, 3); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10], &Default::default()); + let _ = conv.forward(input); + } + + #[test] + fn asymmetric_padding_forward() { + let device = Default::default(); + // Create conv with asymmetric padding: left=1, right=2 + let config = Conv1dConfig::new(2, 3, 3) + .with_padding(PaddingConfig1d::Explicit(1, 2)) + .with_initializer(Initializer::Constant { value: 1.0 }) + .with_bias(false); + let conv = config.init::(&device); + + // Input: [batch=1, channels=2, length=4] + let input = Tensor::::ones([1, 2, 4], &device); + let output = conv.forward(input); + + // With asymmetric padding (1, 2), input length 4 becomes 4+1+2=7 + // Output length = (7 - 3) / 1 + 1 = 5 + assert_eq!(output.dims(), [1, 3, 5]); + } + + #[test] + fn symmetric_explicit_padding_forward() { + let device = Default::default(); + // Create conv with symmetric explicit padding: left=2, right=2 + let config = Conv1dConfig::new(2, 3, 3) + .with_padding(PaddingConfig1d::Explicit(2, 2)) + .with_initializer(Initializer::Constant { value: 1.0 }) + .with_bias(false); + let conv = config.init::(&device); + + // Input: [batch=1, channels=2, length=4] + let input = Tensor::::ones([1, 2, 4], &device); + let output = conv.forward(input); + + // With symmetric padding (2, 2), input length 4 becomes 4+2+2=8 + // Output length = (8 - 3) / 1 + 1 = 6 + assert_eq!(output.dims(), [1, 3, 6]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv2d.rs new file mode 100644 index 0000000..be909b1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv2d.rs @@ -0,0 +1,349 @@ +use alloc::format; + +use burn_core as burn; + +use crate::PaddingConfig2d; +use burn::config::Config; +use burn::module::Initializer; +use burn::module::{Content, DisplaySettings, Ignored, Module, ModuleDisplay, Param}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::conv2d; +use burn::tensor::ops::PaddedConvOptions; + +use crate::conv::checks; + +/// Configuration to create a [2D convolution](Conv2d) layer, using the [init function](Conv2dConfig::init). +#[derive(Config, Debug)] +pub struct Conv2dConfig { + /// The number of channels. + pub channels: [usize; 2], + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The stride of the convolution. + #[config(default = "[1, 1]")] + pub stride: [usize; 2], + /// Spacing between kernel elements. + #[config(default = "[1, 1]")] + pub dilation: [usize; 2], + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub groups: usize, + /// The padding configuration. + /// + /// Supports symmetric and asymmetric padding. `Same` padding with even kernel sizes + /// will automatically use asymmetric padding to preserve input dimensions. + #[config(default = "PaddingConfig2d::Valid")] + pub padding: PaddingConfig2d, + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a 2D convolution over input tensors. +/// +/// Should be created with [Conv2dConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Conv2d { + /// Tensor of shape `[channels_out, channels_in / groups, kernel_size_1, kernel_size_2]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: [usize; 2], + /// Size of the kernel. + pub kernel_size: [usize; 2], + /// Spacing between kernel elements. + pub dilation: [usize; 2], + /// Controls the connections between input and output channels. + pub groups: usize, + /// The padding configuration. + pub padding: Ignored, +} + +impl Conv2dConfig { + /// Initialize a new [conv2d](Conv2d) module. + pub fn init(&self, device: &B::Device) -> Conv2d { + checks::checks_channels_div_groups(self.channels[0], self.channels[1], self.groups); + + let shape = [ + self.channels[1], + self.channels[0] / self.groups, + self.kernel_size[0], + self.kernel_size[1], + ]; + + let k = self.kernel_size.iter().product::(); + let fan_in = self.channels[0] / self.groups * k; + let fan_out = self.channels[1] / self.groups * k; + + let weight = self + .initializer + .init_with(shape, Some(fan_in), Some(fan_out), device); + let mut bias = None; + + if self.bias { + bias = Some(self.initializer.init_with( + [self.channels[1]], + Some(fan_in), + Some(fan_out), + device, + )); + } + + Conv2d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + dilation: self.dilation, + padding: Ignored(self.padding.clone()), + groups: self.groups, + } + } +} + +impl ModuleDisplay for Conv2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + // Since padding does not implement ModuleDisplay, we need to format it manually. + let padding_formatted = format!("{}", &self.padding); + + // Format the stride, kernel_size and dilation as strings, formatted as arrays instead of indexed. + let stride = format!("{:?}", self.stride); + let kernel_size = format!("{:?}", self.kernel_size); + let dilation = format!("{:?}", self.dilation); + let [channels_out, group_channels_in, _, _] = self.weight.dims(); + let channels_in = group_channels_in * self.groups; + let ch_out = format!("{:?}", channels_out); + let ch_in = format!("{:?}", channels_in); + content + .add("ch_in", &ch_in) + .add("ch_out", &ch_out) + .add("stride", &stride) + .add("kernel_size", &kernel_size) + .add("dilation", &dilation) + .add("groups", &self.groups) + .add("padding", &padding_formatted) + .optional() + } +} + +impl Conv2d { + /// Applies the forward pass on the input tensor. + /// + /// See [conv2d](burn::tensor::module::conv2d) for more information. + /// + /// # Shapes + /// - `input`: `[batch_size, channels_in, height_in, width_in]` + /// - `output`: `[batch_size, channels_out, height_out, width_out]` + /// + /// # Example + /// ```rust,ignore + /// use burn::nn::conv::Conv2dConfig; + /// use burn::tensor::Tensor; + /// + /// // Assuming backend type alias `B` + /// let device = Default::default(); + /// let conv = Conv2dConfig::new([3, 8], [3, 3]).init::(&device); + /// + /// let x = Tensor::::zeros([1, 3, 28, 28], &device); + /// let y = conv.forward(x); + /// + /// println!("{:?}", y.dims()); // [1, 8, 26, 26] + /// ``` + pub fn forward(&self, input: Tensor) -> Tensor { + let [_batch_size, _channels_in, height_in, width_in] = input.dims(); + + // Calculate padding as pairs - handles Same, Valid, and Explicit uniformly + let ((top, bottom), (left, right)) = self.padding.calculate_padding_2d_pairs( + height_in, + width_in, + &self.kernel_size, + &self.stride, + ); + + let options = PaddedConvOptions::asymmetric( + self.stride, + [top, left], + [bottom, right], + self.dilation, + self.groups, + ); + + conv2d( + input, + self.weight.val(), + self.bias.as_ref().map(|bias| bias.val()), + options, + ) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::ops::FloatElem; + use burn::tensor::{ElementConversion, Tolerance}; + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + type FT = FloatElem; // Float test + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = Conv2dConfig::new([5, 1], [5, 5]); + let k = (config.channels[0] * config.kernel_size[0] * config.kernel_size[1]) as f64; + let k = (config.groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&device); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = Conv2dConfig::new([5, 2], [5, 5]).with_initializer(Initializer::Zeros); + let conv = config.init::(&device); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(conv.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn initializer_fan_out() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let init = Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: true, // test that fan_out is passed to `init_with()` + }; + + let config = Conv2dConfig::new([5, 1], [5, 5]).with_initializer(init.clone()); + let _ = config.init::(&device); + + assert_eq!(config.initializer, init); + } + + #[test] + fn initializer_fan_with_groups_is_valid() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let init = Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: true, + }; + + let config = Conv2dConfig::new([4, 4], [1, 1]) + .with_initializer(init.clone()) + .with_groups(4); + let _ = config.init::(&device); + + assert_eq!(config.initializer, init); + } + + #[test] + #[should_panic = "Both channels must be divisible by the number of groups."] + fn channels_with_groups_is_invalid() { + let device = Default::default(); + let config = Conv2dConfig::new([1, 4], [1, 1]).with_groups(4); + let _ = config.init::(&device); + } + + #[test] + fn same_with_even_kernel_uses_asymmetric_padding() { + let device = Default::default(); + let config = Conv2dConfig::new([4, 4], [2, 2]) + .with_padding(PaddingConfig2d::Same) + .with_initializer(Initializer::Constant { value: 1.0 }) + .with_bias(false); + let conv = config.init::(&device); + + // Input: [batch=1, channels=4, height=5, width=5] + let input = Tensor::::ones([1, 4, 5, 5], &device); + let output = conv.forward(input); + + // Same padding should preserve spatial dimensions + assert_eq!(output.dims(), [1, 4, 5, 5]); + } + + #[test] + fn display() { + let config = Conv2dConfig::new([5, 1], [5, 5]); + let conv = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{conv}"), + "Conv2d {ch_in: 5, ch_out: 1, stride: [1, 1], kernel_size: [5, 5], dilation: [1, 1], groups: 1, padding: Valid, params: 126}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = Conv2dConfig::new([5, 3], [3, 3]); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10, 10], &Default::default()); + let _ = conv.forward(input); + } + + #[test] + fn asymmetric_padding_forward() { + let device = Default::default(); + // Create conv with asymmetric padding: top=1, left=2, bottom=3, right=4 + let config = Conv2dConfig::new([2, 3], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 2, 3, 4)) + .with_initializer(Initializer::Constant { value: 1.0 }) + .with_bias(false); + let conv = config.init::(&device); + + // Input: [batch=1, channels=2, height=4, width=5] + let input = Tensor::::ones([1, 2, 4, 5], &device); + let output = conv.forward(input); + + // Height: 4 + 1 + 3 = 8, output = (8 - 3) / 1 + 1 = 6 + // Width: 5 + 2 + 4 = 11, output = (11 - 3) / 1 + 1 = 9 + assert_eq!(output.dims(), [1, 3, 6, 9]); + } + + #[test] + fn symmetric_explicit_padding_forward() { + let device = Default::default(); + // Create conv with symmetric explicit padding: top=2, left=2, bottom=2, right=2 + let config = Conv2dConfig::new([2, 3], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(2, 2, 2, 2)) + .with_initializer(Initializer::Constant { value: 1.0 }) + .with_bias(false); + let conv = config.init::(&device); + + // Input: [batch=1, channels=2, height=4, width=5] + let input = Tensor::::ones([1, 2, 4, 5], &device); + let output = conv.forward(input); + + // Height: 4 + 2 + 2 = 8, output = (8 - 3) / 1 + 1 = 6 + // Width: 5 + 2 + 2 = 9, output = (9 - 3) / 1 + 1 = 7 + assert_eq!(output.dims(), [1, 3, 6, 7]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv3d.rs new file mode 100644 index 0000000..e882d38 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv3d.rs @@ -0,0 +1,276 @@ +use alloc::format; + +use burn_core as burn; + +use crate::PaddingConfig3d; +use burn::config::Config; +use burn::module::Initializer; +use burn::module::{Content, DisplaySettings, Ignored, Module, ModuleDisplay, Param}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::conv3d; +use burn::tensor::ops::ConvOptions; + +use crate::conv::checks; + +/// Configuration to create a [3D convolution](Conv3d) layer, using the [init function](Conv3dConfig::init). +#[derive(Config, Debug)] +pub struct Conv3dConfig { + /// The number of channels. + pub channels: [usize; 2], + /// The size of the kernel. + pub kernel_size: [usize; 3], + /// The stride of the convolution. + #[config(default = "[1, 1, 1]")] + pub stride: [usize; 3], + /// Spacing between kernel elements. + #[config(default = "[1, 1, 1]")] + pub dilation: [usize; 3], + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub groups: usize, + /// The padding configuration. + #[config(default = "PaddingConfig3d::Valid")] + pub padding: PaddingConfig3d, + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a 3D convolution over input tensors. +/// +/// Should be created with [Conv3dConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Conv3d { + /// Tensor of shape `[channels_out, channels_in / groups, kernel_size_1, kernel_size_2, kernel_size_3]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: [usize; 3], + /// Size of the kernel. + pub kernel_size: [usize; 3], + /// Spacing between kernel elements. + pub dilation: [usize; 3], + /// Controls the connections between input and output channels. + pub groups: usize, + /// The padding configuration. + pub padding: Ignored, +} + +impl Conv3dConfig { + /// Initialize a new [conv3d](Conv3d) module. + pub fn init(&self, device: &B::Device) -> Conv3d { + checks::checks_channels_div_groups(self.channels[0], self.channels[1], self.groups); + if self.padding == PaddingConfig3d::Same { + checks::check_same_padding_support(&self.kernel_size); + } + + let shape = [ + self.channels[1], + self.channels[0] / self.groups, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ]; + + let k = self.kernel_size.iter().product::(); + let fan_in = self.channels[0] / self.groups * k; + let fan_out = self.channels[1] / self.groups * k; + + let weight = self + .initializer + .init_with(shape, Some(fan_in), Some(fan_out), device); + let mut bias = None; + + if self.bias { + bias = Some(self.initializer.init_with( + [self.channels[1]], + Some(fan_in), + Some(fan_out), + device, + )); + } + + Conv3d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + dilation: self.dilation, + padding: Ignored(self.padding.clone()), + groups: self.groups, + } + } +} + +impl ModuleDisplay for Conv3d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + // Padding doesn't implement ModuleDisplay, so format manually. + let padding_formatted = format!("{}", &self.padding); + + // Format arrays as strings (consistent with Conv2d/Conv1d). + let stride = format!("{:?}", self.stride); + let kernel_size = format!("{:?}", self.kernel_size); + let dilation = format!("{:?}", self.dilation); + + // Weight dims: [channels_out, channels_in/groups, k1, k2, k3] + let [channels_out, group_channels_in, _, _, _] = self.weight.dims(); + let channels_in = group_channels_in * self.groups; + let ch_out = format!("{:?}", channels_out); + let ch_in = format!("{:?}", channels_in); + + content + .add("ch_in", &ch_in) + .add("ch_out", &ch_out) + .add("stride", &stride) + .add("kernel_size", &kernel_size) + .add("dilation", &dilation) + .add("groups", &self.groups) + .add("padding", &padding_formatted) + .optional() + } +} + +impl Conv3d { + /// Applies the forward pass on the input tensor. + /// + /// See [conv3d](burn::tensor::module::conv3d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels_in, depth_in, height_in, width_in]` + /// - output: `[batch_size, channels_out, depth_out, height_out, width_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + let [_batch_size, _channels_in, depth_in, height_in, width_in] = input.dims(); + let padding = self.padding.calculate_padding_3d( + depth_in, + height_in, + width_in, + &self.kernel_size, + &self.stride, + ); + conv3d( + input, + self.weight.val(), + self.bias.as_ref().map(|bias| bias.val()), + ConvOptions::new(self.stride, padding, self.dilation, self.groups), + ) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::{ElementConversion, Tolerance, ops::FloatElem}; + type FT = FloatElem; + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = Conv3dConfig::new([5, 1], [5, 5, 5]); + let k = (config.channels[0] + * config.kernel_size[0] + * config.kernel_size[1] + * config.kernel_size[2]) as f64; + let k = (config.groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&device); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = Conv3dConfig::new([5, 2], [5, 5, 5]).with_initializer(Initializer::Zeros); + let device = Default::default(); + let conv = config.init::(&device); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(conv.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn initializer_fan_out() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let init = Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: true, // test that fan_out is passed to `init_with()` + }; + let config = Conv3dConfig::new([5, 1], [5, 5, 5]).with_initializer(init.clone()); + let _ = config.init::(&device); + + assert_eq!(config.initializer, init); + } + + #[test] + fn initializer_fan_with_groups_is_valid() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let init = Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: true, + }; + + let config = Conv3dConfig::new([4, 4], [1, 1, 1]) + .with_initializer(init.clone()) + .with_groups(4); + let _ = config.init::(&device); + + assert_eq!(config.initializer, init); + } + + #[test] + #[should_panic = "Same padding with an even kernel size is not supported"] + fn same_with_even_kernel_is_invalid() { + let device = Default::default(); + let config = Conv3dConfig::new([4, 4], [2, 2, 2]).with_padding(PaddingConfig3d::Same); + let _ = config.init::(&device); + } + + #[test] + fn display() { + let config = Conv3dConfig::new([5, 1], [5, 5, 5]); + let conv = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{conv}"), + "Conv3d {ch_in: 5, ch_out: 1, stride: [1, 1, 1], kernel_size: [5, 5, 5], dilation: [1, 1, 1], groups: 1, padding: Valid, params: 626}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = Conv3dConfig::new([5, 3], [3, 3, 3]); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10, 10, 10], &Default::default()); + let _ = conv.forward(input); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose1d.rs new file mode 100644 index 0000000..308e553 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose1d.rs @@ -0,0 +1,216 @@ +use alloc::format; + +use burn_core as burn; + +use crate::conv::checks; +use burn::config::Config; +use burn::module::Content; +use burn::module::DisplaySettings; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::ModuleDisplay; +use burn::module::Param; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::conv_transpose1d; +use burn::tensor::ops::ConvTransposeOptions; + +/// Configuration to create an [1D transposed convolution](ConvTranspose1d) layer +/// using the [init function](ConvTranspose1dConfig::init). +#[derive(Config, Debug)] +pub struct ConvTranspose1dConfig { + /// The number of channels. + pub channels: [usize; 2], + /// The size of the kernel. + pub kernel_size: usize, + /// The stride of the convolution. + #[config(default = "1")] + pub stride: usize, + /// Spacing between kernel elements. + #[config(default = "1")] + pub dilation: usize, + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub groups: usize, + /// The padding configuration. + #[config(default = "0")] + pub padding: usize, + /// The padding output configuration. + #[config(default = "0")] + pub padding_out: usize, + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a 1D transposed convolution over input tensors. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct ConvTranspose1d { + /// Tensor of shape `[channels_in, channels_out / groups, kernel_size]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: usize, + /// Size of the kernel. + pub kernel_size: usize, + /// Spacing between kernel elements. + pub dilation: usize, + /// Controls the connections between input and output channels. + pub groups: usize, + /// The padding configuration. + pub padding: usize, + /// The padding output configuration. + pub padding_out: usize, + /// The number of channels. + pub channels: [usize; 2], +} + +impl ModuleDisplay for ConvTranspose1d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("channels", &format!("{:?}", &self.channels)) + .add("stride", &self.stride) + .add("kernel_size", &self.kernel_size) + .add("dilation", &self.dilation) + .add("groups", &self.groups) + .add("padding", &self.padding) + .add("padding_out", &self.padding_out) + .optional() + } +} + +impl ConvTranspose1dConfig { + /// Initialize a new [conv transpose 1d](ConvTranspose1d) module. + pub fn init(&self, device: &B::Device) -> ConvTranspose1d { + checks::checks_channels_div_groups(self.channels[0], self.channels[1], self.groups); + + let shape = [ + self.channels[0], + self.channels[1] / self.groups, + self.kernel_size, + ]; + + let fan_in = self.channels[1] / self.groups * self.kernel_size; + let weight = self + .initializer + .init_with(shape, Some(fan_in), None, device); + let mut bias = None; + + if self.bias { + bias = Some( + self.initializer + .init_with([self.channels[1]], Some(fan_in), None, device), + ); + } + + ConvTranspose1d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + dilation: self.dilation, + groups: self.groups, + padding: self.padding, + padding_out: self.padding_out, + channels: self.channels, + } + } +} + +impl ConvTranspose1d { + /// Applies the forward pass on the input tensor. + /// + /// See also [conv_transpose1d](burn::tensor::module::conv_transpose1d). + /// + /// # Shapes + /// + /// - input: `[batch_size, channels_in, length_in]` + /// - output: `[batch_size, channels_out, length_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + conv_transpose1d( + input, + self.weight.val(), + self.bias.as_ref().map(|bias| bias.val()), + ConvTransposeOptions::new( + [self.stride], + [self.padding], + [self.padding_out], + [self.dilation], + self.groups, + ), + ) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::ops::FloatElem; + use burn::tensor::{ElementConversion, Tolerance}; + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + type FT = FloatElem; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = ConvTranspose1dConfig::new([5, 1], 5); + let k = (config.channels[1] * config.kernel_size) as f64; + let k = (config.groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&Default::default()); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = ConvTranspose1dConfig::new([5, 2], 5).with_initializer(Initializer::Zeros); + let conv = config.init::(&Default::default()); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(conv.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn display() { + let config = ConvTranspose1dConfig::new([5, 2], 5); + let conv = config.init::(&Default::default()); + + assert_eq!( + format!("{conv}"), + "ConvTranspose1d {channels: [5, 2], stride: 1, kernel_size: 5, dilation: 1, groups: 1, padding: 0, padding_out: 0, params: 52}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = ConvTranspose1dConfig::new([5, 3], 3); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10], &Default::default()); + let _ = conv.forward(input); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose2d.rs new file mode 100644 index 0000000..74e6bb1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose2d.rs @@ -0,0 +1,216 @@ +use alloc::format; + +use burn_core as burn; + +use crate::conv::checks; +use burn::config::Config; +use burn::module::Content; +use burn::module::DisplaySettings; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::ModuleDisplay; +use burn::module::Param; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::conv_transpose2d; +use burn::tensor::ops::ConvTransposeOptions; + +/// Configuration to create an [2D transposed convolution](ConvTranspose2d) layer +/// using the [init function](ConvTranspose2dConfig::init). +#[derive(Config, Debug)] +pub struct ConvTranspose2dConfig { + /// The number of channels. + pub channels: [usize; 2], + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The stride of the convolution. + #[config(default = "[1, 1]")] + pub stride: [usize; 2], + /// Spacing between kernel elements. + #[config(default = "[1, 1]")] + pub dilation: [usize; 2], + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub groups: usize, + /// The padding configuration. + #[config(default = "[0, 0]")] + pub padding: [usize; 2], + /// The padding output configuration. + #[config(default = "[0, 0]")] + pub padding_out: [usize; 2], + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a 2D transposed convolution over input tensors. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct ConvTranspose2d { + /// Tensor of shape `[channels_in, channels_out / groups, kernel_size_1, kernel_size_2]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: [usize; 2], + /// Size of the kernel. + pub kernel_size: [usize; 2], + /// Spacing between kernel elements. + pub dilation: [usize; 2], + /// Controls the connections between input and output channels. + pub groups: usize, + /// Padding configuration. + pub padding: [usize; 2], + /// Padding output configuration. + pub padding_out: [usize; 2], + /// Number of channels. + pub channels: [usize; 2], +} + +impl ModuleDisplay for ConvTranspose2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("channels", &format!("{:?}", &self.channels)) + .add("stride", &format!("{:?}", &self.stride)) + .add("kernel_size", &format!("{:?}", &self.kernel_size)) + .add("dilation", &format!("{:?}", &self.dilation)) + .add("groups", &self.groups) + .add("padding", &format!("{:?}", &self.padding)) + .add("padding_out", &format!("{:?}", &self.padding_out)) + .optional() + } +} + +impl ConvTranspose2dConfig { + /// Initialize a new [conv transpose 2d](ConvTranspose2d) module. + pub fn init(&self, device: &B::Device) -> ConvTranspose2d { + checks::checks_channels_div_groups(self.channels[0], self.channels[1], self.groups); + + let shape = [ + self.channels[0], + self.channels[1] / self.groups, + self.kernel_size[0], + self.kernel_size[1], + ]; + + let fan_in = self.channels[1] / self.groups * self.kernel_size.iter().product::(); + let weight = self + .initializer + .init_with(shape, Some(fan_in), None, device); + let mut bias = None; + + if self.bias { + bias = Some( + self.initializer + .init_with([self.channels[1]], Some(fan_in), None, device), + ); + } + + ConvTranspose2d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + dilation: self.dilation, + groups: self.groups, + padding: self.padding, + padding_out: self.padding_out, + channels: self.channels, + } + } +} + +impl ConvTranspose2d { + /// Applies the forward pass on the input tensor. + /// + /// See also [conv_transpose2d](burn::tensor::module::conv_transpose2d). + /// + /// # Shapes + /// + /// - input: `[batch_size, channels_in, height_in, width_in]` + /// - output: `[batch_size, channels_out, height_out, width_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + conv_transpose2d( + input, + self.weight.val(), + self.bias.as_ref().map(|bias| bias.val()), + ConvTransposeOptions::new( + self.stride, + self.padding, + self.padding_out, + self.dilation, + self.groups, + ), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{ElementConversion, Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = ConvTranspose2dConfig::new([5, 1], [5, 5]); + let k = (config.channels[1] * config.kernel_size[0] * config.kernel_size[1]) as f64; + let k = (config.groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&Default::default()); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = + ConvTranspose2dConfig::new([5, 2], [5, 5]).with_initializer(Initializer::Zeros); + let conv = config.init::(&Default::default()); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(conv.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn display() { + let config = ConvTranspose2dConfig::new([5, 2], [5, 5]); + let conv = config.init::(&Default::default()); + + assert_eq!( + format!("{conv}"), + "ConvTranspose2d {channels: [5, 2], stride: [1, 1], kernel_size: [5, 5], dilation: [1, 1], groups: 1, padding: [0, 0], padding_out: [0, 0], params: 252}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = ConvTranspose2dConfig::new([5, 3], [3, 3]); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10, 10], &Default::default()); + let _ = conv.forward(input); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose3d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose3d.rs new file mode 100644 index 0000000..e8f1b47 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/conv_transpose3d.rs @@ -0,0 +1,221 @@ +use alloc::format; + +use burn_core as burn; + +use crate::conv::checks; +use burn::config::Config; +use burn::module::Content; +use burn::module::DisplaySettings; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::ModuleDisplay; +use burn::module::Param; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::conv_transpose3d; +use burn::tensor::ops::ConvTransposeOptions; + +/// Configuration to create an [3D transposed convolution](ConvTranspose3d) layer +/// using the [init function](ConvTranspose3dConfig::init). +#[derive(Config, Debug)] +pub struct ConvTranspose3dConfig { + /// The number of channels. + pub channels: [usize; 2], + /// The size of the kernel. + pub kernel_size: [usize; 3], + /// The stride of the convolution. + #[config(default = "[1, 1, 1]")] + pub stride: [usize; 3], + /// Spacing between kernel elements. + #[config(default = "[1, 1, 1]")] + pub dilation: [usize; 3], + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub groups: usize, + /// The padding configuration. + #[config(default = "[0, 0, 0]")] + pub padding: [usize; 3], + /// The padding output configuration. + #[config(default = "[0, 0, 0]")] + pub padding_out: [usize; 3], + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a 3D transposed convolution over input tensors. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct ConvTranspose3d { + /// Tensor of shape `[channels_in, channels_out / groups, kernel_size_1, kernel_size_2, kernel_size_3]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: [usize; 3], + /// Size of the kernel. + pub kernel_size: [usize; 3], + /// Spacing between kernel elements. + pub dilation: [usize; 3], + /// Controls the connections between input and output channels. + pub groups: usize, + /// Padding configuration. + pub padding: [usize; 3], + /// Padding output configuration. + pub padding_out: [usize; 3], + /// Number of channels. + pub channels: [usize; 2], +} + +impl ModuleDisplay for ConvTranspose3d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("channels", &format!("{:?}", &self.channels)) + .add("stride", &format!("{:?}", &self.stride)) + .add("kernel_size", &format!("{:?}", &self.kernel_size)) + .add("dilation", &format!("{:?}", &self.dilation)) + .add("groups", &self.groups) + .add("padding", &format!("{:?}", &self.padding)) + .add("padding_out", &format!("{:?}", &self.padding_out)) + .optional() + } +} + +impl ConvTranspose3dConfig { + /// Initialize a new [conv transpose 2d](ConvTranspose3d) module. + pub fn init(&self, device: &B::Device) -> ConvTranspose3d { + checks::checks_channels_div_groups(self.channels[0], self.channels[1], self.groups); + + let shape = [ + self.channels[0], + self.channels[1] / self.groups, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ]; + + let fan_in = self.channels[1] / self.groups * self.kernel_size.iter().product::(); + let weight = self + .initializer + .init_with(shape, Some(fan_in), None, device); + let mut bias = None; + + if self.bias { + bias = Some( + self.initializer + .init_with([self.channels[1]], Some(fan_in), None, device), + ); + } + + ConvTranspose3d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + dilation: self.dilation, + groups: self.groups, + padding: self.padding, + padding_out: self.padding_out, + channels: self.channels, + } + } +} + +impl ConvTranspose3d { + /// Applies the forward pass on the input tensor. + /// + /// See also [conv_transpose3d](burn::tensor::module::conv_transpose3d). + /// + /// # Shapes + /// + /// - input: `[batch_size, channels_in, depth_in, height_in, width_in]` + /// - output: `[batch_size, channels_out, depth_out, height_out, width_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + conv_transpose3d( + input, + self.weight.val(), + self.bias.as_ref().map(|bias| bias.val()), + ConvTransposeOptions::new( + self.stride, + self.padding, + self.padding_out, + self.dilation, + self.groups, + ), + ) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::{ElementConversion, Tolerance, ops::FloatElem}; + type FT = FloatElem; + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = ConvTranspose3dConfig::new([5, 1], [5, 5, 5]); + let k = (config.channels[1] + * config.kernel_size[0] + * config.kernel_size[1] + * config.kernel_size[2]) as f64; + let k = (config.groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&Default::default()); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = + ConvTranspose3dConfig::new([5, 2], [5, 5, 5]).with_initializer(Initializer::Zeros); + let conv = config.init::(&Default::default()); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(conv.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn display() { + let config = ConvTranspose3dConfig::new([5, 2], [5, 5, 5]); + let conv = config.init::(&Default::default()); + + assert_eq!( + format!("{conv}"), + "ConvTranspose3d {channels: [5, 2], stride: [1, 1, 1], kernel_size: [5, 5, 5], dilation: [1, 1, 1], groups: 1, padding: [0, 0, 0], padding_out: [0, 0, 0], params: 1252}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = ConvTranspose3dConfig::new([5, 3], [3, 3, 3]); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10, 10, 10], &Default::default()); + let _ = conv.forward(input); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/deform_conv2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/deform_conv2d.rs new file mode 100644 index 0000000..16a2e6a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/deform_conv2d.rs @@ -0,0 +1,295 @@ +use alloc::format; +use burn::tensor::ops::DeformConvOptions; + +use burn_core as burn; + +use crate::PaddingConfig2d; +use burn::config::Config; +use burn::module::Initializer; +use burn::module::{Content, DisplaySettings, Ignored, Module, ModuleDisplay, Param}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::deform_conv2d; + +use crate::conv::checks; + +/// Configuration to create a [deformable 2D convolution](DeformConv2d) layer, using the [init function](DeformConv2dConfig::init). +#[derive(Config, Debug)] +pub struct DeformConv2dConfig { + /// The number of channels. + pub channels: [usize; 2], + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The stride of the convolution. + #[config(default = "[1, 1]")] + pub stride: [usize; 2], + /// Spacing between kernel elements. + #[config(default = "[1, 1]")] + pub dilation: [usize; 2], + /// Controls the connections between input and output channels. + #[config(default = "1")] + pub weight_groups: usize, + /// Offset groups. + #[config(default = "1")] + pub offset_groups: usize, + /// The padding configuration. + /// + /// ### Warning + /// Only symmetric padding is currently supported. As such, using `Same` padding with an even kernel + /// size is not supported as it will not produce the same output size. + #[config(default = "PaddingConfig2d::Valid")] + pub padding: PaddingConfig2d, + /// If bias should be added to the output. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0),fan_out_only:false}" + )] + pub initializer: Initializer, +} + +/// Applies a deformable 2D convolution over input tensors. +/// +/// Should be created with [DeformConv2dConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct DeformConv2d { + /// Tensor of shape `[channels_out, channels_in / groups, kernel_size_1, kernel_size_2]` + pub weight: Param>, + /// Tensor of shape `[channels_out]` + pub bias: Option>>, + /// Stride of the convolution. + pub stride: [usize; 2], + /// Size of the kernel. + pub kernel_size: [usize; 2], + /// Spacing between kernel elements. + pub dilation: [usize; 2], + /// Controls the connections between input and output channels. + pub weight_groups: usize, + /// Offset groups. + pub offset_groups: usize, + /// The padding configuration. + pub padding: Ignored, +} + +impl DeformConv2dConfig { + /// Initialize a new [DeformConv2d](DeformConv2d) module. + pub fn init(&self, device: &B::Device) -> DeformConv2d { + checks::checks_channels_div_groups(self.channels[0], self.channels[1], self.weight_groups); + if self.padding == PaddingConfig2d::Same { + checks::check_same_padding_support(&self.kernel_size); + } + + let shape = [ + self.channels[1], + self.channels[0] / self.weight_groups, + self.kernel_size[0], + self.kernel_size[1], + ]; + + let k = self.kernel_size.iter().product::(); + let fan_in = self.channels[0] / self.weight_groups * k; + let fan_out = self.channels[1] / self.weight_groups * k; + + let weight = self + .initializer + .init_with(shape, Some(fan_in), Some(fan_out), device); + let mut bias = None; + + if self.bias { + bias = Some(self.initializer.init_with( + [self.channels[1]], + Some(fan_in), + Some(fan_out), + device, + )); + } + + DeformConv2d { + weight, + bias, + stride: self.stride, + kernel_size: self.kernel_size, + dilation: self.dilation, + padding: Ignored(self.padding.clone()), + weight_groups: self.weight_groups, + offset_groups: self.weight_groups, + } + } +} + +impl ModuleDisplay for DeformConv2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + // Since padding does not implement ModuleDisplay, we need to format it manually. + let padding_formatted = format!("{}", &self.padding); + + // Format the stride, kernel_size and dilation as strings, formatted as arrays instead of indexed. + let stride = format!("{:?}", self.stride); + let kernel_size = format!("{:?}", self.kernel_size); + let dilation = format!("{:?}", self.dilation); + + content + .add("stride", &stride) + .add("kernel_size", &kernel_size) + .add("dilation", &dilation) + .add("weight_groups", &self.weight_groups) + .add("offset_groups", &self.offset_groups) + .add("padding", &padding_formatted) + .optional() + } +} + +impl DeformConv2d { + /// Applies the forward pass on the input tensor. + /// + /// See [deform_conv2d](burn::tensor::module::deform_conv2d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels_in, height_in, width_in]` + /// - offset: `[batch_size, 2 * offset_groups * kernel_height * kernel_width, height_out, width_out]` + /// - mask: `[batch_size, offset_groups * kernel_height * kernel_width, height_out, width_out]` + /// - output: `[batch_size, channels_out, height_out, width_out]` + pub fn forward( + &self, + input: Tensor, + offset: Tensor, + mask: Option>, + ) -> Tensor { + let [_batch_size, _channels_in, height_in, width_in] = input.dims(); + let padding = + self.padding + .calculate_padding_2d(height_in, width_in, &self.kernel_size, &self.stride); + deform_conv2d( + input, + offset, + self.weight.val(), + mask, + self.bias.as_ref().map(|bias| bias.val()), + DeformConvOptions::new( + self.stride, + padding, + self.dilation, + self.weight_groups, + self.offset_groups, + ), + ) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::{ElementConversion, Tolerance, ops::FloatElem}; + type FT = FloatElem; + + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = DeformConv2dConfig::new([5, 1], [5, 5]); + let k = (config.channels[0] * config.kernel_size[0] * config.kernel_size[1]) as f64; + let k = (config.offset_groups as f64 / k).sqrt().elem::(); + let conv = config.init::(&device); + + conv.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = DeformConv2dConfig::new([5, 2], [5, 5]).with_initializer(Initializer::Zeros); + let conv = config.init::(&device); + + assert_eq!(config.initializer, Initializer::Zeros); + conv.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(conv.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn initializer_fan_out() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let init = Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: true, // test that fan_out is passed to `init_with()` + }; + + let config = DeformConv2dConfig::new([5, 1], [5, 5]).with_initializer(init.clone()); + let _ = config.init::(&device); + + assert_eq!(config.initializer, init); + } + + #[test] + fn initializer_fan_with_groups_is_valid() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let init = Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: true, + }; + + let config = DeformConv2dConfig::new([4, 4], [1, 1]) + .with_initializer(init.clone()) + .with_weight_groups(4); + let _ = config.init::(&device); + + assert_eq!(config.initializer, init); + } + + #[test] + #[should_panic = "Both channels must be divisible by the number of groups."] + fn channels_with_groups_is_invalid() { + let device = Default::default(); + let config = DeformConv2dConfig::new([1, 4], [1, 1]).with_weight_groups(4); + let _ = config.init::(&device); + } + + #[test] + #[should_panic = "Same padding with an even kernel size is not supported"] + fn same_with_even_kernel_is_invalid() { + let device = Default::default(); + let config = DeformConv2dConfig::new([4, 4], [2, 2]).with_padding(PaddingConfig2d::Same); + let _ = config.init::(&device); + } + + #[test] + fn display() { + let config = DeformConv2dConfig::new([5, 1], [5, 5]); + let conv = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{conv}"), + "DeformConv2d {stride: [1, 1], kernel_size: [5, 5], dilation: [1, 1], weight_groups: 1, offset_groups: 1, padding: Valid, params: 126}" + ); + } + + #[test] + #[should_panic = "Number of channels in input tensor and input channels of convolution must be equal. got: 4, expected: 5"] + fn input_channels_mismatch() { + let config = DeformConv2dConfig::new([5, 3], [3, 3]); + let conv = config.init::(&Default::default()); + + let input = Tensor::::zeros([1, 4, 10, 10], &Default::default()); + let offset = Tensor::::zeros([1, 2 * 3 * 3, 10, 10], &Default::default()); + let _ = conv.forward(input, offset, None); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/mod.rs new file mode 100644 index 0000000..9c83141 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/conv/mod.rs @@ -0,0 +1,17 @@ +mod conv1d; +mod conv2d; +mod conv3d; +mod conv_transpose1d; +mod conv_transpose2d; +mod conv_transpose3d; +mod deform_conv2d; + +pub(crate) mod checks; + +pub use conv_transpose1d::*; +pub use conv_transpose2d::*; +pub use conv_transpose3d::*; +pub use conv1d::*; +pub use conv2d::*; +pub use conv3d::*; +pub use deform_conv2d::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/dropout.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/dropout.rs new file mode 100644 index 0000000..9f558c7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/dropout.rs @@ -0,0 +1,124 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay}; +use burn::tensor::backend::Backend; +use burn::tensor::{Distribution, Tensor}; + +/// Configuration to create a [Dropout](Dropout) layer using the [init function](DropoutConfig::init). +#[derive(Config, Debug)] +pub struct DropoutConfig { + /// The probability of randomly zeroes some elements of the input tensor during training. + pub prob: f64, +} + +/// Set at random some elements of the input tensor to zero during training. +/// +/// This is an effective regularization technique as describe in the paper +/// [Improving neural networks by preventing co-adaptation of feature detectors](https://arxiv.org/abs/1207.0580). +/// +/// The input is also scaled during training to `1 / (1 - prob_keep)`. +/// +/// Should be created with [DropoutConfig]. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Dropout { + /// The probability of randomly zeroes some elements of the input tensor during training. + pub prob: f64, +} + +impl DropoutConfig { + /// Initialize a new [dropout](Dropout) module. + pub fn init(&self) -> Dropout { + if self.prob < 0.0 || self.prob > 1.0 { + panic!( + "Dropout probability should be between 0 and 1, but got {}", + self.prob + ); + } + Dropout { prob: self.prob } + } +} + +impl Dropout { + /// Applies the forward pass on the input tensor. + /// + /// See [Dropout](Dropout) for more information. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + if !B::ad_enabled(&input.device()) || self.prob == 0.0 { + return input; + } + + let prob_keep = 1.0 - self.prob; + let random = input.random_like(Distribution::Bernoulli(prob_keep)); + let x = input * random; + + x * (1.0 / prob_keep) + } +} + +impl ModuleDisplay for Dropout { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("prob", &self.prob).optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn::tensor::Shape; + + #[cfg(feature = "std")] + use crate::{TestAutodiffBackend, TestBackend}; + + #[cfg(not(feature = "std"))] + use crate::TestBackend; + + #[cfg(feature = "std")] + #[test] + fn with_ad_backend_should_mark_input() { + let tensor = + Tensor::::ones(Shape::new([100, 100]), &Default::default()); + let dropout = DropoutConfig::new(0.5).init(); + + let output = dropout.forward(tensor.clone()); + + assert_ne!(tensor.to_data(), output.to_data()); + } + + #[test] + fn without_ad_backend_should_not_change_input() { + let tensor = Tensor::::ones(Shape::new([100, 100]), &Default::default()); + let dropout = DropoutConfig::new(0.5).init(); + + let output = dropout.forward(tensor.clone()); + + assert_eq!(tensor.to_data(), output.to_data()); + } + + #[test] + fn display() { + let config = DropoutConfig::new(0.5); + let layer = config.init(); + + assert_eq!(alloc::format!("{layer}"), "Dropout {prob: 0.5}"); + } + + #[test] + #[should_panic = "Dropout probability should be between 0 and 1,"] + fn dropout_prob_invalid() { + let config = DropoutConfig::new(-10.); + let _layer = config.init(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/embedding.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/embedding.rs new file mode 100644 index 0000000..58a6ea7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/embedding.rs @@ -0,0 +1,111 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::Param; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Int; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +use burn::tensor::module::embedding; + +/// Configuration to create an [Embedding](Embedding) layer using the [init function](EmbeddingConfig::init). +#[derive(Config, Debug)] +pub struct EmbeddingConfig { + /// The number of embedding vectors. + pub n_embedding: usize, + /// The size of each vector. + pub d_model: usize, + /// The type of function used to initialize neural network parameters + #[config(default = "Initializer::Normal{mean:0.0, std:1.0}")] + pub initializer: Initializer, +} + +/// Lookup table to store a fix number of vectors. +/// +/// Should be created with [EmbeddingConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Embedding { + /// The learnable weights of the module of shape `[n_embedding, d_model]` initialized + /// from a normal distribution `N(0, 1)`. + pub weight: Param>, +} + +impl ModuleDisplay for Embedding { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [n_embedding, d_model] = self.weight.shape().dims(); + content + .add("n_embedding", &n_embedding) + .add("d_model", &d_model) + .optional() + } +} + +impl EmbeddingConfig { + /// Initialize a new [embedding](Embedding) module. + pub fn init(&self, device: &B::Device) -> Embedding { + let weight = self + .initializer + .init([self.n_embedding, self.d_model], device); + + Embedding { weight } + } +} + +impl Embedding { + /// Applies the forward pass on the input tensor. + /// + /// See also [embedding](burn::tensor::module::embedding). + /// + /// # Shapes + /// + /// - input: `[batch_size, seq_length]` + /// - output: `[batch_size, seq_length, d_model]` + pub fn forward(&self, input: Tensor) -> Tensor { + embedding(self.weight.val(), input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = EmbeddingConfig::new(5, 5).with_initializer(Initializer::Zeros); + let embed = config.init::(&Default::default()); + + assert_eq!(config.initializer, Initializer::Zeros); + embed.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(embed.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn display() { + let config = EmbeddingConfig::new(100, 10); + let embed = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{embed}"), + "Embedding {n_embedding: 100, d_model: 10, params: 1000}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate1d.rs new file mode 100644 index 0000000..68958d5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate1d.rs @@ -0,0 +1,258 @@ +use alloc::format; + +use burn::tensor::module::interpolate; + +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Ignored, Module, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::ops::InterpolateOptions; + +use super::InterpolateMode; + +/// Configuration for the 1D interpolation module. +/// +/// This struct defines the configuration options for the 1D interpolation operation. +/// It allows specifying the output size, scale factor, and interpolation mode. +#[derive(Config, Debug)] +pub struct Interpolate1dConfig { + /// Output size of the interpolated tensor. + /// If specified, this takes precedence over `scale_factor`. + #[config(default = "None")] + pub output_size: Option, + + /// Scale factor for resizing the input tensor. + /// This is used when `output_size` is not specified. + #[config(default = "None")] + pub scale_factor: Option, + + /// Interpolation mode to use for resizing. + /// Determines how the output values are calculated. + #[config(default = "InterpolateMode::Nearest")] + pub mode: InterpolateMode, + + /// If `true`, the input and output tensors are aligned by their corner pixels. + /// If `false`, half-pixel coordinate mapping is used instead. + #[config(default = true)] + pub align_corners: bool, +} + +/// Interpolate module for resizing 1D tensors with shape [N, C, L]. +/// +/// This struct represents a 1D interpolation module that can resize tensors +/// using various interpolation methods. It provides flexibility in specifying +/// either an output size or a scale factor for resizing, along with options +/// for the interpolation mode. +/// +/// The module can be used to upsample or downsample 1D tensors, preserving the +/// number of channels and batch size while adjusting the length dimension. +/// +/// The module can be created using the [Interpolate1dConfig] struct and the +/// `init` method, which returns an instance of the [Interpolate1d] struct. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Interpolate1d { + /// Output size of the interpolated tensor + pub output_size: Option, + + /// Scale factor for resizing the input tensor + pub scale_factor: Option, + + /// Interpolation mode used for resizing + pub mode: Ignored, + + /// Whether to align corner pixels + pub align_corners: bool, +} + +impl Interpolate1dConfig { + /// Initialize the interpolation module + pub fn init(self) -> Interpolate1d { + Interpolate1d { + output_size: self.output_size, + scale_factor: self.scale_factor, + mode: Ignored(self.mode), + align_corners: self.align_corners, + } + } +} + +impl Interpolate1d { + /// Performs the forward pass of the 1D interpolation module + /// + /// # Arguments + /// + /// * `input` - Input tensor with shape [N, C, L] + /// + /// # Returns + /// + /// Resized tensor with shape [N, C, L'], where L' is determined by + /// the output_size or scale_factor specified in the module configuration + /// + /// # Example + /// + /// ```ignore + /// let input = Tensor::::random([1, 3, 64], Distribution::Uniform(0.0, 1.0), &device); + /// let interpolate = Interpolate1dConfig::new() + /// .with_output_size(Some(128)) + /// .init(); + /// let output = interpolate.forward(input); + /// assert_eq!(output.dims(), [1, 3, 128]); + /// ``` + pub fn forward(&self, input: Tensor) -> Tensor { + let output_size = calculate_output_size(input.dims(), self.output_size, self.scale_factor); + + // Use the interpolate operation to resize the temporal input tensor + // by adding a new dimension for the interpolation axis + let input = input.unsqueeze_dim(2); + + let result = interpolate( + input, + [1, output_size], + InterpolateOptions::new(self.mode.0.clone().into()) + .with_align_corners(self.align_corners), + ); + + result.squeeze_dims(&[2]) + } +} + +/// Calculate output size based on input dimensions, output size, and scale factor +/// +/// # Arguments +/// +/// * `input_dims` - Input dimensions of the tensor +/// * `output_size` - Output size for the interpolated tensor +/// * `scale_factor` - Scale factor for resizing the tensor +/// +/// # Returns +/// +/// Output size for the interpolated tensor +/// +/// # Panics +/// +/// Panics if neither output_size nor scale_factor is provided +/// or if the scale factor is too large +fn calculate_output_size( + input_dims: [usize; 3], + output_size: Option, + scale_factor: Option, +) -> usize { + match (output_size, scale_factor) { + (Some(output_size), None) => { + // Use provided + output_size + } + (None, Some(scale_factor)) => { + // Calculate output size based on scale factor + let [_, _, l] = input_dims; + + let new_dim = (l as f64) * (scale_factor as f64); + + if new_dim > usize::MAX as f64 { + panic!("Scale factor is too large"); + } + + new_dim as usize + } + _ => panic!("Either output_size or scale_factor must be provided"), + } +} + +impl ModuleDisplay for Interpolate1d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("mode", &self.mode) + .add("output_size", &format!("{:?}", self.output_size)) + .add("scale_factor", &self.scale_factor) + .optional() + } +} + +#[cfg(test)] +mod tests { + + use burn::tensor::Distribution; + + use super::*; + use crate::TestBackend; + #[test] + fn test_calculate_output_size() { + let input_dims = [1, 1, 4]; + + let output_size = calculate_output_size(input_dims, Some(2), None); + assert_eq!(output_size, 2); + + let output_size = calculate_output_size(input_dims, None, Some(2.0)); + assert_eq!(output_size, 8); + + let output_size = calculate_output_size(input_dims, None, Some(0.5)); + assert_eq!(output_size, 2); + + let output_size = calculate_output_size(input_dims, None, Some(1.5)); + assert_eq!(output_size, 6); + } + + #[test] + #[should_panic(expected = "Either output_size or scale_factor must be provided")] + fn test_panic() { + let input_dims = [1, 1, 4]; + calculate_output_size(input_dims, None, None); + } + + #[test] + #[should_panic(expected = "Scale factor is too large")] + fn test_large_scale_factor() { + let input_dims = [1, 1, usize::MAX - 1]; + calculate_output_size(input_dims, None, Some(2.0)); + } + + #[test] + fn test_module() { + let input = Tensor::::random( + [2, 3, 4], + Distribution::Uniform(0.0, 1.0), + &Default::default(), + ); + + // Test with output_size + let config = Interpolate1dConfig::new().with_output_size(Some(8)); + let interpolate = config.init(); + let output = interpolate.forward(input.clone()); + assert_eq!(output.dims(), [2, 3, 8]); + + // Test with scale_factor + let config = Interpolate1dConfig::new().with_scale_factor(Some(0.5)); + let interpolate = config.init(); + let output = interpolate.forward(input.clone()); + assert_eq!(output.dims(), [2, 3, 2]); + + // Test with different interpolation mode + let config = Interpolate1dConfig::new() + .with_output_size(Some(6)) + .with_mode(InterpolateMode::Linear); + let interpolate = config.init(); + let output = interpolate.forward(input); + assert_eq!(output.dims(), [2, 3, 6]); + } + + #[test] + fn display() { + let config = Interpolate1dConfig::new().with_output_size(Some(20)); + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "Interpolate1d {mode: Nearest, output_size: Some(20), \ + scale_factor: None}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate2d.rs new file mode 100644 index 0000000..a347425 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/interpolate2d.rs @@ -0,0 +1,261 @@ +use alloc::format; + +use burn::tensor::module::interpolate; + +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Ignored, Module, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::ops::InterpolateOptions; + +use super::InterpolateMode; + +/// Configuration for the 2D interpolation module. +/// +/// This struct defines the configuration options for the 2D interpolation operation. +/// It allows specifying the output size, scale factor, and interpolation mode. +#[derive(Config, Debug)] +pub struct Interpolate2dConfig { + /// Output size of the interpolated tensor. + /// If specified, this takes precedence over `scale_factor`. + #[config(default = "None")] + pub output_size: Option<[usize; 2]>, + + /// Scale factor for resizing the input tensor. + /// This is used when `output_size` is not specified. + #[config(default = "None")] + pub scale_factor: Option<[f32; 2]>, + + /// Interpolation mode to use for resizing. + /// Determines how the output values are calculated. + #[config(default = "InterpolateMode::Nearest")] + pub mode: InterpolateMode, + + /// If `true`, the input and output tensors are aligned by their corner pixels. + /// If `false`, half-pixel coordinate mapping is used instead. + #[config(default = true)] + pub align_corners: bool, +} + +/// Interpolate module for resizing tensors with shape [N, C, H, W]. +/// +/// This struct represents an interpolation module that can resize tensors +/// using various interpolation methods. It provides flexibility in specifying +/// either an output size or a scale factor for resizing, along with options +/// for the interpolation mode. +/// +/// The module can be used to upsample or downsample tensors, preserving the +/// number of channels and batch size while adjusting the height and width +/// dimensions. +/// +/// The module can be created using the [Interpolate2dConfig] struct and the +/// `init` method, which returns an instance of the [Interpolate2d] struct. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Interpolate2d { + /// Output size of the interpolated tensor + pub output_size: Option<[usize; 2]>, + + /// Scale factor for resizing the input tensor + pub scale_factor: Option<[f32; 2]>, + + /// Interpolation mode used for resizing + pub mode: Ignored, + + /// Whether to align corner pixels + pub align_corners: bool, +} + +impl Interpolate2dConfig { + /// Initialize the interpolation module + pub fn init(self) -> Interpolate2d { + Interpolate2d { + output_size: self.output_size, + scale_factor: self.scale_factor, + mode: Ignored(self.mode), + align_corners: self.align_corners, + } + } +} +impl Interpolate2d { + /// Performs the forward pass of the interpolation module + /// + /// # Arguments + /// + /// * `input` - Input tensor with shape [N, C, H, W] + /// + /// # Returns + /// + /// Resized tensor with shape [N, C, H', W'], where H' and W' are determined by + /// the output_size or scale_factor specified in the module configuration + /// + /// # Example + /// + /// ```ignore + /// let input = Tensor::::random([1, 3, 64, 64], Distribution::Uniform(0.0, 1.0), &device); + /// let interpolate = Interpolate2dConfig::new() + /// .with_output_size(Some([128, 128])) + /// .init(); + /// let output = interpolate.forward(input); + /// assert_eq!(output.dims(), [1, 3, 128, 128]); + /// ``` + pub fn forward(&self, input: Tensor) -> Tensor { + let output_size = calculate_output_size(input.dims(), self.output_size, self.scale_factor); + interpolate( + input, + output_size, + InterpolateOptions::new(self.mode.0.clone().into()) + .with_align_corners(self.align_corners), + ) + } +} + +/// Calculates the output size for tensor interpolation. +/// +/// # Arguments +/// +/// * `input_dims` - The dimensions of the input tensor [N, C, H, W]. +/// * `output_size` - Optional desired output size [H', W']. +/// * `scale_factor` - Optional scale factor for height and width [scale_h, scale_w]. +/// +/// # Returns +/// +/// A tuple [H', W'] representing the calculated output size. +/// +/// # Panics +/// +/// Panics if neither `output_size` nor `scale_factor` is provided, +/// or if the scale factor results in dimensions exceeding usize::MAX. +fn calculate_output_size( + input_dims: [usize; 4], + output_size: Option<[usize; 2]>, + scale_factor: Option<[f32; 2]>, +) -> [usize; 2] { + match (output_size, scale_factor) { + (Some(output_size), None) => { + // Use provided + output_size + } + (None, Some(scale_factor)) => { + // Calculate output size based on scale factor + let [_, _, h, w] = input_dims; + + let new_dim_h = (h as f64) * (scale_factor[0] as f64); + + if new_dim_h > usize::MAX as f64 { + panic!("Scale factor for height is too large"); + } + + let new_dim_w = (w as f64) * (scale_factor[1] as f64); + + if new_dim_w > usize::MAX as f64 { + panic!("Scale factor for width is too large"); + } + + [new_dim_h as usize, new_dim_w as usize] + } + _ => panic!("Either output_size or scale_factor must be provided"), + } +} + +impl ModuleDisplay for Interpolate2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("mode", &self.mode) + .add("output_size", &format!("{:?}", self.output_size)) + .add("scale_factor", &self.scale_factor) + .optional() + } +} +#[cfg(test)] +mod tests { + use burn::tensor::Distribution; + + use crate::TestBackend; + + use super::*; + + #[test] + fn test_calculate_output_size() { + let input_dims = [1, 1, 4, 4]; + + let output_size = calculate_output_size(input_dims, Some([2, 2]), None); + assert_eq!(output_size, [2, 2]); + + let output_size = calculate_output_size(input_dims, None, Some([2.0, 2.0])); + assert_eq!(output_size, [8, 8]); + + let output_size = calculate_output_size([1, 1, 4, 4], None, Some([0.5, 0.5])); + assert_eq!(output_size, [2, 2]); + + let output_size = calculate_output_size([1, 1, 4, 4], None, Some([2.0, 1.5])); + assert_eq!(output_size, [8, 6]); + } + + #[test] + #[should_panic(expected = "Either output_size or scale_factor must be provided")] + fn test_missing_params() { + calculate_output_size([1, 1, 4, 4], None, None); + } + + #[test] + #[should_panic(expected = "Scale factor for height is too large")] + fn test_infinite_height() { + calculate_output_size([1, 1, usize::MAX - 1, 4], None, Some([2.0, 1.0])); + } + + #[test] + #[should_panic(expected = "Scale factor for width is too large")] + fn test_infinite_width() { + calculate_output_size([1, 1, 4, usize::MAX - 1], None, Some([1.0, 2.0])); + } + + #[test] + fn test_module() { + let input = Tensor::::random( + [2, 3, 4, 4], + Distribution::Uniform(0.0, 1.0), + &Default::default(), + ); + + // Test with output_size + let config = Interpolate2dConfig::new().with_output_size(Some([8, 8])); + let interpolate = config.init(); + let output = interpolate.forward(input.clone()); + assert_eq!(output.dims(), [2, 3, 8, 8]); + + // Test with scale_factor + let config = Interpolate2dConfig::new().with_scale_factor(Some([0.5, 0.5])); + let interpolate = config.init(); + let output = interpolate.forward(input.clone()); + assert_eq!(output.dims(), [2, 3, 2, 2]); + + // Test with different interpolation mode + let config = Interpolate2dConfig::new() + .with_output_size(Some([6, 6])) + .with_mode(InterpolateMode::Linear); + let interpolate = config.init(); + let output = interpolate.forward(input); + assert_eq!(output.dims(), [2, 3, 6, 6]); + } + + #[test] + fn display() { + let config = Interpolate2dConfig::new().with_output_size(Some([20, 20])); + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "Interpolate2d {mode: Nearest, output_size: Some([20, 20]), \ + scale_factor: None}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/mod.rs new file mode 100644 index 0000000..8dd64d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/interpolate/mod.rs @@ -0,0 +1,49 @@ +mod interpolate1d; +mod interpolate2d; + +pub use interpolate1d::*; +pub use interpolate2d::*; + +use burn_core as burn; + +use burn::config::Config; +use burn::tensor::ops::InterpolateMode as OpsInterpolateMode; + +/// Algorithm used for downsampling and upsampling +/// +/// This enum defines different interpolation modes for resampling data. +#[derive(Config, Debug)] +pub enum InterpolateMode { + /// Nearest-neighbor interpolation + /// + /// This mode selects the value of the nearest sample point for each output pixel. + /// It is applicable for both temporal and spatial data. + Nearest, + + /// Linear interpolation + /// + /// This mode calculates the output value using linear + /// interpolation between nearby sample points. + /// + /// It is applicable for both temporal and spatial data. + Linear, + + /// Cubic interpolation + /// + /// This mode uses cubic interpolation to calculate the output value + /// based on surrounding sample points. + /// + /// It is applicable for both temporal and spatial data and generally + /// provides smoother results than linear interpolation. + Cubic, +} + +impl From for OpsInterpolateMode { + fn from(mode: InterpolateMode) -> Self { + match mode { + InterpolateMode::Nearest => OpsInterpolateMode::Nearest, + InterpolateMode::Linear => OpsInterpolateMode::Bilinear, + InterpolateMode::Cubic => OpsInterpolateMode::Bicubic, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/linear.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/linear.rs new file mode 100644 index 0000000..cb19a05 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/linear.rs @@ -0,0 +1,340 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Param; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::module::linear; +use burn::tensor::{Tensor, backend::Backend}; + +/// Configuration to create a [`Linear`] layer using the [init function](LinearConfig::init). +#[derive(Config, Debug)] +pub struct LinearConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the output features. + pub d_output: usize, + /// If a bias should be applied during the linear transformation. + #[config(default = true)] + pub bias: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}" + )] + pub initializer: Initializer, + /// The layout in which the linear parameters are stored. + #[config(default = "LinearLayout::Row")] + pub layout: LinearLayout, +} + +#[derive(Config, Debug, Copy)] +/// The layout in which the linear parameters are stored. +/// +/// This can have performance impacts. +pub enum LinearLayout { + /// Parameters are stored in Row major. + Row, + /// Parameters are stored in Col major. + Col, +} + +/// Applies a linear transformation to the input tensor. +/// +/// Should be created with [LinearConfig] +/// +/// `O = IW + b` +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Linear { + /// Matrix of shape `[d_input, d_output]` initialized from a uniform distribution: + /// `U(-k, k)`, where `k = sqrt(1 / d_input)` + pub weight: Param>, + /// Vector of size `d_output` initialized from a uniform distribution: + /// `U(-k, k)`, where `k = sqrt(1 / d_input)` + pub bias: Option>>, +} + +impl LinearConfig { + /// Initialize a new [`Linear`] module. + pub fn init(&self, device: &B::Device) -> Linear { + let weight = match self.layout { + LinearLayout::Row => { + let shape = [self.d_input, self.d_output]; + self.initializer + .init_with(shape, Some(self.d_input), Some(self.d_output), device) + } + LinearLayout::Col => { + let shape = [self.d_output, self.d_input]; + + self.initializer + .init_with(shape, Some(self.d_output), Some(self.d_input), device) + // The param is already transposed when init. We re-transpose to have + // [d_output, d_input] while saving. + .save_mapper(move |tensor| { + B::sync(&tensor.device()).unwrap(); + let tensor = tensor.transpose(); + B::sync(&tensor.device()).unwrap(); + tensor + }) + // When loading from record we have to transpose. + .load_mapper(move |tensor| { + B::sync(&tensor.device()).unwrap(); + let tensor = tensor.transpose(); + B::sync(&tensor.device()).unwrap(); + + tensor + }) + // When loading from initialization, we have to transpose. + .init_mapper(|tensor| { + B::sync(&tensor.device()).unwrap(); + let tensor = tensor.transpose(); + B::sync(&tensor.device()).unwrap(); + tensor + }) + } + }; + let bias = if self.bias { + Some(self.initializer.init_with( + [self.d_output], + Some(self.d_input), + Some(self.d_output), + device, + )) + } else { + None + }; + + Linear { weight, bias } + } +} + +impl Linear { + /// Applies the forward pass on the input tensor. + /// + /// # Arguments + /// + /// - `input` - The input tensor of shape `[..., d_input]`. + /// + /// # Shapes + /// + /// - input: `[..., d_input]` + /// - output: `[..., d_output]` + /// + /// # Returns + /// + /// The transformed tensor of shape `[..., d_output]`. + pub fn forward(&self, input: Tensor) -> Tensor { + linear( + input, + self.weight.val(), + self.bias.as_ref().map(|b| b.val()), + ) + } +} + +impl ModuleDisplay for Linear { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, d_output] = self.weight.shape().dims(); + content + .add("d_input", &d_input) + .add("d_output", &d_output) + .add("bias", &self.bias.is_some()) + .optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::module::ParamId; + use burn::record::{BinBytesRecorder, FullPrecisionSettings, Recorder}; + use burn::tensor::ElementConversion; + use burn::tensor::{Shape, TensorData}; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn initializer_default() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = LinearConfig::new(5, 5); + let k = (1.0 / config.d_input as f64).sqrt().elem::(); + let linear = config.init::(&device); + + assert_eq!( + config.initializer, + Initializer::KaimingUniform { + gain: 1.0 / 3.0f64.sqrt(), + fan_out_only: false + } + ); + linear.weight.to_data().assert_within_range(-k..k); + } + + #[test] + fn initializer_zeros() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = LinearConfig::new(5, 5).with_initializer(Initializer::Zeros); + let linear = config.init::(&device); + + assert_eq!(config.initializer, Initializer::Zeros); + linear.weight.to_data().assert_approx_eq::( + &TensorData::zeros::(linear.weight.shape()), + Tolerance::default(), + ); + } + + #[test] + fn test_linear_forward_no_bias() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let value = 2.; + let config = LinearConfig::new(2, 3) + .with_initializer(Initializer::Constant { value }) + .with_bias(false); + let linear = config.init::(&device); + + let input = Tensor::::ones(Shape::new([1, 2]), &device); + let result = linear.forward(input); + let expected_result = Tensor::::from_data([[4., 4., 4.]], &device); + + assert_eq!(result.into_data(), expected_result.into_data()); + } + + #[test] + fn test_linear_forward_with_bias() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let device = Default::default(); + + let value = 2.; + let config = LinearConfig::new(2, 3).with_initializer(Initializer::Constant { value }); + let linear = config.init::(&device); + + let input = Tensor::::ones(Shape::new([1, 2]), &device); + let result = linear.forward(input); + let expected_result = Tensor::::from_data([[6., 6., 6.]], &device); + + assert_eq!(result.into_data(), expected_result.into_data()); + } + + #[test] + fn test_linear_1d() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let device = Default::default(); + + let value = 2.; + let config = LinearConfig::new(2, 3).with_initializer(Initializer::Constant { value }); + let linear = config.init::(&device); + + let input_1d = Tensor::::ones(Shape::new([2]), &device); + let input_2d = Tensor::::ones(Shape::new([1, 2]), &device); + + let result_1d = linear.forward(input_1d).unsqueeze::<2>(); + let result_2d = linear.forward(input_2d); + + assert_eq!(result_1d.into_data(), result_2d.into_data()); + } + + #[test] + fn display() { + let config = LinearConfig::new(3, 5); + let linear = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{linear}"), + "Linear {d_input: 3, d_output: 5, bias: true, params: 20}" + ); + } + + #[test] + fn layout() { + let device = Default::default(); + let config = LinearConfig::new(6, 12).with_layout(LinearLayout::Col); + let linear = config.init::(&device); + + assert_eq!(linear.weight.dims(), [6, 12], "Shape is as configured"); + + let recorder = BinBytesRecorder::::new(); + + // We go through serialization to trigger the mappers.. + let record = linear.into_record(); + let data = recorder.record(record, ()).unwrap(); + let record = recorder.load(data.clone(), &device).unwrap(); + + let config = LinearConfig::new(12, 6).with_layout(LinearLayout::Row); + let linear_row = config.init::(&device).load_record(record); + + assert_eq!( + linear_row.weight.dims(), + [12, 6], + "Shape should be transposed" + ); + + let record = recorder.load(data.clone(), &device).unwrap(); + let config = LinearConfig::new(6, 12).with_layout(LinearLayout::Col); + let linear_col = config.init::(&device).load_record(record); + + assert_eq!( + linear_col.weight.dims(), + [6, 12], + "Shape should be as configured" + ); + + // We go through serialization to trigger the mappers. + // + // The test will fail if the mapper is not correctly given to the module after loading a + // record. + let record = linear_col.into_record(); + let data = recorder.record(record, ()).unwrap(); + + let record = recorder.load(data, &device).unwrap(); + let config = LinearConfig::new(6, 12).with_layout(LinearLayout::Col); + let linear_col = config.init::(&device).load_record(record); + + assert_eq!( + linear_col.weight.dims(), + [6, 12], + "Shape should be as configured" + ); + } + + #[test] + fn col_row_same_result() { + let device = Default::default(); + let config_col = LinearConfig::new(6, 12).with_layout(LinearLayout::Col); + let linear_col = config_col.init::(&device); + let signal = Tensor::<_, 2>::random([8, 6], burn::tensor::Distribution::Default, &device); + let value = linear_col.forward(signal.clone()); + + let data_1 = value.into_data(); + + let weights = linear_col.weight.val().into_data(); + let weights = Tensor::from_data(weights, &device); + + let linear = Linear { + weight: Param::initialized(ParamId::new(), weights), + bias: linear_col + .bias + .map(|b| Param::initialized(ParamId::new(), b.val())), + }; + + let value = linear.forward(signal); + let data_2 = value.into_data(); + + data_1.assert_approx_eq::(&data_2, Default::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/mod.rs new file mode 100644 index 0000000..8fcd95d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/mod.rs @@ -0,0 +1,38 @@ +/// Attention module +pub mod attention; + +/// Cache module +pub mod cache; + +/// Convolution module +pub mod conv; + +/// Pooling module +pub mod pool; + +/// Transformer module +pub mod transformer; + +/// Interpolate module +pub mod interpolate; + +mod dropout; +mod embedding; +mod linear; +mod noise; +mod pos_encoding; +mod rnn; +mod rope_encoding; +mod unfold; + +pub mod norm; +pub use norm::{batch::*, group::*, instance::*, layer::*, rms::*}; + +pub use dropout::*; +pub use embedding::*; +pub use linear::*; +pub use noise::*; +pub use pos_encoding::*; +pub use rnn::*; +pub use rope_encoding::*; +pub use unfold::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/noise.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/noise.rs new file mode 100644 index 0000000..4dc1042 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/noise.rs @@ -0,0 +1,123 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay}; +use burn::tensor::backend::Backend; +use burn::tensor::{Distribution, Tensor}; + +/// Configuration to create a [GaussianNoise](GaussianNoise) layer using the [init function](GaussianNoiseConfig::init). +#[derive(Config, Debug)] +pub struct GaussianNoiseConfig { + /// Standard deviation of the normal noise distribution. + pub std: f64, +} + +/// Add pseudorandom Gaussian noise to an arbitrarily shaped tensor. +/// +/// This is an effective regularization technique that also contributes to data augmentation. +/// Please keep in mind that the value of [std](GaussianNoise::std) should be chosen with care in order to avoid +/// distortion. +/// +/// Should be created with [GaussianNoiseConfig]. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct GaussianNoise { + /// Standard deviation of the normal noise distribution. + pub std: f64, +} + +impl GaussianNoiseConfig { + /// Initialize a new [Gaussian noise](GaussianNoise) module. + pub fn init(&self) -> GaussianNoise { + if self.std.is_sign_negative() { + panic!( + "Standard deviation is required to be non-negative, but got {}", + self.std + ); + } + GaussianNoise { std: self.std } + } +} + +impl GaussianNoise { + /// Applies the forward pass on the input tensor. + /// + /// See [GaussianNoise](GaussianNoise) for more information. + /// + /// # Shapes + /// + /// - input: `[..., any]` + /// - output: `[..., any]` + pub fn forward(&self, input: Tensor) -> Tensor { + if B::ad_enabled(&input.device()) && self.std != 0.0 { + let noise = Tensor::random( + input.shape(), + Distribution::Normal(0.0, self.std), + &input.device(), + ); + input + noise + } else { + input + } + } +} + +impl ModuleDisplay for GaussianNoise { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("std", &self.std).optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn::tensor::Shape; + + #[cfg(feature = "std")] + use crate::{TestAutodiffBackend, TestBackend}; + + #[cfg(not(feature = "std"))] + use crate::TestBackend; + + #[cfg(feature = "std")] + #[test] + fn with_ad_backend_should_mark_input() { + let tensor = + Tensor::::ones(Shape::new([100, 100]), &Default::default()); + let noise = GaussianNoiseConfig::new(0.5).init(); + + let output = noise.forward(tensor.clone()); + + assert_ne!(tensor.to_data(), output.to_data()); + } + + #[test] + fn without_ad_backend_should_not_change_input() { + let tensor = Tensor::::ones(Shape::new([100, 100]), &Default::default()); + let noise = GaussianNoiseConfig::new(0.5).init(); + + let output = noise.forward(tensor.clone()); + + assert_eq!(tensor.to_data(), output.to_data()); + } + + #[test] + #[should_panic(expected = "Standard deviation is required to be non-negative")] + fn negative_std_should_panic() { + GaussianNoiseConfig { std: -0.5 }.init(); + } + + #[test] + fn display() { + let config = GaussianNoiseConfig::new(0.5); + let layer = config.init(); + + assert_eq!(alloc::format!("{layer}"), "GaussianNoise {std: 0.5}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/batch.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/batch.rs new file mode 100644 index 0000000..d218075 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/batch.rs @@ -0,0 +1,484 @@ +use burn_core as burn; + +use burn::module::Initializer; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::{Tensor, backend::Backend}; +use burn::{ + config::Config, + module::{Module, Param, RunningState}, +}; + +/// [`BatchNorm`] Configuration. +/// +/// Used to create a [`BatchNorm`] layer using the [`BatchNormConfig::init`]. +#[derive(Config, Debug)] +pub struct BatchNormConfig { + /// The number of features. + pub num_features: usize, + /// A value required for numerical stability. Default: 1e-5 + #[config(default = 1e-5)] + pub epsilon: f64, + /// Momentum used to update the metrics. Default: 0.1 + #[config(default = 0.1)] + pub momentum: f64, +} + +/// Applies Batch Normalization over a tensor. +/// +/// Based upon the paper [Batch Normalization](https://arxiv.org/abs/1502.03167). +/// +/// Assumes input tensor is of shape ``[batch_size, channels, ...]``. +/// +/// `Y = norm(X) * γ + β` +/// +/// Where: +/// - `X` is the input tensor +/// - `Y` is the output tensor +/// - `norm` is the normalization function +/// - `γ` is the learnable weight +/// - `β` is the learnable bias +/// +/// Should be created using [`BatchNormConfig`]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct BatchNorm { + /// The learnable weight gamma. + pub gamma: Param>, + /// The learnable weight beta. + pub beta: Param>, + /// The running mean. + pub running_mean: RunningState>, + /// The running variance. + pub running_var: RunningState>, + /// Momentum used to update the metrics. + pub momentum: f64, + /// A value required for numerical stability. + pub epsilon: f64, +} + +impl BatchNormConfig { + /// Initializes a new [batch norm](BatchNorm) module. + pub fn init(&self, device: &B::Device) -> BatchNorm { + let gamma = Initializer::Ones.init([self.num_features], device); + let beta = Initializer::Zeros.init([self.num_features], device); + + let running_mean = Tensor::zeros([self.num_features], device); + let running_var = Tensor::ones([self.num_features], device); + + BatchNorm { + gamma, + beta, + running_mean: RunningState::new(running_mean), + running_var: RunningState::new(running_var), + momentum: self.momentum, + epsilon: self.epsilon, + } + } +} + +impl BatchNorm { + /// Applies the forward pass on the input tensor. + /// + /// See [`BatchNorm`] for more information. + /// + /// # Shapes + /// + /// - `input`: ``[batch_size, channels, ...]`` + /// - `output`: ``[batch_size, channels, ...]`` + /// + /// # Panics + /// + /// This function will panic if the input tensor has rank < 2. + pub fn forward(&self, input: Tensor) -> Tensor { + // Should be move to a compilation error when const generic support that kind of + // validation. https://github.com/rust-lang/rust/issues/76560 + if D < 2 { + panic!( + "BatchNorm can only be applied on tensors of rank >= 2 with the following shape \ + [batch_size, channels, ...], received {}D tensor", + D + ); + } + + match B::ad_enabled(&input.device()) { + true => self.forward_train(input), + false => self.forward_inference(input), + } + } + + fn forward_inference(&self, input: Tensor) -> Tensor { + let device = input.device(); + let channels = input.dims()[1]; + let mean = self.running_mean.value().to_device(&device); + let var = self.running_var.value().to_device(&device); + + let mut shape = [1; D]; + shape[1] = channels; + + self.forward_shared(input, mean.reshape(shape), var.reshape(shape)) + } + + fn forward_train(&self, input: Tensor) -> Tensor { + let device = input.device(); + let dims = input.dims(); + let batch_size = dims[0]; + let channels = dims[1]; + + let mut shape_unsqueeze = [1; D]; + let mut flatten_size = batch_size; + shape_unsqueeze[1] = channels; + + for dim in dims.iter().take(D).skip(2) { + flatten_size *= dim; + } + + let mean = input + .clone() + .swap_dims(0, 1) + .reshape([channels, flatten_size]) + .mean_dim(1) + .reshape(shape_unsqueeze); + + let var = input + .clone() + .sub(mean.clone()) + .square() + .swap_dims(0, 1) + .reshape([channels, flatten_size]) + .mean_dim(1) + .reshape(shape_unsqueeze); + + let running_mean = self.running_mean.value_sync().to_device(&device); + let running_var = self.running_var.value_sync().to_device(&device); + + let running_mean = running_mean.mul_scalar(1.0 - self.momentum).add( + mean.clone() + .detach() + .mul_scalar(self.momentum) + .reshape([channels]), + ); + let running_var = running_var.mul_scalar(1.0 - self.momentum).add( + var.clone() + .detach() + .mul_scalar(self.momentum) + .reshape([channels]), + ); + + self.running_mean.update(running_mean.detach()); + self.running_var.update(running_var.detach()); + + self.forward_shared(input, mean, var) + } + + fn forward_shared( + &self, + x: Tensor, + mean: Tensor, + var: Tensor, + ) -> Tensor { + let channels = x.dims()[1]; + let mut shape = [1; D]; + shape[1] = channels; + + let std = var.add_scalar(self.epsilon).sqrt(); + + let x = x.sub(mean); + let x = x.div(std); + + let x = x.mul(self.gamma.val().reshape(shape)); + + x.add(self.beta.val().reshape(shape)) + } +} + +impl ModuleDisplay for BatchNorm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [num_features] = self.beta.shape().dims(); + + content + .add("num_features", &num_features) + .add("momentum", &self.momentum) + .add("epsilon", &self.epsilon) + .optional() + } +} + +#[cfg(feature = "std")] +#[cfg(test)] +mod tests_1d { + use super::*; + use crate::TestAutodiffBackend; + use burn::module::AutodiffModule; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn batch_norm_forward_train() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + let output = module.forward(input_tensor(&device)); + + output + .to_data() + .assert_approx_eq::(&expected_train(), Tolerance::rel_abs(0.1, 0.001)); + } + + #[test] + fn batch_norm_forward_inference() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + module.forward(input_tensor(&device)); + let module = module.valid(); + let output = module.forward(input_tensor(&device)); + + output + .to_data() + .assert_approx_eq::(&expected_valid(), Tolerance::default()); + } + + fn expected_valid() -> TensorData { + TensorData::from([ + [[0.9409, 0.6976], [0.5892, 0.8774], [0.9106, 0.6844]], + [[0.6012, 0.0782], [-0.0394, 0.9270], [0.6181, 0.5492]], + ]) + } + + fn expected_train() -> TensorData { + TensorData::from([ + [ + [1.1483e+00, 3.7521e-01], + [1.6272e-03, 7.5067e-01], + [1.6204e+00, -4.5168e-02], + ], + [ + [6.8856e-02, -1.5923e+00], + [-1.6318e+00, 8.7949e-01], + [-5.3368e-01, -1.0416e+00], + ], + ]) + } + + fn input_tensor(device: &B::Device) -> Tensor { + Tensor::::from_floats( + [ + [[0.9601, 0.7277], [0.6272, 0.9034], [0.9378, 0.7230]], + [[0.6356, 0.1362], [0.0249, 0.9509], [0.6600, 0.5945]], + ], + device, + ) + } + + #[test] + fn batch_norm_forward_train_inference() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + module.forward(input_tensor(&device)); + let module = module.valid(); + let output = module.forward(input_tensor(&device)); + + output + .to_data() + .assert_approx_eq::(&expected_valid(), Tolerance::default()); + + let module = module.train::(); + let output = module.forward(input_tensor(&device)); + output + .to_data() + .assert_approx_eq::(&expected_train(), Tolerance::default()); + } +} + +#[cfg(feature = "std")] +#[cfg(test)] +mod tests_2d { + use super::*; + use crate::TestAutodiffBackend; + use burn::module::AutodiffModule; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn batch_norm_forward_train() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + let output = module.forward(input_tensor(&device)); + + let expected = TensorData::from([ + [ + [[1.5136, 0.7506], [-1.2216, 0.1477]], + [[0.3135, 1.2252], [-0.4150, 0.6130]], + [[1.4186, 0.3372], [-1.5183, 1.5262]], + ], + [ + [[0.4483, -1.1914], [-1.2010, 0.7537]], + [[-1.6752, 1.3822], [-0.5058, -0.9381]], + [[0.0200, -0.3097], [-0.5715, -0.9026]], + ], + ]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::rel_abs(0.1, 0.001)); + } + + #[test] + fn batch_norm_forward_inference() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + module.forward(input_tensor(&device)); + let module = module.valid(); + let output = module.forward(input_tensor(&device)); + + let expected = TensorData::from([ + [ + [[0.9538, 0.7103], [0.0808, 0.5179]], + [[0.6015, 0.8910], [0.3703, 0.6966]], + [[0.9171, 0.6912], [0.3037, 0.9395]], + ], + [ + [[0.6138, 0.0904], [0.0874, 0.7113]], + [[-0.0297, 0.9408], [0.3415, 0.2042]], + [[0.6250, 0.5561], [0.5013, 0.4323]], + ], + ]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn batch_norm_running_mean() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + let _output = module.forward(input_tensor(&device)); + + let running_mean = module.running_mean.value_sync(); + + let expected = TensorData::from([0.0499, 0.0532, 0.0656]); + running_mean + .reshape([3]) + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn batch_norm_running_var() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + let _output = module.forward(input_tensor(&device)); + + let running_var = module.running_var.value_sync(); + + let expected = TensorData::from([0.9106, 0.9105, 0.9045]); + running_var + .reshape([3]) + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn batch_norm_running_mean_inner_module() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + + let _output = module.forward(input_tensor(&device)); + + let module_valid = module.valid(); + let running_mean = module_valid.running_mean.value(); + let running_mean_after = module.running_mean.value(); + + running_mean_after + .into_data() + .assert_approx_eq::(&running_mean.into_data(), Tolerance::default()); + } + + #[test] + fn batch_norm_grads() { + let device = Default::default(); + let module = BatchNormConfig::new(3).init::(&device); + let input = input_tensor(&device).require_grad(); + + let output = module.forward(input.clone()); + + let grads = output.backward(); + + let tolerance = Tolerance::rel_abs(0.1, 0.001); + let expected = TensorData::from([0.0000e+00, -5.9035e-07, -6.0011e-07]); + module + .gamma + .grad(&grads) + .unwrap() + .reshape([3]) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([8., 8., 8.]); + module + .beta + .grad(&grads) + .unwrap() + .reshape([3]) + .into_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([ + [ + [[0.0000e+00, 0.0000e+00], [0.0000e+00, 0.0000e+00]], + [[7.6400e-08, 2.9848e-07], [-1.0110e-07, 1.4933e-07]], + [[5.3570e-07, 1.2732e-07], [-5.7336e-07, 5.7632e-07]], + ], + [ + [[0.0000e+00, 0.0000e+00], [0.0000e+00, 0.0000e+00]], + [[-4.0807e-07, 3.3673e-07], [-1.2323e-07, -2.2854e-07]], + [[7.5642e-09, -1.1695e-07], [-2.1582e-07, -3.4078e-07]], + ], + ]); + input + .grad(&grads) + .unwrap() + .into_data() + .assert_approx_eq::(&expected, tolerance); + } + + fn input_tensor(device: &B::Device) -> Tensor { + Tensor::::from_floats( + [ + [ + [[0.9601, 0.7277], [0.1270, 0.5441]], + [[0.6272, 0.9034], [0.4066, 0.7179]], + [[0.9378, 0.7230], [0.3544, 0.9591]], + ], + [ + [[0.6356, 0.1362], [0.1333, 0.7287]], + [[0.0249, 0.9509], [0.3791, 0.2481]], + [[0.6600, 0.5945], [0.5424, 0.4767]], + ], + ], + device, + ) + } + + #[test] + fn display() { + let batch_norm = BatchNormConfig::new(3).init::(&Default::default()); + + assert_eq!( + format!("{batch_norm}"), + "BatchNorm {num_features: 3, momentum: 0.1, epsilon: 0.00001, params: 12}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/group.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/group.rs new file mode 100644 index 0000000..4da360d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/group.rs @@ -0,0 +1,336 @@ +use burn::module::Initializer; +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::Param; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Configuration to create a [GroupNorm](GroupNorm) layer using the [init function](GroupNormConfig::init). +#[derive(Debug, Config)] +pub struct GroupNormConfig { + /// The number of groups to separate the channels into + pub num_groups: usize, + /// The number of channels expected in the input + pub num_channels: usize, + /// A value required for numerical stability. Default: 1e-5 + #[config(default = 1e-5)] + pub epsilon: f64, + /// A boolean value that when set to `true`, this module has learnable + /// per-channel affine parameters initialized to ones (for weights) + /// and zeros (for biases). Default: `true` + #[config(default = true)] + pub affine: bool, +} + +/// Applies Group Normalization over a mini-batch of inputs as described in the paper [Group Normalization](https://arxiv.org/abs/1803.08494). +/// +/// `Y = groupnorm(X) * γ + β` +/// +/// Where: +/// - `X` is the input tensor +/// - `Y` is the output tensor +/// - `γ` is the learnable weight +/// - `β` is the learnable bias +/// +/// Should be created using [GroupNormConfig](GroupNormConfig). +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct GroupNorm { + /// The learnable weight + pub gamma: Option>>, + /// The learnable bias + pub beta: Option>>, + /// The number of groups to separate the channels into + pub num_groups: usize, + /// The number of channels expected in the input + pub num_channels: usize, + /// A value required for numerical stability + pub epsilon: f64, + /// A boolean value that when set to `true`, this module has learnable + pub affine: bool, +} + +impl ModuleDisplay for GroupNorm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("num_groups", &self.num_groups) + .add("num_channels", &self.num_channels) + .add("epsilon", &self.epsilon) + .add("affine", &self.affine) + .optional() + } +} + +impl GroupNormConfig { + /// Initialize a new [group norm](GroupNorm) module. + pub fn init(&self, device: &B::Device) -> GroupNorm { + assert_eq!( + self.num_channels % self.num_groups, + 0, + "The number of channels must be divisible by the number of groups" + ); + + let (gamma, beta) = if self.affine { + let gamma = Initializer::Ones.init([self.num_channels], device); + let beta = Initializer::Zeros.init([self.num_channels], device); + + (Some(gamma), Some(beta)) + } else { + (None, None) + }; + + GroupNorm { + num_groups: self.num_groups, + num_channels: self.num_channels, + gamma, + beta, + epsilon: self.epsilon, + affine: self.affine, + } + } +} + +impl GroupNorm { + /// Applies the forward pass on the input tensor. + /// + /// See [GroupNorm](GroupNorm) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, num_channels, *]` + /// - output: `[batch_size, num_channels, *]` + pub fn forward(&self, input: Tensor) -> Tensor { + if input.shape()[1] != self.num_channels { + panic!( + "The number of channels in the input tensor should be equal to the number of channels in the GroupNorm module. Expected {}, got {}", + self.num_channels, + input.shape()[1] + ); + } + + let gamma = self.gamma.as_ref().map(|x| x.val()); + let beta = self.beta.as_ref().map(|x| x.val()); + + group_norm( + input, + gamma, + beta, + self.num_groups, + self.epsilon, + self.affine, + ) + } +} + +/// Applies Group Normalization over a mini-batch of inputs as described in the paper [Group Normalization](https://arxiv.org/abs/1803.08494). +/// +/// `Y = groupnorm(X) * γ + β` +/// +/// Where: +/// - `X` is the input tensor +/// - `Y` is the output tensor +/// - `γ` is the learnable weight +/// - `β` is the learnable bias +/// +pub(crate) fn group_norm( + input: Tensor, + gamma: Option>, + beta: Option>, + num_groups: usize, + epsilon: f64, + affine: bool, +) -> Tensor { + if (beta.is_none() || gamma.is_none()) && affine { + panic!("Affine is set to true, but gamma or beta is None"); + } + + let shape = input.shape(); + if shape.num_elements() <= 2 { + panic!( + "input rank for GroupNorm should be at least 3, but got {}", + shape.num_elements() + ); + } + + let batch_size = shape[0]; + let num_channels = shape[1]; + + let hidden_size = shape[2..].iter().product::() * num_channels / num_groups; + let input = input.reshape([batch_size, num_groups, hidden_size]); + + let mean = input.clone().sum_dim(2) / hidden_size as f64; + let input = input.sub(mean); + + let var = input.clone().square().sum_dim(2) / hidden_size as f64; + let input_normalized = input.div(var.add_scalar(epsilon).sqrt()); + + if affine { + let mut affine_shape = [1; D]; + affine_shape[1] = num_channels; + + input_normalized + .reshape(shape) + .mul(gamma.clone().unwrap().reshape(affine_shape)) + .add(beta.clone().unwrap().reshape(affine_shape)) + } else { + input_normalized.reshape(shape) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use alloc::format; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn group_norm_forward_affine_false() { + let device = Default::default(); + let module = GroupNormConfig::new(2, 6) + .with_affine(false) + .init::(&device); + + assert!(module.gamma.is_none()); + assert!(module.beta.is_none()); + + let input = Tensor::::from_data( + TensorData::from([ + [ + [-0.3034, 0.2726, -0.9659], + [-1.1845, -1.3236, 0.0172], + [1.9507, 1.2554, -0.8625], + [1.0682, 0.3604, 0.3985], + [-0.4957, -0.4461, -0.9721], + [1.5157, -0.1546, -0.5596], + ], + [ + [-1.6698, -0.4040, -0.7927], + [0.3736, -0.0975, -0.1351], + [-0.9461, 0.5461, -0.6334], + [-1.0919, -0.1158, 0.1213], + [-0.9535, 0.1281, 0.4372], + [-0.2845, 0.3488, 0.5641], + ], + ]), + &device, + ); + + let output = module.forward(input); + + let expected = TensorData::from([ + [ + [-0.1653, 0.3748, -0.7866], + [-0.9916, -1.1220, 0.1353], + [1.9485, 1.2965, -0.6896], + [1.2769, 0.3628, 0.4120], + [-0.7427, -0.6786, -1.3578], + [1.8547, -0.3022, -0.8252], + ], + [ + [-1.9342, 0.0211, -0.5793], + [1.2223, 0.4945, 0.4365], + [-0.8163, 1.4887, -0.3333], + [-1.7960, -0.0392, 0.3875], + [-1.5469, 0.3998, 0.9561], + [-0.3428, 0.7970, 1.1845], + ], + ]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn group_norm_forward_affine_true() { + let device = Default::default(); + let module = GroupNormConfig::new(3, 6) + .with_affine(true) + .init::(&device); + + let tolerance = Tolerance::permissive(); + module + .gamma + .as_ref() + .expect("gamma should not be None") + .val() + .to_data() + .assert_approx_eq::(&TensorData::ones::([6]), tolerance); + + module + .beta + .as_ref() + .expect("beta should not be None") + .val() + .to_data() + .assert_approx_eq::(&TensorData::zeros::([6]), tolerance); + + let input = Tensor::::from_data( + TensorData::from([ + [ + [0.3345, 0.4429, 0.6639], + [0.5041, 0.4175, 0.8437], + [0.6159, 0.3758, 0.4071], + [0.5417, 0.5785, 0.7671], + [0.3837, 0.9883, 0.0420], + [0.4808, 0.8989, 0.6144], + ], + [ + [0.3930, 0.2098, 0.0602], + [0.2298, 0.9425, 0.0333], + [0.7409, 0.8172, 0.8879], + [0.4846, 0.0486, 0.2029], + [0.6741, 0.9765, 0.6864], + [0.2827, 0.5534, 0.2125], + ], + ]), + &device, + ); + + let output = module.forward(input); + + let expected = TensorData::from([ + [ + [-1.1694, -0.5353, 0.7572], + [-0.1775, -0.6838, 1.8087], + [0.5205, -1.3107, -1.0723], + [-0.0459, 0.2351, 1.6734], + [-0.5796, 1.3218, -1.6544], + [-0.2744, 1.0406, 0.1459], + ], + [ + [0.2665, -0.3320, -0.8205], + [-0.2667, 2.0612, -0.9085], + [0.6681, 0.9102, 1.1345], + [-0.1453, -1.5287, -1.0389], + [0.4253, 1.5962, 0.4731], + [-1.0903, -0.0419, -1.3623], + ], + ]); + output + .to_data() + .assert_approx_eq::(&expected, tolerance); + } + + #[test] + fn display() { + let config = GroupNormConfig::new(3, 6); + let group_norm = config.init::(&Default::default()); + + assert_eq!( + format!("{group_norm}"), + "GroupNorm {num_groups: 3, num_channels: 6, epsilon: 0.00001, affine: true, params: 12}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/instance.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/instance.rs new file mode 100644 index 0000000..f0acc1c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/instance.rs @@ -0,0 +1,228 @@ +use burn_core as burn; + +use crate::norm::group_norm; +use burn::config::Config; +use burn::module::Initializer; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::module::{Module, Param}; +use burn::tensor::{Tensor, backend::Backend}; + +/// Configuration to create a [InstanceNorm](InstanceNorm) layer using the [init function](InstanceNormConfig::init). +#[derive(Debug, Config)] +pub struct InstanceNormConfig { + /// The number of channels expected in the input + pub num_channels: usize, + /// A value required for numerical stability. Default: 1e-5 + #[config(default = 1e-5)] + pub epsilon: f64, + /// A boolean value that when set to `true`, this module has learnable + /// per-channel affine parameters initialized to ones (for weights) + /// and zeros (for biases). Default: `true` + #[config(default = true)] + pub affine: bool, +} + +/// Applies Instance Normalization over a tensor as described in the paper [Instance Normalization](https://arxiv.org/abs/1607.08022) +/// +/// Should be created using [InstanceNormConfig](InstanceNormConfig). +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct InstanceNorm { + /// The learnable weight + pub gamma: Option>>, + /// The learnable bias + pub beta: Option>>, + /// The number of channels expected in the input + pub num_channels: usize, + /// A value required for numerical stability + pub epsilon: f64, + /// A boolean value that when set to `true`, this module has learnable + pub affine: bool, +} + +impl ModuleDisplay for InstanceNorm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("num_channels", &self.num_channels) + .add("epsilon", &self.epsilon) + .add("affine", &self.affine) + .optional() + } +} + +impl InstanceNormConfig { + /// Initialize a new [instance norm](InstanceNorm) module. + pub fn init(&self, device: &B::Device) -> InstanceNorm { + let (gamma, beta) = if self.affine { + let gamma = Initializer::Ones.init([self.num_channels], device); + let beta = Initializer::Zeros.init([self.num_channels], device); + + (Some(gamma), Some(beta)) + } else { + (None, None) + }; + + InstanceNorm { + gamma, + beta, + num_channels: self.num_channels, + epsilon: self.epsilon, + affine: self.affine, + } + } +} + +impl InstanceNorm { + /// Applies the forward pass on the input tensor. + /// + /// See also [InstanceNormConfig](InstanceNormConfig) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, num_channels, *]` + /// - output: `[batch_size, num_channels, *]` + pub fn forward(&self, input: Tensor) -> Tensor { + // Instance norm is equivalent to group norm when the number of groups is equal to the number of channels. + let num_groups = self.num_channels; + + let gamma = self.gamma.as_ref().map(|x| x.val()); + let beta = self.beta.as_ref().map(|x| x.val()); + + group_norm(input, gamma, beta, num_groups, self.epsilon, self.affine) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use alloc::format; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn instance_norm_forward_affine_false() { + let device = Default::default(); + let module = InstanceNormConfig::new(6) + .with_affine(false) + .init::(&device); + + let input = Tensor::::from_data( + TensorData::from([ + [ + [-0.3034, 0.2726, -0.9659], + [-1.1845, 1.4078, 0.9774], + [0.3963, -1.3738, 1.4125], + [1.0682, 0.3604, 0.3985], + [-0.4957, -0.4461, -0.9721], + [1.5157, -0.1546, -0.5596], + ], + [ + [-1.6698, -0.4040, -0.7927], + [0.3736, -0.0975, -0.1351], + [-0.9461, 0.5461, -0.6334], + [-1.0919, -0.1158, 0.1213], + [-0.9535, 0.1281, 0.4372], + [-0.2845, 0.3488, 0.5641], + ], + ]), + &device, + ); + + let output = module.forward(input); + + let expected = TensorData::from([ + [ + [0.0569, 1.1952, -1.2522], + [-1.3971, 0.8883, 0.5088], + [0.2183, -1.3192, 1.1009], + [1.4126, -0.7649, -0.6477], + [0.5999, 0.8091, -1.409], + [1.39, -0.4696, -0.9205], + ], + [ + [-1.3492, 1.0417, 0.3075], + [1.411, -0.6243, -0.7867], + [-0.9363, 1.386, -0.4497], + [-1.3899, 0.4692, 0.9208], + [-1.3822, 0.4319, 0.9503], + [-1.3714, 0.3868, 0.9846], + ], + ]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn instance_norm_forward_affine_true() { + let device = Default::default(); + let module = InstanceNormConfig::new(6) + .with_affine(true) + .init::(&device); + + let input = Tensor::::from_data( + TensorData::from([ + [ + [0.3345, 0.4429, 0.6639], + [0.5041, 0.4175, 0.8437], + [0.6159, 0.3758, 0.4071], + [0.5417, 0.5785, 0.7671], + [0.3837, 0.9883, 0.0420], + [0.4808, 0.8989, 0.6144], + ], + [ + [0.3930, 0.2098, 0.0602], + [0.2298, 0.9425, 0.0333], + [0.7409, 0.8172, 0.8879], + [0.4846, 0.0486, 0.2029], + [0.6741, 0.9765, 0.6864], + [0.2827, 0.5534, 0.2125], + ], + ]), + &device, + ); + + let output = module.forward(input); + + let expected = TensorData::from([ + [ + [-1.06458, -0.2738, 1.33838], + [-0.45848, -0.92929, 1.38777], + [1.40388, -0.84877, -0.55511], + [-0.88515, -0.51245, 1.3976], + [-0.22397, 1.32124, -1.09727], + [-1.05468, 1.34316, -0.28848], + ], + [ + [1.26372, -0.08229, -1.18144], + [-0.44049, 1.38403, -0.94354], + [-1.23828, 0.03109, 1.2072], + [1.32524, -1.08999, -0.23524], + [-0.75061, 1.4132, -0.66259], + [-0.45469, 1.38697, -0.93228], + ], + ]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = InstanceNormConfig::new(6); + let instance_norm = config.init::(&Default::default()); + + assert_eq!( + format!("{instance_norm}"), + "InstanceNorm {num_channels: 6, epsilon: 0.00001, affine: true, params: 12}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/layer.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/layer.rs new file mode 100644 index 0000000..98e5b47 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/layer.rs @@ -0,0 +1,232 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Content; +use burn::module::DisplaySettings; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::ModuleDisplay; +use burn::module::Param; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Configuration to create a [LayerNorm](LayerNorm) layer using the [init function](LayerNormConfig::init). +#[derive(Debug, Config)] +pub struct LayerNormConfig { + /// The size of the input features. + pub d_model: usize, + /// A value required for numerical stability. Default: 1e-5 + #[config(default = 1e-5)] + pub epsilon: f64, + /// If a bias (beta) should be applied during the normalization. Default: true + #[config(default = true)] + pub bias: bool, +} + +/// Applies Layer Normalization over an input tensor as described in the paper [Layer Normalization](https://arxiv.org/abs/1607.06450). +/// +/// `Y = norm(X) * γ + β` +/// +/// Where: +/// - `X` is the input tensor +/// - `Y` is the output tensor +/// - `γ` is the learnable weight (scale) +/// - `β` is the learnable bias (optional) +/// +/// Should be created using [LayerNormConfig](LayerNormConfig). +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct LayerNorm { + /// The learnable weight (scale). + pub gamma: Param>, + /// The learnable bias (optional). + pub beta: Option>>, + /// A value required for numerical stability. + epsilon: f64, +} + +impl LayerNormConfig { + /// Initialize a new [layer norm](LayerNorm) module. + pub fn init(&self, device: &B::Device) -> LayerNorm { + let gamma = Initializer::Ones.init([self.d_model], device); + let beta = if self.bias { + Some(Initializer::Zeros.init([self.d_model], device)) + } else { + None + }; + + LayerNorm { + gamma, + beta, + epsilon: self.epsilon, + } + } +} + +impl LayerNorm { + /// Applies the forward pass on the input tensor. + /// + /// See the [LayerNorm](LayerNorm) documentation for more information. + /// + /// # Shapes + /// + /// - input: `[..., any, d_model]` + /// - output: `[..., any, d_model]` + pub fn forward(&self, input: Tensor) -> Tensor { + let (var, mean) = input.clone().var_mean_bias(D - 1); + + let input_normalized = input.sub(mean).div(var.add_scalar(self.epsilon).sqrt()); + + let output = input_normalized.mul(self.gamma.val().unsqueeze()); + + match &self.beta { + Some(beta) => output.add(beta.val().unsqueeze()), + None => output, + } + } +} + +impl ModuleDisplay for LayerNorm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_model] = self.gamma.shape().dims(); + content + .add("d_model", &d_model) + .add("epsilon", &self.epsilon) + .add("bias", &self.beta.is_some()) + .optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::format; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[cfg(feature = "std")] + use crate::{TestAutodiffBackend, TestBackend}; + + #[cfg(not(feature = "std"))] + use crate::TestBackend; + + #[test] + fn layer_norm_forward() { + let device = Default::default(); + let module = LayerNormConfig::new(10).init::(&device); + let input = Tensor::::from_data( + TensorData::from([[ + -0.6897, -2.7106, 2.2222, -1.0330, -0.8933, 1.1765, 0.0601, 1.5252, -0.3630, 0.6728, + ]]), + &device, + ); + + let output = module.forward(input); + + let expected = TensorData::from([[ + -0.4990, -1.9680, 1.6178, -0.7486, -0.6470, 0.8576, 0.0461, 1.1111, -0.2614, 0.4915, + ]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn layer_norm_forward_large_epsilon() { + let device = Default::default(); + let module = LayerNormConfig::new(10) + .with_epsilon(1e-1) + .init::(&device); + let input = Tensor::::from_data( + TensorData::from([[ + -0.6897, -2.7106, 2.2222, -1.0330, -0.8933, 1.1765, 0.0601, 1.5252, -0.3630, 0.6728, + ]]), + &device, + ); + + let output = module.forward(input); + + let expected = TensorData::from([[ + -0.4863, -1.9180, 1.5766, -0.7295, -0.6305, 0.8358, 0.0449, 1.0828, -0.2548, 0.4790, + ]]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[cfg(feature = "std")] + #[test] + fn layer_norm_backward() { + let device = Default::default(); + let module = LayerNormConfig::new(2).init::(&device); + let tensor_1 = Tensor::::from_data( + TensorData::from([[0.0, 1.0], [3.0, 4.0]]), + &device, + ) + .require_grad(); + let tensor_2 = Tensor::::from_data( + TensorData::from([[6.0, 7.0], [9.0, 10.0]]), + &device, + ) + .require_grad(); + + let x = tensor_1.clone().matmul(tensor_2.clone()); + + let output = module.forward(x); + let grads = output.backward(); + + let tensor_1_grad = tensor_1.grad(&grads).unwrap(); + let tensor_2_grad = tensor_2.grad(&grads).unwrap(); + let gamma_grad = module.gamma.grad(&grads).unwrap(); + let beta_grad = module.beta.as_ref().unwrap().grad(&grads).unwrap(); + + let expected = TensorData::from([-2.0, 2.0]); + gamma_grad + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::from([2.0, 2.0]); + beta_grad + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::zeros::(tensor_1_grad.shape()); + tensor_1_grad + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + + let expected = TensorData::zeros::(tensor_2_grad.shape()); + tensor_2_grad + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = LayerNormConfig::new(6); + let layer_norm = config.init::(&Default::default()); + + assert_eq!( + format!("{layer_norm}"), + "LayerNorm {d_model: 6, epsilon: 0.00001, bias: true, params: 12}" + ); + } + + #[test] + fn display_no_bias() { + let config = LayerNormConfig::new(6).with_bias(false); + let layer_norm = config.init::(&Default::default()); + + assert_eq!( + format!("{layer_norm}"), + "LayerNorm {d_model: 6, epsilon: 0.00001, bias: false, params: 6}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/mod.rs new file mode 100644 index 0000000..d26fb95 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/mod.rs @@ -0,0 +1,28 @@ +//! # Normalization Layers +//! +//! Users who wish to provide an abstraction over swappable normalization +//! layers can use the [`Normalization`] wrapper, with support for: +//! * [`Normalization::Batch`] - [`BatchNorm`] +//! * [`Normalization::Group`] - [`GroupNorm`] +//! * [`Normalization::Instance`] - [`InstanceNorm`] +//! * [`Normalization::Layer`] - [`LayerNorm`] +//! * [`Normalization::Rms`] - [`RmsNorm`] +//! +//! [`NormalizationConfig`] can be used as a generic normalization policy: +//! * Construct a config with arbitrary input features (we suggest `0`). +//! * Clone and match that config to the target input layer, +//! using the [`NormalizationConfig::with_num_features()`] method. +pub(crate) mod batch; +pub(crate) mod group; +pub(crate) mod instance; +pub(crate) mod layer; +pub(crate) mod rms; + +mod normalization_wrapper; + +pub use batch::*; +pub use group::*; +pub use instance::*; +pub use layer::*; +pub use normalization_wrapper::*; +pub use rms::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/normalization_wrapper.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/normalization_wrapper.rs new file mode 100644 index 0000000..6122986 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/normalization_wrapper.rs @@ -0,0 +1,368 @@ +use burn_core as burn; + +use crate::{ + BatchNorm, BatchNormConfig, GroupNorm, GroupNormConfig, InstanceNorm, InstanceNormConfig, + LayerNorm, LayerNormConfig, RmsNorm, RmsNormConfig, +}; +use burn::prelude::{Config, Module}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// ['Normalization'] Configuration. +/// +/// The enum is non-exhaustive to prepare for future additions. +/// +/// Can be used as a generic configuration for normalization layers: +/// * Construct a config with arbitrary input features (we suggest `0`). +/// * Clone and match that config to the target input layer, +/// using the [`NormalizationConfig::with_num_features()`] method. +#[derive(Config, Debug)] +#[non_exhaustive] +pub enum NormalizationConfig { + /// ['BatchNorm'] Configuration. + Batch(BatchNormConfig), + + /// ['GroupNorm'] Configuration. + Group(GroupNormConfig), + + /// ['InstanceNorm'] Configuration. + Instance(InstanceNormConfig), + + /// ['LayerNorm'] Configuration. + Layer(LayerNormConfig), + + /// ['RmsNorm'] Configuration. + Rms(RmsNormConfig), +} + +impl From for NormalizationConfig { + fn from(config: BatchNormConfig) -> Self { + Self::Batch(config) + } +} + +impl From for NormalizationConfig { + fn from(config: GroupNormConfig) -> Self { + Self::Group(config) + } +} + +impl From for NormalizationConfig { + fn from(config: InstanceNormConfig) -> Self { + Self::Instance(config) + } +} + +impl From for NormalizationConfig { + fn from(config: LayerNormConfig) -> Self { + Self::Layer(config) + } +} + +impl From for NormalizationConfig { + fn from(config: RmsNormConfig) -> Self { + Self::Rms(config) + } +} + +impl NormalizationConfig { + /// Initialize a ['Norm'] layer. + pub fn init(&self, device: &B::Device) -> Normalization { + match self { + NormalizationConfig::Batch(config) => config.init(device).into(), + NormalizationConfig::Group(config) => config.init(device).into(), + NormalizationConfig::Instance(config) => config.init(device).into(), + NormalizationConfig::Layer(config) => config.init(device).into(), + NormalizationConfig::Rms(config) => config.init(device).into(), + } + } + + /// Set the number of features. + pub fn with_num_features(self, num_features: usize) -> Self { + match self { + NormalizationConfig::Batch(config) => BatchNormConfig { + num_features, + ..config + } + .into(), + NormalizationConfig::Group(config) => GroupNormConfig { + num_channels: num_features, + ..config + } + .into(), + NormalizationConfig::Instance(config) => InstanceNormConfig { + num_channels: num_features, + ..config + } + .into(), + NormalizationConfig::Layer(config) => LayerNormConfig { + d_model: num_features, + ..config + } + .into(), + NormalizationConfig::Rms(config) => RmsNormConfig { + d_model: num_features, + ..config + } + .into(), + } + } + + /// Get the number of features. + pub fn num_features(&self) -> usize { + match self { + NormalizationConfig::Batch(config) => config.num_features, + NormalizationConfig::Group(config) => config.num_channels, + NormalizationConfig::Instance(config) => config.num_channels, + NormalizationConfig::Layer(config) => config.d_model, + NormalizationConfig::Rms(config) => config.d_model, + } + } +} + +/// Normalization Layer Wrapper +/// +/// Provides support for built-in ``burn::nn::norm`` norm layers: +/// * [`Normalization::Batch`] - [`BatchNorm`] +/// * [`Normalization::Group`] - [`GroupNorm`] +/// * [`Normalization::Instance`] - [`InstanceNorm`] +/// * [`Normalization::Layer`] - [`LayerNorm`] +/// * [`Normalization::Rms`] - [`RmsNorm`] +/// +/// The enum is non-exhaustive, to prepare for future additions. +#[derive(Module, Debug)] +#[non_exhaustive] +pub enum Normalization { + /// [`BatchNorm`] layer. + Batch(BatchNorm), + + /// [`GroupNorm`] layer. + Group(GroupNorm), + + /// ['InstanceNorm'] layer. + Instance(InstanceNorm), + + /// [`LayerNorm`] layer. + Layer(LayerNorm), + + /// ['RmsNorm'] layer. + Rms(RmsNorm), +} + +impl From> for Normalization { + fn from(layer: BatchNorm) -> Self { + Self::Batch(layer) + } +} + +impl From> for Normalization { + fn from(layer: GroupNorm) -> Self { + Self::Group(layer) + } +} + +impl From> for Normalization { + fn from(layer: InstanceNorm) -> Self { + Self::Instance(layer) + } +} + +impl From> for Normalization { + fn from(layer: LayerNorm) -> Self { + Self::Layer(layer) + } +} + +impl From> for Normalization { + fn from(layer: RmsNorm) -> Self { + Self::Rms(layer) + } +} + +impl Normalization { + /// Applies normalization to a tensor. + /// + /// The normalization contract depends upon the wrapped norm layer; + /// but all norm layers assume an input of at least rank 2; + /// and produce an output of the same rank and shape. + pub fn forward(&self, input: Tensor) -> Tensor { + match self { + Normalization::Batch(norm) => norm.forward(input), + Normalization::Group(norm) => norm.forward(input), + Normalization::Instance(norm) => norm.forward(input), + Normalization::Layer(norm) => norm.forward(input), + Normalization::Rms(norm) => norm.forward(input), + } + } + + /// Get the number of features. + pub fn num_features(&self) -> usize { + match self { + Normalization::Batch(norm) => norm.gamma.shape()[0], + Normalization::Group(norm) => norm.num_channels, + Normalization::Instance(norm) => norm.num_channels, + Normalization::Layer(norm) => norm.gamma.shape()[0], + Normalization::Rms(norm) => norm.gamma.shape()[0], + } + } +} + +#[cfg(feature = "std")] +#[cfg(test)] +mod tests { + use super::*; + use crate::TestAutodiffBackend; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_match_feature_size() { + let config: NormalizationConfig = BatchNormConfig::new(0).into(); + assert_eq!(config.num_features(), 0); + let config = config.with_num_features(12); + assert_eq!(config.num_features(), 12); + + let config: NormalizationConfig = GroupNormConfig::new(4, 0).into(); + assert_eq!(config.num_features(), 0); + let config = config.with_num_features(12); + assert_eq!(config.num_features(), 12); + + let config: NormalizationConfig = InstanceNormConfig::new(0).into(); + assert_eq!(config.num_features(), 0); + let config = config.with_num_features(12); + assert_eq!(config.num_features(), 12); + + let config: NormalizationConfig = LayerNormConfig::new(0).into(); + assert_eq!(config.num_features(), 0); + let config = config.with_num_features(12); + assert_eq!(config.num_features(), 12); + + let config: NormalizationConfig = RmsNormConfig::new(0).into(); + assert_eq!(config.num_features(), 0); + let config = config.with_num_features(12); + assert_eq!(config.num_features(), 12); + } + + #[test] + fn test_batch_norm() { + type B = TestAutodiffBackend; + let device = Default::default(); + + let num_features = 12; + let input: Tensor = Tensor::ones([2, num_features, 3, 4], &device); + + let config: NormalizationConfig = BatchNormConfig::new(12).into(); + + let layer: Normalization = config.init(&device); + assert_eq!(layer.num_features(), 12); + + let expected = match &layer { + Normalization::Batch(inner) => inner.forward(input.clone()), + _ => panic!("Unexpected layer type"), + }; + + let output = layer.forward(input); + + output.to_data().assert_eq(&expected.to_data(), true); + } + + #[test] + fn test_group_norm() { + type B = TestAutodiffBackend; + let device = Default::default(); + + let num_features = 12; + let input: Tensor = Tensor::ones([2, num_features, 3, 4], &device); + + let config: NormalizationConfig = GroupNormConfig::new(3, num_features).into(); + + let layer: Normalization = config.init(&device); + assert_eq!(layer.num_features(), 12); + + let expected = match &layer { + Normalization::Group(inner) => inner.forward(input.clone()), + _ => panic!("Unexpected layer type"), + }; + + let output = layer.forward(input); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } + + #[test] + fn test_instance_norm() { + type B = TestAutodiffBackend; + let device = Default::default(); + + let num_features = 12; + let input: Tensor = Tensor::ones([2, num_features, 3, 4], &device); + + let config: NormalizationConfig = InstanceNormConfig::new(num_features).into(); + + let layer: Normalization = config.init(&device); + assert_eq!(layer.num_features(), 12); + + let expected = match &layer { + Normalization::Instance(inner) => inner.forward(input.clone()), + _ => panic!("Unexpected layer type"), + }; + + let output = layer.forward(input); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } + + #[test] + fn test_layer_norm() { + type B = TestAutodiffBackend; + let device = Default::default(); + + let num_features = 12; + let input: Tensor = Tensor::ones([2, 3, 4, num_features], &device); + + let config: NormalizationConfig = LayerNormConfig::new(num_features).into(); + + let layer: Normalization = config.init(&device); + assert_eq!(layer.num_features(), 12); + + let expected = match &layer { + Normalization::Layer(inner) => inner.forward(input.clone()), + _ => panic!("Unexpected layer type"), + }; + + let output = layer.forward(input); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } + + #[test] + fn test_rms_norm() { + type B = TestAutodiffBackend; + let device = Default::default(); + + let num_features = 12; + let input: Tensor = Tensor::ones([2, 3, 4, num_features], &device); + + let config: NormalizationConfig = RmsNormConfig::new(num_features).into(); + + let layer: Normalization = config.init(&device); + assert_eq!(layer.num_features(), 12); + + let expected = match &layer { + Normalization::Rms(inner) => inner.forward(input.clone()), + _ => panic!("Unexpected layer type"), + }; + + let output = layer.forward(input); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/rms.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/rms.rs new file mode 100644 index 0000000..6f5c6ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/norm/rms.rs @@ -0,0 +1,134 @@ +use burn::tensor::DType; + +use burn_core as burn; + +use burn::config::Config; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::Param; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Configuration to create a [RMS Norm](RmsNorm) layer using the [init function](RmsNormConfig::init). +#[derive(Config, Debug)] +pub struct RmsNormConfig { + /// The size of the input features. + pub d_model: usize, + /// A value required for numerical stability. Default: 1e-5 + #[config(default = 1e-5)] + pub epsilon: f64, +} + +impl RmsNormConfig { + /// Initialize a new [RMS Norm](RmsNorm) module. + /// + /// # Panics + /// + /// Panics if `epsilon` is not positive. + pub fn init(&self, device: &B::Device) -> RmsNorm { + assert!(self.epsilon > 0.0, "epsilon must be positive."); + + let gamma = Initializer::Ones.init([self.d_model], device); + + RmsNorm { + gamma, + epsilon: self.epsilon, + } + } +} + +/// Applies RMS Normalization over an input tensor along the last dimension. +/// +/// `Y = X / sqrt(mean(X^2) + eps) * gamma` +/// +/// Where: +/// - `X` is the input tensor +/// - `Y` is the output tensor +/// - `gamma` is the learnable weight +/// - `mean` is the mean operation +/// - `eps` is a small value to avoid division by zero. +/// +/// Should be created using the [RmsNormConfig](RmsNormConfig) configuration. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct RmsNorm { + /// The learnable parameter to scale the normalized tensor + pub gamma: Param>, + /// A value required for numerical stability + pub epsilon: f64, +} + +impl RmsNorm { + /// Applies the forward pass on the input tensor. + /// + /// See the [RmsNorm](RmsNorm) documentation for more information. + /// + /// # Shapes + /// + /// - input: `[..., any, d_model]` + /// - output: `[..., any, d_model]` + pub fn forward(&self, x: Tensor) -> Tensor { + // Calculate the root-mean-square norm of the input tensor along the last dimension + let dtype = x.dtype(); + let rms = (x.clone().cast(DType::F32).square().mean_dim(D - 1) + self.epsilon).sqrt(); + (x / rms.cast(dtype)) * self.gamma.val().unsqueeze() + } +} + +impl ModuleDisplay for RmsNorm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_model] = self.gamma.shape().dims(); + content + .add("d_model", &d_model) + .add("epsilon", &self.epsilon) + .optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use alloc::format; + use burn::tensor::TensorData; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn rms_norm_forward() { + let device = Default::default(); + let module = RmsNormConfig::new(3) + .with_epsilon(1e-5) + .init::(&device); + + let input = Tensor::arange(0..9, &device).float().reshape([3, 3]); + let output = module.forward(input); + + let expected = TensorData::from([ + [0.0000, 0.7746, 1.5492], + [0.7348, 0.9798, 1.2247], + [0.8514, 0.9933, 1.1352], + ]); + output + .to_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn display() { + let config = RmsNormConfig::new(6); + let layer_norm = config.init::(&Default::default()); + + assert_eq!( + format!("{layer_norm}"), + "RmsNorm {d_model: 6, epsilon: 0.00001, params: 6}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool1d.rs new file mode 100644 index 0000000..d591bf6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool1d.rs @@ -0,0 +1,77 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +use burn::tensor::module::adaptive_avg_pool1d; + +/// Configuration to create a [1D adaptive avg pooling](AdaptiveAvgPool1d) layer using the [init function](AdaptiveAvgPool1dConfig::init). +#[derive(Config, Debug)] +pub struct AdaptiveAvgPool1dConfig { + /// The size of the output. + pub output_size: usize, +} + +/// Applies a 1D adaptive avg pooling over input tensors. +/// +/// Should be created with [AdaptiveAvgPool1dConfig]. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct AdaptiveAvgPool1d { + /// The size of the output. + pub output_size: usize, +} + +impl ModuleDisplay for AdaptiveAvgPool1d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content.add("output_size", &self.output_size).optional() + } +} + +impl AdaptiveAvgPool1dConfig { + /// Initialize a new [adaptive avg pool 1d](AdaptiveAvgPool1d) module. + pub fn init(&self) -> AdaptiveAvgPool1d { + AdaptiveAvgPool1d { + output_size: self.output_size, + } + } +} + +impl AdaptiveAvgPool1d { + /// Applies the forward pass on the input tensor. + /// + /// See [adaptive_avg_pool1d](burn::tensor::module::adaptive_avg_pool1d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels, length]` + /// - output: `[batch_size, channels, length_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + adaptive_avg_pool1d(input, self.output_size) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let config = AdaptiveAvgPool1dConfig::new(3); + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "AdaptiveAvgPool1d {output_size: 3}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool2d.rs new file mode 100644 index 0000000..72dd80b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/adaptive_avg_pool2d.rs @@ -0,0 +1,79 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +use burn::tensor::module::adaptive_avg_pool2d; + +/// Configuration to create a [2D adaptive avg pooling](AdaptiveAvgPool2d) layer using the [init function](AdaptiveAvgPool2dConfig::init). +#[derive(Config, Debug)] +pub struct AdaptiveAvgPool2dConfig { + /// The size of the output. + pub output_size: [usize; 2], +} + +/// Applies a 2D adaptive avg pooling over input tensors. +/// +/// Should be created with [AdaptiveAvgPool2dConfig]. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct AdaptiveAvgPool2d { + /// The size of the output. + pub output_size: [usize; 2], +} + +impl ModuleDisplay for AdaptiveAvgPool2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let output_size = alloc::format!("{:?}", self.output_size); + + content.add("output_size", &output_size).optional() + } +} + +impl AdaptiveAvgPool2dConfig { + /// Initialize a new [adaptive avg pool 2d](AdaptiveAvgPool2d) module. + pub fn init(&self) -> AdaptiveAvgPool2d { + AdaptiveAvgPool2d { + output_size: self.output_size, + } + } +} + +impl AdaptiveAvgPool2d { + /// Applies the forward pass on the input tensor. + /// + /// See [adaptive_avg_pool2d](burn::tensor::module::adaptive_avg_pool2d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels, height_in, width_in]` + /// - output: `[batch_size, channels, height_out, width_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + adaptive_avg_pool2d(input, self.output_size) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let config = AdaptiveAvgPool2dConfig::new([3, 3]); + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "AdaptiveAvgPool2d {output_size: [3, 3]}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool1d.rs new file mode 100644 index 0000000..89205a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool1d.rs @@ -0,0 +1,220 @@ +use burn_core as burn; + +use crate::PaddingConfig1d; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::module::{Ignored, Module}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::ops::PadMode; + +use burn::tensor::module::avg_pool1d; + +/// Configuration to create a [1D avg pooling](AvgPool1d) layer using the [init function](AvgPool1dConfig::init). +#[derive(Config, Debug)] +pub struct AvgPool1dConfig { + /// The size of the kernel. + pub kernel_size: usize, + /// The stride. + #[config(default = "kernel_size")] + pub stride: usize, + /// The padding configuration. + /// + /// Supports symmetric and asymmetric padding. `Same` padding with even kernel sizes + /// will automatically use asymmetric padding to preserve input dimensions. + #[config(default = "PaddingConfig1d::Valid")] + pub padding: PaddingConfig1d, + /// If the padding is counted in the denominator when computing the average. + #[config(default = "true")] + pub count_include_pad: bool, + /// If true, use ceiling instead of floor for output size calculation. + #[config(default = "false")] + pub ceil_mode: bool, +} + +/// Applies a 1D avg pooling over input tensors. +/// +/// Should be created with [AvgPool1dConfig](AvgPool1dConfig). +/// +/// # Remarks +/// +/// The zero-padding values will be included in the calculation +/// of the average. This means that the zeros are counted as +/// legitimate values, and they contribute to the denominator +/// when calculating the average. This is equivalent to +/// `torch.nn.AvgPool2d` with `count_include_pad=True`. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct AvgPool1d { + /// The stride. + pub stride: usize, + /// The size of the kernel. + pub kernel_size: usize, + /// The padding configuration. + pub padding: Ignored, + /// If the padding is counted in the denominator when computing the average. + pub count_include_pad: bool, + /// If true, use ceiling instead of floor for output size calculation. + pub ceil_mode: bool, +} + +impl ModuleDisplay for AvgPool1d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("kernel_size", &self.kernel_size) + .add("stride", &self.stride) + .add("padding", &self.padding) + .add("count_include_pad", &self.count_include_pad) + .add("ceil_mode", &self.ceil_mode) + .optional() + } +} + +impl AvgPool1dConfig { + /// Initialize a new [avg pool 1d](AvgPool1d) module. + pub fn init(&self) -> AvgPool1d { + AvgPool1d { + stride: self.stride, + kernel_size: self.kernel_size, + padding: Ignored(self.padding.clone()), + count_include_pad: self.count_include_pad, + ceil_mode: self.ceil_mode, + } + } +} + +impl AvgPool1d { + /// Applies the forward pass on the input tensor. + /// + /// See [avg_pool1d](burn::tensor::module::avg_pool1d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels, length_in]` + /// - output: `[batch_size, channels, length_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + let [_batch_size, _channels, length] = input.dims(); + + // Calculate padding as pair - handles Same, Valid, and Explicit uniformly + let (left, right) = + self.padding + .calculate_padding_1d_pair(length, self.kernel_size, self.stride); + + // TODO: Move asymmetric padding to functional level via PoolOptions + // See: https://github.com/tracel-ai/burn/issues/4362 + // Handle asymmetric padding by applying explicit pad operation first + if left != right { + // Burn's pad takes (left, right, top, bottom) for the last two dimensions + // For 1D (NCL format), we only pad L (last dim), so top/bottom = 0 + let padded = input.pad((left, right, 0, 0), PadMode::Constant(0.0)); + // Use zero padding for the pool operation since we already padded + avg_pool1d( + padded, + self.kernel_size, + self.stride, + 0, + self.count_include_pad, + self.ceil_mode, + ) + } else { + // Symmetric padding + avg_pool1d( + input, + self.kernel_size, + self.stride, + left, + self.count_include_pad, + self.ceil_mode, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use rstest::rstest; + + #[test] + fn same_with_even_kernel_uses_asymmetric_padding() { + let device = Default::default(); + let config = AvgPool1dConfig::new(2) + .with_stride(1) + .with_padding(PaddingConfig1d::Same); + let pool = config.init(); + + // Input: [batch=1, channels=2, length=5] + let input = Tensor::::ones([1, 2, 5], &device); + let output = pool.forward(input); + + // Same padding should preserve spatial dimensions + assert_eq!(output.dims(), [1, 2, 5]); + } + + #[test] + fn display() { + let config = AvgPool1dConfig::new(3); + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "AvgPool1d {kernel_size: 3, stride: 3, padding: Valid, count_include_pad: true, ceil_mode: false}" + ); + } + + #[rstest] + #[case(1)] + #[case(2)] + fn default_strides_match_kernel_size(#[case] kernel_size: usize) { + let config = AvgPool1dConfig::new(kernel_size); + + assert_eq!( + config.stride, kernel_size, + "Expected stride ({:?}) to match kernel size ({:?}) in default AvgPool1dConfig::new constructor", + config.stride, config.kernel_size + ); + } + + #[test] + fn asymmetric_padding_forward() { + let device = Default::default(); + // Create avg pool with asymmetric padding: left=1, right=2 + let config = AvgPool1dConfig::new(3) + .with_stride(1) + .with_padding(PaddingConfig1d::Explicit(1, 2)); + let pool = config.init(); + + // Input: [batch=1, channels=2, length=4] + let input = Tensor::::ones([1, 2, 4], &device); + let output = pool.forward(input); + + // With asymmetric padding (1, 2), input length 4 becomes 4+1+2=7 + // Output length = (7 - 3) / 1 + 1 = 5 + assert_eq!(output.dims(), [1, 2, 5]); + } + + #[test] + fn symmetric_explicit_padding_forward() { + let device = Default::default(); + // Create avg pool with symmetric explicit padding: left=2, right=2 + let config = AvgPool1dConfig::new(3) + .with_stride(1) + .with_padding(PaddingConfig1d::Explicit(2, 2)); + let pool = config.init(); + + // Input: [batch=1, channels=2, length=4] + let input = Tensor::::ones([1, 2, 4], &device); + let output = pool.forward(input); + + // With symmetric padding (2, 2), input length 4 becomes 4+2+2=8 + // Output length = (8 - 3) / 1 + 1 = 6 + assert_eq!(output.dims(), [1, 2, 6]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool2d.rs new file mode 100644 index 0000000..9929672 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/avg_pool2d.rs @@ -0,0 +1,223 @@ +use burn_core as burn; + +use crate::PaddingConfig2d; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::module::{Ignored, Module}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::ops::PadMode; + +use burn::tensor::module::avg_pool2d; + +/// Configuration to create a [2D avg pooling](AvgPool2d) layer using the [init function](AvgPool2dConfig::init). +#[derive(Config, Debug)] +pub struct AvgPool2dConfig { + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The strides. + #[config(default = "kernel_size")] + pub strides: [usize; 2], + /// The padding configuration. + /// + /// Supports symmetric and asymmetric padding. `Same` padding with even kernel sizes + /// will automatically use asymmetric padding to preserve input dimensions. + #[config(default = "PaddingConfig2d::Valid")] + pub padding: PaddingConfig2d, + /// If the padding is counted in the denominator when computing the average. + #[config(default = "true")] + pub count_include_pad: bool, + /// If true, use ceiling instead of floor for output size calculation. + #[config(default = "false")] + pub ceil_mode: bool, +} + +/// Applies a 2D avg pooling over input tensors. +/// +/// Should be created with [AvgPool2dConfig](AvgPool2dConfig). +/// +/// # Remarks +/// +/// The zero-padding values will be included in the calculation +/// of the average. This means that the zeros are counted as +/// legitimate values, and they contribute to the denominator +/// when calculating the average. This is equivalent to +/// `torch.nn.AvgPool2d` with `count_include_pad=True`. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct AvgPool2d { + /// Stride of the pooling. + pub stride: [usize; 2], + /// Size of the kernel. + pub kernel_size: [usize; 2], + /// Padding configuration. + pub padding: Ignored, + /// If the padding is counted in the denominator when computing the average. + pub count_include_pad: bool, + /// If true, use ceiling instead of floor for output size calculation. + pub ceil_mode: bool, +} + +impl ModuleDisplay for AvgPool2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("kernel_size", &alloc::format!("{:?}", &self.kernel_size)) + .add("stride", &alloc::format!("{:?}", &self.stride)) + .add("padding", &self.padding) + .add("count_include_pad", &self.count_include_pad) + .add("ceil_mode", &self.ceil_mode) + .optional() + } +} + +impl AvgPool2dConfig { + /// Initialize a new [avg pool 2d](AvgPool2d) module. + pub fn init(&self) -> AvgPool2d { + AvgPool2d { + stride: self.strides, + kernel_size: self.kernel_size, + padding: Ignored(self.padding.clone()), + count_include_pad: self.count_include_pad, + ceil_mode: self.ceil_mode, + } + } +} + +impl AvgPool2d { + /// Applies the forward pass on the input tensor. + /// + /// See [avg_pool2d](burn::tensor::module::avg_pool2d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels, height_in, width_in]` + /// - output: `[batch_size, channels, height_out, width_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + let [_batch_size, _channels_in, height_in, width_in] = input.dims(); + + // Calculate padding as pairs - handles Same, Valid, and Explicit uniformly + let ((top, bottom), (left, right)) = self.padding.calculate_padding_2d_pairs( + height_in, + width_in, + &self.kernel_size, + &self.stride, + ); + + // TODO: Move asymmetric padding to functional level via PoolOptions + // See: https://github.com/tracel-ai/burn/issues/4362 + // Handle asymmetric padding by applying explicit pad operation first + if top != bottom || left != right { + // Burn's pad takes (left, right, top, bottom) for the last two dimensions + let padded = input.pad((left, right, top, bottom), PadMode::Constant(0.0)); + // Use zero padding for the pool operation since we already padded + avg_pool2d( + padded, + self.kernel_size, + self.stride, + [0, 0], + self.count_include_pad, + self.ceil_mode, + ) + } else { + // Symmetric padding + avg_pool2d( + input, + self.kernel_size, + self.stride, + [top, left], + self.count_include_pad, + self.ceil_mode, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use rstest::rstest; + + #[test] + fn same_with_even_kernel_uses_asymmetric_padding() { + let device = Default::default(); + let config = AvgPool2dConfig::new([2, 2]) + .with_strides([1, 1]) + .with_padding(PaddingConfig2d::Same); + let pool = config.init(); + + // Input: [batch=1, channels=2, height=5, width=5] + let input = Tensor::::ones([1, 2, 5, 5], &device); + let output = pool.forward(input); + + // Same padding should preserve spatial dimensions + assert_eq!(output.dims(), [1, 2, 5, 5]); + } + + #[test] + fn display() { + let config = AvgPool2dConfig::new([3, 3]); + + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "AvgPool2d {kernel_size: [3, 3], stride: [3, 3], padding: Valid, count_include_pad: true, ceil_mode: false}" + ); + } + + #[rstest] + #[case([2, 2])] + #[case([1, 2])] + fn default_strides_match_kernel_size(#[case] kernel_size: [usize; 2]) { + let config = AvgPool2dConfig::new(kernel_size); + + assert_eq!( + config.strides, kernel_size, + "Expected strides ({:?}) to match kernel size ({:?}) in default AvgPool2dConfig::new constructor", + config.strides, config.kernel_size + ); + } + + #[test] + fn asymmetric_padding_forward() { + let device = Default::default(); + // Create avg pool with asymmetric padding: top=1, left=2, bottom=3, right=4 + let config = AvgPool2dConfig::new([3, 3]) + .with_strides([1, 1]) + .with_padding(PaddingConfig2d::Explicit(1, 2, 3, 4)); + let pool = config.init(); + + // Input: [batch=1, channels=2, height=4, width=5] + let input = Tensor::::ones([1, 2, 4, 5], &device); + let output = pool.forward(input); + + // Height: 4 + 1 + 3 = 8, output = (8 - 3) / 1 + 1 = 6 + // Width: 5 + 2 + 4 = 11, output = (11 - 3) / 1 + 1 = 9 + assert_eq!(output.dims(), [1, 2, 6, 9]); + } + + #[test] + fn symmetric_explicit_padding_forward() { + let device = Default::default(); + // Create avg pool with symmetric explicit padding: top=2, left=2, bottom=2, right=2 + let config = AvgPool2dConfig::new([3, 3]) + .with_strides([1, 1]) + .with_padding(PaddingConfig2d::Explicit(2, 2, 2, 2)); + let pool = config.init(); + + // Input: [batch=1, channels=2, height=4, width=5] + let input = Tensor::::ones([1, 2, 4, 5], &device); + let output = pool.forward(input); + + // Height: 4 + 2 + 2 = 8, output = (8 - 3) / 1 + 1 = 6 + // Width: 5 + 2 + 2 = 9, output = (9 - 3) / 1 + 1 = 7 + assert_eq!(output.dims(), [1, 2, 6, 7]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool1d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool1d.rs new file mode 100644 index 0000000..2dc5dd9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool1d.rs @@ -0,0 +1,214 @@ +use burn_core as burn; + +use crate::PaddingConfig1d; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::module::{Ignored, Module}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::ops::PadMode; + +use burn::tensor::module::max_pool1d; + +/// Configuration to create a [1D max pooling](MaxPool1d) layer using the [init function](MaxPool1dConfig::init). +#[derive(Config, Debug)] +pub struct MaxPool1dConfig { + /// The size of the kernel. + pub kernel_size: usize, + /// The stride. + #[config(default = "kernel_size")] + pub stride: usize, + /// The padding configuration. + /// + /// Supports symmetric and asymmetric padding. `Same` padding with even kernel sizes + /// will automatically use asymmetric padding to preserve input dimensions. + #[config(default = "PaddingConfig1d::Valid")] + pub padding: PaddingConfig1d, + /// The dilation. + #[config(default = "1")] + pub dilation: usize, + /// If true, use ceiling instead of floor for output size calculation. + #[config(default = "false")] + pub ceil_mode: bool, +} + +/// Applies a 1D max pooling over input tensors. +/// +/// Should be created with [MaxPool1dConfig](MaxPool1dConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct MaxPool1d { + /// The stride. + pub stride: usize, + /// The size of the kernel. + pub kernel_size: usize, + /// The padding configuration. + pub padding: Ignored, + /// The dilation. + pub dilation: usize, + /// If true, use ceiling instead of floor for output size calculation. + pub ceil_mode: bool, +} + +impl ModuleDisplay for MaxPool1d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("kernel_size", &self.kernel_size) + .add("stride", &self.stride) + .add("padding", &self.padding) + .add("dilation", &self.dilation) + .add("ceil_mode", &self.ceil_mode) + .optional() + } +} + +impl MaxPool1dConfig { + /// Initialize a new [max pool 1d](MaxPool1d) module. + pub fn init(&self) -> MaxPool1d { + MaxPool1d { + stride: self.stride, + kernel_size: self.kernel_size, + padding: Ignored(self.padding.clone()), + dilation: self.dilation, + ceil_mode: self.ceil_mode, + } + } +} + +impl MaxPool1d { + /// Applies the forward pass on the input tensor. + /// + /// See [max_pool1d](burn::tensor::module::max_pool1d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels, length_in]` + /// - output: `[batch_size, channels, length_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + let [_batch_size, _channels, length] = input.dims(); + + // Calculate padding as pair - handles Same, Valid, and Explicit uniformly + let (left, right) = + self.padding + .calculate_padding_1d_pair(length, self.kernel_size, self.stride); + + // TODO: Move asymmetric padding to functional level via PoolOptions + // See: https://github.com/tracel-ai/burn/issues/4362 + // Handle asymmetric padding by applying explicit pad operation first + if left != right { + // For 1D (NCL format), pad the length dimension with (left, right) + // and no padding for channel dimension (top=0, bottom=0) + // Use -inf for max pooling so padded values don't affect the max + let padded = input.pad((left, right, 0, 0), PadMode::Constant(f32::NEG_INFINITY)); + // Use zero padding for the pool operation since we already padded + max_pool1d( + padded, + self.kernel_size, + self.stride, + 0, + self.dilation, + self.ceil_mode, + ) + } else { + // Symmetric padding + max_pool1d( + input, + self.kernel_size, + self.stride, + left, + self.dilation, + self.ceil_mode, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use rstest::rstest; + + #[test] + fn same_with_even_kernel_uses_asymmetric_padding() { + let device = Default::default(); + let config = MaxPool1dConfig::new(2) + .with_stride(1) + .with_padding(PaddingConfig1d::Same); + let pool = config.init(); + + // Input: [batch=1, channels=2, length=5] + let input = Tensor::::ones([1, 2, 5], &device); + let output = pool.forward(input); + + // Same padding should preserve spatial dimensions + assert_eq!(output.dims(), [1, 2, 5]); + } + + #[test] + fn display() { + let config = MaxPool1dConfig::new(3); + + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "MaxPool1d {kernel_size: 3, stride: 3, padding: Valid, dilation: 1, ceil_mode: false}" + ); + } + + #[rstest] + #[case(1)] + #[case(2)] + fn default_strides_match_kernel_size(#[case] kernel_size: usize) { + let config = MaxPool1dConfig::new(kernel_size); + + assert_eq!( + config.stride, kernel_size, + "Expected stride ({:?}) to match kernel size ({:?}) in default MaxPool1dConfig::new constructor", + config.stride, config.kernel_size + ); + } + + #[test] + fn asymmetric_padding_forward() { + let device = Default::default(); + // Create max pool with asymmetric padding: left=1, right=2 + let config = MaxPool1dConfig::new(3) + .with_stride(1) + .with_padding(PaddingConfig1d::Explicit(1, 2)); + let pool = config.init(); + + // Input: [batch=1, channels=2, length=4] + let input = Tensor::::ones([1, 2, 4], &device); + let output = pool.forward(input); + + // With asymmetric padding (1, 2), input length 4 becomes 4+1+2=7 + // Output length = (7 - 3) / 1 + 1 = 5 + assert_eq!(output.dims(), [1, 2, 5]); + } + + #[test] + fn symmetric_explicit_padding_forward() { + let device = Default::default(); + // Create max pool with symmetric explicit padding: left=2, right=2 + let config = MaxPool1dConfig::new(3) + .with_stride(1) + .with_padding(PaddingConfig1d::Explicit(2, 2)); + let pool = config.init(); + + // Input: [batch=1, channels=2, length=4] + let input = Tensor::::ones([1, 2, 4], &device); + let output = pool.forward(input); + + // With symmetric padding (2, 2), input length 4 becomes 4+2+2=8 + // Output length = (8 - 3) / 1 + 1 = 6 + assert_eq!(output.dims(), [1, 2, 6]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool2d.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool2d.rs new file mode 100644 index 0000000..3acb79a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/max_pool2d.rs @@ -0,0 +1,219 @@ +use burn_core as burn; + +use crate::PaddingConfig2d; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::module::{Ignored, Module}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::ops::PadMode; + +use burn::tensor::module::max_pool2d; + +/// Configuration to create a [2D max pooling](MaxPool2d) layer using the [init function](MaxPool2dConfig::init). +#[derive(Debug, Config)] +pub struct MaxPool2dConfig { + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The strides. + #[config(default = "kernel_size")] + pub strides: [usize; 2], + /// The padding configuration. + /// + /// Supports symmetric and asymmetric padding. `Same` padding with even kernel sizes + /// will automatically use asymmetric padding to preserve input dimensions. + #[config(default = "PaddingConfig2d::Valid")] + pub padding: PaddingConfig2d, + /// The dilation. + #[config(default = "[1, 1]")] + pub dilation: [usize; 2], + /// If true, use ceiling instead of floor for output size calculation. + #[config(default = "false")] + pub ceil_mode: bool, +} + +/// Applies a 2D max pooling over input tensors. +/// +/// Should be created with [MaxPool2dConfig](MaxPool2dConfig). +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct MaxPool2d { + /// The strides. + pub stride: [usize; 2], + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The padding configuration. + pub padding: Ignored, + /// The dilation. + pub dilation: [usize; 2], + /// If true, use ceiling instead of floor for output size calculation. + pub ceil_mode: bool, +} + +impl ModuleDisplay for MaxPool2d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("kernel_size", &alloc::format!("{:?}", &self.kernel_size)) + .add("stride", &alloc::format!("{:?}", &self.stride)) + .add("padding", &self.padding) + .add("dilation", &alloc::format!("{:?}", &self.dilation)) + .add("ceil_mode", &self.ceil_mode) + .optional() + } +} + +impl MaxPool2dConfig { + /// Initialize a new [max pool 2d](MaxPool2d) module. + pub fn init(&self) -> MaxPool2d { + MaxPool2d { + stride: self.strides, + kernel_size: self.kernel_size, + padding: Ignored(self.padding.clone()), + dilation: self.dilation, + ceil_mode: self.ceil_mode, + } + } +} + +impl MaxPool2d { + /// Applies the forward pass on the input tensor. + /// + /// See [max_pool2d](burn::tensor::module::max_pool2d) for more information. + /// + /// # Shapes + /// + /// - input: `[batch_size, channels, height_in, width_in]` + /// - output: `[batch_size, channels, height_out, width_out]` + pub fn forward(&self, input: Tensor) -> Tensor { + let [_batch_size, _channels_in, height_in, width_in] = input.dims(); + + // Calculate padding as pairs - handles Same, Valid, and Explicit uniformly + let ((top, bottom), (left, right)) = self.padding.calculate_padding_2d_pairs( + height_in, + width_in, + &self.kernel_size, + &self.stride, + ); + + // TODO: Move asymmetric padding to functional level via PoolOptions + // See: https://github.com/tracel-ai/burn/issues/4362 + // Handle asymmetric padding by applying explicit pad operation first + if top != bottom || left != right { + // Burn's pad takes (left, right, top, bottom) for the last two dimensions + // Use -inf for max pooling so padded values don't affect the max + let padded = input.pad( + (left, right, top, bottom), + PadMode::Constant(f32::NEG_INFINITY), + ); + // Use zero padding for the pool operation since we already padded + max_pool2d( + padded, + self.kernel_size, + self.stride, + [0, 0], + self.dilation, + self.ceil_mode, + ) + } else { + // Symmetric padding + max_pool2d( + input, + self.kernel_size, + self.stride, + [top, left], + self.dilation, + self.ceil_mode, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use rstest::rstest; + + #[test] + fn same_with_even_kernel_uses_asymmetric_padding() { + let device = Default::default(); + let config = MaxPool2dConfig::new([2, 2]) + .with_strides([1, 1]) + .with_padding(PaddingConfig2d::Same); + let pool = config.init(); + + // Input: [batch=1, channels=2, height=5, width=5] + let input = Tensor::::ones([1, 2, 5, 5], &device); + let output = pool.forward(input); + + // Same padding should preserve spatial dimensions + assert_eq!(output.dims(), [1, 2, 5, 5]); + } + + #[test] + fn display() { + let config = MaxPool2dConfig::new([3, 3]); + + let layer = config.init(); + + assert_eq!( + alloc::format!("{layer}"), + "MaxPool2d {kernel_size: [3, 3], stride: [3, 3], padding: Valid, dilation: [1, 1], ceil_mode: false}" + ); + } + + #[rstest] + #[case([2, 2])] + #[case([1, 2])] + fn default_strides_match_kernel_size(#[case] kernel_size: [usize; 2]) { + let config = MaxPool2dConfig::new(kernel_size); + + assert_eq!( + config.strides, kernel_size, + "Expected strides ({:?}) to match kernel size ({:?}) in default MaxPool2dConfig::new constructor", + config.strides, config.kernel_size + ); + } + + #[test] + fn asymmetric_padding_forward() { + let device = Default::default(); + // Create max pool with asymmetric padding: top=1, left=2, bottom=3, right=4 + let config = MaxPool2dConfig::new([3, 3]) + .with_strides([1, 1]) + .with_padding(PaddingConfig2d::Explicit(1, 2, 3, 4)); + let pool = config.init(); + + // Input: [batch=1, channels=2, height=4, width=5] + let input = Tensor::::ones([1, 2, 4, 5], &device); + let output = pool.forward(input); + + // Height: 4 + 1 + 3 = 8, output = (8 - 3) / 1 + 1 = 6 + // Width: 5 + 2 + 4 = 11, output = (11 - 3) / 1 + 1 = 9 + assert_eq!(output.dims(), [1, 2, 6, 9]); + } + + #[test] + fn symmetric_explicit_padding_forward() { + let device = Default::default(); + // Create max pool with symmetric explicit padding: top=2, left=2, bottom=2, right=2 + let config = MaxPool2dConfig::new([3, 3]) + .with_strides([1, 1]) + .with_padding(PaddingConfig2d::Explicit(2, 2, 2, 2)); + let pool = config.init(); + + // Input: [batch=1, channels=2, height=4, width=5] + let input = Tensor::::ones([1, 2, 4, 5], &device); + let output = pool.forward(input); + + // Height: 4 + 2 + 2 = 8, output = (8 - 3) / 1 + 1 = 6 + // Width: 5 + 2 + 2 = 9, output = (9 - 3) / 1 + 1 = 7 + assert_eq!(output.dims(), [1, 2, 6, 7]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/mod.rs new file mode 100644 index 0000000..622a4b6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pool/mod.rs @@ -0,0 +1,13 @@ +mod adaptive_avg_pool1d; +mod adaptive_avg_pool2d; +mod avg_pool1d; +mod avg_pool2d; +mod max_pool1d; +mod max_pool2d; + +pub use adaptive_avg_pool1d::*; +pub use adaptive_avg_pool2d::*; +pub use avg_pool1d::*; +pub use avg_pool2d::*; +pub use max_pool1d::*; +pub use max_pool2d::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pos_encoding.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pos_encoding.rs new file mode 100644 index 0000000..16a959d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/pos_encoding.rs @@ -0,0 +1,291 @@ +use burn_core as burn; + +use alloc::vec::Vec; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay}; + +use burn::tensor::Tensor; +use burn::tensor::TensorData; +use burn::tensor::backend::Backend; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Configuration to create a [PositionalEncoding](PositionalEncoding) layer using the [init function](PositionalEncodingConfig::init). +#[derive(Config, Debug)] +pub struct PositionalEncodingConfig { + /// Maximum sequence size to use. + #[config(default = "5_000")] + pub max_sequence_size: usize, + + /// The size of each vector. + pub d_model: usize, + + /// Max time scale to use. + #[config(default = "10_000")] + pub max_timescale: usize, +} + +/// Positional encoding layer for transformer models. +/// +/// This layer adds positional information to the input embeddings, allowing the transformer model +/// to take into account the order of the sequence. The positional encoding is added to the input +/// embeddings by computing a set of sinusoidal functions with different frequencies and phases. +/// +/// Sinusoids are used for positional embedding introduced in +/// [Attention is all you need](https://arxiv.org/abs/1706.03762). +/// +/// The reference implementation can be found here: +/// [LANGUAGE MODELING WITH NN.TRANSFORMER AND TORCHTEXT +/// ](https://pytorch.org/tutorials/beginner/transformer_tutorial.html) +/// +/// Should be created using [PositionalEncodingConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct PositionalEncoding { + /// The sinusoids used to add positional information to the input embeddings. + pub sinusoids: Tensor, + /// The maximum sequence size to use. + pub max_sequence_size: usize, + /// Max time scale to use. + pub max_timescale: usize, +} + +impl ModuleDisplay for PositionalEncoding { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [_, _, d_model] = self.sinusoids.shape().dims(); + content + .add("d_model", &d_model) + .add("max_sequence_size", &self.max_sequence_size) + .add("max_timescale", &self.max_timescale) + .optional() + } +} + +impl PositionalEncodingConfig { + /// Initialize a new [PositionalEncoding](PositionalEncoding) module. + pub fn init(&self, device: &B::Device) -> PositionalEncoding { + let sinusoids = generate_sinusoids::( + self.max_sequence_size, + self.d_model, + self.max_timescale, + device, + ) + .unsqueeze::<3>(); + + PositionalEncoding { + sinusoids, + max_sequence_size: self.max_sequence_size, + max_timescale: self.max_timescale, + } + } +} + +impl PositionalEncoding { + /// Applies the forward pass on the input tensor by adding the sinusoids to the input. + /// + /// # Shapes + /// + /// * input: [batch_size, seq_length, d_model] + /// * output: [batch_size, seq_length, d_model] + /// + /// + /// # Panics + /// + /// * Panics if the input sequence length is greater than the maximum sequence size. + /// * Panics if the input d_model is not equal to the d_model of the sinusoids. + pub fn forward(&self, input: Tensor) -> Tensor { + let [_, seq_length, d_model_input] = input.dims(); + + let [batch_size, max_sequence_size, d_model] = self.sinusoids.dims(); + + assert!( + max_sequence_size >= seq_length, + "max_sequence_size({max_sequence_size}) must be greater or equal than length({seq_length})" + ); + + assert!( + d_model_input == d_model, + "d_model({d_model_input}) of the input must be equal to d_model of encoding({d_model})" + ); + + let slices = [0..batch_size, 0..seq_length, 0..d_model]; + + input.add(self.sinusoids.clone().slice(slices)) + } +} + +/// Returns sinusoids for positional embedding introduced in +/// [Attention is all you need](https://arxiv.org/abs/1706.03762). +/// +/// The reference implementation can be found here: +/// [LANGUAGE MODELING WITH NN.TRANSFORMER AND TORCHTEXT +/// ](https://pytorch.org/tutorials/beginner/transformer_tutorial.html) +/// +/// # Arguments +/// +/// * `length` - The length of the sequence. +/// * `d_model` - The size of each vector. +/// * `max_timescale` - The maximum time scale to use. +/// +/// # Returns +/// +/// A tensor of shape [length, d_model] containing the sinusoids. +pub fn generate_sinusoids( + length: usize, + d_model: usize, + max_timescale: usize, + device: &B::Device, +) -> Tensor { + assert!(d_model.is_multiple_of(2), "d_model must be even"); + assert!( + max_timescale >= length, + "max_timescale must be greater than length" + ); + + // Calculate the increment for the logarithmic timescale + let log_timescale_increment = -(max_timescale as f32).ln() / d_model as f32; + + // Create a vector to hold the sinusoids + let mut scaled_time_sin_cos = Vec::with_capacity(length); + + // Loop over each position in the sequence + for i in 0..length { + // Create a vector to hold the sinusoids for this position + let mut row = Vec::with_capacity(d_model / 2); + // Loop over each dimension of the sinusoids + for k in (0..d_model).step_by(2) { + // Calculate the division term for this dimension + let div_term = (k as f32 * log_timescale_increment).exp(); + // Calculate the sine and cosine values for this dimension and position + row.push((div_term * i as f32).sin()); + row.push((div_term * i as f32).cos()); + } + + // Add the sinusoids for this position to the vector + scaled_time_sin_cos.push(row); + } + + // Convert the sinusoids to a tensor and return it + let data = TensorData::new( + scaled_time_sin_cos.into_iter().flatten().collect(), + [length, d_model], + ); + + Tensor::::from_data(data, device) +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::TestBackend; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_module() { + let d_model = 6; + let length = 3; + + // expected to broadcast + let batch_size = 2; + + let device = Default::default(); + let pe = PositionalEncodingConfig::new(d_model).init::(&device); + + // Use a tensor of zeros as input for easy verification of the output + // The output should be the sinusoids broadcasted to the input shape + let tensor = Tensor::zeros([batch_size, length, d_model], &device); + + let output = pe.forward(tensor); + + assert_eq!(&*output.shape(), [batch_size, length, d_model]); + + let expected = Tensor::::from_floats( + [ + [ + [0.00000, 1.00000, 0.00000, 1.00000, 0.00000, 1.00000], + [0.84147, 0.54030, 0.04640, 0.99892, 0.00215, 1.00000], + [0.90930, -0.41615, 0.09270, 0.99569, 0.00431, 0.99999], + ], + [ + [0.00000, 1.00000, 0.00000, 1.00000, 0.00000, 1.00000], + [0.84147, 0.54030, 0.04640, 0.99892, 0.00215, 1.00000], + [0.90930, -0.41615, 0.09270, 0.99569, 0.00431, 0.99999], + ], + ], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } + + #[test] + fn test_generate_sinusoids() { + let device = Default::default(); + let sinusoids = generate_sinusoids::(12, 6, 10_000, &device); + + // The values are taken from the pytorch reference implementation + let expected = Tensor::::from_floats( + [ + [0.00000, 1.00000, 0.00000, 1.00000, 0.00000, 1.00000], + [0.84147, 0.54030, 0.04640, 0.99892, 0.00215, 1.00000], + [0.90930, -0.41615, 0.09270, 0.99569, 0.00431, 0.99999], + [0.14112, -0.98999, 0.13880, 0.99032, 0.00646, 0.99998], + [-0.75680, -0.65364, 0.18460, 0.98281, 0.00862, 0.99996], + [-0.95892, 0.28366, 0.23000, 0.97319, 0.01077, 0.99994], + [-0.27942, 0.96017, 0.27491, 0.96147, 0.01293, 0.99992], + [0.65699, 0.75390, 0.31922, 0.94768, 0.01508, 0.99989], + [0.98936, -0.14550, 0.36285, 0.93185, 0.01723, 0.99985], + [0.41212, -0.91113, 0.40570, 0.91401, 0.01939, 0.99981], + [-0.54402, -0.83907, 0.44767, 0.89420, 0.02154, 0.99977], + [-0.99999, 0.00443, 0.48868, 0.87246, 0.02370, 0.99972], + ], + &device, + ); + sinusoids + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } + + #[test] + #[should_panic] + fn d_model_input_should_match() { + let d_model = 8; + let device = Default::default(); + let pe = PositionalEncodingConfig::new(d_model).init::(&device); + let input = Tensor::zeros([1, 5, 10], &device); + let _output = pe.forward(input); + } + + #[test] + #[should_panic] + fn input_length_should_be_less_than_max_len() { + let d_model = 8; + let device = Default::default(); + let pe = PositionalEncodingConfig::new(d_model).init::(&device); + let input = Tensor::zeros([1, 6_000, d_model], &device); + let _output = pe.forward(input); + } + + #[test] + fn display() { + let config = PositionalEncodingConfig::new(4); + let pe = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{pe}"), + "PositionalEncoding {d_model: 4, max_sequence_size: 5000, max_timescale: 10000}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/basic.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/basic.rs new file mode 100644 index 0000000..ac969fd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/basic.rs @@ -0,0 +1,742 @@ +use burn_core as burn; + +use crate::GateController; +use crate::activation::{Activation, ActivationConfig}; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// A RnnState is used to store hidden state in RNN. +pub struct RnnState { + /// The hidden state. + pub hidden: Tensor, +} + +impl RnnState { + /// Initialize a new [RNN State](RnnState). + pub fn new(hidden: Tensor) -> Self { + Self { hidden } + } +} + +/// Configuration to create a [Rnn](Rnn) module using the [init function](RnnConfig::init). +#[derive(Config, Debug)] +pub struct RnnConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the hidden state. + pub d_hidden: usize, + /// If a bias should be applied during the Rnn transformation. + pub bias: bool, + /// Rnn initializer + #[config(default = "Initializer::XavierNormal{gain:1.0}")] + pub initializer: Initializer, + /// If true, the input tensor is expected to be `[batch_size, seq_length, input_size]`. + /// If false, the input tensor is expected to be `[seq_length, batch_size, input_size]`. + #[config(default = true)] + pub batch_first: bool, + /// If true, process the sequence in reverse order. + /// This is useful for implementing reverse-direction RNNs (e.g., ONNX reverse direction). + #[config(default = false)] + pub reverse: bool, + /// Optional hidden state clip threshold. If provided, hidden state values are clipped + /// to the range `[-clip, +clip]` after each timestep. This can help prevent + /// exploding values during inference. + pub clip: Option, + /// Activation function applied to the hidden state before computing hidden output. + /// Default is Tanh, which is standard for Rnn. + #[config(default = "ActivationConfig::Tanh")] + pub hidden_activation: ActivationConfig, +} + +/// The Rnn module. This implementation is for a unidirectional, stateless, Rnn. +/// Should be created with [RnnConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Rnn { + /// gate controller for Rnn (has single gate). + pub gate: GateController, + /// The hidden state of the Rnn. + pub d_hidden: usize, + /// If true, input is `[batch_size, seq_length, input_size]`. + /// If false, input is `[seq_length, batch_size, input_size]`. + pub batch_first: bool, + /// If true, process the sequence in reverse order. + pub reverse: bool, + /// Optional hidden state clip threshold. + pub clip: Option, + /// Activation function for hidden output. + pub hidden_activation: Activation, +} + +impl ModuleDisplay for Rnn { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, _] = self.gate.input_transform.weight.shape().dims(); + let bias = self.gate.input_transform.bias.is_some(); + + content + .add("d_input", &d_input) + .add("d_hidden", &self.d_hidden) + .add("bias", &bias) + .optional() + } +} + +impl RnnConfig { + /// Initialize a new [Rnn](Rnn) module. + pub fn init(&self, device: &B::Device) -> Rnn { + let d_output = self.d_hidden; + + let new_gate = || { + GateController::new( + self.d_input, + d_output, + self.bias, + self.initializer.clone(), + device, + ) + }; + + Rnn { + gate: new_gate(), + d_hidden: self.d_hidden, + batch_first: self.batch_first, + reverse: self.reverse, + clip: self.clip, + hidden_activation: self.hidden_activation.init(device), + } + } +} + +impl Rnn { + /// Applies the forward pass on the input tensor. This RNN implementation + /// returns the state for each element in a sequence (i.e., across seq_length) and a final state. + /// + /// ## Parameters: + /// - batched_input: The input tensor of shape: + /// - `[batch_size, sequence_length, input_size]` if `batch_first` is true (default) + /// - `[sequence_length, batch_size, input_size]` if `batch_first` is false + /// - state: An optional `RnnState` representing the initial hidden state. + /// The state tensor has shape `[batch_size, hidden_size]`. + /// If no initial state is provided, these tensors are initialized to zeros. + /// + /// ## Returns: + /// - output: A tensor represents the output features of Rnn. Shape: + /// - `[batch_size, sequence_length, hidden_size]` if `batch_first` is true + /// - `[sequence_length, batch_size, hidden_size]` if `batch_first` is false + /// - state: A `RnnState` represents the final hidden state. The hidden state tensor has the shape + /// `[batch_size, hidden_size]`. + pub fn forward( + &self, + batched_input: Tensor, + state: Option>, + ) -> (Tensor, RnnState) { + // Convert to batch-first layout internally if needed + let batched_input = if self.batch_first { + batched_input + } else { + batched_input.swap_dims(0, 1) + }; + + let device = batched_input.device(); + let [batch_size, seq_length, _] = batched_input.dims(); + + // Process sequence in forward or reverse order based on config + let (output, state) = if self.reverse { + self.forward_iter( + batched_input.iter_dim(1).rev().zip((0..seq_length).rev()), + state, + batch_size, + seq_length, + &device, + ) + } else { + self.forward_iter( + batched_input.iter_dim(1).zip(0..seq_length), + state, + batch_size, + seq_length, + &device, + ) + }; + + // Convert output back to seq-first layout if needed + let output = if self.batch_first { + output + } else { + output.swap_dims(0, 1) + }; + + (output, state) + } + + fn forward_iter, usize)>>( + &self, + input_timestep_iter: I, + state: Option>, + batch_size: usize, + seq_length: usize, + device: &B::Device, + ) -> (Tensor, RnnState) { + let mut batched_hidden_state = + Tensor::empty([batch_size, seq_length, self.d_hidden], device); + + let mut hidden_state = match state { + Some(state) => state.hidden, + None => Tensor::zeros([batch_size, self.d_hidden], device), + }; + + for (input_t, t) in input_timestep_iter { + let input_t = input_t.squeeze_dim(1); + + // Compute gate output: h_t = activation(W_i @ x_t + W_h @ h_{t-1} + b) + let biased_gate_sum = self + .gate + .gate_product(input_t.clone(), hidden_state.clone()); + + let output_values = self.hidden_activation.forward(biased_gate_sum); + + // Update hidden state + hidden_state = output_values; + + // Apply hidden state clipping if configured + if let Some(clip) = self.clip { + hidden_state = hidden_state.clamp(-clip, clip); + } + + let unsqueezed_hidden_state = hidden_state.clone().unsqueeze_dim(1); + + // store the hidden state for this timestep + batched_hidden_state = batched_hidden_state.slice_assign( + [0..batch_size, t..(t + 1), 0..self.d_hidden], + unsqueezed_hidden_state.clone(), + ); + } + + (batched_hidden_state, RnnState::new(hidden_state)) + } +} + +/// Configuration to create a [BiRnn](BiRnn) module using the [init function](BiRnnConfig::init). +#[derive(Config, Debug)] +pub struct BiRnnConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the hidden state. + pub d_hidden: usize, + /// If a bias should be applied during the BiRnn transformation. + pub bias: bool, + /// BiRnn initializer + #[config(default = "Initializer::XavierNormal{gain:1.0}")] + pub initializer: Initializer, + /// If true, the input tensor is expected to be `[batch_size, seq_length, input_size]`. + /// If false, the input tensor is expected to be `[seq_length, batch_size, input_size]`. + #[config(default = true)] + pub batch_first: bool, + /// Optional hidden state clip threshold. + pub clip: Option, + /// Activation function applied to the hidden state before computing hidden output. + #[config(default = "ActivationConfig::Tanh")] + pub hidden_activation: ActivationConfig, +} + +/// The BiRnn module. This implementation is for Bidirectional RNN. +/// Should be created with [BiRnnConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct BiRnn { + /// RNN for the forward direction. + pub forward: Rnn, + /// RNN for the reverse direction. + pub reverse: Rnn, + /// The size of the hidden state. + pub d_hidden: usize, + /// If true, input is `[batch_size, seq_length, input_size]`. + /// If false, input is `[seq_length, batch_size, input_size]`. + pub batch_first: bool, +} + +impl ModuleDisplay for BiRnn { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, _] = self.forward.gate.input_transform.weight.shape().dims(); + let bias = self.forward.gate.input_transform.bias.is_some(); + + content + .add("d_input", &d_input) + .add("d_hidden", &self.d_hidden) + .add("bias", &bias) + .optional() + } +} + +impl BiRnnConfig { + /// Initialize a new [Bidirectional RNN](BiRnn) module. + pub fn init(&self, device: &B::Device) -> BiRnn { + // Internal RNNs always use batch_first=true; BiRnn handles layout conversion + let base_config = RnnConfig::new(self.d_input, self.d_hidden, self.bias) + .with_initializer(self.initializer.clone()) + .with_batch_first(true) + .with_clip(self.clip) + .with_hidden_activation(self.hidden_activation.clone()); + + BiRnn { + forward: base_config.clone().init(device), + reverse: base_config.init(device), + d_hidden: self.d_hidden, + batch_first: self.batch_first, + } + } +} + +impl BiRnn { + /// Applies the forward pass on the input tensor. This Bidirectional RNN implementation + /// returns the state for each element in a sequence (i.e., across seq_length) and a final state. + /// + /// ## Parameters: + /// - batched_input: The input tensor of shape: + /// - `[batch_size, sequence_length, input_size]` if `batch_first` is true (default) + /// - `[sequence_length, batch_size, input_size]` if `batch_first` is false + /// - state: An optional `RnnState` representing the hidden state. + /// Each state tensor has shape `[2, batch_size, hidden_size]`. + /// If no initial state is provided, these tensors are initialized to zeros. + /// + /// ## Returns: + /// - output: A tensor represents the output features of RNN. Shape: + /// - `[batch_size, sequence_length, hidden_size * 2]` if `batch_first` is true + /// - `[sequence_length, batch_size, hidden_size * 2]` if `batch_first` is false + /// - state: A `RnnState` represents the final forward and reverse states. + /// The `state.hidden` have the shape `[2, batch_size, hidden_size]`. + pub fn forward( + &self, + batched_input: Tensor, + state: Option>, + ) -> (Tensor, RnnState) { + // Convert to batch-first layout internally if needed + let batched_input = if self.batch_first { + batched_input + } else { + batched_input.swap_dims(0, 1) + }; + + let device = batched_input.clone().device(); + let [batch_size, seq_length, _] = batched_input.shape().dims(); + + let [init_state_forward, init_state_reverse] = match state { + Some(state) => { + let hidden_state_forward = state + .hidden + .clone() + .slice([0..1, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + let hidden_state_reverse = state + .hidden + .slice([1..2, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + + [ + Some(RnnState::new(hidden_state_forward)), + Some(RnnState::new(hidden_state_reverse)), + ] + } + None => [None, None], + }; + + // forward direction + let (batched_hidden_state_forward, final_state_forward) = self + .forward + .forward(batched_input.clone(), init_state_forward); + + // reverse direction + let (batched_hidden_state_reverse, final_state_reverse) = self.reverse.forward_iter( + batched_input.iter_dim(1).rev().zip((0..seq_length).rev()), + init_state_reverse, + batch_size, + seq_length, + &device, + ); + + let output = Tensor::cat( + [batched_hidden_state_forward, batched_hidden_state_reverse].to_vec(), + 2, + ); + + // Convert output back to seq-first layout if needed + let output = if self.batch_first { + output + } else { + output.swap_dims(0, 1) + }; + + let state = RnnState::new(Tensor::stack( + [final_state_forward.hidden, final_state_reverse.hidden].to_vec(), + 0, + )); + + (output, state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{LinearRecord, TestBackend}; + use burn::module::Param; + use burn::tensor::{Device, Distribution, TensorData}; + use burn::tensor::{ElementConversion, Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[cfg(feature = "std")] + use crate::TestAutodiffBackend; + + fn create_single_feature_gate_controller( + weights: f32, + biases: f32, + d_input: usize, + d_output: usize, + bias: bool, + initializer: Initializer, + device: &Device, + ) -> GateController { + let record_1 = LinearRecord { + weight: Param::from_data(TensorData::from([[weights]]), device), + bias: Some(Param::from_data(TensorData::from([biases]), device)), + }; + let record_2 = LinearRecord { + weight: Param::from_data(TensorData::from([[weights]]), device), + bias: Some(Param::from_data(TensorData::from([biases]), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + bias, + initializer, + record_1, + record_2, + ) + } + + #[test] + fn test_with_uniform_initializer() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = RnnConfig::new(5, 5, false) + .with_initializer(Initializer::Uniform { min: 0.0, max: 1.0 }); + let rnn = config.init::(&Default::default()); + + let gate_to_data = + |gate: GateController| gate.input_transform.weight.val().to_data(); + + gate_to_data(rnn.gate).assert_within_range::(0.elem()..1.elem()); + } + + /// Test forward pass with simple input vector. + /// + /// Simple RNN: h_t = tanh(W_input @ x_t + W_hidden @ h_{t-1} + b) + /// With input=0.1, weight_input=0.5, bias=0.0, h_0=0.0, weight_hidden=0.5 + /// h_t = tanh(0.5*0.1 + 0.5*0) = tanh(0.05) = 0.04995 + #[test] + fn test_forward_single_input_single_feature() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = RnnConfig::new(1, 1, false); + let device = Default::default(); + let mut rnn = config.init::(&device); + + rnn.gate = create_single_feature_gate_controller( + 0.5, + 0.0, + 1, + 1, + false, + Initializer::XavierUniform { gain: 1.0 }, + &device, + ); + + // single timestep with single feature + let input = Tensor::::from_data(TensorData::from([[[0.1]]]), &device); + + let (output, state) = rnn.forward(input, None); + + let tolerance = Tolerance::default(); + let expected = TensorData::from([[0.04995]]); + state + .hidden + .to_data() + .assert_approx_eq::(&expected, tolerance); + + output + .select(0, Tensor::arange(0..1, &device)) + .squeeze_dim::<2>(0) + .to_data() + .assert_approx_eq::(&state.hidden.to_data(), tolerance); + } + + #[test] + fn test_batched_forward_pass_batch_of_one() { + let device = Default::default(); + let rnn = RnnConfig::new(64, 1024, true).init(&device); + let batched_input = + Tensor::::random([1, 2, 64], Distribution::Default, &device); + + let (output, state) = rnn.forward(batched_input, None); + assert_eq!(output.dims(), [1, 2, 1024]); + assert_eq!(state.hidden.dims(), [1, 1024]); + } + + #[test] + #[cfg(feature = "std")] + fn test_batched_backward_pass() { + use burn::tensor::Shape; + let device = Default::default(); + let rnn = RnnConfig::new(64, 32, true).init(&device); + let shape: Shape = [8, 10, 64].into(); + let batched_input = + Tensor::::random(shape, Distribution::Default, &device); + + let (output, _) = rnn.forward(batched_input.clone(), None); + let fake_loss = output; + let grads = fake_loss.backward(); + + let some_gradient = rnn.gate.hidden_transform.weight.grad(&grads).unwrap(); + + // Asserts that the gradients exist and are non-zero + assert_ne!( + some_gradient + .any() + .into_data() + .iter::() + .next() + .unwrap(), + 0.0 + ); + } + + #[test] + fn test_bidirectional() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = BiRnnConfig::new(2, 3, true); + let mut rnn = config.init(&device); + + fn create_gate_controller( + input_weights: [[f32; D1]; D2], + input_biases: [f32; D1], + hidden_weights: [[f32; D1]; D1], + hidden_biases: [f32; D1], + device: &Device, + ) -> GateController { + let d_input = input_weights[0].len(); + let d_output = input_weights.len(); + + let input_record = LinearRecord { + weight: Param::from_data(TensorData::from(input_weights), device), + bias: Some(Param::from_data(TensorData::from(input_biases), device)), + }; + let hidden_record = LinearRecord { + weight: Param::from_data(TensorData::from(hidden_weights), device), + bias: Some(Param::from_data(TensorData::from(hidden_biases), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + true, + Initializer::XavierUniform { gain: 1.0 }, + input_record, + hidden_record, + ) + } + + // [batch_size=1, seq_length=4, input_size=2] + let input = Tensor::::from_data( + TensorData::from([[ + [0.949, -0.861], + [0.892, 0.927], + [-0.173, -0.301], + [-0.081, 0.992], + ]]), + &device, + ); + + // [2, batch_size=1, hidden_size=3] + let h0 = Tensor::::from_data( + TensorData::from([[[0.280, 0.360, -1.242]], [[-0.588, 0.729, -0.788]]]), + &device, + ); + + rnn.forward.gate = create_gate_controller( + // input_weights: [input_size=2, hidden_size=3] + [[0.367, 0.091, 0.342], [0.322, 0.533, 0.059]], + // input_biases: [hidden_size=3] + [-0.196, 0.354, 0.209], + // hidden_weights: [hidden_size=3, hidden_size=3] + [ + [-0.320, 0.232, -0.165], + [0.093, -0.572, -0.315], + [-0.467, 0.325, 0.046], + ], + // hidden_biases: [hidden_size=3] + [0.181, -0.190, -0.245], + &device, + ); + + rnn.reverse.gate = create_gate_controller( + [[-0.055, 0.506, 0.247], [-0.369, 0.178, -0.258]], + [0.540, -0.164, 0.033], + [ + [0.159, 0.180, -0.037], + [-0.443, 0.485, -0.488], + [0.098, -0.085, -0.140], + ], + [-0.510, 0.105, 0.114], + &device, + ); + + // [batch_size=1, sequence_length=4, hidden_size * 2 = 6] + // The expected output values were computed from PyTorch + let expected_output_with_init_state = TensorData::from([[ + [0.5226, -0.6370, 0.0210, 0.0685, 0.3867, 0.3602], + [0.3580, 0.8431, 0.4129, -0.3175, 0.4374, 0.1766], + [-0.3837, -0.2703, -0.3957, -0.1542, -0.1122, 0.0725], + [0.5059, 0.5527, 0.1244, -0.6779, 0.3725, -0.3387], + ]]); + let expected_output_without_init_state = TensorData::from([[ + [0.0560, -0.2056, 0.2334, 0.0892, 0.3912, 0.3607], + [0.4340, 0.7378, 0.3714, -0.2394, 0.4235, 0.2002], + [-0.3962, -0.2097, -0.3798, 0.0532, -0.2067, 0.1727], + [0.5075, 0.5298, 0.1083, -0.3200, 0.0764, -0.1282], + ]]); + + //`[2, batch_size=1, hidden_size=3]` + let expected_hn_with_init_state = + TensorData::from([[[0.5059, 0.5527, 0.1244]], [[0.0685, 0.3867, 0.3602]]]); + let expected_hn_without_init_state = + TensorData::from([[[0.5075, 0.5298, 0.1083]], [[0.0892, 0.3912, 0.3607]]]); + + let (output_with_init_state, state_with_init_state) = + rnn.forward(input.clone(), Some(RnnState::new(h0))); + let (output_without_init_state, state_without_init_state) = rnn.forward(input, None); + + let tolerance = Tolerance::permissive(); + output_with_init_state + .to_data() + .assert_approx_eq::(&expected_output_with_init_state, tolerance); + output_without_init_state + .to_data() + .assert_approx_eq::(&expected_output_without_init_state, tolerance); + state_with_init_state + .hidden + .to_data() + .assert_approx_eq::(&expected_hn_with_init_state, tolerance); + state_without_init_state + .hidden + .to_data() + .assert_approx_eq::(&expected_hn_without_init_state, tolerance); + } + + #[test] + fn display_rnn() { + let config = RnnConfig::new(2, 3, true); + + let layer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "Rnn {d_input: 2, d_hidden: 3, bias: true, params: 21}" + ); + } + + #[test] + fn display_birnn() { + let config = BiRnnConfig::new(2, 3, true); + + let layer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "BiRnn {d_input: 2, d_hidden: 3, bias: true, params: 42}" + ); + } + + #[test] + fn test_rnn_clipping() { + let device = Default::default(); + + // Create Rnn with clipping enabled + let clip_value = 0.3; + let config = RnnConfig::new(4, 8, true).with_clip(Some(clip_value)); + let rnn = config.init::(&device); + + let input = Tensor::::random([2, 5, 4], Distribution::Default, &device); + let (_, state) = rnn.forward(input, None); + + // Verify output values are within the clip range + let hidden_state: Vec = state.hidden.to_data().to_vec().unwrap(); + for val in hidden_state { + assert!( + val >= -clip_value as f32 && val <= clip_value as f32, + "Value {} is outside clip range [-{}, {}]", + val, + clip_value, + clip_value + ); + } + } + + #[test] + fn test_forward_reverse_sequence() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + // Create RNN with reverse=true to process sequence in reverse order + let config = RnnConfig::new(1, 1, false).with_reverse(true); + let mut rnn = config.init::(&device); + + rnn.gate = create_single_feature_gate_controller( + 0.5, + 0.0, + 1, + 1, + false, + Initializer::XavierUniform { gain: 1.0 }, + &device, + ); + + // Create input with 3 timesteps: [0.1, 0.2, 0.3] + // Shape: [batch_size=1, seq_length=3, input_features=1] + let input = + Tensor::::from_data(TensorData::from([[[0.1], [0.2], [0.3]]]), &device); + + let (output, state) = rnn.forward(input, None); + + // With reverse=true and weight=0.5, sequence is processed in reverse: + // t=2 (last): h = tanh(0.5*0.3 + 0.5*0) = tanh(0.15) ≈ 0.1488850 + // t=1 (mid): h = tanh(0.5*0.2 + 0.5*0.1488850) ≈ 0.17269433 + // t=0 (first): h = tanh(0.5*0.1 + 0.5*0.17269433) ≈ 0.135508 + let expected_final_hidden = TensorData::from([[0.135508]]); + + let tolerance = Tolerance::default(); + state + .hidden + .to_data() + .assert_approx_eq::(&expected_final_hidden, tolerance); + + // Verify output tensor has correct shape and matches state at final timestep + assert_eq!(output.dims(), [1, 3, 1]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gate_controller.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gate_controller.rs new file mode 100644 index 0000000..19e7ac6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gate_controller.rs @@ -0,0 +1,101 @@ +use burn_core as burn; + +use crate::{Linear, LinearConfig, LinearLayout}; +use burn::module::{Initializer, Module}; +use burn::tensor::{Tensor, backend::Backend}; + +/// A GateController represents a gate in an LSTM cell. An +/// LSTM cell generally contains three gates: an input gate, +/// forget gate, and output gate. Additionally, cell gate +/// is just used to compute the cell state. +/// +/// An Lstm gate is modeled as two linear transformations. +/// The results of these transformations are used to calculate +/// the gate's output. +#[derive(Module, Debug)] +pub struct GateController { + /// Represents the affine transformation applied to input vector + pub input_transform: Linear, + /// Represents the affine transformation applied to the hidden state + pub hidden_transform: Linear, +} + +impl GateController { + /// Initialize a new [gate_controller](GateController) module. + pub fn new( + d_input: usize, + d_output: usize, + bias: bool, + initializer: Initializer, + device: &B::Device, + ) -> Self { + Self { + input_transform: LinearConfig { + d_input, + d_output, + bias, + initializer: initializer.clone(), + layout: LinearLayout::Row, + } + .init(device), + hidden_transform: LinearConfig { + d_input: d_output, + d_output, + bias, + initializer, + layout: LinearLayout::Row, + } + .init(device), + } + } + + /// Helper function for performing weighted matrix product for a gate and adds + /// bias, if any. + /// + /// Mathematically, performs `Wx*X + Wh*H + b`, where: + /// Wx = weight matrix for the connection to input vector X + /// Wh = weight matrix for the connection to hidden state H + /// X = input vector + /// H = hidden state + /// b = bias terms + pub fn gate_product(&self, input: Tensor, hidden: Tensor) -> Tensor { + self.input_transform.forward(input) + self.hidden_transform.forward(hidden) + } + + /// Used to initialize a gate controller with known weight layers, + /// allowing for predictable behavior. Used only for testing in + /// lstm. + #[cfg(test)] + pub fn create_with_weights( + d_input: usize, + d_output: usize, + bias: bool, + initializer: Initializer, + input_record: crate::LinearRecord, + hidden_record: crate::LinearRecord, + ) -> Self { + let l1 = LinearConfig { + d_input, + d_output, + bias, + initializer: initializer.clone(), + layout: LinearLayout::Row, + } + .init(&input_record.weight.device()) + .load_record(input_record); + let l2 = LinearConfig { + d_input, + d_output, + bias, + initializer, + layout: LinearLayout::Row, + } + .init(&hidden_record.weight.device()) + .load_record(hidden_record); + + Self { + input_transform: l1, + hidden_transform: l2, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gru.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gru.rs new file mode 100644 index 0000000..a5c7bfe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/gru.rs @@ -0,0 +1,1074 @@ +use burn_core as burn; + +use super::gate_controller::GateController; +use crate::activation::{Activation, ActivationConfig}; +use burn::config::Config; +use burn::module::Initializer; +use burn::module::Module; +use burn::module::{Content, DisplaySettings, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Configuration to create a [gru](Gru) module using the [init function](GruConfig::init). +#[derive(Config, Debug)] +pub struct GruConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the hidden state. + pub d_hidden: usize, + /// If a bias should be applied during the Gru transformation. + pub bias: bool, + /// If reset gate should be applied after weight multiplication. + /// + /// This configuration option controls how the reset gate is applied to the hidden state. + /// * `true` - (Default) Match the initial arXiv version of the paper [Learning Phrase Representations using RNN Encoder-Decoder for + /// Statistical Machine Translation (v1)](https://arxiv.org/abs/1406.1078v1) and apply the reset gate after multiplication by + /// the weights. This matches the behavior of [PyTorch GRU](https://pytorch.org/docs/stable/generated/torch.nn.GRU.html#torch.nn.GRU). + /// * `false` - Match the most recent revision of [Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine + /// Translation (v3)](https://arxiv.org/abs/1406.1078) and apply the reset gate before the weight multiplication. + /// + /// The differing implementations can give slightly different numerical results and have different efficiencies. For more + /// motivation for why the `true` can be more efficient see [Optimizing RNNs with Differentiable Graphs](https://svail.github.io/diff_graphs). + /// + /// To set this field to `false` use [`with_reset_after`](`GruConfig::with_reset_after`). + #[config(default = "true")] + pub reset_after: bool, + /// Gru initializer + #[config(default = "Initializer::XavierNormal{gain:1.0}")] + pub initializer: Initializer, + /// Activation function for the update and reset gates. + /// Default is Sigmoid, which is standard for GRU gates. + #[config(default = "ActivationConfig::Sigmoid")] + pub gate_activation: ActivationConfig, + /// Activation function for the new/candidate gate. + /// Default is Tanh, which is standard for GRU. + #[config(default = "ActivationConfig::Tanh")] + pub hidden_activation: ActivationConfig, + /// Optional hidden state clip threshold. If provided, hidden state values are clipped + /// to the range `[-clip, +clip]` after each timestep. This can help prevent + /// exploding values during inference. + pub clip: Option, +} + +/// The Gru (Gated recurrent unit) module. This implementation is for a unidirectional, stateless, Gru. +/// +/// Introduced in the paper: [Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation](https://arxiv.org/abs/1406.1078). +/// +/// Should be created with [GruConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Gru { + /// The update gate controller. + pub update_gate: GateController, + /// The reset gate controller. + pub reset_gate: GateController, + /// The new gate controller. + pub new_gate: GateController, + /// The size of the hidden state. + pub d_hidden: usize, + /// If reset gate should be applied after weight multiplication. + pub reset_after: bool, + /// Activation function for gates (update, reset). + pub gate_activation: Activation, + /// Activation function for new/candidate gate. + pub hidden_activation: Activation, + /// Optional hidden state clip threshold. + pub clip: Option, +} + +impl ModuleDisplay for Gru { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, _] = self.update_gate.input_transform.weight.shape().dims(); + let bias = self.update_gate.input_transform.bias.is_some(); + + content + .add("d_input", &d_input) + .add("d_hidden", &self.d_hidden) + .add("bias", &bias) + .add("reset_after", &self.reset_after) + .optional() + } +} + +impl GruConfig { + /// Initialize a new [gru](Gru) module. + pub fn init(&self, device: &B::Device) -> Gru { + let d_output = self.d_hidden; + + let update_gate = GateController::new( + self.d_input, + d_output, + self.bias, + self.initializer.clone(), + device, + ); + let reset_gate = GateController::new( + self.d_input, + d_output, + self.bias, + self.initializer.clone(), + device, + ); + let new_gate = GateController::new( + self.d_input, + d_output, + self.bias, + self.initializer.clone(), + device, + ); + + Gru { + update_gate, + reset_gate, + new_gate, + d_hidden: self.d_hidden, + reset_after: self.reset_after, + gate_activation: self.gate_activation.init(device), + hidden_activation: self.hidden_activation.init(device), + clip: self.clip, + } + } +} + +impl Gru { + /// Applies the forward pass on the input tensor. This GRU implementation + /// returns a state tensor with dimensions `[batch_size, sequence_length, hidden_size]`. + /// + /// # Parameters + /// - batched_input: `[batch_size, sequence_length, input_size]`. + /// - state: An optional tensor representing an initial cell state with dimensions + /// `[batch_size, hidden_size]`. If none is provided, an empty state will be used. + /// + /// # Returns + /// - output: `[batch_size, sequence_length, hidden_size]` + pub fn forward( + &self, + batched_input: Tensor, + state: Option>, + ) -> Tensor { + let device = batched_input.device(); + let [batch_size, seq_length, _] = batched_input.shape().dims(); + + self.forward_iter( + batched_input.iter_dim(1).zip(0..seq_length), + state, + batch_size, + seq_length, + &device, + ) + .0 + } + + /// Forward pass variant that accepts an iterator over timesteps. + /// Used by BiGru to process sequences in either direction. + /// + /// # Parameters + /// - input_timestep_iter: Iterator yielding (input_tensor, timestep_index) pairs. + /// The timestep_index determines where in the output tensor to store results. + /// - state: Optional initial hidden state with shape `[batch_size, hidden_size]`. + /// - batch_size: Batch size of the input. + /// - seq_length: Sequence length of the input. + /// - device: Device to create tensors on. + /// + /// # Returns + /// - output: `[batch_size, sequence_length, hidden_size]` + /// - final_hidden: Final hidden state `[batch_size, hidden_size]` + pub(crate) fn forward_iter, usize)>>( + &self, + input_timestep_iter: I, + state: Option>, + batch_size: usize, + seq_length: usize, + device: &B::Device, + ) -> (Tensor, Tensor) { + let mut batched_hidden_state = + Tensor::empty([batch_size, seq_length, self.d_hidden], device); + + let mut hidden_t = match state { + Some(state) => state, + None => Tensor::zeros([batch_size, self.d_hidden], device), + }; + + for (input_t, t) in input_timestep_iter { + let input_t = input_t.squeeze_dim(1); + + // u(pdate)g(ate) tensors + let biased_ug_input_sum = + self.gate_product(&input_t, &hidden_t, None, &self.update_gate); + let update_values = self.gate_activation.forward(biased_ug_input_sum); + + // r(eset)g(ate) tensors + let biased_rg_input_sum = + self.gate_product(&input_t, &hidden_t, None, &self.reset_gate); + let reset_values = self.gate_activation.forward(biased_rg_input_sum); + + // n(ew)g(ate) tensor + let biased_ng_input_sum = if self.reset_after { + self.gate_product(&input_t, &hidden_t, Some(&reset_values), &self.new_gate) + } else { + let reset_t = hidden_t.clone().mul(reset_values); + self.gate_product(&input_t, &reset_t, None, &self.new_gate) + }; + let candidate_state = self.hidden_activation.forward(biased_ng_input_sum); + + // calculate linear interpolation between previous hidden state and candidate state: + // h_t = (1 - z_t) * g_t + z_t * h_{t-1} + let one_minus_z = update_values.clone().neg().add_scalar(1.0); + hidden_t = candidate_state.mul(one_minus_z) + update_values.mul(hidden_t); + + // Apply hidden state clipping if configured + if let Some(clip) = self.clip { + hidden_t = hidden_t.clamp(-clip, clip); + } + + let unsqueezed_hidden_state = hidden_t.clone().unsqueeze_dim(1); + + batched_hidden_state = batched_hidden_state.slice_assign( + [0..batch_size, t..(t + 1), 0..self.d_hidden], + unsqueezed_hidden_state, + ); + } + + (batched_hidden_state, hidden_t) + } + + /// Helper function for performing weighted matrix product for a gate and adds + /// bias, if any, and optionally applies reset to hidden state. + /// + /// Mathematically, performs `Wx*X + r .* (Wh*H + b)`, where: + /// Wx = weight matrix for the connection to input vector X + /// Wh = weight matrix for the connection to hidden state H + /// X = input vector + /// H = hidden state + /// b = bias terms + /// r = reset state + fn gate_product( + &self, + input: &Tensor, + hidden: &Tensor, + reset: Option<&Tensor>, + gate: &GateController, + ) -> Tensor { + let input_product = input.clone().matmul(gate.input_transform.weight.val()); + let hidden_product = hidden.clone().matmul(gate.hidden_transform.weight.val()); + + let input_part = match &gate.input_transform.bias { + Some(bias) => input_product + bias.val().unsqueeze(), + None => input_product, + }; + + let hidden_part = match &gate.hidden_transform.bias { + Some(bias) => hidden_product + bias.val().unsqueeze(), + None => hidden_product, + }; + + match reset { + Some(r) => input_part + r.clone().mul(hidden_part), + None => input_part + hidden_part, + } + } +} + +/// Configuration to create a [BiGru](BiGru) module using the [init function](BiGruConfig::init). +#[derive(Config, Debug)] +pub struct BiGruConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the hidden state. + pub d_hidden: usize, + /// If a bias should be applied during the BiGru transformation. + pub bias: bool, + /// If reset gate should be applied after weight multiplication. + #[config(default = "true")] + pub reset_after: bool, + /// BiGru initializer + #[config(default = "Initializer::XavierNormal{gain:1.0}")] + pub initializer: Initializer, + /// If true, the input tensor is expected to be `[batch_size, seq_length, input_size]`. + /// If false, the input tensor is expected to be `[seq_length, batch_size, input_size]`. + #[config(default = true)] + pub batch_first: bool, + /// Activation function for the update and reset gates. + #[config(default = "ActivationConfig::Sigmoid")] + pub gate_activation: ActivationConfig, + /// Activation function for the new/candidate gate. + #[config(default = "ActivationConfig::Tanh")] + pub hidden_activation: ActivationConfig, + /// Optional hidden state clip threshold. + pub clip: Option, +} + +/// The BiGru module. This implementation is for Bidirectional GRU. +/// +/// Based on the paper: [Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation](https://arxiv.org/abs/1406.1078). +/// +/// Should be created with [BiGruConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct BiGru { + /// GRU for the forward direction. + pub forward: Gru, + /// GRU for the reverse direction. + pub reverse: Gru, + /// The size of the hidden state. + pub d_hidden: usize, + /// If true, input is `[batch_size, seq_length, input_size]`. + /// If false, input is `[seq_length, batch_size, input_size]`. + pub batch_first: bool, +} + +impl ModuleDisplay for BiGru { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, _] = self + .forward + .update_gate + .input_transform + .weight + .shape() + .dims(); + let bias = self.forward.update_gate.input_transform.bias.is_some(); + + content + .add("d_input", &d_input) + .add("d_hidden", &self.d_hidden) + .add("bias", &bias) + .optional() + } +} + +impl BiGruConfig { + /// Initialize a new [Bidirectional GRU](BiGru) module. + pub fn init(&self, device: &B::Device) -> BiGru { + // Internal GRUs always use batch_first=true; BiGru handles layout conversion + let base_config = GruConfig::new(self.d_input, self.d_hidden, self.bias) + .with_initializer(self.initializer.clone()) + .with_reset_after(self.reset_after) + .with_gate_activation(self.gate_activation.clone()) + .with_hidden_activation(self.hidden_activation.clone()) + .with_clip(self.clip); + + BiGru { + forward: base_config.clone().init(device), + reverse: base_config.init(device), + d_hidden: self.d_hidden, + batch_first: self.batch_first, + } + } +} + +impl BiGru { + /// Applies the forward pass on the input tensor. This Bidirectional GRU implementation + /// returns the state for each element in a sequence (i.e., across seq_length) and a final state. + /// + /// ## Parameters: + /// - batched_input: The input tensor of shape: + /// - `[batch_size, sequence_length, input_size]` if `batch_first` is true (default) + /// - `[sequence_length, batch_size, input_size]` if `batch_first` is false + /// - state: An optional tensor representing the initial hidden state with shape + /// `[2, batch_size, hidden_size]`. If no initial state is provided, it is initialized to zeros. + /// + /// ## Returns: + /// - output: A tensor representing the output features. Shape: + /// - `[batch_size, sequence_length, hidden_size * 2]` if `batch_first` is true + /// - `[sequence_length, batch_size, hidden_size * 2]` if `batch_first` is false + /// - state: The final forward and reverse hidden states stacked along dimension 0 + /// with shape `[2, batch_size, hidden_size]`. + pub fn forward( + &self, + batched_input: Tensor, + state: Option>, + ) -> (Tensor, Tensor) { + // Convert to batch-first layout internally if needed + let batched_input = if self.batch_first { + batched_input + } else { + batched_input.swap_dims(0, 1) + }; + + let device = batched_input.clone().device(); + let [batch_size, seq_length, _] = batched_input.shape().dims(); + + let [init_state_forward, init_state_reverse] = match state { + Some(state) => { + let hidden_state_forward = state + .clone() + .slice([0..1, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + let hidden_state_reverse = state + .slice([1..2, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + + [Some(hidden_state_forward), Some(hidden_state_reverse)] + } + None => [None, None], + }; + + // forward direction + let (batched_hidden_state_forward, final_state_forward) = self.forward.forward_iter( + batched_input.clone().iter_dim(1).zip(0..seq_length), + init_state_forward, + batch_size, + seq_length, + &device, + ); + + // reverse direction + let (batched_hidden_state_reverse, final_state_reverse) = self.reverse.forward_iter( + batched_input.iter_dim(1).rev().zip((0..seq_length).rev()), + init_state_reverse, + batch_size, + seq_length, + &device, + ); + + let output = Tensor::cat( + [batched_hidden_state_forward, batched_hidden_state_reverse].to_vec(), + 2, + ); + + // Convert output back to seq-first layout if needed + let output = if self.batch_first { + output + } else { + output.swap_dims(0, 1) + }; + + let state = Tensor::stack([final_state_forward, final_state_reverse].to_vec(), 0); + + (output, state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{LinearRecord, TestBackend}; + use burn::module::Param; + use burn::tensor::{Distribution, TensorData}; + use burn::tensor::{Tolerance, ops::FloatElem}; + + type FT = FloatElem; + + fn init_gru(reset_after: bool, device: &B::Device) -> Gru { + fn create_gate_controller( + weights: f32, + biases: f32, + d_input: usize, + d_output: usize, + bias: bool, + initializer: Initializer, + device: &B::Device, + ) -> GateController { + let record_1 = LinearRecord { + weight: Param::from_data(TensorData::from([[weights]]), device), + bias: Some(Param::from_data(TensorData::from([biases]), device)), + }; + let record_2 = LinearRecord { + weight: Param::from_data(TensorData::from([[weights]]), device), + bias: Some(Param::from_data(TensorData::from([biases]), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + bias, + initializer, + record_1, + record_2, + ) + } + + let config = GruConfig::new(1, 1, false).with_reset_after(reset_after); + let mut gru = config.init::(device); + + gru.update_gate = create_gate_controller( + 0.5, + 0.0, + 1, + 1, + false, + Initializer::XavierNormal { gain: 1.0 }, + device, + ); + gru.reset_gate = create_gate_controller( + 0.6, + 0.0, + 1, + 1, + false, + Initializer::XavierNormal { gain: 1.0 }, + device, + ); + gru.new_gate = create_gate_controller( + 0.7, + 0.0, + 1, + 1, + false, + Initializer::XavierNormal { gain: 1.0 }, + device, + ); + gru + } + + /// Test forward pass with simple input vector. + /// + /// z_t = sigmoid(0.5*0.1 + 0.5*0) = 0.5125 + /// r_t = sigmoid(0.6*0.1 + 0.*0) = 0.5150 + /// g_t = tanh(0.7*0.1 + 0.7*0) = 0.0699 + /// + /// h_t = z_t * h' + (1 - z_t) * g_t = 0.0341 + #[test] + fn tests_forward_single_input_single_feature() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let mut gru = init_gru::(false, &device); + + let input = Tensor::::from_data(TensorData::from([[[0.1]]]), &device); + let expected = TensorData::from([[0.034]]); + + // Reset gate applied to hidden state before the matrix multiplication + let state = gru.forward(input.clone(), None); + + let output = state + .select(0, Tensor::arange(0..1, &device)) + .squeeze_dim::<2>(0); + + let tolerance = Tolerance::default(); + output + .to_data() + .assert_approx_eq::(&expected, tolerance); + + // Reset gate applied to hidden state after the matrix multiplication + gru.reset_after = true; // override forward behavior + let state = gru.forward(input, None); + + let output = state + .select(0, Tensor::arange(0..1, &device)) + .squeeze_dim::<2>(0); + + output + .to_data() + .assert_approx_eq::(&expected, tolerance); + } + + #[test] + fn tests_forward_seq_len_3() { + let device = Default::default(); + TestBackend::seed(&device, 0); + let mut gru = init_gru::(true, &device); + + let input = + Tensor::::from_data(TensorData::from([[[0.1], [0.2], [0.3]]]), &device); + let expected = TensorData::from([[0.0341], [0.0894], [0.1575]]); + + let result = gru.forward(input.clone(), None); + let output = result + .select(0, Tensor::arange(0..1, &device)) + .squeeze_dim::<2>(0); + + let tolerance = Tolerance::default(); + output + .to_data() + .assert_approx_eq::(&expected, tolerance); + + // Reset gate applied to hidden state before the matrix multiplication + gru.reset_after = false; // override forward behavior + let state = gru.forward(input, None); + + let output = state + .select(0, Tensor::arange(0..1, &device)) + .squeeze_dim::<2>(0); + + output + .to_data() + .assert_approx_eq::(&expected, tolerance); + } + + #[test] + fn test_batched_forward_pass() { + let device = Default::default(); + let gru = GruConfig::new(64, 1024, true).init::(&device); + let batched_input = + Tensor::::random([8, 10, 64], Distribution::Default, &device); + + let hidden_state = gru.forward(batched_input, None); + + assert_eq!(&*hidden_state.shape(), [8, 10, 1024]); + } + + #[test] + fn display() { + let config = GruConfig::new(2, 8, true); + + let layer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "Gru {d_input: 2, d_hidden: 8, bias: true, reset_after: true, params: 288}" + ); + } + + #[test] + fn test_bigru_batched_forward_pass() { + let device = Default::default(); + let bigru = BiGruConfig::new(64, 1024, true).init::(&device); + let batched_input = + Tensor::::random([8, 10, 64], Distribution::Default, &device); + + let (output, state) = bigru.forward(batched_input, None); + + // Output should have hidden_size * 2 features (forward + reverse concatenated) + assert_eq!(&*output.shape(), [8, 10, 2048]); + // State should have shape [2, batch_size, hidden_size] + assert_eq!(&*state.shape(), [2, 8, 1024]); + } + + #[test] + fn test_bigru_with_initial_state() { + let device = Default::default(); + let bigru = BiGruConfig::new(32, 64, true).init::(&device); + let batched_input = + Tensor::::random([4, 5, 32], Distribution::Default, &device); + let initial_state = + Tensor::::random([2, 4, 64], Distribution::Default, &device); + + let (output, state) = bigru.forward(batched_input, Some(initial_state)); + + assert_eq!(&*output.shape(), [4, 5, 128]); + assert_eq!(&*state.shape(), [2, 4, 64]); + } + + #[test] + fn test_bigru_seq_first() { + let device = Default::default(); + let bigru = BiGruConfig::new(32, 64, true) + .with_batch_first(false) + .init::(&device); + // Input shape: [seq_length, batch_size, input_size] when batch_first=false + let batched_input = + Tensor::::random([5, 4, 32], Distribution::Default, &device); + + let (output, state) = bigru.forward(batched_input, None); + + // Output shape: [seq_length, batch_size, hidden_size * 2] + assert_eq!(&*output.shape(), [5, 4, 128]); + assert_eq!(&*state.shape(), [2, 4, 64]); + } + + /// Test BiGru against PyTorch reference implementation. + /// Expected values computed with PyTorch nn.GRU(bidirectional=True). + #[test] + fn test_bigru_against_pytorch() { + use burn::tensor::Device; + + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = BiGruConfig::new(2, 3, true); + let mut bigru = config.init::(&device); + + fn create_gate_controller( + input_weights: [[f32; D1]; D2], + input_biases: [f32; D1], + hidden_weights: [[f32; D1]; D1], + hidden_biases: [f32; D1], + device: &Device, + ) -> GateController { + let d_input = input_weights[0].len(); + let d_output = input_weights.len(); + + let input_record = LinearRecord { + weight: Param::from_data(TensorData::from(input_weights), device), + bias: Some(Param::from_data(TensorData::from(input_biases), device)), + }; + let hidden_record = LinearRecord { + weight: Param::from_data(TensorData::from(hidden_weights), device), + bias: Some(Param::from_data(TensorData::from(hidden_biases), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + true, + Initializer::XavierUniform { gain: 1.0 }, + input_record, + hidden_record, + ) + } + + let input = Tensor::::from_data( + TensorData::from([[ + [0.949, -0.861], + [0.892, 0.927], + [-0.173, -0.301], + [-0.081, 0.992], + ]]), + &device, + ); + let h0 = Tensor::::from_data( + TensorData::from([[[0.280, 0.360, -1.242]], [[-0.588, 0.729, -0.788]]]), + &device, + ); + + // Forward GRU gates (weights from PyTorch with seed 42, transposed for burn) + bigru.forward.update_gate = create_gate_controller( + [[-0.2811, 0.5090, 0.5018], [0.3391, -0.4236, 0.1081]], + [0.2932, -0.3519, -0.5715], + [ + [-0.3471, 0.5214, 0.0961], + [0.0545, -0.4904, -0.1875], + [-0.5702, 0.4457, 0.3568], + ], + [-0.0100, 0.4518, -0.4102], + &device, + ); + + bigru.forward.reset_gate = create_gate_controller( + [[0.4414, -0.1353, -0.1265], [0.4792, 0.5304, 0.1165]], + [-0.2524, 0.3333, 0.1033], + [ + [-0.2695, -0.0677, -0.4557], + [0.1472, -0.2345, -0.2662], + [-0.2660, 0.3830, -0.1630], + ], + [0.1663, 0.2391, 0.1826], + &device, + ); + + bigru.forward.new_gate = create_gate_controller( + [[0.4266, 0.2784, 0.4451], [0.0782, -0.0815, 0.0853]], + [-0.2231, -0.4428, 0.4737], + [ + [0.0900, -0.1821, 0.2430], + [0.4665, 0.1551, 0.5155], + [0.0631, -0.1566, 0.3337], + ], + [0.0364, -0.3941, 0.1780], + &device, + ); + + // Reverse GRU gates + bigru.reverse.update_gate = create_gate_controller( + [[-0.3444, 0.1924, -0.4765], [0.5193, 0.5556, -0.5727]], + [0.1090, 0.1779, -0.5385], + [ + [0.1221, 0.3925, 0.5287], + [-0.1472, -0.4187, -0.1948], + [0.3441, -0.3082, -0.2047], + ], + [0.0016, -0.2148, -0.0400], + &device, + ); + + bigru.reverse.reset_gate = create_gate_controller( + [[-0.1988, -0.1203, -0.3422], [0.1769, 0.4788, -0.3443]], + [-0.5053, -0.3676, 0.5771], + [ + [-0.3936, 0.3504, -0.4486], + [0.3063, -0.1370, -0.2914], + [-0.2334, 0.3303, 0.1760], + ], + [-0.5080, -0.2488, -0.3456], + &device, + ); + + bigru.reverse.new_gate = create_gate_controller( + [[-0.4517, 0.2339, 0.4797], [-0.3884, 0.2067, -0.2982]], + [-0.3792, -0.1922, 0.0903], + [ + [-0.5586, -0.0762, -0.3944], + [-0.3306, -0.4191, -0.4898], + [0.1442, 0.0135, -0.3179], + ], + [-0.3912, -0.3963, -0.3368], + &device, + ); + + // Expected values from PyTorch + let expected_output_with_init = TensorData::from([[ + [0.24537, 0.14018, 0.19449, -0.49777, -0.15647, 0.48392], + [0.27468, -0.14514, 0.56205, -0.60381, -0.04986, 0.15683], + [-0.04062, -0.33486, 0.52330, -0.42244, -0.12644, -0.12034], + [-0.11743, -0.53873, 0.54429, -0.64943, 0.30127, -0.41943], + ]]); + + let expected_hn_with_init = TensorData::from([ + [[-0.11743, -0.53873, 0.54429]], + [[-0.49777, -0.15647, 0.48392]], + ]); + + let expected_output_without_init = TensorData::from([[ + [0.07452, -0.08247, 0.46677, -0.46770, -0.18086, 0.47519], + [0.15843, -0.27144, 0.65781, -0.50286, -0.12806, 0.14884], + [-0.10704, -0.41573, 0.53954, -0.24794, -0.24003, -0.10294], + [-0.16505, -0.57952, 0.53565, -0.23598, -0.07137, -0.28937], + ]]); + + let expected_hn_without_init = TensorData::from([ + [[-0.16505, -0.57952, 0.53565]], + [[-0.46770, -0.18086, 0.47519]], + ]); + + let (output_with_init, hn_with_init) = bigru.forward(input.clone(), Some(h0)); + let (output_without_init, hn_without_init) = bigru.forward(input, None); + + let tolerance = Tolerance::permissive(); + output_with_init + .to_data() + .assert_approx_eq::(&expected_output_with_init, tolerance); + output_without_init + .to_data() + .assert_approx_eq::(&expected_output_without_init, tolerance); + hn_with_init + .to_data() + .assert_approx_eq::(&expected_hn_with_init, tolerance); + hn_without_init + .to_data() + .assert_approx_eq::(&expected_hn_without_init, tolerance); + } + + #[test] + fn bigru_display() { + let config = BiGruConfig::new(2, 8, true); + + let layer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "BiGru {d_input: 2, d_hidden: 8, bias: true, params: 576}" + ); + } + + #[test] + fn test_gru_custom_activations() { + let device = Default::default(); + + // Create GRU with custom activations (ReLU instead of Sigmoid/Tanh) + let config = GruConfig::new(4, 8, true) + .with_gate_activation(ActivationConfig::Relu) + .with_hidden_activation(ActivationConfig::Relu); + let gru = config.init::(&device); + + let input = Tensor::::random([2, 3, 4], Distribution::Default, &device); + + // Should run without panicking and produce valid output + let output = gru.forward(input, None); + assert_eq!(&*output.shape(), [2, 3, 8]); + } + + #[test] + fn test_bigru_custom_activations() { + let device = Default::default(); + + // Create BiGRU with custom activations + let config = BiGruConfig::new(4, 8, true) + .with_gate_activation(ActivationConfig::Relu) + .with_hidden_activation(ActivationConfig::Relu); + let bigru = config.init::(&device); + + let input = Tensor::::random([2, 3, 4], Distribution::Default, &device); + + let (output, state) = bigru.forward(input, None); + assert_eq!(&*output.shape(), [2, 3, 16]); // hidden_size * 2 + assert_eq!(&*state.shape(), [2, 2, 8]); + } + + #[test] + fn test_gru_clipping() { + let device = Default::default(); + + // Create GRU with clipping enabled + let clip_value = 0.5; + let config = GruConfig::new(4, 8, true).with_clip(Some(clip_value)); + let gru = config.init::(&device); + + let input = Tensor::::random([2, 5, 4], Distribution::Default, &device); + + let output = gru.forward(input, None); + + // Verify output values are within the clip range + let output_data: Vec = output.to_data().to_vec().unwrap(); + for val in output_data { + assert!( + val >= -clip_value as f32 && val <= clip_value as f32, + "Value {} is outside clip range [-{}, {}]", + val, + clip_value, + clip_value + ); + } + } + + #[test] + fn test_bigru_clipping() { + let device = Default::default(); + + // Create BiGRU with clipping enabled + let clip_value = 0.3; + let config = BiGruConfig::new(4, 8, true).with_clip(Some(clip_value)); + let bigru = config.init::(&device); + + let input = Tensor::::random([2, 5, 4], Distribution::Default, &device); + + let (output, state) = bigru.forward(input, None); + + // Verify output values are within the clip range + let output_data: Vec = output.to_data().to_vec().unwrap(); + for val in output_data { + assert!( + val >= -clip_value as f32 && val <= clip_value as f32, + "Output value {} is outside clip range [-{}, {}]", + val, + clip_value, + clip_value + ); + } + + // Verify state values are within the clip range + let state_data: Vec = state.to_data().to_vec().unwrap(); + for val in state_data { + assert!( + val >= -clip_value as f32 && val <= clip_value as f32, + "State value {} is outside clip range [-{}, {}]", + val, + clip_value, + clip_value + ); + } + } + + /// Test Gru against PyTorch reference implementation. + /// Expected values computed with PyTorch nn.GRU (seed=42 for weights, seed=123 for input). + #[test] + fn test_gru_against_pytorch() { + use burn::tensor::Device; + + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = GruConfig::new(2, 3, true); + let mut gru = config.init::(&device); + + fn create_gate_controller( + input_weights: [[f32; D1]; D2], + input_biases: [f32; D1], + hidden_weights: [[f32; D1]; D1], + hidden_biases: [f32; D1], + device: &Device, + ) -> GateController { + let d_input = input_weights[0].len(); + let d_output = input_weights.len(); + + let input_record = LinearRecord { + weight: Param::from_data(TensorData::from(input_weights), device), + bias: Some(Param::from_data(TensorData::from(input_biases), device)), + }; + let hidden_record = LinearRecord { + weight: Param::from_data(TensorData::from(hidden_weights), device), + bias: Some(Param::from_data(TensorData::from(hidden_biases), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + true, + Initializer::XavierUniform { gain: 1.0 }, + input_record, + hidden_record, + ) + } + + // Input: [batch=1, seq=4, input=2] + let input = Tensor::::from_data( + TensorData::from([[ + [-0.11147, 0.12036], + [-0.36963, -0.24042], + [-1.19692, 0.20927], + [-0.97236, -0.75505], + ]]), + &device, + ); + + // Initial hidden state: [batch=1, hidden=3] + let h0 = Tensor::::from_data( + TensorData::from([[0.3239, -0.10852, 0.21033]]), + &device, + ); + + // Update gate (z) - weights from PyTorch, transposed for Burn's Row layout + gru.update_gate = create_gate_controller( + [[-0.2811, 0.5090, 0.5018], [0.3391, -0.4236, 0.1081]], + [0.2932, -0.3519, -0.5715], + [ + [-0.3471, 0.5214, 0.0961], + [0.0545, -0.4904, -0.1875], + [-0.5702, 0.4457, 0.3568], + ], + [-0.0100, 0.4518, -0.4102], + &device, + ); + + // Reset gate (r) + gru.reset_gate = create_gate_controller( + [[0.4414, -0.1353, -0.1265], [0.4792, 0.5304, 0.1165]], + [-0.2524, 0.3333, 0.1033], + [ + [-0.2695, -0.0677, -0.4557], + [0.1472, -0.2345, -0.2662], + [-0.2660, 0.3830, -0.1630], + ], + [0.1663, 0.2391, 0.1826], + &device, + ); + + // New gate (n) + gru.new_gate = create_gate_controller( + [[0.4266, 0.2784, 0.4451], [0.0782, -0.0815, 0.0853]], + [-0.2231, -0.4428, 0.4737], + [ + [0.0900, -0.1821, 0.2430], + [0.4665, 0.1551, 0.5155], + [0.0631, -0.1566, 0.3337], + ], + [0.0364, -0.3941, 0.1780], + &device, + ); + + // Expected values from PyTorch + let expected_output_with_h0 = TensorData::from([[ + [0.05665, -0.34932, 0.43267], + [-0.1737, -0.49246, 0.38099], + [-0.35401, -0.68099, 0.05061], + [-0.47854, -0.70427, -0.13648], + ]]); + + let expected_output_no_h0 = TensorData::from([[ + [-0.0985, -0.31661, 0.36126], + [-0.24563, -0.47784, 0.34609], + [-0.39497, -0.67659, 0.03083], + [-0.50146, -0.70066, -0.14894], + ]]); + + let output_with_h0 = gru.forward(input.clone(), Some(h0)); + let output_no_h0 = gru.forward(input, None); + + let tolerance = Tolerance::permissive(); + output_with_h0 + .to_data() + .assert_approx_eq::(&expected_output_with_h0, tolerance); + output_no_h0 + .to_data() + .assert_approx_eq::(&expected_output_no_h0, tolerance); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/lstm.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/lstm.rs new file mode 100644 index 0000000..4b3c6b0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/lstm.rs @@ -0,0 +1,922 @@ +use burn_core as burn; + +use crate::GateController; +use crate::activation::{Activation, ActivationConfig}; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// A LstmState is used to store cell state and hidden state in LSTM. +pub struct LstmState { + /// The cell state. + pub cell: Tensor, + /// The hidden state. + pub hidden: Tensor, +} + +impl LstmState { + /// Initialize a new [LSTM State](LstmState). + pub fn new(cell: Tensor, hidden: Tensor) -> Self { + Self { cell, hidden } + } +} + +/// Configuration to create a [Lstm](Lstm) module using the [init function](LstmConfig::init). +#[derive(Config, Debug)] +pub struct LstmConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the hidden state. + pub d_hidden: usize, + /// If a bias should be applied during the Lstm transformation. + pub bias: bool, + /// Lstm initializer + #[config(default = "Initializer::XavierNormal{gain:1.0}")] + pub initializer: Initializer, + /// If true, the input tensor is expected to be `[batch_size, seq_length, input_size]`. + /// If false, the input tensor is expected to be `[seq_length, batch_size, input_size]`. + #[config(default = true)] + pub batch_first: bool, + /// If true, process the sequence in reverse order. + /// This is useful for implementing reverse-direction LSTMs (e.g., ONNX reverse direction). + #[config(default = false)] + pub reverse: bool, + /// Optional cell state clip threshold. If provided, cell state values are clipped + /// to the range `[-clip, +clip]` after each timestep. This can help prevent + /// exploding values during inference. + pub clip: Option, + /// If true, couples the input and forget gates: `f_t = 1 - i_t`. + /// This reduces the number of parameters and is based on GRU-style simplification. + #[config(default = false)] + pub input_forget: bool, + /// Activation function for the input, forget, and output gates. + /// Default is Sigmoid, which is standard for LSTM gates. + #[config(default = "ActivationConfig::Sigmoid")] + pub gate_activation: ActivationConfig, + /// Activation function for the cell gate (candidate cell state). + /// Default is Tanh, which is standard for LSTM. + #[config(default = "ActivationConfig::Tanh")] + pub cell_activation: ActivationConfig, + /// Activation function applied to the cell state before computing hidden output. + /// Default is Tanh, which is standard for LSTM. + #[config(default = "ActivationConfig::Tanh")] + pub hidden_activation: ActivationConfig, +} + +/// The Lstm module. This implementation is for a unidirectional, stateless, Lstm. +/// +/// Introduced in the paper: [Long Short-Term Memory](https://www.researchgate.net/publication/13853244). +/// +/// Should be created with [LstmConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Lstm { + /// The input gate regulates which information to update and store in the cell state at each time step. + pub input_gate: GateController, + /// The forget gate is used to control which information to discard or keep in the memory cell at each time step. + /// Note: When `input_forget` is true, this gate is not used (forget = 1 - input). + pub forget_gate: GateController, + /// The output gate determines which information from the cell state to output at each time step. + pub output_gate: GateController, + /// The cell gate is used to compute the cell state that stores and carries information through time. + pub cell_gate: GateController, + /// The hidden state of the LSTM. + pub d_hidden: usize, + /// If true, input is `[batch_size, seq_length, input_size]`. + /// If false, input is `[seq_length, batch_size, input_size]`. + pub batch_first: bool, + /// If true, process the sequence in reverse order. + pub reverse: bool, + /// Optional cell state clip threshold. + pub clip: Option, + /// If true, couples input and forget gates: f_t = 1 - i_t. + pub input_forget: bool, + /// Activation function for gates (input, forget, output). + pub gate_activation: Activation, + /// Activation function for cell gate (candidate cell state). + pub cell_activation: Activation, + /// Activation function for hidden output. + pub hidden_activation: Activation, +} + +impl ModuleDisplay for Lstm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, _] = self.input_gate.input_transform.weight.shape().dims(); + let bias = self.input_gate.input_transform.bias.is_some(); + + content + .add("d_input", &d_input) + .add("d_hidden", &self.d_hidden) + .add("bias", &bias) + .optional() + } +} + +impl LstmConfig { + /// Initialize a new [lstm](Lstm) module. + pub fn init(&self, device: &B::Device) -> Lstm { + let d_output = self.d_hidden; + + let new_gate = || { + GateController::new( + self.d_input, + d_output, + self.bias, + self.initializer.clone(), + device, + ) + }; + + Lstm { + input_gate: new_gate(), + forget_gate: new_gate(), + output_gate: new_gate(), + cell_gate: new_gate(), + d_hidden: self.d_hidden, + batch_first: self.batch_first, + reverse: self.reverse, + clip: self.clip, + input_forget: self.input_forget, + gate_activation: self.gate_activation.init(device), + cell_activation: self.cell_activation.init(device), + hidden_activation: self.hidden_activation.init(device), + } + } +} + +impl Lstm { + /// Applies the forward pass on the input tensor. This LSTM implementation + /// returns the state for each element in a sequence (i.e., across seq_length) and a final state. + /// + /// ## Parameters: + /// - batched_input: The input tensor of shape: + /// - `[batch_size, sequence_length, input_size]` if `batch_first` is true (default) + /// - `[sequence_length, batch_size, input_size]` if `batch_first` is false + /// - state: An optional `LstmState` representing the initial cell state and hidden state. + /// Each state tensor has shape `[batch_size, hidden_size]`. + /// If no initial state is provided, these tensors are initialized to zeros. + /// + /// ## Returns: + /// - output: A tensor represents the output features of LSTM. Shape: + /// - `[batch_size, sequence_length, hidden_size]` if `batch_first` is true + /// - `[sequence_length, batch_size, hidden_size]` if `batch_first` is false + /// - state: A `LstmState` represents the final states. Both `state.cell` and `state.hidden` have the shape + /// `[batch_size, hidden_size]`. + pub fn forward( + &self, + batched_input: Tensor, + state: Option>, + ) -> (Tensor, LstmState) { + // Convert to batch-first layout internally if needed + let batched_input = if self.batch_first { + batched_input + } else { + batched_input.swap_dims(0, 1) + }; + + let device = batched_input.device(); + let [batch_size, seq_length, _] = batched_input.dims(); + + // Process sequence in forward or reverse order based on config + let (output, state) = if self.reverse { + self.forward_iter( + batched_input.iter_dim(1).rev().zip((0..seq_length).rev()), + state, + batch_size, + seq_length, + &device, + ) + } else { + self.forward_iter( + batched_input.iter_dim(1).zip(0..seq_length), + state, + batch_size, + seq_length, + &device, + ) + }; + + // Convert output back to seq-first layout if needed + let output = if self.batch_first { + output + } else { + output.swap_dims(0, 1) + }; + + (output, state) + } + + fn forward_iter, usize)>>( + &self, + input_timestep_iter: I, + state: Option>, + batch_size: usize, + seq_length: usize, + device: &B::Device, + ) -> (Tensor, LstmState) { + let mut batched_hidden_state = + Tensor::empty([batch_size, seq_length, self.d_hidden], device); + + let (mut cell_state, mut hidden_state) = match state { + Some(state) => (state.cell, state.hidden), + None => ( + Tensor::zeros([batch_size, self.d_hidden], device), + Tensor::zeros([batch_size, self.d_hidden], device), + ), + }; + + for (input_t, t) in input_timestep_iter { + let input_t = input_t.squeeze_dim(1); + + // i(nput)g(ate) tensors + let biased_ig_input_sum = self + .input_gate + .gate_product(input_t.clone(), hidden_state.clone()); + let input_values = self.gate_activation.forward(biased_ig_input_sum); + + // f(orget)g(ate) tensors - either computed or coupled to input gate + let forget_values = if self.input_forget { + // Coupled mode: f_t = 1 - i_t + input_values.clone().neg().add_scalar(1.0) + } else { + let biased_fg_input_sum = self + .forget_gate + .gate_product(input_t.clone(), hidden_state.clone()); + self.gate_activation.forward(biased_fg_input_sum) + }; + + // o(output)g(ate) tensors + let biased_og_input_sum = self + .output_gate + .gate_product(input_t.clone(), hidden_state.clone()); + let output_values = self.gate_activation.forward(biased_og_input_sum); + + // c(ell)g(ate) tensors + let biased_cg_input_sum = self + .cell_gate + .gate_product(input_t.clone(), hidden_state.clone()); + let candidate_cell_values = self.cell_activation.forward(biased_cg_input_sum); + + cell_state = forget_values * cell_state.clone() + input_values * candidate_cell_values; + + // Apply cell state clipping if configured + if let Some(clip) = self.clip { + cell_state = cell_state.clamp(-clip, clip); + } + + hidden_state = output_values * self.hidden_activation.forward(cell_state.clone()); + + let unsqueezed_hidden_state = hidden_state.clone().unsqueeze_dim(1); + + // store the hidden state for this timestep + batched_hidden_state = batched_hidden_state.slice_assign( + [0..batch_size, t..(t + 1), 0..self.d_hidden], + unsqueezed_hidden_state.clone(), + ); + } + + ( + batched_hidden_state, + LstmState::new(cell_state, hidden_state), + ) + } +} + +/// Configuration to create a [BiLstm](BiLstm) module using the [init function](BiLstmConfig::init). +#[derive(Config, Debug)] +pub struct BiLstmConfig { + /// The size of the input features. + pub d_input: usize, + /// The size of the hidden state. + pub d_hidden: usize, + /// If a bias should be applied during the BiLstm transformation. + pub bias: bool, + /// BiLstm initializer + #[config(default = "Initializer::XavierNormal{gain:1.0}")] + pub initializer: Initializer, + /// If true, the input tensor is expected to be `[batch_size, seq_length, input_size]`. + /// If false, the input tensor is expected to be `[seq_length, batch_size, input_size]`. + #[config(default = true)] + pub batch_first: bool, + /// Optional cell state clip threshold. + pub clip: Option, + /// If true, couples the input and forget gates. + #[config(default = false)] + pub input_forget: bool, + /// Activation function for the input, forget, and output gates. + #[config(default = "ActivationConfig::Sigmoid")] + pub gate_activation: ActivationConfig, + /// Activation function for the cell gate (candidate cell state). + #[config(default = "ActivationConfig::Tanh")] + pub cell_activation: ActivationConfig, + /// Activation function applied to the cell state before computing hidden output. + #[config(default = "ActivationConfig::Tanh")] + pub hidden_activation: ActivationConfig, +} + +/// The BiLstm module. This implementation is for Bidirectional LSTM. +/// +/// Introduced in the paper: [Framewise phoneme classification with bidirectional LSTM and other neural network architectures](https://www.cs.toronto.edu/~graves/ijcnn_2005.pdf). +/// +/// Should be created with [BiLstmConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct BiLstm { + /// LSTM for the forward direction. + pub forward: Lstm, + /// LSTM for the reverse direction. + pub reverse: Lstm, + /// The size of the hidden state. + pub d_hidden: usize, + /// If true, input is `[batch_size, seq_length, input_size]`. + /// If false, input is `[seq_length, batch_size, input_size]`. + pub batch_first: bool, +} + +impl ModuleDisplay for BiLstm { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_input, _] = self + .forward + .input_gate + .input_transform + .weight + .shape() + .dims(); + let bias = self.forward.input_gate.input_transform.bias.is_some(); + + content + .add("d_input", &d_input) + .add("d_hidden", &self.d_hidden) + .add("bias", &bias) + .optional() + } +} + +impl BiLstmConfig { + /// Initialize a new [Bidirectional LSTM](BiLstm) module. + pub fn init(&self, device: &B::Device) -> BiLstm { + // Internal LSTMs always use batch_first=true; BiLstm handles layout conversion + let base_config = LstmConfig::new(self.d_input, self.d_hidden, self.bias) + .with_initializer(self.initializer.clone()) + .with_batch_first(true) + .with_clip(self.clip) + .with_input_forget(self.input_forget) + .with_gate_activation(self.gate_activation.clone()) + .with_cell_activation(self.cell_activation.clone()) + .with_hidden_activation(self.hidden_activation.clone()); + + BiLstm { + forward: base_config.clone().init(device), + reverse: base_config.init(device), + d_hidden: self.d_hidden, + batch_first: self.batch_first, + } + } +} + +impl BiLstm { + /// Applies the forward pass on the input tensor. This Bidirectional LSTM implementation + /// returns the state for each element in a sequence (i.e., across seq_length) and a final state. + /// + /// ## Parameters: + /// - batched_input: The input tensor of shape: + /// - `[batch_size, sequence_length, input_size]` if `batch_first` is true (default) + /// - `[sequence_length, batch_size, input_size]` if `batch_first` is false + /// - state: An optional `LstmState` representing the initial cell state and hidden state. + /// Each state tensor has shape `[2, batch_size, hidden_size]`. + /// If no initial state is provided, these tensors are initialized to zeros. + /// + /// ## Returns: + /// - output: A tensor represents the output features of LSTM. Shape: + /// - `[batch_size, sequence_length, hidden_size * 2]` if `batch_first` is true + /// - `[sequence_length, batch_size, hidden_size * 2]` if `batch_first` is false + /// - state: A `LstmState` represents the final forward and reverse states. Both `state.cell` and + /// `state.hidden` have the shape `[2, batch_size, hidden_size]`. + pub fn forward( + &self, + batched_input: Tensor, + state: Option>, + ) -> (Tensor, LstmState) { + // Convert to batch-first layout internally if needed + let batched_input = if self.batch_first { + batched_input + } else { + batched_input.swap_dims(0, 1) + }; + + let device = batched_input.clone().device(); + let [batch_size, seq_length, _] = batched_input.shape().dims(); + + let [init_state_forward, init_state_reverse] = match state { + Some(state) => { + let cell_state_forward = state + .cell + .clone() + .slice([0..1, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + let hidden_state_forward = state + .hidden + .clone() + .slice([0..1, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + let cell_state_reverse = state + .cell + .slice([1..2, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + let hidden_state_reverse = state + .hidden + .slice([1..2, 0..batch_size, 0..self.d_hidden]) + .squeeze_dim(0); + + [ + Some(LstmState::new(cell_state_forward, hidden_state_forward)), + Some(LstmState::new(cell_state_reverse, hidden_state_reverse)), + ] + } + None => [None, None], + }; + + // forward direction + let (batched_hidden_state_forward, final_state_forward) = self + .forward + .forward(batched_input.clone(), init_state_forward); + + // reverse direction + let (batched_hidden_state_reverse, final_state_reverse) = self.reverse.forward_iter( + batched_input.iter_dim(1).rev().zip((0..seq_length).rev()), + init_state_reverse, + batch_size, + seq_length, + &device, + ); + + let output = Tensor::cat( + [batched_hidden_state_forward, batched_hidden_state_reverse].to_vec(), + 2, + ); + + // Convert output back to seq-first layout if needed + let output = if self.batch_first { + output + } else { + output.swap_dims(0, 1) + }; + + let state = LstmState::new( + Tensor::stack( + [final_state_forward.cell, final_state_reverse.cell].to_vec(), + 0, + ), + Tensor::stack( + [final_state_forward.hidden, final_state_reverse.hidden].to_vec(), + 0, + ), + ); + + (output, state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{LinearRecord, TestBackend}; + use burn::module::Param; + use burn::tensor::{Device, Distribution, TensorData}; + use burn::tensor::{ElementConversion, Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[cfg(feature = "std")] + use crate::TestAutodiffBackend; + + #[test] + fn test_with_uniform_initializer() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = LstmConfig::new(5, 5, false) + .with_initializer(Initializer::Uniform { min: 0.0, max: 1.0 }); + let lstm = config.init::(&Default::default()); + + let gate_to_data = + |gate: GateController| gate.input_transform.weight.val().to_data(); + + gate_to_data(lstm.input_gate).assert_within_range::(0.elem()..1.elem()); + gate_to_data(lstm.forget_gate).assert_within_range::(0.elem()..1.elem()); + gate_to_data(lstm.output_gate).assert_within_range::(0.elem()..1.elem()); + gate_to_data(lstm.cell_gate).assert_within_range::(0.elem()..1.elem()); + } + + /// Test forward pass with simple input vector. + /// + /// f_t = sigmoid(0.7*0.1 + 0.7*0) = sigmoid(0.07) = 0.5173928 + /// i_t = sigmoid(0.5*0.1 + 0.5*0) = sigmoid(0.05) = 0.5123725 + /// o_t = sigmoid(1.1*0.1 + 1.1*0) = sigmoid(0.11) = 0.5274723 + /// c_t = tanh(0.9*0.1 + 0.9*0) = tanh(0.09) = 0.0892937 + /// C_t = f_t * 0 + i_t * c_t = 0 + 0.5123725 * 0.0892937 = 0.04575243 + /// h_t = o_t * tanh(C_t) = 0.5274723 * tanh(0.04575243) = 0.5274723 * 0.04568173 = 0.024083648 + #[test] + fn test_forward_single_input_single_feature() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = LstmConfig::new(1, 1, false); + let device = Default::default(); + let mut lstm = config.init::(&device); + + fn create_gate_controller( + weights: f32, + biases: f32, + d_input: usize, + d_output: usize, + bias: bool, + initializer: Initializer, + device: &Device, + ) -> GateController { + let record_1 = LinearRecord { + weight: Param::from_data(TensorData::from([[weights]]), device), + bias: Some(Param::from_data(TensorData::from([biases]), device)), + }; + let record_2 = LinearRecord { + weight: Param::from_data(TensorData::from([[weights]]), device), + bias: Some(Param::from_data(TensorData::from([biases]), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + bias, + initializer, + record_1, + record_2, + ) + } + + lstm.input_gate = create_gate_controller( + 0.5, + 0.0, + 1, + 1, + false, + Initializer::XavierUniform { gain: 1.0 }, + &device, + ); + lstm.forget_gate = create_gate_controller( + 0.7, + 0.0, + 1, + 1, + false, + Initializer::XavierUniform { gain: 1.0 }, + &device, + ); + lstm.cell_gate = create_gate_controller( + 0.9, + 0.0, + 1, + 1, + false, + Initializer::XavierUniform { gain: 1.0 }, + &device, + ); + lstm.output_gate = create_gate_controller( + 1.1, + 0.0, + 1, + 1, + false, + Initializer::XavierUniform { gain: 1.0 }, + &device, + ); + + // single timestep with single feature + let input = Tensor::::from_data(TensorData::from([[[0.1]]]), &device); + + let (output, state) = lstm.forward(input, None); + + let expected = TensorData::from([[0.046]]); + let tolerance = Tolerance::default(); + state + .cell + .to_data() + .assert_approx_eq::(&expected, tolerance); + + let expected = TensorData::from([[0.0242]]); + state + .hidden + .to_data() + .assert_approx_eq::(&expected, tolerance); + + output + .select(0, Tensor::arange(0..1, &device)) + .squeeze_dim::<2>(0) + .to_data() + .assert_approx_eq::(&state.hidden.to_data(), tolerance); + } + + #[test] + fn test_batched_forward_pass() { + let device = Default::default(); + let lstm = LstmConfig::new(64, 1024, true).init(&device); + let batched_input = + Tensor::::random([8, 10, 64], Distribution::Default, &device); + + let (output, state) = lstm.forward(batched_input, None); + + assert_eq!(output.dims(), [8, 10, 1024]); + assert_eq!(state.cell.dims(), [8, 1024]); + assert_eq!(state.hidden.dims(), [8, 1024]); + } + + #[test] + fn test_batched_forward_pass_batch_of_one() { + let device = Default::default(); + let lstm = LstmConfig::new(64, 1024, true).init(&device); + let batched_input = + Tensor::::random([1, 2, 64], Distribution::Default, &device); + + let (output, state) = lstm.forward(batched_input, None); + + assert_eq!(output.dims(), [1, 2, 1024]); + assert_eq!(state.cell.dims(), [1, 1024]); + assert_eq!(state.hidden.dims(), [1, 1024]); + } + + #[test] + #[cfg(feature = "std")] + fn test_batched_backward_pass() { + use burn::tensor::Shape; + let device = Default::default(); + let lstm = LstmConfig::new(64, 32, true).init(&device); + let shape: Shape = [8, 10, 64].into(); + let batched_input = + Tensor::::random(shape, Distribution::Default, &device); + + let (output, _) = lstm.forward(batched_input.clone(), None); + let fake_loss = output; + let grads = fake_loss.backward(); + + let some_gradient = lstm + .output_gate + .hidden_transform + .weight + .grad(&grads) + .unwrap(); + + // Asserts that the gradients exist and are non-zero + assert_ne!( + some_gradient + .any() + .into_data() + .iter::() + .next() + .unwrap(), + 0.0 + ); + } + + #[test] + fn test_bidirectional() { + let device = Default::default(); + TestBackend::seed(&device, 0); + + let config = BiLstmConfig::new(2, 3, true); + let device = Default::default(); + let mut lstm = config.init(&device); + + fn create_gate_controller( + input_weights: [[f32; D1]; D2], + input_biases: [f32; D1], + hidden_weights: [[f32; D1]; D1], + hidden_biases: [f32; D1], + device: &Device, + ) -> GateController { + let d_input = input_weights[0].len(); + let d_output = input_weights.len(); + + let input_record = LinearRecord { + weight: Param::from_data(TensorData::from(input_weights), device), + bias: Some(Param::from_data(TensorData::from(input_biases), device)), + }; + let hidden_record = LinearRecord { + weight: Param::from_data(TensorData::from(hidden_weights), device), + bias: Some(Param::from_data(TensorData::from(hidden_biases), device)), + }; + GateController::create_with_weights( + d_input, + d_output, + true, + Initializer::XavierUniform { gain: 1.0 }, + input_record, + hidden_record, + ) + } + + let input = Tensor::::from_data( + TensorData::from([[ + [0.949, -0.861], + [0.892, 0.927], + [-0.173, -0.301], + [-0.081, 0.992], + ]]), + &device, + ); + let h0 = Tensor::::from_data( + TensorData::from([[[0.280, 0.360, -1.242]], [[-0.588, 0.729, -0.788]]]), + &device, + ); + let c0 = Tensor::::from_data( + TensorData::from([[[0.723, 0.397, -0.262]], [[0.471, 0.613, 1.885]]]), + &device, + ); + + lstm.forward.input_gate = create_gate_controller( + [[0.367, 0.091, 0.342], [0.322, 0.533, 0.059]], + [-0.196, 0.354, 0.209], + [ + [-0.320, 0.232, -0.165], + [0.093, -0.572, -0.315], + [-0.467, 0.325, 0.046], + ], + [0.181, -0.190, -0.245], + &device, + ); + + lstm.forward.forget_gate = create_gate_controller( + [[-0.342, -0.084, -0.420], [-0.432, 0.119, 0.191]], + [0.315, -0.413, -0.041], + [ + [0.453, 0.063, 0.561], + [0.211, 0.149, 0.213], + [-0.499, -0.158, 0.068], + ], + [-0.431, -0.535, 0.125], + &device, + ); + + lstm.forward.cell_gate = create_gate_controller( + [[-0.046, -0.382, 0.321], [-0.533, 0.558, 0.004]], + [-0.358, 0.282, -0.078], + [ + [-0.358, 0.109, 0.139], + [-0.345, 0.091, -0.368], + [-0.508, 0.221, -0.507], + ], + [0.502, -0.509, -0.247], + &device, + ); + + lstm.forward.output_gate = create_gate_controller( + [[-0.577, -0.359, 0.216], [-0.550, 0.268, 0.243]], + [-0.227, -0.274, 0.039], + [ + [-0.383, 0.449, 0.222], + [-0.357, -0.093, 0.449], + [-0.106, 0.236, 0.360], + ], + [-0.361, -0.209, -0.454], + &device, + ); + + lstm.reverse.input_gate = create_gate_controller( + [[-0.055, 0.506, 0.247], [-0.369, 0.178, -0.258]], + [0.540, -0.164, 0.033], + [ + [0.159, 0.180, -0.037], + [-0.443, 0.485, -0.488], + [0.098, -0.085, -0.140], + ], + [-0.510, 0.105, 0.114], + &device, + ); + + lstm.reverse.forget_gate = create_gate_controller( + [[-0.154, -0.432, -0.547], [-0.369, -0.310, -0.175]], + [0.141, 0.004, 0.055], + [ + [-0.005, -0.277, -0.515], + [-0.011, -0.101, -0.365], + [0.426, 0.379, 0.337], + ], + [-0.382, 0.331, -0.176], + &device, + ); + + lstm.reverse.cell_gate = create_gate_controller( + [[-0.571, 0.228, -0.287], [-0.331, 0.110, 0.219]], + [-0.206, -0.546, 0.462], + [ + [0.449, -0.240, 0.071], + [-0.045, 0.131, 0.124], + [0.138, -0.201, 0.191], + ], + [-0.030, 0.211, -0.352], + &device, + ); + + lstm.reverse.output_gate = create_gate_controller( + [[0.491, -0.442, 0.333], [0.313, -0.121, -0.070]], + [-0.387, -0.250, 0.066], + [ + [-0.030, 0.268, 0.299], + [-0.019, -0.280, -0.314], + [0.466, -0.365, -0.248], + ], + [-0.398, -0.199, -0.566], + &device, + ); + + let expected_output_with_init_state = TensorData::from([[ + [0.23764, -0.03442, 0.04414, -0.15635, -0.03366, -0.05798], + [0.00473, -0.02254, 0.02988, -0.16510, -0.00306, 0.08742], + [0.06210, -0.06509, -0.05339, -0.01710, 0.02091, 0.16012], + [-0.03420, 0.07774, -0.09774, -0.02604, 0.12584, 0.20872], + ]]); + let expected_output_without_init_state = TensorData::from([[ + [0.08679, -0.08776, -0.00528, -0.15969, -0.05322, -0.08863], + [-0.02577, -0.05057, 0.00033, -0.17558, -0.03679, 0.03142], + [0.02942, -0.07411, -0.06044, -0.03601, -0.09998, 0.04846], + [-0.04026, 0.07178, -0.10189, -0.07349, -0.04576, 0.05550], + ]]); + let expected_hn_with_init_state = TensorData::from([ + [[-0.03420, 0.07774, -0.09774]], + [[-0.15635, -0.03366, -0.05798]], + ]); + let expected_cn_with_init_state = TensorData::from([ + [[-0.13593, 0.17125, -0.22395]], + [[-0.45425, -0.11206, -0.12908]], + ]); + let expected_hn_without_init_state = TensorData::from([ + [[-0.04026, 0.07178, -0.10189]], + [[-0.15969, -0.05322, -0.08863]], + ]); + let expected_cn_without_init_state = TensorData::from([ + [[-0.15839, 0.15923, -0.23569]], + [[-0.47407, -0.17493, -0.19643]], + ]); + + let (output_with_init_state, state_with_init_state) = + lstm.forward(input.clone(), Some(LstmState::new(c0, h0))); + let (output_without_init_state, state_without_init_state) = lstm.forward(input, None); + + let tolerance = Tolerance::permissive(); + output_with_init_state + .to_data() + .assert_approx_eq::(&expected_output_with_init_state, tolerance); + output_without_init_state + .to_data() + .assert_approx_eq::(&expected_output_without_init_state, tolerance); + state_with_init_state + .hidden + .to_data() + .assert_approx_eq::(&expected_hn_with_init_state, tolerance); + state_with_init_state + .cell + .to_data() + .assert_approx_eq::(&expected_cn_with_init_state, tolerance); + state_without_init_state + .hidden + .to_data() + .assert_approx_eq::(&expected_hn_without_init_state, tolerance); + state_without_init_state + .cell + .to_data() + .assert_approx_eq::(&expected_cn_without_init_state, tolerance); + } + + #[test] + fn display_lstm() { + let config = LstmConfig::new(2, 3, true); + + let layer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "Lstm {d_input: 2, d_hidden: 3, bias: true, params: 84}" + ); + } + + #[test] + fn display_bilstm() { + let config = BiLstmConfig::new(2, 3, true); + + let layer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{layer}"), + "BiLstm {d_input: 2, d_hidden: 3, bias: true, params: 168}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/mod.rs new file mode 100644 index 0000000..b651874 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rnn/mod.rs @@ -0,0 +1,15 @@ +mod gate_controller; + +/// Basic RNN. +pub mod basic; + +/// Gated Recurrent Unit module. +pub mod gru; + +/// Long Short-Term Memory module. +pub mod lstm; + +pub use basic::*; +pub use gate_controller::*; +pub use gru::*; +pub use lstm::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rope_encoding.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rope_encoding.rs new file mode 100644 index 0000000..3b20142 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/rope_encoding.rs @@ -0,0 +1,581 @@ +use burn_core as burn; + +use alloc::vec; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay}; +use burn::tensor::Int; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use core::ops::Range; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Configuration to create a [RotaryEncoding](RotaryEncoding) layer using the [init function](RotaryEncodingConfig::init). +#[derive(Config, Debug)] +pub struct RotaryEncodingConfig { + /// Maximum sequence length of input + pub max_sequence_length: usize, + + /// Size of the input embedding or hidden dimension + pub d_model: usize, + + /// Scaling factor for frequency computation. Defaults to 10000.0 + #[config(default = "10000.0")] + pub theta: f32, +} + +impl RotaryEncodingConfig { + /// Initialize a new [RotaryEncoding](RotaryEncoding) module. + /// + /// # Panics + /// + /// Panics if the size of input embedding dimension is not even. + /// Panics if the theta parameter is not positive. + pub fn init(&self, device: &B::Device) -> RotaryEncoding { + self.initialize(|x| x, device) + } + + /// Initialize a new [RotaryEncoding](RotaryEncoding) module with a custom frequency scaling function. + /// This is useful to apply different RoPE extensions. + /// + /// # Panics + /// + /// Panics if the size of input embedding dimension is not even. + /// Panics if the theta parameter is not positive. + pub fn init_with_frequency_scaling( + &self, + scaling: impl Fn(Tensor) -> Tensor, + device: &B::Device, + ) -> RotaryEncoding { + self.initialize(scaling, device) + } + + /// Initialize a new [RotaryEncoding](RotaryEncoding) module. + /// + /// # Panics + /// + /// Panics if the size of input embedding dimension is not even. + /// Panics if the theta parameter is not positive. + fn initialize( + &self, + scaling: impl Fn(Tensor) -> Tensor, + device: &B::Device, + ) -> RotaryEncoding { + assert_eq!( + self.d_model % 2, + 0, + "The input embedding dimension must be even" + ); + assert!( + self.theta > 0.0, + "Theta parameter must be positive (default: 10000)." + ); + + // Calculate the rotation frequencies for positional embeddings based on the formula + // `theta = 1 / (theta ^ (2i / d_model)) for i in [0..d_model/2]` + let exponent = Tensor::::arange_step(0..self.d_model as i64, 2, device) + .float() + .div_scalar(self.d_model as f32); + + // Calculate (10000 ^ (2i / d_model)) by using the log base property `exp(log(10000) * (2i / d_model))` + // This is done since burn doesn't support exponentiation of scalar to tensor + let theta = exponent.mul_scalar(self.theta.ln()).exp().recip(); + + let theta = scaling(theta); + + let freq_complex = + RotaryEncoding::compute_rotary_frequencies(0..self.max_sequence_length, theta.clone()); + + RotaryEncoding { + freq_complex, + theta, + start_offset: 0, + } + } +} + +/// A module that applies rotary positional encoding to a tensor. +/// Rotary Position Encoding or Embedding (RoPE), is a type of position embedding which encodes +/// absolute positional information with rotation matrix and naturally incorporates +/// explicit relative position dependency in self-attention formulation. +/// +/// Introduced in the paper: [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://arxiv.org/abs/2104.09864) +/// +/// Should be created using [RotaryEncodingConfig]. +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct RotaryEncoding { + /// Complex frequency tensor of shape (max_sequence_length, d_model, 2) with real and imaginary components + // Essentially a cache of pre-computed RoPE values. + pub freq_complex: Tensor, + /// Frequency vector used to compute/apply the complex rotations. + pub theta: Tensor, + start_offset: usize, +} + +impl ModuleDisplay for RotaryEncoding { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [max_sequence_length, d_model, _] = self.freq_complex.shape().dims(); + content + .add("d_model", &d_model) + .add("max_sequence_length", &max_sequence_length) + .optional() + } +} + +#[allow(clippy::single_range_in_vec_init)] +impl RotaryEncoding { + /// Applies rotary positional encoding to a tensor of dimensions (..., seq_len, d_model) + /// + /// # Arguments: + /// * `x` - Input tensor of shape (..., seq_len, d_model). Accommodate both 3D and 4D tensors + /// for (batch size, seq_len, hidden_dim) or (batch size, num_heads, seq_len, hidden_dim) + /// respectively. + /// + /// # Returns: + /// Output tensor with the same shape as input tensor after applying rotary encoding. + /// + /// # Panics + /// If the input tensor does not have at least 2 dimensions for sequence length and hidden dimension. + pub fn forward(&self, x: Tensor) -> Tensor { + self.apply(x, 0) + } + + /// Applies rotary positional encoding to a tensor of dimensions (..., seq_len, d_model) + /// + /// # Arguments: + /// * `x` - Input tensor of shape (..., seq_len, d_model). Accommodate both 3D and 4D tensors + /// for (batch size, seq_len, hidden_dim) or (batch size, num_heads, seq_len, hidden_dim) + /// respectively. + /// * `start` - Sequence start position index. + /// + /// # Returns: + /// Output tensor with the same shape as input tensor after applying rotary encoding. + /// + /// # Panics + /// If the input tensor does not have at least 2 dimensions for sequence length and hidden dimension. + pub fn apply(&self, x: Tensor, start: usize) -> Tensor { + assert!( + D >= 2, + "Input tensor must have at least 2 dimensions for sequence length and hidden dimension" + ); + + let device = x.device(); + let input_shape = x.shape(); + + // Extract the sequence length and embedding dimension, other dimensions are kept generic + // to allow both 3D and 4D tensors i.e. batch_size or (batch_size, num_heads) + let (seq_len, d_model) = (x.dims()[D - 2], x.dims()[D - 1]); + let dummy_dim_size = input_shape.num_elements() / (seq_len * d_model); + + // Create a dummy tensor with signed ones based on the 2D rotation matrix + // [[cos, -sin], [sin, cos]] + let sign_tensor = + Tensor::::from_floats([[1.0, 0.0, 0.0, 1.0], [0.0, -1.0, 1.0, 0.0]], &device); + + // Rotate input using the frequency tensor. Slice the frequencies till input sequence length + let out: Tensor = x + .reshape([dummy_dim_size, seq_len, d_model / 2, 2]) + .matmul(sign_tensor.unsqueeze()) + .reshape([dummy_dim_size, seq_len, d_model, 2]) + * self + .freq_complex + .clone() + .slice([start..start + seq_len]) + .unsqueeze(); + + // Sum the real and imaginary components to get output tensor and reshape to original shape + out.sum_dim(-1).reshape(input_shape) + } + + /// Shifts the pre-computed rotary frequency to cover a new range of positions. + /// + /// This method updates the internal frequency tensor `freq_complex` to store + /// the rotary positional encodings for a new window of positions starting at `start`. + pub fn shift(&mut self, start: usize) { + let max_seq_len = self.freq_complex.dims()[0]; + assert!( + start > self.start_offset, + "Shift start position must be monotonically increasing" + ); + + let current_end = self.start_offset + max_seq_len; + + if start >= current_end { + // Overwrite the whole buffer + let new_freqs = + Self::compute_rotary_frequencies(start..start + max_seq_len, self.theta.clone()); + self.freq_complex + .inplace(|freqs| freqs.slice_assign([0..max_seq_len], new_freqs)); + } else { + // Shift the tail + let num_keep = current_end - start; + let start_rel = start - self.start_offset; + let tail_freqs = self.freq_complex.clone().slice([start_rel..max_seq_len]); + self.freq_complex + .inplace(|freqs| freqs.slice_assign([0..num_keep], tail_freqs)); + // Compute the rest and assign + let new_freqs = Self::compute_rotary_frequencies( + current_end..start + max_seq_len, + self.theta.clone(), + ); + self.freq_complex + .inplace(|freqs| freqs.slice_assign([num_keep..max_seq_len], new_freqs)); + } + self.start_offset = start; + } + + /// Computes the positional rotation frequencies (cosine and sine values) used in RoPE. + /// + /// # Arguments + /// - `range`: Range of position indices `[start, end)`. + /// - `theta`: 1D tensor of shape `(d_model / 2)` containing base angular frequencies. + /// + /// # Returns + /// Tensor of shape `(range.len(), d_model, 2)` containing `[cos, sin]` pairs for each position and frequency. + fn compute_rotary_frequencies(range: Range, theta: Tensor) -> Tensor { + let d_model = theta.dims()[0] * 2; + let num_positions = range.end - range.start; + + // Generate frequency values for positional embeddings + let frequencies: Tensor = + Tensor::::arange(range.start as i64..range.end as i64, &theta.device()) + .float() + .unsqueeze() + .transpose() + .repeat_dim(1, d_model / 2) + * theta.unsqueeze(); + + // Convert frequency values to complex numbers (polar form) + let p_cos = frequencies.clone().cos(); + let p_sin = frequencies.sin(); + + Tensor::cat(vec![p_cos, p_sin], 1) + .reshape([num_positions, 2, d_model / 2]) + .transpose() + .unsqueeze_dim::<4>(2) + .repeat_dim(2, 2) + .reshape([num_positions, d_model, 2]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_rotary_encoding_forward() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(10, 4).init::(&device); + + let input = Tensor::::from_floats( + [ + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], + [[9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]], + ], + &device, + ); + + // Input = [Batch size, Num of heads, Seq_len, d_model] + let input = input.unsqueeze::<4>(); + + let output = rotary_encoding.forward(input); + let expected_output = Tensor::::from_floats( + [ + [ + [1.0000, 2.0000, 3.0000, 4.0000], + [-2.3473, 7.4492, 6.9197, 8.0696], + ], + [ + [9.0000, 10.0000, 11.0000, 12.0000], + [-4.7567, 18.5034, 14.8393, 16.1492], + ], + ], + &device, + ); + + output + .squeeze_dim::<3>(0) + .to_data() + .assert_approx_eq::(&expected_output.to_data(), Tolerance::default()); + } + + #[test] + fn test_rotary_encoding_3d() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(10, 4).init::(&device); + + let input = Tensor::::from_floats( + [ + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], + [[9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]], + ], + &device, + ); + + // Input = [Batch size, Num of heads, Seq_len, d_model] + // let input = input.unsqueeze::<4>(); + + let output = rotary_encoding.forward(input); + let expected_output = Tensor::::from_floats( + [ + [ + [1.0000, 2.0000, 3.0000, 4.0000], + [-2.3473, 7.4492, 6.9197, 8.0696], + ], + [ + [9.0000, 10.0000, 11.0000, 12.0000], + [-4.7567, 18.5034, 14.8393, 16.1492], + ], + ], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected_output.to_data(), Tolerance::default()); + } + + #[test] + fn test_zero_input_rotary_encoding_forward() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(10, 4).init::(&device); + + // Use a tensor of exact zeros as input. The output rotary embedding should be zeros as well + let input = Tensor::::zeros([1, 2, 2, 4], &device); + + let output = rotary_encoding.forward(input); + let expected_output = Tensor::::from_floats( + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000], + ], + ], + &device, + ); + + output + .squeeze_dim::<3>(0) + .to_data() + .assert_approx_eq::(&expected_output.to_data(), Tolerance::default()); + } + + #[test] + #[should_panic] + fn test_valid_input_hidden_dim() { + // Hidden dimension must be even to be able to split into real and imaginary components + // for rotation + let d_model = 15; + let device = Default::default(); + let pe = RotaryEncodingConfig::new(10, d_model).init::(&device); + let input = Tensor::::zeros([1, 5, d_model], &device); + let _output = pe.forward(input); + } + + #[test] + fn test_rotary_encoding_frequencies() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(2, 8).init::(&device); + + let expected_freqs = Tensor::::from_floats( + [ + [ + [1.0000, 0.0000], + [1.0000, 0.0000], + [1.0000, 0.0000], + [1.0000, 0.0000], + ], + [ + [5.4030e-01, 8.4147e-01], + [9.9500e-01, 9.9833e-02], + [9.9995e-01, 9.9998e-03], + [9.9999e-01, 9.9999e-04], + ], + ], + &device, + ) + .unsqueeze_dim::<4>(2) + .repeat_dim(2, 2) + .reshape([2, 8, 2]); + + rotary_encoding + .freq_complex + .to_data() + .assert_approx_eq::(&expected_freqs.to_data(), Tolerance::default()); + } + + fn apply_freq_scaling_by_parts(freqs: Tensor) -> Tensor { + // Adapted from: https://github.com/meta-llama/llama-models/blob/main/models/llama3/reference_impl/model.py#L45 + let scale_factor = 8.; + let low_freq_factor = 1.; + let high_freq_factor = 4.; + let old_context_len = 8192.; + + let low_freq_wavelen = old_context_len / low_freq_factor; + let high_freq_wavelen = old_context_len / high_freq_factor; + + let wavelen = freqs.clone().recip().mul_scalar(2. * core::f32::consts::PI); + + // if wavelen >= high_freq_wavelen + let cond = wavelen.clone().greater_equal_elem(high_freq_wavelen); + let smooth = wavelen + .clone() + .recip() + .mul_scalar(old_context_len) + .sub_scalar(low_freq_factor) + .div_scalar(high_freq_factor - low_freq_factor); + // (1 - smooth) * freq / scale_factor + smooth * freq + let new_freqs = smooth + .clone() + .neg() + .add_scalar(1.) + .mul(freqs.clone().div_scalar(scale_factor)) + .add(smooth.clone().mul(freqs.clone())); + let new_freqs = freqs.clone().mask_where(cond, new_freqs); + + // if wavelen > low_freq_wavelen + let cond = wavelen.clone().greater_elem(low_freq_wavelen); + let new_freqs = new_freqs.mask_where(cond, freqs.clone().div_scalar(scale_factor)); + + // if wavelen < high_freq_wavelen + let cond = wavelen.lower_elem(high_freq_wavelen); + new_freqs.mask_where(cond, freqs) + } + + #[test] + fn test_rotary_encoding_with_frequency_scaling() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(2, 8) + .init_with_frequency_scaling::(apply_freq_scaling_by_parts, &device); + + let expected_freqs = Tensor::::from_floats( + [ + [ + [1.0000, 0.0000], + [1.0000, 0.0000], + [1.0000, 0.0000], + [1.0000, 0.0000], + ], + [ + [5.4030e-01, 8.4148e-01], + [9.9500e-01, 9.9833e-02], + [9.9995e-01, 9.9998e-03], + [1.0000, 2.1361e-04], + ], + ], + &device, + ) + .unsqueeze_dim::<4>(2) + .repeat_dim(2, 2) + .reshape([2, 8, 2]); + + rotary_encoding + .freq_complex + .to_data() + .assert_approx_eq::(&expected_freqs.to_data(), Tolerance::default()); + } + + #[test] + fn test_rotary_encoding_shift_full() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(10, 4).init::(&device); + + // Input = [Batch size, Num of heads, Seq_len, d_model] + let input = Tensor::::from_floats( + [ + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], + [[9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]], + ], + &device, + ) + .unsqueeze::<4>(); + + // Initializing for a bigger cache (e.g., max_seq_len = 10) should give the same result + // as using a smaller cache of pre-computed RoPE frequencies that are shifted to the same + // initial position + let expected_output = rotary_encoding.apply(input.clone(), 6); + + let mut rotary_encoding = RotaryEncodingConfig::new(4, 4).init::(&device); + rotary_encoding.shift(6); // start > 4 will perform a full re-compute + + let output = rotary_encoding.apply(input, 0); + + output + .into_data() + .assert_approx_eq::(&expected_output.into_data(), Tolerance::default()); + } + + #[test] + fn test_rotary_encoding_shift() { + let device = Default::default(); + let rotary_encoding = RotaryEncodingConfig::new(10, 4).init::(&device); + + // Input = [Batch size, Num of heads, Seq_len, d_model] + let input = Tensor::::from_floats( + [ + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]], + [[9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]], + ], + &device, + ) + .unsqueeze::<4>(); + + // Initializing for a bigger cache (e.g., max_seq_len = 10) should give the same result + // as using a smaller cache of pre-computed RoPE frequencies that are shifted to the same + // initial position + let expected_output = rotary_encoding.apply(input.clone(), 2); + + let mut rotary_encoding = RotaryEncodingConfig::new(4, 4).init::(&device); + rotary_encoding.shift(2); // start < 4 will shift the (current_end - start) freqs and compute the rest + + let output = rotary_encoding.apply(input, 0); + + output + .into_data() + .assert_approx_eq::(&expected_output.into_data(), Tolerance::default()); + } + + #[test] + fn test_rotary_encoding_shift_multiple() { + let device = Default::default(); + let mut rotary_encoding = RotaryEncodingConfig::new(4, 4).init::(&device); + rotary_encoding.shift(2); + rotary_encoding.shift(5); + } + + #[test] + #[should_panic = "Shift start position must be monotonically increasing"] + fn test_rotary_encoding_shift_should_increase() { + let device = Default::default(); + let mut rotary_encoding = RotaryEncodingConfig::new(4, 4).init::(&device); + rotary_encoding.shift(6); + rotary_encoding.shift(4); // should be monotonically increasing + } + + #[test] + fn display() { + let config = RotaryEncodingConfig::new(10, 4); + let pe = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{pe}"), + "RotaryEncoding {d_model: 4, max_sequence_length: 10}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/decoder.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/decoder.rs new file mode 100644 index 0000000..00155c8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/decoder.rs @@ -0,0 +1,573 @@ +use burn_core as burn; + +use alloc::vec::Vec; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::{Bool, Tensor, backend::Backend}; + +use crate::activation::ActivationConfig; +use crate::cache::TensorCache; +use crate::{ + Dropout, DropoutConfig, LayerNorm, LayerNormConfig, + attention::{MhaCache, MhaInput, MultiHeadAttention, MultiHeadAttentionConfig}, +}; + +use super::{PositionWiseFeedForward, PositionWiseFeedForwardConfig}; + +/// Configuration to create a [Transformer Decoder](TransformerDecoder) layer using the [init function](TransformerDecoderConfig::init). +#[derive(Config, Debug)] +pub struct TransformerDecoderConfig { + /// The size of the model. + pub d_model: usize, + /// The size of the position-wise feed-forward network. + pub d_ff: usize, + /// The number of attention heads. + pub n_heads: usize, + /// The number of layers. + pub n_layers: usize, + /// The dropout rate. Default: 0.1 + #[config(default = 0.1)] + pub dropout: f64, + /// Layer norm will be applied first instead of after the other modules. + #[config(default = false)] + pub norm_first: bool, + /// Use "quiet softmax" instead of regular softmax. + /// + /// - Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head). + /// - Usage may reduce the entropy of weights in the model, enhancing quantization and compression. + /// + /// Reference: + #[config(default = false)] + pub quiet_softmax: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}" + )] + pub initializer: Initializer, + /// The activation function used in the position-wise feed-forward network. Default: Gelu + #[config(default = "ActivationConfig::Gelu")] + pub activation: ActivationConfig, + /// The epsilon value for layer normalization. Default: 1e-5 + #[config(default = 1e-5)] + pub layer_norm_eps: f64, +} + +/// The transformer decoder module as describe in the paper [Attention Is All You Need](https://arxiv.org/abs/1706.03762). +/// +/// # Params +/// +/// - layers: transformer decoder layers with `d_model` input and output features. +/// +/// Should be created using [TransformerDecoderConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct TransformerDecoder { + /// Transformer decoder layers. + pub layers: Vec>, + + /// The size of the model. + pub d_model: usize, + + /// The size of the position-wise feed-forward network. + pub d_ff: usize, + + /// The number of attention heads. + pub n_heads: usize, + + /// The number of layers. + pub n_layers: usize, + + /// The dropout rate. Default: 0.1 + pub dropout: f64, + + /// Layer norm will be applied first instead of after the other modules. + pub norm_first: bool, + + /// Use "quiet softmax" instead of regular softmax. + pub quiet_softmax: bool, +} + +impl ModuleDisplay for TransformerDecoder { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("d_model", &self.d_model) + .add("d_ff", &self.d_ff) + .add("n_heads", &self.n_heads) + .add("n_layers", &self.n_layers) + .add("dropout", &self.dropout) + .add("norm_first", &self.norm_first) + .add("quiet_softmax", &self.quiet_softmax) + .optional() + } +} + +impl TransformerDecoderConfig { + /// Initialize a new [Transformer Decoder](TransformerDecoder) module. + pub fn init(&self, device: &B::Device) -> TransformerDecoder { + let layers = (0..self.n_layers) + .map(|_| TransformerDecoderLayer::new(self, device)) + .collect::>(); + + TransformerDecoder { + layers, + d_model: self.d_model, + d_ff: self.d_ff, + n_heads: self.n_heads, + n_layers: self.n_layers, + dropout: self.dropout, + norm_first: self.norm_first, + quiet_softmax: self.quiet_softmax, + } + } +} + +/// [Transformer Decoder](TransformerDecoder) forward pass input argument. +#[derive(Debug)] +pub struct TransformerDecoderInput { + target: Tensor, + target_mask_pad: Option>, + target_mask_attn: Option>, + memory: Tensor, + memory_mask_pad: Option>, + memory_mask_attn: Option>, +} + +impl TransformerDecoderInput { + /// Create a [transformer decoder](TransformerDecoder) input argument. + pub fn new(target: Tensor, memory: Tensor) -> Self { + Self { + target, + target_mask_pad: None, + target_mask_attn: None, + memory, + memory_mask_pad: None, + memory_mask_attn: None, + } + } + + /// Register the memory padding mask. + pub fn memory_mask_pad(mut self, mask_pad: Tensor) -> Self { + self.memory_mask_pad = Some(mask_pad); + self + } + + /// Register the memory attention mask. + pub fn memory_mask_attn(mut self, mask_attn: Tensor) -> Self { + self.memory_mask_attn = Some(mask_attn); + self + } + + /// Register the target padding mask. + pub fn target_mask_pad(mut self, mask_pad: Tensor) -> Self { + self.target_mask_pad = Some(mask_pad); + self + } + + /// Register the target attention mask. + pub fn target_mask_attn(mut self, mask_attn: Tensor) -> Self { + self.target_mask_attn = Some(mask_attn); + self + } +} + +/// [Transformer Decoder](TransformerDecoder) layer module. +#[derive(Module, Debug)] +pub struct TransformerDecoderLayer { + /// Cross-attention module. + pub cross_attn: MultiHeadAttention, + /// Self-attention module. + pub self_attn: MultiHeadAttention, + /// Position-wise feed-forward module. + pub pwff: PositionWiseFeedForward, + /// First layer norm. + pub norm_1: LayerNorm, + /// Second layer norm. + pub norm_2: LayerNorm, + /// Third layer norm. + pub norm_3: LayerNorm, + /// Dropout. + pub dropout: Dropout, + /// Whether to apply norm first. + pub norm_first: bool, +} + +/// Autoregressive cache for a single [Transformer Decoder Layer](TransformerDecoderLayer). +pub struct TransformerDecoderLayerAutoregressiveCache { + /// Cross-attention cache. + pub cross_attn: MhaCache, + /// Self-attention cache. + pub self_attn: MhaCache, + /// Position-wise feed-forward cache. + pub pwff: TensorCache, + /// First layer norm cache. + pub norm_1: TensorCache, + /// Second layer norm cache. + pub norm_2: TensorCache, + /// Third layer norm cache. + pub norm_3: TensorCache, +} + +impl TransformerDecoderLayerAutoregressiveCache { + /// Create an empty cache. + pub fn empty() -> Self { + Self { + cross_attn: MhaCache::autoregressive_cross_attention(), + self_attn: MhaCache::autoregressive(), + pwff: TensorCache::empty(), + norm_1: TensorCache::empty(), + norm_2: TensorCache::empty(), + norm_3: TensorCache::empty(), + } + } +} + +/// Autoregressive cache for the [Transformer Decoder](TransformerDecoder) layer. +/// +/// To be used during inference when decoding tokens. +pub struct TransformerDecoderAutoregressiveCache { + layers: Vec>, +} + +impl TransformerDecoderAutoregressiveCache { + fn empty(num_layers: usize) -> Self { + Self { + layers: (0..num_layers) + .map(|_| TransformerDecoderLayerAutoregressiveCache::empty()) + .collect(), + } + } +} + +impl TransformerDecoderLayer { + /// Create a new [TransformerDecoderLayer](TransformerDecoderLayer). + pub fn new(config: &TransformerDecoderConfig, device: &B::Device) -> Self { + let self_attn = MultiHeadAttentionConfig::new(config.d_model, config.n_heads) + .with_initializer(config.initializer.clone()) + .with_dropout(config.dropout) + .with_quiet_softmax(config.quiet_softmax) + .init(device); + + let cross_attn = MultiHeadAttentionConfig::new(config.d_model, config.n_heads) + .with_initializer(config.initializer.clone()) + .with_dropout(config.dropout) + .with_quiet_softmax(config.quiet_softmax) + .init(device); + let norm_1 = LayerNormConfig::new(config.d_model) + .with_epsilon(config.layer_norm_eps) + .init(device); + let norm_2 = LayerNormConfig::new(config.d_model) + .with_epsilon(config.layer_norm_eps) + .init(device); + let norm_3 = LayerNormConfig::new(config.d_model) + .with_epsilon(config.layer_norm_eps) + .init(device); + let dropout = DropoutConfig::new(config.dropout).init(); + let pwff = PositionWiseFeedForwardConfig::new(config.d_model, config.d_ff) + .with_initializer(config.initializer.clone()) + .with_dropout(config.dropout) + .with_activation(config.activation.clone()) + .init(device); + + Self { + cross_attn, + self_attn, + norm_1, + norm_2, + norm_3, + pwff, + dropout, + norm_first: config.norm_first, + } + } + + /// Applies the TransformerDecoder forward pass to the input tensor. + pub fn forward(&self, mut input: TransformerDecoderInput) -> TransformerDecoderInput { + // Self attention residual path. + let x = input.target; + let mut residual_path = x.clone(); + + // Normalize. + if self.norm_first { + residual_path = self.norm_3.forward(residual_path); + } + + // Self attention. + let mut self_attn_input = MhaInput::self_attn(residual_path); + if let Some(mask_pad) = &input.target_mask_pad { + self_attn_input = self_attn_input.mask_pad(mask_pad.clone()); + } + if let Some(mask_attn) = &input.target_mask_attn { + self_attn_input = self_attn_input.mask_attn(mask_attn.clone()); + } + let residual_path = self.self_attn.forward(self_attn_input).context; + + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Cross attention residual path. + // Normalize. + let residual_path = if self.norm_first { + self.norm_1.forward(x.clone()) + } else { + x = self.norm_1.forward(x); + x.clone() + }; + + // Cross attention. + let mut cross_attn_input = + MhaInput::new(residual_path, input.memory.clone(), input.memory.clone()); + if let Some(mask_pad) = &input.memory_mask_pad { + cross_attn_input = cross_attn_input.mask_pad(mask_pad.clone()); + } + if let Some(mask_attn) = &input.memory_mask_attn { + cross_attn_input = cross_attn_input.mask_attn(mask_attn.clone()); + } + let residual_path = self.cross_attn.forward(cross_attn_input).context; + + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Feed forward residual path. + // Normalize. + let residual_path = if self.norm_first { + self.norm_2.forward(x.clone()) + } else { + x = self.norm_2.forward(x); + x.clone() + }; + + let residual_path = self.pwff.forward(residual_path); + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Main path. + // Normalize. + if !self.norm_first { + x = self.norm_3.forward(x) + } + + input.target = x; + input + } + + /// Applies the forward pass using an autoregressive cache. + pub fn forward_autoregressive_inference( + &self, + mut input: TransformerDecoderInput, + cache: &mut TransformerDecoderLayerAutoregressiveCache, + ) -> TransformerDecoderInput { + // Self attention residual path. + let x = input.target; + let mut residual_path = x.clone(); + + // Normalize. + if self.norm_first { + residual_path = cache + .norm_3 + .forward_autoregressive(residual_path, 1, |x| self.norm_3.forward(x)); + } + + // Self attention. + let mut self_attn_input = MhaInput::self_attn(residual_path); + if let Some(mask_pad) = &input.target_mask_pad { + self_attn_input = self_attn_input.mask_pad(mask_pad.clone()); + } + if let Some(mask_attn) = &input.target_mask_attn { + self_attn_input = self_attn_input.mask_attn(mask_attn.clone()); + } + let residual_path = self + .self_attn + .forward_cache(self_attn_input, &mut cache.self_attn) + .context; + + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Cross attention residual path. + // Normalize. + let residual_path = if self.norm_first { + cache + .norm_1 + .forward_autoregressive(x.clone(), 1, |x| self.norm_1.forward(x)) + } else { + x = cache + .norm_1 + .forward_autoregressive(x, 1, |x| self.norm_1.forward(x)); + x.clone() + }; + + // Cross attention. + let mut cross_attn_input = + MhaInput::new(residual_path, input.memory.clone(), input.memory.clone()); + if let Some(mask_pad) = &input.memory_mask_pad { + cross_attn_input = cross_attn_input.mask_pad(mask_pad.clone()); + } + if let Some(mask_attn) = &input.memory_mask_attn { + cross_attn_input = cross_attn_input.mask_attn(mask_attn.clone()); + } + let residual_path = self + .cross_attn + .forward_cache(cross_attn_input, &mut cache.cross_attn) + .context; + + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Feed forward residual path. + // Normalize. + let residual_path = if self.norm_first { + cache + .norm_2 + .forward_autoregressive(x.clone(), 1, |x| self.norm_2.forward(x)) + } else { + x = cache + .norm_2 + .forward_autoregressive(x, 1, |x| self.norm_2.forward(x)); + x.clone() + }; + + let residual_path = cache + .pwff + .forward_autoregressive(residual_path, 1, |x| self.pwff.forward(x)); + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Main path. + // Normalize. + if !self.norm_first { + x = cache + .norm_3 + .forward_autoregressive(x, 1, |x| self.norm_3.forward(x)) + } + + input.target = x; + input + } +} + +impl TransformerDecoder { + /// Applies the forward pass. + pub fn forward(&self, mut input: TransformerDecoderInput) -> Tensor { + for layer in self.layers.iter() { + input = layer.forward(input); + } + + input.target + } + + /// Applies the forward pass on the input using autoregressive cache. + pub fn forward_autoregressive_inference( + &self, + mut input: TransformerDecoderInput, + cache: &mut TransformerDecoderAutoregressiveCache, + ) -> Tensor { + for i in 0..self.layers.len() { + let layer = self.layers.get(i).unwrap(); + let cache = cache.layers.get_mut(i).unwrap(); + + input = layer.forward_autoregressive_inference(input, cache); + } + + input.target + } + /// Create an empty autoregressive cache. + pub fn new_autoregressive_cache(&self) -> TransformerDecoderAutoregressiveCache { + TransformerDecoderAutoregressiveCache::empty(self.layers.len()) + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::Device; + + use super::*; + use crate::{TestBackend, attention::generate_autoregressive_mask}; + + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_autoregressive_norm_last() { + let [d_model, d_ff, n_heads, num_layers] = [12, 24, 2, 3]; + let device = Default::default(); + TestBackend::seed(&device, 0); + + test_autoregressive( + TransformerDecoderConfig::new(d_model, d_ff, n_heads, num_layers) + .with_norm_first(false), + ) + } + + #[test] + fn test_autoregressive_norm_first() { + let [d_model, d_ff, n_heads, num_layers] = [12, 24, 2, 3]; + let device = Default::default(); + TestBackend::seed(&device, 0); + + test_autoregressive( + TransformerDecoderConfig::new(d_model, d_ff, n_heads, num_layers).with_norm_first(true), + ) + } + + fn test_autoregressive(config: TransformerDecoderConfig) { + let device: Device = Default::default(); + let [batch_size, seq_length, d_model] = [3, 4, config.d_model]; + let transformer = config.init::(&device); + + let memory = Tensor::arange(0..(batch_size * seq_length * d_model) as i64, &device) + .float() + .reshape([batch_size, seq_length, d_model]); + let target = Tensor::arange(0..(batch_size * seq_length * d_model) as i64, &device) + .float() + .reshape([batch_size, seq_length, d_model]); + let mask_attn = generate_autoregressive_mask(batch_size, seq_length, &target.device()); + let input = TransformerDecoderInput::new(target.clone(), memory.clone()) + .target_mask_attn(mask_attn); + + // Normal forward using masking. + let output_1 = transformer.forward(input); + + // Forward using the autoregressive cache. + let mut output_2 = Vec::new(); + let mut cache = transformer.new_autoregressive_cache(); + + for i in 1..seq_length + 1 { + let target = target.clone().slice([0..batch_size, 0..i, 0..d_model]); + + let mask_attn = generate_autoregressive_mask(batch_size, i, &target.device()); + let input = TransformerDecoderInput::new(target.clone(), memory.clone()) + .target_mask_attn(mask_attn); + let next_tok = transformer // Greedy sampling + .forward_autoregressive_inference(input, &mut cache) + .slice([0..batch_size, i - 1..i, 0..d_model]); + output_2.push(next_tok); + } + + let output_2 = Tensor::cat(output_2, 1); + + // Should produce the same tokens. + let tolerance = Tolerance::rel_abs(5e-3, 1e-4); + output_1 + .into_data() + .assert_approx_eq::(&output_2.into_data(), tolerance); + } + + #[test] + fn display() { + let config = TransformerDecoderConfig::new(2, 4, 2, 3); + let transformer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{transformer}"), + "TransformerDecoder {d_model: 2, d_ff: 4, n_heads: 2, n_layers: 3, \ + dropout: 0.1, norm_first: false, quiet_softmax: false, params: 246}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/encoder.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/encoder.rs new file mode 100644 index 0000000..d10c1ca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/encoder.rs @@ -0,0 +1,489 @@ +use burn_core as burn; + +use alloc::vec::Vec; + +use super::{PositionWiseFeedForward, PositionWiseFeedForwardConfig}; +use crate::{ + Dropout, DropoutConfig, LayerNorm, LayerNormConfig, + activation::ActivationConfig, + attention::{MhaCache, MhaInput, MultiHeadAttention, MultiHeadAttentionConfig}, + cache::TensorCache, +}; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::{Bool, Tensor, backend::Backend}; + +/// Configuration to create a [Transformer Encoder](TransformerEncoder) layer using the [init function](TransformerEncoderConfig::init). +#[derive(Config, Debug)] +pub struct TransformerEncoderConfig { + /// The size of the model. + pub d_model: usize, + /// The size of the position-wise feed-forward network. + pub d_ff: usize, + /// The number of attention heads. + pub n_heads: usize, + /// The number of layers. + pub n_layers: usize, + /// The dropout rate. Default: 0.1 + #[config(default = 0.1)] + pub dropout: f64, + /// Layer norm will be applied first instead of after the other modules. + #[config(default = false)] + pub norm_first: bool, + /// Use "quiet softmax" instead of regular softmax. + /// + /// - Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head). + /// - Usage may reduce the entropy of weights in the model, enhancing quantization and compression. + /// + /// Reference: + #[config(default = false)] + pub quiet_softmax: bool, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}" + )] + pub initializer: Initializer, + /// The activation function used in the position-wise feed-forward network. Default: Gelu + #[config(default = "ActivationConfig::Gelu")] + pub activation: ActivationConfig, + /// The epsilon value for layer normalization. Default: 1e-5 + #[config(default = 1e-5)] + pub layer_norm_eps: f64, +} + +/// The transformer encoder module as describe in the paper [Attention Is All You Need](https://arxiv.org/abs/1706.03762). +/// +/// # Params +/// +/// - layers: transformer encoder layers with `d_model` input and output features. +/// +/// Should be created using [TransformerEncoderConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct TransformerEncoder { + /// The transformer encoder layers. + pub layers: Vec>, + + /// The size of the model. + pub d_model: usize, + + /// The size of the position-wise feed-forward network. + pub d_ff: usize, + + /// The number of attention heads. + pub n_heads: usize, + + /// The number of layers. + pub n_layers: usize, + + /// The dropout rate. Default: 0.1 + pub dropout: f64, + + /// Layer norm will be applied first instead of after the other modules. + pub norm_first: bool, + + /// Use "quiet softmax" instead of regular softmax. + pub quiet_softmax: bool, +} + +impl ModuleDisplay for TransformerEncoder { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("d_model", &self.d_model) + .add("d_ff", &self.d_ff) + .add("n_heads", &self.n_heads) + .add("n_layers", &self.n_layers) + .add("dropout", &self.dropout) + .add("norm_first", &self.norm_first) + .add("quiet_softmax", &self.quiet_softmax) + .optional() + } +} + +/// [Transformer Encoder](TransformerEncoder) forward pass input argument. +#[derive(Debug)] +pub struct TransformerEncoderInput { + tensor: Tensor, + mask_pad: Option>, + mask_attn: Option>, +} + +impl TransformerEncoderInput { + /// Create a [transformer encoder](TransformerEncoder) input argument. + pub fn new(tensor: Tensor) -> Self { + Self { + tensor, + mask_pad: None, + mask_attn: None, + } + } + + /// Register the padding mask. + pub fn mask_pad(mut self, mask_pad: Tensor) -> Self { + self.mask_pad = Some(mask_pad); + self + } + + /// Register the attention mask. + pub fn mask_attn(mut self, mask_attn: Tensor) -> Self { + self.mask_attn = Some(mask_attn); + self + } +} +impl TransformerEncoderConfig { + /// Initialize a new [transformer encoder](TransformerEncoder) module. + pub fn init(&self, device: &B::Device) -> TransformerEncoder { + let layers = (0..self.n_layers) + .map(|_| TransformerEncoderLayer::new(self, device)) + .collect::>(); + + TransformerEncoder { + layers, + d_model: self.d_model, + d_ff: self.d_ff, + n_heads: self.n_heads, + n_layers: self.n_layers, + dropout: self.dropout, + norm_first: self.norm_first, + quiet_softmax: self.quiet_softmax, + } + } +} + +impl TransformerEncoder { + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - tensor: `[batch_size, seq_length, d_model]` + /// - output: `[batch_size, seq_length, d_model]` + pub fn forward(&self, input: TransformerEncoderInput) -> Tensor { + let mut x = input.tensor; + + for layer in self.layers.iter() { + x = layer.forward(x, input.mask_pad.clone(), input.mask_attn.clone()); + } + + x + } + /// Applies the forward pass on the input tensor using autoregressive cache. + /// + /// # Shapes + /// + /// - tensor: `[batch_size, seq_length, d_model]` + /// - output: `[batch_size, seq_length, d_model]` + pub fn forward_autoregressive_inference( + &self, + input: TransformerEncoderInput, + cache: &mut TransformerEncoderAutoregressiveCache, + ) -> Tensor { + let mut x = input.tensor; + + for i in 0..self.layers.len() { + let layer = self.layers.get(i).unwrap(); + let cache = cache.layers.get_mut(i).unwrap(); + + x = layer.forward_autoregressive_inference( + x, + input.mask_pad.clone(), + input.mask_attn.clone(), + cache, + ); + } + + x + } + + /// Create an empty autoregressive cache. + pub fn new_autoregressive_cache(&self) -> TransformerEncoderAutoregressiveCache { + TransformerEncoderAutoregressiveCache::empty(self.layers.len()) + } +} + +/// Transformer encoder layer module. +#[derive(Module, Debug)] +pub struct TransformerEncoderLayer { + /// Multi-head self-attention sub-layer. + pub mha: MultiHeadAttention, + /// Position-wise feed-forward sub-layer. + pub pwff: PositionWiseFeedForward, + /// Layer normalization applied around the feed-forward sub-layer. + pub norm_1: LayerNorm, + /// Layer normalization applied around the attention sub-layer. + pub norm_2: LayerNorm, + /// Dropout module applied to residual connections. + pub dropout: Dropout, + /// If `true`, apply layer normalization before sub-layers (pre-norm), + /// otherwise apply it after (post-norm). + pub norm_first: bool, +} + +impl TransformerEncoderLayer { + /// Create a new transformer encoder layer from the given configuration. + pub fn new(config: &TransformerEncoderConfig, device: &B::Device) -> Self { + let mha = MultiHeadAttentionConfig::new(config.d_model, config.n_heads) + .with_initializer(config.initializer.clone()) + .with_dropout(config.dropout) + .with_quiet_softmax(config.quiet_softmax) + .init(device); + let norm_1 = LayerNormConfig::new(config.d_model) + .with_epsilon(config.layer_norm_eps) + .init(device); + let norm_2 = LayerNormConfig::new(config.d_model) + .with_epsilon(config.layer_norm_eps) + .init(device); + let dropout = DropoutConfig::new(config.dropout).init(); + let pwff = PositionWiseFeedForwardConfig::new(config.d_model, config.d_ff) + .with_initializer(config.initializer.clone()) + .with_dropout(config.dropout) + .with_activation(config.activation.clone()) + .init(device); + + Self { + mha, + norm_1, + norm_2, + pwff, + dropout, + norm_first: config.norm_first, + } + } + + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[batch_size, seq_length, d_model]` + /// - output: `[batch_size, seq_length, d_model]` + pub fn forward( + &self, + input: Tensor, + mask_pad: Option>, + mask_attn: Option>, + ) -> Tensor { + // Multi-head attention residual path. + let x = input; + let mut residual_path = x.clone(); + + // Normalize. + if self.norm_first { + residual_path = self.norm_2.forward(residual_path) + } + + // Multi-head attention. + let mut input_mhs = MhaInput::self_attn(residual_path); + if let Some(mask_pad) = mask_pad { + input_mhs = input_mhs.mask_pad(mask_pad); + } + if let Some(mask_attn) = mask_attn { + input_mhs = input_mhs.mask_attn(mask_attn); + } + let residual_path = self.mha.forward(input_mhs).context; + + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Feed forward residual path. + // Normalize. + let residual_path = if self.norm_first { + self.norm_1.forward(x.clone()) + } else { + x = self.norm_1.forward(x); + x.clone() + }; + + // Feed forward. + let residual_path = self.pwff.forward(residual_path); + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Main path. + // Normalize. + if !self.norm_first { + x = self.norm_2.forward(x) + } + + x + } + + /// Applies the forward pass using an autoregressive cache. + pub fn forward_autoregressive_inference( + &self, + input: Tensor, + mask_pad: Option>, + mask_attn: Option>, + cache: &mut TransformerEncoderLayerAutoregressiveCache, + ) -> Tensor { + // Multi-head attention residual path. + let x = input; + let mut residual_path = x.clone(); + + // Normalize. + if self.norm_first { + residual_path = cache + .norm_2 + .forward_autoregressive(residual_path, 1, |x| self.norm_2.forward(x)) + } + + // Multi-head attention. + let mut input_mhs = MhaInput::self_attn(residual_path); + if let Some(mask_pad) = mask_pad { + input_mhs = input_mhs.mask_pad(mask_pad); + } + if let Some(mask_attn) = mask_attn { + input_mhs = input_mhs.mask_attn(mask_attn); + } + let residual_path = self.mha.forward_cache(input_mhs, &mut cache.mha).context; + + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Feed forward residual path. + // Normalize. + let residual_path = if self.norm_first { + cache + .norm_1 + .forward_autoregressive(x.clone(), 1, |x| self.norm_1.forward(x)) + } else { + x = cache + .norm_1 + .forward_autoregressive(x, 1, |x| self.norm_1.forward(x)); + x.clone() + }; + + // Feed forward. + let residual_path = cache + .pwff + .forward_autoregressive(residual_path, 1, |x| self.pwff.forward(x)); + let residual_path = self.dropout.forward(residual_path); + let mut x = x + residual_path; + + // Main path. + // Normalize. + if !self.norm_first { + x = cache + .norm_2 + .forward_autoregressive(x, 1, |x| self.norm_2.forward(x)) + } + + x + } +} + +/// Autoregressive cache for a single [Transformer Encoder Layer](TransformerEncoderLayer). +pub struct TransformerEncoderLayerAutoregressiveCache { + /// Multi-head attention cache. + pub mha: MhaCache, + /// Position-wise feed-forward cache. + pub pwff: TensorCache, + /// First layer norm cache. + pub norm_1: TensorCache, + /// Second layer norm cache. + pub norm_2: TensorCache, +} + +impl TransformerEncoderLayerAutoregressiveCache { + /// Create an empty cache. + pub fn empty() -> Self { + Self { + mha: MhaCache::autoregressive(), + pwff: TensorCache::empty(), + norm_1: TensorCache::empty(), + norm_2: TensorCache::empty(), + } + } +} + +/// Autoregressive cache for the [Transformer Encoder](TransformerEncoder) layer. +/// +/// To be used during inference when decoding tokens. +pub struct TransformerEncoderAutoregressiveCache { + layers: Vec>, +} + +impl TransformerEncoderAutoregressiveCache { + fn empty(num_layers: usize) -> Self { + Self { + layers: (0..num_layers) + .map(|_| TransformerEncoderLayerAutoregressiveCache::empty()) + .collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TestBackend, attention::generate_autoregressive_mask}; + use burn::tensor::Distribution; + use burn::tensor::{Tolerance, ops::FloatElem}; + type FT = FloatElem; + + #[test] + fn test_autoregressive_norm_last() { + let [d_model, d_ff, n_heads, num_layers] = [12, 24, 2, 3]; + test_autoregressive( + TransformerEncoderConfig::new(d_model, d_ff, n_heads, num_layers) + .with_norm_first(false), + ) + } + + #[test] + fn test_autoregressive_norm_first() { + let [d_model, d_ff, n_heads, num_layers] = [12, 24, 2, 3]; + test_autoregressive( + TransformerEncoderConfig::new(d_model, d_ff, n_heads, num_layers).with_norm_first(true), + ) + } + + fn test_autoregressive(config: TransformerEncoderConfig) { + let [batch_size, seq_length, d_model] = [3, 4, config.d_model]; + let device = Default::default(); + let transformer = config.init(&device); + + let tensor = Tensor::::random( + [batch_size, seq_length, d_model], + Distribution::Default, + &device, + ); + let mask_attn = generate_autoregressive_mask(batch_size, seq_length, &tensor.device()); + let input = TransformerEncoderInput::new(tensor.clone()).mask_attn(mask_attn); + + let output_1 = transformer.forward(input); + let mut output_2 = Vec::new(); + let mut cache = transformer.new_autoregressive_cache(); + + for i in 1..seq_length + 1 { + let tensor = tensor.clone().slice([0..batch_size, 0..i, 0..d_model]); + let input = TransformerEncoderInput::new(tensor.clone()); + let next_tok = transformer + .forward_autoregressive_inference(input, &mut cache) + .slice([0..batch_size, i - 1..i, 0..d_model]); + output_2.push(next_tok); + } + + let output_2 = Tensor::cat(output_2, 1); + + output_1 + .into_data() + .assert_approx_eq::(&output_2.into_data(), Tolerance::permissive()); + } + + #[test] + fn display() { + let config = TransformerEncoderConfig::new(2, 4, 2, 3); + let transformer = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{transformer}"), + "TransformerEncoder {d_model: 2, d_ff: 4, n_heads: 2, \ + n_layers: 3, dropout: 0.1, norm_first: false, quiet_softmax: false, params: 162}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/mod.rs new file mode 100644 index 0000000..54397c6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/mod.rs @@ -0,0 +1,7 @@ +mod decoder; +mod encoder; +mod pwff; + +pub use decoder::*; +pub use encoder::*; +pub use pwff::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/pwff.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/pwff.rs new file mode 100644 index 0000000..ef9be3e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/transformer/pwff.rs @@ -0,0 +1,117 @@ +use burn_core as burn; + +use crate::activation::{Activation, ActivationConfig}; +use crate::{Dropout, DropoutConfig, Linear, LinearConfig}; +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Initializer, Module, ModuleDisplay}; +use burn::tensor::{Tensor, backend::Backend}; + +/// Configuration to create a [position-wise feed-forward](PositionWiseFeedForward) layer using the [init function](PositionWiseFeedForwardConfig::init). +#[derive(Config, Debug)] +pub struct PositionWiseFeedForwardConfig { + /// The size of the input and output features. + pub d_model: usize, + /// The size of the hidden inner features. + pub d_ff: usize, + /// The dropout rate. Default: 0.1 + #[config(default = 0.1)] + pub dropout: f64, + /// The type of function used to initialize neural network parameters + #[config( + default = "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}" + )] + pub initializer: Initializer, + /// The activation function used between the two linear layers. Default: Gelu + #[config(default = "ActivationConfig::Gelu")] + pub activation: ActivationConfig, +} + +/// Applies the position-wise feed-forward network to the input tensor from the paper [Attention Is All You Need](https://arxiv.org/pdf/1706.03762v7). +/// +/// # Params +/// +/// - linear inner: Linear layer with `d_model` input features and `d_ff` output features. +/// - linear outer: Linear layer with `d_ff` input features and `d_model` output features. +/// +/// `FFN(x) = max(0, xW1 + b1)W2 + b2` +/// +/// Should be created using [PositionWiseFeedForwardConfig] +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct PositionWiseFeedForward { + /// Linear layer with `d_model` input features and `d_ff` output features. + pub linear_inner: Linear, + /// Linear layer with `d_ff` input features and `d_model` output features. + pub linear_outer: Linear, + /// Dropout layer. + pub dropout: Dropout, + /// Activation function. + pub activation: Activation, +} + +impl ModuleDisplay for PositionWiseFeedForward { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let [d_model, dff] = self.linear_inner.weight.shape().dims(); + + content + .add("d_model", &d_model) + .add("d_ff", &dff) + .add("prob", &self.dropout.prob) + .optional() + } +} + +impl PositionWiseFeedForwardConfig { + /// Initialize a new [position-wise feed-forward](PositionWiseFeedForward) module. + pub fn init(&self, device: &B::Device) -> PositionWiseFeedForward { + PositionWiseFeedForward { + linear_inner: LinearConfig::new(self.d_model, self.d_ff) + .with_initializer(self.initializer.clone()) + .init(device), + linear_outer: LinearConfig::new(self.d_ff, self.d_model) + .with_initializer(self.initializer.clone()) + .init(device), + dropout: DropoutConfig::new(self.dropout).init(), + activation: self.activation.init(device), + } + } +} + +impl PositionWiseFeedForward { + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - tensor: `[batch_size, seq_length, d_model]` + /// - output: `[batch_size, seq_length, d_model]` + pub fn forward(&self, input: Tensor) -> Tensor { + let x = self.linear_inner.forward(input); + let x = self.activation.forward(x); + let x = self.dropout.forward(x); + + self.linear_outer.forward(x) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn display() { + let config = PositionWiseFeedForwardConfig::new(2, 4); + let pwff = config.init::(&Default::default()); + + assert_eq!( + alloc::format!("{pwff}"), + "PositionWiseFeedForward {d_model: 2, d_ff: 4, prob: 0.1, params: 22}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/unfold.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/unfold.rs new file mode 100644 index 0000000..8e701bf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/modules/unfold.rs @@ -0,0 +1,104 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay}; + +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn::tensor::module::unfold4d; +use burn::tensor::ops::UnfoldOptions; + +/// Configuration to create an [unfold 4d](Unfold4d) layer using the [init function](Unfold4dConfig::init). +#[derive(Config, Debug)] +pub struct Unfold4dConfig { + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The stride of the convolution. + #[config(default = "[1, 1]")] + pub stride: [usize; 2], + /// Spacing between kernel elements. + #[config(default = "[1, 1]")] + pub dilation: [usize; 2], + /// The padding configuration. + #[config(default = "[0, 0]")] + pub padding: [usize; 2], +} + +/// Four-dimensional unfolding. +/// +/// Should be created with [Unfold4dConfig]. +#[derive(Module, Clone, Debug)] +#[module(custom_display)] +pub struct Unfold4d { + /// The size of the kernel. + pub kernel_size: [usize; 2], + /// The stride of the convolution. + pub stride: [usize; 2], + /// Spacing between kernel elements. + pub dilation: [usize; 2], + /// The padding configuration. + pub padding: [usize; 2], +} + +impl ModuleDisplay for Unfold4d { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("kernel_size", &alloc::format!("{:?}", &self.kernel_size)) + .add("stride", &alloc::format!("{:?}", &self.stride)) + .add("dilation", &alloc::format!("{:?}", &self.dilation)) + .add("padding", &alloc::format!("{:?}", &self.padding)) + .optional() + } +} + +impl Unfold4dConfig { + /// Initializes a new [Unfold4d] module. + pub fn init(&self) -> Unfold4d { + Unfold4d { + kernel_size: self.kernel_size, + stride: self.stride, + dilation: self.dilation, + padding: self.padding, + } + } +} + +impl Unfold4d { + /// Applies the forward pass on the input tensor. + /// + /// See [unfold4d](burn::tensor::module::unfold4d) for more information. + /// + /// # Shapes + /// + /// input: `[batch_size, channels_in, height, width]` + /// returns: `[batch_size, channels_in * kernel_size_1 * kernel_size_2, number of blocks]` + pub fn forward(&self, input: Tensor) -> Tensor { + unfold4d( + input, + self.kernel_size, + UnfoldOptions::new(self.stride, self.padding, self.dilation), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let config = Unfold4dConfig::new([3, 3]); + let unfold = config.init(); + + assert_eq!( + alloc::format!("{unfold}"), + "Unfold4d {kernel_size: [3, 3], stride: [1, 1], dilation: [1, 1], padding: [0, 0]}" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/src/padding.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/padding.rs new file mode 100644 index 0000000..cfacfc6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/src/padding.rs @@ -0,0 +1,247 @@ +use burn_core as burn; + +use burn::config::Config; + +/// Calculate asymmetric padding for "same" convolution. +/// Returns (start_padding, end_padding) where start is applied first (top/left). +/// For odd total padding, the extra pad goes to the end (bottom/right) following ONNX convention. +fn calculate_same_padding(kernel_size: usize, stride: usize, size_in: usize) -> (usize, usize) { + let size_out = size_in.div_ceil(stride); // ceil division for same padding + let total_padding = if size_out > 0 { + let needed = (size_out - 1) * stride + kernel_size; + needed.saturating_sub(size_in) + } else { + 0 + }; + let pad_start = total_padding / 2; + let pad_end = total_padding - pad_start; + (pad_start, pad_end) +} + +/// Padding configuration for 1D operators. +#[derive(Config, Debug, PartialEq)] +pub enum PaddingConfig1d { + /// Dynamically calculates padding to ensure output size matches input size. + Same, + /// No padding applied. + Valid, + /// Applies explicit padding values. + /// Format: (left, right) + /// For symmetric padding, use the same value for both (e.g., `Explicit(1, 1)`). + Explicit(usize, usize), +} + +impl PaddingConfig1d { + /// Calculate padding as (left, right) pair for 1D operations. + /// For `Same` padding, this computes the actual asymmetric padding if needed. + pub(crate) fn calculate_padding_1d_pair( + &self, + length: usize, + kernel_size: usize, + stride: usize, + ) -> (usize, usize) { + match self { + Self::Valid => (0, 0), + Self::Same => calculate_same_padding(kernel_size, stride, length), + Self::Explicit(left, right) => (*left, *right), + } + } +} + +/// Padding configuration for 2D operators. +#[derive(Config, Debug, PartialEq)] +pub enum PaddingConfig2d { + /// Dynamically calculates padding to preserve input dimensions in output. + Same, + /// No padding applied. + Valid, + /// Applies explicit padding values. + /// Format: (top, left, bottom, right) + /// For symmetric padding, use matching values (e.g., `Explicit(1, 1, 1, 1)`). + Explicit(usize, usize, usize, usize), +} + +impl PaddingConfig2d { + /// Calculate padding as ((top, bottom), (left, right)) pairs for 2D operations. + /// For `Same` padding, this computes the actual asymmetric padding if needed. + pub(crate) fn calculate_padding_2d_pairs( + &self, + height: usize, + width: usize, + kernel_size: &[usize; 2], + stride: &[usize; 2], + ) -> ((usize, usize), (usize, usize)) { + match self { + Self::Valid => ((0, 0), (0, 0)), + Self::Same => { + let (top, bottom) = calculate_same_padding(kernel_size[0], stride[0], height); + let (left, right) = calculate_same_padding(kernel_size[1], stride[1], width); + ((top, bottom), (left, right)) + } + Self::Explicit(top, left, bottom, right) => ((*top, *bottom), (*left, *right)), + } + } + + /// Calculate symmetric padding for 2D operations. + /// Returns padding values [height, width] (same for both sides). + /// Panics if asymmetric padding is detected. + pub(crate) fn calculate_padding_2d( + &self, + height: usize, + width: usize, + kernel_size: &[usize; 2], + stride: &[usize; 2], + ) -> [usize; 2] { + let ((top, bottom), (left, right)) = + self.calculate_padding_2d_pairs(height, width, kernel_size, stride); + if top != bottom || left != right { + panic!("Asymmetric padding should be handled via calculate_padding_2d_pairs()") + } + [top, left] + } +} + +/// Padding configuration for 3D operators. +#[derive(Config, Debug, PartialEq)] +pub enum PaddingConfig3d { + /// Dynamically calculates padding to preserve input dimensions in output. + Same, + /// No padding applied. + Valid, + /// Applies explicit symmetric padding values. + /// Format: (depth, height, width) — same padding on both sides of each dimension. + Explicit(usize, usize, usize), +} + +impl PaddingConfig3d { + /// Calculate symmetric padding for 3D operations. + /// Returns padding values [depth, height, width] (same for both sides). + pub(crate) fn calculate_padding_3d( + &self, + depth: usize, + height: usize, + width: usize, + kernel_size: &[usize; 3], + stride: &[usize; 3], + ) -> [usize; 3] { + match self { + Self::Valid => [0, 0, 0], + Self::Same => { + let (front, back) = calculate_same_padding(kernel_size[0], stride[0], depth); + let (top, bottom) = calculate_same_padding(kernel_size[1], stride[1], height); + let (left, right) = calculate_same_padding(kernel_size[2], stride[2], width); + if front != back || top != bottom || left != right { + panic!( + "Asymmetric 3D 'Same' padding is not supported. \ + Use odd kernel sizes for symmetric padding." + ) + } + [front, top, left] + } + Self::Explicit(depth, height, width) => [*depth, *height, *width], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ==================== PaddingConfig1d Tests ==================== + + #[test] + fn test_padding_config_1d_calculate_pair_valid() { + let padding = PaddingConfig1d::Valid; + assert_eq!(padding.calculate_padding_1d_pair(10, 3, 1), (0, 0)); + } + + #[test] + fn test_padding_config_1d_calculate_pair_explicit() { + let padding = PaddingConfig1d::Explicit(1, 2); + assert_eq!(padding.calculate_padding_1d_pair(10, 3, 1), (1, 2)); + } + + #[test] + fn test_padding_config_1d_calculate_pair_same() { + let padding = PaddingConfig1d::Same; + // kernel=3, stride=1, length=10: total=2, start=1, end=1 + assert_eq!(padding.calculate_padding_1d_pair(10, 3, 1), (1, 1)); + } + + // ==================== PaddingConfig2d Tests ==================== + + #[test] + fn test_padding_config_2d_calculate_pairs_valid() { + let padding = PaddingConfig2d::Valid; + assert_eq!( + padding.calculate_padding_2d_pairs(10, 10, &[3, 3], &[1, 1]), + ((0, 0), (0, 0)) + ); + } + + #[test] + fn test_padding_config_2d_calculate_pairs_explicit() { + let padding = PaddingConfig2d::Explicit(1, 2, 3, 4); + assert_eq!( + padding.calculate_padding_2d_pairs(10, 10, &[3, 3], &[1, 1]), + ((1, 3), (2, 4)) + ); + } + + #[test] + fn test_padding_config_2d_calculate_symmetric_valid() { + let padding = PaddingConfig2d::Valid; + assert_eq!( + padding.calculate_padding_2d(10, 10, &[3, 3], &[1, 1]), + [0, 0] + ); + } + + #[test] + fn test_padding_config_2d_calculate_symmetric_explicit() { + let padding = PaddingConfig2d::Explicit(2, 3, 2, 3); + assert_eq!( + padding.calculate_padding_2d(10, 10, &[3, 3], &[1, 1]), + [2, 3] + ); + } + + #[test] + #[should_panic( + expected = "Asymmetric padding should be handled via calculate_padding_2d_pairs" + )] + fn test_padding_config_2d_calculate_symmetric_asymmetric_panics() { + let padding = PaddingConfig2d::Explicit(1, 2, 3, 4); + let _ = padding.calculate_padding_2d(10, 10, &[3, 3], &[1, 1]); + } + + // ==================== PaddingConfig3d Tests ==================== + + #[test] + fn test_padding_config_3d_calculate_valid() { + let padding = PaddingConfig3d::Valid; + assert_eq!( + padding.calculate_padding_3d(10, 10, 10, &[3, 3, 3], &[1, 1, 1]), + [0, 0, 0] + ); + } + + #[test] + fn test_padding_config_3d_calculate_explicit() { + let padding = PaddingConfig3d::Explicit(1, 2, 3); + assert_eq!( + padding.calculate_padding_3d(10, 10, 10, &[3, 3, 3], &[1, 1, 1]), + [1, 2, 3] + ); + } + + #[test] + fn test_padding_config_3d_calculate_same_odd_kernel() { + let padding = PaddingConfig3d::Same; + // kernel=3, stride=1: total=2, symmetric (1,1) per dim + assert_eq!( + padding.calculate_padding_3d(10, 10, 10, &[3, 3, 3], &[1, 1, 1]), + [1, 1, 1] + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-nn/tests/quantize.rs b/crates/stable-diffusion-burn/burn-crates/burn-nn/tests/quantize.rs new file mode 100644 index 0000000..c671c5a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-nn/tests/quantize.rs @@ -0,0 +1,167 @@ +use burn_core as burn; + +use burn::module::{Module, Quantizer}; +use burn::tensor::{ + Device, Distribution, Tensor, Tolerance, + ops::{FloatElem, QuantizedTensor}, + quantization::{ + Calibration, QTensorPrimitive, QuantLevel, QuantParam, QuantScheme, QuantValue, + }, +}; +use burn_nn::{ + Linear, LinearConfig, + transformer::{TransformerEncoder, TransformerEncoderConfig, TransformerEncoderInput}, +}; + +#[cfg(all( + test, + not(feature = "test-wgpu"), + not(feature = "test-cuda"), + not(feature = "test-rocm") +))] +pub type B = burn_ndarray::NdArray; + +#[cfg(all(test, feature = "test-wgpu"))] +/// Backend for test cases +pub type B = burn_wgpu::Wgpu; + +#[cfg(all(test, feature = "test-cuda"))] +/// Backend for test cases +pub type B = burn_cuda::Cuda; + +#[cfg(all(test, feature = "test-rocm"))] +/// Backend for test cases +pub type B = burn_rocm::Rocm; + +fn should_quantize_module, const D: usize, F: Fn(&M) -> Tensor>( + module: M, + scheme: QuantScheme, + func: F, + tolerance: Tolerance>, +) { + let result = func(&module); + + let calibration = Calibration::MinMax; + let mut quantizer = Quantizer { + calibration, + scheme, + }; + let q_module = module.quantize_weights(&mut quantizer); + let q_result = func(&q_module); + + result + .into_data() + .assert_approx_eq::(&q_result.into_data(), tolerance); +} + +#[test] +fn should_quantize_transformer() { + let device: Device = Default::default(); + let transformer: TransformerEncoder = + TransformerEncoderConfig::new(128, 256, 2, 2).init(&device); + let signal = Tensor::random([2, 32, 128], Distribution::Default, &device); + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([32])) + .with_param(QuantParam::F32); + + should_quantize_module( + transformer, + scheme, + |tr| tr.forward(TransformerEncoderInput::new(signal.clone())), + Tolerance::rel_abs(1e-2, 2e-2), // slightly higher abs tolerance (permissive: 1e-2) + ); +} + +#[test] +fn should_quantize_linear_128_256() { + let device: Device = Default::default(); + let transformer: Linear = LinearConfig::new(128, 256).with_bias(false).init(&device); + let signal = Tensor::::random([1, 128], Distribution::Default, &device); + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::Tensor) + .with_param(QuantParam::F32); + + should_quantize_module( + transformer, + scheme, + |tr| tr.forward(signal.clone()), + Tolerance::permissive(), + ); +} + +#[test] +fn should_quantize_linear() { + let device: Device = Default::default(); + let transformer: Linear = LinearConfig::new(32, 32).with_bias(false).init(&device); + let signal = Tensor::::random([1, 32], Distribution::Default, &device); + // Default scheme should select supported QuantStore default + // TODO: set native if dtype is supported by the test backend + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::Tensor) + // .with_store(QuantStore::Native) + .with_param(QuantParam::F32); + + should_quantize_module( + transformer, + scheme, + |tr| tr.forward(signal.clone()), + Tolerance::permissive(), + ); +} + +#[test] +fn should_quantize_linear_weights() { + let device: Device = Default::default(); + let transformer: Linear = LinearConfig::new(32, 32).with_bias(false).init(&device); + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::Tensor) + .with_param(QuantParam::F32); + + should_quantize_module( + transformer, + scheme, + |tr| tr.weight.val().dequantize(), + Tolerance::permissive(), + ); +} + +#[test] +fn should_quantize_linear_blocks() { + let device: Device = Default::default(); + let transformer: Linear = LinearConfig::new(32, 32).with_bias(false).init(&device); + let signal = Tensor::::random([1, 32], Distribution::Default, &device); + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([16])) + // .with_store(QuantStore::Native) + .with_param(QuantParam::F32); + + should_quantize_module( + transformer, + scheme, + |tr| tr.forward(signal.clone()), + Tolerance::permissive(), + ); +} + +#[test] +fn should_quantize_linear_weights_blocks() { + let device: Device = Default::default(); + let transformer: Linear = LinearConfig::new(32, 32).with_bias(false).init(&device); + let scheme = as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([16])) + // .with_store(QuantStore::Native) + .with_param(QuantParam::F32); + + should_quantize_module( + transformer, + scheme, + |tr| tr.weight.val().dequantize(), + Tolerance::permissive(), + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/Cargo.toml new file mode 100644 index 0000000..47f3878 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/Cargo.toml @@ -0,0 +1,32 @@ +[package] +authors = [ + "nathanielsimard ", + "Dilshod Tadjibaev (@antimora)", +] +edition.workspace = true +license.workspace = true +name = "burn-no-std-tests" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-no-std-tests" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [] + +tracing = [ + "burn/tracing", + "burn-ndarray/tracing", + "burn-store/tracing", +] + +[dependencies] + +# ** Please make sure all dependencies support no_std ** + +burn = { path = "../burn", version = "=0.21.0-pre.2", default-features = false } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2", default-features = false } + +burn-store = { path = "../burn-store", version = "=0.21.0-pre.2", default-features = false, features = ["safetensors", "burnpack"]} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/README.md b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/README.md new file mode 100644 index 0000000..9fce45a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/README.md @@ -0,0 +1,29 @@ +The `burn-no-std-tests` contains integration tests aimed to check `no_std` compatibility of `burn`, `burn-core`, `burn-tensor` and `burn-ndarray` packages. + +Currently there is only a minimal test that checks if mnist model can be built with `no_std`. More tests should be added to check completeness. + +The continuous integration (CI) should build with additional targets: + + * `wasm32-unknown-unknown` - WebAssembly + * `thumbv7m-none-eabi` - ARM Cortex-M3 + * `thumbv6m-none-eabi` - ARM Cortex-M0+ + +Shell commands to build and test the package: + +```sh + +# install the new targets if not installed previously +rustup target add thumbv6m-none-eabi +rustup target add thumbv7m-none-eabi +rustup target add wasm32-unknown-unknown + +# build for various targets +cargo build # regular build +cargo build --target thumbv7m-none-eabi +cargo build --target wasm32-unknown-unknown +RUSTFLAGS="--cfg portable_atomic_unsafe_assume_single_core" cargo build --target thumbv6m-none-eabi + +# test +cargo test + + ``` \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/burnpack.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/burnpack.rs new file mode 100644 index 0000000..7f30981 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/burnpack.rs @@ -0,0 +1,158 @@ +// Test Burnpack storage in no-std environment + +use burn::{ + module::Module, + nn, + tensor::{Tensor, backend::Backend}, +}; + +use burn_store::{BurnpackStore, ModuleSnapshot, PathFilter}; + +/// Simple model for testing Burnpack storage +#[derive(Module, Debug)] +pub struct TestModel { + linear1: nn::Linear, + linear2: nn::Linear, + batch_norm: nn::BatchNorm, +} + +impl TestModel { + pub fn new(device: &B::Device) -> Self { + Self { + linear1: nn::LinearConfig::new(10, 20).init(device), + linear2: nn::LinearConfig::new(20, 10).init(device), + batch_norm: nn::BatchNormConfig::new(10).init(device), + } + } + + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.linear1.forward(x); + let x = self.linear2.forward(x); + // Apply batch norm (expand to 3D, apply, then squeeze back) + let x: Tensor = x.unsqueeze_dim(2); + let x = self.batch_norm.forward(x); + x.squeeze_dim(2) + } +} + +/// Test basic Burnpack save and load in no-std +pub fn test_burnpack_basic(device: &B::Device) { + // Create a model + let model = TestModel::::new(device); + + // Save to bytes (no file I/O in no-std) + let mut save_store = BurnpackStore::from_bytes(None); + model + .save_into(&mut save_store) + .expect("Failed to save model"); + + // Get the serialized bytes + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load from bytes + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut loaded_model = TestModel::::new(device); + let result = loaded_model + .load_from(&mut load_store) + .expect("Failed to load model"); + + // Verify all tensors were loaded + assert!(result.is_success(), "Should have no errors"); + assert!(!result.applied.is_empty(), "Should have loaded tensors"); + + // Test that the model still works + let input = Tensor::::ones([2, 10], device); + let _output = loaded_model.forward(input); +} + +/// Test Burnpack with filtering in no-std +pub fn test_burnpack_filtering(device: &B::Device) { + let model = TestModel::::new(device); + + // Save only linear1 weights + let filter = PathFilter::new() + .with_full_path("linear1.weight") + .with_full_path("linear1.bias"); + let mut save_store = BurnpackStore::from_bytes(None).with_filter(filter); + model + .save_into(&mut save_store) + .expect("Failed to save filtered model"); + + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load with partial loading allowed + let mut load_store = BurnpackStore::from_bytes(Some(bytes)).allow_partial(true); + let mut partial_model = TestModel::::new(device); + let result = partial_model + .load_from(&mut load_store) + .expect("Failed to load partial model"); + + // Verify that only linear1 was loaded + assert_eq!(result.applied.len(), 2, "Should have loaded 2 tensors"); + assert!(!result.missing.is_empty(), "Should have missing tensors"); +} + +/// Test Burnpack with metadata in no-std +pub fn test_burnpack_metadata(device: &B::Device) { + let model = TestModel::::new(device); + + // Save with metadata + let mut save_store = BurnpackStore::from_bytes(None) + .metadata("version", "1.0.0") + .metadata("environment", "no-std") + .metadata("model_type", "test"); + model + .save_into(&mut save_store) + .expect("Failed to save model with metadata"); + + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load and verify it works + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut loaded_model = TestModel::::new(device); + let result = loaded_model + .load_from(&mut load_store) + .expect("Failed to load model with metadata"); + + assert!(result.is_success(), "Should load successfully"); +} + +// Note: Key remapping test is omitted as KeyRemapper requires std feature + +// Note: Regex filtering test is omitted as with_regex requires std feature + +/// Test Burnpack with match_all in no-std +pub fn test_burnpack_match_all(device: &B::Device) { + let model = TestModel::::new(device); + + // Save with match_all (should save everything) + let mut save_store = BurnpackStore::from_bytes(None).match_all(); + model + .save_into(&mut save_store) + .expect("Failed to save model"); + + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load everything + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut loaded_model = TestModel::::new(device); + let result = loaded_model + .load_from(&mut load_store) + .expect("Failed to load model"); + + assert!(result.is_success(), "Should load successfully"); + // linear1 (weight, bias) + linear2 (weight, bias) + batch_norm (4 params) + assert_eq!(result.applied.len(), 8, "Should load all 8 tensors"); + assert!(result.missing.is_empty(), "Should have no missing tensors"); + assert!(result.unused.is_empty(), "Should have no unused tensors"); +} + +/// Run all Burnpack no-std tests +pub fn run_all_tests(device: &B::Device) { + test_burnpack_basic::(device); + test_burnpack_filtering::(device); + test_burnpack_metadata::(device); + // test_burnpack_remapping requires KeyRemapper which needs std + // test_burnpack_regex_filter requires with_regex which needs std + test_burnpack_match_all::(device); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/conv.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/conv.rs new file mode 100644 index 0000000..60ae230 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/conv.rs @@ -0,0 +1,49 @@ +// Originally copied from the burn/examples/mnist package + +use burn::{ + config::Config, + module::Module, + nn, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct ConvBlock { + conv: nn::conv::Conv2d, + pool: nn::pool::MaxPool2d, + activation: nn::Gelu, +} + +#[derive(Config, Debug)] +pub struct ConvBlockConfig { + channels: [usize; 2], + #[config(default = "[3, 3]")] + kernel_size: [usize; 2], +} + +impl ConvBlock { + pub fn new(config: &ConvBlockConfig, device: &B::Device) -> Self { + let conv = nn::conv::Conv2dConfig::new(config.channels, config.kernel_size) + .with_padding(nn::PaddingConfig2d::Same) + .init(device); + let pool = nn::pool::MaxPool2dConfig::new(config.kernel_size) + .with_strides([1, 1]) + .with_padding(nn::PaddingConfig2d::Same) + .init(); + let activation = nn::Gelu::new(); + + Self { + conv, + pool, + activation, + } + } + + pub fn forward(&self, input: Tensor) -> Tensor { + let x = self.conv.forward(input.clone()); + let x = self.pool.forward(x); + let x = self.activation.forward(x); + + (x + input) / 2.0 + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/lib.rs new file mode 100644 index 0000000..58de35e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/lib.rs @@ -0,0 +1,9 @@ +#![no_std] + +pub mod burnpack; +pub mod conv; +pub mod mlp; +pub mod model; +pub mod safetensors; + +extern crate alloc; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/mlp.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/mlp.rs new file mode 100644 index 0000000..45cc6cb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/mlp.rs @@ -0,0 +1,67 @@ +// Originally copied from the burn/examples/mnist package + +use alloc::vec::Vec; + +use burn::{ + config::Config, + module::Module, + nn, + tensor::{Tensor, backend::Backend}, +}; + +/// Configuration to create a [Multilayer Perceptron](Mlp) layer. +#[derive(Config, Debug)] +pub struct MlpConfig { + /// The number of layers. + #[config(default = 3)] + pub num_layers: usize, + /// The dropout rate. + #[config(default = 0.5)] + pub dropout: f64, + /// The size of each layer. + #[config(default = 256)] + pub d_model: usize, +} + +/// Multilayer Perceptron module. +#[derive(Module, Debug)] +pub struct Mlp { + linears: Vec>, + dropout: nn::Dropout, + activation: nn::Relu, +} + +impl Mlp { + /// Create the module from the given configuration. + pub fn new(config: &MlpConfig, device: &B::Device) -> Self { + let mut linears = Vec::with_capacity(config.num_layers); + + for _ in 0..config.num_layers { + linears.push(nn::LinearConfig::new(config.d_model, config.d_model).init(device)); + } + + Self { + linears, + dropout: nn::DropoutConfig::new(0.3).init(), + activation: nn::Relu::new(), + } + } + + /// Applies the forward pass on the input tensor. + /// + /// # Shapes + /// + /// - input: `[batch_size, d_model]` + /// - output: `[batch_size, d_model]` + pub fn forward(&self, input: Tensor) -> Tensor { + let mut x = input; + + for linear in self.linears.iter() { + x = linear.forward(x); + x = self.dropout.forward(x); + x = self.activation.forward(x); + } + + x + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/model.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/model.rs new file mode 100644 index 0000000..6e05cf1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/model.rs @@ -0,0 +1,66 @@ +// Originally copied from the burn/examples/mnist package + +use crate::{ + conv::{ConvBlock, ConvBlockConfig}, + mlp::{Mlp, MlpConfig}, +}; + +use burn::{ + config::Config, + module::Module, + nn, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Config, Debug)] +pub struct MnistConfig { + #[config(default = 42)] + pub seed: u64, + + pub mlp: MlpConfig, + + #[config(default = 784)] + pub input_size: usize, + + #[config(default = 10)] + pub output_size: usize, +} + +#[derive(Module, Debug)] +pub struct Model { + mlp: Mlp, + conv: ConvBlock, + input: nn::Linear, + output: nn::Linear, + num_classes: usize, +} + +impl Model { + pub fn new(config: &MnistConfig, device: &B::Device) -> Self { + let mlp = Mlp::new(&config.mlp, device); + let input = nn::LinearConfig::new(config.input_size, config.mlp.d_model).init(device); + let output = nn::LinearConfig::new(config.mlp.d_model, config.output_size).init(device); + let conv = ConvBlock::new(&ConvBlockConfig::new([1, 1]), device); + + Self { + mlp, + conv, + output, + input, + num_classes: config.output_size, + } + } + + pub fn forward(&self, input: Tensor) -> Tensor { + let [batch_size, height, width] = input.dims(); + + let x = input.reshape([batch_size, 1, height, width]).detach(); + let x = self.conv.forward(x); + let x = x.reshape([batch_size, height * width]); + + let x = self.input.forward(x); + let x = self.mlp.forward(x); + + self.output.forward(x) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/safetensors.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/safetensors.rs new file mode 100644 index 0000000..b70378d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/src/safetensors.rs @@ -0,0 +1,111 @@ +// Test SafeTensors storage in no-std environment + +use burn::{ + module::Module, + nn, + tensor::{Tensor, backend::Backend}, +}; + +use burn_store::{ModuleSnapshot, SafetensorsStore}; + +/// Simple model for testing SafeTensors storage +#[derive(Module, Debug)] +pub struct TestModel { + linear1: nn::Linear, + linear2: nn::Linear, +} + +impl TestModel { + pub fn new(device: &B::Device) -> Self { + Self { + linear1: nn::LinearConfig::new(10, 20).init(device), + linear2: nn::LinearConfig::new(20, 10).init(device), + } + } + + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.linear1.forward(x); + self.linear2.forward(x) + } +} + +/// Test basic SafeTensors save and load in no-std +pub fn test_safetensors_basic(device: &B::Device) { + // Create a model + let model = TestModel::::new(device); + + // Save to bytes (no file I/O in no-std) + let mut save_store = SafetensorsStore::from_bytes(None); + model + .save_into(&mut save_store) + .expect("Failed to save model"); + + // Get the serialized bytes + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load from bytes + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = TestModel::::new(device); + loaded_model + .load_from(&mut load_store) + .expect("Failed to load model"); + + // Test that the model still works + let input = Tensor::::ones([2, 10], device); + let _output = loaded_model.forward(input); +} + +/// Test SafeTensors with filtering in no-std +pub fn test_safetensors_filtering(device: &B::Device) { + let model = TestModel::::new(device); + + // Save only linear1 weights + let mut save_store = SafetensorsStore::from_bytes(None) + .with_full_path("linear1.weight") + .with_full_path("linear1.bias"); + model + .save_into(&mut save_store) + .expect("Failed to save filtered model"); + + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load with partial loading allowed + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)).allow_partial(true); + let mut partial_model = TestModel::::new(device); + let result = partial_model + .load_from(&mut load_store) + .expect("Failed to load partial model"); + + // Verify that only linear1 was loaded + assert_eq!(result.applied.len(), 2, "Should have loaded 2 tensors"); + assert!(!result.missing.is_empty(), "Should have missing tensors"); +} + +/// Test SafeTensors with metadata in no-std +pub fn test_safetensors_metadata(device: &B::Device) { + let model = TestModel::::new(device); + + // Save with metadata + let mut save_store = SafetensorsStore::from_bytes(None) + .metadata("version", "1.0.0") + .metadata("environment", "no-std"); + model + .save_into(&mut save_store) + .expect("Failed to save model with metadata"); + + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load and verify it works + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = TestModel::::new(device); + loaded_model + .load_from(&mut load_store) + .expect("Failed to load model with metadata"); +} + +/// Run all SafeTensors no-std tests +pub fn run_all_tests(device: &B::Device) { + test_safetensors_basic::(device); + test_safetensors_filtering::(device); + test_safetensors_metadata::(device); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/burnpack_tests.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/burnpack_tests.rs new file mode 100644 index 0000000..c050288 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/burnpack_tests.rs @@ -0,0 +1,12 @@ +extern crate alloc; + +#[test] +fn test_burnpack_no_std() { + use burn_ndarray::NdArray; + use burn_no_std_tests::burnpack; + type Backend = NdArray; + let device = Default::default(); + + // Run all Burnpack tests + burnpack::run_all_tests::(&device); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/safetensors_tests.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/safetensors_tests.rs new file mode 100644 index 0000000..f33ab82 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/safetensors_tests.rs @@ -0,0 +1,12 @@ +extern crate alloc; + +#[test] +fn test_safetensors_no_std() { + use burn_ndarray::NdArray; + use burn_no_std_tests::safetensors; + type Backend = NdArray; + let device = Default::default(); + + // Run all SafeTensors tests + safetensors::run_all_tests::(&device); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/test_integration.rs b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/test_integration.rs new file mode 100644 index 0000000..4f1dd41 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-no-std-tests/tests/test_integration.rs @@ -0,0 +1,31 @@ +#![no_std] // Must keep it for testing + +use burn_no_std_tests::mlp::*; +use burn_no_std_tests::model::*; + +use burn::tensor::{Distribution, Tensor, backend::Backend}; +use burn_ndarray::NdArray; + +#[test] +fn test_mnist_model_with_random_input() { + type Backend = NdArray; + + // Model configurations + let device = Default::default(); + let mlp_config = MlpConfig::new(); + let mnist_config = MnistConfig::new(mlp_config); + let mnist_model: Model = Model::new(&mnist_config, &device); + + // Pass a fixed seed for random, otherwise a build generated random seed is used + Backend::seed(&device, mnist_config.seed); + + // Some random input + let input_shape = [1, 28, 28]; + let input = Tensor::::random(input_shape, Distribution::Default, &device); + + // Run through the model + let output = mnist_model.forward(input); + + assert_eq!(&*output.shape(), [1, 10]); + assert!(output.to_data().iter::().all(|x| x <= 1.0)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-optim/Cargo.toml new file mode 100644 index 0000000..2ad46d0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/Cargo.toml @@ -0,0 +1,102 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Optimizer building blocks for the Burn deep learning framework" +documentation = "https://docs.rs/burn-optim" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-optim" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-optim" +version.workspace = true + +[lints] +workspace = true + +[features] +default = [ + "std", + "burn-core/default", +] +doc = [ + "std", + # Doc features + "burn-core/doc", +] +std = [ + "burn-core/std", + "num-traits/std", + "serde/std", + "log", +] +tracing = [ + "burn-collective?/tracing", + "burn-core/tracing", + "burn-cuda?/tracing", + "burn-fusion?/tracing", + "burn-remote?/tracing", + "burn-rocm?/tracing", + "burn-router?/tracing", + "burn-tch?/tracing", + "burn-wgpu?/tracing", +] + +collective = ["burn-collective"] + +test-cuda = [ + "burn-cuda/default", +] # To use cuda during testing, default uses ndarray. +test-rocm = [ + "burn-rocm/default", +] # To use hip during testing, default uses ndarray. +test-tch = [ + "burn-tch/default", +] # To use tch during testing, default uses ndarray. +test-wgpu = [ + "burn-wgpu/default", +] # To use wgpu during testing, default uses ndarray. +test-vulkan = [ + "test-wgpu", + "burn-wgpu/vulkan", +] # To use wgpu-spirv during testing, default uses ndarray. +test-metal = [ + "test-wgpu", + "burn-wgpu/metal", +] # To use wgpu-spirv during testing, default uses ndarray. + +# Memory checks are disabled by default +test-memory-checks = ["burn-fusion/memory-checks"] + +[dependencies] + +# ** Please make sure all dependencies support no_std when std is disabled ** +burn-core = { path = "../burn-core", version = "=0.21.0-pre.2", default-features = false } +burn-collective = { path = "../burn-collective", version = "=0.21.0-pre.2", optional = true, default-features = false } + +num-traits = { workspace = true } +derive-new = { workspace = true } +log = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"] } + +# The same implementation of HashMap in std but with no_std support (only alloc crate is needed) +hashbrown = { workspace = true, features = ["serde"] } # no_std compatible + +# FOR TESTING +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-rocm = { path = "../burn-rocm", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-remote = { path = "../burn-remote", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-router = { path = "../burn-router", version = "=0.21.0-pre.2", default-features = false, optional = true } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } + +[dev-dependencies] +burn-nn = { path = "../burn-nn", version = "=0.21.0-pre.2" } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2" } +rstest = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/README.md b/crates/stable-diffusion-burn/burn-crates/burn-optim/README.md new file mode 100644 index 0000000..84e45eb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/README.md @@ -0,0 +1,3 @@ +# Burn Optimizers + +Core building blocks for Burn optimizers. \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/base.rs new file mode 100644 index 0000000..ae8a185 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/base.rs @@ -0,0 +1,144 @@ +use burn_core as burn; + +use burn::tensor::backend::Backend; +use burn::{config::Config, tensor::Tensor}; + +/// Gradient Clipping provides a way to mitigate exploding gradients +#[derive(Config, Debug)] +pub enum GradientClippingConfig { + /// Clip the gradient by value. + Value(f32), + + /// Clip the gradient by norm. + Norm(f32), +} + +impl GradientClippingConfig { + /// Initialize the gradient clipping. + /// + /// # Returns + /// + /// The gradient clipping. + pub fn init(&self) -> GradientClipping { + match self { + GradientClippingConfig::Value(val) => GradientClipping::Value(*val), + GradientClippingConfig::Norm(val) => GradientClipping::Norm(*val), + } + } +} + +/// Gradient Clipping provides a way to mitigate exploding gradients +/// by clipping every component of the gradient by value or by norm during +/// backpropagation. +#[derive(Clone)] +pub enum GradientClipping { + /// Clip the gradient by value. + Value(f32), + + /// Clip the gradient by norm. + Norm(f32), +} + +impl GradientClipping { + /// Clip the gradient. + /// + /// # Arguments + /// + /// * `grad` - The gradient to clip. + /// + /// # Returns + /// + /// The clipped gradient. + pub fn clip_gradient(&self, grad: Tensor) -> Tensor { + match self { + GradientClipping::Value(threshold) => self.clip_by_value(grad, *threshold), + GradientClipping::Norm(max_norm) => self.clip_by_norm(grad, *max_norm), + } + } + + fn clip_by_value( + &self, + grad: Tensor, + threshold: f32, + ) -> Tensor { + let greater_mask = grad.clone().greater_elem(threshold); + let lower_mask = grad.clone().lower_elem(-threshold); + + let clipped_grad = grad.mask_fill(greater_mask, threshold); + + clipped_grad.mask_fill(lower_mask, -threshold) + } + + fn clip_by_norm( + &self, + grad: Tensor, + threshold: f32, + ) -> Tensor { + let norm = Self::l2_norm(grad.clone()); + let clip_coef = threshold / norm.add_scalar(1e-6); // avoid div by zero + let clip_coef_clamped = clip_coef.clamp_max(1.0); + grad.mul(clip_coef_clamped.unsqueeze()) + } + + fn l2_norm(tensor: Tensor) -> Tensor { + let squared = tensor.square(); + let sum = squared.sum(); + sum.sqrt() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + use burn::tensor::Tensor; + + #[test] + fn test_clip_by_value() { + let gradient: Tensor = Tensor::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &Default::default(), + ); + + let clipped_gradient = GradientClipping::Value(0.5).clip_gradient(gradient); + let clipped_gradient_data = clipped_gradient.into_data(); + + for value in clipped_gradient_data.iter::() { + assert!(value <= 0.5); + } + } + + #[test] + fn test_clip_by_norm() { + let gradient: Tensor = Tensor::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &Default::default(), + ); + + let clipped_gradient = GradientClipping::Norm(2.2).clip_gradient(gradient); + let clipped_gradient_data = clipped_gradient.into_data(); + + for value in clipped_gradient_data.iter::() { + assert!(value <= 0.88); + } + } + #[test] + fn test_clip_by_norm_no_clipping() { + let gradient: Tensor = Tensor::from_floats( + [[0.3, 0.4, 0.5, 0.2], [0.1, 0.6, 0.3, 0.4]], + &Default::default(), + ); + + let clipped_gradient = GradientClipping::Norm(2.2).clip_gradient(gradient.clone()); + + clipped_gradient + .into_data() + .assert_eq(&gradient.into_data(), true); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/mod.rs new file mode 100644 index 0000000..096c94e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/grad_clipping/mod.rs @@ -0,0 +1,2 @@ +mod base; +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lib.rs new file mode 100644 index 0000000..399e50f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lib.rs @@ -0,0 +1,63 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![recursion_limit = "256"] + +//! Burn optimizers. + +#[macro_use] +extern crate derive_new; + +extern crate alloc; + +/// Optimizer module. +pub mod optim; +pub use optim::*; + +/// Gradient clipping module. +pub mod grad_clipping; + +/// Learning rate scheduler module. +#[cfg(feature = "std")] +pub mod lr_scheduler; + +/// Type alias for the learning rate. +/// +/// LearningRate also implements [learning rate scheduler](crate::lr_scheduler::LrScheduler) so it +/// can be used for constant learning rate. +pub type LearningRate = f64; // We could potentially change the type. + +/// Backend for test cases +#[cfg(all( + test, + not(feature = "test-tch"), + not(feature = "test-wgpu"), + not(feature = "test-cuda"), + not(feature = "test-rocm") +))] +pub type TestBackend = burn_ndarray::NdArray; + +#[cfg(all(test, feature = "test-tch"))] +/// Backend for test cases +pub type TestBackend = burn_tch::LibTorch; + +#[cfg(all(test, feature = "test-wgpu"))] +/// Backend for test cases +pub type TestBackend = burn_wgpu::Wgpu; + +#[cfg(all(test, feature = "test-cuda"))] +/// Backend for test cases +pub type TestBackend = burn_cuda::Cuda; + +#[cfg(all(test, feature = "test-rocm"))] +/// Backend for test cases +pub type TestBackend = burn_rocm::Rocm; + +/// Backend for autodiff test cases +#[cfg(test)] +pub type TestAutodiffBackend = burn_autodiff::Autodiff; + +#[cfg(all(test, feature = "test-memory-checks"))] +mod tests { + burn_fusion::memory_checks!(); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/base.rs new file mode 100644 index 0000000..55d69e7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/base.rs @@ -0,0 +1,85 @@ +pub(super) use alloc::string::String; +use burn_core as burn; + +use burn::record::Record; +use burn::tensor::backend::Backend; + +use crate::LearningRate; + +/// Learning rate scheduler defines how the learning rate will evolve during training. +pub trait LrScheduler: Clone + Send + Sync { + /// Scheduler associative type to be used when saving and loading the state. + type Record: Record; + + /// Perform the scheduler step, potentially updating its state, and returning the effective + /// learning rate. + fn step(&mut self) -> LearningRate; + + /// Get the current state of the scheduler as a [record](Record). + fn to_record(&self) -> Self::Record; + + /// Load the state of the scheduler as a [record](Record). + fn load_record(self, record: Self::Record) -> Self; +} + +#[cfg(test)] +pub(super) mod test_utils { + use super::*; + use crate::TestBackend; + + // A small tolerance for learning rate comparisons. Depending on how learning rates are + // computed, floating-point arithmetic error might exceed f64::EPSILON, so a larger value is + // used here. + const LOOSE_EPSILON: LearningRate = 1e-10; + + pub fn check_lr_sequence(mut scheduler: S, expected_lrs: I) + where + I: IntoIterator, + S: LrScheduler, + { + expected_lrs + .into_iter() + .enumerate() + .for_each(|(i, expected)| { + let lr = scheduler.step(); + assert!( + (lr - expected).abs() < LOOSE_EPSILON, + "Scheduled learning rate {lr} is not approximately equal to the expected value \ + {expected} at step {i}", + ); + }); + } + + // save_at_step is the number of steps to run the scheduler before saving and loading back its + // state. + pub fn check_save_load(mut scheduler: S, save_at_step: usize) + where + S: Clone + LrScheduler, + { + let mut truth = scheduler.clone(); + // Consume some steps before saving and loading back + (0..save_at_step).for_each(|_| { + truth.step(); + scheduler.step(); + }); + let rec = scheduler.to_record::(); + scheduler = scheduler.load_record::(rec); + + // Validate that the scheduler resumes from where it left off. + compare_steps(&mut scheduler, &mut truth, save_at_step); + } + + // Check if two schedulers produce the same learning rate sequences over the specified number of + // steps. + pub fn compare_steps(a: &mut S, b: &mut S, num_steps: usize) { + (0..num_steps).for_each(|i| { + let lr_a = a.step(); + let lr_b = b.step(); + assert!( + (lr_a - lr_b).abs() < LOOSE_EPSILON, + "The two learning rates ({lr_a}, {lr_b}) at position {i} in the remaining \ + sequences are not approximately equal", + ); + }); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/composed.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/composed.rs new file mode 100644 index 0000000..b6b6866 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/composed.rs @@ -0,0 +1,195 @@ +use burn_core as burn; + +use super::cosine::{CosineAnnealingLrScheduler, CosineAnnealingLrSchedulerConfig}; +use super::exponential::{ExponentialLrScheduler, ExponentialLrSchedulerConfig}; +use super::linear::{LinearLrScheduler, LinearLrSchedulerConfig}; +use super::noam::{NoamLrScheduler, NoamLrSchedulerConfig}; +use super::{LrScheduler, String}; +use crate::LearningRate; + +use burn::config::Config; +use burn::record::Record; +use burn::tensor::backend::Backend; + +/// Compose multiple [learning rate schedulers](LrScheduler) together. +#[derive(Config, Debug)] +pub struct ComposedLrSchedulerConfig { + #[config(default = "Vec::new()")] + schedulers: Vec, + #[config(default = "SchedulerReduction::Prod")] + reduction: SchedulerReduction, +} + +/// Compose multiple [learning rate schedulers](LrScheduler) together. +#[derive(Clone)] +pub struct ComposedLrScheduler { + schedulers: Vec, + reduction: SchedulerReduction, +} + +/// Defines how the learning rates generated by the schedulers are combined. +#[derive(Config, Debug, Copy)] +pub enum SchedulerReduction { + /// All learning rates are averaged. + Avg, + /// All learning rates are summed. + Sum, + /// All learning rates are multiplied. + Prod, +} + +impl ComposedLrSchedulerConfig { + /// Initialize the learning rate scheduler. + pub fn init(&self) -> Result { + let mut schedulers = Vec::with_capacity(self.schedulers.len()); + for config in self.schedulers.iter() { + let config = match config { + LrSchedulerConfig::Linear(config) => LrSchedulerItem::Linear(config.init()?), + LrSchedulerConfig::Cosine(config) => LrSchedulerItem::Cosine(config.init()?), + LrSchedulerConfig::Exponential(config) => { + LrSchedulerItem::Exponential(config.init()?) + } + LrSchedulerConfig::Noam(config) => LrSchedulerItem::Noam(config.init()?), + }; + schedulers.push(config); + } + + Ok(ComposedLrScheduler { + schedulers, + reduction: self.reduction, + }) + } + + /// Appends a [linear scheduler](LinearLrScheduler). + pub fn linear(mut self, config: LinearLrSchedulerConfig) -> Self { + self.schedulers.push(LrSchedulerConfig::Linear(config)); + self + } + + /// Appends a [cosine scheduler](ComposedLrSchedulerConfig). + pub fn cosine(mut self, config: CosineAnnealingLrSchedulerConfig) -> Self { + self.schedulers.push(LrSchedulerConfig::Cosine(config)); + self + } + + /// Appends an [exponential scheduler](ExponentialLrScheduler). + pub fn exponential(mut self, config: ExponentialLrSchedulerConfig) -> Self { + self.schedulers.push(LrSchedulerConfig::Exponential(config)); + self + } + + /// Appends a [noam scheduler](NoamLrScheduler). + pub fn noam(mut self, config: NoamLrSchedulerConfig) -> Self { + self.schedulers.push(LrSchedulerConfig::Noam(config)); + self + } +} + +#[derive(Config, Debug)] +enum LrSchedulerConfig { + Linear(LinearLrSchedulerConfig), + Cosine(CosineAnnealingLrSchedulerConfig), + Exponential(ExponentialLrSchedulerConfig), + Noam(NoamLrSchedulerConfig), +} + +#[derive(Clone)] +enum LrSchedulerItem { + Linear(LinearLrScheduler), + Cosine(CosineAnnealingLrScheduler), + Exponential(ExponentialLrScheduler), + Noam(NoamLrScheduler), +} + +#[derive(Record)] +/// Record item for the [composed learning rate scheduler](ComposedLrScheduler). +pub enum LrSchedulerRecord { + /// The linear variant. + Linear(::Record), + /// The cosine variant. + Cosine(::Record), + /// The exponential variant. + Exponential(::Record), + /// The noam variant. + Noam(::Record), +} + +#[derive(Record)] +/// Records for the [composed learning rate scheduler](ComposedLrScheduler). +pub struct ComposedLrSchedulerRecord { + schedulers: Vec>, +} + +impl LrScheduler for ComposedLrScheduler { + type Record = ComposedLrSchedulerRecord; + + fn step(&mut self) -> LearningRate { + let mut step = match self.reduction { + SchedulerReduction::Avg => 0.0, + SchedulerReduction::Sum => 0.0, + SchedulerReduction::Prod => 1.0, + }; + let num_scheduler = self.schedulers.len() as f64; + + for lr in self.schedulers.iter_mut().map(|s| match s { + LrSchedulerItem::Linear(item) => item.step(), + LrSchedulerItem::Cosine(item) => item.step(), + LrSchedulerItem::Exponential(item) => item.step(), + LrSchedulerItem::Noam(item) => item.step(), + }) { + step = match self.reduction { + SchedulerReduction::Avg => step + (lr / num_scheduler), + SchedulerReduction::Sum => step + lr, + SchedulerReduction::Prod => step * lr, + } + } + + step + } + + fn to_record(&self) -> Self::Record { + ComposedLrSchedulerRecord:: { + schedulers: self + .schedulers + .iter() + .map(|s| match s { + LrSchedulerItem::Linear(item) => { + LrSchedulerRecord::Linear(item.to_record::()) + } + LrSchedulerItem::Cosine(item) => { + LrSchedulerRecord::Linear(item.to_record::()) + } + LrSchedulerItem::Exponential(item) => { + LrSchedulerRecord::Exponential(item.to_record::()) + } + LrSchedulerItem::Noam(item) => LrSchedulerRecord::Noam(item.to_record::()), + }) + .collect(), + } + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.schedulers = self + .schedulers + .into_iter() + .zip(record.schedulers) + .map(|scheduler| match scheduler { + (LrSchedulerItem::Linear(item), LrSchedulerRecord::Linear(record)) => { + LrSchedulerItem::Linear(item.load_record::(record)) + } + (LrSchedulerItem::Cosine(item), LrSchedulerRecord::Cosine(record)) => { + LrSchedulerItem::Cosine(item.load_record::(record)) + } + (LrSchedulerItem::Exponential(item), LrSchedulerRecord::Exponential(record)) => { + LrSchedulerItem::Exponential(item.load_record::(record)) + } + (LrSchedulerItem::Noam(item), LrSchedulerRecord::Noam(record)) => { + LrSchedulerItem::Noam(item.load_record::(record)) + } + _ => panic!("Invalid state"), + }) + .collect(); + + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/constant.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/constant.rs new file mode 100644 index 0000000..0880e85 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/constant.rs @@ -0,0 +1,50 @@ +use burn_core as burn; + +use burn::tensor::backend::Backend; + +use super::LrScheduler; +use crate::LearningRate; + +/// Constant learning rate implementing [learning rate scheduler](LrScheduler). +/// +/// # Notes +/// +/// You can also use [learning rate](LearningRate) which the same effect. +#[derive(new, Clone, Debug)] +pub struct ConstantLr { + lr: LearningRate, +} + +impl From for ConstantLr { + fn from(lr: LearningRate) -> Self { + Self { lr } + } +} + +impl LrScheduler for ConstantLr { + type Record = (); + + fn step(&mut self) -> LearningRate { + self.lr + } + + fn to_record(&self) -> Self::Record {} + + fn load_record(self, _record: Self::Record) -> Self { + self + } +} + +impl LrScheduler for LearningRate { + type Record = (); + + fn step(&mut self) -> LearningRate { + *self + } + + fn to_record(&self) -> Self::Record {} + + fn load_record(self, _record: Self::Record) -> Self { + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/cosine.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/cosine.rs new file mode 100644 index 0000000..638ece1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/cosine.rs @@ -0,0 +1,191 @@ +use burn_core as burn; + +use super::{LrScheduler, String}; +use crate::LearningRate; +use burn::config::Config; +use burn::tensor::backend::Backend; + +/// The configuration for creating a [Cosine Annealing learning rate scheduler with warm +/// restarts](CosineAnnealingLrScheduler). +/// +/// This scheduler returns the learning rate `initial_lr` at the first step, then changes it by +/// following a cosine function. After `num_iters` iterations, the learning rate is reset to +/// `initial_lr`. +#[derive(Config, Debug)] +pub struct CosineAnnealingLrSchedulerConfig { + // The initial learning rate. + initial_lr: LearningRate, + // The final learning rate. + #[config(default = 0.0)] + min_lr: LearningRate, + // The number of iterations between two restarts. The two restart iterations themselves are not + // included. + num_iters: usize, +} + +impl CosineAnnealingLrSchedulerConfig { + /// Initializes a [Cosine learning rate scheduler](CosineAnnealingLrScheduler). + /// + /// # Errors + /// + /// An error will be returned if any of the following conditions is true: + /// + /// * `initial_lr` is out of range (0.0, 1.0] + /// * `min_lr` is out of range [0.0, `initial_lr`] + /// * `num_iters` is 0 + pub fn init(&self) -> Result { + if self.initial_lr <= 0. || self.initial_lr > 1. { + return Err("Initial learning rate must be greater than 0 and at most 1".into()); + } + if self.min_lr < 0.0 || self.min_lr > self.initial_lr { + return Err( + "Minimum learning rate must be at least 0 and at most equal to the initial \ + learning rate" + .into(), + ); + } + if self.num_iters == 0 { + return Err("Number of iterations must be at least 1".into()); + } + + Ok(CosineAnnealingLrScheduler { + min_lr: self.min_lr, + max_lr: self.initial_lr, + num_iters: self.num_iters, + current_iter: usize::MAX, + }) + } +} + +/// A Cosine Annealing learning rate scheduler. +/// +/// This scheduler is described in [SGDR: Stochastic Gradient Descent with Warm +/// Restarts](https://arxiv.org/abs/1608.03983). See [CosineAnnealingLrSchedulerConfig] for more +/// information. +#[derive(Clone, Copy, Debug)] +pub struct CosineAnnealingLrScheduler { + min_lr: LearningRate, + max_lr: LearningRate, + num_iters: usize, + current_iter: usize, +} + +impl LrScheduler for CosineAnnealingLrScheduler { + type Record = usize; + + fn step(&mut self) -> LearningRate { + // Make current_iter overflow from usize::MAX to 0 to get the initial learning rate on the + // first call. We could've used i64 with an initial value -1, but keeping it in usize saves + // us from some type casting here. + self.current_iter = self.current_iter.wrapping_add(1) % (self.num_iters + 1); + self.min_lr + + 0.5 + * (self.max_lr - self.min_lr) + * (1.0 + + (self.current_iter as f64 / self.num_iters as f64 * std::f64::consts::PI) + .cos()) + } + + fn to_record(&self) -> Self::Record { + self.current_iter + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.current_iter = record; + self + } +} + +#[cfg(test)] +mod tests { + use super::super::test_utils; + use super::*; + + #[test] + fn config_initial_lr_too_low() { + let r = CosineAnnealingLrSchedulerConfig::new(0., 10).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Initial learning rate must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_initial_lr_too_high() { + let r = CosineAnnealingLrSchedulerConfig::new(1.5, 10).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Initial learning rate must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_min_lr_too_low() { + let r = CosineAnnealingLrSchedulerConfig::new(0.5, 10) + .with_min_lr(-0.1) + .init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Minimum learning rate must be at least 0 and at most equal to the initial learning \ + rate", + "Error messages should match", + ); + } + + #[test] + fn config_min_lr_too_high() { + let r = CosineAnnealingLrSchedulerConfig::new(0.5, 10) + .with_min_lr(0.6) + .init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Minimum learning rate must be at least 0 and at most equal to the initial learning \ + rate", + "Error messages should match", + ); + } + + #[test] + fn config_num_iters_too_low() { + let r = CosineAnnealingLrSchedulerConfig::new(0.5, 0).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Number of iterations must be at least 1", + "Error messages should match", + ); + } + + #[test] + fn test_lr_change() { + const INITIAL_LR: LearningRate = 0.5; + const MIN_LR: LearningRate = 0.1; + + let scheduler = CosineAnnealingLrSchedulerConfig::new(INITIAL_LR, 2) + .with_min_lr(MIN_LR) + .init() + .unwrap(); + let expected_lrs = [ + INITIAL_LR, // cos(0) + (INITIAL_LR + MIN_LR) * 0.5, // cos(PI/2) + MIN_LR, // cos(PI) + INITIAL_LR, // restart + ]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_save_and_load() { + const NUM_ITERS: usize = 9; + let scheduler = CosineAnnealingLrSchedulerConfig::new(1.0, NUM_ITERS) + .init() + .unwrap(); + test_utils::check_save_load(scheduler, NUM_ITERS / 3 * 2); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/exponential.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/exponential.rs new file mode 100644 index 0000000..21fc412 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/exponential.rs @@ -0,0 +1,139 @@ +use burn_core as burn; + +use super::{LrScheduler, String}; +use crate::LearningRate; +use burn::config::Config; +use burn::tensor::backend::Backend; + +/// The configuration for creating an [exponential learning rate scheduler](ExponentialLrScheduler). +/// +/// This scheduler returns the learning rate `initial_lr` at the first step, then multiplies it by +/// a constant `gamma` at every iteration. At any iteration `i` (which starts from 0), the learning +/// rate is given by `initial_lr * gamma^i`. +#[derive(Config, Debug)] +pub struct ExponentialLrSchedulerConfig { + // The initial learning rate. + initial_lr: LearningRate, + // The constant that the learning rate is multiplied by on each iteration. + gamma: f64, +} + +impl ExponentialLrSchedulerConfig { + /// Initializes a [exponential learning rate scheduler](ExponentialLrScheduler). + /// + /// # Errors + /// + /// An error will be returned if any of the following conditions is true: + /// + /// * `initial_lr` is out of range (0.0, 1.0] + /// * `gamma` is out of range (0.0, 1.0] + pub fn init(&self) -> Result { + if self.initial_lr <= 0. || self.initial_lr > 1. { + return Err("Initial learning rate must be greater than 0 and at most 1".into()); + } + if self.gamma <= 0. || self.gamma > 1. { + return Err("Gamma must be greater than 0 and at most 1".into()); + } + + Ok(ExponentialLrScheduler { + // Such an initial value eliminates the need for special-case handling of the first + // learning rate. + previous_lr: self.initial_lr / self.gamma, + gamma: self.gamma, + }) + } +} + +/// A exponential learning rate scheduler. +/// +/// See [ExponentialLrSchedulerConfig] for more information. +#[derive(Clone, Copy, Debug)] +pub struct ExponentialLrScheduler { + // The previous iteration's learning rate. + previous_lr: LearningRate, + // The constant that the learning rate is multiplied by on each iteration. + gamma: f64, +} + +impl LrScheduler for ExponentialLrScheduler { + type Record = LearningRate; + + fn step(&mut self) -> LearningRate { + self.previous_lr *= self.gamma; + self.previous_lr + } + + fn to_record(&self) -> Self::Record { + self.previous_lr + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.previous_lr = record; + self + } +} + +#[cfg(test)] +mod tests { + use super::super::test_utils; + use super::*; + + #[test] + fn config_initial_lr_too_low() { + let r = ExponentialLrSchedulerConfig::new(0., 0.5).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Initial learning rate must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_initial_lr_too_high() { + let r = ExponentialLrSchedulerConfig::new(1.5, 0.5).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Initial learning rate must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_gamma_too_low() { + let r = ExponentialLrSchedulerConfig::new(0.5, 0.0).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Gamma must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_gamma_too_high() { + let r = ExponentialLrSchedulerConfig::new(0.5, 1.5).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Gamma must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn test_lr_change() { + let scheduler = ExponentialLrSchedulerConfig::new(0.8, 0.1).init().unwrap(); + let expected_lrs = [0.8, 0.08, 0.008, 0.0008, 0.00008]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_save_and_load() { + let scheduler = ExponentialLrSchedulerConfig::new(0.083, 0.3) + .init() + .unwrap(); + test_utils::check_save_load(scheduler, 7); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/linear.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/linear.rs new file mode 100644 index 0000000..bc3e9e2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/linear.rs @@ -0,0 +1,173 @@ +use burn_core as burn; + +use super::{LrScheduler, String}; +use crate::LearningRate; +use burn::config::Config; +use burn::tensor::backend::Backend; + +/// The configuration for creating a [linear learning rate scheduler](LinearLrScheduler). +/// +/// This scheduler returns the learning rate `initial_lr` at the first step, then changes it by a +/// constant amount on each iteration until reaching a final learning rate `final_lr`. The +/// `num_iters` parameter controls how many iterations are needed to go from `initial_lr` to +/// `final_lr`. +#[derive(Config, Debug)] +pub struct LinearLrSchedulerConfig { + // The initial learning rate. + initial_lr: LearningRate, + // The final learning rate. + final_lr: LearningRate, + // The number of iterations before reaching the final learning rate. + num_iters: usize, +} + +impl LinearLrSchedulerConfig { + /// Initializes a [linear learning rate scheduler](LinearLrScheduler). + /// + /// # Errors + /// + /// An error will be returned if any of the following conditions is true: + /// + /// * `initial_lr` is out of range (0.0, 1.0] + /// * `final_lr` is out of range [0.0, 1.0] + /// * `num_iters` is 0 + pub fn init(&self) -> Result { + if self.initial_lr <= 0. || self.initial_lr > 1. { + return Err("Initial learning rate must be greater than 0 and at most 1".into()); + } + if self.final_lr < 0. || self.final_lr > 1. { + return Err("Final learning rate must be at least 0 and at most 1".into()); + } + if self.num_iters == 0 { + return Err("Number of iterations must be at least 1".into()); + } + + Ok(LinearLrScheduler { + final_lr: self.final_lr, + step_size: (self.final_lr - self.initial_lr) / self.num_iters as f64, + remaining_iters: self.num_iters + 1, + }) + } +} + +/// A linear learning rate scheduler. +/// +/// See [LinearLrSchedulerConfig] for more information. +#[derive(Clone, Copy, Debug)] +pub struct LinearLrScheduler { + // The final learning rate after the linear changing process stops. + final_lr: LearningRate, + // The amount that the learning rate changes by on each iteration. + step_size: f64, + // The number of iterations left before reaching the final learning rate. + remaining_iters: usize, +} + +impl LrScheduler for LinearLrScheduler { + type Record = usize; + + fn step(&mut self) -> LearningRate { + self.remaining_iters -= (self.remaining_iters != 0) as usize; + self.final_lr - self.step_size * self.remaining_iters as f64 + } + + fn to_record(&self) -> Self::Record { + self.remaining_iters + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.remaining_iters = record; + self + } +} + +#[cfg(test)] +mod tests { + use super::super::test_utils; + use super::*; + + #[test] + fn config_initial_lr_too_low() { + let r = LinearLrSchedulerConfig::new(0., 0.5, 100).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Initial learning rate must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_initial_lr_too_high() { + let r = LinearLrSchedulerConfig::new(1.5, 0.5, 100).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Initial learning rate must be greater than 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_final_lr_too_low() { + let r = LinearLrSchedulerConfig::new(0.5, -0.5, 100).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Final learning rate must be at least 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_final_lr_too_high() { + let r = LinearLrSchedulerConfig::new(0.5, 1.5, 100).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Final learning rate must be at least 0 and at most 1", + "Error messages should match", + ); + } + + #[test] + fn config_num_iters_too_low() { + let r = LinearLrSchedulerConfig::new(0.9, 0.1, 0).init(); + assert!(r.is_err(), "Should return an error"); + assert_eq!( + r.unwrap_err(), + "Number of iterations must be at least 1", + "Error messages should match", + ); + } + + #[test] + fn test_lr_decreasing() { + let scheduler = LinearLrSchedulerConfig::new(0.9, 0.5, 4).init().unwrap(); + let expected_lrs = [0.9, 0.8, 0.7, 0.6, 0.5, 0.5]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_lr_increasing() { + let scheduler = LinearLrSchedulerConfig::new(0.01, 0.04, 3).init().unwrap(); + let expected_lrs = [0.01, 0.02, 0.03, 0.04, 0.04]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_lr_unchanging() { + let scheduler = LinearLrSchedulerConfig::new(0.3, 0.3, 2).init().unwrap(); + let expected_lrs = [0.3, 0.3, 0.3, 0.3]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_save_and_load() { + const NUM_ITERS: usize = 6; + let scheduler = LinearLrSchedulerConfig::new(1.0, 0.01, NUM_ITERS) + .init() + .unwrap(); + test_utils::check_save_load(scheduler, NUM_ITERS / 3 * 2); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/mod.rs new file mode 100644 index 0000000..c16724e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/mod.rs @@ -0,0 +1,24 @@ +/// Constant learning rate scheduler +pub mod constant; + +/// Composed learning rate scheduler +pub mod composed; + +/// Linear learning rate scheduler +pub mod linear; + +/// Noam learning rate scheduler +pub mod noam; + +/// Exponential learning rate scheduler +pub mod exponential; + +/// Cosine learning rate scheduler +pub mod cosine; + +/// Step learning rate scheduler +pub mod step; + +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/noam.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/noam.rs new file mode 100644 index 0000000..9d13b74 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/noam.rs @@ -0,0 +1,136 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::tensor::backend::Backend; + +use super::{LrScheduler, String}; +use crate::LearningRate; + +/// Configuration to create a [noam](NoamLrScheduler) learning rate scheduler. +#[derive(Config, Debug)] +pub struct NoamLrSchedulerConfig { + /// The overall scale factor for the learning rate decay. + factor: f64, + /// The number of steps before the exponential decay stats. + #[config(default = 4000)] + warmup_steps: usize, + /// The size of the model. + #[config(default = 512)] + model_size: usize, +} + +/// Noam learning rate scheduler as described in [Attention Is All You Need](https://arxiv.org/abs/1706.03762). +#[derive(Clone, Debug)] +pub struct NoamLrScheduler { + warmup_steps: f64, + embedding_size: f64, + factor: f64, + step: f64, +} + +impl NoamLrSchedulerConfig { + /// Initialize a new [noam](NoamLrScheduler) learning rate scheduler. + /// + /// # Errors + /// + /// An error will be returned if any of the following conditions is true: + /// + /// * `warmup_steps` is 0 + /// * `model_size` is 0 + pub fn init(&self) -> Result { + if self.warmup_steps == 0 { + return Err( + "Number of steps before exponential decay starts must be greater than 0".into(), + ); + } + if self.model_size == 0 { + return Err("Model size must be greater than 0".into()); + } + + Ok(NoamLrScheduler { + warmup_steps: self.warmup_steps as f64, + embedding_size: self.model_size as f64, + factor: self.factor, + step: 0.0, + }) + } +} + +impl LrScheduler for NoamLrScheduler { + type Record = usize; + + fn step(&mut self) -> LearningRate { + self.step += 1.0; + + let arg1 = self.step.powf(-0.5); + let arg2 = self.step * self.warmup_steps.powf(-1.5); + + self.factor * self.embedding_size.powf(-0.5) * f64::min(arg1, arg2) + } + + fn to_record(&self) -> Self::Record { + self.step as usize + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.step = record as f64; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_warmup_steps_invalid() { + let r = NoamLrSchedulerConfig::new(0.1).with_warmup_steps(0).init(); + assert!(r.is_err(), "Should return an error"); + } + + #[test] + fn test_config_warmup_steps_valid() { + let r = NoamLrSchedulerConfig::new(0.1).with_warmup_steps(1).init(); + assert!(r.is_ok(), "Should return a success value"); + } + + #[test] + fn test_config_model_size_invalid() { + let r = NoamLrSchedulerConfig::new(0.1).with_model_size(0).init(); + assert!(r.is_err(), "Should return an error"); + } + + #[test] + fn test_config_model_size_valid() { + let r = NoamLrSchedulerConfig::new(0.1).with_model_size(1).init(); + assert!(r.is_ok(), "Should return a success value"); + } + + #[test] + fn test_function_increase_and_decrease() { + let warmup_steps = 100; + let mut scheduler = NoamLrSchedulerConfig::new(10.0) + .with_warmup_steps(warmup_steps) + .init() + .unwrap(); + let mut lr_current = 0.0; + + for _ in 0..warmup_steps { + let lr = scheduler.step(); + assert!( + lr > lr_current, + "Learning rate should increase before the warmup_steps is reached." + ); + lr_current = lr; + } + + for _ in 0..warmup_steps { + let lr = scheduler.step(); + assert!( + lr < lr_current, + "Learning rate should decrease after the warmup_steps is reached." + ); + lr_current = lr; + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/step.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/step.rs new file mode 100644 index 0000000..be3a3a6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/lr_scheduler/step.rs @@ -0,0 +1,218 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::tensor::backend::Backend; + +use super::{LrScheduler, String}; +use crate::LearningRate; + +/// The configuration for create a [step learning rate scheduler](StepLrScheduler). +/// +/// This scheduler returns the learning rate `initial_lr` from the start, and keeps doing so until +/// the same value has been given for `step_size` times. Then it multiplies the learning rate by +/// `gamma` before repeating the process. +/// +/// Gamma values out of range (0.0, 1.0) and non-positive initial learning rates are acceptable, but +/// a warning log will be output for such a value in case of mistyping. +/// +/// ## Notes +/// +/// The [step](StepLrScheduler::step) method of the scheduler panics if it is called more than +/// `i32::MAX + 1` times. +#[derive(Config, Debug)] +pub struct StepLrSchedulerConfig { + // The learning rate at the initial step. + initial_lr: LearningRate, + // The number of iterations over which the learning rate remains unchanged before the next + // update. + step_size: usize, + /// The factor by which the learning rate is multiplied with each update. Default: 0.1. + #[config(default = 0.1)] + gamma: f64, +} + +impl StepLrSchedulerConfig { + /// Initializes a [step learning rate scheduler](StepLrScheduler). + /// + /// # Errors + /// + /// An error will be returned if `step_size` is 0. + pub fn init(&self) -> Result { + if self.step_size == 0 { + return Err("Step size must be greater than 0".into()); + } + + // Atypical values of `initial_lr` and `gamma` are not rejected because they might be useful + // in some cases like debugging (e.g., https://datascience.stackexchange.com/q/89518). + if self.initial_lr <= 0.0 { + log::warn!( + "Initial learning rate value of {} is not a positive number. Ignore this warning \ + if it is intended.", + self.initial_lr + ); + } + if self.gamma <= 0.0 || self.gamma >= 1.0 { + log::warn!( + "Gamma value of {} is out of range (0.0, 1.0). Ignore this warning if it is \ + intended.", + self.gamma + ); + } + + Ok(StepLrScheduler { + init_lr: self.initial_lr, + step_size: self.step_size, + gamma: self.gamma, + iter_idx: -1, + }) + } +} + +/// Step learning rate scheduler. +#[derive(Clone, Debug)] +pub struct StepLrScheduler { + init_lr: LearningRate, + step_size: usize, + gamma: f64, + // The index of the current iteration. + // `i32` is used for avoiding truncating the exponent when taking powers of `gamma`. + iter_idx: i32, +} + +impl LrScheduler for StepLrScheduler { + type Record = i32; + + fn step(&mut self) -> LearningRate { + self.iter_idx = self + .iter_idx + .checked_add(1) + .expect("`.step()` should be called no more than `i32::MAX + 1` times"); + // Type casting below causes no truncation, as all the values fall within the ranges. + self.init_lr + * self + .gamma + .powi((self.iter_idx as usize / self.step_size) as i32) + } + + fn to_record(&self) -> Self::Record { + self.iter_idx + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.iter_idx = record; + self + } +} + +#[cfg(test)] +mod tests { + use super::super::test_utils; + use super::*; + use crate::TestBackend; + + // Warning logs for initial LR and gamma are not tested because there seems no straightforward + // way to do it. + // + // Creating a mock logger that collects logs into `String` for later examination seems a possible + // solution, but unit tests run in the same process in parallel, where the single logger would + // be shared by multiple tests, so logs from different tests would be mixed up with no easy way + // to separate them. + // Using "--test-threads=1" could prevent mixup, but whether the ability to test logging is + // worth the slowdown would be a question. Also, using a primitive provided by `std` to + // synchronize the logger across tests is not an option since we need to support `no-std`. + // Maybe the mocking approach can be reconsidered after we are given an option to run tests in + // separate processes like what the issue below is proposing: + // https://github.com/rust-lang/rust/issues/47506 + // + // As a side note, a helper crate exists for the exact purpose: + // https://crates.io/crates/testing_logger + // but the crate has been unmaintained and using it would introduce another dependency. + + #[test] + fn test_config_step_size_zero() { + let r = StepLrSchedulerConfig::new(1.0, 0).init(); + assert!(r.is_err(), "Should return an error"); + } + + #[test] + fn test_config_step_size_nonzero() { + let r = StepLrSchedulerConfig::new(1.0, 1).init(); + assert!(r.is_ok(), "Should return a success value"); + } + + #[test] + fn test_config_default_gamma() { + const INIT_LR: LearningRate = 0.4; + const STEP_SIZE: usize = 2; + + let mut default = StepLrSchedulerConfig::new(INIT_LR, STEP_SIZE) + .init() + .unwrap(); + let mut explicit = StepLrSchedulerConfig::new(INIT_LR, STEP_SIZE) + .with_gamma(0.1) + .init() + .unwrap(); + test_utils::compare_steps(&mut default, &mut explicit, 3 * STEP_SIZE); + } + + #[test] + fn test_lr_decreasing() { + let scheduler = StepLrSchedulerConfig::new(0.5, 3) + .with_gamma(0.1) + .init() + .unwrap(); + let expected_lrs = [0.5, 0.5, 0.5, 0.05, 0.05, 0.05, 0.005, 0.005, 0.005]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_lr_increasing() { + let scheduler = StepLrSchedulerConfig::new(0.1, 2) + .with_gamma(2.0) + .init() + .unwrap(); + let expected_lrs = [0.1, 0.1, 0.2, 0.2, 0.4, 0.4]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_lr_unchanging() { + let scheduler = StepLrSchedulerConfig::new(3.1, 1) + .with_gamma(1.0) + .init() + .unwrap(); + let expected_lrs = [3.1, 3.1, 3.1]; + test_utils::check_lr_sequence(scheduler, expected_lrs); + } + + #[test] + fn test_save_and_load() { + const STEP_SIZE: usize = 10; + + let scheduler = StepLrSchedulerConfig::new(0.007, STEP_SIZE) + .with_gamma(0.03) + .init() + .unwrap(); + test_utils::check_save_load(scheduler, 3 * STEP_SIZE / 2); + } + + // It's too time consuming to actually run a scheduler `i32::MAX` steps, so an approach that + // depends on private fields is used to implement the test. + #[test] + fn test_number_of_calls_within_limit() { + // Create a scheduler that has already run `i32::MAX` steps + let mut scheduler = StepLrSchedulerConfig::new(0.1, 2).init().unwrap(); + scheduler = scheduler.load_record::(i32::MAX - 1); + scheduler.step(); + } + + #[test] + #[should_panic = "i32::MAX"] + fn test_number_of_calls_over_limit() { + // Create a scheduler that has already run `i32::MAX` steps + let mut scheduler = StepLrSchedulerConfig::new(0.1, 2).init().unwrap(); + scheduler = scheduler.load_record::(i32::MAX - 1); + scheduler.step(); + scheduler.step(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adagrad.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adagrad.rs new file mode 100644 index 0000000..2cc53e9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adagrad.rs @@ -0,0 +1,306 @@ +use burn_core as burn; + +use burn::{module::AutodiffModule, record::Record}; + +use burn::config::Config; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use burn::tensor::{backend::Backend, ops::Device}; + +use super::{ + SimpleOptimizer, + adaptor::OptimizerAdaptor, + decay::{WeightDecay, WeightDecayConfig}, +}; +use crate::{LearningRate, grad_clipping::GradientClippingConfig}; + +/// AdaGrad configuration. +#[derive(Config, Debug)] +pub struct AdaGradConfig { + #[config(default = 0.)] + lr_decay: f64, + #[config(default = 1e-5)] + epsilon: f32, + /// [Weight decay](WeightDecayConfig) config. + weight_decay: Option, + /// [Gradient Clipping](GradientClippingConfig) config. + grad_clipping: Option, +} + +/// AdaGrad optimizer +#[derive(Clone)] +pub struct AdaGrad { + lr_decay: LrDecay, + weight_decay: Option, +} + +/// AdaGrad state. +#[derive(Record, Clone, new)] +pub struct AdaGradState { + lr_decay: LrDecayState, +} + +impl SimpleOptimizer for AdaGrad { + type State = AdaGradState; + + fn step( + &self, + lr: LearningRate, + tensor: Tensor, + mut grad: Tensor, + state: Option>, + ) -> (Tensor, Option>) { + let mut state_lr_decay = None; + + if let Some(state) = state { + state_lr_decay = Some(state.lr_decay); + } + + if let Some(weight_decay) = &self.weight_decay { + grad = weight_decay.transform(grad, tensor.clone()); + } + + let (grad, state_lr_decay) = self.lr_decay.transform(grad, lr, state_lr_decay); + + let state = AdaGradState::new(state_lr_decay); + + (tensor - grad, Some(state)) + } + + fn to_device(mut state: Self::State, device: &Device) -> Self::State { + state.lr_decay = state.lr_decay.to_device(device); + state + } +} + +impl AdaGradConfig { + /// Initialize AdaGrad optimizer. + /// + /// # Returns + /// + /// Returns an optimizer that can be used to optimize a module. + pub fn init>( + &self, + ) -> OptimizerAdaptor { + let optim = AdaGrad { + lr_decay: LrDecay { + lr_decay: self.lr_decay, + epsilon: self.epsilon, + }, + weight_decay: self.weight_decay.as_ref().map(WeightDecay::new), + }; + + let mut optim = OptimizerAdaptor::from(optim); + if let Some(config) = &self.grad_clipping { + optim = optim.with_grad_clipping(config.init()); + } + optim + } +} + +/// Learning rate decay state (also includes sum state). +#[derive(Record, new, Clone)] +pub struct LrDecayState { + time: usize, + sum: Tensor, +} + +#[derive(Clone)] +struct LrDecay { + lr_decay: f64, + epsilon: f32, +} + +impl LrDecay { + pub fn transform( + &self, + grad: Tensor, + lr: LearningRate, + lr_decay_state: Option>, + ) -> (Tensor, LrDecayState) { + let state = if let Some(mut state) = lr_decay_state { + state.sum = state.sum.add(grad.clone().square()); + state.time += 1; + state + } else { + LrDecayState::new(1, grad.clone().square()) + }; + + let new_lr = lr / (1. + (state.time as f64 - 1.) * self.lr_decay); + + let grad = grad + .div(state.sum.clone().sqrt().add_scalar(self.epsilon)) + .mul_scalar(new_lr); + + (grad, state) + } +} + +impl LrDecayState { + /// Move state to device. + /// + /// # Arguments + /// + /// * `device` - Device to move state to. + /// + /// # Returns + /// + /// Returns state moved to device. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.sum = self.sum.to_device(device); + self + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::Tolerance; + use burn::tensor::ops::FloatElem; + + use super::*; + use crate::TestAutodiffBackend; + use crate::{GradientsParams, Optimizer}; + use burn::module::{Module, Param}; + use burn::tensor::{Distribution, Tensor, TensorData}; + use burn_nn::{Linear, LinearConfig, LinearRecord}; + + const LEARNING_RATE: LearningRate = 0.01; + + #[test] + fn test_adagrad_optimizer_save_load_state() { + let device = Default::default(); + let linear = LinearConfig::new(6, 6).init(&device); + let x = Tensor::::random([2, 6], Distribution::Default, &device); + let mut optimizer = create_adagrad(); + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let _linear = optimizer.step(LEARNING_RATE, linear, grads); + + #[cfg(feature = "std")] + { + use burn::record::{BinFileRecorder, FullPrecisionSettings, Recorder}; + + BinFileRecorder::::default() + .record( + optimizer.to_record(), + std::env::temp_dir().as_path().join("test_optim_adagrad"), + ) + .unwrap(); + } + #[cfg(not(feature = "std"))] + { + use burn::record::{BinBytesRecorder, FullPrecisionSettings, Recorder}; + + let result = BinBytesRecorder::::default() + .record(optimizer.to_record(), ()) + .unwrap(); + assert!(!result.is_empty()); + } + + let state_optim_before = optimizer.to_record(); + let state_optim_before_copy = optimizer.to_record(); + let optimizer = create_adagrad(); + let optimizer = optimizer.load_record(state_optim_before_copy); + let state_optim_after = optimizer.to_record(); + + assert_eq!(state_optim_before.len(), state_optim_after.len()); + } + + #[test] + fn test_adagrad_optimizer_with_numbers() { + let device = Default::default(); + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + let x_1 = Tensor::::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &device, + ) + .require_grad(); + let x_2 = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &device, + ) + .require_grad(); + + let mut optimizer = AdaGradConfig::new() + .with_epsilon(1e-8) + .with_lr_decay(0.5) + .init(); + + let grads = linear.forward(x_1).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x_2).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + let weights_expected = TensorData::from([ + [-0.334989, 0.123011, 0.389911, 0.305611, 0.071511, 0.052711], + [ + 0.066144, -0.030056, -0.378256, 0.243444, 0.183944, -0.303756, + ], + [ + -0.033462, 0.020138, -0.310662, 0.233938, -0.292462, 0.298538, + ], + [ + -0.312636, -0.236036, -0.386136, -0.312736, -0.090736, 0.147964, + ], + [ + 0.315896, -0.232304, 0.357596, -0.187004, 0.365496, -0.044504, + ], + [-0.030305, -0.026405, 0.111395, 0.177695, 0.014895, 0.368895], + ]); + let bias_expected = TensorData::from([ + -0.405214, 0.073686, -0.111714, 0.102886, 0.121886, -0.001714, + ]); + + let (weight_updated, bias_updated) = ( + state_updated.weight.val().into_data(), + state_updated.bias.unwrap().val().into_data(), + ); + + type FT = FloatElem; + let tolerance = Tolerance::absolute(1e-6); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + } + + fn given_linear_layer(weight: TensorData, bias: TensorData) -> Linear { + let device = Default::default(); + let record = LinearRecord { + weight: Param::from_data(weight, &device), + bias: Some(Param::from_data(bias, &device)), + }; + + LinearConfig::new(6, 6).init(&device).load_record(record) + } + + fn create_adagrad() + -> OptimizerAdaptor, TestAutodiffBackend> { + let config = AdaGradConfig::new(); + AdaGrad { + lr_decay: LrDecay { + lr_decay: config.lr_decay, + epsilon: config.epsilon, + }, + weight_decay: config.weight_decay.as_ref().map(WeightDecay::new), + } + .into() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adam.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adam.rs new file mode 100644 index 0000000..e58ae5c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adam.rs @@ -0,0 +1,521 @@ +use burn_core as burn; + +use burn::{module::AutodiffModule, record::Record}; + +use burn::config::Config; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use burn::tensor::{backend::Backend, ops::Device}; + +use super::{ + SimpleOptimizer, + adaptor::OptimizerAdaptor, + decay::{WeightDecay, WeightDecayConfig}, +}; +use crate::{LearningRate, grad_clipping::GradientClippingConfig}; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Adam configuration. +#[derive(Config, Debug)] +pub struct AdamConfig { + /// Parameter for Adam. + #[config(default = 0.9)] + beta_1: f32, + /// Parameter for Adam. + #[config(default = 0.999)] + beta_2: f32, + /// A value required for numerical stability. + #[config(default = 1e-5)] + epsilon: f32, + /// Whether to use AMSGrad algorithm + #[config(default = false)] + amsgrad: bool, + /// [Weight decay](WeightDecayConfig) config. + weight_decay: Option, + /// [Gradient Clipping](GradientClippingConfig) config. + grad_clipping: Option, +} + +/// Adam optimizer. +/// +/// See: +/// - [Adam: A Method for Stochastic Optimization](https://arxiv.org/pdf/1412.6980.pdf). +/// - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ) +#[derive(Clone)] +pub struct Adam { + momentum: AdaptiveMomentum, + weight_decay: Option, +} + +/// Adam state. +#[derive(Record, Clone, new)] +pub struct AdamState { + /// The current adaptive momentum. + pub momentum: AdaptiveMomentumState, +} + +impl SimpleOptimizer for Adam { + type State = AdamState; + + fn step( + &self, + lr: LearningRate, + tensor: Tensor, + mut grad: Tensor, + state: Option>, + ) -> (Tensor, Option>) { + let mut state_momentum = None; + + if let Some(state) = state { + state_momentum = Some(state.momentum); + } + + if let Some(weight_decay) = &self.weight_decay { + grad = weight_decay.transform(grad, tensor.clone()); + } + + let (grad, state_momentum) = self.momentum.transform(grad, state_momentum); + + let state = AdamState::new(state_momentum); + let delta = grad.mul_scalar(lr); + + (tensor - delta, Some(state)) + } + + fn to_device(mut state: Self::State, device: &Device) -> Self::State { + state.momentum = state.momentum.to_device(device); + state + } +} + +impl AdamConfig { + /// Initialize Adam optimizer. + /// + /// # Returns + /// + /// Returns an optimizer that can be used to optimize a module. + pub fn init>(&self) -> OptimizerAdaptor { + let optim = Adam { + momentum: AdaptiveMomentum { + beta_1: self.beta_1, + beta_2: self.beta_2, + epsilon: self.epsilon, + amsgrad: self.amsgrad, + }, + weight_decay: self.weight_decay.as_ref().map(WeightDecay::new), + }; + + let mut optim = OptimizerAdaptor::from(optim); + if let Some(config) = &self.grad_clipping { + optim = optim.with_grad_clipping(config.init()); + } + optim + } +} + +/// Adaptive momentum state. +#[derive(Record, new, Clone)] +pub struct AdaptiveMomentumState { + /// The number of iterations aggregated. + pub time: usize, + /// The first order momentum. + pub moment_1: Tensor, + /// The second order momentum. + pub moment_2: Tensor, + /// Max of second order momentum (for AMSGrad) + #[new(default)] + pub max_moment_2: Option>, +} + +#[derive(Clone)] +struct AdaptiveMomentum { + beta_1: f32, + beta_2: f32, + epsilon: f32, + amsgrad: bool, +} + +impl AdaptiveMomentum { + pub fn transform( + &self, + grad: Tensor, + momentum_state: Option>, + ) -> (Tensor, AdaptiveMomentumState) { + let state = if let Some(mut state) = momentum_state { + let factor = 1.0 - self.beta_1; + state.moment_1 = state + .moment_1 + .mul_scalar(self.beta_1) + .add(grad.clone().mul_scalar(factor)); + + let factor = 1.0 - self.beta_2; + state.moment_2 = state + .moment_2 + .mul_scalar(self.beta_2) + .add(grad.square().mul_scalar(factor)); + if self.amsgrad { + let max_v = state + .max_moment_2 + .take() + .unwrap_or_else(|| state.moment_2.clone()); + + let new_max = max_v.max_pair(state.moment_2.clone()); + state.max_moment_2 = Some(new_max); + } + + state.time += 1; + + state + } else { + let factor = 1.0 - self.beta_1; + let moment_1 = grad.clone().mul_scalar(factor); + + let factor = 1.0 - self.beta_2; + let moment_2 = grad.square().mul_scalar(factor); + let max_moment_2 = self.amsgrad.then(|| moment_2.clone()); + AdaptiveMomentumState { + time: 1, + moment_1, + moment_2, + max_moment_2, + } + }; + + let time = state.time as i32; + let bias_correction2_sqrt = (1.0 - self.beta_2.powi(time)).sqrt(); + let combined_factor = bias_correction2_sqrt / (1.0 - self.beta_1.powi(time)); + + let v_to_use = if self.amsgrad { + state.max_moment_2.as_ref().unwrap_or(&state.moment_2) + } else { + &state.moment_2 + }; + + let grad = state.moment_1.clone().mul_scalar(combined_factor).div( + v_to_use + .clone() + .sqrt() + .add_scalar(self.epsilon * bias_correction2_sqrt), + ); + (grad, state) + } +} + +impl AdaptiveMomentumState { + /// Move state to device. + /// + /// # Arguments + /// + /// * `device` - Device to move state to. + /// + /// # Returns + /// + /// Returns state moved to device. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.moment_1 = self.moment_1.to_device(device); + self.moment_2 = self.moment_2.to_device(device); + self.max_moment_2 = self.max_moment_2.map(|tensor| tensor.to_device(device)); + self + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::Tolerance; + use burn::tensor::ops::FloatElem; + + use super::*; + use crate::TestAutodiffBackend; + use crate::{GradientsParams, Optimizer}; + use burn::module::{Module, Param}; + use burn::tensor::{Distribution, Tensor, TensorData}; + use burn_nn::{Linear, LinearConfig, LinearRecord}; + + const LEARNING_RATE: LearningRate = 0.01; + + #[test] + fn test_adam_optimizer_save_load_state() { + let device = Default::default(); + let linear = LinearConfig::new(6, 6).init(&device); + let x = Tensor::::random([2, 6], Distribution::Default, &device); + let mut optimizer = create_adam(); + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let _linear = optimizer.step(LEARNING_RATE, linear, grads); + + #[cfg(feature = "std")] + { + use burn::record::{BinFileRecorder, FullPrecisionSettings, Recorder}; + + BinFileRecorder::::default() + .record( + optimizer.to_record(), + std::env::temp_dir().as_path().join("test_optim_adam"), + ) + .unwrap(); + } + #[cfg(not(feature = "std"))] + { + use burn::record::{BinBytesRecorder, FullPrecisionSettings, Recorder}; + + let result = BinBytesRecorder::::default() + .record(optimizer.to_record(), ()) + .unwrap(); + assert!(!result.is_empty()); + } + + let state_optim_before = optimizer.to_record(); + let state_optim_before_copy = optimizer.to_record(); + let optimizer = create_adam(); + let optimizer = optimizer.load_record(state_optim_before_copy); + let state_optim_after = optimizer.to_record(); + + assert_eq!(state_optim_before.len(), state_optim_after.len()); + } + #[test] + fn test_adam_optimizer_with_amsgrad_50_steps() { + let device = Default::default(); + let mut linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + + let mut optimizer = AdamConfig::new() + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_amsgrad(true) + .with_weight_decay(Some(WeightDecayConfig::new(0.5))) + .init(); + + for i in 1..=50 { + let x = Tensor::::ones([2, 6], &device) + .mul_scalar(i as f32 * 0.1) + .require_grad(); + + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + linear = optimizer.step(LEARNING_RATE, linear, grads); + } + + let state_updated = linear.into_record(); + let weight_updated = state_updated.weight.to_data(); + let bias_updated = state_updated.bias.unwrap().to_data(); + + let weights_expected = TensorData::from([ + [ + -0.9125810265541077, + -0.45855265855789185, + -0.1915993094444275, + -0.2759990692138672, + -0.5099529027938843, + -0.5287043452262878, + ], + [ + -0.5181325674057007, + -0.6139854788780212, + -0.9574727416038513, + -0.34102925658226013, + -0.400514155626297, + -0.8847861886024475, + ], + [ + -0.614483118057251, + -0.5611032247543335, + -0.8887064456939697, + -0.34762972593307495, + -0.8708556890487671, + -0.2830044627189636, + ], + [ + -0.8904699683189392, + -0.8151527643203735, + -0.9621278643608093, + -0.8905676603317261, + -0.671261191368103, + -0.4333854615688324, + ], + [ + -0.26599061489105225, + -0.8119961023330688, + -0.22424538433551788, + -0.7672406435012817, + -0.2163349837064743, + -0.6258266568183899, + ], + [ + -0.611397922039032, + -0.6075160503387451, + -0.4701341986656189, + -0.4039117991924286, + -0.5663845539093018, + -0.21262989938259125, + ], + ]); + let bias_expected = TensorData::from([ + -0.8817203044891357, + -0.4038999378681183, + -0.5889149308204651, + -0.37475723028182983, + -0.3557940721511841, + -0.47914788126945496, + ]); + + type FT = FloatElem; + let tolerance = Tolerance::absolute(1e-5); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + } + #[test] + fn test_adam_optimizer_with_numbers() { + let device = Default::default(); + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + let x_1 = Tensor::::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &device, + ) + .require_grad(); + let x_2 = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &device, + ) + .require_grad(); + + let mut optimizer = AdamConfig::new() + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_weight_decay(Some(WeightDecayConfig::new(0.5))) + .init(); + + let grads = linear.forward(x_1).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x_2).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + let weights_expected = TensorData::from([ + [-0.340528, 0.118929, 0.384336, 0.300010, 0.066034, 0.047154], + [ + 0.057757, -0.036690, -0.386649, 0.235010, 0.175624, -0.312133, + ], + [ + -0.038940, 0.016306, -0.316151, 0.228410, -0.297819, 0.293047, + ], + [ + -0.317929, -0.239100, -0.391449, -0.318087, -0.095948, 0.142651, + ], + [ + 0.310050, -0.235909, 0.351736, -0.192888, 0.359710, -0.050343, + ], + [-0.035840, -0.030203, 0.105840, 0.172110, 0.009440, 0.363346], + ]); + let bias_expected = TensorData::from([ + -0.410499, 0.068401, -0.116999, 0.097601, 0.116601, -0.006999, + ]); + + let (weight_updated, bias_updated) = ( + state_updated.weight.to_data(), + state_updated.bias.unwrap().to_data(), + ); + + type FT = FloatElem; + let tolerance = Tolerance::absolute(1e-2); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + } + + #[test] + fn test_adam_optimizer_no_nan() { + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + + let x = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &Default::default(), + ) + .require_grad(); + + let mut optimizer = AdamConfig::new() + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_weight_decay(Some(WeightDecayConfig::new(0.5))) + .init(); + + let grads = linear.forward(x.clone()).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + assert!(!state_updated.weight.to_data().as_slice::().unwrap()[0].is_nan()); + } + + fn given_linear_layer(weight: TensorData, bias: TensorData) -> Linear { + let device = Default::default(); + let record = LinearRecord { + weight: Param::from_data(weight, &device), + bias: Some(Param::from_data(bias, &device)), + }; + + LinearConfig::new(6, 6).init(&device).load_record(record) + } + + fn create_adam() -> OptimizerAdaptor, TestAutodiffBackend> { + let config = AdamConfig::new(); + Adam { + momentum: AdaptiveMomentum { + beta_1: config.beta_1, + beta_2: config.beta_2, + epsilon: config.epsilon, + amsgrad: config.amsgrad, + }, + weight_decay: config.weight_decay.as_ref().map(WeightDecay::new), + } + .into() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adamw.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adamw.rs new file mode 100644 index 0000000..5c9d679 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/adamw.rs @@ -0,0 +1,598 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use burn::tensor::{backend::Backend, ops::Device}; +use burn::{module::AutodiffModule, record::Record}; + +use super::{AdaptiveMomentumState, SimpleOptimizer, adaptor::OptimizerAdaptor}; +use crate::{LearningRate, grad_clipping::GradientClippingConfig}; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// [`AdamW`] Configuration. +#[derive(Config, Debug)] +pub struct AdamWConfig { + /// Parameter for AdamW. + #[config(default = 0.9)] + beta_1: f32, + /// Parameter for AdamW. + #[config(default = 0.999)] + beta_2: f32, + /// A value required for numerical stability. + #[config(default = 1e-5)] + epsilon: f32, + /// Weight decay config. + #[config(default = 1e-4)] + weight_decay: f32, + + /// Cautious weight decay config. + /// + /// See: + #[config(default = false)] + cautious_weight_decay: bool, + + /// Whether to use AMSGrad algorithm + #[config(default = false)] + amsgrad: bool, + /// [Gradient Clipping](GradientClippingConfig) config. + grad_clipping: Option, +} + +/// AdamW optimizer. +/// +/// See: +/// - [Decoupled Weight Decay Regularization, Loshchilov and Hutter, 2019](https://arxiv.org/abs/1711.05101). +/// - [Cautious Weight Decay, 2025](https://arxiv.org/abs/2510.12402) +/// - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ) +/// +/// Configured by [`AdamWConfig`]. +#[derive(Clone)] +pub struct AdamW { + momentum: AdaptiveMomentumW, + weight_decay: f32, + cautious_weight_decay: bool, +} + +/// AdamW state. +#[derive(Record, Clone, new)] +pub struct AdamWState { + /// Th current adaptive momentum state. + pub momentum: AdaptiveMomentumState, +} + +impl SimpleOptimizer for AdamW { + type State = AdamWState; + + /// A single optimization step for any tensor that represents the parameters of a model. + fn step( + &self, + // Learning rate. + lr: LearningRate, + // Any tensor that represents the parameters of a model. + tensor: Tensor, + // Gradient of the loss w.r.t. the parameters. + grad: Tensor, + // State of the optimizer. + state: Option>, + ) -> (Tensor, Option>) { + let (raw_delta, momentum_state) = self.momentum.transform(grad, state.map(|s| s.momentum)); + + let decay_rate = lr * (self.weight_decay as f64); + + let decayed_tensor = if decay_rate == 0.0 { + tensor.clone() + } else if self.cautious_weight_decay { + // Cautious weight decay. + // See: https://arxiv.org/abs/2510.12402 + let tensor_pos = tensor.clone().greater_equal_elem(0.0); + let grad_pos = momentum_state.moment_1.clone().greater_equal_elem(0.0); + let differ = tensor_pos.not_equal(grad_pos); + + // Zero out the decay where the decay is counter to the update direction. + tensor.clone() - tensor.mul_scalar(decay_rate).mask_fill(differ, 0.0) + } else { + tensor.clone().mul_scalar(1.0 - decay_rate) + }; + + let tensor_updated = decayed_tensor - raw_delta.mul_scalar(lr); + + let state = AdamWState { + momentum: momentum_state, + }; + + (tensor_updated, Some(state)) + } + + fn to_device(mut state: Self::State, device: &Device) -> Self::State { + state.momentum = state.momentum.to_device(device); + state + } +} + +impl AdamWConfig { + /// Initialize AdamW optimizer. + /// + /// # Returns + /// + /// Returns an optimizer that can be used to optimize a module. + pub fn init>(&self) -> OptimizerAdaptor { + let optim = AdamW { + momentum: AdaptiveMomentumW { + beta_1: self.beta_1, + beta_2: self.beta_2, + epsilon: self.epsilon, + amsgrad: self.amsgrad, + }, + weight_decay: self.weight_decay, + cautious_weight_decay: self.cautious_weight_decay, + }; + + let mut optim = OptimizerAdaptor::from(optim); + if let Some(config) = &self.grad_clipping { + optim = optim.with_grad_clipping(config.init()); + } + optim + } +} + +#[derive(Clone)] +struct AdaptiveMomentumW { + beta_1: f32, + beta_2: f32, + epsilon: f32, + amsgrad: bool, +} + +impl AdaptiveMomentumW { + pub fn transform( + &self, + grad: Tensor, + state: Option>, + ) -> (Tensor, AdaptiveMomentumState) { + let factor_1 = 1.0 - self.beta_1; + let factor_2 = 1.0 - self.beta_2; + + let state = if let Some(mut state) = state { + // Update first moment estimate. + state.moment_1 = state + .moment_1 + .mul_scalar(self.beta_1) + .add(grad.clone().mul_scalar(factor_1)); + + // Update second moment estimate. + state.moment_2 = state + .moment_2 + .mul_scalar(self.beta_2) + .add(grad.square().mul_scalar(factor_2)); + + if self.amsgrad { + let max_v = state + .max_moment_2 + .take() + .unwrap_or_else(|| state.moment_2.clone()); + state.max_moment_2 = Some(max_v.max_pair(state.moment_2.clone())); + } + + // Update time. + state.time += 1; + + state + } else { + // Initialize first moment estimate. + let moment_1 = grad.clone().mul_scalar(factor_1); + + // Initialize second moment estimate. + let moment_2 = grad.square().mul_scalar(factor_2); + let max_moment_2 = self.amsgrad.then(|| moment_2.clone()); + AdaptiveMomentumState { + time: 1, + moment_1, + moment_2, + max_moment_2, + } + }; + + let time: i32 = state.time as i32; + + // Compute bias-corrected first and second moment estimates. + let moment_1_corrected = state + .moment_1 + .clone() + .div_scalar(1f32 - self.beta_1.powi(time)); + + let v_to_use = if self.amsgrad { + state.max_moment_2.as_ref().unwrap_or(&state.moment_2) + } else { + &state.moment_2 + }; + + let moment_2_corrected = v_to_use.clone().div_scalar(1f32 - self.beta_2.powi(time)); + + let update_delta = + moment_1_corrected.div(moment_2_corrected.sqrt().add_scalar(self.epsilon)); + + (update_delta, state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestAutodiffBackend; + use crate::{GradientsParams, Optimizer}; + use burn::module::{Module, Param}; + use burn::tensor::{Distribution, Tensor, TensorData}; + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_nn::{Linear, LinearConfig, LinearRecord}; + + type FT = FloatElem; + + const LEARNING_RATE: LearningRate = 0.01; + + #[test] + fn test_adamw_optimizer_save_load_state() { + let device = Default::default(); + let linear = LinearConfig::new(6, 6).init(&device); + let x = Tensor::::random([2, 6], Distribution::Default, &device); + let mut optimizer = create_adamw(); + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let _linear = optimizer.step(LEARNING_RATE, linear, grads); + + #[cfg(feature = "std")] + { + use burn::record::{BinFileRecorder, FullPrecisionSettings, Recorder}; + + BinFileRecorder::::default() + .record( + optimizer.to_record(), + std::env::temp_dir().as_path().join("test_optim_adamw"), + ) + .unwrap(); + } + #[cfg(not(feature = "std"))] + { + use burn::record::{BinBytesRecorder, FullPrecisionSettings, Recorder}; + + let result = BinBytesRecorder::::default() + .record(optimizer.to_record(), ()) + .unwrap(); + assert!(!result.is_empty()); + } + + let state_optim_before = optimizer.to_record(); + let state_optim_before_copy = optimizer.to_record(); + let optimizer = create_adamw(); + let optimizer = optimizer.load_record(state_optim_before_copy); + let state_optim_after = optimizer.to_record(); + + assert_eq!(state_optim_before.len(), state_optim_after.len()); + } + #[test] + fn test_adamw_optimizer_with_amsgrad_50_steps() { + let device = Default::default(); + let mut linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + + let mut optimizer = AdamWConfig::new() + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_amsgrad(true) + .with_weight_decay(0.5) + .init(); + + for i in 1..=50 { + let x = Tensor::::ones([2, 6], &device) + .mul_scalar(i as f32 * 0.1) + .require_grad(); + + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + linear = optimizer.step(LEARNING_RATE, linear, grads); + } + + let state_updated = linear.into_record(); + let weight_updated = state_updated.weight.to_data(); + let bias_updated = state_updated.bias.unwrap().to_data(); + + let weights_expected = TensorData::from([ + [ + -0.7822558283805847, + -0.42578864097595215, + -0.21805696189403534, + -0.28366872668266296, + -0.46587175130844116, + -0.4805040955543518, + ], + [ + -0.4722539782524109, + -0.5471276640892029, + -0.8181359767913818, + -0.33425918221473694, + -0.3805687427520752, + -0.7601516842842102, + ], + [ + -0.5475167632102966, + -0.5057991743087769, + -0.763265073299408, + -0.3393959403038025, + -0.7490996718406677, + -0.28911691904067993, + ], + [ + -0.7646660208702087, + -0.7050473093986511, + -0.8218720555305481, + -0.7647438049316406, + -0.5919585227966309, + -0.40617525577545166, + ], + [ + -0.27588561177253723, + -0.7025567889213562, + -0.24343004822731018, + -0.6672990918159485, + -0.23728127777576447, + -0.556389570236206, + ], + [ + -0.5451040267944336, + -0.5420684814453125, + -0.4348171353340149, + -0.3832150399684906, + -0.5099242925643921, + -0.23440153896808624, + ], + ]); + let bias_expected = TensorData::from([ + -0.7473056316375732, + -0.3745720386505127, + -0.5188710689544678, + -0.35184532403945923, + -0.33705732226371765, + -0.4332566559314728, + ]); + + type FT = FloatElem; + let tolerance = Tolerance::absolute(1e-5); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + } + #[test] + fn test_adamw_optimizer_with_numbers() { + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + let device = Default::default(); + let x_1 = Tensor::::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &device, + ) + .require_grad(); + let x_2 = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &device, + ) + .require_grad(); + + let mut optimizer = AdamWConfig::new() + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_weight_decay(0.5) + .init(); + + let grads = linear.forward(x_1).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x_2).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + let weights_expected = TensorData::from([ + [-0.337295, 0.117827, 0.380358, 0.296868, 0.065232, 0.046534], + [ + 0.057032, -0.036518, -0.382951, 0.232516, 0.173738, -0.309182, + ], + [ + -0.038703, 0.016052, -0.313155, 0.225982, -0.295039, 0.289981, + ], + [ + -0.314920, -0.237394, -0.387704, -0.315067, -0.095153, 0.141081, + ], + [ + 0.306815, -0.234226, 0.348083, -0.191115, 0.356002, -0.049993, + ], + [-0.035634, -0.030083, 0.104636, 0.170244, 0.009196, 0.359580], + ]); + let bias_expected = TensorData::from([ + -0.406555, 0.067568, -0.115982, 0.096477, 0.115287, -0.007080, + ]); + + let (weight_updated, bias_updated) = ( + state_updated.weight.to_data(), + state_updated.bias.unwrap().to_data(), + ); + + let tolerance = Tolerance::absolute(1e-2); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + } + + #[test] + fn test_adamw_optimizer_with_numbers_cautious() { + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + let device = Default::default(); + let x_1 = Tensor::::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &device, + ) + .require_grad(); + let x_2 = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, -0.9085], + ], + &device, + ) + .require_grad(); + + let mut optimizer = AdamWConfig::new() + .with_cautious_weight_decay(true) + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_weight_decay(0.5) + .init(); + + let grads = linear.forward(x_1).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x_2).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + let weights_expected = TensorData::from([ + [-0.337295, 0.117827, 0.380358, 0.296868, 0.065232, 0.046534], + [ + 0.057032, -0.036518, -0.382951, 0.232516, 0.173738, -0.309182, + ], + [ + -0.038703, 0.016052, -0.313155, 0.225982, -0.295039, 0.289981, + ], + [ + -0.314920, -0.237394, -0.387704, -0.315067, -0.095153, 0.141081, + ], + [ + 0.306815, -0.234226, 0.348083, -0.191115, 0.356002, -0.049993, + ], + [ + -0.035634, -0.030083, 0.104636, 0.170244, 0.009196, 0.37061332, + ], + ]); + let bias_expected = TensorData::from([ + -0.406555, 0.067568, -0.115982, 0.096477, 0.115287, -0.007080, + ]); + + let (weight_updated, bias_updated) = ( + state_updated.weight.to_data(), + state_updated.bias.unwrap().to_data(), + ); + + let tolerance = Tolerance::absolute(1e-2); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + } + + #[test] + fn test_adam_optimizer_no_nan() { + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + + let x = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &Default::default(), + ) + .require_grad(); + + let mut optimizer = AdamWConfig::new() + .with_epsilon(1e-8) + .with_beta_1(0.9) + .with_beta_2(0.999) + .with_weight_decay(0.5) + .init(); + + let grads = linear.forward(x.clone()).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + assert!(!state_updated.weight.to_data().as_slice::().unwrap()[0].is_nan()); + } + + fn given_linear_layer(weight: TensorData, bias: TensorData) -> Linear { + let device = Default::default(); + let record = LinearRecord { + weight: Param::from_data(weight, &device), + bias: Some(Param::from_data(bias, &device)), + }; + + LinearConfig::new(6, 6).init(&device).load_record(record) + } + + fn create_adamw() -> OptimizerAdaptor, TestAutodiffBackend> { + let config = AdamWConfig::new(); + AdamW { + momentum: AdaptiveMomentumW { + beta_1: config.beta_1, + beta_2: config.beta_2, + epsilon: config.epsilon, + amsgrad: config.amsgrad, + }, + weight_decay: config.weight_decay, + cautious_weight_decay: false, + } + .into() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/base.rs new file mode 100644 index 0000000..0f366c2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/base.rs @@ -0,0 +1,88 @@ +use burn_core::{self as burn, Tensor}; + +use burn_core::module::ParamId; +use burn_core::prelude::{Backend, DeviceOps}; +use burn_core::tensor::Device; +use burn_core::tensor::backend::DeviceId; + +use super::GradientsParams; +use crate::LearningRate; +use alloc::vec::Vec; +use burn::module::AutodiffModule; +use burn::record::Record; +use burn::tensor::backend::AutodiffBackend; + +#[derive(Default)] +/// Exposes multiple gradients for each parameter. +pub struct MultiGradientsParams { + /// Each [GradientsParams] has its associated [DeviceId]. + pub grads: Vec<(GradientsParams, DeviceId)>, +} + +impl MultiGradientsParams { + /// Removes the gradients for the given [parameter id](ParamId). + /// + /// Potentially accumulates the gradients from multiple sources using a device associated with + /// a parameter id. The same parameter will be accumulated using the same device during + /// all training. + pub fn remove( + &mut self, + id: ParamId, + ) -> Option<(Tensor, Device)> { + let (mut tensor, device, index) = self.select(id)?; + + for (i, (grads, _)) in self.grads.iter_mut().enumerate() { + if i == index { + continue; + } + + if let Some(grad) = grads.remove::(id) { + tensor = tensor + grad.to_device(&device); + } + } + + Some((tensor, device)) + } + + fn select( + &mut self, + id: ParamId, + ) -> Option<(Tensor, Device, usize)> { + let id_val = id.val() as usize; + for i in 0..self.grads.len() { + let selected_device_index = (id_val + i) % self.grads.len(); + + if let Some(acc) = self.grads[selected_device_index].0.remove::(id) { + let device_id = self.grads[selected_device_index].1; + let device = ::from_id(device_id); + return Some((acc.to_device(&device), device, selected_device_index)); + } + } + + None + } +} + +/// General trait to optimize [module](AutodiffModule). +pub trait Optimizer: Send + Clone +where + M: AutodiffModule, + B: AutodiffBackend, +{ + /// Optimizer associative type to be used when saving and loading the state. + type Record: Record; + + /// Perform the optimizer step using the given learning rate and gradients. + /// The updated module is returned. + fn step(&mut self, lr: LearningRate, module: M, grads: GradientsParams) -> M; + + /// Perform the optimizer step using the given learning rate and gradients. + /// The updated module is returned. + fn step_multi(&mut self, lr: LearningRate, module: M, grads: MultiGradientsParams) -> M; + + /// Get the current state of the optimizer as a [record](Record). + fn to_record(&self) -> Self::Record; + + /// Load the state of the optimizer as a [record](Record). + fn load_record(self, record: Self::Record) -> Self; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/decay.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/decay.rs new file mode 100644 index 0000000..cc3bb10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/decay.rs @@ -0,0 +1,68 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::record::Record; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; + +/// Configuration to create [weight decay](WeightDecay). +#[derive(Config, Debug)] +pub struct WeightDecayConfig { + /// L2 penalty. + pub penalty: f32, +} + +/// State of [weight decay](WeightDecay). +#[derive(Record, Clone, new)] +pub struct WeightDecayState { + pub(crate) grad_last_step: Tensor, +} + +/// Weight decay implementation that transforms gradients. +#[derive(Clone)] +pub struct WeightDecay { + penalty: f32, +} + +impl WeightDecay { + /// Creates a new [weight decay](WeightDecay) from a [config](WeightDecayConfig). + pub fn new(config: &WeightDecayConfig) -> Self { + Self { + penalty: config.penalty, + } + } + + /// Transforms a gradient. + /// + /// # Arguments + /// + /// * `grad` - Gradient to transform. + /// * `tensor` - Tensor param of the last iteration. + /// + /// # Returns + /// + /// * `grad` - Transformed gradient. + pub fn transform( + &self, + grad: Tensor, + tensor: Tensor, + ) -> Tensor { + tensor.mul_scalar(self.penalty).add(grad) + } +} + +impl WeightDecayState { + /// Moves the state to a device. + /// + /// # Arguments + /// + /// * `device` - Device to move the state to. + /// + /// # Returns + /// + /// * `self` - Moved state. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.grad_last_step = self.grad_last_step.to_device(device); + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grad_accum.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grad_accum.rs new file mode 100644 index 0000000..8317463 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grad_accum.rs @@ -0,0 +1,121 @@ +use burn_core as burn; + +use core::marker::PhantomData; + +use burn::module::{AutodiffModule, ModuleVisitor, Param}; +use burn::tensor::{Tensor, backend::AutodiffBackend}; + +use super::GradientsParams; + +/// Accumulate gradients into a single [Gradients](AutodiffBackend::Gradients) object. +pub struct GradientsAccumulator { + grads: GradientsParams, + phantom: PhantomData, +} + +impl Default for GradientsAccumulator { + fn default() -> Self { + Self::new() + } +} + +impl GradientsAccumulator { + /// Create a new gradients accumulator. + pub fn new() -> Self { + Self { + grads: GradientsParams::new(), + phantom: PhantomData, + } + } +} + +impl GradientsAccumulator { + /// Accumulate the given gradients for each parameter in the given module. + pub fn accumulate(&mut self, module: &M, grads: GradientsParams) + where + M: AutodiffModule, + { + let mut visitor = ModuleGradsAccumulator::::new(&mut self.grads, grads); + module.visit(&mut visitor); + } + + /// Return the accumulated gradients and reset the accumulator state. + pub fn grads(&mut self) -> GradientsParams { + let mut grads = GradientsParams::new(); + core::mem::swap(&mut self.grads, &mut grads); + + grads + } +} + +#[derive(new)] +struct ModuleGradsAccumulator<'a, M> { + grads: &'a mut GradientsParams, + grads_new: GradientsParams, + phantom: PhantomData, +} + +impl> ModuleVisitor for ModuleGradsAccumulator<'_, M> { + fn visit_float(&mut self, param: &Param>) { + let grad_updated = match self.grads_new.remove::(param.id) { + Some(new) => match self.grads.remove::(param.id) { + Some(grad) => grad.add(new), + None => new, + }, + None => match self.grads.remove::(param.id) { + Some(grad) => grad, + None => return, + }, + }; + + self.grads + .register::(param.id, grad_updated); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestAutodiffBackend; + use burn::tensor::{Distribution, backend::Backend}; + use burn_nn::{Linear, LinearConfig}; + + #[test] + fn test_accumulate_gradients_one_step() { + let device = Default::default(); + let mut accumulator = GradientsAccumulator::new(); + let layer = layer::(&device); + let loss = layer.forward(random_tensor::(&device)); + let grads = GradientsParams::from_grads(loss.backward(), &layer); + + accumulator.accumulate(&layer, grads); + + let grads = accumulator.grads(); + assert!(!grads.is_empty()) + } + + #[test] + fn test_accumulate_gradients_two_steps() { + let device = Default::default(); + let mut accumulator = GradientsAccumulator::new(); + let layer = layer::(&device); + let loss_1 = layer.forward(random_tensor(&device)); + let loss_2 = layer.forward(random_tensor(&device)); + let grads_1 = GradientsParams::from_grads(loss_1.backward(), &layer); + let grads_2 = GradientsParams::from_grads(loss_2.backward(), &layer); + + accumulator.accumulate(&layer, grads_1); + accumulator.accumulate(&layer, grads_2); + + let grads = accumulator.grads(); + assert_eq!(grads.len(), 2) + } + + fn layer(device: &B::Device) -> Linear { + LinearConfig::new(20, 20).init(device) + } + + fn random_tensor(device: &B::Device) -> Tensor { + Tensor::::random([2, 20], Distribution::Default, device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grads.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grads.rs new file mode 100644 index 0000000..888e225 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/grads.rs @@ -0,0 +1,192 @@ +use burn_core as burn; + +#[cfg(feature = "collective")] +use burn_collective::{CollectiveError, PeerId, ReduceOperation, all_reduce}; + +use burn::{ + Tensor, + tensor::{ + backend::{AutodiffBackend, Backend}, + container::TensorContainer, + }, +}; + +use burn::module::{AutodiffModule, ParamId}; + +use super::visitor::{GradientsParamsChangeDevice, GradientsParamsConverter}; + +/// Data type that contains gradients for parameters. +#[derive(Default, Debug)] +pub struct GradientsParams { + container: TensorContainer, +} + +impl GradientsParams { + /// Creates a new [GradientsParams](GradientsParams). + pub fn new() -> Self { + Self::default() + } + + /// Extract each tensor gradients for the given [module](AutodiffModule). + /// + /// Note: This consumes the gradients. See ['from_module'] to extract gradients only for + /// a specific module. + pub fn from_grads>( + grads: B::Gradients, + module: &M, + ) -> Self { + let mut grads = grads; + Self::from_module(&mut grads, module) + } + + /// Extract each tensor gradients for the given [module](AutodiffModule). + pub fn from_module>( + grads: &mut B::Gradients, + module: &M, + ) -> Self { + let mut grads_params = GradientsParams::new(); + let mut visitor = GradientsParamsConverter::::new(grads, &mut grads_params, None); + module.visit(&mut visitor); + grads_params + } + + /// Extract tensor gradients for the given [module](AutodiffModule) and given parameters. + pub fn from_params>( + grads: &mut B::Gradients, + module: &M, + params: &[ParamId], + ) -> Self { + let mut grads_params = GradientsParams::new(); + let mut visitor = + GradientsParamsConverter::::new(grads, &mut grads_params, Some(params.to_vec())); + module.visit(&mut visitor); + grads_params + } + + /// Get the gradients for the given [parameter id](ParamId). + /// + /// # Notes + /// + /// You should use [remove](GradientsParams::remove) if you want to get the gradients + /// only one time. + pub fn get(&self, id: ParamId) -> Option> + where + B: Backend, + { + self.container.get(&id).map(Tensor::from_primitive) + } + + /// Remove the gradients for the given [parameter id](ParamId). + pub fn remove(&mut self, id: ParamId) -> Option> + where + B: Backend, + { + self.container.remove(&id).map(Tensor::from_primitive) + } + + /// Register a gradients tensor for the given [parameter id](ParamId). + /// + /// # Notes + /// + /// If a tensor is already registered for the given [parameter id](ParamId), it will be replaced. + pub fn register(&mut self, id: ParamId, value: Tensor) + where + B: Backend, + { + self.container.register(id, value.into_primitive()) + } + + /// The number of gradients tensors registered. + pub fn len(&self) -> usize { + self.container.len() + } + + /// If any tensor is contained. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Change the device of each tensor gradients registered for the given [module](AutodiffModule). + pub fn to_device>( + mut self, + device: &B::Device, + module: &M, + ) -> Self { + let mut visitor = GradientsParamsChangeDevice::::new(device, &mut self); + module.visit(&mut visitor); + self + } + + /// Syncs the gradient params with the other peers in the collective. + #[cfg(feature = "collective")] + pub fn all_reduce( + mut self, + peer_id: PeerId, + op: ReduceOperation, + ) -> Result { + let mut ids = self + .container + .ids() + .into_iter() + .copied() + .collect::>(); + // This is crucial, since the all-reduce operations need to happen in the same order for the same parameters on all nodes! + ids.sort(); + + for id in ids { + let Some(grad) = self.container.remove::(&id) else { + todo!() + }; + + let grad = match grad { + burn::tensor::TensorPrimitive::Float(grad) => { + let grad = all_reduce::(peer_id, grad, op)?; + burn::tensor::TensorPrimitive::Float(grad) + } + burn::tensor::TensorPrimitive::QFloat(_grad) => { + unimplemented!("quantized all-reduce unimplemented") + } + }; + + self.container.register::(id, grad); + } + + Ok(self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestAutodiffBackend; + use burn::module::{Module, list_param_ids}; + use burn::tensor::{Distribution, backend::Backend}; + use burn_nn::{Linear, LinearConfig}; + + #[test] + fn test_convert_grads() { + let device = Default::default(); + let layer_1 = layer::(&device); + let mut layer_2 = layer_1.clone(); + layer_2 = layer_2.fork(&device); + let loss_1 = layer_1.forward(random_tensor(&device)); + let loss_2 = layer_2.forward(random_tensor(&device)); + let grads_1 = GradientsParams::from_grads(loss_1.backward(), &layer_1); + let grads_2 = GradientsParams::from_grads(loss_2.backward(), &layer_2); + + let param_ids_1 = list_param_ids(&layer_1); + let param_ids_2 = list_param_ids(&layer_2); + + assert_eq!(param_ids_1, param_ids_2); + assert_eq!(grads_1.len(), param_ids_1.len()); + assert_eq!(grads_2.len(), param_ids_2.len()); + } + + fn layer(device: &B::Device) -> Linear { + LinearConfig::new(20, 20).init(device) + } + + fn random_tensor(device: &B::Device) -> Tensor { + Tensor::::random([2, 20], Distribution::Default, device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/lbfgs.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/lbfgs.rs new file mode 100644 index 0000000..76d6a1f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/lbfgs.rs @@ -0,0 +1,978 @@ +#![allow(clippy::excessive_precision)] + +use burn_core as burn; + +use super::GradientsParams; +use crate::LearningRate; +use burn::config::Config; +use burn::module::{AutodiffModule, Module, ModuleMapper, ModuleVisitor, Param}; +use burn::prelude::ToElement; +use burn::record::Record; +use burn::tensor::backend::Backend; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use serde::{Deserialize, Serialize}; + +use alloc::vec; +use alloc::vec::Vec; +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Cubic Interpolate +/// +/// Uses two points (x1, f1), (x2, f2) and their first derivatives g1,g2 to construct +/// a cubic interpolant and return its minimum within the given bounds. +fn cubic_interpolate( + x1: f64, + f1: f64, + g1: f64, + x2: f64, + f2: f64, + g2: f64, + bounds: Option<(f64, f64)>, +) -> f64 { + // Compute bounds of interpolation area + let (min_bound, max_bound) = bounds.unwrap_or(if x1 <= x2 { (x1, x2) } else { (x2, x1) }); + // Code for most common case: cubic interpolation of 2 points + // with function and derivative values for both + // Solution in this case (where x2 is the farthest point) + // d1 = g1 + g2 - 3*(f1 - f2) / (x1-x2); + // d2 = sqrt(d1^2 - g1 * g2); + // min_pos = x2 - (x2 - x1)*((g2 + d2 - d1)/(g2 - g1 + 2*d2)); + // t_new = min(max(min_pos,min_bound), max_bound); + let d1 = g1 + g2 - 3.0 * (f1 - f2) / (x1 - x2); + let d2_square = d1 * d1 - g1 * g2; + + if d2_square >= 0.0 { + let d2 = d2_square.sqrt(); + let min_pos = if x1 <= x2 { + x2 - (x2 - x1) * ((g2 + d2 - d1) / (g2 - g1 + 2.0 * d2)) + } else { + x1 - (x1 - x2) * ((g1 + d2 - d1) / (g1 - g2 + 2.0 * d2)) + }; + min_pos.max(min_bound).min(max_bound) + } else { + (min_bound + max_bound) / 2.0 + } +} +/// Auxiliary Struct For Strong_Wolfe +struct LineSearchSample { + // step size + t: f64, + // loss + f: f64, + // gradient + g: Tensor, + // directional derivative + gtd: f64, +} + +#[allow(clippy::too_many_arguments)] +fn strong_wolfe( + // obj_func(x,step size,direction) -> (loss,grad) + obj_func: &mut F, + x: &Tensor, + // initial step size + mut t: f64, + d: &Tensor, + f: f64, + g: Tensor, + gtd: f64, + c1: f64, + c2: f64, + tolerance_change: f64, + max_ls: usize, +) -> (f64, Tensor, f64, usize) +where + F: FnMut(&Tensor, f64, &Tensor) -> (f64, Tensor), +{ + let d_norm = d.clone().abs().max().into_scalar().to_f64(); + + // evaluate objective and gradient using initial step + let (mut f_new, mut g_new) = obj_func(x, t, d); + let mut ls_func_evals = 1; + let mut gtd_new = g_new.clone().dot(d.clone()).into_scalar().to_f64(); + + // bracket an interval [t_prev,t] containing a point satisfying the Wolfe criteria + let (mut t_prev, mut f_prev, mut g_prev, mut gtd_prev) = (0.0, f, g.clone(), gtd); + let mut done = false; + let mut ls_iter = 0; + + // the interval [low,high] using for Zoom phase + let mut bracket: Option<[LineSearchSample; 2]> = None; + // point which satisfy the wolfe condition + let mut wolfe_bracket: Option> = None; + while ls_iter < max_ls { + // Checking Conditions. + + // Checking the Armijo Condition and function value increasing condition. + // Armijo: f(x+t*d) <= f(x) + c_1 t gtd + if f_new > (f + c1 * t * gtd) || (ls_iter > 1 && f_new >= f_prev) { + bracket = Some([ + LineSearchSample { + t: t_prev, + f: f_prev, + g: g_prev, + gtd: gtd_prev, + }, + LineSearchSample { + t, + f: f_new, + g: g_new.clone(), + gtd: gtd_new, + }, + ]); + break; + } + + // Checking Strong Wolfe Condition + // |gtd_new| <= -c_2 gtd + if gtd_new.abs() <= -c2 * gtd { + wolfe_bracket = Some(LineSearchSample { + t, + f: f_new, + g: g_new.clone(), + gtd: gtd_new, + }); + done = true; + break; + } + + // gtd_new >=0 , there must be a local minimum in the interval. + if gtd_new >= 0.0 { + bracket = Some([ + LineSearchSample { + t: t_prev, + f: f_prev, + g: g_prev, + gtd: gtd_prev, + }, + LineSearchSample { + t, + f: f_new, + g: g_new.clone(), + gtd: gtd_new, + }, + ]); + break; + } + + // interpolate + let min_step = t + 0.01 * (t - t_prev); + let max_step = t * 10.0; + let t_next = cubic_interpolate( + t_prev, + f_prev, + gtd_prev, + t, + f_new, + gtd_new, + Some((min_step, max_step)), + ); + t_prev = t; + f_prev = f_new; + g_prev = g_new; + gtd_prev = gtd_new; + + // next step + t = t_next; + (f_new, g_new) = obj_func(x, t, d); + ls_func_evals += 1; + gtd_new = g_new.clone().dot(d.clone()).into_scalar().to_f64(); + ls_iter += 1; + } + if let Some(sample) = wolfe_bracket { + return (sample.f, sample.g, sample.t, ls_func_evals); + } + + let mut bracket = bracket.unwrap_or_else(|| { + [ + LineSearchSample { + t: 0.0, + f, + g: g.clone(), + gtd, + }, + LineSearchSample { + t, + f: f_new, + g: g_new.clone(), + gtd: gtd_new, + }, + ] + }); + + // zoom phase + let mut insuf_progress = false; + + // find high and low points in bracket + let (mut low_idx, mut high_idx) = if bracket[0].f <= bracket[1].f { + (0, 1) + } else { + (1, 0) + }; + + while !done && ls_iter < max_ls { + let diff = (bracket[1].t - bracket[0].t).abs(); + // line-search bracket is so small + if diff * d_norm < tolerance_change { + break; + } + + // compute new trial value + t = cubic_interpolate( + bracket[0].t, + bracket[0].f, + bracket[0].gtd, + bracket[1].t, + bracket[1].f, + bracket[1].gtd, + None, + ); + + let b_min = bracket[0].t.min(bracket[1].t); + let b_max = bracket[0].t.max(bracket[1].t); + let eps = 0.1 * (b_max - b_min); + + if (b_max - t).min(t - b_min) < eps { + // interpolation close to boundary + if insuf_progress || t >= b_max || t <= b_min { + t = if (t - b_max).abs() < (t - b_min).abs() { + b_max - eps + } else { + b_min + eps + }; + insuf_progress = false; + } else { + insuf_progress = true; + } + } else { + insuf_progress = false; + } + + // Evaluate new point + (f_new, g_new) = obj_func(x, t, d); + + ls_func_evals += 1; + gtd_new = g_new.clone().dot(d.clone()).into_scalar().to_f64(); + ls_iter += 1; + + let armijo_holds = f_new <= (f + c1 * t * gtd) && f_new < bracket[low_idx].f; + + if !armijo_holds { + bracket[high_idx] = LineSearchSample { + t, + f: f_new, + g: g_new, + gtd: gtd_new, + }; + } else { + if gtd_new.abs() <= -c2 * gtd { + return (f_new, g_new, t, ls_func_evals); + } + + if gtd_new * (bracket[high_idx].t - bracket[low_idx].t) >= 0.0 { + bracket[high_idx] = LineSearchSample { + t: bracket[low_idx].t, + f: bracket[low_idx].f, + g: bracket[low_idx].g.clone(), + gtd: bracket[low_idx].gtd, + }; + } + bracket[low_idx] = LineSearchSample { + t, + f: f_new, + g: g_new, + gtd: gtd_new, + }; + } + + if bracket[0].f <= bracket[1].f { + low_idx = 0; + high_idx = 1; + } else { + low_idx = 1; + high_idx = 0; + } + } + // return stuff + ( + bracket[low_idx].f, + bracket[low_idx].g.clone(), + bracket[low_idx].t, + ls_func_evals, + ) +} + +/// Strategy for the line search optimization phase +#[derive(Clone, Default, Debug, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LineSearchFn { + /// No line search performed + #[default] + None, + /// strong wolfe conditions + /// + /// See: + StrongWolfe, +} + +/// LBFGS Configuration. +#[derive(Config, Debug)] +pub struct LBFGSConfig { + /// Maximal number of iterations per optimization step (default: 20) + #[config(default = 20)] + pub max_iter: usize, + /// Update history size (default: 100). + #[config(default = 100)] + pub history_size: usize, + /// Termination tolerance on first order optimality (default: 1e-7). + #[config(default = 1e-7)] + pub tolerance_grad: f64, + /// Termination tolerance on function value/parameter changes (default: 1e-9). + #[config(default = 1e-9)] + pub tolerance_change: f64, + /// Maximal number of function evaluations per optimization step (default: max_iter * 1.25). + #[config(default = "None")] + pub max_eval: Option, + /// Either ‘strong_wolfe’ or None (default: None). + #[config(default = "LineSearchFn::None")] + pub line_search_fn: LineSearchFn, +} + +impl LBFGSConfig { + /// Initialize AdamW optimizer + /// + /// # Returns + /// + /// Returns an optimizer that can be used to optimize a module + pub fn init(&self) -> LBFGS { + // by default max_eval = max_iter * 5/4 + let max_eval = self.max_eval.unwrap_or(self.max_iter * 5 / 4); + LBFGS { + config: LBFGSConfig { + max_iter: self.max_iter, + history_size: self.history_size, + tolerance_grad: self.tolerance_grad, + tolerance_change: self.tolerance_change, + max_eval: Some(max_eval), + line_search_fn: self.line_search_fn, + }, + state: Default::default(), + } + } +} + +/// Collects gradients in module visit order. +struct FlattenGradsVisitorInner<'a, B: AutodiffBackend> { + grads: &'a GradientsParams, + tensors: &'a mut Vec>, +} + +impl ModuleVisitor for FlattenGradsVisitorInner<'_, B> { + fn visit_float(&mut self, param: &Param>) { + if let Some(g) = self.grads.get::(param.id) { + let numel = g.shape().num_elements(); + self.tensors.push(g.reshape([numel])); + } + } +} + +/// Flatten params to inner backend 1D tensor. +fn flatten_params_inner>( + module: &M, +) -> Tensor { + let mut tensors = Vec::new(); + let mut visitor = FlattenParamsVisitorInner:: { + tensors: &mut tensors, + }; + module.visit(&mut visitor); + if tensors.is_empty() { + return Tensor::empty([0], &module.devices()[0]); + } + Tensor::cat(tensors, 0) +} + +struct FlattenParamsVisitorInner<'a, B: AutodiffBackend> { + tensors: &'a mut Vec>, +} + +impl ModuleVisitor for FlattenParamsVisitorInner<'_, B> { + fn visit_float(&mut self, param: &Param>) { + let t = param.val().inner(); + let numel = t.shape().num_elements(); + self.tensors.push(t.reshape([numel])); + } +} + +/// Flatten gradients for a module. +fn flatten_grads_inner>( + module: &M, + grads: &GradientsParams, +) -> Tensor { + let mut tensors = Vec::new(); + let mut visitor = FlattenGradsVisitorInner { + grads, + tensors: &mut tensors, + }; + module.visit(&mut visitor); + if tensors.is_empty() { + return Tensor::empty([0], &module.devices()[0]); + } + Tensor::cat(tensors, 0) +} + +/// Mapper that assigns each float param from a flat inner-backend 1D tensor. +struct ParamsFromFlatMapperInner<'a, B: AutodiffBackend> { + flat: &'a Tensor, + offset: &'a mut usize, +} + +impl ParamsFromFlatMapperInner<'_, B> { + fn take_slice(&mut self, numel: usize) -> Tensor { + let start = *self.offset; + *self.offset += numel; + self.flat.clone().slice(start..*self.offset) + } +} + +impl ModuleMapper for ParamsFromFlatMapperInner<'_, B> { + fn map_float(&mut self, param: Param>) -> Param> { + let (id, tensor, mapper) = param.consume(); + let numel = tensor.shape().num_elements(); + let slice_1d = self.take_slice(numel); + let new_inner = slice_1d.reshape(tensor.shape()); + let new_tensor = Tensor::from_inner(new_inner).require_grad(); + Param::from_mapped_value(id, new_tensor, mapper) + } +} + +/// Overwrite module parameters from a flat inner-backend 1D tensor +fn set_params_from_flat_inner>( + module: M, + flat: Tensor, +) -> M { + let mut offset = 0; + let mut mapper = ParamsFromFlatMapperInner { + flat: &flat, + offset: &mut offset, + }; + module.map(&mut mapper) +} + +/// L-BFGS optimizer state +#[derive(Clone, Record)] +pub struct LBFGSState { + /// Historical displacement vectors + pub history_s: Vec>, + /// Historical gradient difference vectors + pub history_y: Vec>, + /// Search direction + pub d: Option>, + /// Step size from the previous iteration + pub t: Option, + /// Flattened gradient from the previous iteration + pub prev_flat_grad: Option>, + /// Loss value from the previous iteration + pub prev_loss: Option, + /// Global iteration count + pub g_iter: usize, +} + +impl LBFGSState { + /// Moves all historical tensors to the target device. + pub fn to_device(self, device: &B::Device) -> Self { + Self { + history_s: self + .history_s + .into_iter() + .map(|t| t.to_device(device)) + .collect(), + history_y: self + .history_y + .into_iter() + .map(|t| t.to_device(device)) + .collect(), + d: self.d.map(|t| t.to_device(device)), + t: self.t, + prev_flat_grad: self.prev_flat_grad.map(|t| t.to_device(device)), + prev_loss: self.prev_loss, + g_iter: self.g_iter, + } + } +} +impl Default for LBFGSState { + fn default() -> Self { + Self { + history_s: Vec::new(), + history_y: Vec::new(), + d: None, + t: Some(1.0), + prev_flat_grad: None, + prev_loss: None, + g_iter: 0, + } + } +} + +/// L-BFGS optimizer. +/// +/// Ported from [pytorch](https://github.com/pytorch/pytorch/torch/optim/lbfgs.py). Heavily inspired by [miniFunc](https://www.cs.ubc.ca/~schmidtm/Software/minFunc.html) +/// +/// See also: +/// - [L-BFGS](https://en.wikipedia.org/wiki/Limited-memory_BFGS) +/// +/// # Note +/// This optimizer is memory intensive +#[derive(Clone)] +pub struct LBFGS { + config: LBFGSConfig, + state: LBFGSState, +} + +impl LBFGS { + /// A single optimization step for any tensor that represents the parameters of a model. + pub fn step(&mut self, lr: LearningRate, mut module: M, mut closure: F) -> (M, f64) + where + M: AutodiffModule + Clone, + F: FnMut(M) -> (f64, GradientsParams), + { + // evaluate initial f(x) and df/dx + let (mut loss, grads) = closure(module.clone()); + let mut current_evals = 1; + + let mut flat_grad = flatten_grads_inner::(&module, &grads); + let mut x_flat = flatten_params_inner::(&module); + + let opt_cond = + flat_grad.clone().abs().max().into_scalar().to_f64() <= self.config.tolerance_grad; + // optimal condition + if opt_cond { + return (module, loss); + } + + // tensors cached in state + let mut d = self + .state + .d + .take() + .unwrap_or_else(|| flat_grad.clone().neg()); + let mut t = self.state.t.unwrap_or(lr); + let mut prev_flat_grad = self.state.prev_flat_grad.take(); + + let mut n_iter = 0; + + // optimize for a max of max_iter iterations + while n_iter < self.config.max_iter { + // keep track of nb of iterations + n_iter += 1; + self.state.g_iter += 1; + + // compute gradient descent direction + if self.state.g_iter == 1 { + d = flat_grad.clone().neg(); + self.state.history_s.clear(); + self.state.history_y.clear(); + } else { + // do lbfgs update (update memory) + if let Some(pg) = prev_flat_grad.as_ref() { + let y = flat_grad.clone().sub(pg.clone()); + let s = d.clone().mul_scalar(t); + + let ys = y.clone().dot(s.clone()).into_scalar().to_f64(); + + if ys > 1e-10 { + // updating memory + if self.state.history_s.len() >= self.config.history_size { + // shift history by one (limited-memory) + self.state.history_s.remove(0); + self.state.history_y.remove(0); + } + self.state.history_s.push(s); + self.state.history_y.push(y); + } + } + + // compute the approximate (L-BFGS) inverse Hessian + // multiplied by the gradient + let num_old = self.state.history_s.len(); + let mut q = flat_grad.clone().neg(); + let mut alphas: Vec> = + vec![Tensor::zeros([1], &flat_grad.device()); num_old]; + + if num_old > 0 { + // multiply by initial Hessian + // r/d is the final direction + for i in (0..num_old).rev() { + let s = &self.state.history_s[i]; + let y = &self.state.history_y[i]; + let rho = y.clone().dot(s.clone()).powf_scalar(-1.0); + let alpha = rho.clone().mul(s.clone().dot(q.clone())); + alphas[i] = alpha.clone(); + q = q.sub(y.clone().mul(alpha)); + } + + let last_s = &self.state.history_s[num_old - 1]; + let last_y = &self.state.history_y[num_old - 1]; + let ys = last_y.clone().dot(last_s.clone()); + let yy = last_y.clone().dot(last_y.clone()); + let h_diag = ys.div(yy); + + let mut r = q.mul(h_diag); + + for ((s, y), alpha) in self + .state + .history_s + .iter() + .zip(self.state.history_y.iter()) + .zip(alphas.into_iter()) + .take(num_old) + { + let rho = y.clone().dot(s.clone()).powf_scalar(-1.0); + + let beta = rho.mul(y.clone().dot(r.clone())); + + r = r.add(s.clone().mul(alpha.sub(beta))); + } + d = r; + } else { + d = q; + } + } + + prev_flat_grad = Some(flat_grad.clone()); + let prev_loss_iter = loss; + + // compute step len + if self.state.g_iter == 1 { + let grad_l1 = flat_grad.clone().abs().sum().into_scalar().to_f64(); + t = (1.0f64 / grad_l1).min(1.0) * lr; + } else { + t = lr; + } + + // directional derivative + let gtd = flat_grad.clone().dot(d.clone()).into_scalar().to_f64(); + + if gtd > -self.config.tolerance_change { + break; + } + + let ls_func_evals; + + if let LineSearchFn::StrongWolfe = self.config.line_search_fn { + // perform line search, using user function + let mut obj_func = + |current_x: &Tensor, + step: f64, + dir: &Tensor| { + let update = dir.clone().mul_scalar(step); + let new_x = current_x.clone().add(update); + let tmp_module = set_params_from_flat_inner::(module.clone(), new_x); + let (l, g) = closure(tmp_module); + (l, flatten_grads_inner::(&module, &g)) + }; + + let (ls_f, ls_g, ls_t, evals) = strong_wolfe( + &mut obj_func, + &x_flat, + t, + &d, + loss, + flat_grad.clone(), + gtd, + 1e-4, + 0.9, + self.config.tolerance_change, + self.config.max_eval.unwrap() - current_evals, + ); + + loss = ls_f; + flat_grad = ls_g; + t = ls_t; + ls_func_evals = evals; + + x_flat = x_flat.add(d.clone().mul_scalar(t)); + module = set_params_from_flat_inner::(module, x_flat.clone()); + } else { + // no line search, simply move with fixed-step + let step_vec = d.clone().mul_scalar(t); + x_flat = x_flat.add(step_vec); + module = set_params_from_flat_inner::(module, x_flat.clone()); + // re-evaluate function only if not in last iteration + // the reason we do this: in a stochastic setting, + // no use to re-evaluate that function here + let (new_loss, new_grads) = closure(module.clone()); + loss = new_loss; + flat_grad = flatten_grads_inner::(&module, &new_grads); + ls_func_evals = 1; + } + + // update func eval + current_evals += ls_func_evals; + + // check conditions + + if current_evals >= self.config.max_eval.unwrap() { + break; + } + + if flat_grad.clone().abs().max().into_scalar().to_f64() <= self.config.tolerance_grad { + break; + } + + if d.clone().mul_scalar(t).abs().max().into_scalar().to_f64() + <= self.config.tolerance_change + { + break; + } + + if (loss - prev_loss_iter).abs() < self.config.tolerance_change { + break; + } + } + self.state.d = Some(d); + self.state.t = Some(t); + self.state.prev_flat_grad = prev_flat_grad; + self.state.prev_loss = Some(loss); + (module, loss) + } + /// Moves the optimizer state to the specified device. + pub fn to_device(self, device: &B::Device) -> Self { + Self { + config: self.config, + // History tensors reside in InnerBackend, so we convert the device accordingly + state: self.state.to_device(device), + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::GradientsParams; + use crate::TestAutodiffBackend; + use burn::module::{Module, Param}; + use burn::tensor::{Tensor, TensorData}; + use burn_nn::{Linear, LinearConfig, LinearRecord}; + + fn given_linear_layer(weight: TensorData, bias: TensorData) -> Linear { + let device = Default::default(); + let record = LinearRecord { + weight: Param::from_data(weight, &device), + bias: Some(Param::from_data(bias, &device)), + }; + + LinearConfig::new(6, 6).init(&device).load_record(record) + } + #[test] + fn test_cubic_interpolate() { + let tolerance = 1e-8; + + // basic + let (x1, f1, g1, x2, f2, g2) = (-1.0, 1.0, -2.0, 1.0, 1.0, 2.0); + let result = cubic_interpolate(x1, f1, g1, x2, f2, g2, None); + assert!( + (result - 0.00000).abs() < tolerance, + "Basic: Result {} should be close to 0.0", + result + ); + + // bound + let (x1, f1, g1, x2, f2, g2) = (0.0, 0.25, -1.0, 1.0, 0.25, 1.0); + let bounds = Some((0.6, 1.0)); + let result = cubic_interpolate(x1, f1, g1, x2, f2, g2, bounds); + assert!( + (result - 0.6000000000).abs() < tolerance, + "Bound: Result {} should be clamped to 0.6", + result + ); + + // d2_square < 0,should return mid value + let (x1, f1, g1, x2, f2, g2) = (0.0, 0.0, 10.0, 1.0, 5.0, 10.0); + let result = cubic_interpolate(x1, f1, g1, x2, f2, g2, Some((0.0, 1.0))); + assert!( + (result - 0.5000000).abs() < tolerance, + "Fallback: Result {} should be midpoint 0.5", + result + ); + + // asymmetric + let (x1, f1, g1, x2, f2, g2) = (0.0, 1.0, -5.0, 1.0, 0.5, 1.0); + let result = cubic_interpolate(x1, f1, g1, x2, f2, g2, None); + assert!( + (result - 0.4606553370833684).abs() < tolerance, + "Asymmetric: Result {} should be 0.4606553370833684", + result + ); + + // not good value + let (x1, f1, g1, x2, f2, g2) = ( + 1.231232145, + -0.12567458754, + 9.1231243007, + 8.239105015, + -100.9012398021, + 123201321.0293982, + ); + let result_1 = cubic_interpolate(x1, f1, g1, x2, f2, g2, None); + let result_2 = cubic_interpolate(x1, f1, g1, x2, f2, g2, Some((-4.4, 4.4))); + assert!( + (result_1 - 5.9031480234724434).abs() < tolerance, + "not good value 1: Result {} should be 5.9031480234724434", + result + ); + assert!( + (result_2 - 4.4000000000000004).abs() < tolerance, + "not good value 2: Result {} should be 4.4000000000000004", + result + ); + } + #[test] + fn test_strong_wolfe_direct_comparison() { + let device = Default::default(); + let tol = 1e-8; + + { + let x = Tensor::::from_floats([2.1321912957_f64], &device); + let d = Tensor::::from_floats([0.91312321_f64], &device); + let t_initial = 1.213132_f64; + fn func( + x_base: &Tensor, + t_val: f64, + d_vec: &Tensor, + ) -> (f64, Tensor) { + let curr_x = x_base.clone().add(d_vec.clone().mul_scalar(t_val)); + let x2 = curr_x.clone().mul(curr_x.clone()); + let x3 = x2.clone().mul(curr_x.clone()); + let x4 = x2.clone().mul(x2.clone()); + + // f(x) = x^4 - 2*x^2 + x + let f_elements = x4 - x2.mul_scalar(2.0) + curr_x.clone(); + + let f_val = f_elements.sum().into_scalar().to_f64(); + + // g(x) = 4*x^3 - 4*x + 1 + let g = x3.mul_scalar(4.0) - curr_x.clone().mul_scalar(4.0) + + Tensor::ones_like(&curr_x); + + (f_val, g) + } + let (f_init, g_init) = func(&x, 0.0, &d); + let gtd_init = g_init.clone().dot(d.clone()).into_scalar().to_f64(); + println!("Initial State: f={},gtd = {}", f_init, gtd_init); + assert!((f_init - 13.7080059052).abs() < tol); + assert!((gtd_init - 28.5305728912).abs() < tol); + let mut obj_func = + |xb: &Tensor, + tv: f64, + dv: &Tensor| func(xb, tv, dv); + + let (f_final, _g_final, t_final, evals) = strong_wolfe( + &mut obj_func, + &x, + t_initial, + &d, + f_init, + g_init, + gtd_init, + 1e-4, // c1 + 0.9, // c2 + 1e-9, // tolerance_change + 10, // max_ls + ); + let g_f = _g_final.into_scalar().to_f64(); + println!( + "f_final:{:?},_g_final:{:?},t_final:{:?},evals:{:?}", + f_final, g_f, t_final, evals + ); + assert!((f_final - 13.708005905151367).abs() < tol); + assert!((g_f - 31.2450428009).abs() < tol); + assert!((t_final.to_f64() - 0.0).abs() < tol); + assert!((evals == 11)); + } + } + #[test] + fn test_lbfgs_strong_wolfe_comparison() { + let device = Default::default(); + let tol = 1e-5; + let x_data = Tensor::::from_data([[1.0], [2.0], [3.0]], &device); + let y_true = Tensor::::from_data([[3.0], [5.0], [7.0]], &device); + let weight = TensorData::from([[0.5f64]]); + let bias = TensorData::from([0.1f64]); + let module = given_linear_layer(weight, bias); + + let mut optimizer: LBFGS = LBFGSConfig::new() + .with_line_search_fn(LineSearchFn::StrongWolfe) + .init(); + let mut closure = |mod_in: Linear| { + let output = mod_in.forward(x_data.clone()); + let loss = burn_nn::loss::MseLoss::new().forward( + output, + y_true.clone(), + burn_nn::loss::Reduction::Sum, + ); + + let grads = loss.backward(); + let grads_params = GradientsParams::from_grads(grads, &mod_in); + + (loss.into_scalar().to_f64(), grads_params) + }; + let initial_loss = closure(module.clone()).0; + assert!((initial_loss - 50.1300048828).abs() < tol); + let (updated_module, final_loss) = optimizer.step(0.001, module, &mut closure); + assert!((final_loss - 0.0234732367).abs() < tol); + let optimized_data: f64 = updated_module.weight.val().into_scalar().to_f64(); + let optimized_bias: f64 = updated_module + .bias + .as_ref() + .unwrap() + .val() + .into_scalar() + .to_f64(); + assert!((optimized_data - 2.0570652485).abs() < tol); + assert!((optimized_bias - 0.8106800914).abs() < tol); + } + #[test] + fn test_lbfgs_no_strong_wolfe_comparison() { + let device = Default::default(); + let tol = 1e-5; + let x_data = Tensor::::from_data([[1.0], [2.0], [3.0]], &device); + let y_true = Tensor::::from_data([[3.0], [5.0], [7.0]], &device); + let weight = TensorData::from([[0.5f64]]); + let bias = TensorData::from([0.1f64]); + let module = given_linear_layer(weight, bias); + + let mut optimizer: LBFGS = LBFGSConfig::new() + .with_line_search_fn(LineSearchFn::None) + .init(); + let mut closure = |mod_in: Linear| { + let output = mod_in.forward(x_data.clone()); + let loss = burn_nn::loss::MseLoss::new().forward( + output, + y_true.clone(), + burn_nn::loss::Reduction::Sum, + ); + + let grads = loss.backward(); + let grads_params = GradientsParams::from_grads(grads, &mod_in); + + (loss.into_scalar().to_f64(), grads_params) + }; + let initial_loss = closure(module.clone()).0; + assert!((initial_loss - 50.1300048828).abs() < tol); + let (updated_module, final_loss) = optimizer.step(0.001, module, &mut closure); + assert!((final_loss - 48.2181930542).abs() < tol); + let optimized_data: f64 = updated_module.weight.val().into_scalar().to_f64(); + let optimized_bias: f64 = updated_module + .bias + .as_ref() + .unwrap() + .val() + .into_scalar() + .to_f64(); + + assert!((optimized_data - 0.5302446192).abs() < tol); + assert!((optimized_bias - 0.1142520783).abs() < tol); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/mod.rs new file mode 100644 index 0000000..4dfa9b2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/mod.rs @@ -0,0 +1,30 @@ +/// Weight decay module for optimizers. +pub mod decay; + +/// Momentum module for optimizers. +pub mod momentum; + +mod adagrad; +mod adam; +mod adamw; +mod base; +mod grad_accum; +mod grads; +mod lbfgs; +mod muon; +mod rmsprop; +mod sgd; +mod simple; +mod visitor; + +pub use adagrad::*; +pub use adam::*; +pub use adamw::*; +pub use base::*; +pub use grad_accum::*; +pub use grads::*; +pub use lbfgs::*; +pub use muon::*; +pub use rmsprop::*; +pub use sgd::*; +pub use simple::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/momentum.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/momentum.rs new file mode 100644 index 0000000..57cf9b8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/momentum.rs @@ -0,0 +1,94 @@ +use burn_core as burn; + +use burn::config::Config; +use burn::record::Record; +use burn::tensor::backend::Backend; +use burn::tensor::{ElementConversion, Tensor}; + +/// Configuration to create [momentum](Momentum). +#[derive(Config, Debug)] +pub struct MomentumConfig { + /// Momentum factor + #[config(default = 0.9)] + pub momentum: f64, + /// Dampening factor. + #[config(default = 0.1)] + pub dampening: f64, + /// Enables Nesterov momentum, see [On the importance of initialization and + /// momentum in deep learning](http://www.cs.toronto.edu/~hinton/absps/momentum.pdf). + #[config(default = false)] + pub nesterov: bool, +} + +/// State of [momentum](Momentum). +#[derive(Record, Clone, new)] +pub struct MomentumState { + velocity: Tensor, +} + +/// Momentum implementation that transforms gradients. +#[derive(Clone)] +pub struct Momentum { + momentum: B::FloatElem, + dampening: f64, + nesterov: bool, +} + +impl Momentum { + /// Creates a new [momentum](Momentum) from a [config](MomentumConfig). + pub fn new(config: &MomentumConfig) -> Self { + Self { + momentum: config.momentum.elem(), + dampening: config.dampening, + nesterov: config.nesterov, + } + } + + /// Transforms a gradient. + /// + /// # Arguments + /// + /// * `grad` - Gradient to transform. + /// * `state` - State of the optimizer. + /// + /// # Returns + /// + /// * `grad` - Transformed gradient. + /// * `state` - State of the optimizer. + pub fn transform( + &self, + grad: Tensor, + state: Option>, + ) -> (Tensor, MomentumState) { + let velocity = if let Some(state) = state { + grad.clone() + .mul_scalar(1.0 - self.dampening) + .add(state.velocity.mul_scalar(self.momentum)) + } else { + grad.clone() + }; + + let grad = match self.nesterov { + true => velocity.clone().mul_scalar(self.momentum).add(grad), + false => velocity.clone(), + }; + + (grad, MomentumState::new(velocity)) + } +} + +impl MomentumState { + /// Moves the state to a device. + /// + /// # Arguments + /// + /// * `device` - Device to move the state to. + /// + /// # Returns + /// + /// * `self` - Moved state. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.velocity = self.velocity.to_device(device); + self + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/muon.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/muon.rs new file mode 100644 index 0000000..4c8fa72 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/muon.rs @@ -0,0 +1,775 @@ +use burn_core as burn; + +use burn::{module::AutodiffModule, record::Record}; + +use burn::config::Config; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use burn::tensor::{backend::Backend, ops::Device}; +use serde::{Deserialize, Serialize}; + +use super::{ + SimpleOptimizer, + adaptor::OptimizerAdaptor, + decay::WeightDecayConfig, + momentum::{Momentum, MomentumConfig, MomentumState}, +}; +use crate::LearningRate; + +#[cfg(not(feature = "std"))] +#[allow(unused_imports)] +use num_traits::Float as _; + +/// Learning rate adjustment method for Muon optimizer. +/// +/// Muon adjusts the learning rate based on parameter shape to maintain consistent +/// RMS across rectangular matrices. +/// +/// # References +/// +/// - Original: [Muon: An optimizer for hidden layers](https://kellerjordan.github.io/posts/muon/) +/// - Moonshot: [Muon is Scalable for LLM Training](https://arxiv.org/pdf/2502.16982) +#[derive(Clone, Default, Debug, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum AdjustLrFn { + /// Keller Jordan's original method: `lr * sqrt(max(1, A/B))` + /// + /// This scales the learning rate based on the aspect ratio of the weight matrix, + /// ensuring that tall matrices (more rows than columns) get proportionally larger + /// learning rates. + /// + /// # Example + /// + /// For a [1024, 512] matrix: `lr * sqrt(1024/512) = lr * 1.414` + #[default] + Original, + + /// Moonshot's method: `lr * 0.2 * sqrt(max(A, B))` + /// + /// This method is designed to match AdamW's RMS, allowing Muon to directly reuse + /// learning rates and weight decay values tuned for AdamW without retuning. + /// + /// # Example + /// + /// For a [1024, 512] matrix: `lr * 0.2 * sqrt(1024) = lr * 6.4` + MatchRmsAdamW, +} + +impl AdjustLrFn { + /// Calculate the learning rate adjustment ratio for a given parameter shape. + /// + /// # Arguments + /// + /// * `shape` - Parameter shape (uses first two dimensions) + /// + /// # Returns + /// + /// Adjustment ratio to multiply with the base learning rate + fn adjustment_ratio(&self, shape: &[usize]) -> f64 { + if shape.len() < 2 { + return 1.0; + } + + let a = shape[0] as f64; + let b = shape[1] as f64; + + match self { + Self::Original => { + // sqrt(max(1, A/B)) + let ratio = a / b; + ratio.max(1.0).sqrt() + } + Self::MatchRmsAdamW => { + // 0.2 * sqrt(max(A, B)) + 0.2 * a.max(b).sqrt() + } + } + } +} + +/// Muon configuration. +/// +/// Muon is an optimizer specifically designed for 2D parameters of neural network +/// hidden layers (weight matrices). Other parameters such as biases and embeddings +/// should be optimized using a standard method such as AdamW. +/// +/// # Learning Rate Adjustment +/// +/// Muon adjusts the learning rate based on parameter shape to maintain consistent +/// RMS across rectangular matrices. Two methods are available: +/// +/// - **Original**: Uses `sqrt(max(1, A/B))` where A and B are the first two dimensions. +/// This is Keller Jordan's method and is the default. +/// +/// - **MatchRmsAdamW**: Uses `0.2 * sqrt(max(A, B))`. This is Moonshot's method +/// designed to match AdamW's RMS, allowing direct reuse of AdamW hyperparameters. +/// +/// # Example +/// +/// ```ignore +/// use burn_optim::{MuonConfig, AdjustLrFn}; +/// +/// // Using default (Original) method +/// let optimizer = MuonConfig::new().init(); +/// +/// // Using MatchRmsAdamW for AdamW-compatible hyperparameters +/// let optimizer = MuonConfig::new() +/// .with_adjust_lr_fn(AdjustLrFn::MatchRmsAdamW) +/// .init(); +/// ``` +/// +/// # References +/// +/// - [Muon: An optimizer for hidden layers in neural networks](https://kellerjordan.github.io/posts/muon/) +/// - [Muon is Scalable for LLM Training](https://arxiv.org/pdf/2502.16982) +/// - [PyTorch Implementation](https://github.com/pytorch/pytorch/blob/main/torch/optim/muon.py) +/// - [Original Implementation](https://github.com/KellerJordan/Muon) +#[derive(Config, Debug)] +pub struct MuonConfig { + /// [Weight decay](WeightDecayConfig) config. + weight_decay: Option, + + /// [Momentum](MomentumConfig) config. + /// + /// Muon always uses momentum. Default configuration: + /// - momentum: 0.95 + /// - dampening: 0.0 + /// - nesterov: true + #[config(default = "MomentumConfig { momentum: 0.95, dampening: 0.0, nesterov: true }")] + momentum: MomentumConfig, + + /// Newton-Schulz iteration coefficients (a, b, c). + /// + /// These coefficients are selected to maximize the slope at zero for the + /// quintic iteration. Default values are from Keller Jordan's implementation. + #[config(default = "(3.4445, -4.775, 2.0315)")] + ns_coefficients: (f32, f32, f32), + + /// Epsilon for numerical stability. + #[config(default = 1e-7)] + epsilon: f32, + + /// Number of Newton-Schulz iteration steps. + #[config(default = 5)] + ns_steps: usize, + + /// Learning rate adjustment method. + /// + /// Controls how the learning rate is adjusted based on parameter shape. + /// See [`AdjustLrFn`] for available methods. + #[config(default = "AdjustLrFn::Original")] + adjust_lr_fn: AdjustLrFn, +} + +impl MuonConfig { + /// Initialize Muon optimizer. + /// + /// # Returns + /// + /// Returns an optimizer adaptor that can be used to optimize a module. + /// + /// # Example + /// + /// ```ignore + /// use burn_optim::{MuonConfig, AdjustLrFn, decay::WeightDecayConfig}; + /// + /// // Basic configuration with default (Original) LR adjustment + /// let optimizer = MuonConfig::new() + /// .with_weight_decay(Some(WeightDecayConfig::new(0.01))) + /// .init(); + /// + /// // With AdamW-compatible settings using MatchRmsAdamW + /// let optimizer = MuonConfig::new() + /// .with_adjust_lr_fn(AdjustLrFn::MatchRmsAdamW) + /// .with_weight_decay(Some(WeightDecayConfig::new(0.1))) + /// .init(); + /// + /// // Custom momentum and NS settings + /// let optimizer = MuonConfig::new() + /// .with_momentum(MomentumConfig { + /// momentum: 0.9, + /// dampening: 0.1, + /// nesterov: false, + /// }) + /// .with_ns_steps(7) + /// .init(); + /// ``` + pub fn init>( + &self, + ) -> OptimizerAdaptor, M, B> { + let momentum = Momentum::new(&self.momentum); + let weight_decay_penalty = self.weight_decay.as_ref().map(|wd| wd.penalty); + + let optim = Muon { + momentum, + ns_params: NewtonSchulzParams::new(self.ns_coefficients, self.ns_steps), + weight_decay_penalty, + epsilon: self.epsilon, + adjust_lr_fn: self.adjust_lr_fn, + }; + + OptimizerAdaptor::from(optim) + } +} + +/// Parameters for Newton-Schulz orthogonalization. +#[derive(Clone, Copy)] +struct NewtonSchulzParams { + a: f32, + b: f32, + c: f32, + steps: usize, +} + +impl NewtonSchulzParams { + fn new(coefficients: (f32, f32, f32), steps: usize) -> Self { + Self { + a: coefficients.0, + b: coefficients.1, + c: coefficients.2, + steps, + } + } +} + +/// Muon optimizer. +/// +/// Muon internally runs standard SGD-momentum, and then performs an orthogonalization +/// post-processing step, in which each 2D parameter's update is replaced with the +/// nearest orthogonal matrix. For efficient orthogonalization we use a Newton-Schulz +/// iteration, which has the advantage that it can be stably run in bfloat16 on the GPU. +/// +/// # Important Notes +/// +/// 1. **Only for 2D+ parameters**: Muon is designed for weight matrices. Use AdamW +/// or SGD for biases, embeddings, and layer norms. +/// +/// 2. **Learning rate adjustment**: Muon automatically adjusts the learning rate based +/// on parameter shape. See [`AdjustLrFn`] for details. +/// +/// 3. **Weight decay timing**: Unlike typical optimizers, Muon applies weight decay +/// AFTER orthogonalization but uses the original (unadjusted) learning rate for it. +#[derive(Clone)] +pub struct Muon { + momentum: Momentum, + ns_params: NewtonSchulzParams, + weight_decay_penalty: Option, + epsilon: f32, + adjust_lr_fn: AdjustLrFn, +} + +impl Muon { + /// Adjust learning rate based on parameter shape. + /// + /// # Arguments + /// + /// * `lr` - Base learning rate + /// * `shape` - Parameter shape (uses first two dimensions) + /// + /// # Returns + /// + /// Adjusted learning rate + /// + /// ```ignore + /// // For a [1024, 512] weight matrix with lr=0.01: + /// // Original: 0.01 * sqrt(1024/512) = 0.01 * 1.414 = 0.01414 + /// // MatchRmsAdamW: 0.01 * 0.2 * sqrt(1024) = 0.01 * 0.2 * 32 = 0.064 + /// ``` + fn adjust_lr(&self, lr: LearningRate, shape: &[usize]) -> LearningRate { + lr * self.adjust_lr_fn.adjustment_ratio(shape) + } + + /// Perform Newton-Schulz orthogonalization on a gradient tensor. + /// + /// This computes the zeroth power (orthogonalization) of the input matrix G + /// using a quintic Newton-Schulz iteration. + /// + /// # Algorithm + /// + /// 1. Transpose if tall matrix (A > B) + /// 2. Normalize: X = X / ||X|| + /// 3. For k steps: + /// - A = X @ X^T + /// - B = b*A + c*A^2 + /// - X = a*X + B@X + /// 4. Transpose back if needed + /// + /// # References + /// + /// - Original: https://github.com/KellerJordan/Muon/blob/master/muon.py + /// - PyTorch: https://github.com/pytorch/pytorch/blob/main/torch/optim/muon.py + fn zeropower_via_newtonschulz(&self, g: Tensor) -> Tensor { + let shape = g.shape(); + let dim_m2 = shape[D - 2]; + let dim_m1 = shape[D - 1]; + + // Step 1: Transpose if tall matrix (more rows than columns) + let (mut x, needs_transpose) = if dim_m2 > dim_m1 { + (g.swap_dims(D - 2, D - 1), true) + } else { + (g, false) + }; + + // Step 2: Normalize by Frobenius norm + // X = X / (||X|| + epsilon) + let norm = x + .clone() + .powf_scalar(2.0) + .sum() + .sqrt() + .clamp_min(self.epsilon) + .unsqueeze(); + + x = x.div(norm); + + // Step 3: Newton-Schulz iteration + // This is the quintic iteration with coefficients (a, b, c) + let NewtonSchulzParams { a, b, c, steps } = self.ns_params; + + for _ in 0..steps { + // A = X @ X^T + let x_t = x.clone().swap_dims(D - 2, D - 1); + let a_matrix = x.clone().matmul(x_t); + + // B = b*A + c*A@A + let a_squared = a_matrix.clone().matmul(a_matrix.clone()); + let b_matrix = a_matrix.mul_scalar(b).add(a_squared.mul_scalar(c)); + + // X = a*X + B@X + x = x.clone().mul_scalar(a).add(b_matrix.matmul(x.clone())); + } + + // Step 4: Restore transpose if it was a tall matrix + if needs_transpose { + x = x.swap_dims(D - 2, D - 1); + } + + x + } +} + +/// Muon state. +#[derive(Record, Clone, new)] +pub struct MuonState { + /// Current momentum state + pub momentum: MomentumState, +} + +impl SimpleOptimizer for Muon { + type State = MuonState; + + /// Perform a single Muon optimization step. + /// + /// # Algorithm + /// + /// 1. Apply momentum to gradient + /// 2. Orthogonalize update via Newton-Schulz + /// 3. Adjust learning rate based on parameter shape + /// 4. Apply weight decay (using original lr) + /// 5. Update parameter (using adjusted lr) + /// + /// # Notes + /// + /// Unlike typical optimizers, the weight decay and parameter update use + /// different learning rates: + /// - Weight decay uses the original `lr` + /// - Parameter update uses the shape-adjusted `lr` + /// + /// # Panics + /// This function will panic if the input tensors are not 2D. + fn step( + &self, + lr: LearningRate, + tensor: Tensor, + grad: Tensor, + state: Option>, + ) -> (Tensor, Option>) { + assert!( + D == 2, + "Newton-Schulz iteration requires 2D tensors, got {}D", + D + ); + + // Step 1: Apply momentum + let state_momentum = state.map(|s| s.momentum); + let (grad, new_momentum_state) = self.momentum.transform(grad, state_momentum); + + // Step 2: Orthogonalize via Newton-Schulz + let update = self.zeropower_via_newtonschulz(grad); + + // Step 3: Adjust learning rate based on parameter shape + let adjusted_lr = self.adjust_lr(lr, &tensor.shape()); + + // Step 4: Apply weight decay (using ORIGINAL lr, not adjusted) + // Muon applies weight decay AFTER orthogonalization + let tensor = if let Some(penalty) = self.weight_decay_penalty { + let decay_factor = 1.0 - lr * penalty as f64; + tensor.mul_scalar(decay_factor) + } else { + tensor + }; + + // Step 5: Update parameter (using ADJUSTED lr) + let delta = update.mul_scalar(adjusted_lr); + let new_state = MuonState::new(new_momentum_state); + + (tensor - delta, Some(new_state)) + } + + fn to_device(mut state: Self::State, device: &Device) -> Self::State { + state.momentum = state.momentum.to_device(device); + state + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestAutodiffBackend; + use crate::{GradientsParams, Optimizer}; + use burn::module::{Module, Param}; + use burn::tensor::{Distribution, Tensor, TensorData}; + use burn_nn::{Linear, LinearConfig, LinearRecord}; + + type TestBackend = burn_ndarray::NdArray; + + const TOLERANCE: f64 = 1e-8; + + fn given_linear_layer_no_bias(weight: TensorData) -> Linear { + let device = Default::default(); + let record = LinearRecord { + weight: Param::from_data(weight, &device), + bias: None, //No bias for Muon optimizer + }; + + LinearConfig::new(4, 4) + .with_bias(false) + .init(&device) + .load_record(record) + } + + #[test] + fn test_adjust_lr_fn_original() { + let method = AdjustLrFn::Original; + + // Square matrix [512, 512] -> sqrt(1) = 1.0 + let ratio = method.adjustment_ratio(&[512, 512]); + assert!((ratio - 1.0).abs() < TOLERANCE); + + // Tall matrix [1024, 512] -> sqrt(2) ≈ 1.414 + let ratio = method.adjustment_ratio(&[1024, 512]); + let expected = (2.0f64).sqrt(); + assert!((ratio - expected).abs() < TOLERANCE); + + // Wide matrix [512, 1024] -> max(1, 0.5) = 1.0 + let ratio = method.adjustment_ratio(&[512, 1024]); + assert!((ratio - 1.0).abs() < TOLERANCE); + } + + #[test] + fn test_adjust_lr_fn_match_rms_adamw() { + let method = AdjustLrFn::MatchRmsAdamW; + + // [1024, 512] -> 0.2 * sqrt(1024) = 6.4 + let ratio = method.adjustment_ratio(&[1024, 512]); + let expected = 0.2 * 1024.0f64.sqrt(); + assert!((ratio - expected).abs() < TOLERANCE); + + // [512, 512] -> 0.2 * sqrt(512) ≈ 4.525 + let ratio = method.adjustment_ratio(&[512, 512]); + let expected = 0.2 * 512.0f64.sqrt(); + assert!((ratio - expected).abs() < TOLERANCE); + } + + #[test] + #[should_panic(expected = "Newton-Schulz iteration requires 2D tensors, got 1D")] + fn test_1d_tensor_panics() { + let device = Default::default(); + let config = MuonConfig::new(); + let optim: Muon = Muon { + momentum: Momentum::new(&config.momentum), + ns_params: NewtonSchulzParams::new(config.ns_coefficients, config.ns_steps), + weight_decay_penalty: None, + epsilon: config.epsilon, + adjust_lr_fn: config.adjust_lr_fn, + }; + + let tensor_1d = Tensor::::zeros([512], &device); + let grad_1d = Tensor::::ones([512], &device); + + let _ = optim.step(0.01, tensor_1d, grad_1d, None); + } + + #[test] + fn test_muon_optimizer_save_load_state() { + let device = Default::default(); + // Use Linear layer WITHOUT bias for Muon optimizer + let linear = LinearConfig::new(6, 6) + .with_bias(false) // No bias - only 2D weight matrix + .init::(&device); + + let x = Tensor::::random([2, 6], Distribution::Default, &device); + + let mut optimizer = + MuonConfig::new().init::>(); + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let _linear = optimizer.step(0.01, linear, grads); + + let state_before = optimizer.to_record(); + let state_before_copy = optimizer.to_record(); + + let optimizer_new = + MuonConfig::new().init::>(); + let optimizer_loaded = optimizer_new.load_record(state_before_copy); + let state_after = optimizer_loaded.to_record(); + + assert_eq!(state_before.len(), state_after.len()); + } + + #[test] + fn test_muon_with_weight_decay() { + let device = Default::default(); + // Create Linear layer WITHOUT bias for Muon + let linear = given_linear_layer_no_bias(TensorData::from([ + [1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + ])); + + let x = Tensor::::from_floats( + [[0.5, 0.5, 0.5, 0.5], [0.5, 0.5, 0.5, 0.5]], + &device, + ) + .require_grad(); + + let mut optimizer = MuonConfig::new() + .with_weight_decay(Some(WeightDecayConfig::new(0.01))) + .init::>(); + + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(0.01, linear, grads); + + let state = linear.into_record(); + let weight = state.weight.to_data(); + + for val in weight.as_slice::().unwrap() { + assert!( + *val < 1.0, + "Weight should be reduced by weight decay, got {}", + val + ); + } + } + + #[test] + fn test_newton_schulz_orthogonalization() { + let device = Default::default(); + let matrix = Tensor::::from_floats([[1.0, 0.5], [0.5, 1.0]], &device); + + let config = MuonConfig::new(); + let muon: Muon = Muon { + momentum: Momentum::new(&config.momentum), + ns_params: NewtonSchulzParams::new(config.ns_coefficients, config.ns_steps), + weight_decay_penalty: None, + epsilon: config.epsilon, + adjust_lr_fn: config.adjust_lr_fn, + }; + + let orthogonalized = muon.zeropower_via_newtonschulz(matrix); + let o_t = orthogonalized.clone().transpose(); + let product = orthogonalized.matmul(o_t); + + let data = product.into_data(); + let values = data.as_slice::().unwrap(); + + assert!( + (values[0] - 1.0).abs() < 0.1, + "Product[0,0] should be ~1.0, got {}", + values[0] + ); + assert!( + (values[3] - 1.0).abs() < 0.1, + "Product[1,1] should be ~1.0, got {}", + values[3] + ); + } + + #[test] + fn test_tall_matrix_transpose() { + // Test that tall matrices (A > B) are transposed during Newton-Schulz iteration + // and then transposed back + let device = Default::default(); + + // Create a tall matrix: [8, 4] (more rows than columns) + let tall_matrix = Tensor::::from_floats( + [ + [1.0, 0.5, 0.3, 0.2], + [0.5, 1.0, 0.4, 0.1], + [0.3, 0.4, 1.0, 0.5], + [0.2, 0.1, 0.5, 1.0], + [0.1, 0.2, 0.3, 0.4], + [0.4, 0.3, 0.2, 0.1], + [0.2, 0.4, 0.1, 0.3], + [0.3, 0.1, 0.4, 0.2], + ], + &device, + ); + + let config = MuonConfig::new(); + let muon: Muon = Muon { + momentum: Momentum::new(&config.momentum), + ns_params: NewtonSchulzParams::new(config.ns_coefficients, config.ns_steps), + weight_decay_penalty: None, + epsilon: config.epsilon, + adjust_lr_fn: config.adjust_lr_fn, + }; + + // Perform Newton-Schulz orthogonalization + let orthogonalized = muon.zeropower_via_newtonschulz(tall_matrix.clone()); + + // Verify shape is preserved (should be transposed internally but returned in original shape) + let original_shape = tall_matrix.shape(); + let result_shape = orthogonalized.shape(); + assert_eq!( + original_shape.dims::<2>(), + result_shape.dims::<2>(), + "Shape should be preserved: [8, 4]" + ); + + // Verify output is different from input (orthogonalization happened) + let original_data = tall_matrix.into_data(); + let result_data = orthogonalized.into_data(); + assert_ne!( + original_data.as_slice::().unwrap(), + result_data.as_slice::().unwrap(), + "Orthogonalized matrix should differ from input" + ); + + // For comparison, test a wide matrix [4, 8] should NOT be transposed + let wide_matrix = Tensor::::from_floats( + [ + [1.0, 0.5, 0.3, 0.2, 0.1, 0.4, 0.2, 0.3], + [0.5, 1.0, 0.4, 0.1, 0.2, 0.3, 0.4, 0.1], + [0.3, 0.4, 1.0, 0.5, 0.3, 0.2, 0.1, 0.4], + [0.2, 0.1, 0.5, 1.0, 0.4, 0.1, 0.3, 0.2], + ], + &device, + ); + + let orthogonalized_wide = muon.zeropower_via_newtonschulz(wide_matrix.clone()); + + // Verify wide matrix shape is also preserved + let wide_original_shape = wide_matrix.shape(); + let wide_result_shape = orthogonalized_wide.shape(); + assert_eq!( + wide_original_shape.dims::<2>(), + wide_result_shape.dims::<2>(), + "Wide matrix shape should be preserved: [4, 8]" + ); + } + + #[test] + fn test_zero_gradient() { + // Test that Muon handles zero gradients gracefully + let device = Default::default(); + + let tensor = Tensor::::from_floats( + [ + [1.0, 0.5, 0.3, 0.2], + [0.5, 1.0, 0.4, 0.1], + [0.3, 0.4, 1.0, 0.5], + [0.2, 0.1, 0.5, 1.0], + ], + &device, + ); + + // Zero gradient - all zeros + let zero_grad = Tensor::::zeros([4, 4], &device); + + let config = MuonConfig::new(); + let muon: Muon = Muon { + momentum: Momentum::new(&config.momentum), + ns_params: NewtonSchulzParams::new(config.ns_coefficients, config.ns_steps), + weight_decay_penalty: None, + epsilon: config.epsilon, + adjust_lr_fn: config.adjust_lr_fn, + }; + + // Should not panic or produce NaN + let (updated_tensor, state) = muon.step(0.01, tensor.clone(), zero_grad, None); + + // Verify state was created + assert!(state.is_some()); + + // With zero gradient and no weight decay, tensor should remain unchanged + let original_data = tensor.into_data(); + let updated_data = updated_tensor.clone().into_data(); + + let original_vals = original_data.as_slice::().unwrap(); + let updated_vals = updated_data.as_slice::().unwrap(); + + for (orig, upd) in original_vals.iter().zip(updated_vals.iter()) { + assert!( + (orig - upd).abs() < 1e-6, + "With zero gradient, tensor should remain unchanged (or very close)" + ); + } + + // Verify no NaN values + for val in updated_vals { + assert!( + !val.is_nan(), + "Result should not contain NaN values with zero gradient" + ); + } + + // Test with weight decay - should still work + let muon_with_decay: Muon = Muon { + momentum: Momentum::new(&config.momentum), + ns_params: NewtonSchulzParams::new(config.ns_coefficients, config.ns_steps), + weight_decay_penalty: Some(0.01), + epsilon: config.epsilon, + adjust_lr_fn: config.adjust_lr_fn, + }; + + let tensor2 = Tensor::::from_floats( + [ + [1.0, 0.5, 0.3, 0.2], + [0.5, 1.0, 0.4, 0.1], + [0.3, 0.4, 1.0, 0.5], + [0.2, 0.1, 0.5, 1.0], + ], + &device, + ); + let zero_grad2 = Tensor::::zeros([4, 4], &device); + + let (updated_tensor_decay, _) = + muon_with_decay.step(0.01, tensor2.clone(), zero_grad2, None); + + // With zero gradient but with weight decay, tensor should be slightly reduced + let updated_decay_data = updated_tensor_decay.into_data(); + let updated_decay_vals = updated_decay_data.as_slice::().unwrap(); + + for val in updated_decay_vals { + assert!( + !val.is_nan(), + "Result should not contain NaN with zero gradient and weight decay" + ); + } + + // With weight decay, values should be slightly smaller than original + let original_vals2 = tensor2.into_data().as_slice::().unwrap().to_vec(); + for (orig, upd) in original_vals2.iter().zip(updated_decay_vals.iter()) { + if orig.abs() > 1e-6 { + // Non-zero values should be reduced by weight decay + assert!( + upd.abs() < orig.abs(), + "Weight decay should reduce magnitude: original={}, updated={}", + orig, + upd + ); + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/rmsprop.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/rmsprop.rs new file mode 100644 index 0000000..99f2a59 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/rmsprop.rs @@ -0,0 +1,566 @@ +use burn_core as burn; + +use burn::{module::AutodiffModule, record::Record}; + +use super::{ + SimpleOptimizer, + adaptor::OptimizerAdaptor, + decay::{WeightDecay, WeightDecayConfig}, +}; +use crate::{LearningRate, grad_clipping::GradientClippingConfig}; + +use burn::config::Config; +use burn::tensor::backend::Backend; +use burn::tensor::{Tensor, backend::AutodiffBackend, ops::Device}; + +/// Configuration to create the [RmsProp](RmsProp) optimizer. +#[derive(Config, Debug)] +pub struct RmsPropConfig { + /// Smoothing constant. + #[config(default = 0.99)] + alpha: f32, + /// momentum for RmsProp. + #[config(default = 0.9)] + momentum: f32, + /// A value required for numerical stability. + #[config(default = 1e-5)] + epsilon: f32, + /// if True, compute the centered RmsProp, the gradient is normalized by an estimation of its variance + #[config(default = false)] + centered: bool, + /// [Weight decay](WeightDecayConfig) config. + weight_decay: Option, + /// [Gradient Clipping](GradientClippingConfig) config. + grad_clipping: Option, +} + +impl RmsPropConfig { + /// Initialize RmsProp optimizer. + /// + /// # Returns + /// + /// Returns an optimizer that can be used to optimize a module. + pub fn init>( + &self, + ) -> OptimizerAdaptor { + let weight_decay = self.weight_decay.as_ref().map(WeightDecay::new); + + let mut optim = OptimizerAdaptor::from(RmsProp { + alpha: self.alpha, + centered: self.centered, + weight_decay, + momentum: RmsPropMomentum { + momentum: self.momentum, + epsilon: self.epsilon, + }, + }); + + if let Some(config) = &self.grad_clipping { + optim = optim.with_grad_clipping(config.init()); + } + + optim + } +} + +/// Optimizer that implements stochastic gradient descent with momentum. +/// The optimizer can be configured with [RmsPropConfig](RmsPropConfig). +#[derive(Clone)] +pub struct RmsProp { + alpha: f32, + // epsilon: f32, + centered: bool, + // momentum: Option>, + momentum: RmsPropMomentum, + weight_decay: Option, +} + +impl SimpleOptimizer for RmsProp { + type State = RmsPropState; + + fn step( + &self, + lr: LearningRate, + tensor: Tensor, + mut grad: Tensor, + state: Option>, + ) -> (Tensor, Option>) { + // fetch state for params + let mut state_square_avg = None; + let mut state_centered = None; + let mut state_momentum = None; + if let Some(state) = state { + state_square_avg = Some(state.square_avg); + state_centered = Some(state.centered); + state_momentum = state.momentum; + } + + // weight_decay transform + if let Some(weight_decay) = &self.weight_decay { + grad = weight_decay.transform(grad, tensor.clone()); + } + + // square_avg transform + let (grad, state_square_avg) = + SquareAvgState::transform(self.alpha, grad, state_square_avg); + + // centered transform + let (grad, state_square_avg, state_centered) = CenteredState::transform( + self.alpha, + self.centered, + grad, + state_square_avg, + state_centered, + ); + + // momentum transform + let (grad, state_centered, state_momentum) = + self.momentum + .transform(grad, state_centered, state_momentum); + + // transition state + let state = RmsPropState::new(state_square_avg, state_centered, state_momentum); + + // tensor param transform + let delta = grad.mul_scalar(lr); + (tensor - delta, Some(state)) + } + + fn to_device(mut state: Self::State, device: &Device) -> Self::State { + state.square_avg = state.square_avg.to_device(device); + state.centered = state.centered.to_device(device); + state.momentum = state.momentum.map(|momentum| momentum.to_device(device)); + state + } +} + +/// State of [RmsProp](RmsProp) +#[derive(Record, Clone, new)] +pub struct RmsPropState { + /// Current squared average state. + pub square_avg: SquareAvgState, + /// Current centered state + pub centered: CenteredState, + /// Current gradient momentum, if any. + pub momentum: Option>, +} + +/// [SquareAvgState](SquareAvgState) is to store and pass optimizer step params. +#[derive(Record, Clone, new)] +pub struct SquareAvgState { + /// Current squared average. + pub square_avg: Tensor, +} + +impl SquareAvgState { + /// transform [SquareAvgState] to the next step + fn transform(alpha: f32, grad: Tensor, state: Option) -> (Tensor, Self) { + match state { + Some(state) => { + let square_avg = state + .square_avg + .mul_scalar(alpha) + .add(grad.clone().square().mul_scalar(1. - alpha)); + (grad, Self { square_avg }) + } + _ => { + let square_avg = grad.clone().square().mul_scalar(1. - alpha); + (grad, Self { square_avg }) + } + } + } + + /// Moves the state to a device. + /// + /// # Arguments + /// + /// * `device` - Device to move the state to. + /// + /// # Returns + /// + /// * `self` - Moved state. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.square_avg = self.square_avg.to_device(device); + self + } +} + +/// [CenteredState](CenteredState) is to store and pass optimizer step params. +#[derive(Record, Clone, new)] +pub struct CenteredState { + /// The averaged gradient to calculate the centered gradient, if available. + pub grad_avg: Option>, + /// The current average value. + pub avg: Tensor, +} + +impl CenteredState { + /// transform [CenteredState] to the next step + fn transform( + alpha: f32, + centered: bool, + grad: Tensor, + square_avg_state: SquareAvgState, + centered_state: Option, + ) -> (Tensor, SquareAvgState, Self) { + if centered { + let grad_avg_constant = grad.clone().mul_scalar(1. - alpha); + let grad_avg = match centered_state { + Some(state) => state + .grad_avg + .map_or(grad_avg_constant.clone(), move |grad_avg| { + grad_avg.mul_scalar(alpha).add(grad_avg_constant) + }), + _ => grad_avg_constant, + }; + let avg = square_avg_state + .square_avg + .clone() + .sub(grad_avg.clone().square()); + + ( + grad, + square_avg_state, + Self { + grad_avg: Some(grad_avg), + avg, + }, + ) + } else { + ( + grad, + square_avg_state.clone(), + Self { + grad_avg: None, + avg: square_avg_state.square_avg, + }, + ) + } + } + + /// Moves the state to a device. + /// + /// # Arguments + /// + /// * `device` - Device to move the state to. + /// + /// # Returns + /// + /// * `self` - Moved state. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.grad_avg = self.grad_avg.map(|grad_avg| grad_avg.to_device(device)); + self.avg = self.avg.to_device(device); + self + } +} + +/// [RmsPropMomentum](RmsPropMomentum) is to store config status for optimizer. +/// (, which is stored in [optimizer](RmsProp) itself and not passed in during `step()` calculation) +#[derive(Clone)] +pub struct RmsPropMomentum { + momentum: f32, + epsilon: f32, +} + +impl RmsPropMomentum { + /// transform [grad](Tensor) and [RmsPropMomentumState] to the next step + fn transform( + &self, + grad: Tensor, + centered_state: CenteredState, + momentum_state: Option>, + ) -> ( + Tensor, + CenteredState, + Option>, + ) { + let grad = grad.div(centered_state.avg.clone().sqrt().add_scalar(self.epsilon)); + + if self.momentum > 0. { + let buf = match momentum_state { + Some(state) => state.buf.mul_scalar(self.momentum).add(grad), + _ => grad, + }; + ( + buf.clone(), + centered_state, + Some(RmsPropMomentumState { buf }), + ) + } else { + (grad, centered_state, None) + } + } +} + +/// [RmsPropMomentumState](RmsPropMomentumState) is to store and pass optimizer step params. +#[derive(Record, Clone, new)] +pub struct RmsPropMomentumState { + buf: Tensor, +} + +impl RmsPropMomentumState { + /// Moves the state to a device. + /// + /// # Arguments + /// + /// * `device` - Device to move the state to. + /// + /// # Returns + /// + /// * `self` - Moved state. + pub fn to_device(mut self, device: &B::Device) -> Self { + self.buf = self.buf.to_device(device); + self + } +} + +#[cfg(test)] +mod tests { + use burn::tensor::ops::FloatElem; + use burn::tensor::{Shape, Tolerance}; + + use super::*; + use crate::TestAutodiffBackend; + use crate::optim::{GradientsParams, Optimizer}; + use burn::module::{Module, Param}; + use burn::tensor::{Distribution, Tensor, TensorData}; + use burn_nn::{Linear, LinearConfig, LinearRecord}; + + type FT = FloatElem; + + const LEARNING_RATE: LearningRate = 0.01; + + #[test] + fn test_rmsprop_optimizer_save_load_state() { + let device = Default::default(); + let linear = LinearConfig::new(6, 6).init(&device); + let x = Tensor::::random([2, 6], Distribution::Default, &device); + let mut optimizer = create_rmsprop(); + let grads = linear.forward(x).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let _linear = optimizer.step(LEARNING_RATE, linear, grads); + + #[cfg(feature = "std")] + { + use burn::record::{BinFileRecorder, FullPrecisionSettings, Recorder}; + + BinFileRecorder::::default() + .record( + optimizer.to_record(), + std::env::temp_dir().as_path().join("test_optim_rmsprop"), + ) + .unwrap(); + } + #[cfg(not(feature = "std"))] + { + use burn::record::{BinBytesRecorder, FullPrecisionSettings, Recorder}; + + let result = BinBytesRecorder::::default() + .record(optimizer.to_record(), ()) + .unwrap(); + assert!(!result.is_empty()); + } + + let state_optim_before = optimizer.to_record(); + let state_optim_before_copy = optimizer.to_record(); + let optimizer = create_rmsprop(); + let optimizer = optimizer.load_record(state_optim_before_copy); + let state_optim_after = optimizer.to_record(); + + assert_eq!(state_optim_before.len(), state_optim_after.len()); + } + + /// used for test differences and debug + #[test] + fn test_rmsprop_optimizer_with_numbers_basic() { + let linear = given_linear_layer( + TensorData::from([ + [1., 1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1., 1.], + ]), + TensorData::from([0.5, 0.5, 0.5, 0.5, 0.5, 0.5]), + ); + let device = Default::default(); + let x_1 = Tensor::::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &device, + ) + .require_grad(); + let x_2 = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &device, + ) + .require_grad(); + + let mut optimizer = RmsPropConfig::new() + .with_alpha(0.99) + .with_epsilon(1e-8) + .with_weight_decay(WeightDecayConfig::new(0.05).into()) + .with_momentum(0.9) + .with_centered(false) + .init(); + + // println!("linear is {:?}", linear); + let grads = linear.forward(x_1).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + // println!("linear is {:?}", linear); + let grads = linear.forward(x_2).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + // println!("linear is {:?}", linear); + let state_updated = linear.into_record(); + + let (weight_updated, bias_updated) = ( + state_updated.weight.to_data(), + state_updated.bias.unwrap().to_data(), + ); + + // println!("\nweight_updated\n{:?}", weight_updated); + // println!("\nbias_updated\n{:?}", bias_updated); + + let weights_expected = TensorData::from([ + [0.743937, 0.743937, 0.743937, 0.743937, 0.743937, 0.743937], + [0.783809, 0.783809, 0.783809, 0.783809, 0.783809, 0.783809], + [0.742881, 0.742881, 0.742881, 0.742881, 0.742881, 0.742881], + [0.740366, 0.740366, 0.740366, 0.740366, 0.740366, 0.740366], + [0.748005, 0.748005, 0.748005, 0.748005, 0.748005, 0.748005], + [0.743710, 0.743710, 0.743710, 0.743710, 0.743710, 0.743710], + ]); + let bias_expected = + TensorData::from([0.239199, 0.239199, 0.239199, 0.239199, 0.239199, 0.239199]); + + let tolerance = Tolerance::absolute(1e-6); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + } + + #[test] + fn test_rmsprop_optimizer_with_numbers() { + let linear = given_linear_layer( + TensorData::from([ + [-0.3206, 0.1374, 0.4043, 0.3200, 0.0859, 0.0671], + [0.0777, -0.0185, -0.3667, 0.2550, 0.1955, -0.2922], + [-0.0190, 0.0346, -0.2962, 0.2484, -0.2780, 0.3130], + [-0.2980, -0.2214, -0.3715, -0.2981, -0.0761, 0.1626], + [0.3300, -0.2182, 0.3717, -0.1729, 0.3796, -0.0304], + [-0.0159, -0.0120, 0.1258, 0.1921, 0.0293, 0.3833], + ]), + TensorData::from([-0.3905, 0.0884, -0.0970, 0.1176, 0.1366, 0.0130]), + ); + let device = Default::default(); + let x_1 = Tensor::::from_floats( + [ + [0.6294, 0.0940, 0.8176, 0.8824, 0.5228, 0.4310], + [0.7152, 0.9559, 0.7893, 0.5684, 0.5939, 0.8883], + ], + &device, + ) + .require_grad(); + let x_2 = Tensor::::from_floats( + [ + [0.8491, 0.2108, 0.8939, 0.4433, 0.5527, 0.2528], + [0.3270, 0.0412, 0.5538, 0.9605, 0.3195, 0.9085], + ], + &device, + ) + .require_grad(); + + let mut optimizer = RmsPropConfig::new() + .with_alpha(0.99) + .with_epsilon(1e-8) + .with_weight_decay(WeightDecayConfig::new(0.05).into()) + .with_momentum(0.9) + .with_centered(false) + .init(); + + let grads = linear.forward(x_1).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let grads = linear.forward(x_2).backward(); + let grads = GradientsParams::from_grads(grads, &linear); + let linear = optimizer.step(LEARNING_RATE, linear, grads); + + let state_updated = linear.into_record(); + let weights_expected = TensorData::from([ + [ + -0.576399, -0.118494, 0.148353, 0.064070, -0.169983, -0.188779, + ], + [ + -0.135571, -0.231448, -0.578445, 0.041143, -0.018162, -0.504207, + ], + [ + -0.275990, -0.222397, -0.553153, -0.008625, -0.534956, 0.055967, + ], + [ + -0.557575, -0.480979, -0.631072, -0.557675, -0.335686, -0.096997, + ], + [ + 0.078313, -0.469618, 0.119993, -0.424341, 0.127890, -0.281912, + ], + [ + -0.271996, -0.268097, -0.130324, -0.064037, -0.226805, 0.127126, + ], + ]); + let bias_expected = TensorData::from([ + -0.651299, -0.172400, -0.357800, -0.143200, -0.124200, -0.247800, + ]); + + let (weight_updated, bias_updated) = ( + state_updated.weight.to_data(), + state_updated.bias.unwrap().to_data(), + ); + + // println!("\nweight_updated\n{:?}", weight_updated); + // println!("\nbias_updated\n{:?}", bias_updated); + + let tolerance = Tolerance::absolute(1e-6); + bias_updated.assert_approx_eq::(&bias_expected, tolerance); + weight_updated.assert_approx_eq::(&weights_expected, tolerance); + } + + fn given_linear_layer(weight: TensorData, bias: TensorData) -> Linear { + let device = Default::default(); + let record = LinearRecord { + weight: Param::from_data(weight, &device), + bias: Some(Param::from_data(bias, &device)), + }; + + LinearConfig::new(6, 6).init(&device).load_record(record) + } + + #[allow(dead_code)] + fn create_random_tensor() -> Tensor { + Tensor::::random( + Shape::new([2, 20]), + Distribution::Default, + &Default::default(), + ) + } + + fn create_rmsprop() + -> OptimizerAdaptor, TestAutodiffBackend> { + RmsPropConfig { + alpha: 0.99, + epsilon: 1e-9, + centered: false, + weight_decay: Some(WeightDecayConfig { penalty: 0.05 }), + momentum: 0.9, + grad_clipping: None, + } + .init() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/sgd.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/sgd.rs new file mode 100644 index 0000000..9da74e1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/sgd.rs @@ -0,0 +1,181 @@ +use burn_core as burn; + +use super::SimpleOptimizer; +use super::adaptor::OptimizerAdaptor; +use super::decay::{WeightDecay, WeightDecayConfig}; +use super::momentum::{Momentum, MomentumConfig, MomentumState}; +use crate::LearningRate; +use crate::grad_clipping::GradientClippingConfig; +use burn::config::Config; +use burn::module::AutodiffModule; +use burn::record::Record; +use burn::tensor::Tensor; +use burn::tensor::backend::{AutodiffBackend, Backend}; + +/// Configuration to create the [Sgd](Sgd) optimizer. +#[derive(Config, Debug)] +pub struct SgdConfig { + /// [Weight decay](WeightDecayConfig) config. + weight_decay: Option, + /// [Momentum](MomentumConfig) config. + momentum: Option, + /// [Gradient Clipping](GradientClippingConfig) config. + gradient_clipping: Option, +} + +/// Optimizer that implements stochastic gradient descent with momentum. +/// +/// The optimizer can be configured with [SgdConfig](SgdConfig). +#[derive(Clone)] +pub struct Sgd { + momentum: Option>, + weight_decay: Option, +} + +/// State of [Sgd](Sgd). +#[derive(Record, Clone, new)] +pub struct SgdState { + /// The current state of the momentum (if any). + pub momentum: Option>, +} + +impl SgdConfig { + /// Creates a new [SgdConfig](SgdConfig) with default values. + pub fn init>( + &self, + ) -> OptimizerAdaptor, M, B> { + let momentum = self.momentum.as_ref().map(Momentum::new); + let weight_decay = self.weight_decay.as_ref().map(WeightDecay::new); + + let mut optim = OptimizerAdaptor::from(Sgd { + momentum, + weight_decay, + }); + if let Some(config) = &self.gradient_clipping { + optim = optim.with_grad_clipping(config.init()); + } + optim + } +} + +impl SimpleOptimizer for Sgd { + type State = SgdState; + + fn step( + &self, + lr: LearningRate, + tensor: Tensor, + mut grad: Tensor, + state: Option>, + ) -> (Tensor, Option>) { + let mut state_momentum = None; + + if let Some(state) = state { + state_momentum = state.momentum; + } + + if let Some(weight_decay) = &self.weight_decay { + grad = weight_decay.transform(grad, tensor.clone()); + } + + if let Some(momentum) = &self.momentum { + let (grad_out, state) = momentum.transform(grad, state_momentum); + state_momentum = Some(state); + grad = grad_out; + } + + let state = SgdState::new(state_momentum); + let delta = grad.mul_scalar(lr); + + (tensor - delta, Some(state)) + } + + fn to_device(mut state: Self::State, device: &B::Device) -> Self::State { + state.momentum = state.momentum.map(|state| state.to_device(device)); + state + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + TestAutodiffBackend, TestBackend, + grad_clipping::GradientClipping, + optim::{GradientsParams, Optimizer}, + }; + use burn::tensor::{Distribution, Shape}; + use burn_nn::{Linear, LinearConfig}; + + const LEARNING_RATE: LearningRate = 0.02; + + #[test] + fn with_updated_params_should_have_state() { + let device = Default::default(); + let layer = layer::(&device); + let mut optim = sgd_with_all(); + let loss = layer.forward(random_tensor::(&device)); + let grads = loss.backward(); + let grads = GradientsParams::from_grads(grads, &layer); + let _layer = optim.step(LEARNING_RATE, layer, grads); + + let record = optim.to_record(); + + assert!(!record.is_empty()); + } + + #[test] + fn without_updated_params_should_not_have_state() { + let optim = sgd_with_all(); + let record = optim.to_record(); + assert!(record.is_empty()); + } + + #[test] + fn can_attach_gradient_clipping() { + let optim = sgd_with_all().with_grad_clipping(GradientClipping::Value(0.5)); + assert!(optim.has_gradient_clipping()); + } + + #[test] + fn should_load_state() { + let device = Default::default(); + let layer = layer::(&device); + let mut optim = sgd_with_all(); + let loss = layer.forward(random_tensor(&device)); + let grads = loss.backward(); + let grads = GradientsParams::from_grads(grads, &layer); + let _layer = optim.step(LEARNING_RATE, layer, grads); + + let record = optim.to_record(); + let optim_new = sgd_with_all(); + let record_new = optim_new.to_record(); + let optim_new = optim_new.load_record(record.clone()); + let state_restored = optim_new.to_record(); + + assert_ne!(record.len(), record_new.len()); + assert_eq!(record.len(), state_restored.len()); + } + + fn random_tensor(device: &B::Device) -> Tensor { + Tensor::::random(Shape::new([2, 20]), Distribution::Default, device) + } + + fn layer(device: &B::Device) -> Linear { + LinearConfig::new(20, 20).init(device) + } + + fn sgd_with_all() + -> OptimizerAdaptor, Linear, TestAutodiffBackend> { + SgdConfig { + weight_decay: Some(WeightDecayConfig { penalty: 0.05 }), + momentum: Some(MomentumConfig { + momentum: 0.9, + dampening: 0.1, + nesterov: true, + }), + gradient_clipping: None, + } + .init() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/adaptor.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/adaptor.rs new file mode 100644 index 0000000..83e485d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/adaptor.rs @@ -0,0 +1,210 @@ +use burn_core::{self as burn, prelude::Backend, tensor::Device}; + +use super::{SimpleOptimizer, record::AdaptorRecord}; +use crate::{ + LearningRate, MultiGradientsParams, + grad_clipping::GradientClipping, + optim::{GradientsParams, Optimizer}, +}; + +use burn::module::{AutodiffModule, ModuleMapper, Param, ParamId}; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use core::marker::PhantomData; +use hashbrown::HashMap; + +/// Wrapper struct that adapts any [simple optimizer](SimpleOptimizer) into +/// an [optimizer](Optimizer). +#[derive(Clone)] +pub struct OptimizerAdaptor +where + O: SimpleOptimizer, + M: AutodiffModule, + B: AutodiffBackend, +{ + optim: O, + records: HashMap>, + module: PhantomData, + grad_clipping: Option, +} + +impl From for OptimizerAdaptor +where + B: AutodiffBackend, + M: AutodiffModule, + O: SimpleOptimizer, +{ + fn from(optim: O) -> Self { + Self { + optim, + records: HashMap::new(), + module: PhantomData, + grad_clipping: None, + } + } +} + +impl OptimizerAdaptor +where + O: SimpleOptimizer, + M: AutodiffModule, + B: AutodiffBackend, +{ + /// Sets the gradient clipping. + /// + /// # Arguments + /// + /// * `gradient_clipping` - The gradient clipping. + /// + /// # Returns + /// + /// The optimizer. + pub fn with_grad_clipping(mut self, gradient_clipping: GradientClipping) -> Self { + self.grad_clipping = Some(gradient_clipping); + self + } + + #[cfg(test)] + pub(crate) fn has_gradient_clipping(&self) -> bool { + self.grad_clipping.is_some() + } +} + +impl Optimizer for OptimizerAdaptor +where + B: AutodiffBackend, + M: AutodiffModule, + O: SimpleOptimizer, +{ + type Record = HashMap>; + + fn step(&mut self, lr: LearningRate, module: M, grads: GradientsParams) -> M { + let mut grads = GradAdaptor::Single(grads); + + let mut mapper = SimpleOptimizerMapper::::new( + &self.optim, + &mut self.records, + &mut grads, + lr, + self.grad_clipping.as_ref(), + ); + module.map(&mut mapper) + } + + fn step_multi(&mut self, lr: LearningRate, module: M, grads: crate::MultiGradientsParams) -> M { + let mut grads = GradAdaptor::Multi(grads); + + let mut mapper = SimpleOptimizerMapper::::new( + &self.optim, + &mut self.records, + &mut grads, + lr, + self.grad_clipping.as_ref(), + ); + module.map(&mut mapper) + } + + fn to_record(&self) -> Self::Record { + self.records.clone() + } + + fn load_record(mut self, record: Self::Record) -> Self { + self.records = record; + self + } +} + +enum GradAdaptor { + Single(GradientsParams), + Multi(MultiGradientsParams), +} + +impl GradAdaptor { + fn remove( + &mut self, + id: ParamId, + ) -> Option<(Tensor, Device)> { + match self { + GradAdaptor::Single(grads) => grads.remove(id).map(|t| { + let device = t.device(); + (t, device) + }), + GradAdaptor::Multi(grads) => grads.remove(id), + } + } +} + +#[derive(new)] +struct SimpleOptimizerMapper<'a, M, B, O> +where + M: AutodiffModule, + B: AutodiffBackend, + O: SimpleOptimizer, +{ + optimizer: &'a O, + records: &'a mut HashMap>, + grads: &'a mut GradAdaptor, + lr: LearningRate, + phantom: PhantomData, + grad_clipping: Option<&'a GradientClipping>, +} + +impl ModuleMapper for SimpleOptimizerMapper<'_, M, B, O> +where + M: AutodiffModule, + B: AutodiffBackend, + O: SimpleOptimizer, +{ + fn map_float(&mut self, param: Param>) -> Param> { + let (id, tensor, mapper) = param.consume(); + let grad = self.grads.remove(id); + + let tensor = if let Some((grad, device)) = grad { + let is_require_grad = tensor.is_require_grad(); + let (key, record) = self.records.remove_entry(&id).unzip(); + let tensor = if tensor.device() != device { + tensor.to_device(&device) + } else { + tensor + }; + + debug_assert_eq!( + grad.device(), + device, + "The gradient is on the provided device" + ); + let clipped_grad = if let Some(g_clipping) = self.grad_clipping { + g_clipping.clip_gradient(grad) + } else { + grad + }; + + debug_assert_eq!( + tensor.device(), + device, + "Tensor and gradients are on the same device." + ); + + let (tensor, state) = self.optimizer.step( + self.lr, + tensor.inner(), + clipped_grad, + record.map(|record| O::to_device(record.into_state(), &device)), + ); + + if let Some(state) = state { + self.records + .insert(key.unwrap_or(id), AdaptorRecord::from_state(state)); + } + + let mut tensor = Tensor::from_inner(tensor); + if is_require_grad { + tensor = tensor.require_grad(); + } + tensor + } else { + tensor + }; + + Param::from_mapped_value(id, tensor, mapper) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/base.rs new file mode 100644 index 0000000..69b190b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/base.rs @@ -0,0 +1,36 @@ +use burn_core as burn; + +use crate::LearningRate; +use burn::record::Record; +use burn::tensor::{Tensor, backend::Backend}; + +/// Simple optimizer is an opinionated trait to simplify the process of implementing an +/// optimizer. +/// +/// Implementations don't have to handle missing gradients, loading and exporting records, navigate the +/// module parameter structure, handle tracked and untracked tensors, and the likes. +pub trait SimpleOptimizer: Send + Sync + Clone +where + B: Backend, +{ + /// The state of the optimizer. It also implements [record](Record), so that it can be saved. + type State: Record + Clone + 'static; + + /// The optimizer step is performed for one tensor at a time with its gradient and state. + /// + /// Note that the state is passed as parameter, so implementations don't have to handle + /// the saving and loading of recorded states. + fn step( + &self, + lr: LearningRate, + tensor: Tensor, + grad: Tensor, + state: Option>, + ) -> (Tensor, Option>); + + /// Change the device of the state. + /// + /// This function will be called accordingly to have the state on the same device as the + /// gradient and the tensor when the [step](SimpleOptimizer::step) function is called. + fn to_device(state: Self::State, device: &B::Device) -> Self::State; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/mod.rs new file mode 100644 index 0000000..bfc1ff6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/mod.rs @@ -0,0 +1,8 @@ +mod base; +pub use base::*; + +/// Adaptor module for optimizers. +pub mod adaptor; + +/// Record module for optimizers. +pub mod record; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/base.rs new file mode 100644 index 0000000..cd58b76 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/base.rs @@ -0,0 +1,93 @@ +use burn_core as burn; + +use super::{AdaptorRecordItemV1, AdaptorRecordV1}; +use crate::optim::SimpleOptimizer; +use burn::record::{PrecisionSettings, Record}; +use burn::tensor::backend::AutodiffBackend; +use serde::{Deserialize, Serialize}; + +/// [Optimizer adaptor](crate::optim::simple::adaptor::OptimizerAdaptor) record. +/// +/// Records are versioned for backward compatibility, so old records can be loaded. +pub enum AdaptorRecord +where + O: SimpleOptimizer, + B: AutodiffBackend, +{ + /// Version 1. + V1(AdaptorRecordV1), +} + +/// [Optimizer adaptor](crate::optim::simple::adaptor::OptimizerAdaptor) record item. +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +pub enum AdaptorRecordItem< + O: SimpleOptimizer, + B: AutodiffBackend, + S: PrecisionSettings, +> { + /// Version 1. + V1(AdaptorRecordItemV1), +} + +impl Record for AdaptorRecord +where + O: SimpleOptimizer, + B: AutodiffBackend, +{ + type Item = AdaptorRecordItem; + + fn into_item(self) -> Self::Item { + match self { + AdaptorRecord::V1(record) => AdaptorRecordItem::V1(record.into_item()), + } + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + match item { + AdaptorRecordItem::V1(item) => Self::V1(AdaptorRecordV1::from_item(item, device)), + } + } +} + +impl Clone for AdaptorRecord +where + O: SimpleOptimizer, + B: AutodiffBackend, +{ + fn clone(&self) -> Self { + match self { + AdaptorRecord::V1(record) => Self::V1(record.clone()), + } + } +} + +impl AdaptorRecord +where + O: SimpleOptimizer, + B: AutodiffBackend, +{ + /// Converts the record into the optimizer state. + /// + /// # Returns + /// + /// The optimizer state. + pub fn into_state(self) -> O::State { + match self { + AdaptorRecord::V1(record) => record.into_state(), + } + } + + /// Converts the optimizer state into the record. + /// + /// # Arguments + /// + /// * `state`: The optimizer state. + /// + /// # Returns + /// + /// The record. + pub fn from_state(state: O::State) -> Self { + Self::V1(AdaptorRecordV1::from_state(state)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/mod.rs new file mode 100644 index 0000000..ffe5531 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod v1; + +pub use base::*; +pub use v1::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/v1.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/v1.rs new file mode 100644 index 0000000..913120f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/simple/record/v1.rs @@ -0,0 +1,201 @@ +use burn_core as burn; + +use crate::optim::SimpleOptimizer; +use burn::record::{PrecisionSettings, Record}; +use burn::tensor::backend::Backend; +use core::any::Any; +use serde::{Deserialize, Serialize}; + +#[cfg(not(feature = "std"))] +use alloc::boxed::Box; + +/// [Optimizer adaptor](crate::optim::simple::adaptor::OptimizerAdaptor) record item. +pub enum AdaptorRecordV1, B: Backend> { + /// Rank 0. + Rank0(O::State<0>), + + /// Rank 1. + Rank1(O::State<1>), + + /// Rank 2. + Rank2(O::State<2>), + + /// Rank 3. + Rank3(O::State<3>), + + /// Rank 4. + Rank4(O::State<4>), + + /// Rank 5. + Rank5(O::State<5>), + + /// Rank 6. + Rank6(O::State<6>), + + /// Rank 7. + Rank7(O::State<7>), + + /// Rank 8. + Rank8(O::State<8>), +} + +impl, B: Backend> Clone for AdaptorRecordV1 { + fn clone(&self) -> Self { + match self { + AdaptorRecordV1::Rank0(record) => AdaptorRecordV1::Rank0(record.clone()), + AdaptorRecordV1::Rank1(record) => AdaptorRecordV1::Rank1(record.clone()), + AdaptorRecordV1::Rank2(record) => AdaptorRecordV1::Rank2(record.clone()), + AdaptorRecordV1::Rank3(record) => AdaptorRecordV1::Rank3(record.clone()), + AdaptorRecordV1::Rank4(record) => AdaptorRecordV1::Rank4(record.clone()), + AdaptorRecordV1::Rank5(record) => AdaptorRecordV1::Rank5(record.clone()), + AdaptorRecordV1::Rank6(record) => AdaptorRecordV1::Rank6(record.clone()), + AdaptorRecordV1::Rank7(record) => AdaptorRecordV1::Rank7(record.clone()), + AdaptorRecordV1::Rank8(record) => AdaptorRecordV1::Rank8(record.clone()), + } + } +} + +/// [Optimizer adaptor](crate::optim::simple::adaptor::OptimizerAdaptor) record item. +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +pub enum AdaptorRecordItemV1, B: Backend, S: PrecisionSettings> { + /// Rank 0. + Rank0( as Record>::Item), + + /// Rank 1. + Rank1( as Record>::Item), + + /// Rank 2. + Rank2( as Record>::Item), + + /// Rank 3. + Rank3( as Record>::Item), + + /// Rank 4. + Rank4( as Record>::Item), + + /// Rank 5. + Rank5( as Record>::Item), + + /// Rank 6. + Rank6( as Record>::Item), + + /// Rank 7. + Rank7( as Record>::Item), + + /// Rank 8. + Rank8( as Record>::Item), +} + +impl AdaptorRecordV1 +where + O: SimpleOptimizer, + B: Backend, +{ + /// Convert the record into the state. + /// + /// # Returns + /// + /// The state. + /// + /// # Panics + /// + /// Panics if the state dimension is not supported. + pub fn into_state(self) -> O::State { + let boxed_state: Box = match self { + AdaptorRecordV1::Rank0(s) => Box::new(s), + AdaptorRecordV1::Rank1(s) => Box::new(s), + AdaptorRecordV1::Rank2(s) => Box::new(s), + AdaptorRecordV1::Rank3(s) => Box::new(s), + AdaptorRecordV1::Rank4(s) => Box::new(s), + AdaptorRecordV1::Rank5(s) => Box::new(s), + AdaptorRecordV1::Rank6(s) => Box::new(s), + AdaptorRecordV1::Rank7(s) => Box::new(s), + AdaptorRecordV1::Rank8(s) => Box::new(s), + }; + let state = boxed_state + .downcast::>() + .expect("Unsupported state dimension, dimension up to 8 are supported."); + *state + } + + /// Convert the state into the record. + /// + /// # Arguments + /// + /// * `state`: The state. + /// + /// # Returns + /// + /// The record. + pub fn from_state(state: O::State) -> Self { + let state: Box = Box::new(state); + + match D { + 0 => AdaptorRecordV1::Rank0(*state.downcast().unwrap()), + 1 => AdaptorRecordV1::Rank1(*state.downcast().unwrap()), + 2 => AdaptorRecordV1::Rank2(*state.downcast().unwrap()), + 3 => AdaptorRecordV1::Rank3(*state.downcast().unwrap()), + 4 => AdaptorRecordV1::Rank4(*state.downcast().unwrap()), + 5 => AdaptorRecordV1::Rank5(*state.downcast().unwrap()), + 6 => AdaptorRecordV1::Rank6(*state.downcast().unwrap()), + 7 => AdaptorRecordV1::Rank7(*state.downcast().unwrap()), + 8 => AdaptorRecordV1::Rank8(*state.downcast().unwrap()), + _ => panic!("Unsupported state dimension, dimension up to 8 are supported."), + } + } +} + +impl Record for AdaptorRecordV1 +where + O: SimpleOptimizer, + B: Backend, +{ + type Item = AdaptorRecordItemV1; + + fn into_item(self) -> Self::Item { + match self { + AdaptorRecordV1::Rank0(record) => AdaptorRecordItemV1::Rank0(record.into_item()), + AdaptorRecordV1::Rank1(record) => AdaptorRecordItemV1::Rank1(record.into_item()), + AdaptorRecordV1::Rank2(record) => AdaptorRecordItemV1::Rank2(record.into_item()), + AdaptorRecordV1::Rank3(record) => AdaptorRecordItemV1::Rank3(record.into_item()), + AdaptorRecordV1::Rank4(record) => AdaptorRecordItemV1::Rank4(record.into_item()), + AdaptorRecordV1::Rank5(record) => AdaptorRecordItemV1::Rank5(record.into_item()), + AdaptorRecordV1::Rank6(record) => AdaptorRecordItemV1::Rank6(record.into_item()), + AdaptorRecordV1::Rank7(record) => AdaptorRecordItemV1::Rank7(record.into_item()), + AdaptorRecordV1::Rank8(record) => AdaptorRecordItemV1::Rank8(record.into_item()), + } + } + + fn from_item(item: Self::Item, device: &B::Device) -> Self { + match item { + AdaptorRecordItemV1::Rank0(item) => { + AdaptorRecordV1::Rank0( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank1(item) => { + AdaptorRecordV1::Rank1( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank2(item) => { + AdaptorRecordV1::Rank2( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank3(item) => { + AdaptorRecordV1::Rank3( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank4(item) => { + AdaptorRecordV1::Rank4( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank5(item) => { + AdaptorRecordV1::Rank5( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank6(item) => { + AdaptorRecordV1::Rank6( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank7(item) => { + AdaptorRecordV1::Rank7( as Record>::from_item(item, device)) + } + AdaptorRecordItemV1::Rank8(item) => { + AdaptorRecordV1::Rank8( as Record>::from_item(item, device)) + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/visitor.rs b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/visitor.rs new file mode 100644 index 0000000..603cb62 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-optim/src/optim/visitor.rs @@ -0,0 +1,60 @@ +use burn_core as burn; + +use super::GradientsParams; +use burn::module::{AutodiffModule, ModuleVisitor, Param, ParamId}; +use burn::tensor::{Tensor, backend::AutodiffBackend}; +use core::marker::PhantomData; + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +#[derive(new)] +pub struct GradientsParamsConverter<'a, M: AutodiffModule, B: AutodiffBackend> { + grads: &'a mut B::Gradients, + grads_params: &'a mut GradientsParams, + phatom: PhantomData, + filter: Option>, +} + +#[derive(new)] +pub struct GradientsParamsChangeDevice<'a, M: AutodiffModule, B: AutodiffBackend> { + device: &'a B::Device, + grads: &'a mut GradientsParams, + phatom: PhantomData, +} + +impl ModuleVisitor for GradientsParamsConverter<'_, M, B> +where + B: AutodiffBackend, + M: AutodiffModule, +{ + fn visit_float(&mut self, param: &Param>) { + if let Some(filter) = self.filter.as_ref() + && !filter.contains(¶m.id) + { + return; + } + + let Some(grad) = param.val().grad_remove(self.grads) else { + return; + }; + + self.grads_params + .register::(param.id, grad); + } +} + +impl ModuleVisitor for GradientsParamsChangeDevice<'_, M, B> +where + B: AutodiffBackend, + M: AutodiffModule, +{ + fn visit_float(&mut self, param: &Param>) { + let Some(grad) = self.grads.remove::(param.id) else { + return; + }; + + self.grads + .register::(param.id, grad.to_device(self.device)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-remote/Cargo.toml new file mode 100644 index 0000000..314414b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/Cargo.toml @@ -0,0 +1,78 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Backend router decorator over the network." +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-remote" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-router-remote" +documentation = "https://docs.rs/burn-router-remote" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["client", "server"] +doc = [] +tracing = [ + "burn-communication/tracing", + "burn-ir/tracing", + "burn-router/tracing", + "burn-std/tracing", + "burn-backend/tracing", +] + +client = ["tokio-tungstenite", "async-channel", "tokio/sync"] +server = [ + "tokio-tungstenite", + "async-channel", + "tokio/sync", + "axum", + "tracing-core/default", + "tracing-subscriber/default", +] + + +[dependencies] +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2", default-features = true } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = true } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = true } +burn-router = { path = "../burn-router", version = "=0.21.0-pre.2", default-features = true } +burn-communication = { path = "../burn-communication", version = "=0.21.0-pre.2", features = [ + "data-service", + "websocket", +] } + +bytes = { workspace = true } + +# Basic dependencies +derive-new = { workspace = true } +log = { workspace = true } + +# Shared dependencies +tokio = { workspace = true, features = ["rt-multi-thread"] } +serde = { workspace = true, features = ["derive"] } +serde_bytes = { workspace = true } +rmp-serde = { workspace = true } +futures-util = { workspace = true } + +# Client dependencies +async-channel = { workspace = true, optional = true } +tokio-tungstenite = { workspace = true, optional = true } + +# Server dependencies +axum = { workspace = true, features = ["ws"], optional = true } +tracing-core = { workspace = true, optional = true } +tracing-subscriber = { workspace = true, optional = true } +tokio-util = { workspace = true } + +[dev-dependencies] +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", default-features = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/README.md b/crates/stable-diffusion-burn/burn-crates/burn-remote/README.md new file mode 100644 index 0000000..e69de29 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/base.rs new file mode 100644 index 0000000..07033d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/base.rs @@ -0,0 +1,123 @@ +pub use super::RemoteDevice; +use super::worker::{ClientRequest, ClientWorker}; +use crate::shared::{ComputeTask, ConnectionId, SessionId, Task, TaskResponseContent}; +use async_channel::{RecvError, SendError, Sender}; +use burn_communication::ProtocolClient; +use burn_ir::TensorId; +use burn_std::id::StreamId; +use std::{ + future::Future, + sync::{Arc, atomic::AtomicU64}, +}; + +#[derive(Clone)] +pub struct RemoteClient { + pub(crate) device: RemoteDevice, + pub(crate) sender: Arc, + pub(crate) runtime: Arc, +} + +impl RemoteClient { + pub fn init(device: RemoteDevice) -> Self { + ClientWorker::::start(device) + } + + pub(crate) fn new( + device: RemoteDevice, + sender: Sender, + runtime: Arc, + session_id: SessionId, + ) -> Self { + Self { + device, + runtime, + sender: Arc::new(RemoteSender { + sender, + position_counter: AtomicU64::new(0), + tensor_id_counter: AtomicU64::new(0), + session_id, + }), + } + } +} + +pub(crate) struct RemoteSender { + sender: Sender, + position_counter: AtomicU64, + tensor_id_counter: AtomicU64, + session_id: SessionId, +} + +#[allow(unused)] +#[derive(Debug)] +pub enum RemoteSendError { + SendError(SendError), + RecvError(RecvError), +} + +impl RemoteSender { + /// Generate a new unique (for this [`RemoteSender`] [`TensorId`]. + pub(crate) fn new_tensor_id(&self) -> TensorId { + TensorId::new( + self.tensor_id_counter + .fetch_add(1, std::sync::atomic::Ordering::Relaxed), + ) + } + + /// Give the next operation sequence number. + fn next_position(&self) -> u64 { + self.position_counter + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + } + + pub(crate) fn send(&self, task: ComputeTask) { + self.sender + .send_blocking(ClientRequest::WithoutCallback(Task::Compute( + task, + ConnectionId::new(self.next_position(), StreamId::current()), + ))) + .unwrap(); + } + + pub(crate) fn send_async( + &self, + task: ComputeTask, + ) -> impl Future> + Send + use<> { + let stream_id = StreamId::current(); + let position = self.next_position(); + let sender = self.sender.clone(); + + async move { + let (tx, rx) = async_channel::bounded(1); + + if let Err(e) = sender + .send(ClientRequest::WithSyncCallback( + Task::Compute(task, ConnectionId::new(position, stream_id)), + tx, + )) + .await + { + return Err(RemoteSendError::SendError(e)); + } + + match rx.recv().await { + Ok(response) => Ok(response), + Err(e) => Err(RemoteSendError::RecvError(e)), + } + } + } + + pub(crate) fn close(&mut self) { + let sender = self.sender.clone(); + + let close_task = ClientRequest::WithoutCallback(Task::Close(self.session_id)); + + sender.send_blocking(close_task).unwrap(); + } +} + +impl Drop for RemoteSender { + fn drop(&mut self) { + self.close(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/channel.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/channel.rs new file mode 100644 index 0000000..14cb2e2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/channel.rs @@ -0,0 +1,82 @@ +use std::marker::PhantomData; + +use burn_backend::Shape; +use burn_communication::ProtocolClient; +use burn_ir::TensorIr; +use burn_router::{RouterTensor, RunnerChannel, get_client}; + +use super::{ + RemoteClient, + runner::{RemoteBridge, RemoteDevice, RemoteTensorHandle}, +}; + +/// A local channel with direct connection to the backend runner clients. +pub struct RemoteChannel { + _p: PhantomData, +} + +impl RunnerChannel for RemoteChannel { + type Device = RemoteDevice; + type Bridge = RemoteBridge; + type Client = RemoteClient; + + type FloatElem = f32; + + type IntElem = i32; + + type BoolElem = u32; + + fn name(device: &Self::Device) -> String { + format!("remote-{device:?}") + } + + fn init_client(device: &Self::Device) -> Self::Client { + RemoteClient::init::(device.clone()) + } + + fn get_tensor_handle(tensor: &TensorIr, client: &Self::Client) -> RemoteTensorHandle { + RemoteTensorHandle { + client: client.clone(), + tensor: tensor.clone(), + _p: PhantomData, + } + } + + fn register_tensor( + _client: &Self::Client, + _handle: RemoteTensorHandle, + _shape: Shape, + _dtype: burn_backend::DType, + ) -> RouterTensor { + // This function is normally only used to move a tensor from a device to another. + // + // In other words, to change the client. + panic!("Can't register manually a tensor on a remote channel."); + } + + fn change_client_backend( + tensor: RouterTensor, + target_device: &Self::Device, // target device + ) -> RouterTensor { + // Get tensor handle from current client + let original_client = tensor.client.clone(); + let desc = tensor.into_ir(); + let handle = Self::get_tensor_handle(&desc, &original_client); + + let handle = handle.change_backend(target_device); + + let id = handle.tensor.id; + + let target_client = get_client::(target_device); + let router_tensor: RouterTensor = + RouterTensor::new(id, handle.tensor.shape, handle.tensor.dtype, target_client); + + router_tensor + } +} + +impl Clone for RemoteChannel { + fn clone(&self) -> Self { + RemoteChannel { _p: PhantomData } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/mod.rs new file mode 100644 index 0000000..37516c2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/mod.rs @@ -0,0 +1,8 @@ +mod base; +mod channel; +mod runner; +mod worker; + +pub use base::*; +pub use channel::*; +pub use runner::RemoteDevice; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/runner.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/runner.rs new file mode 100644 index 0000000..a9f24db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/runner.rs @@ -0,0 +1,294 @@ +use super::{RemoteChannel, RemoteClient}; +use crate::shared::{ComputeTask, TaskResponseContent, TensorRemote}; +use burn_backend::{DeviceId, DeviceOps, ExecutionError, Shape, TensorData}; +use burn_communication::{Address, ProtocolClient, data_service::TensorTransferId}; +use burn_ir::TensorIr; +use burn_router::{MultiBackendBridge, RouterTensor, RunnerClient, get_client}; +use burn_std::{backtrace::BackTrace, future::DynFut}; +use std::sync::OnceLock; +use std::{collections::HashMap, marker::PhantomData, str::FromStr, sync::Mutex}; + +// TODO: we should work with the parsed structure of Address, not the string. +static ADDRESS_REGISTRY: OnceLock>> = OnceLock::new(); + +fn get_address_registry() -> &'static Mutex> { + ADDRESS_REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Map a string network address to a (local runtime) global unique u32. +/// +/// Globally stable over the lifetime of the process, shared between threads, +/// If the address has never been seen, a new id will be created. +/// If the address has been seen, the previous id will be returned. +pub fn address_to_id>(address: S) -> u32 { + let registry = get_address_registry(); + let mut registry = registry.lock().unwrap(); + let next_id = registry.len() as u32; + *registry + .entry(address.as_ref().to_string()) + .or_insert_with(|| next_id) +} + +/// Look up an address by id. +/// +/// Returns the same address given ids by [`address_to_id`]. +pub fn id_to_address(id: u32) -> Option { + let registry = get_address_registry(); + let registry = registry.lock().unwrap(); + for entry in registry.iter() { + if entry.1 == &id { + return Some(entry.0.clone()); + } + } + None +} + +// It is very important to block on any request made with the sender, since ordering is crucial +// when registering operation or creating tensors. +// +// The overhead is minimal, since we only wait for the task to be sent to the async +// channel, but not sent to the server and even less processed by the server. +impl RunnerClient for RemoteClient { + type Device = RemoteDevice; + + fn register_op(&self, op: burn_ir::OperationIr) { + self.sender + .send(ComputeTask::RegisterOperation(Box::new(op))); + } + + fn read_tensor_async( + &self, + tensor: burn_ir::TensorIr, + ) -> DynFut> { + // Important for ordering to call the creation of the future sync. + let fut = self.sender.send_async(ComputeTask::ReadTensor(tensor)); + + Box::pin(async move { + match fut.await { + Ok(response) => match response { + TaskResponseContent::ReadTensor(res) => res, + _ => panic!("Invalid message type"), + }, + Err(e) => Err(ExecutionError::Generic { + reason: format!("Failed to read tensor: {:?}", e), + backtrace: BackTrace::capture(), + }), + } + }) + } + + fn register_tensor_data(&self, data: TensorData) -> RouterTensor { + let id = self.sender.new_tensor_id(); + let shape = data.shape.clone(); + let dtype = data.dtype; + + self.sender.send(ComputeTask::RegisterTensor(id, data)); + + RouterTensor::new(id, Shape::from(shape), dtype, self.clone()) + } + + fn device(&self) -> Self::Device { + self.device.clone() + } + + fn sync(&self) -> Result<(), ExecutionError> { + // Important for ordering to call the creation of the future sync. + let fut = self.sender.send_async(ComputeTask::SyncBackend); + + match self.runtime.block_on(fut) { + Ok(response) => match response { + TaskResponseContent::SyncBackend(res) => res, + _ => panic!("Invalid message type"), + }, + Err(e) => Err(ExecutionError::Generic { + reason: format!("Failed to sync: {:?}", e), + backtrace: BackTrace::capture(), + }), + } + } + + fn seed(&self, seed: u64) { + self.sender.send(ComputeTask::Seed(seed)); + } + + fn create_empty_handle(&self) -> burn_ir::TensorId { + self.sender.new_tensor_id() + } + + fn dtype_usage(&self, dtype: burn_std::DType) -> burn_backend::DTypeUsageSet { + let fut = self.sender.send_async(ComputeTask::SupportsDType(dtype)); + + match self.runtime.block_on(fut) { + Ok(_response) => panic!("Invalid message type"), + Err(e) => panic!("Failed to check dtype support: {:?}", e), + } + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +/// The device contains the connection information of the server. +pub struct RemoteDevice { + pub(crate) address: Address, + /// The id of the device in the local registry, see [`address_to_id`]. + pub(crate) id: u32, +} + +impl RemoteDevice { + /// Create a device from an url. + pub fn new(address: &str) -> Self { + let id = address_to_id(address); + Self { + address: Address::from_str(address).unwrap(), + id, + } + } +} + +impl Default for RemoteDevice { + fn default() -> Self { + let address = match std::env::var("BURN_REMOTE_ADDRESS") { + Ok(address) => address, + Err(_) => String::from("ws://127.0.0.1:3000"), + }; + + Self::new(&address) + } +} + +impl burn_std::device::Device for RemoteDevice { + fn from_id(device_id: DeviceId) -> Self { + if device_id.type_id != 0 { + panic!("Invalid device id: {device_id} (expected type 0)"); + } + let address = id_to_address(device_id.index_id) + .unwrap_or_else(|| panic!("Invalid device id: {device_id}")); + Self::new(&address) + } + + fn to_id(&self) -> DeviceId { + DeviceId { + type_id: 0, + index_id: self.id, + } + } + + fn device_count(_type_id: u16) -> usize { + 1 + } +} + +impl DeviceOps for RemoteDevice {} + +pub struct RemoteBridge { + _p: PhantomData, +} + +pub struct RemoteTensorHandle { + pub(crate) client: RemoteClient, + pub(crate) tensor: TensorIr, + pub(crate) _p: PhantomData, +} + +static TRANSFER_COUNTER: Mutex> = Mutex::new(None); + +fn get_next_transfer_id() -> TensorTransferId { + let mut transfer_counter = TRANSFER_COUNTER.lock().unwrap(); + if transfer_counter.is_none() { + *transfer_counter = Some(0.into()); + + transfer_counter.unwrap() + } else { + let mut transfer_counter = transfer_counter.unwrap(); + transfer_counter.next(); + + transfer_counter + } +} + +impl RemoteTensorHandle { + /// Changes the backend of the tensor via a dWebSocket. + /// We ask the original server to expose the tensor, then ask the target server to fetch + /// the tensor. The target server will open a new network connection to the original server + /// to download the data. + /// This way the client never sees the tensor's data, and we avoid a bottleneck. + pub(crate) fn change_backend(mut self, target_device: &RemoteDevice) -> Self { + let transfer_id = get_next_transfer_id(); + self.client.sender.send(ComputeTask::ExposeTensorRemote { + tensor: self.tensor.clone(), + count: 1, + transfer_id, + }); + + let target_client = get_client::>(target_device); + + let new_id = target_client.sender.new_tensor_id(); + + let remote_tensor = TensorRemote { + transfer_id, + address: self.client.device.address.clone(), + }; + target_client + .sender + .send(ComputeTask::RegisterTensorRemote(remote_tensor, new_id)); + + self.tensor.id = new_id; + self.client = target_client; + + self + } +} + +impl MultiBackendBridge for RemoteBridge { + type TensorHandle = RemoteTensorHandle; + type Device = RemoteDevice; + + fn change_backend_float( + tensor: Self::TensorHandle, + _shape: burn_backend::Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle { + tensor.change_backend(target_device) + } + + fn change_backend_int( + tensor: Self::TensorHandle, + _shape: burn_backend::Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle { + tensor.change_backend(target_device) + } + + fn change_backend_bool( + tensor: Self::TensorHandle, + _shape: burn_backend::Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle { + tensor.change_backend(target_device) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_address_to_id() { + let address1 = "ws://127.0.0.1:3000"; + let address2 = "ws://127.0.0.1:3001"; + + let id1 = address_to_id(address1); + let id2 = address_to_id(address2); + + assert_ne!(id1, id2); + + assert_eq!(address_to_id(address1), id1); + assert_eq!(id_to_address(id1), Some(address1.to_string())); + + assert_eq!(address_to_id(address2), id2); + assert_eq!(id_to_address(id2), Some(address2.to_string())); + + let unused_id = u32::MAX; + + assert_eq!(id_to_address(unused_id), None); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/worker.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/worker.rs new file mode 100644 index 0000000..37694af --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/client/worker.rs @@ -0,0 +1,129 @@ +use super::{RemoteClient, runner::RemoteDevice}; +use crate::shared::{ConnectionId, SessionId, Task, TaskResponse, TaskResponseContent}; +use burn_communication::{CommunicationChannel, Message, ProtocolClient}; +use std::{collections::HashMap, marker::PhantomData, sync::Arc}; + +pub type CallbackSender = async_channel::Sender; + +#[derive(Debug)] +pub enum ClientRequest { + WithSyncCallback(Task, CallbackSender), + WithoutCallback(Task), +} + +pub(crate) struct ClientWorker { + requests: HashMap, + _p: PhantomData, +} + +impl ClientWorker { + async fn on_response(&mut self, response: TaskResponse) { + match self.requests.remove(&response.id) { + Some(request) => { + request.send(response.content).await.unwrap(); + } + None => { + panic!("Can't ignore message from the server."); + } + } + } + + fn register_callback(&mut self, id: ConnectionId, callback: CallbackSender) { + self.requests.insert(id, callback); + } +} + +impl ClientWorker { + pub fn start(device: RemoteDevice) -> RemoteClient { + let runtime = Arc::new( + tokio::runtime::Builder::new_multi_thread() + .enable_io() + .build() + .unwrap(), + ); + + let (sender, rec) = async_channel::bounded(10); + + let session_id = SessionId::new(); + let address = device.address.clone(); + + #[allow(deprecated)] + runtime.spawn(async move { + log::info!("Connecting to {} ...", address.clone()); + let mut stream_request = C::connect(address.clone(), "request") + .await + .expect("Server to be accessible"); + let mut stream_response = C::connect(address, "response") + .await + .expect("Server to be accessible"); + + let state = Arc::new(tokio::sync::Mutex::new(ClientWorker::::default())); + + // Init the connection. + let bytes: bytes::Bytes = rmp_serde::to_vec(&Task::Init(session_id)) + .expect("Can serialize tasks to bytes.") + .into(); + stream_request + .send(Message::new(bytes.clone())) + .await + .expect("Can send the message over the comms channel."); + stream_response + .send(Message::new(bytes)) + .await + .expect("Can send the message on the websocket."); + + // Async worker loading callbacks from the server. + let state_ws = state.clone(); + tokio::spawn(async move { + while let Ok(msg) = stream_response.recv().await { + let msg = match msg { + Some(msg) => msg, + None => { + log::warn!("Closed connection"); + return; + } + }; + + let response: TaskResponse = rmp_serde::from_slice(&msg.data) + .expect("Can deserialize messages from the websocket."); + let mut state = state_ws.lock().await; + state.on_response(response).await; + } + }); + + // Channel async worker sending operations to the server. + tokio::spawn(async move { + while let Ok(req) = rec.recv().await { + let task = match req { + ClientRequest::WithSyncCallback(task, callback) => { + if let Task::Compute(_content, id) = &task { + let mut state = state.lock().await; + state.register_callback(*id, callback); + } + task + } + ClientRequest::WithoutCallback(task) => task, + }; + let bytes = rmp_serde::to_vec(&task) + .expect("Can serialize tasks to bytes.") + .into(); + stream_request + .send(Message::new(bytes)) + .await + .expect("Can send the message on the websocket."); + } + }); + }); + + RemoteClient::new(device, sender, runtime, session_id) + } +} + +impl Default for ClientWorker { + fn default() -> Self { + Self { + requests: Default::default(), + _p: PhantomData, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/lib.rs new file mode 100644 index 0000000..514e116 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/lib.rs @@ -0,0 +1,86 @@ +#[macro_use] +extern crate derive_new; + +#[cfg(feature = "client")] +pub(crate) mod client; + +#[cfg(feature = "server")] +pub mod server; + +pub(crate) mod shared; + +#[cfg(feature = "client")] +mod __client { + use super::*; + + use crate::{client::RemoteChannel, shared::RemoteProtocol}; + use burn_communication::Protocol; + use burn_router::BackendRouter; + + /// The remote backend allows you to run computation on a remote device. + /// + /// Make sure there is a running server before trying to connect to it. + /// + /// ```rust, ignore + /// fn main() { + /// let device = Default::default(); + /// let port = 3000; + /// + /// // You need to activate the `server` feature flag to have access to this function. + /// burn::server::start::(device, port); + /// } + ///``` + pub type RemoteBackend = BackendRouter::Client>>; + + pub use client::RemoteDevice; +} +#[cfg(feature = "client")] +pub use __client::*; + +#[cfg(all(test, feature = "client", feature = "server"))] +mod tests { + use crate::RemoteBackend; + use burn_ndarray::NdArray; + use burn_tensor::{Distribution, Tensor}; + + #[test] + pub fn test_to_device_over_websocket() { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_io() + .build() + .unwrap(); + + rt.spawn(crate::server::start_websocket_async::( + Default::default(), + 3000, + )); + rt.spawn(crate::server::start_websocket_async::( + Default::default(), + 3010, + )); + + let remote_device_1 = super::RemoteDevice::new("ws://localhost:3000"); + let remote_device_2 = super::RemoteDevice::new("ws://localhost:3010"); + + // Some random input + let input_shape = [1, 28, 28]; + let input = Tensor::::random( + input_shape, + Distribution::Default, + &remote_device_1, + ); + let numbers_expected: Vec = input.to_data().to_vec().unwrap(); + + // Move tensor to device 2 + let input = input.to_device(&remote_device_2); + let numbers: Vec = input.to_data().to_vec().unwrap(); + assert_eq!(numbers, numbers_expected); + + // Move tensor back to device 1 + let input = input.to_device(&remote_device_1); + let numbers: Vec = input.to_data().to_vec().unwrap(); + assert_eq!(numbers, numbers_expected); + + rt.shutdown_background(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/base.rs new file mode 100644 index 0000000..7ffc5d7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/base.rs @@ -0,0 +1,182 @@ +use burn_communication::{ + CommunicationChannel, Message, Protocol, ProtocolServer, + data_service::{TensorDataServer, TensorDataService}, + util::os_shutdown_signal, + websocket::{WebSocket, WsServer}, +}; +use std::{marker::PhantomData, sync::Arc}; +use tokio_util::sync::CancellationToken; + +use burn_backend::tensor::Device; +use burn_ir::BackendIr; + +use crate::shared::{ComputeTask, Task}; + +use super::session::SessionManager; + +pub struct RemoteServer +where + B: BackendIr, + P: Protocol, +{ + _b: PhantomData, + _n: PhantomData

, +} + +impl RemoteServer +where + B: BackendIr, + P: Protocol, +{ + /// Start the server on the given address. + pub async fn start(device: Device, server: P::Server) { + let cancel_token = CancellationToken::new(); + let data_service = Arc::new(TensorDataService::::new(cancel_token)); + let session_manager = Arc::new(SessionManager::::new(device, data_service.clone())); + + let _server = server + .route("/response", { + let session_manager = session_manager.clone(); + move |stream| Self::handle_socket_response(session_manager, stream) + }) + .route("/request", { + let session_manager = session_manager.clone(); + move |stream| Self::handle_socket_request(session_manager, stream) + }) + .route_tensor_data_service(data_service) + .serve(os_shutdown_signal()) + .await; + } + + async fn handle_socket_response( + session_manager: Arc>, + mut socket: ::Channel, + ) { + log::info!("[Response Handler] On new connection."); + + let packet = socket.recv().await; + let msg = match packet { + Ok(Some(msg)) => msg, + Ok(None) => { + log::info!("Response stream closed"); + return; + } + Err(e) => { + log::info!("Response stream error on init: {e:?}"); + return; + } + }; + + let id = match rmp_serde::from_slice::(&msg.data) { + Ok(Task::Init(session_id)) => session_id, + msg => { + log::error!("Message is not a valid initialization task {msg:?}"); + return; + } + }; + + let mut receiver = session_manager.register_responder(id).await; + + log::info!("Response handler connection active"); + + while let Some(mut callback) = receiver.recv().await { + let response = callback.recv().await.unwrap(); + let bytes = rmp_serde::to_vec(&response).unwrap(); + + socket.send(Message::new(bytes.into())).await.unwrap(); + } + } + + async fn handle_socket_request( + session_manager: Arc>, + mut socket: ::Channel, + ) { + log::info!("[Request Handler] On new connection."); + let mut session_id = None; + + loop { + let packet = socket.recv().await; + let msg = match packet { + Ok(Some(msg)) => msg, + Ok(None) => { + log::info!("Request stream closed"); + break; + } + Err(e) => { + log::info!("Request stream error: {e:?}, Closing."); + break; + } + }; + + let task = match rmp_serde::from_slice::(&msg.data) { + Ok(val) => val, + Err(err) => { + log::info!("Only bytes message in the json format are supported {err:?}"); + break; + } + }; + + if let Task::Close(id) = task { + session_id = Some(id); + break; + } + + let (stream, connection_id, task) = + match session_manager.stream(&mut session_id, task).await { + Some(val) => val, + None => { + log::info!("Ops session activated {session_id:?}"); + continue; + } + }; + + match task { + ComputeTask::RegisterOperation(op) => { + stream.register_operation(op).await; + } + ComputeTask::RegisterTensor(id, data) => { + stream.register_tensor(id, data).await; + } + ComputeTask::ReadTensor(tensor) => { + stream.read_tensor(connection_id, tensor).await; + } + ComputeTask::SyncBackend => { + stream.sync(connection_id).await; + } + ComputeTask::RegisterTensorRemote(tensor, new_id) => { + stream.register_tensor_remote(tensor, new_id).await; + } + ComputeTask::ExposeTensorRemote { + tensor, + count, + transfer_id, + } => { + stream + .expose_tensor_remote(tensor, count, transfer_id) + .await; + } + ComputeTask::Seed(seed) => { + stream.seed(seed).await; + } + ComputeTask::SupportsDType(dtype) => { + stream.supports_dtype(connection_id, dtype).await + } + } + } + + log::info!("Closing session {session_id:?}"); + session_manager.close(session_id).await; + } +} + +/// Start the server on the given port and [device](Device). +pub async fn start_websocket_async(device: Device, port: u16) { + let server = WsServer::new(port); + RemoteServer::::start(device, server).await; +} + +#[tokio::main] +/// Start the server on the given port and [device](Device). +pub async fn start_websocket(device: Device, port: u16) { + start_websocket_async::(device, port).await; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/mod.rs new file mode 100644 index 0000000..7b8cf64 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod processor; +pub(crate) mod session; +pub(crate) mod stream; + +mod base; + +pub use base::{start_websocket, start_websocket_async}; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/processor.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/processor.rs new file mode 100644 index 0000000..e59405c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/processor.rs @@ -0,0 +1,132 @@ +use burn_backend::TensorData; +use burn_communication::{ + Protocol, + data_service::{TensorDataService, TensorTransferId}, +}; +use burn_ir::{BackendIr, OperationIr, TensorId, TensorIr}; +use burn_router::{Runner, RunnerClient}; +use burn_std::DType; +use core::marker::PhantomData; +use std::sync::Arc; +use tokio::sync::mpsc::Sender; + +use crate::shared::{ConnectionId, TaskResponse, TaskResponseContent, TensorRemote}; + +/// The goal of the processor is to asynchronously process compute tasks on it own thread. +pub struct Processor +where + B: BackendIr, + P: Protocol, +{ + p: PhantomData, + n: PhantomData

, +} + +pub type Callback = Sender; + +pub enum ProcessorTask { + RegisterOperation(Box), + RegisterTensor(TensorId, TensorData), + RegisterTensorRemote(TensorRemote, TensorId), + ExposeTensorRemote { + tensor: TensorIr, + transfer_id: TensorTransferId, + count: u32, + }, + ReadTensor(ConnectionId, TensorIr, Callback), + Sync(ConnectionId, Callback), + Seed(u64), + SupportsDType(ConnectionId, DType, Callback), + Close, +} + +impl Processor +where + B: BackendIr, + P: Protocol, +{ + pub async fn start( + runner: Runner, + data_service: Arc>, + ) -> Sender { + // channel for tasks to execute + let (task_sender, mut task_rec) = tokio::sync::mpsc::channel(1); + + tokio::spawn(async move { + while let Some(item) = task_rec.recv().await { + match item { + ProcessorTask::RegisterOperation(op) => { + runner.register_op(*op); + } + ProcessorTask::Sync(id, callback) => { + let result = runner.sync(); + callback + .send(TaskResponse { + content: TaskResponseContent::SyncBackend(result), + id, + }) + .await + .unwrap(); + } + ProcessorTask::RegisterTensor(id, data) => { + runner.register_tensor_data_id(id, data); + } + ProcessorTask::RegisterTensorRemote(remote_tensor, new_id) => { + log::info!( + "Registering remote tensor...(id: {:?})", + remote_tensor.transfer_id + ); + let data = data_service + .download_tensor(remote_tensor.address, remote_tensor.transfer_id) + .await + .expect("Can't download tensor: error"); // TODO all these panics should be server errors + runner.register_tensor_data_id(new_id, data); + } + ProcessorTask::ExposeTensorRemote { + tensor, + transfer_id, + count, + } => { + log::info!("Exposing tensor: (id: {transfer_id:?})"); + let data = runner.read_tensor_async(tensor).await; + data_service + .expose_data(data.unwrap(), count, transfer_id) + .await; + } + ProcessorTask::ReadTensor(id, tensor, callback) => { + let tensor = runner.read_tensor_async(tensor).await; + callback + .send(TaskResponse { + content: TaskResponseContent::ReadTensor(tensor), + id, + }) + .await + .unwrap(); + } + ProcessorTask::Close => { + let device = runner.device(); + runner.sync().unwrap(); + core::mem::drop(runner); + B::sync(&device).unwrap(); + break; + } + ProcessorTask::Seed(seed) => runner.seed(seed), + ProcessorTask::SupportsDType(id, dtype, callback) => { + let _result = runner.dtype_usage(dtype); + callback + .send(TaskResponse { + // content: TaskResponseContent::SupportsDType(result), + // TODO: Update to result. + content: TaskResponseContent::SupportsDType(()), + id, + }) + .await + .unwrap(); + } + } + } + }); + + task_sender + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/session.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/session.rs new file mode 100644 index 0000000..401983b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/session.rs @@ -0,0 +1,170 @@ +use burn_backend::tensor::Device; +use burn_communication::{Protocol, data_service::TensorDataService}; +use burn_ir::BackendIr; +use burn_router::Runner; +use burn_std::id::StreamId; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::{ + Mutex, + mpsc::{Receiver, Sender}, +}; + +use crate::shared::{ComputeTask, ConnectionId, SessionId, Task, TaskResponse}; + +use super::stream::Stream; + +/// A session manager control the creation of sessions. +/// +/// Each session manages its own stream, spawning one thread per stream to mimic the same behavior +/// a native backend would have. +pub struct SessionManager +where + B: BackendIr, + P: Protocol, +{ + runner: Runner, + sessions: Mutex>>, + data_service: Arc>, +} + +struct Session +where + B: BackendIr, + P: Protocol, +{ + runner: Runner, + streams: HashMap>, + sender: Sender>, + receiver: Option>>, + data_service: Arc>, +} + +impl SessionManager +where + B: BackendIr, + P: Protocol, +{ + pub fn new(device: Device, data_service: Arc>) -> Self { + Self { + runner: Runner::new(device), + sessions: Mutex::new(Default::default()), + data_service, + } + } + + /// Register a new responder for the session. Only one responder can exist for a session for + /// now. + pub async fn register_responder( + &self, + session_id: SessionId, + ) -> Receiver> { + log::info!("Register responder for session {session_id}"); + let mut sessions = self.sessions.lock().await; + self.register_session(&mut sessions, session_id); + + let session = sessions.get_mut(&session_id).unwrap(); + session.init_responder() + } + + /// Get the stream for the current session and task. + pub async fn stream( + &self, + session_id: &mut Option, + task: Task, + ) -> Option<(Stream, ConnectionId, ComputeTask)> { + let mut sessions = self.sessions.lock().await; + + let session_id = match session_id { + Some(id) => *id, + None => match task { + Task::Init(id) => { + log::info!("Init requester for session {id}"); + *session_id = Some(id); + self.register_session(&mut sessions, id); + return None; + } + _ => panic!("The first message should initialize the session"), + }, + }; + + match sessions.get_mut(&session_id) { + Some(session) => { + let (task, connection_id) = match task { + Task::Compute(task, connection_id) => (task, connection_id), + _ => panic!("Only support compute tasks."), + }; + let stream = session.select(connection_id.stream_id).await; + Some((stream, connection_id, task)) + } + None => panic!("To be initialized"), + } + } + + /// Close the session with the given id. + pub async fn close(&self, session_id: Option) { + if let Some(id) = session_id { + let mut sessions = self.sessions.lock().await; + if let Some(session) = sessions.get_mut(&id) { + session.close().await; + } + } + } + + fn register_session(&self, sessions: &mut HashMap>, id: SessionId) { + sessions.entry(id).or_insert_with(|| { + log::info!("Creating a new session {id}"); + + Session::new(self.runner.clone(), self.data_service.clone()) + }); + } +} + +impl Session +where + B: BackendIr, + P: Protocol, +{ + fn new(runner: Runner, data_service: Arc>) -> Self { + let (sender, receiver) = tokio::sync::mpsc::channel(1); + + Self { + runner, + streams: Default::default(), + sender, + receiver: Some(receiver), + data_service, + } + } + + fn init_responder(&mut self) -> Receiver> { + let mut receiver = None; + core::mem::swap(&mut receiver, &mut self.receiver); + receiver.expect("Only one responder per session is possible.") + } + + /// Select the current [stream](Stream) based on the given task. + async fn select(&mut self, stream_id: StreamId) -> Stream { + // We return the stream. + match self.streams.get(&stream_id) { + Some(stream) => stream.clone(), + None => { + let stream = Stream::::new( + self.runner.clone(), + self.sender.clone(), + self.data_service.clone(), + ) + .await; + self.streams.insert(stream_id, stream.clone()); + stream + } + } + } + + // Close all streams created in the session. + async fn close(&mut self) { + for (id, stream) in self.streams.drain() { + log::info!("Closing stream {id}"); + stream.close().await; + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/stream.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/stream.rs new file mode 100644 index 0000000..7313ffd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/server/stream.rs @@ -0,0 +1,134 @@ +use core::marker::PhantomData; +use std::sync::Arc; + +use crate::shared::{ConnectionId, TaskResponse, TensorRemote}; + +use super::processor::{Processor, ProcessorTask}; +use burn_backend::TensorData; +use burn_communication::{ + Protocol, + data_service::{TensorDataService, TensorTransferId}, +}; +use burn_ir::{BackendIr, OperationIr, TensorId, TensorIr}; +use burn_router::Runner; +use burn_std::DType; +use tokio::sync::mpsc::{Receiver, Sender}; + +/// A stream makes sure all operations registered are executed in the order they were sent to the +/// server, potentially waiting to reconstruct consistency. +#[derive(Clone)] +pub struct Stream +where + B: BackendIr, + P: Protocol, +{ + compute_sender: Sender, + writer_sender: Sender>, + _p: PhantomData, + _n: PhantomData

, +} + +impl Stream +where + B: BackendIr, + P: Protocol, +{ + pub async fn new( + runner: Runner, + writer_sender: Sender>, + data_service: Arc>, + ) -> Self { + let sender = Processor::::start(runner, data_service).await; + + Self { + compute_sender: sender, + writer_sender, + _p: PhantomData, + _n: PhantomData, + } + } + + pub async fn register_operation(&self, op: Box) { + self.compute_sender + .send(ProcessorTask::RegisterOperation(op)) + .await + .unwrap(); + } + + pub async fn register_tensor(&self, tensor_id: TensorId, data: TensorData) { + self.compute_sender + .send(ProcessorTask::RegisterTensor(tensor_id, data)) + .await + .unwrap(); + } + + pub async fn register_tensor_remote(&self, tensor: TensorRemote, new_id: TensorId) { + self.compute_sender + .send(ProcessorTask::RegisterTensorRemote(tensor, new_id)) + .await + .unwrap(); + } + + pub async fn expose_tensor_remote( + &self, + tensor: TensorIr, + count: u32, + transfer_id: TensorTransferId, + ) { + self.compute_sender + .send(ProcessorTask::ExposeTensorRemote { + tensor, + count, + transfer_id, + }) + .await + .unwrap(); + } + + pub async fn read_tensor(&self, id: ConnectionId, desc: TensorIr) { + let (callback_sender, callback_rec) = tokio::sync::mpsc::channel(1); + + self.compute_sender + .send(ProcessorTask::ReadTensor(id, desc, callback_sender)) + .await + .unwrap(); + + self.writer_sender.send(callback_rec).await.unwrap(); + } + + pub async fn sync(&self, id: ConnectionId) { + let (callback_sender, callback_rec) = tokio::sync::mpsc::channel(1); + + self.compute_sender + .send(ProcessorTask::Sync(id, callback_sender)) + .await + .unwrap(); + + self.writer_sender.send(callback_rec).await.unwrap(); + } + + pub async fn close(&self) { + self.compute_sender + .send(ProcessorTask::Close) + .await + .unwrap(); + } + + pub async fn seed(&self, seed: u64) { + self.compute_sender + .send(ProcessorTask::Seed(seed)) + .await + .unwrap(); + } + + pub async fn supports_dtype(&self, id: ConnectionId, dtype: DType) { + let (callback_sender, callback_rec) = tokio::sync::mpsc::channel(1); + + self.compute_sender + .send(ProcessorTask::SupportsDType(id, dtype, callback_sender)) + .await + .unwrap(); + + self.writer_sender.send(callback_rec).await.unwrap(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/mod.rs new file mode 100644 index 0000000..27d8b56 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/mod.rs @@ -0,0 +1,7 @@ +mod task; + +#[allow(unused_imports)] +pub(crate) use task::*; + +/// We define the communication protocol here +pub(crate) type RemoteProtocol = burn_communication::websocket::WebSocket; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/task.rs b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/task.rs new file mode 100644 index 0000000..504dd18 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-remote/src/shared/task.rs @@ -0,0 +1,87 @@ +use burn_backend::{ExecutionError, TensorData}; +use burn_communication::{Address, data_service::TensorTransferId}; +use burn_ir::{OperationIr, TensorId, TensorIr}; +use burn_std::{ + DType, + id::{IdGenerator, StreamId}, +}; +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +#[allow(missing_docs)] +#[derive(new, Serialize, Deserialize, Debug, Hash, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)] +pub struct ConnectionId { + pub position: u64, + pub stream_id: StreamId, +} + +/// Unique identifier that can represent a session. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize, PartialOrd, Ord)] +pub struct SessionId { + id: u64, +} + +impl Display for SessionId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "SessionId({})", self.id) + } +} + +impl SessionId { + /// Create a new [session id](SessionId). + #[allow(dead_code)] + pub fn new() -> Self { + Self { + id: IdGenerator::generate(), + } + } +} + +#[allow(missing_docs)] +#[derive(Serialize, Deserialize, Debug)] +pub enum Task { + Compute(ComputeTask, ConnectionId), + Init(SessionId), + Close(SessionId), +} + +#[allow(missing_docs)] +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TensorRemote { + pub transfer_id: TensorTransferId, + pub address: Address, +} + +#[allow(missing_docs)] +#[derive(Serialize, Deserialize, Debug)] +pub enum ComputeTask { + Seed(u64), + RegisterOperation(Box), + RegisterTensor(TensorId, TensorData), + RegisterTensorRemote(TensorRemote, TensorId), + ExposeTensorRemote { + tensor: TensorIr, + count: u32, + transfer_id: TensorTransferId, + }, + ReadTensor(TensorIr), + SyncBackend, + SupportsDType(DType), +} + +#[allow(missing_docs)] +#[derive(Serialize, Deserialize, Debug)] +pub struct TaskResponse { + pub content: TaskResponseContent, + pub id: ConnectionId, +} + +#[allow(missing_docs)] +#[derive(Serialize, Deserialize, Debug)] +pub enum TaskResponseContent { + ReadTensor(Result), + SyncBackend(Result<(), ExecutionError>), + // SupportsDType(DTypeUsageSet), + // TODO: Update to `DTypeUsageSet` when it implements `serde`. + SupportsDType(()), +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-rl/Cargo.toml new file mode 100644 index 0000000..d8f876d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/Cargo.toml @@ -0,0 +1,31 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "RL crate for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-rl" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-rl" +documentation = "https://docs.rs/burn-rl" +version.workspace = true + +[dependencies] +burn-core = { path = "../burn-core", version = "=0.21.0-pre.2", features = [ + "dataset", + "std", +], default-features = false } +burn-optim = { path = "../burn-optim", version = "=0.21.0-pre.2", features = [ + "std", +], default-features = false } + +derive-new.workspace = true +log = { workspace = true } +rand.workspace = true + +[dev-dependencies] +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } + +[lints] +workspace = true diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/README.md b/crates/stable-diffusion-burn/burn-crates/burn-rl/README.md new file mode 100644 index 0000000..0b0221b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/README.md @@ -0,0 +1,6 @@ +# Burn RL + + + + diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/base.rs new file mode 100644 index 0000000..16af869 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/base.rs @@ -0,0 +1,46 @@ +/// The result of taking a step in an environment. +pub struct StepResult { + /// The updated state. + pub next_state: S, + /// The reward. + pub reward: f64, + /// If the environment reached a terminal state. + pub done: bool, + /// If the environment reached its max length. + pub truncated: bool, +} + +/// Trait to be implemented for a RL environment. +pub trait Environment { + /// The type of the state. + type State; + /// The type of actions. + type Action; + + /// The maximum number of step for one episode. + const MAX_STEPS: usize; + + /// Returns the current state. + fn state(&self) -> Self::State; + /// Take a step in the environment given an action. + fn step(&mut self, action: Self::Action) -> StepResult; + /// Reset the environment to an initial state. + fn reset(&mut self); +} + +/// Trait to define how to initialize an environment. +/// By default, any function returning an environment implements it. +pub trait EnvironmentInit: Clone { + /// Initialize the environment. + fn init(&self) -> E; +} + +impl EnvironmentInit for F +where + F: Fn() -> E + Clone, + E: Environment, +{ + fn init(&self) -> E { + (self)() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/mod.rs new file mode 100644 index 0000000..cbcb6ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/environment/mod.rs @@ -0,0 +1,3 @@ +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/lib.rs new file mode 100644 index 0000000..46744ab --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/lib.rs @@ -0,0 +1,166 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! A library for training reinforcement learning agents. + +/// Module for implementing an environment. +pub mod environment; +/// Module for implementing a policy. +pub mod policy; +/// Transition buffer. +pub mod transition_buffer; + +pub use environment::*; +pub use policy::*; +pub use transition_buffer::*; + +#[cfg(test)] +pub(crate) type TestBackend = burn_ndarray::NdArray; + +#[cfg(test)] +pub(crate) mod tests { + use crate::{Batchable, Policy, PolicyState, TestBackend}; + + use burn_core::record::Record; + use burn_core::{self as burn}; + + /// Mock policy for testing + /// + /// Calling `forward()` with a [MockObservation](MockObservation) (list of f32) returns a [MockActionDistribution](MockActionDistribution) + /// containing a list of 0s of the same length as the observation. + /// + /// Calling `action()` with a [MockObservation](MockObservation) (list of f32) returns a [MockAction](MockAction) with a list of actions of the same length as the observation. + /// The actions are all 1 if the call is requested as deterministic, or else 0. + #[derive(Clone)] + pub(crate) struct MockPolicy {} + + impl MockPolicy { + pub fn new() -> Self { + Self {} + } + } + + impl Policy for MockPolicy { + type Observation = MockObservation; + type ActionDistribution = MockActionDistribution; + type Action = MockAction; + type ActionContext = MockActionContext; + type PolicyState = MockPolicyState; + + fn forward(&mut self, obs: Self::Observation) -> Self::ActionDistribution { + let mut dists = vec![]; + + for _ in obs.0 { + dists.push(MockActionDistribution(vec![0.])); + } + MockActionDistribution::batch(dists) + } + + fn action( + &mut self, + obs: Self::Observation, + deterministic: bool, + ) -> (Self::Action, Vec) { + let mut actions = vec![]; + let mut contexts = vec![]; + + for _ in obs.0 { + if deterministic { + actions.push(MockAction(vec![1])); + } else { + actions.push(MockAction(vec![0])); + } + contexts.push(MockActionContext); + } + + (MockAction::batch(actions), contexts) + } + + fn update(&mut self, _update: Self::PolicyState) {} + + fn state(&self) -> Self::PolicyState { + MockPolicyState + } + + fn load_record( + self, + _record: >::Record, + ) -> Self { + self + } + } + + /// Mock observation for testing represented as a vector of f32. Can call `batch()` and `unbatch` on it. + #[derive(Clone)] + pub(crate) struct MockObservation(pub Vec); + + /// Mock action for testing represented as a vector of i32. Can call `batch()` and `unbatch` on it. + #[derive(Clone)] + pub(crate) struct MockAction(pub Vec); + + /// Mock action distribution for testing represented as a vector of i32. Can call `batch()` and `unbatch` on it. + #[derive(Clone)] + pub(crate) struct MockActionDistribution(Vec); + + #[derive(Clone)] + pub(crate) struct MockActionContext; + + /// Mock policy state for testing represented as an arbitrary `usize` that has no effect on the policy. + #[derive(Clone)] + pub(crate) struct MockPolicyState; + + #[derive(Clone, Record)] + pub(crate) struct MockRecord { + item: usize, + } + + impl PolicyState for MockPolicyState { + type Record = MockRecord; + + fn into_record(self) -> Self::Record { + MockRecord { item: 0 } + } + + fn load_record(&self, _record: Self::Record) -> Self { + self.clone() + } + } + + impl Batchable for MockObservation { + fn batch(items: Vec) -> Self { + MockObservation(items.iter().flat_map(|m| m.0.clone()).collect()) + } + + fn unbatch(self) -> Vec { + vec![MockObservation(self.0)] + } + } + + impl Batchable for MockAction { + fn batch(items: Vec) -> Self { + MockAction(items.iter().flat_map(|m| m.0.clone()).collect()) + } + + fn unbatch(self) -> Vec { + let mut actions = vec![]; + for a in self.0 { + actions.push(MockAction(vec![a])); + } + actions + } + } + + impl Batchable for MockActionDistribution { + fn batch(items: Vec) -> Self { + MockActionDistribution(items.iter().flat_map(|m| m.0.clone()).collect()) + } + + fn unbatch(self) -> Vec { + let mut dists = vec![]; + for _ in self.0 { + dists.push(MockActionDistribution(vec![0.])); + } + dists + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/async_policy.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/async_policy.rs new file mode 100644 index 0000000..ca01430 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/async_policy.rs @@ -0,0 +1,485 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc::{self, Sender}, + }, + thread::spawn, +}; + +use burn_core::prelude::Backend; + +use crate::{ActionContext, Batchable, Policy, PolicyState}; + +#[derive(Clone)] +struct PolicyInferenceServer> { + // `num_agents` used to make sure autobatching doesn't block the agents if they are less than the autobatch size. + num_agents: Arc, + max_autobatch_size: usize, + inner_policy: P, + batch_action: Vec>, + batch_logits: Vec>, +} + +impl PolicyInferenceServer +where + B: Backend, + P: Policy, + P::Observation: Clone + Batchable, + P::ActionDistribution: Clone + Batchable, + P::Action: Clone + Batchable, + P::ActionContext: Clone, +{ + pub fn new(max_autobatch_size: usize, inner_policy: P) -> Self { + Self { + num_agents: Arc::new(AtomicUsize::new(0)), + max_autobatch_size, + inner_policy, + batch_action: vec![], + batch_logits: vec![], + } + } + + pub fn push_action(&mut self, item: ActionItem) { + self.batch_action.push(item); + if self.len_actions() + >= self + .num_agents + .load(Ordering::Relaxed) + .min(self.max_autobatch_size) + { + self.flush_actions(); + } + } + + pub fn push_logits(&mut self, item: ForwardItem) { + self.batch_logits.push(item); + if self.len_logits() + >= self + .num_agents + .load(Ordering::Relaxed) + .min(self.max_autobatch_size) + { + self.flush_logits(); + } + } + + pub fn len_actions(&self) -> usize { + self.batch_action.len() + } + + pub fn len_logits(&self) -> usize { + self.batch_logits.len() + } + + pub fn flush_actions(&mut self) { + if self.len_actions() == 0 { + return; + } + let input: Vec<_> = self + .batch_action + .iter() + .map(|m| m.inference_state.clone()) + .collect(); + // Only deterministic if all actions are requested as deterministic. + let deterministic = self.batch_action.iter().all(|item| item.deterministic); + let (actions, context) = self + .inner_policy + .action(P::Observation::batch(input), deterministic); + let actions: Vec<_> = actions.unbatch(); + + for (i, item) in self.batch_action.iter().enumerate() { + item.sender + .send(ActionContext { + context: vec![context[i].clone()], + action: actions[i].clone(), + }) + .expect("Autobatcher should be able to send resulting actions."); + } + self.batch_action.clear(); + } + + pub fn flush_logits(&mut self) { + if self.len_logits() == 0 { + return; + } + let input: Vec<_> = self + .batch_logits + .iter() + .map(|m| m.inference_state.clone()) + .collect(); + let output = self.inner_policy.forward(P::Observation::batch(input)); + let logits: Vec<_> = output.unbatch(); + for (i, item) in self.batch_logits.iter().enumerate() { + item.sender + .send(logits[i].clone()) + .expect("Autobatcher should be able to send resulting probabilities."); + } + self.batch_logits.clear(); + } + + pub fn update_policy(&mut self, policy_update: P::PolicyState) { + if self.len_actions() > 0 { + self.flush_actions(); + } + if self.len_logits() > 0 { + self.flush_logits(); + } + self.inner_policy.update(policy_update); + } + + pub fn state(&self) -> P::PolicyState { + self.inner_policy.state() + } + + pub fn increment_agents(&mut self, num: usize) { + self.num_agents.fetch_add(num, Ordering::Relaxed); + } + + pub fn decrement_agents(&mut self, num: usize) { + self.num_agents.fetch_sub(num, Ordering::Relaxed); + if self.len_actions() + >= self + .num_agents + .load(Ordering::Relaxed) + .min(self.max_autobatch_size) + { + self.flush_actions(); + } + if self.len_logits() + >= self + .num_agents + .load(Ordering::Relaxed) + .min(self.max_autobatch_size) + { + self.flush_logits(); + } + } +} + +enum InferenceMessage> { + ActionMessage(ActionItem), + ForwardMessage(ForwardItem), + PolicyUpdate(P::PolicyState), + PolicyRequest(Sender), + IncrementAgents(usize), + DecrementAgents(usize), +} + +#[derive(Clone)] +struct ActionItem { + sender: Sender>>, + inference_state: S, + deterministic: bool, +} + +#[derive(Clone)] +struct ForwardItem { + sender: Sender, + inference_state: S, +} + +/// An asynchronous policy using an inference server with autobatching. +#[derive(Clone)] +pub struct AsyncPolicy> { + inference_state_sender: Sender>, +} + +impl AsyncPolicy +where + B: Backend, + P: Policy + Clone + Send + 'static, + P::ActionContext: Clone + Send, + P::PolicyState: Send, + P::Observation: Clone + Send + Batchable, + P::ActionDistribution: Clone + Send + Batchable, + P::Action: Clone + Send + Batchable, +{ + /// Create the policy. + /// + /// # Arguments + /// + /// * `autobatch_size` - Number of observations to accumulate before running a pass of inference. + /// * `inner_policy` - The policy used to take actions. + pub fn new(autobatch_size: usize, inner_policy: P) -> Self { + let (sender, receiver) = std::sync::mpsc::channel(); + let mut autobatcher = PolicyInferenceServer::new(autobatch_size, inner_policy.clone()); + spawn(move || { + loop { + match receiver.recv() { + Ok(msg) => match msg { + InferenceMessage::ActionMessage(item) => autobatcher.push_action(item), + InferenceMessage::ForwardMessage(item) => autobatcher.push_logits(item), + InferenceMessage::PolicyUpdate(update) => autobatcher.update_policy(update), + InferenceMessage::PolicyRequest(sender) => sender + .send(autobatcher.state()) + .expect("Autobatcher should be able to send current policy state."), + InferenceMessage::IncrementAgents(num) => autobatcher.increment_agents(num), + InferenceMessage::DecrementAgents(num) => autobatcher.decrement_agents(num), + }, + Err(err) => { + log::error!("Error in AsyncPolicy : {}", err); + break; + } + } + } + }); + + Self { + inference_state_sender: sender, + } + } + + /// Increment the number of agents using the inference server. + pub fn increment_agents(&self, num: usize) { + self.inference_state_sender + .send(InferenceMessage::IncrementAgents(num)) + .expect("Can send message to autobatcher.") + } + + /// Decrement the number of agents using the inference server. + pub fn decrement_agents(&self, num: usize) { + self.inference_state_sender + .send(InferenceMessage::DecrementAgents(num)) + .expect("Can send message to autobatcher.") + } +} + +impl Policy for AsyncPolicy +where + B: Backend, + P: Policy + Send + 'static, +{ + type ActionContext = P::ActionContext; + type PolicyState = P::PolicyState; + + type Observation = P::Observation; + type ActionDistribution = P::ActionDistribution; + type Action = P::Action; + + fn forward(&mut self, states: Self::Observation) -> Self::ActionDistribution { + let (action_sender, action_receiver) = std::sync::mpsc::channel(); + let item = ForwardItem { + sender: action_sender, + inference_state: states, + }; + self.inference_state_sender + .send(InferenceMessage::ForwardMessage(item)) + .expect("Should be able to send message to inference_server"); + action_receiver + .recv() + .expect("AsyncPolicy should receive queued probabilities.") + } + + fn action( + &mut self, + states: Self::Observation, + deterministic: bool, + ) -> (Self::Action, Vec) { + let (action_sender, action_receiver) = std::sync::mpsc::channel(); + let item = ActionItem { + sender: action_sender, + inference_state: states, + deterministic, + }; + self.inference_state_sender + .send(InferenceMessage::ActionMessage(item)) + .expect("should be able to send message to inference_server."); + let action = action_receiver + .recv() + .expect("AsyncPolicy should receive queued actions."); + (action.action, action.context) + } + + fn update(&mut self, update: Self::PolicyState) { + self.inference_state_sender + .send(InferenceMessage::PolicyUpdate(update)) + .expect("AsyncPolicy should be able to send policy state.") + } + + fn state(&self) -> Self::PolicyState { + let (sender, receiver) = mpsc::channel(); + self.inference_state_sender + .send(InferenceMessage::PolicyRequest(sender)) + .expect("should be able to send message to inference_server."); + receiver + .recv() + .expect("AsyncPolicy should be able to receive policy state.") + } + + fn load_record(self, _record: >::Record) -> Self { + // Not needed for now + todo!() + } +} + +#[cfg(test)] +#[allow(clippy::needless_range_loop)] +mod tests { + use std::thread::JoinHandle; + use std::time::Duration; + + use crate::TestBackend; + use crate::tests::{MockAction, MockObservation, MockPolicy}; + + use super::*; + + #[test] + fn test_multiple_actions_before_flush() { + fn launch_thread( + policy: &AsyncPolicy, + handles: &mut Vec>, + ) { + let mut thread_policy = policy.clone(); + let handle = spawn(move || { + thread_policy.action(MockObservation(vec![0.]), false); + }); + handles.push(handle); + } + + let policy = AsyncPolicy::new(8, MockPolicy::new()); + policy.increment_agents(1000); + + let mut handles = vec![]; + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + assert!(!handles[0].is_finished()); + + for _ in 0..6 { + launch_thread(&policy, &mut handles); + } + std::thread::sleep(Duration::from_millis(10)); + for i in 0..7 { + assert!(!handles[i].is_finished()); + } + + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + for i in 0..8 { + assert!(handles[i].is_finished()); + } + + let mut handles = vec![]; + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + assert!(!handles[0].is_finished()); + } + + #[test] + fn test_multiple_forward_before_flush() { + fn launch_thread( + policy: &AsyncPolicy, + handles: &mut Vec>, + ) { + let mut thread_policy = policy.clone(); + let handle = spawn(move || { + thread_policy.forward(MockObservation(vec![0.])); + }); + handles.push(handle); + } + + let policy = AsyncPolicy::new(8, MockPolicy::new()); + policy.increment_agents(1000); + + let mut handles = vec![]; + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + assert!(!handles[0].is_finished()); + + for _ in 0..6 { + launch_thread(&policy, &mut handles); + } + std::thread::sleep(Duration::from_millis(10)); + for i in 0..7 { + assert!(!handles[i].is_finished()); + } + + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + for i in 0..8 { + assert!(handles[i].is_finished()); + } + + let mut handles = vec![]; + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + assert!(!handles[0].is_finished()); + } + + #[test] + fn test_async_policy_deterministic_behaviour() { + fn launch_thread( + policy: &AsyncPolicy, + handles: &mut Vec>, + deterministic: bool, + ) { + let mut thread_policy = policy.clone(); + let handle = spawn(move || { + let (action, _) = thread_policy.action(MockObservation(vec![0.]), deterministic); + action + }); + handles.push(handle); + } + + let policy = AsyncPolicy::new(2, MockPolicy::new()); + policy.increment_agents(1000); + + let mut handles = vec![]; + launch_thread(&policy, &mut handles, true); + launch_thread(&policy, &mut handles, false); + for _ in 0..2 { + let action = handles.pop().unwrap().join().unwrap(); + assert_eq!(action.0, vec![0]); + } + + let mut handles = vec![]; + launch_thread(&policy, &mut handles, true); + launch_thread(&policy, &mut handles, true); + for _ in 0..2 { + let action = handles.pop().unwrap().join().unwrap(); + assert_eq!(action.0, vec![1]); + } + } + + #[test] + fn flush_when_running_agents_smaller_than_autobatch_size() { + fn launch_thread( + policy: &AsyncPolicy, + handles: &mut Vec>, + ) { + let mut thread_policy = policy.clone(); + let handle = spawn(move || { + thread_policy.action(MockObservation(vec![0.]), false); + }); + handles.push(handle); + } + + let policy = AsyncPolicy::new(8, MockPolicy::new()); + policy.increment_agents(3); + + let mut handles = vec![]; + launch_thread(&policy, &mut handles); + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + assert!(!handles[0].is_finished()); + assert!(!handles[1].is_finished()); + + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + for i in 0..3 { + assert!(handles[i].is_finished()); + } + + let mut handles = vec![]; + launch_thread(&policy, &mut handles); + launch_thread(&policy, &mut handles); + std::thread::sleep(Duration::from_millis(10)); + assert!(!handles[0].is_finished()); + assert!(!handles[1].is_finished()); + + policy.decrement_agents(1); + std::thread::sleep(Duration::from_millis(10)); + assert!(handles[0].is_finished()); + assert!(handles[1].is_finished()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/base.rs new file mode 100644 index 0000000..26873f2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/base.rs @@ -0,0 +1,108 @@ +use derive_new::new; + +use burn_core::{prelude::*, record::Record, tensor::backend::AutodiffBackend}; + +use crate::TransitionBatch; + +/// An action along with additional context about the decision. +#[derive(Clone, new)] +pub struct ActionContext { + /// The context. + pub context: C, + /// The action. + pub action: A, +} + +/// The state of a policy. +pub trait PolicyState { + /// The type of the record. + type Record: Record; + + /// Convert the state to a record. + fn into_record(self) -> Self::Record; + /// Load the state from a record. + fn load_record(&self, record: Self::Record) -> Self; +} + +/// Trait for a RL policy. +pub trait Policy: Clone { + /// The observation given as input to the policy. + type Observation; + /// The action distribution parameters defining how the action will be sampled. + type ActionDistribution; + /// The action. + type Action; + + /// Additional context on the policy's decision. + type ActionContext; + /// The current parameterization of the policy. + type PolicyState: PolicyState; + + /// Produces the action distribution from a batch of observations. + fn forward(&mut self, obs: Self::Observation) -> Self::ActionDistribution; + /// Gives the action from a batch of observations. + fn action( + &mut self, + obs: Self::Observation, + deterministic: bool, + ) -> (Self::Action, Vec); + + /// Update the policy's parameters. + fn update(&mut self, update: Self::PolicyState); + /// Returns the current parameterization. + fn state(&self) -> Self::PolicyState; + + /// Loads the policy parameters from a record. + fn load_record(self, record: >::Record) -> Self; +} + +/// Trait for a type that can be batched and unbatched (split). +pub trait Batchable: Sized { + /// Create a batch from a list of items. + fn batch(value: Vec) -> Self; + /// Create a list from batched items. + fn unbatch(self) -> Vec; +} + +/// A training output. +pub struct RLTrainOutput { + /// The policy. + pub policy: P, + /// The item. + pub item: TO, +} + +/// Batched transitions for a PolicyLearner. +pub type LearnerTransitionBatch = + TransitionBatch>::Observation,

>::Action>; + +/// Learner for a policy. +pub trait PolicyLearner +where + B: AutodiffBackend, + >::Observation: Clone + Batchable, + >::ActionDistribution: Clone + Batchable, + >::Action: Clone + Batchable, +{ + /// Additional context of a training step. + type TrainContext; + /// The policy to train. + type InnerPolicy: Policy; + /// The record of the learner. + type Record: Record; + + /// Execute a training step on the policy. + fn train( + &mut self, + input: LearnerTransitionBatch, + ) -> RLTrainOutput>::PolicyState>; + /// Returns the learner's current policy. + fn policy(&self) -> Self::InnerPolicy; + /// Update the learner's policy. + fn update_policy(&mut self, update: Self::InnerPolicy); + + /// Convert the learner's state into a record. + fn record(&self) -> Self::Record; + /// Load the learner's state from a record. + fn load_record(self, record: Self::Record) -> Self; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/mod.rs new file mode 100644 index 0000000..fa7e967 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/policy/mod.rs @@ -0,0 +1,5 @@ +mod async_policy; +mod base; + +pub use async_policy::*; +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/base.rs new file mode 100644 index 0000000..264ae0a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/base.rs @@ -0,0 +1,244 @@ +use burn_core::{Tensor, prelude::Backend, tensor::Distribution}; +use derive_new::new; + +use super::SliceAccess; + +/// A state transition in an environment. +#[derive(Clone, new)] +pub struct Transition { + /// The initial state. + pub state: S, + /// The state after the step was taken. + pub next_state: S, + /// The action taken in the step. + pub action: A, + /// The reward. + pub reward: Tensor, + /// If the environment has reached a terminal state. + pub done: Tensor, +} + +/// A batch of transitions. +pub struct TransitionBatch { + /// Batched initial states. + pub states: SB, + /// Batched resulting states. + pub next_states: SB, + /// Batched actions. + pub actions: AB, + /// Batched rewards. + pub rewards: Tensor, + /// Batched flags for terminal states. + pub dones: Tensor, +} + +/// A tensor-backed circular buffer for transitions. +/// +/// Uses [`SliceAccess`] to store state and action batches in contiguous +/// tensor storage, enabling efficient random sampling via `select`. +/// The buffer lazily initializes its storage on the first `push` call. +pub struct TransitionBuffer, AB: SliceAccess> { + states: Option, + next_states: Option, + actions: Option, + rewards: Option>, + dones: Option>, + capacity: usize, + write_head: usize, + len: usize, + device: B::Device, +} + +impl, AB: SliceAccess> TransitionBuffer { + /// Creates a new buffer. Storage is lazily allocated on the first `push`. + pub fn new(capacity: usize, device: &B::Device) -> Self { + Self { + states: None, + next_states: None, + actions: None, + rewards: None, + dones: None, + capacity, + write_head: 0, + len: 0, + device: device.clone(), + } + } + + fn ensure_init(&mut self, state: &SB, next_state: &SB, action: &AB) { + if self.states.is_none() { + self.states = Some(SB::zeros_like(state, self.capacity, &self.device)); + self.next_states = Some(SB::zeros_like(next_state, self.capacity, &self.device)); + self.actions = Some(AB::zeros_like(action, self.capacity, &self.device)); + self.rewards = Some(Tensor::zeros([self.capacity, 1], &self.device)); + self.dones = Some(Tensor::zeros([self.capacity, 1], &self.device)); + } + } + + /// Add a transition, overwriting the oldest if full. + pub fn push(&mut self, state: SB, next_state: SB, action: AB, reward: f32, done: bool) { + self.ensure_init(&state, &next_state, &action); + + let idx = self.write_head % self.capacity; + + self.states + .as_mut() + .unwrap() + .slice_assign_inplace(idx, state); + self.next_states + .as_mut() + .unwrap() + .slice_assign_inplace(idx, next_state); + self.actions + .as_mut() + .unwrap() + .slice_assign_inplace(idx, action); + + let reward = Tensor::from_data([[reward]], &self.device); + self.rewards + .as_mut() + .unwrap() + .inplace(|r| r.slice_assign(idx..idx + 1, reward)); + + let done_val = if done { 1.0f32 } else { 0.0 }; + let done = Tensor::from_data([[done_val]], &self.device); + self.dones + .as_mut() + .unwrap() + .inplace(|d| d.slice_assign(idx..idx + 1, done)); + + self.write_head += 1; + if self.len < self.capacity { + self.len += 1; + } + } + + /// Sample a random batch of transitions. + pub fn sample(&self, batch_size: usize) -> TransitionBatch { + assert!(batch_size <= self.len, "batch_size exceeds buffer length"); + + let indices = Tensor::::random( + [batch_size], + Distribution::Uniform(0.0, self.len as f64), + &self.device, + ) + .int(); + + TransitionBatch { + states: self + .states + .as_ref() + .unwrap() + .clone() + .select(0, indices.clone()), + next_states: self + .next_states + .as_ref() + .unwrap() + .clone() + .select(0, indices.clone()), + actions: self + .actions + .as_ref() + .unwrap() + .clone() + .select(0, indices.clone()), + rewards: self + .rewards + .as_ref() + .unwrap() + .clone() + .select(0, indices.clone()), + dones: self.dones.as_ref().unwrap().clone().select(0, indices), + } + } + + /// Current number of stored transitions. + pub fn len(&self) -> usize { + self.len + } + + /// Whether the buffer is empty. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Buffer capacity. + pub fn capacity(&self) -> usize { + self.capacity + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + type TB = Tensor; + + fn push_transition( + buffer: &mut TransitionBuffer, + device: &::Device, + val: f32, + ) { + let state = Tensor::::from_data([[val, val]], device); + let next_state = Tensor::::from_data([[val + 1.0, val + 1.0]], device); + let action = Tensor::::from_data([[val]], device); + buffer.push(state, next_state, action, val, false); + } + + #[test] + fn push_increment_len() { + let device = Default::default(); + let mut buffer = TransitionBuffer::::new(5, &device); + + assert_eq!(buffer.len(), 0); + assert!(buffer.is_empty()); + + push_transition(&mut buffer, &device, 1.0); + assert_eq!(buffer.len(), 1); + + push_transition(&mut buffer, &device, 2.0); + assert_eq!(buffer.len(), 2); + } + + #[test] + fn push_overwrites_when_full() { + let device = Default::default(); + let mut buffer = TransitionBuffer::::new(3, &device); + + for i in 0..5 { + push_transition(&mut buffer, &device, i as f32); + } + + assert_eq!(buffer.len(), 3); + assert_eq!(buffer.capacity(), 3); + } + + #[test] + fn sample_returns_correct_shapes() { + let device = Default::default(); + let mut buffer = TransitionBuffer::::new(10, &device); + + for i in 0..5 { + push_transition(&mut buffer, &device, i as f32); + } + + let batch = buffer.sample(3); + assert_eq!(batch.states.dims(), [3, 2]); + assert_eq!(batch.next_states.dims(), [3, 2]); + assert_eq!(batch.actions.dims(), [3, 1]); + assert_eq!(batch.rewards.dims(), [3, 1]); + assert_eq!(batch.dones.dims(), [3, 1]); + } + + #[test] + #[should_panic(expected = "batch_size exceeds buffer length")] + fn sample_panics_when_batch_too_large() { + let device = Default::default(); + let mut buffer = TransitionBuffer::::new(5, &device); + + push_transition(&mut buffer, &device, 1.0); + buffer.sample(5); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/mod.rs new file mode 100644 index 0000000..806834d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod slice_access; + +pub use base::*; +pub use slice_access::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/slice_access.rs b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/slice_access.rs new file mode 100644 index 0000000..65b3f51 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rl/src/transition_buffer/slice_access.rs @@ -0,0 +1,36 @@ +use burn_core::prelude::*; + +/// Trait for types that support tensor-like slice operations, +/// enabling storage in a [`TransitionBuffer`](super::TransitionBuffer). +/// +/// Implement this trait for any type that wraps tensors and can be stored +/// in a replay buffer. The buffer uses these operations for: +/// - Pre-allocating storage (`zeros_like`) +/// - Writing transitions (`slice_assign_inplace`) +/// - Sampling batches (`select`) +pub trait SliceAccess: Clone + Sized { + /// Create zeroed storage matching the shape of `sample` but with `capacity` rows + /// along the first dimension. + fn zeros_like(sample: &Self, capacity: usize, device: &B::Device) -> Self; + + /// Select rows at the given indices along the specified dimension. + fn select(self, dim: usize, indices: Tensor) -> Self; + + /// Assign `value` at row `index` along the first dimension, in place. + fn slice_assign_inplace(&mut self, index: usize, value: Self); +} + +impl SliceAccess for Tensor { + fn zeros_like(sample: &Self, capacity: usize, device: &B::Device) -> Self { + let feature_dim = sample.dims()[1]; + Tensor::zeros([capacity, feature_dim], device) + } + + fn select(self, dim: usize, indices: Tensor) -> Self { + Tensor::select(self, dim, indices) + } + + fn slice_assign_inplace(&mut self, index: usize, value: Self) { + self.inplace(|t| t.slice_assign(index..index + 1, value)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rocm/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-rocm/Cargo.toml new file mode 100644 index 0000000..c97930e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rocm/Cargo.toml @@ -0,0 +1,42 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "ROCm HIP backend for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "gpu", "rocm", "hip"] +license.workspace = true +name = "burn-rocm" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-rocm" +documentation = "https://docs.rs/burn-rocm" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["fusion", "burn-cubecl/default", "cubecl/default"] +tracing = [ + "cubecl/tracing", + "burn-cubecl/tracing", + "burn-backend/tracing", + "burn-fusion?/tracing", +] + +fusion = ["burn-fusion", "burn-cubecl/fusion"] +autotune = ["burn-cubecl/autotune"] +autotune-checks = ["burn-cubecl/autotune-checks"] +doc = ["burn-cubecl/doc"] +std = ["burn-cubecl/std", "cubecl/std"] + +[dependencies] +cubecl = { workspace = true, features = ["hip"] } +burn-cubecl = { path = "../burn-cubecl", version = "=0.21.0-pre.2", default-features = true } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", features = [ + "cubecl-hip", +] } +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rocm/README.md b/crates/stable-diffusion-burn/burn-crates/burn-rocm/README.md new file mode 100644 index 0000000..583bbec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rocm/README.md @@ -0,0 +1,7 @@ +# burn-rocm + +Backend using ROCm HIP runtime. + +To execute the tests for this backend set an environment variable called `ROCM_PATH` or `CUBECL_ROCM_PATH` to the installation path of ROCm. It is often `/opt/rocm`. + +For now this backend requires the version `6.2.2` of ROCm or a compatible version. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-rocm/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-rocm/src/lib.rs new file mode 100644 index 0000000..0019b15 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-rocm/src/lib.rs @@ -0,0 +1,14 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] +extern crate alloc; + +use burn_cubecl::CubeBackend; + +pub use cubecl::hip::AmdDevice as RocmDevice; + +use cubecl::hip::HipRuntime; + +#[cfg(not(feature = "fusion"))] +pub type Rocm = CubeBackend; + +#[cfg(feature = "fusion")] +pub type Rocm = burn_fusion::Fusion>; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-router/Cargo.toml new file mode 100644 index 0000000..a63a469 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/Cargo.toml @@ -0,0 +1,48 @@ +[package] +authors = [ + "laggui ", + "nathanielsimard ", +] +categories = ["science"] +description = "Multi-backend router decorator for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-router" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-router" +documentation = "https://docs.rs/burn-router" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std"] +std = ["burn-backend/std", "burn-std/std", "burn-ir/std"] +doc = ["default"] +tracing = [ + "burn-backend/tracing", + "burn-ir/tracing", + "burn-std/tracing", +] + +[dependencies] +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2", default-features = false } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } +hashbrown = { workspace = true } +spin = { workspace = true } +log = { workspace = true } + +[dev-dependencies] +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", default-features = false } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", default-features = false, features = [ + "std", +] } + + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/README.md b/crates/stable-diffusion-burn/burn-crates/burn-router/README.md new file mode 100644 index 0000000..be3e69a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/README.md @@ -0,0 +1,3 @@ +# Burn Router + +A multi-backend extension that forwards the tensor operations to the appropriate backend. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/backend.rs new file mode 100644 index 0000000..1dba2a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/backend.rs @@ -0,0 +1,75 @@ +use super::{RouterTensor, RunnerChannel, RunnerClient, get_client}; +use alloc::{format, string::String}; +use burn_backend::{Backend, DType, ExecutionError, QTensorPrimitive, quantization::QuantScheme}; +use core::marker::PhantomData; + +/// A backend that forwards the tensor operations to the appropriate backend (given multiple backends). +pub struct BackendRouter { + r: PhantomData, +} + +impl core::fmt::Debug for BackendRouter { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("router")) + } +} + +impl Clone for BackendRouter { + fn clone(&self) -> Self { + Self { r: PhantomData } + } +} + +impl Default for BackendRouter { + fn default() -> Self { + Self { r: PhantomData } + } +} + +impl QTensorPrimitive for RouterTensor { + fn scheme(&self) -> &QuantScheme { + if let DType::QFloat(scheme) = &self.dtype { + scheme + } else { + // TODO: maybe `tensor.scheme()` should return an option + panic!("Expected quantized float dtype, got {:?}", self.dtype) + } + } +} + +impl Backend for BackendRouter { + type Device = R::Device; + + type FloatTensorPrimitive = RouterTensor; + + type FloatElem = R::FloatElem; + + type IntTensorPrimitive = RouterTensor; + + type IntElem = R::IntElem; + + type BoolTensorPrimitive = RouterTensor; + + type BoolElem = R::BoolElem; + + type QuantizedTensorPrimitive = RouterTensor; + + fn name(device: &Self::Device) -> String { + format!("router<{}>", R::name(device)) + } + + fn seed(device: &Self::Device, seed: u64) { + let client = get_client::(device); + client.seed(seed); + } + + fn sync(device: &Self::Device) -> Result<(), ExecutionError> { + let client = get_client::(device); + client.sync() + } + + fn dtype_usage(device: &Self::Device, dtype: DType) -> burn_backend::DTypeUsageSet { + let client = get_client::(device); + client.dtype_usage(dtype) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/base.rs new file mode 100644 index 0000000..8e61854 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/base.rs @@ -0,0 +1,32 @@ +use burn_backend::{Shape, backend::DeviceOps}; + +/// Allows tensors to be transferred between multiple backends. +pub trait MultiBackendBridge: Send + Sync + 'static { + /// The type that can be used to point to a tensor of any kind. + type TensorHandle; + /// Device type used by the backends. + type Device: DeviceOps; + + /// Change the backend of the given float tensor. + fn change_backend_float( + tensor: Self::TensorHandle, + shape: Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle; + + /// Change the backend of the given int tensor. + fn change_backend_int( + tensor: Self::TensorHandle, + shape: Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle; + + /// Change the backend of the given bool tensor. + fn change_backend_bool( + tensor: Self::TensorHandle, + shape: Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle; + + // TODO: change_backend_quantized +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/byte.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/byte.rs new file mode 100644 index 0000000..18f0d05 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/byte.rs @@ -0,0 +1,6 @@ +use core::marker::PhantomData; + +/// Simply transfers tensors between backends via the underlying [tensor data](burn_backend::TensorData). +pub struct ByteBridge { + backends: PhantomData, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/mod.rs new file mode 100644 index 0000000..d43da0e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/bridge/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod byte; + +pub use base::*; +pub use byte::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/base.rs new file mode 100644 index 0000000..b2cb350 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/base.rs @@ -0,0 +1,66 @@ +use alloc::string::String; +use burn_backend::{DType, Element, Shape, backend::DeviceOps}; +use burn_ir::TensorIr; + +use crate::{MultiBackendBridge, RouterTensor, RunnerClient, get_client}; + +/// Type alias for `
::TensorHandle`. +pub type TensorHandle
=
::TensorHandle; + +/// Defines the connection channel and operations for a setup with multiple backend runner clients. +pub trait RunnerChannel: Clone + Send + Sync + 'static + Sized { + /// Device type. + type Device: DeviceOps; + /// A bridge that can transfer tensors between multiple backends. + type Bridge: MultiBackendBridge; + /// Client type. + type Client: RunnerClient; + /// Float element type. + type FloatElem: Element; + /// Int element type. + type IntElem: Element; + /// Bool element type. + type BoolElem: Element; + + /// Name of the channel. + fn name(device: &Self::Device) -> String; + + /// Initialize a new client for the given device. + fn init_client(device: &Self::Device) -> Self::Client; + + /// Get the tensor handle corresponding to the [tensor representation](TensorIr). + fn get_tensor_handle(tensor: &TensorIr, client: &Self::Client) -> TensorHandle; + + /// Create a tensor with the given handle and shape. + fn register_tensor( + client: &Self::Client, + handle: TensorHandle, + shape: Shape, + dtype: DType, + ) -> RouterTensor; + + /// Change the tensor to a different client backend. + fn change_client_backend( + tensor: RouterTensor, + device: &Self::Device, // target device + ) -> RouterTensor { + // Get tensor handle from current client + let original_client = tensor.client.clone(); + let desc = tensor.into_ir(); + let mut handle = Self::get_tensor_handle(&desc, &original_client); + + if desc.dtype.is_float() { + handle = Self::Bridge::change_backend_float(handle, desc.shape.clone(), device); + } else if desc.dtype.is_int() { + handle = Self::Bridge::change_backend_int(handle, desc.shape.clone(), device); + } else if desc.dtype.is_bool() { + handle = Self::Bridge::change_backend_bool(handle, desc.shape.clone(), device); + } else { + unimplemented!() + } + + // Register tensor handle on target client + let target_client = get_client::(device); + Self::register_tensor(&target_client, handle, desc.shape, desc.dtype) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/direct.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/direct.rs new file mode 100644 index 0000000..a0f8814 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/direct.rs @@ -0,0 +1,16 @@ +use core::marker::PhantomData; + +/// A local channel with direct connection to the backend runner clients. +pub struct DirectChannel { + backends: PhantomData, + bridge: PhantomData, +} + +impl Clone for DirectChannel { + fn clone(&self) -> Self { + Self { + backends: self.backends, + bridge: self.bridge, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/mod.rs new file mode 100644 index 0000000..5617df4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/channel/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod direct; + +pub use base::*; +pub use direct::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/client/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/client/base.rs new file mode 100644 index 0000000..71efbb6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/client/base.rs @@ -0,0 +1,128 @@ +use crate::{RouterTensor, RunnerChannel}; +use alloc::boxed::Box; +use alloc::vec::Vec; +use burn_backend::{ + DType, TensorData, + backend::{DeviceId, DeviceOps, ExecutionError}, +}; +use burn_ir::{OperationIr, TensorId, TensorIr}; +use burn_std::future::DynFut; +use core::ops::DerefMut; +use hashbrown::HashMap; +use spin::Mutex; + +/// Type alias for `::Client`. +pub type Client = ::Client; +pub(crate) static CLIENTS: RunnerClientLocator = RunnerClientLocator::new(); + +type Key = (core::any::TypeId, DeviceId); + +/// Define how to interact with the runner. +pub trait RunnerClient: Clone + Send + Sync + Sized { + /// Device type. + type Device: DeviceOps; + + /// Register a new tensor operation to be executed by the (runner) server. + fn register_op(&self, op: OperationIr); + /// Register a new tensor operation to be executed by the (runner) server. + /// + /// Returns the new (uninitialized) output tensor(s) generated by the registered operation. + fn register(&self, op: OperationIr) -> Vec> { + let out = op + .outputs() + .map(|output| { + RouterTensor::new(output.id, output.shape.clone(), output.dtype, self.clone()) + }) + .collect(); + self.register_op(op); + + out + } + /// Read the values contained by a tensor. + fn read_tensor_async(&self, tensor: TensorIr) -> DynFut>; + /// Sync the runner, ensure that all computations are finished. + fn sync(&self) -> Result<(), ExecutionError>; + /// Create a new (uninitialized) empty tensor and returns its corresponding [tensor id](TensorId). + fn create_empty_handle(&self) -> TensorId; + /// Create a new [RouterTensor] from the tensor data. + fn register_tensor_data(&self, data: TensorData) -> RouterTensor; + /// Get the current device used by all operations handled by this client. + fn device(&self) -> Self::Device; + /// Seed the runner. + fn seed(&self, seed: u64); + /// Returns the supported data type usage set + fn dtype_usage(&self, dtype: DType) -> burn_backend::DTypeUsageSet; +} + +pub(crate) struct RunnerClientLocator { + clients: Mutex>>>, +} + +/// Get the client for the given device +pub fn get_client(device: &R::Device) -> Client { + CLIENTS.client::(device) +} + +/// Initialize a new client for the given device. +/// +/// If a (global) seed was previously set, the client seed is set. +fn new_client(device: &R::Device) -> Client { + R::init_client(device) +} + +impl RunnerClientLocator { + /// Create a new client locator. + pub const fn new() -> Self { + Self { + clients: Mutex::new(None), + } + } + + /// Get the runner client for the given device. + /// + /// If a client isn't already initialized, it is created. + pub fn client(&self, device: &R::Device) -> Client { + let device_id = device.id(); + let client_id = (core::any::TypeId::of::(), device_id); + let mut clients = self.clients.lock(); + + if clients.is_none() { + let client = new_client::(device); + Self::register_inner::(client_id, client, &mut clients); + } + + match clients.deref_mut() { + Some(clients) => match clients.get(&client_id) { + Some(client) => { + let client: &Client = client.downcast_ref().unwrap(); + client.clone() + } + None => { + let client = new_client::(device); + let any = Box::new(client.clone()); + clients.insert(client_id, any); + client + } + }, + _ => unreachable!(), + } + } + + fn register_inner( + key: Key, + client: Client, + clients: &mut Option>>, + ) { + if clients.is_none() { + *clients = Some(HashMap::new()); + } + + if let Some(clients) = clients { + if clients.contains_key(&key) { + panic!("Client already created for device {key:?}"); + } + + clients.insert(key, Box::new(client)); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/client/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/client/mod.rs new file mode 100644 index 0000000..cbcb6ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/client/mod.rs @@ -0,0 +1,3 @@ +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/lib.rs new file mode 100644 index 0000000..21b63d2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/lib.rs @@ -0,0 +1,49 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![recursion_limit = "138"] + +//! Burn multi-backend router. + +mod backend; +mod bridge; +mod channel; +mod client; +mod ops; +mod runner; +mod tensor; +mod types; + +pub use backend::*; +pub use bridge::*; +pub use channel::*; +pub use client::*; +pub use runner::*; +pub use tensor::*; +pub use types::*; + +/// A local channel with a simple byte bridge between backends. +/// It transfers tensors between backends via the underlying [tensor data](burn_backend::TensorData). +pub type DirectByteChannel = DirectChannel>; + +/// Router backend. +/// +/// # Example +/// +/// ```ignore +/// type MyBackend = Router<(NdArray, Wgpu)>; +/// ``` +pub type Router = BackendRouter>; + +extern crate alloc; + +#[cfg(test)] +#[allow(unused)] +mod tests { + use crate::BackendRouter; + use crate::DirectByteChannel; + + pub type TestBackend1 = burn_ndarray::NdArray; + pub type TestBackend2 = burn_wgpu::Wgpu; + pub type TestBackend = BackendRouter>; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/activation.rs new file mode 100644 index 0000000..97d92cd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/activation.rs @@ -0,0 +1,4 @@ +use crate::{BackendRouter, RunnerChannel}; +use burn_backend::ops::ActivationOps; + +impl ActivationOps for BackendRouter {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/binary.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/binary.rs new file mode 100644 index 0000000..3af45fd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/binary.rs @@ -0,0 +1,69 @@ +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_float_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.lhs); + let rhs = $handles.get_float_tensor::(&$desc.rhs); + let output = $ops(lhs, rhs); + + $handles.register_float_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_float_cmp_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.lhs); + let rhs = $handles.get_float_tensor::(&$desc.rhs); + let output = $ops(lhs, rhs); + + $handles.register_bool_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_int_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_int_tensor::(&$desc.lhs); + let rhs = $handles.get_int_tensor::(&$desc.rhs); + let output = $ops(lhs, rhs); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_int_cmp_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_int_tensor::(&$desc.lhs); + let rhs = $handles.get_int_tensor::(&$desc.rhs); + let output = $ops(lhs, rhs); + + $handles.register_bool_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! binary_bool_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_bool_tensor::(&$desc.lhs); + let rhs = $handles.get_bool_tensor::(&$desc.rhs); + let output = $ops(lhs, rhs); + + $handles.register_bool_tensor::(&$desc.out.id, output); + }}; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/bool_tensor.rs new file mode 100644 index 0000000..3a314cf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/bool_tensor.rs @@ -0,0 +1,333 @@ +use alloc::vec::Vec; +use burn_backend::backend::ExecutionError; + +use crate::{BackendRouter, RunnerChannel, RunnerClient, get_client}; +use burn_backend::ops::BoolTensorOps; +use burn_backend::tensor::{ + BoolTensor, Device, FloatElem, FloatTensor, IndexingUpdateOp, IntElem, IntTensor, +}; +use burn_backend::{Element, Scalar, Shape, Slice, TensorData}; +use burn_ir::{ + BaseOperationIr, BinaryOpIr, BoolOperationIr, CastOpIr, CatOpIr, CreationOpIr, FlipOpIr, + GatherOpIr, InitOperationIr, MaskFillOpIr, MaskWhereOpIr, OperationIr, OperationOutput, + PermuteOpIr, RepeatDimOpIr, ScalarOpIr, ScatterOpIr, ShapeOpIr, SliceAssignOpIr, SliceOpIr, + SwapDimsOpIr, UnaryOpIr, UnfoldOpIr, +}; + +impl BoolTensorOps for BackendRouter { + fn bool_empty(shape: Shape, device: &Device) -> BoolTensor { + let client = get_client::(device); + let desc = + CreationOpIr::create(shape, R::BoolElem::dtype(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Empty(desc))) + .output() + } + + fn bool_zeros(shape: Shape, device: &Device) -> BoolTensor { + let client = get_client::(device); + let desc = + CreationOpIr::create(shape, R::BoolElem::dtype(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Zeros(desc))) + .output() + } + + fn bool_ones(shape: Shape, device: &Device) -> BoolTensor { + let client = get_client::(device); + let desc = + CreationOpIr::create(shape, R::BoolElem::dtype(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Ones(desc))) + .output() + } + + async fn bool_into_data(tensor: BoolTensor) -> Result { + tensor.into_data().await + } + + fn bool_from_data(data: TensorData, device: &Device) -> BoolTensor { + let client = get_client::(device); + let out = client.register_tensor_data(data); + let desc = InitOperationIr { + out: out.to_ir_out(), + }; + + // Call register op when output is already initialized + client.register_op(OperationIr::Init(desc)); + + out + } + + fn bool_into_int(tensor: BoolTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), IntElem::::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Bool(BoolOperationIr::IntoInt(desc))) + .output() + } + + fn bool_into_float(tensor: BoolTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), FloatElem::::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Bool(BoolOperationIr::IntoFloat(desc))) + .output() + } + + fn bool_device(tensor: &BoolTensor) -> Device { + tensor.client.device() + } + + fn bool_to_device(tensor: BoolTensor, device: &Device) -> BoolTensor { + if &tensor.client.device() == device { + return tensor; + } + R::change_client_backend(tensor, device) + } + + fn bool_reshape(tensor: BoolTensor, shape: Shape) -> BoolTensor { + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Reshape(desc))) + .output() + } + + fn bool_slice(tensor: BoolTensor, slices: &[Slice]) -> BoolTensor { + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Slice(desc))) + .output() + } + + fn bool_slice_assign( + tensor: BoolTensor, + slices: &[burn_backend::Slice], + value: BoolTensor, + ) -> BoolTensor { + let client = tensor.client.clone(); + let desc = + SliceAssignOpIr::create(tensor.into_ir(), slices.into(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::SliceAssign(desc))) + .output() + } + + fn bool_equal(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Equal(desc))) + .output() + } + + fn bool_not(tensor: BoolTensor) -> BoolTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Bool(BoolOperationIr::Not(desc))) + .output() + } + + fn bool_and(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Bool(BoolOperationIr::And(desc))) + .output() + } + + fn bool_or(lhs: BoolTensor, rhs: BoolTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Bool(BoolOperationIr::Or(desc))) + .output() + } + + fn bool_swap_dims(tensor: BoolTensor, dim1: usize, dim2: usize) -> BoolTensor { + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::SwapDims(desc))) + .output() + } + + fn bool_permute(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Permute(desc))) + .output() + } + + fn bool_flip(tensor: BoolTensor, axes: &[usize]) -> BoolTensor { + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Flip(desc))) + .output() + } + + fn bool_expand(tensor: BoolTensor, shape: Shape) -> BoolTensor { + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Expand(desc))) + .output() + } + + fn bool_cat(tensors: Vec>, dim: usize) -> BoolTensor { + let client = tensors.first().unwrap().client.clone(); + let tensors = tensors.into_iter().map(|t| t.into_ir()).collect(); + let desc = CatOpIr::create(tensors, dim, || client.create_empty_handle()); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Cat(desc))) + .output() + } + + fn bool_repeat_dim(tensor: BoolTensor, dim: usize, times: usize) -> BoolTensor { + let client = tensor.client.clone(); + let desc = RepeatDimOpIr::create(tensor.into_ir(), dim, times, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::RepeatDim(desc))) + .output() + } + + fn bool_unfold( + tensor: BoolTensor, + dim: usize, + size: usize, + step: usize, + ) -> BoolTensor { + let client = tensor.client.clone(); + let desc = UnfoldOpIr::create(tensor.into_ir(), dim, size, step, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Unfold(desc))) + .output() + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + let client = tensor.client.clone(); + let desc = MaskWhereOpIr::create(tensor.into_ir(), mask.into_ir(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::MaskWhere(desc))) + .output() + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + let client = tensor.client.clone(); + let value = value.into(); + let desc = MaskFillOpIr::create(tensor.into_ir(), mask.into_ir(), value, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::MaskFill(desc))) + .output() + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Gather(desc))) + .output() + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + let client = tensor.client.clone(); + let desc = ScatterOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseBool(BaseOperationIr::Scatter(desc))) + .output() + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseBool(BaseOperationIr::EqualElem(desc))) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/int_tensor.rs new file mode 100644 index 0000000..fcbebbe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/int_tensor.rs @@ -0,0 +1,1037 @@ +use alloc::vec::Vec; +use burn_backend::backend::{Backend, ExecutionError}; + +use crate::{BackendRouter, RunnerChannel, RunnerClient, get_client}; +use burn_backend::tensor::{ + BoolTensor, Device, FloatElem, FloatTensor, IndexingUpdateOp, IntElem, IntTensor, +}; +use burn_backend::{ + Distribution, Element, IntDType, Scalar, Shape, Slice, TensorData, ops::IntTensorOps, +}; +use burn_ir::{ + BaseOperationIr, BinaryOpIr, CastOpIr, CatOpIr, ClampOpIr, CreationOpIr, DimOpIr, FlipOpIr, + GatherOpIr, InitOperationIr, IntOperationIr, MaskFillOpIr, MaskWhereOpIr, MatmulOpIr, + NumericOperationIr, OperationIr, OperationOutput, PermuteOpIr, RandomOpIr, ReduceDimOpIr, + ReduceDimWithIndicesOpIr, ReduceOpIr, RepeatDimOpIr, ScalarOpIr, ScatterOpIr, SelectAssignOpIr, + SelectOpIr, ShapeOpIr, SliceAssignOpIr, SliceOpIr, SwapDimsOpIr, UnaryOpIr, UnfoldOpIr, +}; + +impl IntTensorOps for BackendRouter { + fn int_empty(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Empty(desc))) + .output() + } + + async fn int_into_data(tensor: IntTensor) -> Result { + Ok(tensor + .into_data() + .await? + // Since underlying backends can have different data types, we convert to the current elem + .convert::<::IntElem>()) + } + + fn int_from_data(data: TensorData, device: &Device) -> IntTensor { + let client = get_client::(device); + let out = client.register_tensor_data(data); + let desc = InitOperationIr { + out: out.to_ir_out(), + }; + + // Call register op when output is already initialized + client.register_op(OperationIr::Init(desc)); + + out + } + + fn int_device(tensor: &IntTensor) -> Device { + tensor.client.device() + } + + fn int_to_device(tensor: IntTensor, device: &Device) -> IntTensor { + if &tensor.client.device() == device { + return tensor; + } + R::change_client_backend(tensor, device) + } + + fn int_reshape(tensor: IntTensor, shape: Shape) -> IntTensor { + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Reshape(desc))) + .output() + } + + fn int_slice(tensor: IntTensor, slices: &[Slice]) -> IntTensor { + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Slice(desc))) + .output() + } + + fn int_slice_assign( + tensor: IntTensor, + slices: &[burn_backend::Slice], + value: IntTensor, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = + SliceAssignOpIr::create(tensor.into_ir(), slices.into(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::SliceAssign(desc))) + .output() + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = MatmulOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::Matmul(desc))) + .output() + } + + fn int_mask_where( + tensor: IntTensor, + mask: BoolTensor, + value: IntTensor, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = MaskWhereOpIr::create(tensor.into_ir(), mask.into_ir(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::MaskWhere(desc))) + .output() + } + + fn int_mask_fill( + tensor: IntTensor, + mask: BoolTensor, + value: Scalar, + ) -> IntTensor { + let client = tensor.client.clone(); + let value = value.into(); + let desc = MaskFillOpIr::create(tensor.into_ir(), mask.into_ir(), value, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::MaskFill(desc))) + .output() + } + + fn int_gather( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Gather(desc))) + .output() + } + + fn int_scatter_add( + dim: usize, + tensor: IntTensor, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = ScatterOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Scatter(desc))) + .output() + } + + fn int_select( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = SelectOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Select(desc))) + .output() + } + + fn int_select_add( + tensor: IntTensor, + dim: usize, + indices: IntTensor, + value: IntTensor, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = SelectAssignOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseInt(BaseOperationIr::SelectAssign(desc))) + .output() + } + + fn int_cat(tensors: Vec>, dim: usize) -> IntTensor { + let client = tensors.first().unwrap().client.clone(); + let tensors = tensors.into_iter().map(|t| t.into_ir()).collect(); + let desc = CatOpIr::create(tensors, dim, || client.create_empty_handle()); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Cat(desc))) + .output() + } + + fn int_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Equal(desc))) + .output() + } + + fn int_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::EqualElem(desc))) + .output() + } + + fn int_greater(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::Greater(desc), + )) + .output() + } + + fn int_greater_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::GreaterElem(desc), + )) + .output() + } + + fn int_greater_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::GreaterEqual(desc), + )) + .output() + } + + fn int_greater_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::GreaterEqualElem(desc), + )) + .output() + } + + fn int_lower(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::Lower(desc), + )) + .output() + } + + fn int_lower_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::LowerElem(desc), + )) + .output() + } + + fn int_lower_equal(lhs: IntTensor, rhs: IntTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::LowerEqual(desc), + )) + .output() + } + + fn int_lower_equal_elem(lhs: IntTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.lhs.dtype, + NumericOperationIr::LowerEqualElem(desc), + )) + .output() + } + + fn int_add(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Add(desc), + )) + .output() + } + + fn int_add_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::AddScalar(desc), + )) + .output() + } + + fn int_sub(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Sub(desc), + )) + .output() + } + + fn int_sub_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::SubScalar(desc), + )) + .output() + } + + fn int_mul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Mul(desc), + )) + .output() + } + + fn int_mul_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MulScalar(desc), + )) + .output() + } + + fn int_div(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Div(desc), + )) + .output() + } + + fn int_div_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::DivScalar(desc), + )) + .output() + } + + fn int_remainder(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Rem(desc), + )) + .output() + } + + fn int_remainder_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::RemScalar(desc), + )) + .output() + } + + fn int_zeros(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Zeros(desc))) + .output() + } + + fn int_ones(shape: Shape, device: &Device, dtype: IntDType) -> IntTensor { + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Ones(desc))) + .output() + } + + fn int_sum(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Sum(desc), + )) + .output() + } + + fn int_sum_dim(tensor: IntTensor, axis: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), axis, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::SumDim(desc), + )) + .output() + } + + fn int_prod(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Prod(desc), + )) + .output() + } + + fn int_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::ProdDim(desc), + )) + .output() + } + + fn int_mean(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Mean(desc), + )) + .output() + } + + fn int_mean_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MeanDim(desc), + )) + .output() + } + + fn int_cumsum(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::CumSum(desc), + )) + .output() + } + + fn int_cumprod(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::CumProd(desc), + )) + .output() + } + + fn int_cummin(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::CumMin(desc), + )) + .output() + } + + fn int_cummax(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::CumMax(desc), + )) + .output() + } + + fn int_argmax(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::ArgMax(desc), + )) + .output() + } + + fn int_argmin(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::ArgMin(desc), + )) + .output() + } + + fn int_clamp(tensor: IntTensor, min: Scalar, max: Scalar) -> IntTensor { + let client = tensor.client.clone(); + let min = min.into(); + let max = max.into(); + let desc = ClampOpIr::create(tensor.into_ir(), min, max, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Clamp(desc), + )) + .output() + } + + fn int_abs(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Abs(desc), + )) + .output() + } + + fn int_into_float(tensor: IntTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), FloatElem::::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::IntoFloat(desc))) + .output() + } + + fn int_swap_dims(tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::SwapDims(desc))) + .output() + } + + fn int_max(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Max(desc), + )) + .output() + } + + fn int_max_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MaxDim(desc), + )) + .output() + } + + fn int_max_dim_with_indices( + tensor: IntTensor, + dim: usize, + ) -> (IntTensor, IntTensor) { + let client = tensor.client.clone(); + let desc = ReduceDimWithIndicesOpIr::create( + tensor.into_ir(), + dim, + IntElem::::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericInt( + desc.tensor.dtype, + NumericOperationIr::MaxDimWithIndices(desc), + )) + .outputs() + .into() + } + + fn int_max_abs(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MaxAbs(desc), + )) + .output() + } + + fn int_max_abs_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MaxAbsDim(desc), + )) + .output() + } + + fn int_min(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::Min(desc), + )) + .output() + } + + fn int_min_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MinDim(desc), + )) + .output() + } + + fn int_min_dim_with_indices( + tensor: IntTensor, + dim: usize, + ) -> (IntTensor, IntTensor) { + let client = tensor.client.clone(); + let desc = ReduceDimWithIndicesOpIr::create( + tensor.into_ir(), + dim, + IntElem::::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericInt( + desc.out.dtype, + NumericOperationIr::MinDimWithIndices(desc), + )) + .outputs() + .into() + } + + fn int_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> IntTensor { + let client = get_client::(device); + let dtype = IntElem::::dtype(); + let desc = RandomOpIr::create(shape, dtype, distribution, || client.create_empty_handle()); + + client + .register(OperationIr::NumericInt( + dtype, + NumericOperationIr::IntRandom(desc), + )) + .output() + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Permute(desc))) + .output() + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Expand(desc))) + .output() + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Flip(desc))) + .output() + } + + fn int_repeat_dim(tensor: IntTensor, dim: usize, times: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = RepeatDimOpIr::create(tensor.into_ir(), dim, times, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::RepeatDim(desc))) + .output() + } + + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseAnd(desc))) + .output() + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseOr(desc))) + .output() + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseXor(desc))) + .output() + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseNot(desc))) + .output() + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseAndScalar(desc))) + .output() + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseOrScalar(desc))) + .output() + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseXorScalar(desc))) + .output() + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseLeftShift(desc))) + .output() + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseLeftShiftScalar( + desc, + ))) + .output() + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseRightShift(desc))) + .output() + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::Int(IntOperationIr::BitwiseRightShiftScalar( + desc, + ))) + .output() + } + + fn int_cast(tensor: IntTensor, dtype: burn_backend::IntDType) -> IntTensor { + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), dtype.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Cast(desc))) + .output() + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + let client = tensor.client.clone(); + let desc = UnfoldOpIr::create(tensor.into_ir(), dim, size, step, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseInt(BaseOperationIr::Unfold(desc))) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/mod.rs new file mode 100644 index 0000000..b98aa82 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/mod.rs @@ -0,0 +1,9 @@ +mod activation; +mod binary; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; +mod unary; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/module.rs new file mode 100644 index 0000000..37a33bb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/module.rs @@ -0,0 +1,796 @@ +use alloc::boxed::Box; + +use burn_backend::Element; +use burn_backend::ops::{ + AttentionModuleOptions, ConvOptions, ConvTransposeOptions, DeformConv2dBackward, + DeformConvOptions, InterpolateOptions, MaxPool1dBackward, MaxPool1dWithIndices, + MaxPool2dBackward, MaxPool2dWithIndices, ModuleOps, +}; +use burn_backend::tensor::{BoolTensor, FloatTensor, IntElem, IntTensor}; +use burn_ir::*; + +use crate::{BackendRouter, RunnerChannel, RunnerClient}; + +impl ModuleOps for BackendRouter { + fn conv1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<1>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv1dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv1d(desc))) + .output() + } + + fn conv1d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv1dXBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv1dXBackward( + desc, + ))) + .output() + } + + fn conv1d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<1>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv1dWeightBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module( + ModuleOperationIr::Conv1dWeightBackward(desc), + )) + .output() + } + + fn conv1d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv1dBiasBackwardOpIr::create( + x.into_ir(), + bias.into_ir(), + output_grad.into_ir(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv1dBiasBackward( + desc, + ))) + .output() + } + + fn conv2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<2>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv2dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv2d(desc))) + .output() + } + + fn conv2d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv2dXBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv2dXBackward( + desc, + ))) + .output() + } + + fn conv2d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<2>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv2dWeightBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module( + ModuleOperationIr::Conv2dWeightBackward(desc), + )) + .output() + } + + fn conv2d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv2dBiasBackwardOpIr::create( + x.into_ir(), + bias.into_ir(), + output_grad.into_ir(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv2dBiasBackward( + desc, + ))) + .output() + } + + fn conv3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvOptions<3>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv3dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv3d(desc))) + .output() + } + + fn conv3d_x_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv3dXBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv3dXBackward( + desc, + ))) + .output() + } + + fn conv3d_weight_backward( + x: FloatTensor, + weight: FloatTensor, + output_grad: FloatTensor, + options: ConvOptions<3>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv3dWeightBackwardOpIr::create( + x.into_ir(), + weight.into_ir(), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module( + ModuleOperationIr::Conv3dWeightBackward(desc), + )) + .output() + } + + fn conv3d_bias_backward( + x: FloatTensor, + bias: FloatTensor, + output_grad: FloatTensor, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = Conv3dBiasBackwardOpIr::create( + x.into_ir(), + bias.into_ir(), + output_grad.into_ir(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Conv3dBiasBackward( + desc, + ))) + .output() + } + + fn conv_transpose1d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<1>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = ConvTranspose1dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::ConvTranspose1d( + desc, + ))) + .output() + } + + fn conv_transpose2d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<2>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = ConvTranspose2dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::ConvTranspose2d( + desc, + ))) + .output() + } + + fn conv_transpose3d( + x: FloatTensor, + weight: FloatTensor, + bias: Option>, + options: ConvTransposeOptions<3>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = ConvTranspose3dOpIr::create( + x.into_ir(), + weight.into_ir(), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::ConvTranspose3d( + desc, + ))) + .output() + } + + fn avg_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = AvgPool1dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::AvgPool1d(desc))) + .output() + } + + fn avg_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = AvgPool2dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::AvgPool2d(desc))) + .output() + } + + fn avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = AvgPool1dBackwardOpIr::create( + x.into_ir(), + grad.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::AvgPool1dBackward( + desc, + ))) + .output() + } + + fn avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = AvgPool2dBackwardOpIr::create( + x.into_ir(), + grad.into_ir(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::AvgPool2dBackward( + desc, + ))) + .output() + } + + fn max_pool1d( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = MaxPool1dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::MaxPool1d(desc))) + .output() + } + + fn max_pool2d( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = MaxPool2dOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::MaxPool2d(desc))) + .output() + } + + fn max_pool1d_with_indices( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> MaxPool1dWithIndices { + let client = x.client.clone(); + let desc = MaxPool1dWithIndicesOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + IntElem::::dtype(), + || client.create_empty_handle(), + ); + + let [out, out_indices] = client + .register(OperationIr::Module( + ModuleOperationIr::MaxPool1dWithIndices(desc), + )) + .outputs(); + + MaxPool1dWithIndices::new(out, out_indices) + } + + fn max_pool2d_with_indices( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices { + let client = x.client.clone(); + let desc = MaxPool2dWithIndicesOpIr::create( + x.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + IntElem::::dtype(), + || client.create_empty_handle(), + ); + + let [out, out_indices] = client + .register(OperationIr::Module( + ModuleOperationIr::MaxPool2dWithIndices(desc), + )) + .outputs(); + + MaxPool2dWithIndices::new(out, out_indices) + } + + fn max_pool1d_with_indices_backward( + x: FloatTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool1dBackward { + let client = x.client.clone(); + + let desc = MaxPool1dWithIndicesBackwardOpIr::create( + x.into_ir(), + output_grad.into_ir(), + indices.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + let out = client + .register(OperationIr::Module( + ModuleOperationIr::MaxPool1dWithIndicesBackward(desc), + )) + .output(); + + MaxPool1dBackward::new(out) + } + + fn max_pool2d_with_indices_backward( + x: FloatTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: FloatTensor, + indices: IntTensor, + ) -> MaxPool2dBackward { + let client = x.client.clone(); + + let desc = MaxPool2dWithIndicesBackwardOpIr::create( + x.into_ir(), + output_grad.into_ir(), + indices.into_ir(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + || client.create_empty_handle(), + ); + + let out = client + .register(OperationIr::Module( + ModuleOperationIr::MaxPool2dWithIndicesBackward(desc), + )) + .output(); + + MaxPool2dBackward::new(out) + } + + fn adaptive_avg_pool1d(x: FloatTensor, output_size: usize) -> FloatTensor { + let client = x.client.clone(); + + let desc = AdaptiveAvgPool1dOpIr::create(x.into_ir(), output_size, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Module(ModuleOperationIr::AdaptiveAvgPool1d( + desc, + ))) + .output() + } + + fn adaptive_avg_pool2d(x: FloatTensor, output_size: [usize; 2]) -> FloatTensor { + let client = x.client.clone(); + + let desc = AdaptiveAvgPool2dOpIr::create(x.into_ir(), output_size, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Module(ModuleOperationIr::AdaptiveAvgPool2d( + desc, + ))) + .output() + } + + fn adaptive_avg_pool1d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + let client = x.client.clone(); + + let desc = AdaptiveAvgPool1dBackwardOpIr::create(x.into_ir(), grad.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Module( + ModuleOperationIr::AdaptiveAvgPool1dBackward(desc), + )) + .output() + } + + fn adaptive_avg_pool2d_backward( + x: FloatTensor, + grad: FloatTensor, + ) -> FloatTensor { + let client = x.client.clone(); + + let desc = AdaptiveAvgPool2dBackwardOpIr::create(x.into_ir(), grad.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Module( + ModuleOperationIr::AdaptiveAvgPool2dBackward(desc), + )) + .output() + } + + fn interpolate( + x: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = InterpolateOpIr::create(x.into_ir(), output_size, options.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Module(ModuleOperationIr::Interpolate(desc))) + .output() + } + + fn interpolate_backward( + x: FloatTensor, + grad: FloatTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = InterpolateBackwardOpIr::create( + x.into_ir(), + grad.into_ir(), + output_size, + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::InterpolateBackward( + desc, + ))) + .output() + } + + fn deform_conv2d( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, + ) -> FloatTensor { + let client = x.client.clone(); + let desc = DeformConv2dOpIr::create( + x.into_ir(), + offset.into_ir(), + weight.into_ir(), + mask.map(|mask| mask.into_ir()), + bias.map(|bias| bias.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::DeformableConv2d( + Box::new(desc), + ))) + .output() + } + + fn deform_conv2d_backward( + x: FloatTensor, + offset: FloatTensor, + weight: FloatTensor, + mask: Option>, + bias: Option>, + output_grad: FloatTensor, + options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + let client = x.client.clone(); + let has_bias = bias.is_some(); + let has_mask = mask.is_some(); + + let desc = DeformConv2dBackwardOpIr::create( + x.into_ir(), + offset.into_ir(), + weight.into_ir(), + mask.map(|mask| mask.into_ir()), + bias.map(|bias| bias.into_ir()), + output_grad.into_ir(), + options.into(), + || client.create_empty_handle(), + ); + let mut outputs = client + .register(OperationIr::Module( + ModuleOperationIr::DeformableConv2dBackward(Box::new(desc)), + )) + .into_iter(); + + // When the number of outputs is variable, the order is important + let input_grad = outputs.next().unwrap(); + let offset_grad = outputs.next().unwrap(); + let weight_grad = outputs.next().unwrap(); + let mask_grad = has_mask.then(|| outputs.next().unwrap()); + let bias_grad = has_bias.then(|| outputs.next().unwrap()); + + DeformConv2dBackward::new(input_grad, offset_grad, weight_grad, mask_grad, bias_grad) + } + + fn attention( + query: FloatTensor, + key: FloatTensor, + value: FloatTensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, + ) -> FloatTensor { + let client = query.client.clone(); + let desc = AttentionOpIr::create( + query.into_ir(), + key.into_ir(), + value.into_ir(), + mask.map(|m: BoolTensor| m.into_ir()), + attn_bias.map(|ab| ab.into_ir()), + options.into(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::Module(ModuleOperationIr::Attention(desc))) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/qtensor.rs new file mode 100644 index 0000000..e2ad4cc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/qtensor.rs @@ -0,0 +1,92 @@ +use burn_backend::{ + ExecutionError, Shape, Slice, TensorData, + ops::QTensorOps, + quantization::{QuantScheme, QuantizationParametersPrimitive}, + tensor::{Device, FloatTensor, IntTensor, QuantizedTensor}, +}; + +use crate::{BackendRouter, RunnerChannel}; + +impl QTensorOps for BackendRouter { + fn q_from_data(_data: TensorData, _device: &Device) -> QuantizedTensor { + unimplemented!() + } + + fn quantize( + _tensor: FloatTensor, + _scheme: &QuantScheme, + _qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + unimplemented!() + } + + fn quantize_dynamic( + _tensor: FloatTensor, + _scheme: &QuantScheme, + ) -> QuantizedTensor { + unimplemented!() + } + + fn dequantize(_tensor: QuantizedTensor) -> FloatTensor { + unimplemented!() + } + + fn q_device(_tensor: &QuantizedTensor) -> Device { + unimplemented!() + } + + fn q_to_device( + _tensor: QuantizedTensor, + _device: &Device, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_reshape(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } + + async fn q_into_data(_tensor: QuantizedTensor) -> Result { + unimplemented!() + } + + fn q_swap_dims( + _tensor: QuantizedTensor, + _dim1: usize, + _dim2: usize, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_permute(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_flip(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_gather( + _dim: usize, + _tensor: QuantizedTensor, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_select( + _tensor: QuantizedTensor, + _dim: usize, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_slice(_tensor: QuantizedTensor, _slices: &[Slice]) -> QuantizedTensor { + unimplemented!() + } + + fn q_expand(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/tensor.rs new file mode 100644 index 0000000..ce8cf5d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/tensor.rs @@ -0,0 +1,1248 @@ +use alloc::vec::Vec; +use burn_backend::Scalar; +use burn_backend::backend::{Backend, ExecutionError}; + +use crate::{BackendRouter, RunnerChannel, RunnerClient, get_client}; +use burn_backend::tensor::{ + BoolTensor, Device, FloatElem, FloatTensor, IndexingUpdateOp, IntElem, IntTensor, +}; +use burn_backend::{ + Distribution, Element, FloatDType, Shape, Slice, TensorData, ops::FloatTensorOps, +}; +use burn_ir::{ + BaseOperationIr, BinaryOpIr, CastOpIr, CatOpIr, ClampOpIr, CreationOpIr, CrossOpIr, DimOpIr, + FlipOpIr, FloatOperationIr, FullOpIr, GatherOpIr, InitOperationIr, MaskFillOpIr, MaskWhereOpIr, + MatmulOpIr, NumericOperationIr, OperationIr, OperationOutput, PermuteOpIr, RandomOpIr, + ReduceDimOpIr, ReduceDimWithIndicesOpIr, ReduceOpIr, RepeatDimOpIr, ScalarOpIr, ScatterOpIr, + SelectAssignOpIr, SelectOpIr, ShapeOpIr, SliceAssignOpIr, SliceOpIr, SwapDimsOpIr, UnaryOpIr, + UnfoldOpIr, +}; + +impl FloatTensorOps for BackendRouter { + fn float_from_data(data: TensorData, device: &Device) -> FloatTensor { + let client = get_client::(device); + let out = client.register_tensor_data(data); + let desc = InitOperationIr { + out: out.to_ir_out(), + }; + + // Call register op when output is already initialized + client.register_op(OperationIr::Init(desc)); + + out + } + + fn float_random( + shape: Shape, + distribution: Distribution, + device: &Device, + ) -> FloatTensor { + let client = get_client::(device); + let dtype = FloatElem::::dtype(); + let desc = RandomOpIr::create(shape, dtype, distribution, || client.create_empty_handle()); + + client + .register(OperationIr::Float(dtype, FloatOperationIr::Random(desc))) + .output() + } + + fn float_zeros(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Zeros(desc))) + .output() + } + + fn float_ones(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Ones(desc))) + .output() + } + + fn float_full( + shape: Shape, + fill_value: Scalar, + device: &Device, + dtype: FloatDType, + ) -> FloatTensor { + let client = get_client::(device); + let dtype = dtype.into(); + let value = fill_value.into(); + let desc = FullOpIr::create(shape, dtype, value, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Full(desc), + )) + .output() + } + + async fn float_into_data(tensor: FloatTensor) -> Result { + Ok(tensor + .into_data() + .await? + // Since underlying backends can have different data types, we convert to the current elem + .convert::<::FloatElem>()) + } + + fn float_device(tensor: &FloatTensor) -> Device { + tensor.client.device() + } + + fn float_to_device(tensor: FloatTensor, device: &Device) -> FloatTensor { + if &tensor.client.device() == device { + return tensor; + } + R::change_client_backend(tensor, device) + } + + fn float_into_int(tensor: FloatTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), IntElem::::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Float( + desc.input.dtype, + FloatOperationIr::IntoInt(desc), + )) + .output() + } + + fn float_empty(shape: Shape, device: &Device, dtype: FloatDType) -> FloatTensor { + let client = get_client::(device); + let desc = CreationOpIr::create(shape, dtype.into(), || client.create_empty_handle()); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Empty(desc))) + .output() + } + + fn float_add(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Add(desc), + )) + .output() + } + + fn float_add_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::AddScalar(desc), + )) + .output() + } + + fn float_clamp(tensor: FloatTensor, min: Scalar, max: Scalar) -> FloatTensor { + let client = tensor.client.clone(); + let min = min.into(); + let max = max.into(); + let desc = ClampOpIr::create(tensor.into_ir(), min, max, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Clamp(desc), + )) + .output() + } + + fn float_sub(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Sub(desc), + )) + .output() + } + + fn float_sub_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::SubScalar(desc), + )) + .output() + } + + fn float_mul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Mul(desc), + )) + .output() + } + + fn float_mul_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MulScalar(desc), + )) + .output() + } + + fn float_div(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Div(desc), + )) + .output() + } + + fn float_div_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::DivScalar(desc), + )) + .output() + } + + fn float_remainder(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Rem(desc), + )) + .output() + } + + fn float_remainder_scalar(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::RemScalar(desc), + )) + .output() + } + + fn float_matmul(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = MatmulOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Matmul(desc), + )) + .output() + } + + fn float_cross( + lhs: FloatTensor, + rhs: FloatTensor, + dim: usize, + ) -> FloatTensor { + let client = lhs.client.clone(); + let desc = CrossOpIr::create(lhs.into_ir(), rhs.into_ir(), dim, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Cross(desc), + )) + .output() + } + + fn float_swap_dims(tensor: FloatTensor, dim1: usize, dim2: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = SwapDimsOpIr::create(tensor.into_ir(), dim1, dim2, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::SwapDims(desc))) + .output() + } + + fn float_reshape(tensor: FloatTensor, shape: Shape) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ShapeOpIr::reshape(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Reshape(desc))) + .output() + } + + fn float_gather( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = GatherOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Gather(desc))) + .output() + } + + fn float_scatter_add( + dim: usize, + tensor: FloatTensor, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ScatterOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Scatter(desc))) + .output() + } + + fn float_select( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = SelectOpIr::create(tensor.into_ir(), dim, indices.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Select(desc))) + .output() + } + + fn float_select_add( + tensor: FloatTensor, + dim: usize, + indices: IntTensor, + value: FloatTensor, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = SelectAssignOpIr::create( + tensor.into_ir(), + dim, + indices.into_ir(), + value.into_ir(), + IndexingUpdateOp::Add, + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::SelectAssign(desc))) + .output() + } + + fn float_slice(tensor: FloatTensor, slices: &[Slice]) -> FloatTensor { + let client = tensor.client.clone(); + let desc = SliceOpIr::create(tensor.into_ir(), slices.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Slice(desc))) + .output() + } + + fn float_slice_assign( + tensor: FloatTensor, + slices: &[burn_backend::Slice], + value: FloatTensor, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = + SliceAssignOpIr::create(tensor.into_ir(), slices.into(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::SliceAssign(desc))) + .output() + } + + fn float_mask_where( + tensor: FloatTensor, + mask: BoolTensor, + value: FloatTensor, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = MaskWhereOpIr::create(tensor.into_ir(), mask.into_ir(), value.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::MaskWhere(desc))) + .output() + } + + fn float_mask_fill( + tensor: FloatTensor, + mask: BoolTensor, + value: Scalar, + ) -> FloatTensor { + let client = tensor.client.clone(); + let value = value.into(); + let desc = MaskFillOpIr::create(tensor.into_ir(), mask.into_ir(), value, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::MaskFill(desc))) + .output() + } + + fn float_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Equal(desc))) + .output() + } + + fn float_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::EqualElem(desc))) + .output() + } + + fn float_greater(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::Greater(desc), + )) + .output() + } + + fn float_greater_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::GreaterElem(desc), + )) + .output() + } + + fn float_greater_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::GreaterEqual(desc), + )) + .output() + } + + fn float_greater_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::GreaterEqualElem(desc), + )) + .output() + } + + fn float_lower(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::Lower(desc), + )) + .output() + } + + fn float_lower_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::LowerElem(desc), + )) + .output() + } + + fn float_lower_equal(lhs: FloatTensor, rhs: FloatTensor) -> BoolTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create_comparison( + lhs.into_ir(), + rhs.into_ir(), + R::BoolElem::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::LowerEqual(desc), + )) + .output() + } + + fn float_lower_equal_elem(lhs: FloatTensor, rhs: Scalar) -> BoolTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create_comparison(lhs.into_ir(), rhs, R::BoolElem::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.lhs.dtype, + NumericOperationIr::LowerEqualElem(desc), + )) + .output() + } + + fn float_sum(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Sum(desc), + )) + .output() + } + + fn float_sum_dim(tensor: FloatTensor, axis: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), axis, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::SumDim(desc), + )) + .output() + } + + fn float_prod(tensor: IntTensor) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Prod(desc), + )) + .output() + } + + fn float_prod_dim(tensor: IntTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::ProdDim(desc), + )) + .output() + } + + fn float_mean(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Mean(desc), + )) + .output() + } + + fn float_mean_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MeanDim(desc), + )) + .output() + } + + fn float_cumsum(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::CumSum(desc), + )) + .output() + } + + fn float_cumprod(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::CumProd(desc), + )) + .output() + } + + fn float_cummin(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::CumMin(desc), + )) + .output() + } + + fn float_cummax(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = DimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::CumMax(desc), + )) + .output() + } + + fn float_exp(lhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = UnaryOpIr::create(lhs.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Exp(desc), + )) + .output() + } + + fn float_log(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Log(desc), + )) + .output() + } + + fn float_log1p(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Log1p(desc), + )) + .output() + } + + fn float_powf_scalar_impl(lhs: FloatTensor, rhs: Scalar) -> FloatTensor { + let client = lhs.client.clone(); + let rhs = rhs.into(); + let desc = ScalarOpIr::create(lhs.into_ir(), rhs, || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::PowfScalar(desc), + )) + .output() + } + + fn float_sqrt(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Sqrt(desc), + )) + .output() + } + + fn float_abs(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Abs(desc), + )) + .output() + } + + fn float_cos(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Cos(desc), + )) + .output() + } + + fn float_cosh(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Cosh(desc), + )) + .output() + } + + fn float_sin(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Sin(desc), + )) + .output() + } + + fn float_sinh(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Sinh(desc), + )) + .output() + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Tan(desc), + )) + .output() + } + + fn float_tanh(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Tanh(desc), + )) + .output() + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcCos(desc), + )) + .output() + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcCosh(desc), + )) + .output() + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcSin(desc), + )) + .output() + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcSinh(desc), + )) + .output() + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcTan(desc), + )) + .output() + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcTanh(desc), + )) + .output() + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::ArcTan2(desc), + )) + .output() + } + + fn float_round(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Round(desc), + )) + .output() + } + + fn float_floor(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Floor(desc), + )) + .output() + } + + fn float_ceil(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Ceil(desc), + )) + .output() + } + + fn float_trunc(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Trunc(desc), + )) + .output() + } + + fn float_recip(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Recip(desc), + )) + .output() + } + + fn float_erf(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnaryOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::Float( + desc.out.dtype, + FloatOperationIr::Erf(desc), + )) + .output() + } + + fn float_cat(tensors: Vec>, dim: usize) -> FloatTensor { + let client = tensors.first().unwrap().client.clone(); + let tensors = tensors.into_iter().map(|t| t.into_ir()).collect(); + let desc = CatOpIr::create(tensors, dim, || client.create_empty_handle()); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Cat(desc))) + .output() + } + + fn float_argmax(tensor: FloatTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = + ReduceDimOpIr::create_arg(tensor.into_ir(), dim, IntElem::::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.input.dtype, + NumericOperationIr::ArgMax(desc), + )) + .output() + } + + fn float_repeat_dim(tensor: FloatTensor, dim: usize, times: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = RepeatDimOpIr::create(tensor.into_ir(), dim, times, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::RepeatDim(desc))) + .output() + } + + fn float_argmin(tensor: FloatTensor, dim: usize) -> IntTensor { + let client = tensor.client.clone(); + let desc = + ReduceDimOpIr::create_arg(tensor.into_ir(), dim, IntElem::::dtype(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.input.dtype, + NumericOperationIr::ArgMin(desc), + )) + .output() + } + + fn float_max(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Max(desc), + )) + .output() + } + + fn float_max_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MaxDim(desc), + )) + .output() + } + + fn float_max_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + let client = tensor.client.clone(); + let desc = ReduceDimWithIndicesOpIr::create( + tensor.into_ir(), + dim, + IntElem::::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericFloat( + desc.tensor.dtype, + NumericOperationIr::MaxDimWithIndices(desc), + )) + .outputs() + .into() + } + + fn float_min(tensor: FloatTensor) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceOpIr::create(tensor.into_ir(), || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Min(desc), + )) + .output() + } + + fn float_min_dim(tensor: FloatTensor, dim: usize) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ReduceDimOpIr::create(tensor.into_ir(), dim, || client.create_empty_handle()); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::MinDim(desc), + )) + .output() + } + + fn float_min_dim_with_indices( + tensor: FloatTensor, + dim: usize, + ) -> (FloatTensor, IntTensor) { + let client = tensor.client.clone(); + let desc = ReduceDimWithIndicesOpIr::create( + tensor.into_ir(), + dim, + IntElem::::dtype(), + || client.create_empty_handle(), + ); + + client + .register(OperationIr::NumericFloat( + desc.tensor.dtype, + NumericOperationIr::MinDimWithIndices(desc), + )) + .outputs() + .into() + } + + fn float_powf(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + let client = lhs.client.clone(); + let desc = BinaryOpIr::create(lhs.into_ir(), rhs.into_ir(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::NumericFloat( + desc.out.dtype, + NumericOperationIr::Powf(desc), + )) + .output() + } + + fn float_permute(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + let client = tensor.client.clone(); + let desc = PermuteOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Permute(desc))) + .output() + } + + fn float_expand(tensor: FloatTensor, shape: Shape) -> FloatTensor { + let client = tensor.client.clone(); + let desc = ShapeOpIr::expand(tensor.into_ir(), shape, || client.create_empty_handle()); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Expand(desc))) + .output() + } + + fn float_flip(tensor: FloatTensor, axes: &[usize]) -> FloatTensor { + let client = tensor.client.clone(); + let desc = FlipOpIr::create(tensor.into_ir(), axes.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Flip(desc))) + .output() + } + + fn float_cast(tensor: FloatTensor, dtype: burn_backend::FloatDType) -> FloatTensor { + let client = tensor.client.clone(); + let desc = CastOpIr::create(tensor.into_ir(), dtype.into(), || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Cast(desc))) + .output() + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + let client = tensor.client.clone(); + let desc = UnfoldOpIr::create(tensor.into_ir(), dim, size, step, || { + client.create_empty_handle() + }); + + client + .register(OperationIr::BaseFloat(BaseOperationIr::Unfold(desc))) + .output() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/transaction.rs new file mode 100644 index 0000000..037733c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/transaction.rs @@ -0,0 +1,5 @@ +use burn_backend::ops::TransactionOps; + +use crate::{BackendRouter, RunnerChannel}; + +impl TransactionOps for BackendRouter {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/unary.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/unary.rs new file mode 100644 index 0000000..c44179e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/ops/unary.rs @@ -0,0 +1,155 @@ +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs.into()); + + $handles.register_float_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float_dim_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs); + + $handles.register_float_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! reduce_float_dim_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let input = $handles.get_float_tensor::(&$desc.input); + let output = $ops(input, $desc.axis); + + $handles.register_float_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! reduce_float2int_dim_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let input = $handles.get_float_tensor::(&$desc.input); + let output = $ops(input, $desc.axis); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! reduce_int_dim_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let input = $handles.get_int_tensor::(&$desc.input); + let output = $ops(input, $desc.axis); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float2int_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_float_cmp_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs.into()); + + $handles.register_bool_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! unary_float_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_float_tensor::(&$desc.input); + let output = $ops(lhs); + + $handles.register_float_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_int_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_int_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs.into()); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_int_dim_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_int_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! scalar_int_cmp_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_int_tensor::(&$desc.lhs); + let output = $ops(lhs, $desc.rhs.into()); + + $handles.register_bool_tensor::(&$desc.out.id, output); + }}; +} + +#[allow(missing_docs)] +#[macro_export(local_inner_macros)] +macro_rules! unary_int_ops { + ( + $handles:expr, $desc:expr, $ops:expr + ) => {{ + let lhs = $handles.get_int_tensor::(&$desc.input); + let output = $ops(lhs); + + $handles.register_int_tensor::(&$desc.out.id, output); + }}; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/runner.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/runner.rs new file mode 100644 index 0000000..9314e0f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/runner.rs @@ -0,0 +1,1575 @@ +use super::{RouterTensor, RunnerClient}; +use crate::{ + binary_bool_ops, binary_float_cmp_ops, binary_float_ops, binary_int_cmp_ops, binary_int_ops, + reduce_float_dim_ops, reduce_float2int_dim_ops, reduce_int_dim_ops, scalar_float_cmp_ops, + scalar_float_ops, scalar_int_cmp_ops, scalar_int_ops, unary_float_ops, unary_int_ops, +}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use burn_backend::{Backend, DType, ExecutionError, Shape, TensorData, tensor::IndexingUpdateOp}; +use burn_ir::{ + BackendIr, BaseOperationIr, BoolOperationIr, FloatOperationIr, HandleContainer, IntOperationIr, + ModuleOperationIr, NumericOperationIr, OperationIr, TensorId, TensorIr, TensorStatus, +}; +use burn_std::{future::DynFut, stub::Mutex}; + +/// A runner's context contains a [handle container](HandleContainer) to manage +/// (i.e., fetch and update) existing tensors. +pub struct RunnerContext { + /// Handle container to retrieve tensors based on their intermediate representation. + handles: HandleContainer, +} + +impl RunnerContext { + /// Create a new (uninitialized) empty tensor and returns its corresponding [tensor id](TensorId). + fn create_empty_handle(&mut self) -> TensorId { + self.handles.create_tensor_uninit() + } +} + +/// A runner is responsible for executing tensor operations for a given [intermediate backend](BackendIr). +#[derive(Clone)] +pub struct Runner { + // Mutex for the mutable handles + context: Arc>>, + device: B::Device, +} + +impl core::fmt::Debug for Runner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Runner") + .field("device", &self.device) + .finish() + } +} + +impl Runner { + /// Create a new runner. + pub fn new(device: B::Device) -> Self { + Self { + context: Arc::new(Mutex::new(RunnerContext { + handles: HandleContainer::new(), + })), + device, + } + } + + /// Get the tensor handle for the given [tensor representation](TensorIr). + pub fn get_tensor_handle(&self, tensor: &TensorIr) -> B::Handle { + let handles = &mut self.context.lock().unwrap().handles; + handles.get_tensor_handle(tensor).handle + } + + /// Create a tensor with the given handle and shape. + pub fn register_tensor( + &self, + handle: B::Handle, + shape: Shape, + dtype: DType, + client: C, + ) -> RouterTensor { + let mut ctx = self.context.lock().unwrap(); + let id = ctx.create_empty_handle(); + + ctx.handles.register_handle(id, handle); + core::mem::drop(ctx); + + RouterTensor::new(id, shape, dtype, client) + } + + /// Register a tensor from its data and id. + pub fn register_tensor_data_id(&self, id: TensorId, data: TensorData) { + let mut ctx = self.context.lock().unwrap(); + let dtype = data.dtype; + + if dtype.is_float() { + let tensor = B::float_from_data(data, &self.device); + ctx.handles.register_float_tensor::(&id, tensor) + } else if dtype.is_int() { + let tensor = B::int_from_data(data, &self.device); + ctx.handles.register_int_tensor::(&id, tensor) + } else if dtype.is_bool() { + let tensor = B::bool_from_data(data, &self.device); + ctx.handles.register_bool_tensor::(&id, tensor) + } else if let DType::QFloat(_) = dtype { + todo!(); + } + + core::mem::drop(ctx); + } + + /// Register a tensor and returns its intermediate representation. + pub fn register_tensor_data_desc(&self, data: TensorData) -> TensorIr { + let mut ctx = self.context.lock().unwrap(); + let id = ctx.create_empty_handle(); + let shape = data.shape.clone(); + let dtype = data.dtype; + + if dtype.is_float() { + let tensor = B::float_from_data(data, &self.device); + ctx.handles.register_float_tensor::(&id, tensor) + } else if dtype.is_int() { + let tensor = B::int_from_data(data, &self.device); + ctx.handles.register_int_tensor::(&id, tensor) + } else if dtype.is_bool() { + let tensor = B::bool_from_data(data, &self.device); + ctx.handles.register_bool_tensor::(&id, tensor) + } else if let DType::QFloat(_) = dtype { + todo!(); + } + + core::mem::drop(ctx); + + TensorIr { + id, + shape: Shape::from(shape), + status: TensorStatus::ReadWrite, + dtype, + } + } +} + +// This is a Remote Runner +impl RunnerClient for Runner { + type Device = B::Device; + + /// Execute a tensor operation. + fn register_op(&self, op: OperationIr) { + // Remove unused tensor handles + let mut ctx = self.context.lock().unwrap(); + + let handles = &mut ctx.handles; + match &op { + // For every op: get the input(s), execute the operation and register the output(s) + OperationIr::BaseFloat(op) => match op { + BaseOperationIr::Reshape(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_reshape(tensor, desc.out.shape.clone()); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::SwapDims(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_swap_dims(tensor, desc.dim1, desc.dim2); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Permute(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_permute(tensor, &desc.axes); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Flip(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_flip(tensor, &desc.axes); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Expand(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_expand(tensor, desc.out.shape.clone()); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Unfold(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_unfold(tensor, desc.dim, desc.size, desc.step); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Slice(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + + let output = B::float_slice(tensor, &desc.ranges); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::SliceAssign(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let value = handles.get_float_tensor::(&desc.value); + + let output = B::float_slice_assign(tensor, &desc.ranges, value); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Gather(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::float_gather(desc.dim, tensor, indices); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Scatter(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + let value = handles.get_float_tensor::(&desc.value); + + let output = match desc.update { + IndexingUpdateOp::Add => { + B::float_scatter_add(desc.dim, tensor, indices, value) + } + }; + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Select(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::float_select(tensor, desc.dim, indices); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::SelectAssign(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + let value = handles.get_float_tensor::(&desc.value); + + let output = match desc.update { + IndexingUpdateOp::Add => { + B::float_select_add(tensor, desc.dim, indices, value) + } + }; + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::MaskWhere(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let mask = handles.get_bool_tensor::(&desc.mask); + let value = handles.get_float_tensor::(&desc.value); + + let output = B::float_mask_where(tensor, mask, value); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::MaskFill(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let mask = handles.get_bool_tensor::(&desc.mask); + + let output = B::float_mask_fill(tensor, mask, desc.value.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Equal(desc) => { + binary_float_cmp_ops!(handles, desc, B::float_equal) + } + BaseOperationIr::EqualElem(desc) => { + scalar_float_cmp_ops!(handles, desc, B::float_equal_elem) + } + BaseOperationIr::RepeatDim(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + + let output = B::float_repeat_dim(tensor, desc.dim, desc.times); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Cat(desc) => { + let tensors = desc + .tensors + .iter() + .map(|tensor| handles.get_float_tensor::(tensor)) + .collect(); + + let output = B::float_cat(tensors, desc.dim); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Cast(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + let output = B::float_cast(tensor, desc.out.dtype.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Empty(desc) => { + let shape = desc.out.shape.clone(); + let output = B::float_empty(shape, &self.device, desc.out.dtype.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Ones(desc) => { + let shape = desc.out.shape.clone(); + let output = B::float_ones(shape, &self.device, desc.out.dtype.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + BaseOperationIr::Zeros(desc) => { + let shape = desc.out.shape.clone(); + let output = B::float_zeros(shape, &self.device, desc.out.dtype.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + }, + OperationIr::BaseInt(op) => match op { + BaseOperationIr::Reshape(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_reshape(tensor, desc.out.shape.clone()); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::SwapDims(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_swap_dims(tensor, desc.dim1, desc.dim2); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Permute(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_permute(tensor, &desc.axes); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Flip(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_flip(tensor, &desc.axes); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Expand(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_expand(tensor, desc.out.shape.clone()); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Unfold(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_unfold(tensor, desc.dim, desc.size, desc.step); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Slice(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + + let output = B::int_slice(tensor, &desc.ranges); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::SliceAssign(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let value = handles.get_int_tensor::(&desc.value); + + let output = B::int_slice_assign(tensor, &desc.ranges, value); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Gather(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::int_gather(desc.dim, tensor, indices); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Scatter(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + let value = handles.get_int_tensor::(&desc.value); + + let output = match desc.update { + IndexingUpdateOp::Add => { + B::int_scatter_add(desc.dim, tensor, indices, value) + } + }; + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Select(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::int_select(tensor, desc.dim, indices); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::SelectAssign(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + let value = handles.get_int_tensor::(&desc.value); + + let output = match desc.update { + IndexingUpdateOp::Add => { + B::int_select_add(tensor, desc.dim, indices, value) + } + }; + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::MaskWhere(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let mask = handles.get_bool_tensor::(&desc.mask); + let value = handles.get_int_tensor::(&desc.value); + + let output = B::int_mask_where(tensor, mask, value); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::MaskFill(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + let mask = handles.get_bool_tensor::(&desc.mask); + + let output = B::int_mask_fill(tensor, mask, desc.value.into()); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Equal(desc) => { + binary_int_cmp_ops!(handles, desc, B::int_equal) + } + BaseOperationIr::EqualElem(desc) => { + scalar_int_cmp_ops!(handles, desc, B::int_equal_elem) + } + BaseOperationIr::RepeatDim(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + + let output = B::int_repeat_dim(tensor, desc.dim, desc.times); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Cat(desc) => { + let tensors = desc + .tensors + .iter() + .map(|tensor| handles.get_int_tensor::(tensor)) + .collect(); + + let output = B::int_cat(tensors, desc.dim); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Cast(_) => unreachable!(), + BaseOperationIr::Empty(desc) => { + let shape = desc.out.shape.clone(); + let output = B::int_empty(shape, &self.device, desc.out.dtype.into()); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Ones(desc) => { + let shape = desc.out.shape.clone(); + let output = B::int_ones(shape, &self.device, desc.out.dtype.into()); + handles.register_int_tensor::(&desc.out.id, output); + } + BaseOperationIr::Zeros(desc) => { + let shape = desc.out.shape.clone(); + let output = B::int_zeros(shape, &self.device, desc.out.dtype.into()); + handles.register_int_tensor::(&desc.out.id, output); + } + }, + OperationIr::BaseBool(op) => match op { + BaseOperationIr::Reshape(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_reshape(tensor, desc.out.shape.clone()); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::SwapDims(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_swap_dims(tensor, desc.dim1, desc.dim2); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Permute(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_permute(tensor, &desc.axes); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Flip(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_flip(tensor, &desc.axes); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Expand(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_expand(tensor, desc.out.shape.clone()); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Unfold(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_unfold(tensor, desc.dim, desc.size, desc.step); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Slice(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + + let output = B::bool_slice(tensor, &desc.ranges); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::SliceAssign(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let value = handles.get_bool_tensor::(&desc.value); + + let output = B::bool_slice_assign(tensor, &desc.ranges, value); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Gather(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::bool_gather(desc.dim, tensor, indices); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Scatter(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + let value = handles.get_bool_tensor::(&desc.value); + + let output = match desc.update { + IndexingUpdateOp::Add => { + B::bool_scatter_or(desc.dim, tensor, indices, value) + } + }; + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Select(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::bool_select(tensor, desc.dim, indices); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::SelectAssign(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let indices = handles.get_int_tensor::(&desc.indices); + let value = handles.get_bool_tensor::(&desc.value); + + let output = match desc.update { + IndexingUpdateOp::Add => { + B::bool_select_or(tensor, desc.dim, indices, value) + } + }; + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::MaskWhere(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let mask = handles.get_bool_tensor::(&desc.mask); + let value = handles.get_bool_tensor::(&desc.value); + + let output = B::bool_mask_where(tensor, mask, value); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::MaskFill(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + let mask = handles.get_bool_tensor::(&desc.mask); + + let output = B::bool_mask_fill(tensor, mask, desc.value.into()); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Equal(desc) => { + let lhs = handles.get_bool_tensor::(&desc.lhs); + let rhs = handles.get_bool_tensor::(&desc.rhs); + + let output = B::bool_equal(lhs, rhs); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::EqualElem(desc) => { + let lhs = handles.get_bool_tensor::(&desc.lhs); + + let output = B::bool_equal_elem(lhs, desc.rhs.into()); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::RepeatDim(desc) => { + let tensor = handles.get_bool_tensor::(&desc.tensor); + + let output = B::bool_repeat_dim(tensor, desc.dim, desc.times); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Cat(desc) => { + let tensors = desc + .tensors + .iter() + .map(|tensor| handles.get_bool_tensor::(tensor)) + .collect(); + + let output = B::bool_cat(tensors, desc.dim); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Cast(_) => unreachable!(), + BaseOperationIr::Empty(desc) => { + let shape = desc.out.shape.clone(); + let output = B::bool_empty(shape, &self.device); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Zeros(desc) => { + let shape = desc.out.shape.clone(); + let output = B::bool_zeros(shape, &self.device); + handles.register_bool_tensor::(&desc.out.id, output); + } + BaseOperationIr::Ones(desc) => { + let shape = desc.out.shape.clone(); + let output = B::bool_ones(shape, &self.device); + handles.register_bool_tensor::(&desc.out.id, output); + } + }, + OperationIr::NumericFloat(_dtype, op) => match op { + NumericOperationIr::Add(desc) => { + binary_float_ops!(handles, desc, B::float_add) + } + NumericOperationIr::AddScalar(desc) => { + scalar_float_ops!(handles, desc, B::float_add_scalar) + } + NumericOperationIr::Sub(desc) => { + binary_float_ops!(handles, desc, B::float_sub) + } + NumericOperationIr::SubScalar(desc) => { + scalar_float_ops!(handles, desc, B::float_sub_scalar) + } + NumericOperationIr::Div(desc) => { + binary_float_ops!(handles, desc, B::float_div) + } + NumericOperationIr::DivScalar(desc) => { + scalar_float_ops!(handles, desc, B::float_div_scalar) + } + NumericOperationIr::Rem(desc) => { + binary_float_ops!(handles, desc, B::float_remainder) + } + NumericOperationIr::RemScalar(desc) => { + scalar_float_ops!(handles, desc, B::float_remainder_scalar) + } + NumericOperationIr::Mul(desc) => { + binary_float_ops!(handles, desc, B::float_mul) + } + NumericOperationIr::MulScalar(desc) => { + scalar_float_ops!(handles, desc, B::float_mul_scalar) + } + NumericOperationIr::Abs(desc) => { + unary_float_ops!(handles, desc, B::float_abs) + } + NumericOperationIr::Full(desc) => { + let shape = desc.out.shape.clone(); + let output = B::float_full( + shape, + desc.value.into(), + &self.device, + desc.out.dtype.into(), + ); + handles.register_float_tensor::(&desc.out.id, output); + } + NumericOperationIr::MeanDim(desc) => { + reduce_float_dim_ops!(handles, desc, B::float_mean_dim) + } + NumericOperationIr::Mean(desc) => { + unary_float_ops!(handles, desc, B::float_mean) + } + NumericOperationIr::Sum(desc) => { + unary_float_ops!(handles, desc, B::float_sum) + } + NumericOperationIr::SumDim(desc) => { + reduce_float_dim_ops!(handles, desc, B::float_sum_dim) + } + NumericOperationIr::Prod(desc) => { + unary_float_ops!(handles, desc, B::float_prod) + } + NumericOperationIr::ProdDim(desc) => { + reduce_float_dim_ops!(handles, desc, B::float_prod_dim) + } + NumericOperationIr::Greater(desc) => { + binary_float_cmp_ops!(handles, desc, B::float_greater) + } + NumericOperationIr::GreaterElem(desc) => { + scalar_float_cmp_ops!(handles, desc, B::float_greater_elem) + } + NumericOperationIr::GreaterEqual(desc) => { + binary_float_cmp_ops!(handles, desc, B::float_greater_equal) + } + NumericOperationIr::GreaterEqualElem(desc) => { + scalar_float_cmp_ops!(handles, desc, B::float_greater_equal_elem) + } + NumericOperationIr::Lower(desc) => { + binary_float_cmp_ops!(handles, desc, B::float_lower) + } + NumericOperationIr::LowerElem(desc) => { + scalar_float_cmp_ops!(handles, desc, B::float_lower_elem) + } + NumericOperationIr::LowerEqual(desc) => { + binary_float_cmp_ops!(handles, desc, B::float_lower_equal) + } + NumericOperationIr::LowerEqualElem(desc) => { + scalar_float_cmp_ops!(handles, desc, B::float_lower_equal_elem) + } + NumericOperationIr::ArgMax(desc) => { + reduce_float2int_dim_ops!(handles, desc, B::float_argmax) + } + NumericOperationIr::ArgMin(desc) => { + reduce_float2int_dim_ops!(handles, desc, B::float_argmin) + } + NumericOperationIr::Max(desc) => { + unary_float_ops!(handles, desc, B::float_max) + } + NumericOperationIr::MaxDimWithIndices(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + + let (output, output_idx) = B::float_max_dim_with_indices(tensor, desc.dim); + handles.register_float_tensor::(&desc.out.id, output); + handles.register_int_tensor::(&desc.out_indices.id, output_idx); + } + NumericOperationIr::MinDimWithIndices(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + + let (output, output_idx) = B::float_min_dim_with_indices(tensor, desc.dim); + handles.register_float_tensor::(&desc.out.id, output); + handles.register_int_tensor::(&desc.out_indices.id, output_idx); + } + NumericOperationIr::Min(desc) => { + unary_float_ops!(handles, desc, B::float_min) + } + NumericOperationIr::MaxDim(desc) => { + reduce_float_dim_ops!(handles, desc, B::float_max_dim) + } + NumericOperationIr::MinDim(desc) => { + reduce_float_dim_ops!(handles, desc, B::float_min_dim) + } + NumericOperationIr::MaxAbs(desc) => { + unary_float_ops!(handles, desc, B::float_max_abs) + } + NumericOperationIr::MaxAbsDim(desc) => { + reduce_float_dim_ops!(handles, desc, B::float_max_abs_dim) + } + NumericOperationIr::Clamp(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + + let output = B::float_clamp(tensor, desc.min.into(), desc.max.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + NumericOperationIr::IntRandom(_) => unreachable!(), + NumericOperationIr::Powf(desc) => { + binary_float_ops!(handles, desc, B::float_powf) + } + NumericOperationIr::CumSum(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + let output = B::float_cumsum(tensor, desc.axis); + handles.register_float_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumProd(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + let output = B::float_cumprod(tensor, desc.axis); + handles.register_float_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumMin(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + let output = B::float_cummin(tensor, desc.axis); + handles.register_float_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumMax(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + let output = B::float_cummax(tensor, desc.axis); + handles.register_float_tensor::(&desc.out.id, output); + } + }, + OperationIr::NumericInt(_dtype, op) => match op { + NumericOperationIr::Add(desc) => { + binary_int_ops!(handles, desc, B::int_add) + } + NumericOperationIr::AddScalar(desc) => { + scalar_int_ops!(handles, desc, B::int_add_scalar) + } + NumericOperationIr::Sub(desc) => { + binary_int_ops!(handles, desc, B::int_sub) + } + NumericOperationIr::SubScalar(desc) => { + scalar_int_ops!(handles, desc, B::int_sub_scalar) + } + NumericOperationIr::Div(desc) => { + binary_int_ops!(handles, desc, B::int_div) + } + NumericOperationIr::DivScalar(desc) => { + scalar_int_ops!(handles, desc, B::int_div_scalar) + } + NumericOperationIr::Rem(desc) => { + binary_int_ops!(handles, desc, B::int_remainder) + } + NumericOperationIr::RemScalar(desc) => { + scalar_int_ops!(handles, desc, B::int_remainder_scalar) + } + NumericOperationIr::Mul(desc) => { + binary_int_ops!(handles, desc, B::int_mul) + } + NumericOperationIr::MulScalar(desc) => { + scalar_int_ops!(handles, desc, B::int_mul_scalar) + } + NumericOperationIr::Abs(desc) => { + unary_int_ops!(handles, desc, B::int_abs) + } + NumericOperationIr::Full(desc) => { + let shape = desc.out.shape.clone(); + let output = B::int_full( + shape, + desc.value.into(), + &self.device, + desc.out.dtype.into(), + ); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::MeanDim(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_mean_dim) + } + NumericOperationIr::Mean(desc) => { + unary_int_ops!(handles, desc, B::int_mean) + } + NumericOperationIr::Sum(desc) => { + unary_int_ops!(handles, desc, B::int_sum) + } + NumericOperationIr::SumDim(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_sum_dim) + } + NumericOperationIr::Prod(desc) => { + unary_int_ops!(handles, desc, B::int_prod) + } + NumericOperationIr::ProdDim(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_prod_dim) + } + NumericOperationIr::Greater(desc) => { + binary_int_cmp_ops!(handles, desc, B::int_greater) + } + NumericOperationIr::GreaterElem(desc) => { + scalar_int_cmp_ops!(handles, desc, B::int_greater_elem) + } + NumericOperationIr::GreaterEqual(desc) => { + binary_int_cmp_ops!(handles, desc, B::int_greater_equal) + } + NumericOperationIr::GreaterEqualElem(desc) => { + scalar_int_cmp_ops!(handles, desc, B::int_greater_equal_elem) + } + NumericOperationIr::Lower(desc) => { + binary_int_cmp_ops!(handles, desc, B::int_lower) + } + NumericOperationIr::LowerElem(desc) => { + scalar_int_cmp_ops!(handles, desc, B::int_lower_elem) + } + NumericOperationIr::LowerEqual(desc) => { + binary_int_cmp_ops!(handles, desc, B::int_lower_equal) + } + NumericOperationIr::LowerEqualElem(desc) => { + scalar_int_cmp_ops!(handles, desc, B::int_lower_equal_elem) + } + NumericOperationIr::ArgMax(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_argmax) + } + NumericOperationIr::ArgMin(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_argmin) + } + NumericOperationIr::Max(desc) => { + unary_int_ops!(handles, desc, B::int_max) + } + NumericOperationIr::MaxDimWithIndices(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + + let (output, output_idx) = B::int_max_dim_with_indices(tensor, desc.dim); + handles.register_int_tensor::(&desc.out.id, output); + handles.register_int_tensor::(&desc.out_indices.id, output_idx); + } + NumericOperationIr::MinDimWithIndices(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + + let (output, output_idx) = B::int_min_dim_with_indices(tensor, desc.dim); + handles.register_int_tensor::(&desc.out.id, output); + handles.register_int_tensor::(&desc.out_indices.id, output_idx); + } + NumericOperationIr::Min(desc) => { + unary_int_ops!(handles, desc, B::int_min) + } + NumericOperationIr::MaxDim(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_max_dim) + } + NumericOperationIr::MinDim(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_min_dim) + } + NumericOperationIr::MaxAbs(desc) => { + unary_int_ops!(handles, desc, B::int_max_abs) + } + NumericOperationIr::MaxAbsDim(desc) => { + reduce_int_dim_ops!(handles, desc, B::int_max_abs_dim) + } + NumericOperationIr::Clamp(desc) => { + let tensor = handles.get_int_tensor::(&desc.tensor); + + let output = B::int_clamp(tensor, desc.min.into(), desc.max.into()); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::IntRandom(desc) => { + let shape = desc.out.shape.clone(); + + let output = B::int_random(shape, desc.distribution, &self.device); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::Powf(desc) => { + let lhs = handles.get_int_tensor::(&desc.lhs); + let rhs = handles.get_float_tensor::(&desc.rhs); + + let output = B::int_powf(lhs, rhs); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumSum(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + let output = B::int_cumsum(tensor, desc.axis); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumProd(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + let output = B::int_cumprod(tensor, desc.axis); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumMin(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + let output = B::int_cummin(tensor, desc.axis); + handles.register_int_tensor::(&desc.out.id, output); + } + NumericOperationIr::CumMax(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + let output = B::int_cummax(tensor, desc.axis); + handles.register_int_tensor::(&desc.out.id, output); + } + }, + OperationIr::Bool(op) => match op { + BoolOperationIr::IntoFloat(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_into_float(tensor); + handles.register_float_tensor::(&desc.out.id, output); + } + BoolOperationIr::IntoInt(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_into_int(tensor); + handles.register_int_tensor::(&desc.out.id, output); + } + BoolOperationIr::Not(desc) => { + let tensor = handles.get_bool_tensor::(&desc.input); + + let output = B::bool_not(tensor); + handles.register_bool_tensor::(&desc.out.id, output); + } + BoolOperationIr::And(desc) => { + binary_bool_ops!(handles, desc, B::bool_and) + } + BoolOperationIr::Or(desc) => { + binary_bool_ops!(handles, desc, B::bool_or) + } + }, + OperationIr::Int(op) => match op { + IntOperationIr::IntoFloat(desc) => { + let tensor = handles.get_int_tensor::(&desc.input); + + let output = B::int_into_float(tensor); + handles.register_float_tensor::(&desc.out.id, output); + } + IntOperationIr::Matmul(desc) => { + binary_int_ops!(handles, desc, B::int_matmul) + } + IntOperationIr::BitwiseAnd(desc) => { + binary_int_ops!(handles, desc, B::bitwise_and) + } + IntOperationIr::BitwiseAndScalar(desc) => { + scalar_int_ops!(handles, desc, B::bitwise_and_scalar) + } + IntOperationIr::BitwiseOr(desc) => { + binary_int_ops!(handles, desc, B::bitwise_or) + } + IntOperationIr::BitwiseOrScalar(desc) => { + scalar_int_ops!(handles, desc, B::bitwise_or_scalar) + } + IntOperationIr::BitwiseXor(desc) => { + binary_int_ops!(handles, desc, B::bitwise_xor) + } + IntOperationIr::BitwiseXorScalar(desc) => { + scalar_int_ops!(handles, desc, B::bitwise_xor_scalar) + } + IntOperationIr::BitwiseNot(desc) => { + unary_int_ops!(handles, desc, B::bitwise_not) + } + IntOperationIr::BitwiseLeftShift(desc) => { + binary_int_ops!(handles, desc, B::bitwise_left_shift) + } + IntOperationIr::BitwiseRightShift(desc) => { + binary_int_ops!(handles, desc, B::bitwise_right_shift) + } + IntOperationIr::BitwiseLeftShiftScalar(desc) => { + scalar_int_ops!(handles, desc, B::bitwise_left_shift_scalar) + } + IntOperationIr::BitwiseRightShiftScalar(desc) => { + scalar_int_ops!(handles, desc, B::bitwise_right_shift_scalar) + } + }, + OperationIr::Float(_dtype, op) => match op { + FloatOperationIr::Exp(desc) => { + unary_float_ops!(handles, desc, B::float_exp) + } + FloatOperationIr::Log(desc) => { + unary_float_ops!(handles, desc, B::float_log) + } + FloatOperationIr::Log1p(desc) => { + unary_float_ops!(handles, desc, B::float_log1p) + } + FloatOperationIr::Erf(desc) => { + unary_float_ops!(handles, desc, B::float_erf) + } + FloatOperationIr::PowfScalar(desc) => { + scalar_float_ops!(handles, desc, B::float_powf_scalar) + } + FloatOperationIr::Sqrt(desc) => { + unary_float_ops!(handles, desc, B::float_sqrt) + } + FloatOperationIr::Cos(desc) => { + unary_float_ops!(handles, desc, B::float_cos) + } + FloatOperationIr::Sin(desc) => { + unary_float_ops!(handles, desc, B::float_sin) + } + FloatOperationIr::Tanh(desc) => { + unary_float_ops!(handles, desc, B::float_tanh) + } + FloatOperationIr::Tan(desc) => unary_float_ops!(handles, desc, B::float_tan), + FloatOperationIr::Cosh(desc) => unary_float_ops!(handles, desc, B::float_cosh), + FloatOperationIr::Sinh(desc) => unary_float_ops!(handles, desc, B::float_sinh), + FloatOperationIr::ArcCos(desc) => unary_float_ops!(handles, desc, B::float_acos), + FloatOperationIr::ArcCosh(desc) => unary_float_ops!(handles, desc, B::float_acosh), + FloatOperationIr::ArcSin(desc) => unary_float_ops!(handles, desc, B::float_asin), + FloatOperationIr::ArcSinh(desc) => unary_float_ops!(handles, desc, B::float_asinh), + FloatOperationIr::ArcTan(desc) => unary_float_ops!(handles, desc, B::float_atan), + FloatOperationIr::ArcTanh(desc) => unary_float_ops!(handles, desc, B::float_atanh), + FloatOperationIr::ArcTan2(desc) => binary_float_ops!(handles, desc, B::float_atan2), + FloatOperationIr::Round(desc) => { + unary_float_ops!(handles, desc, B::float_round) + } + FloatOperationIr::Floor(desc) => { + unary_float_ops!(handles, desc, B::float_floor) + } + FloatOperationIr::Ceil(desc) => { + unary_float_ops!(handles, desc, B::float_ceil) + } + FloatOperationIr::Trunc(desc) => { + unary_float_ops!(handles, desc, B::float_trunc) + } + FloatOperationIr::IntoInt(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_into_int(tensor); + handles.register_int_tensor::(&desc.out.id, output); + } + FloatOperationIr::Matmul(desc) => { + binary_float_ops!(handles, desc, B::float_matmul) + } + FloatOperationIr::Cross(desc) => { + let lhs = handles.get_float_tensor::(&desc.lhs); + let rhs = handles.get_float_tensor::(&desc.rhs); + let output = B::float_cross(lhs, rhs, desc.dim); + handles.register_float_tensor::(&desc.out.id, output); + } + FloatOperationIr::Random(desc) => { + let shape = desc.out.shape.clone(); + + let output = B::float_random(shape, desc.distribution, &self.device); + handles.register_float_tensor::(&desc.out.id, output); + } + FloatOperationIr::Recip(desc) => { + unary_float_ops!(handles, desc, B::float_recip) + } + FloatOperationIr::Quantize(_) => todo!(), + FloatOperationIr::Dequantize(_) => todo!(), + FloatOperationIr::IsNan(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_is_nan(tensor); + handles.register_bool_tensor::(&desc.out.id, output); + } + FloatOperationIr::IsInf(desc) => { + let tensor = handles.get_float_tensor::(&desc.input); + + let output = B::float_is_inf(tensor); + handles.register_bool_tensor::(&desc.out.id, output); + } + FloatOperationIr::GridSample2d(desc) => { + let tensor = handles.get_float_tensor::(&desc.tensor); + let grid = handles.get_float_tensor::(&desc.grid); + + let output = B::float_grid_sample_2d(tensor, grid, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + }, + OperationIr::Module(op) => match op { + ModuleOperationIr::Embedding(desc) => { + let weights = handles.get_float_tensor::(&desc.weights); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::embedding(weights, indices); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::EmbeddingBackward(desc) => { + let weights = handles.get_float_tensor::(&desc.weights); + let indices = handles.get_int_tensor::(&desc.indices); + let output_grad = handles.get_float_tensor::(&desc.out_grad); + + let output = B::embedding_backward(weights, output_grad, indices); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv1d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv1d(x, weight, bias, desc.clone().options.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv1dXBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = + B::conv1d_x_backward(x, weight, output_grad, desc.clone().options.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv1dWeightBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = B::conv1d_weight_backward( + x, + weight, + output_grad, + desc.clone().options.into(), + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv1dBiasBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let bias = handles.get_float_tensor::(&desc.bias); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = B::conv1d_bias_backward(x, bias, output_grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv2d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv2d(x, weight, bias, desc.clone().options.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv2dXBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = + B::conv2d_x_backward(x, weight, output_grad, desc.clone().options.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv2dWeightBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = B::conv2d_weight_backward( + x, + weight, + output_grad, + desc.clone().options.into(), + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv2dBiasBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let bias = handles.get_float_tensor::(&desc.bias); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = B::conv2d_bias_backward(x, bias, output_grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv3d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv3d(x, weight, bias, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv3dXBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = + B::conv3d_x_backward(x, weight, output_grad, desc.clone().options.into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv3dWeightBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = B::conv3d_weight_backward( + x, + weight, + output_grad, + desc.clone().options.into(), + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Conv3dBiasBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let bias = handles.get_float_tensor::(&desc.bias); + let output_grad = handles.get_float_tensor::(&desc.output_grad); + + let output = B::conv3d_bias_backward(x, bias, output_grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::DeformableConv2d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let offset = handles.get_float_tensor::(&desc.offset); + let mask = desc + .mask + .as_ref() + .map(|mask| handles.get_float_tensor::(mask)); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::deform_conv2d( + x, + offset, + weight, + mask, + bias, + desc.options.clone().into(), + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::DeformableConv2dBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let offset = handles.get_float_tensor::(&desc.offset); + let mask = desc + .mask + .as_ref() + .map(|mask| handles.get_float_tensor::(mask)); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + let output_grad = handles.get_float_tensor::(&desc.out_grad); + + let output = B::deform_conv2d_backward( + x, + offset, + weight, + mask, + bias, + output_grad, + desc.options.clone().into(), + ); + + handles.register_float_tensor::(&desc.input_grad.id, output.x_grad); + handles.register_float_tensor::(&desc.offset_grad.id, output.offset_grad); + handles.register_float_tensor::(&desc.weight_grad.id, output.weight_grad); + if let Some((mask_grad, field)) = output.mask_grad.zip(desc.mask_grad.as_ref()) + { + handles.register_float_tensor::(&field.id, mask_grad); + } + if let Some((bias_grad, field)) = output.bias_grad.zip(desc.bias_grad.as_ref()) + { + handles.register_float_tensor::(&field.id, bias_grad); + } + } + ModuleOperationIr::ConvTranspose1d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv_transpose1d(x, weight, bias, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::ConvTranspose2d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv_transpose2d(x, weight, bias, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::ConvTranspose3d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let weight = handles.get_float_tensor::(&desc.weight); + let bias = desc + .bias + .as_ref() + .map(|bias| handles.get_float_tensor::(bias)); + + let output = B::conv_transpose3d(x, weight, bias, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AvgPool1d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::avg_pool1d( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.count_include_pad, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AvgPool2d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::avg_pool2d( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.count_include_pad, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AvgPool1dBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let grad = handles.get_float_tensor::(&desc.grad); + + let output = B::avg_pool1d_backward( + x, + grad, + desc.kernel_size, + desc.stride, + desc.padding, + desc.count_include_pad, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AvgPool2dBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let grad = handles.get_float_tensor::(&desc.grad); + + let output = B::avg_pool2d_backward( + x, + grad, + desc.kernel_size, + desc.stride, + desc.padding, + desc.count_include_pad, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AdaptiveAvgPool1d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::adaptive_avg_pool1d(x, desc.output_size); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AdaptiveAvgPool2d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::adaptive_avg_pool2d(x, desc.output_size); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AdaptiveAvgPool1dBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let grad = handles.get_float_tensor::(&desc.grad); + + let output = B::adaptive_avg_pool1d_backward(x, grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::AdaptiveAvgPool2dBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let grad = handles.get_float_tensor::(&desc.grad); + + let output = B::adaptive_avg_pool2d_backward(x, grad); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::MaxPool1d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::max_pool1d( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.dilation, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::MaxPool1dWithIndices(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::max_pool1d_with_indices( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.dilation, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output.output); + handles.register_int_tensor::(&desc.out_indices.id, output.indices); + } + ModuleOperationIr::MaxPool1dWithIndicesBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let output_grad = handles.get_float_tensor::(&desc.grad); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::max_pool1d_with_indices_backward( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.dilation, + desc.ceil_mode, + output_grad, + indices, + ); + handles.register_float_tensor::(&desc.out.id, output.x_grad); + } + ModuleOperationIr::MaxPool2d(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::max_pool2d( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.dilation, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::MaxPool2dWithIndices(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::max_pool2d_with_indices( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.dilation, + desc.ceil_mode, + ); + handles.register_float_tensor::(&desc.out.id, output.output); + handles.register_int_tensor::(&desc.out_indices.id, output.indices); + } + ModuleOperationIr::MaxPool2dWithIndicesBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let output_grad = handles.get_float_tensor::(&desc.grad); + let indices = handles.get_int_tensor::(&desc.indices); + + let output = B::max_pool2d_with_indices_backward( + x, + desc.kernel_size, + desc.stride, + desc.padding, + desc.dilation, + desc.ceil_mode, + output_grad, + indices, + ); + handles.register_float_tensor::(&desc.out.id, output.x_grad); + } + ModuleOperationIr::Interpolate(desc) => { + let x = handles.get_float_tensor::(&desc.x); + + let output = B::interpolate(x, desc.output_size, desc.options.clone().into()); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::InterpolateBackward(desc) => { + let x = handles.get_float_tensor::(&desc.x); + let grad = handles.get_float_tensor::(&desc.grad); + + let output = B::interpolate_backward( + x, + grad, + desc.output_size, + desc.options.clone().into(), + ); + handles.register_float_tensor::(&desc.out.id, output); + } + ModuleOperationIr::Attention(desc) => { + let query = handles.get_float_tensor::(&desc.query); + let key = handles.get_float_tensor::(&desc.key); + let value = handles.get_float_tensor::(&desc.value); + let mask = desc.mask.as_ref().map(|m| handles.get_bool_tensor::(m)); + let attn_bias = desc + .attn_bias + .as_ref() + .map(|ab| handles.get_float_tensor::(ab)); + + let output = B::attention( + query, + key, + value, + mask, + attn_bias, + desc.options.clone().into(), + ); + + handles.register_float_tensor::(&desc.out.id, output); + } + }, + OperationIr::Custom(_) => { + panic!("Can't execute custom operation here") + } + OperationIr::Init(_) => { + // Nothing to do. + } + OperationIr::Drop(repr) => { + handles.remove_handle(repr.id); + } + } + } + + fn read_tensor_async(&self, tensor: TensorIr) -> DynFut> { + let mut ctx = self.context.lock().unwrap(); + + enum Output { + Float(B::FloatTensorPrimitive), + Int(B::IntTensorPrimitive), + Bool(B::BoolTensorPrimitive), + } + + let tensor = if tensor.dtype.is_float() { + let tensor = ctx.handles.get_float_tensor::(&tensor); + Output::::Float(tensor) + } else if tensor.dtype.is_int() { + let tensor = ctx.handles.get_int_tensor::(&tensor); + Output::Int(tensor) + } else if tensor.dtype.is_bool() { + let tensor = ctx.handles.get_bool_tensor::(&tensor); + Output::Bool(tensor) + } else if let DType::QFloat(_) = tensor.dtype { + todo!() + } else { + unimplemented!() + }; + + match tensor { + Output::Float(val) => Box::pin(B::float_into_data(val)), + Output::Int(val) => Box::pin(B::int_into_data(val)), + Output::Bool(val) => Box::pin(B::bool_into_data(val)), + } + } + + fn register_tensor_data(&self, data: TensorData) -> RouterTensor { + let desc = self.register_tensor_data_desc(data); + RouterTensor::new(desc.id, desc.shape, desc.dtype, self.clone()) + } + + fn device(&self) -> Self::Device { + self.device.clone() + } + + fn sync(&self) -> Result<(), ExecutionError> { + B::sync(&self.device) + } + + fn seed(&self, seed: u64) { + B::seed(&self.device, seed) + } + + fn create_empty_handle(&self) -> TensorId { + let mut ctx = self.context.lock().unwrap(); + ctx.create_empty_handle() + } + + fn dtype_usage(&self, dtype: DType) -> burn_backend::DTypeUsageSet { + B::dtype_usage(&self.device, dtype) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/tensor.rs new file mode 100644 index 0000000..6bce82a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/tensor.rs @@ -0,0 +1,142 @@ +use core::sync::atomic::{AtomicU32, Ordering}; + +use alloc::format; +use alloc::{sync::Arc, vec::Vec}; + +use super::RunnerClient; +use burn_backend::{DType, Shape, TensorData, TensorMetadata, backend::ExecutionError}; +use burn_ir::{TensorId, TensorIr, TensorStatus}; + +/// Tensor primitive for the [router backend](crate::BackendRouter). +pub struct RouterTensor { + pub(crate) id: TensorId, + pub(crate) shape: Shape, + pub(crate) dtype: DType, + /// The client that has this tensor + pub client: C, + pub(crate) count: Arc, +} + +impl TensorMetadata for RouterTensor { + fn dtype(&self) -> DType { + self.dtype + } + + fn shape(&self) -> Shape { + self.shape.clone() + } + + fn rank(&self) -> usize { + self.shape.num_dims() + } +} + +impl RouterTensor { + /// Create a new router tensor. + pub fn new(id: TensorId, shape: Shape, dtype: DType, client: C) -> Self { + Self { + id, + shape, + dtype, + client, + count: Arc::new(AtomicU32::new(1)), + } + } + + pub(crate) async fn into_data(self) -> Result { + self.client.clone().read_tensor_async(self.into_ir()).await + } + + /// Get the ir for this tensor + pub fn into_ir(mut self) -> TensorIr { + let count = self.count.load(Ordering::Relaxed); + let status = self.status(count); + let mut shape_out = Shape::from(Vec::::new()); + core::mem::swap(&mut self.shape, &mut shape_out); + + if let TensorStatus::ReadWrite = status { + // Avoids an unwanted drop on the same thread. + // + // Since `drop` is called after `into_ir`, we must not register a drop if the tensor + // was consumed with a `ReadWrite` status. + self.count.fetch_add(1, Ordering::Relaxed); + } + + TensorIr { + status, + shape: shape_out, + id: self.id, + dtype: self.dtype, + } + } + + pub(crate) fn to_ir_out(&self) -> TensorIr { + TensorIr { + status: TensorStatus::NotInit, + shape: self.shape.clone(), + id: self.id, + dtype: self.dtype, + } + } + + pub(crate) fn status(&self, count: u32) -> TensorStatus { + if count <= 1 { + TensorStatus::ReadWrite + } else { + TensorStatus::ReadOnly + } + } +} + +impl core::fmt::Debug for RouterTensor { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str( + format!( + "{{ id: {:?}, shape: {:?}, dtype: {:?}, device: {:?} }}", + self.id, + self.shape, + self.dtype, + self.client.device().clone(), + ) + .as_str(), + ) + } +} + +impl Clone for RouterTensor { + fn clone(&self) -> Self { + self.count.fetch_add(1, Ordering::Relaxed); + + Self { + id: self.id, + shape: self.shape.clone(), + client: self.client.clone(), + dtype: self.dtype, + count: self.count.clone(), + } + } +} + +impl Drop for RouterTensor { + fn drop(&mut self) { + let count = self.count.fetch_sub(1, Ordering::Relaxed); + + match self.status(count) { + TensorStatus::ReadWrite => { + let id = self.id; + let mut shape = Shape::from(Vec::::new()); + core::mem::swap(&mut shape, &mut self.shape); + + let ir = TensorIr { + id, + shape, + status: TensorStatus::ReadWrite, + dtype: self.dtype, + }; + self.client.register_op(burn_ir::OperationIr::Drop(ir)); + } + TensorStatus::ReadOnly => {} + TensorStatus::NotInit => {} + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-router/src/types.rs b/crates/stable-diffusion-burn/burn-crates/burn-router/src/types.rs new file mode 100644 index 0000000..f42b9fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-router/src/types.rs @@ -0,0 +1,386 @@ +use alloc::format; +use alloc::string::String; +use burn_backend::{ + DType, Shape, TensorData, + backend::{Backend, DeviceId, DeviceOps, ExecutionError}, + try_read_sync, +}; +use burn_ir::{BackendIr, OperationIr, TensorHandle, TensorId, TensorIr}; +use burn_std::future::DynFut; + +use crate::{ + ByteBridge, DirectChannel, MultiBackendBridge, RouterTensor, Runner, RunnerChannel, + RunnerClient, +}; + +/// Implement multi backend types, with enums having one variant per backend. +macro_rules! impl_multi_backend_types { + // Match the default backend and at least one other backend, with rest being optional + ($module_name:ident, $DefaultBackend:ident, $($OtherBackend:ident),+) => { + /// Module containing the essential types for multi-backend operations. + /// + /// - `Handle`: the type used to point to a tensor (defined for all backends). + /// - `MultiRunnerClient`: a client for multiple runners (each responsible to execute tensor operations on a given backend). + /// - `DirectChannel`: a local channel with direct connection to the backend runner clients. + /// - `ByteBridge`: a simple multi-backend bridge that transfers tensors via the underlying [tensor data](burn_backend::TensorData). + /// + /// Each enum type is defined with backend identifiers as variant names (e.g., `B1` and `B2` for dual backends). + pub mod $module_name { + use super::*; + + /// The type that can be used to point to a tensor of any kind. + /// Each backend has its own variant. + pub enum Handle<$DefaultBackend: BackendIr, $($OtherBackend: BackendIr),+> { + #[allow(missing_docs)] + $DefaultBackend($DefaultBackend::Handle), + $( + #[allow(missing_docs)] + $OtherBackend($OtherBackend::Handle), + )+ + } + + /// The device type used by a backend. + /// Each backend has its own variant. + #[derive(Clone, Debug)] + pub enum MultiDevice<$DefaultBackend: Backend, $($OtherBackend: Backend),+> { + #[allow(missing_docs)] + $DefaultBackend($DefaultBackend::Device), + $( + #[allow(missing_docs)] + $OtherBackend($OtherBackend::Device), + )+ + } + impl<$DefaultBackend: Backend, $($OtherBackend: Backend),+> PartialEq for MultiDevice<$DefaultBackend, $($OtherBackend),+> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::$DefaultBackend(lhs), Self::$DefaultBackend(rhs)) => lhs == rhs, + $( + (Self::$OtherBackend(lhs), Self::$OtherBackend(rhs)) => lhs == rhs, + )+ + _ => false, + } + } + } + + // Default implementation always returns the first backend's device + impl<$DefaultBackend: Backend, $($OtherBackend: Backend),+> Default for MultiDevice<$DefaultBackend, $($OtherBackend),+> { + fn default() -> Self { + Self::$DefaultBackend($DefaultBackend::Device::default()) + } + } + + impl<$DefaultBackend: Backend, $($OtherBackend: Backend),+> burn_std::device::Device for MultiDevice<$DefaultBackend, $($OtherBackend),+> { + fn from_id(_device_id: DeviceId) -> Self { + // TODO: Should be fix with the new router backend. + Default::default() + } + + fn to_id(&self) -> DeviceId { + match self { + Self::$DefaultBackend(device) => device.id(), + $( + Self::$OtherBackend(device) => device.id(), + )+ + } + } + + fn device_count(_type_id: u16) -> usize { + 1 + } + } + + impl<$DefaultBackend: Backend, $($OtherBackend: Backend),+> DeviceOps for MultiDevice<$DefaultBackend, $($OtherBackend),+> {} + + /// A local client with multiple runners (each responsible to execute tensor operations on a given backend). + #[derive(Clone)] + pub enum MultiRunnerClient<$DefaultBackend: BackendIr, $($OtherBackend: BackendIr),+> { + #[allow(missing_docs)] + $DefaultBackend(Runner<$DefaultBackend>), + $( + #[allow(missing_docs)] + $OtherBackend(Runner<$OtherBackend>), + )+ + } + + impl<$DefaultBackend: BackendIr, $($OtherBackend: BackendIr),+> RunnerClient for MultiRunnerClient<$DefaultBackend, $($OtherBackend),+> + { + type Device = MultiDevice<$DefaultBackend, $($OtherBackend),+>; + + fn register_op(&self, op: OperationIr) { + match self { + Self::$DefaultBackend(runner) => runner.register_op(op), + $( + Self::$OtherBackend(runner) => runner.register_op(op), + )+ + } + } + + fn read_tensor_async(&self, tensor: TensorIr) -> DynFut> { + match self { + Self::$DefaultBackend(runner) => runner.read_tensor_async(tensor), + $( + Self::$OtherBackend(runner) => runner.read_tensor_async(tensor), + )+ + } + } + + fn register_tensor_data(&self, data: TensorData) -> RouterTensor { + match self { + Self::$DefaultBackend(runner) => { + let desc = runner.register_tensor_data_desc(data); + RouterTensor::new(desc.id, desc.shape, desc.dtype, self.clone()) + } + $( + Self::$OtherBackend(runner) => { + let desc = runner.register_tensor_data_desc(data); + RouterTensor::new(desc.id, desc.shape, desc.dtype, self.clone()) + } + )+ + } + } + + fn device(&self) -> Self::Device { + match self { + Self::$DefaultBackend(runner) => MultiDevice::$DefaultBackend(runner.device()), + $( + Self::$OtherBackend(runner) => MultiDevice::$OtherBackend(runner.device()), + )+ + } + } + + fn sync(&self) -> Result<(), ExecutionError> { + match self { + Self::$DefaultBackend(runner) => runner.sync(), + $( + Self::$OtherBackend(runner) => runner.sync(), + )+ + } + } + + fn seed(&self, seed: u64) { + match self { + Self::$DefaultBackend(runner) => runner.seed(seed), + $( + Self::$OtherBackend(runner) => runner.seed(seed), + )+ + } + } + + fn create_empty_handle(&self) -> TensorId { + match self { + Self::$DefaultBackend(runner) => runner.create_empty_handle(), + $( + Self::$OtherBackend(runner) => runner.create_empty_handle(), + )+ + } + } + + fn dtype_usage(&self, dtype: burn_std::DType) -> burn_backend::DTypeUsageSet { + match self { + Self::$DefaultBackend(runner) => runner.dtype_usage(dtype), + $( + Self::$OtherBackend(runner) => runner.dtype_usage(dtype), + )+ + } + } + } + + impl<$DefaultBackend: BackendIr, $($OtherBackend: BackendIr),+, Br> RunnerChannel for DirectChannel<($DefaultBackend, $($OtherBackend),+), Br> + where + Br: MultiBackendBridge, Device = MultiDevice<$DefaultBackend, $($OtherBackend),+>>, + { + type Device = Br::Device; + + type Bridge = Br; + + type FloatElem = $DefaultBackend::FloatElem; + type IntElem = $DefaultBackend::IntElem; + type BoolElem = $DefaultBackend::BoolElem; + + type Client = MultiRunnerClient<$DefaultBackend, $($OtherBackend),+>; + + fn init_client(device: &Self::Device) -> Self::Client { + match device { + MultiDevice::$DefaultBackend(device) => MultiRunnerClient::$DefaultBackend(Runner::new(device.clone())), + $( + MultiDevice::$OtherBackend(device) => MultiRunnerClient::$OtherBackend(Runner::new(device.clone())), + )+ + } + } + + fn get_tensor_handle( + tensor: &TensorIr, + client: &Self::Client, + ) -> ::TensorHandle { + match client { + MultiRunnerClient::$DefaultBackend(runner) => Handle::$DefaultBackend(runner.get_tensor_handle(tensor)), + $( + MultiRunnerClient::$OtherBackend(runner) => Handle::$OtherBackend(runner.get_tensor_handle(tensor)), + )+ + } + } + + fn register_tensor( + client: &Self::Client, + handle: ::TensorHandle, + shape: Shape, + dtype: DType, + ) -> RouterTensor { + match client { + MultiRunnerClient::$DefaultBackend(runner) => match handle { + Handle::$DefaultBackend(handle) => runner.register_tensor(handle, shape, dtype, client.clone()), + _ => unreachable!("Can't register tensor handle for another backend."), + }, + $( + MultiRunnerClient::$OtherBackend(runner) => match handle { + Handle::$OtherBackend(handle) => runner.register_tensor(handle, shape, dtype, client.clone()), + _ => unreachable!("Can't register tensor handle for another backend."), + }, + )+ + } + } + + fn name(_device: &Self::Device) -> String { + let mut name = format!("{}", $DefaultBackend::name(&<$DefaultBackend::Device as Default>::default())); + $( + name.push_str(&format!(", {}", $OtherBackend::name(&<$OtherBackend::Device as Default>::default()))); + )+ + format!("direct<({})>", name) + } + } + + impl<$DefaultBackend: BackendIr, $($OtherBackend: BackendIr),+> MultiBackendBridge for ByteBridge<($DefaultBackend, $($OtherBackend),+)> { + type TensorHandle = Handle<$DefaultBackend, $($OtherBackend),+>; + type Device = MultiDevice<$DefaultBackend, $($OtherBackend),+>; + + fn change_backend_float( + tensor: Self::TensorHandle, + shape: Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle { + multi_backend_match!(shape, (tensor, target_device) : $DefaultBackend, $($OtherBackend),+) + } + + fn change_backend_int( + tensor: Self::TensorHandle, + shape: Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle { + multi_backend_match!(shape, (tensor, target_device) : $DefaultBackend, $($OtherBackend),+) + } + + fn change_backend_bool( + tensor: Self::TensorHandle, + shape: Shape, + target_device: &Self::Device, + ) -> Self::TensorHandle { + multi_backend_match!(shape, (tensor, target_device) : $DefaultBackend, $($OtherBackend),+) + } + + } + } + }; +} + +macro_rules! bridge { + ($Backend:ident, $handle:expr, $device:expr, $shape:expr) => {{ + // Bridge for the same backend + let tensor = $Backend::float_tensor(TensorHandle { + handle: $handle, + shape: $shape, + }); + let tensor = $Backend::float_to_device(tensor, $device); + let handle = $Backend::float_tensor_handle(tensor); + Handle::$Backend(handle) + }}; + ($BackendA:ident, $BackendB:ident, $handle:expr, $device:expr, $shape:expr) => {{ + // Byte bridge between two backends + let tensor = $BackendA::float_tensor(TensorHandle { handle: $handle, shape: $shape }); + let data = try_read_sync($BackendA::float_into_data(tensor)).unwrap().expect( + "Failed to read tensor data synchronously. This can happen on platforms that don't support blocking futures like WASM." + ); + let tensor = $BackendB::float_from_data(data, $device); + let handle = $BackendB::float_tensor_handle(tensor); + Handle::$BackendB(handle) + }}; +} + +macro_rules! multi_backend_match { + ($shape:expr, ($handle:expr, $device:expr) : $DefaultBackend:ident, $($OtherBackend:ident),+) => { + multi_backend_match! ( + @step + $shape, + ($handle, $device); + { + (Handle::$DefaultBackend(handle), MultiDevice::$DefaultBackend(device)) => bridge!($DefaultBackend, handle, device, $shape), + $( + (Handle::$DefaultBackend(handle), MultiDevice::$OtherBackend(device)) => bridge!($DefaultBackend, $OtherBackend, handle, device, $shape), + (Handle::$OtherBackend(handle), MultiDevice::$DefaultBackend(device)) => bridge!($OtherBackend, $DefaultBackend, handle, device, $shape), + (Handle::$OtherBackend(handle), MultiDevice::$OtherBackend(device)) => bridge!($OtherBackend, handle, device, $shape), + )+ + }; + $($OtherBackend),+ + ) + }; + + (@step + $shape:expr, + $pats:tt; + { $($arms:tt)* }; + $BackendA:ident, + $($OtherBackend:ident),+ + ) => { + multi_backend_match! ( + @step + $shape, + $pats; + { + $($arms)* + $( + (Handle::$BackendA(handle), MultiDevice::$OtherBackend(device)) => bridge!($BackendA, $OtherBackend, handle, device, $shape), + (Handle::$OtherBackend(handle), MultiDevice::$BackendA(device)) => bridge!($OtherBackend, $BackendA, handle, device, $shape), + )* + }; + $($OtherBackend),* + ) + }; + + (@step + $shape:expr, + ($handle:expr, $device:expr); + { $($arms:tt)* }; + $($BackendA:ident)? + ) => { + match ($handle, $device) { + $($arms)* + } + }; +} + +// Implement multi-backend types and byte bridge for up to 4 backends +impl_multi_backend_types!(duo, B1, B2); +impl_multi_backend_types!(trio, B1, B2, B3); +impl_multi_backend_types!(quad, B1, B2, B3, B4); + +#[cfg(not(target_os = "windows"))] // cannot find a wgpu adapter on windows CI +#[cfg(test)] +mod tests { + use burn_tensor::{Tensor, backend::Backend}; + + use super::*; + use crate::tests::{TestBackend, TestBackend1, TestBackend2}; + + #[test] + fn should_support_dual_byte_bridge() { + let device1 = duo::MultiDevice::B1(::Device::default()); + let device2 = duo::MultiDevice::B2(::Device::default()); + let tensor1 = Tensor::::from_floats([1.0, 2.0, 3.0, 4.0], &device1); + let tensor2 = Tensor::::from_floats([5.0, 6.0, 7.0, 8.0], &device2); + + let tensor1_2 = tensor1.clone().to_device(&device2); + tensor1.into_data().assert_eq(&tensor1_2.into_data(), true); + + let tensor2_1 = tensor2.clone().to_device(&device1); + tensor2.into_data().assert_eq(&tensor2_1.into_data(), true); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-std/Cargo.toml new file mode 100644 index 0000000..ba5ff9a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/Cargo.toml @@ -0,0 +1,57 @@ +[package] +authors = ["Dilshod Tadjibaev (@antimora)"] +categories = [] +description = "Core types and utilities shared across the Burn ecosystem." +documentation = "https://docs.rs/burn-std" +edition.workspace = true +keywords = [] +license.workspace = true +name = "burn-std" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-std" +version.workspace = true + +[lints] +workspace = true + +[features] +cubecl = ["dep:cubecl"] +default = ["std", "cubecl-common/default"] +doc = ["default"] +std = ["cubecl-common/std", "num-traits/std"] +tracing = ["cubecl?/tracing", "cubecl-common/tracing"] + +network = ["dep:indicatif", "dep:reqwest", "dep:tokio"] + +[dependencies] +bytemuck = { workspace = true, features = ["extern_crate_alloc"] } +half = { workspace = true, features = ["bytemuck"] } +num-traits = { workspace = true } +serde = { workspace = true } +smallvec = { workspace = true, features = ["serde"] } + +cubecl = { workspace = true, optional = true, default-features = false } +cubecl-common = { workspace = true, default-features = false, features = [ + "serde", + "shared-bytes", +] } +cubecl-zspace = { workspace = true, default-features = false } +# Enable extra-platforms for portable-atomic support on targets without native atomics (e.g., thumbv6m) +# This is needed because cubecl-common's shared-bytes feature pulls in bytes +bytes = { workspace = true } + +# Network downloader +indicatif = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } + +[dev-dependencies] +dashmap = { workspace = true } + +# Enable extra-platforms for bytes on targets without native atomics (e.g., thumbv6m-none-eabi) +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +bytes = { workspace = true, features = ["extra-platforms"] } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/README.md b/crates/stable-diffusion-burn/burn-crates/burn-std/README.md new file mode 100644 index 0000000..6969baf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/README.md @@ -0,0 +1,7 @@ +# Burn Standard Library + +`burn-std` provides the core types and utilities shared across the Burn ecosystem. +It includes foundational definitions for shapes, indexing, and data types. + +This crate supports both `std` and `no_std` environments and must compile with +`cargo build --no-default-features` as well. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/id.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/id.rs new file mode 100644 index 0000000..5cda467 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/id.rs @@ -0,0 +1,69 @@ +//! # Unique Identifiers +use crate::rand::gen_random; + +/// Simple ID generator. +pub struct IdGenerator {} + +impl IdGenerator { + /// Generates a new ID. + pub fn generate() -> u64 { + // Generate a random u64 (18,446,744,073,709,551,615 combinations) + let random_bytes: [u8; 8] = gen_random(); + u64::from_le_bytes(random_bytes) + } +} + +pub use cubecl_common::stream_id::StreamId; + +#[cfg(test)] +mod tests { + use super::*; + + use alloc::collections::BTreeSet; + + #[cfg(feature = "std")] + use dashmap::DashSet; //Concurrent HashMap + #[cfg(feature = "std")] + use std::{sync::Arc, thread}; + + #[test] + fn uniqueness_test() { + const IDS_CNT: usize = 10_000; + + let mut set: BTreeSet = BTreeSet::new(); + + for _i in 0..IDS_CNT { + assert!(set.insert(IdGenerator::generate())); + } + + assert_eq!(set.len(), IDS_CNT); + } + + #[cfg(feature = "std")] + #[test] + fn thread_safety_test() { + const NUM_THREADS: usize = 10; + const NUM_REPEATS: usize = 1_000; + const EXPECTED_TOTAL_IDS: usize = NUM_THREADS * NUM_REPEATS; + + let set: Arc> = Arc::new(DashSet::new()); + + let mut handles = vec![]; + + for _ in 0..NUM_THREADS { + let set = set.clone(); + + let handle = thread::spawn(move || { + for _i in 0..NUM_REPEATS { + assert!(set.insert(IdGenerator::generate())); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + assert_eq!(set.len(), EXPECTED_TOTAL_IDS); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/lib.rs new file mode 100644 index 0000000..e16bce8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/lib.rs @@ -0,0 +1,97 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! # Burn Standard Library +//! +//! This library contains core types and utilities shared across Burn, including shapes, indexing, +//! and data types. + +extern crate alloc; + +/// Id module contains types for unique identifiers. +pub mod id; + +/// Tensor utilities. +pub mod tensor; +pub use tensor::*; + +/// Common Errors. +pub use cubecl_zspace::errors::{self, *}; + +/// Network utilities. +#[cfg(feature = "network")] +pub mod network; + +// Re-exported types +pub use cubecl_common::bytes::*; +pub use cubecl_common::*; +pub use half::{bf16, f16}; + +#[cfg(feature = "cubecl")] +pub use cubecl::flex32; + +#[cfg(feature = "cubecl")] +mod cube { + use cubecl::ir::{ElemType, FloatKind, IntKind, StorageType, UIntKind}; + use cubecl_common::quant::scheme::QuantScheme; + + use crate::tensor::DType; + use crate::tensor::quantization::{QuantStore, QuantValue}; + + impl From for cubecl::ir::ElemType { + fn from(dtype: DType) -> Self { + match dtype { + DType::F64 => ElemType::Float(FloatKind::F64), + DType::F32 => ElemType::Float(FloatKind::F32), + DType::Flex32 => ElemType::Float(FloatKind::Flex32), + DType::F16 => ElemType::Float(FloatKind::F16), + DType::BF16 => ElemType::Float(FloatKind::BF16), + DType::I64 => ElemType::Int(IntKind::I64), + DType::I32 => ElemType::Int(IntKind::I32), + DType::I16 => ElemType::Int(IntKind::I16), + DType::I8 => ElemType::Int(IntKind::I8), + DType::U64 => ElemType::UInt(UIntKind::U64), + DType::U32 => ElemType::UInt(UIntKind::U32), + DType::U16 => ElemType::UInt(UIntKind::U16), + DType::U8 => ElemType::UInt(UIntKind::U8), + DType::Bool => ElemType::Bool, + DType::QFloat(scheme) => match scheme.store { + QuantStore::Native => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S => Self::Int(IntKind::I8), + QuantValue::E4M3 => Self::Float(FloatKind::E4M3), + QuantValue::E5M2 => Self::Float(FloatKind::E5M2), + QuantValue::Q4F + | QuantValue::Q4S + | QuantValue::Q2F + | QuantValue::Q2S + | QuantValue::E2M1 => { + panic!("Can't store native sub-byte values") + } + }, + QuantStore::PackedU32(_) => Self::UInt(UIntKind::U32), + QuantStore::PackedNative(_) => match scheme.value { + QuantValue::E2M1 => panic!("Can't store native sub-byte values"), + other => panic!("{other:?} doesn't support native packing"), + }, + }, + } + } + } + + impl From for cubecl::ir::StorageType { + fn from(dtype: DType) -> cubecl::ir::StorageType { + match dtype { + DType::QFloat(QuantScheme { + store: QuantStore::PackedNative(_), + value: QuantValue::E2M1, + .. + }) => StorageType::Packed(ElemType::Float(FloatKind::E2M1), 2), + _ => { + let elem: ElemType = dtype.into(); + elem.into() + } + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/network.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/network.rs new file mode 100644 index 0000000..621cc10 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/network.rs @@ -0,0 +1,57 @@ +//! # Common Network Utilities + +/// Network download utilities. +pub mod downloader { + use indicatif::{ProgressBar, ProgressState, ProgressStyle}; + use reqwest::Client; + use std::io::Write; + + /// Download the file at the specified url. + /// File download progress is reported with the help of a [progress bar](indicatif). + /// + /// # Arguments + /// + /// * `url` - The file URL to download. + /// * `message` - The message to display on the progress bar during download. + /// + /// # Returns + /// + /// A vector of bytes containing the downloaded file data. + #[tokio::main(flavor = "current_thread")] + pub async fn download_file_as_bytes(url: &str, message: &str) -> Vec { + // Get file from web + let mut response = Client::new().get(url).send().await.unwrap(); + let total_size = response.content_length().unwrap(); + + // Pretty progress bar + let pb = ProgressBar::new(total_size); + let msg = message.to_owned(); + pb.set_style( + ProgressStyle::with_template( + "{msg}\n {wide_bar:.cyan/blue} {bytes}/{total_bytes} ({eta})", + ) + .unwrap() + .with_key( + "eta", + |state: &ProgressState, w: &mut dyn std::fmt::Write| { + write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap() + }, + ) + .progress_chars("▬ "), + ); + pb.set_message(msg.clone()); + + // Read stream into bytes + let mut downloaded: u64 = 0; + let mut bytes: Vec = Vec::with_capacity(total_size as usize); + while let Some(chunk) = response.chunk().await.unwrap() { + let num_bytes = bytes.write(&chunk).unwrap(); + let new = std::cmp::min(downloaded + (num_bytes as u64), total_size); + downloaded = new; + pb.set_position(new); + } + pb.finish_with_message(msg); + + bytes + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/dtype.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/dtype.rs new file mode 100644 index 0000000..94c8c9e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/dtype.rs @@ -0,0 +1,224 @@ +//! Tensor data type. + +use serde::{Deserialize, Serialize}; + +use crate::tensor::quantization::{QuantScheme, QuantStore, QuantValue}; +use crate::{bf16, f16}; + +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum DType { + F64, + F32, + Flex32, + F16, + BF16, + I64, + I32, + I16, + I8, + U64, + U32, + U16, + U8, + Bool, + QFloat(QuantScheme), +} + +#[cfg(feature = "cubecl")] +impl From for DType { + fn from(value: cubecl::ir::ElemType) -> Self { + match value { + cubecl::ir::ElemType::Float(float_kind) => match float_kind { + cubecl::ir::FloatKind::F16 => DType::F16, + cubecl::ir::FloatKind::BF16 => DType::BF16, + cubecl::ir::FloatKind::Flex32 => DType::Flex32, + cubecl::ir::FloatKind::F32 => DType::F32, + cubecl::ir::FloatKind::F64 => DType::F64, + cubecl::ir::FloatKind::TF32 => panic!("Not a valid DType for tensors."), + cubecl::ir::FloatKind::E2M1 + | cubecl::ir::FloatKind::E2M3 + | cubecl::ir::FloatKind::E3M2 + | cubecl::ir::FloatKind::E4M3 + | cubecl::ir::FloatKind::E5M2 + | cubecl::ir::FloatKind::UE8M0 => { + unimplemented!("Not yet supported, will be used for quantization") + } + }, + cubecl::ir::ElemType::Int(int_kind) => match int_kind { + cubecl::ir::IntKind::I8 => DType::I8, + cubecl::ir::IntKind::I16 => DType::I16, + cubecl::ir::IntKind::I32 => DType::I32, + cubecl::ir::IntKind::I64 => DType::I64, + }, + cubecl::ir::ElemType::UInt(uint_kind) => match uint_kind { + cubecl::ir::UIntKind::U8 => DType::U8, + cubecl::ir::UIntKind::U16 => DType::U16, + cubecl::ir::UIntKind::U32 => DType::U32, + cubecl::ir::UIntKind::U64 => DType::U64, + }, + _ => panic!("Not a valid DType for tensors."), + } + } +} + +impl DType { + /// Returns the size of a type in bytes. + pub const fn size(&self) -> usize { + match self { + DType::F64 => core::mem::size_of::(), + DType::F32 => core::mem::size_of::(), + DType::Flex32 => core::mem::size_of::(), + DType::F16 => core::mem::size_of::(), + DType::BF16 => core::mem::size_of::(), + DType::I64 => core::mem::size_of::(), + DType::I32 => core::mem::size_of::(), + DType::I16 => core::mem::size_of::(), + DType::I8 => core::mem::size_of::(), + DType::U64 => core::mem::size_of::(), + DType::U32 => core::mem::size_of::(), + DType::U16 => core::mem::size_of::(), + DType::U8 => core::mem::size_of::(), + DType::Bool => core::mem::size_of::(), + DType::QFloat(scheme) => match scheme.store { + QuantStore::Native => match scheme.value { + QuantValue::Q8F | QuantValue::Q8S => core::mem::size_of::(), + // e2m1 native is automatically packed by the kernels, so the actual storage is + // 8 bits wide. + QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1 => { + core::mem::size_of::() + } + QuantValue::Q4F | QuantValue::Q4S | QuantValue::Q2F | QuantValue::Q2S => { + // Sub-byte values have fractional size + 0 + } + }, + QuantStore::PackedU32(_) => core::mem::size_of::(), + QuantStore::PackedNative(_) => match scheme.value { + QuantValue::E2M1 => core::mem::size_of::(), + _ => 0, + }, + }, + } + } + /// Returns true if the data type is a floating point type. + pub fn is_float(&self) -> bool { + matches!( + self, + DType::F64 | DType::F32 | DType::Flex32 | DType::F16 | DType::BF16 + ) + } + /// Returns true if the data type is a signed integer type. + pub fn is_int(&self) -> bool { + matches!(self, DType::I64 | DType::I32 | DType::I16 | DType::I8) + } + /// Returns true if the data type is an unsigned integer type. + pub fn is_uint(&self) -> bool { + matches!(self, DType::U64 | DType::U32 | DType::U16 | DType::U8) + } + + /// Returns true if the data type is a boolean type + pub fn is_bool(&self) -> bool { + matches!(self, DType::Bool) + } + + /// Returns the data type name. + pub fn name(&self) -> &'static str { + match self { + DType::F64 => "f64", + DType::F32 => "f32", + DType::Flex32 => "flex32", + DType::F16 => "f16", + DType::BF16 => "bf16", + DType::I64 => "i64", + DType::I32 => "i32", + DType::I16 => "i16", + DType::I8 => "i8", + DType::U64 => "u64", + DType::U32 => "u32", + DType::U16 => "u16", + DType::U8 => "u8", + DType::Bool => "bool", + DType::QFloat(_) => "qfloat", + } + } +} + +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum FloatDType { + F64, + F32, + Flex32, + F16, + BF16, +} + +impl From for FloatDType { + fn from(value: DType) -> Self { + match value { + DType::F64 => FloatDType::F64, + DType::F32 => FloatDType::F32, + DType::Flex32 => FloatDType::Flex32, + DType::F16 => FloatDType::F16, + DType::BF16 => FloatDType::BF16, + _ => panic!("Expected float data type, got {value:?}"), + } + } +} + +impl From for DType { + fn from(value: FloatDType) -> Self { + match value { + FloatDType::F64 => DType::F64, + FloatDType::F32 => DType::F32, + FloatDType::Flex32 => DType::Flex32, + FloatDType::F16 => DType::F16, + FloatDType::BF16 => DType::BF16, + } + } +} + +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum IntDType { + I64, + I32, + I16, + I8, + U64, + U32, + U16, + U8, +} + +impl From for IntDType { + fn from(value: DType) -> Self { + match value { + DType::I64 => IntDType::I64, + DType::I32 => IntDType::I32, + DType::I16 => IntDType::I16, + DType::I8 => IntDType::I8, + DType::U64 => IntDType::U64, + DType::U32 => IntDType::U32, + DType::U16 => IntDType::U16, + DType::U8 => IntDType::U8, + _ => panic!("Expected int data type, got {value:?}"), + } + } +} + +impl From for DType { + fn from(value: IntDType) -> Self { + match value { + IntDType::I64 => DType::I64, + IntDType::I32 => DType::I32, + IntDType::I16 => DType::I16, + IntDType::I8 => DType::I8, + IntDType::U64 => DType::U64, + IntDType::U32 => DType::U32, + IntDType::U16 => DType::U16, + IntDType::U8 => DType::U8, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/mod.rs new file mode 100644 index 0000000..c11d911 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/mod.rs @@ -0,0 +1,221 @@ +pub mod dtype; +pub mod quantization; +pub mod shape; +pub mod slice; + +pub use dtype::*; +pub use quantization::*; +pub use shape::*; +pub use slice::*; + +pub use cubecl_zspace::indexing::{self, *}; +pub use cubecl_zspace::{Strides, metadata::Metadata, strides}; + +/// Check if the current tensor is contiguous. +/// +/// A tensor is considered contiguous if its elements are stored in memory +/// such that the stride at position `k` is equal to the product of the shapes +/// of all dimensions greater than `k`. +/// +/// This means that strides increase as you move from the rightmost to the leftmost dimension. +pub fn is_contiguous(shape: &[usize], strides: &[usize]) -> bool { + if shape.is_empty() { + return true; + } + + for (&expected, &stride) in contiguous_strides(shape).iter().zip(strides) { + if expected != stride { + return false; + } + } + + true +} + +/// Computes the strides for a contiguous tensor with the given shape. +/// +/// In a contiguous row-major tensor, the stride for each dimension +/// equals the product of all dimension sizes to its right. +pub fn contiguous_strides(shape: &[usize]) -> Strides { + let mut strides = strides![0; shape.len()]; + let mut current = 1; + + for (i, &dim) in shape.iter().enumerate().rev() { + strides[i] = current; + current *= dim; + } + + strides +} + +/// The action to take for a reshape operation. +#[derive(Debug)] +pub enum ReshapeAction { + /// Updating the strides is sufficient to handle the reshape. + UpdateStrides { + /// The new strides. + strides: Strides, + }, + /// The strides are not compatible, we should recompute the buffer. + Recompute, + /// The strides are already correct. + NoChange, +} + +/// The reshape kind. +#[derive(Debug)] +pub enum ReshapeAnalysis { + /// Original tensor is contiguous, can update the strides. + IsContiguous, + /// Original tensor is highly permutated, can't update the strides. + HighlyPermuted, + /// Only batch dimensions are added, can update the strides. + Broadcasted, + /// Dimensions are only split, can update the strides. + Split, + /// Original tensor is bigger than output shape. + SmallerRank, + /// New shape is the same. + NoChange, +} + +impl ReshapeAnalysis { + /// Returns the proper action to take for the current analysis. + fn action(self, shape: &[usize], strides: &[usize], shape_new: &[usize]) -> ReshapeAction { + match self { + ReshapeAnalysis::IsContiguous => ReshapeAction::UpdateStrides { + strides: contiguous_strides(shape_new), + }, + ReshapeAnalysis::NoChange => ReshapeAction::NoChange, + ReshapeAnalysis::HighlyPermuted | ReshapeAnalysis::SmallerRank => { + ReshapeAction::Recompute + } + ReshapeAnalysis::Broadcasted => { + let shape_rank = shape.len(); + let shape_new_rank = shape_new.len(); + let n_new_batch = shape_new_rank - shape_rank; + let num_elems = shape.iter().product::(); + let strides_new = broadcast_strides(n_new_batch, shape_rank, num_elems, strides); + + ReshapeAction::UpdateStrides { + strides: strides_new, + } + } + ReshapeAnalysis::Split => { + let strides_new = split_strides(shape, strides, shape_new); + + ReshapeAction::UpdateStrides { + strides: strides_new, + } + } + } + } +} + +/// Returns the proper action to take when reshaping a tensor. +pub fn reshape_action(shape: &[usize], strides: &[usize], shape_new: &[usize]) -> ReshapeAction { + reshape_analysis(shape, Some(strides), shape_new).action(shape, strides, shape_new) +} + +/// Calculate the new strides given added batch dimensions. +pub fn broadcast_strides( + n_new_batch: usize, + rank_prev: usize, + num_elems: usize, + strides: &[usize], +) -> Strides { + let mut strides_new = strides![num_elems; rank_prev + n_new_batch]; + + for (i, s) in strides.iter().enumerate() { + strides_new[i + n_new_batch] = *s; + } + + strides_new +} + +/// Calculate the new strides given added split dimensions. +pub fn split_strides(shape: &[usize], strides: &[usize], shape_new: &[usize]) -> Strides { + let mut strides_new = strides![1; shape_new.len()]; + + let mut old_idx = shape.len() - 1; + let mut current_stride = strides[old_idx]; + let mut dim_prod = 1; + + for (i, dim) in shape_new.iter().enumerate().rev() { + dim_prod *= *dim; + strides_new[i] = current_stride; + if *dim == 1 { + continue; + } else if dim_prod == shape[old_idx] { + old_idx = old_idx.saturating_sub(1); + current_stride = strides[old_idx]; + dim_prod = 1; + } else { + current_stride *= *dim; + } + } + + strides_new +} + +/// Returns the analysis of a reshape operation. +pub fn reshape_analysis( + shape: &[usize], + strides: Option<&[usize]>, + shape_new: &[usize], +) -> ReshapeAnalysis { + let shape_rank = shape.len(); + let shape_new_rank = shape_new.len(); + + let is_contiguous = match strides { + Some(strides) => is_contiguous(shape, strides), + None => false, + }; + + if is_contiguous { + return ReshapeAnalysis::IsContiguous; + } + + if shape_new_rank < shape_rank { + return ReshapeAnalysis::SmallerRank; + } + + let n_new_batch = shape_new_rank - shape_rank; + + match n_new_batch > 0 { + true => { + if shape == &shape_new[n_new_batch..shape_new_rank] + && shape_new[0..n_new_batch].iter().all(|it| *it == 1) + { + return ReshapeAnalysis::Broadcasted; + } else { + let mut dim_prod = 1; + let mut old_idx = 0; + for dim in shape_new { + dim_prod *= *dim; + + // We need to ignore unit dims because they don't affect analysis and break + // things because they match the default `dim_prod`. If we don't do this, + // reshapes like [2, 3] to [2, 3, 1] will panic from out of bounds access. + if *dim == 1 { + continue; + } else if dim_prod == shape[old_idx] { + dim_prod = 1; + old_idx += 1; + } else if dim_prod > shape[old_idx] { + return ReshapeAnalysis::HighlyPermuted; + } + } + return ReshapeAnalysis::Split; + } + } + + false => { + if shape == shape_new { + return ReshapeAnalysis::NoChange; + } + } + }; + + ReshapeAnalysis::HighlyPermuted +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/quantization.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/quantization.rs new file mode 100644 index 0000000..7048552 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/quantization.rs @@ -0,0 +1,393 @@ +//! Quantization data representation. + +// Re-exported types +pub use cubecl_common::quant::scheme::{ + BlockSize, QuantLevel, QuantMode, QuantParam, QuantScheme, QuantStore, QuantValue, +}; + +/// Alignment (in bytes) for quantization parameters in serialized tensor data. +/// +/// NOTE: This is currently f32-based since scales were originally always f32. +/// With `QuantParam` now supporting different precisions (F16, BF16, etc.), +/// this alignment may need to be revisited in the future. +pub const QPARAM_ALIGN: usize = core::mem::align_of::(); + +use alloc::vec::Vec; +use core::any::TypeId; +use num_traits::PrimInt; +use serde::{Deserialize, Serialize}; + +use crate::{DType, Metadata, Shape, bytes::Bytes}; + +#[derive( + Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default, +)] +/// The precision of accumulating elements. +pub enum QuantAcc { + /// Full precision. + #[default] + F32, + /// Half precision. + F16, + /// bfloat16 precision. + BF16, +} + +/// Specify if the output of an operation is quantized using the scheme of the input +/// or returned unquantized. +#[derive( + Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default, +)] +pub enum QuantPropagation { + /// The output is quantized using the scheme of the input. + Propagate, + /// The output is not quantized. + #[default] + Inhibit, +} + +/// The quantization tensor data parameters. +#[derive(Clone, Debug)] +pub struct QParams { + /// The scaling factor. + pub scales: S, +} + +/// A quantization parameter tensor descriptor. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QParamTensor { + /// Start of the tensor in the buffer + pub offset_start: usize, + /// Offset of tensor end from the end of the buffer + pub offset_end: usize, + /// Metadata of the tensor + pub metadata: Metadata, + /// Data type of the tensor + pub dtype: DType, +} + +/// Calculate the shape of the quantization parameters for a given tensor and level +pub fn params_shape(data_shape: &Shape, level: QuantLevel) -> Shape { + match level { + QuantLevel::Tensor => Shape::new([1]), + QuantLevel::Block(block_size) => { + let mut params_shape = data_shape.clone(); + let block_size = block_size.to_dim_vec(data_shape.num_dims()); + + for (shape, block_size) in params_shape.iter_mut().zip(block_size) { + *shape = (*shape).div_ceil(block_size as usize); + } + + params_shape + } + } +} + +/// Quantized data bytes representation. +/// +/// # Notes +/// 1) The quantized values are packed into 32-bit unsigned integers. For example, int8 +/// quantized values pack 4 grouped values into a single `u32`. When unpacking these values, +/// we make sure to retrieve only the meaningful values (and ignore the alignment padding). +/// 2) Quantization parameters are appended to the tensor data. +/// As such, the last bytes always correspond to the scale parameter. +/// If the quantization scheme includes an offset (zero-point) parameter, it is next to last. +pub struct QuantizedBytes { + /// The quantized values and quantization parameters represented as bytes. + pub bytes: Bytes, + /// The quantization scheme. + pub scheme: QuantScheme, + /// The number of quantized elements. + pub num_elements: usize, +} + +impl QuantizedBytes { + /// Creates a new quantized bytes representation. + pub fn new( + value: Vec, + scheme: QuantScheme, + scales: &[f32], + ) -> Self { + let num_elements = value.len(); + // Only used for 8-bit quantization data comparison in tests + if TypeId::of::() != TypeId::of::() { + panic!("Invalid quantized type"); + } + + // Re-interpret `Vec` as `Vec` with `Vec::from_raw_parts` + let i8s: Vec = bytemuck::allocation::cast_vec(value); + let mut bytes = Bytes::from_elems(i8s); + + match scheme.level { + QuantLevel::Tensor => { + let scale_bytes = bytemuck::bytes_of(&scales[0]); + bytes.extend_from_byte_slice_aligned(scale_bytes, QPARAM_ALIGN); + } + QuantLevel::Block(_block_size) => { + let mut scale_bytes = Vec::with_capacity(size_of_val(scales)); + for scale in scales { + scale_bytes.extend_from_slice(bytemuck::bytes_of(scale)); + } + bytes.extend_from_byte_slice_aligned(scale_bytes.as_slice(), QPARAM_ALIGN); + } + } + + Self { + bytes, + scheme, + num_elements, + } + } + + /// Returns the int8 quantized values with the quantization parameters. + pub fn into_vec_i8(self) -> (Vec, QParams>) { + let (values, (qparams, num_params)) = self.split_values_off(); + + // Quantization parameters are added at the end of the tensor data. + // As such, the last bytes always correspond to the scale parameter(s). + // For example, per-block quantization can have multiple parameters for a single tensor: + // [scale, scale, scale, ...] + let scale_size = core::mem::size_of::(); // scale is stored as f32 + let qparams_bytes: &[u8] = bytemuck::cast_slice(&qparams); + let total_bytes = qparams_bytes.len(); + + let scales_size = scale_size * num_params; + + let scales = bytemuck::cast_slice(&qparams_bytes[total_bytes - scales_size..]).to_vec(); + + (values, QParams { scales }) + } + + fn split_i8_values(self, num_params: usize) -> (Vec, Vec) { + let mut values = read_bytes_to_i8(self.bytes); + + let scale_size = num_params * size_of::(); + let values_end = values.len() - scale_size; + + let qparams = values.split_off(values_end); + + let qparams = if (qparams.as_ptr() as usize).is_multiple_of(4) { + let mut qparams = core::mem::ManuallyDrop::new(qparams); + unsafe { + Vec::::from_raw_parts( + qparams.as_mut_ptr() as _, + qparams.len() / 4, + qparams.capacity() / 4, + ) + } + } else { + #[cfg(target_endian = "little")] + { + // SAFETY: quantized bytes representation is created from packed u32 values in little endian + bytemuck::cast_vec(qparams) + } + #[cfg(target_endian = "big")] + { + crate::quantization::pack_i8s_to_u32s(bytemuck::cast_vec(qparams)) + } + }; + (values, qparams) + } + + /// Splits the quantized values of the tensor from the quantization parameters. + /// + /// Returns the values in i8 and a newly allocated vector containing the quantization parameters. + fn split_values_off(self) -> (Vec, (Vec, usize)) { + let num_params = match self.scheme.level { + QuantLevel::Tensor => 1, + QuantLevel::Block(block_size) => self.num_elements / block_size.num_elements(), + }; + + if let QuantStore::PackedU32(packed_dim) = self.scheme.store { + assert_eq!( + packed_dim, 0, + "Packing must be on innermost dimension for splitting off values" + ); + } + + let (values, qparams) = match self.scheme.store { + QuantStore::Native => self.split_i8_values(num_params), + QuantStore::PackedU32(_) => match self.scheme.value { + QuantValue::Q8F | QuantValue::Q8S => self.split_i8_values(num_params), + QuantValue::Q4F | QuantValue::Q4S | QuantValue::Q2F | QuantValue::Q2S => { + let mut values = self.bytes.try_into_vec::().unwrap(); + let scale_size = num_params; // size of f32 same as u32 + let values_end = values.len() - scale_size; + + let qparams = values.split_off(values_end); + // Sub-byte values are unpacked as i8s for value equality tests + let values = unpack_q_to_i8s(&values, self.num_elements, &self.scheme.value); + (values, qparams) + } + QuantValue::E4M3 | QuantValue::E5M2 | QuantValue::E2M1 => { + unimplemented!("Not yet supported") + } + }, + QuantStore::PackedNative(_) => unimplemented!("Not yet supported"), + }; + + (values, (qparams, num_params)) + } +} + +fn read_bytes_to_i8(bytes: Bytes) -> Vec { + match bytes.try_into_vec::() { + Ok(val) => val, + // Safety, + // + // `Vec` can be Re-interpreted as `Vec` since they share the same alignment. + Err(bytes) => unsafe { core::mem::transmute::, Vec>(bytes.to_vec()) }, + } +} + +/// Pack signed 8-bit integer values into a sequence of unsigned 32-bit integers. +pub fn pack_i8s_to_u32s(values: Vec) -> Vec { + // Shift and combine groups of four 8-bit values into a u32. + // Same as doing this: + // let result = (d_u8 & 0xFF) << 24 | (c_u8 & 0xFF) << 16 | (b_u8 & 0xFF) << 8 | (a_u8 & 0xFF); + #[cfg(target_endian = "big")] + { + values + .chunks(4) + .map(|x| { + x.iter() + .enumerate() + .fold(0u32, |acc, (i, x)| acc | (*x as u32 & 0xFF) << (i * 8)) + }) + .collect() + } + + // The order of bytes in little endian matches the above description, we just need to + // handle padding when the number of values is not a factor of 4 + #[cfg(target_endian = "little")] + { + let mut values = values; + let remainder = values.len() % 4; + if remainder != 0 { + // Pad with zeros + values.extend(core::iter::repeat_n(0, 4 - remainder)); + } + + let len = values.len() / 4; + let capacity = values.capacity() / 4; + + // Pre-forget the old vec and re-interpret as u32 + let mut values = core::mem::ManuallyDrop::new(values); + let ptr = values.as_mut_ptr() as *mut u32; + + unsafe { Vec::from_raw_parts(ptr, len, capacity) } + } +} + +/// Unpack integer values into a sequence of signed 8-bit integers. +pub(crate) fn unpack_q_to_i8s( + values: &[Q], + numel: usize, + value: &QuantValue, +) -> Vec { + let size_store = size_of::() * 8; + let size_quant = value.size_bits(); + let num_quants = size_store / size_quant; + let mask = Q::from((1 << size_quant) - 1).unwrap(); + let sign_shift = 8 - size_quant; // sign extension for sub-byte values + values + .iter() + .enumerate() + .flat_map(|(i, &packed)| { + // A single u32 could contain less than four 8-bit values... + let n = core::cmp::min(num_quants, numel - i * num_quants); + // Extract each 8-bit segment from u32 and cast back to i8 + // Same as doing this (when 4 values are fully packed): + // let a = (packed & 0xFF) as i8; + // let b = ((packed >> 8) & 0xFF) as i8; + // let c = ((packed >> 16) & 0xFF) as i8; + // let d = ((packed >> 24) & 0xFF) as i8; + (0..n).map(move |i| { + let raw = (packed >> (i * size_quant) & mask).to_u8().unwrap(); + ((raw << sign_shift) as i8) >> sign_shift + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + + use super::*; + use alloc::vec; + + #[test] + fn should_pack_i8s_to_u32() { + let packed = pack_i8s_to_u32s(vec![-128, 2, -3, 127]); + + assert_eq!(packed, vec![2147287680]); + } + + #[test] + fn should_pack_i8s_to_u32_padded() { + let packed = pack_i8s_to_u32s(vec![-128, 2, -3, 127, 55]); + let packed_padded = pack_i8s_to_u32s(vec![-128, 2, -3, 127, 55, 0, 0, 0]); + + assert_eq!(packed, vec![2147287680, 55]); + assert_eq!(packed, packed_padded); + } + + #[test] + fn should_unpack_u32s_to_i8s() { + let unpacked = unpack_q_to_i8s(&[2147287680u32], 4, &QuantValue::Q8S); + + assert_eq!(unpacked, vec![-128, 2, -3, 127]); + } + + #[test] + fn should_unpack_u32s_to_i8s_padded() { + let unpacked = unpack_q_to_i8s(&[55u32], 1, &QuantValue::Q8S); + + assert_eq!(unpacked, vec![55]); + } + + #[test] + fn should_unpack_u32s_to_i8s_arange() { + let unpacked = unpack_q_to_i8s( + &[ + 0u32, 286331136, 286331153, 572657937, 572662306, 857874978, 858993459, 858993459, + 1145324612, 1145324612, 1431655748, 1431655765, 1717982549, 1717986918, 2003199590, + 2004318071, + ], + 128, + &QuantValue::Q4S, + ); + + assert_eq!( + unpacked, + vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 + ] + ); + } + + #[test] + fn should_pack_unpack_quantization_parameters_per_tensor_symmetric() { + // Quantized [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]] + let scale = 0.03937008; + let values = vec![0i8, 25, 51, 76, 102, 127]; + + let q_bytes = QuantizedBytes::new( + values.clone(), + QuantScheme::default() + .with_value(QuantValue::Q8S) + .with_store(QuantStore::Native), + &[scale], + ); + + let (q_values, qparams) = q_bytes.into_vec_i8(); + + assert_eq!(qparams.scales, vec![scale]); + + assert_eq!(q_values, values); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/shape.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/shape.rs new file mode 100644 index 0000000..7f2cdce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/shape.rs @@ -0,0 +1,271 @@ +//! Tensor shape definition. + +use super::{Slice, SliceArg}; +use alloc::vec::Vec; +use core::ops::Range; + +pub use crate::errors::ExpressionError; + +pub use cubecl_zspace::{MetadataError, Shape, calculate_matmul_output, shape}; + +/// Slice-related ops on [`Shape`] +pub trait SliceOps: Sized { + /// Convert shape dimensions to full covering ranges (0..dim) for each dimension. + fn into_ranges(self) -> Vec>; + /// Converts slice arguments into an array of slice specifications for the shape. + /// + /// This method returns an array of `Slice` objects that can be used for slicing operations. + /// The slices are clamped to the shape's dimensions. Similar to `into_ranges()`, but + /// allows custom slice specifications instead of full ranges. + /// For creating complex slice specifications, use the [`s!`] macro. + /// + /// # Arguments + /// + /// * `slices` - An array of slice specifications, where each element can be: + /// - A range (e.g., `2..5`) + /// - An index + /// - A `Slice` object + /// - The output of the [`s!`] macro for advanced slicing + /// + /// # Behavior + /// + /// - Supports partial and full slicing in any number of dimensions. + /// - Missing ranges are treated as full slices if D > D2. + /// - Handles negative indices by wrapping around from the end of the dimension. + /// - Clamps ranges to the shape's dimensions if they exceed the bounds. + /// + /// # Returns + /// + /// An array of `Slice` objects corresponding to the provided slice specifications, + /// clamped to the shape's actual dimensions. + /// + /// # Examples + /// + /// ```rust + /// use burn_std::{Shape, Slice, s, SliceOps}; + /// + /// fn example() { + /// // 1D slicing + /// let slices = Shape::new([4]).into_slices(1..4); + /// assert_eq!(slices[0].to_range(4), 1..3); + /// + /// // 2D slicing + /// let slices = Shape::new([3, 4]).into_slices(s![1..4, 0..2]); + /// assert_eq!(slices[0].to_range(3), 1..3); + /// assert_eq!(slices[1].to_range(4), 0..2); + /// + /// // Using negative indices + /// let slices = Shape::new([3]).into_slices(..-2); + /// assert_eq!(slices[0].to_range(3), 0..1); + /// + /// // Using the slice macro to select different ranges + /// let slices = Shape::new([2, 3, 4]).into_slices(s![.., 1..-1]); + /// assert_eq!(slices[0].to_range(2), 0..2); + /// assert_eq!(slices[1].to_range(3), 1..2); + /// } + /// ``` + /// + /// # See Also + /// + /// - [`s!`] - The recommended macro for creating slice specifications + /// - [`Shape::into_ranges`] - Convert to full covering ranges + /// + /// [`s!`]: crate::s! + fn into_slices(self, slices: S) -> Vec + where + S: SliceArg; + /// Compute the output shape from the given slices. + fn slice(self, slices: &[Slice]) -> Result; +} + +impl SliceOps for Shape { + fn into_ranges(self) -> Vec> { + self.iter().map(|&d| 0..d).collect() + } + + fn into_slices(self, slices: S) -> Vec + where + S: SliceArg, + { + slices.into_slices(&self) + } + + fn slice(mut self, slices: &[Slice]) -> Result { + if slices.len() > self.rank() { + return Err(MetadataError::RankMismatch { + left: self.rank(), + right: slices.len(), + }); + } + + slices + .iter() + .zip(self.iter_mut()) + .for_each(|(slice, dim_size)| *dim_size = slice.output_size(*dim_size)); + + Ok(self) + } +} + +#[cfg(test)] +#[allow(clippy::identity_op, reason = "useful for clarity")] +mod tests { + use super::*; + use crate::s; + use alloc::vec; + + #[test] + fn test_into_ranges() { + let dims = [2, 3, 4, 5]; + let shape = Shape::new(dims); + assert_eq!(shape.into_ranges(), vec![0..2, 0..3, 0..4, 0..5]); + } + + #[allow(clippy::single_range_in_vec_init)] + #[test] + fn test_into_slices() { + let slices = Shape::new([3]).into_slices(1..4); + assert_eq!(slices[0].to_range(3), 1..3); + + let slices = Shape::new([3, 4]).into_slices(s![1..4, 0..2]); + assert_eq!(slices[0].to_range(3), 1..3); + assert_eq!(slices[1].to_range(4), 0..2); + + let slices = Shape::new([3]).into_slices(..-2); + assert_eq!(slices[0].to_range(3), 0..1); + + let slices = Shape::new([2, 3, 4]).into_slices(s![.., 1..-1]); + assert_eq!(slices[0].to_range(2), 0..2); + assert_eq!(slices[1].to_range(3), 1..2); + + let slices = Shape::new([2, 3, 4]).into_slices(s![..20, 2]); + assert_eq!(slices[0].to_range(2), 0..2); + assert_eq!(slices[1].to_range(3), 2..3); + } + + #[test] + fn test_shape_as_slice() { + let dims = [2, 3, 4, 5]; + let shape = Shape::new(dims); + + assert_eq!(shape.as_slice(), dims.as_slice()); + + // Deref coercion + let shape_slice: &[usize] = &shape; + assert_eq!(shape_slice, *&[2, 3, 4, 5]); + } + + #[test] + fn test_shape_as_mut_slice() { + let mut dims = [2, 3, 4, 5]; + let mut shape = Shape::new(dims); + + let shape_mut = shape.as_mut_slice(); + assert_eq!(shape_mut, dims.as_mut_slice()); + shape_mut[1] = 6; + + assert_eq!(shape_mut, &[2, 6, 4, 5]); + + let mut shape = Shape::new(dims); + let shape = &mut shape[..]; + shape[1] = 6; + + assert_eq!(shape, shape_mut) + } + + #[test] + fn test_shape_slice_output_shape_basic() { + // Test basic slicing with step=1 + let slices = [ + Slice::new(0, Some(5), 1), // 5 elements + Slice::new(2, Some(8), 1), // 6 elements + ]; + let original_shape = Shape::new([10, 10, 10]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([5, 6, 10])); + } + + #[test] + fn test_shape_slice_output_shape_with_positive_steps() { + // Test slicing with various positive steps + let slices = [ + Slice::new(0, Some(10), 2), // [0,2,4,6,8] -> 5 elements + Slice::new(1, Some(9), 3), // [1,4,7] -> 3 elements + Slice::new(0, Some(7), 4), // [0,4] -> 2 elements + ]; + let original_shape = Shape::new([20, 20, 20, 30]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([5, 3, 2, 30])); + } + + #[test] + fn test_shape_slice_output_shape_with_negative_steps() { + // Test slicing with negative steps (backward iteration) + let slices = [ + Slice::new(0, Some(10), -1), // 10 elements traversed backward + Slice::new(2, Some(8), -2), // [7,5,3] -> 3 elements + ]; + let original_shape = Shape::new([20, 20, 20]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([10, 3, 20])); + } + + #[test] + fn test_shape_slice_output_shape_mixed_steps() { + // Test with a mix of positive, negative, and unit steps + let slices = [ + Slice::from_range_stepped(1..6, 1), // 5 elements + Slice::from_range_stepped(0..10, -3), // [9,6,3,0] -> 4 elements + Slice::from_range_stepped(2..14, 4), // [2,6,10] -> 3 elements + ]; + let original_shape = Shape::new([20, 20, 20]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([5, 4, 3])); + } + + #[test] + fn test_shape_slice_output_shape_partial_dims() { + // Test when slices has fewer dimensions than original shape + let slices = [ + Slice::from_range_stepped(2..7, 2), // [2,4,6] -> 3 elements + ]; + let original_shape = Shape::new([10, 20, 30, 40]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([3, 20, 30, 40])); + } + + #[test] + fn test_shape_slice_output_shape_edge_cases() { + // Test edge cases with small ranges and large steps + let slices = [ + Slice::from_range_stepped(0..1, 1), // Single element + Slice::from_range_stepped(0..10, 100), // Step larger than range -> 1 element + Slice::from_range_stepped(5..5, 1), // Empty range -> 0 elements + ]; + let original_shape = Shape::new([10, 20, 30]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([1, 1, 0])); + } + + #[test] + fn test_shape_slice_output_shape_empty() { + // Test with no slice infos (should return original shape) + let slices = []; + let original_shape = Shape::new([10, 20, 30]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([10, 20, 30])); + } + + #[test] + fn test_shape_slice_output_shape_uneven_division() { + // Test cases where range size doesn't divide evenly by step + let slices = [ + Slice::from_range_stepped(0..7, 3), // ceil(7/3) = 3 elements: [0,3,6] + Slice::from_range_stepped(0..11, 4), // ceil(11/4) = 3 elements: [0,4,8] + Slice::from_range_stepped(1..10, 5), // ceil(9/5) = 2 elements: [1,6] + ]; + let original_shape = Shape::new([20, 20, 20]); + let result = original_shape.slice(&slices).unwrap(); + assert_eq!(result, Shape::new([3, 3, 2])); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/slice.rs b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/slice.rs new file mode 100644 index 0000000..7a90e44 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-std/src/tensor/slice.rs @@ -0,0 +1,937 @@ +//! Tensor slice utilities. + +use crate::Shape; +use crate::indexing::AsIndex; +use alloc::format; +use alloc::vec::Vec; +use core::fmt::{Display, Formatter}; +use core::ops::{Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive}; +use core::str::FromStr; + +/// Trait for slice arguments that can be converted into an array of slices. +/// This allows the `slice` method to accept both single slices (from `s![..]`) +/// and arrays of slices (from `s![.., ..]` or `[0..5, 1..3]`). +pub trait SliceArg { + /// Convert to an vec of slices with clamping to shape dimensions. + /// + /// Returns a [Slice] for each dimension in `shape`. + fn into_slices(self, shape: &Shape) -> Vec; +} + +impl + Clone> SliceArg for &[S] { + fn into_slices(self, shape: &Shape) -> Vec { + assert!( + self.len() <= shape.num_dims(), + "Too many slices provided for shape, got {} but expected at most {}", + self.len(), + shape.num_dims() + ); + + shape + .iter() + .enumerate() + .map(|(i, dim_size)| { + let slice = if i >= self.len() { + Slice::full() + } else { + self[i].clone().into() + }; + // Apply shape clamping by converting to range and back + let clamped_range = slice.to_range(*dim_size); + Slice::new( + clamped_range.start as isize, + Some(clamped_range.end as isize), + slice.step(), + ) + }) + .collect::>() + } +} + +impl SliceArg for &Vec { + fn into_slices(self, shape: &Shape) -> Vec { + self.as_slice().into_slices(shape) + } +} + +impl SliceArg for [T; R] +where + T: Into + Clone, +{ + fn into_slices(self, shape: &Shape) -> Vec { + self.as_slice().into_slices(shape) + } +} + +impl SliceArg for T +where + T: Into, +{ + fn into_slices(self, shape: &Shape) -> Vec { + let slice: Slice = self.into(); + [slice].as_slice().into_slices(shape) + } +} + +/// Slice argument constructor for tensor indexing. +/// +/// The `s![]` macro is used to create multi-dimensional slice specifications for tensors. +/// It converts various range syntax forms into a `&[Slice]` that can be used with +/// `tensor.slice()` and `tensor.slice_assign()` operations. +/// +/// # Syntax Overview +/// +/// ## Basic Forms +/// +/// * **`s![index]`** - Index a single element (produces a subview with that axis removed) +/// * **`s![range]`** - Slice a range of elements +/// * **`s![range;step]`** - Slice a range with a custom step +/// * **`s![dim1, dim2, ...]`** - Multiple dimensions, each can be any of the above forms +/// +/// ## Range Types +/// +/// All standard Rust range types are supported: +/// * **`a..b`** - From `a` (inclusive) to `b` (exclusive) +/// * **`a..=b`** - From `a` to `b` (both inclusive) +/// * **`a..`** - From `a` to the end +/// * **`..b`** - From the beginning to `b` (exclusive) +/// * **`..=b`** - From the beginning to `b` (inclusive) +/// * **`..`** - The full range (all elements) +/// +/// ## Negative Indices +/// +/// Negative indices count from the end of the axis: +/// * **`-1`** refers to the last element +/// * **`-2`** refers to the second-to-last element +/// * And so on... +/// +/// This works in all range forms: `s![-3..-1]`, `s![-2..]`, `s![..-1]` +/// +/// ## Step Syntax +/// +/// Steps control the stride between selected elements: +/// * **`;step`** after a range specifies the step +/// * **Positive steps** select every nth element going forward +/// * **Negative steps** select every nth element going backward +/// * Default step is `1` when not specified +/// * Step cannot be `0` +/// +/// ### Negative Step Behavior +/// +/// With negative steps, the range bounds still specify *which* elements to include, +/// but the traversal order is reversed: +/// +/// * `s![0..5;-1]` selects indices `[4, 3, 2, 1, 0]` (not `[0, 1, 2, 3, 4]`) +/// * `s![2..8;-2]` selects indices `[7, 5, 3]` (starting from 7, going backward by 2) +/// * `s![..;-1]` reverses the entire axis +/// +/// This matches the semantics of NumPy and the ndarray crate. +/// +/// # Examples +/// +/// ## Basic Slicing +/// +/// ```rust,ignore +/// use burn_tensor::{Tensor, s}; +/// +/// # fn example(tensor: Tensor) { +/// // Select rows 0-5 (exclusive) +/// let subset = tensor.slice(s![0..5, .., ..]); +/// +/// // Select the last row +/// let last_row = tensor.slice(s![-1, .., ..]); +/// +/// // Select columns 2, 3, 4 +/// let cols = tensor.slice(s![.., 2..5, ..]); +/// +/// // Select a single element at position [1, 2, 3] +/// let element = tensor.slice(s![1, 2, 3]); +/// # } +/// ``` +/// +/// ## Slicing with Steps +/// +/// ```rust,ignore +/// use burn_tensor::{Tensor, s}; +/// +/// # fn example(tensor: Tensor) { +/// // Select every 2nd row +/// let even_rows = tensor.slice(s![0..10;2, ..]); +/// +/// // Select every 3rd column +/// let cols = tensor.slice(s![.., 0..9;3]); +/// +/// // Select every 2nd element in reverse order +/// let reversed_even = tensor.slice(s![10..0;-2, ..]); +/// # } +/// ``` +/// +/// ## Reversing Dimensions +/// +/// ```rust,ignore +/// use burn_tensor::{Tensor, s}; +/// +/// # fn example(tensor: Tensor) { +/// // Reverse the first dimension +/// let reversed = tensor.slice(s![..;-1, ..]); +/// +/// // Reverse both dimensions +/// let fully_reversed = tensor.slice(s![..;-1, ..;-1]); +/// +/// // Reverse a specific range +/// let range_reversed = tensor.slice(s![2..8;-1, ..]); +/// # } +/// ``` +/// +/// ## Complex Multi-dimensional Slicing +/// +/// ```rust,ignore +/// use burn_tensor::{Tensor, s}; +/// +/// # fn example(tensor: Tensor) { +/// // Mix of different slice types +/// let complex = tensor.slice(s![ +/// 0..10;2, // Every 2nd element from 0 to 10 +/// .., // All elements in dimension 1 +/// 5..15;-3, // Every 3rd element from 14 down to 5 +/// -1 // Last element in dimension 3 +/// ]); +/// +/// // Using inclusive ranges +/// let inclusive = tensor.slice(s![2..=5, 1..=3, .., ..]); +/// +/// // Negative indices with steps +/// let from_end = tensor.slice(s![-5..-1;2, .., .., ..]); +/// # } +/// ``` +/// +/// ## Slice Assignment +/// +/// ```rust,ignore +/// use burn_tensor::{Tensor, s}; +/// +/// # fn example(tensor: Tensor, values: Tensor) { +/// // Assign to every 2nd row +/// let tensor = tensor.slice_assign(s![0..10;2, ..], values); +/// +/// // Assign to a reversed slice +/// let tensor = tensor.slice_assign(s![..;-1, 0..5], values); +/// # } +/// ``` +#[macro_export] +macro_rules! s { + // Empty - should not happen + [] => { + compile_error!("Empty slice specification") + }; + + // Single expression with step + [$range:expr; $step:expr] => { + { + #[allow(clippy::reversed_empty_ranges)] + { + $crate::tensor::Slice::from_range_stepped($range, $step) + } + } + }; + + // Single expression without step (no comma after) + [$range:expr] => { + { + #[allow(clippy::reversed_empty_ranges)] + { + $crate::tensor::Slice::from($range) + } + } + }; + + // Two or more expressions with first having step + [$range:expr; $step:expr, $($rest:tt)*] => { + { + #[allow(clippy::reversed_empty_ranges)] + { + $crate::s!(@internal [$crate::tensor::Slice::from_range_stepped($range, $step)] $($rest)*) + } + } + }; + + // Two or more expressions with first not having step + [$range:expr, $($rest:tt)*] => { + { + #[allow(clippy::reversed_empty_ranges)] + { + $crate::s!(@internal [$crate::tensor::Slice::from($range)] $($rest)*) + } + } + }; + + // Internal: finished parsing + (@internal [$($acc:expr),*]) => { + [$($acc),*] + }; + + // Internal: parse range with step followed by comma + (@internal [$($acc:expr),*] $range:expr; $step:expr, $($rest:tt)*) => { + $crate::s!(@internal [$($acc,)* $crate::tensor::Slice::from_range_stepped($range, $step as isize)] $($rest)*) + }; + + // Internal: parse range with step at end + (@internal [$($acc:expr),*] $range:expr; $step:expr) => { + $crate::s!(@internal [$($acc,)* $crate::tensor::Slice::from_range_stepped($range, $step as isize)]) + }; + + // Internal: parse range without step followed by comma + (@internal [$($acc:expr),*] $range:expr, $($rest:tt)*) => { + $crate::s!(@internal [$($acc,)* $crate::tensor::Slice::from($range)] $($rest)*) + }; + + // Internal: parse range without step at end + (@internal [$($acc:expr),*] $range:expr) => { + $crate::s!(@internal [$($acc,)* $crate::tensor::Slice::from($range)]) + }; +} + +/// A slice specification for a single tensor dimension. +/// +/// This struct represents a range with an optional step, used for advanced indexing +/// operations on tensors. It is typically created using the [`s!`] macro rather than +/// constructed directly. +/// +/// # Fields +/// +/// * `start` - The starting index (inclusive). Negative values count from the end. +/// * `end` - The ending index (exclusive). `None` means to the end of the dimension. +/// * `step` - The stride between elements. Must be non-zero. +/// +/// # Index Interpretation +/// +/// - **Positive indices**: Count from the beginning (0-based) +/// - **Negative indices**: Count from the end (-1 is the last element) +/// - **Bounds checking**: Indices are clamped to valid ranges +/// +/// # Step Behavior +/// +/// - **Positive step**: Traverse forward through the range +/// - **Negative step**: Traverse backward through the range +/// - **Step size**: Determines how many elements to skip +/// +/// # Examples +/// +/// While you typically use the [`s!`] macro, you can also construct slices directly: +/// +/// ```rust,ignore +/// use burn_tensor::Slice; +/// +/// // Equivalent to s![2..8] +/// let slice1 = Slice::new(2, Some(8), 1); +/// +/// // Equivalent to s![0..10;2] +/// let slice2 = Slice::new(0, Some(10), 2); +/// +/// // Equivalent to s![..;-1] (reverse) +/// let slice3 = Slice::new(0, None, -1); +/// ``` +/// +/// See also the [`s!`] macro for the preferred way to create slices. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Slice { + /// Slice start index. + pub start: isize, + /// Slice end index (exclusive). + pub end: Option, + /// Step between elements (default: 1). + pub step: isize, +} + +/// Defines an [`Iterator`] over a [`Slice`]. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SliceIter { + slice: Slice, + current: isize, +} + +impl Iterator for SliceIter { + type Item = isize; + + fn next(&mut self) -> Option { + let next = self.current; + self.current += self.slice.step; + + if let Some(end) = self.slice.end { + if self.slice.is_reversed() { + if next <= end { + return None; + } + } else if next >= end { + return None; + } + } + + Some(next) + } +} + +/// Note: Unbounded [`Slice`]s produce infinite iterators. +impl IntoIterator for Slice { + type Item = isize; + type IntoIter = SliceIter; + + fn into_iter(self) -> Self::IntoIter { + SliceIter { + slice: self, + current: self.start, + } + } +} + +impl Default for Slice { + fn default() -> Self { + Self::full() + } +} + +impl Slice { + /// Creates a new slice with start, end, and step + pub const fn new(start: isize, end: Option, step: isize) -> Self { + assert!(step != 0, "Step cannot be zero"); + Self { start, end, step } + } + + /// Creates a slice that represents the full range. + pub const fn full() -> Self { + Self::new(0, None, 1) + } + + /// Creates a slice that represents a single index + pub fn index(idx: isize) -> Self { + Self { + start: idx, + end: handle_signed_inclusive_end(idx), + step: 1, + } + } + + /// Converts the slice to a vector. + pub fn into_vec(self) -> Vec { + assert!( + self.end.is_some(), + "Slice must have an end to convert to a vector: {self:?}" + ); + self.into_iter().collect() + } + + /// Clips the slice to a maximum size. + /// + /// # Example + /// + /// ```rust,ignore + /// assert_eq!( + /// Slice::new(0, None, 1).bound_to(10), + /// Slice::new(0, Some(10), 1)); + /// assert_eq!( + /// Slice::new(0, Some(5), 1).bound_to(10), + /// Slice::new(0, Some(5), 1)); + /// assert_eq!( + /// Slice::new(0, None, -1).bound_to(10), + /// Slice::new(0, Some(-11), -1)); + /// assert_eq!( + /// Slice::new(0, Some(-5), -1).bound_to(10), + /// Slice::new(0, Some(-5), -1)); + /// ``` + pub fn bound_to(self, size: usize) -> Self { + let mut bounds = size as isize; + + if let Some(end) = self.end { + if end > 0 { + bounds = end.min(bounds); + } else { + bounds = end.max(-(bounds + 1)); + } + } else if self.is_reversed() { + bounds = -(bounds + 1); + } + + Self { + end: Some(bounds), + ..self + } + } + + /// Creates a slice with a custom step + pub fn with_step(start: isize, end: Option, step: isize) -> Self { + assert!(step != 0, "Step cannot be zero"); + Self { start, end, step } + } + + /// Creates a slice from a range with a specified step + pub fn from_range_stepped>(range: R, step: isize) -> Self { + assert!(step != 0, "Step cannot be zero"); + let mut slice = range.into(); + slice.step = step; + slice + } + + /// Returns the step of the slice + pub fn step(&self) -> isize { + self.step + } + + /// Returns the range for this slice given a dimension size + pub fn range(&self, size: usize) -> Range { + self.to_range(size) + } + + /// Convert this slice to a range for a dimension of the given size. + /// + /// # Arguments + /// + /// * `size` - The size of the dimension to slice. + /// + /// # Returns + /// + /// A `Range` representing the slice bounds. + pub fn to_range(&self, size: usize) -> Range { + // Always return a valid range with start <= end + // The step information will be handled separately + let start = convert_signed_index(self.start, size); + let end = match self.end { + Some(end) => convert_signed_index(end, size), + None => size, + }; + start..end + } + + /// Converts the slice into a range and step tuple + pub fn to_range_and_step(&self, size: usize) -> (Range, isize) { + let range = self.to_range(size); + (range, self.step) + } + + /// Returns true if the step is negative + pub fn is_reversed(&self) -> bool { + self.step < 0 + } + + /// Calculates the output size for this slice operation + pub fn output_size(&self, dim_size: usize) -> usize { + let range = self.to_range(dim_size); + // Handle empty slices (start >= end) + if range.start >= range.end { + return 0; + } + let len = range.end - range.start; + if self.step.unsigned_abs() == 1 { + len + } else { + len.div_ceil(self.step.unsigned_abs()) + } + } +} + +fn convert_signed_index(index: isize, size: usize) -> usize { + if index < 0 { + (size as isize + index).max(0) as usize + } else { + (index as usize).min(size) + } +} + +fn handle_signed_inclusive_end(end: isize) -> Option { + match end { + -1 => None, + end => Some(end + 1), + } +} + +impl From> for Slice { + fn from(r: Range) -> Self { + Self { + start: r.start.as_index(), + end: Some(r.end.as_index()), + step: 1, + } + } +} + +impl From> for Slice { + fn from(r: RangeInclusive) -> Self { + Self { + start: r.start().as_index(), + end: handle_signed_inclusive_end(r.end().as_index()), + step: 1, + } + } +} + +impl From> for Slice { + fn from(r: RangeFrom) -> Self { + Self { + start: r.start.as_index(), + end: None, + step: 1, + } + } +} + +impl From> for Slice { + fn from(r: RangeTo) -> Self { + Self { + start: 0, + end: Some(r.end.as_index()), + step: 1, + } + } +} + +impl From> for Slice { + fn from(r: RangeToInclusive) -> Self { + Self { + start: 0, + end: handle_signed_inclusive_end(r.end.as_index()), + step: 1, + } + } +} + +impl From for Slice { + fn from(_: RangeFull) -> Self { + Self { + start: 0, + end: None, + step: 1, + } + } +} + +impl From for Slice { + fn from(i: usize) -> Self { + Slice::index(i as isize) + } +} + +impl From for Slice { + fn from(i: isize) -> Self { + Slice::index(i) + } +} + +impl From for Slice { + fn from(i: i32) -> Self { + Slice::index(i as isize) + } +} + +impl From for Slice { + fn from(i: i64) -> Self { + Slice::index(i as isize) + } +} + +impl Display for Slice { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + if self.step == 1 + && let Some(end) = self.end + && self.start == end - 1 + { + f.write_fmt(format_args!("{}", self.start)) + } else { + if self.start != 0 { + f.write_fmt(format_args!("{}", self.start))?; + } + f.write_str("..")?; + if let Some(end) = self.end { + f.write_fmt(format_args!("{}", end))?; + } + if self.step != 1 { + f.write_fmt(format_args!(";{}", self.step))?; + } + Ok(()) + } + } +} + +impl FromStr for Slice { + type Err = crate::ExpressionError; + + fn from_str(source: &str) -> Result { + let mut s = source.trim(); + + let parse_int = |v: &str| -> Result { + v.parse::().map_err(|e| { + crate::ExpressionError::parse_error( + format!("Invalid integer: '{v}': {}", e), + source, + ) + }) + }; + + let mut start: isize = 0; + let mut end: Option = None; + let mut step: isize = 1; + + if let Some((head, tail)) = s.split_once(";") { + step = parse_int(tail)?; + s = head; + } + + if s.is_empty() { + return Err(crate::ExpressionError::parse_error( + "Empty expression", + source, + )); + } + + if let Some((start_s, end_s)) = s.split_once("..") { + if !start_s.is_empty() { + start = parse_int(start_s)?; + } + if !end_s.is_empty() { + if let Some(end_s) = end_s.strip_prefix('=') { + end = Some(parse_int(end_s)? + 1); + } else { + end = Some(parse_int(end_s)?); + } + } + } else { + start = parse_int(s)?; + end = Some(start + 1); + } + + if step == 0 { + return Err(crate::ExpressionError::invalid_expression( + "Step cannot be zero", + source, + )); + } + + Ok(Slice::new(start, end, step)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::string::ToString; + use alloc::vec; + + #[test] + fn test_slice_to_str() { + assert_eq!(Slice::new(0, None, 1).to_string(), ".."); + + assert_eq!(Slice::new(0, Some(1), 1).to_string(), "0"); + + assert_eq!(Slice::new(0, Some(10), 1).to_string(), "..10"); + assert_eq!(Slice::new(1, Some(10), 1).to_string(), "1..10"); + + assert_eq!(Slice::new(-3, Some(10), -2).to_string(), "-3..10;-2"); + } + + #[test] + fn test_slice_from_str() { + assert_eq!("1".parse::(), Ok(Slice::new(1, Some(2), 1))); + assert_eq!("..".parse::(), Ok(Slice::new(0, None, 1))); + assert_eq!("..3".parse::(), Ok(Slice::new(0, Some(3), 1))); + assert_eq!("..=3".parse::(), Ok(Slice::new(0, Some(4), 1))); + + assert_eq!("-12..3".parse::(), Ok(Slice::new(-12, Some(3), 1))); + assert_eq!("..;-1".parse::(), Ok(Slice::new(0, None, -1))); + + assert_eq!("..=3;-2".parse::(), Ok(Slice::new(0, Some(4), -2))); + + assert_eq!( + "..;0".parse::(), + Err(crate::ExpressionError::invalid_expression( + "Step cannot be zero", + "..;0" + )) + ); + + assert_eq!( + "".parse::(), + Err(crate::ExpressionError::parse_error("Empty expression", "")) + ); + assert_eq!( + "a".parse::(), + Err(crate::ExpressionError::parse_error( + "Invalid integer: 'a': invalid digit found in string", + "a" + )) + ); + assert_eq!( + "..a".parse::(), + Err(crate::ExpressionError::parse_error( + "Invalid integer: 'a': invalid digit found in string", + "..a" + )) + ); + assert_eq!( + "a:b:c".parse::(), + Err(crate::ExpressionError::parse_error( + "Invalid integer: 'a:b:c': invalid digit found in string", + "a:b:c" + )) + ); + } + + #[test] + fn test_slice_output_size() { + // Test the output_size method directly + assert_eq!(Slice::new(0, Some(10), 1).output_size(10), 10); + assert_eq!(Slice::new(0, Some(10), 2).output_size(10), 5); + assert_eq!(Slice::new(0, Some(10), 3).output_size(10), 4); // ceil(10/3) + assert_eq!(Slice::new(0, Some(10), -1).output_size(10), 10); + assert_eq!(Slice::new(0, Some(10), -2).output_size(10), 5); + assert_eq!(Slice::new(2, Some(8), -3).output_size(10), 2); // ceil(6/3) + assert_eq!(Slice::new(5, Some(5), 1).output_size(10), 0); // empty range + } + + #[test] + fn test_bound_to() { + assert_eq!( + Slice::new(0, None, 1).bound_to(10), + Slice::new(0, Some(10), 1) + ); + assert_eq!( + Slice::new(0, Some(5), 1).bound_to(10), + Slice::new(0, Some(5), 1) + ); + + assert_eq!( + Slice::new(0, None, -1).bound_to(10), + Slice::new(0, Some(-11), -1) + ); + assert_eq!( + Slice::new(0, Some(-5), -1).bound_to(10), + Slice::new(0, Some(-5), -1) + ); + } + + #[test] + fn test_slice_iter() { + assert_eq!( + Slice::new(2, Some(3), 1).into_iter().collect::>(), + vec![2] + ); + assert_eq!( + Slice::new(3, Some(-1), -1).into_iter().collect::>(), + vec![3, 2, 1, 0] + ); + + assert_eq!(Slice::new(3, Some(-1), -1).into_vec(), vec![3, 2, 1, 0]); + + assert_eq!( + Slice::new(3, None, 2) + .into_iter() + .take(3) + .collect::>(), + vec![3, 5, 7] + ); + assert_eq!( + Slice::new(3, None, 2) + .bound_to(8) + .into_iter() + .collect::>(), + vec![3, 5, 7] + ); + } + + #[test] + #[should_panic( + expected = "Slice must have an end to convert to a vector: Slice { start: 0, end: None, step: 1 }" + )] + fn test_unbound_slice_into_vec() { + Slice::new(0, None, 1).into_vec(); + } + + #[test] + fn into_slices_should_return_for_all_shape_dims() { + let slice = s![1]; + let shape = Shape::new([2, 3, 1]); + + let slices = slice.into_slices(&shape); + + assert_eq!(slices.len(), shape.len()); + + assert_eq!(slices[0], Slice::new(1, Some(2), 1)); + assert_eq!(slices[1], Slice::new(0, Some(3), 1)); + assert_eq!(slices[2], Slice::new(0, Some(1), 1)); + + let slice = s![1, 0..2]; + let slices = slice.into_slices(&shape); + + assert_eq!(slices.len(), shape.len()); + + assert_eq!(slices[0], Slice::new(1, Some(2), 1)); + assert_eq!(slices[1], Slice::new(0, Some(2), 1)); + assert_eq!(slices[2], Slice::new(0, Some(1), 1)); + + let slice = s![..]; + let slices = slice.into_slices(&shape); + + assert_eq!(slices.len(), shape.len()); + + assert_eq!(slices[0], Slice::new(0, Some(2), 1)); + assert_eq!(slices[1], Slice::new(0, Some(3), 1)); + assert_eq!(slices[2], Slice::new(0, Some(1), 1)); + } + + #[test] + fn into_slices_all_dimensions() { + let slice = s![1, ..2, ..]; + let shape = Shape::new([2, 3, 1]); + + let slices = slice.into_slices(&shape); + + assert_eq!(slices.len(), shape.len()); + + assert_eq!(slices[0], Slice::new(1, Some(2), 1)); + assert_eq!(slices[1], Slice::new(0, Some(2), 1)); + assert_eq!(slices[2], Slice::new(0, Some(1), 1)); + } + + #[test] + fn into_slices_supports_empty_dimensions() { + let slice = s![.., 1, ..]; + let shape = Shape::new([0, 3, 1]); + + let slices = slice.into_slices(&shape); + + assert_eq!(slices.len(), shape.len()); + + assert_eq!(slices[0], Slice::new(0, Some(0), 1)); + assert_eq!(slices[1], Slice::new(1, Some(2), 1)); + assert_eq!(slices[2], Slice::new(0, Some(1), 1)); + } + + #[test] + #[should_panic = "Too many slices provided for shape"] + fn into_slices_should_match_shape_rank() { + let slice = s![.., 1, ..]; + let shape = Shape::new([3, 1]); + + let _ = slice.into_slices(&shape); + } + + #[test] + fn should_support_const_and_full() { + static SLICES: [Slice; 2] = [Slice::full(), Slice::new(2, None, 1)]; + assert_eq!(SLICES[0], Slice::new(0, None, 1)); + assert_eq!(SLICES[1], Slice::new(2, None, 1)); + } + + #[test] + fn should_support_default() { + assert_eq!(Slice::default(), Slice::new(0, None, 1)); + } + + #[test] + fn should_support_copy() { + let mut slice = Slice::new(1, Some(3), 2); + let slice_copy = slice; + + slice.end = Some(4); + + assert_eq!(slice, Slice::new(1, Some(4), 2)); + assert_eq!(slice_copy, Slice::new(1, Some(3), 2)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-store/Cargo.toml new file mode 100644 index 0000000..a2f9bc0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/Cargo.toml @@ -0,0 +1,106 @@ +[package] +authors = ["Dilshod Tadjibaev (@antimora)"] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Storage and serialization infrastructure for Burn" +documentation = "https://docs.rs/burn-store" +edition.workspace = true +keywords = [ + "deep-learning", + "machine-learning", + "tensor", + "storage", + "serialization", +] +license.workspace = true +name = "burn-store" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-store" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std", "pytorch", "safetensors", "burnpack", "memmap"] +memmap = ["std", "dep:memmap2"] +std = [ + "dep:memmap2", + "safetensors/std", + "burn-core/std", + "burn-tensor/std", + "dep:regex", + "byteorder/std", +] +tracing = [ + "burn-core/tracing", + "burn-cuda?/tracing", + "burn-nn/tracing", + "burn-tch?/tracing", + "burn-tensor/tracing", + "burn-wgpu?/tracing", +] + + +burnpack = ["serde", "ciborium"] +cuda = ["burn-cuda"] +metal = ["wgpu", "burn-wgpu/metal"] +tch = ["burn-tch"] +wgpu = ["burn-wgpu"] + +safetensors = ["dep:safetensors"] + +pytorch = ["burn-core/record-item-custom-serde", "zip", "serde", "tar"] + +[dependencies] +burn-core = { path = "../burn-core", version = "=0.21.0-pre.2", default-features = false } +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2", default-features = false } + +# External dependencies +byteorder = { workspace = true, default-features = false } +bytes = { workspace = true } +ciborium = { workspace = true, optional = true } +half = { workspace = true } +hashbrown = { workspace = true, features = ["serde"] } +memmap2 = { workspace = true, optional = true } +regex = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +textdistance = { workspace = true } +zip = { workspace = true, optional = true } +tar = { workspace = true, optional = true } + +# Workaround to force broken minor version to update +lzma-rust2 = { workspace = true, optional = true } + +safetensors = { workspace = true, optional = true } + +# Optional backend dependencies for benchmarks +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", optional = true } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", optional = true } + +[dev-dependencies] +# burn-import = { path = "../burn-import", version = "=0.21.0-pre.2" } # disabled (circular dep in publish, only for bench) +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-nn = { path = "../burn-nn", version = "=0.21.0-pre.2", default-features = false } +divan = "0.1" +tempfile = { workspace = true } + +[[bench]] +harness = false +name = "resnet18_loading" + +[[bench]] +harness = false +name = "unified_loading" + +[[bench]] +harness = false +name = "unified_saving" + +[[bench]] +harness = false +name = "zero_copy_loading" + +# Enable extra-platforms for bytes on targets without native atomics (e.g., thumbv6m-none-eabi) +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +bytes = { workspace = true, features = ["extra-platforms"] } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/MIGRATION.md b/crates/stable-diffusion-burn/burn-crates/burn-store/MIGRATION.md new file mode 100644 index 0000000..2defff5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/MIGRATION.md @@ -0,0 +1,325 @@ +# Migration Guide: burn-import to burn-store + +This guide helps you migrate from the deprecated `burn-import` recorders (`PyTorchFileRecorder`, +`SafetensorsFileRecorder`) to the new `burn-store` API (`PytorchStore`, `SafetensorsStore`). + +## Overview + +The new `burn-store` API provides: + +- **Simpler API**: Load directly into models instead of records +- **Fluent builder pattern**: Chain configuration methods +- **Better error handling**: Detailed load results with applied/missing/errors info +- **Bidirectional support**: Both load and save operations +- **More features**: Filtering, partial loading, metadata, zero-copy loading + +## Quick Migration + +### PyTorch Files (.pt/.pth) + +**Before (burn-import):** + +```rust +use burn::record::{FullPrecisionSettings, Recorder}; +use burn_import::pytorch::{LoadArgs, PyTorchFileRecorder}; + +// Load into a record, then create model from record +let record: ModelRecord = PyTorchFileRecorder::::default() + .load("model.pt".into(), &device) + .expect("Failed to load"); + +let model = Model::init(&device).load_record(record); +``` + +**After (burn-store):** + +```rust +use burn_store::{ModuleSnapshot, PytorchStore}; + +// Initialize model, then load weights directly +let mut model = Model::init(&device); +let mut store = PytorchStore::from_file("model.pt"); +model.load_from(&mut store).expect("Failed to load"); +``` + +### SafeTensors Files (.safetensors) + +**Before (burn-import):** + +```rust +use burn::record::{FullPrecisionSettings, Recorder}; +use burn_import::safetensors::{AdapterType, LoadArgs, SafetensorsFileRecorder}; + +let record: ModelRecord = SafetensorsFileRecorder::::default() + .load("model.safetensors".into(), &device) + .expect("Failed to load"); + +let model = Model::init(&device).load_record(record); +``` + +**After (burn-store):** + +```rust +use burn_store::{ModuleSnapshot, PyTorchToBurnAdapter, SafetensorsStore}; + +let mut model = Model::init(&device); + +// For SafeTensors exported from PyTorch, use the adapter +let mut store = SafetensorsStore::from_file("model.safetensors") + .with_from_adapter(PyTorchToBurnAdapter); +model.load_from(&mut store).expect("Failed to load"); + +// For native Burn SafeTensors, no adapter needed +let mut store = SafetensorsStore::from_file("model.safetensors"); +model.load_from(&mut store).expect("Failed to load"); +``` + +## API Mapping + +### PyTorchFileRecorder Options + +| burn-import | burn-store | +| ---------------------------------------------- | ------------------------------------------- | +| `LoadArgs::new(path)` | `PytorchStore::from_file(path)` | +| `.with_key_remap(pattern, replacement)` | `.with_key_remapping(pattern, replacement)` | +| `.with_top_level_key(key)` | `.with_top_level_key(key)` | +| `.with_debug_print()` | _(use tracing/logging instead)_ | +| `PyTorchFileRecorder::` | _(precision handled automatically)_ | + +### SafetensorsFileRecorder Options + +| burn-import | burn-store | +| -------------------------------------------------- | ------------------------------------------- | +| `LoadArgs::new(path)` | `SafetensorsStore::from_file(path)` | +| `.with_key_remap(pattern, replacement)` | `.with_key_remapping(pattern, replacement)` | +| `.with_adapter_type(AdapterType::PyTorch)` | `.with_from_adapter(PyTorchToBurnAdapter)` | +| `.with_adapter_type(AdapterType::NoAdapter)` | _(default, no adapter)_ | +| `.with_debug_print()` | _(use tracing/logging instead)_ | +| `SafetensorsFileRecorder::` | _(precision handled automatically)_ | + +## Detailed Examples + +### Key Remapping + +**Before:** + +```rust +let args = LoadArgs::new("model.pt".into()) + .with_key_remap("conv\\.(.*)", "$1") + .with_key_remap("^old_prefix\\.", "new_prefix."); + +let record: ModelRecord = PyTorchFileRecorder::::default() + .load(args, &device)?; +``` + +**After:** + +```rust +let mut store = PytorchStore::from_file("model.pt") + .with_key_remapping("conv\\.(.*)", "$1") + .with_key_remapping("^old_prefix\\.", "new_prefix."); + +model.load_from(&mut store)?; +``` + +### Top-Level Key Access + +**Before:** + +```rust +let args = LoadArgs::new("checkpoint.pt".into()) + .with_top_level_key("state_dict"); + +let record: ModelRecord = PyTorchFileRecorder::::default() + .load(args, &device)?; +``` + +**After:** + +```rust +let mut store = PytorchStore::from_file("checkpoint.pt") + .with_top_level_key("state_dict"); + +model.load_from(&mut store)?; +``` + +### PyTorch Adapter for SafeTensors + +**Before:** + +```rust +use burn_import::safetensors::{AdapterType, LoadArgs}; + +let args = LoadArgs::new("pytorch_model.safetensors".into()) + .with_adapter_type(AdapterType::PyTorch); + +let record: ModelRecord = SafetensorsFileRecorder::::default() + .load(args, &device)?; +``` + +**After:** + +```rust +use burn_store::{PyTorchToBurnAdapter, SafetensorsStore}; + +let mut store = SafetensorsStore::from_file("pytorch_model.safetensors") + .with_from_adapter(PyTorchToBurnAdapter); + +model.load_from(&mut store)?; +``` + +## New Features in burn-store + +### Partial Loading + +Handle missing tensors gracefully: + +```rust +let mut store = PytorchStore::from_file("model.pt") + .allow_partial(true); + +let result = model.load_from(&mut store)?; +println!("Loaded: {:?}", result.applied); +println!("Missing: {:?}", result.missing); +``` + +### Filtering + +Load only specific tensors: + +```rust +let mut store = SafetensorsStore::from_file("model.safetensors") + .with_regex(r"^encoder\..*") // Only encoder layers + .allow_partial(true); + +model.load_from(&mut store)?; +``` + +### Saving Models + +Save models (not supported by old recorders): + +```rust +// Save to SafeTensors +let mut store = SafetensorsStore::from_file("output.safetensors") + .metadata("version", "1.0"); +model.save_into(&mut store)?; + +// Save to Burnpack (native format) +let mut store = BurnpackStore::from_file("output.bpk"); +model.save_into(&mut store)?; +``` + +### Load Results + +Get detailed information about loading: + +```rust +let result = model.load_from(&mut store)?; + +// Print the full result for debugging - shows applied, skipped, missing, and errors +println!("{}", result); + +// Or access individual fields +println!("Applied: {} tensors", result.applied.len()); +println!("Skipped: {} tensors", result.skipped.len()); +println!("Missing: {:?}", result.missing); +println!("Errors: {:?}", result.errors); + +// Check if fully successful +if result.is_success() { + println!("All tensors loaded successfully"); +} +``` + +The `LoadResult` implements `Display`, so printing it shows a formatted summary with suggestions for +common issues (e.g., using `allow_partial(true)` for missing tensors). + +## Updating Cargo.toml + +**Before:** + +```toml +[dependencies] +burn-import = { version = "0.x", features = ["pytorch", "safetensors"] } +``` + +**After:** + +```toml +[dependencies] +burn-store = { version = "0.x", features = ["pytorch", "safetensors"] } +``` + +## Common Migration Issues + +### 1. Model vs Record + +The new API loads directly into models, not records. Update your model initialization: + +```rust +// Before: Create record, then model from record +let record = recorder.load(...)?; +let model = Model::init(&device).load_record(record); + +// After: Create model, then load into it +let mut model = Model::init(&device); +model.load_from(&mut store)?; +``` + +### 2. Inference Functions + +If you had functions that took `ModelRecord`, update them to take `Model`: + +```rust +// Before +fn infer(record: ModelRecord) { + let model = Model::init(&device).load_record(record); + // ... +} + +// After +fn infer(model: Model) { + // Model already has weights loaded + // ... +} +``` + +### 3. Precision Settings + +The old API required explicit precision settings. The new API handles this automatically: + +```rust +// Before: Had to specify FullPrecisionSettings or HalfPrecisionSettings +PyTorchFileRecorder::::default() + +// After: Precision handled automatically based on tensor dtype +PytorchStore::from_file("model.pt") +``` + +### 4. Error Handling + +The new API provides richer error information: + +```rust +// Before: Simple Result +let record = recorder.load(args, &device)?; + +// After: LoadResult with detailed info +let result = model.load_from(&mut store)?; + +// Print the result to see a helpful summary with suggestions +println!("{}", result); + +// Or handle specific issues programmatically +if !result.errors.is_empty() { + for (path, error) in &result.errors { + eprintln!("Error loading {}: {}", path, error); + } +} +``` + +## See Also + +- [burn-store README](README.md) - Full documentation +- [import-model-weights example](../../examples/import-model-weights/) - Working example diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/README.md b/crates/stable-diffusion-burn/burn-crates/burn-store/README.md new file mode 100644 index 0000000..06290fd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/README.md @@ -0,0 +1,77 @@ +# Burn Store + +> Advanced model storage and serialization for the Burn deep learning framework + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-store.svg)](https://crates.io/crates/burn-store) +[![Documentation](https://docs.rs/burn-store/badge.svg)](https://docs.rs/burn-store) + +A comprehensive storage library for Burn that enables efficient model serialization, cross-framework +interoperability, and advanced tensor management. + +> **Migrating from burn-import?** See the [Migration Guide](MIGRATION.md) for help moving from +> `PyTorchFileRecorder`/`SafetensorsFileRecorder` to the new Store API. + +## Features + +- **Burnpack Format** - Native Burn format with CBOR metadata, memory-mapped loading, ParamId + persistence for stateful training, and no-std support +- **SafeTensors Format** - Industry-standard format for secure and efficient tensor serialization +- **PyTorch Support** - Direct loading of PyTorch .pth/.pt files with automatic weight + transformation +- **Zero-Copy Loading** - Memory-mapped files and lazy tensor materialization for optimal + performance +- **Flexible Filtering** - Load/save specific model subsets with regex, exact paths, or custom + predicates +- **Tensor Remapping** - Rename tensors during load/save for framework compatibility +- **No-std Support** - Burnpack and SafeTensors formats available in embedded and WASM environments + +## Quick Start + +```rust +use burn_store::{ModuleSnapshot, PytorchStore, SafetensorsStore, BurnpackStore}; + +// Load from PyTorch +let mut store = PytorchStore::from_file("model.pt"); +model.load_from(&mut store)?; + +// Load from SafeTensors (with PyTorch adapter) +let mut store = SafetensorsStore::from_file("model.safetensors") + .with_from_adapter(PyTorchToBurnAdapter); +model.load_from(&mut store)?; + +// Save to Burnpack +let mut store = BurnpackStore::from_file("model.bpk"); +model.save_into(&mut store)?; +``` + +## Documentation + +For comprehensive documentation including: + +- Exporting weights from PyTorch +- Loading weights into Burn models +- Saving models to various formats +- Advanced features (filtering, remapping, partial loading, zero-copy) +- API reference and troubleshooting + +See the **[Burn Book - Model Weights](https://burn.dev/book/import/model-weights.html)** chapter. + +## Running Benchmarks + +```bash +# Generate model files (one-time setup) +uv run benches/generate_unified_models.py + +# Run loading benchmarks +cargo bench --bench unified_loading + +# Run saving benchmarks +cargo bench --bench unified_saving + +# With specific backend +cargo bench --bench unified_loading --features metal +``` + +## License + +This project is dual-licensed under MIT and Apache-2.0. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/benches/download_resnet18.py b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/download_resnet18.py new file mode 100644 index 0000000..6393465 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/download_resnet18.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.8" +# dependencies = [ +# "torch", +# "torchvision", +# ] +# /// +""" +Download ResNet18 PyTorch model for benchmarking. +This script downloads a pre-trained ResNet18 model from PyTorch Hub +and saves it in a format suitable for benchmarking. +""" + +import os +import sys +import tempfile +from pathlib import Path + +import torch +import torchvision.models as models + +def download_resnet18(): + """Download ResNet18 model and save to temp directory.""" + + # Create a temporary directory for the model + temp_dir = Path(tempfile.gettempdir()) / "burn_resnet18_benchmark" + temp_dir.mkdir(parents=True, exist_ok=True) + + output_path = temp_dir / "resnet18.pth" + + # Check if already downloaded + if output_path.exists(): + file_size_mb = output_path.stat().st_size / (1024 * 1024) + print(f"✅ ResNet18 already exists at: {output_path}") + print(f" Size: {file_size_mb:.1f} MB") + return str(output_path) + + print("📥 Downloading ResNet18 model...") + + try: + # Download pre-trained ResNet18 model + model = models.resnet18(pretrained=True) + + # Save the model state dict (this is what burn-store reads) + # Using the legacy format for compatibility + torch.save(model.state_dict(), output_path, _use_new_zipfile_serialization=False) + + file_size_mb = output_path.stat().st_size / (1024 * 1024) + print(f"✅ Successfully downloaded ResNet18 to: {output_path}") + print(f" Size: {file_size_mb:.1f} MB") + print(f" Format: PyTorch legacy format") + + # Verify it's readable + state_dict = torch.load(output_path, map_location='cpu') + print(f" Tensors: {len(state_dict)} tensors") + + # Print a few tensor names and shapes for verification + print("\n Sample tensors:") + for i, (name, tensor) in enumerate(state_dict.items()): + if i < 3: + print(f" - {name}: {list(tensor.shape)}") + + return str(output_path) + + except Exception as e: + print(f"❌ Failed to download ResNet18: {e}") + sys.exit(1) + +def main(): + """Main entry point.""" + path = download_resnet18() + + # Write the path to a file that the benchmark can read + bench_config = Path(tempfile.gettempdir()) / "burn_resnet18_benchmark" / "path.txt" + bench_config.write_text(path) + + print(f"\n💡 Model ready for benchmarking") + print(f" Run: cargo bench --bench resnet18_loading") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/benches/generate_unified_models.py b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/generate_unified_models.py new file mode 100644 index 0000000..6216d22 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/generate_unified_models.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.8" +# dependencies = [ +# "torch", +# "safetensors", +# "packaging", +# "numpy", +# ] +# /// +""" +Generate a large model (~312MB) in both PyTorch and SafeTensors formats for unified benchmarking. + +Usage: + uv run benches/generate_unified_models.py + +The script will create model files in /tmp/simple_bench_models/ directory. +""" + +import torch +import torch.nn as nn +import os +from pathlib import Path +import tempfile +from safetensors.torch import save_file + +def get_temp_dir(): + """Get the appropriate temp directory.""" + temp_dir = Path(tempfile.gettempdir()) / "simple_bench_models" + temp_dir.mkdir(parents=True, exist_ok=True) + return temp_dir + +class LargeModel(nn.Module): + """Large model with 20 layers to match Rust benchmark.""" + def __init__(self): + super().__init__() + self.layers = nn.ModuleList() + + # Create a model with 20 layers matching the Rust LargeModel + for i in range(20): + in_size = 1024 if i == 0 else 2048 + out_size = 2048 + self.layers.append(nn.Linear(in_size, out_size)) + + print(f"Created model with {len(self.layers)} layers") + + def forward(self, x): + for layer in self.layers: + x = layer(x) + return x + +def calculate_model_size(model): + """Calculate the size of the model in MB.""" + total_params = sum(p.numel() for p in model.parameters()) + size_mb = (total_params * 4) / (1024 * 1024) # 4 bytes per float32 + return total_params, size_mb + +def initialize_weights(model): + """Initialize model weights with random values.""" + for param in model.parameters(): + if param.dim() > 1: + nn.init.xavier_uniform_(param) + else: + nn.init.zeros_(param) + +def save_pytorch_format(model, output_dir): + """Save model in PyTorch format.""" + pt_path = output_dir / "large_model.pt" + + # Save as checkpoint with model_state_dict (common format) + checkpoint = { + 'model_state_dict': model.state_dict(), + 'metadata': { + 'model_type': 'large_benchmark_model', + 'num_layers': len(model.layers), + } + } + torch.save(checkpoint, pt_path) + + return pt_path + +def save_safetensors_format(model, output_dir): + """Save model in SafeTensors format.""" + st_path = output_dir / "large_model.safetensors" + + # Convert state dict to safetensors format + state_dict = model.state_dict() + # Ensure all tensors are contiguous and on CPU + state_dict = {k: v.contiguous().cpu() for k, v in state_dict.items()} + + # Save with metadata + metadata = { + 'model_type': 'large_benchmark_model', + 'num_layers': str(len(model.layers)), + } + save_file(state_dict, st_path, metadata=metadata) + + return st_path + +def verify_files(pt_path, st_path): + """Verify the saved files can be loaded.""" + # Verify PyTorch file + checkpoint = torch.load(pt_path, map_location='cpu') + pt_keys = set(checkpoint['model_state_dict'].keys()) + print(f" PyTorch file: {len(pt_keys)} tensors") + + # Verify SafeTensors file + from safetensors import safe_open + with safe_open(st_path, framework="pt", device="cpu") as f: + st_keys = set(f.keys()) + print(f" SafeTensors file: {len(st_keys)} tensors") + + # Check keys match + if pt_keys != st_keys: + print(" ⚠️ Warning: Keys don't match between formats!") + else: + print(" ✓ Keys match between formats") + +def main(): + print("🔧 Generating unified benchmark model files...") + print("") + + output_dir = get_temp_dir() + print(f"📁 Output directory: {output_dir}") + print("") + + # Set random seed for reproducibility + torch.manual_seed(42) + + # Create the large model + print("📝 Creating large model...") + model = LargeModel() + + # Calculate and display model size + total_params, size_mb = calculate_model_size(model) + print(f" Total parameters: {total_params:,}") + print(f" Model size: {size_mb:.2f} MB") + print("") + + # Initialize weights + print("🎲 Initializing weights...") + initialize_weights(model) + + # Save in PyTorch format + print("💾 Saving PyTorch format...") + pt_path = save_pytorch_format(model, output_dir) + pt_size_mb = pt_path.stat().st_size / (1024 * 1024) + print(f" Saved: {pt_path}") + print(f" File size: {pt_size_mb:.2f} MB") + print("") + + # Save in SafeTensors format + print("💾 Saving SafeTensors format...") + st_path = save_safetensors_format(model, output_dir) + st_size_mb = st_path.stat().st_size / (1024 * 1024) + print(f" Saved: {st_path}") + print(f" File size: {st_size_mb:.2f} MB") + print("") + + # Verify files + print("🔍 Verifying saved files...") + verify_files(pt_path, st_path) + print("") + + print(f"✅ Model files generated successfully!") + print("") + print("📊 Summary:") + print(f" PyTorch file: {pt_path.name} ({pt_size_mb:.2f} MB)") + print(f" SafeTensors file: {st_path.name} ({st_size_mb:.2f} MB)") + print("") + print("💡 To run the unified benchmark:") + print(" cargo bench --bench unified_loading") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/benches/resnet18_loading.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/resnet18_loading.rs new file mode 100644 index 0000000..ff992f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/resnet18_loading.rs @@ -0,0 +1,213 @@ +//! Benchmark for ResNet18 loading to verify lazy loading memory usage. +//! +//! resnet18.pth is pytorch's legacy file format. +//! +//! This benchmark loads a ResNet18 model and materializes all tensors +//! to ensure memory usage stays reasonable with lazy loading. +//! +//! Run the benchmark: +//! ```bash +//! cargo bench --bench resnet18_loading +//! ``` + +use burn_store::pytorch::PytorchReader; +use divan::{AllocProfiler, Bencher}; +use std::path::PathBuf; + +#[global_allocator] +static ALLOC: AllocProfiler = AllocProfiler::system(); + +#[allow(clippy::manual_range_contains)] +fn main() { + // Check if ResNet18 file exists + let path = resnet18_path(); + if !path.exists() { + eprintln!("❌ ResNet18 model not found!"); + eprintln!(); + eprintln!("Please download it first by running:"); + eprintln!(" python benches/download_resnet18.py"); + eprintln!(); + eprintln!("Or if you don't have Python/PyTorch installed:"); + eprintln!(" uv run benches/download_resnet18.py"); + eprintln!(); + eprintln!("Expected location: {}", path.display()); + std::process::exit(1); + } + + // Verify file size is reasonable + let metadata = std::fs::metadata(&path).expect("Failed to read file metadata"); + let size_mb = metadata.len() as f64 / 1_048_576.0; + + if size_mb < 40.0 || size_mb > 50.0 { + eprintln!( + "⚠️ Warning: ResNet18 file size ({:.1} MB) seems unusual", + size_mb + ); + eprintln!("Expected size is around 45 MB"); + } + + println!("✅ Found ResNet18 model at: {}", path.display()); + println!("📦 File size: {:.1} MB", size_mb); + println!("📊 Running ResNet18 loading benchmarks...\n"); + + // Run divan benchmarks + divan::main(); +} + +/// Get the path to ResNet18 model file +fn resnet18_path() -> PathBuf { + // First try to read from the path file created by download script + let temp_dir = std::env::temp_dir(); + let config_file = temp_dir.join("burn_resnet18_benchmark").join("path.txt"); + + if config_file.exists() + && let Ok(path_str) = std::fs::read_to_string(&config_file) + { + let path = PathBuf::from(path_str.trim()); + if path.exists() { + return path; + } + } + + // Fallback to default location + temp_dir + .join("burn_resnet18_benchmark") + .join("resnet18.pth") +} + +#[divan::bench(sample_count = 10)] +fn load_resnet18_metadata(bencher: Bencher) { + let path = resnet18_path(); + + bencher.bench_local(|| { + let reader = PytorchReader::new(&path).expect("Failed to load ResNet18"); + let metadata = reader.metadata(); + + // Just access metadata without materializing tensors + assert_eq!(metadata.tensor_count, 122); + }); +} + +#[divan::bench(sample_count = 5)] +fn load_resnet18_materialize_all(bencher: Bencher) { + let path = resnet18_path(); + + bencher.bench_local(|| { + let reader = PytorchReader::new(&path).expect("Failed to load ResNet18"); + let keys = reader.keys(); + + let mut total_bytes = 0usize; + + // Materialize all tensors one by one + for key in &keys { + let tensor = reader.get(key).expect("Failed to get tensor"); + // Materialize the tensor data + let _data = tensor.to_data().expect("Failed to materialize tensor data"); + total_bytes += tensor.data_len(); + } + + // Verify we processed all the data + assert!(total_bytes > 40_000_000); // Should be ~45MB + }); +} + +#[divan::bench(sample_count = 5)] +fn load_resnet18_materialize_sequential(bencher: Bencher) { + let path = resnet18_path(); + + bencher.bench_local(|| { + let reader = PytorchReader::new(&path).expect("Failed to load ResNet18"); + let keys = reader.keys(); + + // Materialize tensors one at a time, letting previous ones be dropped + // This simulates processing tensors sequentially without keeping all in memory + for key in &keys { + let tensor = reader.get(key).expect("Failed to get tensor"); + let data = tensor.to_data().expect("Failed to materialize tensor data"); + + // Do minimal work with the data to prevent optimization + let sum = match data.dtype { + burn_tensor::DType::F32 => data + .as_slice::() + .map(|s| s.iter().sum::()) + .unwrap_or(0.0) as f64, + burn_tensor::DType::F64 => data + .as_slice::() + .map(|s| s.iter().sum::()) + .unwrap_or(0.0), + _ => 0.0, + }; + + // Use the sum to prevent dead code elimination + std::hint::black_box(sum); + } + }); +} + +#[divan::bench(sample_count = 10)] +fn load_resnet18_largest_tensor(bencher: Bencher) { + let path = resnet18_path(); + + bencher.bench_local(|| { + let reader = PytorchReader::new(&path).expect("Failed to load ResNet18"); + + // Find and materialize only the largest tensor + // This tests peak memory for a single tensor operation + let keys = reader.keys(); + let mut largest_key = String::new(); + let mut largest_size = 0usize; + + for key in &keys { + let tensor = reader.get(key).expect("Failed to get tensor"); + let size = tensor.data_len(); + if size > largest_size { + largest_size = size; + largest_key = key.clone(); + } + } + + // Materialize the largest tensor + let tensor = reader + .get(&largest_key) + .expect("Failed to get largest tensor"); + let _data = tensor.to_data().expect("Failed to materialize tensor data"); + + assert!(largest_size > 9_000_000); // Should be ~9MB for layer4.0.conv2.weight + }); +} + +#[divan::bench(sample_count = 10)] +fn load_resnet18_memory_profile(bencher: Bencher) { + let path = resnet18_path(); + + bencher + .with_inputs(|| path.clone()) + .bench_local_values(|path| { + let reader = PytorchReader::new(&path).expect("Failed to load ResNet18"); + let keys = reader.keys(); + + let mut peak_single_tensor = 0usize; + let mut total_data = 0usize; + + // Process each tensor and track memory + for key in &keys { + let tensor = reader.get(key).expect("Failed to get tensor"); + let tensor_size = tensor.data_len(); + + // Track largest single tensor + if tensor_size > peak_single_tensor { + peak_single_tensor = tensor_size; + } + + // Materialize the tensor + let data = tensor.to_data().expect("Failed to materialize tensor data"); + total_data += tensor_size; + + // Drop data immediately to test lazy loading memory efficiency + drop(data); + } + + // Return stats for verification + (peak_single_tensor, total_data) + }); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_loading.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_loading.rs new file mode 100644 index 0000000..3467fff --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_loading.rs @@ -0,0 +1,332 @@ +#![recursion_limit = "256"] + +//! Unified benchmark comparing all loading methods: +//! - BurnpackStore (new native format) +//! - NamedMpkFileRecorder (old native format) +//! - SafetensorsStore (new) +//! - SafetensorsFileRecorder (old) +//! - PytorchStore (new) +//! - PyTorchFileRecorder (old) +//! +//! Before running this benchmark, generate the model files: +//! ```bash +//! cd crates/burn-store +//! uv run benches/generate_unified_models.py +//! ``` +//! +//! Then run the benchmark: +//! ```bash +//! cargo bench --bench unified_loading +//! ``` + +use burn_core as burn; + +use burn_core::module::Module; +use burn_core::prelude::*; +use burn_core::record::{FullPrecisionSettings, NamedMpkFileRecorder, Recorder}; +// use burn_import::pytorch::{LoadArgs, PyTorchFileRecorder}; +// use burn_import::safetensors::SafetensorsFileRecorder; +use burn_nn as nn; +use burn_store::{ + BurnpackStore, ModuleSnapshot, PyTorchToBurnAdapter, PytorchStore, SafetensorsStore, +}; +use divan::{AllocProfiler, Bencher}; +use std::fs; +use std::path::{Path, PathBuf}; + +#[global_allocator] +static ALLOC: AllocProfiler = AllocProfiler::system(); + +// Backend type aliases +type NdArrayBackend = burn_ndarray::NdArray; + +#[cfg(feature = "wgpu")] +type WgpuBackend = burn_wgpu::Wgpu; + +#[cfg(feature = "cuda")] +type CudaBackend = burn_cuda::Cuda; + +#[cfg(feature = "tch")] +type TchBackend = burn_tch::LibTorch; + +#[cfg(feature = "metal")] +type MetalBackend = burn_wgpu::Metal; + +// Use the same LargeModel as other benchmarks for fair comparison +#[derive(Module, Debug)] +struct LargeModel { + layers: Vec>, +} + +impl LargeModel { + fn new(device: &B::Device) -> Self { + let mut layers = Vec::new(); + // Create a model with 20 layers - same as safetensor_loading benchmark + for i in 0..20 { + let in_size = if i == 0 { 1024 } else { 2048 }; + layers.push(nn::LinearConfig::new(in_size, 2048).init(device)); + } + Self { layers } + } +} + +/// Get the path to the model files +fn get_model_dir() -> PathBuf { + std::env::temp_dir().join("simple_bench_models") +} + +/// Generate Burnpack and NamedMpk files from existing SafeTensors file +fn generate_burn_formats(st_path: &Path, bp_path: &Path, mpk_path: &Path) { + type TestBackend = NdArrayBackend; + let device = Default::default(); + + // Load the model from SafeTensors + let mut model = LargeModel::::new(&device); + let mut store = SafetensorsStore::from_file(st_path).with_from_adapter(PyTorchToBurnAdapter); + model + .load_from(&mut store) + .expect("Failed to load from SafeTensors"); + + // Save as Burnpack + if !bp_path.exists() { + println!(" Creating Burnpack file..."); + let mut burnpack_store = BurnpackStore::from_file(bp_path); + model + .save_into(&mut burnpack_store) + .expect("Failed to save as Burnpack"); + } + + // Save as NamedMpk + if !mpk_path.exists() { + println!(" Creating NamedMpk file..."); + let recorder = NamedMpkFileRecorder::::default(); + model + .save_file(mpk_path, &recorder) + .expect("Failed to save as NamedMpk"); + } +} + +/// Get paths to the model files +fn get_model_paths() -> (PathBuf, PathBuf, PathBuf, PathBuf) { + let dir = get_model_dir(); + ( + dir.join("large_model.bpk"), + dir.join("large_model.mpk"), + dir.join("large_model.safetensors"), + dir.join("large_model.pt"), + ) +} + +/// Check if model files exist +fn check_model_files() -> Result<(), String> { + let (_, _, st_path, pt_path) = get_model_paths(); + + // For now, only check safetensors and pytorch files (will generate burnpack/mpk later) + if !st_path.exists() || !pt_path.exists() { + return Err(format!( + "\n❌ Model files not found!\n\ + \n\ + Please generate the model files first by running:\n\ + \n\ + cd crates/burn-store\n\ + uv run benches/generate_unified_models.py\n\ + \n\ + Expected files:\n\ + - {}\n\ + - {}\n", + st_path.display(), + pt_path.display() + )); + } + + Ok(()) +} + +fn main() { + // Check if model files exist before running benchmarks + match check_model_files() { + Ok(()) => { + let (bp_path, mpk_path, st_path, pt_path) = get_model_paths(); + + // First, generate Burnpack and MPK files if they don't exist + if !bp_path.exists() || !mpk_path.exists() { + println!("⏳ Generating Burnpack and NamedMpk files from SafeTensors..."); + generate_burn_formats(&st_path, &bp_path, &mpk_path); + } + + let bp_size = fs::metadata(&bp_path) + .ok() + .map(|m| m.len() as f64 / 1_048_576.0); + let mpk_size = fs::metadata(&mpk_path) + .ok() + .map(|m| m.len() as f64 / 1_048_576.0); + let st_size = fs::metadata(&st_path).unwrap().len() as f64 / 1_048_576.0; + let pt_size = fs::metadata(&pt_path).unwrap().len() as f64 / 1_048_576.0; + + println!("✅ Found model files:"); + if let Some(size) = bp_size { + println!(" Burnpack: {} ({:.1} MB)", bp_path.display(), size); + } + if let Some(size) = mpk_size { + println!(" NamedMpk: {} ({:.1} MB)", mpk_path.display(), size); + } + println!(" SafeTensors: {} ({:.1} MB)", st_path.display(), st_size); + println!(" PyTorch: {} ({:.1} MB)", pt_path.display(), pt_size); + println!(); + println!("🚀 Running unified loading benchmarks..."); + println!(); + println!("Comparing 6 loading methods:"); + println!(" 1. BurnpackStore (new native format - lazy loading)"); + println!(" 2. NamedMpkFileRecorder (old native format - loads all to memory)"); + println!(" 3. SafetensorsStore (new)"); + println!(" 4. SafetensorsFileRecorder (old)"); + println!(" 5. PytorchStore (new)"); + println!(" 6. PyTorchFileRecorder (old)"); + println!(); + println!("Available backends:"); + println!(" - NdArray (CPU)"); + #[cfg(feature = "wgpu")] + println!(" - WGPU (GPU)"); + #[cfg(feature = "cuda")] + println!(" - CUDA (NVIDIA GPU)"); + #[cfg(feature = "tch")] + println!(" - LibTorch"); + #[cfg(feature = "metal")] + println!(" - Metal (Apple GPU)"); + println!(); + + divan::main(); + } + Err(msg) => { + eprintln!("{}", msg); + std::process::exit(1); + } + } +} + +// Macro to generate benchmarks for each backend +macro_rules! bench_backend { + ($backend:ty, $mod_name:ident, $backend_name:literal) => { + #[divan::bench_group(name = $backend_name, sample_count = 10)] + mod $mod_name { + use super::*; + + type TestBackend = $backend; + type TestDevice = ::Device; + + #[divan::bench] + fn burnpack_store(bencher: Bencher) { + let (bp_path, _, _, _) = get_model_paths(); + let file_size = fs::metadata(&bp_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_file(bp_path.clone()); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + #[divan::bench] + fn namedmpk_recorder(bencher: Bencher) { + let (_, mpk_path, _, _) = get_model_paths(); + let file_size = fs::metadata(&mpk_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let recorder = NamedMpkFileRecorder::::default(); + let record = recorder + .load(mpk_path.clone().into(), &device) + .expect("Failed to load"); + let _model = LargeModel::::new(&device).load_record(record); + }); + } + + #[divan::bench] + fn safetensors_store(bencher: Bencher) { + let (_, _, st_path, _) = get_model_paths(); + let file_size = fs::metadata(&st_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = SafetensorsStore::from_file(st_path.clone()) + .with_from_adapter(PyTorchToBurnAdapter); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + // #[divan::bench] + // fn safetensors_recorder(bencher: Bencher) { + // let (_, _, st_path, _) = get_model_paths(); + // let file_size = fs::metadata(&st_path).unwrap().len(); + + // bencher + // .counter(divan::counter::BytesCount::new(file_size)) + // .bench(|| { + // let device: TestDevice = Default::default(); + // let recorder = SafetensorsFileRecorder::::default(); + // let record = recorder + // .load(st_path.clone().into(), &device) + // .expect("Failed to load"); + // let _model = LargeModel::::new(&device).load_record(record); + // }); + // } + + #[divan::bench] + fn pytorch_store(bencher: Bencher) { + let (_, _, _, pt_path) = get_model_paths(); + let file_size = fs::metadata(&pt_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = PytorchStore::from_file(pt_path.clone()) + .with_top_level_key("model_state_dict") + .allow_partial(true); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + // #[divan::bench] + // fn pytorch_recorder(bencher: Bencher) { + // let (_, _, _, pt_path) = get_model_paths(); + // let file_size = fs::metadata(&pt_path).unwrap().len(); + + // bencher + // .counter(divan::counter::BytesCount::new(file_size)) + // .bench(|| { + // let device: TestDevice = Default::default(); + // let recorder = PyTorchFileRecorder::::default(); + // let load_args = + // LoadArgs::new(pt_path.clone()).with_top_level_key("model_state_dict"); + // let record = recorder.load(load_args, &device).expect("Failed to load"); + // let _model = LargeModel::::new(&device).load_record(record); + // }); + // } + } + }; +} + +// Generate benchmarks for each backend +bench_backend!(NdArrayBackend, ndarray_backend, "NdArray Backend (CPU)"); + +#[cfg(feature = "wgpu")] +bench_backend!(WgpuBackend, wgpu_backend, "WGPU Backend (GPU)"); + +#[cfg(feature = "cuda")] +bench_backend!(CudaBackend, cuda_backend, "CUDA Backend (NVIDIA GPU)"); + +#[cfg(feature = "tch")] +bench_backend!(TchBackend, tch_backend, "LibTorch Backend"); + +#[cfg(feature = "metal")] +bench_backend!(MetalBackend, metal_backend, "Metal Backend (Apple GPU)"); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_saving.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_saving.rs new file mode 100644 index 0000000..34fc075 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/unified_saving.rs @@ -0,0 +1,183 @@ +#![recursion_limit = "256"] + +//! Unified benchmark comparing all saving methods: +//! - BurnpackStore (new native format) +//! - NamedMpkFileRecorder (old native format) +//! - SafetensorsStore (new) +//! +//! Before running this benchmark, ensure the directory exists: +//! ```bash +//! mkdir -p /tmp/simple_bench_models +//! ``` +//! +//! Then run the benchmark: +//! ```bash +//! cargo bench --bench unified_saving +//! ``` +use burn_core as burn; + +use burn_core::module::Module; +use burn_core::prelude::*; +use burn_core::record::{FullPrecisionSettings, NamedMpkFileRecorder}; +use burn_nn as nn; +use burn_store::{BurnpackStore, ModuleSnapshot, SafetensorsStore}; +use divan::{AllocProfiler, Bencher}; +use std::fs; +use std::path::PathBuf; + +#[global_allocator] +static ALLOC: AllocProfiler = AllocProfiler::system(); + +// Backend type aliases +type NdArrayBackend = burn_ndarray::NdArray; + +#[cfg(feature = "wgpu")] +type WgpuBackend = burn_wgpu::Wgpu; + +#[cfg(feature = "cuda")] +type CudaBackend = burn_cuda::Cuda; + +#[cfg(feature = "tch")] +type TchBackend = burn_tch::LibTorch; + +#[cfg(feature = "metal")] +type MetalBackend = burn_wgpu::Metal; + +// Use the same LargeModel as other benchmarks for fair comparison +#[derive(Module, Debug)] +struct LargeModel { + layers: Vec>, +} + +impl LargeModel { + fn new(device: &B::Device) -> Self { + let mut layers = Vec::new(); + // Create a model with 20 layers - same as loading benchmarks + for i in 0..20 { + let in_size = if i == 0 { 1024 } else { 2048 }; + layers.push(nn::LinearConfig::new(in_size, 2048).init(device)); + } + Self { layers } + } +} + +/// Get the path to the output directory +fn get_output_dir() -> PathBuf { + std::env::temp_dir().join("simple_bench_models_saving") +} + +/// Ensure output directory exists +fn ensure_output_dir() -> Result<(), String> { + let dir = get_output_dir(); + if !dir.exists() { + fs::create_dir_all(&dir) + .map_err(|e| format!("Failed to create output directory: {}", e))?; + } + Ok(()) +} + +fn main() { + match ensure_output_dir() { + Ok(()) => { + println!("✅ Output directory ready: {}", get_output_dir().display()); + println!(); + println!("🚀 Running unified saving benchmarks..."); + println!(); + println!("Comparing 3 saving methods:"); + println!(" 1. BurnpackStore (new native format)"); + println!(" 2. NamedMpkFileRecorder (old native format)"); + println!(" 3. SafetensorsStore (new)"); + println!(); + println!("Available backends:"); + println!(" - NdArray (CPU)"); + #[cfg(feature = "wgpu")] + println!(" - WGPU (GPU)"); + #[cfg(feature = "cuda")] + println!(" - CUDA (NVIDIA GPU)"); + #[cfg(feature = "tch")] + println!(" - LibTorch"); + #[cfg(feature = "metal")] + println!(" - Metal (Apple GPU)"); + println!(); + + divan::main(); + } + Err(msg) => { + eprintln!("❌ {}", msg); + std::process::exit(1); + } + } +} + +// Macro to generate benchmarks for each backend +macro_rules! bench_backend { + ($backend:ty, $mod_name:ident, $backend_name:literal) => { + #[divan::bench_group(name = $backend_name, sample_count = 10)] + mod $mod_name { + use super::*; + + type TestBackend = $backend; + type TestDevice = ::Device; + + #[divan::bench] + fn burnpack_store(bencher: Bencher) { + bencher.bench(|| { + let device: TestDevice = Default::default(); + let model = LargeModel::::new(&device); + let output_path = get_output_dir().join("test_burnpack.bpk"); + let mut store = BurnpackStore::from_file(output_path.clone()).overwrite(true); + model + .save_into(&mut store) + .expect("Failed to save with BurnpackStore"); + // Clean up + let _ = fs::remove_file(output_path); + }); + } + + #[divan::bench] + fn namedmpk_recorder(bencher: Bencher) { + bencher.bench(|| { + let device: TestDevice = Default::default(); + let model = LargeModel::::new(&device); + let output_path = get_output_dir().join("test_namedmpk.mpk"); + let recorder = NamedMpkFileRecorder::::default(); + model + .save_file(output_path.clone(), &recorder) + .expect("Failed to save with NamedMpkFileRecorder"); + // Clean up + let _ = fs::remove_file(output_path); + }); + } + + #[divan::bench] + fn safetensors_store(bencher: Bencher) { + bencher.bench(|| { + let device: TestDevice = Default::default(); + let model = LargeModel::::new(&device); + let output_path = get_output_dir().join("test_safetensors_store.safetensors"); + let mut store = SafetensorsStore::from_file(output_path.clone()); + model + .save_into(&mut store) + .expect("Failed to save with SafetensorsStore"); + // Clean up + let _ = fs::remove_file(output_path); + }); + } + } + }; +} + +// Generate benchmarks for each backend +bench_backend!(NdArrayBackend, ndarray_backend, "NdArray Backend (CPU)"); + +#[cfg(feature = "wgpu")] +bench_backend!(WgpuBackend, wgpu_backend, "WGPU Backend (GPU)"); + +#[cfg(feature = "cuda")] +bench_backend!(CudaBackend, cuda_backend, "CUDA Backend (NVIDIA GPU)"); + +#[cfg(feature = "tch")] +bench_backend!(TchBackend, tch_backend, "LibTorch Backend"); + +#[cfg(feature = "metal")] +bench_backend!(MetalBackend, metal_backend, "Metal Backend (Apple GPU)"); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/benches/zero_copy_loading.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/zero_copy_loading.rs new file mode 100644 index 0000000..c5a070e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/benches/zero_copy_loading.rs @@ -0,0 +1,596 @@ +#![recursion_limit = "256"] + +//! Benchmark comparing zero-copy vs copy loading modes for BurnpackStore. +//! +//! This benchmark measures the performance difference between: +//! - `zero_copy(false)` - Default mode, copies tensor data into new allocations +//! - `zero_copy(true)` - Zero-copy mode, slices tensor data without copying +//! +//! ## Understanding the Results +//! +//! **IMPORTANT**: For NdArray backend, you'll see similar allocation numbers because: +//! - NdArray uses `ndarray::ArrayD` which MUST own data as `Vec` +//! - Even with zero-copy, the backend eventually copies data into its own format +//! +//! The zero-copy benefit is: +//! - **Without zero-copy**: File → Copy to heap (Bytes) → Copy to Vec (backend) +//! - **With zero-copy**: File → Zero-copy slice → Copy to Vec (backend) +//! +//! So zero-copy saves ONE memory copy at the store level. The `store_only_*` benchmarks +//! show the raw store performance without backend allocation overhead. +//! +//! GPU backends that can consume `Bytes` directly will show larger benefits. +//! +//! ## Running the benchmark +//! +//! Before running this benchmark, generate the model files: +//! ```bash +//! cd crates/burn-store +//! uv run benches/generate_unified_models.py +//! ``` +//! +//! Then run the benchmark: +//! ```bash +//! cargo bench --bench zero_copy_loading +//! ``` + +use burn_core as burn; + +use burn_core::module::Module; +use burn_core::prelude::*; +use burn_nn as nn; +use burn_store::{ + BurnpackStore, ModuleSnapshot, ModuleStore, PyTorchToBurnAdapter, SafetensorsStore, +}; +use burn_tensor::{AllocationProperty, Bytes}; +use divan::{AllocProfiler, Bencher}; +use std::fs; +use std::path::PathBuf; +use std::sync::OnceLock; + +#[global_allocator] +static ALLOC: AllocProfiler = AllocProfiler::system(); + +// Static storage for embedded model bytes (simulating include_bytes!) +static STATIC_MODEL_BYTES: OnceLock<&'static [u8]> = OnceLock::new(); + +// Backend type aliases +type NdArrayBackend = burn_ndarray::NdArray; + +#[cfg(feature = "wgpu")] +type WgpuBackend = burn_wgpu::Wgpu; + +#[cfg(feature = "cuda")] +type CudaBackend = burn_cuda::Cuda; + +#[cfg(feature = "tch")] +type TchBackend = burn_tch::LibTorch; + +#[cfg(feature = "metal")] +type MetalBackend = burn_wgpu::Metal; + +// Use the same LargeModel as other benchmarks for fair comparison +#[derive(Module, Debug)] +struct LargeModel { + layers: Vec>, +} + +impl LargeModel { + fn new(device: &B::Device) -> Self { + let mut layers = Vec::new(); + // Create a model with 20 layers - same as unified_loading benchmark + for i in 0..20 { + let in_size = if i == 0 { 1024 } else { 2048 }; + layers.push(nn::LinearConfig::new(in_size, 2048).init(device)); + } + Self { layers } + } +} + +/// Get the path to the model files +fn get_model_dir() -> PathBuf { + std::env::temp_dir().join("simple_bench_models") +} + +/// Get path to Burnpack model file +fn get_burnpack_path() -> PathBuf { + get_model_dir().join("large_model.bpk") +} + +/// Generate Burnpack file from existing SafeTensors file if needed +fn ensure_burnpack_file() { + let bp_path = get_burnpack_path(); + let st_path = get_model_dir().join("large_model.safetensors"); + + if bp_path.exists() { + return; + } + + if !st_path.exists() { + panic!( + "\n❌ SafeTensors model file not found!\n\ + \n\ + Please generate the model files first by running:\n\ + \n\ + cd crates/burn-store\n\ + uv run benches/generate_unified_models.py\n\ + \n\ + Expected file: {}\n", + st_path.display() + ); + } + + println!("⏳ Generating Burnpack file from SafeTensors..."); + + type TestBackend = NdArrayBackend; + let device = Default::default(); + + // Load from SafeTensors + let mut model = LargeModel::::new(&device); + let mut store = SafetensorsStore::from_file(&st_path).with_from_adapter(PyTorchToBurnAdapter); + model + .load_from(&mut store) + .expect("Failed to load from SafeTensors"); + + // Save as Burnpack + let mut burnpack_store = BurnpackStore::from_file(&bp_path); + model + .save_into(&mut burnpack_store) + .expect("Failed to save as Burnpack"); + + println!("✅ Created Burnpack file: {}", bp_path.display()); +} + +/// Initialize static model bytes (simulating include_bytes! at runtime for benchmarks) +fn get_static_model_bytes() -> &'static [u8] { + STATIC_MODEL_BYTES.get_or_init(|| { + let bp_path = get_burnpack_path(); + let bytes = fs::read(&bp_path).expect("Failed to read Burnpack file"); + // Leak the bytes to get a 'static lifetime (acceptable for benchmarks) + Box::leak(bytes.into_boxed_slice()) + }) +} + +fn main() { + // Ensure Burnpack file exists + ensure_burnpack_file(); + + let bp_path = get_burnpack_path(); + let file_size = fs::metadata(&bp_path).unwrap().len() as f64 / 1_048_576.0; + + println!("✅ Found Burnpack model file:"); + println!(" Path: {}", bp_path.display()); + println!(" Size: {:.1} MB", file_size); + println!(); + println!("🚀 Running zero-copy loading benchmarks..."); + println!(); + println!("Comparing loading modes:"); + println!(" 1. file_copy - from_file().zero_copy(false) - copies tensor data"); + println!(" 2. file_zero_copy - from_file().zero_copy(true) - zero-copy via mmap"); + println!(" 3. static_copy - from_bytes() with Vec copy - copies from static"); + println!(" 4. static_zero_copy - from_static() - zero-copy from static"); + println!(); + println!("Available backends:"); + println!(" - NdArray (CPU)"); + #[cfg(feature = "wgpu")] + println!(" - WGPU (GPU)"); + #[cfg(feature = "cuda")] + println!(" - CUDA (NVIDIA GPU)"); + #[cfg(feature = "tch")] + println!(" - LibTorch"); + #[cfg(feature = "metal")] + println!(" - Metal (Apple GPU)"); + println!(); + + // Pre-initialize static bytes before benchmarks + let _ = get_static_model_bytes(); + + divan::main(); +} + +// Macro to generate benchmarks for each backend +macro_rules! bench_backend { + ($backend:ty, $mod_name:ident, $backend_name:literal) => { + #[divan::bench_group(name = $backend_name, sample_count = 10)] + mod $mod_name { + use super::*; + + type TestBackend = $backend; + type TestDevice = ::Device; + + /// File-based loading with copy mode (default) + #[divan::bench] + fn file_copy(bencher: Bencher) { + let bp_path = get_burnpack_path(); + let file_size = fs::metadata(&bp_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_file(&bp_path).zero_copy(false); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + /// File-based loading with zero-copy mode (mmap + bytes::Bytes) + #[divan::bench] + fn file_zero_copy(bencher: Bencher) { + let bp_path = get_burnpack_path(); + let file_size = fs::metadata(&bp_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_file(&bp_path).zero_copy(true); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + /// Static bytes with copy mode (simulating old behavior) + #[divan::bench] + fn static_copy(bencher: Bencher) { + let static_bytes = get_static_model_bytes(); + let file_size = static_bytes.len() as u64; + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + + // Simulate old behavior: copy static bytes to Vec, then load + let bytes = Bytes::from_bytes_vec(static_bytes.to_vec()); + let mut store = BurnpackStore::from_bytes(Some(bytes)).zero_copy(false); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + /// Static bytes with zero-copy mode (new from_static) + #[divan::bench] + fn static_zero_copy(bencher: Bencher) { + let static_bytes = get_static_model_bytes(); + let file_size = static_bytes.len() as u64; + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + + // Zero-copy: use from_static which keeps data in .rodata + let mut store = BurnpackStore::from_static(static_bytes); + model.load_from(&mut store).expect("Failed to load"); + }); + } + + /// In-memory shared bytes with zero-copy + #[divan::bench] + fn memory_shared_zero_copy(bencher: Bencher) { + let static_bytes = get_static_model_bytes(); + let file_size = static_bytes.len() as u64; + + // Pre-create shared bytes outside the benchmark loop + let shared = bytes::Bytes::from_static(static_bytes); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let device: TestDevice = Default::default(); + let mut model = LargeModel::::new(&device); + + // Create Bytes from shared (cheap clone of Arc) + let bytes = Bytes::from_shared(shared.clone(), AllocationProperty::Other); + let mut store = BurnpackStore::from_bytes(Some(bytes)).zero_copy(true); + model.load_from(&mut store).expect("Failed to load"); + }); + } + } + }; +} + +// ============================================================================= +// Zero-copy verification (proves operations use static region data) +// ============================================================================= + +/// Verify that zero-copy loading actually uses data from the static region. +/// This runs once at startup to prove correctness before benchmarking. +#[divan::bench_group(name = "Zero-Copy Verification", sample_count = 1)] +mod verification { + use super::*; + use burn_ndarray::NdArray; + + type B = NdArray; + + /// Verify zero-copy: tensor storage is borrowed (not owned) + #[divan::bench] + fn verify_storage_is_borrowed() { + let static_bytes = get_static_model_bytes(); + + // Load model with zero-copy from static bytes + let device = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_static(static_bytes); + model.load_from(&mut store).expect("Failed to load"); + + // Get the first layer's weight tensor and verify it uses borrowed storage + let weight = model.layers[0].weight.val(); + // .into_primitive() returns TensorPrimitive, .tensor() extracts B::FloatTensorPrimitive + let ndarray_tensor = weight.into_primitive().tensor(); + + // Verify the storage is borrowed (zero-copy from static region) + assert!( + ndarray_tensor.is_borrowed(), + "ZERO-COPY FAILURE: Tensor storage is NOT borrowed. \ + Data was copied instead of being zero-copy!" + ); + + println!("✅ Verified: Tensor storage is borrowed (zero-copy from static region)"); + } + + /// Verify ALL layers use borrowed (zero-copy) storage. + /// This is the key proof that loaded weights point to static memory. + #[divan::bench] + fn verify_all_layers_borrowed() { + let static_bytes = get_static_model_bytes(); + + // Load model with zero-copy + let device = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_static(static_bytes); + model.load_from(&mut store).expect("Failed to load"); + + // Check ALL layers have borrowed storage + let mut total_elements = 0usize; + for (i, layer) in model.layers.iter().enumerate() { + let weight = layer.weight.val(); + total_elements += weight.shape().num_elements(); + + assert!( + weight.into_primitive().tensor().is_borrowed(), + "Layer {} weight should be borrowed (zero-copy)", + i + ); + } + + let total_mb = (total_elements * 4) as f64 / 1_048_576.0; + println!( + "✅ Verified: All {} layers use borrowed storage", + model.layers.len() + ); + println!( + " - Model size: {:.2} MB - all pointing to static region", + total_mb + ); + } + + /// Verify data is readable and correct using sum().into_scalar(). + /// Note: sum() triggers COW copy, so this shows ops work correctly on zero-copy data. + #[divan::bench] + fn verify_ops_produce_correct_results() { + let static_bytes = get_static_model_bytes(); + + let device = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_static(static_bytes); + model.load_from(&mut store).expect("Failed to load"); + + // Compute sum of first layer weight - proves data is valid + let weight = model.layers[0].weight.val(); + let sum: f32 = weight.sum().into_scalar(); + + assert!(sum.is_finite(), "Sum should be finite"); + println!("✅ Verified: Operations on zero-copy data produce valid results"); + println!(" - First layer sum: {:.4}", sum); + } + + /// Verify operations produce correct results on zero-copy data + #[divan::bench] + fn verify_operations_on_static_data() { + let static_bytes = get_static_model_bytes(); + + // Load model with zero-copy + let device = Default::default(); + let mut model = LargeModel::::new(&device); + let mut store = BurnpackStore::from_static(static_bytes); + model.load_from(&mut store).expect("Failed to load"); + + // Perform operations on the loaded weights + let weight = model.layers[0].weight.val(); + let shape = weight.shape(); + + // Test 1: Sum should be finite (not NaN or Inf) + let sum: f32 = weight.clone().sum().to_data().to_vec().unwrap()[0]; + assert!( + sum.is_finite(), + "Operation failed: sum is not finite ({})", + sum + ); + + // Test 2: Matrix multiply with itself transposed (W @ W.T) + let transposed = weight.clone().transpose(); + let matmul_result = weight.clone().matmul(transposed); + let matmul_sum: f32 = matmul_result.sum().to_data().to_vec().unwrap()[0]; + assert!( + matmul_sum.is_finite(), + "Matmul failed: result sum is not finite ({})", + matmul_sum + ); + + // Test 3: Element-wise operations + let doubled = weight.clone() * 2.0; + let doubled_sum: f32 = doubled.sum().to_data().to_vec().unwrap()[0]; + assert!( + (doubled_sum - sum * 2.0).abs() < 1e-3, + "Element-wise op failed: doubled_sum ({}) != sum*2 ({})", + doubled_sum, + sum * 2.0 + ); + + println!("✅ Verified: Operations on zero-copy data produce correct results"); + println!(" - Weight shape: {:?}", shape.as_slice()); + println!(" - Sum: {:.4}", sum); + println!(" - Matmul result sum: {:.4}", matmul_sum); + } + + /// Compare zero-copy vs copy: verify both produce identical results + #[divan::bench] + fn verify_copy_vs_zero_copy_equality() { + let static_bytes = get_static_model_bytes(); + let device: ::Device = Default::default(); + + // Load with zero-copy + let mut model_zc = LargeModel::::new(&device); + let mut store_zc = BurnpackStore::from_static(static_bytes); + model_zc + .load_from(&mut store_zc) + .expect("Failed to load zero-copy"); + + // Load with copy (simulate old behavior) + let mut model_copy = LargeModel::::new(&device); + let bytes = Bytes::from_bytes_vec(static_bytes.to_vec()); + let mut store_copy = BurnpackStore::from_bytes(Some(bytes)).zero_copy(false); + model_copy + .load_from(&mut store_copy) + .expect("Failed to load copy"); + + // Compare weights from both models + for (i, (layer_zc, layer_copy)) in model_zc + .layers + .iter() + .zip(model_copy.layers.iter()) + .enumerate() + { + let weight_zc = layer_zc.weight.val(); + let weight_copy = layer_copy.weight.val(); + + // Check shapes match + assert_eq!( + weight_zc.shape(), + weight_copy.shape(), + "Layer {} weight shapes don't match", + i + ); + + // Check values match (using sum as a proxy) + let sum_zc: f32 = weight_zc.clone().sum().to_data().to_vec().unwrap()[0]; + let sum_copy: f32 = weight_copy.clone().sum().to_data().to_vec().unwrap()[0]; + assert!( + (sum_zc - sum_copy).abs() < 1e-6, + "Layer {} weight sums don't match: zero-copy={}, copy={}", + i, + sum_zc, + sum_copy + ); + } + + println!( + "✅ Verified: Zero-copy and copy loading produce identical results for all {} layers", + model_zc.layers.len() + ); + } +} + +// ============================================================================= +// Store-only benchmarks (no backend allocation overhead) +// These show the TRUE zero-copy benefit at the store level +// ============================================================================= + +#[divan::bench_group(name = "Store Only (no backend)", sample_count = 10)] +mod store_only { + use super::*; + + /// File-based store with copy mode - measures store overhead only + #[divan::bench] + fn file_copy(bencher: Bencher) { + let bp_path = get_burnpack_path(); + let file_size = fs::metadata(&bp_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let mut store = BurnpackStore::from_file(&bp_path).zero_copy(false); + // Just iterate through all tensor snapshots, calling to_data() on each + // This forces the store to read and materialize all tensor data + let snapshots = store.get_all_snapshots().expect("Failed to get snapshots"); + for snapshot in snapshots.values() { + let _data = snapshot.to_data().expect("Failed to get tensor data"); + } + }); + } + + /// File-based store with zero-copy mode - measures store overhead only + #[divan::bench] + fn file_zero_copy(bencher: Bencher) { + let bp_path = get_burnpack_path(); + let file_size = fs::metadata(&bp_path).unwrap().len(); + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let mut store = BurnpackStore::from_file(&bp_path).zero_copy(true); + let snapshots = store.get_all_snapshots().expect("Failed to get snapshots"); + for snapshot in snapshots.values() { + let _data = snapshot.to_data().expect("Failed to get tensor data"); + } + }); + } + + /// Static bytes with copy mode - measures store overhead only + #[divan::bench] + fn static_copy(bencher: Bencher) { + let static_bytes = get_static_model_bytes(); + let file_size = static_bytes.len() as u64; + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + // Simulate old behavior: copy static bytes to Vec + let bytes = Bytes::from_bytes_vec(static_bytes.to_vec()); + let mut store = BurnpackStore::from_bytes(Some(bytes)).zero_copy(false); + let snapshots = store.get_all_snapshots().expect("Failed to get snapshots"); + for snapshot in snapshots.values() { + let _data = snapshot.to_data().expect("Failed to get tensor data"); + } + }); + } + + /// Static bytes with zero-copy mode - measures store overhead only + #[divan::bench] + fn static_zero_copy(bencher: Bencher) { + let static_bytes = get_static_model_bytes(); + let file_size = static_bytes.len() as u64; + + bencher + .counter(divan::counter::BytesCount::new(file_size)) + .bench(|| { + let mut store = BurnpackStore::from_static(static_bytes); + let snapshots = store.get_all_snapshots().expect("Failed to get snapshots"); + for snapshot in snapshots.values() { + let _data = snapshot.to_data().expect("Failed to get tensor data"); + } + }); + } +} + +// ============================================================================= +// Full model loading benchmarks (includes backend allocation) +// ============================================================================= + +// Generate benchmarks for each backend +bench_backend!(NdArrayBackend, ndarray_backend, "NdArray Backend (CPU)"); + +#[cfg(feature = "wgpu")] +bench_backend!(WgpuBackend, wgpu_backend, "WGPU Backend (GPU)"); + +#[cfg(feature = "cuda")] +bench_backend!(CudaBackend, cuda_backend, "CUDA Backend (NVIDIA GPU)"); + +#[cfg(feature = "tch")] +bench_backend!(TchBackend, tch_backend, "LibTorch Backend"); + +#[cfg(feature = "metal")] +bench_backend!(MetalBackend, metal_backend, "Metal Backend (Apple GPU)"); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/examples/burnpack_inspect.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/examples/burnpack_inspect.rs new file mode 100644 index 0000000..066f3ec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/examples/burnpack_inspect.rs @@ -0,0 +1,148 @@ +//! Example: Generate a Burnpack file for inspection +//! +//! This example creates a simple Burnpack file that you can examine to understand the format. +//! +//! Usage: +//! cargo run --example burnpack-inspect [output_path] +//! +//! Example: +//! cargo run --example burnpack-inspect sample.bpk +//! cargo run --example burnpack-inspect /tmp/test.bpk +//! +//! After generating the file, examine it with: +//! hexdump -C sample.bpk | head -100 +//! xxd sample.bpk | head -100 +//! hexyl sample.bpk +use burn_core as burn; + +use burn_core::module::Module; +use burn_ndarray::NdArray; +use burn_nn::{Linear, LinearConfig}; +use burn_store::{BurnpackStore, ModuleSnapshot}; +use burn_tensor::backend::Backend; +use std::env; + +// Simple model with a few layers +#[derive(Module, Debug)] +struct SampleModel { + linear1: Linear, + linear2: Linear, + linear3: Linear, +} + +impl SampleModel { + fn new(device: &B::Device) -> Self { + Self { + linear1: LinearConfig::new(128, 64).init(device), + linear2: LinearConfig::new(64, 32).init(device), + linear3: LinearConfig::new(32, 10).init(device), + } + } +} + +fn main() { + type Backend = NdArray; + + // Get output path from command line or use default + let output_path = env::args() + .nth(1) + .unwrap_or_else(|| "sample.bpk".to_string()); + + println!("Creating sample Burnpack file: {}", output_path); + println!(); + + // Create a simple model + let device = Default::default(); + let model = SampleModel::::new(&device); + + // Save to Burnpack format with metadata + let mut store = BurnpackStore::from_file(&output_path) + .overwrite(true) + .metadata("format", "burnpack") + .metadata("description", "Sample file for examining Burnpack format") + .metadata("version", env!("CARGO_PKG_VERSION")) + .metadata("author", "Burn Example"); + + model.save_into(&mut store).expect("Failed to save model"); + + println!("✅ Successfully created: {}", output_path); + println!(); + println!("📋 File Structure:"); + println!(" ┌─────────────────────────────────────┐"); + println!(" │ Header (10 bytes) │"); + println!(" ├─────────────────────────────────────┤"); + println!(" │ - Magic: 0x4E525542 (BURN in LE) │"); + println!(" │ - Version: 0x0001 (2 bytes) │"); + println!(" │ - Metadata size: (4 bytes, u32 LE) │"); + println!(" ├─────────────────────────────────────┤"); + println!(" │ Metadata (CBOR format) │"); + println!(" ├─────────────────────────────────────┤"); + println!(" │ - Tensor descriptors │"); + println!(" │ * name, dtype, shape, offsets │"); + println!(" │ - User metadata │"); + println!(" ├─────────────────────────────────────┤"); + println!(" │ Tensor Data (raw bytes, LE) │"); + println!(" ├─────────────────────────────────────┤"); + println!(" │ - linear1.weight [64, 128] │"); + println!(" │ - linear1.bias [64] │"); + println!(" │ - linear2.weight [32, 64] │"); + println!(" │ - linear2.bias [32] │"); + println!(" │ - linear3.weight [10, 32] │"); + println!(" │ - linear3.bias [10] │"); + println!(" └─────────────────────────────────────┘"); + println!(); + println!("📊 Model Contents:"); + println!(" - linear1.weight: [64, 128] = 8,192 params → 32,768 bytes"); + println!(" - linear1.bias: [64] = 64 params → 256 bytes"); + println!(" - linear2.weight: [32, 64] = 2,048 params → 8,192 bytes"); + println!(" - linear2.bias: [32] = 32 params → 128 bytes"); + println!(" - linear3.weight: [10, 32] = 320 params → 1,280 bytes"); + println!(" - linear3.bias: [10] = 10 params → 40 bytes"); + println!(" ───────────────────────────────────────────────────────"); + + let total_params = 8192 + 64 + 2048 + 32 + 320 + 10; + let total_bytes = total_params * 4; + println!( + " Total: {} parameters = {} KB", + total_params, + total_bytes / 1024 + ); + println!(); + + // Get actual file size + if let Ok(metadata) = std::fs::metadata(&output_path) { + let file_size = metadata.len(); + println!( + "📦 File size: {} bytes ({:.2} KB)", + file_size, + file_size as f64 / 1024.0 + ); + } + + println!(); + println!("🔍 Inspection Commands:"); + println!(); + println!(" # View first 100 bytes in hex:"); + println!(" hexdump -C {} | head -20", output_path); + println!(); + println!(" # View header only (10 bytes):"); + println!(" head -c 10 {} | hexdump -C", output_path); + println!(); + println!(" # View with prettier hex viewer (if installed):"); + println!(" hexyl {} | head -50", output_path); + println!(); + println!(" # View in binary format:"); + println!(" xxd -b {} | head -20", output_path); + println!(); + println!(" # Extract and examine header:"); + println!(" # Magic (bytes 0-3): Should be 42 55 52 4E (BURN)"); + println!(" # Version (bytes 4-5): Should be 01 00"); + println!(" # Metadata size (bytes 6-9): u32 little-endian"); + println!(); + println!(" # Load back the model:"); + println!( + " # let mut store = BurnpackStore::from_file(\"{}\");", + output_path + ); + println!(" # model.load_from(&mut store)?;"); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/Cargo.toml new file mode 100644 index 0000000..ec06ba1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "pytorch-tests" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dev-dependencies] +burn = { path = "../../burn" } +burn-ndarray = { path = "../../burn-ndarray" } +burn-autodiff = { path = "../../burn-autodiff" } +burn-store = { path = "../", features = ["std", "pytorch"] } +serde = { workspace = true } +float-cmp = { workspace = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/src/lib.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/backend.rs new file mode 100644 index 0000000..a264f96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/backend.rs @@ -0,0 +1 @@ +pub type TestBackend = burn_ndarray::NdArray; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/export_weights.py new file mode 100755 index 0000000..24a65c9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/export_weights.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.norm1 = nn.BatchNorm2d(5) + + def forward(self, x): + x = self.norm1(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + # Condition batch norm (each forward will affect the running stats) + x1 = torch.ones(1, 5, 2, 2) - 0.5 + _ = model(x1) + model.eval() # Set to eval mode to freeze running stats + # Save the model after the first forward + torch.save(model.state_dict(), "batch_norm2d.pt") + + x2 = torch.ones(1, 5, 2, 2) - 0.3 + print("Input shape: {}", x2.shape) + output = model(x2) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/mod.rs new file mode 100644 index 0000000..ac4053c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/batch_norm/mod.rs @@ -0,0 +1,62 @@ +use burn::{ + module::Module, + nn::{BatchNorm, BatchNormConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + norm1: BatchNorm, +} + +impl Net { + pub fn new(device: &B::Device) -> Self { + Self { + norm1: BatchNormConfig::new(5).init(device), // Python model uses BatchNorm2d(5) + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.norm1.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + #[test] + fn batch_norm2d() { + let device = Default::default(); + let mut model = Net::::new(&device); + let mut store = PytorchStore::from_file("tests/batch_norm/batch_norm2d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::ones([1, 5, 2, 2], &device) - 0.3; + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [[0.68515635, 0.68515635], [0.68515635, 0.68515635]], + [[0.68515635, 0.68515635], [0.68515635, 0.68515635]], + [[0.68515635, 0.68515635], [0.68515635, 0.68515635]], + [[0.68515635, 0.68515635], [0.68515635, 0.68515635]], + [[0.68515635, 0.68515635], [0.68515635, 0.68515635]], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/export_weights.py new file mode 100755 index 0000000..1b2e020 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/export_weights.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + buffer = torch.tensor([True, False, True]) + self.register_buffer("buffer", buffer, persistent=True) + + def forward(self, x): + x = self.buffer + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "boolean.pt") + + input = torch.ones(3, 3) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/mod.rs new file mode 100644 index 0000000..31cd027 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/boolean/mod.rs @@ -0,0 +1,58 @@ +use burn::{ + module::{Module, Param, ParamId}, + tensor::{Bool, Tensor, TensorData, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + buffer: Param>, +} + +impl Net { + /// Create a new model with placeholder values. + pub fn init(device: &B::Device) -> Self { + Self { + buffer: Param::initialized( + ParamId::new(), + Tensor::from_bool(TensorData::from([false, false, false]), device), + ), + } + } + + /// Forward pass of the model. + pub fn forward(&self, _x: Tensor) -> Tensor { + self.buffer.val() + } +} + +#[cfg(test)] +mod tests { + + use burn::tensor::TensorData; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + use crate::backend::TestBackend; + + #[test] + fn boolean() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/boolean/boolean.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::ones([3, 3], &device); + + let output = model.forward(input); + + let expected = Tensor::::from_bool( + TensorData::from([true, false, true]), + &device, + ); + + assert_eq!(output.to_data(), expected.to_data()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/export_weights.py new file mode 100755 index 0000000..ecd678c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/export_weights.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + buffer = torch.ones(3, 3) + self.register_buffer("buffer", buffer, persistent=True) + + def forward(self, x): + x = self.buffer + x + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "buffer.pt") + + input = torch.ones(3, 3) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/mod.rs new file mode 100644 index 0000000..590354b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/buffer/mod.rs @@ -0,0 +1,53 @@ +use burn::{ + module::{Module, Param}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + buffer: Param>, +} + +impl Net { + /// Create a new model with placeholder values. + pub fn init(device: &B::Device) -> Self { + Self { + buffer: Param::from_tensor(Tensor::zeros([3, 3], device)), + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.buffer.val() + x + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + #[test] + fn buffer() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/buffer/buffer.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::ones([3, 3], &device); + + let output = model.forward(input); + + let expected = Tensor::::ones([3, 3], &device) * 2.0; + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/export_weights.py new file mode 100755 index 0000000..9bae3bf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/export_weights.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class ConvBlock(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size): + super(ConvBlock, self).__init__() + self.conv = nn.Conv2d(in_channels, out_channels, kernel_size) + self.norm = nn.BatchNorm2d(out_channels) + + def forward(self, x): + x = self.conv(x) + x = self.norm(x) + return x + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + + self.conv_blocks = nn.Sequential( + ConvBlock(2, 4, (3, 2)), + ConvBlock(4, 6, (3, 2)), + ) + self.norm1 = nn.BatchNorm2d(6) + + self.fc1 = nn.Linear(120, 12) + self.fc2 = nn.Linear(12, 10) + + def forward(self, x): + x = self.conv_blocks(x) + x = self.norm1(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = F.relu(x) + x = self.fc2(x) + x = F.log_softmax(x, dim=1) + return x + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(2) + + + model = Net().to(torch.device("cpu")) + + # Condition the model (batch norm requires a forward pass to compute the mean and variance) + x1 = torch.ones(1, 2, 9, 6) - 0.1 + x2 = torch.ones(1, 2, 9, 6) - 0.3 + output = model(x1) + output = model(x2) + model.eval() # set to eval mode + + torch.save(model.state_dict(), "complex_nested.pt") + + # feed test data + x = torch.ones(1, 2, 9, 6) - 0.5 + output = model(x) + print("Input shape: {}", x.shape) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/mod.rs new file mode 100644 index 0000000..ef09f3e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/complex_nested/mod.rs @@ -0,0 +1,240 @@ +use burn::tensor::Tolerance; +use burn::tensor::ops::FloatElem; +use burn::{ + module::Module, + nn::{ + BatchNorm, BatchNormConfig, Linear, LinearConfig, + conv::{Conv2d, Conv2dConfig}, + }, + tensor::{ + Tensor, + activation::{log_softmax, relu}, + backend::Backend, + }, +}; +use burn_autodiff::Autodiff; +use burn_store::{ModuleSnapshot, PytorchStore}; + +#[derive(Module, Debug)] +pub struct ConvBlock { + conv: Conv2d, + norm: BatchNorm, +} + +#[derive(Module, Debug)] +pub struct Net { + conv_blocks: Vec>, + norm1: BatchNorm, + fc1: Linear, + fc2: Linear, +} + +impl Net { + pub fn init(device: &B::Device) -> Self { + let conv_blocks = vec![ + ConvBlock { + conv: Conv2dConfig::new([2, 4], [3, 2]).init(device), + norm: BatchNormConfig::new(4).init(device), // matches conv output channels + }, + ConvBlock { + conv: Conv2dConfig::new([4, 6], [3, 2]).init(device), + norm: BatchNormConfig::new(6).init(device), // matches conv output channels + }, + ]; + let norm1 = BatchNormConfig::new(6).init(device); + let fc1 = LinearConfig::new(120, 12).init(device); + let fc2 = LinearConfig::new(12, 10).init(device); + + Self { + conv_blocks, + norm1, + fc1, + fc2, + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv_blocks[0].forward(x); + let x = self.conv_blocks[1].forward(x); + let x = self.norm1.forward(x); + let x = x.reshape([0, -1]); + let x = self.fc1.forward(x); + let x = relu(x); + let x = self.fc2.forward(x); + + log_softmax(x, 1) + } +} + +impl ConvBlock { + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv.forward(x); + + self.norm.forward(x) + } +} + +/// Partial model to test loading of partial records. +#[derive(Module, Debug)] +pub struct PartialNet { + conv1: ConvBlock, +} + +impl PartialNet { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let conv1 = ConvBlock { + conv: Conv2dConfig::new([2, 4], [3, 2]).init(device), + norm: BatchNormConfig::new(4).init(device), // matches conv output channels + }; + Self { conv1 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.conv1.forward(x) + } +} + +/// Model with extra fields to test loading of records (e.g. from a different model). +#[derive(Module, Debug)] +pub struct PartialWithExtraNet { + conv1: ConvBlock, + extra_field: bool, // This field is not present in the pytorch model +} + +impl PartialWithExtraNet { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let conv1 = ConvBlock { + conv: Conv2dConfig::new([2, 4], [3, 2]).init(device), + norm: BatchNormConfig::new(4).init(device), // matches conv output channels + }; + + Self { + conv1, + extra_field: true, + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.conv1.forward(x) + } +} + +type TestBackend = burn_ndarray::NdArray; + +fn model_test(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::ones([1, 2, 9, 6], &device) - 0.5; + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + -2.306_613, + -2.058_945_4, + -2.298_372_7, + -2.358_294, + -2.296_395_5, + -2.416_090_5, + -2.107_669, + -2.428_420_8, + -2.526_469, + -2.319_918_6, + ]], + &device, + ); + + output.to_data().assert_approx_eq::>( + &expected.to_data(), + Tolerance::absolute(precision), + ); +} + +#[test] +fn full_record() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/complex_nested/complex_nested.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + model_test(model, 1e-8); +} + +#[test] +fn full_record_autodiff() { + let device = Default::default(); + let mut model = Net::>::init(&device); + let mut store = PytorchStore::from_file("tests/complex_nested/complex_nested.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); +} + +#[test] +fn half_record() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/complex_nested/complex_nested.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + model_test(model, 1e-4); +} + +#[test] +fn partial_model_loading() { + let device = Default::default(); + let mut model = PartialNet::::init(&device); + + // Load the full model but rename "conv_blocks.0.*" to "conv1.*" + let mut store = PytorchStore::from_file("tests/complex_nested/complex_nested.pt") + .with_key_remapping("conv_blocks\\.0\\.(.*)", "conv1.$1") + .allow_partial(true); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::ones([1, 2, 9, 6], &device) - 0.5; + + let output = model.forward(input); + + // get the sum of all elements in the output tensor for quick check + let sum = output.sum(); + + assert!((sum.into_scalar() - 4.871538).abs() < 0.000002); +} + +#[test] +fn extra_field_model_loading() { + let device = Default::default(); + let mut model = PartialWithExtraNet::::init(&device); + + // Load the full model but rename "conv_blocks.0.*" to "conv1.*" + let mut store = PytorchStore::from_file("tests/complex_nested/complex_nested.pt") + .with_key_remapping("conv_blocks\\.0\\.(.*)", "conv1.$1") + .allow_partial(true); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::ones([1, 2, 9, 6], &device) - 0.5; + + let output = model.forward(input); + + // get the sum of all elements in the output tensor for quick check + let sum = output.sum(); + + assert!((sum.into_scalar() - 4.871538).abs() < 0.000002); + + assert!(model.extra_field); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/export_weights.py new file mode 100755 index 0000000..8dd194b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/export_weights.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.fc1 = nn.Linear(2, 3) + self.fc2 = nn.Linear(3, 4, bias=False) + + def forward(self, x): + x = self.fc1(x) + x = F.relu(x) # Add relu so that PyTorch optimizer does not combine fc1 and fc2 + x = self.fc2(x) + + return x + +CONFIG = { + "n_head": 2, + "n_layer": 3, + "d_model": 512, + "some_float": 0.1, + "some_int": 1, + "some_bool": True, + "some_str": "hello", + "some_list_int": [1, 2, 3], + "some_list_str": ["hello", "world"], + "some_list_float": [0.1, 0.2, 0.3], + "some_dict": { + "some_key": "some_value" + } +} + +class ModelWithBias(nn.Module): + def __init__(self): + super(ModelWithBias, self).__init__() + self.fc1 = nn.Linear(2, 3) + + def forward(self, x): + x = self.fc1(x) + + return x + + +def main(): + + model = Model().to(torch.device("cpu")) + + weights_with_config = { + "my_model": model.state_dict(), + "my_config": CONFIG + } + + torch.save(weights_with_config, "weights_with_config.pt") + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/mod.rs new file mode 100644 index 0000000..6c8deb3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/config/mod.rs @@ -0,0 +1,53 @@ +#![allow(clippy::too_many_arguments)] // To mute derive Config warning +use std::collections::HashMap; + +use burn::config::Config; + +#[allow(clippy::too_many_arguments)] +#[derive(Debug, PartialEq, Config)] +struct NetConfig { + n_head: usize, + n_layer: usize, + d_model: usize, + some_float: f64, + some_int: i32, + some_bool: bool, + some_str: String, + some_list_int: Vec, + some_list_str: Vec, + some_list_float: Vec, + some_dict: HashMap, +} + +#[cfg(test)] +mod tests { + use burn_store::pytorch::PytorchReader; + + use super::*; + + #[test] + fn test_net_config() { + let config_expected = NetConfig { + n_head: 2, + n_layer: 3, + d_model: 512, + some_float: 0.1, + some_int: 1, + some_bool: true, + some_str: "hello".to_string(), + some_list_int: vec![1, 2, 3], + some_list_str: vec!["hello".to_string(), "world".to_string()], + some_list_float: vec![0.1, 0.2, 0.3], + some_dict: { + let mut map = HashMap::new(); + map.insert("some_key".to_string(), "some_value".to_string()); + map + }, + }; + let path = "tests/config/weights_with_config.pt"; + let top_level_key = Some("my_config"); + let config: NetConfig = PytorchReader::load_config(path, top_level_key).unwrap(); + + assert_eq!(config, config_expected); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/export_weights.py new file mode 100755 index 0000000..bda180e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/export_weights.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv1d(2, 2, 2) + self.conv2 = nn.Conv1d(2, 2, 2, bias=False) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "conv1d.pt") + + input = torch.rand(1, 2, 6) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/mod.rs new file mode 100644 index 0000000..52e042d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv1d/mod.rs @@ -0,0 +1,97 @@ +use burn::{ + module::Module, + nn::conv::{Conv1d, Conv1dConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + conv1: Conv1d, + conv2: Conv1d, +} + +impl Net { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let conv1 = Conv1dConfig::new(2, 2, 2).init(device); + let conv2 = Conv1dConfig::new(2, 2, 2).with_bias(false).init(device); + + Self { conv1, conv2 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + + self.conv2.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + fn conv1d(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[ + [ + 0.93708336, 0.65559506, 0.31379688, 0.19801933, 0.41619217, 0.28432965, + ], + [ + 0.33977574, + 0.523_940_8, + 0.798_063_9, + 0.77176833, + 0.01122457, + 0.80996025, + ], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [0.02987457, 0.03134188, 0.04234261, -0.02437721], + [-0.03788019, -0.02972012, -0.00806090, -0.01981254], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn conv1d_full_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv1d/conv1d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv1d(model, 1e-7); + } + + #[test] + fn conv1d_half_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv1d/conv1d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv1d(model, 1e-4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/export_weights.py new file mode 100755 index 0000000..cdc3f76 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/export_weights.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv2d(2, 2, (2,2)) + self.conv2 = nn.Conv2d(2, 2, (2,2), bias=False) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "conv2d.pt") + + input = torch.rand(1, 2, 5, 5) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/mod.rs new file mode 100644 index 0000000..b14e17f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv2d/mod.rs @@ -0,0 +1,134 @@ +use burn::{ + module::Module, + nn::conv::{Conv2d, Conv2dConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + conv1: Conv2d, + conv2: Conv2d, +} + +impl Net { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let conv1 = Conv2dConfig::new([2, 2], [2, 2]).init(device); + let conv2 = Conv2dConfig::new([2, 2], [2, 2]) + .with_bias(false) + .init(device); + + Self { conv1, conv2 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + + self.conv2.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + fn conv2d(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[ + [ + [ + 0.024_595_8, + 0.25883394, + 0.93905586, + 0.416_715_5, + 0.713_979_7, + ], + [0.267_644_3, 0.990_609, 0.28845078, 0.874_962_4, 0.505_920_8], + [0.23659128, 0.757_007_4, 0.23458993, 0.64705235, 0.355_621_4], + [0.445_182_8, 0.01930594, 0.26160914, 0.771_317, 0.37846136], + [ + 0.99802476, + 0.900_794_2, + 0.476_588_2, + 0.16625845, + 0.804_481_1, + ], + ], + [ + [ + 0.65517855, + 0.17679012, + 0.824_772_3, + 0.803_550_9, + 0.943_447_5, + ], + [0.21972018, 0.417_697, 0.49031407, 0.57302874, 0.12054086], + [0.14518881, 0.772_002_3, 0.38275403, 0.744_236_7, 0.52850497], + [0.664_172_4, 0.60994434, 0.681_799_7, 0.74785537, 0.03694397], + [ + 0.751_675_7, + 0.148_438_4, + 0.12274551, + 0.530_407_2, + 0.414_796_4, + ], + ], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [-0.02502128, 0.00250649, 0.04841233], + [0.04589614, -0.00296854, 0.01991477], + [0.02920526, 0.059_497_3, 0.04326791], + ], + [ + [-0.04825336, 0.080_190_9, -0.02375088], + [0.02885434, 0.09638263, -0.07460806], + [0.02004079, 0.06244051, 0.035_887_1], + ], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn conv2d_full_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv2d/conv2d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv2d(model, 1e-7); + } + + #[test] + fn conv2d_half_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv2d/conv2d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv2d(model, 1e-4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/export_weights.py new file mode 100755 index 0000000..a15250c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/export_weights.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.ConvTranspose1d(2, 2, 2) + self.conv2 = nn.ConvTranspose1d(2, 2, 2, bias=False) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "conv_transpose1d.pt") + + input = torch.rand(1, 2, 2) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/mod.rs new file mode 100644 index 0000000..ce7d24f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose1d/mod.rs @@ -0,0 +1,87 @@ +use burn::{ + module::Module, + nn::conv::{ConvTranspose1d, ConvTranspose1dConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + conv1: ConvTranspose1d, + conv2: ConvTranspose1d, +} + +impl Net { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let conv1 = ConvTranspose1dConfig::new([2, 2], 2).init(device); + let conv2 = ConvTranspose1dConfig::new([2, 2], 2) + .with_bias(false) + .init(device); + + Self { conv1, conv2 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + + self.conv2.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + fn conv_transpose1d(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[[0.93708336, 0.65559506], [0.31379688, 0.19801933]]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [0.02935525, 0.01119324, -0.01356167, -0.00682688], + [0.01644749, -0.01429807, 0.00083987, 0.00279229], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn conv_transpose1d_full() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv_transpose1d/conv_transpose1d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv_transpose1d(model, 1e-8); + } + + #[test] + fn conv_transpose1d_half() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv_transpose1d/conv_transpose1d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv_transpose1d(model, 1e-4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/export_weights.py new file mode 100755 index 0000000..8dc5ff8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/export_weights.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.ConvTranspose2d(2, 2, (2, 2)) + self.conv2 = nn.ConvTranspose2d(2, 2, (2, 2), bias=False) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "conv_transpose2d.pt") + + input = torch.rand(1, 2, 2, 2) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/mod.rs new file mode 100644 index 0000000..e86f690 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/conv_transpose2d/mod.rs @@ -0,0 +1,99 @@ +use burn::{ + module::Module, + nn::conv::{ConvTranspose2d, ConvTranspose2dConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + conv1: ConvTranspose2d, + conv2: ConvTranspose2d, +} + +impl Net { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let conv1 = ConvTranspose2dConfig::new([2, 2], [2, 2]).init(device); + let conv2 = ConvTranspose2dConfig::new([2, 2], [2, 2]) + .with_bias(false) + .init(device); + + Self { conv1, conv2 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + + self.conv2.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + fn conv_transpose2d(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[ + [[0.024_595_8, 0.25883394], [0.93905586, 0.416_715_5]], + [[0.713_979_7, 0.267_644_3], [0.990_609, 0.28845078]], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [0.04547675, 0.01879685, -0.01636661, 0.00310803], + [0.02090115, 0.01192738, -0.048_240_2, 0.02252235], + [0.03249975, -0.00460748, 0.05003899, 0.04029131], + [0.02185687, -0.10226749, -0.06508022, -0.01267705], + ], + [ + [0.00277598, -0.00513832, -0.059_048_3, 0.00567626], + [-0.03149522, -0.195_757_4, 0.03474613, 0.01997269], + [-0.10096474, 0.00679589, 0.041_919_7, -0.02464108], + [-0.03174751, 0.02963913, -0.02703723, -0.01860938], + ], + ]], + &device, + ); + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn conv_transpose2d_full() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv_transpose2d/conv_transpose2d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv_transpose2d(model, 1e-7); + } + + #[test] + fn conv_transpose2d_half() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/conv_transpose2d/conv_transpose2d.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + conv_transpose2d(model, 1e-4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/export_weights.py new file mode 100755 index 0000000..9510e3d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/export_weights.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.embed = nn.Embedding(10, 3) + + def forward(self, x): + x = self.embed(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "embedding.pt") + + input = torch.LongTensor([[1, 2, 4, 5], [4, 3, 2, 9]]) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/mod.rs new file mode 100644 index 0000000..888c768 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/embedding/mod.rs @@ -0,0 +1,86 @@ +use burn::{ + module::Module, + nn::{Embedding, EmbeddingConfig}, + tensor::{Int, Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + embed: Embedding, +} + +impl Net { + /// Create a new model. + pub fn init(device: &B::Device) -> Self { + let embed = EmbeddingConfig::new(10, 3).init(device); + Self { embed } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.embed.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + fn embedding(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data([[1, 2, 4, 5], [4, 3, 2, 9]], &device); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [ + [ + [-1.609_484_9, -0.10016718, -0.609_188_9], + [-0.97977227, -1.609_096_3, -0.712_144_6], + [-0.22227049, 1.687_113_4, -0.32062083], + [-0.29934573, 1.879_345_7, -0.07213178], + ], + [ + [-0.22227049, 1.687_113_4, -0.32062083], + [0.303_722, -0.777_314_3, -0.25145486], + [-0.97977227, -1.609_096_3, -0.712_144_6], + [-0.02878714, 2.357_111, -1.037_338_7], + ], + ], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn embedding_full_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/embedding/embedding.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + embedding(model, 1e-3); + } + + #[test] + fn embedding_half_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/embedding/embedding.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + embedding(model, 1e-3); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/export_weights.py new file mode 100755 index 0000000..7a22623 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/export_weights.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import torch +from torch import nn, Tensor + +class DwsConv(nn.Module): + """Depthwise separable convolution.""" + + def __init__(self, in_channels: int, out_channels: int, kernel_size: int) -> None: + super().__init__() + # Depthwise conv + self.dconv = nn.Conv2d(in_channels, in_channels, kernel_size, groups=in_channels) + # Pointwise conv + self.pconv = nn.Conv2d(in_channels, out_channels, kernel_size=1, groups=1) + + def forward(self, x: Tensor) -> Tensor: + x = self.dconv(x) + return self.pconv(x) + + +class Model(nn.Module): + def __init__(self, depthwise: bool = False) -> None: + super().__init__() + self.conv = DwsConv(2, 2, 3) if depthwise else nn.Conv2d(2, 2, 3) + + def forward(self, x: Tensor) -> Tensor: + return self.conv(x) + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "enum_depthwise_false.pt") + + input = torch.rand(1, 2, 5, 5) + + print("Depthwise is False") + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + print("Depthwise is True") + model = Model(depthwise=True).to(torch.device("cpu")) + torch.save(model.state_dict(), "enum_depthwise_true.pt") + + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/mod.rs new file mode 100644 index 0000000..8639b1e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/enum_module/mod.rs @@ -0,0 +1,197 @@ +use burn::{ + module::Module, + nn::conv::{Conv2d, Conv2dConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +#[allow(clippy::large_enum_variant)] +pub enum Conv { + DwsConv(DwsConv), + Conv(Conv2d), +} + +#[derive(Module, Debug)] +pub struct DwsConv { + dconv: Conv2d, + pconv: Conv2d, +} + +#[derive(Module, Debug)] +pub struct Net { + conv: Conv, +} + +impl Net { + /// Create a new model with DwsConv variant. + pub fn init_dws_conv(device: &B::Device) -> Self { + let dconv = Conv2dConfig::new([2, 2], [3, 3]) + .with_groups(2) + .init(device); + let pconv = Conv2dConfig::new([2, 2], [1, 1]) + .with_groups(1) + .init(device); + Net { + conv: Conv::DwsConv(DwsConv { dconv, pconv }), + } + } + + /// Create a new model with Conv variant. + pub fn init_conv(device: &B::Device) -> Self { + let conv2d_config = Conv2dConfig::new([2, 2], [3, 3]); + Net { + conv: Conv::Conv(conv2d_config.init(device)), + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + match &self.conv { + Conv::DwsConv(dws_conv) => { + let x = dws_conv.dconv.forward(x); + dws_conv.pconv.forward(x) + } + Conv::Conv(conv) => conv.forward(x), + } + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + #[test] + fn depthwise_false() { + let device = Default::default(); + let mut model = Net::::init_conv(&device); + let mut store = PytorchStore::from_file("tests/enum_module/enum_depthwise_false.pt"); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + let input = Tensor::::from_data( + [[ + [ + [0.713_979_7, 0.267_644_3, 0.990_609, 0.28845078, 0.874_962_4], + [0.505_920_8, 0.23659128, 0.757_007_4, 0.23458993, 0.64705235], + [0.355_621_4, 0.445_182_8, 0.01930594, 0.26160914, 0.771_317], + [0.37846136, 0.99802476, 0.900_794_2, 0.476_588_2, 0.16625845], + [ + 0.804_481_1, + 0.65517855, + 0.17679012, + 0.824_772_3, + 0.803_550_9, + ], + ], + [ + [0.943_447_5, 0.21972018, 0.417_697, 0.49031407, 0.57302874], + [0.12054086, 0.14518881, 0.772_002_3, 0.38275403, 0.744_236_7], + [0.52850497, 0.664_172_4, 0.60994434, 0.681_799_7, 0.74785537], + [ + 0.03694397, + 0.751_675_7, + 0.148_438_4, + 0.12274551, + 0.530_407_2, + ], + [0.414_796_4, 0.793_662, 0.21043217, 0.05550903, 0.863_884_4], + ], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [0.35449377, -0.02832414, 0.490_976_1], + [0.29709217, 0.332_586_3, 0.30594018], + [0.18101373, 0.30932188, 0.30558896], + ], + [ + [-0.17683622, -0.13244139, -0.05608707], + [0.23467252, -0.07038684, 0.255_044_1], + [-0.241_931_3, -0.20476191, -0.14468731], + ], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } + + #[test] + fn depthwise_true() { + let device = Default::default(); + let mut model = Net::::init_dws_conv(&device); + let mut store = PytorchStore::from_file("tests/enum_module/enum_depthwise_true.pt"); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::from_data( + [[ + [ + [0.713_979_7, 0.267_644_3, 0.990_609, 0.28845078, 0.874_962_4], + [0.505_920_8, 0.23659128, 0.757_007_4, 0.23458993, 0.64705235], + [0.355_621_4, 0.445_182_8, 0.01930594, 0.26160914, 0.771_317], + [0.37846136, 0.99802476, 0.900_794_2, 0.476_588_2, 0.16625845], + [ + 0.804_481_1, + 0.65517855, + 0.17679012, + 0.824_772_3, + 0.803_550_9, + ], + ], + [ + [0.943_447_5, 0.21972018, 0.417_697, 0.49031407, 0.57302874], + [0.12054086, 0.14518881, 0.772_002_3, 0.38275403, 0.744_236_7], + [0.52850497, 0.664_172_4, 0.60994434, 0.681_799_7, 0.74785537], + [ + 0.03694397, + 0.751_675_7, + 0.148_438_4, + 0.12274551, + 0.530_407_2, + ], + [0.414_796_4, 0.793_662, 0.21043217, 0.05550903, 0.863_884_4], + ], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [0.77874625, 0.859_017_6, 0.834_283_5], + [0.773_056_4, 0.73817325, 0.78292674], + [0.710_775_2, 0.747_187_2, 0.733_264_4], + ], + [ + [-0.44891885, -0.49027523, -0.394_170_7], + [-0.43836114, -0.33961445, -0.387_311_5], + [-0.581_134_3, -0.34197026, -0.535_035_7], + ], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/export_weights.py new file mode 100755 index 0000000..9e44950 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/export_weights.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.norm1 = nn.GroupNorm(2, 6) + + def forward(self, x): + x = self.norm1(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "group_norm.pt") + + x2 = torch.rand(1, 6, 2, 2) + print("Input shape: {}", x2.shape) + print("Input: {}", x2) + output = model(x2) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/mod.rs new file mode 100644 index 0000000..4b3b9d3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/group_norm/mod.rs @@ -0,0 +1,90 @@ +use burn::{ + module::Module, + nn::{GroupNorm, GroupNormConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + norm1: GroupNorm, +} + +impl Net { + /// Create a new model from the given record. + pub fn init(device: &B::Device) -> Self { + let norm1 = GroupNormConfig::new(2, 6).init(device); + Self { norm1 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.norm1.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + fn group_norm(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[ + [[0.757_631_6, 0.27931088], [0.40306926, 0.73468447]], + [[0.02928156, 0.799_858_6], [0.39713734, 0.75437194]], + [[0.569_508_5, 0.43877792], [0.63868046, 0.524_665_9]], + [[0.682_614_1, 0.305_149_5], [0.46354562, 0.45498633]], + [[0.572_472, 0.498_002_6], [0.93708336, 0.65559506]], + [[0.31379688, 0.19801933], [0.41619217, 0.28432965]], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [[1.042_578_5, -1.122_016_7], [-0.56195974, 0.938_733_6]], + [[-2.253_500_7, 1.233_672_9], [-0.588_804_1, 1.027_827_3]], + [[0.19124532, -0.40036356], [0.504_276_5, -0.01168585]], + [[1.013_829_2, -0.891_984_6], [-0.09224463, -0.13546038]], + [[0.45772314, 0.08172822], [2.298_641_4, 0.877_410_4]], + [[-0.84832406, -1.432_883_4], [-0.331_331_5, -0.997_103_7]], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn group_norm_full() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/group_norm/group_norm.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + group_norm(model, 1e-3); + } + + #[test] + fn group_norm_half() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/group_norm/group_norm.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + group_norm(model, 1e-3); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/export_weights.py new file mode 100755 index 0000000..151dda3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/export_weights.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + buffer = torch.tensor([1, 2, 3]) + self.register_buffer("buffer", buffer, persistent=True) + + def forward(self, x): + x = self.buffer + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "integer.pt") + + input = torch.ones(3, 3) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/mod.rs new file mode 100644 index 0000000..3e24cb3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/integer/mod.rs @@ -0,0 +1,72 @@ +use burn::{ + module::{Module, Param, ParamId}, + tensor::{Int, Tensor, TensorData, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + buffer: Param>, +} + +impl Net { + /// Create a new model with placeholder values. + pub fn init(device: &B::Device) -> Self { + Self { + buffer: Param::initialized( + ParamId::new(), + Tensor::::from_data(TensorData::from([0, 0, 0]), device), + ), + } + } + + /// Forward pass of the model. + pub fn forward(&self, _x: Tensor) -> Tensor { + self.buffer.val() + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + use burn::tensor::TensorData; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + fn integer(model: Net) { + let device = Default::default(); + + let input = Tensor::::ones([3, 3], &device); + + let output = model.forward(input); + + let expected = + Tensor::::from_data(TensorData::from([1, 2, 3]), &device); + + assert_eq!(output.to_data(), expected.to_data()); + } + + #[test] + fn integer_full_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/integer/integer.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + integer(model); + } + + #[test] + fn integer_half_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/integer/integer.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + integer(model); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/export_weights.py new file mode 100755 index 0000000..1b7490b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/export_weights.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class ConvModule(nn.Module): + def __init__(self): + super(ConvModule, self).__init__() + self.conv1 = nn.Conv2d(2, 2, (2,2)) + self.conv2 = nn.Conv2d(2, 2, (2,2), bias=False) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + return x + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv = ConvModule() + + def forward(self, x): + x = self.conv(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "key_remap.pt") + + input = torch.rand(1, 2, 5, 5) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/mod.rs new file mode 100644 index 0000000..5de33c6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap/mod.rs @@ -0,0 +1,118 @@ +use burn::{ + module::Module, + nn::conv::{Conv2d, Conv2dConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + conv1: Conv2d, + conv2: Conv2d, +} + +impl Net { + /// Create a new model. + pub fn init(device: &B::Device) -> Self { + let conv1 = Conv2dConfig::new([2, 2], [2, 2]).init(device); + let conv2 = Conv2dConfig::new([2, 2], [2, 2]) + .with_bias(false) + .init(device); + Self { conv1, conv2 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + + self.conv2.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + #[test] + fn key_remap() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/key_remap/key_remap.pt") + .with_key_remapping("conv\\.(.*)", "$1"); // Remove "conv" prefix, e.g. "conv.conv1" -> "conv1" + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::from_data( + [[ + [ + [ + 0.024_595_8, + 0.25883394, + 0.93905586, + 0.416_715_5, + 0.713_979_7, + ], + [0.267_644_3, 0.990_609, 0.28845078, 0.874_962_4, 0.505_920_8], + [0.23659128, 0.757_007_4, 0.23458993, 0.64705235, 0.355_621_4], + [0.445_182_8, 0.01930594, 0.26160914, 0.771_317, 0.37846136], + [ + 0.99802476, + 0.900_794_2, + 0.476_588_2, + 0.16625845, + 0.804_481_1, + ], + ], + [ + [ + 0.65517855, + 0.17679012, + 0.824_772_3, + 0.803_550_9, + 0.943_447_5, + ], + [0.21972018, 0.417_697, 0.49031407, 0.57302874, 0.12054086], + [0.14518881, 0.772_002_3, 0.38275403, 0.744_236_7, 0.52850497], + [0.664_172_4, 0.60994434, 0.681_799_7, 0.74785537, 0.03694397], + [ + 0.751_675_7, + 0.148_438_4, + 0.12274551, + 0.530_407_2, + 0.414_796_4, + ], + ], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [-0.02502128, 0.00250649, 0.04841233], + [0.04589614, -0.00296854, 0.01991477], + [0.02920526, 0.059_497_3, 0.04326791], + ], + [ + [-0.04825336, 0.080_190_9, -0.02375088], + [0.02885434, 0.09638263, -0.07460806], + [0.02004079, 0.06244051, 0.035_887_1], + ], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/export_weights.py new file mode 100755 index 0000000..95da5a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/export_weights.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import torch +from torch import nn, Tensor + + +class ConvBlock(nn.Module): + def __init__(self, in_channels: int, out_channels: int): + super().__init__() + self.block = nn.Sequential( + nn.Conv2d(in_channels, out_channels, 1, bias=False), + nn.BatchNorm2d(out_channels), + ) + + def forward(self, x: Tensor) -> Tensor: + return self.block(x) + + +class Model(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 6, 3, bias=False) + self.bn = nn.BatchNorm2d(6) + self.layer = nn.Sequential(ConvBlock(6, 6), ConvBlock(6, 6)) + + def forward(self, x: Tensor) -> Tensor: + x = self.conv(x) + x = self.bn(x) + x = self.layer(x) + + return x + + +def main(): + torch.set_printoptions(precision=8) + torch.manual_seed(42) + + model = Model() + + input = torch.rand(1, 3, 4, 4) + model(input) # condition batch norm + model.eval() + + with torch.no_grad(): + print(f"Input shape: {input.shape}") + print("Input type: {}", input.dtype) + print(f"Input: {input}") + output = model(input) + + print(f"Output: {output}") + print(f"Output Shape: {output.shape}") + + torch.save(model.state_dict(), "key_remap.pt") + + +if __name__ == "__main__": + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/mod.rs new file mode 100644 index 0000000..2d35a3d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/key_remap_chained/mod.rs @@ -0,0 +1,179 @@ +use std::marker::PhantomData; + +use burn::{ + module::Module, + nn::{ + BatchNorm, BatchNormConfig, + conv::{Conv2d, Conv2dConfig}, + }, + tensor::{Device, Tensor, backend::Backend}, +}; + +/// Some module that implements a specific method so it can be used in a sequential block. +pub trait ForwardModule { + fn forward(&self, input: Tensor) -> Tensor; +} + +/// Conv2d + BatchNorm block. +#[derive(Module, Debug)] +pub struct ConvBlock { + conv: Conv2d, + bn: BatchNorm, +} + +impl ForwardModule for ConvBlock { + fn forward(&self, input: Tensor) -> Tensor { + let out = self.conv.forward(input); + self.bn.forward(out) + } +} + +impl ConvBlock { + pub fn new(in_channels: usize, out_channels: usize, device: &Device) -> Self { + let conv = Conv2dConfig::new([in_channels, out_channels], [1, 1]) + .with_bias(false) + .init(device); + let bn = BatchNormConfig::new(out_channels).init(device); + + Self { conv, bn } + } +} + +/// Collection of sequential blocks. +#[derive(Module, Debug)] +pub struct ModuleBlock { + blocks: Vec, + _backend: PhantomData, +} + +impl> ModuleBlock { + pub fn forward(&self, input: Tensor) -> Tensor { + let mut out = input; + for block in &self.blocks { + out = block.forward(out); + } + out + } +} + +impl ModuleBlock> { + pub fn new(device: &Device) -> Self { + let blocks = vec![ConvBlock::new(6, 6, device), ConvBlock::new(6, 6, device)]; + + Self { + blocks, + _backend: PhantomData, + } + } +} + +#[derive(Module, Debug)] +pub struct Model { + conv: Conv2d, + bn: BatchNorm, + layer: ModuleBlock, +} + +impl Model> { + pub fn new(device: &Device) -> Self { + let conv = Conv2dConfig::new([3, 6], [3, 3]) + .with_bias(false) + .init(device); + let bn = BatchNormConfig::new(6).init(device); + + let layer = ModuleBlock::new(device); + + Self { conv, bn, layer } + } + + pub fn forward(&self, input: Tensor) -> Tensor { + let out = self.conv.forward(input); + let out = self.bn.forward(out); + self.layer.forward(out) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + #[test] + #[should_panic] + fn key_remap_chained_missing_pattern() { + // Loading record should fail due to missing pattern to map the layer.blocks + let device = Default::default(); + let mut model: Model = Model::new(&device); + let mut store = PytorchStore::from_file("tests/key_remap_chained/key_remap.pt") + // Map *.block.0.* -> *.conv.* + .with_key_remapping("(.+)\\.block\\.0\\.(.+)", "$1.conv.$2") + // Map *.block.1.* -> *.bn.* + .with_key_remapping("(.+)\\.block\\.1\\.(.+)", "$1.bn.$2"); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + } + + #[test] + fn key_remap_chained() { + let device = Default::default(); + let mut model: Model = Model::new(&device); + let mut store = PytorchStore::from_file("tests/key_remap_chained/key_remap.pt") + // Map *.block.0.* -> *.conv.* + .with_key_remapping("(.+)\\.block\\.0\\.(.+)", "$1.conv.$2") + // Map *.block.1.* -> *.bn.* + .with_key_remapping("(.+)\\.block\\.1\\.(.+)", "$1.bn.$2") + // Map layer.[i].* -> layer.blocks.[i].* + .with_key_remapping("layer\\.([0-9])\\.(.+)", "layer.blocks.$1.$2"); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::from_data( + [[ + [ + [0.76193494, 0.626_546_1, 0.49510366, 0.11974698], + [0.07161391, 0.03232569, 0.704_681, 0.254_516], + [0.399_373_7, 0.21224737, 0.40888822, 0.14808255], + [0.17329216, 0.665_855_4, 0.351_401_8, 0.808_671_6], + ], + [ + [0.33959562, 0.13321638, 0.41178054, 0.257_626_3], + [0.347_029_2, 0.02400219, 0.77974546, 0.15189773], + [0.75130886, 0.726_892_1, 0.85721636, 0.11647397], + [0.859_598_4, 0.263_624_2, 0.685_534_6, 0.96955734], + ], + [ + [0.42948407, 0.49613327, 0.38488472, 0.08250773], + [0.73995143, 0.00364107, 0.81039995, 0.87411255], + [0.972_853_2, 0.38206023, 0.08917904, 0.61241513], + [0.77621365, 0.00234562, 0.38650817, 0.20027226], + ], + ]], + &device, + ); + let expected = Tensor::::from_data( + [[ + [[0.198_967_1, 0.17847246], [0.06883702, 0.20012866]], + [[0.17582723, 0.11344293], [0.05444185, 0.13307181]], + [[0.192_229_5, 0.20391327], [0.06150475, 0.22688155]], + [[0.00230906, -0.02177845], [0.01129148, 0.00925517]], + [[0.14751078, 0.14433631], [0.05498439, 0.29049855]], + [[0.16868964, 0.133_269_3], [0.06917118, 0.35094324]], + ]], + &device, + ); + + let output = model.forward(input); + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/export_weights.py new file mode 100755 index 0000000..04b4234 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/export_weights.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.norm1 = nn.LayerNorm(2) + + def forward(self, x): + x = self.norm1(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "layer_norm.pt") + + x2 = torch.rand(1, 2, 2, 2) + print("Input shape: {}", x2.shape) + print("Input: {}", x2) + output = model(x2) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/mod.rs new file mode 100644 index 0000000..22a35bf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/layer_norm/mod.rs @@ -0,0 +1,82 @@ +use burn::{ + module::Module, + nn::{LayerNorm, LayerNormConfig}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + norm1: LayerNorm, +} + +impl Net { + /// Create a new model. + pub fn init(device: &B::Device) -> Self { + let norm1 = LayerNormConfig::new(2).init(device); + Self { norm1 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.norm1.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + fn layer_norm(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[ + [[0.757_631_6, 0.27931088], [0.40306926, 0.73468447]], + [[0.02928156, 0.799_858_6], [0.39713734, 0.75437194]], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [[0.99991274, -0.999_912_5], [-0.999_818_3, 0.999_818_3]], + [[-0.999_966_2, 0.99996626], [-0.99984336, 0.99984336]], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn layer_norm_full() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/layer_norm/layer_norm.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + layer_norm(model, 1e-3); + } + + #[test] + fn layer_norm_half() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/layer_norm/layer_norm.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + layer_norm(model, 1e-3); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/export_weights.py new file mode 100755 index 0000000..5d76638 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/export_weights.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.fc1 = nn.Linear(2, 3) + self.fc2 = nn.Linear(3, 4, bias=False) + + def forward(self, x): + x = self.fc1(x) + x = F.relu(x) # Add relu so that PyTorch optimizer does not combine fc1 and fc2 + x = self.fc2(x) + + return x + + +class ModelWithBias(nn.Module): + def __init__(self): + super(ModelWithBias, self).__init__() + self.fc1 = nn.Linear(2, 3) + + def forward(self, x): + x = self.fc1(x) + + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + model_with_bias = ModelWithBias().to(torch.device("cpu")) + + torch.save(model.state_dict(), "linear.pt") + torch.save(model_with_bias.state_dict(), "linear_with_bias.pt") + + input = torch.rand(1, 2, 2, 2) + print("Input shape: {}", input.shape) + print("Input: {}", input) + + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + print("Model with bias") + output = model_with_bias(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + + + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/mod.rs new file mode 100644 index 0000000..ec64f1d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/linear/mod.rs @@ -0,0 +1,154 @@ +use burn::{ + module::Module, + nn::{Linear, LinearConfig, Relu}, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + fc1: Linear, + fc2: Linear, + relu: Relu, +} + +impl Net { + /// Create a new model. + pub fn init(device: &B::Device) -> Self { + let fc1 = LinearConfig::new(2, 3).init(device); + let fc2 = LinearConfig::new(3, 4).with_bias(false).init(device); + let relu = Relu; + + Self { fc1, fc2, relu } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.fc1.forward(x); + let x = self.relu.forward(x); + + self.fc2.forward(x) + } +} + +#[derive(Module, Debug)] +struct NetWithBias { + fc1: Linear, +} + +impl NetWithBias { + /// Create a new model. + pub fn init(device: &B::Device) -> Self { + let fc1 = LinearConfig::new(2, 3).init(device); + + Self { fc1 } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.fc1.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + fn linear_test(model: Net, precision: f32) { + let device = Default::default(); + + let input = Tensor::::from_data( + [[ + [[0.63968194, 0.97427773], [0.830_029_9, 0.04443115]], + [[0.024_595_8, 0.25883394], [0.93905586, 0.416_715_5]], + ]], + &device, + ); + + let output = model.forward(input); + let expected = Tensor::::from_data( + [[ + [ + [0.09778349, -0.13756673, 0.04962806, 0.08856435], + [0.03163241, -0.02848549, 0.01437942, 0.11905234], + ], + [ + [0.07628226, -0.10757702, 0.03656857, 0.03824598], + [0.05443089, -0.06904714, 0.02744314, 0.09997337], + ], + ]], + &device, + ); + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(precision)); + } + + #[test] + fn linear_full_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/linear/linear.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + linear_test(model, 1e-7); + } + + #[test] + fn linear_half_precision() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/linear/linear.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + linear_test(model, 1e-4); + } + + #[test] + fn linear_with_bias() { + let device = Default::default(); + + let mut model = NetWithBias::::init(&device); + let mut store = PytorchStore::from_file("tests/linear/linear_with_bias.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::from_data( + [[ + [[0.63968194, 0.97427773], [0.830_029_9, 0.04443115]], + [[0.024_595_8, 0.25883394], [0.93905586, 0.416_715_5]], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [-0.00432095, -1.107_101_2, 0.870_691_4], + [0.024_595_5, -0.954_462_9, 0.48518157], + ], + [ + [0.34315687, -0.757_384_2, 0.548_288], + [-0.06608963, -1.072_072_7, 0.645_800_5], + ], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/export_weights.py new file mode 100755 index 0000000..e65585c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/export_weights.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv2d(2, 2, (2,2)) + + def forward(self, x): + x = self.conv1(x) + return x + + +def main(): + torch.set_printoptions(precision=8) + torch.manual_seed(1) + model = Model().to(torch.device("cpu")) + torch.save(model.state_dict(), "missing_module_field.pt") + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/mod.rs new file mode 100644 index 0000000..53659d3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/missing_module_field/mod.rs @@ -0,0 +1,37 @@ +use burn::{module::Module, nn::conv::Conv2d, tensor::backend::Backend}; + +#[derive(Module, Debug)] +#[allow(unused)] +pub struct Net { + do_not_exist_in_pt: Conv2d, +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::nn::conv::Conv2dConfig; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + impl Net { + pub fn init(device: &B::Device) -> Self { + Self { + do_not_exist_in_pt: Conv2dConfig::new([2, 2], [2, 2]).init(device), + } + } + } + + #[test] + #[should_panic(expected = "do_not_exist_in_pt")] + fn should_fail_if_struct_field_is_missing() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = + PytorchStore::from_file("tests/missing_module_field/missing_module_field.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/export_weights.py new file mode 100755 index 0000000..9a76930 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/export_weights.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + num_layers = 5 # Number of repeated convolutional layers + + # Create a list to store the layers + layers = [] + for _ in range(num_layers): + layers.append(nn.Conv2d(2, 2, kernel_size=3, padding=1, bias=True)) + layers.append(nn.ReLU(inplace=True)) + + # Use nn.Sequential to create a single module from the layers + self.fc = nn.Sequential(*layers) + + def forward(self, x): + x = self.fc(x) + return x + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + torch.save(model.state_dict(), "non_contiguous_indexes.pt") + + input = torch.rand(1, 2, 5, 5) + print("Input shape: {}", input.shape) + print("Input: {}", input) + output = model(input) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/mod.rs new file mode 100644 index 0000000..0ac4ff2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/non_contiguous_indexes/mod.rs @@ -0,0 +1,110 @@ +use burn::{ + module::Module, + nn::{ + PaddingConfig2d, + conv::{Conv2d, Conv2dConfig}, + }, + tensor::{Tensor, activation::relu, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + fc: Vec>, +} + +impl Net { + /// Create a new model with placeholder values. + pub fn init(device: &B::Device) -> Self { + let conv2d_config = Conv2dConfig::new([2, 2], [3, 3]).with_padding(PaddingConfig2d::Same); + // The PyTorch file has 5 Conv2d layers at non-contiguous indices (0, 2, 4, 6, 8) + // in the Sequential (alternating with ReLU layers) + let fc = vec![ + conv2d_config.init(device), + conv2d_config.init(device), + conv2d_config.init(device), + conv2d_config.init(device), + conv2d_config.init(device), + ]; + Net { fc } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + self.fc.iter().fold(x, |x_i, conv| relu(conv.forward(x_i))) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::{Tolerance, ops::FloatElem}; + use burn_store::{ModuleSnapshot, PytorchStore}; + type FT = FloatElem; + + use super::*; + + #[test] + fn non_contiguous_indexes() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = + PytorchStore::from_file("tests/non_contiguous_indexes/non_contiguous_indexes.pt"); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::from_data( + [[ + [ + [ + 0.67890584, + 0.307_537_2, + 0.265_156_2, + 0.528_318_8, + 0.86194897, + ], + [0.14828813, 0.73480314, 0.821_220_7, 0.989_098_6, 0.15003455], + [0.62109494, 0.13028657, 0.926_875_1, 0.30604684, 0.80117637], + [0.514_885_7, 0.46105868, 0.484_046_1, 0.58499724, 0.73569804], + [0.58018994, 0.65252745, 0.05023766, 0.864_268_7, 0.935_932], + ], + [ + [0.913_302_9, 0.869_611_3, 0.139_184_3, 0.314_65, 0.94086266], + [0.11917073, 0.953_610_6, 0.10675198, 0.14779574, 0.744_439], + [0.14075547, 0.38544965, 0.863_745_9, 0.89604443, 0.97287786], + [0.39854127, 0.11136961, 0.99230546, 0.39348692, 0.29428244], + [0.621_886_9, 0.15033776, 0.828_640_1, 0.81336635, 0.10325938], + ], + ]], + &device, + ); + + let output = model.forward(input); + + let expected = Tensor::::from_data( + [[ + [ + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.04485746, 0.03582812, 0.03432692, 0.02892298, 0.013_844_3], + ], + [ + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + [0.00000000, 0.00000000, 0.00000000, 0.00000000, 0.00000000], + ], + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::absolute(1e-7)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/test_mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/test_mod.rs new file mode 100644 index 0000000..57d614c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/test_mod.rs @@ -0,0 +1,22 @@ +mod backend; + +mod batch_norm; +mod boolean; +mod buffer; +mod complex_nested; +mod config; +mod conv1d; +mod conv2d; +mod conv_transpose1d; +mod conv_transpose2d; +mod embedding; +mod enum_module; +mod group_norm; +mod integer; +mod key_remap; +mod key_remap_chained; +mod layer_norm; +mod linear; +mod missing_module_field; +mod non_contiguous_indexes; +mod top_level_key; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/export_weights.py b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/export_weights.py new file mode 100755 index 0000000..4045629 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/export_weights.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv2d(2, 2, (2,2)) + + def forward(self, x): + x = self.conv1(x) + return x + + +def main(): + torch.set_printoptions(precision=8) + torch.manual_seed(1) + model = Model().to(torch.device("cpu")) + torch.save({"my_state_dict": model.state_dict()}, "top_level_key.pt") + +if __name__ == '__main__': + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/mod.rs new file mode 100644 index 0000000..e6abec8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/pytorch-tests/tests/top_level_key/mod.rs @@ -0,0 +1,48 @@ +use burn::{module::Module, nn::conv::Conv2d, tensor::backend::Backend}; + +#[derive(Module, Debug)] +#[allow(unused)] +pub struct Net { + conv1: Conv2d, +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::nn::conv::Conv2dConfig; + use burn_store::{ModuleSnapshot, PytorchStore}; + + use super::*; + + impl Net { + pub fn init(device: &B::Device) -> Self { + Self { + conv1: Conv2dConfig::new([2, 2], [2, 2]).init(device), + } + } + } + + #[test] + #[should_panic] + fn should_fail_if_not_found() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/top_level_key/top_level_key.pt"); + model + .load_from(&mut store) + .expect("Should decode state successfully"); + } + + #[test] + fn should_load() { + let device = Default::default(); + let mut model = Net::::init(&device); + let mut store = PytorchStore::from_file("tests/top_level_key/top_level_key.pt") + .with_top_level_key("my_state_dict"); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/Cargo.toml new file mode 100644 index 0000000..2af9990 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "safetensors-tests" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dev-dependencies] +burn = { path = "../../burn" } +burn-ndarray = { path = "../../burn-ndarray" } +burn-autodiff = { path = "../../burn-autodiff" } +burn-store = { path = "../", features = ["std", "safetensors"] } +serde = { workspace = true } +float-cmp = { workspace = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/src/lib.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/backend.rs new file mode 100644 index 0000000..a264f96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/backend.rs @@ -0,0 +1 @@ +pub type TestBackend = burn_ndarray::NdArray; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/mod.rs new file mode 100644 index 0000000..5e3298a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/mod.rs @@ -0,0 +1,92 @@ +use burn::{ + module::Module, + nn::{ + BatchNorm, BatchNormConfig, Linear, LinearConfig, PaddingConfig2d, Relu, + conv::{Conv2d, Conv2dConfig}, + }, + tensor::{Tensor, backend::Backend}, +}; + +#[derive(Module, Debug)] +pub struct Net { + conv1: Conv2d, + norm1: BatchNorm, + fc1: Linear, + relu: Relu, +} + +impl Net { + pub fn new(device: &B::Device) -> Self { + Self { + conv1: Conv2dConfig::new([3, 4], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .init(device), + norm1: BatchNormConfig::new(4).init(device), + fc1: LinearConfig::new(4 * 8 * 8, 16).init(device), + relu: Relu::new(), + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + let x = self.norm1.forward(x); + let x = self.relu.forward(x); + // Flatten all dimensions except the batch dimension + let x = x.flatten(1, 3); + self.fc1.forward(x) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::TestBackend; + + use burn::tensor::Tolerance; + use burn_store::{ModuleSnapshot, PyTorchToBurnAdapter, SafetensorsStore}; + + use super::*; + + #[test] + fn multi_layer_model() { + let device = Default::default(); + let mut model = Net::::new(&device); + let mut store = SafetensorsStore::from_file("tests/multi_layer/multi_layer.safetensors") + .with_from_adapter(PyTorchToBurnAdapter); + + model + .load_from(&mut store) + .expect("Should decode state successfully"); + + let input = Tensor::::ones([1, 3, 8, 8], &device); + + let output = model.forward(input); + + // Note: Expected values should be updated based on the actual output from the PyTorch model + let expected = Tensor::::from_data( + [[ + 0.04971555, + -0.16849735, + 0.05182848, + -0.18032673, + 0.23138367, + 0.05041867, + 0.13005908, + -0.32202929, + -0.07915690, + -0.03232457, + -0.19790289, + -0.17476529, + -0.19627589, + -0.21757686, + -0.31376451, + 0.08377837, + ]], + &device, + ); + + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/multi_layer.py b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/multi_layer.py new file mode 100755 index 0000000..487462d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/multi_layer/multi_layer.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +import torch +import torch.nn as nn +import torch.nn.functional as F +from safetensors.torch import save_file + + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv2d(3, 4, kernel_size=3, padding=1) + self.norm1 = nn.BatchNorm2d(4) + self.flatten = nn.Flatten() + self.fc1 = nn.Linear(4 * 8 * 8, 16) # Changed for smaller input size + + def forward(self, x): + x = self.conv1(x) + x = self.norm1(x) + x = F.relu(x) + x = self.flatten(x) + x = self.fc1(x) + return x + + +def main(): + + torch.set_printoptions(precision=8) + torch.manual_seed(1) + + model = Model().to(torch.device("cpu")) + + # Use a smaller input size + # 1 batch, 3 channels (RGB), 8x8 image (small input) + x1 = torch.ones(1, 3, 8, 8) + _ = model(x1) + model.eval() # Set to eval mode to freeze running stats + # Save the model to safetensors after the first forward + save_file(model.state_dict(), "multi_layer.safetensors") + + x2 = torch.ones(1, 3, 8, 8) + print("Input shape: {}", x2.shape) + output = model(x2) + print("Output: {}", output) + print("Output Shape: {}", output.shape) + + +if __name__ == "__main__": + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/test_mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/test_mod.rs new file mode 100644 index 0000000..d1b86bf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/safetensors-tests/tests/test_mod.rs @@ -0,0 +1,3 @@ +mod backend; + +mod multi_layer; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/adapter.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/adapter.rs new file mode 100644 index 0000000..8efd91f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/adapter.rs @@ -0,0 +1,663 @@ +//! Module adapters for transforming tensors between different formats +//! +//! This module provides adapters that handle differences between PyTorch and Burn: +//! - Linear layer weight transposition +//! - Normalization parameter naming (weight/bias vs gamma/beta) + +use crate::TensorSnapshot; + +use alloc::boxed::Box; +use alloc::rc::Rc; +use alloc::string::String; +use alloc::string::ToString; +use alloc::vec; + +use burn_tensor::TensorData; + +// Module type names as they appear in the container_type field +// These come from the Module derive macro which uses stringify! on the struct name +// Format: "Struct:TypeName" for user-defined structs +mod module_names { + // The actual string constants that match what the Module derive macro produces + pub const LINEAR: &str = "Struct:Linear"; + pub const BATCH_NORM: &str = "Struct:BatchNorm"; + pub const LAYER_NORM: &str = "Struct:LayerNorm"; + pub const GROUP_NORM: &str = "Struct:GroupNorm"; +} + +/// Trait for adapting tensor snapshots between different module formats +pub trait ModuleAdapter: Send + Sync { + /// Adapt a tensor snapshot based on its container type and parameter name + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot; + + /// Get alternative parameter name to try during matching + /// + /// When looking for a parameter in a module, this method provides an alternative + /// name to try if the direct name doesn't match. This enables matching parameters + /// with different naming conventions (e.g., PyTorch's "weight" vs Burn's "gamma"). + /// + /// # Arguments + /// * `param_name` - The parameter name we're looking for + /// * `container_type` - The type of container module (e.g., "BatchNorm") + /// + /// # Returns + /// Alternative parameter name to try, or None if no alternative exists + fn get_alternative_param_name( + &self, + _param_name: &str, + _container_type: &str, + ) -> Option { + None + } + + /// Clone the adapter into a boxed trait object + fn clone_box(&self) -> Box; + + /// Chain adapters together, applying `self` first and then `next`. + /// + /// This is useful when multiple transformations are required when importing model weights + /// (e.g. PyTorch -> Burn layout conversion, then dtype casting, then custom remapping). + /// + /// The semantics follow a simple pipeline: + /// - `adapt`: `next.adapt(&self.adapt(snapshot))` + /// - `get_alternative_param_name`: try `self` first; if it returns an alternative name, + /// try `next` with that name, otherwise return the first alternative name. + fn chain(self, next: A) -> ChainAdapter + where + Self: Sized + 'static, + A: ModuleAdapter + 'static, + { + ChainAdapter::new(self, next) + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +/// Adapter that applies two adapters in sequence. +/// +/// This allows composing smaller adapters instead of creating one large monolithic adapter. +#[derive(Clone)] +pub struct ChainAdapter { + first: Box, + second: Box, +} + +impl ChainAdapter { + /// Create a new adapter chain. + pub fn new(first: A, second: B) -> Self + where + A: ModuleAdapter + 'static, + B: ModuleAdapter + 'static, + { + Self { + first: Box::new(first), + second: Box::new(second), + } + } +} + +impl ModuleAdapter for ChainAdapter { + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot { + let snapshot = self.first.adapt(snapshot); + self.second.adapt(&snapshot) + } + + fn get_alternative_param_name(&self, param_name: &str, container_type: &str) -> Option { + if let Some(name) = self + .first + .get_alternative_param_name(param_name, container_type) + { + self.second + .get_alternative_param_name(&name, container_type) + .or(Some(name)) + } else { + self.second + .get_alternative_param_name(param_name, container_type) + } + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Identity adapter that passes tensors through unchanged +#[derive(Debug, Clone, Default)] +pub struct IdentityAdapter; + +impl ModuleAdapter for IdentityAdapter { + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot { + snapshot.clone() + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Adapter for converting from PyTorch format to Burn format +/// +/// Handles: +/// - Linear layer weight transposition (PyTorch: [out, in] → Burn: [in, out]) +/// - Normalization parameter renaming (weight → gamma, bias → beta) +#[derive(Debug, Clone, Default)] +pub struct PyTorchToBurnAdapter; + +impl ModuleAdapter for PyTorchToBurnAdapter { + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot { + adapt_pytorch_tensor(snapshot, PyTorchConversionDirection::PyTorchToBurn) + } + + fn get_alternative_param_name(&self, param_name: &str, container_type: &str) -> Option { + // For PyTorch->Burn: When looking for Burn names (gamma/beta), try PyTorch names (weight/bias) + if is_normalization_layer(container_type) { + burn_norm_param_to_pytorch(param_name).map(|s| s.to_string()) + } else { + None + } + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Adapter for converting from Burn format to PyTorch format +/// +/// Handles: +/// - Linear layer weight transposition (Burn: [in, out] → PyTorch: [out, in]) +/// - Normalization parameter renaming (gamma → weight, beta → bias) +#[derive(Debug, Clone, Default)] +pub struct BurnToPyTorchAdapter; + +impl ModuleAdapter for BurnToPyTorchAdapter { + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot { + adapt_pytorch_tensor(snapshot, PyTorchConversionDirection::BurnToPyTorch) + } + + fn get_alternative_param_name(&self, param_name: &str, container_type: &str) -> Option { + // For Burn->PyTorch: When looking for PyTorch names (weight/bias), try Burn names (gamma/beta) + if is_normalization_layer(container_type) { + pytorch_norm_param_to_burn(param_name).map(|s| s.to_string()) + } else { + None + } + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Direction of PyTorch conversion for parameter naming +#[derive(Debug, Clone, Copy)] +enum PyTorchConversionDirection { + PyTorchToBurn, + BurnToPyTorch, +} + +/// Check if container type is a normalization layer +fn is_normalization_layer(container_type: &str) -> bool { + matches!( + container_type, + module_names::BATCH_NORM | module_names::LAYER_NORM | module_names::GROUP_NORM + ) +} + +/// Map PyTorch normalization parameter name to Burn +fn pytorch_norm_param_to_burn(param_name: &str) -> Option<&'static str> { + match param_name { + "weight" => Some("gamma"), + "bias" => Some("beta"), + _ => None, + } +} + +/// Map Burn normalization parameter name to PyTorch +fn burn_norm_param_to_pytorch(param_name: &str) -> Option<&'static str> { + match param_name { + "gamma" => Some("weight"), + "beta" => Some("bias"), + _ => None, + } +} + +/// Core tensor adaptation logic for PyTorch format conversions +fn adapt_pytorch_tensor( + snapshot: &TensorSnapshot, + direction: PyTorchConversionDirection, +) -> TensorSnapshot { + // Extract path and parameter name + let (path_stack, param_name) = match get_path_and_param(snapshot) { + Some(result) => result, + None => return snapshot.clone(), + }; + + // Get module type for matching (ignores Vec/Array wrappers) + let module_type = match snapshot.module_type() { + Some(mt) => mt, + None => return snapshot.clone(), // No user-defined module found + }; + + // Linear: transpose weight (bidirectional - same operation both ways) + if module_type == module_names::LINEAR && param_name == "weight" && snapshot.shape.len() == 2 { + return transpose_2d_tensor(snapshot); + } + + // Normalization layers: rename parameters based on direction + if is_normalization_layer(&module_type) { + let new_name = match direction { + PyTorchConversionDirection::PyTorchToBurn => pytorch_norm_param_to_burn(param_name), + PyTorchConversionDirection::BurnToPyTorch => burn_norm_param_to_pytorch(param_name), + }; + + if let Some(new_name) = new_name { + return rename_parameter(snapshot, path_stack, new_name); + } + } + + snapshot.clone() +} + +/// Extract path stack and parameter name from snapshot +fn get_path_and_param(snapshot: &TensorSnapshot) -> Option<(&[String], &str)> { + let path_stack = snapshot.path_stack.as_ref()?; + let param_name = path_stack.last()?.as_str(); + Some((path_stack.as_slice(), param_name)) +} + +/// Rename a parameter in the snapshot +fn rename_parameter( + snapshot: &TensorSnapshot, + path_stack: &[String], + new_name: &str, +) -> TensorSnapshot { + let mut new_path = path_stack.to_vec(); + *new_path.last_mut().unwrap() = new_name.to_string(); + + TensorSnapshot::from_closure( + snapshot.clone_data_fn(), + snapshot.dtype, + snapshot.shape.clone(), + new_path, + snapshot.container_stack.clone().unwrap_or_default(), + snapshot.tensor_id.unwrap_or_default(), + ) +} + +/// Transpose a 2D tensor +fn transpose_2d_tensor(snapshot: &TensorSnapshot) -> TensorSnapshot { + if snapshot.shape.len() != 2 { + return snapshot.clone(); + } + + let original_data_fn = snapshot.clone_data_fn(); + let dtype = snapshot.dtype; + let transposed_shape = vec![snapshot.shape[1], snapshot.shape[0]]; + + // Create a lazy closure that transposes when called + let transposed_data_fn = Rc::new(move || { + let data = original_data_fn()?; + Ok(transpose_tensor_data(data)) + }); + + TensorSnapshot::from_closure( + transposed_data_fn, + dtype, + transposed_shape, + snapshot.path_stack.clone().unwrap_or_default(), + snapshot.container_stack.clone().unwrap_or_default(), + snapshot.tensor_id.unwrap_or_default(), + ) +} + +/// Transpose tensor data (assumes 2D shape is already validated) +fn transpose_tensor_data(data: TensorData) -> TensorData { + let shape = &data.shape; + let rows = shape[0]; + let cols = shape[1]; + let transposed_shape = vec![cols, rows]; + + // Get the raw bytes and element size + let bytes = data.as_bytes(); + let element_size = data.dtype.size(); + + // Create a new buffer for transposed data + let mut transposed_bytes = vec![0u8; bytes.len()]; + + // Transpose at the byte level - works for any data type + for i in 0..rows { + for j in 0..cols { + let src_idx = (i * cols + j) * element_size; + let dst_idx = (j * rows + i) * element_size; + + // Copy the bytes for this element + transposed_bytes[dst_idx..dst_idx + element_size] + .copy_from_slice(&bytes[src_idx..src_idx + element_size]); + } + } + + // Create new TensorData from transposed bytes + TensorData::from_bytes_vec(transposed_bytes, transposed_shape, data.dtype) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::rc::Rc; + use alloc::sync::Arc; + use burn_tensor::{DType, TensorData}; + use core::sync::atomic::{AtomicUsize, Ordering}; + + #[test] + fn test_module_names_match_burn_nn() { + // If these types are renamed or moved in `burn-nn`, this test will fail to compile. + // This use statement replicates the previous check/alarm system. + #[allow(unused_imports)] + use burn_nn::{BatchNorm, GroupNorm, LayerNorm, Linear}; + + // These assert statements work as extra checks that should remind maintainers more + // clearly that the hardcoded strings needs get updated. + assert_eq!(module_names::LINEAR, "Struct:Linear"); + assert_eq!(module_names::BATCH_NORM, "Struct:BatchNorm"); + assert_eq!(module_names::LAYER_NORM, "Struct:LayerNorm"); + assert_eq!(module_names::GROUP_NORM, "Struct:GroupNorm"); + } + + fn create_test_snapshot(path: &str, shape: Vec, container_type: &str) -> TensorSnapshot { + let path_parts: Vec = path.split('.').map(|s| s.to_string()).collect(); + let values = vec![1.0f32; shape.iter().product()]; + let data = TensorData::new(values, shape.clone()); + + TensorSnapshot::from_closure( + Rc::new(move || Ok(data.clone())), + DType::F32, + shape, + path_parts, + vec![container_type.to_string()], + burn_core::module::ParamId::new(), + ) + } + + #[test] + fn test_pytorch_to_burn_linear_weight() { + let adapter = PyTorchToBurnAdapter; + + // Linear layer weight should be transposed + let snapshot = create_test_snapshot("fc.weight", vec![10, 5], module_names::LINEAR); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.shape, vec![5, 10]); + + // Linear layer bias should not be transposed + let snapshot = create_test_snapshot("fc.bias", vec![10], module_names::LINEAR); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.shape, vec![10]); + } + + #[test] + fn test_pytorch_to_burn_norm_params() { + let adapter = PyTorchToBurnAdapter; + + // BatchNorm weight -> gamma + let snapshot = create_test_snapshot("norm.weight", vec![10], module_names::BATCH_NORM); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.full_path(), "norm.gamma"); + + // BatchNorm bias -> beta + let snapshot = create_test_snapshot("norm.bias", vec![10], module_names::BATCH_NORM); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.full_path(), "norm.beta"); + } + + #[test] + fn test_burn_to_pytorch_linear_weight() { + let adapter = BurnToPyTorchAdapter; + + // Linear layer weight should be transposed + let snapshot = create_test_snapshot("fc.weight", vec![5, 10], module_names::LINEAR); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.shape, vec![10, 5]); + } + + #[test] + fn test_burn_to_pytorch_norm_params() { + let adapter = BurnToPyTorchAdapter; + + // BatchNorm gamma -> weight + let snapshot = create_test_snapshot("norm.gamma", vec![10], module_names::BATCH_NORM); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.full_path(), "norm.weight"); + + // BatchNorm beta -> bias + let snapshot = create_test_snapshot("norm.beta", vec![10], module_names::BATCH_NORM); + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.full_path(), "norm.bias"); + } + + #[test] + fn test_transpose_different_dtypes() { + // Test that transpose works for different data types + + // Test with F32 + let f32_data = TensorData::new(vec![1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3]); + let transposed = transpose_tensor_data(f32_data); + assert_eq!(transposed.shape, vec![3, 2]); + let values = transposed.to_vec::().unwrap(); + assert_eq!(values, vec![1.0, 4.0, 2.0, 5.0, 3.0, 6.0]); + + // Test with I32 + let i32_data = TensorData::new(vec![1i32, 2, 3, 4, 5, 6], vec![2, 3]); + let transposed = transpose_tensor_data(i32_data); + assert_eq!(transposed.shape, vec![3, 2]); + let values = transposed.to_vec::().unwrap(); + assert_eq!(values, vec![1, 4, 2, 5, 3, 6]); + + // Test with F64 + let f64_data = TensorData::new(vec![1.0f64, 2.0, 3.0, 4.0], vec![2, 2]); + let transposed = transpose_tensor_data(f64_data); + assert_eq!(transposed.shape, vec![2, 2]); + let values = transposed.to_vec::().unwrap(); + assert_eq!(values, vec![1.0, 3.0, 2.0, 4.0]); + } + + #[test] + fn test_no_container_info() { + let adapter = PyTorchToBurnAdapter; + + // Without container info, adapter returns unchanged for non-norm parameters + let mut snapshot = create_test_snapshot("fc.weight", vec![10, 5], module_names::LINEAR); + snapshot.container_stack = None; + + // Without container info, no transformation occurs for linear layers + let adapted = adapter.adapt(&snapshot); + assert_eq!(adapted.shape, vec![10, 5]); // No transposition without container info + + // Test a non-linear, non-norm parameter - should pass through unchanged + let mut snapshot2 = create_test_snapshot("other.weight", vec![10, 5], "Struct:Other"); + snapshot2.container_stack = None; + let adapted2 = adapter.adapt(&snapshot2); + assert_eq!(adapted2.shape, vec![10, 5]); // No transposition + } + + #[derive(Clone)] + struct RenameParamAdapter { + from: &'static str, + to: &'static str, + called: Arc, + } + + impl ModuleAdapter for RenameParamAdapter { + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot { + self.called.fetch_add(1, Ordering::Relaxed); + + let path_stack = match snapshot.path_stack.as_ref() { + Some(stack) => stack, + None => return snapshot.clone(), + }; + let param = match path_stack.last() { + Some(p) => p.as_str(), + None => return snapshot.clone(), + }; + if param != self.from { + return snapshot.clone(); + } + + let mut new_path = path_stack.to_vec(); + *new_path.last_mut().unwrap() = self.to.to_string(); + + TensorSnapshot::from_closure( + snapshot.clone_data_fn(), + snapshot.dtype, + snapshot.shape.clone(), + new_path, + snapshot.container_stack.clone().unwrap_or_default(), + snapshot.tensor_id.unwrap_or_default(), + ) + } + + fn get_alternative_param_name( + &self, + _param_name: &str, + _container_type: &str, + ) -> Option { + None + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone)] + struct AltNameAdapter { + from: &'static str, + to: &'static str, + called: Arc, + } + + impl ModuleAdapter for AltNameAdapter { + fn adapt(&self, snapshot: &TensorSnapshot) -> TensorSnapshot { + TensorSnapshot::from_closure( + snapshot.clone_data_fn(), + snapshot.dtype, + snapshot.shape.clone(), + snapshot.path_stack.clone().unwrap_or_default(), + snapshot.container_stack.clone().unwrap_or_default(), + snapshot.tensor_id.unwrap_or_default(), + ) + } + + fn get_alternative_param_name( + &self, + param_name: &str, + _container_type: &str, + ) -> Option { + self.called.fetch_add(1, Ordering::Relaxed); + if param_name == self.from { + Some(self.to.to_string()) + } else { + None + } + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + } + + #[test] + fn test_chain_adapter_pipes_adapt() { + let called1 = Arc::new(AtomicUsize::new(0)); + let called2 = Arc::new(AtomicUsize::new(0)); + + let a = RenameParamAdapter { + from: "weight", + to: "a", + called: called1.clone(), + }; + let b = RenameParamAdapter { + from: "a", + to: "b", + called: called2.clone(), + }; + + let chain = a.chain(b); + let snapshot = create_test_snapshot("fc.weight", vec![2, 2], module_names::LINEAR); + let adapted = chain.adapt(&snapshot); + + assert_eq!(adapted.full_path(), "fc.b"); + assert_eq!(called1.load(Ordering::Relaxed), 1); + assert_eq!(called2.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_chain_adapter_alternative_name_pipes_and_fallbacks() { + let called1 = Arc::new(AtomicUsize::new(0)); + let called2 = Arc::new(AtomicUsize::new(0)); + + let a = AltNameAdapter { + from: "gamma", + to: "weight", + called: called1.clone(), + }; + let b = AltNameAdapter { + from: "weight", + to: "scale", + called: called2.clone(), + }; + + let chain = a.chain(b); + let alt = chain.get_alternative_param_name("gamma", module_names::LAYER_NORM); + assert_eq!(alt.as_deref(), Some("scale")); + assert_eq!(called1.load(Ordering::Relaxed), 1); + assert_eq!(called2.load(Ordering::Relaxed), 1); + + // If the second adapter doesn't have a mapping for the first alternative, + // fall back to the first alternative name. + let called1 = Arc::new(AtomicUsize::new(0)); + let called2 = Arc::new(AtomicUsize::new(0)); + let a = AltNameAdapter { + from: "gamma", + to: "weight", + called: called1.clone(), + }; + let b = AltNameAdapter { + from: "something-else", + to: "unused", + called: called2.clone(), + }; + let chain = a.chain(b); + let alt = chain.get_alternative_param_name("gamma", module_names::LAYER_NORM); + assert_eq!(alt.as_deref(), Some("weight")); + assert_eq!(called1.load(Ordering::Relaxed), 1); + assert_eq!(called2.load(Ordering::Relaxed), 1); + + // If the first adapter doesn't provide an alternative, try the second with the original name. + let called1 = Arc::new(AtomicUsize::new(0)); + let called2 = Arc::new(AtomicUsize::new(0)); + let a = AltNameAdapter { + from: "something-else", + to: "unused", + called: called1.clone(), + }; + let b = AltNameAdapter { + from: "gamma", + to: "weight", + called: called2.clone(), + }; + let chain = a.chain(b); + let alt = chain.get_alternative_param_name("gamma", module_names::LAYER_NORM); + assert_eq!(alt.as_deref(), Some("weight")); + assert_eq!(called1.load(Ordering::Relaxed), 1); + assert_eq!(called2.load(Ordering::Relaxed), 1); + + // clone_box must preserve behavior. + let boxed = chain.clone_box(); + let alt = boxed.get_alternative_param_name("gamma", module_names::LAYER_NORM); + assert_eq!(alt.as_deref(), Some("weight")); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/applier.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/applier.rs new file mode 100644 index 0000000..b0a929a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/applier.rs @@ -0,0 +1,608 @@ +//! Applier that correctly applies tensor snapshots with adapter support + +use alloc::boxed::Box; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; + +use hashbrown::{HashMap, HashSet}; + +use burn_core::module::{ModuleMapper, Param}; +use burn_tensor::{Bool, Int, Shape, Tensor, backend::Backend}; + +use crate::apply_result::{ApplyError, ApplyResult}; +use crate::{ModuleAdapter, PathFilter, TensorSnapshot}; + +/// Applier that applies tensor snapshots to module parameters +/// with proper adapter support using container type information +pub struct Applier { + /// Map of tensor paths to their snapshots + snapshots: HashMap, + /// Current path in the module hierarchy + path_stack: Vec, + /// Current container type stack in the module hierarchy + container_stack: Vec, + /// Optional filter for selective application + filter: Option, + /// Optional adapter to transform tensors based on container types + adapter: Option>, + /// Successfully applied tensor paths + applied: Vec, + /// Skipped tensor paths + skipped: HashSet, + /// Errors encountered during application + errors: Vec, + /// Track visited paths with their container stacks (in dot notation) to find missing tensors + visited_paths: HashMap, + /// Skip enum variant names when matching paths + /// When true, "feature.BaseConv.weight" will also try to match "feature.weight" + skip_enum_variants: bool, + /// Phantom data for backend type + _backend: core::marker::PhantomData, +} + +impl Applier { + /// Create a new applier with snapshots, optional filter, and optional adapter + /// + /// # Arguments + /// + /// * `views` - A vector of TensorSnapshot objects to apply + /// * `filter` - An optional [`PathFilter`] to determine which tensors to apply. + /// When `None`, all available tensors are applied. + /// * `adapter` - Optional adapter to transform tensors based on container types + /// * `skip_enum_variants` - Skip enum variant names when matching paths + pub fn new( + views: Vec, + filter: Option, + adapter: Option>, + skip_enum_variants: bool, + ) -> Self { + let views_map: HashMap = views + .into_iter() + .map(|view| (view.full_path(), view)) + .collect(); + + Self { + snapshots: views_map, + path_stack: Vec::new(), + container_stack: Vec::new(), + filter, + adapter, + applied: Vec::new(), + skipped: HashSet::new(), + errors: Vec::new(), + visited_paths: HashMap::new(), + skip_enum_variants, + _backend: core::marker::PhantomData, + } + } + + /// Get the current path in the module hierarchy + fn current_path(&self) -> String { + self.path_stack.join(".") + } + + /// Get the current module type (last Struct/Enum in container stack) + fn current_module_type(&self) -> Option<&str> { + self.container_stack + .iter() + .rev() + .find(|ct| ct.starts_with("Struct:") || ct.starts_with("Enum:")) + .map(|s| s.as_str()) + } + + /// Check if a tensor should be applied based on filter + fn should_apply(&self) -> bool { + match &self.filter { + None => true, + Some(f) => f.matches_with_container_path(&self.path_stack, &self.container_stack), + } + } + + /// Convert the applier into a result + pub fn into_result(self) -> ApplyResult { + let mut unused: Vec = self + .snapshots + .keys() + .filter(|path| !self.visited_paths.contains_key(*path) && !self.skipped.contains(*path)) + .cloned() + .collect(); + // Sort for stable output order + unused.sort(); + + // Create a set of successfully applied paths for efficient lookup + let applied_set: HashSet = self.applied.iter().cloned().collect(); + + // Extract paths that have errors - these are not "missing", they were found but had issues + let errored_paths: HashSet = self + .errors + .iter() + .map(|e| match e { + ApplyError::ShapeMismatch { path, .. } => path.clone(), + ApplyError::DTypeMismatch { path, .. } => path.clone(), + ApplyError::AdapterError { path, .. } => path.clone(), + ApplyError::LoadError { path, .. } => path.clone(), + }) + .collect(); + + // A path is missing if it was visited but not successfully applied, not skipped, and didn't have an error + // Store both the path and its container stack (in dot notation) + let mut missing: Vec<(String, String)> = self + .visited_paths + .into_iter() + .filter(|(p, _)| { + !applied_set.contains(p) && !self.skipped.contains(p) && !errored_paths.contains(p) + }) + .collect(); + // Sort for stable output order (by path) + missing.sort_by(|a, b| a.0.cmp(&b.0)); + + // Convert skipped HashSet to sorted Vec for stable output + let mut skipped: Vec = self.skipped.into_iter().collect(); + skipped.sort(); + + ApplyResult { + applied: self.applied, + skipped, + missing, + unused, + errors: self.errors, + } + } + + /// Apply a tensor snapshot with shape validation and optional adapter transformation + /// Returns None if snapshot not found, filtered, or validation fails + fn apply_tensor( + &mut self, + target_device: &B::Device, + target_shape: Shape, + ) -> Option> + where + K: burn_tensor::TensorKind, + K: burn_tensor::BasicOps, + { + let path = self.current_path(); + let container_stack_str = self.container_stack.join("."); + self.visited_paths.insert(path.clone(), container_stack_str); + + // Try to get snapshot with original path first + let mut snapshot = self.snapshots.get(&path).cloned(); + + // If not found and we have an adapter, try alternative parameter names + if snapshot.is_none() + && let Some(ref adapter) = self.adapter + && let Some(module_type) = self.current_module_type() + { + // Get alternative name based on current module type (user-defined module only) + let param_name = self.path_stack.last()?; + + if let Some(alt_name) = adapter.get_alternative_param_name(param_name, module_type) { + // Build alternative path with parameter name substitution + let mut alt_path_stack = self.path_stack.clone(); + *alt_path_stack.last_mut().unwrap() = alt_name.clone(); + let alt_path = alt_path_stack.join("."); + + // Try to get snapshot with alternative name + snapshot = self.snapshots.get(&alt_path).cloned(); + + // Don't mark the alternative path as visited - only the original Burn path + // should be tracked. The alternative path is just for lookup. + } + } + + let mut snapshot = snapshot?; + + // Apply adapter transformation using current container_stack context (for data transformation like transpose) + if let Some(ref adapter) = self.adapter { + // Create a temporary snapshot with current context for adaptation + let snapshot_with_context = TensorSnapshot::from_closure( + snapshot.clone_data_fn(), + snapshot.dtype, + snapshot.shape.clone(), + self.path_stack.clone(), + self.container_stack.clone(), + snapshot.tensor_id.unwrap_or_default(), + ); + + // Transform using adapter (handles transpose) + snapshot = adapter.adapt(&snapshot_with_context); + } + + // Check if we should apply based on filter + if !self.should_apply() { + self.skipped.insert(path.clone()); + return None; + } + + // Load tensor data + let data = match snapshot.to_data() { + Ok(data) => data, + Err(e) => { + self.errors.push(ApplyError::LoadError { + path: path.clone(), + message: format!("Failed to load tensor data: {:?}", e), + }); + return None; // Signal caller to fall back to initialization + } + }; + + // Validate shape + if data.shape != *target_shape { + self.errors.push(ApplyError::ShapeMismatch { + path: path.clone(), + expected: target_shape.to_vec(), + found: data.shape.clone(), + }); + return None; // Signal caller to fall back to initialization + } + + self.applied.push(path); + Some(Tensor::from_data_dtype(data, target_device, snapshot.dtype)) + } +} + +impl ModuleMapper for Applier { + fn enter_module(&mut self, name: &str, container_type: &str) { + // Always track the container type for proper module type detection + self.container_stack.push(container_type.to_string()); + + // Only add to path if it's not an enum variant (when skip_enum_variants is enabled) + // This ensures paths are built without enum variant names from the start + if !self.skip_enum_variants || !container_type.starts_with("Enum:") { + self.path_stack.push(name.to_string()); + } + } + + fn exit_module(&mut self, _name: &str, container_type: &str) { + self.container_stack.pop(); + + // Only pop from path if we added it (not an enum variant when skip_enum_variants is enabled) + if !self.skip_enum_variants || !container_type.starts_with("Enum:") { + self.path_stack.pop(); + } + } + + fn map_float(&mut self, param: Param>) -> Param> { + let param_id = param.id; + let target_device = param.lazy_device(); + let target_shape = param.lazy_shape(); + + // Try to apply snapshot with shape validation + match self.apply_tensor(&target_device, target_shape) { + Some(tensor) => { + // We have a tensor to apply - load it + param.transform_for_load(tensor, param_id) + } + None => { + // No snapshot, filtered, or validation failed - return param unchanged + param + } + } + } + + fn map_int( + &mut self, + param: Param>, + ) -> Param> { + let param_id = param.id; + let target_device = param.lazy_device(); + let target_shape = param.lazy_shape(); + + // Try to apply snapshot with shape validation + match self.apply_tensor(&target_device, target_shape) { + Some(tensor) => { + // We have a tensor to apply - load it + param.transform_for_load(tensor, param_id) + } + None => { + // No snapshot, filtered, or validation failed - return param unchanged + param + } + } + } + + fn map_bool( + &mut self, + param: Param>, + ) -> Param> { + let param_id = param.id; + let target_device = param.lazy_device(); + let target_shape = param.lazy_shape(); + + // Try to apply snapshot with shape validation + match self.apply_tensor(&target_device, target_shape) { + Some(tensor) => { + // We have a tensor to apply - load it + param.transform_for_load(tensor, param_id) + } + None => { + // No snapshot, filtered, or validation failed - return param unchanged + param + } + } + } +} + +#[cfg(all(test, feature = "std", target_has_atomic = "ptr"))] +mod tests { + use super::*; + use burn_core::module::{ModuleMapper, Param, ParamId}; + use burn_tensor::{DType, Tensor, TensorData}; + + type TestBackend = burn_ndarray::NdArray; + + #[test] + fn root_level_parameters() { + let device = Default::default(); + + // Create root-level parameters (not inside any module) + let weight = Param::>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let bias = Param::>::from_data([5.0, 6.0], &device); + + // Create snapshots with root-level paths (single-element path, no nested modules) + let weight_snapshot = crate::TensorSnapshot::from_data( + weight.val().to_data(), + vec!["weight".to_string()], // root-level parameter name + vec![], // no container + ParamId::new(), + ); + + let bias_snapshot = crate::TensorSnapshot::from_data( + bias.val().to_data(), + vec!["bias".to_string()], // root-level parameter name + vec![], // no container + ParamId::new(), + ); + + // Create applier with root-level snapshots + let mut applier = + Applier::::new(vec![weight_snapshot, bias_snapshot], None, None, false); + + // Create new params to load into + let weight_target = Param::initialized( + ParamId::new(), + Tensor::::zeros([2, 2], &device), + ); + let bias_target = Param::initialized( + ParamId::new(), + Tensor::::zeros([2], &device), + ); + + // Apply using the ModuleMapper interface - simulate module traversal + // Enter "weight" path (as if we're visiting a field named "weight") + applier.enter_module("weight", ""); + let weight_loaded = applier.map_float(weight_target); + applier.exit_module("weight", ""); + + // Enter "bias" path (as if we're visiting a field named "bias") + applier.enter_module("bias", ""); + let bias_loaded = applier.map_float(bias_target); + applier.exit_module("bias", ""); + + // Verify values were loaded + let weight_data = weight_loaded.val().to_data().to_vec::().unwrap(); + let bias_data = bias_loaded.val().to_data().to_vec::().unwrap(); + + assert_eq!(weight_data, vec![1.0, 2.0, 3.0, 4.0]); + assert_eq!(bias_data, vec![5.0, 6.0]); + + // Verify applier result + let result = applier.into_result(); + assert_eq!(result.applied.len(), 2); + assert_eq!(result.errors.len(), 0); + } + + /// Test that the applier preserves dtype when loading tensor data. + /// This is a regression test for the bug where F16 tensors were being + /// loaded as F32 because `Tensor::from_data` was used instead of + /// `Tensor::from_data_dtype`. + #[test] + fn dtype_preservation_f64() { + // Use NdArray backend to properly test F64 dtype preservation + type TestBackendF64 = burn_ndarray::NdArray; + let device = Default::default(); + + // Create TensorData with F64 dtype explicitly + let f64_data = TensorData::new(vec![1.0f64, 2.0, 3.0, 4.0], [2, 2]); + assert_eq!(f64_data.dtype, DType::F64, "Test setup: data should be F64"); + + // Create a snapshot with F64 data + let snapshot = crate::TensorSnapshot::from_data( + f64_data.clone(), + vec!["weight".to_string()], + vec![], + ParamId::new(), + ); + assert_eq!( + snapshot.dtype, + DType::F64, + "Snapshot should preserve F64 dtype" + ); + + // Create applier with the F64 snapshot + let mut applier = Applier::::new(vec![snapshot], None, None, false); + + // Create target parameter + let target = Param::initialized( + ParamId::new(), + Tensor::::zeros([2, 2], &device), + ); + + // Apply the snapshot + applier.enter_module("weight", ""); + let loaded = applier.map_float(target); + applier.exit_module("weight", ""); + + // Verify dtype is preserved - this would fail before the fix + // because the data would be converted to the backend's default FloatElem + assert_eq!( + loaded.val().dtype(), + DType::F64, + "Loaded tensor should have F64 dtype" + ); + + // Verify data values are correct + let loaded_data = loaded.val().to_data().to_vec::().unwrap(); + assert_eq!(loaded_data, vec![1.0, 2.0, 3.0, 4.0]); + + // Verify applier result + let result = applier.into_result(); + assert_eq!(result.applied.len(), 1); + assert_eq!(result.errors.len(), 0); + } + + /// Test that F32 dtype is preserved when loading (verifies we didn't break F32 handling) + #[test] + fn dtype_preservation_f32() { + let device = Default::default(); + + // Create TensorData with F32 dtype + let f32_data = TensorData::new(vec![1.0f32, 2.0, 3.0, 4.0], [2, 2]); + assert_eq!(f32_data.dtype, DType::F32); + + // Create a snapshot with F32 data + let snapshot = crate::TensorSnapshot::from_data( + f32_data.clone(), + vec!["weight".to_string()], + vec![], + ParamId::new(), + ); + assert_eq!(snapshot.dtype, DType::F32); + + // Create applier with the F32 snapshot + let mut applier = Applier::::new(vec![snapshot], None, None, false); + + // Create target parameter + let target = Param::initialized( + ParamId::new(), + Tensor::::zeros([2, 2], &device), + ); + + // Apply the snapshot + applier.enter_module("weight", ""); + let loaded = applier.map_float(target); + applier.exit_module("weight", ""); + + // Verify dtype is F32 + assert_eq!(loaded.val().dtype(), DType::F32); + + // Verify data values + let loaded_data = loaded.val().to_data().to_vec::().unwrap(); + assert_eq!(loaded_data, vec![1.0, 2.0, 3.0, 4.0]); + } + + /// Test that F16 dtype is correctly preserved in TensorSnapshot. + /// + /// Note: Full F16 tensor loading requires a backend that supports F16 + /// (e.g., CUDA, WebGPU). The NdArray backend does not support F16. + /// This test verifies that the snapshot correctly preserves F16 dtype, + /// which is the key part of the dtype preservation fix. + #[test] + fn dtype_preservation_f16_snapshot() { + use half::f16; + + // Create TensorData with F16 dtype using the half crate + let f16_values: Vec = vec![ + f16::from_f32(1.0), + f16::from_f32(2.0), + f16::from_f32(3.0), + f16::from_f32(4.0), + ]; + let f16_data = TensorData::new(f16_values.clone(), [2, 2]); + assert_eq!( + f16_data.dtype, + DType::F16, + "TensorData should have F16 dtype" + ); + + // Create a snapshot with F16 data + let snapshot = crate::TensorSnapshot::from_data( + f16_data.clone(), + vec!["weight".to_string()], + vec![], + ParamId::new(), + ); + + // Verify snapshot preserves F16 dtype + assert_eq!( + snapshot.dtype, + DType::F16, + "TensorSnapshot should preserve F16 dtype" + ); + + // Verify the data can be retrieved with correct dtype + let retrieved_data = snapshot.to_data().expect("Should be able to retrieve data"); + assert_eq!( + retrieved_data.dtype, + DType::F16, + "Retrieved data should have F16 dtype" + ); + + // Verify the actual values are preserved + let retrieved_values: Vec = retrieved_data + .to_vec() + .expect("Should be able to convert to f16 vec"); + assert_eq!( + retrieved_values, f16_values, + "F16 values should be preserved" + ); + + // Note: To fully test F16 tensor creation, you would need a backend + // that supports F16 (like CUDA or WebGPU). The applier fix ensures + // that `Tensor::from_data_dtype(data, device, snapshot.dtype)` is + // called with DType::F16, which will correctly create an F16 tensor + // on backends that support it. + } + + /// Test that BF16 dtype is correctly preserved in TensorSnapshot. + #[test] + fn dtype_preservation_bf16_snapshot() { + use half::bf16; + + // Create TensorData with BF16 dtype + let bf16_values: Vec = vec![ + bf16::from_f32(1.0), + bf16::from_f32(2.0), + bf16::from_f32(3.0), + bf16::from_f32(4.0), + ]; + let bf16_data = TensorData::new(bf16_values.clone(), [2, 2]); + assert_eq!( + bf16_data.dtype, + DType::BF16, + "TensorData should have BF16 dtype" + ); + + // Create a snapshot with BF16 data + let snapshot = crate::TensorSnapshot::from_data( + bf16_data.clone(), + vec!["weight".to_string()], + vec![], + ParamId::new(), + ); + + // Verify snapshot preserves BF16 dtype + assert_eq!( + snapshot.dtype, + DType::BF16, + "TensorSnapshot should preserve BF16 dtype" + ); + + // Verify the data can be retrieved with correct dtype + let retrieved_data = snapshot.to_data().expect("Should be able to retrieve data"); + assert_eq!( + retrieved_data.dtype, + DType::BF16, + "Retrieved data should have BF16 dtype" + ); + + // Verify the actual values are preserved + let retrieved_values: Vec = retrieved_data + .to_vec() + .expect("Should be able to convert to bf16 vec"); + assert_eq!( + retrieved_values, bf16_values, + "BF16 values should be preserved" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/apply_result.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/apply_result.rs new file mode 100644 index 0000000..c88f18c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/apply_result.rs @@ -0,0 +1,300 @@ +//! Result types and diagnostics for tensor application operations + +use alloc::string::String; +use alloc::vec; +use alloc::vec::Vec; + +use burn_tensor::DType; + +/// Error types that can occur during tensor application +#[derive(Debug, Clone)] +pub enum ApplyError { + /// Shape mismatch between expected and actual tensor + ShapeMismatch { + /// Path of the tensor + path: String, + /// Expected shape + expected: Vec, + /// Found shape + found: Vec, + }, + /// Data type mismatch between expected and actual tensor + DTypeMismatch { + /// Path of the tensor + path: String, + /// Expected data type + expected: DType, + /// Found data type + found: DType, + }, + /// Error from adapter transformation + AdapterError { + /// Path of the tensor + path: String, + /// Error message + message: String, + }, + /// Error loading tensor data + LoadError { + /// Path of the tensor + path: String, + /// Error message + message: String, + }, +} + +impl core::fmt::Display for ApplyError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::ShapeMismatch { + path, + expected, + found, + } => { + write!( + f, + "Shape mismatch for '{}': expected {:?}, found {:?}", + path, expected, found + ) + } + Self::DTypeMismatch { + path, + expected, + found, + } => { + write!( + f, + "DType mismatch for '{}': expected {:?}, found {:?}", + path, expected, found + ) + } + Self::AdapterError { path, message } => { + write!(f, "Adapter error for '{}': {}", path, message) + } + Self::LoadError { path, message } => { + write!(f, "Load error for '{}': {}", path, message) + } + } + } +} + +impl core::error::Error for ApplyError {} + +/// Result of applying tensor snapshots to a module +#[derive(Clone)] +pub struct ApplyResult { + /// Successfully applied tensor paths + pub applied: Vec, + /// Skipped tensor paths (due to filter) + pub skipped: Vec, + /// Missing tensor paths with their container stacks in dot notation (path, container_stack) + /// Container stack shows the hierarchy: "Struct:Model.Struct:Linear" or "Struct:Model.Enum:ConvType.Struct:Linear" + pub missing: Vec<(String, String)>, + /// Unused tensor paths (in snapshots but not in module) + pub unused: Vec, + /// Errors encountered during application + pub errors: Vec, +} + +impl ApplyResult { + /// Try to strip enum variant from a path + /// e.g., "field.BaseConv.weight" -> "field.weight" + fn strip_enum_variant(path: &str) -> Option { + let segments: Vec<&str> = path.split('.').collect(); + + // Find segments that look like enum variants (CamelCase in middle of path) + let variant_indices: Vec = segments + .iter() + .enumerate() + .filter(|(i, segment)| { + *i > 0 && *i < segments.len() - 1 // Not first or last + && !segment.is_empty() + && segment.chars().next().map(|c| c.is_uppercase()).unwrap_or(false) + && segment.len() > 1 + && segment.chars().skip(1).any(|c| c.is_lowercase()) + }) + .map(|(i, _)| i) + .collect(); + + if variant_indices.is_empty() { + return None; + } + + // Remove the first found variant and return the modified path + let mut result_segments = segments.clone(); + result_segments.remove(variant_indices[0]); + Some(result_segments.join(".")) + } + + /// Find similar paths for a given missing path (for "Did you mean?" suggestions) + fn find_similar_paths(&self, missing_path: &str, max_suggestions: usize) -> Vec { + // First, try exact match with enum variant stripped + if let Some(stripped) = Self::strip_enum_variant(missing_path) + && self.unused.contains(&stripped) + { + return vec![stripped]; + } + + // Fall back to Jaro similarity (used by Elixir for "did you mean?" suggestions) + // Jaro gives higher weight to matching prefixes, ideal for hierarchical tensor paths + let mut similarities: Vec<(String, f64)> = self + .unused + .iter() + .map(|available| { + let similarity = textdistance::nstr::jaro(missing_path, available); + (available.clone(), similarity) + }) + .collect(); + + // Sort by similarity (higher = more similar) + similarities + .sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(core::cmp::Ordering::Equal)); + + // Only suggest paths with >= 70% similarity + const SIMILARITY_THRESHOLD: f64 = 0.7; + similarities + .into_iter() + .filter(|(_, sim)| *sim >= SIMILARITY_THRESHOLD) + .take(max_suggestions) + .map(|(path, _)| path) + .collect() + } +} + +impl ApplyResult { + /// Check if the apply operation was successful (no errors) + /// Note: Missing tensors are not considered errors when allow_partial is true + pub fn is_success(&self) -> bool { + self.errors.is_empty() + } +} + +impl core::fmt::Debug for ApplyResult { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // Delegate to Display for comprehensive output + core::fmt::Display::fmt(self, f) + } +} + +impl core::fmt::Display for ApplyResult { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + writeln!(f, "┌─ Tensor Loading Summary ─────────────────────────")?; + writeln!(f, "│")?; + writeln!( + f, + "│ ✓ Successfully applied: {} tensors", + self.applied.len() + )?; + writeln!(f, "│ ⊘ Skipped (filtered): {} tensors", self.skipped.len())?; + writeln!( + f, + "│ ✗ Missing in source: {} tensors", + self.missing.len() + )?; + writeln!(f, "│ ? Unused in target: {} tensors", self.unused.len())?; + writeln!(f, "│ ! Errors: {} errors", self.errors.len())?; + + if !self.missing.is_empty() { + writeln!(f, "│")?; + writeln!( + f, + "├─ Missing Tensors (requested by model but not found in source)" + )?; + writeln!(f, "│")?; + + // Use actual container stack data to detect enum variants + // Count how many missing paths have "Enum:" in their container stack + let enum_variant_missing: Vec<_> = self + .missing + .iter() + .filter(|(_, stack)| stack.contains("Enum:")) + .collect(); + + if !enum_variant_missing.is_empty() { + writeln!( + f, + "│ ⚠️ {} paths contain enum variants (detected from container stack)", + enum_variant_missing.len() + )?; + writeln!( + f, + "│ Burn includes enum variant names in paths, but PyTorch doesn't." + )?; + writeln!( + f, + "│ Example: Burn has 'field.BaseConv.weight', PyTorch has 'field.weight'" + )?; + writeln!(f, "│")?; + writeln!( + f, + "│ 💡 Solution 1: Enable skip_enum_variants flag (simplest):" + )?; + writeln!(f, "│")?; + writeln!( + f, + "│ let mut store = PytorchStore::from_file(\"model.pth\")" + )?; + writeln!(f, "│ .skip_enum_variants(true); // ← Add this")?; + writeln!(f, "│")?; + writeln!( + f, + "│ 💡 Solution 2: Remap enum keys in source (most precise):" + )?; + writeln!(f, "│")?; + writeln!( + f, + "│ let mut store = SafetensorsStore::from_file(\"model.safetensors\")" + )?; + writeln!( + f, + "│ .with_key_remapping(r\"field\\.(\\w+)\", \"field.BaseConv.$1\");" + )?; + writeln!(f, "│")?; + } + + writeln!(f, "│ First 10 missing tensors:")?; + for (path, _) in self.missing.iter().take(10) { + writeln!(f, "│ • {}", path)?; + + // Show "Did you mean?" suggestions for this path + let suggestions = self.find_similar_paths(path, 1); + if !suggestions.is_empty() { + writeln!(f, "│ Did you mean: '{}'?", suggestions[0])?; + } + } + if self.missing.len() > 10 { + writeln!(f, "│ ... and {} more", self.missing.len() - 10)?; + } + } + + if !self.unused.is_empty() { + writeln!(f, "│")?; + writeln!(f, "├─ Unused Tensors (in source but not used by model)")?; + writeln!(f, "│")?; + writeln!(f, "│ First 10 unused tensors:")?; + for path in self.unused.iter().take(10) { + writeln!(f, "│ • {}", path)?; + } + if self.unused.len() > 10 { + writeln!(f, "│ ... and {} more", self.unused.len() - 10)?; + } + } + + if !self.errors.is_empty() { + writeln!(f, "│")?; + writeln!(f, "├─ Errors")?; + writeln!(f, "│")?; + for error in self.errors.iter().take(10) { + writeln!(f, "│ ⚠️ {}", error)?; + } + if self.errors.len() > 10 { + writeln!(f, "│ ... and {} more", self.errors.len() - 10)?; + } + } + + writeln!(f, "│")?; + write!(f, "└───────────────────────────────────────────────────")?; + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/base.rs new file mode 100644 index 0000000..773eefd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/base.rs @@ -0,0 +1,231 @@ +//! Core types and constants for the Burnpack file format. +//! +//! See the [parent module](crate::burnpack) for the complete file format specification. + +use alloc::collections::BTreeMap; +use alloc::string::String; +use alloc::vec::Vec; +use burn_tensor::DType; +use byteorder::{ByteOrder, LittleEndian}; +use serde::{Deserialize, Serialize}; + +/// Magic number identifying a Burnpack file: "BURN" in ASCII (0x4255524E) +/// When written to file in little-endian format, appears as "NRUB" bytes +pub const MAGIC_NUMBER: u32 = 0x4255524E; + +/// Current format version +pub const FORMAT_VERSION: u16 = 0x0001; + +/// Size of the magic number in bytes +pub const MAGIC_SIZE: usize = 4; + +/// Size of the format version in bytes +pub const VERSION_SIZE: usize = 2; + +/// Size of the metadata size field in bytes +pub const METADATA_SIZE_FIELD_SIZE: usize = 4; + +/// Total header size (computed from components) +pub const HEADER_SIZE: usize = MAGIC_SIZE + VERSION_SIZE + METADATA_SIZE_FIELD_SIZE; + +/// Alignment for tensor data in bytes. +/// +/// All tensor data is aligned to 256-byte boundaries to enable efficient +/// memory-mapped (mmap) zero-copy loading. This alignment ensures: +/// - Proper pointer alignment for all tensor element types (f64 requires 8-byte alignment) +/// - Cache-line friendly access (most CPUs use 64-byte cache lines) +/// - GPU memory alignment (CUDA prefers 256-byte for coalesced access) +/// - Future-proofing for wider SIMD (AVX-512 = 64 bytes, future AVX-1024 = 128 bytes) +/// +/// Industry alignment choices: +/// - 256-byte: GGUF, MLX, ncnn, MNN, TNN, vLLM-AWQ, Marlin (15+ formats) +/// - 64-byte: SafeTensors (minimum for AVX-512) +/// - 4096-byte: Core ML +/// +/// 256-byte alignment has negligible overhead for typical tensor sizes while +/// providing maximum compatibility with current and future hardware. +pub const TENSOR_ALIGNMENT: u64 = 256; + +/// Calculate the byte offset where the tensor data section starts. +/// +/// The data section is padded to start at a 256-byte aligned position +/// so that all tensor offsets (which are relative to data section) result +/// in properly aligned absolute file positions for mmap zero-copy access. +/// +/// This function must be used consistently by both writer and reader. +#[inline] +pub fn aligned_data_section_start(metadata_size: usize) -> usize { + let unaligned_start = (HEADER_SIZE + metadata_size) as u64; + // Keep multiplication in u64 space to avoid overflow on 32-bit systems + (unaligned_start.div_ceil(TENSOR_ALIGNMENT) * TENSOR_ALIGNMENT) as usize +} + +// Security limits to prevent DoS attacks via resource exhaustion +// These can be adjusted based on your use case + +/// Maximum allowed metadata size (100 MB) +/// Prevents memory exhaustion attacks via oversized metadata claims +pub const MAX_METADATA_SIZE: u32 = 100 * 1024 * 1024; + +/// Maximum allowed tensor size per tensor +/// Prevents memory exhaustion attacks via oversized tensor claims +/// 32-bit platforms: 2 GB limit (to fit within usize range) +/// 64-bit platforms: 10 GB limit +#[cfg(target_pointer_width = "32")] +pub const MAX_TENSOR_SIZE: usize = 2 * 1024 * 1024 * 1024; +#[cfg(not(target_pointer_width = "32"))] +pub const MAX_TENSOR_SIZE: usize = 10 * 1024 * 1024 * 1024; + +/// Maximum allowed number of tensors (100,000) +/// Prevents resource exhaustion via excessive tensor counts +pub const MAX_TENSOR_COUNT: usize = 100_000; + +/// Maximum CBOR deserialization recursion depth (128 levels) +/// Prevents stack overflow attacks via deeply nested CBOR structures +pub const MAX_CBOR_RECURSION_DEPTH: usize = 128; + +/// Maximum allowed file size (100 GB) +/// Prevents resource exhaustion from extremely large files +/// This limit applies to file-based loading (mmap and buffered) +#[cfg(feature = "std")] +pub const MAX_FILE_SIZE: u64 = 100 * 1024 * 1024 * 1024; + +/// Byte range for magic number in header +pub const fn magic_range() -> core::ops::Range { + let start = 0; + let end = start + MAGIC_SIZE; + start..end +} + +/// Byte range for format version in header +pub const fn version_range() -> core::ops::Range { + let start = MAGIC_SIZE; + let end = start + VERSION_SIZE; + start..end +} + +/// Byte range for metadata size field in header +pub const fn metadata_size_range() -> core::ops::Range { + let start = MAGIC_SIZE + VERSION_SIZE; + let end = start + METADATA_SIZE_FIELD_SIZE; + start..end +} + +// Compile-time validation that ranges are correct +const _: () = assert!(MAGIC_SIZE + VERSION_SIZE + METADATA_SIZE_FIELD_SIZE == HEADER_SIZE); + +/// Header structure for Burnpack files +#[derive(Debug, Clone, Copy)] +pub struct BurnpackHeader { + /// Magic number (4 bytes): 0x4255524E ("BURN") + pub magic: u32, + /// Format version (2 bytes) + pub version: u16, + /// Size of CBOR metadata in bytes (4 bytes) + pub metadata_size: u32, +} + +impl BurnpackHeader { + /// Create a new header with the given metadata size + #[allow(dead_code)] + pub fn new(metadata_size: u32) -> Self { + Self { + magic: MAGIC_NUMBER, + version: FORMAT_VERSION, + metadata_size, + } + } + + /// Serialize header into bytes + pub fn into_bytes(self) -> [u8; HEADER_SIZE] { + let mut bytes = [0u8; HEADER_SIZE]; + LittleEndian::write_u32(&mut bytes[magic_range()], self.magic); + LittleEndian::write_u16(&mut bytes[version_range()], self.version); + LittleEndian::write_u32(&mut bytes[metadata_size_range()], self.metadata_size); + bytes + } + + /// Deserialize header from bytes + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() < HEADER_SIZE { + return Err(BurnpackError::InvalidHeader); + } + + let magic = LittleEndian::read_u32(&bytes[magic_range()]); + if magic != MAGIC_NUMBER { + return Err(BurnpackError::InvalidMagicNumber); + } + + let version = LittleEndian::read_u16(&bytes[version_range()]); + let metadata_size = LittleEndian::read_u32(&bytes[metadata_size_range()]); + + Ok(Self { + magic, + version, + metadata_size, + }) + } +} + +/// Metadata structure serialized with CBOR +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BurnpackMetadata { + /// Tensor descriptors mapped by name for efficient lookup + pub tensors: BTreeMap, + /// Optional additional metadata + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub metadata: BTreeMap, +} + +/// Individual tensor descriptor +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TensorDescriptor { + /// Data type of the tensor + pub dtype: DType, + /// Tensor shape dimensions + pub shape: Vec, + /// Byte offsets in data section (start, end) + pub data_offsets: (u64, u64), + /// Parameter ID for training state persistence matching. + /// Generated automatically if not present during loading. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub param_id: Option, +} + +/// Error types for Burnpack operations +#[derive(Debug)] +pub enum BurnpackError { + InvalidHeader, + InvalidMagicNumber, + InvalidVersion, + MetadataSerializationError(String), + MetadataDeserializationError(String), + IoError(String), + TensorNotFound(String), + TensorBytesSizeMismatch(String), + ValidationError(String), +} + +impl core::fmt::Display for BurnpackError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + BurnpackError::InvalidHeader => write!(f, "Invalid header: insufficient bytes"), + BurnpackError::InvalidMagicNumber => write!(f, "Invalid magic number"), + BurnpackError::InvalidVersion => write!(f, "Unsupported version"), + BurnpackError::MetadataSerializationError(e) => { + write!(f, "Metadata serialization error: {}", e) + } + BurnpackError::MetadataDeserializationError(e) => { + write!(f, "Metadata deserialization error: {}", e) + } + BurnpackError::IoError(e) => write!(f, "I/O error: {}", e), + BurnpackError::TensorNotFound(name) => write!(f, "Tensor not found: {}", name), + BurnpackError::TensorBytesSizeMismatch(e) => { + write!(f, "Tensor bytes size mismatch: {}", e) + } + BurnpackError::ValidationError(e) => write!(f, "Validation error: {}", e), + } + } +} + +impl core::error::Error for BurnpackError {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/mod.rs new file mode 100644 index 0000000..94d5828 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/mod.rs @@ -0,0 +1,62 @@ +//! # Burnpack - Native Burn Model Storage Format +//! +//! Burnpack is the native binary storage format for Burn models, designed for efficient +//! serialization, fast loading, and cross-platform compatibility. +//! +//! ## Key Features +//! +//! - **CBOR Metadata**: Structured metadata with efficient binary encoding +//! - **Memory-Mapped Loading**: Zero-copy loading for optimal performance +//! - **256-byte Tensor Alignment**: Enables efficient mmap zero-copy access +//! - **No-std Support**: Works in embedded and WASM environments +//! - **ParamId Persistence**: Preserves parameter identities for stateful training +//! - **Lazy Tensor Loading**: Deferred data materialization for efficient memory usage +//! +//! ## File Format Structure +//! +//! ```text +//! ┌──────────────────────────────────┐ +//! │ Header (10 bytes) │ +//! ├──────────────────────────────────┤ +//! │ - Magic number (4 bytes) │ 0x4E525542 ("NRUB" in LE) +//! │ - Version (2 bytes) │ Format version (0x0001) +//! │ - Metadata size (4 bytes) │ Size of CBOR metadata (u32) +//! ├──────────────────────────────────┤ +//! │ Metadata (CBOR) │ +//! ├──────────────────────────────────┤ +//! │ - Tensor descriptors (BTreeMap) │ Order-preserving map of tensor metadata +//! │ Key: tensor name (string) │ e.g., "model.layer1.weight" +//! │ Value: TensorDescriptor │ +//! │ - dtype: DType │ Data type (F32, F64, I32, etc.) +//! │ - shape: Vec │ Tensor dimensions +//! │ - data_offsets: (u64, u64) │ (start, end) byte offsets (256-byte aligned) +//! │ - param_id: Option │ Parameter ID (for training state) +//! │ - Additional metadata(BTreeMap) │ User-defined key-value pairs +//! ├──────────────────────────────────┤ +//! │ Tensor Data Section │ +//! ├──────────────────────────────────┤ +//! │ [padding][tensor1][padding]... │ Each tensor aligned to 256-byte boundary +//! │ Raw tensor bytes (little-endian)│ Enables mmap zero-copy loading +//! └──────────────────────────────────┘ +//! ``` +//! +//! ## Tensor Alignment +//! +//! All tensor data is aligned to 256-byte boundaries to enable efficient memory-mapped +//! (mmap) zero-copy loading. This alignment ensures: +//! +//! - Proper pointer alignment for all tensor element types (f64 requires 8 bytes) +//! - Cache-line friendly access (most CPUs use 64-byte cache lines) +//! - GPU memory alignment (CUDA prefers 256-byte for coalesced access) +//! - Future-proofing for wider SIMD instructions (AVX-512, future AVX-1024) +//! +//! The 256-byte alignment matches industry standards used by GGUF, MLX, ncnn, MNN, +//! and other major model formats. + +pub mod base; +pub mod reader; +pub mod store; +pub mod writer; + +#[cfg(test)] +mod tests; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/reader.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/reader.rs new file mode 100644 index 0000000..e6c4eb2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/reader.rs @@ -0,0 +1,761 @@ +#[cfg(feature = "std")] +use super::base::MAX_FILE_SIZE; +use super::base::{ + BurnpackError, BurnpackHeader, BurnpackMetadata, FORMAT_VERSION, HEADER_SIZE, MAGIC_NUMBER, + MAX_CBOR_RECURSION_DEPTH, MAX_METADATA_SIZE, MAX_TENSOR_COUNT, MAX_TENSOR_SIZE, + aligned_data_section_start, +}; +use crate::TensorSnapshot; +use alloc::format; +use alloc::rc::Rc; +use alloc::string::ToString; +use alloc::vec; +use alloc::vec::Vec; +use burn_core::module::ParamId; +use burn_tensor::{Bytes, TensorData}; + +#[cfg(feature = "std")] +use std::cell::RefCell; +#[cfg(feature = "std")] +use std::fs::File; +#[cfg(feature = "std")] +use std::io::{Read, Seek}; +#[cfg(feature = "std")] +use std::path::Path; + +/// Storage backend for BurnpackReader +pub(crate) enum StorageBackend { + /// Memory-based storage (also used for memory-mapped files converted to bytes::Bytes) + Memory(Rc), + /// File-based storage with buffered reading + #[cfg(feature = "std")] + #[allow(dead_code)] + FileBuffered { file: Rc> }, +} + +impl StorageBackend { + /// Read data from storage into the provided buffer at the given offset. + /// + /// # Arguments + /// * `bytes` - The buffer to read into (caller-allocated) + /// * `offset` - Absolute file/data position to start reading from + /// + /// # Errors + /// + /// Returns an error if: + /// - The requested data range is out of bounds + /// - Less data is available than requested (indicates corruption or incorrect offset) + /// - File I/O fails + /// + /// # Notes + /// + /// The caller allocates the buffer, which allows for buffer reuse and future optimizations + /// like memory pools and pinned memory. + /// + /// This method ensures all backends have consistent behavior: if the exact number of + /// requested bytes cannot be read, an error is returned to prevent data corruption. + pub(crate) fn read_into(&self, bytes: &mut [u8], offset: usize) -> Result<(), BurnpackError> { + match self { + StorageBackend::Memory(data) => { + let data_bytes = data.as_ref(); + let end = offset.checked_add(bytes.len()).ok_or_else(|| { + BurnpackError::IoError(format!( + "Offset overflow: offset {} + length {} exceeds maximum", + offset, + bytes.len() + )) + })?; + + if end > data_bytes.len() { + return Err(BurnpackError::IoError(format!( + "Read out of bounds: requested {}..{} but data length is {}", + offset, + end, + data_bytes.len() + ))); + } + + bytes.copy_from_slice(&data_bytes[offset..end]); + Ok(()) + } + #[cfg(feature = "std")] + StorageBackend::FileBuffered { file } => { + use std::io::SeekFrom; + + let mut file = file.borrow_mut(); + file.seek(SeekFrom::Start(offset as u64)).map_err(|e| { + BurnpackError::IoError(format!("Failed to seek in file: {}", e)) + })?; + + file.read_exact(bytes).map_err(|e| { + BurnpackError::IoError(format!("Failed to read from file: {}", e)) + })?; + Ok(()) + } + } + } + + /// Get full data reference for raw access + #[allow(dead_code)] + pub(crate) fn as_bytes(&self) -> Result<&[u8], BurnpackError> { + match self { + StorageBackend::Memory(data) => Ok(data.as_ref()), + #[cfg(feature = "std")] + StorageBackend::FileBuffered { .. } => Err(BurnpackError::IoError( + "Cannot get full bytes reference for FileBuffered backend".into(), + )), + } + } + + /// Attempt to slice bytes without copying (zero-copy). + /// + /// This uses `Bytes::clone()` + `split()` which is zero-copy when the underlying + /// `Bytes` was created via `Bytes::from_shared()` (backed by `bytes::Bytes`). + /// + /// # Returns + /// - `Ok(bytes)` - Successfully created a zero-copy slice + /// - `Err(_)` - Backend doesn't support zero-copy or split failed + pub(crate) fn slice_bytes(&self, start: usize, end: usize) -> Result { + if end < start { + return Err(BurnpackError::IoError(format!( + "Invalid slice range: end ({}) < start ({})", + end, start + ))); + } + + match self { + StorageBackend::Memory(data) => { + // Clone the Bytes - cheap if backed by SharedBytesAllocationController + let cloned = (**data).clone(); + + // Split at start offset to get (_, right) + let (_, right) = cloned.split(start).map_err(|(_, e)| { + BurnpackError::IoError(format!("Failed to split at start {}: {:?}", start, e)) + })?; + + // Split right at (end - start) to get (middle, _) + let slice_len = end - start; + let (middle, _) = right.split(slice_len).map_err(|(_, e)| { + BurnpackError::IoError(format!( + "Failed to split at length {}: {:?}", + slice_len, e + )) + })?; + + Ok(middle) + } + #[cfg(feature = "std")] + StorageBackend::FileBuffered { .. } => Err(BurnpackError::IoError( + "Zero-copy not supported for buffered file reading. Use from_file() with memmap feature for zero-copy loading.".into(), + )), + } + } +} + +/// Reader for loading Burnpack files +pub struct BurnpackReader { + /// Parsed metadata + pub(crate) metadata: BurnpackMetadata, + /// Storage backend + pub(crate) storage: StorageBackend, + /// Offset to the start of tensor data + pub(crate) data_offset: usize, +} + +impl BurnpackReader { + /// Load from bytes + pub fn from_bytes(bytes: Bytes) -> Result { + // Validate minimum size + if bytes.len() < HEADER_SIZE { + return Err(BurnpackError::InvalidHeader); + } + + // Parse header + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE])?; + + // Verify magic number + if header.magic != MAGIC_NUMBER { + return Err(BurnpackError::InvalidMagicNumber); + } + + // Verify version compatibility + if header.version > FORMAT_VERSION { + return Err(BurnpackError::InvalidVersion); + } + + // Validate metadata size against security limit + if header.metadata_size > MAX_METADATA_SIZE { + return Err(BurnpackError::ValidationError(format!( + "Metadata size {} exceeds maximum allowed size of {} bytes (potential DoS attack)", + header.metadata_size, MAX_METADATA_SIZE + ))); + } + + // Parse metadata + let metadata_start = HEADER_SIZE; + let metadata_end = metadata_start + .checked_add(header.metadata_size as usize) + .ok_or_else(|| { + BurnpackError::IoError(format!( + "Metadata size overflow: {} + {}", + metadata_start, header.metadata_size + )) + })?; + + if bytes.len() < metadata_end { + return Err(BurnpackError::InvalidHeader); + } + + let metadata: BurnpackMetadata = ciborium::de::from_reader_with_recursion_limit( + &bytes[metadata_start..metadata_end], + MAX_CBOR_RECURSION_DEPTH, + ) + .map_err(|e| BurnpackError::MetadataDeserializationError(e.to_string()))?; + + // Validate tensor count against security limit + if metadata.tensors.len() > MAX_TENSOR_COUNT { + return Err(BurnpackError::ValidationError(format!( + "File contains {} tensors, exceeding maximum of {} (potential DoS attack)", + metadata.tensors.len(), + MAX_TENSOR_COUNT + ))); + } + + // Validate total file size - ensure file is large enough for all claimed tensor data + if !metadata.tensors.is_empty() { + let max_data_offset = metadata + .tensors + .values() + .map(|t| t.data_offsets.1) + .max() + .unwrap_or(0); + + let max_data_offset_usize: usize = max_data_offset.try_into().map_err(|_| { + BurnpackError::ValidationError(format!( + "Data offset {} exceeds platform maximum", + max_data_offset + )) + })?; + + let min_file_size = + metadata_end + .checked_add(max_data_offset_usize) + .ok_or_else(|| { + BurnpackError::ValidationError("File size calculation overflow".into()) + })?; + + if bytes.len() < min_file_size { + return Err(BurnpackError::ValidationError(format!( + "File truncated: expected at least {} bytes, got {} bytes", + min_file_size, + bytes.len() + ))); + } + } + + Ok(Self { + metadata, + storage: StorageBackend::Memory(Rc::new(bytes)), + data_offset: aligned_data_section_start(header.metadata_size as usize), + }) + } + + /// Load from file with memory mapping (most efficient for large files) + #[cfg(all(feature = "std", feature = "memmap"))] + pub(crate) fn from_file_mmap>(path: P) -> Result { + let file = File::open(&path).map_err(|e| BurnpackError::IoError(e.to_string()))?; + + // Validate maximum file size to prevent resource exhaustion + let file_size = file + .metadata() + .map_err(|e| BurnpackError::IoError(e.to_string()))? + .len(); + + if file_size > MAX_FILE_SIZE { + return Err(BurnpackError::ValidationError(format!( + "File size {} bytes exceeds maximum allowed size of {} bytes", + file_size, MAX_FILE_SIZE + ))); + } + + // Memory map the file + let mmap = unsafe { + memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| BurnpackError::IoError(e.to_string()))? + }; + + // Parse header + if mmap.len() < HEADER_SIZE { + return Err(BurnpackError::InvalidHeader); + } + + let header = BurnpackHeader::from_bytes(&mmap[..HEADER_SIZE])?; + + // Verify magic number and version + if header.magic != MAGIC_NUMBER { + return Err(BurnpackError::InvalidMagicNumber); + } + + if header.version > FORMAT_VERSION { + return Err(BurnpackError::InvalidVersion); + } + + // Validate metadata size against security limit + if header.metadata_size > MAX_METADATA_SIZE { + return Err(BurnpackError::ValidationError(format!( + "Metadata size {} exceeds maximum allowed size of {} bytes (potential DoS attack)", + header.metadata_size, MAX_METADATA_SIZE + ))); + } + + // Parse metadata + let metadata_start = HEADER_SIZE; + let metadata_end = metadata_start + .checked_add(header.metadata_size as usize) + .ok_or_else(|| { + BurnpackError::IoError(format!( + "Metadata size overflow: {} + {}", + metadata_start, header.metadata_size + )) + })?; + + if mmap.len() < metadata_end { + return Err(BurnpackError::InvalidHeader); + } + + let metadata: BurnpackMetadata = ciborium::de::from_reader_with_recursion_limit( + &mmap[metadata_start..metadata_end], + MAX_CBOR_RECURSION_DEPTH, + ) + .map_err(|e| BurnpackError::MetadataDeserializationError(e.to_string()))?; + + // Validate tensor count against security limit + if metadata.tensors.len() > MAX_TENSOR_COUNT { + return Err(BurnpackError::ValidationError(format!( + "File contains {} tensors, exceeding maximum of {} (potential DoS attack)", + metadata.tensors.len(), + MAX_TENSOR_COUNT + ))); + } + + // Validate total file size - ensure file is large enough for all claimed tensor data + if !metadata.tensors.is_empty() { + let max_data_offset = metadata + .tensors + .values() + .map(|t| t.data_offsets.1) + .max() + .unwrap_or(0); + + let max_data_offset_usize: usize = max_data_offset.try_into().map_err(|_| { + BurnpackError::ValidationError(format!( + "Data offset {} exceeds platform maximum", + max_data_offset + )) + })?; + + let min_file_size = + metadata_end + .checked_add(max_data_offset_usize) + .ok_or_else(|| { + BurnpackError::ValidationError("File size calculation overflow".into()) + })?; + + if mmap.len() < min_file_size { + return Err(BurnpackError::ValidationError(format!( + "File truncated: expected at least {} bytes, got {} bytes", + min_file_size, + mmap.len() + ))); + } + } + + // Convert mmap to bytes::Bytes for zero-copy slicing support + // bytes::Bytes::from_owner takes ownership and enables efficient slicing + let shared_bytes = bytes::Bytes::from_owner(mmap); + let bytes = Bytes::from_shared(shared_bytes, burn_tensor::AllocationProperty::File); + + Ok(Self { + metadata, + storage: StorageBackend::Memory(Rc::new(bytes)), + data_offset: aligned_data_section_start(header.metadata_size as usize), + }) + } + + /// Load from file - automatically uses memory mapping if available, otherwise uses buffered reading + #[cfg(feature = "std")] + pub fn from_file>(path: P) -> Result { + #[cfg(feature = "memmap")] + { + // Use memory mapping for efficient access + Self::from_file_mmap(path) + } + #[cfg(not(feature = "memmap"))] + { + // Fall back to buffered reading for memory efficiency + Self::from_file_buffered(path) + } + } + + /// Load from file with buffered reading (memory efficient but slower) + /// This is less efficient than memory mapping but works everywhere + #[cfg(feature = "std")] + #[allow(dead_code)] + pub(crate) fn from_file_buffered>(path: P) -> Result { + let mut file = File::open(&path).map_err(|e| BurnpackError::IoError(e.to_string()))?; + + // Validate maximum file size to prevent resource exhaustion + let file_size = file + .metadata() + .map_err(|e| BurnpackError::IoError(e.to_string()))? + .len(); + + if file_size > MAX_FILE_SIZE { + return Err(BurnpackError::ValidationError(format!( + "File size {} bytes exceeds maximum allowed size of {} bytes", + file_size, MAX_FILE_SIZE + ))); + } + + // Read header + let mut header_bytes = [0u8; HEADER_SIZE]; + file.read_exact(&mut header_bytes) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + + let header = BurnpackHeader::from_bytes(&header_bytes)?; + + // Verify version + if header.version > FORMAT_VERSION { + return Err(BurnpackError::InvalidVersion); + } + + // Validate metadata size against security limit + if header.metadata_size > MAX_METADATA_SIZE { + return Err(BurnpackError::ValidationError(format!( + "Metadata size {} exceeds maximum allowed size of {} bytes (potential DoS attack)", + header.metadata_size, MAX_METADATA_SIZE + ))); + } + + // Read metadata + let mut metadata_bytes = vec![0u8; header.metadata_size as usize]; + file.read_exact(&mut metadata_bytes) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + + let metadata: BurnpackMetadata = ciborium::de::from_reader_with_recursion_limit( + metadata_bytes.as_slice(), + MAX_CBOR_RECURSION_DEPTH, + ) + .map_err(|e| BurnpackError::MetadataDeserializationError(e.to_string()))?; + + // Validate tensor count against security limit + if metadata.tensors.len() > MAX_TENSOR_COUNT { + return Err(BurnpackError::ValidationError(format!( + "File contains {} tensors, exceeding maximum of {} (potential DoS attack)", + metadata.tensors.len(), + MAX_TENSOR_COUNT + ))); + } + + // Calculate metadata end offset + let metadata_end = HEADER_SIZE + .checked_add(header.metadata_size as usize) + .ok_or_else(|| { + BurnpackError::IoError(format!( + "Metadata size overflow: {} + {}", + HEADER_SIZE, header.metadata_size + )) + })?; + + // Validate total file size - ensure file is large enough for all claimed tensor data + if !metadata.tensors.is_empty() { + let max_data_offset = metadata + .tensors + .values() + .map(|t| t.data_offsets.1) + .max() + .unwrap_or(0); + + let max_data_offset_usize: usize = max_data_offset.try_into().map_err(|_| { + BurnpackError::ValidationError(format!( + "Data offset {} exceeds platform maximum", + max_data_offset + )) + })?; + + let min_file_size = + metadata_end + .checked_add(max_data_offset_usize) + .ok_or_else(|| { + BurnpackError::ValidationError("File size calculation overflow".into()) + })?; + + // Get actual file size + let file_size = file + .metadata() + .map_err(|e| BurnpackError::IoError(e.to_string()))? + .len() as usize; + + if file_size < min_file_size { + return Err(BurnpackError::ValidationError(format!( + "File truncated: expected at least {} bytes, got {} bytes", + min_file_size, file_size + ))); + } + } + + Ok(Self { + metadata, + storage: StorageBackend::FileBuffered { + file: Rc::new(RefCell::new(file)), + }, + data_offset: aligned_data_section_start(header.metadata_size as usize), + }) + } + + /// Get all tensor snapshots at once for efficient loading (always copies data) + pub fn get_snapshots(&self) -> Result, BurnpackError> { + self.get_snapshots_internal(false) + } + + /// Get all tensor snapshots with optional zero-copy loading. + /// + /// When `zero_copy` is true and the backend supports it (Memory backend with + /// `Bytes::from_shared()`), tensor data is sliced without copying. This keeps + /// the original data alive as long as any tensor holds a reference. + /// + /// When `zero_copy` is false or the backend doesn't support it, data is copied + /// into newly allocated buffers (default behavior). + pub fn get_snapshots_zero_copy( + &self, + zero_copy: bool, + ) -> Result, BurnpackError> { + self.get_snapshots_internal(zero_copy) + } + + /// Internal implementation with optional zero-copy support + fn get_snapshots_internal( + &self, + zero_copy: bool, + ) -> Result, BurnpackError> { + let mut snapshots = Vec::new(); + + for (name, descriptor) in &self.metadata.tensors { + // Clone metadata for use in closure + // Convert shape dimensions with overflow checking + let shape: Vec = descriptor + .shape + .iter() + .map(|&s| { + s.try_into().map_err(|_| { + BurnpackError::ValidationError(format!( + "Tensor '{}' has corrupted shape data: dimension {} exceeds platform maximum", + name, s + )) + }) + }) + .collect::, BurnpackError>>()?; + + let dtype = descriptor.dtype; + + // Clone storage reference for the closure + let storage = match &self.storage { + StorageBackend::Memory(data) => StorageBackend::Memory(data.clone()), + #[cfg(feature = "std")] + StorageBackend::FileBuffered { file } => { + StorageBackend::FileBuffered { file: file.clone() } + } + }; + + // Always use absolute positions for all backends + // Convert offsets with overflow checking + let offset_start: usize = descriptor.data_offsets.0.try_into().map_err(|_| { + BurnpackError::ValidationError(format!( + "Tensor '{}' has corrupted offset data: start offset {} exceeds platform maximum", + name, descriptor.data_offsets.0 + )) + })?; + + let offset_end: usize = descriptor.data_offsets.1.try_into().map_err(|_| { + BurnpackError::ValidationError(format!( + "Tensor '{}' has corrupted offset data: end offset {} exceeds platform maximum", + name, descriptor.data_offsets.1 + )) + })?; + + let start = self.data_offset.checked_add(offset_start).ok_or_else(|| { + BurnpackError::ValidationError(format!( + "Tensor '{}' has corrupted offset data: start offset overflow {} + {}", + name, self.data_offset, offset_start + )) + })?; + + let end = self.data_offset.checked_add(offset_end).ok_or_else(|| { + BurnpackError::ValidationError(format!( + "Tensor '{}' has corrupted offset data: end offset overflow {} + {}", + name, self.data_offset, offset_end + )) + })?; + + // Clone shape for the closure (TensorSnapshot::from_closure will also need it) + let shape_for_closure = shape.clone(); + + // Validate offset range + if end < start { + return Err(BurnpackError::ValidationError(format!( + "Tensor '{}' has corrupted offset data: end offset {} < start offset {}", + name, end, start + ))); + } + + // Validate tensor size against security limit + let tensor_size = end - start; + if tensor_size > MAX_TENSOR_SIZE { + return Err(BurnpackError::ValidationError(format!( + "Tensor '{}' size {} exceeds maximum allowed size of {} bytes (potential DoS attack)", + name, tensor_size, MAX_TENSOR_SIZE + ))); + } + + // Restore param_id if it was saved, otherwise generate + let tensor_id = descriptor + .param_id + .map(ParamId::from) + .unwrap_or_else(ParamId::new); + + // Create the data-loading closure based on zero_copy flag + let data_fn: Rc Result> = + if zero_copy { + // Zero-copy closure: slice without copying, error if not supported + Rc::new(move || { + let bytes = storage.slice_bytes(start, end).map_err(|e| { + crate::TensorSnapshotError::IoError(format!( + "Zero-copy slice failed: {}", + e + )) + })?; + Ok(TensorData::from_bytes( + bytes, + shape_for_closure.clone(), + dtype, + )) + }) + } else { + // Copying closure: always allocate and copy + Rc::new(move || { + let len = end - start; + // TODO Should be allocated by the backend in the future + // See https://github.com/tracel-ai/burn/pull/3792#discussion_r2416812091 + let mut data_bytes = vec![0u8; len]; + storage.read_into(&mut data_bytes, start).map_err(|e| { + crate::TensorSnapshotError::IoError(format!( + "Failed to read tensor data: {}", + e + )) + })?; + Ok(TensorData::from_bytes_vec( + data_bytes, + shape_for_closure.clone(), + dtype, + )) + }) + }; + + // Create lazy TensorSnapshot + let snapshot = TensorSnapshot::from_closure( + data_fn, + dtype, + shape, + name.split('.').map(|s| s.to_string()).collect(), + vec![], // empty container_stack + tensor_id, // restored or newly generated param id + ); + + snapshots.push(snapshot); + } + + Ok(snapshots) + } + + // Legacy methods for test compatibility - will be removed + + /// Get tensor as TensorSnapshot with lazy loading + #[allow(dead_code)] + pub(crate) fn get_tensor_snapshot(&self, name: &str) -> Result { + let snapshots = self.get_snapshots()?; + snapshots + .into_iter() + .find(|s| s.full_path() == name) + .ok_or_else(|| BurnpackError::TensorNotFound(name.to_string())) + } + + /// Get list of tensor names + #[allow(dead_code)] + pub(crate) fn tensor_names(&self) -> Vec<&str> { + self.metadata + .tensors + .keys() + .map(|name| name.as_str()) + .collect() + } + + /// Get metadata + #[allow(dead_code)] + pub(crate) fn metadata(&self) -> &BurnpackMetadata { + &self.metadata + } + + /// Get tensor data as raw bytes + #[allow(dead_code)] + pub(crate) fn get_tensor_data(&self, name: &str) -> Result, BurnpackError> { + let descriptor = self + .metadata + .tensors + .get(name) + .ok_or_else(|| BurnpackError::TensorNotFound(name.to_string()))?; + + // Always use absolute positions for all backends + // Convert offsets with overflow checking + let offset_start: usize = descriptor.data_offsets.0.try_into().map_err(|_| { + BurnpackError::IoError(format!( + "Tensor '{}' has corrupted offset data: start offset {} exceeds platform maximum", + name, descriptor.data_offsets.0 + )) + })?; + + let offset_end: usize = descriptor.data_offsets.1.try_into().map_err(|_| { + BurnpackError::IoError(format!( + "Tensor '{}' has corrupted offset data: end offset {} exceeds platform maximum", + name, descriptor.data_offsets.1 + )) + })?; + + let start = self.data_offset.checked_add(offset_start).ok_or_else(|| { + BurnpackError::IoError(format!( + "Tensor '{}' has corrupted offset data: start offset overflow {} + {}", + name, self.data_offset, offset_start + )) + })?; + + let end = self.data_offset.checked_add(offset_end).ok_or_else(|| { + BurnpackError::IoError(format!( + "Tensor '{}' has corrupted offset data: end offset overflow {} + {}", + name, self.data_offset, offset_end + )) + })?; + + // Validate offset range + if end < start { + return Err(BurnpackError::IoError(format!( + "Tensor '{}' has corrupted offset data: end offset {} < start offset {}", + name, end, start + ))); + } + + let len = end - start; + let mut buffer = vec![0u8; len]; + self.storage.read_into(&mut buffer, start)?; + Ok(buffer) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/store.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/store.rs new file mode 100644 index 0000000..7ec47c5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/store.rs @@ -0,0 +1,507 @@ +#[cfg(feature = "std")] +use std::path::PathBuf; + +use super::reader::BurnpackReader; +use super::writer::BurnpackWriter; +#[cfg(feature = "std")] +use crate::KeyRemapper; +use crate::burnpack::base::BurnpackError; +use crate::{ModuleSnapshot, ModuleStore, PathFilter, TensorSnapshot}; +use alloc::collections::BTreeMap; +use alloc::format; +use alloc::string::String; +use alloc::vec::Vec; +use burn_core::prelude::Backend; +use burn_tensor::Bytes; + +/// Store mode for BurnpackStore +enum StoreMode { + #[cfg(feature = "std")] + File(PathBuf), + Bytes(Option), +} + +/// BurnpackStore - A Burn-specific file format store using CBOR for metadata +pub struct BurnpackStore { + /// Store mode - either file path or bytes + mode: StoreMode, + /// Optional filter for selective loading/saving + filter: Option, + /// Additional metadata + metadata: BTreeMap, + /// Allow partial loading (ignore missing tensors) + allow_partial: bool, + /// Validate tensors during loading (check shapes and dtypes) + validate: bool, + /// Allow overwriting existing files (default: false) + overwrite: bool, + /// Enable zero-copy tensor loading (default: false) + /// + /// When enabled and the backend supports it, tensor data is sliced from + /// the source without copying. This requires keeping the source data alive. + zero_copy: bool, + /// Automatically append .bpk extension if not present (default: true) + #[cfg(feature = "std")] + auto_extension: bool, + /// Key remapper for tensor name transformations + #[cfg(feature = "std")] + remapper: KeyRemapper, + /// Writer for saving + writer: Option, + /// Reader for loading + reader: Option, + /// Cached tensor snapshots (parsed once, reused) + snapshots_cache: Option>, +} + +impl BurnpackStore { + /// Get the default metadata that includes Burn framework information. + /// + /// This includes: + /// - `format`: "burnpack" + /// - `producer`: "burn" + /// - `version`: The version of burn-store crate (from CARGO_PKG_VERSION) + /// + /// These metadata fields are automatically added to all saved models. + pub fn default_metadata() -> BTreeMap { + let mut metadata = BTreeMap::new(); + metadata.insert("format".into(), "burnpack".into()); + metadata.insert("producer".into(), "burn".into()); + metadata.insert("version".into(), env!("CARGO_PKG_VERSION").into()); + metadata + } + /// Create a new store from a file path + /// + /// By default, automatically appends `.bpk` extension if the path doesn't have one. + /// Use `.auto_extension(false)` to disable this behavior. + /// + /// # Examples + /// + /// ```no_run + /// # use burn_store::BurnpackStore; + /// // Automatically appends .bpk + /// let store = BurnpackStore::from_file("model"); // creates "model.bpk" + /// + /// // Already has extension, no append + /// let store = BurnpackStore::from_file("model.bpk"); // uses "model.bpk" + /// let store = BurnpackStore::from_file("model.myext"); // uses "model.myext" + /// + /// // Disable auto-extension + /// let store = BurnpackStore::from_file("model").auto_extension(false); // uses "model" + /// ``` + #[cfg(feature = "std")] + pub fn from_file>(path: P) -> Self { + Self { + mode: StoreMode::File(path.as_ref().to_path_buf()), + filter: None, + metadata: Self::default_metadata(), + allow_partial: false, + validate: true, + overwrite: false, + zero_copy: false, + #[cfg(feature = "std")] + auto_extension: true, + #[cfg(feature = "std")] + remapper: KeyRemapper::new(), + writer: None, + reader: None, + snapshots_cache: None, + } + } + + /// Create a new store from bytes (for reading) or empty (for writing) + pub fn from_bytes(bytes: Option) -> Self { + Self { + mode: StoreMode::Bytes(bytes), + filter: None, + metadata: Self::default_metadata(), + allow_partial: false, + validate: true, + overwrite: false, + zero_copy: false, + #[cfg(feature = "std")] + auto_extension: false, // Not used for bytes mode + #[cfg(feature = "std")] + remapper: KeyRemapper::new(), + writer: None, + reader: None, + snapshots_cache: None, + } + } + + /// Create a new store from static bytes with zero-copy loading enabled. + /// + /// This is optimized for embedded model weights where the data lives in the + /// binary's `.rodata` section. Tensor data is sliced without copying, keeping + /// the static reference alive. + /// + /// # Example + /// + /// ```ignore + /// static MODEL_DATA: &[u8] = include_bytes!("model.bpk"); + /// let store = BurnpackStore::from_static(MODEL_DATA); + /// ``` + pub fn from_static(data: &'static [u8]) -> Self { + use burn_tensor::AllocationProperty; + + // Create bytes::Bytes from static data (zero-copy, stays in .rodata) + let shared = bytes::Bytes::from_static(data); + + // Wrap in cubecl Bytes with shared-bytes allocation controller + let bytes = Bytes::from_shared(shared, AllocationProperty::Other); + + Self { + mode: StoreMode::Bytes(Some(bytes)), + filter: None, + metadata: Self::default_metadata(), + allow_partial: false, + validate: true, + overwrite: false, + zero_copy: true, // Enable zero-copy by default for static data + #[cfg(feature = "std")] + auto_extension: false, + #[cfg(feature = "std")] + remapper: KeyRemapper::new(), + writer: None, + reader: None, + snapshots_cache: None, + } + } + + /// Add metadata key-value pair + pub fn metadata(mut self, key: impl Into, value: impl Into) -> Self { + self.metadata.insert(key.into(), value.into()); + self + } + + /// Clear all metadata (including defaults) + /// + /// This removes all metadata including the default format, producer, and version fields. + /// Use with caution as some tools may expect these fields to be present. + pub fn clear_metadata(mut self) -> Self { + self.metadata.clear(); + self + } + + /// Allow partial loading (ignore missing tensors) + /// + /// When set to `true`, the store will not fail if some tensors are missing + /// during loading. This is useful when loading a subset of a model's parameters. + /// + /// Default: `false` + pub fn allow_partial(mut self, allow: bool) -> Self { + self.allow_partial = allow; + self + } + + /// Enable or disable validation during loading + /// + /// When validation is enabled, the store will check that loaded tensors + /// match the expected shapes and data types. Disabling validation can + /// improve performance but may lead to runtime errors if data is corrupted. + /// + /// Default: `true` + pub fn validate(mut self, validate: bool) -> Self { + self.validate = validate; + self + } + + /// Allow overwriting existing files when saving + /// + /// When set to `false`, attempting to save to an existing file will result in an error. + /// When set to `true`, existing files will be overwritten without warning. + /// + /// Default: `false` + pub fn overwrite(mut self, overwrite: bool) -> Self { + self.overwrite = overwrite; + self + } + + /// Enable or disable zero-copy tensor loading. + /// + /// When enabled and the backend supports it (memory-backed with shared bytes), + /// tensor data is sliced from the source without copying. This keeps the source + /// data alive as long as any tensor holds a reference. + /// + /// Zero-copy is automatically enabled when using [`from_static`](Self::from_static). + /// Use this method to enable it for other memory-backed stores created with + /// [`from_bytes`](Self::from_bytes) when using `Bytes::from_shared()`. + /// + /// Default: `false` (except for `from_static` which defaults to `true`) + pub fn zero_copy(mut self, enable: bool) -> Self { + self.zero_copy = enable; + self + } + + /// Enable or disable automatic .bpk extension appending + /// + /// When enabled (default), automatically appends `.bpk` to the file path + /// if no extension is detected. If an extension is already present, it is preserved. + /// + /// When disabled, uses the exact path provided without modification. + /// + /// Default: `true` + /// + /// # Examples + /// + /// ```no_run + /// # use burn_store::BurnpackStore; + /// // With auto_extension enabled (default) + /// let store = BurnpackStore::from_file("model"); // -> "model.bpk" + /// + /// // With auto_extension disabled + /// let store = BurnpackStore::from_file("model") + /// .auto_extension(false); // -> "model" + /// ``` + #[cfg(feature = "std")] + pub fn auto_extension(mut self, enable: bool) -> Self { + self.auto_extension = enable; + self + } + + /// Set path filter for selective loading/saving + pub fn with_filter(mut self, filter: PathFilter) -> Self { + self.filter = Some(filter); + self + } + + /// Add regex pattern to filter + #[cfg(feature = "std")] + pub fn with_regex(mut self, pattern: &str) -> Self { + let filter = self.filter.unwrap_or_default(); + self.filter = Some(filter.with_regex(pattern)); + self + } + + /// Add exact path to filter + pub fn with_full_path(mut self, path: impl Into) -> Self { + let filter = self.filter.unwrap_or_default(); + self.filter = Some(filter.with_full_path(path)); + self + } + + /// Match all tensors (no filtering) + pub fn match_all(mut self) -> Self { + self.filter = Some(PathFilter::new().match_all()); + self + } + + /// Set key remapper for tensor name transformations during loading + #[cfg(feature = "std")] + pub fn remap(mut self, remapper: KeyRemapper) -> Self { + self.remapper = remapper; + self + } + + /// Add a single regex pattern for key remapping + #[cfg(feature = "std")] + pub fn with_remap_pattern(mut self, from: S1, to: S2) -> Self + where + S1: AsRef, + S2: Into, + { + self.remapper = self + .remapper + .add_pattern(from.as_ref(), to.into()) + .expect("Invalid regex pattern"); + self + } + + /// Set the path filter + pub fn filter(mut self, filter: PathFilter) -> Self { + self.filter = Some(filter); + self + } + + /// Get the bytes after writing (only valid for bytes mode after collecting) + pub fn get_bytes(&self) -> Result { + if let Some(writer) = &self.writer { + return writer.to_bytes(); + } + + match &self.mode { + StoreMode::Bytes(Some(bytes)) => Ok(bytes.clone()), + _ => Err(BurnpackError::IoError("No bytes available".into())), + } + } + + /// Process the file path with auto-extension logic + #[cfg(feature = "std")] + fn process_path(&self, path: &std::path::Path) -> PathBuf { + if !self.auto_extension { + return path.to_path_buf(); + } + + // Check if path already has an extension + if path.extension().is_some() { + // Has extension, use as-is + return path.to_path_buf(); + } + + // No extension, append .bpk + let mut new_path = path.to_path_buf(); + new_path.set_extension("bpk"); + new_path + } + + /// Ensure the reader is initialized, loading from storage if needed + fn ensure_reader(&mut self) -> Result<&BurnpackReader, BurnpackError> { + if self.reader.is_none() { + let reader = match &self.mode { + #[cfg(feature = "std")] + StoreMode::File(path) => { + let final_path = self.process_path(path); + BurnpackReader::from_file(&final_path)? + } + StoreMode::Bytes(Some(bytes)) => BurnpackReader::from_bytes(bytes.clone())?, + StoreMode::Bytes(None) => { + return Err(BurnpackError::IoError("No bytes to read from".into())); + } + }; + self.reader = Some(reader); + } + + self.reader + .as_ref() + .ok_or_else(|| BurnpackError::IoError("Reader not initialized".into())) + } +} + +impl ModuleStore for BurnpackStore { + type Error = BurnpackError; + + fn collect_from>( + &mut self, + module: &M, + ) -> Result<(), Self::Error> { + // Invalidate cache since we're writing new data + self.snapshots_cache = None; + self.reader = None; + + // Collect snapshots from module + let snapshots = module.collect(self.filter.clone(), None, false); + + // Initialize writer with snapshots + let mut writer = BurnpackWriter::new(snapshots); + + // Add metadata using builder pattern + for (key, value) in &self.metadata { + writer = writer.with_metadata(key.as_str(), value.as_str()); + } + + // Store the writer for finalization + self.writer = Some(writer); + + // Write to storage based on mode + if let Some(writer) = &self.writer { + match &self.mode { + #[cfg(feature = "std")] + StoreMode::File(path) => { + // Process path with auto-extension logic + let final_path = self.process_path(path); + + // Check if file exists and overwrite is disabled + if final_path.exists() && !self.overwrite { + return Err(BurnpackError::IoError(format!( + "File already exists: {}. Use .overwrite(true) to overwrite.", + final_path.display() + ))); + } + writer.write_to_file(&final_path)?; + } + StoreMode::Bytes(_) => { + // Generate and store the bytes + let bytes_data = writer.to_bytes()?; + // Update mode with bytes - this pattern is irrefutable in no-std mode + #[cfg_attr(not(feature = "std"), allow(irrefutable_let_patterns))] + let StoreMode::Bytes(bytes_ref) = &mut self.mode else { + unreachable!("We just matched Bytes variant"); + }; + *bytes_ref = Some(bytes_data); + } + } + } + + Ok(()) + } + + fn apply_to>( + &mut self, + module: &mut M, + ) -> Result { + // Get all snapshots using the cached method + let snapshots: Vec = self.get_all_snapshots()?.values().cloned().collect(); + + // Apply all snapshots at once to the module + // Burnpack is Burn's native format, so no enum variant skipping needed + // Filter is applied here during apply, not during cache population + let result = module.apply(snapshots, self.filter.clone(), None, false); + + // Validate if needed + if self.validate && !result.errors.is_empty() { + return Err(BurnpackError::ValidationError(format!( + "Import errors: {:?}", + result.errors + ))); + } + + // Check for missing tensors if partial loading is not allowed + if !self.allow_partial && !result.missing.is_empty() { + return Err(BurnpackError::ValidationError(format!( + "Missing tensors: {:?}", + result.missing + ))); + } + + Ok(result) + } + + fn get_snapshot(&mut self, name: &str) -> Result, Self::Error> { + // Ensure cache is populated + self.ensure_snapshots_cache()?; + Ok(self.snapshots_cache.as_ref().unwrap().get(name)) + } + + fn get_all_snapshots(&mut self) -> Result<&BTreeMap, Self::Error> { + // Ensure cache is populated + self.ensure_snapshots_cache()?; + Ok(self.snapshots_cache.as_ref().unwrap()) + } + + fn keys(&mut self) -> Result, Self::Error> { + // Always use the cache to ensure remapping is applied consistently + Ok(self.get_all_snapshots()?.keys().cloned().collect()) + } +} + +impl BurnpackStore { + /// Ensure the snapshots cache is populated + fn ensure_snapshots_cache(&mut self) -> Result<(), BurnpackError> { + if self.snapshots_cache.is_some() { + return Ok(()); + } + + // Ensure reader is loaded + self.ensure_reader()?; + + // Get snapshots from reader with zero-copy if enabled + let reader = self.reader.as_ref().unwrap(); + let snapshots = reader.get_snapshots_zero_copy(self.zero_copy)?; + + // Apply remapping if configured (but NOT filtering - that's done at apply time) + #[cfg(feature = "std")] + let snapshots = if !self.remapper.patterns.is_empty() { + let (remapped, _remapped_names) = self.remapper.remap(snapshots); + remapped + } else { + snapshots + }; + + // Build the cache as BTreeMap + let cache: BTreeMap = + snapshots.into_iter().map(|s| (s.full_path(), s)).collect(); + + self.snapshots_cache = Some(cache); + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/alignment.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/alignment.rs new file mode 100644 index 0000000..5367d75 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/alignment.rs @@ -0,0 +1,434 @@ +//! Tests for tensor data alignment in burnpack format. +//! +//! These tests verify that tensor data is properly aligned for mmap zero-copy access. + +use crate::TensorSnapshot; +use crate::burnpack::{ + base::{ + BurnpackHeader, BurnpackMetadata, HEADER_SIZE, TENSOR_ALIGNMENT, aligned_data_section_start, + }, + reader::BurnpackReader, + writer::BurnpackWriter, +}; +use burn_core::module::ParamId; +use burn_tensor::{DType, TensorData}; + +/// Verify that aligned_data_section_start always returns 256-byte aligned values +#[test] +fn test_aligned_data_section_start_is_always_aligned() { + // Test various metadata sizes + for metadata_size in 0..1024 { + let result = aligned_data_section_start(metadata_size); + assert_eq!( + result % TENSOR_ALIGNMENT as usize, + 0, + "aligned_data_section_start({}) = {} is not aligned to {}", + metadata_size, + result, + TENSOR_ALIGNMENT + ); + } +} + +/// Verify data section starts at correct aligned position +#[test] +fn test_data_section_alignment() { + // Create a tensor + let data = [1.0f32, 2.0, 3.0, 4.0]; + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![4], DType::F32), + vec!["tensor".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let file_bytes = writer.to_bytes().unwrap(); + + // Parse header to get metadata size + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + + // Verify data section starts at 256-byte aligned position + assert_eq!( + data_section_start % TENSOR_ALIGNMENT as usize, + 0, + "Data section start {} is not 256-byte aligned", + data_section_start + ); + + // Verify the file is large enough + assert!( + file_bytes.len() >= data_section_start, + "File too small: {} < {}", + file_bytes.len(), + data_section_start + ); +} + +/// Verify that first tensor's absolute file position is 256-byte aligned +#[test] +fn test_first_tensor_absolute_position_aligned() { + let data: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![8], DType::U8), + vec!["first".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let file_bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&file_bytes[HEADER_SIZE..metadata_end]).unwrap(); + + let tensor_desc = metadata.tensors.get("first").unwrap(); + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + + // Absolute file position of first tensor + let absolute_pos = data_section_start + tensor_desc.data_offsets.0 as usize; + + assert_eq!( + absolute_pos % TENSOR_ALIGNMENT as usize, + 0, + "First tensor absolute position {} is not 256-byte aligned", + absolute_pos + ); +} + +/// Verify that all tensors in a multi-tensor file have 256-byte aligned absolute positions +#[test] +fn test_all_tensors_absolute_positions_aligned() { + // Create multiple tensors of different sizes (all U8 to simplify shape calculation) + let tensors = vec![ + ("tensor_a", vec![1u8, 2, 3]), // 3 bytes + ("tensor_b", vec![0u8; 16]), // 16 bytes + ("tensor_c", vec![0u8; 64]), // 64 bytes + ("tensor_d", vec![42u8]), // 1 byte + ("tensor_e", vec![0u8; 400]), // 400 bytes + ]; + + let snapshots: Vec = tensors + .into_iter() + .map(|(name, data)| { + let len = data.len(); + TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![len], DType::U8), + vec![name.to_string()], + vec![], + ParamId::new(), + ) + }) + .collect(); + + let writer = BurnpackWriter::new(snapshots); + let file_bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&file_bytes[HEADER_SIZE..metadata_end]).unwrap(); + + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + + // Check every tensor has aligned absolute position + for (name, desc) in &metadata.tensors { + let absolute_pos = data_section_start + desc.data_offsets.0 as usize; + assert_eq!( + absolute_pos % TENSOR_ALIGNMENT as usize, + 0, + "Tensor '{}' at absolute position {} is not 256-byte aligned (offset in data section: {})", + name, + absolute_pos, + desc.data_offsets.0 + ); + } +} + +/// Test edge case: metadata size that results in no padding needed +#[test] +fn test_alignment_with_minimal_padding() { + // We can't control metadata size directly, but we can verify the math works + // When HEADER_SIZE + metadata_size is already a multiple of 256, no padding needed + let aligned_metadata_size = TENSOR_ALIGNMENT as usize - HEADER_SIZE; // 256 - 10 = 246 + + let result = aligned_data_section_start(aligned_metadata_size); + assert_eq!(result, TENSOR_ALIGNMENT as usize); // Should be exactly 256 + + // One byte more should still round up to 256 + let result_plus_one = aligned_data_section_start(aligned_metadata_size + 1); + assert_eq!(result_plus_one, 2 * TENSOR_ALIGNMENT as usize); // Should be 512 +} + +/// Verify padding bytes in the file are zeros +#[test] +fn test_padding_bytes_are_zeros() { + let data: Vec = vec![0xAA; 16]; // Distinctive pattern + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), vec![16], DType::U8), + vec!["tensor".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let file_bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + + // Check padding between metadata and data section + if data_section_start > metadata_end { + let padding = &file_bytes[metadata_end..data_section_start]; + assert!( + padding.iter().all(|&b| b == 0), + "Padding bytes between metadata and data section contain non-zero values" + ); + } +} + +/// Verify alignment is sufficient for all primitive types +/// 256-byte alignment is a multiple of all primitive type alignments: +/// - f64/i64/u64: 8 bytes +/// - f32/i32/u32: 4 bytes +/// - f16/bf16/i16/u16: 2 bytes +/// - i8/u8/bool: 1 byte +#[test] +#[allow(clippy::modulo_one)] +fn test_alignment_covers_all_primitive_types() { + // 256 must be divisible by all common alignments + assert_eq!( + TENSOR_ALIGNMENT % 8, + 0, + "256 not divisible by 8 (f64 alignment)" + ); + assert_eq!( + TENSOR_ALIGNMENT % 4, + 0, + "256 not divisible by 4 (f32 alignment)" + ); + assert_eq!( + TENSOR_ALIGNMENT % 2, + 0, + "256 not divisible by 2 (f16 alignment)" + ); + assert_eq!( + TENSOR_ALIGNMENT % 1, + 0, + "256 not divisible by 1 (u8 alignment)" + ); +} + +/// Verify that tensor data can be read correctly after alignment +#[test] +fn test_aligned_tensor_data_readable() { + // Create f32 tensor + let f32_data = vec![1.0f32, 2.0, 3.0, 4.0]; + let f32_bytes: Vec = f32_data.iter().flat_map(|f| f.to_le_bytes()).collect(); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f32_bytes.clone(), vec![4], DType::F32), + vec!["floats".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let file_bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&file_bytes[HEADER_SIZE..metadata_end]).unwrap(); + + let tensor_desc = metadata.tensors.get("floats").unwrap(); + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + + let start = data_section_start + tensor_desc.data_offsets.0 as usize; + let end = data_section_start + tensor_desc.data_offsets.1 as usize; + let tensor_bytes = &file_bytes[start..end]; + + // Verify the bytes match what we wrote + assert_eq!(tensor_bytes, f32_bytes.as_slice()); + + // Verify we can interpret them as floats + let mut floats = Vec::new(); + for chunk in tensor_bytes.chunks_exact(4) { + floats.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + assert_eq!(floats, f32_data); +} + +/// Verify alignment works with f64 data +#[test] +fn test_aligned_f64_tensor_data_readable() { + let f64_data = vec![1.0f64, 2.0, 3.0, 4.0]; + let f64_bytes: Vec = f64_data.iter().flat_map(|f| f.to_le_bytes()).collect(); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f64_bytes.clone(), vec![4], DType::F64), + vec!["doubles".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let file_bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&file_bytes[HEADER_SIZE..metadata_end]).unwrap(); + + let tensor_desc = metadata.tensors.get("doubles").unwrap(); + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + + let start = data_section_start + tensor_desc.data_offsets.0 as usize; + let end = data_section_start + tensor_desc.data_offsets.1 as usize; + let tensor_bytes = &file_bytes[start..end]; + + // Verify the bytes match + assert_eq!(tensor_bytes, f64_bytes.as_slice()); + + // Verify we can interpret them as doubles + let mut doubles = Vec::new(); + for chunk in tensor_bytes.chunks_exact(8) { + doubles.push(f64::from_le_bytes([ + chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], + ])); + } + assert_eq!(doubles, f64_data); +} + +/// Test round-trip preserves alignment (write then read) +#[test] +fn test_round_trip_maintains_alignment() { + let f32_data = vec![1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let f32_bytes: Vec = f32_data.iter().flat_map(|f| f.to_le_bytes()).collect(); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f32_bytes, vec![2, 4], DType::F32), + vec!["matrix".to_string()], + vec![], + ParamId::new(), + ); + + // Write + let writer = BurnpackWriter::new(vec![snapshot]); + let file_bytes = writer.to_bytes().unwrap(); + + // Read back + let reader = BurnpackReader::from_bytes(file_bytes.clone()).unwrap(); + let snapshots = reader.get_snapshots().unwrap(); + + assert_eq!(snapshots.len(), 1); + let loaded = &snapshots[0]; + assert_eq!(loaded.full_path(), "matrix"); + + // Verify the loaded data is correct + let tensor_data = loaded.to_data().unwrap(); + let mut loaded_floats = Vec::new(); + for chunk in tensor_data.bytes.chunks_exact(4) { + loaded_floats.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + assert_eq!(loaded_floats, f32_data); +} + +/// Test that tensor offsets within data section are also aligned +#[test] +fn test_tensor_relative_offsets_are_aligned() { + // Create several small tensors to force multiple alignment padding + let tensors: Vec<_> = (0..5) + .map(|i| { + let data = vec![i as u8; 7]; // 7 bytes each - not aligned + TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![7], DType::U8), + vec![format!("tensor_{}", i)], + vec![], + ParamId::new(), + ) + }) + .collect(); + + let writer = BurnpackWriter::new(tensors); + let file_bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&file_bytes[HEADER_SIZE..metadata_end]).unwrap(); + + // All tensor start offsets within data section should be multiples of 256 + for (name, desc) in &metadata.tensors { + assert_eq!( + desc.data_offsets.0 % TENSOR_ALIGNMENT, + 0, + "Tensor '{}' relative offset {} is not 256-byte aligned", + name, + desc.data_offsets.0 + ); + } +} + +#[cfg(feature = "std")] +mod file_tests { + use super::*; + use std::fs; + use tempfile::tempdir; + + /// Test alignment is preserved when writing to and reading from file + #[test] + fn test_file_io_preserves_alignment() { + let dir = tempdir().unwrap(); + let file_path = dir.path().join("aligned.bpk"); + + let f32_data = [1.0f32, 2.0, 3.0, 4.0]; + let f32_bytes: Vec = f32_data.iter().flat_map(|f| f.to_le_bytes()).collect(); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f32_bytes, vec![4], DType::F32), + vec!["floats".to_string()], + vec![], + ParamId::new(), + ); + + // Write to file + let writer = BurnpackWriter::new(vec![snapshot]); + writer.write_to_file(&file_path).unwrap(); + + // Read file bytes directly + let file_bytes = fs::read(&file_path).unwrap(); + + let header = BurnpackHeader::from_bytes(&file_bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&file_bytes[HEADER_SIZE..metadata_end]).unwrap(); + + let tensor_desc = metadata.tensors.get("floats").unwrap(); + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + let absolute_pos = data_section_start + tensor_desc.data_offsets.0 as usize; + + assert_eq!( + absolute_pos % TENSOR_ALIGNMENT as usize, + 0, + "Tensor absolute position in file {} is not 256-byte aligned", + absolute_pos + ); + + // Verify data is correct + let start = data_section_start + tensor_desc.data_offsets.0 as usize; + let end = data_section_start + tensor_desc.data_offsets.1 as usize; + let tensor_bytes = &file_bytes[start..end]; + + let mut floats = Vec::new(); + for chunk in tensor_bytes.chunks_exact(4) { + floats.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + assert_eq!(floats, vec![1.0f32, 2.0, 3.0, 4.0]); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/edge_cases.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/edge_cases.rs new file mode 100644 index 0000000..3c257a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/edge_cases.rs @@ -0,0 +1,365 @@ +use crate::TensorSnapshot; +use crate::burnpack::{ + base::{BurnpackHeader, HEADER_SIZE}, + reader::BurnpackReader, + writer::BurnpackWriter, +}; +use burn_core::module::ParamId; +use burn_tensor::{DType, TensorData}; + +#[test] +fn test_maximum_metadata_size() { + // Create metadata that approaches u32::MAX (4GB limit) + // In practice, we'll test with a reasonably large metadata + let large_key = "x".repeat(1000); + let large_value = "y".repeat(10000); + + let mut writer = BurnpackWriter::new(vec![]); + + for i in 0..100 { + writer = writer.with_metadata(&format!("{}_{}", large_key, i), &large_value); + } + + let result = writer.to_bytes(); + assert!(result.is_ok()); + + let bytes = result.unwrap(); + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + + // Metadata size should be large but within u32 bounds + assert!(header.metadata_size > 1000000); // At least 1MB of metadata + assert!(header.metadata_size < u32::MAX); +} + +#[test] +fn test_zero_size_tensor_shapes() { + // Test various zero-dimensional shapes + let test_cases = [ + (vec![0], vec![]), // Empty 1D + (vec![0, 10], vec![]), // Zero rows + (vec![10, 0], vec![]), // Zero columns + (vec![0, 0], vec![]), // Zero both dimensions + (vec![5, 0, 10], vec![]), // Zero in middle dimension + ]; + + let mut snapshots = vec![]; + for (i, (shape, data)) in test_cases.iter().enumerate() { + let name = format!("zero_tensor_{}", i); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), shape.clone(), DType::F32), + vec![name.clone()], + vec![], + ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + + // Read back and verify + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let names = reader.tensor_names(); + assert_eq!(names.len(), 5); +} + +#[test] +fn test_extremely_long_tensor_names() { + // Create a tensor with an extremely long name + let long_name = "a".repeat(10000); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![4], DType::U8), + vec![long_name.clone()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let names = reader.tensor_names(); + assert_eq!(names[0].len(), 10000); +} + +#[test] +fn test_unicode_in_names_and_metadata() { + // Test various Unicode characters in tensor names and metadata + let unicode_names = vec![ + "测试_tensor", // Chinese + "тест_tensor", // Cyrillic + "テスト_tensor", // Japanese + "🔥_burn_tensor", // Emoji + "αβγδ_tensor", // Greek + "한글_tensor", // Korean + ]; + + let mut snapshots = vec![]; + for name in &unicode_names { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1], vec![1], DType::U8), + vec![name.to_string()], + vec![], + ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots) + .with_metadata("模型名称", "测试模型") + .with_metadata("מודל", "בדיקה") + .with_metadata("🔥", "fire"); + + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Verify all Unicode names are preserved + let names = reader.tensor_names(); + assert_eq!(names.len(), unicode_names.len()); + + // Verify metadata + assert_eq!( + reader.metadata().metadata.get("模型名称"), + Some(&"测试模型".to_string()) + ); + assert_eq!( + reader.metadata().metadata.get("🔥"), + Some(&"fire".to_string()) + ); +} + +#[test] +fn test_all_supported_dtypes() { + // Test all DTypes with their boundary values + let dtypes_with_data = [ + ( + DType::F32, + [ + f32::MIN.to_le_bytes().to_vec(), + f32::MAX.to_le_bytes().to_vec(), + ] + .concat(), + ), + ( + DType::F64, + [ + f64::MIN.to_le_bytes().to_vec(), + f64::MAX.to_le_bytes().to_vec(), + ] + .concat(), + ), + ( + DType::I32, + [ + i32::MIN.to_le_bytes().to_vec(), + i32::MAX.to_le_bytes().to_vec(), + ] + .concat(), + ), + ( + DType::I64, + [ + i64::MIN.to_le_bytes().to_vec(), + i64::MAX.to_le_bytes().to_vec(), + ] + .concat(), + ), + ( + DType::U32, + [ + u32::MIN.to_le_bytes().to_vec(), + u32::MAX.to_le_bytes().to_vec(), + ] + .concat(), + ), + ( + DType::U64, + [ + u64::MIN.to_le_bytes().to_vec(), + u64::MAX.to_le_bytes().to_vec(), + ] + .concat(), + ), + (DType::U8, vec![u8::MIN, u8::MAX]), + (DType::Bool, vec![0, 1]), + ]; + + let mut snapshots = vec![]; + for (i, (dtype, data)) in dtypes_with_data.iter().enumerate() { + let name = format!("dtype_test_{}", i); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), vec![2], *dtype), + vec![name], + vec![], + ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + assert_eq!(reader.tensor_names().len(), dtypes_with_data.len()); + + // Verify dtypes are preserved + for (i, (expected_dtype, _)) in dtypes_with_data.iter().enumerate() { + let name = format!("dtype_test_{}", i); + let snapshot = reader.get_tensor_snapshot(&name).unwrap(); + assert_eq!(snapshot.dtype, *expected_dtype); + } +} + +#[test] +fn test_special_float_values() { + // Test special floating-point values (NaN, Inf, -Inf) + let special_values = [ + f32::NAN, + f32::INFINITY, + f32::NEG_INFINITY, + 0.0_f32, + -0.0_f32, + ]; + + let data: Vec = special_values + .iter() + .flat_map(|f| f.to_le_bytes()) + .collect(); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), vec![5], DType::F32), + vec!["special_floats".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let tensor_data = reader.get_tensor_data("special_floats").unwrap(); + + // Check data is preserved exactly (bit-for-bit) + assert_eq!(tensor_data, data); +} + +#[test] +fn test_metadata_with_empty_values() { + let writer = BurnpackWriter::new(vec![]) + .with_metadata("empty_value", "") + .with_metadata("", "empty_key") + .with_metadata("normal", "value"); + + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + let metadata = &reader.metadata().metadata; + assert_eq!(metadata.get("empty_value"), Some(&"".to_string())); + assert_eq!(metadata.get(""), Some(&"empty_key".to_string())); + assert_eq!(metadata.get("normal"), Some(&"value".to_string())); +} + +#[test] +fn test_single_byte_tensor() { + // Test the smallest possible tensor (1 byte) + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![42], vec![1], DType::U8), + vec!["single_byte".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let data = reader.get_tensor_data("single_byte").unwrap(); + assert_eq!(data, vec![42]); +} + +#[test] +fn test_high_dimensional_tensor() { + // Test a tensor with many dimensions (10D) + let shape = vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2]; // 10 dimensions, 1024 elements total + let data = vec![1u8; 1024]; + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), shape.clone(), DType::U8), + vec!["high_dim".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let loaded_snapshot = reader.get_tensor_snapshot("high_dim").unwrap(); + assert_eq!(loaded_snapshot.shape, shape); +} + +#[test] +fn test_metadata_key_collision() { + // Test that later values override earlier ones for the same key + let writer = BurnpackWriter::new(vec![]) + .with_metadata("key", "value1") + .with_metadata("key", "value2") + .with_metadata("key", "value3"); + + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + assert_eq!( + reader.metadata().metadata.get("key"), + Some(&"value3".to_string()) + ); +} + +#[test] +fn test_tensor_name_with_path_separators() { + // Test tensor names that look like file paths + let path_like_names = vec![ + "model/encoder/layer1/weights", + "model\\decoder\\layer1\\bias", + "model::module::param", + "model.submodule.weight", + ]; + + let mut snapshots = vec![]; + for name in &path_like_names { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![4], DType::U8), + vec![name.to_string()], + vec![], + ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let names = reader.tensor_names(); + + // All names should be preserved exactly + for expected_name in &path_like_names { + assert!(names.contains(expected_name)); + } +} + +// The following tests are commented out as they test error conditions +// that might be handled differently in the new API + +// #[test] +// fn test_data_overflow_protection() { +// // Test that we handle potential integer overflows in offset calculations +// ... +// } + +// #[test] +// fn test_reading_corrupted_header() { +// // Test reading files with corrupted headers +// ... +// } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/header.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/header.rs new file mode 100644 index 0000000..a7ac2ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/header.rs @@ -0,0 +1,61 @@ +use crate::burnpack::base::*; + +#[test] +fn test_header_serialization() { + let header = BurnpackHeader::new(12345); + + // Check fields + assert_eq!(header.magic, MAGIC_NUMBER); + assert_eq!(header.version, FORMAT_VERSION); + assert_eq!(header.metadata_size, 12345); + + // Serialize to bytes + let bytes = header.into_bytes(); + assert_eq!(bytes.len(), HEADER_SIZE); + + // Deserialize back + let header2 = BurnpackHeader::from_bytes(&bytes).unwrap(); + assert_eq!(header2.magic, header.magic); + assert_eq!(header2.version, header.version); + assert_eq!(header2.metadata_size, header.metadata_size); +} + +#[test] +fn test_header_invalid_magic() { + let mut bytes = [0u8; HEADER_SIZE]; + // Write wrong magic number + bytes[0..4].copy_from_slice(&[0x00, 0x00, 0x00, 0x00]); + + let result = BurnpackHeader::from_bytes(&bytes); + match result { + Err(BurnpackError::InvalidMagicNumber) => {} + _ => panic!("Expected InvalidMagicNumber error"), + } +} + +#[test] +fn test_header_insufficient_bytes() { + let bytes = [0u8; 5]; // Too short + + let result = BurnpackHeader::from_bytes(&bytes); + match result { + Err(BurnpackError::InvalidHeader) => {} + _ => panic!("Expected InvalidHeader error"), + } +} + +#[test] +fn test_version_compatibility() { + // Create a header with current version + let header = BurnpackHeader::new(100); + let bytes = header.into_bytes(); + + // Should succeed with current version + let result = BurnpackHeader::from_bytes(&bytes); + assert!(result.is_ok()); + + // Test with future version (should fail in real implementation) + // For now, we just verify the version field is correctly set + let header = result.unwrap(); + assert_eq!(header.version, FORMAT_VERSION); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/helpers.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/helpers.rs new file mode 100644 index 0000000..3f13d7b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/helpers.rs @@ -0,0 +1,19 @@ +use crate::TensorSnapshot; +use burn_core::module::ParamId; +use burn_tensor::{DType, TensorData}; + +/// Helper to create a test TensorSnapshot +#[allow(dead_code)] +pub fn create_test_snapshot( + name: String, + data: Vec, + shape: Vec, + dtype: DType, +) -> TensorSnapshot { + TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, shape, dtype), + vec![name], + vec![], + ParamId::new(), + ) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/mod.rs new file mode 100644 index 0000000..d6f3890 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/mod.rs @@ -0,0 +1,11 @@ +use crate::TensorSnapshot; + +mod alignment; +mod edge_cases; +mod header; +mod helpers; +mod reader; +mod round_trip; +mod store; +mod writer; +mod zero_copy; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/reader.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/reader.rs new file mode 100644 index 0000000..8936313 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/reader.rs @@ -0,0 +1,775 @@ +use crate::burnpack::{ + base::{ + BurnpackError, FORMAT_VERSION, HEADER_SIZE, MAGIC_NUMBER, magic_range, metadata_size_range, + version_range, + }, + reader::BurnpackReader, + writer::BurnpackWriter, +}; + +use super::*; +use burn_tensor::{Bytes, DType, TensorData}; + +#[test] +fn test_reader_from_bytes_empty() { + // Create empty burnpack data + let writer = BurnpackWriter::new(Vec::new()); + let bytes = writer.to_bytes().unwrap(); + + // Read it back + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + assert_eq!(reader.metadata().tensors.len(), 0); + assert!(reader.metadata().metadata.is_empty()); +} + +#[test] +fn test_reader_from_bytes_with_data() { + // Create test data + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["test_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]).with_metadata("test", "value"); + + let bytes = writer.to_bytes().unwrap(); + + // Read it back + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + assert_eq!(reader.metadata().tensors.len(), 1); + assert_eq!( + reader.metadata().metadata.get("test"), + Some(&"value".to_string()) + ); + + // Get tensor data + let tensor_data = reader.get_tensor_data("test_tensor").unwrap(); + assert_eq!(tensor_data, &[1, 2, 3, 4]); +} + +#[test] +fn test_reader_invalid_magic_number() { + let mut bytes = vec![0u8; 100]; + // Write invalid magic number + bytes[magic_range()].copy_from_slice(b"NOPE"); + + let result = BurnpackReader::from_bytes(Bytes::from_bytes_vec(bytes)); + assert!(matches!(result, Err(BurnpackError::InvalidMagicNumber))); +} + +#[test] +fn test_reader_invalid_version() { + let mut bytes = vec![0u8; 100]; + // Write correct magic but invalid version + bytes[magic_range()].copy_from_slice(&MAGIC_NUMBER.to_le_bytes()); + bytes[version_range()].copy_from_slice(&999u16.to_le_bytes()); // Invalid version + bytes[metadata_size_range()].copy_from_slice(&10u32.to_le_bytes()); // Metadata size + + let result = BurnpackReader::from_bytes(Bytes::from_bytes_vec(bytes)); + assert!(matches!(result, Err(BurnpackError::InvalidVersion))); +} + +#[test] +fn test_reader_header_too_short() { + let bytes = vec![0u8; 5]; // Less than HEADER_SIZE + + let result = BurnpackReader::from_bytes(Bytes::from_bytes_vec(bytes)); + assert!(matches!(result, Err(BurnpackError::InvalidHeader))); +} + +#[test] +fn test_reader_metadata_truncated() { + let mut bytes = vec![0u8; HEADER_SIZE + 10]; + // Write valid header + bytes[magic_range()].copy_from_slice(&MAGIC_NUMBER.to_le_bytes()); + bytes[version_range()].copy_from_slice(&FORMAT_VERSION.to_le_bytes()); + bytes[metadata_size_range()].copy_from_slice(&100u32.to_le_bytes()); // Claims 100 bytes of metadata + + // But only provide 10 bytes after header + let result = BurnpackReader::from_bytes(Bytes::from_bytes_vec(bytes)); + assert!(matches!(result, Err(BurnpackError::InvalidHeader))); +} + +#[test] +fn test_reader_get_tensor_not_found() { + let writer = BurnpackWriter::new(Vec::new()); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + let result = reader.get_tensor_data("non_existent"); + assert!(matches!(result, Err(BurnpackError::TensorNotFound(_)))); +} + +#[test] +fn test_reader_get_tensor_snapshot() { + let data = [1.0f32, 2.0, 3.0, 4.0]; + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![2, 2], DType::F32), + vec!["weights".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let writer_bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(writer_bytes).unwrap(); + + // Get tensor as snapshot + let loaded_snapshot = reader.get_tensor_snapshot("weights").unwrap(); + + // Verify snapshot metadata + assert_eq!(loaded_snapshot.full_path(), "weights"); + assert_eq!(loaded_snapshot.dtype, DType::F32); + assert_eq!(loaded_snapshot.shape, vec![2, 2]); + + // Verify data through closure + let tensor_data = loaded_snapshot.to_data().unwrap(); + assert_eq!(tensor_data.shape, vec![2, 2]); +} + +#[test] +fn test_reader_multiple_tensors() { + // Add multiple tensors + let mut snapshots = Vec::new(); + for i in 0..10 { + let name = format!("tensor_{}", i); + let data = vec![i as u8; 100]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![100], DType::U8), + vec![name.clone()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Verify all tensors can be read + for i in 0..10 { + let name = format!("tensor_{}", i); + let data = reader.get_tensor_data(&name).unwrap(); + assert_eq!(data.len(), 100); + assert!(data.iter().all(|&b| b == i as u8)); + } +} + +#[test] +fn test_reader_lazy_loading() { + // Create large tensor + let size = 1024 * 1024; // 1MB + let data = vec![42u8; size]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), vec![size], DType::U8), + vec!["large".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Get snapshot (should be lazy) + let snapshot = reader.get_tensor_snapshot("large").unwrap(); + + // Data should only be accessed when to_data is called + let tensor_data = snapshot.to_data().unwrap(); + assert_eq!(tensor_data.bytes.len(), size); + assert!(tensor_data.bytes.iter().all(|&b| b == 42)); +} + +#[test] +fn test_reader_all_dtypes() { + // Test all data types + let test_data = [ + (DType::F32, [1.0f32.to_le_bytes().to_vec()].concat()), + (DType::F64, [2.0f64.to_le_bytes().to_vec()].concat()), + (DType::I32, [3i32.to_le_bytes().to_vec()].concat()), + (DType::I64, [4i64.to_le_bytes().to_vec()].concat()), + (DType::U32, [5u32.to_le_bytes().to_vec()].concat()), + (DType::U64, [6u64.to_le_bytes().to_vec()].concat()), + (DType::U8, vec![7u8]), + (DType::Bool, vec![1u8]), + ]; + + let mut snapshots = Vec::new(); + for (i, (dtype, data)) in test_data.iter().enumerate() { + let name = format!("tensor_{}", i); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), vec![1], *dtype), + vec![name.clone()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Verify all dtypes are preserved + for (i, (expected_dtype, expected_data)) in test_data.iter().enumerate() { + let name = format!("tensor_{}", i); + let snapshot = reader.get_tensor_snapshot(&name).unwrap(); + assert_eq!(snapshot.dtype, *expected_dtype); + + let data = reader.get_tensor_data(&name).unwrap(); + assert_eq!(data, expected_data.as_slice()); + } +} + +#[test] +fn test_reader_empty_tensor() { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![], vec![0], DType::F32), + vec!["empty".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + let data = reader.get_tensor_data("empty").unwrap(); + assert_eq!(data.len(), 0); + + let snapshot = reader.get_tensor_snapshot("empty").unwrap(); + assert_eq!(snapshot.shape, vec![0]); +} + +#[cfg(feature = "std")] +#[test] +fn test_reader_from_file() { + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("test.bpk"); + + // Create test file + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![10, 20, 30], vec![3], DType::U8), + vec!["file_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]).with_metadata("from_file_test", "true"); + + writer.write_to_file(&file_path).unwrap(); + + // Read from file + let reader = BurnpackReader::from_file(&file_path).unwrap(); + + assert_eq!( + reader.metadata().metadata.get("from_file_test"), + Some(&"true".to_string()) + ); + + let data = reader.get_tensor_data("file_tensor").unwrap(); + assert_eq!(data, &[10, 20, 30]); +} + +#[cfg(all(feature = "std", feature = "memmap"))] +#[test] +fn test_reader_from_file_mmap() { + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("test_mmap.bpk"); + + // Create large test file + let size = 1024 * 1024; // 1MB + let data = vec![99u8; size]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![size], DType::U8), + vec!["large_mmap".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + writer.write_to_file(&file_path).unwrap(); + + // Read using mmap + let reader = BurnpackReader::from_file_mmap(&file_path).unwrap(); + + let data = reader.get_tensor_data("large_mmap").unwrap(); + assert_eq!(data.len(), size); + assert!(data.iter().all(|&b| b == 99)); +} + +#[cfg(feature = "std")] +#[test] +fn test_reader_from_file_buffered() { + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("test_buffered.bpk"); + + // Create test file + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![5, 10, 15], vec![3], DType::U8), + vec!["buffered_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + writer.write_to_file(&file_path).unwrap(); + + // Read using buffered reader + let reader = BurnpackReader::from_file_buffered(&file_path).unwrap(); + + let data = reader.get_tensor_data("buffered_tensor").unwrap(); + assert_eq!(data, &[5, 10, 15]); +} + +#[test] +fn test_reader_metadata_access() { + // Add various metadata using builder pattern + let writer = BurnpackWriter::new(Vec::new()) + .with_metadata("model_name", "test_model") + .with_metadata("version", "1.2.3") + .with_metadata("author", "test_author") + .with_metadata("description", "A test model"); + + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + let metadata = reader.metadata(); + assert_eq!(metadata.metadata.len(), 4); + assert_eq!( + metadata.metadata.get("model_name"), + Some(&"test_model".to_string()) + ); + assert_eq!(metadata.metadata.get("version"), Some(&"1.2.3".to_string())); + assert_eq!( + metadata.metadata.get("author"), + Some(&"test_author".to_string()) + ); + assert_eq!( + metadata.metadata.get("description"), + Some(&"A test model".to_string()) + ); +} + +#[test] +fn test_reader_tensor_iteration() { + // Add tensors + let tensor_names = vec!["weights", "bias", "running_mean", "running_var"]; + let mut snapshots = Vec::new(); + for name in &tensor_names { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![4], DType::U8), + vec![name.to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Iterate through all tensors + let metadata = reader.metadata(); + assert_eq!(metadata.tensors.len(), 4); + + // Check that all expected tensor names are present + for name in &tensor_names { + let tensor_desc = metadata.tensors.get(*name).unwrap(); + assert_eq!(tensor_desc.shape, vec![4u64]); + assert_eq!(tensor_desc.dtype, DType::U8); + } + + // Verify the keys match the expected names + let mut actual_names: Vec<_> = metadata.tensors.keys().cloned().collect(); + actual_names.sort(); + let mut expected_names = tensor_names + .iter() + .map(|s| s.to_string()) + .collect::>(); + expected_names.sort(); + assert_eq!(actual_names, expected_names); +} + +#[test] +fn test_reader_corrupt_metadata() { + let mut bytes = vec![0u8; 100]; + + // Write valid header + bytes[magic_range()].copy_from_slice(&MAGIC_NUMBER.to_le_bytes()); + bytes[version_range()].copy_from_slice(&FORMAT_VERSION.to_le_bytes()); + bytes[metadata_size_range()].copy_from_slice(&50u32.to_le_bytes()); // 50 bytes of metadata + + // Write garbage as metadata + #[allow(clippy::needless_range_loop)] + for i in HEADER_SIZE..HEADER_SIZE + 50 { + bytes[i] = 0xFF; + } + + let result = BurnpackReader::from_bytes(Bytes::from_bytes_vec(bytes)); + assert!(result.is_err()); +} + +#[test] +fn test_reader_data_offsets_validation() { + // Add two tensors + let snapshot1 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![4], DType::U8), + vec!["tensor1".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + let snapshot2 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![5, 6, 7, 8], vec![4], DType::U8), + vec!["tensor2".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot1, snapshot2]); + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Verify offsets don't overlap and are properly aligned + let metadata = reader.metadata(); + let tensor1_desc = metadata.tensors.get("tensor1").unwrap(); + let tensor2_desc = metadata.tensors.get("tensor2").unwrap(); + + // First tensor starts at offset 0 (already aligned to 256 bytes) + assert_eq!(tensor1_desc.data_offsets, (0, 4)); + // Second tensor starts at next 256-byte aligned offset + assert_eq!(tensor2_desc.data_offsets, (256, 260)); +} + +#[test] +fn test_reader_out_of_bounds_error() { + use crate::burnpack::reader::StorageBackend; + use alloc::rc::Rc; + + // Create a small data buffer + let data = Bytes::from_bytes_vec(vec![1, 2, 3, 4, 5]); + let backend = StorageBackend::Memory(Rc::new(data)); + + // Try to read beyond the available data + let mut buffer = vec![0u8; 10]; + let result = backend.read_into(&mut buffer, 0); + + // Should return an error + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains("out of bounds")); +} + +#[test] +fn test_reader_offset_overflow_error() { + use crate::burnpack::reader::StorageBackend; + use alloc::rc::Rc; + + let data = Bytes::from_bytes_vec(vec![1, 2, 3, 4, 5]); + let backend = StorageBackend::Memory(Rc::new(data)); + + // Try to read with an offset that would overflow + let mut buffer = vec![0u8; 10]; + let result = backend.read_into(&mut buffer, usize::MAX - 5); + + // Should return an error about overflow + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains("overflow")); +} + +#[test] +fn test_reader_corrupted_shape_returns_error() { + // Only test this on platforms where usize is smaller than u64 + // On 64-bit platforms, u64 values can fit in usize + #[cfg(target_pointer_width = "32")] + { + use crate::burnpack::base::{BurnpackMetadata, TensorDescriptor}; + use alloc::collections::BTreeMap; + use alloc::rc::Rc; + use burn_tensor::DType; + + // Create metadata with a shape dimension that exceeds usize::MAX on 32-bit platforms + let mut tensors = BTreeMap::new(); + tensors.insert( + "corrupted_tensor".to_string(), + TensorDescriptor { + dtype: DType::F32, + shape: vec![u64::MAX, 2, 3], // First dimension exceeds usize::MAX on 32-bit + data_offsets: (0, 100), + param_id: None, + }, + ); + + let metadata = BurnpackMetadata { + tensors, + metadata: BTreeMap::new(), + }; + + // Create a small data buffer + let data = Bytes::from_bytes_vec(vec![0u8; 1000]); + let backend = crate::burnpack::reader::StorageBackend::Memory(Rc::new(data)); + + let reader = BurnpackReader { + metadata, + storage: backend, + data_offset: 0, + }; + + // This should return an error, not panic + let result = reader.get_snapshots(); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, BurnpackError::ValidationError(_))); + assert!( + err.to_string().contains("corrupted shape data") + || err.to_string().contains("exceeds platform maximum") + ); + } + + #[cfg(not(target_pointer_width = "32"))] + { + // On 64-bit platforms, just pass the test + // The conversion logic is still correct, but u64 fits in usize + } +} + +#[test] +fn test_reader_corrupted_offsets_returns_error() { + // Only test this on platforms where usize is smaller than u64 + #[cfg(target_pointer_width = "32")] + { + use crate::burnpack::base::{BurnpackMetadata, TensorDescriptor}; + use alloc::collections::BTreeMap; + use alloc::rc::Rc; + use burn_tensor::DType; + + // Create metadata with offsets that would overflow + let mut tensors = BTreeMap::new(); + tensors.insert( + "tensor_bad_offset".to_string(), + TensorDescriptor { + dtype: DType::F32, + shape: vec![2, 2], + data_offsets: (u64::MAX - 10, u64::MAX), // Offsets that exceed usize::MAX on 32-bit + param_id: None, + }, + ); + + let metadata = BurnpackMetadata { + tensors, + metadata: BTreeMap::new(), + }; + + let data = Bytes::from_bytes_vec(vec![0u8; 1000]); + let backend = crate::burnpack::reader::StorageBackend::Memory(Rc::new(data)); + + let reader = BurnpackReader { + metadata, + storage: backend, + data_offset: 0, + }; + + // This should return an error, not panic + let result = reader.get_snapshots(); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, BurnpackError::ValidationError(_))); + assert!( + err.to_string().contains("corrupted offset data") + || err.to_string().contains("exceeds platform maximum") + ); + } + + #[cfg(not(target_pointer_width = "32"))] + { + use crate::burnpack::base::{BurnpackMetadata, TensorDescriptor}; + use alloc::collections::BTreeMap; + use alloc::rc::Rc; + use burn_tensor::DType; + + // On 64-bit platforms, test offset overflow during addition + let mut tensors = BTreeMap::new(); + tensors.insert( + "tensor_overflow".to_string(), + TensorDescriptor { + dtype: DType::F32, + shape: vec![2, 2], + data_offsets: (0, 100), + param_id: None, + }, + ); + + let metadata = BurnpackMetadata { + tensors, + metadata: BTreeMap::new(), + }; + + let data = Bytes::from_bytes_vec(vec![0u8; 1000]); + let backend = crate::burnpack::reader::StorageBackend::Memory(Rc::new(data)); + + // Use a data_offset that will overflow when added to the tensor offset + let reader = BurnpackReader { + metadata, + storage: backend, + data_offset: usize::MAX - 50, // Will overflow when added to 100 + }; + + // This should return an error, not panic + let result = reader.get_snapshots(); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, BurnpackError::ValidationError(_))); + assert!(err.to_string().contains("overflow")); + } +} + +#[test] +fn test_reader_inverted_offsets_returns_error() { + use crate::burnpack::base::{BurnpackMetadata, TensorDescriptor}; + use alloc::collections::BTreeMap; + use alloc::rc::Rc; + use burn_tensor::DType; + + // Create metadata with end offset < start offset (corrupted) + let mut tensors = BTreeMap::new(); + tensors.insert( + "inverted_tensor".to_string(), + TensorDescriptor { + dtype: DType::F32, + shape: vec![2, 2], + data_offsets: (100, 50), // End offset < start offset + param_id: None, + }, + ); + + let metadata = BurnpackMetadata { + tensors, + metadata: BTreeMap::new(), + }; + + let data = Bytes::from_bytes_vec(vec![0u8; 1000]); + let backend = crate::burnpack::reader::StorageBackend::Memory(Rc::new(data)); + + let reader = BurnpackReader { + metadata, + storage: backend, + data_offset: 0, + }; + + // This should return an error, not panic + let result = reader.get_snapshots(); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, BurnpackError::ValidationError(_))); + assert!(err.to_string().contains("end offset") && err.to_string().contains("start offset")); +} + +#[test] +fn test_reader_truncated_file_from_bytes() { + // Create a valid burnpack with tensor data + let tensor_size = 1024; // 1KB of data + let data = vec![42u8; tensor_size]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![tensor_size], DType::U8), + vec!["large_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let full_bytes = writer.to_bytes().unwrap(); + + // Truncate the bytes - remove the last 512 bytes of tensor data + let truncated_len = full_bytes.len() - 512; + let truncated_bytes = Bytes::from_bytes_vec(full_bytes.to_vec()[..truncated_len].to_vec()); + + // This should fail with a validation error indicating file truncation + let result = BurnpackReader::from_bytes(truncated_bytes); + assert!(result.is_err()); + if let Err(err) = result { + assert!(matches!(err, BurnpackError::ValidationError(_))); + assert!(err.to_string().contains("File truncated")); + assert!(err.to_string().contains("expected at least")); + } +} + +#[cfg(feature = "std")] +#[test] +fn test_reader_truncated_file_from_file() { + use std::fs::OpenOptions; + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("truncated.bpk"); + + // Create a valid burnpack file with tensor data + let tensor_size = 2048; // 2KB of data + let data = vec![99u8; tensor_size]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![tensor_size], DType::U8), + vec!["data_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + writer.write_to_file(&file_path).unwrap(); + + // Read the full file to get its size + let full_size = std::fs::metadata(&file_path).unwrap().len(); + + // Truncate the file - remove the last 1KB + let truncated_size = full_size - 1024; + let truncated_file = OpenOptions::new().write(true).open(&file_path).unwrap(); + truncated_file.set_len(truncated_size).unwrap(); + drop(truncated_file); + + // Try to read the truncated file - should fail with validation error + let result = BurnpackReader::from_file(&file_path); + assert!(result.is_err()); + if let Err(err) = result { + assert!(matches!(err, BurnpackError::ValidationError(_))); + assert!(err.to_string().contains("File truncated")); + assert!(err.to_string().contains("expected at least")); + } +} + +#[test] +fn test_reader_file_size_exactly_correct() { + // Test that a file with exactly the right size passes validation + let tensor_size = 100; + let data = vec![77u8; tensor_size]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![tensor_size], DType::U8), + vec!["exact_size".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + + // This should succeed - file is exactly the right size + let reader = BurnpackReader::from_bytes(bytes); + assert!(reader.is_ok()); + + // Verify we can read the data + let reader = reader.unwrap(); + let tensor_data = reader.get_tensor_data("exact_size").unwrap(); + assert_eq!(tensor_data.len(), tensor_size); + assert!(tensor_data.iter().all(|&b| b == 77)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/round_trip.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/round_trip.rs new file mode 100644 index 0000000..5ec07b2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/round_trip.rs @@ -0,0 +1,606 @@ +use crate::burnpack::{reader::BurnpackReader, writer::BurnpackWriter}; + +use super::*; +use alloc::collections::BTreeMap; +use alloc::string::String; +use burn_tensor::{DType, TensorData}; + +/// Helper function to perform round-trip test +fn round_trip_test(setup: F) +where + F: FnOnce(&mut Vec, &mut BTreeMap), +{ + // Collect snapshots and metadata + let mut snapshots = Vec::new(); + let mut metadata = BTreeMap::new(); + setup(&mut snapshots, &mut metadata); + + // Sort snapshots by name to ensure consistent ordering + // This is necessary because BTreeMap will store them sorted + snapshots.sort_by_key(|a| a.full_path()); + + // Create writer with snapshots and metadata + let mut writer = BurnpackWriter::new(snapshots); + for (key, value) in &metadata { + writer = writer.with_metadata(key, value); + } + + let bytes = writer.to_bytes().unwrap(); + let reader = BurnpackReader::from_bytes(bytes.clone()).unwrap(); + + // Write to bytes again from reader data + let mut snapshots2 = Vec::new(); + + // Copy tensors (metadata.tensors is now BTreeMap) + // They will come out in sorted order from tensor_names() + for tensor_name in reader.tensor_names() { + let snapshot = reader.get_tensor_snapshot(tensor_name).unwrap(); + snapshots2.push(snapshot); + } + + // Create writer2 with collected snapshots and metadata + let mut writer2 = BurnpackWriter::new(snapshots2); + for (key, value) in &reader.metadata().metadata { + writer2 = writer2.with_metadata(key, value); + } + + let bytes2 = writer2.to_bytes().unwrap(); + + // Both byte representations should be identical + assert_eq!(bytes, bytes2, "Round-trip produced different bytes"); +} + +#[test] +fn test_round_trip_empty() { + round_trip_test(|_snapshots, _metadata| { + // Empty writer + }); +} + +#[test] +fn test_round_trip_metadata_only() { + round_trip_test(|_snapshots, metadata| { + metadata.insert("key1".to_string(), "value1".to_string()); + metadata.insert("key2".to_string(), "value2".to_string()); + metadata.insert("key3".to_string(), "value3".to_string()); + }); +} + +#[test] +fn test_round_trip_f32() { + round_trip_test(|snapshots, _metadata| { + let data = [1.0f32, 2.0, 3.0, 4.0, 5.0]; + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![5], DType::F32), + vec!["f32_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_f64() { + round_trip_test(|snapshots, _metadata| { + let data = [1.0f64, 2.0, 3.0]; + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![3], DType::F64), + vec!["f64_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_i32() { + round_trip_test(|snapshots, _metadata| { + let data = [-10i32, 0, 10, 20]; + let bytes: Vec = data.iter().flat_map(|i| i.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![4], DType::I32), + vec!["i32_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_i64() { + round_trip_test(|snapshots, _metadata| { + let data = [i64::MIN, 0, i64::MAX]; + let bytes: Vec = data.iter().flat_map(|i| i.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![3], DType::I64), + vec!["i64_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_u32() { + round_trip_test(|snapshots, _metadata| { + let data = [0u32, 100, 1000, u32::MAX]; + let bytes: Vec = data.iter().flat_map(|u| u.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![4], DType::U32), + vec!["u32_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_u64() { + round_trip_test(|snapshots, _metadata| { + let data = [0u64, u64::MAX / 2, u64::MAX]; + let bytes: Vec = data.iter().flat_map(|u| u.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![3], DType::U64), + vec!["u64_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_u8() { + round_trip_test(|snapshots, _metadata| { + let data = vec![0u8, 127, 255]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![3], DType::U8), + vec!["u8_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_bool() { + round_trip_test(|snapshots, _metadata| { + let data = vec![0u8, 1, 0, 1, 1]; + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data, vec![5], DType::Bool), + vec!["bool_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[test] +fn test_round_trip_mixed_dtypes() { + round_trip_test(|snapshots, _metadata| { + // F32 + let f32_data = [1.0f32, 2.0]; + let f32_bytes: Vec = f32_data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let f32_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f32_bytes, vec![2], DType::F32), + vec!["f32".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(f32_snapshot); + + // I64 + let i64_data = [100i64, 200]; + let i64_bytes: Vec = i64_data.iter().flat_map(|i| i.to_le_bytes()).collect(); + let i64_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(i64_bytes, vec![2], DType::I64), + vec!["i64".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(i64_snapshot); + + // Bool + let bool_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 0, 1], vec![3], DType::Bool), + vec!["bool".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(bool_snapshot); + }); +} + +#[test] +fn test_round_trip_multidimensional() { + round_trip_test(|snapshots, _metadata| { + // 2D tensor + let data_2d = [1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0]; + let bytes_2d: Vec = data_2d.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot_2d = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes_2d, vec![2, 3], DType::F32), + vec!["tensor_2d".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot_2d); + + // 3D tensor + let data_3d = [1.0f32; 24]; + let bytes_3d: Vec = data_3d.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot_3d = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes_3d, vec![2, 3, 4], DType::F32), + vec!["tensor_3d".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot_3d); + + // 4D tensor (common for CNNs) + let data_4d = vec![1.0f32; 120]; + let bytes_4d: Vec = data_4d.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot_4d = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes_4d, vec![2, 3, 4, 5], DType::F32), + vec!["tensor_4d".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot_4d); + }); +} + +#[test] +fn test_round_trip_with_metadata_and_tensors() { + round_trip_test(|snapshots, metadata| { + // Add metadata + metadata.insert("model_name".to_string(), "test_model".to_string()); + metadata.insert("version".to_string(), "1.0.0".to_string()); + metadata.insert( + "description".to_string(), + "A test model for round-trip testing".to_string(), + ); + + // Add tensors + let weights = [0.1f32, 0.2, 0.3, 0.4]; + let weights_bytes: Vec = weights.iter().flat_map(|f| f.to_le_bytes()).collect(); + let weights_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(weights_bytes, vec![2, 2], DType::F32), + vec!["layer1".to_string(), "weights".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(weights_snapshot); + + let bias = [0.5f32, 0.6]; + let bias_bytes: Vec = bias.iter().flat_map(|f| f.to_le_bytes()).collect(); + let bias_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bias_bytes, vec![2], DType::F32), + vec!["layer1".to_string(), "bias".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(bias_snapshot); + }); +} + +#[test] +fn test_round_trip_special_values() { + round_trip_test(|snapshots, _metadata| { + // Test special float values + let special_f32 = [ + 0.0f32, + -0.0, + f32::INFINITY, + f32::NEG_INFINITY, + f32::NAN, + f32::MIN, + f32::MAX, + f32::EPSILON, + ]; + let f32_bytes: Vec = special_f32.iter().flat_map(|f| f.to_le_bytes()).collect(); + let f32_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f32_bytes, vec![8], DType::F32), + vec!["special_f32".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(f32_snapshot); + + // Test special f64 values + let special_f64 = [ + 0.0f64, + -0.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + f64::MIN, + f64::MAX, + f64::EPSILON, + ]; + let f64_bytes: Vec = special_f64.iter().flat_map(|f| f.to_le_bytes()).collect(); + let f64_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f64_bytes, vec![8], DType::F64), + vec!["special_f64".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(f64_snapshot); + }); +} + +#[test] +fn test_round_trip_large_tensors() { + round_trip_test(|snapshots, _metadata| { + // Large tensor (100KB) + let size = 25600; // 100KB / 4 bytes per f32 + let data: Vec = (0..size).map(|i| i as f32).collect(); + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![size], DType::F32), + vec!["large_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + }); +} + +#[cfg(feature = "std")] +#[test] +fn test_round_trip_file_io() { + use std::fs; + use tempfile::tempdir; + + use crate::burnpack::writer::BurnpackWriter; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("round_trip.bpk"); + + // Create original data + let data = [1.0f32, 2.0, 3.0, 4.0]; + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![2, 2], DType::F32), + vec!["weights".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]).with_metadata("test", "round_trip"); + + // Write to file + writer.write_to_file(&file_path).unwrap(); + + // Read from file + let reader = BurnpackReader::from_file(&file_path).unwrap(); + + // Write to another file + let file_path2 = dir.path().join("round_trip2.bpk"); + + // Collect snapshots from reader + let mut snapshots2 = Vec::new(); + for tensor_name in reader.tensor_names() { + let snapshot = reader.get_tensor_snapshot(tensor_name).unwrap(); + snapshots2.push(snapshot); + } + + // Create writer2 with snapshots and metadata + let mut writer2 = BurnpackWriter::new(snapshots2); + for (key, value) in &reader.metadata().metadata { + writer2 = writer2.with_metadata(key, value); + } + + writer2.write_to_file(&file_path2).unwrap(); + + // Compare files + let bytes1 = fs::read(&file_path).unwrap(); + let bytes2 = fs::read(&file_path2).unwrap(); + + assert_eq!( + bytes1, bytes2, + "Round-trip through files produced different content" + ); +} + +#[test] +fn test_round_trip_empty_shapes() { + round_trip_test(|snapshots, _metadata| { + // Scalar (0-dimensional) + let scalar = [42.0f32]; + let scalar_bytes: Vec = scalar.iter().flat_map(|f| f.to_le_bytes()).collect(); + let scalar_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(scalar_bytes, vec![], DType::F32), + vec!["scalar".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(scalar_snapshot); + + // Empty tensor + let empty_snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![], vec![0], DType::F32), + vec!["empty".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(empty_snapshot); + }); +} + +#[test] +fn test_param_id_persistence() { + use burn_core::module::ParamId; + + // Create a specific ParamId with a known value + let original_param_id = ParamId::from(123456789u64); + + let data = [1.0f32, 2.0, 3.0, 4.0]; + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes, vec![2, 2], DType::F32), + vec!["weights".to_string()], + vec![], + original_param_id, + ); + + // Write to burnpack + let writer = BurnpackWriter::new(vec![snapshot]); + let bytes = writer.to_bytes().unwrap(); + + // Read back from burnpack + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + let loaded_snapshot = reader.get_tensor_snapshot("weights").unwrap(); + + // Verify ParamId was preserved + assert!( + loaded_snapshot.tensor_id.is_some(), + "ParamId should be present" + ); + let loaded_param_id = loaded_snapshot.tensor_id.unwrap(); + assert_eq!( + loaded_param_id.val(), + original_param_id.val(), + "ParamId value should be preserved: expected {}, got {}", + original_param_id.val(), + loaded_param_id.val() + ); +} + +#[test] +fn test_param_id_backward_compatibility() { + use crate::burnpack::base::{BurnpackMetadata, TensorDescriptor}; + use alloc::collections::BTreeMap; + + // Create metadata without param_id (simulating old burnpack format) + let mut tensors = BTreeMap::new(); + tensors.insert( + "old_tensor".to_string(), + TensorDescriptor { + dtype: DType::F32, + shape: vec![2, 2], + data_offsets: (0, 16), + param_id: None, // No param_id stored (old format) + }, + ); + + let metadata = BurnpackMetadata { + tensors, + metadata: BTreeMap::new(), + }; + + // Serialize metadata + let mut metadata_bytes = Vec::new(); + ciborium::ser::into_writer(&metadata, &mut metadata_bytes).unwrap(); + + // Create a complete burnpack with header and data + use crate::burnpack::base::{BurnpackHeader, FORMAT_VERSION, MAGIC_NUMBER}; + + let metadata_size = metadata_bytes.len() as u32; + let header = BurnpackHeader { + magic: MAGIC_NUMBER, + version: FORMAT_VERSION, + metadata_size, + }; + + let mut full_bytes = Vec::new(); + full_bytes.extend_from_slice(&header.into_bytes()); + full_bytes.extend_from_slice(&metadata_bytes); + + // Add tensor data (4 f32 values = 16 bytes) + let tensor_data = vec![1.0f32, 2.0, 3.0, 4.0]; + for value in tensor_data { + full_bytes.extend_from_slice(&value.to_le_bytes()); + } + + // Read the old format burnpack + let reader = + BurnpackReader::from_bytes(burn_tensor::Bytes::from_bytes_vec(full_bytes)).unwrap(); + let loaded_snapshot = reader.get_tensor_snapshot("old_tensor").unwrap(); + + // Verify that a new ParamId was generated (backward compatibility) + assert!( + loaded_snapshot.tensor_id.is_some(), + "ParamId should be generated for old format" + ); + + // The generated ParamId should be different each time (it's new), but we can't test the exact value + // We just verify it exists and has a valid u64 value + let generated_param_id = loaded_snapshot.tensor_id.unwrap(); + assert!( + generated_param_id.val() > 0, + "Generated ParamId should have a valid value" + ); +} + +#[test] +fn test_multiple_tensors_preserve_distinct_param_ids() { + use burn_core::module::ParamId; + + // Create multiple tensors with distinct ParamIds + let param_id_1 = ParamId::from(111111u64); + let param_id_2 = ParamId::from(222222u64); + let param_id_3 = ParamId::from(333333u64); + + let mut snapshots = Vec::new(); + + let data1 = [1.0f32, 2.0]; + let bytes1: Vec = data1.iter().flat_map(|f| f.to_le_bytes()).collect(); + snapshots.push(TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes1, vec![2], DType::F32), + vec!["tensor1".to_string()], + vec![], + param_id_1, + )); + + let data2 = [3.0f32, 4.0]; + let bytes2: Vec = data2.iter().flat_map(|f| f.to_le_bytes()).collect(); + snapshots.push(TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes2, vec![2], DType::F32), + vec!["tensor2".to_string()], + vec![], + param_id_2, + )); + + let data3 = [5.0f32, 6.0]; + let bytes3: Vec = data3.iter().flat_map(|f| f.to_le_bytes()).collect(); + snapshots.push(TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes3, vec![2], DType::F32), + vec!["tensor3".to_string()], + vec![], + param_id_3, + )); + + // Write to burnpack + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + + // Read back + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + let snapshot1 = reader.get_tensor_snapshot("tensor1").unwrap(); + let snapshot2 = reader.get_tensor_snapshot("tensor2").unwrap(); + let snapshot3 = reader.get_tensor_snapshot("tensor3").unwrap(); + + // Verify each ParamId was preserved correctly + assert_eq!(snapshot1.tensor_id.unwrap().val(), param_id_1.val()); + assert_eq!(snapshot2.tensor_id.unwrap().val(), param_id_2.val()); + assert_eq!(snapshot3.tensor_id.unwrap().val(), param_id_3.val()); + + // Verify they are distinct + let id1 = snapshot1.tensor_id.unwrap().val(); + let id2 = snapshot2.tensor_id.unwrap().val(); + let id3 = snapshot3.tensor_id.unwrap().val(); + + assert_ne!(id1, id2, "ParamIds should be distinct"); + assert_ne!(id2, id3, "ParamIds should be distinct"); + assert_ne!(id1, id3, "ParamIds should be distinct"); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/store.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/store.rs new file mode 100644 index 0000000..8e5114f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/store.rs @@ -0,0 +1,1175 @@ +#[cfg(feature = "std")] +use crate::KeyRemapper; +use crate::burnpack::store::BurnpackStore; +use crate::{ModuleSnapshot, ModuleStore, PathFilter}; + +use burn_core as burn; +use burn_core::module::{Module, Param}; +use burn_tensor::{Tensor, backend::Backend}; + +type TestBackend = burn_ndarray::NdArray; + +#[derive(Module, Debug)] +struct TestModule { + weight: Param>, + bias: Param>, + nested: NestedModule, +} + +#[derive(Module, Debug)] +struct NestedModule { + gamma: Param>, + beta: Param>, +} + +impl TestModule { + fn new(device: &B::Device) -> Self { + Self { + weight: Param::from_data([[1.0, 2.0], [3.0, 4.0]], device), + bias: Param::from_data([0.1, 0.2], device), + nested: NestedModule { + gamma: Param::from_data([1.0, 1.0], device), + beta: Param::from_data([0.0, 0.0], device), + }, + } + } + + fn new_zeros(device: &B::Device) -> Self { + Self { + weight: Param::from_tensor(Tensor::zeros([2, 2], device)), + bias: Param::from_tensor(Tensor::zeros([2], device)), + nested: NestedModule { + gamma: Param::from_tensor(Tensor::zeros([2], device)), + beta: Param::from_tensor(Tensor::zeros([2], device)), + }, + } + } + + fn new_uninitialized(device: &B::Device) -> Self { + use burn_core::module::ParamId; + let device_clone = device.clone(); + let device_clone2 = device.clone(); + let device_clone3 = device.clone(); + let device_clone4 = device.clone(); + + Self { + weight: Param::uninitialized( + ParamId::new(), + move |d, _| Tensor::zeros([2, 2], d), + device_clone, + true, + [2, 2].into(), + ), + bias: Param::uninitialized( + ParamId::new(), + move |d, _| Tensor::zeros([2], d), + device_clone2, + true, + [2].into(), + ), + nested: NestedModule { + gamma: Param::uninitialized( + ParamId::new(), + move |d, _| Tensor::zeros([2], d), + device_clone3, + true, + [2].into(), + ), + beta: Param::uninitialized( + ParamId::new(), + move |d, _| Tensor::zeros([2], d), + device_clone4, + true, + [2].into(), + ), + }, + } + } +} + +#[test] +fn test_store_from_bytes_round_trip() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load from bytes + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + // Verify success + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); // weight, bias, nested.gamma, nested.beta + assert!(result.errors.is_empty()); + + // Verify data was loaded correctly + let weight1 = module.weight.val().to_data().to_vec::().unwrap(); + let weight2 = module2.weight.val().to_data().to_vec::().unwrap(); + assert_eq!(weight1, weight2); +} + +#[test] +fn test_store_with_metadata() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save with metadata + let mut save_store = BurnpackStore::from_bytes(None) + .metadata("version", "1.0.0") + .metadata("model_name", "test_model") + .metadata("author", "burn_team"); + + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load and verify metadata is preserved + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_with_path_filter() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save all tensors + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load with filter - only load weight and bias (not nested) + let mut load_store = BurnpackStore::from_bytes(Some(bytes)).with_regex("^(weight|bias)$"); + + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 2); // Only weight and bias + assert_eq!(result.skipped.len(), 2); // nested.gamma and nested.beta skipped + + // Verify weight and bias were loaded + let weight2 = module2.weight.val().to_data().to_vec::().unwrap(); + assert_eq!(weight2, vec![1.0, 2.0, 3.0, 4.0]); + + // Verify nested module was NOT loaded (should still be zeros) + let gamma2 = module2 + .nested + .gamma + .val() + .to_data() + .to_vec::() + .unwrap(); + assert_eq!(gamma2, vec![0.0, 0.0]); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_with_key_remapping() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save with original names + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load with remapping: nested.gamma -> nested.new_gamma, nested.beta -> nested.new_beta + let remapper = KeyRemapper::new() + .add_pattern(r"nested\.gamma", "nested.new_gamma") + .unwrap() + .add_pattern(r"nested\.beta", "nested.new_beta") + .unwrap(); + + let mut load_store = BurnpackStore::from_bytes(Some(bytes)) + .remap(remapper) + .allow_partial(true); + + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + // The remapping should cause missing tensors since names don't match + assert_eq!(result.applied.len(), 2); // Only weight and bias match + assert_eq!(result.unused.len(), 2); // nested.new_gamma and nested.new_beta are unused + assert_eq!(result.missing.len(), 2); // nested.gamma and nested.beta are missing +} + +#[test] +fn test_store_allow_partial() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save only weight and bias + let filter = PathFilter::new() + .with_full_path("weight") + .with_full_path("bias"); + let mut save_store = BurnpackStore::from_bytes(None).with_filter(filter); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load with allow_partial + let mut load_store = BurnpackStore::from_bytes(Some(bytes)).allow_partial(true); + + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 2); + assert_eq!(result.missing.len(), 2); // nested.gamma and nested.beta are missing but that's OK + + // Verify loaded tensors + let weight2 = module2.weight.val().to_data().to_vec::().unwrap(); + assert_eq!(weight2, vec![1.0, 2.0, 3.0, 4.0]); +} + +#[test] +fn test_store_match_all() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save with match_all filter (should save everything) + let mut save_store = BurnpackStore::from_bytes(None).match_all(); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load everything + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); + assert!(result.errors.is_empty()); + assert!(result.missing.is_empty()); + assert!(result.unused.is_empty()); +} + +#[test] +fn test_store_with_full_path() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save everything + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load only specific tensors by full path + let mut load_store = BurnpackStore::from_bytes(Some(bytes)) + .with_full_path("weight") + .with_full_path("nested.gamma"); + + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 2); // Only weight and nested.gamma + assert_eq!(result.skipped.len(), 2); // bias and nested.beta skipped +} + +#[test] +#[cfg(feature = "std")] +fn test_store_chain_multiple_patterns() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save with chained metadata and filters + let mut save_store = BurnpackStore::from_bytes(None) + .metadata("version", "1.0") + .metadata("format", "burnpack") + .with_regex(r"^(weight|nested\.)") + .match_all(); // This overrides the previous filter + + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load everything since match_all was called last + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); // All tensors loaded +} + +#[test] +#[cfg(feature = "std")] +fn test_store_with_remap_pattern() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save normally + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load with single remap pattern using the convenience method + let mut load_store = BurnpackStore::from_bytes(Some(bytes)) + .with_remap_pattern(r"^nested\.", "sub_module.") + .allow_partial(true); + + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + // After remapping, nested.* becomes sub_module.*, which won't match + assert_eq!(result.applied.len(), 2); // Only weight and bias + assert_eq!(result.unused.len(), 2); // sub_module.gamma and sub_module.beta unused +} + +#[test] +fn test_store_default_metadata() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save without adding custom metadata + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Verify default metadata is included + // We can't directly inspect metadata from bytes, but we can verify + // that the model loads successfully which means metadata was written correctly + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); +} + +#[test] +fn test_store_default_metadata_with_custom() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save with custom metadata (should preserve defaults) + let mut save_store = BurnpackStore::from_bytes(None) + .metadata("custom_field", "custom_value") + .metadata("author", "test_author"); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load and verify it works (metadata including defaults was saved) + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); +} + +#[test] +fn test_store_clear_metadata() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save with cleared metadata (no defaults) + let mut save_store = BurnpackStore::from_bytes(None).clear_metadata(); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Verify it still loads correctly + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); +} + +#[test] +fn test_store_validate_enabled() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save normally + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load with validation enabled (default) + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert!(result.errors.is_empty()); +} + +#[test] +fn test_store_validate_disabled() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save normally + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load with validation disabled + let mut load_store = BurnpackStore::from_bytes(Some(bytes)).validate(false); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + // Should still succeed + assert!(result.is_success()); +} + +#[test] +fn test_store_allow_partial_missing_tensors() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save only weight (not bias or nested) + let filter = PathFilter::new().with_full_path("weight"); + let mut save_store = BurnpackStore::from_bytes(None).with_filter(filter); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Try to load without allow_partial - should fail due to missing tensors + let mut load_store = BurnpackStore::from_bytes(Some(bytes.clone())); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2); + + // Should fail because of missing tensors + assert!(result.is_err()); + + // Now try with allow_partial - should succeed + let mut load_store = BurnpackStore::from_bytes(Some(bytes)).allow_partial(true); + let mut module3 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module3).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 1); // Only weight + assert!(!result.missing.is_empty()); // Has missing tensors +} + +#[test] +#[cfg(feature = "std")] +fn test_store_file_round_trip() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory and file path + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_file_round_trip.bpk"); + + // Save to file + let mut save_store = BurnpackStore::from_file(&path).metadata("test", "value"); + save_store.collect_from(&module).unwrap(); + + // Verify file exists + assert!(path.exists()); + + // Load from file + let mut load_store = BurnpackStore::from_file(&path); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); + + // Verify data + let weight1 = module.weight.val().to_data().to_vec::().unwrap(); + let weight2 = module2.weight.val().to_data().to_vec::().unwrap(); + assert_eq!(weight1, weight2); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_overwrite_protection() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory and file path (file doesn't exist yet) + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_model.bpk"); + + // First save - should succeed + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + assert!(path.exists()); + + // Second save without overwrite flag - should fail + let mut save_store2 = BurnpackStore::from_file(&path); + let result = save_store2.collect_from(&module); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("File already exists") + ); + + // Third save with overwrite flag - should succeed + let mut save_store3 = BurnpackStore::from_file(&path).overwrite(true); + save_store3.collect_from(&module).unwrap(); + + // Verify file still exists and is valid + let mut load_store = BurnpackStore::from_file(&path); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_overwrite_with_metadata() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory and file path + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_model_metadata.bpk"); + + // First save with v1 metadata + let mut save_store = BurnpackStore::from_file(&path) + .metadata("version", "1.0") + .overwrite(true); + save_store.collect_from(&module).unwrap(); + + // Second save with v2 metadata and overwrite enabled + let mut save_store2 = BurnpackStore::from_file(&path) + .metadata("version", "2.0") + .overwrite(true); + save_store2.collect_from(&module).unwrap(); + + // Verify file loads correctly + let mut load_store = BurnpackStore::from_file(&path); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_auto_extension_default() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("model"); + + // Save without extension - should auto-append .bpk + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + // Verify that model.bpk was created + let expected_path = temp_dir.path().join("model.bpk"); + assert!(expected_path.exists()); + assert!(!path.exists()); // Original path without extension should not exist + + // Load using the path without extension - should work + let mut load_store = BurnpackStore::from_file(&path); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_auto_extension_with_existing_extension() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("model.bpk"); + + // Save with .bpk extension - should not double append + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + // Verify that only model.bpk was created + assert!(path.exists()); + let double_ext_path = temp_dir.path().join("model.bpk.bpk"); + assert!(!double_ext_path.exists()); + + // Load and verify + let mut load_store = BurnpackStore::from_file(&path); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_auto_extension_with_custom_extension() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("model.mpk"); + + // Save with .mpk extension - should preserve it + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + // Verify that model.mpk was created (not model.mpk.bpk) + assert!(path.exists()); + let burnpack_path = temp_dir.path().join("model.mpk.bpk"); + assert!(!burnpack_path.exists()); + + // Load and verify + let mut load_store = BurnpackStore::from_file(&path); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_auto_extension_disabled() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Create temp directory + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("model"); + + // Save with auto_extension disabled - should use exact path + let mut save_store = BurnpackStore::from_file(&path).auto_extension(false); + save_store.collect_from(&module).unwrap(); + + // Verify that "model" (without extension) was created + assert!(path.exists()); + let burnpack_path = temp_dir.path().join("model.bpk"); + assert!(!burnpack_path.exists()); + + // Load with auto_extension disabled + let mut load_store = BurnpackStore::from_file(&path).auto_extension(false); + let mut module2 = TestModule::::new_zeros(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +#[cfg(feature = "std")] +fn test_partial_loading_preserves_lazy_initialization() { + use tempfile::tempdir; + + let device = Default::default(); + + // Create and save a full module + let module = TestModule::::new(&device); + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("model.bpk"); + + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + // Create an uninitialized module (all params lazy) + let mut load_module = TestModule::::new_uninitialized(&device); + + // Before loading: verify ALL params are uninitialized (lazy) + assert!( + !load_module.weight.is_initialized(), + "weight should be uninitialized before loading" + ); + assert!( + !load_module.bias.is_initialized(), + "bias should be uninitialized before loading" + ); + assert!( + !load_module.nested.gamma.is_initialized(), + "nested.gamma should be uninitialized before loading" + ); + assert!( + !load_module.nested.beta.is_initialized(), + "nested.beta should be uninitialized before loading" + ); + + // Partial load: only load weight and bias (skip nested.*) + let filter = PathFilter::new().with_regex("^(weight|bias)$"); + let mut load_store = BurnpackStore::from_file(&path).filter(filter); + let result = load_module.load_from(&mut load_store).unwrap(); + + // Verify only weight and bias were loaded + assert_eq!(result.applied.len(), 2); + assert!(result.applied.contains(&"weight".to_string())); + assert!(result.applied.contains(&"bias".to_string())); + assert_eq!(result.skipped.len(), 2); + assert!(result.skipped.contains(&"nested.gamma".to_string())); + assert!(result.skipped.contains(&"nested.beta".to_string())); + + // After loading: verify loaded params are initialized, skipped remain lazy + assert!( + load_module.weight.is_initialized(), + "weight should be initialized after loading" + ); + assert!( + load_module.bias.is_initialized(), + "bias should be initialized after loading" + ); + assert!( + !load_module.nested.gamma.is_initialized(), + "nested.gamma should remain uninitialized (was skipped)" + ); + assert!( + !load_module.nested.beta.is_initialized(), + "nested.beta should remain uninitialized (was skipped)" + ); + + // Verify the loaded values are correct + let weight_data = load_module.weight.val().to_data().to_vec::().unwrap(); + assert_eq!(weight_data, vec![1.0, 2.0, 3.0, 4.0]); + + let bias_data = load_module.bias.val().to_data().to_vec::().unwrap(); + assert_eq!(bias_data, vec![0.1, 0.2]); + + // Now check that nested params can still be initialized on first access + let gamma_data = load_module + .nested + .gamma + .val() + .to_data() + .to_vec::() + .unwrap(); + assert_eq!(gamma_data, vec![0.0, 0.0]); // Initialized to zeros via the init function + + // After accessing, they should be initialized + assert!( + load_module.nested.gamma.is_initialized(), + "nested.gamma should be initialized after first access" + ); +} + +// Model with forward pass for testing weight preservation +#[derive(Module, Debug)] +struct ForwardTestModel { + linear1: burn_nn::Linear, + linear2: burn_nn::Linear, +} + +impl ForwardTestModel { + /// Forward pass: input -> linear1 -> gelu -> linear2 + fn forward(&self, input: Tensor) -> Tensor { + let x = self.linear1.forward(input); + let x = burn::tensor::activation::gelu(x); + self.linear2.forward(x) + } +} + +#[derive(burn::config::Config, Debug)] +struct ForwardTestModelConfig { + input_size: usize, + hidden_size: usize, + output_size: usize, +} + +impl ForwardTestModelConfig { + fn init(&self, device: &B::Device) -> ForwardTestModel { + ForwardTestModel { + linear1: burn_nn::LinearConfig::new(self.input_size, self.hidden_size) + .with_bias(true) + .init(device), + linear2: burn_nn::LinearConfig::new(self.hidden_size, self.output_size) + .with_bias(true) + .init(device), + } + } +} + +#[test] +#[cfg(feature = "std")] +fn test_forward_pass_preservation_after_save_load() { + use tempfile::tempdir; + + let device = Default::default(); + + // Create model config + let config = ForwardTestModelConfig { + input_size: 4, + hidden_size: 8, + output_size: 2, + }; + + // Initialize model1 with random weights + let model1 = config.init::(&device); + + // Create random input + let input = Tensor::::random( + [1, 4], + burn_tensor::Distribution::Uniform(-1.0, 1.0), + &device, + ); + + // Forward pass with model1 -> output1 + let output1 = model1.forward(input.clone()); + + // Save model1 weights + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("forward_test_model.bpk"); + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&model1).unwrap(); + + // Initialize model2 with different random weights + let mut model2 = config.init::(&device); + + // Forward pass with model2 -> output2 (should differ from output1) + let output2 = model2.forward(input.clone()); + + // Verify output2 differs from output1 (different random weights) + assert!( + !output1 + .clone() + .all_close(output2.clone(), Some(1e-6), Some(1e-6)), + "output2 should differ from output1 (different random initializations)" + ); + + // Load model1 weights into model2 + let mut load_store = BurnpackStore::from_file(&path); + let result = load_store.apply_to(&mut model2).unwrap(); + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); // 2 weights + 2 biases + + // Forward pass with model2 (now has model1 weights) -> output3 + let output3 = model2.forward(input.clone()); + + // Verify output3 equals output1 (same weights) + assert!( + output1.all_close(output3, Some(1e-6), Some(1e-6)), + "output3 should equal output1 after loading weights" + ); +} + +#[test] +fn test_store_get_all_snapshots() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save module to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Get all snapshots (returns &BTreeMap) + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let snapshots = load_store.get_all_snapshots().unwrap(); + + // Should have 4 tensors + assert_eq!(snapshots.len(), 4); + + // Verify tensor names exist (BTreeMap keys) + assert!(snapshots.contains_key("weight")); + assert!(snapshots.contains_key("bias")); + assert!(snapshots.contains_key("nested.gamma")); + assert!(snapshots.contains_key("nested.beta")); +} + +#[test] +fn test_store_get_snapshot_existing() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save module to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Get a specific snapshot (returns Option<&TensorSnapshot>) + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let snapshot = load_store.get_snapshot("weight").unwrap(); + + // Should find the tensor + assert!(snapshot.is_some()); + let snapshot = snapshot.unwrap(); + assert_eq!(snapshot.full_path(), "weight"); + assert_eq!(snapshot.shape, vec![2, 2]); + + // Verify data can be loaded + let data = snapshot.to_data().unwrap(); + assert_eq!(data.to_vec::().unwrap(), vec![1.0, 2.0, 3.0, 4.0]); +} + +#[test] +fn test_store_get_snapshot_nested() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save module to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Get a nested snapshot + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let snapshot = load_store.get_snapshot("nested.gamma").unwrap(); + + assert!(snapshot.is_some()); + let snapshot = snapshot.unwrap(); + assert_eq!(snapshot.full_path(), "nested.gamma"); + assert_eq!(snapshot.shape, vec![2]); +} + +#[test] +fn test_store_get_snapshot_not_found() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save module to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Try to get a non-existent snapshot + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let snapshot = load_store.get_snapshot("nonexistent").unwrap(); + + // Should return None + assert!(snapshot.is_none()); +} + +#[test] +fn test_store_keys() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save module to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Get all keys + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let keys = load_store.keys().unwrap(); + + // Should have 4 keys + assert_eq!(keys.len(), 4); + assert!(keys.contains(&"weight".to_string())); + assert!(keys.contains(&"bias".to_string())); + assert!(keys.contains(&"nested.gamma".to_string())); + assert!(keys.contains(&"nested.beta".to_string())); +} + +#[test] +#[cfg(feature = "std")] +fn test_store_get_all_snapshots_from_file() { + use tempfile::tempdir; + + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save to file + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_get_all_snapshots.bpk"); + + let mut save_store = BurnpackStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + // Get snapshots from file (returns &BTreeMap) + let mut load_store = BurnpackStore::from_file(&path); + let snapshots = load_store.get_all_snapshots().unwrap(); + + assert_eq!(snapshots.len(), 4); + + // Verify we can load data from a snapshot (use get() on BTreeMap) + let weight_snapshot = snapshots.get("weight").unwrap(); + let data = weight_snapshot.to_data().unwrap(); + assert_eq!(data.to_vec::().unwrap(), vec![1.0, 2.0, 3.0, 4.0]); +} + +#[test] +fn test_store_caching_behavior() { + let device = Default::default(); + let module = TestModule::::new(&device); + + // Save module to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Create store and call get_snapshots multiple times + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + + // First call should populate cache + let snapshots1 = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots1.len(), 4); + + // Second call should return cached data (same reference) + let snapshots2 = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots2.len(), 4); + + // get_snapshot should also use the cache + let weight = load_store.get_snapshot("weight").unwrap(); + assert!(weight.is_some()); +} + +#[test] +fn test_store_cache_invalidation_on_save() { + let device = Default::default(); + + // Create first module with specific weights + let module1 = TestModule::::new(&device); + + // Save module1 to bytes store + let mut store = BurnpackStore::from_bytes(None); + store.collect_from(&module1).unwrap(); + + // Populate cache by calling get_snapshots + let snapshots1 = store.get_all_snapshots().unwrap(); + assert_eq!(snapshots1.len(), 4); + let weight1_data = snapshots1.get("weight").unwrap().to_data().unwrap(); + let weight1_values: Vec = weight1_data.to_vec().unwrap(); + + // Create a different module with different weights + let module2 = TestModule:: { + weight: Param::from_tensor(Tensor::from_data([[10.0, 20.0], [30.0, 40.0]], &device)), + bias: Param::from_tensor(Tensor::from_data([100.0, 200.0], &device)), + nested: NestedModule { + gamma: Param::from_tensor(Tensor::from_data([1000.0, 2000.0], &device)), + beta: Param::from_tensor(Tensor::from_data([3000.0, 4000.0], &device)), + }, + }; + + // Save module2 - this should invalidate the cache + store.collect_from(&module2).unwrap(); + + // Get snapshots again - should return NEW data, not cached old data + let snapshots2 = store.get_all_snapshots().unwrap(); + assert_eq!(snapshots2.len(), 4); + let weight2_data = snapshots2.get("weight").unwrap().to_data().unwrap(); + let weight2_values: Vec = weight2_data.to_vec().unwrap(); + + // Verify the data changed (cache was invalidated) + assert_ne!(weight1_values, weight2_values); + assert_eq!(weight2_values, vec![10.0, 20.0, 30.0, 40.0]); +} + +/// Test storing and loading quantized weights with BurnpackStore. +/// Regression test for https://github.com/tracel-ai/burn/issues/4179 +#[test] +fn test_store_quantized_module_round_trip() { + use burn_core::module::Quantizer; + use burn_nn::LinearConfig; + use burn_tensor::quantization::{ + Calibration, QTensorPrimitive, QuantLevel, QuantParam, QuantValue, + }; + + let device = Default::default(); + + // Create a simple linear module (512x512 as in the bug report) + let linear = LinearConfig::new(512, 512) + .with_bias(false) + .init::(&device); + + // Define quantization scheme (Q8S with tensor-level quantization) + let scheme = <::QuantizedTensorPrimitive as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::Tensor) + .with_param(QuantParam::F32); + + // Quantize the module + let calibration = Calibration::MinMax; + let mut quantizer = Quantizer { + calibration, + scheme, + }; + let quantized_linear = linear.quantize_weights(&mut quantizer); + + // Save the quantized module + let mut save_store = BurnpackStore::from_bytes(None); + let result = save_store.collect_from(&quantized_linear); + assert!( + result.is_ok(), + "Failed to save quantized module: {:?}", + result.err() + ); + + // Get the bytes + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load the bytes and verify we can read the tensor metadata + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let snapshots = load_store + .get_all_snapshots() + .expect("Failed to get snapshots"); + + // Verify we have the weight tensor + assert_eq!(snapshots.len(), 1, "Expected 1 tensor (weight)"); + assert!(snapshots.contains_key("weight"), "Expected 'weight' tensor"); + + // Verify the tensor metadata + let weight_snapshot = snapshots.get("weight").unwrap(); + assert_eq!(weight_snapshot.shape, vec![512, 512]); + + // Verify we can load the tensor data + let weight_data = weight_snapshot + .to_data() + .expect("Failed to load tensor data"); + assert_eq!(weight_data.shape, vec![512, 512]); +} + +/// Test storing quantized weights with block-level quantization. +#[test] +fn test_store_quantized_module_block_level() { + use burn_core::module::Quantizer; + use burn_nn::LinearConfig; + use burn_tensor::quantization::{ + Calibration, QTensorPrimitive, QuantLevel, QuantParam, QuantValue, + }; + + let device = Default::default(); + + // Create a linear module + let linear = LinearConfig::new(128, 128) + .with_bias(false) + .init::(&device); + + // Define quantization scheme with block-level quantization + let scheme = <::QuantizedTensorPrimitive as QTensorPrimitive>::default_scheme() + .with_value(QuantValue::Q8S) + .with_level(QuantLevel::block([32])) // Block size of 32 + .with_param(QuantParam::F32); + + // Quantize the module + let calibration = Calibration::MinMax; + let mut quantizer = Quantizer { + calibration, + scheme, + }; + let quantized_linear = linear.quantize_weights(&mut quantizer); + + // Save the quantized module + let mut save_store = BurnpackStore::from_bytes(None); + let result = save_store.collect_from(&quantized_linear); + assert!( + result.is_ok(), + "Failed to save quantized module with block-level quantization: {:?}", + result.err() + ); + + // Get the bytes and verify round-trip + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let snapshots = load_store + .get_all_snapshots() + .expect("Failed to get snapshots"); + + assert_eq!(snapshots.len(), 1); + let weight_snapshot = snapshots.get("weight").unwrap(); + assert_eq!(weight_snapshot.shape, vec![128, 128]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/writer.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/writer.rs new file mode 100644 index 0000000..4faa021 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/writer.rs @@ -0,0 +1,744 @@ +use crate::burnpack::{ + base::{ + BurnpackHeader, BurnpackMetadata, FORMAT_VERSION, HEADER_SIZE, MAGIC_NUMBER, + aligned_data_section_start, magic_range, + }, + writer::BurnpackWriter, +}; + +use super::*; +use burn_core::module::ParamId; +use burn_tensor::{DType, TensorData}; +use std::rc::Rc; + +#[test] +fn test_writer_new() { + let writer = BurnpackWriter::new(vec![]); + assert_eq!(writer.snapshots.len(), 0); + assert!(writer.metadata.is_empty()); +} + +#[test] +fn test_writer_add_metadata() { + let writer = BurnpackWriter::new(vec![]) + .with_metadata("model_name", "test_model") + .with_metadata("version", "1.0.0") + .with_metadata("author", "test_author"); + + assert_eq!(writer.metadata.len(), 3); + assert_eq!( + writer.metadata.get("model_name"), + Some(&"test_model".to_string()) + ); + assert_eq!(writer.metadata.get("version"), Some(&"1.0.0".to_string())); + assert_eq!( + writer.metadata.get("author"), + Some(&"test_author".to_string()) + ); +} + +#[test] +fn test_writer_add_tensor_snapshot() { + // Create test tensor snapshots + let snapshot1 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["layer1".to_string(), "weights".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let snapshot2 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![5, 6, 7, 8], vec![4], DType::U8), + vec!["layer1".to_string(), "bias".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot1, snapshot2]); + + assert_eq!(writer.snapshots.len(), 2); + assert_eq!(writer.snapshots[0].full_path(), "layer1.weights"); + assert_eq!(writer.snapshots[1].full_path(), "layer1.bias"); +} + +#[test] +fn test_writer_to_bytes_empty() { + let writer = BurnpackWriter::new(vec![]); + let bytes = writer.to_bytes().unwrap(); + + // Verify header + assert!(bytes.len() >= HEADER_SIZE); + assert_eq!(&bytes[magic_range()], &MAGIC_NUMBER.to_le_bytes()); + + // Parse header + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + assert_eq!(header.magic, MAGIC_NUMBER); + assert_eq!(header.version, FORMAT_VERSION); + + // Verify metadata + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata_bytes = &bytes[HEADER_SIZE..metadata_end]; + let metadata: BurnpackMetadata = ciborium::de::from_reader(metadata_bytes).unwrap(); + + assert_eq!(metadata.tensors.len(), 0); + assert!(metadata.metadata.is_empty()); +} + +#[test] +fn test_writer_to_bytes_with_tensors() { + // Add tensors with different data types + let f32_data = [1.0f32, 2.0, 3.0, 4.0]; + let f32_bytes: Vec = f32_data.iter().flat_map(|f| f.to_le_bytes()).collect(); + let snapshot_f32 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(f32_bytes.clone(), vec![2, 2], DType::F32), + vec!["weights".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let i64_data = [10i64, 20, 30]; + let i64_bytes: Vec = i64_data.iter().flat_map(|i| i.to_le_bytes()).collect(); + let snapshot_i64 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(i64_bytes.clone(), vec![3], DType::I64), + vec!["bias".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot_f32, snapshot_i64]) + .with_metadata("test_key", "test_value"); + + let bytes = writer.to_bytes().unwrap(); + + // Parse and verify + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&bytes[HEADER_SIZE..metadata_end]).unwrap(); + + // Verify metadata + assert_eq!( + metadata.metadata.get("test_key"), + Some(&"test_value".to_string()) + ); + + // Verify tensors + assert_eq!(metadata.tensors.len(), 2); + + let weights = metadata.tensors.get("weights").unwrap(); + assert_eq!(weights.dtype, DType::F32); + assert_eq!(weights.shape, vec![2, 2]); + assert_eq!(weights.data_offsets.1 - weights.data_offsets.0, 16); // 4 * 4 bytes + + let bias = metadata.tensors.get("bias").unwrap(); + assert_eq!(bias.dtype, DType::I64); + assert_eq!(bias.shape, vec![3]); + assert_eq!(bias.data_offsets.1 - bias.data_offsets.0, 24); // 3 * 8 bytes + + // Verify actual tensor data + // Data section starts at aligned position after metadata + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + let weights = metadata.tensors.get("weights").unwrap(); + let bias = metadata.tensors.get("bias").unwrap(); + let weights_data = &bytes[data_section_start + weights.data_offsets.0 as usize + ..data_section_start + weights.data_offsets.1 as usize]; + assert_eq!(weights_data, f32_bytes); + + let bias_data = &bytes[data_section_start + bias.data_offsets.0 as usize + ..data_section_start + bias.data_offsets.1 as usize]; + assert_eq!(bias_data, i64_bytes); +} + +#[test] +fn test_writer_all_dtypes() { + use half::{bf16, f16}; + + // Test all supported data types (excluding QFloat which is tested separately) + // Format: (DType, expected_size_per_element, sample_data_bytes) + let test_cases = vec![ + // Floating point types + (DType::F64, 8, 1.0f64.to_le_bytes().to_vec()), + (DType::F32, 4, 1.0f32.to_le_bytes().to_vec()), + (DType::F16, 2, f16::from_f32(1.0).to_le_bytes().to_vec()), + (DType::BF16, 2, bf16::from_f32(1.0).to_le_bytes().to_vec()), + // Signed integers + (DType::I64, 8, 1i64.to_le_bytes().to_vec()), + (DType::I32, 4, 1i32.to_le_bytes().to_vec()), + (DType::I16, 2, 1i16.to_le_bytes().to_vec()), + (DType::I8, 1, 1i8.to_le_bytes().to_vec()), + // Unsigned integers + (DType::U64, 8, 255u64.to_le_bytes().to_vec()), + (DType::U32, 4, 255u32.to_le_bytes().to_vec()), + (DType::U16, 2, 255u16.to_le_bytes().to_vec()), + (DType::U8, 1, vec![255u8]), + // Boolean + (DType::Bool, 1, vec![1u8]), + ]; + + let mut snapshots = vec![]; + let mut expected_data = vec![]; + for (i, (dtype, expected_size, data)) in test_cases.into_iter().enumerate() { + let name = format!("tensor_{}", i); + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), vec![1], dtype), + vec![name.clone()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + expected_data.push((name, dtype, expected_size, data)); + } + + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + + // Parse and verify metadata + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&bytes[HEADER_SIZE..HEADER_SIZE + header.metadata_size as usize]) + .unwrap(); + + assert_eq!( + metadata.tensors.len(), + 13, + "Expected 13 dtypes to be tested" + ); + + // Verify each tensor's metadata and data + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + for (name, expected_dtype, expected_size, expected_bytes) in expected_data { + let tensor = metadata + .tensors + .get(&name) + .unwrap_or_else(|| panic!("Missing tensor: {}", name)); + assert_eq!(tensor.dtype, expected_dtype, "DType mismatch for {}", name); + assert_eq!(tensor.shape, vec![1], "Shape mismatch for {}", name); + + // Verify data size matches expected + let data_size = (tensor.data_offsets.1 - tensor.data_offsets.0) as usize; + assert_eq!( + data_size, expected_size, + "Data size mismatch for {} ({:?})", + name, expected_dtype + ); + + // Verify actual data bytes match + let actual_bytes = &bytes[data_section_start + tensor.data_offsets.0 as usize + ..data_section_start + tensor.data_offsets.1 as usize]; + assert_eq!( + actual_bytes, + expected_bytes.as_slice(), + "Data mismatch for {} ({:?})", + name, + expected_dtype + ); + } +} + +#[test] +fn test_writer_all_dtypes_round_trip() { + use crate::burnpack::reader::BurnpackReader; + use half::{bf16, f16}; + + // Test all dtypes can be written and read back correctly + let test_cases = vec![ + // Floating point types - use multiple elements to better test + ( + "f64_tensor", + DType::F64, + [1.0f64, 2.0, 3.0, 4.0] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![4], + ), + ( + "f32_tensor", + DType::F32, + [1.0f32, 2.0, 3.0, 4.0] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![2, 2], + ), + ( + "f16_tensor", + DType::F16, + [f16::from_f32(1.0), f16::from_f32(2.0)] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![2], + ), + ( + "bf16_tensor", + DType::BF16, + [bf16::from_f32(1.0), bf16::from_f32(2.0)] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![2], + ), + // Signed integers + ( + "i64_tensor", + DType::I64, + [1i64, -2, 3, -4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![4], + ), + ( + "i32_tensor", + DType::I32, + [1i32, -2, 3, -4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![2, 2], + ), + ( + "i16_tensor", + DType::I16, + [1i16, -2, 3, -4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![4], + ), + ( + "i8_tensor", + DType::I8, + [1i8, -2, 3, -4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![2, 2], + ), + // Unsigned integers + ( + "u64_tensor", + DType::U64, + [1u64, 2, 3, 4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![4], + ), + ( + "u32_tensor", + DType::U32, + [1u32, 2, 3, 4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![2, 2], + ), + ( + "u16_tensor", + DType::U16, + [1u16, 2, 3, 4] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + vec![4], + ), + ("u8_tensor", DType::U8, vec![1u8, 2, 3, 4], vec![2, 2]), + // Boolean + ("bool_tensor", DType::Bool, vec![1u8, 0, 1, 0], vec![4]), + ]; + + let mut snapshots = vec![]; + let mut expected_results: Vec<(&str, DType, Vec, Vec)> = vec![]; + + for (name, dtype, data, shape) in test_cases.into_iter() { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(data.clone(), shape.clone(), dtype), + vec![name.to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + snapshots.push(snapshot); + expected_results.push((name, dtype, data, shape)); + } + + // Write to bytes + let writer = BurnpackWriter::new(snapshots); + let bytes = writer.to_bytes().unwrap(); + + // Read back using BurnpackReader + let reader = BurnpackReader::from_bytes(bytes).unwrap(); + + // Verify each tensor can be read back with correct data + for (name, expected_dtype, expected_data, expected_shape) in expected_results { + let snapshot = reader + .get_tensor_snapshot(name) + .unwrap_or_else(|e| panic!("Failed to get tensor snapshot {}: {}", name, e)); + let tensor_data = snapshot + .to_data() + .unwrap_or_else(|e| panic!("Failed to read tensor data {}: {}", name, e)); + + assert_eq!( + tensor_data.dtype, expected_dtype, + "DType mismatch for {}", + name + ); + assert_eq!( + tensor_data.shape, expected_shape, + "Shape mismatch for {}", + name + ); + assert_eq!( + &tensor_data.bytes[..], + expected_data.as_slice(), + "Data mismatch for {}", + name + ); + } +} + +#[test] +fn test_writer_large_tensor() { + // Create a large tensor (1MB) + let size = 256 * 1024; // 256K floats = 1MB + let data: Vec = (0..size).map(|i| i as f32).collect(); + let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(bytes.clone(), vec![size], DType::F32), + vec!["large_tensor".to_string()], + vec![], + burn_core::module::ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + + let result = writer.to_bytes().unwrap(); + + // Verify the large tensor is correctly stored + let header = BurnpackHeader::from_bytes(&result[..HEADER_SIZE]).unwrap(); + let metadata: BurnpackMetadata = ciborium::de::from_reader( + &result[HEADER_SIZE..HEADER_SIZE + header.metadata_size as usize], + ) + .unwrap(); + + assert_eq!(metadata.tensors.len(), 1); + let tensor = metadata.tensors.get("large_tensor").unwrap(); + assert_eq!(tensor.shape, vec![size as u64]); + assert_eq!( + tensor.data_offsets.1 - tensor.data_offsets.0, + (size * 4) as u64 + ); +} + +#[test] +fn test_writer_empty_tensors() { + // Add tensor with empty data + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![], vec![0], DType::F32), + vec!["empty".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + + let bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&bytes[HEADER_SIZE..HEADER_SIZE + header.metadata_size as usize]) + .unwrap(); + + assert_eq!(metadata.tensors.len(), 1); + let tensor = metadata.tensors.get("empty").unwrap(); + assert_eq!(tensor.shape, vec![0]); + assert_eq!(tensor.data_offsets.1 - tensor.data_offsets.0, 0); +} + +#[test] +fn test_writer_special_characters_in_names() { + // Test various special characters in tensor names + let special_names = vec![ + "layer.0.weight", + "model/encoder/layer1", + "model::layer::weight", + "layer[0].bias", + "layer_1_weight", + "layer-1-bias", + "layer@1#weight", + "emoji_😀_tensor", + "unicode_测试_tensor", + "spaces in name", + ]; + + let mut snapshots = vec![]; + for name in &special_names { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![4], DType::U8), + vec![name.to_string()], + vec![], + ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + + let bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&bytes[HEADER_SIZE..HEADER_SIZE + header.metadata_size as usize]) + .unwrap(); + + assert_eq!(metadata.tensors.len(), 10); + for (tensor_name, _tensor) in metadata.tensors.iter() { + assert!(!tensor_name.is_empty()); + // Names should be preserved exactly + assert!( + tensor_name.contains("layer") + || tensor_name.contains("model") + || tensor_name.contains("emoji") + || tensor_name.contains("unicode") + || tensor_name.contains("spaces") + ); + } +} + +#[test] +fn test_writer_metadata_overwrite() { + let writer = BurnpackWriter::new(vec![]) + .with_metadata("key", "value1") + .with_metadata("key", "value2"); + + assert_eq!(writer.metadata.get("key"), Some(&"value2".to_string())); + assert_eq!(writer.metadata.len(), 1); +} + +#[test] +fn test_writer_tensor_order_preserved() { + // Add tensors in specific order + let names = vec!["z_tensor", "a_tensor", "m_tensor", "b_tensor"]; + + let mut snapshots = vec![]; + for name in &names { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1], vec![1], DType::U8), + vec![name.to_string()], + vec![], + ParamId::new(), + ); + snapshots.push(snapshot); + } + + let writer = BurnpackWriter::new(snapshots); + + let bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&bytes[HEADER_SIZE..HEADER_SIZE + header.metadata_size as usize]) + .unwrap(); + + // Verify all tensors are present (BTreeMap stores in sorted order by key) + let expected_sorted = vec!["a_tensor", "b_tensor", "m_tensor", "z_tensor"]; + let actual_names: Vec<_> = metadata.tensors.keys().collect(); + assert_eq!(actual_names, expected_sorted); +} + +#[test] +fn test_writer_lazy_snapshot_evaluation() { + // Create a lazy snapshot using closure + let data = Rc::new(vec![1.0f32, 2.0, 3.0, 4.0]); + let data_clone = data.clone(); + + let snapshot = TensorSnapshot::from_closure( + Rc::new(move || { + let bytes: Vec = data_clone.iter().flat_map(|f| f.to_le_bytes()).collect(); + Ok(TensorData::from_bytes_vec(bytes, vec![2, 2], DType::F32)) + }), + DType::F32, + vec![2, 2], + vec!["lazy".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + + // The closure should only be evaluated when to_bytes is called + let bytes = writer.to_bytes().unwrap(); + + let header = BurnpackHeader::from_bytes(&bytes[..HEADER_SIZE]).unwrap(); + let metadata_end = HEADER_SIZE + header.metadata_size as usize; + let metadata: BurnpackMetadata = + ciborium::de::from_reader(&bytes[HEADER_SIZE..metadata_end]).unwrap(); + + assert_eq!(metadata.tensors.len(), 1); + let tensor = metadata.tensors.get("lazy").unwrap(); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![2, 2]); + + // Verify the data was correctly written + // Data section starts at aligned position after metadata + let data_section_start = aligned_data_section_start(header.metadata_size as usize); + let tensor_data = &bytes[data_section_start..data_section_start + 16]; + let expected: Vec = [1.0f32, 2.0, 3.0, 4.0] + .iter() + .flat_map(|f| f.to_le_bytes()) + .collect(); + assert_eq!(tensor_data, expected.as_slice()); +} + +#[cfg(feature = "std")] +#[test] +fn test_writer_write_to_file() { + use std::fs; + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("test.bpk"); + + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["test".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]).with_metadata("file_test", "true"); + + writer.write_to_file(&file_path).unwrap(); + + // Verify file exists and has correct content + assert!(file_path.exists()); + + let file_bytes = fs::read(&file_path).unwrap(); + let memory_bytes = writer.to_bytes().unwrap(); + + assert_eq!(file_bytes.as_slice(), &*memory_bytes); +} + +#[test] +fn test_writer_size() { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["test".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]).with_metadata("test", "value"); + + let size = writer.size().unwrap(); + let bytes = writer.to_bytes().unwrap(); + + // Size should match actual bytes length + assert_eq!(size, bytes.len()); +} + +#[test] +fn test_writer_write_into() { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["test".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]).with_metadata("test", "value"); + + // Get size and allocate buffer + let size = writer.size().unwrap(); + let mut buffer = vec![0u8; size]; + + // Write into buffer + writer.write_into(&mut buffer).unwrap(); + + // Compare with to_bytes() + let bytes = writer.to_bytes().unwrap(); + assert_eq!(buffer.as_slice(), &*bytes); +} + +#[test] +fn test_writer_write_into_buffer_too_small() { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["test".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + + // Allocate a buffer that's too small + let mut buffer = vec![0u8; 10]; + + // Should fail with buffer too small error + let result = writer.write_into(&mut buffer); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Buffer too small")); +} + +#[test] +fn test_writer_write_into_buffer_larger_than_needed() { + let snapshot = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["test".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot]); + + // Allocate a larger buffer + let size = writer.size().unwrap(); + let mut buffer = vec![0u8; size + 100]; // Extra 100 bytes + + // Should succeed and only write the necessary bytes + writer.write_into(&mut buffer).unwrap(); + + // Compare the written portion with to_bytes() + let bytes = writer.to_bytes().unwrap(); + assert_eq!(&buffer[..size], &*bytes); +} + +#[test] +fn test_writer_write_into_multiple_tensors() { + let snapshot1 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![1, 2, 3, 4], vec![2, 2], DType::U8), + vec!["tensor1".to_string()], + vec![], + ParamId::new(), + ); + + let snapshot2 = TensorSnapshot::from_data( + TensorData::from_bytes_vec(vec![5, 6, 7, 8, 9, 10], vec![2, 3], DType::U8), + vec!["tensor2".to_string()], + vec![], + ParamId::new(), + ); + + let writer = BurnpackWriter::new(vec![snapshot1, snapshot2]).with_metadata("test", "multiple"); + + let size = writer.size().unwrap(); + let mut buffer = vec![0u8; size]; + writer.write_into(&mut buffer).unwrap(); + + let bytes = writer.to_bytes().unwrap(); + assert_eq!(buffer.as_slice(), &*bytes); +} + +#[test] +fn test_writer_write_into_empty() { + let writer = BurnpackWriter::new(vec![]); + + let size = writer.size().unwrap(); + let mut buffer = vec![0u8; size]; + writer.write_into(&mut buffer).unwrap(); + + let bytes = writer.to_bytes().unwrap(); + assert_eq!(buffer.as_slice(), &*bytes); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/zero_copy.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/zero_copy.rs new file mode 100644 index 0000000..b18520c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/tests/zero_copy.rs @@ -0,0 +1,211 @@ +//! Tests for zero-copy tensor loading functionality. + +use crate::ModuleStore; +use crate::burnpack::store::BurnpackStore; + +use burn_core as burn; +use burn_core::module::{Module, Param}; +use burn_tensor::{AllocationProperty, Bytes, Tensor, backend::Backend}; + +type TestBackend = burn_ndarray::NdArray; + +#[derive(Module, Debug)] +struct SimpleModule { + weight: Param>, + bias: Param>, +} + +impl SimpleModule { + fn new(device: &B::Device) -> Self { + Self { + weight: Param::from_data([[1.0f32, 2.0], [3.0, 4.0]], device), + bias: Param::from_data([0.5f32, 1.5], device), + } + } + + fn new_zeros(device: &B::Device) -> Self { + Self { + weight: Param::from_tensor(Tensor::zeros([2, 2], device)), + bias: Param::from_tensor(Tensor::zeros([2], device)), + } + } +} + +/// Test that from_static creates a store with zero_copy enabled by default. +#[test] +fn test_from_static_enables_zero_copy() { + let device = Default::default(); + let module = SimpleModule::::new(&device); + + // Save to bytes first + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Convert to Vec and then leak to get &'static [u8] + let bytes_vec: Vec = bytes.to_vec(); + let static_bytes: &'static [u8] = Box::leak(bytes_vec.into_boxed_slice()); + + // Create store from static - zero_copy should be enabled + let mut load_store = BurnpackStore::from_static(static_bytes); + + // Load into a new module + let mut loaded_module = SimpleModule::::new_zeros(&device); + load_store.apply_to(&mut loaded_module).unwrap(); + + // Verify data is correct + let loaded_weight = loaded_module.weight.val().to_data(); + let loaded_bias = loaded_module.bias.val().to_data(); + + assert_eq!( + loaded_weight.to_vec::().unwrap(), + vec![1.0, 2.0, 3.0, 4.0] + ); + assert_eq!(loaded_bias.to_vec::().unwrap(), vec![0.5, 1.5]); +} + +/// Test that zero_copy builder method works. +#[test] +fn test_zero_copy_builder_method() { + let device = Default::default(); + let module = SimpleModule::::new(&device); + + // Save to bytes first + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Create shared bytes for zero-copy + let shared = bytes::Bytes::from(bytes.to_vec()); + let cubecl_bytes = Bytes::from_shared(shared, AllocationProperty::Other); + + // Create store with zero_copy enabled + let mut load_store = BurnpackStore::from_bytes(Some(cubecl_bytes)).zero_copy(true); + + // Load into a new module + let mut loaded_module = SimpleModule::::new_zeros(&device); + load_store.apply_to(&mut loaded_module).unwrap(); + + // Verify data is correct + let loaded_weight = loaded_module.weight.val().to_data(); + assert_eq!( + loaded_weight.to_vec::().unwrap(), + vec![1.0, 2.0, 3.0, 4.0] + ); +} + +/// Test that zero_copy(false) uses copying even with shared bytes. +#[test] +fn test_zero_copy_disabled_uses_copy() { + let device = Default::default(); + let module = SimpleModule::::new(&device); + + // Save to bytes first + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Convert to Vec and then leak to get &'static [u8] + let bytes_vec: Vec = bytes.to_vec(); + let static_bytes: &'static [u8] = Box::leak(bytes_vec.into_boxed_slice()); + + // Create store from static but disable zero_copy + let mut load_store = BurnpackStore::from_static(static_bytes).zero_copy(false); + + // Load into a new module + let mut loaded_module = SimpleModule::::new_zeros(&device); + load_store.apply_to(&mut loaded_module).unwrap(); + + // Verify data is correct (copied, not zero-copy) + let loaded_weight = loaded_module.weight.val().to_data(); + assert_eq!( + loaded_weight.to_vec::().unwrap(), + vec![1.0, 2.0, 3.0, 4.0] + ); +} + +/// Test that from_bytes with regular Bytes uses copying by default. +#[test] +fn test_from_bytes_uses_copy_by_default() { + let device = Default::default(); + let module = SimpleModule::::new(&device); + + // Save to bytes + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Load from bytes (default: zero_copy = false) + let mut load_store = BurnpackStore::from_bytes(Some(bytes)); + let mut loaded_module = SimpleModule::::new_zeros(&device); + load_store.apply_to(&mut loaded_module).unwrap(); + + // Verify data is correct + let loaded_weight = loaded_module.weight.val().to_data(); + assert_eq!( + loaded_weight.to_vec::().unwrap(), + vec![1.0, 2.0, 3.0, 4.0] + ); +} + +/// Test that slice_bytes works correctly on StorageBackend. +#[test] +fn test_storage_backend_slice_bytes() { + use crate::burnpack::reader::BurnpackReader; + + let device = Default::default(); + let module = SimpleModule::::new(&device); + + // Save to bytes first + let mut save_store = BurnpackStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + // Create shared bytes + let shared = bytes::Bytes::from(bytes.to_vec()); + let cubecl_bytes = Bytes::from_shared(shared, AllocationProperty::Other); + + // Create reader and get snapshots with zero-copy + let reader = BurnpackReader::from_bytes(cubecl_bytes).unwrap(); + let snapshots = reader.get_snapshots_zero_copy(true).unwrap(); + + // Verify we got the expected number of tensors + assert_eq!(snapshots.len(), 2); + + // Load the tensor data + for snapshot in &snapshots { + let data = snapshot.to_data().unwrap(); + // Just verify we can access the data - the actual content depends on tensor order + assert!(!data.bytes.is_empty()); + } +} + +/// Test that zero_copy=true with file-based loading works (via mmap + bytes::Bytes). +#[test] +fn test_zero_copy_file_based_works() { + use tempfile::NamedTempFile; + + let device = Default::default(); + let module = SimpleModule::::new(&device); + + // Save to a temporary file + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + let mut save_store = BurnpackStore::from_file(path).overwrite(true); + save_store.collect_from(&module).unwrap(); + + // Load with zero_copy=true - should work because mmap is converted to bytes::Bytes + let mut load_store = BurnpackStore::from_file(path).zero_copy(true); + let mut loaded_module = SimpleModule::::new_zeros(&device); + + // The apply should succeed - mmap now supports zero-copy via bytes::Bytes::from_owner() + load_store.apply_to(&mut loaded_module).unwrap(); + + // Verify data is correct + let loaded_weight = loaded_module.weight.val().to_data(); + assert_eq!( + loaded_weight.to_vec::().unwrap(), + vec![1.0, 2.0, 3.0, 4.0] + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/writer.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/writer.rs new file mode 100644 index 0000000..367d53b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/burnpack/writer.rs @@ -0,0 +1,331 @@ +use super::base::{ + BurnpackError, BurnpackHeader, BurnpackMetadata, FORMAT_VERSION, HEADER_SIZE, MAGIC_NUMBER, + TENSOR_ALIGNMENT, TensorDescriptor, aligned_data_section_start, +}; +use crate::TensorSnapshot; +use alloc::collections::BTreeMap; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec; +use alloc::vec::Vec; +use burn_tensor::Bytes; + +#[cfg(feature = "std")] +use std::fs::File; +#[cfg(feature = "std")] +use std::io::Write; +#[cfg(feature = "std")] +use std::path::Path; + +/// Align an offset to the specified alignment boundary. +/// +/// Returns the smallest value >= `offset` that is a multiple of `alignment`. +#[inline] +const fn align_offset(offset: u64, alignment: u64) -> u64 { + offset.div_ceil(alignment) * alignment +} + +/// Writer for creating Burnpack files +pub struct BurnpackWriter { + /// Tensors to write + pub(crate) snapshots: Vec, + /// Metadata key-value pairs + pub(crate) metadata: BTreeMap, +} + +impl BurnpackWriter { + /// Create a new writer + pub fn new(snapshots: Vec) -> Self { + Self { + snapshots, + metadata: BTreeMap::new(), + } + } + + /// Builder pattern: add metadata and return self + pub fn with_metadata(mut self, key: &str, value: &str) -> Self { + self.metadata.insert(key.to_string(), value.to_string()); + self + } + + /// Build tensor descriptors and metadata + fn build_metadata(&self) -> Result<(BurnpackMetadata, Vec), BurnpackError> { + // Build tensor descriptors and calculate offsets with alignment + let mut tensors = BTreeMap::new(); + let mut current_offset = 0u64; + + for snapshot in &self.snapshots { + let data_len = snapshot.data_len() as u64; + + // Align the start offset for mmap zero-copy support + let aligned_start = align_offset(current_offset, TENSOR_ALIGNMENT); + let end = aligned_start.checked_add(data_len).ok_or_else(|| { + BurnpackError::IoError(format!( + "Tensor offset overflow: {} + {} exceeds maximum", + aligned_start, data_len + )) + })?; + + tensors.insert( + snapshot.full_path(), + TensorDescriptor { + dtype: snapshot.dtype, + shape: snapshot.shape.iter().map(|&s| s as u64).collect(), + data_offsets: (aligned_start, end), + param_id: snapshot.tensor_id.map(|id| id.val()), + }, + ); + + current_offset = end; + } + + // Create metadata structure + let metadata = BurnpackMetadata { + tensors, + metadata: self.metadata.clone(), + }; + + // Serialize metadata with CBOR + let mut metadata_bytes = Vec::new(); + ciborium::ser::into_writer(&metadata, &mut metadata_bytes) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + + Ok((metadata, metadata_bytes)) + } + + /// Calculate the total size needed for the burnpack data + /// + /// This is useful when you want to pre-allocate a buffer for `write_into()`. + /// The size includes padding bytes for both metadata alignment and tensor alignment. + pub fn size(&self) -> Result { + let (metadata, metadata_bytes) = self.build_metadata()?; + + // Data section starts at aligned position after header + metadata + let data_section_start = aligned_data_section_start(metadata_bytes.len()); + + // Calculate total data section size from aligned offsets + // The last tensor's end offset gives us the total data section size + let data_size = metadata + .tensors + .values() + .map(|t| t.data_offsets.1) + .max() + .unwrap_or(0) as usize; + + Ok(data_section_start + data_size) + } + + /// Write burnpack data into a caller-provided buffer + /// + /// The buffer must be large enough to hold all data. Use `size()` to determine + /// the required buffer size. If the buffer is too small, this will return an error. + /// + /// This allows the caller to control buffer allocation, enabling optimizations like: + /// - Buffer reuse across multiple writes + /// - Custom allocators + /// - Pinned memory for GPU transfers + /// + /// # Arguments + /// + /// * `buffer` - Mutable slice to write data into. Must be at least `size()` bytes. + pub fn write_into(&self, buffer: &mut [u8]) -> Result<(), BurnpackError> { + let (metadata, metadata_bytes) = self.build_metadata()?; + + // Check metadata size fits in u32 + let metadata_size: u32 = metadata_bytes.len().try_into().map_err(|_| { + BurnpackError::IoError(format!( + "Metadata size {} exceeds maximum of {} bytes", + metadata_bytes.len(), + u32::MAX + )) + })?; + + // Create header + let header = BurnpackHeader { + magic: MAGIC_NUMBER, + version: FORMAT_VERSION, + metadata_size, + }; + + // Data section starts at aligned position after header + metadata + let data_section_start = aligned_data_section_start(metadata_bytes.len()); + + // Calculate required size from aligned offsets + let data_size = metadata + .tensors + .values() + .map(|t| t.data_offsets.1) + .max() + .unwrap_or(0) as usize; + let total_size = data_section_start + data_size; + + // Check buffer size + if buffer.len() < total_size { + return Err(BurnpackError::IoError(format!( + "Buffer too small: need {} bytes, got {} bytes", + total_size, + buffer.len() + ))); + } + + let mut offset = 0; + + // Write header + let header_bytes = header.into_bytes(); + buffer[offset..offset + HEADER_SIZE].copy_from_slice(&header_bytes); + offset += HEADER_SIZE; + + // Write metadata + buffer[offset..offset + metadata_bytes.len()].copy_from_slice(&metadata_bytes); + offset += metadata_bytes.len(); + + // Write padding to align data section start + if data_section_start > offset { + buffer[offset..data_section_start].fill(0); + offset = data_section_start; + } + + // Write tensor data with alignment padding + for snapshot in &self.snapshots { + // Get the aligned offset from metadata + let descriptor = metadata.tensors.get(&snapshot.full_path()).ok_or_else(|| { + BurnpackError::IoError(format!( + "Internal error: tensor '{}' not found in metadata", + snapshot.full_path() + )) + })?; + let aligned_offset = descriptor.data_offsets.0 as usize; + let target_offset = data_section_start + aligned_offset; + + // Write padding zeros if needed + if target_offset > offset { + buffer[offset..target_offset].fill(0); + offset = target_offset; + } + + let expected_len = snapshot.data_len(); + let data = snapshot.to_data().map_err(|e| { + BurnpackError::IoError(format!("Failed to get tensor data: {:?}", e)) + })?; + let actual_len = data.bytes.len(); + + // Validate data length consistency + if actual_len != expected_len { + return Err(BurnpackError::IoError(format!( + "Data corruption: tensor '{}' has inconsistent length (expected {}, got {})", + snapshot.full_path(), + expected_len, + actual_len + ))); + } + + buffer[offset..offset + actual_len].copy_from_slice(&data.bytes); + offset += actual_len; + } + + Ok(()) + } + + /// Write to a byte buffer (convenience method) + /// + /// This allocates a buffer internally and writes the burnpack data. + /// For more control over buffer allocation, use `size()` + `write_into()`. + pub fn to_bytes(&self) -> Result { + let size = self.size()?; + let mut buffer = vec![0u8; size]; + self.write_into(&mut buffer)?; + Ok(Bytes::from_bytes_vec(buffer)) + } + + /// Write directly to a file (more memory efficient for large models) + #[cfg(feature = "std")] + pub fn write_to_file>(&self, path: P) -> Result<(), BurnpackError> { + let mut file = File::create(path).map_err(|e| BurnpackError::IoError(e.to_string()))?; + + let (metadata, metadata_bytes) = self.build_metadata()?; + + // Check metadata size fits in u32 + let metadata_size: u32 = metadata_bytes.len().try_into().map_err(|_| { + BurnpackError::IoError(format!( + "Metadata size {} exceeds maximum of {} bytes", + metadata_bytes.len(), + u32::MAX + )) + })?; + + // Create and write header + let header = BurnpackHeader { + magic: MAGIC_NUMBER, + version: FORMAT_VERSION, + metadata_size, + }; + + file.write_all(&header.into_bytes()) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + + // Write metadata + file.write_all(&metadata_bytes) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + + // Data section starts at aligned position after header + metadata + let data_section_start = aligned_data_section_start(metadata_bytes.len()); + let current_file_pos = HEADER_SIZE + metadata_bytes.len(); + + // Write padding to align data section start + if data_section_start > current_file_pos { + let padding_size = data_section_start - current_file_pos; + let padding = vec![0u8; padding_size]; + file.write_all(&padding) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + } + + // Track current position within data section (relative to data_section_start) + let mut data_offset = 0usize; + + // Stream tensor data directly to file with alignment padding + for snapshot in &self.snapshots { + // Get the aligned offset from metadata + let descriptor = metadata.tensors.get(&snapshot.full_path()).ok_or_else(|| { + BurnpackError::IoError(format!( + "Internal error: tensor '{}' not found in metadata", + snapshot.full_path() + )) + })?; + let aligned_offset = descriptor.data_offsets.0 as usize; + + // Write padding zeros if needed + if aligned_offset > data_offset { + let padding_size = aligned_offset - data_offset; + let padding = vec![0u8; padding_size]; + file.write_all(&padding) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + data_offset = aligned_offset; + } + + let expected_len = snapshot.data_len(); + let data = snapshot.to_data().map_err(|e| { + BurnpackError::IoError(format!("Failed to get tensor data: {:?}", e)) + })?; + let actual_len = data.bytes.len(); + + // Validate data length consistency + if actual_len != expected_len { + return Err(BurnpackError::IoError(format!( + "Data corruption: tensor '{}' has inconsistent length (expected {}, got {})", + snapshot.full_path(), + expected_len, + actual_len + ))); + } + + file.write_all(&data.bytes) + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + data_offset += actual_len; + } + + file.flush() + .map_err(|e| BurnpackError::IoError(e.to_string()))?; + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/collector.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/collector.rs new file mode 100644 index 0000000..e76ed57 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/collector.rs @@ -0,0 +1,1137 @@ +use alloc::boxed::Box; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; + +use burn_tensor::{Bool, Int, Tensor, backend::Backend}; + +use crate::{ModuleAdapter, PathFilter, TensorSnapshot}; +use burn_core::module::{ModuleVisitor, Param, ParamId}; + +/// Collects tensor views from modules without copying data. +/// +/// This collector traverses a module hierarchy and creates lightweight views +/// of tensors that can be materialized to `TensorData` on demand. +/// +/// # Examples +/// +/// ## Collect all tensors +/// ```rust,no_run +/// # use burn_store::Collector; +/// let collector = Collector::new(None, None, false); +/// // Use with module.visit(&mut collector); +/// let all_tensors = collector.tensors; +/// ``` +/// +/// ## Filter with single pattern +/// ```rust,no_run +/// # use burn_store::{Collector, PathFilter}; +/// let filter = PathFilter::new().with_regex(r"^encoder\..*"); +/// let collector = Collector::new(Some(filter), None, false); +/// // Use with module.visit(&mut collector); +/// // Only collects tensors starting with "encoder." +/// ``` +/// +/// ## Filter with multiple patterns (OR union) +/// ```rust,no_run +/// # use burn_store::{Collector, PathFilter}; +/// let filter = PathFilter::new() +/// .with_regex(r"^encoder\..*") // Match all encoder tensors +/// .with_regex(r".*\.bias$"); // OR match any bias tensors +/// let collector = Collector::new(Some(filter), None, false); +/// // Use with module.visit(&mut collector); +/// // Collects tensors matching ANY of the patterns +/// ``` +pub struct Collector { + /// Collection of tensor views + pub tensors: Vec, + path_stack: Vec, + container_stack: Vec, + filter: Option, + adapter: Option>, + /// Skip enum variant names when building paths + /// When true, enum variant names are not included in tensor paths + skip_enum_variants: bool, +} + +impl Default for Collector { + fn default() -> Self { + Self::new(None, None, false) + } +} + +impl Collector { + /// Create a new tensor view collector with an optional filter and adapter. + /// + /// # Arguments + /// + /// * `filter` - An optional [`PathFilter`] to determine which tensors to collect. + /// When `None`, all tensors are collected. + /// * `adapter` - Optional adapter to transform tensors based on container types. + /// Applied to all collected tensors before returning. + /// * `skip_enum_variants` - Skip enum variant names when building paths. + /// When true, paths will not include enum variant names (e.g., "feature.weight" + /// instead of "feature.BaseConv.weight"). Useful when exporting to formats + /// like PyTorch that don't use enum variants. + /// + /// # Examples + /// + /// ```rust,no_run + /// # use burn_store::{Collector, PathFilter}; + /// // Collect all tensors without adapter + /// let collector = Collector::new(None, None, false); + /// + /// // Use PathFilter builder + /// let filter = PathFilter::new() + /// .with_regex(r"^encoder\..*") + /// .with_full_path("decoder.weight"); + /// let collector = Collector::new(Some(filter), None, false); + /// + /// // Skip enum variants for PyTorch export + /// let collector = Collector::new(None, None, true); + /// ``` + pub fn new( + filter: Option, + adapter: Option>, + skip_enum_variants: bool, + ) -> Self { + Self { + tensors: Vec::new(), + path_stack: Vec::new(), + container_stack: Vec::new(), + filter, + adapter, + skip_enum_variants, + } + } + + /// Apply the adapter to collected tensors and return the result. + pub fn into_tensors(self) -> Vec { + if let Some(adapter) = self.adapter { + self.tensors + .into_iter() + .map(|snapshot| adapter.adapt(&snapshot)) + .collect() + } else { + self.tensors + } + } + + fn should_collect(&self, path: &[String], container_stack: &[String]) -> bool { + // If filter is present, use it; otherwise collect all + match &self.filter { + None => true, + Some(f) => f.matches_with_container_path(path, container_stack), + } + } +} + +impl ModuleVisitor for Collector { + fn enter_module(&mut self, name: &str, container_type: &str) { + // Always track the container type for proper filtering and module type detection + self.container_stack.push(container_type.to_string()); + + // Only add to path if it's not an enum variant (when skip_enum_variants is enabled) + // This ensures paths are built without enum variant names from the start + if !self.skip_enum_variants || !container_type.starts_with("Enum:") { + self.path_stack.push(name.to_string()); + } + } + + fn exit_module(&mut self, _name: &str, container_type: &str) { + self.container_stack.pop(); + + // Only pop from path if we added it (not an enum variant when skip_enum_variants is enabled) + if !self.skip_enum_variants || !container_type.starts_with("Enum:") { + self.path_stack.pop(); + } + } + + fn visit_float(&mut self, param: &Param>) { + if self.should_collect(&self.path_stack, &self.container_stack) { + self.tensors.push(TensorSnapshot::from_float( + ¶m.transform_for_save().val(), + self.path_stack.clone(), + self.container_stack.clone(), + param.id, + )); + } + } + + fn visit_int(&mut self, param: &Param>) { + if self.should_collect(&self.path_stack, &self.container_stack) { + self.tensors.push(TensorSnapshot::from_int( + ¶m.transform_for_save().val(), + self.path_stack.clone(), + self.container_stack.clone(), + param.id, + )); + } + } + + fn visit_bool(&mut self, param: &Param>) { + if self.should_collect(&self.path_stack, &self.container_stack) { + self.tensors.push(TensorSnapshot::from_bool( + ¶m.transform_for_save().val(), + self.path_stack.clone(), + self.container_stack.clone(), + param.id, + )); + } + } + + fn visit_float_with_path( + &mut self, + path: &[String], + id: ParamId, + tensor: &Tensor, + ) { + // For path-based visits, we use the current container stack for filtering + if self.should_collect(path, &self.container_stack) { + self.tensors.push(TensorSnapshot::from_float( + tensor, + path.to_vec(), + self.container_stack.clone(), + id, + )); + } + } + + fn visit_int_with_path( + &mut self, + path: &[String], + id: ParamId, + tensor: &Tensor, + ) { + if self.should_collect(path, &self.container_stack) { + self.tensors.push(TensorSnapshot::from_int( + tensor, + path.to_vec(), + self.container_stack.clone(), + id, + )); + } + } + + fn visit_bool_with_path( + &mut self, + path: &[String], + id: ParamId, + tensor: &Tensor, + ) { + if self.should_collect(path, &self.container_stack) { + self.tensors.push(TensorSnapshot::from_bool( + tensor, + path.to_vec(), + self.container_stack.clone(), + id, + )); + } + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + + use burn_core as burn; + + type TestBackend = burn_ndarray::NdArray; + use alloc::collections::BTreeMap; + use alloc::string::String; + use burn_core::module::{Module, Param}; + use burn_nn::LinearConfig; + + #[test] + fn tensor_snapshot_collector() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + + let mut collector = Collector::new(None, None, false); + let id = ParamId::new(); + + // Collect a tensor + collector.visit_float_with_path(&["model".to_string(), "weight".to_string()], id, &tensor); + + assert_eq!(collector.tensors.len(), 1); + assert_eq!(collector.tensors[0].full_path(), "model.weight"); + + // Verify the tensor can be converted to data + let view = &collector.tensors[0]; + let data = view.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 2]); + } + + #[test] + fn root_level_parameters() { + use burn_core::module::ModuleVisitor; + + let device = Default::default(); + + // Create root-level parameters (single-element path, not nested in modules) + let weight = Param::>::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let bias = Param::>::from_data([5.0, 6.0], &device); + + let mut collector = Collector::new(None, None, false); + + // Simulate module traversal for root-level parameters + // Enter "weight" path (as if we're visiting a field named "weight") + ModuleVisitor::::enter_module(&mut collector, "weight", ""); + ModuleVisitor::::visit_float(&mut collector, &weight); + ModuleVisitor::::exit_module(&mut collector, "weight", ""); + + // Enter "bias" path (as if we're visiting a field named "bias") + ModuleVisitor::::enter_module(&mut collector, "bias", ""); + ModuleVisitor::::visit_float(&mut collector, &bias); + ModuleVisitor::::exit_module(&mut collector, "bias", ""); + + // Verify both parameters were collected + assert_eq!(collector.tensors.len(), 2); + + // Verify paths are correct (single-element paths) + assert_eq!(collector.tensors[0].full_path(), "weight"); + assert_eq!(collector.tensors[1].full_path(), "bias"); + + // Verify data is correct + let weight_data = collector.tensors[0] + .to_data() + .unwrap() + .to_vec::() + .unwrap(); + let bias_data = collector.tensors[1] + .to_data() + .unwrap() + .to_vec::() + .unwrap(); + + assert_eq!(weight_data, vec![1.0, 2.0, 3.0, 4.0]); + assert_eq!(bias_data, vec![5.0, 6.0]); + } + + #[test] + #[cfg(target_has_atomic = "ptr")] + fn tensor_snapshot_collector_with_filter() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + + let filter = PathFilter::new().with_regex(r"^encoder\..*"); + let mut collector = Collector::new(Some(filter), None, false); + let id = ParamId::new(); + + // This should be collected + collector.visit_float_with_path( + &["encoder".to_string(), "weight".to_string()], + id, + &tensor, + ); + // This should NOT be collected + collector.visit_float_with_path( + &["decoder".to_string(), "weight".to_string()], + id, + &tensor, + ); + + assert_eq!(collector.tensors.len(), 1); + assert_eq!(collector.tensors[0].full_path(), "encoder.weight"); + } + + #[test] + #[cfg(target_has_atomic = "ptr")] + fn tensor_snapshot_collector_with_multiple_filters() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + + // Multiple patterns - collect if matches ANY (OR union) + let filter = PathFilter::new() + .with_regex(r"^encoder\..*") // Match encoder.* + .with_regex(r".*\.bias$"); // Match *.bias + let mut collector = Collector::new(Some(filter), None, false); + let id = ParamId::new(); + + // These should be collected + collector.visit_float_with_path( + &["encoder".to_string(), "weight".to_string()], + id, + &tensor, + ); // matches first pattern + collector.visit_float_with_path(&["decoder".to_string(), "bias".to_string()], id, &tensor); // matches second pattern + collector.visit_float_with_path(&["encoder".to_string(), "bias".to_string()], id, &tensor); // matches both patterns + + // This should NOT be collected + collector.visit_float_with_path( + &["decoder".to_string(), "weight".to_string()], + id, + &tensor, + ); // matches neither + + assert_eq!(collector.tensors.len(), 3); + let paths: Vec = collector.tensors.iter().map(|v| v.full_path()).collect(); + assert!(paths.contains(&"encoder.weight".to_string())); + assert!(paths.contains(&"decoder.bias".to_string())); + assert!(paths.contains(&"encoder.bias".to_string())); + assert!(!paths.contains(&"decoder.weight".to_string())); + } + + #[test] + fn tensor_snapshot_collector_with_predicate() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + + // Use predicate function for filtering + fn filter_fn(path: &str, _container_path: &str) -> bool { + path.starts_with("encoder.") || path == "decoder.bias" + } + let filter = PathFilter::new().with_predicate(filter_fn); + let mut collector = Collector::new(Some(filter), None, false); + let id = ParamId::new(); + + // These should be collected + collector.visit_float_with_path( + &["encoder".to_string(), "weight".to_string()], + id, + &tensor, + ); + collector.visit_float_with_path(&["encoder".to_string(), "bias".to_string()], id, &tensor); + collector.visit_float_with_path(&["decoder".to_string(), "bias".to_string()], id, &tensor); + + // This should NOT be collected + collector.visit_float_with_path( + &["decoder".to_string(), "weight".to_string()], + id, + &tensor, + ); + collector.visit_float_with_path(&["other".to_string(), "tensor".to_string()], id, &tensor); + + assert_eq!(collector.tensors.len(), 3); + let paths: Vec = collector.tensors.iter().map(|v| v.full_path()).collect(); + assert!(paths.contains(&"encoder.weight".to_string())); + assert!(paths.contains(&"encoder.bias".to_string())); + assert!(paths.contains(&"decoder.bias".to_string())); + assert!(!paths.contains(&"decoder.weight".to_string())); + assert!(!paths.contains(&"other.tensor".to_string())); + } + + #[test] + fn tensor_snapshot_collector_predicate_with_complex_logic() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + + // Complex predicate with multiple conditions + fn complex_filter(path: &str, _container_path: &str) -> bool { + let parts: Vec<&str> = path.split('.').collect(); + if parts.len() != 3 { + return false; + } + // Only collect if it's layer1 or layer2, and it's a weight tensor + (parts[1] == "layer1" || parts[1] == "layer2") && parts[2] == "weight" + } + let filter = PathFilter::new().with_predicate(complex_filter); + let mut collector = Collector::new(Some(filter), None, false); + let id = ParamId::new(); + + // These should be collected + collector.visit_float_with_path( + &[ + "model".to_string(), + "layer1".to_string(), + "weight".to_string(), + ], + id, + &tensor, + ); + collector.visit_float_with_path( + &[ + "model".to_string(), + "layer2".to_string(), + "weight".to_string(), + ], + id, + &tensor, + ); + + // These should NOT be collected + collector.visit_float_with_path( + &[ + "model".to_string(), + "layer1".to_string(), + "bias".to_string(), + ], + id, + &tensor, + ); + collector.visit_float_with_path( + &[ + "model".to_string(), + "layer3".to_string(), + "weight".to_string(), + ], + id, + &tensor, + ); + collector.visit_float_with_path( + &["encoder".to_string(), "weight".to_string()], + id, + &tensor, + ); // wrong structure + + assert_eq!(collector.tensors.len(), 2); + let paths: Vec = collector.tensors.iter().map(|v| v.full_path()).collect(); + assert!(paths.contains(&"model.layer1.weight".to_string())); + assert!(paths.contains(&"model.layer2.weight".to_string())); + assert!(!paths.contains(&"model.layer1.bias".to_string())); + assert!(!paths.contains(&"model.layer3.weight".to_string())); + assert!(!paths.contains(&"encoder.weight".to_string())); + } + + // Test visitor that collects tensor paths + struct TensorPathCollector { + pub paths: BTreeMap)>, + path_stack: Vec, + } + + impl TensorPathCollector { + fn new() -> Self { + Self { + paths: BTreeMap::new(), + path_stack: Vec::new(), + } + } + + fn current_path(&self) -> String { + self.path_stack.join(".") + } + } + + impl ModuleVisitor for TensorPathCollector { + fn enter_module(&mut self, name: &str, _container_type: &str) { + self.path_stack.push(name.to_string()); + } + + fn exit_module(&mut self, _name: &str, _container_type: &str) { + self.path_stack.pop(); + } + + fn visit_float(&mut self, param: &Param>) { + let path = self.current_path(); + if !path.is_empty() { + self.paths.insert( + path, + (param.id, param.transform_for_save().val().shape().to_vec()), + ); + } + } + + fn visit_int(&mut self, param: &Param>) { + let path = self.current_path(); + if !path.is_empty() { + self.paths.insert( + path, + (param.id, param.transform_for_save().val().shape().to_vec()), + ); + } + } + + fn visit_bool(&mut self, param: &Param>) { + let path = self.current_path(); + if !path.is_empty() { + self.paths.insert( + path, + (param.id, param.transform_for_save().val().shape().to_vec()), + ); + } + } + } + + // Simple nested module for testing + #[derive(Module, Debug)] + struct InnerModule { + weight: Param>, + bias: Param>, + } + + #[derive(Module, Debug)] + struct OuterModule { + layer1: InnerModule, + layer2: InnerModule, + } + + impl InnerModule { + fn new(device: &B::Device) -> Self { + Self { + weight: Param::from_data([[1.0, 2.0], [3.0, 4.0]], device), + bias: Param::from_data([5.0, 6.0], device), + } + } + } + + impl OuterModule { + fn new(device: &B::Device) -> Self { + Self { + layer1: InnerModule::new(device), + layer2: InnerModule::new(device), + } + } + } + + #[test] + fn nested_module_path_tracking() { + let device = Default::default(); + let module = OuterModule::::new(&device); + + let mut collector = TensorPathCollector::new(); + module.visit(&mut collector); + + let paths = collector.paths; + + // Verify we have the expected paths + // Note: Param fields are themselves modules, so we get an extra level + assert!(paths.contains_key("layer1.weight"), "Missing layer1.weight"); + assert!(paths.contains_key("layer1.bias"), "Missing layer1.bias"); + assert!(paths.contains_key("layer2.weight"), "Missing layer2.weight"); + assert!(paths.contains_key("layer2.bias"), "Missing layer2.bias"); + + // Verify the shapes are correct + assert_eq!(paths.get("layer1.weight").unwrap().1, vec![2, 2]); + assert_eq!(paths.get("layer1.bias").unwrap().1, vec![2]); + assert_eq!(paths.get("layer2.weight").unwrap().1, vec![2, 2]); + assert_eq!(paths.get("layer2.bias").unwrap().1, vec![2]); + } + + #[test] + fn linear_module_paths() { + let device = Default::default(); + let config = LinearConfig::new(10, 20).with_bias(true); + let linear = config.init::(&device); + + let mut collector = TensorPathCollector::new(); + linear.visit(&mut collector); + + let paths = collector.paths; + + // Linear module has weight and optional bias + assert!(paths.contains_key("weight")); + assert!(paths.contains_key("bias")); + + // Check dimensions + assert_eq!(paths.get("weight").unwrap().1, vec![10, 20]); + assert_eq!(paths.get("bias").unwrap().1, vec![20]); + } + + // Deep nesting test structures (4+ levels) + #[derive(Module, Debug)] + struct Level4Module { + weight: Param>, + bias: Param>, + } + + #[derive(Module, Debug)] + struct Level3Module { + layer: Level4Module, + extra: Level4Module, + } + + #[derive(Module, Debug)] + struct Level2Module { + block1: Level3Module, + block2: Level3Module, + } + + #[derive(Module, Debug)] + struct Level1Module { + encoder: Level2Module, + decoder: Level2Module, + } + + #[derive(Module, Debug)] + struct DeepModel { + backbone: Level1Module, + head: Level4Module, + } + + impl Level4Module { + fn new(device: &B::Device) -> Self { + Self { + weight: Param::from_data([[1.0, 2.0], [3.0, 4.0]], device), + bias: Param::from_data([5.0, 6.0], device), + } + } + } + + impl Level3Module { + fn new(device: &B::Device) -> Self { + Self { + layer: Level4Module::new(device), + extra: Level4Module::new(device), + } + } + } + + impl Level2Module { + fn new(device: &B::Device) -> Self { + Self { + block1: Level3Module::new(device), + block2: Level3Module::new(device), + } + } + } + + impl Level1Module { + fn new(device: &B::Device) -> Self { + Self { + encoder: Level2Module::new(device), + decoder: Level2Module::new(device), + } + } + } + + impl DeepModel { + fn new(device: &B::Device) -> Self { + Self { + backbone: Level1Module::new(device), + head: Level4Module::new(device), + } + } + } + + #[test] + fn deep_module_path_tracking() { + let device = Default::default(); + let model = DeepModel::::new(&device); + + let mut collector = Collector::new(None, None, false); + model.visit(&mut collector); + + let views = collector.tensors; + let paths: Vec = views.iter().map(|v| v.full_path()).collect(); + + // Test 5-level deep paths + assert!(paths.contains(&"backbone.encoder.block1.layer.weight".to_string())); + assert!(paths.contains(&"backbone.encoder.block1.layer.bias".to_string())); + assert!(paths.contains(&"backbone.encoder.block1.extra.weight".to_string())); + assert!(paths.contains(&"backbone.encoder.block1.extra.bias".to_string())); + + assert!(paths.contains(&"backbone.encoder.block2.layer.weight".to_string())); + assert!(paths.contains(&"backbone.encoder.block2.layer.bias".to_string())); + assert!(paths.contains(&"backbone.encoder.block2.extra.weight".to_string())); + assert!(paths.contains(&"backbone.encoder.block2.extra.bias".to_string())); + + assert!(paths.contains(&"backbone.decoder.block1.layer.weight".to_string())); + assert!(paths.contains(&"backbone.decoder.block1.layer.bias".to_string())); + assert!(paths.contains(&"backbone.decoder.block1.extra.weight".to_string())); + assert!(paths.contains(&"backbone.decoder.block1.extra.bias".to_string())); + + assert!(paths.contains(&"backbone.decoder.block2.layer.weight".to_string())); + assert!(paths.contains(&"backbone.decoder.block2.layer.bias".to_string())); + assert!(paths.contains(&"backbone.decoder.block2.extra.weight".to_string())); + assert!(paths.contains(&"backbone.decoder.block2.extra.bias".to_string())); + + // Test 2-level paths + assert!(paths.contains(&"head.weight".to_string())); + assert!(paths.contains(&"head.bias".to_string())); + + // Total should be 18 tensors (16 from backbone + 2 from head) + assert_eq!(views.len(), 18); + + // Verify data can be materialized + let view = views + .iter() + .find(|v| v.full_path() == "backbone.encoder.block1.layer.weight") + .unwrap(); + let data = view.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 2]); + } + + #[test] + fn deep_module_filtered_export() { + let device = Default::default(); + let model = DeepModel::::new(&device); + + // Test filtering at different depths + #[cfg(target_has_atomic = "ptr")] + { + let filter = PathFilter::new().with_regex(r"^backbone\.encoder\..*"); + let mut collector = Collector::new(Some(filter), None, false); + model.visit(&mut collector); + assert_eq!(collector.tensors.len(), 8); // Only encoder tensors + } + + // Test filtering specific blocks + #[cfg(target_has_atomic = "ptr")] + { + let filter = PathFilter::new().with_regex(r".*\.block1\..*"); + let mut collector = Collector::new(Some(filter), None, false); + model.visit(&mut collector); + assert_eq!(collector.tensors.len(), 8); // block1 in both encoder and decoder + } + + // Test filtering by tensor type at any depth + #[cfg(target_has_atomic = "ptr")] + { + let filter = PathFilter::new().with_regex(r".*\.weight$"); + let mut collector = Collector::new(Some(filter), None, false); + model.visit(&mut collector); + assert_eq!(collector.tensors.len(), 9); // All weight tensors + } + + // Test complex multi-pattern filtering + #[cfg(target_has_atomic = "ptr")] + { + let filter = PathFilter::new() + .with_regex(r"^backbone\.encoder\.block1\..*") // All encoder.block1 tensors + .with_regex(r"^backbone\.decoder\..*\.bias$") // All decoder biases + .with_regex(r"^head\.weight$"); // Head weight only + let mut collector = Collector::new(Some(filter), None, false); + model.visit(&mut collector); + + // Should have: + // - 4 from encoder.block1 (2 weights + 2 biases) + // - 4 decoder biases + // - 1 head weight + assert_eq!(collector.tensors.len(), 9); + + let paths: Vec = collector.tensors.iter().map(|v| v.full_path()).collect(); + assert!(paths.contains(&"backbone.encoder.block1.layer.weight".to_string())); + assert!(paths.contains(&"backbone.decoder.block1.layer.bias".to_string())); + assert!(paths.contains(&"head.weight".to_string())); + assert!(!paths.contains(&"head.bias".to_string())); // Not included + } + } + + use crate::traits::ModuleSnapshot; + use burn_nn::Linear; + use hashbrown::HashMap; + + // Test module with Option fields + #[derive(Module, Debug)] + struct OptionalFieldModule { + required: Param>, + optional: Option>>, + } + + impl OptionalFieldModule { + fn new_with_optional(device: &B::Device) -> Self { + Self { + required: Param::from_data([[1.0, 2.0], [3.0, 4.0]], device), + optional: Some(Param::from_data([5.0, 6.0], device)), + } + } + + fn new_without_optional(device: &B::Device) -> Self { + Self { + required: Param::from_data([[1.0, 2.0], [3.0, 4.0]], device), + optional: None, + } + } + } + + #[test] + fn optional_field_module_with_value() { + let device = Default::default(); + let module = OptionalFieldModule::::new_with_optional(&device); + + let views: HashMap = module + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + assert_eq!(views.len(), 2); + assert!(views.contains_key("required")); + assert!(views.contains_key("optional")); + } + + #[test] + fn optional_field_module_without_value() { + let device = Default::default(); + let module = OptionalFieldModule::::new_without_optional(&device); + + let views: HashMap = module + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + assert_eq!(views.len(), 1); + assert!(views.contains_key("required")); + assert!(!views.contains_key("optional")); + } + + // Test Vec of modules + #[derive(Module, Debug)] + struct VecModule { + layers: Vec>, + } + + impl VecModule { + fn new(device: &B::Device, num_layers: usize) -> Self { + Self { + layers: (0..num_layers) + .map(|_| LinearConfig::new(10, 10).init(device)) + .collect(), + } + } + } + + // Test tuple of modules + #[derive(Module, Debug)] + struct TupleModule { + layers: (Linear, Linear, Linear), + } + + impl TupleModule { + fn new(device: &B::Device) -> Self { + Self { + layers: ( + LinearConfig::new(10, 10).init(device), + LinearConfig::new(10, 10).init(device), + LinearConfig::new(10, 10).init(device), + ), + } + } + } + + #[test] + fn vec_module_collect() { + let device = Default::default(); + let module = VecModule::::new(&device, 3); + + let views: HashMap = module + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + // With the fix, all Vec items should now be properly indexed and visited + assert_eq!(views.len(), 6); // 3 layers × 2 tensors each = 6 tensors + + // Check that all indexed paths exist + assert!(views.contains_key("layers.0.weight")); + assert!(views.contains_key("layers.0.bias")); + assert!(views.contains_key("layers.1.weight")); + assert!(views.contains_key("layers.1.bias")); + assert!(views.contains_key("layers.2.weight")); + assert!(views.contains_key("layers.2.bias")); + } + + #[test] + fn tuple_module_collect() { + let device = Default::default(); + let module = TupleModule::::new(&device); + + let snapshots = module.collect(None, None, false); + assert_eq!(snapshots.len(), 6); + + let views: HashMap = + snapshots.into_iter().map(|v| (v.full_path(), v)).collect(); + + assert_eq!(views.len(), 6); + + assert!(views.contains_key("layers.0.weight")); + assert!(views.contains_key("layers.0.bias")); + assert!(views.contains_key("layers.1.weight")); + assert!(views.contains_key("layers.1.bias")); + assert!(views.contains_key("layers.2.weight")); + assert!(views.contains_key("layers.2.bias")); + } + + // Test array of modules + #[derive(Module, Debug)] + struct ArrayModule { + layers: [Linear; 3], + } + + impl ArrayModule { + fn new(device: &B::Device) -> Self { + Self { + layers: [ + LinearConfig::new(10, 10).init(device), + LinearConfig::new(10, 10).init(device), + LinearConfig::new(10, 10).init(device), + ], + } + } + } + + #[test] + fn array_module_collect() { + let device = Default::default(); + let module = ArrayModule::::new(&device); + + let views: HashMap = module + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + // All array items should be properly indexed + assert_eq!(views.len(), 6); // 3 layers × 2 tensors each = 6 tensors + + // Check indexed paths + for i in 0..3 { + assert!(views.contains_key(&format!("layers.{}.weight", i))); + assert!(views.contains_key(&format!("layers.{}.bias", i))); + } + } + + // Test enum modules + #[derive(Module, Debug)] + enum EnumModule { + LayerA(Linear), + LayerB(Linear), + LayerC(Linear), + } + + #[test] + fn enum_module_collect() { + let device = Default::default(); + + // Test variant A + let module_a = EnumModule::::LayerA(LinearConfig::new(10, 20).init(&device)); + let views_a: HashMap = module_a + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + // Should have the variant name in the path + assert_eq!(views_a.len(), 2); + assert!(views_a.contains_key("LayerA.weight")); + assert!(views_a.contains_key("LayerA.bias")); + + // Test variant B + let module_b = EnumModule::::LayerB(LinearConfig::new(10, 20).init(&device)); + let views_b: HashMap = module_b + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + assert_eq!(views_b.len(), 2); + assert!(views_b.contains_key("LayerB.weight")); + assert!(views_b.contains_key("LayerB.bias")); + } + + // Container type tracking tests + #[test] + fn linear_container_type() { + let device = Default::default(); + + #[derive(Module, Debug)] + struct ModelWithLinear { + linear: Linear, + } + + impl ModelWithLinear { + fn new(device: &B::Device) -> Self { + Self { + linear: LinearConfig::new(10, 20).init(device), + } + } + } + + let model = ModelWithLinear::::new(&device); + + let views: HashMap = model + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + // Check that tensors inside Linear layers have "Struct:Linear" as their module type + for (path, view) in views.iter() { + if path == "linear.weight" || path == "linear.bias" { + assert_eq!( + view.module_type(), + Some("Struct:Linear".to_string()), + "Tensor '{}' should have module type 'Struct:Linear'", + path + ); + } + } + } + + #[test] + fn complex_model_container_types() { + let device = Default::default(); + + #[derive(Module, Debug)] + struct ComplexModel { + linear_layers: [Linear; 2], + vec_layers: Vec>, + single_linear: Linear, + } + + impl ComplexModel { + fn new(device: &B::Device) -> Self { + Self { + linear_layers: [ + LinearConfig::new(100, 50).init(device), + LinearConfig::new(50, 10).init(device), + ], + vec_layers: vec![ + LinearConfig::new(10, 10).init(device), + LinearConfig::new(10, 10).init(device), + ], + single_linear: LinearConfig::new(10, 1).init(device), + } + } + } + + let model = ComplexModel::::new(&device); + + let views: HashMap = model + .collect(None, None, false) + .into_iter() + .map(|v| (v.full_path(), v)) + .collect(); + + // Should have 10 tensors total + assert_eq!(views.len(), 10); + + // Verify different module types + for (_path, view) in views.iter() { + assert_eq!(view.module_type(), Some("Struct:Linear".to_string())); + } + } + + #[test] + fn collect_with_container_filter() { + let device = Default::default(); + + #[derive(Module, Debug)] + struct FilterTestModel { + layers: Vec>, + } + + impl FilterTestModel { + fn new(device: &B::Device) -> Self { + Self { + layers: vec![ + LinearConfig::new(10, 10).init(device), + LinearConfig::new(10, 10).init(device), + ], + } + } + } + + let model = FilterTestModel::::new(&device); + + // Filter to only collect tensors from Linear modules + let filter = PathFilter::new().with_predicate(|_path, container_path| { + container_path.split('.').next_back() == Some("Struct:Linear") + }); + + let linear_views: Vec = model.collect(Some(filter), None, false); + + // All collected tensors should be from Linear modules + for view in linear_views.iter() { + assert_eq!( + view.module_type(), + Some("Struct:Linear".to_string()), + "All tensors should be from Linear modules" + ); + } + + // Should have collected all Linear tensors + assert_eq!(linear_views.len(), 4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/filter.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/filter.rs new file mode 100644 index 0000000..1d995b6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/filter.rs @@ -0,0 +1,625 @@ +use alloc::format; +use alloc::string::String; +use alloc::vec::Vec; +use core::fmt; + +#[cfg(feature = "std")] +use regex::Regex; + +/// A sophisticated path filter that supports multiple matching strategies. +/// +/// The filter uses an OR logic - a path is included if it matches ANY of the configured criteria. +/// This allows for flexible and powerful filtering configurations. +/// +/// # Examples +/// +/// ```rust,no_run +/// # use burn_store::PathFilter; +/// // Create a filter that matches encoder paths or any weight path +/// let filter = PathFilter::new() +/// .with_regex(r"^encoder\..*") +/// .with_regex(r".*\.weight$") +/// .with_full_path("special_tensor"); +/// +/// // Check if a path should be included +/// if filter.matches("encoder.layer1.weight") { +/// // This will match due to both regex patterns +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct PathFilter { + /// Compiled regex patterns for matching paths + #[cfg(feature = "std")] + regex_patterns: Vec, + + /// Exact full paths to match + exact_paths: Vec, + + /// Predicate functions for custom matching logic based on path and container path + /// Note: These cannot be cloned, so we store them separately + predicates: Vec bool>, + + /// If true, matches all paths (overrides other filters) + match_all: bool, +} + +impl PathFilter { + /// Create a new empty filter (matches nothing by default) + pub fn new() -> Self { + Self::default() + } + + /// Create a filter that matches all paths + pub fn all() -> Self { + Self { + match_all: true, + ..Default::default() + } + } + + /// Create a filter that matches nothing + pub fn none() -> Self { + Self::default() + } + + /// Add a regex pattern for matching paths + #[cfg(feature = "std")] + pub fn with_regex>(mut self, pattern: S) -> Self { + if let Ok(regex) = Regex::new(pattern.as_ref()) { + self.regex_patterns.push(regex); + } + // TODO: Consider returning Result to handle regex compilation errors + self + } + + /// Add multiple regex patterns + #[cfg(feature = "std")] + pub fn with_regexes(mut self, patterns: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + for pattern in patterns { + if let Ok(regex) = Regex::new(pattern.as_ref()) { + self.regex_patterns.push(regex); + } + } + self + } + + /// Add an exact full path to match + pub fn with_full_path>(mut self, path: S) -> Self { + self.exact_paths.push(path.into()); + self + } + + /// Add multiple exact full paths + pub fn with_full_paths(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.exact_paths.extend(paths.into_iter().map(|p| p.into())); + self + } + + /// Add a predicate function for custom matching based on path and container path + pub fn with_predicate(mut self, predicate: fn(&str, &str) -> bool) -> Self { + self.predicates.push(predicate); + self + } + + /// Add multiple predicates + pub fn with_predicates(mut self, predicates: I) -> Self + where + I: IntoIterator bool>, + { + self.predicates.extend(predicates); + self + } + + /// Set to match all paths + pub fn match_all(mut self) -> Self { + self.match_all = true; + self + } + + /// Check if a path matches this filter (assumes empty container path for backward compatibility) + pub fn matches(&self, path: &str) -> bool { + self.matches_with_container_path_str(path, "") + } + + /// Check if a path and container type match this filter (for backward compatibility) + pub fn matches_with_container(&self, path: &str, container_type: &str) -> bool { + // For backward compatibility, treat single container type as the full path + self.matches_with_container_path_str(path, container_type) + } + + /// Check if a path and container path match this filter + pub fn matches_with_container_path(&self, path: &[String], container_stack: &[String]) -> bool { + let path_str = path.join("."); + let container_path = container_stack.join("."); + self.matches_with_container_path_str(&path_str, &container_path) + } + + /// Check if a path and container path (dot-notated strings) match this filter + pub fn matches_with_container_path_str(&self, path: &str, container_path: &str) -> bool { + // If match_all is set, always return true + if self.match_all { + return true; + } + + // If no filters are configured, match nothing + if self.is_empty() { + return false; + } + + // Check exact path matches + if self.exact_paths.iter().any(|p| p == path) { + return true; + } + + // Check regex patterns (on the path) + #[cfg(feature = "std")] + { + for regex in &self.regex_patterns { + if regex.is_match(path) { + return true; + } + } + } + + // Check predicates with container path + if self + .predicates + .iter() + .any(|pred| pred(path, container_path)) + { + return true; + } + + false + } + + /// Check if the filter is empty (matches nothing) + pub fn is_empty(&self) -> bool { + if self.match_all { + return false; + } + + #[cfg(feature = "std")] + let regex_empty = self.regex_patterns.is_empty(); + #[cfg(not(feature = "std"))] + let regex_empty = true; + + self.exact_paths.is_empty() && self.predicates.is_empty() && regex_empty + } + + /// Get the number of filter criteria configured + pub fn criteria_count(&self) -> usize { + if self.match_all { + return 1; + } + + #[allow(unused_mut)] + let mut count = self.exact_paths.len() + self.predicates.len(); + + #[cfg(feature = "std")] + { + count += self.regex_patterns.len(); + } + + count + } + + /// Clear all regex patterns + #[cfg(feature = "std")] + pub fn clear_regex(&mut self) -> &mut Self { + self.regex_patterns.clear(); + self + } + + /// Clear all exact paths + pub fn clear_paths(&mut self) -> &mut Self { + self.exact_paths.clear(); + self + } + + /// Clear all predicates + pub fn clear_predicates(&mut self) -> &mut Self { + self.predicates.clear(); + self + } + + /// Clear all filters + pub fn clear(&mut self) -> &mut Self { + #[cfg(feature = "std")] + self.clear_regex(); + + self.clear_paths().clear_predicates(); + self.match_all = false; + self + } + + /// Create a filter from regex patterns only + #[cfg(feature = "std")] + pub fn from_regex_patterns(patterns: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + Self::new().with_regexes(patterns) + } + + /// Create a filter from exact paths only + pub fn from_paths(paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + Self::new().with_full_paths(paths) + } + + /// Create a filter from a single predicate + pub fn from_predicate(predicate: fn(&str, &str) -> bool) -> Self { + Self::new().with_predicate(predicate) + } + + /// Combine with another filter using OR logic + pub fn or(mut self, other: Self) -> Self { + if self.match_all || other.match_all { + return Self::all(); + } + + #[cfg(feature = "std")] + { + self.regex_patterns.extend(other.regex_patterns); + } + + self.exact_paths.extend(other.exact_paths); + self.predicates.extend(other.predicates); + + self + } +} + +impl fmt::Display for PathFilter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.match_all { + return write!(f, "PathFilter::all()"); + } + + if self.is_empty() { + return write!(f, "PathFilter::none()"); + } + + write!(f, "PathFilter[")?; + + let mut parts = Vec::new(); + + #[cfg(feature = "std")] + if !self.regex_patterns.is_empty() { + parts.push(format!("regex: {:?}", self.regex_patterns)); + } + + if !self.exact_paths.is_empty() { + parts.push(format!("paths: {:?}", self.exact_paths)); + } + + if !self.predicates.is_empty() { + parts.push(format!("predicates: {}", self.predicates.len())); + } + + write!(f, "{}]", parts.join(", ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_filter() { + let filter = PathFilter::new(); + assert!(filter.is_empty()); + assert!(!filter.matches("encoder.weight")); + assert!(!filter.matches("decoder.bias")); + } + + #[test] + fn match_all() { + let filter = PathFilter::all(); + assert!(!filter.is_empty()); + assert!(filter.matches("encoder.weight")); + assert!(filter.matches("decoder.bias")); + assert!(filter.matches("anything")); + } + + #[test] + fn exact_paths() { + let filter = PathFilter::new() + .with_full_path("encoder.weight") + .with_full_path("decoder.bias"); + + assert!(filter.matches("encoder.weight")); + assert!(filter.matches("decoder.bias")); + assert!(!filter.matches("encoder.bias")); + assert!(!filter.matches("decoder.weight")); + } + + #[test] + #[cfg(feature = "std")] + fn regex_patterns() { + let filter = PathFilter::new() + .with_regex(r"^encoder\..*") + .with_regex(r".*\.weight$"); + + assert!(filter.matches("encoder.layer1.bias")); + assert!(filter.matches("decoder.weight")); + assert!(filter.matches("encoder.weight")); + assert!(!filter.matches("decoder.bias")); + } + + #[test] + fn predicates() { + fn contains_norm(path: &str, _container_path: &str) -> bool { + path.contains("norm") + } + + fn is_short(path: &str, _container_path: &str) -> bool { + path.len() < 10 + } + + let filter = PathFilter::new() + .with_predicate(contains_norm) + .with_predicate(is_short); + + assert!(filter.matches("norm.weight")); + assert!(filter.matches("layer.norm.bias")); + assert!(filter.matches("bias")); + assert!(!filter.matches("encoder.decoder.weight.long.name")); + } + + #[test] + fn combined_filters() { + let filter = PathFilter::new() + .with_full_path("special.tensor") + .with_predicate(|path, _container_path| path.contains("attention")); + + #[cfg(feature = "std")] + let filter = filter.with_regex(r"^encoder\..*"); + + assert!(filter.matches("special.tensor")); + assert!(filter.matches("self_attention.query")); + + #[cfg(feature = "std")] + assert!(filter.matches("encoder.anything")); + + assert!(!filter.matches("decoder.weight")); + } + + #[test] + fn or_combination() { + let encoder_filter = PathFilter::new().with_full_path("encoder.weight"); + let decoder_filter = PathFilter::new().with_full_path("decoder.bias"); + + let combined = encoder_filter.or(decoder_filter); + + assert!(combined.matches("encoder.weight")); + assert!(combined.matches("decoder.bias")); + assert!(!combined.matches("model.head.weight")); + } + + #[test] + #[cfg(feature = "std")] + fn common_patterns() { + // Test encoder pattern + let encoder = PathFilter::new().with_regex(r"^encoder\..*"); + assert!(encoder.matches("encoder.weight")); + assert!(!encoder.matches("decoder.weight")); + + // Test weights-only pattern + let weights = PathFilter::new().with_regex(r".*\.weight$"); + assert!(weights.matches("encoder.weight")); + assert!(weights.matches("decoder.weight")); + assert!(!weights.matches("encoder.bias")); + + // Test layer-specific patterns + let layers = PathFilter::new() + .with_regex(r"(^|.*\.)layers\.0\.") + .with_regex(r"(^|.*\.)layers\.2\.") + .with_regex(r"(^|.*\.)layers\.4\."); + assert!(layers.matches("model.layers.0.weight")); + assert!(layers.matches("layers.2.bias")); + assert!(!layers.matches("layers.1.weight")); + } + + #[test] + fn criteria_count() { + let filter = PathFilter::new() + .with_full_path("path1") + .with_full_path("path2") + .with_predicate(|_, _| true); + + #[cfg(feature = "std")] + let filter = filter.with_regex(".*"); + + #[cfg(feature = "std")] + assert_eq!(filter.criteria_count(), 4); + + #[cfg(not(feature = "std"))] + assert_eq!(filter.criteria_count(), 3); + } + + #[test] + fn clear_operations() { + let mut filter = PathFilter::new().with_full_path("test"); + + filter.clear_paths(); + assert!(!filter.matches("test")); + + filter.clear(); + assert!(filter.is_empty()); + } + + #[test] + fn container_predicates() { + // Filter that matches only Linear module weights + let linear_weights = PathFilter::new().with_predicate(|path, container_path| { + container_path.split('.').next_back() == Some("Linear") && path.ends_with(".weight") + }); + + assert!(linear_weights.matches_with_container("layer1.weight", "Linear")); + assert!(!linear_weights.matches_with_container("layer1.weight", "Conv2d")); + assert!(!linear_weights.matches_with_container("layer1.bias", "Linear")); + + // Filter for specific container types + let conv_only = PathFilter::new().with_predicate(|_path, container_path| { + let last = container_path.split('.').next_back(); + last == Some("Conv2d") || last == Some("ConvTranspose2d") + }); + + assert!(conv_only.matches_with_container("encoder.weight", "Conv2d")); + assert!(conv_only.matches_with_container("decoder.weight", "ConvTranspose2d")); + assert!(!conv_only.matches_with_container("fc.weight", "Linear")); + + // Combine path and container predicates + let combined = PathFilter::new() + .with_predicate(|path, _container_path| path.starts_with("encoder.")) + .with_predicate(|_path, container_path| { + container_path.split('.').next_back() == Some("BatchNorm2d") + }); + + // Should match either condition (OR logic) + assert!(combined.matches_with_container("encoder.layer1", "Linear")); + assert!(combined.matches_with_container("decoder.bn", "BatchNorm2d")); + assert!(!combined.matches_with_container("decoder.layer", "Linear")); + } + + #[test] + fn container_predicate_with_regex() { + // Combine regex patterns with container predicates + #[cfg(feature = "std")] + { + let filter = PathFilter::new() + .with_regex(r"^encoder\..*") + .with_predicate(|path, container_path| { + container_path.split('.').next_back() == Some("Linear") + && path.contains(".bias") + }); + + // Matches due to regex + assert!(filter.matches_with_container("encoder.layer1.weight", "Conv2d")); + // Matches due to container predicate + assert!(filter.matches_with_container("decoder.fc.bias", "Linear")); + // Doesn't match either + assert!(!filter.matches_with_container("decoder.conv.weight", "Conv2d")); + } + } + + #[test] + fn container_stack_predicates() { + // Filter using full container path - only tensors nested in a specific hierarchy + let nested_filter = PathFilter::new().with_predicate(|_path, container_path| { + // Check if tensor is nested within: Model -> TransformerBlock -> Linear + let parts: Vec<&str> = container_path.split('.').collect(); + parts.len() >= 3 + && parts[0] == "Model" + && parts[1] == "TransformerBlock" + && parts[2] == "Linear" + }); + + assert!(nested_filter.matches_with_container_path_str( + "encoder.weight", + "Model.TransformerBlock.Linear.Param" + )); + assert!( + !nested_filter + .matches_with_container_path_str("decoder.weight", "Model.Decoder.Linear.Param") + ); + assert!(!nested_filter.matches_with_container_path_str( + "encoder.weight", + "Model.TransformerBlock.Conv2d.Param" + )); + + // Filter that checks for specific depth in hierarchy + let depth_filter = PathFilter::new().with_predicate(|_path, container_path| { + let parts: Vec<&str> = container_path.split('.').collect(); + parts.len() == 4 && parts.get(2) == Some(&"Linear") + }); + + assert!(depth_filter.matches_with_container_path_str( + "model.layer.weight", + "Model.TransformerBlock.Linear.Param" + )); + assert!( + !depth_filter + .matches_with_container_path_str("model.weight", "Model.TransformerBlock.Conv2d") + ); // Too shallow + + // Filter that checks any Linear in the path (not just the last) + let any_linear = PathFilter::new() + .with_predicate(|_path, container_path| container_path.contains("Linear")); + + assert!( + any_linear.matches_with_container_path_str( + "some.path", + "Model.TransformerBlock.Linear.Param" + ) + ); + assert!( + any_linear.matches_with_container_path_str("other.path", "Model.Decoder.Linear.Param") + ); + assert!( + !any_linear.matches_with_container_path_str( + "conv.path", + "Model.TransformerBlock.Conv2d.Param" + ) + ); + } + + #[test] + fn container_path_dot_notation() { + // Filter using dot-notated container path + let dot_filter = PathFilter::new().with_predicate(|_path, container_path| { + container_path.starts_with("Model.TransformerBlock") + }); + + // Test with matches_with_container_path + assert!( + dot_filter.matches_with_container_path_str("weight", "Model.TransformerBlock.Linear") + ); + assert!(!dot_filter.matches_with_container_path_str("weight", "Model.Decoder.Linear")); + + // Filter that checks for specific patterns in container path + let pattern_filter = PathFilter::new().with_predicate(|_path, container_path| { + // Match any path that has Linear after a block + container_path.contains("Block.Linear") || container_path.contains("Block.Conv") + }); + + assert!( + pattern_filter + .matches_with_container_path_str("weight", "Model.TransformerBlock.Linear") + ); + assert!(pattern_filter.matches_with_container_path_str("weight", "Model.ResBlock.Conv2d")); + assert!(!pattern_filter.matches_with_container_path_str("weight", "Model.Linear.Param")); + + // Filter combining path and container path patterns + let combined = PathFilter::new().with_predicate(|path, container_path| { + // Only weights in Linear layers that are inside blocks + path.ends_with(".weight") + && container_path.contains("Block") + && container_path.split('.').next_back() == Some("Linear") + }); + + assert!( + combined + .matches_with_container_path_str("layer.weight", "Model.TransformerBlock.Linear") + ); + assert!( + !combined + .matches_with_container_path_str("layer.bias", "Model.TransformerBlock.Linear") + ); + assert!(!combined.matches_with_container_path_str("layer.weight", "Model.Decoder.Linear")); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/keyremapper.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/keyremapper.rs new file mode 100644 index 0000000..12537aa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/keyremapper.rs @@ -0,0 +1,674 @@ +use alloc::collections::BTreeMap; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; + +use regex::{self, Regex}; + +use crate::TensorSnapshot; + +/// Key remapper for transforming tensor names. +/// +/// This allows mapping tensor names from one naming convention to another, +/// which is useful for loading models from different frameworks or versions. +/// +/// # Examples +/// +/// ```rust +/// # use burn_store::KeyRemapper; +/// // Create a key remapper +/// let remapper = KeyRemapper::new() +/// .add_pattern(r"^pytorch\.(.*)", "burn.$1").expect("valid regex") // pytorch.layer -> burn.layer +/// .add_pattern(r"\.gamma$", ".weight").expect("valid regex"); // layer.gamma -> layer.weight +/// +/// // Use remapper with stores +/// // store.remap(remapper) +/// ``` +#[derive(Debug, Clone, Default)] +pub struct KeyRemapper { + /// Pattern-based remapping rules (regex pattern, replacement string) + pub patterns: Vec<(Regex, String)>, +} + +impl KeyRemapper { + /// Create a new empty key remapper + pub fn new() -> Self { + Self::default() + } + + /// Add a remapping pattern (compiles regex) + /// + /// # Arguments + /// + /// * `from` - Source pattern (regex string) + /// * `to` - Replacement string (can include capture groups like `$1`) + /// + /// # Returns + /// + /// * `Ok(Self)` - Updated remapping configuration + /// * `Err(regex::Error)` - If regex compilation fails + pub fn add_pattern(mut self, from: S1, to: S2) -> Result + where + S1: AsRef, + S2: Into, + { + let regex = Regex::new(from.as_ref())?; + self.patterns.push((regex, to.into())); + Ok(self) + } + + /// Create from a list of compiled regex patterns + pub fn from_compiled_patterns(patterns: Vec<(Regex, String)>) -> Self { + Self { patterns } + } + + /// Create from string patterns (will compile to regex) + /// + /// # Arguments + /// + /// * `patterns` - Vector of (pattern, replacement) tuples + /// + /// # Returns + /// + /// * `Ok(Self)` - New remapping configuration + /// * `Err(regex::Error)` - If any regex compilation fails + pub fn from_patterns(patterns: Vec<(S1, S2)>) -> Result + where + S1: AsRef, + S2: Into, + { + let mut compiled_patterns = Vec::new(); + for (pattern, replacement) in patterns { + let regex = Regex::new(pattern.as_ref())?; + compiled_patterns.push((regex, replacement.into())); + } + Ok(Self { + patterns: compiled_patterns, + }) + } + + /// Create from an iterator of patterns + /// + /// # Arguments + /// + /// * `iter` - Iterator yielding (pattern, replacement) tuples + /// + /// # Returns + /// + /// * `Ok(Self)` - New remapping configuration + /// * `Err(regex::Error)` - If any regex compilation fails + pub fn from_pattern_iter(iter: I) -> Result + where + I: IntoIterator, + S1: AsRef, + S2: Into, + { + let patterns: Result, _> = iter + .into_iter() + .map(|(from, to)| Ok((Regex::new(from.as_ref())?, to.into()))) + .collect(); + Ok(Self { + patterns: patterns?, + }) + } + + /// Check if the remapping is empty + pub fn is_empty(&self) -> bool { + self.patterns.is_empty() + } + + /// Convert to the format expected by remap_tensor_paths_with_patterns + pub fn to_regex_pairs(&self) -> Vec<(Regex, String)> { + self.patterns.clone() + } + + /// Remap tensor paths using the configured patterns. + /// + /// # Arguments + /// + /// * `tensors` - Vec of TensorSnapshots to remap + /// + /// # Returns + /// + /// A tuple containing: + /// * The remapped Vec of TensorSnapshots with updated paths + /// * A vector of (new_path, original_path) showing the transformations + pub fn remap( + &self, + mut tensors: Vec, + ) -> (Vec, Vec<(String, String)>) { + if self.patterns.is_empty() { + let remapped_names = tensors + .iter() + .map(|v| { + let path = v.full_path(); + (path.clone(), path) + }) + .collect(); + return (tensors, remapped_names); + } + + let mut remapped_snapshots = Vec::new(); + let mut remapped_names = Vec::new(); + + for mut snapshot in tensors.drain(..) { + let original_path = snapshot.full_path(); + let mut new_path = original_path.clone(); + + // Apply all patterns to get the new path + for (pattern, replacement) in &self.patterns { + if pattern.is_match(&new_path) { + new_path = pattern + .replace_all(&new_path, replacement.as_str()) + .to_string(); + } + } + + // Update the snapshot's internal path_stack if the path changed + if new_path != original_path + && let Some(ref mut path_stack) = snapshot.path_stack + { + *path_stack = new_path.split('.').map(|s| s.to_string()).collect(); + } + + remapped_names.push((new_path.clone(), original_path)); + remapped_snapshots.push(snapshot); + } + + (remapped_snapshots, remapped_names) + } +} + +/// Map tensor paths to have contiguous numeric indices. +/// +/// This function detects numeric indices in tensor paths and renumbers them +/// to be contiguous (0, 1, 2, ...) while preserving their relative order. +/// It handles nested sequential structures by processing ALL numeric indices +/// in each path independently based on their position context. +/// +/// This is useful when loading PyTorch models that have gaps in layer numbering, +/// such as when using `nn.Sequential` with mixed layer types (e.g., Conv2d + ReLU +/// where only Conv2d has parameters). +/// +/// # Example +/// +/// Simple case - input paths: +/// - `fc.0.weight`, `fc.0.bias` +/// - `fc.2.weight`, `fc.2.bias` +/// - `fc.4.weight`, `fc.4.bias` +/// +/// Output paths: +/// - `fc.0.weight`, `fc.0.bias` +/// - `fc.1.weight`, `fc.1.bias` +/// - `fc.2.weight`, `fc.2.bias` +/// +/// Nested case - input paths: +/// - `feature.layers.0.conv_block.0.weight` +/// - `feature.layers.0.conv_block.2.weight` +/// - `feature.layers.2.conv_block.0.weight` +/// - `feature.layers.2.conv_block.2.weight` +/// +/// Output paths: +/// - `feature.layers.0.conv_block.0.weight` +/// - `feature.layers.0.conv_block.1.weight` +/// - `feature.layers.1.conv_block.0.weight` +/// - `feature.layers.1.conv_block.1.weight` +/// +/// # Arguments +/// +/// * `tensors` - Vec of TensorSnapshots to map +/// +/// # Returns +/// +/// A tuple containing: +/// * The mapped Vec of TensorSnapshots with updated paths +/// * A vector of (new_path, original_path) showing the transformations +pub fn map_indices_contiguous( + mut tensors: Vec, +) -> (Vec, Vec<(String, String)>) { + if tensors.is_empty() { + return (tensors, Vec::new()); + } + + // Step 1: Collect all paths and find all index positions + // For each index position (identified by prefix using ORIGINAL indices), + // collect all indices seen at that position. + // + // Key: prefix using original path (e.g., "feature.layers." or "feature.layers.0.conv_block.") + // Value: BTreeMap of original_index -> new_index + let mut index_maps: BTreeMap> = BTreeMap::new(); + + // First pass: collect all indices at each position using original prefixes + for snapshot in &tensors { + let path = snapshot.full_path(); + let parts: Vec<&str> = path.split('.').collect(); + + // Check each part for numeric indices + for (i, part) in parts.iter().enumerate() { + if let Ok(index) = part.parse::() { + // The prefix is everything before this index (using original path) + let prefix = if i > 0 { + format!("{}.", parts[..i].join(".")) + } else { + String::new() + }; + + index_maps + .entry(prefix) + .or_default() + .entry(index) + .or_insert(usize::MAX); // Placeholder + } + } + } + + // Second pass: assign contiguous indices for each position + for indices in index_maps.values_mut() { + let mut sorted_indices: Vec = indices.keys().cloned().collect(); + sorted_indices.sort(); + + for (new_idx, old_idx) in sorted_indices.into_iter().enumerate() { + indices.insert(old_idx, new_idx); + } + } + + // Third pass: apply the remapping to all tensors + // We use original prefixes for lookup since that's how we collected indices + let mut mapped_snapshots = Vec::new(); + let mut transformations = Vec::new(); + + for mut snapshot in tensors.drain(..) { + let original_path = snapshot.full_path(); + let new_path = remap_all_indices_with_original_prefix(&original_path, &index_maps); + + // Update the snapshot's internal path_stack if the path changed + if new_path != original_path + && let Some(ref mut path_stack) = snapshot.path_stack + { + *path_stack = new_path.split('.').map(|s| s.to_string()).collect(); + } + + transformations.push((new_path, original_path)); + mapped_snapshots.push(snapshot); + } + + (mapped_snapshots, transformations) +} + +/// Remap all numeric indices in a path using the provided index maps. +/// Uses original path prefixes for lookup. +fn remap_all_indices_with_original_prefix( + path: &str, + index_maps: &BTreeMap>, +) -> String { + let parts: Vec<&str> = path.split('.').collect(); + let mut result_parts: Vec = Vec::with_capacity(parts.len()); + + for (i, part) in parts.iter().enumerate() { + if let Ok(index) = part.parse::() { + // Build the prefix from ORIGINAL parts (not remapped) + let prefix = if i > 0 { + format!("{}.", parts[..i].join(".")) + } else { + String::new() + }; + + // Look up the new index using original prefix + if let Some(index_map) = index_maps.get(&prefix) + && let Some(&new_index) = index_map.get(&index) + { + result_parts.push(new_index.to_string()); + continue; + } + } + // Not a numeric index or no mapping found, keep as-is + result_parts.push((*part).to_string()); + } + + result_parts.join(".") +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + use burn_core::module::ParamId; + use burn_tensor::TensorData; + + fn create_test_tensor_snapshot(name: &str) -> TensorSnapshot { + let data = TensorData { + bytes: burn_tensor::Bytes::from_bytes_vec(vec![1, 2, 3, 4]), + shape: vec![2, 2], + dtype: burn_tensor::DType::F32, + }; + let path_parts: Vec = name.split('.').map(|s| s.to_string()).collect(); + TensorSnapshot::from_data(data, path_parts, vec!["Test".to_string()], ParamId::new()) + } + + #[test] + fn test_key_remapper_basic() { + let remapper = KeyRemapper::new() + .add_pattern(r"^encoder\.", "transformer.encoder.") + .expect("valid regex"); + + let tensors = vec![ + create_test_tensor_snapshot("encoder.layer1.weight"), + create_test_tensor_snapshot("decoder.layer1.weight"), + ]; + + let (remapped, transformations) = remapper.remap(tensors); + + // Check that remapped views exist with correct paths + assert!( + remapped + .iter() + .any(|v| v.full_path() == "transformer.encoder.layer1.weight") + ); + assert!( + remapped + .iter() + .any(|v| v.full_path() == "decoder.layer1.weight") + ); + assert_eq!(remapped.len(), 2); + + // Check transformations + let encoder_transform = transformations + .iter() + .find(|(_new, old)| old == "encoder.layer1.weight") + .expect("should find encoder transformation"); + assert_eq!(encoder_transform.0, "transformer.encoder.layer1.weight"); + } + + #[test] + fn test_key_remapper_multiple_patterns() { + let remapper = KeyRemapper::new() + .add_pattern(r"^encoder\.", "transformer.encoder.") + .expect("valid regex") + .add_pattern(r"\.gamma$", ".weight") + .expect("valid regex"); + + let tensors = vec![create_test_tensor_snapshot("encoder.layer1.gamma")]; + + let (remapped, _) = remapper.remap(tensors); + + assert!( + remapped + .iter() + .any(|v| v.full_path() == "transformer.encoder.layer1.weight") + ); + assert_eq!(remapped.len(), 1); + } + + #[test] + fn test_key_remapper_from_patterns() { + let patterns = vec![(r"^pytorch\.", "burn."), (r"\.bias$", ".bias_param")]; + let remapper = KeyRemapper::from_patterns(patterns).expect("valid patterns"); + + let tensors = vec![create_test_tensor_snapshot("pytorch.linear.bias")]; + + let (remapped, _) = remapper.remap(tensors); + + assert!( + remapped + .iter() + .any(|v| v.full_path() == "burn.linear.bias_param") + ); + } + + #[test] + fn test_key_remapper_empty() { + let remapper = KeyRemapper::new(); + assert!(remapper.is_empty()); + + let tensors = vec![create_test_tensor_snapshot("test.weight")]; + + let (remapped, transformations) = remapper.remap(tensors); + + assert!(remapped.iter().any(|v| v.full_path() == "test.weight")); + assert_eq!(remapped.len(), 1); + assert_eq!(transformations.len(), 1); + assert_eq!( + transformations[0], + ("test.weight".to_string(), "test.weight".to_string()) + ); + } + + #[test] + fn test_map_indices_contiguous_basic() { + // Simulate PyTorch nn.Sequential with Conv2d (0, 2, 4) and ReLU (1, 3, 5) + // Only Conv2d layers have parameters + let tensors = vec![ + create_test_tensor_snapshot("fc.0.weight"), + create_test_tensor_snapshot("fc.0.bias"), + create_test_tensor_snapshot("fc.2.weight"), + create_test_tensor_snapshot("fc.2.bias"), + create_test_tensor_snapshot("fc.4.weight"), + create_test_tensor_snapshot("fc.4.bias"), + ]; + + let (reindexed, transformations) = map_indices_contiguous(tensors); + + // Check that indices are now contiguous + assert!(reindexed.iter().any(|v| v.full_path() == "fc.0.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.0.bias")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.1.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.1.bias")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.2.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.2.bias")); + assert_eq!(reindexed.len(), 6); + + // Check transformations + let transform_2_to_1 = transformations + .iter() + .find(|(_, old)| old == "fc.2.weight") + .expect("should find fc.2.weight transformation"); + assert_eq!(transform_2_to_1.0, "fc.1.weight"); + + let transform_4_to_2 = transformations + .iter() + .find(|(_, old)| old == "fc.4.weight") + .expect("should find fc.4.weight transformation"); + assert_eq!(transform_4_to_2.0, "fc.2.weight"); + } + + #[test] + fn test_map_indices_contiguous_already_contiguous() { + // Already contiguous indices should remain unchanged + let tensors = vec![ + create_test_tensor_snapshot("fc.0.weight"), + create_test_tensor_snapshot("fc.1.weight"), + create_test_tensor_snapshot("fc.2.weight"), + ]; + + let (reindexed, transformations) = map_indices_contiguous(tensors); + + assert!(reindexed.iter().any(|v| v.full_path() == "fc.0.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.1.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.2.weight")); + assert_eq!(reindexed.len(), 3); + + // All transformations should have same old and new paths + for (new, old) in &transformations { + assert_eq!(new, old); + } + } + + #[test] + fn test_map_indices_contiguous_multiple_prefixes() { + // Different prefixes should be mapped independently + let tensors = vec![ + create_test_tensor_snapshot("encoder.0.weight"), + create_test_tensor_snapshot("encoder.2.weight"), + create_test_tensor_snapshot("decoder.1.weight"), + create_test_tensor_snapshot("decoder.5.weight"), + ]; + + let (reindexed, _) = map_indices_contiguous(tensors); + + // encoder: 0, 2 -> 0, 1 + assert!( + reindexed + .iter() + .any(|v| v.full_path() == "encoder.0.weight") + ); + assert!( + reindexed + .iter() + .any(|v| v.full_path() == "encoder.1.weight") + ); + + // decoder: 1, 5 -> 0, 1 + assert!( + reindexed + .iter() + .any(|v| v.full_path() == "decoder.0.weight") + ); + assert!( + reindexed + .iter() + .any(|v| v.full_path() == "decoder.1.weight") + ); + } + + #[test] + fn test_map_indices_contiguous_no_indices() { + // Paths without indices should remain unchanged + let tensors = vec![ + create_test_tensor_snapshot("encoder.weight"), + create_test_tensor_snapshot("decoder.bias"), + ]; + + let (reindexed, transformations) = map_indices_contiguous(tensors); + + assert!(reindexed.iter().any(|v| v.full_path() == "encoder.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "decoder.bias")); + + for (new, old) in &transformations { + assert_eq!(new, old); + } + } + + #[test] + fn test_map_indices_contiguous_empty() { + let tensors: Vec = vec![]; + let (reindexed, transformations) = map_indices_contiguous(tensors); + + assert!(reindexed.is_empty()); + assert!(transformations.is_empty()); + } + + #[test] + fn test_map_indices_contiguous_mixed_indexed_and_non_indexed() { + // Mix of indexed and non-indexed paths + let tensors = vec![ + create_test_tensor_snapshot("fc.0.weight"), + create_test_tensor_snapshot("fc.2.weight"), + create_test_tensor_snapshot("output.weight"), // no index + ]; + + let (reindexed, _) = map_indices_contiguous(tensors); + + assert!(reindexed.iter().any(|v| v.full_path() == "fc.0.weight")); + assert!(reindexed.iter().any(|v| v.full_path() == "fc.1.weight")); // 2 -> 1 + assert!(reindexed.iter().any(|v| v.full_path() == "output.weight")); // unchanged + } + + #[test] + fn test_map_indices_contiguous_nested_sequential() { + // Test nested sequential structures like: + // feature = nn.Sequential(ConvBlock, ReLU, ConvBlock, ReLU, ConvBlock) + // where ConvBlock = nn.Sequential(Conv2d, ReLU, Conv2d) + // + // This produces paths like: + // feature.layers.0.conv_block.0.weight (layer 0, conv 0) + // feature.layers.0.conv_block.2.weight (layer 0, conv 2 - skipping ReLU at 1) + // feature.layers.2.conv_block.0.weight (layer 2 - skipping ReLU at 1, conv 0) + // feature.layers.2.conv_block.2.weight (layer 2, conv 2) + let tensors = vec![ + create_test_tensor_snapshot("feature.layers.0.conv_block.0.weight"), + create_test_tensor_snapshot("feature.layers.0.conv_block.2.weight"), + create_test_tensor_snapshot("feature.layers.2.conv_block.0.weight"), + create_test_tensor_snapshot("feature.layers.2.conv_block.2.weight"), + ]; + + let (mapped, transformations) = map_indices_contiguous(tensors); + + // Expected mapping: + // feature.layers: 0, 2 -> 0, 1 + // feature.layers.0.conv_block: 0, 2 -> 0, 1 + // feature.layers.2.conv_block: 0, 2 -> 0, 1 + // + // Result: + // feature.layers.0.conv_block.0.weight -> feature.layers.0.conv_block.0.weight + // feature.layers.0.conv_block.2.weight -> feature.layers.0.conv_block.1.weight + // feature.layers.2.conv_block.0.weight -> feature.layers.1.conv_block.0.weight + // feature.layers.2.conv_block.2.weight -> feature.layers.1.conv_block.1.weight + + assert!( + mapped + .iter() + .any(|v| v.full_path() == "feature.layers.0.conv_block.0.weight"), + "0.0 should stay as 0.0" + ); + assert!( + mapped + .iter() + .any(|v| v.full_path() == "feature.layers.0.conv_block.1.weight"), + "0.2 should become 0.1" + ); + assert!( + mapped + .iter() + .any(|v| v.full_path() == "feature.layers.1.conv_block.0.weight"), + "2.0 should become 1.0" + ); + assert!( + mapped + .iter() + .any(|v| v.full_path() == "feature.layers.1.conv_block.1.weight"), + "2.2 should become 1.1" + ); + + // Verify specific transformations + let t1 = transformations + .iter() + .find(|(_, old)| old == "feature.layers.2.conv_block.2.weight"); + assert_eq!( + t1.map(|(new, _)| new.as_str()), + Some("feature.layers.1.conv_block.1.weight"), + "2.2 should map to 1.1" + ); + } + + #[test] + fn test_map_indices_contiguous_deeply_nested() { + // Test with three levels of nesting + let tensors = vec![ + create_test_tensor_snapshot("a.0.b.0.c.0.weight"), + create_test_tensor_snapshot("a.0.b.0.c.2.weight"), + create_test_tensor_snapshot("a.0.b.2.c.0.weight"), + create_test_tensor_snapshot("a.2.b.0.c.0.weight"), + ]; + + let (mapped, _) = map_indices_contiguous(tensors); + + // a: 0, 2 -> 0, 1 + // a.0.b: 0, 2 -> 0, 1 + // a.2.b: 0 -> 0 + // a.0.b.0.c: 0, 2 -> 0, 1 + // a.0.b.2.c: 0 -> 0 + // a.2.b.0.c: 0 -> 0 + + assert!(mapped.iter().any(|v| v.full_path() == "a.0.b.0.c.0.weight")); + assert!( + mapped.iter().any(|v| v.full_path() == "a.0.b.0.c.1.weight"), + "a.0.b.0.c.2 should become a.0.b.0.c.1" + ); + assert!( + mapped.iter().any(|v| v.full_path() == "a.0.b.1.c.0.weight"), + "a.0.b.2.c.0 should become a.0.b.1.c.0" + ); + assert!( + mapped.iter().any(|v| v.full_path() == "a.1.b.0.c.0.weight"), + "a.2.b.0.c.0 should become a.1.b.0.c.0" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/lib.rs new file mode 100644 index 0000000..dde16a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/lib.rs @@ -0,0 +1,118 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +//! # Burn Store +//! +//! Advanced model storage and serialization infrastructure for the Burn deep learning framework. +//! +//! This crate provides comprehensive functionality for storing and loading Burn modules +//! and their tensor data, with support for cross-framework interoperability, flexible filtering, +//! and efficient memory management through lazy materialization. +//! +//! ## Key Features +//! +//! - **Burnpack Format**: Native Burn format with CBOR metadata, ParamId persistence for stateful training, and no-std support +//! - **SafeTensors Format**: Industry-standard format for secure and efficient tensor serialization +//! - **PyTorch Compatibility**: Load PyTorch models directly into Burn with automatic weight transformation +//! - **Zero-Copy Loading**: Memory-mapped files and lazy tensor materialization for optimal performance +//! - **Flexible Filtering**: Load/save specific model subsets using regex, exact paths, or custom predicates +//! - **Tensor Remapping**: Rename tensors during load/save operations for framework compatibility +//! - **No-std Support**: Core functionality available in embedded and WASM environments +//! +//! ## Quick Start +//! +//! ### Basic Save and Load +//! +//! ```rust,ignore +//! use burn_store::{ModuleSnapshot, SafetensorsStore}; +//! +//! // Save a model +//! let mut store = SafetensorsStore::from_file("model.safetensors"); +//! model.save_into(&mut store)?; +//! +//! // Load a model +//! let mut store = SafetensorsStore::from_file("model.safetensors"); +//! model.load_from(&mut store)?; +//! ``` +//! +//! ### Loading PyTorch Models +//! +//! ```rust,ignore +//! use burn_store::PytorchStore; +//! +//! // Load PyTorch model (automatic weight transformation via PyTorchToBurnAdapter) +//! let mut store = PytorchStore::from_file("pytorch_model.pth") +//! .with_top_level_key("state_dict") // Access nested state dict if needed +//! .allow_partial(true); // Skip unknown tensors +//! +//! model.load_from(&mut store)?; +//! ``` +//! +//! ### Filtering and Remapping +//! +//! ```rust,no_run +//! # use burn_store::SafetensorsStore; +//! // Save only specific layers with renaming +//! let mut store = SafetensorsStore::from_file("encoder.safetensors") +//! .with_regex(r"^encoder\..*") // Filter: only encoder layers +//! .with_key_remapping(r"^encoder\.", "transformer.") // Rename: encoder.X -> transformer.X +//! .metadata("subset", "encoder_only"); +//! +//! // Use store with model.save_into(&mut store)?; +//! ``` +//! +//! ## Core Components +//! +//! - [`ModuleSnapshot`]: Extension trait for Burn modules providing `collect()` and `apply()` methods +//! - [`BurnpackStore`]: Native Burn format with ParamId persistence for stateful training workflows +//! - [`SafetensorsStore`]: Primary storage implementation supporting the SafeTensors format +//! - [`PytorchStore`]: PyTorch model loader supporting .pth and .pt files +//! - [`PathFilter`]: Flexible filtering system for selective tensor loading/saving +//! - [`KeyRemapper`]: Advanced tensor name remapping with regex patterns +//! - [`ModuleAdapter`]: Framework adapters for cross-framework compatibility +//! +//! ## Feature Flags +//! +//! - `std`: Enables file I/O and other std-only features (default) +//! - `safetensors`: Enables SafeTensors format support (default) + +extern crate alloc; + +mod adapter; +mod applier; +mod apply_result; +mod collector; +mod filter; +mod tensor_snapshot; +mod traits; + +pub use adapter::{ + BurnToPyTorchAdapter, ChainAdapter, IdentityAdapter, ModuleAdapter, PyTorchToBurnAdapter, +}; +pub use applier::Applier; +pub use apply_result::{ApplyError, ApplyResult}; +pub use collector::Collector; +pub use filter::PathFilter; +pub use tensor_snapshot::{TensorSnapshot, TensorSnapshotError}; +pub use traits::{ModuleSnapshot, ModuleStore}; + +#[cfg(feature = "std")] +mod keyremapper; +#[cfg(feature = "std")] +pub use keyremapper::{KeyRemapper, map_indices_contiguous}; + +#[cfg(feature = "pytorch")] +pub mod pytorch; +#[cfg(feature = "pytorch")] +pub use pytorch::{PytorchStore, PytorchStoreError}; + +#[cfg(feature = "safetensors")] +mod safetensors; +#[cfg(feature = "safetensors")] +pub use safetensors::{SafetensorsStore, SafetensorsStoreError}; + +#[cfg(feature = "burnpack")] +mod burnpack; +#[cfg(feature = "burnpack")] +pub use burnpack::writer::BurnpackWriter; +#[cfg(feature = "burnpack")] +pub use burnpack::{base::BurnpackError, store::BurnpackStore}; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/lazy_data.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/lazy_data.rs new file mode 100644 index 0000000..a6c73a9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/lazy_data.rs @@ -0,0 +1,567 @@ +//! Lazy data loading support for PyTorch files. +//! +//! This module provides abstractions for lazy loading of tensor data from PyTorch files, +//! avoiding the need to load all data into memory upfront. + +use alloc::string::String; +use alloc::vec::Vec; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufReader, Read, Seek}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, RwLock}; +use zip::ZipArchive; + +/// A data source that can lazily load tensor data. +#[derive(Clone)] +pub enum LazyDataSource { + /// ZIP archive with lazy loading + Zip(Arc>), + /// TAR archive format (older torchvision models) + Tar(Arc>), + /// Legacy format with multiple storages in single blob + LegacyMultiStorage(Arc>), +} + +/// ZIP archive source for lazy loading +pub struct ZipSource { + path: PathBuf, + // Cache the file list to avoid reopening archive repeatedly + file_list: Vec<(String, u64, u64)>, // (name, offset, compressed_size) +} + +/// TAR archive source for lazy loading (older torchvision models like AlexNet, SqueezeNet) +/// +/// Older PyTorch/torchvision models (pre-1.6) use TAR format instead of ZIP. +/// The TAR archive contains: +/// - `sys_info`: System info pickle (endianness, type sizes) +/// - `pickle`: OrderedDict mapping tensor names to storage keys +/// - `tensors`: Tensor metadata pickles (unused, metadata is embedded in pickle) +/// - `storages`: Storage count + sequential (metadata pickle, element count, raw data) +pub struct TarSource { + /// Cached storage map: storage_key -> (offset_in_storages, size_bytes) + storage_map: HashMap, + /// The raw storages data (kept in memory for TAR format) + storages_data: Vec, +} + +/// Legacy multi-storage source for old PyTorch format (0.1.10 - 1.5) +/// +/// Legacy format stores tensor data as concatenated raw binary without explicit +/// storage boundaries. This source tracks storage usage during tensor parsing +/// to build a storage map for lazy loading. +/// +/// ## Storage Layout +/// - Pickle metadata with tensor definitions +/// - List of storage keys (determines concatenation order) +/// - Raw binary blob with all storages concatenated +pub struct LegacyMultiStorageSource { + path: PathBuf, + data_offset: u64, + #[allow(dead_code)] + data_size: u64, + // Map of storage_key -> (offset_in_blob, size) + storage_map: RwLock>>, + // Storage keys in order (for boundary calculation) + storage_keys: RwLock>>, + // Track storage usage as tensors are accessed + storage_usage: RwLock>, // key -> max_bytes_needed +} + +impl ZipSource { + /// Create a new ZIP source + pub fn new(path: PathBuf) -> std::io::Result { + let file = File::open(&path)?; + let reader = BufReader::new(file); + let mut archive = ZipArchive::new(reader)?; + + // Cache file metadata + let mut file_list = Vec::new(); + for i in 0..archive.len() { + let file = archive.by_index(i)?; + let name = file.name().to_string(); + let offset = file.data_start(); + let compressed_size = file.compressed_size(); + file_list.push(( + name, + offset.expect("should have an offset"), + compressed_size, + )); + } + + Ok(Self { path, file_list }) + } + + /// Check if a file exists in the archive + pub fn contains(&self, name: &str) -> bool { + self.file_list.iter().any(|(n, _, _)| n == name) + } + + /// Get list of data files (excluding pickle files) + pub fn data_files(&self) -> Vec { + self.file_list + .iter() + .filter(|(name, _, _)| name.starts_with("data/") || name.contains("/data/")) + .filter(|(name, _, _)| !name.ends_with(".pkl") && !name.ends_with("/")) + .map(|(name, _, _)| name.clone()) + .collect() + } + + /// Read a specific file from the archive + pub fn read_file(&self, name: &str) -> std::io::Result> { + let file = File::open(&self.path)?; + let reader = BufReader::new(file); + let mut archive = ZipArchive::new(reader)?; + + let mut file = archive.by_name(name)?; + let mut contents = Vec::with_capacity(file.size() as usize); + file.read_to_end(&mut contents)?; + Ok(contents) + } + + /// Read a portion of a file + pub fn read_file_range( + &self, + name: &str, + offset: usize, + length: usize, + ) -> std::io::Result> { + let file = File::open(&self.path)?; + let reader = BufReader::new(file); + let mut archive = ZipArchive::new(reader)?; + + let mut file = archive.by_name(name)?; + let mut buffer = vec![0u8; length]; + + // Skip to offset + let mut skip_buffer = vec![0u8; offset.min(8192)]; + let mut skipped = 0; + while skipped < offset { + let to_skip = (offset - skipped).min(skip_buffer.len()); + file.read_exact(&mut skip_buffer[..to_skip])?; + skipped += to_skip; + } + + // Read the requested data + file.read_exact(&mut buffer)?; + Ok(buffer) + } +} + +impl LegacyMultiStorageSource { + /// Create a new legacy multi-storage source + pub fn new(path: PathBuf, data_offset: u64, data_size: u64) -> Self { + Self { + path, + data_offset, + data_size, + storage_map: RwLock::new(None), + storage_keys: RwLock::new(None), + storage_usage: RwLock::new(HashMap::new()), + } + } + + /// Set the ordered storage keys from the pickle + pub fn set_storage_keys(&self, keys: Vec) { + let mut storage_keys = self + .storage_keys + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + *storage_keys = Some(keys); + } + + /// Track storage usage from tensor access + /// This is called from within tensor loading closures + pub fn track_storage_usage(&self, storage_key: &str, offset: usize, size: usize) { + let mut usage = self + .storage_usage + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let max_extent = offset + size; + usage + .entry(storage_key.to_string()) + .and_modify(|current| *current = (*current).max(max_extent)) + .or_insert(max_extent); + + // Try to build storage map if we have enough information + drop(usage); + self.try_build_storage_map(); + } + + /// Try to build the storage map from tracked usage + fn try_build_storage_map(&self) { + // Only build if we don't already have a map + if self + .storage_map + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .is_some() + { + return; + } + + // Check if we have storage keys + let keys_guard = self + .storage_keys + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(ref keys) = *keys_guard { + let usage = self + .storage_usage + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + // Only build if we have usage info for all storages + if keys.iter().all(|k| usage.contains_key(k)) { + let mut map = HashMap::new(); + let mut current_offset = 0u64; + + for key in keys { + if let Some(&size) = usage.get(key) { + map.insert(key.clone(), (current_offset, size as u64)); + current_offset += size as u64; + } + } + + // Set the storage map + drop(keys_guard); + drop(usage); + let mut storage_map = self + .storage_map + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + *storage_map = Some(map); + } + } + } + + /// Read data for a specific storage key + /// Only loads the specific storage portion, never the entire blob + pub fn read(&self, key: &str) -> std::io::Result> { + // Extract numeric key from paths like "data/0" or just "0" + let storage_key = key.split('/').next_back().unwrap_or(key); + + // Get storage map - must be available for lazy loading to work + let storage_map = self + .storage_map + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + if let Some(ref map) = *storage_map + && let Some(&(offset, size)) = map.get(storage_key) + { + // Load only this specific storage + let mut file = File::open(&self.path)?; + file.seek(std::io::SeekFrom::Start(self.data_offset + offset))?; + + let mut buffer = vec![0u8; size as usize]; + file.read_exact(&mut buffer)?; + return Ok(buffer); + } + + // NO FALLBACK! If we don't have storage boundaries, we cannot load data lazily + // The storage map MUST be built from tensor metadata for lazy loading to work + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "Storage boundaries not available for key '{}'. Cannot perform lazy loading.", + storage_key + ), + )) + } +} + +impl TarSource { + /// Create a new TAR source by parsing storages data. + /// + /// # Arguments + /// * `storages_data` - Raw storages blob with structure: + /// - Count pickle (number of storages) + /// - For each storage: metadata pickle + u64 num_elements + raw binary data + pub fn new(storages_data: Vec) -> std::io::Result { + use super::pickle_reader::{read_pickle, storage_type_to_element_size}; + use std::io::Cursor; + + let mut storage_map = HashMap::new(); + let mut pos = 0usize; + + // First, read the count of storages + let mut cursor = Cursor::new(&storages_data[pos..]); + let storage_count = + if let Ok(super::pickle_reader::Object::Int(count)) = read_pickle(&mut cursor) { + pos += cursor.position() as usize; + count as usize + } else { + 0 + }; + + // Parse each storage entry + for _i in 0..storage_count { + if pos >= storages_data.len() { + break; + } + + // Read the storage metadata pickle: (storage_key, device, storage_type) + let mut cursor = Cursor::new(&storages_data[pos..]); + if let Ok(obj) = read_pickle(&mut cursor) { + let pickle_size = cursor.position() as usize; + pos += pickle_size; + + // Extract storage info from pickle tuple + let (storage_key, storage_type) = match obj { + super::pickle_reader::Object::Tuple(tuple) if tuple.len() >= 3 => { + let key = match &tuple[0] { + super::pickle_reader::Object::Int(i) => i.to_string(), + super::pickle_reader::Object::String(s) => s.clone(), + _ => continue, + }; + // tuple[1] is device (e.g., "cpu") + // tuple[2] is storage type class + let stype = match &tuple[2] { + super::pickle_reader::Object::Class { name, .. } => name.clone(), + other => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Expected Class for storage type, got {:?}", other), + )); + } + }; + (key, stype) + } + _ => continue, + }; + + // Read the number of elements (u64 little-endian) + if pos + 8 > storages_data.len() { + break; + } + let num_elements = u64::from_le_bytes([ + storages_data[pos], + storages_data[pos + 1], + storages_data[pos + 2], + storages_data[pos + 3], + storages_data[pos + 4], + storages_data[pos + 5], + storages_data[pos + 6], + storages_data[pos + 7], + ]) as usize; + pos += 8; + + // Determine element size from storage type + let element_size = storage_type_to_element_size(&storage_type) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let data_size = num_elements * element_size; + + // Store the offset to raw data and its size + storage_map.insert(storage_key, (pos, data_size)); + + // Skip the raw binary data + pos += data_size; + } else { + break; + } + } + + Ok(Self { + storage_map, + storages_data, + }) + } + + /// Read data for a specific storage key + pub fn read_file(&self, key: &str) -> std::io::Result> { + // Extract the storage key from paths like "data/0" + let storage_key = key.split('/').next_back().unwrap_or(key); + + if let Some(&(offset, size)) = self.storage_map.get(storage_key) + && offset + size <= self.storages_data.len() + { + return Ok(self.storages_data[offset..offset + size].to_vec()); + } + + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("Storage key '{}' not found in TAR archive", storage_key), + )) + } + + /// Read a range of data for a specific storage key (avoids double allocation) + pub fn read_file_range( + &self, + key: &str, + offset: usize, + length: usize, + ) -> std::io::Result> { + let storage_key = key.split('/').next_back().unwrap_or(key); + + if let Some(&(storage_offset, storage_size)) = self.storage_map.get(storage_key) + && storage_offset + storage_size <= self.storages_data.len() + { + let start = storage_offset + offset; + let end = (storage_offset + offset + length).min(storage_offset + storage_size); + return Ok(self.storages_data[start..end].to_vec()); + } + + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("Storage key '{}' not found in TAR archive", storage_key), + )) + } + + /// Check if a storage key exists + pub fn contains(&self, key: &str) -> bool { + let storage_key = key.split('/').next_back().unwrap_or(key); + self.storage_map.contains_key(storage_key) + } + + /// Get list of storage keys + pub fn keys(&self) -> Vec { + self.storage_map.keys().cloned().collect() + } +} + +impl LazyDataSource { + /// Create from a ZIP file + pub fn from_zip(path: impl AsRef) -> std::io::Result { + Ok(Self::Zip(Arc::new(Mutex::new(ZipSource::new( + path.as_ref().to_path_buf(), + )?)))) + } + + /// Create from a TAR archive's storages data + pub fn from_tar(storages_data: &[u8]) -> std::io::Result { + Ok(Self::Tar(Arc::new(Mutex::new(TarSource::new( + storages_data.to_vec(), + )?)))) + } + + /// Create from a legacy multi-storage file + pub fn from_legacy_multi_storage( + path: impl AsRef, + data_offset: u64, + data_size: u64, + ) -> Self { + Self::LegacyMultiStorage(Arc::new(Mutex::new(LegacyMultiStorageSource::new( + path.as_ref().to_path_buf(), + data_offset, + data_size, + )))) + } + + /// Read data for a specific key + pub fn read(&self, key: &str) -> std::io::Result> { + match self { + Self::Zip(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.read_file(key) + } + Self::Tar(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.read_file(key) + } + Self::LegacyMultiStorage(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.read(key) + } + } + } + + /// Read a portion of data for a specific key + pub fn read_range(&self, key: &str, offset: usize, length: usize) -> std::io::Result> { + match self { + Self::Zip(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.read_file_range(key, offset, length) + } + Self::Tar(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.read_file_range(key, offset, length) + } + Self::LegacyMultiStorage(source) => { + // For legacy format, read only the requested range + let storage_key = key.split('/').next_back().unwrap_or(key); + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + // Get storage boundaries + let storage_map = source + .storage_map + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(ref map) = *storage_map + && let Some(&(storage_offset, storage_size)) = map.get(storage_key) + { + // Calculate actual file position + let file_offset = source.data_offset + storage_offset + offset as u64; + let read_length = length.min((storage_size as usize).saturating_sub(offset)); + + // Read only the requested range + let mut file = File::open(&source.path)?; + file.seek(std::io::SeekFrom::Start(file_offset))?; + + let mut buffer = vec![0u8; read_length]; + file.read_exact(&mut buffer)?; + Ok(buffer) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "Storage boundaries not available for key '{}'. Cannot perform lazy loading.", + storage_key + ), + )) + } + } + } + } + + /// Check if a key exists + pub fn contains(&self, key: &str) -> bool { + match self { + Self::Zip(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.contains(key) + } + Self::Tar(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.contains(key) + } + Self::LegacyMultiStorage(_) => true, // Legacy format has all data + } + } + + /// Get list of available keys (for ZIP sources) + pub fn keys(&self) -> Vec { + match self { + Self::Zip(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.data_files() + } + Self::Tar(source) => { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.keys() + } + Self::LegacyMultiStorage(_) => vec![], // Legacy format doesn't have distinct keys + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/mod.rs new file mode 100644 index 0000000..5eecf6b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/mod.rs @@ -0,0 +1,48 @@ +//! PyTorch format support for burn-store. +//! +//! This module provides comprehensive support for loading PyTorch model files (.pth, .pt) +//! into Burn, with automatic weight transformation and flexible configuration options. +//! +//! ## Features +//! +//! - **Direct .pth/.pt file loading**: Load PyTorch checkpoint and state dict files +//! - **Automatic weight transformation**: `PyTorchToBurnAdapter` is applied by default: +//! - Linear layer weights are automatically transposed +//! - Normalization parameters are renamed (gamma → weight, beta → bias) +//! - Conv2d weights maintain their format +//! - **Flexible filtering**: Load only specific layers or parameters +//! - **Key remapping**: Rename tensors during loading to match your model structure +//! - **Partial loading**: Continue even when some tensors are missing +//! +//! ## Example +//! +//! ```rust,ignore +//! use burn_store::PytorchStore; +//! +//! // Load a PyTorch model (PyTorchToBurnAdapter is applied automatically) +//! let mut store = PytorchStore::from_file("model.pth") +//! .with_top_level_key("state_dict") // Access nested state dict +//! .with_regex(r"^encoder\..*") // Only load encoder layers +//! .with_key_remapping(r"^fc\.", "linear.") // Rename fc -> linear +//! .allow_partial(true); // Skip missing tensors +//! +//! let mut model = MyModel::new(&device); +//! let result = model.load_from(&mut store)?; +//! +//! println!("Loaded {} tensors", result.applied.len()); +//! if !result.missing.is_empty() { +//! println!("Missing tensors: {:?}", result.missing); +//! } +//! ``` + +pub mod lazy_data; +pub mod pickle_reader; +pub mod reader; +pub mod store; + +#[cfg(test)] +pub mod tests; + +// Main public interface +pub use reader::{PytorchError, PytorchReader}; +pub use store::{PytorchStore, PytorchStoreError}; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/pickle_reader.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/pickle_reader.rs new file mode 100644 index 0000000..b33bf26 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/pickle_reader.rs @@ -0,0 +1,1528 @@ +//! Just enough pickle support to be able to read PyTorch checkpoints. +//! +//! This implementation is based on the candle project's pickle loader with significant +//! modifications for improved separation of concerns and extended PyTorch compatibility. +//! +//! Original source: +//! +//! Modifications include: +//! - Lazy tensor data loading for memory efficiency +//! - Extended PyTorch version compatibility (0.1.10 - 2.x) +//! - Better separation of pickle parsing and tensor extraction +//! - Support for both legacy and modern PyTorch formats +use crate::TensorSnapshot; +use crate::pytorch::lazy_data::LazyDataSource; +use alloc::rc::Rc; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use burn_core::module::ParamId; +use burn_tensor::{DType, TensorData}; +use byteorder::{LittleEndian, ReadBytesExt}; +use half::{bf16, f16}; +use std::collections::HashMap; +use std::io::{self, BufRead}; +use std::sync::Arc; + +/// Error type for pickle operations +#[derive(Debug)] +pub enum PickleError { + Io(io::Error), + InvalidOpCode(u8), + InvalidProtocol(u8), + UnexpectedOpCode(OpCode), + UnsupportedType(String), + InvalidData(String), + StackUnderflow, + MemoNotFound(u32), + InvalidShapeOrType, +} + +impl From for PickleError { + fn from(e: io::Error) -> Self { + PickleError::Io(e) + } +} + +impl std::fmt::Display for PickleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PickleError::Io(e) => write!(f, "IO error: {}", e), + PickleError::InvalidOpCode(code) => write!( + f, + "Invalid pickle opcode: 0x{:02x}. The file may be corrupted or use an unsupported pickle protocol.", + code + ), + PickleError::InvalidProtocol(proto) => write!( + f, + "Invalid or unsupported pickle protocol version: {}. Supported versions are 2-5.", + proto + ), + PickleError::UnexpectedOpCode(op) => { + write!(f, "Unexpected pickle opcode {:?} in current context", op) + } + PickleError::UnsupportedType(ty) => write!( + f, + "Unsupported Python type '{}'. This may indicate a full model save rather than a state_dict.", + ty + ), + PickleError::InvalidData(msg) => write!(f, "Invalid data in pickle file: {}", msg), + PickleError::StackUnderflow => { + write!(f, "Pickle stack underflow - the file may be corrupted") + } + PickleError::MemoNotFound(idx) => write!( + f, + "Pickle memo reference {} not found - the file may be corrupted", + idx + ), + PickleError::InvalidShapeOrType => { + write!(f, "Invalid tensor shape or data type in PyTorch file") + } + } + } +} + +impl std::error::Error for PickleError {} + +type Result = std::result::Result; + +/// Convert PyTorch storage type name to element size in bytes. +/// +/// This is used to calculate storage sizes for lazy loading. +/// The storage type names follow PyTorch's naming convention (e.g., "FloatStorage", "BFloat16Storage"). +/// +/// Returns an error for unknown storage types to avoid silently loading garbage data. +pub fn storage_type_to_element_size(storage_type: &str) -> std::result::Result { + match storage_type { + "DoubleStorage" | "LongStorage" | "ComplexFloatStorage" => Ok(8), + "FloatStorage" | "IntStorage" | "ComplexHalfStorage" => Ok(4), + "HalfStorage" | "BFloat16Storage" | "ShortStorage" => Ok(2), + "ByteStorage" | "CharStorage" | "BoolStorage" => Ok(1), + _ => Err(format!("Unknown storage type: {}", storage_type)), + } +} + +// https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/ +#[repr(u8)] +#[derive(Debug, Eq, PartialEq, Clone)] +pub enum OpCode { + // https://github.com/python/cpython/blob/ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a/Lib/pickletools.py#L2123 + Proto = 0x80, + Global = b'c', + BinPut = b'q', + LongBinPut = b'r', + EmptyTuple = b')', + Reduce = b'R', + Mark = b'(', + BinUnicode = b'X', + ShortBinString = b'U', + BinInt = b'J', + Int = b'I', + Tuple = b't', + BinPersId = b'Q', + BinInt1 = b'K', + BinInt2 = b'M', + Tuple1 = 0x85, + Tuple2 = 0x86, + Tuple3 = 0x87, + NewTrue = 0x88, + NewFalse = 0x89, + None = b'N', + BinGet = b'h', + LongBinGet = b'j', + SetItem = b's', + SetItems = b'u', + EmptyDict = b'}', + Dict = b'd', + Build = b'b', + Stop = b'.', + NewObj = 0x81, + EmptyList = b']', + List = b'l', + BinFloat = b'G', + Append = b'a', + Appends = b'e', + Long1 = 0x8a, + Memoize = 0x94, +} + +// Avoid using FromPrimitive so as not to drag another dependency. +impl TryFrom for OpCode { + type Error = u8; + fn try_from(value: u8) -> std::result::Result { + match value { + 0x80 => Ok(Self::Proto), + b'c' => Ok(Self::Global), + b'q' => Ok(Self::BinPut), + b'r' => Ok(Self::LongBinPut), + b')' => Ok(Self::EmptyTuple), + b'R' => Ok(Self::Reduce), + b'(' => Ok(Self::Mark), + b'X' => Ok(Self::BinUnicode), + b'U' => Ok(Self::ShortBinString), + b'J' => Ok(Self::BinInt), + b'I' => Ok(Self::Int), + b't' => Ok(Self::Tuple), + b'Q' => Ok(Self::BinPersId), + b'K' => Ok(Self::BinInt1), + b'M' => Ok(Self::BinInt2), + b'N' => Ok(Self::None), + 0x85 => Ok(Self::Tuple1), + 0x86 => Ok(Self::Tuple2), + 0x87 => Ok(Self::Tuple3), + 0x88 => Ok(Self::NewTrue), + 0x89 => Ok(Self::NewFalse), + b'h' => Ok(Self::BinGet), + b'j' => Ok(Self::LongBinGet), + b's' => Ok(Self::SetItem), + b'u' => Ok(Self::SetItems), + b'}' => Ok(Self::EmptyDict), + b'd' => Ok(Self::Dict), + b'b' => Ok(Self::Build), + b'.' => Ok(Self::Stop), + 0x81 => Ok(Self::NewObj), + b']' => Ok(Self::EmptyList), + b'l' => Ok(Self::List), + b'G' => Ok(Self::BinFloat), + b'a' => Ok(Self::Append), + b'e' => Ok(Self::Appends), + 0x8a => Ok(Self::Long1), + 0x94 => Ok(Self::Memoize), + value => Err(value), + } + } +} + +fn read_to_newline(r: &mut R) -> Result> { + let mut data: Vec = Vec::with_capacity(32); + r.read_until(b'\n', &mut data)?; + data.pop(); + if data.last() == Some(&b'\r') { + data.pop(); + } + Ok(data) +} + +fn buf_to_str(buf: &[u8]) -> Result { + String::from_utf8(buf.to_vec()) + .map_err(|e| PickleError::InvalidData(format!("Invalid UTF-8: {}", e))) +} + +#[derive(Debug, Clone)] +pub enum Object { + Class { + module_name: String, + name: String, + }, + String(String), + Int(i64), + Float(f64), + Bool(bool), + None, + Tuple(Vec), + List(Vec), + Dict(HashMap), + Persistent(Vec), + PersistentTuple(Vec), + Reduce { + callable: Box, + args: Box, + }, + Build { + callable: Box, + args: Box, + }, + TorchParam(TensorSnapshot), +} + +fn rebuild_from_type_v2( + o: Object, + memo: &mut HashMap, + data_source: &Option>, +) -> Result { + let args = if let Object::Tuple(args) = o { + if args.is_empty() { + return Err(PickleError::InvalidData( + "rebuild_from_type_v2: empty args".to_string(), + )); + } + args + } else { + return Err(PickleError::InvalidData(format!( + "rebuild_from_type_v2: expected tuple got {:?}", + o + ))); + }; + let func = &args[0]; + match func { + Object::Class { module_name, name } => { + let module_name = module_name.as_str(); + let name = name.as_str(); + // For rebuild_tensor_v2, the args might already be in a tuple + let actual_args = if args.len() == 2 && matches!(&args[1], Object::Tuple(_)) { + // If there's only one arg and it's a tuple, use it directly + args[1].clone() + } else { + // Otherwise, wrap the remaining args in a tuple + Object::Tuple(args[1..].to_vec()) + }; + if module_name == "torch._utils" && name == "_rebuild_tensor_v2" { + rebuild_tensor_v2(actual_args, memo, data_source) + } else if module_name == "torch._utils" && name == "_rebuild_tensor" { + // Legacy _rebuild_tensor (PyTorch < 1.6) + // Same as v2 but with fewer arguments: (storage, storage_offset, size, stride) + rebuild_tensor(actual_args, memo, data_source) + } else if module_name == "torch._tensor" && name == "_rebuild_from_type_v2" { + rebuild_from_type_v2(actual_args, memo, data_source) + } else if module_name == "torch._utils" && name == "_rebuild_parameter" { + rebuild_parameter(actual_args, memo, data_source) + } else if module_name == "collections" && name == "OrderedDict" { + // OrderedDict is treated as a regular Dict in our implementation + Ok(Object::Dict(HashMap::new())) + } else { + Err(PickleError::UnsupportedType(format!( + "{}.{}", + module_name, name + ))) + } + } + _ => Err(PickleError::InvalidData(format!( + "rebuild_from_type_v2: expected class got {:?}", + func + ))), + } +} + +fn rebuild_parameter( + args: Object, + memo: &mut HashMap, + data_source: &Option>, +) -> Result { + let args = if let Object::Tuple(args) = args { + if args.is_empty() { + return Err(PickleError::InvalidData( + "rebuild_parameter: empty args".to_string(), + )); + } + args + } else { + return Err(PickleError::InvalidData(format!( + "rebuild_parameter: expected tuple got {:?}", + args + ))); + }; + let data = &args[0]; + let tensor = match data { + Object::Reduce { + callable: _, + args: _, + } => rebuild_from_type_v2(data.clone(), memo, data_source)?, + _ => data.clone(), + }; + Ok(tensor) +} + +/// Parse storage argument and extract storage info and tuple. +fn parse_storage_arg(arg: &Object, fn_name: &str) -> Result<(Vec, Option>)> { + match arg { + Object::Persistent(data) => Ok((data.clone(), None)), + Object::PersistentTuple(tuple) => Ok((vec![], Some(tuple.clone()))), + // Also accept regular Tuple for TAR format compatibility + Object::Tuple(tuple) => Ok((vec![], Some(tuple.clone()))), + _ => Err(PickleError::InvalidData(format!( + "{}: expected persistent id got {:?}", + fn_name, arg + ))), + } +} + +/// Parse shape argument. +fn parse_shape_arg(arg: &Object, fn_name: &str) -> Result> { + match arg { + Object::Tuple(shape) => shape + .iter() + .map(|x| match x { + Object::Int(i) => Ok(*i as usize), + _ => Err(PickleError::InvalidData( + "shape must contain ints".to_string(), + )), + }) + .collect::>>(), + _ => Err(PickleError::InvalidData(format!( + "{}: expected shape tuple got {:?}", + fn_name, arg + ))), + } +} + +/// Legacy _rebuild_tensor function for PyTorch < 1.6. +/// Thin wrapper that parses 4 arguments and calls rebuild_tensor_impl. +fn rebuild_tensor( + args: Object, + _memo: &mut HashMap, + data_source: &Option>, +) -> Result { + let args = if let Object::Tuple(args) = args { + args + } else { + return Err(PickleError::InvalidData(format!( + "rebuild_tensor: expected tuple got {:?}", + args + ))); + }; + + if args.len() < 4 { + return Err(PickleError::InvalidData(format!( + "rebuild_tensor: expected at least 4 args, got {}", + args.len() + ))); + } + + let (storage_info, storage_tuple) = parse_storage_arg(&args[0], "rebuild_tensor")?; + let storage_offset = match &args[1] { + Object::Int(offset) => *offset as usize, + _ => 0, + }; + let shape = parse_shape_arg(&args[2], "rebuild_tensor")?; + + rebuild_tensor_impl( + storage_info, + storage_tuple, + storage_offset, + shape, + data_source, + ) +} + +/// Modern _rebuild_tensor_v2 function for PyTorch >= 1.6. +/// Thin wrapper that parses 5+ arguments and calls rebuild_tensor_impl. +fn rebuild_tensor_v2( + args: Object, + _memo: &mut HashMap, + data_source: &Option>, +) -> Result { + let args = if let Object::Tuple(args) = args { + args + } else { + return Err(PickleError::InvalidData(format!( + "rebuild_tensor_v2: expected tuple got {:?}", + args + ))); + }; + + if args.len() < 5 { + return Err(PickleError::InvalidData(format!( + "rebuild_tensor_v2: expected at least 5 args, got {}", + args.len() + ))); + } + + let (storage_info, storage_tuple) = parse_storage_arg(&args[0], "rebuild_tensor_v2")?; + let storage_offset = match &args[1] { + Object::Int(offset) => *offset as usize, + _ => 0, + }; + let shape = parse_shape_arg(&args[2], "rebuild_tensor_v2")?; + // args[3] is stride (unused) + // args[4] is requires_grad (unused) + // args[5] is backward_hooks (unused) + + rebuild_tensor_impl( + storage_info, + storage_tuple, + storage_offset, + shape, + data_source, + ) +} + +/// Helper to convert storage type name to DType. +fn storage_type_to_dtype(storage_type: &str) -> Result { + match storage_type { + "FloatStorage" => Ok(DType::F32), + "DoubleStorage" => Ok(DType::F64), + "HalfStorage" => Ok(DType::F16), + "BFloat16Storage" => Ok(DType::BF16), + "LongStorage" => Ok(DType::I64), + "IntStorage" => Ok(DType::I32), + "ShortStorage" => Ok(DType::I16), + "CharStorage" => Ok(DType::I8), + "ByteStorage" => Ok(DType::U8), + "BoolStorage" => Ok(DType::Bool), + _ => Err(PickleError::InvalidData(format!( + "Unknown storage type: {}", + storage_type + ))), + } +} + +/// Core implementation for rebuilding tensors. +/// Shared by both rebuild_tensor (legacy) and rebuild_tensor_v2 (modern). +fn rebuild_tensor_impl( + storage_info: Vec, + storage_tuple: Option>, + storage_offset: usize, + shape: Vec, + data_source: &Option>, +) -> Result { + // Parse the storage info to extract dtype and storage key + // The persistent ID is typically a tuple like: ('storage', 'FloatStorage', '0', 'cpu', 4) + let (dtype, storage_key) = if let Some(tuple) = storage_tuple { + // Direct tuple access + if tuple.len() >= 3 { + let storage_type = match &tuple[1] { + Object::String(s) => s.as_str(), + Object::Class { + module_name: _, + name, + } => name.as_str(), + other => { + return Err(PickleError::InvalidData(format!( + "Expected storage type as String or Class, got {:?}", + other + ))); + } + }; + let dtype = storage_type_to_dtype(storage_type)?; + let key = match &tuple[2] { + Object::String(s) => s.clone(), + other => { + return Err(PickleError::InvalidData(format!( + "Expected storage key as String, got {:?}", + other + ))); + } + }; + (dtype, key) + } else { + return Err(PickleError::InvalidData(format!( + "Storage tuple too short, expected at least 3 elements, got {}", + tuple.len() + ))); + } + } else if !storage_info.is_empty() { + // Legacy string-based parsing + let storage_str = String::from_utf8_lossy(&storage_info); + if storage_str.starts_with("Tuple(") { + // Parse from the debug representation we stored + let parts: Vec<&str> = storage_str + .trim_start_matches("Tuple(") + .trim_end_matches(")") + .split(", ") + .map(|s| { + let trimmed = s.trim_matches('"'); + if let Some(inner) = trimmed + .strip_prefix("Object::String(\"") + .and_then(|s| s.strip_suffix("\")")) + { + inner + } else { + trimmed + } + }) + .collect(); + + if parts.len() >= 3 { + let dtype = storage_type_to_dtype(parts[1])?; + (dtype, parts[2].to_string()) + } else { + return Err(PickleError::InvalidData(format!( + "Storage info tuple too short, expected at least 3 parts, got {}", + parts.len() + ))); + } + } else { + return Err(PickleError::InvalidData(format!( + "Invalid storage info format: {}", + storage_str + ))); + } + } else { + return Err(PickleError::InvalidData( + "No storage information available".to_string(), + )); + }; + + // If no data source, we can't load tensor data + let data_source = match data_source { + Some(ds) => ds.clone(), + None => { + return Err(PickleError::InvalidData( + "Cannot load tensor data without a data source".to_string(), + )); + } + }; + + // Create clones for the closure + let data_source_clone = data_source.clone(); + let shape_clone = shape.clone(); + + // Find the correct data file key + let data_file_key = { + let exact_key = format!("data/{}", storage_key); + if data_source.contains(&exact_key) { + exact_key + } else { + // Try other patterns + data_source + .keys() + .into_iter() + .find(|key| { + key.ends_with(&format!("/data/{}", storage_key)) + || (key.contains("/data/") && key.rsplit('/').next() == Some(&storage_key)) + }) + .unwrap_or_else(|| format!("data/{}", storage_key)) + } + }; + + // Track storage usage IMMEDIATELY for lazy boundary detection + // This must happen BEFORE creating the closure, not inside it! + if let LazyDataSource::LegacyMultiStorage(ref source) = *data_source { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let num_elements: usize = shape.iter().product(); + let bytes_needed = storage_offset * dtype.size() + num_elements * dtype.size(); + source.track_storage_usage(&storage_key, 0, bytes_needed); + } + + // Create a TensorSnapshot with a closure that loads the actual data on-demand + Ok(Object::TorchParam(TensorSnapshot::from_closure( + Rc::new(move || { + // Load data only when needed + if let Ok(data) = data_source_clone.read(&data_file_key) { + // Parse the binary data based on dtype + let num_elements = shape_clone.iter().product::().max(1); + + // Use dtype.size() to get element size in bytes + let element_size = dtype.size(); + + // Apply storage offset + let offset_bytes = storage_offset * element_size; + if offset_bytes >= data.len() { + return Ok(TensorData::new( + vec![0.0f32; num_elements], + shape_clone.clone(), + )); + } + + let data_slice = &data[offset_bytes..]; + let available_elements = data_slice.len() / element_size; + let elements_to_read = num_elements.min(available_elements); + + // Convert bytes to the appropriate type + match dtype { + DType::F32 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let bytes = [ + data_slice[i * element_size], + data_slice[i * element_size + 1], + data_slice[i * element_size + 2], + data_slice[i * element_size + 3], + ]; + values.push(f32::from_le_bytes(bytes)); + } + // Pad with zeros if needed + values.resize(num_elements, 0.0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::F64 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 8]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(f64::from_le_bytes(bytes)); + } + values.resize(num_elements, 0.0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::I64 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 8]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(i64::from_le_bytes(bytes)); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::I32 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 4]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(i32::from_le_bytes(bytes)); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::I16 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 2]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(i16::from_le_bytes(bytes)); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::I8 => { + let mut values = Vec::with_capacity(num_elements); + for &byte in data_slice.iter().take(elements_to_read) { + values.push(byte as i8); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::Bool => { + let mut values = Vec::with_capacity(num_elements); + for &byte in data_slice.iter().take(elements_to_read) { + values.push(byte != 0); + } + values.resize(num_elements, false); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::F16 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 2]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(f16::from_le_bytes(bytes)); + } + values.resize(num_elements, f16::ZERO); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::BF16 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 2]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(bf16::from_le_bytes(bytes)); + } + values.resize(num_elements, bf16::ZERO); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::U8 => { + let mut values = Vec::with_capacity(num_elements); + for &byte in data_slice.iter().take(elements_to_read) { + values.push(byte); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::U16 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 2]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(u16::from_le_bytes(bytes)); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::U32 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 4]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(u32::from_le_bytes(bytes)); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + DType::U64 => { + let mut values = Vec::with_capacity(num_elements); + for i in 0..elements_to_read { + let mut bytes = [0u8; 8]; + bytes.copy_from_slice( + &data_slice[i * element_size..(i + 1) * element_size], + ); + values.push(u64::from_le_bytes(bytes)); + } + values.resize(num_elements, 0); + Ok(TensorData::new(values, shape_clone.clone())) + } + _ => { + // For any remaining unsupported types, return an error + Err(crate::TensorSnapshotError::DataError(format!( + "Unsupported dtype for tensor data reading: {:?}", + dtype + ))) + } + } + } else { + // If no data file found, return zeros of the appropriate type + let num_elements = shape_clone.iter().product::().max(1); + match dtype { + DType::F32 => Ok(TensorData::new( + vec![0.0f32; num_elements], + shape_clone.clone(), + )), + DType::F64 => Ok(TensorData::new( + vec![0.0f64; num_elements], + shape_clone.clone(), + )), + DType::F16 => Ok(TensorData::new( + vec![f16::ZERO; num_elements], + shape_clone.clone(), + )), + DType::BF16 => Ok(TensorData::new( + vec![bf16::ZERO; num_elements], + shape_clone.clone(), + )), + DType::I64 => Ok(TensorData::new( + vec![0i64; num_elements], + shape_clone.clone(), + )), + DType::I32 => Ok(TensorData::new( + vec![0i32; num_elements], + shape_clone.clone(), + )), + DType::I16 => Ok(TensorData::new( + vec![0i16; num_elements], + shape_clone.clone(), + )), + DType::I8 => Ok(TensorData::new( + vec![0i8; num_elements], + shape_clone.clone(), + )), + DType::U8 => Ok(TensorData::new( + vec![0u8; num_elements], + shape_clone.clone(), + )), + DType::U16 => Ok(TensorData::new( + vec![0u16; num_elements], + shape_clone.clone(), + )), + DType::U32 => Ok(TensorData::new( + vec![0u32; num_elements], + shape_clone.clone(), + )), + DType::U64 => Ok(TensorData::new( + vec![0u64; num_elements], + shape_clone.clone(), + )), + DType::Bool => Ok(TensorData::new( + vec![false; num_elements], + shape_clone.clone(), + )), + _ => { + // For any remaining unsupported types, return an error + Err(crate::TensorSnapshotError::DataError(format!( + "Unsupported dtype for tensor data reading: {:?}", + dtype + ))) + } + } + } + }), + dtype, + shape, + vec![], // path_stack + vec![], // container_stack + ParamId::new(), // tensor_id + ))) +} + +pub struct Stack { + stack: Vec, + memo: HashMap, + data_source: Option>, +} + +impl Default for Stack { + fn default() -> Self { + Self::new() + } +} + +impl Stack { + pub fn new() -> Self { + // For cases where no data source is needed (pure pickle without tensor data) + Self { + stack: Vec::new(), + memo: HashMap::new(), + data_source: None, + } + } + + pub fn with_data_source(data_source: Arc) -> Self { + Self { + stack: Vec::new(), + memo: HashMap::new(), + data_source: Some(data_source), + } + } + + fn push(&mut self, o: Object) { + self.stack.push(o) + } + + fn pop(&mut self) -> Result { + match self.stack.pop() { + None => Err(PickleError::StackUnderflow), + Some(o) => Ok(o), + } + } + + fn top(&self) -> Result { + match self.stack.last() { + None => Err(PickleError::StackUnderflow), + Some(o) => Ok(o.clone()), + } + } + + fn pop_to_marker(&mut self) -> Result> { + let marker_pos = self + .stack + .iter() + .rposition(|o| { + matches!(o, Object::Class { module_name, name } + if module_name == "mark" && name == "mark") + }) + .ok_or(PickleError::InvalidData("marker not found".to_string()))?; + + let result = self.stack.split_off(marker_pos + 1); + self.stack.pop(); // Remove the marker + Ok(result) + } + + fn last_mut(&mut self) -> Result<&mut Object> { + match self.stack.last_mut() { + None => Err(PickleError::StackUnderflow), + Some(o) => Ok(o), + } + } + + fn push_mark(&mut self) { + self.stack.push(Object::Class { + module_name: "mark".to_string(), + name: "mark".to_string(), + }); + } + + fn memo_get(&self, idx: u32) -> Result { + self.memo + .get(&idx) + .cloned() + .ok_or(PickleError::MemoNotFound(idx)) + } + + fn memo_put(&mut self, idx: u32, obj: Object) { + self.memo.insert(idx, obj); + } + + fn memo_len(&self) -> usize { + self.memo.len() + } +} + +fn read_global(r: &mut R, stack: &mut Stack) -> Result<()> { + let module_name = buf_to_str(&read_to_newline(r)?)?; + let name = buf_to_str(&read_to_newline(r)?)?; + stack.push(Object::Class { module_name, name }); + Ok(()) +} + +fn read_long1(r: &mut R, stack: &mut Stack) -> Result<()> { + let len = r.read_u8()? as usize; + let mut data = vec![0u8; len]; + r.read_exact(&mut data)?; + // Handle little-endian signed integer + let mut value = 0i64; + for (i, &byte) in data.iter().enumerate().take(8) { + // Only process up to 8 bytes for i64, and use wrapping to avoid overflow + value |= (byte as i64).wrapping_shl((i as u32) * 8); + } + // Handle sign extension for negative numbers + if len < 8 && data.last().is_some_and(|&b| b & 0x80 != 0) { + // Sign extend + for i in len..8 { + value |= 0xffi64.wrapping_shl((i as u32) * 8); + } + } + stack.push(Object::Int(value)); + Ok(()) +} + +fn read_string(r: &mut R, stack: &mut Stack, len: usize) -> Result<()> { + let mut data = vec![0u8; len]; + r.read_exact(&mut data)?; + let s = buf_to_str(&data)?; + stack.push(Object::String(s)); + Ok(()) +} + +fn read_bin_int(r: &mut R, stack: &mut Stack) -> Result<()> { + let v = r.read_i32::()?; + stack.push(Object::Int(v as i64)); + Ok(()) +} + +fn read_int(r: &mut R, stack: &mut Stack) -> Result<()> { + // INT opcode reads an integer as ASCII string followed by newline + let line = read_to_newline(r)?; + let s = buf_to_str(&line)?; + let v = s + .parse::() + .map_err(|e| PickleError::InvalidData(format!("Invalid INT value '{}': {}", s, e)))?; + stack.push(Object::Int(v)); + Ok(()) +} + +fn read_bin_int1(r: &mut R, stack: &mut Stack) -> Result<()> { + let v = r.read_u8()?; + stack.push(Object::Int(v as i64)); + Ok(()) +} + +fn read_bin_int2(r: &mut R, stack: &mut Stack) -> Result<()> { + let v = r.read_u16::()?; + stack.push(Object::Int(v as i64)); + Ok(()) +} + +fn read_bin_float(r: &mut R, stack: &mut Stack) -> Result<()> { + // Python's BINFLOAT uses big-endian encoding + let v = r.read_f64::()?; + stack.push(Object::Float(v)); + Ok(()) +} + +pub fn read_pickle(r: &mut R) -> Result { + // For pure pickle without tensor data, no data source is needed + read_pickle_with_optional_data(r, None) +} + +/// Skip over a pickle without parsing it fully +/// This is useful for legacy format where we need to skip the main object +/// that contains tensors but we don't have a data source yet +pub fn skip_pickle(r: &mut R) -> Result<()> { + // Read the protocol marker if present + let mut first_byte = [0u8; 1]; + r.read_exact(&mut first_byte)?; + + if first_byte[0] == 0x80 { + // PROTO marker - read protocol version + let mut proto_version = [0u8; 1]; + r.read_exact(&mut proto_version)?; + } + // If not PROTO, the first byte is an opcode - continue to main loop + + // Helper to skip until newline + fn skip_line(r: &mut R) -> Result<()> { + let mut buf = Vec::new(); + r.read_until(b'\n', &mut buf)?; + Ok(()) + } + + // Helper to skip length-prefixed data + fn skip_length_prefixed(r: &mut R, length: usize) -> Result<()> { + let mut skip_buf = vec![0u8; length.min(8192)]; + let mut skipped = 0; + while skipped < length { + let to_skip = (length - skipped).min(skip_buf.len()); + r.read_exact(&mut skip_buf[..to_skip])?; + skipped += to_skip; + } + Ok(()) + } + + // Process first byte if it wasn't PROTO + let mut pending_byte = if first_byte[0] != 0x80 { + Some(first_byte[0]) + } else { + None + }; + + // Scan until we find STOP (0x2e) opcode + loop { + let byte = if let Some(b) = pending_byte.take() { + b + } else { + let mut byte = [0u8; 1]; + r.read_exact(&mut byte)?; + byte[0] + }; + + match byte { + 0x2e => { + // STOP - end of pickle + break; + } + // === Newline-terminated string opcodes === + 0x63 => { + // GLOBAL - two newline-terminated strings (module\nname\n) + skip_line(r)?; + skip_line(r)?; + } + 0x69 => { + // INST - two newline-terminated strings + skip_line(r)?; + skip_line(r)?; + } + 0x53 => { + // STRING - quoted string ending with newline + skip_line(r)?; + } + 0x46 | 0x49 | 0x4c => { + // FLOAT, INT, LONG - newline-terminated ASCII + skip_line(r)?; + } + 0x50 => { + // PERSID - newline-terminated persistent ID + skip_line(r)?; + } + // === Length-prefixed binary opcodes === + 0x58 | 0x42 | 0x43 | 0x54 | 0x55 | 0x56 | 0x8c | 0x8d | 0x8e => { + // String/bytes opcodes with length prefixes + let length = match byte { + 0x43 | 0x55 | 0x8c => { + // SHORT versions - 1 byte length + let mut len_byte = [0u8; 1]; + r.read_exact(&mut len_byte)?; + len_byte[0] as usize + } + 0x42 | 0x54 | 0x58 | 0x56 => { + // Regular versions - 4 byte length + let mut len_bytes = [0u8; 4]; + r.read_exact(&mut len_bytes)?; + u32::from_le_bytes(len_bytes) as usize + } + 0x8d | 0x8e => { + // 8-byte length versions + let mut len_bytes = [0u8; 8]; + r.read_exact(&mut len_bytes)?; + u64::from_le_bytes(len_bytes) as usize + } + _ => 0, + }; + skip_length_prefixed(r, length)?; + } + // === Fixed-size integer opcodes === + 0x4b => { + // BININT1 - 1 byte + let mut buf = [0u8; 1]; + r.read_exact(&mut buf)?; + } + 0x4d => { + // BININT2 - 2 bytes + let mut buf = [0u8; 2]; + r.read_exact(&mut buf)?; + } + 0x4a => { + // BININT - 4 bytes (signed int) + let mut buf = [0u8; 4]; + r.read_exact(&mut buf)?; + } + 0x47 => { + // BINFLOAT - 8 bytes + let mut buf = [0u8; 8]; + r.read_exact(&mut buf)?; + } + // === Variable-length integer opcodes === + 0x8a => { + // LONG1 - 1 byte length, then that many bytes + let mut len_byte = [0u8; 1]; + r.read_exact(&mut len_byte)?; + let length = len_byte[0] as usize; + skip_length_prefixed(r, length)?; + } + 0x8b => { + // LONG4 - 4 byte length, then that many bytes + let mut len_bytes = [0u8; 4]; + r.read_exact(&mut len_bytes)?; + let length = u32::from_le_bytes(len_bytes) as usize; + skip_length_prefixed(r, length)?; + } + // === Memo opcodes === + 0x71 | 0x68 => { + // BINPUT, BINGET - 1 byte index + let mut buf = [0u8; 1]; + r.read_exact(&mut buf)?; + } + 0x72 | 0x6a => { + // LONG_BINPUT, LONG_BINGET - 4 byte index + let mut buf = [0u8; 4]; + r.read_exact(&mut buf)?; + } + 0x67 | 0x70 => { + // GET, PUT - newline-terminated decimal index + skip_line(r)?; + } + // === Extension opcodes === + 0x82 => { + // EXT1 - 1 byte code + let mut buf = [0u8; 1]; + r.read_exact(&mut buf)?; + } + 0x83 => { + // EXT2 - 2 byte code + let mut buf = [0u8; 2]; + r.read_exact(&mut buf)?; + } + 0x84 => { + // EXT4 - 4 byte code + let mut buf = [0u8; 4]; + r.read_exact(&mut buf)?; + } + // === Frame opcode (protocol 4+) === + 0x95 => { + // FRAME - 8 byte frame size (we don't actually use framing, just skip the size) + let mut buf = [0u8; 8]; + r.read_exact(&mut buf)?; + } + // === Opcodes with no additional data === + // These just manipulate the stack or are markers + 0x28 | 0x29 | 0x30 | 0x31 | 0x32 | // MARK, TUPLE, POP, POP_MARK, DUP + 0x4e | 0x52 | 0x5d | 0x5b | 0x7d | // NONE, REDUCE, LIST, EMPTY_LIST, EMPTY_DICT + 0x61 | 0x62 | 0x64 | 0x65 | 0x73 | // APPEND, BUILD, DICT, APPENDS, SETITEM + 0x74 | 0x75 | 0x85 | 0x86 | 0x87 | // TUPLE, SETITEMS, TUPLE1, TUPLE2, TUPLE3 + 0x88 | 0x89 | 0x8f | 0x90 | 0x91 | // NEWTRUE, NEWFALSE, STACK_GLOBAL, MEMOIZE, EMPTY_SET + 0x92 | 0x93 | 0x94 | 0x51 | 0x81 => { // ADDITEMS, FROZENSET, NEWOBJ, BINPERSID, NEWOBJ_EX + // No additional data to skip + } + _ => { + // Unknown opcode - assume no additional data + // This is a best-effort approach + } + } + } + + Ok(()) +} + +pub fn read_pickle_with_data( + r: &mut R, + data_source: Arc, +) -> Result { + read_pickle_with_optional_data(r, Some(data_source)) +} + +fn get_dict_key(obj: Object) -> Result { + match obj { + Object::String(s) => Ok(s), + Object::Int(i) => Ok(i.to_string()), + _ => Err(PickleError::InvalidData(format!( + "dict key must be a valid type, got {obj:?}" + ))), + } +} + +pub fn read_pickle_with_optional_data( + r: &mut R, + data_source: Option>, +) -> Result { + let mut stack = match data_source { + Some(ds) => Stack::with_data_source(ds), + None => Stack::new(), + }; + loop { + let op_code = r.read_u8()?; + let op_code = OpCode::try_from(op_code).map_err(PickleError::InvalidOpCode)?; + match op_code { + OpCode::Proto => { + let version = r.read_u8()?; + if version > 5 { + return Err(PickleError::InvalidProtocol(version)); + } + } + OpCode::Global => read_global(r, &mut stack)?, + OpCode::BinInt => read_bin_int(r, &mut stack)?, + OpCode::Int => read_int(r, &mut stack)?, + OpCode::BinInt1 => read_bin_int1(r, &mut stack)?, + OpCode::BinInt2 => read_bin_int2(r, &mut stack)?, + OpCode::BinFloat => read_bin_float(r, &mut stack)?, + OpCode::BinUnicode => { + let len = r.read_u32::()? as usize; + read_string(r, &mut stack, len)? + } + OpCode::ShortBinString => { + let len = r.read_u8()? as usize; + read_string(r, &mut stack, len)? + } + OpCode::Long1 => read_long1(r, &mut stack)?, + OpCode::None => stack.push(Object::None), + OpCode::NewTrue => stack.push(Object::Bool(true)), + OpCode::NewFalse => stack.push(Object::Bool(false)), + OpCode::EmptyTuple => stack.push(Object::Tuple(Vec::new())), + OpCode::EmptyList => stack.push(Object::List(Vec::new())), + OpCode::EmptyDict => stack.push(Object::Dict(HashMap::new())), + OpCode::Tuple => { + let objs = stack.pop_to_marker()?; + stack.push(Object::Tuple(objs)) + } + OpCode::Tuple1 => { + let obj = stack.pop()?; + stack.push(Object::Tuple(vec![obj])) + } + OpCode::Tuple2 => { + let obj2 = stack.pop()?; + let obj1 = stack.pop()?; + stack.push(Object::Tuple(vec![obj1, obj2])) + } + OpCode::Tuple3 => { + let obj3 = stack.pop()?; + let obj2 = stack.pop()?; + let obj1 = stack.pop()?; + stack.push(Object::Tuple(vec![obj1, obj2, obj3])) + } + OpCode::Append => { + let value = stack.pop()?; + match stack.last_mut()? { + Object::List(list) => list.push(value), + _ => return Err(PickleError::UnexpectedOpCode(op_code)), + } + } + OpCode::Appends => { + let objs = stack.pop_to_marker()?; + match stack.last_mut()? { + Object::List(list) => list.extend(objs), + _ => return Err(PickleError::UnexpectedOpCode(op_code)), + } + } + OpCode::SetItem => { + let value = stack.pop()?; + let key = stack.pop()?; + match stack.last_mut()? { + Object::Dict(dict) => { + if let Object::String(key) = key { + dict.insert(key, value); + } else { + return Err(PickleError::InvalidData( + "dict key must be a string".to_string(), + )); + } + } + _ => return Err(PickleError::UnexpectedOpCode(op_code)), + } + } + OpCode::SetItems => { + let mut objs = stack.pop_to_marker()?; + if objs.len() % 2 != 0 { + return Err(PickleError::InvalidData( + "setitems requires even number of objects".to_string(), + )); + } + match stack.last_mut()? { + Object::Dict(dict) => { + while !objs.is_empty() { + let key = objs.remove(0); + let value = objs.remove(0); + let key = get_dict_key(key)?; + dict.insert(key, value); + } + } + _ => return Err(PickleError::UnexpectedOpCode(op_code)), + } + } + OpCode::BinPut => { + let idx = r.read_u8()? as u32; + let obj = stack.top()?; + stack.memo_put(idx, obj); + } + OpCode::LongBinPut => { + let idx = r.read_u32::()?; + let obj = stack.top()?; + stack.memo_put(idx, obj); + } + OpCode::BinGet => { + let idx = r.read_u8()? as u32; + let obj = stack.memo_get(idx)?; + stack.push(obj); + } + OpCode::LongBinGet => { + let idx = r.read_u32::()?; + let obj = stack.memo_get(idx)?; + stack.push(obj); + } + OpCode::Mark => stack.push_mark(), + OpCode::BinPersId => { + let pid = stack.pop()?; + match pid { + Object::String(s) => { + stack.push(Object::Persistent(s.into_bytes())); + } + Object::Tuple(tuple) => { + // The persistent ID is a tuple (e.g., ('storage', 'FloatStorage', '0', 'cpu', 4)) + // Store it as a PersistentTuple for proper handling + stack.push(Object::PersistentTuple(tuple)); + } + _ => { + return Err(PickleError::InvalidData(format!( + "persistent id must be a string or tuple, got {:?}", + pid + ))); + } + } + } + OpCode::Reduce => { + let args = stack.pop()?; + let callable = stack.pop()?; + + // Check if this is an OrderedDict + if let Object::Class { module_name, name } = &callable { + if module_name == "collections" && name == "OrderedDict" { + // OrderedDict can be created with items: OrderedDict([(key1, val1), ...]) + // The args is typically a tuple containing a list of [key, value] pairs + let mut dict = HashMap::new(); + + // Extract items from args + let items = match &args { + Object::Tuple(tuple) if !tuple.is_empty() => { + // Args is a tuple, get the first element (the list of items) + match &tuple[0] { + Object::List(list) => Some(list.clone()), + _ => None, + } + } + Object::List(list) => Some(list.clone()), + _ => None, + }; + + if let Some(items) = items { + for item in items { + // Each item is a list/tuple of [key, value] + match item { + Object::List(pair) | Object::Tuple(pair) if pair.len() >= 2 => { + if let Object::String(key) = &pair[0] { + dict.insert(key.clone(), pair[1].clone()); + } + } + _ => {} + } + } + } + + stack.push(Object::Dict(dict)); + } else { + let _obj = Object::Reduce { + callable: Box::new(callable.clone()), + args: Box::new(args.clone()), + }; + let obj = rebuild_from_type_v2( + Object::Tuple(vec![callable, args]), + &mut stack.memo, + &stack.data_source, + )?; + stack.push(obj); + } + } else { + let _obj = Object::Reduce { + callable: Box::new(callable.clone()), + args: Box::new(args.clone()), + }; + let obj = rebuild_from_type_v2( + Object::Tuple(vec![callable, args]), + &mut stack.memo, + &stack.data_source, + )?; + stack.push(obj); + } + } + OpCode::Build => { + let args = stack.pop()?; + let obj = stack.pop()?; + match obj { + Object::Dict(mut dict) => { + // For dicts, BUILD updates with the args + if let Object::Dict(update) = args { + dict.extend(update); + } + stack.push(Object::Dict(dict)); + } + _ => { + stack.push(Object::Build { + callable: Box::new(obj), + args: Box::new(args), + }); + } + } + } + OpCode::NewObj => { + let args = stack.pop()?; + let cls = stack.pop()?; + stack.push(Object::Reduce { + callable: Box::new(cls), + args: Box::new(args), + }); + } + OpCode::Dict => { + let objs = stack.pop_to_marker()?; + let mut dict = HashMap::new(); + if objs.len() % 2 != 0 { + return Err(PickleError::InvalidData( + "dict requires even number of objects".to_string(), + )); + } + for chunk in objs.chunks(2) { + let key = get_dict_key(chunk[0].clone())?; + dict.insert(key, chunk[1].clone()); + } + stack.push(Object::Dict(dict)); + } + OpCode::List => { + let objs = stack.pop_to_marker()?; + stack.push(Object::List(objs)); + } + OpCode::Memoize => { + // Store top of stack in memo without popping + // The memo index is the current number of items in the memo + let obj = stack.top()?; + let idx = stack.memo_len() as u32; + stack.memo_put(idx, obj); + } + OpCode::Stop => break, + } + } + stack.pop() +} + +/// Load tensors from a pickle file (PyTorch checkpoint format) +pub fn read_pickle_tensors(reader: &mut R) -> Result> { + let obj = read_pickle(reader)?; + + // Extract tensors from the loaded object + let mut tensors = HashMap::new(); + let mut path = Vec::new(); + extract_tensors(&obj, &mut path, &mut tensors); + + Ok(tensors) +} + +fn extract_tensors<'a>( + obj: &'a Object, + path: &mut Vec<&'a str>, + tensors: &mut HashMap, +) { + match obj { + Object::Dict(dict) => { + for (key, value) in dict { + path.push(key); + extract_tensors(value, path, tensors); + path.pop(); + } + } + Object::TorchParam(snapshot) => { + // Only allocate the string here when we actually insert + tensors.insert(path.join("."), snapshot.clone()); + } + _ => {} + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/reader.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/reader.rs new file mode 100644 index 0000000..4e21013 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/reader.rs @@ -0,0 +1,1133 @@ +//! PyTorch file reader implementation. +//! +//! This module provides support for reading PyTorch checkpoint files (.pt/.pth). +//! +//! # Supported Formats +//! +//! ## 1. Modern ZIP Format (PyTorch 1.6+) +//! Files are ZIP archives containing: +//! - `data.pkl` or `archive/data.pkl`: Pickled tensor metadata +//! - `data/` directory: Binary tensor data files +//! +//! ## 2. TAR Format (older torchvision models like AlexNet, SqueezeNet) +//! TAR archives containing: +//! - `sys_info`: System info pickle (endianness, type sizes) +//! - `pickle`: OrderedDict mapping tensor names to storage keys +//! - `tensors`: Tensor metadata (unused, metadata is in pickle) +//! - `storages`: Count pickle + sequential (metadata, num_elements, raw data) +//! +//! ## 3. Legacy Pickle Format (PyTorch 0.1.10 - 1.5) +//! Sequential pickle streams with the structure: +//! - Magic number pickle (0x1950a86a20f9469cfc6c) +//! - Protocol version pickle (e.g., 1001) +//! - System info pickle (endianness, type sizes) +//! - Model data pickle (state_dict or full model) +//! +//! ## 4. Simple Pickle Format +//! Direct pickle file with a dictionary at the root, commonly used for +//! manually saved state_dicts. +//! +//! # Compatibility +//! +//! The reader handles backward compatibility by detecting the file format +//! automatically. Files from PyTorch 0.1.10 through current versions are +//! supported, though full model saves (vs state_dict) may have limitations +//! as they contain Python code references. + +use crate::TensorSnapshot; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use burn_core::record::serde::{adapter::DefaultAdapter, data::NestedValue, de::Deserializer}; +use serde::de::DeserializeOwned; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufReader, Read, Seek, SeekFrom}; +use std::path::Path; + +use super::lazy_data::LazyDataSource; +use super::pickle_reader::{Object, PickleError, read_pickle, read_pickle_with_data}; +use std::sync::Arc; + +/// Error type for PyTorch file operations +#[derive(Debug)] +pub enum PytorchError { + /// IO error + Io(std::io::Error), + /// Pickle parsing error + Pickle(PickleError), + /// Zip archive error + Zip(zip::result::ZipError), + /// TAR archive error + Tar(std::io::Error), + /// Invalid file format + InvalidFormat(String), + /// Key not found + KeyNotFound(String), + /// Serde deserialization error + Serde(burn_core::record::serde::error::Error), +} + +impl From for PytorchError { + fn from(e: std::io::Error) -> Self { + PytorchError::Io(e) + } +} + +impl From for PytorchError { + fn from(e: PickleError) -> Self { + PytorchError::Pickle(e) + } +} + +impl From for PytorchError { + fn from(e: zip::result::ZipError) -> Self { + PytorchError::Zip(e) + } +} + +impl From for PytorchError { + fn from(e: burn_core::record::serde::error::Error) -> Self { + PytorchError::Serde(e) + } +} + +impl std::fmt::Display for PytorchError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PytorchError::Io(e) => write!(f, "IO error: {}", e), + PytorchError::Pickle(e) => write!( + f, + "Pickle parsing error: {}. This may indicate an unsupported PyTorch file format or corrupted file.", + e + ), + PytorchError::Zip(e) => write!(f, "Zip archive error: {}", e), + PytorchError::Tar(e) => write!(f, "TAR archive error: {}", e), + PytorchError::InvalidFormat(msg) => write!(f, "Invalid PyTorch file format: {}", msg), + PytorchError::KeyNotFound(key) => write!( + f, + "Key '{}' not found in PyTorch file. Available keys may be listed with the keys() method.", + key + ), + PytorchError::Serde(e) => write!(f, "Serde deserialization error: {}", e), + } + } +} + +impl std::error::Error for PytorchError {} + +type Result = std::result::Result; + +/// Metadata about a PyTorch file +/// +/// Contains information about the file format, version, and other properties +/// that can be useful for debugging or compatibility checking. +#[derive(Debug, Clone)] +pub struct PytorchMetadata { + /// Format version (e.g., "1.0" for modern ZIP format) + pub format_version: Option, + /// File format type (ZIP, Legacy, or Pickle) + pub format_type: FileFormat, + /// Byte order (endianness) - currently only LittleEndian is supported + pub byte_order: ByteOrder, + /// Whether the file has storage alignment information + pub has_storage_alignment: bool, + /// PyTorch version that saved the file (if available) + pub pytorch_version: Option, + /// Number of tensors in the file + pub tensor_count: usize, + /// Total size of tensor data in bytes (if available) + pub total_data_size: Option, +} + +impl PytorchMetadata { + /// Check if this is a modern format file (ZIP-based, PyTorch 1.6+) + pub fn is_modern_format(&self) -> bool { + matches!(self.format_type, FileFormat::Zip) + } + + /// Check if this is a legacy format file (PyTorch 0.1.10 - 1.5) + pub fn is_legacy_format(&self) -> bool { + matches!(self.format_type, FileFormat::Legacy) + } +} + +/// File format type +#[derive(Debug, Clone, PartialEq)] +pub enum FileFormat { + /// ZIP-based format (PyTorch 1.6+) + Zip, + /// TAR-based format (older torchvision models) + Tar, + /// Legacy format (PyTorch 0.1.10 - 1.5) + Legacy, + /// Simple pickle file + Pickle, +} + +/// Byte order (endianness) +#[derive(Debug, Clone, PartialEq)] +pub enum ByteOrder { + LittleEndian, + BigEndian, +} + +/// PyTorch checkpoint reader +/// +/// This is the main interface for reading PyTorch checkpoint files (.pt/.pth). +/// It supports multiple PyTorch formats including modern ZIP-based format (1.6+), +/// legacy format (0.1.10-1.5), and simple pickle files. +/// +/// # Example +/// ```rust,no_run +/// # use burn_store::pytorch::PytorchReader; +/// # fn example() -> Result<(), Box> { +/// // Load a checkpoint file +/// let reader = PytorchReader::new("model.pt")?; +/// +/// // Get tensor names +/// let keys = reader.keys(); +/// +/// // Access a specific tensor +/// if let Some(tensor) = reader.get("conv1.weight") { +/// let data = tensor.to_data(); // Materializes the tensor +/// } +/// +/// // Check file metadata +/// println!("Format: {:?}", reader.metadata().format_type); +/// println!("Tensor count: {}", reader.metadata().tensor_count); +/// # Ok(()) +/// # } +/// ``` +pub struct PytorchReader { + tensors: HashMap, + metadata: PytorchMetadata, +} + +impl PytorchReader { + /// Load a PyTorch checkpoint file + /// + /// # Arguments + /// * `path` - Path to the PyTorch file (.pt or .pth) + /// + /// # Returns + /// A `PytorchReader` with lazy-loaded tensors and metadata + pub fn new>(path: P) -> Result { + let (tensors, metadata) = load_pytorch_file_with_metadata(path.as_ref(), None)?; + Ok(Self { tensors, metadata }) + } + + /// Load a PyTorch checkpoint with a specific top-level key + /// + /// Many PyTorch checkpoints store the model weights under a specific key + /// like "state_dict", "model", or "model_state_dict". + /// + /// # Arguments + /// * `path` - Path to the PyTorch file + /// * `key` - Top-level key to extract (e.g., "state_dict") + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::pytorch::PytorchReader; + /// # fn example() -> Result<(), Box> { + /// let reader = PytorchReader::with_top_level_key("checkpoint.pt", "state_dict")?; + /// # Ok(()) + /// # } + /// ``` + pub fn with_top_level_key>(path: P, key: &str) -> Result { + let (tensors, metadata) = load_pytorch_file_with_metadata(path.as_ref(), Some(key))?; + Ok(Self { tensors, metadata }) + } + + /// Load from a reader + /// + /// This method is useful when loading from non-file sources like memory buffers. + /// Note: Metadata detection is limited when loading from a reader. + /// + /// # Arguments + /// * `reader` - Any type implementing `Read` + /// * `top_level_key` - Optional key to extract + pub fn from_reader(reader: R, top_level_key: Option<&str>) -> Result { + // For reader-based loading, we don't have full metadata access + let tensors = load_from_reader(reader, top_level_key)?; + let metadata = PytorchMetadata { + format_version: None, + format_type: FileFormat::Pickle, // Default assumption + byte_order: ByteOrder::LittleEndian, + has_storage_alignment: false, + pytorch_version: None, + tensor_count: tensors.len(), + total_data_size: None, + }; + Ok(Self { tensors, metadata }) + } + + /// Get all tensor names + pub fn keys(&self) -> Vec { + self.tensors.keys().cloned().collect() + } + + /// Get a tensor by name + pub fn get(&self, name: &str) -> Option<&TensorSnapshot> { + self.tensors.get(name) + } + + /// Get all tensors + pub fn tensors(&self) -> &HashMap { + &self.tensors + } + + /// Take ownership of all tensors + pub fn into_tensors(self) -> HashMap { + self.tensors + } + + /// Get metadata about the loaded file + /// + /// Provides information about the file format, version, endianness, etc. + pub fn metadata(&self) -> &PytorchMetadata { + &self.metadata + } + + /// Get the number of tensors in the file + pub fn len(&self) -> usize { + self.tensors.len() + } + + /// Check if the file contains no tensors + pub fn is_empty(&self) -> bool { + self.tensors.is_empty() + } + + /// Read raw pickle data from a PyTorch file + /// + /// This is useful for extracting configuration or metadata that isn't tensor data. + /// Returns a simplified JSON-like structure that can be easily converted to other formats. + /// + /// # Arguments + /// * `path` - Path to the PyTorch file + /// * `top_level_key` - Optional key to extract from the top-level dictionary + /// + /// # Returns + /// A `PickleValue` representing the pickle data structure + pub fn read_pickle_data>( + path: P, + top_level_key: Option<&str>, + ) -> Result { + read_pickle_as_value(path.as_ref(), top_level_key) + } + + /// Load and deserialize configuration data from a PyTorch file + /// + /// This method reads configuration or metadata stored in PyTorch checkpoint files + /// and deserializes it into the specified type. It's particularly useful for + /// extracting model configurations that might be saved alongside model weights. + /// + /// # Arguments + /// * `path` - Path to the PyTorch file (.pt or .pth) + /// * `top_level_key` - Optional key to extract specific data within the pickle file. + /// If `None`, the entire content is deserialized. + /// + /// # Type Parameters + /// * `D` - The target type to deserialize into. Must implement `DeserializeOwned`. + /// + /// # Returns + /// A `Result` containing the deserialized configuration data, or an `Error` if + /// reading or deserialization fails. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::pytorch::PytorchReader; + /// # use serde::Deserialize; + /// # fn example() -> Result<(), Box> { + /// #[derive(Debug, Deserialize)] + /// struct ModelConfig { + /// hidden_size: usize, + /// num_layers: usize, + /// } + /// + /// let config: ModelConfig = PytorchReader::load_config("model.pth", Some("config"))?; + /// # Ok(()) + /// # } + /// ``` + pub fn load_config(path: P, top_level_key: Option<&str>) -> Result + where + D: DeserializeOwned, + P: AsRef, + { + // Read the PyTorch file and extract the pickle data + let pickle_value = Self::read_pickle_data(path, top_level_key)?; + + // Convert PickleValue to NestedValue + let nested_value = convert_pickle_to_nested_value(pickle_value)?; + + // Create a deserializer with the default adapter + let deserializer = Deserializer::::new(nested_value, false); + + // Deserialize the nested value into the target type + let value = D::deserialize(deserializer)?; + Ok(value) + } +} + +/// Simplified representation of pickle data +/// +/// This enum provides a JSON-like structure that's easier to work with +/// than the internal pickle Object type. +#[derive(Debug, Clone, PartialEq)] +pub enum PickleValue { + /// None/null value + None, + /// Boolean value + Bool(bool), + /// Integer value + Int(i64), + /// Floating point value + Float(f64), + /// String value + String(String), + /// List/array of values + List(Vec), + /// Dictionary/map of string keys to values + Dict(HashMap), + /// Binary data + Bytes(Vec), +} + +/// Internal function to load a PyTorch file with metadata +fn load_pytorch_file_with_metadata( + path: &Path, + top_level_key: Option<&str>, +) -> Result<(HashMap, PytorchMetadata)> { + // First, try to read as a zip file + if let Ok(file) = File::open(path) + && let Ok(mut archive) = zip::ZipArchive::new(BufReader::new(file)) + { + // PyTorch saves the main data in various locations within the zip + let mut pickle_data = Vec::new(); + let mut pickle_found = false; + + // Try different common pickle file locations + let possible_pickle_paths = [ + "data.pkl", + "archive/data.pkl", + // Look for any .pkl file in the root or first-level directories + ]; + + for pickle_path in &possible_pickle_paths { + if archive.by_name(pickle_path).is_ok() { + let mut pickle_file = archive.by_name(pickle_path)?; + pickle_file.read_to_end(&mut pickle_data)?; + pickle_found = true; + break; + } + } + + // If not found in standard locations, search for any .pkl file + if !pickle_found { + for i in 0..archive.len() { + let file = archive.by_index(i)?; + let name = file.name().to_string(); + drop(file); // Release the borrow + + if name.ends_with("data.pkl") { + let mut file = archive.by_index(i)?; + file.read_to_end(&mut pickle_data)?; + pickle_found = true; + break; + } + } + } + + if !pickle_found { + return Err(PytorchError::InvalidFormat( + "No data.pkl file found in ZIP archive. Expected PyTorch 1.6+ format with data.pkl or archive/data.pkl".to_string(), + )); + } + + // Check for format version (optional) + let format_version = if let Ok(mut version_file) = archive.by_name(".format_version") { + let mut version_data = Vec::new(); + version_file.read_to_end(&mut version_data)?; + let version_str = String::from_utf8_lossy(&version_data); + let version = version_str.trim().to_string(); + Some(version) + } else { + None + }; + + // Check for byteorder file to detect endianness + let is_big_endian = if let Ok(mut byteorder_file) = archive.by_name("byteorder") { + let mut byteorder_data = Vec::new(); + byteorder_file.read_to_end(&mut byteorder_data)?; + let byteorder_str = String::from_utf8_lossy(&byteorder_data); + byteorder_str.trim() == "big" + } else { + false // Default to little-endian if no byteorder file + }; + + if is_big_endian { + // Big-endian files are not yet supported as they require different byte order conversion + // TODO: To support big-endian files, we need to: + // 1. Pass endianness info through to pickle_reader + // 2. Use from_be_bytes instead of from_le_bytes for tensor data + // 3. Handle byte swapping for all numeric types (f32, f64, i32, etc.) + return Err(PytorchError::InvalidFormat( + "Big-endian PyTorch files are not yet supported. The file was saved on a big-endian system and requires byte order conversion.".to_string() + )); + } + + // Check for storage alignment file + let has_storage_alignment = archive.by_name(".storage_alignment").is_ok(); + + // Check for PyTorch version (if saved) + let pytorch_version = if let Ok(mut version_file) = archive.by_name("version") { + let mut version_data = Vec::new(); + version_file.read_to_end(&mut version_data)?; + Some(String::from_utf8_lossy(&version_data).trim().to_string()) + } else { + None + }; + + // Create a lazy data source instead of loading all data upfront + let data_source = Arc::new(LazyDataSource::from_zip(path)?); + + // Calculate total data size without loading + let mut total_data_size = 0usize; + for i in 0..archive.len() { + let file = archive.by_index(i)?; + let name = file.name(); + + // Look for data files - they can be in various locations + let is_data_file = (name.contains("/data/") + || name.starts_with("data/") + || name.starts_with("archive/data/")) + && !name.ends_with(".pkl") + && !name.ends_with("/"); + + if is_data_file { + total_data_size += file.size() as usize; + } + } + + // Parse the pickle data with lazy data source + let mut pickle_reader = BufReader::new(pickle_data.as_slice()); + let obj = read_pickle_with_data(&mut pickle_reader, data_source)?; + + // Extract tensors with their data + let tensors = extract_tensors_with_data(obj, top_level_key)?; + + // Create metadata + let metadata = PytorchMetadata { + format_version, + format_type: FileFormat::Zip, + byte_order: if is_big_endian { + ByteOrder::BigEndian + } else { + ByteOrder::LittleEndian + }, + has_storage_alignment, + pytorch_version, + tensor_count: tensors.len(), + total_data_size: Some(total_data_size), + }; + + return Ok((tensors, metadata)); + } + + // If not a zip or zip reading failed, try TAR format + if is_tar_file(path) { + return load_tar_pytorch_file_with_metadata(path, top_level_key); + } + + // Try reading as a plain pickle file + let mut file = File::open(path)?; + + // Check for PyTorch legacy format (starts with magic number as pickled integer) + let mut header = [0u8; 15]; + // Use read() instead of read_exact() to handle files smaller than 15 bytes + let bytes_read = file.read(&mut header)?; + file.seek(std::io::SeekFrom::Start(0))?; + + // Only check for legacy format if we have enough bytes + // PyTorch legacy format detection (PyTorch 0.1.10 - 1.3) + // Reference: https://github.com/pytorch/pytorch/blob/main/torch/serialization.py#L65 + // + // These files use sequential pickle streams with metadata before the actual data. + // Format structure: + // 1. Magic number (0x1950a86a20f9469cfc6c) stored as LONG1 pickle + // 2. Protocol version (e.g., 1001) + // 3. System info dict (protocol_version, little_endian, type_sizes) + // 4. Actual model data (state_dict or full model) + // 5. Storage keys list (pickle) + // 6. Raw binary data for each storage + // + // The pattern is: 0x80 0x02 0x8a 0x0a (PROTO 2, LONG1 with 10 bytes) + // followed by 10 bytes of magic number (little-endian), then 0x2e (STOP) + let is_legacy_format = bytes_read >= 15 + && header[0] == 0x80 // PROTO opcode + && header[1] == 0x02 // Protocol version 2 + && header[2] == 0x8a // LONG1 opcode + && header[3] == 0x0a // 10 bytes follow + // Magic number 0x1950a86a20f9469cfc6c in little-endian + && header[4] == 0x6c + && header[5] == 0xfc + && header[6] == 0x9c + && header[7] == 0x46 + && header[8] == 0xf9 + && header[9] == 0x20 + && header[10] == 0x6a + && header[11] == 0xa8 + && header[12] == 0x50 + && header[13] == 0x19 + && header[14] == 0x2e; // STOP opcode + + if is_legacy_format { + return load_legacy_pytorch_file_with_metadata(path, top_level_key); + } + + // Standard pickle file + // This might be a pickle with tensor references, so we need to handle that case + // For plain pickle files without a separate data section, we can't use lazy loading + // so we'll just create empty placeholder tensors for the structure + let file = File::open(path)?; + let mut reader = BufReader::new(file); + + // Try reading without data source first + match read_pickle(&mut reader) { + Ok(obj) => { + let tensors = extract_tensors_with_data(obj, top_level_key)?; + let tensor_count = tensors.len(); + Ok(( + tensors, + PytorchMetadata { + format_version: None, + format_type: FileFormat::Pickle, + byte_order: ByteOrder::LittleEndian, + has_storage_alignment: false, + pytorch_version: None, + tensor_count, + total_data_size: None, + }, + )) + } + Err(e) + if e.to_string() + .contains("Cannot load tensor data without a data source") => + { + // This pickle file contains tensor data but we're trying to read it without + // providing a data source. This shouldn't happen in normal usage as PyTorch + // files with actual tensor data should be in ZIP or legacy format. + Err(PytorchError::InvalidFormat( + "Pickle file contains tensor data but no data source is available. This file should be loaded as ZIP or legacy format.".to_string() + )) + } + Err(e) => Err(PytorchError::Pickle(e)), + } +} + +/// Load from a reader +fn load_from_reader( + reader: R, + top_level_key: Option<&str>, +) -> Result> { + let mut buf_reader = BufReader::new(reader); + + // Try reading without data source + match read_pickle(&mut buf_reader) { + Ok(obj) => extract_tensors_with_data(obj, top_level_key), + Err(e) + if e.to_string() + .contains("Cannot load tensor data without a data source") => + { + // This reader contains tensor data but we can't load it without a file path + Err(PytorchError::InvalidFormat( + "Reader contains tensor data but no data source is available. Use file-based loading instead.".to_string() + )) + } + Err(e) => Err(PytorchError::Pickle(e)), + } +} + +/// Extract tensors from a parsed pickle object +fn extract_tensors_with_data( + obj: Object, + top_level_key: Option<&str>, +) -> Result> { + let dict = match obj { + Object::Dict(dict) => { + if let Some(key) = top_level_key { + // Extract the nested dictionary if a top-level key is specified + match dict.get(key) { + Some(Object::Dict(nested)) => nested.clone(), + _ => { + return Err(PytorchError::KeyNotFound(format!( + "Top-level key '{}' not found or is not a dictionary. Available top-level keys in file: {:?}", + key, + dict.keys().collect::>() + ))); + } + } + } else { + dict + } + } + _ => { + return Err(PytorchError::InvalidFormat( + "Expected a dictionary at the root of the PyTorch file, but found a different type. The file may be a full model save rather than a state_dict.".to_string(), + )); + } + }; + + let mut tensors = HashMap::new(); + let mut path = Vec::new(); + extract_tensors_recursive(&Object::Dict(dict), &mut path, &mut tensors); + Ok(tensors) +} + +/// Recursively extract tensors from an object +fn extract_tensors_recursive<'a>( + obj: &'a Object, + path: &mut Vec<&'a str>, + tensors: &mut HashMap, +) { + match obj { + Object::Dict(dict) => { + for (key, value) in dict { + path.push(key); + extract_tensors_recursive(value, path, tensors); + path.pop(); + } + } + Object::TorchParam(snapshot) => { + // The TensorSnapshot already contains the data loading closure + // Only allocate the string here when we actually insert + tensors.insert(path.join("."), snapshot.clone()); + } + _ => {} + } +} + +/// Load a legacy PyTorch file with metadata +fn load_legacy_pytorch_file_with_metadata( + path: &Path, + top_level_key: Option<&str>, +) -> Result<(HashMap, PytorchMetadata)> { + let file = File::open(path)?; + let mut reader = BufReader::new(file); + + // Skip metadata pickles + // 1. Magic number + let _ = read_pickle(&mut reader).map_err(|e| { + PytorchError::InvalidFormat(format!( + "Failed to read magic number from legacy format: {}", + e + )) + })?; + + // 2. Protocol version + let _ = read_pickle(&mut reader).map_err(|e| { + PytorchError::InvalidFormat(format!( + "Failed to read protocol version from legacy format: {}", + e + )) + })?; + + // 3. System info + let _ = read_pickle(&mut reader).map_err(|e| { + PytorchError::InvalidFormat(format!( + "Failed to read system info from legacy format: {}", + e + )) + })?; + + // Save position before main pickle + let main_pickle_pos = reader.stream_position()?; + + // 4. Skip main object - it might contain tensors so we can't parse it yet + // We'll re-read it with a data source later + use crate::pytorch::pickle_reader::skip_pickle; + skip_pickle(&mut reader).map_err(|e| { + PytorchError::InvalidFormat(format!( + "Failed to skip main object in legacy format: {}", + e + )) + })?; + + // 5. Storage keys list (sorted keys as written by PyTorch) + let storage_keys = match read_pickle(&mut reader) { + Ok(Object::List(keys)) => keys + .into_iter() + .filter_map(|obj| match obj { + Object::String(s) => Some(s), + _ => None, + }) + .collect::>(), + _ => vec![], + }; + + // 6. Skip 8-byte header before raw binary data + // PyTorch legacy format has an 8-byte header (possibly protocol version or alignment) + // between the storage keys list and the actual tensor data + let mut header = [0u8; 8]; + if reader.read(&mut header).is_ok() { + // Header read successfully, data starts after this + } + + // 7. Raw binary data starts here + let data_start_pos = reader.stream_position()?; + let file_size = reader.seek(SeekFrom::End(0))?; + let data_size = file_size - data_start_pos; + + // Create a lazy data source for legacy multi-storage format + let data_source = Arc::new(LazyDataSource::from_legacy_multi_storage( + path, + data_start_pos, + data_size, + )); + + // Set storage keys BEFORE parsing the main pickle + // This is critical because track_storage_usage() is called during parsing + // and it needs storage_keys to build the storage map + if let LazyDataSource::LegacyMultiStorage(ref source) = *data_source + && !storage_keys.is_empty() + { + let source = source + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + source.set_storage_keys(storage_keys.clone()); + } + + // Now re-read the main pickle with lazy data source + reader.seek(SeekFrom::Start(main_pickle_pos))?; + let main_obj = read_pickle_with_data(&mut reader, data_source.clone())?; + + // Extract tensors normally + let tensors = extract_tensors_with_data(main_obj, top_level_key)?; + + // Create metadata for legacy format + let metadata = PytorchMetadata { + format_version: None, // Legacy format doesn't have version files + format_type: FileFormat::Legacy, + byte_order: ByteOrder::LittleEndian, // Legacy format is little-endian + has_storage_alignment: false, + pytorch_version: None, // Could parse from protocol version, but not reliable + tensor_count: tensors.len(), + total_data_size: Some(data_size as usize), + }; + + Ok((tensors, metadata)) +} + +/// Check if a file is a TAR archive +fn is_tar_file(path: &Path) -> bool { + if let Ok(mut file) = File::open(path) { + // TAR files have "ustar" magic at offset 257 + let mut header = [0u8; 263]; + if file.read_exact(&mut header).is_ok() { + // Check for "ustar" magic at offset 257 + return &header[257..262] == b"ustar"; + } + } + false +} + +/// Load a TAR format PyTorch file with metadata +fn load_tar_pytorch_file_with_metadata( + path: &Path, + top_level_key: Option<&str>, +) -> Result<(HashMap, PytorchMetadata)> { + use tar::Archive; + + let file = File::open(path)?; + let mut archive = Archive::new(BufReader::new(file)); + + // Extract the main entries from the TAR archive + let mut sys_info_data: Option> = None; + let mut pickle_data: Option> = None; + let mut storages_data: Option> = None; + + for entry in archive.entries().map_err(PytorchError::Tar)? { + let mut entry = entry.map_err(PytorchError::Tar)?; + let entry_path = entry + .path() + .map_err(PytorchError::Tar)? + .to_string_lossy() + .to_string(); + + // Skip PAX headers + if entry_path.contains("@PaxHeader") { + continue; + } + + // Normalize path (remove ./ prefix if present) + let normalized = entry_path.trim_start_matches("./"); + + match normalized { + "sys_info" => { + let mut data = Vec::new(); + entry.read_to_end(&mut data).map_err(PytorchError::Tar)?; + sys_info_data = Some(data); + } + "pickle" => { + let mut data = Vec::new(); + entry.read_to_end(&mut data).map_err(PytorchError::Tar)?; + pickle_data = Some(data); + } + "storages" => { + let mut data = Vec::new(); + entry.read_to_end(&mut data).map_err(PytorchError::Tar)?; + storages_data = Some(data); + } + _ => {} + } + } + + // Validate required entries + let pickle_data = pickle_data.ok_or_else(|| { + PytorchError::InvalidFormat("TAR file missing 'pickle' entry".to_string()) + })?; + let storages_data = storages_data.ok_or_else(|| { + PytorchError::InvalidFormat("TAR file missing 'storages' entry".to_string()) + })?; + + // Parse sys_info to check endianness + let is_little_endian = if let Some(ref data) = sys_info_data { + parse_tar_sys_info(data)? + } else { + true // Default to little-endian + }; + + if !is_little_endian { + return Err(PytorchError::InvalidFormat( + "Big-endian TAR PyTorch files are not supported".to_string(), + )); + } + + // Create TarSource for lazy loading + let data_source = Arc::new(LazyDataSource::from_tar(&storages_data)?); + + // Parse the pickle (OrderedDict of name -> storage_key) + let mut pickle_reader = BufReader::new(pickle_data.as_slice()); + let obj = read_pickle_with_data(&mut pickle_reader, data_source)?; + + // Extract tensors + let tensors = extract_tensors_with_data(obj, top_level_key)?; + + let metadata = PytorchMetadata { + format_version: None, + format_type: FileFormat::Tar, + byte_order: ByteOrder::LittleEndian, + has_storage_alignment: false, + pytorch_version: None, + tensor_count: tensors.len(), + total_data_size: Some(storages_data.len()), + }; + + Ok((tensors, metadata)) +} + +/// Parse sys_info pickle from TAR format to extract endianness +fn parse_tar_sys_info(data: &[u8]) -> Result { + let mut reader = BufReader::new(data); + let obj = read_pickle(&mut reader)?; + + if let Object::Dict(dict) = obj + && let Some(Object::Bool(little_endian)) = dict.get("little_endian") + { + return Ok(*little_endian); + } + + Ok(true) // Default assumption +} + +/// Read pickle data from a PyTorch file as a simplified value +fn read_pickle_as_value(path: &Path, top_level_key: Option<&str>) -> Result { + use crate::pytorch::lazy_data::LazyDataSource; + use crate::pytorch::pickle_reader::{read_pickle, read_pickle_with_data}; + use std::sync::Arc; + + // Try to open as ZIP first + if let Ok(file) = File::open(path) + && let Ok(mut archive) = zip::ZipArchive::new(BufReader::new(file)) + { + // Read pickle data from ZIP + let mut pickle_data = Vec::new(); + + // Try standard locations + for pickle_path in &["data.pkl", "archive/data.pkl"] { + if let Ok(mut pickle_file) = archive.by_name(pickle_path) { + pickle_file.read_to_end(&mut pickle_data)?; + break; + } + } + + // If not found, search for any .pkl file + if pickle_data.is_empty() { + for i in 0..archive.len() { + let file = archive.by_index(i)?; + let name = file.name().to_string(); + drop(file); + + if name.ends_with("data.pkl") { + let mut file = archive.by_index(i)?; + file.read_to_end(&mut pickle_data)?; + break; + } + } + } + + if !pickle_data.is_empty() { + // Create a data source for the ZIP file + let data_source = LazyDataSource::from_zip(path)?; + let data_source_arc = Arc::new(data_source); + + let mut reader = BufReader::new(pickle_data.as_slice()); + let obj = read_pickle_with_data(&mut reader, data_source_arc)?; + return convert_object_to_value(obj, top_level_key); + } + } + + // Try as plain pickle file + // First attempt without data source (for pure metadata files) + let file = File::open(path)?; + let mut reader = BufReader::new(file); + + match read_pickle(&mut reader) { + Ok(obj) => convert_object_to_value(obj, top_level_key), + Err(e) + if e.to_string() + .contains("Cannot load tensor data without a data source") => + { + // File contains tensors, need to use full PytorchReader + // Use the regular reader to get proper tensor handling + let reader = PytorchReader::new(path)?; + + // Convert tensors to PickleValue structure + let mut result = std::collections::HashMap::new(); + for key in reader.keys() { + // For pickle value extraction, we just need the structure, not the actual data + result.insert( + key.clone(), + PickleValue::String(format!("", key)), + ); + } + + if let Some(key) = top_level_key { + Ok(PickleValue::Dict( + [(key.to_string(), PickleValue::Dict(result))] + .into_iter() + .collect(), + )) + } else { + Ok(PickleValue::Dict(result)) + } + } + Err(e) => Err(PytorchError::Pickle(e)), + } +} + +/// Convert internal Object to public PickleValue +fn convert_object_to_value(obj: Object, top_level_key: Option<&str>) -> Result { + use crate::pytorch::pickle_reader::Object; + + // If a top-level key is specified, extract it first + if let Some(key) = top_level_key + && let Object::Dict(dict) = obj + { + if let Some(value) = dict.get(key) { + return object_to_pickle_value(value.clone()); + } else { + return Err(PytorchError::KeyNotFound(format!( + "Key '{}' not found in pickle data", + key + ))); + } + } + + object_to_pickle_value(obj) +} + +/// Convert Object to PickleValue +fn object_to_pickle_value(obj: Object) -> Result { + use crate::pytorch::pickle_reader::Object; + + Ok(match obj { + Object::None => PickleValue::None, + Object::Bool(b) => PickleValue::Bool(b), + Object::Int(i) => PickleValue::Int(i), + Object::Float(f) => PickleValue::Float(f), + Object::String(s) => PickleValue::String(s), + Object::Persistent(data) => { + // Persistent data is raw bytes + PickleValue::Bytes(data) + } + Object::PersistentTuple(tuple) => { + // Convert persistent tuples to lists + let mut values = Vec::new(); + for item in tuple { + values.push(object_to_pickle_value(item)?); + } + PickleValue::List(values) + } + Object::List(list) => { + let mut values = Vec::new(); + for item in list { + values.push(object_to_pickle_value(item)?); + } + PickleValue::List(values) + } + Object::Dict(dict) => { + let mut map = HashMap::new(); + for (k, v) in dict { + map.insert(k, object_to_pickle_value(v)?); + } + PickleValue::Dict(map) + } + Object::Tuple(tuple) => { + // Convert tuples to lists in the public API + let mut values = Vec::new(); + for item in tuple { + values.push(object_to_pickle_value(item)?); + } + PickleValue::List(values) + } + Object::TorchParam(_) => { + // Skip tensor parameters in config reading + PickleValue::None + } + Object::Class { .. } | Object::Build { .. } | Object::Reduce { .. } => { + // Complex objects are represented as None for simplicity + PickleValue::None + } + }) +} + +/// Convert PickleValue to NestedValue for deserialization +fn convert_pickle_to_nested_value(value: PickleValue) -> Result { + Ok(match value { + PickleValue::None => NestedValue::Default(None), + PickleValue::Bool(b) => NestedValue::Bool(b), + PickleValue::Int(i) => NestedValue::I64(i), + PickleValue::Float(f) => NestedValue::F64(f), + PickleValue::String(s) => NestedValue::String(s), + PickleValue::List(list) => { + let mut vec = Vec::new(); + for item in list { + vec.push(convert_pickle_to_nested_value(item)?); + } + NestedValue::Vec(vec) + } + PickleValue::Dict(dict) => { + let mut map = HashMap::new(); + for (k, v) in dict { + map.insert(k, convert_pickle_to_nested_value(v)?); + } + NestedValue::Map(map) + } + PickleValue::Bytes(data) => { + // Convert bytes to a list of u8 values + let vec: Vec = data.into_iter().map(NestedValue::U8).collect(); + NestedValue::Vec(vec) + } + }) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/store.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/store.rs new file mode 100644 index 0000000..21224a9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/store.rs @@ -0,0 +1,442 @@ +//! PyTorch store implementation for saving and loading models in PyTorch format. + +use crate::{ + ApplyResult, KeyRemapper, ModuleSnapshot, ModuleStore, PathFilter, PyTorchToBurnAdapter, + TensorSnapshot, map_indices_contiguous, +}; + +use alloc::collections::BTreeMap; + +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use burn_tensor::backend::Backend; +use core::fmt; +use std::path::PathBuf; + +use super::reader::{PytorchError as ReaderError, PytorchReader}; + +/// Errors that can occur during PyTorch operations. +#[derive(Debug)] +pub enum PytorchStoreError { + /// Reader error. + Reader(ReaderError), + + /// I/O error. + Io(std::io::Error), + + /// Tensor not found. + TensorNotFound(String), + + /// Validation failed. + ValidationFailed(String), + + /// Other error. + Other(String), +} + +impl fmt::Display for PytorchStoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Reader(e) => write!(f, "PyTorch reader error: {}", e), + Self::Io(e) => write!(f, "I/O error: {}", e), + Self::TensorNotFound(name) => write!(f, "Tensor not found: {}", name), + Self::ValidationFailed(msg) => write!(f, "Validation failed: {}", msg), + Self::Other(msg) => write!(f, "{}", msg), + } + } +} + +impl std::error::Error for PytorchStoreError {} + +impl From for PytorchStoreError { + fn from(e: ReaderError) -> Self { + PytorchStoreError::Reader(e) + } +} + +impl From for PytorchStoreError { + fn from(e: std::io::Error) -> Self { + PytorchStoreError::Io(e) + } +} + +/// PyTorch store for file-based storage only. +/// +/// This store allows loading models from PyTorch checkpoint files (.pt/.pth) +/// with automatic weight transformation using `PyTorchToBurnAdapter`. +/// Linear weights are automatically transposed and normalization parameters +/// are renamed (gamma -> weight, beta -> bias). +/// +/// Note that saving to PyTorch format is not yet supported. +pub struct PytorchStore { + pub(crate) path: PathBuf, + pub(crate) filter: PathFilter, + pub(crate) remapper: KeyRemapper, + pub(crate) validate: bool, + pub(crate) allow_partial: bool, + pub(crate) top_level_key: Option, + pub(crate) skip_enum_variants: bool, + /// Enable contiguous mapping of layer indices (default: true) + pub(crate) map_indices_contiguous: bool, + /// Cached tensor snapshots (parsed once, reused) + snapshots_cache: Option>, +} + +impl PytorchStore { + /// Create a store for loading from a PyTorch file. + /// + /// # Arguments + /// * `path` - Path to the PyTorch checkpoint file (.pt or .pth) + /// + /// # Example + /// ```rust,no_run + /// use burn_store::PytorchStore; + /// + /// let store = PytorchStore::from_file("model.pth"); + /// ``` + pub fn from_file(path: impl Into) -> Self { + Self { + path: path.into(), + filter: PathFilter::new(), + remapper: KeyRemapper::new(), + validate: true, + allow_partial: false, + top_level_key: None, + // PyTorch models never include enum variant names in paths + skip_enum_variants: true, + // Enable contiguous index mapping by default for PyTorch files + // This handles nn.Sequential models with gaps in layer indices + map_indices_contiguous: true, + snapshots_cache: None, + } + } + + /// Set a top-level key to extract tensors from. + /// + /// PyTorch files often contain nested dictionaries. Use this to extract + /// tensors from a specific top-level key like "state_dict" or "model_state_dict". + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// let store = PytorchStore::from_file("checkpoint.pth") + /// .with_top_level_key("model_state_dict"); + /// ``` + pub fn with_top_level_key(mut self, key: impl Into) -> Self { + self.top_level_key = Some(key.into()); + self + } + + /// Filter which tensors to load. + pub fn filter(mut self, filter: PathFilter) -> Self { + self.filter = filter; + self + } + + /// Add a regex pattern to filter tensors. + /// + /// Multiple patterns can be added and they work with OR logic. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// let store = PytorchStore::from_file("model.pth") + /// .with_regex(r"^encoder\..*") // Match all encoder tensors + /// .with_regex(r".*\.weight$"); // OR match any weight tensors + /// ``` + pub fn with_regex>(mut self, pattern: S) -> Self { + self.filter = self.filter.with_regex(pattern); + self + } + + /// Add multiple regex patterns to filter tensors. + pub fn with_regexes(mut self, patterns: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + self.filter = self.filter.with_regexes(patterns); + self + } + + /// Add an exact full path to match. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// let store = PytorchStore::from_file("model.pth") + /// .with_full_path("encoder.layer1.weight") + /// .with_full_path("decoder.output.bias"); + /// ``` + pub fn with_full_path>(mut self, path: S) -> Self { + self.filter = self.filter.with_full_path(path); + self + } + + /// Add multiple exact full paths to match. + pub fn with_full_paths(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.filter = self.filter.with_full_paths(paths); + self + } + + /// Add a predicate function for custom filtering logic. + /// + /// The predicate receives the tensor path and container path. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// let store = PytorchStore::from_file("model.pth") + /// .with_predicate(|path, _| path.starts_with("encoder.") || path.ends_with(".bias")); + /// ``` + pub fn with_predicate(mut self, predicate: fn(&str, &str) -> bool) -> Self { + self.filter = self.filter.with_predicate(predicate); + self + } + + /// Add multiple predicate functions. + pub fn with_predicates(mut self, predicates: I) -> Self + where + I: IntoIterator bool>, + { + self.filter = self.filter.with_predicates(predicates); + self + } + + /// Set the filter to match all paths (disables filtering). + pub fn match_all(mut self) -> Self { + self.filter = self.filter.match_all(); + self + } + + /// Remap tensor names during load. + pub fn remap(mut self, remapper: KeyRemapper) -> Self { + self.remapper = remapper; + self + } + + /// Add a regex pattern to remap tensor names during load. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// let store = PytorchStore::from_file("model.pth") + /// .with_key_remapping(r"^encoder\.", "transformer.encoder.") // encoder.X -> transformer.encoder.X + /// .with_key_remapping(r"\.gamma$", ".weight"); // X.gamma -> X.weight + /// ``` + pub fn with_key_remapping( + mut self, + from_pattern: impl AsRef, + to_pattern: impl Into, + ) -> Self { + self.remapper = self + .remapper + .add_pattern(from_pattern, to_pattern) + .expect("Invalid regex pattern"); + self + } + + /// Set whether to validate tensors during loading (default: true). + pub fn validate(mut self, validate: bool) -> Self { + self.validate = validate; + self + } + + /// Allow partial loading of tensors (continue even if some tensors are missing). + pub fn allow_partial(mut self, allow: bool) -> Self { + self.allow_partial = allow; + self + } + + /// Skip enum variant names when matching tensor paths (default: true). + /// + /// When enabled, tensor paths from PyTorch that don't include enum variants + /// can be matched against Burn module paths that do include them. + /// For example, PyTorch path "feature.weight" can match Burn path "feature.BaseConv.weight". + /// + /// This defaults to `true` for PytorchStore since PyTorch models never include + /// enum variant names in their parameter paths. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// // Disable enum variant skipping (not typical) + /// let store = PytorchStore::from_file("model.pth") + /// .skip_enum_variants(false); + /// ``` + pub fn skip_enum_variants(mut self, skip: bool) -> Self { + self.skip_enum_variants = skip; + self + } + + /// Enable or disable automatic contiguous mapping of layer indices (default: true). + /// + /// When enabled, non-contiguous numeric indices in tensor paths are renumbered + /// to be contiguous. This is useful when loading PyTorch models that have gaps + /// in layer numbering, such as when using `nn.Sequential` with mixed layer types + /// (e.g., Conv2d layers at indices 0, 2, 4 with ReLU layers at 1, 3, 5). + /// + /// # Example + /// + /// With index mapping enabled (default): + /// - `fc.0.weight` → `fc.0.weight` + /// - `fc.2.weight` → `fc.1.weight` (gap filled) + /// - `fc.4.weight` → `fc.2.weight` (gap filled) + /// + /// # Arguments + /// + /// * `map` - `true` to enable contiguous index mapping, `false` to disable + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::PytorchStore; + /// // Disable contiguous index mapping if your model already has contiguous indices + /// let store = PytorchStore::from_file("model.pth") + /// .map_indices_contiguous(false); + /// ``` + pub fn map_indices_contiguous(mut self, map: bool) -> Self { + self.map_indices_contiguous = map; + self + } + + /// Apply remapping to tensor snapshots. + fn apply_remapping(&self, snapshots: Vec) -> Vec { + if self.remapper.is_empty() { + return snapshots; + } + + let (remapped, _) = self.remapper.remap(snapshots); + remapped + } + + /// Create a PytorchReader for the configured path and options. + fn create_reader(&self) -> Result { + let reader = if let Some(ref key) = self.top_level_key { + PytorchReader::with_top_level_key(&self.path, key)? + } else { + PytorchReader::new(&self.path)? + }; + Ok(reader) + } +} + +impl ModuleStore for PytorchStore { + type Error = PytorchStoreError; + + fn collect_from>( + &mut self, + _module: &M, + ) -> Result<(), Self::Error> { + // Saving to PyTorch format is not yet supported + Err(PytorchStoreError::Other( + "Saving to PyTorch format is not yet supported. Use other formats for saving." + .to_string(), + )) + } + + fn apply_to>( + &mut self, + module: &mut M, + ) -> Result { + // Get snapshots from cache + let snapshots: Vec = self.get_all_snapshots()?.values().cloned().collect(); + + // Get filter (convert to Option for apply) + let filter_opt = if self.filter.is_empty() { + None + } else { + Some(self.filter.clone()) + }; + + // Apply to module with PyTorchToBurnAdapter (always used for PyTorch files) + // This adapter handles: + // - Transposing linear weights from PyTorch format to Burn format + // - Renaming normalization parameters (gamma -> weight, beta -> bias) + // Filter is applied here during apply, not during cache population + let result = module.apply( + snapshots, + filter_opt, + Some(Box::new(PyTorchToBurnAdapter)), + self.skip_enum_variants, + ); + + // Validate if needed + if self.validate && !result.errors.is_empty() { + return Err(PytorchStoreError::ValidationFailed(format!( + "Import errors:\n{}", + result + ))); + } + + if !self.allow_partial && !result.missing.is_empty() { + return Err(PytorchStoreError::TensorNotFound(format!("\n{}", result))); + } + + Ok(result) + } + + fn get_snapshot(&mut self, name: &str) -> Result, Self::Error> { + self.ensure_snapshots_cache()?; + Ok(self.snapshots_cache.as_ref().unwrap().get(name)) + } + + fn get_all_snapshots(&mut self) -> Result<&BTreeMap, Self::Error> { + self.ensure_snapshots_cache()?; + Ok(self.snapshots_cache.as_ref().unwrap()) + } + + fn keys(&mut self) -> Result, Self::Error> { + // Always use the cache to ensure remapping is applied consistently + Ok(self.get_all_snapshots()?.keys().cloned().collect()) + } +} + +impl PytorchStore { + /// Ensure the snapshots cache is populated + fn ensure_snapshots_cache(&mut self) -> Result<(), PytorchStoreError> { + if self.snapshots_cache.is_some() { + return Ok(()); + } + + let reader = self.create_reader()?; + + // Convert to tensor snapshots + let mut snapshots: Vec = reader + .into_tensors() + .into_iter() + .map(|(key, mut snapshot)| { + // Parse the key into path parts (split by '.') + let path_parts: Vec = key.split('.').map(|s| s.to_string()).collect(); + + // Set the path stack from the key + snapshot.path_stack = Some(path_parts); + snapshot.container_stack = None; + snapshot.tensor_id = None; + + snapshot + }) + .collect(); + + // Apply remapping (but NOT filtering - that's done at apply time) + snapshots = self.apply_remapping(snapshots); + + // Apply contiguous index mapping if enabled + // This must be done after remapping so that remapped paths are mapped + if self.map_indices_contiguous { + let (mapped, _) = map_indices_contiguous(snapshots); + snapshots = mapped; + } + + // Build cache as BTreeMap + let cache: BTreeMap = + snapshots.into_iter().map(|s| (s.full_path(), s)).collect(); + + self.snapshots_cache = Some(cache); + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/mod.rs new file mode 100644 index 0000000..b9d78f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/mod.rs @@ -0,0 +1,2 @@ +pub mod reader; +pub mod store; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_legacy_with_offsets.py b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_legacy_with_offsets.py new file mode 100644 index 0000000..4e690b0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_legacy_with_offsets.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = ["torch"] +# /// +"""Create a legacy format PyTorch file with specific storage offsets to test offset handling.""" + +import torch + +# Create tensors with known values at specific storage offsets +# This will help us verify we're reading from the correct location + +# Create a state dict with tensors that share storage +# This is common in PyTorch models (e.g., weight and transposed weight views) +state_dict = {} + +# Create a base tensor with known pattern +base_data = torch.arange(100, dtype=torch.float32) + +# tensor1: uses elements 10-19 (offset 10*4 = 40 bytes) +tensor1 = base_data[10:20].clone() +tensor1[:] = torch.arange(1.0, 1.1, 0.01)[:10] # 1.00, 1.01, 1.02, ... + +# tensor2: uses elements 30-35 (offset 30*4 = 120 bytes) +tensor2 = base_data[30:35].clone() +tensor2[:] = torch.arange(2.0, 2.5, 0.1)[:5] # 2.0, 2.1, 2.2, 2.3, 2.4 + +# tensor3: starts at beginning (offset 0) +tensor3 = base_data[:5].clone() +tensor3[:] = torch.arange(3.0, 3.5, 0.1)[:5] # 3.0, 3.1, 3.2, 3.3, 3.4 + +state_dict['tensor1'] = tensor1 +state_dict['tensor2'] = tensor2 +state_dict['tensor3'] = tensor3 + +# Save in legacy format +output_file = 'test_data/legacy_with_offsets.pt' +torch.save(state_dict, output_file, _use_new_zipfile_serialization=False) + +print(f"Created {output_file}") + +# Verify by loading +loaded = torch.load(output_file, weights_only=False) +print("\nVerification - expected values:") +for key, tensor in loaded.items(): + print(f" {key}: {tensor.tolist()}") + print(f" Storage offset: {tensor.storage_offset()}") + print(f" Storage size: {len(tensor.storage())}") + +# Also create a test with multiple tensors sharing the same storage +# This is important for proper offset handling +shared_storage = torch.randn(1000) + +# Create views into the same storage at different offsets +view1 = shared_storage[100:110] # offset 100 +view2 = shared_storage[500:520] # offset 500 +view3 = shared_storage[0:10] # offset 0 + +# Need to save these properly - PyTorch will handle the storage sharing +shared_dict = { + 'view1': view1.clone(), # Clone to avoid view issues + 'view2': view2.clone(), + 'view3': view3.clone(), +} + +output_file2 = 'test_data/legacy_shared_storage.pt' +torch.save(shared_dict, output_file2, _use_new_zipfile_serialization=False) +print(f"\nCreated {output_file2}") + +# Print exact values for test verification +print("\nExact test values for legacy_with_offsets.pt:") +print("tensor1 (10 elements starting at 1.0):") +print(" First 3 values: [1.00, 1.01, 1.02]") +print("tensor2 (5 elements starting at 2.0):") +print(" All values: [2.0, 2.1, 2.2, 2.3, 2.4]") +print("tensor3 (5 elements starting at 3.0):") +print(" All values: [3.0, 3.1, 3.2, 3.3, 3.4]") \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_tar_format.py b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_tar_format.py new file mode 100644 index 0000000..28549f7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/create_tar_format.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +""" +Create TAR format test fixtures for burn-store integration tests. + +The TAR format was used by very early versions of PyTorch (pre 0.1.10). +Modern torch.save cannot create this format, so we construct it manually. + +TAR format structure: + - sys_info: pickle with {protocol_version, little_endian, type_sizes} + - pickle: pickle with OrderedDict containing _rebuild_tensor_v2 REDUCE calls + - storages: count_pickle + for each storage: (key, device, class) pickle + u64 num_elements + raw data +""" + +import io +import pickle +import struct +import tarfile +import os +from collections import OrderedDict + + +def create_sys_info(): + """Create sys_info pickle data.""" + sys_info = { + "protocol_version": 1000, + "little_endian": True, + "type_sizes": { + "short": 2, + "int": 4, + "long": 8, + }, + } + return pickle.dumps(sys_info, protocol=2) + + +def encode_tensor_data(values: list, storage_type: str) -> tuple: + """Encode tensor values to bytes and return (bytes, element_size).""" + fmt_map = { + "FloatStorage": (" bytes: + """ + Create the storages binary blob manually. + + Args: + tensors: List of (key, storage_type, element_size, data_bytes) tuples + """ + buffer = io.BytesIO() + + # Write storage count as pickle (simple integer) + pickle.dump(len(tensors), buffer, protocol=2) + + for key, storage_type, element_size, data_bytes in tensors: + # Manually construct the tuple pickle with GLOBAL class reference + # Format: (key, "cpu", ) + + tuple_buffer = io.BytesIO() + # Protocol 2 header + tuple_buffer.write(b'\x80\x02') + + # Build tuple with MARK + items + TUPLE + tuple_buffer.write(b'(') # MARK + + # First item: storage key (string) + write_string(tuple_buffer, key) + + # Second item: device "cpu" + tuple_buffer.write(b'U\x03cpu') + + # Third item: class reference using GLOBAL + tuple_buffer.write(b'c') # GLOBAL opcode + tuple_buffer.write(b'torch\n') # module + tuple_buffer.write(storage_type.encode('ascii') + b'\n') # name + + # End tuple + tuple_buffer.write(b't') # TUPLE + tuple_buffer.write(b'.') # STOP + + buffer.write(tuple_buffer.getvalue()) + + # Write num_elements as u64 little-endian + num_elements = len(data_bytes) // element_size + buffer.write(struct.pack(" bytes: + """ + Create the main pickle containing _rebuild_tensor_v2 REDUCE calls. + + For each tensor, we need: + - GLOBAL torch._utils _rebuild_tensor_v2 + - MARK + - args tuple: (persistent_id, offset, shape, stride, requires_grad, hooks) + - TUPLE + - REDUCE + + The persistent_id is a PersistentTuple: ('storage', , key, device, num_elements) + """ + buffer = io.BytesIO() + + # Protocol 2 header + buffer.write(b'\x80\x02') + + # Build OrderedDict: GLOBAL + EMPTY_LIST + items + TUPLE + REDUCE + # OrderedDict([('name1', tensor1), ('name2', tensor2)]) + + # GLOBAL collections OrderedDict + buffer.write(b'ccollections\nOrderedDict\n') + + # Start list for items + buffer.write(b'(') # MARK + buffer.write(b']') # EMPTY_LIST + + # For each tensor, add (name, rebuilt_tensor) to the list + for name, storage_key, storage_type, shape, num_elements in tensors_info: + # Calculate stride for row-major (C) order + stride = [] + s = 1 + for dim in reversed(shape): + stride.insert(0, s) + s *= dim + + # Build inner tuple: (name, tensor_value) + buffer.write(b'(') # MARK for (name, value) tuple + + # Write name + write_string(buffer, name) + + # Now build the tensor using _rebuild_tensor_v2 REDUCE + # GLOBAL torch._utils _rebuild_tensor_v2 + buffer.write(b'ctorch._utils\n_rebuild_tensor_v2\n') + + # Build args tuple for _rebuild_tensor_v2 + # (persistent_id, offset, shape, stride, requires_grad, backward_hooks) + buffer.write(b'(') # MARK for args tuple + + # arg 0: persistent_id tuple: ('storage', class, key, device, num_elements) + # This will be converted to PersistentTuple by the reader + buffer.write(b'(') # MARK for persistent_id + + write_string(buffer, 'storage') + + # Class reference - GLOBAL torch FloatStorage + buffer.write(b'c') + buffer.write(b'torch\n') + buffer.write(storage_type.encode('ascii') + b'\n') + + # Storage key + write_string(buffer, storage_key) + + # Device + buffer.write(b'U\x03cpu') + + # num_elements + write_int(buffer, num_elements) + + buffer.write(b't') # TUPLE - end persistent_id + + # arg 1: storage offset (0) + buffer.write(b'K\x00') + + # arg 2: shape tuple + buffer.write(b'(') + for dim in shape: + write_int(buffer, dim) + buffer.write(b't') + + # arg 3: stride tuple + buffer.write(b'(') + for s_val in stride: + write_int(buffer, s_val) + buffer.write(b't') + + # arg 4: requires_grad (False) + buffer.write(b'\x89') # NEWFALSE + + # arg 5: backward_hooks (empty OrderedDict) + buffer.write(b'ccollections\nOrderedDict\n') + buffer.write(b'(') + buffer.write(b']') + buffer.write(b't') + buffer.write(b'R') # REDUCE to create empty OrderedDict + + buffer.write(b't') # TUPLE - end args tuple + + buffer.write(b'R') # REDUCE - call _rebuild_tensor_v2 with args + + buffer.write(b't') # TUPLE - end (name, tensor) tuple + + buffer.write(b'a') # APPEND to list + + buffer.write(b't') # TUPLE - wrap list in tuple for REDUCE + buffer.write(b'R') # REDUCE - call OrderedDict with the list + buffer.write(b'.') # STOP + + return buffer.getvalue() + + +def create_tar_pytorch_file(filename: str, tensors: dict, dtypes: dict): + """ + Create a TAR format PyTorch file. + + Args: + filename: Output file path + tensors: Dict of tensor_name -> (values_list, shape) + dtypes: Dict of tensor_name -> storage_type + """ + # Prepare storage data + storage_list = [] # (key, storage_type, element_size, data_bytes) + tensors_info = [] # (name, storage_key, storage_type, shape, num_elements) + + for idx, (name, (values, shape)) in enumerate(tensors.items()): + storage_key = str(idx) + storage_type = dtypes[name] + data_bytes, element_size = encode_tensor_data(values, storage_type) + num_elements = len(values) + + storage_list.append((storage_key, storage_type, element_size, data_bytes)) + tensors_info.append((name, storage_key, storage_type, shape, num_elements)) + + # Create the three main entries + sys_info_data = create_sys_info() + pickle_data = create_main_pickle_manual(tensors_info) + storages_data = create_storages_blob_manual(storage_list) + + # Write TAR archive + os.makedirs(os.path.dirname(filename) or ".", exist_ok=True) + + with tarfile.open(filename, "w") as tar: + # Add sys_info + tarinfo = tarfile.TarInfo(name="sys_info") + tarinfo.size = len(sys_info_data) + tar.addfile(tarinfo, io.BytesIO(sys_info_data)) + + # Add pickle + tarinfo = tarfile.TarInfo(name="pickle") + tarinfo.size = len(pickle_data) + tar.addfile(tarinfo, io.BytesIO(pickle_data)) + + # Add storages + tarinfo = tarfile.TarInfo(name="storages") + tarinfo.size = len(storages_data) + tar.addfile(tarinfo, io.BytesIO(storages_data)) + + size = os.path.getsize(filename) + print(f"Created {filename} ({size} bytes)") + print(f" Tensors: {list(tensors.keys())}") + + +def main(): + # Create test_data directory + os.makedirs("test_data", exist_ok=True) + + # Test 1: Single float32 tensor + create_tar_pytorch_file( + "test_data/tar_float32.tar", + {"tensor": ([1.0, 2.5, -3.7, 0.0], [4])}, + {"tensor": "FloatStorage"}, + ) + + # Test 2: Single float64 tensor + create_tar_pytorch_file( + "test_data/tar_float64.tar", + {"tensor": ([1.1, 2.2, 3.3], [3])}, + {"tensor": "DoubleStorage"}, + ) + + # Test 3: Single int64 tensor + create_tar_pytorch_file( + "test_data/tar_int64.tar", + {"tensor": ([100, -200, 300, 0], [4])}, + {"tensor": "LongStorage"}, + ) + + # Test 4: Multiple tensors (weight + bias) + create_tar_pytorch_file( + "test_data/tar_weight_bias.tar", + { + "weight": ([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], [2, 3]), + "bias": ([0.01, 0.02], [2]), + }, + { + "weight": "FloatStorage", + "bias": "FloatStorage", + }, + ) + + # Test 5: Different dtypes in one file + create_tar_pytorch_file( + "test_data/tar_multi_dtype.tar", + { + "float_tensor": ([1.5, 2.5, 3.5], [3]), + "double_tensor": ([1.111, 2.222], [2]), + "int_tensor": ([10, 20, 30, 40], [4]), + }, + { + "float_tensor": "FloatStorage", + "double_tensor": "DoubleStorage", + "int_tensor": "LongStorage", + }, + ) + + # Test 6: 2D tensor for shape verification + create_tar_pytorch_file( + "test_data/tar_2d_tensor.tar", + { + "matrix": ([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0], [3, 4]), + }, + {"matrix": "FloatStorage"}, + ) + + print("\nAll TAR format test files created!") + print("\nTo run tests: cargo test -p burn-store --features pytorch test_tar") + + +if __name__ == "__main__": + main() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/mod.rs new file mode 100644 index 0000000..734a2ba --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/mod.rs @@ -0,0 +1,1223 @@ +//! Tests for PyTorch file reader functionality +//! +//! Floating-point comparison tolerances: +//! - F16/BF16: 1e-2 (~3 decimal digits precision) +//! - F32: 1e-6 (~7 decimal digits precision) +//! - F64: 1e-10 (~16 decimal digits precision) + +use crate::pytorch::PytorchReader; +// Import internal types for testing only +use crate::pytorch::reader::{ByteOrder, FileFormat}; +use burn_tensor::DType; +use std::path::PathBuf; + +fn test_data_path(filename: &str) -> PathBuf { + // Get the path relative to the crate root + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("pytorch") + .join("tests") + .join("reader") + .join("test_data") + .join(filename) +} + +#[test] +fn test_float32_tensor() { + let path = test_data_path("float32.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load float32.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![4]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 4); + assert!((values[0] - 1.0).abs() < 1e-6); + assert!((values[1] - 2.5).abs() < 1e-6); + assert!((values[2] - (-3.7)).abs() < 1e-6); + assert!((values[3] - 0.0).abs() < 1e-6); +} + +#[test] +fn test_float64_tensor() { + let path = test_data_path("float64.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load float64.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F64); + assert_eq!(tensor.shape, vec![3]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 3); + assert!((values[0] - 1.1).abs() < 1e-10); + assert!((values[1] - 2.2).abs() < 1e-10); + assert!((values[2] - 3.3).abs() < 1e-10); +} + +#[test] +fn test_int64_tensor() { + let path = test_data_path("int64.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load int64.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::I64); + assert_eq!(tensor.shape, vec![4]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[100, -200, 300, 0]); +} + +#[test] +fn test_int32_tensor() { + let path = test_data_path("int32.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load int32.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::I32); + assert_eq!(tensor.shape, vec![3]); + + let data = tensor.to_data().unwrap(); + // Convert to the appropriate element type + let data_converted = data.convert::(); + let values = data_converted.as_slice::().unwrap(); + assert_eq!(values, &[10, 20, -30]); +} + +#[test] +fn test_int16_tensor() { + let path = test_data_path("int16.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load int16.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::I16); + assert_eq!(tensor.shape, vec![3]); + + let data = tensor.to_data().unwrap(); + let data_converted = data.convert::(); + let values = data_converted.as_slice::().unwrap(); + assert_eq!(values, &[1000, -2000, 3000]); +} + +#[test] +fn test_int8_tensor() { + let path = test_data_path("int8.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load int8.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::I8); + assert_eq!(tensor.shape, vec![4]); + + let data = tensor.to_data().unwrap(); + let data_converted = data.convert::(); + let values = data_converted.as_slice::().unwrap(); + assert_eq!(values, &[127, -128, 0, 50]); +} + +#[test] +fn test_bool_tensor() { + let path = test_data_path("bool.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load bool.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::Bool); + assert_eq!(tensor.shape, vec![5]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[true, false, true, true, false]); +} + +#[test] +fn test_uint8_tensor() { + let path = test_data_path("uint8.pt"); + + let reader = PytorchReader::new(&path).expect("Failed to load uint8.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::U8); + assert_eq!(tensor.shape, vec![4]); + + // Verify actual U8 values [0, 128, 255, 42] from test_data.py + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[0, 128, 255, 42]); +} + +#[test] +fn test_float16_tensor() { + use half::f16; + + let path = test_data_path("float16.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load float16.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F16); + assert_eq!(tensor.shape, vec![3]); + + // Verify actual F16 values [1.5, -2.25, 3.125] from test_data.py + let data = tensor.to_data().unwrap(); + assert_eq!(data.shape, vec![3]); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 3); + assert!((values[0].to_f32() - 1.5).abs() < 1e-2); + assert!((values[1].to_f32() - (-2.25)).abs() < 1e-2); + assert!((values[2].to_f32() - 3.125).abs() < 1e-2); +} + +#[test] +fn test_bfloat16_tensor() { + use half::bf16; + + let path = test_data_path("bfloat16.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load bfloat16.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::BF16); + assert_eq!(tensor.shape, vec![3]); + + // Verify actual BF16 values [1.5, -2.5, 3.5] from test_data.py + let data = tensor.to_data().unwrap(); + assert_eq!(data.shape, vec![3]); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 3); + assert!((values[0].to_f32() - 1.5).abs() < 1e-2); + assert!((values[1].to_f32() - (-2.5)).abs() < 1e-2); + assert!((values[2].to_f32() - 3.5).abs() < 1e-2); +} + +#[test] +fn test_2d_tensor() { + let path = test_data_path("tensor_2d.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load tensor_2d.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![3, 2]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 6); + // Check flattened values [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + for (i, expected) in [1.0, 2.0, 3.0, 4.0, 5.0, 6.0].iter().enumerate() { + assert!((values[i] - expected).abs() < 1e-6); + } +} + +#[test] +fn test_3d_tensor() { + let path = test_data_path("tensor_3d.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load tensor_3d.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![2, 3, 4]); + + let data = tensor.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 3, 4]); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 24); +} + +#[test] +fn test_4d_tensor() { + let path = test_data_path("tensor_4d.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load tensor_4d.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![2, 3, 2, 2]); + + let data = tensor.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 3, 2, 2]); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 24); +} + +#[test] +fn test_state_dict() { + let path = test_data_path("state_dict.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load state_dict.pt"); + let keys = reader.keys(); + + assert_eq!(keys.len(), 4); + assert!(keys.contains(&"weight".to_string())); + assert!(keys.contains(&"bias".to_string())); + assert!(keys.contains(&"running_mean".to_string())); + assert!(keys.contains(&"running_var".to_string())); + + // Check weight tensor + let weight = reader.get("weight").unwrap(); + assert_eq!(weight.shape, vec![3, 4]); + assert_eq!(weight.dtype, DType::F32); + + // Check bias tensor + let bias = reader.get("bias").unwrap(); + assert_eq!(bias.shape, vec![3]); + assert_eq!(bias.dtype, DType::F32); + + // Check running_mean (should be zeros) + let running_mean = reader.get("running_mean").unwrap(); + assert_eq!(running_mean.shape, vec![3]); + let mean_data = running_mean.to_data().unwrap(); + let mean_values = mean_data.as_slice::().unwrap(); + assert!(mean_values.iter().all(|&v| v.abs() < 1e-6)); + + // Check running_var (should be ones) + let running_var = reader.get("running_var").unwrap(); + assert_eq!(running_var.shape, vec![3]); + let var_data = running_var.to_data().unwrap(); + let var_values = var_data.as_slice::().unwrap(); + assert!(var_values.iter().all(|&v| (v - 1.0).abs() < 1e-6)); +} + +#[test] +fn test_nested_dict() { + let path = test_data_path("nested_dict.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load nested_dict.pt"); + let keys = reader.keys(); + + assert_eq!(keys.len(), 4); + assert!(keys.contains(&"layer1.weight".to_string())); + assert!(keys.contains(&"layer1.bias".to_string())); + assert!(keys.contains(&"layer2.weight".to_string())); + assert!(keys.contains(&"layer2.bias".to_string())); + + // Check layer1.weight and load data + let layer1_weight = reader.get("layer1.weight").unwrap(); + assert_eq!(layer1_weight.shape, vec![2, 3]); + assert_eq!(layer1_weight.dtype, DType::F32); + let data = layer1_weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 6); // 2x3 = 6 elements + + // Check layer2.weight and load data + let layer2_weight = reader.get("layer2.weight").unwrap(); + assert_eq!(layer2_weight.shape, vec![4, 2]); + assert_eq!(layer2_weight.dtype, DType::F32); + let data = layer2_weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 8); // 4x2 = 8 elements +} + +#[test] +fn test_checkpoint() { + let path = test_data_path("checkpoint.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load checkpoint.pt"); + let keys = reader.keys(); + + // Should have model_state_dict entries and optimizer entries + assert!(keys.contains(&"model_state_dict.fc1.weight".to_string())); + assert!(keys.contains(&"model_state_dict.fc1.bias".to_string())); + assert!(keys.contains(&"model_state_dict.fc2.weight".to_string())); + assert!(keys.contains(&"model_state_dict.fc2.bias".to_string())); + + // Check fc1.weight dimensions and load data + let fc1_weight = reader.get("model_state_dict.fc1.weight").unwrap(); + assert_eq!(fc1_weight.shape, vec![10, 5]); + let data = fc1_weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 50); // 10x5 = 50 elements + + // Check fc2.weight dimensions and load data + let fc2_weight = reader.get("model_state_dict.fc2.weight").unwrap(); + assert_eq!(fc2_weight.shape, vec![3, 10]); + let data = fc2_weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 30); // 3x10 = 30 elements +} + +#[test] +fn test_empty_tensor() { + let path = test_data_path("empty.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load empty.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.shape, vec![0]); // Empty tensor has shape [0] + assert_eq!(tensor.dtype, DType::F32); + + // Note: Empty tensors cannot be loaded with to_data() due to TensorData validation + // We verify the metadata is correct, which confirms the .pt file is being read +} + +#[test] +fn test_scalar_tensor() { + let path = test_data_path("scalar.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load scalar.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.shape, Vec::::new()); // Scalar has empty shape + assert_eq!(tensor.dtype, DType::F32); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 1); + assert!((values[0] - 42.0).abs() < 1e-6); +} + +#[test] +fn test_large_shape() { + let path = test_data_path("large_shape.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load large_shape.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.shape, vec![100, 100]); + assert_eq!(tensor.dtype, DType::F32); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 10000); + + // Check specific non-zero values + assert!((values[0] - 1.0).abs() < 1e-6); // [0, 0] = 1.0 + assert!((values[5050] - 2.0).abs() < 1e-6); // [50, 50] = 2.0 + assert!((values[9999] - 3.0).abs() < 1e-6); // [99, 99] = 3.0 +} + +#[test] +fn test_mixed_types() { + let path = test_data_path("mixed_types.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load mixed_types.pt"); + let tensors = reader.tensors(); + + assert_eq!(tensors.len(), 4); + + // Check float32 tensor [1.0, 2.0] from test_data.py + let float32 = reader.get("float32").unwrap(); + assert_eq!(float32.dtype, DType::F32); + assert_eq!(float32.shape, vec![2]); + let data = float32.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert!((values[0] - 1.0).abs() < 1e-6); + assert!((values[1] - 2.0).abs() < 1e-6); + + // Check int64 tensor [100, 200] from test_data.py + let int64 = reader.get("int64").unwrap(); + assert_eq!(int64.dtype, DType::I64); + assert_eq!(int64.shape, vec![2]); + let data = int64.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[100, 200]); + + // Check bool tensor [True, False] from test_data.py + let bool_tensor = reader.get("bool").unwrap(); + assert_eq!(bool_tensor.dtype, DType::Bool); + assert_eq!(bool_tensor.shape, vec![2]); + let data = bool_tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[true, false]); + + // Check float64 tensor [1.1, 2.2] from test_data.py + let float64 = reader.get("float64").unwrap(); + assert_eq!(float64.dtype, DType::F64); + assert_eq!(float64.shape, vec![2]); + let data = float64.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert!((values[0] - 1.1).abs() < 1e-10); + assert!((values[1] - 2.2).abs() < 1e-10); +} + +#[test] +fn test_special_values() { + let path = test_data_path("special_values.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load special_values.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![5]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 5); + + // Check for special values + assert!(values[0].is_nan()); + assert!(values[1].is_infinite() && values[1] > 0.0); + assert!(values[2].is_infinite() && values[2] < 0.0); + assert!((values[3] - 0.0).abs() < 1e-6); + assert!((values[4] - 1.0).abs() < 1e-6); +} + +#[test] +fn test_extreme_values() { + let path = test_data_path("extreme_values.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load extreme_values.pt"); + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![4]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 4); + + // Very small positive + assert!(values[0] > 0.0 && values[0] < 1e-20); + // Very large positive + assert!(values[1] > 1e20); + // Very small negative + assert!(values[2] < 0.0 && values[2] > -1e-20); + // Very large negative + assert!(values[3] < -1e20); +} + +#[test] +fn test_parameter() { + let path = test_data_path("parameter.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load parameter.pt"); + let tensors = reader.tensors(); + + // nn.Parameter is typically saved as a regular tensor + assert_eq!(tensors.len(), 1); + let param = reader.get("param").unwrap(); + assert_eq!(param.shape, vec![3, 3]); + assert_eq!(param.dtype, DType::F32); + + let data = param.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 9); +} + +#[test] +fn test_buffers() { + let path = test_data_path("buffers.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load buffers.pt"); + let tensors = reader.tensors(); + + assert_eq!(tensors.len(), 2); + + // Check buffer1 (int32) + let buffer1 = reader.get("buffer1").unwrap(); + assert_eq!(buffer1.dtype, DType::I32); + assert_eq!(buffer1.shape, vec![3]); + let data1 = buffer1.to_data().unwrap(); + let data1_converted = data1.convert::(); + let values1 = data1_converted.as_slice::().unwrap(); + assert_eq!(values1, &[1, 2, 3]); + + // Check buffer2 (bool) + let buffer2 = reader.get("buffer2").unwrap(); + assert_eq!(buffer2.dtype, DType::Bool); + assert_eq!(buffer2.shape, vec![2]); + let data2 = buffer2.to_data().unwrap(); + let values2 = data2.as_slice::().unwrap(); + assert_eq!(values2, &[true, false]); +} + +#[test] +fn test_complex_structure() { + let path = test_data_path("complex_structure.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load complex_structure.pt"); + let keys = reader.keys(); + + // Should have nested structure tensors + assert!(keys.contains(&"state.encoder.layer_0.weight".to_string())); + assert!(keys.contains(&"state.encoder.layer_0.bias".to_string())); + assert!(keys.contains(&"state.encoder.layer_1.weight".to_string())); + assert!(keys.contains(&"state.encoder.layer_1.bias".to_string())); + assert!(keys.contains(&"state.decoder.weight".to_string())); + assert!(keys.contains(&"state.decoder.bias".to_string())); + + // Check encoder layer_0 weight and load data + let layer0_weight = reader.get("state.encoder.layer_0.weight").unwrap(); + assert_eq!(layer0_weight.shape, vec![4, 3]); + let data = layer0_weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 12); // 4x3 = 12 elements + + // Check decoder weight and load data + let decoder_weight = reader.get("state.decoder.weight").unwrap(); + assert_eq!(decoder_weight.shape, vec![3, 2]); + let data = decoder_weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 6); // 3x2 = 6 elements +} + +#[test] +fn test_read_pytorch_tensors_convenience() { + // Test reading and materializing tensors into memory + let path = test_data_path("state_dict.pt"); + let reader = PytorchReader::new(&path).expect("Failed to read file"); + + let keys = reader.keys(); + assert_eq!(keys.len(), 4); + assert!(keys.contains(&"weight".to_string())); + assert!(keys.contains(&"bias".to_string())); + + // Check that data can be materialized + let weight = reader.get("weight").unwrap(); + let weight_data = weight.to_data().unwrap(); + assert_eq!(weight_data.shape, vec![3, 4]); + assert_eq!(weight_data.dtype, DType::F32); +} + +#[test] +fn test_with_top_level_key() { + // Test loading with a specific top-level key + let path = test_data_path("checkpoint.pt"); + + // Load only model_state_dict + let reader = PytorchReader::with_top_level_key(&path, "model_state_dict") + .expect("Failed to load with top-level key"); + + let keys = reader.keys(); + // Should only have model weights, not optimizer state + assert!(keys.contains(&"fc1.weight".to_string())); + assert!(keys.contains(&"fc1.bias".to_string())); + assert!(keys.contains(&"fc2.weight".to_string())); + assert!(keys.contains(&"fc2.bias".to_string())); + + // Should NOT have nested paths with model_state_dict prefix + assert!(!keys.contains(&"model_state_dict.fc1.weight".to_string())); +} + +#[test] +fn test_legacy_format() { + // Test loading PyTorch legacy format (pre-1.6) + let path = test_data_path("simple_legacy.pt"); + + // This file has the sequential pickle structure of legacy PyTorch format + let reader = PytorchReader::new(&path).expect("Failed to load legacy format"); + let keys = reader.keys(); + + // Should have the tensors from the state dict + assert!(keys.contains(&"weight".to_string()), "Missing 'weight' key"); + assert!(keys.contains(&"bias".to_string()), "Missing 'bias' key"); + assert!( + keys.contains(&"running_mean".to_string()), + "Missing 'running_mean' key" + ); + + // Check weight tensor + let weight = reader.get("weight").expect("weight not found"); + assert_eq!(weight.shape, vec![2, 3]); + assert_eq!(weight.dtype, DType::F32); + + // Check bias tensor + let bias = reader.get("bias").expect("bias not found"); + assert_eq!(bias.shape, vec![2]); + assert_eq!(bias.dtype, DType::F32); + + // Verify bias values are all ones + let bias_data = bias.to_data().unwrap(); + let bias_values = bias_data.as_slice::().unwrap(); + // Note: values in simple_legacy.pt are randomly generated, not necessarily 1.0 + assert_eq!(bias_values.len(), 2); + + // Check running_mean tensor + let running_mean = reader.get("running_mean").expect("running_mean not found"); + assert_eq!(running_mean.shape, vec![2]); + assert_eq!(running_mean.dtype, DType::F32); + + // Verify running_mean values are accessible + let mean_data = running_mean.to_data().unwrap(); + let mean_values = mean_data.as_slice::().unwrap(); + assert_eq!(mean_values.len(), 2); +} + +#[test] +fn test_legacy_with_offsets() { + // Test with legacy format file that has storage offsets + let path = test_data_path("legacy_with_offsets.pt"); + let reader = PytorchReader::new(&path).expect("Should read legacy file with offsets"); + + let keys = reader.keys(); + assert_eq!(keys.len(), 3, "Should have 3 tensors"); + + // Check that tensors exist + for key in &keys { + assert!(reader.get(key).is_some(), "Should have tensor: {}", key); + let tensor = reader.get(key).unwrap(); + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert!(!values.is_empty(), "Tensor {} should have data", key); + } +} + +#[test] +fn test_legacy_shared_storage() { + // Test with legacy format file that has shared storage + let path = test_data_path("legacy_shared_storage.pt"); + let reader = PytorchReader::new(&path).expect("Should read legacy file with shared storage"); + + let keys = reader.keys(); + assert!(keys.len() >= 2, "Should have at least 2 tensors"); + + // Check that tensors exist and can be loaded + for key in &keys { + assert!(reader.get(key).is_some(), "Should have tensor: {}", key); + let tensor = reader.get(key).unwrap(); + let data = tensor.to_data().unwrap(); + + // Verify tensor data can be accessed + match tensor.dtype { + DType::F32 => { + let values = data.as_slice::().unwrap(); + assert!(!values.is_empty(), "Tensor {} should have data", key); + } + DType::I64 => { + let values = data.as_slice::().unwrap(); + assert!(!values.is_empty(), "Tensor {} should have data", key); + } + _ => { + // For other types, just verify we can convert to data + assert!(!data.shape.is_empty(), "Tensor {} should have shape", key); + } + } + } +} + +#[test] +fn test_metadata_zip_format() { + // Test that metadata is properly populated for ZIP format files + let path = test_data_path("float32.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load float32.pt"); + + // Check metadata + let metadata = reader.metadata(); + assert_eq!(metadata.format_type, FileFormat::Zip); + assert_eq!(metadata.byte_order, ByteOrder::LittleEndian); + assert_eq!(metadata.tensor_count, 1); + assert!(metadata.total_data_size.is_some()); + + // Check that metadata is accessible + assert!(metadata.is_modern_format()); + assert!(!metadata.is_legacy_format()); +} + +#[test] +fn test_metadata_legacy_format() { + // Test that metadata is properly populated for legacy format files + let path = test_data_path("simple_legacy.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load legacy file"); + + // Check metadata + let metadata = reader.metadata(); + assert_eq!(metadata.format_type, FileFormat::Legacy); + assert_eq!(metadata.byte_order, ByteOrder::LittleEndian); + assert_eq!(metadata.tensor_count, 3); // weight, bias, running_mean + assert!(metadata.total_data_size.is_some()); +} + +#[test] +fn test_legacy_metadata_detailed() { + // Detailed test to prove we load all metadata for legacy format files + let path = test_data_path("simple_legacy.pt"); + let reader = PytorchReader::new(&path).expect("Failed to load legacy file"); + + // Get and examine metadata + let metadata = reader.metadata(); + + // Verify the metadata is correct for legacy format + assert_eq!( + metadata.format_type, + FileFormat::Legacy, + "Should be Legacy format" + ); + assert_eq!( + metadata.byte_order, + ByteOrder::LittleEndian, + "Legacy format is little-endian" + ); + assert_eq!( + metadata.tensor_count, 3, + "Should have 3 tensors: weight, bias, running_mean" + ); + assert!( + metadata.total_data_size.is_some(), + "Should have total data size" + ); + assert!( + metadata.total_data_size.unwrap() > 0, + "Data size should be positive" + ); + + // Legacy format specifics + assert_eq!( + metadata.format_version, None, + "Legacy format doesn't have version file" + ); + assert_eq!( + metadata.pytorch_version, None, + "Legacy format doesn't store PyTorch version reliably" + ); + assert!( + !metadata.has_storage_alignment, + "Legacy format doesn't have storage alignment" + ); + + // Also verify we can access the tensors + let keys = reader.keys(); + assert!( + keys.contains(&"weight".to_string()), + "Should have weight tensor" + ); + assert!( + keys.contains(&"bias".to_string()), + "Should have bias tensor" + ); + assert!( + keys.contains(&"running_mean".to_string()), + "Should have running_mean tensor" + ); +} + +#[test] +fn test_small_invalid_file() { + // Test that we handle broken/invalid files gracefully + let path = test_data_path("broken.pt"); + + // Should fail gracefully with an appropriate error + let result = PytorchReader::new(&path); + assert!(result.is_err(), "Expected error for broken file"); + + // The error should be a pickle error since the file is too small to be valid + if let Err(e) = result { + let err_str = format!("{}", e); + assert!( + err_str.contains("Pickle") || err_str.contains("Invalid"), + "Error should mention pickle or invalid format: {}", + err_str + ); + } +} + +#[test] +fn test_read_pickle_data_basic() { + use crate::pytorch::reader::PickleValue; + + // Test reading pickle data from a checkpoint file + let path = test_data_path("checkpoint.pt"); + + // Read the entire pickle data + let data = PytorchReader::read_pickle_data(&path, None).expect("Failed to read pickle data"); + + // Should be a dictionary at the root + if let PickleValue::Dict(dict) = data { + // Check that expected keys exist + assert!(dict.contains_key("model_state_dict")); + assert!(dict.contains_key("optimizer_state_dict")); + assert!(dict.contains_key("epoch")); + assert!(dict.contains_key("loss")); + + // Check epoch value + if let Some(PickleValue::Int(epoch)) = dict.get("epoch") { + assert_eq!(*epoch, 42); + } else { + panic!("Expected epoch to be an integer"); + } + + // Check loss value + if let Some(PickleValue::Float(loss)) = dict.get("loss") { + assert!(*loss > 0.0 && *loss < 1.0, "Loss should be between 0 and 1"); + } else { + panic!("Expected loss to be a float"); + } + } else { + panic!("Expected root to be a dictionary"); + } +} + +#[test] +fn test_read_pickle_data_with_key() { + use crate::pytorch::reader::PickleValue; + + // Test reading specific key from checkpoint + let path = test_data_path("checkpoint.pt"); + + // Read only the model_state_dict + let data = PytorchReader::read_pickle_data(&path, Some("model_state_dict")) + .expect("Failed to read pickle data with key"); + + // Should get the model_state_dict directly + if let PickleValue::Dict(dict) = data { + // Should have model weights + assert!(dict.contains_key("fc1.weight")); + assert!(dict.contains_key("fc1.bias")); + assert!(dict.contains_key("fc2.weight")); + assert!(dict.contains_key("fc2.bias")); + + // Should NOT have optimizer keys + assert!(!dict.contains_key("optimizer_state_dict")); + assert!(!dict.contains_key("epoch")); + } else { + panic!("Expected model_state_dict to be a dictionary"); + } +} + +#[test] +fn test_read_pickle_data_nested_structure() { + use crate::pytorch::reader::PickleValue; + + // Test reading nested dictionary structure + let path = test_data_path("nested_dict.pt"); + + let data = + PytorchReader::read_pickle_data(&path, None).expect("Failed to read nested structure"); + + if let PickleValue::Dict(dict) = data { + // nested_dict.pt has a nested structure, not flat keys + // It should have layer1 and layer2 as nested dicts + assert!(!dict.is_empty(), "Dictionary should not be empty"); + + // The structure depends on how the file was saved + // It could be flat keys like "layer1.weight" or nested dicts + // Just verify it's a valid dict structure + for (_key, value) in dict.iter() { + // Values could be None (tensors), nested dicts, or other types + assert!( + matches!(value, PickleValue::None | PickleValue::Dict(_)), + "Values should be None or nested dicts" + ); + } + } else { + panic!("Expected nested_dict to be a dictionary"); + } +} + +#[test] +fn test_read_pickle_data_types() { + use crate::pytorch::reader::PickleValue; + + // Test various data types in mixed_types.pt + let path = test_data_path("mixed_types.pt"); + + let data = PytorchReader::read_pickle_data(&path, None).expect("Failed to read mixed types"); + + if let PickleValue::Dict(dict) = data { + // The file contains different tensor types + assert!(dict.len() >= 3, "Should have at least 3 tensor types"); + + // All tensor values should be None in pickle data + for (_key, value) in dict.iter() { + // All values should be None (tensors are not included in pickle data) + assert!( + matches!(value, PickleValue::None), + "Tensors should be None in pickle data" + ); + } + } else { + panic!("Expected mixed_types to be a dictionary"); + } +} + +#[test] +fn test_read_pickle_data_key_not_found() { + // Test error handling when key doesn't exist + let path = test_data_path("checkpoint.pt"); + + let result = PytorchReader::read_pickle_data(&path, Some("nonexistent_key")); + assert!(result.is_err()); + + if let Err(e) = result { + let err_str = format!("{}", e); + assert!( + err_str.contains("not found"), + "Error should mention key not found: {}", + err_str + ); + } +} + +#[test] +fn test_read_pickle_data_simple_pickle() { + use crate::pytorch::reader::PickleValue; + + // Test reading a simple pickle file (not ZIP) + // Note: simple_legacy.pt is a legacy format file, not a simple pickle + // It may return None because legacy format reading is different + let path = test_data_path("state_dict.pt"); // Use a proper simple pickle file + + let data = PytorchReader::read_pickle_data(&path, None).expect("Failed to read simple pickle"); + + // Should contain state dict entries + if let PickleValue::Dict(dict) = data { + // state_dict.pt has weight, bias, running_mean, running_var + assert!(dict.len() >= 3); + assert!(dict.contains_key("weight")); + assert!(dict.contains_key("bias")); + + // All tensor values should be None in pickle data + for (_key, value) in dict.iter() { + assert!(matches!(value, PickleValue::None)); + } + } else { + panic!("Expected state_dict to contain a dictionary"); + } +} + +#[test] +fn test_load_config_basic() { + let path = test_data_path("checkpoint.pt"); + + // Define a struct that matches part of the checkpoint data + #[derive(Debug, serde::Deserialize, PartialEq)] + struct CheckpointConfig { + epoch: i64, + loss: f64, + } + + // Load config + let config: CheckpointConfig = + PytorchReader::load_config(&path, None).expect("Failed to load config"); + + // Verify values - based on test_read_pickle_data_basic + assert_eq!(config.epoch, 42); + assert!((config.loss - 0.123).abs() < 1e-6); +} + +#[test] +fn test_load_config_with_top_level_key() { + // Test that we can extract a non-existent key and get an appropriate error + let path = test_data_path("checkpoint.pt"); + + #[derive(Debug, serde::Deserialize, PartialEq)] + struct DummyConfig { + field: String, + } + + // Try loading with a valid top-level key that exists but has wrong structure + let result: Result = PytorchReader::load_config(&path, Some("epoch")); + + // This should fail because epoch is an integer, not a struct with a field + assert!(result.is_err()); + + // Now test that we can load with a real key that has the right structure + // Since checkpoint.pt doesn't have nested configs, let's use nested_dict.pt + let path2 = test_data_path("nested_dict.pt"); + + // Try to extract a specific nested key if it exists + // Since nested_dict has complex structure, let's just verify we can read it + let data = PytorchReader::read_pickle_data(&path2, None).unwrap(); + + // Verify it's a dict + if let crate::pytorch::reader::PickleValue::Dict(dict) = data { + assert!(!dict.is_empty()); + } else { + panic!("Expected a dict"); + } +} + +#[test] +fn test_load_config_complex_types() { + // For this test, let's create a comprehensive test using checkpoint.pt + // which has both metadata and state_dict fields + let path = test_data_path("checkpoint.pt"); + + // Define a partial config that only captures metadata fields + #[derive(Debug, serde::Deserialize, PartialEq)] + struct PartialCheckpoint { + epoch: i64, + loss: f64, + // We skip model_state_dict and optimizer_state_dict + // as they contain tensor references that become None + } + + // Load partial config + let config: PartialCheckpoint = + PytorchReader::load_config(&path, None).expect("Failed to load config"); + + // Verify we can extract the metadata + assert_eq!(config.epoch, 42); + assert!((config.loss - 0.123).abs() < 1e-6); +} + +#[test] +fn test_load_config_key_not_found() { + let path = test_data_path("checkpoint.pt"); + + #[derive(Debug, serde::Deserialize)] + struct DummyConfig { + #[allow(dead_code)] + field: String, + } + + // Try to load with non-existent key + let result: Result = PytorchReader::load_config(&path, Some("nonexistent")); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error.to_string().contains("not found") || error.to_string().contains("Key")); +} + +#[test] +fn test_pickle_value_conversion() { + use crate::pytorch::reader::PickleValue; + + // Test that PickleValue provides useful data structures + let path = test_data_path("checkpoint.pt"); + let data = PytorchReader::read_pickle_data(&path, None).unwrap(); + + // Test pattern matching and data extraction + match data { + PickleValue::Dict(dict) => { + // Extract epoch as integer + if let Some(PickleValue::Int(epoch)) = dict.get("epoch") { + assert!(*epoch >= 0); + } + + // Extract loss as float + if let Some(PickleValue::Float(loss)) = dict.get("loss") { + assert!(loss.is_finite()); + } + + // Test nested access + if let Some(PickleValue::Dict(model_dict)) = dict.get("model_state_dict") { + assert!(!model_dict.is_empty()); + } + } + _ => panic!("Unexpected root type"), + } +} + +// ============================================================================ +// TAR Format Tests +// ============================================================================ +// The TAR format was used by very early versions of PyTorch (pre 0.1.10). +// These tests verify that we can correctly load models saved in this format. + +#[test] +fn test_tar_format_detection() { + // Test that is_tar_file correctly detects TAR files + let tar_path = test_data_path("tar_float32.tar"); + let zip_path = test_data_path("float32.pt"); + + // TAR file should be detected as TAR + let reader = PytorchReader::new(&tar_path).expect("Failed to load TAR file"); + let metadata = reader.metadata(); + assert_eq!(metadata.format_type, FileFormat::Tar); + + // ZIP file should NOT be detected as TAR + let reader = PytorchReader::new(&zip_path).expect("Failed to load ZIP file"); + let metadata = reader.metadata(); + assert_ne!(metadata.format_type, FileFormat::Tar); +} + +#[test] +fn test_tar_float32_tensor() { + let path = test_data_path("tar_float32.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_float32.tar"); + + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F32); + assert_eq!(tensor.shape, vec![4]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 4); + assert!((values[0] - 1.0).abs() < 1e-6); + assert!((values[1] - 2.5).abs() < 1e-6); + assert!((values[2] - (-3.7)).abs() < 1e-6); + assert!((values[3] - 0.0).abs() < 1e-6); +} + +#[test] +fn test_tar_float64_tensor() { + let path = test_data_path("tar_float64.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_float64.tar"); + + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::F64); + assert_eq!(tensor.shape, vec![3]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 3); + assert!((values[0] - 1.1).abs() < 1e-10); + assert!((values[1] - 2.2).abs() < 1e-10); + assert!((values[2] - 3.3).abs() < 1e-10); +} + +#[test] +fn test_tar_int64_tensor() { + let path = test_data_path("tar_int64.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_int64.tar"); + + let tensor = reader.get("tensor").expect("tensor key not found"); + assert_eq!(tensor.dtype, DType::I64); + assert_eq!(tensor.shape, vec![4]); + + let data = tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[100, -200, 300, 0]); +} + +#[test] +fn test_tar_multiple_tensors() { + // Test loading multiple tensors (weight + bias) with correct shapes + let path = test_data_path("tar_weight_bias.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_weight_bias.tar"); + + // Check weight tensor (2x3 matrix) + let weight = reader.get("weight").expect("weight key not found"); + assert_eq!(weight.dtype, DType::F32); + assert_eq!(weight.shape, vec![2, 3]); + + let data = weight.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 6); + assert!((values[0] - 0.1).abs() < 1e-6); + assert!((values[1] - 0.2).abs() < 1e-6); + assert!((values[5] - 0.6).abs() < 1e-6); + + // Check bias tensor (2-element vector) + let bias = reader.get("bias").expect("bias key not found"); + assert_eq!(bias.dtype, DType::F32); + assert_eq!(bias.shape, vec![2]); + + let data = bias.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 2); + assert!((values[0] - 0.01).abs() < 1e-6); + assert!((values[1] - 0.02).abs() < 1e-6); +} + +#[test] +fn test_tar_multi_dtype() { + // Test loading different dtypes from the same TAR file + let path = test_data_path("tar_multi_dtype.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_multi_dtype.tar"); + + // Float32 tensor + let float_tensor = reader + .get("float_tensor") + .expect("float_tensor key not found"); + assert_eq!(float_tensor.dtype, DType::F32); + let data = float_tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert!((values[0] - 1.5).abs() < 1e-6); + + // Float64 tensor + let double_tensor = reader + .get("double_tensor") + .expect("double_tensor key not found"); + assert_eq!(double_tensor.dtype, DType::F64); + let data = double_tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert!((values[0] - 1.111).abs() < 1e-10); + + // Int64 tensor + let int_tensor = reader.get("int_tensor").expect("int_tensor key not found"); + assert_eq!(int_tensor.dtype, DType::I64); + let data = int_tensor.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values, &[10, 20, 30, 40]); +} + +#[test] +fn test_tar_2d_tensor_shape() { + // Test that 2D tensor shapes are correctly preserved + let path = test_data_path("tar_2d_tensor.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_2d_tensor.tar"); + + let matrix = reader.get("matrix").expect("matrix key not found"); + assert_eq!(matrix.dtype, DType::F32); + assert_eq!(matrix.shape, vec![3, 4]); // 3 rows, 4 columns + + let data = matrix.to_data().unwrap(); + let values = data.as_slice::().unwrap(); + assert_eq!(values.len(), 12); + + // Verify values in row-major order + for i in 0..12 { + assert!((values[i] - (i as f32 + 1.0)).abs() < 1e-6); + } +} + +#[test] +fn test_tar_metadata() { + // Test that TAR metadata is correctly populated + let path = test_data_path("tar_float32.tar"); + let reader = PytorchReader::new(&path).expect("Failed to load tar_float32.tar"); + + let metadata = reader.metadata(); + assert_eq!(metadata.format_type, FileFormat::Tar); + assert_eq!(metadata.byte_order, ByteOrder::LittleEndian); + assert_eq!(metadata.tensor_count, 1); + assert!(metadata.total_data_size.is_some()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/simple_legacy.py b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/simple_legacy.py new file mode 100644 index 0000000..662f594 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/simple_legacy.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = ["torch"] +# /// +"""Create a simple legacy format PyTorch file.""" + +import torch + +# Create a simple state dict +state_dict = { + 'weight': torch.randn(2, 3), + 'bias': torch.ones(2), + 'running_mean': torch.zeros(2), +} + +# Save without using zip format (legacy format) +torch.save(state_dict, 'test_data/simple_legacy.pt', _use_new_zipfile_serialization=False) + +print("Created simple_legacy.pt") + +# Verify +loaded = torch.load('test_data/simple_legacy.pt', weights_only=False) +print(f"Loaded {len(loaded)} tensors") +for key, val in loaded.items(): + print(f" {key}: shape {val.shape}, dtype {val.dtype}") \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data.py b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data.py new file mode 100644 index 0000000..6020590 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = ["torch", "numpy"] +# /// +""" +Generate test PyTorch .pt files for testing the burn-store PyTorch reader. +Run with: uv run test_files.py +""" + +import torch +import numpy as np +import os +from pathlib import Path + +# Create test directory +test_dir = Path(__file__).parent / "test_data" +test_dir.mkdir(exist_ok=True) + +def save_test_file(filename, data, description): + """Save a test file and print what was saved.""" + filepath = test_dir / filename + torch.save(data, filepath) + print(f"✓ {filename}: {description}") + return filepath + +# Test 1: Simple tensors of different types +print("\n=== Generating Basic Tensor Tests ===") + +# Float32 tensor (wrap in dict for compatibility) +float32_tensor = torch.tensor([1.0, 2.5, -3.7, 0.0], dtype=torch.float32) +save_test_file("float32.pt", {"tensor": float32_tensor}, "Float32 tensor [1.0, 2.5, -3.7, 0.0]") + +# Float64 tensor +float64_tensor = torch.tensor([1.1, 2.2, 3.3], dtype=torch.float64) +save_test_file("float64.pt", {"tensor": float64_tensor}, "Float64 tensor [1.1, 2.2, 3.3]") + +# Int64 tensor +int64_tensor = torch.tensor([100, -200, 300, 0], dtype=torch.int64) +save_test_file("int64.pt", {"tensor": int64_tensor}, "Int64 tensor [100, -200, 300, 0]") + +# Int32 tensor +int32_tensor = torch.tensor([10, 20, -30], dtype=torch.int32) +save_test_file("int32.pt", {"tensor": int32_tensor}, "Int32 tensor [10, 20, -30]") + +# Int16 tensor +int16_tensor = torch.tensor([1000, -2000, 3000], dtype=torch.int16) +save_test_file("int16.pt", {"tensor": int16_tensor}, "Int16 tensor [1000, -2000, 3000]") + +# Int8 tensor +int8_tensor = torch.tensor([127, -128, 0, 50], dtype=torch.int8) +save_test_file("int8.pt", {"tensor": int8_tensor}, "Int8 tensor [127, -128, 0, 50]") + +# Boolean tensor +bool_tensor = torch.tensor([True, False, True, True, False], dtype=torch.bool) +save_test_file("bool.pt", {"tensor": bool_tensor}, "Bool tensor [True, False, True, True, False]") + +# Float16 tensor (half precision) +float16_tensor = torch.tensor([1.5, -2.25, 3.125], dtype=torch.float16) +save_test_file("float16.pt", {"tensor": float16_tensor}, "Float16 tensor [1.5, -2.25, 3.125]") + +# BFloat16 tensor +bfloat16_tensor = torch.tensor([1.5, -2.5, 3.5], dtype=torch.bfloat16) +save_test_file("bfloat16.pt", {"tensor": bfloat16_tensor}, "BFloat16 tensor [1.5, -2.5, 3.5]") + +# UInt8 tensor +uint8_tensor = torch.tensor([0, 128, 255, 42], dtype=torch.uint8) +save_test_file("uint8.pt", {"tensor": uint8_tensor}, "UInt8 tensor [0, 128, 255, 42]") + +# Test 2: Multi-dimensional tensors +print("\n=== Generating Multi-dimensional Tensor Tests ===") + +# 2D tensor +tensor_2d = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=torch.float32) +save_test_file("tensor_2d.pt", {"tensor": tensor_2d}, "2D tensor shape (3, 2)") + +# 3D tensor +torch.manual_seed(42) +tensor_3d = torch.randn(2, 3, 4) * 10 +save_test_file("tensor_3d.pt", {"tensor": tensor_3d}, "3D tensor shape (2, 3, 4)") + +# 4D tensor (common for conv weights) +tensor_4d = torch.randn(2, 3, 2, 2) +save_test_file("tensor_4d.pt", {"tensor": tensor_4d}, "4D tensor shape (2, 3, 2, 2)") + +# Test 3: State dict (multiple tensors) +print("\n=== Generating State Dict Tests ===") + +state_dict = { + "weight": torch.randn(3, 4), + "bias": torch.randn(3), + "running_mean": torch.zeros(3), + "running_var": torch.ones(3), +} +save_test_file("state_dict.pt", state_dict, "State dict with 4 tensors") + +# Nested state dict +nested_dict = { + "layer1": { + "weight": torch.randn(2, 3), + "bias": torch.randn(2) + }, + "layer2": { + "weight": torch.randn(4, 2), + "bias": torch.randn(4) + } +} +save_test_file("nested_dict.pt", nested_dict, "Nested state dict") + +# Test 4: Model checkpoint format +print("\n=== Generating Model Checkpoint Tests ===") + +# Typical checkpoint format (use string keys for compatibility) +checkpoint = { + "model_state_dict": { + "fc1.weight": torch.randn(10, 5), + "fc1.bias": torch.randn(10), + "fc2.weight": torch.randn(3, 10), + "fc2.bias": torch.randn(3), + }, + "optimizer_state_dict": { + "state": { + "0": { # Use string key instead of integer + "momentum_buffer": torch.randn(10, 5) + } + } + }, + "epoch": 42, + "loss": 0.123 +} +save_test_file("checkpoint.pt", checkpoint, "Full checkpoint with model and optimizer state") + +# Test 5: Edge cases +print("\n=== Generating Edge Case Tests ===") + +# Empty tensor (1D with 0 elements) +empty_tensor = torch.zeros(0) +save_test_file("empty.pt", {"tensor": empty_tensor}, "Empty tensor") + +# Scalar tensor (0-dimensional) +scalar_tensor = torch.tensor(42.0) +save_test_file("scalar.pt", {"tensor": scalar_tensor}, "Scalar tensor (0-dim)") + +# Large shape but small data (testing shape vs actual data) +sparse_like = torch.zeros(100, 100) +sparse_like[0, 0] = 1.0 +sparse_like[50, 50] = 2.0 +sparse_like[99, 99] = 3.0 +save_test_file("large_shape.pt", {"tensor": sparse_like}, "Large shape (100, 100) mostly zeros") + +# Test 6: Mixed types in dict +print("\n=== Generating Mixed Type Tests ===") + +mixed_types = { + "float32": torch.tensor([1.0, 2.0], dtype=torch.float32), + "int64": torch.tensor([100, 200], dtype=torch.int64), + "bool": torch.tensor([True, False], dtype=torch.bool), + "float64": torch.tensor([1.1, 2.2], dtype=torch.float64), +} +save_test_file("mixed_types.pt", mixed_types, "Dict with mixed tensor types") + +# Test 7: Special values +print("\n=== Generating Special Value Tests ===") + +# NaN and Inf values +special_values = torch.tensor([float('nan'), float('inf'), float('-inf'), 0.0, 1.0]) +save_test_file("special_values.pt", {"tensor": special_values}, "Tensor with NaN and Inf") + +# Very small and very large values +extreme_values = torch.tensor([1e-30, 1e30, -1e-30, -1e30], dtype=torch.float32) +save_test_file("extreme_values.pt", {"tensor": extreme_values}, "Tensor with extreme values") + +# Test 8: Parameter wrapper (common in models) +print("\n=== Generating Parameter Tests ===") + +import torch.nn as nn +param = nn.Parameter(torch.randn(3, 3)) +param_dict = {"param": param} +save_test_file("parameter.pt", param_dict, "nn.Parameter wrapped tensor") + +# Test 9: Buffer-style tensors +print("\n=== Generating Buffer Tests ===") + +# Simulate model buffers +buffers = { + "buffer1": torch.tensor([1, 2, 3], dtype=torch.int32), + "buffer2": torch.tensor([True, False], dtype=torch.bool), +} +save_test_file("buffers.pt", buffers, "Model buffers") + +# Test 10: Complex nested structure +print("\n=== Generating Complex Structure Tests ===") + +complex_structure = { + "metadata": { + "version": 1, + "name": "test_model" + }, + "state": { + "encoder": { + "layer_0": { + "weight": torch.randn(4, 3), + "bias": torch.randn(4) + }, + "layer_1": { + "weight": torch.randn(2, 4), + "bias": torch.randn(2) + } + }, + "decoder": { + "weight": torch.randn(3, 2), + "bias": torch.randn(3) + } + }, + "config": { + "hidden_size": 4, + "num_layers": 2 + } +} +save_test_file("complex_structure.pt", complex_structure, "Complex nested structure") + +print(f"\n✅ Generated {len(list(test_dir.glob('*.pt')))} test files in {test_dir}") +print("\nTest files can be used to verify PyTorch reader functionality:") +print("- Different data types (float32, int64, bool, etc.)") +print("- Multi-dimensional tensors") +print("- State dicts and nested structures") +print("- Edge cases (empty, scalar, special values)") +print("- Model checkpoints and parameters") diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_2d_tensor.tar b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_2d_tensor.tar new file mode 100644 index 0000000000000000000000000000000000000000..abe84590c108c41e5f1702abfa5958208af434eb GIT binary patch literal 10240 zcmeH}(Msb$6o%6z+9kN)BX~JiC9T?CZUymXSQeB*ZY0EX)WMi(W==#Ggj~9B;iLE{ zUiuiGT3ca}-ep2rD{X^Nx*lyq-s-!NjT48%8RNiztwe*?-2h9 zftk*)3f1mp=B8ff|G~%Ju=Ia_cLe{pyDE$9ji0mS3zZW)-bKIcR9Q7W)Hy}sR7^In zGqUCGS#V$yZoDZAiI^S;qhVKO>dR7I0cJPGny zEw@N|LIyGzdRrg;fAHm)PgcWvWIeH-S>A!?ZL@Y*F>B01fB*=900@8p2!H?x lfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900{gW0O=bP5BGaQssG+!0R2}xE8Fb39T~|>9+6SJc5c%dPx#dsXzN@{71G_-q)puR)&0%#(RF zdOgrHP0SV21baori(ofty2@_;(!V<%mHz~#{&D_SIU(P2=eAet<~A!8`kQ*GTO8n9 z%P%aebNTTu-%1Zk5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X J0D)s5@C7=wmbm}` literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_float64.tar b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_float64.tar new file mode 100644 index 0000000000000000000000000000000000000000..b6acf66feeafd43a8a48c4b17adcc61a2530a68e GIT binary patch literal 10240 zcmeH|%SyvQ6ozx@MFN74;BrLchAy7dj5 z^tLEcS3$x5gv?~l8O}_;lYgRCDiq078YP;mtRiz%(_1<^{@26CTBimJuJl_kGD$L$ak%7}5;956yAM0K)R9;a#XRK6M2wQudI=ou zq}76lN<4Fw+UvB!f~t0^xy&f7`c`1;kw_@5`IVqjPr_X=Ts9rXhIOSUjb=JiG1uwKd>r$KozEGX?$C}Fxo?wiFx@iW zq}MY`(?DM%8sM-f_bb>JO_jOIPx|-V?y3IUUK{!^vnGe!8_PEv8;czT`mK7YQT)vd zdw(nJ^LE#pz3JAR-ZqE^1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck) N1V8`;K;ZWXd;w;qp6CDo literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_int64.tar b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_int64.tar new file mode 100644 index 0000000000000000000000000000000000000000..abf626cc16f15a211ca772bad528237b579f4d8b GIT binary patch literal 10240 zcmeH|!AiqG5QdX9Z6zT12p;z!rKxG_19*!fD1}@k)MRZJL(*g?g$jip^-X;XoshPN zBJ~s${3j%nompl#-|oNBD;=uzIWroxKHJFijLNee`o25b=eGIra4;~=`wh-%U%J#Z zEqF@%!2f#LSnJ#n!5M$`qR1$VvLsx}LaQvz&D)PwaLSoP5hXH|X{@GcZoLH#2WhpC zp;oU_=dE>ai=fRxJ~*~wI* zKBoV{<#=}{?DX$?F7)5*s?^mtR>YD|WJIiZ?c5cyEM$DGB67r|m~=>{Ix8aLd)^C6 zQi*oLLOzu$iNiV>E(eYfldk4Y(-~K)`Z_nP_lG*-Ow9hI6)j5NCf^YJ6uwFCM+T;Q zzNU1KqoV0Ouw69W}qc;-|THHv6JXG>ZM-w|E?^qopJS9 z+l9Fez<~e=fB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*>mHi0ia CCYj3s literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_multi_dtype.tar b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_multi_dtype.tar new file mode 100644 index 0000000000000000000000000000000000000000..4524991d7f3f88b23d0745e4826d083d4d140e61 GIT binary patch literal 10240 zcmeH}L2J}N6vtI+U~?_`fz5-gV1(b|3%syi=e`Rp*zY(^56<=-C%P+ryb! z0@NZGO(lz__1*1!fJ^oVUBl z-M)`-kK!L8Fqh@)S_~`3n0lN4XS2zwC#?KGnUep9y(n(=g;%2GYf%C^UWd=SN_3(+ zmL-JzB~S02)vACTM59$#0)8)|ya!q9Fz>`^FYC%p=9h8EdAh6dpyC|YGslKcO!*5t z6sDtb>X-9A@gZRxe|a*3cU#U6C-^CNx>u?G6dTa3WgIW-H##O(hqu(>-B?G64ejZ)f5h0Uqh*mbk}LYzWe|C;=F!LZUq5{Qc4+g+R@(dDRsLuVh|KGP3G>Y5g-CYfCvx)B0vO)01+SpM1Tko0V1#= F@Ex#H6iNU9 literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_weight_bias.tar b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/reader/test_data/tar_weight_bias.tar new file mode 100644 index 0000000000000000000000000000000000000000..24961dcf2cbd11ab4abc5789f8c2f151d14a2b35 GIT binary patch literal 10240 zcmeH{L2KJE6vrJoO^aZ#>u!TxbP9=^CYLU~j9s)*=rH(FjO%ELn6YB%xub-@9v8jK#sa4_)JBgXx)*B@@z zHtVBba(3#`0RcEEnCNZeo85^GmT!?Vj;ihDYO6$G+eP$B)RHb;> z>V!CrxhDU$ zr~Ka^j_&*aAlN7WH+d$t_JT>)T`*BQ^da{`x3k%PyIQ@yU40l|FPCGSk1>68y?k}_ zOQ15?!vbt0!0+B1^1GYEx;%|UfCvx)B0vO)01+SpM1Tko0U|&IhyW2F0z`la5CI}U U1c(3;AOb{y2oM1x@P`Qe1le`i4*&oF literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/mod.rs new file mode 100644 index 0000000..657e668 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/mod.rs @@ -0,0 +1,1200 @@ +//! Comprehensive tests for PytorchStore with real model application +use burn_core as burn; + +use std::path::PathBuf; + +use crate::ModuleStore; +use crate::pytorch::PytorchStore; +use burn_core::module::Module; +use burn_nn::conv::{Conv2d, Conv2dConfig}; +use burn_nn::{Linear, LinearConfig}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +/// Path to pytorch test files (now under burn-store) +fn pytorch_test_path(subdir: &str, filename: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("pytorch-tests") + .join("tests") + .join(subdir) + .join(filename) +} + +/// Path to burn-store test data files +fn test_data_path(filename: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("pytorch") + .join("tests") + .join("reader") + .join("test_data") + .join(filename) +} + +/// Path to store test data files +fn store_test_data_path(filename: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("pytorch") + .join("tests") + .join("store") + .join("test_data") + .join(filename) +} + +#[cfg(test)] +mod basic_tests { + use super::*; + + #[test] + fn test_store_creation() { + let store = PytorchStore::from_file("model.pth"); + assert!(store.validate); + assert!(!store.allow_partial); + assert!(store.top_level_key.is_none()); + // Contiguous index mapping is enabled by default for PyTorch files + assert!(store.map_indices_contiguous); + } + + #[test] + fn test_store_map_indices_contiguous_default() { + // Verify that map_indices_contiguous is enabled by default + let store = PytorchStore::from_file("model.pth"); + assert!( + store.map_indices_contiguous, + "map_indices_contiguous should be enabled by default" + ); + } + + #[test] + fn test_store_map_indices_contiguous_disabled() { + // Verify that we can disable map_indices_contiguous + let store = PytorchStore::from_file("model.pth").map_indices_contiguous(false); + assert!( + !store.map_indices_contiguous, + "map_indices_contiguous should be disabled after explicit call" + ); + } + + #[test] + fn test_store_with_top_level_key() { + let store = PytorchStore::from_file("model.pth").with_top_level_key("state_dict"); + assert_eq!(store.top_level_key, Some("state_dict".to_string())); + } + + #[test] + fn test_store_configuration() { + let store = PytorchStore::from_file("model.pth") + .validate(false) + .allow_partial(true) + .with_regex(r"^encoder\.") + .with_full_path("decoder.weight"); + + assert!(!store.validate); + assert!(store.allow_partial); + assert!(!store.filter.is_empty()); + } + + #[test] + fn test_store_with_remapping() { + let store = PytorchStore::from_file("model.pth").with_key_remapping(r"^old\.", "new."); + + assert!(!store.remapper.is_empty()); + } + + #[test] + fn test_store_save_not_supported() { + // Currently, saving to PyTorch format is not implemented + // The collect_from method always returns an error + let store = PytorchStore::from_file("test.pth"); + + // Just verify that store creation works + assert!(store.validate); + + // Note: Actually testing save would require a proper Module implementation + // which is complex. The implementation guarantees it returns an error. + } +} + +#[cfg(test)] +mod linear_model_tests { + use super::*; + type TestBackend = burn_ndarray::NdArray; + + #[derive(Module, Debug)] + pub struct SimpleLinearModel { + fc1: Linear, + fc2: Linear, + } + + impl SimpleLinearModel { + pub fn new(device: &B::Device) -> Self { + Self { + fc1: LinearConfig::new(2, 3).init(device), + fc2: LinearConfig::new(3, 4).init(device), + } + } + + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.fc1.forward(x); + self.fc2.forward(x) + } + } + + #[test] + fn test_load_linear_model() { + let device = Default::default(); + let path = pytorch_test_path("linear", "linear.pt"); + + // Create a model and load weights from PyTorch + let mut model = SimpleLinearModel::::new(&device); + let mut store = PytorchStore::from_file(path).allow_partial(true); + + // Apply the PyTorch weights to our model + let result = store.apply_to::(&mut model); + + assert!( + result.is_ok(), + "Failed to load linear model: {:?}", + result.err() + ); + + let result = result.unwrap(); + assert!(!result.applied.is_empty(), "No tensors were applied"); + + // Test forward pass with loaded weights + let input = Tensor::::ones([1, 2], &device); + let output = model.forward(input); + + // Verify output shape + assert_eq!(&*output.shape(), [1, 4]); + } + + #[test] + fn test_load_linear_with_bias() { + let device = Default::default(); + let path = pytorch_test_path("linear", "linear_with_bias.pt"); + + // Single linear layer with bias + #[derive(Module, Debug)] + struct LinearWithBias { + fc1: Linear, + } + + let mut model = LinearWithBias { + fc1: LinearConfig::new(2, 3).init(&device), + }; + + let mut store = PytorchStore::from_file(path).allow_partial(true); + + let result = store.apply_to::(&mut model); + assert!(result.is_ok(), "Failed to load model with bias"); + + // Verify biases were loaded + let result = result.unwrap(); + let bias_loaded = result.applied.iter().any(|s| s.contains("bias")); + assert!(bias_loaded, "Bias parameters not loaded"); + } + + #[test] + fn test_filter_layers() { + let device = Default::default(); + let path = pytorch_test_path("linear", "linear.pt"); + + let mut model = SimpleLinearModel::::new(&device); + + // Only load fc1 layers + let mut store = PytorchStore::from_file(path) + .with_regex(r"^fc1\.") + .allow_partial(true); + + let result = store.apply_to::(&mut model).unwrap(); + + // Should only have fc1 tensors + for tensor in &result.applied { + assert!(tensor.contains("fc1")); + assert!(!tensor.contains("fc2")); + } + } + + #[test] + fn test_remap_layer_names() { + let device = Default::default(); + let path = pytorch_test_path("linear", "linear.pt"); + + // Model with different layer names + #[derive(Module, Debug)] + struct RemappedModel { + linear1: Linear, + linear2: Linear, + } + + let mut model = RemappedModel { + linear1: LinearConfig::new(2, 3).init(&device), + linear2: LinearConfig::new(3, 4).init(&device), + }; + + let mut store = PytorchStore::from_file(path) + .with_key_remapping(r"^fc1\.", "linear1.") + .with_key_remapping(r"^fc2\.", "linear2.") + .allow_partial(true); + + let result = store.apply_to::(&mut model); + assert!(result.is_ok(), "Failed to load with remapped names"); + + let result = result.unwrap(); + // Verify remapped names were applied + let has_linear1 = result.applied.iter().any(|s| s.contains("linear1")); + assert!(has_linear1, "Remapped names not applied"); + } +} + +#[cfg(test)] +mod conv_model_tests { + use super::*; + + type TestBackend = burn_ndarray::NdArray; + + #[derive(Module, Debug)] + struct SimpleConvModel { + conv1: Conv2d, + conv2: Conv2d, + } + + impl SimpleConvModel { + pub fn new(device: &B::Device) -> Self { + Self { + conv1: Conv2dConfig::new([3, 16], [3, 3]).init(device), + conv2: Conv2dConfig::new([16, 32], [3, 3]).init(device), + } + } + } + + #[test] + fn test_load_conv2d_model() { + let device = Default::default(); + let path = pytorch_test_path("conv2d", "conv2d.pt"); + + // Check if file exists, skip if not + if !path.exists() { + println!("Skipping conv2d test - file not found: {:?}", path); + return; + } + + let mut model = SimpleConvModel::::new(&device); + let mut store = PytorchStore::from_file(path).allow_partial(true); + + let result = store.apply_to::(&mut model); + + if let Ok(result) = result { + assert!(!result.applied.is_empty(), "No conv tensors applied"); + + // Check for conv weights + let has_conv_weights = result.applied.iter().any(|s| s.contains("weight")); + assert!(has_conv_weights, "Conv weights not loaded"); + } + } + + #[test] + fn test_load_conv1d_model() { + let path = pytorch_test_path("conv1d", "conv1d.pt"); + + if !path.exists() { + println!("Skipping conv1d test - file not found: {:?}", path); + return; + } + + // Just test that we can create a store for conv1d files + let store = PytorchStore::from_file(path).allow_partial(true); + + assert!(store.allow_partial); + } +} + +#[cfg(test)] +mod complex_model_tests { + use super::*; + type TestBackend = burn_ndarray::NdArray; + + #[test] + fn test_load_with_top_level_key() { + let path = test_data_path("checkpoint.pt"); + + // Just verify that we can create a store with top-level key + let store = PytorchStore::from_file(path) + .with_top_level_key("model_state_dict") + .allow_partial(true); + + assert_eq!(store.top_level_key, Some("model_state_dict".to_string())); + } + + #[test] + fn test_load_nested_structure() { + let path = test_data_path("complex_structure.pt"); + + // Just verify that we can create a store for nested structure + let store = PytorchStore::from_file(path).allow_partial(true); + + assert!(store.allow_partial); + } + + #[test] + fn test_legacy_format() { + let path = test_data_path("simple_legacy.pt"); + + if !path.exists() { + println!("Skipping legacy format test - file not found: {:?}", path); + return; + } + + // Just verify that we can create a store for legacy format + let store = PytorchStore::from_file(path).allow_partial(true); + + assert!(store.allow_partial); + + // Could load into an actual model if we had legacy model structure + } + + #[test] + fn test_key_remap_chained() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping key remap test - file not found: {:?}", path); + return; + } + + let device = Default::default(); + + // Model with different layer names that need remapping + #[derive(Module, Debug)] + struct RemappedChainModel { + convolution1: Linear, // Will be remapped from fc1 + linear2: Linear, // Will be remapped from fc2 + } + + let mut model = RemappedChainModel { + convolution1: LinearConfig::new(2, 3).init(&device), + linear2: LinearConfig::new(3, 4).init(&device), + }; + + // Chain multiple remappings + let mut store = PytorchStore::from_file(path) + .with_key_remapping(r"^fc1\.", "convolution1.") + .with_key_remapping(r"^fc2\.", "linear2.") + .allow_partial(true); + + let result = store.apply_to::(&mut model); + + if let Ok(result) = result { + // Check that remapped names were applied + assert!( + !result.applied.is_empty(), + "No tensors were applied after remapping" + ); + } + } +} + +#[cfg(test)] +mod adapter_tests { + use super::*; + + type TestBackend = burn_ndarray::NdArray; + + #[derive(Module, Debug)] + pub struct SimpleLinearModel { + fc1: Linear, + fc2: Linear, + } + + impl SimpleLinearModel { + pub fn new(device: &B::Device) -> Self { + Self { + fc1: LinearConfig::new(2, 3).init(device), + fc2: LinearConfig::new(3, 4).init(device), + } + } + } + + #[test] + fn test_pytorch_adapter_always_applied() { + // Test that PyTorchToBurnAdapter is always applied internally + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping adapter test - file not found: {:?}", path); + return; + } + + let device = Default::default(); + let mut model = SimpleLinearModel::::new(&device); + + let mut store = PytorchStore::from_file(path).allow_partial(true); + + let result = store.apply_to::(&mut model); + + // PyTorchToBurnAdapter is always applied internally + assert!( + result.is_ok(), + "Failed to load with internal PyTorchToBurnAdapter: {:?}", + result.err() + ); + assert!(!result.unwrap().applied.is_empty()); + } + + #[test] + fn test_pytorch_adapter_with_filtering() { + // Test that PyTorchToBurnAdapter works with filtering + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping filtering test - file not found: {:?}", path); + return; + } + + let device = Default::default(); + let mut model = SimpleLinearModel::::new(&device); + + // Filter to exclude bias tensors + let mut store = PytorchStore::from_file(path) + .with_predicate(|path, _| !path.contains("bias")) + .allow_partial(true); + + let result = store.apply_to::(&mut model).unwrap(); + + // Should not have any bias tensors due to filtering + for applied_path in &result.applied { + assert!( + !applied_path.contains("bias"), + "Bias tensor was not filtered: {}", + applied_path + ); + } + } +} + +#[cfg(test)] +mod error_handling_tests { + use super::*; + use burn_ndarray::NdArray; + + #[derive(Module, Debug)] + pub struct SimpleLinearModel { + fc1: Linear, + fc2: Linear, + } + + impl SimpleLinearModel { + pub fn new(device: &B::Device) -> Self { + Self { + fc1: LinearConfig::new(2, 3).init(device), + fc2: LinearConfig::new(3, 4).init(device), + } + } + } + + #[test] + fn test_missing_file() { + let device = Default::default(); + let mut model = SimpleLinearModel::::new(&device); + let mut store = PytorchStore::from_file("nonexistent.pth"); + + let result = store.apply_to::(&mut model); + + assert!(result.is_err()); + match result { + Err(crate::pytorch::PytorchStoreError::Reader(_)) => {} + _ => panic!("Expected reader error for missing file"), + } + } + + #[test] + fn test_invalid_top_level_key() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!( + "Skipping invalid top level key test - file not found: {:?}", + path + ); + return; + } + + let device = Default::default(); + let mut model = SimpleLinearModel::::new(&device); + + let mut store = PytorchStore::from_file(path).with_top_level_key("nonexistent_key"); + + let result = store.apply_to::(&mut model); + + assert!(result.is_err(), "Should fail with invalid top level key"); + } + + #[test] + fn test_strict_validation() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!( + "Skipping strict validation test - file not found: {:?}", + path + ); + return; + } + + let device = Default::default(); + let mut model = SimpleLinearModel::::new(&device); + + // Apply very restrictive filter that matches nothing + let mut store = PytorchStore::from_file(path) + .with_regex(r"^this_will_never_match$") + .validate(true) + .allow_partial(false); + + let result = store.apply_to::(&mut model); + + // Should fail because no tensors match and allow_partial is false + assert!( + result.is_err(), + "Should fail when no tensors match with allow_partial=false" + ); + } +} + +#[cfg(test)] +mod enum_variant_tests { + use super::*; + use crate::ModuleSnapshot; + use burn_ndarray::NdArray; + + /// Enum representing different convolution block types (similar to YOLOX architecture) + #[derive(Module, Debug)] + pub enum ConvBlock { + /// Base convolution block + BaseConv(Linear), + /// Depthwise separable convolution block + DwsConv(Linear), + } + + /// Model with enum field that will have variant names in Burn paths + #[derive(Module, Debug)] + pub struct ModelWithEnum { + /// Feature extractor with enum variants + feature: ConvBlock, + /// Output classifier + classifier: Linear, + } + + impl ModelWithEnum { + pub fn new(device: &B::Device) -> Self { + Self { + feature: ConvBlock::BaseConv(LinearConfig::new(3, 64).init(device)), + classifier: LinearConfig::new(64, 10).init(device), + } + } + } + + #[test] + fn test_enum_variant_path_mismatch() { + let device = Default::default(); + let mut model = ModelWithEnum::::new(&device); + + // Load PyTorch model that was generated without enum variant names + // PyTorch paths: "feature.weight", "feature.bias", "classifier.weight", "classifier.bias" + // Burn paths: "feature.BaseConv.weight", "feature.BaseConv.bias", "classifier.weight", "classifier.bias" + // ^^^^^^^^ enum variant name is included in Burn but not PyTorch + + let pytorch_file = store_test_data_path("model_without_enum_variants.pt"); + + // Try to load from PyTorch format (without enum variants) + // Explicitly disable skip_enum_variants to demonstrate the mismatch problem + let mut store = PytorchStore::from_file(pytorch_file) + .skip_enum_variants(false) // Disable to show the mismatch + .allow_partial(true) // Allow partial to see what's missing + .validate(false); // Disable validation to get detailed missing info + + let result = store.apply_to::(&mut model); + + // The load should succeed (allow_partial=true) but report missing tensors + match result { + Ok(apply_result) => { + // Verify we have missing tensors + assert!( + !apply_result.missing.is_empty(), + "Should have missing tensors due to enum variant path mismatch" + ); + + // Check that missing paths contain enum variants + let enum_missing: Vec<_> = apply_result + .missing + .iter() + .filter(|(_, container_stack)| container_stack.contains("Enum:")) + .collect(); + + assert!( + !enum_missing.is_empty(), + "Missing tensors should be detected as having enum containers" + ); + + // Verify the paths look like what we expect + let has_base_conv_path = apply_result + .missing + .iter() + .any(|(path, _)| path.contains("BaseConv")); + + assert!( + has_base_conv_path, + "Should have missing paths with 'BaseConv' enum variant. Missing: {:?}", + apply_result + .missing + .iter() + .map(|(p, _)| p) + .collect::>() + ); + + // Print the diagnostic output to show enum detection + println!("\n{}", apply_result); + + // Verify the diagnostic message mentions enum variants + let display_output = format!("{}", apply_result); + assert!( + display_output.contains("enum variant"), + "Display output should mention enum variants" + ); + } + Err(e) => panic!( + "Load should succeed with allow_partial=true, got error: {}", + e + ), + } + } + + #[test] + fn test_enum_variant_detection_in_container_stack() { + let device = Default::default(); + + // Create model with enum + let model = ModelWithEnum::::new(&device); + + // Collect snapshots to inspect container stacks + let snapshots = model.collect(None, None, false); + + // Find a snapshot from inside the enum + let enum_snapshot = snapshots + .iter() + .find(|s| s.full_path().contains("feature")) + .expect("Should have feature snapshots"); + + // Verify container stack contains enum marker + if let Some(container_stack) = &enum_snapshot.container_stack { + let container_str = container_stack.join("."); + assert!( + container_str.contains("Enum:ConvBlock"), + "Container stack should contain Enum:ConvBlock marker. Got: {}", + container_str + ); + } else { + panic!("Snapshot should have container_stack"); + } + } + + #[test] + fn test_skip_enum_variants_feature() { + let device = Default::default(); + let mut model = ModelWithEnum::::new(&device); + + // Load PyTorch model that was generated without enum variant names + // PyTorch paths: "feature.weight", "feature.bias", "classifier.weight", "classifier.bias" + // Burn paths: "feature.BaseConv.weight", "feature.BaseConv.bias", "classifier.weight", "classifier.bias" + + let pytorch_file = store_test_data_path("model_without_enum_variants.pt"); + + // Try to load with skip_enum_variants enabled + let mut store = PytorchStore::from_file(pytorch_file) + .skip_enum_variants(true) // Enable enum variant skipping + .allow_partial(true) + .validate(false); + + let result = store.apply_to::(&mut model); + + // The load should succeed and all tensors should be loaded + match result { + Ok(apply_result) => { + println!("\n{}", apply_result); + + // With skip_enum_variants enabled, we should successfully load the feature tensors + let feature_applied = apply_result + .applied + .iter() + .filter(|path| path.contains("feature")) + .count(); + + assert!( + feature_applied > 0, + "Should have applied feature tensors with skip_enum_variants=true. Applied: {:?}", + apply_result.applied + ); + + // The feature tensors should NOT be in missing anymore + let feature_missing = apply_result + .missing + .iter() + .filter(|(path, _)| path.contains("feature")) + .count(); + + assert_eq!( + feature_missing, 0, + "Feature tensors should not be missing with skip_enum_variants=true. Missing: {:?}", + apply_result.missing + ); + } + Err(e) => panic!( + "Load with skip_enum_variants should succeed, got error: {}", + e + ), + } + } +} + +#[cfg(test)] +mod direct_access_tests { + use super::*; + + #[test] + fn test_get_all_snapshots() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let mut store = PytorchStore::from_file(path); + let snapshots = store.get_all_snapshots().unwrap(); + + // linear.pt should have fc1.weight, fc1.bias, fc2.weight, fc2.bias + assert!(!snapshots.is_empty(), "Should have snapshots"); + assert!( + snapshots.contains_key("fc1.weight"), + "Should contain fc1.weight" + ); + assert!( + snapshots.contains_key("fc1.bias"), + "Should contain fc1.bias" + ); + } + + #[test] + fn test_get_snapshot_existing() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let mut store = PytorchStore::from_file(path); + + // Get existing snapshot + let snapshot = store.get_snapshot("fc1.weight").unwrap(); + assert!(snapshot.is_some(), "Should find fc1.weight"); + + let snapshot = snapshot.unwrap(); + // Linear weight should be 2D + assert_eq!(snapshot.shape.len(), 2, "Weight should be 2D tensor"); + + // Verify we can load data + let data = snapshot.to_data().unwrap(); + assert!(!data.bytes.is_empty(), "Data should not be empty"); + } + + #[test] + fn test_get_snapshot_not_found() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let mut store = PytorchStore::from_file(path); + + // Get non-existent snapshot + let snapshot = store.get_snapshot("nonexistent.weight").unwrap(); + assert!(snapshot.is_none(), "Should not find nonexistent tensor"); + } + + #[test] + fn test_keys() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let mut store = PytorchStore::from_file(path); + let keys = store.keys().unwrap(); + + assert!(!keys.is_empty(), "Should have keys"); + assert!( + keys.contains(&"fc1.weight".to_string()), + "Keys should contain fc1.weight" + ); + assert!( + keys.contains(&"fc1.bias".to_string()), + "Keys should contain fc1.bias" + ); + } + + #[test] + fn test_keys_fast_path() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + // Create fresh store - cache should be empty + let mut store = PytorchStore::from_file(&path); + + // keys() should work without populating the full cache (fast path) + let keys = store.keys().unwrap(); + assert!(!keys.is_empty(), "Should have keys via fast path"); + + // Now call get_all_snapshots to populate cache + let snapshots = store.get_all_snapshots().unwrap(); + assert!(!snapshots.is_empty(), "Should have snapshots"); + + // keys() should now use the cached data + let keys2 = store.keys().unwrap(); + assert_eq!(keys.len(), keys2.len(), "Keys count should match"); + } + + #[test] + fn test_caching_behavior() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let mut store = PytorchStore::from_file(path); + + // First call populates cache + let snapshots1 = store.get_all_snapshots().unwrap(); + let count1 = snapshots1.len(); + + // Second call uses cache + let snapshots2 = store.get_all_snapshots().unwrap(); + let count2 = snapshots2.len(); + + assert_eq!(count1, count2, "Cached results should match"); + } + + #[test] + fn test_get_all_snapshots_with_remapping() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + // Create store with key remapping + let mut store = PytorchStore::from_file(path).with_key_remapping(r"^fc1\.", "linear1."); + + let snapshots = store.get_all_snapshots().unwrap(); + + // Should have remapped keys + assert!( + snapshots.contains_key("linear1.weight"), + "Should contain remapped key linear1.weight. Keys: {:?}", + snapshots.keys().collect::>() + ); + assert!( + snapshots.contains_key("linear1.bias"), + "Should contain remapped key linear1.bias" + ); + + // Original keys should not exist + assert!( + !snapshots.contains_key("fc1.weight"), + "Should not contain original key fc1.weight" + ); + } + + #[test] + fn test_get_snapshot_with_remapped_name() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + // Create store with key remapping + let mut store = PytorchStore::from_file(path).with_key_remapping(r"^fc1\.", "linear1."); + + // Should find by remapped name + let snapshot = store.get_snapshot("linear1.weight").unwrap(); + assert!(snapshot.is_some(), "Should find tensor by remapped name"); + + // Should NOT find by original name + let snapshot_orig = store.get_snapshot("fc1.weight").unwrap(); + assert!( + snapshot_orig.is_none(), + "Should not find tensor by original name after remapping" + ); + } + + #[test] + fn test_get_all_snapshots_ignores_filter() { + let path = pytorch_test_path("linear", "linear.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + // Create store with filter that only matches fc1 + let mut store = PytorchStore::from_file(path).with_regex(r"^fc1\."); + + // get_all_snapshots should return ALL tensors regardless of filter + let snapshots = store.get_all_snapshots().unwrap(); + + // Should have both fc1 and fc2 tensors + assert!( + snapshots.contains_key("fc1.weight"), + "Should contain fc1.weight" + ); + assert!( + snapshots.contains_key("fc2.weight"), + "Should contain fc2.weight (filter not applied to get_all_snapshots)" + ); + } +} + +/// Tests for contiguous index mapping feature +#[cfg(test)] +mod map_indices_contiguous_tests { + use super::*; + type TestBackend = burn_ndarray::NdArray; + + /// Model with a Vec of Conv2d layers that expects contiguous indices + #[derive(Module, Debug)] + struct SequentialConvModel { + fc: Vec>, + } + + impl SequentialConvModel { + pub fn new(device: &B::Device, num_layers: usize) -> Self { + Self { + fc: (0..num_layers) + .map(|_| { + Conv2dConfig::new([2, 2], [3, 3]) + .with_bias(true) + .init(device) + }) + .collect(), + } + } + } + + #[test] + fn test_load_non_contiguous_indexes_with_mapping() { + // This test uses the non_contiguous_indexes.pt file which has: + // fc.0.weight, fc.0.bias, fc.2.weight, fc.2.bias, fc.4.weight, ... (non-contiguous) + // The Burn model expects fc.0, fc.1, fc.2, ... (contiguous) + + let path = pytorch_test_path("non_contiguous_indexes", "non_contiguous_indexes.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let device = Default::default(); + + // Create model with 5 conv layers (matching the PyTorch model) + let mut model = SequentialConvModel::::new(&device, 5); + + // Load with contiguous index mapping enabled (default) + let mut store = PytorchStore::from_file(&path) + .map_indices_contiguous(true) + .allow_partial(true) + .validate(false); + + let result = store.apply_to::(&mut model); + + match result { + Ok(apply_result) => { + println!("Applied tensors: {:?}", apply_result.applied); + println!("Missing tensors: {:?}", apply_result.missing); + println!("Unused tensors: {:?}", apply_result.unused); + + // All fc layers should be loaded successfully + assert!( + !apply_result.applied.is_empty(), + "Should have applied tensors" + ); + + // Verify we have tensors from all 5 layers + // With mapping: fc.0, fc.1, fc.2, fc.3, fc.4 + for i in 0..5 { + let has_weight = apply_result + .applied + .iter() + .any(|p| p.contains(&format!("fc.{}.weight", i))); + let has_bias = apply_result + .applied + .iter() + .any(|p| p.contains(&format!("fc.{}.bias", i))); + + assert!( + has_weight, + "Should have applied fc.{}.weight, applied: {:?}", + i, apply_result.applied + ); + assert!( + has_bias, + "Should have applied fc.{}.bias, applied: {:?}", + i, apply_result.applied + ); + } + + // There should be no missing tensors (assuming model matches) + let missing_fc: Vec<_> = apply_result + .missing + .iter() + .filter(|(p, _)| p.starts_with("fc.")) + .collect(); + assert!( + missing_fc.is_empty(), + "Should have no missing fc tensors with index mapping. Missing: {:?}", + missing_fc + ); + } + Err(e) => panic!("Failed to load with index mapping: {}", e), + } + } + + #[test] + fn test_load_non_contiguous_indexes_without_mapping() { + // This test verifies that loading fails or has missing tensors when + // map_indices_contiguous is disabled + + let path = pytorch_test_path("non_contiguous_indexes", "non_contiguous_indexes.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + let device = Default::default(); + + // Create model with 5 conv layers + let mut model = SequentialConvModel::::new(&device, 5); + + // Load with contiguous index mapping DISABLED + let mut store = PytorchStore::from_file(&path) + .map_indices_contiguous(false) // Disable index mapping + .allow_partial(true) + .validate(false); + + let result = store.apply_to::(&mut model); + + match result { + Ok(apply_result) => { + println!( + "Without index mapping - Applied tensors: {:?}", + apply_result.applied + ); + println!( + "Without index mapping - Missing tensors: {:?}", + apply_result.missing + ); + + // Without index mapping, we should have missing tensors for fc.1, fc.3 + // because the source has fc.0, fc.2, fc.4, fc.6, fc.8 but model expects fc.0-4 + let missing_fc: Vec<_> = apply_result + .missing + .iter() + .filter(|(p, _)| p.starts_with("fc.")) + .collect(); + + assert!( + !missing_fc.is_empty(), + "Should have missing fc tensors without index mapping (indices 1, 3 don't exist in file)" + ); + + // Specifically, fc.1 and fc.3 should be missing + let has_fc1_missing = apply_result + .missing + .iter() + .any(|(p, _)| p.starts_with("fc.1.")); + let has_fc3_missing = apply_result + .missing + .iter() + .any(|(p, _)| p.starts_with("fc.3.")); + + assert!( + has_fc1_missing || has_fc3_missing, + "Should have fc.1 or fc.3 missing. Missing: {:?}", + apply_result.missing + ); + } + Err(e) => panic!("Unexpected error: {}", e), + } + } + + #[test] + fn test_mapping_applied_to_keys() { + // Verify that the keys returned by the store are mapped + let path = pytorch_test_path("non_contiguous_indexes", "non_contiguous_indexes.pt"); + + if !path.exists() { + println!("Skipping test - file not found: {:?}", path); + return; + } + + // With index mapping enabled (default) + let mut store_mapped = PytorchStore::from_file(&path).map_indices_contiguous(true); + + let keys_mapped = store_mapped.keys().unwrap(); + println!("Keys with index mapping: {:?}", keys_mapped); + + // Should have contiguous keys: fc.0, fc.1, fc.2, fc.3, fc.4 + assert!( + keys_mapped.iter().any(|k| k.starts_with("fc.1.")), + "With index mapping, should have fc.1 (from fc.2)" + ); + assert!( + keys_mapped.iter().any(|k| k.starts_with("fc.2.")), + "With index mapping, should have fc.2 (from fc.4)" + ); + + // Without index mapping + let mut store_no_mapping = PytorchStore::from_file(&path).map_indices_contiguous(false); + + let keys_no_mapping = store_no_mapping.keys().unwrap(); + println!("Keys without index mapping: {:?}", keys_no_mapping); + + // Should have original non-contiguous keys: fc.0, fc.2, fc.4, fc.6, fc.8 + assert!( + keys_no_mapping.iter().any(|k| k.starts_with("fc.2.")), + "Without index mapping, should have original fc.2" + ); + assert!( + keys_no_mapping.iter().any(|k| k.starts_with("fc.4.")), + "Without index mapping, should have original fc.4" + ); + assert!( + !keys_no_mapping.iter().any(|k| k.starts_with("fc.1.")), + "Without index mapping, should NOT have fc.1 (not in original file)" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/test_data/generate_enum_test.py b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/test_data/generate_enum_test.py new file mode 100644 index 0000000..f735153 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/pytorch/tests/store/test_data/generate_enum_test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Generate PyTorch test data for enum variant path mismatch testing. + +This script creates a PyTorch checkpoint that simulates how PyTorch models +export their state dicts WITHOUT enum variant names in the paths. + +Example: +- PyTorch path: "feature.weight" +- Burn path: "feature.BaseConv.weight" (includes enum variant "BaseConv") + +Run with: uv run generate_enum_test.py +""" + +import torch +import torch.nn as nn + + +class SimpleModel(nn.Module): + """ + Simple PyTorch model that represents what a Burn enum model would look like + WITHOUT the enum variant names in the path. + + In Burn, this would be: + struct ModelWithEnum { + feature: ConvBlock, // enum with BaseConv, DwsConv variants + classifier: Linear, + } + + But PyTorch exports it as flat paths without the enum variant names. + """ + def __init__(self): + super().__init__() + # This represents the "feature" field which is an enum in Burn + # PyTorch doesn't have enums, so it's just a Linear layer + # Path will be: "feature.weight" and "feature.bias" + self.feature = nn.Linear(3, 64) + + # This represents the "classifier" field + # Path will be: "classifier.weight" and "classifier.bias" + self.classifier = nn.Linear(64, 10) + + def forward(self, x): + x = self.feature(x) + x = torch.relu(x) + x = self.classifier(x) + return x + + +def generate_enum_variant_mismatch_test(): + """Generate test file demonstrating enum variant path mismatch.""" + model = SimpleModel() + + # Initialize with some deterministic weights for testing + torch.manual_seed(42) + for param in model.parameters(): + param.data.normal_(0, 0.1) + + # Save the state dict + # PyTorch paths: "feature.weight", "feature.bias", "classifier.weight", "classifier.bias" + # Burn paths: "feature.BaseConv.weight", "feature.BaseConv.bias", ... + # ^^^^^^^^ enum variant is missing in PyTorch + torch.save(model.state_dict(), "model_without_enum_variants.pt") + + print("Generated: model_without_enum_variants.pt") + print("\nPyTorch state dict keys:") + for key in model.state_dict().keys(): + shape = tuple(model.state_dict()[key].shape) + print(f" {key}: {shape}") + + print("\nExpected Burn paths (with enum variant):") + print(" feature.BaseConv.weight: (3, 64)") + print(" feature.BaseConv.bias: (64,)") + print(" classifier.weight: (64, 10)") + print(" classifier.bias: (10,)") + + print("\n⚠️ Notice: Burn includes 'BaseConv' enum variant, PyTorch doesn't!") + + +if __name__ == "__main__": + generate_enum_variant_mismatch_test() diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/mod.rs new file mode 100644 index 0000000..b077ed1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/mod.rs @@ -0,0 +1,322 @@ +//! SafeTensors format support for Burn deep learning framework. +//! +//! [SafeTensors](https://github.com/huggingface/safetensors) is a simple, safe, and efficient format +//! for storing and loading tensors. It provides fast zero-copy deserialization and strong safety +//! guarantees, making it ideal for production environments. +//! +//! # Features +//! +//! - **Fast Loading**: Zero-copy tensor access using safetensors' built-in mechanisms +//! - **Safety**: Prevents arbitrary code execution during model loading +//! - **Efficiency**: Memory-mapped files enable lazy loading without reading entire file +//! - **Filtering**: Load only specific tensors using path filters +//! - **Remapping**: Transform tensor names during load/save operations +//! - **Metadata**: Store and retrieve custom metadata alongside tensors (automatic `format`, `producer` and `version` metadata included) +//! - **Cross-Platform**: Works on all platforms including no-std environments +//! +//! # Usage Examples +//! +//! ## Basic Save and Load +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot}; +//! +//! // Save a model to a file +//! let mut store = SafetensorsStore::from_file("model.safetensors"); +//! model.save_into(&mut store)?; +//! +//! // Load a model from a file +//! let mut store = SafetensorsStore::from_file("model.safetensors"); +//! let mut model = Model::new(&device); +//! model.load_from(&mut store)?; +//! ``` +//! +//! ## Memory-Based Operations +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot}; +//! +//! // Save to memory buffer +//! let mut store = SafetensorsStore::from_bytes(None); +//! model.save_into(&mut store)?; +//! let bytes = store.get_bytes()?; +//! +//! // Load from memory buffer +//! let mut store = SafetensorsStore::from_bytes(Some(bytes)); +//! let mut model = Model::new(&device); +//! model.load_from(&mut store)?; +//! ``` +//! +//! ## Advanced Features +//! +//! ### Filter Configuration with Builder Pattern +//! +//! ```rust,no_run +//! # use burn_store::SafetensorsStore; +//! // Filter with regex patterns (OR logic - matches any pattern) +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .with_regex(r"^encoder\..*") // Match all encoder tensors +//! .with_regex(r".*\.bias$"); // OR match any bias tensors +//! +//! // Filter with exact paths +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .with_full_path("encoder.weight") +//! .with_full_path("encoder.bias") +//! .with_full_paths(vec!["decoder.scale", "decoder.norm"]); +//! +//! // Custom filter logic with predicate +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .with_predicate(|path, _dtype| { +//! // Only save layer weights (not biases) +//! path.contains("layer") && path.ends_with("weight") +//! }); +//! +//! // Combine multiple filter methods +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .with_regex(r"^encoder\..*") // All encoder tensors +//! .with_full_path("decoder.scale") // Plus specific decoder.scale +//! .with_predicate(|path, _| { // Plus any projection tensors +//! path.contains("projection") +//! }); +//! +//! // Save or load all tensors (no filtering) +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .match_all(); +//! ``` +//! +//! ### Tensor Name Remapping +//! +//! Remap tensor names during load/save operations for compatibility between different frameworks: +//! +//! ```rust,no_run +//! # use burn_store::{SafetensorsStore, KeyRemapper}; +//! // Using builder pattern for common remapping patterns +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .with_key_remapping(r"^encoder\.", "transformer.encoder.") // encoder.X -> transformer.encoder.X +//! .with_key_remapping(r"\.gamma$", ".weight") // X.gamma -> X.weight +//! .with_key_remapping(r"\.beta$", ".bias"); // X.beta -> X.bias +//! +//! // Or using a pre-configured KeyRemapper for complex transformations +//! let remapper = KeyRemapper::new() +//! .add_pattern(r"^pytorch\.(.*)", "burn.$1").expect("valid regex") // pytorch.layer -> burn.layer +//! .add_pattern(r"^(.*)\.running_mean$", "$1.mean").expect("valid regex") // layer.running_mean -> layer.mean +//! .add_pattern(r"^(.*)\.running_var$", "$1.variance").expect("valid regex"); // layer.running_var -> layer.variance +//! +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! .remap(remapper); +//! ``` +//! +//! ### Framework Adapters +//! +//! Use adapters for automatic framework-specific transformations: +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot, PyTorchToBurnAdapter, BurnToPyTorchAdapter}; +//! +//! // Loading PyTorch model into Burn +//! let mut store = SafetensorsStore::from_file("pytorch_model.safetensors") +//! .with_from_adapter(PyTorchToBurnAdapter) // Transposes linear weights, renames norm params +//! .allow_partial(true); // PyTorch models may have extra tensors +//! +//! let mut burn_model = Model::new(&device); +//! burn_model.load_from(&mut store)?; +//! +//! // Saving Burn model for PyTorch +//! let mut store = SafetensorsStore::from_file("for_pytorch.safetensors") +//! .with_to_adapter(BurnToPyTorchAdapter); // Transposes weights back, renames for PyTorch +//! +//! burn_model.save_into(&mut store)?; +//! ``` +//! +//! ### Additional Configuration Options +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot}; +//! +//! let mut store = SafetensorsStore::from_file("model.safetensors") +//! // Add custom metadata +//! .metadata("version", "1.0.0") +//! .metadata("producer", "burn") +//! // Allow partial loading (continue even if some tensors are missing) +//! .allow_partial(true) +//! // Disable validation for faster loading +//! .validate(false); +//! +//! // Use the configured store +//! model.save_into(&mut store)?; // For saving +//! // or +//! model.load_from(&mut store)?; // For loading +//! ``` +//! +//! # Efficient Loading with SafeTensors +//! +//! SafeTensors provides efficient tensor loading through its zero-copy design: +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot}; +//! +//! let mut store = SafetensorsStore::from_file("large_model.safetensors"); +//! // Uses memory mapping (when available) for zero-copy access +//! // Falls back to buffered reading when mmap is not available +//! let mut model = Model::new(&device); +//! model.load_from(&mut store)?; +//! ``` +//! +//! The safetensors approach provides: +//! - Zero-copy views - tensors are accessed directly from the mapped file +//! - Lazy loading - only accessed tensors are materialized +//! - Efficient memory usage - no unnecessary data duplication +//! +//! # Lazy Loading and Inspection +//! +//! SafeTensors provides efficient inspection and selective loading through its +//! zero-copy design and built-in metadata handling: +//! +//! ```rust,ignore +//! use burn_store::SafetensorsStore; +//! +//! // Open a file - uses safetensors' efficient header reading +//! let store = SafetensorsStore::from_file("large_model.safetensors"); +//! +//! // List all tensor names from the metadata +//! let tensor_names = store.list_tensors()?; +//! println!("Model contains {} tensors", tensor_names.len()); +//! +//! // Get tensor metadata without loading tensor data +//! if let Some((shape, dtype)) = store.tensor_info("encoder.weight")? { +//! println!("Encoder weight shape: {:?}, dtype: {:?}", shape, dtype); +//! } +//! +//! // Selectively load tensors - safetensors handles efficient access +//! let encoder_tensors = store.load_tensors(&[ +//! "encoder.weight", +//! "encoder.bias", +//! "encoder.norm" +//! ])?; +//! +//! // Distributed loading: each worker loads only its assigned layers +//! // SafeTensors' zero-copy views ensure minimal memory usage +//! let worker_layers = match worker_id { +//! 0 => vec!["encoder.layer1", "encoder.layer2"], +//! 1 => vec!["encoder.layer3", "encoder.layer4"], +//! 2 => vec!["decoder.layer1", "decoder.layer2"], +//! _ => vec!["head.weight", "head.bias"], +//! }; +//! let worker_tensors = store.load_tensors(&worker_layers)?; +//! ``` +//! +//! # Builder Pattern API Reference +//! +//! The SafetensorsStore provides a fluent builder API for configuration: +//! +//! ## Filtering Methods +//! +//! - **`with_regex(pattern)`** - Add regex pattern to match tensor names (OR logic with multiple patterns) +//! - **`with_full_path(path)`** - Add exact tensor path to include +//! - **`with_full_paths(paths)`** - Add multiple exact tensor paths to include +//! - **`with_predicate(fn)`** - Add custom filter function `fn(&str, &str) -> bool` +//! - **`match_all()`** - Disable filtering, include all tensors +//! +//! ## Remapping Methods +//! +//! - **`with_key_remapping(from, to)`** - Add regex pattern to rename tensors +//! - **`remap(KeyRemapper)`** - Use a pre-configured KeyRemapper for complex transformations +//! +//! ## Adapter Methods +//! +//! - **`with_from_adapter(adapter)`** - Set adapter for loading (e.g., PyTorchToBurnAdapter) +//! - **`with_to_adapter(adapter)`** - Set adapter for saving (e.g., BurnToPyTorchAdapter) +//! +//! ## Configuration Methods +//! +//! - **`metadata(key, value)`** - Add custom metadata to saved files (in addition to automatic `format`, `producer` and `version`) +//! - **`allow_partial(bool)`** - Allow loading even if some tensors are missing +//! - **`validate(bool)`** - Enable/disable tensor validation during loading +//! +//! All methods return `Self` for chaining: +//! +//! ```rust,no_run +//! use burn_store::{SafetensorsStore, PyTorchToBurnAdapter}; +//! +//! let store = SafetensorsStore::from_file("model.safetensors") +//! .with_regex(r"^encoder\..*") +//! .with_key_remapping(r"\.gamma$", ".weight") +//! .with_from_adapter(PyTorchToBurnAdapter) +//! .allow_partial(true) +//! .metadata("version", "2.0"); +//! ``` +//! +//! # Working with Bytes +//! +//! For direct byte operations without files: +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot}; +//! +//! // Save to bytes with filtering and remapping +//! let mut store = SafetensorsStore::from_bytes(None) +//! .with_regex(r"^encoder\..*") // Only save encoder tensors +//! .with_key_remapping(r"^encoder\.", "transformer.") // Rename encoder.X -> transformer.X +//! .metadata("subset", "encoder_only"); +//! model.save_into(&mut store)?; +//! let bytes = store.get_bytes()?; +//! +//! // Load from bytes (allow partial since we only saved encoder) +//! let mut store = SafetensorsStore::from_bytes(Some(bytes)) +//! .with_key_remapping(r"^transformer\.", "encoder.") // Rename back: transformer.X -> encoder.X +//! .allow_partial(true); +//! let mut model = Model::new(&device); +//! let result = model.load_from(&mut store)?; +//! println!("Applied {} tensors", result.applied.len()); +//! ``` +//! +//! # Complete Example: PyTorch Model Migration +//! +//! Migrating a PyTorch model to Burn with filtering, remapping, and adapters: +//! +//! ```rust,ignore +//! use burn_store::{SafetensorsStore, ModuleSnapshot, PyTorchToBurnAdapter}; +//! +//! // Load PyTorch model with all transformations +//! let mut store = SafetensorsStore::from_file("pytorch_model.safetensors") +//! // Use PyTorch adapter for automatic transformations +//! .with_from_adapter(PyTorchToBurnAdapter) +//! // Only load transformer layers +//! .with_regex(r"^transformer\..*") +//! // Rename old layer names to new structure +//! .with_key_remapping(r"^transformer\.h\.(\d+)\.", "transformer.layer$1.") +//! // Skip unexpected tensors from PyTorch +//! .allow_partial(true) +//! // Add metadata about the conversion +//! .metadata("source", "pytorch") +//! .metadata("converted_by", "burn-store"); +//! +//! let mut model = TransformerModel::new(&device); +//! let result = model.load_from(&mut store)?; +//! +//! println!("Successfully loaded {} tensors", result.applied.len()); +//! if !result.missing.is_empty() { +//! println!("Missing tensors: {:?}", result.missing); +//! } +//! ``` +//! +//! # Format Details +//! +//! SafeTensors uses a simple binary format: +//! - **8 bytes**: Header size (unsigned little-endian 64-bit integer) +//! - **N bytes**: JSON header with tensor metadata +//! - Contains: `{"tensor_name": {"dtype": "F32", "shape": [1, 2, 3], "data_offsets": [start, end]}, ...}` +//! - Special key `__metadata__` for user-defined string metadata +//! - **Rest**: Raw tensor data (referenced by offsets in header) +//! +//! The format enables: +//! - **Secure loading**: No code execution, just data +//! - **Efficient access**: Use offsets to read only needed tensors +//! - **Simple parsing**: Standard JSON header with fixed structure + +mod store; + +pub use store::{SafetensorsStore, SafetensorsStoreError}; + +#[cfg(test)] +mod tests; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/store.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/store.rs new file mode 100644 index 0000000..7db2344 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/store.rs @@ -0,0 +1,1091 @@ +//! SafeTensors store implementation using the official safetensors crate. + +use crate::{ + ApplyResult, IdentityAdapter, ModuleAdapter, ModuleSnapshot, ModuleStore, PathFilter, + TensorSnapshot, +}; + +#[cfg(feature = "std")] +use crate::{KeyRemapper, map_indices_contiguous}; +use alloc::boxed::Box; +use alloc::collections::BTreeMap; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec; +use alloc::vec::Vec; +use burn_core::module::ParamId; +use burn_tensor::backend::Backend; +use burn_tensor::{DType, TensorData}; +use core::fmt; +use core::ops::Deref; +use hashbrown::HashMap; + +// Arc is only available on targets with atomic pointers +#[cfg(target_has_atomic = "ptr")] +use alloc::sync::Arc; + +// For targets without atomic pointers, we use Box instead +#[cfg(not(target_has_atomic = "ptr"))] +type Arc = Box; + +/// Errors that can occur during SafeTensors operations. +#[derive(Debug)] +pub enum SafetensorsStoreError { + /// SafeTensors crate error. + Safetensors(safetensors::SafeTensorError), + + /// I/O error. + #[cfg(feature = "std")] + Io(std::io::Error), + + /// Tensor not found. + TensorNotFound(String), + + /// Validation failed. + ValidationFailed(String), + + /// Other error. + Other(String), +} + +impl fmt::Display for SafetensorsStoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Safetensors(e) => write!(f, "SafeTensors error: {}", e), + #[cfg(feature = "std")] + Self::Io(e) => write!(f, "I/O error: {}", e), + Self::TensorNotFound(name) => write!(f, "Tensor not found: {}", name), + Self::ValidationFailed(msg) => write!(f, "Validation failed: {}", msg), + Self::Other(msg) => write!(f, "{}", msg), + } + } +} + +impl core::error::Error for SafetensorsStoreError {} + +impl From for SafetensorsStoreError { + fn from(e: safetensors::SafeTensorError) -> Self { + SafetensorsStoreError::Safetensors(e) + } +} + +#[cfg(feature = "std")] +impl From for SafetensorsStoreError { + fn from(e: std::io::Error) -> Self { + SafetensorsStoreError::Io(e) + } +} + +/// SafeTensors store supporting both file and memory storage. +pub enum SafetensorsStore { + /// File-based storage. + #[cfg(feature = "std")] + File(FileStore), + + /// Memory-based storage. + Memory(MemoryStore), +} + +impl Default for SafetensorsStore { + /// Create a default memory-based store. + fn default() -> Self { + Self::from_bytes(None) + } +} + +impl SafetensorsStore { + /// Get the default metadata that includes Burn framework information. + /// + /// This includes: + /// - `format`: "safetensors" + /// - `producer`: "burn" + /// - `version`: The version of burn-store crate (from CARGO_PKG_VERSION) + /// + /// These metadata fields are automatically added to all saved models. + pub fn default_metadata() -> HashMap { + let mut metadata = HashMap::new(); + metadata.insert("format".to_string(), "safetensors".to_string()); + metadata.insert("producer".to_string(), "burn".to_string()); + metadata.insert("version".to_string(), env!("CARGO_PKG_VERSION").to_string()); + metadata + } + + /// Create a store for loading from or saving to a file. + #[cfg(feature = "std")] + pub fn from_file(path: impl Into) -> Self { + Self::File(FileStore { + path: path.into(), + filter: PathFilter::new(), + remapper: KeyRemapper::new(), + metadata: Self::default_metadata(), + validate: true, + allow_partial: false, + overwrite: false, + skip_enum_variants: false, + // Contiguous index mapping is off by default for SafeTensors + // (SafeTensors files typically have clean, contiguous indices) + map_indices_contiguous: false, + from_adapter: Box::new(IdentityAdapter), + to_adapter: Box::new(IdentityAdapter), + snapshots_cache: None, + }) + } + + /// Create a store for working with bytes in memory. + pub fn from_bytes(bytes: Option>) -> Self { + Self::Memory(MemoryStore { + data: bytes.map(Arc::new), + filter: PathFilter::new(), + #[cfg(feature = "std")] + remapper: KeyRemapper::new(), + metadata: Self::default_metadata(), + validate: true, + allow_partial: false, + skip_enum_variants: false, + // Contiguous index mapping is off by default for SafeTensors + #[cfg(feature = "std")] + map_indices_contiguous: false, + from_adapter: Box::new(IdentityAdapter), + to_adapter: Box::new(IdentityAdapter), + snapshots_cache: None, + }) + } + + /// Filter which tensors to load/save. + pub fn filter(mut self, filter: PathFilter) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = filter, + Self::Memory(p) => p.filter = filter, + } + self + } + + /// Add a regex pattern to filter tensors. + /// + /// Multiple patterns can be added and they work with OR logic. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// let store = SafetensorsStore::from_file("model.safetensors") + /// .with_regex(r"^encoder\..*") // Match all encoder tensors + /// .with_regex(r".*\.weight$"); // OR match any weight tensors + /// ``` + #[cfg(feature = "std")] + pub fn with_regex>(mut self, pattern: S) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().with_regex(pattern), + Self::Memory(p) => p.filter = p.filter.clone().with_regex(pattern), + } + self + } + + /// Add multiple regex patterns to filter tensors. + #[cfg(feature = "std")] + pub fn with_regexes(mut self, patterns: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().with_regexes(patterns), + Self::Memory(p) => p.filter = p.filter.clone().with_regexes(patterns), + } + self + } + + /// Add an exact full path to match. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// let store = SafetensorsStore::from_file("model.safetensors") + /// .with_full_path("encoder.layer1.weight") + /// .with_full_path("decoder.output.bias"); + /// ``` + pub fn with_full_path>(mut self, path: S) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().with_full_path(path), + Self::Memory(p) => p.filter = p.filter.clone().with_full_path(path), + } + self + } + + /// Add multiple exact full paths to match. + pub fn with_full_paths(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().with_full_paths(paths), + Self::Memory(p) => p.filter = p.filter.clone().with_full_paths(paths), + } + self + } + + /// Add a predicate function for custom filtering logic. + /// + /// The predicate receives the tensor path and container path. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// let store = SafetensorsStore::from_file("model.safetensors") + /// .with_predicate(|path, _| path.starts_with("encoder.") || path.ends_with(".bias")); + /// ``` + pub fn with_predicate(mut self, predicate: fn(&str, &str) -> bool) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().with_predicate(predicate), + Self::Memory(p) => p.filter = p.filter.clone().with_predicate(predicate), + } + self + } + + /// Add multiple predicate functions. + pub fn with_predicates(mut self, predicates: I) -> Self + where + I: IntoIterator bool>, + { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().with_predicates(predicates), + Self::Memory(p) => p.filter = p.filter.clone().with_predicates(predicates), + } + self + } + + /// Set the filter to match all paths (disables filtering). + pub fn match_all(mut self) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.filter = p.filter.clone().match_all(), + Self::Memory(p) => p.filter = p.filter.clone().match_all(), + } + self + } + + /// Remap tensor names during load/save. + #[cfg(feature = "std")] + pub fn remap(mut self, remapper: KeyRemapper) -> Self { + match &mut self { + Self::File(p) => p.remapper = remapper, + Self::Memory(p) => p.remapper = remapper, + } + self + } + + /// Add a regex pattern to remap tensor names during load/save. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// let store = SafetensorsStore::from_file("model.safetensors") + /// .with_key_remapping(r"^encoder\.", "transformer.encoder.") // encoder.X -> transformer.encoder.X + /// .with_key_remapping(r"\.gamma$", ".weight"); // X.gamma -> X.weight + /// ``` + #[cfg(feature = "std")] + pub fn with_key_remapping( + mut self, + from_pattern: impl AsRef, + to_pattern: impl Into, + ) -> Self { + match &mut self { + Self::File(p) => { + p.remapper = p + .remapper + .clone() + .add_pattern(from_pattern, to_pattern) + .expect("Invalid regex pattern"); + } + Self::Memory(p) => { + p.remapper = p + .remapper + .clone() + .add_pattern(from_pattern, to_pattern) + .expect("Invalid regex pattern"); + } + } + self + } + + /// Add metadata to be saved with the tensors. + pub fn metadata(mut self, key: impl Into, value: impl Into) -> Self { + let key = key.into(); + let value = value.into(); + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => { + p.metadata.insert(key, value); + } + Self::Memory(p) => { + p.metadata.insert(key, value); + } + } + self + } + + /// Clear all metadata including the default Burn framework metadata. + /// + /// This removes the automatic `format`, `producer` and `version` fields. + /// Use this when you need complete control over metadata or when + /// saving models for use with other frameworks. + pub fn clear_metadata(mut self) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => { + p.metadata.clear(); + } + Self::Memory(p) => { + p.metadata.clear(); + } + } + self + } + + /// Set whether to validate tensors during loading (default: true). + pub fn validate(mut self, validate: bool) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.validate = validate, + Self::Memory(p) => p.validate = validate, + } + self + } + + /// Allow partial loading of tensors (continue even if some tensors are missing). + pub fn allow_partial(mut self, allow: bool) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.allow_partial = allow, + Self::Memory(p) => p.allow_partial = allow, + } + self + } + + /// Skip enum variant names when loading or saving tensor paths. + /// + /// When enabled during **loading**, tensor paths from the source that don't include enum variants + /// can be matched against Burn module paths that do include them. + /// For example, source path "feature.weight" can match Burn path "feature.BaseConv.weight". + /// + /// When enabled during **saving**, enum variant names are omitted from the exported tensor paths, + /// making them compatible with PyTorch naming conventions. + /// For example, "feature.BaseConv.weight" becomes "feature.weight" in the exported file. + /// + /// This is useful when working with models from/to formats that don't include enum variant + /// names in their parameter paths (like PyTorch models). + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// // For PyTorch compatibility + /// let store = SafetensorsStore::from_file("model.safetensors") + /// .skip_enum_variants(true); + /// ``` + pub fn skip_enum_variants(mut self, skip: bool) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.skip_enum_variants = skip, + Self::Memory(p) => p.skip_enum_variants = skip, + } + self + } + + /// Enable or disable automatic contiguous mapping of layer indices (default: false). + /// + /// When enabled, non-contiguous numeric indices in tensor paths are renumbered + /// to be contiguous. This is useful when loading models that have gaps + /// in layer numbering, such as PyTorch models using `nn.Sequential` with mixed + /// layer types (e.g., Conv2d layers at indices 0, 2, 4 with ReLU layers at 1, 3, 5). + /// + /// # Example + /// + /// With index mapping enabled: + /// - `fc.0.weight` → `fc.0.weight` + /// - `fc.2.weight` → `fc.1.weight` (gap filled) + /// - `fc.4.weight` → `fc.2.weight` (gap filled) + /// + /// # Arguments + /// + /// * `map` - `true` to enable contiguous index mapping, `false` to disable + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// // Enable contiguous index mapping for PyTorch-exported safetensors + /// let store = SafetensorsStore::from_file("model.safetensors") + /// .map_indices_contiguous(true); + /// ``` + #[cfg(feature = "std")] + pub fn map_indices_contiguous(mut self, map: bool) -> Self { + match &mut self { + Self::File(p) => p.map_indices_contiguous = map, + Self::Memory(p) => p.map_indices_contiguous = map, + } + self + } + + /// Set whether to overwrite existing files when saving (default: false). + /// + /// When set to `false`, attempting to save to an existing file will result in an error. + /// When set to `true`, existing files will be overwritten without warning. + /// + /// This setting only applies to file-based stores. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// let mut store = SafetensorsStore::from_file("model.safetensors") + /// .overwrite(true); + /// // Will overwrite if file exists when saving + /// ``` + #[cfg(feature = "std")] + pub fn overwrite(mut self, overwrite: bool) -> Self { + match &mut self { + Self::File(p) => p.overwrite = overwrite, + Self::Memory(_) => { + // Memory stores don't have overwrite semantics, ignore + } + } + self + } + + /// Set the adapter for loading tensors (converting from source format to Burn). + pub fn with_from_adapter(mut self, adapter: impl ModuleAdapter + 'static) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.from_adapter = Box::new(adapter), + Self::Memory(p) => p.from_adapter = Box::new(adapter), + } + self + } + + /// Set the adapter for saving tensors (converting from Burn to target format). + pub fn with_to_adapter(mut self, adapter: impl ModuleAdapter + 'static) -> Self { + match &mut self { + #[cfg(feature = "std")] + Self::File(p) => p.to_adapter = Box::new(adapter), + Self::Memory(p) => p.to_adapter = Box::new(adapter), + } + self + } + + /// Get saved bytes from memory-based store. + /// + /// # Example + /// ```rust,no_run + /// # use burn_store::SafetensorsStore; + /// # fn example() -> Result<(), Box> { + /// let mut store = SafetensorsStore::from_bytes(None); + /// // After saving model with collect_to()... + /// let bytes = store.get_bytes()?; + /// # Ok(()) + /// # } + /// ``` + pub fn get_bytes(&self) -> Result, SafetensorsStoreError> { + match self { + #[cfg(feature = "std")] + Self::File(_) => Err(SafetensorsStoreError::Other( + "Cannot get bytes from file-based store".to_string(), + )), + Self::Memory(p) => p + .data() + .map(|arc| arc.as_ref().clone()) + .ok_or_else(|| SafetensorsStoreError::Other("No data available".to_string())), + } + } +} + +/// File-based store. +#[cfg(feature = "std")] +pub struct FileStore { + path: std::path::PathBuf, + filter: PathFilter, + remapper: KeyRemapper, + metadata: HashMap, + validate: bool, + allow_partial: bool, + overwrite: bool, + skip_enum_variants: bool, + /// Enable contiguous mapping of layer indices (default: false) + map_indices_contiguous: bool, + from_adapter: Box, + to_adapter: Box, + /// Cached tensor snapshots (parsed once, reused) + snapshots_cache: Option>, +} + +/// Memory-based store. +pub struct MemoryStore { + data: Option>>, + filter: PathFilter, + #[cfg(feature = "std")] + remapper: KeyRemapper, + metadata: HashMap, + validate: bool, + allow_partial: bool, + skip_enum_variants: bool, + /// Enable contiguous mapping of layer indices (default: false) + #[cfg(feature = "std")] + map_indices_contiguous: bool, + from_adapter: Box, + to_adapter: Box, + /// Cached tensor snapshots (parsed once, reused) + snapshots_cache: Option>, +} + +impl Default for MemoryStore { + fn default() -> Self { + Self { + data: None, + filter: PathFilter::new(), + #[cfg(feature = "std")] + remapper: KeyRemapper::new(), + metadata: HashMap::new(), + validate: true, + allow_partial: false, + skip_enum_variants: false, + #[cfg(feature = "std")] + map_indices_contiguous: false, + from_adapter: Box::new(IdentityAdapter), + to_adapter: Box::new(IdentityAdapter), + snapshots_cache: None, + } + } +} + +impl MemoryStore { + #[cfg(test)] + pub(crate) fn data(&self) -> Option>> { + self.data.clone() + } + + #[cfg(not(test))] + fn data(&self) -> Option>> { + self.data.clone() + } + + #[cfg(test)] + pub(crate) fn set_data(&mut self, data: Vec) { + self.data = Some(Arc::new(data)); + } +} + +// Adapter to use TensorSnapshot directly with safetensors +#[derive(Debug)] +struct TensorSnapshotAdapter(TensorSnapshot); + +impl safetensors::View for TensorSnapshotAdapter { + fn dtype(&self) -> safetensors::Dtype { + // Convert from burn dtype to safetensors dtype + dtype_to_safetensors(self.0.dtype).unwrap_or(safetensors::Dtype::F32) + } + + fn shape(&self) -> &[usize] { + &self.0.shape + } + + fn data(&self) -> alloc::borrow::Cow<'_, [u8]> { + // Only materialize data when actually needed for serialization + let data = self + .0 + .to_data() + .unwrap_or_else(|e| panic!("Failed to get tensor data: {:?}", e)); + alloc::borrow::Cow::Owned(data.bytes.deref().to_vec()) + } + + fn data_len(&self) -> usize { + // Use the efficient data_len method from TensorSnapshot + self.0.data_len() + } +} + +impl ModuleStore for SafetensorsStore { + type Error = SafetensorsStoreError; + + fn collect_from>( + &mut self, + module: &M, + ) -> Result<(), Self::Error> { + // Invalidate cache since we're writing new data + match self { + #[cfg(feature = "std")] + Self::File(p) => p.snapshots_cache = None, + Self::Memory(p) => p.snapshots_cache = None, + } + + // Collect tensor snapshots from module with adapter + // The to_adapter converts from Burn format to target format for saving + let to_adapter = match self { + #[cfg(feature = "std")] + Self::File(p) => p.to_adapter.clone(), + Self::Memory(p) => p.to_adapter.clone(), + }; + let mut snapshots = module.collect(None, Some(to_adapter), self.get_skip_enum_variants()); + + // Apply filtering + snapshots = apply_filter(snapshots, self.get_filter()); + + // Apply remapping + #[cfg(feature = "std")] + { + snapshots = apply_remapping(snapshots, self.get_remapper()); + } + + // Get metadata (already includes format, producer and version from default_metadata) + let metadata = self.get_metadata().clone(); + + #[cfg(feature = "std")] + let std_metadata: std::collections::HashMap = metadata + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + // Write to storage + match self { + #[cfg(feature = "std")] + Self::File(p) => { + // Check if file exists and overwrite is disabled + if p.path.exists() && !p.overwrite { + return Err(SafetensorsStoreError::Other(format!( + "File already exists: {}. Use .overwrite(true) to overwrite.", + p.path.display() + ))); + } + + // Convert to safetensors format + let tensors = snapshots_to_safetensors(snapshots)?; + + // Use serialize_to_file which streams directly to disk + // This calls the lazy closures on-demand without buffering everything + safetensors::serialize_to_file(tensors, Some(std_metadata), &p.path)?; + Ok(()) + } + Self::Memory(p) => { + // For memory, we need to serialize to bytes + let tensors = snapshots_to_safetensors(snapshots)?; + // For no-std, serialize still needs std HashMap when std feature is enabled + #[cfg(feature = "std")] + let data = safetensors::serialize(tensors, Some(std_metadata))?; + + #[cfg(not(feature = "std"))] + let data = safetensors::serialize(tensors, Some(metadata))?; + p.data = Some(Arc::new(data)); + Ok(()) + } + } + } + + fn apply_to>( + &mut self, + module: &mut M, + ) -> Result { + // Get snapshots from cache + let snapshots: Vec = self.get_all_snapshots()?.values().cloned().collect(); + + // Get the adapter + let adapter: Box = match self { + #[cfg(feature = "std")] + Self::File(p) => p.from_adapter.clone(), + Self::Memory(p) => p.from_adapter.clone(), + }; + + // Get filter (cloned to Option for apply) + let filter = self.get_filter(); + let filter_opt = if filter.is_empty() { + None + } else { + Some(filter.clone()) + }; + + // Apply to module with adapter + // The adapter will be applied during module traversal with proper container info + // Filter is applied here during apply, not during cache population + let result = module.apply( + snapshots, + filter_opt, + Some(adapter), + self.get_skip_enum_variants(), + ); + + // Validate if needed + if self.get_validate() && !result.errors.is_empty() { + return Err(SafetensorsStoreError::ValidationFailed(format!( + "Import errors: {:?}", + result.errors + ))); + } + + if !self.get_allow_partial() && !result.missing.is_empty() { + return Err(SafetensorsStoreError::TensorNotFound(format!( + "\n{}", + result + ))); + } + + Ok(result) + } + + fn get_snapshot(&mut self, name: &str) -> Result, Self::Error> { + // Ensure cache is populated + self.ensure_snapshots_cache()?; + let cache = match self { + #[cfg(feature = "std")] + Self::File(p) => p.snapshots_cache.as_ref().unwrap(), + Self::Memory(p) => p.snapshots_cache.as_ref().unwrap(), + }; + Ok(cache.get(name)) + } + + fn get_all_snapshots(&mut self) -> Result<&BTreeMap, Self::Error> { + // Ensure cache is populated + self.ensure_snapshots_cache()?; + let cache = match self { + #[cfg(feature = "std")] + Self::File(p) => p.snapshots_cache.as_ref().unwrap(), + Self::Memory(p) => p.snapshots_cache.as_ref().unwrap(), + }; + Ok(cache) + } + + fn keys(&mut self) -> Result, Self::Error> { + // Always use the cache to ensure remapping is applied consistently + Ok(self.get_all_snapshots()?.keys().cloned().collect()) + } +} + +impl SafetensorsStore { + fn get_filter(&self) -> &PathFilter { + match self { + #[cfg(feature = "std")] + Self::File(p) => &p.filter, + Self::Memory(p) => &p.filter, + } + } + + #[cfg(feature = "std")] + fn get_remapper(&self) -> &KeyRemapper { + match self { + Self::File(p) => &p.remapper, + Self::Memory(p) => &p.remapper, + } + } + + fn get_metadata(&self) -> &HashMap { + match self { + #[cfg(feature = "std")] + Self::File(p) => &p.metadata, + Self::Memory(p) => &p.metadata, + } + } + + fn get_validate(&self) -> bool { + match self { + #[cfg(feature = "std")] + Self::File(p) => p.validate, + Self::Memory(p) => p.validate, + } + } + + fn get_allow_partial(&self) -> bool { + match self { + #[cfg(feature = "std")] + Self::File(p) => p.allow_partial, + Self::Memory(p) => p.allow_partial, + } + } + + fn get_skip_enum_variants(&self) -> bool { + match self { + #[cfg(feature = "std")] + Self::File(p) => p.skip_enum_variants, + Self::Memory(p) => p.skip_enum_variants, + } + } + + #[cfg(feature = "std")] + fn get_map_indices_contiguous(&self) -> bool { + match self { + Self::File(p) => p.map_indices_contiguous, + Self::Memory(p) => p.map_indices_contiguous, + } + } + + /// Ensure the snapshots cache is populated + fn ensure_snapshots_cache(&mut self) -> Result<(), SafetensorsStoreError> { + // Check if cache exists + let has_cache = match self { + #[cfg(feature = "std")] + Self::File(p) => p.snapshots_cache.is_some(), + Self::Memory(p) => p.snapshots_cache.is_some(), + }; + + if has_cache { + return Ok(()); + } + + // Load snapshots + #[allow(unused_mut)] + let mut snapshots = match self { + #[cfg(feature = "std")] + Self::File(p) => safetensors_to_snapshots_lazy_file(&p.path)?, + Self::Memory(p) => { + let data_arc = p + .data + .clone() + .ok_or_else(|| SafetensorsStoreError::Other("No data loaded".to_string()))?; + safetensors_to_snapshots_lazy(data_arc)? + } + }; + + // Apply remapping (but NOT filtering - that's done at apply time) + #[cfg(feature = "std")] + { + snapshots = match self { + Self::File(p) => apply_remapping(snapshots, &p.remapper), + Self::Memory(p) => apply_remapping(snapshots, &p.remapper), + }; + } + + // Apply contiguous index mapping if enabled + // This must be done after remapping so that remapped paths are mapped + #[cfg(feature = "std")] + if self.get_map_indices_contiguous() { + let (mapped, _) = map_indices_contiguous(snapshots); + snapshots = mapped; + } + + // Build cache as BTreeMap + let cache: BTreeMap = + snapshots.into_iter().map(|s| (s.full_path(), s)).collect(); + + // Store cache + match self { + #[cfg(feature = "std")] + Self::File(p) => p.snapshots_cache = Some(cache), + Self::Memory(p) => p.snapshots_cache = Some(cache), + } + + Ok(()) + } +} + +/// Apply filter to tensor snapshots. +fn apply_filter(mut snapshots: Vec, filter: &PathFilter) -> Vec { + if filter.is_empty() { + return snapshots; + } + + snapshots.retain(|snapshot| { + let path = snapshot.full_path(); + filter.matches(&path) + }); + + snapshots +} + +/// Apply remapping to tensor snapshots. +#[cfg(feature = "std")] +fn apply_remapping(snapshots: Vec, remapper: &KeyRemapper) -> Vec { + if remapper.is_empty() { + return snapshots; + } + + let (remapped, _) = remapper.remap(snapshots); + remapped +} + +/// Convert TensorSnapshots to safetensors format lazily. +fn snapshots_to_safetensors( + snapshots: Vec, +) -> Result, SafetensorsStoreError> { + let mut tensors = Vec::new(); + + for snapshot in snapshots { + let name = snapshot.full_path(); + // No need to materialize data - TensorSnapshot now has dtype and shape cached! + tensors.push((name, TensorSnapshotAdapter(snapshot))); + } + + Ok(tensors) +} + +/// Convert safetensors to TensorSnapshots with lazy loading. +fn safetensors_to_snapshots_lazy( + data_arc: Arc>, +) -> Result, SafetensorsStoreError> { + // Parse to get metadata + let tensors = safetensors::SafeTensors::deserialize(&data_arc)?; + let mut snapshots = Vec::new(); + + for (name, tensor_snapshot) in tensors.tensors() { + // Extract metadata without materializing data + let dtype = safetensor_dtype_to_burn(tensor_snapshot.dtype())?; + let shape = tensor_snapshot.shape().to_vec(); + let path_parts: Vec = name.split('.').map(|s| s.to_string()).collect(); + + // Create a lazy closure that will deserialize only this tensor when needed + #[cfg(target_has_atomic = "ptr")] + let data_clone = Arc::clone(&data_arc); + #[cfg(not(target_has_atomic = "ptr"))] + let data_clone = data_arc.clone(); + let name_clone = name.to_string(); + let data_fn = alloc::rc::Rc::new(move || { + // Re-deserialize when needed (this is cheap, just parsing header) + let tensors = safetensors::SafeTensors::deserialize(&data_clone).map_err(|e| { + crate::TensorSnapshotError::IoError(format!( + "Failed to re-deserialize safetensors: {}", + e + )) + })?; + + // Find our specific tensor + let tensor = tensors.tensor(&name_clone).map_err(|e| { + crate::TensorSnapshotError::DataError(format!( + "Tensor '{}' not found: {}", + name_clone, e + )) + })?; + + // Now materialize just this tensor's data + let bytes = burn_tensor::Bytes::from_bytes_vec(tensor.data().to_vec()); + Ok(TensorData { + bytes, + shape: tensor.shape().to_vec(), + dtype: safetensor_dtype_to_burn(tensor.dtype()) + .map_err(|_| crate::TensorSnapshotError::DataError("Invalid dtype".into()))?, + }) + }); + + let snapshot = TensorSnapshot::from_closure( + data_fn, + dtype, + shape, + path_parts, + vec![], // Empty container_stack - will be filled during module traversal + ParamId::new(), + ); + snapshots.push(snapshot); + } + + Ok(snapshots) +} + +/// Convert safetensors to TensorSnapshots with true on-demand loading from file. +/// This reads only the header initially, then loads tensor data on demand. +#[cfg(feature = "std")] +fn safetensors_to_snapshots_lazy_file( + path: &std::path::Path, +) -> Result, SafetensorsStoreError> { + // Always use memory mapping for the most efficient access + use memmap2::MmapOptions; + + // Memory map the file for efficient access + let file = std::fs::File::open(path)?; + let mmap = unsafe { MmapOptions::new().map(&file)? }; + let mmap_arc = Arc::new(mmap); + + // Parse just to get metadata (safetensors won't copy data with mmap) + let tensors = safetensors::SafeTensors::deserialize(&mmap_arc)?; + let mut snapshots = Vec::new(); + + for (name, tensor_snapshot) in tensors.tensors() { + let dtype = safetensor_dtype_to_burn(tensor_snapshot.dtype())?; + let shape = tensor_snapshot.shape().to_vec(); + let path_parts: Vec = name.split('.').map(|s| s.to_string()).collect(); + + // Create a lazy closure that accesses the mmap'd data + let mmap_clone = Arc::clone(&mmap_arc); + let name_clone = name.to_string(); + + let data_fn = alloc::rc::Rc::new(move || { + // Re-parse to get the tensor snapshot (this is cheap with mmap) + let tensors = safetensors::SafeTensors::deserialize(&mmap_clone).map_err(|e| { + crate::TensorSnapshotError::IoError(format!("Failed to deserialize: {}", e)) + })?; + let tensor = tensors.tensor(&name_clone).map_err(|e| { + crate::TensorSnapshotError::DataError(format!( + "Tensor '{}' not found: {}", + name_clone, e + )) + })?; + + // Only now do we actually copy the tensor data + Ok(TensorData { + bytes: burn_tensor::Bytes::from_bytes_vec(tensor.data().to_vec()), + shape: tensor.shape().to_vec(), + dtype: safetensor_dtype_to_burn(tensor.dtype()) + .map_err(|_| crate::TensorSnapshotError::DataError("Invalid dtype".into()))?, + }) + }); + + let snapshot = TensorSnapshot::from_closure( + data_fn, + dtype, + shape, + path_parts, + vec![], // Empty container_stack - will be filled during module traversal + ParamId::new(), + ); + snapshots.push(snapshot); + } + + Ok(snapshots) +} + +/// Helper to convert safetensors Dtype to burn DType. +fn safetensor_dtype_to_burn(dtype: safetensors::Dtype) -> Result { + use safetensors::Dtype; + + match dtype { + Dtype::F64 => Ok(DType::F64), + Dtype::F32 => Ok(DType::F32), + Dtype::F16 => Ok(DType::F16), + Dtype::BF16 => Ok(DType::BF16), + Dtype::I64 => Ok(DType::I64), + Dtype::I32 => Ok(DType::I32), + Dtype::I16 => Ok(DType::I16), + Dtype::I8 => Ok(DType::I8), + Dtype::U64 => Ok(DType::U64), + Dtype::U32 => Ok(DType::U32), + Dtype::U8 => Ok(DType::U8), + Dtype::BOOL => Ok(DType::Bool), + _ => Err(SafetensorsStoreError::Other(format!( + "Unsupported dtype: {:?}", + dtype + ))), + } +} + +/// Helper to convert DType to safetensors Dtype. +fn dtype_to_safetensors(dtype: DType) -> Result { + use safetensors::Dtype; + + match dtype { + DType::F64 => Ok(Dtype::F64), + DType::F32 | DType::Flex32 => Ok(Dtype::F32), // Flex32 is stored as F32 + DType::F16 => Ok(Dtype::F16), + DType::BF16 => Ok(Dtype::BF16), + DType::I64 => Ok(Dtype::I64), + DType::I32 => Ok(Dtype::I32), + DType::I16 => Ok(Dtype::I16), + DType::I8 => Ok(Dtype::I8), + DType::U64 => Ok(Dtype::U64), + DType::U32 => Ok(Dtype::U32), + DType::U16 => Err(SafetensorsStoreError::Other( + "U16 dtype not yet supported in safetensors".to_string(), + )), + DType::U8 => Ok(Dtype::U8), + DType::Bool => Ok(Dtype::BOOL), + DType::QFloat(_) => Err(SafetensorsStoreError::Other( + "Quantized tensors not yet supported in safetensors".to_string(), + )), + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/adapter.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/adapter.rs new file mode 100644 index 0000000..f2846f3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/adapter.rs @@ -0,0 +1,193 @@ +use burn_core as burn; + +use crate::{BurnToPyTorchAdapter, ModuleSnapshot, PyTorchToBurnAdapter, SafetensorsStore}; +use burn_core::module::{Module, Param}; +use burn_nn::{Linear, LinearConfig}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +type TestBackend = burn_ndarray::NdArray; + +#[derive(Module, Debug)] +struct TestModel { + linear: Linear, + norm_weight: Param>, + norm_bias: Param>, +} + +impl TestModel { + fn new(device: &B::Device) -> Self { + Self { + linear: LinearConfig::new(4, 2).with_bias(true).init(device), + norm_weight: Param::from_data([1.0, 1.0], device), + norm_bias: Param::from_data([0.0, 0.0], device), + } + } +} + +#[test] +fn pytorch_to_burn_adapter_linear_transpose() { + let device = Default::default(); + let model = TestModel::::new(&device); + + // Save with BurnToPyTorch adapter (will transpose linear weights) + let mut save_store = SafetensorsStore::from_bytes(None).with_to_adapter(BurnToPyTorchAdapter); + model.save_into(&mut save_store).unwrap(); + + // Load with PyTorchToBurn adapter (will transpose back) + let mut load_store = SafetensorsStore::from_bytes(None).with_from_adapter(PyTorchToBurnAdapter); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + p.set_data(p_save.data().unwrap().as_ref().clone()); + } + + let mut model2 = TestModel::::new(&device); + let result = model2.load_from(&mut load_store).unwrap(); + + // Should successfully load all tensors + assert!(!result.applied.is_empty()); + + // Verify the linear weights are the same after round-trip + let weight1 = model.linear.weight.val().to_data(); + let weight2 = model2.linear.weight.val().to_data(); + + assert_eq!(weight1.shape, weight2.shape); + let data1 = weight1.to_vec::().unwrap(); + let data2 = weight2.to_vec::().unwrap(); + + for (a, b) in data1.iter().zip(data2.iter()) { + assert!( + (a - b).abs() < 1e-6, + "Weights differ after adapter round-trip" + ); + } +} + +#[test] +fn pytorch_to_burn_adapter_norm_rename() { + let device = Default::default(); + + // Create a model with norm-like naming + #[derive(Module, Debug)] + struct NormModel { + norm_gamma: Param>, + norm_beta: Param>, + } + + impl NormModel { + fn new(device: &B::Device) -> Self { + Self { + norm_gamma: Param::from_data([1.0, 2.0, 3.0], device), + norm_beta: Param::from_data([0.1, 0.2, 0.3], device), + } + } + } + + let model = NormModel::::new(&device); + + // Save with BurnToPyTorch adapter (will rename gamma->weight, beta->bias) + let mut save_store = SafetensorsStore::from_bytes(None).with_to_adapter(BurnToPyTorchAdapter); + model.save_into(&mut save_store).unwrap(); + + // The saved data should have PyTorch naming convention + // We can't directly verify the internal names, but we can verify round-trip works + + // Load with PyTorchToBurn adapter (will rename weight->gamma, bias->beta) + let mut load_store = SafetensorsStore::from_bytes(None).with_from_adapter(PyTorchToBurnAdapter); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + p.set_data(p_save.data().unwrap().as_ref().clone()); + } + + let mut model2 = NormModel::::new(&device); + let result = model2.load_from(&mut load_store).unwrap(); + + // Should load successfully + assert!(!result.applied.is_empty()); + + // Verify data is preserved + let gamma1 = model.norm_gamma.val().to_data().to_vec::().unwrap(); + let gamma2 = model2.norm_gamma.val().to_data().to_vec::().unwrap(); + let beta1 = model.norm_beta.val().to_data().to_vec::().unwrap(); + let beta2 = model2.norm_beta.val().to_data().to_vec::().unwrap(); + + assert_eq!(gamma1, gamma2); + assert_eq!(beta1, beta2); +} + +#[test] +fn no_adapter_preserves_original() { + let device = Default::default(); + let model = TestModel::::new(&device); + + // Save without adapter + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).unwrap(); + + // Load without adapter + let mut load_store = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + p.set_data(p_save.data().unwrap().as_ref().clone()); + } + + let mut model2 = TestModel::::new(&device); + let result = model2.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); + assert!(!result.applied.is_empty()); + + // Verify data is exactly the same + let weight1 = model.linear.weight.val().to_data(); + let weight2 = model2.linear.weight.val().to_data(); + + assert_eq!(weight1.shape, weight2.shape); + assert_eq!( + weight1.to_vec::().unwrap(), + weight2.to_vec::().unwrap() + ); +} + +#[test] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +fn adapter_with_pytorch_import() { + use crate::PyTorchToBurnAdapter; + + let device = Default::default(); + + // Reference the safetensors file from burn-store + let safetensors_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/safetensors-tests/tests/multi_layer/multi_layer.safetensors" + ); + + // Simple test model that matches some of the PyTorch structure + #[derive(Module, Debug)] + struct SimpleNet { + fc1: Linear, + } + + impl SimpleNet { + fn new(device: &B::Device) -> Self { + Self { + fc1: LinearConfig::new(4 * 8 * 8, 16).init(device), + } + } + } + + // Load with PyTorchToBurn adapter + let mut store = SafetensorsStore::from_file(safetensors_path) + .with_from_adapter(PyTorchToBurnAdapter) + .validate(false) + .allow_partial(true); + + let mut model = SimpleNet::::new(&device); + let result = model.load_from(&mut store).unwrap(); + + // Should load some tensors (fc1 if it exists in the file) + // This mainly tests that the adapter works with real PyTorch files + assert!(!result.applied.is_empty() || !result.missing.is_empty()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/direct_access.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/direct_access.rs new file mode 100644 index 0000000..ff1e0d7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/direct_access.rs @@ -0,0 +1,341 @@ +use burn_core as burn; + +use crate::{ModuleStore, SafetensorsStore}; +use burn_core::module::{Module, Param}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +type TestBackend = burn_ndarray::NdArray; + +// Test module for direct access tests +#[derive(Module, Debug)] +struct DirectAccessTestModule { + weight: Param>, + bias: Param>, + nested: DirectAccessNestedModule, +} + +#[derive(Module, Debug)] +struct DirectAccessNestedModule { + gamma: Param>, + beta: Param>, +} + +impl DirectAccessTestModule { + fn new(device: &B::Device) -> Self { + Self { + weight: Param::from_data([[1.0, 2.0], [3.0, 4.0]], device), + bias: Param::from_data([0.1, 0.2], device), + nested: DirectAccessNestedModule { + gamma: Param::from_data([1.0, 2.0], device), + beta: Param::from_data([0.5, 0.5], device), + }, + } + } +} + +#[test] +fn test_memory_get_all_snapshots() { + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + // Save module to memory + let mut save_store = SafetensorsStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + + // Get bytes and create load store + let bytes = save_store.get_bytes().unwrap(); + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + + // Get all snapshots + let snapshots = load_store.get_all_snapshots().unwrap(); + + assert_eq!(snapshots.len(), 4); + assert!(snapshots.contains_key("weight")); + assert!(snapshots.contains_key("bias")); + assert!(snapshots.contains_key("nested.gamma")); + assert!(snapshots.contains_key("nested.beta")); +} + +#[test] +fn test_memory_get_snapshot_existing() { + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let mut save_store = SafetensorsStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + + // Get existing snapshot + let snapshot = load_store.get_snapshot("weight").unwrap(); + assert!(snapshot.is_some()); + + let snapshot = snapshot.unwrap(); + assert_eq!(snapshot.shape, vec![2, 2]); + + // Verify data + let data = snapshot.to_data().unwrap(); + let values: Vec = data.to_vec().unwrap(); + assert_eq!(values, vec![1.0, 2.0, 3.0, 4.0]); +} + +#[test] +fn test_memory_get_snapshot_nested() { + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let mut save_store = SafetensorsStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + + // Get nested snapshot + let snapshot = load_store.get_snapshot("nested.gamma").unwrap(); + assert!(snapshot.is_some()); + + let snapshot = snapshot.unwrap(); + let data = snapshot.to_data().unwrap(); + let values: Vec = data.to_vec().unwrap(); + assert_eq!(values, vec![1.0, 2.0]); +} + +#[test] +fn test_memory_get_snapshot_not_found() { + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let mut save_store = SafetensorsStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + + // Get non-existent snapshot + let snapshot = load_store.get_snapshot("nonexistent").unwrap(); + assert!(snapshot.is_none()); +} + +#[test] +fn test_memory_keys() { + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let mut save_store = SafetensorsStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + + let keys = load_store.keys().unwrap(); + assert_eq!(keys.len(), 4); + assert!(keys.contains(&"weight".to_string())); + assert!(keys.contains(&"bias".to_string())); + assert!(keys.contains(&"nested.gamma".to_string())); + assert!(keys.contains(&"nested.beta".to_string())); +} + +#[test] +fn test_memory_caching_behavior() { + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let mut save_store = SafetensorsStore::from_bytes(None); + save_store.collect_from(&module).unwrap(); + let bytes = save_store.get_bytes().unwrap(); + + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + + // Call get_all_snapshots multiple times - should return same cached data + let snapshots1 = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots1.len(), 4); + + let snapshots2 = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots2.len(), 4); + + // Verify we can still access individual snapshots after caching + let snapshot = load_store.get_snapshot("bias").unwrap(); + assert!(snapshot.is_some()); +} + +// ============================================================================ +// Tests for FileStore variant +// ============================================================================ + +#[test] +#[cfg(feature = "std")] +fn test_file_get_all_snapshots() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_get_all_snapshots.safetensors"); + + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + let mut load_store = SafetensorsStore::from_file(&path); + let snapshots = load_store.get_all_snapshots().unwrap(); + + assert_eq!(snapshots.len(), 4); + assert!(snapshots.contains_key("weight")); + assert!(snapshots.contains_key("bias")); + assert!(snapshots.contains_key("nested.gamma")); + assert!(snapshots.contains_key("nested.beta")); +} + +#[test] +#[cfg(feature = "std")] +fn test_file_get_snapshot_existing() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_get_snapshot.safetensors"); + + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + let mut load_store = SafetensorsStore::from_file(&path); + + let snapshot = load_store.get_snapshot("weight").unwrap(); + assert!(snapshot.is_some()); + + let snapshot = snapshot.unwrap(); + assert_eq!(snapshot.shape, vec![2, 2]); + + let data = snapshot.to_data().unwrap(); + let values: Vec = data.to_vec().unwrap(); + assert_eq!(values, vec![1.0, 2.0, 3.0, 4.0]); +} + +#[test] +#[cfg(feature = "std")] +fn test_file_get_snapshot_not_found() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_not_found.safetensors"); + + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + let mut load_store = SafetensorsStore::from_file(&path); + + let snapshot = load_store.get_snapshot("nonexistent").unwrap(); + assert!(snapshot.is_none()); +} + +#[test] +#[cfg(feature = "std")] +fn test_file_keys() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_keys.safetensors"); + + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + let mut load_store = SafetensorsStore::from_file(&path); + + let keys = load_store.keys().unwrap(); + assert_eq!(keys.len(), 4); + assert!(keys.contains(&"weight".to_string())); + assert!(keys.contains(&"bias".to_string())); + assert!(keys.contains(&"nested.gamma".to_string())); + assert!(keys.contains(&"nested.beta".to_string())); +} + +#[test] +#[cfg(feature = "std")] +fn test_file_keys_fast_path() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_keys_fast.safetensors"); + + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + // Create fresh store - cache should be empty + let mut load_store = SafetensorsStore::from_file(&path); + + // keys() should work without populating the full cache (fast path) + let keys = load_store.keys().unwrap(); + assert_eq!(keys.len(), 4); + + // Now call get_all_snapshots to populate cache + let snapshots = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots.len(), 4); + + // keys() should now use the cached data + let keys2 = load_store.keys().unwrap(); + assert_eq!(keys2.len(), 4); +} + +#[test] +#[cfg(feature = "std")] +fn test_file_caching_behavior() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_caching.safetensors"); + + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + + let mut load_store = SafetensorsStore::from_file(&path); + + // First call populates cache + let snapshots1 = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots1.len(), 4); + + // Second call uses cache + let snapshots2 = load_store.get_all_snapshots().unwrap(); + assert_eq!(snapshots2.len(), 4); +} + +#[test] +#[cfg(feature = "std")] +fn test_file_cache_invalidation_on_save() { + use tempfile::tempdir; + + let device = Default::default(); + let module = DirectAccessTestModule::::new(&device); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_invalidation.safetensors"); + + // Create store, save, and populate cache + let mut store = SafetensorsStore::from_file(&path).overwrite(true); + store.collect_from(&module).unwrap(); + + let snapshots1 = store.get_all_snapshots().unwrap(); + assert_eq!(snapshots1.len(), 4); + + // Save again (this should invalidate cache) + store.collect_from(&module).unwrap(); + + // Cache should be repopulated with fresh data + let snapshots2 = store.get_all_snapshots().unwrap(); + assert_eq!(snapshots2.len(), 4); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/error_handling.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/error_handling.rs new file mode 100644 index 0000000..ce9aaec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/error_handling.rs @@ -0,0 +1,51 @@ +use crate::{ModuleSnapshot, SafetensorsStore}; +use burn_nn::LinearConfig; + +type TestBackend = burn_ndarray::NdArray; + +#[test] +fn shape_mismatch_errors() { + let device = Default::default(); + + // Create a module + let module = LinearConfig::new(2, 2) + .with_bias(true) + .init::(&device); + + // Save module + let mut save_store = SafetensorsStore::from_bytes(None); + module.save_into(&mut save_store).unwrap(); + + // Try to load into incompatible module (different dimensions) + let mut incompatible_module = LinearConfig::new(3, 3) + .with_bias(true) + .init::(&device); + + // Load without validation - should return errors in the result + let mut load_store = SafetensorsStore::from_bytes(None).validate(false); // Disable validation to get errors in result + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + // Get Arc and extract data + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + + let result = incompatible_module.load_from(&mut load_store).unwrap(); + + // Should have errors due to shape mismatch + assert!(!result.errors.is_empty()); + + // Try again with validation enabled - should return Err + let mut load_store_with_validation = SafetensorsStore::from_bytes(None).validate(true); + if let SafetensorsStore::Memory(ref mut p) = load_store_with_validation + && let SafetensorsStore::Memory(ref p_save) = save_store + { + // Get Arc and extract data + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + + let validation_result = incompatible_module.load_from(&mut load_store_with_validation); + assert!(validation_result.is_err()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/file_io.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/file_io.rs new file mode 100644 index 0000000..8e12383 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/file_io.rs @@ -0,0 +1,288 @@ +use burn_core as burn; + +use crate::{ModuleSnapshot, ModuleStore, SafetensorsStore}; +use burn_core::module::{Module, Param}; +use burn_nn::{Initializer, LinearConfig}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +use tempfile::tempdir; + +type TestBackend = burn_ndarray::NdArray; + +// Define a test model with forward pass +#[derive(Module, Debug)] +struct ForwardTestModel { + linear1: burn_nn::Linear, + linear2: burn_nn::Linear, +} + +impl ForwardTestModel { + fn forward(&self, input: Tensor) -> Tensor { + let x = self.linear1.forward(input); + let x = burn::tensor::activation::gelu(x); + self.linear2.forward(x) + } +} + +// Define config for the model +#[derive(burn::config::Config, Debug)] +struct ForwardTestModelConfig { + input_size: usize, + hidden_size: usize, + output_size: usize, +} + +impl ForwardTestModelConfig { + fn init(&self, device: &B::Device) -> ForwardTestModel { + ForwardTestModel { + linear1: LinearConfig::new(self.input_size, self.hidden_size) + .with_bias(true) + .init(device), + linear2: LinearConfig::new(self.hidden_size, self.output_size) + .with_bias(true) + .init(device), + } + } +} + +#[derive(Module, Debug)] +pub struct ModuleBasic { + weight_basic: Param>, +} + +impl ModuleBasic { + fn new(device: &B::Device) -> Self { + Self { + weight_basic: Initializer::Normal { + std: 1.0, + mean: 0.0, + } + .init([20, 20], device), + } + } +} + +#[derive(Module, Debug)] +pub struct ModuleComposed { + weight: Param>, + basic: ModuleBasic, + tuple: (ModuleBasic, ModuleBasic), +} + +impl ModuleComposed { + fn new(device: &B::Device) -> Self { + let weight = Initializer::Normal { + std: 1.0, + mean: 0.0, + } + .init([20, 20], device); + + Self { + weight, + basic: ModuleBasic::new(device), + tuple: (ModuleBasic::new(device), ModuleBasic::new(device)), + } + } +} + +#[test] +fn file_based_loading() { + use std::fs; + + let device = Default::default(); + let module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + // Create temp file path + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join("test_safetensors.st"); + + // Save to file + let mut save_store = SafetensorsStore::from_file(&file_path).metadata("test", "file_loading"); + + module.save_into(&mut save_store).unwrap(); + + // Verify file exists + assert!(file_path.exists()); + + // Load from file (will use memory-mapped loading if available) + let mut load_store = SafetensorsStore::from_file(&file_path); + + let mut loaded_module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + let result = loaded_module.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 2); // weight and bias + + // Clean up + fs::remove_file(file_path).ok(); +} + +#[test] +fn test_store_overwrite_protection() { + use tempfile::tempdir; + + let device = Default::default(); + let module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + // Create temp directory and file path (file doesn't exist yet) + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_model.safetensors"); + + // First save - should succeed + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&module).unwrap(); + assert!(path.exists()); + + // Second save without overwrite flag - should fail + let mut save_store2 = SafetensorsStore::from_file(&path); + let result = save_store2.collect_from(&module); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("File already exists") + ); + + // Third save with overwrite flag - should succeed + let mut save_store3 = SafetensorsStore::from_file(&path).overwrite(true); + save_store3.collect_from(&module).unwrap(); + + // Verify file still exists and is valid + let mut load_store = SafetensorsStore::from_file(&path); + let mut module2 = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + let result = load_store.apply_to(&mut module2).unwrap(); + assert!(result.is_success()); +} + +#[test] +fn test_store_overwrite_with_metadata() { + use tempfile::tempdir; + + let device = Default::default(); + let module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + // Create temp directory and file path + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("test_model_metadata.safetensors"); + + // First save with v1 metadata and overwrite enabled + let mut save_store = SafetensorsStore::from_file(&path) + .metadata("model_version", "v1") + .overwrite(true); + save_store.collect_from(&module).unwrap(); + + // Second save with v2 metadata and overwrite enabled + let mut save_store2 = SafetensorsStore::from_file(&path) + .metadata("model_version", "v2") + .overwrite(true); + save_store2.collect_from(&module).unwrap(); + + // Load and verify the metadata was updated to v2 + let mut load_store = SafetensorsStore::from_file(&path); + // Since we can't easily access metadata after loading, we just verify the file loads successfully + let mut module2 = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + let result = module2.load_from(&mut load_store).unwrap(); + assert!(result.is_success()); +} + +#[test] +fn test_forward_pass_preservation_after_save_load() { + let device = Default::default(); + + // Create model config + let config = ForwardTestModelConfig { + input_size: 4, + hidden_size: 8, + output_size: 2, + }; + + // Initialize model1 with random weights + let model1 = config.init::(&device); + + // Create random input + let input = Tensor::::random( + [1, 4], + burn_tensor::Distribution::Uniform(-1.0, 1.0), + &device, + ); + + // Forward pass with model1 -> output1 + let output1 = model1.forward(input.clone()); + + // Save model1 weights + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("forward_test_model.safetensors"); + let mut save_store = SafetensorsStore::from_file(&path); + save_store.collect_from(&model1).unwrap(); + + // Initialize model2 with different random weights + let mut model2 = config.init::(&device); + + // Forward pass with model2 -> output2 (should differ from output1) + let output2 = model2.forward(input.clone()); + + // Verify output2 differs from output1 (different random weights) + assert!( + !output1 + .clone() + .all_close(output2.clone(), Some(1e-6), Some(1e-6)), + "output2 should differ from output1 (different random initializations)" + ); + + // Load model1 weights into model2 + let mut load_store = SafetensorsStore::from_file(&path); + let result = load_store.apply_to(&mut model2).unwrap(); + assert!(result.is_success()); + assert_eq!(result.applied.len(), 4); // 2 weights + 2 biases + + // Forward pass with model2 (now has model1 weights) -> output3 + let output3 = model2.forward(input.clone()); + + // Verify output3 equals output1 (same weights) + assert!( + output1.all_close(output3, Some(1e-6), Some(1e-6)), + "output3 should equal output1 after loading weights" + ); +} + +#[test] +fn should_save_load_compose() { + let device = ::Device::default(); + let module_1 = ModuleComposed::::new(&device); + let mut module_2 = ModuleComposed::::new(&device); + assert_ne!(module_1.weight.to_data(), module_2.weight.to_data()); + assert_ne!( + module_1.basic.weight_basic.to_data(), + module_2.basic.weight_basic.to_data() + ); + + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path().join("save_load_compose.safetensors"); + let mut store = SafetensorsStore::from_file(&path); + module_1.save_into(&mut store).unwrap(); + + let mut load_store = SafetensorsStore::from_file(&path); + let result = module_2.load_from(&mut load_store).unwrap(); + assert!(result.is_success()); + + assert_eq!(module_1.weight.to_data(), module_2.weight.to_data()); + assert_eq!( + module_1.basic.weight_basic.to_data(), + module_2.basic.weight_basic.to_data() + ); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/filtering.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/filtering.rs new file mode 100644 index 0000000..c79528f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/filtering.rs @@ -0,0 +1,169 @@ +use crate::{ModuleSnapshot, SafetensorsStore}; + +use super::round_trip::ComplexModule; + +type TestBackend = burn_ndarray::NdArray; + +#[test] +#[cfg(target_has_atomic = "ptr")] +fn filtered_export_import() { + let device = Default::default(); + let module1 = ComplexModule::::new(&device); + let mut module2 = ComplexModule::::new_zeros(&device); + + // Export only encoder tensors using the builder pattern + let mut save_store = SafetensorsStore::from_bytes(None).with_regex(r"^encoder\..*"); + module1.save_into(&mut save_store).unwrap(); + + // Import filtered tensors - need to allow partial since we only saved encoder tensors + let mut load_store = SafetensorsStore::from_bytes(None).allow_partial(true); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + // Get Arc and extract data + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + let result = module2.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 3); // encoder.weight, encoder.bias, encoder.norm + assert!(!result.missing.is_empty()); // decoder and layers tensors are missing +} + +#[test] +#[cfg(target_has_atomic = "ptr")] +fn builder_pattern_filtering() { + let device = Default::default(); + let module = ComplexModule::::new(&device); + + // Test with_regex - multiple patterns (OR logic) + let mut store = SafetensorsStore::from_bytes(None) + .with_regex(r"^encoder\..*") // Match encoder tensors + .with_regex(r".*\.bias$"); // OR match any bias tensors + + let views = module.collect(None, None, false); + let filtered_count = views + .iter() + .filter(|v| { + let path = v.full_path(); + path.starts_with("encoder.") || path.ends_with(".bias") + }) + .count(); + + module.save_into(&mut store).unwrap(); + + // Verify we saved the expected number of tensors + if let SafetensorsStore::Memory(ref p) = store { + let data = p.data().unwrap(); + let tensors = safetensors::SafeTensors::deserialize(&data).unwrap(); + assert_eq!(tensors.len(), filtered_count); + } +} + +#[test] +fn builder_pattern_exact_paths() { + let device = Default::default(); + let module = ComplexModule::::new(&device); + + // Test with_full_path and with_full_paths + let paths = vec!["encoder.weight", "decoder.scale"]; + let mut store = SafetensorsStore::from_bytes(None) + .with_full_path("encoder.norm") + .with_full_paths(paths.clone()); + + module.save_into(&mut store).unwrap(); + + // Verify only specified tensors were saved + if let SafetensorsStore::Memory(ref p) = store { + let data = p.data().unwrap(); + let tensors = safetensors::SafeTensors::deserialize(&data).unwrap(); + assert_eq!(tensors.len(), 3); // encoder.norm + encoder.weight + decoder.scale + + for (name, _) in tensors.tensors() { + assert!(name == "encoder.norm" || name == "encoder.weight" || name == "decoder.scale"); + } + } +} + +#[test] +fn builder_pattern_with_predicate() { + let device = Default::default(); + let module = ComplexModule::::new(&device); + + // Test with_predicate - custom logic + let mut store = SafetensorsStore::from_bytes(None).with_predicate(|path, _| { + // Only save tensors with "layer" in the path and ending with "weight" + path.contains("layer") && path.ends_with("weight") + }); + + module.save_into(&mut store).unwrap(); + + // Verify only layer weights were saved + if let SafetensorsStore::Memory(ref p) = store { + let data = p.data().unwrap(); + let tensors = safetensors::SafeTensors::deserialize(&data).unwrap(); + + for (name, _) in tensors.tensors() { + assert!(name.contains("layer")); + assert!(name.ends_with("weight")); + } + } +} + +#[test] +fn builder_pattern_combined() { + let device = Default::default(); + let module = ComplexModule::::new(&device); + + // Combine multiple filter methods + #[cfg(target_has_atomic = "ptr")] + { + let mut store = SafetensorsStore::from_bytes(None) + .with_regex(r"^encoder\..*") // All encoder tensors + .with_full_path("decoder.scale") // Plus specific decoder.scale + .with_predicate(|path, _| { + // Plus any projection tensors + path.contains("projection") + }); + + module.save_into(&mut store).unwrap(); + + if let SafetensorsStore::Memory(ref p) = store { + let data = p.data().unwrap(); + let tensors = safetensors::SafeTensors::deserialize(&data).unwrap(); + + // Should have encoder.*, decoder.scale, and projection tensors + let mut names = Vec::new(); + for (name, _) in tensors.tensors() { + names.push(name); + } + assert!(names.iter().any(|n| n == "encoder.weight")); + assert!(names.iter().any(|n| n == "encoder.bias")); + assert!(names.iter().any(|n| n == "encoder.norm")); + assert!(names.iter().any(|n| n == "decoder.scale")); + // decoder.projection.* should also be included due to predicate + assert!(names.iter().any(|n| n.contains("projection"))); + } + } +} + +#[test] +fn builder_pattern_match_all() { + let device = Default::default(); + let module = ComplexModule::::new(&device); + + let all_views = module.collect(None, None, false); + let total_count = all_views.len(); + + // Test match_all - should save everything + let mut store = SafetensorsStore::from_bytes(None).match_all(); + + module.save_into(&mut store).unwrap(); + + if let SafetensorsStore::Memory(ref p) = store { + let data = p.data().unwrap(); + let tensors = safetensors::SafeTensors::deserialize(&data).unwrap(); + assert_eq!(tensors.len(), total_count); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/integration.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/integration.rs new file mode 100644 index 0000000..00602d2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/integration.rs @@ -0,0 +1,172 @@ +use burn_core as burn; + +use crate::{ModuleSnapshot, SafetensorsStore}; +use burn_core::module::{Module, Param}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +type TestBackend = burn_ndarray::NdArray; + +// Integration tests demonstrating the SafeTensors store API +#[derive(Module, Debug)] +struct IntegrationTestModel { + encoder: IntegrationEncoderModule, + decoder: IntegrationDecoderModule, + head: IntegrationHeadModule, +} + +#[derive(Module, Debug)] +struct IntegrationEncoderModule { + layer1: IntegrationLinearLayer, + layer2: IntegrationLinearLayer, + norm: IntegrationNormLayer, +} + +#[derive(Module, Debug)] +struct IntegrationDecoderModule { + layer1: IntegrationLinearLayer, + layer2: IntegrationLinearLayer, + norm: IntegrationNormLayer, +} + +#[derive(Module, Debug)] +struct IntegrationHeadModule { + weight: Param>, + bias: Param>, +} + +#[derive(Module, Debug)] +struct IntegrationLinearLayer { + weight: Param>, + bias: Param>, +} + +#[derive(Module, Debug)] +struct IntegrationNormLayer { + scale: Param>, + shift: Param>, +} + +impl IntegrationTestModel { + fn new(device: &B::Device) -> Self { + Self { + encoder: IntegrationEncoderModule::new(device), + decoder: IntegrationDecoderModule::new(device), + head: IntegrationHeadModule::new(device), + } + } +} + +impl IntegrationEncoderModule { + fn new(device: &B::Device) -> Self { + Self { + layer1: IntegrationLinearLayer::new(device, 1), + layer2: IntegrationLinearLayer::new(device, 2), + norm: IntegrationNormLayer::new(device), + } + } +} + +impl IntegrationDecoderModule { + fn new(device: &B::Device) -> Self { + Self { + layer1: IntegrationLinearLayer::new(device, 3), + layer2: IntegrationLinearLayer::new(device, 4), + norm: IntegrationNormLayer::new(device), + } + } +} + +impl IntegrationHeadModule { + fn new(device: &B::Device) -> Self { + Self { + weight: Param::from_data([[5.0, 6.0], [7.0, 8.0]], device), + bias: Param::from_data([9.0, 10.0], device), + } + } +} + +impl IntegrationLinearLayer { + fn new(device: &B::Device, seed: i32) -> Self { + let weight_data = [ + [seed as f32, (seed + 1) as f32], + [(seed + 2) as f32, (seed + 3) as f32], + ]; + let bias_data = [(seed + 4) as f32, (seed + 5) as f32]; + + Self { + weight: Param::from_data(weight_data, device), + bias: Param::from_data(bias_data, device), + } + } +} + +impl IntegrationNormLayer { + fn new(device: &B::Device) -> Self { + Self { + scale: Param::from_data([1.0, 2.0], device), + shift: Param::from_data([0.1, 0.2], device), + } + } +} + +#[test] +fn basic_usage() { + let device = Default::default(); + let model = IntegrationTestModel::::new(&device); + + // Save using new API (format, producer and version are automatically added) + let mut save_store = SafetensorsStore::from_bytes(None).metadata("model_name", "test_model"); + + // Use collect_to method + model.save_into(&mut save_store).unwrap(); + + // Load using new API + let mut load_store = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + p.set_data(p_save.data().unwrap().as_ref().clone()); + } + + let mut target_model = IntegrationTestModel::::new(&device); + let result = target_model.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); + assert_eq!(result.applied.len(), 14); // All tensors should be applied + assert_eq!(result.errors.len(), 0); + assert_eq!(result.unused.len(), 0); +} + +#[test] +#[cfg(target_has_atomic = "ptr")] +fn with_filtering() { + let device = Default::default(); + let model = IntegrationTestModel::::new(&device); + + // Save only encoder tensors using the builder pattern + let mut save_store = SafetensorsStore::from_bytes(None) + .with_regex(r"^encoder\..*") + .metadata("subset", "encoder_only"); + + model.save_into(&mut save_store).unwrap(); + + // Load into new model - need to allow partial loading since we only saved encoder tensors + let mut load_store = SafetensorsStore::from_bytes(None).allow_partial(true); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + p.set_data(p_save.data().unwrap().as_ref().clone()); + } + + let mut target_model = IntegrationTestModel::::new(&device); + let result = target_model.load_from(&mut load_store).unwrap(); + + // Only encoder tensors should be applied + assert_eq!(result.applied.len(), 6); // encoder has 6 tensors (2 layers × 2 + norm × 2) + + // Check that only encoder tensors were applied + for tensor_name in &result.applied { + assert!(tensor_name.starts_with("encoder.")); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/metadata.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/metadata.rs new file mode 100644 index 0000000..ccb6dfd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/metadata.rs @@ -0,0 +1,115 @@ +use crate::{ModuleSnapshot, SafetensorsStore}; +use burn_nn::LinearConfig; + +type TestBackend = burn_ndarray::NdArray; + +#[test] +fn default_metadata_included() { + // Verify that default metadata is automatically included + let default_metadata = SafetensorsStore::default_metadata(); + + // Check that format, producer and version are present + assert_eq!(default_metadata.get("format").unwrap(), "safetensors"); + assert_eq!(default_metadata.get("producer").unwrap(), "burn"); + assert!(default_metadata.contains_key("version")); + + // The version should be the crate version + let version = default_metadata.get("version").unwrap(); + assert!(!version.is_empty()); +} + +#[test] +fn metadata_preservation() { + let device = Default::default(); + let module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + // Write with metadata - note that format, producer and version are automatically added + let mut save_store = SafetensorsStore::from_bytes(None) + .metadata("model_type", "linear") + .metadata("custom_field", "test_value"); + + module.save_into(&mut save_store).unwrap(); + + // Verify metadata was saved (would need to add a method to check metadata) + // For now, just verify the round trip works + let mut load_store = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + // Get Arc and extract data + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + + let mut module2 = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + let result = module2.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); +} + +#[test] +fn clear_metadata_removes_all() { + let device = Default::default(); + let module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + // Create store with custom metadata, then clear all + let mut save_store = SafetensorsStore::from_bytes(None) + .metadata("model_type", "linear") + .metadata("custom_field", "test_value") + .clear_metadata(); // Should remove all metadata including defaults + + module.save_into(&mut save_store).unwrap(); + + // Load and verify the module still works (metadata is optional) + let mut load_store = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + + let mut module2 = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + let result = module2.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); +} + +#[test] +fn clear_then_add_custom_metadata() { + let device = Default::default(); + let module = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + + // Clear all metadata, then add only custom ones + let mut save_store = SafetensorsStore::from_bytes(None) + .clear_metadata() + .metadata("only_custom", "value"); + + module.save_into(&mut save_store).unwrap(); + + // Verify round-trip works + let mut load_store = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + + let mut module2 = LinearConfig::new(4, 2) + .with_bias(true) + .init::(&device); + let result = module2.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mixed_datatypes.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mixed_datatypes.rs new file mode 100644 index 0000000..a4ee5e0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mixed_datatypes.rs @@ -0,0 +1,442 @@ +use burn_core as burn; + +use burn_core::module::{Module, Param, ParamId}; +use burn_nn as nn; +use burn_tensor::{Bool, Int, Tensor, backend::Backend}; + +use crate::{ModuleSnapshot, SafetensorsStore}; + +/// Simple model with different data types for testing +#[derive(Module, Debug)] +pub struct MixedDtypeModel { + // Standard neural network layers (float tensors) + linear: nn::Linear, + + // Direct tensor parameters of different types + float_tensor: Param>, + + int_tensor: Param>, + + bool_tensor: Param>, +} + +impl MixedDtypeModel { + pub fn new(device: &B::Device) -> Self { + Self { + linear: nn::LinearConfig::new(3, 3).init(device), + + // Simple float values + float_tensor: Param::from_tensor(Tensor::from_floats( + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + device, + )), + + // Simple integer values + int_tensor: Param::initialized( + ParamId::new(), + Tensor::from_ints([[1, 2, 3], [4, 5, 6]], device), + ), + + // Simple boolean values + bool_tensor: Param::initialized( + ParamId::new(), + Tensor::from_bool( + burn::tensor::TensorData::new( + vec![true, false, true, false, true, false], + [2, 3], + ), + device, + ), + ), + } + } +} + +#[cfg(test)] +#[allow(clippy::excessive_precision)] +mod tests { + use super::*; + + #[test] + fn test_mixed_dtypes_round_trip() { + type TestBackend = burn_ndarray::NdArray; + let device = Default::default(); + + // Create model with mixed data types + let model = MixedDtypeModel::::new(&device); + + // Save to bytes + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).expect("Failed to save"); + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load into a new model + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = MixedDtypeModel::::new(&device); + loaded_model + .load_from(&mut load_store) + .expect("Failed to load"); + + // Verify float tensor is preserved + let orig_float = model.float_tensor.val().into_data(); + let loaded_float = loaded_model.float_tensor.val().into_data(); + assert_eq!(orig_float, loaded_float, "Float tensor not preserved"); + + // Verify integer tensor is preserved + let orig_int = model.int_tensor.val().into_data(); + let loaded_int = loaded_model.int_tensor.val().into_data(); + assert_eq!(orig_int, loaded_int, "Integer tensor not preserved"); + + // Verify boolean tensor is preserved + let orig_bool = model.bool_tensor.val().into_data(); + let loaded_bool = loaded_model.bool_tensor.val().into_data(); + assert_eq!(orig_bool, loaded_bool, "Boolean tensor not preserved"); + } + + #[test] + fn test_dtype_detection() { + type TestBackend = burn_ndarray::NdArray; + let device = Default::default(); + + let model = MixedDtypeModel::::new(&device); + let snapshots = model.collect(None, None, false); + + for snapshot in snapshots { + let path = snapshot.full_path(); + let dtype = snapshot.dtype; + + if path.contains("float_tensor") || path.contains("linear") { + assert_eq!( + dtype, + burn::tensor::DType::F32, + "Float tensor {} should have F32 dtype", + path + ); + } else if path.contains("int_tensor") { + assert!( + matches!( + dtype, + burn::tensor::DType::I64 + | burn::tensor::DType::I32 + | burn::tensor::DType::I16 + | burn::tensor::DType::I8 + ), + "Integer tensor {} should have integer dtype, got {:?}", + path, + dtype + ); + } else if path.contains("bool_tensor") { + assert_eq!( + dtype, + burn::tensor::DType::Bool, + "Boolean tensor {} should have Bool dtype", + path + ); + } + } + } + + #[test] + fn test_extreme_values() { + type TestBackend = burn_ndarray::NdArray; + let device = ::Device::default(); + + #[derive(Module, Debug)] + struct ExtremeValueModel { + large_floats: Param>, + small_floats: Param>, + large_ints: Param>, + } + + impl ExtremeValueModel { + fn new(device: &B::Device) -> Self { + Self { + large_floats: Param::from_tensor(Tensor::from_floats( + [1e30, -1e30, f32::MAX, f32::MIN], + device, + )), + small_floats: Param::from_tensor(Tensor::from_floats( + [1e-30, -1e-30, f32::MIN_POSITIVE, f32::EPSILON], + device, + )), + large_ints: Param::initialized( + ParamId::new(), + Tensor::from_ints([i32::MAX, i32::MIN, 0, -1], device), + ), + } + } + } + + let model = ExtremeValueModel::::new(&device); + + // Save and load + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).expect("Failed to save"); + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = ExtremeValueModel::::new(&device); + loaded_model + .load_from(&mut load_store) + .expect("Failed to load"); + + // Check exact preservation + assert_eq!( + model.large_floats.val().into_data(), + loaded_model.large_floats.val().into_data(), + "Large floats not preserved" + ); + assert_eq!( + model.small_floats.val().into_data(), + loaded_model.small_floats.val().into_data(), + "Small floats not preserved" + ); + assert_eq!( + model.large_ints.val().into_data(), + loaded_model.large_ints.val().into_data(), + "Large integers not preserved" + ); + } + + #[test] + fn test_mixed_precision_floats() { + // Note: While SafeTensors format supports storing tensors with different precisions + // (F16, BF16, F32, F64, etc.) in the same file, Burn's backend architecture currently + // requires all tensors in a model instance to share the same floating-point precision. + // This is determined at the backend level (e.g., NdArray or NdArray). + // + // However, for storage purposes, SafeTensors can correctly save and load tensors + // with their original precision, preserving the data type information in the file format. + // This test demonstrates that different precision backends work correctly with SafeTensors. + + // Test with f32 backend + { + type TestBackend = burn_ndarray::NdArray; + let device = Default::default(); + + let model = MixedDtypeModel::::new(&device); + + // Save to bytes + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).expect("Failed to save"); + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load and verify + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = MixedDtypeModel::::new(&device); + loaded_model + .load_from(&mut load_store) + .expect("Failed to load"); + + assert_eq!( + model.float_tensor.val().into_data(), + loaded_model.float_tensor.val().into_data(), + "F32 float tensor not preserved" + ); + } + + // Test with f64 backend + { + type TestBackend = burn_ndarray::NdArray; + let device = Default::default(); + + #[derive(Module, Debug)] + struct F64Model { + linear: nn::Linear, + double_precision: Param>, + } + + let model = F64Model:: { + linear: nn::LinearConfig::new(2, 2).init(&device), + double_precision: Param::from_tensor(Tensor::from_floats( + [ + [1.234567890123456789, 2.345678901234567890], + [3.456789012345678901, 4.567890123456789012], + ], + &device, + )), + }; + + // Save to bytes + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).expect("Failed to save"); + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load and verify + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = F64Model:: { + linear: nn::LinearConfig::new(2, 2).init(&device), + double_precision: Param::from_tensor(Tensor::zeros([2, 2], &device)), + }; + loaded_model + .load_from(&mut load_store) + .expect("Failed to load"); + + let orig = model.double_precision.val().into_data(); + let loaded = loaded_model.double_precision.val().into_data(); + assert_eq!(orig, loaded, "F64 double precision not preserved"); + } + } + + #[test] + fn test_mixed_precision_integers() { + type TestBackend = burn_ndarray::NdArray; + let device = Default::default(); + + #[derive(Module, Debug)] + struct MultiIntModel { + // Note: Burn's Tensor uses the backend's default int type + // We can't directly specify i8, i16, etc. in the type system + // But we can test with different values that would fit in different ranges + small_ints: Param>, // Values that fit in i8 + medium_ints: Param>, // Values that fit in i16 + large_ints: Param>, // Values that need i32/i64 + } + + let model = MultiIntModel:: { + small_ints: Param::initialized( + ParamId::new(), + Tensor::from_ints([127i32, -128, 0, 42], &device), + ), + medium_ints: Param::initialized( + ParamId::new(), + Tensor::from_ints([32767i32, -32768, 1000, -1000], &device), + ), + large_ints: Param::initialized( + ParamId::new(), + Tensor::from_ints([i32::MAX, i32::MIN, 1_000_000, -1_000_000], &device), + ), + }; + + // Save to bytes + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).expect("Failed to save"); + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load and verify + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = MultiIntModel:: { + small_ints: Param::initialized(ParamId::new(), Tensor::zeros([4], &device)), + medium_ints: Param::initialized(ParamId::new(), Tensor::zeros([4], &device)), + large_ints: Param::initialized(ParamId::new(), Tensor::zeros([4], &device)), + }; + loaded_model + .load_from(&mut load_store) + .expect("Failed to load"); + + assert_eq!( + model.small_ints.val().into_data(), + loaded_model.small_ints.val().into_data(), + "Small ints (i8 range) not preserved" + ); + assert_eq!( + model.medium_ints.val().into_data(), + loaded_model.medium_ints.val().into_data(), + "Medium ints (i16 range) not preserved" + ); + assert_eq!( + model.large_ints.val().into_data(), + loaded_model.large_ints.val().into_data(), + "Large ints (i32 range) not preserved" + ); + } + + #[test] + fn test_comprehensive_mixed_types() { + type TestBackend = burn_ndarray::NdArray; + let device = Default::default(); + + #[derive(Module, Debug)] + struct ComprehensiveModel { + // Neural network layers + linear1: nn::Linear, + conv2d: nn::conv::Conv2d, + + // Different tensor types + float32_weights: Param>, + integer_indices: Param>, + boolean_mask: Param>, + } + + let model = ComprehensiveModel:: { + linear1: nn::LinearConfig::new(4, 8).init(&device), + conv2d: nn::conv::Conv2dConfig::new([3, 16], [3, 3]).init(&device), + + float32_weights: Param::from_tensor(Tensor::from_floats( + [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], + &device, + )), + + integer_indices: Param::initialized( + ParamId::new(), + Tensor::from_ints( + [[0, 1, 2, 3], [10, 20, 30, 40], [100, 200, 300, 400]], + &device, + ), + ), + + boolean_mask: Param::initialized( + ParamId::new(), + Tensor::from_bool( + burn::tensor::TensorData::new( + vec![true, false, false, true, false, true, true, false], + [2, 4], + ), + &device, + ), + ), + }; + + // Collect all tensors + let snapshots = model.collect(None, None, false); + + // Verify we have all expected tensors + let paths: Vec = snapshots.iter().map(|s| s.full_path()).collect(); + assert!(paths.iter().any(|p| p.contains("linear1"))); + assert!(paths.iter().any(|p| p.contains("conv2d"))); + assert!(paths.iter().any(|p| p.contains("float32_weights"))); + assert!(paths.iter().any(|p| p.contains("integer_indices"))); + assert!(paths.iter().any(|p| p.contains("boolean_mask"))); + + // Save to bytes + let mut save_store = SafetensorsStore::from_bytes(None); + model.save_into(&mut save_store).expect("Failed to save"); + let bytes = save_store.get_bytes().expect("Failed to get bytes"); + + // Load into fresh model + let mut load_store = SafetensorsStore::from_bytes(Some(bytes)); + let mut loaded_model = ComprehensiveModel:: { + linear1: nn::LinearConfig::new(4, 8).init(&device), + conv2d: nn::conv::Conv2dConfig::new([3, 16], [3, 3]).init(&device), + float32_weights: Param::from_tensor(Tensor::zeros([2, 2, 2], &device)), + integer_indices: Param::initialized(ParamId::new(), Tensor::zeros([3, 4], &device)), + boolean_mask: Param::initialized( + ParamId::new(), + Tensor::from_bool( + burn::tensor::TensorData::new(vec![false; 8], [2, 4]), + &device, + ), + ), + }; + loaded_model + .load_from(&mut load_store) + .expect("Failed to load"); + + // Verify all data is preserved + assert_eq!( + model.float32_weights.val().into_data(), + loaded_model.float32_weights.val().into_data(), + "Float32 weights not preserved" + ); + assert_eq!( + model.integer_indices.val().into_data(), + loaded_model.integer_indices.val().into_data(), + "Integer indices not preserved" + ); + assert_eq!( + model.boolean_mask.val().into_data(), + loaded_model.boolean_mask.val().into_data(), + "Boolean mask not preserved" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mod.rs new file mode 100644 index 0000000..1ac3037 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/mod.rs @@ -0,0 +1,12 @@ +mod adapter; +mod direct_access; +mod error_handling; +#[cfg(feature = "std")] +mod file_io; +mod filtering; +mod integration; +mod metadata; +mod mixed_datatypes; +mod multi_layer_verify; +mod pytorch_import; +mod round_trip; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/multi_layer_verify.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/multi_layer_verify.rs new file mode 100644 index 0000000..6ca2b2f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/multi_layer_verify.rs @@ -0,0 +1,114 @@ +//! Tests for multi-layer model loading with SafeTensors format +use burn_core as burn; + +use burn_core::module::Module; +use burn_tensor::{Tensor, backend::Backend}; + +use burn_nn::{ + BatchNorm, BatchNormConfig, Linear, LinearConfig, PaddingConfig2d, Relu, + conv::{Conv2d, Conv2dConfig}, +}; + +/// Multi-layer neural network model for testing +#[derive(Module, Debug)] +pub struct Net { + conv1: Conv2d, + norm1: BatchNorm, + fc1: Linear, + relu: Relu, +} + +impl Net { + /// Create a new network instance + pub fn new(device: &B::Device) -> Self { + Self { + conv1: Conv2dConfig::new([3, 4], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .init(device), + norm1: BatchNormConfig::new(4).init(device), + fc1: LinearConfig::new(4 * 8 * 8, 16).init(device), + relu: Relu::new(), + } + } + + /// Forward pass of the model + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + let x = self.norm1.forward(x); + let x = self.relu.forward(x); + // Flatten all dimensions except the batch dimension + let x = x.flatten(1, 3); + self.fc1.forward(x) + } +} + +use crate::{ModuleSnapshot, PyTorchToBurnAdapter, SafetensorsStore}; +use burn_tensor::Tolerance; + +type TestBackend = burn_ndarray::NdArray; + +/// Path to the multi_layer.safetensors test file +fn get_safetensors_path() -> &'static str { + concat!( + env!("CARGO_MANIFEST_DIR"), + "/safetensors-tests/tests/multi_layer/multi_layer.safetensors" + ) +} + +#[test] +fn multi_layer_model() { + let device = Default::default(); + let safetensors_path = get_safetensors_path(); + + // Load model from SafeTensors file with PyTorch adapter + let mut store = SafetensorsStore::from_file(safetensors_path) + .with_from_adapter(PyTorchToBurnAdapter) + .validate(false) + .allow_partial(true); + + let mut model = Net::::new(&device); + let result = model.load_from(&mut store).unwrap(); + + // Verify loading was successful + assert!( + !result.applied.is_empty(), + "Should have loaded some tensors" + ); + assert!( + result.errors.is_empty(), + "Should have no errors: {:?}", + result.errors + ); + + // Test forward pass + let input = Tensor::::ones([1, 3, 8, 8], &device); + let output = model.forward(input); + + // Expected output values from PyTorch model + let expected = Tensor::::from_data( + [[ + 0.04971555, + -0.16849735, + 0.05182848, + -0.18032673, + 0.23138367, + 0.05041867, + 0.13005908, + -0.32202929, + -0.07915690, + -0.03232457, + -0.19790289, + -0.17476529, + -0.19627589, + -0.21757686, + -0.31376451, + 0.08377837, + ]], + &device, + ); + + // Verify output matches expected values + output + .to_data() + .assert_approx_eq::(&expected.to_data(), Tolerance::default()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/pytorch_import.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/pytorch_import.rs new file mode 100644 index 0000000..9f0575e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/pytorch_import.rs @@ -0,0 +1,204 @@ +use burn_core as burn; + +use crate::{ModuleSnapshot, SafetensorsStore}; +use burn_core::module::Module; +use burn_nn::{ + BatchNorm, BatchNormConfig, Linear, LinearConfig, PaddingConfig2d, Relu, + conv::{Conv2d, Conv2dConfig}, +}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +type TestBackend = burn_ndarray::NdArray; + +#[derive(Module, Debug)] +pub struct Net { + conv1: Conv2d, + norm1: BatchNorm, + fc1: Linear, + relu: Relu, +} + +impl Net { + pub fn new(device: &B::Device) -> Self { + Self { + conv1: Conv2dConfig::new([3, 4], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .init(device), + norm1: BatchNormConfig::new(4).init(device), + fc1: LinearConfig::new(4 * 8 * 8, 16).init(device), + relu: Relu::new(), + } + } + + /// Forward pass of the model. + pub fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv1.forward(x); + let x = self.norm1.forward(x); + let x = self.relu.forward(x); + // Flatten all dimensions except the batch dimension + let x = x.flatten(1, 3); + self.fc1.forward(x) + } +} + +#[test] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +fn multi_layer_model_import() { + let device = Default::default(); + + // Reference the safetensors file from burn-import + let safetensors_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/safetensors-tests/tests/multi_layer/multi_layer.safetensors" + ); + + // Load the model using SafetensorsStore + // Note: PyTorch and Burn have different conventions for linear layer weights + // PyTorch stores as [out_features, in_features], Burn as [in_features, out_features] + // Also, tensor names may differ (e.g., PyTorch uses different names for BatchNorm params) + let mut store = SafetensorsStore::from_file(safetensors_path) + .with_from_adapter(crate::PyTorchToBurnAdapter) // Use adapter to handle PyTorch format + .allow_partial(true); // Allow partial loading due to naming differences + let mut model = Net::::new(&device); + + let result = model.load_from(&mut store).unwrap(); + + // With the adapter, weights should load correctly + assert!(!result.applied.is_empty()); + assert!( + result.errors.is_empty(), + "Should have no errors with adapter: {:?}", + result.errors + ); + + // Test forward pass with the loaded weights + // Note: Due to shape mismatches (PyTorch vs Burn conventions for linear layers), + // we can't directly compare outputs with PyTorch model. + // This test mainly verifies that the loading mechanism works. + let input = Tensor::::ones([1, 3, 8, 8], &device); + let _output = model.forward(input); + + // Verify that some tensors were loaded successfully + // Conv and BatchNorm layers should load correctly + assert!(result.applied.iter().any(|n| n.contains("conv1"))); + assert!(result.applied.iter().any(|n| n.contains("norm1"))); +} + +#[test] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +fn safetensors_round_trip_with_pytorch_model() { + let device = Default::default(); + + // Reference the safetensors file from burn-import + let safetensors_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/safetensors-tests/tests/multi_layer/multi_layer.safetensors" + ); + + // Load the model from PyTorch safetensors + let mut load_store = SafetensorsStore::from_file(safetensors_path) + .with_from_adapter(crate::PyTorchToBurnAdapter) // Use adapter to handle PyTorch format + .allow_partial(true); // Allow partial loading due to naming differences + let mut model = Net::::new(&device); + let load_result = model.load_from(&mut load_store).unwrap(); + // With the adapter, weights should load correctly + assert!(!load_result.applied.is_empty()); + assert!( + load_result.errors.is_empty(), + "Should have no errors with adapter: {:?}", + load_result.errors + ); + + // Save the model to memory + // Note: format, producer and version are automatically added + let mut save_store = SafetensorsStore::from_bytes(None).metadata("source", "pytorch"); + model.save_into(&mut save_store).unwrap(); + + // Load into a new model + let mut model2 = Net::::new(&device); + let mut load_store2 = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store2 + && let SafetensorsStore::Memory(ref p_save) = save_store + { + p.set_data(p_save.data().unwrap().as_ref().clone()); + } + + let result = model2.load_from(&mut load_store2).unwrap(); + assert!(!result.applied.is_empty()); + + // Verify both models produce the same output + let input = Tensor::::ones([1, 3, 8, 8], &device); + let output1 = model.forward(input.clone()); + let output2 = model2.forward(input); + + // Check outputs are identical + let output1_data = output1.to_data().to_vec::().unwrap(); + let output2_data = output2.to_data().to_vec::().unwrap(); + + for (a, b) in output1_data.iter().zip(output2_data.iter()) { + assert!((a - b).abs() < 1e-7, "Outputs differ after round trip"); + } +} + +#[test] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +fn partial_load_from_pytorch_model() { + let device = Default::default(); + + // Reference the safetensors file from burn-import + let safetensors_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/safetensors-tests/tests/multi_layer/multi_layer.safetensors" + ); + + // Load only conv1 and norm1 parameters (not fc1) + let mut store = SafetensorsStore::from_file(safetensors_path) + .validate(false) // Disable validation due to shape differences + .allow_partial(true); + + let mut model = Net::::new(&device); + + // Save initial fc1 weights for comparison + let _initial_fc1_weight = model.fc1.weight.val().to_data(); + + let result = model.load_from(&mut store).unwrap(); + + // Should load available tensors (with some errors due to shape mismatch) + assert!(!result.applied.is_empty()); + + // fc1 weight should remain unchanged if not in the file + // or should be updated if it is in the file + // This test verifies that partial loading works correctly +} + +#[test] +#[cfg(all(feature = "std", target_has_atomic = "ptr"))] +fn verify_tensor_names_from_pytorch() { + let device = Default::default(); + + // Reference the safetensors file from burn-import + let safetensors_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/safetensors-tests/tests/multi_layer/multi_layer.safetensors" + ); + + // Create a model and load from PyTorch + let mut model = Net::::new(&device); + let mut store = SafetensorsStore::from_file(safetensors_path) + .validate(false) // Disable validation due to shape differences + .allow_partial(true); // Allow partial loading due to naming differences + let result = model.load_from(&mut store).unwrap(); + + // Check that we loaded some tensors (with errors due to shape mismatch) + assert!(!result.applied.is_empty()); + + // Collect tensor names from the model + let views = model.collect(None, None, false); + let tensor_names: Vec = views.iter().map(|v| v.full_path()).collect(); + + // Verify expected tensor names are present + assert!(tensor_names.iter().any(|n| n.contains("conv1"))); + assert!(tensor_names.iter().any(|n| n.contains("norm1"))); + assert!(tensor_names.iter().any(|n| n.contains("fc1"))); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/round_trip.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/round_trip.rs new file mode 100644 index 0000000..9e5c7ca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/safetensors/tests/round_trip.rs @@ -0,0 +1,106 @@ +use burn_core as burn; + +use crate::{ModuleSnapshot, SafetensorsStore}; +use burn_core::module::{Module, Param}; +use burn_nn::{Linear, LinearConfig}; +use burn_tensor::Tensor; +use burn_tensor::backend::Backend; + +type TestBackend = burn_ndarray::NdArray; + +#[derive(Module, Debug)] +pub(super) struct ComplexModule { + pub encoder: EncoderModule, + pub decoder: DecoderModule, + pub layers: Vec>, +} + +#[derive(Module, Debug)] +pub(super) struct EncoderModule { + pub weight: Param>, + pub bias: Param>, + pub norm: Param>, +} + +#[derive(Module, Debug)] +pub(super) struct DecoderModule { + pub projection: Linear, + pub scale: Param>, +} + +impl ComplexModule { + pub fn new(device: &B::Device) -> Self { + Self { + encoder: EncoderModule { + weight: Param::from_data( + [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]], + device, + ), + bias: Param::from_data([0.1, 0.2, 0.3], device), + norm: Param::from_data([1.0, 1.0, 1.0], device), + }, + decoder: DecoderModule { + projection: LinearConfig::new(4, 2).with_bias(true).init(device), + scale: Param::from_data([[0.5, 0.5], [0.5, 0.5]], device), + }, + layers: vec![ + LinearConfig::new(3, 4).with_bias(false).init(device), + LinearConfig::new(4, 3).with_bias(true).init(device), + ], + } + } + + pub fn new_zeros(device: &B::Device) -> Self { + Self { + encoder: EncoderModule { + weight: Param::from_tensor(Tensor::zeros([2, 2, 2], device)), + bias: Param::from_tensor(Tensor::zeros([3], device)), + norm: Param::from_tensor(Tensor::zeros([3], device)), + }, + decoder: DecoderModule { + projection: LinearConfig::new(4, 2).with_bias(true).init(device), + scale: Param::from_tensor(Tensor::zeros([2, 2], device)), + }, + layers: vec![ + LinearConfig::new(3, 4).with_bias(false).init(device), + LinearConfig::new(4, 3).with_bias(true).init(device), + ], + } + } +} + +#[test] +fn complex_module_round_trip() { + let device = Default::default(); + let module1 = ComplexModule::::new(&device); + let mut module2 = ComplexModule::::new_zeros(&device); + + // Save module1 using new store API + let mut save_store = SafetensorsStore::from_bytes(None); + module1.save_into(&mut save_store).unwrap(); + + // Load into module2 + let mut load_store = SafetensorsStore::from_bytes(None); + if let SafetensorsStore::Memory(ref mut p) = load_store + && let SafetensorsStore::Memory(ref p_save) = save_store + { + // Get Arc and extract data + let data_arc = p_save.data().unwrap(); + p.set_data(data_arc.as_ref().clone()); + } + let result = module2.load_from(&mut load_store).unwrap(); + + assert!(result.is_success()); + assert!(result.applied.len() > 5); + assert_eq!(result.errors.len(), 0); + + // Verify data was imported correctly + let module2_views = module2.collect(None, None, false); + let encoder_weight = module2_views + .iter() + .find(|v| v.full_path() == "encoder.weight") + .unwrap() + .to_data() + .unwrap(); + assert_eq!(encoder_weight.shape, vec![2, 2, 2]); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/tensor_snapshot.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/tensor_snapshot.rs new file mode 100644 index 0000000..0e72df9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/tensor_snapshot.rs @@ -0,0 +1,618 @@ +use alloc::rc::Rc; +use alloc::string::String; +use alloc::string::ToString; +use alloc::vec::Vec; +use burn_core::module::ParamId; +use burn_tensor::quantization::{QPARAM_ALIGN, QuantParam, params_shape}; +use burn_tensor::{Bool, DType, Int, Shape, Tensor, TensorData, backend::Backend}; +use half::f16; + +/// Returns the byte size of a quantization parameter type. +// TODO: Add `size_bytes()` method to `QuantParam` in cubecl and use it here. +const fn quant_param_size(param: QuantParam) -> usize { + match param { + QuantParam::F32 => core::mem::size_of::(), + QuantParam::F16 | QuantParam::BF16 => core::mem::size_of::(), + QuantParam::UE8M0 | QuantParam::UE4M3 => core::mem::size_of::(), + } +} + +/// Error type for TensorSnapshot operations +#[derive(Debug, Clone)] +pub enum TensorSnapshotError { + /// I/O error occurred while loading tensor data + IoError(String), + /// Data corruption or invalid format + DataError(String), + /// Panic occurred while loading tensor data + PanicError(String), +} + +impl core::fmt::Display for TensorSnapshotError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::IoError(e) => write!(f, "I/O error: {}", e), + Self::DataError(e) => write!(f, "Data error: {}", e), + Self::PanicError(e) => write!(f, "Panic error: {}", e), + } + } +} + +impl core::error::Error for TensorSnapshotError {} + +/// A lightweight snapshot of a tensor that can lazily produce TensorData. +/// +/// TensorSnapshot stores a cloned tensor internally (which is cheap due to reference counting) +/// and only materializes the actual data when `to_data()` is called. This allows +/// efficient inspection of module structure without the overhead of copying all tensor data. +/// +/// The dtype and shape are cached for efficient access without requiring data materialization, +/// which is particularly useful for serialization formats that need metadata upfront. +pub struct TensorSnapshot { + /// Function to get tensor data when needed (Rc allows cloning) + data_fn: Rc Result>, + /// Data type of the tensor (cached for efficient access) + pub dtype: burn_tensor::DType, + /// Shape of the tensor (cached for efficient access) + pub shape: Vec, + /// Path stack representing the module hierarchy + pub path_stack: Option>, + /// Container stack representing the container types at each level + pub container_stack: Option>, + /// Unique identifier for the tensor parameter + pub tensor_id: Option, +} + +impl TensorSnapshot { + /// Create a new tensor snapshot from a float tensor + pub fn from_float( + tensor: &Tensor, + path_stack: Vec, + container_stack: Vec, + tensor_id: ParamId, + ) -> Self { + let dtype = tensor.dtype(); + let shape = tensor.shape().to_vec(); + let tensor = tensor.clone(); // Clone is cheap (reference counted) + Self { + data_fn: Rc::new(move || Ok(tensor.to_data())), + dtype, + shape, + path_stack: Some(path_stack), + container_stack: Some(container_stack), + tensor_id: Some(tensor_id), + } + } + + /// Create a new tensor snapshot from an int tensor + pub fn from_int( + tensor: &Tensor, + path_stack: Vec, + container_stack: Vec, + tensor_id: ParamId, + ) -> Self { + let dtype = tensor.dtype(); + let shape = tensor.shape().to_vec(); + let tensor = tensor.clone(); // Clone is cheap (reference counted) + Self { + data_fn: Rc::new(move || Ok(tensor.to_data())), + dtype, + shape, + path_stack: Some(path_stack), + container_stack: Some(container_stack), + tensor_id: Some(tensor_id), + } + } + + /// Create a new tensor snapshot from a bool tensor + pub fn from_bool( + tensor: &Tensor, + path_stack: Vec, + container_stack: Vec, + tensor_id: ParamId, + ) -> Self { + let dtype = tensor.dtype(); + let shape = tensor.shape().to_vec(); + let tensor = tensor.clone(); // Clone is cheap (reference counted) + Self { + data_fn: Rc::new(move || Ok(tensor.to_data())), + dtype, + shape, + path_stack: Some(path_stack), + container_stack: Some(container_stack), + tensor_id: Some(tensor_id), + } + } + + /// Convert to TensorData (this is where actual data copy happens) + #[cfg(feature = "std")] + pub fn to_data(&self) -> Result { + // Use AssertUnwindSafe since we're working with Rc which is not UnwindSafe + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (self.data_fn)())).unwrap_or_else( + |_| { + Err(TensorSnapshotError::PanicError( + "Panic occurred while loading tensor data".to_string(), + )) + }, + ) + } + + /// Convert to TensorData (this is where actual data copy happens) + #[cfg(not(feature = "std"))] + pub fn to_data(&self) -> Result { + (self.data_fn)() // Can't catch panics in no-std, do it when core::panic::AssertUnwindSafe is available + } + + /// Get the full path by joining the path stack + pub fn full_path(&self) -> String { + self.path_stack + .as_ref() + .map(|stack| stack.join(".")) + .unwrap_or_default() + } + + /// Get the full container path by joining the container stack + pub fn container_path(&self) -> String { + self.container_stack + .as_ref() + .map(|stack| stack.join(".")) + .unwrap_or_default() + } + + /// Get the module type (last Struct/Enum in the hierarchy) + /// + /// Returns the last user-defined module type, skipping primitive containers + /// like "Vec", "Array". This is useful for determining which user-defined + /// module a tensor belongs to. + /// + /// # Examples + /// - `Linear.weight` → `Some("Struct:Linear")` + /// - `Vec[0].weight` → `Some("Struct:Linear")` + /// - `Linear.bias` (Optional) → `Some("Struct:Linear")` + /// - `Vec[0]` (no module) → `None` + pub fn module_type(&self) -> Option { + self.container_stack.as_ref().and_then(|stack| { + // Find the last user-defined type (Struct: or Enum:) + stack + .iter() + .rev() + .find(|ct| ct.starts_with("Struct:") || ct.starts_with("Enum:")) + .cloned() + }) + } + + /// Get the immediate container type (last in the container stack) + /// + /// Returns the last element in the container stack, which could be a + /// user-defined type ("Struct:", "Enum:") or a collection type ("Vec", "Array"). + /// This is useful for understanding the full container hierarchy. + /// + /// # Examples + /// - `Linear.weight` → `"Struct:Linear"` + /// - `Vec[0].weight` → `"Struct:Linear"` (the Linear, not the Vec) + /// - `Vec[0]` → `"Vec"` + pub fn container_type(&self) -> String { + self.container_stack + .as_ref() + .and_then(|stack| stack.last()) + .cloned() + .unwrap_or_else(|| "Unknown".to_string()) + } + + /// Create a TensorSnapshot from a closure that produces TensorData + /// This is used internally for lazy loading + pub fn from_closure( + data_fn: Rc Result>, + dtype: burn_tensor::DType, + shape: Vec, + path_stack: Vec, + container_stack: Vec, + tensor_id: ParamId, + ) -> Self { + Self { + data_fn, + dtype, + shape, + path_stack: Some(path_stack), + container_stack: Some(container_stack), + tensor_id: Some(tensor_id), + } + } + + /// Create a TensorSnapshot from TensorData directly + pub fn from_data( + data: TensorData, + path_stack: Vec, + container_stack: Vec, + tensor_id: ParamId, + ) -> Self { + let dtype = data.dtype; + let shape = data.shape.clone(); + Self { + data_fn: Rc::new(move || Ok(data.clone())), + dtype, + shape, + path_stack: Some(path_stack), + container_stack: Some(container_stack), + tensor_id: Some(tensor_id), + } + } + + /// Get the size of the tensor data in bytes without materializing it. + /// + /// For regular (non-quantized) types, this is simply `shape.product() * dtype.size()`. + /// + /// For quantized types (`QFloat`), this accounts for: + /// - The quantized values (packed according to the quantization scheme) + /// - Alignment padding (values are aligned to 4-byte boundary) + /// - Quantization parameters (scale values appended to the data) + pub fn data_len(&self) -> usize { + const BITS_PER_BYTE: usize = 8; + + let num_elements: usize = self.shape.iter().product(); + + match self.dtype { + DType::QFloat(scheme) => { + // Calculate value bytes using scheme's packing information + let num_storage_elements = num_elements.div_ceil(scheme.num_quants()); + let value_bytes = + num_storage_elements * (scheme.size_bits_stored() / BITS_PER_BYTE); + + // Calculate number of quantization parameters (scales) + let num_params = + params_shape(&Shape::from(self.shape.clone()), scheme.level).num_elements(); + + let aligned_value_bytes = value_bytes.div_ceil(QPARAM_ALIGN) * QPARAM_ALIGN; + let scale_bytes = num_params * quant_param_size(scheme.param); + + aligned_value_bytes + scale_bytes + } + _ => num_elements * self.dtype.size(), + } + } + + /// Clone the data function for lazy composition + pub fn clone_data_fn(&self) -> Rc Result> { + self.data_fn.clone() + } +} + +impl Clone for TensorSnapshot { + fn clone(&self) -> Self { + // Clone lazily - keep the same data function + Self { + data_fn: self.data_fn.clone(), + dtype: self.dtype, + shape: self.shape.clone(), + path_stack: self.path_stack.clone(), + container_stack: self.container_stack.clone(), + tensor_id: self.tensor_id, + } + } +} + +impl core::fmt::Debug for TensorSnapshot { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TensorSnapshot") + .field("dtype", &self.dtype) + .field("shape", &self.shape) + .field("path_stack", &self.path_stack) + .field("container_stack", &self.container_stack) + .field("tensor_id", &self.tensor_id) + .finish() + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + type TestBackend = burn_ndarray::NdArray; + use alloc::string::ToString; + use burn_tensor::DType; + + #[test] + fn tensor_view_float() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + + let snapshot = TensorSnapshot::from_float( + &tensor, + vec!["test".to_string(), "weight".to_string()], + vec!["TestModule".to_string(), "Param".to_string()], + ParamId::new(), + ); + + // Test metadata access without materialization + assert_eq!(snapshot.dtype, DType::F32); + assert_eq!(snapshot.shape, vec![2, 2]); + assert_eq!(snapshot.full_path(), "test.weight"); + assert_eq!(snapshot.container_path(), "TestModule.Param"); + + // Test data materialization + let data = snapshot.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 2]); + assert_eq!(data.dtype, DType::F32); + } + + #[test] + fn tensor_view_int() { + let device = Default::default(); + let tensor = Tensor::::from_data([[1, 2], [3, 4]], &device); + + let snapshot = TensorSnapshot::from_int( + &tensor, + vec!["test".to_string(), "int".to_string()], + vec!["TestModule".to_string(), "Param".to_string()], + ParamId::new(), + ); + + // Test metadata access without materialization + // TestBackend uses I64 for integers + assert_eq!(snapshot.dtype, DType::I64); + assert_eq!(snapshot.shape, vec![2, 2]); + + let data = snapshot.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 2]); + assert_eq!(data.dtype, DType::I64); + } + + #[test] + fn tensor_view_bool() { + let device = Default::default(); + let tensor = + Tensor::::from_data([[true, false], [false, true]], &device); + + let snapshot = TensorSnapshot::from_bool( + &tensor, + vec!["test".to_string(), "bool".to_string()], + vec!["TestModule".to_string(), "Param".to_string()], + ParamId::new(), + ); + + // Test metadata access without materialization + assert_eq!(snapshot.dtype, DType::Bool); + assert_eq!(snapshot.shape, vec![2, 2]); + + let data = snapshot.to_data().unwrap(); + assert_eq!(data.shape, vec![2, 2]); + assert_eq!(data.dtype, DType::Bool); + } + + #[test] + fn data_len() { + let device = Default::default(); + + // Test F32 tensor (4 bytes per element) + let tensor_f32 = Tensor::::from_data([[1.0, 2.0], [3.0, 4.0]], &device); + let view_f32 = TensorSnapshot::from_float( + &tensor_f32, + vec!["test".to_string()], + vec!["Module".to_string()], + ParamId::new(), + ); + assert_eq!(view_f32.data_len(), 16); // 4 elements * 4 bytes + + // Test I64 tensor (8 bytes per element) - TestBackend uses I64 for Int + let tensor_i64 = + Tensor::::from_data([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], &device); + let view_i64 = TensorSnapshot::from_int( + &tensor_i64, + vec!["test".to_string()], + vec!["Module".to_string()], + ParamId::new(), + ); + assert_eq!(view_i64.data_len(), 64); // 8 elements * 8 bytes (I64) + + // Test Bool tensor (1 byte per element) + let tensor_bool = + Tensor::::from_data([[true, false], [false, true]], &device); + let view_bool = TensorSnapshot::from_bool( + &tensor_bool, + vec!["test".to_string()], + vec!["Module".to_string()], + ParamId::new(), + ); + assert_eq!(view_bool.data_len(), 4); // 4 elements * 1 byte + } + + #[test] + fn from_closure() { + let data = TensorData::from([1.0f32, 2.0, 3.0, 4.0]); + let dtype = data.dtype; + let shape = data.shape.clone(); + + let snapshot = TensorSnapshot::from_closure( + Rc::new(move || Ok(data.clone())), + dtype, + shape.clone(), + vec!["model".to_string(), "layer".to_string()], + vec!["Model".to_string(), "Layer".to_string()], + ParamId::new(), + ); + + // Test metadata access + assert_eq!(snapshot.dtype, DType::F32); + assert_eq!(snapshot.shape, vec![4]); + assert_eq!(snapshot.full_path(), "model.layer"); + assert_eq!(snapshot.data_len(), 16); // 4 * 4 bytes + + // Test data materialization + let materialized = snapshot.to_data().unwrap(); + assert_eq!(materialized.shape, vec![4]); + } + + #[test] + fn from_data() { + let data = TensorData::from([1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0]); + let original_dtype = data.dtype; + let original_shape = data.shape.clone(); + + let snapshot = TensorSnapshot::from_data( + data, + vec!["encoder".to_string(), "weight".to_string()], + vec!["Struct:Encoder".to_string(), "Struct:Dense".to_string()], + ParamId::new(), + ); + + // Test metadata + assert_eq!(snapshot.dtype, original_dtype); + assert_eq!(snapshot.shape, original_shape); + assert_eq!(snapshot.full_path(), "encoder.weight"); + assert_eq!(snapshot.container_type(), "Struct:Dense"); + assert_eq!(snapshot.data_len(), 24); // 6 * 4 bytes + + // Test data materialization + let materialized = snapshot.to_data().unwrap(); + assert_eq!(materialized.shape, original_shape); + } + + #[test] + #[cfg(feature = "std")] + fn panic_catching_in_to_data() { + use alloc::rc::Rc; + + // Create a TensorSnapshot with a closure that panics + let snapshot = TensorSnapshot { + data_fn: Rc::new(|| panic!("Test panic in data_fn")), + dtype: DType::F32, + shape: vec![2, 2], + path_stack: Some(vec!["test".to_string()]), + container_stack: Some(vec!["Test".to_string()]), + tensor_id: Some(ParamId::new()), + }; + + // When std is available, to_data should catch the panic and return an error + let result = snapshot.to_data(); + assert!(result.is_err()); + + match result { + Err(TensorSnapshotError::PanicError(msg)) => { + assert!(msg.contains("Panic occurred")); + } + _ => panic!("Expected PanicError with panic message"), + } + } + + #[test] + fn error_propagation_in_closure() { + use alloc::rc::Rc; + + // Create a snapshot with a closure that returns an error + let snapshot = TensorSnapshot::from_closure( + Rc::new(|| Err(TensorSnapshotError::IoError("Simulated IO error".into()))), + DType::F32, + vec![2, 2], + vec!["error_test".into()], + vec![], + ParamId::new(), + ); + + // Should return an error when trying to get data + let result = snapshot.to_data(); + assert!(result.is_err()); + match result { + Err(TensorSnapshotError::IoError(msg)) => { + assert!(msg.contains("Simulated IO error")); + } + _ => panic!("Expected IoError"), + } + } + + #[test] + fn container_type_extraction() { + let device = Default::default(); + let tensor = Tensor::::from_data([1.0, 2.0, 3.0], &device); + + let snapshot = TensorSnapshot::from_float( + &tensor, + vec![ + "model".to_string(), + "layer1".to_string(), + "weight".to_string(), + ], + vec![ + "Struct:Model".to_string(), + "Struct:Conv2d".to_string(), + "Struct:Param".to_string(), + ], + ParamId::new(), + ); + + assert_eq!(snapshot.container_type(), "Struct:Param"); + assert_eq!(snapshot.module_type(), Some("Struct:Param".to_string())); + assert_eq!( + snapshot.container_path(), + "Struct:Model.Struct:Conv2d.Struct:Param" + ); + assert_eq!(snapshot.full_path(), "model.layer1.weight"); + } + + #[test] + fn container_type_vs_module_type() { + let device = Default::default(); + let tensor = Tensor::::from_data([1.0, 2.0, 3.0], &device); + + // Test case 1: Tensor inside a Vec + // container_stack: ["Struct:Model", "Vec", "Struct:Linear"] + let snapshot = TensorSnapshot::from_float( + &tensor, + vec![ + "model".to_string(), + "layers".to_string(), + "0".to_string(), + "weight".to_string(), + ], + vec![ + "Struct:Model".to_string(), + "Vec".to_string(), + "Struct:Linear".to_string(), + ], + ParamId::new(), + ); + + // container_type() returns the last element (Struct:Linear in this case) + assert_eq!(snapshot.container_type(), "Struct:Linear"); + // module_type() also returns Some(Struct:Linear) (skipping Vec) + assert_eq!(snapshot.module_type(), Some("Struct:Linear".to_string())); + + // Test case 2: Tensor that's just in a Vec + // container_stack: ["Vec"] + let snapshot2 = TensorSnapshot::from_float( + &tensor, + vec!["data".to_string(), "0".to_string()], + vec!["Vec".to_string()], + ParamId::new(), + ); + + // container_type() returns Vec + assert_eq!(snapshot2.container_type(), "Vec"); + // module_type() returns None (no Struct/Enum found) + assert_eq!(snapshot2.module_type(), None); + + // Test case 3: Nested collections + // container_stack: ["Struct:Model", "Vec", "Array", "Struct:Linear"] + let snapshot3 = TensorSnapshot::from_float( + &tensor, + vec![ + "model".to_string(), + "layers".to_string(), + "0".to_string(), + "sublayers".to_string(), + "1".to_string(), + "weight".to_string(), + ], + vec![ + "Struct:Model".to_string(), + "Vec".to_string(), + "Array".to_string(), + "Struct:Linear".to_string(), + ], + ParamId::new(), + ); + + // container_type() returns the immediate container + assert_eq!(snapshot3.container_type(), "Struct:Linear"); + // module_type() returns the last Struct/Enum + assert_eq!(snapshot3.module_type(), Some("Struct:Linear".to_string())); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-store/src/traits.rs b/crates/stable-diffusion-burn/burn-crates/burn-store/src/traits.rs new file mode 100644 index 0000000..bb4fd91 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-store/src/traits.rs @@ -0,0 +1,288 @@ +use alloc::boxed::Box; +use alloc::collections::BTreeMap; +use alloc::string::String; +use alloc::vec::Vec; + +use super::applier::Applier; +use super::apply_result::ApplyResult; +use crate::collector::Collector; +use crate::{ModuleAdapter, PathFilter, TensorSnapshot}; +use burn_core::module::Module; +use burn_tensor::backend::Backend; + +/// Extension trait for modules that provides tensor storage functionality. +/// +/// This trait provides convenient methods to collect and apply tensor snapshots from any Burn module. +/// Collection operations create lightweight tensor snapshots without immediately copying data. +/// Apply operations apply tensor data from snapshots to the corresponding tensors in the module. +pub trait ModuleSnapshot: Module { + /// Collects tensor snapshots for inspection without copying data. + /// + /// Returns a vector of `TensorSnapshot` objects that can lazily materialize the tensor data. + /// Each `TensorSnapshot` contains the full path accessible via `snapshot.full_path()`. + /// + /// # Arguments + /// + /// * `filter` - An optional [`PathFilter`] to determine which tensors to collect. + /// When `None`, all tensors are collected. + /// * `adapter` - Optional adapter to transform tensors based on container types. + /// Applied to all collected tensors before returning. + /// * `skip_enum_variants` - Skip enum variant names when building paths. + /// When true, paths will not include enum variant names (e.g., "feature.weight" + /// instead of "feature.BaseConv.weight"). Useful when exporting to formats + /// like PyTorch/SafeTensors that don't use enum variants. + fn collect( + &self, + filter: Option, + adapter: Option>, + skip_enum_variants: bool, + ) -> Vec { + let mut collector = Collector::new(filter, adapter, skip_enum_variants); + self.visit(&mut collector); + collector.into_tensors() + } + + /// Applies tensor snapshots to the module. + /// + /// This is the primary apply method that applies tensor data from `TensorSnapshot`s + /// to the corresponding tensors in the module. The snapshots are typically obtained + /// from `collect()` or loaded from storage. + /// + /// # Arguments + /// + /// * `snapshots` - A vector of TensorSnapshot objects + /// * `filter` - An optional [`PathFilter`] to determine which tensors to apply. + /// When `None`, all available tensors are applied. + /// * `adapter` - Optional adapter to transform tensors based on container types + /// * `skip_enum_variants` - Skip enum variant names when matching tensor paths + /// + /// # Returns + /// + /// An [`ApplyResult`] containing information about applied, skipped, missing, + /// and unused tensors, as well as any errors encountered. + /// + /// # Examples + /// + /// ```rust,ignore + /// use burn_store::PathFilter; + /// + /// // Apply all tensors + /// let result = model.apply(snapshots, None, None, false); + /// + /// // Apply only encoder tensors + /// let filter = PathFilter::new().with_regex(r"^encoder\..*"); + /// let result = model.apply(snapshots, Some(filter), None, false); + /// + /// // Apply with complex filter + /// let filter = PathFilter::new() + /// .with_regex(r"^encoder\..*") + /// .with_regex(r"^decoder\..*") + /// .with_full_path("head.weight"); + /// let result = model.apply(snapshots, Some(filter), None, false); + /// + /// // Apply with enum variant skipping (for PyTorch models) + /// let result = model.apply(snapshots, None, None, true); + /// ``` + fn apply( + &mut self, + snapshots: Vec, + filter: Option, + adapter: Option>, + skip_enum_variants: bool, + ) -> ApplyResult + where + Self: Sized, + { + let mut applier = Applier::new(snapshots, filter, adapter, skip_enum_variants); + + // Use unsafe to avoid cloning the entire module, which would double the memory usage + // We read the module out, map it, then write it back + // See https://github.com/tracel-ai/burn/issues/3754 + unsafe { + // Read the module out of self (moves it, leaving self in undefined state) + let module = core::ptr::read(self as *const Self); + + // Map the module to create a new one with updated tensors + let new_module = module.map(&mut applier); + + // Write the new module back to self + core::ptr::write(self as *mut Self, new_module); + } + + applier.into_result() + } + + /// Saves tensor snapshots into a [`ModuleStore`]. + /// + /// This method allows using a `ModuleStore` implementation to handle the + /// collection and writing logic in a configurable way. + /// + /// # Arguments + /// + /// * `store` - A mutable reference to a [`ModuleStore`] that will collect and save the tensors + fn save_into

(&self, store: &mut P) -> Result<(), P::Error> + where + P: ModuleStore, + { + store.collect_from(self) + } + + /// Loads tensor data from a [`ModuleStore`]. + /// + /// This method allows using a `ModuleStore` implementation to handle the + /// loading and application logic in a configurable way. + /// + /// # Arguments + /// + /// * `store` - A mutable reference to a [`ModuleStore`] that will load and apply tensors + fn load_from

(&mut self, store: &mut P) -> Result + where + P: ModuleStore, + { + store.apply_to(self) + } +} + +/// A trait for handling module storage operations. +/// +/// `ModuleStore` provides a unified interface for saving and loading module +/// tensor data with support for various storage formats and advanced features like filtering, +/// remapping, and metadata handling. +pub trait ModuleStore { + /// The error type that can be returned during storage operations. + /// + /// This should be a format-specific error type that provides detailed + /// information about what went wrong (e.g., I/O errors, format violations, + /// unsupported tensor types). + type Error: core::fmt::Debug + core::fmt::Display; + + /// Collect tensor data from a module and store it to storage. + /// + /// This method traverses the module structure, collects all tensor data + /// according to the store's configuration (filters, remapping, etc.), + /// and writes it to the underlying storage. + /// + /// # Arguments + /// + /// * `module` - The module to collect tensor data from. The module must + /// implement `ModuleSnapshot` to provide tensor access. + /// + /// # Returns + /// + /// * `Ok(())` - If all tensors were successfully collected and stored + /// * `Err(Self::Error)` - If an error occurred during collection or writing + fn collect_from>( + &mut self, + module: &M, + ) -> Result<(), Self::Error>; + + /// Load stored tensor data and apply it to a module. + /// + /// This method reads tensor data from storage and applies it to the provided + /// module. The operation is flexible and can handle partial matches, missing + /// tensors, and extra tensors in the storage. + /// + /// # Arguments + /// + /// * `module` - The module to apply tensor data to. The module must + /// implement `ModuleSnapshot` to allow tensor updates. + /// + /// # Returns + /// + /// * `Ok(ApplyResult)` - Detailed information about the apply operation: + /// - `applied`: List of successfully applied tensor names + /// - `missing`: Tensors expected by the module but not found in storage + /// - `skipped`: Tensors in storage that were not applied (filtered or not needed) + /// - `errors`: Non-critical errors that occurred during apply + /// * `Err(Self::Error)` - If a critical error prevented the apply operation + fn apply_to>( + &mut self, + module: &mut M, + ) -> Result; + + /// Get a single tensor snapshot by name. + /// + /// This method provides direct access to individual tensors in storage without + /// requiring a module. The returned `TensorSnapshot` uses lazy loading - tensor + /// data is only materialized when `to_data()` is called. + /// + /// **Note:** Key remapping is applied, so use the remapped name if configured. + /// Filters are NOT applied - use `apply_to()` for filtered loading. + /// + /// Results are cached after the first call for efficient repeated access. + /// + /// # Arguments + /// + /// * `name` - The tensor name/path (e.g., "encoder.layer1.weight") + /// + /// # Returns + /// + /// * `Ok(Some(&TensorSnapshot))` - Reference to the tensor snapshot if found + /// * `Ok(None)` - If no tensor with that name exists + /// * `Err(Self::Error)` - If an error occurred accessing storage + /// + /// # Example + /// + /// ```rust,ignore + /// let mut store = BurnpackStore::from_file("model.bpk"); + /// if let Some(snapshot) = store.get_snapshot("encoder.weight")? { + /// println!("Shape: {:?}", snapshot.shape); + /// println!("Dtype: {:?}", snapshot.dtype); + /// let data = snapshot.to_data()?; // Lazy load + /// } + /// ``` + fn get_snapshot(&mut self, name: &str) -> Result, Self::Error>; + + /// Get all tensor snapshots from storage as an ordered map. + /// + /// This method returns all tensors in storage as lazy-loading snapshots, + /// organized in a `BTreeMap` for efficient lookup by name. The map preserves + /// alphabetical ordering of tensor names. + /// + /// **Note:** This returns ALL tensors in storage, regardless of any filter + /// settings. Filters are only applied during `apply_to()`. Key remapping + /// IS applied, so tensor names reflect any configured remapping. + /// + /// Results are cached after the first call for efficient repeated access. + /// + /// # Returns + /// + /// * `Ok(&BTreeMap)` - Reference to all tensor snapshots + /// * `Err(Self::Error)` - If an error occurred accessing storage + /// + /// # Example + /// + /// ```rust,ignore + /// let mut store = SafetensorsStore::from_file("model.safetensors"); + /// let snapshots = store.get_all_snapshots()?; + /// for (name, snapshot) in snapshots { + /// println!("{}: {:?}", name, snapshot.shape); + /// } + /// ``` + fn get_all_snapshots(&mut self) -> Result<&BTreeMap, Self::Error>; + + /// Get all tensor names/keys in storage. + /// + /// This method returns the names of all tensors in storage. + /// Useful for inspecting storage contents or checking if specific tensors exist. + /// + /// **Note:** Returns ALL tensor names regardless of filter settings. + /// Key remapping IS applied, so names reflect any configured remapping. + /// + /// # Returns + /// + /// * `Ok(Vec)` - All tensor names in storage + /// * `Err(Self::Error)` - If an error occurred accessing storage + /// + /// # Example + /// + /// ```rust,ignore + /// let mut store = PytorchStore::from_file("model.pth"); + /// let keys = store.keys()?; + /// println!("Tensors in file: {:?}", keys); + /// ``` + fn keys(&mut self) -> Result, Self::Error>; +} + +// Blanket implementation for all modules +impl> ModuleSnapshot for M {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-tch/Cargo.toml new file mode 100644 index 0000000..86c5a00 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/Cargo.toml @@ -0,0 +1,38 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "LibTorch backend for the Burn framework using the tch bindings." +documentation = "https://docs.rs/burn-tch" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "burn-tch" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-tch" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std"] +std = ["burn-backend/std"] +doc = ["tch/doc-only"] +tracing = [ + "burn-backend/tracing", +] + +[dependencies] +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } + +libc = { workspace = true } +log = { workspace = true } +tch = { workspace = true, features = ["download-libtorch"] } +torch-sys = { workspace = true } # for build script lib dir detection + +[build-dependencies] +cc = "1.2.56" + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/README.md b/crates/stable-diffusion-burn/burn-crates/burn-tch/README.md new file mode 100644 index 0000000..559a18e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/README.md @@ -0,0 +1,246 @@ +# Burn Torch Backend + +[Burn](https://github.com/tracel-ai/burn) Torch backend + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-tch.svg)](https://crates.io/crates/burn-tch) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-tch/blob/master/README.md) + +This crate provides a Torch backend for [Burn](https://github.com/tracel-ai/burn) utilizing the +[`tch-rs`](https://github.com/LaurentMazare/tch-rs) crate, which offers a Rust interface to the +[PyTorch](https://pytorch.org/) C++ API. + +The backend supports CPU (multithreaded), [CUDA](https://pytorch.org/docs/stable/notes/cuda.html) +(multiple GPUs), and [MPS](https://pytorch.org/docs/stable/notes/mps.html) devices (MacOS). + +## Installation + +[`tch-rs`](https://github.com/LaurentMazare/tch-rs) requires the C++ PyTorch library (LibTorch) to +be available on your system. + +By default, the CPU distribution is installed for LibTorch v2.9.0 as required by `tch-rs`. + +

+CUDA + +To install the latest compatible CUDA distribution, set the `TORCH_CUDA_VERSION` environment +variable before the `tch-rs` dependency is retrieved with `cargo`. + +```shell +export TORCH_CUDA_VERSION=cu128 +``` + +On Windows: + +```powershell +$Env:TORCH_CUDA_VERSION = "cu128" +``` + +> Note: `tch` doesn't expose the downloaded libtorch directory on Windows when using the automatic +> download feature, so the `torch_cuda.dll` cannot be detected properly during build. In this case, +> you can set the `LIBTORCH` environment variable to point to the `libtorch/` folder in `torch-sys` +> `OUT_DIR` (or move the downloaded lib to a different folder and point to it). + +For example, running the validation sample for the first time could be done with the following +commands: + +```shell +export TORCH_CUDA_VERSION=cu128 +cargo run --bin cuda --release +``` + +**Important:** make sure your driver version is compatible with the selected CUDA version. A CUDA +Toolkit installation is not required since LibTorch ships with the appropriate CUDA runtimes. Having +the latest driver version is recommended, but you can always take a look at the +[toolkit driver version table](https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html#id4) +or +[minimum required driver version](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#minor-version-compatibility) +(limited feature-set, might not work with all operations). + +

+ +Once your installation is complete, you should be able to build/run your project. You can also +validate your installation by running the appropriate `cpu`, `cuda` or `mps` sample as below. + +```shell +cargo run --bin cpu --release +cargo run --bin cuda --release +cargo run --bin mps --release +``` + +_Note: no MPS distribution is available for automatic download at this time, please check out the +[manual instructions](#metal-mps)._ + +### Manual Download + +To install `tch-rs` with a different LibTorch distribution, you will have to manually download the +desired LibTorch distribution. The instructions are detailed in the sections below for each +platform. + +| Compute Platform | CPU | GPU | Linux | MacOS | Windows | Android | iOS | WASM | +| :------------------------ | :----------------------------: | :-: | :---: | :---: | :-----: | :-----: | :-: | :--: | +| [CPU](#cpu) | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| [CUDA](#cuda) | Yes [[1]](#cpu-sup) | Yes | Yes | No | Yes | No | No | No | +| [Metal (MPS)](#metal-mps) | No | Yes | No | Yes | No | No | No | No | +| Vulkan | Yes | Yes | Yes | Yes | Yes | Yes | No | No | + +[1] The LibTorch CUDA distribution also comes with CPU support. + +#### CPU + +
+🐧 Linux + +First, download the LibTorch CPU distribution. + +```shell +wget -O libtorch.zip https://download.pytorch.org/libtorch/cpu/libtorch-shared-with-deps-2.9.0%2Bcpu.zip +unzip libtorch.zip +``` + +Then, point to that installation using the `LIBTORCH` and `LD_LIBRARY_PATH` environment variables +before building `burn-tch` or a crate which depends on it. + +```shell +export LIBTORCH=/absolute/path/to/libtorch/ +export LD_LIBRARY_PATH=/absolute/path/to/libtorch/lib:$LD_LIBRARY_PATH +``` + +

+ +
+🍎 Mac + +First, download the LibTorch CPU distribution. + +```shell +wget -O libtorch.zip https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-2.9.0.zip +unzip libtorch.zip +``` + +Then, point to that installation using the `LIBTORCH` and `DYLD_LIBRARY_PATH` environment variables +before building `burn-tch` or a crate which depends on it. + +```shell +export LIBTORCH=/absolute/path/to/libtorch/ +export DYLD_LIBRARY_PATH=/absolute/path/to/libtorch/lib:$DYLD_LIBRARY_PATH +``` + +

+ +
+🪟 Windows + +First, download the LibTorch CPU distribution. + +```powershell +wget https://download.pytorch.org/libtorch/cpu/libtorch-win-shared-with-deps-2.9.0%2Bcpu.zip -OutFile libtorch.zip +Expand-Archive libtorch.zip +``` + +Then, set the `LIBTORCH` environment variable and append the library to your path as with the +PowerShell commands below before building `burn-tch` or a crate which depends on it. + +```powershell +$Env:LIBTORCH = "/absolute/path/to/libtorch/" +$Env:Path += ";/absolute/path/to/libtorch/" +``` + +

+ +#### CUDA + +LibTorch 2.9.0 currently includes binary distributions with CUDA 12.6, 12.8 or 13.0 runtimes. The +manual installation instructions are detailed below for CUDA 12.6, but can be applied to the other +CUDA versions by replacing `cu126` with the corresponding version string (e.g., `cu130`). + +
+🐧 Linux + +First, download the LibTorch CUDA 12.6 distribution. + +```shell +wget -O libtorch.zip https://download.pytorch.org/libtorch/cu126/libtorch-shared-with-deps-2.9.0%2Bcu126.zip +unzip libtorch.zip +``` + +Then, point to that installation using the `LIBTORCH` and `LD_LIBRARY_PATH` environment variables +before building `burn-tch` or a crate which depends on it. + +```shell +export LIBTORCH=/absolute/path/to/libtorch/ +export LD_LIBRARY_PATH=/absolute/path/to/libtorch/lib:$LD_LIBRARY_PATH +``` + +**Note:** make sure your CUDA installation is in your `PATH` and `LD_LIBRARY_PATH`. + +

+ +
+🪟 Windows + +First, download the LibTorch CUDA 12.6 distribution. + +```powershell +wget https://download.pytorch.org/libtorch/cu126/libtorch-win-shared-with-deps-2.9.0%2Bcu126.zip -OutFile libtorch.zip +Expand-Archive libtorch.zip +``` + +Then, set the `LIBTORCH` environment variable and append the library to your path as with the +PowerShell commands below before building `burn-tch` or a crate which depends on it. + +```powershell +$Env:LIBTORCH = "/absolute/path/to/libtorch/" +$Env:Path += ";/absolute/path/to/libtorch/" +``` + +

+ +#### Metal (MPS) + +There is no official LibTorch distribution with MPS support at this time, so the easiest alternative +is to use a PyTorch installation. This requires a Python installation. + +_Note: MPS acceleration is available on MacOS 12.3+._ + +```shell +pip install torch==2.9.0 numpy==1.26.4 setuptools +export LIBTORCH_USE_PYTORCH=1 +export DYLD_LIBRARY_PATH=/path/to/pytorch/lib:$DYLD_LIBRARY_PATH +``` + +**Note:** if `venv` is used, it should be activated during coding and building, or the compiler may +not work properly. + +## Example Usage + +For a simple example, check out any of the test programs in [`src/bin/`](./src/bin/). Each program +sets the device to use and performs a simple element-wise addition. + +For a more complete example using the `tch` backend, take a loot at the +[Burn mnist example](https://github.com/tracel-ai/burn/tree/main/examples/mnist). + +## Too many environment variables? + +Try `.cargo/config.toml` ([cargo book](https://doc.rust-lang.org/cargo/reference/config.html#env)). + +Instead of setting the environments in your shell, you can manually add them to your +`.cargo/config.toml`: + +```toml +[env] +LD_LIBRARY_PATH = "/absolute/path/to/libtorch/lib" +LIBTORCH = "/absolute/path/to/libtorch/libtorch" +``` + +Or use bash commands below: + +```bash +mkdir .cargo +cat < .cargo/config.toml +[env] +LD_LIBRARY_PATH = "/absolute/path/to/libtorch/lib:$LD_LIBRARY_PATH" +LIBTORCH = "/absolute/path/to/libtorch/libtorch" +EOF +``` + +This will automatically include the old `LD_LIBRARY_PATH` value in the new one. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/build.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/build.rs new file mode 100644 index 0000000..7fb432b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/build.rs @@ -0,0 +1,243 @@ +// The LIBTORCH environment variable can be used to specify the directory +// where libtorch has been installed. +// When not specified this script downloads the cpu version for libtorch +// and extracts it in OUT_DIR. +// +// On Linux, the TORCH_CUDA_VERSION environment variable can be used, +// like 9.0, 90, or cu90 to specify the version of CUDA to use for libtorch. + +use std::path::{Path, PathBuf}; +use std::{env, fs}; + +const PYTHON_PRINT_PYTORCH_DETAILS: &str = r" +import torch +from torch.utils import cpp_extension +print('LIBTORCH_VERSION:', torch.__version__.split('+')[0]) +print('LIBTORCH_CXX11:', torch._C._GLIBCXX_USE_CXX11_ABI) +for include_path in cpp_extension.include_paths(): + print('LIBTORCH_INCLUDE:', include_path) +for library_path in cpp_extension.library_paths(): + print('LIBTORCH_LIB:', library_path) +"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Os { + Linux, + Macos, + Windows, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +struct SystemInfo { + os: Os, + cxx11_abi: String, + libtorch_include_dirs: Vec, + libtorch_lib_dir: PathBuf, +} + +fn env_var_rerun(name: &str) -> Result { + println!("cargo:rerun-if-env-changed={name}"); + env::var(name) +} + +impl SystemInfo { + fn new() -> Option { + let os = match env::var("CARGO_CFG_TARGET_OS") + .expect("Unable to get TARGET_OS") + .as_str() + { + "linux" => Os::Linux, + "windows" => Os::Windows, + "macos" => Os::Macos, + os => panic!("unsupported TARGET_OS '{os}'"), + }; + // Locate the currently active Python binary, similar to: + // https://github.com/PyO3/maturin/blob/243b8ec91d07113f97a6fe74d9b2dcb88086e0eb/src/target.rs#L547 + let python_interpreter = match os { + Os::Windows => PathBuf::from("python.exe"), + Os::Linux | Os::Macos => { + if env::var_os("VIRTUAL_ENV").is_some() { + PathBuf::from("python") + } else { + PathBuf::from("python3") + } + } + }; + let mut libtorch_include_dirs = vec![]; + let mut libtorch_lib_dir = None; + let cxx11_abi = if env_var_rerun("LIBTORCH_USE_PYTORCH").is_ok() { + let output = std::process::Command::new(&python_interpreter) + .arg("-c") + .arg(PYTHON_PRINT_PYTORCH_DETAILS) + .output() + .expect("error running python interpreter"); + let mut cxx11_abi = None; + for line in String::from_utf8_lossy(&output.stdout).lines() { + match line.strip_prefix("LIBTORCH_CXX11: ") { + Some("True") => cxx11_abi = Some("1".to_owned()), + Some("False") => cxx11_abi = Some("0".to_owned()), + _ => {} + } + if let Some(path) = line.strip_prefix("LIBTORCH_INCLUDE: ") { + libtorch_include_dirs.push(PathBuf::from(path)) + } + if let Some(path) = line.strip_prefix("LIBTORCH_LIB: ") { + libtorch_lib_dir = Some(PathBuf::from(path)) + } + } + match cxx11_abi { + Some(cxx11_abi) => cxx11_abi, + None => panic!("no cxx11 abi returned by python {output:?}"), + } + } else { + let libtorch = Self::prepare_libtorch_dir(os)?; + let includes = env_var_rerun("LIBTORCH_INCLUDE") + .map(PathBuf::from) + .unwrap_or_else(|_| libtorch.clone()); + let lib = env_var_rerun("LIBTORCH_LIB") + .map(PathBuf::from) + .unwrap_or_else(|_| libtorch.clone()); + libtorch_include_dirs.push(includes.join("include")); + libtorch_include_dirs.push(includes.join("include/torch/csrc/api/include")); + if lib.ends_with("lib") { + // DEP_TCH_LIBTORCH_LIB might already point to /lib + libtorch_lib_dir = Some(lib); + } else { + libtorch_lib_dir = Some(lib.join("lib")); + } + env_var_rerun("LIBTORCH_CXX11_ABI").unwrap_or_else(|_| "1".to_owned()) + }; + let libtorch_lib_dir = libtorch_lib_dir?; + Some(Self { + os, + cxx11_abi, + libtorch_include_dirs, + libtorch_lib_dir, + }) + } + + fn check_system_location(os: Os) -> Option { + match os { + Os::Linux => Path::new("/usr/lib/libtorch.so") + .exists() + .then(|| PathBuf::from("/usr")), + _ => None, + } + } + + fn prepare_libtorch_dir(os: Os) -> Option { + if let Ok(libtorch) = env_var_rerun("DEP_TCH_LIBTORCH_LIB") { + Some(PathBuf::from(libtorch)) + } else if let Ok(libtorch) = env_var_rerun("LIBTORCH") { + Some(PathBuf::from(libtorch)) + } else if let Some(pathbuf) = Self::check_system_location(os) { + Some(pathbuf) + } else { + check_out_dir() + } + } + + fn make(&self, use_cuda: bool, use_hip: bool) { + let cuda_dependency = if use_cuda || use_hip { + "src/cuda_hack/dummy_cuda_dependency.cpp" + } else { + "src/cuda_hack/fake_cuda_dependency.cpp" + }; + println!("cargo:rerun-if-changed={cuda_dependency}"); + + match self.os { + Os::Linux | Os::Macos => { + cc::Build::new() + .cpp(true) + .pic(true) + .warnings(false) + .includes(&self.libtorch_include_dirs) + .flag(format!("-Wl,-rpath={}", self.libtorch_lib_dir.display())) + .flag("-std=c++17") + .flag(format!("-D_GLIBCXX_USE_CXX11_ABI={}", self.cxx11_abi)) + .files(&[cuda_dependency]) + .compile("burn-tch"); + } + Os::Windows => { + cc::Build::new() + .cpp(true) + .pic(true) + .warnings(false) + .includes(&self.libtorch_include_dirs) + .flag("/std:c++17") + .files(&[cuda_dependency]) + .compile("burn-tch"); + } + }; + } + + fn make_cpu() { + let cuda_dependency = "src/cuda_hack/fake_cuda_dependency.cpp"; + println!("cargo:rerun-if-changed={cuda_dependency}"); + + let os = env::var("CARGO_CFG_TARGET_OS").expect("Unable to get TARGET_OS"); + + match os.as_str() { + "windows" => { + cc::Build::new() + .cpp(true) + .pic(true) + .warnings(false) + .flag("/std:c++17") + .files(&[cuda_dependency]) + .compile("burn-tch"); + } + _ => { + cc::Build::new() + .cpp(true) + .pic(true) + .warnings(false) + .flag("-std=c++17") + .files(&[cuda_dependency]) + .compile("tch"); + } + }; + } +} + +fn check_out_dir() -> Option { + let out_dir = env_var_rerun("OUT_DIR").ok()?; + let libtorch_dir = PathBuf::from(out_dir).join("libtorch"); + libtorch_dir.exists().then_some(libtorch_dir) +} + +fn main() { + let system_info = SystemInfo::new(); + let out_dir = env_var_rerun("OUT_DIR").expect("Failed to get out dir"); + + let mut gpu_found = false; + let found_dir = system_info.is_some(); + if let Some(system_info) = &system_info { + let si_lib = &system_info.libtorch_lib_dir; + let use_cuda = + si_lib.join("libtorch_cuda.so").exists() || si_lib.join("torch_cuda.dll").exists(); + let use_hip = + si_lib.join("libtorch_hip.so").exists() || si_lib.join("torch_hip.dll").exists(); + + system_info.make(use_cuda, use_hip); + gpu_found = use_cuda || use_hip; + } else { + SystemInfo::make_cpu(); + } + let check_file = PathBuf::from(out_dir).join("tch_gpu_check.rs"); + if gpu_found { + fs::write(check_file, "#[allow(clippy::no_effect)]\n()").unwrap(); + } else { + let message = if !found_dir { + r#"Could not find libtorch dir. + + If you are trying to use the automatically downloaded version, the path is not directly available on Windows. Instead, try setting the `LIBTORCH` environment variable for the manual download instructions. + + If the library has already been downloaded in the torch-sys OUT_DIR, you can point the variable to this path (or move the downloaded lib and point to it)."# + } else { + "No libtorch_cuda or libtorch_hip found. Download the GPU version of libtorch to use a GPU device" + }; + fs::write(check_file, format!("panic!(\"{message}\")")).unwrap(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/backend.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/backend.rs new file mode 100644 index 0000000..2d12bf4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/backend.rs @@ -0,0 +1,175 @@ +use std::marker::PhantomData; + +use crate::IntoKind; + +use super::TchTensor; +use super::element::TchElement; +use burn_backend::backend::{Backend, DeviceId, DeviceOps, ExecutionError}; +use burn_backend::ops::IntTensorOps; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// The device struct when using the `tch` backend. +/// +/// Note that you need to provide the device index when using Cuda. +/// +/// # Example +/// +/// ```no_run +/// use burn_tch::LibTorchDevice; +/// +/// let device_gpu_1 = LibTorchDevice::Cuda(0); // First GPU +/// let device_gpu_2 = LibTorchDevice::Cuda(1); // Second GPU +/// let device_cpu = LibTorchDevice::Cpu; // CPU +/// let device_mps = LibTorchDevice::Mps; // Metal Performance Shaders +/// let device_vulkan = LibTorchDevice::Vulkan; // Vulkan +/// ``` +#[derive(Default)] +pub enum LibTorchDevice { + /// CPU device. + #[default] + Cpu, + + /// Cuda device with the given index. The index is the index of the Cuda device in the list of + /// all Cuda devices found on the system. + Cuda(usize), + + /// Metal Performance Shaders device. + Mps, + + /// Vulkan device. + Vulkan, +} + +impl From for tch::Device { + #[allow( + unreachable_code, + reason = "CUDA branch always panics if the library is missing" + )] + fn from(device: LibTorchDevice) -> Self { + match device { + LibTorchDevice::Cpu => tch::Device::Cpu, + LibTorchDevice::Cuda(_num) => { + include!(concat!(env!("OUT_DIR"), "/tch_gpu_check.rs")); + tch::Device::Cuda(_num) + } + LibTorchDevice::Mps => tch::Device::Mps, + LibTorchDevice::Vulkan => tch::Device::Vulkan, + } + } +} + +impl From for LibTorchDevice { + fn from(device: tch::Device) -> Self { + match device { + tch::Device::Cpu => LibTorchDevice::Cpu, + tch::Device::Cuda(num) => LibTorchDevice::Cuda(num), + tch::Device::Mps => LibTorchDevice::Mps, + tch::Device::Vulkan => LibTorchDevice::Vulkan, + } + } +} + +impl burn_backend::Device for LibTorchDevice { + fn from_id(device_id: DeviceId) -> Self { + match device_id.type_id { + 0 => Self::Cuda(device_id.index_id as usize), + 1 => Self::Mps, + 2 => Self::Cpu, + 3 => Self::Vulkan, + _ => LibTorchDevice::Cpu, + } + } + + fn to_id(&self) -> DeviceId { + match self { + LibTorchDevice::Cuda(index) => DeviceId::new(0, *index as u32), + LibTorchDevice::Mps => DeviceId::new(1, 0), + LibTorchDevice::Cpu => DeviceId::new(2, 0), + LibTorchDevice::Vulkan => DeviceId::new(3, 0), + } + } + + fn device_count(_type_id: u16) -> usize { + // TODO: Somehow find the info using the tch API. + 1 + } +} + +impl DeviceOps for LibTorchDevice {} + +/// Tensor backend that uses `LibTorch` with the [tch] crate for executing tensor operations. +/// +/// This backend is compatible with a wide range of hardwares ranging from CPUs to GPUs, but +/// requires `LibTorch` to be installed correctly. The CPU version can be downloaded +/// automatically and the CUDA version as well by setting the `TORCH_CUDA_VERSION` environment +/// variable. For more complex configurations, check out the manual installation for +/// [burn-tch](https://github.com/tracel-ai/burn/tree/main/crates/burn-tch). +/// +/// Refer to the [tch] crate for more information. +#[derive(Clone, Copy, Default, Debug)] +pub struct LibTorch { + _e: PhantomData, +} + +impl Backend for LibTorch { + type Device = LibTorchDevice; + + type FloatTensorPrimitive = TchTensor; + type FloatElem = E; + + type IntTensorPrimitive = TchTensor; + type IntElem = i64; + type BoolTensorPrimitive = TchTensor; + type BoolElem = bool; + + type QuantizedTensorPrimitive = TchTensor; + + fn seed(_device: &Self::Device, seed: u64) { + tch::manual_seed(seed as i64); + } + + fn ad_enabled(_device: &Self::Device) -> bool { + false + } + + fn name(device: &Self::Device) -> String { + match device { + LibTorchDevice::Cpu => "libtorch", + LibTorchDevice::Cuda(_) => "libtorch", + LibTorchDevice::Mps => "libtorch", + LibTorchDevice::Vulkan => "libtorch", + } + .to_string() + } + + fn sync(device: &Self::Device) -> Result<(), ExecutionError> { + match device { + LibTorchDevice::Cpu => (), + LibTorchDevice::Cuda(index) => { + tch::Cuda::synchronize(*index as i64); + } + _ => { + // When there is no explicit way to synchronize, we write and read one value to sync + burn_backend::read_sync(Self::int_into_data(Self::int_zeros( + [1].into(), + device, + E::dtype().into(), + ))) + .unwrap(); + } + }; + + Ok(()) + } + + fn dtype_usage( + _device: &Self::Device, + dtype: burn_backend::DType, + ) -> burn_backend::DTypeUsageSet { + if dtype.try_into_kind().is_ok() { + burn_backend::DTypeUsage::general() + } else { + burn_backend::DTypeUsageSet::empty() + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cpu.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cpu.rs new file mode 100644 index 0000000..64daa1e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cpu.rs @@ -0,0 +1,14 @@ +use burn_backend::{TensorMetadata, ops::FloatTensorOps}; +use burn_tch::{LibTorch, LibTorchDevice}; + +fn main() { + type B = LibTorch; + let device = LibTorchDevice::Cpu; + + // Creation of two tensors, the first with explicit values and the second one with ones, with the same shape as the first + let tensor_1 = B::float_from_data([[2f32, 3.], [4., 5.]].into(), &device); + let tensor_2 = B::float_ones(tensor_1.shape(), &device, tensor_1.dtype().into()); + + // Print the element-wise addition of the two tensors. + println!("{}", B::float_add(tensor_1, tensor_2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cuda.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cuda.rs new file mode 100644 index 0000000..4d96718 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/cuda.rs @@ -0,0 +1,19 @@ +use burn_backend::{TensorMetadata, ops::FloatTensorOps}; +use burn_tch::{LibTorch, LibTorchDevice}; + +fn main() { + assert!( + tch::utils::has_cuda(), + "Could not detect valid CUDA configuration" + ); + + type B = LibTorch; + let device = LibTorchDevice::Cuda(0); + + // Creation of two tensors, the first with explicit values and the second one with ones, with the same shape as the first + let tensor_1 = B::float_from_data([[2f32, 3.], [4., 5.]].into(), &device); + let tensor_2 = B::float_ones(tensor_1.shape(), &device, tensor_1.dtype().into()); + + // Print the element-wise addition of the two tensors. + println!("{}", B::float_add(tensor_1, tensor_2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/mps.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/mps.rs new file mode 100644 index 0000000..92932ae --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/bin/mps.rs @@ -0,0 +1,16 @@ +use burn_backend::{TensorMetadata, ops::FloatTensorOps}; +use burn_tch::{LibTorch, LibTorchDevice}; + +fn main() { + assert!(tch::utils::has_mps(), "Could not detect MPS"); + + type B = LibTorch; + let device = LibTorchDevice::Mps; + + // Creation of two tensors, the first with explicit values and the second one with ones, with the same shape as the first + let tensor_1 = B::float_from_data([[2f32, 3.], [4., 5.]].into(), &device); + let tensor_2 = B::float_ones(tensor_1.shape(), &device, tensor_1.dtype().into()); + + // Print the element-wise addition of the two tensors. + println!("{}", B::float_add(tensor_1, tensor_2)); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/dummy_cuda_dependency.cpp b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/dummy_cuda_dependency.cpp new file mode 100644 index 0000000..54c98e4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/dummy_cuda_dependency.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include +using namespace std; +extern "C" { +void dummy_cuda_dependency(); +} + +struct cublasContext; + +namespace at { +namespace cuda { +cublasContext *getCurrentCUDABlasHandle(); +int warp_size(); +} // namespace cuda +} // namespace at +char *magma_strerror(int err); +void dummy_cuda_dependency() { + try { + at::cuda::getCurrentCUDABlasHandle(); + at::cuda::warp_size(); + } catch (std::exception &e) { + if (getenv("TCH_PRINT_CUDA_INIT_ERROR") != nullptr) { + std::cerr << "error initializing cuda: " << e.what() << std::endl; + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/fake_cuda_dependency.cpp b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/fake_cuda_dependency.cpp new file mode 100644 index 0000000..57c077e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/cuda_hack/fake_cuda_dependency.cpp @@ -0,0 +1,5 @@ +extern "C" { +void dummy_cuda_dependency(); +} + +void dummy_cuda_dependency() {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/element.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/element.rs new file mode 100644 index 0000000..36db5ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/element.rs @@ -0,0 +1,51 @@ +use burn_backend::Element; +use burn_backend::{bf16, f16}; + +/// The element type for the tch backend. +pub trait TchElement: Element + tch::kind::Element { + /// Returns the associated tensor kind for [`tch::kind::Element`]. + fn kind() -> tch::Kind { + Self::KIND + } +} + +impl TchElement for f64 {} +impl TchElement for f32 {} +impl TchElement for f16 {} +impl TchElement for bf16 { + fn kind() -> tch::Kind { + let mut kind = ::KIND; + // Incorrect kind mapping in tch definitions, force bfloat16 + if matches!(Self::dtype(), burn_backend::DType::BF16) && kind == tch::Kind::Half { + kind = tch::Kind::BFloat16 + } + kind + } +} + +impl TchElement for i64 {} +impl TchElement for i32 {} +impl TchElement for i16 {} +impl TchElement for i8 {} + +impl TchElement for u8 {} + +impl TchElement for bool {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_elem_kinds() { + assert_eq!(f64::kind(), tch::Kind::Double); + assert_eq!(f32::kind(), tch::Kind::Float); + assert_eq!(f16::kind(), tch::Kind::Half); + assert_eq!(bf16::kind(), tch::Kind::BFloat16); + assert_eq!(i64::kind(), tch::Kind::Int64); + assert_eq!(i32::kind(), tch::Kind::Int); + assert_eq!(i16::kind(), tch::Kind::Int16); + assert_eq!(i8::kind(), tch::Kind::Int8); + assert_eq!(bool::kind(), tch::Kind::Bool); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/lib.rs new file mode 100644 index 0000000..4424a6b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/lib.rs @@ -0,0 +1,14 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![allow(clippy::single_range_in_vec_init)] + +//! Burn Tch Backend + +mod backend; +mod element; +mod ops; +mod tensor; + +pub use backend::*; +pub use element::*; +pub use tensor::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/activation.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/activation.rs new file mode 100644 index 0000000..b0636bd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/activation.rs @@ -0,0 +1,37 @@ +use crate::{LibTorch, TchTensor, element::TchElement}; +use burn_backend::ops::ActivationOps; + +impl ActivationOps for LibTorch { + fn relu(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.relu_(), |tensor| tensor.relu()) + } + + fn gelu(tensor: TchTensor) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.gelu_("none"), + |tensor| tensor.gelu("none"), + ) + } + + fn gelu_backward(tensor: TchTensor, grad: TchTensor) -> TchTensor { + let storage = tensor.storage.clone(); + let tensor = tensor.tensor.gelu_backward(&grad.tensor, "none"); + + TchTensor::from_existing(tensor, storage) + } + + fn sigmoid(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.sigmoid_(), |tensor| tensor.sigmoid()) + } + + fn log_sigmoid(tensor: TchTensor) -> TchTensor { + // NOTE: we don't override log_sigmoid_backward because Torch has a special backward + // formula that uses a buffer with computed values from the forward pass + + // no in-place log_sigmoid_ + let storage = tensor.storage.clone(); + let tensor = tensor.tensor.log_sigmoid(); + + TchTensor::from_existing(tensor, storage) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/base.rs new file mode 100644 index 0000000..53f1062 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/base.rs @@ -0,0 +1,737 @@ +use burn_backend::{Shape, TensorMetadata}; +use tch::Scalar; + +use crate::{LibTorchDevice, TchShape, TchTensor}; + +pub struct TchOps { + // e: PhantomData, +} + +impl TchOps { + pub fn to_device(tensor: TchTensor, device: &LibTorchDevice) -> TchTensor { + let device = (*device).into(); + + // We have to manually check if the device is the same, since when it's the case, we need to keep + // the same storage reference and not create a new one. + if tensor.tensor.device() == device { + return tensor; + } + + TchTensor::new(tensor.tensor.to(device)) + } + + pub fn reshape(tensor: TchTensor, shape: Shape) -> TchTensor { + let shape_tch: TchShape = shape.into(); + + TchTensor::from_existing(tensor.tensor.reshape(shape_tch.dims), tensor.storage) + } + + pub fn repeat_dim(tensor: TchTensor, dim: usize, times: usize) -> TchTensor { + let mut dims = vec![1; tensor.shape().num_dims()]; + dims[dim] = times as i64; + let tensor = tch::Tensor::repeat(&tensor.tensor, dims); + TchTensor::new(tensor) + } + + pub fn slice_with_steps(tensor: TchTensor, slices: &[burn_backend::Slice]) -> TchTensor { + let storage = tensor.storage.clone(); + let mut tensor = tensor.tensor.shallow_clone(); + + for (dim, slice) in slices.iter().enumerate() { + let dim_i64 = dim as i64; + // Convert slice to range using a dummy size (we'll use tensor dimensions) + let dim_size = tensor.size()[dim]; + let range = slice.to_range(dim_size as usize); + let start = range.start as i64; + let end = range.end as i64; + let step = slice.step as i64; + + if step > 0 { + // Forward stepping - use native slice + tensor = tensor.slice(dim_i64, Some(start), Some(end), step); + } else { + // Negative stepping - we need to handle the semantics correctly + // For negative steps, we iterate backwards from end-1 + // PyTorch's negative step works differently than our semantics + // We need to reverse the selected range + + // First get the slice with positive step + tensor = tensor.slice(dim_i64, Some(start), Some(end), 1); + + // Then reverse it and apply the step + if step == -1 { + // Simple reversal + tensor = tensor.flip([dim_i64]); + } else { + // Reverse and then take every nth element + tensor = tensor.flip([dim_i64]); + let abs_step = step.abs(); + tensor = tensor.slice(dim_i64, None, None, abs_step); + } + } + } + + TchTensor::partial(tensor, storage) + } + + pub fn slice_assign( + tensor: TchTensor, + slices: &[burn_backend::Slice], + value: TchTensor, + ) -> TchTensor { + // PyTorch's narrow operation only supports contiguous slices (step=1) + // For non-unit steps, we use advanced indexing as a workaround + let all_unit_steps = slices.iter().all(|s| s.step == 1); + + if all_unit_steps { + // Fast path: use narrow and copy_ for unit steps + let tch_shape = TchShape::from(tensor.shape()); + + // Copy the input tensor if we can't mutate it + let tensor_original: TchTensor = + tensor.unary_ops(|tensor| tensor, |tensor| tensor.copy()); + let tensor_original = tensor_original.tensor; + + let mut tensor = tensor_original.view_(tch_shape.dims); + + for (i, slice) in slices.iter().enumerate().take(slices.len()) { + // Convert Slice to range for narrow operation + let dim_size = tensor.size()[i] as usize; + let range = slice.to_range(dim_size); + let start = range.start as i64; + let length = (range.end - range.start) as i64; + + tensor = tensor.narrow(i as i64, start, length); + } + + tensor.copy_(&value.tensor); + TchTensor::new(tensor_original) + } else { + // Workaround for non-unit steps: use PyTorch's index_put operation + // This generates explicit indices for the slice and uses advanced indexing + let tensor_shape = tensor.shape(); + let dims = tensor_shape.clone(); + + // Copy the tensor since we'll modify it + let result_tensor = tensor.tensor.shallow_clone(); + + // Use advanced indexing to set the values + Self::slice_assign_with_advanced_indexing(result_tensor, slices, value.tensor, &dims) + } + } + + /// Generate indices for a slice with potentially non-unit step. + /// For negative steps, generates indices in reverse order. + fn generate_slice_indices(slice: &burn_backend::Slice, dim_size: usize) -> Vec { + let step = slice.step; + let range = slice.to_range(dim_size); + + let mut indices = Vec::new(); + + if step > 0 { + let mut idx = range.start as i64; + while idx < range.end as i64 { + indices.push(idx); + idx += step as i64; + } + } else if step < 0 { + // For negative steps, iterate backwards through the range + let mut idx = (range.end - 1) as i64; + while idx >= range.start as i64 { + indices.push(idx); + idx += step as i64; // step is negative, so this decreases + } + } + + indices + } + + /// Implementation using advanced indexing for non-unit steps. + /// Uses PyTorch's index_put operation to assign values at specific indices. + fn slice_assign_with_advanced_indexing( + mut tensor: tch::Tensor, + slices: &[burn_backend::Slice], + value: tch::Tensor, + dims: &[usize], + ) -> TchTensor { + // Generate all index combinations for the sliced regions + let mut index_sets: Vec> = Vec::new(); + for (i, slice) in slices.iter().enumerate() { + let dim_size = if i < dims.len() { dims[i] } else { 1 }; + let indices = Self::generate_slice_indices(slice, dim_size); + index_sets.push(indices); + } + + // For unsliced dimensions, include all indices + for &dim_size in dims.iter().skip(slices.len()) { + let indices: Vec = (0..dim_size as i64).collect(); + index_sets.push(indices); + } + + // Convert index sets to tensors for index_put + let mut final_indices = Vec::new(); + let total_elements = index_sets.iter().map(|s| s.len()).product::(); + + // Build flattened index arrays for each dimension using cartesian product + // This creates the index tensors needed for PyTorch's index_put operation + for dim_idx in 0..index_sets.len() { + let mut dim_indices = Vec::with_capacity(total_elements); + let repeat = index_sets[dim_idx + 1..] + .iter() + .map(|s| s.len()) + .product::() + .max(1); + let tile = index_sets[..dim_idx] + .iter() + .map(|s| s.len()) + .product::() + .max(1); + + for _ in 0..tile { + for &idx in &index_sets[dim_idx] { + for _ in 0..repeat { + dim_indices.push(idx); + } + } + } + + let indices_tensor = tch::Tensor::from_slice(&dim_indices).to_device(tensor.device()); + final_indices.push(indices_tensor); + } + + // PyTorch's index_put handles assignment correctly for negative steps + // following NumPy semantics: values[i] goes to selected_indices[i] + let value_flat = value.view(-1); + + // Use index_put to assign values - convert to Option + let final_indices_opt: Vec> = + final_indices.into_iter().map(Some).collect(); + tensor = tensor.index_put(&final_indices_opt, &value_flat, false); + + TchTensor::new(tensor) + } + + pub fn gather(dim: usize, tensor: TchTensor, indices: TchTensor) -> TchTensor { + let storage = tensor.storage.clone(); + let tensor = tensor.tensor.gather(dim as i64, &indices.tensor, false); + + TchTensor::from_existing(tensor, storage) + } + + pub fn scatter( + dim: usize, + tensor: TchTensor, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + let storage = tensor.storage.clone(); + let tensor = tensor + .tensor + .scatter_add(dim as i64, &indices.tensor, &value.tensor); + + TchTensor::from_existing(tensor, storage) + } + + pub fn index_select_dim(tensor: TchTensor, dim: usize, indices: TchTensor) -> TchTensor { + let storage = tensor.storage.clone(); + let tensor = tensor.tensor.index_select(dim as i64, &indices.tensor); + + TchTensor::from_existing(tensor, storage) + } + + pub fn select_assign( + tensor: TchTensor, + dim: usize, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + tensor.clone().unary_ops( + |mut tensor| tensor.index_add_(dim as i64, &indices.tensor, &value.tensor), + |tensor| tensor.index_add(dim as i64, &indices.tensor, &value.tensor), + ) + } + + pub fn cat(tensors: Vec, dim: usize) -> TchTensor { + let tensors: Vec = tensors + .into_iter() + .map(|t| t.tensor.shallow_clone()) + .collect(); + let tensor = tch::Tensor::cat(&tensors, dim as i64); + + TchTensor::new(tensor) + } + + pub fn equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.eq_tensor_(rhs).to_kind(tch::Kind::Bool), + |lhs, rhs| rhs.eq_tensor_(lhs).to_kind(tch::Kind::Bool), + |lhs, rhs| lhs.eq_tensor(rhs), + ) + } + + pub fn equal_elem + Clone>(lhs: TchTensor, rhs: S) -> TchTensor { + lhs.unary_ops( + |mut tensor| tensor.eq_(rhs.clone().into()).to_kind(tch::Kind::Bool), + |tensor| tensor.eq(rhs.clone().into()), + ) + } + + pub fn greater(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.greater_tensor_(rhs).to_kind(tch::Kind::Bool), + |lhs, rhs| rhs.less_tensor_(lhs).to_kind(tch::Kind::Bool), + |lhs, rhs| lhs.greater_tensor(rhs), + ) + } + + pub fn greater_elem + Clone>(lhs: TchTensor, rhs: S) -> TchTensor { + lhs.unary_ops( + |mut tensor| tensor.greater_(rhs.clone().into()).to_kind(tch::Kind::Bool), + |tensor| tensor.greater(rhs.clone().into()), + ) + } + + pub fn greater_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.greater_equal_tensor_(rhs).to_kind(tch::Kind::Bool), + |lhs, rhs| rhs.less_equal_tensor_(lhs).to_kind(tch::Kind::Bool), + |lhs, rhs| lhs.greater_equal_tensor(rhs), + ) + } + + pub fn greater_equal_elem + Clone>(lhs: TchTensor, rhs: S) -> TchTensor { + lhs.unary_ops( + |mut tensor| { + tensor + .greater_equal_(rhs.clone().into()) + .to_kind(tch::Kind::Bool) + }, + |tensor| tensor.greater_equal(rhs.clone().into()), + ) + } + + pub fn lower(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.less_tensor_(rhs).to_kind(tch::Kind::Bool), + |lhs, rhs| rhs.greater_tensor_(lhs).to_kind(tch::Kind::Bool), + |lhs, rhs| lhs.less_tensor(rhs), + ) + } + + pub fn lower_elem + Clone>(lhs: TchTensor, rhs: S) -> TchTensor { + lhs.unary_ops( + |mut tensor| tensor.less_(rhs.clone().into()).to_kind(tch::Kind::Bool), + |tensor| tensor.less(rhs.clone().into()), + ) + } + + pub fn lower_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.less_equal_tensor_(rhs).to_kind(tch::Kind::Bool), + |lhs, rhs| rhs.greater_equal_tensor_(lhs).to_kind(tch::Kind::Bool), + |lhs, rhs| lhs.less_equal_tensor(rhs), + ) + } + + pub fn lower_equal_elem + Clone>(lhs: TchTensor, rhs: S) -> TchTensor { + lhs.unary_ops( + |mut tensor| { + tensor + .less_equal_(rhs.clone().into()) + .to_kind(tch::Kind::Bool) + }, + |tensor| tensor.less_equal(rhs.clone().into()), + ) + } + + pub fn add(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_add_(rhs).unwrap(), + |lhs, rhs| rhs.f_add_(lhs).unwrap(), + |lhs, rhs| lhs.f_add(rhs).unwrap(), + ) + } + + pub fn sub(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_sub_(rhs).unwrap(), + |lhs, rhs| lhs.f_sub(rhs).unwrap(), + |lhs, rhs| lhs.f_sub(rhs).unwrap(), + ) + } + + pub fn mul(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_mul_(rhs).unwrap(), + |lhs, rhs| rhs.f_mul_(lhs).unwrap(), + |lhs, rhs| lhs.f_mul(rhs).unwrap(), + ) + } + + pub fn div(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_div_(rhs).unwrap(), + |lhs, rhs| lhs.f_div(rhs).unwrap(), + |lhs, rhs| lhs.f_div(rhs).unwrap(), + ) + } + + pub fn remainder(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_remainder_tensor_(rhs).unwrap(), + |lhs, rhs| lhs.f_remainder_tensor(rhs).unwrap(), + |lhs, rhs| lhs.f_remainder_tensor(rhs).unwrap(), + ) + } + + pub fn mean(tensor: TchTensor) -> TchTensor { + // view as 1d tensor + let tensor = tensor.tensor.mean(tensor.tensor.kind()).view(1); + TchTensor::new(tensor) + } + + pub fn mean_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchTensor::from_existing( + tensor + .tensor + .mean_dim(Some([dim as i64].as_slice()), true, tensor.tensor.kind()), + tensor.storage, + ) + } + + pub fn sum(tensor: TchTensor) -> TchTensor { + // view as 1d tensor + let tensor = tensor.tensor.sum(tensor.tensor.kind()).view(1); + TchTensor::new(tensor) + } + + pub fn sum_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchTensor::from_existing( + tensor.tensor.sum_dim_intlist( + Some([dim as i64].as_slice()), + true, + tensor.tensor.kind(), + ), + tensor.storage, + ) + } + + pub fn prod(tensor: TchTensor) -> TchTensor { + // view as 1d tensor + let tensor = tensor.tensor.prod(tensor.tensor.kind()).view(1); + TchTensor::new(tensor) + } + + pub fn prod_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchTensor::from_existing( + tensor + .tensor + .prod_dim_int(dim as i64, true, tensor.tensor.kind()), + tensor.storage, + ) + } + + pub fn cumsum(tensor: TchTensor, dim: usize) -> TchTensor { + TchTensor::from_existing( + tensor.tensor.cumsum(dim as i64, tensor.tensor.kind()), + tensor.storage, + ) + } + + pub fn cumprod(tensor: TchTensor, dim: usize) -> TchTensor { + TchTensor::from_existing( + tensor.tensor.cumprod(dim as i64, tensor.tensor.kind()), + tensor.storage, + ) + } + + pub fn cummin(tensor: TchTensor, dim: usize) -> TchTensor { + let (values, _indices) = tensor.tensor.cummin(dim as i64); + TchTensor::from_existing(values, tensor.storage) + } + + pub fn cummax(tensor: TchTensor, dim: usize) -> TchTensor { + // cummax returns (values, indices) tuple in PyTorch, we only need values + let (values, _indices) = tensor.tensor.cummax(dim as i64); + TchTensor::from_existing(values, tensor.storage) + } + + pub fn argmax(tensor: TchTensor, dim: usize) -> TchTensor { + let storage = tensor.storage.clone(); + let tensor = tensor.tensor.argmax(dim as i64, true); + + TchTensor::from_existing(tensor, storage) + } + + pub fn argmin(tensor: TchTensor, dim: usize) -> TchTensor { + let storage = tensor.storage.clone(); + let tensor = tensor.tensor.argmin(dim as i64, true); + + TchTensor::from_existing(tensor, storage) + } + + pub fn max_dim(tensor: TchTensor, dim: usize) -> TchTensor { + let storage = tensor.storage.clone(); + let (tensor, _indices) = tensor.tensor.max_dim(dim as i64, true); + + TchTensor::from_existing(tensor, storage) + } + + pub fn max_dim_with_indices(tensor: TchTensor, dim: usize) -> (TchTensor, TchTensor) { + let storage = tensor.storage.clone(); + let (tensor, indices) = tensor.tensor.max_dim(dim as i64, true); + + let tensor = TchTensor::from_existing(tensor, storage); + let indices = TchTensor::new(indices); + + (tensor, indices) + } + + pub fn min_dim(tensor: TchTensor, dim: usize) -> TchTensor { + let storage = tensor.storage.clone(); + let (tensor, _indices) = tensor.tensor.min_dim(dim as i64, true); + + TchTensor::from_existing(tensor, storage) + } + + pub fn min_dim_with_indices(tensor: TchTensor, dim: usize) -> (TchTensor, TchTensor) { + let storage = tensor.storage.clone(); + let (tensor, indices) = tensor.tensor.min_dim(dim as i64, true); + + let tensor = TchTensor::from_existing(tensor, storage); + let indices = TchTensor::new(indices); + + (tensor, indices) + } + + pub fn clamp_min + Clone + Copy>(tensor: TchTensor, min: S) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.clamp_min_(min), + |tensor| tensor.clamp_min(min), + ) + } + + pub fn clamp_max + Clone + Copy>(tensor: TchTensor, max: S) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.clamp_max_(max), + |tensor| tensor.clamp_max(max), + ) + } + + pub fn clamp + Clone + Copy>( + tensor: TchTensor, + min: S, + max: S, + ) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.clamp_(min, max), + |tensor| tensor.clamp(min, max), + ) + } + + pub fn swap_dims(tensor: TchTensor, dim1: usize, dim2: usize) -> TchTensor { + let tensor = tensor.tensor.transpose(dim1 as i64, dim2 as i64); + TchTensor::new(tensor) + } + + pub fn permute(tensor: TchTensor, axes: &[usize]) -> TchTensor { + let tensor = tensor + .tensor + .permute(axes.iter().map(|x| *x as i64).collect::>()); + TchTensor::new(tensor) + } + + pub fn flip(tensor: TchTensor, axes: &[usize]) -> TchTensor { + let dims = axes.iter().map(|x| *x as i64).collect::>(); + let tensor = tensor.tensor.flip(dims); + TchTensor::new(tensor) + } + + pub fn powf(tensor: TchTensor, exponent: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + tensor, + exponent, + |lhs, rhs| lhs.f_pow_tensor_(rhs).unwrap(), + |lhs, rhs| lhs.f_pow(rhs).unwrap(), + |lhs, rhs| lhs.f_pow(rhs).unwrap(), + ) + } + + pub fn sign(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.sign_(), |tensor| tensor.sign()) + } + + pub fn expand(tensor: TchTensor, shape: Shape) -> TchTensor { + let storage = tensor.storage.clone(); + let broadcasted_tensor = tensor.tensor.broadcast_to(TchShape::from(shape).dims); + TchTensor::from_existing(broadcasted_tensor, storage) + } + + pub fn unfold(tensor: TchTensor, dim: usize, size: usize, step: usize) -> TchTensor { + let storage = tensor.storage.clone(); + let uf_tensor = tensor.tensor.unfold(dim as i64, size as i64, step as i64); + + TchTensor::from_existing(uf_tensor, storage) + } + + pub fn sort(tensor: TchTensor, dim: usize, descending: bool) -> TchTensor { + TchTensor::new(tensor.tensor.sort(dim as i64, descending).0) + } + + pub fn sort_with_indices( + tensor: TchTensor, + dim: usize, + descending: bool, + ) -> (TchTensor, TchTensor) { + let sorted = tensor.tensor.sort(dim as i64, descending); + (TchTensor::new(sorted.0), TchTensor::new(sorted.1)) + } + + pub fn argsort(tensor: TchTensor, dim: usize, descending: bool) -> TchTensor { + TchTensor::new(tensor.tensor.argsort(dim as i64, descending)) + } + + pub fn bitwise_and(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_bitwise_and_tensor_(rhs).unwrap(), + |lhs, rhs| rhs.f_bitwise_and_tensor_(lhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_and_tensor(rhs).unwrap(), + ) + } + + pub fn bitwise_and_scalar + Clone>(tensor: TchTensor, scalar: S) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.f_bitwise_and_(scalar.clone().into()).unwrap(), + |tensor| tensor.f_bitwise_and(scalar.clone().into()).unwrap(), + ) + } + + pub fn bitwise_or(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_bitwise_or_tensor_(rhs).unwrap(), + |lhs, rhs| rhs.f_bitwise_or_tensor_(lhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_or_tensor(rhs).unwrap(), + ) + } + + pub fn bitwise_or_scalar + Clone>(tensor: TchTensor, scalar: S) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.f_bitwise_or_(scalar.clone().into()).unwrap(), + |tensor| tensor.f_bitwise_or(scalar.clone().into()).unwrap(), + ) + } + + pub fn bitwise_xor(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_bitwise_xor_tensor_(rhs).unwrap(), + |lhs, rhs| rhs.f_bitwise_xor_tensor_(lhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_xor_tensor(rhs).unwrap(), + ) + } + + pub fn bitwise_xor_scalar + Clone>(tensor: TchTensor, scalar: S) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.f_bitwise_xor_(scalar.clone().into()).unwrap(), + |tensor| tensor.f_bitwise_xor(scalar.clone().into()).unwrap(), + ) + } + + pub fn bitwise_not(tensor: TchTensor) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.f_bitwise_not_().unwrap(), + |tensor| tensor.f_bitwise_not().unwrap(), + ) + } + + pub fn bitwise_left_shift(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_bitwise_left_shift_(rhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_left_shift(rhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_left_shift(rhs).unwrap(), + ) + } + + pub fn bitwise_left_shift_scalar + Clone>( + tensor: TchTensor, + scalar: S, + ) -> TchTensor { + tensor.unary_ops( + |mut tensor| { + tensor + .f_bitwise_left_shift_tensor_scalar_(scalar.clone().into()) + .unwrap() + }, + |tensor| { + tensor + .f_bitwise_left_shift_tensor_scalar(scalar.clone().into()) + .unwrap() + }, + ) + } + + pub fn bitwise_right_shift(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_bitwise_right_shift_(rhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_right_shift(rhs).unwrap(), + |lhs, rhs| lhs.f_bitwise_right_shift(rhs).unwrap(), + ) + } + + pub fn bitwise_right_shift_scalar + Clone>( + tensor: TchTensor, + scalar: S, + ) -> TchTensor { + tensor.unary_ops( + |mut tensor| { + tensor + .f_bitwise_right_shift_tensor_scalar_(scalar.clone().into()) + .unwrap() + }, + |tensor| { + tensor + .f_bitwise_right_shift_tensor_scalar(scalar.clone().into()) + .unwrap() + }, + ) + } + + pub fn atan2(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.f_atan2_(rhs).unwrap(), + |lhs, rhs| lhs.f_atan2(rhs).unwrap(), + |lhs, rhs| lhs.f_atan2(rhs).unwrap(), + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/bool_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/bool_tensor.rs new file mode 100644 index 0000000..da2fe65 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/bool_tensor.rs @@ -0,0 +1,219 @@ +use super::TchOps; +use crate::{LibTorch, LibTorchDevice, TchShape, TchTensor, element::TchElement}; +use burn_backend::ExecutionError; +use burn_backend::Scalar; +use burn_backend::tensor::BoolTensor; +use burn_backend::tensor::IntTensor; +use burn_backend::{Shape, TensorData, TensorMetadata, ops::BoolTensorOps}; + +impl BoolTensorOps for LibTorch { + fn bool_from_data(data: TensorData, device: &LibTorchDevice) -> TchTensor { + match data.dtype { + burn_backend::DType::Bool => TchTensor::from_data::(data, (*device).into()), + _ => unimplemented!("Unsupported dtype for `bool_from_data`"), + } + } + + fn bool_repeat_dim(tensor: TchTensor, dim: usize, times: usize) -> TchTensor { + TchOps::repeat_dim(tensor, dim, times) + } + + async fn bool_into_data(tensor: TchTensor) -> Result { + let shape = tensor.shape(); + let tensor = Self::bool_reshape(tensor.clone(), Shape::new([shape.num_elements()])); + let values: Result, tch::TchError> = tensor.tensor.shallow_clone().try_into(); + Ok(TensorData::new(values.unwrap(), shape)) + } + + fn bool_to_device(tensor: TchTensor, device: &LibTorchDevice) -> TchTensor { + TchOps::to_device(tensor, device) + } + + fn bool_reshape(tensor: TchTensor, shape: Shape) -> TchTensor { + TchOps::reshape(tensor, shape) + } + + fn bool_device(tensor: &TchTensor) -> LibTorchDevice { + tensor.tensor.device().into() + } + + fn bool_empty(shape: Shape, device: &LibTorchDevice) -> TchTensor { + let tensor = tch::Tensor::empty( + TchShape::from(shape).dims, + (tch::Kind::Bool, (*device).into()), + ); + + TchTensor::new(tensor) + } + + fn bool_zeros(shape: Shape, device: &LibTorchDevice) -> TchTensor { + let tensor = tch::Tensor::zeros( + TchShape::from(shape).dims, + (tch::Kind::Bool, (*device).into()), + ); + + TchTensor::new(tensor) + } + + fn bool_ones(shape: Shape, device: &LibTorchDevice) -> TchTensor { + let tensor = tch::Tensor::ones( + TchShape::from(shape).dims, + (tch::Kind::Bool, (*device).into()), + ); + + TchTensor::new(tensor) + } + + fn bool_slice(tensor: TchTensor, slices: &[burn_backend::Slice]) -> TchTensor { + TchOps::slice_with_steps(tensor, slices) + } + + fn bool_slice_assign( + tensor: TchTensor, + slices: &[burn_backend::Slice], + value: TchTensor, + ) -> TchTensor { + TchOps::slice_assign(tensor, slices, value) + } + + fn bool_cat(tensors: Vec, dim: usize) -> TchTensor { + TchOps::cat(tensors, dim) + } + + fn bool_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::equal(lhs, rhs) + } + + fn bool_not(tensor: TchTensor) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.eq_(0).to_kind(tch::Kind::Bool), + |tensor| tensor.eq(0), + ) + } + + fn bool_and(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.logical_and_(rhs), + |lhs, rhs| rhs.logical_and_(lhs), + |lhs, rhs| lhs.logical_and(rhs), + ) + } + + fn bool_or(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + lhs, + rhs, + |lhs, rhs| lhs.logical_or_(rhs), + |lhs, rhs| rhs.logical_or_(lhs), + |lhs, rhs| lhs.logical_or(rhs), + ) + } + + fn bool_into_int(tensor: TchTensor) -> TchTensor { + let tensor = tensor.tensor.to_kind(tch::Kind::Int64); + TchTensor::new(tensor) + } + + fn bool_into_float(tensor: TchTensor) -> TchTensor { + let tensor = tensor.tensor.to_kind(E::kind()); + TchTensor::new(tensor) + } + + fn bool_swap_dims(tensor: TchTensor, dim1: usize, dim2: usize) -> TchTensor { + TchOps::swap_dims(tensor, dim1, dim2) + } + + fn bool_permute(tensor: TchTensor, axes: &[usize]) -> TchTensor { + TchOps::permute(tensor, axes) + } + + fn bool_flip(tensor: TchTensor, axes: &[usize]) -> TchTensor { + TchOps::flip(tensor, axes) + } + + async fn bool_argwhere(tensor: TchTensor) -> TchTensor { + TchTensor::new(tensor.tensor.argwhere()) + } + + fn bool_select(tensor: TchTensor, dim: usize, indices: TchTensor) -> TchTensor { + TchOps::index_select_dim(tensor, dim, indices) + } + + fn bool_select_or( + tensor: TchTensor, + dim: usize, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + TchOps::select_assign(tensor, dim, indices, value) + } + + fn bool_expand(tensor: TchTensor, shape: Shape) -> TchTensor { + TchOps::expand(tensor, shape) + } + + fn bool_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + TchOps::unfold(tensor, dim, size, step) + } + + fn bool_mask_where( + tensor: BoolTensor, + mask: BoolTensor, + value: BoolTensor, + ) -> BoolTensor { + TchTensor::binary_ops_tensor( + tensor, + value, + |tensor, source| source.f_where_self(&mask.tensor, tensor).unwrap(), + |tensor, source| source.f_where_self(&mask.tensor, tensor).unwrap(), + |tensor, source| source.f_where_self(&mask.tensor, tensor).unwrap(), + ) + } + + fn bool_mask_fill( + tensor: BoolTensor, + mask: BoolTensor, + value: Scalar, + ) -> BoolTensor { + tensor.unary_ops( + |mut tensor| { + tensor + .f_masked_fill_(&mask.tensor, value.elem::()) + .unwrap() + }, + |tensor| { + tensor + .f_masked_fill(&mask.tensor, value.elem::()) + .unwrap() + }, + ) + } + + fn bool_gather( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + ) -> BoolTensor { + TchOps::gather(dim, tensor, indices) + } + + fn bool_scatter_or( + dim: usize, + tensor: BoolTensor, + indices: IntTensor, + value: BoolTensor, + ) -> BoolTensor { + TchOps::scatter(dim, tensor, indices, value) + } + + fn bool_equal_elem(lhs: BoolTensor, rhs: Scalar) -> BoolTensor { + TchOps::equal_elem(lhs, rhs.elem::()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/int_tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/int_tensor.rs new file mode 100644 index 0000000..a2eb3d9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/int_tensor.rs @@ -0,0 +1,501 @@ +use std::ops::Range; + +use burn_backend::{ + Distribution, ExecutionError, IntDType, Scalar, Shape, TensorData, TensorMetadata, + ops::{FloatTensorOps, IntTensorOps}, + tensor::IntTensor, +}; + +use crate::{IntoKind, LibTorch, LibTorchDevice, TchShape, TchTensor, element::TchElement}; + +use super::TchOps; + +impl IntTensorOps for LibTorch { + fn int_from_data(data: TensorData, device: &LibTorchDevice) -> TchTensor { + match data.dtype { + burn_backend::DType::I64 => TchTensor::from_data::(data, (*device).into()), + _ => unimplemented!("Unsupported dtype for `int_from_data`"), + } + } + + fn int_repeat_dim(tensor: TchTensor, dim: usize, times: usize) -> TchTensor { + TchOps::repeat_dim(tensor, dim, times) + } + + async fn int_into_data(tensor: TchTensor) -> Result { + let shape = tensor.shape(); + let tensor = Self::int_reshape(tensor.clone(), Shape::new([shape.num_elements()])); + let values: Result, tch::TchError> = tensor.tensor.shallow_clone().try_into(); + Ok(TensorData::new(values.unwrap(), shape)) + } + + fn int_to_device(tensor: TchTensor, device: &LibTorchDevice) -> TchTensor { + TchOps::to_device(tensor, device) + } + + fn int_reshape(tensor: TchTensor, shape: Shape) -> TchTensor { + TchOps::reshape(tensor, shape) + } + + fn int_device(tensor: &TchTensor) -> LibTorchDevice { + tensor.tensor.device().into() + } + + fn int_empty(shape: Shape, device: &LibTorchDevice, dtype: IntDType) -> TchTensor { + let tensor = tch::Tensor::empty( + TchShape::from(shape).dims, + (dtype.into_kind(), (*device).into()), + ); + + TchTensor::new(tensor) + } + + fn int_slice(tensor: TchTensor, slices: &[burn_backend::Slice]) -> TchTensor { + TchOps::slice_with_steps(tensor, slices) + } + + fn int_slice_assign( + tensor: TchTensor, + slices: &[burn_backend::Slice], + value: TchTensor, + ) -> TchTensor { + TchOps::slice_assign(tensor, slices, value) + } + + fn int_cat(tensors: Vec, dim: usize) -> TchTensor { + TchOps::cat(tensors, dim) + } + + fn int_matmul(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + let lhs = Self::int_into_float(lhs); + let rhs = Self::int_into_float(rhs); + let out = lhs.tensor.f_matmul(&rhs.tensor).unwrap(); + Self::float_into_int(TchTensor::new(out)) + } + + fn int_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::equal(lhs, rhs) + } + + fn int_equal_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::equal_elem(lhs, rhs.elem::()) + } + + fn int_greater(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::greater(lhs, rhs) + } + + fn int_greater_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::greater_elem(lhs, rhs.elem::()) + } + + fn int_greater_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::greater_equal(lhs, rhs) + } + + fn int_greater_equal_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::greater_equal_elem(lhs, rhs.elem::()) + } + + fn int_lower(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::lower(lhs, rhs) + } + + fn int_lower_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::lower_elem(lhs, rhs.elem::()) + } + + fn int_lower_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::lower_equal(lhs, rhs) + } + + fn int_lower_equal_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::lower_equal_elem(lhs, rhs.elem::()) + } + + fn int_add(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::add(lhs, rhs) + } + + fn int_add_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + lhs.unary_ops( + |mut tensor| tensor.f_add_scalar_(rhs.elem::()).unwrap(), + |tensor| tensor.f_add_scalar(rhs.elem::()).unwrap(), + ) + } + + fn int_sub(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::sub(lhs, rhs) + } + + fn int_sub_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + lhs.unary_ops( + |mut tensor| tensor.f_sub_scalar_(rhs.elem::()).unwrap(), + |tensor| tensor.f_sub_scalar(rhs.elem::()).unwrap(), + ) + } + + fn int_mul(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::mul(lhs, rhs) + } + + fn int_mul_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + lhs.unary_ops( + |mut tensor| tensor.f_mul_scalar_(rhs.elem::()).unwrap(), + |tensor| tensor.f_mul_scalar(rhs.elem::()).unwrap(), + ) + } + + fn int_div(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + let dtype = lhs.tensor.kind(); + let copy = false; + let non_blocking = true; + let lhs: TchTensor = + TchTensor::new(lhs.tensor.to_dtype(tch::Kind::Float, non_blocking, copy)); + let rhs: TchTensor = + TchTensor::new(rhs.tensor.to_dtype(tch::Kind::Float, non_blocking, copy)); + + let out = TchOps::div(lhs, rhs); + + TchTensor::new(out.tensor.to_dtype(dtype, non_blocking, copy)) + } + + fn int_div_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + let dtype = lhs.tensor.kind(); + let copy = false; + let non_blocking = true; + let lhs: TchTensor = + TchTensor::new(lhs.tensor.to_dtype(tch::Kind::Float, non_blocking, copy)); + + let out: TchTensor = lhs.unary_ops( + |mut tensor| tensor.f_div_scalar_(rhs.elem::()).unwrap(), + |tensor| tensor.f_div_scalar(rhs.elem::()).unwrap(), + ); + + TchTensor::new(out.tensor.to_dtype(dtype, non_blocking, copy)) + } + + fn int_remainder(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + let dtype = lhs.tensor.kind(); + let copy = false; + let non_blocking = true; + let lhs: TchTensor = + TchTensor::new(lhs.tensor.to_dtype(tch::Kind::Float, non_blocking, copy)); + let rhs: TchTensor = + TchTensor::new(rhs.tensor.to_dtype(tch::Kind::Float, non_blocking, copy)); + + let out = TchOps::remainder(lhs, rhs); + + TchTensor::new(out.tensor.to_dtype(dtype, non_blocking, copy)) + } + + fn int_remainder_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + lhs.unary_ops( + |tensor| tensor.f_remainder(rhs.elem::()).unwrap(), + |tensor| tensor.f_remainder(rhs.elem::()).unwrap(), + ) + } + + fn int_zeros(shape: Shape, device: &LibTorchDevice, dtype: IntDType) -> TchTensor { + let shape = TchShape::from(shape); + let device: tch::Device = (*device).into(); + + TchTensor::new(tch::Tensor::zeros(shape.dims, (dtype.into_kind(), device))) + } + + fn int_ones(shape: Shape, device: &LibTorchDevice, dtype: IntDType) -> TchTensor { + let shape = TchShape::from(shape); + let device: tch::Device = (*device).into(); + + TchTensor::new(tch::Tensor::ones(shape.dims, (dtype.into_kind(), device))) + } + + fn int_full( + shape: Shape, + fill_value: Scalar, + device: &LibTorchDevice, + dtype: IntDType, + ) -> TchTensor { + let shape = TchShape::from(shape); + let device: tch::Device = (*device).into(); + + TchTensor::new(tch::Tensor::full( + shape.dims, + fill_value.elem::(), + (dtype.into_kind(), device), + )) + } + + fn int_sum(tensor: TchTensor) -> TchTensor { + TchOps::sum(tensor) + } + + fn int_sum_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::sum_dim(tensor, dim) + } + + fn int_prod(tensor: TchTensor) -> TchTensor { + TchOps::prod(tensor) + } + + fn int_prod_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::prod_dim(tensor, dim) + } + + fn int_mean(tensor: TchTensor) -> TchTensor { + let dtype = tensor.tensor.kind(); + let tensor: TchTensor = + TchTensor::new(tensor.tensor.to_dtype(tch::Kind::Float, true, false)); + let output: TchTensor = TchTensor::new(TchOps::mean(tensor).tensor); + + TchTensor::new(output.tensor.to_dtype(dtype, true, false)) + } + + fn int_mean_dim(tensor: TchTensor, dim: usize) -> TchTensor { + let dtype = tensor.tensor.kind(); + let tensor: TchTensor = + TchTensor::new(tensor.tensor.to_dtype(tch::Kind::Float, true, false)); + + let output: TchTensor = TchTensor::new(TchOps::mean_dim(tensor, dim).tensor); + + TchTensor::new(output.tensor.to_dtype(dtype, true, false)) + } + + fn int_cumsum(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cumsum(tensor, dim) + } + + fn int_cumprod(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cumprod(tensor, dim) + } + + fn int_cummin(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cummin(tensor, dim) + } + + fn int_cummax(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cummax(tensor, dim) + } + + fn int_gather(dim: usize, tensor: TchTensor, indices: TchTensor) -> TchTensor { + TchOps::gather(dim, tensor, indices) + } + + fn int_scatter_add( + dim: usize, + tensor: TchTensor, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + TchOps::scatter(dim, tensor, indices, value) + } + + fn int_select(tensor: TchTensor, dim: usize, indices: TchTensor) -> TchTensor { + TchOps::index_select_dim(tensor, dim, indices) + } + + fn int_select_add( + tensor: TchTensor, + dim: usize, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + TchOps::select_assign(tensor, dim, indices, value) + } + + fn int_mask_where(tensor: TchTensor, mask: TchTensor, source: TchTensor) -> TchTensor { + TchTensor::binary_ops_tensor( + tensor, + source, + |tensor, source| source.f_where_self(&mask.tensor, tensor).unwrap(), + |tensor, source| source.f_where_self(&mask.tensor, tensor).unwrap(), + |tensor, source| source.f_where_self(&mask.tensor, tensor).unwrap(), + ) + } + + fn int_mask_fill(tensor: TchTensor, mask: TchTensor, value: Scalar) -> TchTensor { + let value = value.elem::(); + tensor.unary_ops( + |mut tensor| tensor.f_masked_fill_(&mask.tensor, value).unwrap(), + |tensor| tensor.f_masked_fill(&mask.tensor, value).unwrap(), + ) + } + + fn int_argmax(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::argmax(tensor, dim) + } + + fn int_argmin(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::argmin(tensor, dim) + } + + fn int_max_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::max_dim(tensor, dim) + } + + fn int_max_dim_with_indices(tensor: TchTensor, dim: usize) -> (TchTensor, TchTensor) { + TchOps::max_dim_with_indices(tensor, dim) + } + + fn int_min_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::min_dim(tensor, dim) + } + + fn int_min_dim_with_indices(tensor: TchTensor, dim: usize) -> (TchTensor, TchTensor) { + TchOps::min_dim_with_indices(tensor, dim) + } + + fn int_clamp_min(tensor: TchTensor, min: Scalar) -> TchTensor { + TchOps::clamp_min(tensor, min.elem::()) + } + + fn int_clamp_max(tensor: TchTensor, max: Scalar) -> TchTensor { + TchOps::clamp_max(tensor, max.elem::()) + } + + fn int_clamp(tensor: TchTensor, min: Scalar, max: Scalar) -> TchTensor { + TchOps::clamp(tensor, min.elem::(), max.elem::()) + } + + fn int_abs(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.abs_(), |tensor| tensor.abs()) + } + + fn int_into_float(tensor: TchTensor) -> TchTensor { + let tensor = tensor.tensor.to_kind(E::kind()); + TchTensor::new(tensor) + } + + fn int_swap_dims(tensor: IntTensor, dim1: usize, dim2: usize) -> IntTensor { + TchOps::swap_dims(tensor, dim1, dim2) + } + + fn int_random(shape: Shape, distribution: Distribution, device: &LibTorchDevice) -> TchTensor { + match distribution { + Distribution::Default => TchTensor::new(tch::Tensor::randint_low( + 0, + 255, + shape.iter().map(|i| *i as i64).collect::>(), + (tch::Kind::Int64, (*device).into()), + )), + Distribution::Bernoulli(prob) => { + let mut tensor = TchTensor::empty::(shape, *device); + tensor + .mut_ops(|tensor| tensor.f_bernoulli_float_(prob).unwrap()) + .unwrap() + } + Distribution::Uniform(from, to) => TchTensor::new(tch::Tensor::randint_low( + from as i64, + to as i64, + shape.iter().map(|i| *i as i64).collect::>(), + (tch::Kind::Int64, (*device).into()), + )), + Distribution::Normal(mean, std) => { + let mut tensor = TchTensor::empty::(shape, *device); + tensor.mut_ops(|tensor| tensor.normal_(mean, std)).unwrap() + } + } + } + + fn int_arange(range: Range, device: &LibTorchDevice) -> TchTensor { + let device: tch::Device = (*device).into(); + let mut tensor = tch::Tensor::arange(range.end - range.start, (tch::Kind::Int64, device)); + + if range.start != 0 { + tensor = tensor.f_add_scalar_(range.start).unwrap(); + } + + TchTensor::new(tensor) + } + + fn int_permute(tensor: IntTensor, axes: &[usize]) -> IntTensor { + TchOps::permute(tensor, axes) + } + + fn int_flip(tensor: IntTensor, axes: &[usize]) -> IntTensor { + TchOps::flip(tensor, axes) + } + + fn int_sign(tensor: IntTensor) -> IntTensor { + TchOps::sign(tensor) + } + + fn int_expand(tensor: IntTensor, shape: Shape) -> IntTensor { + TchOps::expand(tensor, shape) + } + + fn int_sort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + TchOps::sort(tensor, dim, descending) + } + + fn int_argsort(tensor: IntTensor, dim: usize, descending: bool) -> IntTensor { + TchOps::argsort(tensor, dim, descending) + } + + fn bitwise_and(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + TchOps::bitwise_and(lhs, rhs) + } + + fn bitwise_or(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + TchOps::bitwise_or(lhs, rhs) + } + + fn bitwise_xor(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + TchOps::bitwise_xor(lhs, rhs) + } + + fn bitwise_not(tensor: IntTensor) -> IntTensor { + TchOps::bitwise_not(tensor) + } + + fn bitwise_and_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + TchOps::bitwise_and_scalar(lhs, rhs.elem::()) + } + + fn bitwise_or_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + TchOps::bitwise_or_scalar(lhs, rhs.elem::()) + } + + fn bitwise_xor_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + TchOps::bitwise_xor_scalar(lhs, rhs.elem::()) + } + + fn bitwise_left_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + TchOps::bitwise_left_shift(lhs, rhs) + } + + fn bitwise_right_shift(lhs: IntTensor, rhs: IntTensor) -> IntTensor { + TchOps::bitwise_right_shift(lhs, rhs) + } + + fn bitwise_left_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + TchOps::bitwise_left_shift_scalar(lhs, rhs.elem::()) + } + + fn bitwise_right_shift_scalar(lhs: IntTensor, rhs: Scalar) -> IntTensor { + TchOps::bitwise_right_shift_scalar(lhs, rhs.elem::()) + } + + fn int_cast(tensor: IntTensor, dtype: IntDType) -> IntTensor { + // NOTE: when dtypes of inputs to an arithmetic operation differ, tch handles type + // promotion based on a set of rules: https://pytorch.org/docs/stable/tensor_attributes.html#type-promotion-doc + + // Type promotion is not automatic on all backends so this behavior might differ + let kind = dtype.into_kind(); + + if tensor.tensor.kind() == kind { + tensor + } else { + TchTensor::new(tensor.tensor.to_kind(kind)) + } + } + + fn int_unfold( + tensor: IntTensor, + dim: usize, + size: usize, + step: usize, + ) -> IntTensor { + TchOps::unfold(tensor, dim, size, step) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/mod.rs new file mode 100644 index 0000000..cebe9d2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/mod.rs @@ -0,0 +1,10 @@ +mod activation; +mod base; +mod bool_tensor; +mod int_tensor; +mod module; +mod qtensor; +mod tensor; +mod transaction; + +pub(crate) use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/module.rs new file mode 100644 index 0000000..e8f398b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/module.rs @@ -0,0 +1,473 @@ +use crate::{LibTorch, TchTensor, element::TchElement}; +use burn_backend::{ + TensorMetadata, + ops::{ + AttentionModuleOptions, ConvOptions, ConvTransposeOptions, DeformConv2dBackward, + DeformConvOptions, InterpolateMode, InterpolateOptions, MaxPool1dWithIndices, + MaxPool2dBackward, MaxPool2dWithIndices, ModuleOps, attention::attention_fallback, + }, +}; + +impl ModuleOps for LibTorch { + fn embedding(weights: TchTensor, indices: TchTensor) -> TchTensor { + // Workaround for MPS "Placeholder storage has not been allocated" error. + // See: https://github.com/pytorch/pytorch/issues/123995 + // MPS uses lazy allocation and the embedding operation (which uses index_select) + // can fail if the tensors haven't been materialized yet. + // We work around this by performing the embedding on CPU and transferring back to MPS. + if matches!(weights.tensor.device(), tch::Device::Mps) { + let cpu_weights = weights.tensor.to(tch::Device::Cpu); + let cpu_indices = indices.tensor.to(tch::Device::Cpu); + let result = tch::Tensor::embedding(&cpu_weights, &cpu_indices, -1, false, false) + .to(tch::Device::Mps); + return TchTensor::new(result); + } + + let tensor = tch::Tensor::embedding(&weights.tensor, &indices.tensor, -1, false, false); + TchTensor::new(tensor) + } + + fn embedding_backward(weights: TchTensor, output: TchTensor, indices: TchTensor) -> TchTensor { + let [n_embedding, _d_model] = weights.shape().dims(); + + // Workaround for MPS "Placeholder storage has not been allocated" error. + // See: https://github.com/pytorch/pytorch/issues/123995 + if matches!(output.tensor.device(), tch::Device::Mps) { + let cpu_output = output.tensor.to(tch::Device::Cpu); + let cpu_indices = indices.tensor.to(tch::Device::Cpu); + let result = tch::Tensor::embedding_backward( + &cpu_output, + &cpu_indices, + n_embedding as i64, + -1, + false, + false, + ) + .to(tch::Device::Mps); + return TchTensor::new(result); + } + + let tensor = tch::Tensor::embedding_backward( + &output.tensor, + &indices.tensor, + n_embedding as i64, + -1, + false, + false, + ); + + TchTensor::new(tensor) + } + + fn conv1d( + x: TchTensor, + weight: TchTensor, + bias: Option, + options: ConvOptions<1>, + ) -> TchTensor { + let tensor = tch::Tensor::conv1d( + &x.tensor, + &weight.tensor, + bias.map(|t| t.tensor), + options.stride.map(|i| i as i64), + options.padding.map(|i| i as i64), + options.dilation.map(|i| i as i64), + options.groups as i64, + ); + + TchTensor::new(tensor) + } + + fn conv2d( + x: TchTensor, + weight: TchTensor, + bias: Option, + options: ConvOptions<2>, + ) -> TchTensor { + let tensor = tch::Tensor::conv2d( + &x.tensor, + &weight.tensor, + bias.map(|t| t.tensor), + options.stride.map(|i| i as i64), + options.padding.map(|i| i as i64), + options.dilation.map(|i| i as i64), + options.groups as i64, + ); + + TchTensor::new(tensor) + } + + fn conv3d( + x: TchTensor, + weight: TchTensor, + bias: Option, + options: ConvOptions<3>, + ) -> TchTensor { + let tensor = tch::Tensor::conv3d( + &x.tensor, + &weight.tensor, + bias.map(|t| t.tensor), + options.stride.map(|i| i as i64), + options.padding.map(|i| i as i64), + options.dilation.map(|i| i as i64), + options.groups as i64, + ); + + TchTensor::new(tensor) + } + + fn deform_conv2d( + _x: TchTensor, + _offset: TchTensor, + _weight: TchTensor, + _mask: Option, + _bias: Option, + _options: DeformConvOptions<2>, + ) -> TchTensor { + unimplemented!("Torch bindings don't support deform_conv2d"); + } + + fn deform_conv2d_backward( + _x: TchTensor, + _offset: TchTensor, + _weight: TchTensor, + _mask: Option, + _bias: Option, + _out_grad: TchTensor, + _options: DeformConvOptions<2>, + ) -> DeformConv2dBackward { + unimplemented!("Torch bindings don't support deform_conv2d"); + } + + fn conv_transpose1d( + x: TchTensor, + weight: TchTensor, + bias: Option, + options: ConvTransposeOptions<1>, + ) -> TchTensor { + let tensor = tch::Tensor::conv_transpose1d( + &x.tensor, + &weight.tensor, + bias.map(|t| t.tensor), + options.stride.map(|i| i as i64), + options.padding.map(|i| i as i64), + options.padding_out.map(|i| i as i64), + options.groups as i64, + options.dilation.map(|i| i as i64), + ); + + TchTensor::new(tensor) + } + + fn conv_transpose2d( + x: TchTensor, + weight: TchTensor, + bias: Option, + options: ConvTransposeOptions<2>, + ) -> TchTensor { + let tensor = tch::Tensor::conv_transpose2d( + &x.tensor, + &weight.tensor, + bias.map(|t| t.tensor), + options.stride.map(|i| i as i64), + options.padding.map(|i| i as i64), + options.padding_out.map(|i| i as i64), + options.groups as i64, + options.dilation.map(|i| i as i64), + ); + + TchTensor::new(tensor) + } + + fn conv_transpose3d( + x: TchTensor, + weight: TchTensor, + bias: Option, + options: ConvTransposeOptions<3>, + ) -> TchTensor { + let tensor = tch::Tensor::conv_transpose3d( + &x.tensor, + &weight.tensor, + bias.map(|t| t.tensor), + options.stride.map(|i| i as i64), + options.padding.map(|i| i as i64), + options.padding_out.map(|i| i as i64), + options.groups as i64, + options.dilation.map(|i| i as i64), + ); + + TchTensor::new(tensor) + } + + fn avg_pool1d( + x: TchTensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, + ) -> TchTensor { + let tensor = tch::Tensor::avg_pool1d( + &x.tensor, + [kernel_size as i64], + [stride as i64], + [padding as i64], + ceil_mode, + count_include_pad, + ); + + TchTensor::new(tensor) + } + fn avg_pool2d( + x: TchTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> TchTensor { + let tensor = tch::Tensor::avg_pool2d( + &x.tensor, + [kernel_size[0] as i64, kernel_size[1] as i64], + [stride[0] as i64, stride[1] as i64], + [padding[0] as i64, padding[1] as i64], + ceil_mode, + count_include_pad, + None, + ); + + TchTensor::new(tensor) + } + + fn avg_pool2d_backward( + x: TchTensor, + grad: TchTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, + ) -> TchTensor { + let tensor = tch::Tensor::avg_pool2d_backward( + &x.tensor, + &grad.tensor, + [kernel_size[0] as i64, kernel_size[1] as i64], + [stride[0] as i64, stride[1] as i64], + [padding[0] as i64, padding[1] as i64], + ceil_mode, + count_include_pad, + None, + ); + + TchTensor::new(tensor) + } + + fn max_pool1d( + x: TchTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> TchTensor { + let tensor = tch::Tensor::max_pool1d( + &x.tensor, + kernel_size as i64, + stride as i64, + padding as i64, + dilation as i64, + ceil_mode, + ); + + TchTensor::new(tensor) + } + + fn max_pool1d_with_indices( + x: TchTensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, + ) -> MaxPool1dWithIndices { + let (tensor, indices) = tch::Tensor::max_pool1d_with_indices( + &x.tensor, + kernel_size as i64, + stride as i64, + padding as i64, + dilation as i64, + ceil_mode, + ); + + MaxPool1dWithIndices::new(TchTensor::new(tensor), TchTensor::new(indices)) + } + + fn max_pool2d( + x: TchTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> TchTensor { + let tensor = tch::Tensor::max_pool2d( + &x.tensor, + [kernel_size[0] as i64, kernel_size[1] as i64], + [stride[0] as i64, stride[1] as i64], + [padding[0] as i64, padding[1] as i64], + [dilation[0] as i64, dilation[1] as i64], + ceil_mode, + ); + + TchTensor::new(tensor) + } + + fn max_pool2d_with_indices( + x: TchTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + ) -> MaxPool2dWithIndices { + let (tensor, indices) = tch::Tensor::max_pool2d_with_indices( + &x.tensor, + [kernel_size[0] as i64, kernel_size[1] as i64], + [stride[0] as i64, stride[1] as i64], + [padding[0] as i64, padding[1] as i64], + [dilation[0] as i64, dilation[1] as i64], + ceil_mode, + ); + + MaxPool2dWithIndices::new(TchTensor::new(tensor), TchTensor::new(indices)) + } + + fn max_pool2d_with_indices_backward( + x: TchTensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, + output_grad: TchTensor, + indices: TchTensor, + ) -> MaxPool2dBackward { + let grad = tch::Tensor::max_pool2d_with_indices_backward( + &x.tensor, + &output_grad.tensor, + [kernel_size[0] as i64, kernel_size[1] as i64], + [stride[0] as i64, stride[1] as i64], + [padding[0] as i64, padding[1] as i64], + [dilation[0] as i64, dilation[1] as i64], + ceil_mode, + &indices.tensor, + ); + + MaxPool2dBackward::new(TchTensor::new(grad)) + } + + fn adaptive_avg_pool2d(x: TchTensor, output_size: [usize; 2]) -> TchTensor { + let tensor = tch::Tensor::adaptive_avg_pool2d(&x.tensor, output_size.map(|e| e as i64)); + + TchTensor::new(tensor) + } + + fn adaptive_avg_pool2d_backward(x: TchTensor, grad: TchTensor) -> TchTensor { + let tensor = tch::Tensor::internal_adaptive_avg_pool2d_backward(&x.tensor, &grad.tensor); + + TchTensor::new(tensor) + } + + fn adaptive_avg_pool1d(x: TchTensor, output_size: usize) -> TchTensor { + let tensor = tch::Tensor::adaptive_avg_pool1d(&x.tensor, output_size as i64); + + TchTensor::new(tensor) + } + + fn interpolate( + x: TchTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> TchTensor { + let output_size = output_size.map(|e| e as i64); + + let align_corners = options.align_corners; + let tensor = match options.mode { + InterpolateMode::Nearest => { + tch::Tensor::upsample_nearest2d(&x.tensor, output_size, None, None) + } + InterpolateMode::Bilinear => { + tch::Tensor::upsample_bilinear2d(&x.tensor, output_size, align_corners, None, None) + } + InterpolateMode::Bicubic => { + tch::Tensor::upsample_bicubic2d(&x.tensor, output_size, align_corners, None, None) + } + }; + + TchTensor::new(tensor) + } + + fn interpolate_backward( + x: TchTensor, + grad: TchTensor, + output_size: [usize; 2], + options: InterpolateOptions, + ) -> TchTensor { + let output_size = output_size.map(|e| e as i64); + let [n, c, h_in, w_in] = x.shape().dims(); + let input_size = [n as i64, c as i64, h_in as i64, w_in as i64]; + let align_corners = options.align_corners; + + let tensor = match options.mode { + InterpolateMode::Nearest => tch::Tensor::upsample_nearest2d_backward( + &grad.tensor, + output_size, + input_size, + None, + None, + ), + InterpolateMode::Bilinear => tch::Tensor::upsample_bilinear2d_backward( + &grad.tensor, + output_size, + input_size, + align_corners, + None, + None, + ), + InterpolateMode::Bicubic => tch::Tensor::upsample_bicubic2d_backward( + &grad.tensor, + output_size, + input_size, + align_corners, + None, + None, + ), + }; + + TchTensor::new(tensor) + } + + fn attention( + query: TchTensor, + key: TchTensor, + value: TchTensor, + mask: Option, + attn_bias: Option, + options: AttentionModuleOptions, + ) -> TchTensor { + if attn_bias.is_some() { + return attention_fallback::(query, key, value, mask, attn_bias, options); + } + + TchTensor::new(tch::Tensor::scaled_dot_product_attention( + &query.tensor, + &key.tensor, + &value.tensor, + mask.map(|m| m.tensor), + 0., + options.is_causal, + options.scale, + false, + )) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/qtensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/qtensor.rs new file mode 100644 index 0000000..7d82b7d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/qtensor.rs @@ -0,0 +1,140 @@ +use burn_backend::{ + ExecutionError, Shape, TensorData, + ops::QTensorOps, + quantization::{QuantScheme, QuantizationParametersPrimitive}, + tensor::{Device, FloatTensor, IntTensor, QuantizedTensor}, +}; + +use crate::{LibTorch, LibTorchDevice, TchElement}; + +impl QTensorOps for LibTorch { + fn q_from_data(_data: TensorData, _device: &LibTorchDevice) -> QuantizedTensor { + unimplemented!() + } + + fn quantize( + _tensor: FloatTensor, + _scheme: &QuantScheme, + _qparams: QuantizationParametersPrimitive, + ) -> QuantizedTensor { + unimplemented!() + } + + fn quantize_dynamic( + _tensor: FloatTensor, + _scheme: &QuantScheme, + ) -> QuantizedTensor { + unimplemented!() + } + + fn dequantize(_tensor: QuantizedTensor) -> FloatTensor { + unimplemented!() + } + + fn q_device(_tensor: &QuantizedTensor) -> LibTorchDevice { + unimplemented!() + } + + fn q_to_device( + _tensor: QuantizedTensor, + _device: &Device, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_reshape(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } + + async fn q_into_data(_tensor: QuantizedTensor) -> Result { + unimplemented!() + } + fn q_swap_dims( + _tensor: QuantizedTensor, + _dim1: usize, + _dim2: usize, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_permute(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_flip(_tensor: QuantizedTensor, _axes: &[usize]) -> QuantizedTensor { + unimplemented!() + } + + fn q_select( + _tensor: QuantizedTensor, + _dim: usize, + _indices: IntTensor, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_slice( + _tensor: QuantizedTensor, + _slices: &[burn_backend::Slice], + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_argmax(_tensor: QuantizedTensor, _dim: usize) -> IntTensor { + unimplemented!() + } + + fn q_argmin(_tensor: QuantizedTensor, _dim: usize) -> IntTensor { + unimplemented!() + } + + fn q_max_dim_with_indices( + _tensor: QuantizedTensor, + _dim: usize, + ) -> (QuantizedTensor, IntTensor) { + unimplemented!() + } + + fn q_max_dim(_tensor: QuantizedTensor, _dim: usize) -> QuantizedTensor { + unimplemented!() + } + + fn q_min_dim(_tensor: QuantizedTensor, _dim: usize) -> QuantizedTensor { + unimplemented!() + } + + fn q_min_dim_with_indices( + _tensor: QuantizedTensor, + _dim: usize, + ) -> (QuantizedTensor, IntTensor) { + unimplemented!() + } + + fn q_expand(_tensor: QuantizedTensor, _shape: Shape) -> QuantizedTensor { + unimplemented!() + } + + fn q_sort( + _tensor: QuantizedTensor, + _dim: usize, + _descending: bool, + ) -> QuantizedTensor { + unimplemented!() + } + + fn q_sort_with_indices( + _tensor: QuantizedTensor, + _dim: usize, + _descending: bool, + ) -> (QuantizedTensor, IntTensor) { + unimplemented!() + } + + fn q_argsort( + _tensor: QuantizedTensor, + _dim: usize, + _descending: bool, + ) -> IntTensor { + unimplemented!() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/tensor.rs new file mode 100644 index 0000000..d99c97a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/tensor.rs @@ -0,0 +1,539 @@ +use super::TchOps; +use crate::{IntoKind, LibTorch, LibTorchDevice, TchShape, TchTensor, element::TchElement}; +use burn_backend::backend::ExecutionError; +use burn_backend::tensor::{BoolTensor, FloatTensor, IntTensor}; +use burn_backend::{ + DType, Distribution, FloatDType, Shape, TensorData, TensorMetadata, ops::FloatTensorOps, +}; +use burn_backend::{Scalar, bf16, f16}; + +impl FloatTensorOps for LibTorch { + fn float_from_data(data: TensorData, device: &LibTorchDevice) -> TchTensor { + match data.dtype { + DType::F64 => TchTensor::from_data::(data, (*device).into()), + DType::F32 => TchTensor::from_data::(data, (*device).into()), + DType::F16 => TchTensor::from_data::(data, (*device).into()), + DType::BF16 => TchTensor::from_data::(data, (*device).into()), + _ => unimplemented!("Unsupported dtype for `float_from_data`"), + } + } + + fn float_random( + shape: Shape, + distribution: Distribution, + device: &LibTorchDevice, + ) -> TchTensor { + match distribution { + Distribution::Default => { + let mut tensor = TchTensor::empty::(shape, *device); + tensor + .mut_ops(|tensor| tensor.rand_like_out(tensor)) + .unwrap() + } + Distribution::Bernoulli(prob) => { + let mut tensor = TchTensor::empty::(shape, *device); + tensor + .mut_ops(|tensor| tensor.f_bernoulli_float_(prob).unwrap()) + .unwrap() + } + Distribution::Uniform(from, to) => { + let mut tensor = TchTensor::empty::(shape, *device); + tensor.mut_ops(|tensor| tensor.uniform_(from, to)).unwrap() + } + Distribution::Normal(mean, std) => { + let mut tensor = TchTensor::empty::(shape, *device); + tensor.mut_ops(|tensor| tensor.normal_(mean, std)).unwrap() + } + } + } + + fn float_repeat_dim(tensor: TchTensor, dim: usize, times: usize) -> TchTensor { + TchOps::repeat_dim(tensor, dim, times) + } + + fn float_zeros(shape: Shape, device: &LibTorchDevice, dtype: FloatDType) -> TchTensor { + let shape = TchShape::from(shape); + let device: tch::Device = (*device).into(); + + TchTensor::new(tch::Tensor::zeros(shape.dims, (dtype.into_kind(), device))) + } + + fn float_ones(shape: Shape, device: &LibTorchDevice, dtype: FloatDType) -> TchTensor { + let shape = TchShape::from(shape); + let device: tch::Device = (*device).into(); + + TchTensor::new(tch::Tensor::ones(shape.dims, (dtype.into_kind(), device))) + } + + async fn float_into_data(tensor: TchTensor) -> Result { + let shape = tensor.shape(); + let tensor = Self::float_reshape(tensor.clone(), Shape::new([shape.num_elements()])); + Ok(match tensor.tensor.kind() { + tch::Kind::Half => { + let values = Vec::::try_from(&tensor).unwrap(); + TensorData::new(values, shape) + } + tch::Kind::Float => { + let values = Vec::::try_from(&tensor).unwrap(); + TensorData::new(values, shape) + } + tch::Kind::Double => { + let values = Vec::::try_from(&tensor).unwrap(); + TensorData::new(values, shape) + } + tch::Kind::BFloat16 => { + let values = Vec::::try_from(&tensor).unwrap(); + TensorData::new(values, shape) + } + _ => panic!("Not a valid float kind"), + }) + } + + fn float_device(tensor: &TchTensor) -> LibTorchDevice { + tensor.tensor.device().into() + } + + fn float_to_device(tensor: TchTensor, device: &LibTorchDevice) -> TchTensor { + TchOps::to_device(tensor, device) + } + + fn float_empty(shape: Shape, device: &LibTorchDevice, dtype: FloatDType) -> TchTensor { + let tensor = tch::Tensor::empty( + TchShape::from(shape).dims, + (dtype.into_kind(), (*device).into()), + ); + + TchTensor::new(tensor) + } + + fn float_add(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::add(lhs, rhs) + } + + fn float_add_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + let rhs: f64 = rhs.elem(); + + lhs.unary_ops( + |mut tensor| tensor.f_add_scalar_(rhs).unwrap(), + |tensor| tensor.f_add_scalar(rhs).unwrap(), + ) + } + + fn float_sub(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::sub(lhs, rhs) + } + + fn float_sub_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + let rhs: f64 = rhs.elem(); + + lhs.unary_ops( + |mut tensor| tensor.f_sub_scalar_(rhs).unwrap(), + |tensor| tensor.f_sub_scalar(rhs).unwrap(), + ) + } + + fn float_mul(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::mul(lhs, rhs) + } + + fn float_mul_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + let rhs: f64 = rhs.elem(); + + lhs.unary_ops( + |mut tensor| tensor.f_mul_scalar_(rhs).unwrap(), + |tensor| tensor.f_mul_scalar(rhs).unwrap(), + ) + } + + fn float_div(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::div(lhs, rhs) + } + + fn float_div_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + let rhs: f64 = rhs.elem(); + + lhs.unary_ops( + |mut tensor| tensor.f_div_scalar_(rhs).unwrap(), + |tensor| tensor.f_div_scalar(rhs).unwrap(), + ) + } + + fn float_remainder(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::remainder(lhs, rhs) + } + + fn float_remainder_scalar(lhs: TchTensor, rhs: Scalar) -> TchTensor { + let rhs: f64 = rhs.elem(); + + lhs.unary_ops( + |tensor| tensor.f_remainder(rhs).unwrap(), + |tensor| tensor.f_remainder(rhs).unwrap(), + ) + } + + fn float_matmul(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + let tensor = lhs.tensor.matmul(&rhs.tensor); + TchTensor::new(tensor) + } + + fn float_cross(lhs: TchTensor, rhs: TchTensor, dim: usize) -> TchTensor { + let tensor = lhs.tensor.cross(&rhs.tensor, dim as i64); + TchTensor::new(tensor) + } + + fn float_recip(tensor: TchTensor) -> TchTensor { + TchTensor::new(tensor.tensor.reciprocal()) + } + + fn float_swap_dims(tensor: TchTensor, dim1: usize, dim2: usize) -> TchTensor { + TchOps::swap_dims(tensor, dim1, dim2) + } + + fn float_reshape(tensor: TchTensor, shape: Shape) -> TchTensor { + TchOps::reshape(tensor, shape) + } + + fn float_gather(dim: usize, tensor: TchTensor, indices: TchTensor) -> TchTensor { + TchOps::gather(dim, tensor, indices) + } + + fn float_scatter_add( + dim: usize, + tensor: TchTensor, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + TchOps::scatter(dim, tensor, indices, value) + } + + fn float_select(tensor: TchTensor, dim: usize, indices: TchTensor) -> TchTensor { + TchOps::index_select_dim(tensor, dim, indices) + } + + fn float_select_add( + tensor: TchTensor, + dim: usize, + indices: TchTensor, + value: TchTensor, + ) -> TchTensor { + TchOps::select_assign(tensor, dim, indices, value) + } + + fn float_slice(tensor: TchTensor, slices: &[burn_backend::Slice]) -> TchTensor { + TchOps::slice_with_steps(tensor, slices) + } + + fn float_slice_assign( + tensor: TchTensor, + slices: &[burn_backend::Slice], + value: TchTensor, + ) -> TchTensor { + TchOps::slice_assign(tensor, slices, value) + } + + fn float_mask_where(tensor: TchTensor, mask: TchTensor, value: TchTensor) -> TchTensor { + let output = value.tensor.where_self(&mask.tensor, &tensor.tensor); + + TchTensor::new(output) + } + + fn float_mask_fill(tensor: TchTensor, mask: TchTensor, value: Scalar) -> TchTensor { + let value: f64 = value.elem(); + + tensor.unary_ops( + |mut tensor| tensor.f_masked_fill_(&mask.tensor, value).unwrap(), + |tensor| tensor.f_masked_fill(&mask.tensor, value).unwrap(), + ) + } + + fn float_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::equal(lhs, rhs) + } + + fn float_equal_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::equal_elem(lhs, rhs.elem::()) + } + + fn float_greater(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::greater(lhs, rhs) + } + + fn float_greater_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::greater_elem(lhs, rhs.elem::()) + } + + fn float_greater_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::greater_equal(lhs, rhs) + } + + fn float_greater_equal_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::greater_equal_elem(lhs, rhs.elem::()) + } + + fn float_lower(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::lower(lhs, rhs) + } + + fn float_lower_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::lower_elem(lhs, rhs.elem::()) + } + + fn float_lower_equal(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::lower_equal(lhs, rhs) + } + + fn float_lower_equal_elem(lhs: TchTensor, rhs: Scalar) -> TchTensor { + TchOps::lower_equal_elem(lhs, rhs.elem::()) + } + + fn float_mean(tensor: TchTensor) -> TchTensor { + TchOps::mean(tensor) + } + + fn float_sum(tensor: TchTensor) -> TchTensor { + TchOps::sum(tensor) + } + + fn float_sum_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::sum_dim(tensor, dim) + } + + fn float_mean_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::mean_dim(tensor, dim) + } + + fn float_cumsum(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cumsum(tensor, dim) + } + + fn float_cumprod(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cumprod(tensor, dim) + } + + fn float_cummin(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cummin(tensor, dim) + } + + fn float_cummax(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::cummax(tensor, dim) + } + + fn float_prod(tensor: TchTensor) -> TchTensor { + TchOps::prod(tensor) + } + + fn float_prod_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::prod_dim(tensor, dim) + } + + fn float_argmax(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::argmax(tensor, dim) + } + + fn float_argmin(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::argmin(tensor, dim) + } + + fn float_max_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::max_dim(tensor, dim) + } + + fn float_max_dim_with_indices(tensor: TchTensor, dim: usize) -> (TchTensor, TchTensor) { + TchOps::max_dim_with_indices(tensor, dim) + } + + fn float_min_dim(tensor: TchTensor, dim: usize) -> TchTensor { + TchOps::min_dim(tensor, dim) + } + + fn float_min_dim_with_indices(tensor: TchTensor, dim: usize) -> (TchTensor, TchTensor) { + TchOps::min_dim_with_indices(tensor, dim) + } + + fn float_exp(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.exp_(), |tensor| tensor.exp()) + } + + fn float_log(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.log_(), |tensor| tensor.log()) + } + + fn float_log1p(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.log1p_(), |tensor| tensor.log1p()) + } + + fn float_powf_scalar_impl(tensor: TchTensor, value: Scalar) -> TchTensor { + tensor.unary_ops( + |mut tensor| tensor.f_pow_(value.elem::()).unwrap(), + |tensor| tensor.pow_tensor_scalar(value.elem::()), + ) + } + + fn float_sqrt(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.sqrt_(), |tensor| tensor.sqrt()) + } + + fn float_abs(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.abs_(), |tensor| tensor.abs()) + } + + fn float_cos(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.cos_(), |tensor| tensor.cos()) + } + + fn float_cosh(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.cosh_(), |tensor| tensor.cosh()) + } + + fn float_sin(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.sin_(), |tensor| tensor.sin()) + } + + fn float_sinh(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.sinh_(), |tensor| tensor.sinh()) + } + + fn float_tan(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.tan_(), |tensor| tensor.tan()) + } + + fn float_tanh(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.tanh_(), |tensor| tensor.tanh()) + } + + fn float_acos(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.acos_(), |tensor| tensor.acos()) + } + + fn float_acosh(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.acosh_(), |tensor| tensor.acosh()) + } + + fn float_asin(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.asin_(), |tensor| tensor.asin()) + } + + fn float_asinh(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.asinh_(), |tensor| tensor.asinh()) + } + + fn float_atan(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.atan_(), |tensor| tensor.atan()) + } + + fn float_atanh(tensor: FloatTensor) -> FloatTensor { + tensor.unary_ops(|mut tensor| tensor.atanh_(), |tensor| tensor.atanh()) + } + + fn float_atan2(lhs: FloatTensor, rhs: FloatTensor) -> FloatTensor { + TchOps::atan2(lhs, rhs) + } + + fn float_round(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.round_(), |tensor| tensor.round()) + } + + fn float_floor(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.floor_(), |tensor| tensor.floor()) + } + + fn float_ceil(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.ceil_(), |tensor| tensor.ceil()) + } + + fn float_trunc(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.trunc_(), |tensor| tensor.trunc()) + } + + fn float_erf(tensor: TchTensor) -> TchTensor { + tensor.unary_ops(|mut tensor| tensor.erf_(), |tensor| tensor.erf()) + } + + fn float_cat(tensors: Vec, dim: usize) -> TchTensor { + TchOps::cat(tensors, dim) + } + + fn float_clamp_min(tensor: TchTensor, min: Scalar) -> TchTensor { + TchOps::clamp_min(tensor, min.elem::()) + } + + fn float_clamp_max(tensor: TchTensor, max: Scalar) -> TchTensor { + TchOps::clamp_max(tensor, max.elem::()) + } + + fn float_clamp(tensor: TchTensor, min: Scalar, max: Scalar) -> TchTensor { + TchOps::clamp(tensor, min.elem::(), max.elem::()) + } + + fn float_into_int(tensor: TchTensor) -> TchTensor { + let tensor = tensor.tensor.to_kind(tch::Kind::Int64); + TchTensor::new(tensor) + } + + fn float_powf(lhs: TchTensor, rhs: TchTensor) -> TchTensor { + TchOps::powf(lhs, rhs) + } + + fn float_permute(tensor: TchTensor, axes: &[usize]) -> TchTensor { + TchOps::permute(tensor, axes) + } + + fn float_flip(tensor: TchTensor, axes: &[usize]) -> TchTensor { + TchOps::flip(tensor, axes) + } + + fn float_sign(tensor: TchTensor) -> TchTensor { + TchOps::sign(tensor) + } + + fn float_expand(tensor: TchTensor, shape: Shape) -> TchTensor { + TchOps::expand(tensor, shape) + } + + fn float_sort(tensor: TchTensor, dim: usize, descending: bool) -> TchTensor { + TchOps::sort(tensor, dim, descending) + } + + fn float_sort_with_indices( + tensor: TchTensor, + dim: usize, + descending: bool, + ) -> (TchTensor, TchTensor) { + TchOps::sort_with_indices(tensor, dim, descending) + } + + fn float_argsort(tensor: TchTensor, dim: usize, descending: bool) -> IntTensor { + TchOps::argsort(tensor, dim, descending) + } + + fn float_cast(tensor: TchTensor, dtype: FloatDType) -> TchTensor { + // NOTE: when dtypes of inputs to an arithmetic operation differ, tch handles type + // promotion based on a set of rules: https://pytorch.org/docs/stable/tensor_attributes.html#type-promotion-doc + + // Type promotion is not automatic on all backends so this behavior might differ + let kind = dtype.into_kind(); + + if tensor.tensor.kind() == kind { + tensor + } else { + TchTensor::new(tensor.tensor.to_kind(kind)) + } + } + + fn float_unfold( + tensor: FloatTensor, + dim: usize, + size: usize, + step: usize, + ) -> FloatTensor { + TchOps::unfold(tensor, dim, size, step) + } + + fn float_is_nan(tensor: FloatTensor) -> BoolTensor { + TchTensor::new(tensor.tensor.isnan()) + } + + fn float_is_inf(tensor: FloatTensor) -> BoolTensor { + TchTensor::new(tensor.tensor.isinf()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/transaction.rs new file mode 100644 index 0000000..323c25b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/ops/transaction.rs @@ -0,0 +1,5 @@ +use burn_backend::ops::TransactionOps; + +use crate::{LibTorch, TchElement}; + +impl TransactionOps for LibTorch {} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tch/src/tensor.rs b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/tensor.rs new file mode 100644 index 0000000..a6b1bee --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tch/src/tensor.rs @@ -0,0 +1,507 @@ +use crate::{LibTorchDevice, TchElement}; +use burn_backend::{DType, FloatDType, IntDType, Shape, TensorData, TensorMetadata}; +use libc::c_void; +use std::sync::Arc; + +/// A reference to a tensor storage. +/// +/// We manually implement `Sync` and `Send` unsafely, so even if we could use `Rc`, it isn't safe. +#[allow(clippy::arc_with_non_send_sync)] +pub type StorageRef = Arc<*mut c_void>; + +/// A reference to a tensor storage. +#[derive(PartialEq, Debug, Clone)] +pub enum Storage { + /// When a tensor is a partial view of another tensor. + View { + /// Storage reference for the whole buffer. + buffer_ref: StorageRef, + /// Storage reference for the partial buffer. + view_ref: StorageRef, + }, + /// When a tensor use all of its buffer. + Owned { + /// Storage reference for the whole buffer. + buffer_ref: StorageRef, + }, +} + +impl Storage { + /// Check if the storage can be used inplace. + pub fn can_mut(&self) -> bool { + match self { + Storage::View { + buffer_ref: start_ref, + view_ref, + } => Arc::strong_count(start_ref) == 1 && Arc::strong_count(view_ref) == 1, + Storage::Owned { + buffer_ref: start_ref, + } => Arc::strong_count(start_ref) == 1, + } + } + + /// Get the whole buffer reference. + pub fn buffer_ref(&self) -> &StorageRef { + match self { + Storage::View { + buffer_ref: start_ref, + view_ref: _, + } => start_ref, + Storage::Owned { + buffer_ref: start_ref, + } => start_ref, + } + } +} + +/// A tensor using the tch backend. +#[derive(Debug, PartialEq)] +pub struct TchTensor { + /// Handle to the tensor. Call methods on this field. + pub tensor: tch::Tensor, + + /// The tensor's storage + pub storage: Storage, +} + +impl TensorMetadata for TchTensor { + fn dtype(&self) -> DType { + match self.tensor.kind() { + tch::Kind::Uint8 => DType::U8, + tch::Kind::Int8 => DType::I8, + tch::Kind::Int16 => DType::I16, + tch::Kind::Int => DType::I32, + tch::Kind::Int64 => DType::I64, + tch::Kind::Half => DType::F16, + tch::Kind::Float => DType::F32, + tch::Kind::Double => DType::F64, + tch::Kind::Bool => DType::Bool, + tch::Kind::BFloat16 => DType::BF16, + // Complex and quantization types are not valid/implemented. + _ => unimplemented!(), + } + } + + fn shape(&self) -> Shape { + Shape::from(self.tensor.size()) + } + + fn rank(&self) -> usize { + self.tensor.dim() + } +} + +impl burn_backend::QTensorPrimitive for TchTensor { + fn scheme(&self) -> &burn_backend::quantization::QuantScheme { + unimplemented!("Quantization is not supported") + } +} + +impl core::fmt::Display for TchTensor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.tensor) + } +} + +pub(crate) trait IntoKind { + fn try_into_kind(self) -> Result; + fn into_kind(self) -> tch::Kind + where + Self: Sized, + { + self.try_into_kind().unwrap() + } +} + +impl IntoKind for IntDType { + fn try_into_kind(self) -> Result { + let dtype: DType = self.into(); + dtype.try_into_kind() + } +} + +impl IntoKind for FloatDType { + fn try_into_kind(self) -> Result { + let dtype: DType = self.into(); + dtype.try_into_kind() + } +} + +impl IntoKind for DType { + fn try_into_kind(self) -> Result { + match self { + DType::F64 => Ok(tch::Kind::Double), + DType::F32 => Ok(tch::Kind::Float), + DType::Flex32 => Ok(tch::Kind::Float), + DType::F16 => Ok(tch::Kind::Half), + DType::BF16 => Ok(tch::Kind::BFloat16), + DType::I64 => Ok(tch::Kind::Int64), + DType::I32 => Ok(tch::Kind::Int), + DType::I16 => Ok(tch::Kind::Int16), + DType::I8 => Ok(tch::Kind::Int8), + DType::U8 => Ok(tch::Kind::Uint8), + DType::Bool => Ok(tch::Kind::Bool), + other => Err(tch::TchError::Kind(format!("Unsupported dtype {other:?}"))), + } + } +} + +impl TchTensor { + /// Create a new tensor. + /// + /// Note that if the tensor was created from an operation that may reuse the same tensor + /// storage as the parent, you should use [from_existing](TchTensor::from_existing) + /// instead. + pub fn new(tensor: tch::Tensor) -> Self { + #[allow(clippy::arc_with_non_send_sync)] + let storage = Storage::Owned { + buffer_ref: Arc::new(tensor.data_ptr()), + }; + + Self { tensor, storage } + } + + /// Create a tensor that was created from an operation executed on a parent tensor. + /// + /// If the child tensor shared the same storage as its parent, it will be cloned, effectively + /// tracking how much tensors point to the same memory space. + pub fn from_existing(tensor: tch::Tensor, storage_parent: Storage) -> Self { + let storage_child = tensor.data_ptr(); + let mut is_a_new_tensor = true; + + match &storage_parent { + Storage::View { + buffer_ref: start_ref, + view_ref, + } => { + if storage_child == *start_ref.as_ref() || storage_child == *view_ref.as_ref() { + is_a_new_tensor = false; + } + } + Storage::Owned { + buffer_ref: start_ref, + } => { + if storage_child == *start_ref.as_ref() { + is_a_new_tensor = false; + } + } + }; + + let storage = match is_a_new_tensor { + true => Storage::Owned { + #[allow(clippy::arc_with_non_send_sync)] + buffer_ref: Arc::new(storage_child), + }, + false => storage_parent.clone(), + }; + + Self { tensor, storage } + } + + /// Create a tensor that uses a part of its parent tensor such as slice and narrow. + pub fn partial(tensor: tch::Tensor, storage_parent: Storage) -> Self { + let storage = Storage::View { + buffer_ref: storage_parent.buffer_ref().clone(), + #[allow(clippy::arc_with_non_send_sync)] + view_ref: Arc::new(tensor.data_ptr()), + }; + Self { tensor, storage } + } +} + +// This is safe since we don't use autodiff from LibTorch. +// Also, atomic reference counting is used to know if the tensor's data can be reused. +// If there are multiple reference on the same tensor, it becomes read only. +unsafe impl Send for TchTensor {} +unsafe impl Sync for TchTensor {} + +impl TchTensor { + /// Checks if the tensor can be mutated in-place. + /// + /// Returns `true` if the tensor's stride does not contain zero (no broadcasting) + /// and the storage can be mutated. + pub fn can_mut(&self) -> bool { + let stride_contains_zero = self.tensor.stride().contains(&0); + + !stride_contains_zero && self.storage.can_mut() + } + + /// Executes an operation on a tensor if the data can be reused. + pub fn mut_ops tch::Tensor>( + &mut self, + func: F, + ) -> Option { + if !self.can_mut() { + return None; + } + + let data = self.storage.clone(); + Some(TchTensor::from_existing(func(&mut self.tensor), data)) + } + + /// Executes a unary operation, reusing the tensor data if possible. + pub fn unary_ops(self, fown: FOwn, fref: FRef) -> TchTensor + where + FOwn: Fn(tch::Tensor) -> tch::Tensor, + FRef: Fn(&tch::Tensor) -> tch::Tensor, + { + if !self.can_mut() { + return TchTensor::from_existing(fref(&self.tensor), self.storage); + } + + TchTensor::from_existing(fown(self.tensor), self.storage) + } + + /// Executes a binary operation, reusing the tensor data if possible. + pub fn binary_ops_tensor( + mut lhs: Self, + mut rhs: Self, + flmut: FLMut, + frmut: FRMut, + fref: FRef, + ) -> TchTensor + where + FLMut: Fn(&mut tch::Tensor, &tch::Tensor) -> tch::Tensor, + FRMut: Fn(&tch::Tensor, &mut tch::Tensor) -> tch::Tensor, + FRef: Fn(&tch::Tensor, &tch::Tensor) -> tch::Tensor, + { + let lhs_shape = lhs.shape(); + let rhs_shape = rhs.shape(); + + // Both lhs and rhs are expected to have the same rank + let d_out = lhs_shape.num_dims(); + let mut out_shape = Shape::from(vec![1usize; d_out]); + + for i in 0..d_out { + out_shape[i] = usize::max(lhs_shape[i], rhs_shape[i]); + } + + let num_elements_out = out_shape.num_elements(); + + // Attempt to mutate lhs tensor + if lhs_shape.num_elements() == num_elements_out + && let Some(output) = lhs.mut_ops(|lhs| flmut(lhs, &rhs.tensor)) + { + return output; + } + + // Attempt to mutate rhs tensor + if rhs_shape.num_elements() == num_elements_out + && let Some(output) = rhs.mut_ops(|rhs| frmut(&lhs.tensor, rhs)) + { + return output; + } + + let storage = lhs.storage; + let tensor = fref(&lhs.tensor, &rhs.tensor); + + TchTensor::from_existing(tensor, storage) + } +} + +impl Clone for TchTensor { + fn clone(&self) -> Self { + Self { + tensor: self.tensor.shallow_clone(), + storage: self.storage.clone(), + } + } +} + +/// A shape that can be used by LibTorch. +#[derive(Debug)] +pub struct TchShape { + /// The shape's dimensions. + pub dims: Vec, +} + +impl From for TchShape { + fn from(shape: Shape) -> Self { + TchShape { + dims: shape.iter().map(|d| *d as i64).collect(), + } + } +} + +impl From<&[usize]> for TchShape { + fn from(shape: &[usize]) -> Self { + TchShape { + dims: shape.iter().map(|d| *d as i64).collect(), + } + } +} + +impl TchTensor { + /// Creates a new tensor from a shape and a device. + /// + /// # Arguments + /// + /// * `data` - The tensor's data. + /// * `device` - The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// A new tensor. + pub fn from_data(data: TensorData, device: tch::Device) -> Self { + let shape_tch = TchShape::from(data.shape.as_slice()); + let tensor = + tch::Tensor::from_data_size(&data.bytes, &shape_tch.dims, E::kind()).to(device); + + Self::new(tensor) + } +} + +impl TchTensor { + /// Creates an empty tensor from a shape and a device. + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// A new empty tensor. + pub fn empty(shape: Shape, device: LibTorchDevice) -> Self { + let shape_tch = TchShape::from(shape); + let tensor = tch::Tensor::empty(shape_tch.dims, (E::kind(), device.into())); + + Self::new(tensor) + } +} + +// Adapted from `tch` to use patched `T::kind()` instead of `T::KIND` which is incorrect for bf16. +// TODO: remove when fixed in `tch` release (https://github.com/LaurentMazare/tch-rs/pull/996). +impl TryFrom<&TchTensor> for Vec { + type Error = tch::TchError; + fn try_from(tensor: &TchTensor) -> Result { + let tensor = &tensor.tensor; + let size = tensor.size(); + if size.len() != 1 { + Err(tch::TchError::Convert(format!( + "Attempting to convert a Tensor with {} dimensions to flat vector", + size.len() + )))?; + } + let numel = size[0] as usize; + let mut vec = vec![T::ZERO; numel]; + // Adapted to use patched `T::kind()` instead + // TODO: tensor.f_to_kind(T::KIND)?.f_copy_data(&mut vec, numel)?; + f_copy_data(&mut tensor.f_to_kind(T::kind())?, &mut vec, numel)?; + Ok(vec) + } +} + +unsafe fn ptr_to_string(ptr: *mut libc::c_char) -> Option { + if !ptr.is_null() { + unsafe { + let str = std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned(); + libc::free(ptr as *mut libc::c_void); + Some(str) + } + } else { + None + } +} + +/// Copies `numel` elements from `self` to `dst`. +fn f_copy_data( + tensor: &mut tch::Tensor, + dst: &mut [T], + numel: usize, +) -> Result<(), tch::TchError> { + if T::kind() != tensor.f_kind()? { + return Err(tch::TchError::Kind(format!( + "incoherent elt kind, {:?} != {:?}", + tensor.f_kind(), + T::kind() + ))); + } + if dst.len() < numel { + return Err(tch::TchError::Shape(format!("slice len < {numel}"))); + } + + unsafe { + torch_sys::at_copy_data( + tensor.as_mut_ptr(), + dst.as_mut_ptr() as *const c_void, + numel, + T::kind().elt_size_in_bytes(), + ); + match ptr_to_string(torch_sys::get_and_reset_last_err()) { + None => Ok(()), + Some(c_error) => Err(tch::TchError::Torch(c_error)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn_backend::ops::FloatTensorOps; + use burn_backend::{Backend, quantization::QuantScheme, read_sync}; + + type B = crate::LibTorch; + + #[test] + fn should_have_bf16_kind() { + let data = TensorData::from([4.0, 4.0]); + let tensor_1: TchTensor = B::float_from_data(data, &Default::default()); + let tensor_2 = B::float_cast(tensor_1, DType::BF16.into()); + + assert_eq!(tensor_2.tensor.kind(), tch::Kind::BFloat16); + + let out = read_sync(B::float_into_data(tensor_2)).unwrap(); + + out.assert_eq(&TensorData::from([4.0, 4.0]), false); + } + + #[test] + fn should_support_dtypes() { + let device = Default::default(); + + assert!(B::supports_dtype(&device, DType::F64)); + assert!(B::supports_dtype(&device, DType::F32)); + assert!(B::supports_dtype(&device, DType::Flex32)); + assert!(B::supports_dtype(&device, DType::F16)); + assert!(B::supports_dtype(&device, DType::BF16)); + assert!(B::supports_dtype(&device, DType::I64)); + assert!(B::supports_dtype(&device, DType::I32)); + assert!(B::supports_dtype(&device, DType::I16)); + assert!(B::supports_dtype(&device, DType::I8)); + assert!(B::supports_dtype(&device, DType::U8)); + assert!(B::supports_dtype(&device, DType::Bool)); + + assert!(!B::supports_dtype(&device, DType::U64)); + assert!(!B::supports_dtype(&device, DType::U32)); + assert!(!B::supports_dtype(&device, DType::U16)); + assert!(!B::supports_dtype( + &device, + DType::QFloat(QuantScheme::default()) + )); + } + + #[test] + fn should_support_from_bf16() { + let data = TensorData::from([[1.0], [1.]]).convert_dtype(DType::BF16); + let tensor_1: TchTensor = B::float_from_data(data, &Default::default()); + let data = TensorData::from([[2.0], [2.]]).convert_dtype(DType::BF16); + let tensor_2 = B::float_from_data(data, &Default::default()); + + let tensor_3 = B::float_add(tensor_1, tensor_2); + + assert_eq!(tensor_3.tensor.kind(), tch::Kind::BFloat16); + + let out = read_sync(B::float_into_data(tensor_3)).unwrap(); + + out.assert_eq(&TensorData::from([[3.0], [3.0]]), false); + } +} + +unsafe extern "C" { + /// Dummy function to get CUDA to link properly + pub fn dummy_cuda_dependency(); +} + +#[used] +static INIT_ARRAY: [unsafe extern "C" fn(); 1] = [dummy_cuda_dependency]; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/Cargo.toml new file mode 100644 index 0000000..f784ecf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/Cargo.toml @@ -0,0 +1,20 @@ +[package] +authors = ["nathanielsimard "] +description = "Test generation crate for burn-tensor" +edition.workspace = true +license.workspace = true +name = "burn-tensor-testgen" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-tensor-testgen" +version.workspace = true + +[lints] +workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/README.md b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/README.md new file mode 100644 index 0000000..662fca6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/README.md @@ -0,0 +1,6 @@ +# Burn Tensor Test Generation + +> [Burn](https://github.com/tracel-ai/burn) tensor test generation + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-tensor-testgen.svg)](https://crates.io/crates/burn-tensor-testgen) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-tensor-testgen/blob/master/README.md) diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/src/lib.rs new file mode 100644 index 0000000..95e433c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor-testgen/src/lib.rs @@ -0,0 +1,130 @@ +use proc_macro::TokenStream; +use quote::{format_ident, quote}; + +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::token::Comma; +use syn::{Attribute, Expr, ItemFn, Lit, Meta, MetaNameValue, parse_macro_input}; + +// Define a structure to parse the attribute arguments +struct AttributeArgs { + args: Punctuated, +} + +impl Parse for AttributeArgs { + fn parse(input: ParseStream) -> syn::Result { + Ok(AttributeArgs { + args: Punctuated::parse_terminated(input)?, + }) + } +} + +#[allow(clippy::test_attr_in_doctest)] +/// **This is only meaningful when the `reason` is specific and clear.** +/// +/// A proc macro attribute that adds panic handling to test functions. +/// +/// # Usage +/// ```rust, ignore +/// #[might_panic(reason = "expected panic message prefix")] +/// #[test] +/// fn test_that_might_panic() { +/// // test code that might panic (with acceptable reason) +/// } +/// ``` +/// +/// # Behavior +/// - If the test does not panic, it passes. +/// - If the test panics with a message starting with the expected prefix, the failure is ignored. +/// - If the test panics with a different message, the test fails. +/// +/// # Note +/// This proc macro uses [`std::panic::catch_unwind`]. As such, it does not work in a no-std environment. +/// Make sure it is feature gated when an `"std"` feature is available. +#[proc_macro_attribute] +pub fn might_panic(args: TokenStream, input: TokenStream) -> TokenStream { + // Parse the attribute arguments + let args = parse_macro_input!(args as AttributeArgs); + let input_fn = parse_macro_input!(input as ItemFn); + + // Extract the expected panic reason + let mut expected_reason = None; + for arg in args.args.iter() { + if let Meta::NameValue(MetaNameValue { path, value, .. }) = arg + && path.is_ident("reason") + && let Expr::Lit(lit) = value + && let Lit::Str(ref lit_str) = lit.lit + { + expected_reason = Some(lit_str.value()); + } + } + + let expected_reason = match expected_reason { + Some(reason) => reason, + None => { + return syn::Error::new( + proc_macro2::Span::call_site(), + "The #[might_panic] attribute requires a 'reason' parameter", + ) + .to_compile_error() + .into(); + } + }; + + let fn_name = &input_fn.sig.ident; + let fn_vis = &input_fn.vis; + let fn_generics = &input_fn.sig.generics; + let fn_block = &input_fn.block; + let fn_attrs = input_fn + .attrs + .iter() + .filter(|attr| !attr.path().is_ident("test")) + .collect::>(); + + // Create a wrapped test function + let wrapper_name = format_ident!("{}_might_panic", fn_name); + + let expanded = quote! { + #(#fn_attrs)* + #fn_vis fn #fn_name #fn_generics() { + #fn_block + } + + #[test] + #fn_vis fn #wrapper_name #fn_generics() { + use std::panic::{self, AssertUnwindSafe}; + + let expected_reason = #expected_reason; + let result = panic::catch_unwind(AssertUnwindSafe(|| { + #fn_name(); + })); + + match result { + Ok(_) => { + // Test passed without panic - this is OK + } + Err(e) => { + // Convert the panic payload to a string + let panic_msg = if let Some(s) = e.downcast_ref::() { + s.to_string() + } else if let Some(s) = e.downcast_ref::<&str>() { + s.to_string() + } else { + "Unknown panic".to_string() + }; + + // Check if the panic message starts with the expected reason + if !panic_msg.starts_with(expected_reason) { + panic!( + "Test '{}' marked as 'might_panic' failed. Expected reason: '{}'", + stringify!(#fn_name), + expected_reason + ); + } + } + } + } + }; + + expanded.into() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-tensor/Cargo.toml new file mode 100644 index 0000000..3eb7f72 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/Cargo.toml @@ -0,0 +1,62 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science", "no-std", "embedded", "wasm"] +description = "Tensor library with user-friendly APIs and automatic differentiation support" +documentation = "https://docs.rs/burn-tensor" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-tensor" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-tensor" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["std"] +doc = ["default"] +std = [ + "num-traits/std", + "burn-std/std", + "burn-backend/std", + "colored", +] +tracing = [ + "burn-std/tracing", + "burn-backend/tracing", +] + +cubecl = ["burn-std/cubecl", "burn-backend/cubecl"] +cubecl-cuda = ["burn-backend/cubecl-cuda"] +cubecl-hip = ["burn-backend/cubecl-hip"] +cubecl-wgpu = ["burn-backend/cubecl-wgpu"] +cubecl-cpu = ["burn-backend/cubecl-cpu"] +experimental-named-tensor = [] + +[dependencies] +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", default-features = false } +burn-backend = { path = "../burn-backend", version = "=0.21.0-pre.2", default-features = false } + +colored = { workspace = true, optional = true } +derive-new = { workspace = true } +num-traits = { workspace = true } + +# Device +hashbrown = { workspace = true } +spin = { workspace = true } +thiserror = { workspace = true } + +# Serialization +serde = { workspace = true } + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +portable-atomic-util = { workspace = true } + +[dev-dependencies] +serial_test = { workspace = true } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs", "--html-in-header", "katex-header.html"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/README.md b/crates/stable-diffusion-burn/burn-crates/burn-tensor/README.md new file mode 100644 index 0000000..b97bffd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/README.md @@ -0,0 +1,12 @@ +# Burn Tensor + +> [Burn](https://github.com/tracel-ai/burn) Tensor Library + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-tensor.svg)](https://crates.io/crates/burn-tensor) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-tensor/blob/master/README.md) + +This library provides the core abstractions required to run tensor operations with Burn. + +`Tensor`s are generic over the backend to allow users to perform operations using different +`Backend` implementations. Burn's tensors also support auto-differentiation thanks to the +`AutodiffBackend` trait. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/katex-header.html b/crates/stable-diffusion-burn/burn-crates/burn-tensor/katex-header.html new file mode 120000 index 0000000..4a9eb97 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/katex-header.html @@ -0,0 +1 @@ +../../docs/katex-header.html \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/device.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/device.rs new file mode 100644 index 0000000..abcb80d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/device.rs @@ -0,0 +1,465 @@ +use alloc::format; +use alloc::string::String; +use burn_backend::{Backend, Device, DeviceId, DeviceOps}; +use burn_std::stub::RwLock; +use burn_std::{DType, FloatDType, IntDType}; + +#[cfg(target_has_atomic = "ptr")] +use alloc::sync::Arc; + +#[cfg(not(target_has_atomic = "ptr"))] +use portable_atomic_util::Arc; +use thiserror::Error; + +use core::any::TypeId; + +#[cfg(feature = "std")] +pub use std::collections::HashMap; +#[cfg(feature = "std")] +use std::sync::LazyLock; + +#[cfg(not(feature = "std"))] +pub use hashbrown::HashMap; +#[cfg(not(feature = "std"))] +use spin::Lazy as LazyLock; + +/// Policy controlling default device behavior. +/// +/// This includes default data types used for tensor creation. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct DevicePolicy { + /// Default floating-point data type for tensor creation. + float_dtype: Option, + /// Default integer data type for tensor creation. + int_dtype: Option, +} + +impl DevicePolicy { + /// Returns the default floating-point data type used for tensor creation. + pub(crate) fn float_dtype(&self) -> Option { + self.float_dtype + } + + /// Returns the default integer data type used for tensor creation. + pub(crate) fn int_dtype(&self) -> Option { + self.int_dtype + } + + /// Sets the default floating-point data type. + pub(crate) fn set_float_dtype(&mut self, dtype: FloatDType) { + self.float_dtype = Some(dtype); + } + + /// Sets the default integer data type. + pub(crate) fn set_int_dtype(&mut self, dtype: IntDType) { + self.int_dtype = Some(dtype); + } +} + +/// Key for the registry: physical device type + device id +type RegistryKey = (DeviceId, TypeId); + +/// Global registry mapping devices to their policies. +static REGISTRY: LazyLock>>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +/// Device policy management for controlling default tensor creation behavior. +/// +/// # Policy Semantics +/// +/// Device policies use snapshot semantics: when you retrieve a policy with +/// [`get_device_policy`], you get an immutable snapshot of the current configuration. +/// Updates to the policy (via [`set_default_dtypes`], [`set_default_float_dtype`], etc.) +/// only affect future policy retrievals, not existing references. +/// +/// This is intended for the common case where policies are set once during +/// initialization and then read frequently during tensor creation. +struct DevicePolicyRegistry; + +impl DevicePolicyRegistry { + /// Get the policy for a physical device type and device id. + /// + /// If no policy exists yet, a default one is created and stored. + fn get(device: &D) -> Arc { + let key = Self::key(device); + + if let Some(policy) = REGISTRY.read().unwrap().get(&key) { + return Arc::clone(policy); + } + + let mut map = REGISTRY.write().unwrap(); + Arc::clone( + map.entry(key) + .or_insert_with(|| Arc::new(DevicePolicy::default())), + ) + } + + /// Mutate the policy for a given device. + fn update(device: &D, update_fn: impl FnOnce(&mut DevicePolicy)) { + let key = Self::key(device); + let mut map = REGISTRY.write().unwrap(); + + let policy = map + .entry(key) + .or_insert_with(|| Arc::new(DevicePolicy::default())); + + // Update the policy + let policy_mut = Arc::make_mut(policy); + update_fn(policy_mut); + } + + /// Returns the device registry key. + fn key(device: &D) -> RegistryKey { + (device.to_id(), TypeId::of::()) + } +} + +/// Get the [`device`'s policy](DevicePolicy). +/// +/// Returns an immutable snapshot of the device's current policy. If the policy +/// is updated after retrieval, this snapshot will not reflect those changes. +pub(crate) fn get_device_policy(device: &D) -> Arc { + DevicePolicyRegistry::get(device) +} + +/// Errors that can occur during device-related operations. +/// +/// This covers errors related to hardware capability mismatches, such as +/// requesting a data type not supported by the device, and configuration +/// errors like attempting to change a policy in an invalid context. +#[derive(Debug, Error)] +pub enum DeviceError { + /// Unsupported data type by the device. + #[error("Device {device} does not support the requested data type {dtype:?}")] + UnsupportedDType { + /// The string representation of the device. + device: String, + /// The data type that caused the error. + dtype: DType, + }, + // TODO: `InvalidContext` if a device policy cannot be changed after init / during training / etc. +} + +impl DeviceError { + /// Helper to create a [`DeviceError::UnsupportedDType`] from any device. + pub fn unsupported_dtype(device: &D, dtype: DType) -> Self { + Self::UnsupportedDType { + device: format!("{device:?}"), + dtype, + } + } +} + +fn check_dtype_support( + device: &B::Device, + dtype: impl Into, +) -> Result<(), DeviceError> { + let dtype = dtype.into(); + // Default dtypes should have `DTypeUsage::general()`. Types restricted to specialized + // operations should not be used as default. + if B::supports_dtype(device, dtype) { + Ok(()) + } else { + Err(DeviceError::unsupported_dtype(device, dtype)) + } +} + +/// Sets the default data types for the device. +/// +/// This updates the device's default data types used for tensor creation. +/// The policy should typically be set once during initialization and then +/// remains global for all subsequent operations on that device. +/// +/// # Example +/// +/// ```rust +/// use burn_tensor::backend::Backend; +/// use burn_tensor::{DType, Int, Tensor, set_default_dtypes}; +/// +/// fn example() { +/// let device = B::Device::default(); +/// +/// // Update the device policy +/// set_default_dtypes::(&device, DType::F16, DType::I32); +/// +/// // All float tensors created after this will use F16 by default +/// let tensor = Tensor::::zeros([2, 3], &device); +/// // All int tensors created after this will use I32 default +/// let tensor = Tensor::::zeros([2, 3], &device); +/// } +/// ``` +pub fn set_default_dtypes( + device: &B::Device, + float_dtype: impl Into, + int_dtype: impl Into, +) -> Result<(), DeviceError> { + let float_dtype = float_dtype.into(); + let int_dtype = int_dtype.into(); + check_dtype_support::(device, float_dtype)?; + check_dtype_support::(device, int_dtype)?; + + set_default_dtypes_unchecked(device, float_dtype, int_dtype); + Ok(()) +} + +/// Sets the default floating-point data type for the device. +/// +/// This updates the device's default data types used for tensor creation. +/// The policy should typically be set once during initialization and then +/// remains global for all subsequent operations on that device. +/// +/// # Example +/// +/// ```rust +/// use burn_tensor::backend::Backend; +/// use burn_tensor::{DType, Tensor, set_default_float_dtype}; +/// +/// fn example() { +/// let device = B::Device::default(); +/// +/// // Update the device policy +/// set_default_float_dtype::(&device, DType::F16); +/// +/// // All float tensors created after this will use F16 by default +/// let tensor = Tensor::::zeros([2, 3], &device); +/// } +/// ``` +pub fn set_default_float_dtype( + device: &B::Device, + dtype: impl Into, +) -> Result<(), DeviceError> { + let dtype = dtype.into(); + check_dtype_support::(device, dtype)?; + + set_default_float_dtype_unchecked(device, dtype); + Ok(()) +} + +/// Sets the default integer data type for the device. +/// +/// This updates the device's default data types used for tensor creation. +/// The policy should typically be set once during initialization and then +/// remains global for all subsequent operations on that device. +/// +/// # Example +/// +/// ```rust +/// use burn_tensor::backend::Backend; +/// use burn_tensor::{DType, Int, Tensor, set_default_int_dtype}; +/// +/// fn example() { +/// let device = B::Device::default(); +/// +/// // Update the device policy +/// set_default_int_dtype::(&device, DType::I32); +/// +/// // All int tensors created after this will use I32 default +/// let tensor = Tensor::::zeros([2, 3], &device); +/// } +/// ``` +pub fn set_default_int_dtype( + device: &B::Device, + dtype: impl Into, +) -> Result<(), DeviceError> { + let dtype = dtype.into(); + check_dtype_support::(device, dtype)?; + + set_default_int_dtype_unchecked(device, dtype); + Ok(()) +} + +// Unchecked versions +fn set_default_dtypes_unchecked( + device: &D, + float_dtype: FloatDType, + int_dtype: IntDType, +) { + DevicePolicyRegistry::update(device, |p| { + p.set_float_dtype(float_dtype); + p.set_int_dtype(int_dtype); + }); +} + +fn set_default_float_dtype_unchecked(device: &D, dtype: FloatDType) { + DevicePolicyRegistry::update(device, |p| { + p.set_float_dtype(dtype); + }); +} + +fn set_default_int_dtype_unchecked(device: &D, dtype: IntDType) { + DevicePolicyRegistry::update(device, |p| { + p.set_int_dtype(dtype); + }); +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use serial_test::serial; + + use super::*; + + fn clear_registry() { + REGISTRY.write().unwrap().clear(); + } + + #[derive(Clone, Debug, Default, PartialEq, new)] + pub struct TestDeviceA { + index: u32, + } + + impl Device for TestDeviceA { + fn from_id(device_id: DeviceId) -> Self { + Self { + index: device_id.index_id, + } + } + + fn to_id(&self) -> DeviceId { + DeviceId { + type_id: 0, + index_id: self.index, + } + } + + fn device_count(_type_id: u16) -> usize { + 1 + } + } + + impl DeviceOps for TestDeviceA {} + + #[derive(Clone, Debug, Default, PartialEq, new)] + pub struct TestDeviceB { + index: u32, + } + + impl Device for TestDeviceB { + fn from_id(device_id: DeviceId) -> Self { + Self { + index: device_id.index_id, + } + } + + fn to_id(&self) -> DeviceId { + DeviceId { + type_id: 0, + index_id: self.index, + } + } + + fn device_count(_type_id: u16) -> usize { + 1 + } + } + + impl DeviceOps for TestDeviceB {} + + #[test] + #[serial] + fn default_policy_is_created_and_shared() { + clear_registry(); // reset registry for each test + + let device = TestDeviceA::new(0); + + let p1 = get_device_policy(&device); + let p2 = get_device_policy(&device); + + assert!(Arc::ptr_eq(&p1, &p2)); + // Not explicitly set + assert!(p1.float_dtype().is_none()); + assert!(p1.int_dtype().is_none()); + assert!(p2.float_dtype().is_none()); + assert!(p2.int_dtype().is_none()); + } + + #[test] + #[serial] + fn updated_policy_is_shared() { + clear_registry(); // reset registry for each test + + let device = TestDeviceA::new(0); + + // The device policy is meant to be set once at initialization + set_default_dtypes_unchecked(&device, FloatDType::BF16, IntDType::I32); + let p1 = get_device_policy(&device); + let p2 = get_device_policy(&device); + + assert!(Arc::ptr_eq(&p1, &p2)); + assert_eq!(p1.float_dtype(), Some(FloatDType::BF16)); + assert_eq!(p1.int_dtype(), Some(IntDType::I32)); + assert_eq!(p2.float_dtype(), Some(FloatDType::BF16)); + assert_eq!(p2.int_dtype(), Some(IntDType::I32)); + } + + #[test] + #[serial] + fn policy_is_device_id_specific() { + clear_registry(); // reset registry for each test + + let d1 = TestDeviceA::new(0); + let d2 = TestDeviceA::new(1); + + set_default_float_dtype_unchecked(&d1, FloatDType::F16); + + let p1 = get_device_policy(&d1); + let p2 = get_device_policy(&d2); + + assert!(!Arc::ptr_eq(&p1, &p2)); + assert_eq!(p1.float_dtype(), Some(FloatDType::F16)); + assert!(p1.int_dtype().is_none()); + assert!(p2.float_dtype().is_none()); + assert!(p2.int_dtype().is_none()); + } + + #[test] + #[serial] + fn policy_is_device_type_specific() { + clear_registry(); // reset registry for each test + + let d1 = TestDeviceA::new(0); + let d2 = TestDeviceB::new(0); + + set_default_float_dtype_unchecked(&d2, FloatDType::F16); + + let p1 = get_device_policy(&d1); + let p2 = get_device_policy(&d2); + + assert!(p1.float_dtype().is_none()); + assert!(p1.int_dtype().is_none()); + assert_eq!(p2.float_dtype(), Some(FloatDType::F16)); + assert!(p2.int_dtype().is_none()); + } + + #[test] + #[serial] + fn updating_policy_should_not_affect_snapshot() { + clear_registry(); // reset registry for each test + + // The device policy is meant to be set once at initialization + let device = TestDeviceA::new(0); + let before = get_device_policy(&device); + + set_default_float_dtype_unchecked(&device, FloatDType::BF16); + + let after = get_device_policy(&device); + + assert!(!Arc::ptr_eq(&before, &after)); + assert_eq!(after.float_dtype(), Some(FloatDType::BF16)); + assert!(before.float_dtype().is_none()); + } + + #[test] + #[serial] + fn set_default_dtypes_overwrites_fields() { + clear_registry(); // reset registry for each test + + let device = TestDeviceA::new(0); + + set_default_dtypes_unchecked(&device, FloatDType::F16, IntDType::I64); + + let policy = get_device_policy(&device); + + assert_eq!(policy.float_dtype(), Some(FloatDType::F16)); + assert_eq!(policy.int_dtype(), Some(IntDType::I64)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/lib.rs new file mode 100644 index 0000000..61b77f5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/lib.rs @@ -0,0 +1,23 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! This library provides the core abstractions required to run tensor operations with Burn. +//! `Tensor`s are generic over the backend to allow users to perform operations using different `Backend` implementations. +//! Burn's tensors also support auto-differentiation thanks to the `AutodiffBackend` trait. + +#[macro_use] +extern crate derive_new; + +extern crate alloc; + +mod tensor; + +pub(crate) use tensor::check::macros::check; +pub use tensor::*; + +// Re-exported types +pub use burn_backend::{AllocationProperty, Bytes, StreamId, bf16, f16, read_sync, try_read_sync}; + +mod device; +pub use device::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/base.rs new file mode 100644 index 0000000..71b4381 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/base.rs @@ -0,0 +1,647 @@ +use crate::backend::Backend; +use crate::check::TensorCheck; +use crate::{Tensor, TensorPrimitive, check, s}; + +/// Applies the rectified linear unit function element-wise +/// as described in the paper [Deep Learning using Rectified Linear Units (ReLU)](https://arxiv.org/pdf/1803.08375). +/// +#[cfg_attr(doc, doc = "$$\\text{ReLU}\\(x\\) = \\(x\\)^+ = \\max\\(0, x\\)$$")] +#[cfg_attr(not(doc), doc = "`ReLU(x) = max(0, x)`")] +pub fn relu(tensor: Tensor) -> Tensor { + tensor.relu() +} + +/// Applies the leaky rectified linear unit function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{LeakyReLU}\(x\) = \max\(0,x\) + \text{negative\\_slope} \cdot \min\(0, x\) +$$ + +or + +$$ +\text{LeakyReLU}(x) = + \begin{cases} + x & \text{if } x \geq 0 \newline + \text{negative\\_slope} \cdot x & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`f(x) =`\n- `x for x >= 0`\n- `negative_slope * x if x < 0`" +)] +pub fn leaky_relu( + tensor: Tensor, + negative_slope: f64, +) -> Tensor { + Tensor::from_primitive(TensorPrimitive::Float(B::leaky_relu( + tensor.primitive.tensor(), + negative_slope.into(), + ))) +} + +/// Applies the Gaussian Error Linear Units function as described in the paper +/// [Gaussian Error Linear Units (GELUs)](https://arxiv.org/pdf/1606.08415v3.pdf). +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{GELU}(x) += x \cdot \Phi(x) += x \cdot \frac{1}{2}\left(1 + \text{erf}\left(\frac{x}{\sqrt{2}}\right)\right) +$$ + +where $\Phi(x)$ is the cumulative distribution function for the Gaussian distribution. +"# +)] +#[cfg_attr( + not(doc), + doc = r#" +`GELU(x) = x * Φ(x) = x * 1/2 * (1 + erf(x / sqrt(2)))` + +where `Φ(x)` is the cumulative distribution function for the Gaussian distribution. +"# +)] +pub fn gelu(tensor: Tensor) -> Tensor { + Tensor::from_primitive(TensorPrimitive::Float(B::gelu(tensor.primitive.tensor()))) +} + +/// Applies the tanh-based approximate GELU function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{GELU\_approx}(x) += \frac{x}{2}\left(1 + \tanh\left(\sqrt{\frac{2}{\pi}}\left(x + 0.044715\,x^3\right)\right)\right) +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`GELU_approx(x) = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))`" +)] +pub fn gelu_approximate(tensor: Tensor) -> Tensor { + /// sqrt(2/π) precomputed as FRAC_2_SQRT_PI * FRAC_1_SQRT_2 + const SQRT_2_OVER_PI: f64 = + core::f64::consts::FRAC_2_SQRT_PI * core::f64::consts::FRAC_1_SQRT_2; + + let x = tensor; + let inner = x.clone() + x.clone().powf_scalar(3.0) * 0.044715; + let inner = inner * SQRT_2_OVER_PI; + (x.clone() * (inner.tanh() + 1)) * 0.5 +} + +/// Applies Parametric ReLu activation function as described in the paper +/// [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](https://arxiv.org/pdf/1502.01852). +/// +/// - The tensor is assumed to be of shape `[batch_size, channels, ...]`. +/// - `alpha` is assumed to be of shape `[channels]` or `[1]`. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{PReLU}\(x\) = \max\(0,x\) + \alpha \cdot \min\(0, x\) +$$ + +or + +$$ +\text{PReLU}(x) = + \begin{cases} + x & \text{if } x \geq 0 \newline + \alpha x & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`PReLu(x) = max(0,x) + alpha * min(0,x)`")] +pub fn prelu( + tensor: Tensor, + alpha: Tensor, +) -> Tensor { + check!(TensorCheck::check_prelu_shape::( + &tensor.shape(), + &alpha.shape() + )); + + let weight = if alpha.dims()[0] == 1 { + // if there is only 1 weight, then reshape it to (1,1,1... D times) so that the rank is D + alpha.reshape([1; D]) + } else { + // D>=2 because the case where D==1 and num_weights >1 is handled by check function + // there is more than 1 weight and rank is more than 2 + let num_weights = alpha.dims()[0]; + let mut s = [1; D]; + s[1] = num_weights; + // reshape the weights to (1, channels,1 ...) + alpha.reshape(s) + }; + + Tensor::from_primitive(TensorPrimitive::Float(B::prelu( + tensor.primitive.tensor(), + weight.primitive.tensor(), + ))) +} + +/// Applies the softmax function on the input tensor along the given dimension. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{softmax}\(x_i\) = \frac{\exp\(x_i\)}{\sum_j \exp\(x_j\)} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`softmax(x_i) = exp(x_i) / sum_j(exp(x_j))`")] +/// +/// # Arguments +/// - `dim`: the dimension along which Softmax will be computed. +/// +/// # Panics +/// - If `dim` is outside [0, D) +pub fn softmax(tensor: Tensor, dim: usize) -> Tensor { + check!(TensorCheck::dim_ops::("softmax", dim)); + + let tensor = tensor.clone() - tensor.detach().max_dim(dim); + let tensor = tensor.exp(); + let tensor_tmp = tensor.clone().sum_dim(dim); + + tensor.div(tensor_tmp) +} + +/// Applies the softmin function on the input tensor along the given dimension. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{softmin}\(x_i\) = \frac{\exp\(-x_i\)}{\sum_j \exp\(-x_j\)} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`softmin(x_i) = exp(-x_i) / sum_j(exp(-x_j)`")] +/// +/// # Arguments +/// - `dim`: the dimension along which Softmax will be computed. +/// +/// # Panics +/// - If `dim` is outside [0, D) +pub fn softmin(tensor: Tensor, dim: usize) -> Tensor { + check!(TensorCheck::dim_ops::("softmin", dim)); + softmax(tensor.neg(), dim) +} + +/// Applies the SoftPlus function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{softplus}\(x\) = \frac{1}{\beta}\log\(1 + \exp\(\beta x\)\) +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`softplus(x_i) = log(1 + exp(beta * x_i)) / beta`")] +/// +/// The SoftPlus function is a smooth approximation of the ReLU function. +pub fn softplus(tensor: Tensor, beta: f64) -> Tensor { + let tensor = (tensor.mul_scalar(beta).exp() + 1).log(); + tensor.div_scalar(beta) +} + +/// Applies the "quiet softmax" function on the input tensor along the given dimension. +/// +/// Also referred to as [`softmax1`](https://www.evanmiller.org/attention-is-off-by-one.html). +/// +/// This function is similar to the softmax function, but it allows for "no selection" when +/// all the outputs are close to zero. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{quiet\\_softmax}\(x_i\) = \frac{\exp\(x_i\)}{1 + \sum_j \exp\(x_j\)} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`quiet_softmax(x_i) = exp(x_i) / [ 1 + sum_j(exp(x_j)) ]`" +)] +/// +/// # Arguments +/// - `dim`: the dimension along which Softmax will be computed. +/// +/// # Panics +/// - If `dim` is outside [0, D) +pub fn quiet_softmax(tensor: Tensor, dim: usize) -> Tensor { + check!(TensorCheck::dim_ops::("softmax", dim)); + + let max_vals = tensor.clone().detach().max_dim(dim); + let exp_x = (tensor - max_vals.clone()).exp(); + let sum_exp = exp_x.clone().sum_dim(dim); + + exp_x.div(sum_exp + max_vals.neg().exp()) +} + +/// Applies the log softmax function on the input tensor along the given dimension. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{log\\_softmax}\(x_i\) += \log\left(\text{softmax}\(x_i\)\right) += \log\left(\frac{\exp\(x_i\)}{\sum_j \exp\(x_j\)}\right) +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`log_softmax(x_i) = log(softmax(x_i)) = log(exp(x_i) / sum_j(exp(x_j)))`" +)] +/// +/// # Arguments +/// - `dim`: the dimension along which Softmax will be computed. +/// +/// # Panics +/// - If `dim` is outside [0, D) +pub fn log_softmax(tensor: Tensor, dim: usize) -> Tensor { + check!(TensorCheck::dim_ops::("log softmax", dim)); + + let tensor = tensor.clone() - tensor.detach().max_dim(dim); + let tensor_tmp = tensor.clone().exp().sum_dim(dim).log(); + + tensor.sub(tensor_tmp) +} + +/// Applies the sigmoid function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{sigmoid}\(x\) += \sigma(x) += \frac{1}{1 + \exp(-x)} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`sigmoid(x) = 1 / (1 + exp(-x))`")] +pub fn sigmoid(tensor: Tensor) -> Tensor { + Tensor::from_primitive(TensorPrimitive::Float(B::sigmoid( + tensor.primitive.tensor(), + ))) +} + +/// Applies the hard sigmoid function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{hard\\_sigmoid}\(x\) = \max(0, \min(1, \alpha \cdot x + \beta)) +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`hard_sigmoid(x) = max(0, min(1, alpha * x + beta))`")] +pub fn hard_sigmoid( + tensor: Tensor, + alpha: f64, + beta: f64, +) -> Tensor { + Tensor::from_primitive(TensorPrimitive::Float(B::hard_sigmoid( + tensor.primitive.tensor(), + alpha.into(), + beta.into(), + ))) +} + +/// Applies the log sigmoid function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{log\\_sigmoid}\(x\) = \log\left(\frac{1}{1 + \exp(-x)}\right) +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`log_sigmoid(x) = log(1 / (1 + exp(-x)))`")] +pub fn log_sigmoid(tensor: Tensor) -> Tensor { + Tensor::from_primitive(TensorPrimitive::Float(B::log_sigmoid( + tensor.primitive.tensor(), + ))) +} + +/// Applies the SiLU function (also known as the swish function) element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{SiLU}\(x\) = x \cdot \sigma(x) = \frac{x}{1 + \exp(-x)} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`SiLU(x) = x * sigmoid(x) = x / (1 + exp(-x))`")] +pub fn silu(tensor: Tensor) -> Tensor { + tensor.clone().mul(sigmoid(tensor)) +} + +/// Applies the hard swish function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{hard\_swish}\(x\) = x \cdot \text{hard\_sigmoid}(x) = x \cdot \max(0, \min(1, \frac{x}{6} + 0.5)) +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`hard_swish(x) = x * hard_sigmoid(x) = x * max(0, min(1, x/6 + 0.5))`" +)] +pub fn hard_swish(tensor: Tensor) -> Tensor { + tensor.clone().mul(hard_sigmoid(tensor, 1.0 / 6.0, 0.5)) +} + +/// Applies the Mish function as described in the paper in +/// [Mish: A Self Regularized Non-Monotonic Neural Activation Function](https://arxiv.org/abs/1908.08681). +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{Mish}\(x\) += x \cdot \tanh(\text{Softplus}(x)) += \tanh\left(\log\(1 + \exp\(x\)\)\right) +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`mish(x) = x * tanh(softplus(x)) = tanh(log(1 + exp(x)))`" +)] +pub fn mish(tensor: Tensor) -> Tensor { + tensor.clone().mul(softplus(tensor, 1.0).tanh()) +} + +/// Applies the tanh function element-wise. +pub fn tanh(tensor: Tensor) -> Tensor { + tensor.tanh() +} + +/// Applies the Exponential Linear Unit function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{ELU}\(x\) = + \begin{cases} + x & \text{if } x > 0 \newline + \alpha \cdot (\exp(x) - 1) & \text{if } x \leq 0 + \end{cases} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`f(x) =`\n- `x for x > 0`\n- `alpha * (exp(x) - 1) for x <= 0`" +)] +pub fn elu(tensor: Tensor, alpha: f64) -> Tensor { + let mask = tensor.clone().lower_equal_elem(0); + let scaled = tensor.clone().exp().sub_scalar(1).mul_scalar(alpha); + tensor.mask_where(mask, scaled) +} + +/// Applies the Continuously Differentiable Exponential Linear Unit function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{CELU}(x) = + \begin{cases} + x & \text{if } x \geq 0 \newline + \alpha \cdot \left(\exp\left(\frac{x}{\alpha}\right) - 1\right) & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`celu(x) = max(0, x) + min(0, alpha * (exp(x / alpha) - 1))`" +)] +/// +/// See also [CELU](https://pytorch.org/docs/stable/generated/torch.nn.CELU.html) +/// +/// # Arguments +/// - `alpha`: scaling parameter for the negative part. +pub fn celu(tensor: Tensor, alpha: f64) -> Tensor { + let mask = tensor.clone().lower_equal_elem(0); + let scaled = tensor + .clone() + .div_scalar(alpha) + .exp() + .sub_scalar(1) + .mul_scalar(alpha); + tensor.mask_where(mask, scaled) +} + +/// Applies the Scaled Exponential Linear Unit function element-wise +/// as described in the paper [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515). +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{SELU}\(x\) = \gamma \cdot + \begin{cases} + x & \text{if } x > 0 \newline + \alpha \cdot (\exp(x) - 1) & \text{if } x \leq 0 + \end{cases} +$$ + +where $\alpha \approx 1.6733$ and $\gamma \approx 1.0507$. +"# +)] +#[cfg_attr( + not(doc), + doc = "`selu(x) = gamma * x if x > 0, gamma * alpha * (exp(x) - 1) if x <= 0`" +)] +pub fn selu(tensor: Tensor) -> Tensor { + // Constants from the SELU paper / ONNX spec + const ALPHA: f64 = 1.6732632423543772848170429916717_f64; + const GAMMA: f64 = 1.0507009873554804934193349852946_f64; + + let mask = tensor.clone().greater_equal_elem(0.0); + let positive = tensor.clone().mul_scalar(GAMMA); + let negative = tensor.exp().sub_scalar(1.0).mul_scalar(ALPHA * GAMMA); + + negative.mask_where(mask, positive) +} + +/// Applies the thresholded rectified linear unit function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{ThresholdedReLU}(x) = + \begin{cases} + x & \text{if } x > \alpha \newline + 0 & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`f(x) =`\n- `x if x > alpha`\n- `0 otherwise`")] +/// +/// # Arguments +/// - `alpha`: threshold value (default in ONNX is 1.0). +pub fn thresholded_relu( + tensor: Tensor, + alpha: f64, +) -> Tensor { + let mask = tensor.clone().lower_equal_elem(alpha); + tensor.mask_fill(mask, 0) +} + +/// Applies the gated linear unit function. +/// +/// GLU(a,b)=a⊗σ(b) where `a` is the first half of the input matrices and `b` is the second half. +/// +/// **Note**: +/// * The size of the input tensor along `dim` must be divisible by 2. +/// +/// ### Arguments +/// * `tensor` - The input tensor. +/// +/// ### Returns +/// * A tensor with the same shape as the input, except the size along `dim` is halved. +pub fn glu(tensor: Tensor, dim: usize) -> Tensor { + // TODO: Handle negative indices with AsIndex for compatibility with Pytorch nn.GLU. + + assert!( + tensor.dims()[dim].is_multiple_of(2), + "Input tensor along dimension {dim} must have an even size. N is divisible by 2." + ); + let new_len = tensor.dims()[dim] / 2; + + let a = tensor.clone().slice_dim(dim, s![0..new_len]); + let b = tensor.slice_dim(dim, s![new_len..new_len * 2]); + + a.mul(sigmoid(b)) +} + +/// Applies the Softsign function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{softsign}(x) = \frac{x}{1 + |x|} +$$ +"# +)] +#[cfg_attr(not(doc), doc = "`softsign(x_i) = x_i / (1 + |x_i|)`")] +pub fn softsign(tensor: Tensor) -> Tensor { + tensor.clone().div(tensor.abs() + 1) +} + +/// Applies the HardShrink function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{hard\_shrink}(x) = + \begin{cases} + x & \text{if } x > \lambda \newline + x & \text{if } x < -\lambda \newline + 0 & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`hard_shrink(x) = x if x > lambda, x if x < -lambda, 0 otherwise`" +)] +/// # Arguments +/// - `lambda`: the lambda value for the Hard Shrink formulation. Default is 0.5. +pub fn hard_shrink(tensor: Tensor, lambda: f64) -> Tensor { + let mask = tensor.clone().abs().lower_equal_elem(lambda); + tensor.mask_fill(mask, 0) +} + +/// Applies the SoftShrink function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{soft\_shrink}(x) = + \begin{cases} + x - \lambda & \text{if } x > \lambda \newline + x + \lambda & \text{if } x < -\lambda \newline + 0 & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`soft_shrink(x) = x - lambda if x > lambda, x + lambda if x < -lambda, 0 otherwise`" +)] +/// # Arguments +/// - `lambda`: the lambda value for the Soft Shrink formulation. Default is 0.5. +pub fn soft_shrink(tensor: Tensor, lambda: f64) -> Tensor { + shrink(tensor, lambda, lambda) +} + +/// Applies the Shrink function element-wise. +/// +#[cfg_attr( + doc, + doc = r#" +$$ +\text{shrink}(x) = + \begin{cases} + x - \text{bias} & \text{if } x > \lambda \newline + x + \text{bias} & \text{if } x < -\lambda \newline + 0 & \text{otherwise} + \end{cases} +$$ +"# +)] +#[cfg_attr( + not(doc), + doc = "`shrink(x) = x - bias if x > lambda, x + bias if x < -lambda, 0 otherwise`" +)] +/// # Arguments +/// - `lambda`: the lambda value for the Shrink formulation. +/// - `bias`: the bias value for the Shrink formulation. +pub fn shrink( + tensor: Tensor, + lambda: f64, + bias: f64, +) -> Tensor { + let abs_tensor = tensor.clone().abs(); + let sign = tensor.clone().sign(); + let shrunk = tensor.sub(sign.mul_scalar(bias)); + let mask = abs_tensor.lower_equal_elem(lambda); + shrunk.mask_fill(mask, 0) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/mod.rs new file mode 100644 index 0000000..cbcb6ac --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/activation/mod.rs @@ -0,0 +1,3 @@ +mod base; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/autodiff.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/autodiff.rs new file mode 100644 index 0000000..46d7e1f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/autodiff.rs @@ -0,0 +1,75 @@ +pub use burn_backend::tensor::BasicAutodiffOps; + +use crate::{Tensor, TensorPrimitive, backend::AutodiffBackend}; + +impl Tensor { + /// Backward pass of the tensor. + pub fn backward(&self) -> B::Gradients { + B::backward(self.primitive.clone().tensor()) + } + + /// Get the gradients of a tensor if it exist. + /// + /// Returns a new reference to the same tensor. Therefore the same grad tensor can + /// be accessed multiple times. If you only need to get the gradients one time, + /// consider using [grad_remove](Tensor::grad_remove) for better performance. + pub fn grad(&self, grads: &B::Gradients) -> Option> { + match &self.primitive { + TensorPrimitive::Float(tensor) => B::grad(tensor, grads) + .map(TensorPrimitive::Float) + .map(Tensor::new), + TensorPrimitive::QFloat(_tensor) => B::grad(&self.primitive.clone().tensor(), grads) + .map(TensorPrimitive::Float) + .map(Tensor::new), + } + } + + /// Remove the grad tensor from the [grads](AutodiffBackend::Gradients) struct returning the result. + pub fn grad_remove(&self, grads: &mut B::Gradients) -> Option> { + match &self.primitive { + TensorPrimitive::Float(tensor) => B::grad_remove(tensor, grads) + .map(TensorPrimitive::Float) + .map(Tensor::new), + TensorPrimitive::QFloat(_tensor) => { + B::grad_remove(&self.primitive.clone().tensor(), grads) + .map(TensorPrimitive::Float) + .map(Tensor::new) + } + } + } + + /// Replace the grad tensor from the [grads](AutodiffBackend::Gradients) struct with the provided + /// gradient. + pub fn grad_replace(&self, grads: &mut B::Gradients, grad: Tensor) { + match &self.primitive { + TensorPrimitive::Float(tensor) => { + B::grad_replace(tensor, grads, grad.primitive.tensor()) + } + TensorPrimitive::QFloat(_tensor) => B::grad_replace( + &self.primitive.clone().tensor(), + grads, + grad.primitive.tensor(), + ), + } + } +} + +impl> Tensor { + /// Returns the inner tensor without the autodiff information. + pub fn inner(self) -> Tensor { + Tensor::new(K::inner(self.primitive)) + } + + /// Convert a tensor to the autodiff backend. + /// + /// # Arguments + /// + /// * `inner` - The tensor to convert. + /// + /// # Returns + /// + /// The tensor converted to the autodiff backend. + pub fn from_inner(inner: Tensor) -> Self { + Self::new(K::from_inner(inner.primitive)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/base.rs new file mode 100644 index 0000000..12807c9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/base.rs @@ -0,0 +1,3319 @@ +#![allow(clippy::single_range_in_vec_init)] +use crate::backend::ExecutionError; +use crate::check::unwrap_shape_reshape; + +use burn_backend::Scalar; +pub use burn_backend::tensor::BasicOps; + +use alloc::vec::Vec; + +use alloc::format; +use alloc::string::String; +use alloc::vec; + +use burn_std::{SliceOps, stub::RwLock}; +use core::iter::repeat; +use core::{fmt::Debug, ops::Range}; +use serde::{Deserialize, Deserializer}; + +use crate::{AsIndex, Slice, SliceArg, wrap_index}; +use crate::{ + Bool, ElementConversion, Float, Int, Shape, TensorData, TensorKind, TensorMetadata, + backend::Backend, check, +}; +use crate::{DType, Element}; +use crate::{IndexingUpdateOp, TensorCreationOptions}; +use crate::{cast::ToElement, check::TensorCheck}; +use serde::{Serialize, Serializer}; + +/// A tensor with a given backend, shape and data type. +/// +/// # Indexing +/// Indexing a tensor can be done using [`slice`](Tensor::slice) for all tensor types +/// or [`select`](Tensor::select) for numeric types. +/// +/// ## Example +/// +/// ```rust +/// use burn_tensor::backend::Backend; +/// use burn_tensor::Tensor; +/// use burn_tensor::Int; +/// +/// fn example() { +/// let device = Default::default(); +/// +/// let tensor = Tensor::::from_data( +/// [ +/// [3.0, 4.9, 2.0], +/// [2.0, 1.9, 3.0], +/// [6.0, 1.5, 7.0], +/// [3.0, 4.9, 9.0], +/// ], +/// &device, +/// ); +/// +/// // Slice the tensor to get the second and third rows: +/// // [[2.0, 1.9, 3.0], [6.0, 1.5, 7.0]] +/// // The resulting tensor will have dimensions [2, 3]. +/// let slice = tensor.clone().slice([1..3]); +/// println!("{slice}"); +/// +/// // Slice the tensor to get the first two rows and the first 2 columns: +/// // [[3.0, 4.9], [2.0, 1.9]] +/// // The resulting tensor will have dimensions [2, 2]. +/// let slice = tensor.clone().slice([0..2, 0..2]); +/// println!("{slice}"); +/// +/// // Index the tensor along the dimension 1 to get the elements 0 and 2: +/// // [[3.0, 2.0], [2.0, 3.0], [6.0, 7.0], [3.0, 9.0]] +/// // The resulting tensor will have dimensions [4, 2] +/// let indices = Tensor::::from_data([0, 2], &device); +/// let indexed = tensor.select(1, indices); +/// println!("{indexed}"); +/// } +/// ``` +#[derive(new, Clone, Debug)] +pub struct Tensor +where + B: Backend, + K: TensorKind, +{ + pub(crate) primitive: K::Primitive, +} + +impl From for Tensor +where + B: Backend, + K: BasicOps, + T: Into, +{ + fn from(value: T) -> Self { + Tensor::from_data(value.into(), &Default::default()) + } +} + +impl Tensor +where + B: Backend, + K: BasicOps, + K::Elem: Element, +{ + /// Executes an operation on the tensor and modifies its value. + /// + /// # Notes + /// + /// This won't necessarily reuse the same tensor data/buffer, but it should if there is + /// no other reference pointing to the same tensor. + /// + /// Wrapping operations with inplace is not an optimization, it's mainly there if you + /// want to mutate a tensor by using owned operations. A plausible usage would be to + /// update the weights of a mutable model reference. + pub fn inplace Self>(&mut self, func: F) { + let mut tensor_owned = Tensor::empty([0; D], &self.device()); + core::mem::swap(&mut tensor_owned, self); + + let mut tensor_new = func(tensor_owned); + core::mem::swap(&mut tensor_new, self); + } + + /// Converts the tensor into a primitive tensor. + pub fn into_primitive(self) -> K::Primitive { + self.primitive + } + + /// Converts from a primitive tensor into a tensor. + pub fn from_primitive(tensor: K::Primitive) -> Self { + Self::new(tensor) + } + + /// Returns the number of dimensions of the tensor. + pub fn rank(&self) -> usize { + self.primitive.rank() + } + + /// Returns the tensor primitive data type. + /// + /// # Note + /// Some element types are encoded in different primitive types depending on the backend + /// (e.g., bool could be encoded as `u8` or `u32`). + pub fn dtype(&self) -> DType { + self.primitive.dtype() + } + + /// Create an empty tensor of the given shape. + /// + /// # Arguments + /// + /// - `shape`: The shape of the tensor. + /// - `device`: The device where the tensor will be created. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create an empty tensor with dimensions [2, 3, 4]. + /// let tensor = Tensor::::empty([2, 3, 4], &device); + /// } + /// ``` + pub fn empty>(shape: S, options: impl Into>) -> Self { + let opt = options.into(); + let shape = shape.into(); + let dtype = opt.resolve_policy(K::Elem::dtype()); + check!(TensorCheck::creation_ops::("Empty", &shape)); + Self::new(K::empty(shape, &opt.device, dtype)) + } + + /// Create a tensor of the given shape where each element is zero. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::zeros(Shape::new([2, 3]), &device); + /// println!("{tensor}"); + /// // [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + /// } + /// ``` + pub fn zeros>(shape: S, options: impl Into>) -> Self { + let opt = options.into(); + let shape = shape.into(); + let dtype = opt.resolve_policy(K::Elem::dtype()); + check!(TensorCheck::creation_ops::("Zeros", &shape)); + Self::new(K::zeros(shape, &opt.device, dtype)) + } + + /// Returns a new tensor with the same shape, dtype, and device as the current tensor filled with zeros. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.zeros_like(); + /// println!("{tensor}"); + /// // [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + /// } + /// ``` + pub fn zeros_like(&self) -> Self { + Self::new(K::zeros(self.shape(), &self.device(), self.dtype())) + } + + /// Create a tensor of the given shape where each element is one. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::ones(Shape::new([2, 3]), &device); + /// println!("{tensor}"); + /// // [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + /// } + /// ``` + pub fn ones>(shape: S, options: impl Into>) -> Self { + let opt = options.into(); + let shape = shape.into(); + let dtype = opt.resolve_policy(K::Elem::dtype()); + check!(TensorCheck::creation_ops::("Ones", &shape)); + Self::new(K::ones(shape, &opt.device, dtype)) + } + + /// Returns a new tensor with the same shape, dtype, and device as the current tensor filled with ones. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.ones_like(); + /// println!("{tensor}"); + /// // [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + /// } + /// ``` + pub fn ones_like(&self) -> Self { + Self::new(K::ones(self.shape(), &self.device(), self.dtype())) + } + + /// Create a tensor of the given shape where each element is equal to the provided value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::full(Shape::new([2, 3]), 5.0, &device); + /// println!("{tensor}"); + /// // [[5.0, 5.0, 5.0], [5.0, 5.0, 5.0]] + /// } + /// ``` + pub fn full, E: ElementConversion>( + shape: S, + fill_value: E, + options: impl Into>, + ) -> Self { + let opt = options.into(); + let shape = shape.into(); + let dtype = opt.resolve_policy(K::Elem::dtype()); + check!(TensorCheck::creation_ops::("Full", &shape)); + Self::new(K::full( + shape, + Scalar::new(fill_value, &dtype), + &opt.device, + dtype, + )) + } + + /// Returns a new tensor with the same shape, dtype, and device as the current tensor, + /// filled with the provided value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.full_like(5.0); + /// println!("{tensor}"); + /// // [[5.0, 5.0, 5.0], [5.0, 5.0, 5.0]] + /// } + /// ``` + pub fn full_like(&self, fill_value: E) -> Self { + let dtype = self.dtype(); + Self::new(K::full( + self.shape(), + Scalar::new(fill_value, &dtype), + &self.device(), + dtype, + )) + } + + /// Returns the dimensions of the current tensor. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::ones([2, 3, 4], &device); + /// let dims = tensor.dims(); // [2, 3, 4] + /// println!("{dims:?}"); + /// } + /// ``` + pub fn dims(&self) -> [usize; D] { + Self::shape(self).dims() + } + + /// Returns the shape of the current tensor. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::ones([2, 3, 4], &device); + /// // Shape { dims: [2, 3, 4] } + /// let shape = tensor.shape(); + /// } + /// ``` + pub fn shape(&self) -> Shape { + self.primitive.shape() + } + + /// Reshape the tensor to have the given shape. + /// + /// The tensor has the same data and number of elements as the input. + /// + /// A `-1` in the shape is used to infer the remaining dimensions, e.g.: `[2, -1]` + /// will reshape the tensor with [2, 3, 4] dimensions to [2, 12]. + /// + /// A `0` in the shape instructs to keep the current dimension from the original tensor, + /// e.g.: `[2, 0, 4]` will reshape the tensor with [2, 3, 4] dimensions to [2, 3, 4]. + /// This is useful when reshaping tensors with unknown dimensions and combining with `-1` + /// to infer the remaining dimensions, e.g. `[0, -1]` will reshape the tensor + /// with [1, 3, 4] dimensions to [1, 12]. + /// + /// # Arguments + /// - `shape`: The new shape of the tensor. + /// + /// # Panics + /// - If the tensor contains more than one `-1` in the shape. + /// - If the tensor contains values that are not positive (other than -1). + /// - If the shape does not match the number of elements of the original shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a tensor with dimensions [2, 3, 4] + /// let tensor = Tensor::::ones([2, 3, 4], &device); + /// // Reshape it to [2, 12], where 12 is inferred from the number of elements. + /// let reshaped = tensor.reshape([2, -1]); + /// println!("{reshaped}"); + /// } + /// ``` + pub fn reshape>(self, shape: S) -> Tensor { + // Convert reshape args to shape + let shape = shape.into_shape::(self.shape()); + Tensor::new(K::reshape(self.primitive, shape)) + } + + /// Transpose the tensor. + /// + /// For a 2D tensor, this is the standard matrix transpose. For `D > 2`, the transpose is + /// applied on the last two dimensions. For example, the transpose of a tensor with shape + /// `[1, 2, 3, 4]` will have shape `[1, 2, 4, 3]`. + /// + /// See also [`permute`](Tensor::permute). + /// + /// # Arguments + /// + /// * `tensor` - The tensor to transpose. + /// + /// # Returns + /// + /// The transposed tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor of shape [2, 3] + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// + /// // Transpose the tensor: + /// // [[1.0, 5.0], [-2.0, 9.0], [3.0, 6.0]] + /// // The resulting tensor will have dimensions [3, 2]. + /// let transposed = tensor.transpose(); + /// println!("{transposed}"); + /// } + /// ``` + pub fn transpose(self) -> Tensor { + Tensor::new(K::transpose(self.primitive)) + } + + /// Alias for `transpose`. + #[inline(always)] + pub fn t(self) -> Tensor { + self.transpose() + } + + /// Swaps two dimensions of a tensor. + /// + /// This is a no-op when `dim1 == dim2`, assuming both are within bounds. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to swap the dimensions of. + /// * `dim1` - The first dimension to swap, supports negative indexing. + /// * `dim2` - The second dimension to swap, supports negative indexing. + /// + /// # Returns + /// + /// The tensor with the dimensions swapped. + /// + /// # Panics + /// + /// When dimensions are out of bounds. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor of shape [2, 3] + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// + /// // Swap the dimensions 0 and -1 (equivalent to `tensor.transpose()`): + /// // [[1.0, 5.0], [-2.0, 9.0], [3.0, 6.0]] + /// // The resulting tensor will have dimensions [3, 2]. + /// let swapped = tensor.swap_dims(0, -1); + /// println!("{swapped}"); + /// } + /// ``` + pub fn swap_dims(self, dim1: Dim1, dim2: Dim2) -> Tensor + where + Dim1: AsIndex, + Dim2: AsIndex, + { + let dim1 = dim1.expect_dim_index(D); + let dim2 = dim2.expect_dim_index(D); + check!(TensorCheck::swap_dims::(dim1, dim2)); + if dim1 == dim2 { + self + } else { + Tensor::new(K::swap_dims(self.primitive, dim1, dim2)) + } + } + + /// Permute the dimensions of the tensor. + /// + /// This is a no-op when the resolved `axes` match the current order. + /// + /// # Arguments + /// + /// * `axes` - The new order of the dimensions. The length of the axes + /// must be equal to the number of dimensions of the tensor. + /// The values must be unique and in the range of the number of dimensions. + /// The values can be negative, in which case they are used as an offset from the end. + /// + /// # Returns + /// + /// The tensor with the dimensions permuted. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor of shape [3, 2] + /// let tensor = Tensor::::from_data([[1.0, 5.0], [-2.0, 9.0], [3.0, 6.0]], &device); + /// + /// // Permute the dimensions 1 and 0: + /// // [[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]] + /// // The resulting tensor will have dimensions [3, 2]. + /// let permuted = tensor.permute([1, 0]); + /// println!("{permuted}"); + /// } + /// ``` + pub fn permute(self, axes: [Dim; D]) -> Tensor + where + Dim: AsIndex, + { + let mut no_op = true; + let mut fixed_axes = [0; D]; + for (i, axis) in axes.into_iter().enumerate() { + let dim = axis.expect_dim_index(D); + no_op &= dim == i; + fixed_axes[i] = dim; + } + + if no_op { + self + } else { + check!(TensorCheck::permute(fixed_axes)); + Tensor::new(K::permute(self.primitive, &fixed_axes)) + } + } + + /// Moves the dimension(s) of input at the position(s) in source to the position(s) in destination. + /// + /// Other dimensions of input that are not explicitly moved remain in their original order and appear + /// at the positions not specified in destination. + /// + /// # Arguments + /// + /// * `src` - The dimension(s) to move. The values must be unique and in the range of the number of dimensions. + /// The values can be negative, in which case they are used as an offset from the end. + /// + /// * `dst` - Destination positions for each of the original dims. These must also be unique. + /// + /// # Panics + /// + /// - If the source and destination dimensions are not of the same length. + /// - If the source and destination vectors contain duplicate values. + /// - If the source and destination vectors contain values that are out of bounds. + /// + /// # Returns + /// + /// The tensor with the dimensions moved. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 3D tensor of shape [3, 2, 1] + /// let tensor = Tensor::::from_data([[[1.0], [5.0]], [[-2.0], [9.0]], [[3.0], [6.0]]], &device); + /// + /// // Move the dimensions 0 and 1: + /// // [[[1.0], [-2.0], [3.0]], [[5.0], [9.0], [6.0]]] + /// // The resulting tensor will have dimensions [2, 3, 1]. + /// let moved = tensor.movedim(1, 0); + /// println!("{moved}"); + /// } + /// ``` + /// + /// # Note + /// + /// This is a syntactic sugar for `permute`. It is used widely enough, so we define a separate Op + /// for it + pub fn movedim(self, src: S1, dst: S2) -> Tensor { + let source_dims = src.into_dim_vec::(); + let destination_dims = dst.into_dim_vec::(); + + check!(TensorCheck::movedim_args_length( + &source_dims, + &destination_dims + )); + + let mut m = [-1; D]; + for (&d, &s) in destination_dims.iter().zip(source_dims.iter()) { + m[d] = s as isize; + } + let mut axes: [isize; D] = [0; D]; + let mut source_i = 0; + for (dest_i, item) in axes.iter_mut().enumerate().take(D) { + *item = if m[dest_i] != -1 { + m[dest_i] + } else { + while source_dims.contains(&source_i) { + source_i += 1; + } + let result = source_i as isize; + source_i += 1; + result + }; + } + + self.permute(axes) + } + + /// Reverse the order of elements in the tensor along the given dimensions. + /// + /// # Arguments + /// + /// * `axes` - The dimensions to reverse. The values must be unique and in the range of the number of dimensions. + /// The values can be negative, in which case they are used as an offset from the end. + /// + /// # Returns + /// + /// The tensor with the axes flipped. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [4, 3] + /// let tensor = Tensor::::from_data( + /// [ + /// [3.0, 4.9, 2.0], + /// [2.0, 1.9, 3.0], + /// [4.0, 5.9, 8.0], + /// [1.4, 5.8, 6.0], + /// ], + /// &device, + /// ); + /// + /// // Flip the elements in dimensions 0 and 1: + /// // [[6.0, 5.8, 1.4], + /// // [8.0, 5.9, 4.0], + /// // [3.0, 1.9, 2.0], + /// // [2.0, 4.9, 3.0]] + /// // The resulting tensor will have dimensions [4, 3]. + /// let flipped = tensor.flip([0, 1]); + /// println!("{flipped}"); + /// } + /// ``` + pub fn flip(self, axes: [isize; N]) -> Tensor { + // Convert the axes to usize and handle negative values without using vector + let mut transformed_axes: [usize; N] = [0; N]; + for (i, &x) in axes.iter().enumerate() { + transformed_axes[i] = if x < 0 { + (D as isize + x) as usize + } else { + x as usize + }; + } + + // Check if the axes are valid + check!(TensorCheck::flip(D, &transformed_axes)); + + Tensor::new(K::flip(self.primitive, &transformed_axes)) + } + + /// Flatten the tensor along a given range of dimensions. + /// + /// This function collapses the specified range of dimensions into a single dimension, + /// effectively flattening the tensor in that range. + /// + /// # Arguments + /// + /// - `start_dim`: The starting dimension of the range to be flattened, + /// supports negative indexing. + /// - `end_dim`: The ending dimension of the range to be flattened (inclusive), + /// supports negative indexing. + /// + /// # Type Parameters + /// + /// - `D2`: The resulting number of dimensions in the flattened tensor. + /// + /// # Returns + /// + /// A new `Tensor` instance with the specified range of dimensions flattened. + /// + /// # Example + /// + /// ```rust + /// + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 3D tensor with dimensions [2, 3, 4] + /// let tensor = Tensor::::ones(Shape::new([2, 3, 4]), &device); + /// + /// // Flatten the tensor from dimensions 1 to 2 (inclusive). + /// // The resulting tensor will have dimensions [2, 12] + /// let flattened: Tensor = tensor.flatten(1, 2); + /// println!("{flattened}"); + /// } + /// ``` + pub fn flatten( + self, + start_dim: impl AsIndex, + end_dim: impl AsIndex, + ) -> Tensor { + let start_dim = start_dim.expect_dim_index(D); + let end_dim = end_dim.expect_dim_index(D); + check!(TensorCheck::flatten::(start_dim, end_dim)); + let new_shape = self.shape().flatten_dims(start_dim, end_dim); + + Tensor::new(K::reshape(self.primitive, new_shape)) + } + + /// Squeeze the tensor along all dimensions, removing dimensions + /// of size one, and effectively reducing the rank of the tensor. + /// + /// # Type Parameters + /// + /// - `D2`: The resulting number of dimensions in the squeezed tensor. + /// + /// # Returns + /// + /// A new `Tensor` instance with the specified dimension removed. + /// + /// # Example + /// + /// ```rust + /// + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 4D tensor with dimensions [1, 3, 1, 3] + /// let tensor = Tensor::::from_data( + /// [[[[3.0, 4.9, 2.0]], [[2.0, 1.9, 3.0]], [[4.0, 5.9, 8.0]]]], + /// &device, + /// ); + /// + /// // Squeeze the tensor dimensions. + /// // The resulting tensor will have dimensions [3, 3]. + /// let squeezed = tensor.squeeze::<2>(); + /// println!("{squeezed}"); + /// } + /// ``` + pub fn squeeze(self) -> Tensor { + let new_dims = self + .shape() + .iter() + .filter_map(|&dim| if dim == 1 { None } else { Some(dim) }) + .collect::>(); + check!(TensorCheck::squeeze_dims_len::(new_dims.len())); + + Tensor::new(K::reshape(self.primitive, new_dims.into())) + } + + /// Squeeze the tensor along the given dimension, removing the specified dimension + /// of size one, and effectively reducing the rank of the tensor by one. + /// + /// # Arguments + /// + /// - `dim`: The dimension to be squeezed. + /// + /// # Type Parameters + /// + /// - `D2`: The resulting number of dimensions in the squeezed tensor. + /// + /// # Panics + /// + /// If the size in the squeezed dimension is not 1. + /// + /// # Returns + /// + /// A new `Tensor` instance with the specified dimension removed. + /// + /// # Example + /// + /// ```rust + /// + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 3D tensor with dimensions [3, 1, 3] + /// let tensor = Tensor::::from_data( + /// [[[3.0, 4.9, 2.0]], [[2.0, 1.9, 3.0]], [[4.0, 5.9, 8.0]]], + /// &device, + /// ); + /// + /// // Squeeze the dimension 1. + /// // The resulting tensor will have dimensions [3, 3]. + /// let squeezed = tensor.squeeze_dim::<2>(1); + /// println!("{squeezed}"); + /// } + /// ``` + pub fn squeeze_dim(self, dim: usize) -> Tensor { + check!(TensorCheck::squeeze::(dim, &self.shape())); + + let current_dims = self.shape(); + let mut new_dims: [usize; D2] = [0; D2]; + + new_dims[..dim].copy_from_slice(¤t_dims[..dim]); + new_dims[dim..].copy_from_slice(¤t_dims[dim + 1..]); + + check!(TensorCheck::squeeze_dims_len::(new_dims.len())); + Tensor::new(K::reshape(self.primitive, new_dims.into())) + } + + /// Removes specified dimensions of size 1 from a tensor's shape. This function takes a tensor and + /// an array of dimensions (`dims`) to be squeezed. If `dims` is provided, only the dimensions + /// specified in this array will be removed. Each dimension in `dims` should correspond to a size of 1 + /// in the tensor; otherwise, the dimension will not be squeezed. If `dims` is empty, all single-dimensional entries + /// in the tensor will be removed. If entries in `dims` are negative, then dimensions will be counted + /// from the back. + /// + /// # Arguments + /// + /// - `dims`: The dimension(s) to be squeezed. + /// + /// # Type Parameters + /// + /// - `D2`: The resulting number of dimensions in the squeezed tensor. + /// + /// # Returns + /// + /// A new `Tensor` instance with the specified dimensions removed. + /// + /// # Example + /// + /// ```rust + /// + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 4D tensor with dimensions [2, 1, 4, 1] + /// let tensor = Tensor::::ones(Shape::new([2, 1, 4, 1]), &device); + /// + /// // Squeeze the dimensions 1 and 3. + /// // The resulting tensor will have dimensions [2, 4]. + /// let squeezed: Tensor = tensor.squeeze_dims(&[1, 3]); + /// println!("{squeezed}"); + /// } + /// ``` + pub fn squeeze_dims(self, dims: &[isize]) -> Tensor { + let current_dims = self.shape(); + let mut dim_indices: Vec; + + // Check if dims is empty, if yes then assign dim_indices all single-dimensional entries + if dims.is_empty() { + dim_indices = current_dims + .iter() + .enumerate() + .filter_map(|(index, &dim)| if dim == 1 { Some(index) } else { None }) + .collect(); + } else { + // If negative dims, count from the back + dim_indices = dims + .iter() + .map(|&d| { + if d < 0 { + (current_dims.len() as isize + d) as usize + } else { + d as usize + } + }) + .collect(); + } + + // Sort indices and remove duplicates + dim_indices.sort_unstable(); + dim_indices.dedup(); + + // Make sure squeeze_dims doesn't result in a tensor with < 1 dimensions + check!(TensorCheck::squeeze_dims_input::( + &dim_indices, + ¤t_dims + )); + + // Calculate new dimensions + let mut new_dims = Vec::new(); + for (index, &dim_size) in current_dims.iter().enumerate() { + // Exclude the dimension if it's explicitly marked for squeezing + if dim_indices.contains(&index) { + check!(TensorCheck::squeeze::(index, ¤t_dims)); + continue; + } + new_dims.push(dim_size); + } + + // Check that after squeezing, we still respect the D2 size + check!(TensorCheck::squeeze_dims_len::(new_dims.len())); + + Tensor::new(K::reshape(self.primitive, new_dims.into())) + } + + /// Unsqueeze the current tensor. Create new leading dimensions to fit the given size. + /// + /// # Type Parameters + /// + /// - `D2`: The resulting number of dimensions in the unsqueezed tensor. + /// + /// # Panics + /// + /// If the output size `D2` is smaller than the current number of dimensions. + /// + /// # Returns + /// + /// A new `Tensor` instance with the specified dimensions added. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [3, 3] + /// let tensor = Tensor::::ones(Shape::new([3, 3]), &device); + /// // Unsqueeze the tensor up to 4 dimensions. + /// // The resulting tensor will have dimensions [1, 1, 3, 3]. + /// let unsqueezed = tensor.unsqueeze::<4>(); + /// println!("{unsqueezed}"); + /// } + /// ``` + pub fn unsqueeze(self) -> Tensor { + check!(TensorCheck::unsqueeze::()); + + let mut dims = [1; D2]; + let num_ones = D2 - D; + let shape = self.shape(); + + dims[num_ones..(D + num_ones)].copy_from_slice(&shape[..D]); + + let shape = Shape::new(dims); + self.reshape(shape) + } + + /// Creates a new tensor with a dimension of size one inserted at the specified position. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [3, 3] + /// let tensor = Tensor::::ones(Shape::new([3, 3]), &device); + /// // Unsqueeze the dimension 1. + /// // The resulting tensor will have dimensions [3, 1, 3]. + /// let unsqueezed: Tensor = tensor.unsqueeze_dim(1); + /// println!("{unsqueezed}"); + /// } + /// ``` + pub fn unsqueeze_dim(self, dim: usize) -> Tensor { + check!(TensorCheck::unsqueeze_dim::(dim)); + + let mut dims = [1; D2]; + let shape = self.shape(); + + dims[0..dim].copy_from_slice(&shape[0..dim]); + + if dim < D { + dims[dim] = 1; + dims[(dim + 1)..].copy_from_slice(&shape[dim..]); + } else { + dims[dim] = 1; + } + + let shape = Shape::new(dims); + self.reshape(shape) + } + + /// Creates a new tensor with added dimensions of size one inserted at the specified indices. + /// The indices can be negative, in which case they are counted from the last to the first dimension. + /// the axes can contain duplicates, in which case the number of dimensions inserted at the index + /// is the number of duplicates. + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 3D tensor with dimensions [3, 4, 5] + /// let tensor = Tensor::::ones(Shape::new([3, 4, 5]), &device); + /// // Unsqueeze the leading dimension (0) once and the trailing dimension (-1) twice. + /// // The resulting tensor will have dimensions [1, 3, 4, 5, 1, 1]. + /// let unsqueezed: Tensor = tensor.unsqueeze_dims(&[0, -1, -1]); + /// println!("{unsqueezed}"); + /// } + /// ``` + pub fn unsqueeze_dims(self, axes: &[impl AsIndex]) -> Tensor { + let mut new_dims = [1; D2]; + let old_dims = self.shape(); + //for checking if the dimension is in the acceptable range + + //part 1: convert the negative indices to positive + let mut neg_offset = D2; + let mut dim_indices = axes + .iter() + .map(|d| { + let d = d.as_index(); + // check if the dimension is in the acceptable range + check!(TensorCheck::unsqueeze_dims::<{ D2 }>(d)); + (if d < 0 { + neg_offset -= 1; // handle multiple negative indices (decrease dim value in reverse) + d + neg_offset as isize + 1 + } else { + d + }) as usize + }) + .collect::>(); + + //sort the indices + dim_indices.sort_unstable(); + + //Now use this to copy the chunks of the dims + let mut prev_idx: usize = 0; + let mut current_left_b: usize = 0; + let mut current_right_b: usize = 0; + let mut offset: usize = 0; + dim_indices.iter().for_each(|d| { + //check if there is space for at least one dimension + if prev_idx < *d { + current_right_b = *d - offset; + //copy the chunks of the dims + if current_right_b < D { + new_dims[prev_idx..*d] + .copy_from_slice(&old_dims[current_left_b..current_right_b]); + } else { + new_dims[prev_idx..*d].copy_from_slice(&old_dims[current_left_b..]); + } + prev_idx = *d + 1; + //offset is equal to the number of extracted elements from the original shape + offset += current_right_b - current_left_b; + current_left_b = current_right_b; + } else { + //it's sorted so the only reason this would happen + //is if multiple indices are the same + prev_idx += 1; + } + }); + //copy over anything past the index of the last new dimension + if current_left_b < D { + new_dims[prev_idx..].copy_from_slice(&old_dims[current_left_b..]); + } + + //lastly, create the shape and reshape + let shape = Shape::new(new_dims); + self.reshape(shape) + } + + /// Roll operation along a specific dimension; wrapping around the elements. + /// + /// ## Parameters + /// + /// - `shift`: The roll extent; supports negative values and wraps around. + /// - `dim`: The dimension to roll; supports negative indexing. + /// + /// ## Returns + /// + /// A new tensor with the specified dimension rolled by the given shift amount. + pub fn roll_dim(self, shift: Shift, dim: Dim) -> Self + where + Shift: AsIndex, + Dim: AsIndex, + { + let dim = dim.expect_dim_index(D); + let size = self.shape()[dim]; + if size == 0 { + // If the dimension is empty, return the tensor as is. + return self; + } + + let shift = wrap_index(shift, size); + if shift == 0 { + // If the shift is zero, return the tensor as is. + return self; + } + + self.unchecked_roll_dim(shift, dim) + } + + /// Internal implementation of `roll_dim` that does not canonicalize dimensions or shifts. + /// + /// ## Parameters + /// + /// - `shift`: The number of positions to shift; must be (0 < shift < size). + /// - `dim`: The dimension to roll; must be a valid index for the tensor's shape. + /// + /// ## Returns + /// + /// A new tensor with the specified dimension rolled by the given shift amount. + #[inline(always)] + fn unchecked_roll_dim(self, shift: usize, dim: usize) -> Self { + #[cfg(debug_assertions)] + { + let size = self.shape()[dim]; + assert!( + 0 < shift && shift < size, + "Expected: 0 < shift < size: found shift={shift}, size={size}", + ); + assert!( + dim < self.shape().num_dims(), + "Expected: dim < num_dims: found dim={dim}, num_dims={size}", + ); + } + + Tensor::cat( + vec![ + self.clone().slice_dim(dim, shift..), + self.slice_dim(dim, ..shift), + ], + dim, + ) + } + + /// Roll operation. + /// + /// Note: unlike ``pytorch``, `dims` and `shifts` must have the same length. + /// + /// A given `dim` may be rolled multiple times, and the shifts will be applied sequentially. + /// + /// ## Parameters + /// + /// - `shifts`: A slice of shifts corresponding to each dimension; + /// supports negative values and wraps around. + /// - `dims`: A slice of dimensions to roll; supports negative indexing. + /// + /// ## Returns + /// + /// A new tensor with the specified dimensions rolled by the given shifts. + pub fn roll(self, shifts: &[Shift], dims: &[Dim]) -> Self + where + Shift: AsIndex, + Dim: AsIndex, + { + assert_eq!( + dims.len(), + shifts.len(), + "Dimensions and shifts must align; found dims={dims:#?}, shifts={shifts:#?}", + ); + + // This is a fair amount of complexity, which could be replaced + // by a simple canonicalization of `dims` and wrapping of `shifts`. + // The work is done here to ensure that any roll operation + // which could be a no-op is a no-op; simplifying the accounting + // needed by backend-specific implementations of the inner roll op. + + let item_count = dims.len(); + + let shape = self.shape(); + + // Accumulate the effective shifts for each dimension. + let mut accumulated_shifts: Vec = vec![0; shape.len()]; + for i in 0..item_count { + let dim = dims[i].expect_dim_index(D); + accumulated_shifts[dim] += shifts[i].as_index(); + } + + // Do this after we've checked the validity of `dims` and `shifts`. + if self.shape().num_elements() == 0 { + // If the tensor is empty, return it as is. + return self; + } + + // Wrap the accumulated shifts, and filter out empty dimensions. + let mut effective_dims: Vec = Vec::with_capacity(item_count); + let mut effective_shifts: Vec = Vec::with_capacity(item_count); + for dim in 0..shape.len() { + // `wrap_index` should inline, and has a fast-exit path for zero shifts. + let shift = wrap_index(accumulated_shifts[dim], shape[dim]); + if shift == 0 { + continue; + } + + effective_dims.push(dim); + effective_shifts.push(shift); + } + + // If no shifts are needed, return the original tensor. + if effective_shifts.is_empty() { + return self; + } + + // At this point: + // - `dims` contains the effective dimensions to roll, in index order, + // - `shifts` contains the effective usize shifts for each dimension. + // - Every shift is non-zero, and less than the size of the corresponding dimension. + self.unchecked_roll(&effective_shifts, &effective_dims) + } + + /// `roll` internal implementation. + /// + /// ## Parameters + /// + /// - `shifts`: A slice of shifts corresponding to each dimension; + /// must be non-empty, the same length as `dims`, and all ``1..``. + /// - `dims`: A slice of dimensions to roll; must be non-empty; + /// the same length as `shifts`, and must not contain repeats. + /// + /// ## Panics + /// + /// Panics if the shifts and dimensions do not align, or if dimensions contain repeats. + /// + /// ## Returns + /// + /// A new tensor with the specified dimensions rolled by the given shifts. + #[inline(always)] + fn unchecked_roll(self, shifts: &[usize], dims: &[usize]) -> Self { + #[cfg(debug_assertions)] + { + assert!(!shifts.is_empty()); + assert_eq!( + shifts.len(), + dims.len(), + "Shifts and dimensions must align; found {} shifts and {} dims", + shifts.len(), + dims.len() + ); + + let mut unique_dims = dims.to_vec(); + unique_dims.dedup(); + + assert_eq!( + unique_dims.len(), + dims.len(), + "Dimensions must not contain repeats; found {} unique dims and {} total dims", + unique_dims.len(), + dims.len() + ) + } + + let x = self.unchecked_roll_dim(shifts[0], dims[0]); + + if dims.len() == 1 { + x + } else { + x.unchecked_roll(&shifts[1..], &dims[1..]) + } + } + + /// Returns a tensor containing the elements selected from the given slices. + /// + /// This method provides flexible tensor slicing with support for various range types, + /// negative indices, and stepped slicing. The method accepts both single slices and + /// arrays of slices, with the [`s!`] macro providing convenient syntax for complex patterns. + /// + /// # Arguments + /// + /// * `slices` - Can be: + /// - A single range for 1D slicing (e.g., `0..5`, `..`, `2..`) + /// - An array of ranges (e.g., `[0..2, 1..4]`) + /// - The [`s!`] macro output for advanced slicing with steps + /// - a `&Vec` or `&[Slice]` + /// + /// # Behavior + /// + /// - Supports partial and full slicing in any number of dimensions + /// - Handles negative indices by wrapping from the end (-1 is the last element) + /// - Automatically clamps ranges that exceed tensor dimensions + /// - Supports stepped slicing for selecting every nth element + /// - Negative steps reverse the selection order + /// + /// # Panics + /// + /// - If the number of slices exceeds the tensor's dimensions + /// - If a range is descending (e.g., 2..1) or empty (e.g., 1..1) without negative step + /// - If a step is zero + /// + /// # Examples + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape, s}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// + /// // Single dimension slicing - no brackets needed! + /// let tensor = Tensor::::arange(0..10, &device); + /// let slice = tensor.clone().slice(2..8); // Simple range + /// assert_eq!(slice.into_data().to_vec::().unwrap(), vec![2, 3, 4, 5, 6, 7]); + /// + /// // Using s! macro for single dimension with step + /// let slice = tensor.clone().slice(s![0..10;2]); // Every 2nd element + /// assert_eq!(slice.into_data().to_vec::().unwrap(), vec![0, 2, 4, 6, 8]); + /// + /// // Reverse a dimension with negative step + /// let slice = tensor.slice(s![..;-1]); // Reverse entire tensor + /// assert_eq!(slice.into_data().to_vec::().unwrap(), vec![9, 8, 7, 6, 5, 4, 3, 2, 1, 0]); + /// + /// // Multi-dimensional slicing + /// let tensor = Tensor::::ones(Shape::new([4, 6]), &device); + /// + /// // Array syntax for simple ranges + /// let slice = tensor.clone().slice([1..3, 2..5]); + /// assert_eq!(slice.dims(), [2, 3]); + /// + /// // Advanced multi-dimensional with s! macro + /// let slice = tensor.clone().slice(s![0..4;2, ..;-1]); // Every 2nd row, reverse columns + /// assert_eq!(slice.dims(), [2, 6]); + /// + /// // Complex 3D example with mixed slice types + /// let tensor = Tensor::::ones(Shape::new([4, 6, 8]), &device); + /// let slice = tensor.slice(s![1..3, ..;2, -3..]); // Rows 1-2, every 2nd col, last 3 depth + /// assert_eq!(slice.dims(), [2, 3, 3]); + /// + /// // Using negative indices + /// let tensor = Tensor::::ones(Shape::new([4, 6]), &device); + /// let slice = tensor.slice(s![-2.., ..-1]); // Last 2 rows, all but last column + /// assert_eq!(slice.dims(), [2, 5]); + /// } + /// ``` + /// + /// # See Also + /// + /// - [`s!`] - The recommended macro for creating complex slice specifications + /// - [`slice_assign`](Self::slice_assign) - Assign values to a slice + /// - [`slice_fill`](Self::slice_fill) - Fill a slice with a constant value + /// - [`slice_dim`](Self::slice_dim) - Slice a single dimension + /// + /// [`s!`]: crate::s! + pub fn slice(self, slices: S) -> Self + where + S: SliceArg, + { + let shape = self.shape(); + let slices = slices.into_slices(&shape); + + // Validate slices + check!(TensorCheck::slice::(&shape, &slices)); + + // Calculate output shape and check for empty slices + let mut output_dims = shape.clone(); + for (dim, slice) in slices.iter().enumerate() { + output_dims[dim] = slice.output_size(shape[dim]); + } + + // Return empty tensor if any dimension is 0 (empty slice) + if output_dims.contains(&0) { + return Self::empty(output_dims, &self.device()); + } + Self::new(K::slice(self.primitive, &slices)) + } + + /// Assigns values to a slice of the tensor and returns the updated tensor. + /// + /// This method supports advanced slicing with steps, including negative steps for reverse + /// assignment. Like `slice`, it accepts both single slices and arrays, with the [`s!`] macro + /// providing powerful syntax for complex patterns. + /// + /// # Arguments + /// + /// * `slices` - Slice specification (same format as `slice` method) + /// * `values` - Tensor with values to assign (must match slice dimensions) + /// + /// # Panics + /// + /// - If slices exceed tensor dimensions + /// - If values dimensions don't match the selected slice shape + /// - If a step is zero + /// + /// # Examples + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, s}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// + /// // Simple assignment to a sub-region + /// let mut tensor = Tensor::::zeros([4, 6], &device); + /// let values = Tensor::::ones([2, 3], &device); + /// tensor = tensor.slice_assign([1..3, 2..5], values); + /// // Now tensor[1..3, 2..5] contains ones + /// + /// // Single dimension assignment with step + /// let mut tensor = Tensor::::zeros([10], &device); + /// let values = Tensor::::ones([5], &device); + /// tensor = tensor.slice_assign(s![0..10;2], values); + /// // Now every 2nd element is 1: [1, 0, 1, 0, 1, 0, 1, 0, 1, 0] + /// + /// // Reverse assignment with negative step + /// let mut tensor = Tensor::::from_data([0.0, 1.0, 2.0, 3.0, 4.0], &device); + /// let values = Tensor::::from_data([10.0, 11.0, 12.0, 13.0, 14.0], &device); + /// tensor = tensor.slice_assign(s![..;-1], values); + /// // Assigns in reverse: [14, 13, 12, 11, 10] + /// + /// // Complex multi-dimensional assignment + /// let mut tensor = Tensor::::zeros([4, 6, 8], &device); + /// let values = Tensor::::ones([2, 3, 3], &device); + /// tensor = tensor.slice_assign(s![0..4;2, ..;2, -3..], values); + /// // Assigns to every 2nd row, every 2nd column, last 3 in depth + /// + /// // Mixed syntax example + /// let mut tensor = Tensor::::zeros([8, 8], &device); + /// let pattern = Tensor::::ones([4, 4], &device); + /// tensor = tensor.slice_assign(s![..;2, ..;2], pattern); + /// // Creates a checkerboard pattern with ones + /// } + /// ``` + /// + /// # See Also + /// + /// - [`s!`] - The recommended macro for creating complex slice specifications + /// - [`slice`](Self::slice) - Extract a slice from a tensor + /// - [`slice_fill`](Self::slice_fill) - Fill a slice with a constant value + /// + /// [`s!`]: crate::s! + pub fn slice_assign(self, slices: S, values: Self) -> Self + where + S: SliceArg, + { + let shape = self.shape(); + let slices = slices.into_slices(&shape); + + // Check if any slice produces 0 elements (empty assignment). + // Empty assignments are no-ops and would cause issues in backend implementations. + let is_empty_assignment = slices + .iter() + .enumerate() + .any(|(i, slice)| slice.output_size(shape[i]) == 0); + + if is_empty_assignment { + return self; + } + + check!(TensorCheck::slice_assign::( + &shape, + &values.shape(), + &slices + )); + + Self::new(K::slice_assign(self.primitive, &slices, values.primitive)) + } + + /// Fills a slice of the tensor with a constant value and returns the updated tensor. + /// + /// Like other slice methods, accepts both single slices and arrays. However, this method + /// currently **does not support stepped slicing** - use [`slice_assign`](Self::slice_assign) + /// with a constant tensor for stepped patterns. + /// + /// # Arguments + /// + /// * `slices` - Slice specification (same format as `slice` method, but no steps) + /// * `value` - The value to fill the slice with + /// + /// # Panics + /// + /// - If slices exceed tensor dimensions + /// - If any slice has a step != 1 (not yet supported) + /// + /// # Examples + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, s}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// + /// // Simple fill for a single dimension + /// let mut tensor = Tensor::::zeros([10], &device); + /// tensor = tensor.slice_fill(2..5, 1.0); + /// // Now tensor is [0, 0, 1, 1, 1, 0, 0, 0, 0, 0] + /// + /// // Multi-dimensional fill + /// let mut tensor = Tensor::::zeros([4, 6], &device); + /// tensor = tensor.slice_fill([1..3, 2..5], -1.0); + /// // Fills the rectangle at rows 1-2, columns 2-4 with -1 + /// + /// // Using negative indices + /// let mut tensor = Tensor::::zeros([10], &device); + /// tensor = tensor.slice_fill(-3.., 2.0); + /// // Fills the last 3 elements with 2.0 + /// + /// // Complex multi-dimensional example + /// let mut tensor = Tensor::::ones([4, 6, 8], &device); + /// tensor = tensor.slice_fill(s![1..3, .., -2..], 0.0); + /// // Sets rows 1-2, all columns, last 2 in depth to 0 + /// + /// // Stepped slicing is supported + /// let mut tensor = Tensor::::zeros([10], &device); + /// tensor = tensor.slice_fill(s![0..10;2], 1.0); + /// // Now every 2nd element is 1: [1, 0, 1, 0, 1, 0, 1, 0, 1, 0] + /// } + /// ``` + /// + /// # See Also + /// + /// - [`s!`] - The macro for creating slice specifications with steps + /// - [`slice`](Self::slice) - Extract a slice from a tensor + /// - [`slice_assign`](Self::slice_assign) - Assign tensor values to a slice + /// + /// [`s!`]: crate::s! + pub fn slice_fill(self, slices: S, value: E) -> Self + where + S: SliceArg, + { + let shape = self.shape(); + let slices = slices.into_slices(&shape); + + check!(TensorCheck::slice::(&shape, &slices)); + + let slice_shape = shape.slice(&slices).unwrap(); + let value = Tensor::::from_data_dtype( + [value.elem::()], + &self.device(), + self.dtype(), + ); + let value = value.expand(slice_shape); + self.slice_assign(&slices, value) + } + + /// Returns a new tensor with the specified dimension sliced. + /// + /// # Arguments + /// + /// * `dim`: The dimension to slice. + /// * `slice`: The slice specification for the dimension. Can be a range (e.g., `2..5`), + /// slice with step (via `s!` macro, e.g., `s![0..10;2]`), or any type that implements `Into`. + /// + /// # Returns + /// + /// A new tensor with the specified dimension sliced. + /// + /// # Panics + /// + /// If the slice is out of bounds for the specified dimension. + /// + /// # Examples + /// + /// ```rust + /// # use burn_tensor::{Tensor, s}; + /// # use burn_tensor::backend::Backend; + /// # + /// # fn example() { + /// # let device = B::Device::default(); + /// let tensor = Tensor::::zeros([3, 4, 5], &device); + /// + /// // Simple range slicing + /// let sliced = tensor.clone().slice_dim(1, 1..3); + /// assert_eq!(sliced.shape().as_slice(), [3, 2, 5]); + /// + /// // Slicing with step - take every 2nd element + /// let sliced = tensor.clone().slice_dim(2, s![0..5;2]); + /// assert_eq!(sliced.shape().as_slice(), [3, 4, 3]); // Takes indices 0, 2, 4 + /// + /// // Reverse slicing with negative step + /// let sliced = tensor.clone().slice_dim(1, s![..;-1]); + /// assert_eq!(sliced.shape().as_slice(), [3, 4, 5]); // Reverses dimension 1 + /// + /// // Select from index 2 with step 3 + /// let sliced = tensor.clone().slice_dim(0, s![2..;3]); + /// assert_eq!(sliced.shape().as_slice(), [1, 4, 5]); // Takes only index 2 + /// + /// // Select single index (reduces dimension to size 1) + /// let sliced = tensor.slice_dim(0, 1); + /// assert_eq!(sliced.shape().as_slice(), [1, 4, 5]); + /// # } + /// ``` + /// + /// # See Also + /// + /// - [`slice`](Self::slice) - Slice multiple dimensions simultaneously + /// - [`s!`] - The macro for creating complex slice specifications + /// + /// [`s!`]: crate::s! + pub fn slice_dim(self, dim: usize, slice: S) -> Self + where + S: Into, + { + check!(TensorCheck::check_dim::(dim)); + let slice: Slice = slice.into(); + + let mut slices = vec![Slice::full(); D]; + slices[dim] = slice; + + self.slice(&slices) + } + + /// Returns the device of the current tensor. + pub fn device(&self) -> B::Device { + K::device(&self.primitive) + } + + /// Move the tensor to the given device. + pub fn to_device(self, device: &B::Device) -> Self { + Self::new(K::to_device(self.primitive, device)) + } + + /// Select tensor elements along the given dimension corresponding to the given indices. + /// + /// # Arguments + /// + /// * `dim` - The dimension to select from. Supports negative indexing. + /// * `indices` - The indices of the elements to select. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Int}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [4.0, 5.0, 6.0]], &device); + /// let indices = Tensor::::from_data([0], &device); + /// let tensor = tensor.select(0, indices); + /// println!("{tensor}"); + /// // [[1.0, -2.0, 3.0]] + /// } + /// ``` + pub fn select(self, dim: impl AsIndex, indices: Tensor) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::select::(dim)); + Self::new(K::select(self.primitive, dim, indices.primitive)) + } + + /// Assign the selected elements along the given dimension corresponding to the given indices + /// from the value tensor to the original tensor using sum reduction. + /// + /// # Note + /// For booleans, the sum operator is logical or. + /// + /// # Arguments + /// + /// * `dim` - The dimension along which to select. Supports negative indexing. + /// * `indices` - The indices to select from the tensor. + /// * `values` - The values to assign to the selected indices. + /// * `update` - The operation used to update the existing values at the indexed positions (e.g., add). + /// + /// # Example + /// + /// Example using a 3D tensor: + /// + /// `input[indices[i], j, k] += values[i, j, k]; // dim = 0` + /// `input[i, indices[j], k] += values[i, j, k]; // dim = 1` + /// `input[i, j, indices[k]] += values[i, j, k]; // dim = 2` + /// `input[i, j, indices[k]] += values[i, j, k]; // dim = -1 (same as dim = 2)` + /// + /// # Warning + /// + /// Not all backends have runtime bound checks for the indices, so make sure they are valid. + /// Otherwise, out of bounds indices could lead to unexpected results instead of panicking. + pub fn select_assign( + self, + dim: impl AsIndex, + indices: Tensor, + values: Tensor, + update: IndexingUpdateOp, + ) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::select_assign::( + dim, + &indices.shape(), + &values.shape() + )); + + Self::new(K::select_assign( + self.primitive, + dim, + indices.primitive, + values.primitive, + update, + )) + } + + /// Update the given tensor with the value tensor where the mask is true. + /// + /// This is similar to [mask_fill](Tensor::mask_fill), however the value is a tensor instead of + /// a scalar. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape, Bool}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let mask = Tensor::::from_data([[true, false, true], [false, true, false]], &device); + /// let value = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor.mask_where(mask, value); + /// println!("{tensor}"); + /// // [[2.0, -2.0, 4.0], [5.0, 2.0, 6.0]] + /// } + /// ``` + pub fn mask_where(self, mask: Tensor, value: Self) -> Self { + Self::new(K::mask_where( + self.primitive, + mask.primitive, + value.primitive, + )) + } + + /// Update the given tensor with the value where the mask is true. + /// + /// This is similar to [mask_where](Tensor::mask_where), however the value is a scalar instead of + /// a tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape, Bool}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let mask = Tensor::::from_data([[true, false, true], [false, true, false]], &device); + /// let tensor = tensor.mask_fill(mask, 3.0); + /// println!("{tensor}"); + /// // [[3.0, -2.0, 3.0], [5.0, 3.0, 6.0]] + /// } + /// ``` + pub fn mask_fill(self, mask: Tensor, value: E) -> Self { + let value = Scalar::new(value, &self.dtype()); + Self::new(K::mask_fill(self.primitive, mask.primitive, value)) + } + + /// Gather tensor elements corresponding to the given indices from the specified dim. + /// + /// Example using a 3D tensor: + /// + /// `output[i, j, k] = input[indices[i, j, k], j, k]; // dim = 0` + /// `output[i, j, k] = input[i, indices[i, j, k], k]; // dim = 1` + /// `output[i, j, k] = input[i, j, indices[i, j, k]]; // dim = 2` + /// + /// # Notes + /// + /// The index tensor should have the same shape as the original tensor except for the dim + /// specified. + /// + /// # Warning + /// Not all backends have runtime bound checks for the indices, so make sure the they are valid. + /// Otherwise, out of bounds indices could lead to unexpected results instead of panicking. + pub fn gather(self, dim: usize, indices: Tensor) -> Self { + check!(TensorCheck::gather::( + dim, + &self.shape(), + &indices.shape() + )); + + Self::new(K::gather(dim, self.primitive, indices.primitive)) + } + + /// Assign the gathered elements corresponding to the given indices along the specified dimension + /// from the value tensor to the original tensor using sum reduction. + /// + /// Example using a 3D tensor: + /// + /// `input[indices[i, j, k], j, k] += values[i, j, k]; // dim = 0` + /// `input[i, indices[i, j, k], k] += values[i, j, k]; // dim = 1` + /// `input[i, j, indices[i, j, k]] += values[i, j, k]; // dim = 2` + /// + /// # Arguments + /// * `dim` - The axis along which to scatter elements. + /// * `indices` - The indices of the elements to scatter. + /// * `values` - The values to scatter into the tensor. + /// * `update` - The operation used to update the existing values at the indexed positions (e.g., add). + /// + /// # Notes + /// + /// The index tensor should have the same shape as the original tensor except for the specified + /// dimension. The value and index tensors should have the same shape. + /// + /// Other references to the input tensor will not be modified by this operation. + /// + /// # Warning + /// Not all backends have runtime bound checks for the indices, so make sure the they are valid. + /// Otherwise, out of bounds indices could lead to unexpected results instead of panicking. + pub fn scatter( + self, + dim: usize, + indices: Tensor, + values: Self, + update: IndexingUpdateOp, + ) -> Self { + check!(TensorCheck::scatter::( + dim, + &self.shape(), + &indices.shape(), + &values.shape() + )); + + Self::new(K::scatter( + dim, + self.primitive, + indices.primitive, + values.primitive, + update, + )) + } + + /// Converts the data of the current tensor. + /// + /// # Note + /// + /// For better performance, prefer using a [Transaction](crate::Transaction) when reading multiple + /// tensors at once. This may improve laziness, especially if executed on a different + /// thread in native environments. + pub fn into_data(self) -> TensorData { + self.try_into_data().expect( + "Error while reading data: use `try_into_data` instead to catch the error at runtime", + ) + } + + /// Converts the data of the current tensor and returns any error that might have occurred since the + /// last time the device was synchronized. + /// + /// # Note + /// + /// For better performance, prefer using a [Transaction](crate::Transaction) when reading multiple + /// tensors at once. This may improve laziness, especially if executed on a different + /// thread in native environments. + pub fn try_into_data(self) -> Result { + crate::try_read_sync(self.into_data_async()).expect( + "Failed to read tensor data synchronously. + This can happen on platforms that don't support blocking futures like WASM. + If possible, try using into_data_async instead.", + ) + } + + /// Converts the data of the current tensor. + /// + /// # Note + /// + /// For better performance, prefer using a [Transaction](crate::Transaction) when reading multiple + /// tensors at once. This may improve laziness, especially if executed on a different + /// thread in native environments. + pub fn to_data(&self) -> TensorData { + self.clone().into_data() + } + + /// Returns the data of the current tensor. + pub async fn into_data_async(self) -> Result { + K::into_data_async(self.primitive).await + } + + /// Returns the data of the current tensor. + pub async fn to_data_async(&self) -> Result { + self.clone().into_data_async().await + } + + /// Create a tensor from the given data on the given device. + pub fn from_data(data: T, device: &B::Device) -> Self + where + T: Into, + { + let data = data.into(); + check!(TensorCheck::creation_ops::( + "From Data", + data.shape.as_slice() + )); + Self::new(K::from_data(data, device)) + } + + /// Create a tensor from the given data on the given device enforcing the given data type. + pub fn from_data_dtype(data: T, device: &B::Device, dtype: DType) -> Self + where + T: Into, + { + let data = data.into(); + check!(TensorCheck::creation_ops::( + "From Data", + data.shape.as_slice() + )); + Self::new(K::from_data_dtype(data, device, dtype)) + } + + /// Repeat the tensor along the given dimension. + /// + /// The output tensor has the same shape, except along the given dimension. + /// + /// # Arguments + /// - `dim`: The dimension to repeat. + /// - `times`: The number of times to repeat the tensor along the given dimension in the new tensor. + /// + /// # Returns + /// + /// A new tensor with the given dimension repeated `times` times. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [3, 2] + /// let tensor = Tensor::::from_data([[3.0, 4.9], [2.0, 1.9], [4.0, 5.9]], &device); + /// + /// // Repeat the tensor along the dimension 0 twice. + /// // [[3.0, 4.9], [2.0, 1.9], [4.0, 5.9], [3.0, 4.9], [2.0, 1.9], [4.0, 5.9]] + /// // The resulting tensor will have dimensions [6, 2]. + /// let repeated = tensor.repeat_dim(0, 2); + /// println!("{repeated}"); + /// } + /// ``` + pub fn repeat_dim(self, dim: usize, times: usize) -> Self { + if times > 0 { + Self::new(K::repeat_dim(self.primitive, dim, times)) + } else { + let shape = self.shape().repeat(dim, times).unwrap(); + Self::empty(shape, &self.device()) + } + } + + /// Repeat the tensor along the given dimensions. + /// # Arguments + /// - `sizes`: Borrowed slice of the number of times to repeat each dimension. + /// + /// # Returns + /// + /// A new tensor with the given dimensions repeated `times` times. + /// + /// # Panics + /// + /// If `sizes` contains more elements than the number of dimensions. + /// + /// # Example + /// + /// ```rust + /// + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [3, 2] + /// let tensor = Tensor::::from_data([[3.0, 4.9], [2.0, 1.9], [4.0, 5.9]], &device); + /// + /// // Repeat the tensor along the dimension 0 twice and the dimension 0 once. + /// // [[3.0, 4.9], [2.0, 1.9], [4.0, 5.9], [3.0, 4.9], [2.0, 1.9], [4.0, 5.9]] + /// // The resulting tensor will have dimensions [6, 2]. + /// let repeated = tensor.repeat(&[2, 1]); + /// } + /// ``` + pub fn repeat(self, sizes: &[usize]) -> Self { + if sizes.contains(&0) { + let mut shape = self.shape(); + for (dim, ×) in sizes.iter().enumerate() { + shape = shape.repeat(dim, times).unwrap(); + } + + return Self::empty(shape, &self.device()); + } + + let mut tensor = self; + for (dim, ×) in sizes.iter().enumerate() { + if times > 1 { + tensor = tensor.repeat_dim(dim, times); + } + } + tensor + } + + /// Applies element-wise equal comparison. + /// + /// # Returns + /// A boolean tensor that is `true` where input is equal to `other` and `false` elsewhere. + /// + /// # Panics + /// + /// If the two tensors don't have the same shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let t1 = Tensor::::from_data([[2.0, 4.9], [2.0, 1.9], [4.0, 5.9]], &device); + /// let t2 = Tensor::::from_data([[3.0, 4.9], [2.0, 1.9], [4.0, 5.9]], &device); + /// // Compare the elements of the two 2D tensors with dimensions [3, 2]. + /// // [[false, true], [true, true], [true, true]] + /// let equal = t1.equal(t2); + /// println!("{equal}"); + /// } + /// ``` + pub fn equal(self, other: Self) -> Tensor { + check!(TensorCheck::binary_ops_ew("Equal", &self, &other)); + Tensor::new(K::equal(self.primitive, other.primitive)) + } + + /// Applies element-wise non-equality comparison. + /// + /// # Returns + /// A boolean tensor that is `true` where input is not equal to `other` and `false` elsewhere. + /// + /// # Panics + /// + /// If the two tensors don't have the same shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let t1 = Tensor::::from_data([[2.0, 4.9], [2.0, 1.9], [4.0, 5.9]], &device); + /// let t2 = Tensor::::from_data([[3.0, 4.9], [2.0, 1.9], [4.0, 5.9]], &device); + /// // Compare the elements of the two 2D tensors for inequality. + /// // [[true, false], [false, false], [false, false]] + /// let not_equal = t1.not_equal(t2); + /// println!("{not_equal}"); + /// } + /// ``` + pub fn not_equal(self, other: Self) -> Tensor { + check!(TensorCheck::binary_ops_ew("NotEqual", &self, &other)); + Tensor::new(K::not_equal(self.primitive, other.primitive)) + } + + /// Applies element wise equal comparison and returns a boolean tensor. + /// + /// # Arguments + /// + /// * `other` - The element to compare. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.equal_elem(3.0); + /// println!("{tensor}"); + /// // [[false, false, true], [false, false, false]] + /// } + /// ``` + pub fn equal_elem(self, other: E) -> Tensor { + let other = Scalar::new(other, &self.dtype()); + Tensor::new(K::equal_elem(self.primitive, other)) + } + + /// Applies element wise non-equality comparison and returns a boolean tensor. + /// + /// # Arguments + /// + /// * `other` - The element to compare. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.not_equal_elem(3.0); + /// println!("{tensor}"); + /// // [[true, true, false], [true, true, true]] + /// } + /// ``` + pub fn not_equal_elem(self, other: E) -> Tensor { + let other = Scalar::new(other, &self.dtype()); + Tensor::new(K::not_equal_elem(self.primitive, other)) + } + + /// Concatenates all tensors into a new one along the given dimension. + /// + /// # Panics + /// + /// - If `dim` is higher than the rank. + /// - If `tensors` is an empty vector. + /// - If all tensors don't have the same shape (the dimension `dim` is ignored). + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let t1 = Tensor::::from_data([[3.0, 4.9, 2.0, 1.0], [2.0, 1.9, 3.0, 1.0]], &device); + /// let t2 = Tensor::::from_data([[4.0, 5.9, 8.0], [1.4, 5.8, 6.0]], &device); + /// + /// // Concatenate the two tensors with shapes [2, 4] and [2, 3] along the dimension 1. + /// // [[3.0, 4.9, 2.0, 1.0, 4.0, 5.9, 8.0], [2.0, 1.9, 3.0, 1.0, 1.4, 5.8, 6.0]] + /// // The resulting tensor will have shape [2, 7]. + /// let concat = Tensor::cat(vec![t1, t2], 1); + /// println!("{concat}"); + /// } + /// ``` + pub fn cat(tensors: Vec, dim: usize) -> Self { + check!(TensorCheck::cat(&tensors, dim)); + + // Filter out tensors with size 0 along the concatenation dimension. + // Empty tensors don't contribute to the output and would cause issues + // in backend implementations (e.g., division by zero in slice_assign). + // Safety: TensorCheck::cat ensures tensors is non-empty + let first_tensor = tensors.first().unwrap(); + let device = first_tensor.device(); + let mut shape = first_tensor.shape(); + + let non_empty_primitives: Vec<_> = tensors + .into_iter() + .filter(|t| t.shape()[dim] > 0) + .map(|t| t.primitive) + .collect(); + + // If all tensors were empty, return an empty tensor with size 0 on concat dim + if non_empty_primitives.is_empty() { + shape[dim] = 0; + return Self::empty(shape, &device); + } + + Self::new(K::cat(non_empty_primitives, dim)) + } + + /// Concatenates all tensors into a new one along a new dimension. + /// + /// # Panics + /// + /// - If all tensors don't have the same shape. + /// - If given dimension is not with range of 0..D2 + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let t1 = Tensor::::from_data([[3.0, 4.9, 2.0], [2.0, 1.9, 3.0]], &device); + /// let t2 = Tensor::::from_data([[4.0, 5.9, 8.0], [1.4, 5.8, 6.0]], &device); + /// let t3 = Tensor::::from_data([[4.0, 5.9, 8.0], [1.4, 5.8, 6.0]], &device); + /// + /// // Concatenate the three tensors with shape [2, 3] along a new dimension, 0. + /// // [[[3.0, 4.9, 2.0], [2.0, 1.9, 3.0]], + /// // [[4.0, 5.9, 8.0], [1.4, 5.8, 6.0]], + /// // [[4.0, 5.9, 8.0], [1.4, 5.8, 6.0]]] + /// // The resulting tensor will have shape [3, 2, 3]. + /// let stacked= Tensor::stack::<3>(vec![t1, t2, t3], 0); + /// println!("{stacked}"); + /// } + /// ``` + pub fn stack(tensors: Vec>, dim: usize) -> Tensor { + check!(TensorCheck::stack::(&tensors, dim)); + let tensors = tensors.into_iter().map(|t| t.unsqueeze_dim(dim)).collect(); + Tensor::::cat(tensors, dim) + } + + /// Iterate over slices of tensors alongside a given dimension. + /// + /// # Panics + /// + /// If given dimension is greater than or equal to tensor rank. + /// + /// # Returns + /// + /// A tensor iterator. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_data([[3.0, 4.9, 2.0], [2.0, 1.9, 3.0]], &device); + /// // Given a 2D tensor with dimensions [2, 3], iterate over slices of tensors along the dimension 0. + /// let iter = tensor.iter_dim(0); + /// for (i,tensor) in iter.enumerate() { + /// println!("Tensor {}: {}", i, tensor); + /// // Tensor 0: Tensor { data: [[3.0, 4.9, 2.0]], ... } + /// // Tensor 1: Tensor { data: [[2.0, 1.9, 3.0]], ... } + /// } + /// } + /// ``` + pub fn iter_dim(self, dim: usize) -> DimIter { + check!(TensorCheck::dim_ops::("iter_dim", dim)); + DimIter::new(self, dim) + } + + /// Returns a new tensor with the given dimension narrowed to the given range. + /// + /// # Panics + /// + /// - If the dimension is greater than the number of dimensions of the tensor. + /// - If the given range exceeds the number of elements on the given dimension. + /// + /// # Returns + /// + /// A new tensor with the given dimension narrowed to the given range. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [4, 3] + /// let tensor = Tensor::::from_data( + /// [ + /// [3.0, 4.9, 2.0], + /// [2.0, 1.9, 3.0], + /// [6.0, 1.5, 7.0], + /// [3.0, 4.9, 9.0], + /// ], + /// &device, + /// ); + /// // Narrow the tensor along the dimension 0, keeping 3 elements starting from index 1. + /// // [[2.0, 1.9, 3.0], [6.0, 1.5, 7.0], [3.0, 4.9, 9.0]] + /// // The resulting tensor will have dimensions [3, 3]. + /// let narrowed = tensor.narrow(0, 1, 3); + /// println!("{narrowed}"); + /// } + /// ``` + pub fn narrow(self, dim: usize, start: usize, length: usize) -> Self { + check!(TensorCheck::dim_ops::("narrow", dim)); + check!(TensorCheck::narrow(&self, dim, start, length)); + let dims = self.dims(); + + let ranges: [Range; D] = dims + .iter() + .enumerate() + .map(|(i, d)| { + if i == dim { + start..(start + length) + } else { + 0..*d + } + }) + .collect::>() + .try_into() + .unwrap(); + + Self::slice(self, ranges) + } + + /// Attempts to split the tensor into a specified number of chunks along a given dimension. + /// May return less chunks than requested if the tensor size is not divisible by the number of chunks. + /// + /// When the given dimension is evenly divisible by the number of chunks, the chunks will be of equal size. + /// Otherwise all chunks will be of equal size except for the last one. + /// + /// # Panics + /// + /// If the dimension is greater than the number of dimensions of the tensor. + /// + /// # Returns + /// A vector of tensors. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [4, 3] + /// let tensor = Tensor::::from_data( + /// [ + /// [3.0, 4.9, 2.0], + /// [2.0, 1.9, 3.0], + /// [6.0, 1.5, 7.0], + /// [3.0, 4.9, 9.0], + /// ], + /// &device, + /// ); + /// // Split the tensor along the dimension 1 into 2 chunks. + /// // The first chuck will have shape [4, 2]: + /// // [[3.0, 4.9], [2.0, 1.9], [6.0, 1.5], [3.0, 4.9]] + /// // The second chunk will have shape [4, 1]: + /// // [[2.0], [3.0], [7.0], [9.0]] + /// let chunks = tensor.chunk(2, 1); + /// println!("{chunks:?}"); + /// } + /// ``` + pub fn chunk(self, chunks: usize, dim: usize) -> Vec { + check!(TensorCheck::dim_ops::("chunk", dim)); + let size = self.shape()[dim]; + if size < chunks { + return (0..size) + .map(|i| Self::narrow(self.clone(), dim, i, 1)) + .collect(); + } + + let mut tensors = Vec::with_capacity(chunks); + let mut sum_chunk_size = 0; + if size.is_multiple_of(chunks) { + let chunk_size = size / chunks; + for _ in 0..chunks { + tensors.push(Self::narrow(self.clone(), dim, sum_chunk_size, chunk_size)); + sum_chunk_size += chunk_size; + } + } else { + let chunk_size = (size / chunks) + 1; // assumes not divisible + for _ in 0..chunks - 1 { + tensors.push(Self::narrow(self.clone(), dim, sum_chunk_size, chunk_size)); + sum_chunk_size += chunk_size; + } + let remainder = size % chunk_size; + tensors.push(Self::narrow(self.clone(), dim, sum_chunk_size, remainder)); + } + + tensors + } + + /// Splits the tensor into chunks of a specified size along a given dimension. + /// Each chunk is a view of the original tensor. + /// + /// If the tensor size along the given dimension is not divisible by `split_size`, + /// then the last chunk will be smaller. + /// + /// # Panics + /// + /// If the specified dimension to split along is greater than the number of dimensions of the tensor. + /// + /// # Returns + /// + /// A vector of tensors. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 1D tensor with 5 elements + /// let tensor = Tensor::::from_data([0.0, 1.0, 2.0, 3.0, 4.0], &device); + /// // Split the tensor into chunks of size 2 along dimension 0 + /// let chunks = tensor.split(2, 0); + /// // The result is a vector of tensors: + /// // [Tensor([0.0, 1.0]), Tensor([2.0, 3.0]), Tensor([4.0])] + /// println!("{:?}", chunks); + /// } + /// ``` + pub fn split(self, split_size: usize, dim: usize) -> Vec { + check!(TensorCheck::split::(&self.shape(), split_size, dim)); + let size = self.shape()[dim]; + let mut tensors = Vec::new(); + + let mut start = 0; + while start < size { + let length = usize::min(split_size, size - start); + tensors.push(Self::narrow(self.clone(), dim, start, length)); + start += length; + } + + tensors + } + + /// Splits the tensor into chunks with the specified sizes along a given dimension. + /// Each chunk is a view of the original tensor. + /// + /// The sizes of the chunks are specified in the `split_sizes` vector. The sum of the sizes + /// in `split_sizes` must equal the size of the tensor along the specified dimension. + /// + /// # Panics + /// + /// If the specified dimension to split along is greater than the number of dimensions of the tensor or + /// if the sum of `dim_sizes` does not equal the size of the tensor along `dim`. + /// + /// # Returns + /// + /// A vector of tensors. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 1D tensor with 5 elements + /// let tensor = Tensor::::from_data([0.0, 1.0, 2.0, 3.0, 4.0], &device); + /// // Split the tensor into chunks with sizes [2, 3] along dimension 0 + /// let chunks = tensor.split_with_sizes(vec![2, 3], 0); + /// // The result is a vector of tensors: + /// // [Tensor([0.0, 1.0]), Tensor([2.0, 3.0, 4.0])] + /// println!("{:?}", chunks); + /// } + /// ``` + pub fn split_with_sizes(self, split_sizes: Vec, dim: usize) -> Vec { + check!(TensorCheck::split_with_sizes::( + &self.shape(), + &split_sizes, + dim + )); + let mut tensors = Vec::new(); + + let mut start = 0; + for length in split_sizes { + if length == 0 { + continue; + } + tensors.push(Self::narrow(self.clone(), dim, start, length)); + start += length; + } + + tensors + } + + /// Tests if any element in the `tensor` evaluates to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. All input tensor types (Float, Int, Bool) are supported. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` containing a single element, True if any element in the input tensor + /// evaluates to True, False otherwise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_data([[true,false,true],[false,true,false]], &device); + /// let tensor_two = Tensor::::from_data([[false,false,false],[false,false,false]], &device); + /// + /// // Given a 2D tensor with dimensions [2, 3], test if any element in the tensor evaluates to True. + /// let any_tensor = tensor.any(); + /// println!("{}", any_tensor); + /// // Tensor { data: [true], ... } + /// + /// // Given a 2D tensor with dimensions [2, 3], test if any element in the tensor evaluates to True. + /// let any_tensor_two = tensor_two.any(); + /// println!("{}", any_tensor_two); + /// // Tensor { data: [false], ... } + /// } + /// ``` + pub fn any(self) -> Tensor { + Tensor::new(K::any(self.primitive)) + } + + /// Tests if any element in the `tensor` evaluates to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. All input tensor types (Float, Int, Bool) are supported. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same shape as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if any element along this dim in the input + /// evaluates to True, False otherwise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = + /// Tensor::::from_data([[true, false, false], [false, true, false]], &device); + /// // Check if any element in the tensor evaluates to True along the dimension 1. + /// // [[true], [true]], + /// let any_dim = tensor.clone().any_dim(1); + /// println!("{any_dim}"); + /// } + /// ``` + pub fn any_dim(self, dim: usize) -> Tensor { + Tensor::new(K::any_dim(self.primitive, dim)) + } + + /// Tests if all elements in the `tensor` evaluate to True. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. All input tensor types (Float, Int, Bool) are supported. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with a single element, True if all elements in the input tensor + /// evaluate to True, False otherwise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = + /// Tensor::::from_data([[true, false, true], [true, true, true]], &device); + /// // Check if all elements in the tensor evaluate to True (which is not the case). + /// // [false] + /// let all = tensor.all(); + /// println!("{all}"); + /// } + /// ``` + pub fn all(self) -> Tensor { + Tensor::new(K::all(self.primitive)) + } + + /// Tests if all elements in the `tensor` evaluate to True along a given dimension `dim`. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to test. All input tensor types (Float, Int, Bool) are supported. + /// * `dim` - The axis along which to test. + /// + /// # Returns + /// + /// A boolean tensor `Tensor` with the same shape as input `tensor`, except in the `dim` axis + /// where the size is 1. The elem in the `dim` axis is True if all elements along this dim in the input + /// evaluates to True, False otherwise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = + /// Tensor::::from_data([[true, true, false], [true, true, true]], &device); + /// // Check if all elements in the tensor evaluate to True along the dimension 1. + /// // [[true, true, false]] + /// let all_dim = tensor.clone().all_dim(0); + /// println!("{all_dim}"); + /// } + /// ``` + pub fn all_dim(self, dim: usize) -> Tensor { + Tensor::new(K::all_dim(self.primitive, dim)) + } + + /// Convert the tensor into a scalar. + /// + /// # Panics + /// + /// - If the tensor doesn't have one element. + /// - If the backend fails to read the tensor data synchronously. + /// + /// # Returns + /// + /// The scalar value of the tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_data([[3.0]], &device); + /// // Convert the tensor with a single element into a scalar. + /// let scalar = tensor.into_scalar(); + /// println!("{scalar}"); + /// } + /// ``` + pub fn into_scalar(self) -> K::Elem { + crate::try_read_sync(self.into_scalar_async()) + .expect( + "Failed to read tensor data synchronously. This can happen on platforms + that don't support blocking futures like WASM. Try into_scalar_async instead.", + ) + .expect("Error while reading data: use `try_into_scalar` instead to catch the error at runtime") + } + + /// Convert the tensor into a scalar and returns any error that might have occurred since the + /// last time the device was synchronized. + /// + /// # Panics + /// + /// - If the tensor doesn't have one element. + /// - If the backend fails to read the tensor data synchronously. + /// + /// # Returns + /// + /// The scalar value of the tensor. + pub fn try_into_scalar(self) -> Result { + crate::try_read_sync(self.into_scalar_async()).expect( + "Failed to read tensor data synchronously. This can happen on platforms + that don't support blocking futures like WASM. Try into_scalar_async instead.", + ) + } + + /// Convert the tensor into a scalar. + /// + /// # Panics + /// + /// If the tensor doesn't have one element. + pub async fn into_scalar_async(self) -> Result { + check!(TensorCheck::into_scalar::(&self.shape())); + + Ok(self.into_data_async().await?.iter().next().unwrap()) + } + + /// Broadcast the tensor to the given shape. + /// + /// Only singleton dimensions can be expanded to a larger size. Other dimensions must have the same size + /// (which can be inferred with `-1`). + /// + /// # Arguments + /// + /// * `shape` - The shape to broadcast the tensor to. + /// Can contain -1 for dimensions that should be inferred. + /// The number of elements in the shape must be greater or equal as + /// the number of dimensions of the tensor. + /// + /// # Panics + /// + /// If the tensor cannot be broadcasted to the given shape. + /// + /// # Returns + /// + /// A new tensor with the given shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// // Create a 2D tensor with dimensions [3, 1] + /// let tensor = Tensor::::from_data([[1.], [2.], [3.]], &device); + /// // Expand the tensor to a new shape [3, 4] + /// // [[1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0], [3.0, 3.0, 3.0, 3.0]] + /// let expanded = tensor.expand([3, 4]); + /// println!("{}", expanded); + /// } + /// ``` + pub fn expand>(self, shape: S) -> Tensor { + let shape = shape.into_shape(&self.shape()); + check!(TensorCheck::expand::( + "expand", + &self.shape(), + &shape, + )); + + Tensor::::new(K::expand(self.primitive, shape)) + } + + /// Unfold windows along a dimension. + /// + /// Returns a view of the tensor with all complete windows of size `size` in dimension `dim`; + /// where windows are advanced by `step` at each index. + /// + /// The number of windows is `max(0, (shape[dim] - size).ceil_div(step))`. + /// + /// The new view will have the unfolded dimension replaced by two dimensions; + /// one in the position of the original dimension, with size equal to the number of windows, + /// and one appended to the right-most position, with size equal to `size`. + /// + /// # Warning + /// + /// For the `ndarray` backend; this is not a view but a copy + /// with duplicated data. + /// + /// # Arguments + /// + /// * `dim` - the dimension to unfold. + /// * `size` - the size of each unfolded window. + /// * `step` - the step between each window. + /// + /// # Returns + /// + /// A tensor view with the shape ``[pre=..., windows, post=..., size]``. + pub fn unfold( + self, + dim: I, + size: usize, + step: usize, + ) -> Tensor { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::unfold::( + "unfold", + &self.shape(), + dim, + size, + step, + )); + Tensor::::new(K::unfold(self.primitive, dim, size, step)) + } +} + +/// Iterator given by (Tensor::iter_dim). +pub struct DimIter +where + B: Backend, + K: BasicOps, +{ + start: usize, + end: usize, + dim: usize, + ranges: [Range; D], + tensor: Tensor, +} + +impl> Iterator for DimIter { + type Item = Tensor; + + fn next(&mut self) -> Option { + if self.start >= self.end { + return None; + } + + let mut ranges = self.ranges.clone(); + ranges[self.dim] = self.start..(self.start + 1); + + let slice = self.tensor.clone().slice(ranges); + self.start += 1; + + Some(slice) + } +} + +impl> DoubleEndedIterator for DimIter { + fn next_back(&mut self) -> Option { + if self.start >= self.end { + return None; + } + + let mut ranges = self.ranges.clone(); + ranges[self.dim] = (self.end - 1)..self.end; + + let slice = self.tensor.clone().slice(ranges); + self.end = self.end.saturating_sub(1); + + Some(slice) + } +} + +impl> DimIter { + fn new(tensor: Tensor, dim: usize) -> Self { + let dims = tensor.dims(); + let ranges = dims + .iter() + .map(|&dim| 0..dim) + .collect::>>(); + let ranges: [Range; D] = ranges.try_into().unwrap(); + Self { + end: dims[dim], + ranges, + start: 0, + dim, + tensor, + } + } +} + +impl Tensor +where + B: Backend, + K: BasicOps, + >::Elem: Debug, +{ + #[inline] + fn push_newline_indent(acc: &mut String, indent: usize) { + acc.push('\n'); + for _ in 0..indent { + acc.push(' '); + } + } + fn fmt_inner_tensor( + &self, + acc: &mut String, + depth: usize, + multi_index: &mut [usize], + range: (usize, usize), + precision: Option, + ) { + let (start, end) = range; + for i in start..end { + if i > 0 { + acc.push_str(", "); + } + multi_index[depth] = i; + let range: [Range; D] = + core::array::from_fn(|i| multi_index[i]..multi_index[i] + 1); + + let data = burn_std::reader::try_read_sync(self.clone().slice(range).into_data_async()); + + if let Some(Ok(data)) = data { + let elem = data.iter::<>::Elem>().next().unwrap(); + match (precision, K::name()) { + (Some(p), "Float") => acc.push_str(&format!("{elem:.p$}")), + (_, "Bool") => acc.push_str(&format!("{}", elem.to_bool())), + _ => acc.push_str(&format!("{elem:?}")), + } + } else { + acc.push_str(""); + } + } + } + + fn fmt_outer_tensor( + &self, + acc: &mut String, + depth: usize, + multi_index: &mut [usize], + print_options: &PrintOptions, + summarize: bool, + range: (usize, usize), + ) { + let (start, end) = range; + for i in start..end { + if i > start { + acc.push(','); + Self::push_newline_indent(acc, depth + 1); + } + acc.push('['); + multi_index[depth] = i; + self.display_recursive(acc, depth + 1, multi_index, print_options, summarize); + acc.push(']'); + } + } + + /// Recursively formats the tensor data for display and appends it to the provided accumulator string. + /// + /// This function is designed to work with tensors of any dimensionality. + /// It traverses the tensor dimensions recursively, converting the elements + /// to strings and appending them to the accumulator string with the + /// appropriate formatting. + /// + /// # Arguments + /// + /// * `acc` - A mutable reference to a `String` used as an accumulator for the formatted output. + /// * `depth` - The current depth of the tensor dimensions being processed. + /// * `multi_index` - A mutable slice of `usize` representing the current indices in each dimension. + fn display_recursive( + &self, + acc: &mut String, + depth: usize, + multi_index: &mut [usize], + print_options: &PrintOptions, + summarize: bool, + ) { + let edge_items = print_options.edge_items; + + if depth == 0 { + acc.push('['); + } + + if depth == self.dims().len() - 1 { + // if we are at the innermost dimension, just push its elements into the accumulator + if summarize && self.dims()[depth] > 2 * edge_items { + // print the starting `edge_items` elements + self.fmt_inner_tensor( + acc, + depth, + multi_index, + (0, edge_items), + print_options.precision, + ); + acc.push_str(", ..."); + // print the last `edge_items` elements + self.fmt_inner_tensor( + acc, + depth, + multi_index, + (self.dims()[depth] - edge_items, self.dims()[depth]), + print_options.precision, + ); + } else { + // print all the elements + self.fmt_inner_tensor( + acc, + depth, + multi_index, + (0, self.dims()[depth]), + print_options.precision, + ); + } + } else { + // otherwise, iterate through the current dimension and recursively display the inner tensors + if summarize && self.dims()[depth] > 2 * edge_items { + self.fmt_outer_tensor( + acc, + depth, + multi_index, + print_options, + summarize, + (0, edge_items), + ); + + acc.push(','); + Self::push_newline_indent(acc, depth + 1); + acc.push_str("..."); + Self::push_newline_indent(acc, depth + 1); + + self.fmt_outer_tensor( + acc, + depth, + multi_index, + print_options, + summarize, + (self.dims()[depth] - edge_items, self.dims()[depth]), + ); + } else { + self.fmt_outer_tensor( + acc, + depth, + multi_index, + print_options, + summarize, + (0, self.dims()[depth]), + ); + } + } + + if depth == 0 { + acc.push(']'); + } + } +} + +#[derive(Clone, Debug)] +/// Options for Tensor pretty printing +pub struct PrintOptions { + /// number of elements to start summarizing tensor + pub threshold: usize, + + /// number of starting elements and ending elements to display + pub edge_items: usize, + + /// Precision for floating point numbers + pub precision: Option, +} + +static PRINT_OPTS: RwLock = RwLock::new(PrintOptions::const_default()); + +impl PrintOptions { + /// Print options with default values + pub const fn const_default() -> Self { + Self { + threshold: 1000, + edge_items: 3, + precision: None, + } + } +} + +impl Default for PrintOptions { + fn default() -> Self { + Self::const_default() + } +} + +/// Set print options +pub fn set_print_options(options: PrintOptions) { + let mut print_opts = PRINT_OPTS.write().unwrap(); + *print_opts = options; +} + +/// Pretty print tensors +impl core::fmt::Display for Tensor +where + B: Backend, + B::IntElem: core::fmt::Display, + K: BasicOps, + >::Elem: Debug, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + writeln!(f, "Tensor {{")?; + + { + // Do not lock the mutex for the whole function + let mut po = { PRINT_OPTS.read().unwrap().clone() }; + + // Override the precision if it is set from the formatter + // This will be possible when the tensor is printed using the `{:.*}` syntax + if let Some(precision) = f.precision() { + po.precision = Some(precision); + } + + let mut acc = String::new(); + let mut multi_index = vec![0; D]; + let summarize = self.shape().num_elements() > po.threshold; + + self.display_recursive(&mut acc, 0, &mut multi_index, &po, summarize); + + writeln!(f, " data:")?; + write!(f, "{acc}")?; + writeln!(f, ",")?; + } + + writeln!(f, " shape: {:?},", self.dims())?; + writeln!(f, " device: {:?},", self.device())?; + writeln!(f, " backend: {:?},", B::name(&self.device()))?; + writeln!(f, " kind: {:?},", K::name())?; + + let dtype = self.primitive.dtype(); + + writeln!(f, " dtype: {:?},", dtype.name())?; + write!(f, "}}") + } +} + +/// Trait used for movedim arguments +pub trait MovedimArgs { + /// Converts into a set of dimensions `Vec` for the `tensor.movedim()` function + fn into_dim_vec(self) -> Vec; +} + +impl MovedimArgs for Vec { + fn into_dim_vec(self) -> Vec { + let set = self + .iter() + .map(|&dim| { + if dim < 0 { + (D as i32 + dim) as usize + } else { + dim as usize + } + }) + .collect::>(); + check!(TensorCheck::movedim_args_vec::(&set)); + + set + } +} + +impl MovedimArgs for Vec { + fn into_dim_vec(self) -> Vec { + check!(TensorCheck::movedim_args_vec::(&self)); + self + } +} + +impl MovedimArgs for usize { + #[allow(clippy::vec_init_then_push)] + fn into_dim_vec(self) -> Vec { + check!(TensorCheck::movedim_args_usize::(self)); + + let mut set = Vec::with_capacity(1); + set.push(self); + + set + } +} + +impl MovedimArgs for i32 { + #[allow(clippy::vec_init_then_push)] + fn into_dim_vec(self) -> Vec { + check!(TensorCheck::movedim_args_i32::(self)); + + let dim = if self < 0 { + (D as i32 + self) as usize + } else { + self as usize + }; + + let mut set = Vec::with_capacity(1); + set.push(dim); + + set + } +} + +/// Trait used for reshape arguments. +pub trait ReshapeArgs: Debug { + /// Converts to a shape. + fn into_shape(self, source: Shape) -> Shape; +} + +impl ReshapeArgs for [I; D2] { + fn into_shape(self, source: Shape) -> Shape { + unwrap_shape_reshape(source.reshape(self)) + } +} + +impl ReshapeArgs for Shape { + fn into_shape(self, source: Shape) -> Shape { + unwrap_shape_reshape(source.reshape(self)) + } +} + +/// Trait used for broadcast arguments. +pub trait BroadcastArgs { + /// Converts to a shape. + fn into_shape(self, shape: &Shape) -> Shape; +} + +impl BroadcastArgs for Shape { + fn into_shape(self, _shape: &Shape) -> Shape { + self + } +} + +impl BroadcastArgs for [E; D2] { + // Passing -1 as the size for a dimension means not changing the size of that dimension. + fn into_shape(self, shape: &Shape) -> Shape { + if self.len() < shape.num_dims() { + panic!("Broadcast arguments must be greater than the number of dimensions"); + } + + // Zip the two shapes in reverse order and replace -1 with the actual dimension value. + let new_shape: Vec<_> = self + .iter() + .rev() + .map(|x| { + let primitive = x.as_index(); + if primitive < -1 || primitive == 0 { + panic!("Broadcast arguments must be positive or -1"); + } + primitive + }) + .zip(shape.iter().rev().chain(repeat(&0)).take(self.len())) // Pad the original shape with 0s + .map(|(x, &y)| if x == -1 { y } else { x as usize }) + .collect::>() + .into_iter() + .rev() + .collect(); + + if new_shape.contains(&0) { + panic!("Cannot substitute -1 for a non-existing dimension"); + } + + let new_shape: [usize; D2] = new_shape.try_into().unwrap(); + + Shape::from(new_shape) + } +} + +impl Serialize for Tensor +where + B: Backend, + K: BasicOps, + K::Elem: Debug + Copy + Serialize, +{ + fn serialize(&self, serializer: S) -> Result { + let data = self.to_data(); + data.serialize(serializer) + } +} + +impl<'de, B, const D: usize, K> Deserialize<'de> for Tensor +where + B: Backend, + K: BasicOps, + K::Elem: Debug + Copy + Deserialize<'de>, +{ + fn deserialize>(deserializer: De) -> Result { + let tensor = Tensor::from_data( + TensorData::deserialize(deserializer)?, + &::default(), + ); + Ok(tensor) + } +} + +#[cfg(test)] +mod tests { + use burn_std::SliceOps; + + use crate::{Shape, s}; + + #[test] + fn slice_range_single_dim_leading() { + let shape = Shape::new([8, 4]); + + // Half-open range + let slices = shape.clone().into_slices([0..5]); + assert_eq!(slices[0].to_range(8), 0..5); + let slices = shape.clone().into_slices([-3..-1]); + assert_eq!(slices[0].to_range(8), 5..7); + + // Inclusive range + let slices = shape.clone().into_slices([0..=4]); + assert_eq!(slices[0].to_range(8), 0..5); + let slices = shape.clone().into_slices([-2..=-1]); + assert_eq!(slices[0].to_range(8), 6..8); + + // Unbounded start + let slices = shape.clone().into_slices([..3]); + assert_eq!(slices[0].to_range(8), 0..3); + let slices = shape.clone().into_slices([..-5]); + assert_eq!(slices[0].to_range(8), 0..3); + + // Unbounded end + let slices = shape.clone().into_slices([5..]); + assert_eq!(slices[0].to_range(8), 5..8); + let slices = shape.clone().into_slices([-3..]); + assert_eq!(slices[0].to_range(8), 5..8); + + // Full range + let slices = shape.into_slices([..]); + assert_eq!(slices[0].to_range(8), 0..8); + } + + #[test] + fn test_negative_slice_indices() { + use crate::Slice; + + // Test negative indices conversion + let slice: Slice = (-3..-1).into(); + assert_eq!(slice.start, -3); + assert_eq!(slice.end, Some(-1)); + + // Test to_range conversion with size 8 + let range = slice.to_range(8); + assert_eq!(range, 5..7); + + // Test with shape slice + let shape = Shape::new([8, 4]); + let result = shape.clone().into_slices([-3..-1]); + assert_eq!(result[0].to_range(8), 5..7); + + // Test more negative index cases + let slice2: Slice = (-5..).into(); + assert_eq!(slice2.to_range(10), 5..10); + + let slice3: Slice = (..-2).into(); + assert_eq!(slice3.to_range(10), 0..8); + + // Test with s! macro - single dimension returns Slice directly + let slice4 = s![-3..-1]; + assert_eq!(slice4.start, -3); + assert_eq!(slice4.end, Some(-1)); + } + + #[test] + fn slice_range_multi_dim() { + let shape = Shape::new([8, 4]); + + // Multiple ways to provide ranges + let slices = shape.clone().into_slices([0..5, 0..4]); + assert_eq!(slices[0].to_range(8), 0..5); + assert_eq!(slices[1].to_range(4), 0..4); + + let slices = shape.clone().into_slices([0.., 0..]); + assert_eq!(slices[0].to_range(8), 0..8); + assert_eq!(slices[1].to_range(4), 0..4); + + let slices = shape.clone().into_slices([0..=7, 0..=3]); + assert_eq!(slices[0].to_range(8), 0..8); + assert_eq!(slices[1].to_range(4), 0..4); + + let slices = shape.clone().into_slices([0..5, 0..3]); + assert_eq!(slices[0].to_range(8), 0..5); + assert_eq!(slices[1].to_range(4), 0..3); + + let slices = shape.into_slices([0.., 0..]); + assert_eq!(slices[0].to_range(8), 0..8); + assert_eq!(slices[1].to_range(4), 0..4); + } + + #[test] + fn slice_range_multi_dim_index() { + let shape = Shape::new([8, 4]); + + // Indices (single integer) should also convert to correct range + let slices = shape.clone().into_slices([0, 2]); + assert_eq!(slices[0].to_range(8), 0..1); + assert_eq!(slices[1].to_range(4), 2..3); + + let slices = shape.into_slices([-1, -1]); + assert_eq!(slices[0].to_range(8), 7..8); + assert_eq!(slices[1].to_range(4), 3..4); + } + + #[test] + fn slice_range_multi_dim_heterogeneous() { + // Slice macro `s![]` can be used to provide different range types + let shape = Shape::new([8, 4, 2]); + let slice = s![0..5, .., -1]; + let slices = shape.into_slices(slice); + assert_eq!(slices[0].to_range(8), 0..5); + assert_eq!(slices[1].to_range(4), 0..4); + assert_eq!(slices[2].to_range(2), 1..2); + + let shape = Shape::new([8, 4, 2, 3]); + let slice = s![..=4, 0..=3, .., -2..]; + let slices = shape.into_slices(slice); + assert_eq!(slices[0].to_range(8), 0..5); + assert_eq!(slices[1].to_range(4), 0..4); + assert_eq!(slices[2].to_range(2), 0..2); + assert_eq!(slices[3].to_range(3), 1..3); + + let shape = Shape::new([3, 4]); + let slice = s![1..-1, ..]; + let slices = shape.into_slices(slice); + assert_eq!(slices[0].to_range(3), 1..2); + assert_eq!(slices[1].to_range(4), 0..4); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/bool.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/bool.rs new file mode 100644 index 0000000..9adb818 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/bool.rs @@ -0,0 +1,429 @@ +use crate::{Bool, Int, Shape, Tensor, TensorData, TensorPrimitive, backend::Backend}; +use alloc::{vec, vec::Vec}; + +use crate::try_read_sync; + +/// The part of the tensor to keep when creating a triangular mask. +enum TriPart { + /// Upper triangular part. + Upper, + + /// Lower triangular part. + Lower, + + /// Diagonal part. + Diagonal, +} + +impl Tensor +where + B: Backend, +{ + /// Create a boolean tensor from data on the given device. + /// + /// # Arguments + /// + /// * `data` - The tensor data. + /// * `device` - The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// A boolean tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_bool([[true, false], [false, true]].into(), &device); + /// println!("{tensor}"); + /// } + /// ``` + pub fn from_bool(data: TensorData, device: &B::Device) -> Self { + Self::new(B::bool_from_data(data.convert::(), device)) + } + + /// Convert the bool tensor into an int tensor. + /// + /// # Returns + /// + /// An integer tensor where `true` is converted to `1` and `false` to `0`. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let bool_tensor = Tensor::::from_bool([true, false, true].into(), &device); + /// let int_tensor = bool_tensor.int(); + /// println!("{int_tensor}"); // [1, 0, 1] + /// } + /// ``` + pub fn int(self) -> Tensor { + Tensor::new(B::bool_into_int(self.primitive)) + } + + /// Convert the bool tensor into a float tensor. + /// + /// # Returns + /// + /// A float tensor where `true` is converted to `1.0` and `false` to `0.0`. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let bool_tensor = Tensor::::from_bool([true, false, true].into(), &device); + /// let float_tensor = bool_tensor.float(); + /// println!("{float_tensor}"); // [1.0, 0.0, 1.0] + /// } + /// ``` + pub fn float(self) -> Tensor { + Tensor::new(TensorPrimitive::Float(B::bool_into_float(self.primitive))) + } + + /// Inverses boolean values. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_bool([[true, false], [false, true]].into(), &device); + /// let inverted = tensor.bool_not(); + /// println!("{inverted}"); // [[false, true], [true, false]] + /// } + /// ``` + pub fn bool_not(self) -> Self { + Tensor::new(B::bool_not(self.primitive)) + } + + /// Performs logical and (`&&`) on two boolean tensors. + /// + /// # Arguments + /// + /// * `rhs` - The right-hand side tensor for the AND operation. + /// + /// # Returns + /// + /// A boolean tensor where each element is the result of `self[i] && rhs[i]`. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let a = Tensor::::from_bool([[true, true], [false, false]].into(), &device); + /// let b = Tensor::::from_bool([[true, false], [true, false]].into(), &device); + /// let result = a.bool_and(b); + /// println!("{result}"); // [[true, false], [false, false]] + /// } + /// ``` + pub fn bool_and(self, rhs: Tensor) -> Tensor { + Tensor::new(B::bool_and(self.primitive, rhs.primitive)) + } + + /// Performs logical or (`||`) on two boolean tensors. + /// + /// # Arguments + /// + /// * `rhs` - The right-hand side tensor for the OR operation. + /// + /// # Returns + /// + /// A boolean tensor where each element is the result of `self[i] || rhs[i]`. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let a = Tensor::::from_bool([[true, true], [false, false]].into(), &device); + /// let b = Tensor::::from_bool([[true, false], [true, false]].into(), &device); + /// let result = a.bool_or(b); + /// println!("{result}"); // [[true, true], [true, false]] + /// } + /// ``` + pub fn bool_or(self, rhs: Tensor) -> Tensor { + Tensor::new(B::bool_or(self.primitive, rhs.primitive)) + } + + /// Performs logical xor (`^`) on two boolean tensors. + /// + /// # Arguments + /// + /// * `rhs` - The right-hand side tensor for the XOR operation. + /// + /// # Returns + /// + /// A boolean tensor where each element is the result of `self[i] ^ rhs[i]`. + /// Returns `true` when exactly one of the operands is `true`. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let a = Tensor::::from_bool([[true, true], [false, false]].into(), &device); + /// let b = Tensor::::from_bool([[true, false], [true, false]].into(), &device); + /// let result = a.bool_xor(b); + /// println!("{result}"); // [[false, true], [true, false]] + /// } + /// ``` + pub fn bool_xor(self, rhs: Tensor) -> Tensor { + Tensor::new(B::bool_xor(self.primitive, rhs.primitive)) + } + + /// Compute the indices of `true` elements in the tensor (i.e., non-zero for boolean tensors). + /// + /// # Returns + /// + /// A vector of tensors, one for each dimension of the given tensor, containing the indices of + /// the non-zero elements in that dimension. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_bool( + /// [[true, false, true], [false, true, false], [false, true, false]].into(), + /// &device, + /// ); + /// let indices = tensor.nonzero(); + /// println!("{}", indices[0]); // [0, 0, 1, 2] + /// println!("{}", indices[1]); // [0, 2, 1, 1] + /// } + /// ``` + pub fn nonzero(self) -> Vec> { + try_read_sync(self.nonzero_async()) + .expect("Failed to read tensor data synchronously. Try using nonzero_async instead.") + } + + /// Compute the indices of `true` elements in the tensor (i.e., non-zero for boolean tensors). + /// + /// # Returns + /// + /// A vector of tensors, one for each dimension of the given tensor, containing the indices of + /// the non-zero elements in that dimension. + pub async fn nonzero_async(self) -> Vec> { + let indices = self.argwhere_async().await; + + if indices.shape().num_elements() == 0 { + // Return empty vec when all elements are zero + return vec![]; + } + + let dims = indices.shape(); + indices + .chunk(dims[1], 1) + .into_iter() + .map(|t| t.reshape(Shape::new([dims[0]]))) + .collect() + } + + /// Compute the indices of the elements that are true, grouped by element. + /// + /// # Returns + /// + /// A tensor containing the indices of all non-zero elements of the given tensor. Each row in the + /// result contains the indices of a non-zero element. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_bool( + /// [[true, false, true], [false, true, false], [false, true, false]].into(), + /// &device, + /// ); + /// let indices = tensor.argwhere(); + /// println!("{indices}"); // [[0, 0], [0, 2], [1, 1], [2, 1]] + /// } + /// ``` + pub fn argwhere(self) -> Tensor { + try_read_sync(self.argwhere_async()) + .expect("Failed to read tensor data synchronously. Try using argwhere_async instead.") + } + + /// Compute the indices of the elements that are true, grouped by element. + /// + /// # Returns + /// + /// A tensor containing the indices of all non-zero elements of the given tensor. Each row in the + /// result contains the indices of a non-zero element. + pub async fn argwhere_async(self) -> Tensor { + Tensor::new(B::bool_argwhere(self.primitive).await) + } + + /// Creates a mask for the upper, lower triangle, or diagonal of a matrix, which can be used to + /// fill the specified area with a value. + fn tri_mask>( + shape: S, + tri_part: TriPart, + offset: i64, + device: &B::Device, + ) -> Self { + let shape: Shape = shape.into(); + let height = shape[D - 2]; + let width = shape[D - 1]; + + // Generate row and column index tensors. + let row_indices: Tensor = Tensor::arange(0..height as i64, device); + let col_indices: Tensor = Tensor::arange(0..width as i64, device); + + // Prepare shapes for broadcasting. + let mut row_shape = [1; D]; + row_shape[D - 2] = height; + let mut col_shape = [1; D]; + col_shape[D - 1] = width; + + // Reshape for broadcasting. + let row_broadcast: Tensor = row_indices.reshape(Shape::new(row_shape)); + let col_broadcast = col_indices.reshape(Shape::new(col_shape)); + + // Broadcasting trick to create a matrix that facilitates comparison for mask generation. + let matrix = row_broadcast.clone() - (col_broadcast.clone() - offset); + + // Select the appropriate comparison function based on `tri_part`. + let compare = match tri_part { + TriPart::Upper => Tensor::greater_elem, + TriPart::Lower => Tensor::lower_elem, + TriPart::Diagonal => Tensor::not_equal_elem, + }; + + // Generate and return the mask by applying the comparison to the matrix. + compare(matrix, 0).unsqueeze() + } + + /// Creates a mask for the upper triangle of a matrix, which can be used to fill the specified + /// area with a value. + /// + /// This function generates a boolean tensor representing the mask of the upper triangle of a matrix. + /// + /// # Arguments + /// + /// * `shape`: The shape of the matrix. + /// * `offset`: The offset from the diagonal, where 0 means the diagonal, and positive values shift + /// towards the upper triangle. + /// * `device`: The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// Returns a boolean tensor where `false` indicates the elements of the matrix that are part of the + /// upper triangle taking into account the specified `offset`. All other elements are `true`. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let mask = Tensor::::triu_mask([3, 3], 0, &Default::default()); + /// println!("{mask}"); + /// // [[false, false, false], + /// // [true, false, false], + /// // [true, true, false]] + /// } + /// ``` + pub fn triu_mask>(shape: S, offset: i64, device: &B::Device) -> Self { + Self::tri_mask(shape, TriPart::Upper, offset, device) + } + + /// Creates a mask for the lower triangle of a matrix, which can be used to fill the specified + /// area with a value. + /// + /// This function generates a boolean tensor representing the mask of the lower triangle of a matrix. + /// + /// # Arguments + /// + /// * `shape`: The shape of the matrix. + /// * `offset`: The offset from the diagonal, where 0 means the diagonal, and negative values shift + /// towards the lower triangle. + /// * `device`: The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// Returns a boolean tensor where `false` indicates the elements of the matrix that are part of the + /// lower triangle taking into account the specified `offset`. All other elements are `true`. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let mask = Tensor::::tril_mask([3, 3], 0, &Default::default()); + /// println!("{mask}"); + /// // [[false, true, true], + /// // [false, false, true], + /// // [false, false, false]] + /// } + /// ``` + pub fn tril_mask>(shape: S, offset: i64, device: &B::Device) -> Self { + Self::tri_mask(shape, TriPart::Lower, offset, device) + } + + /// Creates a mask for the diagonal of a matrix, which can be used to fill the specified + /// area with a value. + /// + /// This function generates a boolean tensor representing the mask of the diagonal of a matrix. + /// + /// # Arguments + /// + /// * `shape`: The shape of the matrix. + /// * `offset`: The offset from the diagonal, where 0 means the diagonal, and positive values shift + /// towards the upper triangle. + /// * `device`: The device on which the tensor will be allocated. + /// + /// # Returns + /// + /// Returns a boolean tensor where `false` indicates the elements of the matrix that are part of the + /// diagonal. All other elements are `true`. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool}; + /// + /// fn example() { + /// let mask = Tensor::::diag_mask([3, 3], 0, &Default::default()); + /// println!("{mask}"); + /// // [[false, true, true], + /// // [true, false, true], + /// // [true, true, false]] + /// } + /// ``` + pub fn diag_mask>(shape: S, offset: i64, device: &B::Device) -> Self { + Self::tri_mask(shape, TriPart::Diagonal, offset, device) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/cartesian_grid.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/cartesian_grid.rs new file mode 100644 index 0000000..90b850a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/cartesian_grid.rs @@ -0,0 +1,56 @@ +use crate::{Int, Shape, Tensor, backend::Backend}; +use alloc::vec::Vec; + +/// Generates a cartesian grid for the given tensor shape on the specified device. +/// The generated tensor is of dimension `D2 = D + 1`, where each element at dimension D contains the cartesian grid coordinates for that element. +/// +/// # Arguments +/// +/// * `shape` - The shape specifying the dimensions of the tensor. +/// * `device` - The device to create the tensor on. +/// +/// # Panics +/// +/// Panics if `D2` is not equal to `D+1`. +/// +/// # Examples +/// +/// ```rust +/// use burn_tensor::Int; +/// use burn_tensor::{backend::Backend, Shape, Tensor}; +/// fn example() { +/// let device = Default::default(); +/// let result: Tensor = Tensor::::cartesian_grid([2, 3], &device); +/// println!("{}", result); +/// } +/// ``` +pub fn cartesian_grid, const D: usize, const D2: usize>( + shape: S, + device: &B::Device, +) -> Tensor { + if D2 != D + 1 { + panic!("D2 must equal D + 1 for Tensor::cartesian_grid") + } + + let dims = shape.into(); + let mut indices: Vec> = Vec::new(); + + for dim in 0..D { + let dim_range: Tensor = Tensor::arange(0..dims[dim] as i64, device); + + let mut shape = [1; D]; + shape[dim] = dims[dim]; + let mut dim_range = dim_range.reshape(shape); + + for (i, &item) in dims.iter().enumerate() { + if i == dim { + continue; + } + dim_range = dim_range.repeat_dim(i, item); + } + + indices.push(dim_range); + } + + Tensor::stack::(indices, D) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/check.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/check.rs new file mode 100644 index 0000000..09bd9ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/check.rs @@ -0,0 +1,1551 @@ +use crate::ops::FloatElem; +use crate::{BasicOps, Shape, Slice, Tensor, backend::Backend, cast::ToElement}; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec; +use alloc::vec::Vec; +use burn_backend::tensor::Ordered; + +/// The struct should always be used with the [check](crate::check) macro. +/// +/// This is a simple pub(crate) data structure that efficiently checks tensor operations and +/// formats clear error messages. It's crucial that the checks are really fast, but it doesn't matter +/// when a failed check is discovered since the program will panic. +/// +/// # Notes +/// +/// Failing tensor checks will always result in a panic. +/// As mentioned in [The Rust Programming Language book](https://doc.rust-lang.org/book/ch09-03-to-panic-or-not-to-panic.html), +/// when there is no way to recover, panic should be used instead of a result. +/// +/// Most users will unwrap the results anyway, which will worsen the clarity of the code. Almost +/// all checks highlight programming errors, which means invalid programs that should be fixed. +/// Checks are not the ideal way to help users write correct programs, but they are still better +/// than backend errors. Other forms of compile-time validation could be developed, such as named +/// tensors, but we have to carefully evaluate the ease of use of the Tensor API. Adding overly +/// complex type validation checks might drastically worsen the API and result in harder-to-maintain +/// programs. +/// +/// # Design +/// +/// Maybe the Backend API should return a result for each operation, which would allow handling +/// all checks, even the ones that can't be efficiently checked before performing an operation, +/// such as the `index_select` operation. The downside of that approach is that all backend +/// implementation might re-implement the same checks, which may result in unnecessary code +/// duplication. Maybe a combination of both strategies could help to cover all use cases. +pub(crate) enum TensorCheck { + Ok, + Failed(FailedTensorCheck), +} + +impl TensorCheck { + /// Checks device and shape compatibility for element wise binary operations. + pub(crate) fn binary_ops_ew>( + ops: &str, + lhs: &Tensor, + rhs: &Tensor, + ) -> Self { + Self::Ok + .binary_ops_device(ops, &lhs.device(), &rhs.device()) + .binary_ops_ew_shape::(ops, &lhs.shape(), &rhs.shape()) + } + + pub(crate) fn into_scalar(shape: &Shape) -> Self { + let mut check = Self::Ok; + + if shape.num_elements() != 1 { + check = check.register( + "Into Scalar", + TensorError::new("Only tensors with 1 element can be converted into scalar.") + .details(format!( + "Current tensor has {} elements", + shape.num_elements() + )), + ); + } + + check + } + + pub(crate) fn dim_ops(ops: &str, dim: usize) -> Self { + let mut check = Self::Ok; + + if dim >= D { + check = check.register( + ops, + TensorError::new("Given dimension is higher than the tensor rank.") + .details(format!("Tensor rank: '{D}', given dimension: '{dim}'.")), + ); + } + + check + } + + pub(crate) fn creation_ops(ops: &str, dims: &[usize]) -> Self { + let mut check = Self::Ok; + + if D == 0 { + check = check.register( + ops, + TensorError::new("Tried to create a 0-dim tensor, which is invalid.") + .details(format!("Tensor rank: '{D}', given dimensions: '{dims:?}'.")), + ); + } + + if dims.len() != D { + check = check.register( + ops, + TensorError::new("Given dimensions differ from the tensor rank.") + .details(format!("Tensor rank: '{D}', given dimensions: '{dims:?}'.")), + ); + } + + check + } + + pub(crate) fn narrow>( + tensor: &Tensor, + dim: usize, + start: usize, + length: usize, + ) -> Self { + let mut check = Self::Ok; + + if length == 0 { + check = check.register( + "Narrow", + TensorError::new(format!( + "Can't narrow at dimension {dim}, length must be greater than 0", + )), + ); + } + + if start >= tensor.shape()[dim] { + check = check.register( + "Narrow", + TensorError::new(format!( + "Can't narrow at dimension {dim}, start exceeds the size of the tensor along \ + this dimension (Size={})", + tensor.shape()[dim] + )), + ); + } + + if start + length > tensor.shape()[dim] { + check = check.register( + "Narrow", + TensorError::new(format!( + "Can't narrow at dimension {dim}, start + length exceeds the size of the tensor \ + along this dimension (Size={})", + tensor.shape()[dim] + )), + ); + } + + check + } + + pub(crate) fn movedim_args_usize(dim: usize) -> Self { + let mut check = Self::Ok; + + if dim >= D { + check = check.register( + "Movedim", + TensorError::new( + "The given dimension exceeds the number of dimensions of the current tensor.", + ) + .details(format!( + "Current tensor has {D} dimensions, but the given dimension is {dim}.", + )), + ); + } + + check + } + + pub(crate) fn movedim_args_i32(dim: i32) -> Self { + let mut check = Self::Ok; + + if dim < -(D as i32) || dim >= D as i32 { + check = check.register( + "Movedim", + TensorError::new( + "The given dimension is out of bounds for the current tensor dimensions.", + ) + .details(format!( + "Current tensor has {D} dimensions, but the given dimension is {dim}.", + )), + ); + } + + check + } + + pub(crate) fn movedim_args_vec(dims: &Vec) -> Self { + let mut check = Self::Ok; + + // Check out of bounds + if dims.iter().any(|&x| x >= D) { + check = check.register( + "Movedim", + TensorError::new("The given dimensions are out of bounds.").details(format!( + "Current tensor has {D} dimensions, but the given dimensions are {dims:?}.", + )), + ); + } + + // Check there are no duplicates + for (i, &dim_i) in dims.iter().enumerate() { + for &dim_j in dims.iter().skip(i + 1) { + if dim_i == dim_j { + check = check.register( + "Movedim", + TensorError::new("The given dimensions contain duplicates.").details( + format!( + "The dimension {dim_i} is duplicated in the given dimensions {dims:?}.", + ), + ), + ); + } + } + } + + check + } + + pub(crate) fn movedim_args_length( + source_dims: &Vec, + destination_dims: &Vec, + ) -> Self { + let mut check = Self::Ok; + + if source_dims.len() != destination_dims.len() { + check = check.register( + "Movedim", + TensorError::new( + "The number of dimensions in source and destination must be equal.", + ) + .details(format!( + "Source dimensions: {source_dims:?}, Destination dimensions: {destination_dims:?}.", + )), + ) + } + + check + } + + pub(crate) fn flatten( + start_dim: usize, + end_dim: usize, + ) -> Self { + let mut check = Self::Ok; + + if start_dim > end_dim { + check = check.register( + "Flatten", + TensorError::new(format!( + "The start dim ({start_dim}) must be smaller than or equal to the end dim ({end_dim})" + )), + ); + } + + if D2 > D1 { + check = check.register( + "Flatten", + TensorError::new(format!( + "Result dim ({D2}) must be smaller than or equal to ({D1})" + )), + ); + } + + if D1 < end_dim + 1 { + check = check.register( + "Flatten", + TensorError::new(format!( + "The end dim ({end_dim}) must be smaller than the tensor dim ({D1})" + )), + ); + } + + if (D2 as i32) < (D1 as i32 - (end_dim as i32 - start_dim as i32)) { + check = check.register( + "Flatten", + TensorError::new(format!( + "The destination dimension ({D2}) must be large enough to accommodate the \ + flattening operation." + )), + ); + } + + check + } + + pub(crate) fn tri() -> Self { + let mut check = Self::Ok; + + if D < 2 { + check = check.register( + "Tri", + TensorError::new(format!( + "The input tensor must have at least 2 dimensions, got {D}" + )), + ); + } + + check + } + + pub(crate) fn squeeze(dim: usize, tensor_dims: &[usize]) -> Self { + let mut check = Self::Ok; + // This should actually be to check that the dimension to squeeze + // has a size of 1 + if tensor_dims[dim] != 1 { + check = check.register( + "Squeeze", + TensorError::new(format!( + "Can't squeeze dimension {dim} because its size is not 1", + )), + ); + } + + if dim >= tensor_dims.len() { + check = check.register( + "Squeeze", + TensorError::new(format!( + "Dimension index {dim} is out of bounds for tensor dimensions {tensor_dims:?}.", + )), + ); + } + + check + } + + pub(crate) fn squeeze_dims_input( + dim_indices: &[usize], + current_dims: &[usize], + ) -> Self { + let mut check = Self::Ok; + if dim_indices.len() >= current_dims.len() { + check = check.register( + "Squeeze", + TensorError::new("Attempted to squeeze too many dimensions!"), + ); + } + + check + } + + pub(crate) fn squeeze_dims_len(new_dims_len: usize) -> Self { + let mut check = Self::Ok; + if new_dims_len == 0 { + // 0-dim tensor not supported + check = check.register( + "Squeeze", + TensorError::new( + "Resulting dimensions cannot be zero. To remove specific singleton dimensions while preserving at least one, use `squeeze_dims` instead.".to_string() + ), + ); + } + + if new_dims_len != D2 { + check = check.register( + "Squeeze", + TensorError::new(format!( + "Resulting dimensions {new_dims_len} do not match the required D2 size {D2}.", + )), + ); + } + + check + } + + pub(crate) fn unsqueeze() -> Self { + let mut check = Self::Ok; + if D2 < D1 { + check = check.register( + "Unsqueeze", + TensorError::new(format!( + "Can't unsqueeze smaller tensor, got dim {D2}, expected > {D1}", + )), + ); + } + + check + } + + pub(crate) fn unsqueeze_dim(dim: usize) -> Self { + let mut check = Self::Ok; + if D2 <= D1 { + check = check.register( + "Unsqueeze", + TensorError::new(format!( + "The unsqueezed rank must be greater than the input rank (D={D1}; D2={D2})", + )), + ); + } + + if dim > D1 { + check = check.register( + "Unsqueeze", + TensorError::new(format!( + "Can't unsqueeze at dimension {dim}, exceeds tensor dimensions (D={D1})", + )), + ); + } + + if dim >= D2 { + check = check.register( + "Unsqueeze", + TensorError::new(format!( + "Can't unsqueeze at dimension {dim}, exceeds output tensor dimensions (D2={D2})", + )), + ); + } + + check + } + + pub(crate) fn unsqueeze_dims(dim: isize) -> Self { + let mut check = Self::Ok; + let output_rank = D as isize; + //contains is right exclusive, so this is to spec + if !(-output_rank..output_rank).contains(&dim) { + check = check.register( + "Unsqueeze", + TensorError::new(format!( + "unsqueeze arg {dim} is out of range for the output tensor of rank {output_rank}", + )), + ); + } + check + } + + pub(crate) fn one_hot_tensor>( + index_tensor: Tensor, + num_classes: usize, + ) -> Self { + let mut check = Self::Ok; + if index_tensor + .clone() + .greater_equal_elem(num_classes as i32) + .any() + .into_scalar() + .to_bool() + { + check = check.register( + "One Hot", + TensorError::new(format!( + "Can't create a one hot tensor from ({index_tensor:?}) containing indexes greater or equal to the number of classes ({num_classes})", + )), + ); + } else if num_classes <= 1 { + check = check.register( + "One Hot", + TensorError::new("Can't create a one hot tensor with less then 2 classes"), + ) + } + check + } + + pub(crate) fn one_hot_tensor_rank() -> Self { + let mut check = Self::Ok; + if D + 1 != D2 { + check = check.register( + "One Hot", + TensorError::new( + "The one-hot tensor rank must correspond to the rank of the tensor + 1", + ) + .details(format!("Expected D2={}, got {D2}", D + 1)), + ); + } + check + } + + pub(crate) fn swap_dims(dim1: usize, dim2: usize) -> Self { + let mut check = Self::Ok; + + if dim1 > D || dim2 > D { + check = check.register( + "Swap Dims", + TensorError::new("The swap dimensions must be smaller than the tensor dimension") + .details(format!( + "Swap dims ({dim1}, {dim2}) on tensor with ({D}) dimensions." + )), + ); + } + + check + } + + pub(crate) fn permute(axes: [usize; D]) -> Self { + let check = Self::Ok; + + // Check if the axes are within the tensor dimensions + if let Some(axis) = axes.iter().find(|&x| *x >= D) { + return check.register( + "permute", + TensorError::new("The axes must be smaller than the tensor dimension.") + .details(format!("The '{axis}' axis is greater than {D} dimensions.")), + ); + } + + // Check if the axes are unique + let mut seen = [false; D]; + axes.iter().for_each(|&x| seen[x] = true); + if seen.iter().any(|&x| !x) { + return check.register( + "permute", + TensorError::new("The axes must be unique.") + .details(format!("The axes '{axes:?}' are not unique.")), + ); + } + + check + } + + pub(crate) fn flip(rank: usize, axes: &[usize]) -> Self { + let check = Self::Ok; + + // Check if the axes are within the tensor dimensions + if let Some(axis) = axes.iter().find(|&x| *x >= rank) { + return check.register( + "flip", + TensorError::new("The axes must be smaller than the tensor dimension.").details( + format!("The '{axis}' axis is greater than {rank} dimensions."), + ), + ); + } + + // Check if the axes are unique + let mut dedup = axes.to_vec(); + dedup.sort_unstable(); + dedup.dedup(); + if dedup.len() != axes.len() { + return check.register( + "flip", + TensorError::new("The axes must be unique.") + .details(format!("The axes '{axes:?}' are not unique.")), + ); + } + + check + } + + pub(crate) fn matmul( + lhs: &Tensor, + rhs: &Tensor, + ) -> Self + where + K: BasicOps, + { + let mut check = Self::Ok; + + check = check.binary_ops_device("Matmul", &lhs.device(), &rhs.device()); + + if D < 2 { + return check; + } + + let shape_lhs = lhs.shape(); + let shape_rhs = rhs.shape(); + + let dim_lhs = shape_lhs[D - 1]; + let dim_rhs = shape_rhs[D - 2]; + + if dim_lhs != dim_rhs { + check = check.register( + "Matmul", + TensorError::new(format!( + "The inner dimension of matmul should be the same, but got {dim_lhs} and \ + {dim_rhs}." + )) + .details(format!( + "Lhs shape {:?}, rhs shape {:?}.", + shape_lhs, shape_rhs + )), + ); + } + + check + } + + pub(crate) fn cross( + lhs: &Tensor, + rhs: &Tensor, + dim: usize, + ) -> Self + where + K: BasicOps, + { + let mut check = Self::Ok; + + check = check.binary_ops_device("Cross", &lhs.device(), &rhs.device()); + + let shape_lhs = lhs.shape(); + let shape_rhs = rhs.shape(); + + if dim >= D { + check = check.register( + "Cross", + TensorError::new(format!( + "Dimension {dim} is out of bounds for tensors with {D} dimensions." + )), + ); + return check; + } + + let dim_size_lhs = shape_lhs[dim]; + let dim_size_rhs = shape_rhs[dim]; + + if dim_size_lhs != 3 || dim_size_rhs != 3 { + check = check.register( + "Cross", + TensorError::new(format!( + "Cross product requires dimension {dim} to have size 3, but got {dim_size_lhs} and {dim_size_rhs}." + )), + ); + } + + // Check broadcastability of other dimensions + for i in 0..D { + if i != dim { + let l = shape_lhs[i]; + let r = shape_rhs[i]; + if l != r && l != 1 && r != 1 { + check = check.register( + "Cross", + TensorError::new(format!( + "Tensors are not broadcastable along dimension {i}: {l} and {r}." + )), + ); + } + } + } + + check + } + + pub(crate) fn stack, const D2: usize>( + tensors: &[Tensor], + dim: usize, + ) -> Self { + let mut check = Self::Ok; + + if dim > D1 { + check = check.register( + "Stack", + TensorError::new( + "Can't stack tensors on a dim that exceeds the tensors dimension (inclusive)", + ) + .details(format!( + "Trying to concatenate tensors with {D1} dimensions on axis {dim}." + )), + ); + } + + if D1 == D2 { + check = check.register( + "Stack", + TensorError::new(format!( + "Can't stack tensors on existing dimension {dim}, the input and output ranks are the same (D={D1}; D2={D2}).\ + If you want to concatenate the tensors along the specified dimension ({dim}), use `Tensor::cat` instead.", + )), + ); + } + + if tensors.is_empty() { + return check.register( + "Stack", + TensorError::new("Can't stack an empty list of tensors."), + ); + } + + let shape_reference = tensors.first().unwrap().shape(); + + for tensor in tensors { + let shape = tensor.shape(); + + if shape_reference != shape { + return check.register( + "Stack", + TensorError::new("Can't stack tensors with different shapes").details(format!( + "Provided dimension ({dim}), tensors shapes: {:?}", + tensors.iter().map(Tensor::shape).collect::>() + )), + ); + } + } + + check + } + + pub(crate) fn cat>( + tensors: &[Tensor], + dim: usize, + ) -> Self { + let mut check = Self::Ok; + + if dim >= D { + check = check.register( + "Cat", + TensorError::new( + "Can't concatenate tensors on a dim that exceeds the tensors dimension", + ) + .details(format!( + "Trying to concatenate tensors with {D} dimensions on axis {dim}." + )), + ); + } + + if tensors.is_empty() { + return check.register( + "Cat", + TensorError::new("Can't concatenate an empty list of tensors."), + ); + } + + let mut shape_reference = tensors.first().unwrap().shape(); + shape_reference[dim] = 1; // We want to check every dims except the one where the + // concatenation happens. + + for tensor in tensors { + let mut shape = tensor.shape(); + shape[dim] = 1; // Ignore the concatenate dim. + + if shape_reference != shape { + return check.register( + "Cat", + TensorError::new( + "Can't concatenate tensors with different shapes, except for the provided \ + dimension", + ) + .details(format!( + "Provided dimension ({dim}), tensors shapes: {:?}", + tensors.iter().map(Tensor::shape).collect::>() + )), + ); + } + } + + check + } + + pub(crate) fn slice(shape: &Shape, slices: &[Slice]) -> Self { + let mut check = Self::Ok; + let n_dims_tensor = R; + let n_dims_slices = slices.len(); + + if n_dims_tensor < n_dims_slices { + check = check.register( + "Slice", + TensorError::new( + "The provided slices array has a higher number of dimensions than the current \ + tensor.", + ) + .details(format!( + "The slices array must be smaller or equal to the tensor number of \ + dimensions. Tensor number of dimensions: {n_dims_tensor}, slices array \ + length {n_dims_slices}." + )), + ); + } + + for (i, slice) in slices.iter().enumerate().take(R) { + let d_tensor = shape[i]; + + // Check the raw end value before conversion + if let Some(end) = slice.end + && end > 0 + && end as usize > d_tensor + { + check = check.register( + "Slice", + TensorError::new( + "The provided slice has a range that exceeds the current tensor \ + size.", + ) + .details(format!( + "The slice end index {} exceeds the size of the tensor ({}) at dimension {}. \ + Tensor shape {:?}.", + end, d_tensor, i, shape, + )), + ); + } + + // Empty slices (start >= end) are allowed and produce a tensor with size 0 + // in that dimension. This matches PyTorch behavior and is required for ONNX + // compatibility where dynamic slice ranges may become empty at runtime. + + if slice.step() == 0 { + check = check.register( + "Slice", + TensorError::new("The provided slice has a step of 0.").details(format!( + "The slice at dimension '{i}' has a step of 0. Step must be non-zero.", + )), + ); + } + } + + check + } + + pub(crate) fn slice_assign( + shape: &Shape, + shape_value: &Shape, + slices: &[crate::Slice], + ) -> Self { + let mut check = Self::Ok; + let n_dims_slices = slices.len(); + + if R < n_dims_slices { + check = check.register( + "Slice Assign", + TensorError::new( + "The provided slices array has a higher number of dimensions than the current \ + tensor.", + ) + .details(format!( + "The slices array must be smaller or equal to the tensor number of \ + dimensions. Tensor number of dimensions: {R}, slices array length {n_dims_slices}." + )), + ); + } + + for (i, slice) in slices.iter().enumerate().take(usize::min(R, n_dims_slices)) { + let d_tensor = shape[i]; + let d_tensor_value = shape_value[i]; + let range = slice.to_range(d_tensor); + + if range.end > d_tensor { + check = check.register( + "Range Assign", + TensorError::new( + "The provided slice has a range that exceeds the current tensor \ + size.", + ) + .details(format!( + "The range ({}..{}) exceeds the size of the tensor ({}) at dimension {}. \ + Current tensor shape {:?}, value tensor shape {:?}.", + range.start, range.end, d_tensor, i, shape, shape_value, + )), + ); + } + + // Calculate the number of elements selected with the given step + let num_elements = slice.output_size(d_tensor); + + if num_elements != d_tensor_value { + check = check.register( + "Slice Assign", + TensorError::new( + "The value tensor must match the amount of elements selected with the \ + slices array", + ) + .details(format!( + "The slice with range ({}..{}) and step {} selects {} elements but the value \ + tensor has {} elements at dimension {}. Current tensor shape {:?}, value tensor \ + shape {:?}.", + range.start, + range.end, + slice.step, + num_elements, + d_tensor_value, + i, + shape, + shape_value, + )), + ); + } + + // Note: Empty slices (start >= end with positive step) are handled at the API level + // by returning the original tensor unchanged, so we don't check for them here. + } + + check + } + + pub(crate) fn check_dim(dim: usize) -> Self { + let mut check = Self::Ok; + + if dim >= D { + check = check.register( + "Check Dim", + TensorError::new("The provided dimension exceeds the tensor dimensions.").details( + format!("Tensor has {D} dimensions, but the provided dimension is {dim}."), + ), + ); + } + + check + } + + pub(crate) fn gather(dim: usize, shape: &Shape, shape_indices: &Shape) -> Self { + Self::check_gather_scatter_indices::(Self::Ok, "Gather", dim, shape, shape_indices) + } + + pub(crate) fn scatter( + dim: usize, + shape: &Shape, + shape_indices: &Shape, + shape_value: &Shape, + ) -> Self { + let ops = "Scatter"; + let mut check = + Self::check_gather_scatter_indices::(Self::Ok, ops, dim, shape, shape_indices); + + if shape_indices != shape_value { + check = check.register( + ops, + TensorError::new( + "Indices tensor shape should be the same as the value tensor shape." + .to_string(), + ) + .details(format!( + "The shape differs: {:?} != {:?}", + shape_indices, shape_value + )), + ); + } + + check + } + + pub(crate) fn select(dim: usize) -> Self { + Self::check_select_basic::(Self::Ok, "select", dim) + } + + pub(crate) fn take(dim: usize) -> Self { + let mut check = Self::check_select_basic::(Self::Ok, "Take", dim); + + // Calculate expected output dimensions + // DO = D - 1 + DI (remove 1 dim, add DI dims) + let expected_do = D + DI - 1; + if DO != expected_do { + check = check.register( + "Take", + TensorError::new("Output dimension mismatch").details(format!( + "Expected output dimension {} (D={} + DI={} - 1) but got DO={}", + expected_do, D, DI, DO + )), + ); + } + + check + } + + pub(crate) fn diag() -> Self { + let mut check = Self::Ok; + + if D < 2 { + check = check.register( + "Diag", + TensorError::new( + "Diagonal operations require + tensors with at least 2 dimensions.", + ) + .details(format!( + "Got tensor with {D} dimensions, + expected at least 2" + )), + ); + } + + if DO != D - 1 { + check = check.register( + "Diag", + TensorError::new("Output rank must be input rank minus 1 for diagonal") + .details(format!("Expected output rank {}, got {DO}", D - 1)), + ); + } + + check + } + + pub(crate) fn select_assign( + dim: usize, + shape_indices: &Shape, + shape_value: &Shape, + ) -> Self { + let mut check = Self::check_select_basic::(Self::Ok, "Select Assign", dim); + + if shape_value[dim] != shape_indices[0] { + check = check.register( + "Select Assign", + TensorError::new( + format!( + "Number of indices ({}) should be equal to value tensor dimensions {:?} on axis (dim={dim})", + shape_indices[0], + shape_value + ), + ) + ); + } + + check + } + + fn check_select_basic(mut check: Self, ops: &str, dim: usize) -> Self { + if dim > D { + check = check.register( + ops, + TensorError::new(format!( + "Can't index a tensor with ({D}) dimensions on axis ({dim})" + )), + ); + } + + check + } + fn check_gather_scatter_indices( + mut check: Self, + ops: &str, + dim: usize, + shape: &Shape, + shape_indices: &Shape, + ) -> Self { + if dim > D { + check = check.register( + ops, + TensorError::new(format!( + "Can't index a tensor with ({D}) dimensions on axis ({dim})" + )), + ); + } + + for i in 0..D { + if i == dim { + continue; + } + + let tensor_dim_i = shape[i]; + let indices_dim_i = shape_indices[i]; + + if tensor_dim_i != indices_dim_i { + check = check.register( + ops, + TensorError::new( + "The tensor shape should be the same as the index tensor shape." + .to_string(), + ) + .details(format!( + "The shape differs at dimension {i}: {tensor_dim_i} != {indices_dim_i}" + )), + ); + } + } + + check + } + + pub(crate) fn check_prelu_shape( + shape_tensor: &Shape, + shape_weight: &Shape, + ) -> Self { + let mut check = Self::Ok; + if shape_weight[0] == 1 { + check + } else if D >= 2 { + let channels = shape_tensor[1]; + let num_weights = shape_weight[0]; + if channels != num_weights { + check = check.register( + "PReLu", + TensorError::new( + "Number of channels in input tensor and number of weights must be equal", + ) + .details(format!( + "Got no. of channels: {channels}, no. of weights: {num_weights}", + )), + ); + return check; + } + check + } else { + check = check.register( + "PReLu", + TensorError::new( + "Number of channels in input tensor and number of weights must be equal", + ) + .details(format!( + "Got no. of channels: 1, no. of weights: {}", + shape_weight[0] + )), + ); + check + } + } + + /// Checks aggregate dimension such as mean and sum. + pub(crate) fn aggregate_dim(ops: &str, dim: usize) -> Self { + let mut check = Self::Ok; + + if dim > D { + check = check.register( + ops, + TensorError::new(format!( + "Can't aggregate a tensor with ({D}) dimensions on axis ({dim})" + )), + ); + } + + check + } + + pub(crate) fn sort_dim(ops: &str, dim: usize) -> Self { + let mut check = Self::Ok; + + if dim > D { + check = check.register( + ops, + TensorError::new(format!( + "Can't sort a tensor with ({D}) dimensions on axis ({dim})" + )), + ); + } + + check + } + + pub(crate) fn split( + tensor_dims: &[usize], + split_size: usize, + dim: usize, + ) -> Self { + let mut check = Self::Ok; + let op = "split"; + let tensor_rank = tensor_dims.len(); + + if dim >= tensor_rank { + check = check.register( + op, + TensorError::new("Given dimension is greater than or equal to the tensor rank.") + .details(format!("Tensor rank: '{D}', given dimension: '{dim}'")), + ); + } else { + let tensor_size = tensor_dims[dim]; + if split_size == 0 && tensor_size != 0 { + check = check.register( + op, + TensorError::new("split_size must be greater than 0 unless the tensor size along the dimension is 0.") + .details(format!("split_size: '{split_size}', tensor size along dim '{dim}': '{tensor_size}'.")), + ); + } + } + + check + } + + pub(crate) fn split_with_sizes( + tensor_dims: &[usize], + split_sizes: &[usize], + dim: usize, + ) -> Self { + let mut check = Self::Ok; + let op = "split_with_sizes"; + let tensor_rank = tensor_dims.len(); + + if dim >= tensor_rank { + check = check.register( + op, + TensorError::new("Given dimension is greater than or equal to the tensor rank.") + .details(format!("Tensor rank: '{D}', given dimension: '{dim}'.")), + ); + } else { + // Validate split_sizes add up to size of dimension to split along + let tensor_size = tensor_dims[dim]; + let total_split_size: usize = split_sizes.iter().sum(); + if total_split_size != tensor_size { + check = check.register( + op, + TensorError::new("The sum of split_sizes must equal the tensor size along the specified dimension.") + .details(format!("Sum of split_sizes: '{total_split_size}', tensor size along dim '{dim}': '{tensor_size}'.")), + ); + } + } + + check + } + + /// The goal is to minimize the cost of checks when there are no error, but it's way less + /// important when an error occurred, crafting a comprehensive error message is more important + /// than optimizing string manipulation. + fn register(self, ops: &str, error: TensorError) -> Self { + let errors = match self { + Self::Ok => vec![error], + Self::Failed(mut failed) => { + failed.errors.push(error); + failed.errors + } + }; + + Self::Failed(FailedTensorCheck { + ops: ops.to_string(), + errors, + }) + } + + /// Checks if shapes are compatible for element wise operations supporting broadcasting. + pub(crate) fn binary_ops_ew_shape( + self, + ops: &str, + lhs: &Shape, + rhs: &Shape, + ) -> Self { + let mut check = self; + + for i in 0..D { + let d_lhs = lhs[i]; + let d_rhs = rhs[i]; + + if d_lhs != d_rhs { + let is_broadcast = d_lhs == 1 || d_rhs == 1; + + if is_broadcast { + continue; + } + + check = check.register( + ops, + TensorError::new("The provided tensors have incompatible shapes.").details( + format!( + "Incompatible size at dimension '{}' => '{} != {}', which can't be \ + broadcasted. Lhs tensor shape {:?}, Rhs tensor shape {:?}.", + i, d_lhs, d_rhs, lhs, rhs, + ), + ), + ); + } + } + + check + } + + /// Checks if tensor devices are equal. + fn binary_ops_device( + self, + ops: &str, + lhs: &Device, + rhs: &Device, + ) -> Self { + match lhs != rhs { + true => self.register( + ops, + TensorError::new("The provided tensors are not on the same device.").details( + format!("Lhs tensor device {lhs:?}, Rhs tensor device {rhs:?}.",), + ), + ), + false => self, + } + } + + /// Checks if expand operation is possible for the given shapes. + pub fn expand(ops: &str, shape: &Shape, to: &Shape) -> Self { + let mut check = TensorCheck::Ok; + let max_dims = core::cmp::max(D1, D2); + + // Calculate the starting indices for each shape array, ensuring alignment from the right. + let start_index_shape = max_dims.saturating_sub(D1); + let start_index_to = max_dims.saturating_sub(D2); + + for i in 0..max_dims { + // Use 1 as the default dimension size for dimensions beyond the tensor's rank. + let d_shape = if i >= start_index_shape { + shape[i - start_index_shape] + } else { + 1 + }; + let d_to = if i >= start_index_to { + to[i - start_index_to] + } else { + 1 + }; + + if d_shape != d_to && d_shape != 1 && d_to != 1 { + // Register an incompatibility error. + check = check.register( + ops, + TensorError::new( + "The provided tensor can't be broadcasted to the target shape.", + ) + .details(format!( + "Incompatible size at dimension '{}' => '{} != {}', which can't be \ + broadcasted. Tensor shape {:?}, Target shape {:?}.", + max_dims - i - 1, + d_shape, + d_to, + shape, + to, + )), + ); + break; // Incompatibility found, no need to check further. + } + } + + check + } + + /// Checks if unfold operation is possible for the given shapes. + pub fn unfold( + ops: &str, + _shape: &Shape, + _dim: usize, + _size: usize, + _step: usize, + ) -> Self { + let mut check = TensorCheck::Ok; + + if D2 != D1 + 1 { + check = check.register( + ops, + TensorError::new("The unfold rank is incompatible with the input tensor rank.") + .details(format!( + "The output rank '{D2}' != the input rank + 1 '{D1}'.", + )), + ); + } + + check + } + + /// Checks if input is compatible with convolution weights. + pub fn conv( + ops: &str, + x: [usize; D1], + weight: [usize; D2], + groups: usize, + ) -> Self { + let mut check = TensorCheck::Ok; + let channels = x[1]; + let expected = weight[1] * groups; + if channels != expected { + check = check.register( + ops, + TensorError::new("Number of channels in input tensor and input channels of convolution must be equal.") + .details(format!("got: {channels}, expected: {expected}")), + ); + } + check + } + + /// Checks if input is compatible with transposed convolution weights. + pub fn conv_transpose( + ops: &str, + x: [usize; D1], + weight: [usize; D2], + ) -> Self { + let mut check = TensorCheck::Ok; + let channels = x[1]; + let expected = weight[0]; + if channels != expected { + check = check.register( + ops, + TensorError::new("Number of channels in input tensor and input channels of convolution must be equal.") + .details(format!("got: {channels}, expected: {expected}")), + ); + } + check + } + + /// Check if input is compatible with LU decomposition. + pub fn is_square(ops: &str, shape: &Shape) -> Self { + let mut check = TensorCheck::Ok; + if shape[D - 1] != shape[D - 2] { + check = check.register( + ops, + TensorError::new("The input tensor must be square.").details(format!( + "Got tensor with shape {:?}, expected last two dimensions to be equal", + shape + )), + ); + } + check + } + + /// Check pivot is valid for LU decomposition. + pub fn lu_decomposition_pivot(pivot: FloatElem) -> Self { + let mut check = TensorCheck::Ok; + if pivot.to_f64().abs() <= 1e-6 { + check = check.register( + "lu_decomposition", + TensorError::new("LU decomposition requires a valid pivot.") + .details(format!("Got pivot value too close to zero: {}", pivot)), + ); + } + check + } +} + +pub(crate) struct FailedTensorCheck { + ops: String, + errors: Vec, +} + +impl FailedTensorCheck { + /// Format all the checks into a single message ready to be printed by a [panic](core::panic). + pub(crate) fn format(self) -> String { + self.errors.into_iter().enumerate().fold( + format!( + "=== Tensor Operation Error ===\n Operation: '{}'\n Reason:", + self.ops + ), + |accum, (number, error)| accum + error.format(number + 1).as_str(), + ) + "\n" + } +} + +struct TensorError { + description: String, + details: Option, +} + +impl TensorError { + pub(crate) fn new>(description: S) -> Self { + TensorError { + description: description.into(), + details: None, + } + } + + pub(crate) fn details>(mut self, details: S) -> Self { + self.details = Some(details.into()); + self + } + + fn format(self, number: usize) -> String { + let mut message = format!("\n {number}. "); + message += self.description.as_str(); + message += " "; + + if let Some(details) = self.details { + message += details.as_str(); + message += " "; + } + + message + } +} + +/// Module where we defined macros that can be used only in the project. +pub(crate) mod macros { + /// We use a macro for all checks, since the panic message file and line number will match the + /// function that does the check instead of a generic error.rs crate private unrelated file + /// and line number. + macro_rules! check { + ($check:expr) => { + if let TensorCheck::Failed(check) = $check { + core::panic!("{}", check.format()); + } + }; + } + pub(crate) use check; +} + +pub(crate) fn unwrap_shape_reshape(result: Result) -> Shape { + match result { + Ok(shape) => shape, + // `shape.reshape(new_shape)` should only return `MetadataError::Invalid`. + Err(burn_std::MetadataError::Invalid { reason }) => { + macros::check!({ + TensorCheck::Ok.register("Reshape", crate::check::TensorError::new(reason)) + }); + unreachable!() + } + Err(e) => panic!("{e:?}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use macros::check; + + #[test] + #[should_panic] + fn index_range_exceed_dimension() { + let slices = vec![Slice::from(0..2), Slice::from(0..4), Slice::from(1..8)]; + check!(TensorCheck::slice::<3>(&Shape::new([3, 5, 7]), &slices)); + } + + #[test] + #[should_panic] + fn index_range_exceed_number_of_dimensions() { + let slices = vec![Slice::from(0..1), Slice::from(0..1), Slice::from(0..1)]; + check!(TensorCheck::slice::<2>(&Shape::new([3, 5]), &slices)); + } + + #[test] + #[should_panic] + fn binary_ops_shapes_no_broadcast() { + check!(TensorCheck::binary_ops_ew_shape::<2>( + TensorCheck::Ok, + "TestOps", + &Shape::new([3, 5]), + &Shape::new([3, 6]) + )); + } + + #[test] + fn binary_ops_shapes_with_broadcast() { + check!(TensorCheck::binary_ops_ew_shape::<2>( + TensorCheck::Ok, + "Test", + &Shape::new([3, 5]), + &Shape::new([1, 5]) + )); + } + + #[test] + #[should_panic] + fn binary_ops_devices() { + check!(TensorCheck::binary_ops_device( + TensorCheck::Ok, + "Test", + &5, // We can pass anything that implements PartialEq as device + &8 + )); + } + + #[test] + #[should_panic] + fn movedim_args_out_of_bounds() { + check!(TensorCheck::movedim_args_usize::<3>(5)); + } + + #[test] + fn movedim_args_i32() { + check!(TensorCheck::movedim_args_i32::<3>(-3)); + } + + #[test] + #[should_panic] + fn movedim_args_too_negative() { + check!(TensorCheck::movedim_args_i32::<3>(-4)); + } + + #[test] + #[should_panic] + fn movedim_args_vec_out_of_bounds() { + check!(TensorCheck::movedim_args_vec::<3>(&vec![0, 1, 3])); + } + + #[test] + #[should_panic] + fn movedim_args_vec_duplicates() { + check!(TensorCheck::movedim_args_vec::<3>(&vec![0, 1, 1])); + } + + #[test] + #[should_panic] + fn movedim_args_length() { + check!(TensorCheck::movedim_args_length( + &vec![0, 1], + &vec![0, 1, 2] + )); + } + + #[test] + #[should_panic] + fn unsqueeze_dim_same_rank() { + check!(TensorCheck::unsqueeze_dim::<3, 3>(2)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/float.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/float.rs new file mode 100644 index 0000000..1dd3de9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/float.rs @@ -0,0 +1,1011 @@ +use crate::AsIndex; +use crate::FloatDType; +use crate::Tensor; +use crate::cast::ToElement; +use crate::check; +use crate::check::TensorCheck; +use crate::ops::GridSampleOptions; +use crate::quantization::{QuantScheme, QuantizationParameters}; +use crate::tensor::backend::Backend; +use crate::tensor::stats; +use crate::tensor::{Distribution, TensorData}; +use crate::{Bool, Int, TensorPrimitive}; +use burn_backend::tensor::quantization::QuantizationParametersPrimitive; +use core::f32; + +/// Default RTOL value for `is_close` and `all_close`. +pub const DEFAULT_RTOL: f64 = 1e-5; + +/// Default ATOL value for `is_close` and `all_close`. +pub const DEFAULT_ATOL: f64 = 1e-8; + +impl Tensor +where + B: Backend, +{ + /// Applies element wise exponential operation. + /// + #[cfg_attr(doc, doc = "$y_i = e^{x_i}$")] + #[cfg_attr(not(doc), doc = "`y = e^x`")] + pub fn exp(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_exp( + self.primitive.tensor(), + ))) + } + + /// Applies element wise natural log operation *ln*. + /// + #[cfg_attr(doc, doc = r#"$y_i = \log_e\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = log(x_i)`")] + pub fn log(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_log( + self.primitive.tensor(), + ))) + } + + /// Applies the natural logarithm of one plus the input tensor, element-wise. + /// + #[cfg_attr(doc, doc = r#"$y_i = \log_e\(x_i + 1\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = log(x_i + 1)`")] + pub fn log1p(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_log1p( + self.primitive.tensor(), + ))) + } + + /// Applies the [error function](https://en.wikipedia.org/wiki/Error_function) element wise. + /// + #[cfg_attr( + doc, + doc = r#" +$y_i = \text{erf}\(x_i\)$ + +The error function is defined as: + +$$\text{erf}\(x\) = \frac{2}{\sqrt{\pi}} \int_0^x e^{-t^2} dt$$ +"# + )] + #[cfg_attr(not(doc), doc = "`y_i = erf(x_i)`")] + pub fn erf(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_erf( + self.primitive.tensor(), + ))) + } + + /// Applies [reciprocal operation](https://en.wikipedia.org/wiki/Multiplicative_inverse) + /// (or multiplicative inverse) element wise. + /// + #[cfg_attr(doc, doc = r#"$y_i = \frac{1}{x_i}$"#)] + #[cfg_attr(not(doc), doc = "`y_i = 1/x_i`")] + pub fn recip(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_recip( + self.primitive.tensor(), + ))) + } + + /// Applies element wise square operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = x_i * x_i$"#)] + #[cfg_attr(not(doc), doc = "`y_i = x_i * x_i`")] + pub fn square(self) -> Self { + self.powi_scalar(2) + } + + /// Applies element wise root square operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \sqrt{x_i}$"#)] + #[cfg_attr(not(doc), doc = "`y_i = sqrt(x_i)`")] + pub fn sqrt(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_sqrt( + self.primitive.tensor(), + ))) + } + + /// Applies element wise cosine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \cos\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = cos(x_i)`")] + pub fn cos(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_cos( + self.primitive.tensor(), + ))) + } + + /// Applies element wise sine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \sin\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = sin(x_i)`")] + pub fn sin(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_sin( + self.primitive.tensor(), + ))) + } + + /// Applies element wise tangent operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \tan\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = tan(x_i)`")] + pub fn tan(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_tan( + self.primitive.tensor(), + ))) + } + + /// Applies element wise hyperbolic cosine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \cosh\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = cosh(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 2.0], &device); + /// println!("{}", tensor.cosh()); // [1.0, 1.5430, 3.7621] + /// } + /// ``` + pub fn cosh(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_cosh( + self.primitive.tensor(), + ))) + } + + /// Applies element wise hyperbolic sine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \sinh\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = sinh(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 2.0], &device); + /// println!("{}", tensor.sinh()); // [0.0, -1.1752, 3.6269] + /// } + /// ``` + pub fn sinh(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_sinh( + self.primitive.tensor(), + ))) + } + + /// Applies element wise hyperbolic tangent operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \tanh\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = tanh(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 2.0], &device); + /// println!("{}", tensor.tanh()); // [0.0, -0.7616, 0.9640] + /// } + /// ``` + pub fn tanh(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_tanh( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse sine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \asin\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = asin(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 1.0], &device); + /// println!("{}", tensor.asin()); // [ 0.0000, -1.5708, 1.5708] + /// } + /// ``` + pub fn asin(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_asin( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse hyperbolic sine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \asinh\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = asinh(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 1.0], &device); + /// println!("{}", tensor.asinh()); // [ 0.0000, -0.8814, 0.8814] + /// } + /// ``` + pub fn asinh(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_asinh( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse cosine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \acos\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = acos(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 1.0], &device); + /// println!("{}", tensor.acos()); // [1.5708, 3.1416, 0.0] + /// } + /// ``` + pub fn acos(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_acos( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse hyperbolic cosine operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \acosh\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = acosh(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([1.0, 2.0, 3.0], &device); + /// println!("{}", tensor.sinh()); // [0.0000, 1.3170, 1.7627] + /// } + /// ``` + pub fn acosh(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_acosh( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse tangent operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \atan\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = atan(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -1.0, 2.0], &device); + /// println!("{}", tensor.sinh()); // [ 0.0, -0.7854, 1.1071] + /// } + /// ``` + pub fn atan(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_atan( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse hyperbolic tangent operation. + /// + #[cfg_attr(doc, doc = r#"$y_i = \atan\(x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`y_i = atan(x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let tensor = Tensor::::from_data([0.0, -0.5, 0.5], &device); + /// println!("{}", tensor.sinh()); // [ 0.0, -0.5493, 0.5493] + /// } + /// ``` + pub fn atanh(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_atanh( + self.primitive.tensor(), + ))) + } + + /// Applies element wise inverse tangent operation using the signs of arguments to determine the correct quadrant. + /// + #[cfg_attr(doc, doc = r#"$z_i = \atan2\(y_i, x_i\)$"#)] + #[cfg_attr(not(doc), doc = "`z_i = atan2(y_i, x_i)`")] + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// + /// let lhs = Tensor::::from_data([-2.0, 2.0, -2.0], &device); + /// let rhs = Tensor::::from_data([1.0, -1.0, -1.0], &device); + /// println!("{}", lhs.atan2(rhs)); // [-1.1071, 2.0344, -2.0344] + /// } + /// ``` + pub fn atan2(self, other: Self) -> Self { + Self::new(TensorPrimitive::Float(B::float_atan2( + self.primitive.tensor(), + other.primitive.tensor(), + ))) + } + + /// Converts each of the elements of the input tensor from angles in degrees to radians. + /// + /// # Example + /// ```ignore + /// let tensor_in_radians = tensor.deg2rad(); + /// ``` + pub fn deg2rad(self) -> Self { + self.mul_scalar(f32::consts::PI / 180.0) + } + + /// Converts each of the elements of the input tensor from angles in radians to degrees. + /// + /// # Example + /// ```ignore + /// let tensor_in_degrees = tensor.rad2deg(); + /// ``` + pub fn rad2deg(self) -> Self { + self.mul_scalar(180.0 / f32::consts::PI) + } + + /// Applies element wise round operation. + /// + /// This function implements the [round half to even](https://en.wikipedia.org/wiki/Rounding#Rounding_half_to_even) + /// strategy, with halfway cases rounded to the nearest even integer value. + pub fn round(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_round( + self.primitive.tensor(), + ))) + } + + /// Applies element wise floor operation. + pub fn floor(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_floor( + self.primitive.tensor(), + ))) + } + + /// Applies element wise ceil operation. + pub fn ceil(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_ceil( + self.primitive.tensor(), + ))) + } + + /// Create a tensor from floats (f32) on a given device. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let _ = Tensor::::from_floats([1.0, 2.0], &device); + /// let _ = Tensor::::from_floats([[1.0, 2.0], [3.0, 4.0]], &device); + /// } + /// ``` + pub fn from_floats>(floats: A, device: &B::Device) -> Self { + Self::from_data(floats.into().convert::(), device) + } + + /// Returns a new tensor with the same shape and device as the current tensor and the data + /// cast to Integer. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = Default::default(); + /// let float_tensor = Tensor::::from_floats([1.0, 2.0], &device); + /// let int_tensor = float_tensor.int(); + /// } + /// ``` + pub fn int(self) -> Tensor { + Tensor::new(B::float_into_int(self.primitive.tensor())) + } + + /// Returns a new tensor with the same shape, dtype, and device as the current tensor filled random + /// values sampled from the given distribution. + pub fn random_like(&self, distribution: Distribution) -> Self { + Self::new(TensorPrimitive::Float(B::float_random( + self.shape(), + distribution, + &self.device(), + ))) + .cast(self.dtype()) + } + + /// Calculate the variance along the given dimension. + pub fn var(self, dim: usize) -> Self { + stats::var(self, dim) + } + + /// Calculate the variance along the given dimension without applying the Bessel’s correction. + pub fn var_bias(self, dim: usize) -> Self { + stats::var_bias(self, dim) + } + + /// Calculate the variance along the given dimension and also returns the mean. + pub fn var_mean(self, dim: usize) -> (Self, Self) { + let mean = self.clone().mean_dim(dim); + let var = stats::var_with_mean(self, mean.clone(), dim); + (var, mean) + } + + /// Calculate the variance along the given dimension without applying the Bessel’s correction and also returns the mean. + pub fn var_mean_bias(self, dim: usize) -> (Self, Self) { + let mean = self.clone().mean_dim(dim); + let var = stats::var_with_mean_bias(self, mean.clone(), dim); + (var, mean) + } + + /// Returns the median value along the specified dimension. + /// + /// The median is not unique for input tensors with an even number of elements + /// in the reduced dimension. In this case, the lower of the two medians is returned, + /// following PyTorch's behavior. + /// + /// # Note + /// + /// The current implementation performs a full sort along the specified dimension, + /// which has O(nlog(n)) complexity. Additionally, most backends currently fall back + /// to CPU for the sort operation, which may result in slower performance compared + /// to native GPU operations. + /// + /// # Arguments + /// + /// - `dim` - The dimension along which to compute the median. + /// + /// # Returns + /// + /// - A tensor containing the median values along the specified dimension. + /// + /// # Example 1 + /// + /// ```ignore + /// // Assuming backend B + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data( + /// [[1.0, 5.0, 3.0, 2.0], [8.0, 4.0, 6.0, 7.0]], + /// &device, + /// ); + /// + /// // Median along dimension 0: + /// // sorted columns are [1.0, 8.0], [4.0, 5.0], [3.0, 6.0], [2.0, 7.0] + /// let median = tensor.median(0); + /// // Result: [[1.0, 4.0, 3.0, 2.0]] + /// + /// // Median along dimension 1: + /// // sorted rows are [1.0, 2.0, 3.0, 5.0] and [4.0, 6.0, 7.0, 8.0] + /// let median = tensor.median(1); + /// // Result: [[2.0], [6.0]] + /// ``` + /// + /// # Example 2 + /// + /// The median across all elements can be calculated as follows: + /// + /// ```ignore + /// // D is the number of dimensions of the tensor + /// let flattened_tensor: Tensor = tensor.flatten(0, D - 1); + /// + /// // Calculate median for dim 0 since the tensor has become 1 dimensional + /// let median = flattened_tensor.median(0); + /// // Result: [4.0] + /// ``` + pub fn median(self, dim: usize) -> Self { + // TODO: Allow backend specialization. Optimally, implement a median kernel for cubecl + // instead of leveraging a full sort to get the median. + stats::median(self, dim) + } + + /// Returns the median value along the specified dimension and its index. + /// + /// The median is not unique for input tensors with an even number of elements + /// in the reduced dimension. In this case, the lower of the two medians is returned, + /// following PyTorch's behavior. + /// + /// # Note + /// + /// The current implementation performs a full sort along the specified dimension, + /// which has O(nlog(n)) complexity. Additionally, most backends currently fall back + /// to CPU for the sort operation, which may result in slower performance compared + /// to native GPU operations. + /// + /// # Arguments + /// + /// - `dim` - The dimension along which to compute the median. + /// + /// # Returns + /// + /// A tuple containing: + /// - A tensor with the median values. + /// - A tensor with the indices of the median values in the original tensor. + /// + /// # Example + /// + /// ```ignore + /// // Assuming backend B + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data( + /// [[1.0, 5.0, 3.0, 2.0], [8.0, 4.0, 6.0, 7.0]], + /// &device, + /// ); + /// + /// // Median along dimension 1: + /// // sorted rows are [1.0, 2.0, 3.0, 5.0] and [4.0, 6.0, 7.0, 8.0] + /// let (values, indices) = tensor.median_with_indices(1); + /// // values: [[2.0], [6.0]], indices: [[3], [2]] (position in the original tensor) + /// ``` + pub fn median_with_indices(self, dim: usize) -> (Self, Tensor) { + // TODO: Allow backend specialization. Optimally, implement a median kernel for cubecl + // instead of leveraging a full sort to get the median. + stats::median_with_indices(self, dim) + } + + /// Converts a tensor to the specified floating point data type. + /// + /// This is always a no-op when casting to the current dtype. + /// + /// # Warning + /// Most backends don't have automatic type promotion at this time, so make sure that all tensors + /// have the same floating point precision data type for operations multiple input tensors (e.g., binary ops). + pub fn cast>(self, dtype: F) -> Tensor { + let dtype = dtype.into(); + let self_type: FloatDType = self.dtype().into(); + if dtype == self_type { + // no-op. + return self; + } + + Tensor::new(TensorPrimitive::Float(B::float_cast( + self.primitive.tensor(), + dtype, + ))) + } + + /// Detach the current tensor from the autodiff graph. + /// + /// This function does nothing when autodiff is not enabled. + /// This can be used in batchers or elsewhere to ensure that previous operations are not + /// considered in the autodiff graph. + pub fn detach(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_detach( + self.primitive.tensor(), + ))) + } + + /// Mark the tensor to keep gradients during the backward pass. + /// + /// This function does nothing when autodiff is not enabled. + pub fn require_grad(self) -> Self { + self.set_require_grad(true) + } + + /// Returns true if the tensor requires gradients during the backward pass. + pub fn is_require_grad(&self) -> bool { + match &self.primitive { + TensorPrimitive::Float(tensor) => B::float_is_require_grad(tensor), + TensorPrimitive::QFloat(tensor) => B::q_is_require_grad(tensor), + } + } + + /// Mark the tensor as tracked or untracked depending on the require_grad argument. + /// When tracked, the gradients will be available after the backward pass. + /// + /// This function does nothing when autodiff is not enabled. + pub fn set_require_grad(self, require_grad: bool) -> Self { + let primitive = match self.primitive { + TensorPrimitive::Float(tensor) => { + TensorPrimitive::Float(B::float_set_require_grad(tensor, require_grad)) + } + TensorPrimitive::QFloat(tensor) => { + TensorPrimitive::QFloat(B::q_set_require_grad(tensor, require_grad)) + } + }; + Self::new(primitive) + } + + /// Applies the relu function to the tensor. + pub(crate) fn relu(self) -> Self { + Self::new(TensorPrimitive::Float(B::relu(self.primitive.tensor()))) + } + + /// Calculate covaraince matrix between different entries alongside a given dimension. + /// + /// # Arguments + /// + /// * `size` - The size of the square matrix. + /// * `correction_factor` - Is usually 1 for samples and 0 for population. + pub fn cov(self, dim: usize, correction_factor: usize) -> Tensor { + let n = self.dims()[dim]; + let centered = (self.clone() - self.mean_dim(dim)).swap_dims(dim, 0); + centered + .clone() + .transpose() + .matmul(centered) + .div_scalar(n as f32 - correction_factor as f32) + } + + /// Convert the tensor to a lower precision data type based on the quantization scheme. + /// + /// # Arguments + /// + /// * `scheme` - The quantization scheme. + /// * `qparams` - The pre-computed quantization parameters. + /// + /// # Returns + /// + /// The quantized tensor. + pub fn quantize( + self, + scheme: &QuantScheme, + qparams: QuantizationParameters, + ) -> Tensor { + Tensor::new(TensorPrimitive::QFloat(B::quantize( + self.primitive.tensor(), + scheme, + QuantizationParametersPrimitive { + scales: qparams.scales.primitive.tensor(), + }, + ))) + } + + /// Dynamically convert the tensor to a lower precision data type based on the quantization scheme. + /// + /// # Arguments + /// + /// * `scheme` - The quantization scheme. + /// + /// # Returns + /// + /// The quantized tensor. + /// + /// # Notes + /// This uses [min-max calibration](crate::quantization::Calibration::MinMax). + pub fn quantize_dynamic(self, scheme: &QuantScheme) -> Tensor { + Tensor::new(TensorPrimitive::QFloat(B::quantize_dynamic( + self.primitive.tensor(), + scheme, + ))) + } + + /// Convert the tensor back to a higher precision data type. + /// + /// If the tensor is not quantized, its value is simply returned. + /// + /// # Returns + /// + /// The dequantized tensor. + pub fn dequantize(self) -> Tensor { + Tensor::new(TensorPrimitive::Float(self.primitive.tensor())) + } + + /// Checks element wise if the tensor is close to another tensor. + /// + /// The tolerance is defined by the following equation: + /// + /// ```text + /// abs(a - b) <= (atol + rtol * abs(b)) + /// + /// where `a` is the first tensor, `b` is the second tensor, `rtol` is the relative tolerance, + /// and `atol` is the absolute tolerance. + /// ``` + /// + /// # Arguments + /// + /// * `other` - The tensor to compare with. + /// * `rtol` - Optional relative tolerance. Default is 1e-5; see `DEFAULT_RTOL`. + /// * `atol` - Optional absolute tolerance. Default is 1e-8; see `DEFAULT_ATOL`. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensors. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor1.is_close(tensor2, None, None); + /// println!("{tensor}"); + /// // [[true, true, true], [true, true, true]] + /// } + /// ``` + pub fn is_close(self, other: Self, rtol: Option, atol: Option) -> Tensor { + let rtol = rtol.unwrap_or(DEFAULT_RTOL); + let atol = atol.unwrap_or(DEFAULT_ATOL); + + // check finite difference is close + let is_close_finite_val = self + .clone() + .sub(other.clone()) + .abs() + .lower_equal(other.clone().abs().mul_scalar(rtol).add_scalar(atol)) + .bool_and(self.clone().is_finite()) + .bool_and(other.clone().is_finite()); + + // check if both are infinite and have same sign + let inf_same_sign = self + .clone() + .is_finite() + .bool_not() + .bool_and(other.clone().is_finite().bool_not()) + .bool_and(self.equal(other)); + + is_close_finite_val.bool_or(inf_same_sign) + } + + /// Checks if all elements are close to another tensor. + /// + /// The tolerance is defined by the following equation: + /// + /// ```text + /// + /// abs(a - b) <= (atol + rtol * abs(b)) + /// + /// where `a` is the first tensor, `b` is the second tensor, `rtol` is the relative tolerance, + /// and `atol` is the absolute tolerance. + /// + /// ``` + /// + /// # Arguments + /// + /// * `other` - The tensor to compare with. + /// * `rtol` - Optional relative tolerance. Default is 1e-5; see `DEFAULT_RTOL`. + /// * `atol` - Optional absolute tolerance. Default is 1e-8; see `DEFAULT_ATOL`. + /// + /// # Returns + /// + /// A boolean scalar. + /// + /// # Remarks + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let result = tensor1.all_close(tensor2, None, None); + /// println!("{}", result); + /// // true + /// } + /// ``` + pub fn all_close(self, other: Self, rtol: Option, atol: Option) -> bool { + self.is_close(other, rtol, atol) + .all() + .into_scalar() + .to_bool() + } + + /// Returns a new tensor with boolean elements indicating whether each element of the input is NaN. + /// + /// # Returns + /// + /// A boolean tensor where `true` indicates NaN and `false` indicates a non-NaN value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, f64::NAN, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.is_nan(); + /// println!("{tensor}"); + /// // [[false, true, false], [false, false, false]] + /// } + /// ``` + pub fn is_nan(self) -> Tensor { + Tensor::new(B::float_is_nan(self.primitive.tensor())) + } + + /// Checks if the tensor contains any NaN values. + /// + /// # Returns + /// + /// A boolean tensor with a single element indicating whether the tensor contains any NaN values. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [f64::NAN, 9.0, 6.0]], &device); + /// let tensor = tensor.contains_nan(); + /// println!("{tensor}"); + /// // [true] + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.contains_nan(); + /// println!("{tensor}"); + /// // [false] + /// } + /// ``` + pub fn contains_nan(self) -> Tensor { + // Summing the tensor will result in NaN if the tensor contains any NaN values + // This is faster than checking each element individually + // because it rolls up the NaN values into a single value + let sum = self.sum(); + + sum.is_nan() + } + + /// Returns a new tensor with boolean elements indicating whether each element of the input is infinite (either +INF or -INF). + /// + /// # Returns + /// + /// A boolean tensor where `true` indicates that the value is infinite + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, f64::INFINITY, 3.0], [f64::NAN, 9.0, 6.0]], &device); + /// let tensor = tensor.is_finite(); + /// println!("{tensor}"); + /// // [[false, true, false], [false, false, false]] + /// } + /// ``` + pub fn is_inf(self) -> Tensor { + Tensor::new(B::float_is_inf(self.primitive.tensor())) + } + + /// Returns a new tensor with boolean elements indicating whether each element of the input is finite + /// + /// # Returns + /// + /// A boolean tensor where `true` indicates that the value is finite and `false` indicates + /// either INF, -INF or NAN + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Bool, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, f64::INFINITY, 3.0], [f64::NAN, 9.0, 6.0]], &device); + /// let tensor = tensor.is_finite(); + /// println!("{tensor}"); + /// // [[true, false, true], [false, true, true]] + /// } + /// ``` + pub fn is_finite(self) -> Tensor { + self.clone() + .is_nan() + .bool_not() + .bool_and(self.is_inf().bool_not()) + } + + /// Samples tensor as a two-dimensional spatial grid of (possibly multi-channel) values, + /// using the given locations in [-1, 1]. + /// + /// # Arguments + /// + /// * `grid` - A tensor of locations, with shape (N, H_out, W_out, 2). Values are [-1, 1]. + /// A [x = -1, y = -1] means top-left, and [x = 1, y = 1] means bottom-right + /// * `options` - Grid sampling options (mode, padding_mode, align_corners) + /// + /// # Returns + /// + /// A tensor with shape (N, C, H_out, W_out) + /// + /// # Example + /// + /// ```ignore + /// use burn_tensor::ops::{GridSampleOptions, GridSamplePaddingMode, InterpolateMode}; + /// + /// // Default options (bilinear, zeros padding, align_corners=false) + /// let output = tensor.grid_sample_2d(grid, GridSampleOptions::default()); + /// + /// // Custom options + /// let options = GridSampleOptions::new(InterpolateMode::Bilinear) + /// .with_padding_mode(GridSamplePaddingMode::Border) + /// .with_align_corners(true); + /// let output = tensor.grid_sample_2d(grid, options); + /// ``` + pub fn grid_sample_2d( + self, + grid: Tensor, + options: impl Into, + ) -> Tensor { + Tensor::new(TensorPrimitive::Float(B::float_grid_sample_2d( + self.primitive.tensor(), + grid.primitive.tensor(), + options.into(), + ))) + } + + /// Computes the cross product of `self` and another tensor along a given dimension. + /// + /// Both `self` and `other` **must have size 3** along the specified `dim`, + /// because the cross product is only defined in three-dimensional space. + /// + /// # Arguments + /// + /// * `other` - The other tensor to take the cross product with. + /// * `dim` - The dimension along which to compute the cross product. + /// + /// # Returns + /// + /// A tensor containing the cross product of `self` and `other` along `dim`. + pub fn cross(self, other: Tensor, dim: Dim) -> Tensor { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::cross(&self, &other, dim)); + Tensor::new(TensorPrimitive::Float(B::float_cross( + self.primitive.tensor(), + other.primitive.tensor(), + dim, + ))) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/fmod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/fmod.rs new file mode 100644 index 0000000..efb9c0f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/fmod.rs @@ -0,0 +1,111 @@ +use crate::{Float, Tensor, backend::Backend}; + +impl Tensor +where + B: Backend, +{ + /// Computes the floating-point remainder of dividing `self` by `other`. + /// + /// The result has the same sign as `self` and magnitude less than `other`. + /// This is equivalent to the IEEE 754 remainder operation. + /// + /// # Special Cases (IEEE 754 compliant) + /// + /// - If `self` is ±∞ and `other` is not NaN, NaN is returned + /// - If `other` is ±0 and `self` is not NaN, NaN is returned + /// - If `other` is ±∞ and `self` is finite, `self` is returned + /// - If either argument is NaN, NaN is returned + /// + /// # Arguments + /// + /// * `other` - The divisor tensor. Must have the same shape as `self`. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the floating-point remainder. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let dividend = Tensor::::from_data([5.3, -5.3, 5.3, -5.3], &device); + /// let divisor = Tensor::::from_data([2.0, 2.0, -2.0, -2.0], &device); + /// let result = dividend.fmod(divisor); + /// + /// // Result: [1.3, -1.3, 1.3, -1.3] + /// } + /// ``` + pub fn fmod(self, other: Self) -> Self { + // Normal case: fmod(x, y) = x - y * trunc(x / y) + let quotient = self.clone().div(other.clone()); + let truncated = quotient.trunc(); + let product = other.clone() * truncated.clone(); + + // When divisor is infinity and dividend is finite: + // - quotient is 0, truncated is 0 + // - but 0 * infinity = NaN, which is wrong + // We need to handle this case by replacing NaN with 0 when appropriate + + // Check if the product is NaN due to 0 * inf + let is_zero_times_inf = truncated.equal_elem(0.0).bool_and(other.is_inf()); + let zero_tensor = self.clone().mul_scalar(0.0); + let corrected_product = product.mask_where(is_zero_times_inf, zero_tensor); + + self - corrected_product + } + + /// Computes the floating-point remainder of dividing `self` by a scalar. + /// + /// The result has the same sign as `self` and magnitude less than the scalar. + /// + /// # Special Cases (IEEE 754 compliant) + /// + /// - If `self` is ±∞ and scalar is not NaN, NaN is returned + /// - If scalar is ±0 and `self` is not NaN, NaN is returned + /// - If scalar is ±∞ and `self` is finite, `self` is returned + /// - If either argument is NaN, NaN is returned + /// + /// # Arguments + /// + /// * `scalar` - The scalar divisor. + /// + /// # Returns + /// + /// A tensor with the same shape where each element is the floating-point remainder. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([5.3, -5.3, 7.5, -7.5], &device); + /// let result = tensor.fmod_scalar(2.0); + /// + /// // Result: [1.3, -1.3, 1.5, -1.5] + /// } + /// ``` + pub fn fmod_scalar(self, scalar: f32) -> Self { + // Normal case: fmod(x, y) = x - y * trunc(x / y) + let quotient = self.clone().div_scalar(scalar); + let truncated = quotient.trunc(); + let product = truncated.mul_scalar(scalar); + + // Handle the special case where scalar is infinity + // When scalar is ±∞ and self is finite, quotient is 0, truncated is 0 + // but 0 * infinity = NaN, which is wrong - it should be 0 + if scalar.is_infinite() { + // For finite values, fmod(x, ±∞) = x + // For infinite values, fmod(±∞, ±∞) = NaN (which is handled by arithmetic) + return self; + } + + self - product + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/int.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/int.rs new file mode 100644 index 0000000..fe76fe7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/int.rs @@ -0,0 +1,182 @@ +use burn_backend::Scalar; + +use crate::{ + Float, Int, IntDType, Shape, Tensor, TensorData, TensorPrimitive, backend::Backend, + cartesian_grid, +}; + +use core::ops::Range; + +impl Tensor +where + B: Backend, +{ + /// Returns a new integer tensor on the specified device. + /// + /// # Arguments + /// + /// * `range` - The range of values to generate. + /// * `device` - The device to create the tensor on. + pub fn arange(range: Range, device: &B::Device) -> Self { + Tensor::new(B::int_arange(range, device)) + } + + /// Returns a new integer tensor on the specified device. + /// + /// # Arguments + /// + /// * `range` - The range of values to generate. + /// * `step` - The step between each value. + pub fn arange_step(range: Range, step: usize, device: &B::Device) -> Self { + Tensor::new(B::int_arange_step(range, step, device)) + } +} + +impl Tensor +where + B: Backend, +{ + /// Create a tensor from integers (i32), placing it on a given device. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Int}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let _x: Tensor = Tensor::from_ints([1, 2], &device); + /// let _y: Tensor = Tensor::from_ints([[1, 2], [3, 4]], &device); + /// } + /// ``` + pub fn from_ints>(ints: A, device: &B::Device) -> Self { + Self::from_data(ints.into().convert::(), device) + } + + /// Returns a new tensor with the same shape and device as the current tensor and the data + /// cast to Float. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let int_tensor = Tensor::::arange(0..5, &device); + /// let float_tensor = int_tensor.float(); + /// } + /// ``` + pub fn float(self) -> Tensor { + Tensor::new(TensorPrimitive::Float(B::int_into_float(self.primitive))) + } + + /// Generates a cartesian grid for the given tensor shape on the specified device. + /// The generated tensor is of dimension `D2 = D + 1`, where each element at dimension D contains the cartesian grid coordinates for that element. + /// + /// # Arguments + /// + /// * `shape` - The shape specifying the dimensions of the tensor. + /// * `device` - The device to create the tensor on. + /// + /// # Panics + /// + /// Panics if `D2` is not equal to `D+1`. + /// + /// # Examples + /// + /// ```rust + /// use burn_tensor::Int; + /// use burn_tensor::{backend::Backend, Shape, Tensor}; + /// fn example() { + /// let device = Default::default(); + /// let result: Tensor = Tensor::::cartesian_grid([2, 3], &device); + /// println!("{}", result); + /// } + /// ``` + pub fn cartesian_grid, const D2: usize>( + shape: S, + device: &B::Device, + ) -> Tensor { + cartesian_grid::(shape, device) + } + + /// Applies the bitwise logical and operation with each bit representing the integer. + pub fn bitwise_and(self, other: Self) -> Self { + Self::new(B::bitwise_and(self.primitive, other.primitive)) + } + + /// Applies the bitwise logical or operation with another tensor. + pub fn bitwise_or(self, other: Self) -> Self { + Self::new(B::bitwise_or(self.primitive, other.primitive)) + } + + /// Applies the bitwise logical xor operation with another tensor. + pub fn bitwise_xor(self, other: Self) -> Self { + Self::new(B::bitwise_xor(self.primitive, other.primitive)) + } + + /// Applies the bitwise logical not operation. + pub fn bitwise_not(self) -> Self { + Self::new(B::bitwise_not(self.primitive)) + } + + /// Applies the bitwise logical and operation with each bit in the scalar and the integers in the tensor. + pub fn bitwise_and_scalar(self, other: B::IntElem) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(B::bitwise_and_scalar(self.primitive, other)) + } + + /// Applies the bitwise logical or operation with each bit in the scalar and the integers in the tensor. + pub fn bitwise_or_scalar(self, other: B::IntElem) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(B::bitwise_or_scalar(self.primitive, other)) + } + + /// Applies bitwise logical xor operation with each bit in the scalar and the integers in the tensor. + pub fn bitwise_xor_scalar(self, other: B::IntElem) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(B::bitwise_xor_scalar(self.primitive, other)) + } + + /// Applies the bitwise left shift operation with the integers in the tensor. + pub fn bitwise_left_shift(self, other: Self) -> Self { + Self::new(B::bitwise_left_shift(self.primitive, other.primitive)) + } + + /// Applies the bitwise right shift operation with the integers in the tensor. + pub fn bitwise_right_shift(self, other: Self) -> Self { + Self::new(B::bitwise_right_shift(self.primitive, other.primitive)) + } + + /// Applies the bitwise left shift operation with the scalar. + pub fn bitwise_left_shift_scalar(self, other: B::IntElem) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(B::bitwise_left_shift_scalar(self.primitive, other)) + } + + /// Applies the bitwise right shift operation with the scalar. + pub fn bitwise_right_shift_scalar(self, other: B::IntElem) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(B::bitwise_right_shift_scalar(self.primitive, other)) + } + + /// Converts a tensor to the specified integer data type. + /// + /// This is always a no-op when casting to the current dtype. + /// + /// # Warning + /// Most backends don't have automatic type promotion at this time, so make sure that all tensors + /// have the same integer data type for operations multiple input tensors (e.g., binary ops). + pub fn cast>(self, dtype: F) -> Tensor { + let dtype = dtype.into(); + let self_dtype: IntDType = self.dtype().into(); + if dtype == self_dtype { + // no-op. + return self; + } + Tensor::new(B::int_cast(self.primitive, dtype)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/mod.rs new file mode 100644 index 0000000..dfb4f5a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/mod.rs @@ -0,0 +1,27 @@ +pub(crate) mod check; + +mod autodiff; +mod base; +mod bool; +mod cartesian_grid; +mod float; +mod fmod; +mod int; +mod numeric; +mod options; +mod orderable; +mod pad; +pub use pad::IntoPadding; +mod take; +mod transaction; +mod trunc; + +pub use autodiff::*; +pub use base::*; +pub use cartesian_grid::cartesian_grid; +pub use float::{DEFAULT_ATOL, DEFAULT_RTOL}; +pub use numeric::*; +pub use options::*; +pub use transaction::*; + +pub use burn_backend::tensor::IndexingUpdateOp; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/numeric.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/numeric.rs new file mode 100644 index 0000000..dd7a3a0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/numeric.rs @@ -0,0 +1,1255 @@ +use burn_backend::Scalar; +pub use burn_backend::tensor::Numeric; + +use crate::alloc::borrow::ToOwned; +use alloc::vec::Vec; + +use crate::IndexingUpdateOp; +use crate::{ + AsIndex, Bool, Distribution, Element, ElementConversion, Int, Shape, Tensor, backend::Backend, + check, check::TensorCheck, +}; + +impl Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + /// Applies element wise addition operation. + /// + /// `y = x2 + x1` + /// + /// # Arguments + /// + /// * `other` - The tensor to add. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1 + tensor2; + /// println!("{tensor}"); + /// // [[3.0, 1.0, 7.0], [6.0, 11.0, 9.0]] + /// } + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn add(self, other: Self) -> Self { + check!(TensorCheck::binary_ops_ew("Add", &self, &other)); + Self::new(K::add(self.primitive, other.primitive)) + } + + /// Applies element wise addition operation with a scalar. + /// + /// `y = x + s` + /// + /// # Arguments + /// + /// * `other` - The scalar to add, element wise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let scalar = 2.0; + /// let tensor = tensor + scalar; + /// println!("{tensor}"); + /// // [[3.0, 0.0, 5.0], [7.0, 11.0, 8.0]] + /// } + /// ``` + pub fn add_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::add_scalar(self.primitive, other)) + } + + /// Applies element wise subtraction operation. + /// + /// `y = x2 - x1` + /// + /// # Arguments + /// + /// * `other` - The tensor to subtract. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1 - tensor2; + /// println!("{tensor}"); + /// // [[-1.0, -5.0, -1.0], [4.0, 7.0, 3.0]] + /// } + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn sub(self, other: Self) -> Self { + check!(TensorCheck::binary_ops_ew("Sub", &self, &other)); + Self::new(K::sub(self.primitive, other.primitive)) + } + + /// Applies element wise subtraction operation with a scalar. + /// + /// `y = x - s` + /// + /// # Arguments + /// + /// * `other` - The scalar to subtract, element wise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let scalar = 2.0; + /// let tensor = tensor - scalar; + /// println!("{tensor}"); + /// // [[-1.0, -4.0, 1.0], [3.0, 7.0, 4.0]] + /// } + /// ``` + pub fn sub_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::sub_scalar(self.primitive, other)) + } + + /// Applies element wise division operation. + /// + /// `y = x2 / x1` + /// + /// # Arguments + /// + /// * `other` - The tensor to divide. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1 / tensor2; + /// println!("{tensor}"); + /// // [[0.5, -0.6666667, 0.75], [5.0, 4.5, 2.0]] + /// } + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn div(self, other: Self) -> Self { + check!(TensorCheck::binary_ops_ew("Div", &self, &other)); + Self::new(K::div(self.primitive, other.primitive)) + } + + /// Applies element wise division operation with a scalar. + /// + /// `y = x / s` + /// + /// # Arguments + /// + /// * `other` - The scalar to divide, element wise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let scalar = 2.0; + /// let tensor = tensor / scalar; + /// println!("{tensor}"); + /// // [[0.5, -1.0, 1.5], [2.5, 4.5, 3.0]] + /// } + /// ``` + pub fn div_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::div_scalar(self.primitive, other)) + } + + /// Applies element wise the remainder operation with a scalar. + /// + /// `y = x2 % x1` + pub fn remainder(self, other: Self) -> Self { + Self::new(K::remainder(self.primitive, other.primitive)) + } + + /// Applies element wise the remainder operation with a scalar. + /// + /// `y = x % s` + /// + /// # Arguments + /// + /// * `other` - The scalar to divide, element wise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let scalar = 2.0; + /// let tensor = tensor1 % scalar; + /// println!("{tensor}"); + /// // [[1.0, 0.0, 1.0], [1.0, 1.0, 0.0]] + /// } + /// ``` + pub fn remainder_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::remainder_scalar(self.primitive, other)) + } + + /// Applies element wise multiplication operation. + /// + /// `y = x2 * x1` + /// + /// # Arguments + /// + /// * `other` - The tensor to multiply. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1 * tensor2; + /// println!("{tensor}"); + /// // [[2.0, -6.0, 12.0], [5.0, 18.0, 18.0]] + /// } + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn mul(self, other: Self) -> Self { + check!(TensorCheck::binary_ops_ew("Mul", &self, &other)); + Self::new(K::mul(self.primitive, other.primitive)) + } + + /// Applies element wise multiplication operation with a scalar. + /// + /// `y = x * s` + /// + /// # Arguments + /// + /// * `other` - The scalar to multiply, element wise. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let scalar = 2.0; + /// let tensor = tensor * scalar; + /// println!("{tensor}"); + /// // [[2.0, -4.0, 6.0], [10.0, 18.0, 12.0]] + /// } + /// ``` + pub fn mul_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::mul_scalar(self.primitive, other)) + } + + /// Switch sign of each element in the tensor. + /// + /// `y = -x` + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = -tensor; + /// println!("{tensor}"); + /// // [[-1.0, 2.0, -3.0], [-5.0, -9.0, -6.0]] + /// } + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn neg(self) -> Self { + Self::new(K::neg(self.primitive)) + } + + /// Returns the signs of the elements of the input tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.sign(); + /// println!("{tensor}"); + /// // [[1.0, -1.0, 1.0], [1.0, 1.0, 1.0]] + /// } + /// ``` + pub fn sign(self) -> Self { + Self::new(K::sign(self.primitive)) + } + + /// Aggregate all elements in the tensor with the mean operation. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.mean(); + /// println!("{tensor}"); + /// // [3.6666667] + /// } + /// ``` + pub fn mean(self) -> Tensor { + Tensor::new(K::mean(self.primitive)) + } + + /// Aggregate all elements in the tensor with the sum operation. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.sum(); + /// println!("{tensor}"); + /// // [22.0] + /// } + /// ``` + pub fn sum(self) -> Tensor { + Tensor::new(K::sum(self.primitive)) + } + + /// Aggregate all elements along the given *dimension* or *axis* + /// in the tensor with the mean operation. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to aggregate the elements; + /// supports negative indexing. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.clone().mean_dim(0); + /// println!("{tensor}"); + /// // [[3.0, 3.5, 4.5]] + /// let tensor = tensor.clone().mean_dim(1); + /// println!("{tensor}"); + /// // [[0.6666667], [6.6666665]] + /// } + /// ``` + pub fn mean_dim(self, dim: I) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Mean", dim)); + Self::new(K::mean_dim(self.primitive, dim)) + } + + /// Aggregate all elements along the given *axes* + /// in the tensor with the mean operation. + /// + /// # Arguments + /// + /// * `dims` - the dimensions to aggregate; supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[2.0, 4.0], [6.0, -4.0]], &device); + /// let tensor = tensor.clone().mean_dims(&[0, 1]); + /// println!("{tensor}"); + /// // [[2.0]] + /// } + /// ``` + pub fn mean_dims(self, dims: &[I]) -> Self { + dims.iter().fold(self, |tensor, &dim| tensor.mean_dim(dim)) + } + + /// Aggregate all elements along the given *dimension* or *axis* + /// in the tensor with the sum operation. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to aggregate the elements; + /// supports negative indexing. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.clone().sum_dim(0); + /// println!("{tensor}"); + /// // [[6.0, 7.0, 9.0]] + /// let tensor = tensor.clone().sum_dim(1); + /// println!("{tensor}"); + /// // [[2.0], [20.0]] + /// } + /// ``` + pub fn sum_dim(self, dim: I) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Sum", dim)); + Self::new(K::sum_dim(self.primitive, dim)) + } + + /// Aggregate all elements along the given *axes* + /// in the tensor with the sum operation. + /// + /// # Arguments + /// + /// * `dims` - the dimensions to aggregate; supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.clone().sum_dims(&[0, 1]); + /// println!("{tensor}"); + /// // [[27]] + /// } + /// ``` + pub fn sum_dims(self, dims: &[I]) -> Self { + dims.iter().fold(self, |tensor, &dim| tensor.sum_dim(dim)) + } + + /// Aggregate and squeeze along the given dimensions. + /// + /// This is equivalent to ``tensor.sum_dims(dims).squeeze_dims(dims)`` + /// + /// # Arguments + /// + /// * `dims` - the dimensions to aggregate; supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([ + /// [[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], + /// [[9.0, 2.0, 5.0], [5.0, 7.0, 7.0]], + /// ], &device); + /// let tensor = tensor.clone().sum_dims_squeeze::<1, _>(&[0, 1]); + /// println!("{tensor}"); + /// // [20.0, 16.0, 21.0] + /// } + /// ``` + pub fn sum_dims_squeeze(self, dims: &[I]) -> Tensor { + // TODO: remove idims when squeeze_dims uses AsIndex. + let idims = dims + .iter() + .map(|&dim| (dim.expect_dim_index(D)) as isize) + .collect::>(); + self.sum_dims(dims).squeeze_dims::(&idims) + } + + /// Aggregate all elements in the tensor with the product operation. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.prod(); + /// println!("{tensor}"); + /// // [-1620.0] + /// } + /// ``` + pub fn prod(self) -> Tensor { + Tensor::new(K::prod(self.primitive)) + } + + /// Aggregate all elements along the given *dimension* or *axis* + /// in the tensor with the product operation. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to aggregate the elements, + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimension will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.clone().prod_dim(0); + /// println!("{tensor}"); + /// // [[5.0, -18.0, 18.0]] + /// let tensor = tensor.clone().prod_dim(1); + /// println!("{tensor}"); + /// // [[-6.0], [270.0]] + /// } + /// ``` + pub fn prod_dim(self, dim: I) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Prod", dim)); + Self::new(K::prod_dim(self.primitive, dim)) + } + + /// Aggregate all elements along the given *axes* + /// in the tensor with the prod operation. + /// + /// # Arguments + /// + /// * `dims` - the dimensions to aggregate, supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.clone().sum_dims(&[0, 1]); + /// println!("{tensor}"); + /// // [[-1620.0]] + /// } + /// ``` + pub fn prod_dims(self, dims: &[I]) -> Self { + dims.iter().fold(self, |tensor, &dim| tensor.prod_dim(dim)) + } + + /// Computes the cumulative sum of elements along the given *dimension* or *axis*. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to compute the cumulative sum. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + /// let result = tensor.clone().cumsum(0); + /// println!("{result}"); + /// // [[1.0, 2.0, 3.0], [5.0, 7.0, 9.0]] + /// let result = tensor.cumsum(1); + /// println!("{result}"); + /// // [[1.0, 3.0, 6.0], [4.0, 9.0, 15.0]] + /// } + /// ``` + pub fn cumsum(self, dim: usize) -> Self { + check!(TensorCheck::aggregate_dim::("CumSum", dim)); + Self::new(K::cumsum(self.primitive, dim)) + } + + /// Computes the cumulative product of elements along the given *dimension* or *axis*. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to compute the cumulative product. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + /// let result = tensor.clone().cumprod(0); + /// println!("{result}"); + /// // [[1.0, 2.0, 3.0], [4.0, 10.0, 18.0]] + /// let result = tensor.cumprod(1); + /// println!("{result}"); + /// // [[1.0, 2.0, 6.0], [4.0, 20.0, 120.0]] + /// } + /// ``` + pub fn cumprod(self, dim: usize) -> Self { + check!(TensorCheck::aggregate_dim::("CumProd", dim)); + Self::new(K::cumprod(self.primitive, dim)) + } + + /// Apply element wise absolute value operation. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_ints([[1, -2, 3], [4, -5, 6], [7, -8, 9]], &device); + /// let tensor = tensor.abs(); + /// println!("{tensor}"); + /// // [[1, 2, 3], [4, 5, 6], [7, 8, 9]] + /// } + /// ``` + /// + /// # Notes + /// + /// For signed integer dtypes, this operation uses two's-complement wraparound semantics, similar to + /// `x.wrapping_abs()`. For example, `abs(i64::MIN) == i64::MIN`. + pub fn abs(self) -> Self { + Self::new(K::abs(self.primitive)) + } + + /// Returns the upper triangular part of a matrix (2-D tensor) or batch of matrices input, + /// the other elements of the result tensor out are set to 0. + /// + /// See also [`triu_mask`](Tensor::triu_mask). + /// + /// # Arguments + /// + /// * `diagonal` - The offset from the diagonal, where 0 means the diagonal, and positive values shift + /// towards the upper triangle. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_ints( + /// [ + /// [1, 2, 3], + /// [4, 5, 6], + /// [7, 8, 9] + /// ], + /// &device + /// ); + /// let tensor = tensor.triu(1); + /// println!("{tensor}"); + /// // [ + /// // [0, 2, 3], + /// // [0, 0, 6], + /// // [0, 0, 0] + /// // ] + /// } + /// ``` + pub fn triu(self, diagonal: i64) -> Self { + check!(TensorCheck::tri::<{ D }>()); + + // last two dimensions + let shape = &self.shape()[D - 2..].to_owned(); + + let mask = Tensor::::triu_mask(shape, diagonal, &self.device()).unsqueeze(); + self.mask_fill(mask, 0) + } + + /// Returns the lower triangular part of a matrix (2-D tensor) or batch of matrices input, + /// the other elements of the result tensor out are set to 0. + /// + /// See also [`tril_mask`](Tensor::tril_mask). + /// + /// # Arguments + /// + /// * `diagonal` - The offset from the diagonal, where 0 means the diagonal, and positive values shift + /// towards the upper triangle. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_ints( + /// [ + /// [1, 2, 3], + /// [4, 5, 6], + /// [7, 8, 9] + /// ], + /// &device + /// ); + /// + /// let tensor = tensor.tril(-1); + /// println!("{tensor}"); + /// // [ + /// // [0, 0, 0], + /// // [4, 0, 0], + /// // [7, 8, 0] + /// // ] + /// } + /// ``` + pub fn tril(self, diagonal: i64) -> Self { + check!(TensorCheck::tri::<{ D }>()); + + // last two dimensions + let shape = &self.shape()[D - 2..].to_owned(); + let mask = Tensor::::tril_mask(shape, diagonal, &self.device()).unsqueeze(); + + self.mask_fill(mask, 0) + } + + /// Applies element wise power operation with a float Tensor + /// + /// # Arguments + /// + /// * `other` - The tensor to apply the power operation with. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.powf(tensor2); + /// println!("{tensor}"); + /// // [[1.0, 8.0, 81.0], [5.0, 81.0, 216.0]] + /// } + /// ``` + pub fn powf(self, other: Self) -> Self { + Self::new(K::powf(self.primitive, other.primitive)) + } + + /// Applies element wise power operation with a float scalar + /// + /// # Arguments + /// + /// * `other` - The scalar to apply the power operation with. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.powf_scalar(2.0); + /// println!("{tensor}"); + /// // [[1.0, 4.0, 9.0], [25.0, 81.0, 36.0]] + /// } + /// ``` + pub fn powf_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::powf_scalar(self.primitive, other)) + } + + /// Applies element wise power operation with a integer Tensor + /// + /// # Arguments + /// + /// * `other` - The tensor to apply the power operation with. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape, Int}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_ints([[1, -2, 3], [5, 9, 6]], &device); + /// let tensor2 = Tensor::::from_ints([[2, 3, 4], [1, 2, 3]], &device); + /// let tensor = tensor1.powi(tensor2); + /// println!("{tensor}"); + /// // [[1, -8, 81], [5, 81, 216]] + /// } + /// ``` + pub fn powi(self, other: Self) -> Self { + Self::new(K::powi(self.primitive, other.primitive)) + } + + /// Applies element wise power operation with a integer scalar + /// + /// # Arguments + /// + /// * `other` - The scalar to apply the power operation with. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape, Int}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_ints([[1, -2, 3], [5, 9, 6]], &device); + /// let tensor = tensor.powi_scalar(2); + /// println!("{tensor}"); + /// + /// // [[1, 4, 9], [25, 81, 36]] + /// let tensor = Tensor::::from_data([[1.5, -2., 3.], [5., 9., 6.]], &device); + /// let tensor = tensor.powi_scalar(2); + /// println!("{tensor}"); + /// // [[2.25, 4., 9.], [25., 81., 36.]] + /// } + /// ``` + pub fn powi_scalar(self, other: E) -> Self { + let other = Scalar::new(other, &self.dtype()); + Self::new(K::powi_scalar(self.primitive, other)) + } + + /// Converts the tensor to a boolean tensor by checking if the elements are non-zero. + /// + /// # Returns + /// + /// A boolean tensor with the same shape as the input tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [0.0, 9.0, 6.0]], &device); + /// let tensor = tensor.bool(); + /// println!("{tensor}"); + /// // [ + /// // [true, true, true], + /// // [false, true, true] + /// // ] + /// } + pub fn bool(self) -> Tensor { + self.not_equal_elem(0) + } + + /// Create a random tensor of the given shape on the given device where each element is + /// sampled from the given distribution. + /// + /// See also [`random_like`](Tensor::random_like). + /// + /// # Arguments + /// + /// * `shape` - The shape of the tensor. + /// * `distribution` - The distribution to sample from. + /// * `device` - The device to create the tensor on. + /// + /// # Returns + /// + /// A new tensor with the given shape and elements sampled from the given distribution. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape, Distribution}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let distribution = Distribution::Uniform(0.0, 1.0); // Any random value between 0.0 and 1.0 + /// let tensor = Tensor::::random(Shape::new([2, 3]), distribution, &device); + /// println!("{tensor}"); + /// // [ + /// // [0.08347523, 0.70498955, 0.60332155], + /// // [0.08173251, 0.18028641, 0.97942924] + /// // ] + /// } + /// ``` + pub fn random>( + shape: S, + distribution: Distribution, + device: &B::Device, + ) -> Self { + Self::new(K::random(shape.into(), distribution, device)) + } + + /// Applies the matrix multiplication operation. + /// + /// ```math + /// C = AB + /// ``` + /// + /// Shapes of the form `[..., B, 1, K] @ [..., 1, K, N]` are reinterpreted as + /// `[..., 1, B, K] @ [..., 1, K, N]`, turning a batched vec-mat into a general + /// matmul, which is often faster. + pub fn matmul(self, other: Self) -> Self { + check!(TensorCheck::matmul(&self, &other)); + + if D >= 3 { + let batch_index = D - 3; + let vector_index = D - 2; + let lhs_dims = &self.shape()[batch_index..D]; + let rhs_dims = &other.shape()[batch_index..D]; + + if let ([_, 1, k1], [1, k2, _]) = (lhs_dims, rhs_dims) + && k1 == k2 + { + return Tensor::new(K::matmul( + self.swap_dims(batch_index, vector_index).primitive, + other.primitive, + )) + .swap_dims(batch_index, vector_index); + } + } + + Tensor::new(K::matmul(self.primitive, other.primitive)) + } +} + +impl Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + /// Calculates the dot product with another tensor. + /// + /// `y = x2.dot(x1)` + /// + /// # Arguments + /// + /// * `other` - The tensor to compute dot product with. + /// + /// # Notes + /// + /// Both tensors must have the same number of elements. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([1.0, 2.0], &device); + /// let tensor2 = Tensor::::from_data([-2.0, 3.0], &device); + /// let tensor = tensor1.dot(tensor2); + /// println!("{tensor}"); + /// // [4] + /// } + /// ``` + pub fn dot(self, other: Self) -> Self { + self.mul(other).sum() + } +} + +impl Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + /// Creates a new 2D tensor with ones on the diagonal and zeros elsewhere. + /// + /// # Arguments + /// + /// * `size` - The size of the square matrix. + pub fn eye(size: usize, device: &B::Device) -> Self { + let indices = Tensor::::arange(0..size as i64, device).unsqueeze::<2>(); + let ones = Self::ones([1, size], device); + let zeros = Self::zeros([size, size], device); + + zeros.scatter(0, indices, ones, IndexingUpdateOp::Add) + } +} + +// Tensor + tensor +impl> core::ops::Add for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self::add(self, rhs) + } +} + +// Tensor + scalar +impl> core::ops::Add + for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn add(self, other: E) -> Self::Output { + Tensor::add_scalar(self, other) + } +} + +// Scalar + tensor +macro_rules! impl_tensor_scalar_add { + ($($t:ty),*) => { + $( + impl> core::ops::Add> for $t + where + K::Elem: Element, + { + type Output = Tensor; + + fn add(self, tensor: Tensor) -> Self::Output { + Tensor::add_scalar(tensor, self) + } + } + )* + } +} +impl_tensor_scalar_add!(f32, f64, i32, i64, u32, u64); + +// Tensor - tensor +impl> core::ops::Sub for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Tensor::sub(self, rhs) + } +} + +// Tensor - scalar +impl> core::ops::Sub + for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn sub(self, other: E) -> Self::Output { + Tensor::sub_scalar(self, other) + } +} + +// Scalar - tensor +macro_rules! impl_tensor_scalar_sub { + ($($t:ty),*) => { + $( + impl> core::ops::Sub> for $t + where + K::Elem: Element, + { + type Output = Tensor; + + fn sub(self, tensor: Tensor) -> Self::Output { + Tensor::add_scalar(Tensor::neg(tensor), self) + } + } + )* + } +} +impl_tensor_scalar_sub!(f32, f64, i32, i64, u32, u64); + +// Tensor / tensor +impl> core::ops::Div for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn div(self, rhs: Self) -> Self::Output { + Tensor::div(self, rhs) + } +} + +// Tensor / scalar +impl> core::ops::Div + for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn div(self, other: E) -> Self::Output { + Tensor::div_scalar(self, other) + } +} + +// Scalar / tensor (float only) +macro_rules! impl_tensor_scalar_div { + ($($t:ty),*) => { + $( + impl core::ops::Div> for $t + { + type Output = Tensor; + + fn div(self, tensor: Tensor) -> Self::Output { + tensor.recip().mul_scalar(self) + } + } + )* + } +} + +impl_tensor_scalar_div!(f32, f64); + +// Tensor % tensor. +impl> core::ops::Rem for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn rem(self, rhs: Self) -> Self::Output { + Tensor::remainder(self, rhs) + } +} + +// Tensor % scalar. +impl> core::ops::Rem + for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn rem(self, other: E) -> Self::Output { + Tensor::remainder_scalar(self, other) + } +} + +// Tensor * tensor. +impl> core::ops::Mul for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn mul(self, rhs: Self) -> Self::Output { + Tensor::mul(self, rhs) + } +} + +// Tensor * scalar. +impl> core::ops::Mul + for Tensor +where + K::Elem: Element, +{ + type Output = Self; + + fn mul(self, other: E) -> Self::Output { + Tensor::mul_scalar(self, other) + } +} + +macro_rules! impl_tensor_scalar_mul { + ($($t:ty),*) => { + $( + impl> core::ops::Mul> for $t + where + K::Elem: Element, + { + type Output = Tensor; + + fn mul(self, other: Tensor) -> Self::Output { + Tensor::mul_scalar(other, self) + } + } + )* + } +} + +impl_tensor_scalar_mul!(f32, f64, i32, i64, u32, u64); + +impl core::ops::Neg for Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + type Output = Self; + + fn neg(self) -> Self::Output { + Tensor::neg(self) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/options.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/options.rs new file mode 100644 index 0000000..e78b631 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/options.rs @@ -0,0 +1,116 @@ +use burn_backend::{Backend, Element, tensor::Device}; +use burn_std::DType; + +use crate::get_device_policy; + +/// Options for tensor creation. +/// +/// This struct allows specifying the `device` and overriding the data type when creating a tensor. +/// When the `dtype` is not specified, the [device's default policy](crate::set_default_dtypes) is used. +#[derive(Debug, Clone)] +pub struct TensorCreationOptions { + /// Device where the tensor will be created. + pub device: Device, + /// Optional data type. + /// If `None`, the dtype will be inferred on creation from the [device policy](crate::set_default_dtypes). + pub dtype: Option, +} + +impl Default for TensorCreationOptions { + /// Returns new options with the backend's default device. + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl TensorCreationOptions { + /// Create new options with a specific device. + /// + /// Data type will follow the [device policy](crate::set_default_dtypes) on tensor creation. + pub fn new(device: Device) -> Self { + Self { + device, + dtype: None, + } + } + + /// Set the tensor creation data type. + pub fn with_dtype(mut self, dtype: DType) -> Self { + self.dtype = Some(dtype); + + self + } + + /// Set the tensor creation device. + pub fn with_device(mut self, device: Device) -> Self { + self.device = device; + + self + } + + /// Create options with backend's default device and float dtype. + pub fn float() -> Self { + Self::default().with_dtype(::dtype()) + } + + /// Create options with backend's default device and int dtype. + pub fn int() -> Self { + Self::default().with_dtype(::dtype()) + } + + /// Create options with backend's default device and bool dtype. + pub fn bool() -> Self { + Self::default().with_dtype(::dtype()) + } + + /// Returns the tensor data type, or a provided default if not set. + /// + /// This is useful for cases where [`TensorCreationOptions`] may not have an explicit `dtype`. + pub fn dtype_or(&self, dtype: DType) -> DType { + self.dtype.unwrap_or(dtype) + } + + /// Returns the tensor data type, or the default from the [device policy](crate::set_default_dtypes). + pub(crate) fn resolve_policy(&self, dtype: DType) -> DType { + // TODO: should rely on tensor kind, not element dtype + self.dtype.unwrap_or_else(|| { + let policy = get_device_policy(&self.device); + if dtype.is_float() + && let Some(float_dtype) = policy.float_dtype() + { + float_dtype.into() + } else if (dtype.is_int() || dtype.is_uint()) + && let Some(int_dtype) = policy.int_dtype() + { + int_dtype.into() + } else { + // If policy was not explicitly set, use the fallback dtype (default backend elem type) + dtype + } + }) + } +} + +impl From<&Device> for TensorCreationOptions { + /// Convenience conversion from a reference to a device. + /// + /// Example: + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::TensorCreationOptions; + /// + /// fn example(device: B::Device) { + /// let options: TensorCreationOptions = (&device).into(); + /// } + /// ``` + fn from(device: &Device) -> Self { + TensorCreationOptions::new(device.clone()) + } +} + +impl From<(&Device, DType)> for TensorCreationOptions { + /// Convenience conversion for a specified `(&device, dtype)` tuple. + fn from(args: (&Device, DType)) -> Self { + TensorCreationOptions::new(args.0.clone()).with_dtype(args.1) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/orderable.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/orderable.rs new file mode 100644 index 0000000..dddf6b9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/orderable.rs @@ -0,0 +1,1155 @@ +use burn_backend::{ + Backend, ElementConversion, Scalar, + tensor::{Bool, IndexingUpdateOp, Int, Ordered}, +}; +use burn_std::AsIndex; + +use crate::check; +use crate::{Tensor, check::TensorCheck}; + +impl Tensor +where + B: Backend, + K: Ordered, +{ + /// Sort the elements by value in ascending order along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `dim` - The dimension to sort along. + /// + /// # Returns + /// + /// A new tensor with the elements sorted in ascending order along the given dimension. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let tensor = tensor.sort(0); + /// println!("{tensor}"); + /// // [[5.0, -2.0, 3.0], [12.0, 3.0, 6.0]] + /// let tensor = tensor.sort(1); + /// println!("{tensor}"); + /// // [[-2.0, 3.0, 12.0], [3.0, 5.0, 6.0]] + /// } + /// ``` + pub fn sort(self, dim: usize) -> Self { + check!(TensorCheck::sort_dim::("Sort", dim)); + Tensor::new(K::sort(self.primitive, dim, /*descending*/ false)) + } + + /// Sort the elements by value in descending order along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `dim` - The dimension to sort along. + /// + /// # Returns + /// + /// A new tensor with the elements sorted in descending order along the given dimension. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let tensor = tensor.sort_descending(0); + /// println!("{tensor}"); + /// // [[12.0, 3.0, 6.0], [5.0, -2.0, 3.0]] + /// let tensor = tensor.sort_descending(1); + /// println!("{tensor}"); + /// // [[12.0, 3.0, -2.0], [6.0, 5.0, 3.0]] + /// } + /// ``` + pub fn sort_descending(self, dim: usize) -> Self { + check!(TensorCheck::sort_dim::("Sort", dim)); + Tensor::new(K::sort(self.primitive, dim, /*descending*/ true)) + } + + /// Sort the elements by value in ascending order along a given dimension. + /// Also returns the indices. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `dim` - The dimension to sort along. + /// + /// # Returns + /// + /// A tuple containing the sorted tensor and the indices tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let (tensor, indices) = tensor.sort_with_indices(0); + /// println!("{tensor}"); + /// // [[5.0, -2.0, 3.0], [12.0, 3.0, 6.0]] + /// println!("{}", indices); + /// // [[1, 0, 0], [0, 1, 1]] + /// } + /// ``` + pub fn sort_with_indices(self, dim: usize) -> (Self, Tensor) { + check!(TensorCheck::sort_dim::("Sort_with_indices", dim)); + let (values, indices) = + K::sort_with_indices(self.primitive, dim, /*descending*/ false); + (Tensor::new(values), Tensor::new(indices)) + } + + /// Sort the elements by value in descending order along a given dimension. + /// Also returns the indices. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `dim` - The dimension to sort along. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let (tensor, indices) = tensor.sort_descending_with_indices(0); + /// println!("{tensor}"); + /// // [[12.0, 3.0, 6.0], [5.0, -2.0, 3.0]] + /// println!("{}", indices); + /// // [[0, 1, 1], [1, 0, 0]] + /// } + /// ``` + pub fn sort_descending_with_indices(self, dim: usize) -> (Self, Tensor) { + check!(TensorCheck::sort_dim::("Sort_with_indices", dim)); + let (values, indices) = K::sort_with_indices(self.primitive, dim, /*descending*/ true); + (Tensor::new(values), Tensor::new(indices)) + } + + /// Returns the indices that sort the elements by value in ascending order along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `dim` - The dimension to sort along. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let tensor = tensor.argsort(0); + /// println!("{tensor}"); + /// // [[1, 0, 0], [0, 1, 1]] + /// } + /// ``` + pub fn argsort(self, dim: usize) -> Tensor { + check!(TensorCheck::sort_dim::("Argsort", dim)); + Tensor::new(K::argsort(self.primitive, dim, /*descending*/ false)) + } + + /// Returns the indices that sort the elements by value in descending order along a given dimension. + /// + /// This sort is unstable (i.e., may reorder equal elements). + /// + /// # Arguments + /// + /// * `dim` - The dimension to sort along. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let tensor = tensor.argsort_descending(0); + /// println!("{tensor}"); + /// // [[0, 1, 1], [1, 0, 0]] + /// let tensor = tensor.argsort_descending(1); + /// println!("{tensor}"); + /// // [[0, 2, 1], [2, 0, 1]] + /// } + /// ``` + pub fn argsort_descending(self, dim: usize) -> Tensor { + check!(TensorCheck::sort_dim::("Argsort", dim)); + Tensor::new(K::argsort(self.primitive, dim, /*descending*/ true)) + } + + /// Returns the `k` largest elements of the given input tensor along a given dimension. + /// + /// # Arguments + /// + /// * `k` - The number of elements to return. + /// + /// # Returns + /// + /// A new tensor with the `k` largest elements along the given dimension. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let tensor = tensor.topk(2, 0); + /// println!("{tensor}"); + /// // [[12.0, 3.0, 6.0], [5.0, -2.0, 3.0]] + /// let tensor = tensor.topk(1, 1); + /// println!("{tensor}"); + /// // [[12.0], [6.0]] + /// } + /// ``` + pub fn topk(self, k: usize, dim: usize) -> Self { + let k_indices = Tensor::arange(0..k as i64, &self.device()); + self.sort_descending(dim).select(dim, k_indices) + } + + /// Returns the `k` largest elements of the given input tensor along a given dimension. + /// Also returns the indices. + /// + /// # Arguments + /// + /// * `k` - The number of elements to return. + /// * `dim` - The dimension to sort along. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// let (tensor, indices) = tensor.topk_with_indices(2, 0); + /// println!("{tensor}"); + /// // [[12.0, 3.0, 6.0], [5.0, -2.0, 3.0]] + /// println!("{}", indices); + /// // [[0, 1, 1], [1, 0, 0]] + /// let (tensor, indices) = tensor.topk_with_indices(1, 1); + /// println!("{tensor}"); + /// // [[12.0], [6.0]] + /// println!("{indices}"); + /// // [[0], [2]] + /// } + /// ``` + pub fn topk_with_indices(self, k: usize, dim: usize) -> (Self, Tensor) { + let k_indices = Tensor::arange(0..k as i64, &self.device()); + let (values, indices) = self.sort_descending_with_indices(dim); + ( + values.select(dim, k_indices.clone()), + indices.select(dim, k_indices), + ) + } + + /// Create a one hot tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example(){ + /// let device = Default::default(); + /// let indices: Tensor = Tensor::from_floats([0.0, 1.0, 2.0, 3.0], &device); + /// let one_hot: Tensor = indices.one_hot(4); + /// println!("{}", one_hot.to_data()); + /// // [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]] + /// } + /// ``` + pub fn one_hot(self, num_classes: usize) -> Tensor { + check!(TensorCheck::one_hot_tensor(self.clone(), num_classes)); + self.one_hot_fill(num_classes, 1.0, 0.0, -1) + } + + /// Create a one-hot encoded tensor with configurable `num_classes`, `on_value`, `off_value`, and `axis` including high-ranked tensors. + /// + /// # Arguments + /// + /// * `num_classes`: The number of classes for the one-hot encoding, which defines the size of the one-hot dimension. + /// * `on_value`: The value to assign for active positions (corresponding to indices). + /// * `off_value`: The value to assign for inactive positions. + /// * `axis`: The axis along which the one-hot dimension is added. Supports negative indexing. + /// + /// # Returns + /// + /// A tensor with one additional dimension for the one-hot encoding, where active positions are filled with `on_value` and others with `off_value`. + /// + /// # Example + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Float}; + /// fn example>>() { + /// let device = B::Device::default(); + /// let indices: Tensor = Tensor::from_floats([[0., 2.], [1., -1.]], &device); + /// // One-hot encoding + /// let tensor:Tensor = indices.one_hot_fill(3, 5.0.into(), 0.0.into(), -1); + /// println!("{tensor}"); + /// // [[[5.0, 0.0, 0.0], + /// // [0.0, 0.0, 5.0]], + /// // [[0.0, 5.0, 0.0], + /// // [0.0, 0.0, 5.0]]] + /// } + /// ``` + pub fn one_hot_fill( + self, + num_classes: usize, + on_value: f32, + off_value: f32, + axis: i64, + ) -> Tensor { + check!(TensorCheck::one_hot_tensor_rank::()); + // Initialize shape from the current tensor dimensions and prepare for modification + let mut shape = self.shape(); + let device = self.device(); + let rank = self.dims().len(); + + // Adjust negative axis to a positive index + let axis = if axis < 0 { + axis + rank as i64 + 1 + } else { + axis + }; + + // Ensure axis is within valid range + if axis < 0 || axis > rank as i64 { + panic!("Axis out of range. Accepted range is [-r-1, r] where r = rank(indices)."); + } + // Convert the input tensor to integer indices + let indices: Tensor = + Tensor::from_data(self.to_data().convert::(), &device); + // Insert the new dimension for the one-hot representation + shape.insert(axis as usize, num_classes); + // Adjust indices to valid range and handle invalid indices + let adjusted_indices = indices + .clone() + .mask_fill(self.clone().lower_elem(0), num_classes as i64) // Handle negative indices + .add(indices.clone().mask_fill(self.clone().greater_elem(0), 0)); // Handle positive indices + // Unsqueeze the indices tensor along the specified axis + let indices_unsqueezed: Tensor = adjusted_indices.unsqueeze_dim(axis as usize); + + // Initialize the output tensor with the off_value + let output = Tensor::full(shape.clone(), off_value, &device); + + // Prepare scatter tensor for on_value and off_value adjustments + let scatter_on_values = Tensor::full(indices_unsqueezed.shape(), on_value, &device) + - Tensor::full(indices_unsqueezed.shape(), off_value, &self.device()); + + // Scatter on_value at the appropriate indices to create the one-hot representation + output.scatter( + axis as usize, + indices_unsqueezed, + scatter_on_values, + IndexingUpdateOp::Add, + ) + } + + /// Applies element wise greater comparison and returns a boolean tensor. + /// + /// # Panics + /// + /// If the two tensors don't have the same shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[1.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.greater(tensor2); + /// println!("{tensor}"); + /// // [[false, false, false], [true, true, true]] + /// } + /// ``` + pub fn greater(self, other: Self) -> Tensor { + check!(TensorCheck::binary_ops_ew("Greater", &self, &other)); + Tensor::new(K::greater(self.primitive, other.primitive)) + } + + /// Applies element wise greater-equal comparison and returns a boolean tensor. + /// + /// # Panics + /// + /// If the two tensors don't have the same shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[1.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.greater_equal(tensor2); + /// println!("{tensor}"); + /// // [[true, false, false], [true, true, true]] + /// } + /// ``` + pub fn greater_equal(self, other: Self) -> Tensor { + check!(TensorCheck::binary_ops_ew("Greater_equal", &self, &other)); + Tensor::new(K::greater_equal(self.primitive, other.primitive)) + } + + /// Applies element wise lower comparison and returns a boolean tensor. + /// + /// # Panics + /// + /// If the two tensors don't have the same shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[1.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.lower(tensor2); + /// println!("{tensor}"); + /// // [[false, true, true], [false, false, false]] + /// } + /// ``` + pub fn lower(self, other: Self) -> Tensor { + check!(TensorCheck::binary_ops_ew("Lower", &self, &other)); + Tensor::new(K::lower(self.primitive, other.primitive)) + } + + /// Applies element wise lower-equal comparison and returns a boolean tensor. + /// + /// # Panics + /// + /// If the two tensors don't have the same shape. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[1.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.lower_equal(tensor2); + /// println!("{tensor}"); + /// // [[true, true, true], [false, false, false]] + /// } + /// ``` + pub fn lower_equal(self, other: Self) -> Tensor { + check!(TensorCheck::binary_ops_ew("Lower_equal", &self, &other)); + Tensor::new(K::lower_equal(self.primitive, other.primitive)) + } + + /// Applies greater than `other` comparison and returns a boolean tensor. + /// + /// # Arguments + /// + /// * `other` - The element to compare. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.greater_elem(3.0); + /// println!("{tensor}"); + /// // [[false, false, true], [true, true, true]] + /// } + /// ``` + pub fn greater_elem(self, other: E) -> Tensor { + let other = Scalar::new(other, &self.dtype()); + Tensor::new(K::greater_elem(self.primitive, other)) + } + + /// Applies greater-equal than `other` comparison and returns a boolean tensor. + /// + /// # Arguments + /// + /// * `other` - The element to compare. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.greater_equal_elem(3.0); + /// println!("{tensor}"); + /// // [[false, false, true], [true, true, true]] + /// } + /// ``` + pub fn greater_equal_elem(self, other: E) -> Tensor { + let other = Scalar::new(other, &self.dtype()); + Tensor::new(K::greater_equal_elem(self.primitive, other)) + } + + /// Applies lower than `other` comparison and returns a boolean tensor. + /// + /// # Arguments + /// + /// * `other` - The element to compare. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.lower_elem(3.0); + /// println!("{tensor}"); + /// // [[true, true, false], [false, false, false]] + /// } + /// ``` + pub fn lower_elem(self, other: E) -> Tensor { + let other = Scalar::new(other, &self.dtype()); + Tensor::new(K::lower_elem(self.primitive, other)) + } + + /// Applies lower-equal than `other` comparison and returns a boolean tensor. + /// + /// # Arguments + /// + /// * `other` - The element to compare. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.lower_equal_elem(3.0); + /// println!("{tensor}"); + /// // [[true, true, true], [false, false, false]] + /// } + /// ``` + pub fn lower_equal_elem(self, other: E) -> Tensor { + let other = Scalar::new(other, &self.dtype()); + Tensor::new(K::lower_equal_elem(self.primitive, other)) + } + + /// Applies the argmax function along the given dimension and returns an integer tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::ones(Shape::new([2, 3, 3]), &device); + /// let tensor = tensor.argmax(1); + /// println!("{:?}", tensor.shape()); + /// // Shape { dims: [2, 1, 3] } + /// } + /// ``` + pub fn argmax(self, dim: usize) -> Tensor { + Tensor::new(K::argmax(self.primitive, dim)) + } + + /// Find the maximum value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.max(); + /// println!("{tensor}"); + /// // [9.0] + /// } + /// ``` + pub fn max(self) -> Tensor { + Tensor::new(K::max(self.primitive)) + } + + /// Find the maximum value along the given dimension. + /// + /// Also returns the indices. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let (tensor, index) = tensor.max_dim_with_indices(0); + /// // [[5.0, 9.0, 6.0]] + /// println!("{tensor}"); + /// // [[1, 1, 1]] + /// println!("{index}"); + /// } + /// ``` + pub fn max_dim_with_indices(self, dim: I) -> (Self, Tensor) { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Max", dim)); + + let (tensor, index) = K::max_dim_with_indices(self.primitive, dim); + + let tensor = Tensor::new(tensor); + let index = Tensor::new(index); + + (tensor, index) + } + + /// Find the maximum absolute value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -7.0, 3.0], [5.0, -1.0, 6.0]], &device); + /// let tensor = tensor.max_abs(); + /// println!("{tensor}"); + /// // [7.0] + /// } + /// ``` + pub fn max_abs(self) -> Tensor { + Tensor::new(K::max_abs(self.primitive)) + } + + /// Finds the maximum pair wise values with another tensor. + /// + /// # Arguments + /// + /// * `other` - Other tensor to find maximum elements with + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensors containing the maximum value found + /// in the input tensors. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.max_pair(tensor2); + /// println!("{tensor}"); + /// // [[2.0, 3.0, 4.0], [5.0, 9.0, 6.0]] + /// } + /// ``` + pub fn max_pair(self, other: Self) -> Self { + let mask = self.clone().lower(other.clone()); + self.mask_where(mask, other) + } + + /// Find the maximum absolute value along the given dimension. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to aggregate the elements, + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimension will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.max_dim(0); + /// println!("{tensor}"); + /// // [[5.0, 9.0, 6.0]] + /// } + /// ``` + pub fn max_abs_dim(self, dim: I) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("MaxAbs", dim)); + + Tensor::new(K::max_abs_dim(self.primitive, dim)) + } + + /// Find the maximum absolute value along the given dimensions. + /// + /// # Arguments + /// + /// * `dims` - The dimensions or axes along which to aggregate the elements, + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.max_abs_dims(&[0, 1]); + /// println!("{tensor}"); + /// // [[9.0]] + /// } + /// ``` + pub fn max_abs_dims(self, dims: &[I]) -> Self { + dims.iter() + .fold(self, |tensor, &dim| tensor.max_abs_dim(dim)) + } + + /// Applies the argmin function along the given dimension and returns an integer tensor. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::ones(Shape::new([2, 3, 3]), &device); + /// let tensor = tensor.argmin(1); + /// println!("{:?}", tensor.shape()); + /// // Shape { dims: [2, 1, 3] } + /// } + /// ``` + pub fn argmin(self, dim: usize) -> Tensor { + Tensor::new(K::argmin(self.primitive, dim)) + } + + /// Find the minimum value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.min(); + /// println!("{tensor}"); + /// // [-2.0] + /// } + /// ``` + pub fn min(self) -> Tensor { + Tensor::new(K::min(self.primitive)) + } + + /// Find the minimum value along the given dimension. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to aggregate the elements; + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimension will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.min_dim(0); + /// println!("{tensor}"); + /// // [[1.0, -2.0, 3.0]] + /// } + /// ``` + pub fn min_dim(self, dim: I) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Min", dim)); + Tensor::new(K::min_dim(self.primitive, dim)) + } + + /// Find the minimum value along the given dimensions. + /// + /// # Arguments + /// + /// * `dims` - The dimensions or axes along which to aggregate the elements; + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.min_dims(&[0, 1]); + /// println!("{tensor}"); + /// // [[-2.0]] + /// } + /// ``` + pub fn min_dims(self, dims: &[I]) -> Self { + dims.iter().fold(self, |tensor, &dim| tensor.min_dim(dim)) + } + + /// Find the minimum value along the given dimension. + /// + /// Also returns the indices. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[7.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let (tensor, index) = tensor.min_dim_with_indices(0); + /// println!("{tensor}"); + /// // [[5.0, -2.0, 3.0]] + /// println!("{}", index); + /// // [[1, 0, 0]] + /// } + /// ``` + pub fn min_dim_with_indices(self, dim: I) -> (Self, Tensor) { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Min", dim)); + + let (tensor, index) = K::min_dim_with_indices(self.primitive, dim); + + let tensor = Tensor::new(tensor); + let index = Tensor::new(index); + + (tensor, index) + } + + /// Finds the minimum pair wise values with another tensor. + /// + /// # Arguments + /// + /// * `other` - Other tensor to find minimum elements with + /// + /// # Returns + /// + /// A tensor with the same shape as the input tensors containing the minimum value found + /// between each element of the two source tensors. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor1 = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor2 = Tensor::::from_data([[2.0, 3.0, 4.0], [1.0, 2.0, 3.0]], &device); + /// let tensor = tensor1.min_pair(tensor2); + /// println!("{tensor}"); + /// // [[1.0, -2.0, 3.0], [1.0, 2.0, 3.0]] + /// } + pub fn min_pair(self, other: Self) -> Self { + let mask = other.clone().lower(self.clone()); + self.mask_where(mask, other) + } + + /// Clamp element wise between the given min and max values. + /// + /// # Arguments + /// + /// * `min` - The minimum value. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// A new tensor with the values clamped between the given min and max values. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_ints( + /// [ + /// [1, 2, 3], + /// [4, 5, 6], + /// [7, 8, 9] + /// ], + /// &device); + /// let tensor = tensor.clamp(2, 6); + /// println!("{tensor}"); + /// // [[2, 2, 3], [4, 5, 6], [6, 6, 6]] + /// } + /// ``` + pub fn clamp(self, min: E, max: E) -> Self { + let dtype = self.dtype(); + Self::new(K::clamp( + self.primitive, + Scalar::new(min, &dtype), + Scalar::new(max, &dtype), + )) + } + + /// Clamp element wise under a minimum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `min` - The minimum value. + /// + /// # Returns + /// + /// A new tensor with the values clamped under the given min value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_ints( + /// [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + /// &device); + /// let tensor = tensor.clamp_min(4); + /// println!("{tensor}"); + /// // [[4, 4, 4], [4, 5, 6], [7, 8, 9]] + /// } + /// ``` + pub fn clamp_min(self, min: E) -> Self { + let min = Scalar::new(min, &self.dtype()); + Self::new(K::clamp_min(self.primitive, min)) + } + + /// Clamp element wise over a maximum value. + /// + /// # Arguments + /// + /// * `tensor` - The tensor to clamp. + /// * `max` - The maximum value. + /// + /// # Returns + /// + /// A new tensor with the values clamped over the given max value. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Int, Tensor}; + /// + /// fn example() { + /// let device = Default::default(); + /// let tensor = Tensor::::from_ints( + /// [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + /// &device); + /// let tensor = tensor.clamp_max(5); + /// println!("{tensor}"); + /// // [[1, 2, 3], [4, 5, 5], [5, 5, 5]] + /// } + /// ``` + pub fn clamp_max(self, max: E) -> Self { + let max = Scalar::new(max, &self.dtype()); + Self::new(K::clamp_max(self.primitive, max)) + } + + /// Computes the cumulative minimum of elements along the given *dimension* or *axis*. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to compute the cumulative minimum. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[3.0, 5.0, 2.0], [4.0, 1.0, 6.0]], &device); + /// let result = tensor.clone().cummin(0); + /// println!("{result}"); + /// // [[3.0, 5.0, 2.0], [3.0, 1.0, 2.0]] + /// let result = tensor.cummin(1); + /// println!("{result}"); + /// // [[3.0, 3.0, 2.0], [4.0, 1.0, 1.0]] + /// } + /// ``` + pub fn cummin(self, dim: usize) -> Self { + check!(TensorCheck::aggregate_dim::("CumMin", dim)); + Self::new(K::cummin(self.primitive, dim)) + } + + /// Computes the cumulative maximum of elements along the given *dimension* or *axis*. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to compute the cumulative maximum. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[3.0, 1.0, 2.0], [4.0, 5.0, 2.0]], &device); + /// let result = tensor.clone().cummax(0); + /// println!("{result}"); + /// // [[3.0, 1.0, 2.0], [4.0, 5.0, 2.0]] + /// let result = tensor.cummax(1); + /// println!("{result}"); + /// // [[3.0, 3.0, 3.0], [4.0, 5.0, 5.0]] + /// } + /// ``` + pub fn cummax(self, dim: usize) -> Self { + check!(TensorCheck::aggregate_dim::("CumMax", dim)); + Self::new(K::cummax(self.primitive, dim)) + } + /// Find the maximum value along the given dimension. + /// + /// # Arguments + /// + /// * `dim` - The dimension or axis along which to aggregate the elements; + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimension will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.max_dim(0); + /// println!("{tensor}"); + /// // [[5.0, 9.0, 6.0]] + /// } + /// ``` + pub fn max_dim(self, dim: I) -> Self { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::aggregate_dim::("Max", dim)); + Tensor::new(K::max_dim(self.primitive, dim)) + } + + /// Find the maximum value along the given dimensions. + /// + /// # Arguments + /// + /// * `dims` - The dimensions or axis along which to aggregate the elements; + /// supports negative indexing. + /// + /// # Returns + /// + /// The returned tensor will have the same rank, + /// but the aggregated dimensions will have size 1. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[1.0, -2.0, 3.0], [5.0, 9.0, 6.0]], &device); + /// let tensor = tensor.max_dims(&[0, 1]); + /// println!("{tensor}"); + /// // [[9.0]] + /// } + /// ``` + pub fn max_dims(self, dims: &[I]) -> Self { + dims.iter().fold(self, |tensor, &dim| tensor.max_dim(dim)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/pad.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/pad.rs new file mode 100644 index 0000000..6b6eddf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/pad.rs @@ -0,0 +1,363 @@ +use alloc::vec::Vec; +use core::ops::Range; + +use crate::{Element, ElementConversion, Tensor, backend::Backend, ops::PadMode}; + +use super::Numeric; + +/// Trait for types that can be used as padding specifications. +/// +/// Padding is specified as `(before, after)` pairs per dimension, returned as a +/// fixed-size array `[(usize, usize); D]`. If fewer pairs than dimensions are provided, +/// they apply to the **last** N dimensions (earlier dimensions are left unpadded). +pub trait IntoPadding { + /// Converts into a fixed-size array of `(before, after)` padding pairs. + fn into_padding(self) -> [(usize, usize); D]; +} + +impl IntoPadding for [(usize, usize); N] { + fn into_padding(self) -> [(usize, usize); D] { + assert!( + N <= D, + "Padding has {} pairs but tensor only has {} dimensions", + N, + D + ); + let mut result = [(0usize, 0usize); D]; + let offset = D - N; + for (i, pair) in self.into_iter().enumerate() { + result[offset + i] = pair; + } + result + } +} + +/// Backward-compatible: `(left, right, top, bottom)` maps to last 2 dimensions. +/// +/// Equivalent to `[(top, bottom), (left, right)]`. +impl IntoPadding for (usize, usize, usize, usize) { + fn into_padding(self) -> [(usize, usize); D] { + let (left, right, top, bottom) = self; + let mut result = [(0usize, 0usize); D]; + result[D - 2] = (top, bottom); + result[D - 1] = (left, right); + result + } +} + +impl IntoPadding for &[(usize, usize)] { + fn into_padding(self) -> [(usize, usize); D] { + assert!( + self.len() <= D, + "Padding has {} pairs but tensor only has {} dimensions", + self.len(), + D + ); + let mut result = [(0usize, 0usize); D]; + let offset = D - self.len(); + for (i, &pair) in self.iter().enumerate() { + result[offset + i] = pair; + } + result + } +} + +impl IntoPadding for Vec<(usize, usize)> { + fn into_padding(self) -> [(usize, usize); D] { + assert!( + self.len() <= D, + "Padding has {} pairs but tensor only has {} dimensions", + self.len(), + D + ); + let mut result = [(0usize, 0usize); D]; + let offset = D - self.len(); + for (i, pair) in self.into_iter().enumerate() { + result[offset + i] = pair; + } + result + } +} + +/// Helper to build a range array for slice_assign, selecting a portion of one dimension. +fn build_slice_ranges( + dims: [usize; D], + target_dim: usize, + start: usize, + len: usize, +) -> [Range; D] { + dims.iter() + .enumerate() + .map(|(i, &size)| { + if i == target_dim { + start..start + len + } else { + 0..size + } + }) + .collect::>>() + .try_into() + .unwrap() +} + +impl Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + /// Pads the tensor using the specified padding mode. + /// + /// Padding is specified as `(before, after)` pairs. If fewer pairs than tensor dimensions + /// are provided, they apply to the **last** N dimensions (unspecified leading dimensions + /// are left unpadded). + /// + /// For backward compatibility, a `(left, right, top, bottom)` tuple is also accepted, + /// which pads the last two dimensions. + /// + /// # Arguments + /// + /// * `padding` - Padding specification. Accepts: + /// - `[(before, after); N]` fixed-size array of pairs (N <= D) + /// - `&[(before, after)]` slice of pairs per dimension + /// - `Vec<(before, after)>` vector of pairs + /// - `(left, right, top, bottom)` tuple for last-2-dim backward compatibility + /// * `mode` - The padding mode: `Constant(value)`, `Reflect`, or `Edge`. + /// + /// # Returns + /// + /// A new tensor with the specified padding applied. + /// + /// # Panics + /// + /// - Panics if more padding pairs are provided than tensor dimensions. + /// - `Reflect` mode panics if padding exceeds `dimension_size - 1`. + /// - `Edge` mode panics if padding is applied to a zero-sized dimension. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Shape}; + /// use burn_tensor::ops::PadMode; + /// + /// fn example>>() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([[12.0, -2.0, 3.0], [5.0, 3.0, 6.0]], &device); + /// + /// // Constant padding with value 0.0 (backward-compatible tuple) + /// let padded = tensor.clone().pad((1, 1, 1, 1), PadMode::Constant(0.0)); + /// + /// // Pad arbitrary dimensions with slice of (before, after) pairs + /// let padded = tensor.clone().pad([(1, 1), (2, 2)], PadMode::Constant(0.0)); + /// + /// // Pad only the last dimension + /// let padded = tensor.pad([(1, 1)], PadMode::Reflect); + /// } + /// ``` + pub fn pad(self, padding: impl IntoPadding, mode: impl Into) -> Self { + let pairs = padding.into_padding(); + match mode.into() { + PadMode::Constant(value) => pad_constant(self, &pairs, value), + PadMode::Reflect => pad_reflect(self, &pairs), + PadMode::Edge => pad_edge(self, &pairs), + } + } +} + +/// Pad with a constant value. +fn pad_constant( + tensor: Tensor, + padding: &[(usize, usize); D], + value: E, +) -> Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, + E: ElementConversion, +{ + let mut padded_dims: [usize; D] = tensor.dims(); + + for (i, &(before, after)) in padding.iter().enumerate() { + padded_dims[i] += before + after; + } + + let ranges: [Range; D] = padded_dims + .iter() + .enumerate() + .map(|(i, &dim)| { + let (before, after) = padding[i]; + before..dim - after + }) + .collect::>>() + .try_into() + .unwrap(); + + let padded_tensor = Tensor::full(padded_dims, value, &tensor.device()); + + padded_tensor.slice_assign(ranges, tensor) +} + +/// Pad using reflection at the boundaries (excluding edge values). +/// +/// For ONNX "reflect" mode: mirrors from index 1, not index 0. +/// Example: `[1, 2, 3, 4]` with left padding 2 becomes `[3, 2, 1, 2, 3, 4]` +fn pad_reflect( + tensor: Tensor, + padding: &[(usize, usize); D], +) -> Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + let dims = tensor.dims(); + + for (i, &(before, after)) in padding.iter().enumerate() { + if before > 0 || after > 0 { + assert!( + before < dims[i] && after < dims[i], + "Reflect padding ({}, {}) must be less than dimension {} size ({})", + before, + after, + i, + dims[i] + ); + } + } + + let mut result = tensor; + + for (i, &(before, after)) in padding.iter().enumerate() { + if before > 0 || after > 0 { + result = pad_reflect_dim(result, i, before, after); + } + } + + result +} + +/// Helper to pad a single dimension using reflection. +fn pad_reflect_dim( + tensor: Tensor, + dim: usize, + pad_before: usize, + pad_after: usize, +) -> Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + let dims = tensor.dims(); + let dim_size = dims[dim]; + + // Calculate output dimensions + let mut output_dims = dims; + output_dims[dim] += pad_before + pad_after; + + // Create output tensor and place original in the center + let output = Tensor::zeros(output_dims, &tensor.device()); + let original_range = build_slice_ranges(output_dims, dim, pad_before, dim_size); + let mut output = output.slice_assign(original_range, tensor.clone()); + + // Assign reflected "before" padding (e.g., top or left) + // Reflect excludes the edge, so we take indices [1..pad_before+1] and flip + if pad_before > 0 { + let before_slice = tensor.clone().narrow(dim, 1, pad_before); + let before_flipped = before_slice.flip([dim as isize]); + let before_range = build_slice_ranges(output_dims, dim, 0, pad_before); + output = output.slice_assign(before_range, before_flipped); + } + + // Assign reflected "after" padding (e.g., bottom or right) + // Take indices [dim_size - pad_after - 1..dim_size - 1] and flip + if pad_after > 0 { + let start = dim_size - pad_after - 1; + let after_slice = tensor.narrow(dim, start, pad_after); + let after_flipped = after_slice.flip([dim as isize]); + let after_range = build_slice_ranges(output_dims, dim, pad_before + dim_size, pad_after); + output = output.slice_assign(after_range, after_flipped); + } + + output +} + +/// Pad by replicating edge values. +/// +/// Example: `[1, 2, 3, 4]` with left padding 2 becomes `[1, 1, 1, 2, 3, 4]` +fn pad_edge( + tensor: Tensor, + padding: &[(usize, usize); D], +) -> Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + let dims = tensor.dims(); + + for (i, &(before, after)) in padding.iter().enumerate() { + if before > 0 || after > 0 { + assert!( + dims[i] > 0, + "Cannot apply edge padding to zero-sized dimension {}", + i + ); + } + } + + let mut result = tensor; + + for (i, &(before, after)) in padding.iter().enumerate() { + if before > 0 || after > 0 { + result = pad_edge_dim(result, i, before, after); + } + } + + result +} + +/// Helper to pad a single dimension by replicating edge values. +fn pad_edge_dim( + tensor: Tensor, + dim: usize, + pad_before: usize, + pad_after: usize, +) -> Tensor +where + B: Backend, + K: Numeric, + K::Elem: Element, +{ + let dims = tensor.dims(); + let dim_size = dims[dim]; + + // Calculate output dimensions + let mut output_dims = dims; + output_dims[dim] += pad_before + pad_after; + + // Create output tensor and place original in the center + let output = Tensor::zeros(output_dims, &tensor.device()); + let original_range = build_slice_ranges(output_dims, dim, pad_before, dim_size); + let mut output = output.slice_assign(original_range, tensor.clone()); + + // Assign "before" padding by repeating the first element + if pad_before > 0 { + let first_slice = tensor.clone().narrow(dim, 0, 1); + let before_pad = first_slice.repeat_dim(dim, pad_before); + let before_range = build_slice_ranges(output_dims, dim, 0, pad_before); + output = output.slice_assign(before_range, before_pad); + } + + // Assign "after" padding by repeating the last element + if pad_after > 0 { + let last_slice = tensor.narrow(dim, dim_size - 1, 1); + let after_pad = last_slice.repeat_dim(dim, pad_after); + let after_range = build_slice_ranges(output_dims, dim, pad_before + dim_size, pad_after); + output = output.slice_assign(after_range, after_pad); + } + + output +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/take.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/take.rs new file mode 100644 index 0000000..e476d1f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/take.rs @@ -0,0 +1,98 @@ +use crate::{AsIndex, BasicOps, Int, Tensor, backend::Backend, check, check::TensorCheck}; +use alloc::vec::Vec; + +impl Tensor +where + B: Backend, + K: BasicOps, +{ + /// Takes elements from the tensor along the given dimension using indices of any dimensionality. + /// + /// This behaves like numpy's take function. When indices is multi-dimensional, + /// the output shape will be: input.shape\[:dim\] + indices.shape + input.shape\[dim+1:\] + /// + /// # Arguments + /// + /// * `dim` - The dimension along which to select elements. Supports negative indexing. + /// * `indices` - The indices of elements to select. Can be any dimensionality. + /// Must be valid indices in the range [0, dim_size). + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::{Tensor, Int}; + /// + /// fn example() { + /// let device = B::Device::default(); + /// + /// // Example with 1D indices + /// let tensor = Tensor::::from_data([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], &device); + /// let indices = Tensor::::from_data([2, 0, 1], &device); + /// let result: Tensor = tensor.clone().take::<1, 2>(-1, indices); // -1 refers to last dimension + /// println!("{result}"); + /// // [[3.0, 1.0, 2.0], [6.0, 4.0, 5.0]] + /// + /// // Example with 2D indices - output will have +1 dimension (2D -> 3D) + /// let indices_2d = Tensor::::from_data([[0, 2], [1, 0]], &device); + /// let result: Tensor = tensor.take::<2, 3>(1, indices_2d); + /// println!("{result}"); + /// // [[[1.0, 3.0], [2.0, 1.0]], [[4.0, 6.0], [5.0, 4.0]]] + /// } + /// ``` + pub fn take( + self, + dim: impl AsIndex, + indices: Tensor, + ) -> Tensor { + let dim = dim.expect_dim_index(D); + check!(TensorCheck::take::(dim)); + + // Store the indices shape for reshaping later + let indices_shape = indices.shape(); + let indices_dims = indices_shape.clone(); + + // Flatten indices to 1D for processing + let indices_flat = indices.reshape([indices_shape.num_elements()]); + + // Perform the selection with the flattened indices + let selected = self.select(dim, indices_flat); + + // Build the output shape + // Output shape = input.shape[:dim] + indices.shape + input.shape[dim+1:] + let selected_shape = selected.shape(); + let mut new_shape = Vec::with_capacity(DO); + + // Add dimensions before the selected dimension + for i in 0..dim { + new_shape.push(selected_shape[i]); + } + + // Add all indices dimensions + for &idx_dim in indices_dims.iter() { + new_shape.push(idx_dim); + } + + // Add dimensions after the selected dimension + for i in (dim + 1)..D { + new_shape.push(selected_shape[i]); + } + + // Verify we have the correct number of dimensions + assert_eq!( + new_shape.len(), + DO, + "Internal error: shape calculation resulted in {} dims but expected {}", + new_shape.len(), + DO + ); + + // Convert to fixed-size array for reshape + let mut shape_array = [0; DO]; + for (i, &s) in new_shape.iter().enumerate() { + shape_array[i] = s; + } + + selected.reshape(shape_array) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/transaction.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/transaction.rs new file mode 100644 index 0000000..23e08ad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/transaction.rs @@ -0,0 +1,57 @@ +use super::{BasicOps, Tensor}; +use crate::{ + TensorData, + backend::{Backend, ExecutionError}, + ops::TransactionPrimitive, +}; +use alloc::vec::Vec; + +#[derive(Default)] +/// A transaction can [read](Self::register) multiple tensors at once with a single operation improving +/// compute utilization with optimized laziness. +/// +/// # Example +/// +/// ```rust,ignore +/// let [output_data, loss_data, targets_data] = Transaction::default() +/// .register(output) +/// .register(loss) +/// .register(targets) +/// .execute() +/// .try_into() +/// .expect("Correct amount of tensor data"); +/// ``` +pub struct Transaction { + op: TransactionPrimitive, +} + +impl Transaction { + /// Add a [tensor](Tensor) to the transaction to be read. + pub fn register>(mut self, tensor: Tensor) -> Self { + K::register_transaction(&mut self.op, tensor.into_primitive()); + self + } + + /// Executes the transaction synchronously and returns the [data](TensorData) in the same order + /// in which they were [registered](Self::register). + pub fn execute(self) -> Vec { + burn_std::future::block_on(self.execute_async()) + .expect("Error while reading data: use `try_execute` to handle error at runtime") + } + + /// Executes the transaction synchronously and returns the [data](TensorData) in the same + /// order in which they were [registered](Self::register). + /// + /// # Returns + /// + /// Any error that might have occurred since the last time the device was synchronized. + pub fn try_execute(self) -> Result, ExecutionError> { + burn_std::future::block_on(self.execute_async()) + } + + /// Executes the transaction asynchronously and returns the [data](TensorData) in the same order + /// in which they were [registered](Self::register). + pub async fn execute_async(self) -> Result, ExecutionError> { + self.op.execute_async().await + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/trunc.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/trunc.rs new file mode 100644 index 0000000..593c66a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/api/trunc.rs @@ -0,0 +1,42 @@ +use crate::{Float, Tensor, TensorPrimitive, backend::Backend}; + +impl Tensor +where + B: Backend, +{ + /// Truncates the tensor element-wise, rounding toward zero. + /// + /// This function returns a new tensor with the same shape as the input tensor, + /// where each element is truncated toward zero. For positive values, this is + /// equivalent to floor, and for negative values, it's equivalent to ceil. + /// + /// # Special Cases (IEEE 754 compliant) + /// + /// - `trunc(±0)` returns ±0 (preserves sign of zero) + /// - `trunc(±∞)` returns ±∞ + /// - `trunc(NaN)` returns NaN + /// + /// # Returns + /// + /// A tensor with the same shape where each element has been truncated toward zero. + /// + /// # Example + /// + /// ```rust + /// use burn_tensor::backend::Backend; + /// use burn_tensor::Tensor; + /// + /// fn example() { + /// let device = B::Device::default(); + /// let tensor = Tensor::::from_data([2.3, -1.7, 0.5, -0.5, 3.9], &device); + /// let truncated = tensor.trunc(); + /// + /// // Result: [2.0, -1.0, 0.0, -0.0, 3.0] + /// } + /// ``` + pub fn trunc(self) -> Self { + Self::new(TensorPrimitive::Float(B::float_trunc( + self.primitive.tensor(), + ))) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/affine_grid.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/affine_grid.rs new file mode 100644 index 0000000..3789a1d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/affine_grid.rs @@ -0,0 +1,60 @@ +use crate::ElementConversion; +use crate::backend::Backend; +use crate::s; +use crate::tensor::{Int, Tensor}; +use alloc::vec; + +/// Generate a tensor with homogeonous coordinates of each element's +/// transformed location +/// +/// +/// See: +/// - [torch.nn.functional.affine_grid](https://docs.pytorch.org/docs/stable/generated/torch.nn.functional.affine_grid.html) +/// +/// * `transform` - Transformation with shape (batch_size, 2, 3) +/// * `dims` - dimensions as (batch_size, channels, height, width) +/// +/// # Returns +/// +/// Tensor with shape (batch_size, height, width, 2), where dim 2 is (x, y) +/// All coordinates are broadcast on the batch dim +pub fn affine_grid_2d(transform: Tensor, dims: [usize; 4]) -> Tensor { + let [batch_size, _c, height, width] = dims; + + let device = &transform.device(); + + let x = Tensor::::arange(0..width as i64, device) + .reshape([1, width]) + .expand([height, width]); + let y = Tensor::::arange(0..height as i64, device) + .reshape([height, 1]) + .expand([height, width]); + + // from ints (0..(width-1)) and (0..(height-1)), to (-1.0..1.0) + let x = x + .float() + .div_scalar(((width - 1) as f32 / 2.0).elem::()) + .sub_scalar((1_f32).elem::()); + let y = y + .float() + .div_scalar(((height - 1) as f32 / 2.0).elem::()) + .sub_scalar((1_f32).elem::()); + + // Broadcast to batch dimension + let x = x.unsqueeze_dim::<3>(0).expand([batch_size, height, width]); // [B, H, W] + let y = y.unsqueeze_dim::<3>(0).expand([batch_size, height, width]); // [B, H, W] + + // Apply affine transform + let a_11 = transform.clone().slice(s![.., 0, 0]); + let a_12 = transform.clone().slice(s![.., 0, 1]); + let trans_x = transform.clone().slice(s![.., 0, 2]); + + let a_21 = transform.clone().slice(s![.., 1, 0]); + let a_22 = transform.clone().slice(s![.., 1, 1]); + let trans_y = transform.slice(s![.., 1, 2]); + + let grid_x = a_11.mul(x.clone()).add(a_12.mul(y.clone())).add(trans_x); + let grid_y = a_21.mul(x).add(a_22.mul(y)).add(trans_y); + + Tensor::stack(vec![grid_x, grid_y], 3) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/meshgrid.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/meshgrid.rs new file mode 100644 index 0000000..d2e1b89 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/meshgrid.rs @@ -0,0 +1,107 @@ +use crate::backend::Backend; +use crate::tensor::grid::{GridIndexing, GridOptions, GridSparsity, IndexPos}; +use crate::tensor::{BasicOps, Tensor}; +use alloc::vec::Vec; + +/// Return a collection of coordinate matrices for coordinate vectors. +/// +/// Takes N 1D tensors and returns N tensors where each tensor represents the coordinates +/// in one dimension across an N-dimensional grid. +/// +/// Based upon `options.sparse`, the generated coordinate tensors can either be `Sparse` or `Dense`: +/// * In `Sparse` mode, output tensors will have shape 1 everywhere except their cardinal dimension. +/// * In `Dense` mode, output tensors will be expanded to the full grid shape. +/// +/// Based upon `options.indexing`, the generated coordinate tensors will use either: +/// * `Matrix` indexing, where dimensions are in the same order as their cardinality. +/// * `Cartesian` indexing; where the first two dimensions are swapped. +/// +/// See: +/// - [numpy.meshgrid](https://numpy.org/doc/stable/reference/generated/numpy.meshgrid.html) +/// - [torch.meshgrid](https://pytorch.org/docs/stable/generated/torch.meshgrid.html) +/// +/// # Arguments +/// +/// * `tensors` - A slice of 1D tensors +/// * `options` - the options. +/// +/// # Returns +/// +/// A vector of N N-dimensional tensors representing the grid coordinates. +pub fn meshgrid( + tensors: &[Tensor; N], + options: O, +) -> [Tensor; N] +where + K: BasicOps, + O: Into, +{ + let options = options.into(); + let swap_dims = options.indexing == GridIndexing::Cartesian && N > 1; + let dense = options.sparsity == GridSparsity::Dense; + + let grid_shape: [usize; N] = tensors + .iter() + .map(|t| t.dims()[0]) + .collect::>() + .try_into() + .unwrap(); + + tensors + .iter() + .enumerate() + .map(|(i, tensor)| { + let mut coord_tensor_shape = [1; N]; + coord_tensor_shape[i] = grid_shape[i]; + + // Reshape the tensor to have singleton dimensions in all but the i-th dimension + let mut tensor = tensor.clone().reshape(coord_tensor_shape); + + if dense { + tensor = tensor.expand(grid_shape); + } + if swap_dims { + tensor = tensor.swap_dims(0, 1); + } + + tensor + }) + .collect::>() + .try_into() + .unwrap() +} + +/// Return a coordinate matrix for a given set of 1D coordinate tensors. +/// +/// Equivalent to stacking a dense matrix `meshgrid`, +/// where the stack is along the first or last dimension. +/// +/// # Arguments +/// +/// * `tensors`: A slice of 1D tensors. +/// * `index_pos`: The position of the index in the output tensor. +/// +/// # Returns +/// +/// A tensor of either ``(N, ..., |T[i]|, ...)`` or ``(..., |T[i]|, ..., N)``, +/// of coordinates, indexed on the first or last dimension. +pub fn meshgrid_stack( + tensors: &[Tensor; D], + index_pos: IndexPos, +) -> Tensor +where + K: BasicOps, +{ + assert_eq!(D2, D + 1, "D2 ({D2}) != D ({D}) + 1"); + + let xs: Vec> = meshgrid(tensors, GridOptions::default()) + .into_iter() + .collect(); + + let dim = match index_pos { + IndexPos::First => 0, + IndexPos::Last => D, + }; + + Tensor::stack(xs, dim) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/mod.rs new file mode 100644 index 0000000..3750962 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/grid/mod.rs @@ -0,0 +1,68 @@ +mod affine_grid; +mod meshgrid; + +pub use meshgrid::*; + +pub use affine_grid::*; + +/// Enum to specify index cardinal layout. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub enum GridIndexing { + /// Dimensions are in the same order as the cardinality of the inputs. + /// Equivalent to "ij" indexing in NumPy and PyTorch. + #[default] + Matrix, + + /// The same as Matrix, but the first two dimensions are swapped. + /// Equivalent to "xy" indexing in NumPy and PyTorch. + Cartesian, +} + +/// Enum to specify grid sparsity mode. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub enum GridSparsity { + /// The grid is fully expanded to the full cartesian product shape. + #[default] + Dense, + + /// The grid is sparse, expanded only at the cardinal dimensions. + Sparse, +} + +/// Grid policy options. +#[derive(new, Default, Debug, Copy, Clone)] +pub struct GridOptions { + /// Indexing mode. + pub indexing: GridIndexing, + + /// Sparsity mode. + pub sparsity: GridSparsity, +} + +impl From for GridOptions { + fn from(value: GridIndexing) -> Self { + Self { + indexing: value, + ..Default::default() + } + } +} +impl From for GridOptions { + fn from(value: GridSparsity) -> Self { + Self { + sparsity: value, + ..Default::default() + } + } +} + +/// Enum to specify the index dimension position. +#[derive(Default, Debug, Copy, Clone)] +pub enum IndexPos { + /// The index is in the first dimension. + #[default] + First, + + /// The index is in the last dimension. + Last, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/cosine_similarity.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/cosine_similarity.rs new file mode 100644 index 0000000..f12c80b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/cosine_similarity.rs @@ -0,0 +1,48 @@ +use crate::ElementConversion; +use crate::backend::Backend; +use crate::tensor::Tensor; + +use super::l2_norm; + +/// Default epsilon value to avoid division by zero +pub const DEFAULT_EPSILON: f64 = 1e-8; +/// Computes the cosine similarity between two tensors along a specified dimension. +/// +/// Calculates the cosine of the angle between inputs as their dot product divided +/// by the product of their L2 norms. +/// +/// # Arguments +/// +/// * `x1` - First input tensor +/// * `x2` - Second input tensor +/// * `dim` - Dimension along which to compute the similarity +/// (negative indices allowed: -1 for last dimension) +/// * `eps` - Small value to avoid division by zero (default: 1e-8) +/// +/// # Returns +/// +/// Tensor containing the cosine similarity between x1 and x2 +pub fn cosine_similarity( + x1: Tensor, + x2: Tensor, + dim: i32, + eps: Option, +) -> Tensor { + let eps = eps.unwrap_or_else(|| B::FloatElem::from_elem(DEFAULT_EPSILON)); + + // Convert negative dimension to positive + let dim_idx = if dim < 0 { D as i32 + dim } else { dim } as usize; + + // Compute dot product: sum(x1 * x2) along the specified dimension + let dot_product = (x1.clone() * x2.clone()).sum_dim(dim_idx); + + // Compute L2 norms: ||x1|| and ||x2|| + let norm_x1 = l2_norm(x1, dim_idx); + let norm_x2 = l2_norm(x2, dim_idx); + + // Calculate the denominator (product of the norms) with epsilon to avoid division by zero + let denominator = norm_x1.clamp_min(eps) * norm_x2.clamp_min(eps); + + // Return the cosine similarity (dot product divided by the product of norms) + dot_product / denominator +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/diag.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/diag.rs new file mode 100644 index 0000000..807a9e6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/diag.rs @@ -0,0 +1,44 @@ +use crate::backend::Backend; +use crate::check; +use crate::check::TensorCheck; +use crate::tensor::{Int, Tensor}; +use crate::{BasicOps, TensorKind}; + +/// Returns the diag of a matrix. +/// +/// For batched inputs, returns of each matrix in the batch independently. +/// +/// The diag operation extracts the diagonal elements of the last two dimensions, +/// treating them as the matrix dimensions, while preserving all leading batch dimensions. +/// +/// # Arguments +/// +/// * `tensor` - The input tensor with at least 2 dimensions. +/// +/// # Returns +/// A tensor of rank `D - 1`, where the last dimension contains the diagonal elements of the input. +pub fn diag( + tensor: Tensor, +) -> Tensor +where + K: TensorKind + BasicOps, +{ + check!(TensorCheck::diag::()); + + let shape = tensor.shape(); + let rows = shape[D - 2]; + let cols = shape[D - 1]; + let diag_len = rows.min(cols); + let device = tensor.device(); + + // create the indices for the diag + let mut flat_shape = shape.clone(); + flat_shape[D - 2] = rows * cols; + flat_shape[D - 1] = 1; + let flat: Tensor = tensor.reshape(flat_shape); + + let range = Tensor::::arange(0..diag_len as i64, &device); + let step_tensor = Tensor::::from_data([cols as i64 + 1], &device); + let indices = range * step_tensor; + flat.take::<1, D>(D - 2, indices).squeeze_dim(D - 1) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/lu_decomposition.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/lu_decomposition.rs new file mode 100644 index 0000000..eec76c3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/lu_decomposition.rs @@ -0,0 +1,79 @@ +use crate::{ + Int, backend::Backend, cast::ToElement, check, check::TensorCheck, linalg::swap_slices, s, + tensor::Tensor, +}; +/// Performs PLU decomposition of a square matrix. +/// +/// The function decomposes a given square matrix `A` into three matrices: a permutation vector `p`, +/// a lower triangular matrix `L`, and an upper triangular matrix `U`, such that `PA = LU`. +/// The permutation vector `p` represents the row swaps made during the decomposition process. +/// The lower triangular matrix `L` has ones on its diagonal and contains the multipliers used +/// during the elimination process below the diagonal. The upper triangular matrix `U` contains +/// the resulting upper triangular form of the matrix after the elimination process. +/// +/// # Arguments +/// * `tensor` - A square matrix to decompose, represented as a 2D tensor. +/// +/// # Returns +/// A tuple containing: +/// - A 2D tensor representing the combined `L` and `U` matrices. +/// - A 1D tensor representing the permutation vector `p`. +/// +/// # Panics and numerical issues +/// - The function will panic if the input matrix is singular or near-singular. +/// - The function will panic if the input matrix is not square. +/// # Performance note (synchronization / device transfers) +/// This function may involve multiple synchronizations and device transfers, especially +/// when determining pivot elements and performing row swaps. This can impact performance, +pub fn lu_decomposition(tensor: Tensor) -> (Tensor, Tensor) { + check!(TensorCheck::is_square::<2>( + "lu_decomposition", + &tensor.shape() + )); + let dims = tensor.shape().dims::<2>(); + let n = dims[0]; + + let mut permutations = Tensor::arange(0..n as i64, &tensor.device()); + let mut tensor = tensor; + + for k in 0..n { + // Find the pivot row + let p = tensor + .clone() + .slice(s![k.., k]) + .abs() + .argmax(0) + .into_scalar() + .to_usize() + + k; + let max = tensor.clone().slice(s![p, k]).abs(); + + // Avoid division by zero + let pivot = max.into_scalar(); + check!(TensorCheck::lu_decomposition_pivot::(pivot)); + + if p != k { + tensor = swap_slices(tensor, s![k, ..], s![p, ..]); + permutations = swap_slices(permutations, s![k], s![p]); + } + + // Normalize k-th column under the diagonal + if k < n - 1 { + let a_kk = tensor.clone().slice(s![k, k]); + let column = tensor.clone().slice(s![(k + 1).., k]) / a_kk; + tensor = tensor.slice_assign(s![(k + 1).., k], column); + } + + // Update the trailing submatrix + for i in (k + 1)..n { + // a[i, k+1..] -= a[i, k] * a[k, k+1..] + let a_ik = tensor.clone().slice(s![i, k]); + let row_k = tensor.clone().slice(s![k, (k + 1)..]); + let update = a_ik * row_k; + let row_i = tensor.clone().slice(s![i, (k + 1)..]); + tensor = tensor.slice_assign(s![i, (k + 1)..], row_i - update); + } + } + + (tensor, permutations) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/matvec.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/matvec.rs new file mode 100644 index 0000000..54abc9c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/matvec.rs @@ -0,0 +1,59 @@ +use crate::Numeric; +use crate::backend::Backend; +use crate::tensor::{BasicOps, Shape, Tensor}; + +/// Performs matrix-vector multiplication with optional batch dimensions. +/// +/// The `matrix` tensor is expected to have rank `DM` with the last two dimensions representing +/// the matrix rows and columns. The `vector` tensor should have rank `DV = DM - 1`, sharing +/// broadcast-compatible batch dimensions and matching the last dimension of the matrix. +/// +/// # Panics +/// +/// * If the matrix rank is lower than 2. +/// * If the vector rank isn't one less than the matrix rank. +/// * If batch dimensions differ between the operands. +/// * If the inner dimensions are incompatible for multiplication. +pub fn matvec( + matrix: Tensor, + vector: Tensor, +) -> Tensor +where + K: BasicOps + Numeric, +{ + assert!( + DM >= 2, + "matvec expects the matrix to be at least rank 2 (got {DM})" + ); + assert!( + DM == DV + 1, + "matvec expects the vector rank ({DV}) to be exactly one less than the matrix rank ({DM})", + ); + + let matrix_dims = matrix.shape().dims::(); + let vector_dims = vector.shape().dims::(); + + // Validate batch dimensions (all leading dimensions prior to the matrix axes). + let batch_rank = DM.saturating_sub(2); + if batch_rank > 0 { + let matrix_batch = Shape::from(&matrix_dims[..batch_rank]); + let vector_batch = Shape::from(&vector_dims[..batch_rank]); + + assert!( + matrix_batch.broadcast(&vector_batch).is_ok(), + "Batch dimensions are not broadcast-compatible: matrix {:?} vs vector {:?}", + &matrix_dims[..batch_rank], + &vector_dims[..batch_rank] + ); + } + + let matrix_inner = matrix_dims[DM - 1]; + let vector_inner = vector_dims[DV - 1]; + assert!( + matrix_inner == vector_inner, + "Inner dimension mismatch: matrix has {matrix_inner} columns but vector has {vector_inner} entries", + ); + + let vector_expanded = vector.unsqueeze_dim::(DV); + matrix.matmul(vector_expanded).squeeze_dim::(DM - 1) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/mod.rs new file mode 100644 index 0000000..ac235bb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/mod.rs @@ -0,0 +1,42 @@ +mod cosine_similarity; +mod diag; +mod lu_decomposition; +mod matvec; +mod outer; +mod trace; +mod vector_norm; + +pub use cosine_similarity::*; +pub use diag::*; +pub use lu_decomposition::*; +pub use matvec::*; +pub use outer::*; +pub use trace::*; +pub use vector_norm::*; + +use crate::{BasicOps, SliceArg, Tensor, TensorKind, backend::Backend}; + +/// Swaps two slices of a tensor. +/// # Arguments +/// * `tensor` - The input tensor. +/// * `slices1` - The first slice to swap. +/// * `slices2` - The second slice to swap. +/// # Returns +/// A new tensor with the specified slices swapped. +/// # Notes +/// This method will be useful for matrix factorization algorithms. +fn swap_slices( + tensor: Tensor, + slices1: S, + slices2: S, +) -> Tensor +where + S: SliceArg + Clone, + K: TensorKind + BasicOps, +{ + let temporary = tensor.clone().slice(slices1.clone()); + let tensor = tensor + .clone() + .slice_assign(slices1, tensor.slice(slices2.clone())); + tensor.slice_assign(slices2, temporary) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/outer.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/outer.rs new file mode 100644 index 0000000..d686b44 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/outer.rs @@ -0,0 +1,77 @@ +use crate::backend::Backend; +use crate::tensor::{BasicOps, Tensor}; +use crate::{AsIndex, Numeric}; + +/// Computes the outer product for the last columns of 2 tensors. +/// +/// See also: [`outer_dim`]. +/// +/// # Arguments +/// - `lhs`: the "row" tensor, with shape ``[..., i]``. +/// - `rhs`: the "col" tensor, with shape ``[..., j]``. +/// - `dim`: the dimension to product. +/// +/// # Returns +/// +/// A tensor of rank `R = D + 1`, where: +/// +/// `` +/// result[..., i, j] = lhs[..., i] * rhs[..., j] +/// `` +pub fn outer( + x: Tensor, + y: Tensor, +) -> Tensor +where + K: BasicOps + Numeric, +{ + outer_dim(x, y, -1) +} + +/// Computes the outer product along a specific dimension, broadcasting over others. +/// +/// For the given `dim`, computes the outer product of elements along that dimension, +/// expanding it into two dimensions of size ``M × N`` at positions ``(dim, dim + 1)``. +/// +/// # Arguments +/// +/// - `lhs`: left operand, the "row" tensor, with size `M` at dimension `dim`. +/// - `rhs`: right operand, the "col" tensor, with size `N` at dimension `dim`. +/// - `dim`: dimension to compute the outer product along (supports negative indexing). +/// +/// # Returns +/// +/// A tensor of rank `R = D + 1`, where: +/// +/// `` +/// result[..., i, j, ...] = lhs[..., i, ...] * rhs[..., j, ...] +/// `` +// +// Notes: +// - For large batched inputs, `x_col.matmul(y_row)` *might* be more performant +// than broadcasted elemwise multiply; benchmarking needed to confirm. +pub fn outer_dim( + lhs: Tensor, + rhs: Tensor, + dim: Dim, +) -> Tensor +where + K: BasicOps + Numeric, +{ + assert_eq!( + R, + D + 1, + "`outer` with D={D} expects R={} (got R={R})", + D + 1 + ); + let dim = dim.expect_dim_index(D); + + // (..., i, 1, ...) + let x = lhs.unsqueeze_dim::(dim + 1); + + // (..., 1, j, ...) + let y = rhs.unsqueeze_dim::(dim); + + // (..., i, j, ...) + x * y +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/trace.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/trace.rs new file mode 100644 index 0000000..ccac2d2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/trace.rs @@ -0,0 +1,24 @@ +use super::diag; +use crate::backend::Backend; +use crate::tensor::Tensor; + +/// Computes the trace of a matrix. +/// +/// For batched inputs, computes the trace of each matrix in the batch independently. +/// +/// The trace operation sums the diagonal elements of the last two dimensions, +/// treating them as the matrix dimensions, while preserving all leading batch dimensions. +/// +/// # Arguments +/// +/// * `tensor` - The input tensor with at least 2 dimensions. +/// +/// # Returns +/// +/// A tensor of rank `D - 1`, where the last dimension contains the sum along the diagonals +/// of the input. +pub fn trace(tensor: Tensor) -> Tensor { + let diag_tensor = diag::<_, D, DO, _>(tensor); + + diag_tensor.sum_dim(DO - 1) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/vector_norm.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/vector_norm.rs new file mode 100644 index 0000000..670096a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/linalg/vector_norm.rs @@ -0,0 +1,291 @@ +use burn_backend::tensor::Ordered; + +use crate::backend::Backend; +use crate::tensor::{BasicOps, Tensor}; +use crate::{ElementConversion, Numeric}; +#[allow(unused_imports)] +use num_traits::float::Float; +/// Specifies the type of norm to compute. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Norm { + /// L0 norm (count of non-zero elements) + L0, + + /// L1 norm (sum of absolute values) + L1, + + /// L2 norm (Euclidean norm) + L2, + + /// L:INFINITY norm (maximum absolute value) + LInf, + + /// L:NEG_INFINITY norm (minimum absolute value) + LNegInf, + + /// Lp norm (generalized norm) + Lp(f64), +} + +impl Norm { + /// Get the exponent of the norm. + pub fn to_exponent(self) -> f64 { + use Norm::*; + match self { + L0 => 0.0, + L1 => 1.0, + L2 => 2.0, + LInf => f64::INFINITY, + LNegInf => f64::NEG_INFINITY, + Lp(p) => p, + } + } +} + +impl From for Norm { + fn from(value: u32) -> Self { + use Norm::*; + match value { + 0 => L0, + 1 => L1, + 2 => L2, + u32::MAX => LInf, + _ => Lp(value as f64), + } + } +} + +impl From for Norm { + fn from(value: i32) -> Self { + use Norm::*; + match value { + 0 => L0, + 1 => L1, + 2 => L2, + i32::MAX => LInf, + i32::MIN => LNegInf, + _ => Lp(value as f64), + } + } +} + +impl From for Norm { + fn from(value: f32) -> Self { + use Norm::*; + match value { + 0.0 => L0, + 1.0 => L1, + 2.0 => L2, + f32::INFINITY => LInf, + f32::NEG_INFINITY => LNegInf, + _ => Lp(value as f64), + } + } +} + +impl From for Norm { + fn from(value: f64) -> Self { + use Norm::*; + match value { + 0.0 => L0, + 1.0 => L1, + 2.0 => L2, + f64::INFINITY => LInf, + f64::NEG_INFINITY => LNegInf, + _ => Lp(value), + } + } +} + +/// Computes the vector norm of a tensor along a specified dimension. +/// +/// Generic dispatch wrapper over specialized / optimized norms. +/// +/// See: +/// - [torch.linalg.vector_norm](https://pytorch.org/docs/stable/generated/torch.linalg.vector_norm.html) +/// - [numpy.linalg.vector_norm](https://numpy.org/doc/stable/reference/generated/numpy.linalg.vector_norm.html) +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `norm` - The selected norm. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The vector norm of the input tensor. +pub fn vector_norm( + x: Tensor, + norm: impl Into, + dim: usize, +) -> Tensor { + lp_norm(x, norm.into().to_exponent(), dim) +} + +/// Computes the general ``L(p)`` norm of a tensor along a specified dimension. +/// +/// Uses the specialized implementations for: +/// * 0.0 +/// * 1.0 +/// * 2.0 +/// * 2 * N for integral N, +/// * f64::INFINITY, +/// * f64::NEG_INFINITY, +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `p` - The exponent of the Lp norm. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The ``L(p)`` norm of the input tensor. +pub fn lp_norm(x: Tensor, p: f64, dim: usize) -> Tensor { + match p { + 0.0 => l0_norm(x, dim), + 1.0 => l1_norm(x, dim), + 2.0 => l2_norm(x, dim), + p if is_even_integer(p) => lp_signed_norm(x, p as u32, dim), + f64::INFINITY => max_abs_norm(x, dim), + f64::NEG_INFINITY => min_abs_norm(x, dim), + _ => lp_norm_base(x, p, dim), + } +} + +/// Normalize a tensor versus its `vector_norm`. +/// +/// Equivalent to ``x.clone() / vector_norm(x, norm, dim).clamp_min(eps)``. +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `norm` - The selected norm. +/// * `dim` - The dimension to compute the norm over. +/// * `eps` - The epsilon for the norm. +/// +/// # Returns +/// +/// The normalized tensor. +pub fn vector_normalize( + x: Tensor, + norm: impl Into, + dim: usize, + eps: E, +) -> Tensor { + let norm = vector_norm(x.clone(), norm, dim).clamp_min(eps); + x / norm +} + +/// Computes the L0 norm of a tensor along a specified dimension. +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The L0 norm of the input tensor. +pub fn l0_norm(x: Tensor, dim: usize) -> Tensor +where + K: BasicOps + Numeric, +{ + x.zeros_like() + .mask_fill(x.not_equal_elem(0), 1) + .sum_dim(dim) +} + +/// Computes the L1 norm of a tensor along a specified dimension. +/// +/// This is a convenience function that wraps `vector_norm` with `p = 1.0`. +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The L1 norm of the input tensor. +pub fn l1_norm(x: Tensor, dim: usize) -> Tensor +where + K: BasicOps + Numeric, +{ + x.abs().sum_dim(dim) +} + +/// Computes the L2 norm of a tensor along a specified dimension. +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The L2 norm of the input tensor. +pub fn l2_norm(x: Tensor, dim: usize) -> Tensor { + x.square().sum_dim(dim).sqrt() +} + +fn is_even_integer(x: f64) -> bool { + x.fract() == 0.0 && (x as i64) % 2 == 0 +} + +/// Computes ``L(2*n)`` for even integer ``n``. +/// +/// This lets us skip the abs. +fn lp_signed_norm(x: Tensor, p: u32, dim: usize) -> Tensor { + x.powi_scalar(p).sum_dim(dim).powf_scalar(1. / (p as f64)) +} + +/// Computes the general ``L(p)`` using the generalized method. +/// +/// This uses no specialized implementations and cannot handle: +/// * 0.0 +/// * f64::INFINITY, +/// * f64::NEG_INFINITY, +fn lp_norm_base(x: Tensor, p: f64, dim: usize) -> Tensor { + x.abs().powf_scalar(p).sum_dim(dim).powf_scalar(1. / p) +} + +/// Computes the L:INFINITY norm of a tensor along a specified dimension. +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The L:INFINITY norm of the input tensor. +pub fn max_abs_norm( + x: Tensor, + dim: usize, +) -> Tensor +where + K: Ordered, +{ + x.max_abs_dim(dim) +} + +/// Computes the L:NEG_INFINITY norm of a tensor along a specified dimension. +/// +/// # Arguments +/// +/// * `x` - The input tensor. +/// * `dim` - The dimension to compute the norm over. +/// +/// # Returns +/// +/// The L:NEG_INFINITY norm of the input tensor. +pub fn min_abs_norm( + x: Tensor, + dim: usize, +) -> Tensor +where + K: Ordered, +{ + x.abs().min_dim(dim) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/loss/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/loss/mod.rs new file mode 100644 index 0000000..6f1afff --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/loss/mod.rs @@ -0,0 +1,23 @@ +use crate::backend::Backend; +use crate::{Tensor, activation}; + +/// Computes the log softmax cross entropy between logits and target probabilities. +/// +/// # Arguments +/// +/// * `logits` - The logits. +/// * `target_probs` - The target probabilities. +/// +/// # Returns +/// +/// The log softmax cross entropy. +pub fn cross_entropy_with_logits( + logits: Tensor, + target_probs: Tensor, +) -> Tensor { + let tensor = activation::log_softmax(logits, D - 1); + let tensor = tensor.mul(target_probs); + let tensor = tensor.sum_dim(D - 1); + + tensor.mean().neg() +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/mod.rs new file mode 100644 index 0000000..dfa88dc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/mod.rs @@ -0,0 +1,67 @@ +pub(crate) mod stats; + +mod api; + +pub use api::*; + +// Re-exported types +pub use burn_backend::{ + DType, DataError, FloatDType, IntDType, TensorData, TensorMetadata, TensorPrimitive, Tolerance, + distribution::*, + element::*, + indexing::*, + ops::TransactionPrimitive, + shape::*, + slice::*, + tensor::{Bool, Float, Int, TensorKind}, +}; + +/// The activation module. +pub mod activation; + +/// The backend module. +pub mod backend { + pub use burn_backend::backend::*; +} + +/// The container module. +pub mod container { + pub use burn_backend::tensor::TensorContainer; +} + +/// The grid module. +pub mod grid; + +/// The linalg module. +pub mod linalg; + +/// The loss module. +pub mod loss; + +/// The neural network module. +pub mod module; + +/// Operations on tensors module. +pub mod ops { + pub use burn_backend::backend::ops::*; + pub use burn_backend::tensor::{ + BoolElem, BoolTensor, Device, FloatElem, FloatTensor, IntElem, IntTensor, QuantizedTensor, + }; +} + +/// Tensor quantization module. +pub mod quantization; + +#[cfg(feature = "std")] +pub use report::*; + +#[cfg(feature = "std")] +mod report; + +#[cfg(feature = "experimental-named-tensor")] +mod named; + +#[cfg(feature = "experimental-named-tensor")] +pub use named::*; + +pub use ops::Device; // Re-export device so that it's available from `burn_tensor::Device`. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/module.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/module.rs new file mode 100644 index 0000000..3df0168 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/module.rs @@ -0,0 +1,555 @@ +use crate::{ + Bool, Int, Tensor, TensorPrimitive, + backend::Backend, + check, + check::TensorCheck, + ops::{ + AttentionModuleOptions, ConvOptions, ConvTransposeOptions, InterpolateOptions, PadMode, + PaddedConvOptions, UnfoldOptions, + }, +}; + +use super::ops::DeformConvOptions; + +/// Applies the [embedding module](crate::ops::ModuleOps::embedding). +pub fn embedding(weights: Tensor, indices: Tensor) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::embedding( + weights.primitive.tensor(), + indices.primitive, + ))) +} + +/// Applies a [1D convolution](crate::ops::ModuleOps::conv1d). +/// +/// Accepts [`ConvOptions`] for symmetric padding, or [`PaddedConvOptions`] for +/// asymmetric padding. When asymmetric padding is specified, an explicit pad +/// operation is applied before the convolution backend op. +pub fn conv1d( + x: Tensor, + weight: Tensor, + bias: Option>, + options: impl Into>, +) -> Tensor +where + B: Backend, +{ + let padded_options = options.into(); + check!(TensorCheck::conv( + "conv1d", + x.dims(), + weight.dims(), + padded_options.options.groups, + )); + + if let Some(padding_end) = padded_options.padding_end { + let left = padded_options.options.padding[0]; + let right = padding_end[0]; + // For 1D (NCL format), pad the length dimension + let padded = x.pad((left, right, 0, 0), PadMode::Constant(0.0)); + let zero_options = ConvOptions::new( + padded_options.options.stride, + [0], + padded_options.options.dilation, + padded_options.options.groups, + ); + Tensor::new(TensorPrimitive::Float(B::conv1d( + padded.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + zero_options, + ))) + } else { + Tensor::new(TensorPrimitive::Float(B::conv1d( + x.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + padded_options.options, + ))) + } +} + +/// Applies a [2D convolution](crate::ops::ModuleOps::conv2d). +/// +/// Accepts [`ConvOptions`] for symmetric padding, or [`PaddedConvOptions`] for +/// asymmetric padding. When asymmetric padding is specified, an explicit pad +/// operation is applied before the convolution backend op. +pub fn conv2d( + x: Tensor, + weight: Tensor, + bias: Option>, + options: impl Into>, +) -> Tensor +where + B: Backend, +{ + let padded_options = options.into(); + check!(TensorCheck::conv( + "conv2d", + x.dims(), + weight.dims(), + padded_options.options.groups, + )); + + if let Some(padding_end) = padded_options.padding_end { + let top = padded_options.options.padding[0]; + let left = padded_options.options.padding[1]; + let bottom = padding_end[0]; + let right = padding_end[1]; + // For 2D (NCHW format), pad height and width + let padded = x.pad((left, right, top, bottom), PadMode::Constant(0.0)); + let zero_options = ConvOptions::new( + padded_options.options.stride, + [0, 0], + padded_options.options.dilation, + padded_options.options.groups, + ); + Tensor::new(TensorPrimitive::Float(B::conv2d( + padded.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + zero_options, + ))) + } else { + Tensor::new(TensorPrimitive::Float(B::conv2d( + x.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + padded_options.options, + ))) + } +} + +/// Applies a [3D convolution](crate::ops::ModuleOps::conv3d). +/// +/// Accepts [`ConvOptions`] for symmetric padding, or [`PaddedConvOptions`] for +/// asymmetric padding. Asymmetric 3D padding is not yet supported. +pub fn conv3d( + x: Tensor, + weight: Tensor, + bias: Option>, + options: impl Into>, +) -> Tensor +where + B: Backend, +{ + let padded_options = options.into(); + check!(TensorCheck::conv( + "conv3d", + x.dims(), + weight.dims(), + padded_options.options.groups, + )); + + if padded_options.is_asymmetric() { + panic!("Asymmetric padding is not yet supported for conv3d"); + } + + Tensor::new(TensorPrimitive::Float(B::conv3d( + x.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + padded_options.options, + ))) +} + +/// Applies a [Deformable 2D convolution](crate::ops::ModuleOps::deform_conv2d). +pub fn deform_conv2d( + x: Tensor, + offset: Tensor, + weight: Tensor, + mask: Option>, + bias: Option>, + options: DeformConvOptions<2>, +) -> Tensor +where + B: Backend, +{ + check!(TensorCheck::conv( + "deform_conv2d", + x.dims(), + weight.dims(), + options.weight_groups, + )); + Tensor::new(TensorPrimitive::Float(B::deform_conv2d( + x.primitive.tensor(), + offset.primitive.tensor(), + weight.primitive.tensor(), + mask.map(|m| m.primitive.tensor()), + bias.map(|b| b.primitive.tensor()), + options, + ))) +} + +/// Applies a [1D transposed convolution](crate::ops::ModuleOps::conv_transpose1d). +pub fn conv_transpose1d( + x: Tensor, + weight: Tensor, + bias: Option>, + options: ConvTransposeOptions<1>, +) -> Tensor +where + B: Backend, +{ + check!(TensorCheck::conv_transpose( + "conv_transpose1d", + x.dims(), + weight.dims(), + )); + Tensor::new(TensorPrimitive::Float(B::conv_transpose1d( + x.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + options, + ))) +} + +/// Applies a [2D transposed convolution](crate::ops::ModuleOps::conv_transpose2d). +pub fn conv_transpose2d( + x: Tensor, + weight: Tensor, + bias: Option>, + options: ConvTransposeOptions<2>, +) -> Tensor +where + B: Backend, +{ + check!(TensorCheck::conv_transpose( + "conv_transpose2d", + x.dims(), + weight.dims(), + )); + Tensor::new(TensorPrimitive::Float(B::conv_transpose2d( + x.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + options, + ))) +} + +/// Applies a 3D transposed convolution](crate::ops::ModuleOps::conv_transpose3d). +pub fn conv_transpose3d( + x: Tensor, + weight: Tensor, + bias: Option>, + options: ConvTransposeOptions<3>, +) -> Tensor +where + B: Backend, +{ + check!(TensorCheck::conv_transpose( + "conv_transpose3d", + x.dims(), + weight.dims(), + )); + Tensor::new(TensorPrimitive::Float(B::conv_transpose3d( + x.primitive.tensor(), + weight.primitive.tensor(), + bias.map(|b| b.primitive.tensor()), + options, + ))) +} + +/// Applies a [4D to 3D unfold](crate::ops::ModuleOps::unfold4d). +pub fn unfold4d(x: Tensor, kernel_size: [usize; 2], options: UnfoldOptions) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::unfold4d( + x.primitive.tensor(), + kernel_size, + options, + ))) +} + +/// Applies a [1D max pooling](crate::ops::ModuleOps::max_pool1d). +pub fn max_pool1d( + x: Tensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, +) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::max_pool1d( + x.primitive.tensor(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ))) +} + +/// Applies a [2D max pooling](crate::ops::ModuleOps::max_pool2d). +pub fn max_pool2d( + x: Tensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, +) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::max_pool2d( + x.primitive.tensor(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ))) +} + +/// Applies a [2D avg pooling](crate::ops::ModuleOps::avg_pool2d). +pub fn avg_pool2d( + x: Tensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + count_include_pad: bool, + ceil_mode: bool, +) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::avg_pool2d( + x.primitive.tensor(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ))) +} + +/// Applies a [1D avg pooling](crate::ops::ModuleOps::avg_pool1d). +pub fn avg_pool1d( + x: Tensor, + kernel_size: usize, + stride: usize, + padding: usize, + count_include_pad: bool, + ceil_mode: bool, +) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::avg_pool1d( + x.primitive.tensor(), + kernel_size, + stride, + padding, + count_include_pad, + ceil_mode, + ))) +} + +/// Applies a [1D max pooling](crate::ops::ModuleOps::max_pool1d). +pub fn max_pool1d_with_indices( + x: Tensor, + kernel_size: usize, + stride: usize, + padding: usize, + dilation: usize, + ceil_mode: bool, +) -> (Tensor, Tensor) +where + B: Backend, +{ + let output = B::max_pool1d_with_indices( + x.primitive.tensor(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + + ( + Tensor::new(TensorPrimitive::Float(output.output)), + Tensor::new(output.indices), + ) +} + +/// Applies a [2D max pooling with indices](crate::ops::ModuleOps::max_pool2d_with_indices). +pub fn max_pool2d_with_indices( + x: Tensor, + kernel_size: [usize; 2], + stride: [usize; 2], + padding: [usize; 2], + dilation: [usize; 2], + ceil_mode: bool, +) -> (Tensor, Tensor) +where + B: Backend, +{ + let output = B::max_pool2d_with_indices( + x.primitive.tensor(), + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ); + + ( + Tensor::new(TensorPrimitive::Float(output.output)), + Tensor::new(output.indices), + ) +} + +/// Applies a [2D adaptive avg pooling](crate::ops::ModuleOps::adaptive_avg_pool2d). +pub fn adaptive_avg_pool2d(x: Tensor, output_size: [usize; 2]) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::adaptive_avg_pool2d( + x.primitive.tensor(), + output_size, + ))) +} + +/// Applies a [1D adaptive avg pooling](crate::ops::ModuleOps::adaptive_avg_pool1d). +pub fn adaptive_avg_pool1d(x: Tensor, output_size: usize) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::adaptive_avg_pool1d( + x.primitive.tensor(), + output_size, + ))) +} + +/// Applies a [2D interpolation](crate::ops::ModuleOps::interpolate). +pub fn interpolate( + x: Tensor, + output_size: [usize; 2], + options: InterpolateOptions, +) -> Tensor +where + B: Backend, +{ + Tensor::new(TensorPrimitive::Float(B::interpolate( + x.primitive.tensor(), + output_size, + options, + ))) +} + +/// Applies a linear transformation to the input tensor using the given weight and bias. +/// +/// ```math +/// y = x @ weight + [bias] +/// ``` +/// +/// # Arguments: +/// +/// - `input` is the input tensor, ``[..., d_input]``. +/// - `weight` is the weight tensor, ``[d_input, d_output]``. +/// - `bias` is the bias tensor (optional), ``[d_output]``. +/// +/// # Returns: +/// +/// The transformed tensor, ``[..., d_output]``. +/// +/// # Compatibility +/// +/// This function differs from PyTorch's ``torch.nn.functional.linear`` in that it does not +/// transpose the weight matrix. In PyTorch, the weight matrix is transposed before +/// multiplication: +/// +/// ```math +/// y = x @ weight^T + [bias] +/// ``` +pub fn linear( + input: Tensor, + weight: Tensor, + bias: Option>, +) -> Tensor { + if D == 1 { + // Insert and remove an extra batch dimension for the batch matmul to work. + let input = input.unsqueeze::<2>(); + let output = linear(input, weight, bias); + return output.squeeze_dim(0); + } + + // Perform broadcasting + // + // Important to be done before doing operations to easily fuse. + let weight = weight.unsqueeze::(); + let bias = bias.map(|bias| bias.unsqueeze::()); + + let output = input.matmul(weight); + match bias { + Some(bias) => output.add(bias), + None => output, + } +} + +/// Computes scaled dot-product attention: softmax(QKᵗ * scale) · V, +/// where scale defaults to 1/sqrt(head_dim) (configurable via `options.scale`). +/// Optionally applies masking, additive bias, causal masking, and softcap. +/// +/// # Arguments +/// - `query`: Query tensor of shape `[batch_size, num_heads, seq_len_q, head_dim]` +/// - `key`: Key tensor of shape `[batch_size, num_heads, seq_len_k, head_dim]` +/// - `value`: Value tensor of shape `[batch_size, num_heads, seq_len_k, val_dim]` +/// - `mask`: Optional boolean mask of shape `[batch_size, num_heads, seq_len_q, seq_len_k]`, +/// where `true` indicates positions to mask (i.e. set to -inf before softmax). +/// - `attn_bias`: Optional float tensor of shape `[batch_size, num_heads, seq_len_q, seq_len_k]` +/// added to the attention scores before softmax (e.g. ALiBi, relative position biases). +/// - `options`: Additional attention options (custom scale, softcap, causal masking). +/// +/// # Returns +/// A tensor of shape `[batch_size, num_heads, seq_len_q, val_dim]` +/// representing the attended context per head. +/// +/// # Note +/// This implementation does not support dropout and is intended for inference or +/// use cases where dropout is not needed. +pub fn attention( + query: Tensor, + key: Tensor, + value: Tensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, +) -> Tensor { + Tensor::new(TensorPrimitive::Float(B::attention( + query.primitive.tensor(), + key.primitive.tensor(), + value.primitive.tensor(), + mask.map(|mask| mask.primitive), + attn_bias.map(|bias| bias.primitive.tensor()), + options, + ))) +} + +/// Exports attention fallback to test backend's attention against. +pub fn attention_fallback( + query: Tensor, + key: Tensor, + value: Tensor, + mask: Option>, + attn_bias: Option>, + options: AttentionModuleOptions, +) -> Tensor { + Tensor::new(TensorPrimitive::Float( + crate::ops::attention::attention_fallback::( + query.primitive.tensor(), + key.primitive.tensor(), + value.primitive.tensor(), + mask.map(|mask| mask.primitive), + attn_bias.map(|bias| bias.primitive.tensor()), + options, + ), + )) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/base.rs new file mode 100644 index 0000000..7f91a3c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/base.rs @@ -0,0 +1,85 @@ +use alloc::format; + +use crate::backend::Backend; +use crate::{Distribution, NamedDims, Shape, Tensor}; + +/// A tensor with named dimensions. +#[derive(Debug, Clone)] +pub struct NamedTensor> { + pub(crate) tensor: D::Tensor, +} + +impl>, const D: usize> From> + for Tensor +{ + fn from(nt: NamedTensor) -> Self { + nt.tensor + } +} + +impl>, const D: usize> From> + for NamedTensor +{ + fn from(tensor: Tensor) -> Self { + Self::from_tensor(tensor) + } +} + +impl> core::fmt::Display for NamedTensor +where + ND: NamedDims>, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(&format!( + "NamedTensor[shape={:?}, dims={}]", + self.shape(), + ND::to_string(), + )) + } +} + +impl NamedTensor +where + ND: NamedDims>, +{ + /// Create a named tensor from a tensor. + pub fn from_tensor(tensor: Tensor) -> Self { + Self { tensor } + } + + /// Create a random named tensor of the given shape where each element is sampled from + /// the given distribution. + pub fn random>( + shape: S, + distribution: Distribution, + device: &B::Device, + ) -> Self { + Self::from_tensor(Tensor::random(shape, distribution, device)) + } + + /// Returns the shape of the current tensor. + pub fn shape(&self) -> Shape { + self.tensor.shape() + } + + /// Applies element wise multiplication operation. + /// + /// `y = x2 * x1` + #[allow(clippy::should_implement_trait)] + pub fn mul(self, rhs: Self) -> Self { + Self::from_tensor(self.tensor.mul(rhs.tensor)) + } + + /// Reshape the tensor to have the given shape. + /// + /// # Panics + /// + /// If the tensor can not be reshape to the given shape. + pub fn reshape(self, shape: S, _: ND2) -> NamedTensor + where + S: Into, + ND2: NamedDims>, + { + NamedTensor::from_tensor(self.tensor.reshape(shape.into())) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/dims.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/dims.rs new file mode 100644 index 0000000..d427d2b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/dims.rs @@ -0,0 +1,95 @@ +use alloc::format; +use alloc::string::String; + +use crate::Tensor; +use crate::backend::Backend; + +/// Dimension trait. +pub trait Dim: core::fmt::Debug { + /// Converts the dimension to a string. + fn to_string() -> String; +} + +/// Named dimensions trait. +pub trait NamedDims: core::fmt::Debug { + /// Tensor type. + type Tensor; + + /// Converts the named dimensions to a string. + fn to_string() -> String; +} + +/// Named dimension macro. +#[macro_export] +macro_rules! NamedDim { + ($name:ident) => { + #[derive(Debug, Clone)] + pub struct $name; + impl Dim for $name { + fn to_string() -> String { + stringify!($name).to_string() + } + } + }; +} + +impl NamedDims for (D1,) +where + B: Backend, + D1: Dim, +{ + type Tensor = Tensor; + fn to_string() -> String { + format!("[{}]", D1::to_string()) + } +} + +impl NamedDims for (D1, D2) +where + B: Backend, + D1: Dim, + D2: Dim, +{ + type Tensor = Tensor; + fn to_string() -> String { + format!("[{}, {}]", D1::to_string(), D2::to_string()) + } +} + +impl NamedDims for (D1, D2, D3) +where + B: Backend, + D1: Dim, + D2: Dim, + D3: Dim, +{ + type Tensor = Tensor; + fn to_string() -> String { + format!( + "[{}, {}, {}]", + D1::to_string(), + D2::to_string(), + D3::to_string() + ) + } +} + +impl NamedDims for (D1, D2, D3, D4) +where + B: Backend, + D1: Dim, + D2: Dim, + D3: Dim, + D4: Dim, +{ + type Tensor = Tensor; + fn to_string() -> String { + format!( + "[{}, {}, {}, {}]", + D1::to_string(), + D2::to_string(), + D3::to_string(), + D4::to_string() + ) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/matmul.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/matmul.rs new file mode 100644 index 0000000..ef8e984 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/matmul.rs @@ -0,0 +1,59 @@ +use crate::backend::Backend; +use crate::{Dim, NamedDims, NamedTensor, Tensor}; + +pub trait Matmul { + fn matmul(self, rhs: Rhs) -> Out; +} + +impl NamedTensor +where + ND: NamedDims>, +{ + /// Applies the matrix multiplication operation. + /// + /// `C = AB` + /// + /// # Panics + /// + /// If the two tensors dont' have a compatible shape. + pub fn matmul( + self, + rhs: NamedTensor, + ) -> NamedTensor + where + NamedDimsRhs: NamedDims>, + NamedDimsOut: NamedDims>, + Self: Matmul, NamedTensor>, + { + Matmul::matmul(self, rhs) + } +} + +impl Matmul, NamedTensor> + for NamedTensor +{ + fn matmul(self, rhs: NamedTensor) -> NamedTensor { + NamedTensor::from_tensor(self.tensor.matmul(rhs.tensor)) + } +} + +impl + Matmul, NamedTensor> + for NamedTensor +{ + fn matmul(self, rhs: NamedTensor) -> NamedTensor { + NamedTensor::from_tensor(self.tensor.matmul(rhs.tensor)) + } +} + +impl + Matmul, NamedTensor> + for NamedTensor +{ + fn matmul( + self, + rhs: NamedTensor, + ) -> NamedTensor { + NamedTensor::from_tensor(self.tensor.matmul(rhs.tensor)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/mod.rs new file mode 100644 index 0000000..b262c2f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/mod.rs @@ -0,0 +1,7 @@ +mod base; +mod dims; +mod matmul; +mod swap_dims; + +pub use base::*; +pub use dims::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/swap_dims.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/swap_dims.rs new file mode 100644 index 0000000..2a9a356 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/named/swap_dims.rs @@ -0,0 +1,62 @@ +use crate::backend::Backend; +use crate::{Dim, NamedDims, NamedTensor, Tensor}; + +pub trait SwapDims { + fn swap_dims(self) -> N; +} + +impl NamedTensor +where + ND: NamedDims>, +{ + /// Swap two dimensions. + pub fn swap_dims(self) -> NamedTensor + where + ND2: NamedDims>, + Self: SwapDims, D1, D2>, + { + SwapDims::swap_dims(self) + } +} + +macro_rules! generate_permute { + (2 => $output:ty, ($dim1:expr, $dim2:expr)) => { + impl SwapDims, $dim1, $dim2> + for NamedTensor + { + fn swap_dims(self) -> NamedTensor { + NamedTensor::from_tensor(self.tensor.swap_dims($dim1, $dim2)) + } + } + }; + + (3 => $output:ty, ($dim1:expr, $dim2:expr)) => { + impl SwapDims, $dim1, $dim2> + for NamedTensor + { + fn swap_dims(self) -> NamedTensor { + NamedTensor::from_tensor(self.tensor.swap_dims($dim1, $dim2)) + } + } + }; + + (4 => $output:ty, ($dim1:expr, $dim2:expr)) => { + impl + SwapDims, $dim1, $dim2> for NamedTensor + { + fn swap_dims(self) -> NamedTensor { + NamedTensor::from_tensor(self.tensor.swap_dims($dim1, $dim2)) + } + } + }; +} + +generate_permute!(2 => (D2, D1), (0, 1)); +generate_permute!(3 => (D2, D1, D3), (0, 1)); +generate_permute!(3 => (D3, D2, D1), (0, 2)); +generate_permute!(3 => (D1, D3, D2), (1, 2)); +generate_permute!(4 => (D2, D1, D3, D4), (0, 1)); +generate_permute!(4 => (D3, D2, D1, D4), (0, 2)); +generate_permute!(4 => (D4, D2, D3, D1), (0, 3)); +generate_permute!(4 => (D1, D3, D2, D4), (1, 2)); +generate_permute!(4 => (D1, D4, D3, D2), (1, 3)); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/quantization.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/quantization.rs new file mode 100644 index 0000000..8373bcd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/quantization.rs @@ -0,0 +1,52 @@ +use crate::{Tensor, TensorPrimitive, backend::Backend}; +use burn_backend::tensor::quantization; + +// We re-export those types. +pub use burn_backend::{QTensorPrimitive, quantization::*}; + +/// The tensor quantization parameters. +pub type QuantizationParameters = QParams>; + +/// The observed input calibration range. +#[derive(Clone, Debug)] +pub struct CalibrationRange { + /// Minimum observed value(s). + pub min: Tensor, + /// Maximum observed value(s). + pub max: Tensor, +} + +/// Compute the quantization range mapping. +pub fn compute_range( + scheme: &QuantScheme, + tensor: &Tensor, + calibration: &Calibration, +) -> CalibrationRange { + let (min, max) = match &tensor.primitive { + TensorPrimitive::Float(tensor) => { + quantization::compute_range::(scheme, tensor.clone(), calibration) + } + TensorPrimitive::QFloat(_) => unreachable!(), + }; + + CalibrationRange { + min: Tensor::from_primitive(TensorPrimitive::Float(min)), + max: Tensor::from_primitive(TensorPrimitive::Float(max)), + } +} + +/// Compute the quantization parameters. +pub fn compute_q_params( + scheme: &QuantScheme, + range: CalibrationRange, +) -> QuantizationParameters { + match (range.min.primitive, range.max.primitive) { + (TensorPrimitive::Float(min), TensorPrimitive::Float(max)) => { + let qparams = quantization::compute_q_params::(scheme, min, max); + QuantizationParameters { + scales: Tensor::from_primitive(TensorPrimitive::Float(qparams.scales)), + } + } + _ => unreachable!(), + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/report.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/report.rs new file mode 100644 index 0000000..9860940 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/report.rs @@ -0,0 +1,106 @@ +use super::{Tensor, backend::Backend}; + +use colored::*; + +/// Checks the closeness of two tensors and prints the results. +/// +/// Compares tensors by checking the absolute difference between each element. +/// Prints the percentage of elements within specified tolerances. +/// +/// # Arguments +/// +/// * `output` - The output tensor. +/// * `expected` - The expected tensor. +/// +/// # Example +/// +/// ```no_run +/// use burn_tensor::backend::Backend; +/// use burn_tensor::{check_closeness, Tensor}; +/// +/// fn example() { +/// let device = Default::default(); +/// let tensor1 = Tensor::::from_floats( +/// [1.0, 2.0, 3.0, 4.0, 5.0, 6.001, 7.002, 8.003, 9.004, 10.1], +/// &device, +/// ); +/// let tensor2 = Tensor::::from_floats( +/// [1.0, 2.0, 3.0, 4.000, 5.0, 6.0, 7.001, 8.002, 9.003, 10.004], +/// &device, +/// ); +/// check_closeness(&tensor1, &tensor2); +///} +/// ``` +/// +/// # Output +/// +/// ```text +/// Tensor Closeness Check Results: +/// =============================== +/// Epsilon: 1e-1 +/// Close elements: 10/10 (100.00%) +/// [PASS] All elements are within tolerance +/// +/// Epsilon: 1e-2 +/// Close elements: 10/10 (100.00%) +/// [PASS] All elements are within tolerance +/// +/// Epsilon: 1e-3 +/// Close elements: 9/10 (90.00%) +/// [WARN] Most elements are within tolerance +/// +/// Epsilon: 1e-4 +/// Close elements: 6/10 (60.00%) +/// [FAIL] Significant differences detected +/// +/// Epsilon: 1e-5 +/// Close elements: 5/10 (50.00%) +/// [FAIL] Significant differences detected +/// +/// Epsilon: 1e-6 +/// Close elements: 5/10 (50.00%) +/// [FAIL] Significant differences detected +/// +/// Epsilon: 1e-7 +/// Close elements: 5/10 (50.00%) +/// [FAIL] Significant differences detected +/// +/// Epsilon: 1e-8 +/// Close elements: 5/10 (50.00%) +/// [FAIL] Significant differences detected +/// +/// Closeness check complete. +/// ``` +pub fn check_closeness(output: &Tensor, expected: &Tensor) { + println!("{}", "Tensor Closeness Check Results:".bold()); + println!("==============================="); + + for epsilon in [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8].iter() { + println!("{} {:e}", "Epsilon:".bold(), epsilon); + + let close = output + .clone() + .is_close(expected.clone(), Some(*epsilon), Some(*epsilon)); + let data = close.clone().into_data(); + let num_elements = data.num_elements(); + + // Count the number of elements that are close (true) + let count = data.iter::().filter(|x| *x).count(); + + let percentage = (count as f64 / num_elements as f64) * 100.0; + + println!(" Close elements: {count}/{num_elements} ({percentage:.2}%)"); + + if percentage == 100.0 { + println!(" {} All elements are within tolerance", "[PASS]".green()); + } else if percentage >= 90.0 { + println!(" {} Most elements are within tolerance", "[WARN]".yellow()); + } else { + println!(" {} Significant differences detected", "[FAIL]".red()); + } + + println!(); + } + + println!("{}", "Closeness check complete.".bold()); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/stats/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/stats/mod.rs new file mode 100644 index 0000000..17e895f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-tensor/src/tensor/stats/mod.rs @@ -0,0 +1,74 @@ +use crate::{Tensor, backend::Backend}; +use burn_backend::tensor::Int; + +pub fn var(tensor: Tensor, dim: usize) -> Tensor { + let mean = tensor.clone().mean_dim(dim); + var_with_mean(tensor, mean, dim) +} + +pub fn var_with_mean( + tensor: Tensor, + mean: Tensor, + dim: usize, +) -> Tensor { + let n = tensor.shape()[dim] - 1; + var_with_mean_n(tensor, mean, dim, n) +} + +pub fn var_bias(tensor: Tensor, dim: usize) -> Tensor { + let mean = tensor.clone().mean_dim(dim); + var_with_mean_bias(tensor, mean, dim) +} + +pub fn var_with_mean_bias( + tensor: Tensor, + mean: Tensor, + dim: usize, +) -> Tensor { + let n = tensor.shape()[dim]; + var_with_mean_n(tensor, mean, dim, n) +} + +pub fn var_with_mean_n( + tensor: Tensor, + mean: Tensor, + dim: usize, + n: usize, +) -> Tensor { + tensor.sub(mean).square().sum_dim(dim).div_scalar(n as f32) +} + +pub fn median(tensor: Tensor, dim: usize) -> Tensor { + let total_elem_numbers = tensor.dims()[dim]; + let sorted_tensor = tensor.sort(dim); + + // Following the PyTorch behavior: + // - Odd count: the median + // - Even count: the lower of the two median elements + // + // Example: + // - 5 elements: (5 - 1) / 2 = 4 / 2 = 2 + // - 4 elements: (4 - 1) / 2 = 3 / 2 = 1 + let median_index = (total_elem_numbers - 1) / 2; + sorted_tensor.narrow(dim, median_index, 1) +} + +pub fn median_with_indices( + tensor: Tensor, + dim: usize, +) -> (Tensor, Tensor) { + let total_elem_numbers = tensor.dims()[dim]; + let (sorted_tensor, indices) = tensor.sort_with_indices(dim); + + // Following the PyTorch behavior: + // - Odd count: the median + // - Even count: the lower of the two median elements + // + // Example: + // - 5 elements: (5 - 1) / 2 = 4 / 2 = 2 + // - 4 elements: (4 - 1) / 2 = 3 / 2 = 1 + let median_index = (total_elem_numbers - 1) / 2; + let median_values = sorted_tensor.narrow(dim, median_index, 1); + let median_indices = indices.narrow(dim, median_index, 1); + (median_values, median_indices) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-train/Cargo.toml new file mode 100644 index 0000000..0d08e44 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/Cargo.toml @@ -0,0 +1,80 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "Training crate for the Burn framework" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "tensor", "pytorch", "ndarray"] +license.workspace = true +name = "burn-train" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-train" +documentation = "https://docs.rs/burn-train" +version.workspace = true + +[lints] +workspace = true + +[features] +default = ["sys-metrics", "tui", "rl"] +doc = ["default"] +vision = ["burn-nn", "burn-store/pytorch", "burn-std/network", "dirs"] +tracing = [ + "burn-core/tracing", + "burn-optim/tracing", + "burn-collective?/tracing", +] + + +sys-metrics = ["nvml-wrapper", "sysinfo", "systemstat"] +tui = ["ratatui"] +rl = ["burn-rl"] +# Distributed Data Parallel +ddp = ["burn-collective", "burn-optim/collective"] + +[dependencies] +burn-core = { path = "../burn-core", version = "=0.21.0-pre.2", features = [ + "dataset", + "std", +], default-features = false } +burn-optim = { path = "../burn-optim", version = "=0.21.0-pre.2", features = [ + "std", +], default-features = false } +burn-rl = { path = "../burn-rl", version = "=0.21.0-pre.2", optional = true, default-features = false } +burn-collective = { path = "../burn-collective", version = "=0.21.0-pre.2", optional = true } +burn-nn = { path = "../burn-nn", version = "=0.21.0-pre.2", optional = true, default-features = false, features = ["std"] } +burn-store = { path = "../burn-store", version = "=0.21.0-pre.2", optional = true, default-features = false, features = ["std"] } +burn-std = { path = "../burn-std", version = "=0.21.0-pre.2", optional = true, default-features = false, features = ["std"] } +dirs = { workspace = true, optional = true } + +log = { workspace = true } +tracing-subscriber = { workspace = true } +tracing-appender = { workspace = true } +tracing-core = { workspace = true } + +# System Metrics +nvml-wrapper = { workspace = true, optional = true } +sysinfo = { workspace = true, optional = true } +systemstat = { workspace = true, optional = true } + +# Text UI +ratatui = { workspace = true, optional = true, features = [ + "all-widgets", + "crossterm", +] } + +# Utilities +derive-new = { workspace = true } +serde = { workspace = true, features = ["std", "derive"] } +async-channel = { workspace = true } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +rstest.workspace = true +thiserror.workspace = true +rand.workspace = true + +[dev-dependencies] +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-autodiff = { path = "../burn-autodiff", version = "=0.21.0-pre.2" } + +[package.metadata.docs.rs] +features = ["doc"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-APACHE b/crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-MIT b/crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/README.md b/crates/stable-diffusion-burn/burn-crates/burn-train/README.md new file mode 100644 index 0000000..e693c68 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/README.md @@ -0,0 +1,6 @@ +# Burn Train + +This crate should be used with [burn](https://github.com/tracel-ai/burn). + +[![Current Crates.io Version](https://img.shields.io/crates/v/burn-train.svg)](https://crates.io/crates/burn-train) +[![license](https://shields.io/badge/license-MIT%2FApache--2.0-blue)](https://github.com/tracel-ai/burn-train/blob/master/README.md) diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/async_checkpoint.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/async_checkpoint.rs new file mode 100644 index 0000000..c08f487 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/async_checkpoint.rs @@ -0,0 +1,171 @@ +use super::{Checkpointer, CheckpointerError}; +use crate::Interrupter; +use burn_core::{record::Record, tensor::backend::Backend}; +use std::sync::mpsc; + +enum Message { + Restore( + usize, + B::Device, + mpsc::SyncSender>, + Option, + ), + Save(usize, R, Option), + Delete(usize, Option), + End, +} + +#[derive(new)] +struct CheckpointerThread { + checkpointer: C, + receiver: mpsc::Receiver>, +} + +impl CheckpointerThread +where + C: Checkpointer, + R: Record, + B: Backend, +{ + fn run(self) { + for item in self.receiver.iter() { + match item { + Message::Restore(epoch, device, callback, interrupter) => { + let record = self.checkpointer.restore(epoch, &device); + callback.send(record).unwrap_or_else(|err| { + interrupter.map_or_else( + || { + panic!( + "Error when sending response through callback channel: {err}" + ) + }, + |int| int.stop(Some(&err.to_string())), + ) + }); + } + Message::Save(epoch, state, interrupter) => { + self.checkpointer.save(epoch, state).unwrap_or_else(|err| { + interrupter.map_or_else( + || panic!("Error when saving the state: {err}"), + |int| int.stop(Some(&err.to_string())), + ) + }); + } + Message::Delete(epoch, interrupter) => { + self.checkpointer.delete(epoch).unwrap_or_else(|err| { + interrupter.map_or_else( + || panic!("Error when deleting the state: {err}"), + |int| int.stop(Some(&err.to_string())), + ) + }); + } + + Message::End => { + return; + } + }; + } + } +} + +/// Async checkpointer. +pub struct AsyncCheckpointer { + sender: mpsc::SyncSender>, + handler: Option>, + interrupter: Option, +} + +impl AsyncCheckpointer +where + R: Record + 'static, + B: Backend, +{ + /// Create a new async checkpointer. + /// + /// # Arguments + /// + /// * `checkpointer` - The checkpointer. + /// + /// # Returns + /// + /// The async checkpointer. + pub fn new(checkpointer: C) -> Self + where + C: Checkpointer + Send + 'static, + { + // Only on checkpoint can be done in advance. + let (sender, receiver) = mpsc::sync_channel(0); + let thread = CheckpointerThread::new(checkpointer, receiver); + let handler = Some(std::thread::spawn(move || thread.run())); + + Self { + sender, + handler, + interrupter: None, + } + } + + /// Assign a handle used to interrupt training in case of checkpointing error. + pub fn with_interrupter(mut self, interrupter: Interrupter) -> Self { + self.interrupter = Some(interrupter); + self + } +} + +impl Checkpointer for AsyncCheckpointer +where + R: Record + 'static, + B: Backend, +{ + fn save(&self, epoch: usize, record: R) -> Result<(), CheckpointerError> { + self.sender + .send(Message::Save(epoch, record, self.interrupter.clone())) + .expect("Can send message to checkpointer thread."); + + Ok(()) + } + + fn restore(&self, epoch: usize, device: &B::Device) -> Result { + let (sender, receiver) = mpsc::sync_channel(1); + self.sender + .send(Message::Restore( + epoch, + device.clone(), + sender, + self.interrupter.clone(), + )) + .map_err(|e| CheckpointerError::Unknown(e.to_string()))?; + + if let Ok(record) = receiver.recv() { + return record; + }; + + Err(CheckpointerError::Unknown("Channel error.".to_string())) + } + + fn delete(&self, epoch: usize) -> Result<(), CheckpointerError> { + self.sender + .send(Message::Delete(epoch, self.interrupter.clone())) + .map_err(|e| CheckpointerError::Unknown(e.to_string()))?; + + Ok(()) + } +} + +impl Drop for AsyncCheckpointer +where + B: Backend, +{ + fn drop(&mut self) { + self.sender + .send(Message::End) + .expect("Can send the end message to the checkpointer thread."); + let handler = self.handler.take(); + + if let Some(handler) = handler { + handler + .join() + .expect("The checkpointer thread should stop."); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/base.rs new file mode 100644 index 0000000..db2196b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/base.rs @@ -0,0 +1,51 @@ +use burn_core::{ + record::{Record, RecorderError}, + tensor::backend::Backend, +}; +use thiserror::Error; + +/// The error type for checkpointer. +#[derive(Error, Debug)] +pub enum CheckpointerError { + /// IO error. + #[error("I/O Error: `{0}`")] + IOError(std::io::Error), + + /// Recorder error. + #[error("Recorder error: `{0}`")] + RecorderError(RecorderError), + + /// Other errors. + #[error("Unknown error: `{0}`")] + Unknown(String), +} + +/// The trait for checkpointer. +pub trait Checkpointer: Send + Sync +where + R: Record, + B: Backend, +{ + /// Save the record. + /// + /// # Arguments + /// + /// * `epoch` - The epoch. + /// * `record` - The record. + fn save(&self, epoch: usize, record: R) -> Result<(), CheckpointerError>; + + /// Delete the record at the given epoch if present. + fn delete(&self, epoch: usize) -> Result<(), CheckpointerError>; + + /// Restore the record. + /// + /// # Arguments + /// + /// * `epoch` - The epoch. + /// * `device` - The device used to restore the record. + /// + /// # Returns + /// + /// The record. + fn restore(&self, epoch: usize, device: &B::Device) -> Result; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/file.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/file.rs new file mode 100644 index 0000000..77dfd67 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/file.rs @@ -0,0 +1,86 @@ +use std::path::{Path, PathBuf}; + +use super::{Checkpointer, CheckpointerError}; +use burn_core::{ + record::{FileRecorder, Record}, + tensor::backend::Backend, +}; + +/// The file checkpointer. +pub struct FileCheckpointer { + directory: PathBuf, + name: String, + recorder: FR, +} + +impl FileCheckpointer { + /// Creates a new file checkpointer. + /// + /// # Arguments + /// + /// * `recorder` - The file recorder. + /// * `directory` - The directory to save the checkpoints. + /// * `name` - The name of the checkpoint. + pub fn new(recorder: FR, directory: impl AsRef, name: &str) -> Self { + let directory = directory.as_ref(); + std::fs::create_dir_all(directory).ok(); + + Self { + directory: directory.to_path_buf(), + name: name.to_string(), + recorder, + } + } + + fn path_for_epoch(&self, epoch: usize) -> PathBuf { + self.directory.join(format!("{}-{}", self.name, epoch)) + } +} + +impl Checkpointer for FileCheckpointer +where + R: Record, + FR: FileRecorder, + B: Backend, +{ + fn save(&self, epoch: usize, record: R) -> Result<(), CheckpointerError> { + let file_path = self.path_for_epoch(epoch); + log::trace!("Saving checkpoint {} to {}", epoch, file_path.display()); + + self.recorder + .record(record, file_path) + .map_err(CheckpointerError::RecorderError)?; + + Ok(()) + } + + fn restore(&self, epoch: usize, device: &B::Device) -> Result { + let file_path = self.path_for_epoch(epoch); + log::info!( + "Restoring checkpoint {} from {}", + epoch, + file_path.display() + ); + let record = self + .recorder + .load(file_path, device) + .map_err(CheckpointerError::RecorderError)?; + + Ok(record) + } + + fn delete(&self, epoch: usize) -> Result<(), CheckpointerError> { + let file_to_remove = format!( + "{}.{}", + self.path_for_epoch(epoch).display(), + FR::file_extension(), + ); + + if std::path::Path::new(&file_to_remove).exists() { + log::trace!("Removing checkpoint {file_to_remove}"); + std::fs::remove_file(file_to_remove).map_err(CheckpointerError::IOError)?; + } + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/mod.rs new file mode 100644 index 0000000..14b8f15 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/mod.rs @@ -0,0 +1,9 @@ +mod async_checkpoint; +mod base; +mod file; +mod strategy; + +pub use async_checkpoint::*; +pub use base::*; +pub use file::*; +pub use strategy::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/base.rs new file mode 100644 index 0000000..07d6213 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/base.rs @@ -0,0 +1,34 @@ +use std::ops::DerefMut; + +use crate::metric::store::EventStoreClient; + +/// Action to be taken by a [checkpointer](crate::checkpoint::Checkpointer). +#[derive(Clone, PartialEq, Debug)] +pub enum CheckpointingAction { + /// Delete the given epoch. + Delete(usize), + /// Save the current record. + Save, +} + +/// Define when checkpoint should be saved and deleted. +pub trait CheckpointingStrategy: Send { + /// Based on the epoch, determine if the checkpoint should be saved. + fn checkpointing( + &mut self, + epoch: usize, + collector: &EventStoreClient, + ) -> Vec; +} + +// We make dyn box implement the checkpointing strategy so that it can be used with generic, but +// still be dynamic. +impl CheckpointingStrategy for Box { + fn checkpointing( + &mut self, + epoch: usize, + collector: &EventStoreClient, + ) -> Vec { + self.deref_mut().checkpointing(epoch, collector) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/composed.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/composed.rs new file mode 100644 index 0000000..8029c9e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/composed.rs @@ -0,0 +1,146 @@ +use crate::metric::store::EventStoreClient; + +use super::{CheckpointingAction, CheckpointingStrategy}; +use std::collections::HashSet; + +/// Compose multiple checkpointing strategy and only delete checkpoints when both strategy flag an +/// epoch to be deleted. +pub struct ComposedCheckpointingStrategy { + strategies: Vec>, + deleted: Vec>, +} + +/// Help building a [checkpointing strategy](CheckpointingStrategy) by combining multiple ones. +#[derive(Default)] +pub struct ComposedCheckpointingStrategyBuilder { + strategies: Vec>, +} + +impl ComposedCheckpointingStrategyBuilder { + /// Add a new [checkpointing strategy](CheckpointingStrategy). + #[allow(clippy::should_implement_trait)] + pub fn add(mut self, strategy: S) -> Self + where + S: CheckpointingStrategy + 'static, + { + self.strategies.push(Box::new(strategy)); + self + } + + /// Create a new [composed checkpointing strategy](ComposedCheckpointingStrategy). + pub fn build(self) -> ComposedCheckpointingStrategy { + ComposedCheckpointingStrategy::new(self.strategies) + } +} + +impl ComposedCheckpointingStrategy { + fn new(strategies: Vec>) -> Self { + Self { + deleted: strategies.iter().map(|_| HashSet::new()).collect(), + strategies, + } + } + /// Create a new builder which help compose multiple + /// [checkpointing strategies](CheckpointingStrategy). + pub fn builder() -> ComposedCheckpointingStrategyBuilder { + ComposedCheckpointingStrategyBuilder::default() + } +} + +impl CheckpointingStrategy for ComposedCheckpointingStrategy { + fn checkpointing( + &mut self, + epoch: usize, + collector: &EventStoreClient, + ) -> Vec { + let mut saved = false; + let mut actions = Vec::new(); + let mut epochs_to_check = Vec::new(); + + for (i, strategy) in self.strategies.iter_mut().enumerate() { + let actions = strategy.checkpointing(epoch, collector); + // We assume that the strategy would not want the current epoch to be saved. + // So we flag it as deleted. + if actions.is_empty() { + self.deleted + .get_mut(i) + .expect("As many 'deleted' as 'strategies'.") + .insert(epoch); + } + + for action in actions { + match action { + CheckpointingAction::Delete(epoch) => { + self.deleted + .get_mut(i) + .expect("As many 'deleted' as 'strategies'.") + .insert(epoch); + epochs_to_check.push(epoch); + } + CheckpointingAction::Save => saved = true, + } + } + } + + if saved { + actions.push(CheckpointingAction::Save); + } + + for epoch in epochs_to_check.into_iter() { + let mut num_true = 0; + for i in 0..self.strategies.len() { + if self + .deleted + .get(i) + .expect("Ad many 'deleted' as 'strategies'.") + .contains(&epoch) + { + num_true += 1; + } + } + + if num_true == self.strategies.len() { + actions.push(CheckpointingAction::Delete(epoch)); + + for i in 0..self.strategies.len() { + self.deleted + .get_mut(i) + .expect("As many 'deleted' as 'strategies'.") + .remove(&epoch); + } + } + } + + actions + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{checkpoint::KeepLastNCheckpoints, metric::store::LogEventStore}; + + #[test] + fn should_delete_when_both_deletes() { + let store = EventStoreClient::new(LogEventStore::default()); + let mut strategy = ComposedCheckpointingStrategy::builder() + .add(KeepLastNCheckpoints::new(1)) + .add(KeepLastNCheckpoints::new(2)) + .build(); + + assert_eq!( + vec![CheckpointingAction::Save], + strategy.checkpointing(1, &store) + ); + + assert_eq!( + vec![CheckpointingAction::Save], + strategy.checkpointing(2, &store) + ); + + assert_eq!( + vec![CheckpointingAction::Save, CheckpointingAction::Delete(1)], + strategy.checkpointing(3, &store) + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/lastn.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/lastn.rs new file mode 100644 index 0000000..6e8b46a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/lastn.rs @@ -0,0 +1,56 @@ +use super::CheckpointingStrategy; +use crate::{checkpoint::CheckpointingAction, metric::store::EventStoreClient}; + +/// Keep the last N checkpoints. +/// +/// Very useful when training, minimizing disk space while ensuring that the training can be +/// resumed even if something goes wrong. +#[derive(new)] +pub struct KeepLastNCheckpoints { + num_keep: usize, +} + +impl CheckpointingStrategy for KeepLastNCheckpoints { + fn checkpointing( + &mut self, + epoch: usize, + _store: &EventStoreClient, + ) -> Vec { + let mut actions = vec![CheckpointingAction::Save]; + + if let Some(epoch) = usize::checked_sub(epoch, self.num_keep) + && epoch > 0 + { + actions.push(CheckpointingAction::Delete(epoch)); + } + + actions + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metric::store::LogEventStore; + + #[test] + fn should_always_delete_lastn_epoch_if_higher_than_one() { + let mut strategy = KeepLastNCheckpoints::new(2); + let store = EventStoreClient::new(LogEventStore::default()); + + assert_eq!( + vec![CheckpointingAction::Save], + strategy.checkpointing(1, &store) + ); + + assert_eq!( + vec![CheckpointingAction::Save], + strategy.checkpointing(2, &store) + ); + + assert_eq!( + vec![CheckpointingAction::Save, CheckpointingAction::Delete(1)], + strategy.checkpointing(3, &store) + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/metric.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/metric.rs new file mode 100644 index 0000000..a1d0c9a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/metric.rs @@ -0,0 +1,136 @@ +use super::CheckpointingStrategy; +use crate::{ + checkpoint::CheckpointingAction, + metric::{ + Metric, MetricName, + store::{Aggregate, Direction, EventStoreClient, Split}, + }, +}; + +/// Keep the best checkpoint based on a metric. +pub struct MetricCheckpointingStrategy { + current: Option, + aggregate: Aggregate, + direction: Direction, + split: Split, + name: MetricName, +} + +impl MetricCheckpointingStrategy { + /// Create a new metric checkpointing strategy. + pub fn new(metric: &M, aggregate: Aggregate, direction: Direction, split: Split) -> Self + where + M: Metric, + { + Self { + current: None, + name: metric.name(), + aggregate, + direction, + split, + } + } +} + +impl CheckpointingStrategy for MetricCheckpointingStrategy { + fn checkpointing( + &mut self, + epoch: usize, + store: &EventStoreClient, + ) -> Vec { + let best_epoch = + match store.find_epoch(&self.name, self.aggregate, self.direction, &self.split) { + Some(epoch_best) => epoch_best, + None => epoch, + }; + + let mut actions = Vec::new(); + + if let Some(current) = self.current + && current != best_epoch + { + actions.push(CheckpointingAction::Delete(current)); + } + + if best_epoch == epoch { + actions.push(CheckpointingAction::Save); + } + + self.current = Some(best_epoch); + + actions + } +} + +#[cfg(test)] +mod tests { + use crate::{ + EventProcessorTraining, TestBackend, + logger::InMemoryMetricLogger, + metric::{ + LossMetric, + processor::{ + MetricsTraining, MinimalEventProcessor, + test_utils::{end_epoch, process_train}, + }, + store::LogEventStore, + }, + }; + + use super::*; + use std::sync::Arc; + + #[test] + fn always_keep_the_best_epoch() { + let loss = LossMetric::::new(); + let mut store = LogEventStore::default(); + let mut strategy = MetricCheckpointingStrategy::new( + &loss, + Aggregate::Mean, + Direction::Lowest, + Split::Train, + ); + let mut metrics = MetricsTraining::::default(); + // Register an in memory logger. + store.register_logger(InMemoryMetricLogger::default()); + // Register the loss metric. + metrics.register_train_metric_numeric(loss); + let store = Arc::new(EventStoreClient::new(store)); + let mut processor = MinimalEventProcessor::new(metrics, store.clone()); + processor.process_train(crate::LearnerEvent::Start); + + // Two points for the first epoch. Mean 0.75 + let mut epoch = 1; + process_train(&mut processor, 1.0, epoch); + process_train(&mut processor, 0.5, epoch); + end_epoch(&mut processor, epoch); + + // Should save the current record. + assert_eq!( + vec![CheckpointingAction::Save], + strategy.checkpointing(epoch, &store) + ); + + // Two points for the second epoch. Mean 0.4 + epoch += 1; + process_train(&mut processor, 0.5, epoch); + process_train(&mut processor, 0.3, epoch); + end_epoch(&mut processor, epoch); + + // Should save the current record and delete the previous one. + assert_eq!( + vec![CheckpointingAction::Delete(1), CheckpointingAction::Save], + strategy.checkpointing(epoch, &store) + ); + + // Two points for the last epoch. Mean 2.0 + epoch += 1; + process_train(&mut processor, 1.0, epoch); + process_train(&mut processor, 3.0, epoch); + end_epoch(&mut processor, epoch); + + // Should not delete the previous record, since it's the best one, and should not save a + // new one. + assert!(strategy.checkpointing(epoch, &store).is_empty()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/mod.rs new file mode 100644 index 0000000..2915a71 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/checkpoint/strategy/mod.rs @@ -0,0 +1,9 @@ +mod base; +mod composed; +mod lastn; +mod metric; + +pub use base::*; +pub use composed::*; +pub use lastn::*; +pub use metric::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/components.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/components.rs new file mode 100644 index 0000000..05fad97 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/components.rs @@ -0,0 +1,66 @@ +use crate::{InferenceStep, TrainStep}; +use burn_core::{module::AutodiffModule, tensor::backend::AutodiffBackend}; +use burn_optim::{Optimizer, lr_scheduler::LrScheduler}; +use std::marker::PhantomData; + +/// Components used for a model to learn, grouped in one trait. +pub trait LearningComponentsTypes { + /// The backend used for training. + type Backend: AutodiffBackend; + /// The learning rate scheduler used for training. + type LrScheduler: LrScheduler + 'static; + /// The model to train. + type TrainingModel: TrainStep + + AutodiffModule + + core::fmt::Display + + 'static; + /// The non-autodiff type of the model. + type InferenceModel: InferenceStep; + /// The optimizer used for training. + type Optimizer: Optimizer + 'static; +} + +/// Concrete type that implements the [LearningComponentsTypes](LearningComponentsTypes) trait. +pub struct LearningComponentsMarker { + _backend: PhantomData, + _lr_scheduler: PhantomData, + _model: PhantomData, + _optimizer: PhantomData, +} + +impl LearningComponentsTypes for LearningComponentsMarker +where + B: AutodiffBackend, + LR: LrScheduler + 'static, + M: TrainStep + AutodiffModule + core::fmt::Display + 'static, + M::InnerModule: InferenceStep, + O: Optimizer + 'static, +{ + type Backend = B; + type LrScheduler = LR; + type TrainingModel = M; + type InferenceModel = M::InnerModule; + type Optimizer = O; +} + +/// The training backend. +pub type TrainingBackend = ::Backend; +/// The inference backend. +pub(crate) type InferenceBackend = + <::Backend as AutodiffBackend>::InnerBackend; +/// The model used for training. +pub type TrainingModel = ::TrainingModel; +/// The non-autodiff model. +pub(crate) type InferenceModel = ::InferenceModel; +/// Type for training input. +pub(crate) type TrainingModelInput = + <::TrainingModel as TrainStep>::Input; +/// Type for inference input. +pub(crate) type InferenceModelInput = + <::InferenceModel as InferenceStep>::Input; +/// Type for training output. +pub(crate) type TrainingModelOutput = + <::TrainingModel as TrainStep>::Output; +/// Type for inference output. +pub(crate) type InferenceModelOutput = + <::InferenceModel as InferenceStep>::Output; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/base.rs new file mode 100644 index 0000000..0ad78a2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/base.rs @@ -0,0 +1,72 @@ +use crate::{ + AsyncProcessorEvaluation, EvaluationItem, FullEventProcessorEvaluation, InferenceStep, + Interrupter, LearnerSummaryConfig, + evaluator::components::EvaluatorComponentTypes, + metric::processor::{EvaluatorEvent, EventProcessorEvaluation}, + renderer::{EvaluationName, MetricsRenderer}, +}; +use burn_core::{data::dataloader::DataLoader, module::Module}; +use std::sync::Arc; + +pub(crate) type TestBackend = ::Backend; +pub(crate) type TestInput = <::Model as InferenceStep>::Input; +pub(crate) type TestOutput = <::Model as InferenceStep>::Output; + +pub(crate) type TestLoader = Arc, TestInput>>; + +/// Evaluates a model on a specific dataset. +pub struct Evaluator { + pub(crate) model: EC::Model, + pub(crate) interrupter: Interrupter, + pub(crate) event_processor: + AsyncProcessorEvaluation>>, + /// Config for creating a summary of the evaluation + pub summary: Option, +} + +impl Evaluator { + /// Run the evaluation on the given dataset. + /// + /// The data will be stored and displayed under the provided name. + pub fn eval( + mut self, + name: S, + dataloader: TestLoader, + ) -> Box { + // Move dataloader to the model device + let dataloader = dataloader.to_device(self.model.devices().first().unwrap()); + + let name = EvaluationName::new(name); + let mut iterator = dataloader.iter(); + let mut iteration = 0; + + self.event_processor.process_test(EvaluatorEvent::Start); + while let Some(item) = iterator.next() { + let progress = iterator.progress(); + iteration += 1; + + let item = self.model.step(item); + let item = EvaluationItem::new(item, progress, Some(iteration)); + + self.event_processor + .process_test(EvaluatorEvent::ProcessedItem(name.clone(), item)); + + if self.interrupter.should_stop() { + log::info!("Testing interrupted."); + break; + } + } + + let summary = self.summary.and_then(|summary| { + summary + .init() + .map(|summary| summary.with_model(self.model.to_string())) + .ok() + }); + + self.event_processor + .process_test(EvaluatorEvent::End(summary)); + + self.event_processor.renderer() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/builder.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/builder.rs new file mode 100644 index 0000000..ebd2ddf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/builder.rs @@ -0,0 +1,212 @@ +use crate::{ + ApplicationLoggerInstaller, Evaluator, FileApplicationLoggerInstaller, InferenceStep, + Interrupter, LearnerSummaryConfig, TestOutput, + evaluator::components::{EvaluatorComponentTypes, EvaluatorComponentTypesMarker}, + logger::FileMetricLogger, + metric::{ + Adaptor, ItemLazy, Metric, Numeric, + processor::{AsyncProcessorEvaluation, FullEventProcessorEvaluation, MetricsEvaluation}, + store::{EventStoreClient, LogEventStore}, + }, + renderer::{MetricsRenderer, default_renderer}, +}; +use burn_core::{module::Module, prelude::Backend}; +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + sync::Arc, +}; + +/// Struct to configure and create an [evaluator](Evaluator). +/// +/// The generics components of the builder should probably not be set manually, as they are +/// optimized for Rust type inference. +pub struct EvaluatorBuilder { + tracing_logger: Option>, + event_store: LogEventStore, + summary_metrics: BTreeSet, + renderer: Option>, + interrupter: Interrupter, + metrics: MetricsEvaluation>, + directory: PathBuf, + summary: bool, +} + +impl EvaluatorBuilder> +where + B: Backend, + M: Module + InferenceStep + core::fmt::Display + 'static, +{ + /// Creates a new evaluator builder. + /// + /// # Arguments + /// + /// * `directory` - The directory to save the checkpoints. + pub fn new(directory: impl AsRef) -> Self { + let directory = directory.as_ref().to_path_buf(); + let log_file = directory.join("evaluation.log"); + + Self { + tracing_logger: Some(Box::new(FileApplicationLoggerInstaller::new(log_file))), + event_store: LogEventStore::default(), + summary_metrics: Default::default(), + renderer: None, + interrupter: Interrupter::new(), + summary: false, + metrics: MetricsEvaluation::default(), + directory, + } + } +} + +impl EvaluatorBuilder { + /// Registers [numeric](crate::metric::Numeric) test [metrics](Metric). + pub fn metrics>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Registers text [metrics](Metric). + pub fn metrics_text>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// By default, Rust logs are captured and written into + /// `evaluation.log`. If disabled, standard Rust log handling + /// will apply. + pub fn with_application_logger( + mut self, + logger: Option>, + ) -> Self { + self.tracing_logger = logger; + self + } + + /// Register a [numeric](crate::metric::Numeric) test [metric](Metric). + pub fn metric_numeric(mut self, metric: Me) -> Self + where + Me: Metric + Numeric + 'static, + as ItemLazy>::ItemSync: Adaptor, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_test_metric_numeric(metric); + self + } + + /// Register a text test [metric](Metric). + pub fn metric(mut self, metric: Me) -> Self + where + Me: Metric + 'static, + as ItemLazy>::ItemSync: Adaptor, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_test_metric(metric); + self + } + + /// Replace the default CLI renderer with a custom one. + /// + /// # Arguments + /// + /// * `renderer` - The custom renderer. + pub fn renderer(mut self, renderer: Box) -> Self { + self.renderer = Some(renderer); + self + } + + /// Enable the evaluation summary report. + /// + /// The summary will be displayed at the end of `.eval()`. + pub fn summary(mut self) -> Self { + self.summary = true; + self + } + + /// Builds the evaluator. + #[allow(clippy::type_complexity)] + pub fn build(mut self, model: EC::Model) -> Evaluator { + let renderer = self + .renderer + .unwrap_or_else(|| default_renderer(self.interrupter.clone(), None)); + + self.event_store + .register_logger(FileMetricLogger::new_eval(self.directory.clone())); + let event_store = Arc::new(EventStoreClient::new(self.event_store)); + + let event_processor = AsyncProcessorEvaluation::new(FullEventProcessorEvaluation::new( + self.metrics, + renderer, + event_store, + )); + + let summary = if self.summary { + Some(LearnerSummaryConfig { + directory: self.directory, + metrics: self.summary_metrics.into_iter().collect::>(), + }) + } else { + None + }; + + Evaluator { + model, + interrupter: self.interrupter, + event_processor, + summary, + } + } +} + +/// Trait to fake variadic generics. +pub trait EvalMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: EvaluatorBuilder) -> EvaluatorBuilder; +} + +/// Trait to fake variadic generics. +pub trait EvalTextMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: EvaluatorBuilder) -> EvaluatorBuilder; +} + +macro_rules! gen_tuple { + ($($M:ident),*) => { + impl<$($M,)* EC: EvaluatorComponentTypes> EvalTextMetricRegistration for ($($M,)*) + where + $( as ItemLazy>::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: EvaluatorBuilder, + ) -> EvaluatorBuilder { + let ($($M,)*) = self; + $(let builder = builder.metric($M);)* + builder + } + } + + impl<$($M,)* EC: EvaluatorComponentTypes> EvalMetricRegistration for ($($M,)*) + where + $( as ItemLazy>::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + $crate::metric::Numeric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: EvaluatorBuilder, + ) -> EvaluatorBuilder { + let ($($M,)*) = self; + $(let builder = builder.metric_numeric($M);)* + builder + } + } + }; +} + +gen_tuple!(M1); +gen_tuple!(M1, M2); +gen_tuple!(M1, M2, M3); +gen_tuple!(M1, M2, M3, M4); +gen_tuple!(M1, M2, M3, M4, M5); +gen_tuple!(M1, M2, M3, M4, M5, M6); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/components.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/components.rs new file mode 100644 index 0000000..c3a051e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/components.rs @@ -0,0 +1,25 @@ +use crate::InferenceStep; +use burn_core::{module::Module, prelude::Backend}; +use std::marker::PhantomData; + +/// All components necessary to evaluate a model grouped in one trait. +pub trait EvaluatorComponentTypes { + /// The backend in used for the evaluation. + type Backend: Backend; + /// The model to evaluate. + type Model: Module + InferenceStep + core::fmt::Display + 'static; +} + +/// A marker type used to provide [evaluation components](EvaluatorComponentTypes). +pub struct EvaluatorComponentTypesMarker { + _p: PhantomData<(B, M)>, +} + +impl EvaluatorComponentTypes for EvaluatorComponentTypesMarker +where + B: Backend, + M: Module + InferenceStep + core::fmt::Display + 'static, +{ + type Backend = B; + type Model = M; +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/mod.rs new file mode 100644 index 0000000..aa20339 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/evaluator/mod.rs @@ -0,0 +1,7 @@ +mod base; +mod builder; + +pub(crate) mod components; + +pub use base::*; +pub use builder::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/application_logger.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/application_logger.rs new file mode 100644 index 0000000..42f725c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/application_logger.rs @@ -0,0 +1,69 @@ +use std::path::{Path, PathBuf}; +use tracing_core::{Level, LevelFilter}; +use tracing_subscriber::filter::filter_fn; +use tracing_subscriber::prelude::*; +use tracing_subscriber::{Layer, registry}; + +/// This trait is used to install an application logger. +pub trait ApplicationLoggerInstaller { + /// Install the application logger. + fn install(&self) -> Result<(), String>; +} + +/// This struct is used to install a local file application logger to output logs to a given file path. +pub struct FileApplicationLoggerInstaller { + path: PathBuf, +} + +impl FileApplicationLoggerInstaller { + /// Create a new file application logger. + pub fn new(path: impl AsRef) -> Self { + Self { + path: path.as_ref().to_path_buf(), + } + } +} + +impl ApplicationLoggerInstaller for FileApplicationLoggerInstaller { + fn install(&self) -> Result<(), String> { + let path = Path::new(&self.path); + let writer = tracing_appender::rolling::never( + path.parent().unwrap_or_else(|| Path::new(".")), + path.file_name().unwrap_or_else(|| { + panic!("The path '{}' to point to a file.", self.path.display()) + }), + ); + let layer = tracing_subscriber::fmt::layer() + .with_ansi(false) + .with_writer(writer) + .with_filter(LevelFilter::INFO) + .with_filter(filter_fn(|m| { + if let Some(path) = m.module_path() { + // The wgpu crate is logging too much, so we skip `info` level. + if path.starts_with("wgpu") && *m.level() >= Level::INFO { + return false; + } + } + true + })); + + if registry().with(layer).try_init().is_err() { + return Err("Failed to install the file logger.".to_string()); + } + + let hook = std::panic::take_hook(); + let file_path = self.path.to_owned(); + + std::panic::set_hook(Box::new(move |info| { + log::error!("PANIC => {info}"); + eprintln!( + "=== PANIC ===\nA fatal error happened, you can check the experiment logs here => \ + '{}'\n=============", + file_path.display() + ); + hook(info); + })); + + Ok(()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/base.rs new file mode 100644 index 0000000..2228b94 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/base.rs @@ -0,0 +1,255 @@ +use crate::LearningComponentsMarker; +use crate::checkpoint::{ + AsyncCheckpointer, Checkpointer, CheckpointingAction, CheckpointingStrategy, +}; +use crate::components::{LearningComponentsTypes, TrainingBackend}; +use crate::metric::store::EventStoreClient; +use crate::{ + CloneEarlyStoppingStrategy, InferenceStep, TrainOutput, TrainStep, TrainingModelInput, + TrainingModelOutput, +}; +use burn_core::module::{AutodiffModule, Module}; +use burn_core::prelude::Backend; +use burn_core::tensor::Device; +use burn_core::tensor::backend::AutodiffBackend; +use burn_optim::lr_scheduler::LrScheduler; +use burn_optim::{GradientsParams, MultiGradientsParams, Optimizer}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +/// The record of the learner's model. +pub type LearnerModelRecord = + <::TrainingModel as Module>>::Record; +/// The record of the optimizer. +pub type LearnerOptimizerRecord = <::Optimizer as Optimizer< + ::TrainingModel, + TrainingBackend, +>>::Record; +/// The record of the LR scheduler. +pub type LearnerSchedulerRecord = + <::LrScheduler as LrScheduler>::Record>; + +/// Learner struct encapsulating all components necessary to train a Neural Network model. +pub struct Learner { + pub(crate) model: LC::TrainingModel, + optim: LC::Optimizer, + lr_scheduler: LC::LrScheduler, + lr: f64, +} + +impl Clone for Learner { + fn clone(&self) -> Self { + Self { + model: self.model.clone(), + optim: self.optim.clone(), + lr_scheduler: self.lr_scheduler.clone(), + lr: self.lr, + } + } +} + +impl Learner> +where + B: AutodiffBackend, + LR: LrScheduler + 'static, + M: TrainStep + AutodiffModule + core::fmt::Display + 'static, + M::InnerModule: InferenceStep, + O: Optimizer + 'static, +{ + /// Create a learner. + pub fn new(model: M, optim: O, lr_scheduler: LR) -> Self { + Self { + model, + optim, + lr_scheduler, + lr: 0.0, + } + } +} + +impl Learner { + /// Fork the learner's model to the given device. + pub fn fork(&mut self, device: & as Backend>::Device) { + self.model = self.model().fork(device); + } + + /// Returns the current model. + pub fn model(&self) -> LC::TrainingModel { + self.model.clone() + } + + /// Returns the current learning rate. + pub fn lr_current(&self) -> f64 { + self.lr + } + + /// Executes a step of the learning rate scheduler. + pub fn lr_step(&mut self) { + self.lr = self.lr_scheduler.step(); + } + + /// Runs a step of the model for training, which executes the forward and backward passes. + /// + /// # Arguments + /// + /// * `item` - The input for the model. + /// + /// # Returns + /// + /// The output containing the model output and the gradients. + pub fn train_step(&self, item: TrainingModelInput) -> TrainOutput> { + self.model.step(item) + } + + /// Optimize the current module with the provided gradients and learning rate. + /// + /// # Arguments + /// + /// * `optim`: Optimizer used for learning. + /// * `lr`: The learning rate used for this step. + /// * `grads`: The gradients of each parameter in the current model. + pub fn optimizer_step(&mut self, grads: GradientsParams) { + self.model = self.model().optimize(&mut self.optim, self.lr, grads); + } + + /// Optimize the current module with the provided gradients and learning rate. + /// + /// # Arguments + /// + /// * `optim`: Optimizer used for learning. + /// * `lr`: The learning rate used for this step. + /// * `grads`: Multiple gradients associated to each parameter in the current model. + pub fn optimizer_step_multi(&mut self, grads: MultiGradientsParams) { + self.model = self.model().optimize_multi(&mut self.optim, self.lr, grads); + } + + /// Load the module state from a [record](LearnerModelRecord). + pub fn load_model(&mut self, record: LearnerModelRecord) { + self.model = self.model.clone().load_record(record); + } + + /// Load the state of the learner's optimizer as a [record](LearnerOptimizerRecord). + pub fn load_optim(&mut self, record: LearnerOptimizerRecord) { + self.optim = self.optim.clone().load_record(record); + } + + /// Load the state of the learner's scheduler as a [record](LearnerSchedulerRecord). + pub fn load_scheduler(&mut self, record: LearnerSchedulerRecord) { + self.lr_scheduler = self.lr_scheduler.clone().load_record(record); + } +} + +#[derive(new)] +/// Used to create, delete, or load checkpoints of the training process. +pub struct LearningCheckpointer { + model: AsyncCheckpointer, LC::Backend>, + optim: AsyncCheckpointer, LC::Backend>, + lr_scheduler: AsyncCheckpointer, LC::Backend>, + strategy: Box, +} + +impl LearningCheckpointer { + /// Create checkpoint for the training process. + pub fn checkpoint(&mut self, learner: &Learner, epoch: usize, store: &EventStoreClient) { + let actions = self.strategy.checkpointing(epoch, store); + + for action in actions { + match action { + CheckpointingAction::Delete(epoch) => { + self.model + .delete(epoch) + .expect("Can delete model checkpoint."); + self.optim + .delete(epoch) + .expect("Can delete optimizer checkpoint."); + self.lr_scheduler + .delete(epoch) + .expect("Can delete learning rate scheduler checkpoint."); + } + CheckpointingAction::Save => { + self.model + .save(epoch, learner.model.clone().into_record()) + .expect("Can save model checkpoint."); + self.optim + .save(epoch, learner.optim.to_record()) + .expect("Can save optimizer checkpoint."); + self.lr_scheduler + .save(epoch, learner.lr_scheduler.to_record()) + .expect("Can save learning rate scheduler checkpoint."); + } + } + } + } + + /// Load a training checkpoint. + pub fn load_checkpoint( + &self, + mut learner: Learner, + device: &Device, + epoch: usize, + ) -> Learner { + let record = self + .model + .restore(epoch, device) + .expect("Can load model checkpoint."); + learner.load_model(record); + + let record = self + .optim + .restore(epoch, device) + .expect("Can load optimizer checkpoint."); + learner.load_optim(record); + + let record = self + .lr_scheduler + .restore(epoch, device) + .expect("Can load learning rate scheduler checkpoint."); + learner.load_scheduler(record); + + learner + } +} + +/// Cloneable reference to an early stopping strategy +pub(crate) type EarlyStoppingStrategyRef = Box; + +#[derive(Clone, Default)] +/// A handle that allows aborting the training/evaluation process early. +pub struct Interrupter { + state: Arc, + message: Arc>>, +} + +impl Interrupter { + /// Create a new instance. + pub fn new() -> Self { + Self::default() + } + + /// Notify the learner that it should stop. + /// # Arguments + /// * `reason` - A string describing the reason the training was stopped. + pub fn stop(&self, reason: Option<&str>) { + self.state.store(true, Ordering::Relaxed); + reason.inspect(|r| { + let mut message = self.message.lock().unwrap(); + *message = Some(String::from(*r)); + }); + } + + /// Reset the interrupter. + pub fn reset(&self) { + self.state.store(false, Ordering::Relaxed); + } + + /// True if .stop() has been called. + pub fn should_stop(&self) -> bool { + self.state.load(Ordering::Relaxed) + } + + /// Get the message associated with the interrupt. + pub fn get_message(&self) -> Option { + let message = self.message.lock().unwrap(); + message.clone() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/classification.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/classification.rs new file mode 100644 index 0000000..504c1b0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/classification.rs @@ -0,0 +1,159 @@ +use crate::metric::{ + AccuracyInput, Adaptor, AurocInput, ConfusionStatsInput, HammingScoreInput, LossInput, + PerplexityInput, TopKAccuracyInput, processor::ItemLazy, +}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{Int, Tensor, Transaction}; +use burn_ndarray::NdArray; + +/// Simple classification output adapted for multiple metrics. +/// +/// Supported metrics: +/// - Accuracy +/// - AUROC +/// - TopKAccuracy +/// - Perplexity +/// - Precision (via ConfusionStatsInput) +/// - Recall (via ConfusionStatsInput) +/// - FBetaScore (via ConfusionStatsInput) +/// - Loss. +#[derive(new)] +pub struct ClassificationOutput { + /// The loss. + pub loss: Tensor, + + /// The class logits or probabilities. Shape: \[batch_size, num_classes\]. + pub output: Tensor, + + /// The ground truth class index for each sample. Shape: \[batch_size\]. + pub targets: Tensor, +} + +impl ItemLazy for ClassificationOutput { + type ItemSync = ClassificationOutput; + + fn sync(self) -> Self::ItemSync { + let [output, loss, targets] = Transaction::default() + .register(self.output) + .register(self.loss) + .register(self.targets) + .execute() + .try_into() + .expect("Correct amount of tensor data"); + + let device = &Default::default(); + + ClassificationOutput { + output: Tensor::from_data(output, device), + loss: Tensor::from_data(loss, device), + targets: Tensor::from_data(targets, device), + } + } +} + +impl Adaptor> for ClassificationOutput { + fn adapt(&self) -> AccuracyInput { + AccuracyInput::new(self.output.clone(), self.targets.clone()) + } +} + +impl Adaptor> for ClassificationOutput { + fn adapt(&self) -> AurocInput { + AurocInput::new(self.output.clone(), self.targets.clone()) + } +} + +impl Adaptor> for ClassificationOutput { + fn adapt(&self) -> LossInput { + LossInput::new(self.loss.clone()) + } +} + +impl Adaptor> for ClassificationOutput { + fn adapt(&self) -> TopKAccuracyInput { + TopKAccuracyInput::new(self.output.clone(), self.targets.clone()) + } +} + +impl Adaptor> for ClassificationOutput { + fn adapt(&self) -> PerplexityInput { + PerplexityInput::new(self.output.clone(), self.targets.clone()) + } +} + +impl Adaptor> for ClassificationOutput { + fn adapt(&self) -> ConfusionStatsInput { + let [_, num_classes] = self.output.dims(); + if num_classes > 1 { + ConfusionStatsInput::new( + self.output.clone(), + self.targets.clone().one_hot(num_classes).bool(), + ) + } else { + ConfusionStatsInput::new( + self.output.clone(), + self.targets.clone().unsqueeze_dim(1).bool(), + ) + } + } +} + +/// Multi-label classification output adapted for multiple metrics. +/// +/// Supported metrics: +/// - HammingScore +/// - Precision (via ConfusionStatsInput) +/// - Recall (via ConfusionStatsInput) +/// - FBetaScore (via ConfusionStatsInput) +/// - Loss +#[derive(new)] +pub struct MultiLabelClassificationOutput { + /// The loss. + pub loss: Tensor, + + /// The label logits or probabilities. Shape: \[batch_size, num_classes\]. + pub output: Tensor, + + /// The ground truth labels. Shape: \[batch_size, num_classes\]. + pub targets: Tensor, +} + +impl ItemLazy for MultiLabelClassificationOutput { + type ItemSync = MultiLabelClassificationOutput; + + fn sync(self) -> Self::ItemSync { + let [output, loss, targets] = Transaction::default() + .register(self.output) + .register(self.loss) + .register(self.targets) + .execute() + .try_into() + .expect("Correct amount of tensor data"); + + let device = &Default::default(); + + MultiLabelClassificationOutput { + output: Tensor::from_data(output, device), + loss: Tensor::from_data(loss, device), + targets: Tensor::from_data(targets, device), + } + } +} + +impl Adaptor> for MultiLabelClassificationOutput { + fn adapt(&self) -> HammingScoreInput { + HammingScoreInput::new(self.output.clone(), self.targets.clone()) + } +} + +impl Adaptor> for MultiLabelClassificationOutput { + fn adapt(&self) -> LossInput { + LossInput::new(self.loss.clone()) + } +} + +impl Adaptor> for MultiLabelClassificationOutput { + fn adapt(&self) -> ConfusionStatsInput { + ConfusionStatsInput::new(self.output.clone(), self.targets.clone().bool()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/early_stopping.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/early_stopping.rs new file mode 100644 index 0000000..2028339 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/early_stopping.rs @@ -0,0 +1,298 @@ +use crate::metric::{ + Metric, MetricName, + store::{Aggregate, Direction, EventStoreClient, Split}, +}; + +/// The condition that [early stopping strategies](EarlyStoppingStrategy) should follow. +#[derive(Clone)] +pub enum StoppingCondition { + /// When no improvement has happened since the given number of epochs. + NoImprovementSince { + /// The number of epochs allowed to worsen before it gets better. + n_epochs: usize, + }, +} + +/// A strategy that checks if the training should be stopped. +pub trait EarlyStoppingStrategy: Send { + /// Update its current state and returns if the training should be stopped. + fn should_stop(&mut self, epoch: usize, store: &EventStoreClient) -> bool; +} + +/// A helper trait to provide type-erased cloning. +pub trait CloneEarlyStoppingStrategy: EarlyStoppingStrategy + Send { + /// Clone into a boxed trait object. + fn clone_box(&self) -> Box; +} + +/// Blanket-implement `CloneEarlyStoppingStrategy` for any `T` that +/// already implements your strategy + `Clone` + `Send` + `'static`. +impl CloneEarlyStoppingStrategy for T +where + T: EarlyStoppingStrategy + Clone + Send + 'static, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Now you can `impl Clone` for the boxed trait object. +impl Clone for Box { + fn clone(&self) -> Box { + self.clone_box() + } +} + +/// An [early stopping strategy](EarlyStoppingStrategy) based on a metrics collected +/// during training or validation. +#[derive(Clone)] +pub struct MetricEarlyStoppingStrategy { + condition: StoppingCondition, + metric_name: MetricName, + aggregate: Aggregate, + direction: Direction, + split: Split, + best_epoch: usize, + best_value: f64, + warmup_epochs: Option, +} + +impl EarlyStoppingStrategy for MetricEarlyStoppingStrategy { + fn should_stop(&mut self, epoch: usize, store: &EventStoreClient) -> bool { + let current_value = + match store.find_metric(&self.metric_name, epoch, self.aggregate, &self.split) { + Some(value) => value, + None => { + log::warn!("Can't find metric for early stopping."); + return false; + } + }; + + let is_best = match self.direction { + Direction::Lowest => current_value < self.best_value, + Direction::Highest => current_value > self.best_value, + }; + + if is_best { + log::info!( + "New best epoch found {} {}: {}", + epoch, + self.metric_name, + current_value + ); + self.best_value = current_value; + self.best_epoch = epoch; + return false; + } + + if let Some(warmup_epochs) = self.warmup_epochs + && epoch <= warmup_epochs + { + return false; + } + + match self.condition { + StoppingCondition::NoImprovementSince { n_epochs } => { + let should_stop = epoch - self.best_epoch >= n_epochs; + + if should_stop { + log::info!( + "Stopping training loop, no improvement since epoch {}, {}: {}, current \ + epoch {}, {}: {}", + self.best_epoch, + self.metric_name, + self.best_value, + epoch, + self.metric_name, + current_value + ); + } + + should_stop + } + } + } +} + +impl MetricEarlyStoppingStrategy { + /// Create a new [early stopping strategy](EarlyStoppingStrategy) based on a metrics collected + /// during training or validation. + /// + /// # Notes + /// + /// The metric should be registered for early stopping to work, otherwise no data is collected. + pub fn new( + metric: &Me, + aggregate: Aggregate, + direction: Direction, + split: Split, + condition: StoppingCondition, + ) -> Self { + let init_value = match direction { + Direction::Lowest => f64::MAX, + Direction::Highest => f64::MIN, + }; + + Self { + metric_name: metric.name(), + condition, + aggregate, + direction, + split, + best_epoch: 1, + best_value: init_value, + warmup_epochs: None, + } + } + + /// Get the warmup period. + /// + /// Early stopping will not trigger during the warmup epochs. + pub fn warmup_epochs(&self) -> Option { + self.warmup_epochs + } + + /// Set the warmup epochs. + /// + /// Early stopping will not trigger during the warmup epochs. + /// + /// # Arguments + /// - `warmup`: the number of warmup epochs, or None. + pub fn with_warmup_epochs(self, warmup: Option) -> Self { + Self { + warmup_epochs: warmup, + ..self + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::{ + EventProcessorTraining, TestBackend, + logger::InMemoryMetricLogger, + metric::{ + LossMetric, + processor::{ + MetricsTraining, MinimalEventProcessor, + test_utils::{end_epoch, process_train}, + }, + store::LogEventStore, + }, + }; + + use super::*; + + #[test] + fn never_early_stop_while_it_is_improving() { + test_early_stopping( + None, + 1, + &[ + (&[0.5, 0.3], false, "Should not stop first epoch"), + (&[0.4, 0.3], false, "Should not stop when improving"), + (&[0.3, 0.3], false, "Should not stop when improving"), + (&[0.2, 0.3], false, "Should not stop when improving"), + ], + ); + } + + #[test] + fn early_stop_when_no_improvement_since_two_epochs() { + test_early_stopping( + None, + 2, + &[ + (&[1.0, 0.5], false, "Should not stop first epoch"), + (&[0.5, 0.3], false, "Should not stop when improving"), + ( + &[1.0, 3.0], + false, + "Should not stop first time it gets worse", + ), + ( + &[1.0, 2.0], + true, + "Should stop since two following epochs didn't improve", + ), + ], + ); + } + + #[test] + fn early_stopping_with_warmup() { + test_early_stopping( + Some(3), + 2, + &[ + (&[1.0, 0.5], false, "Should not stop during warmup"), + (&[1.0, 0.5], false, "Should not stop during warmup"), + (&[1.0, 0.5], false, "Should not stop during warmup"), + ( + &[1.0, 0.5], + true, + "Should stop when not improving after warmup", + ), + ], + ) + } + + #[test] + fn early_stop_when_stays_equal() { + test_early_stopping( + None, + 2, + &[ + (&[0.5, 0.3], false, "Should not stop first epoch"), + ( + &[0.5, 0.3], + false, + "Should not stop first time it stars the same", + ), + ( + &[0.5, 0.3], + true, + "Should stop since two following epochs didn't improve", + ), + ], + ); + } + + fn test_early_stopping(warmup: Option, n_epochs: usize, data: &[(&[f64], bool, &str)]) { + let loss = LossMetric::::new(); + let mut early_stopping = MetricEarlyStoppingStrategy::new( + &loss, + Aggregate::Mean, + Direction::Lowest, + Split::Train, + StoppingCondition::NoImprovementSince { n_epochs }, + ) + .with_warmup_epochs(warmup); + let mut store = LogEventStore::default(); + let mut metrics = MetricsTraining::::default(); + + store.register_logger(InMemoryMetricLogger::default()); + metrics.register_train_metric_numeric(loss); + + let store = Arc::new(EventStoreClient::new(store)); + let mut processor = MinimalEventProcessor::new(metrics, store.clone()); + + let mut epoch = 1; + processor.process_train(crate::LearnerEvent::Start); + for (points, should_start, comment) in data { + for point in points.iter() { + process_train(&mut processor, *point, epoch); + } + end_epoch(&mut processor, epoch); + + assert_eq!( + *should_start, + early_stopping.should_stop(epoch, &store), + "{comment}" + ); + epoch += 1; + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/mod.rs new file mode 100644 index 0000000..293bb0d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/mod.rs @@ -0,0 +1,24 @@ +#[cfg(feature = "rl")] +mod rl; +#[cfg(feature = "rl")] +pub use rl::*; + +mod application_logger; +mod base; +mod classification; +mod early_stopping; +mod regression; +mod sequence; +mod summary; +mod supervised; +mod train_val; + +pub use application_logger::*; +pub use base::*; +pub use classification::*; +pub use early_stopping::*; +pub use regression::*; +pub use sequence::*; +pub use summary::*; +pub use supervised::*; +pub use train_val::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/regression.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/regression.rs new file mode 100644 index 0000000..9782bf0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/regression.rs @@ -0,0 +1,46 @@ +use crate::metric::processor::ItemLazy; +use crate::metric::{Adaptor, LossInput}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{Tensor, Transaction}; +use burn_ndarray::NdArray; + +/// Regression output adapted for the loss metric. +#[derive(new)] +pub struct RegressionOutput { + /// The loss. + pub loss: Tensor, + + /// The predicted values. Shape: \[batch_size, num_targets\]. + pub output: Tensor, + + /// The ground truth values. Shape: \[batch_size, num_targets\]. + pub targets: Tensor, +} + +impl Adaptor> for RegressionOutput { + fn adapt(&self) -> LossInput { + LossInput::new(self.loss.clone()) + } +} + +impl ItemLazy for RegressionOutput { + type ItemSync = RegressionOutput; + + fn sync(self) -> Self::ItemSync { + let [output, loss, targets] = Transaction::default() + .register(self.output) + .register(self.loss) + .register(self.targets) + .execute() + .try_into() + .expect("Correct amount of tensor data"); + + let device = &Default::default(); + + RegressionOutput { + output: Tensor::from_data(output, device), + loss: Tensor::from_data(loss, device), + targets: Tensor::from_data(targets, device), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/checkpointer.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/checkpointer.rs new file mode 100644 index 0000000..0c0bbe8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/checkpointer.rs @@ -0,0 +1,75 @@ +use burn_core::tensor::Device; +use burn_rl::{Policy, PolicyLearner, PolicyState}; + +use crate::RLAgentRecord; +use crate::{ + RLComponentsTypes, RLPolicyRecord, + checkpoint::Checkpointer, + checkpoint::{AsyncCheckpointer, CheckpointingAction, CheckpointingStrategy}, + metric::store::EventStoreClient, +}; + +#[derive(new)] +/// Used to create, delete, or load checkpoints of the training process. +pub struct RLCheckpointer { + policy: AsyncCheckpointer, RLC::Backend>, + learning_agent: AsyncCheckpointer, RLC::Backend>, + strategy: Box, +} + +impl RLCheckpointer { + /// Create checkpoint for the training process. + pub fn checkpoint( + &mut self, + policy: &RLC::PolicyState, + learning_agent: &RLC::LearningAgent, + epoch: usize, + store: &EventStoreClient, + ) { + let actions = self.strategy.checkpointing(epoch, store); + + for action in actions { + match action { + CheckpointingAction::Delete(epoch) => { + self.policy + .delete(epoch) + .expect("Can delete policy checkpoint."); + self.learning_agent + .delete(epoch) + .expect("Can delete learning agent checkpoint.") + } + CheckpointingAction::Save => { + self.policy + .save(epoch, policy.clone().into_record()) + .expect("Can save policy checkpoint."); + self.learning_agent + .save(epoch, learning_agent.record()) + .expect("Can save learning agent checkpoint."); + } + } + } + } + + /// Load a training checkpoint. + pub fn load_checkpoint( + &self, + learning_agent: RLC::LearningAgent, + device: &Device, + epoch: usize, + ) -> RLC::LearningAgent { + let record = self + .policy + .restore(epoch, device) + .expect("Can load model checkpoint."); + let policy = learning_agent.policy().load_record(record); + + let record = self + .learning_agent + .restore(epoch, device) + .expect("Can load learning agent checkpoint."); + let mut learning_agent = learning_agent.load_record(record); + learning_agent.update_policy(policy); + + learning_agent + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/components.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/components.rs new file mode 100644 index 0000000..3377ded --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/components.rs @@ -0,0 +1,115 @@ +use std::marker::PhantomData; + +use burn_core::tensor::backend::AutodiffBackend; +use burn_rl::{Batchable, Environment, EnvironmentInit, Policy, PolicyLearner, PolicyState}; + +use crate::{AgentEvaluationEvent, AsyncProcessorTraining, ItemLazy, RLEvent}; + +/// All components used by the reinforcement learning paradigm, grouped in one trait. +pub trait RLComponentsTypes { + /// The backend used for training. + type Backend: AutodiffBackend; + /// The learning environment. + type Env: Environment + 'static; + /// Specifies how to initialize the environment. + type EnvInit: EnvironmentInit + Send + 'static; + /// The type of the environment state. + type State: Into<>::Observation> + Clone + Send + 'static; + /// The type of the environment action. + type Action: From<>::Action> + + Into<>::Action> + + Clone + + Send + + 'static; + + /// The policy used to take actions in the environment. + type Policy: Policy< + Self::Backend, + Observation = Self::PolicyObs, + ActionDistribution = Self::PolicyAD, + Action = Self::PolicyAction, + ActionContext = Self::ActionContext, + PolicyState = Self::PolicyState, + > + Send + + 'static; + /// The policy's observation type. + type PolicyObs: Clone + Send + Batchable + 'static; + /// The policy's action distribution type. + type PolicyAD: Clone + Send + Batchable; + /// The policy's action type. + type PolicyAction: Clone + Send + Batchable; + /// Additional data as context for an agent's action. + type ActionContext: ItemLazy + Clone + Send + 'static; + /// The state of the parameterized policy. + type PolicyState: Clone + Send + PolicyState + 'static; + + /// The learning agent. + type LearningAgent: PolicyLearner< + Self::Backend, + TrainContext = Self::TrainingOutput, + InnerPolicy = Self::Policy, + > + Send + + 'static; + /// The output data of a training step. + type TrainingOutput: ItemLazy + Clone + Send; +} + +/// Concrete type that implements the [RLComponentsTypes](RLComponentsTypes) trait. +pub struct RLComponentsMarker { + _backend: PhantomData, + _env: PhantomData, + _env_init: PhantomData, + _agent: PhantomData, +} + +impl RLComponentsTypes for RLComponentsMarker +where + B: AutodiffBackend, + E: Environment + 'static, + EI: EnvironmentInit + Send + 'static, + A: PolicyLearner + Send + 'static, + A::TrainContext: ItemLazy + Clone + Send, + A::InnerPolicy: Policy + Send, + >::Observation: Batchable + Clone + Send, + >::ActionDistribution: Batchable + Clone + Send, + >::Action: Batchable + Clone + Send, + >::ActionContext: ItemLazy + Clone + Send + 'static, + >::PolicyState: Clone + Send, + E::State: Into<>::Observation> + Clone + Send + 'static, + E::Action: From<>::Action> + + Into<>::Action> + + Clone + + Send + + 'static, +{ + type Backend = B; + type Env = E; + type EnvInit = EI; + type LearningAgent = A; + type Policy = A::InnerPolicy; + type PolicyObs = >::Observation; + type PolicyAD = >::ActionDistribution; + type PolicyAction = >::Action; + type ActionContext = >::ActionContext; + type PolicyState = >::PolicyState; + type TrainingOutput = A::TrainContext; + type State = E::State; + type Action = E::Action; +} + +pub(crate) type RlPolicy = <::LearningAgent as PolicyLearner< + ::Backend, +>>::InnerPolicy; +/// The event processor type for reinforcement learning. +pub type RLEventProcessorType = AsyncProcessorTraining< + RLEvent<::TrainingOutput, ::ActionContext>, + AgentEvaluationEvent<::ActionContext>, +>; +/// The record of the policy. +pub type RLPolicyRecord = <<::Policy as Policy< + ::Backend, +>>::PolicyState as PolicyState<::Backend>>::Record; +/// The record of the learning agent. +pub type RLAgentRecord = <::LearningAgent as PolicyLearner< + ::Backend, +>>::Record; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/async_runner.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/async_runner.rs new file mode 100644 index 0000000..dded356 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/async_runner.rs @@ -0,0 +1,703 @@ +use rand::prelude::SliceRandom; +use std::{ + sync::mpsc::{Receiver, Sender}, + thread::spawn, +}; + +use burn_core::{Tensor, data::dataloader::Progress, prelude::Backend, tensor::Device}; +use burn_rl::EnvironmentInit; +use burn_rl::Policy; +use burn_rl::Transition; +use burn_rl::{AsyncPolicy, Environment}; + +use crate::{ + AgentEnvLoop, AgentEvaluationEvent, EpisodeSummary, EvaluationItem, EventProcessorTraining, + Interrupter, RLComponentsTypes, RLEvent, RLEventProcessorType, RLTimeStep, RLTrajectory, + RlPolicy, TimeStep, Trajectory, +}; + +enum RequestMessage { + Step(), + Episode(), +} + +/// Configuration for an async agent/environment loop. +pub struct AsyncAgentEnvLoopConfig { + /// If the loop is used for evaluation (as opposed to training). + pub eval: bool, + /// If the agent should take action deterministically. + pub deterministic: bool, + /// An arbitrary ID for the loop. + pub id: usize, +} + +/// An asynchronous agent/environement interface. +pub struct AgentEnvAsyncLoop { + eval: bool, + agent: AsyncPolicy>, + transition_receiver: Receiver>, + trajectory_receiver: Receiver>, + request_sender: Sender, +} + +impl AgentEnvAsyncLoop { + /// Create a new asynchronous runner. + /// + /// # Arguments + /// + /// * `env_init` - A function returning an environment instance. + /// * `agent` - An [AsyncPolicy](AsyncPolicy) taking actions in the loop. + /// * `config` - An [AsyncAgentEnvLoopConfig](AsyncAgentEnvLoopConfig). + /// * `transition_sender` - Optional sender for transitions if you want to drive the requests from outside of the loop instance. + /// * `trajectory_sender` - Optional sender for trajectories if you want to drive the requests from outside of the loop instance. + /// + /// # Returns + /// + /// An async Agent/Environement loop. + pub fn new( + env_init: RLC::EnvInit, + agent: AsyncPolicy>, + config: AsyncAgentEnvLoopConfig, + transition_device: &Device, + transition_sender: Option>>, + trajectory_sender: Option>>, + ) -> Self { + let (loop_transition_sender, transition_receiver) = std::sync::mpsc::channel(); + let (loop_trajectory_sender, trajectory_receiver) = std::sync::mpsc::channel(); + let (request_sender, request_receiver) = std::sync::mpsc::channel(); + let loop_transition_sender = transition_sender.unwrap_or(loop_transition_sender); + let loop_trajectory_sender = trajectory_sender.unwrap_or(loop_trajectory_sender); + + let device = transition_device.clone(); + let mut loop_agent = agent.clone(); + let eval = config.eval; + + let mut current_steps = vec![]; + let mut current_reward = 0.0; + let mut step_num = 0; + spawn(move || { + let mut env = env_init.init(); + env.reset(); + + let mut request_episode = false; + loop { + let state = env.state(); + let (action, context) = + loop_agent.action(state.clone().into(), config.deterministic); + + let env_action = RLC::Action::from(action); + let step_result = env.step(env_action.clone()); + + current_reward += step_result.reward; + step_num += 1; + + let transition = Transition::new( + state.clone(), + step_result.next_state, + env_action, + Tensor::from_data([step_result.reward], &device), + Tensor::from_data( + [(step_result.done || step_result.truncated) as i32 as f64], + &device, + ), + ); + + if !request_episode { + loop_agent.decrement_agents(1); + let request = match request_receiver.recv() { + Ok(req) => req, + Err(err) => { + log::error!("Error in env runner : {}", err); + break; + } + }; + loop_agent.increment_agents(1); + + match request { + RequestMessage::Step() => (), + RequestMessage::Episode() => request_episode = true, + } + } + + let time_step = TimeStep { + env_id: config.id, + transition, + done: step_result.done, + ep_len: step_num, + cum_reward: current_reward, + action_context: context[0].clone(), + }; + current_steps.push(time_step.clone()); + + if !request_episode && let Err(err) = loop_transition_sender.send(time_step) { + log::error!("Error in env runner : {}", err); + break; + } + + if step_result.done || step_result.truncated { + if request_episode { + request_episode = false; + loop_trajectory_sender + .send(Trajectory { + timesteps: current_steps.clone(), + }) + .expect("Can send trajectory to main thread."); + } + current_steps.clear(); + + env.reset(); + current_reward = 0.; + step_num = 0; + } + } + }); + + Self { + eval, + agent, + transition_receiver, + trajectory_receiver, + request_sender, + } + } +} + +impl AgentEnvLoop for AgentEnvAsyncLoop +where + BT: Backend, + RLC: RLComponentsTypes, +{ + fn run_steps( + &mut self, + num_steps: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + progress: &mut Progress, + ) -> Vec> { + let mut items = vec![]; + for _ in 0..num_steps { + self.request_sender + .send(RequestMessage::Step()) + .expect("Can request transitions."); + let transition = self + .transition_receiver + .recv() + .expect("Can receive transitions."); + items.push(transition.clone()); + + if !self.eval { + progress.items_processed += 1; + processor.process_train(RLEvent::TimeStep(EvaluationItem::new( + transition.action_context, + progress.clone(), + None, + ))); + + if transition.done { + processor.process_train(RLEvent::EpisodeEnd(EvaluationItem::new( + EpisodeSummary { + episode_length: transition.ep_len, + cum_reward: transition.cum_reward, + }, + progress.clone(), + None, + ))); + } + } + + if interrupter.should_stop() { + break; + } + } + items + } + + fn run_episodes( + &mut self, + num_episodes: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + _progress: &mut Progress, + ) -> Vec> { + let mut items = vec![]; + self.agent.increment_agents(1); + for episode_num in 0..num_episodes { + self.request_sender + .send(RequestMessage::Episode()) + .expect("Can request episodes."); + let trajectory = self + .trajectory_receiver + .recv() + .expect("Main thread can receive trajectory."); + + for (i, step) in trajectory.timesteps.iter().enumerate() { + // TODO : clean this. + if self.eval { + processor.process_valid(AgentEvaluationEvent::TimeStep(EvaluationItem::new( + step.action_context.clone(), + Progress::new(i, i), + None, + ))); + + if step.done { + processor.process_valid(AgentEvaluationEvent::EpisodeEnd( + EvaluationItem::new( + EpisodeSummary { + episode_length: step.ep_len, + cum_reward: step.cum_reward, + }, + Progress::new(episode_num + 1, num_episodes), + None, + ), + )); + } + } else { + processor.process_train(RLEvent::TimeStep(EvaluationItem::new( + step.action_context.clone(), + Progress::new(i, i), + None, + ))); + + if step.done { + processor.process_train(RLEvent::EpisodeEnd(EvaluationItem::new( + EpisodeSummary { + episode_length: step.ep_len, + cum_reward: step.cum_reward, + }, + Progress::new(episode_num + 1, num_episodes), + None, + ))); + } + } + } + + items.push(trajectory); + if interrupter.should_stop() { + break; + } + } + self.agent.decrement_agents(1); + items + } + + fn update_policy(&mut self, update: RLC::PolicyState) { + self.agent.update(update); + } + + fn policy(&self) -> RLC::PolicyState { + self.agent.state() + } +} + +/// An asynchronous runner for multiple agent/environement interfaces. +pub struct MultiAgentEnvLoop { + num_envs: usize, + eval: bool, + agent: AsyncPolicy, + transition_receiver: Receiver>, + trajectory_receiver: Receiver>, + request_senders: Vec>, +} + +impl MultiAgentEnvLoop { + /// Create a new asynchronous runner for multiple agent/environement interfaces. + pub fn new( + num_envs: usize, + env_init: RLC::EnvInit, + agent: AsyncPolicy, + eval: bool, + deterministic: bool, + device: &Device, + ) -> Self { + let (transition_sender, transition_receiver) = std::sync::mpsc::channel(); + let (trajectory_sender, trajectory_receiver) = std::sync::mpsc::channel(); + let mut request_senders = vec![]; + + // Double batching : The environments are always one step ahead of requests. This allows inference for the first batch of steps. + agent.increment_agents(num_envs); + + for i in 0..num_envs { + let config = AsyncAgentEnvLoopConfig { + eval, + deterministic, + id: i, + }; + let runner = AgentEnvAsyncLoop::::new( + env_init.clone(), + agent.clone(), + config, + &device.clone(), + Some(transition_sender.clone()), + Some(trajectory_sender.clone()), + ); + request_senders.push(runner.request_sender.clone()); + } + + // Double batching : The environments are always one step ahead. + request_senders.iter().for_each(|s| { + s.send(RequestMessage::Step()) + .expect("Main thread can send step requests.") + }); + + Self { + num_envs, + eval, + agent: agent.clone(), + transition_receiver, + trajectory_receiver, + request_senders, + } + } +} + +impl AgentEnvLoop for MultiAgentEnvLoop +where + BT: Backend, + RLC: RLComponentsTypes, +{ + fn run_steps( + &mut self, + num_steps: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + progress: &mut Progress, + ) -> Vec> { + let mut items = vec![]; + for _ in 0..num_steps { + let transition = self + .transition_receiver + .recv() + .expect("Can receive transitions."); + items.push(transition.clone()); + + self.request_senders[transition.env_id] + .send(RequestMessage::Step()) + .expect("Main thread can request steps."); + + if !self.eval { + progress.items_processed += 1; + processor.process_train(RLEvent::TimeStep(EvaluationItem::new( + transition.action_context, + progress.clone(), + None, + ))); + + if transition.done { + processor.process_train(RLEvent::EpisodeEnd(EvaluationItem::new( + EpisodeSummary { + episode_length: transition.ep_len, + cum_reward: transition.cum_reward, + }, + progress.clone(), + None, + ))); + } + } + + if interrupter.should_stop() { + break; + } + } + items + } + + fn update_policy(&mut self, update: RLC::PolicyState) { + self.agent.update(update); + } + + fn run_episodes( + &mut self, + num_episodes: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + _progress: &mut Progress, + ) -> Vec> { + // Send `num_episodes` initial requests. + let mut idx = vec![]; + if num_episodes < self.num_envs { + let mut rng = rand::rng(); + let mut vec: Vec = (0..self.num_envs).collect(); + vec.shuffle(&mut rng); + idx = vec.into_iter().take(num_episodes).collect(); + } else { + idx = (0..self.num_envs).collect(); + } + let num_requests = self.num_envs.min(num_episodes); + idx.into_iter().for_each(|i| { + self.request_senders[i] + .send(RequestMessage::Episode()) + .expect("Main thread can request steps."); + }); + + let mut items = vec![]; + for episode_num in 0..num_episodes { + let trajectory = self + .trajectory_receiver + .recv() + .expect("Can receive trajectory."); + items.push(trajectory.clone()); + if items.len() + num_requests <= num_episodes { + self.request_senders[trajectory.timesteps[0].env_id] + .send(RequestMessage::Episode()) + .expect("Main thread can request steps."); + } + for (i, step) in trajectory.timesteps.iter().enumerate() { + if self.eval { + processor.process_valid(AgentEvaluationEvent::TimeStep(EvaluationItem::new( + step.action_context.clone(), + Progress::new(i, i), + None, + ))); + + if step.done { + processor.process_valid(AgentEvaluationEvent::EpisodeEnd( + EvaluationItem::new( + EpisodeSummary { + episode_length: step.ep_len, + cum_reward: step.cum_reward, + }, + Progress::new(episode_num + 1, num_episodes), + None, + ), + )); + } + } else { + processor.process_train(RLEvent::TimeStep(EvaluationItem::new( + step.action_context.clone(), + Progress::new(i, i), + None, + ))); + + if step.done { + processor.process_train(RLEvent::EpisodeEnd(EvaluationItem::new( + EpisodeSummary { + episode_length: step.ep_len, + cum_reward: step.cum_reward, + }, + Progress::new(episode_num + 1, num_episodes), + None, + ))); + } + } + } + + if interrupter.should_stop() { + break; + } + } + + items + } + + fn policy(&self) -> RLC::PolicyState { + self.agent.state() + } +} + +#[cfg(test)] +#[allow(clippy::needless_range_loop)] +mod tests { + use burn_core::data::dataloader::Progress; + use burn_rl::AsyncPolicy; + + use crate::learner::rl::env_runner::async_runner::AsyncAgentEnvLoopConfig; + use crate::learner::rl::env_runner::base::AgentEnvLoop; + use crate::learner::tests::{MockPolicyState, MockProcessor}; + use crate::{ + AgentEnvAsyncLoop, TestBackend, + learner::tests::{MockEnvInit, MockPolicy, MockRLComponents}, + }; + use crate::{AsyncProcessorTraining, Interrupter, MultiAgentEnvLoop}; + + fn setup_async_loop( + state: usize, + eval: bool, + deterministic: bool, + ) -> AgentEnvAsyncLoop { + let env_init = MockEnvInit; + let agent = MockPolicy(state); + let config = AsyncAgentEnvLoopConfig { + eval, + deterministic, + id: 0, + }; + AgentEnvAsyncLoop::::new( + env_init, + AsyncPolicy::new(1, agent), + config, + &Default::default(), + None, + None, + ) + } + + fn setup_multi_loop( + num_envs: usize, + autobatch_size: usize, + state: usize, + eval: bool, + deterministic: bool, + ) -> MultiAgentEnvLoop { + let env_init = MockEnvInit; + let agent = MockPolicy(state); + MultiAgentEnvLoop::::new( + num_envs, + env_init, + AsyncPolicy::new(autobatch_size, agent), + eval, + deterministic, + &Default::default(), + ) + } + + #[test] + fn test_policy_async_loop() { + let runner = setup_async_loop(1000, false, false); + let policy_state = runner.policy(); + assert_eq!(policy_state.0, 1000); + } + + #[test] + fn test_update_policy_async_loop() { + let mut runner = setup_async_loop(0, false, false); + + runner.update_policy(MockPolicyState(1)); + assert_eq!(runner.policy().0, 1); + } + + #[test] + fn run_steps_returns_requested_number_async_loop() { + let mut runner = setup_async_loop(0, false, false); + let mut processor = AsyncProcessorTraining::new(MockProcessor); + let interrupter = Interrupter::new(); + let mut progress = Progress { + items_processed: 0, + items_total: 1, + }; + + let steps = runner.run_steps(1, &mut processor, &interrupter, &mut progress); + assert_eq!(steps.len(), 1); + let steps = runner.run_steps(8, &mut processor, &interrupter, &mut progress); + assert_eq!(steps.len(), 8); + } + + #[test] + fn run_episodes_returns_requested_number_async_loop() { + let mut runner = setup_async_loop(0, false, false); + let mut processor = AsyncProcessorTraining::new(MockProcessor); + let interrupter = Interrupter::new(); + let mut progress = Progress { + items_processed: 0, + items_total: 1, + }; + + let trajectories = runner.run_episodes(1, &mut processor, &interrupter, &mut progress); + assert_eq!(trajectories.len(), 1); + assert_ne!(trajectories[0].timesteps.len(), 0); + let trajectories = runner.run_episodes(8, &mut processor, &interrupter, &mut progress); + assert_eq!(trajectories.len(), 8); + for i in 0..8 { + assert_ne!(trajectories[i].timesteps.len(), 0); + } + } + + #[test] + fn test_policy_multi_loop() { + let runner = setup_multi_loop(4, 4, 1000, false, false); + let policy_state = runner.policy(); + assert_eq!(policy_state.0, 1000); + } + + #[test] + fn test_update_policy_multi_loop() { + let mut runner = setup_multi_loop(4, 4, 0, false, false); + + runner.update_policy(MockPolicyState(1)); + assert_eq!(runner.policy().0, 1); + } + + #[test] + fn run_steps_returns_requested_number_multi_loop() { + fn run_test(num_envs: usize, autobatch_size: usize) { + let mut runner = setup_multi_loop(num_envs, autobatch_size, 0, false, false); + let mut processor = AsyncProcessorTraining::new(MockProcessor); + let interrupter = Interrupter::new(); + let mut progress = Progress { + items_processed: 0, + items_total: 1, + }; + + // Kickstart tests by running some steps to make sure it's not a double batching edge case success. + let steps = runner.run_steps(8, &mut processor, &interrupter, &mut progress); + assert_eq!(steps.len(), 8); + + for i in 0..16 { + let steps = runner.run_steps(i, &mut processor, &interrupter, &mut progress); + assert_eq!(steps.len(), i); + } + } + + // num_envs == autobatch_size + run_test(1, 1); + run_test(4, 4); + // num_envs < autobatch_size + run_test(1, 2); + run_test(1, 3); + run_test(2, 3); + run_test(2, 4); + run_test(5, 19); + // num_envs > autobatch_size + run_test(2, 1); + run_test(8, 1); + run_test(3, 2); + run_test(8, 2); + run_test(8, 3); + run_test(8, 7); + } + + #[test] + fn run_episodes_returns_requested_number_multi_loop() { + fn run_test(num_envs: usize, autobatch_size: usize) { + let mut runner = setup_multi_loop(num_envs, autobatch_size, 0, false, false); + let mut processor = AsyncProcessorTraining::new(MockProcessor); + let interrupter = Interrupter::new(); + let mut progress = Progress { + items_processed: 0, + items_total: 1, + }; + + // Kickstart tests by running some episodes to make sure it's not a double batching edge case success. + let trajectories = runner.run_episodes(8, &mut processor, &interrupter, &mut progress); + assert_eq!(trajectories.len(), 8); + for j in 0..8 { + assert_ne!(trajectories[j].timesteps.len(), 0); + } + + for i in 0..16 { + let trajectories = + runner.run_episodes(i, &mut processor, &interrupter, &mut progress); + assert_eq!(trajectories.len(), i); + for j in 0..i { + assert_ne!(trajectories[j].timesteps.len(), 0); + } + } + } + + // num_envs == autobatch_size + run_test(1, 1); + run_test(4, 4); + // num_envs < autobatch_size + run_test(1, 2); + run_test(1, 3); + run_test(2, 3); + run_test(2, 4); + run_test(5, 19); + // num_envs > autobatch_size + run_test(2, 1); + run_test(8, 1); + run_test(3, 2); + run_test(8, 2); + run_test(8, 3); + run_test(8, 7); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/base.rs new file mode 100644 index 0000000..d83f372 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/base.rs @@ -0,0 +1,343 @@ +use std::marker::PhantomData; + +use burn_core::data::dataloader::Progress; +use burn_core::{Tensor, prelude::Backend}; +use burn_rl::Policy; +use burn_rl::Transition; +use burn_rl::{Environment, EnvironmentInit}; + +use crate::RLEvent; +use crate::{ + AgentEvaluationEvent, EpisodeSummary, EvaluationItem, EventProcessorTraining, + RLEventProcessorType, +}; +use crate::{Interrupter, RLComponentsTypes}; + +/// A trajectory, i.e. a list of ordered [TimeStep](TimeStep). +#[derive(Clone, new)] +pub struct Trajectory { + /// A list of ordered [TimeStep](TimeStep)s. + pub timesteps: Vec>, +} + +/// A timestep debscribing an iteration of the state/decision process. +#[derive(Clone)] +pub struct TimeStep { + /// The environment id. + pub env_id: usize, + /// The [burn_rl::Transition](burn_rl::Transition). + pub transition: Transition, + /// True if the environment reaches a terminal state. + pub done: bool, + /// The running length of the current episode. + pub ep_len: usize, + /// The running cumulative reward. + pub cum_reward: f64, + /// The action's context for this timestep. + pub action_context: C, +} + +pub(crate) type RLTimeStep = TimeStep< + B, + ::State, + ::Action, + ::ActionContext, +>; + +pub(crate) type RLTrajectory = Trajectory< + B, + ::State, + ::Action, + ::ActionContext, +>; + +/// Trait for a structure that implements an agent/environement interface. +pub trait AgentEnvLoop { + /// Run a certain number of timesteps. + /// + /// # Arguments + /// + /// * `num_steps` - The number of time_steps to run. + /// * `processor` - An [crate::EventProcessorTraining](crate::EventProcessorTraining). + /// * `interrupter` - An [crate::Interrupter](crate::Interrupter). + /// * `num_steps` - The number of time_steps to run. + /// * `progress` - A mutable reference to the learning progress. + /// + /// # Returns + /// + /// A list of ordered timesteps. + fn run_steps( + &mut self, + num_steps: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + progress: &mut Progress, + ) -> Vec>; + /// Run a certain number of episodes. + /// + /// # Arguments + /// + /// * `num_episodes` - The number of episodes to run. + /// * `processor` - An [crate::EventProcessorTraining](crate::EventProcessorTraining). + /// * `interrupter` - An [crate::Interrupter](crate::Interrupter). + /// * `progress` - A mutable reference to the learning progress. + /// + /// # Returns + /// + /// A list of ordered timesteps. + fn run_episodes( + &mut self, + num_episodes: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + progress: &mut Progress, + ) -> Vec>; + /// Update the runner's agent. + fn update_policy(&mut self, update: RLC::PolicyState); + /// Get the state of the runner's agent. + fn policy(&self) -> RLC::PolicyState; +} + +/// A simple, synchronized agent/environement interface. +pub struct AgentEnvBaseLoop { + env: RLC::Env, + eval: bool, + agent: RLC::Policy, + deterministic: bool, + current_reward: f64, + run_num: usize, + step_num: usize, + _backend: PhantomData, +} + +impl AgentEnvBaseLoop { + /// Create a new base runner. + pub fn new( + env_init: RLC::EnvInit, + agent: RLC::Policy, + eval: bool, + deterministic: bool, + ) -> Self { + let mut env = env_init.init(); + env.reset(); + + Self { + env, + eval, + agent: agent.clone(), + deterministic, + current_reward: 0.0, + run_num: 0, + step_num: 0, + _backend: PhantomData, + } + } +} + +impl AgentEnvLoop for AgentEnvBaseLoop +where + BT: Backend, + RLC: RLComponentsTypes, +{ + fn run_steps( + &mut self, + num_steps: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + progress: &mut Progress, + ) -> Vec> { + let mut items = vec![]; + let device = Default::default(); + for _ in 0..num_steps { + let state = self.env.state(); + let (action, context) = self.agent.action(state.clone().into(), self.deterministic); + + let step_result = self.env.step(RLC::Action::from(action.clone())); + + self.current_reward += step_result.reward; + self.step_num += 1; + + let transition = Transition::new( + state.clone(), + step_result.next_state, + RLC::Action::from(action), + Tensor::from_data([step_result.reward], &device), + Tensor::from_data( + [(step_result.done || step_result.truncated) as i32 as f64], + &device, + ), + ); + items.push(TimeStep { + env_id: 0, + transition, + done: step_result.done, + ep_len: self.step_num, + cum_reward: self.current_reward, + action_context: context[0].clone(), + }); + + if !self.eval { + progress.items_processed += 1; + processor.process_train(RLEvent::TimeStep(EvaluationItem::new( + context[0].clone(), + progress.clone(), + None, + ))); + + if step_result.done { + processor.process_train(RLEvent::EpisodeEnd(EvaluationItem::new( + EpisodeSummary { + episode_length: self.step_num, + cum_reward: self.current_reward, + }, + progress.clone(), + None, + ))); + } + } + + if interrupter.should_stop() { + break; + } + + if step_result.done || step_result.truncated { + self.env.reset(); + self.current_reward = 0.; + self.step_num = 0; + self.run_num += 1; + } + } + items + } + + fn update_policy(&mut self, update: RLC::PolicyState) { + self.agent.update(update); + } + + fn run_episodes( + &mut self, + num_episodes: usize, + processor: &mut RLEventProcessorType, + interrupter: &Interrupter, + progress: &mut Progress, + ) -> Vec> { + self.env.reset(); + + let mut items = vec![]; + for ep in 0..num_episodes { + let mut steps = vec![]; + loop { + let step = self.run_steps(1, processor, interrupter, progress)[0].clone(); + steps.push(step.clone()); + + if self.eval { + processor.process_valid(AgentEvaluationEvent::TimeStep(EvaluationItem::new( + step.action_context.clone(), + Progress::new(steps.len() + 1, steps.len() + 1), + None, + ))); + + if step.done { + processor.process_valid(AgentEvaluationEvent::EpisodeEnd( + EvaluationItem::new( + EpisodeSummary { + episode_length: step.ep_len, + cum_reward: step.cum_reward, + }, + Progress::new(ep + 1, num_episodes), + None, + ), + )); + } + } + + if interrupter.should_stop() || step.done { + break; + } + } + items.push(Trajectory::new(steps)); + + if interrupter.should_stop() { + break; + } + } + items + } + + fn policy(&self) -> RLC::PolicyState { + self.agent.state() + } +} + +#[cfg(test)] +#[allow(clippy::needless_range_loop)] +mod tests { + use crate::{AsyncProcessorTraining, TestBackend}; + + use crate::learner::tests::{ + MockEnvInit, MockPolicy, MockPolicyState, MockProcessor, MockRLComponents, + }; + + use super::*; + + fn setup( + state: usize, + eval: bool, + deterministic: bool, + ) -> AgentEnvBaseLoop { + let env_init = MockEnvInit; + let agent = MockPolicy(state); + AgentEnvBaseLoop::::new(env_init, agent, eval, deterministic) + } + + #[test] + fn test_policy_returns_agent_state() { + let runner = setup(1000, false, false); + let policy_state = runner.policy(); + assert_eq!(policy_state.0, 1000); + } + + #[test] + fn test_update_policy() { + let mut runner = setup(0, false, false); + + runner.update_policy(MockPolicyState(1)); + assert_eq!(runner.policy().0, 1); + } + + #[test] + fn run_steps_returns_requested_number() { + let mut runner = setup(0, false, false); + let mut processor = AsyncProcessorTraining::new(MockProcessor); + let interrupter = Interrupter::new(); + let mut progress = Progress { + items_processed: 0, + items_total: 1, + }; + + let steps = runner.run_steps(1, &mut processor, &interrupter, &mut progress); + assert_eq!(steps.len(), 1); + let steps = runner.run_steps(8, &mut processor, &interrupter, &mut progress); + assert_eq!(steps.len(), 8); + } + + #[test] + fn run_episodes_returns_requested_number() { + let mut runner = setup(0, false, false); + let mut processor = AsyncProcessorTraining::new(MockProcessor); + let interrupter = Interrupter::new(); + let mut progress = Progress { + items_processed: 0, + items_total: 1, + }; + + let trajectories = runner.run_episodes(1, &mut processor, &interrupter, &mut progress); + assert_eq!(trajectories.len(), 1); + assert_ne!(trajectories[0].timesteps.len(), 0); + let trajectories = runner.run_episodes(8, &mut processor, &interrupter, &mut progress); + assert_eq!(trajectories.len(), 8); + for i in 0..8 { + assert_ne!(trajectories[i].timesteps.len(), 0); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/mod.rs new file mode 100644 index 0000000..a50195a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/env_runner/mod.rs @@ -0,0 +1,302 @@ +mod async_runner; +mod base; + +pub use async_runner::*; +pub use base::*; + +#[cfg(test)] +pub(crate) mod tests { + use burn_rl::{Batchable, Environment, EnvironmentInit, Policy, PolicyState}; + + use crate::tests::TestAutodiffBackend; + use crate::{ + AgentEvaluationEvent, EventProcessorTraining, ItemLazy, RLComponentsTypes, RLEvent, + }; + use burn_rl::{LearnerTransitionBatch, PolicyLearner, RLTrainOutput, StepResult}; + + /// Mock policy for testing + /// + /// Calling `forward()` with a [MockObservation](MockObservation) (list of f32) returns a [MockActionDistribution](MockActionDistribution) + /// containing a list of 0s of the same length as the observation. + /// + /// Calling `action()` with a [MockObservation](MockObservation) (list of f32) returns a [MockPolicyAction](MockPolicyAction) with a list of actions of the same length as the observation. + /// The actions are all 1 if the call is requested as deterministic, or else 0. + + #[derive(Clone)] + pub(crate) struct MockPolicy(pub usize); + + impl Policy for MockPolicy { + type Observation = MockObservation; + type ActionDistribution = MockActionDistribution; + type Action = MockPolicyAction; + type ActionContext = MockActionContext; + type PolicyState = MockPolicyState; + + fn forward(&mut self, obs: Self::Observation) -> Self::ActionDistribution { + let mut dists = vec![]; + for _ in obs.0 { + dists.push(MockActionDistribution(vec![0.])); + } + MockActionDistribution::batch(dists) + } + + fn action( + &mut self, + obs: Self::Observation, + deterministic: bool, + ) -> (Self::Action, Vec) { + let mut actions = vec![]; + let mut contexts = vec![]; + + for _ in obs.0 { + if deterministic { + actions.push(MockPolicyAction(vec![1])); + } else { + actions.push(MockPolicyAction(vec![0])); + } + contexts.push(MockActionContext); + } + + (MockPolicyAction::batch(actions), contexts) + } + + fn update(&mut self, update: Self::PolicyState) { + self.0 = update.0; + } + + fn state(&self) -> Self::PolicyState { + MockPolicyState(self.0) + } + + fn load_record( + self, + _record: >::Record, + ) -> Self { + self + } + } + + /// Mock observation for testing represented as a vector of f32. Can call `batch()` and `unbatch` on it. + #[derive(Clone)] + pub(crate) struct MockObservation(pub Vec); + + /// Mock action for testing represented as a vector of i32. Can call `batch()` and `unbatch` on it. + #[derive(Clone)] + pub(crate) struct MockPolicyAction(pub Vec); + + /// Mock action distribution for testing represented as a vector of i32. Can call `batch()` and `unbatch` on it. + #[derive(Clone)] + pub(crate) struct MockActionDistribution(Vec); + + #[derive(Clone)] + pub(crate) struct MockActionContext; + + /// Mock policy state for testing represented as an arbitrary `usize` that has no effect on the policy. + #[derive(Clone)] + pub(crate) struct MockPolicyState(pub usize); + + impl PolicyState for MockPolicyState { + type Record = (); + + fn into_record(self) -> Self::Record {} + + fn load_record(&self, _record: Self::Record) -> Self { + self.clone() + } + } + + impl Batchable for MockObservation { + fn batch(items: Vec) -> Self { + MockObservation(items.iter().flat_map(|m| m.0.clone()).collect()) + } + + fn unbatch(self) -> Vec { + vec![MockObservation(self.0)] + } + } + + impl Batchable for MockPolicyAction { + fn batch(items: Vec) -> Self { + MockPolicyAction(items.iter().flat_map(|m| m.0.clone()).collect()) + } + + fn unbatch(self) -> Vec { + let mut actions = vec![]; + for a in self.0 { + actions.push(MockPolicyAction(vec![a])); + } + actions + } + } + + impl Batchable for MockActionDistribution { + fn batch(items: Vec) -> Self { + MockActionDistribution(items.iter().flat_map(|m| m.0.clone()).collect()) + } + + fn unbatch(self) -> Vec { + let mut dists = vec![]; + for _ in self.0 { + dists.push(MockActionDistribution(vec![0.])); + } + dists + } + } + + /// Mock environment for testing + #[derive(Clone)] + pub(crate) struct MockEnv { + counter: usize, + } + + #[derive(Clone, Debug)] + pub(crate) struct MockState; + + #[derive(Clone, Debug)] + pub(crate) struct MockAction(pub i32); + + impl From for MockObservation { + fn from(_value: MockState) -> Self { + MockObservation(vec![0.]) + } + } + + impl From for MockAction { + fn from(value: MockPolicyAction) -> Self { + MockAction(value.0[0]) + } + } + + impl From for MockPolicyAction { + fn from(value: MockAction) -> Self { + MockPolicyAction(vec![value.0]) + } + } + + impl ItemLazy for MockActionContext { + type ItemSync = MockActionContext; + + fn sync(self) -> Self::ItemSync { + self + } + } + + impl MockEnv { + fn new() -> Self { + Self { counter: 0 } + } + } + + impl Environment for MockEnv { + type State = MockState; + type Action = MockAction; + const MAX_STEPS: usize = 5; + + fn reset(&mut self) { + self.counter = 0; + } + + fn step(&mut self, _action: Self::Action) -> StepResult { + self.counter += 1; + let done = self.counter >= Self::MAX_STEPS; + + burn_rl::StepResult { + next_state: MockState, + reward: 1.0, + done, + truncated: false, + } + } + + fn state(&self) -> Self::State { + MockState + } + } + + /// Mock environment init for testing + #[derive(Clone)] + pub(crate) struct MockEnvInit; + + impl EnvironmentInit for MockEnvInit { + fn init(&self) -> MockEnv { + MockEnv::new() + } + } + + // Mock RLComponentsTypes for testing + pub(crate) struct MockRLComponents; + + impl RLComponentsTypes for MockRLComponents { + type Backend = TestAutodiffBackend; + type Env = MockEnv; + type EnvInit = MockEnvInit; + type State = MockState; + type Action = MockAction; + type Policy = MockPolicy; + type PolicyObs = MockObservation; + type PolicyAD = MockActionDistribution; + type PolicyAction = MockPolicyAction; + type ActionContext = MockActionContext; + type PolicyState = MockPolicyState; + type LearningAgent = MockLearningAgent; + type TrainingOutput = (); + } + + // Mock learning agent for testing + #[derive(Clone)] + pub(crate) struct MockLearningAgent; + + impl PolicyLearner for MockLearningAgent { + type InnerPolicy = MockPolicy; + type TrainContext = (); + type Record = (); + + fn train( + &mut self, + _input: LearnerTransitionBatch, + ) -> RLTrainOutput< + Self::TrainContext, + >::PolicyState, + > { + unimplemented!() + } + + fn policy(&self) -> Self::InnerPolicy { + unimplemented!() + } + + fn update_policy(&mut self, _update: Self::InnerPolicy) { + unimplemented!() + } + + fn record(&self) -> Self::Record { + unimplemented!() + } + + fn load_record(self, _record: Self::Record) -> Self { + unimplemented!() + } + } + + // Mock event processor for testing + pub(crate) struct MockProcessor; + + impl + EventProcessorTraining< + RLEvent<(), MockActionContext>, + AgentEvaluationEvent, + > for MockProcessor + { + fn process_train(&mut self, _event: RLEvent<(), MockActionContext>) { + // Mock process train + } + + fn process_valid(&mut self, _event: AgentEvaluationEvent) { + // Mock process valid + } + + fn renderer(self) -> Box { + unimplemented!() + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/mod.rs new file mode 100644 index 0000000..34874dc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/mod.rs @@ -0,0 +1,15 @@ +mod checkpointer; +mod components; +mod env_runner; +mod off_policy; +mod output; +mod paradigm; +mod strategy; + +pub use checkpointer::*; +pub use components::*; +pub use env_runner::*; +pub use off_policy::*; +pub use output::*; +pub use paradigm::*; +pub use strategy::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/off_policy.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/off_policy.rs new file mode 100644 index 0000000..726e2f2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/off_policy.rs @@ -0,0 +1,189 @@ +use std::marker::PhantomData; + +use crate::{ + AgentEnvAsyncLoop, AgentEnvLoop, AsyncAgentEnvLoopConfig, EvaluationItem, + EventProcessorTraining, MultiAgentEnvLoop, RLComponents, RLComponentsTypes, RLEvent, + RLEventProcessorType, RLStrategy, +}; +use burn_core::{self as burn}; +use burn_core::{config::Config, data::dataloader::Progress}; +use burn_ndarray::NdArray; +use burn_rl::{AsyncPolicy, Policy, PolicyLearner, SliceAccess, TransitionBuffer}; + +/// Parameters of an on policy training with multi environments and double-batching. +#[derive(Config, Debug)] +pub struct OffPolicyConfig { + /// The number of environments to run simultaneously for experience collection. + #[config(default = 1)] + pub num_envs: usize, + /// Number of environment state to accumulate before running one step of inference with the policy. + /// Must be equal or less than the number of simultaneous environments. + #[config(default = 1)] + pub autobatch_size: usize, + /// Max number of transitions stored in the replay buffer. + #[config(default = 1024)] + pub replay_buffer_size: usize, + /// The number of steps to collect between each step of training. + #[config(default = 1)] + pub train_interval: usize, + /// Number of optimization steps done each `train_interval`. + #[config(default = 1)] + pub train_steps: usize, + /// The number of steps to collect between each evaluation. + #[config(default = 10_000)] + pub eval_interval: usize, + /// The number of episodes to run for each evaluation. + #[config(default = 1)] + pub eval_episodes: usize, + /// The number of transition to train on. + #[config(default = 32)] + pub train_batch_size: usize, + /// Number of steps to collect before starting to train. + #[config(default = 0)] + pub warmup_steps: usize, +} + +/// Off-policy reinforcement learning strategy with multi-env experience collection and double-batching. +pub struct OffPolicyStrategy { + config: OffPolicyConfig, + _components: PhantomData, +} +impl OffPolicyStrategy { + /// Create a new off-policy base strategy. + pub fn new(config: OffPolicyConfig) -> Self { + Self { + config, + _components: PhantomData, + } + } +} + +impl RLStrategy for OffPolicyStrategy +where + RLC: RLComponentsTypes, + RLC::PolicyObs: SliceAccess, + RLC::PolicyAction: SliceAccess, +{ + fn train_loop( + &self, + training_components: RLComponents, + learner_agent: &mut RLC::LearningAgent, + starting_epoch: usize, + env_init: RLC::EnvInit, + ) -> (RLC::Policy, RLEventProcessorType) { + let mut event_processor = training_components.event_processor; + let mut checkpointer = training_components.checkpointer; + let num_steps_total = training_components.num_steps; + + let mut env_runner = MultiAgentEnvLoop::::new( + self.config.num_envs, + env_init.clone(), + AsyncPolicy::new( + self.config.num_envs.min(self.config.autobatch_size), + learner_agent.policy(), + ), + false, + false, + &Default::default(), + ); + let runner_config = AsyncAgentEnvLoopConfig { + eval: true, + deterministic: true, + id: 0, + }; + let mut env_runner_valid = AgentEnvAsyncLoop::::new( + env_init, + AsyncPolicy::new(1, learner_agent.policy()), + runner_config, + &Default::default(), + None, + None, + ); + + let device: ::Device = Default::default(); + let mut transition_buffer = TransitionBuffer::< + RLC::Backend, + RLC::PolicyObs, + RLC::PolicyAction, + >::new(self.config.replay_buffer_size, &device); + + let mut valid_next = self.config.eval_interval + starting_epoch - 1; + let mut progress = Progress { + items_processed: starting_epoch, + items_total: num_steps_total, + }; + + let mut intermediary_update: Option<>::PolicyState> = + None; + while progress.items_processed < num_steps_total { + if training_components.interrupter.should_stop() { + let reason = training_components + .interrupter + .get_message() + .unwrap_or(String::from("Reason unknown")); + log::info!("Training interrupted: {reason}"); + break; + } + + let previous_steps = progress.items_processed; + let items = env_runner.run_steps( + self.config.train_interval, + &mut event_processor, + &training_components.interrupter, + &mut progress, + ); + + for item in &items { + let t = &item.transition; + let state: RLC::PolicyObs = t.state.clone().into(); + let next_state: RLC::PolicyObs = t.next_state.clone().into(); + let action: RLC::PolicyAction = t.action.clone().into(); + let reward = t.reward.to_data().to_vec::().unwrap()[0]; + let done = t.done.to_data().to_vec::().unwrap()[0] > 0.5; + transition_buffer.push(state, next_state, action, reward, done); + } + + if transition_buffer.len() >= self.config.train_batch_size + && progress.items_processed >= self.config.warmup_steps + { + if let Some(ref u) = intermediary_update { + env_runner.update_policy(u.clone()); + } + for _ in 0..self.config.train_steps { + let batch = transition_buffer.sample(self.config.train_batch_size); + let train_item = learner_agent.train(batch); + intermediary_update = Some(train_item.policy); + + event_processor.process_train(RLEvent::TrainStep(EvaluationItem::new( + train_item.item, + progress.clone(), + None, + ))); + } + } + + if valid_next > previous_steps && valid_next <= progress.items_processed { + env_runner_valid.update_policy(learner_agent.policy().state()); + env_runner_valid.run_episodes( + self.config.eval_episodes, + &mut event_processor, + &training_components.interrupter, + &mut progress, + ); + + if let Some(checkpointer) = &mut checkpointer { + checkpointer.checkpoint( + &env_runner.policy(), + learner_agent, + valid_next, + &training_components.event_store, + ); + } + + valid_next += self.config.eval_interval; + } + } + + (learner_agent.policy(), event_processor) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/output.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/output.rs new file mode 100644 index 0000000..01177e0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/output.rs @@ -0,0 +1,32 @@ +use crate::{ + ItemLazy, + metric::{Adaptor, CumulativeRewardInput, EpisodeLengthInput}, +}; + +/// Summary of an episode. +pub struct EpisodeSummary { + /// The total length of the episode. + pub episode_length: usize, + /// The final cumulative reward. + pub cum_reward: f64, +} + +impl ItemLazy for EpisodeSummary { + type ItemSync = EpisodeSummary; + + fn sync(self) -> Self::ItemSync { + self + } +} + +impl Adaptor for EpisodeSummary { + fn adapt(&self) -> EpisodeLengthInput { + EpisodeLengthInput::new(self.episode_length as f64) + } +} + +impl Adaptor for EpisodeSummary { + fn adapt(&self) -> CumulativeRewardInput { + CumulativeRewardInput::new(self.cum_reward) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/paradigm.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/paradigm.rs new file mode 100644 index 0000000..965c3d7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/paradigm.rs @@ -0,0 +1,525 @@ +use crate::checkpoint::{ + AsyncCheckpointer, CheckpointingStrategy, ComposedCheckpointingStrategy, FileCheckpointer, + KeepLastNCheckpoints, MetricCheckpointingStrategy, +}; +use crate::learner::base::Interrupter; +use crate::logger::{FileMetricLogger, MetricLogger}; +use crate::metric::store::{Aggregate, Direction, EventStoreClient, LogEventStore, Split}; +use crate::metric::{Adaptor, EpisodeLengthMetric, Metric, Numeric}; +use crate::renderer::{MetricsRenderer, default_renderer}; +use crate::{ + ApplicationLoggerInstaller, AsyncProcessorTraining, FileApplicationLoggerInstaller, ItemLazy, + LearnerSummaryConfig, OffPolicyConfig, OffPolicyStrategy, RLAgentRecord, RLCheckpointer, + RLComponents, RLComponentsMarker, RLComponentsTypes, RLEventProcessor, RLMetrics, + RLPolicyRecord, RLStrategy, +}; +use crate::{EpisodeSummary, RLStrategies}; +use burn_core::record::FileRecorder; +use burn_core::tensor::backend::AutodiffBackend; +use burn_rl::{Batchable, Environment, EnvironmentInit, Policy, PolicyLearner, SliceAccess}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +/// Structure to configure and launch reinforcement learning trainings. +pub struct RLTraining { + // Not that complex. Extracting into yet another type would only make it more confusing. + #[allow(clippy::type_complexity)] + checkpointers: Option<( + AsyncCheckpointer, RLC::Backend>, + AsyncCheckpointer, RLC::Backend>, + )>, + num_steps: usize, + checkpoint: Option, + directory: PathBuf, + grad_accumulation: Option, + renderer: Option>, + metrics: RLMetrics, + event_store: LogEventStore, + interrupter: Interrupter, + tracing_logger: Option>, + checkpointer_strategy: Box, + learning_strategy: RLStrategies, + // Use BTreeSet instead of HashSet for consistent (alphabetical) iteration order + summary_metrics: BTreeSet, + summary: bool, + env_initializer: RLC::EnvInit, +} + +impl RLTraining> +where + B: AutodiffBackend, + E: Environment + 'static, + EI: EnvironmentInit + Send + 'static, + A: PolicyLearner + Send + 'static, + A::TrainContext: ItemLazy + Clone + Send, + A::InnerPolicy: Policy + Send, + >::Observation: Batchable + Clone + Send, + >::ActionDistribution: Batchable + Clone + Send, + >::Action: Batchable + Clone + Send, + >::ActionContext: ItemLazy + Clone + Send + 'static, + >::PolicyState: Clone + Send, + E::State: Into<>::Observation> + Clone + Send + 'static, + E::Action: From<>::Action> + + Into<>::Action> + + Clone + + Send + + 'static, +{ + /// Creates a new runner for reinforcement learning. + /// + /// # Arguments + /// + /// * `directory` - The directory to save the checkpoints. + /// * `env_init` - Specifies how to initialize the environment. + pub fn new(directory: impl AsRef, env_initializer: EI) -> Self { + let directory = directory.as_ref().to_path_buf(); + let experiment_log_file = directory.join("experiment.log"); + Self { + num_steps: 1, + checkpoint: None, + checkpointers: None, + directory, + grad_accumulation: None, + metrics: RLMetrics::default(), + event_store: LogEventStore::default(), + renderer: None, + interrupter: Interrupter::new(), + tracing_logger: Some(Box::new(FileApplicationLoggerInstaller::new( + experiment_log_file, + ))), + checkpointer_strategy: Box::new( + ComposedCheckpointingStrategy::builder() + .add(KeepLastNCheckpoints::new(2)) + .add(MetricCheckpointingStrategy::new( + &EpisodeLengthMetric::new(), // default to evaluations' cumulative reward. + Aggregate::Mean, + Direction::Lowest, + Split::Valid, + )) + .build(), + ), + learning_strategy: RLStrategies::OffPolicyStrategy(OffPolicyConfig::new()), + summary_metrics: BTreeSet::new(), + summary: false, + env_initializer, + } + } +} + +impl RLTraining { + /// Replace the default learning strategy (Off Policy learning) with the provided one. + /// + /// # Arguments + /// + /// * `training_strategy` - The training strategy. + pub fn with_learning_strategy(mut self, learning_strategy: RLStrategies) -> Self { + self.learning_strategy = learning_strategy; + self + } + + /// Replace the default metric loggers with the provided ones. + /// + /// # Arguments + /// + /// * `logger` - The training logger. + pub fn with_metric_logger(mut self, logger: ML) -> Self + where + ML: MetricLogger + 'static, + { + self.event_store.register_logger(logger); + self + } + + /// Update the checkpointing_strategy. + pub fn with_checkpointing_strategy( + mut self, + strategy: CS, + ) -> Self { + self.checkpointer_strategy = Box::new(strategy); + self + } + + /// Replace the default CLI renderer with a custom one. + /// + /// # Arguments + /// + /// * `renderer` - The custom renderer. + pub fn renderer(mut self, renderer: MR) -> Self + where + MR: MetricsRenderer + 'static, + { + self.renderer = Some(Box::new(renderer)); + self + } + + /// Register numerical metrics for a training step of the agent. + pub fn metrics_train>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register textual metrics for a training step of the agent. + pub fn text_metrics_train>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register numerical metrics for each action of the agent. + pub fn metrics_agent>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register textual metrics for each action of the agent. + pub fn text_metrics_agent>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register numerical metrics for a completed episode. + pub fn metrics_episode>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register textual metrics for a completed episode. + pub fn text_metrics_episode>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register a textual metric for a training step. + pub fn text_metric_train(mut self, metric: Me) -> Self + where + ::ItemSync: Adaptor, + { + self.metrics.register_text_metric_train(metric); + self + } + + /// Register a [numeric](crate::metric::Numeric) [metric](Metric) for a training step. + pub fn metric_train(mut self, metric: Me) -> Self + where + Me: Metric + Numeric + 'static, + ::ItemSync: Adaptor, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_metric_train(metric); + self + } + + /// Register a textual metric for each action taken by the agent. + pub fn text_metric_agent(mut self, metric: Me) -> Self + where + ::ItemSync: Adaptor, + { + self.metrics.register_text_metric_agent(metric.clone()); + self.metrics.register_text_metric_agent_valid(metric); + self + } + + /// Register a [numeric](crate::metric::Numeric) [metric](Metric) for each action taken by the agent. + pub fn metric_agent(mut self, metric: Me) -> Self + where + Me: Metric + Numeric + 'static, + ::ItemSync: Adaptor, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_agent_metric(metric.clone()); + self.metrics.register_agent_metric_valid(metric); + self + } + + /// Register a textual metric for a completed episode. + pub fn text_metric_episode(mut self, metric: Me) -> Self + where + EpisodeSummary: Adaptor + 'static, + { + self.metrics.register_text_metric_episode(metric.clone()); + self.metrics.register_text_metric_episode_valid(metric); + self + } + + /// Register a [numeric](crate::metric::Numeric) [metric](Metric) for a completed episode. + pub fn metric_episode(mut self, metric: Me) -> Self + where + Me: Metric + Numeric + 'static, + EpisodeSummary: Adaptor + 'static, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_episode_metric(metric.clone()); + self.metrics.register_episode_metric_valid(metric); + self + } + + /// The number of environment steps to train for. + pub fn num_steps(mut self, num_steps: usize) -> Self { + self.num_steps = num_steps; + self + } + + /// The step from which the training must resume. + pub fn checkpoint(mut self, checkpoint: usize) -> Self { + self.checkpoint = Some(checkpoint); + self + } + + /// Provides a handle that can be used to interrupt training. + pub fn interrupter(&self) -> Interrupter { + self.interrupter.clone() + } + + /// Override the handle for stopping training with an externally provided handle + pub fn with_interrupter(mut self, interrupter: Interrupter) -> Self { + self.interrupter = interrupter; + self + } + + /// By default, Rust logs are captured and written into + /// `experiment.log`. If disabled, standard Rust log handling + /// will apply. + pub fn with_application_logger( + mut self, + logger: Option>, + ) -> Self { + self.tracing_logger = logger; + self + } + + /// Register a checkpointer that will save the environment runner's [policy](Policy) + /// and the [PolicyLearner](PolicyLearner) state to different files. + pub fn with_file_checkpointer(mut self, recorder: FR) -> Self + where + FR: FileRecorder + 'static, + FR: FileRecorder<::InnerBackend> + 'static, + { + let checkpoint_dir = self.directory.join("checkpoint"); + let checkpointer_policy = + FileCheckpointer::new(recorder.clone(), &checkpoint_dir, "policy"); + let checkpointer_learning = + FileCheckpointer::new(recorder.clone(), &checkpoint_dir, "learning-agent"); + + self.checkpointers = Some(( + AsyncCheckpointer::new(checkpointer_policy), + AsyncCheckpointer::new(checkpointer_learning), + )); + + self + } + + /// Enable the training summary report. + /// + /// The summary will be displayed after `.launch()`, when the renderer is dropped. + pub fn summary(mut self) -> Self { + self.summary = true; + self + } + + /// Launch the training with the specified [PolicyLearner](PolicyLearner) on the specified environment. + pub fn launch(mut self, learner_agent: RLC::LearningAgent) -> RLResult + where + RLC::PolicyObs: SliceAccess, + RLC::PolicyAction: SliceAccess, + { + if self.tracing_logger.is_some() + && let Err(e) = self.tracing_logger.as_ref().unwrap().install() + { + log::warn!("Failed to install the experiment logger: {e}"); + } + let renderer = self + .renderer + .unwrap_or_else(|| default_renderer(self.interrupter.clone(), self.checkpoint)); + + if !self.event_store.has_loggers() { + self.event_store + .register_logger(FileMetricLogger::new(self.directory.clone())); + } + + let event_store = Arc::new(EventStoreClient::new(self.event_store)); + let event_processor = AsyncProcessorTraining::new(RLEventProcessor::new( + self.metrics, + renderer, + event_store.clone(), + )); + + let checkpointer = self.checkpointers.map(|(policy, learning_agent)| { + RLCheckpointer::new(policy, learning_agent, self.checkpointer_strategy) + }); + + let summary = if self.summary { + Some(LearnerSummaryConfig { + directory: self.directory, + metrics: self.summary_metrics.into_iter().collect::>(), + }) + } else { + None + }; + + let components = RLComponents:: { + checkpoint: self.checkpoint, + checkpointer, + interrupter: self.interrupter, + event_processor, + event_store, + num_steps: self.num_steps, + grad_accumulation: self.grad_accumulation, + summary, + }; + + match self.learning_strategy { + RLStrategies::OffPolicyStrategy(config) => { + let strategy = OffPolicyStrategy::new(config); + strategy.train(learner_agent, components, self.env_initializer) + } + RLStrategies::Custom(strategy) => { + strategy.train(learner_agent, components, self.env_initializer) + } + } + } +} + +/// The result of reinforcement learning, containing the final policy along with the [renderer](MetricsRenderer). +pub struct RLResult

{ + /// The learned policy. + pub policy: P, + /// The renderer that can be used for follow up training and evaluation. + pub renderer: Box, +} + +/// Trait to fake variadic generics for train step metrics. +pub trait AgentMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: RLTraining) -> RLTraining; +} + +/// Trait to fake variadic generics for train step text metrics. +pub trait AgentTextMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: RLTraining) -> RLTraining; +} + +/// Trait to fake variadic generics for env step metrics. +pub trait TrainMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: RLTraining) -> RLTraining; +} + +/// Trait to fake variadic generics for env step text metrics. +pub trait TrainTextMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: RLTraining) -> RLTraining; +} + +/// Trait to fake variadic generics for episode metrics. +pub trait EpisodeMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: RLTraining) -> RLTraining; +} + +/// Trait to fake variadic generics for episode text metrics. +pub trait EpisodeTextMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: RLTraining) -> RLTraining; +} + +macro_rules! gen_tuple { + ($($M:ident),*) => { + impl<$($M,)* RLC: RLComponentsTypes + 'static> TrainTextMetricRegistration for ($($M,)*) + where + $(::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: RLTraining, + ) -> RLTraining { + let ($($M,)*) = self; + $(let builder = builder.text_metric_train($M.clone());)* + builder + } + } + + impl<$($M,)* RLC: RLComponentsTypes + 'static> TrainMetricRegistration for ($($M,)*) + where + $(::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + Numeric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: RLTraining, + ) -> RLTraining { + let ($($M,)*) = self; + $(let builder = builder.metric_train($M.clone());)* + builder + } + } + + impl<$($M,)* RLC: RLComponentsTypes + 'static> AgentTextMetricRegistration for ($($M,)*) + where + $(::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: RLTraining, + ) -> RLTraining { + let ($($M,)*) = self; + $(let builder = builder.text_metric_agent($M.clone());)* + builder + } + } + + impl<$($M,)* RLC: RLComponentsTypes + 'static> AgentMetricRegistration for ($($M,)*) + where + $(::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + Numeric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: RLTraining, + ) -> RLTraining { + let ($($M,)*) = self; + $(let builder = builder.metric_agent($M.clone());)* + builder + } + } + + impl<$($M,)* RLC: RLComponentsTypes + 'static> EpisodeTextMetricRegistration for ($($M,)*) + where + $(EpisodeSummary: Adaptor<$M::Input> + 'static,)* + $($M: Metric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: RLTraining, + ) -> RLTraining { + let ($($M,)*) = self; + $(let builder = builder.text_metric_episode($M.clone());)* + builder + } + } + + impl<$($M,)* RLC: RLComponentsTypes + 'static> EpisodeMetricRegistration for ($($M,)*) + where + $(EpisodeSummary: Adaptor<$M::Input> + 'static,)* + $($M: Metric + Numeric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: RLTraining, + ) -> RLTraining { + let ($($M,)*) = self; + $(let builder = builder.metric_episode($M.clone());)* + builder + } + } + }; +} + +gen_tuple!(M1); +gen_tuple!(M1, M2); +gen_tuple!(M1, M2, M3); +gen_tuple!(M1, M2, M3, M4); +gen_tuple!(M1, M2, M3, M4, M5); +gen_tuple!(M1, M2, M3, M4, M5, M6); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/strategy.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/strategy.rs new file mode 100644 index 0000000..dc866c0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/rl/strategy.rs @@ -0,0 +1,99 @@ +use std::sync::Arc; + +use crate::{ + Interrupter, LearnerSummaryConfig, OffPolicyConfig, RLCheckpointer, RLComponentsTypes, RLEvent, + RLEventProcessorType, RLResult, + metric::{processor::EventProcessorTraining, store::EventStoreClient}, +}; + +/// Struct to minimise parameters passed to [RLStrategy::train]. +pub struct RLComponents { + /// The total number of environment steps. + pub num_steps: usize, + /// The step number from which to continue the training. + pub checkpoint: Option, + /// A checkpointer used to load and save learning checkpoints. + pub checkpointer: Option>, + /// Enables gradients accumulation. + pub grad_accumulation: Option, + /// An [Interupter](Interrupter) that allows aborting the training/evaluation process early. + pub interrupter: Interrupter, + /// An [EventProcessor](crate::EventProcessorTraining) that processes events happening during training and evaluation. + pub event_processor: RLEventProcessorType, + /// A reference to an [EventStoreClient](EventStoreClient). + pub event_store: Arc, + /// Config for creating a summary of the learning + pub summary: Option, +} + +/// The strategy for reinforcement learning. +#[derive(Clone)] +pub enum RLStrategies { + /// Training on one device + OffPolicyStrategy(OffPolicyConfig), + /// Training using a custom learning strategy + Custom(CustomRLStrategy), +} + +/// A reference to an implementation of [RLStrategy]. +pub type CustomRLStrategy = Arc>; + +/// Provides the `fit` function for any learning strategy +pub trait RLStrategy { + /// Train the learner agent with this strategy. + fn train( + &self, + mut learner_agent: RLC::LearningAgent, + mut training_components: RLComponents, + env_init: RLC::EnvInit, + ) -> RLResult { + let starting_epoch = match training_components.checkpoint { + Some(checkpoint) => { + if let Some(checkpointer) = &mut training_components.checkpointer { + learner_agent = checkpointer.load_checkpoint( + learner_agent, + &Default::default(), + checkpoint, + ); + } + checkpoint + 1 + } + None => 1, + }; + + let summary_config = training_components.summary.clone(); + + // Event processor start training + training_components + .event_processor + .process_train(RLEvent::Start); + + // Training loop + let (policy, mut event_processor) = self.train_loop( + training_components, + &mut learner_agent, + starting_epoch, + env_init, + ); + + let summary = summary_config.and_then(|summary| summary.init().ok()); + + // Signal training end. For the TUI renderer, this handles the exit & return to main screen. + // TODO: summary makes sense for RL? + event_processor.process_train(RLEvent::End(summary)); + + // let model = model.valid(); + let renderer = event_processor.renderer(); + + RLResult { policy, renderer } + } + + /// Training loop for this strategy + fn train_loop( + &self, + training_components: RLComponents, + learner_agent: &mut RLC::LearningAgent, + starting_epoch: usize, + env_init: RLC::EnvInit, + ) -> (RLC::Policy, RLEventProcessorType); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/sequence.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/sequence.rs new file mode 100644 index 0000000..2717fd8 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/sequence.rs @@ -0,0 +1,131 @@ +use crate::metric::{AccuracyInput, PerplexityInput, TopKAccuracyInput}; +use crate::metric::{Adaptor, CerInput, LossInput, WerInput, processor::ItemLazy}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{Int, Tensor, Transaction}; +use burn_ndarray::NdArray; + +/// Sequence prediction output adapted for multiple metrics. +/// +/// Supported metrics: +/// - Accuracy +/// - TopKAccuracy +/// - Perplexity +/// - Loss +/// - CER +/// - WER +#[derive(new)] +pub struct SequenceOutput { + /// The loss. + pub loss: Tensor, + + /// Raw logits. Shape: `[batch_size, seq_len, vocab_size]` + pub logits: Tensor, + + /// Optional predicted token indices. Shape: `[batch_size, seq_length]`. + /// If not provided, predictions default to argmax of `logits` along the last dimension. + pub predictions: Option>, + + /// The target token indices. Shape: `[batch_size, seq_length]` + pub targets: Tensor, +} + +impl SequenceOutput { + fn predicted_tokens(&self) -> Tensor { + match &self.predictions { + Some(preds) => preds.clone(), + None => self.logits.clone().argmax(2).squeeze_dim::<2>(2), + } + } + + fn flat_logits(&self) -> Tensor { + let [batch_size, seq_len, vocab_size] = self.logits.dims(); + self.logits + .clone() + .reshape([batch_size * seq_len, vocab_size]) + } + + fn flat_targets(&self) -> Tensor { + let [batch_size, seq_len] = self.targets.dims(); + self.targets.clone().reshape([batch_size * seq_len]) + } +} + +impl ItemLazy for SequenceOutput { + type ItemSync = SequenceOutput; + + fn sync(self) -> Self::ItemSync { + let device = &Default::default(); + + match self.predictions { + Some(preds) => { + let [logits, loss, targets, predictions] = Transaction::default() + .register(self.logits) + .register(self.loss) + .register(self.targets) + .register(preds) + .execute() + .try_into() + .expect("Correct amount of tensor data"); + + SequenceOutput { + logits: Tensor::from_data(logits, device), + loss: Tensor::from_data(loss, device), + targets: Tensor::from_data(targets, device), + predictions: Some(Tensor::from_data(predictions, device)), + } + } + None => { + let [logits, loss, targets] = Transaction::default() + .register(self.logits) + .register(self.loss) + .register(self.targets) + .execute() + .try_into() + .expect("Correct amount of tensor data"); + + SequenceOutput { + logits: Tensor::from_data(logits, device), + loss: Tensor::from_data(loss, device), + targets: Tensor::from_data(targets, device), + predictions: None, + } + } + } + } +} + +impl Adaptor> for SequenceOutput { + fn adapt(&self) -> LossInput { + LossInput::new(self.loss.clone()) + } +} + +impl Adaptor> for SequenceOutput { + fn adapt(&self) -> CerInput { + CerInput::new(self.predicted_tokens(), self.targets.clone()) + } +} + +impl Adaptor> for SequenceOutput { + fn adapt(&self) -> WerInput { + WerInput::new(self.predicted_tokens(), self.targets.clone()) + } +} + +impl Adaptor> for SequenceOutput { + fn adapt(&self) -> AccuracyInput { + AccuracyInput::new(self.flat_logits(), self.flat_targets()) + } +} + +impl Adaptor> for SequenceOutput { + fn adapt(&self) -> TopKAccuracyInput { + TopKAccuracyInput::new(self.flat_logits(), self.flat_targets()) + } +} + +impl Adaptor> for SequenceOutput { + fn adapt(&self) -> PerplexityInput { + PerplexityInput::new(self.flat_logits(), self.flat_targets()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/summary.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/summary.rs new file mode 100644 index 0000000..478fbb0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/summary.rs @@ -0,0 +1,475 @@ +use core::cmp::Ordering; +use std::{ + collections::{HashMap, hash_map::Entry}, + fmt::Display, + path::{Path, PathBuf}, +}; + +use crate::{ + logger::FileMetricLogger, + metric::store::{Aggregate, EventStore, LogEventStore, Split}, +}; + +/// Contains the metric value at a given time. +#[derive(Debug)] +pub struct MetricEntry { + /// The step at which the metric was recorded (i.e., epoch). + pub step: usize, + /// The metric value. + pub value: f64, +} + +/// Contains the summary of recorded values for a given metric. +#[derive(Debug)] +pub struct MetricSummary { + /// The metric name. + pub name: String, + /// The metric entries. + pub entries: Vec, +} + +impl MetricSummary { + fn collect( + event_store: &mut E, + metric: &str, + split: &Split, + num_epochs: usize, + ) -> Option { + let entries = (1..=num_epochs) + .filter_map(|epoch| { + event_store + .find_metric(metric, epoch, Aggregate::Mean, split) + .map(|value| MetricEntry { step: epoch, value }) + }) + .collect::>(); + + if entries.is_empty() { + None + } else { + Some(Self { + name: metric.to_string(), + entries, + }) + } + } +} + +/// Contains the summary of recorded metrics for the training and validation steps. +pub struct SummaryMetrics { + /// Training metrics summary. + pub train: Vec, + /// Validation metrics summary. + pub valid: Vec, + /// Test metrics summary per test split tag. + /// + /// Each key corresponds to a `Split::Test(Some(tag))`. + /// The empty string represents `Split::Test(None)`. + pub test: HashMap>, +} + +/// Detailed training summary. +pub struct LearnerSummary { + /// The number of epochs completed. + pub epochs: usize, + /// The summary of recorded metrics during training. + pub metrics: SummaryMetrics, + /// The model name (only recorded within the learner). + pub(crate) model: Option, +} + +impl LearnerSummary { + /// Creates a new learner summary for the specified metrics. + /// + /// # Arguments + /// + /// * `directory` - The directory containing the training artifacts (checkpoints and logs). + /// * `metrics` - The list of metrics to collect for the summary. + pub fn new>(directory: impl AsRef, metrics: &[S]) -> Result { + let directory = directory.as_ref(); + if !directory.exists() { + return Err(format!( + "Artifact directory does not exist at: {}", + directory.display() + )); + } + + let mut event_store = LogEventStore::default(); + let train_split = Split::Train; + let valid_split = Split::Valid; + + let logger = FileMetricLogger::new(directory); + let test_split_root = logger.split_dir(&Split::Test(None)); + if !logger.split_exists(&train_split) + && !logger.split_exists(&valid_split) + && test_split_root.is_none() + { + return Err(format!( + "No training, validation or test artifacts found at: {}", + directory.display() + )); + } + + // Number of recorded epochs + let epochs = logger.epochs(); + + event_store.register_logger(logger); + + let train_summary = metrics + .iter() + .filter_map(|metric| { + MetricSummary::collect(&mut event_store, metric.as_ref(), &train_split, epochs) + }) + .collect::>(); + + let valid_summary = metrics + .iter() + .filter_map(|metric| { + MetricSummary::collect(&mut event_store, metric.as_ref(), &valid_split, epochs) + }) + .collect::>(); + + let test_summary = match test_split_root { + Some(root) => collect_test_split_metrics(root, metrics, &mut event_store, epochs), + None => Default::default(), + }; + + Ok(Self { + epochs, + metrics: SummaryMetrics { + train: train_summary, + valid: valid_summary, + test: test_summary, + }, + model: None, + }) + } + + pub(crate) fn with_model(mut self, name: String) -> Self { + self.model = Some(name); + self + } + + /// Merges another summary into this one, combining all metric entries. + pub(crate) fn merge(mut self, other: LearnerSummary) -> Self { + fn merge_metrics( + base: Vec, + incoming: Vec, + ) -> Vec { + let mut map: HashMap = + base.into_iter().map(|m| (m.name.clone(), m)).collect(); + + for metric in incoming { + match map.entry(metric.name.clone()) { + Entry::Occupied(mut entry) => { + entry.get_mut().entries.extend(metric.entries); + } + Entry::Vacant(entry) => { + entry.insert(metric); + } + } + } + map.into_values().collect() + } + + self.metrics.train = merge_metrics(self.metrics.train, other.metrics.train); + self.metrics.valid = merge_metrics(self.metrics.valid, other.metrics.valid); + + for (tag, metrics) in other.metrics.test { + match self.metrics.test.entry(tag) { + Entry::Occupied(mut entry) => { + let current = std::mem::take(entry.get_mut()); + let merged = merge_metrics(current, metrics); + *entry.get_mut() = merged; + } + Entry::Vacant(entry) => { + entry.insert(metrics); + } + } + } + + if self.model != other.model { + self.model = None; + } + + self + } +} + +fn collect_test_split_metrics, S: AsRef>( + root: P, + metrics: &[S], + event_store: &mut LogEventStore, + epochs: usize, +) -> HashMap> { + // Collect immediate child directories + let dirs = match std::fs::read_dir(root) { + Ok(entries) => entries + .filter_map(|entry| { + let entry = entry.ok()?; + let file_type = entry.file_type().ok()?; + if file_type.is_dir() { + Some(entry.file_name().to_string_lossy().to_string()) + } else { + None + } + }) + .collect::>(), + Err(_) => Vec::new(), + }; + + let mut map = HashMap::new(); + + if dirs.is_empty() { + return map; + } + + // Detect if all directories are epoch directories + let all_epochs = dirs.iter().all(FileMetricLogger::is_epoch_dir); + + if all_epochs { + // Single untagged test split + let split = Split::Test(None); + + let summaries = metrics + .iter() + .filter_map(|metric| { + MetricSummary::collect(event_store, metric.as_ref(), &split, epochs) + }) + .collect::>(); + + // Untagged marked with empty string + map.insert("".to_string(), summaries); + } else { + // Tagged splits + for tag in dirs { + let split = Split::Test(Some(tag.clone().into())); + + let summaries = metrics + .iter() + .filter_map(|metric| { + MetricSummary::collect(event_store, metric.as_ref(), &split, epochs) + }) + .collect::>(); + + map.insert(tag, summaries); + } + } + + map +} + +impl Display for LearnerSummary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Compute the max length for each column + let mut max_split_len = 5; // "Train" + let mut max_metric_len = "Metric".len(); + for metric in self.metrics.train.iter() { + max_metric_len = max_metric_len.max(metric.name.len()); + } + for metric in self.metrics.valid.iter() { + max_metric_len = max_metric_len.max(metric.name.len()); + } + for (tag, metrics) in self.metrics.test.iter() { + let split_name = if tag.is_empty() { + "Test".to_string() + } else { + format!("Test ({tag})") + }; + + max_split_len = max_split_len.max(split_name.len()); + + for metric in metrics { + max_metric_len = max_metric_len.max(metric.name.len()); + } + } + + // Summary header + writeln!( + f, + "{:=>width_symbol$} Learner Summary {:=>width_symbol$}", + "", + "", + width_symbol = 24, + )?; + + if let Some(model) = &self.model { + writeln!(f, "Model:\n{model}")?; + } + writeln!(f, "Total Epochs: {epochs}\n\n", epochs = self.epochs)?; + + // Metrics table header + writeln!( + f, + "| {:width_split$}--|{:->width_metric$}--|----------|----------|----------|----------|", + "Split", + "Metric", + "", + "", + width_split = max_split_len, + width_metric = max_metric_len, + )?; + + // Table entries + fn cmp_f64(a: &f64, b: &f64) -> Ordering { + match (a.is_nan(), b.is_nan()) { + (true, true) => Ordering::Equal, + (true, false) => Ordering::Greater, + (false, true) => Ordering::Less, + _ => a.partial_cmp(b).unwrap(), + } + } + + fn fmt_val(val: f64) -> String { + if val < 1e-2 { + // Use scientific notation for small values which would otherwise be truncated + format!("{val:<9.3e}") + } else { + format!("{val:<9.3}") + } + } + + let mut write_metrics_summary = + |metrics: &[MetricSummary], split: String| -> std::fmt::Result { + for metric in metrics.iter() { + if metric.entries.is_empty() { + continue; // skip metrics with no recorded values + } + + // Compute the min & max for each metric + let metric_min = metric + .entries + .iter() + .min_by(|a, b| cmp_f64(&a.value, &b.value)) + .unwrap(); + let metric_max = metric + .entries + .iter() + .max_by(|a, b| cmp_f64(&a.value, &b.value)) + .unwrap(); + + writeln!( + f, + "| {:, +} + +impl LearnerSummaryConfig { + /// Create the learning summary. + pub fn init(&self) -> Result { + LearnerSummary::new(&self.directory, &self.metrics[..]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic = "Summary artifacts should exist"] + fn test_artifact_dir_should_exist() { + let dir = "/tmp/learner-summary-not-found"; + let _summary = LearnerSummary::new(dir, &["Loss"]).expect("Summary artifacts should exist"); + } + + #[test] + #[should_panic = "Summary artifacts should exist"] + fn test_train_valid_artifacts_should_exist() { + let dir = "/tmp/test-learner-summary-empty"; + std::fs::create_dir_all(dir).ok(); + let _summary = LearnerSummary::new(dir, &["Loss"]).expect("Summary artifacts should exist"); + } + + #[test] + fn test_summary_should_be_empty() { + let dir = Path::new("/tmp/test-learner-summary-empty-metrics"); + std::fs::create_dir_all(dir).unwrap(); + std::fs::create_dir_all(dir.join("train/epoch-1")).unwrap(); + std::fs::create_dir_all(dir.join("valid/epoch-1")).unwrap(); + let summary = LearnerSummary::new(dir.to_str().unwrap(), &["Loss"]) + .expect("Summary artifacts should exist"); + + assert_eq!(summary.epochs, 1); + + assert_eq!(summary.metrics.train.len(), 0); + assert_eq!(summary.metrics.valid.len(), 0); + + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn test_summary_should_be_collected() { + let dir = Path::new("/tmp/test-learner-summary"); + let train_dir = dir.join("train/epoch-1"); + let valid_dir = dir.join("valid/epoch-1"); + std::fs::create_dir_all(dir).unwrap(); + std::fs::create_dir_all(&train_dir).unwrap(); + std::fs::create_dir_all(&valid_dir).unwrap(); + + std::fs::write(train_dir.join("Loss.log"), "1.0\n2.0").expect("Unable to write file"); + std::fs::write(valid_dir.join("Loss.log"), "1.0").expect("Unable to write file"); + + let summary = LearnerSummary::new(dir.to_str().unwrap(), &["Loss"]) + .expect("Summary artifacts should exist"); + + assert_eq!(summary.epochs, 1); + + // Only Loss metric + assert_eq!(summary.metrics.train.len(), 1); + assert_eq!(summary.metrics.valid.len(), 1); + + // Aggregated train metric entries for 1 epoch + let train_metric = &summary.metrics.train[0]; + assert_eq!(train_metric.name, "Loss"); + assert_eq!(train_metric.entries.len(), 1); + let entry = &train_metric.entries[0]; + assert_eq!(entry.step, 1); // epoch = 1 + assert_eq!(entry.value, 1.5); // (1 + 2) / 2 + + // Aggregated valid metric entries for 1 epoch + let valid_metric = &summary.metrics.valid[0]; + assert_eq!(valid_metric.name, "Loss"); + assert_eq!(valid_metric.entries.len(), 1); + let entry = &valid_metric.entries[0]; + assert_eq!(entry.step, 1); // epoch = 1 + assert_eq!(entry.value, 1.0); + + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/mod.rs new file mode 100644 index 0000000..56b2f0f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/mod.rs @@ -0,0 +1,7 @@ +mod paradigm; +mod step; +mod strategies; + +pub use paradigm::*; +pub use step::*; +pub use strategies::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/paradigm.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/paradigm.rs new file mode 100644 index 0000000..beb3370 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/paradigm.rs @@ -0,0 +1,488 @@ +use crate::checkpoint::{ + AsyncCheckpointer, CheckpointingStrategy, ComposedCheckpointingStrategy, FileCheckpointer, + KeepLastNCheckpoints, MetricCheckpointingStrategy, +}; +use crate::components::{InferenceModelOutput, TrainingModelOutput}; +use crate::learner::EarlyStoppingStrategy; +use crate::learner::base::Interrupter; +use crate::logger::{FileMetricLogger, MetricLogger}; +use crate::metric::processor::{ + AsyncProcessorTraining, FullEventProcessorTraining, ItemLazy, MetricsTraining, +}; +use crate::metric::store::{Aggregate, Direction, EventStoreClient, LogEventStore, Split}; +use crate::metric::{Adaptor, LossMetric, Metric, Numeric}; +use crate::multi::MultiDeviceLearningStrategy; +use crate::renderer::{MetricsRenderer, default_renderer}; +use crate::single::SingleDeviceTrainingStrategy; +use crate::{ + ApplicationLoggerInstaller, EarlyStoppingStrategyRef, FileApplicationLoggerInstaller, + InferenceBackend, InferenceModel, InferenceModelInput, InferenceStep, LearnerEvent, + LearnerModelRecord, LearnerOptimizerRecord, LearnerSchedulerRecord, LearnerSummaryConfig, + LearningCheckpointer, LearningComponentsMarker, LearningComponentsTypes, LearningResult, + TrainStep, TrainingBackend, TrainingComponents, TrainingModelInput, TrainingStrategy, +}; +use crate::{Learner, SupervisedLearningStrategy}; +use burn_core::data::dataloader::DataLoader; +use burn_core::module::{AutodiffModule, Module}; +use burn_core::record::FileRecorder; +use burn_core::tensor::backend::AutodiffBackend; +use burn_optim::Optimizer; +use burn_optim::lr_scheduler::LrScheduler; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +/// A reference to the training split [DataLoader](DataLoader). +pub type TrainLoader = Arc, TrainingModelInput>>; +/// A reference to the validation split [DataLoader](DataLoader). +pub type ValidLoader = Arc, InferenceModelInput>>; +/// The event processor type for supervised learning. +pub type SupervisedTrainingEventProcessor = AsyncProcessorTraining< + LearnerEvent>, + LearnerEvent>, +>; + +/// Structure to configure and launch supervised learning trainings. +pub struct SupervisedTraining +where + LC: LearningComponentsTypes, +{ + // Not that complex. Extracting into another type would only make it more confusing. + #[allow(clippy::type_complexity)] + checkpointers: Option<( + AsyncCheckpointer, TrainingBackend>, + AsyncCheckpointer, TrainingBackend>, + AsyncCheckpointer, TrainingBackend>, + )>, + num_epochs: usize, + checkpoint: Option, + directory: PathBuf, + grad_accumulation: Option, + renderer: Option>, + metrics: MetricsTraining, InferenceModelOutput>, + event_store: LogEventStore, + interrupter: Interrupter, + tracing_logger: Option>, + checkpointer_strategy: Box, + early_stopping: Option, + training_strategy: Option>, + dataloader_train: TrainLoader, + dataloader_valid: ValidLoader, + // Use BTreeSet instead of HashSet for consistent (alphabetical) iteration order + summary_metrics: BTreeSet, + summary: bool, +} + +impl SupervisedTraining> +where + B: AutodiffBackend, + LR: LrScheduler + 'static, + M: TrainStep + AutodiffModule + core::fmt::Display + 'static, + M::InnerModule: InferenceStep, + O: Optimizer + 'static, +{ + /// Creates a new runner for a supervised training. + /// + /// # Arguments + /// + /// * `directory` - The directory to save the checkpoints. + /// * `dataloader_train` - The dataloader for the training split. + /// * `dataloader_valid` - The dataloader for the validation split. + pub fn new( + directory: impl AsRef, + dataloader_train: Arc>, + dataloader_valid: Arc< + dyn DataLoader::Input>, + >, + ) -> Self { + let directory = directory.as_ref().to_path_buf(); + let experiment_log_file = directory.join("experiment.log"); + Self { + num_epochs: 1, + checkpoint: None, + checkpointers: None, + directory, + grad_accumulation: None, + metrics: MetricsTraining::default(), + event_store: LogEventStore::default(), + renderer: None, + interrupter: Interrupter::new(), + tracing_logger: Some(Box::new(FileApplicationLoggerInstaller::new( + experiment_log_file, + ))), + checkpointer_strategy: Box::new( + ComposedCheckpointingStrategy::builder() + .add(KeepLastNCheckpoints::new(2)) + .add(MetricCheckpointingStrategy::new( + &LossMetric::::new(), // default to valid loss + Aggregate::Mean, + Direction::Lowest, + Split::Valid, + )) + .build(), + ), + early_stopping: None, + training_strategy: None, + summary_metrics: BTreeSet::new(), + summary: false, + dataloader_train, + dataloader_valid, + } + } +} + +impl SupervisedTraining { + /// Replace the default training strategy (SingleDeviceTrainingStrategy) with the provided one. + /// + /// # Arguments + /// + /// * `training_strategy` - The training strategy. + pub fn with_training_strategy(mut self, training_strategy: TrainingStrategy) -> Self { + self.training_strategy = Some(training_strategy); + self + } + + /// Replace the default metric loggers with the provided ones. + /// + /// # Arguments + /// + /// * `logger` - The training logger. + pub fn with_metric_logger(mut self, logger: ML) -> Self + where + ML: MetricLogger + 'static, + { + self.event_store.register_logger(logger); + self + } + + /// Update the checkpointing_strategy. + pub fn with_checkpointing_strategy( + mut self, + strategy: CS, + ) -> Self { + self.checkpointer_strategy = Box::new(strategy); + self + } + + /// Replace the default CLI renderer with a custom one. + /// + /// # Arguments + /// + /// * `renderer` - The custom renderer. + pub fn renderer(mut self, renderer: MR) -> Self + where + MR: MetricsRenderer + 'static, + { + self.renderer = Some(Box::new(renderer)); + self + } + + /// Register all metrics as numeric for the training and validation set. + pub fn metrics>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register all metrics as text for the training and validation set. + pub fn metrics_text>(self, metrics: Me) -> Self { + metrics.register(self) + } + + /// Register a training metric. + pub fn metric_train(mut self, metric: Me) -> Self + where + as ItemLazy>::ItemSync: Adaptor, + { + self.metrics.register_train_metric(metric); + self + } + + /// Register a validation metric. + pub fn metric_valid(mut self, metric: Me) -> Self + where + as ItemLazy>::ItemSync: Adaptor, + { + self.metrics.register_valid_metric(metric); + self + } + + /// Enable gradients accumulation. + /// + /// # Notes + /// + /// When you enable gradients accumulation, the gradients object used by the optimizer will be + /// the sum of all gradients generated by each backward pass. It might be a good idea to + /// reduce the learning to compensate. + /// + /// The effect is similar to increasing the `batch size` and the `learning rate` by the `accumulation` + /// amount. + pub fn grads_accumulation(mut self, accumulation: usize) -> Self { + self.grad_accumulation = Some(accumulation); + self + } + + /// Register a [numeric](crate::metric::Numeric) training [metric](Metric). + pub fn metric_train_numeric(mut self, metric: Me) -> Self + where + Me: Metric + Numeric + 'static, + as ItemLazy>::ItemSync: Adaptor, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_train_metric_numeric(metric); + self + } + + /// Register a [numeric](crate::metric::Numeric) validation [metric](Metric). + pub fn metric_valid_numeric(mut self, metric: Me) -> Self + where + as ItemLazy>::ItemSync: Adaptor, + { + self.summary_metrics.insert(metric.name().to_string()); + self.metrics.register_valid_metric_numeric(metric); + self + } + + /// The number of epochs the training should last. + pub fn num_epochs(mut self, num_epochs: usize) -> Self { + self.num_epochs = num_epochs; + self + } + + /// The epoch from which the training must resume. + pub fn checkpoint(mut self, checkpoint: usize) -> Self { + self.checkpoint = Some(checkpoint); + self + } + + /// Provides a handle that can be used to interrupt training. + pub fn interrupter(&self) -> Interrupter { + self.interrupter.clone() + } + + /// Override the handle for stopping training with an externally provided handle + pub fn with_interrupter(mut self, interrupter: Interrupter) -> Self { + self.interrupter = interrupter; + self + } + + /// Register an [early stopping strategy](EarlyStoppingStrategy) to stop the training when the + /// conditions are meet. + pub fn early_stopping(mut self, strategy: Strategy) -> Self + where + Strategy: EarlyStoppingStrategy + Clone + Send + Sync + 'static, + { + self.early_stopping = Some(Box::new(strategy)); + self + } + + /// By default, Rust logs are captured and written into + /// `experiment.log`. If disabled, standard Rust log handling + /// will apply. + pub fn with_application_logger( + mut self, + logger: Option>, + ) -> Self { + self.tracing_logger = logger; + self + } + + /// Register a checkpointer that will save the [optimizer](Optimizer), the + /// [model](AutodiffModule) and the [scheduler](LrScheduler) to different files. + pub fn with_file_checkpointer(mut self, recorder: FR) -> Self + where + FR: FileRecorder<::Backend> + 'static, + FR: FileRecorder< + <::Backend as AutodiffBackend>::InnerBackend, + > + 'static, + { + let checkpoint_dir = self.directory.join("checkpoint"); + let checkpointer_model = FileCheckpointer::new(recorder.clone(), &checkpoint_dir, "model"); + let checkpointer_optimizer = + FileCheckpointer::new(recorder.clone(), &checkpoint_dir, "optim"); + let checkpointer_scheduler: FileCheckpointer = + FileCheckpointer::new(recorder, &checkpoint_dir, "scheduler"); + + self.checkpointers = Some(( + AsyncCheckpointer::new(checkpointer_model), + AsyncCheckpointer::new(checkpointer_optimizer), + AsyncCheckpointer::new(checkpointer_scheduler), + )); + + self + } + + /// Enable the training summary report. + /// + /// The summary will be displayed after `.fit()`, when the renderer is dropped. + pub fn summary(mut self) -> Self { + self.summary = true; + self + } +} + +impl SupervisedTraining { + /// Launch this training with the given [Learner](Learner). + pub fn launch(mut self, learner: Learner) -> LearningResult> { + if self.tracing_logger.is_some() + && let Err(e) = self.tracing_logger.as_ref().unwrap().install() + { + log::warn!("Failed to install the experiment logger: {e}"); + } + let renderer = self + .renderer + .unwrap_or_else(|| default_renderer(self.interrupter.clone(), self.checkpoint)); + + if !self.event_store.has_loggers() { + self.event_store + .register_logger(FileMetricLogger::new(self.directory.clone())); + } + + let event_store = Arc::new(EventStoreClient::new(self.event_store)); + let event_processor = AsyncProcessorTraining::new(FullEventProcessorTraining::new( + self.metrics, + renderer, + event_store.clone(), + )); + + let checkpointer = self.checkpointers.map(|(model, optim, scheduler)| { + LearningCheckpointer::new( + model.with_interrupter(self.interrupter.clone()), + optim.with_interrupter(self.interrupter.clone()), + scheduler.with_interrupter(self.interrupter.clone()), + self.checkpointer_strategy, + ) + }); + + let summary = if self.summary { + Some(LearnerSummaryConfig { + directory: self.directory, + metrics: self.summary_metrics.into_iter().collect::>(), + }) + } else { + None + }; + + let components = TrainingComponents { + checkpoint: self.checkpoint, + checkpointer, + interrupter: self.interrupter, + early_stopping: self.early_stopping, + event_processor, + event_store, + num_epochs: self.num_epochs, + grad_accumulation: self.grad_accumulation, + summary, + }; + + // Default to single device based on model + let training_strategy = self + .training_strategy + .unwrap_or(TrainingStrategy::SingleDevice( + learner.model.devices()[0].clone(), + )); + + match training_strategy { + TrainingStrategy::SingleDevice(device) => { + let single_device: SingleDeviceTrainingStrategy = + SingleDeviceTrainingStrategy::new(device); + single_device.train( + learner, + self.dataloader_train, + self.dataloader_valid, + components, + ) + } + TrainingStrategy::Custom(learning_paradigm) => learning_paradigm.train( + learner, + self.dataloader_train, + self.dataloader_valid, + components, + ), + TrainingStrategy::MultiDevice(devices, multi_device_optim) => { + let strategy: Box> = match devices.len() == 1 { + true => Box::new(SingleDeviceTrainingStrategy::new(devices[0].clone())), + false => Box::new(MultiDeviceLearningStrategy::new( + devices, + multi_device_optim, + )), + }; + strategy.train( + learner, + self.dataloader_train, + self.dataloader_valid, + components, + ) + } + #[cfg(feature = "ddp")] + TrainingStrategy::DistributedDataParallel { devices, config } => { + use crate::ddp::DdpTrainingStrategy; + + let ddp = DdpTrainingStrategy::new(devices.clone(), config.clone()); + ddp.train( + learner, + self.dataloader_train, + self.dataloader_valid, + components, + ) + } + } + } +} + +/// Trait to fake variadic generics. +pub trait MetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: SupervisedTraining) -> SupervisedTraining; +} + +/// Trait to fake variadic generics. +pub trait TextMetricRegistration: Sized { + /// Register the metrics. + fn register(self, builder: SupervisedTraining) -> SupervisedTraining; +} + +macro_rules! gen_tuple { + ($($M:ident),*) => { + impl<$($M,)* LC: LearningComponentsTypes> TextMetricRegistration for ($($M,)*) + where + $( as ItemLazy>::ItemSync: Adaptor<$M::Input>,)* + $( as ItemLazy>::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: SupervisedTraining, + ) -> SupervisedTraining { + let ($($M,)*) = self; + $(let builder = builder.metric_train($M.clone());)* + $(let builder = builder.metric_valid($M);)* + builder + } + } + + impl<$($M,)* LC: LearningComponentsTypes> MetricRegistration for ($($M,)*) + where + $( as ItemLazy>::ItemSync: Adaptor<$M::Input>,)* + $( as ItemLazy>::ItemSync: Adaptor<$M::Input>,)* + $($M: Metric + Numeric + 'static,)* + { + #[allow(non_snake_case)] + fn register( + self, + builder: SupervisedTraining, + ) -> SupervisedTraining { + let ($($M,)*) = self; + $(let builder = builder.metric_train_numeric($M.clone());)* + $(let builder = builder.metric_valid_numeric($M);)* + builder + } + } + }; +} + +gen_tuple!(M1); +gen_tuple!(M1, M2); +gen_tuple!(M1, M2, M3); +gen_tuple!(M1, M2, M3, M4); +gen_tuple!(M1, M2, M3, M4, M5); +gen_tuple!(M1, M2, M3, M4, M5, M6); diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/mod.rs new file mode 100644 index 0000000..066b064 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/mod.rs @@ -0,0 +1,2 @@ +/// The trainer module. +pub mod train; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/train.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/train.rs new file mode 100644 index 0000000..d6255a4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/step/train.rs @@ -0,0 +1,151 @@ +use crate::{LearningComponentsTypes, TrainingModel}; +use crate::{TrainOutput, TrainStep, TrainingBackend, TrainingModelInput, TrainingModelOutput}; +use burn_core::data::dataloader::DataLoaderIterator; +use burn_core::data::dataloader::Progress; +use burn_core::module::Module; +use burn_core::prelude::DeviceOps; +use burn_core::tensor::Device; +use burn_core::tensor::backend::DeviceId; +use std::sync::mpsc::{Receiver, Sender}; +use std::thread::spawn; + +/// Multi devices train step. +pub struct MultiDevicesTrainStep { + workers: Vec>, + receiver: Receiver>>, +} + +struct Message { + item: TI, + model: M, +} + +struct Worker { + // Not that complex. Extracting into another type would only make it more confusing. + #[allow(clippy::type_complexity)] + sender_input: Sender, TrainingModelInput>>, + device: Device>, +} + +impl Worker { + fn register(&self, item: TrainingModelInput, model: &TrainingModel) { + let message = Message { + item, + model: model.clone(), + }; + self.sender_input.send(message).unwrap(); + } + + // Not that complex. Extracting into another type would only make it more confusing. + #[allow(clippy::type_complexity)] + fn start( + &self, + sender_output: Sender>>, + receiver_input: Receiver, TrainingModelInput>>, + ) { + let device = self.device.clone(); + + spawn(move || { + loop { + match receiver_input.recv() { + Ok(item) => { + let model = item.model.fork(&device); + let output = model.step(item.item); + let item = MultiTrainOutput { + output, + device: device.to_id(), + }; + + sender_output.send(item).unwrap(); + } + Err(_err) => { + log::info!("Closing thread on device {device:?}"); + break; + } + } + } + }); + } +} + +/// Multiple output items. +pub struct MultiTrainOutput { + /// The training output. + pub output: TrainOutput, + /// The device on which the computing happened. + pub device: DeviceId, +} + +impl MultiDevicesTrainStep { + /// Create a new multi devices train step. + /// + /// # Arguments + /// + /// * `devices` - Devices. + /// + /// # Returns + /// + /// MultiDevicesTrainStep instance. + pub fn new(devices: &[Device>]) -> Self { + let (sender_output, receiver_output) = std::sync::mpsc::channel(); + let workers = devices + .iter() + .map(|device| { + let (sender_input, receiver_input) = std::sync::mpsc::channel(); + let worker = Worker { + sender_input, + device: device.clone(), + }; + + worker.start(sender_output.clone(), receiver_input); + worker + }) + .collect(); + + Self { + workers, + receiver: receiver_output, + } + } + + /// Collect outputs from workers for one step. + /// + /// # Arguments + /// + /// * `model` - Model. + /// * `dataloaders` - The data loader for each worker. + /// + /// # Returns + /// + /// Outputs. + pub fn step<'a>( + &self, + dataloaders: &mut [Box> + 'a>], + model: &TrainingModel, + ) -> (Vec>>, Progress) { + let mut num_send = 0; + + let mut items_total = 0; + let mut items_processed = 0; + + for (i, worker) in self.workers.iter().enumerate() { + let dataloader = &mut dataloaders[i]; + if let Some(item) = dataloader.next() { + worker.register(item, model); + num_send += 1; + let progress = dataloader.progress(); + items_total += progress.items_total; + items_processed += progress.items_processed; + } + } + + let mut outputs = Vec::with_capacity(num_send); + + for _ in 0..num_send { + let output = self.receiver.recv().unwrap(); + outputs.push(output); + } + + (outputs, Progress::new(items_processed, items_total)) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/base.rs new file mode 100644 index 0000000..ea7d833 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/base.rs @@ -0,0 +1,154 @@ +use std::sync::Arc; + +#[cfg(feature = "ddp")] +use burn_collective::CollectiveConfig; +use burn_core::{module::AutodiffModule, prelude::Backend}; + +use crate::{ + EarlyStoppingStrategyRef, InferenceModel, Interrupter, Learner, LearnerSummaryConfig, + LearningCheckpointer, LearningResult, SupervisedTrainingEventProcessor, TrainLoader, + TrainingModel, ValidLoader, + components::LearningComponentsTypes, + metric::{ + processor::{EventProcessorTraining, LearnerEvent}, + store::EventStoreClient, + }, +}; + +type LearnerDevice = <::Backend as Backend>::Device; + +/// A reference to an implementation of SupervisedLearningStrategy. +pub type CustomLearningStrategy = Arc>; + +#[derive(Clone, Copy, Debug)] +/// Determine how the optimization is performed when training with multiple devices. +pub enum MultiDeviceOptim { + /// The optimization is done on an elected device. + OptimMainDevice, + /// The optimization is sharded across all devices. + OptimSharded, +} + +/// How should the learner run the learning for the model +#[derive(Clone)] +pub enum TrainingStrategy { + /// Training on one device + SingleDevice(LearnerDevice), + /// Performs data-parallel distributed training where the optimization is + /// done on an elected master device. + MultiDevice(Vec>, MultiDeviceOptim), + /// Training using a custom learning strategy + Custom(CustomLearningStrategy), + /// Training with input distributed across devices, each device has its own copy of the model. + /// Collective ops are used to sync the gradients after each pass. + #[cfg(feature = "ddp")] + DistributedDataParallel { + /// Devices on this node for the DDP + devices: Vec>, + + /// The configuration for collective operations + /// num_devices is ignored + config: CollectiveConfig, + }, +} + +/// Constructor for a distributed data parallel (DDP) learning strategy +#[cfg(feature = "ddp")] +pub fn ddp( + devices: Vec>, + config: CollectiveConfig, +) -> TrainingStrategy { + TrainingStrategy::DistributedDataParallel { devices, config } +} + +impl Default for TrainingStrategy { + fn default() -> Self { + Self::SingleDevice(Default::default()) + } +} + +/// Struct to minimise parameters passed to [SupervisedLearningStrategy::train]. +/// These components are used during training. +pub struct TrainingComponents { + /// The total number of epochs + pub num_epochs: usize, + /// The epoch number from which to continue the training. + pub checkpoint: Option, + /// A checkpointer used to load and save learner checkpoints. + pub checkpointer: Option>, + /// Enables gradients accumulation. + pub grad_accumulation: Option, + /// An [Interupter](Interrupter) that allows aborting the training/evaluation process early. + pub interrupter: Interrupter, + /// Cloneable reference to an early stopping strategy. + pub early_stopping: Option, + /// An [EventProcessor](crate::EventProcessorTraining) that processes events happening during training and validation. + pub event_processor: SupervisedTrainingEventProcessor, + /// A reference to an [EventStoreClient](EventStoreClient). + pub event_store: Arc, + /// Config for creating a summary of the learning + pub summary: Option, +} + +/// Provides the `fit` function for any learning strategy +pub trait SupervisedLearningStrategy { + /// Train the learner's model with this strategy. + fn train( + &self, + mut learner: Learner, + dataloader_train: TrainLoader, + dataloader_valid: ValidLoader, + mut training_components: TrainingComponents, + ) -> LearningResult> { + let starting_epoch = match training_components.checkpoint { + Some(checkpoint) => { + if let Some(checkpointer) = &mut training_components.checkpointer { + learner = + checkpointer.load_checkpoint(learner, &Default::default(), checkpoint); + } + checkpoint + 1 + } + None => 1, + }; + + let summary_config = training_components.summary.clone(); + + // Event processor start training + training_components + .event_processor + .process_train(LearnerEvent::Start); + // Training loop + let (model, mut event_processor) = self.fit( + training_components, + learner, + dataloader_train, + dataloader_valid, + starting_epoch, + ); + + let summary = summary_config.and_then(|summary| { + summary + .init() + .map(|summary| summary.with_model(model.to_string())) + .ok() + }); + + // Signal training end. For the TUI renderer, this handles the exit & return to main screen. + event_processor.process_train(LearnerEvent::End(summary)); + + let model = model.valid(); + let renderer = event_processor.renderer(); + + LearningResult::> { model, renderer } + } + + /// Training loop for this strategy + fn fit( + &self, + training_components: TrainingComponents, + learner: Learner, + dataloader_train: TrainLoader, + dataloader_valid: ValidLoader, + starting_epoch: usize, + ) -> (TrainingModel, SupervisedTrainingEventProcessor); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/README.md b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/README.md new file mode 100644 index 0000000..b38c9c7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/README.md @@ -0,0 +1,17 @@ +## DDP +Distributed Data Parallel + +The DDP is a learning strategy that trains a replica of the model on each device. + +The DDP launches threads for each local device. Each thread on each node will run the model. +After the forward and backward passes, the gradients are synced between all peers on all nodes +with an `all-reduce` operation. + +While the DDP launches threads for each local device, it is the user's responsibility to launch the +DDP on each node, and assure the collective configuration matches. + +## Main device vs secondary devices + +The main device is responsible for validation, as well as event processing, which is used in the UI. + +The first device is chosen as the main device. diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/epoch.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/epoch.rs new file mode 100644 index 0000000..588dde5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/epoch.rs @@ -0,0 +1,234 @@ +use burn_collective::{PeerId, ReduceOperation}; +use burn_core::data::dataloader::Progress; +use burn_core::module::AutodiffModule; +use burn_core::tensor::backend::AutodiffBackend; +use burn_optim::GradientsAccumulator; +use burn_optim::GradientsParams; +use std::marker::PhantomData; +use std::sync::mpsc::{Receiver, SyncSender}; +use std::sync::{Arc, Mutex}; + +use crate::SupervisedTrainingEventProcessor; +use crate::learner::base::Interrupter; +use crate::metric::processor::{EventProcessorTraining, LearnerEvent, TrainingItem}; +use crate::{ + InferenceStep, Learner, LearningComponentsTypes, TrainLoader, TrainingBackend, ValidLoader, +}; + +/// A validation epoch. +#[derive(new)] +pub struct DdpValidEpoch { + dataloader: ValidLoader, +} + +/// A training epoch. +#[derive(new)] +pub struct DdpTrainEpoch { + dataloader: TrainLoader, + grad_accumulation: Option, +} + +impl DdpValidEpoch { + /// Runs the validation epoch. + /// + /// # Arguments + /// + /// * `model` - The model to validate. + /// * `processor` - The event processor to use. + pub fn run( + &self, + model: &::TrainingModel, + global_progress: &Progress, + processor: &mut SupervisedTrainingEventProcessor, + interrupter: &Interrupter, + ) { + let epoch = global_progress.items_processed; + log::info!("Executing validation step for epoch {}", epoch); + let model = model.valid(); + + let mut iterator = self.dataloader.iter(); + let mut iteration = 0; + + while let Some(item) = iterator.next() { + let progress = iterator.progress(); + iteration += 1; + + let item = model.step(item); + let item = TrainingItem::new( + item, + progress, + global_progress.clone(), + Some(iteration), + None, + ); + + processor.process_valid(LearnerEvent::ProcessedItem(item)); + + if interrupter.should_stop() { + log::info!("Training interrupted."); + break; + } + } + processor.process_valid(LearnerEvent::EndEpoch(epoch)); + } +} + +impl DdpTrainEpoch { + /// Runs the training epoch. + /// + /// # Arguments + /// + /// * `model` - The model to train. + /// * `optim` - The optimizer to use. + /// * `scheduler` - The learning rate scheduler to use. + /// * `processor` - The event processor to use. + /// + /// # Returns + /// + /// The trained model and the optimizer. + #[allow(clippy::too_many_arguments)] + pub fn run( + &self, + learner: &mut Learner, + global_progress: &Progress, + processor: Arc>>, + interrupter: &Interrupter, + peer_id: PeerId, + peer_count: usize, + is_main: bool, + ) { + let epoch = global_progress.items_processed; + log::info!("Executing training step for epoch {}", epoch,); + + let mut iterator = self.dataloader.iter(); + let mut iteration = 0; + let mut accumulator = GradientsAccumulator::new(); + let mut accumulation_current = 0; + + let grads_syncer = GradsSyncer::< + TrainingBackend, + ::TrainingModel, + >::new(false, peer_id); + + while let Some(item) = iterator.next() { + for _ in 0..peer_count { + iteration += 1; + learner.lr_step(); + } + log::info!("Iteration {iteration}"); + + let mut progress = iterator.progress(); + progress.items_processed *= peer_count; + progress.items_total *= peer_count; + + let item = learner.train_step(item); + + match self.grad_accumulation { + Some(accumulation) => { + accumulator.accumulate(&learner.model(), item.grads); + accumulation_current += 1; + + if accumulation <= accumulation_current { + let grads = accumulator.grads(); + + // With double buffering, these are the previous iteration's gradients + let grads = grads_syncer.sync(grads); + if let Some(grads) = grads { + learner.optimizer_step(grads); + } + + accumulation_current = 0; + } + } + None => { + // With double buffering, these are the previous iteration's gradients + let grads = grads_syncer.sync(item.grads); + + if let Some(grads) = grads { + learner.optimizer_step(grads); + } + } + } + + let item = TrainingItem::new( + item.item, + progress, + global_progress.clone(), + Some(iteration), + Some(learner.lr_current()), + ); + + { + let mut processor = processor.lock().unwrap(); + processor.process_train(LearnerEvent::ProcessedItem(item)); + } + + if interrupter.should_stop() { + log::info!("Training interrupted."); + break; + } + } + + if is_main { + let mut processor = processor.lock().unwrap(); + processor.process_train(LearnerEvent::EndEpoch(epoch)); + } + } +} + +/// Worker that is responsible for syncing gradients for the DDP worker. With double buffering, +/// this allows for more optimization. +struct GradsSyncer + 'static> { + msg_send: SyncSender, + // Optional because with double buffering, the first iteration yields no gradients. + result_recv: Receiver>, + + _p: PhantomData<(B, M)>, +} + +impl + 'static> GradsSyncer { + fn new(double_buffering: bool, peer_id: PeerId) -> Self { + let (msg_send, msg_recv) = std::sync::mpsc::sync_channel::(1); + let (result_send, result_recv) = + std::sync::mpsc::sync_channel::>(1); + std::thread::spawn(move || { + Self::run_worker(double_buffering, peer_id, result_send, msg_recv) + }); + Self { + msg_send, + result_recv, + _p: PhantomData, + } + } + + fn sync(&self, grads: GradientsParams) -> Option { + self.msg_send.send(grads).unwrap(); + self.result_recv.recv().unwrap() + } + + fn run_worker( + double_buffering: bool, + peer_id: PeerId, + send: SyncSender>, + recv: Receiver, + ) { + let mut grads_buffer = None; + + while let Ok(new_grads) = recv.recv() { + // Sync grads with collective + let new_grads = new_grads + .all_reduce::(peer_id, ReduceOperation::Mean) + .expect("DDP worker could not sync gradients!"); + + if double_buffering { + let old_grads = grads_buffer.take(); + grads_buffer = Some(new_grads); + + send.send(old_grads).unwrap(); + } else { + send.send(Some(new_grads)).unwrap(); + } + } + // GradsSyncer dropped, channel closed, this thread can end + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/mod.rs new file mode 100644 index 0000000..1828753 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/mod.rs @@ -0,0 +1,5 @@ +mod epoch; +mod strategy; +mod worker; + +pub use strategy::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/strategy.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/strategy.rs new file mode 100644 index 0000000..15d13a0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/strategy.rs @@ -0,0 +1,140 @@ +use core::panic; +use std::sync::{Arc, Mutex}; + +use burn_collective::CollectiveConfig; +use burn_core::tensor::Device; +use burn_core::tensor::backend::DeviceOps; + +use crate::ddp::worker::DdpWorker; +use crate::metric::store::EventStoreClient; +use crate::{ + EarlyStoppingStrategyRef, Interrupter, Learner, LearningComponentsTypes, + SupervisedLearningStrategy, SupervisedTrainingEventProcessor, TrainLoader, TrainingBackend, + TrainingComponents, TrainingModel, ValidLoader, +}; +use burn_core::data::dataloader::split::split_dataloader; + +#[derive(Clone)] +pub(crate) struct WorkerComponents { + /// The total number of epochs + pub num_epochs: usize, + /// Enables gradients accumulation. + pub grad_accumulation: Option, + /// An [Interupter](Interrupter) that allows aborting the training/evaluation process early. + pub interrupter: Interrupter, + /// Cloneable reference to an early stopping strategy. + pub early_stopping: Option, + /// A reference to an [EventStoreClient](EventStoreClient). + pub event_store: Arc, +} + +pub struct DdpTrainingStrategy { + devices: Vec>>, + config: CollectiveConfig, +} +impl DdpTrainingStrategy { + pub fn new(devices: Vec>>, config: CollectiveConfig) -> Self { + let config = config.with_num_devices(devices.len()); + Self { devices, config } + } +} + +impl SupervisedLearningStrategy + for DdpTrainingStrategy +{ + fn fit( + &self, + training_components: TrainingComponents, + learner: Learner, + dataloader_train: TrainLoader, + dataloader_valid: ValidLoader, + starting_epoch: usize, + ) -> (TrainingModel, SupervisedTrainingEventProcessor) { + // The reference model is always on the first device provided. + let main_device = self.devices.first().unwrap(); + // One worker per device, so we use a fixed device strategy + // for each (worker) data loader. This matches the expected device on the worker, so we + // don't have to move the data between devices. + let mut dataloaders_train = split_dataloader(dataloader_train, &self.devices); + let dataloader_valid = dataloader_valid.to_device(main_device.inner()); + + let main_device = self.devices[0].clone(); + let peer_count = self.devices.len(); + let event_processor = Arc::new(Mutex::new(training_components.event_processor)); + + let interrupter = training_components.interrupter; + let worker_components = WorkerComponents { + num_epochs: training_components.num_epochs, + grad_accumulation: training_components.grad_accumulation, + interrupter: interrupter.clone(), + early_stopping: training_components.early_stopping, + event_store: training_components.event_store, + }; + + // Start worker for main device + // First training dataloader corresponds to main device + let main_handle = DdpWorker::::start( + 0.into(), + main_device, + learner.clone(), + event_processor.clone(), + worker_components.clone(), + training_components.checkpointer, + dataloaders_train.remove(0), + Some(dataloader_valid), + self.config.clone(), + starting_epoch, + peer_count, + true, + ); + + // Spawn other workers for the other devices, starting with peer id 1 + let mut peer_id = 1; + let mut secondary_workers = vec![]; + for device in &self.devices[1..] { + let handle = DdpWorker::::start( + peer_id.into(), + device.clone(), + learner.clone(), + event_processor.clone(), + worker_components.clone(), + None, + dataloaders_train.remove(0), + None, + self.config.clone(), + starting_epoch, + peer_count, + false, + ); + + peer_id += 1; + + secondary_workers.push(handle); + } + + // Wait for all devices to finish + for worker in secondary_workers { + worker + .join() + .expect("Distributed data parallel worker failed"); + } + // Main worker had the event processor + let model = main_handle + .join() + .expect("Distributed data parallel main worker failed"); + + if interrupter.should_stop() { + let reason = interrupter + .get_message() + .unwrap_or(String::from("Reason unknown")); + log::info!("Training interrupted: {reason}"); + } + let Ok(event_processor) = Arc::try_unwrap(event_processor) else { + panic!("Event processor still held!"); + }; + let Ok(event_processor) = event_processor.into_inner() else { + panic!("Event processor lock poisoned"); + }; + (model, event_processor) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/worker.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/worker.rs new file mode 100644 index 0000000..1b9ee96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/ddp/worker.rs @@ -0,0 +1,135 @@ +use crate::ddp::epoch::{DdpTrainEpoch, DdpValidEpoch}; +use crate::ddp::strategy::WorkerComponents; +use crate::single::TrainingLoop; +use crate::{ + Learner, LearningCheckpointer, LearningComponentsTypes, SupervisedTrainingEventProcessor, + TrainLoader, TrainingBackend, ValidLoader, +}; +use burn_collective::{self, CollectiveConfig, PeerId}; +use burn_core::tensor::Device; +use burn_core::tensor::backend::AutodiffBackend; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +/// A worker runs the model, syncing gradients using collective operations. +/// Event processing and validation is optional too. +pub(crate) struct DdpWorker +where + LC: LearningComponentsTypes + Send + 'static, +{ + peer_id: PeerId, + device: Device>, + learner: Learner, + event_processor: Arc>>, + components: WorkerComponents, + checkpointer: Option>, + dataloader_train: TrainLoader, + dataloader_valid: Option>, + collective_config: CollectiveConfig, + starting_epoch: usize, + peer_count: usize, + is_main: bool, +} + +impl DdpWorker +where + LC: LearningComponentsTypes + Send + 'static, +{ + /// Starts a worker that runs the model in a data distributed parallel + #[allow(clippy::too_many_arguments)] + pub fn start( + peer_id: PeerId, + device: Device>, + learner: Learner, + event_processor: Arc>>, + components: WorkerComponents, + checkpointer: Option>, + dataloader_train: TrainLoader, + dataloader_valid: Option>, + collective_config: CollectiveConfig, + starting_epoch: usize, + peer_count: usize, + is_main: bool, + ) -> JoinHandle<::TrainingModel> { + let worker = Self { + peer_id, + device, + learner, + event_processor, + components, + checkpointer, + dataloader_train, + dataloader_valid, + collective_config, + starting_epoch, + peer_count, + is_main, + }; + + std::thread::spawn(|| worker.fit()) + } + + /// Fits the model, + pub fn fit(mut self) -> ::TrainingModel { + burn_collective::register::< as AutodiffBackend>::InnerBackend>( + self.peer_id, + self.device.clone(), + self.collective_config.clone(), + ) + .expect("Couldn't register for collective operations!"); + + let num_epochs = self.components.num_epochs; + let interrupter = self.components.interrupter; + + // Changed the train epoch to keep the dataloaders + let epoch_train = DdpTrainEpoch::::new( + self.dataloader_train.clone(), + self.components.grad_accumulation, + ); + let epoch_valid = self + .dataloader_valid + .map(|dataloader| DdpValidEpoch::::new(dataloader)); + self.learner.fork(&self.device); + + for training_progress in TrainingLoop::new(self.starting_epoch, num_epochs) { + let epoch = training_progress.items_processed; + + epoch_train.run( + &mut self.learner, + &training_progress, + self.event_processor.clone(), + &interrupter, + self.peer_id, + self.peer_count, + self.is_main, + ); + + if interrupter.should_stop() { + break; + } + + // Validation + if let Some(runner) = &epoch_valid { + let mut event_processor = self.event_processor.lock().unwrap(); + runner.run( + &self.learner.model(), + &training_progress, + &mut event_processor, + &interrupter, + ); + } + + if let Some(checkpointer) = &mut self.checkpointer { + checkpointer.checkpoint(&self.learner, epoch, &self.components.event_store); + } + + if let Some(early_stopping) = &mut self.components.early_stopping + && early_stopping.should_stop(epoch, &self.components.event_store) + { + break; + } + } + + self.learner.model() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/mod.rs new file mode 100644 index 0000000..3148df7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/mod.rs @@ -0,0 +1,8 @@ +mod base; + +#[cfg(feature = "ddp")] +pub(crate) mod ddp; +pub(crate) mod multi; +pub(crate) mod single; + +pub use base::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/epoch.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/epoch.rs new file mode 100644 index 0000000..18d758e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/epoch.rs @@ -0,0 +1,219 @@ +use crate::learner::base::Interrupter; +use crate::metric::processor::{EventProcessorTraining, LearnerEvent, TrainingItem}; +use crate::train::MultiDevicesTrainStep; +use crate::{ + Learner, LearningComponentsTypes, MultiDeviceOptim, SupervisedTrainingEventProcessor, + TrainLoader, TrainingBackend, +}; +use burn_core::data::dataloader::Progress; +use burn_core::prelude::DeviceOps; +use burn_core::tensor::Device; +use burn_core::tensor::backend::DeviceId; +use burn_optim::GradientsAccumulator; +use burn_optim::MultiGradientsParams; +use std::collections::HashMap; + +/// A training epoch. +#[derive(new)] +pub struct MultiDeviceTrainEpoch { + dataloaders: Vec>, + grad_accumulation: Option, +} + +impl MultiDeviceTrainEpoch { + /// Runs the training epoch on multiple devices. + /// + /// # Arguments + /// + /// * `model` - The model to train. + /// * `optim` - The optimizer to use. + /// * `lr_scheduler` - The learning rate scheduler to use. + /// * `processor` - The event processor to use. + /// * `devices` - The devices to use. + /// + /// # Returns + /// + /// The trained model and the optimizer. + #[allow(clippy::too_many_arguments)] + pub fn run( + &self, + learner: &mut Learner, + global_progress: &Progress, + event_processor: &mut SupervisedTrainingEventProcessor, + interrupter: &Interrupter, + devices: Vec>>, + strategy: MultiDeviceOptim, + ) { + match strategy { + MultiDeviceOptim::OptimMainDevice => self.run_optim_main( + learner, + global_progress, + event_processor, + interrupter, + devices, + ), + MultiDeviceOptim::OptimSharded => self.run_optim_distr( + learner, + global_progress, + event_processor, + interrupter, + devices, + ), + } + } + + fn run_optim_main( + &self, + learner: &mut Learner, + global_progress: &Progress, + event_processor: &mut SupervisedTrainingEventProcessor, + interrupter: &Interrupter, + devices: Vec>>, + ) { + let epoch = global_progress.items_processed; + log::info!( + "Executing training step for epoch {} on devices {:?}", + epoch, + devices + ); + + let mut iterators = self + .dataloaders + .iter() + .map(|d| d.iter()) + .collect::>(); + let mut iteration = 0; + let mut accumulator = GradientsAccumulator::new(); + let mut accumulation_current = 0; + + let accumulation = self.grad_accumulation.unwrap_or(1); + let step = MultiDevicesTrainStep::::new(&devices); + + // The main device is always the first in the list. + let device_main = devices.first().expect("A minimum of one device.").clone(); + + loop { + let (items, progress) = step.step(iterators.as_mut_slice(), &learner.model()); + if items.is_empty() { + break; + } + + learner.lr_step(); + + let mut progress_items = Vec::with_capacity(items.len()); + for item in items.into_iter() { + let grads = item.output.grads.to_device(&device_main, &learner.model()); + accumulator.accumulate(&learner.model(), grads); + progress_items.push(item.output.item); + } + + accumulation_current += 1; + + if accumulation <= accumulation_current { + let grads = accumulator.grads(); + learner.optimizer_step(grads); + accumulation_current = 0; + } + + for item in progress_items { + iteration += 1; + let item = TrainingItem::new( + item, + progress.clone(), + global_progress.clone(), + Some(iteration), + Some(learner.lr_current()), + ); + + event_processor.process_train(LearnerEvent::ProcessedItem(item)); + } + + if interrupter.should_stop() { + break; + } + } + + event_processor.process_train(LearnerEvent::EndEpoch(epoch)); + } + + fn run_optim_distr( + &self, + learner: &mut Learner, + global_progress: &Progress, + event_processor: &mut SupervisedTrainingEventProcessor, + interrupter: &Interrupter, + devices: Vec>>, + ) { + let epoch = global_progress.items_processed; + log::info!( + "Executing training step for epoch {} on devices {:?}", + epoch, + devices + ); + + let mut iterators = self + .dataloaders + .iter() + .map(|d| d.iter()) + .collect::>(); + let mut iteration = 0; + let mut accumulators = HashMap::< + DeviceId, + GradientsAccumulator<::TrainingModel>, + >::new(); + for device in devices.iter() { + accumulators.insert(device.to_id(), GradientsAccumulator::new()); + } + let mut accumulation_current = 0; + + let accumulation = self.grad_accumulation.unwrap_or(1); + let step = MultiDevicesTrainStep::::new(&devices); + + loop { + let (items, progress) = step.step(iterators.as_mut_slice(), &learner.model()); + if items.is_empty() { + break; + } + + learner.lr_step(); + + let mut progress_items = Vec::with_capacity(items.len()); + for item in items.into_iter() { + let accumulator = accumulators.get_mut(&item.device).unwrap(); + accumulator.accumulate(&learner.model(), item.output.grads); + progress_items.push(item.output.item); + } + + accumulation_current += 1; + + if accumulation <= accumulation_current { + let mut grads = MultiGradientsParams::default(); + for (device_id, accumulator) in accumulators.iter_mut() { + let grad = accumulator.grads(); + grads.grads.push((grad, *device_id)); + } + learner.optimizer_step_multi(grads); + accumulation_current = 0; + } + + for item in progress_items { + iteration += 1; + let item = TrainingItem::new( + item, + progress.clone(), + global_progress.clone(), + Some(iteration), + Some(learner.lr_current()), + ); + + event_processor.process_train(LearnerEvent::ProcessedItem(item)); + } + + if interrupter.should_stop() { + break; + } + } + + event_processor.process_train(LearnerEvent::EndEpoch(epoch)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/mod.rs new file mode 100644 index 0000000..b141f9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod epoch; +mod strategy; + +pub use strategy::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/strategy.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/strategy.rs new file mode 100644 index 0000000..fb08024 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/multi/strategy.rs @@ -0,0 +1,100 @@ +use crate::{ + Learner, LearningComponentsTypes, MultiDeviceOptim, SupervisedLearningStrategy, + SupervisedTrainingEventProcessor, TrainLoader, TrainingBackend, TrainingComponents, + TrainingModel, ValidLoader, + multi::epoch::MultiDeviceTrainEpoch, + single::{TrainingLoop, epoch::SingleDeviceValidEpoch}, +}; +use burn_core::{ + data::dataloader::split::split_dataloader, + tensor::{Device, backend::DeviceOps}, +}; + +pub struct MultiDeviceLearningStrategy { + devices: Vec>>, + optim: MultiDeviceOptim, +} +impl MultiDeviceLearningStrategy { + pub fn new(devices: Vec>>, optim: MultiDeviceOptim) -> Self { + Self { devices, optim } + } +} + +impl SupervisedLearningStrategy + for MultiDeviceLearningStrategy +{ + fn fit( + &self, + training_components: TrainingComponents, + mut learner: Learner, + dataloader_train: TrainLoader, + dataloader_valid: ValidLoader, + starting_epoch: usize, + ) -> (TrainingModel, SupervisedTrainingEventProcessor) { + let main_device = self.devices.first().unwrap(); + + // `MultiDevicesTrainStep` has one worker per device, so we use a fixed device strategy + // for each (worker) data loader. This matches the expected device on the worker, so we + // don't have to move the data between devices. + let dataloader_train = split_dataloader(dataloader_train, &self.devices); + let dataloader_valid = dataloader_valid.to_device(main_device.inner()); + + learner.fork(main_device); + let mut event_processor = training_components.event_processor; + let mut checkpointer = training_components.checkpointer; + let mut early_stopping = training_components.early_stopping; + + let epoch_train = MultiDeviceTrainEpoch::::new( + dataloader_train.clone(), + training_components.grad_accumulation, + ); + let epoch_valid: SingleDeviceValidEpoch = + SingleDeviceValidEpoch::new(dataloader_valid.clone()); + + for training_progress in TrainingLoop::new(starting_epoch, training_components.num_epochs) { + let epoch = training_progress.items_processed; + epoch_train.run( + &mut learner, + &training_progress, + &mut event_processor, + &training_components.interrupter, + self.devices.to_vec(), + self.optim, + ); + + if training_components.interrupter.should_stop() { + let reason = training_components + .interrupter + .get_message() + .unwrap_or(String::from("Reason unknown")); + log::info!("Training interrupted: {reason}"); + break; + } + + // After OptimSharded training, model parameters are scattered across + // devices. Fork back to main_device before single-device validation. + if matches!(self.optim, MultiDeviceOptim::OptimSharded) { + learner.fork(main_device); + } + + epoch_valid.run( + &learner, + &training_progress, + &mut event_processor, + &training_components.interrupter, + ); + + if let Some(checkpointer) = &mut checkpointer { + checkpointer.checkpoint(&learner, epoch, &training_components.event_store); + } + + if let Some(early_stopping) = &mut early_stopping + && early_stopping.should_stop(epoch, &training_components.event_store) + { + break; + } + } + + (learner.model(), event_processor) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/epoch.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/epoch.rs new file mode 100644 index 0000000..ad86bec --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/epoch.rs @@ -0,0 +1,136 @@ +use crate::learner::base::Interrupter; +use crate::metric::processor::{EventProcessorTraining, LearnerEvent, TrainingItem}; +use crate::{ + InferenceStep, Learner, LearningComponentsTypes, SupervisedTrainingEventProcessor, TrainLoader, + ValidLoader, +}; +use burn_core::data::dataloader::Progress; +use burn_core::module::AutodiffModule; +use burn_optim::GradientsAccumulator; + +/// A validation epoch. +#[derive(new)] +pub struct SingleDeviceValidEpoch { + dataloader: ValidLoader, +} + +/// A training epoch. +#[derive(new)] +pub struct SingleDeviceTrainEpoch { + dataloader: TrainLoader, + grad_accumulation: Option, +} + +impl SingleDeviceValidEpoch { + /// Runs the validation epoch. + /// + /// # Arguments + /// + /// * `model` - The model to validate. + /// * `processor` - The event processor to use. + pub fn run( + &self, + learner: &Learner, + global_progress: &Progress, + processor: &mut SupervisedTrainingEventProcessor, + interrupter: &Interrupter, + ) { + let epoch = global_progress.items_processed; + log::info!("Executing validation step for epoch {}", epoch); + let model = learner.model().valid(); + + let mut iterator = self.dataloader.iter(); + let mut iteration = 0; + + while let Some(item) = iterator.next() { + let progress = iterator.progress(); + iteration += 1; + + let item = model.step(item); + let item = TrainingItem::new( + item, + progress, + global_progress.clone(), + Some(iteration), + None, + ); + + processor.process_valid(LearnerEvent::ProcessedItem(item)); + + if interrupter.should_stop() { + break; + } + } + processor.process_valid(LearnerEvent::EndEpoch(epoch)); + } +} + +impl SingleDeviceTrainEpoch { + /// Runs the training epoch. + /// + /// # Arguments + /// + /// * `model` - The model to train. + /// * `optim` - The optimizer to use. + /// * `scheduler` - The learning rate scheduler to use. + /// * `processor` - The event processor to use. + /// + /// # Returns + /// + /// The trained model and the optimizer. + pub fn run( + &self, + learner: &mut Learner, + global_progress: &Progress, + processor: &mut SupervisedTrainingEventProcessor, + interrupter: &Interrupter, + ) { + let epoch = global_progress.items_processed; + log::info!("Executing training step for epoch {}", epoch,); + + // Single device / dataloader + let mut iterator = self.dataloader.iter(); + let mut iteration = 0; + let mut accumulator = GradientsAccumulator::new(); + let mut accumulation_current = 0; + + while let Some(item) = iterator.next() { + iteration += 1; + learner.lr_step(); + log::info!("Iteration {iteration}"); + + let progress = iterator.progress(); + let item = learner.train_step(item); + + match self.grad_accumulation { + Some(accumulation) => { + accumulator.accumulate(&learner.model(), item.grads); + accumulation_current += 1; + + if accumulation <= accumulation_current { + let grads = accumulator.grads(); + + learner.optimizer_step(grads); + accumulation_current = 0; + } + } + None => learner.optimizer_step(item.grads), + } + + let item = TrainingItem::new( + item.item, + progress, + global_progress.clone(), + Some(iteration), + Some(learner.lr_current()), + ); + + processor.process_train(LearnerEvent::ProcessedItem(item)); + + if interrupter.should_stop() { + break; + } + } + processor.process_train(LearnerEvent::EndEpoch(epoch)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/mod.rs new file mode 100644 index 0000000..b141f9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod epoch; +mod strategy; + +pub use strategy::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/strategy.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/strategy.rs new file mode 100644 index 0000000..edb926a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/supervised/strategies/single/strategy.rs @@ -0,0 +1,108 @@ +use crate::{ + Learner, LearningComponentsTypes, SupervisedLearningStrategy, SupervisedTrainingEventProcessor, + TrainLoader, TrainingBackend, TrainingComponents, TrainingModel, ValidLoader, + single::epoch::{SingleDeviceTrainEpoch, SingleDeviceValidEpoch}, +}; +use burn_core::{ + data::dataloader::Progress, + tensor::{Device, backend::DeviceOps}, +}; + +/// Simplest learning strategy possible, with only a single devices doing both the training and +/// validation. +pub struct SingleDeviceTrainingStrategy { + device: Device>, +} +impl SingleDeviceTrainingStrategy { + pub fn new(device: Device>) -> Self { + Self { device } + } +} + +#[derive(new)] +pub(crate) struct TrainingLoop { + next_iteration: usize, + total_iteration: usize, +} + +/// An iterator that returns the progress of the training. +impl Iterator for TrainingLoop { + type Item = Progress; + + fn next(&mut self) -> Option { + if self.next_iteration > self.total_iteration { + return None; + } + + let progress = Progress { + items_processed: self.next_iteration, + items_total: self.total_iteration, + }; + + self.next_iteration += 1; + Some(progress) + } +} + +impl SupervisedLearningStrategy + for SingleDeviceTrainingStrategy +{ + fn fit( + &self, + training_components: TrainingComponents, + mut learner: Learner, + dataloader_train: TrainLoader, + dataloader_valid: ValidLoader, + starting_epoch: usize, + ) -> (TrainingModel, SupervisedTrainingEventProcessor) { + let dataloader_train = dataloader_train.to_device(&self.device); + let dataloader_valid = dataloader_valid.to_device(self.device.inner()); + learner.fork(&self.device); + let mut event_processor = training_components.event_processor; + let mut checkpointer = training_components.checkpointer; + let mut early_stopping = training_components.early_stopping; + + let epoch_train: SingleDeviceTrainEpoch = + SingleDeviceTrainEpoch::new(dataloader_train, training_components.grad_accumulation); + let epoch_valid: SingleDeviceValidEpoch = + SingleDeviceValidEpoch::new(dataloader_valid.clone()); + + for training_progress in TrainingLoop::new(starting_epoch, training_components.num_epochs) { + let epoch = training_progress.items_processed; + epoch_train.run( + &mut learner, + &training_progress, + &mut event_processor, + &training_components.interrupter, + ); + + if training_components.interrupter.should_stop() { + let reason = training_components + .interrupter + .get_message() + .unwrap_or(String::from("Reason unknown")); + log::info!("Training interrupted: {reason}"); + break; + } + + epoch_valid.run( + &learner, + &training_progress, + &mut event_processor, + &training_components.interrupter, + ); + + if let Some(checkpointer) = &mut checkpointer { + checkpointer.checkpoint(&learner, epoch, &training_components.event_store); + } + + if let Some(early_stopping) = &mut early_stopping + && early_stopping.should_stop(epoch, &training_components.event_store) + { + break; + } + } + + (learner.model(), event_processor) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/train_val.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/train_val.rs new file mode 100644 index 0000000..ff266cc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/learner/train_val.rs @@ -0,0 +1,129 @@ +use crate::{ItemLazy, renderer::MetricsRenderer}; +use burn_core::module::AutodiffModule; +use burn_core::tensor::backend::AutodiffBackend; +use burn_optim::{GradientsParams, MultiGradientsParams, Optimizer}; + +/// A training output. +pub struct TrainOutput { + /// The gradients. + pub grads: GradientsParams, + + /// The item. + pub item: TO, +} + +impl TrainOutput { + /// Creates a new training output. + /// + /// # Arguments + /// + /// * `module` - The module. + /// * `grads` - The gradients. + /// * `item` - The item. + /// + /// # Returns + /// + /// A new training output. + pub fn new>( + module: &M, + grads: B::Gradients, + item: TO, + ) -> Self { + let grads = GradientsParams::from_grads(grads, module); + Self { grads, item } + } +} + +/// Trait to be implemented for models to be able to be trained. +/// +/// The [step](TrainStep::step) method needs to be manually implemented for all structs. +/// +/// The [optimize](TrainStep::optimize) method can be overridden if you want to control how the +/// optimizer is used to update the model. This can be useful if you want to call custom mutable +/// functions on your model (e.g., clipping the weights) before or after the optimizer is used. +/// +/// # Notes +/// +/// To be used with the [Learner](crate::Learner) struct, the struct which implements this trait must +/// also implement the [AutodiffModule] trait, which is done automatically with the +/// [Module](burn_core::module::Module) derive. +pub trait TrainStep { + /// Type of input for a step of the training stage. + type Input: Send + 'static; + /// Type of output for a step of the training stage. + type Output: ItemLazy + 'static; + /// Runs a step for training, which executes the forward and backward passes. + /// + /// # Arguments + /// + /// * `item` - The input for the model. + /// + /// # Returns + /// + /// The output containing the model output and the gradients. + fn step(&self, item: Self::Input) -> TrainOutput; + /// Optimize the current module with the provided gradients and learning rate. + /// + /// # Arguments + /// + /// * `optim`: Optimizer used for learning. + /// * `lr`: The learning rate used for this step. + /// * `grads`: The gradients of each parameter in the current model. + /// + /// # Returns + /// + /// The updated model. + fn optimize(self, optim: &mut O, lr: f64, grads: GradientsParams) -> Self + where + B: AutodiffBackend, + O: Optimizer, + Self: AutodiffModule, + { + optim.step(lr, self, grads) + } + /// Optimize the current module with the provided gradients and learning rate. + /// + /// # Arguments + /// + /// * `optim`: Optimizer used for learning. + /// * `lr`: The learning rate used for this step. + /// * `grads`: Multiple gradients associated to each parameter in the current model. + /// + /// # Returns + /// + /// The updated model. + fn optimize_multi(self, optim: &mut O, lr: f64, grads: MultiGradientsParams) -> Self + where + B: AutodiffBackend, + O: Optimizer, + Self: AutodiffModule, + { + optim.step_multi(lr, self, grads) + } +} + +/// Trait to be implemented for validating models. +pub trait InferenceStep { + /// Type of input for an inference step. + type Input: Send + 'static; + /// Type of output for an inference step. + type Output: ItemLazy + 'static; + /// Runs a validation step. + /// + /// # Arguments + /// + /// * `item` - The item to validate on. + /// + /// # Returns + /// + /// The validation output. + fn step(&self, item: Self::Input) -> Self::Output; +} + +/// The result of a training, containing the model along with the [renderer](MetricsRenderer). +pub struct LearningResult { + /// The model with the learned weights. + pub model: M, + /// The renderer that can be used for follow up training and evaluation. + pub renderer: Box, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/lib.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/lib.rs new file mode 100644 index 0000000..76b1da0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/lib.rs @@ -0,0 +1,118 @@ +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! A library for training neural networks using the burn crate. + +#[macro_use] +extern crate derive_new; + +/// The checkpoint module. +pub mod checkpoint; + +pub(crate) mod components; + +/// Renderer modules to display metrics and training information. +pub mod renderer; + +/// The logger module. +pub mod logger; + +/// The metric module. +pub mod metric; + +pub use metric::processor::*; + +mod learner; + +pub use learner::*; + +mod evaluator; + +pub use evaluator::*; + +pub use components::*; + +#[cfg(test)] +pub(crate) type TestBackend = burn_ndarray::NdArray; + +#[cfg(test)] +pub(crate) mod tests { + use crate::TestBackend; + use burn_core::{prelude::Tensor, tensor::Bool}; + use std::default::Default; + + pub type TestAutodiffBackend = burn_autodiff::Autodiff; + + /// Probability of tp before adding errors + pub const THRESHOLD: f64 = 0.5; + + #[derive(Debug, Default)] + pub enum ClassificationType { + #[default] + Binary, + Multiclass, + Multilabel, + } + + /// Sample x Class shaped matrix for use in + /// classification metrics testing + pub fn dummy_classification_input( + classification_type: &ClassificationType, + ) -> (Tensor, Tensor) { + match classification_type { + ClassificationType::Binary => { + ( + Tensor::from_data([[0.3], [0.2], [0.7], [0.1], [0.55]], &Default::default()), + // targets + Tensor::from_data([[0], [1], [0], [0], [1]], &Default::default()), + // predictions @ threshold=0.5 + // [[0], [0], [1], [0], [1]] + ) + } + ClassificationType::Multiclass => { + ( + Tensor::from_data( + [ + [0.2, 0.8, 0.0], + [0.3, 0.6, 0.1], + [0.7, 0.25, 0.05], + [0.1, 0.15, 0.8], + [0.9, 0.03, 0.07], + ], + &Default::default(), + ), + Tensor::from_data( + // targets + [[0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0]], + // predictions @ top_k=1 + // [[0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0]] + // predictions @ top_k=2 + // [[1, 1, 0], [1, 1, 0], [1, 1, 0], [0, 1, 1], [1, 0, 1]] + &Default::default(), + ), + ) + } + ClassificationType::Multilabel => { + ( + Tensor::from_data( + [ + [0.1, 0.7, 0.6], + [0.3, 0.9, 0.05], + [0.8, 0.9, 0.4], + [0.7, 0.5, 0.9], + [1.0, 0.3, 0.2], + ], + &Default::default(), + ), + // targets + Tensor::from_data( + [[1, 1, 0], [1, 0, 1], [1, 1, 1], [0, 0, 1], [1, 0, 0]], + // predictions @ threshold=0.5 + // [[0, 1, 1], [0, 1, 0], [1, 1, 0], [1, 0, 1], [1, 0, 0]] + &Default::default(), + ), + ) + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/async_logger.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/async_logger.rs new file mode 100644 index 0000000..c659098 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/async_logger.rs @@ -0,0 +1,91 @@ +use super::Logger; +use std::sync::mpsc; + +enum Message { + Log(T), + End, + Sync(mpsc::Sender<()>), +} +/// Async logger. +pub struct AsyncLogger { + sender: mpsc::Sender>, + handler: Option>, +} + +#[derive(new)] +struct LoggerThread> { + logger: L, + receiver: mpsc::Receiver>, +} + +impl LoggerThread +where + L: Logger, +{ + fn run(mut self) { + for item in self.receiver.iter() { + match item { + Message::Log(item) => { + self.logger.log(item); + } + Message::End => { + return; + } + Message::Sync(callback) => { + callback + .send(()) + .expect("Can return result with the callback channel."); + } + } + } + } +} + +impl AsyncLogger { + /// Create a new async logger. + pub fn new(logger: L) -> Self + where + L: Logger + 'static, + { + let (sender, receiver) = mpsc::channel(); + let thread = LoggerThread::new(logger, receiver); + + let handler = Some(std::thread::spawn(move || thread.run())); + + Self { sender, handler } + } + + /// Sync the async logger. + pub(crate) fn sync(&self) { + let (sender, receiver) = mpsc::channel(); + + self.sender + .send(Message::Sync(sender)) + .expect("Can send message to logger thread."); + + receiver + .recv() + .expect("Should sync, otherwise the thread is dead."); + } +} + +impl Logger for AsyncLogger { + fn log(&mut self, item: T) { + self.sender + .send(Message::Log(item)) + .expect("Can log using the logger thread."); + } +} + +impl Drop for AsyncLogger { + fn drop(&mut self) { + self.sender + .send(Message::End) + .expect("Can send the end message to the logger thread."); + let handler = self.handler.take(); + + if let Some(handler) = handler { + handler.join().expect("The logger thread should stop."); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/base.rs new file mode 100644 index 0000000..5bb98ff --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/base.rs @@ -0,0 +1,9 @@ +/// The logger trait. +pub trait Logger: Send { + /// Logs an item. + /// + /// # Arguments + /// + /// * `item` - The item. + fn log(&mut self, item: T); +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/file.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/file.rs new file mode 100644 index 0000000..c852c21 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/file.rs @@ -0,0 +1,46 @@ +use super::Logger; +use std::{fs::File, io::Write, path::Path}; + +/// File logger. +pub struct FileLogger { + file: File, +} + +impl FileLogger { + /// Create a new file logger. + /// + /// # Arguments + /// + /// * `path` - The path. + /// + /// # Returns + /// + /// The file logger. + pub fn new(path: impl AsRef) -> Self { + let path = path.as_ref(); + let mut options = std::fs::File::options(); + let file = options + .write(true) + .truncate(true) + .create(true) + .open(path) + .unwrap_or_else(|err| { + panic!( + "Should be able to create the new file '{}': {}", + path.display(), + err + ) + }); + + Self { file } + } +} + +impl Logger for FileLogger +where + T: std::fmt::Display, +{ + fn log(&mut self, item: T) { + writeln!(&mut self.file, "{item}").expect("Can log an item."); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/in_memory.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/in_memory.rs new file mode 100644 index 0000000..31cf3f1 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/in_memory.rs @@ -0,0 +1,16 @@ +use super::Logger; + +/// In memory logger. +#[derive(Default)] +pub struct InMemoryLogger { + pub(crate) values: Vec, +} + +impl Logger for InMemoryLogger +where + T: std::fmt::Display, +{ + fn log(&mut self, item: T) { + self.values.push(item.to_string()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/metric.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/metric.rs new file mode 100644 index 0000000..70393d3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/metric.rs @@ -0,0 +1,375 @@ +use super::{AsyncLogger, FileLogger, InMemoryLogger, Logger}; +use crate::metric::{ + MetricDefinition, MetricEntry, MetricId, NumericEntry, + store::{EpochSummary, MetricsUpdate, Split}, +}; +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; + +const EPOCH_PREFIX: &str = "epoch-"; + +/// Metric logger. +pub trait MetricLogger: Send { + /// Logs an item. + /// + /// # Arguments + /// + /// * `update` - Update information for all registered metrics. + /// * `epoch` - Current epoch. + /// * `split` - Current dataset split. + fn log(&mut self, update: MetricsUpdate, epoch: usize, split: &Split); + + /// Read the logs for an epoch. + fn read_numeric( + &mut self, + name: &str, + epoch: usize, + split: &Split, + ) -> Result, String>; + + /// Logs the metric definition information (name, description, unit, etc.) + fn log_metric_definition(&mut self, definition: MetricDefinition); + + /// Logs summary at the end of the epoch. + fn log_epoch_summary(&mut self, summary: EpochSummary); +} + +/// The file metric logger. +pub struct FileMetricLogger { + loggers: HashMap>, + directory: PathBuf, + metric_definitions: HashMap, + is_eval: bool, + last_epoch: Option, +} + +impl FileMetricLogger { + /// Create a new file metric logger. + /// + /// # Arguments + /// + /// * `directory` - The directory. + /// + /// # Returns + /// + /// The file metric logger. + pub fn new(directory: impl AsRef) -> Self { + Self { + loggers: HashMap::new(), + directory: directory.as_ref().to_path_buf(), + metric_definitions: HashMap::default(), + is_eval: false, + last_epoch: None, + } + } + + /// Create a new file metric logger. + /// + /// # Arguments + /// + /// * `directory` - The directory. + /// + /// # Returns + /// + /// The file metric logger. + pub fn new_eval(directory: impl AsRef) -> Self { + Self { + loggers: HashMap::new(), + directory: directory.as_ref().to_path_buf(), + metric_definitions: HashMap::default(), + is_eval: true, + last_epoch: None, + } + } + + pub(crate) fn split_exists(&self, split: &Split) -> bool { + self.split_dir(split).is_some() + } + + pub(crate) fn split_dir(&self, split: &Split) -> Option { + let split_path = match split { + Split::Test(Some(tag)) => self.directory.join(split.to_string()).join(tag.as_str()), + other => self.directory.join(other.to_string()), + }; + (split_path.exists() && split_path.is_dir()).then_some(split_path) + } + + pub(crate) fn is_epoch_dir>(dirname: P) -> bool { + dirname.as_ref().starts_with(EPOCH_PREFIX) + } + + /// Number of epochs recorded. + pub(crate) fn epochs(&self) -> usize { + if self.is_eval { + log::warn!("Number of epochs not available when testing."); + return 0; + } + + let mut max_epoch = 0; + + // with split + for path in fs::read_dir(&self.directory).unwrap() { + let path = path.unwrap(); + + if fs::metadata(path.path()).unwrap().is_dir() { + for split_path in fs::read_dir(path.path()).unwrap() { + let split_path = split_path.unwrap(); + + if fs::metadata(split_path.path()).unwrap().is_dir() { + let dir_name = split_path.file_name().into_string().unwrap(); + + if !dir_name.starts_with(EPOCH_PREFIX) { + continue; + } + + let epoch = dir_name.replace(EPOCH_PREFIX, "").parse::().ok(); + + if let Some(epoch) = epoch + && epoch > max_epoch + { + max_epoch = epoch; + } + } + } + } + } + + max_epoch + } + + fn train_directory(&self, epoch: usize, split: &Split) -> PathBuf { + let name = format!("{EPOCH_PREFIX}{epoch}"); + + match split { + Split::Train | Split::Valid | Split::Test(None) => { + self.directory.join(split.to_string()).join(name) + } + Split::Test(Some(tag)) => { + let tag = format_tag(tag); + self.directory.join(split.to_string()).join(tag).join(name) + } + } + } + + fn eval_directory(&self, split: &Split) -> PathBuf { + match split { + Split::Train | Split::Valid | Split::Test(None) => self.directory.clone(), + Split::Test(Some(tag)) => self.directory.join(split.to_string()).join(format_tag(tag)), + } + } + + fn file_path(&self, name: &str, epoch: Option, split: &Split) -> PathBuf { + let directory = match epoch { + Some(epoch) => self.train_directory(epoch, split), + None => self.eval_directory(split), + }; + let name = name.replace(' ', "_"); + let name = format!("{name}.log"); + directory.join(name) + } + + fn create_directory(&self, epoch: Option, split: &Split) { + let directory = match epoch { + Some(epoch) => self.train_directory(epoch, split), + None => self.eval_directory(split), + }; + std::fs::create_dir_all(directory).ok(); + } +} + +impl FileMetricLogger { + fn log_item(&mut self, item: &MetricEntry, epoch: Option, split: &Split) { + let name = &self.metric_definitions.get(&item.metric_id).unwrap().name; + let key = logger_key(name, split); + let value = &item.serialized_entry.serialized; + + let logger = match self.loggers.get_mut(&key) { + Some(val) => val, + None => { + self.create_directory(epoch, split); + + let file_path = self.file_path(name, epoch, split); + let logger = FileLogger::new(file_path); + let logger = AsyncLogger::new(logger); + + self.loggers.insert(key.clone(), logger); + self.loggers + .get_mut(&key) + .expect("Can get the previously saved logger.") + } + }; + + logger.log(value.clone()); + } +} + +fn format_tag(tag: &str) -> String { + tag.trim().replace(' ', "-").to_lowercase() +} + +impl MetricLogger for FileMetricLogger { + fn log(&mut self, update: MetricsUpdate, epoch: usize, split: &Split) { + if !self.is_eval && self.last_epoch != Some(epoch) { + self.loggers.clear(); + self.last_epoch = Some(epoch); + } + + let entries: Vec<_> = update + .entries + .iter() + .chain( + update + .entries_numeric + .iter() + .map(|numeric_update| &numeric_update.entry), + ) + .cloned() + .collect(); + + for item in entries.iter() { + self.log_item(item, Some(epoch), split); + } + } + + fn read_numeric( + &mut self, + name: &str, + epoch: usize, + split: &Split, + ) -> Result, String> { + if let Some(value) = self.loggers.get(name) { + value.sync() + } + + let file_path = self.file_path(name, Some(epoch), split); + + let mut errors = false; + + let data = std::fs::read_to_string(file_path) + .unwrap_or_default() + .split('\n') + .filter_map(|value| { + if value.is_empty() { + None + } else { + match NumericEntry::deserialize(value) { + Ok(value) => Some(value), + Err(err) => { + log::error!("{err}"); + errors = true; + None + } + } + } + }) + .collect(); + + if errors { + Err("Parsing numeric entry errors".to_string()) + } else { + Ok(data) + } + } + + fn log_metric_definition(&mut self, definition: MetricDefinition) { + self.metric_definitions + .insert(definition.metric_id.clone(), definition); + } + + fn log_epoch_summary(&mut self, _summary: EpochSummary) { + if !self.is_eval { + self.loggers.clear(); + } + } +} + +fn logger_key(name: &str, split: &Split) -> String { + format!("{name}_{split}") +} + +/// In memory metric logger, useful when testing and debugging. +#[derive(Default)] +pub struct InMemoryMetricLogger { + values: HashMap>, + last_epoch: Option, + metric_definitions: HashMap, +} + +impl InMemoryMetricLogger { + /// Create a new in-memory metric logger. + pub fn new() -> Self { + Self::default() + } +} + +impl MetricLogger for InMemoryMetricLogger { + fn log(&mut self, update: MetricsUpdate, epoch: usize, split: &Split) { + if self.last_epoch != Some(epoch) { + self.values + .values_mut() + .for_each(|loggers| loggers.push(InMemoryLogger::default())); + self.last_epoch = Some(epoch); + } + + let entries: Vec<_> = update + .entries + .iter() + .chain( + update + .entries_numeric + .iter() + .map(|numeric_update| &numeric_update.entry), + ) + .cloned() + .collect(); + + for item in entries.iter() { + let name = &self.metric_definitions.get(&item.metric_id).unwrap().name; + let key = logger_key(name, split); + + if !self.values.contains_key(&key) { + self.values + .insert(key.to_string(), vec![InMemoryLogger::default()]); + } + + let values = self.values.get_mut(&key).unwrap(); + + values + .last_mut() + .unwrap() + .log(item.serialized_entry.serialized.clone()); + } + } + + fn read_numeric( + &mut self, + name: &str, + epoch: usize, + split: &Split, + ) -> Result, String> { + let key = logger_key(name, split); + let values = match self.values.get(&key) { + Some(values) => values, + None => return Ok(Vec::new()), + }; + + match values.get(epoch - 1) { + Some(logger) => Ok(logger + .values + .iter() + .filter_map(|value| NumericEntry::deserialize(value).ok()) + .collect()), + None => Ok(Vec::new()), + } + } + + fn log_metric_definition(&mut self, definition: MetricDefinition) { + self.metric_definitions + .insert(definition.metric_id.clone(), definition); + } + + fn log_epoch_summary(&mut self, _summary: EpochSummary) {} +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/mod.rs new file mode 100644 index 0000000..df727a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/logger/mod.rs @@ -0,0 +1,11 @@ +mod async_logger; +mod base; +mod file; +mod in_memory; +mod metric; + +pub use async_logger::*; +pub use base::*; +pub use file::*; +pub use in_memory::*; +pub use metric::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/acc.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/acc.rs new file mode 100644 index 0000000..328a1ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/acc.rs @@ -0,0 +1,164 @@ +use core::marker::PhantomData; + +use super::MetricMetadata; +use super::state::{FormatOptions, NumericMetricState}; +use crate::metric::{Metric, MetricAttributes, MetricName, Numeric, SerializedEntry}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{ElementConversion, Int, Tensor}; + +/// The accuracy metric. +#[derive(Clone)] +pub struct AccuracyMetric { + name: MetricName, + state: NumericMetricState, + pad_token: Option, + _b: PhantomData, +} + +/// The [accuracy metric](AccuracyMetric) input type. +#[derive(new)] +pub struct AccuracyInput { + outputs: Tensor, + targets: Tensor, +} + +impl Default for AccuracyMetric { + fn default() -> Self { + Self::new() + } +} + +impl AccuracyMetric { + /// Creates the metric. + pub fn new() -> Self { + Self { + name: MetricName::new("Accuracy".to_string()), + state: Default::default(), + pad_token: Default::default(), + _b: PhantomData, + } + } + + /// Sets the pad token. + pub fn with_pad_token(mut self, index: usize) -> Self { + self.pad_token = Some(index); + self + } +} + +impl Metric for AccuracyMetric { + type Input = AccuracyInput; + + fn update(&mut self, input: &AccuracyInput, _metadata: &MetricMetadata) -> SerializedEntry { + let targets = input.targets.clone(); + let outputs = input.outputs.clone(); + + let [batch_size, _n_classes] = outputs.dims(); + + let outputs = outputs.argmax(1).reshape([batch_size]); + + let accuracy = match self.pad_token { + Some(pad_token) => { + let mask = targets.clone().equal_elem(pad_token as i64); + let matches = outputs.equal(targets).float().mask_fill(mask.clone(), 0); + let num_pad = mask.float().sum(); + + let acc = matches.sum() / (num_pad.neg() + batch_size as f32); + + acc.into_scalar().elem::() + } + None => { + outputs + .equal(targets) + .int() + .sum() + .into_scalar() + .elem::() + / batch_size as f64 + } + }; + + self.state.update( + 100.0 * accuracy, + batch_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + super::NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for AccuracyMetric { + fn value(&self) -> super::NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> super::NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn test_accuracy_without_padding() { + let device = Default::default(); + let mut metric = AccuracyMetric::::new(); + let input = AccuracyInput::new( + Tensor::from_data( + [ + [0.0, 0.2, 0.8], // 2 + [1.0, 2.0, 0.5], // 1 + [0.4, 0.1, 0.2], // 0 + [0.6, 0.7, 0.2], // 1 + ], + &device, + ), + Tensor::from_data([2, 2, 1, 1], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(50.0, metric.value().current()); + } + + #[test] + fn test_accuracy_with_padding() { + let device = Default::default(); + let mut metric = AccuracyMetric::::new().with_pad_token(3); + let input = AccuracyInput::new( + Tensor::from_data( + [ + [0.0, 0.2, 0.8, 0.0], // 2 + [1.0, 2.0, 0.5, 0.0], // 1 + [0.4, 0.1, 0.2, 0.0], // 0 + [0.6, 0.7, 0.2, 0.0], // 1 + [0.0, 0.1, 0.2, 5.0], // Predicted padding should not count + [0.0, 0.1, 0.2, 0.0], // Error on padding should not count + [0.6, 0.0, 0.2, 0.0], // Error on padding should not count + ], + &device, + ), + Tensor::from_data([2, 2, 1, 1, 3, 3, 3], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(50.0, metric.value().current()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/auroc.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/auroc.rs new file mode 100644 index 0000000..9d3dd96 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/auroc.rs @@ -0,0 +1,231 @@ +use core::f64; +use core::marker::PhantomData; + +use super::MetricMetadata; +use super::state::{FormatOptions, NumericMetricState}; +use crate::metric::{Metric, MetricName, Numeric, SerializedEntry}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{ElementConversion, Int, Tensor}; + +/// The Area Under the Receiver Operating Characteristic Curve (AUROC, also referred to as [ROC AUC](https://en.wikipedia.org/wiki/Receiver_operating_characteristic)) for binary classification. +#[derive(Clone)] +pub struct AurocMetric { + name: MetricName, + state: NumericMetricState, + _b: PhantomData, +} + +/// The [AUROC metric](AurocMetric) input type. +#[derive(new)] +pub struct AurocInput { + outputs: Tensor, + targets: Tensor, +} + +impl Default for AurocMetric { + fn default() -> Self { + Self::new() + } +} + +impl AurocMetric { + /// Creates the metric. + pub fn new() -> Self { + Self { + name: MetricName::new("AUROC".to_string()), + state: Default::default(), + _b: PhantomData, + } + } + + fn binary_auroc(&self, probabilities: &Tensor, targets: &Tensor) -> f64 { + let n = targets.dims()[0]; + + let n_pos = targets.clone().sum().into_scalar().elem::() as usize; + + // Early return if we don't have both positive and negative samples + if n_pos == 0 || n_pos == n { + if n_pos == 0 { + log::warn!("Metric cannot be computed because all target values are negative.") + } else { + log::warn!("Metric cannot be computed because all target values are positive.") + } + return 0.0; + } + + let pos_mask = targets.clone().equal_elem(1).int().reshape([n, 1]); + let neg_mask = targets.clone().equal_elem(0).int().reshape([1, n]); + + let valid_pairs = pos_mask * neg_mask; + + let prob_i = probabilities.clone().reshape([n, 1]).repeat_dim(1, n); + let prob_j = probabilities.clone().reshape([1, n]).repeat_dim(0, n); + + let correct_order = prob_i.clone().greater(prob_j.clone()).int(); + + let ties = prob_i.equal(prob_j).int(); + + // Calculate AUC components + let num_pairs = valid_pairs.clone().sum().into_scalar().elem::(); + let correct_pairs = (correct_order * valid_pairs.clone()) + .sum() + .into_scalar() + .elem::(); + let tied_pairs = (ties * valid_pairs).sum().into_scalar().elem::(); + + (correct_pairs + 0.5 * tied_pairs) / num_pairs + } +} + +impl Metric for AurocMetric { + type Input = AurocInput; + + fn update(&mut self, input: &AurocInput, _metadata: &MetricMetadata) -> SerializedEntry { + let [batch_size, num_classes] = input.outputs.dims(); + + assert_eq!( + num_classes, 2, + "Currently only binary classification is supported" + ); + + let probabilities = { + let exponents = input.outputs.clone().exp(); + let sum = exponents.clone().sum_dim(1); + (exponents / sum) + .select(1, Tensor::arange(1..2, &input.outputs.device())) + .squeeze_dim(1) + }; + + let area_under_curve = self.binary_auroc(&probabilities, &input.targets); + + self.state.update( + 100.0 * area_under_curve, + batch_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } +} + +impl Numeric for AurocMetric { + fn value(&self) -> super::NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> super::NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn test_auroc() { + let device = Default::default(); + let mut metric = AurocMetric::::new(); + + let input = AurocInput::new( + Tensor::from_data( + [ + [0.1, 0.9], // High confidence positive + [0.7, 0.3], // Low confidence negative + [0.6, 0.4], // Low confidence negative + [0.2, 0.8], // High confidence positive + ], + &device, + ), + Tensor::from_data([1, 0, 0, 1], &device), // True labels + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(metric.value().current(), 100.0); + } + + #[test] + fn test_auroc_perfect_separation() { + let device = Default::default(); + let mut metric = AurocMetric::::new(); + + let input = AurocInput::new( + Tensor::from_data([[0.0, 1.0], [1.0, 0.0], [1.0, 0.0], [0.0, 1.0]], &device), + Tensor::from_data([1, 0, 0, 1], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(metric.value().current(), 100.0); // Perfect AUC + } + + #[test] + fn test_auroc_random() { + let device = Default::default(); + let mut metric = AurocMetric::::new(); + + let input = AurocInput::new( + Tensor::from_data( + [ + [0.5, 0.5], // Random predictions + [0.5, 0.5], + [0.5, 0.5], + [0.5, 0.5], + ], + &device, + ), + Tensor::from_data([1, 0, 0, 1], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(metric.value().current(), 50.0); + } + + #[test] + fn test_auroc_all_one_class() { + let device = Default::default(); + let mut metric = AurocMetric::::new(); + + let input = AurocInput::new( + Tensor::from_data( + [ + [0.1, 0.9], // All positives predictions + [0.2, 0.8], + [0.3, 0.7], + [0.4, 0.6], + ], + &device, + ), + Tensor::from_data([1, 1, 1, 1], &device), // All positive class + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(metric.value().current(), 0.0); + } + + #[test] + #[should_panic(expected = "Currently only binary classification is supported")] + fn test_auroc_multiclass_error() { + let device = Default::default(); + let mut metric = AurocMetric::::new(); + + let input = AurocInput::new( + Tensor::from_data( + [ + [0.1, 0.2, 0.7], // More than 2 classes not supported + [0.3, 0.5, 0.2], + ], + &device, + ), + Tensor::from_data([2, 1], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/base.rs new file mode 100644 index 0000000..1855987 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/base.rs @@ -0,0 +1,278 @@ +use std::sync::Arc; + +use burn_core::data::dataloader::Progress; +use burn_optim::LearningRate; + +/// Metric metadata that can be used when computing metrics. +pub struct MetricMetadata { + /// The current progress. + pub progress: Progress, + + /// The global progress of the training (e.g. epochs). + pub global_progress: Progress, + + /// The current iteration. + pub iteration: Option, + + /// The current learning rate. + pub lr: Option, +} + +impl MetricMetadata { + /// Fake metric metadata + #[cfg(test)] + pub fn fake() -> Self { + Self { + progress: Progress { + items_processed: 1, + items_total: 1, + }, + global_progress: Progress { + items_processed: 0, + items_total: 1, + }, + iteration: Some(0), + lr: None, + } + } +} + +/// Metric id that can be used to compare metrics and retrieve entries of the same metric. +/// For now we take the name as id to make sure that the same metric has the same id across different runs. +#[derive(Debug, Clone, new, PartialEq, Eq, Hash)] +pub struct MetricId { + /// The metric id. + id: Arc, +} + +/// Metric attributes define the properties intrinsic to different types of metric. +#[derive(Clone, Debug)] +pub enum MetricAttributes { + /// Numeric attributes. + Numeric(NumericAttributes), + /// No attributes. + None, +} + +/// Definition of a metric. +#[derive(Clone, Debug)] +pub struct MetricDefinition { + /// The metric's id. + pub metric_id: MetricId, + /// The name of the metric. + pub name: String, + /// The description of the metric. + pub description: Option, + /// The attributes of the metric. + pub attributes: MetricAttributes, +} + +impl MetricDefinition { + /// Create a new metric definition given the metric and a unique id. + pub fn new(metric_id: MetricId, metric: &Me) -> Self { + Self { + metric_id, + name: metric.name().to_string(), + description: metric.description(), + attributes: metric.attributes(), + } + } +} + +/// Metric trait. +/// +/// # Notes +/// +/// Implementations should define their own input type only used by the metric. +/// This is important since some conflict may happen when the model output is adapted for each +/// metric's input type. +pub trait Metric: Send + Sync + Clone { + /// The input type of the metric. + type Input; + + /// The parameterized name of the metric. + /// + /// This should be unique, so avoid using short generic names, prefer using the long name. + /// + /// For a metric that can exist at different parameters (e.g., top-k accuracy for different + /// values of k), the name should be unique for each instance. + fn name(&self) -> MetricName; + + /// A short description of the metric. + fn description(&self) -> Option { + None + } + + /// Attributes of the metric. + /// + /// By default, metrics have no attributes. + fn attributes(&self) -> MetricAttributes { + MetricAttributes::None + } + + /// Update the metric state and returns the current metric entry. + fn update(&mut self, item: &Self::Input, metadata: &MetricMetadata) -> SerializedEntry; + + /// Clear the metric state. + fn clear(&mut self); +} + +/// Type used to store metric names efficiently. +pub type MetricName = Arc; + +/// Adaptor are used to transform types so that they can be used by metrics. +/// +/// This should be implemented by a model's output type for all [metric inputs](Metric::Input) that are +/// registered with the specific learning paradigm (i.e. [SupervisedTraining](crate::SupervisedTraining)). +pub trait Adaptor { + /// Adapt the type to be passed to a [metric](Metric). + fn adapt(&self) -> T; +} + +impl Adaptor<()> for T { + fn adapt(&self) {} +} + +/// Attributes that describe intrinsic properties of a numeric metric. +#[derive(Clone, Debug)] +pub struct NumericAttributes { + /// Optional unit (e.g. "%", "ms", "pixels") + pub unit: Option, + /// Whether larger values are better (true) or smaller are better (false). + pub higher_is_better: bool, +} + +impl From for MetricAttributes { + fn from(attr: NumericAttributes) -> Self { + MetricAttributes::Numeric(attr) + } +} + +impl Default for NumericAttributes { + fn default() -> Self { + Self { + unit: None, + higher_is_better: true, + } + } +} + +/// Declare a metric to be numeric. +/// +/// This is useful to plot the values of a metric during training. +pub trait Numeric { + /// Returns the numeric value of the metric. + fn value(&self) -> NumericEntry; + /// Returns the current aggregated value of the metric over the global step (epoch). + fn running_value(&self) -> NumericEntry; +} + +/// Serialized form of a metric entry. +#[derive(Debug, Clone, new)] +pub struct SerializedEntry { + /// The string to be displayed. + pub formatted: String, + /// The string to be saved. + pub serialized: String, +} + +/// Data type that contains the current state of a metric at a given time. +#[derive(Debug, Clone)] +pub struct MetricEntry { + /// Id of the entry's metric. + pub metric_id: MetricId, + /// The serialized form of the entry. + pub serialized_entry: SerializedEntry, +} + +impl MetricEntry { + /// Create a new metric. + pub fn new(metric_id: MetricId, serialized_entry: SerializedEntry) -> Self { + Self { + metric_id, + serialized_entry, + } + } +} + +/// Numeric metric entry. +#[derive(Debug, Clone)] +pub enum NumericEntry { + /// Single numeric value. + Value(f64), + /// Aggregated numeric (value, number of elements). + Aggregated { + /// The aggregated value of all entries. + aggregated_value: f64, + /// The number of entries present in the aggregated value. + count: usize, + }, +} + +impl NumericEntry { + /// Gets the current aggregated value of the metric. + pub fn current(&self) -> f64 { + match self { + NumericEntry::Value(val) => *val, + NumericEntry::Aggregated { + aggregated_value, .. + } => *aggregated_value, + } + } + + /// Returns a String representing the NumericEntry + pub fn serialize(&self) -> String { + match self { + Self::Value(v) => v.to_string(), + Self::Aggregated { + aggregated_value, + count, + } => format!("{aggregated_value},{count}"), + } + } + + /// De-serializes a string representing a NumericEntry and returns a Result containing the corresponding NumericEntry. + pub fn deserialize(entry: &str) -> Result { + // Check for comma separated values + let values = entry.split(',').collect::>(); + let num_values = values.len(); + + if num_values == 1 { + // Numeric value + match values[0].parse::() { + Ok(value) => Ok(NumericEntry::Value(value)), + Err(err) => Err(err.to_string()), + } + } else if num_values == 2 { + // Aggregated numeric (value, number of elements) + let (value, numel) = (values[0], values[1]); + match value.parse::() { + Ok(value) => match numel.parse::() { + Ok(numel) => Ok(NumericEntry::Aggregated { + aggregated_value: value, + count: numel, + }), + Err(err) => Err(err.to_string()), + }, + Err(err) => Err(err.to_string()), + } + } else { + Err("Invalid number of values for numeric entry".to_string()) + } + } + + /// Compare this numeric metric's value with another one using the specified direction. + pub fn better_than(&self, other: &NumericEntry, higher_is_better: bool) -> bool { + (self.current() > other.current()) == higher_is_better + } +} + +/// Format a float with the given precision. Will use scientific notation if necessary. +pub fn format_float(float: f64, precision: usize) -> String { + let scientific_notation_threshold = 0.1_f64.powf(precision as f64 - 1.0); + + match scientific_notation_threshold >= float { + true => format!("{float:.precision$e}"), + false => format!("{float:.precision$}"), + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cer.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cer.rs new file mode 100644 index 0000000..fba732a --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cer.rs @@ -0,0 +1,239 @@ +use super::state::{FormatOptions, NumericMetricState}; +use super::{MetricMetadata, SerializedEntry}; +use crate::metric::{Metric, MetricAttributes, MetricName, Numeric, NumericEntry}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{Int, Tensor}; +use core::marker::PhantomData; +use std::sync::Arc; + +/// Computes the edit distance (Levenshtein distance) between two sequences of integers. +/// +/// The edit distance is defined as the minimum number of single-element edits (insertions, +/// deletions, or substitutions) required to change one sequence into the other. This +/// implementation is optimized for space, using only two rows of the dynamic programming table. +/// +pub(crate) fn edit_distance(reference: &[i32], prediction: &[i32]) -> usize { + let mut prev = (0..=prediction.len()).collect::>(); + let mut curr = vec![0; prediction.len() + 1]; + + for (i, &r) in reference.iter().enumerate() { + curr[0] = i + 1; + for (j, &p) in prediction.iter().enumerate() { + curr[j + 1] = if r == p { + prev[j] // no operation needed + } else { + 1 + prev[j].min(prev[j + 1]).min(curr[j]) // substitution, insertion, deletion + }; + } + core::mem::swap(&mut prev, &mut curr); + } + prev[prediction.len()] +} + +/// Character error rate (CER) is defined as the edit distance (e.g. Levenshtein distance) between the predicted +/// and reference character sequences, divided by the total number of characters in the reference. +/// This metric is commonly used in tasks such as speech recognition, OCR, or text generation +/// to quantify how closely the predicted output matches the ground truth at a character level. +/// +#[derive(Clone)] +pub struct CharErrorRate { + name: MetricName, + state: NumericMetricState, + pad_token: Option, + _b: PhantomData, +} + +/// The [character error rate metric](CharErrorRate) input type. +#[derive(new)] +pub struct CerInput { + /// The predicted token sequences (as a 2-D tensor of token indices). + pub outputs: Tensor, + /// The target token sequences (as a 2-D tensor of token indices). + pub targets: Tensor, +} + +impl Default for CharErrorRate { + fn default() -> Self { + Self::new() + } +} + +impl CharErrorRate { + /// Creates the metric. + pub fn new() -> Self { + Self { + name: Arc::new("CER".to_string()), + state: NumericMetricState::default(), + pad_token: None, + _b: PhantomData, + } + } + + /// Sets the pad token. + pub fn with_pad_token(mut self, index: usize) -> Self { + self.pad_token = Some(index); + self + } +} + +/// The [character error rate metric](CharErrorRate) implementation. +impl Metric for CharErrorRate { + type Input = CerInput; + + fn update(&mut self, input: &CerInput, _metadata: &MetricMetadata) -> SerializedEntry { + let outputs = &input.outputs; + let targets = &input.targets; + let [batch_size, seq_len] = targets.dims(); + + let (output_lengths, target_lengths) = if let Some(pad) = self.pad_token { + // Create boolean masks for non-padding tokens. + let output_mask = outputs.clone().not_equal_elem(pad as i64); + let target_mask = targets.clone().not_equal_elem(pad as i64); + + let output_lengths_tensor = output_mask.int().sum_dim(1); + let target_lengths_tensor = target_mask.int().sum_dim(1); + + ( + output_lengths_tensor.to_data().to_vec::().unwrap(), + target_lengths_tensor.to_data().to_vec::().unwrap(), + ) + } else { + // If there's no padding, all sequences have the full length. + ( + vec![seq_len as i64; batch_size], + vec![seq_len as i64; batch_size], + ) + }; + + let outputs_data = outputs.to_data().to_vec::().unwrap(); + let targets_data = targets.to_data().to_vec::().unwrap(); + + let total_edit_distance: usize = (0..batch_size) + .map(|i| { + let start = i * seq_len; + + // Get pre-calculated lengths for the current sequence. + let output_len = output_lengths[i] as usize; + let target_len = target_lengths[i] as usize; + + let output_seq_slice = &outputs_data[start..(start + output_len)]; + let target_seq_slice = &targets_data[start..(start + target_len)]; + let output_seq: Vec = output_seq_slice.iter().map(|&x| x as i32).collect(); + let target_seq: Vec = target_seq_slice.iter().map(|&x| x as i32).collect(); + + edit_distance(&target_seq, &output_seq) + }) + .sum(); + + let total_target_length = target_lengths.iter().map(|&x| x as f64).sum::(); + + let value = if total_target_length > 0.0 { + 100.0 * total_edit_distance as f64 / total_target_length + } else { + 0.0 + }; + + self.state.update( + value, + batch_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset(); + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + super::NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: false, + } + .into() + } +} + +impl Numeric for CharErrorRate { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + /// Perfect match ⇒ CER = 0 %. + #[test] + fn test_cer_without_padding() { + let device = Default::default(); + let mut metric = CharErrorRate::::new(); + + // Batch size = 2, sequence length = 2 + let preds = Tensor::from_data([[1, 2], [3, 4]], &device); + let tgts = Tensor::from_data([[1, 2], [3, 4]], &device); + + metric.update(&CerInput::new(preds, tgts), &MetricMetadata::fake()); + + assert_eq!(0.0, metric.value().current()); + } + + /// Two edits in four target tokens ⇒ 50 %. + #[test] + fn test_cer_without_padding_two_errors() { + let device = Default::default(); + let mut metric = CharErrorRate::::new(); + + // One substitution in each sequence. + let preds = Tensor::from_data([[1, 2], [3, 5]], &device); + let tgts = Tensor::from_data([[1, 3], [3, 4]], &device); + + metric.update(&CerInput::new(preds, tgts), &MetricMetadata::fake()); + + // 2 edits / 4 tokens = 50 % + assert_eq!(50.0, metric.value().current()); + } + + /// Same scenario as above, but with right-padding (token 9) ignored. + #[test] + fn test_cer_with_padding() { + let device = Default::default(); + let pad = 9_i64; + let mut metric = CharErrorRate::::new().with_pad_token(pad as usize); + + // Each row has three columns, last one is the pad token. + let preds = Tensor::from_data([[1, 2, pad], [3, 5, pad]], &device); + let tgts = Tensor::from_data([[1, 3, pad], [3, 4, pad]], &device); + + metric.update(&CerInput::new(preds, tgts), &MetricMetadata::fake()); + assert_eq!(50.0, metric.value().current()); + } + + /// `clear()` must reset the running statistics to zero. + #[test] + fn test_clear_resets_state() { + let device = Default::default(); + let mut metric = CharErrorRate::::new(); + + let preds = Tensor::from_data([[1, 2]], &device); + let tgts = Tensor::from_data([[1, 3]], &device); // one error + + metric.update( + &CerInput::new(preds.clone(), tgts.clone()), + &MetricMetadata::fake(), + ); + assert!(metric.value().current() > 0.0); + + metric.clear(); + assert!(metric.value().current().is_nan()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/classification.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/classification.rs new file mode 100644 index 0000000..6d66c63 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/classification.rs @@ -0,0 +1,33 @@ +use std::num::NonZeroUsize; + +/// Necessary data for classification metrics. +#[derive(Default, Debug, Clone)] +pub struct ClassificationMetricConfig { + pub decision_rule: DecisionRule, + pub class_reduction: ClassReduction, +} + +/// The prediction decision rule for classification metrics. +#[derive(Debug, Clone)] +pub enum DecisionRule { + /// Consider a class predicted if its probability exceeds the threshold. + Threshold(f64), + /// Consider a class predicted correctly if it is within the top k predicted classes based on scores. + TopK(NonZeroUsize), +} + +impl Default for DecisionRule { + fn default() -> Self { + Self::Threshold(0.5) + } +} + +/// The reduction strategy for classification metrics. +#[derive(Copy, Clone, Default, Debug)] +pub enum ClassReduction { + /// Computes the statistics over all classes before averaging + Micro, + /// Computes the statistics independently for each class before averaging + #[default] + Macro, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/confusion_stats.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/confusion_stats.rs new file mode 100644 index 0000000..7c4ebab --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/confusion_stats.rs @@ -0,0 +1,351 @@ +use super::classification::{ClassReduction, ClassificationMetricConfig, DecisionRule}; +use burn_core::{ + prelude::{Backend, Bool, Int, Tensor}, + tensor::IndexingUpdateOp, +}; +use std::fmt::{self, Debug}; + +/// Input for confusion statistics error types. +#[derive(new, Debug, Clone)] +pub struct ConfusionStatsInput { + /// Sample x Class Non thresholded normalized predictions. + pub predictions: Tensor, + /// Sample x Class one-hot encoded target. + pub targets: Tensor, +} + +impl From> for (Tensor, Tensor) { + fn from(input: ConfusionStatsInput) -> Self { + (input.predictions, input.targets) + } +} + +impl From<(Tensor, Tensor)> for ConfusionStatsInput { + fn from(value: (Tensor, Tensor)) -> Self { + Self::new(value.0, value.1) + } +} + +#[derive(Clone)] +pub struct ConfusionStats { + confusion_classes: Tensor, + class_reduction: ClassReduction, +} + +impl Debug for ConfusionStats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let to_vec = |tensor_data: Tensor| { + tensor_data + .to_data() + .to_vec::() + .expect("A vector representation of the input Tensor is expected") + }; + let ratio_of_support_vec = + |metric: Tensor| to_vec(self.clone().ratio_of_support(metric)); + f.debug_struct("ConfusionStats") + .field("tp", &ratio_of_support_vec(self.clone().true_positive())) + .field("fp", &ratio_of_support_vec(self.clone().false_positive())) + .field("tn", &ratio_of_support_vec(self.clone().true_negative())) + .field("fn", &ratio_of_support_vec(self.clone().false_negative())) + .field("support", &to_vec(self.clone().support())) + .finish() + } +} + +impl ConfusionStats { + /// Expects `predictions` to be normalized. + pub fn new(input: &ConfusionStatsInput, config: &ClassificationMetricConfig) -> Self { + let prediction_mask = match config.decision_rule { + DecisionRule::Threshold(threshold) => input.predictions.clone().greater_elem(threshold), + DecisionRule::TopK(top_k) => { + let mask = input.predictions.zeros_like(); + let indexes = + input + .predictions + .clone() + .argsort_descending(1) + .narrow(1, 0, top_k.get()); + let values = indexes.ones_like().float(); + mask.scatter(1, indexes, values, IndexingUpdateOp::Add) + .bool() + } + }; + Self { + confusion_classes: prediction_mask.int() + input.targets.clone().int() * 2, + class_reduction: config.class_reduction, + } + } + + /// sum over samples + fn aggregate( + sample_class_mask: Tensor, + class_reduction: ClassReduction, + ) -> Tensor { + use ClassReduction::{Macro, Micro}; + match class_reduction { + Micro => sample_class_mask.float().sum(), + Macro => sample_class_mask.float().sum_dim(0).squeeze_dim(0), + } + } + + pub fn true_positive(self) -> Tensor { + Self::aggregate(self.confusion_classes.equal_elem(3), self.class_reduction) + } + + pub fn true_negative(self) -> Tensor { + Self::aggregate(self.confusion_classes.equal_elem(0), self.class_reduction) + } + + pub fn false_positive(self) -> Tensor { + Self::aggregate(self.confusion_classes.equal_elem(1), self.class_reduction) + } + + pub fn false_negative(self) -> Tensor { + Self::aggregate(self.confusion_classes.equal_elem(2), self.class_reduction) + } + + pub fn positive(self) -> Tensor { + self.clone().true_positive() + self.false_negative() + } + + pub fn negative(self) -> Tensor { + self.clone().true_negative() + self.false_positive() + } + + pub fn predicted_positive(self) -> Tensor { + self.clone().true_positive() + self.false_positive() + } + + pub fn support(self) -> Tensor { + self.clone().positive() + self.negative() + } + + pub fn ratio_of_support(self, metric: Tensor) -> Tensor { + metric / self.clone().support() + } +} + +#[cfg(test)] +mod tests { + use super::{ConfusionStats, ConfusionStatsInput}; + use crate::{ + TestBackend, + metric::classification::{ClassReduction, ClassificationMetricConfig, DecisionRule}, + tests::{ClassificationType, THRESHOLD, dummy_classification_input}, + }; + use burn_core::prelude::TensorData; + use rstest::{fixture, rstest}; + use std::num::NonZeroUsize; + + fn top_k_config( + top_k: NonZeroUsize, + class_reduction: ClassReduction, + ) -> ClassificationMetricConfig { + ClassificationMetricConfig { + decision_rule: DecisionRule::TopK(top_k), + class_reduction, + } + } + #[fixture] + #[once] + fn top_k_config_k1_micro() -> ClassificationMetricConfig { + top_k_config(NonZeroUsize::new(1).unwrap(), ClassReduction::Micro) + } + + #[fixture] + #[once] + fn top_k_config_k1_macro() -> ClassificationMetricConfig { + top_k_config(NonZeroUsize::new(1).unwrap(), ClassReduction::Macro) + } + #[fixture] + #[once] + fn top_k_config_k2_micro() -> ClassificationMetricConfig { + top_k_config(NonZeroUsize::new(2).unwrap(), ClassReduction::Micro) + } + #[fixture] + #[once] + fn top_k_config_k2_macro() -> ClassificationMetricConfig { + top_k_config(NonZeroUsize::new(2).unwrap(), ClassReduction::Macro) + } + + fn threshold_config( + threshold: f64, + class_reduction: ClassReduction, + ) -> ClassificationMetricConfig { + ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + class_reduction, + } + } + #[fixture] + #[once] + fn threshold_config_micro() -> ClassificationMetricConfig { + threshold_config(THRESHOLD, ClassReduction::Micro) + } + #[fixture] + #[once] + fn threshold_config_macro() -> ClassificationMetricConfig { + threshold_config(THRESHOLD, ClassReduction::Macro) + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [1].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [1].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [3].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [1, 1, 1].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [4].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [2, 1, 1].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [5].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [2, 2, 1].into())] + fn test_true_positive( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .true_positive() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [2].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [2].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [8].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [2, 3, 3].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [4].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [1, 1, 2].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [3].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [0, 2, 1].into())] + fn test_true_negative( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .true_negative() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [1].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [1].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [2].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [1, 1, 0].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [6].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [2, 3, 1].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [3].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [1, 1, 1].into())] + fn test_false_positive( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .false_positive() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [1].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [1].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [2].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [1, 0, 1].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [1].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [0, 0, 1].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [4].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [2, 0, 2].into())] + fn test_false_negatives( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .false_negative() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [2].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [2].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [5].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [2, 1, 2].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [5].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [2, 1, 2].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [9].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [4, 2, 3].into())] + fn test_positive( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .positive() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [3].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [3].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [10].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [3, 4, 3].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [10].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [3, 4, 3].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [6].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [1, 3, 2].into())] + fn test_negative( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .negative() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } + + #[rstest] + #[case::binary_micro(ClassificationType::Binary, threshold_config_micro(), [2].into())] + #[case::binary_macro(ClassificationType::Binary, threshold_config_macro(), [2].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k1_micro(), [5].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k1_macro(), [2, 2, 1].into())] + #[case::multiclass_micro(ClassificationType::Multiclass, top_k_config_k2_micro(), [10].into())] + #[case::multiclass_macro(ClassificationType::Multiclass, top_k_config_k2_macro(), [4, 4, 2].into())] + #[case::multilabel_micro(ClassificationType::Multilabel, threshold_config_micro(), [8].into())] + #[case::multilabel_macro(ClassificationType::Multilabel, threshold_config_macro(), [3, 3, 2].into())] + fn test_predicted_positive( + #[case] classification_type: ClassificationType, + #[case] config: ClassificationMetricConfig, + #[case] expected: Vec, + ) { + let input: ConfusionStatsInput = + dummy_classification_input(&classification_type).into(); + ConfusionStats::new(&input, &config) + .predicted_positive() + .int() + .into_data() + .assert_eq(&TensorData::from(expected.as_slice()), true); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_temp.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_temp.rs new file mode 100644 index 0000000..61a3c4e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_temp.rs @@ -0,0 +1,76 @@ +use std::sync::Arc; + +/// CPU Temperature metric +use super::MetricMetadata; +use crate::metric::{Metric, MetricAttributes, MetricName, Numeric, NumericEntry, SerializedEntry}; +use systemstat::{Platform, System}; + +/// CPU Temperature in celsius degrees +#[derive(Clone)] +pub struct CpuTemperature { + name: MetricName, + temp_celsius: f32, + sys: Arc, +} + +impl CpuTemperature { + /// Creates a new CPU temp metric + pub fn new() -> Self { + let name = Arc::new("CPU Temperature".to_string()); + + Self { + name, + temp_celsius: 0., + sys: Arc::new(System::new()), + } + } +} + +impl Default for CpuTemperature { + fn default() -> Self { + CpuTemperature::new() + } +} + +impl Metric for CpuTemperature { + type Input = (); + + fn update(&mut self, _item: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + match self.sys.cpu_temp() { + Ok(temp) => self.temp_celsius = temp, + Err(_) => self.temp_celsius = f32::NAN, + } + + let formatted = match self.temp_celsius.is_nan() { + true => format!("{}: NaN °C", self.name()), + false => format!("{}: {:.2} °C", self.name(), self.temp_celsius), + }; + let raw = format!("{:.2}", self.temp_celsius); + + SerializedEntry::new(formatted, raw) + } + + fn clear(&mut self) {} + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + super::NumericAttributes { + unit: Some("°C".to_string()), + higher_is_better: false, + } + .into() + } +} + +impl Numeric for CpuTemperature { + fn value(&self) -> NumericEntry { + NumericEntry::Value(self.temp_celsius as f64) + } + + fn running_value(&self) -> NumericEntry { + NumericEntry::Value(self.temp_celsius as f64) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_use.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_use.rs new file mode 100644 index 0000000..67bf5ca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cpu_use.rs @@ -0,0 +1,103 @@ +use super::MetricMetadata; +use crate::metric::{Metric, MetricAttributes, MetricName, Numeric, NumericEntry, SerializedEntry}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use sysinfo::{CpuRefreshKind, RefreshKind, System}; + +/// General CPU Usage metric +pub struct CpuUse { + name: MetricName, + last_refresh: Instant, + refresh_frequency: Duration, + sys: System, + current: f64, +} + +impl Clone for CpuUse { + fn clone(&self) -> Self { + Self { + name: self.name.clone(), + last_refresh: self.last_refresh, + refresh_frequency: self.refresh_frequency, + sys: System::new(), + current: self.current, + } + } +} + +impl CpuUse { + /// Creates a new CPU metric + pub fn new() -> Self { + let mut sys = System::new(); + let current = Self::refresh(&mut sys); + let name = "CPU Usage".to_string(); + + Self { + name: Arc::new(name), + last_refresh: Instant::now(), + refresh_frequency: Duration::from_millis(200), + sys, + current, + } + } + + fn refresh(sys: &mut System) -> f64 { + sys.refresh_specifics( + RefreshKind::nothing().with_cpu(CpuRefreshKind::nothing().with_cpu_usage()), + ); + + let cpus = sys.cpus(); + let num_cpus = cpus.len(); + let use_percentage = cpus.iter().fold(0.0, |acc, cpu| acc + cpu.cpu_usage()) as f64; + + use_percentage / num_cpus as f64 + } +} + +impl Default for CpuUse { + fn default() -> Self { + CpuUse::new() + } +} + +impl Metric for CpuUse { + type Input = (); + + fn update(&mut self, _item: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + if self.last_refresh.elapsed() >= self.refresh_frequency { + self.current = Self::refresh(&mut self.sys); + self.last_refresh = Instant::now(); + } + + let formatted = format!("{}: {:.2} %", self.name(), self.current); + let raw = format!("{:.2}", self.current); + + SerializedEntry::new(formatted, raw) + } + + fn clear(&mut self) {} + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + super::NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: false, + } + .into() + } +} + +impl Numeric for CpuUse { + fn value(&self) -> NumericEntry { + NumericEntry::Value(self.current) + } + + fn running_value(&self) -> NumericEntry { + NumericEntry::Value(self.current) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cuda.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cuda.rs new file mode 100644 index 0000000..3bbcf08 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/cuda.rs @@ -0,0 +1,108 @@ +use std::sync::Arc; + +use super::MetricMetadata; +use crate::metric::{Metric, MetricName, SerializedEntry}; +use nvml_wrapper::Nvml; + +/// Track basic cuda infos. +#[derive(Clone)] +pub struct CudaMetric { + name: MetricName, + nvml: Arc>, +} + +impl CudaMetric { + /// Creates a new metric for CUDA. + pub fn new() -> Self { + Self { + name: Arc::new("Cuda".to_string()), + nvml: Arc::new(Nvml::init().map(Some).unwrap_or_else(|err| { + log::warn!("Unable to initialize CUDA Metric: {err}"); + None + })), + } + } +} + +impl Default for CudaMetric { + fn default() -> Self { + Self::new() + } +} + +impl Metric for CudaMetric { + type Input = (); + + fn update(&mut self, _item: &(), _metadata: &MetricMetadata) -> SerializedEntry { + let not_available = + || SerializedEntry::new("Unavailable".to_string(), "Unavailable".to_string()); + + let available = |nvml: &Nvml| { + let mut formatted = String::new(); + let mut raw_running = String::new(); + + let device_count = match nvml.device_count() { + Ok(val) => val, + Err(err) => { + log::warn!("Unable to get the number of cuda devices: {err}"); + return not_available(); + } + }; + + for index in 0..device_count { + let device = match nvml.device_by_index(index) { + Ok(val) => val, + Err(err) => { + log::warn!("Unable to get device {index}: {err}"); + return not_available(); + } + }; + let memory_info = match device.memory_info() { + Ok(info) => info, + Err(err) => { + log::warn!("Unable to get memory info from device {index}: {err}"); + return not_available(); + } + }; + + let used_gb = memory_info.used as f64 * 1e-9; + let total_gb = memory_info.total as f64 * 1e-9; + + let memory_info_formatted = format!("{used_gb:.2}/{total_gb:.2} Gb"); + let memory_info_raw = format!("{used_gb}/{total_gb}"); + + formatted = format!("{formatted} GPU #{index} - Memory {memory_info_formatted}"); + raw_running = format!("{memory_info_raw} "); + + let utilization_rates = match device.utilization_rates() { + Ok(rate) => rate, + Err(err) => { + log::warn!("Unable to get utilization rates from device {index}: {err}"); + return not_available(); + } + }; + let utilization_rate_formatted = format!("{}%", utilization_rates.gpu); + formatted = format!("{formatted} - Usage {utilization_rate_formatted}"); + + // Power is the currency for perf/W. NVML reports milliwatts. + if let Ok(power_mw) = device.power_usage() { + let power_w = power_mw as f64 / 1000.0; + formatted = format!("{formatted} - Power {power_w:.1} W"); + } + } + + SerializedEntry::new(formatted, raw_running) + }; + + match self.nvml.as_ref() { + Some(nvml) => available(nvml), + None => not_available(), + } + } + + fn clear(&mut self) {} + + fn name(&self) -> MetricName { + self.name.clone() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/fbetascore.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/fbetascore.rs new file mode 100644 index 0000000..232e51b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/fbetascore.rs @@ -0,0 +1,259 @@ +use crate::metric::{MetricName, Numeric}; + +use super::{ + Metric, MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, SerializedEntry, + classification::{ClassReduction, ClassificationMetricConfig, DecisionRule}, + confusion_stats::{ConfusionStats, ConfusionStatsInput}, + state::{FormatOptions, NumericMetricState}, +}; +use burn_core::{ + prelude::{Backend, Tensor}, + tensor::cast::ToElement, +}; +use core::marker::PhantomData; +use std::{num::NonZeroUsize, sync::Arc}; + +/// The [F-beta score](https://en.wikipedia.org/wiki/F-score) metric. +/// +/// The `beta` parameter represents the ratio of recall importance to precision importance. +/// `beta > 1` gives more weight to recall, while `beta < 1` favors precision. +#[derive(Clone)] +pub struct FBetaScoreMetric { + name: MetricName, + state: NumericMetricState, + _b: PhantomData, + config: ClassificationMetricConfig, + beta: f64, +} + +impl Default for FBetaScoreMetric { + fn default() -> Self { + Self::new(Default::default(), Default::default()) + } +} + +impl FBetaScoreMetric { + #[allow(dead_code)] + fn new(config: ClassificationMetricConfig, beta: f64) -> Self { + let name = Arc::new(format!( + "FBetaScore ({}) @ {:?} [{:?}]", + beta, config.decision_rule, config.class_reduction + )); + Self { + name, + config, + beta, + state: Default::default(), + _b: PhantomData, + } + } + + /// F-beta score metric for binary classification. + /// + /// # Arguments + /// + /// * `beta` - Positive real factor to weight recall's importance. + /// * `threshold` - The threshold to transform a probability into a binary prediction. + #[allow(dead_code)] + pub fn binary(beta: f64, threshold: f64) -> Self { + Self::new( + ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + // binary classification results are the same independently of class_reduction + ..Default::default() + }, + beta, + ) + } + + /// F-beta score metric for multiclass classification. + /// + /// # Arguments + /// + /// * `beta` - Positive real factor to weight recall's importance. + /// * `top_k` - The number of highest predictions considered to find the correct label (typically `1`). + /// * `class_reduction` - [Class reduction](ClassReduction) type. + #[allow(dead_code)] + pub fn multiclass(beta: f64, top_k: usize, class_reduction: ClassReduction) -> Self { + Self::new( + ClassificationMetricConfig { + decision_rule: DecisionRule::TopK( + NonZeroUsize::new(top_k).expect("top_k must be non-zero"), + ), + class_reduction, + }, + beta, + ) + } + + /// F-beta score metric for multi-label classification. + /// + /// # Arguments + /// + /// * `beta` - Positive real factor to weight recall's importance. + /// * `threshold` - The threshold to transform a probability into a binary prediction. + /// * `class_reduction` - [Class reduction](ClassReduction) type. + #[allow(dead_code)] + pub fn multilabel(beta: f64, threshold: f64, class_reduction: ClassReduction) -> Self { + Self::new( + ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + class_reduction, + }, + beta, + ) + } + + fn class_average(&self, mut aggregated_metric: Tensor) -> f64 { + use ClassReduction::{Macro, Micro}; + let avg_tensor = match self.config.class_reduction { + Micro => aggregated_metric, + Macro => { + if aggregated_metric + .clone() + .contains_nan() + .any() + .into_scalar() + .to_bool() + { + let nan_mask = aggregated_metric.clone().is_nan(); + aggregated_metric = aggregated_metric + .clone() + .select(0, nan_mask.bool_not().argwhere().squeeze_dim(1)) + } + aggregated_metric.mean() + } + }; + avg_tensor.into_scalar().to_f64() + } +} + +impl Metric for FBetaScoreMetric { + type Input = ConfusionStatsInput; + + fn update(&mut self, input: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + let [sample_size, _] = input.predictions.dims(); + + let cf_stats = ConfusionStats::new(input, &self.config); + let scaled_true_positive = cf_stats.clone().true_positive() * (1.0 + self.beta.powi(2)); + let metric = self.class_average( + scaled_true_positive.clone() + / (scaled_true_positive + + cf_stats.clone().false_negative() * self.beta.powi(2) + + cf_stats.false_positive()), + ); + + self.state.update( + 100.0 * metric, + sample_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for FBetaScoreMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::{ + ClassReduction::{self, *}, + FBetaScoreMetric, Metric, MetricMetadata, + }; + use crate::metric::Numeric; + use crate::{ + TestBackend, + tests::{ClassificationType, THRESHOLD, dummy_classification_input}, + }; + use burn_core::tensor::TensorData; + use burn_core::tensor::Tolerance; + use rstest::rstest; + + #[rstest] + #[case::binary_b1(1.0, THRESHOLD, 0.5)] + #[case::binary_b2(2.0, THRESHOLD, 0.5)] + fn test_binary_fscore(#[case] beta: f64, #[case] threshold: f64, #[case] expected: f64) { + let input = dummy_classification_input(&ClassificationType::Binary).into(); + let mut metric = FBetaScoreMetric::binary(beta, threshold); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[rstest] + #[case::multiclass_b1_micro_k1(1.0, Micro, 1, 3.0/5.0)] + #[case::multiclass_b1_micro_k2(1.0, Micro, 2, 2.0/(5.0/4.0 + 10.0/4.0))] + #[case::multiclass_b1_macro_k1(1.0, Macro, 1, (0.5 + 2.0/(1.0 + 2.0) + 2.0/(2.0 + 1.0))/3.0)] + #[case::multiclass_b1_macro_k2(1.0, Macro, 2, (2.0/(1.0 + 2.0) + 2.0/(1.0 + 4.0) + 0.5)/3.0)] + #[case::multiclass_b2_micro_k1(2.0, Micro, 1, 3.0/5.0)] + #[case::multiclass_b2_micro_k2(2.0, Micro, 2, 5.0*4.0/(4.0*5.0 + 10.0))] + #[case::multiclass_b2_macro_k1(2.0, Macro, 1, (0.5 + 5.0/(4.0 + 2.0) + 5.0/(8.0 + 1.0))/3.0)] + #[case::multiclass_b2_macro_k2(2.0, Macro, 2, (5.0/(4.0 + 2.0) + 5.0/(4.0 + 4.0) + 0.5)/3.0)] + fn test_multiclass_fscore( + #[case] beta: f64, + #[case] class_reduction: ClassReduction, + #[case] top_k: usize, + #[case] expected: f64, + ) { + let input = dummy_classification_input(&ClassificationType::Multiclass).into(); + let mut metric = FBetaScoreMetric::multiclass(beta, top_k, class_reduction); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[rstest] + #[case::multilabel_micro(1.0, Micro, THRESHOLD, 2.0/(9.0/5.0 + 8.0/5.0))] + #[case::multilabel_macro(1.0, Macro, THRESHOLD, (2.0/(2.0 + 3.0/2.0) + 2.0/(1.0 + 3.0/2.0) + 2.0/(3.0+2.0))/3.0)] + #[case::multilabel_micro(2.0, Micro, THRESHOLD, 5.0/(4.0*9.0/5.0 + 8.0/5.0))] + #[case::multilabel_macro(2.0, Macro, THRESHOLD, (5.0/(8.0 + 3.0/2.0) + 5.0/(4.0 + 3.0/2.0) + 5.0/(12.0+2.0))/3.0)] + fn test_multilabel_fscore( + #[case] beta: f64, + #[case] class_reduction: ClassReduction, + #[case] threshold: f64, + #[case] expected: f64, + ) { + let input = dummy_classification_input(&ClassificationType::Multilabel).into(); + let mut metric = FBetaScoreMetric::multilabel(beta, threshold, class_reduction); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[test] + fn test_parameterized_unique_name() { + let metric_a = FBetaScoreMetric::::multiclass(0.5, 1, ClassReduction::Macro); + let metric_b = FBetaScoreMetric::::multiclass(0.5, 2, ClassReduction::Macro); + let metric_c = FBetaScoreMetric::::multiclass(0.5, 1, ClassReduction::Macro); + + assert_ne!(metric_a.name(), metric_b.name()); + assert_eq!(metric_a.name(), metric_c.name()); + + let metric_a = FBetaScoreMetric::::binary(0.5, 0.5); + let metric_b = FBetaScoreMetric::::binary(0.75, 0.5); + assert_ne!(metric_a.name(), metric_b.name()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/hamming.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/hamming.rs new file mode 100644 index 0000000..f162e9e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/hamming.rs @@ -0,0 +1,198 @@ +use core::marker::PhantomData; +use std::sync::Arc; + +use super::state::{FormatOptions, NumericMetricState}; +use super::{MetricMetadata, SerializedEntry}; +use crate::metric::{ + Metric, MetricAttributes, MetricName, Numeric, NumericAttributes, NumericEntry, +}; +use burn_core::tensor::{ElementConversion, Int, Tensor, activation::sigmoid, backend::Backend}; + +/// The hamming score, sometimes referred to as multi-label or label-based accuracy. +#[derive(Clone)] +pub struct HammingScore { + name: MetricName, + state: NumericMetricState, + threshold: f32, + sigmoid: bool, + _b: PhantomData, +} + +/// The [hamming score](HammingScore) input type. +#[derive(new)] +pub struct HammingScoreInput { + outputs: Tensor, + targets: Tensor, +} + +impl HammingScore { + /// Creates the metric. + pub fn new() -> Self { + Self::default() + } + + fn update_name(&mut self) { + self.name = Arc::new(format!("Hamming Score @ Threshold({})", self.threshold)); + } + + /// Sets the threshold. + pub fn with_threshold(mut self, threshold: f32) -> Self { + self.threshold = threshold; + self.update_name(); + self + } + + /// Sets the sigmoid activation function usage. + pub fn with_sigmoid(mut self, sigmoid: bool) -> Self { + self.sigmoid = sigmoid; + self.update_name(); + self + } +} + +impl Default for HammingScore { + /// Creates a new metric instance with default values. + fn default() -> Self { + let threshold = 0.5; + let name = Arc::new(format!("Hamming Score @ Threshold({})", threshold)); + + Self { + name, + state: NumericMetricState::default(), + threshold, + sigmoid: false, + _b: PhantomData, + } + } +} + +impl Metric for HammingScore { + type Input = HammingScoreInput; + + fn update( + &mut self, + input: &HammingScoreInput, + _metadata: &MetricMetadata, + ) -> SerializedEntry { + let [batch_size, _n_classes] = input.outputs.dims(); + + let targets = input.targets.clone(); + + let mut outputs = input.outputs.clone(); + + if self.sigmoid { + outputs = sigmoid(outputs); + } + + let score = outputs + .greater_elem(self.threshold) + .equal(targets.bool()) + .float() + .mean() + .into_scalar() + .elem::(); + + self.state.update( + 100.0 * score, + batch_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for HammingScore { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn test_hamming_score() { + let device = Default::default(); + let mut metric = HammingScore::::new(); + + let x = Tensor::from_data( + [ + [0.32, 0.52, 0.38, 0.68, 0.61], // with x > 0.5: [0, 1, 0, 1, 1] + [0.43, 0.31, 0.21, 0.63, 0.53], // [0, 0, 0, 1, 1] + [0.44, 0.25, 0.71, 0.39, 0.73], // [0, 0, 1, 0, 1] + [0.49, 0.37, 0.68, 0.39, 0.31], // [0, 0, 1, 0, 0] + ], + &device, + ); + let y = Tensor::from_data( + [ + [0, 1, 0, 1, 1], + [0, 0, 0, 1, 1], + [0, 0, 1, 0, 1], + [0, 0, 1, 0, 0], + ], + &device, + ); + + let _entry = metric.update( + &HammingScoreInput::new(x.clone(), y.clone()), + &MetricMetadata::fake(), + ); + assert_eq!(100.0, metric.value().current()); + + // Invert all targets: y = (1 - y) + let y = y.neg().add_scalar(1); + let _entry = metric.update( + &HammingScoreInput::new(x.clone(), y), // invert targets (1 - y) + &MetricMetadata::fake(), + ); + assert_eq!(0.0, metric.value().current()); + + // Invert 5 target values -> 1 - (5/20) = 0.75 + let y = Tensor::from_data( + [ + [0, 1, 1, 0, 1], + [0, 0, 0, 0, 1], + [0, 0, 0, 0, 1], + [0, 1, 1, 0, 0], + ], + &device, + ); + let _entry = metric.update( + &HammingScoreInput::new(x, y), // invert targets (1 - y) + &MetricMetadata::fake(), + ); + assert_eq!(75.0, metric.value().current()); + } + + #[test] + fn test_parameterized_unique_name() { + let metric_a = HammingScore::::new().with_threshold(0.5); + let metric_b = HammingScore::::new().with_threshold(0.75); + let metric_c = HammingScore::::new().with_threshold(0.5); + + assert_ne!(metric_a.name(), metric_b.name()); + assert_eq!(metric_a.name(), metric_c.name()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/iteration.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/iteration.rs new file mode 100644 index 0000000..077ddde --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/iteration.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use super::MetricMetadata; +use super::SerializedEntry; +use super::state::FormatOptions; +use super::state::NumericMetricState; +use crate::metric::MetricName; +use crate::metric::Numeric; +use crate::metric::{Metric, MetricAttributes, NumericAttributes, NumericEntry}; + +/// The loss metric. +#[derive(Clone)] +pub struct IterationSpeedMetric { + name: MetricName, + state: NumericMetricState, + instant: Option, +} + +impl Default for IterationSpeedMetric { + fn default() -> Self { + Self::new() + } +} + +impl IterationSpeedMetric { + /// Create the metric. + pub fn new() -> Self { + Self { + name: Arc::new("Iteration Speed".to_string()), + state: Default::default(), + instant: Default::default(), + } + } +} + +impl Metric for IterationSpeedMetric { + type Input = (); + + fn update(&mut self, _: &Self::Input, metadata: &MetricMetadata) -> SerializedEntry { + let raw = match self.instant { + Some(val) => { + // If iteration is not logged, compute the speed over the number of items processed. + // 1 iteration should equal 1 item when iteration is not logged. + metadata + .iteration + .unwrap_or(metadata.progress.items_processed) as f64 + / val.elapsed().as_secs_f64() + } + None => { + self.instant = Some(std::time::Instant::now()); + 0.0 + } + }; + + self.state.update( + raw, + 1, + FormatOptions::new(self.name()) + .unit("iter/sec") + .precision(2), + ) + } + + fn clear(&mut self) { + self.instant = None; + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("iter/sec".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for IterationSpeedMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/learning_rate.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/learning_rate.rs new file mode 100644 index 0000000..bbb30ff --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/learning_rate.rs @@ -0,0 +1,67 @@ +use std::sync::Arc; + +use super::{ + MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, + state::{FormatOptions, NumericMetricState}, +}; +use crate::metric::{Metric, MetricName, Numeric, SerializedEntry}; + +/// Track the learning rate across iterations. +#[derive(Clone)] +pub struct LearningRateMetric { + name: MetricName, + state: NumericMetricState, +} + +impl LearningRateMetric { + /// Creates a new learning rate metric. + pub fn new() -> Self { + Self { + name: Arc::new("Learning Rate".to_string()), + state: NumericMetricState::new(), + } + } +} + +impl Default for LearningRateMetric { + fn default() -> Self { + Self::new() + } +} + +impl Metric for LearningRateMetric { + type Input = (); + + fn update(&mut self, _item: &(), metadata: &MetricMetadata) -> SerializedEntry { + let lr = metadata.lr.unwrap_or(0.0); + + self.state + .update(lr, 1, FormatOptions::new(self.name()).precision(2)) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: None, + higher_is_better: false, + } + .into() + } +} + +impl Numeric for LearningRateMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/loss.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/loss.rs new file mode 100644 index 0000000..8b61587 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/loss.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use super::MetricMetadata; +use super::SerializedEntry; +use super::state::FormatOptions; +use super::state::NumericMetricState; +use crate::metric::MetricName; +use crate::metric::{Metric, MetricAttributes, Numeric, NumericAttributes, NumericEntry}; +use burn_core::tensor::Tensor; +use burn_core::tensor::backend::Backend; + +/// The loss metric. +#[derive(Clone)] +pub struct LossMetric { + name: Arc, + state: NumericMetricState, + _b: B, +} + +/// The [loss metric](LossMetric) input type. +#[derive(new)] +pub struct LossInput { + tensor: Tensor, +} + +impl Default for LossMetric { + fn default() -> Self { + Self::new() + } +} + +impl LossMetric { + /// Create the metric. + pub fn new() -> Self { + Self { + name: Arc::new("Loss".to_string()), + state: NumericMetricState::default(), + _b: Default::default(), + } + } +} + +impl Metric for LossMetric { + type Input = LossInput; + + fn update(&mut self, loss: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + let [batch_size] = loss.tensor.dims(); + let loss = loss + .tensor + .clone() + .mean() + .into_data() + .iter::() + .next() + .unwrap(); + + self.state.update( + loss, + batch_size, + FormatOptions::new(self.name()).precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: None, + higher_is_better: false, + } + .into() + } +} + +impl Numeric for LossMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/memory_use.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/memory_use.rs new file mode 100644 index 0000000..364887b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/memory_use.rs @@ -0,0 +1,111 @@ +/// RAM use metric +use super::{MetricAttributes, MetricMetadata, NumericAttributes}; +use crate::metric::{Metric, Numeric, NumericEntry, SerializedEntry}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use sysinfo::System; + +/// Memory information +pub struct CpuMemory { + name: Arc, + last_refresh: Instant, + refresh_frequency: Duration, + sys: System, + ram_bytes_total: u64, + ram_bytes_used: u64, +} + +impl Clone for CpuMemory { + fn clone(&self) -> Self { + Self { + name: self.name.clone(), + last_refresh: self.last_refresh, + refresh_frequency: self.refresh_frequency, + sys: System::new(), + ram_bytes_total: self.ram_bytes_total, + ram_bytes_used: self.ram_bytes_used, + } + } +} + +impl CpuMemory { + /// Creates a new memory metric + pub fn new() -> Self { + let mut metric = Self { + name: Arc::new("CPU Memory".into()), + last_refresh: Instant::now(), + refresh_frequency: Duration::from_millis(200), + sys: System::new(), + ram_bytes_total: 0, + ram_bytes_used: 0, + }; + metric.refresh(); + metric + } + + fn refresh(&mut self) { + self.sys.refresh_memory(); + self.last_refresh = Instant::now(); + + // bytes of RAM available + self.ram_bytes_total = self.sys.total_memory(); + + // bytes of RAM in use + self.ram_bytes_used = self.sys.used_memory(); + } +} + +impl Default for CpuMemory { + fn default() -> Self { + CpuMemory::new() + } +} + +impl Metric for CpuMemory { + type Input = (); + + fn update(&mut self, _item: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + if self.last_refresh.elapsed() >= self.refresh_frequency { + self.refresh(); + } + + let raw = bytes2gb(self.ram_bytes_used); + let formatted = format!( + "RAM Used: {:.2} / {:.2} Gb", + raw, + bytes2gb(self.ram_bytes_total), + ); + + SerializedEntry::new(formatted, raw.to_string()) + } + + fn clear(&mut self) {} + + fn name(&self) -> Arc { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("Gb".to_string()), + higher_is_better: false, + } + .into() + } +} + +impl Numeric for CpuMemory { + fn value(&self) -> NumericEntry { + NumericEntry::Value(bytes2gb(self.ram_bytes_used)) + } + + fn running_value(&self) -> NumericEntry { + NumericEntry::Value(bytes2gb(self.ram_bytes_used)) + } +} + +fn bytes2gb(bytes: u64) -> f64 { + bytes as f64 / 1e9 +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/mod.rs new file mode 100644 index 0000000..5b2b14b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/mod.rs @@ -0,0 +1,71 @@ +/// State module. +pub mod state; +/// Module responsible to save and exposes data collected during training. +pub mod store; +/// Metrics module for vision tasks. +#[cfg(feature = "vision")] +pub mod vision; + +//Metrics for reinforcement learning. +#[cfg(feature = "rl")] +mod rl; +#[cfg(feature = "rl")] +pub use rl::*; + +// System metrics +#[cfg(feature = "sys-metrics")] +mod cpu_temp; +#[cfg(feature = "sys-metrics")] +mod cpu_use; +#[cfg(feature = "sys-metrics")] +mod cuda; +#[cfg(feature = "sys-metrics")] +mod memory_use; +#[cfg(feature = "sys-metrics")] +pub use cpu_temp::*; +#[cfg(feature = "sys-metrics")] +pub use cpu_use::*; +#[cfg(feature = "sys-metrics")] +pub use cuda::*; +#[cfg(feature = "sys-metrics")] +pub use memory_use::*; + +// Training metrics +mod acc; +mod auroc; +mod base; +mod cer; +mod confusion_stats; +mod fbetascore; +mod hamming; +mod iteration; +mod learning_rate; +mod loss; +mod perplexity; +mod precision; +mod recall; +mod top_k_acc; +mod wer; + +pub use acc::*; +pub use auroc::*; +pub use base::*; +pub use cer::*; +pub use confusion_stats::ConfusionStatsInput; +pub use fbetascore::*; +pub use hamming::*; +pub use iteration::*; +pub use learning_rate::*; +pub use loss::*; +pub use perplexity::*; +pub use precision::*; +pub use recall::*; +pub use top_k_acc::*; +pub use wer::*; + +pub(crate) mod classification; +pub(crate) mod processor; + +pub use crate::metric::classification::ClassReduction; +// Expose `ItemLazy` so it can be implemented for custom types +pub use processor::ItemLazy; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/perplexity.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/perplexity.rs new file mode 100644 index 0000000..f0a4426 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/perplexity.rs @@ -0,0 +1,438 @@ +use core::marker::PhantomData; + +use super::state::FormatOptions; +use super::{MetricMetadata, NumericEntry, SerializedEntry, format_float}; +use crate::metric::{Metric, MetricAttributes, MetricName, Numeric, NumericAttributes}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{ElementConversion, Int, Tensor}; + +/// Custom state for perplexity metric that correctly accumulates negative log-likelihood. +/// +/// Unlike other metrics that can be averaged, perplexity requires special handling: +/// - Accumulate total negative log-likelihood across all tokens +/// - Accumulate total number of effective tokens +/// - Compute perplexity as exp(total_nll / total_tokens) only at the end +#[derive(Clone)] +struct PerplexityState { + /// Sum of negative log-likelihood across all tokens + sum_nll: f64, + /// Total number of effective tokens (excluding padding) + total_tokens: usize, + /// Current batch perplexity (for display purposes) + current: f64, +} + +impl PerplexityState { + fn new() -> Self { + Self { + sum_nll: 0.0, + total_tokens: 0, + current: f64::NAN, + } + } + + fn reset(&mut self) { + self.sum_nll = 0.0; + self.total_tokens = 0; + self.current = f64::NAN; + } + + /// Update state with negative log-likelihood and token count from current batch + fn update( + &mut self, + sum_log_prob: f64, + effective_tokens: usize, + format: FormatOptions, + ) -> SerializedEntry { + // sum_log_prob is already the sum of log probabilities (negative values) + // We need to negate it to get negative log-likelihood + let batch_nll = -sum_log_prob; + + // Accumulate across batches + self.sum_nll += batch_nll; + self.total_tokens += effective_tokens; + + // Compute current batch perplexity for display + let batch_perplexity = if effective_tokens > 0 { + (batch_nll / effective_tokens as f64).exp() + } else { + f64::INFINITY + }; + self.current = batch_perplexity; + + // Compute running epoch perplexity + let epoch_perplexity = if self.total_tokens > 0 { + (self.sum_nll / self.total_tokens as f64).exp() + } else { + f64::INFINITY + }; + + // Format for display + let (formatted_current, formatted_running) = match format.precision_value() { + Some(precision) => ( + format_float(batch_perplexity, precision), + format_float(epoch_perplexity, precision), + ), + None => (format!("{batch_perplexity}"), format!("{epoch_perplexity}")), + }; + + let formatted = match format.unit_value() { + Some(unit) => { + format!("epoch {formatted_running} {unit} - batch {formatted_current} {unit}") + } + None => format!("epoch {formatted_running} - batch {formatted_current}"), + }; + + // Serialize the state for aggregation + let serialized = NumericEntry::Aggregated { + aggregated_value: epoch_perplexity, + count: self.total_tokens, + } + .serialize(); + + SerializedEntry::new(formatted, serialized) + } + + fn value(&self) -> NumericEntry { + let perplexity = if self.total_tokens > 0 { + (self.sum_nll / self.total_tokens as f64).exp() + } else { + f64::INFINITY + }; + + NumericEntry::Aggregated { + aggregated_value: perplexity, + count: self.total_tokens, + } + } + + fn running_value(&self) -> NumericEntry { + self.value() + } +} + +/// The perplexity metric. +/// +/// Perplexity is a measure of how well a probability distribution or probability model +/// predicts a sample. It's commonly used to evaluate language models. A lower perplexity +/// indicates that the model is more confident in its predictions. +/// +/// Mathematically, perplexity is defined as the exponentiation of the cross-entropy loss: +/// PPL = exp(H(p, q)) = exp(-1/N * Σ log(p(x_i))) +/// +/// where: +/// - H(p, q) is the cross-entropy between the true distribution p and predicted distribution q +/// - N is the number of tokens +/// - p(x_i) is the predicted probability of the i-th token +/// +/// # Aggregation +/// Unlike other metrics, perplexity cannot be simply averaged across batches. +/// This implementation correctly accumulates the total negative log-likelihood and +/// total token count across batches, then computes perplexity as exp(total_nll / total_tokens). +#[derive(Clone)] +pub struct PerplexityMetric { + name: MetricName, + state: PerplexityState, + pad_token: Option, + _b: PhantomData, +} + +/// The [perplexity metric](PerplexityMetric) input type. +#[derive(new)] +pub struct PerplexityInput { + /// Logits tensor of shape [batch_size * sequence_length, vocab_size] + outputs: Tensor, + /// Target tokens tensor of shape [batch_size * sequence_length] + targets: Tensor, +} + +impl Default for PerplexityMetric { + fn default() -> Self { + Self::new() + } +} + +impl PerplexityMetric { + /// Creates the metric. + pub fn new() -> Self { + Self { + name: MetricName::new("Perplexity".to_string()), + state: PerplexityState::new(), + pad_token: Default::default(), + _b: PhantomData, + } + } + + /// Sets the pad token to exclude from perplexity calculation. + /// + /// When a pad token is set, predictions for padding tokens are masked out + /// and do not contribute to the perplexity calculation. This is important + /// for variable-length sequences where padding is used. + pub fn with_pad_token(mut self, index: usize) -> Self { + self.pad_token = Some(index); + self + } +} + +impl Metric for PerplexityMetric { + type Input = PerplexityInput; + + fn update( + &mut self, + input: &PerplexityInput, + _metadata: &MetricMetadata, + ) -> SerializedEntry { + let targets = input.targets.clone(); + let outputs = input.outputs.clone(); + + let [total_tokens, _vocab_size] = outputs.dims(); + + // Convert logits to log probabilities using log_softmax for numerical stability + let log_probs = burn_core::tensor::activation::log_softmax(outputs, 1); + + // Gather the log probabilities for the target tokens + let target_log_probs = log_probs + .gather(1, targets.clone().unsqueeze_dim(1)) + .squeeze_dim(1); + + let (sum_log_prob, effective_tokens) = match self.pad_token { + Some(pad_token) => { + // Create a mask for non-padding tokens + let mask = targets.clone().not_equal_elem(pad_token as i64); + + // Apply mask to log probabilities (set padding log probs to 0) + let masked_log_probs = target_log_probs.mask_fill(mask.clone().bool_not(), 0.0); + + // Sum the log probabilities and count effective tokens + let sum_log_prob = masked_log_probs.sum().into_scalar().elem::(); + let effective_tokens = mask.int().sum().into_scalar().elem::() as usize; + + (sum_log_prob, effective_tokens) + } + None => { + // No padding, use all tokens + let sum_log_prob = target_log_probs.sum().into_scalar().elem::(); + (sum_log_prob, total_tokens) + } + }; + + // Pass the sum_log_prob and effective_tokens to the state + // The state will handle the correct accumulation and perplexity calculation + self.state.update( + sum_log_prob, + effective_tokens, + FormatOptions::new(self.name()).precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: None, + higher_is_better: false, + } + .into() + } +} + +impl Numeric for PerplexityMetric { + fn value(&self) -> NumericEntry { + self.state.value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn test_perplexity_perfect_prediction() { + let device = Default::default(); + let mut metric = PerplexityMetric::::new(); + + // Perfect prediction: target is always the highest probability class + let input = PerplexityInput::new( + Tensor::from_data( + [ + [10.0, 0.0, 0.0], // Very confident prediction for class 0 + [0.0, 10.0, 0.0], // Very confident prediction for class 1 + [0.0, 0.0, 10.0], // Very confident prediction for class 2 + ], + &device, + ), + Tensor::from_data([0, 1, 2], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + let perplexity = metric.value().current(); + + // Perfect predictions should result in very low perplexity (close to 1.0) + assert!( + perplexity < 1.1, + "Perfect predictions should have low perplexity, got {}", + perplexity + ); + } + + #[test] + fn test_perplexity_uniform_prediction() { + let device = Default::default(); + let mut metric = PerplexityMetric::::new(); + + // Uniform prediction: all classes have equal probability + let input = PerplexityInput::new( + Tensor::from_data( + [ + [0.0, 0.0, 0.0], // Uniform distribution (after softmax) + [0.0, 0.0, 0.0], // Uniform distribution (after softmax) + [0.0, 0.0, 0.0], // Uniform distribution (after softmax) + ], + &device, + ), + Tensor::from_data([0, 1, 2], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + let perplexity = metric.value().current(); + + // Uniform distribution over 3 classes should have perplexity ≈ 3.0 + assert!( + (perplexity - 3.0).abs() < 0.1, + "Uniform distribution perplexity should be ~3.0, got {}", + perplexity + ); + } + + #[test] + fn test_perplexity_with_padding() { + let device = Default::default(); + let mut metric = PerplexityMetric::::new().with_pad_token(3); + + let input = PerplexityInput::new( + Tensor::from_data( + [ + [10.0, 0.0, 0.0, 0.0], // Good prediction for class 0 + [0.0, 10.0, 0.0, 0.0], // Good prediction for class 1 + [0.0, 0.0, 0.0, 1.0], // This is padding - should be ignored + [0.0, 0.0, 0.0, 1.0], // This is padding - should be ignored + ], + &device, + ), + Tensor::from_data([0, 1, 3, 3], &device), // 3 is pad token + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + let perplexity = metric.value().current(); + + // Should only consider the first two predictions, both of which are confident + assert!( + perplexity < 1.1, + "Good predictions with padding should have low perplexity, got {}", + perplexity + ); + } + + #[test] + fn test_perplexity_wrong_prediction() { + let device = Default::default(); + let mut metric = PerplexityMetric::::new(); + + // Wrong predictions: target class has very low probability + let input = PerplexityInput::new( + Tensor::from_data( + [ + [0.0, 10.0, 0.0], // Predicts class 1, but target is 0 + [10.0, 0.0, 0.0], // Predicts class 0, but target is 1 + [0.0, 0.0, 10.0], // Predicts class 2, but target is 0 + ], + &device, + ), + Tensor::from_data([0, 1, 0], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + let perplexity = metric.value().current(); + + // Wrong predictions should result in high perplexity + assert!( + perplexity > 10.0, + "Wrong predictions should have high perplexity, got {}", + perplexity + ); + } + + #[test] + fn test_perplexity_multi_batch_aggregation() { + let device = Default::default(); + let mut metric = PerplexityMetric::::new(); + + // First batch: 2 tokens with uniform distribution (log_prob ≈ -1.0986 each) + let input1 = PerplexityInput::new( + Tensor::from_data( + [ + [0.0, 0.0, 0.0], // Uniform distribution (log_prob ≈ -1.0986) + [0.0, 0.0, 0.0], // Uniform distribution (log_prob ≈ -1.0986) + ], + &device, + ), + Tensor::from_data([0, 1], &device), + ); + + // Second batch: 1 token with uniform distribution + let input2 = PerplexityInput::new( + Tensor::from_data( + [ + [0.0, 0.0, 0.0], // Uniform distribution (log_prob ≈ -1.0986) + ], + &device, + ), + Tensor::from_data([2], &device), + ); + + // Update with both batches + let _entry1 = metric.update(&input1, &MetricMetadata::fake()); + let _entry2 = metric.update(&input2, &MetricMetadata::fake()); + + let aggregated_perplexity = metric.value().current(); + + // For uniform distribution over 3 classes: log_prob ≈ -log(3) ≈ -1.0986 + // Total negative log-likelihood: 3 * 1.0986 ≈ 3.2958 + // Total tokens: 3 + // Expected perplexity: exp(3.2958 / 3) = exp(1.0986) ≈ 3.0 + assert!( + (aggregated_perplexity - 3.0).abs() < 0.1, + "Multi-batch aggregated perplexity should be ~3.0, got {}", + aggregated_perplexity + ); + + // Compare with single batch containing all data + let mut single_batch_metric = PerplexityMetric::::new(); + let single_input = PerplexityInput::new( + Tensor::from_data([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], &device), + Tensor::from_data([0, 1, 2], &device), + ); + + let _single_entry = single_batch_metric.update(&single_input, &MetricMetadata::fake()); + let single_batch_perplexity = single_batch_metric.value().current(); + + // Multi-batch and single-batch should give the same result + assert!( + (aggregated_perplexity - single_batch_perplexity).abs() < 0.01, + "Multi-batch ({}) and single-batch ({}) perplexity should match", + aggregated_perplexity, + single_batch_perplexity + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/precision.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/precision.rs new file mode 100644 index 0000000..66816a7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/precision.rs @@ -0,0 +1,231 @@ +use crate::metric::{MetricName, Numeric}; + +use super::{ + Metric, MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, SerializedEntry, + classification::{ClassReduction, ClassificationMetricConfig, DecisionRule}, + confusion_stats::{ConfusionStats, ConfusionStatsInput}, + state::{FormatOptions, NumericMetricState}, +}; +use burn_core::{ + prelude::{Backend, Tensor}, + tensor::cast::ToElement, +}; +use core::marker::PhantomData; +use std::{num::NonZeroUsize, sync::Arc}; + +/// The Precision Metric +#[derive(Clone)] +pub struct PrecisionMetric { + name: MetricName, + state: NumericMetricState, + _b: PhantomData, + config: ClassificationMetricConfig, +} + +impl Default for PrecisionMetric { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl PrecisionMetric { + fn new(config: ClassificationMetricConfig) -> Self { + let state = Default::default(); + let name = Arc::new(format!( + "Precision @ {:?} [{:?}]", + config.decision_rule, config.class_reduction + )); + + Self { + state, + config, + name, + _b: Default::default(), + } + } + /// Precision metric for binary classification. + /// + /// # Arguments + /// + /// * `threshold` - The threshold to transform a probability into a binary prediction. + #[allow(dead_code)] + pub fn binary(threshold: f64) -> Self { + Self::new(ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + // binary classification results are the same independently of class_reduction + ..Default::default() + }) + } + + /// Precision metric for multiclass classification. + /// + /// # Arguments + /// + /// * `top_k` - The number of highest predictions considered to find the correct label (typically `1`). + /// * `class_reduction` - [Class reduction](ClassReduction) type. + #[allow(dead_code)] + pub fn multiclass(top_k: usize, class_reduction: ClassReduction) -> Self { + Self::new(ClassificationMetricConfig { + decision_rule: DecisionRule::TopK( + NonZeroUsize::new(top_k).expect("top_k must be non-zero"), + ), + class_reduction, + }) + } + + /// Precision metric for multi-label classification. + /// + /// # Arguments + /// + /// * `threshold` - The threshold to transform a probability into a binary value. + /// * `class_reduction` - [Class reduction](ClassReduction) type. + #[allow(dead_code)] + pub fn multilabel(threshold: f64, class_reduction: ClassReduction) -> Self { + Self { + config: ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + class_reduction, + }, + ..Default::default() + } + } + + fn class_average(&self, mut aggregated_metric: Tensor) -> f64 { + use ClassReduction::{Macro, Micro}; + let avg_tensor = match self.config.class_reduction { + Micro => aggregated_metric, + Macro => { + if aggregated_metric + .clone() + .contains_nan() + .any() + .into_scalar() + .to_bool() + { + let nan_mask = aggregated_metric.clone().is_nan(); + aggregated_metric = aggregated_metric + .clone() + .select(0, nan_mask.bool_not().argwhere().squeeze_dim(1)) + } + aggregated_metric.mean() + } + }; + avg_tensor.into_scalar().to_f64() + } +} + +impl Metric for PrecisionMetric { + type Input = ConfusionStatsInput; + + fn update(&mut self, input: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + let [sample_size, _] = input.predictions.dims(); + + let cf_stats = ConfusionStats::new(input, &self.config); + let metric = + self.class_average(cf_stats.clone().true_positive() / cf_stats.predicted_positive()); + + self.state.update( + 100.0 * metric, + sample_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for PrecisionMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::{ + ClassReduction::{self, *}, + Metric, MetricMetadata, PrecisionMetric, + }; + use crate::metric::Numeric; + use crate::{ + TestBackend, + tests::{ClassificationType, THRESHOLD, dummy_classification_input}, + }; + use burn_core::tensor::TensorData; + use burn_core::tensor::Tolerance; + use rstest::rstest; + + #[rstest] + #[case::binary(THRESHOLD, 0.5)] + fn test_binary_precision(#[case] threshold: f64, #[case] expected: f64) { + let input = dummy_classification_input(&ClassificationType::Binary).into(); + let mut metric = PrecisionMetric::binary(threshold); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[rstest] + #[case::multiclass_micro_k1(Micro, 1, 3.0/5.0)] + #[case::multiclass_micro_k2(Micro, 2, 4.0/10.0)] + #[case::multiclass_macro_k1(Macro, 1, (0.5 + 0.5 + 1.0)/3.0)] + #[case::multiclass_macro_k2(Macro, 2, (0.5 + 1.0/4.0 + 0.5)/3.0)] + fn test_multiclass_precision( + #[case] class_reduction: ClassReduction, + #[case] top_k: usize, + #[case] expected: f64, + ) { + let input = dummy_classification_input(&ClassificationType::Multiclass).into(); + let mut metric = PrecisionMetric::multiclass(top_k, class_reduction); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[rstest] + #[case::multilabel_micro(Micro, THRESHOLD, 5.0/8.0)] + #[case::multilabel_macro(Macro, THRESHOLD, (2.0/3.0 + 2.0/3.0 + 0.5)/3.0)] + fn test_multilabel_precision( + #[case] class_reduction: ClassReduction, + #[case] threshold: f64, + #[case] expected: f64, + ) { + let input = dummy_classification_input(&ClassificationType::Multilabel).into(); + let mut metric = PrecisionMetric::multilabel(threshold, class_reduction); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[test] + fn test_parameterized_unique_name() { + let metric_a = PrecisionMetric::::multiclass(1, ClassReduction::Macro); + let metric_b = PrecisionMetric::::multiclass(2, ClassReduction::Macro); + let metric_c = PrecisionMetric::::multiclass(1, ClassReduction::Macro); + + assert_ne!(metric_a.name(), metric_b.name()); + assert_eq!(metric_a.name(), metric_c.name()); + + let metric_a = PrecisionMetric::::binary(0.5); + let metric_b = PrecisionMetric::::binary(0.75); + assert_ne!(metric_a.name(), metric_b.name()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/async_wrapper.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/async_wrapper.rs new file mode 100644 index 0000000..f6ae8f2 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/async_wrapper.rs @@ -0,0 +1,137 @@ +use crate::metric::processor::{EvaluatorEvent, EventProcessorEvaluation}; + +use super::EventProcessorTraining; +use async_channel::{Receiver, Sender}; + +/// Event processor for the training process. +pub struct AsyncProcessorTraining { + sender: Sender>, +} + +/// Event processor for the model evaluation. +pub struct AsyncProcessorEvaluation { + sender: Sender>, +} + +struct WorkerTraining> { + processor: P, + rec: Receiver>, +} + +struct WorkerEvaluation { + processor: P, + rec: Receiver>, +} + +impl + 'static> + WorkerTraining +{ + pub fn start(processor: P, rec: Receiver>) { + let mut worker = Self { processor, rec }; + + std::thread::spawn(move || { + while let Ok(msg) = worker.rec.recv_blocking() { + match msg { + Message::Train(event) => worker.processor.process_train(event), + Message::Valid(event) => worker.processor.process_valid(event), + Message::Renderer(callback) => { + callback.send_blocking(worker.processor.renderer()).unwrap(); + return; + } + } + } + }); + } +} +impl WorkerEvaluation

{ + pub fn start(processor: P, rec: Receiver>) { + let mut worker = Self { processor, rec }; + + std::thread::spawn(move || { + while let Ok(event) = worker.rec.recv_blocking() { + match event { + EvalMessage::Test(event) => worker.processor.process_test(event), + EvalMessage::Renderer(sender) => { + sender.send_blocking(worker.processor.renderer()).unwrap(); + return; + } + } + } + }); + } +} + +impl AsyncProcessorTraining { + /// Create an event processor for training. + pub fn new + 'static>(processor: P) -> Self { + let (sender, rec) = async_channel::bounded(1); + + WorkerTraining::start(processor, rec); + + Self { sender } + } +} + +impl AsyncProcessorEvaluation

{ + /// Create an event processor for model evaluation. + pub fn new(processor: P) -> Self { + let (sender, rec) = async_channel::bounded(1); + + WorkerEvaluation::start(processor, rec); + + Self { sender } + } +} + +enum Message { + Train(EventTrain), + Valid(EventValid), + Renderer(Sender>), +} + +enum EvalMessage { + Test(EvaluatorEvent), + Renderer(Sender>), +} + +impl EventProcessorTraining for AsyncProcessorTraining { + fn process_train(&mut self, event: ET) { + self.sender.send_blocking(Message::Train(event)).unwrap(); + } + + fn process_valid(&mut self, event: EV) { + self.sender.send_blocking(Message::Valid(event)).unwrap(); + } + + fn renderer(self) -> Box { + let (sender, rec) = async_channel::bounded(1); + self.sender + .send_blocking(Message::Renderer(sender)) + .unwrap(); + + match rec.recv_blocking() { + Ok(value) => value, + Err(err) => panic!("{err:?}"), + } + } +} + +impl EventProcessorEvaluation for AsyncProcessorEvaluation

{ + type ItemTest = P::ItemTest; + + fn process_test(&mut self, event: EvaluatorEvent) { + self.sender.send_blocking(EvalMessage::Test(event)).unwrap(); + } + + fn renderer(self) -> Box { + let (sender, rec) = async_channel::bounded(1); + self.sender + .send_blocking(EvalMessage::Renderer(sender)) + .unwrap(); + + match rec.recv_blocking() { + Ok(value) => value, + Err(err) => panic!("{err:?}"), + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/base.rs new file mode 100644 index 0000000..19a7906 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/base.rs @@ -0,0 +1,126 @@ +use burn_core::data::dataloader::Progress; +use burn_optim::LearningRate; + +use crate::{ + LearnerSummary, + renderer::{EvaluationName, MetricsRenderer}, +}; + +/// Event happening during the training/validation process. +pub enum LearnerEvent { + /// Signal the start of the process (e.g., training start) + Start, + /// Signal that an item have been processed. + ProcessedItem(TrainingItem), + /// Signal the end of an epoch. + EndEpoch(usize), + /// Signal the end of the process (e.g., training end). + End(Option), +} + +/// Event happening during the evaluation process. +pub enum EvaluatorEvent { + /// Signal the start of the process (e.g., evaluation start) + Start, + /// Signal that an item have been processed. + ProcessedItem(EvaluationName, EvaluationItem), + /// Signal the end of the process (e.g., evaluation end). + End(Option), +} + +/// Items that are lazy are not ready to be processed by metrics. +/// +/// We want to sync them on a different thread to avoid blocking training. +pub trait ItemLazy: Send { + /// Item that is properly synced and ready to be processed by metrics. + type ItemSync: Send; + + /// Sync the item. + fn sync(self) -> Self::ItemSync; +} + +/// Process events happening during training and validation. +pub trait EventProcessorTraining: Send { + /// Collect a training event. + fn process_train(&mut self, event: TrainEvent); + /// Collect a validation event. + fn process_valid(&mut self, event: ValidEvent); + /// Returns the renderer used for training. + fn renderer(self) -> Box; +} + +/// Process events happening during evaluation. +pub trait EventProcessorEvaluation: Send { + /// The test item. + type ItemTest: ItemLazy; + + /// Collect a test event. + fn process_test(&mut self, event: EvaluatorEvent); + + /// Returns the renderer used for evaluation. + fn renderer(self) -> Box; +} + +/// A learner item. +#[derive(new)] +pub struct TrainingItem { + /// The item. + pub item: T, + + /// The progress. + pub progress: Progress, + + /// The global progress of the training (e.g. epochs). + pub global_progress: Progress, + + /// The iteration, if it it different from the items processed. + pub iteration: Option, + + /// The learning rate. + pub lr: Option, +} + +impl ItemLazy for TrainingItem { + type ItemSync = TrainingItem; + + fn sync(self) -> Self::ItemSync { + TrainingItem { + item: self.item.sync(), + progress: self.progress, + global_progress: self.global_progress, + iteration: self.iteration, + lr: self.lr, + } + } +} + +/// An evaluation item. +#[derive(new)] +pub struct EvaluationItem { + /// The item. + pub item: T, + + /// The progress. + pub progress: Progress, + + /// The iteration, if it it different from the items processed. + pub iteration: Option, +} + +impl ItemLazy for EvaluationItem { + type ItemSync = EvaluationItem; + + fn sync(self) -> Self::ItemSync { + EvaluationItem { + item: self.item.sync(), + progress: self.progress, + iteration: self.iteration, + } + } +} + +impl ItemLazy for () { + type ItemSync = (); + + fn sync(self) -> Self::ItemSync {} +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/full.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/full.rs new file mode 100644 index 0000000..687f33d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/full.rs @@ -0,0 +1,257 @@ +use super::{EventProcessorTraining, ItemLazy, LearnerEvent, MetricsTraining}; +use crate::metric::processor::{EvaluatorEvent, EventProcessorEvaluation, MetricsEvaluation}; +use crate::metric::store::{EpochSummary, EventStoreClient, Split}; +use crate::renderer::{ + EvaluationProgress, MetricState, MetricsRenderer, ProgressType, TrainingProgress, +}; +use std::sync::Arc; + +/// An [event processor](EventProcessorTraining) that handles: +/// - Computing and storing metrics in an [event store](crate::metric::store::EventStore). +/// - Render metrics using a [metrics renderer](MetricsRenderer). +pub struct FullEventProcessorTraining { + metrics: MetricsTraining, + renderer: Box, + store: Arc, +} + +/// An [event processor](EventProcessorEvaluation) that handles: +/// - Computing and storing metrics in an [event store](crate::metric::store::EventStore). +/// - Render metrics using a [metrics renderer](MetricsRenderer). +pub struct FullEventProcessorEvaluation { + metrics: MetricsEvaluation, + renderer: Box, + store: Arc, +} + +impl FullEventProcessorTraining { + pub(crate) fn new( + metrics: MetricsTraining, + renderer: Box, + store: Arc, + ) -> Self { + Self { + metrics, + renderer, + store, + } + } + + fn progress_indicators(&self, progress: &TrainingProgress) -> Vec { + let mut indicators = vec![]; + indicators.push(ProgressType::Detailed { + tag: String::from("Epoch"), + progress: progress.global_progress.clone(), + }); + + if let Some(iteration) = progress.iteration { + indicators.push(ProgressType::Value { + tag: String::from("Iteration"), + value: iteration, + }); + }; + + if let Some(p) = &progress.progress { + indicators.push(ProgressType::Detailed { + tag: String::from("Items"), + progress: p.clone(), + }); + }; + + indicators + } +} + +impl FullEventProcessorEvaluation { + pub(crate) fn new( + metrics: MetricsEvaluation, + renderer: Box, + store: Arc, + ) -> Self { + Self { + metrics, + renderer, + store, + } + } + + fn progress_indicators(&self, progress: &EvaluationProgress) -> Vec { + let mut indicators = vec![]; + if let Some(iteration) = progress.iteration { + indicators.push(ProgressType::Value { + tag: String::from("Iteration"), + value: iteration, + }); + }; + + indicators.push(ProgressType::Detailed { + tag: String::from("Items"), + progress: progress.progress.clone(), + }); + + indicators + } +} + +impl EventProcessorEvaluation for FullEventProcessorEvaluation { + type ItemTest = T; + + fn process_test(&mut self, event: EvaluatorEvent) { + match event { + EvaluatorEvent::Start => { + let definitions = self.metrics.metric_definitions(); + self.store + .add_event_train(crate::metric::store::Event::MetricsInit( + definitions.clone(), + )); + definitions + .iter() + .for_each(|definition| self.renderer.register_metric(definition.clone())); + } + EvaluatorEvent::ProcessedItem(name, item) => { + let item = item.sync(); + let progress = (&item).into(); + let metadata = (&item).into(); + + let update = self.metrics.update_test(&item, &metadata); + + self.store.add_event_test( + crate::metric::store::Event::MetricsUpdate(update.clone()), + name.name.clone(), + ); + + update.entries.into_iter().for_each(|entry| { + self.renderer + .update_test(name.clone(), MetricState::Generic(entry)) + }); + + update + .entries_numeric + .into_iter() + .for_each(|numeric_update| { + self.renderer.update_test( + name.clone(), + MetricState::Numeric( + numeric_update.entry, + numeric_update.numeric_entry, + ), + ) + }); + + let indicators = self.progress_indicators(&progress); + self.renderer.render_test(progress, indicators); + } + EvaluatorEvent::End(summary) => { + self.renderer.on_test_end(summary).ok(); + } + } + } + + fn renderer(self) -> Box { + self.renderer + } +} + +impl EventProcessorTraining, LearnerEvent> + for FullEventProcessorTraining +{ + fn process_train(&mut self, event: LearnerEvent) { + match event { + LearnerEvent::Start => { + let definitions = self.metrics.metric_definitions(); + self.store + .add_event_train(crate::metric::store::Event::MetricsInit( + definitions.clone(), + )); + definitions + .iter() + .for_each(|definition| self.renderer.register_metric(definition.clone())); + } + LearnerEvent::ProcessedItem(item) => { + let item = item.sync(); + let progress = (&item).into(); + let metadata = (&item).into(); + + let update = self.metrics.update_train(&item, &metadata); + + self.store + .add_event_train(crate::metric::store::Event::MetricsUpdate(update.clone())); + + update + .entries + .into_iter() + .for_each(|entry| self.renderer.update_train(MetricState::Generic(entry))); + + update + .entries_numeric + .into_iter() + .for_each(|numeric_update| { + self.renderer.update_train(MetricState::Numeric( + numeric_update.entry, + numeric_update.numeric_entry, + )) + }); + + let indicators = self.progress_indicators(&progress); + self.renderer.render_train(progress, indicators); + } + LearnerEvent::EndEpoch(epoch) => { + self.store + .add_event_train(crate::metric::store::Event::EndEpoch(EpochSummary::new( + epoch, + Split::Train, + ))); + self.metrics.end_epoch_train(); + } + LearnerEvent::End(summary) => { + self.renderer.on_train_end(summary).ok(); + } + } + } + + fn process_valid(&mut self, event: LearnerEvent) { + match event { + LearnerEvent::Start => {} // no-op for now + LearnerEvent::ProcessedItem(item) => { + let item = item.sync(); + let progress = (&item).into(); + let metadata = (&item).into(); + + let update = self.metrics.update_valid(&item, &metadata); + + self.store + .add_event_valid(crate::metric::store::Event::MetricsUpdate(update.clone())); + + update + .entries + .into_iter() + .for_each(|entry| self.renderer.update_valid(MetricState::Generic(entry))); + + update + .entries_numeric + .into_iter() + .for_each(|numeric_update| { + self.renderer.update_valid(MetricState::Numeric( + numeric_update.entry, + numeric_update.numeric_entry, + )) + }); + + let indicators = self.progress_indicators(&progress); + self.renderer.render_valid(progress, indicators); + } + LearnerEvent::EndEpoch(epoch) => { + self.store + .add_event_valid(crate::metric::store::Event::EndEpoch(EpochSummary::new( + epoch, + Split::Valid, + ))); + self.metrics.end_epoch_valid(); + } + LearnerEvent::End(_) => {} // no-op for now + } + } + fn renderer(self) -> Box { + self.renderer + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/metrics.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/metrics.rs new file mode 100644 index 0000000..935e0cd --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/metrics.rs @@ -0,0 +1,341 @@ +use std::collections::HashMap; + +use super::{ItemLazy, TrainingItem}; +use crate::{ + EvaluationItem, + metric::{ + Adaptor, Metric, MetricDefinition, MetricEntry, MetricId, MetricMetadata, Numeric, + store::{MetricsUpdate, NumericMetricUpdate}, + }, + renderer::{EvaluationProgress, TrainingProgress}, +}; + +pub(crate) struct MetricsTraining { + train: Vec>>, + valid: Vec>>, + train_numeric: Vec>>, + valid_numeric: Vec>>, + metric_definitions: HashMap, +} + +pub(crate) struct MetricsEvaluation { + test: Vec>>, + test_numeric: Vec>>, + metric_definitions: HashMap, +} + +impl Default for MetricsEvaluation { + fn default() -> Self { + Self { + test: Default::default(), + test_numeric: Default::default(), + metric_definitions: HashMap::default(), + } + } +} + +impl Default for MetricsTraining { + fn default() -> Self { + Self { + train: Vec::default(), + valid: Vec::default(), + train_numeric: Vec::default(), + valid_numeric: Vec::default(), + metric_definitions: HashMap::default(), + } + } +} + +impl MetricsEvaluation { + /// Register a testing metric. + pub(crate) fn register_test_metric(&mut self, metric: Me) + where + T::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.test.push(Box::new(metric)) + } + + /// Register a numeric testing metric. + pub(crate) fn register_test_metric_numeric( + &mut self, + metric: Me, + ) where + T::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.test_numeric.push(Box::new(metric)) + } + + fn register_definition(&mut self, metric: &MetricWrapper) { + self.metric_definitions.insert( + metric.id.clone(), + MetricDefinition::new(metric.id.clone(), &metric.metric), + ); + } + + /// Get metric definitions. + pub(crate) fn metric_definitions(&mut self) -> Vec { + self.metric_definitions.values().cloned().collect() + } + + /// Update the testing information from the testing item. + pub(crate) fn update_test( + &mut self, + item: &EvaluationItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.test.len()); + let mut entries_numeric = Vec::with_capacity(self.test_numeric.len()); + + for metric in self.test.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.test_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } +} + +impl MetricsTraining { + /// Register a training metric. + pub(crate) fn register_train_metric(&mut self, metric: Me) + where + T::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.train.push(Box::new(metric)) + } + + /// Register a validation metric. + pub(crate) fn register_valid_metric(&mut self, metric: Me) + where + V::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.valid.push(Box::new(metric)) + } + + /// Register a numeric training metric. + pub(crate) fn register_train_metric_numeric( + &mut self, + metric: Me, + ) where + T::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.train_numeric.push(Box::new(metric)) + } + + /// Register a numeric validation metric. + pub(crate) fn register_valid_metric_numeric(&mut self, metric: Me) + where + V::ItemSync: Adaptor + 'static, + Me: Metric + Numeric + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.valid_numeric.push(Box::new(metric)) + } + + fn register_definition(&mut self, metric: &MetricWrapper) { + self.metric_definitions.insert( + metric.id.clone(), + MetricDefinition::new(metric.id.clone(), &metric.metric), + ); + } + + /// Get metric definitions for all splits + pub(crate) fn metric_definitions(&mut self) -> Vec { + self.metric_definitions.values().cloned().collect() + } + + /// Update the training information from the training item. + pub(crate) fn update_train( + &mut self, + item: &TrainingItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.train.len()); + let mut entries_numeric = Vec::with_capacity(self.train_numeric.len()); + + for metric in self.train.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.train_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } + + /// Update the training information from the validation item. + pub(crate) fn update_valid( + &mut self, + item: &TrainingItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.valid.len()); + let mut entries_numeric = Vec::with_capacity(self.valid_numeric.len()); + + for metric in self.valid.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.valid_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } + + /// Signal the end of a training epoch. + pub(crate) fn end_epoch_train(&mut self) { + for metric in self.train.iter_mut() { + metric.clear(); + } + for metric in self.train_numeric.iter_mut() { + metric.clear(); + } + } + + /// Signal the end of a validation epoch. + pub(crate) fn end_epoch_valid(&mut self) { + for metric in self.valid.iter_mut() { + metric.clear(); + } + for metric in self.valid_numeric.iter_mut() { + metric.clear(); + } + } +} + +impl From<&TrainingItem> for TrainingProgress { + fn from(item: &TrainingItem) -> Self { + Self { + progress: Some(item.progress.clone()), + global_progress: item.global_progress.clone(), + iteration: item.iteration, + } + } +} + +impl From<&EvaluationItem> for TrainingProgress { + fn from(item: &EvaluationItem) -> Self { + Self { + progress: None, + global_progress: item.progress.clone(), + iteration: item.iteration, + } + } +} + +impl From<&EvaluationItem> for EvaluationProgress { + fn from(item: &EvaluationItem) -> Self { + Self { + progress: item.progress.clone(), + iteration: item.iteration, + } + } +} + +impl From<&TrainingItem> for MetricMetadata { + fn from(item: &TrainingItem) -> Self { + Self { + progress: item.progress.clone(), + global_progress: item.global_progress.clone(), + iteration: item.iteration, + lr: item.lr, + } + } +} + +impl From<&EvaluationItem> for MetricMetadata { + fn from(item: &EvaluationItem) -> Self { + Self { + progress: item.progress.clone(), + global_progress: item.progress.clone(), + iteration: item.iteration, + lr: None, + } + } +} + +pub(crate) trait NumericMetricUpdater: Send + Sync { + fn update(&mut self, item: &T, metadata: &MetricMetadata) -> NumericMetricUpdate; + fn clear(&mut self); +} + +pub(crate) trait MetricUpdater: Send + Sync { + fn update(&mut self, item: &T, metadata: &MetricMetadata) -> MetricEntry; + fn clear(&mut self); +} + +pub(crate) struct MetricWrapper { + pub id: MetricId, + pub metric: M, +} + +impl MetricWrapper { + pub fn new(metric: M) -> Self { + Self { + id: MetricId::new(metric.name()), + metric, + } + } +} + +impl NumericMetricUpdater for MetricWrapper +where + T: 'static, + M: Metric + Numeric + 'static, + T: Adaptor, +{ + fn update(&mut self, item: &T, metadata: &MetricMetadata) -> NumericMetricUpdate { + let serialized_entry = self.metric.update(&item.adapt(), metadata); + let update = MetricEntry::new(self.id.clone(), serialized_entry); + let numeric = self.metric.value(); + let running = self.metric.running_value(); + + NumericMetricUpdate { + entry: update, + numeric_entry: numeric, + running_entry: running, + } + } + + fn clear(&mut self) { + self.metric.clear() + } +} + +impl MetricUpdater for MetricWrapper +where + T: 'static, + M: Metric + 'static, + T: Adaptor, +{ + fn update(&mut self, item: &T, metadata: &MetricMetadata) -> MetricEntry { + let serialized_entry = self.metric.update(&item.adapt(), metadata); + MetricEntry::new(self.id.clone(), serialized_entry) + } + + fn clear(&mut self) { + self.metric.clear() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/minimal.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/minimal.rs new file mode 100644 index 0000000..b68d0d5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/minimal.rs @@ -0,0 +1,76 @@ +use super::{EventProcessorTraining, ItemLazy, LearnerEvent, MetricsTraining}; +use crate::{ + metric::store::{EpochSummary, EventStoreClient, Split}, + renderer::cli::CliMetricsRenderer, +}; +use std::sync::Arc; + +/// An [event processor](EventProcessor) that handles: +/// - Computing and storing metrics in an [event store](crate::metric::store::EventStore). +#[allow(dead_code)] +#[derive(new)] +pub(crate) struct MinimalEventProcessor { + metrics: MetricsTraining, + store: Arc, +} + +impl EventProcessorTraining, LearnerEvent> + for MinimalEventProcessor +{ + fn process_train(&mut self, event: LearnerEvent) { + match event { + LearnerEvent::Start => { + let definitions = self.metrics.metric_definitions(); + self.store + .add_event_train(crate::metric::store::Event::MetricsInit(definitions)); + } + + LearnerEvent::ProcessedItem(item) => { + let item = item.sync(); + let metadata = (&item).into(); + + let update = self.metrics.update_train(&item, &metadata); + + self.store + .add_event_train(crate::metric::store::Event::MetricsUpdate(update)); + } + LearnerEvent::EndEpoch(epoch) => { + self.metrics.end_epoch_train(); + self.store + .add_event_train(crate::metric::store::Event::EndEpoch(EpochSummary::new( + epoch, + Split::Train, + ))); + } + LearnerEvent::End(_summary) => {} // no-op for now + } + } + + fn process_valid(&mut self, event: LearnerEvent) { + match event { + LearnerEvent::Start => {} // no-op for now + LearnerEvent::ProcessedItem(item) => { + let item = item.sync(); + let metadata = (&item).into(); + + let update = self.metrics.update_valid(&item, &metadata); + + self.store + .add_event_valid(crate::metric::store::Event::MetricsUpdate(update)); + } + LearnerEvent::EndEpoch(epoch) => { + self.metrics.end_epoch_valid(); + self.store + .add_event_valid(crate::metric::store::Event::EndEpoch(EpochSummary::new( + epoch, + Split::Valid, + ))); + } + LearnerEvent::End(_) => {} // no-op for now + } + } + fn renderer(self) -> Box { + // TODO: Check for another default. + Box::new(CliMetricsRenderer::new()) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/mod.rs new file mode 100644 index 0000000..f0752c6 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/mod.rs @@ -0,0 +1,77 @@ +mod async_wrapper; +mod base; +mod full; +mod metrics; +mod minimal; +#[cfg(feature = "rl")] +mod rl_metrics; +#[cfg(feature = "rl")] +mod rl_processor; + +pub use base::*; +pub(crate) use full::*; +pub(crate) use metrics::*; +#[cfg(feature = "rl")] +pub(crate) use rl_metrics::*; +#[cfg(feature = "rl")] +pub(crate) use rl_processor::*; + +#[cfg(test)] +pub(crate) use minimal::*; + +pub use async_wrapper::{AsyncProcessorEvaluation, AsyncProcessorTraining}; + +#[cfg(test)] +pub(crate) mod test_utils { + use crate::metric::{ + Adaptor, LossInput, + processor::{EventProcessorTraining, LearnerEvent, MinimalEventProcessor, TrainingItem}, + }; + use burn_core::tensor::{ElementConversion, Tensor, backend::Backend}; + + use super::ItemLazy; + + impl ItemLazy for f64 { + type ItemSync = f64; + + fn sync(self) -> Self::ItemSync { + self + } + } + + impl Adaptor> for f64 { + fn adapt(&self) -> LossInput { + let device = B::Device::default(); + LossInput::new(Tensor::from_data([self.elem::()], &device)) + } + } + + pub(crate) fn process_train( + processor: &mut MinimalEventProcessor, + value: f64, + epoch: usize, + ) { + let dummy_progress = burn_core::data::dataloader::Progress { + items_processed: 1, + items_total: 10, + }; + let dummy_global_progress = burn_core::data::dataloader::Progress { + items_processed: epoch, + items_total: 3, + }; + let dummy_iteration = Some(1); + + processor.process_train(LearnerEvent::ProcessedItem(TrainingItem::new( + value, + dummy_progress, + dummy_global_progress, + dummy_iteration, + None, + ))); + } + + pub(crate) fn end_epoch(processor: &mut MinimalEventProcessor, epoch: usize) { + processor.process_train(LearnerEvent::EndEpoch(epoch)); + processor.process_valid(LearnerEvent::EndEpoch(epoch)); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_metrics.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_metrics.rs new file mode 100644 index 0000000..d520aca --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_metrics.rs @@ -0,0 +1,268 @@ +use std::collections::HashMap; + +use crate::{ + EpisodeSummary, EvaluationItem, ItemLazy, MetricUpdater, MetricWrapper, NumericMetricUpdater, + metric::{ + Adaptor, Metric, MetricDefinition, MetricId, MetricMetadata, Numeric, store::MetricsUpdate, + }, +}; + +pub(crate) struct RLMetrics { + train_step: Vec>>, + env_step: Vec>>, + env_step_valid: Vec>>, + episode_end: Vec>>, + episode_end_valid: Vec>>, + + train_step_numeric: Vec>>, + env_step_numeric: Vec>>, + env_step_valid_numeric: Vec>>, + episode_end_numeric: Vec>>, + episode_end_valid_numeric: Vec>>, + + metric_definitions: HashMap, +} + +impl Default for RLMetrics { + fn default() -> Self { + Self { + train_step: Vec::default(), + env_step: Vec::default(), + env_step_valid: Vec::default(), + episode_end: Vec::default(), + episode_end_valid: Vec::default(), + train_step_numeric: Vec::default(), + env_step_numeric: Vec::default(), + env_step_valid_numeric: Vec::default(), + episode_end_numeric: Vec::default(), + episode_end_valid_numeric: Vec::default(), + metric_definitions: HashMap::default(), + } + } +} + +impl RLMetrics { + /// Register a training metric. + pub(crate) fn register_text_metric_agent(&mut self, metric: Me) + where + ES::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.env_step.push(Box::new(metric)) + } + + /// Register a training metric. + pub(crate) fn register_agent_metric(&mut self, metric: Me) + where + ES::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.env_step_numeric.push(Box::new(metric)) + } + + /// Register a training metric. + pub(crate) fn register_text_metric_train(&mut self, metric: Me) + where + TS::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.train_step.push(Box::new(metric)) + } + + /// Register a training metric. + pub(crate) fn register_metric_train(&mut self, metric: Me) + where + TS::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.train_step_numeric.push(Box::new(metric)) + } + + /// Register a validation env-step metric. + pub(crate) fn register_text_metric_agent_valid(&mut self, metric: Me) + where + ES::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.env_step_valid.push(Box::new(metric)) + } + + /// Register a validation env-step numeric metric. + pub(crate) fn register_agent_metric_valid(&mut self, metric: Me) + where + ES::ItemSync: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.env_step_valid_numeric.push(Box::new(metric)) + } + + /// Register an episode-end metric. + pub(crate) fn register_text_metric_episode(&mut self, metric: Me) + where + EpisodeSummary: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.episode_end.push(Box::new(metric)) + } + + /// Register an episode-end numeric metric. + pub(crate) fn register_episode_metric(&mut self, metric: Me) + where + EpisodeSummary: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.episode_end_numeric.push(Box::new(metric)) + } + + /// Register an episode-end metric for validation. + pub(crate) fn register_text_metric_episode_valid(&mut self, metric: Me) + where + EpisodeSummary: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.episode_end_valid.push(Box::new(metric)) + } + + /// Register an episode-end numeric metric for validation. + pub(crate) fn register_episode_metric_valid( + &mut self, + metric: Me, + ) where + EpisodeSummary: Adaptor + 'static, + { + let metric = MetricWrapper::new(metric); + self.register_definition(&metric); + self.episode_end_valid_numeric.push(Box::new(metric)) + } + + fn register_definition(&mut self, metric: &MetricWrapper) { + self.metric_definitions.insert( + metric.id.clone(), + MetricDefinition::new(metric.id.clone(), &metric.metric), + ); + } + + /// Get metric definitions for all splits + pub(crate) fn metric_definitions(&mut self) -> Vec { + self.metric_definitions.values().cloned().collect() + } + + /// Update the training information from the training item. + pub(crate) fn update_train_step( + &mut self, + item: &EvaluationItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.train_step.len()); + let mut entries_numeric = Vec::with_capacity(self.train_step_numeric.len()); + + for metric in self.train_step.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.train_step_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } + + /// Update the env-step metrics from an environment step item. + pub(crate) fn update_env_step( + &mut self, + item: &EvaluationItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.env_step.len()); + let mut entries_numeric = Vec::with_capacity(self.env_step_numeric.len()); + + for metric in self.env_step.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.env_step_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } + + /// Update the env-step metrics for validation from an environment step item. + pub(crate) fn update_env_step_valid( + &mut self, + item: &EvaluationItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.env_step_valid.len()); + let mut entries_numeric = Vec::with_capacity(self.env_step_valid_numeric.len()); + + for metric in self.env_step_valid.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.env_step_valid_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } + + /// Update the episode-end metrics from an episode summary. + pub(crate) fn update_episode_end( + &mut self, + item: &EvaluationItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.episode_end.len()); + let mut entries_numeric = Vec::with_capacity(self.episode_end_numeric.len()); + + for metric in self.episode_end.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.episode_end_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } + + /// Update the episode-end metrics for validation from an episode summary. + pub(crate) fn update_episode_end_valid( + &mut self, + item: &EvaluationItem, + metadata: &MetricMetadata, + ) -> MetricsUpdate { + let mut entries = Vec::with_capacity(self.episode_end_valid.len()); + let mut entries_numeric = Vec::with_capacity(self.episode_end_valid_numeric.len()); + + for metric in self.episode_end_valid.iter_mut() { + let state = metric.update(&item.item, metadata); + entries.push(state); + } + + for metric in self.episode_end_valid_numeric.iter_mut() { + let numeric_update = metric.update(&item.item, metadata); + entries_numeric.push(numeric_update); + } + + MetricsUpdate::new(entries, entries_numeric) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_processor.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_processor.rs new file mode 100644 index 0000000..e2b4b9b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/processor/rl_processor.rs @@ -0,0 +1,177 @@ +use std::sync::Arc; + +use crate::{ + EpisodeSummary, EvaluationItem, EventProcessorTraining, ItemLazy, LearnerSummary, RLMetrics, + metric::store::{Event, EventStoreClient, MetricsUpdate}, + renderer::{MetricState, MetricsRenderer, ProgressType, TrainingProgress}, +}; + +/// Event happening during reinforcement learning. +pub enum RLEvent { + /// Signal the start of the process (e.g., learning starts). + Start, + /// Signal an agent's training step. + TrainStep(EvaluationItem), + /// Signal a timestep of the agent-environment interface. + TimeStep(EvaluationItem), + /// Signal an episode end. + EpisodeEnd(EvaluationItem), + /// Signal the end of the process (e.g., learning ends). + End(Option), +} + +/// Event happening during evaluation of a reinforcement learning's agent. +pub enum AgentEvaluationEvent { + /// Signal the start of the process (e.g., training start) + Start, + /// Signal a timestep of the agent-environment interface. + TimeStep(EvaluationItem), + /// Signal an episode end. + EpisodeEnd(EvaluationItem), + /// Signal the end of the process (e.g., training end). + End, +} + +/// An [event processor](EventProcessorTraining) that handles: +/// - Computing and storing metrics in an [event store](crate::metric::store::EventStore). +/// - Render metrics using a [metrics renderer](MetricsRenderer). +#[derive(new)] +pub struct RLEventProcessor { + metrics: RLMetrics, + renderer: Box, + store: Arc, +} + +impl RLEventProcessor { + fn progress_indicators(&self, progress: &TrainingProgress) -> Vec { + let indicators = vec![ProgressType::Detailed { + tag: String::from("Step"), + progress: progress.global_progress.clone(), + }]; + + indicators + } + + fn progress_indicators_eval(&self, progress: &TrainingProgress) -> Vec { + let indicators = vec![ProgressType::Detailed { + tag: String::from("Step"), + progress: progress.global_progress.clone(), + }]; + + indicators + } +} + +impl RLEventProcessor { + fn process_update_train(&mut self, update: MetricsUpdate) { + self.store + .add_event_train(crate::metric::store::Event::MetricsUpdate(update.clone())); + + update + .entries + .into_iter() + .for_each(|entry| self.renderer.update_train(MetricState::Generic(entry))); + + update + .entries_numeric + .into_iter() + .for_each(|numeric_update| { + self.renderer.update_train(MetricState::Numeric( + numeric_update.entry, + numeric_update.numeric_entry, + )) + }); + } + + fn process_update_valid(&mut self, update: MetricsUpdate) { + self.store + .add_event_valid(crate::metric::store::Event::MetricsUpdate(update.clone())); + + update + .entries + .into_iter() + .for_each(|entry| self.renderer.update_valid(MetricState::Generic(entry))); + + update + .entries_numeric + .into_iter() + .for_each(|numeric_update| { + self.renderer.update_valid(MetricState::Numeric( + numeric_update.entry, + numeric_update.numeric_entry, + )) + }); + } +} + +impl EventProcessorTraining, AgentEvaluationEvent> + for RLEventProcessor +{ + fn process_train(&mut self, event: RLEvent) { + match event { + RLEvent::Start => { + let definitions = self.metrics.metric_definitions(); + self.store + .add_event_train(Event::MetricsInit(definitions.clone())); + definitions + .iter() + .for_each(|definition| self.renderer.register_metric(definition.clone())); + } + RLEvent::TrainStep(item) => { + let item = item.sync(); + let metadata = (&item).into(); + + let update = self.metrics.update_train_step(&item, &metadata); + self.process_update_train(update); + } + RLEvent::TimeStep(item) => { + let item = item.sync(); + let progress = (&item).into(); + let metadata = (&item).into(); + + let update = self.metrics.update_env_step(&item, &metadata); + self.process_update_train(update); + let status = self.progress_indicators(&progress); + self.renderer.render_train(progress, status); + } + RLEvent::EpisodeEnd(item) => { + let item = item.sync(); + let metadata = (&item).into(); + + let update = self.metrics.update_episode_end(&item, &metadata); + self.process_update_train(update); + } + RLEvent::End(learner_summary) => { + self.renderer.on_train_end(learner_summary).ok(); + } + } + } + + fn process_valid(&mut self, event: AgentEvaluationEvent) { + match event { + AgentEvaluationEvent::Start => {} // no-op for now + AgentEvaluationEvent::TimeStep(item) => { + let item = item.sync(); + let metadata = (&item).into(); + + let update = self.metrics.update_env_step_valid(&item, &metadata); + self.process_update_valid(update); + } + AgentEvaluationEvent::EpisodeEnd(item) => { + let item = item.sync(); + let progress = (&item).into(); + let metadata = (&item).into(); + + let update = self.metrics.update_episode_end_valid(&item, &metadata); + self.process_update_valid(update); + let status = self.progress_indicators_eval(&progress); + self.renderer.render_valid(progress, status); + } + AgentEvaluationEvent::End => {} // no-op for now + } + } + + fn renderer(self) -> Box { + self.renderer + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/recall.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/recall.rs new file mode 100644 index 0000000..b037f7e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/recall.rs @@ -0,0 +1,226 @@ +use crate::metric::{MetricName, Numeric}; + +use super::{ + Metric, MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, SerializedEntry, + classification::{ClassReduction, ClassificationMetricConfig, DecisionRule}, + confusion_stats::{ConfusionStats, ConfusionStatsInput}, + state::{FormatOptions, NumericMetricState}, +}; +use burn_core::{ + prelude::{Backend, Tensor}, + tensor::cast::ToElement, +}; +use core::marker::PhantomData; +use std::{num::NonZeroUsize, sync::Arc}; + +///The Recall Metric +#[derive(Clone)] +pub struct RecallMetric { + name: MetricName, + state: NumericMetricState, + _b: PhantomData, + config: ClassificationMetricConfig, +} + +impl Default for RecallMetric { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl RecallMetric { + fn new(config: ClassificationMetricConfig) -> Self { + let state = Default::default(); + let name = Arc::new(format!( + "Recall @ {:?} [{:?}]", + config.decision_rule, config.class_reduction + )); + + Self { + state, + config, + name, + _b: Default::default(), + } + } + /// Recall metric for binary classification. + /// + /// # Arguments + /// + /// * `threshold` - The threshold to transform a probability into a binary prediction. + #[allow(dead_code)] + pub fn binary(threshold: f64) -> Self { + Self::new(ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + // binary classification results are the same independently of class_reduction + ..Default::default() + }) + } + + /// Recall metric for multiclass classification. + /// + /// # Arguments + /// + /// * `top_k` - The number of highest predictions considered to find the correct label (typically `1`). + /// * `class_reduction` - [Class reduction](ClassReduction) type. + #[allow(dead_code)] + pub fn multiclass(top_k: usize, class_reduction: ClassReduction) -> Self { + Self::new(ClassificationMetricConfig { + decision_rule: DecisionRule::TopK( + NonZeroUsize::new(top_k).expect("top_k must be non-zero"), + ), + class_reduction, + }) + } + + /// Recall metric for multi-label classification. + /// + /// # Arguments + /// + /// * `threshold` - The threshold to transform a probability into a binary prediction. + /// * `class_reduction` - [Class reduction](ClassReduction) type. + #[allow(dead_code)] + pub fn multilabel(threshold: f64, class_reduction: ClassReduction) -> Self { + Self::new(ClassificationMetricConfig { + decision_rule: DecisionRule::Threshold(threshold), + class_reduction, + }) + } + + fn class_average(&self, mut aggregated_metric: Tensor) -> f64 { + use ClassReduction::{Macro, Micro}; + let avg_tensor = match self.config.class_reduction { + Micro => aggregated_metric, + Macro => { + if aggregated_metric + .clone() + .contains_nan() + .any() + .into_scalar() + .to_bool() + { + let nan_mask = aggregated_metric.clone().is_nan(); + aggregated_metric = aggregated_metric + .clone() + .select(0, nan_mask.bool_not().argwhere().squeeze_dim(1)) + } + aggregated_metric.mean() + } + }; + avg_tensor.into_scalar().to_f64() + } +} + +impl Metric for RecallMetric { + type Input = ConfusionStatsInput; + + fn update(&mut self, input: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + let [sample_size, _] = input.predictions.dims(); + + let cf_stats = ConfusionStats::new(input, &self.config); + let metric = self.class_average(cf_stats.clone().true_positive() / cf_stats.positive()); + + self.state.update( + 100.0 * metric, + sample_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for RecallMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::{ + ClassReduction::{self, *}, + Metric, MetricMetadata, RecallMetric, + }; + use crate::metric::Numeric; + use crate::{ + TestBackend, + tests::{ClassificationType, THRESHOLD, dummy_classification_input}, + }; + use burn_core::tensor::{TensorData, Tolerance}; + use rstest::rstest; + + #[rstest] + #[case::binary(THRESHOLD, 0.5)] + fn test_binary_recall(#[case] threshold: f64, #[case] expected: f64) { + let input = dummy_classification_input(&ClassificationType::Binary).into(); + let mut metric = RecallMetric::binary(threshold); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[rstest] + #[case::multiclass_micro_k1(Micro, 1, 3.0/5.0)] + #[case::multiclass_micro_k2(Micro, 2, 4.0/5.0)] + #[case::multiclass_macro_k1(Macro, 1, (0.5 + 1.0 + 0.5)/3.0)] + #[case::multiclass_macro_k2(Macro, 2, (1.0 + 1.0 + 0.5)/3.0)] + fn test_multiclass_recall( + #[case] class_reduction: ClassReduction, + #[case] top_k: usize, + #[case] expected: f64, + ) { + let input = dummy_classification_input(&ClassificationType::Multiclass).into(); + let mut metric = RecallMetric::multiclass(top_k, class_reduction); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[rstest] + #[case::multilabel_micro(Micro, THRESHOLD, 5.0/9.0)] + #[case::multilabel_macro(Macro, THRESHOLD, (0.5 + 1.0 + 1.0/3.0)/3.0)] + fn test_multilabel_recall( + #[case] class_reduction: ClassReduction, + #[case] threshold: f64, + #[case] expected: f64, + ) { + let input = dummy_classification_input(&ClassificationType::Multilabel).into(); + let mut metric = RecallMetric::multilabel(threshold, class_reduction); + let _entry = metric.update(&input, &MetricMetadata::fake()); + TensorData::from([metric.value().current()]) + .assert_approx_eq::(&TensorData::from([expected * 100.0]), Tolerance::default()) + } + + #[test] + fn test_parameterized_unique_name() { + let metric_a = RecallMetric::::multiclass(1, ClassReduction::Macro); + let metric_b = RecallMetric::::multiclass(2, ClassReduction::Macro); + let metric_c = RecallMetric::::multiclass(1, ClassReduction::Macro); + + assert_ne!(metric_a.name(), metric_b.name()); + assert_eq!(metric_a.name(), metric_c.name()); + + let metric_a = RecallMetric::::binary(0.5); + let metric_b = RecallMetric::::binary(0.75); + assert_ne!(metric_a.name(), metric_b.name()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/cum_reward.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/cum_reward.rs new file mode 100644 index 0000000..28505e3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/cum_reward.rs @@ -0,0 +1,78 @@ +use std::sync::Arc; + +use super::super::{ + MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, + state::{FormatOptions, NumericMetricState}, +}; +use crate::metric::{Metric, MetricName, Numeric, SerializedEntry}; + +/// Metric for the cumulative reward of the last completed episode. +#[derive(Clone)] +pub struct CumulativeRewardMetric { + name: MetricName, + state: NumericMetricState, +} + +impl CumulativeRewardMetric { + /// Creates a new episode length metric. + pub fn new() -> Self { + Self { + name: Arc::new("Cum. Reward".to_string()), + state: NumericMetricState::new(), + } + } +} + +impl Default for CumulativeRewardMetric { + fn default() -> Self { + Self::new() + } +} + +/// The [CumulativeRewardMetric](CumulativeRewardMetric) input type. +#[derive(new)] +pub struct CumulativeRewardInput { + cum_reward: f64, +} + +impl Metric for CumulativeRewardMetric { + type Input = CumulativeRewardInput; + + fn update( + &mut self, + item: &CumulativeRewardInput, + _metadata: &MetricMetadata, + ) -> SerializedEntry { + self.state.update( + item.cum_reward, + 1, + FormatOptions::new(self.name()).precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: None, + higher_is_better: true, + } + .into() + } +} + +impl Numeric for CumulativeRewardMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/ep_len.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/ep_len.rs new file mode 100644 index 0000000..90d90c4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/ep_len.rs @@ -0,0 +1,71 @@ +use std::sync::Arc; + +use super::super::{ + MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, + state::{FormatOptions, NumericMetricState}, +}; +use crate::metric::{Metric, MetricName, Numeric, SerializedEntry}; + +/// Metric for the length of the last completed episode. +#[derive(Clone)] +pub struct EpisodeLengthMetric { + name: MetricName, + state: NumericMetricState, +} + +impl EpisodeLengthMetric { + /// Creates a new episode length metric. + pub fn new() -> Self { + Self { + name: Arc::new("Episode length".to_string()), + state: NumericMetricState::new(), + } + } +} + +impl Default for EpisodeLengthMetric { + fn default() -> Self { + Self::new() + } +} + +/// The [EpisodeLengthMetric](EpisodeLengthMetric) input type. +#[derive(new)] +pub struct EpisodeLengthInput { + ep_len: f64, +} + +impl Metric for EpisodeLengthMetric { + type Input = EpisodeLengthInput; + + fn update(&mut self, item: &EpisodeLengthInput, _metadata: &MetricMetadata) -> SerializedEntry { + self.state + .update(item.ep_len, 1, FormatOptions::new(self.name()).precision(0)) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some(String::from("steps")), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for EpisodeLengthMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/exploration_rate.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/exploration_rate.rs new file mode 100644 index 0000000..66a7acf --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/exploration_rate.rs @@ -0,0 +1,78 @@ +use std::sync::Arc; + +use super::super::{ + MetricAttributes, MetricMetadata, NumericAttributes, NumericEntry, + state::{FormatOptions, NumericMetricState}, +}; +use crate::metric::{Metric, MetricName, Numeric, SerializedEntry}; + +/// Metric for the length of the last completed episode. +#[derive(Clone)] +pub struct ExplorationRateMetric { + name: MetricName, + state: NumericMetricState, +} + +impl ExplorationRateMetric { + /// Creates a new episode length metric. + pub fn new() -> Self { + Self { + name: Arc::new("Exploration rate".to_string()), + state: NumericMetricState::new(), + } + } +} + +impl Default for ExplorationRateMetric { + fn default() -> Self { + Self::new() + } +} + +/// The [ExplorationRateMetric](ExplorationRateMetric) input type. +#[derive(new)] +pub struct ExplorationRateInput { + exploration_rate: f64, +} + +impl Metric for ExplorationRateMetric { + type Input = ExplorationRateInput; + + fn update( + &mut self, + item: &ExplorationRateInput, + _metadata: &MetricMetadata, + ) -> SerializedEntry { + self.state.update( + item.exploration_rate, + 1, + FormatOptions::new(self.name()).precision(3), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some(String::from("%")), + higher_is_better: false, + } + .into() + } +} + +impl Numeric for ExplorationRateMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/mod.rs new file mode 100644 index 0000000..f2a2d22 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/rl/mod.rs @@ -0,0 +1,7 @@ +mod cum_reward; +mod ep_len; +mod exploration_rate; + +pub use cum_reward::*; +pub use ep_len::*; +pub use exploration_rate::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/state.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/state.rs new file mode 100644 index 0000000..8791fe0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/state.rs @@ -0,0 +1,144 @@ +use std::sync::Arc; + +use crate::metric::{MetricName, NumericEntry, SerializedEntry, format_float}; + +/// Useful utility to implement numeric metrics. +/// +/// # Notes +/// +/// The numeric metric store values inside floats. +/// Even if some metric are integers, their mean are floats. +#[derive(Clone)] +pub struct NumericMetricState { + sum: f64, + count: usize, + current: f64, + current_count: usize, +} + +/// Formatting options for the [numeric metric state](NumericMetricState). +pub struct FormatOptions { + name: Arc, + unit: Option, + precision: Option, +} + +impl FormatOptions { + /// Create the [formatting options](FormatOptions) with a name. + pub fn new(name: MetricName) -> Self { + Self { + name: name.clone(), + unit: None, + precision: None, + } + } + + /// Specify the metric unit. + pub fn unit(mut self, unit: &str) -> Self { + self.unit = Some(unit.to_string()); + self + } + + /// Specify the floating point precision. + pub fn precision(mut self, precision: usize) -> Self { + self.precision = Some(precision); + self + } + + /// Get the metric name. + pub fn name(&self) -> &Arc { + &self.name + } + + /// Get the metric unit. + pub fn unit_value(&self) -> &Option { + &self.unit + } + + /// Get the precision. + pub fn precision_value(&self) -> Option { + self.precision + } +} + +impl NumericMetricState { + /// Create a new [numeric metric state](NumericMetricState). + pub fn new() -> Self { + Self { + sum: 0.0, + count: 0, + current: f64::NAN, + current_count: 0, + } + } + + /// Reset the state. + pub fn reset(&mut self) { + self.sum = 0.0; + self.count = 0; + self.current = f64::NAN; + self.current_count = 0; + } + + /// Update the state. + pub fn update( + &mut self, + value: f64, + batch_size: usize, + format: FormatOptions, + ) -> SerializedEntry { + self.sum += value * batch_size as f64; + self.count += batch_size; + self.current = value; + self.current_count = batch_size; + + let value_current = value; + let value_running = self.sum / self.count as f64; + // Numeric metric state is an aggregated value + let serialized = NumericEntry::Aggregated { + aggregated_value: value_current, + count: batch_size, + } + .serialize(); + + let (formatted_current, formatted_running) = match format.precision { + Some(precision) => ( + format_float(value_current, precision), + format_float(value_running, precision), + ), + None => (format!("{value_current}"), format!("{value_running}")), + }; + + // TODO: naming inconsistent with RL. + let formatted = match format.unit { + Some(unit) => { + format!("epoch {formatted_running} {unit} - batch {formatted_current} {unit}") + } + None => format!("epoch {formatted_running} - batch {formatted_current}"), + }; + + SerializedEntry::new(formatted, serialized) + } + + /// Get the numeric value. + pub fn current_value(&self) -> NumericEntry { + NumericEntry::Aggregated { + aggregated_value: self.current, + count: self.current_count, + } + } + + /// Get the running aggregated value. + pub fn running_value(&self) -> NumericEntry { + NumericEntry::Aggregated { + aggregated_value: self.sum / self.count as f64, + count: self.count, + } + } +} + +impl Default for NumericMetricState { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/aggregate.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/aggregate.rs new file mode 100644 index 0000000..2ccdd7f --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/aggregate.rs @@ -0,0 +1,251 @@ +use crate::{ + logger::MetricLogger, + metric::{NumericEntry, store::Split}, +}; +use std::collections::HashMap; + +use super::{Aggregate, Direction}; + +/// Type that can be used to fetch and use numeric metric aggregates. +#[derive(Default, Debug)] +pub(crate) struct NumericMetricsAggregate { + value_for_each_epoch: HashMap, +} + +#[derive(new, Hash, PartialEq, Eq, Debug)] +struct Key { + name: String, + epoch: usize, + split: Split, + aggregate: Aggregate, +} + +impl NumericMetricsAggregate { + pub(crate) fn aggregate( + &mut self, + name: &str, + epoch: usize, + split: &Split, + aggregate: Aggregate, + loggers: &mut [Box], + ) -> Option { + let key = Key::new(name.to_string(), epoch, split.clone(), aggregate); + + if let Some(value) = self.value_for_each_epoch.get(&key) { + return Some(*value); + } + + let points = || { + let mut errors = Vec::new(); + for logger in loggers { + match logger.read_numeric(name, epoch, split) { + Ok(points) => return Ok(points), + Err(err) => errors.push(err), + }; + } + + Err(errors.join(" ")) + }; + + let points = points().expect("Can read values"); + + if points.is_empty() { + return None; + } + + // Accurately compute the aggregated value based on the *actual* number of points + // since not all mini-batches are guaranteed to have the specified batch size + let (sum, num_points) = points + .into_iter() + .map(|entry| match entry { + NumericEntry::Value(v) => (v, 1), + // Right now the mean is the only aggregate available, so we can assume that the sum + // of an entry corresponds to (value * number of elements) + NumericEntry::Aggregated { + aggregated_value, + count, + } => (aggregated_value * count as f64, count), + }) + .reduce(|(acc_v, acc_n), (v, n)| (acc_v + v, acc_n + n)) + .unwrap(); + let value = match aggregate { + Aggregate::Mean => sum / num_points as f64, + }; + + self.value_for_each_epoch.insert(key, value); + Some(value) + } + + pub(crate) fn find_epoch( + &mut self, + name: &str, + split: &Split, + aggregate: Aggregate, + direction: Direction, + loggers: &mut [Box], + ) -> Option { + let mut data = Vec::new(); + let mut current_epoch = 1; + + while let Some(value) = self.aggregate(name, current_epoch, split, aggregate, loggers) { + data.push(value); + current_epoch += 1; + } + + if data.is_empty() { + return None; + } + + let mut current_value = match &direction { + Direction::Lowest => f64::MAX, + Direction::Highest => f64::MIN, + }; + + for (i, value) in data.into_iter().enumerate() { + match &direction { + Direction::Lowest => { + if value < current_value { + current_value = value; + current_epoch = i + 1; + } + } + Direction::Highest => { + if value > current_value { + current_value = value; + current_epoch = i + 1; + } + } + } + } + + Some(current_epoch) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::{ + logger::{FileMetricLogger, InMemoryMetricLogger}, + metric::{MetricDefinition, MetricEntry, MetricId, SerializedEntry, store::MetricsUpdate}, + }; + + use super::*; + + struct TestLogger { + logger: FileMetricLogger, + epoch: usize, + } + const NAME: &str = "test-logger"; + + impl TestLogger { + fn new() -> Self { + Self { + logger: FileMetricLogger::new("/tmp"), + epoch: 1, + } + } + fn log(&mut self, num: f64) { + let entry = MetricEntry::new( + MetricId::new(Arc::new(NAME.into())), + SerializedEntry::new(num.to_string(), num.to_string()), + ); + let entries = Vec::from([entry]); + let metrics_update = MetricsUpdate::new(entries, vec![]); + self.logger.log(metrics_update, self.epoch, &Split::Train); + } + fn log_definition(&mut self) { + let definition = MetricDefinition { + metric_id: MetricId::new(Arc::new(NAME.into())), + name: NAME.into(), + attributes: crate::metric::MetricAttributes::None, + description: None, + }; + self.logger.log_metric_definition(definition); + } + fn new_epoch(&mut self) { + self.epoch += 1; + } + } + + #[test] + fn should_find_epoch() { + let mut logger = TestLogger::new(); + let mut aggregate = NumericMetricsAggregate::default(); + logger.log_definition(); + + logger.log(500.); // Epoch 1 + logger.log(1000.); // Epoch 1 + logger.new_epoch(); + logger.log(200.); // Epoch 2 + logger.log(1000.); // Epoch 2 + logger.new_epoch(); + logger.log(10000.); // Epoch 3 + + let value = aggregate + .find_epoch( + NAME, + &Split::Train, + Aggregate::Mean, + Direction::Lowest, + &mut [Box::new(logger.logger)], + ) + .unwrap(); + + assert_eq!(value, 2); + } + + #[test] + fn should_aggregate_numeric_entry() { + let mut logger = InMemoryMetricLogger::default(); + let mut aggregate = NumericMetricsAggregate::default(); + let metric_name = Arc::new("Loss".to_string()); + let metric_id = MetricId::new(metric_name.clone()); + let definition = MetricDefinition { + metric_id: metric_id.clone(), + name: metric_name.to_string(), + attributes: crate::metric::MetricAttributes::None, + description: None, + }; + logger.log_metric_definition(definition); + + // Epoch 1 + let loss_1 = 0.5; + let loss_2 = 1.25; // (1.5 + 1.0) / 2 = 2.5 / 2 + let entry = MetricEntry::new( + metric_id.clone(), + SerializedEntry::new(loss_1.to_string(), NumericEntry::Value(loss_1).serialize()), + ); + let entries = Vec::from([entry]); + let metrics_update = MetricsUpdate::new(entries, vec![]); + logger.log(metrics_update, 1, &Split::Train); + let entry = MetricEntry::new( + metric_id.clone(), + SerializedEntry::new( + loss_2.to_string(), + NumericEntry::Aggregated { + aggregated_value: loss_2, + count: 2, + } + .serialize(), + ), + ); + let entries = Vec::from([entry]); + let metrics_update = MetricsUpdate::new(entries, vec![]); + logger.log(metrics_update, 1, &Split::Train); + + let value = aggregate + .aggregate( + &metric_name, + 1, + &Split::Train, + Aggregate::Mean, + &mut [Box::new(logger)], + ) + .unwrap(); + + // Average should be (0.5 + 1.25 * 2) / 3 = 1.0, not (0.5 + 1.25) / 2 = 0.875 + assert_eq!(value, 1.0); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/base.rs new file mode 100644 index 0000000..0e74c7d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/base.rs @@ -0,0 +1,105 @@ +use std::sync::Arc; + +use crate::metric::{MetricDefinition, MetricEntry, NumericEntry}; + +/// Event happening during the training/validation process. +pub enum Event { + /// Signal the iniialization of the metrics + MetricsInit(Vec), + /// Signal that metrics have been updated. + MetricsUpdate(MetricsUpdate), + /// Signal the end of an epoch. + EndEpoch(EpochSummary), +} + +/// Contains all metric information. +#[derive(new, Clone, Debug)] +pub struct NumericMetricUpdate { + /// Generic metric information. + pub entry: MetricEntry, + /// The numeric information. + pub numeric_entry: NumericEntry, + /// Numeric value averaged over the global step (epoch). + pub running_entry: NumericEntry, +} + +/// Contains all metric information. +#[derive(new, Clone, Debug)] +pub struct MetricsUpdate { + /// Metrics information related to non-numeric metrics. + pub entries: Vec, + /// Metrics information related to numeric metrics. + pub entries_numeric: Vec, +} + +/// Summary information about a given epoch +#[derive(new, Clone, Debug)] +pub struct EpochSummary { + /// Epoch number. + pub epoch_number: usize, + /// Dataset split (train, valid, test). + pub split: Split, +} + +/// Defines how training and validation events are collected and searched. +/// +/// This trait also exposes methods that uses the collected data to compute useful information. +pub trait EventStore: Send { + /// Collect a training/validation event. + fn add_event(&mut self, event: Event, split: Split); + + /// Find the epoch following the given criteria from the collected data. + fn find_epoch( + &mut self, + name: &str, + aggregate: Aggregate, + direction: Direction, + split: &Split, + ) -> Option; + + /// Find the metric value for the current epoch following the given criteria. + fn find_metric( + &mut self, + name: &str, + epoch: usize, + aggregate: Aggregate, + split: &Split, + ) -> Option; +} + +#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] +/// How to aggregate the metric. +pub enum Aggregate { + /// Compute the average. + Mean, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +/// The split to use. +pub enum Split { + /// The training split. + Train, + /// The validation split. + Valid, + /// The testing split, which might be tagged. + Test(Option>), +} + +impl std::fmt::Display for Split { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Split::Train => write!(f, "train"), + Split::Valid => write!(f, "valid"), + Split::Test(_) => write!(f, "test"), + } + } +} + +#[derive(Copy, Clone)] +/// The direction of the query. +pub enum Direction { + /// Lower is better. + Lowest, + /// Higher is better. + Highest, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/client.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/client.rs new file mode 100644 index 0000000..3135cf4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/client.rs @@ -0,0 +1,171 @@ +use super::EventStore; +use super::{Aggregate, Direction, Event, Split}; +use std::sync::Arc; +use std::{sync::mpsc, thread::JoinHandle}; + +/// Type that allows to communicate with an [event store](EventStore). +pub struct EventStoreClient { + sender: mpsc::Sender, + handler: Option>, +} + +impl EventStoreClient { + /// Create a new [event store](EventStore) client. + pub(crate) fn new(store: C) -> Self + where + C: EventStore + 'static, + { + let (sender, receiver) = mpsc::channel(); + let thread = WorkerThread::new(store, receiver); + + let handler = std::thread::spawn(move || thread.run()); + let handler = Some(handler); + + Self { sender, handler } + } +} + +impl EventStoreClient { + /// Add a training event to the [event store](EventStore). + pub(crate) fn add_event_train(&self, event: Event) { + self.sender + .send(Message::OnEventTrain(event)) + .expect("Can send event to event store thread."); + } + + /// Add a validation event to the [event store](EventStore). + pub(crate) fn add_event_valid(&self, event: Event) { + self.sender + .send(Message::OnEventValid(event)) + .expect("Can send event to event store thread."); + } + + /// Add a testing event to the [event store](EventStore). + pub(crate) fn add_event_test(&self, event: Event, tag: Arc) { + self.sender + .send(Message::OnEventTest(event, tag)) + .expect("Can send event to event store thread."); + } + + /// Find the epoch following the given criteria from the collected data. + pub fn find_epoch( + &self, + name: &str, + aggregate: Aggregate, + direction: Direction, + split: &Split, + ) -> Option { + let (sender, receiver) = mpsc::sync_channel(1); + self.sender + .send(Message::FindEpoch( + name.to_string(), + aggregate, + direction, + split.clone(), + sender, + )) + .expect("Can send event to event store thread."); + + match receiver.recv() { + Ok(value) => value, + Err(err) => panic!("Event store thread crashed: {err:?}"), + } + } + + /// Find the metric value for the current epoch following the given criteria. + pub fn find_metric( + &self, + name: &str, + epoch: usize, + aggregate: Aggregate, + split: &Split, + ) -> Option { + let (sender, receiver) = mpsc::sync_channel(1); + self.sender + .send(Message::FindMetric( + name.to_string(), + epoch, + aggregate, + split.clone(), + sender, + )) + .expect("Can send event to event store thread."); + + match receiver.recv() { + Ok(value) => value, + Err(err) => panic!("Event store thread crashed: {err:?}"), + } + } +} + +#[derive(new)] +struct WorkerThread { + store: S, + receiver: mpsc::Receiver, +} + +impl WorkerThread +where + C: EventStore, +{ + fn run(mut self) { + for item in self.receiver.iter() { + match item { + Message::End => { + return; + } + Message::FindEpoch(name, aggregate, direction, split, callback) => { + let response = self.store.find_epoch(&name, aggregate, direction, &split); + callback + .send(response) + .expect("Can send response using callback channel."); + } + Message::FindMetric(name, epoch, aggregate, split, callback) => { + let response = self.store.find_metric(&name, epoch, aggregate, &split); + callback + .send(response) + .expect("Can send response using callback channel."); + } + Message::OnEventTrain(event) => self.store.add_event(event, Split::Train), + Message::OnEventValid(event) => self.store.add_event(event, Split::Valid), + Message::OnEventTest(event, tag) => { + self.store.add_event(event, Split::Test(Some(tag))) + } + } + } + } +} + +enum Message { + OnEventTest(Event, Arc), + OnEventTrain(Event), + OnEventValid(Event), + End, + FindEpoch( + String, + Aggregate, + Direction, + Split, + mpsc::SyncSender>, + ), + FindMetric( + String, + usize, + Aggregate, + Split, + mpsc::SyncSender>, + ), +} + +impl Drop for EventStoreClient { + fn drop(&mut self) { + self.sender + .send(Message::End) + .expect("Can send the end message to the event store thread."); + let handler = self.handler.take(); + + if let Some(handler) = handler { + handler.join().expect("The event store thread should stop."); + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/log.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/log.rs new file mode 100644 index 0000000..341c2a5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/log.rs @@ -0,0 +1,72 @@ +use std::collections::HashMap; + +use super::{Aggregate, Direction, Event, EventStore, Split, aggregate::NumericMetricsAggregate}; +use crate::logger::MetricLogger; + +#[derive(Default)] +pub(crate) struct LogEventStore { + loggers: Vec>, + aggregate: NumericMetricsAggregate, + epochs: HashMap, +} + +impl EventStore for LogEventStore { + fn add_event(&mut self, event: Event, split: Split) { + let epoch = *self.epochs.entry(split.clone()).or_insert(1); + + match event { + Event::MetricsInit(definitions) => { + definitions.iter().for_each(|def| { + self.loggers + .iter_mut() + .for_each(|logger| logger.log_metric_definition(def.clone())); + }); + } + Event::MetricsUpdate(update) => { + self.loggers + .iter_mut() + .for_each(|logger| logger.log(update.clone(), epoch, &split)); + } + Event::EndEpoch(summary) => { + self.epochs.insert(split, summary.epoch_number + 1); + self.loggers + .iter_mut() + .for_each(|logger| logger.log_epoch_summary(summary.clone())); + } + } + } + + fn find_epoch( + &mut self, + name: &str, + aggregate: Aggregate, + direction: Direction, + split: &Split, + ) -> Option { + self.aggregate + .find_epoch(name, split, aggregate, direction, &mut self.loggers) + } + + fn find_metric( + &mut self, + name: &str, + epoch: usize, + aggregate: Aggregate, + split: &Split, + ) -> Option { + self.aggregate + .aggregate(name, epoch, split, aggregate, &mut self.loggers) + } +} + +impl LogEventStore { + /// Register a logger for metrics. + pub(crate) fn register_logger(&mut self, logger: ML) { + self.loggers.push(Box::new(logger)); + } + + /// Returns whether any loggers are registered. + pub(crate) fn has_loggers(&self) -> bool { + !self.loggers.is_empty() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/mod.rs new file mode 100644 index 0000000..b86c0f4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/store/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod aggregate; + +mod base; +mod client; +mod log; + +pub(crate) use self::log::*; +pub use base::*; +pub use client::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/top_k_acc.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/top_k_acc.rs new file mode 100644 index 0000000..e81d1c9 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/top_k_acc.rs @@ -0,0 +1,185 @@ +use core::marker::PhantomData; +use std::sync::Arc; + +use super::state::{FormatOptions, NumericMetricState}; +use super::{MetricMetadata, SerializedEntry}; +use crate::metric::{ + Metric, MetricAttributes, MetricName, Numeric, NumericAttributes, NumericEntry, +}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{ElementConversion, Int, Tensor}; + +/// The Top-K accuracy metric. +/// +/// For K=1, this is equivalent to the [accuracy metric](`super::acc::AccuracyMetric`). +#[derive(Default, Clone)] +pub struct TopKAccuracyMetric { + name: Arc, + k: usize, + state: NumericMetricState, + /// If specified, targets equal to this value will be considered padding and will not count + /// towards the metric + pad_token: Option, + _b: PhantomData, +} + +/// The [top-k accuracy metric](TopKAccuracyMetric) input type. +#[derive(new)] +pub struct TopKAccuracyInput { + /// The outputs (batch_size, num_classes) + outputs: Tensor, + /// The labels (batch_size) + targets: Tensor, +} + +impl TopKAccuracyMetric { + /// Creates the metric. + pub fn new(k: usize) -> Self { + Self { + name: Arc::new(format!("Top-K Accuracy @ TopK({})", k)), + k, + ..Default::default() + } + } + + /// Sets the pad token. + pub fn with_pad_token(mut self, index: usize) -> Self { + self.pad_token = Some(index); + self + } +} + +impl Metric for TopKAccuracyMetric { + type Input = TopKAccuracyInput; + + fn update( + &mut self, + input: &TopKAccuracyInput, + _metadata: &MetricMetadata, + ) -> SerializedEntry { + let [batch_size, _n_classes] = input.outputs.dims(); + + let targets = input.targets.clone().to_device(&B::Device::default()); + + let outputs = input + .outputs + .clone() + .argsort_descending(1) + .narrow(1, 0, self.k) + .to_device(&B::Device::default()) + .reshape([batch_size, self.k]); + + let (targets, num_pad) = match self.pad_token { + Some(pad_token) => { + // we ignore the samples where the target is equal to the pad token + let mask = targets.clone().equal_elem(pad_token as i64); + let num_pad = mask.clone().int().sum().into_scalar().elem::(); + (targets.clone().mask_fill(mask, -1_i64), num_pad) + } + None => (targets.clone(), 0_f64), + }; + + let accuracy = targets + .reshape([batch_size, 1]) + .repeat_dim(1, self.k) + .equal(outputs) + .int() + .sum() + .into_scalar() + .elem::() + / (batch_size as f64 - num_pad); + + self.state.update( + 100.0 * accuracy, + batch_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn clear(&mut self) { + self.state.reset() + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for TopKAccuracyMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + #[test] + fn test_accuracy_without_padding() { + let device = Default::default(); + let mut metric = TopKAccuracyMetric::::new(2); + let input = TopKAccuracyInput::new( + Tensor::from_data( + [ + [0.0, 0.2, 0.8], // 2, 1 + [1.0, 2.0, 0.5], // 1, 0 + [0.4, 0.1, 0.2], // 0, 2 + [0.6, 0.7, 0.2], // 1, 0 + ], + &device, + ), + Tensor::from_data([2, 2, 1, 1], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(50.0, metric.value().current()); + } + + #[test] + fn test_accuracy_with_padding() { + let device = Default::default(); + let mut metric = TopKAccuracyMetric::::new(2).with_pad_token(3); + let input = TopKAccuracyInput::new( + Tensor::from_data( + [ + [0.0, 0.2, 0.8, 0.0], // 2, 1 + [1.0, 2.0, 0.5, 0.0], // 1, 0 + [0.4, 0.1, 0.2, 0.0], // 0, 2 + [0.6, 0.7, 0.2, 0.0], // 1, 0 + [0.0, 0.1, 0.2, 5.0], // Predicted padding should not count + [0.0, 0.1, 0.2, 0.0], // Error on padding should not count + [0.6, 0.0, 0.2, 0.0], // Error on padding should not count + ], + &device, + ), + Tensor::from_data([2, 2, 1, 1, 3, 3, 3], &device), + ); + + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert_eq!(50.0, metric.value().current()); + } + + #[test] + fn test_parameterized_unique_name() { + let metric_a = TopKAccuracyMetric::::new(2); + let metric_b = TopKAccuracyMetric::::new(1); + let metric_c = TopKAccuracyMetric::::new(2); + + assert_ne!(metric_a.name(), metric_b.name()); + assert_eq!(metric_a.name(), metric_c.name()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dice.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dice.rs new file mode 100644 index 0000000..1f3bd54 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dice.rs @@ -0,0 +1,345 @@ +use crate::metric::{MetricAttributes, MetricName, SerializedEntry}; + +use super::super::{ + Metric, MetricMetadata, + state::{FormatOptions, NumericMetricState}, +}; +use burn_core::{ + prelude::{Backend, Tensor}, + tensor::{ElementConversion, Int, s}, +}; +use core::marker::PhantomData; + +/// Input type for the [DiceMetric]. +/// +/// # Type Parameters +/// - `B`: Backend type. +/// - `D`: Number of dimensions. Should be more than, or equal to 3 (default 4). +pub struct DiceInput { + /// Model outputs (predictions), as a tensor. + outputs: Tensor, + /// Ground truth targets, as a tensor. + targets: Tensor, +} + +impl DiceInput { + /// Creates a new DiceInput with the given outputs and targets. + /// + /// Inputs are expected to have the dimensions `[B, C, ...]` + /// where `B` is the batch size, `C` is the number of classes, + /// and `...` represents additional dimensions (e.g., height, width for images). + /// + /// If `C` is more than 1, the first class (index 0) is considered the background. + /// Additionally, one-hot encoding is the responsibility of the caller. + /// + /// # Arguments + /// - `outputs`: The model outputs as a tensor. + /// - `targets`: The ground truth targets as a tensor. + /// + /// # Returns + /// A new instance of `DiceInput`. + /// + /// # Panics + /// - If `D` is less than 3. + /// - If `outputs` and `targets` do not have the same dimensions. + /// - If `outputs` or `targets` do not have exactly `D` dimensions. + /// - If `outputs` and `targets` do not have the same shape. + pub fn new(outputs: Tensor, targets: Tensor) -> Self { + assert!(D >= 3, "DiceInput requires at least 3 dimensions."); + assert!( + outputs.dims() == targets.dims(), + "Outputs and targets must have the same dimensions. Got {:?} and {:?}", + outputs.dims(), + targets.dims() + ); + Self { outputs, targets } + } +} + +/// Configuration for the [DiceMetric]. +#[derive(Debug, Clone, Copy)] +pub struct DiceMetricConfig { + /// Epsilon value to avoid division by zero. + pub epsilon: f64, + /// Whether to include the background class in the metric calculation. + /// The background is assumed to be the first class (index 0). + /// if `true`, will panic if there are fewer than 2 classes. + pub include_background: bool, +} + +impl Default for DiceMetricConfig { + fn default() -> Self { + Self { + epsilon: 1e-7, + include_background: false, + } + } +} + +/// The Dice-Sorenson coefficient (DSC) for evaluating overlap between two binary masks. +/// The DSC is defined as: +/// `DSC = 2 * (|X ∩ Y|) / (|X| + |Y|)` +/// where `X` is the model output and `Y` is the ground truth target. +/// +/// # Type Parameters +/// - `B`: Backend type. +/// - `D`: Number of dimensions. Should be more than, or equal to 3 (default 4). +#[derive(Default, Clone)] +pub struct DiceMetric { + name: MetricName, + /// Internal state for numeric metric aggregation. + state: NumericMetricState, + /// Marker for backend type. + _b: PhantomData, + /// Configuration for the metric. + config: DiceMetricConfig, +} + +impl DiceMetric { + /// Creates a new Dice metric instance with default config. + pub fn new() -> Self { + Self::with_config(DiceMetricConfig::default()) + } + + /// Creates a new Dice metric with a custom config. + pub fn with_config(config: DiceMetricConfig) -> Self { + let name = MetricName::new(format!("{D}D Dice Metric")); + assert!(D >= 3, "DiceMetric requires at least 3 dimensions."); + Self { + name, + config, + ..Default::default() + } + } +} + +impl Metric for DiceMetric { + type Input = DiceInput; + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn update(&mut self, item: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + // Dice coefficient: 2 * (|X ∩ Y|) / (|X| + |Y|) + if item.outputs.dims() != item.targets.dims() { + panic!( + "Outputs and targets must have the same dimensions. Got {:?} and {:?}", + item.outputs.dims(), + item.targets.dims() + ); + } + + let dims = item.outputs.dims(); + let batch_size = dims[0]; + let n_classes = dims[1]; + + let mut outputs = item.outputs.clone(); + let mut targets = item.targets.clone(); + + if !self.config.include_background && n_classes > 1 { + // If not including background, we can ignore the first class + outputs = outputs.slice(s![.., 1..]); + targets = targets.slice(s![.., 1..]); + } else if self.config.include_background && n_classes < 2 { + // If including background, we need at least 2 classes + panic!("Dice metric requires at least 2 classes when including background."); + } + + let intersection = (outputs.clone() * targets.clone()).sum(); + let outputs_sum = outputs.sum(); + let targets_sum = targets.sum(); + + // Convert to f64 + let intersection_val = intersection.into_scalar().elem::(); + let outputs_sum_val = outputs_sum.into_scalar().elem::(); + let targets_sum_val = targets_sum.into_scalar().elem::(); + + // Use epsilon from config + let epsilon = self.config.epsilon; + let dice = + (2.0 * intersection_val + epsilon) / (outputs_sum_val + targets_sum_val + epsilon); + + self.state.update( + dice, + batch_size, + FormatOptions::new(self.name()).precision(4), + ) + } + + /// Clears the metric state. + fn clear(&mut self) { + self.state.reset(); + } + + fn attributes(&self) -> MetricAttributes { + crate::metric::NumericAttributes { + unit: None, + higher_is_better: true, + } + .into() + } +} + +impl crate::metric::Numeric for DiceMetric { + fn value(&self) -> crate::metric::NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> crate::metric::NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TestBackend, metric::Numeric}; + use burn_core::tensor::{Shape, Tensor}; + + #[test] + fn test_dice_perfect_overlap() { + let device = Default::default(); + let mut metric = DiceMetric::::new(); + let input = DiceInput::new( + Tensor::from_data([[[[1, 0], [1, 0]]]], &device), + Tensor::from_data([[[[1, 0], [1, 0]]]], &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert!((metric.value().current() - 1.0).abs() < 1e-6); + } + + #[test] + fn test_dice_no_overlap() { + let device = Default::default(); + let mut metric = DiceMetric::::new(); + let input = DiceInput::new( + Tensor::from_data([[[[1, 0], [1, 0]]]], &device), + Tensor::from_data([[[[0, 1], [0, 1]]]], &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert!(metric.value().current() < 1e-6); + } + + #[test] + fn test_dice_partial_overlap() { + let device = Default::default(); + let mut metric = DiceMetric::::new(); + let input = DiceInput::new( + Tensor::from_data([[[[1, 1], [0, 0]]]], &device), + Tensor::from_data([[[[1, 0], [1, 0]]]], &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + // intersection = 1, sum = 2+2=4, dice = 2*1/4 = 0.5 + assert!((metric.value().current() - 0.5).abs() < 1e-6); + } + + #[test] + fn test_dice_empty_masks() { + let device = Default::default(); + let mut metric = DiceMetric::::new(); + let input = DiceInput::new( + Tensor::from_data([[[[0, 0], [0, 0]]]], &device), + Tensor::from_data([[[[0, 0], [0, 0]]]], &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert!((metric.value().current() - 1.0).abs() < 1e-6); + } + + #[test] + fn test_dice_no_background() { + let device = Default::default(); + let mut metric = DiceMetric::::new(); + let input = DiceInput::new( + Tensor::ones(Shape::new([1, 1, 2, 2]), &device), + Tensor::ones(Shape::new([1, 1, 2, 2]), &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert!((metric.value().current() - 1.0).abs() < 1e-6); + } + + #[test] + fn test_dice_with_background() { + let device = Default::default(); + let config = DiceMetricConfig { + epsilon: 1e-7, + include_background: true, + }; + let mut metric = DiceMetric::::with_config(config); + let input = DiceInput::new( + Tensor::ones(Shape::new([1, 2, 2, 2]), &device), + Tensor::ones(Shape::new([1, 2, 2, 2]), &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert!((metric.value().current() - 1.0).abs() < 1e-6); + } + + #[test] + fn test_dice_ignored_background() { + let device = Default::default(); + let config = DiceMetricConfig { + epsilon: 1e-7, + include_background: false, + }; + let mut metric = DiceMetric::::with_config(config); + let input = DiceInput::new( + Tensor::ones(Shape::new([1, 2, 2, 2]), &device), + Tensor::ones(Shape::new([1, 2, 2, 2]), &device), + ); + let _entry = metric.update(&input, &MetricMetadata::fake()); + assert!((metric.value().current() - 1.0).abs() < 1e-6); + } + + #[test] + #[should_panic(expected = "DiceInput requires at least 3 dimensions.")] + fn test_invalid_input_dimensions() { + let device = Default::default(); + // D = 2, should panic + let _ = DiceInput::::new( + Tensor::from_data([[0.0, 0.0]], &device), + Tensor::from_data([[0.0, 0.0]], &device), + ); + } + + #[test] + #[should_panic( + expected = "Outputs and targets must have the same dimensions. Got [1, 1, 2, 2] and [1, 1, 2, 3]" + )] + fn test_mismatched_shape() { + let device = Default::default(); + // shapes differ + let _ = DiceInput::::new( + Tensor::from_data([[[[0.0; 2]; 2]; 1]; 1], &device), + Tensor::from_data([[[[0.0; 3]; 2]; 1]; 1], &device), + ); + } + + #[test] + #[should_panic(expected = "Dice metric requires at least 2 classes when including background.")] + fn test_include_background_panic() { + let device = Default::default(); + let config = DiceMetricConfig { + epsilon: 1e-7, + include_background: true, + }; + let mut metric = DiceMetric::::with_config(config); + let input = DiceInput::new( + Tensor::from_data([[[[1.0; 2]; 1]; 1]; 1], &device), + Tensor::from_data([[[[1.0; 2]; 1]; 1]; 1], &device), + ); + // n_classes = 2, should not panic + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let config = DiceMetricConfig { + epsilon: 1e-7, + include_background: true, + }; + let mut metric = DiceMetric::::with_config(config); + let input = DiceInput::new( + Tensor::from_data([[[[1.0; 1]; 1]; 1]; 1], &device), + Tensor::from_data([[[[1.0; 1]; 1]; 1]; 1], &device), + ); + // n_classes = 1, should panic + let _entry = metric.update(&input, &MetricMetadata::fake()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/l2pool.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/l2pool.rs new file mode 100644 index 0000000..6ba2a9d --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/l2pool.rs @@ -0,0 +1,164 @@ +//! L2 Pooling layer for DISTS. +//! +//! L2 Pooling applies a Hanning window filter and computes the L2 norm +//! across the pooling window. This is used in DISTS instead of MaxPooling. + +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_nn::PaddingConfig2d; +use burn_nn::conv::{Conv2d, Conv2dConfig}; + +/// L2 Pooling layer configuration. +#[derive(Debug, Clone)] +pub struct L2Pool2dConfig { + /// Kernel size for pooling + pub kernel_size: usize, + /// Stride for pooling + pub stride: usize, + /// Padding for pooling + pub padding: usize, +} + +impl Default for L2Pool2dConfig { + fn default() -> Self { + Self { + kernel_size: 5, + stride: 2, + padding: 2, + } + } +} + +impl L2Pool2dConfig { + /// Create a new L2Pool2d configuration. + #[allow(dead_code)] + pub fn new(kernel_size: usize, stride: usize, padding: usize) -> Self { + Self { + kernel_size, + stride, + padding, + } + } + + /// Initialize the L2Pool2d layer. + pub fn init(&self, channels: usize, device: &B::Device) -> L2Pool2d { + L2Pool2d::new( + channels, + self.kernel_size, + self.stride, + self.padding, + device, + ) + } +} + +/// L2 Pooling layer. +/// +/// Applies a 2D Hanning window filter followed by L2 normalization. +/// This provides smoother downsampling compared to MaxPooling. +#[derive(Module, Debug)] +pub struct L2Pool2d { + /// Depthwise convolution with Hanning kernel + conv: Conv2d, +} + +impl L2Pool2d { + /// Create a new L2Pool2d layer with Hanning window kernel. + pub fn new( + channels: usize, + kernel_size: usize, + stride: usize, + padding: usize, + device: &B::Device, + ) -> Self { + // Create Hanning kernel + let kernel = Self::create_hanning_kernel(channels, kernel_size, device); + + // Create depthwise convolution (groups = channels) + let mut conv = Conv2dConfig::new([channels, channels], [kernel_size, kernel_size]) + .with_stride([stride, stride]) + .with_padding(PaddingConfig2d::Explicit( + padding, padding, padding, padding, + )) + .with_groups(channels) + .with_bias(false) + .init(device); + + // Set the kernel weights to Hanning window + conv.weight = burn::module::Param::from_tensor(kernel); + + Self { conv } + } + + /// Create a Hanning kernel for depthwise convolution. + /// Output shape: [channels, 1, kernel_size, kernel_size] + fn create_hanning_kernel( + channels: usize, + kernel_size: usize, + device: &B2::Device, + ) -> Tensor { + // Create 1D Hanning window + let mut hanning_1d = Vec::with_capacity(kernel_size); + for i in 0..kernel_size { + let n = i as f32; + let n_minus_1 = (kernel_size - 1) as f32; + let value = if n_minus_1 == 0.0 { + 1.0 + } else { + 0.5 * (1.0 - (2.0 * std::f32::consts::PI * n / n_minus_1).cos()) + }; + hanning_1d.push(value); + } + + // Create 2D Hanning window by outer product + let mut hanning_2d = Vec::with_capacity(kernel_size * kernel_size); + let mut sum = 0.0; + for i in 0..kernel_size { + for j in 0..kernel_size { + let value = hanning_1d[i] * hanning_1d[j]; + hanning_2d.push(value); + sum += value; + } + } + + // Normalize + for v in hanning_2d.iter_mut() { + *v /= sum; + } + + // Create tensor of shape [1, 1, kernel_size, kernel_size] + let kernel_single = Tensor::::from_floats(hanning_2d.as_slice(), device).reshape([ + 1, + 1, + kernel_size, + kernel_size, + ]); + + // Expand to [channels, 1, kernel_size, kernel_size] + kernel_single.repeat_dim(0, channels) + } + + /// Apply L2 pooling to the input tensor. + /// + /// # Arguments + /// + /// * `x` - Input tensor of shape `[batch, channels, height, width]` + /// + /// # Returns + /// + /// Pooled tensor with reduced spatial dimensions. + pub fn forward(&self, x: Tensor) -> Tensor { + // Square the input + let x_sq = x.clone().mul(x); + + // Apply depthwise convolution with Hanning kernel + let pooled = self.conv.forward(x_sq); + + // Take square root for L2 norm + // Add small epsilon to avoid sqrt of negative numbers due to numerical errors + pooled.clamp_min(1e-10).sqrt() + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/metric.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/metric.rs new file mode 100644 index 0000000..d89cd83 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/metric.rs @@ -0,0 +1,498 @@ +//! DISTS (Deep Image Structure and Texture Similarity) metric. +//! +//! DISTS is a full-reference image quality assessment metric that combines +//! structure and texture similarity using deep features from VGG16. +//! +//! Reference: "Image Quality Assessment: Unifying Structure and Texture Similarity" +//! https://arxiv.org/abs/2004.07728 + +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay, Param}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_nn::loss::Reduction; + +use super::vgg16_l2pool::Vgg16L2PoolExtractor; + +/// Channel counts for each stage: [input, stage1, stage2, stage3, stage4, stage5] +const CHANNELS: [usize; 6] = [3, 64, 128, 256, 512, 512]; + +/// Small constant for numerical stability in structure similarity. +const C1: f32 = 1e-6; + +/// Small constant for numerical stability in texture similarity. +const C2: f32 = 1e-6; + +/// ImageNet normalization constants. +const IMAGENET_MEAN: [f32; 3] = [0.485, 0.456, 0.406]; +const IMAGENET_STD: [f32; 3] = [0.229, 0.224, 0.225]; + +/// Image normalizer with pre-initialized mean and std tensors. +/// +/// This struct holds the mean and std tensors for normalization, +/// avoiding the need to create them on each forward pass. +#[derive(Module, Debug)] +pub struct Normalizer { + /// Mean tensor of shape [1, 3, 1, 1] for broadcasting. + pub mean: Tensor, + /// Std tensor of shape [1, 3, 1, 1] for broadcasting. + pub std: Tensor, +} + +impl Normalizer { + /// Create a new ImageNet normalizer. + pub fn imagenet(device: &B::Device) -> Self { + // Shape: [1, 3, 1, 1] for broadcasting over [batch, channels, height, width] + let mean = Tensor::from_floats( + [[ + [[IMAGENET_MEAN[0]]], + [[IMAGENET_MEAN[1]]], + [[IMAGENET_MEAN[2]]], + ]], + device, + ); + let std = Tensor::from_floats( + [[ + [[IMAGENET_STD[0]]], + [[IMAGENET_STD[1]]], + [[IMAGENET_STD[2]]], + ]], + device, + ); + Self { mean, std } + } + + /// Normalize a tensor: (x - mean) / std + pub fn normalize(&self, x: Tensor) -> Tensor { + x.sub(self.mean.clone()).div(self.std.clone()) + } +} + +/// Configuration for DISTS metric. +#[derive(Config, Debug)] +pub struct DistsConfig { + /// Whether to apply ImageNet normalization to input images. + #[config(default = true)] + pub normalize: bool, +} + +impl DistsConfig { + /// Initialize a DISTS module with default weights. + pub fn init(&self, device: &B::Device) -> Dists { + let total_channels: usize = CHANNELS.iter().sum(); + + // Initialize alpha and beta with constant value 0.1 for all channels + let alpha_data: Vec = (0..total_channels).map(|_| 0.1).collect(); + let beta_data: Vec = (0..total_channels).map(|_| 0.1).collect(); + + let normalizer = if self.normalize { + Some(Normalizer::imagenet(device)) + } else { + None + }; + + Dists { + extractor: Vgg16L2PoolExtractor::new(device), + alpha: Param::from_tensor(Tensor::from_floats(alpha_data.as_slice(), device)), + beta: Param::from_tensor(Tensor::from_floats(beta_data.as_slice(), device)), + normalizer, + } + } + + /// Initialize a DISTS module with pretrained weights. + pub fn init_pretrained(&self, device: &B::Device) -> Dists { + let dists = self.init(device); + super::weights::load_pretrained_weights(dists) + } +} + +/// DISTS (Deep Image Structure and Texture Similarity) metric. +/// +/// Computes perceptual similarity between two images by combining +/// structure similarity (based on spatial means) and texture similarity +/// (based on variances and covariances) across VGG16 feature maps. +/// +/// # Example +/// +/// ```ignore +/// use burn_train::metric::vision::{DistsConfig, Reduction}; +/// +/// let device = Default::default(); +/// let dists = DistsConfig::new().init_pretrained(&device); +/// +/// let img1: Tensor = /* [batch, 3, H, W] */; +/// let img2: Tensor = /* [batch, 3, H, W] */; +/// +/// let distance = dists.forward(img1, img2, Reduction::Mean); +/// ``` +#[derive(Module, Debug)] +#[module(custom_display)] +pub struct Dists { + /// VGG16 feature extractor with L2 pooling + pub(crate) extractor: Vgg16L2PoolExtractor, + /// Learned weights for structure similarity (per channel) + pub(crate) alpha: Param>, + /// Learned weights for texture similarity (per channel) + pub(crate) beta: Param>, + /// Optional normalizer for input preprocessing + pub(crate) normalizer: Option>, +} + +impl ModuleDisplay for Dists { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + content + .add("backbone", &"VGG16-L2Pool".to_string()) + .add("normalize", &self.normalizer.is_some().to_string()) + .optional() + } +} + +impl Dists { + /// Compute DISTS distance with reduction. + /// + /// # Arguments + /// + /// * `input` - First image tensor of shape `[batch, 3, H, W]` + /// * `target` - Second image tensor of shape `[batch, 3, H, W]` + /// * `reduction` - How to reduce the output (Mean, Sum, or Auto) + /// + /// # Returns + /// + /// Scalar tensor of shape `[1]`. + pub fn forward( + &self, + input: Tensor, + target: Tensor, + reduction: Reduction, + ) -> Tensor { + let distance = self.forward_no_reduction(input, target); + + match reduction { + Reduction::Mean | Reduction::Auto | Reduction::BatchMean => distance.mean(), + Reduction::Sum => distance.sum(), + } + } + + /// Compute DISTS distance without reduction. + /// + /// # Arguments + /// + /// * `input` - First image tensor of shape `[batch, 3, H, W]` + /// * `target` - Second image tensor of shape `[batch, 3, H, W]` + /// + /// # Returns + /// + /// Per-sample distance tensor of shape `[batch]`. + pub fn forward_no_reduction(&self, input: Tensor, target: Tensor) -> Tensor { + let [batch, _, _, _] = input.dims(); + + // Preprocess inputs + let (input, target) = self.preprocess(input, target); + + // Extract features from both images + let feats_x = self.extractor.forward(input); + let feats_y = self.extractor.forward(target); + + // Get alpha and beta weights + let alpha = self.alpha.val(); + let beta = self.beta.val(); + + // Compute weighted sum of alpha and beta for normalization + let alpha_sum = alpha.clone().sum(); + let beta_sum = beta.clone().sum(); + + let device = feats_x[0].device(); + + // Initialize accumulators + let mut structure_dist = Tensor::::zeros([batch], &device); + let mut texture_dist = Tensor::::zeros([batch], &device); + + let mut channel_offset = 0; + + // Compute similarity for each stage + for (feat_x, feat_y) in feats_x.iter().zip(feats_y.iter()) { + let [_b, c, _h, _w] = feat_x.dims(); + + // Get alpha and beta for this stage + let alpha_stage = alpha.clone().narrow(0, channel_offset, c); + let beta_stage = beta.clone().narrow(0, channel_offset, c); + + // Compute structure and texture similarity for this stage + let (s_dist, t_dist) = self.compute_stage_similarity( + feat_x.clone(), + feat_y.clone(), + alpha_stage, + beta_stage, + ); + + structure_dist = structure_dist.add(s_dist); + texture_dist = texture_dist.add(t_dist); + + channel_offset += c; + } + + // Normalize by sum of weights + structure_dist = structure_dist.div(alpha_sum); + texture_dist = texture_dist.div(beta_sum); + + // DISTS = 1 - (structure_similarity + texture_similarity) + // Since we computed distances (1 - similarity), we return the sum + structure_dist.add(texture_dist) + } + + /// Compute structure and texture similarity for a single stage. + fn compute_stage_similarity( + &self, + feat_x: Tensor, + feat_y: Tensor, + alpha: Tensor, + beta: Tensor, + ) -> (Tensor, Tensor) { + let [batch, channels, height, width] = feat_x.dims(); + let device = feat_x.device(); + + // Reshape to [batch, channels, H*W] for easier computation + let x = feat_x.reshape([batch, channels, height * width]); + let y = feat_y.reshape([batch, channels, height * width]); + + // Compute means: [batch, channels] (squeeze after mean_dim to remove the reduced dimension) + let mean_x = x.clone().mean_dim(2).squeeze_dim::<2>(2); + let mean_y = y.clone().mean_dim(2).squeeze_dim::<2>(2); + + // Compute structure similarity: (2*mean_x*mean_y + c1) / (mean_x^2 + mean_y^2 + c1) + let c1 = Tensor::::full([batch, channels], C1, &device); + let structure_sim = mean_x + .clone() + .mul(mean_y.clone()) + .mul_scalar(2.0) + .add(c1.clone()) + .div( + mean_x + .clone() + .mul(mean_x.clone()) + .add(mean_y.clone().mul(mean_y.clone())) + .add(c1), + ); + + // Compute variances and covariance + // var_x = E[x^2] - E[x]^2, clamped at 0 for numerical stability + let var_x = x + .clone() + .mul(x.clone()) + .mean_dim(2) + .squeeze_dim::<2>(2) + .sub(mean_x.clone().mul(mean_x.clone())) + .clamp_min(0.0); + let var_y = y + .clone() + .mul(y.clone()) + .mean_dim(2) + .squeeze_dim::<2>(2) + .sub(mean_y.clone().mul(mean_y.clone())) + .clamp_min(0.0); + + // cov_xy = E[xy] - E[x]E[y] + let cov_xy = x + .mul(y) + .mean_dim(2) + .squeeze_dim::<2>(2) + .sub(mean_x.clone().mul(mean_y.clone())); + + // Compute texture similarity: (2*cov_xy + c2) / (var_x + var_y + c2) + let c2 = Tensor::::full([batch, channels], C2, &device); + let texture_sim = cov_xy + .mul_scalar(2.0) + .add(c2.clone()) + .div(var_x.add(var_y).add(c2)); + + // Convert similarity to distance: 1 - similarity + let structure_dist = Tensor::::ones([batch, channels], &device).sub(structure_sim); + let texture_dist = Tensor::::ones([batch, channels], &device).sub(texture_sim); + + // Apply weights: [batch, channels] * [channels] -> [batch, channels] + // Then sum over channels -> [batch] + let weighted_structure = structure_dist + .mul(alpha.unsqueeze_dim::<2>(0)) + .sum_dim(1) + .squeeze_dim::<1>(1); + let weighted_texture = texture_dist + .mul(beta.unsqueeze_dim::<2>(0)) + .sum_dim(1) + .squeeze_dim::<1>(1); + + (weighted_structure, weighted_texture) + } + + /// Preprocess input images using the configured normalizer. + fn preprocess( + &self, + input: Tensor, + target: Tensor, + ) -> (Tensor, Tensor) { + match &self.normalizer { + Some(normalizer) => { + let input = normalizer.normalize(input); + let target = normalizer.normalize(target); + (input, target) + } + None => (input, target), + } + } +} + +// ============================================================================= +// Tests +// ============================================================================= + +#[cfg(test)] +mod tests { + use super::*; + use burn_core::tensor::{TensorData, Tolerance, ops::FloatElem}; + use burn_ndarray::NdArray; + + type TestBackend = NdArray; + type FT = FloatElem; + type TestTensor = Tensor; + + #[test] + fn test_dists_identical_images_zero_distance() { + let device = Default::default(); + // Use random image instead of constant to avoid numerical edge cases + let image = TestTensor::<4>::random( + [1, 3, 64, 64], + burn_core::tensor::Distribution::Uniform(0.0, 1.0), + &device, + ); + + let dists: Dists = DistsConfig::new().init(&device); + let distance = dists.forward(image.clone(), image, Reduction::Mean); + + let expected = TensorData::from([0.0]); + distance + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + #[test] + fn test_dists_different_images_nonzero_distance() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([1, 3, 64, 64], &device); + let image2 = TestTensor::<4>::ones([1, 3, 64, 64], &device); + + let dists: Dists = DistsConfig::new().init(&device); + let distance = dists.forward(image1, image2, Reduction::Mean); + + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value.abs() > 1e-6, + "DISTS should be != 0 for different images" + ); + } + + #[test] + fn test_dists_symmetry() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([1, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([1, 3, 32, 32], &device); + + let dists: Dists = DistsConfig::new().init(&device); + let distance_forward = dists.forward(image1.clone(), image2.clone(), Reduction::Mean); + let distance_reverse = dists.forward(image2, image1, Reduction::Mean); + + distance_forward + .into_data() + .assert_approx_eq::(&distance_reverse.into_data(), Tolerance::default()); + } + + #[test] + fn test_dists_batch_processing() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([2, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([2, 3, 32, 32], &device); + + let dists: Dists = DistsConfig::new().init(&device); + let distance = dists.forward(image1, image2, Reduction::Mean); + + assert_eq!(distance.dims(), [1]); + } + + #[test] + fn test_dists_no_reduction() { + let device = Default::default(); + + let batch_size = 4; + let image1 = TestTensor::<4>::zeros([batch_size, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([batch_size, 3, 32, 32], &device); + + let dists: Dists = DistsConfig::new().init(&device); + let distance = dists.forward_no_reduction(image1, image2); + + assert_eq!(distance.dims(), [batch_size]); + } + + #[test] + fn display_dists() { + let device = Default::default(); + let dists: Dists = DistsConfig::new().init(&device); + + let display_str = format!("{dists}"); + assert!(display_str.contains("Dists")); + assert!(display_str.contains("VGG16-L2Pool")); + } + + // ========================================================================= + // Pretrained Weights Tests (requires network) + // ========================================================================= + + /// Test DISTS pretrained weights download and loading. + #[test] + fn test_dists_pretrained() { + let device = Default::default(); + + let dists: Dists = DistsConfig::new().init_pretrained(&device); + + // Test with identical images - should be ~0 + // Use random image to avoid numerical edge cases with constant images + let image = TestTensor::<4>::random( + [1, 3, 64, 64], + burn_core::tensor::Distribution::Uniform(0.0, 1.0), + &device, + ); + let distance = dists.forward(image.clone(), image, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value.abs() < 1e-5, + "Pretrained DISTS should be ~0 for identical images, got {}", + distance_value + ); + + // Test with different images - should be positive + let image1 = TestTensor::<4>::random( + [1, 3, 64, 64], + burn_core::tensor::Distribution::Uniform(0.0, 0.3), + &device, + ); + let image2 = TestTensor::<4>::random( + [1, 3, 64, 64], + burn_core::tensor::Distribution::Uniform(0.7, 1.0), + &device, + ); + let distance = dists.forward(image1, image2, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value > 0.0, + "Pretrained DISTS should be > 0 for different images" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/mod.rs new file mode 100644 index 0000000..4c031ea --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/mod.rs @@ -0,0 +1,14 @@ +//! DISTS (Deep Image Structure and Texture Similarity) metric. +//! +//! This module implements DISTS, a full-reference image quality assessment metric +//! that combines structure and texture similarity using deep features. +//! +//! Reference: "Image Quality Assessment: Unifying Structure and Texture Similarity" +//! https://arxiv.org/abs/2004.07728 + +mod l2pool; +mod metric; +mod vgg16_l2pool; +mod weights; + +pub use metric::{Dists, DistsConfig}; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/vgg16_l2pool.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/vgg16_l2pool.rs new file mode 100644 index 0000000..f7e05f0 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/vgg16_l2pool.rs @@ -0,0 +1,169 @@ +//! VGG16 feature extractor with L2 Pooling for DISTS. +//! +//! This module implements the VGG16 backbone used in DISTS, +//! with L2Pooling replacing MaxPooling for smoother feature extraction. + +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::activation::relu; +use burn::tensor::backend::Backend; +use burn_nn::PaddingConfig2d; +use burn_nn::conv::{Conv2d, Conv2dConfig}; + +use super::l2pool::{L2Pool2d, L2Pool2dConfig}; + +/// VGG16 feature extractor with L2 Pooling for DISTS. +/// +/// Extracts features from 5 stages of VGG16, using L2Pooling +/// instead of MaxPooling for smoother downsampling. +/// +/// Output channels per stage: [64, 128, 256, 512, 512] +#[derive(Module, Debug)] +pub struct Vgg16L2PoolExtractor { + // Stage 1: 2 conv layers, 64 channels + pub(crate) conv1_1: Conv2d, + pub(crate) conv1_2: Conv2d, + pub(crate) pool1: L2Pool2d, + + // Stage 2: 2 conv layers, 128 channels + pub(crate) conv2_1: Conv2d, + pub(crate) conv2_2: Conv2d, + pub(crate) pool2: L2Pool2d, + + // Stage 3: 3 conv layers, 256 channels + pub(crate) conv3_1: Conv2d, + pub(crate) conv3_2: Conv2d, + pub(crate) conv3_3: Conv2d, + pub(crate) pool3: L2Pool2d, + + // Stage 4: 3 conv layers, 512 channels + pub(crate) conv4_1: Conv2d, + pub(crate) conv4_2: Conv2d, + pub(crate) conv4_3: Conv2d, + pub(crate) pool4: L2Pool2d, + + // Stage 5: 3 conv layers, 512 channels + pub(crate) conv5_1: Conv2d, + pub(crate) conv5_2: Conv2d, + pub(crate) conv5_3: Conv2d, +} + +impl Vgg16L2PoolExtractor { + /// Create a new VGG16 feature extractor with L2 Pooling. + pub fn new(device: &B::Device) -> Self { + let pool_config = L2Pool2dConfig::default(); + + Self { + // Stage 1 + conv1_1: Conv2dConfig::new([3, 64], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv1_2: Conv2dConfig::new([64, 64], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + pool1: pool_config.init(64, device), + + // Stage 2 + conv2_1: Conv2dConfig::new([64, 128], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv2_2: Conv2dConfig::new([128, 128], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + pool2: pool_config.init(128, device), + + // Stage 3 + conv3_1: Conv2dConfig::new([128, 256], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv3_2: Conv2dConfig::new([256, 256], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv3_3: Conv2dConfig::new([256, 256], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + pool3: pool_config.init(256, device), + + // Stage 4 + conv4_1: Conv2dConfig::new([256, 512], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv4_2: Conv2dConfig::new([512, 512], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv4_3: Conv2dConfig::new([512, 512], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + pool4: pool_config.init(512, device), + + // Stage 5 + conv5_1: Conv2dConfig::new([512, 512], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv5_2: Conv2dConfig::new([512, 512], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + conv5_3: Conv2dConfig::new([512, 512], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .init(device), + } + } + + /// Extract features from all 5 stages. + /// + /// # Arguments + /// + /// * `x` - Input tensor of shape `[batch, 3, height, width]` + /// + /// # Returns + /// + /// Vector of 6 feature tensors: + /// - Stage 0: Input image [batch, 3, H, W] + /// - Stage 1: After conv1 [batch, 64, H/2, W/2] + /// - Stage 2: After conv2 [batch, 128, H/4, W/4] + /// - Stage 3: After conv3 [batch, 256, H/8, W/8] + /// - Stage 4: After conv4 [batch, 512, H/16, W/16] + /// - Stage 5: After conv5 [batch, 512, H/32, W/32] + pub fn forward(&self, x: Tensor) -> Vec> { + let mut features = Vec::with_capacity(6); + + // Stage 0: Input image + features.push(x.clone()); + + // Stage 1 + let x = relu(self.conv1_1.forward(x)); + let x = relu(self.conv1_2.forward(x)); + features.push(x.clone()); + let x = self.pool1.forward(x); + + // Stage 2 + let x = relu(self.conv2_1.forward(x)); + let x = relu(self.conv2_2.forward(x)); + features.push(x.clone()); + let x = self.pool2.forward(x); + + // Stage 3 + let x = relu(self.conv3_1.forward(x)); + let x = relu(self.conv3_2.forward(x)); + let x = relu(self.conv3_3.forward(x)); + features.push(x.clone()); + let x = self.pool3.forward(x); + + // Stage 4 + let x = relu(self.conv4_1.forward(x)); + let x = relu(self.conv4_2.forward(x)); + let x = relu(self.conv4_3.forward(x)); + features.push(x.clone()); + let x = self.pool4.forward(x); + + // Stage 5 + let x = relu(self.conv5_1.forward(x)); + let x = relu(self.conv5_2.forward(x)); + let x = relu(self.conv5_3.forward(x)); + features.push(x); + + features + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/weights.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/weights.rs new file mode 100644 index 0000000..3948696 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/dists/weights.rs @@ -0,0 +1,128 @@ +//! Pretrained weights loading for DISTS. + +use burn_core as burn; + +use burn::tensor::backend::Backend; +use burn_std::network::downloader::download_file_as_bytes; +use burn_store::{ModuleSnapshot, PytorchStore}; +use std::fs::{File, create_dir_all}; +use std::io::Write; +use std::path::PathBuf; + +use super::metric::Dists; + +/// URL for pretrained DISTS alpha/beta weights from the official repository. +/// Reference: https://github.com/dingkeyan93/DISTS +const DISTS_WEIGHTS_URL: &str = + "https://github.com/dingkeyan93/DISTS/raw/master/DISTS_pytorch/weights.pt"; + +/// URL for ImageNet pretrained VGG16 backbone weights from PyTorch. +const VGG16_IMAGENET_URL: &str = "https://download.pytorch.org/models/vgg16-397923af.pth"; + +/// Get the cache directory for DISTS weights. +fn get_cache_dir() -> PathBuf { + let cache_dir = dirs::cache_dir() + .expect("Could not get cache directory") + .join("burn-dataset") + .join("dists"); + + if !cache_dir.exists() { + create_dir_all(&cache_dir).expect("Failed to create cache directory"); + } + + cache_dir +} + +/// Download file if not cached. +fn download_if_needed(url: &str, cache_path: &PathBuf, message: &str) { + if !cache_path.exists() { + let bytes = download_file_as_bytes(url, message); + let mut file = File::create(cache_path).expect("Failed to create cache file"); + file.write_all(&bytes).expect("Failed to write weights"); + } +} + +/// Download and load pretrained weights into a DISTS module. +/// +/// This loads both: +/// 1. ImageNet pretrained VGG16 backbone weights +/// 2. DISTS trained alpha/beta weights +/// +/// Weights are cached in the user's cache directory to avoid re-downloading. +/// +/// # Arguments +/// +/// * `dists` - The DISTS module to load weights into. +/// +/// # Returns +/// +/// The DISTS module with loaded pretrained weights. +pub fn load_pretrained_weights(mut dists: Dists) -> Dists { + let cache_dir = get_cache_dir(); + + // Step 1: Download and load VGG16 ImageNet backbone weights + let vgg_cache_path = cache_dir.join("vgg16_backbone.pth"); + download_if_needed( + VGG16_IMAGENET_URL, + &vgg_cache_path, + "Downloading VGG16 ImageNet weights for DISTS...", + ); + + // Step 2: Download DISTS alpha/beta weights + let dists_cache_path = cache_dir.join("dists_weights.pt"); + download_if_needed( + DISTS_WEIGHTS_URL, + &dists_cache_path, + "Downloading DISTS alpha/beta weights...", + ); + + // Load VGG16 backbone weights first + dists = load_vgg16_backbone_weights(dists, &vgg_cache_path); + + // Then load DISTS alpha/beta weights + dists = load_dists_weights(dists, &dists_cache_path); + + dists +} + +/// Load VGG16 ImageNet pretrained backbone weights. +fn load_vgg16_backbone_weights(mut dists: Dists, cache_path: &PathBuf) -> Dists { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + .skip_enum_variants(true) + // VGG16 features.X -> extractor.convY_Z + .with_key_remapping(r"^features\.0\.", "extractor.conv1_1.") + .with_key_remapping(r"^features\.2\.", "extractor.conv1_2.") + .with_key_remapping(r"^features\.5\.", "extractor.conv2_1.") + .with_key_remapping(r"^features\.7\.", "extractor.conv2_2.") + .with_key_remapping(r"^features\.10\.", "extractor.conv3_1.") + .with_key_remapping(r"^features\.12\.", "extractor.conv3_2.") + .with_key_remapping(r"^features\.14\.", "extractor.conv3_3.") + .with_key_remapping(r"^features\.17\.", "extractor.conv4_1.") + .with_key_remapping(r"^features\.19\.", "extractor.conv4_2.") + .with_key_remapping(r"^features\.21\.", "extractor.conv4_3.") + .with_key_remapping(r"^features\.24\.", "extractor.conv5_1.") + .with_key_remapping(r"^features\.26\.", "extractor.conv5_2.") + .with_key_remapping(r"^features\.28\.", "extractor.conv5_3."); + + let result = dists.load_from(&mut store); + if let Err(e) = result { + log::warn!("Some VGG16 backbone weights could not be loaded: {:?}", e); + } + + dists +} + +/// Load DISTS trained alpha/beta weights. +fn load_dists_weights(mut dists: Dists, cache_path: &PathBuf) -> Dists { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + .skip_enum_variants(true); + + let result = dists.load_from(&mut store); + if let Err(e) = result { + log::warn!("Some DISTS weights could not be loaded: {:?}", e); + } + + dists +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/alexnet.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/alexnet.rs new file mode 100644 index 0000000..1dd1d7e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/alexnet.rs @@ -0,0 +1,100 @@ +//! AlexNet feature extractor for LPIPS. + +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::activation::relu; +use burn::tensor::backend::Backend; +use burn_nn::PaddingConfig2d; +use burn_nn::conv::{Conv2d, Conv2dConfig}; + +/// AlexNet feature extractor for LPIPS. +/// +/// Extracts features from 5 layers: +/// - conv1: 64 channels (after ReLU) +/// - conv2: 192 channels (after ReLU) +/// - conv3: 384 channels (after ReLU) +/// - conv4: 256 channels (after ReLU) +/// - conv5: 256 channels (after ReLU) +#[derive(Module, Debug)] +pub struct AlexFeatureExtractor { + /// Conv1: 3 -> 64, kernel 11x11, stride 4, padding 2 + conv1: Conv2d, + /// Conv2: 64 -> 192, kernel 5x5, stride 1, padding 2 + conv2: Conv2d, + /// Conv3: 192 -> 384, kernel 3x3, stride 1, padding 1 + conv3: Conv2d, + /// Conv4: 384 -> 256, kernel 3x3, stride 1, padding 1 + conv4: Conv2d, + /// Conv5: 256 -> 256, kernel 3x3, stride 1, padding 1 + conv5: Conv2d, +} + +impl AlexFeatureExtractor { + /// Create a new AlexNet feature extractor. + pub fn new(device: &B::Device) -> Self { + Self { + // Conv1: 3 -> 64, 11x11, stride 4, padding 2 + conv1: Conv2dConfig::new([3, 64], [11, 11]) + .with_stride([4, 4]) + .with_padding(PaddingConfig2d::Explicit(2, 2, 2, 2)) + .with_bias(true) + .init(device), + // Conv2: 64 -> 192, 5x5, stride 1, padding 2 + conv2: Conv2dConfig::new([64, 192], [5, 5]) + .with_padding(PaddingConfig2d::Explicit(2, 2, 2, 2)) + .with_bias(true) + .init(device), + // Conv3: 192 -> 384, 3x3, stride 1, padding 1 + conv3: Conv2dConfig::new([192, 384], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .with_bias(true) + .init(device), + // Conv4: 384 -> 256, 3x3, stride 1, padding 1 + conv4: Conv2dConfig::new([384, 256], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .with_bias(true) + .init(device), + // Conv5: 256 -> 256, 3x3, stride 1, padding 1 + conv5: Conv2dConfig::new([256, 256], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .with_bias(true) + .init(device), + } + } + + /// Extract features from 5 AlexNet layers. + pub fn forward(&self, x: Tensor) -> Vec> { + let mut features = Vec::with_capacity(5); + + // Slice 1: Conv1 + ReLU + let x = relu(self.conv1.forward(x)); + features.push(x.clone()); + + // Slice 2: MaxPool + Conv2 + ReLU + let x = max_pool2d_alex(x); + let x = relu(self.conv2.forward(x)); + features.push(x.clone()); + + // Slice 3: MaxPool + Conv3 + ReLU + let x = max_pool2d_alex(x); + let x = relu(self.conv3.forward(x)); + features.push(x.clone()); + + // Slice 4: Conv4 + ReLU (no pooling) + let x = relu(self.conv4.forward(x)); + features.push(x.clone()); + + // Slice 5: Conv5 + ReLU (no pooling) + let x = relu(self.conv5.forward(x)); + features.push(x); + + features + } +} + +/// 3x3 max pooling with stride 2 (for AlexNet). +fn max_pool2d_alex(x: Tensor) -> Tensor { + burn_core::tensor::module::max_pool2d(x, [3, 3], [2, 2], [0, 0], [1, 1], false) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/metric.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/metric.rs new file mode 100644 index 0000000..824323b --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/metric.rs @@ -0,0 +1,802 @@ +//! LPIPS (Learned Perceptual Image Patch Similarity) metric module. +//! +//! LPIPS measures perceptual similarity between images using deep features. +//! Supports VGG16, AlexNet, and SqueezeNet as backbone networks. +//! +//! Reference: "The Unreasonable Effectiveness of Deep Features as a Perceptual Metric" +//! + +use burn_core as burn; + +use burn::config::Config; +use burn::module::{Content, DisplaySettings, Module, ModuleDisplay}; +use burn::tensor::Tensor; +use burn::tensor::backend::Backend; +use burn_nn::conv::{Conv2d, Conv2dConfig}; +use burn_nn::loss::Reduction; + +use super::alexnet::AlexFeatureExtractor; +use super::squeezenet::SqueezeFeatureExtractor; +use super::vgg::VggFeatureExtractor; + +/// Network type for LPIPS. +#[derive(Config, Debug, Copy, PartialEq, Eq)] +pub enum LpipsNet { + /// VGG16 network (default) + Vgg, + /// AlexNet network + Alex, + /// SqueezeNet network + Squeeze, +} + +/// Configuration for [Lpips](Lpips) metric module. +/// +/// # Example +/// +/// ```ignore +/// use burn_train::metric::vision::{LpipsConfig, LpipsNet}; +/// +/// // VGG (default) +/// let lpips_vgg = LpipsConfig::new().init(&device); +/// +/// // AlexNet +/// let lpips_alex = LpipsConfig::new() +/// .with_net(LpipsNet::Alex) +/// .init(&device); +/// +/// // SqueezeNet +/// let lpips_squeeze = LpipsConfig::new() +/// .with_net(LpipsNet::Squeeze) +/// .init(&device); +/// ``` +#[derive(Config, Debug)] +pub struct LpipsConfig { + /// Network type for feature extraction. + #[config(default = "LpipsNet::Vgg")] + pub net: LpipsNet, + + /// Whether to normalize input images to [-1, 1] range. + /// Set to true if input is in [0, 1] range. + #[config(default = true)] + pub normalize: bool, +} + +impl LpipsConfig { + /// Initialize a new [Lpips](Lpips) module with pretrained weights. + /// + /// Downloads and loads official LPIPS pretrained weights from the + /// PerceptualSimilarity repository. + /// + /// # Arguments + /// + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A new LPIPS module with pretrained weights loaded. + /// + /// # Example + /// + /// ```ignore + /// use burn_train::metric::vision::{LpipsConfig, LpipsNet}; + /// + /// let lpips = LpipsConfig::new() + /// .with_net(LpipsNet::Vgg) + /// .init_pretrained(&device); + /// ``` + pub fn init_pretrained(&self, device: &B::Device) -> Lpips { + let lpips = self.init(device); + super::weights::load_pretrained_weights(lpips, self.net) + } + + /// Initialize a new [Lpips](Lpips) module with random weights. + /// + /// # Arguments + /// + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A new LPIPS module with random weights. Use `init_pretrained` for accurate results. + pub fn init(&self, device: &B::Device) -> Lpips { + match self.net { + LpipsNet::Vgg => { + // Channel sizes for VGG16: [64, 128, 256, 512, 512] + Lpips::Vgg(LpipsVgg { + extractor: VggFeatureExtractor::new(device), + lin0: Conv2dConfig::new([64, 1], [1, 1]) + .with_bias(false) + .init(device), + lin1: Conv2dConfig::new([128, 1], [1, 1]) + .with_bias(false) + .init(device), + lin2: Conv2dConfig::new([256, 1], [1, 1]) + .with_bias(false) + .init(device), + lin3: Conv2dConfig::new([512, 1], [1, 1]) + .with_bias(false) + .init(device), + lin4: Conv2dConfig::new([512, 1], [1, 1]) + .with_bias(false) + .init(device), + normalize: self.normalize, + }) + } + LpipsNet::Alex => { + // Channel sizes for AlexNet: [64, 192, 384, 256, 256] + Lpips::Alex(LpipsAlex { + extractor: AlexFeatureExtractor::new(device), + lin0: Conv2dConfig::new([64, 1], [1, 1]) + .with_bias(false) + .init(device), + lin1: Conv2dConfig::new([192, 1], [1, 1]) + .with_bias(false) + .init(device), + lin2: Conv2dConfig::new([384, 1], [1, 1]) + .with_bias(false) + .init(device), + lin3: Conv2dConfig::new([256, 1], [1, 1]) + .with_bias(false) + .init(device), + lin4: Conv2dConfig::new([256, 1], [1, 1]) + .with_bias(false) + .init(device), + normalize: self.normalize, + }) + } + LpipsNet::Squeeze => { + // Channel sizes for SqueezeNet: [64, 128, 256, 384, 384, 512, 512] + Lpips::Squeeze(LpipsSqueeze { + extractor: SqueezeFeatureExtractor::new(device), + lin0: Conv2dConfig::new([64, 1], [1, 1]) + .with_bias(false) + .init(device), + lin1: Conv2dConfig::new([128, 1], [1, 1]) + .with_bias(false) + .init(device), + lin2: Conv2dConfig::new([256, 1], [1, 1]) + .with_bias(false) + .init(device), + lin3: Conv2dConfig::new([384, 1], [1, 1]) + .with_bias(false) + .init(device), + lin4: Conv2dConfig::new([384, 1], [1, 1]) + .with_bias(false) + .init(device), + lin5: Conv2dConfig::new([512, 1], [1, 1]) + .with_bias(false) + .init(device), + lin6: Conv2dConfig::new([512, 1], [1, 1]) + .with_bias(false) + .init(device), + normalize: self.normalize, + }) + } + } + } +} + +/// LPIPS (Learned Perceptual Image Patch Similarity) metric module. +/// +/// Computes perceptual distance between two images using deep features. +/// Supports VGG16, AlexNet, and SqueezeNet as backbone networks. +/// +/// # Example +/// +/// ```ignore +/// use burn_train::metric::vision::{LpipsConfig, LpipsNet, Reduction}; +/// +/// let device = Default::default(); +/// let lpips = LpipsConfig::new().init(&device); +/// +/// let img1: Tensor = /* [batch, 3, H, W] */; +/// let img2: Tensor = /* [batch, 3, H, W] */; +/// +/// // Compute LPIPS distance +/// let distance = lpips.forward(img1, img2, Reduction::Mean); +/// ``` +#[derive(Module, Debug)] +#[allow(clippy::large_enum_variant)] +#[module(custom_display)] +pub enum Lpips { + /// VGG16 backbone (5 feature layers) + Vgg(LpipsVgg), + /// AlexNet backbone (5 feature layers) + Alex(LpipsAlex), + /// SqueezeNet backbone (7 feature layers) + Squeeze(LpipsSqueeze), +} + +/// LPIPS with VGG16 backbone. +#[derive(Module, Debug)] +pub struct LpipsVgg { + /// VGG feature extractor + pub(crate) extractor: VggFeatureExtractor, + /// Linear layers for each feature level + pub(crate) lin0: Conv2d, + pub(crate) lin1: Conv2d, + pub(crate) lin2: Conv2d, + pub(crate) lin3: Conv2d, + pub(crate) lin4: Conv2d, + /// Whether to normalize input + pub(crate) normalize: bool, +} + +/// LPIPS with AlexNet backbone. +#[derive(Module, Debug)] +pub struct LpipsAlex { + /// AlexNet feature extractor + pub(crate) extractor: AlexFeatureExtractor, + /// Linear layers for each feature level + pub(crate) lin0: Conv2d, + pub(crate) lin1: Conv2d, + pub(crate) lin2: Conv2d, + pub(crate) lin3: Conv2d, + pub(crate) lin4: Conv2d, + /// Whether to normalize input + pub(crate) normalize: bool, +} + +/// LPIPS with SqueezeNet backbone. +#[derive(Module, Debug)] +pub struct LpipsSqueeze { + /// SqueezeNet feature extractor + pub(crate) extractor: SqueezeFeatureExtractor, + /// Linear layers for each feature level + pub(crate) lin0: Conv2d, + pub(crate) lin1: Conv2d, + pub(crate) lin2: Conv2d, + pub(crate) lin3: Conv2d, + pub(crate) lin4: Conv2d, + pub(crate) lin5: Conv2d, + pub(crate) lin6: Conv2d, + /// Whether to normalize input + pub(crate) normalize: bool, +} + +impl LpipsVgg { + /// Compute LPIPS distance without reduction using VGG backbone. + pub fn forward_no_reduction(&self, input: Tensor, target: Tensor) -> Tensor { + // Preprocess inputs + let (input, target) = preprocess_inputs(input, target, self.normalize); + + // Extract features from both images + let feats0 = self.extractor.forward(input); + let feats1 = self.extractor.forward(target); + + // Compute distance for each layer using stack + sum + let layer_distances: Vec> = vec![ + compute_layer_distance(&feats0[0], &feats1[0], &self.lin0).unsqueeze_dim(1), + compute_layer_distance(&feats0[1], &feats1[1], &self.lin1).unsqueeze_dim(1), + compute_layer_distance(&feats0[2], &feats1[2], &self.lin2).unsqueeze_dim(1), + compute_layer_distance(&feats0[3], &feats1[3], &self.lin3).unsqueeze_dim(1), + compute_layer_distance(&feats0[4], &feats1[4], &self.lin4).unsqueeze_dim(1), + ]; + + Tensor::cat(layer_distances, 1) + .sum_dim(1) + .squeeze_dim::<1>(1) + } +} + +impl LpipsAlex { + /// Compute LPIPS distance without reduction using AlexNet backbone. + pub fn forward_no_reduction(&self, input: Tensor, target: Tensor) -> Tensor { + // Preprocess inputs + let (input, target) = preprocess_inputs(input, target, self.normalize); + + // Extract features from both images + let feats0 = self.extractor.forward(input); + let feats1 = self.extractor.forward(target); + + // Compute distance for each layer using stack + sum + let layer_distances: Vec> = vec![ + compute_layer_distance(&feats0[0], &feats1[0], &self.lin0).unsqueeze_dim(1), + compute_layer_distance(&feats0[1], &feats1[1], &self.lin1).unsqueeze_dim(1), + compute_layer_distance(&feats0[2], &feats1[2], &self.lin2).unsqueeze_dim(1), + compute_layer_distance(&feats0[3], &feats1[3], &self.lin3).unsqueeze_dim(1), + compute_layer_distance(&feats0[4], &feats1[4], &self.lin4).unsqueeze_dim(1), + ]; + + Tensor::cat(layer_distances, 1) + .sum_dim(1) + .squeeze_dim::<1>(1) + } +} + +impl LpipsSqueeze { + /// Compute LPIPS distance without reduction using SqueezeNet backbone. + pub fn forward_no_reduction(&self, input: Tensor, target: Tensor) -> Tensor { + // Preprocess inputs + let (input, target) = preprocess_inputs(input, target, self.normalize); + + // Extract features from both images + let feats0 = self.extractor.forward(input); + let feats1 = self.extractor.forward(target); + + // Compute distance for each layer using stack + sum (7 layers for SqueezeNet) + let layer_distances: Vec> = vec![ + compute_layer_distance(&feats0[0], &feats1[0], &self.lin0).unsqueeze_dim(1), + compute_layer_distance(&feats0[1], &feats1[1], &self.lin1).unsqueeze_dim(1), + compute_layer_distance(&feats0[2], &feats1[2], &self.lin2).unsqueeze_dim(1), + compute_layer_distance(&feats0[3], &feats1[3], &self.lin3).unsqueeze_dim(1), + compute_layer_distance(&feats0[4], &feats1[4], &self.lin4).unsqueeze_dim(1), + compute_layer_distance(&feats0[5], &feats1[5], &self.lin5).unsqueeze_dim(1), + compute_layer_distance(&feats0[6], &feats1[6], &self.lin6).unsqueeze_dim(1), + ]; + + Tensor::cat(layer_distances, 1) + .sum_dim(1) + .squeeze_dim::<1>(1) + } +} + +impl ModuleDisplay for Lpips { + fn custom_settings(&self) -> Option { + DisplaySettings::new() + .with_new_line_after_attribute(false) + .optional() + } + + fn custom_content(&self, content: Content) -> Option { + let (net_name, normalize) = match self { + Lpips::Vgg(inner) => ("Vgg", inner.normalize), + Lpips::Alex(inner) => ("Alex", inner.normalize), + Lpips::Squeeze(inner) => ("Squeeze", inner.normalize), + }; + content + .add("net", &net_name.to_string()) + .add("normalize", &normalize.to_string()) + .optional() + } +} + +impl Lpips { + /// Compute LPIPS distance with reduction. + /// + /// # Arguments + /// + /// * `input` - First image tensor of shape `[batch, 3, H, W]` + /// * `target` - Second image tensor of shape `[batch, 3, H, W]` + /// * `reduction` - How to reduce the output (Mean, Sum, or Auto) + /// + /// # Returns + /// + /// Scalar tensor of shape `[1]`. + /// + /// # Shapes + /// + /// - input: `[batch, 3, H, W]` + /// - target: `[batch, 3, H, W]` + /// - output: `[1]` + pub fn forward( + &self, + input: Tensor, + target: Tensor, + reduction: Reduction, + ) -> Tensor { + let distance = self.forward_no_reduction(input, target); + + match reduction { + Reduction::Mean | Reduction::Auto | Reduction::BatchMean => distance.mean(), + Reduction::Sum => distance.sum(), + } + } + + /// Compute LPIPS distance without reduction. + /// + /// # Arguments + /// + /// * `input` - First image tensor of shape `[batch, 3, H, W]` + /// * `target` - Second image tensor of shape `[batch, 3, H, W]` + /// + /// # Returns + /// + /// Per-sample distance tensor of shape `[batch]`. + /// + /// # Shapes + /// + /// - input: `[batch, 3, H, W]` + /// - target: `[batch, 3, H, W]` + /// - output: `[batch]` + pub fn forward_no_reduction(&self, input: Tensor, target: Tensor) -> Tensor { + match self { + Lpips::Vgg(inner) => inner.forward_no_reduction(input, target), + Lpips::Alex(inner) => inner.forward_no_reduction(input, target), + Lpips::Squeeze(inner) => inner.forward_no_reduction(input, target), + } + } +} + +// ============================================================================= +// Helper Functions +// ============================================================================= + +/// Normalize tensor to unit norm along channel dimension. +fn normalize_tensor(x: Tensor) -> Tensor { + let norm = x.clone().mul(x.clone()).sum_dim(1).sqrt().clamp_min(1e-10); + x.div(norm) +} + +/// Apply ImageNet normalization used by PyTorch lpips. +/// shift = [-.030, -.088, -.188], scale = [.458, .448, .450] +/// output = (input - shift) / scale +fn scaling_layer(x: Tensor) -> Tensor { + let device = x.device(); + let [batch, _, h, w] = x.dims(); + + // Create shift and scale tensors [1, 3, 1, 1] and broadcast + let shift = Tensor::::from_floats([[-0.030], [-0.088], [-0.188]], &device) + .reshape([1, 3, 1, 1]) + .expand([batch, 3, h, w]); + let scale = Tensor::::from_floats([[0.458], [0.448], [0.450]], &device) + .reshape([1, 3, 1, 1]) + .expand([batch, 3, h, w]); + + x.sub(shift).div(scale) +} + +/// Compute normalized L2 distance for a single layer. +fn compute_layer_distance( + feat0: &Tensor, + feat1: &Tensor, + lin: &Conv2d, +) -> Tensor { + // Normalize features (unit norm along channel dimension) + let feat0_norm = normalize_tensor(feat0.clone()); + let feat1_norm = normalize_tensor(feat1.clone()); + + // Compute squared difference + let diff = feat0_norm.sub(feat1_norm); + let diff_sq = diff.clone().mul(diff); + + // Apply linear layer (learned weights) + // Shape: [batch, C, H, W] -> [batch, 1, H, W] + let weighted = lin.forward(diff_sq); + + // Spatial average: compute mean over C, H, W dimensions + // Shape: [batch, 1, H, W] -> [batch] + let [batch, c, h, w] = weighted.dims(); + + // Reshape to [batch, c*h*w] then take mean over last dimension + weighted + .reshape([batch, c * h * w]) + .mean_dim(1) + .squeeze_dim::<1>(1) +} + +/// Preprocess input images for LPIPS computation. +fn preprocess_inputs( + input: Tensor, + target: Tensor, + normalize: bool, +) -> (Tensor, Tensor) { + // Normalize to [-1, 1] if needed + let (input, target) = if normalize { + ( + input.mul_scalar(2.0).sub_scalar(1.0), + target.mul_scalar(2.0).sub_scalar(1.0), + ) + } else { + (input, target) + }; + + // Apply ImageNet normalization (same as PyTorch lpips scaling_layer) + (scaling_layer(input), scaling_layer(target)) +} + +// ============================================================================= +// Tests +// ============================================================================= + +#[cfg(test)] +mod tests { + use super::*; + use burn_core::tensor::{TensorData, Tolerance, ops::FloatElem}; + use burn_ndarray::NdArray; + + type TestBackend = NdArray; + type FT = FloatElem; + type TestTensor = Tensor; + + // ========================================================================= + // Basic Functionality Tests + // ========================================================================= + + /// Identical images should have LPIPS distance of 0. + #[test] + fn test_lpips_identical_images_zero_distance() { + let device = Default::default(); + let image = TestTensor::<4>::ones([1, 3, 32, 32], &device); + + let lpips: Lpips = LpipsConfig::new().init(&device); + let distance = lpips.forward(image.clone(), image, Reduction::Mean); + + // Identical images → distance = 0 + let expected = TensorData::from([0.0]); + distance + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + /// Different images should have LPIPS distance != 0. + /// Note: With random weights, distance can be negative, so we only check != 0. + /// Non-negativity is tested with pretrained weights. + #[test] + fn test_lpips_different_images_nonzero_distance() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([1, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([1, 3, 32, 32], &device); + + let lpips: Lpips = LpipsConfig::new().init(&device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value.abs() > 1e-6, + "LPIPS should be != 0 for different images" + ); + } + + /// Test symmetry: LPIPS(a, b) == LPIPS(b, a). + #[test] + fn test_lpips_symmetry() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([1, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([1, 3, 32, 32], &device); + + let lpips: Lpips = LpipsConfig::new().init(&device); + let distance_forward = lpips.forward(image1.clone(), image2.clone(), Reduction::Mean); + let distance_reverse = lpips.forward(image2, image1, Reduction::Mean); + + distance_forward + .into_data() + .assert_approx_eq::(&distance_reverse.into_data(), Tolerance::default()); + } + + // ========================================================================= + // Reduction Tests + // ========================================================================= + + #[test] + fn test_lpips_forward_mean_reduction() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([2, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([2, 3, 32, 32], &device); + + let lpips: Lpips = LpipsConfig::new().init(&device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + + assert_eq!(distance.dims(), [1]); + } + + #[test] + fn test_lpips_forward_no_reduction() { + let device = Default::default(); + + let batch_size = 4; + let image1 = TestTensor::<4>::zeros([batch_size, 3, 32, 32], &device); + let image2 = TestTensor::<4>::ones([batch_size, 3, 32, 32], &device); + + let lpips: Lpips = LpipsConfig::new().init(&device); + let distance = lpips.forward_no_reduction(image1, image2); + + assert_eq!(distance.dims(), [batch_size]); + } + + // ========================================================================= + // AlexNet Tests + // ========================================================================= + + /// Test AlexNet LPIPS with identical images. + #[test] + fn test_lpips_alex_identical_images_zero_distance() { + let device = Default::default(); + let image = TestTensor::<4>::ones([1, 3, 64, 64], &device); + + let lpips: Lpips = LpipsConfig::new().with_net(LpipsNet::Alex).init(&device); + let distance = lpips.forward(image.clone(), image, Reduction::Mean); + + let expected = TensorData::from([0.0]); + distance + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + /// Test AlexNet LPIPS with different images produces non-zero distance. + #[test] + fn test_lpips_alex_different_images_nonzero_distance() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([1, 3, 64, 64], &device); + let image2 = TestTensor::<4>::ones([1, 3, 64, 64], &device); + + let lpips: Lpips = LpipsConfig::new().with_net(LpipsNet::Alex).init(&device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + // Note: With random weights, non-negativity is not guaranteed. + // We only check that different images produce a non-zero distance. + assert!( + distance_value.abs() > 1e-6, + "LPIPS (Alex) should be != 0 for different images" + ); + } + + // ========================================================================= + // SqueezeNet Tests + // ========================================================================= + + /// Test SqueezeNet LPIPS with identical images. + #[test] + fn test_lpips_squeeze_identical_images_zero_distance() { + let device = Default::default(); + let image = TestTensor::<4>::ones([1, 3, 64, 64], &device); + + let lpips: Lpips = + LpipsConfig::new().with_net(LpipsNet::Squeeze).init(&device); + let distance = lpips.forward(image.clone(), image, Reduction::Mean); + + let expected = TensorData::from([0.0]); + distance + .into_data() + .assert_approx_eq::(&expected, Tolerance::default()); + } + + /// Test SqueezeNet LPIPS with different images produces non-zero distance. + #[test] + fn test_lpips_squeeze_different_images_nonzero_distance() { + let device = Default::default(); + + let image1 = TestTensor::<4>::zeros([1, 3, 64, 64], &device); + let image2 = TestTensor::<4>::ones([1, 3, 64, 64], &device); + + let lpips: Lpips = + LpipsConfig::new().with_net(LpipsNet::Squeeze).init(&device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + // Note: With random weights, non-negativity is not guaranteed. + // We only check that different images produce a non-zero distance. + assert!( + distance_value.abs() > 1e-6, + "LPIPS (Squeeze) should be != 0 for different images" + ); + } + + // ========================================================================= + // Display Tests + // ========================================================================= + + #[test] + fn display_vgg() { + let device = Default::default(); + let lpips: Lpips = LpipsConfig::new().init(&device); + + let display_str = format!("{lpips}"); + assert!(display_str.contains("Lpips")); + assert!(display_str.contains("Vgg")); + } + + #[test] + fn display_alex() { + let device = Default::default(); + let lpips: Lpips = LpipsConfig::new().with_net(LpipsNet::Alex).init(&device); + + let display_str = format!("{lpips}"); + assert!(display_str.contains("Lpips")); + assert!(display_str.contains("Alex")); + } + + #[test] + fn display_squeeze() { + let device = Default::default(); + let lpips: Lpips = + LpipsConfig::new().with_net(LpipsNet::Squeeze).init(&device); + + let display_str = format!("{lpips}"); + assert!(display_str.contains("Lpips")); + assert!(display_str.contains("Squeeze")); + } + + // ========================================================================= + // Pretrained Weights Tests (requires network) + // ========================================================================= + + /// Test VGG pretrained weights download and loading. + #[test] + fn test_lpips_pretrained_vgg() { + let device = Default::default(); + + // This will download ~60MB of weights + let lpips: Lpips = LpipsConfig::new() + .with_net(LpipsNet::Vgg) + .init_pretrained(&device); + + // Test with identical images - should be 0 + let image = TestTensor::<4>::ones([1, 3, 64, 64], &device); + let distance = lpips.forward(image.clone(), image, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value.abs() < 1e-5, + "Pretrained LPIPS (VGG) should be ~0 for identical images, got {}", + distance_value + ); + + // Test with different images - should be positive + let image1 = TestTensor::<4>::zeros([1, 3, 64, 64], &device); + let image2 = TestTensor::<4>::ones([1, 3, 64, 64], &device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value > 0.0, + "Pretrained LPIPS (VGG) should be > 0 for different images, got {}", + distance_value + ); + } + + /// Test AlexNet pretrained weights download and loading. + #[test] + fn test_lpips_pretrained_alex() { + let device = Default::default(); + + let lpips: Lpips = LpipsConfig::new() + .with_net(LpipsNet::Alex) + .init_pretrained(&device); + + // Test with identical images + let image = TestTensor::<4>::ones([1, 3, 64, 64], &device); + let distance = lpips.forward(image.clone(), image, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value.abs() < 1e-5, + "Pretrained LPIPS (Alex) should be ~0 for identical images, got {}", + distance_value + ); + + // Test with different images + let image1 = TestTensor::<4>::zeros([1, 3, 64, 64], &device); + let image2 = TestTensor::<4>::ones([1, 3, 64, 64], &device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value > 0.0, + "Pretrained LPIPS (Alex) should be > 0 for different images" + ); + } + + /// Test SqueezeNet pretrained weights download and loading. + #[test] + fn test_lpips_pretrained_squeeze() { + let device = Default::default(); + + let lpips: Lpips = LpipsConfig::new() + .with_net(LpipsNet::Squeeze) + .init_pretrained(&device); + + // Test with identical images + let image = TestTensor::<4>::ones([1, 3, 64, 64], &device); + let distance = lpips.forward(image.clone(), image, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value.abs() < 1e-5, + "Pretrained LPIPS (Squeeze) should be ~0 for identical images, got {}", + distance_value + ); + + // Test with different images + let image1 = TestTensor::<4>::zeros([1, 3, 64, 64], &device); + let image2 = TestTensor::<4>::ones([1, 3, 64, 64], &device); + let distance = lpips.forward(image1, image2, Reduction::Mean); + let distance_value = distance.into_data().to_vec::().unwrap()[0]; + assert!( + distance_value > 0.0, + "Pretrained LPIPS (Squeeze) should be > 0 for different images, got {}", + distance_value + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/mod.rs new file mode 100644 index 0000000..9e5e6a5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/mod.rs @@ -0,0 +1,21 @@ +//! LPIPS (Learned Perceptual Image Patch Similarity) metric module. +//! +//! LPIPS measures perceptual similarity between images using deep features. +//! Supports VGG16, AlexNet, and SqueezeNet as backbone networks. +//! +//! Reference: "The Unreasonable Effectiveness of Deep Features as a Perceptual Metric" +//! + +mod alexnet; +mod metric; +mod squeezenet; +mod vgg; +mod weights; + +pub use metric::{Lpips, LpipsAlex, LpipsConfig, LpipsNet, LpipsSqueeze, LpipsVgg}; +pub use weights::{get_backbone_weights_url, get_lpips_weights_url, load_pretrained_weights}; + +// Re-export feature extractors for advanced use cases +pub use alexnet::AlexFeatureExtractor; +pub use squeezenet::{FireModule, SqueezeFeatureExtractor}; +pub use vgg::VggFeatureExtractor; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/squeezenet.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/squeezenet.rs new file mode 100644 index 0000000..4f67a51 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/squeezenet.rs @@ -0,0 +1,157 @@ +//! SqueezeNet feature extractor for LPIPS. + +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::activation::relu; +use burn::tensor::backend::Backend; +use burn_nn::PaddingConfig2d; +use burn_nn::conv::{Conv2d, Conv2dConfig}; + +/// Fire module for SqueezeNet. +/// +/// A fire module consists of: +/// - Squeeze layer: 1x1 conv to reduce channels +/// - Expand layers: parallel 1x1 and 3x3 convs, concatenated +#[derive(Module, Debug)] +pub struct FireModule { + /// Squeeze layer: 1x1 conv + squeeze: Conv2d, + /// Expand 1x1 conv + expand1x1: Conv2d, + /// Expand 3x3 conv + expand3x3: Conv2d, +} + +impl FireModule { + /// Create a new Fire module. + pub fn new( + in_channels: usize, + squeeze_channels: usize, + expand1x1_channels: usize, + expand3x3_channels: usize, + device: &B::Device, + ) -> Self { + Self { + squeeze: Conv2dConfig::new([in_channels, squeeze_channels], [1, 1]) + .with_bias(true) + .init(device), + expand1x1: Conv2dConfig::new([squeeze_channels, expand1x1_channels], [1, 1]) + .with_bias(true) + .init(device), + expand3x3: Conv2dConfig::new([squeeze_channels, expand3x3_channels], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1)) + .with_bias(true) + .init(device), + } + } + + /// Forward pass through fire module. + pub fn forward(&self, x: Tensor) -> Tensor { + let squeezed = relu(self.squeeze.forward(x)); + let e1 = relu(self.expand1x1.forward(squeezed.clone())); + let e3 = relu(self.expand3x3.forward(squeezed)); + // Concatenate along channel dimension + Tensor::cat(vec![e1, e3], 1) + } +} + +/// SqueezeNet 1.1 feature extractor for LPIPS. +/// +/// Extracts features from 7 layers: +/// - After conv1+relu: 64 channels +/// - After fire1+fire2: 128 channels +/// - After fire3+fire4: 256 channels +/// - After fire5: 384 channels +/// - After fire6: 384 channels +/// - After fire7: 512 channels +/// - After fire8: 512 channels +#[derive(Module, Debug)] +pub struct SqueezeFeatureExtractor { + /// Conv1: 3 -> 64, kernel 3x3, stride 2 + conv1: Conv2d, + /// Fire1: 64 -> 128 (squeeze=16, expand=64+64) + fire1: FireModule, + /// Fire2: 128 -> 128 (squeeze=16, expand=64+64) + fire2: FireModule, + /// Fire3: 128 -> 256 (squeeze=32, expand=128+128) + fire3: FireModule, + /// Fire4: 256 -> 256 (squeeze=32, expand=128+128) + fire4: FireModule, + /// Fire5: 256 -> 384 (squeeze=48, expand=192+192) + fire5: FireModule, + /// Fire6: 384 -> 384 (squeeze=48, expand=192+192) + fire6: FireModule, + /// Fire7: 384 -> 512 (squeeze=64, expand=256+256) + fire7: FireModule, + /// Fire8: 512 -> 512 (squeeze=64, expand=256+256) + fire8: FireModule, +} + +impl SqueezeFeatureExtractor { + /// Create a new SqueezeNet feature extractor. + pub fn new(device: &B::Device) -> Self { + Self { + // Conv1: 3 -> 64, 3x3, stride 2 + conv1: Conv2dConfig::new([3, 64], [3, 3]) + .with_stride([2, 2]) + .with_bias(true) + .init(device), + // Fire modules (SqueezeNet 1.1 configuration) + fire1: FireModule::new(64, 16, 64, 64, device), // -> 128 + fire2: FireModule::new(128, 16, 64, 64, device), // -> 128 + fire3: FireModule::new(128, 32, 128, 128, device), // -> 256 + fire4: FireModule::new(256, 32, 128, 128, device), // -> 256 + fire5: FireModule::new(256, 48, 192, 192, device), // -> 384 + fire6: FireModule::new(384, 48, 192, 192, device), // -> 384 + fire7: FireModule::new(384, 64, 256, 256, device), // -> 512 + fire8: FireModule::new(512, 64, 256, 256, device), // -> 512 + } + } + + /// Extract features from 7 SqueezeNet layers. + pub fn forward(&self, x: Tensor) -> Vec> { + let mut features = Vec::with_capacity(7); + + // Slice 1: Conv1 + ReLU (64 channels) + let x = relu(self.conv1.forward(x)); + features.push(x.clone()); + + // Slice 2: MaxPool + Fire1 + Fire2 (128 channels) + let x = max_pool2d_squeeze(x); + let x = self.fire1.forward(x); + let x = self.fire2.forward(x); + features.push(x.clone()); + + // Slice 3: MaxPool + Fire3 + Fire4 (256 channels) + let x = max_pool2d_squeeze(x); + let x = self.fire3.forward(x); + let x = self.fire4.forward(x); + features.push(x.clone()); + + // Slice 4: MaxPool + Fire5 (384 channels) + let x = max_pool2d_squeeze(x); + let x = self.fire5.forward(x); + features.push(x.clone()); + + // Slice 5: Fire6 (384 channels) + let x = self.fire6.forward(x); + features.push(x.clone()); + + // Slice 6: Fire7 (512 channels) + let x = self.fire7.forward(x); + features.push(x.clone()); + + // Slice 7: Fire8 (512 channels) + let x = self.fire8.forward(x); + features.push(x); + + features + } +} + +/// 3x3 max pooling with stride 2, ceil mode (for SqueezeNet). +fn max_pool2d_squeeze(x: Tensor) -> Tensor { + burn_core::tensor::module::max_pool2d(x, [3, 3], [2, 2], [0, 0], [1, 1], true) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/vgg.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/vgg.rs new file mode 100644 index 0000000..464a0fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/vgg.rs @@ -0,0 +1,116 @@ +//! VGG16 feature extractor for LPIPS. + +use burn_core as burn; + +use burn::module::Module; +use burn::tensor::Tensor; +use burn::tensor::activation::relu; +use burn::tensor::backend::Backend; +use burn_nn::PaddingConfig2d; +use burn_nn::conv::{Conv2d, Conv2dConfig}; + +/// VGG16 feature extractor for LPIPS. +/// +/// Extracts features from 5 layers: +/// - conv1_2: 64 channels +/// - conv2_2: 128 channels +/// - conv3_3: 256 channels +/// - conv4_3: 512 channels +/// - conv5_3: 512 channels +#[derive(Module, Debug)] +pub struct VggFeatureExtractor { + // Block 1 + conv1_1: Conv2d, + conv1_2: Conv2d, + // Block 2 + conv2_1: Conv2d, + conv2_2: Conv2d, + // Block 3 + conv3_1: Conv2d, + conv3_2: Conv2d, + conv3_3: Conv2d, + // Block 4 + conv4_1: Conv2d, + conv4_2: Conv2d, + conv4_3: Conv2d, + // Block 5 + conv5_1: Conv2d, + conv5_2: Conv2d, + conv5_3: Conv2d, +} + +impl VggFeatureExtractor { + /// Create a new VGG16 feature extractor. + pub fn new(device: &B::Device) -> Self { + let conv_config = |in_ch, out_ch| { + Conv2dConfig::new([in_ch, out_ch], [3, 3]) + .with_padding(PaddingConfig2d::Same) + .with_bias(true) + }; + + Self { + // Block 1: 3 -> 64 + conv1_1: conv_config(3, 64).init(device), + conv1_2: conv_config(64, 64).init(device), + // Block 2: 64 -> 128 + conv2_1: conv_config(64, 128).init(device), + conv2_2: conv_config(128, 128).init(device), + // Block 3: 128 -> 256 + conv3_1: conv_config(128, 256).init(device), + conv3_2: conv_config(256, 256).init(device), + conv3_3: conv_config(256, 256).init(device), + // Block 4: 256 -> 512 + conv4_1: conv_config(256, 512).init(device), + conv4_2: conv_config(512, 512).init(device), + conv4_3: conv_config(512, 512).init(device), + // Block 5: 512 -> 512 + conv5_1: conv_config(512, 512).init(device), + conv5_2: conv_config(512, 512).init(device), + conv5_3: conv_config(512, 512).init(device), + } + } + + /// Extract features from 5 VGG layers. + pub fn forward(&self, x: Tensor) -> Vec> { + let mut features = Vec::with_capacity(5); + + // Block 1 + let x = relu(self.conv1_1.forward(x)); + let x = relu(self.conv1_2.forward(x)); + features.push(x.clone()); + let x = max_pool2d(x); + + // Block 2 + let x = relu(self.conv2_1.forward(x)); + let x = relu(self.conv2_2.forward(x)); + features.push(x.clone()); + let x = max_pool2d(x); + + // Block 3 + let x = relu(self.conv3_1.forward(x)); + let x = relu(self.conv3_2.forward(x)); + let x = relu(self.conv3_3.forward(x)); + features.push(x.clone()); + let x = max_pool2d(x); + + // Block 4 + let x = relu(self.conv4_1.forward(x)); + let x = relu(self.conv4_2.forward(x)); + let x = relu(self.conv4_3.forward(x)); + features.push(x.clone()); + let x = max_pool2d(x); + + // Block 5 + let x = relu(self.conv5_1.forward(x)); + let x = relu(self.conv5_2.forward(x)); + let x = relu(self.conv5_3.forward(x)); + features.push(x); + + features + } +} + +/// 2x2 max pooling with stride 2. +fn max_pool2d(x: Tensor) -> Tensor { + burn_core::tensor::module::max_pool2d(x, [2, 2], [2, 2], [0, 0], [1, 1], false) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/weights.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/weights.rs new file mode 100644 index 0000000..051d9fe --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/lpips/weights.rs @@ -0,0 +1,228 @@ +//! Pretrained weights loading for LPIPS. + +use burn_core as burn; + +use burn::tensor::backend::Backend; +use burn_std::network::downloader::download_file_as_bytes; +use burn_store::{ModuleSnapshot, PytorchStore}; +use std::fs::{File, create_dir_all}; +use std::io::Write; +use std::path::PathBuf; + +use super::metric::{Lpips, LpipsNet}; + +/// URLs for pretrained LPIPS linear layer weights from the official repository. +/// Reference: https://github.com/richzhang/PerceptualSimilarity +const LPIPS_VGG_URL: &str = + "https://github.com/richzhang/PerceptualSimilarity/raw/master/lpips/weights/v0.1/vgg.pth"; +const LPIPS_ALEX_URL: &str = + "https://github.com/richzhang/PerceptualSimilarity/raw/master/lpips/weights/v0.1/alex.pth"; +const LPIPS_SQUEEZE_URL: &str = + "https://github.com/richzhang/PerceptualSimilarity/raw/master/lpips/weights/v0.1/squeeze.pth"; + +/// URLs for ImageNet pretrained backbone weights from PyTorch. +const VGG16_IMAGENET_URL: &str = "https://download.pytorch.org/models/vgg16-397923af.pth"; +const ALEXNET_IMAGENET_URL: &str = "https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth"; +const SQUEEZENET_IMAGENET_URL: &str = + "https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth"; + +/// Get the download URL for LPIPS linear layer weights. +pub fn get_lpips_weights_url(net: LpipsNet) -> &'static str { + match net { + LpipsNet::Vgg => LPIPS_VGG_URL, + LpipsNet::Alex => LPIPS_ALEX_URL, + LpipsNet::Squeeze => LPIPS_SQUEEZE_URL, + } +} + +/// Get the download URL for backbone ImageNet weights. +pub fn get_backbone_weights_url(net: LpipsNet) -> &'static str { + match net { + LpipsNet::Vgg => VGG16_IMAGENET_URL, + LpipsNet::Alex => ALEXNET_IMAGENET_URL, + LpipsNet::Squeeze => SQUEEZENET_IMAGENET_URL, + } +} + +/// Get the cache directory for LPIPS weights. +fn get_cache_dir() -> PathBuf { + let cache_dir = dirs::cache_dir() + .expect("Could not get cache directory") + .join("burn-dataset") + .join("lpips"); + + if !cache_dir.exists() { + create_dir_all(&cache_dir).expect("Failed to create cache directory"); + } + + cache_dir +} + +/// Download file if not cached and return the cache path. +fn download_if_needed(url: &str, cache_path: &PathBuf, message: &str) { + if !cache_path.exists() { + let bytes = download_file_as_bytes(url, message); + let mut file = File::create(cache_path).expect("Failed to create cache file"); + file.write_all(&bytes).expect("Failed to write weights"); + } +} + +/// Download and load pretrained weights into an LPIPS module. +/// +/// This loads both: +/// 1. ImageNet pretrained backbone weights (VGG16/AlexNet/SqueezeNet) +/// 2. LPIPS trained linear layer weights +/// +/// Weights are cached in the user's cache directory to avoid re-downloading. +/// +/// # Arguments +/// +/// * `lpips` - The LPIPS module to load weights into. +/// * `net` - The network type (determines which weights to download). +/// +/// # Returns +/// +/// The LPIPS module with loaded pretrained weights. +pub fn load_pretrained_weights(mut lpips: Lpips, net: LpipsNet) -> Lpips { + let cache_dir = get_cache_dir(); + + // Step 1: Load backbone ImageNet weights + let backbone_url = get_backbone_weights_url(net); + let backbone_cache_path = cache_dir.join(format!("{:?}_backbone.pth", net).to_lowercase()); + let backbone_message = match net { + LpipsNet::Vgg => "Downloading VGG16 ImageNet weights...", + LpipsNet::Alex => "Downloading AlexNet ImageNet weights...", + LpipsNet::Squeeze => "Downloading SqueezeNet ImageNet weights...", + }; + download_if_needed(backbone_url, &backbone_cache_path, backbone_message); + + // Step 2: Load LPIPS linear layer weights + let lpips_url = get_lpips_weights_url(net); + let lpips_cache_path = cache_dir.join(format!("{:?}_lpips.pth", net).to_lowercase()); + let lpips_message = match net { + LpipsNet::Vgg => "Downloading LPIPS VGG weights...", + LpipsNet::Alex => "Downloading LPIPS AlexNet weights...", + LpipsNet::Squeeze => "Downloading LPIPS SqueezeNet weights...", + }; + download_if_needed(lpips_url, &lpips_cache_path, lpips_message); + + // Load backbone weights first + lpips = load_backbone_weights(lpips, &backbone_cache_path); + + // Then load LPIPS linear layer weights + lpips = load_lpips_weights(lpips, &lpips_cache_path); + + lpips +} + +/// Load ImageNet pretrained backbone weights. +fn load_backbone_weights(lpips: Lpips, cache_path: &PathBuf) -> Lpips { + // Load directly into the inner struct to avoid enum variant issues + match lpips { + Lpips::Vgg(mut inner) => { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + // VGG16 features.X -> extractor.convY_Z + .with_key_remapping(r"^features\.0\.", "extractor.conv1_1.") + .with_key_remapping(r"^features\.2\.", "extractor.conv1_2.") + .with_key_remapping(r"^features\.5\.", "extractor.conv2_1.") + .with_key_remapping(r"^features\.7\.", "extractor.conv2_2.") + .with_key_remapping(r"^features\.10\.", "extractor.conv3_1.") + .with_key_remapping(r"^features\.12\.", "extractor.conv3_2.") + .with_key_remapping(r"^features\.14\.", "extractor.conv3_3.") + .with_key_remapping(r"^features\.17\.", "extractor.conv4_1.") + .with_key_remapping(r"^features\.19\.", "extractor.conv4_2.") + .with_key_remapping(r"^features\.21\.", "extractor.conv4_3.") + .with_key_remapping(r"^features\.24\.", "extractor.conv5_1.") + .with_key_remapping(r"^features\.26\.", "extractor.conv5_2.") + .with_key_remapping(r"^features\.28\.", "extractor.conv5_3."); + if let Err(e) = inner.load_from(&mut store) { + log::warn!("Some VGG backbone weights could not be loaded: {:?}", e); + } + Lpips::Vgg(inner) + } + Lpips::Alex(mut inner) => { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + // AlexNet features.X -> extractor.convY + .with_key_remapping(r"^features\.0\.", "extractor.conv1.") + .with_key_remapping(r"^features\.3\.", "extractor.conv2.") + .with_key_remapping(r"^features\.6\.", "extractor.conv3.") + .with_key_remapping(r"^features\.8\.", "extractor.conv4.") + .with_key_remapping(r"^features\.10\.", "extractor.conv5."); + if let Err(e) = inner.load_from(&mut store) { + log::warn!("Some AlexNet backbone weights could not be loaded: {:?}", e); + } + Lpips::Alex(inner) + } + Lpips::Squeeze(mut inner) => { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + // SqueezeNet features.X -> extractor.* + .with_key_remapping(r"^features\.0\.", "extractor.conv1.") + .with_key_remapping(r"^features\.3\.", "extractor.fire1.") + .with_key_remapping(r"^features\.4\.", "extractor.fire2.") + .with_key_remapping(r"^features\.6\.", "extractor.fire3.") + .with_key_remapping(r"^features\.7\.", "extractor.fire4.") + .with_key_remapping(r"^features\.9\.", "extractor.fire5.") + .with_key_remapping(r"^features\.10\.", "extractor.fire6.") + .with_key_remapping(r"^features\.11\.", "extractor.fire7.") + .with_key_remapping(r"^features\.12\.", "extractor.fire8."); + if let Err(e) = inner.load_from(&mut store) { + log::warn!( + "Some SqueezeNet backbone weights could not be loaded: {:?}", + e + ); + } + Lpips::Squeeze(inner) + } + } +} + +/// Load LPIPS trained linear layer weights. +fn load_lpips_weights(lpips: Lpips, cache_path: &PathBuf) -> Lpips { + // Load directly into the inner struct to avoid enum variant issues + match lpips { + Lpips::Vgg(mut inner) => { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + .with_key_remapping(r"^lin0\.model\.1\.", "lin0.") + .with_key_remapping(r"^lin1\.model\.1\.", "lin1.") + .with_key_remapping(r"^lin2\.model\.1\.", "lin2.") + .with_key_remapping(r"^lin3\.model\.1\.", "lin3.") + .with_key_remapping(r"^lin4\.model\.1\.", "lin4."); + if let Err(e) = inner.load_from(&mut store) { + log::warn!("Some VGG LPIPS weights could not be loaded: {:?}", e); + } + Lpips::Vgg(inner) + } + Lpips::Alex(mut inner) => { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + .with_key_remapping(r"^lin0\.model\.1\.", "lin0.") + .with_key_remapping(r"^lin1\.model\.1\.", "lin1.") + .with_key_remapping(r"^lin2\.model\.1\.", "lin2.") + .with_key_remapping(r"^lin3\.model\.1\.", "lin3.") + .with_key_remapping(r"^lin4\.model\.1\.", "lin4."); + if let Err(e) = inner.load_from(&mut store) { + log::warn!("Some AlexNet LPIPS weights could not be loaded: {:?}", e); + } + Lpips::Alex(inner) + } + Lpips::Squeeze(mut inner) => { + let mut store = PytorchStore::from_file(cache_path) + .allow_partial(true) + .with_key_remapping(r"^lin0\.model\.1\.", "lin0.") + .with_key_remapping(r"^lin1\.model\.1\.", "lin1.") + .with_key_remapping(r"^lin2\.model\.1\.", "lin2.") + .with_key_remapping(r"^lin3\.model\.1\.", "lin3.") + .with_key_remapping(r"^lin4\.model\.1\.", "lin4.") + .with_key_remapping(r"^lin5\.model\.1\.", "lin5.") + .with_key_remapping(r"^lin6\.model\.1\.", "lin6."); + if let Err(e) = inner.load_from(&mut store) { + log::warn!("Some SqueezeNet LPIPS weights could not be loaded: {:?}", e); + } + Lpips::Squeeze(inner) + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/mod.rs new file mode 100644 index 0000000..76b08bb --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/mod.rs @@ -0,0 +1,11 @@ +mod dice; +mod dists; +mod lpips; +mod psnr; +mod ssim; + +pub use dice::*; +pub use dists::*; +pub use lpips::*; +pub use psnr::*; +pub use ssim::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/psnr.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/psnr.rs new file mode 100644 index 0000000..1742e20 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/psnr.rs @@ -0,0 +1,616 @@ +use crate::metric::{ + Metric, MetricAttributes, MetricMetadata, MetricName, Numeric, NumericAttributes, NumericEntry, + SerializedEntry, + state::{FormatOptions, NumericMetricState}, +}; +use burn_core::{ + prelude::{Backend, Tensor}, + tensor::ElementConversion, +}; +use core::marker::PhantomData; +use std::f64::consts::LN_10; + +/// Input type for the [PsnrMetric]. +/// +/// Both tensors must have shape `[N, C, H, W]`: +/// - `N`: Batch size +/// - `C`: Number of channels (1 for grayscale, 3 for RGB, etc.) +/// - `H`: Height +/// - `W`: Width +pub struct PsnrInput { + /// Model output (predictions/reconstructions) images with shape `[N, C, H, W]`. + outputs: Tensor, + /// Ground truth images with shape `[N, C, H, W]`. + targets: Tensor, +} + +impl PsnrInput { + /// Creates a new PsnrInput with the given outputs and targets. + /// + /// Inputs are expected to have the dimensions `[N, C, H, W]` + /// where `N` is the batch size, `C` is the number of channels, + /// `H` is the height of the image, and `W` is the width of the image. + /// + /// # Arguments + /// - `outputs`: The model output images with shape `[N, C, H, W]`. + /// - `targets`: The ground truth images with shape `[N, C, H, W]`. + /// + /// # Returns + /// A new instance of `PsnrInput`. + /// + /// # Panics + /// - If `outputs` and `targets` do not have the same shape. + pub fn new(outputs: Tensor, targets: Tensor) -> Self { + assert!( + outputs.dims() == targets.dims(), + "Shape mismatch: outputs {:?}, targets {:?}", + outputs.dims(), + targets.dims() + ); + Self { outputs, targets } + } +} + +/// Configuration for the [PsnrMetric]. +#[derive(Debug, Clone, Copy)] +pub struct PsnrMetricConfig { + /// Maximum possible pixel value. + /// - Use `1.0` for normalized images in range \[0, 1\] + /// - Use `255.0` for 8-bit images in range \[0, 255\] + pub max_pixel_val: f64, + /// Epsilon value for numerical stability when MSE is very small or zero. + /// + /// When MSE falls below this threshold, it is clamped to `epsilon`, + /// resulting in a maximum PSNR of approximately `10 * log10(max_pixel_val² / epsilon)` dB. + /// + /// Default is `1e-10`, which yields ~100 dB for perfect reconstruction with `max_pixel_val = 1.0`. + pub epsilon: f64, +} + +impl PsnrMetricConfig { + /// Creates a configuration with the specified maximum pixel value. + /// + /// # Example + /// ```ignore + /// // Normalized images [0, 1] + /// let config = PsnrMetricConfig::new(1.0); + /// + /// // 8-bit images [0, 255] + /// let config = PsnrMetricConfig::new(255.0); + /// // Also set a custom epsilon value + /// let config = PsnrMetricConfig::new(255.0).with_epsilon(1e-8); + /// ``` + pub fn new(max_pixel_val: f64) -> Self { + assert!(max_pixel_val > 0.0, "max_pixel_val must be positive"); + Self { + max_pixel_val, + epsilon: 1e-10, + } + } + + /// Sets a custom epsilon for numerical stability near zero MSE + pub fn with_epsilon(mut self, epsilon: f64) -> Self { + assert!(epsilon > 0.0, "epsilon must be positive"); + self.epsilon = epsilon; + self + } +} + +/// The peak signal-to-noise ratio (PSNR) metric for image quality assessment. +/// +/// PSNR is commonly used to measure the quality of reconstructed images +/// compared to the original. Higher values (in dB) indicate better quality. +/// +/// # Formula +/// ```text +/// PSNR = 10 * log10(MAX^2 / MSE) +/// ``` +/// where MAX is the maximum possible pixel value and MSE is the mean squared error. +/// +/// # Note +/// - PSNR is computed for each image first, and then it is averaged across all the images in the batch. +/// - For perfect reconstruction (MSE = 0), the MSE is clamped to `epsilon` to avoid division by zero, +/// yielding a maximum PSNR of `10 * log10(MAX^2 / epsilon)` dB. +#[derive(Clone)] +pub struct PsnrMetric { + name: MetricName, + /// Internal state for numeric metric aggregation. + state: NumericMetricState, + /// Marker for backend type. + _b: PhantomData, + /// Configuration for the metric. + config: PsnrMetricConfig, +} + +impl PsnrMetric { + /// Creates a new PSNR metric with the given configuration. + /// + /// # Example + /// ```ignore + /// let config = PsnrMetricConfig::new(1.0); + /// let metric = PsnrMetric::::new(config); + /// ``` + pub fn new(config: PsnrMetricConfig) -> Self { + Self { + name: MetricName::new(format!("PSNR@{}", config.max_pixel_val)), + state: NumericMetricState::default(), + config, + _b: PhantomData, + } + } + + /// Overrides the default metric name which is `PSNR@{max_pixel_val}`. + /// + /// Examples names: + /// - `PSNR@1.0` + /// - `PSNR@255.0` + /// + /// Use this method to provide a custom name. + pub fn with_name(mut self, name: &str) -> Self { + self.name = MetricName::new(name.to_string()); + self + } +} + +impl Metric for PsnrMetric { + type Input = PsnrInput; + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn update(&mut self, item: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + let dims = item.outputs.dims(); + let batch_size = dims[0]; + let outputs = item.outputs.clone(); + let targets = item.targets.clone(); + + // Compute per-image MSE by reducing over all dimensions except batch (dims 1, 2, 3) + // Resulting shape: [N, 1, 1, 1] + let diff = outputs.sub(targets); + let mse_per_image = diff.powi_scalar(2).mean_dims(&[1, 2, 3]); + // Flatten to shape: [N] + let mse_flat = mse_per_image.flatten::<1>(0, 3); + // Clamp MSE to avoid division by 0 in the expression (MAX^2 / MSE) + let mse_clamped = mse_flat.clamp_min(self.config.epsilon); + let max_squared = self.config.max_pixel_val * self.config.max_pixel_val; + + // Compute PSNR for each image and accumulate + // PSNR value in dB (using the change of base formula): + // 10 * log10(MAX^2 / MSE) = 10 * ln(MAX^2 / MSE) / ln(10) + // = ln(MAX^2 / MSE) * (10 / ln(10)) + let psnr_per_image = mse_clamped + .recip() + .mul_scalar(max_squared) + .log() + .mul_scalar(10.0 / LN_10); + let avg_psnr = psnr_per_image.mean().into_scalar().elem::(); + + self.state.update( + avg_psnr, + batch_size, + FormatOptions::new(self.name()).unit("dB").precision(2), + ) + } + + /// Clears the metric state. + fn clear(&mut self) { + self.state.reset(); + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("dB".to_string()), + higher_is_better: true, + } + .into() + } +} + +impl Numeric for PsnrMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TestBackend, metric::Numeric}; + use burn_core::tensor::TensorData; + + #[test] + fn test_psnr_perfect_reconstruction() { + // When outputs exactly match targets, PSNR should be very high + // (limited by epsilon clamping to ~100 dB with default epsilon=1e-10) + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[[[1.0_f32, 0.5], [0.25, 0.75]]]]), + &device, + ); + let targets = outputs.clone(); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + // With epsilon = 1e-10 and max=1.0: + // PSNR = 10 * log10(1.0 / 1e-10) = 100 dB + let psnr = metric.value().current(); + assert!( + psnr >= 99.0, + "PSNR for perfect reconstruction should be ~100 dB, got {} dB", + psnr + ); + } + + #[test] + fn test_psnr_constant_error() { + // Constant error of 0.1 across all pixels + // MSE = 0.01, PSNR = 10 * log10(1.0 / 0.01) = 20 dB + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[[[0.1_f32, 0.1], [0.1, 0.1]]]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[[[0.0_f32, 0.0], [0.0, 0.0]]]]), + &device, + ); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + assert!( + (psnr - 20.0).abs() < 0.01, + "Expected PSNR ~20 dB, got {} dB", + psnr + ); + } + + #[test] + fn test_psnr_varying_error() { + // Errors: 0.1, 0.2, 0.3, 0.4 → squared: 0.01, 0.04, 0.09, 0.16 + // MSE = 0.075, PSNR = 10 * log10(1.0 / 0.075) ≈ 11.249 dB + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[[[0.1_f32, 0.2], [0.3, 0.4]]]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[[[0.0_f32, 0.0], [0.0, 0.0]]]]), + &device, + ); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + let expected_psnr = 10.0 * (1.0_f64 / 0.075).log10(); + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{:.3} dB, got {} dB", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_max_pixel_255() { + // Test with 8-bit image range [0, 255] + // Error = 10 everywhere, MSE = 100 + // PSNR = 10 * log10(255^2 / 100) ≈ 28.13 dB + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[[[10.0_f32, 10.0], [10.0, 10.0]]]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[[[0.0_f32, 0.0], [0.0, 0.0]]]]), + &device, + ); + + let config = PsnrMetricConfig::new(255.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + let expected_psnr = 10.0 * (255.0_f64 * 255.0 / 100.0).log10(); + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{:.3} dB, got {} dB", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_batch_averaging() { + // Batch of 2 images with different MSEs + // Image 1: error 0.1 → MSE = 0.01 → PSNR = 20 dB + // Image 2: error 0.01 → MSE = 0.0001 → PSNR = 40 dB + // Average PSNR = 30 dB + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([ + [[[0.1_f32, 0.1], [0.1, 0.1]]], + [[[0.01_f32, 0.01], [0.01, 0.01]]], + ]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([ + [[[0.0_f32, 0.0], [0.0, 0.0]]], + [[[0.0_f32, 0.0], [0.0, 0.0]]], + ]), + &device, + ); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + let expected_psnr = 30.0; + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected average PSNR ~{} dB, got {} dB", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_multichannel() { + // Test with 3 channels (RGB-like) + // All channels have constant error 0.1 → MSE = 0.01 → PSNR = 20 dB + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[ + [[0.1_f32, 0.1], [0.1, 0.1]], + [[0.1_f32, 0.1], [0.1, 0.1]], + [[0.1_f32, 0.1], [0.1, 0.1]], + ]]), + &device, + ); + let targets = Tensor::::zeros([1, 3, 2, 2], &device); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + let expected_psnr = 20.0; + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{} dB, got {} dB", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_running_average() { + // Test running average across multiple updates + let device = Default::default(); + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + + // First update: error 0.1 → MSE = 0.01 → PSNR = 20 dB + let outputs1 = Tensor::::from_data( + TensorData::from([[[[0.1_f32, 0.1], [0.1, 0.1]]]]), + &device, + ); + let targets1 = Tensor::::zeros([1, 1, 2, 2], &device); + let input1 = PsnrInput::new(outputs1, targets1); + let _entry = metric.update(&input1, &MetricMetadata::fake()); + + let psnr1 = metric.value().current(); + let expected_psnr1 = 20.0; + assert!( + (psnr1 - expected_psnr1).abs() < 0.01, + "First update PSNR should be ~{} dB, got {} dB", + expected_psnr1, + psnr1 + ); + + // Second update: error 0.01 → MSE = 0.0001 → PSNR = 40 dB + let outputs2 = Tensor::::from_data( + TensorData::from([[[[0.01_f32, 0.01], [0.01, 0.01]]]]), + &device, + ); + let targets2 = Tensor::::zeros([1, 1, 2, 2], &device); + let input2 = PsnrInput::new(outputs2, targets2); + let _entry = metric.update(&input2, &MetricMetadata::fake()); + + // Running average: (20 + 40) / 2 = 30 dB + let running_avg_psnr = metric.running_value().current(); + let expected_running_avg_psnr = 30.0; + assert!( + (running_avg_psnr - expected_running_avg_psnr).abs() < 0.01, + "Running average should be ~{} dB, got {} dB", + expected_running_avg_psnr, + running_avg_psnr + ); + } + + #[test] + fn test_psnr_clear() { + // Error 0.1 → MSE = 0.01 → PSNR = 20 dB + let device = Default::default(); + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + + let outputs = Tensor::::from_data( + TensorData::from([[[[0.1_f32, 0.1], [0.1, 0.1]]]]), + &device, + ); + let targets = Tensor::::zeros([1, 1, 2, 2], &device); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + let expected_psnr = 20.0; + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{} dB, got {} dB", + expected_psnr, + psnr + ); + + // Clear and verify reset + metric.clear(); + let psnr = metric.running_value().current(); + assert!(psnr.is_nan(), "Expected NaN after clear, got {} dB", psnr) + } + + #[test] + fn test_psnr_custom_name() { + let config = PsnrMetricConfig::new(1.0); + let metric = PsnrMetric::::new(config).with_name("CustomPSNR"); + + assert_eq!(metric.name().to_string(), "CustomPSNR"); + } + + #[test] + fn test_psnr_custom_epsilon() { + let device = Default::default(); + // With a larger epsilon, perfect reconstruction gives lower PSNR + let config = PsnrMetricConfig::new(1.0).with_epsilon(0.01); + let mut metric = PsnrMetric::::new(config); + + let outputs = Tensor::::from_data( + TensorData::from([[[[0.5_f32, 0.5], [0.5, 0.5]]]]), + &device, + ); + let targets = outputs.clone(); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + // With epsilon = 0.01, PSNR = 10 * log10(1.0 / 0.01) = 20 dB + let psnr = metric.value().current(); + let expected_psnr = 20.0; + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{} dB with epsilon=0.01, got {}", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_negative_errors() { + // Test that negative differences (target > output) work correctly + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[[[0.0_f32, 0.0], [0.0, 0.0]]]]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([[[[0.1_f32, 0.1], [0.1, 0.1]]]]), + &device, + ); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + // Same MSE as positive errors (0.01), so PSNR = 20 dB + let psnr = metric.value().current(); + let expected_psnr = 20.0; + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{} dB, got {}", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_large_batch() { + // Test with a larger batch to verify batch dimension handling + let device = Default::default(); + let batch_size = 8; + + // All images have constant error 0.1 → MSE = 0.01 → PSNR = 20 dB + let outputs = Tensor::::full([batch_size, 3, 4, 4], 0.1, &device); + let targets = Tensor::::zeros([batch_size, 3, 4, 4], &device); + + let config = PsnrMetricConfig::new(1.0); + let mut metric = PsnrMetric::::new(config); + let input = PsnrInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let psnr = metric.value().current(); + let expected_psnr = 20.0; + assert!( + (psnr - expected_psnr).abs() < 0.01, + "Expected PSNR ~{} dB, got {}", + expected_psnr, + psnr + ); + } + + #[test] + fn test_psnr_attributes() { + let config = PsnrMetricConfig::new(1.0); + let metric = PsnrMetric::::new(config); + let attrs = metric.attributes(); + + match attrs { + MetricAttributes::Numeric(numeric_attrs) => { + assert_eq!(numeric_attrs.unit, Some("dB".to_string())); + assert!(numeric_attrs.higher_is_better); + } + _ => panic!("Expected numeric attributes"), + } + } + + #[test] + #[should_panic(expected = "Shape mismatch")] + fn test_psnr_shape_mismatch() { + let device = Default::default(); + let outputs = Tensor::::zeros([1, 1, 2, 2], &device); + let targets = Tensor::::zeros([1, 1, 3, 3], &device); + + let _ = PsnrInput::new(outputs, targets); + } + + #[test] + #[should_panic(expected = "max_pixel_val must be positive")] + fn test_psnr_negative_max_pixel_val() { + let _ = PsnrMetricConfig::new(-1.0); + } + + #[test] + #[should_panic(expected = "max_pixel_val must be positive")] + fn test_psnr_zero_max_pixel_val() { + let _ = PsnrMetricConfig::new(0.0); + } + + #[test] + #[should_panic(expected = "epsilon must be positive")] + fn test_psnr_negative_epsilon() { + let _ = PsnrMetricConfig::new(1.0).with_epsilon(-1e-10); + } + + #[test] + #[should_panic(expected = "epsilon must be positive")] + fn test_psnr_zero_epsilon() { + let _ = PsnrMetricConfig::new(1.0).with_epsilon(0.0); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/ssim.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/ssim.rs new file mode 100644 index 0000000..e053906 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/vision/ssim.rs @@ -0,0 +1,875 @@ +use crate::metric::{ + Metric, MetricAttributes, MetricMetadata, MetricName, Numeric, NumericAttributes, NumericEntry, + SerializedEntry, + state::{FormatOptions, NumericMetricState}, +}; +use burn_core::{ + prelude::{Backend, Tensor}, + tensor::{ElementConversion, module::conv2d, ops::ConvOptions}, +}; +use core::marker::PhantomData; + +/// Input type for the [SsimMetric]. +/// +/// Both tensors must have shape `[N, C, H, W]`: +/// - `N`: Batch size +/// - `C`: Number of channels (1 for grayscale, 3 for RGB, etc.) +/// - `H`: Height +/// - `W`: Width +pub struct SsimInput { + /// Model output (predictions/reconstructions) images with shape [N, C, H, W]. + outputs: Tensor, + /// Ground truth images with shape [N, C, H, W]. + targets: Tensor, +} + +impl SsimInput { + /// Creates a new SsimInput with the given outputs and targets. + /// + /// Inputs are expected to have the dimensions `[N, C, H, W]` + /// where `N` is the batch size, `C` is the number of channels, + /// `H` is the height of the image, and `W` is the width of the image. + /// + /// # Arguments + /// - `outputs`: The model output images with shape [N, C, H, W]. + /// - `targets`: The ground truth images with shape [N, C, H, W]. + /// + /// # Returns + /// A new instance of `SsimInput`. + /// + /// # Panics + /// - If `outputs` and `targets` do not have the same shape. + pub fn new(outputs: Tensor, targets: Tensor) -> Self { + assert!( + outputs.dims() == targets.dims(), + "Shape mismatch: outputs {:?}, targets {:?}", + outputs.dims(), + targets.dims() + ); + Self { outputs, targets } + } +} + +/// Configuration for the [SsimMetric]. +#[derive(Debug, Clone, Copy)] +pub struct SsimMetricConfig { + /// The range of the pixel values in images which can be computed as following: + /// `let data_range = max_pixel_val - min_pixel_val;` + /// where `max_pixel_val` is the maximum possible pixel value and `min_pixel_val` + /// is the minimum possible pixel value. + /// + /// - For normalized images in range [0, 1], it should be set to `1.0 - 0.0 = 1.0` + /// - For normalized images in range [-1, 1], it should be set to `1.0 - (-1.0) = 2.0` + /// - For 8-bit images in range [0, 255], it should be set to `255.0 - 0.0 = 255.0` + pub data_range: f64, + /// A parameter of SSIM used to stabilize the luminance comparison. + /// Default is 0.01. + pub k1: f64, + /// A parameter of SSIM used to stabilize the contrast comparison. + /// Default is 0.03. + pub k2: f64, + /// The SSIM metric involves applying convolution to the input tensors using a Gaussian kernel. + /// This is the window/kernel size of the Gaussian kernel. Default is 11. + pub window_size: usize, + /// The SSIM metric involves applying convolution to the input tensors using a Gaussian kernel. + /// This is the standard deviation of the Gaussian kernel. Default is 1.5. + pub sigma: f64, +} + +impl SsimMetricConfig { + /// Creates a configuration with the specified data range and default parameters. + /// + /// # Default parameters + /// - k1: 0.01 + /// - k2: 0.03 + /// - window_size: 11 + /// - sigma: 1.5 + /// + /// # Panics + /// - If `data_range` is not positive. + /// + /// # Example + /// ```ignore + /// // Normalized images [0, 1] + /// let config1 = SsimMetricConfig::new(1.0); + /// + /// // 8-bit images [0, 255] + /// let config2 = SsimMetricConfig::new(255.0); + /// + /// // Also set custom values for k1 and k2 + /// let config3 = SsimMetricConfig::new(1.0).with_k1_k2(0.015, 0.025); + /// + /// // Also set a custom value for window size + /// config3.with_window_size(13); + /// ``` + pub fn new(data_range: f64) -> Self { + assert!(data_range > 0.0, "data_range must be positive"); + Self { + data_range, + k1: 0.01, + k2: 0.03, + window_size: 11, + sigma: 1.5, + } + } + + /// Sets a custom value for the k1 and k2 parameters of SSIM which are + /// used for numerical stability. + /// + /// # Default values + /// - k1: 0.01 + /// - k2: 0.03 + /// + /// # Panics + /// - If `k1` or `k2` is not positive. + pub fn with_k1_k2(mut self, k1: f64, k2: f64) -> Self { + assert!(k1 > 0.0, "k1 must be positive"); + assert!(k2 > 0.0, "k2 must be positive"); + self.k1 = k1; + self.k2 = k2; + self + } + + /// Sets a custom window size for the Gaussian kernel used in SSIM. The + /// window size must be a positive odd number. + /// + /// # Default value + /// - window_size: 11 + /// + /// # Panics + /// - If `window_size` is not a positive odd number. + pub fn with_window_size(mut self, window_size: usize) -> Self { + assert!( + window_size > 0 && window_size % 2 == 1, + "window_size must be positive and an odd number" + ); + self.window_size = window_size; + self + } + + /// Sets a custom sigma (standard deviation) for the Gaussian kernel used in SSIM. + /// + /// # Default value + /// - sigma: 1.5 + /// + /// # Panics + /// - If `sigma` is not positive. + pub fn with_sigma(mut self, sigma: f64) -> Self { + assert!(sigma > 0.0, "sigma must be positive"); + self.sigma = sigma; + self + } +} + +/// The SSIM (structural similarity index measure) metric for image quality assessment. +/// +/// SSIM measures the perceived quality of images by comparing luminance, +/// contrast, and structure. Values range from -1 to 1, where 1 indicates +/// perfect structural similarity. +/// +/// # Formula +/// ```text +/// SSIM(x, y) = (2μxμy + C1)(2σxy + C2) / (μx² + μy² + C1)(σx² + σy² + C2) +/// ``` +/// +/// # Note +/// - This implementation uses separable Gaussian convolution for efficiency. Instead of a +/// single 2D convolution with a K by K kernel, it applies two 1D convolutions (horizontal +/// then vertical). This reduces the computational complexity from O(K^2) to O(2K) per pixel. +/// - SSIM is computed for each image first, and then it is averaged across all the images in the batch. +#[derive(Clone)] +pub struct SsimMetric { + name: MetricName, + /// Internal state for numeric metric aggregation. + state: NumericMetricState, + /// Marker for backend type. + _b: PhantomData, + /// Configuration for the metric. + config: SsimMetricConfig, +} + +impl SsimMetric { + /// Creates a new SSIM metric with the given configuration. + /// + /// # Note + /// The metric name format is "SSIM (dr={}, w={}, σ={})" + /// where dr is the data range, w is the window size, sigma is the + /// standard deviation. For example, the metric name might be + /// "SSIM (dr=1.0, w=11, σ=1.5)". + /// + /// # Example + /// ```ignore + /// let ssim_config = SsimMetricConfig::new(1.0); + /// let ssim_metric = SsimMetric::::new(ssim_config); + /// ``` + pub fn new(config: SsimMetricConfig) -> Self { + Self { + name: MetricName::new(format!( + "SSIM (dr={}, w={}, σ={})", + config.data_range, config.window_size, config.sigma, + )), + state: NumericMetricState::default(), + config, + _b: PhantomData, + } + } + + /// Overrides the default metric name which is "SSIM". + pub fn with_name(mut self, name: &str) -> Self { + self.name = MetricName::new(name.to_string()); + self + } + + /// Creates a 1D Gaussian kernel as a tensor. + /// + /// Returns a normalized kernel where all values sum to 1. + /// The returned kernel will be reshaped by the `gaussian_conv_separable` + /// associated function later. + fn create_1d_gaussian_kernel(&self) -> Vec { + let size = self.config.window_size; + let sigma = self.config.sigma; + let center = (size / 2) as f64; + + let mut kernel = vec![0.0f32; size]; + let mut sum = 0.0f64; + + for (i, v) in kernel.iter_mut().enumerate() { + let x = i as f64 - center; + let value = (-(x * x) / (2.0 * sigma * sigma)).exp(); + *v = value as f32; + sum += value; + } + + // Normalize so values sum to 1 + for v in kernel.iter_mut() { + *v /= sum as f32; + } + + kernel + } + + /// Applies separable convolution using two 1D Gaussian kernels. + /// + /// # Arguments + /// - `inputs`: Tensor of shape [N, C, H, W] + /// - `kernel_1d`: The 1D Gaussian kernel values + /// - `channels`: Number of channels for depthwise convolution. + fn gaussian_conv_separable( + &self, + input: Tensor, + kernel_1d: &[f32], + channels: usize, + device: &B::Device, + ) -> Tensor { + let size = self.config.window_size; + let padding = size / 2; + + // Create horizontal kernel: shape [C, 1, 1, K] + let horizontal_kernel = Tensor::::from_floats(kernel_1d, device) + .reshape([1, 1, 1, size]) // [1, 1, 1, K] + .repeat_dim(0, channels); // [C, 1, 1, K] + + let vertical_kernel = Tensor::::from_floats(kernel_1d, device) + .reshape([1, 1, size, 1]) // [1, 1, K, 1] + .repeat_dim(0, channels); // [C, 1, K, 1] + + // Apply horizontal convolution + let horizontal_conv_options = ConvOptions::new([1, 1], [0, padding], [1, 1], channels); + let input_after_horizontal_conv = + conv2d(input, horizontal_kernel, None, horizontal_conv_options); + + // Apply vertical convolution + let vertical_conv_options = ConvOptions::new([1, 1], [padding, 0], [1, 1], channels); + conv2d( + input_after_horizontal_conv, + vertical_kernel, + None, + vertical_conv_options, + ) + } +} + +impl Metric for SsimMetric { + type Input = SsimInput; + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn update(&mut self, item: &Self::Input, _metadata: &MetricMetadata) -> SerializedEntry { + let dims = item.outputs.dims(); + let batch_size = dims[0]; + let channels = dims[1]; + let device = item.outputs.device(); + + let img_height = dims[2]; + let img_width = dims[3]; + assert!( + img_height >= self.config.window_size && img_width >= self.config.window_size, + "Image dimensions (H={}, W={}) must be >= window_size ({})", + img_height, + img_width, + self.config.window_size + ); + + // Constants in SSIM formula used for numerical stability + let c1 = (self.config.k1 * self.config.data_range).powi(2); + let c2 = (self.config.k2 * self.config.data_range).powi(2); + + // Create 1D Gaussian kernel to apply separable convolutions twice (horizontally and vertically) + let kernel_1d = self.create_1d_gaussian_kernel(); + + // Compute mu_x and mu_y, their product and squares + let x = item.outputs.clone(); + let y = item.targets.clone(); + let mu_x = self.gaussian_conv_separable(x.clone(), &kernel_1d, channels, &device); + let mu_y = self.gaussian_conv_separable(y.clone(), &kernel_1d, channels, &device); + let mu_x_mu_y = mu_x.clone() * mu_y.clone(); + let square_of_mu_x = mu_x.clone() * mu_x.clone(); + let square_of_mu_y = mu_y.clone() * mu_y.clone(); + + // Compute var_x, var_y (which are the same as (sigma_x)^2 and (sigma_y)^2): + // Var(X) = E[X^2] - E[X]^2 + // var_x = mu_of_x_squared - (mu_x * mu_x) + let mu_of_x_squared = + self.gaussian_conv_separable(x.clone() * x.clone(), &kernel_1d, channels, &device); + let mu_of_y_squared = + self.gaussian_conv_separable(y.clone() * y.clone(), &kernel_1d, channels, &device); + let var_x = (mu_of_x_squared - square_of_mu_x.clone()).clamp_min(0.0); + let var_y = (mu_of_y_squared - square_of_mu_y.clone()).clamp_min(0.0); + + // Compute the sample covariance of x and y: sigma_xy + // Cov(X, Y) = E[XY] - E[X]E[Y] + // sigma_xy = mu_xy - (mu_x * mu_y) + let mu_xy = self.gaussian_conv_separable(x * y, &kernel_1d, channels, &device); + let sigma_xy = mu_xy - mu_x_mu_y.clone(); + + // Compute SSIM: + // SSIM(x, y) = (2μxμy + C1)(2σxy + C2) / (μx² + μy² + C1)(σx² + σy² + C2) + let numerator = (mu_x_mu_y.mul_scalar(2.0) + c1) * (sigma_xy.mul_scalar(2.0) + c2); + let denominator = (square_of_mu_x + square_of_mu_y + c1) * (var_x + var_y + c2); + let ssim_tensor = numerator / denominator; + + // Average SSIM across all dimensions to get a single scalar value + let ssim_per_image = ssim_tensor.mean_dims(&[1, 2, 3]); + let avg_ssim = ssim_per_image.mean().into_scalar().elem::(); + + self.state.update( + avg_ssim, + batch_size, + FormatOptions::new(self.name()).precision(4), + ) + } + + /// Clears the metric state. + fn clear(&mut self) { + self.state.reset(); + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: None, + higher_is_better: true, + } + .into() + } +} + +impl Numeric for SsimMetric { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +#[allow(clippy::manual_range_contains)] +mod tests { + use super::*; + use crate::{TestBackend, metric::Numeric}; + use burn_core::tensor::{Distribution, Shape, TensorData}; + + fn test_config() -> SsimMetricConfig { + SsimMetricConfig::new(1.0) + .with_window_size(3) + .with_sigma(1.0) + } + + #[test] + fn test_ssim_perfect_similarity() { + // When outputs exactly match targets, SSIM should be 1.0 + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[[ + [0.1_f32, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.2, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9], + ]]]), + &device, + ); + let targets = outputs.clone(); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + (ssim - 1.0).abs() < 0.001, + "SSIM for identical images should be 1.0, got {}", + ssim + ); + } + + #[test] + fn test_ssim_completely_different() { + // Constant black vs constant white + // With constant images: SSIM = (2*mu_x*mu_y + C1) / (mu_x^2 + mu_y^2 + C1) + // For x=0, y=1 with C1=(0.01)^2=0.0001: SSIM ≈ 0.0001 / (1 + 0.00001) = 0.00009999 + let device = Default::default(); + let outputs = Tensor::::zeros([1, 1, 4, 4], &device); + let targets = Tensor::::ones([1, 1, 4, 4], &device); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + ssim < 0.0001, + "SSIM for black vs white images should be very low, got {}", + ssim + ); + } + + #[test] + fn test_ssim_similar_images() { + // Small perturbation should give high SSIM + let device = Default::default(); + let outputs = Tensor::::full([1, 1, 4, 4], 0.5, &device); + let targets = Tensor::::full([1, 1, 4, 4], 0.51, &device); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + ssim > 0.99, + "SSIM for very similar images should be close to 1.0, got {}", + ssim + ); + } + + #[test] + fn test_ssim_batch_averaging() { + // Batch of 2 images: + // Image 1: identical (SSIM = 1.0) + // Image 2: black vs white (SSIM ≈ 0) + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([ + [[ + [0.5_f32, 0.5, 0.5, 0.5], + [0.5, 0.5, 0.5, 0.5], + [0.5, 0.5, 0.5, 0.5], + [0.5, 0.5, 0.5, 0.5], + ]], + [[ + [0.0_f32, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + ]], + ]), + &device, + ); + let targets = Tensor::::from_data( + TensorData::from([ + [[ + [0.5_f32, 0.5, 0.5, 0.5], + [0.5, 0.5, 0.5, 0.5], + [0.5, 0.5, 0.5, 0.5], + [0.5, 0.5, 0.5, 0.5], + ]], + [[ + [1.0_f32, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + ]], + ]), + &device, + ); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + // Average of ~1.0 and ~0.0 should be around 0.5 + assert!( + ssim > 0.49 && ssim < 0.51, + "Average SSIM should be around 0.5, got {}", + ssim + ); + } + + #[test] + fn test_ssim_multichannel() { + // Test with 3 channels (e.g., RGB) + let device = Default::default(); + let outputs = Tensor::::from_data( + TensorData::from([[ + [ + [0.5_f32, 0.6, 0.7, 0.8], + [0.4, 0.5, 0.6, 0.7], + [0.3, 0.4, 0.5, 0.6], + [0.2, 0.3, 0.4, 0.5], + ], + [ + [0.3_f32, 0.4, 0.5, 0.6], + [0.2, 0.3, 0.4, 0.5], + [0.1, 0.2, 0.3, 0.4], + [0.0, 0.1, 0.2, 0.3], + ], + [ + [0.7_f32, 0.8, 0.9, 1.0], + [0.6, 0.7, 0.8, 0.9], + [0.5, 0.6, 0.7, 0.8], + [0.4, 0.5, 0.6, 0.7], + ], + ]]), + &device, + ); + let targets = outputs.clone(); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + (ssim - 1.0).abs() < 0.001, + "SSIM for identical RGB images should be 1.0, got {}", + ssim + ); + } + + #[test] + fn test_ssim_symmetry() { + // SSIM(x, y) should equal SSIM(y, x) + // Symmetry is one of the mathematical properties of SSIM + let device = Default::default(); + let img1 = Tensor::::from_data( + TensorData::from([[[ + [0.1_f32, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.2, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9], + ]]]), + &device, + ); + let img2 = Tensor::::from_data( + TensorData::from([[[ + [0.2_f32, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9], + [0.3, 0.4, 0.5, 0.6], + [0.7, 0.8, 0.9, 1.0], + ]]]), + &device, + ); + + let config = test_config(); + + let mut metric1 = SsimMetric::::new(config); + let input1 = SsimInput::new(img1.clone(), img2.clone()); + let _entry = metric1.update(&input1, &MetricMetadata::fake()); + let ssim1 = metric1.value().current(); + + let mut metric2 = SsimMetric::::new(config); + let input2 = SsimInput::new(img2, img1); + let _entry = metric2.update(&input2, &MetricMetadata::fake()); + let ssim2 = metric2.value().current(); + + assert!( + (ssim1 - ssim2).abs() < 0.001, + "SSIM should be symmetric: SSIM(x,y)={} vs SSIM(y,x)={}", + ssim1, + ssim2 + ); + } + + #[test] + fn test_ssim_range() { + // SSIM values should be in [-1, 1] range + let device = Default::default(); + let shape = Shape::new([1, 1, 11, 11]); + let distribution = Distribution::Uniform(0.0, 1.0); + let outputs = Tensor::::random(shape.clone(), distribution, &device); + let targets = Tensor::::random(shape, distribution, &device); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + ssim >= -1.0 && ssim <= 1.0, + "SSIM should be in range [-1, 1], got {}", + ssim + ); + } + + #[test] + fn test_ssim_running_average() { + let device = Default::default(); + let mut metric = SsimMetric::::new(test_config()); + + // First update: identical images (SSIM = 1.0) + let outputs1 = Tensor::::from_data( + TensorData::from([[[ + [0.5_f32, 0.6, 0.7, 0.8], + [0.4, 0.5, 0.6, 0.7], + [0.3, 0.4, 0.5, 0.6], + [0.2, 0.3, 0.4, 0.5], + ]]]), + &device, + ); + let targets1 = outputs1.clone(); + let input1 = SsimInput::new(outputs1, targets1); + let _entry = metric.update(&input1, &MetricMetadata::fake()); + + let ssim1 = metric.value().current(); + assert!( + (ssim1 - 1.0).abs() < 0.001, + "First update SSIM should be ~1.0, got {}", + ssim1 + ); + + // Second update: very different images (SSIM close to 0) + let outputs2 = Tensor::::zeros([1, 1, 4, 4], &device); + let targets2 = Tensor::::ones([1, 1, 4, 4], &device); + let input2 = SsimInput::new(outputs2, targets2); + let _entry = metric.update(&input2, &MetricMetadata::fake()); + + // Running average should be around 0.5 + let running_avg = metric.running_value().current(); + assert!( + running_avg > 0.49 && running_avg < 0.51, + "Running average should be around 0.5, got {}", + running_avg + ); + } + + #[test] + fn test_ssim_clear() { + let device = Default::default(); + let mut metric = SsimMetric::::new(test_config()); + + let outputs = Tensor::::from_data( + TensorData::from([[[ + [0.5_f32, 0.6, 0.7, 0.8], + [0.4, 0.5, 0.6, 0.7], + [0.3, 0.4, 0.5, 0.6], + [0.2, 0.3, 0.4, 0.5], + ]]]), + &device, + ); + let targets = outputs.clone(); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + (ssim - 1.0).abs() < 0.001, + "Expected SSIM ~1.0, got {}", + ssim + ); + + // Clear and verify reset + metric.clear(); + let ssim = metric.running_value().current(); + assert!(ssim.is_nan(), "Expected NaN after clear, got {}", ssim); + } + + #[test] + fn test_ssim_custom_name() { + let config = SsimMetricConfig::new(1.0); + let metric = SsimMetric::::new(config).with_name("CustomSSIM"); + assert_eq!(metric.name().to_string(), "CustomSSIM"); + + let metric = SsimMetric::::new(test_config()); + assert_eq!(metric.name().to_string(), "SSIM (dr=1, w=3, σ=1)"); + + let config = SsimMetricConfig::new(255.0); + let metric = SsimMetric::::new(config); + assert_eq!(metric.name().to_string(), "SSIM (dr=255, w=11, σ=1.5)"); + } + + #[test] + fn test_ssim_data_range_255() { + // Test with 8-bit image range [0, 255] + let device = Default::default(); + let shape = Shape::new([1, 1, 10, 10]); + let distribution = Distribution::Uniform(0.0, 255.0); + let outputs = Tensor::::random(shape.clone(), distribution, &device); + let targets = outputs.clone(); + + let config = SsimMetricConfig::new(255.0).with_window_size(3); + let mut metric = SsimMetric::::new(config); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + (ssim - 1.0).abs() < 0.001, + "SSIM for identical 8-bit images should be 1.0, got {}", + ssim + ); + } + + #[test] + fn test_ssim_large_batch() { + let device = Default::default(); + let shape = Shape::new([20, 3, 30, 30]); + let distribution = Distribution::Uniform(0.0, 1.0); + let outputs = Tensor::::random(shape, distribution, &device); + let targets = outputs.clone(); + + let mut metric = SsimMetric::::new(test_config()); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + (ssim - 1.0).abs() < 0.001, + "SSIM for identical batch should be 1.0, got {}", + ssim + ); + } + + #[test] + fn test_ssim_default_window_size() { + // Test with default window_size=11, need images >= 11x11 + let device = Default::default(); + let shape = Shape::new([1, 1, 1080, 1920]); + let distribution = Distribution::Uniform(0.0, 1.0); + let outputs = Tensor::::random(shape, distribution, &device); + let targets = outputs.clone(); + + let config = SsimMetricConfig::new(1.0); // default window_size=11 + let mut metric = SsimMetric::::new(config); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + + let ssim = metric.value().current(); + assert!( + (ssim - 1.0).abs() < 0.001, + "SSIM with default window size should work and SSIM should be ~0.0, got {}", + ssim + ); + } + + #[test] + fn test_ssim_attributes() { + let config = SsimMetricConfig::new(1.0); + let metric = SsimMetric::::new(config); + let attrs = metric.attributes(); + + match attrs { + MetricAttributes::Numeric(numeric_attrs) => { + assert_eq!(numeric_attrs.unit, None); + assert!(numeric_attrs.higher_is_better); + } + _ => panic!("Expected numeric attributes"), + } + } + + #[test] + #[should_panic(expected = "Shape mismatch")] + fn test_ssim_shape_mismatch() { + let device = Default::default(); + let outputs = Tensor::::zeros([1, 1, 4, 4], &device); + let targets = Tensor::::zeros([1, 1, 5, 5], &device); + + let _ = SsimInput::new(outputs, targets); + } + + #[test] + #[should_panic(expected = "Image dimensions (H=4, W=4) must be >= window_size (11)")] + fn test_ssim_image_too_small() { + let device = Default::default(); + let outputs = Tensor::::zeros([1, 1, 4, 4], &device); + let targets = outputs.clone(); + + // Default window_size=11, but image is only 4x4 + let config = SsimMetricConfig::new(1.0); + let mut metric = SsimMetric::::new(config); + let input = SsimInput::new(outputs, targets); + let _entry = metric.update(&input, &MetricMetadata::fake()); + } + + #[test] + fn test_ssim_valid_k1_k2() { + let config = SsimMetricConfig::new(1.0).with_k1_k2(0.015, 0.035); + assert!( + config.k1 == 0.015 && config.k2 == 0.035, + "Expected k1=0.015 and k2=0.035, got k1={} and k2={}", + config.k1, + config.k2 + ); + } + + #[test] + #[should_panic(expected = "data_range must be positive")] + fn test_ssim_negative_data_range() { + let _ = SsimMetricConfig::new(-1.0); + } + + #[test] + #[should_panic(expected = "data_range must be positive")] + fn test_ssim_zero_data_range() { + let _ = SsimMetricConfig::new(0.0); + } + + #[test] + #[should_panic(expected = "k1 must be positive")] + fn test_ssim_negative_k1() { + let _ = SsimMetricConfig::new(1.0).with_k1_k2(-0.01, 0.03); + } + + #[test] + #[should_panic(expected = "k2 must be positive")] + fn test_ssim_negative_k2() { + let _ = SsimMetricConfig::new(1.0).with_k1_k2(0.01, -0.03); + } + + #[test] + #[should_panic(expected = "window_size must be positive and an odd number")] + fn test_ssim_even_window_size() { + let _ = SsimMetricConfig::new(1.0).with_window_size(10); + } + + #[test] + #[should_panic(expected = "window_size must be positive and an odd number")] + fn test_ssim_zero_window_size() { + let _ = SsimMetricConfig::new(1.0).with_window_size(0); + } + + #[test] + #[should_panic(expected = "sigma must be positive")] + fn test_ssim_negative_sigma() { + let _ = SsimMetricConfig::new(1.0).with_sigma(-1.5); + } + + #[test] + #[should_panic(expected = "sigma must be positive")] + fn test_ssim_zero_sigma() { + let _ = SsimMetricConfig::new(1.0).with_sigma(0.0); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/wer.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/wer.rs new file mode 100644 index 0000000..9d305db --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/metric/wer.rs @@ -0,0 +1,225 @@ +use super::cer::edit_distance; +use super::state::{FormatOptions, NumericMetricState}; +use super::{MetricMetadata, SerializedEntry}; +use crate::metric::{ + Metric, MetricAttributes, MetricName, Numeric, NumericAttributes, NumericEntry, +}; +use burn_core::tensor::backend::Backend; +use burn_core::tensor::{Int, Tensor}; +use core::marker::PhantomData; +use std::sync::Arc; + +// The edit_distance function remains the same as it calculates the Levenshtein distance +// between two sequences. The "units" within the sequences will now be treated as words. +/// The word error rate (WER) metric, similar to the CER, is defined as the edit distance (e.g. Levenshtein distance) between the predicted +/// and reference word sequences, divided by the total number of words in the reference. Here, the "units" within the sequences are words. +/// +#[derive(Clone)] +pub struct WordErrorRate { + name: MetricName, + state: NumericMetricState, + pad_token: Option, + _b: PhantomData, +} + +/// The [word error rate metric](WordErrorRate) input type. +#[derive(new)] +pub struct WerInput { + /// The predicted token sequences (as a 2-D tensor of token indices). + pub outputs: Tensor, + /// The target token sequences (as a 2-D tensor of token indices). + pub targets: Tensor, +} +impl Default for WordErrorRate { + fn default() -> Self { + Self::new() + } +} + +impl WordErrorRate { + /// Creates the metric. + pub fn new() -> Self { + Self { + name: Arc::new("WER".to_string()), + state: NumericMetricState::default(), + pad_token: None, + _b: PhantomData, + } + } + + /// Sets the pad token. + pub fn with_pad_token(mut self, index: usize) -> Self { + self.pad_token = Some(index); + self + } +} + +impl Metric for WordErrorRate { + type Input = WerInput; + + fn update(&mut self, input: &WerInput, _metadata: &MetricMetadata) -> SerializedEntry { + let outputs = input.outputs.clone(); + let targets = input.targets.clone(); + let [batch_size, seq_len] = targets.dims(); + + let outputs_data = outputs + .to_data() + .to_vec::() + .expect("Failed to convert outputs to Vec"); + let targets_data = targets + .to_data() + .to_vec::() + .expect("Failed to convert targets to Vec"); + + let pad_token = self.pad_token; + + let mut total_edit_distance = 0.0; + let mut total_target_length = 0.0; + + // Process each sequence in the batch + for i in 0..batch_size { + let start = i * seq_len; + let end = (i + 1) * seq_len; + let output_seq = &outputs_data[start..end]; + let target_seq = &targets_data[start..end]; + + // Handle padding and map elements to i32. + // These sequences now represent "words" (token IDs). + let output_seq_no_pad = match pad_token { + Some(pad) => output_seq + .iter() + .take_while(|&&x| x != pad as i64) + .map(|&x| x as i32) + .collect::>(), + None => output_seq.iter().map(|&x| x as i32).collect(), + }; + + let target_seq_no_pad = match pad_token { + Some(pad) => target_seq + .iter() + .take_while(|&&x| x != pad as i64) + .map(|&x| x as i32) + .collect::>(), + None => target_seq.iter().map(|&x| x as i32).collect(), + }; + + let ed = edit_distance(&target_seq_no_pad, &output_seq_no_pad); + total_edit_distance += ed as f64; + total_target_length += target_seq_no_pad.len() as f64; + } + + // Compute current WER value as a percentage + let value = if total_target_length > 0.0 { + 100.0 * total_edit_distance / total_target_length + } else { + 0.0 + }; + + self.state.update( + value, + batch_size, + FormatOptions::new(self.name()).unit("%").precision(2), + ) + } + + fn name(&self) -> MetricName { + self.name.clone() + } + + fn clear(&mut self) { + self.state.reset(); + } + + fn attributes(&self) -> MetricAttributes { + NumericAttributes { + unit: Some("%".to_string()), + higher_is_better: false, + } + .into() + } +} + +impl Numeric for WordErrorRate { + fn value(&self) -> NumericEntry { + self.state.current_value() + } + + fn running_value(&self) -> NumericEntry { + self.state.running_value() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TestBackend; + + /// Perfect match => WER = 0 %. + #[test] + fn test_wer_without_padding() { + let device = Default::default(); + let mut metric = WordErrorRate::::new(); + + // Batch size = 2, sequence length = 2 + let preds = Tensor::from_data([[1, 2], [3, 4]], &device); + let tgts = Tensor::from_data([[1, 2], [3, 4]], &device); + + metric.update(&WerInput::new(preds, tgts), &MetricMetadata::fake()); + + assert_eq!(0.0, metric.value().current()); + } + + /// Two word edits in four target words => 50 %. + #[test] + fn test_wer_without_padding_two_errors() { + let device = Default::default(); + let mut metric = WordErrorRate::::new(); + + // One substitution in each sequence. + // Sequence 1: target [1, 3], pred [1, 2] -> 1 error (3 vs 2) + // Sequence 2: target [3, 4], pred [3, 5] -> 1 error (4 vs 5) + let preds = Tensor::from_data([[1, 2], [3, 5]], &device); + let tgts = Tensor::from_data([[1, 3], [3, 4]], &device); + + metric.update(&WerInput::new(preds, tgts), &MetricMetadata::fake()); + + // Total errors = 2, Total target words = 4. WER = (2/4) * 100 = 50 % + assert_eq!(50.0, metric.value().current()); + } + + /// Same scenario as above, but with right-padding (token 9) ignored. + #[test] + fn test_wer_with_padding() { + let device = Default::default(); + let pad = 9_i64; + let mut metric = WordErrorRate::::new().with_pad_token(pad as usize); + + // Each row has three columns, last one is the pad token. + // Target sequences after removing pad: [1, 3] and [3, 4] (total length 4) + // Predicted sequences after removing pad: [1, 2] and [3, 5] + let preds = Tensor::from_data([[1, 2, pad], [3, 5, pad]], &device); + let tgts = Tensor::from_data([[1, 3, pad], [3, 4, pad]], &device); + + metric.update(&WerInput::new(preds, tgts), &MetricMetadata::fake()); + assert_eq!(50.0, metric.value().current()); + } + + /// `clear()` must reset the running statistics to NaN. + #[test] + fn test_clear_resets_state() { + let device = Default::default(); + let mut metric = WordErrorRate::::new(); + + let preds = Tensor::from_data([[1, 2]], &device); + let tgts = Tensor::from_data([[1, 3]], &device); // one error + + metric.update( + &WerInput::new(preds.clone(), tgts.clone()), + &MetricMetadata::fake(), + ); + assert!(metric.value().current() > 0.0); + + metric.clear(); + assert!(metric.value().current().is_nan()); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/base.rs new file mode 100644 index 0000000..ce291a3 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/base.rs @@ -0,0 +1,198 @@ +use std::sync::Arc; + +use crate::{ + LearnerSummary, + metric::{MetricDefinition, MetricEntry, NumericEntry}, +}; +use burn_core::data::dataloader::Progress; + +/// Trait for rendering metrics. +pub trait MetricsRendererTraining: Send + Sync { + /// Updates the training metric state. + /// + /// # Arguments + /// + /// * `state` - The metric state. + fn update_train(&mut self, state: MetricState); + + /// Updates the validation metric state. + /// + /// # Arguments + /// + /// * `state` - The metric state. + fn update_valid(&mut self, state: MetricState); + + /// Renders the training progress. + /// + /// # Arguments + /// + /// * `item` - The training progress. + fn render_train(&mut self, item: TrainingProgress, progress_indicators: Vec); + + /// Renders the validation progress. + /// + /// # Arguments + /// + /// * `item` - The validation progress. + fn render_valid(&mut self, item: TrainingProgress, progress_indicators: Vec); + + /// Callback method invoked when training ends, whether it + /// completed successfully or was interrupted. + /// + /// # Returns + /// + /// A result indicating whether the end-of-training actions were successful. + fn on_train_end( + &mut self, + summary: Option, + ) -> Result<(), Box> { + default_summary_action(summary); + Ok(()) + } +} + +/// A renderer that can be used for both training and evaluation. +pub trait MetricsRenderer: MetricsRendererEvaluation + MetricsRendererTraining { + /// Keep the renderer from automatically closing, requiring manual action to close it. + fn manual_close(&mut self); + /// Register a new metric. + fn register_metric(&mut self, definition: MetricDefinition); +} + +#[derive(Clone)] +/// The name of an evaluation. +/// +/// This is going to group metrics together for easier analysis. +pub struct EvaluationName { + pub(crate) name: Arc, +} + +impl EvaluationName { + /// Creates a new evaluation name. + pub fn new(s: S) -> Self { + Self { + name: Arc::new(format!("{s}")), + } + } + + /// Returns the evaluation name. + pub fn as_str(&self) -> &str { + &self.name + } +} + +impl core::fmt::Display for EvaluationName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(&self.name) + } +} + +/// Trait for rendering metrics. +pub trait MetricsRendererEvaluation: Send + Sync { + /// Updates the testing metric state. + /// + /// # Arguments + /// + /// * `state` - The metric state. + fn update_test(&mut self, name: EvaluationName, state: MetricState); + /// Renders the testing progress. + /// + /// # Arguments + /// + /// * `item` - The training progress. + fn render_test(&mut self, item: EvaluationProgress, progress_indicators: Vec); + + /// Callback method invoked when testing ends, whether it + /// completed successfully or was interrupted. + /// + /// # Returns + /// + /// A result indicating whether the end-of-testing actions were successful. + fn on_test_end( + &mut self, + summary: Option, + ) -> Result<(), Box> { + default_summary_action(summary); + Ok(()) + } +} + +/// The state of a metric. +#[derive(Debug)] +pub enum MetricState { + /// A generic metric. + Generic(MetricEntry), + /// A numeric metric. + Numeric(MetricEntry, NumericEntry), +} + +/// Training progress. +#[derive(Debug)] +pub struct TrainingProgress { + /// The progress. + pub progress: Option, + + /// The progress of the whole training. + pub global_progress: Progress, + + /// The iteration, if it differs from the items processed. + pub iteration: Option, +} + +/// Evaluation progress. +#[derive(Debug)] +pub struct EvaluationProgress { + /// The progress. + pub progress: Progress, + + /// The iteration, if it is different from the processed items. + pub iteration: Option, +} + +impl From<&EvaluationProgress> for TrainingProgress { + fn from(value: &EvaluationProgress) -> Self { + TrainingProgress { + progress: None, + global_progress: value.progress.clone(), + iteration: value.iteration, + } + } +} + +impl TrainingProgress { + /// Creates a new empty training progress. + pub fn none() -> Self { + Self { + progress: None, + global_progress: Progress { + items_processed: 0, + items_total: 0, + }, + iteration: None, + } + } +} + +/// Type of progress indicators. +pub enum ProgressType { + /// Detailed progress. + Detailed { + /// The tag. + tag: String, + /// The progress. + progress: Progress, + }, + /// Simple value. + Value { + /// The tag. + tag: String, + /// The value. + value: usize, + }, +} + +fn default_summary_action(summary: Option) { + if let Some(summary) = summary { + println!("{summary}"); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/cli.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/cli.rs new file mode 100644 index 0000000..9f92bf4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/cli.rs @@ -0,0 +1,45 @@ +use crate::renderer::{ + EvaluationProgress, MetricState, MetricsRenderer, MetricsRendererEvaluation, + MetricsRendererTraining, ProgressType, TrainingProgress, +}; + +/// A simple renderer for when the cli feature is not enabled. +pub struct CliMetricsRenderer; + +#[allow(clippy::new_without_default)] +impl CliMetricsRenderer { + /// Create a new instance. + pub fn new() -> Self { + Self {} + } +} + +impl MetricsRendererTraining for CliMetricsRenderer { + fn update_train(&mut self, _state: MetricState) {} + + fn update_valid(&mut self, _state: MetricState) {} + + fn render_train(&mut self, item: TrainingProgress, _progress_indicators: Vec) { + println!("{item:?}"); + } + + fn render_valid(&mut self, item: TrainingProgress, _progress_indicators: Vec) { + println!("{item:?}"); + } +} + +impl MetricsRendererEvaluation for CliMetricsRenderer { + fn render_test(&mut self, item: EvaluationProgress, _progress_indicators: Vec) { + println!("{item:?}"); + } + + fn update_test(&mut self, _name: super::EvaluationName, _state: MetricState) {} +} + +impl MetricsRenderer for CliMetricsRenderer { + fn manual_close(&mut self) { + // Nothing to do. + } + + fn register_metric(&mut self, _definition: crate::metric::MetricDefinition) {} +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/mod.rs new file mode 100644 index 0000000..6e441fa --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/mod.rs @@ -0,0 +1,34 @@ +#[cfg(feature = "tui")] +use std::io::IsTerminal; + +mod base; +pub use base::*; + +pub(crate) mod cli; + +pub use cli::*; + +/// The tui renderer +#[cfg(feature = "tui")] +pub mod tui; +use crate::Interrupter; + +/// Return the default metrics renderer. +/// +/// This can be either: +/// - `TuiMetricsRenderer`, when the `tui` feature is enabled and `stdout` is +/// a terminal, or +/// - `CliMetricsRenderer`, when the `tui` feature is not enabled, or `stdout` +/// is not a terminal. +#[allow(unused_variables)] +pub(crate) fn default_renderer( + interuptor: Interrupter, + checkpoint: Option, +) -> Box { + #[cfg(feature = "tui")] + if std::io::stdout().is_terminal() { + return Box::new(tui::TuiMetricsRendererWrapper::new(interuptor, checkpoint)); + } + + Box::new(CliMetricsRenderer::new()) +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/base.rs new file mode 100644 index 0000000..c59af6e --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/base.rs @@ -0,0 +1,106 @@ +use std::sync::Arc; + +use super::{ + ControlsView, NumericMetricView, ProgressBarView, StatusView, TerminalFrame, TextMetricView, +}; +use ratatui::{ + prelude::{Constraint, Direction, Layout, Rect}, + style::Color, +}; + +#[derive(new)] +pub(crate) struct MetricsView<'a> { + metric_numeric: NumericMetricView<'a>, + metric_text: TextMetricView, + progress: ProgressBarView, + controls: ControlsView, + status: StatusView, +} + +impl MetricsView<'_> { + pub(crate) fn render(self, frame: &mut TerminalFrame<'_>, size: Rect) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Min(16), Constraint::Max(4)].as_ref()) + .split(size); + let size_other = chunks[0]; + let size_progress = chunks[1]; + + let chunks = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(38), Constraint::Percentage(62)].as_ref()) + .split(size_other); + let size_other = chunks[0]; + let size_metric_numeric = chunks[1]; + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Max(5), Constraint::Min(6), Constraint::Max(6)].as_ref()) + .split(size_other); + let size_controls = chunks[0]; + let size_metric_text = chunks[1]; + let size_status = chunks[2]; + + self.metric_numeric.render(frame, size_metric_numeric); + self.metric_text.render(frame, size_metric_text); + self.controls.render(frame, size_controls); + self.progress.render(frame, size_progress); + self.status.render(frame, size_status); + } +} + +#[derive(Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum TuiSplit { + Train, + Valid, + Test, +} + +#[derive(Hash, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum TuiGroup { + Default, + Named(Arc), +} + +#[derive(new, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct TuiTag { + pub(crate) split: TuiSplit, + pub(crate) group: TuiGroup, +} + +impl core::fmt::Display for TuiTag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.group { + TuiGroup::Default => f.write_fmt(format_args!("{}", self.split)), + TuiGroup::Named(group) => f.write_fmt(format_args!("{} - {}", self.split, group)), + } + } +} +impl core::fmt::Display for TuiGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TuiGroup::Default => f.write_str(""), + TuiGroup::Named(group) => f.write_fmt(format_args!("{group} ")), + } + } +} + +impl core::fmt::Display for TuiSplit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TuiSplit::Train => f.write_str("Train"), + TuiSplit::Valid => f.write_str("Valid"), + TuiSplit::Test => f.write_str("Test"), + } + } +} + +impl TuiSplit { + pub(crate) fn color(&self) -> Color { + match self { + TuiSplit::Train => Color::LightRed, + TuiSplit::Valid => Color::LightBlue, + TuiSplit::Test => Color::LightGreen, + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/controls.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/controls.rs new file mode 100644 index 0000000..e487780 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/controls.rs @@ -0,0 +1,46 @@ +use super::TerminalFrame; +use ratatui::{ + prelude::{Alignment, Rect}, + style::{Color, Style, Stylize}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Wrap}, +}; + +/// Controls view. +pub(crate) struct ControlsView; + +impl ControlsView { + /// Render the view. + pub(crate) fn render(self, frame: &mut TerminalFrame<'_>, size: Rect) { + let lines = vec![ + vec![ + Span::from(" Quit : ").yellow().bold(), + Span::from("q ").bold(), + Span::from(" Stop the training.").italic(), + ], + vec![ + Span::from(" Plots Metrics : ").yellow().bold(), + Span::from("⬅ ➡").bold(), + Span::from(" Switch between metrics.").italic(), + ], + vec![ + Span::from(" Plots Type : ").yellow().bold(), + Span::from("⬆ ⬇").bold(), + Span::from(" Switch between types.").italic(), + ], + ]; + let paragraph = Paragraph::new(lines.into_iter().map(Line::from).collect::>()) + .alignment(Alignment::Left) + .wrap(Wrap { trim: false }) + .style(Style::default().fg(Color::Gray)) + .block( + Block::default() + .borders(Borders::ALL) + .style(Style::default().fg(Color::Gray)) + .title_alignment(Alignment::Left) + .title("Controls"), + ); + + frame.render_widget(paragraph, size); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/full_history.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/full_history.rs new file mode 100644 index 0000000..437c5ce --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/full_history.rs @@ -0,0 +1,295 @@ +use super::PlotAxes; +use crate::{ + metric::NumericEntry, + renderer::tui::{TuiSplit, TuiTag}, +}; +use ratatui::{ + style::{Color, Style}, + symbols, + widgets::{Bar, Dataset, GraphType}, +}; +use std::collections::BTreeMap; + +/// A plot that shows the full history at a reduced resolution. +pub(crate) struct FullHistoryPlot { + pub(crate) axes: PlotAxes, + points: BTreeMap, + max_samples: usize, + max_samples_ratio: BTreeMap, + next_x_state: usize, +} + +struct FullHistoryPoints { + min_x: f64, + max_x: f64, + min_y: f64, + max_y: f64, + avg_sum: f64, + avg_counter: f64, + points: Vec<(f64, f64)>, + max_samples: usize, + step_size: usize, +} + +impl FullHistoryPlot { + /// Create a new history plot. + pub(crate) fn new(max_samples: usize) -> Self { + Self { + points: BTreeMap::default(), + axes: PlotAxes::default(), + max_samples, + max_samples_ratio: BTreeMap::default(), + next_x_state: 0, + } + } + + /// Update the maximum amount of sample to display for the validation points. + /// + /// This is necessary if we want the validation line to have the same point density as the + /// training line. + pub(crate) fn update_max_sample(&mut self, split: TuiSplit, ratio: f64) { + self.max_samples_ratio.insert(split, ratio); + + self.points + .iter_mut() + .filter(|(tag, _)| tag.split == split) + .for_each(|(_, points)| { + points.max_samples = (self.max_samples as f64 * ratio) as usize; + }); + } + + /// Register a training data point. + pub(crate) fn push(&mut self, tag: TuiTag, data: NumericEntry) { + let x_current = self.next_x(); + let points = match self.points.get_mut(&tag) { + Some(val) => val, + None => { + let max_samples = self + .max_samples_ratio + .get(&tag.split) + .map(|ratio| (*ratio * self.max_samples as f64) as usize) + .unwrap_or(self.max_samples); + self.points + .insert(tag.clone(), FullHistoryPoints::new(max_samples)); + self.points.get_mut(&tag).unwrap() + } + }; + + points.push((x_current, data)); + + self.update_bounds(); + } + + pub(crate) fn datasets(&self) -> Vec> { + let mut datasets = Vec::with_capacity(2); + + for (tag, points) in self.points.iter() { + datasets.push(points.dataset(format!("{tag}"), tag.split.color())); + } + + datasets + } + + pub(crate) fn bars(&self, max: u64, bar_width: &mut usize) -> Vec> { + let mut bars = Vec::new(); + + for (tag, points) in self.points.iter() { + if let Some((bar, width)) = points.bar(tag, max) { + *bar_width = usize::max(*bar_width, width); + bars.push(bar); + } + } + + bars + } + + fn next_x(&mut self) -> f64 { + let value = self.next_x_state; + self.next_x_state += 1; + value as f64 + } + + fn update_bounds(&mut self) { + let (mut x_min, mut x_max) = (f64::MAX, f64::MIN); + let (mut y_min, mut y_max) = (f64::MAX, f64::MIN); + + for points in self.points.values() { + x_min = f64::min(x_min, points.min_x); + x_max = f64::max(x_max, points.max_x); + y_min = f64::min(y_min, points.min_y); + y_max = f64::max(y_max, points.max_y); + } + + self.axes.update_bounds((x_min, x_max), (y_min, y_max)); + } +} + +impl FullHistoryPoints { + fn new(max_samples: usize) -> Self { + Self { + min_x: 0., + max_x: 0., + min_y: f64::MAX, + max_y: f64::MIN, + avg_sum: 0.0, + avg_counter: 0.0, + points: Vec::with_capacity(max_samples), + max_samples, + step_size: 1, + } + } + + fn push(&mut self, (x, y): (f64, NumericEntry)) { + if !(x as usize).is_multiple_of(self.step_size) { + return; + } + + let y = match y { + NumericEntry::Value(val) => { + self.avg_sum += val; + self.avg_counter += 1.0; + val + } + NumericEntry::Aggregated { + aggregated_value, + count, + } => { + self.avg_sum += aggregated_value * count as f64; + self.avg_counter += count as f64; + aggregated_value + } + }; + + if x > self.max_x { + self.max_x = x; + } + if x < self.min_x { + self.min_x = x; + } + if y > self.max_y { + self.max_y = y; + } + if y < self.min_y { + self.min_y = y + } + + self.points.push((x, y)); + + if self.points.len() > self.max_samples { + self.resize(); + } + } + + /// We keep only half the points and we double the step size. + /// + /// This ensure that we have the same amount of points across the X axis. + fn resize(&mut self) { + let mut points = Vec::with_capacity(self.max_samples / 2); + let mut max_x = f64::MIN; + let mut max_y = f64::MIN; + let mut min_x = f64::MAX; + let mut min_y = f64::MAX; + + for (i, (x, y)) in self.points.drain(0..self.points.len()).enumerate() { + if i % 2 == 0 { + if x > max_x { + max_x = x; + } + if x < min_x { + min_x = x; + } + if y > max_y { + max_y = y; + } + if y < min_y { + min_y = y; + } + + points.push((x, y)); + } + } + + self.points = points; + self.step_size *= 2; + + self.min_x = min_x; + self.max_x = max_x; + self.min_y = min_y; + self.max_y = max_y; + } + + fn dataset<'a>(&'a self, name: String, color: Color) -> Dataset<'a> { + Dataset::default() + .name(name) + .marker(symbols::Marker::Braille) + .style(Style::default().fg(color).bold()) + .graph_type(GraphType::Line) + .data(&self.points) + } + + fn bar<'a>(&'a self, tag: &TuiTag, max: u64) -> Option<(Bar<'a>, usize)> { + if self.avg_sum == 0.0 { + return None; + } + + let label = format!("{tag}"); + let width = usize::max(label.len(), 7); // 7 min width + + let factor = max as f64; + + let avg = self.avg_sum / self.avg_counter; + + Some(( + Bar::default() + .value((avg * factor) as u64) + .style(tag.split.color()) + .text_value(format!("{:.2}", avg)) + .label(label), + width, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::renderer::tui::{TuiGroup, TuiSplit}; + + #[test] + fn test_points() { + let mut chart = FullHistoryPlot::new(10); + let tag_train = TuiTag::new(TuiSplit::Train, TuiGroup::Default); + let tag_valid = TuiTag::new(TuiSplit::Valid, TuiGroup::Default); + chart.update_max_sample(tag_valid.split, 0.6); + + for i in 0..100 { + chart.push(tag_train.clone(), NumericEntry::Value(i as f64)); + } + for i in 0..60 { + chart.push(tag_valid.clone(), NumericEntry::Value(i as f64)); + } + + let expected_train = vec![ + (0.0, 0.0), + (16.0, 16.0), + (32.0, 32.0), + (48.0, 48.0), + (64.0, 64.0), + (80.0, 80.0), + (96.0, 96.0), + ]; + + let expected_valid = vec![(100.0, 0.0), (116.0, 16.0), (128.0, 28.0), (144.0, 44.0)]; + + assert_eq!( + chart.points.get(&tag_train).unwrap().points, + expected_train, + "Expected train data points" + ); + assert_eq!( + chart.points.get(&tag_valid).unwrap().points, + expected_valid, + "Expected valid data points" + ); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_numeric.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_numeric.rs new file mode 100644 index 0000000..f88b0ad --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_numeric.rs @@ -0,0 +1,326 @@ +use crate::{ + metric::{MetricName, NumericEntry}, + renderer::{EvaluationProgress, TrainingProgress, tui::TuiTag}, +}; + +use super::{FullHistoryPlot, RecentHistoryPlot, TerminalFrame, TuiSplit}; +use ratatui::{ + crossterm::event::{Event, KeyCode, KeyEventKind}, + prelude::{Alignment, Constraint, Direction, Layout, Rect}, + style::{Color, Modifier, Style, Stylize}, + text::Line, + widgets::{ + Axis, BarChart, BarGroup, Block, Borders, Chart, LegendPosition, Padding, Paragraph, Tabs, + }, +}; +use std::collections::BTreeMap; + +/// 1000 seems to be required to see some improvement. +const MAX_NUM_SAMPLES_RECENT: usize = 1000; +/// 250 seems to be the right resolution when plotting all history. +/// Otherwise, there is too much points and the lines arent't smooth enough. +const MAX_NUM_SAMPLES_FULL: usize = 250; + +/// Numeric metrics state that handles creating plots. +#[derive(Default)] +pub(crate) struct NumericMetricsState { + data: BTreeMap, + names: Vec, + selected: usize, + kind: PlotKind, + num_samples_train: Option, + num_samples_valid: Option, + num_samples_test: Option, + epoch: usize, +} + +/// The kind of plot to display. +#[derive(Default, Clone, Copy)] +pub(crate) enum PlotKind { + /// Display the full history of the metric with reduced resolution. + #[default] + Full, + /// Display only the recent history of the metric, but with more resolution. + Recent, + Summary, +} + +impl NumericMetricsState { + /// Register a new training value for the metric with the given name. + pub(crate) fn push(&mut self, tag: TuiTag, name: MetricName, data: NumericEntry) { + if let Some((recent, full)) = self.data.get_mut(name.as_ref()) { + recent.push(tag.clone(), data.current()); + full.push(tag, data); + } else { + let mut recent = RecentHistoryPlot::new(MAX_NUM_SAMPLES_RECENT); + let mut full = FullHistoryPlot::new(MAX_NUM_SAMPLES_FULL); + + recent.push(tag.clone(), data.current()); + full.push(tag, data); + + self.names.push(name.clone()); + self.data.insert(name, (recent, full)); + } + } + + /// Update the state with the training progress. + pub(crate) fn update_progress_train(&mut self, progress: &TrainingProgress) { + self.epoch = progress.global_progress.items_processed; + + if self.num_samples_train.is_some() { + return; + } + + // If the training only has the notion of global progress, num_samples_train remains None. + self.num_samples_train = progress.progress.as_ref().map(|p| p.items_total); + } + + /// Update the state with the validation progress. + pub(crate) fn update_progress_valid(&mut self, progress: &TrainingProgress) { + if self.num_samples_valid.is_some() { + return; + } + + // If num_samples_train is None, keep the default max_samples for validation. + if let Some(num_sample_train) = self.num_samples_train { + for (_, (_recent, full)) in self.data.iter_mut() { + let ratio = match &progress.progress { + Some(p) => p.items_total as f64 / num_sample_train as f64, + None => progress.global_progress.items_total as f64 / num_sample_train as f64, + }; + + full.update_max_sample(TuiSplit::Valid, ratio); + } + } + + self.epoch = progress.global_progress.items_processed; + self.num_samples_valid = progress.progress.as_ref().map(|p| p.items_total); + } + + /// Update the state with the testing progress. + pub(crate) fn update_progress_test(&mut self, progress: &EvaluationProgress) { + if self.num_samples_test.is_some() { + return; + } + + if let Some(num_sample_train) = self.num_samples_train { + for (_, (_recent, full)) in self.data.iter_mut() { + let ratio = progress.progress.items_total as f64 / num_sample_train as f64; + full.update_max_sample(TuiSplit::Test, ratio); + } + } + + self.num_samples_test = Some(progress.progress.items_total); + } + + /// Create a view to display the numeric metrics. + pub(crate) fn view(&self) -> NumericMetricView<'_> { + match self.names.is_empty() { + true => NumericMetricView::None, + false => match self.kind { + PlotKind::Summary => { + NumericMetricView::BarPlots(&self.names, self.selected, self.bar_chart()) + } + _ => NumericMetricView::LinePlots( + &self.names, + self.selected, + self.line_chart(), + self.kind, + ), + }, + } + } + + /// Handle the current event. + pub(crate) fn on_event(&mut self, event: &Event) { + if let Event::Key(key) = event { + match key.kind { + KeyEventKind::Release | KeyEventKind::Repeat => (), + #[cfg(target_os = "windows")] // Fix the double toggle on Windows. + KeyEventKind::Press => return, + #[cfg(not(target_os = "windows"))] + KeyEventKind::Press => (), + } + match key.code { + KeyCode::Right => self.next_metric(), + KeyCode::Left => self.previous_metric(), + KeyCode::Up => self.switch_kind(), + KeyCode::Down => self.switch_kind(), + _ => {} + } + } + } + + fn switch_kind(&mut self) { + self.kind = match self.kind { + PlotKind::Full => PlotKind::Recent, + PlotKind::Recent => PlotKind::Summary, + PlotKind::Summary => PlotKind::Full, + }; + } + + fn next_metric(&mut self) { + self.selected = (self.selected + 1) % { + let this = &self; + this.data.len() + }; + } + + fn previous_metric(&mut self) { + if self.selected > 0 { + self.selected -= 1; + } else { + self.selected = ({ + let this = &self; + this.data.len() + }) - 1; + } + } + + fn line_chart<'a>(&'a self) -> Chart<'a> { + let name = self.names.get(self.selected).unwrap(); + let (recent, full) = self.data.get(name).unwrap(); + + let (datasets, axes) = match self.kind { + PlotKind::Full => (full.datasets(), &full.axes), + PlotKind::Recent => (recent.datasets(), &recent.axes), + _ => unreachable!(), + }; + + Chart::<'a>::new(datasets) + .block(Block::default()) + .x_axis( + Axis::default() + .style(Style::default().fg(Color::DarkGray)) + .title("Iteration") + .labels(axes.labels_x.clone().into_iter().map(|s| s.bold())) + .bounds(axes.bounds_x), + ) + .y_axis( + Axis::default() + .style(Style::default().fg(Color::DarkGray)) + .labels(axes.labels_y.clone().into_iter().map(|s| s.bold())) + .bounds(axes.bounds_y), + ) + .legend_position(Some(LegendPosition::Right)) + } + + fn bar_chart<'a>(&'a self) -> BarChart<'a> { + let name = self.names.get(self.selected).unwrap(); + let (_recent, full) = self.data.get(name).unwrap(); + let mut bar_width = 0; + let bars = full.bars(100, &mut bar_width); + + let data = BarGroup::default().bars(&bars); + BarChart::default() + .block(Block::default().padding(Padding::new(2, 2, 2, 0))) + .bar_width(bar_width as u16) + .bar_gap(2) + .data(data) + } +} + +#[allow(clippy::large_enum_variant)] +#[derive(new)] +pub(crate) enum NumericMetricView<'a> { + LinePlots(&'a [MetricName], usize, Chart<'a>, PlotKind), + BarPlots(&'a [MetricName], usize, BarChart<'a>), + None, +} + +impl NumericMetricView<'_> { + pub(crate) fn render(self, frame: &mut TerminalFrame<'_>, size: Rect) { + match self { + Self::LinePlots(titles, selected, chart, kind) => { + let block = Block::default() + .borders(Borders::ALL) + .title("Plots") + .title_alignment(Alignment::Left); + let size_new = block.inner(size); + frame.render_widget(block, size); + + let size = size_new; + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints( + [ + Constraint::Length(2), + Constraint::Length(1), + Constraint::Min(0), + ] + .as_ref(), + ) + .split(size); + + let tabs = Tabs::new( + titles + .iter() + .map(|i| Line::from(vec![i.to_string().yellow()])), + ) + .select(selected) + .style(Style::default()) + .highlight_style( + Style::default() + .add_modifier(Modifier::BOLD) + .add_modifier(Modifier::UNDERLINED) + .fg(Color::LightYellow), + ); + let title = match kind { + PlotKind::Full => "Full History", + PlotKind::Recent => "Recent History", + _ => unreachable!(), + }; + + let plot_type = + Paragraph::new(Line::from(title.bold())).alignment(Alignment::Center); + + frame.render_widget(tabs, chunks[0]); + frame.render_widget(plot_type, chunks[1]); + frame.render_widget(chart, chunks[2]); + } + Self::BarPlots(titles, selected, chart) => { + let block = Block::default() + .borders(Borders::ALL) + .title("Summary") + .title_alignment(Alignment::Left); + let size_new = block.inner(size); + frame.render_widget(block, size); + + let size = size_new; + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(2), + Constraint::Length(1), + Constraint::Min(0), + ]) + .split(size); + + let tabs = Tabs::new( + titles + .iter() + .map(|i| Line::from(vec![i.to_string().yellow()])), + ) + .select(selected) + .style(Style::default()) + .highlight_style( + Style::default() + .add_modifier(Modifier::BOLD) + .add_modifier(Modifier::UNDERLINED) + .fg(Color::LightYellow), + ); + let title = "Summary"; + + let plot_type = + Paragraph::new(Line::from(title.bold())).alignment(Alignment::Center); + + frame.render_widget(tabs, chunks[0]); + frame.render_widget(plot_type, chunks[1]); + frame.render_widget(chart, chunks[2]); + } + Self::None => {} + }; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_text.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_text.rs new file mode 100644 index 0000000..e938f47 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/metric_text.rs @@ -0,0 +1,124 @@ +use super::TerminalFrame; +use crate::{ + metric::{MetricEntry, MetricName}, + renderer::tui::{TuiGroup, TuiSplit}, +}; +use ratatui::{ + prelude::{Alignment, Rect}, + style::{Color, Style, Stylize}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Wrap}, +}; +use std::{collections::BTreeMap, sync::Arc}; + +#[derive(Default)] +pub(crate) struct TextMetricsState { + data: BTreeMap, + names: Vec, +} + +struct MetricGroup { + groups: BTreeMap, +} + +impl MetricGroup { + fn new(group: TuiGroup, metric: MetricSplits) -> Self { + Self { + groups: BTreeMap::from_iter(Some((group, metric))), + } + } + fn update(&mut self, split: TuiSplit, group: TuiGroup, metric: MetricEntry) { + match self.groups.get_mut(&group) { + Some(value) => value.update(split, metric), + None => { + let value = MetricSplits::new(split, metric); + + self.groups.insert(group, value); + } + } + } +} + +struct MetricSplits { + splits: BTreeMap, +} + +impl MetricSplits { + fn new(split: TuiSplit, metric: MetricEntry) -> Self { + Self { + splits: BTreeMap::from_iter(Some((split, metric))), + } + } + + fn update(&mut self, split: TuiSplit, metric: MetricEntry) { + self.splits.insert(split, metric); + } +} + +impl TextMetricsState { + pub(crate) fn update( + &mut self, + split: TuiSplit, + group: TuiGroup, + metric: MetricEntry, + name: Arc, + ) { + if let Some(existing) = self.data.get_mut(name.as_ref()) { + existing.update(split, group, metric); + } else { + let key = name.clone(); + let value = MetricSplits::new(split, metric); + + self.names.push(key.clone()); + self.data + .insert(key.to_string(), MetricGroup::new(group, value)); + } + } + pub(crate) fn view(&self) -> TextMetricView { + TextMetricView::new(&self.names, &self.data) + } +} + +pub(crate) struct TextMetricView { + lines: Vec>>, +} + +impl TextMetricView { + fn new(names: &[MetricName], data: &BTreeMap) -> Self { + let mut lines = Vec::with_capacity(names.len() * 4); + + let start_line = |title: &str| vec![Span::from(format!(" {title} ")).bold().yellow()]; + let format_line = |group: &TuiGroup, split: &TuiSplit, formatted: &str| { + vec![ + Span::from(format!(" {group}{split} ")).bold(), + Span::from(formatted.to_string()).italic(), + ] + }; + + for name in names { + lines.push(start_line(name)); + + let entry = data.get(name.as_ref()).unwrap(); + + for (name, group) in entry.groups.iter() { + for (split, entry) in group.splits.iter() { + lines.push(format_line(name, split, &entry.serialized_entry.formatted)); + } + } + + lines.push(vec![Span::from("")]); + } + + Self { lines } + } + + pub(crate) fn render(self, frame: &mut TerminalFrame<'_>, size: Rect) { + let paragraph = Paragraph::new(self.lines.into_iter().map(Line::from).collect::>()) + .alignment(Alignment::Left) + .wrap(Wrap { trim: false }) + .block(Block::default().borders(Borders::ALL).title("Metrics")) + .style(Style::default().fg(Color::Gray)); + + frame.render_widget(paragraph, size); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/mod.rs new file mode 100644 index 0000000..b69d0c5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/mod.rs @@ -0,0 +1,23 @@ +mod base; +mod controls; +mod full_history; +mod metric_numeric; +mod metric_text; +mod plot_utils; +mod popup; +mod progress; +mod recent_history; +mod renderer; +mod status; + +pub(crate) use base::*; +pub(crate) use controls::*; +pub(crate) use full_history::*; +pub(crate) use metric_numeric::*; +pub(crate) use metric_text::*; +pub(crate) use plot_utils::*; +pub(crate) use popup::*; +pub(crate) use progress::*; +pub(crate) use recent_history::*; +pub use renderer::*; +pub(crate) use status::*; diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/plot_utils.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/plot_utils.rs new file mode 100644 index 0000000..7a8a6d4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/plot_utils.rs @@ -0,0 +1,37 @@ +use crate::metric::format_float; + +const AXIS_TITLE_PRECISION: usize = 2; + +/// The data describing both X and Y axes. +pub(crate) struct PlotAxes { + pub(crate) labels_x: Vec, + pub(crate) labels_y: Vec, + pub(crate) bounds_x: [f64; 2], + pub(crate) bounds_y: [f64; 2], +} + +impl Default for PlotAxes { + fn default() -> Self { + Self { + bounds_x: [f64::MAX, f64::MIN], + bounds_y: [f64::MAX, f64::MIN], + labels_x: Vec::new(), + labels_y: Vec::new(), + } + } +} + +impl PlotAxes { + /// Update the bounds based on the min max of each X and Y axes with both train and valid data. + pub(crate) fn update_bounds(&mut self, (x_min, x_max): (f64, f64), (y_min, y_max): (f64, f64)) { + self.bounds_x = [x_min, x_max]; + self.bounds_y = [y_min, y_max]; + + // We know x are integers. + self.labels_x = vec![format!("{x_min}"), format!("{x_max}")]; + self.labels_y = vec![ + format_float(y_min, AXIS_TITLE_PRECISION), + format_float(y_max, AXIS_TITLE_PRECISION), + ]; + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/popup.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/popup.rs new file mode 100644 index 0000000..de0902c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/popup.rs @@ -0,0 +1,144 @@ +use ratatui::{ + crossterm::event::{Event, KeyCode}, + prelude::{Alignment, Constraint, Direction, Layout, Rect}, + style::{Color, Modifier, Style, Stylize}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Wrap}, +}; + +use super::TerminalFrame; + +/// Popup callback function. +pub(crate) trait CallbackFn: Send + Sync { + /// Call the function and return if the popup state should be reset. + fn call(&self) -> bool; +} + +/// Popup callback. +pub(crate) struct Callback { + title: String, + description: String, + trigger: char, + callback: Box, +} + +impl Callback { + /// Create a new popup. + pub(crate) fn new(title: T, description: D, trigger: char, callback: C) -> Self + where + T: Into, + D: Into, + C: CallbackFn + 'static, + { + Self { + title: title.into(), + description: description.into(), + trigger, + callback: Box::new(callback), + } + } +} + +/// Popup state. +pub(crate) enum PopupState { + Empty, + Full(String, Vec), +} + +impl PopupState { + /// If the popup is empty. + pub(crate) fn is_empty(&self) -> bool { + matches!(&self, PopupState::Empty) + } + /// Handle popup events. + pub(crate) fn on_event(&mut self, event: &Event) { + let mut reset = false; + + match self { + PopupState::Empty => {} + PopupState::Full(_, callbacks) => { + for callback in callbacks.iter() { + if let Event::Key(key) = event + && let KeyCode::Char(key) = &key.code + && &callback.trigger == key + && callback.callback.call() + { + reset = true; + } + } + } + }; + + if reset { + *self = Self::Empty; + } + } + /// Create the popup view. + pub(crate) fn view(&self) -> Option> { + match self { + PopupState::Empty => None, + PopupState::Full(title, callbacks) => Some(PopupView::new(title, callbacks)), + } + } +} + +#[derive(new)] +pub(crate) struct PopupView<'a> { + title: &'a String, + callbacks: &'a [Callback], +} + +impl<'a> PopupView<'a> { + /// Render the view. + pub(crate) fn render<'b>(&'a self, frame: &mut TerminalFrame<'b>, size: Rect) { + let lines = self + .callbacks + .iter() + .flat_map(|callback| { + vec![ + Line::from(vec![ + Span::from(format!("[{}] ", callback.trigger)).bold(), + Span::from(format!("{} ", callback.title)).yellow().bold(), + ]), + Line::from(Span::from("")), + Line::from(Span::from(callback.description.to_string()).italic()), + Line::from(Span::from("")), + ] + }) + .collect::>(); + + let paragraph = Paragraph::new(lines) + .alignment(Alignment::Left) + .wrap(Wrap { trim: false }) + .style(Style::default().fg(Color::Gray)) + .block( + Block::default() + .borders(Borders::ALL) + .title_alignment(Alignment::Center) + .style(Style::default().fg(Color::Gray)) + .title(Span::styled( + self.title, + Style::default().add_modifier(Modifier::BOLD), + )), + ); + + let area = centered_percent(20, size, Direction::Horizontal); + let area = centered_percent(20, area, Direction::Vertical); + + frame.render_widget(paragraph, area); + } +} + +/// The percent represents the amount of space that will be taken by each side. +fn centered_percent(percent: u16, size: Rect, direction: Direction) -> Rect { + let center = 100 - (percent * 2); + + Layout::default() + .direction(direction) + .constraints([ + Constraint::Percentage(percent), + Constraint::Percentage(center), + Constraint::Percentage(percent), + ]) + .split(size)[1] +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/progress.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/progress.rs new file mode 100644 index 0000000..b596bdc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/progress.rs @@ -0,0 +1,328 @@ +use super::TerminalFrame; +use crate::renderer::{EvaluationProgress, TrainingProgress, tui::TuiSplit}; +use ratatui::{ + prelude::{Alignment, Constraint, Direction, Layout, Rect}, + style::{Color, Style, Stylize}, + text::{Line, Span}, + widgets::{Block, Borders, Gauge, Paragraph}, +}; +use std::time::{Duration, Instant}; + +/// Simple progress bar for the training. +/// +/// We currently ignore the time taken for the validation part. +pub(crate) struct ProgressBarState { + progress_total: f64, // Progress for total execution. + progress_task: f64, // Progress for current task. + split: TuiSplit, + starting_epoch: usize, + estimate: ProgressEstimate, +} + +const MINUTE: u64 = 60; +const HOUR: u64 = 60 * 60; +const DAY: u64 = 24 * 60 * 60; + +impl ProgressBarState { + pub fn new(checkpoint: Option) -> Self { + Self { + progress_total: 0.0, + progress_task: 0.0, + split: TuiSplit::Train, + estimate: ProgressEstimate::new(), + starting_epoch: checkpoint.unwrap_or(0), + } + } + /// Update the training progress. + pub(crate) fn update_train(&mut self, progress: &TrainingProgress) { + self.progress_total = calculate_progress(progress, 0, 0); + let local_progress = progress + .progress + .as_ref() + .unwrap_or(&progress.global_progress); + self.progress_task = + local_progress.items_processed as f64 / local_progress.items_total as f64; + self.estimate.update(progress, self.starting_epoch); + self.split = TuiSplit::Train; + } + + /// Update the validation progress. + pub(crate) fn update_valid(&mut self, progress: &TrainingProgress) { + // We don't use the validation for the total progress yet. + let local_progress = progress + .progress + .as_ref() + .unwrap_or(&progress.global_progress); + self.progress_task = + local_progress.items_processed as f64 / local_progress.items_total as f64; + self.split = TuiSplit::Valid; + } + + /// Update the testing progress. + pub(crate) fn update_test(&mut self, progress: &EvaluationProgress) { + // We don't use the testing for the total progress yet. + self.progress_task = + progress.progress.items_processed as f64 / progress.progress.items_total as f64; + self.split = TuiSplit::Test; + } + + /// Create a view for the current progress. + pub(crate) fn view(&self) -> ProgressBarView { + const NO_ETA: &str = "---"; + + let eta = match self.estimate.secs() { + Some(eta) => format_eta(eta), + None => NO_ETA.to_string(), + }; + ProgressBarView::new( + self.progress_total, + self.progress_task, + self.split.color(), + eta, + ) + } +} + +#[derive(new)] +pub(crate) struct ProgressBarView { + progress: f64, + progress_task: f64, + color_task: Color, + eta: String, +} + +impl ProgressBarView { + /// Render the view. + pub(crate) fn render(self, frame: &mut TerminalFrame<'_>, size: Rect) { + let block = Block::default() + .borders(Borders::ALL) + .title("Progress") + .title_alignment(Alignment::Left); + let size_new = block.inner(size); + frame.render_widget(block, size); + let size = size_new; + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Ratio(1, 2), Constraint::Ratio(1, 2)].as_ref()) + .split(size); + + let size_task = chunks[0]; + let size_total = chunks[1]; + + let calculate_size = |size: Rect| { + Layout::default() + .direction(Direction::Horizontal) + .constraints( + [ + Constraint::Length(1), // Empty space + Constraint::Min(0), + Constraint::Length(self.eta.len() as u16 + 4), + ] + .as_ref(), + ) + .split(size) + }; + + let chunks = calculate_size(size_total); + let size_gauge_total = chunks[1]; + let size_eta = chunks[2]; + let chunks = calculate_size(size_task); + let size_gauge_task = chunks[1]; + + let progress_total = Gauge::default() + .gauge_style(Style::default().fg(Color::Yellow)) + .ratio(self.progress.min(1.0)); + let progress_task = Gauge::default() + .gauge_style(Style::default().fg(self.color_task)) + .ratio(self.progress_task.min(1.0)); + + let eta = Paragraph::new(Line::from(vec![ + Span::from(" ("), + Span::from(self.eta).italic(), + Span::from(") "), + ])); + + frame.render_widget(progress_task, size_gauge_task); + frame.render_widget(progress_total, size_gauge_total); + frame.render_widget(eta, size_eta); + } +} + +struct ProgressEstimate { + started: Instant, + started_after_warmup: Option, + warmup_num_items: usize, + progress: f64, +} + +impl ProgressEstimate { + fn new() -> Self { + Self { + started: Instant::now(), + started_after_warmup: None, + warmup_num_items: 0, + progress: 0.0, + } + } + + fn secs(&self) -> Option { + let eta = self.started_after_warmup?.elapsed(); + + let total_estimated = (eta.as_secs() as f64) / self.progress; + + if total_estimated.is_normal() { + let remaining = 1.0 - self.progress; + let eta = (total_estimated * remaining) as u64; + Some(eta) + } else { + None + } + } + + fn update(&mut self, progress: &TrainingProgress, starting_epoch: usize) { + if self.started_after_warmup.is_some() { + self.progress = calculate_progress(progress, starting_epoch, self.warmup_num_items); + return; + } + + const WARMUP_NUM_ITERATION: usize = 10; + + // When the training has started since 30 seconds. + if self.started.elapsed() > Duration::from_secs(30) { + self.init(progress, starting_epoch); + return; + } + + // When the training has started since at least 10 seconds and completed 10 iterations. + if progress.iteration >= Some(WARMUP_NUM_ITERATION) + && self.started.elapsed() > Duration::from_secs(10) + { + self.init(progress, starting_epoch); + } + } + + fn init(&mut self, progress: &TrainingProgress, starting_epoch: usize) { + let epoch = progress.global_progress.items_processed - starting_epoch; + + self.warmup_num_items = match &progress.progress { + Some(local_progress) => { + let epoch_items = (epoch - 1) * local_progress.items_total; + let iteration_items = local_progress.items_processed; + epoch_items + iteration_items + } + None => epoch, + }; + + self.started_after_warmup = Some(Instant::now()); + self.progress = calculate_progress(progress, starting_epoch, self.warmup_num_items); + } +} + +fn calculate_progress( + progress: &TrainingProgress, + starting_epoch: usize, + ignore_num_items: usize, +) -> f64 { + let epoch_total = progress.global_progress.items_total - starting_epoch; + let epoch = progress.global_progress.items_processed - starting_epoch; + match &progress.progress { + Some(local_progress) => { + let total_items = local_progress.items_total * epoch_total; + let epoch_items = (epoch - 1) * local_progress.items_total; + let iteration_items = local_progress.items_processed; + let num_items = epoch_items + iteration_items - ignore_num_items; + + num_items as f64 / total_items as f64 + } + None => epoch as f64 / epoch_total as f64, + } +} + +fn format_eta(eta_secs: u64) -> String { + let seconds = eta_secs % 60; + let minutes = eta_secs / MINUTE % 60; + let hours = eta_secs / HOUR % 24; + let days = eta_secs / DAY; + + if days > 1 { + format!("{days} days") + } else if days == 1 { + "1 day".to_string() + } else if hours > 1 { + format!("{hours} hours") + } else if hours == 1 { + "1 hour".to_string() + } else if minutes > 1 { + format!("{minutes} mins") + } else if minutes == 1 { + "1 min".to_string() + } else if seconds > 1 { + format!("{seconds} secs") + } else { + "1 sec".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use burn_core::data::dataloader::Progress; + + #[test] + fn test_format_eta() { + assert_eq!("55 secs", format_eta(55), "Less than 1 minutes"); + assert_eq!("1 min", format_eta(61), "More than 1 minutes"); + assert_eq!("2 mins", format_eta(2 * 61), "More than 2 minutes"); + assert_eq!("1 hour", format_eta(3601), "More than 1 hour"); + assert_eq!("2 hours", format_eta(2 * 3601), "More than 2 hour"); + assert_eq!("1 day", format_eta(24 * 3601), "More than 1 day"); + assert_eq!("2 days", format_eta(48 * 3601), "More than 2 day"); + } + + #[test] + fn calculate_progress_for_eta() { + let half = Progress { + items_processed: 5, + items_total: 10, + }; + let global_progress = Progress { + items_processed: 9, + items_total: 10, + }; + let progress = TrainingProgress { + progress: Some(half), + global_progress, + iteration: Some(500), + }; + + let starting_epoch = 8; + let progress = calculate_progress(&progress, starting_epoch, 0); + + // Two epochs remaining while the first is half done. + assert_eq!(0.25, progress); + } + + #[test] + fn calculate_progress_for_eta_with_warmup() { + let half = Progress { + items_processed: 110, + items_total: 1000, + }; + let global_progress = Progress { + items_processed: 9, + items_total: 10, + }; + let progress = TrainingProgress { + progress: Some(half), + global_progress, + iteration: Some(500), + }; + + let starting_epoch = 8; + let progress = calculate_progress(&progress, starting_epoch, 10); + + // Two epochs remaining while the first is half done. + assert_eq!(0.05, progress); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/recent_history.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/recent_history.rs new file mode 100644 index 0000000..909aff5 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/recent_history.rs @@ -0,0 +1,249 @@ +use super::PlotAxes; +use crate::renderer::tui::TuiTag; +use ratatui::{ + style::{Color, Style}, + symbols, + widgets::{Dataset, GraphType}, +}; +use std::collections::BTreeMap; + +const FACTOR_BEFORE_RESIZE: usize = 2; + +/// A plot that shows the recent history at full resolution. +pub(crate) struct RecentHistoryPlot { + pub(crate) axes: PlotAxes, + points: BTreeMap, + max_samples: usize, +} + +struct RecentHistoryPoints { + min_x: f64, + max_x: f64, + min_y: f64, + max_y: f64, + cursor: usize, + points: Vec<(f64, f64)>, + max_samples: usize, + factor_before_resize: usize, +} + +impl RecentHistoryPlot { + pub(crate) fn new(max_samples: usize) -> Self { + Self { + axes: PlotAxes::default(), + points: BTreeMap::default(), + max_samples, + } + } + + pub(crate) fn push(&mut self, tag: TuiTag, data: f64) { + if !self.points.contains_key(&tag) { + self.points + .insert(tag.clone(), RecentHistoryPoints::new(self.max_samples)); + } + + let (x_min, x_current) = self.point_x(); + + for (s, entry) in self.points.iter_mut() { + if s == &tag { + entry.push((x_current, data)); + } + entry.update_cursor(x_min); + } + + self.update_bounds(); + } + + pub(crate) fn datasets(&self) -> Vec> { + let mut datasets = Vec::new(); + + for (tag, points) in self.points.iter() { + datasets.push(points.dataset(format!("{tag}"), tag.split.color())); + } + + datasets + } + + fn point_x(&mut self) -> (f64, f64) { + let mut x_current = f64::MIN; + let mut x_min = f64::MAX; + + for point in self.points.values() { + x_current = f64::max(x_current, point.max_x); + x_min = f64::min(x_min, point.min_x); + } + + if x_current - x_min >= self.max_samples as f64 { + x_min += 1.0; + } + + (x_min, x_current + 1.0) + } + + fn update_bounds(&mut self) { + let (mut x_min, mut x_max) = (f64::MAX, f64::MIN); + let (mut y_min, mut y_max) = (f64::MAX, f64::MIN); + + for points in self.points.values() { + x_min = f64::min(x_min, points.min_x); + x_max = f64::max(x_max, points.max_x); + y_min = f64::min(y_min, points.min_y); + y_max = f64::max(y_max, points.max_y); + } + + self.axes.update_bounds((x_min, x_max), (y_min, y_max)); + } +} + +impl RecentHistoryPoints { + fn new(max_samples: usize) -> Self { + let factor_before_resize = FACTOR_BEFORE_RESIZE; + + Self { + min_x: 0., + max_x: 0., + min_y: f64::MAX, + max_y: f64::MIN, + points: Vec::with_capacity(factor_before_resize * max_samples), + cursor: 0, + max_samples, + factor_before_resize, + } + } + + fn push(&mut self, (x, y): (f64, f64)) { + if x > self.max_x { + self.max_x = x; + } + if x < self.min_x { + self.min_x = x; + } + if y > self.max_y { + self.max_y = y; + } + if y < self.min_y { + self.min_y = y + } + self.points.push((x, y)); + } + + fn update_cursor(&mut self, min_x: f64) { + if self.min_x >= min_x { + return; + } + self.min_x = min_x; + + let mut update_y_max = false; + let mut update_y_min = false; + + while let Some((x, y)) = self.points.get(self.cursor) { + if *x >= self.min_x { + break; + } + + if *y == self.max_y { + update_y_max = true + } + if *y == self.min_y { + update_y_min = true; + } + + self.cursor += 1; + } + + if update_y_max { + self.max_y = self.calculate_max_y(); + } + + if update_y_min { + self.min_y = self.calculate_min_y(); + } + + if self.points.len() >= self.max_samples * self.factor_before_resize { + self.resize(); + } + } + + fn slice(&self) -> &[(f64, f64)] { + &self.points[self.cursor..self.points.len()] + } + + fn calculate_max_y(&self) -> f64 { + let mut max_y = f64::MIN; + + for (_x, y) in self.slice() { + max_y = f64::max(max_y, *y); + } + + max_y + } + + fn calculate_min_y(&self) -> f64 { + let mut min_y = f64::MAX; + + for (_x, y) in self.slice() { + if *y < min_y { + min_y = *y; + } + } + + min_y + } + + fn resize(&mut self) { + let mut points = Vec::with_capacity(self.max_samples * self.factor_before_resize); + + for i in self.cursor..self.points.len() { + points.push(self.points[i]); + } + + self.points = points; + self.cursor = 0; + } + + fn dataset<'a>(&'a self, name: String, color: Color) -> Dataset<'a> { + let data = &self.points[self.cursor..self.points.len()]; + + Dataset::default() + .name(name) + .marker(symbols::Marker::Braille) + .style(Style::default().fg(color).bold()) + .graph_type(GraphType::Scatter) + .data(data) + } +} + +#[cfg(test)] +mod tests { + use crate::renderer::tui::{TuiGroup, TuiSplit}; + + use super::*; + + #[test] + fn test_push_update_bounds_max_y() { + let mut chart = RecentHistoryPlot::new(2); + let tag = TuiTag::new(TuiSplit::Train, TuiGroup::Default); + + chart.push(tag.clone(), 15.0); + chart.push(tag.clone(), 10.0); + chart.push(tag.clone(), 14.0); + + assert_eq!(chart.axes.bounds_y[1], 15.); + chart.push(tag, 10.0); + assert_eq!(chart.axes.bounds_y[1], 14.); + } + + #[test] + fn test_push_update_bounds_min_y() { + let mut chart = RecentHistoryPlot::new(2); + let tag = TuiTag::new(TuiSplit::Train, TuiGroup::Default); + + chart.push(tag.clone(), 5.0); + chart.push(tag.clone(), 10.0); + chart.push(tag.clone(), 14.0); + + assert_eq!(chart.axes.bounds_y[0], 5.); + chart.push(tag, 10.0); + assert_eq!(chart.axes.bounds_y[0], 10.); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/renderer.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/renderer.rs new file mode 100644 index 0000000..337cf04 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/renderer.rs @@ -0,0 +1,533 @@ +use crate::metric::{MetricDefinition, MetricId}; +use crate::renderer::tui::TuiSplit; +use crate::renderer::{ + EvaluationName, EvaluationProgress, MetricState, MetricsRenderer, MetricsRendererEvaluation, + ProgressType, TrainingProgress, +}; +use crate::renderer::{MetricsRendererTraining, tui::NumericMetricsState}; +use crate::{Interrupter, LearnerSummary}; +use ratatui::{ + Terminal, + crossterm::{ + event::{self, Event, KeyCode}, + execute, + terminal::{EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode}, + }, + prelude::*, +}; +use std::collections::HashMap; +use std::panic::{set_hook, take_hook}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::{JoinHandle, spawn}; +use std::{ + error::Error, + io::{self, Stdout}, + time::{Duration, Instant}, +}; + +use super::{ + Callback, CallbackFn, ControlsView, MetricsView, PopupState, ProgressBarState, StatusState, + TextMetricsState, TuiGroup, TuiTag, +}; + +/// The current terminal backend. +pub(crate) type TerminalBackend = CrosstermBackend; +/// The current terminal frame. +pub(crate) type TerminalFrame<'a> = ratatui::Frame<'a>; + +type PanicHook = Box) + 'static + Sync + Send>; + +const MAX_REFRESH_RATE_MILLIS: u64 = 100; + +enum TuiRendererEvent { + MetricRegistration(MetricDefinition), + MetricsUpdate((TuiSplit, TuiGroup, MetricState)), + StatusUpdateTrain((TuiSplit, TrainingProgress, Vec)), + StatusUpdateTest((EvaluationProgress, Vec)), + ProcessEnd { + summary: Option, + /// Interrupter reset. + reset: bool, + }, + ManualClose, + Close, + Persistent, +} + +/// The terminal UI metrics renderer. +pub struct TuiMetricsRendererWrapper { + sender: mpsc::Sender, + interrupter: Interrupter, + handle_join: Option>, + kill_signal: Arc>>, +} + +impl TuiMetricsRendererWrapper { + /// Create a new terminal UI renderer. + pub fn new(interrupter: Interrupter, checkpoint: Option) -> Self { + let (sender, receiver) = mpsc::channel(); + let (kill_signal_sender, kill_signal_receiver) = mpsc::channel(); + + let interrupter_clone = interrupter.clone(); + let handle_join = spawn(move || { + let mut renderer = + TuiMetricsRenderer::new(interrupter_clone, checkpoint, kill_signal_sender); + + let tick_rate = Duration::from_millis(MAX_REFRESH_RATE_MILLIS); + loop { + match receiver.try_recv() { + Ok(event) => renderer.handle_event(event), + Err(mpsc::TryRecvError::Empty) => (), + Err(mpsc::TryRecvError::Disconnected) => { + log::error!("Renderer thread disconnected."); + break; + } + } + + // Render + if renderer.last_update.elapsed() >= tick_rate + && let Err(err) = renderer.render() + { + log::error!("Render error: {err}"); + break; + } + + if (renderer.manual_close && renderer.interrupter.should_stop()) || renderer.close { + break; + } + } + }); + + Self { + sender, + interrupter, + handle_join: Some(handle_join), + kill_signal: Arc::new(Mutex::new(kill_signal_receiver)), + } + } + + fn send_event(&self, event: TuiRendererEvent) { + if self.kill_signal.lock().unwrap().try_recv().is_ok() { + panic!("Killing training from user input.") + } + if let Err(e) = self.sender.send(event) { + log::warn!("Failed to send TUI event: {e}"); + } + } + + /// Set the renderer to persistent mode. + pub fn persistent(self) -> Self { + self.send_event(TuiRendererEvent::Persistent); + self + } +} + +struct TuiMetricsRenderer { + terminal: Terminal, + last_update: std::time::Instant, + progress: ProgressBarState, + metric_definitions: HashMap, + metrics_numeric: NumericMetricsState, + metrics_text: TextMetricsState, + status: StatusState, + interrupter: Interrupter, + popup: PopupState, + previous_panic_hook: Option>, + persistent: bool, + manual_close: bool, + close: bool, + summary: Option, + kill_signal: Sender<()>, +} + +impl MetricsRendererEvaluation for TuiMetricsRendererWrapper { + fn update_test(&mut self, name: EvaluationName, state: MetricState) { + self.send_event(TuiRendererEvent::MetricsUpdate(( + TuiSplit::Test, + TuiGroup::Named(name.name), + state, + ))); + } + + fn render_test(&mut self, item: EvaluationProgress, progress_indicators: Vec) { + self.send_event(TuiRendererEvent::StatusUpdateTest(( + item, + progress_indicators, + ))); + } + + fn on_test_end(&mut self, summary: Option) -> Result<(), Box> { + // Update the summary + self.send_event(TuiRendererEvent::ProcessEnd { + summary, + reset: false, + }); + Ok(()) + } +} + +impl MetricsRenderer for TuiMetricsRendererWrapper { + fn manual_close(&mut self) { + self.send_event(TuiRendererEvent::ManualClose); + let _ = self.handle_join.take().unwrap().join(); + } + + fn register_metric(&mut self, definition: MetricDefinition) { + self.send_event(TuiRendererEvent::MetricRegistration(definition)); + } +} + +impl MetricsRendererTraining for TuiMetricsRendererWrapper { + fn update_train(&mut self, state: MetricState) { + self.send_event(TuiRendererEvent::MetricsUpdate(( + TuiSplit::Train, + TuiGroup::Default, + state, + ))); + } + + fn update_valid(&mut self, state: MetricState) { + self.send_event(TuiRendererEvent::MetricsUpdate(( + TuiSplit::Valid, + TuiGroup::Default, + state, + ))); + } + + fn render_train(&mut self, item: TrainingProgress, progress_indicators: Vec) { + self.send_event(TuiRendererEvent::StatusUpdateTrain(( + TuiSplit::Train, + item, + progress_indicators, + ))); + } + + fn render_valid(&mut self, item: TrainingProgress, progress_indicators: Vec) { + self.send_event(TuiRendererEvent::StatusUpdateTrain(( + TuiSplit::Valid, + item, + progress_indicators, + ))); + } + + fn on_train_end(&mut self, summary: Option) -> Result<(), Box> { + // Reset for following steps. + self.interrupter.reset(); + // Update the summary + self.send_event(TuiRendererEvent::ProcessEnd { + summary, + reset: true, + }); + Ok(()) + } +} + +impl Drop for TuiMetricsRendererWrapper { + fn drop(&mut self) { + if !std::thread::panicking() { + self.send_event(TuiRendererEvent::Close); + let _ = self.handle_join.take().unwrap().join(); + } + } +} + +impl TuiMetricsRenderer { + fn update_metric(&mut self, split: TuiSplit, group: TuiGroup, state: MetricState) { + match state { + MetricState::Generic(entry) => { + let name = self + .metric_definitions + .get(&entry.metric_id) + .unwrap() + .name + .clone() + .into(); + self.metrics_text.update(split, group, entry, name); + } + MetricState::Numeric(entry, value) => { + let name: Arc = self + .metric_definitions + .get(&entry.metric_id) + .unwrap() + .name + .clone() + .into(); + self.metrics_numeric + .push(TuiTag::new(split, group.clone()), name.clone(), value); + self.metrics_text.update(split, group, entry, name); + } + }; + } + + pub fn new( + interrupter: Interrupter, + checkpoint: Option, + kill_signal: Sender<()>, + ) -> Self { + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen).unwrap(); + enable_raw_mode().unwrap(); + let terminal = Terminal::new(CrosstermBackend::new(stdout)).unwrap(); + + // Reset the terminal to raw mode on panic before running the panic handler + // This prevents that the panic message is not visible for the user. + let previous_panic_hook = Arc::new(take_hook()); + set_hook(Box::new({ + let previous_panic_hook = previous_panic_hook.clone(); + move |panic_info| { + let _ = disable_raw_mode(); + let _ = execute!(io::stdout(), LeaveAlternateScreen); + previous_panic_hook(panic_info); + } + })); + + Self { + terminal, + last_update: Instant::now(), + progress: ProgressBarState::new(checkpoint), + metric_definitions: HashMap::default(), + metrics_numeric: NumericMetricsState::default(), + metrics_text: TextMetricsState::default(), + status: StatusState::default(), + interrupter, + popup: PopupState::Empty, + previous_panic_hook: Some(previous_panic_hook), + persistent: false, + manual_close: false, + close: false, + summary: None, + kill_signal, + } + } + + fn handle_event(&mut self, event: TuiRendererEvent) { + match event { + TuiRendererEvent::MetricRegistration(definition) => { + self.metric_definitions + .insert(definition.metric_id.clone(), definition); + } + TuiRendererEvent::MetricsUpdate((split, group, state)) => { + self.update_metric(split, group, state); + } + TuiRendererEvent::StatusUpdateTrain((split, item, status)) => match split { + TuiSplit::Train => { + self.progress.update_train(&item); + self.metrics_numeric.update_progress_train(&item); + self.status.update_train(status); + } + TuiSplit::Valid => { + self.progress.update_valid(&item); + self.metrics_numeric.update_progress_valid(&item); + self.status.update_valid(status); + } + _ => (), + }, + TuiRendererEvent::StatusUpdateTest((item, status)) => { + self.progress.update_test(&item); + self.metrics_numeric.update_progress_test(&item); + self.status.update_test(status); + } + TuiRendererEvent::ProcessEnd { summary, reset } => { + match (self.summary.take(), summary) { + (None, Some(summary)) => { + self.summary = Some(summary); + } + (Some(current), Some(other)) => self.summary = Some(current.merge(other)), + (_, _) => { /* nothing to update */ } + } + + if reset { + self.interrupter.reset(); + } + } + TuiRendererEvent::ManualClose => self.manual_close = true, + TuiRendererEvent::Persistent => self.persistent = true, + TuiRendererEvent::Close => self.close = true, + } + } + + fn render(&mut self) -> Result<(), Box> { + self.draw()?; + self.handle_user_input()?; + + self.last_update = Instant::now(); + + Ok(()) + } + + fn draw(&mut self) -> Result<(), Box> { + self.terminal.draw(|frame| { + let size = frame.area(); + + match self.popup.view() { + Some(view) => view.render(frame, size), + None => { + let view = MetricsView::new( + self.metrics_numeric.view(), + self.metrics_text.view(), + self.progress.view(), + ControlsView, + self.status.view(), + ); + + view.render(frame, size); + } + }; + })?; + + Ok(()) + } + + fn handle_user_input(&mut self) -> Result<(), Box> { + while event::poll(Duration::from_secs(0))? { + let event = event::read()?; + self.popup.on_event(&event); + + if self.popup.is_empty() { + self.metrics_numeric.on_event(&event); + + if let Event::Key(key) = event + && let KeyCode::Char('q') = key.code + { + self.popup = PopupState::Full( + "Quit".to_string(), + vec![ + Callback::new( + "Stop the training.", + "Stop the training immediately. This will break from the \ + training loop, but any remaining code after the loop will be \ + executed.", + 's', + QuitPopupAccept(self.interrupter.clone()), + ), + Callback::new( + "Stop the training immediately.", + "Kill the program. This will create a panic! which will make \ + the current training fails. Any code following the training \ + won't be executed.", + 'k', + KillPopupAccept(self.kill_signal.clone()), + ), + Callback::new( + "Cancel", + "Cancel the action, continue the training.", + 'c', + PopupCancel, + ), + ], + ); + } + } + } + + Ok(()) + } + + fn handle_post_training(&mut self) -> Result<(), Box> { + self.popup = PopupState::Full( + "Training is done".to_string(), + vec![Callback::new( + "Training Done", + "Press 'x' to close this popup. Press 'q' to exit the application after the \ + popup is closed.", + 'x', + PopupCancel, + )], + ); + + self.draw().ok(); + + loop { + if let Ok(true) = event::poll(Duration::from_millis(MAX_REFRESH_RATE_MILLIS)) { + match event::read() { + Ok(event @ Event::Key(key)) => { + if self.popup.is_empty() { + self.metrics_numeric.on_event(&event); + if let KeyCode::Char('q') = key.code { + break; + } + } else { + self.popup.on_event(&event); + } + self.draw().ok(); + } + + Ok(Event::Resize(..)) => { + self.draw().ok(); + } + Err(err) => { + eprintln!("Error reading event: {err}"); + break; + } + _ => continue, + } + } + } + Ok(()) + } + + // Reset the terminal back to raw mode. + fn reset(&mut self) -> Result<(), Box> { + // If previous panic hook has already been re-instated, then the terminal was already reset. + if self.previous_panic_hook.is_some() { + if self.persistent + && let Err(err) = self.handle_post_training() + { + eprintln!("Error in post-training handling: {err}"); + } + + disable_raw_mode()?; + execute!(self.terminal.backend_mut(), LeaveAlternateScreen)?; + self.terminal.show_cursor()?; + + // Reinstall the previous panic hook + let _ = take_hook(); + if let Some(previous_panic_hook) = + Arc::into_inner(self.previous_panic_hook.take().unwrap()) + { + set_hook(previous_panic_hook); + } + } + Ok(()) + } +} + +struct QuitPopupAccept(Interrupter); +struct KillPopupAccept(Sender<()>); +struct PopupCancel; + +impl CallbackFn for KillPopupAccept { + fn call(&self) -> bool { + self.0.send(()).unwrap(); + panic!("Killing training from user input."); + } +} + +impl CallbackFn for QuitPopupAccept { + fn call(&self) -> bool { + self.0.stop(Some("Stopping training from user input.")); + true + } +} + +impl CallbackFn for PopupCancel { + fn call(&self) -> bool { + true + } +} + +impl Drop for TuiMetricsRenderer { + fn drop(&mut self) { + // Reset the terminal back to raw mode. This can be skipped during + // panicking because the panic hook has already reset the terminal + if !std::thread::panicking() { + self.reset().unwrap(); + + if let Some(summary) = &self.summary { + println!("{summary}"); + log::info!("{summary}"); + } + } + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/status.rs b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/status.rs new file mode 100644 index 0000000..98d9d61 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-train/src/renderer/tui/status.rs @@ -0,0 +1,111 @@ +use crate::renderer::ProgressType; + +use super::TerminalFrame; +use ratatui::{ + prelude::{Alignment, Rect}, + style::{Color, Style, Stylize}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Wrap}, +}; + +/// Show the training status with various information. +pub(crate) struct StatusState { + progress_indicators: Vec, + mode: Mode, +} + +enum Mode { + Valid, + Train, + Evaluation, +} + +impl Default for StatusState { + fn default() -> Self { + Self { + progress_indicators: vec![], + mode: Mode::Train, + } + } +} + +impl StatusState { + /// Update the training information. + pub(crate) fn update_train(&mut self, progress_indicators: Vec) { + self.progress_indicators = progress_indicators; + self.mode = Mode::Train; + } + /// Update the validation information. + pub(crate) fn update_valid(&mut self, progress_indicators: Vec) { + self.progress_indicators = progress_indicators; + self.mode = Mode::Valid; + } + /// Update the testing information. + pub(crate) fn update_test(&mut self, progress_indicators: Vec) { + self.progress_indicators = progress_indicators; + self.mode = Mode::Evaluation; + } + /// Create a view. + pub(crate) fn view(&self) -> StatusView { + StatusView::new(&self.progress_indicators, &self.mode) + } +} + +pub(crate) struct StatusView { + lines: Vec>>, +} + +impl StatusView { + fn new(progress_indicators: &[ProgressType], mode: &Mode) -> Self { + let title = |title: &str| Span::from(format!(" {title} ")).bold().yellow(); + let value = |value: String| Span::from(value).italic(); + let mode = match mode { + Mode::Valid => "Validating", + Mode::Train => "Training", + Mode::Evaluation => "Evaluation", + }; + + let width = progress_indicators + .iter() + .map(|p| match p { + ProgressType::Detailed { tag, .. } => tag.len(), + ProgressType::Value { tag, .. } => tag.len(), + }) + .max() + .unwrap_or(4); + + let mut lines = vec![vec![ + title(&format!("{: lines.push(vec![ + title(&format!("{: lines.push(vec![ + title(&format!("{: , size: Rect) { + let paragraph = Paragraph::new(self.lines.into_iter().map(Line::from).collect::>()) + .alignment(Alignment::Left) + .block(Block::default().borders(Borders::ALL).title("Status")) + .wrap(Wrap { trim: false }) + .style(Style::default().fg(Color::Gray)); + + frame.render_widget(paragraph, size); + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/Cargo.toml b/crates/stable-diffusion-burn/burn-crates/burn-vision/Cargo.toml new file mode 100644 index 0000000..d2a5fa4 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/Cargo.toml @@ -0,0 +1,71 @@ +[package] +authors = [ + "nathanielsimard ", + "wingertge ", +] +categories = ["science"] +description = "Vision processing operations for burn tensors" +documentation = "https://docs.rs/burn-vision" +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "gpu"] +license.workspace = true +name = "burn-vision" +readme.workspace = true +repository = "https://github.com/tracel-ai/burn/tree/main/crates/burn-vision" +version.workspace = true + +[lints] +workspace = true + + +[features] +default = ["ndarray", "cubecl-backend", "fusion", "std"] +std = ["aligned-vec/std"] +tracing = [ + "burn-cubecl?/tracing", + "burn-fusion?/tracing", + "burn-ir/tracing", + "burn-ndarray?/tracing", + "burn-tch?/tracing", + "burn-tensor/tracing", + "cubecl/tracing", +] + +cubecl-backend = ["cubecl", "burn-cubecl"] +fusion = ["burn-fusion", "burn-cuda/fusion", "burn-wgpu/fusion"] +ndarray = ["burn-ndarray"] +tch = ["burn-tch"] + +# Test features +test-cpu = [] +test-cuda = ["cubecl-backend", ] +test-wgpu = ["cubecl-backend", ] +test-vulkan = ["burn-wgpu/vulkan", "test-wgpu"] +test-metal = ["burn-wgpu/metal", "test-wgpu"] + +[dependencies] +aligned-vec = { version = "0.6", default-features = false } +bon = { workspace = true } +burn-cubecl = { path = "../burn-cubecl", version = "=0.21.0-pre.2", optional = true } +burn-fusion = { path = "../burn-fusion", version = "=0.21.0-pre.2", optional = true } +burn-ir = { path = "../burn-ir", version = "=0.21.0-pre.2" } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2", optional = true } +burn-tch = { path = "../burn-tch", version = "=0.21.0-pre.2", optional = true } +burn-tensor = { path = "../burn-tensor", version = "=0.21.0-pre.2" } +burn-tensor-testgen = { path = "../burn-tensor-testgen", version = "=0.21.0-pre.2", optional = true } +bytemuck = { workspace = true } +cubecl = { workspace = true, optional = true } +derive-new = { workspace = true } +half = { workspace = true } +image = { version = "0.25" } +macerator = { workspace = true } +ndarray = { workspace = true } +num-traits = { workspace = true } +paste = { workspace = true } +serde = { workspace = true } + +[dev-dependencies] +burn-cuda = { path = "../burn-cuda", version = "=0.21.0-pre.2", default-features = false } +burn-ndarray = { path = "../burn-ndarray", version = "=0.21.0-pre.2" } +burn-wgpu = { path = "../burn-wgpu", version = "=0.21.0-pre.2", default-features = false } +cubecl = { workspace = true } diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/base.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/base.rs new file mode 100644 index 0000000..c295360 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/base.rs @@ -0,0 +1,42 @@ +pub trait MinMax { + fn min(self, other: Self) -> Self; + fn max(self, other: Self) -> Self; +} + +macro_rules! impl_minmax { + ($ty: ty) => { + impl MinMax for $ty { + fn min(self, other: Self) -> Self { + Ord::min(self, other) + } + fn max(self, other: Self) -> Self { + Ord::max(self, other) + } + } + }; + ($($ty: ty),*) => { + $(impl_minmax!($ty);)* + } +} + +impl_minmax!(u8, i8, u16, i16, u32, i32, u64, i64); + +impl MinMax for f32 { + fn min(self, other: Self) -> Self { + self.min(other) + } + + fn max(self, other: Self) -> Self { + self.max(other) + } +} + +impl MinMax for f64 { + fn min(self, other: Self) -> Self { + self.min(other) + } + + fn max(self, other: Self) -> Self { + self.max(other) + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components.rs new file mode 100644 index 0000000..63d3865 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components.rs @@ -0,0 +1,279 @@ +use std::{cmp::Ordering, marker::PhantomData}; + +use alloc::vec::Vec; +use burn_tensor::{ + Bool, DType, Element, ElementConversion, ElementOrdered, Int, Shape, Tensor, TensorData, + backend::Backend, + ops::{BoolTensor, IntTensor}, +}; +use ndarray::Array2; + +use crate::{ConnectedStatsOptions, ConnectedStatsPrimitive, Connectivity}; + +mod spaghetti; +mod spaghetti_4c; + +/// Dispatches connected components based on `B::IntElem::dtype()`, binding a concrete +/// integer type to enable generic instantiations without extra trait bounds (after removing +/// `ElementComparison` from `Element`). +macro_rules! dispatch_int_dtype { + (|$ty:ident| $body:expr) => { + match B::IntElem::dtype() { + DType::I64 => { + type $ty = i64; + $body + } + DType::I32 => { + type $ty = i32; + $body + } + DType::I16 => { + type $ty = i16; + $body + } + DType::I8 => { + type $ty = i8; + $body + } + DType::U64 => { + type $ty = u64; + $body + } + DType::U32 => { + type $ty = u32; + $body + } + DType::U16 => { + type $ty = u16; + $body + } + DType::U8 => { + type $ty = u8; + $body + } + _ => unreachable!("Unsupported dtype"), + } + }; +} + +pub fn connected_components( + img: BoolTensor, + connectivity: Connectivity, +) -> IntTensor { + dispatch_int_dtype!(|I| run::>(img, connectivity, NoOp::default).0) +} + +pub fn connected_components_with_stats( + img: BoolTensor, + connectivity: Connectivity, + _options: ConnectedStatsOptions, +) -> (IntTensor, ConnectedStatsPrimitive) { + let device = B::bool_device(&img); + + dispatch_int_dtype!(|I| { + let (labels, stats) = + run::>(img, connectivity, ConnectedStatsOp::default); + let stats = finalize_stats(&device, stats); + (labels, stats) + }) +} + +fn run>( + img: BoolTensor, + connectivity: Connectivity, + stats: impl Fn() -> Stats, +) -> (IntTensor, Stats) { + let device = B::bool_device(&img); + let img = Tensor::::from_primitive(img); + let [height, width] = img.shape().dims(); + let img = img.into_data(); + let img = img.into_vec::().unwrap(); + + let mut stats = stats(); + + let out = match connectivity { + Connectivity::Four => { + spaghetti_4c::process::>(img, height, width, &mut stats) + } + Connectivity::Eight => { + // SAFETY: This is validated by `TensorData` + let img = unsafe { Array2::from_shape_vec_unchecked((height, width), img) }; + spaghetti::process::>(img, &mut stats) + } + }; + + let (data, _) = out.into_raw_vec_and_offset(); + let data = TensorData::new(data, Shape::new([height, width])); + let labels = Tensor::::from_data(data, &device).into_primitive(); + (labels, stats) +} + +pub trait Solver { + type Label: ElementOrdered; + + fn init(max_labels: usize) -> Self; + /// Hack to get around mutable borrow limitations on methods + fn merge(label_1: Self::Label, label_2: Self::Label, solver: &mut Self) -> Self::Label; + fn new_label(&mut self) -> Self::Label; + fn flatten(&mut self) -> Self::Label; + fn get_label(&self, i_label: Self::Label) -> Self::Label; +} + +pub(crate) struct UnionFind { + labels: Vec, +} + +impl Solver for UnionFind { + type Label = I; + + fn init(max_labels: usize) -> Self { + let mut labels = Vec::with_capacity(max_labels); + labels.push(0.elem()); + Self { labels } + } + + fn merge(mut label_1: I, mut label_2: I, solver: &mut Self) -> I { + use Ordering::Less; + + while matches!(solver.labels[label_1.to_usize()].cmp(&label_1), Less) { + label_1 = solver.labels[label_1.to_usize()]; + } + + while matches!(solver.labels[label_2.to_usize()].cmp(&label_2), Less) { + label_2 = solver.labels[label_2.to_usize()]; + } + + if matches!(label_1.cmp(&label_2), Less) { + solver.labels[label_2.to_usize()] = label_1; + label_1 + } else { + solver.labels[label_1.to_usize()] = label_2; + label_2 + } + } + + fn new_label(&mut self) -> I { + let len = I::from_elem(self.labels.len()); + self.labels.push(len); + len + } + + fn flatten(&mut self) -> I { + let mut k = 1; + for i in 1..self.labels.len() { + if matches!(self.labels[i].cmp(&I::from_elem(i)), Ordering::Less) { + self.labels[i] = self.labels[self.labels[i].to_usize()]; + } else { + self.labels[i] = k.elem(); + k += 1; + } + } + k.elem() + } + + fn get_label(&self, i_label: I) -> I { + self.labels[i_label.to_usize()] + } +} + +pub trait StatsOp { + type Label; + + fn init(&mut self, num_labels: usize); + fn update(&mut self, row: usize, column: usize, label: Self::Label); + fn finish(&mut self); +} + +#[derive(Default)] +struct NoOp { + _i: PhantomData, +} + +impl StatsOp for NoOp { + type Label = I; // placeholder still required + + fn init(&mut self, _num_labels: usize) {} + + fn update(&mut self, _row: usize, _column: usize, _label: Self::Label) {} + + fn finish(&mut self) {} +} + +#[derive(Default, Debug)] +struct ConnectedStatsOp { + pub area: Vec, + pub left: Vec, + pub top: Vec, + pub right: Vec, + pub bottom: Vec, +} + +impl StatsOp for ConnectedStatsOp { + type Label = I; + + fn init(&mut self, num_labels: usize) { + self.area = vec![0.elem(); num_labels]; + self.left = vec![I::MAX; num_labels]; + self.top = vec![I::MAX; num_labels]; + self.right = vec![0.elem(); num_labels]; + self.bottom = vec![0.elem(); num_labels]; + } + + fn update(&mut self, row: usize, column: usize, label: I) { + let l = label.to_usize(); + unsafe { + *self.area.get_unchecked_mut(l) = + I::from_elem((*self.area.get_unchecked(l)).to_usize() + 1); + *self.left.get_unchecked_mut(l) = + I::from_elem((*self.left.get_unchecked(l)).to_usize().min(column)); + *self.top.get_unchecked_mut(l) = + I::from_elem((*self.top.get_unchecked(l)).to_usize().min(row)); + *self.right.get_unchecked_mut(l) = + I::from_elem((*self.right.get_unchecked(l)).to_usize().max(column)); + *self.bottom.get_unchecked_mut(l) = + I::from_elem((*self.bottom.get_unchecked(l)).to_usize().max(row)); + } + } + + fn finish(&mut self) { + // Background shouldn't have stats + self.area[0] = 0.elem(); + self.left[0] = 0.elem(); + self.right[0] = 0.elem(); + self.top[0] = 0.elem(); + self.bottom[0] = 0.elem(); + } +} + +fn finalize_stats( + device: &B::Device, + stats: ConnectedStatsOp, +) -> ConnectedStatsPrimitive { + let labels = stats.area.len(); + + let into_prim = |data: Vec| { + let data = TensorData::new(data, Shape::new([labels])); + Tensor::::from_data(data, device).into_primitive() + }; + + let max_label = { + let data = TensorData::new(vec![I::from_elem(labels - 1)], Shape::new([1])); + Tensor::::from_data(data, device).into_primitive() + }; + + ConnectedStatsPrimitive { + area: into_prim(stats.area), + left: into_prim(stats.left), + top: into_prim(stats.top), + right: into_prim(stats.right), + bottom: into_prim(stats.bottom), + max_label, + } +} + +pub fn max_labels(h: usize, w: usize, conn: Connectivity) -> usize { + match conn { + Connectivity::Four => (h * w).div_ceil(2) + 1, + Connectivity::Eight => h.div_ceil(2) * w.div_ceil(2) + 1, + } +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_center_line_forest_code.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_center_line_forest_code.rs new file mode 100644 index 0000000..6b6a796 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_center_line_forest_code.rs @@ -0,0 +1,1954 @@ +no_analyze!{{ +use centerLabels::*;let mut label = entry; +while let Some(next) = (|label| -> Option { match label { + NODE_1=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_2); + } + else { + return Some(NODE_3); + } + } + NODE_3=> { + if (*img_row01.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_2); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + return Some(cl_tree_1); + } + } + NODE_4=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_5); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_4); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_3); + } + } + } + NODE_6=> { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_2); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_7); + } + } + else { + return Some(NODE_1); + } + } + NODE_2=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + return Some(NODE_4); + } + } + NODE_7=> { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_5); + } + } + NODE_5=> { + if (*img_row12.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_5); + } + } + NODE_8=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + else { + return Some(NODE_9); + } + } + NODE_10=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_11); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_5); + } + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_8); + } + else { + return Some(NODE_11); + } + } + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_12); + } + else { + return Some(NODE_12); + } + } + } + } + NODE_11=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_4); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_3); + } + } + NODE_13=> { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(cl_tree_11); + } + } + NODE_9=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + } + else { + return Some(NODE_11); + } + } + NODE_12=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_10); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_9); + } + } + NODE_14=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_4); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_10); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_9); + } + } + } + } + NODE_15=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + return Some(NODE_13); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_5); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + return Some(NODE_7); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_4); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(cl_tree_3); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_3); + } + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_10); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(cl_tree_9); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_9); + } + } + } + } + } + NODE_16=> { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_8); + } + else { + return Some(NODE_2); + } + } + else { + return Some(NODE_17); + } + } + else { + return Some(NODE_1); + } + } + NODE_18=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + NODE_19=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + return Some(NODE_20); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_8); + } + } + NODE_21=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + if (*img_row11.add((c + 2) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_3); + } + } + } + else { + return Some(NODE_3); + } + } + NODE_22=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + } + NODE_23=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + } + NODE_24=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + NODE_17=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_7); + } + } + NODE_25=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_18); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_8); + } + } + NODE_20=> { + if (*img_row12.add((c + 1) as usize)).to_bool() { + return Some(NODE_26); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + NODE_27=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + NODE_28=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_22); + } + else { + return Some(NODE_19); + } + } + NODE_26=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + } + NODE_29=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_8); + } + } + NODE_30=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_8); + } + } + NODE_31=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_23); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_19); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_12); + } + } + } + NODE_32=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_33); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_34); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_35); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_3); + } + } + } + NODE_36=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + return Some(NODE_33); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_34); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_35); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_3); + } + } + } + NODE_37=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_18); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_8); + } + } + NODE_33=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + return Some(NODE_26); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(cl_tree_5); + } + } + NODE_38=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + return Some(NODE_22); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + NODE_39=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_10); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_10); + } + } + NODE_35=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_4); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_4); + } + } + NODE_40=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + return Some(NODE_23); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + NODE_34=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_5); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_5); + } + } +cl_tree_0 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_0); } else { return Some(cl_break_1_0); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_14); + } + else { + return Some(NODE_6); + } +} +cl_tree_1 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_1); } else { return Some(cl_break_1_1); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_15); + } + else { + return Some(NODE_6); + } +} +cl_tree_2 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_2); } else { return Some(cl_break_1_2); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_10); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_8); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + } + else { + return Some(NODE_1); + } + } +} +cl_tree_3 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_3); } else { return Some(cl_break_1_3); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_29); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_12); + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + else { + return Some(NODE_29); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + } + else { + return Some(NODE_21); + } + } +} +cl_tree_4 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_3); } else { return Some(cl_break_1_4); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_27); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_25); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_12); + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_24); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + else { + return Some(NODE_25); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + } + else { + return Some(NODE_21); + } + } +} +cl_tree_5 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_3); } else { return Some(cl_break_1_5); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_30); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_12); + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + return Some(NODE_30); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_6); + } + else { + if (*img_row11.add((c + 2) as usize)).to_bool() { + return Some(NODE_5); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_4); + } + } + } + else { + return Some(NODE_3); + } + } + } +} +cl_tree_6 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_3); } else { return Some(cl_break_1_6); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_31); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_28); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + } + else { + return Some(NODE_1); + } + } +} +cl_tree_7 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_4); } else { return Some(cl_break_1_7); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_10); + } + else { + return Some(NODE_15); + } + } + else { + return Some(NODE_16); + } +} +cl_tree_8 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_3); } else { return Some(cl_break_1_8); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_27); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_37); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_12); + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_24); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + else { + return Some(NODE_37); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + } + else { + return Some(NODE_21); + } + } +} +cl_tree_9 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_5); } else { return Some(cl_break_1_9); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_9); + } + else { + return Some(NODE_12); + } + } + } + else { + return Some(NODE_14); + } + } + else { + return Some(NODE_16); + } +} +cl_tree_10 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_6); } else { return Some(cl_break_1_10); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_40); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_36); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_39); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_9); + } + } + } + } + else { + return Some(NODE_14); + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_38); + } + else { + return Some(NODE_36); + } + } + else { + return Some(NODE_2); + } + } + else { + return Some(NODE_17); + } + } + else { + return Some(NODE_1); + } + } +} +cl_tree_11 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_7); } else { return Some(cl_break_1_11); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + return Some(NODE_31); + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_31); + } + else { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_11); + } + else { + return Some(NODE_13); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_5); + } + else { + return Some(NODE_7); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_4); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(cl_tree_3); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_10); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(cl_tree_9); + } + } + } + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + return Some(NODE_28); + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_22); + } + else { + if (*img_row11.add((c + 2) as usize)).to_bool() { + return Some(NODE_20); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(cl_tree_4); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_3); + } + } + } + } + else { + return Some(NODE_2); + } + } + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + else { + if (*img_row00.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_7); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(cl_tree_7); + } + } + } + } + else { + return Some(NODE_1); + } + } +} +cl_tree_12 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(cl_break_0_8); } else { return Some(cl_break_1_12); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_40); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_11); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_32); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_39); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_10); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(cl_tree_9); + } + } + } + } + else { + return Some(NODE_14); + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_38); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(cl_tree_6); + } + } + else { + return Some(NODE_32); + } + } + else { + return Some(NODE_2); + } + } + else { + return Some(NODE_17); + } + } + else { + return Some(NODE_1); + } + } +} + NODE_41=> { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + } + else { + return Some(NODE_42); + } + } + NODE_43=> { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + NODE_42=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + NODE_44=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + NODE_45=> { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + NODE_46=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + NODE_47=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + NODE_48=> { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } +cl_break_0_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_47); + } + else { + return Some(NODE_48); + } + return None;} +cl_break_0_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_44); + } + else { + return Some(NODE_48); + } + return None;} +cl_break_0_2 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_41); + } + else { + return Some(NODE_43); + } + return None;} +cl_break_0_3 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + return Some(NODE_43); + } + return None;} +cl_break_0_4 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_41); + } + else { + return Some(NODE_44); + } + } + else { + return Some(NODE_45); + } + return None;} +cl_break_0_5 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_42); + } + else { + return Some(NODE_47); + } + } + else { + return Some(NODE_45); + } + return None;} +cl_break_0_6 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_46); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_47); + } + } + else { + return Some(NODE_45); + } + return None;} +cl_break_0_7 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row00.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + return None;} +cl_break_0_8 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_46); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_47); + } + } + else { + return Some(NODE_45); + } + return None;} + NODE_49=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + NODE_50=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_49); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + NODE_51=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_52); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + NODE_52=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + } + NODE_53=> { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_54); + } + else { + return Some(NODE_55); + } + } + else { + return Some(NODE_56); + } + } + NODE_55=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + NODE_54=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_57); + } + else { + return Some(NODE_58); + } + } + NODE_59=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + else { + return Some(NODE_60); + } + } + NODE_61=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + NODE_62=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + return Some(NODE_63); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_49); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + } + else { + return Some(NODE_58); + } + } + NODE_63=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + return Some(NODE_52); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + NODE_64=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_65); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + NODE_65=> { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_49); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + NODE_66=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_63); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_65); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + } + else { + return Some(NODE_58); + } + } + NODE_67=> { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_58); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + else { + return Some(NODE_56); + } + } + NODE_56=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_58); + } + else { + return Some(NODE_60); + } + } + NODE_58=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + NODE_68=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver), *img_labels_row12.add((c - 2) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + } + else { + return Some(NODE_69); + } + } + } + NODE_70=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + return Some(NODE_71); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + } + NODE_57=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + return Some(NODE_69); + } + } + NODE_60=> { + if (*img_row01.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + NODE_71=> { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c) as usize), *img_labels_row12.add((c - 2) as usize), solver); + } + } + NODE_69=> { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } +cl_break_1_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_58); + } + else { + return Some(NODE_67); + } + return None;} +cl_break_1_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_70); + } + else { + return Some(NODE_67); + } + return None;} +cl_break_1_2 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_68); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_57); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_56); + } + } + return None;} +cl_break_1_3 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_61); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_61); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_59); + } + } + return None;} +cl_break_1_4 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_50); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_50); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_59); + } + } + return None;} +cl_break_1_5 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + return Some(NODE_60); + } + } + } + return None;} +cl_break_1_6 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_51); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_51); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_56); + } + } + return None;} +cl_break_1_7 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_68); + } + else { + return Some(NODE_70); + } + } + else { + return Some(NODE_53); + } + return None;} +cl_break_1_8 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_64); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_64); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_59); + } + } + return None;} +cl_break_1_9 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_54); + } + else { + return Some(NODE_53); + } + return None;} +cl_break_1_10 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_62); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_62); + } + else { + return Some(NODE_55); + } + } + else { + return Some(NODE_56); + } + } + return None;} +cl_break_1_11 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + return Some(NODE_51); + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_51); + } + else { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + return Some(NODE_71); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + } + } + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + return Some(NODE_51); + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_51); + } + else { + return Some(NODE_58); + } + } + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row00.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + } + else { + return Some(NODE_56); + } + } + return None;} +cl_break_1_12 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_66); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_66); + } + else { + return Some(NODE_55); + } + } + else { + return Some(NODE_56); + } + } + return None;} + }; None})(label) +{ +label = next; +} +}} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_first_line_forest_code.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_first_line_forest_code.rs new file mode 100644 index 0000000..6524b28 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_first_line_forest_code.rs @@ -0,0 +1,223 @@ +no_analyze!{{ +use firstLabels::*;let mut label = entry; +while let Some(next) = (|label| -> Option { match label { + NODE_72=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(fl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(fl_tree_2); + } + } + NODE_73=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(fl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(fl_tree_2); + } + } + NODE_74=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(fl_tree_1); + } + else { + if (*img_row01.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(fl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + return Some(fl_tree_0); + } + } + } +fl_tree_0 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(fl_break_0_0); } else { return Some(fl_break_1_0); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_72); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + return Some(NODE_72); + } + else { + return Some(NODE_74); + } + } +} +fl_tree_1 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(fl_break_0_1); } else { return Some(fl_break_1_1); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_73); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + return Some(NODE_73); + } + else { + return Some(NODE_74); + } + } +} +fl_tree_2 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(fl_break_0_2); } else { return Some(fl_break_1_2); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + return Some(NODE_73); + } + else { + return Some(NODE_72); + } + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(fl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(fl_tree_1); + } + } + else { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(fl_tree_2); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(fl_tree_2); + } + } + } + else { + return Some(NODE_74); + } + } +} + NODE_75=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } +fl_break_0_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + return None;} +fl_break_0_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + return None;} +fl_break_0_2 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_75); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + return Some(NODE_75); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + return None;} + NODE_76=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + if (*img_row01.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + } + NODE_77=> { + if (*img_row01.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } +fl_break_1_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + return Some(NODE_76); + } + } + return None;} +fl_break_1_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + return Some(NODE_76); + } + } + return None;} +fl_break_1_2 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_77); + } + else { + if (*img_row01.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_77); + } + else { + return Some(NODE_77); + } + } + else { + return Some(NODE_76); + } + } + return None;} +fl_ => {}, + }; None})(label) +{ +label = next; +} +}} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_forest_labels.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_forest_labels.rs new file mode 100644 index 0000000..6c994fc --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_forest_labels.rs @@ -0,0 +1,191 @@ +/// Workaround for rust-analyzer bug that causes invalid errors on the `include!`. +macro_rules! no_analyze { + ($tokens:tt) => { + $tokens + }; +} + +pub(crate) use no_analyze; + +#[allow(non_snake_case, non_camel_case_types, unused)] +pub enum centerLabels { + NODE_1, + NODE_2, + NODE_3, + NODE_4, + NODE_5, + NODE_6, + NODE_7, + NODE_8, + NODE_9, + NODE_10, + NODE_11, + NODE_12, + NODE_13, + NODE_14, + NODE_15, + NODE_16, + NODE_17, + NODE_18, + NODE_19, + NODE_20, + NODE_21, + NODE_22, + NODE_23, + NODE_24, + NODE_25, + NODE_26, + NODE_27, + NODE_28, + NODE_29, + NODE_30, + NODE_31, + NODE_32, + NODE_33, + NODE_34, + NODE_35, + NODE_36, + NODE_37, + NODE_38, + NODE_39, + NODE_40, + NODE_41, + NODE_42, + NODE_43, + NODE_44, + NODE_45, + NODE_46, + NODE_47, + NODE_48, + NODE_49, + NODE_50, + NODE_51, + NODE_52, + NODE_53, + NODE_54, + NODE_55, + NODE_56, + NODE_57, + NODE_58, + NODE_59, + NODE_60, + NODE_61, + NODE_62, + NODE_63, + NODE_64, + NODE_65, + NODE_66, + NODE_67, + NODE_68, + NODE_69, + NODE_70, + NODE_71, + cl_tree_0, + cl_tree_1, + cl_tree_2, + cl_tree_3, + cl_tree_4, + cl_tree_5, + cl_tree_6, + cl_tree_7, + cl_tree_8, + cl_tree_9, + cl_tree_10, + cl_tree_11, + cl_tree_12, + cl_break_0_0, + cl_break_0_1, + cl_break_0_2, + cl_break_0_3, + cl_break_0_4, + cl_break_0_5, + cl_break_0_6, + cl_break_0_7, + cl_break_0_8, + cl_break_1_0, + cl_break_1_1, + cl_break_1_2, + cl_break_1_3, + cl_break_1_4, + cl_break_1_5, + cl_break_1_6, + cl_break_1_7, + cl_break_1_8, + cl_break_1_9, + cl_break_1_10, + cl_break_1_11, + cl_break_1_12, +} + +#[allow(non_snake_case, non_camel_case_types, unused)] +pub enum firstLabels { + NODE_72, + NODE_73, + NODE_74, + NODE_75, + NODE_76, + NODE_77, + fl_tree_0, + fl_tree_1, + fl_tree_2, + fl_break_0_0, + fl_break_0_1, + fl_break_0_2, + fl_break_1_0, + fl_break_1_1, + fl_break_1_2, + fl_, +} + +#[allow(non_snake_case, non_camel_case_types, unused)] +pub enum lastLabels { + NODE_78, + NODE_79, + NODE_80, + NODE_81, + NODE_82, + NODE_83, + NODE_84, + NODE_85, + NODE_86, + NODE_87, + NODE_88, + NODE_89, + NODE_90, + NODE_91, + NODE_92, + ll_tree_0, + ll_tree_1, + ll_tree_2, + ll_tree_3, + ll_tree_4, + ll_tree_5, + ll_tree_6, + ll_tree_7, + ll_break_0_0, + ll_break_0_1, + ll_break_0_2, + ll_break_0_3, + ll_break_1_0, + ll_break_1_1, + ll_break_1_2, + ll_break_1_3, + ll_break_1_4, + ll_break_1_5, + ll_break_1_6, + ll_break_1_7, + ll_, +} + +#[allow(non_snake_case, non_camel_case_types, unused)] +pub enum singleLabels { + NODE_93, + NODE_94, + sl_tree_0, + sl_tree_1, + sl_break_0_0, + sl_break_0_1, + sl_break_1_0, + sl_break_1_1, + sl_, +} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_last_line_forest_code.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_last_line_forest_code.rs new file mode 100644 index 0000000..9187a4c --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_last_line_forest_code.rs @@ -0,0 +1,787 @@ +no_analyze!{{ +use lastLabels::*;let mut label = entry; +while let Some(next) = (|label| -> Option { match label { + NODE_78=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(ll_tree_4); + } + } + NODE_79=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(ll_tree_6); + } + } + NODE_80=> { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(ll_tree_6); + } + } + NODE_81=> { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_82); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_3); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(ll_tree_2); + } + } + } + NODE_83=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_5); + } + else { + if (*img_row11.add((c + 2) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(ll_tree_2); + } + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + return Some(ll_tree_1); + } + } + NODE_84=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_5); + } + else { + return Some(NODE_81); + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + return Some(ll_tree_1); + } + } + NODE_82=> { + if (*img_row12.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(ll_tree_4); + } + } + NODE_85=> { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c + 2) as usize), *img_labels_row12.add((c - 2) as usize), solver); + return Some(ll_tree_4); + } + } + NODE_86=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(ll_tree_6); + } + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_7); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_0); + } + } + } +ll_tree_0 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_0); } else { return Some(ll_break_1_0); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_81); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_0); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(ll_tree_0); + } + } + } + } + else { + return Some(NODE_84); + } +} +ll_tree_1 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_1); } else { return Some(ll_break_1_1); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + return Some(NODE_80); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_82); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + return Some(NODE_85); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_3); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(ll_tree_2); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(ll_tree_2); + } + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_0); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(ll_tree_0); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(ll_tree_0); + } + } + } + } + } + else { + return Some(NODE_84); + } +} +ll_tree_2 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_2); } else { return Some(ll_break_1_2); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(ll_tree_6); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_7); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_0); + } + } + } + else { + return Some(NODE_83); + } +} +ll_tree_3 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_2); } else { return Some(ll_break_1_3); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_79); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(ll_tree_6); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_78); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_7); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_0); + } + } + } + else { + return Some(NODE_83); + } +} +ll_tree_4 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_2); } else { return Some(ll_break_1_4); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c + 2) as usize); + return Some(ll_tree_4); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_7); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_0); + } + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_5); + } + else { + if (*img_row11.add((c + 2) as usize)).to_bool() { + return Some(NODE_82); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_3); + } + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + return Some(ll_tree_1); + } + } +} +ll_tree_5 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_2); } else { return Some(ll_break_1_5); } } + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_86); + } + else { + return Some(NODE_84); + } +} +ll_tree_6 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_3); } else { return Some(ll_break_1_6); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + return Some(NODE_86); + } + else { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_6); + } + else { + return Some(NODE_80); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + return Some(NODE_82); + } + else { + return Some(NODE_85); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_3); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(ll_tree_2); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + return Some(ll_tree_0); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + return Some(ll_tree_0); + } + } + } + } + } + else { + return Some(NODE_84); + } +} +ll_tree_7 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(ll_break_0_2); } else { return Some(ll_break_1_7); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_79); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(ll_tree_6); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + return Some(ll_tree_6); + } + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 2) as usize)).to_bool() { + if (*img_row12.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_78); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c + 2) as usize), solver); + return Some(ll_tree_4); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_7); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(ll_tree_0); + } + } + } + else { + return Some(NODE_83); + } +} +ll_break_0_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + return None;} +ll_break_0_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + return None;} +ll_break_0_2 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + return None;} +ll_break_0_3 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + return None;} + NODE_87=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + return Some(NODE_88); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + NODE_88=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + NODE_89=> { + if (*img_row12.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c) as usize), *img_labels_row12.add((c - 2) as usize), solver); + } + } + NODE_90=> { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + NODE_91=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + NODE_92=> { + if (*img_row12.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row12.add((c) as usize), *img_labels_row12.add((c - 2) as usize), solver); + } + } +ll_break_1_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_88); + } + else { + return Some(NODE_87); + } + return None;} +ll_break_1_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + return Some(NODE_92); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + if (*img_row11.add((c - 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + } + } + } + else { + return Some(NODE_87); + } + return None;} +ll_break_1_2 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_91); + } + return None;} +ll_break_1_3 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + return Some(NODE_89); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_91); + } + return None;} +ll_break_1_4 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } + return None;} +ll_break_1_5 => { + if (*img_row00.add((c) as usize)).to_bool() { + return Some(NODE_90); + } + else { + return Some(NODE_87); + } + return None;} +ll_break_1_6 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c - 1) as usize)).to_bool() { + return Some(NODE_90); + } + else { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + return Some(NODE_92); + } + } + else { + if (*img_row11.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c) as usize); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row12.add((c - 2) as usize); + } + } + } + } + else { + return Some(NODE_87); + } + return None;} +ll_break_1_7 => { + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row11.add((c + 1) as usize)).to_bool() { + if (*img_row12.add((c) as usize)).to_bool() { + if (*img_row11.add((c - 2) as usize)).to_bool() { + return Some(NODE_89); + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = LabelsSolver::merge(*img_labels_row00.add((c - 2) as usize), *img_labels_row12.add((c) as usize), solver); + } + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + } + else { + return Some(NODE_91); + } + return None;} +ll_ => {}, + }; None})(label) +{ +label = next; +} +}} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_single_line_forest_code.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_single_line_forest_code.rs new file mode 100644 index 0000000..bc36ac7 --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/Spaghetti_single_line_forest_code.rs @@ -0,0 +1,91 @@ +no_analyze!{{ +use singleLabels::*;let mut label = entry; +while let Some(next) = (|label| -> Option { match label { + NODE_93=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(sl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + return Some(sl_tree_0); + } + } +sl_tree_0 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(sl_break_0_0); } else { return Some(sl_break_1_0); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(sl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = solver.new_label(); + return Some(sl_tree_0); + } + } + else { + return Some(NODE_93); + } +} +sl_tree_1 => { +if ({c+=2; c}) >= w - 2 { if c > w - 2 { return Some(sl_break_0_1); } else { return Some(sl_break_1_1); } } + if (*img_row00.add((c) as usize)).to_bool() { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(sl_tree_1); + } + else { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + return Some(sl_tree_0); + } + } + else { + return Some(NODE_93); + } +} +sl_break_0_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + return None;} +sl_break_0_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + return None;} + NODE_94=> { + if (*img_row00.add((c + 1) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + *img_labels_row00.add(c as usize) = 0.elem(); + } + } +sl_break_1_0 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = solver.new_label(); + } + else { + return Some(NODE_94); + } + return None;} +sl_break_1_1 => { + if (*img_row00.add((c) as usize)).to_bool() { + *img_labels_row00.add(c as usize) = *img_labels_row00.add((c - 2) as usize); + } + else { + return Some(NODE_94); + } + return None;} +sl_ => {}, + }; None})(label) +{ +label = next; +} +}} diff --git a/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/mod.rs b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/mod.rs new file mode 100644 index 0000000..a00fbde --- /dev/null +++ b/crates/stable-diffusion-burn/burn-crates/burn-vision/src/backends/cpu/connected_components/spaghetti/mod.rs @@ -0,0 +1,273 @@ +//! Spaghetti algorithm for connected component labeling +//! F. Bolelli, S. Allegretti, L. Baraldi, and C. Grana, +//! "Spaghetti Labeling: Directed Acyclic Graphs for Block-Based Bonnected Components Labeling," +//! IEEE Transactions on Image Processing, vol. 29, no. 1, pp. 1999-2012, 2019. +//! +//! Decision forests are generated using a modified [GRAPHGEN](https://github.com/wingertge/GRAPHGEN) +//! as described in +//! +//! F. Bolelli, S. Allegretti, C. Grana. +//! "One DAG to Rule Them All." +//! IEEE Transactions on Pattern Analysis and Machine Intelligence, 2021 + +#![allow( + unreachable_code, + clippy::collapsible_else_if, + clippy::if_same_then_else +)] + +use std::cmp::Ordering; + +use burn_tensor::{Element, ElementComparison, ElementConversion, cast::ToElement}; +use ndarray::{Array2, Axis, s}; + +#[allow(non_snake_case)] +mod Spaghetti_forest_labels; +pub(crate) use Spaghetti_forest_labels::*; + +use crate::Connectivity; + +use super::{Solver, StatsOp, max_labels}; + +pub fn process( + img_arr: Array2, + stats: &mut impl StatsOp

{xNa@h?xt(y>h_kSjs2W6j<^`CriUX5$>Xc6%M z+0CAU#3@HXEDy(bg-Mfs`^S#A{~Gx4Lmar``JOF*sXg^G_r{}I7r5X5^bKvCx=d5V z#Ljzwj6Fzw35sf^AGs_L?$VCEG@NR^TY_ylH@K^5H1LH#-skR{<%>Wi1<*8^LZIav z(QDq8#HG@vL(|I3Mcn5h@niYyvMHmi&0@zgy1v z`9J^S;Fz~8u*pAI2;K=-7<&)De%4U*{Aks02X7pyy7zs`+u+#9P?|E00)ku;!kwT@ zR`(b77`F5VRC59&E=Z?mek^^aUI?LDUln`kVlkQ(c!t_2!Ys|fyyqz zwRlftxO3LP*@li~V;r5CiA|ai7;=#2Ef|fGj*S;1HyZ62rXkVc%M)iZxRWk3SpaYK z31j|H2hY;zvw7ADbsI@9yiqMuH{&r#d}3?zroT!ZD+HrO;*b-HrEO&nqq;YMxUPk2 zkYVJ$iHR5|(=;-xKZK)jK$(U!j1aTZfvqhE6Sy@d$6Oulj@hIkjV!Qt7mMHk6}?{E z)LTdetZo*+MpT)#)MwuQk!;ec?cWp-Sw+seekap`)I*kS?YERu~NHnCb!s+kOn zc3%VOqbmgj;Sg#yYI>!>0({e!v(9OAI{1)N0e?2d20RPb+O!|dN06ov7-D_!4pUh% zoDvsjx<*==0*0mcZtz~FGlMgJ=`!XPVIHzSmHXn9axiKpq&>+>b^%cbV-=YXY3#{z7`IC z3nnG4+Ai?n?;!$iYbrvi9A2!Bs`k{9O!lY=B@F6h6idU%n z{m$g>mgM)@HHO0jznI29i6X;8rLm_Qa^q`92ELZN@z0gHe+3w1|M&UCKmT|4-R1!} zOF}E}J(_ds{N=dYKNiH?bZhNCUYuj+!j(3MN=HWCOd*5AJ+G-;r+f0`IhIh@7Tj?= z{ZwrIjq#U_Kjv#kn!{gS5gyr^Gs74^k-9>T{PGg_*-^V{N2Yd^A6%RB$FBL;k+l5M zeE%a?&u#tB!5^l(vjU_1OFYLf3wbXMuO@dA^4NagKE6;I8l3i1N`KT(CI5bY`SLfP zwM!q>SDoxR)gl=Et{{KR!_%cDcc$e~&aYq891Q!%FZ<%leh&=z<@ZBZZ{7YYE$Z@~ zLd1`!qkId$?e!cN46w8v<-M#)t2nb+8$>bj^E2D_4x@YqcJxBnAYRV$!u9fpAobWs z#lAp0pzKo0bh3c+BA{C zzbXyb-;*q7sGEt=t*9YN0@b4xyZrRe-N$+79*lV%!rmA_ZahY-BGZQ@$r_L`Gf*vR z3h{9<{$oVqwbenY`8Uq#RuOXU8=zf5S}bi@Dq*{Bs!%9->p>c>;w46VcBW#CG_u#+ zaiM7a1J&AuW~i1_u@x9JE^A2!tDIyg-<#Lwb|@K^ANo!b^>#KHxP5U?LcOQG3UR=P zJ}k-{2H8;p$3Ui#ChftDnwDsOHf&?R{aDT(Do;eL7^@MpeWW!!DN7FCw+@_*UU1A3 z0vd9$Ol=JgO#93h(W1(!i0tA!+_u)`&sHo!3sMDMXb3_|%7ro0tyKix8vtrlw> z%F9Vc9}=OtEtGy0C&8AN*aRxGxGGf+q8~zWRbZ<+P_jy3uNERVHgcR>nh0he(q<3@ z!~y8*;NJPvC(P6i1{j%sacoTt&T62VwXHw!}!176NUE5z!(Wx)^ zg0mh+vH@`|t$U*0?y5_}Btv}$iZxEf5vBGdkp9@9LrfqxVP0^HR<&E~Nd_0^4RF&) z&juYbNQx+jZs4O$TJRn2^@IY_kF(c97?qufXtl?0o=uRsQLg4ZH!giT)FvtNgV+`% zoLNSBk)kr^jHbSj1d4GLO$_*hp1GjivIWJFx26{BwnU1P6k3jfhGe!e z*|S9s;O}2${t43nhxjfk|2pS(LTr8h{HbOj82GL}clLJ4RiEzJEW&rKH?C3mCHz|t z{-`+P_mLYl6P()pAm*p2B7cW>_bv)uWdsfj8-LXZ)CM&4M$=@rJG3hIrgu5EWeE5L zkE>!QC(3yC*v5l-JjTR|%t54U%qRLq<(-(muK*OO^a@N2`?_q}ap15G(k>JcGP zVcS+42r+I>bqL3Xm(nunSMonHLSDWck-ztm)5bwp*c14puSSG3nmtKGRvQ4ZD3<)4os!yB-IsWR^`zkX zU|-+3vJc0(*z3m-4H2Q-ENSvxDY5)-TQ8yF=%DIV@;8*+J%1?b;J$zE`~CUPm-)Ba zM)@fR!?<1pvTIRt#z=qmE6o+8A9=~)xUciU(o=;VfvbZ>5r3x@)&F<8?9}D&6V}I` z^kv;n+)IuLb&J0BdaLX6{asbB_q*QuHtz8GsvVL3FD_pA;nbmT$L`+Sy*I8G)F1*P z#;>8i>=E@<6pLO7MY;_aNfJMTs=i;n>Zsb)q|S{?obuGY86UaBvDMO9>MlB7tz`Ao zp_SprNvcsSvGl1jmLi;MdV)n=;Ia7;thJMb2#U(RK#**uQ;h_$foYTe?4f?sL$@xe zI;Y9mR**VKu*XoH;OaGr=`k9PQxPa6a{x(7CN0~MIK0yltawnLG?6CKcL@E>?~we?J0(GHk6+hR2?eFegm@Lq<* zSVhfT)Mt`SpaE4+(iGA+KL?;cJ5rN$bBtN3ONDvQ9jBPeL4H(X>#1n7XbUT?_T^1h zKYF%#Fi*5r%(2YtMb`J78PCw6UL6PXXw|H-fs7pK1>8rfvH*MsIzNKL2N4z5D)uh3 zuG=RfXXgFvM7u*el3&sFKhOz^t#$@E+CUB>=7WE=ZN^cF*-D@BZaTSj01gZ;fSO)F z^$3i`&B@BaMp|;E2v25$?Ojf#t{II$Pmypk7M@?S*FQa#Z3Pa) zM_(-BP)|;GT#NT!?+LfcwxEH?G2EBN!szlw0FL2IMI`8n$6F-}6w} zMkoCY4xOLPf@QJbs&*h}WYi8Cb8^P5)(tWE4)fPPC6g?VPTHL|oRzHYAI{)9wIP;= zz!y%?%U;NXrh^B!5M?Q%Zn7cP0?01VfEgg++r9SY(t3gW8~S9L&}dI;ltWT@J8Vjb zl5Lr}dDTbBwE!8x0eGz zJ(?|c;Pfw-4|yr*)Rb94b@33&(uAyFOyMkGc^m*y-?g7+rEP*W<_os)lDF)!v-NIy zihh{{qIU#-jRVFz`E>=Q7_TiRjqOKU83Db+RD5tHQ;s89BBsdvla(i06lD4?A=s&)yTpekJlh~VT?dSxj=F7>Z3RC*Pv z&va1V?$Pf5eWP3dSi7GtlrG&Z(k?x9T5U_dJN_Z7wELm^auOy|I6hlKyZvY?aNHkG z85`AFj`CxCe%p5`veAzTES>V!kTjpjNN9OB*!_!GWwH(vOhiv03X zU*wys?{`L~+%Fh^5gGCCoc$pUWkq`e{=m6oRtgdwk8uUd^Bn6m2aveTL%pb#%|68q z+?|0tpm#Iv1ko)OPa{aGdvOJ#^^%bAbs&@hqk*?1vyhZcG?U%KC(shyaFvTiEXnTW zQN-==+@`h)s}5@9G9kEWC}~>KIgcm2pS{Bh0=ZKoC}(Pm30fhu9k0K~?^VKaIA`Y; zEOw-Ea@7qzALt$>J%CmOia*t(70)Q`Rx$IKbsP;y|CVmFml9=Lct4bb$^5WC`amlk zLAarsL7KGzxCWLQ*!I9xNl21284bf#)G!~8zSF1T$>pt6i06rDp) zyqX#b-AFH`@;Pk>#mlL@S`ca2gVw!B)r&wNm;i8;A9mQ*^i&VW)yEG>mMzG!Mxsei zlC%+kSLqJKT7}DOgLN#fvPggxhEKCHqDf~&j;7k9LLYq(K2J**FVf^2KU#bB2!na0FqQz4b8 zoFi~H0yu#2qdiOkGjPrXNC}*vyzG7^h(HlW15_DCr~kNs3!)9#As{ZLX)FaF*bgo$e@19lJ zR_;-$;Vqy|CsgkUp=7Dajxz*dN{e&4WfLIaMJPB< zwt@awC3-rnr8pDJ)hPN*y1aoSlcJ?=AOk|uf|TvX&@_gWAB4Z-5x;H8?3nURH zT417$7l28;5Jgfq!08>cWrl1`7c5j`L2l5Xwuh7qZUqWVT*W(tXy!!t4EU`aqT;8T zJtxF5V0s(p>47!onvUfIgZseAV`4RrWAS0N*{pP1HetF)n>I%EOqR)t+jYkBEiHLC zbo_v}ZX1PI8bs`$uWW4+N!CkFatwTq?l36dv>~*uqr8``l__N@B{N)5OBJPDLxE<) zNS33Zw&ii+Hv!CFzxv0+ebawUd~xx3!=tIkcbANwa-$)hUBUxGHC{A9GVCtP1OE*w ztQ{T4soHXfyve%hi4vZ@NGFrU3vh(AJkRN+=yI(5eI2zYo~CoKWRLLNSGZR>*!ac{ zx9ORwiO_f~0ZGMdgwdB`OHT(5WCs^su0P(8iXEH!ze~AiPipxW`TeJ30NJs)`%T$i z*Ory^P}s}Fh~Imbh|7tk(w~2)VVTYS<0EfEeLF%^GS0<)8OOc%61Xe;+g?s&=ajob4ctwpXC_4)GT-6_~p+vJY9%S*Rgf-n1!YqcqR1qs+ww?LfP z{m02~+v2;I+&fC&{3^IwAae|ei#>U1{KA)g{lQhaV_7#Nxj)bl-v*36AKNZON?|s@ zn3t-;=eLHNWLuE85UU09rbxW(#1S}b^~tMfRR`ub3(J(Nd`Gn^Q9$W8j}z(iAgZKd z;d^)ZibG|>1&l>T_1c_>fQkukMW!|6=A;DQ3$r}fp}j$}L=R`=3kfuiDG$lgD=#gq z0z{Fk&1whHhahdycBG}V;fq@I;%DB=G>G{X*`mN$lw;?G<797LkE&f!Lo$#_$?2+h zG>Gn15V1-Za4zB3H}EEHT|ON^wFvPK#K2 zwQUXO(Cgm7oMiwwI~daZDXHox_Qnk>?1!k+%nTr>5+i`)8&I-!(66zJ$F`bG{4Q}9 zEqsm(r)C8O5!y*<%F5J2E(q#P1!d$HgAA@w6j1F8$}N0InyhGtFkC<8Vi6fkO2!+e z{wW!G3)G8cpy`aUE&#CQc(@YWr_|g*TgDa6e{c8 zNu8bDV$=0>!wVNLH5+diuE|HA3^sQIW7kHDa0V+RXcUT|Y|tiuhei_Fjo979b5AM} z?TbU*{N4hamg=oUh;0c#QY>o`irPriVT(!vc)?%?x*?q>q2=XYg@B)2y8^U|OhKeg zT!deic>!Z9rJLU1!X7PuW5-G^10x*>au$I1*AO@)ZTdEtqpNhs9KANWD4nVTmIyzD zFdsn2x8Qoa3{1!d=s)U720e(Y16cUIviWw$DUbW|m{7a!EZ`})am}u~!?Yy{^tW3sM% z*wj$e(@6z3g`i8%g@;s=*w$&rN7^}b?szB*SV+LC#nu69DT5ZCV2A0006)oIsqq4Ls{;7<$pU6 ze6BUr2O3)*>`)zpSQz=v@u<+JHraNYkwhdUJ#z|QPkKtz)ImqMyNgp~kFMKKCOjgZ zkpx6lUFaL@R2?|V>P(2)53+GqVmS|3fm)&0zj~8TgQ~c!!tR*2 zg?)*1@>{6&E!3JL(VoSH&%j$JI;>ap-CkMgnhw;iZ>k@Ov4FJai$E^uzPn!7>1p0r8+fCR(EY>)&dXSGG3((%KLOc&f9ovyP~WR?UQdA8i3Paapu#0-hZVDr*$025pjSaCFp6J3k>E zK-E(fr4gvj+UouL8cDyV1Gh)&3simHZB#+88qavMC@|X>1K4j^~4ihn5^t-5S-Te5X{F#!+02)F-k zi5~*wz*_OD7Boq%voqE+1@j1;x5Y`K;tfj(v-ZR}J;ZX?0j&*08^$7>HAf_xd4PLd zq9G0QDyFJZWf+2t%LG2e56qJd$HLn^LLNmp2n)iU9i3fwNps`UkA=3s@en~81dFPU zgBv1=CY5elu>)U@-lr-l1t3yixkX)jr_Q7_+IvV4FSRZ)59v-H+MXae>m+r?xfx7=`OteZ%Qh z*0a_9;F9d(!aQGobpMw8$?WjJ*T!VTAIA$W|Lu#&a}_BMYIi&>$NG)G97&0C|90Th z@hx5*Ft>oX$5FuVFwWKmzPjKt-XOS+9FF{~2A`Ah@_fh+#h`61Ow7Hj@6BZ+dWx5WBtwP z-&WDC>RHB8M(iEa-l~H~UEb|`*nAXwI$e5Mtp4Ugeg5vh9F7=>S^utC`_({XYaMw@ z0_zru69!0qkVoT{lw<8q0-mV0J^U>U|tke6_N_ zeIxDB>GK7Zyj_}}_~TRz0EqPf;blHZbY#1i>py=yb^aU-8iqEWAKX=?n}qHZU5mON zTjcvWWg+&Oo9XsxXQvk0XHdKB=HZcW_!KYZ=U{xW{ROePd%O{Xc%6`Fx}?ao|Fnvr{4C(4H&7Sv3!rM^b2)>s-G5aHSyO z_E1d~w_vuaawCM)p&A#+xZs(uS2~AgO!3)^p!OdYZrb`MBUJA+jBd9SZ9%6hSE-22 z8!+$rR(gMm$PF%xPuJkGi|tq%ItGE8Ui%wn{VFB<{SjqloG=up=*Dy`J8+hp{L3ks z%RXj<$~r{1tx_vGXQwXy*}Lf25$#*yl@5j;mRFL^8Z@C^m@)p&!_gIOm7@P*r^&Ae z?=z%nnqy?bGRe4+EbJSv_4g6spy$9;^yx#rNZ`TD;g+*k&O)?LcewagcoHlT;BCB! z)jg`7$r?`3d#m*}^S!nT!utq1X=!MkTTX~Yh~K#3*Toea9UPauYzFdfKOshvk*-$? zvmdvaN-?H+HGs}hI-+KVC6;?GEc4mnqtK(8)ucr}x%{G|xJC}anx`D1p@ucy7aY9K z7TK9IhG%-kUK3=;R5s9e51W(q~_34nnaFes&Jz10mJEiYJWu zWqr+1Xf()cwhz&XBHxNHv8_L9Fz49C^M}}@dG&+9g{hmG)T6^ z)ovv^@*3oVQ0g%hEe#@CaVloUFqoTsT=(QiHx z2*a2JQTDtWdbVXq+=ep+*sLRzajqNxNax(pklA9Ersu`z6<&3;=T-sJlO*Tl@z8Q2dpG0t>fl&(wB|Ze#WS@Xz>rD_=F8 z2pQbfdy&JJ!7H7Pl;y9u;As6Br<^W#IKOpr{88>VaA4r<%Wt@TKBgf*_s=ix{0GlJ zVMoHv(31^M_V7Om_J!P!sJrSHuHreD@e(dZ_FbLYIdsCe&Mx&@?6vXP|LuP57~0=0 z3Wad9hJ6-SyBXf7cA01Fk%U3V2X2nnYaG{hc2^0u^*v6hkXsz>X09H0oc$#U^2ks} z1!XD$CO&>J+QJ2Z^VAW51FE&u9cd99V>JA>WIRz`JlZjI&WTojrTiizBX<8yM{Z2N zbk^g`#_+e^AGU4f3)nv?}3+pJbd-|nSr9R;2#5g3#*eIFFTYz-Iy1+W!1eJFuCQ z0VM(d=ctjShuPK}p{5V$%|1y+1&mO^RJP+<93$eaFp~ZSK&AAWc}M^SL1j*X>#v_( z)*6KMye?`}= z)dw_5Hu=b`Nqf@AuA`g3k1Z+ufuPM(n-2%4v6KgO~^6c8c@Qx z6-|_N36UEweY{t7Kpzhxznf*`PP3v7X{~8PNZHfM60W*lVy$>iVYhNjO&okr5Wa_F z({PZj9T|ghUa}a{x*g~vJkg#R_`)8Vuo_=^I+cGXPk*#nzD1PAA5u@|vDWiY0n>2Y z><_3;*O1d~_{7G>7F-O;lpDbT&4|A!fNORNq%SbPH#2d|&nVeT2`K#3kb1sW3hHBG zmR^*Ztp9GdvT1SpjS?2^#)yEsyyOl5IR>ytROU1-(h->{iyJj$V(uH1u~2fo zVBoHDwStV%G^iG&BW>Aw{S$C4Ww8K7yU1q?pt-CM)&%G)nWrR64J=(*Oo{i>JQAc5 z*Qz;&VxOFFi(9y6xlM$h=BqZ%bn`nGq(RNEGii{-6+Sgx58Bvmnep@BC#%k{H+1#n z!4M1F0HHEIf!Z$Dg@A+A#Rft-UFtY`hg>%emZ;T_VzNQ-J$v;MA{Il!>1UL)cbK_H z!7c#BR-O?iCMsuAdll7*o|*?_Z!?>jh#<~u-5P*2u&b*ob-)CA)K=g$RUeSQvC<@F za$81eG2p5fXRH)fo^Zm3gyGa;le6I=r9DG9J6A@ws8v=41Tl{>HqKz>2+~?K$5xEO z_hYN=xd5jC$@|Elev35n)64KnGE=kz_Mf7!_;*Es`xR#J|_(I}eA1wTK(6^(f{l(Vf4H1r~Tw@BkegmIzv4=2gFDdb7kF1ow zt1tg{Hj%ZF^+Q2ZYh}IN&5r*^(wRUtedS^Mo26r0tBgyf79rC*7F-~8VG#nEx>2Em zfQoEkgbHPifFXpCWTvP{5mF0+7zh>0CPHKh0YZ{NmWU9LHIR@*2wMnAfNW&{uHSc# z=jb^e6-;vP{r~R!K94`CtI8A(>C;ypG)Nqt9t+B7jB9jpw}1KZ4bvrFBx_K6vgybA(ZRF(lM|p**T0^*M>VtU6{$m%FoTh8& zv}-kW(GgE?28G6#4cvML(u}N(RkwIOx^=nzYF=ej(dEMfV@XkmZtk)p z1=U?}ZMbb`ig8T4SaI?^LmFxsIZFPlf6`%Tu5oO1Rl?7F)+`M2ZtR{MI3 z3Y%p|b3SY7|HKX@J})wBSUPd7Ak$#eJxc#F&)=84syK^8uQ>IhwKLiV@VBdiPbR`s_WoADp(r zZi7sH8Y|PeJre8&BXKicDOpJET+~TyJ{P-`+@6tJf}f>fxis9HE3TIDQ4=5+4m1eU zR?Iy@6=-g;?MKTpbitX`035B<)FMD6o{;4c4}x8D$5{(QMhOI-#Dkt5nX$Li4G=Tm z7RuvI>l=gg6dA$BbFZnoo^sNQlIbNrCCg!D?K!!acqsQ#hDL+rs%eS6GTFLVoU&jTSL)| z!8IPDjU{h-Z_g{VjAI4(4w>0Y)vd&=^*j)rSRG({!Cs>U>9HXykeVWNqb0_*pOBci znVDHUsFhu@i_}mfw>}{lg3-bh!~IsYtb(AI+xing@<40=cc2*^+9G>XFNrxQ1ul8k z!i1f+D4LB%tSv!wi}qfd)3tLc85!Tk7&#(yZ+3DQ#;{#_>m~*ES?^nCh&c)h7Jn0c zMZxnp6zq>5pQ!V4y9rknlQd8nbAsNjyy6NxK|ngQ&pc0w)IC8wPM(9I)(SinPf>Y| z``G^FUSPe5O^a{HR{Jtu4>f6bKfYWJ!)!T<79Gi3llGq-cimb-h@TK-T>Oemq_mCH zQ4x}UFn?_a#ED9WdCO2$GXB&0Tg;tbNZg z4@@XD=}3GWwOccs3w9MIAIlQkFqRUyheX?rq79(nAj-Q8pj=|hw6MFU6vnv$-8kq8}xUMi6O@63WmmpDW9M3sZSV%Z9lid zEbOzh6ag0Uc6}i*4apUNa(^2#Wy@cWwXvr#mghRLGi5+@v(Z)y4*yr!{^Tz6jKn-s znlK_bDL{?lz|`{=)buG~Bf%dt9z&wn_#xm8hb|J;@b%M=c(Zpp0KQU=OJ6K za)idG-evezz>k+0-D$HtH{~~!&A(~oh;H>kn%6uMi34$vQk2u_(1Abx?(lFgl|q$v zg~qhb)P)wUo~S0RRu+ePzvRMVwb&1a!NQPKaBzqa;LdLTj)E2wP1T|MV~ z?WchH-uSnVT}~EEfO&M)twS{~|GgfsxXVRF)Kj7FlkM_2Os8&uQFRRqaajA_wR*bn z!AUD`3Onv*Q;qt!xF6ldGy3j0T%tay%>Cr{i|6+S+J@sK>RU}Ev(qn$ytcbGZtfee zV%)A_1YB&QBV+DjddDyLL$H_5jqwGhw)KJQu7`@%WOkMN`s*{uA@}v3Yc)MLKPDy} zDyus9q%dylUty;kd!v6iocw0;pfZ1EPU7Mkt<0cNokcTb0EcXXf-g`yH7ZRUd$Jaq z`4KgcKCZqn`0$aJ6ehh?9CExhmpAkvpTsLRcsorZm(-p zwGVuJJQ23^G(NgUcG8}IWo?Fu-+Zf7PeaWcM8%xy*Dm+&r7@`aG+e`Q&l-OtXP`x< zHV^Xj0LT?35-fD02U<(!h9S3W6lv@r`DG4yyN@NB>i`CAqg91wlCZ4=z*cZXI|2?3 z7S#$6l!&AzkmR-MYU1XNpz%Wuzu`g%pH1GQedf(m`T2L#hQtsj?f26bZFe?Tk2(9%&smr7`j)=Ey* zL1N~tx81r(-3KuioAP?=*|7Bm&k{S`Q)}$&3eRTZRy#y#_W{$0%eXUNLu$ed6q4~< z0|~`+U6DOZyi2H$2y1tc3U;m(*z)mqZUmjHEg*jJ=)vrkOs?5#Ohg zIzm*hc0_b2&h^ClRe%{OUdA#O!VgUY)q#31)RamGDD^owD_OiL1NbuV|2AsY%S1~u zhz0!tcIgFV8XDvQR60n(HOs_AoXXdcJ#SCgsNHR~8&E#TaJ@i;2mS5A{17lPkqkT6 z`UHC}hcTz5kywr-a{IpMENbDx-A!MXQcT%gqYxyog$j!qVi77#W>2&Tt0B{OYrxxu z2q7%=Np^rY5D~OJD`t2?dWHmS3fIpR0Irv(`m#UV-91ak#XsR!nE&YYwXSgy;P~Z) z?OKBQ9*Zz1G6_VMWr{@;X_U!;xxQEyB7mj9Jggcd!yMN#-Z9oTjPfV1kC%qii*<3S6(d$RzZv4!(zt$f92M>%_7+0`V- zU1JZ?J^k+8TQl%b{erN+`T>m5+6usee35Ag$n@sNaafAB9jO+J%mN~2w&dDp0e>7j z>OK`1ji3h_`m^jSHG-E7sLCfFSE?eoqg_p;slXLeH>FjV)%+A1FI&mG$W=$ck4N0a zSw~2Ko;aZt_2PxHX|fJ@b`60(BF=T6;MgPaT=g%ZRnwvIvY*GbN~h%`2cCoubxW7; z2TFd4|ND=Boj;JAe7-<@rfI;XJux}@SQT|WKGL`6EJ+;^9Mc*~cbLj6XO5hbUXAS_ zg+A-_clg=dGe`=j_*5a6RHaUk^W-x2)>LQc=_qZ1gOGSsJ zw<_blXuVzN`|Mv6|9N=p=eow**GlfkvS-{r&)dgy4{-%r;rM9gtZY3oIT3mwtW{SD z-qk*}$p?T*x&7mYANM!>=o+MI6unukJyo$fz4pyi&Ck7YVMPwl{|B;uA&#w>a!xe! z`D{&0o`Y>GC(wG$`gZ#sW#d!iw~?=!dq#3wFehBv-!A)=B40HhipSkI*~!7k;{5+?Hs6BM#6na1Yexl-OxwTK7_n?vyhm zEm~+^1njb6-7*fP{1v8F5oQcR#dfv2A5@(n?KHl5!?j|=0M~{AMVdAJjtEE0m<>J) zmQJ%~6t8Hi6<~(WN3JUXDITzJ{MYHaD_qgG3JBW{Lv(vVES0jU0!LK?z=;df2Yue^FNW%jn=FNEu;T?*g2ibZ)OIwTF0?hCh%62MC9Y%P02-&@qSsSUT zhG_l(M>m6DqFdg0&ge-IMPs%#Z?5%{Ux3Z=A)lv?M|zlX3lL*G#D^ zZWCzarHnxmVc zRHMPLZt)1m0}YdX;MdfZ7jliOG~>H`p!Ec%#Q>ic1b!`+unn<{iRLntr3X4$bXqQa zWglU!-rv+a`*S-D-=M{pph24K13*U3o3JN{xp?PF(AI-GaOX0_IJAi29uAj7%*mF2 zE68+t?rdUclUBQXr0gv9W+M2W98t^yA!hpy~~)% zn%~1lVDp>)Z~_;1Fte{0;_^dpf8LI~Mj1%KX?Xx`3C^V!aK*dW*Sv$#=L^@%J&<6$ z>mlz}YYuUN;bugL9@H2J!&uXzm3gi-XKsT=Fw>xh7nFCS-E(0qA*mZ!IXyu1P6Gh+ zLn635r7~-)1h-aUo<_>bW-x;#nX8g$5>)#O)EtAZh!w@e5NNJQST%6z@gvVI+*ZZ^4`m(@I^)$yauC*+Kt|h=t(pQ#*>$mZ0 zYx6vrmARI{5DYbjYW0+@W3i)hOAm4mFBy+O8VjHkp@!ZHi&80-utDUA&j3T76tUQ% z$#oTOY*36pi_TKd{wRT{-X&bKiT}^~uP@eKHsDylRu&)o{9i-fmJ1csl0Hcxr!Q-aOiIr-46RSR!ie+9U?@RkNh#4HOvydBJ_?OO@o2TU1 zoAGX2Yrm!aGXGBYFvX$4bjT?HpQ*;cp~9&U1VclAcz%oYh)XFbj*>c_6I$<^?V$t@VTpU=Iz)Ha7>jy1dQ0 z(eBa}&2PouIFJW+!N#+s{;VeaC0PrlsXU=J@Y>EJ=YBkT|J5(&s*88W=7z?L^4WH+ z>Ps|FlLRXs+ehC11B`If6ssH4_GXi_D+j+sQ*4h=^qU4UzOMslB2*zkc0iW9@W94S z7mwWL4Kb7i16X(cY%OAue#+h=m=MZLLRpeX4Ok?MNO2sIWdOQsV+&;?)t_Zx250C> zG8PYy8aE>?p3_eD0^&|37HM%p>TFoXut<-wTtdeZOGN*subO-3n8GBq@o;k%2WuPw8sRxKG39Q*=h)e5}Zt8$;*{;hiR6 zlwtUXw<_-uI!s9OvYfEpAh0(spDu7(P9n6s3;oPIE7+1fYXvIrGzmv7rO=FesgoI4 z-Fr!daU08OiFv*{%nA^#)c%=kr$;=a-O`4P)riOW7mcP02;=qfvN?B;2^@dogUnK( z4OE!jMV9Muym4l_)H@QbpK11HE*$m~wkcP^f&X~7@pw5)dijYg3`v((BetGXw%noD z(Yw>zqngJA+rQel_z%-JHwL80a2c7Q@HUBs*b5!_5}#g6?Bz4B~2hlnc;#B4b}=AhfC;_Pb_)CStZIlQ&UuO0Dk&(Bo<=%POGSY!WQ@cCJ9ZBml0_Ei3S5OdhUPyT>?0 z1LX^P-U#q`mSvC-|EuDpg zrU-v}TA*1-F`p|HWZCJ)Fv;j#3$VHD?UL8UmfTdXu~sP zCEdCnd!h(q0Arq&TeWs)u~crS8SDTM#HMqkZjs85D37u6oc;QmZzC0W(CdzUS~*+D z9tS9i?Z`n+29$Fx`cM^ob;eO_7g z^%IBc3%L&ieMRc$$KtOCR~`q*uR7ZO*Sk-Z%{7u~Al^m^ zgP$-P0Cj?Mt?Qbt^v@;EsMLcsH>VpeTq2p_}u+-(<#4({%U$v@d(!> zZFBO*Cb`}9Q~!kgqM$|{5N7+RH#5|c9oJ@))MA^|_EDYlj_<~zjxXD%rgzKc{?6wX zMb~yk+aK^?FMF^L@I6R*A#hRIDDc9?3ZbhpMZXCi@s9cwk6&xigv63LRl}jB_5$|< zjU%=ZduKte$tk=5ApgtmObijcy z95*hlRS}<a=L9#x)8aFiY#_$prJ!=k0*LM`V0*Z)wC2@j4HKOdLIR&gFtGG853lz)c(Qim z{W5zl^$Y@#8Lo^1y*o#j1&|DIW4~w)tLQAjPL;Sdccz`W%eS$|i&yaWm1UkZ4-yJY zx8giVCSZ#|B48|f^G#JPxZVI}P*Ps6iIFpxA*ix5e_Lq_Pt(VNO`#ezYeyo%ogORe zNibHnua*Wb&MbP6v52B*|3<;Ld9BB&xp=8-S21N1)Kb#m#tBk{h$9{9UiXEY737Ua zv2$bJh;Sl7X(9K#N2uo}(lQT<`XruFj)hL^>g`Rk(_I+)9gSSH-A>7*0(1@l#<02# z1v~w5M$cMwyPa|7G|f{7)`iJk3Uf7b>r*IKkj`hgd%?C(T5XLLI&nrSW|-h{A()`6 z@(1RYWD^NHI!qfmMBl6_p=T>PSAjeX^#8#k_NzalMgm<(pqJwiTHQ;=tdM5NSfFAE zzTb_%cr7)W^+fWwukZswBl%||c=U})Yz2=y#}mTP+}Rwj_2!{Nc9s$X>J>UJ{7N06 zfrepY$#|(ROXcgO%k`H!gB=dKsf2HyCT0#-Dd<|`ax;jzJ_#K$!{=8_Cp>v` z>uP(5#1KNG(XB61wqFpLujt&I(YZS>@hy2R)w6aUovl>ohMuhG4`0?< zjDGu)j6#<@6WegT7WP2B_QCasSsTYFx=f+`!=z1*5^Yeaw&MQ<0hUBAz&+5_epbL- zy*y+Z9yPtY@7r6Wz`VI)g;`5N%fjiHJDZm&=7&2NJ$at1%R&6!V^*jpm1!4O-aYIK zUw=%4ZI7_wlGivwfM2nUsPMnm`=hM++x_ z72i&O3aYb#Z)5|Qbyf9=i}!YZihBAz=3c=0Fk;g1^si6WhF-=`Ke>2&I_|&M=>Nwj zKlHWR?Y_mqGdU#QKcgPd{|g+` zz7SkWJ&{zG6!HcA6sKY3G)VMv-kyu*VtcwWJ#EmwPINWmk>TXJcUU?CEydjo9%yp( zEug+^?Vj_6ojR9_TDSqbANoTA<})i)IMyw3;{DWo#~kFr)J)AB2*_}OQ<-=^tp!2* zq2=hFxQ{=Ipx}dS>=~#=bkjs`f_S@LSD)Zgpg7?dnU<9n-Ogz3Y(B{kJ+YkEG;)o; z9<#P`tUN*CUh=l9x%OZ0CVLR%49HGm&Sn1bOR0o)|xIJp!q6dBV_UQ z5%tFcd{j6ts`^4vpwES)0X~!e|0iy}L(BfpIYV20=UqixpqN@R(P2v4#fF-DJ zAhln9=pDt7N#j5Kc#<(O5d6?+UGZZixA^X4Ak@ zu)%iVrTfRIxmRRCO>S)gk~PkcrEdT&_^k6UXr8izAC{rbH#QKZ6f94p9pVCh1zy|9 zcgh+Ep`zshqgu48u9{9r?!2XaEo}6sn?~pg9&a{0afD#5CA>H;0N8M)fF=NX1?>W` zQk+g7g?Irdr1BmdYa9YDV&tXhj^>6**EZH=-qO@E`b zz!pS1@xF>~`F*~eKRP9}-O+VtlN@PW5^evt0`O&7YO=kT$=|YZ(3QEkupBJxTyEaM zhPW7qMMn7$Ba@)jWe0g!6V^P;8;`*-aSebUq>Sw#`~nmq7ucZ;g$n$eE3Ff?y=?yc zw8jI+i%!lM293v9gGw?L9r;TkH>G#l3{-6Q;>5o@m^Xqg^AB>8SZp?fyU()q35ldM zIE4pBj+S*VkL$pKw|W?$prMG46>=-Z@_>S4N6XfhJVu5nB7@&S56G#0I*}ZfMceC&)=4Wt_2Dz(^dXm*59WBA0ZaXkj>l&^PDFKp?0(GCoNygix*0KkisZQfq09>B%ujrCHm5YWea(BLjPF+6SgC?TD1iw4rk35an% z$ghv8j3yB_Rz`nyiSmFLqO1rDI647q9`_9w+48#wZqrdY9`^e3iP~|hrfP&e}8q+)uREvs`=HmA8uGj&5@g zF9*6`t_EDn`f6#{i*ZzhNK2gze4BW3-=qWez^Ut3sd04y4_1=^>*9wW+hcw{k@P8* zTrce&!+I1ecln)YV?1oU-%gWizIIue7`U{q>RheuA12{Q$aGTLfz!wnZ8IlGHn*b@ z%Qi|*zSr9-yR52*DXF_>FAbnb({{m0p;)jGf60EFEvQn*y8|1Nd$>Guu414)yfL^T z#j`EtP1{EU|EKsDAAj25w4@yc5cm6Qo1{&h$?0p83p366Vs?8wEha24IU%k6q+&(c z)F979r9oYM?mGJ9IaT?gZn0a2LnT3`f59)X3h>w97hO9)D+m}{I}~;Ghw3y*AnL{S zwYrF-5tmARR+8;dO?8bG_b2sU`Y&6XDxQQ~hbvt5s~=R$Dka{FqC}Wpj#!(z8(Nu~ zKj`j-|Ll}5?|HHIhr=;}r%5sU*oW=*a?Gv$CXqI30$FGSzb(^Cd8S_7dZfTI(5h@87q%A5db0u}j zfO~_5>z~t%vk=of1;AWDy)Fl493yth3OsV+)Qyn_9R1N~hlw0_mSHk<16<*hD9dP# zU*|J89^dbutEAQM$inN&E7A4%{hi}yHy#+1qZys%}u*`pW$j@3* zDdvsb*{VqG6Z5*Oh1srOgxj#`1Nnslv3Wz|%^Z&=YqseKO9Wz*LRQg9u1)xPZiSt7 zx>>VASlQ$1^@r^vg1Oi7ZN7SDaGyVJ%Tu7)q~R68*w_1#%OhN2YtHmqxVj+0E-+Xi zuV&f06swOQOb_BxR13&SOD_ZD13?v)hz;j=fx~Kh3GOeR)n3^-Wnq3;iW=d%a$lf} zLM9}1DAzkPw`N*wIgoxZ{(gO${9J4eyhQ!_x(LHD`vST97wp`J`LccfS>qELjRsNJOkX0RTLEILgp>xoe(Cu+UtHNiT!NGeum?y~oqm+)H`T`Hg?GMtt5?bS&Sz3@*2&^@Mb@5DTsj?Fd_o)B2y#X|>oKZ3ZWOKEvY| z^^0{foHsqa3yKyvOW8)Am6wI>0Y^9cJvZDG=ec0@pIw_2t`9Z_8TpdtU*y z`94nW zr4h2v0)Kzx@LToXua5dS9y#{uQJ-rUAD{arF~lb~@V?Lf^WUHQ>f(jp9iIPQ{As{1 zUspCzFHDBUAEjPuyyaRZ$$b{oG~p=tpWl^+;rdep9X}ojJDE6nt65N9nvg*EaEVSy zOv$S0hq<`Y8!rlYbzwFFP=02NA#M3??am_M>H)S35$GzXm#3$rV$<3&S%wRkja}TA zIb}DZYigT!c~7j}DVM0nCTrp_Z3wy5P}Ni>6=CaWm2PEa1zhzd4Y$Lr_8s`sCvR?i zqDKhMk=Ut9)!nV0vX0BO$aVjSRyQ1HI=3*)gHHlKNXq_!sP^WQ4H2q{yQhIoF1Bc- zy4^iU66D+(?$GMAw%_yz{%*+hL8tSQz}tb30&E$dAO6z8;fK;c{uz4o`hTmA*0>*k z6G7p52S?wKzLdsPONT03o=(a9Y$60No@tw}J5!!n>nktjhU79YSl>SIBJg4W@MQRu zUz|LbyLv7n_se5@s285>e{tmJZws7UMpe-E=B`_og@_$XLSONgD}IR%JXw(}C#wQT zgTFBc3)!sOAxLA}5>#{oeyht{Z4*W@4Qi7{_kw zVUFkaV=KB?f(CVlY`iHJucQ&wXmC$rzIyxGcHe&P1=bDL2i6~t8yOnwx@E0YR* zIHLpuGlwy5?2p7wM@BYbsXAj1=3T_6G2XuIdC8q|$`QJic@dgikt|2z*9SdpO30}x z3I6QSyTnXJ5W%F?;$Pb{RB)JSZr-*8U?CM}yH{TOvX*_x4DtE|gb;g>e}MC#6a zQ0`xr$;f@0Y;UhTcH!&|Xsr@(7%(d!Tpm%4LQ<_%lx5ywxdTjS0OQ@9||m?j5W zc}%WEAXi~|{|V7x5ZBT)Rt07w)Ux>g zeRp6)2yA_fJGTkUp!7TDf?5#(xSMymMi78i0$g?Bl|b}|18=o_K0v1R)dvByifUS+ zNv5dQz!Q=cOOrxS_VvdwmWwBdxN7ULMq;WP%nfFoh;+jW26D$SmOCp7JRlF53_EWl z-&XavI10KduyWs;uv)QV<;M#ER&>#YgH&0&7t zcIB))IZ^4_PJ-LB8Q61%VFbj3Ma6C03?)q{qUZkWb{HONQ zt<*$GK& zppt2xV*9aUZU;oSHiV78tlQajHCPfKvOngc^52NbV9uh~r1!PX2cJ&*`0pRTaNnVK zq0e}vohSLa0wyRZ?nSX7ZJo^wtobM#nKcWFr6>J$4e%`ezGQ|X4rqEt3F^m0e6_uK zNd$pomW6pGh&n#BGQgtddC z9jG+rwx>VCw9e3)#uUaed9?wdnL@DNMjD=owr3;tQy(4xOK(W8hp|SnEaeb$;5cK_ zmoW^^v5}spX?xhb0-`-e*?J2I@04b{T3>H54JXqk6=$fxe06$y`Y@=@3XQuW4Yxpr zzBIwlYkPS|W3hLwpB-J%L?_5f@HoZmfP23aB*9otwl5uOYN4nCp_bpYFw@|Q!js;4 zHjnvo_jEd+C$S1n!riG?w804T)d(FJK0FQUY7Z5n6{XR&O}KZL(5K!ydjt3*1zj07 zr!T5y0YwsM{Z2QSIh62@f6hbdRfxwv{H-G28~QFK4afS>ckLLY^2`WgB_M{ zeomn(GZou2y&g4KBnyj8a}iL@GFJD-Q*Ur9Vn`OzqcPhZjjU}lRwPy0b#`Y+M)7wP z^CVnY^DAWLY`LJtC}nkNkJzOxjN9%2BE{D_wyeDKyUrew=k2F*rs^#+eR&7dP>=v= zL+QV>4>n*-`FUJCvBdnCVx~s3-;x;O2F4(r(O+R)gJ*5iajL4wt;hbEZ^c%`=`l3g zOSTRG@x3`;p`nO1eG84{^8gHzC)ehZOC9Ci;%Ppp^aT(xeYyMC;LxE(Yw01LZgX(Z z2{6y~m*AfS8w*7Slem^x&JfG-(+UKv%Rn|%4x}m&glK6$aNfF1GqoPw zmK;kD#nKa1lVVG^wn}?6J_^#oqOB!6<(kO!7ZETx#fml@MGNoOJF`Y|K7TgxX^j-wb>l7(pX(i|{QdL*!DQ_Z zkadO8Lk}Hn)PsPxksDNN2TsDCpGl?24l@r}0G4eS$uc@%9mBW5^23{U^3M1Uk*3PzFC9K{#G!-A8)QNxwXOn7gRUWzE zm*4mIBj5h_kB`so`TgqSYn5*I_k8-%y?=dwZ{Nkq!2AT6Pf?{`;L(eF|NcDj%ESN7 zhPuT5B{hDetmfK1o3^#@o^^$zPO7RM_3Z*`zLzWDLwv{szksgA0B;4mmfb1mR7|bsy)O7-Hewthq%(E zd62g5=}ASkkpW%Z*W35mrvaZkAF!#8ya3l`1t}fd3ldg}(woNWZS-PpUn$AOuPhY2 zyQr;AoM@^k{m8N3#`fQL2r)$E2iTz6Zd3=i_&D17CcHgW+VkZ^ zLdk<~VBZw}y!W?Pr!I>>zW9A==DGHuBY%FfO22t6X+*l|=kcW6k3Yq~f6eI-4e_+Y0LMxb~}3-KBSVFfxO!j0ZngO84^gk@F@0qnR3dArsh20l}` zsFXXJz@zvfs|+^q3^Z~pW;2^hvbH#glHZvfD$AA({F0KXS> zGa#0Eh3CSO-Q$K$f!C_QzrxG>K4N8hvrzMCl6AW4+VSHjG2@fzg-Gou6vm%CaW29Dy;fgD^qM$mT zlv}z!IOgrZrcOX>0>}mJ`_=IT7Pn#L6+uqUF2OeBkOeG5weWqvd7Y0ZD0{lc?7PQ@ zx@YqN-j_f%Ok!iBb@dR|9l?N6nCGg`+A*x^yqSR(q^u>%;9} z{P+^^&mrci!w}`GCJIWJ-+(&bDA{g~9_^47{C-Vsg=x-Vj!Ri{vwZo9Rx_Ya_;bCF z9}pbxemexVFxGGJ?co+lg>jH*S=CY=RGH3u%Aw{hBza>y$itqPl$e~H&Mv4pwgws< z0FH0!NNDO@cEy`q-TU)EkjzfxWlmS^73*z&{J2pYHcIjyE0NxsVmzc zU~t^7X*bcy7K|wdb3n;ip0T!%eJjJ*h(=&d^|dhF2#Ml}S7t*3twyg*cUPP(sr85O z!w^}sw2bN;VT)-mG+hZX4@8=W73J9c62@k^*LFTan-nP<*mr?t@PHc{SejUL(+v+$ z+{_yt+6C36QnCO9$W9v@3;yBu{@E}P2$v%*PdL3TDOU*6eR7i?YLuik@CU0-+xzGz z%jy0iVHfx#=CwQ8b1F8nb8~s-*~_V?XO~|gtNR~R(?!xZ zM_XA&o zVNZW!#<7#Av1LT@Z>`VrcrgqYEGPE%SOZ9rjX5r(npY3B@*HDqNO}FsQGmY|M*2Wt zi)x=KLxPm7jME5M#n`k27XbZ9a*SzhnJsk?yM+R=?^y2zn^Xq*hBT$r^hZcc>DxI} z;cSe=HL5x2fQKNlkRzsw{w=*W3ple;dlsUvEhnF!^#3~fYek8FBT-&!&n@@_? zLB%@e|F!$1hEU&6p7`t!a+o?+{4Dy~AFm4rA5Z_3Km8Bt9~Xap_U^(D4$o(=obwrU z>j}8}`_+3t6z+ZX*wLp$D)qCri5g?4XH>t{7wY}H)W&MlQf9aIAeu2oVsY-@ zPPT=cRkd(2_#|;fDvoFgut|XGhJcDqUqFI>L;=LZMdY=qM{fL$8%YpE8mE9ksHj$e z5;1H%*iDT2Vu3-PX$+F zb}L+OL(zi2?*+va3dk=6ixokGw!&?zDcVfr);_rEJO%vT*KOzytVYO3tOL3Q0FVD9 zqY-UAbgEh}6IJ+r7bHc@@yF>Kk>+WIYMaGisMH0qbIQFGbypaPPlM~o`mZ$=^Cap~69NKr658<2A!}<0Q;HR{-!nX~F9VUj+ zWv*XIN0&23d z?0{skIcQ=AgpjSVd9iaza-*T&)aFD;i^K4<``7F2GuKetuVy? zL}dy=J=tD?7G`Q0Xj@O?Ex~ctV5!L1SS#GxXmL;9SPUCAOOQfCoWd02I@eauF}2cH z%tmYSb`A@7Lz?gFZb?PzXFNuPbQsR~1hMj6>(X6w$_kRHyY1&?1Y*GTQqb+NvLEdR zSJhKJ&L+ka{i~9!MO=x-vP?9C52&R7(v9rk{6t4mZDQln>d_x!4^U5h9iK1*PNObA zPml*(c2B&r3m*D#Y#bXKn@{beVdEHww9tr9$6)bfP0FEy8%e`)WglOea4QI^H=St; zb6JmzcNpJwtFZLz^=tRe1yo+RckcPaulMf>@VU2_cjoec{^|3$XX3AJHeY==bond( zgNHx8DNOsVW%=)Rz0r?7@~I%|n!HC2FbXIo22MgNnIvtslgk0+EFj&eajpuwuDfD1 zCPKE{I{OmP(lEhvjH}c-Z|}(ppi&f*zTSWNOxM`S{_e3_&{6U$k1Y?PIg7>yn!9k7 ze0OKQdw`375l6fSZlIImCzo&uFWyg@n$gKdJ?Jh47Lu2TdLq^+IoRxc6GA!^w8#NL0CoV)+7p? zT!bWu>OI^bh9B!6T3YKX1<%m|72qAZmE^yz472E=?&4$0zy{g3cPQ6vI7k zgY=MPYH%N0w8|0wgu1gS%+oTr^B6M^A*NGM&0~nVSx^bTDZ_+{w%)P32c!&*B!Na) zDIsXtghg*BvWIKil!3v`>FyyhQfz`pGu9gI+uITHCz1HrSY-iUj^m@Vq=5EE^B~Nx zLyYDM6OhNv&ZdEQk1D!}Wge!OUL(uTqOzna%@x;jubIe*4uEHTJuftykrp20P-C%n zE=?L~5kw;9_&DP}%Y$SX;OXu(l#sVq2%Gm<*sVqyVul}PccQk|IHeF^_=Ptb#v)+b z{S=mTaHSxEj!~X(GAKnX^@E&b#$=f1LQPI$n<~g`E(pfgdM|PGN_%1YP%qL*hcnf} zyrF2K8}3yyz`{H*@Nl$v!?VowF%Lc_lfgChmRrXdlu9BW02N$alHD6VGC64u#%b9e0EoMs3(4{ZKM3JggDe_m;Ox!BL$-2o=HbYqDO=Adn_p7Q znL$U=gXcR8M_Ifga;A`5##RiyuLEhiE zPms{9IiC&{2nsaM7Ur#R?}7kGZEFnZ($FhOqZy^a^kuMjZ`T8Ch>5y0Oe*uRn91G| ze0i2}*48$;JmiF3<`x+{>ba>Px45OeD3Wo~jq_HZElq;B&|{>}!9?PVR;itGqqjHg z^u2G22BwbHd`oE#So^HREA#V~{69W_Ju>|%iU9&u9`5dm6K$cC0*w`W=}sFfQG6zU zBGc1W2BFpi_t4S@X-?4c8NZ-{z|SUr`hp#BB(de-{)@A(dWr&@$8%q|`~0Wsz~8_4 z--TQkzv4t<@2@T{zm*46-u?Wc!^6bV`Z&+Q{)3OJFK{oGPvv7+t3!z=!-m(ggrlxt zia>Jg>W^~aKjj#jz>7!`1wrfN#~lJY^qY_@aBHfBCeug&K@DF& zN>F>OjHDS(?@ovQz^8fv&iQkI?nTTyrvnwS2Wykgg>UtsVLCtVJlVsFykX^!d&6P- z1VhxPh=*!7e{#-z(WnE-ccfOX*zTezdEok*fRbj0I4!%e7uhacCiq-)pz^H14TcudvgmLe+cGhaxgsxwt~`QPIOe z3MXT$5U|2uR|ELrt<*dOF#{~6bn|n2zeueQW#g1R<*dn0VVHJ>uRph;7IDma8=!mP zl=71q%0;Rx%=nm?Iaoqi0x5SXFUTJ38#W**iqO>i6GEfE45~%KQs(8pKjc5s4RqJO z@9G?*#Q~=y1l~C{07HQ~iXj#zmP`F?JxuS-Jntld_74xxDJ5^+t5qP-EiH0nnJr-f z>SYoV+z7qdO=J4;yBR6c;dx-pG&N^zG>poBDnjuwgF$)E3F8n^2z2uvT-*btxu-W0 zq}J2kXwd3<^0ripnm@Ri2G88yg5r&__^ixA?!!Fx116a%zC6TcVi<v(?Vxqwwm{aQ7mi7ZrsA7n-N`YnT=e+EO;<)r@B;(x}aUS-qNKpU}|}i@`YBb zSuw4DE}E~_Dg=>~zdc{d1Pwgq@ZpNBMLu90b2IrR9fpnd5skBD!<#@beizHTX=mQc z72(*Jm*qiW7e0%&i3VH{Gl#_B*s!*T;F>vGt(OVJNX!s3cwK#S8s^*iAoB{eoqQSE z&Lic~%j~=QVM%aiesJbMp?O{thGdD``J;eX1Q2Q<)$BS{)!RSNotH5#MiSJnnT*mH zf6VJbB<9_foE$88ASQP&&F{!Dc+{>?!vxw%56c>q{B6O+bng8UX~_TO=*+{CPWS$Q z&Uxk>rbGNlTA_6IHh(L<$@a1>+yZWQ6>*_MEgq!>O`Mlq+*TOZdnTnUyfgPk?3hy)| zGgmQeH&ZFW%!M&tJMAE>g(9*CgS+BVM|Amy<+K?FtumW!8|un-l4EC~e&)#D{fq6M z;g~4;LT@4bD&u)5VUACzK{X-`jf8W(s7E_aWO5z6$Y{JnN{<#gWieCNmDu0a9VA`0hzf0u zC`27;^hJQ_?9cu4TYBBU5`Vvo3TWUSvdAtsyS$gTZcn_Rocc%2w*Lkk-tgV`Zvvj5 zb^K}ao6Y;ZDW9MHeCoL4ziy=Ns;e0ZNp0KcpK|+XgYp^v2!F+o`7mjH85U#RhW7yv z=)9^k72{WfyFrzTdzG8N6d&xz#n{#d#y8dcj1!&x`Pt_Ah0u)df`{mMMiZ|j#`B0s zzF-|K&7@{E@~6v_k~AdG^wfsX#EksG7n9BTvXI3@TjXIk;EYEla(h3SkQ1V4q@?Q! z2_2J5Zf@NLvMhLz(?Qp&E5lXyk6&t>*h%8P^>97j($sq7iSL+Akn;W1yL)dRo_jEn z;+=K8sQT)LZ~mFmQNQErmfzg|`{euIvVT8cbmHopnv$$LjKGuOtB03f(QQw=hMnr& zS9N#KxyL_WTF!0G|6|(9=~?XKtM4B?+@7A{kp7!v&Hnm}PN{d#XGC|@jGBfExM2jb zZeCZtL*q?nDmRouOO|-43w}n^f}L!I;U9$VV0j~OURH`EOB(Q3tuDbJJ{&t6+G8M@ zL1RT;f#_aOX)&Ecul-YD;sQD#!R$-OE@p|0I*|rlYD^Bs)`w?^D#SN@`*nIGz8Z#w4yLtJ*MMG4?? z*bp>~7A4Qltp>Re2Ay26vlBFzitMp^Q%~CR>qcuf-y9z%>dGGR_7&SBJ53#)1TmW1 zqED%_Tbp9ne)NyM%-#_Se9e}+A|x_A)A--2p2>zm^IXeiwCWjY5!OVzmFE*iFtGdU zjBD;hdTjYz6MSu96gDOciUxO2k4ZZCWO4BpzjA_kY2^-KS}{}KCer7Ub3X1x0j_S& zD`Br6F{`T-fW%%+_6@ZaR1`$ICY4WR>$jRwiVxSlojH{v=6tPG<6=W!=J(95y?T*h z@}rNObe?YOKTbKz&geMRWvE{)bhhhR;rFc6G&S?{iW5VoUu%cm`5)2|2I9ixKkf#pX|64eUA-+C!WX zmF{@8QND1EmS4clQc|^GS2qYAzQ;T_p)t|`dpKSkWtYAx$5v`B5t*|9z8Op~6a!7@ zMC^1H-u2$;6LuRFDh5(DN6-rLH7lU=&m6vkk5W_$66~=n1@O5`98p&f-mt{Q&#%H) z^D#~G2)SK>NF_$(3`9}wl(T$GW{uBUfj#8FgzkalOKa&+3&=#B9w!@nk8KJTSNErL z!%~&`-nF6=$!p&r1OgEs`#Q>Qd$jHe+r(0FWnroPs`;noCvg3rdfIUH1N&1pPyN!H z(rOwgXB=xv7{w2}r)JV6j=z`duR2ga&wp~_-lJAlRefMeh236@JbFKQ`s7 zQoIYI5_lv1)zxXghmxj_U-;ws+r*}tR^ZhP2#VSNeWUNmgeJ;{`hvZE8MSYZXZYPb zmh^%Gx@K{eiT6(KOMR8$_HABC)1`DuTDUAmkaO8k08|g51BTO}AuMPlkB?|fjn5{g z#z(e~hxl<--~Ct|`22?pU)VoI#^eNyeB86Y<$XHMgcDEjrE*$QNVr(XPni30DQ ztx?i^u@;-7+cNlkYhdYmeMcrOB!hrC_Dw7Uz`t4(=tPGgnv`AQa=#oSiL$VLtko27|@Eg%fR|98syVBZ9+IfDlW0ik__rx1%5uukpwN3>OI@amgDrJu{ck6kDg4o2zhS+P}_G) zmb4OC6o_=@w-`Spps;;w1)|wog0|8fTpJg*YLq!CTx|+akm;F@efc$vwNOzf!<{it zvI9275(_{O^!9=eAlF*)cQ8Cq&ti!!?sws8;pTg}Vr~XTh|OH?OQ_VwE=H_HBf+h2 z1sz{dpj((pp?ke+(^L>t+$_2ST&{pCL2YY>*a>hxiWVPZ8+@5VSOohSn+jkFrDMnf zVL1}RDgZQ?Jma9f7wTbcB5Sz&(r~Y_MyMa(#Az-};)I0SRUqxBv*M05Ja1v0weC5wjzjUy2$WNbh z_WS#n;41Zn_zcR})gYLAZOHiGt6}}=3%?Z{aE*5iD7ybbQuW)G3pIZgo!)l0YEzVT z-s80N^mdzx>fY?G#L?saC{z^bMdWnI$@rj=!>2pa03`YL!`dHH5|VcIxkY>M-MgYY z{^Rc8P}byM^aU@ZV_@QWU40}1Y}XPTZ$&xd>d_18%hL%}4U_7pfb!xn(RwJ7*_ERHafcbf1>#pH@@_U)=i5_G0zi&yJ%} zM_q{ri(K2o{3EC~wKgLgPaNC6`)c6Vfj@t7`yWp!=b9I>`LZ&4W@M>^gl z-n7~ITb58LPHl?GmK(0)Zy#1 zuv-~6&~hS+-=taC+*Z&F)zDKH(S&g>fzmxLFwOBRjToX~9;xLCQZb-B;H8@tUi zCVk3V%R=f*K#cgdxyHZRPty*3F8-o5e>A{M!F6rSkKnt5NTfDCKpg-LA$o|+Oj``c zc+JNlKvB$Of&)VU;Q=>$T%?^x!oxu9{|TFVkFV&F9&uTHiscBf?Ga>}W{hpBL)TK?bN(Blu$C5edh<=He`~{e%v%Z7S%Ic=kXwg4{YY zuc)+v4rnBTBFKogQ%#m;uN9$i@Auljz^wp2c=p?1TyH2@|Ab$;Zdr%&!s|>|1#n3L(r?vgoft}=^eBILguUGM@cAJb+}07&@8|%zg+GE z4N!~?`22&l&^v&_KsfE_)7*xU_ueg1y@tZAr0TxC;QIzy42Q%)%%&1x>B83gEjlMA z+8HEdZH=eG-JB)T@W{qyGGkcMOyekI5szx)u5|D#li3wi(UC^=wQ@H%YC!pvvuS0) z*E>g3+1AfT5lonzyknM(1rVtKK>31-7hb92zS-BnlCGK?9ja_0iRl`d_sRoQ77yda z2($HGj}EYH)-+@9B-opFxu_oc!rl5WP`ip)LCrjHLzSB5L0WpIlwra$2cHHv&3+P~wE4mlX7rY=yW>;mr zUW0RhQ><%*DP5?ZsPkFX-f3cH8H1X;pn-+e3Jdt$jQBYQ>Yy1yx)1PDyqi(k?^UHk zc9qj%)9Q5^$6GcOoTY%7#?eMzDx%x)3g>J7$Stj0ol9*oT_#o_8|Lphki9ijCdME@ z*y6RJL&B7B+#kUnfKHM9{*O92YDft*fS|UBH8Y}dBXop>(>MqDi_{cgAH>hqYiv!u z_QEXwC>hT=h64A`p>hDk4Pxq!0eD(VryvQ;ewnWiF_sgO$XUJBDsgpl7`y7>cvkS< z_>!NtMc+G^OS_Q%O$F_0lAZplKXLQ3&0jqDX_AsQN!hXOr~KZ=hJTOsC7%3GG@aT` ziP$s>CZ{BzVCxBMu1f}DuMUugii=MwKPm`GX!0K|=7HZR10@+T38TeLJiCd6vl%gN zo*PHMuMO>gn@NfIzfYG}XKq7#aDUBr{K;_1LVf{5ov(|zAfcTWaS|kc^?x^$uikz7 z;r5&Jx#!-NCsGr)`J=?{9)d?s8^sR?=UZL6MhBxhfbLi@xdO}R=JkwKqRvo%ku;7} z#awg@d>r`MRsB=f+lL=G);7nwwSeURgTO@pYk@njMmj!zd2x^RU{bXI_4uT;=IzI0 z?*ttDal=oaedho8VAZKNf1UgJ?{B}GIeg)}f8TsnQn5SIp`)US=brT3f6tT8-`>A) z_qWk)2a4@Z20Rk{_uJ@=;NZM@_dw_D=RfSW=!mxF-!J%2{Pq4QLcDs56`F>| zAG|8WnkZ~YzJa;s%QTm2DrnMu5zG7SQGku@HS=*gYihO@T<<=VX1^XKrx=1NQ@D_K zsT@Sjt?6QG@3|QmNW7S5ipC7Zy?Y21a9Jh=AP^<$OVd%!_fnni(S zJpgOmC}@QWRXMS(QP}|dIS+Q|Oa4@^-{?Zz0L$K(E>a&Ns15M+=RQ}QLM1!w)e`$~ zsaDF=TKMJ}1UVf{!22Q?^S~=!)RSq_)(NxaMjaKOrwKzeAhkLqy_1R=&9%=Fb%9_) z9bsk`5|ZdDYkRq!FkKUFNzj5Sue#e4g`2Ww@Iv9n5)!mT#KY<&LM+FHJ&wba_q*Zs z4s_O_CH}QPF>{rhV}f-``bI6VG@8|>T&U8MLp<@hyeGEd5Q}FR@|UiDgJ49hbSN0& zN-Ca9H1Xlaf)_hoBHTamp|fYEd*_dG1gSG~3lf1JZL~9|Pi%?ml#gvB_JH&`hMOB9 zpKFx{_^cwFd*o7|rxRa#AhJYwFs>vQZzXc znSEM)OR~O$>EVUzF*lLnnalPzjH!H4N~4`Q4=$v=w`}=%hK*mc*hJO9i271~f@7R>fL@_RSWc1>FE)aX za+;>8Qe{<_s+rc!iD52wtKBf6@r-4~jyzM@R>HcHpnB1QRg%qWnNk`;2=WqJa%=@u zp|@A|GI<5C*U>#`iY|v~Xfc*3m%(5d@c*o)fv#8(szp%Ivews<158Xt71=BYR;1T? z-pfA{K~Fz^y~Y+|fyS^DlhXjs!CEO0lxgd?0%hrIBs2&g=mssjm3`nd*S766L4?yx zW4ZJwB@QD43QY-!7TD{%+}zyi+}hosaWXu4c#*{Hwe}W&gSpoqEwOS@f8p}>IK>eu zb&beQml*fjCo4UBSR|9q1scvw)@&4-(}f>^k139zx|!#*Fd&rkfajNyJ;l3p9-&vx zg!{}tRtn@s9J}w{cm)@lfC==)aQga(QXhYtaz=)@*ADBU@M`ppf(x~!OsXdtmH9_0 z;VWK*X*<$@W#}>X%3YS86%OVZ4Ox93JUiVXXtZ_Sra16WwpwTZVmPh7_hMq#&0VHl zArc$c3;E9XyF)HLO`U0^eBM2E`p*56e^oWOJ}!x;9k|uHp>)g9$mq6Xb#d<;xeYJh zJ`A#EPt?Xe>Kp&ZqrQ6ojmP|}pA3|Yj=hMse$pLb8&Fo3bj5P?_*5A<`Oc3P4>q|L zv9fpe?dBJ&;I7Zu%$Br$`YmmL zAA0t<>R$8u(JPZR+Z?~{JNFO%4^w4l-P>x86>R?JPTmi@9OE57dwl!b-w>b0e0L_{ zQA+f#SLY8Dl?;zK#r^7-_}^NdQ^ouJ6K^MbDzCtrLYpt_oj6c^n%luvk_b91p{=vI zlKD|Xq?wW%l`e$Ei6WRl`^LgMWpRMT5#TvNmC5S|0cMzKZXg&vnMMZ<0Ihwzey~Ec zC>Dut5KK=TL{KNwlsBJ(UIff7(=6I~2kugi(E)$1xfTv0rf>q^yQVazPJv%O763|a z-Dw{&1U0qPS%A?;vNu_BM8*%H!dZb3@LiVk3TFeQi$qY+MpZ22F-#WFf~C1xs?T4{ zK&}C3a6|AG2n_$^O#pBJlfV&RY{8~85nTxehd2z1J zY;kM{d38OMlr`t)v&JRN$2x=0a8cKn`>9kIsz?1AsxjH?-oim~#skkv?+xoln>#|u z$(eDu741Tb4dCvrdos8NhOmJ#49KCDa=Bfj=U{Apq-$8Jab4$WhnX!IMHn04Lz6a^ zo6y;FDg?nO0F@I0QzpVnq9Xt^NqLlb1?-I+AXQE7mn-2t$4NWPAF=&(rZorZA{${{ zgq^0cQ3USYOc?%SL3s}Vyn#`^j&wdC?yhB4Od?B^?R>UWE}od(oPH0V9`JFvXRv;fY-4 znsHVq@Z?kyK%idIJ5;uny^yy!ghr`|pj_f326KaHpt`RktQEtsG#Gzs@>}CExH**h zu?>9K64VaT@f&M>6{(AIx=DihFms~}n3i&6Fnk*qTf~I~BE0~h$psh@50DilBfSD& zuenIeEG*>%X-)Cj%*`}AW6+Su%7EnFF)MQ@=IN&4nz6=ARqhofvaH$u+7K&HZM9lP>A%xs0 z0P`=v7Yjatg_?Djf&a-xMP=$2sb1<~v2o(N$BsV}{ZDqB@qbsmEAZ>;A6={S_fX2a z8Tq?v9q-*NDA-uIODNSlfN@$Vb!%GDZ|nov>-B`hXE{C#6# z;R}!k2*3F0%;ct-gaoyX%>8z#DmuU^)Z>n0#euhv3n(oI zuO`T1Tz@~n(@+{*_gvib)8Jo`6q_~^9N;O~3R1uv!q z-`%!*_xAFG|NiP~t8yfSI-{q|d3FTMnN+juNC+2)Y{FBCStLipRRrs(JhnXm(6m-5Ugp3Nj(ap7!KZ&!UQYW8E!?^n*ypR`K8dq9eYDE-37Qx zM?e6@H?$C}#>o#{)LUHe?F6kAK~-9`cE|j2Y*40~sE=Ai;rgm0R(_;#%cN8l24H&j@6SWaNh%SMJW`pZoN;Qh=37c&`r4s`q+@P-c`{+{R#^9LzLu}Y*JSLEpF zP5zdI^=eJ23=OSW8k)g4v?8{drf&9>dlBTumo=t180jYA6+B@0=lF`9K3Fdlouq7U=#>CJX zu)4~8jDNb|LD{2IBf$*8;^RQu_pQ>Z5S2CH1oK`3n%?BN#C~7qP?=nxA!|tU$x@es z*#W7mN9jmIWR6rIJbTjgle1HUa{q`ZNXi74KLMb)PBszY>+;OOw@s;>H&-%9TwCl2 zSxCEjD;P(#U~2u!@wvfa)7WIiTZFe)$8w>1QzbN4=WRT~=VVUHG}AO|t*dZOmEwNI zZj+1ooeK>$D!@#8$jQ=J)rad@YN283A@Ivh@r0Rgt*|TMny?&@J=Svx>O~Z9suPRH zm_J+xtlN>7GftMcC7vhH#@7=GGGO{`UI%uvAR<~8h8^`M=Byu@)@$pgsWv;>*6zi6 zMY|^wGqv?Ju=rPlgIxjX<6M+~!Fx48aNhjdv6qjHNN{-tQ3%-0_9QZ{MvJntSFSukM+gvnrMhtx>FwegC5td$Tw~q zM}ckqCyP?`lzX8#+vZjr@NR{2Pj5`8sKZ4VL8H2j7xuLH+n~p8{!XO0(ar#B^3Ktw z`Qrs*_kyHHA*2u>FyA>Lo~)gC;XgWdEAG(*>lZudVbY}wzdWWq`C<6|nLX^dhkcH} zr-eG@EtGtJzMyKMq^!98#roZhh_dOJ4S6bl4=-2}1hQrcO-;SpRnQ;IG?x8~%4&ap28= zo%{JGw>{teyyt?)qq32x_(wef_CLDD`40`>%g8;&Yd?3ne*DS&620$P^7nCqoVhDi zxsJi_c51pu^YfUm8+ztOD~+93dXc7Ku_AjdUNf5s@_pexMsEl_JeWsC1}rO*?@d!S z+iN$pK_j57=x1UfeW39O&HUw3bxRjiTZ%6$m3mZkL+jwt{O=+*!qiKKdV5}5FwDN; zSS2{YX%5AK)Iv@Z^a`p0!s|sRxY94z!F%2wpZ*OnXVoBAMtFP|K{^|7A}sP`+J!rH z0=$C(+hUxsgyVXA^u6euKHJ!+v6*;^%`H|w2)m~n$i@l#2oKD2jC^)j2!6e!aLYet z?=0M8(B)9d3&}oiU$LxvbW+BoAF$(STO+-P^m46wLdu*cc`wbtdO!hjBG*0(Fhs}2 zwIx`K25`Tw6Jpq1fK`h$8JId{uIl3)Ta>|X!jRpwjMEiX#0i?Y_!vRIK3`4jj{{2Y+myTHE;FXpU(j?(^ z3>TFAhu{EV0QJKeh6fQ)Yl~V>)aQ!KG%Ai1jAoO=&>_ZY4zv+7fLy4-1e8yE8R}SM z^J?!gENg}?Jg!VT#}oLr)7ce}H*o zIAkG4b8JYM>Sl$1?_@m%N$G>BbrIQ%v$Hp%+JOA}GIAky_WITCsO)@B=9?DNNLOCZ z`s`!kvjNq;I*ex>Ujh?#x6g41c$coUh52{#pcPsUThu~M;r{Vr6pglZQL2KVnGqEi z?IpKIg;#JGulYHz>+sxQ0_zY{-~az}P#cJZHM04o{gx9Xb`aUvt~&#U+U60@d|d)_ zm}Hve7^lE>^R}<|!Ufjy-xEtT)D^@N8iTelbqs-B+sgmm#@3Ef|3J5L7CdMiqAxUf zsa6q>1{bN`;8vWo+{EXino3LU(oA=b!6H)SLtg^BOwr|jXdhC20Y+D!gqBMy*GZu1 z0&K(HS#cVbXy$>v?wY6H1bQu?<>TwvX=tPczYuMg41^7O1!3JmfuGE6CXg%%l%}p0 z*|~ldAN1^?)grX{j*F?Dif3Ax2g%rXKrj@F)O_KYAA+Atz$?-tRIZgDOA$ja%uh=* zwf!I}E35FH(+YRyMTNH1&MytWsJoTwEvt45i0asX_FU1?j6;|9Z$J0wT;Rjn;f(&v z|El8`zX~i4torOS_d)X^;K$;0U$wF(DYrCNbTf*7cp{qs2jrgXC!8!b>`2g*yh4Q5ZizM7Vyt~ zW84u^7PN?ce=Xn?pA^=Sx;in0ba!&|E7 zc2hvGke;QD!kl*t@%Yo`JTEY5Alb&Ej-77JWsRj!A`pX{ELrZ9J3sWQFCNAlbCSm^%y zWoNJDCR*Q5O7z^i;j{le^2Im1-`@RvgX4FnH~bL!sxWe(#&zGx(e74MqwM^~48O|~ zitCfh^{OYicUW(+Td`XWh2V(2$GTU&9x48Z6KQ8j9v9W4<<-e6nVAp=VBpDm4I+Co zSEv_yR6cA7C-rSWDRZia&&6dCm{!t}+hkRDCmHmF!-C~^$p znDTDPBbuT+sWln&ptRB;hi3Z}a6*}3F3`hL{JLMyYZv1>@zAwjL4J>H)DX;_bXsMy zDK`MV3f@I%vm~@`c7VdAW@IjQIkZP=K!97{#W#ajooUF|J0n4}RH=Z3@GKL|ecZM` zIOjnr(^N<^;2Fb%i*!5usFS^Mn0!MaSd1kJ>1o?M*>Z+*q{iT!lsMo9U}7B^6VhbO zh|Vx1FNne_saO`+-g?hIw(QavBjMzvM0*k3y?asJ#tp|TkMMAJ<3&weixcM^5EYqs zf*HD3-lcA^1sCY33w|+)D#9&xMUmJ-B}l67z*=JOShQ$LT%Z(URw)eYTbSU(zp|+t znkoXT)yTYt>QX$flqnwopDCm8Ed3Y&HqQ&W*(5F)is%KgUaDye+?oL=r9#UhLHf** z7G>b#1WRDx$2JqnLN|3}1e}MPPioS|iK|<@7Aj||$l0s1nxG}+JRo)Cz}NxyKHBY=2=&HhQ=9$O^wKGCO0RuGp!6)f z#F?Lp5k>=@6p(1^wK}rm*xy?*3SFSxkc@1F7V1f`>GdMf8jfk^eyWn`wE{n1U{yRz zB=pJ5H>AEfy%sAzvLU3d26=+f>N>G72FZJwdX^%no#)kpoh!4TFW+$}$G++&A=|QE z*xB=KSwgYKUb)o!wv>!l0+e56hW0*bk)S(CTdYiZjzQ(D^iLd+GTK@6l$1%WRMh?P zTua96z6jk|>+L+RJr;P5XU)b4?S09zgyP{OaOB-Rlk*eFJiW}MGN$2Gv5H>WGz~? zK*SHQDvjDYI~%)l4_x5phMA?Y)~&>x`cmMF(^VVfJ{Zxc+#^{(=a93`1`3DYoWwe; zyg!22HQ!et4&_$(FfiZuEL+2k9~ij?NG=b~Qu+EU584e~>tbZ!(qdGx@Kp;J^DS6J zSO8ojK)Cxj4V8PO_K;12@K}z(wdZxKSQFNbU0m1q0hgnY=Ip(mZg)%xxtK0h>J z-?!6Cl!<0z?2SsnJ?`--0+eWze__boM5%@x*WBs znj000IDdOMs{g`=iJPB)chov1Ch)7yeH*v@c=4R$!AcOQmnPVUBV&}^t*sByCAz8! zXZ_#{_sa{Ol=Z;X;~ObK%7=ErzIj2REqT?+VNG1$ewImRzCLr~Chcn4cui1K+5@mr+ZTW5*}s;nEaS&}`l=3*{Ln{p94}=S13*=(qW=ef>ZG_1(G6 z9X|w~N~09t-t%DBX~1N?vt{>|uGZS~G3N^;z*nYJ`VW9L*?pecD ztbblX?geatUy#jl+vA(wf3N>b`QI41F2F039%>Kb@)>KlDpF~nQY3y>>7KDd#G3)F z_=pSiCK$SpG}X7zp-9UFtV~;05O)PGKFgU5EnsLn=PRVky*xsluQCI7)SB-UfTyck{|WEE$Q?6Jhad9d?AKCfnKk4!AL3} zP-h2Da=cWE4CLxYlrU$quOGNw+Oz9@JGcazm4O7uzcXH8V$(AFod8|biJ09du1sDT zlTfit%P#3e9dg$#jq$&lOjYnF?H3D{Gf2&>NF_3HAx$9cW0!j4#^+s_WK3+TbOBfm zibN)s5+hXl5>`t`VfeVkM|H(jz@Zx>0CS zL36>-fxOF6J&W>B_M4gAVBhixt*s;&v2Yxhj`3L^Ci>8@qE+5%0)nmuGe9l4lPi*_ z0@F)-wNLn>*g7;kvvspHGIN}ZRZ#(x3*y-u-?DqOCqcPJo@FR z%5177jwmu#lD)7quTCP3z0f)r`tbLVW(eu!)p;W^wFB4KGfZQy4O+Ies5*#UX`CRW z+?D8WZIbjt^0eb@o{tc|B zQ{jcQQwH+$#iJF}3Ue9q(T+NQs4S6tts4*cfu>NsBl@P1DZ%sIYv zBE8y#X&cd0%;{p9Di%qC#K#_*#QcIN%Hdl4Om7C4lbYN!hQC|3UAo&+mRmhD@^ZMB z#q!3|so0sNr4LIVf2DYo-Y_ug^t%7EvvGwNC>N}2>L&-m z_QVJdUtCD9E@4zlX$8-+-v2xNsb%Re4-Q@~xcbXa^rt`OMQ{J=V439jfePM4#-6&= z_^s|q{Ja0SfB(DPXYU4n{^i%3|K$rtFc+Ev-;eLw_uXFK$tUL$PCR^D8B#bg=@l|~ zKFIl4NkR0B09${LH|~iZuW!zxM!U>ZW6Qj)EwJ7K00fKQ>vv3P}* zd1;;EK$In#SxB>vWv>j>07tS6!Bj>^7^Cfs%PpE*q(Mf|2NTpo3VorKoF)c14QVKf z0nG}#F#5kSf@y|g!0){IqZPt1uYe4txIc6(bk0(tLR*Z$4f4@+4V5rc?_fxgOpN0! z3|yZSXbjF&l(5UM*y;vWWrMtQHVhr(V<-~oE4F&V1{UXMED0+!!_Xj1@Jk3mW2hXMl94s=qbXv+#!LoofqScfmU*W$W$>B4p0Z+8$Ur%==hb;y;cf%>Kikl z2wxNrMujSydXeUtt?0~;Wu-Zz+?|#d9R1a%@Va9Tp)IXylg=Q=P3EE-mB2Qj^hJY2 ztMxvl&K0R<6PD`GgsJdYOE$1gYAwz0F)pQoiNybzfBBL});|`}GVzKH1XX*T5Vxo> zPKhGSA4fm6@_R7u&BmOLuisB(GFX0O)JnL2>qP!?lcJkGS+i#;Hs1s!xsnxt5?h7b7QjN*UUE!i35nkwQmtR3)_3T_oU8F zg5<}RG<|E!S&A4!Yei;NX)d&G zSp<}OI16I3tgO|y|-0bVVzL`Jhd6hkK zOK62tx1;S7N<`f%_5{8vDlw}%u0ublFp#p+5ZGCnLLb+X7Wu9;=XEtx-9X-qu`TY_ zj*6HGICt+=gkrK=SL;bm`F_PN94nx>&@VvxQnYt>op4I9n$I+~TOff@h6jQU8I?OB zy@h$9gdJfHQJ5~r+7tL-wI;_N#Fiid?n8D_--qrGv&+MTYhMIyyph|Q2wAtXz>y~y^8N8OtNz^ zxtRh;vP|ju3fXzVnV}3X{_27x(5(;3%>a_aEHEE-H0saovA>lnX1ML4y3XVk0W^g z&IwNY^~34@@aS+V$`+TOmCAgUf41%aJVp>l5T&lBZEdZI*tK6`g1Y8k=0834zOTM1 z^y|uzjUm~4`ksCFdo%HAnrk;>-E~3v;shWH@OoPEvzU}tUeJQ6uThOFQ`IQ?tGL%D zl0L(Y_mp&n{Z#X`W%%@Z)$JQ6o2!2QzdfOcmdk(rYRi7v+gGUl`y4<2^oPm>)9pB%{N<$|2}o<)Xj4netY$H&ZemL`2+XISIg`g&z_%ndUSt6 zLQ<2>$?UkMz&!tkvG+M+?dMt|-`TWL-b5r}8HCwx-Cl+?fnXf+bP;B<0A*52@bG7) zw17urOC)Ib%{~f+xT%_XBE;W{qd+SXMf!W88igmeGZ6!V z2uS0Mh(71%y)Z6X+E@rO_e;Mj`YZ%hs*hR+@8*);P^JMZRNrHEs^=q2j5_F@H&u`& z!m+X+6;@1o>7H7XwTm3>JUmngY6Es=%YJLS84LQvNtbC)r&5;0qG2cop`{7vG)tIx z{cRC-TS@6AX94Je)77Dj85scMaS41F{dGQbdZLw=K$ka^XKCw?agkX5vC3q%P|5Y4 zyM;77AZtlX1AjhVH(8o9;K~ixXqh?F**n!7WVgzZnjK#{WIH=?Yya`Xl50?(LYqxc zStBxOm`tjJmt5g8-O6n3${eL;2|oR?`9!KITr-Uk_}6j3=;aU3A%=>Cdq0X|N}Ghs z03rBt zUsQ*YcPd}?_e|&R_!1fBWGR27?+%Z@2eS!in18t}ry5tdsj+Cm*R~)c zp3*efyY{*@CGCPGwl(t&|Mj!baLw1~2+wjugCGb-k>9Ey8{g5InfvA1Q9@^?Mus6T z9F&C+OjLeY)=hO$|=1Zq`)ddrZxHv7#9xs<*yw>%j!|8-IkV#oa z>qjiHuk!&L3tHEz+PJ2V;Q6{$_9yX*H|RSQ!cz=jNHrEoTOUF9td4nJ9_^Lw)ewxY zS4*siz%ae^d2Df zM&U|qv&wVR8V;}>3^{#Ns7nq6Z^cDiQ+TPsZ$A znasq$1=r=$BE;ztIhTlb8Y>#cq#Lzg|Hy`h$tG3-SZEtX@c9oT`T>?(#;5ggm`myC z@~<|Vj`}ufC_dMHOQ+(?q}zwOeI6c9KYl0R;l-zubH5_KNdF`Lv(NXOs~;Y$BAqTf z!oQ6lZA%)&T(lZBo=Pk~kiRv=CUqc>*RcqCr*}|A9(B$KjJ@lMJnT~34Ax+ZO`On|A+$oEL#%C%`hmo&Tqm;rFPJ}t_&Ckd#_Ds^6?SX zp#yH+N4Fk1qzq7A6*NA&)BMzB*rMj?q2=6N2e0gTetqVaf5+vEPyTzZ($U-Tw_Qhn zD}Mgn=~H)rZQzt6<mR_(73_;k8^c`ju3kxFidj_Rd6thI3aq*^}g<&u@E+t zW}rvOY7-FlhJKN`Kge>K+CtiBGw)vE)tWj;$qzNpmDfS8NZC5sq_#C^l5>(*HkfY& zb60^EF@R~NG26nL$i~-f(@V5*sGl+G2Nx%EGF#bR#t8>+RSRTH)_94}wl#wrr5xlJ zP5`{cuA|F+IxL6SpxSG%`cRq!On$GB(COML-ENl*Ttl*4p1Y)40@l^-qLl|AcW7=5 zEw+F!Tp*b92zb^>boM+qYdp^BQ~o) zAp$ca6qJJ9laJxvyx6?(4T3PzL{MW1S&^FSsiQ3#JM#LMaQ*+dsQ=@lD`l?dB|kC9 z!|CgXm;+Hr-Mo1TW?tVOv6^l~2` zkmbUlSr_MULJH$?E(w9Iv!K6# zAkNehsdEzvokV{tzBrjO>pu;?WhAL zpu_`9f`KNP*SsX@MGfi*JY#JPw44rAGM~dmPc5)kcr&=Ej{KqIbShDQx~Pu2U<@(@ zZBb4G1{$Pi{9(&HF)2R}Shc|4B}46XaCHqbPWdPb&kbFQS*w% zyt{Y!51?nOi}o~v%_s#c@9u1M2i{E8qYI^@GS*3?s-zCSD$EeYu%E#@R-eFW38_9l zOW*~-#jjzm*&zs1MaY>;_SiTKd3+_(nvGQVqB7sG0=(Dv5eWi~k8XPe^oZ2af}15P z;5v@HQ+F$gEbNxo`jY9Ij~u*s1E0V8+!t=Nhn5S$Y2+2 zvbcpl|L2SUkwx@Oj>{+eX8&~2Es4zYqRty!IXJYy3`_uQrw$R_?U?7-8_S@!sSt$Q zI_0i4+pi5dxGf(7_B`_(5n2c2_!a?9{VxhrQbNLTM8RWb6B0C5(k&Y1b@_-+@dsg% z1A@p}f7D{SpBh^VS7A#d)*?d@P+mU4Oe#fX&aG%w`SzyA{0Q0h=BwrIDqm&_yFP*F zt>1?j?7)4vfgL zQ~gE6Pi?Ktk#1FJU!(PNb&$zkkSeW+Rq#L`z+BA)^|C{Jkb-zAA~O}!VE@)dPhN8s z$|;5D3VpuFT%qg1_KnJ~D|wilQOx*@>!bR85mHyAgc1RPP8=GMh-N2(p@iB}nb5*b z$(%dnz3$0elm>`0S1z=y?sq^W`pDiQ;J6&I)-|+f>|J7%@T~adOYI!9(-n6^DzGuF z%EJ-UxHtXR#i|arL0#)WwIV!#Eb$Vf%&N8^Xnw)B-LT&_e9GNss0}wew$OKo$&;im zf@!Iz<{I%5p(1Oq!CJlDVKDaLa{q%}{XzQrr;`*<}Aco#W}BPI>C! z_xn7b&-?v)^*F(N(WY=`($E-9u>dg$C>gUbQ~ydfdE%74091+_miU46k)%a`YJ>yG zdeAJ7-2CCl%!DZx)1iR^{hCLzOq;716N0|^KF)|Hp5u0SXX6W^<}p)(3~24_o@f%I zjg3WEzO-Jw2c2rr5){k2XX7p`#csLs2~y9b#*%*};@pEsFl`3ymO(5}TLP$^Fzh|` zSKU(`-8QR$j2(qys==th_!egLQLgL-ii-I5{)-X2FbNJiwYX8D-5G@ewx$`L@5mN{ARg@LM>FMdo=XkO@IJtU|ag_V~w@Y3VPhhs!G6vv!!k*s#lqz+gjDBxn*YTJl)v{|X z<#j`F&da(h1;_rbjbR+^n-na?lf%g=7wm%n9$i(oe;_3~FgIRa)xdpRx*BFIexu$I1VZrA1}qkV;)Hlo2?mbxT z<;yJj$J4u&```Nht$yU++fNGq^2@;=cK?2RJN(A3$m_4G-}h&K(R2IR{+|l>gPX(N zuTECp{{82f|Nigcafd&*o+~>aIlFK-dcfVWrivgQ--1P`**P97)x>!RLR5Qda6bT^ zhNKd)K(RSpD`_xDI3{p>Xo9ydXUF&eO98_MyokFo9cl6MX*|=E>Y#YWA@)w-1`%R5 zGj)hU?Di-Atq?0B`1>0IDQ3_JQ~OU%F4ZW0O~R$Fsx*=ggLiXpJ98^Vzi0&>i*^1) z-Eod`O6$JvOO+u`rak7nsEM?aO7 zr2rAy7H7Z;tnIqhcPgIHNzbUKy3c^9&-(F@)yL=s4O0$Kos@)ESd|{5P6Y%bPKshR z7^O2ffWwdF6IXbt&-fDrr`m2=`I2Am!u~K8-S%N1bKnABu{C#}q-`1ZYzDG=O}wsiB#H|mhaJXT&GNW|>|D>Udu1Q;tq z+7(6JpeA4jA=qvf9ibjs&Zq1Ct4$MQ@sjy=Hz?rFfPfY#r<>;#%I?|OJ1p=#YiYlR zl`~)UUzPeSCk_RBI*?Kz_^wkS3ztENEM&e%67P}%ummDg8@1(i+DB16a$`G9)RwMS zb6mCe(G?ucp_c&a;v<@F6lR>>oyyI8tGRR}jxr)98kUT9Oc%c?d59r@fn#WU?>X-z z(uUFzvgIZ=aR6fceKTQQhocyeK-WBoN(d)|jW%xpPr)gB7n};X-0QJM-K&#esplZv z@B)wAI6}0O$4gpRyT1XFKG);z{@U$jUuPo#oAAwgm@Rv@ufqPw5fIU;rE;|Zm?c&=M`liJp&O#k4dJB45bf_9coM$QEwb1HZ;gX~eg=rFr8-X017-@XO zB~d(rWR&)W5i&pr0~vd@aCVDGMXNA&iK(cH=?Rx~Q*Lou*B*PtJ=oZobQzUCxo}59|Bu&>wY-7h3BeFBO~U)1;4Dqq^D^6p*(qkzBz&?^+h$T zFVsa6;kDz;ms;hd*4L^6<4IqR?^~fBQ^r<3j|q>!Za>P1dEqzqwELm!#lW72uD^J4 z$FG-!OOKtXzfA4 z&`!4|f^fhp1&6CjJt}8(zZ5M2S?r5um(MJ(rJZMd!>;tX@4wS<7E}1Azw!D{Y>Bcd zKX8D0`nTJAD*wLCS9*6Tnq0j9>5=cfmcROp!nxRY_}iy1`U711PM?18_*_i;+1!hv z?NrQu^?a1-ZSY z7%0GZHZ4y|P#sDt;SLUTYoMq@ggMr>uYS-yd#OUFtEyg^ZO zV?>dH*3EDf$5G2+iU^G93fCEBoTpf>bvvN!KqhuqL6MD(4O1Hopv1&pVK*xbtkO6p zpKzC?1?%kwE7_4a7B&H|{CqF#4w?P*R>#C3v6=?vXn+mOLtoucTnsZDgqc5Zr*ADD zuSiaA26+q%oqQyjI;$3$wm^nLGDq?dbMJ}xR5nY1o!6J4L^z3E9jnTd)1`?U8fPXe z)`C4YSQTLZ7u2uf67}>>p}rff4Y&VH5=#RwDVh;LNZ&w>voMN#%6bk*O})~W5Rfma zY9%@&RZ0(2g#%LY=~qxdSM*XFkPjvz;&gz%wZ09ew6=n|&&b^3P5EdG;9i;VeaoM~ zF0*udI9}spnV^NViQzS-Yep#mpLp&%!*FU)$r9Osf zTC#uPiF{!kMcT4vJ5gLkhIGJ_gs=C58CoGjVI()hGmfBRq5mgV+EBMc{Pe4|Q8+@` zM%YG{RVr+lLoaCX3+L4-Deq~ko=c$wT?1y$S?bA9?F)JI8awP;-n z#ifabH?K}e>{Y*Ku{QH7Jdy(W?(^z_b{Gy94H7Rz%Y|nxd3%9t5xBLu}z5 zifR+e0!oEULDgo!>cljb$~q*2fg+UvO)zBZfyomN7bXwv2{Au_X}w7LT#A85CzkPT zvo8w*q_U~8!x6^^49Db^hpII#b=8M==CQ}VABac^3%e2nD1R#DQiSs2j^Nth-Iw&0 zvy<3i-^I&hiTldo-k{;__`1Bi!fI;2Ejc&$VePp)7rk>|`JI@%SQ{MW@|&0A)iW(} zE4(xkk1WF!dU#1MEidBv{Et?T{}btI`1qjybm5EmE0r-#15e_fPhIS%F1_yerPMxE zeJF|MpKX3{Iq$ayCp)KlPiV_k`gf&Qsz{~$0m+Th#8;(NAO`zyyPB&9A0}Rn?}$JC zZM7dcO1aT7b#GvMUVWGn{5lZ+>gGKX@JN_0z)#UXJ9umH)W+!$@>;wQ79Z<=wUa z@x4@E-a#sCUU)ihy^k60^Yf?v`}?o{dLxv2=iAqfzNc%le!0E>r^jB;@BVZ5xr1MM zEr0+0w|BqUZTs-szkfY;qF^FmUwt_EDy!^DbvdXN)_@I|gH$G~FP6YF%e3k1Am?+4 zv*OMEc!+a-Vzq&}u;AdK{t}he3xp~wJb7Ag+&SV(Kk(tT7hi=oXtdhbjUg9W#-dmu zw6Vyciu!8%!w?KddL{20#YNisvCkbD!c;xotU;N^``8N#2M_7=SPB*758Pecllm59 z$FC68R>d!3IQ3k%!keXxVK3iOe7d4XnwtbvqZvVM2)4D?8Kv3_FZzH;%klzd`J>F< zI9^PgaKqpy)$odVzX}#c&cAV_B;7<-AIT_0<<`^F#amA1{jwve-ga+k@*s4Bo zcKhtIP!crvV7d?aVmFR9hsbl5ybpr}Hh@&KKepU;La^>dpH_?ro{l;om!Zx+6`8${ zWOqM?Xy(BVA;t=6c| zH7FBw-Ok(VDwx?yg8+Oew$tf`T03&vf(mN-CKPO8t$JpcJ>L3H7f`s6zgs}E+Jxjb^ZUUlrzSKGeaQvLn* ztmAJk-ah`fhhMH9-uC?V&)c>+Rc`X)Z<9qdO?vXNsgqHqZ{OADmpCR>zN*;DJW~?) zBEId&^St}lBn!`dsGaf)MakETO1@_4fN zqW9s5rphPUJnOyBDy@C@9y*5j^!_=J!uW)s+O4SxsvqZn^zr1KaE9>OdJq5WewnB@WuDP`+mRA;r_Qyu==s90e*c>zZJGNwOqXSj~`o`uHF61 z-!>h*=sR-p#kavFxs#7Qo+pk>9;kd?bn)4(h+n>b`t!pmc-Wb5BV*zKI48USaCF}H zoGa9wD2n2^r4QtZOmjFA*(1d)6Y$f?{3GDBqj}qgIzffsW`69*V?swhaG~F5fs`+4VS-}SPdJhiI20kEYOKK`9~qm(uFarMaG0cTedNfGZGfRpAI%g+bfJC zro2UFn7FP(TGugb!-IWNzZ{D zBqpxC=_)$O)Gf6UPxG{k>hX!qIeg^n;Erb^!=Cub37In~WM&}V_z1*gh1ADL;}I}5 z9EZ+23B!+iL;>mA0|OyN5~`Ht7O(IKp;*ink?OA`k?LZfKd!0!&eRci)490zf_%vs zO#4%wNLS|V0L;Ouwwri&!p*&?3PDr~p_11)bAV)HUVv-uHA`YHog0c!{62v(jR46! z*GG^nb>s`ygDHj`ko2xjEKFuGlFVoOqqp)TEYOh!?DMu6Mqtq^Yfk9p{xBZ~O7W#miELt|R>3&5~B$ zVE3q#mZE4PMU2d!ORz5G9 z6|nV@Uwa623!pYUI-|UCpL#Llv$ya;DGELkQvon#z@w*&`C+qdob^xw+$J*{A*#8>}#W z`DRc{o|W9dY@#EiYa6iuWn(D7T3*2cxhr#nViq3XN}Wg0DO1d}TbYaFb!<(H^Y6@X44oVt$N+($&ie$3)`*ibq~2*9e#2B6PzIZ5(MtuvonV72)G?xTS)l#Ii`15iy{j(1U3~O ziy98lmzb2S0KpUFJc+2hAd(B%QjoDc&Yq1StT+&tLnG~72Q`BT)7o5Cyhr*%Kia$^ zpo`cU5;{|ss|9kpiC>bh3(Dvki$zN)`X(jE zg6Q1TVP1$~t4b(_PPfeU(kemPY`efey)$MIqGjoAHoAqaxLK6lw27aWBeaDO4y5|T z56LIf-cnjNangb;uJaEd6KU&rvJ^#T7@&7=m8V6Um)dT5BAihEALZBpK_*d8Go|E${`vby zqGgi@c=4rgu&@<)pqF6*V>D845}|3IQD{BII9)ML%Fhryiu3^!EF-WV=p^>_LzD>@ zz;a(NfDjr%TI&JH1*b8Fq1X2wk%lnDyXYKqV8_G_QGy6hrE#8~`4ECw*ieu_S0f`(T7j0;vE{>b^ zxjEh_jAe|JVMI=wyBtE6Tv=}{l#CNSyxJ|T*6ON9KBW1iP@g7yH8a6T_k50<~0FL zV)=4m{oR}A%3j4p?Bagy>UP?JU;e6U@0spYf9BY})qdYrTi+@g_$JZn%&;-_LDqA> z+Sbx4mzo3Ir;Z2OexjV$YW(v{>hae&!=^qYwpQKJ_i3);yQiC_tO-NjUEm z6M*8KY^~~i>c0Ba%_guZeh)yxg1DBle&Tv0d%VK4aH6L};#EtY`{f^hgf~y__;xw> ziFMsVb?2DtwR~P;@BD|cQ}&Kj)BMV<7LNcFFwqYG*sPAWo@6|zZ%Kag<(JQ%fA5uh z_j>8~f2urDbg!s+VfexOC((}l_!BKD%<=DdlAaHV(SF{K9$ok~_y6`@{_Xkq(Ir-o z_g}t!3H-VJyAL0D_3hqoFaGwYa7OrFcaeBE-Yp!M`tfU@eeUO7s`;ITSN{6t|9&W# z|0H&!`d|>N#T)}&*8ag>^Sl_Z<7ad?fwof_M;F5_h`fn6WyN<$TXEp0qI}W?NNr-Y zvv^W^Zn4WWE7?HSB0x_JKfcEvST8jEz`Xj4J9Dfw0Fmv z=>z{9ZpNxbBd0VUm1Ejw^i-s-RUinBj1eP{bEg)R-Vl6C4njJHbW_i+QqW`0I6bdj zrBVY$1KB^qC5O9Me;Pk8)=NR}BUS<{hyeyv58BfLwkz1Ry-+CIc7$Kda1zN9F|> za_tckfW}Q)ipVH{vt_(?lOXCz+zD2HA87CD**FG=lY&+12Z$Sr z1&*~@BPTnqF28}WX7rvTS@Sbke2)}n{&bwER4&H1nb$VncMFK3R%d#j#`v}!Cj-Kz zPJU)5q4nsfj1fbVFw8fdzLxB%0``>MhC0@9(7=`phMtEj#^h_xd|-n`8g*z>4+f}g zI#gk(cnnd($7>OE5unz%&sz~@7Eu-K1!KDsoE|vpXH+2}?S!$5WHJVJ%ugXq%XG6; z47EIj8P<=AwM0Yl%5@1)dM-2aP4la70C?;*EmKfpCul*Zwnos50g!g}drw^5Jk&G+ zb%NS)WaB~|OX`>RxDStFs)~um$Q<4WPj}TYKVl+12SQYHEzbTIUegX9MyKD325F|I2IeCGLdE;4`LRv#{#_hDNWYF{)T`EJTkD~ zYHF&itV~HJO9%4v(?vr&lMZ+-SMGFh*?Vx}pAVj&wz|~b8|-Vf;(j1t;Aw&9+%HY% z8tN{W{oQ?ZW)k!Nk8~=KMjh=+Yn!y`8GY6oWY;hz8O=z!)f_|; zN3%lebJQ6(-~V{O^JF-~h1=*f`J<2cM14*|c}?Sd;=8d*_qm09Lc;)=(awvDABrdo zCXYT39~-LN{PFCz-@?PepmXnETy_n7rdiFn@cYgG$6coD1L%Zr}L#GVlNNOEQNWv9DII` zAQ?PqG}#$m4p8^N!^MjgnF??OyjsqlDd3cw=d>^|{0xcqETyk~ zGis8Z-iS9PMhd~5TXs_Y$3aCG(j)W3mX*}4aA=c`&q-1Y!`z6q{w^9U5$MNvDPDl( zM3wNIM3^hw>GVw#J~XZ$g^tILa7~<~3$x9SG}GVQ6tUafV9>W5GS<-(%UU0G-aGsG6Pg z!5DNcgR=1v8!}Dowi30`r6SXf7tZU2M?zG!It`vQg&7)WNT)Zh(^_5A#bcoGxiSo6 zjta)@O-tQXG_s|iFVilr%={FEJ7x@^Bqx{GX}J&-8pz;7Ra9&r9=7|2yek@aYTwt2{I3g9AW2` zlaKTB+rbtd!Sxp6XPYMmW;N?>A|}&3HBj7^PFNi!*{fyT^mi-$Hd3P$O~I+PX?G|a z{cdPe3N5`q21+@iwp}lDtC>I6V}l7ouq|q+UM12uaVak8%I#+3dl(R9Tb^4H-+L%G zG{Ay5>Q4X=B*mvqf@iZaG^Y6+P`1_b`1WpW7RUOCunE}1tHGq3lRi;Q=z1x)%ng~1 zqiDlrph+l&X8=qXjy3&bFHjkt6Ou@JJoKa zwDG$AT4$g5g>h%nV?ooY63mghcw$dCIIl*!tB%5h7-`KRbR~*OE$fycastbmUO#oC zFtE20I8MPyg2t-OteC-aAm@=Y7AE~13ZQxWTO`Z?=EngB0kmDKWBcLJK;mc9qESw~ zgatdc$eG;>XQdf0CkMRX5cMzr_mwJ~S#;*+HrZFl+fy4K96CBU8PM!O{q9Pf?J@tO z_8+fw`vuo0vU=5w`rKXz3pg@vb zb)n*H&6|qv_;J?mTSI8vs?G$&LdLv0+BIPD$Mlv;_l@1)@1Bf{b|%I9GTDXK(9xK@ zv$+iuKl2LD?EapdS{i%!IDcSM{+sjlFSFgqZKOZZ(wBz(uui|~}@8AEquqW~3>x2(iQLeU0MAh(IXNxvF^<>m{HJt}? ziX45N55)M2Qu6+gpE$I#8Sj{9O}Dl8S(> ze)jd*p{n5*_0%AHpT5iXb}ge`oxWQSJ!lL-0aOn8nRQ(2(Y%p6)cJ|X^IAbWV#n2M9h3m_zaM*um@i|C03J)7-cz1%%PsOoO=5S(HVErMIHcSYC^#p%1+= zfu=aB0%+P7pBMDSX)QtQk9C}TC&(P_pveS?Hob?CE89rvw6j z^$|~x_2I}Np!bYTwVQL^hinLdZZ=w~|1!6b` zm}MasjFc6hmv=jpkg96vr7GNt-=N1rw+{xR2D>@)akFZv8(oRCo}yUBx+C4=GUvR} zhGR6{?@Y>iVR1$a*Jz}hb=xTnLljzjjqWjDYg3n%2j*vB0;q!%WT>LzG3EBA8@jV{ z^*sQD&YZ=gjLXoC(Y#IAWIvW<`PA3CW{1Nl(d(qnQ_2KXh;Th|>ugu<%ug8oT^xlm zP3%FC2qhBpI-k(QR3yJoT&%AvLlu_UXYj!FwL1h1<`f2wiCM8x=y2bDGJ0iEIz(KJ z!kSEGKVl@&QY*McnB+vJ^F=SOx2tbE*s7x`x+=Mr$uap8wVR54$Ze^qKu^9ouk8<6 z$Uuo{c*q{GOopKAJNavc6LQP?f?1|XahTvp8uw3RzN7)uuoV=A_DZiOl*!xrTeCLJ^gI#oVE*XMX#yzY*V zPB15k&yhuVJs=Tf2U>hdREdmaql^_|68AgWmWjRy&$v4F{^X=L} zTCx~CRMg?3Pj5KdM}R2nhE_USkur5<6#a2rpP_*8X_IBvczn>JmLu^&(OB(UKg;ZU zu;<4ZBLR*QsZHYOGB6tf;F)*a2atx0_{C_dXUBM+0BqqP0HK@TfLrT?>Q_MTxGYcl zHNX<9PT8-2IxvpFEo#|4{e>+SMHM@joX^BX;a1afwuNX4E3^B4lAkj#QmJTnfkd}I6({{PWoA5W12W= zQLL22qMObI4nB71^73G!AQ!lAy-$D8=Iu{ud@^J>BT@rkHnTx)(7lwq4#0OdrPl`| zDQeS8IpZ3#Yyb=nIJ>dOF|A8JX9wYi=oxj zrVi1{^;Q?8eKMZYI1H7n#0niFQmV$ z=4i*J^4otcf8_S~o9b_WDgD8^v=lNh(ovB-a`L+*#6tAV&XcX9ZT10yM{?BBhzl++ zbGoj1zSwc38gOM!fB836%Ff&OzIfaEvG2yapz5BnqAEz)6-ldG;*X8x9i@}e(R)&z ziznX)`$x=N3*V-C+VIcdYhUc}tMt0;d;ZRkp;K+?z5OSA@+Xc(4W5T|p2&CcadEmL ziE?V5KcJ$polIP|LTRj^+`V;O~WnDgg<9cMoi1qJ>B3zHhzyY{@^pt&S zQnuL)(3|>A=p;==p7=yVLD8cgCWsE)jAL4UhnWAHdPG?(>@=~$ zY{=ePJ~5FW!;yL$dtgYZlSr~-Dv!0g)|F%>zlxZr$pU?9J@f?M45W@`4rL(+^|~;q zdWmGXO=2nwnB4jGdXcQ&ys9l}K zza`w=UE4N~v3x*tx?-j;C5nd#COJV&1)&9k7y}%$#_eX!d$E3rj8qN~44wonyNQ`6 z?VK7!Dt;neJ3Lgu(K>S`0|9s@#I&1_$S8MomK*b$Ae>`W6m%56 zt21Q1M*_`^H&K<{W6kz5eHAB@3c>OHGsym%7v&}eUr53Y4u!}9*&D^;+2P=!Tg{Be37vHp97H$7 z<6+9#hze$td#1RHMwE0*7-3YO7QwQyr*jOunk0ScVWt(~ikmwy_>pYp5&cv6;f`xE zR=u4afKZ(z>Jc6Z9X9;=k9fnPJsRuPF)<~2C3WH%H}Z1cH(lMqUdIcjr zA#Hv4{0!_33n%@F)@W`*G67M(H9cXheZt9EhsOiVO)4^n)2ly%-7VjT;82?lF>)#=pY{|DU+IJy1;`$HWDuZJ457v<67G ziu=VGCEV$h#bB7?A}PJA%HB{++=%RRLJ?;D7=%tjgRdUUNHmeCqLDm#@ zl32BbDaqQGswHV)dQ3%{Y}Zg*eNxfUoz$HG0^E<+3Yg=o8d4G6LyB4q6qp%OIW*-S zu;GQs)gDq3LC%eD)2IOMw4oTk>=}m{Vt&M$UY-TULB|$~8b}D6z8W54G)NfN;AQ;D z?w+2?R6IWgPgV;m&+?1USOs%CqCM)$SUa5;Oc0_#iqr|Rj z;8~kv?{9s@`bq#aE#-IKJ@o_sLs$BOQuzGn*W%Z<7bW)Ow?TQ za<~z;H}pR^+sUUt*W5To!}Pg?;L6oiD0OY7RJfl530p7kt>+rDoI#>z~hK7Z(zrvHYea>W=YQDAC|%3Oa0sbsiQl zV?crv3r4hEiL8#9A{#iJk(F0`51*j@Uz+yq9UDwXDmXw8HAWyyi1b+TO>gc{ewq%3 znePR9@@6%h38r>eA5el%UgmI4hGcIBBfMx!IC^%x9u>eIt7A<&I$$Qp$BFB9eOHI* zDB&7_bFEa54)<|-PhlA861}G|bLELfQ_Nf(B)d8hfj6x^j3fj^0rvzh0%hH!`Q4v2 zGZq0(XG@?N?#|eXH5ChVC5f<_C2Lc!p27&zcteJ4k-ml|u+w5P*G`4V z?g-H6 zWy_@W60UI+YI>i6S3URim<@$++MxiFMg;KJEvGWln!NBr8 zQ4M;N3FF9T-Q9VfIad7eF#+Bn)HQALMH~+EsOxO5h&ybrzZ*Cxq95)rC_3~Yj)Rzc zj9#rqOR5J!irRY-3x@->$Q92`bp!dkc7dVu^(Wb}Aq`U^i*5+Y(R5|pHyN_wggpO= ztNmY9r-moR)a4O|ha&OtvXs1ZI3)#iVdJkvDkq0}d#*c63;`>#&T{X1U*?im;Tif; z)gW`g8jp_gkM?IasDhW7OQ%{+JIq{~eCB@45pkqs?5KVF7T}|?W^|h7lb?_Nv^9%w z;OT!%+F_qj^NmXxKa3oKpD0(`C?_J&&JiiY*v;)DA5#P!g#d=zF!hhe-vm1%iZ0!> z=8I3XM1k1TcM$R9-+jGy{8o7udN8$c>1VPuw)b4z$L4t6|5pA*tu8pSkSNMwhCriR zsG+s<@%M2DVVc=UuPt7+!4Ce+=KIdwC!1}<0wP2AeP5RFe|B*<$|oPV=6jURK92%t z+4bJG7O-zi^zm!@>mjc{r!=uIIm6+VLj?Pkm&a0$w-*%rcI-tfXqOcn8>`ZkZQW<* z6Hb(F4wG&1xcQPB6>k0A{ueF%_q@Y}kMoZ>|0U(9r1$3)uiDDfg*P64b*1k>)bfd! z)T#u4G0%4X8tm_Y|ExT_h&Nu-*m58VK+BU#Cr_rg+t%a?l#=-h$dP7mZL9i7U*6G> z7o8rG>YDI|n%)20J$j_Ff1&ANo_ZUfk~UXQOo@`r4aU)h8GJI`(O&|`b%Jz}2${}q zRDcA11}Fp(@m!uP^A1RSh2n1wu@)(7bRto)dIwGv!0WV7((P9yPbKgWSvXn>Zb*w28i3WGn}HC*);J)WN#_hpkVXYvh&NXtg5OmMGCzc2ebDr@7A$_A71_)&w>TKbh6K(eDMj}n za^t3UH;8%NT7E0lRv7X`IPfZH6Z_J1cd@2t5Pgow%#5LlGTwz@O7pemQM6@VkyD)Y zS{I4LiS~p%3>|fZWOnvp*q`fT%bZMZN*?gzh;bn^YP3=BiJ{>2JaH{HB~P)7L+~W? zbA+l7ELI+EZIb3JVlQmsXsN}xI)DoB1Jhv;v`Y~gh|_K&tONlOeh0IY36nc7bjGTm zaq!cQ5LRRPN#f#hf|g2D4Pn>`hPkj=c}3#YzLoy#{0Wg#t0?3`-IshlboT#sK ziajAg9Az3Bg&#T8&xn_3sI4~kK`4)#1aVA!=2E01L5VtI8p8-TtTF!1!d2s=tz#Su zE(Doi-iP50k)Uy+*Y4^>u8dLd#sa$DzJmDn24QElAN(cFtqCFI)2CR9gZ%PkpRHIj zCa&Wa+TDanh4-#HiI#`y9?Qb4scbZGW3(oiifEdh)BVQeZA->&S+@}VF#d*0!MH4RBp%5(7Wf6rlIP=TDBF@dwK_*o!7z z$Ju0%v2D6L$@jPvU3>5D{*$0gS>aW1Kc;0jwe!iC%V8Tye9%$*g|-u6aQl;!hy4!< zm-^0*W{$;;eB^%|Vx;@)zX8j-i_y9NTv<7P<$3m3|GHB)b-FCEe6qP>tS+>?r#64! z!gptK_iyvbd6)bPb@*A&=!tqnLlO(wu=N_ccYg0!jy2>ev@7|aAzc0QmyoDUgaCG#Yz14S@JC9cH;}#x0-)PM$z3$R{IZTjP{@N+hFCa?% z#r}`6!gK9UMwbwh+N8(F2AnrHzD)Y?@L0>q|Jb#hv}VCK{#d>_GQ7lkpA&YZ?Ci}i zUpK}q-vcLn>LmY6iq*EuPk;G$ty5h0wi7X4*<)7*`WmIAxbJGJio_YQeq}|S(XqS_ z*$0w(V+X%Ikl5Sx_g_mVf1JBDk_}!Q^KV=rxRjI-adqh?K_Z;G2U1tGJaK(2Cgk#> z&6MxG*EFFjo_H%%0t`MoBFh9~^Na$-4CO;m%kBK_Hl@Yju(&fkVO^C%7ijKJRjl9d z9(+%A*X3eC7BQ5Rv4jA*?FEJ(xNV6FXuDGOm4cr-7|wT67{)cMNN|&V5-< zNnRaqyvS5W!eTM8SUgeCLVp5uTr3_HXvsV@Do{Ru+pgr7yRx2e0RZ=dcgMK**QYkG zD8y9fMKzVMvgrh$26CU6coorvML~P=b5>)`qhc)T@r1}I4W?oyPhRTiuM?=&te7)k z(mUwj9;$Urw?+%~HN-d5etj zmNk^tqZ8=_4#o+Z923Mm>i(KI&JDpy9>Ic}=4X~Z4RXxP417A7j#mlsOwL9f7C#+H zoE{)$jsw|v#~U}+uoNruA6h-X2wO#%Wd4LI?x1}#PpIETG#e~y`)JB{6bqn6EQi|A z1$3mFkg&qd@kBlY1=Dn~s*xp_7~UJ7t}@O*Dp;yIwd6q+2kPU&Fq*bc3Xz3ntVvEI z?UYdiMQM!_Yi(1-@bcAozA6WWMhBrtZm~AOE*#M5PbDt=RECuT@SbWM9ukX;POIh1 zC@RR}5E&gGVd=%BMYWlyPr*Nc%tBU;74Ezx4*3eJ+>>I;;|}hWcCC}xi)(A=@^Dy5 z9%nGt{jz%?()6W)?yl-L`%y(36K=4elbK?^hoX%mQoC+V^_LpQ1axVQbE=QWg4o08 z7b4pc65k$#8DJ*F64!v0Yah4{K{D3%z(U#<84}EsAqEv8F4%-H&P*v~-S#Pc5HmlvTYf2bBLdt*3`f1_fPzx&L=5eq z1G+=Lw)W{)#C_sM3{f)zkfiI3?Ql_ML<=stud|_Dk?n^fP9jK|BP}+8N>2~{0$c7S z6)rdm%`l68NajXNz`qhs3SHA93^HJH28A#INkZMD?e|<_{BOYL_P+ud!!={s2`pQF4sYdF z5VUnP)Mo3+j(-PoZ~YOvqy1pqHEzo!W8q+L9%rt_k^JSp`q9MisJ0Gr($IG#=O_2x zFMZFtlD6JkT6$${?%0i&r9C&AYex1@JURcpV731rkAJysrDSUkJA09tW3~Nvk3HU+ z64o=Ge4{@2?|k)==>Vy;*s*vdAzD(K|KD%pkN7esT`qg)KYx0$>QMEV>!VAR`+ume z#0>uMaiFxuHrtjO+czKmq-OMlz5SLx^~w8j=N+Cs3HbTOzHhdj42S?%#*@dke(ep@ zUIT}6n@)C5)_0Jrb_=g!d0%`*swNE#)cZAE^Op+qW(OE_^;wMh0oGmIFfkU}=wcC| zExgI%s;v2*cW!JgpVM8(UZ$CaC}R)JJelZZXMcf2d@DxTBS96`90BG^MVlk#25=~2 zaoJsAaTFxotu-MDm$caRejpu@o`{PH6CXfzs9Qv<7tpgnqyx4|ky;Hz3tJ`eu11zc z%G)p}K?dR*xKJ_TO;k2>$OBm)T}S_)qB9RmI^Ey@IcLrpXQoL`QH{lr<$qMLm;D$eXi7&(Y`senma zyXcY@jlrijXAa~s$CRK0CGc{gh2bQ8VziNBUBcQhQyR4md(s(vI%fn_tXTUn4|GZ8 zv($YeArqKu^wgf=Dff zCE;+JatWNx-RtZsSLmQTY%kIaUqhl`akWReFh*O4j7!oFgLU{5Cu`TpiGcV5a>k^8 zxb=CjqeyMn)l0N6%7uI}>G*zG?2uy_fhP{@j5hX=5i4)Ap|9>EJxpDg&UoTj!Q#k% zR__Vus-YC@BH7j{(q#n|+Ztu3B=Gh#`(y%~Ivb{*f$Ys&Go!NvOkH@!I|^c{kYs=j z39}TVR0adP6yElj&sCVFF^;nti-j_)nJvzT9R{M7loyNmVktd4k1Y5 zl?GInYHP;uaHrk?-;y?_Fi8ZS;~=qzbZAlS65}L1+51ZCU^ysDXpAo+QVQ~CU;2`C zir7vCpJhg|2`9rlQdPm0Xep&h*kPMgs8kX|=#`mon%I4>hi_ z4g4qp!RVS-4l&<@n5z(LU06~f)|OlAt;%)LA8~1FpA>l#4RoS}K9OBvSgmERP?#ou z);p9{SZV6cic$(hzOEVOc{{@Dv|=Q-bGr0Ez$$3VDd(f|fKMRL?%)VdPjK-u=VD`n zoMsxL1Woh(jyh3$OSdCd?lw~$ZmC%pg)Jq>pPi@dm^VB-E`O%IKsgh9<~-x}pA*}XoiD`XKKtq`%725ZwHx+TpZxlJY}U!Y zhsNxEbVcm9cc1ICUt5kHYyn6$BD=PyKG_~O;WkTV;SX8_Fh zWkOd>2w2WvfGS#Ncfk~g50p%#g+5BU#Phtl3zhce;OyZ-->Uz&E2#fze#MtPuPIe6 z!TeoORM(oCnHNFvX+s}uFY(_-0LyE2fHV2pE9#EZp}Rwk|L~~fj@DJyDK9ui<%KZn zv?t@M{U6ov_EfzGU~c{GBh}#O-M!=J(KCDYRu}EruiF;>G=I*$C@#|gd z@$disb75E8lbf=nvUi$e4dm#X@^q%D@usXaSh|fCnR9v%QiPr2M0wDn!G=C!;*T%U~TDb+6vaj!=`zf zNe?qGh^8xUHi1GxR{&lmshvpEUKNHN%7jaYCfr73|Nz1hW70c!$ zNC^&YvLV6+`(6Pvap5{z^C+OQO{=sTl7PPDf}iUtr7;)2EmvfYf}*@Or=ssRMYasM z-Q^0BliT4PMXk_QIvtQ~E2^rdhoS^&rnQoFj3{NhwBZ3aG)2k8Dk^+%*>HR*L($$p zEnd@zCm%aQ)ql98t*qvgOg5zCX%Umd4ORDGjOm5Mw>yW+Jhd-vd=9(?{b8}*id>|R z=Ok&$j_|=UjaynBYK-PptZ@*X>iI-mAGy25jFOzxs*s$~iI<)UXk zuQ?|`Km*kg$*{{(%*5PwCna3LzPBM7ysUjR>tJNKE}kf>B`qx}vgkr0Upu6VGy_9$ zH$SUj9X4g5uIr_eZDj&_j^d?wPtz$mqfK8V2L^Y-gtQpnYuc{FsZWSZm$5fmUw|ZT zhi+bER1g*yO>#e-r~#fSnuzd!LqUU_CUm-xYP*qx#J^FgtqPi(kl_7jgG;ac`Gg=}|8Jd+JaBFbLa$E6|J zJ0X??>sXhab-67f>#4x-N%+BHnrvhf1S2p@5StaNMUft1#V8OBl%m(RbFE`YXa&p7 z8X@jDR)!5XKJLy!y0PGVypYUu#2jWY+pqqR0V!)sRn6f%xVf_Y@}qbiewripS9}e;Ja3x;C&S@ z$%m(aaY6N`3{y5dBLe+OQh_7g#{#S)s%V`sjcb}T4J?$`dM~Kp;zW5ryQg2M9YeuX zkNtclxk%iaKVr#_v3u<`>mbtY) z3w?zkqX2#uy=GyhMZ-i|jP|s4{S)xv!U|iWluUIZ1`M=5aqYe@s)y)q2g?|^tI{=* z<)w!bt;i2o%*dscmWPaBvP1*vh}MlNYq2CwiSDwzzpIN8jpT=^tRaIZdGs4TwsGFRCZDwfsEI+xyKQ{Rg(418gYQ=Xd}7{r-vX{(u}2FRJo6+<#dQ+IF6{$nk#hw54;w%v5#4!PMI~6BdiHX=UVp5p@MqTqqgI2wjhFi#c(mb@PP-&wl^g-s2^=Q+Z#W3W)AK)*SIj zawq94c)tI+{rG>^Ih53FZh3fcb5nlE=BAGOhmSxD`je+8{JTV|jb0Fik;=yq7PrXc zz_Z-F^l?6OV=D<;!aU!;PlhMl-ykt<4~GM-iF{jDC)0+D9|Z+mXid!A><`G)bLYj*rCRb4auZX2yp!=0GS3u7Z$D-K# zLA*N3b#_slGZH`TXI~}{2~#W}cJ{UkZMuL$ zRQ%63vX|H-T{U`TwiTpb-vH{<>**iwlcO_L#l(S10ByDQU@W5S1lHUHzcNWG#F`hx zX1(Z^&l&tmpi8=NLZ2dAhh0xv^CQ>NAITzNhOaT&W&%(-*rZgM7>z3lW~*oxg#c{z z+5rr~raMN8W$XKpMh*co(O+_dIwZ?T6Q;qHVgff3t*N$x@kQN@VacjZ@Pas!oKDKmh}Y$myCXjY53Vj7kTQ8Q`mj^10GEB#8ANoml)hKLid7R;`%clC_l0nvEq|jig0C zmO9F!g;lJsA&90htWY;ujQ<2=n2i4YKJzSghwfJ&%dC%`XiZLIXy?Z5J1uEC-$-}j z0kmbj^XLJqW$XJYeP1BgI*5S@4P*Wg_6^xq`s{jcF*k$$IK38OH7cIcI#MjH{up(L zxzs1fsgM?~%%8&!=!lvIf1%c3dVutvlnEMW#+bZXQ{2+G_6#gJNcFC`VcS9TVCAD? zd>t;AD=STICndUbsu_P&0jjfeVND(~SHk_2>!+=SrOnK6{FyYNrg4e}Q%M z8TXU+nrBCPl7_!b0FdwpmPHYlR9@_rqCO}2m*0MxbxxV67y_9AaTdZD$u@1HS@$Uo zPnB4lj!81pNxD^V#h=WjQmH&UGgz#K#H(sa1|8i;wH3VauvAV}f<=Q~lj{h>bQCSR zU&TZys#rQ1TT^78kny1lcA<;huiAx{KPtpLL?w8n2nQjb>*sU;I00wOtuEi?TNNfn zwa{Nz?w%w+0n@*$^|sW}4DVgC%)t4Tqf~_eCowWpfc7!-N|COcp*+-C>H>N$(KHBPvJLWlS|`=&o~Ea_f#x%k$m1%r?f#`*O?kk zij9oQdI-r{2LF|@?B40+HwuwGbAjR`iKerAWUNs1ku!kGy|;SvkNeL5c&gfEJ9pcK z;2cY|*R@m%&{blgK8m$!XuNNJ7RIf;9HZM4JaP?AG0XrhU@4C!j~4rvIY zXfiHCp^i&15I$h~6Z$m~MbS6?AN};=7mrevduznapD3EPko^8MFVJiATlB?X!{{mY zg&U>SrQKHkx*ENu%VT-}zLdg$zl(x5=DZhKl-SlnQ4 z&Cp_e&RxwR=VDoQ&zSdm$+3LuR)>E-K0o=7$TQSG&cOE6JwADoj5_Lk^um*(guwoP zCWaL8t~r!^c|UpGg;@EWj{ki6uG;hOr+&WEx3T%!*}}*Mr@{w0<1e4QTF+_Ix}T*^ zYbJNEqnsSOg8r+cq3GaJeb$SA|JL|>QcXyIQaN>G_xgT^qKcw}w#ch*^6L%|w~f-?py4UI5|LR%VJ^+X@11~p1)LzQg#sG`~ivG_#DOpz#7cPrO!I0ckJwU7jc z=4-EBIJlje60M^eP#N1UY=R3j)$6%s?Kp9FzFd;VN>YId1@td)0qGF8U`J>an!A9; z2sB5w$p{7*^{*Ix4@SKi>{vmz)%+-2=ZZGESXPF^feFe$129=~QvIE1s>Xs_FeSS$ zwzhh-9%jC!loLw}GgL~;E41K>5(M;HL`;JO%~<_t<4C9@R|2OnGex@EhZ-ttH#cmD zL@8HTVii!_?jfw*XhI+OQ8<}YfeBe}0%OG1IgDkhrLfJi7uj*hYc!rtD{y>7eZciz zq(Zfy5X8myT7gkiX-Y?|IU_pUYfzQxVhy)KgbEkBlkh*NTNO#vO!M9bjpWV#D zVZBWnBzuWpt`}Ca)#Ks7qtH3jWu1W~uu~yXF4V_4*|n8?Z&wBm1V6)Nji1#-)&=^( zaDh)!vVYUs9hl*fm#%X44~TQNt#vsHlXtXQUIRd-f2xIA) z2t1!j5+LRc+A)7Hy~21BQm_}dg^OcE?({2_MI!lw2zTt74R(z9fiK8hISqE08gznW zpy~#D?SXJ7rp~(j&ta_r_lpB#J6gZK17=)UBa07WR73hGyR5l^o(C4~yka)heBuT{ z+U)M?`0Z6vdRIW!o3JK^W!Y5M>f7XFy#;dsq`yRqA}MWQekLrQPL+CfNGDV*1|+$i zIJpz!YkZy^Wg72J@Jt`fhu{}CR;DW3*C|(s$~w)FW5A_iQGxq;R%3A%0Co@LnGG&h z(28II&Q-I>kNGxIq{>F%Vh{60f(}JaDak+}fv>?Fq@eP0+<`gDxS^0J*SSeS_|XQ5 z2OV?^WZ#({&0o$1+}_T~e1!T00w}@awO+QKqZ8_#=xCgvgdEP!xoZ`8<)k=a&ETZ1GM@mfi<0^DwM{n21yTs$2%U zjZcn1oVQG+C{096Q@Elf?vH=`y?XPbv&e-T^;i2AP9D5)=-waI_x^GGW^&%1z<@8M z|4jexyPqze+;aN2O&88=y6xt+>2k|2=bzW^72e$0GS)D01+MH!U|j~ll_`*u9dv`Y zzPvCty8XAFB%jX>w`XS$HM|U_bh#Bs4n5CGOpI;`rvVRFgr~dv<%VJaIwSXY)t&Q` z`NZVixg+Q)8U)=ZCGjg6G5xddZc*Re0g7z?tI50h)Q-mQ^PJ;h+@aI0$Qa`f_Nd0D zqx-?@(QKBf_Skd&)MSL@bB^Wj`fCZeX&WeMo6QFtN=9m0p`Fl`o9`k9M%vC*gyiiB z@E8xdQTnrA?(H2voZof)LRZPhYdh@U*PTlbnCgj$3n+EFA9yCYtM1QxKL)yB6 zZo|`k`*)oD>hFJSs_-t5fB5mpIS*HQs(%^v)cO;x^;sv+Q#$_oYJA_dKga&|`@NUf zuHT3av@g$(UX3a7_---zKir|aZgu&`DC?m$2NK8tPHnX5ig_GVuM4^b$~2#MB(zq~ zPFvi+C8^;g)jhKJTA;6*W1GA%zBo|w074q;Qg#9YKc+1KqcqV>CR#aLTEL?8228L` zOg6asXDzmS4Kq1LW>=s^p>olWy5Kn)2m@HNf!)*#*#5N~t{}135o^+7%)Bblb2t^J zZDVJ?iG&#^=TxUj`nTl|dTES>YhE3j)EUa7LypeD;B>@*&#kf8-rn5872&iq?J3fs zgSvE>ahcpX=Ffg}65m&hua;rlU1c8WafEG0L6(<9F8(qgRdBnnoPc^fQg<$Lk<6n9TUJ}u+Df2Sp`qvq4n@{A*1C(&w$*V_$2g-Nn< zlEn#`2>^TABQBcT!5Lk}j8t)Z{uQits0xG-$O!Y!-l%>8f0=^NJuJpA1U6Z6nslG` zcfQVzPRSgglo==p9xYy0OpL}!;2{SZ`p|0@O{~uizb=u+3l(y(CVP6AWHq_kcveRf z2%(}D>z}<1TBrxQbFBr$)Moobx$!Qx6`9SM`rKqcDOCBp#guHG2QKEz<=AG@gWkLY&mVj=z5gQCyV_oQ>=p=jho*IoD@fEfq0B= z=mCT?ZpXrpkc@ZP&{S|i1{}>FF~BBZU8I@+a>5q)OeY~fb0=w*C7RnDs-mbDsUul} zk?#1Vw{Zkj2WIuJ!pya7SSDv8quoO86=od5N9_cVEjyB`7{MH)lB_oDH=V-OclL_R z9u9iu`-C+)TF+;j^kO1YN8ik=)nt$4vC4ver@lwGF{jq z0fs(OSFRu8(@(o7(Sk|ebXCDRj1{Wvz^X(jXhxvj#;HwPfBgD-K*@uD^nI89)8$*| zzxtbW&*^Wr-_1Y%w^y5nQfnL%qW|Id@9%z%`{CQRO}Bp7aWUa6<;RPquap^2L;iY> z7wekv3yYbo#>Xx@7BgF4R39w?z?~$ABH4IMd-%a(KT1*-t{D*k4%Sd4rK`$38-q_w ze2I;2XJc`x$3l4s0pH6r)E1npido)0|4Vx}{NM7)SZ8wdvAW~G!`%#e?}GBOcmUAO zB^n>P6~;3aATgkSPvnL8#6>_*<*9J zfA)5@KjnP#YZP4PIjJbAXz2yZo{^->7YqGd}cKpy|ktptOdJq%bLz=ss=M&kd0IX zp3SQ|qE1Hwz0F#sWkuzocd;0hn2zaHnDupK0Hw;$#}q-*+L2KEj06JBTti#?12i9F zedPs>D=37TZ#N`Tzo(g35C$IvaJHy|m1_mcOsY1 z%vUuqrdLNsSiuyp;R+wmZST&A%~NQLrZ)2pn2CYsCC5gbDR$I-E&;$(xdb`eoX5@t zvRII`388O7u=H=W*)Y|++!YWxmjJ(bEC~SnrRT&STVSS_sEQ;$ha@c~l^(ot2~>U; z>Ek3?ApxQgFpU*L#G2X3mR#^Et?Qf%A#u8uUy;$*G<%NL)>UC$E>^Y~VL>+ZXiV4y z;YuqUY8-XJa^sOn6c=O2=dphU(r+fXgnE{}V5|E{nUbzT@!Guybab0EiLJiNHj-1Y z7T02}(j#TYu)gxLycJ<;XM2$n;N%QNwC-&va*eVM>#G@^%P>(YGv7DbP{U^&8xT4j z0c-sU%(Gc}!|Z8L2$cD5Na5&LXdFxd0$^-IdwQTO8sC;N8JXZt=%tcWToTGw<_eEv zcH0oIwHm`U<>7N1?45=Z9R9K=B!L(&+cwav5}G#cEwyDoNPHUW75lkH0s#Z{A9?YG4TRXSl_f3L-s-i zgNvMmT@`J1f+HbK{@CuPJNSu>;*J}QZ4!(NbA}It9ka|T2f-6(F&q7)H!MWl#$@83 z-K?sq=^|>E{q_~c*AE#1;2-ONUyY4pX{4QoN|Uy-GK2acul!X~v(J*tXVtHneXh^g`eGi;Q%MO_@>?H<~9cxn3h4s8)^;(fW~sMHcd=2f+xIomA?ab ztC*-qDLV){mfFR<&&93*Th2JNUWUc9Axk%e#UxH)+2rIrHr;T_XA)s5pqXDcNtFU` zH-YB4*dytN5&0REv^2p$>juLUe8t0;EPXJ6>ZJ|hmox&eT!=IG)fLFbaasid)HS0Q zkt8z`L!3NU!qN>f%W}9GO~pROW?@&J9|k07*aB2|8^*=oVM$3C_^DZgTX>^bc!26! z`FVpMxod7>BCulhVM>(p#nYy!VJkSP%{@1msvfdq1NpO<9#Y5v+hOp%UEC0YUOe;N zZ%x;q{ZLa>^|RpK)9)`lKmYqLG5NPcVqV<7efz5o%>#3n<~F7N{NMA=uH|muoPPSh z-ACiI%RL%;-Ohc}lC)%(2g+eLO*OIh19~i<=jnzXh*In0+-x>YyOlUh_;oENKFJwh z{56JSv4~dY6XPTqY3}R07Uyr`+q}Kqd18dOySFF!9a>Nf@(~3+nzl2pV>=vN&pbX< zUYqoUx8+pLVAS=<3v=tr-2pIbZY{SXt=_?FBt9-ydeSHTSwi`qQohJ{Jl>JwRNx1I z!5IK6w)>(u?Lti5rAuM*@6%X^DNVd#-O~LmPs-jyXE!%eME{%y#7O zjI46K_&zmYDDlbejdwX?OK%6G2|j0?eXj3r-do!znb-U^bz~dYqhY>-pvU6`*H!YO?RMRK< zr1_}v?9J75tp9>?Lb{1_+iCW45P?b^-_7H3enr*%YiMA6zDZhRW~R4^Ys}is7{gP9 zbsgYc#$%Zlug;P;;Ef|KLzU(Vg!KV}+iq+U8!5pAO)+t*m{h&aPT4N0U>vLv3VrcQ zImKCTvjHMRoCR9u(@yV7$e29Q6CqgaUx;nM^Ee6MBS3hAVwzE)^$IjzJaqb8BAD%8 ze50wmPDUdDa7s5Jfh`9PL)px&JP-@u6z0qZR+v7)aSyl%{L+|ZoQ%2^g(m08#t&IS*30}NnN1;kz-AOKf^TJr&bdUNGgbSM=BI?|CXuV_Nz zW*%T$H`A716@mgR#*NZaxF}h@(K@{lzwaXJwHHhB`j+va01pz8sv{%|nS|@lFLk0X zhGD7U=IhYaw}=kVJieLkQ*C`XOr!w$kr3d$g7Z16h16_UP6?SBSOTEu;Nx>l&obP?|9w2rU*Kp=kF zshDEWzHLuA!5WmZ2E>-L;TacMyi%~a3AggtmZ~PUX+0$vFE6b$)!J>wo>Qub7~J$$ zN!GBtH3_kndrHkz>qN%8ULgM!F0DpOyNSADfb1ih=|h!_mxPE6kW3q(Sp}62^h%S? z5A>EC$7UFJg(t^aN2i!oxKcp=o5PSxVJ+d}f0NWY%3ybMds9-PQU3+TWabp4X6%%5 zWNg_)7HNz*=|8p(S$+cxWeqVZKitY*$4;?)O&=~mc*3D)5c`n?!Se0iRczb@-xAk3 zmpGNLsqmW6h2803X33QB0tij}SCfQK;LJ_PG9TwerFI1t6lgPr+7T*N#x$YB*LM-fKyAG~9tEdEjG>YqowN>j$%u&qP@9W;Qe^CZ&zt`Km3O!A?jKEgrK zQQ4x4N#G*V|6ZFp*~=czhGrTbAeN36I3m;!Q^1ovmy~X5t4YW8%E8o1vP5~n!mWqT7&}j67F!ek#>unW4*kpm^ znBgT^l&m#X=P@LiBbugsu$15m7TSj)!|m!x$g4j(st{2F<2V{X$g zI|Pp|#@yfNu>Iu8Eho2J3D|nzUk?w(A4xy?!z+&a(R}AayZf3qv?lRNN}$12XZn&3 zTZ_zfjzthLgZI~hId=RI z)WKP&3uf2>Q;)3{5(NI|9QT%YqfSqrtsC33aHKBag21J@@d~{4ea|yRTpGW3@#=%} zjpnOQ2j#CC>*Vr|$v;t_CK^4(2U-qQws#t2zB%ta3&Cn_VXj4+1*fP7@pWmwrG4M* z6&`@ZJCKw9AWzuZ2KDcR$)y&BdqW!T9_N<=+vJ$$)6T*pMg0>?MbzrM6S0uk!faH} z!!zA4-_J%IEIvy;OLc%YCw(tFQBzkCd!54Bl5jngNA(2E7xS^LXZml4ys93V3`wbJ z%B?$>YoGAUbvJ)Q>&BYHwFAdW9^O0OKk$Qm!uuT5DWvD+(D-{<3zrfKy;j=nJKFY) z`fwv>)Z^o$o0raIm2AhXRtITr#uO}mwIw=%j2Fg&N~6vd8S{8n!+52^-I zwGC=Y8e?`lJ+vuhhE9^|8URS+4sCUD7-xvDMV4i7v4mlxf~o5zF84xB_j*&$SwZ@M zJ^2_0ZE%40L`x>MK3Mdgzns-9hQgO5zC}=UCN@72&Bcon!aD%Id+WatOWPqVGf03b zlu#XDz087Q(fMMM91hJ?p955Sa~D+o4rVQY@fbJ@24;msfkDp~6}J?}?ki=gKGSYX zg7J9$B-$#1iPrAfL(PAV6%sL1oshY-KTOw#$m%%(Q~f(_ zn_`QiO-N=?N<}!0c0VhfY$?79h6+VRNnUxKo zOZI_Pu`qPvXgNbV18nVjC(_5_;3(a}a-UB91#ag`sd&{*slV-FTthI|z73zwFSM#? z_5I!HI!mH3O_B}!%gL0DmOBINGcT9A^1%)j=2dx61SJ_IO^PAI7h>t)!rT3wygc&& zS7}|4xS=)ik{>Z+4WKEvuK2P^0dr(dxT3JuskK!!>laV+3N4}YO}1{xdlEvzwJ$Ul z55W)}ZolMckd|Mp0m={N*HLpTa09~X>SNIKIJtlx()*Hg-ln}L$H*@OTTH>I+ZY`}L4Yh~K8u$8% zeftzY+}~GCY1uJ8nthk^oPVr<<9?Qtnwom-EJvYGxYq~wS8uo9o%7UDP7R_YDHGFh z0I3C4gb5InULn+p{@GV!KiwUJ@?^yy>q&DzWi}+V@7@<3YjjRk!_3 zr&5_`()N1rUsUhAGp8m_MBzceXeq8BLz9)i$6PNngcx`l^~GlsL)XN*x`DE9{m6d( zRS~Xd$pB;)VV~m-3eoTLFHiVKy@GPCw!cqWamUAb&yJ4QooyMR;$w!M1HQI> zZS<2@yCh>r`mV*?twN~}c?M1$8re-howN5`PsTg9!v!HOS6ouIT%j>T=8nCct}cr7 z;9=e$acHM5H8y|0t4ekwY{77)eDI9;(Zx+IcQ0Kl9h`T2#t#=Hn(Np`3LCu94C^$; zy0Sd~u50-_+Uk+C%>ZlYXVQp$6@^42#HGxK)O%Dh#rF~jnb&gg2#{P!u)CfTNl9%2 z>@9z$AhvTZ8zfd|6h3_5CIEb_U}6h*(xWl0KO0IEI>W8gN^1{D9KItXv`N7DdL3=4 z76X~WdlNF_jhIs8Z!A`{8<*-odl53LgNfC(2u&Y>qFGN5B{UDzdYiwFZ$ZU4I4mJl zUnBV*t(p#+`3{h~dZ~aWK2RsGH0Gi+M#Wb1db;2c|AlY|&}(%)V3=Ze^FvdFX>7BM zon}}o5ek`3mM-ywz*JVMxSTFd_j}`r4Fjm$Ng+~<5C|E^28V>KP?UA~6Q~1G7H6$U z5{y-11B0~IQVXMMX{0*x0GV0!zQ1!cTe-$3q$?evRw=zAbJ8?yF=)M9K!r*>KrA6j_5J(;y1`3}5PfeMi$M7uh?BbrSKy*8tQF!f7%Q3d2yhnxE$P z20#%V&B0gSHFUo9N0?m2>MxYIl}c|_VdY49Zlu#@^@LC>C4@k z)UVc_1B%2E$5D8h;L+fMm{kRo^DF<^K z+a8+SX{F?yh@%fUp6ExKiG67A)HWS zWbj+XtC|4oxL7WH*`73=5N&N^gocGGd?QFLLZ*Tds>T|zY#E$oMVQ`;2_&P6ZK;J2 z=d)R6K3ka!P+A&z=n1(<%M@F{5;d|npmP-9$m%wF(ZzU1cO0!Hz0V5Moy$s`P< z7si97|I7fO#HE-uNSvus1YWQA!d@h?3ZB&;;m}hxPU+NAh|4yaLv!_BBt@vRWWV=f z4@$|D6tJyTVEjZS;K#aX>3S2EIhor$%z;WGowqN-n`zHY<-b)J_Ev*1Dh>gAwl)7sQcT2O{XsGf7NvBt5Z$o&PP%9a<|XNkHpl{I`QWwsYe7`r! zd;_r`tl3%{uYdwB*;UbS$JX?G&vTp%W<^f4&#e1d8wMqMmbs3z_%rnV;{T?a!&^1) zyB^11EPcGK|5EGj7yZ?>b$3tH+ct+}Tqd-vCigAg%ML?E`tYQqQxWcWt4;*@g}@qp zKQtfkNUJcojX88(Ia1X!^KHAk>$fFabH8==DG2nh-cpL%Ie6&WlZf{b@1jRD#z&u* z{8YFzQhf~nSWt8GMp>ul@}ZmfhqCINN05WFhW}P&e_I^=Nb=2fRmZ^9iRcM#{!}Ne z>4D#OLB0`cU9}P7B-4keys^W=EOC3% zwZ_AQ`9b*k)ZuP5su%t_wYxIe7YM`x7lI*2revrq5M~JipAFAynypSD_iLM-5Y$xt zJ+d&_cvP_vRnPUrimUBN%wXY^XGX3KroxNv5bWTPZ=+l7R}(N=TMUY~Hl)<=8_dxbf3Hc6RIEpP0-c%i;a4MQPZ3xS!+d`F4`bN z*wgmDGXu_fd_^0rEVMp|=gCyb5M~C_RK>1Lo^(%{*@|S#G$MmWiQ@$Z_g6K>vj$Nt z5$JxebS$&-g$(WcFAMSQWb^xBU;X#!xhJ*0s`;X$M4nKd6PqdvEON2jmNdPYPN-lz zix$UcDx4hh!`hXhRw*fCT1D#1U@ygvL(6R5vrXmGJ(1g-CXff=%(w={>Q3Pw-5-Q{~T^dyQ}FO?#4780tHVG%yrU#79a4Q}T6j#fl$K6q>DQnp1ZKhCq%#iEb-X#8Uy8i?R%ftbX5P$}SAHn^w& z(fN}<#1C8C@TG0t#KGr#?tFDB_iz8(y=m9a-%`)nwz%DuM5L6~L|hGlQjetg-0jK` zhBR!9{C;wP$(&Gu>-12dRQj5uw$>jDYL6lbS5ZXkSNHJm&jZ7y`BHTR%2#4CxgWRsM zQMZHTmwG*`#CI+VH6w%m@H}GBT%Kp?DXJo>CH*hQ>W*KY*UVgWjQQ@)fm>t0j*Y9Z46DTLuUTNgEd3xjJXHtSonp#RZa9M z=Nq3D>IGACS`LQ9lhZ}Q>4ueJEP6U5asH+W%4e9si42-%rM3l1d>4ow+@gmIJZzD60sz>wQxiL($W z-Oi7#VThF(I2csllKy6X5Rxri5{ z`Z)Jz#b3n=4|m-U!o+^l(nxnK_A22)ll72XL};6AOHhioPc)gG#EMoGs7!^I;xoMO z&I&0^p$0J4r$@`BzN&RBD|jSQD>G!bbQw~(ajH^3>!p|!@_4@9*y&cn!PdPMs#WGj zd$sUsV@PF3tHm5r&5|3nKbvBTYtk)ifC_p+6@S8)oeq4 zHr6_Za%(m0B{LM4U0haanj%`FwqQIFlT}!VBaiC`HY*JU%0#6j)>{FqIJ6UCd4M?7 z3`C8LT+m7UHOW9HF;e1yioJBW&`9nD?>%d5Csci}T;io%w#5*e1q4kn=~irlJ9q}G zSDXM52Vk)8ob_I5glzQ_8t&GDCKun%#B`6hLY9rCq&O)^5rHu7EQnMPqJ zG|Y2&sD&WHKBa=CYCWxXm^9c<8d@k`?D72x%^gJrHX)wXfs|^Q|C{WWHn`n9+qrb~n zuq;KT&&MCYtgG2fe^BPJbfM4#E2Z`#hhQ3j`$z+L42;PZQ_R`2BYx}NOLWGo0DwB$zk zI5^bT)X8D>=`K)5Xv4QQHvRr~kOGk0cXd$EY<0G^cm0k1nnMMHt*8^z#0Q>Ls6*m? z>OoI&y&Ak;7vOZrNW0YX;NiZyt2f`B-W`4M{Xm?@>FEBzB6yapIQ_=zsSSU6%9buY z-1am?R$tWcux|Z#zvl=0N!qk4zcK~(!3<7Yef*`G?8WQxe%p#a?O}ZDpC8+Ep}%XS zZuV*KNWaNT2jyvfvBnaqrYOjEMi)7HvX`4 zRUY2n|3+V3>yOUx$qvT~LL5DMw-kjq;s7T3knO=wG06TrhqB@EUEn~Mb3?y}99B>wYc%yo#ATQ< zR^@O^u(A6paG!Qgw0B(L4o-QGJF;(_D0iEqJMi7{pr^nxQ)pTJ)EKE$k0{mPd?6sS zc#cK}SvGMH+&bIUQX?@*=n6l}dF@_LZ*>E{fMV|uXeePYmyVa|UgkLYXj=Vz7q82H z%9=r#y0UZ?UaeWl#>u)^pt!Bj*6wt9k|bD?6#POK#AD(T9MRO`yg@z2GOF~JpVrB# ztNtW|AIXX(VTIWkmZT5cIT6cxKQsg)2BVU2`E0nf4h$o&Un{EBHXx+W4YAdM?dW-G z+SW=7gf31}%yg$j@sm$EgJwoZ)+c8G!=<&jC&hV&UZ`P!@F; zILXx8+ZrHRc^M0w0Qr7Ky9vL@)-tS_C%H;c{plx2#7!qgMNw9hrbpY)rB|pxMAO!0 zK|kn{KG+qm{*i8$KR|1p!&RU9OlFv#Ofxfr<83Ij59V~XWNYn>)T zDokWQGF@6}T@_WqrrCIbs4Kj<)}+y=4+DEovssHMBlVGe=T)R6XG_+e2jV6F_*2y1;* z1%vV_ADx^%1cIUZ*%4qb)j5k*)fNR!nYrfbWSOHYvp$`30y0t%7P6gE)^R2>AFfqXRlwGBFLt+NwcvsQBhRNYxgH=ke(e_ zto9}OQ7z&+u1@l~3oCJMvXRL*za)ou#JjNs zI$!hN{nyQ(XhfkMp+ti=&xx{OolN>20u=kQ1jRO;Kmr*bKVBNH$+yf!S%2v5Qqx_O zKAp>1;?<2P`>^%4Tp2^rre5odnVF2sxlNx&(JNJ+<3-iM}Sw-xX*$=w36sP83t827MxIocQyl%PLRyZG=tGNIjZg1 zb}5!~2n1>)TD(IJ22+4jEEFW$LOW7%xHbX8+q)Bp!KugU`LAD3$9U8i%ObWwAQPNp z=K^vY_O}Eh_N(&~TAvO+Xd?FX40TraUZuFWR@DaG^FMs?uf54-A^s4~%3Ev?aegA<<=l*nzLN8sCi%&3>X-BjT^}c|;*aH1;vNL`r^;%!=bY%d6z0r? zrln5iq;36j!Y@AUG3V{Eo8B?WVJ@hKD{`sfp5`~M4G$qMVuXiXWe%tQe0TgrGhby`IXL-lrwCbL#O%(NCGt|J^04QNOcW@*4@UvfBX&Yl3 z^Lm-HUaY}DsIFNkRAV#+m=uMU5iZxDgQgJG%OP&294?rAZ0#0X%-Wx_ z+JRY?6V-r{E5)V{y(SgRq_|}qVoDjzUIJTaM2A!w`9R9@s!*ML1Eda8YvYLXW_$Z4 z!#uzvhHm7V$jpIAnB^*j&<+Zv#D-aPho$GJ6SA^hTEKv*UUCU<2jKvnQ#*hKJp26$ zm}=6`*~N{ntx>KVRCjlVg|G&yXv|u<#;LWYMF|LHz2i(CgE==vNQdbdbk^JcGDAC! zk)jU<&EN~kR;_`K3};IH9%|yUX68UCiu^iSJ(&;LXNR$eX=0+nWoZRg1y;da@PZPS zWX>!@paNF~Zzqxt3%n6Tg=xeQ$@s<2+}Dv}q!vT&q7i2#Uh1o|D>+cG^Dhx-Qn%5>P9*PzL4qDu>8hF(j7!`*}CKs5%vX(7BY7VqomWOJWFy(^9 z8YwbpvuI;WIDt{goG4`u;FPDm!m zS@i%X_kQyXFgBFL22?O53vkiOpMbP%js@MB(2V|bY;`GHM_>a`kr~u5w)G;Clnqfo zAN+UT+9GXj2t2+e273%%$i-zB6BoGpT$n{5DFwt#IRu=~z|9_6J^ zX(WpiDFtWw-&wHO8!QJcCfJB8QBA<4j#DUv_6hC!#P&|5)(BmBSsK0sQi!gl@$qrq zrVWtt-38_DsxvN@Kj@+~T|U>*%iCMv<7qOovp{mJdYzHc6@woH$lcZbOcMuceJJ#| zxd@I+Txu$Adrne6AB z?DPLQdi%Jf@BRP(`@4O9pVRlVEj_I*Tc&)r)^V+pbtR@KXQwsKa>^-JmMD~-w7dYX z5D|gTnJ2B8vds$13sz3enUI;HBJe3fD!j-GM39RV1rY%e0lEIY`u($i+U*uMf)}s% z^Z9t(@3n@4D_sYrhKh>#q(bm&AE#mli}S&Mhx!Rmch}z0Ow7BM^ObE^-~A<5dv^uW zh9^R$d*0wL&(N9Gqefy=cee~*q^+*U-KJ>eSuZ{kw*KYGtC2g4)5G1M@Z9orf7_5Sdg!BHb9S+d%f32wJn9C%=0eBGkAIX)j`mQ(Z(l&rPf17JfZv-M z-fPc0+;L{{G}e9!z2!tm_&%CzKE27l@^o&iZV+By!y(T#AnBRLGt^?{lU4Fj+I{I! zMovy!*I2N1Fr>?zvCVY!EHaI`xH=Pu4(%@!>Q?!Urybh z;k-V3Rs8Gzg!tZ;sBrNR>Z9Aja$Y?cWR%A;@Z^rPvo|QGvum7x@@6V|zS)^E?LjHL zY#Rz{dk;!+ZQMZN_&Hf5UlPLALvy?NxD3+dcK`5z3nas=1u<~@5*kb}DeF1hv0%9V z6_WB&fTc*JLl6xVAt^337wE`dB>Cp|1&w3M zHwBROw^gylsmO!LqaLN&CwbMh9SIhLjLpqCSP<9?z7PP*6+nsY)l!F?zWH*8$p}BL7B}Z^6VT z%Pql)a35ieQT83$)sr~1F_(+55R$qvA0(0C}7k?Mhh){ej)aa|4-^_+9N&4IBc?o9ThVCdkR_)Vn z!MM*ry$`{`WKhP=b2Gquwz8U*)szQXOiduZ8)M)Q+0OTHzW1bKhWYK|h}GdF@WiCb z^1U_qoe+4&&P(Gu^IH~0XZJ=|7G=@eP=j=epbO*IlZq4w=g;fORE}eTg^}+mwlufo zAoukxeBN`}D8FVGA^L!3HQU-$K&Dj-@hq6n_=7U~{$932Nq)=OEQa z+1&*=5N1G34D#@N4p%vB;Ogu-T)XxNSOk%~Y6s`PhEhD|lV$sRoUaZe?Af?Y7mW<) z%Af@6yf^L}Ik=aHItK5%<4C#YBR)blZeSfIDCpn}YUoVj}NapZ#^CsPo(UMGl` zo+%wlHk-pyD4T{YeH<(mtlCN*E9PQI%@IniK_p9@39_3z zE}|&ZJ|D>Hnmr2>6DpJMQrXfuhU-PUqEKAXyhcZoUMFF3qYTSl2wqpmpcwJ(*~z#8 zXC*L+ZEMAVjmI&TaqxwC7-D5($h>FQ-@g6vuX|6w{_h9>cTPW3m<=Wx0fDsI+H>LxU`Hh9 z>z6MQB8QHKrk)Qmu^a?C7|FH>z$8R0u$3um?R3Flh%sdA2^4dN9&R)m!D{Xi7>LsX zq&hRVpD_RS?Ui1FKA%dUQvDafyg3gZl@g%I-YJqiep2|lVCT8crNbyxIIXfHWH;FA zRmRipv~yG7THPq0pLmmEGH&g}2M?##j}Ztvav^4?17Sq9dlNopH#1|JYrSv zB3r)i5_jL+565ewDqB*~uK_-l&@O3htbIMvEG31~e!m(Ua=TaTD| zyhWXo=484&|Ls*q$wvYn0Q)}O$bUHhR}JNZE#0b?uIW4X0&aKiblg7i2+X%L&Wj#( z?gh(B92U?!biu(u(&plA^h^?3ONx7H%-~XunPF1pUC5BXGy$;H4<#jun88vO znAHE+&C_uB;vB9oCv??JHWR#WTF?u^Q1@({+P5(WZnnM``Xpr8WSt^1G6gJj{LcUt zY!9s|s=%BVge_kKiLe=5!TeaevPf!_HuJFl1!0Xaz4hw(UYqMV(7*&y@6wll2cL*g z-by1B;VyQ32gJu80I0%VcmG zun6LVfg}Tg>`=SF1Nc0|e)b+I4=E9dJxVsJ8y!Xiv% z&+9;PU-k9|)iKR?HncaEjo89;4LIi^h=+vm&3(+5iO6r$2yo|S1yc7owXu8_QLc$W zx@;|@%J;!^X$#9%U$y%K4a1-5FARKLvE2YT;KltM%4+Z`oIEhNbUzpde;^5-O9&Q> z(&}+o3HL--b9cFb=4un3_DIa$lPvvK_58>Fb+&*0?Q<9Cu*moGCd7c<$V zqM`F$UXLm40E_%mdn^z&-H_ZK%A0^3T*xnfou@R!J|W5Lh_mrH-!O(QrS7F{paLN1 z2sZ1gF0kC(PnG_!lT;}s*Nj%hYG>b8FzWeRa=5| z%19D`&041IGiv{my8nBqdk|`1D)4jHlF2ryDmK=C59)dXx*h-35ZnL*x)Lc+@)^Mo z6tI=aJGu58gx-R5YSd2m6s&3U-$>W3IUE_>+g&%8eDmO@$z`i;wPr^+)tyYW{0g0oL0DHG9nT^noB@^!^gJ6b- zV|y&0Q{ujd+J2>N{QMq&vpNPKe(gw%{w&5meK!YUSqp-MrvX;vp4b0u{ospJr!s0j z%ec9HXVum70J9hwmD(lEqvD8<*-`!d@WG=Wi2I9u7xA-p#~pEnWaUaKgxUTb;H6ux z+^e+YSH{!SY`=#~(j8ZSHaIhSX!QG%zN@*iwh`M`p`pxntw3@#F&3V^rD_14 z-3e$BuO*Y3KQDEumiDFu-)pSYBt>8zA2bcFVDM$4Q+KC5WV8_nom-cgNhN#gf*AK&lo zHa7i1$;s#*=ss?C6A|WkwKU3**;|bL!FDWa_XiVysj2GfYRc(y%$2@5N76h-qDI@! z1_vGu8SaO4dI!;cf@*Cs+~Q6{>-(3_fmlaTPG7ySyk$g3E^eJZ^V8YCYIb&&>^VX}O&lC# z8goT9wPh=R{b`t2u% oz%RMiWfW!3C+4`$Ei{%qCj!Xk8aTj_KbiD*X#cpC?bP zZ36g?aV0_2$y?6>^X(3>dbYBuUI*HK481hcJqko@^y2-2 z(^>6k8nC;#Jy|pe!hQp?js-B`jbB0F14$mP!}dErWyc!r^wOo|0sSCwN9OQ7&jPvn zCG^V2?YTf(kkP#UA}p0Knh=Z%4ok%PoKGqo0Xey zws1%G^F81wdU17tGC(gCN1Myi?@Dr*)a7dK>0)GH>YD2`7KZHzEw zv+#b0VqHpEzhPW>R@!zjXlS*8k*dDW@mLjFwEQW2J+0*EO{%k~LE5yj$gTOrO_wJ? zyYV_rThl09DfBFQc&KEgXLL@zxE1|Gzxiyy9${YF0mK^uCdVg_}Ne-OlcESth$bvQ_qIN|PGQ*I~DAPP^Z` zo$7{B%nskCRJ~k-2_x4({rlW+m0{ogc0IqZ<8mgdtL^HDEs7qeS3Z8Zq4`WFiQ z3)%R{kdQ~`;sopYF&|91uAC2-=3OyJ9>Y^=f#xZc2?pL?Z-qb*PY~yij=?*uU z!vh0q1Ly)UPTzGdsc$(gYooe8Z>FnBnBG!C?l{}}&5d8DjxsAR)eqkxaol;Y1p?c| zeA%hCj@E1YT5nyyki5f+CH2h7^n|{Z?y1WW8GYM&E=*s*u0AFn&IFBwY1xmI&~K{0 z+<)W~RNDpST-ivse(iBcsJ69Q^NeD`yte9wE8=6}uO+7pCtjm?bMYy%!q2W8jZQfA z-`{+D`tY~+r?2m7O_EAdGG?HECc{pQWcFl!^WFB9zu!KSWj&c3WR=aN^DZ1ijev^A zaC)pco)6mRsRtr80a0iO*mD?)G;v6IoK*k+hy97NWOky%gw!oZzVxMy3;Z#tx*%i`Z;2 zV27|g?(wa0RU8hO_dyHn9w%U5P2xe=s%=Zm#+|8lB*3~MV^R(V1_b(6SP+62FKy)6 zH;&#EF^^Am$PfKaZ~#Qn*c`i-ad#YscHUMmY$mF4#?+yFlgb?)vik!h*T`&O^0EFa zToon;oc*Y9sF5Y=GMbhznhTKSE|$jxVM`tpy@2WBIphF8D5_`I)V|wl&>nPmUQOlA zwm`rn(efuSQgI>N0tDsFs5H&}Td-+BV+ZF%g?Y%9o+z+^;34Qghy4>^g?!(E%^iP0 z!z%(eD0^R)!n#52&u}b!RZYy^Rb@+Hk+lg3{b_Nb>9Q(am&|iF)5&X!BDM9YQg$#7 z;S3gtht5+?wI!a52D5LdF=|Hb+5mKj0TK;4P~yAS6Q>-qqNwE#HNts@fKl1E1li5O zIZYvx0-XBLTdq-^OgO>6BF9$`>^Z z&*B_|qU%CZ$`B2gmpa(~1O#hKTqk(x#1e;MJEPRF2R(nkiooEPj+3L?*=pzevFq9l zj+@YE&LIy-qw$U|c2oJAUTZ~}>!1EW{+$9GY2irJohk5KGjWaIlz3Z`M+jHhw{lF8 z?L~e3$em1vS27M|Lu6y{a(f`Pzp>J1Z|{xcZP*9epL*SG7SHgJ<|hZ}o(%x?x?>Qo z&Q4JsZ!bZQjN;%cT0Eww#(?v1G5trvKR~*sdO}5j_Ye=>qB*}yPj!F=rY?{V%1`P7 z-Tb_83&Qhh9$e8#pgMx6MXUFk+Ci8!jz1mm-9UObUTMPOYY4<~Lch;tpaR?)0MU9M zPf63L`NdZcna3Jqdqey6gGn&O#@h@L8wo*xONDnbkTi?m@*h3Mn(blChLd+c(v6!z zl0Ig>6=OS=Xv_s=g00rku4pXB$mBrbE2r59c36X`deQC%kdKV!PaG4$3!q} zh=T0yfta~i-oOBNRxW#()|J-L3sB`zM-%gQ?m7)RDm&%>{N(eKTfb2Y(|a(v#iH6# zHq9YQJW!H2+Sg|ownVAJQSH6N>?lftY*z28i9+3)+)`p(nN7L%WLHW=NLNZjWJ=%C zX_GNQ2ll4+<5Y~PdAuclH~M0)_U#wRK`0`4o4S9f1)z#uhv#n93BJhRfB2W)bN|_E z00N+ghxY%G)^-13+P>D#G%4Z8zQ29>$6u3OSa9}QWx27P_?#S|bg~lp^`oxIgW77| z%;@MzHJ*&&$nm`!Y5emeoC}x-#XX4+i_iZ3W#+Ct((WSs(=i}13@CiPG?AUURa3ZD zZ)o}SNaa$$90Gq)Nae~QceV(gnI>EpQy_iwZ^ZA9iVv?&1%Og3{sQEw{_lBdGw!FC zpI!z>G}xbBP3hP=0UXKa{=V(rXnIadL*6ZC348Vr<%3&y&l-2`Zh3txWvVs(_2H7D zK_=EAC3HI1zw)~vzRW|2nel~MK+LRSliH8Kb|ZMiC&#=Bq<8sl4sLg)OA4O&puT!K zoa_Ol054)_Q9QH(9#gZslz_@uvcCDPenkv-39($`KB}+Xzjg*{dznmbf|cK(lZ7Db zaJAVNr6~ZRwnvOCaWNELX)Yo<$oaDsZWAj2x2!LIPJM2=%O? zz;iM%NSmU~q|Iu?>b<&l7Jt4Hs61X#?ILP#l-w9(oeC>i$!)3GH8XW0)bbePyatJ7 za|n-9Ri)wtsxOO%AbMZ5T;{C@^Pb-o+9!mP{X=8%jCJi6@<^QD0`bjYJbDei({k?! zY&bpjK(S{32(IkoJY1;zm+{R`SgAB_dde%<0pf5?NK&hG(9Tm<)HN)1)2=w)%y5ij zd??0}A@#2?OloIzXE>b@!)qd~m+9`c)Il{353?A*8z9@IaQn!1e>rtVr}5bd{ zTc#CB1|nY1YnJT^)ytt0`)6?eWg38m_*XnJRV25diJ<(r=gd#ziu{CIz|PAvh_%|8 z!sEBz)F0ms9(#(i7l4$AL-GAql^eV9DZ%H&_&0EVZw6*b*dC5C9c0b%HIo&Xa@{9c zfiMlCScC{Wa^NR8GWT_3`bgD07U+`&D<-7K|Dw# zJ;Ls%l`amy3Z~WC0lNHFGHI@+ir>LTcPw@|SKdwvRJ|zk?H($MMcvdoi!>chZ62SX z`?e<6nuA%m1BPZ&vIa(d--yJyhH>owp_e)7M)!rorp~p!o zU)#dpY=HoCR5UnGRi)k~}^!}7auM_tPjjO{Gi){6${7Hn1oX*GW6 zMFnoacMR?R7_#vj(9Hf47@Jxv8drE6RA(STjW+|EOO6x(Wxdl-FmD8LYPDq!>^nuZ zMv#sLs#PGl=a@Jdz$xwfkQN#c93n>!INapgz7D&0nNPvWo`$if5 z%S_S#{}#xSsj^xi;82-ff+t?%MWTAh%$WoMn(}2?5tXV*keWkwV-K!_dCkXTYVV?y zq@lw!FQX>={?E^;_kZuY9?{y>c$8U0{EC@;{6JCalh~c@y?Nhldk?^XnufRCYde>< zE##Y|(`7%^jvVdF`}*fQzsp71M1Jp)=z`3mqO6kDjp4ByH9~D}2C2os!K!TJp!;(C z!KpWgk9<+p6ey*+Z7u%%fMfTk-bg*nM?(g&#MMZWhIppHgns9f} z@gdqJ+WH-{Z+4&P#^Qw6?-Zvesf1V&TEuGr( zu`f(lbFPU9(33djb25!*O@vKiJDV#5Q zzx}uC;oehGhazO~0L{cicJ}cE&Es7&21(=xf;YqaL}$Us=#p}#^J;JCIWee>oJKSdHfsf2WjPdPS0jTO}!Xgrzk8dY?CRqMw+$=NdO35Xh|6`0-V&Z8`N2F+L z8O|>6JzQswA~}F}k#&$ZgWm@FU>e}|MCb8N`&L>0g#xN09xTnpi4PNu_UPtvKq>6x z%7Tr#P>NGZHGr%vf$9yUVG`+mC5bVn0%U17n9n2{XWCUg-DD4F?SS79(X}F7rqv13 zb9{dl`DFule$KDd_-6!@w?+P??MP+msE}kY8OQ9Ji9?bmW#i5f8u`tMP+K6?N#OhA zsE&V&)Zo5l{gvtur1r}Bo9WO3@qXTD{P3ru&;h(zV;<6c=a#}jK8W5G2{pT_SF4@AqLziEeezG*=C8^KST3(^6!qO z8~#bPsBF1Of;Zu5=l|%`e8miK^Z_me)&94A*>}O&&9Z4(U^Cn7o5Fb98A{8Rv?7gV zX~k2=DAXkdqF01i;!8oH`KkjX)fJY0QgN*J`j%0tU^9@~jc7V+Sie`5ST*%5gz&vI z6x9enSJ+2c$I$qeV<_0PB|j!_cQ{&mOb7VM`eA7*Tx0R*%}e`-7S?VGCj_B*ff2!r z)?f73+SEQfP-#mNX#Msz{Zt^ngi`fRpIEk?_@(RumfLVDRX z`qJ{+7Rs_ty=cRbcFG|j0@W1RSENg$N4N*tEzbuyyKN5eStk$(;GfR~NC$@|gfrl) zYz{}ya)4_#k}Bo0)_#IUo5jK&9gAza=S&BOyDcmXX#^LN(7{8UwDMO~06-Y)+rpb~ zN9z(WULo4Fmy;4|yB?@!9Dk-F`K`zBLi+|U+ja;A5MsD*_)TGd98_(#QM_v|u zguMFx%aJcuGEaWZerO$Kj!ci{@Xzi*rCoTD-VQy|mpIl^)^~wjdvFCF(dQPH5LOq< zbfevjz=KJKFv(%U?Pw&zLra%~7| zcXVttFrrLQ)~a1q=gosdqokNGZ)bk^`p!3d zo_u?7;+3p}o*Iv7iE6w4#ZMm|``c{gnN#P_A5S>*U#}n8gHhG8M{~31!TIcc)Zp+A zgC~OA?(PY&42adCfCROGJ8g5H!s&Cw60j^5Oir2wswz?OZCSzB3fMUb66di=tUzkl}EfBg8r z+kV~s*>4~H^nXsZ{xBYNZ?rH&j1mnEG=(v1G+D(9DIuITEF+z#>`S~gcxZ%>!k>6u zzM|{GycsPl>SP6@bMXz{2gcFL_uKt;3lcCivYJcWz%cWox@l1RHIg{l1!jcR-U+i; z1rku^7|=l|1wa4C=fc4alj^$_hOmXGd`+SL-`fFX)}y^<-O{Q(dJuFE(es;QpkfZ7 zM=wArpi4@0f)yePM9*$ULwvuXREu5e)xGGI%K*#K3`Qg1qit}bUTpXKAil*w|H4+) z+Ezj>u955Z%nb<|VmGuIs5y~;mFDyGMVlg1x}+x(s>EiR3xDVzpgRliKo7%9#g@cI&?x{aHl&Y#pW0EWx;^ z)&5s90I$y)niW5-ptz|~kF+)Qw3sU}yflow1m{sx8 zSl8?`*(Jl^_ANR)vRU`V3B59s7qm#c@@b-SxN>P zQ~10YtWyFDXQPLTb9+2vP~Wh#nqyenyaR~X^l+2J>CvK?RL^z>Xi`&4C#N9RwKyL@ zB&`HdR50{lzN3lV#PP1?!EiVQpr5kbgShfC-R235=RVZ>gyuD>Jq9|_B5LN4%+CU; zvwK2|#ryd_LyY%3r0X`uEugpXx?cboRg%)cWwLEzi)%)&VIpHiOpAJ zU$uUNU|E?5v|jLkW^k~^;<-8R4Zsepvo>w7mfg`dbkD8~)(mz_ptTavGnkptmS;pX zj1Dq8dkrh8CizFQ*JVqR$tqGF{#NH6d-(U7OS=^s0VzXKtv_>ByfZKZ{*&dmjF2Ux z+I;9QQ{wy^e{H?ZEI#dGnt%H2`o$SSnlb0oAAf%Qx%B5n=JbcBKg+(AvL_0k)-<}3 zd#;ZV(MKEARX-UxhRL1o>Y`-yjKRdiCH&b(%I~X<^3mlWIe2-*>EnNN87}wA<6zZH z+7}&v+xNvsXOnWKj`l|E-7fs3E+cza`|f*xp%)Yfx4mjj==wV6(8;}$2kbu#BlrqA zn^Haai}wyEA+uqGM$~*iT)whHhU3g#sm{nNcw|fk^kQrGkyv6~qH1Rt_NNS~esk-A z#G9!0jDjC!i4A#~?0Q#CIkj9pU=!JIX!vVI2yj4_byqC8Ht-_ckZ%};9hhJ2 zI1)65DYr?95@kG{0&Mcm$!lKl+-7#=FpsB(DttGin>`MssU!ldB2E~fX@eUt!ea9> zGC)asbE{)qHIXu4xerUDPIXk%0~o%|uvq(LG+5nvYijex-j&Bq6Hv=elKCuJb2ilS z9u!}WMmDNuBWFev^L!f)uRH^@Dl~L&M-JL*w>R|Kl|Y{<9o<1I)6+{=1!y}k$tzAk z>~GmbI-%ccZ&w$YW!u}4vG#3H=VE&-bL)xXC8%$~5=Sq=ERCZ5%agt_yCu0?lH80V z#F2ue#x*mDCPG}-t5~!mgzLC-6wZzi#G3MzXT3%R!!%%@NE* zmk%WQ)Nfcw{)P?dRA$}WA{G?IVOP@>sNJZilC?jvoR!G3g{g?XJ=qVR3T|hTS1nhJyJOz8&I;GCO}qIJ>fdR?%5L@q`2dqAO;< z7WwYh>VHA=$N}8L3Tg-qS@5O4dV@#x3VD;)kYNa2!p5?}ElO(sDl`@0~mY*;0AG3uuuZ zRbGzne;Nu5P+ft6m4b>(_^)nu)tm@U9*ww|CB61j4kV35IkIv*)IDmDUnWa{g3EnUHW?i z-%P8H;H}tzxRdwrt;HXHd3Iav!${6@-p3z*qdxl|-#(Q~&MfQj-KRmtR`_5}v7&9b z?4B*p=waYql<(Pr@|N$(488|4cE%^0d(h`t;gn<5zQIlszySsbDCNaaF{0bmx#) zU2sYg79&kmjLhJ38JBdGipXC?*tPE9@^^nkZXSFkg`aQ70;a^$H1HcHcyAaMUf^vn z@JpL_BTEKNjbAp1UJvg;MdNv&^c=4~d3QV`f@Q7Y8FJWP^j2R`&MqfHhm-hVNa})7 z-J3>if#AfL$2absNpD9}NAGIFU{iySsi0==ACFnx2IIR)yt$T8??bg4fDH_MpPOSE zG~}r@#`v3HOck@JWUGj)iwk>0-9Mv1FjnLhBqcT-C+io4$)%Fyvi0rFYd!ppV3?xV zxRT%amoZh5l&lNq0+`am`r4$~uVS#TpZtL8$)Z~05o>}`H*4)4tGfrU-3N{#jUSkD zOV{gm1Oq)|23El>HQn<~D>*?RH%@>$0-pzAR)ml;*T^yR`WUXnH&#UTiKsr%dFv$C z(5bRyFza!30z$HO8;!D|1-N}h4AllqQr6lzWb=?6kG3AmtX?NjH(`RoCP1`?dRJI{ zMSP4)qVm+B*IO|jB#;3o?ruPAZlgK5>dmgHXm2Oaw!b4}NV|on#rR%m24x!yIL}KX z3Y1HuL^P!W?Mv%3@eI$$Ud_+&a|BT*`))&KhR4>@S%Yt06-L_W zKGXB~p;bWI5R)QW(1Tod!SX( zD{AlWq1^*%VSR))-wQm=-P=dbG`FISBP_}|_O1k0Do7$3J@LHxOhX&k(oG@#;)21r zEzvr(rGn|?Frpp5^DV(#>&%ZOrT3eN77gFQC4dCa*z$I|snnI^pDz$Zp+Tr==@kGU zX?#9tsOxQm9jd}BYfw5X-!g&h#*tM`Kfn%CIKU9Rgq^>c%HVwo( zNxRqhj$Ag>eau~h*_1GsM^z>zbt8^a`Xk_bA>g(Sq;mEr6k}uBndyIz<$DioO#Evp zxV4mOl@zfkSG$!6Y7LCi1p`o$vMyHfB}!dyj3w@A?B@j#(F!DHO}1K;W6HQR97}`ZS6o$^FFB|xx5n=9Yp{iBkH6IOxQX0J4SCj9ZR0fYY@2{-l0HegW4%@5FIo>VDQ%cSDG zk^@@NXRaCCH|&djpLjF(rm(Ne9^6^swJ@rlpY9pCkc;;|`23T!=?{BvF%7G~p7>Jy zuA2!;>?Z;`@7I(}!ao7RY3r~yw|gQF>u87u?CP}KnT)W^p5u5`X2XhBHLKr&YGrgX zKu!ZshHPt{-gWkye^g3tW$rpKSRm}5cKwgQI<@|{M}!OL7oD}A7G6xf@%jD#d9}`j zY<_Vv{?6%;U-k~y6c+xIWp<}rKm7NNH1*TOO6|VgUk82puRTB4q*lo3@&SfaHZ-#1 zK!i(}sg#E|G$?bop-{@<#JdM)qh=hz_wv5w&ttbsI>rtU&fqSyAFq5wwg>-FBt71W zJKcVGJPMhv#GTeA$r>|JN&KlDVMXqH=Nn(vWT*z1xy>^RpQEy#lG|1rZ!{SqOSX!b zx)(JC;G{0tlNI`V9qh%8U~zZw!ltcpsH1V{e&Z0cagf(M$cv`@B{3~25gpvl+$U8i z(nu#_$Z1?{rR*4NL#pVH1m2tVd%{;aS`iZLbO{Y8@N)!XXY~+AGJ8AkK?lzqYmhR{ z@i%0-=r7Zf-zypxCNWCY^JU=Vv&6Y|4vgtzjE_y#gKSY3hN^op1MxkBB==US%f+C0 z*@9jNbCMtmxcb=JDHfK0V?5MVAEyA^(X)rtQTF`^M>Px(Md_sjbfuc_XyyQdSs!T( zvRh%_h;}#vvky+yo9$ybe<1)&;npfy)>Bab6A`=5HQ-wWD>)<(wXKwA6y<~prRw5! ziE91!4)7=zTv9Tg*023k`Y8M$#&HmEltaBK2uq!M^C{q{RaW7ed6qjli9n)CMqY|V zqz1wiYrmol~) zdV-1IlRj;wmyO2xHrf7kl;J+q_hF$4c;K%hHyflR0n8;pIO{mrXvZ3mvsK_qZ$R9? zK!A!B7fUs8ZSxu%?khh4^>yZ;xd_-Jh6hH<`$fuoHWtq<^8XgsT)u3(3~i_R``#(u zpY_7%T)|Q>&)cEiY~bLCv)b08&K@KxQ6w&%MlV&Y*hJ!_P^zhe`r#aG#&@S6t8dl4 zhkgl`f_ZwrR;24ef$#!G$phI`Z#{irzP7zNL|Rwai_a~x-D-!RBdOL-MWG!BRc>4l zwuh%ig4tc1p;3_%EW#Vf;))1Hfq@$|*2Fb4=MqG`wGl`|u&IUt z12(FMEj~z-57=A#7(N20N{5%G?T>(=jh7Qs|FD8eqO(=GSp(_XZ;n~NFT#0S1XhmV z?h$KPOaY?M&^JfVBe~!1t3qy8SYg~39?$l3VeF^gGhX{K&8P5IH0Lzuhj_6RT^Kn7ws0RHb*hmG!-by z+o8Vs+3qF+(aZ+JXGp}-tdv(=T|`h8MYx7pR+pe`DARqPL}DuoblkWkgIgvZdM)+}%|1&4KtF2mQl33{XpBH{JGgcO zPYVGBWFKhJETyCUpij1b2nu)ru?x@qpw|O%m7a>e6#wM7U%sFJdVd}OuJPt@29oPS72CcsQu$Y;l7Q&pS;eA96oS5##l_yMFmHbnYKn2BzV}NEX@n zS@7)1-hW6wOYRpw-Zl9_=lxbx4oTe8gYE8CLL=PHAy9R46EEaSgx|{KrX8ru-AP%& z=-t)MJKEsAt^<4HgZ>rY@#BZLKTlQU|9Y;Wu|>W!@1a3(-Y9yUEYF^)J*Boax$>{( z1fTr3addmytL`Cp6})Zf&&tnhe)?K?wKL$#qh1O2dj9aGj~XU+E&cH2k=Cc%d%yTB z<)b)8FT-_m_dP7kA*a)H4(LR#DIuoeksS;co#XIxi1vCrhQ+ zL#PchCVnFgX3(BYQ;%{c>FD04DcWGAUhS5I#V(o012ifD#CJWq*?tS`;jLznOsQ>( zvVw9$dvLWghbPMkG53k?mN>u({R+vP;9G!FRPE8L&5dv?hrfP=5Mx{YV=K))g`h5I znoI4g&WK}3)hY+@$qvC(rBmfuKsTiU0Rtgg?@@u#c@tMVm_hYRsYNTL#|adN9?7>Z zkb_j~Wc51Ex9q@@!)GHP*bcA>Brs`7mfGpz=&rl)P2fK5eR4@>J^|m zG(J;Sfp1lJG^0d4#Mww;xE4CspSB4aLY2R0%HXAM1UsrPz`am=-`MV_24&OE0MiAv z=R+^UXI$Tlyon@vjy53g;dSPie(l$&Dly>B-$fY5Go?c>*5y^W8N(fv6gt>4q~Gh} zP6oiLMyvFbPRWg`Tt#VolLW88zipCPS|r_7Sg$c@ST@UF2!{Xta;lNtfSxav&64i2 z%PkzAb~W;Y*{P$qWhzc(oyZLK#Z zl$^^F_oLO;Ti!UHP)pVo`oG&%GFQ#Ft$qcK*%WH4eTzfDDoN6r%gP(0F{@osH?8UgZf)+2u z4X&!k_=Y*2TM@(NZh(mj&AS!|9hBj`!%$OWJ@@&3zKY9M&g9K7fDcWMVmjzmS-=scJn`btgxpUD&zqDN)xEB!=2I8-Nt%G6+ z99%}5N7b8K(K1{>3Gw;R)yr-ELmWhCIkbf_SHrZoF|s^E`<8l;=mP4a3_vLqKDr)KQw9J zJ#S*GEmFQa8fwX=;lq=sz5*Dj;~@t{-ioXKi>V>sbHJOO78x~*0TUKScxX)y!~}VI z=kA59)7fACMRaRdMnvzNy@oVb#-}1&(oj2 zJk!yeaPVQ@%U_dfTYB^t9bY5?noN@T(lKp)a6@#@1tZ9AWV2)s#@H+NS5ECY#N4Bm zA;7O2P02Ksb^K%S!)<@Q|MPnn#dSp^BLHEQlz3Ho>3B^Jd|z@J2G+@;MC|`);NRK% zeyJ-u^TS`B_N2Y$ke4&2wqM@8{^72K5C1FS*omtly^*=Av;js5paL8&YLwUK{r3Cw zmDDd=3HmhmDlA5w+4mg+(nq`P%nXmv@90w3yND&$Rb5v@R#SrVKs*dj-mi>o*sh#* zFv<3!!3zeUUlw5x*->5C50u&;U^B|GAJ~<$#LfmmdX=zbEMf?MU|4w`|Cru&@~1D3Wp_2@ zKL6>!$Z&Zf3Z14=gl=jVG(^=#og{>}Fq0G*ux^wCVx-DI+GbX2A{u!u|2G7G|EM38#e z#JVNl3SPC5Fvf@>rDy0qMr&_Nb0Di}o8XI7!}nG5KUtNXBluKSd?>2AhVR!P##<`l zV(i}_2TYgRA>LCkxP?XO!K#XiANz0dof_cAf&>f{uY4_4SvP08`c;2sC2q_ssrbAT+Hj*n?$0pM$52;ywk((;v$dt)pG$j|`M zXqU#JAdWW-Ng9BIsS2{y&9Z|$cRXTai<`ra(L?>K1j@V!OLZYYs==pi61RI7F+P(3 zCm#YzCJ}>Nj!x~{uAK5^`{kqalQQ1w!SLoZh%p3v(3MPV>NhjtzF+h3g|;AmtWOnq zD~NjCXaDXH|EtE1b_cWz;+n;pYu^`N_1vdf_h5I?DhqQ-v)mY3lQ@}Z8=VVf1(#F` zTlM#9WDftEm9~?fO$bBG-;MUnhXt}dZ-VBC8)9tZ>~7$;0p!AgjW1cYNr3bwXtj}4 ztQoY@vu6rS&GGHbnB$ME`R3meG&x+Mp|PK-C=#_sxF&FGhpB5Ve4tt&i7cb863FJP zySca^>&S|&Ac_7|(Ffs~@9RM-rst5fPxM>HDAge8b`}CdQ)g86-a*GUL&1 z8yF1vGS3Ghi5?A$uXTxd8yRX(0^0itTjd08wB^W`VuzEXk}aPmDQg9`THT!0tLp zWlF83<%yXbH8UksL`7iBl=7H)Kt&)$K}0}AK+eCr=eqo>{?M!N`@Qea=ly=Y&fD_O zq6Xk+c{fDQ^}30{mqVevUx&g}5$11^2dWbeRtEU1Txzmf_KnwENSHXJ{5H<{faogw zxo%n3)KZW689q#wP35z%4LKby$2VUyP{!#+E{#k?~wr1=p z>Gs9yAOCu0%f+BS?YjBw@9+Nhzdz&K78wYR_Py79Po!!@;<${`$MKADMET~bY-I%4 z`o+VHt1!cL8fmSjKsmUH3^wFT!0GPqXT8r@-^s9#fvnvp=8uL-ev2auSr3~Kz*OjF zHMsE%&YaaL_VB4hhJ70&s~xveM_Dapto1R#@4JbL6O$~--~%*~TT%t0f(G(^5DJbP zQZ){VaBJVYtyBRGLZV~cXrKE!QW{fKITozpC|AUM)ciux`EyF!LI6ir0Oq7faL%^J zqe-t-S17Vh=aeq>_rPE34uSzVAOc!YfWQi_iD!snDg!Hg4YgqYa*;Ax2ru%sy&krI zxDDnY!22TYc3T5knuWs<@?-(8&p8q#S$ZPPTQIhV7%h%sDz%K7JW)Asqnn5#7NHxI z$F-FqlZ1CHLQiR~(jQ8Elb#`O3KQSRZYSCIiWleeBIC)kL*#>#iRAYIN#l*tIKOyi z+Vy7rU$qd5cT)Itrw(={j(<kD|47)hn!*X!S^%(Ks|nyom<&G2*3xEXXwBHEM>~pBK1YpUl1LYc{x} z?5|%TorarIY)W3JTO)8hZG{<*I1_Q>+ri7N7ofbWKK7Eea1=ggX)@okUWu_OgMojYU+I(`>nw*%hfs7Ikak#=U-@vA zxfOw~%$YCXt%0*+16XWVtr>B?vYN=+m_`zAI6{uG?uDB2u_l18PGF=7z?l*^yG)=s z=9%!4FDXE(X%FMAh4EHC)Jfp@!t?I;C8HTl2{w7R(^2@P?RqFw@U$> z<$gFo1#mR3V!emD%Tfg{Tl@&zAk#02w-U{>hB95Ul}#{00zt--B7& zi+OU(pSOC@D%RL#0o=)gN^dR*53E~oA_6~{zJ}wYlZ;`2aSH-g2oiyqrCl+S^;nrZ z%Wd^K#PovZ*yg5hs7BBXB@FOI=*|oDlPXw|Ahw&^Y)XewMrG%u$9{h4mq4|Q+D+8~ zY-NEQ0;>`Ud6}sS>+GvnOEBemlJRIo~Be z{90sFh%zmq>WY7V{G)+og(}RBZA%*TRHWTVUTrVC(*LO#$$|FPM|8HMWUV|P zQ7fYQE2EI1|GRt9JUy6*YA6_-N@~%Vo9N@F(;`>SBBcra#3jN8*k%zYo7@3{$otTo`z z2x?lA_ur!l73nviy(L$QIudd^y1Y8PzwDq7?mxjxtqhLyh&{2^I51(0y`8ccdre;N z^U$w4@j$S`g?Z{sLWxg(Y-7=*vtP_!{CaOd(Ee);Uj^4+vn&IwtnD0$5sEB-$c>xe zC)M^mITYJ`b*!f=DaGJikP;o;*1YKHlr_T7jf`tKLhq41p&Mhw4?k5x$<=g^rKF&^ z79VA`@`)iR@uXn8o;<(M_Js9QIAk;a>q)EX6?^3l7i2jD(B4>Lyf-+NH&2GMbxfm2_$&o zs2aEpJ|)sW`DN6@WDDAD?LBpj3*ryCCMs(66~$CXG5pR)Dc%ejKy;d4?VB}toDGgT zlihAvikTjXsZp}Mwt7%iNkz@=g4zBG)35VPO8x4GDa#P}khkoela&XeW)%P$FF}1z za?D}PFF*ZCkhRvE_VpZoqZ4x^E)(agR}hIfMhMuJPa2<^G1UYZOAP0=eKky7np7>a$PuA+2UPB%R8^-DeSvf*9xt!lMJs1}$8_?K>u;QbBHaW#2uOb_?Y#+m;AI#6a#ZbM5Wr)d&bSM{6%)Qwyl79t`_@ zbGuU0Sq$GSz`nuS-;Ufi<;ZImj>CuTAGO^ZD6L_#*<|Z6;O@K+z-xOYsWT^02YA#O9+bVW%?aENUN0Ey^%o*HnZx)VE2M6hiBHxuxmbO$_EME7a!|bT-gQQ;h*vwTbaf+>!tlR|;xBS7)EH;X=K=#5k=?w4*HGsk62F4#5(!3_lLJmb! zi6i&MQ4EEj-U`6>GK-~yL&nM>lwl7l+t5^Se@jcx9s1#fGPX%{?+K1t{e23$1+ z(D(#{;aZ=ei%3UPl%Ucs;V2bOcDoj>Z?71%s2Zc4j6=NDe$H4!*F++6ISTSPImx;X zvHkZTccy=>Tt{DjGt-Zfxl`12m+>Ex(GEZ(X$Z;bF17y(!?##c*f=Id z0kYvTP+14EHbvM{1W zmY0)mKQgwo2JHVL=-IxSOy*F{e>OFIEUy|yFfe{~*+C?H=v)_3yE*^8kMj%`gi(Du#IABw+>JMw+r zxBqYEtHRW;4nBYTmkS5(h2Poz)x*$RJDJb2mlDQy)||fmVf@y!FMqpm`&MYstrwS{ ze5QYL@No6l9Upg2OH5-aEHtccq8w(tk?+JGC-4J^AwSK$2qO zNafEb39Z{k)5)nRx@rHeN!ifcMvCJNA2}Nj;FBkYDkjeyKV|%#@8|}X|JCv^k<>2R zC<4`>$xyXh7og}3z9N{aL0E%@TySgRX=BmkJXTJF$TH-&IsyY?7Ub?rSf2ijf5QoZ zFFsu!6S<&AV{o!z$2=sZ)6^M4P?eI`s-anOP8@@0uT2EM99YMaCakxU$un^%UTLb> z%N&EbCw**}^TIq+84&k)A~cC>?VnknW9aS`uvCsgU=su1>pg0n!^;(gP)*ZUY1Sp}uwjpDn$3obUVAwa4F0|wAbJ(N{G#h|epv@qKp8Xg$7 z9jZ$65_bKxzZ1RdKxvAN%{u_;$T5nGEyWF4JKCVVCjd^B%a`D1&-lZP1X`#5j53PS zm3^2Ox4c=i({6^q#Op7eDe3$XRWL6|_p=+{(>*yEwNS5gh6-`;-=F*Y-Z1geVYVKu z|bz$_(oY50&1WoSTCtx1{o&&sW zFNczQ+kG?RJ>+~a>6O)&Lg zR}xrAslgfBo((`uM)O^Wxm#iy#Ss|rt``W-D4V-5#nvFmvNenyy#2v{|8FJsL9wC{ z$l&PK|3ij=5c8m_5o39W%Bc~M>*4qYL6^LUc*r#(-1Rx_&Ct+5Gk*zUXN=8A!U*k$+8nnSrp%$PYox3lnz%PznIlOSRimW8b z)l*Y59SYe7QMM*R_m+xW+9Ds87!NRPeRSg~i47!GR)O=f8?zddLbgg!_L(W5S?b{t zCacjD3C_3a&9U-IzxHv8wp5-i6EaNCV79d|m>6#WN-$I6`962s)aXzpfW3P8cICXF z*yr9!>*!Vi8qgXE@XSjHX6>)+%ruhr_38)p>Kt$i%MF`1QvvDY=il$9!eK!#GwZ*r zdHn9~U`JIx5Xp@YNUbik66x%#c8Zsa2Q{xCx=5dj^PQn@K2*4YC_N@i_uiyD4Z`mW zua0ecUYeXWna>QN0xpS9S8FJ=t=krrbxfs!HqvUV^V%XmAx#d1Cyq6(k(`cudcxUw zNtP(L`PM}ot{u8DbAg5JRjZnulrx@ z58WTsH+ETnw6jBAa9AeIJsj`3T=8MzTQG}$8(o_ejd&X#I2u~kTXN^p{jszDiORL_ zHaGknj_#hka^n$p>_c_GNc6_Hx_0p}01Ap9k>;5B` zJWlrA#>S{0w(e|vclYAa=$BW{JULD?oIG~)%dI0TKgR!7vU88i{hWX2Cxt!w$5pSh z_kRrA{p818U)TS6?~~J+TO!_cJ;2|NT-x1v>(F)R()oX!**%)>_3HKOjr*xz-G29H zkCvL2noY&RvDydjz1xDTy_{lv@O+LEfM%S)%F9}yI>j+Iv4P3+6|PcPiA_RDci`v< zV$FiIxjVSCZF){TnM6FQ@d`yOapK7 zE-BdFwX=i|$z}O~H=hh35c7{m0We4Q+Wty}tHPG4*_!bE1XBnYkmYEpRcv_?Cxgv= z(}?!5Om5&zjv$wj+Mf_`HCy*_qRw@O#Wu2dAYhA}eg4!PeFy~PB9}MPDCPjr>53bU z>;ykKZL@JWsS-aK?PhtyM-3^C3g|P>%biM9%N?BPxD>yP1f@UaV;!xepAAmNsP%D4 zmu!nbytcSuQ&?PSn(+jPl_Ak6$|Ikw>r!iQHiNe?4Ok{y(3B;P@NyW8iS%1v>yEVV zqOn!n4ZN%2JW&MY*HWM6r<92%-s}y|xa$mCU552LT2$Ju+o<1xRzlNb4Y&LUJ)4p# z1|$EFJr_TRauKz5Fq;TdM)4zL&P4iR%BsXVCqlqo#P*fK1<6AJK_0COyTS@DWg|or z$DN53%2?&hUHqYvEG}{;wJWf{n`h%7Fslo2yUZxe;=`f(m+rNRy>%jjB-ox}yewIi z@z(R$n58X($WVwph1X6L)`YW^fss0J=<|oqSnti=z-Ej@WOPy9pE!o-@W~P-BoebV zm_x%V7!IcL2<#mzN+G;4yxMB0G6->*&dwW%0lg~T`c#;ksTac4dbCxoC$i{%S=Jzk zw=IN+S7%`yvI3(?jhc>`2UqRadm@+eTN)}A%LK>k1S7YAH1*H)Bu@!yJ8iJYHpa4F zKi5sSy_0~MCU{z}OY`)vw$fMD;${eP6-%Rz3_k=h!dy4Yhv<}tf}HVE;$SIZI=G6Q zW_-Ho69#Oi0BP>YI6zWiNxiPpbTFZ?hKiD!iLDG5vQpq?%+~0${{=6C0VzbLq7Tf2 zquT;2xT!fSM4o{dm(s3n5>{o4vLI%yo9%vZoeYqKi^&!oIgZ;ge!hZZtD1_%&uMHT zB}dyZwA7r~#MZ(bW{qS?FV)&y69zN?|AxE4L~?4+d^dkJG6XW zxhMZbb$h9 z$<5hReV=qODmC$(^ipCyGrOfp(>bUrENm<^^#Z>0pqwPZ>A0Fil*TDSwI{(N;=-wo zF0Fm!OWl7%S-nN8}?7P8X z%y2kl@I-%1!ppj~G2dGMGjD(Sb^pctKkQom{7lV{C;yg&h`IFZ(ajIPB%O3xI-&dN z5UzdSv&F#y_1|h`C6g}?Z)cT^9@`t=|EKF^y}uVGyG$PotO`wRp9*gKtI2>$Fqv(E*s%XOnLO&;>){v;5@f@fF zoDx?7wr*uBU3)hTxvYxxX@9}<-Ckm`@GN1J9R6rZ1S5n>r3xgu)fuFA9xl!J(amnv zeropa@Uu?Q*J}_}WWn{sxQgke#Z>NiB@br=I2z>$tsTD&vBRdqiEujW_iTTsB3Hy| zQ5EJWr9PzHG~o{Idfkr#X(GiYbBYBdW9JTX%>7kJoFoFxOGMsf=m}Uen`ac(4OQUZ zRk;CxJFeUEooL27E}u z%5CM`g)mhmCv|mt)cYLPkc~Y}8B--C#S`*acpH5k=)iBgIkZ66**zg+#tm!48N(dr zUAW&`FL>OMHKnQ>O~q}!81|$7>aHX1otJ+LzFuC zusGG01tGoF$H8WZa-@@+@%I9VPezw2NhN9_4rb9rLQ%{Ja+aY^viNJX6IkF}bS!U8 zp)8*m_Ej0;o^K958Y@~>fENu3AbB5RG!L2Cp~@Jx#XvL|sVzp8N)6qttUrG0_LpVA zNX?gB@n}tnZP+y0xsciF5+Z$pOU-=Yl7EKC?l#}5whc5l8Q*5T0MLvu#GUb7%00CY z^JChWSE$M1BUcVXoBi%a)wJEI=6G(3$y!{9D*tm~!`}mnaxVUD*3>l=NAdtYkn74x zK#n;r(HHcb#;{FE;_)VB=Y@?AFS2j_twqC0q?`h(d1m?;oG69VZR!8nBNBXmxdeb^~7EN@I zOy=*W>IofH6aBL{kl*rM1YoWBJU@QnE%@X1bF_EI7CILWRG){|wgsZ=OQ075zIpK1 z@@+ls|N8pMKg(YK^+jWAovdW)+cR7CJ-rz^{@KkRzH<3u_gA}4G_+3p#st>KCWUs! zlz;H34Bo6Li7f-#gwbQ0l6Ug!nI-QYjC!5FS6tC|DJa`D4w@-DPPUX%NiVr2{-HT^gm&V}l39eTA1A^^T9y$;pP za;z(^@^Ji{c^SOi_HiwO*F_ml)=^QR5W>x^qC%;}CQ>^`plsb+ocOaPrL((Wp7hN} zN?WRnJ9z1=h&t4EKxOM6Iq#=W7!BaQHrIx&g;2V>xUJ)Jw_IDd3M)YlHTQZt{4T?s zFEGY!mc#0)U2Ah(L@Z^!+bQLr?&52ff;4GfEX?*o{myuZt-8VroUxVxqtkt-d@x&X z*S)zN%yQiBtwZZZ(_}w~=bD@15L6?yYp%Sj->j}KY)VYl)Z=VZ=iz?Vjc7uCfSq zLH~Zr=V}%4c4^KO3e-)!uWq8)ei5R9vB8L)c6}i`#0PfjW0_go<(0d@n|DD z`^nuG$k@$_*wEPSc*MRx{Q1bM^!tDM@?Xxk8#6r_{O~~h`}A`Yjo?@xo!A>M{0qPg z27>PdX1Sr?+^);q`H!gAL)V7qrYoHUv&a4B>sb zci1GhZao~6A#6nRjIVp@{7mugfy+tX%7EGR;`7q(SySFoK}9YNVt$XcJ;UZa3?Z7T z_4k+}yd?rC10-1hlSUwNSv*S}PhSeUU@sapWoNU6F*ZavsFc95%9 zawK43#TaS>my*e#4rvo{^MvwWPz8h-)E5on^+AEY)t=fjM zb-0lMRNX#`vTIwMh7^;#CJOm&XPu4oHtm{wBw?{Zcgr8j(~E7TIX4HVx|PLhzg(4o zLzep-^N3scXl?ZNMfMxs!6^j_D`zy*D6?w}Xv@PwDZZ79)jvw~MC+y1ENKj3T^{2G69Q_TJLls^K zp(Wp~)8v-IQQmfZhqRyR7gmj_**Pok`OD{s$)09&CCGEjv*MtyBPgVK-C<}+ouB@- zg=9h?6&szFdh{5_v>OV{wvoO7?ylx>6yFOX+rej#cj>`d-*wqmG%q?!Gzi)x>eCS} zb}MZdrP6l)eW(g~d3x3CKVWV;C^+Zu-Y=r0DfUWo;xzv)uk&o8yqJi@x@t7`>S}-P z&`|H(x`;wlZ-J%RBGBwNSH()HUmP(SB=~gx44WrIdj#fK`n(%-)~cxirB!K`;J}I* zJGBmFXSg}w5}hk(c2i7FE!Rj)c&;Ah5J@OAbcapJwIO`xeq)`Z;%0EOK=L@*72B!) zo_{F3T^T}Iuz)da=JK0j^i1mLd6@1g&nlwl*1$;%&TiS-I9kLSAe~d6J#~3Z5kIeSd}K9s%#O*lt9p!fs*nKCtoxh`Hee_j zZC%>2$8{*R0(_k9OpNj&mbf&v0k!^0o`N#&a&tVS0JiHI01&vG!)mj!$mgX{Dp(`% zd@{^y&0E3lS;+f3)IJTctOU+-HBSdvfDj|VD!vM~+duTw1;$}{Ijy)>lJ1=j&DpAN zIn!UgKUUPw83^DEg>}6?rdDkbx?}OpPHe+?zwBH)BP=iRzneGVKl@v2{gKl} z*FUG-xpd*qzrXT0R^sQwda8c3biM8A5<65Lfl|ilOaPheRA9=TNKP4&>t1$5fN$8S z>ST&kE=qJo71mA3YEuG+U9|f)-6;_Hx+Hkk^F$d}vuzOM_aA=w;4eRKc$`@iz z?fAE)-7k1J@WiX+8O2d&iV~y`9;bNxFaA;wbnx%z-%X}RB(%ANIqyDlwr%212?sQ* z-$QmB^=DAiyo;oh`I%vi5b4-|2G))RHjGbx!%F%gLZD}6pSpOqc=zA~=yF#lBNA(l1g0abN&w$<0gXylJS^9WI`{1gtW)BDhh@+W3KJpS>> zuGgNw?7q-C@FuwYi|n%(|KhUi!LD!jocMS0XYO0iWd~25GdzG+Miag`|KP%{-Szwa zhyA!U_>dwA>R%EKE$j$~Y-kvtCA^JK2c+l8ozVQWq{hA9ocW*qzx;jw1h_Aot1(CS~3eIHS9LuDz3vpjjZAS zcA11-86nf7Skm;T$(el>X76lh?&#=X<5mEe{~ka>IEC?-df#nj>YRDKC+xEd`JwQc zf*nOUOo8Jbg06kT_cQDoO)?i_?855oO9)gv+hA`$OcOly)$Pvu}j>*Q3|_8 z=+;+m#r;A}hvs4AYIF2+O|DrQ34-xEPi?krV9O(QS%9AierMeDZ~0tv1L^GxZM#L? zj+L)ZTwhGFb*B$bXh&NrCDx7r2%`UUGmGO=I8nYBtPDFxuVt|}s|{4zfES*M~$ z9w7`b>P~N~a63ObvD8%8denJ3ck+l4LMPKCtAD)CMB01l{6yaHM71O)qhXAfXYHW*J8OZMCC)SA}bK6_&Ytcd(Kus86Vgv-*h>Wu#G&p>F&FpVZzi{11u zX#7lToq zH>gRyHBsyJ8pn`O)2Z5zv+4u7hQgMi$Y{-nVptXFI!<9FlCA#GBBRK3*S*ra-d~cs zY#nvX;T$XIKvze83 z%`g3gPgVRB-2!!us4!@dx~Iu%ycuItz(gy*QhfnfD=LGXlDxD9roYz{cugiw_YGwT z&HMOU&UwW6DYt<jtNMe#ujq)Vae42xTmL+&ZL#{+i-23a5xZ7!`rqA!^+FU3czF*_%kMkk}`-u zS?Y=af$Um|&klD+C1{o*vQ$ZyOH;nDhTcu(mJh1P+z8KXbIO($yvNbAm*EG4_9Q$A ztjWK5^WoOyPbS2x3@RtDI(dE6&(^eESji z{rsqbfm<&l_g42Ka{rrm8c}`T8%P4;`(&PXl~D<=mE*_mdT$2#i(f7#{PJT#Qu0|a z7x4Z@nx54f?olVJ?#|A6pABmbD9u$ofgCdZG~EBk%@CozXLj+{_{*tg9_o1r{ssK* zA20OoIy3*l*n)p7ShV;6W!1xLK|SA}zOd)c^IQKeyuI`BCYSUFxBffzx1i}M-&jv- z%lVU@w0FOh{qm&v_2<{mqCEt)$34nJ9#74b;M<3~_3feOyGICFna2v>CA>YHv7P$v z^TwFdEhqOguZ%yrnCO1WDJz3~uBL3lSRU3=EbJa^DBx4pH;`|1_~tO-lpc4hY3s16 zlC4w3xtT9QfN+ZNnCB>We}2tA0kEEX+{mB^{U(~)Et=+^fM$1da8<~wn-So;1%e4F z`1c;4rUxuix_pF&pELw|w`bl#8Qyj|uq!bRy#(nyiNK{f@+%udh#WNj4O16S zS_ijAC01|1@`!1Sl#2Bha5fGhJq#gz2zJP{6i2`y@zaDflgQR-`$H@rOUvnx^Q*O2 zHOdt6qm&(Of&bYVX{l^N;W8g6^`~dcoLMx{5rkiu!+>=KN2~}R)Hv9Obt;XA8X*kI z@~ks&k1_nO$V|;9l;I89`qBQ_d4li8X_OoQQAVEU(Pu47;+Ge;lDUZc@3--~h^KNuClZuJ|@+Q-D9B<>8NYfO_XK%GoMEffhs*!r7NWd=(FygFUW>X90r#p3QW z*SS?Gu`+qNQdUkk&O8YbvUST!?n0v9;uZ)`v%^T9^e2r0xRhA{ARPdbrobA-nl#%U zHxNUqvfguI+x_Tld6T=k;&G9ygs|K|TGlwGX{^=O0;04Us=J5SY$t^Di`3sc^S6Ps zH_Y&{w5lpinG7Mk>4q4u41;n7-UiPdFFY!+sfPi&)$xIG+|nkXzQ02lPr% zY614PtA{g{FBgL`q3~eQ~<1#fjBOZbVyuBJx z6sanNLNmw?0X@^$D976Rd8D<25p@e=OJDM~Kz4S^cbL+xBmp1XZPMC0k-G0a~w zUb1+(^ZmZH1vZ@ZS9mz1u6kTNl0exmP*x8dZ#?yzfix99;8f7bM8Yn&a1$G^c`WD? z!l6w4@7>*BCx0X04s5eskCRRgO-$U5F0VtoNW^8>Z_TPglR-AO4!qRvaGIM0gpri< z?|`g2JJ3bqhbPG2+#Xb^fYytZoeiGU;O3T`oIDg*F*z-MoUB^oqlakprR>1ct%7Vs z9eXgjEpTG6I7WIhiM|K`7VretEz@aC!;@_{*`{+p9Ph-R-}#5^{i4a!^`DQn?#$49 z{P^)Fd@Cz5a_8ouO`-hNi`$k6oCh<$eZc~!;?ees^nbpp>W$|_Lcr*>IC+yYwsfwk ztJ@NRiV5^yI2IclT~=FPIGCFA0y@#LvsT~fed}CH~!Mq z)wK8@Smaxsv8_Npg;WMO2~(d|2Tb$6Yl0mAHmkK%y8RP??}ZQ=o@;kA>wT0!oR#IBGAVkp+h-k|{B zrCA7hrI29^Wmx4Ddnv_g;jx)`K=zcys5>o&sq3HxJ-3m>^1|B+nL|K~Gz1)Pw}oN) zk5+raao@QfH?m!vk&J9Am6mhF!iaXwPzyj_m5GdwsJJ*gxHc}9P(`_x(W=9* z&a*1YhGU6NYOY~j%cX^o2K~j_*3S}YV7jPF1~ zm5;9jcyAJ|3mQKaS2!$A?^lTO!0OjB+PiEBqe$zA*M$I;)ZWsepMx2vZFjx3zq|X| z%c2V;R+|LDnkEhOqt?&R9Kt=rv2_!1Dc{x&&)^K=&uD;nFD;vzLi8T3Q@9!P+ob<;{+NpK61 zQb$oVN~x^I1AvX6OX-s2^x!I>y0g&~DTuDd0?Rgp$3#}1x|EK#Ly2)7*cCD(L!E|0suiqled%zghds?b$INZ0P`hauwJsmj6wT=0>RBxo{wj!jRLGQa$?q-<; zS{Mo4zC;(ZiFQEsxeJnM8j=u~+mA76F}4+&=>eb)@sZZwAl^hlBY?zGjK6LGBPKp~ zt`bdIl^NcF>`a$>Ix#>&HnQxC5Xby*q#*_aOlX#I?s_n4NhPrtHnG!)4sHQswTwZ~ zegHv*@ep!fa8;&q6gHE&I==~mOq=`JA*YyHqw4}~db;>{zo~{@H~Y^cg~o_SMcH6& zF$n_n6@Pd5+AatRxcs!*nf86}v!$U6@ zawmr4M7^@-^l|c)xR#6T|Gf)HoXU4GkN!a#zn<94%F02hds`8G#wR;TH&zdRwiLQg7T}(%z|*-1JgVC`cn(GTyi%wG6?M3)>Za|Aimh(y1rE|=T%)J1#*~l!}7nFc%I8f6NlyF9v z9{p03{t|g^G=US@2(t3s3+dCmR-eunk#7GDO@CYe;mP~y_v1cyl}V4iV*^KDL^AUA zBkev3$paM+2B)4JPv5bSERnBG)@B{zx0QjBYvQ2PYx9Z3!S0Xa&tkScc{2C;?X-$} zrl5j7Omu3`ydu{O#50~7`&vKbENcIPjv+mry%+Va z^->tilGe7+FRn-6N zzwqn5AM!sx>v8Mp|C_oWy!iy8%=d1}HQ$#hJE?7vfl7UADA9Lk37BG}Z!hgnbENid z3`;)yeB$?uXX|TjOz0!_>wFrPEKRgqPj(CT1tG>4mER$_`Z<&}kCHX`PUVhL#cS5> zSObeZ?=H2~d_hD^d^TKn;_s-4NCmxFMbpYJqf85rh z=;jcLu`W*IO#yJKWLG*JOyJbBCU-yZp&~9d0vA#n&Hf>sB#%JHw^mn=p8$`FYfiw} zM{#@v&5bwOLY}#tZ(AF-PrNdH2Kq>-sjjl#?z9Awq!K~gf9uEO<#{lJ2H_qfaxqBY zw-f~7%0EoeErafy>4QkK(P2=Cwi`v`pJ;}_hC&!56WdC=uV=(8hr9Z&gP z%Rs8>26`jp*?JYY5rp7xho!WuHBd)8A4tV1N*q~!gJS%imYW*rPqWx`;cy3A!d@%E zrkRT}}W_WIb1ZMs^W11n#Z2^e{FH% zD_&Gi`9x6lB(ziirp;ztE+0mok&;Q2j;@?+&3%~t9vdqmS|z~b=q@hvzGuqTtcqDP zFZ7@g5M^w#Vb&jmAg{*q%H2rELu9o#o)E`YV3reDE!j=>J0}t zh}d7jF%CA*;uKL}*<6A)Qvn=ek_+b3al@57#rJRxL|TcfSTevP@7njayN}ftkB!n!c`)MLvE6)XtygdJ}hzPUkdWL6M5>{2= zo1F`u$@AK1A%B~DV3I31*GyGmlX0MxYnYguzYT~hEOe#VJ1s#ud0SOn2Y@qK%WrO1 z{@+PC+tV`^fd~t8cGiIr@zQgEUJJOax)l2~cA&8-CVr`r+jFk))!xe|g9Nys16%fX zE^b_wwEzZ!GFLe7QC(RyVZhJNP%}lLl zCKH?dg4D_j1@!2ZWAiUwhCSbmZ1nwnr1tfN{xFX5_I~HRNBQ-IiloO4`Y(c(Q}ZwV z+V_X!<v+m#Wds?Wzl-QFr@&ptIjf z4?IXXaqB6mo|%+{Yq+4-7Pi{{NJAG&07yHVRJMQnV3%j%5zgU^ZtjhoG{2#r~brU!{d*jPO%SxVI ztUrF~Z(r>S4*0qzcx>_CPisCqb$efp<_T^x;S1HT;_MYzGl}y%7v#xdRHYSbv>P!E#&Yxds0qP6OqY-}5e+d1?=zbME4)SdirX{` z6|6vHjhisIem!F?0Vb)5SmF#(6LVYPg5kfJ+ z2`eX3N`O~y=Ib34T`9WDID@2l;pYs^$!>z1?8l9usoi%ZOEl4S$gF>5Hj-nmR#hvP z6x1HTn^|JGj-ek+s5{msGTL9~OKzLg?D$8PGfz81BQDZ@^o74|6u~*O~{<&B2 zTKJ}62-+|9ZE%Cc(yKOi_jE%?`Lj2JA=g|~*z_Qn-5}D~41meY&AMoM#-LcOV!M)) zWzz9gy!n1N1TmZUu#tnJN8}tqIRY$#nU`Gob#`iY?)w1IIv7rhRL009=tTy8z|cCw zHtj{*CTI(Gjs;gWC?JVKd@T!O}f*UvYhSkC%nNXejVsEMuH5rSE>{uNZN6bib``L~^!DB^hmvLH z#6Tp_@U4nla8;AyM-de3axj+D-UT|vwyg|{46EzZShpF0^AW5f3$V!^fcfv^yA!A+ z)rl#-8R{ywLk3glj8&&h=Bx}xU_errz8XMTf$;Ok7J-h(qOsezf_j8g6FWsoBP%s- zYyS~=1R5at$@3KBy1RysWvrit8U6#aT=zGA3kfM5s+i8Lqmfm>7ZeYCIG;Jeiu$u# zFBOt^Z!8OY=ABe~RisI(^y+Bo=LLV>H59P~nG9ii^&OYtG2c$RJ9A#;~y`H#(Zz+Hl}*&Xei9mV?ZLN5B4RM^yjg;y;1bm-GC0 zO=0D=@u7_&4c!YbQk0xqpQFnLcc0z&VdOupN5NmExlDg|t`Yo|)QA7CC{@(pi`JH( zCp=F#1>lQuLZ~P|JkUEv_Q2x@kB9)bJE8ykl&NI<{p;*!nL9lW3WGzi(^P0VdlaD-PBAE{H;=aJlG98@Q2NWH&Kt zU`tQi&%GXz@y(@VD<<+rLp96%tH4p?W~mHt$}ZSepsEy*yWyf4pTp2bx_Q1%l&u^E z9JTcoaR13`Vk=xF3zcXW;j}N8WB_a`q-lct08SFZuequvKrMxn5Zv>?a1^`{ZPyd8 z$ieAUem%PSX>z9ib&n`o5J^e0te=3U_q$n#+!&jw`>7~f1Mk4$Lg}i8zS17Rmb11M zm}J&nu(q_>xv0`)d9Qhnk=t08`+iU0OuBRvXmHd)5h74RcJv^+>;=h{xo;)bSD3CD z5V5U9lYArH)X*yL3^SLl9vG1FX+eDlxvvj`9Wu(kADz3>-9+jQPGRS0M0MCsLmJDr zV2=V|96un*<5a&qfG}S6m>%ka3jXZn1LPS`FeGU+_V-4J? zEQ*C+O4d+^cw!+o{Vdw5eE5W?G1Sn9{;V>GZR1>Ux>M9;QhDm(B*u9xLle28{QwcK>ZIu0 zeOm$+ZzwA0w~a{Grm^d10g)`*To%mUu)!HoaO$>k{T9Rl(q;djqBD<6I$itlS;yJZ zsivG#;mmYq%u1Oyapz1;IT3olY-PC&iTODP7ovB6D}|22iUqgD^2-8RSk5&9!P-Eu$3{1;FZy3X+_ut0A)P9bN8pd$A`Tc z=@B5}0F2Ms`vtRgs#J6;8V2UOew{2@c4|T*7CcqitU&Q>132ln)2LIv>~1QJMgz*| z)Kq!uFnxP8*v|E(j>5K=_Z6|HmX}H^vO#Yw+|9zT~7f z`CjFBfY)=FwqJ0F_U6#{Tei#%-cA4G*7MMaPVaa+FRJ$8s}9J_?anL(Tea){2s0}D zaLC!iq@>^Knm_T6b}73%H2yD_uOd%0mhNiJl^pk-Om$1}q!=oxQlHAt69xu(W1s%^ z&C4X$pT2!^y7{AsnU|8^)7GCo4E6JSE&UI{amt^FO=Qhhlzg^dp3+UqJSX^g$MEkV z@V(lh=RVB|{M$nbx$NBnE<+KnZIev3fbdwl_g-3Z5bWgNuBRo0?CC#pY|Gc*gr<0o zr(H@?--o{V>3Z6UuXfZOSY?6%PpqPIyUVw4 zpAtu$sce~voflhgphlV7a%g%$uS||?+f$ppT0ToC-j=lPzk$P`=j050#;PS zLr-TKp}K1X`yvG-5sW3&B?EC|gM+Jr<0Now85~p^3yjuuQ!U3ffH0SH@+bL4lVt%Q zID%XA#G6U@Tun_1_@g*OF3Mg3;5K>M-Zm&tQYdUtPec=-0y!NX1_hP*moMg;tkdeD zx_-qeFL0RXBLeU%NW3i$URMlFFFuUv$q^R<($8kxzQJAyi@X-bDSJNDP0g+EgncN(j_y#dx}j9*V|p7IfX?p#VY$7%xvxk zrZ;?~f=M^{5=E;r)Jh^}jiA~Xn%gn*4rnYk#)*s=w;_lz%H7G4WuF+b#b_lY7AY4b ztJIH!)9Vz;)Ax9(K}wYqUP96?>rGwm;JBLbwjNn4&WzCZ^{XS@?3%JNOwmF6P^<&T zmZE+?73P=-*Cm#IP3y@ovf7~8i|1MFxl7tgsGDfvWV@hL&=8RIzm>AOjREl#-lBrC zR`)JD;jJ(8+3EGGyz0v^dnrF~{?cV+P(V!VO_CUwBivN$Y@Z)qxdMpl<~_TtXQbagIcqCN3&=0mzt8D4K96eh7`s(V4Frjr>fSud5h8DNlG@Rf(LAEw6J&6j5JDq&{FK3$md+mScj`$qwj6tRaO2HJfEciozqQ5h$A}O)cn>L2o zvSs+`$aO3u5M)?L|}Huimf z?)%AjWy4Tr-)LKe^Zm*T#Xt9+8bAB;(}Tpm{-0bq&4QrKTqK5DS*?%=d10LF9wKa& zH(b;?dhF!NH-9|jcQ^)0_-~7DgQ;#pNeS9ufk-TzbbaH0Wk2sXrqzYL{i46IW){Ipg-!#OCN{9$4 zJKl9V_t>_v-Py@er2a3*0Xb%4>_NiOXz(Vs_JfI3RT;4Os^+2gc?#g$McXY%+vVIO z!-rkLA-4vfYz-f<;88|U^?xiZftucv0u?RwtIDR5aDJ*=Ejp)|14v$YFufv)Ez7|{I?(Uc@$odSb+QUJo2_pi7l6=#%KXSZ$o6jhFkko#pD!Bw5!+r} zblt2BG@k=Z5xfrbLCZdD8zu&&&1PQ>>3ug;&@WLbm)c15#c13#zxZ7r*3u!+C)YNg z>BgBC=l-Qyy9MnEz^pq$*CRmJr!xMqXnoBE4Bb-Bk_ERx2_ym76KwNmP}&O9kcp2s z)2Dlad}ba~u?_jkMBIu}JF<4Yikd*d^hOuS%MmNEbg*FylJhwVUNCFyX`oMk=Tolb zadiU}|BFQd6S3{D%FT&fv(mqi3vPhrirEr~Xt*Mg4i(~-YPZAe_qx<)^3eHB-sSDF zb+Ko&G(UFF%?vqk=7z^{mwjj9SPNM=5O}#@fR>w`y)hmq3MlCE3)m4&?Xnu0-7QIj zFwoN`>kW8>P_xrTSVD*5p5oVE$65M|`Lo~KnOrvzb(%}zXw}S-N9_w&YUEX$qJ4JAO{=GaS z$7h}T)lWk82AAo=Wc%YZIrQby$>dgq^>uYo*>D-t5Ev)~;F%iT13H}^XDk*MGzgOm z)XV36FoM3pNgW>mn~V>Mto6M;vqI2Gv{{O|@JbHg?+T%|Rb-k{o)b+%b95`87tBw+ zk(%CbLz&NnK?L_E0YSBfm=3eOB?Qug@I`VUrj<|J*FKbtIwv;)kTJ!+0PW06 zUtV-h_bV0-5sF?DKhVb}U`1f+E)UKb0H;|O&V~uSF_U&O&UgkCKw)~It}c=Z#dEo^ zfR&cNk{T>soZ`1&U)##EMhB^#Y=D!ZHOzD`GNK{HBY~KP;#CDeZtgGNNhrL+mhX)< z2oUN=3Ea}^f|7%HwG)Qy1!3{sS#u>&%X?Dv&6)R4s#rsL*jJo-V$qY z&#T2yCxo$4Z22wdM#%NYPo77^Wej#Lng?9<-88^mNgXuRlPjhlrlUvT&Q{*yqJFV| zvA=(D@xp#pr?+!)q&Kh?!o54aI}?Qu3FjisQI{V?O?)`H%959w+w>2j8b3R8f9GM> z&Bq=;o_f2V)K2@l+Uy|laU(kt?zZ;!#v3fFS71&~S+3OSO~Ndd`NQW zj|ZVu+cu9z?cH-iv`jb6V2THq?%eZ+ClBm@a;bXwKS#d2nih4s@6hX`U;2g8o=`*| z51D7G`9)>Zg{i%VrkV~XUT&fBt{vGld8FkD?W@0j&Ga1)qg_wBSiGP+<>25;9pYMJtJEKI?fd?RWbeH$g50Hxoa|7HoRPki6PPQs0FbY`3UG{t`r!-Sgy|mMDHEI5 z_D%KgH@;zbzdbi`!?QN&#I-}czk4nAgv8hA?7QQPjGIzDlYJrTwHC9d&d#@_R- zmnwf>iC_O%_~L4X)jgJp#pPz?X8<%u$>dxrNFX6&?EjTPZLiD7Ty?(-1Z+;h+Qha@ z@e*Yl0u8g~XVH-L|CWieEE_y97`+J;YO1C4{s0N63p>)l)`ZKk%)mz9tNs@fi^HXYUgm?XIQf{MkPx$eF&nqMWc&_ z@Zvr#CXQj78HX84DVh$fs-9n@#o*Nbs_vzQWv>0L!U1yvSuo3@sKa2kL{5>p5qdyD zkhn3+s`UwIG-D~r!Mzg>fvDkwmaFZVvcT( z5~FNzROUZ~>YtOCa~l%8?w(cv)Q}BUaOre! zKI#~yzP=voB^MQoS?{;{mkU5rrNQ0SUWtY*6k@fn>X8WZ8nxN|7`_m__6HxkA7C3g`%H6`LJ;y@v>5$K;iR|X0NrEP z)fRjt1%KVe=l;@&r|dp3%4$7fK^)}^xX^gpst{m!7<7=87hfZs7`9q%u}HIkW2s^~ zHDp>tVhzX98`l5;LW1!kW(@2N(nR9tNy8$lp$xxXJ-!MWp1laUC|fYxV!Y5<1dg=2 zbZpSgV2Zks0>&ElvD#SH;z=&lG8Jr{gW6x*Pv8TrjddfT4gY3xbe}-uWS{qd>G!}u zf5bvJnlnsUJZXOn6o+i{65Y8hbs2GYODgji(=UpRV2%6= zv+*vHsZzm2IayXZe&eRQb+Q|q&N)P3=VMUW^LI&sO8E~=b(xk!n>dwx^7_Y{X}_h(cmI8S($V$gksG@=zw~jt?Hbx&INk4! zVsv1hf!`1l1)Z4ub6N)D(oVsvLt*Jdo@sv@pQ!WhB&DXMhA%EZd)c&ia}EFRZ`)3p zBaWA*R{R_$S;+a@+4QcK+h0X=cpT69cKfJlXuAD;C#~S23jTI4D=GNofOp%$TERV9 z>5qTA_wk*N2Pd9~dWL%DvXayO#x0b(N-_{h?fBueGkVyb$Qf~Y(&t~sLfw%$R7C(T zmR|e>!b#7YUzZQwA4+gv1bS^1kj9N3JDp1%)75)EIPuGl;0HT%WbL8kP2nM1?)7&t zGc|=@m9}q^Jljbs>D7&9fhtDkYmYR+kDdEA=XneMP@E8W|7pvobsqoo_&VmBXUEB3 zFIszk^c2_~V1F_3W~e@)mMt5F5`$Br_0?r)-CfmMhd}9rTg?9+I{;J$d~AK>j=R-&%hGb^*yP%nwt|3fhs>;~L7w58BQ>`)7og}k9531eC4GVnIqj4e5 z7Q8D2QHZkH09>k{^nl=;B`sj#G&4gxSDbi1tr7&f>UAyGn$o)q@+S@NfCf)fDiPMF zO;W;GFM5Igy;EAwIb|FWvsEw3vwm;I5)TCFNX#-V-h+4H@vs;q%&Az5Ueczdw^xj(oC>bL_`GbO#A>amnAWU4rsy(gH)^=(!nHtTw}K6t zgiJ$2G(`8BV=U(cFuE{ZetQy#LT!>LG==FO!`V^@Dv?_+K12$QSh!kWU_P6gNDH!@lqL(iJK5rZixR5*fPyzcKk||jOg`ih`f(V$|M0K~WjBds0W;f0F z;OdD%q4>70^L7ZRx0Cerso&5M+cch%GT4N8S=vZ)apCd>dqUJV*$l+r0! zwW$=kp+Pd2-cZby1cttJlEy~|swoY65_pUOT@idL0A&M7Tp1HD4{>&JpK#+Sb{OP0 zg@yBHUGZ-ZR&Lii+piLh<7Rznb#YTqQ#}jSS*dlprYMo!<+Abtes89FqcXHxB_EnkEe-Klrl+Lx6+GBxJvt9uhj78ou z6QCJfYQLug^TVZL_=l_SUhC80%hYKw9!Q%QyZWi;aS(<&()45BAAk8olG;)FuPx^X zjwfFK)5W^}KT>>N7RUF$J2*&UJ(*n0IW!`@5U}OTNAZCbyCm<%I@4}V|5G|~!tWw| zcl>kSwO{zU&ByXQ*$7>%$4hmTf05?sNZU_JI4P z^1GLOb_#AwZV$x&X;bU2(TbyCyF80K{C09DZF8$*qG$MLC~|XV>c58vQ(wOS_}?T& zPVK&l1HBiiV-IIO{`%M5P1~~;RHi^C@kDXPr!j zrmA$nw#i>@gyL5x$uF_?iNXH}p*5pTiZwM@L_Em~7ULb@UH z6!*1D7=yup(rkkkSdR_|NL3I#KXxh7!9iQ0JR@43taf%HI68SLC`b@O(Oc$}qsh5y zOEkMLrD0|&pm=er|7n2fTU4v!!^2MF`~y>_7giTsm5{=y_-C`=62u(Nh|jet1$?48 zo|3`G*^JPjgcK!b!_4u+b7MDiL1hqYEur``y0CNeqax)T6ga*R>!slUUK_X&&jF_u zlRw9V1zc8f0%%aR6~7)r=t?+QwgzE+Lu#<|2vn&;i%c7rx>=P=jySuDxY=KE$^&n{ zD9kdQa9J&ZbayH94H!IL9}mM?Mu>%A%ao<=oZqV( ziWaC0`1()K!tvFhw}-6=i|MK^8MjzNrsJ#*BXOoA#oZw*=e|t?cSIROO}$%VsML)+ zBQ6BGU)wr~lRtO9dPUTu^}w)Knc{qTX-dO-FlJGdKR4S~)-$&2u5)S0Wd4d}G>gU7 zy>v@s`Ng2l4lYSvtaq1NXx{cmEkwiXS-RJFiD&A^Y-@gqTtH5JH$F{lb@PHgkX z^#%|C|InBhU?t!!FHQ2*|3YT>Hu&T)73hXh2rgDUi!wIJp!Oh`{hCd+YP#!OYu#`n zqvR&PwjA!(YNx`57OA=lV-<>scBv9!dG(^TN4Ualusw>jyxtlOS5C^D!m${k)_`PN zRZ1Vca)Q>Y%(CnKmBOM8`T1F^kkh5@Bn3(abGgbXj%uOz`J}qiAA{~v&ybBjdBB4} zJ9AkYr_?oAA~8V<=x~q@t?b#jDb2!|U$vBdE@PJ$P4Ww-!=t&lr6nAQIC2zU13~S0 zLzX}cK9V1V+z?edTiTpE4ru2utMCd;gk<#r;eMdJUHuL|GwtUcr0FBTU}W%D|4h;E zAoN%>m%adk1ba~ziU zVZTGJPhVO{WrLpBN+K<9!t3F~Et_K_nKO6()wGxPBfK{EWXSFRzK-RO9HIS^yh3ey zR5$jpB0OwUM%2ryw5q)3fBx(4!yhMRhEjFHkG!bq%);#L{%(SL`b(F&?y{Lvt&Of9 z5oEmYDE#mcr`GtLTMJyv0I>N~<&lqbS3#~TrRw44>D=4j{Jr0B2rj(U5#{{pq=rOQx6YZ9Q-1f$%156&KXy5+iu@WA+eNWb?} zQvB<>9#oiQYGxs|CL%rI`=s3ejs86X+wS+z@I5sn7eF~KXxh!+7dX}u0ENdJ#;r^n zbiM?>{I5@ZhMdl-nYrhAxWd;lEr!lJUVA9)=1~{kzOo1R8yoL@DVy}!)E*@r9W^ay zurTWPV@doy(`>!4GiyjzUJk&7rEXMI@tz%@_KztNgF#7%F{%6B3NZ<`WSlY6( z0@^-Jai(^_imuL|2RhqVdOfpe2`jJbmV!V?ITLhioJAHg3d24O#(;|0>gEl8|IBAye9*U#JEp9C2cj%A!A5-Ra(%(CXvbWzGIy6Td1`b&>=AO%(Zp*z?DuCLIG5W3Y6_m+=ZnX zaxElR_#zq84H*_PqgqIp`mkben)vv1=(%D!hHHtAW11lL` zur|RuYI~-c;0?&aT=6uF>DLmbU8%Aye1g_6&^qvPO&sY`R7x{4R98#3(qqg9jx9Jk zTExl`r?7MbpxkJN5jC}(H7#djxeN+s?zTr@c<+o`H$&B1-0kmGxJ#O?D~fw zx1Nq+4a(S>?oQnGgIa%Lo+{TV+O|jxGc>5;G>5%K=3Si;bnSX}Yo8Eot`0}?yWH_A zuAUILbGYXap-|>v6(Ry8I#C)<9Ij&99z;1+D+A|EX;UJ^AILavYOtH|LwwuQAACf8 zg&X&J^xq!vCVX0;X0y)9#g;$#JZ-tyrbd{V2z#xxaJGSnTQXQ?#P$lPZvZxDB+#K% zcc2!S4&4#@Z=sXuAq?M{sJ=~Nt%J<-94=rb6JNYy5jBMbKNU2hl!tNJM4ZZlV~lno zc`MmnMe8XI-CgV)kmFz_sMZz^yKrpLbUI)d8L#?74TdFE@t9^-Bf@W3Nj>u^0Y!Pj zUrP5MN{xXZ9B3Np{I&d5eo=~`Tn?wT$2z-Q>U_{PvK-_0@X=o$ew|u&WXqZRKY!x! zFz4d?sN~3pcV{rP>fWTQbt@hR-RsB|o!K@0NpCw2?hmhhwXNy)mK)QlZP(buJ7Yu4 zW8$=&-2Eyz(@!P)DO2^J&2UHdJm5+~O89|FNmh5m0fEnn-ir&TYY#l%TQm3iV%>qz zfrux^?@*^bNts^(E_#0!XIqC~*6x5iN!ya{FzyX?Ue0CTOUuh0tuFY{1K=>J(v#8T z8Po4`*D{>je*Jyo!0%*Uz~KXLPS*w;CLE2pll07eayg<51%1m)7N=j@xvd4fH6*}U znT4BwwK&Y{S4+wP`GH!n>A|zV?OE|~CnPwAx0Sl`d~dpXawp%8pLTTQwKm3g*YwBN zoJ!No&HZq>z8}|}3j9U3mvc=ay*671WxkD7MzYu_uks+#)`Ep&Ys-2X}9 zj_>F+;WxGNF96N{&CjQnwqJ$ZSkBD-?xXLo?|+%+m=H~mfpMkJP!O-kV9;p`t=1E= zZp5#TK1mxyLQ-Io5aEka#vrp^!c}&g*(Aiu4%vQ@Eluo4;0<^`)f;dt!ML@WTBtEM zZha_G+*tu6g3PWtPF+r_Uu>@Iv7VTBX$x-Bz@Nz8MHXDwP#S~6Q7EM><`l=z# zy)H;1?j^`sDgzC|oB>Y`z)xmrCWea^!xQIM2prX`l1i&ljLL6u#xHvU&n(AS&s60X zMOOwjaH(V4$-_$+%cyODHKq1f&U70K12;fddA>4!S6ChH&GZgEPTp!gYaHN7Wu zF3W~F6MKoL?B~(LZ(-Kujb?Orfw~A=0q>gIfduJ#V6A*JtrKN%(-gNT2RJK}$h9&D zy6w%{y12~&>Bm@VD_o&8ALPQ)IIm33`1J-EbXv(0)as@R-P5O{_JuMja$DasY;d>4 zUFijXT9(AHMk;$&7%3`Cm2}8yrsyg^K;H#Yse_?fj#1~%Z%cI(peVf7Gvz_N@X`dD zE6-VQGAqM&1xtcP<1;dzVWvduu@J1GN*`Sr%V{50k16o$@3~+AKXIR|kfj&3m!W#B zoxwfwy~F_46|6yFpRPh_($Wj%SPn@1+DUke0S{>6_Eo3iKFwZ^jx@4SD-OUKB;xfF zu^h~zCJWYYCN7NISC`QFMu#LH`zoa!H#SYO42k3NU&&H!)3GuZY3(CQ5CV{3F5nZo zA@((7Wm`SkrbJUPDoMkTIv!mhN$D{(0y}j&%Qk4!FPbXLt09Hsi}UdT5>;e*lBl#N zN04Iy0Dv5jmY(Z_;8JM;jE&!76q)*nbaR#baJ?_J{oM9MZ;T&gXE1zUSgKc^w{20? zu*S!mvcpe+GN|LcL9M9%A(OXRIh z_y6-p#o=3*kDUFfqf^EYIfj2X-F)|>)KstR z@4h~^`{AA=pMH6kCQHttIbH?0{9gKc$RYdok^5FvU9&lB>Aw z)1jxrk}kO3|6?)>5E*m~+D&Ql@jH|8E)Eq5_iM&#?xuTY+$>=n!0=LcCl6%0@~TMv zhYDP|o8NvyTl=@U$=l3QDN=K86b2{px0IHo;ImwXrWfjWq5R@*RmGxh-luY9b z+bTG6z?`ZTaGZhQ;Ot^ zbQ_Gd=>Y}YdL(19CIz)BMcL*ET~at+iiB9#98~hUnuyVZbu6ujB^3>@Cw4WgF9_l+ z&Ecf5hN9UxVP+QuhVqj$D*!ZQAs6n3cECC~4}@@58ew&AQ1vx_(Hc;o??o8HonfLp zOU*%M^)RS+$6yqm{Mr1WX8uUNMgktQ|lD7G1dF%V(=39aQ( zQ2@iT!UgwM!9j3W%s2F=i)L7f_A%i=&zG2>1cL`YN7l#G!{m^TU9>aB8w0Ey<}-}C zGMl;Nh_=dj5#!zEt#cXrJx;#rn?!`Yi+EqzRRS-fk1vI=R^r`pORafaQU!iPTWomM z*Tj_E5&4vE4c7tms)WbKmjx?RQyW-Imf@BvZFn_A=U47-f3s#LfM&9e7m9w)HugPs zikd9D!?Ca`;FkH!lMK!VE1v!4XdF&2KrraJS=I%Fhk{ZXVs2N?*K94fVw{jpsWV*KdwpE(Md&RVlQg zgS^un?H55X01HtjNIeH8bl;=(J#EwTT?ElX(wAvVl{^%57NhReR%xL&Iha*WB&PBz zO6J&GxkwW`v1cMBaCwxX-hy9u!Rxoc@`^-S6I%Z)v5qDA8Oabhc~Q*&QT;LQ7M$8) zEs;TFh$2lf%=(>IVoqu+$dyhA(5;k>WI*az0CLpv5KK^d-*dLk1?p0{sZ3kKv5il+ zx*5vR!d%TQ;Efj9*6#m4o*zfPWuCKraMx)(a z_yv50RWz;MhEgzaDWKMznwGOf+#lOjj8n^v7+?w6U?D6+-~%1$-;+P*3GP)5>s{_A zFB5Rn)jiTCh;4~uaK>Bx!MmsnSm>Z2m>i(Y70W#vHsE-jJAMOBLK3?aEp#Dfi41B6 zoFc7Leya;a^R5ko)rc)Zz*#nMG+ey>_h2GhDuu33Ro{#(RwTf**D6Q|;}xn5?$4R= z4}4BGb3hlTU=o8B#F;3N;P&IB&qyv8 z|NW82m){=W{mnPOJ(L|^NrvyG?OQBo19AC~x#6ECgF97> zw&oCW9UI5a{@`5KzFHYlNG@gW+aA4odH2q@NkDQLLm1EU^(@QlXt;aSF=1aMBw_|} zuRr%@D`S?}ocpwLri<6ryeYGTTPPl|KIbrlW@{Zs=#3!-`NIRO1B&3b-tdclnO|mg z?Atm%17^XtCS8|)F5Z8J$P0FjiK6IferwP4U3$&=w6aBXi%61lGfgScbTI|EE;8 zU}{4eXC11gaFk@RE)8WTqW~spLy)}$W3Sfsfb7aTMN#Kh02W)IR;xb<7p>g}guo5; zbjv}~u(%jV)ot6h3qXptiUXJthWEkZ97zH+K=2s9#zdTfuTF;`W7B7;t&Jt%+Jr$| zGO(ke#y?=%w-ogaWT9p&BuCo_vCI?d#(;}T1psw%$=TUUpr_bE3XpatxO;Dp778(9 zCOxJhpuke;53@{$>pL59CI^;494J-uW3X0<-DJ^N=I@}?+q{#5Y?QnF`fjD(U1%#r zYwjY8=b%k_5cLy`SlwZEj36P?Xt<)~kcMuV{OY-!2J;!Q`CCVrob|@s(-&@A z$XrsfJIo7ndhTR3{%Y_|hgN<*rMgFXCf!&j4#JzE1iG~s$b=?Y`rx`Q)yp}eoo5_6 zfE`SM2%aP3E>LchWtW2|xIN9;AxK+qV>M9VU?Qnhdqx*N6F+q>XyXjM!Nl*~mHs#p z)yG7(Yz}5(!MF2F!>H9}$f4-GMAP@xIV??WXDy_gVLq?_gM#lha-kLcr9J_-Hr<1= z)=tf=chSy?{L_mUqQzZ|iJZBYTN}W{oiyE_qrPI?Fnf`!)1rpTjt)6X?^d&oN)j;^ z>c)Dzmop)@TdPKd?f4bZWP7;QCC*gC!g@gOL~9KiaSr#bU4VjD)7wl#wz zT7bR7m-D4BiI$akk-zIKHB)go8jmHYiY^$oI0dPHi`EW_o&-ttMsa|2O($C3h_f4& zFQyx{OqnyjPYMlOSw`FT3C2heYW{L)Imc3Oh8VbJiaCPYum)dpUgB97CmapKwi(yYR2@Nls13B6e4acfb4C% zL4;G0k;OBLkuKHG$o!u?2QSCw2C}xng&C$ZD4mXwR$nj^PSk%GNzKRrI~sub3w0ML z>svVXy&SWNqE1c(^@Uv^$HUnWieEPh;f0h)U}Mq(afqcYB$zug=mpFuPQh2uFaJQgAbAeDey-T=XoZMI$f%c~i)?)($zp+KR?Alr3EzK~o8A|C%6;1o*9Q-;{qo~))LVJpSC8CU zIko4;Om5z%bt@f5uC?WD{?A{ZCEb6_xa9Homb>hn!_ixtE~SLmenjD}EMM7q7eKtC zkg~?iI-e+r^q1s%+F<4huczhk+xcIKFatoOz&GL05&o#1iKHqOTu`7Ha_2R44 z&T|i=Q&WeTeLWzOTd=7*n4OOZg+xMlC`f zJ3>2Zx-o0txZjmy=gOL#fq)~z7k4>Rz964C?Bb4I#tg5I5=Ln)72)3H((j*^W_Is1 zIB^@LT`|(Rxh_cE*72H5Mbee-u|pk6fhUfJ9Ub_ZS4Q^iO$eNc+Z0^T7hxW;}`)&eW{+ zjFqzsFFU`uJ)TwEfuitxSJ&3A9Cj=-w@xPe1l)711PtG<81wR#C~{K5)4ORt$*sNK zuF}=-1+PaHLg32tzhmuLd;!Pv`NacFHXqwIEuQhhYt48ymSYaafc7kq1CG=ox+I_^ z2^HD3LGRy>9cst1;Dxj}YkM418V~1)T-Yy;LTt-cA$WEO@KaMWetDW1v9&4g32oN$ z?w~V4IVh32Ta8d4!V0)p!f`>b6`5aELG7>FpEj)1kwFEEFP2nHb0!jKE$u=?(T5%x z_o_c`Js4CYYtg1wjzwDw$x)0p340qu4iTsP*fX^evYsX7L5f9#UUT3WNuaz)z?s5T zrZNOmClxKG5tRPs1x~??3tPR{HW_EN4Zo3KM7fyjii0Eyo^HU=5f-Uf#L5?sGOR|T z?P%&StWH=wU0txA!g?bEJ$*X=7+zT^K+QLy!5r8+S%+9zr|~C;?m^3=(JbojMQ*?J z7Q|Rs>r{B%b{$_R4k#nbv^((G{2@ZX%6*s+o7srS)AU!6Y?EX&^6Vx$im}L0Fzy-P z4cOJb25f2m+qL)M<$<|)eQWjQOYDLW?2y3#w1`H;MqguLOVCDXpV6O(O?1lbYHyzc zzxLugaV9ngyBO_};5EJbB+eXXP~ z&RP>^HTC-2Bb@3-;+EXdJ%*RXE-u}0QMT!1c{yFyhdR(;TA&(2vGl-8I(J0jtv*vz zoNw4{B=}Dauh!?kt8f{JXT9hM*y*Kg1GjGCfzQYL+id`hma8(FvEG%F2_yoMlkp6H zD-}nh<0V5w(t3_*2p|E~0^)!i~lhMA*aoy98~y%4hqiG53` zxBJ_paS{{BVrweLDC0gr%hb;z;~Egy#Tge+I!85m!d|5YQFQvmfV6g{P5CntKZ~#^ zBsMvr60xvxL>KRbo!I0AW;^1w$q!YAe}@;$ZdDfP;6;;FaG_-aVbzZFW3gOCB2mA= z9AJPVTH(xLK3d-91Tp_U#KFE~)pAMW1WzaI`u@^!yvB^5)jOiBWR%$nC*e1&J>qay zZgHPS^R6CFsNt>lOszirxaI~xxs-YcZl`s-MjrK#sjPOatmMf6N^!0*uwrXlSJzdS z@q1Ydw;voi>bUuvKm7wN$bRvp@c^%duhX^-{`$qqFE{`3?cY6~p4j@I%s&$T>Kk6U z`$5MM&#gPp&!*Dle%`bG}-#=DwJ?!^jcm2U*UEiMT|2k53NB#ZH zGKZ9>Z8C$Q)cbDI#lIYjOy*;#>_O%M!R*xX<)+W>1ov!QU=7AkXI)^OCf};6Pao>_ zYtFqH{-E#s{I3q}`{ZH8i4;+)qqPd3?=9PH3L2X_1uUZ{W=$O$aAcWy2x3F8-#42Wa_ z$&$yt)|Is}pU-_iPdjVQ!1zTa+bvz@o0?&$CDRb7Y=D_>piS!|_Q$Y|M=pIxaiW@MDs*MmJ?I0g5nK6g|dlutr;|h z-{5Ue&>Lf*eWa|$*H#3{NDH~pWv!IlX2;c5JOU#*E^ZHB2el;``S16FUr za2*)^4tk(1!?1P|W<21mWIAWrXO-b#R}(vu55E5Z<%n0ms>V-726fY{HwAXw=H9;x zi~mmW|!W)lqPg^O7s(;b>n2R))iCORGJcOB+enK2tne zB-Xr6fEjC~JCM<_+Q-Jjl#Z!l&6C?h2y-8)PBdFMgLb0WpSas>=m@T+Ltni5s0}-= zd1e?*K&8ldq!?F~uwcC+H_*S!c*)Ep7IvM<-_Tpqg5-+`qriQgzb~xkw3S8-%wH&> zuihxfIM$lGyK_%7=Jq?yblsB&-ifpImV{tBCPk~;L>WuBD49c5cKu7!?t!N{Zk=Vh zK?-_9O>5p%hGU#7>3|K?_82Ppp!>q}G zH%mZP6=rxPP))z?d*`Y0=ZNZbaA8j-JGN)>Qop-Znamt<54b~4&%f^6iex7EU{(2Y zQ(yhkdY2f=#V&Dl39n9uldi7z#yyeOt%xjV=#_U86za0XHW!?}QaPa$P;B2>ZS}G$ zhcpbPXrguSdB4i0{K0qqetm*FVFXYW^&>&|6D^`=R6a*u3&l^gLYU@C;keTb!~AnF zT-?LL#~a+li)ZkLrI9w`ZHi&DO8+Tn>4Ddy*r+4gyGbC1n_4&fpu!3FDkNaB4+SO{ z0u^&f9>QL24eix5JH(A+$^m%H9V{tp;=3^>h64y|1yVGpD3pc<+%X7Z7>@_M9@QfL$3(O}f_kA?A6Y+*6;4)1FBdTS1M<7)Td_ z{#cM2ZPO6M^JGz4HV;`X=Hdd>rj&%NAZPRC`(fsL;Kz{g z!!1^*?dX>c>(0nnx2G6V-VzwqwY#+8uU6yLCAll~wt|HaR)q9@B;MwZTLh|=SE-VN zoH;&my^J#fngW_u)`rApQb)(e;I(gn$-P{qXuw;>i54fMWowGH25Jn{S>gf|N@n#y zr*X~ntQrvk9gMTjk5=JS??)-w7h-Og^=O_lFPYmoF%S_0tc_INixWScOy~BuIHWyQ zj{B^vAhHD!zE(O1%iW6C#Y_+MT6-sxll=>;(fJ!lkV}RbKcc#e3%>UpXoQeCrsCM1 zVqZj%y;qFupT8KedRTVQmi78Y0Rw9xl0^%>D18foDG?#OFqs+Jqx}N(Py&BrV`aBf zbbZmA3W4ehA93T!bk6w!@4+)jHu=#&U61GM+ZVGxbMK-JgZ%PLb?$9$4+Ub-`8FO2_9!T>{7c z9E@*s{quVbxKzC6d>*wlK8oALq1R;Lz{FM_t*Y~*by-IUKhY;g!Mrr+(#Z?j+pwp3 zl!6KQCwM%s0ijbx;MI-J2I79J5UNt%wAfU+{*wL@$ZdGqvRr9e?#zD|GU32F=u~_d zDlGKTDSCqxugDlOF;BKyQyoCfOfxtkyVn1Tz{x|Q)cq$IwksCo4OMb*7U=WBK# z4KX?qw%<>D?;OX_Xx*UK{S&G&`NNDLxF#o8-YL{yU$SrTt0}g{Xr|-^plw>T*}?GG z?A`=$&mt7*p_V?1^$h^=rvRgm79OaX1GGCs7Y7d{J6LeJQxHL6nl)>r8hYN;vWBXZ zg(fnRjXk@Gxc)8Cxa@!c=a>apoMA|u?>B4jMQqdq&KAs#R=zaMdLdx(G{95^7v>8~ zm~i`l4Xz_@fjRjcFNamB5b)-e<|6Mw4R(178^$OK27>k=;Qs=dq3o@B-Lla^h+?dAZN-8<+pp( z&DFKCH>V@2+c)^DagQBM69+!_Zyq3HPL_v5EOzWV#Wnr{E&$45U;9N3DK{cs`h zrsR9a-v2vHJ5l8ejB9&h#UqPNyla09Upo+%wln+eV%OWolB8tc%Cx%I9l_h~EStO^ zold$iHg)eO`oh}s2pcV|e$oFc#S6NvaKd7s@d*~QnS21||Q;!jE^VnqD>2voQfv3~glQ3husG7rz|(7 zEKO0-6j2el@8cAeDbpqzDw(O7DH$RP0#l}Dxa5i=OG++?fQand^X>Vkf4aJ^x~kvr z^ZR_>@7K$?FE0j-{hjPKavdSUM|tMOFawugC3{_jCiCd23j<4mmotfblPd#omx96? zbe}4xv}T&X%7y1Nn1c0B4mB87&Y^5rFAxU<$OJ2}XJyQ*_(Q6g%sEb`NfxAY56@FW!i$Vt_6=i!OCSux{!e6nlD2L^^t^G5g3FLF*%=srs5}t z?cJ1fc{5wi%Ul^f<}QW0obf~CPQ7Ug&xnDg*X5wNfoZY&o`P5U1?2F@M zEpgO_-${V_;5>zKem(Nh`9(|6$4Xq=JzK$+8Qe+sM4Uv|GHI;of8h5m7Z>Ot)2+Yt zXr+e4=(p;hIi@*(%#vnKLI5K7H)mVax_sXO{dy zRxO=^95%q>AOu|%Y}}zlwM1`RS^C5#sW*#)k{{DKtEF%$Xw7f!kR9U)@)sAEV@S%c zny?(pfvYhLBos}t9g|+~OT`mSsM8GnqQ2TzJdV6Jzgbm~-~qlp>6GZ!)>f#$*FG{c z0=IfM)Wc&|f}tqvpK4@Tib1D!Sjx4rWdtQlw(>5tL!+d{CFn{(iM!&#OOBr5Fv6>_fR|$yCP@q<1C7P! z{kD$9_k@k@n7vWQp5~9?iK}}YVw6*=MO<=yPC4*y=vbWHtV3gyeDmIgVioCa*mo)D z177%L9+2v8CK0l%VpY7q*jwR{`MpnbhTu{i0-f$X?*tge>Y{h(aTD9ja<4#MZ9`RGi1CV$9)ZEjz8=eo2k zIQewJ+kvW9RxL|}hJlDea%a^cVcyvLotsC_-ebLsiKt&p#BO_CxRmKhKt%8LdHRsp zkhWC?r{HKkoc>1sswUxR+1>NP%P%jYfDB{qGGOXZQjPi(dp91C-y((ye!na=YGcMkN5Q(6Ad zP|?n^k%_3IH@`X&Zv2KrT%5az#}W3#4kU&*BoZQzxB9G&^ZHix2m-0C4h_bIOz|3) zI4ME_Rr>(@S^j8+YxJl^DN{ zVe4kJ;gz3yakA_b+h>5ReI>BicJSsJ1S?YDWVf~K5NK#TqVsHd5Y~p4>p)!6*mT?aShEdTd^aMz5)?s4ie@8rm;VS zXTObdsReN-5c0OUDo8tsS>=8hKSJ?l*;fYbr-fYuI*TVRuyIv-{X3ZRK8&QmvOCNT zeCrANh2pwI#rZ z+ojP=19!D>Yg{tL#sx%(3?$cCC90J8>FzUhr>Fx`crjVpJSmfP<@}r7enD!5H`E2k zamF_BhK3)7?%U3%_hW}I-?1r{ka8U4`*%p zvTgK?FGw`wt9a=^xI{Pa=IHQ7db6RALsYwr#R#1n=$>viZEY1`-jBjG88`4dZ9knbqTTjHNu2NCJnY}mb4|*yoG^^J&0*`wa0lH1f1-foP2xl zV2UR;DmpJrv*M7bw5EoXEl!zRR6O5&DQ=M)%}w4+L38yQ0r-YKEhvLwaK`F0;| z?tM*L2xgT(QbgG$FnFM^cWCbMe~slqJmBhCrm0;i}WSIJd zGx5M|E7Ih? zmCN+QnB1jA(h{E>*jkzY2#xV?wcZ+R*ED0EkvlHL1p4OD*iyb)1dGXE*$yp9+GMN5 z)32|7jK2JBh5Xpfz#hDrj`SYIvtgB{I zD~9k^qaY0-@&66}4ixlY=Qy1B2%R?R2W0A5{qwY?LA_k_oyq=$ zew`)3^#NF~5}SBNq-y~zYwuI6k5G20$@&E1o85u%_stU{0(D1ZpDKu2lWXZyC7yvd z4;X+Pcm(7mv}|W~`PyLfHopB4)H&l?9+hvU+yswal{Cx#J;kOMq|GmG!{k4Z+bU4J zEFu7%r~&z`?(W#LtP$S>YolQCJP-wXeEz`JBwk-k5{nju*wqA%^85M zli#|)X_A+=x0m~mv+jL6TbjDJ;Yg={WPJ3RH1ER0XEgr<`TXzE`xoCHdA-4~JrBZ4 z_MyX0?IOqr;he5!PA*54klOs6^zQJPeX8y|rHgm@l%|mm7-%-96ac(r_su!x#Kdz} z-N2~f!R6S+C;QM~k&JzKU!R+@x263YMg0);fN9Yg_P+f2oJq;;{!~wdl*~(3Ux`g&PJHOv$XtWvRUd29axcnLQ?OG#LfweLjCG73xvI~a^o{dDzO(0m?L}fC zr}+r%$nUQY6s4_KaEd~&KM%3Rd3FuO<-}6CSADPKzHbSme5h)9^z&)7=hDG%_kX$? zxbi)tBh&uwP^IxVDxMW-Rmh#{$%J-?aVkhY8AL+eK!-{5vvQj0gKPJBlS~Zf=X;Rc zQQ&PwrVR+!dzqf8ngskB3SGj_8r%*)ms;Mg<0y8_Q;4@ig3NCqT!)t4ma86k;AM(_ zW2uwsRwHwZB0JhhN-fE_nLM8$@(LiTivj?FDRX|p2yC|vE(y+&Q`VoLFPf@MpVwFn zp2+s)Qg_nchch|NAwe^r?3P(#@Jchq@c>~SQ8l$2zo+D^9;&=Pe*+JbW@;C|*n}qW zS{bU<=OCbut0TD%B^!eoL}|{&DH*PnL2ZZLcskB;CXUxgP#Y*YOYx|cNa9RgG2P#o z26g1&qrRNKJSNV`pzKY!UP(AdZcAbM-< z&Ad8?ZlH6XIptg#?!?GAy19Qu2=KOtLBe`_7hS4vse745it*5nhwBH{FvnsJRAp1zLo2jHR0h?%Mq1!UE;?;7>myT-$MU z4o;fOyHmg&4C}#M2wXVwA%!5+wI`2+XSy zBV3c(5(CoH*VJ50{`;j@@1U~0d$zg45c&mq zmNbZziQ3cP9+0_WmwTpEe3raftYs5#yoewf&}^QohgfdwYA#aX-Pbbw2(QxyND7m4 z4jZQ)8w63oDhk2;X1X1OIm{}7Ia6Oj1{B%X-e0^90&s4&CLxJ4pp+I0M&$e{e8(V{ z&M~fjtPQ{j6rM82nzW9ZKqXX@-`$u<&+i#Q#XbhF{-O zd+EdY&zsMqPx_q+JNBO_TDQ+yMcy?5N$OgmyyAiD|p%3|}tXSnKL?dlyO?G(lbMc9Nsaf?E4@%Pdfd=j6 z{qwU;$K^QvS!)rI-`>t@F2(Hi0Z7QXxdfE$`FFzFty3=>leRAPDt_<$?YA>w=b!#k z)e@fSo%l(70=1C*@S@;(4dbfU>hwXHcQVic8l!8{TGI-MUwR}o&c>QC+4d;W5H2Nk zwa{-WT5q{)d3ih)n=rOF_Sm2AzI#%7=k-l=4Lu3ZjP7e`4u`{4a|vbMmlN@=6So@G zL^M8_-ayZ7z6fs>c7yLl^ysDL@Ai*-`89_|qHptI;G=P(ml)4X@dPeU?t?lXAGXkV zn+U3M!FzhLTWwpIKv8tQ=6%-gzmDxBgnxA^p?~=Nx8$M6wb+(_bsybx!&ef|dREl+ z=exRLm_@AmKOpMa6|D!=Ez1~K1}F7|$(P{(e&jLL12sPHv4beNmD5!ZUb?FdycOZ0 zyD;5-$`=P2H#)6d`6Gm2j-Wdqc3sxJU^t=}&fN^h1Smjp`O1(U-n0y_{USXYgPY&8 z&0Lh_!5c-WV4|A%1yT3ut)&8ui&LKaaz1cLG5P5ZP?2-iBYEmdz(&y)cPi>AgqcLa z*huB--QkH~!pby+)CbfD_wc52X&`aw5`<$}2O)i21CPZiZpY(LzIo>OIM$>3(SuEq zCK*n*dV2(B9ELe>G7RIUa^S6T&A>tYLY0mDsxw9(3Jr*m8fO<{DbHWgw`(p$fe+ajISNsmKef_KJ`Lfw3Y_1$hmcuP;Y)8Q zPV|2a#Z9W;0UATTqK-4b1$f;2#GA=&X*s`$y!_J%{H2C7#Rj>nIqT)=SW#F>XBubj zd;Q4ZIB(B{m4um1+9zoMs1vB`B|OA<%RhZ`1& zv?etzUmDR+&=flwgbd=C_|x5Fz}s#vg6A-tZ_}I$rq~XpwNVn3v)Vs}w2kONyxLXk z!45-Y?-lh!_2Su4q4E>b=(M|2DxEze@m~L+41)jM8${eZ?OwTITiM&$ef4pgv z#J#praY56Rn@QIMnKsR#wfdSktWE5J;poZ0XGYh^^R1Uib`S?zMdV2;OtJYN04Dv2 z+`NY2mkCmj)A!~+8RVa4&ysN<5b0W`x|TnYgWA^E5IU(K-8hg4WNUA_6FR!l9TJ8^ zqUHB+dCK%IDOm{K@3X?EW0JI`xo-Ap6Y_>bW&hoACH%v z{)e-xUf4Ahtf;Czhltr-=sPrmotcIFX$&It6Bm1u=w5F&I_~$0C`;c__^TS4#A@4{FkeC5?j??Lsy>Y&= zF2B7|h$p+L%Da;ezhJ>?w%ys#y|a5)(RnaL7b^~rpZ#CH(W5Z8(eV=WlhIk*KP+BL z75?Sf&eZFi+0XwfAuICy@^+{G2^)K{k5hCJJ|&h-PE1SyQj%D#v`yywXy-Cd{4MD4*Yi4XZZ28)vM=6AHA$td?~+GpS{ZDJRicvMxx772R`I4 zW|yXNqK_+?#(e}MFY%B84F4DURNX(pX}DPTSZK}P?>*eOW!u>+^~LX_{uU*t@*`0T zFF3)Reb$T3bxp9UyXoGG+~?xt z@t%R^#;y)jpU?<^_W7;A@xu)J?fvkhe_YzSPVWypvaxWu|LOVv-X!!*E|v66{OQzR zBJQHs6i`Fze?ELBG|gAPV7aUZ#QG4jJ0))E9?as;u!))Kil(WZtOuFn-lM6z4Iw0B z-4{h!@5mHr(kq&H%33nQUPW5Ak(MWV0vBsNyjmv2`CV}-5Bc@4;l!7eOx1wnm=U%co@(G zhX6fnY)87PRIR}efZrtL7Xh#rIt$O%MG|jB()&Mz3RXf>($b{HyNQ819nz{2fGVvZ z4CgF%H;&1TQ&`(}Xs+T%T~-Fi$XQRxRgf)OGRoYvgFz{QNbpcjv*ouPXX?Vf zMcQq~uQAUA1go=|oDC7L)F)-X!cRBVX`I6*uL@4KFSdY+jKTG4U7D`Q+`fG0Wvu_9 zU`0~rKxMHe8Y{}|S$}>+mes#aaWiCmuVcOQx}U5)CUz^VJjh=XAgQ?7HP?On2FB5F z(v8e88Ut_@K~d002%9<+X1L5Oif1C6O#S(C#QQ|MWpUkA{k1MEMD3b^+Wj#+{l6H$ zBUZU4+*ogmEHNWu4pbWF4xWJ6BEJS1l}lJMvolgxT6GXFi`mB|G< z8y{Sl#TZPANw(8QEe4brc~ZGi!^yb_}s6%B)!zv}^^NV)%JcBe5it5n!*2U#h zCOk4H4~jyy+S%#t+St{u%ywMK^x4!9=JXR`6`iVHPfG-YrXrvxLHe>V_&LkSJb|;n zGajjHnw%7iA7j$e(lBW$S&oi}TXuhcDdoWxnxuB?4rKpZ?*osNdWe}X^m`r84nC@kiiQjMk@BAWA;VZKYMW&6)Q035?I$(rz*~tzn**_ar2HR>z8#k#~(hgI{U-j`z1glQGs>5 z)P^44wf%>$rI+KGB^M(B8b0B8nL*DV;H1eOHI1GH)Uj3lzB}_w{_35dH(+_3Erhy# ze}g559za63dpXX+bgw~?(4kl{3+5^CWqbqVQeEM{fEz{kH=@v3B@P->eYX&FHn3mjN;^f%$ zbnxtR5n#p%QE;5%08ID$*rdZFtXwh$ibK;2mtfAd>|iBiZHc0x!%>hOr&!nXVG&NU zvOoMVZeB;>IhX&ytn6dB?!fdPdk~u9A?6npA=~*09_opV!>>@BW{7k8HnKgPXwrJQ zo_REJHA$f0;nX-$_Gx-trlJ>!Dm^?5GX!@OkgQm*#@P=DR)m7A!AX5r#*CQe^Zz48 zN9cK~+Yh4s=77#Rt-k5SL+b zXZTqgOu_ef^1@7T@1sqv{1b?ayVPV~E*Wv{#db{IX@oM}78Rs!I*=H+c-6;5Rq+as zsj1C}wC;yY4Jv3eFRJ+1X8QH1SDAb3iaKn&MyjElYa$VxFrTv+(L)ZrVu%z7oDS$+ zB4cz!&$j%*wm)LngoSJdCJ<|7#Vx);m>Ws9-y17zl^T?@DgYcE8mV~81ixHPkfXknAjotY>y&Opr3^=!XLC#pipH!1 z@>LU&k%mFc^~dRY1WN&cvMpaoLK23zf#3wnF-fQ3LrhKQzqtO)%kcc;>|A>l%yckX z4^%`JL|(U=5F)iWKjZ9P={%-s74b0j_yq zhu6bw@5ww%bpG2|wnKUhIOczZ;QyYQq9TO6(#VzD5Z7JH6gO=>AT`&y)Z0)R;M!S9 z=Tx8ROzxVliSc^(rp8xdn7+7A)|<0Iaq4d23(gUFa4R!6ACCvR2-#q3tB*CUwY2|- znRmApJHnQhQc^twh$)%z)!Iasij%f6{D-Hq& zgrFsMq&U8|>S5D{M^B@ko*!${+AcOn*OD)td-|z-{LgdCzmj-ciq}%fHu84=_CLk; zam3yUg<@=B*VBerwSUdsCq-!r-!R36gzdE_Z+ocE<74K-e?J$`sXa_(c-==v3q{ zcS5i1M2CB?`{8Hz*zo#w=k6=Hn;$l1|M2kaSMY=1g@1qQm&xr<{~i0)>02+3z8Ek5 zpIc48)Fg{`jXi48&Q&#i{Oj&;PE*g$>OJfJ{$J02dr@=kbaze3tFM!y2i%!b-*meCR(Vj=?vuaz5zPI zw1`>l7P#C5fEVhl!%4AI3^0cTt4|`^Ksu(Jm+m}*(vMZoUT)5uULtV0^J#g@;9aNz z#FBN4bU6VB4!BO~^mBnlNCIC`7GZ%jv=HqRiBHsaXf>f^VSjzcQsG#ko6s$2RU8eW zk{nETOdCx6VL#s81mZNl#1-Fo7ych70Aiy!51XkltXRg%l+E1`ORaPJ|x}WP>Zpi zLgHsqaWZ@y))*o%O(hPZz}Fba)`7<&GN(J|Rf<q1lCM=@2CjZ{P5A<<4GdS6`OLQbjR`h+;U#t1$s#m)hWI zIEE9j1@_oplI(K|unV#(#tb--O zjGKx#K`oo;9Z4C^jL!JDVGoGOsC><-15{xZgGUrq%I(kBlXULxYawi$7F4p0o@EbI zcBs?2G@|j3(lRrpGv*=>pg_RbE@GT2xjr(ni8gS?{Q@8}jG&zw7>54gCpExjrn(L= zvK)B?hJo3HXk3) zZe(c#ZlB$7@XA-iyXq={y6MB5Pisa{6FFm3UDA=4J@ua6loHi!a!j1RM39w}(-H^W z>gnlu=7Qf?SAICagY&a_zuYwJU#nQTmYdT?%E?KWgA?hR24>=*S7Qknf{zu~v~95R z8QbTWH)ua94xc^D8n4Z0kAZq_UDLU^kls#8_hl)s1zloe9j`ulncoXVJUX zvHpFaSOMsYpka2nsNr0h=|t7xgJlC7J0IS^k}~(`mlUJ#xhE?BlLTuD{-TUVC$Qxa9gNM%~u#tvAR2HUCiYDWmjYbxri2fAAPt@adZPpC6vLwLFa8 z8Gj!BeoNoiLHlY}C(nL;8hx@h=`a7@{m-av;iFzpvkAMd9#EnG|Il6Q#6&q1(TYup zSDCxPikWyBpIQAS5HhjgLs8rNq0UYZzJ=?xRV}xU)|K-$tL@B06zIjz-K3@E^cC_Z zpW@OH@A^y`Htm>L@7=taWT4q>3+5Rub!IC*~7Yc&~$m7@8p+4|*M)cj>l zdo|Dmk);`$%IS1%FGaB9pqK3*lrnD)5TWAM^5;JHdO)2|E=qI8zy*(PzD;j8{AxM? zbKE*4ssK=ZN25Fbb9`7V-riTxgdnKbm;if%OdKXVM!iJM*sNVC)KX9)@M`BY~MBqkZGC~+8itPs_{8nz)xSj z3{;%@k1>r#lpPDzy}~_@rbq(qH2gmMfUNuAT1vM;&gLW= zvH5MomMZ%P@R+k)4xAgZJnQVFK|9(Uu-S3mkejExLj#mZ-S*Fyfqf8_wi6e8yXz;H zL(nF^Xe%yIQb;7{7KQ}$O@=iD+Bxj}vz6aZ49URl1{)m!*K9>Kp>zkZ_&S0d64POO z4y2Oan90&u(DNdZjyda(H2pcSCjPl1%D!wu!r|8I)%Z?&)wE^oP{fNV(QNdlh3-Mt&*A5)Q*E zQb`%k@8pIqI8ytNTf**;#5=9-8h^7bv7^_GFQvP>6;KzNw5~-@#*=ZTKG##3al1m@0gTU-r*(`r&MnAFr&=80WZfBf`c zSAJFa6qj9@8|^MxGyN8D8_H|9M!Y<)YAzCvt)%lL(Ow$0r$+7dQnY#TM145(Vt5Lx zd_1JV@JcKX@STuHH$lJ+(sYeSnx1I~Bk-Fy&b%mys`G5}UuwpivSYn>4~Y)8UP@8y zFD8@8%B^5#UPupWnmT+&F&ZW8OFHFO{qS$EugU?rXWd^*qkQHUmtW29KiyE>S)K9t z!|CMnwY3R5%EFGJFKoY3|IFgiT7&i zuC7k<`+VJRA64)@VvCn{~h&@ zJEnYrbyj z2>YcFE*C&ai%DiO097ZkgBLDAEYnc)wB<5!q`=Gi8s>N%-Gc+IRM+%0B=|l&}IyLaVHCfR$>mXCoA{9YKa-D$zolNjLpU z!nq`q`a~QoZi4iAn^#_EKU|=rb?Dl>h-Oe>Zrvd_Br_~g7|Q@=8PJ8l`H$f+Ffceg zlLbz_AWva{_ggJ2cxC%5z9YS(<#DFY*yel#((+-hWyt)#KtgQ1=_@ugU9>kXj=G*U zXr8(*thRy=#A#uN2sBA>{WXGi+b&8p!Z#rRQpQt=Qk=u?s3^o3+sQ&1i&ic%d;;3y z1MxjAZMO@T9?@FgKE_%;MRJWr^0^^_qXC)sz(g?Hvs`V0&!3O6GBdsL_A!OYwTL?p zqzJxk73&%4OTB^|9Yc_f5j4UXQ-P*;)izDxWJm)o-Q>Q6vn@b{1#VtLm z>y>1I&;<%#O_k*rz!xdJRz>rU6#rAeIMA{N7|M(H|88c!y*85YQb zczt+ES~-vyzmXnmDx>RH-4N~I7h~NO&k;wavPinkHnNJS7cX$t>)C`bqodY4(^aLZPH7R&{+H+3gH~5=GKW`_N2MM zFgt*-Rm&&nngIQbht191G%y%vzY1O935+zg`50qzw|3<==CC|QEw$bS7#2hEIlbpB-*H%K-x1*hc-u#-T?$|L=B*SB7|7S$4wQ z!-%lkHNV&NZ4GMmDf08Ve`{R4(06~s<&F1`CfCnSi;FY1J{<=&Iabo9^KNc#4VSRG znJ#H4S2+FYQ#u$N^foskkLNJFdR;epdEE*qR&auE{i*NcxZ<#9WAV|9^j9e(hfjw@ z)Mgw%J@Prlk?#}sAl2hXdwW~`_oGMt zyPZ*W3&?~{+)y{>M#h6+Z!p;B_8p$O(?9Z<{@zlbq1rMRkjhtfWh*&LwzGFK*8gzy z`yXl#9AX~XQNwDHzP?I(Tu~O0fPRo6Oez=qhI!YWt{Hnwz7!b6fE1s{yOCA5d7&z4 zp-L(R_sR>Nxw(fE8XBqtqrw#al(n@54&CzpXw_Os?WISZcYLx^Unnl+ES33mUk#yA z(5&#u4~LUBUB7_8z%IF4ed2V4m*-3ICe=Cg`qn2cb5Y5=Vnrra7BGZ%$i=1JKJeD+ zIyzPS=-{5{DXDJ#4!EkUqTfTyGfCustgWK6Y zHhpkVtErfnZ1qWN*SsMa5|JE*(bk$c)#HPN#%(D+-w?u=Z1ZwOx|VNZEMMagvl$@% zW-v){W)8!a2T1;^noxK8GS2miy;1?l0t6rpTGL4|$+G9fXmUX(7G2mFuSm~QE_JCZ z)QO=LX>0`4o0>`d82Aww%>LpOYF!G5>j`Ja#AVzxqO zsX&gVQYaCWeWtzR?%CWPnJ?*Gyn7*0lDAZ!Eq?=byo;T$9CDhe#REYDfJ>L&nPK^P z_VX&lRLo`T&N`acLuhg8Gp$hE{7@lysw^W@D#tf%hTz}LX?Pin!^vp8mHZ7Cc^V@y zmx=NUt;?z~DK;B474lzy%So5Z8j%%omNBc=QkyaV^HsJ~KRSI65}YlHjY#gutcT}2 z9J9HOw}Y5iL9CZSJVmmixrweghGm$EvUN@0^q{(DE^ZuI18Z(3eUB{nKahxyA0*Fn zqrFxZLC>-!8cLe4j+?m$bNDk>CnEEg2yAKQig-O2hl-N1aZ0{Jlv_ddwSBggTD!8A zlD_;AiN;~L-~xBQEM0g^HE^b(ZXnHN%N$A&xR@kS>Yyf&)Rvo)e>*a_3_)Ch@J2lZ zD_9737A$T{ zf)gF*Iv0nIX)SagsNg9E+=A@uz_5*Tk5^X2dCmk*0A`qQPK!|Pwvltdj(_UcaZyW6n( zE6b36c&#eqS_oO#Qr9Uv4{p1boty<+TA`OPZN z#z=CA&{0s}=q%uYAEuXo#b#x=W~PURCYkQV=M^=MvHdL>$*k3p{R8g$@`D>vA9rWO zG>TN0SX*z7tz5(UCbZVGdfk5XuK8bdc;mW{xSEEFYXtf9-TxeIFuXX^9tEqrR0`Y= z$Iv6bO{btY{x;eF;O9d}5B{xg+pSyw*uA^vIc;pRV@iB+)dR~v2WP0+O@&3O>lt~H zK((d$rM{lVZgQ(#xRw6Z_U_V@vdPzeH@5iFdi#3+x4gS7tTyQ}$T25vjTWZ#H_qm5 zqAJ>=7^bx)oxj^`Dqd(qD88ZkoR_BY=Kp0Z@4wzs4h~UHS^6u%mhru!+?9oM&vihVI>$U62QBxPo*EKymadK+y zhO;2dan^bicB7!SdG+k^a75FQGXdbxwWM_pFe=)Nk8zGCCVLmuWwB{E%PNFz5BR<| z@{(r94G#N~)O|pb<1jYo88w*{LcRN*Bbs15YIB_jBgn7SAa@uPns&uK@bA!9j*%O@6XXXOi6sQwl$ zOq$ta$#g&1p)U>LPbd#T{L5D`u4RneqLyiDP=+IiJ%eXY;>tnKLoziLtpUaPSdt9_ zTxTd*9x%dt1rTNqz?}bh7H2AwErW$6OxvC*I{FYe{(P*SGF9x~oKA)-ZOhbXj#NZy zOuB}nb8WS5a`4s&Qgf)4u70RgA4$Rjp$kFN%s7nyAYZ#jylAI~B)sIEo%ClZsBh5@4G{~@PHiH!0(P^XSH;WX*C~c6lwNZ22@@ddFH;YYj zaaAh*?Ks7kQNFu7d0F;fohO*dI2qS)A2t_CUAZU9W~>f~0+VnZnk%_nk%l7OZ(qXg z9gOksnCohC*#xnu2zNlFv^>WATuTI>+xJL>YpFTGbQ~dW3z?f%6e4*Z0J}>v&^jb% z)casw5nEQ3}rQw<%CA1r6NMDvA^&0|W`SYNX-ygNqAht$iuNhvPjqsSC`N-uBqn zR2w4bq9T-uu1^a~PP$W~jWPX$o#Gk58@JkHZ4{83i2pu)0VZvYR0wEM6k3 zQg&cUMOCg&s0`l^=P5rA3`udGzqDdBsXeK{zv!Iq$%6ty8Y^2yxd~ z)hKJ>>oJ!G%y(@qhUZbSOP#VV4KbD7p zYuW5>so1x*FX{EumcBpXcVqvxKD_+mhX3yL4eL+cB4Rb1SS7p*WcyH3(+1oq=&DK_NUZP)o1446U|pI!d-~2^-iBupug?g#W|oQ% zpN@zcd3dtrG`DEv?MluL8pvdXGKv`L4b1tfQH66o=c5u*WRq| z%*gc(Tl4t(os8w7>;8d{_r*qOW4BP&iifUunwM=e5TFg02ReWSKoe{^A~l z`FlugTuhCyfj5LkwW*AB%Ot%a2n5bzA0O%oL)kvAuWJnp1Z^Duj1)xxf0q29%?LKT zzBt_*N}icuD(6nudKAr~pEDZg@PANPl&2|Z_9R>2^x)48I^R>U%p7GTesVS(|85(ULwZG890l@$2Z;P; zYK~dg-+$?GSY@DL6>L!&Wpm!yrnTJ$>#w;+49TPeQr9aUb;|vf=69j`t{kJ$+1)s4 zDG7l8;w7k~<-fXfh}{QW@dx|d^Ugk;jG#YLh?alm^mJ(KwqVTw+DFl(7fu!|T8C$r zl?2%&u#zD5b!&#lM_hT5++Rq-P_@K~n~$^n87b(SskJ7sDA;d4??YA9 zhJ%#BWIBb=(1pdkpQ)|d8?#USW~MttHSFstJ7DjXd+`l!D3Z12X2jH%3kVDRwY{re zV69E#V2Reb$>{tyMM!gV^>piNE}NTq6w;nypF(Vg+CH8zEguU*x*J!{$bX%mW^5$c z^d^ZVtAf)Us8b4eIHD$3rs;vROJ|~^nh4hK3l%gjX_-BOopPX@vraV$pOa&n)~%@| z_O#)nm-dPgr2dLR$MlUTk^?(bN|Ovf4<-m-{1i&o z-myaq_DUc?IDOKUE~Yud+K--O`O_0Gr~*Q`O+yMK z_L4&ZM>c9#SQsk=;WCM78oud&BOHySm4}hUF?bX!a|g6zsTi77D4Dorc{zZN?GtwY zGzs2hQ%rS#8oD7((_%RQohtM8nFnF3BQc!+_Wt2){K2zP!jn~hOB5zWAr-HpV6?ii zHvdgjw-7gSLud8ahn4+n`X6kos=u%u&1ixR)n|h__pFlBRC4F=T-5Q-XQv{cKK#dV za54%l+A?H{MWOFm#n_thv&rvXGXA2^MI|3U-njkenItf$Ci`JyckQPSCW653e9xDs zZ(l2%4RANb>XbChQp!FboDX}WzKBojPly!{Y$aE*lD4{i-g$p#?Bs=%hYA|=Z`R%1 z8>3sgH*SBMEjn?$KZRA&HJ;MCvj-DX&bgYsSf%erXWF`7Xa36vSu*(+s~HY`yg`$k zR2hX{H}-HY2GZ&K?QQa0{j6{6SWDRa{+It^XVo5=jL2&bj#;BJjE*<5y02eoEKwDJ z4a(h+kk0+5!O~$YFS#h``tgTn-ws8E6&Ze4d^RK}$9vq2@$La*$UeEwT_Z}ETH1cj}IyvZW=l(`&XPzI}p%fGt+R_xs zTgYFmm@TnO&xcNwT&(!+MEL5Fofkjf+6jzz?%t0oL8L_83DqxvxZalWk;B2O)07Uy zW+b6|gl@$F!xlMssY+#7{E^aeRW;GAMiM1!AbjJhQsp#i=_~hPj{DI80RlaZH1Tn0 z+P7F0nL9?6`L?p!+jBbM#1W{qQX4=@GXds5@EHs`UleNA$X6R(R+w`M@Iy$7fiv@i z6%Fr3=vjk9#3`tv8Jw0Y1H>!0L<9w!;1ZcX)>RM|g*;<}cdL(UA=0I3N{o?6+Kv2Z zZJSC{D3a&T0XRkVX-Bv5My3TUH|`5`OaWd~`AvjkGax<~`OJ{e5}V>fAvN!INU%l8 z<`3`T_Yq8z%p})k>2;FkZHhm$$)FqZ!zstnF<5tA0z(~YMv`sX2tg!y_lc%RIh zfA2{Ht@{+-ibp@4s7mh{QS3q-%3|RqBjcd=I(yO`R_^PA;Os-!sW44Fj56XP=$?Fy zX|}W&oCRLZxk)%l`-z53$9VTrlIv%7$;`VvaL5p(sgjVgSb0$2A&9nO0FkPdB4C&) zPuC}#sHZ6v<_L#(g$@nCP;*ypv$yqJeW)nz5a}&FhGY6_1l&ItS@LAvGx?oN(qf3< z?Mw}-K2RZI1Wu8Xd!~YjK#GcpfQW#yKA#@1{^+tl%rBrrnh;yV$m{iU7^;ru@;?jWsyLl0o}=T<%(X zpG+~?hDyTWIzSWC^%0I$ncrW5=>0M69LLs)Bj>&h4$tg7ofN@j928;fr*S6gv+n%K zT{NwUJeOAJW8d(o1Kl`GY~(vcW2Y2sT1CjufZ?2N7K3(3`~0{qNjIYiL@X>*ARCn& zb1@RPB*<@*TIteR;fW4J0tggs0&%%QJ&j!Fs__eG{&qh5C^BMM?WM*|8}qh9$W``j zs7)IiWfFP#+rETmuYqQ1Ojl_#VhQ4IbU@k@9e@Bl?k`Va*k-QB;nk&JV0C!sPWAu@ zQNF?|wNIMeb?>R}Lw(>9&vA53J9?YM&d3pxP_(UYxl*gA(RvH5AI(xGV{Ke1?K&7c zE83D1d-NX|o0Ftj%8h5hLA|`J1d%@CdK=OHjKh+BN5c;D2+tiNdaRIa=wS^*){Zvo zkU0&ozO;^vb|tH$dHYao0|lGa_1kD0qc;+kJ+xeUoVv_=K$YhM+9IP8p0G3I<*$uX zLnlt{|8E#&_IPmH`IvToOLPQOFl?Pp$J*bd)Au&U)7>vko%Oow2j5%k71nmj<8n*T zzdZl`pQ7T&Yhk9H4K3d5VAEyV&hMxkjirygOM=zj2=T<93JtnYwxWFX&$1ML)4=HV z52r?!J7;Z2hzC#)B%JQq3{ok{cmiqX;=@EB4D;Gzujoe!Nr?4_L<6jecAkFhoKwqTJM$}v> z5q~6SUhAfJ-dUttwCKCGw@yT9TB}i+vN@J`n}?@yuQrLR%BW4t<`M_P!}PB2qr$EYkNRr;R7@Tjc9mRzNSUB&nKelT8sOOb_v=m^32`2 z;7^Nyv-BLs*$nCfE)G2sp;E+|35GvNh_#t3P16m#notBKFqMKv(BkPg>OM^~DsKP} zyiTa7*Q}Z02%Q5bJ4gM2=`J;cGt_OX3}qR|)szbL%lfhDz+iGuy!;cWK)NmPgx=I- z3wBwHoGITh+X`S5S6N(exL6U%0pD{rmQZeEC0>wV*r9Jd#{!n# zPzZ0YXQkw<4P(w7vKzEAE?LpYYbPqO0{|K=XbK>^w} zNVUHvZ>?}lI^q@+>{WFY2I<@L(ycq>yjK*K04~Avc4^qE?{`*~W9+9hGX|`Cqthdsqs0gBO9saCk6zVR{OprO*Uj`}-?_oN; z4qs(j*ZCeb0;1hCCkS;N;pj=Nu@o8sneCwqv4Hc_L z(q3mC**fxhn2(32=!m0IpFodlBNvLTItK@m85|tU3WJlO>F8uD@6(BI-<#Z9|5e+5 zzlbnJ!%c;cqo-H+x52%cH@YS;dJrqTu=dz9+o;1?UcvDeHdQi-6p;PYp zDTiAtqocY^KLMA8?wTiF!C-65v&P6!HwwT+mZ;dpG4LjHyGY{8uEe-XPXGs;mr`C# zs8_~)a=GDWzrTIt5?J)xZ()pUZISyM=2AzmgkEQ_s1DKnemp~}|KVHlxkxmbTBoY{ zeG;Y@NZz1v!vo&khf1jmShv6Q5Iv*0Ft&Q>*#+OKFvpm?L%wtnfj-$z%6{Fb(>clj zT}Xd6k$ugr^hDv)R>#?APd(^INjJK_6g||>o7UHV0;hRPbgygr#_CP+d(3+UEbhhF zK!?EO>sMnP#ZN{Gf=D1Qo$%x4iDx@mO@|uF9)UWhv)%lM7at5akH{%mT^*hH$sKX- zUGyvq8<)aS)-k(Lb+0BsbfThXemPOoqv|SYcV`+J(LjMKnkG zWOBy!R&38xZTCZSwnOKxCZT6VBKQn`CdM5!;8-7#kZa}0?MdY4<&(``0=Js~KS;8O zwvD3au+lb+RiZIZh1-uq&(Vxw0@tVcL;iBs?{{GW^=yAF2P+M+<>?F2y}3QF3OGp# zVzk|`TfV++^3f1u@jqKs~_^HkNv5|R` zD3+~XYF*rwJ#Jo3m#QkdAdD>OEnonN$ufGj!^LVwy;X(WB==n)=c}}8kq=3<+QObs zbpeL*nG#s)az7R3aD`kfOdZj)vxEJ z$K}6}$u7Hl!*4H9trPs%s$j___lqML%lUjO_RwjqA5ZNt- z-zH4eqZcM<>&LjO3$4X6UN;CM*tb4M5m#N<{gfI21eMvB)8fYKtY)N{3H(os9jXkH zl4~u9{;B?DGHM(2NMjKO`nW6S1Y|QIO;?a@7}pSHT4J`sw0*cQaoDEl#6tRO->YR?w~4(^Vr?m<)L9F+s6s&^%;dNE5Pt` zO%&VGl1wGsHc!ZHu`A?6V-ho$N4f3M9-gE*1d25eM=>Xm>h8l9t85IpjT*BFF$jeT zfa*+b&cv!fsmTcyzGE&^riEF5#a53~`&P(|{@vcn8ewZD>vCY0XF|P;nSFYcouS%S zTkkSWUM+8PZEzfz!5lqa@+zeCWINBbS~Nco=-|B^Vy5=Rt@urwtzqQ->1fvD$fDqr zQPty&+5Kl9KfAc={O*%MlTQa9hW!;5cYh@jZT6#%1})8uDcDM8iVxM(^k&>t^4G$O z`5Jn7AU~wBcoY=JBb9zdVYr8f&kCl4J36k8I2Ugx{THYWW2TgeQMolKhFv-Cj8~Xa zt^}RR{N`V(H2?Dv`jfa)ctXCVf&|0`!Wu zTJ=l)(&nhXFfb(wVCX|&>b060XxD1e{Wsy>iT!6kt@ikOIC<|=Ky*t;&N|-8pS0Oh zbB?RgNyA^!C>dvmlh6L+z>RdTXZ%J!kd03Sk2a^(Rt;SOTS4Ji(!cl3mU*TP9FN~S zn{dzb%W3`zP*9ONbfJYin3&t2^An93Uv=Y8LE~wcbi8lra0HYT|MS)0;F95@{U6?I ze4OH!;69Y|@9=?~xBB^<)U>N^=VcFTE(M`7{(b5j|B@3sJSim)FD4?xllZ{oK^|qe z{l>BAkc1a^sardt`VE=s<|}}O*}j!XLwX(5iKuDHZE+`Ri^M?^jkRFdv#&2e36sE% zco5QB&NV%Gk~g=@Mre^T^lIkyi1yIoiu~;gKj5?kxE>^8>pm=bWDi!HA?nRAvc{Bs zlDl=5+oSA{fRTNxFx=hn1ee03XIcTzUBE)YFG@M9N)~s_(7`SXlk-qDg9X!gY(&n2 z)c@AdHn_qZcAHDJdW$?trP-oLsD21)|BI3`CuQbrY%Sb~QrsZB!F^fZ3l5WwFTO!Vuh&!F$)vImcY&y`^S3#OC(BCrU?O`;I zTfyZi_Mmf!b9iYJ#2dE~l#){88f8?)na5x8BCoQoyt!0^Y)Hs#|v5t7zkO5BH7h;tF#!OHR52G|>IWTB;iK8|;dRmQBg5=1seK6@g;N#Sfb8|NdCLIe3&RLiQ8&%$FiYXW znFa%%=6n^3FckT5b?tglAfMMa+kv(yc>bo@L{p-2oG8+W&ufI<*zOlckAAl8CMtUT9$K*TEQkW&-FG+&PPA$kZXVU{AKe;f4`zPp4MT;YHg z`K#VE5HJMsK<2O#q9KnGm#~gBreP*;hAsP?E#n9A+V^`fv!#;N^GMU>g+*IF@Cq6J zfCGa^1ALrqZ?eq;8kSc#a{F^d5Lm)gk5;rn&ekS0UEDqN{{FB_2_=tK&z4<3U4J#} z_x97D9|LFZjcY!hmusq@(kCW;OMq{TG^~HJ*NI%-!i{I0s!QC5uQJlJ@ta=LJ0l^3 zE3Qou-zYaLpZS!6BK9)b@h_umEE)F|;VMm};4-8o1)>EoWWZmQBJZWGwq<>{%qNgy~~?jttD-_g8n{@4{T3yfGJ9Mt#(|=F9$W7%;o#V zi75vJlAMhxMVqXt6r&n3=c-I!yh?hZ*e|Xb&CaWCjISI>3YdXMxR;no+=-`=2>2R$ zW8#jeY;@|t&$rJ1(s?6x_~qb};O5&g?SkxMA6FOFya%ruIJ6t}_A|kWlKDH);(;5( zje48!p$y-f$xB^7husNsi}>t#*5sYfV_2}>>8_3A%b7D*96SRXv%X5ae0g9wX>u1H zraSehCDZ+1fhZ-y?<#2heByrkR|b15Au|$2X*~1m?qE_-*&D$%By}9txJp+2Fgx48h7 zp5_qV+|~40!LG=_5T%IVMkh1b+%i8W6oaHmEel?VfFtH1*~~WeEDTUrer9C~ZWRxzUN(De}rfjK%(cZw8&iU6b4*jO*JN%b#e) zr@FeYtYVb%ODDhO`uf^;WQSJq_LO#>Eb*Js*fumC)F%jM zEkLaEDwvz}ic*bxVG6aatnYBo&{6H;e(lsC(kv^O)E8p$qHbF~GVeOtM95(3HeZtv zi!a9p!dW_aEujA%WKG35HgVz9JP`x{lg0VNdaSgUXlw6;&E<~*(oGCul#AZ>M#~_! zMXYqwEsEH6&xug9%jL#)IB|U{lyH}#?ok13h@xuAv{1}iKB=*ZvNta!`}c*5GsdYl zeQzA1%?;j_!^1KR_asIW#J23wCwD+IiMrJbn5+ch6{s#-&^J}jgILGJ4y-NjV&QUP zpU@Bq68PpxXQ=&Et?6RZVOzf>f#!J$!-&nc@^e%>miD6}*i#4jEK!fGJcKkz*S}oL zG48?W9y_2Qnx0Gp#OTjrkzqUPjrmvn}5Me!+t) zXTKZ_8%*7Q=5}-R(2dChgHx}|K#J*PVxV#}W7l+8}Jm;K_eH02_d%&;FHJ&XQx--Gj?pZd?EQ=Y;9Z&J#rXGV2; zbz}J<0T4j^&&B-b{?0$|wx1Httt=N4e3WOW2l8V&j%t`I3}OBRv`fyT34PPV1BO9i zRU(;+z?-(lnz#86sQTZp7roq*GSF3f;(RhVbQ8E!)jZOXp z6GYKH$|CR83V&F9*sv7aG}jk};{MTg)G>v&5S8YZF3Bz5l|oxKfn!4 zs(rBri>DJKKA<>pJR)QM#jUjIvKzQgX+Q^lb*lrIIsZa#O-t1)5bYz9Z60C;c^lw+ z((NRX0-17SI~*U2OEk}n@$({VV;H*vYTMNK>${r3kjJu!-{p{qIwzMyhH;v@*H{-5 z*Hx$mDSX0ED-<|^$MfjHD5(A{RF5T^uuU|Mp?a49F|VoHdc!!0BhMAr62S?e7y~*uo6jbk zV@X6QYJ*Eb?=Ta>yYv`!?Xxi-JILc)G)LvfZQA9c5**jwI6trx=3uuQX_=|o%mgPMFw zk*aY6sXuictLkXdzY1Mdvx2aaUUN687?w|pmh}rOY3>_8h^J7_8sFQM<@7a`V^grJ zMy|`&BqS(h&ua#r7_{{*tJNn;ejmx`kDQfzvWzc!sWw2SEY=T(!0x>sg+-eG1U*`D?VKf0ghJZ0G@FR=&3G1oli!7O zXC)|qvagJZG_!pvI)QylBi#_Y>wfj9v)7}0GNqb`KERk$<3dq(l1>234QgoMfPldB za>`xb0cI(j><8NdjG8CrnskA~q!Ac|?ZvdsdYMrHb|(}|wvFt5#AFg)#3(`CFJ$vArJY->1aLNW_~tuTDBApXT_ov)0OMYx@j$L}5uw zZmJbUAkpNEjbLDBI6aa&Oy5X2XKmSAu5gzgLR4Sc`A65kZk;V^=eO@W%ilZE{P#}^ zFLhkMNPki4(-l30@V)9d9PhiAPfCtB(X_K}+K)00doU$z)x@$s>BbQozFhlV8D~7 zrjLm#1%xZRJB9fY{rtYh^TNiYDv~*HPqRwR1K%ILkJ zN`2mm-^wmsX0Q#B$d*W9HMSF+*h2#YeblVt?H@p`MmWWVQmndLENgOZYL^NF=xHY4 zdEu360N>HXRx*Cu*}&iT*SX?j-UCHxH`{u@QQr3ql_gA zX+kOKsQbW^pM8hoZ}Hyn-9uYT_dcGSO<(*vtm?{@mgAT2#qhf0A_0_bB|&iS@V{z) zJ+R~9eY)$^x04S}#Dbcln4=noN|js{)^g@=ukT$<;LOaZnb**;>@JjNEN>6RgQ_j_ z2pkN}DG4RMKw5;rzKe!V7~|0Cw`((1AchAC1V z2l0>0e!y365!*IBK*B>5wR#!7or=*I;A%h$!xvQGZdD*7qVl%t-3?En+BdX~$FvOx zqJ?(@CkGvFkD5TJ_^{8iAPRet)wF*M9TA|~kHRTJ^Lst)++>U?7GNu&h9|WY*mz{0 zoL`JhNv_Q?-6iuYhJ^Y&Ruqs7BhUkVveM$XaPcMM49T6cY}0DS0$C#+fqlXqL}Lsy zZ8!HJh!EekK9Fb^xqlG{LBIW!Qko+NAP5lpG}Gw)DYlkQoO zy5XC%SE)>#{x+Hea|}I!<~%1w88=0xg>WN!VO~e#1n15i3?}DoZnPIKeSp-Eq{rkq~W*$1hdbERYeVM&x}e7+UR;i*G($&%&8yYAA%bW5>E6rk$(ecgSw!rE^83dt^^ zQQ?dut>hs}pHLZ~9fxr6v9>L^Z&&BOlNJyHvI?7kU3;!UMBTj2Ds}*`GH2Xc4B0?_ z$NOJc>n-l~*YhjDOcRiG-Fya>ZvxpLQDPsz#eW_^%(J58qstUMgF-t2z+AO4Jg5Zg zu6&zHl#L7&VCGB|p8vc)$~?2fMRX%Kzi-jO-%vQ;9wc{yvPQt)WlfuoWc*m#@TfEKvdWyhe>Kkk|D3s5Gr)gJ?>HYb3{J~|a|Qh2 z?9^xO4vt`zG!k7~^^da@SN@TY+ZuMC`Z?&wt}6*8`({2IHGQ5f7(HafOQ%2l(2MGM zYQ!A%6E1=*JAQiNos1j-!+qAoT}uCK;Qo1#EmrD}&?DLk(<&Ql%4K;Gw6tFm%dSZU zVSx8=Q7Bn6xAVh%_GG=g?#5XkP@&5rI7P&RWe1OS2d7}b>QvdA zXkkK0(x-u4H+N;9`|bv)$j; z9Q{VP`vCVs#xd8f_`llXgKky6SCUxjNQ$o=wu0$y{UYT(K+@*^_X^Iy_*l95&zAa{ z(pR5cJaMcoCM<66pBJBXtRz0^Sb@RJVL_haqyQzOoxgYIMW48V^b@yBxJywk@yP%V zHv*G)3j3&@3DSzN2iXU9pALmLk1_i5Gdh*Aos6qVbys&>W2DRR#;^ab&jX<*B^TLTkA-Y ztv_&MweUG4F&?jdVcXOp&0Jn7GU=#hHCyb<&JcOkY2`#4KuI0va&0*SB*jzyZ< zA2*)g!d`aatWj-hwEJ3AUPEZE5NQI^+DZXNGAG<>;cN}Rq^*ZR1?9s{2z46MT)|)? z7VpET;37WaEvQ4JEqB4kSKa;=2%!dny$Tbw4!Pc?P#)Njo)MId&5C3NPELluY_hfOMyG<-uMT?ICgJT z&g{la7O0m=Fj~3M5uCSsk*d1sWUSi~*T<;d;6@#2#|^4A)Tf-Bk%6xuRq_-XlsH+1 z%wIJz5}WpC0^4YlpIs;=L$!?JCgk>;{5t>*KQ_Ri7B2l$$g)u|2rYTbjQkvHj2+Ke z*jOJi^$fSLZ)~?w;rRoy1z>Q>U0@$}haptjG;4h~s_#wa>auBzGlO5OdK1~18(WW# znZcYl& zXTTL9f)KmdbH$PNMGr_#ZbvzJor|9DKWrThQCrQ~lMl?GE@?B2#`c!HP;&5g03}u2 zh)S2WH+4%bMN2z$rHF{b@-fE7{8#Q7`Pr$`=0sAxHAA`;<1E!@1&h~XMR654&gMY} z+Qu=S({=lcTTjgZyklCd52yMb)||kOZ`Zdu+`I28NgJPEjwLw;*Ro?7Vup)icYSeh-^I%1B*m$&+PY&dGtocaN`j%XTY3D(&jfz6l!WmA z=ytN)!YQJL-P1dNFBG&~rlh-qe~kyx-{8^r_Rk^OGyE5Q^G@6zUP`F!iz0_4TqQp& zoMxQ={6bjbKdZWK)yBGp3G~In2TCSD@mv#l`s|5piwvzl_2uKUq2K@cMJK>VCWp0w z`ee{zuc)6+_da*!gzsGV`N_P!1JUV;@esA!`L-R(?=J41zQ~w8T~t!yd24974;ooJ z4(_)}0aQ0+T7FDMOa?AS2kPf{Q9NS7W-b3HWt7qW*wBA#@96_6A0Gdv?w_?@$NnZ* zO7QkUDZ+4H?~lbJ(VKW$o2q@RZ{~34t%vdK3lD~c5B5E!23>6XKKM{{x#FF)i) zz;r-8z|NoR0VCpe>p^_zgO>jIxJ2ddV|znSKKbY4NKbqf5V5D*`E1R?raUXbl04+f)6V$Mb3;kT~1Vi;XmSs7}0tGR)va{iB0P&=AjGN0=hG24E7hF56 z>D9b>ek4Favzgc#a+?jDtlnt*bks6{>r5$JsGEyT$d!4R{T0@>fU(#hk5szvT{@ovG=s0HY~Ky{t?j znnECUXR=G7wcc9ai52(eCbcN(OdKHQu?)atQOQhU0`L3k8S53SxF97B#4QiA#$4Fh zNNPKiAb%n3X4mR2droZLZM|B&`50QeZ3daYxe z6F76NGjKyFIr9ZX1t`-xN=sdC4>AUr?u27DgPVr8`TzuAIu4A1Kqv=FUknNYen1=qJw8e>LH z6(eIi9Ysc!Y2i$A-&8@}rj-dhENZ^18y$V>H&sIRO*2YAhU$KIaUM%%zlbUC4YwIe z6%n7KSD#NRcP~13SlZ7W?weiPKmOTJ zQnuAwvc?c$&}l~^=g@u24rud1DQvT4{xZ_^66lk$0jl`o3dHuKifXrWfX`ujlM^pv z!=0`S8~&)Epz|0sA*8Fj@ZKJxHO?6Z>$1%|866!NEkal4> zd+Rn+|5k9=fdtr(j?myq)r~GMQ4C%@OeJh!JWmR^zQ6WjE|8=%EqSq6b~jMfA-@QJ244t{N_gw-W6)Jm0?!RmluZ6b^h zPbYFFT5Lj%Z60mgmX7V=1&dQx3rTaEe^F>6NfiX1wzekB5b>O88)38!uGBawSF?aU z{fnh&7I+=w`dK(NB4borLAV#nHm)TTZClK!h3{NC0l7UOJ^%H=I5i|$Ra%^eS9JMi zsS2p;%s`r1ebRVc#?QC3Q5jncP`1^BlluhU$l6RynafC-5$R&k4NTP;_rrP^dwbAq zTU#p%kzj@^@V)(1yTm57;t#d};10b%h$HkQ>b{&Pv3Q0t1Kj={Ey`O8eOlUI2~Lm{wWdk~Wlf z^Q(Cu`fsyYktJ`&^JM?N5fjB=gVIY~E^%SAxvi~adpRj1&PVj- zYiCN6>V8O~k3?@AUpgL3xxMc+5HxA}Lw~>Na_K=-5M1@O zOG?Q|dp)S9=-K?HozVE|-Bv&Vd{w-E~==%m`$9YD>)8M1#HmRwzo?D>W#Zxo&&GN&w z)U{3=mIG#V>q}tB(paBEB5|rw02Lsbz!L3s5NoYPoF+(?ACAhaawY=IEf*l2V+cw) zQHv!Sod~NQgn|^`4Ar232tx7wq4+ARl@(4Uh`vIv-SJM zR7n`NN!d5hFD~eGhSvGxcGHO+fK4c5j@9{QuT>C@Ft+I#1m(vMnP{4pPZuyEkL9BnzeZq z)}1|!>s!(mThRo;E=ybsNPR`-e8@zJh# zBzwGgK*8`1Pm&&GkARtJef@5yfWXXBIzsYJZnMzxZ=^Z+3wajl%4gi`q;Lqbm6WGC zuL~j9L>^GR2q8Ib|JKjUdJWK0Z#GvoYHvGeXsodm*}2ygM^< z_1uX2r252fOvgfwooRaN^t&@owte2^Bh6vrQq$p5VsDmU7qsp04ymbxkYPfJE9Edy2aevDuSBdS}fwiT?Iv z({1&k8F*;ZuM|vmN?yJC12V@KEEe;`8tc$J7hTg=&eh4T2f(p`aIEx6+WC>LJw!{1 zJa}Z5g$q15``=Yhij|39A9D*C4sW5Rt3SY2p3Yq$QvcdU)I&{-+8+En=c+;`u_6q2R|fMf%W~l zy$|^p|J_69*R%#d^FMEwev1VlHqv4Fkg2saf+265GW5#?9%qw5t$z*LGgCH&AVt zj-PL%3tIuI5--4BNK_<910N+pSMw(@$yp)~Sogax?4Iz>?JKny0ik{@HY-4obOq+D zW48bo*?~_!@!CBT67FDo4s7tdBym7RK8eJ1b+%}OUFzT;X#h2e`G^kL!ceSg1Z zS}i|wdvx;Vzt+lLZ9X|$omdnhJ0tTuyPwWaiTeFv>Mp>E z`#8XV+qV|PdKttT36kVbme}9%$qJBLo$xxtq5ufG3{ar$ZvB%imI4N^1>cR>Y9u7a zFkiZGMk$EZZ``xF1sK=T!*%aNkQrRV!VEy@nRy&54j_EV56o7nP1j8@cjpL!cZ-Bv z{RAZ5#}eC_>X|{WD(&m zAC2WTNqVeJz|C3560J^P<7-qT(sE77C~!&1rMtMNMP6p=i%`VD(04k9iYLi3_oA)$ zq$;y*q5mV==2;qWjmk%wWliG|Ol+#+In13iZ$h}L>&$bOFL^`y*+?g-Zh-=|`|wim za}8E&frn+O0fy&N_%I{K5Q8Z^+zCVG4_>2`-z!&Vs~WTJt%?Bv~3lk zY{gbVUP&m~KlWE(IbIKxdsQo@_VA9dgd< zy&zTBqL)9%&M~D6?FIF`zL63#M{aPktcKc1B29!i?cc=IjW0XBoD68&^St~G{-irq z;Q!3Ffl1ZUU_eNhVeK`cISsYur^Huy7h<-k0ZZfw-JHweaeIs^n)~KoHU?$&*)Eaf zYSK$E)S&H!{kw>9*0+f!mW=5U41qI3i^Itb3eQ72Mkmks64|wOb9}B~;3PWDXDwsl zWxl3sNyHf*c%ERn_oJE%u47>aIckGhA`GyHNL`7>WB@lnOR$yO)QNh zs+vVq({0yhD{`bcP-^@TYVdP593C^T`mi^LIY}_GOymLZ`0nQ4NdB8Q>2otPgaug& zOJ&F~`X~Dl$2SI)xwo8p)QVxy2FUR3@gQ4m{rbmGm3wG zSp2Q>vi(uu2r4QxxTe{Rg8Ld?Z#dcif)VcKiupn&iyf0=a;v7k9r+w7TQE&aQfyOj zCt#J#&JS_Iy&BWt?Q_PXQc%RAz*vLaklw^uh`(&+2m2HfxB8X4{yitcOpdPQ5KiR< z`E;i2{SPkR`zhhq;yXXpgt@3*zN$(dN=pbS1u{U&WF>bc?Pu19jl#U{Ja2MWmFMiM zCv}cSa;f7LrQY}BWFWx_xJNz~Hs1ICEtdZ9e0E6iw@iI{n0q_fu|ioNeYF$-2u!I* zMID19(UnP8hP@lT|6BCG5AXdr65AP8KXozj&c24J_ERMtJHSPDe>wL^X~9=lTJ#Al z_i&d&aGDlIv-Vbylo^Jkt57E=a^ncAAS1qE*Am~*tjcwArTK(c^6dYP?>yabD)@L1D6$^~ z|MSGM%F&n(F^1aOl(LdNmTLM=zgY6l=lilmKUFY1=+NIxD)Vwuw&$i2O^w24N*r7; z+BLT6!n@w}srCCTkokI+%Sfs@WZhHU)^<<%IBhZIH}>h|z(CTO#E(z5henmNvx*mL zOzl(OC$>bpo{Z!^y)-p<>Hn@ZKGIzS`%QwLS7&kRc3Z1s>D!t11t)@LQy~LsSt;<$ zKD5>sgP0@Kh$Y({idAng7znXIfV0LSRG-5=P?D_iTB0_Ph?P~jP&cN)p6b)6RV~qI zbZ~Ye%AASzIKr|312Ss$=(PrFzS3l?cH#6R71g)*0bHXr-VNc^&;#w4O7Ia*+^ch;% zt1K)PzkU*})}r#}+5mQp4yy@>pas|Z(#P)S>O~x_sN!&B{%VL?X@2S@>NS_*0Qf?T zRSZJ6|D?6A1EYmJHTScILp(0kG64a#Erhv&^KBwOU*>AIfoFxw#yI2Bt@Z}S(%4DN zte=;e$q6v7I~a9J5!K4?C9yV_6FGx);#M(=I7u_ll%(0}!O%%Fd`lx7cNd$T$(-#^ z+c~&e6$7tw@-2*O}KnFiBQHYpL#l{WUc}fzW+3u75S4Yf-BQ zdnLoCuSu|Sp$lftR*aqz_h4l+Ck1gS%LeAgNLp}B1pa?=r^$!hw7gqRmFQ|G&W0%e zmB`(^jh3*srVK_RhcKsEe4cI{qCLHg8ARF(=BsBT6N=TA=dB0j>Z_&+fG`4|)P&`)#7nG9oe~jqE;SP5h zAXEES#oTAcbXksnMl4s}$gJzC8JArY4=;Ama=;8g1Ov2il;68nXLXnhm%N^jyjGEpB*Jf^CtbWf5ob zZ_PEP5~+}Bt#BsiXhQ^?<+U-UG0Jq5f}N{E*(n95RgLkoxBnvHRNs+v>9#3|r%!oi z%-&=44sOWKZd|yXuK?A}OqS zN7!K=H;5FHh^sJE1h!b;ebsl$$J#1RQl368XH&z1PsVzEp1hG)HN3>|ZYwREzX@Ot zq*om-p(hT7#;^T4lJuQ#ZI$rFi? z(tI9h9Z@B+sq{mhqP)F@X)O0EbpHFNi^39$W~bRLS)ius ze3&zIaBlOh5|poc_%F_P&!uv+oO-t#D{yTUqOcKgPloDyIF@2yEIaDa&NKjOLOs9S z%mh34lkZcRI>}uR?BS>xI9}&pUHZ@93 z(8GF{+oa|hsa>%drOD~5J7IVOh5N@FpNQluV?I9^Nyr@a%)*w8BPWkV5SI`VqO7mv;TIu$7R{%C=G=?e26@-AQv-&i8XU5~CE?0kl!(~3s zLsTL#wKu>ZM{$cS)5vWZB1Kg;YFz?e`UvAnMKdl0tI85f{iCzQ2T>>lGi%(R=eUgH z1!JgYYg*2qQoZM@JilKOl!QU+u3M8batdGw797o`{N@8W!iDwkt$NK{Icvwx{ClxQZ7O*rfr%LsESZVrjqf^l&(pUz$Z&GO}Wg8mB#?B`jfeEfd|{4WHVc z2fpuQ;#H05Y6Nc78#*MH{-Q6vxSPgC1xf36EL`nYF50W`LyVy)-=vn!$7jm2E;9M!U^a9HXph4>qe>a0wmwPhYd zed_l~hxy@WOl~)U<6xiAxLVZY^E7)q#R=E%+oOAXhU4vM{yhvDnFQT_>cr4%H2y+# zXGhj@JI=m-fwqr#dU`EkDvbGN@YDyrik$2?(;1?sm8&k!|@S;2QndS5!i7l8meZ$;w$M|j@{ng8VGxfVva(W6yq zVn?GuS)X=QzVTC1V)vB?)9mmOkalRf{eDvTqyJ2N8F$;oqJHIx_Gu|q5K8+ zSvKX({-ftl{`zJzA-DMlhu>c(@hf(6ecyMIO&#O)?LM{dUuUKI2EH)K=l`SV?BkL? z_y7Ofw&ZMWxl-Y~_1#=6C6|f!oULu1Wy;E>2mDmknR^t1|*8XB#ypEi?a8c>TY8BMzsgHBOIL zj{8PT&rS6ojA@Uk6u9SJZa*9m^(pz8a_ru$_tyTif2-~O;h=BX?dC6Urv4D0dcN@B ziM_QyxLJ&OnuWoF(wp5ljLEQI!QJA2M&Xn~}0yu4v9 zkd~p^)g{H%#U;dG>K8YI##54${Nyzgy{>MhLqS7bo&Ej&+qTTCUX}EFGT}nWbexGM#y+3ht z*KP25-D_kOK>uk1DNq)*K;b2Zb}&$U9FoP`xQ(Jtiz%Jeg#SGE?6*G|A6C|kt zDxk%o!lL1mFM8PPuJnJJb#&sA7+QZ)3E7_z>~{V8aN6=Qm}-6)Kz$G3c<~9b$2NSl zmQ$!fph@gyoET5w@Gp<3g*XccsB^Dh!M+If?yJ= z7qq0UmY~*jirWov0YAR}JZ~io&*Vbu$+F7^;M%blGlL~UwpvBCKK4ScpGYy3lJ?SV zqi%$q-%CBEfg$=?l{~nK3#CVpW{pt)Qghf-bnwzv@x)@$=AAue0D{z7$J(nYP2)I6 z8$YHnSsLVy_h3qi@@V>c9rk*-`MN9f`-+C~w@N}%qE-LtjrQza3xaj z_O&3e_+Vm|WWLxl7-*~86{w+$j#6|mdB(V6+MIA^XtM~G->R?w0^;5Ss>P`0#7_k& zMr8@pJ%rej!Vb~An#HZQW28l!Po%CUp=Jo&rEU!~9ZOm{MGN8VtIS37^|zkxw$Zm+ zSlbto#yXy<4OJgtc^DSF0=75@xHMx7&0%)Ff*}lZ0=r`$P&qe{91zQqYm<>Txr;o@Em8-ifOKq2O0ikDGD~l-$~euz zCoUc-cTxXHorr`Y+q6-hVSLploFNHYlt5Joq3w)+@KLMmW{{ zYbBV!K8$XwZyvkp`TjumOWD^h<`3pj6BOo>fl&2kSLw*kf&!8`v{9<>%!(P9T~eDv9|2PLWXvEh|f znMrKFZ@!9YOs)e!8d00^)Q2f=o_aaEFp}ycFJAx)fT6nflK&Y_n;Sg}Zt@s^_oVhZ zYK+Y?SW{Nk9-DsdPaprP^aJLJ3vkP`;{ujcgBnQkB;}WTNEQ}qkjgs4h4+jgEcQkD z^8K8+`O3_oGyl$*2Dk-=h<|LaN}16ZhT(SuQ(T!>>PgA1(tjRo7Jahy|F&XJ{rJE4 zTu6$0ek0_U{+)sK(MMRK{@}~g&mNvS^%r83$$_f`Mj>eqe|(;Xw-hg1(4jB&-ey0b}0cd`S?NAg|Z7Rp_?N`qUy63 z7tSBG4&ecJp|rhySUG5txOH8KDbGaiH{@(REm~U{$-uHJz;9;h^FYxa1*4^#8{1x` z$k89`#QPF^Om069XZg!$VkrP5DpeRJxfQ&!0a#;827YVs$tZ71?i`oP?7s=Tt8JuE zOSYHrtJ+1g z<3@eShfEvQSXCK9-_7G>{-E1IP>bk_CVZh8fRc=9#uQ$5AGX?+($t{adCZ|tMwAE) z9iDrG7RXOevH}K^oJgo9jIqJ$f9`4;iRtUS86L%()$~OS-IOeRijnzKKWGawWb^tc znrr;_3N+3BQweY8d1cE2iZ=f+J03>ArJI1W1Dz;h|Bf5|yhFj9jiR><+S3V$_!#Fd z+dIMVuC-X+-(06LuLZ-rqbkPGQ`Dg-%L!*Nw!M7aMBmNv5t24aC12 zDyII&F6Z{m)xGM4I5^R2SHLv>L7}@oo*Ef9-H;c*fDZ2W?~L^#P#34+_J^)agnUqi zU@Qof#fY7Y0%3iTLR>gq5@P?@1arYz&SCK~0(uDn=M@&^`ryEUXV;`r$ z8KXr@c{-^p>v=;QJYTPggAMFcS!!v452*C5sdl>MB1`|q>(BM}03n<2k9~%JyW}K) zCb}(=`(pK%s<8gRyL*%4)44mF-%bB3UNq`n;rrlGL_kN&-%tB=`M>?@ z-4l18VOikt7J&!8sjqIo`|2;_0FKaEwrf9 zN3wVuAqn+k5;E|a85!OKV7=2(75Zk+k+#Gu%{w=IgG%50=WX3w`PI~r2QFV7$i10V z6RlcWzf7E4oI3e5_=Y;Q{^)}5t-t*Ew~g;=0$pN}N&WS)o3FA0{p+iaRDdOJZDrhN zV#cl5uA@pr!#+2O#BohwPy|VZ`%7s({bx|97e)KKF61TDf-i$v|Q&!&sBB!5G_2a-x{0G>vZLfg){?Zc}Ff;*a&d8`;1kwZ0 zup^E@9YNdGxQ0RKsg%UPELM*jAL~I%4I_;23Q=v!p<$f(0-LBD10js<>_Xkv75?VU zmg6ar;9qfEN^g&t{w4;eA!3&XRD=Rub8y~PQCr>_DQ(?RSVkDJ2l*cYA9-|rZoxBy61w|S~?~K*sP-T zh9JsB6mS#wR1Bu(N0F9SJm}#0aPUcb@Yb$Tf}vR}^RX8>@CrA0j@3?Gy;@zvwQ|OK z5eq`crS{RG!Mjdq&4e$Ut8M(+J@hc1d<}}U_NAYYNg{)P&rCSv)#6XglU{{1B$ut! zL!*$Upg`UodO@ZPab<^`pjj!s;*ul1No!w$fe;_8pX^e}S6;#zAkDWF12`SYtG{=u z#E*_;dwhNd=dIVh&{R4(m5};-%@%ib?1*? z&L&Z;@`2IzeHk;2gL$ee#=B}}!uEPTKkhUe(eXh^l$)Cb)jNE0ltnUy=seRalKHni_Sn&D*(w3L`DuoaS zdLRiLBvbcZCfeV`T+{;h5_FrUplt3LoVZ5ceJgSXpRMQ=7hGTz&`vC(Bz3)T%X+~! zIAh$}#0BHy(nGQxBim_b?I&2HCG1dYjY$&a?B?#MGUMk${B3zy`z!{%>?5(cUfmq~oON{^eeZ|1GjVmdb2Aia+*okn3-p^TOHF&bTHVlYxhpvj#=f!in7e&n z&3yXi)V`*Vlb-(2;{KaxWXJdIHez|nDEIHGZ1Bgi+8y5mDvzx1Q~Z){)J?xJl6vFa z`(G#Q{mOUk-s~f<>V9l4Kq}a#bJUKvGKv3v*xa5SB_|n%?JVdrmF|azHXQpD6@=Q6lG=zQ@}`l?i8~LW9=1vcA#3-BnZxxp2(c% z&V&zYTGKIchc`WdxEH8+eK1@jq1?rcRVL#CK0;X6!xl-l*8U5UW;>1#37i9= zWu45{UyPqt6fZ|HUK?R7k=Rc0w=8F~q{=?HO=mPB)oi0sK=g&;F| z$d_8#p&w6A9}C^R1KUh1tgWW(wqtl|(9dKHn0}!er#0jeKAfV+!S-_OZ8)A|0bEs) z(9)A0SOWn+Y~R_op-Ip$Fs4wWJg)HvuSzUce-u&|==lkP=cq0ZT?><~v1l95)ytE8 zuTb>uj1ZM4I@Q_E+r2&C=;sZ@i5sq3WJ5~5$N0oC`ugUMs@rH(H{sM{^Ssc51T36> zubipR1Wo+8(Ub;u9RUP3pBsPDU0m$$CytI6UP_^khEr^vieZY&{-Eq*!|0;XkvN`m z1ZUSYHOzwMtoZLRE;bZS+FjtbX12z-EAN2knZq!{3f80bX`tkzE zCO(S88k&d>RU}Z!`Gs=ndK`>k-IU5Kb_wtFCG}W%8PXsO5;GG9fp`Cx4GwOI54HNpmPMn7?R|?-nI}%8<7O- zufpc%>L4Wn&&f0IenN2KD{Nf?=OWT>PwN@5m6sw+J_t%5c&!b>JG;;jXWjTF(m90+ zSsn-rH4Qs=V9w2{NeBTgFl8i3e>E5q+{Gt1IF%^rPB)LPFXh^k>894f{rAFv|HY(r6S430WsZ^%MLqXl|mQnz;+1 z=XZ1#)3TRL;ZZV+OX1#h6lA4xgAa;F6Nxh;|86{4wFj%&*nXF@2A)f^`&o(1rI-B4 zl`{d)y^@cdXb;)jiA^QMa7djLl7@l91@nS~wtZI_}sHM`g^|p z@XDpG;fuXsnmeHdqc1l6@Y(i1NG^P-@SmGsc-aj4!Rh0X`)c8FN2q0g`3|%!tV7!L zJWwPK#f_x^cx;i#XgJ&S-+5q{0LX~>mDGa*<9Aqqd`auA5%>F{*dS7SRvrFXc;1)2 zzxufp--_x8&pY#b=I4U!gVK@g(;tZ1_^afMfbQUfB4FC|scTW5{_bcD-?!<`d$Y0O zf4P1@%uj1*YuvbeYftrvpEZl;5x=9qctIAo@cElfEJtVAu&%7VGENfYMojOxfX0uh ziW=LRMDXdU;uDemw?~r6r!cki3qQn;iWZKQtEw_Hk`|_Y%T=jq?ep!)Zl1Rr-njAY z=N^9u|M~2x!(W##5I?v!pCJ+t)J*I*d>yk{L>ZbpY8d}q-eG88XAjKhH;iBYBn=s} znvr4~$#oS1E;2_uxHfa5#kq;3+p_6Sv*Q7M_ar<|JkJS^sOY!6!`gmW?tHN!669-$f-J;l$^)X2a6RV!h~#G({rJI~?P zi1}Kna}=Dd`@jSTs(fjaBO42DtaG~`!YG}kYse7w23xC1NK0IJ?{|=me;6QP#Bv-P z6F~PqQ;et8fcI;zm|7_YR!3Ud9w6uD`-ehWCsYBwLRM9~X#4Z>8~~ym8V9h!Z3=GbtANAIF zIX5e`pM8wg^^~J`W;y;?Oy0CEU5qsJ3g{agCe$aNuS3+)95K!gjh7ZGDP`$L+z8ue zJehd{H36c*tTIBM@g0~O@@>nE1cqBn&Uf*ZZr^W+iZeE=!co&bEhDpkcFv&s=ug%Q zsjYFB(2#LJvk)C=ROO!JHZuxVtfp``o+GoXq~05#2dejXP%;bGrd@=CN@Wv|w}axn z;|e7jgo0s9OR6&xU(~J}mVsV{0vWoJTrQ;)nBqNT8+$u!3N)whxIDBkfwg-vexL?C zRDol}8)NIT$eowC{lf_8$>t<%!7|QV?W~ZJU-EN5<3H~+5opWnt#luEbTk_t&wyZI zKJCgT%5K|)%}sT(L+xFk5W1oqE<+I~3eIB11p8y;7Ek7uxpGanb8=u`#ix_E0&nK0 zF#M%2G6IK)dIX%!>G2_~8E)?>9GjX;*;a975fcS3u%vu)F@`(}q|rK%L*P+?KptvB z*i}JJh^-yeY6Om1sl=_+6WE{k?6C>TaW=TtARAqM^*hs{W721e38Azi!`85kv3)NN z6yaslg5F+C2JdxjaxaXdIfqrR(l-S($d$el1J^BA%+4s!!3tMSLdzA<@|0{?Ft}2R zP{U;fik>KD_#Y}8&i1wLL6uoyu5-C;yz4nb~vdG>0KA}8B%D5YB1k$+WpRlLR2u_>s*DP4<4 z2cEL2_Yse{4e_6MsV@gtV;mgdI(i#MEnuqG4_6@a?$dX`rzKKs75f)odu6#w>l11fAkN9sQI+p zglnB)J|k3q{wUZMXa{1$Z+^sfZ%O3Fb^xZHetmuYpYA2f-q?=js$Q?}Ke|WrW11aB zo!J>{1&5P!lTu_Pupk0&xxilzj&-ab|xJz&P==AJa&EdnDW^Bv#EbO zkneFd9uX1w@Z3t(y@of~KGkih{X6B5G#hUw)s2k#dWGEXEc~D(yFb>e zA{yKrazfzkg2OEtop8^(u|LHnR{rCiJO5ANoBH{wpD&fY(>i7S>}ypZF+qs3&L7=h z=eRhTnKABKM;z+^RJ!;(D&FwZcv;k*utG^YMty2*e;o$ERDsa6D^}1WW-dxYC|M6l zaxiN;tpsk*Y$gGusCHTwiwq}mHX@F!+Pzjyt_5zZs=ATZhQRE*$|drNhTuYVI#+W9 zTE(GttrheUFI4DCa~vLMj2#&vrB-+*- z`9wF7veSq#1u%6wY#GofYs3W-lv9ecwl+#Y@cpa@+P_g<~-+^%q<(>$1 ze$mi7p|fkU{z^|VQ#Xxts^nxbm$HUa8d>yRzAOI$WU7eovwSEy;t^sYC^1&upe0$! zGF1f`j*qr0y2hi#!4_nRgwegQvBt-9x6Q8Ff-faCR>wdf^#zQRczaw~LANjM6<1D; zjBGt=Xu}@}Jb-k(;6}Q8qZ~P`z#!Xu1lx}AE>D@nV|Pc@Li%cCwtc^8m!iP^->p8bZp0kmzj__!wqZB<&I zN){9dbmn3;hixY4q781i$x^k!<~dSL?a_>&R+f-*d7)Wvr-B#3FkZZ)9T7~vJY&Px zY>Jc{YJW((mV5Twt#+zJY^gY8gU}U{YM<36$cX6|c7lzE`AdJU8eJ{M%Zk=dmOJ^CFowQ_e@P_* zt6!5*?wD+M)o^zGw9pU?P)Z>_Xf?kI^D;~NSABmy_(pVCf9OU3nMp>~o-LUGDy(u< zy!uabmi^ir1mgA!PI_rIXB%fXA;&Zrk5&;2i&511 zKx9`t0IZ*VGpF@L%ZG2`8-tU}yT5HZ%>dFg@Y=SQ|9emOC|GhHds~&*u#fM5I7Jxj zKbQP_kCFeG;PEIw8(bBLE@ACGRqeB(|2mv|KH`(BJ>sx_nRyZqU*zY#96nz`Frs9A zRH?DctwZXcn?qBME$?1?)boDV{6PLbpHIKpeBthrNe^&;&@xMoN3@GUi*Gdt0EgO) z58)%q>qyuqSm zP}Vv)TCxH~zAhq7P2_j^59TdZl>~3L0Fo#wQjIb0R$%5^(0K)?U5Ca>{ZV2clct9Er`2oPCf;4??V}i<{ z7NF0@+F4FLfw4`(^cU#Xx3dP#LJD@P2|M_Ed0XS~#q5mq+`=p!K=ZM{mjcBbI{;wn z78QzC%?_RmvEAd*7YR-ivt86CrWK=rf?;qB0V;jCzAV!aMeEkGv{X@ejRFAh9TLre zSp&2#Q%I+PXA47GPQe!9^GJ*N6B#n}#$Pdmre-%MfF!*)!t zflViCJ{Gz=CJ}HfuQOM-rQf2Kz^HwMqMsPLKQ+iMtJ=vD2Hjabfk-JX1kV(QZQs&4^a-p@g+7c5Cg)oNGi7~4 zit+`foV;~}AalfoRoE&m^59giaXV+Ulpx#P4#megbfgRyHbJQ~#nAf$jrYe@xK!0S zbb$&9t*@^TEpsSoC{;bhf4mX15NtNr4N)vV)v-;^V___2z5u$0r;Tq6XBAm)gjJ#G zi<{eLNsz3}(ekuyP_zNch$o{Tc!M!OD!gkOC&1x_qZx%ySoXA_+BSpv>49N{V@w9d z4DEP6Awxcg-YUZv=$Y`NgQ3)7hTb1vxNXX0mtmkX(*V-^Tb%O;^CJ2*P>V!un1tTU zZB>cerk|hUaYd|qc^E0UXECd6p(*io7VnxTa>AvosFFklru9zHh%^9hx)@cU$2FA3 zgXE$^gmca_hhDUCMIjgpK&ECdKUE%rTmNyXe~G1li3 zlekvZ6cb(%_5emKn6$ah^ImZ|A@G^iE9cdniw55x{8E0c{CCyi!rQ@R*?n>s6Hzx5 z?5}DJ@ve{Ct2&)=;}6IF5HaoB-3lUV@frlcG3@)l|LGc%#2yvKlg3;330Pd^*z-!y z3J#ZP8+qEe{_0s2&z&LMyeU}t1g4txF9a!w&sq|H{rcOj)I2xe2N(M% zHh+8D*4>GSmR>@m3#B6`@uB$!ey=b-rh`W_jMbK@8|I7NnT)dD{k#ddpzf$yBYelw zdRBjYxnlAq5EZ>mxJ#r7f@k&mYtD#S>gXc;OEP#+e1lX{QBeR+diMXdFK7k91n0u z_k#e)j`~wSPRYBDYsXu@po2T!BP?d0;-ZW&CIJI;;YgIW2;fHmFcZkv5H43^ili}A zJI-O;i;_RAK?>z0)ilC+9c^tuLtNahwA0cfGwo%g2HbN+4Z!DHLEb9WBHR~=YavL7 zzT($rw4(t%{Svzjuw>wv3~-ku!FAhF;)cLjj}VXurftL&=mf#blNGdGJ^&1>we|&q z$5F9kWNV?aMP_f8s{oveq$zrF90-W`>7{mXw~XzAD+gjS58y_rLmS7 z;`nLL#KZ&U4mi0DFT^3HS-4$-&L?CBkWx}j%YC)n!z?;e6+`Sjn(mtNE^yDG&ZJI0dNBP3ohXt`PgI9|Bm%@9AyU*)!)@D%t7j#gxS6sE%2`-l1XTEsnJNJsGQ zoU#EU2fDmaP(kO?`8}N(UHSO&rrGC~5iaH+3|?LB;v(PJ2e&>&$2n)Ur(v{pWp^%g zPPXQQHeMlFg$<=_MCeuo+_yeKX1K1OBIGHa?`a6^ByO7IFEyFoQ31Ald^1fP7OFU` zmbG|fFgF>yF}AiQ7nk+fK;QV5#e~2{ectGn#vP68{>eV&@S|U`1g`wmH zia749>Z=E?4bB}Tq*=TomD`Qz5F$AxNd`UxJmE>W;pMa>j&2dtbJZuri_pE6h9Ttu zyI_qpXlP|ame&WS{?#v_H~(0FIXA_ySevO%tLY{7P15J$EI1Vxoa%@ZKMookL%HAJ<&ZT!?_xhFCTz zr`ur8M|F6r9Rl=f!^5zn&0ljYX9Res!nVH-o}!p$1g@XGT=Pj+0()Q~!EiraGS~S4 zv&+w|DBR*Rp{F*$4WgQaZ-v>+T#JoMt3^}iyDktfz_`QRB5FRU!;U%0-5~I{y7Ii4|$}h?9{po|R?w)6rjNSX@ zSyv(Z{LMnS2fCN4T)CFc3d&W^T&rw}1mI$>k9qAL{8N|@>nP`wXhZ@Y`y^4waY!_R#spSb(pcjEUOEf=xrz-RHPFOXq? zwW?p;?D#NPRurYDC;0#y>KZks^hH{aX{Ncg=lW{adrio0{Z#_=?)}-Xzv%UxzMB1T z_x8c~F8Fl0@%OTmZY_S1QR(>c z%$&OJP<@kuztoiNUq1pi%HKvFS#KU8hj@v@REcTA&edZ;T`y?$5|7UI#aE0Pk6mv+ z!4@s}Cik9}jQsigpEv)p(E3DwV`(ZQWo^J=oRm8pa_&eQ54>#Yf8yCjd1u9(y?kg1 zNPY@bJHq3P{Iw7gle9ZBjRKh+&^z)BB}&U98v-4_vh42yU*@HxF7Sh5I-cVkGL~~A z?ZnVn_~7HjI0jF(_*e!F+k{kOjAPF9pFHp%r!7}9^nC&cKMK3`F`Vn)pF0jRz*|l5 zNAHxT1S@*^{9^j9jpre%&PRI848^qz8p`I?Y&4JuIqpS?Cjel&_y0^TTnA1mloVG5qQc%6v8o8QS?F%@sIU_xITKHg1aU} zTa)25{X!B9AZSOX83WE0-7)HthT8%OA727Dd%&8Ia918uG((;dgdJtY?ky(J*52lt z+QtCv*Smh<2#h>dtRY*66P>RX=TjALvz#)RbsWL9rSkH&+sqybx!-B1BYJ>2PzV3$ zhP#g6`Aoc_Gf{~8s&Pt;Q~_PgPmtDl2w9XZs}HNY(ASMj!0z_X7bD$ga;23+REqJf zQd@b^@_an6sDg>d=Y_mwc^^QikGL zG+k`&Tsl072n`8(k_0H*=M7+)(j(JnB0;^;HgVsTS+qUH+WvwBa1Er8MDvs5Ncf$lSR7*Zyfwhe@HqncV9vs=fr zaZ!Wex3@_&T4Q|*Z$=Wt0I z+e4)QcWNNmUofkagVIKii}65q*wH)>J>DED4$dGeDhg&yp)L3W1P*0)7E#|3bO1(g z(Dc&#%(vvLR)N!!Gb*)Eu0nGFBlLK)T<*qzc4xhS>Xg*oI-kZfb`A8mmhhemshI2F z)5$?q`y^biO)-ZgX*+@@8iIR__e&OYbg#CPv8GAR9?U>NK`LQ`Pj&G=wk7Dh?$;NQ z;Pztkth@SFc2mZie>rpN9~atwQr6u4zntc(u+W9+r9UIoY1H-B18qNk35M4DZN#L> z@Bo$7GN?_H){Oh@H;fmueV7PIKrgq4T9lHKl5Tq{)TzhYhl|dH@BQ|(>0`G=Ik6R^ z+4Wav-@KoI3vVD9hQa-{bHgXa@pj6!?{3V0E;;r(;^y04lKOw&r#Fnd8Z392B*O*v zTx>>$&=n-NM5BK?lb)9lbE>sR=w}6oGSh4re$aOA-?_h2<*`SaDo;*&DeFoLak5ul z@HpE~|M{Wy@aF@f^Q1(|X?7K3;e&T-{`Xncc&%ck7vHuDq`nYGR~{Pm|9!92*9cDmqc~WQ+OMiec=9vRhH7A1JmJSq zN}3#Zokae8sxZQL@mbTy~S;_%xijiV|e&5d09hl33H_FR{j|K$(llW@&7cr1! z9z2Ed)8G59ws}r6?g*dO|CE@?e)!XO*9s$(KOlD3N2}_8y;B&{J4=pj7i5nC(q?qI z=t$gn&ga*K*Auhc8lL(l2Y#12?j;bN&S;1aPD?V37}IX4Hde8gkHu}-(u0mXc|!{L zDC0D;E(*Ai(O3EZgE2QqxelRZVli)gbZ9F)b8M3UoE8bebXv_-KxwE(FNV~ZS66DI zDrjS+T>U6-b{Y)Rql3%gjKJInDLr=@+xNwxLiBe?Qrbj_kH~_9?z@%Z{oETSR=E?;FwS4A2UN?NOR1m@^~mrWdAfC6 z;Ltqmnu|wgzNs1syS(;&urXED9ub+z${1&r;p|IAiV`1VP=&KA)$+FU3Cn7&POY#% z2^-S6C75^kfZG#8)u(3WE|BLzdy#PfU+{FRqop?&gDk0t(@3!k`-u8~7L2pyL$t2n?N18OH?GVNBZ2^y}epUWptxG6%GUi(j1R-Y-2YuIcDsy)B1?QgM-5LBjeEs9KMtI9qr+(s5vuF>F~9okJ1xx`8J7L?JtQqcQ=Yl{dOJ~f*WFvJI3vMk5RUei#()o15`JK{ zxvZgG?kGl6*ZCbxZ3b>I3(sAoKiS93GHHT9i(U6y$)X-`CYEFz@9yS(0+eQ`z|>q4 zdfE0=1Tg&I4d2n>ii`*MCA*ZLs$HWkyMjJ`OR(JsHlWy8+ zX9u;_7e2460u`8*CZXMk#E~toygPE1SzT_Uf9_)5az3ljgk1I^hLIQ055lY2gJ1Cu zDz>7`aO2rHw6m0gA?u+q$E(qCW#IseEhFAMgWbBN-=KFD>?WbWyQA|lMhhzm16d0; zAWzyrwg;H2^%1~Q>YAKyn`Z8OLog13EwUpSXGu1>KoBF`6;_q0bypo~{y6h!Tcdl` z16a@CdjVHl{&G0#4_CgQUHse3y?e!{%gYBBQ{u&|@V@VT7ND4eV-dL4j1J{UQEqNJCEd0n6oM!^+YbdCiUBNuUO4qv&swjzbl|%nrYEKE z`|(t*u{0K(o{d1aDm?kl$6uaF`#}lr>i%ylj`co&@m^1xTV0$RsGbE$>f8XTQnCO@ zXgL6Fui6Y={MtJ@$y=<>0?qH~oB%MAyNB$(ivJ+Dru+6t z-o2zAVd6IVR#;p1ZQruDPew-7{yKXsLX{led@dwdp0j9i$ai~sKXo(A7sIVLxQ(lF zFwCO$?^NXHvE#c8FIA|P)Pl_@#+m(C_?B9&4p9I>x|HM3oPb*aq%SmZw*lvny5WcL zT&x`6Nst;3`!cdfj&ly0b52oR zHE!NsR_G3lwml8zp<;(I31M5`6SDh-XC#wBx79jLtDAQtk%WEwiWl;gq?nX40 zB<`x{yDEyd-|d#Ts7&m5j<7tBp>V}K?Gq0{=r9!H^p%A;5Q^Klg8|HKJu zLc!BwJoQPXcLQf`_KSqAM-CX;Y%5tq{x4joY|#v;G=iaJVP{+(jha|^4CmTG)rF(D z_T_IYBr_KNKI0g=NbnktY+lqomAcyg6GPvPL$9xkB=n6v`4tSGZnqq#z3)@wz;x&W zQG#a=y3?&sbD%8UjS|6e=n3pd9E`pz>Otkx?U^bGnZa>X2wDN;^%33A_Nz)_6VyLg02Vh2K_Z;Z2 zgLeJ`A8TI#3QX`Pb)liJLqge3vpt^fYZ@PtFaYUI3@T+W%j6DeDZ%P(fu)$i&#^0P zP_HDy+8p`1N3w#Wo6WS%l!>8Dx5eFiyrAkRuZu~!*#-VXJLyAD_W=g;_Lj+0c3}hV z_!+r+rJOrw466o5ge#3qFZ3HIn_Zf$1vMb9>Y^cU- zQV)GSJY?t9auavVzsLVcsKeOKX+%C8zkaK2?O0E zJ`p9iTo00M7Gu?arBk(TLyv2j&gVgJ!@I`&5ltiWwySQvY(>@_@8RGHLGC!xd9d`! z-ACDYg7a#WTi8kZQ~j{q-+Qq?n1=7S-Q0L0Z4;mi$=6~SXb2NsQLsD6XFFzTs)h6+ z=}G5K46}1pUC1Uvy*9sIl2FR#%!TG^JtqDhPd99|ggOYCKJ$~ujB1WM_J_cq8bfj} za({>|>B^1zN-TQw%GdI+<&M(zdw*ZTf5N_e@&bH>jC>PJtGS|arDJQ?Bvf7;0%|tk zW<#;Tw_|2z2EbtJpLPP-`-rk=(kwNJI){t0H=ouH0~Bv=I{3$jdG7tw=}C1tA?)tFZi3mmshIjRHruXyKC&(%b6-dMz1;BHp{=ahCiq_vjLZuZO#PJFF!; zqISSNE%}$?|NNYEu;sMq$lhXcdi&Sunv~=J2`rs|0OI8P#PO8nt8OIxV5)7Tt{21D z>@EMA_kq%F({?`3@a2bZ2nMoi+FHnJZz8&Fnn@}suQkVTQ09|#PXpq(!B?N=?+}iFBOB6zMBtO2B(xr5FrYaKL@lr)bSR2+Gp8DIl)rT*IDy*A z8eiIrJ36Z2*PO??u>ujZ^7feTby!eY7;G z&LPoQ+tC*~M5y4)R0-8qF}Sp{k5RNSBG|syB5ohHGE%xNJ;{c%W~Sk=ZAmw+Y1X0OGoCsM1qXTP=*fV1!U4#z(5wyYQ%*ijQVfVq;oR)O znK+SdM;3t$IbtA7+6Oe9QePR(_vJzXIX95rTJP9%)o`rWmEI)=I3xsl38O8Xt{yaj zoWU}|A=9Y14yC5ExFkX1n1gu?E~_%onroJO=<(V4jV!4!xSON?Ygqa?*7`AvzRYA& zXJdkM6Klm8%nOKKgIV1`-K3V%XnLz#bX)#-YZS{+&N{xBZ<4h%NasU|f+#Hwa3vM4 zl#OaH9}jPF$y$|XWW(U<9ay1N!8FxW7rxJeF?L~?fX>{lx#v-LH1-fmSjlLOf1z@& z8;dzym_K?5(S3rcuJ_mN@OX}IG5toL3i2GnG06nJEGV&;nzpuk(Ya*-`z(=xLf37w zbo(WO{Yi1so%M7)b%Bd8JV)4F>GqU6MN{o0yNuwJC6E@o&Y-khkP;l6OTsBX#(@bz z_wib(wGpj9RPC(+B)$Ysb~)E3p>3I-lNH4XdrdWYmzpvWU=tSc2ALo1pl$s`2c zt&d}^w@A<{6iL3t$1fzfYvf&_`CGe{%!2=(uzBKJha`_T$Bt}VBQbbOU$E=Zblvhi z-xD%URzk|{JE%Vkj2BsAxVd{))-S3sC?uo9t@AP}f7-hpY|fAK0Gn`gtTl!n=hcEm_z6c<8Utob?FAx%gQ+X?cVCn%iHF!{OPv@LJEd!JR8r0)u z>rD3{4(fxUCGdH+KFQ(a|HU0i8HWu4=j9mDwyQ)&Cq z%vMU%k6Rh-0r9rWnV%mx_4$>Xj1ix%)wyQTsizlxRBO^}(-KKs-Re24^J>-f+||=T zr+xoSZU1`dUX?6@s{ZnwQ^)hq4Er9d@jKnd<$_oov77Gy;G3S`uH5+U!<^IMOWz$P zioR~aE2qVzmPoHiTvxVZub_jQuNr$kOiTgbhGA2rM0ST+VYtGfZS9Y^{@J1Anj-(~ zdjpIdy3{3xKRKBezxVpS_Q`{(r|$gY-om2})AFm2M#J3`UQYfwVsuKBOkk^V+uav? z^!_)qXUHDYe{0I?(Ov?+(f32z5m$rbUD^@@V>_aO4s_|&_xH7%-^8GP-hVdc`^EF{ zNd7M{^8&fu_{htg8=qR}A5j_8wy}NLy`(o#zjS8WKZpxRmb7c&vs%iVn+9)1pipO$ zLq^2w8n7`AoULs`R-5uH7*2X;eS?<}jGL;}(ZQ*3?zC-CA-6x5qjw&|3Mb4(O!_Y9 zDi_Nwm1qAhHHzb%KeO})k+xGv;^T=VY57Wo(?L)j*G&IlvV=T|Y*Iz$1pm}xC z;$g>U&^3dgfkwILaUcuSRfy&3CKO)aaS?Kai|`;O+PQ^vt|=}Z#!-f_Od3Jz{X9TX zFj#h+|6&Mu-7=g?qXE9gn^>*mNtcpb(v{@sKJ%7 zHt!-66`RXdmk5izL0c`J6D&WiA2aPr%A|U}Q@b}s0q!FA)&!mUXA6q^+^ZQ>K&T5X zkOc)VBGTEaA$y@^#UN+WS7N8X+s;tFyhw126;3GAJjKvao6R+B@}~UR$}tODhJX~a zwnAXN3~S<)s8)g*4iVngCm=eZ&P8aH&45CX*MI2V43;~qDVlidh^vR!OF)317qXa1 zF?u1E9|QR5Yz5MMkZMs=m!1QN&r!6}q}V(~E3;j8r3@G5V>Nq$umaCDl&^k^u&Ok2 zO98(mjb*VS$x|fqvSW409teJWu-P5g0wIxhVvtnen8Ub+kc)f%O_PlA^MLz;aq}kvIAEcwMa*W9t!&D87uRY+h@&*9lw!4u=Cp zTHFx}E?dBMw0vtvKnTi@qkJNc?`%QZdIeF*UC^zcE5?{FfP+ASqBzAE@bkDimZscb zak`1~Hg^}ce@^gcVQ3BxS8}_8e>U(2kpIf&@3nhxw^%g_G@Z1^n>wG?Fyf6TCTMk< zX~Q_u?s~-A(2?zvoqXf$4i7F&5m96pc8x@BEW{AH_apP(L&XG^-eE$?y zHaEApcZK`~_ab#TG`N5}OK zL=Re3)Bh+g4q<}72jGAxF~5+Q4VB8H23Y;Qz{z`1x?xYK6uG$9(XB1|^z_kF=R2zb zOj9!SDBZW>Ogwhmk??lY4|{Y#f1Anifzab6<_nvt2bHRUacGs++*Z zCpvQZ2wYY;euabM^(`Ptl3{S9@=Kz+5B2g_lXz>;H~#k-Ix;Vo5#pC1kJ=w~hgY zpAe^F^PEqYmsv3qMXntj_0KrqeFuOH10oD}D~n+~=3~|;KsIVmcO#270d1nkIIiSy z>beC>k5QEMI6Yt4ow-OM9Z?eqBdc4d8TPm5%MXu)h!zW%!CZSKz+;L-ubR;A+pr?7 zyGaf4WHJTTB>I>?z3h5?kK-qS3^CA?mn(5+_(+=ON}gizfw(b@?UY~yi*|Ghh_CNq z83J9owysQFC+9t6>u$S-Fj$JdsY2ac{AKadWv&+0t8z}AA}`XDNfv1DU>j+)?!NwY zMix3YX*GaSHq^dd&vE2>zVniFGxkZEQx-!X_}Ko4*6br#QIumS4_206CBlST?p0g` z_=!WJbp+~MO5!#+nh8L-d3fZD(!?7}B0&wwU2mi&4ELCJEbwRzHL->SiL*HjgVO{m+c$JoK0)|!U4uXLn&Q&!Rj1{_SX0&s`* z&GDr(@zOA~3PB@3ud`lLc$6AOnCZ+&s(jyufx%LA@a%&vaup zHnJfRdFum!bkR{qx-~y5k^x-RHxY#Y*H!UmhwE_&Tn>=WL=u*sUd-|<&JHEVcW(#3M8$_y(a~ zW0n~~1bR1(@@%MWN?`L9m;rBmAE&%)`5EVa#3<&< zXJWCGFCix{>q`DALEMoWpALB`PK)G{AT2x)>{?!n3ylKTIPiWuUY$XkR;dmz_xZHp zE}FnngmKK&%abyA^s|KEcDcH z?l)cO=UCgWj;Bb1f zYoDL`2YafB4E?|3U2jbm?fv@qlQlV@K=K`Su(N-f64Fo^uE`8$^XX@9By#@tk3atD zNbs5;f6=3Ckb*`}^}M+UB*=zUBB9XC)f``IIeVvItu4`i@`v8qvsI6(1ACtjeRVP? zcOaynzyJ2%G6R1tHn+zFi#Ff2pxN19eT9x93><$*GADaO#IJ!9RBfOl0G6Pb&{MB#m&=0!Awjo2+Y* z1!;{@*G5pe+I}OJ(LfJ9lVO~-=_e342Amy#`L+1kPbmxiP$?RxSW(W}u9%wgQ= z?IK#v_cbdo&(Hi3L=#>b5B;(W0a@+4dta|PrU=X$!xrxS+q>qUl18Sn>(X%Hr^2HT z9-+G>`=RJPvtBu&H--JZmrgP9?4_Q7niQj&x|vx)BHn>2Zc z6G4fNz#9m8!+Ez&O}rsTM4a;`M7q`j20}PpVy1$kZ-D?lvI8u;I|c3}aCKEGlw`SS zAF71i-qO-?rbZ(nUzUUwIfdX8vo=&p)aO|IBMLTPo=qslSOHMZ2!apvSOXPp&rQY>!6`nWoIHen}Mw$2dg zk*j5%I_jxrCtYC3;b@G;lwY}oO$nI9NNPQcA1A_xF8-Qtzvtg*Wl7+wqn-3&KWl zRK%+F{H@;^WvRS_QKsFaU1_F|;TCmWG}2hXRx_DULoRZ;iHVPxBU!PycfHXmp4Y=2 zK1tk-B)pptGu=1qJRC|>^`rPYYE zbxsUwJ6x%G5(ELjwH=p{E)Au#F|_!kroszpwc8$`!*`*w^-inaxl}D_3hVE6J)#ji zK<+}|0Luw!UCG8#&>!+x2S7#Kbs4uXX#Pejuz(^c4PnV{aXx0aC#wPfFSwn-D=c|$ z3B0=MB1rlsaLeyLe>1=EhwZdYe!EV;(U(V`nXe{M)1JjBk3xB1CtP4y@4umOp{qy7*hN>dP)@(&s%bxO=~Ers1e{5kkTTT~Cc?1W zM}H}k-sAZ>TK0M?pfO_6cRtkcvLb5vH<;-a(sK>Ta+fE~Dy`3=qg3Pa&3=lqj%s?H zn;evO?%RRWAI7qtS4OvebnD*l^50C?v!6 z$F*4jw|{+_eUN!5Y2m^L{l~uf$)4anrk17hfBuiuthDg<#;;EuU(4)G165ns9SwUv z`_ntuUybx6EQOrRIsM}!`gj(mxEM@4cJ>levds+uCi6$`sJf<{eZDH%?H@5aK$}1{-3(N97u`##{eAC;?>>A~x7PD{ zS3eJj{_Lpg3uzYOKzwIjW8~F7Vd=?`gu%zOmA}ME&i17XqBbYpUv1x0I|N90H49ma&i^`mBjk93Pa-I*luEupj z?W&8829IrQXks(V`~#C^@B-;1f~-_}4V0p6D(?M7`}GJ0%RMcoJMTv5S%8x)yTs#` z7?wwY=mr)~irbxWhQ?iGS8G`@kBB1eCU5u$)DuM7>K*Y~z@)hGK95_#POI>Ndu9DfWu=lKL_}U_ZoOs^w#{x3naML#fiz#ce@KmDnp68&o zEJx0B45t7Rl;jQ%q;E^PQwLf5|>p2I8E(H)G5y(c;Wri&h)WZat zswG!x+7(QS0Xb7 z0Gx0Uym@{;y!JWI(xlIs8$+ZD#akR`)Hmfh1&%uHAx1I%^b+q%erJZ zh^IB(QU<~i|672tva$3$y)7Ve$?QYjXh8v6K@Y8TY%>?n@}W^WM#g-6sjhaa`B`-2 zI$#RXBUz>q>g!Q&zTuQNb#AmTk2+a5vvh?cQZM}gas2_}AH$B==2xoB3?_HBzt3HQ ztBwIzB#S&RcWBN1G;qk@%ggmOQ+5zdElo>Q9|&pd*RhnZ_oA2Uk&a< z%v#L8z3bEYtVCE{Mdx|MgF0nsF>oY}aVTj813--IQ`9p)EEfne$=*O$+Qj$%3@cKDzhLo$ucYjJ_}`OAV23FpE{jY6_os6K{59qtU9F zny!kOtRlcA-poY=vkxH?VeNkn`&(ld@rUX2d$LdLSZ62v@$=7jY~?ldxn_KUbNY_r z$AW-?nqjlyCSI5tk>Q(QUrlw~(@2G22=4eKdT)QM@bj*3>hl`6k6oXC1Oj<1$l0Mm zzmYt|gNnRe<;mOo?gmfo`gVHhbKx0g7TW#^HZk-2 zn~AZv_bvQyNPlonY9dOxG~JdAJu&JE2|9lut7z9^$Mp_zAK0Y93@ZZCRCbH`RLz1l zfVu^K{z%BzVCq{7pvDKKhQlLSyP1ypYUbi=@KIAfL0FtmXdV^by{=!(lln8LxUT?WP z^o~W_%r`u(hNm!j#C4F%ouoDUJ`!UtlY*BUjfwACK{=36eXmwkRdnM?9v&)E*wy zvEfV^;s^-MrB{?b&5?e|do<6h^MRSCI*CTC77J`G1Sle14|Okf4}Ut^+S?iLXsja( z8S?(a&3+RM92=((R;(Nm7_k<4K~IF?(a?~5Wx5Az7r3un7Q=Cic1ve;v!S$LqPN;e z80JwytxNS(!~ZMDNtSFm8l60az*?%Gq(|I;1N|MvJgcMls~C|UC0#vzww6+__>-1t zA8)G00ADn`Y;9AGigY$0lLaf=Y^8MP_sx0M0xCD0`U7k+UcQ2$HE~SM3J;}e_fC}C zxLJYB{Uc`7_Rj_3dhQn$D2#vdF3egQ!%>A{>$il-0WU(U18{QI+T->%&S)!VM4Kj6 z>!pG|ss1>2+UnSNd+^%@i2ns?St3s?lvB@N)PNofn z6)&e5uSw8^(Q3H-^#b0ZA~Wqn3R{!udx-OAUkY%**o)+w{>2kAn+1&G0gp6}Mzqx- zxt1xRSw*9GXb9DB-g1mT#NWT>vYxNe87}IlSOvXon}fPlK{dirj2ldnguq{I)5Ky3 zL&HXnMb`}K=sKX?DjnO;wPR(5pGLGg4A-{gO>&ny!{s5RfK9SPI>A) zgI@dSjkFO|WwVbkEOv$``Sag&UOx69*U_8YK|Zk6Z??;{kaMP zIUehCsuy#;yY5cdp7cXQ*E>7>)4R(fZ??^_{cFUJR@tn*If(7G!o0}Hb4ic7-rc+N zlb(I|I(xf@XGOEK=G1|MFfxpFozH!tY5w@Zv-7g;wG}z_#J3k8Jox47j#jV!u^}#g z^Sy8T+lzZ{!q~($pCaR&P<2+&Gn1n|kkL>y@Z8+fZSKOnw>QS$9@fu3Ym)h~&Yl#X zDt+%SB0p(p_}_o{pV^yY@M}U&0N+$xNZPyqxm4c1>ula&{C3#z6%tt?<6CODg#*dl z{o8e7cLL1YcyT6M`ukgZM#d^l;o6IFCt4oOgp@xRE;~psO40suJc*R3TEZpyAH2GD zftU5+?)S!X{LY@)`{9Z2j>Y1xMY8&n{kIKKy0^DqYF$0=8%-?KW*QS@Hz)9d9DIRh z3;@8CX!}Q)H;w4?$k!=Ix$ta8JHTzngZ#2g){ZW)kGUsXrFbMkflwMZ=r?+Ok8Fs(jyMY&Z*_-d4_6?u z+^N=c3Z-34z7=HbXH$)pRHr0Dg0%hyuEADwk9LRaUrX=P%2q1H?d}ze`y#fNIxi`? zK|${fsG%d7pu(Bv6&-DnlMP!vJM;z|lusy8+W^)80A5B=jZx&$Z3MX;j3RB73@~gm z3-Qjg610F9Ir!z!(LRCIC;(b$h9Zg&Ca|);1x}%UbB%x=)I5DI&WnOY2C)m!^7~y! znA`=Yg+g#TpuV*hKO7W!zsK}zT7<8$R1yc{u1k?C-=RWF#*RYsH5i$r97G(sEQo8O4jCzTf@6b~ZNGuoN;^AIUJ)ECzb-$1AF+G9_OdQ1k zN!xEAS>(2d5|(Z8T20U*K(lkoqq!xnc|An2IkZkkkjYY1H|h+Szrq}7m+0&Evvvm~ zDw5nCwQAn{ogrzhLAzTSDH9^+*o14SC}8U}ZH~Ae=k0cS!}w!TS?*}dtan6zFUefS zBHE^t+A9R|9t{IITLFvAE0XnKb#~bdcpVpkgi6Osv0x#LO$hU4kMN&6wy;R;C1%b< z2&#$Y)2wB^%)F0;eh^v3>#-_$7}7 zHZiimwXGd(03)XNf%ioeEO9@fIagE}o^HLNnBfj*6b{|eN}J2fA2TCW!V3O^7Uki1 zVW=*^Vik+@H8A%F-fNwJ8in2fw}(nFZ@z}-x@!Xvx{isG&A4a=X7wX9_XkWkRrj&C zV6_9qHvOII`6)q-v^F7gI#&S<_O8HPHKNzkNt-+9W)S$fPc?f{xg^7);aX;Gbj3(F zqKD*OG@A_$kv2-l!10WDJsQ*44PJd1_i9R$vPN2F;4hivuWKW8Sp<1;u5PN-I91A; z$x(!oHkIDqx?3>)xGT^0N77DlmU&Q+gkh&qEl)!IW+l{$I$2Uph5D?I*=&I&hZmP} zR-8jo?j{^}wN`5r;XMoZ@%#H$0nZDMSssR%)Kqgkilg2A#r-oWXTlH1{rclSUW5{} z-aR^ScXzAbfA(iwH+Cg*=EA}vinEkYZ&jydssOKjE<9mpz>DRZA*dU+zh^|$=X5Im zCO+UBFh64hna4En*FnX(F*vA-Bp@3saUoCMe!sBQ400FzNkSFBq4Lgw9DpJ={HTAq zNX{e>U=V*uBa);5dFfvc_Y(6jNTT85?B4x*V~R$P_Cne(clciwFNOMdNgv+?xtf^K z8FxMz=#*pAMnPBFe(~blej$5*xwi1FB>v3$n6U|5cE>^5qYAi|3XW~N{#G6MvC;hf z=*kbZUOnM2^6JNaYWqOP{`vc7)gi4}`}#8!H*9w_A+ItICY5An4)g;z$J4SFTy|lS z)RlTFgQxDroz={yPD(y+vUeAf@zvEI^FltoFi?oa-8#07)h#T3*a(!;eu&`mp6c;S zjsI~dCv{KZp!zazj&m=@hpdaGlgeR{O)&)3(pxZZ>GY$$Z#T|eSoqBM*1Zl`E2C8( z6PTXKiO4Heu2h1=N;MJQS=i9~twgeC{gbTfA>R2t3EnNMtLxvt$n&dM{i-@=SAk#q zCE3|%fd0rGWF|sYgZVW#e@{C$nRW4CY%i`L1nGX8Vc5gis%N@h zd`knEEq5uFW<2f*cS)juHil_2A?TnQA0T2W^I=1Kh>d6o|qw4C}BP6{8Wrx}gO*6yavxvaOoIPBe}aJ2b^|)b5U%9U zwRbGZbCCmK4L9madZHzd1~u|GJPeEq1?&!QbiK=!C^C3K8?Gx{Lkd<&pXDK7lUymj}}A#_4JYv4RoYNRqx-uG1 z3?B$ydH$U@vN*f{BI=yWjdyHQ0A4dDUq1;edRn>q_&k&UWt8E&I*2jL^{JxNc*?sO zV|h3aEu&#fA1N=^rdfv}5qN39PFfmVBjp=RG|%%$Hm{2&q&shz($Iku+~K zz@$uFSbM^TU^YVbC6a8-$DeCH*i5|=ANgB&>G~CO0F_B__9*xBwO>b!`ZDP1))|b) zK5R%OZ|ui~*H*>#npivy8eruF?985`Y-0Cp3T9AW^9E-)52Jf_ZjAjPiI{w+o?xYBmRTID+>*PIom|R4zyhS%W1>gKS?(}|vJLCL8 zMcI@B;ih@kE7JsA*KVpmi?S-A+yV;lQ&scAVUnJ-d=1BQ=)nF!(+n@U98a%}2J$bV zq^v2P+8R%+7J!;_NDQ_6&*?=LIZ_{qxjll6f1rUpvNR{5R!oXIZPDR)|P+aw1%<7Tbx+e&)gX;{6 zrx7dTJbdut8m;s-0C0RtUD9~W!s;E86heM~y{^o;l!md~tl8;Zx-m{G8}_AJ+tsnP zxcTw`_hVTAeh#u-m>0~IR8UhR*y>7@KsgYAB;%=`W{&QZb|RVJGNiTRz=b2|4X-w$ z`P45V_783+-uX+{{AVW*hfUV!T={9=88SngmbZK&+4INzHt)n_s(4xGrA;O&Rkt)D zD8*o&bAe5Mg!5-qH64;prv|XA?~z3zqp7Ft#I=3C4R3;L)xdhhpIca5oLiVnKEUSd z)#NqK-@d!wU%OxeHZN@SFWq$J`DDh3%;*D4VHW~2Ik-KmMNPaC_lA5)o)vk*J23v2 zBSnp{A2Pd&6JC6n62oXKo-6yR@3LyDjZzWAfQdp^B_J3BDIR8`HFbxId!Rk0}tvXJc>8SD-{PBVSuVCwq)tfrZ-PgQ;W zZ0bV!%;4An4$^k1UI>ty{XPA_aS|Nsh}N_eMMQ!scw{(WOVn^{;&qYNH*2P%z3_+;<49TVIYqe`ng-zJG?`~U*C-?UHB@wbVfq#@F5xM!Uu?(4EGA-4duM6+nuR+%|ja&#IBRx z^X#?TVWbD1xOg3tiRSZg4@^8f$Aj4?Fb5$ygB7;ML7oE0RcdB3#Lb~ z+B4!ULJ#UpFMO#l4ecJ`6)(k?NwZ5#_Y1~mQ2*lKpeuUha)9GG>K5;@^wsy7^gQXL z|9m&`S3L3ah;{@q)byvWTlOgGCe9U1|A%K~LM?$tgcIVa05DKR>TztfR0((>w@VKR zsiPq=-N@mS_-E5x{#UEN3}0&E^~rf(>Y|nwg_kwgJnq+dCv zDuJ~HYFZW;In@8@yXH`LV4l?oKx`(;d5~sp zQ8<+&9@jPW<_*wPq?K%32WRsF9t3ux%zM%HbZ{@!M^F#aY8m!*#-@ep@(~!4sOCKk zL-?Dwm$MXa`^*xx^c}-q4|8}^?R8!iUT~!Kv9i>Z4>P=ujhG2-L5NbffJu7BA1#uI7frP*-6szn1wbo-co}Cy&EtXOKY3-ZHNFc?q4H-_EAFY zh1=fog88 zE57u39ZR`3Q8ElK8wYtE1wI=eSXXiV_YU6B5Ky-)Vwk3fbFbhu5VuBP{12e~B8W@- zd?1!g!kC)4Jd6jM;VAO6Nb;y~J~P)W;wdHcb4Vi5q0l#T@cpFjYE6rTM2mM-D5&eR z&zpD!%u%-4-k_?=p8%~Zw$VA6k7%$kUcX>CueUE2uTS@IbnO0ZIkP_5598tvoqzYx zvENTI?B6ty*%Z&OZdLRFT$y!to41^+t8LPBE4V6QQ(8M)8z;*iZ5Y8eiD9V& zKie{@gW!aWFRm~1L75L==5aVk11!XzpfIIKW^82#8!UVMPM!Jb%AHyEO?>UW;^+G+ zOr@SF_Z6dhf{r9z(l6WIQ3vVJ+(GIkg6DTcaMwp^TM3!+s1S{M49=PQt^Pzq~3XKh{;QQ@Q{4k>m*IxUr0zcvC;s@EorE+;QV16bgRdsbYjHpD%kaSo` z2)jG(qp-WPaoZshkYd){R<0Dj+nMZ7Bz`&8692BRuPpV{!j~b;O4IdiV1~Z4!M8Vj ze`}D642O|1$JvX^Q?@>Hnmy`os?L@y?_E$yck?gCz`aSnWGZk_WQQ>wI*` zp-Z>U{l`CQ-#Y!F{R88-aK*iFTOK{Yt2Kb&E5gZ4BYVm>DUVu-FviXp||3-3w_EwzBRzLtwgNCH@ zmJPQF-jo3KFT7D#)*8TlkZXO|)7mjH43 zYHsO~ z1Wsjw^*OjME3VP4cLSWW6V9){0QD9vC~^6wotqq2QEkywiq47itpIhwQw&o$zGPxP z0%EZSTyA>9IhsF)cesH|+s#pG=PjMb^uGHa)3dr&pQ-nLa)n}{H) zHZKm?Np+H_$(O;H<)Eq*JJ}_DS{go3w4rya1`0CqoHwhkRd{J<0N61)v-KBi@|!qKAv; zvU%1Rft9PORF2R-0o)Aw>pcvQDz^lzum@RlgKLi^+myKvnH!wV*A+--zhYw>IK0UK zc@YS6-Deao@k#9Q`k_Ld+1Pqr10mlD)Km#JOquA-r<+gYgL+eIq~rP$U%-Fe7toy4 zH8ZYq_;_a;t1tI<{x{Tqv!b*W1n$)4%tezKZxo@(n>+4cu8Obsh2Lnv^W7p&GX)6$ z20Cth67hcL!HU}Ie+7R13qMPJp^@K%2Ip|Omj#Z@qR0Y`4Nx|D+~f<&P3o1k`JNpv zL!7Z%c~CKx9oJjEP*guLnk_F%8S^2-t8Co(+j`D$YFZi}3^g2Jq)}8=Jv^&AhSbCW zE$aE3*^T#Gr>BZ00V%Ym!r=wZVossKNd}vgr6*(u$!7u+PRQ+p6R$2S+cPp_@r|rv z_0zSq_SlVVsA56BusE0a)1Bgv4(iq4>|1A3j_MwrX+w`?=B0KkeN81JsymRR>G5i} z6&yzy4ZbdsFcnL3%w-ni8t-aVX3^Sz)`{}m#X+*pRmx8muH|Fn%?hi0ClNwwq zRarr=ncQ)vzo7c`7rhG~eI3-f+P5b^xb$LJX>C zsgn~vu36C?%u``39C=V08P!<;{Mwue8EJVeFeRZeinwfD({h^gEIS~cpA@be$og6F z;pRNO9%z{wd2nWyrB-kAhnZ7ppuOepD_&)5a3ym?&|-W24y554lJEdoDq|AIqay)b z%v6qdwE=332@cX#0iwvQfe4U<-+@zv618Nej9oA6=m&49qslc6&EFqwS1Lf)I5wMp z%hIdc&6E=jQwW&h1+DT;S+L62;6irEj;uEo^|4DR=~z9O^bH-R8f8hZM~9STE0CZzDKg zTC~UI&S$<9P;jx=2n70nMVm}Zw7JCQCICwxciU0zIh*Ab65f9yOt3KGf8oudh5+2p z^SUi2D$;YWZhnzN2q!V*AhxTI2>$}5Yo=KKiBh#dKH_9@=Gp8~F@x*+H5%8vIOm6w zBi_|qTOE^&cTEIA!ZY<;+(p?A_j#8TDNl)*>KCJ|#u#sM(+-Ot#O>14*I}UdoUM^X z8CECU)p*LEmF)-vXq`xd12W*eVmPOO{J(H{2T!I8#xP~qwDxZydWl*GL1h!v%~8!! zwNz(-X;Qi#dqS3xAd^!x7MiX!nQDC(QZ_of@$nZ)fFm&*V;PUt7yhm`VrixZnq#yb zsu)z2S43sMPXXy+77#C+Rclo~dN+5%ri!T(@x)&I(NCOM(9f_e@(92lGn?VmLIFs_ z((G!b+nUDX*vjW%HPFYAzf6&@w?d7}sItW!)Xf0O`j0#hcps_Z^4ydy8OrkvdUYPh zwjIH{`Ow^4OO6Ch61QMh!GCoNr8Be4NGKWhAzg{?nvhX&`fd%B3}~8mx<{*soEPVg z4&ki5NLvM*s)4gC8`|h0_O$(K61~){b2M*0ekitsT9 z$%-BG1wP(-mbRvAc)7ko6dFb>RTHT0c=<+({6~mP0bt@7f_NLSR=2M_6{#vgXG-F5 zV93k4U&%~YW$mtUvSe|vN#hsYb+*Ovh@5}$-3`5Vg56|$zSn zFIqm)$c@64&25xdfZahEW$36%B0Iz}mk$R^%mJ~}X(OY;ok`Nk@%E{0tqts`m`f@8 z#NNW`DK`4_vFEl2z}Wfo_;^zyXTj9JA51;^uk+~=e%0ST4$#c(y4OFO7&}x_@{s$c zYHw7Vizny`1cyoeF8vpD#0VTk2XE0fEROhRFu0KQ!$Ie#Ujn%CPv3m&O{k5Zx9aT+ zCNg|~>#-l0|4ylSd*V9z0&;&+mCpL$fU2ndPahLmHg{7y=BOU8P6Js!pWehfdTdw! zY{D8lvGZTk_jb*ObxW^z-i|c4qE5~W&+j^$mUOsN26Rd}K^aA}^=~PsKWweVg-GWb z@whRcDoPaaq3mfrk&-f<`1c##KYfP%=-m%)PolT8{G^lj&iAAJyW>aC^90E$;pen^ z#@pw6Z<>0$-}%?w^w`UQ60}P=oBYQ&`%XO~X4iiPt}nxb2co0%42`;y^+NY0ej)zd zitxpW4puPpFTeidFTX!*eee5E*n8ej96PS)DBkD|MB1X44zT3G8uD zpPxzSIyI2l+km64cQuUE00G@dPnVWwR0lP3EYRjcN3f)6ZTZ~!1>f=jgpElzObZ5e zMI z`Hr9s=mGHgm<7=_aC5hfBAr7a-6N~Ob7ApW0^H}Q<$W}V6pN2E>_@sf!Elk`9;B8m z-jNU}6Q~rbs}f!&mQc4^Ad&Py`O6z)sf`Iu>PS{xULxL)2eu!5hJw-M_|3^Dis5sBw6#?q9A;U!Gl}l z<`g5Y;`S^4+688r6$}0i4MvMoF>{*8pJcXO23q5N)Pr)jw?$Sg-F%`|L+n#kbrh;Z zJml{qHDFZ)UGZJdCZpEziKsv_TxMhP+-szz?*9ajrPump(f z14eFNxP1WcQeAefj`E(QLCn8a6iSVNeT-qNrnmVNTbegll00UZCnK}RRRgt&V2)H6 z|J(tzsUO`%ACJW~m)?pm>k!y`1(D=ZFo@QD6cRGfegt4sVeWPO<{__43$4~`qGVAa zr#Sq3o93^%H&J!b6*S88q!-TahHeDe*dHKp0PN6WlbS(#cG9;B><&vTC%t*;iPh0w zYKL|Q)mBICVKPC!!wkoRQeyDCB)rRvw_Od0`uImyvTGF+>E0$^Hc^dr)YX?8{#46P z?U+#N)^Ub05op*I5%>D>Gbi|~AHxM3qr4TUN2JezAc?B64sm8T`PtDof9$A1k!h=4 zW }DX$9kgo3zKURUyGa*@XFOIZz|YKAd}_rV%gw``MDAnahvQc82npss6i68sn; zWP6ZbYR&d`hfd0~^#sIMXhyis@re>Ai*i!XZzzVkLg0SLu>G!wQKf63<33#0a_`eV z*4(+SiUoY2!~tI{jHwY9wKHVAlBL~D1tLB6R`H3rjPkFI#Eoo8)8W+xuNzqClCJXA>Q_ zj(4SCu!%i>+0Zv(19Ir_Qpba44r04cLiO{`~h9saKNk8k_7 z9uZIexl1CRbgxh4hc99venH;bYe!u}Gn1v~w*A-Bva_7iJB`E}?5S&ao38mL+`V(} z)bYU2XR&RkG{4O*zxr=)bD2S2tV3hCaPJ^t7VK=ri5(%w|8e2rnXiTCYOBBLIM{II z$Jyb~uaYV%I){eNQ!Tl?tj;q91(k`@XH4QBQGKeQA6l{}I8EbSp!KP82`ke^LVS|Q zl9~I%LJ(dd0WInm#%`Q;OF9A%VcIDz*&EZHjTEct;|dp$liae_A8tpR&tTl3i+6M8q5TTozTt&*ww@*GTLCH5 zISeZ@etYQ(#b8B}053Fel&g3ZPhxuJQQ$g|!(1FJX^k(Neteu6wYJf-qc^5*d)up6 zG-2GIsVH(NgCbwmR`A71JWX7)0p8v%D;nPOqiXSnxv#M-VjjCt){iCE`)M!kP~hYZO&t5JdQ7&&~jN zCd~L*LjXhJBj(CoBk7E!G1edgMz!qcZ(Sd!B%n1%W1R*J#J1ZP zZ@2Z5S0|{qVzalrNerG3$vEs!SYMDd`;X_(ouAu9m|bPZbC>>39pDJ+O}@$op0VSy zmefsc-TVcAYXld*NZi*%Esw>PZSGj;OlF!$itdU`=a78eA6eg0(943CZy@I7lx%0K~E{hY7dupeTRB12~P&*ctP7Jnvz_y7e-8c^Gt;sBQ_>4ac}|VAk`fj%f`(ylB1tuo}$na}CQH z-ejj|3guaEB?T=OYY?YcW)R}_dXMNimLQzcg%LnWAIZ`ULwLwV$15KW&hz>e zfV^(u8U-6zFp{7+E!1ijAf=f zN(jKJF?O;Si*Lqw{&2IsCKxP!L>}7@d5TTcsCW0jA6Izx zq?f+=pZ|OC^@km$SN|Z?-f8}3I~*mx9=H7s4Y)370|L=Ob$UI9J(vv9=~EUI(qLt3 zCz5QR+Xio9na3(YM7_vW6o8_K7pv-lC)YJQd+fuy*!Q=8D?M*LLR{42NHP|C1-*9RC#e_Ws7^2hW=i#ft|OhuSL#idn0- ztFm)=$^AXwYKnIz0U(l6`i52q98yx>-s&y@(eGi+&CSn+4G_yf>BL@LP3fd1RI`%r z94m-H9hKr)dkPTgK_lSc_54^=k~=6!WB_Kp-=*pUli+j&Ma5a5X~mE2G#=XjHXN0f z6|807fd2fW*OL&YUGNp(V3OcXW50<%3yfF>WPP^X3dd_KK>_1|yb{XX#j z{qNW3-~GR@`^LJn_VCcL6%|8ElO=1*SybvAGCBJC9zgw9%f_nxdOrAcx(=5~N;{ae zk*x(PEoFo>DarRkOYhu#IG@Gp@10v*xY>E4DuLOP01wGb8qt)sk1^K<=ECJgRqQYz zUJnOFyYvFeCd{px$UMV*vp?(RnE^Bt*g)(B&?sVFX(sWeH+e1wv3!uW*a+Kh(Whwp z17wZ@IKukAj|kZlsEPxgDCaf0`?{-&3I311ibmK!nxIFN8fm3QI2bpi?xfn7DBFy} zAqEQTh0=2ZyDwK%xdVt;#S+0gdT4s3(&q(AAF{eI^v8ZhbdtYfWI&U<9_uFv_wD?5bS-m%_-)0cDGIWvWd6 z%6{&c-l@{NWlB7$T2gi^I&$MNGjek;#QZ8caDX#!uqK0lV`} zdI>bUTjx_XE7eiP9}(6YHRH#pO}XdCkk+%j>HRqFI>%dJPO8_B@MuVGNZa{gxvXk_ zpW6W#M|;+Dk54chZO8+J>@L&wMq|06Dq4!ZsA&y93yF zHVl}eL2HAS1dWH-e;>kop8p$=+K&tckb&cIB#5+~fR{zEDFe)8D6&^8CQj%kY1DYq z4c`oAm!ych6fiqBNGSen&;7gy;|*rc@Pl4tENcs*r9PENJ0aO_jnigCtD>9Re%9Yz?5q z8rve8-F+2N&f4*LA@H+h@{JW}2M-P8wW_)|-AnM&vJ$|NhE?8IBuBm(I8~x}1qFPv ztXjeyPU1~sJhHg6%Gddf*S{$9_2s_Z{Hi@baO12JxH{_q;Tx=Q z9lh9CKkBPbL1()GSEl~|9KC5klV{rgJc%OjCCw-5Lb4{+iIm1Ayq0U zTZ&YXHS7UGl4phrRfO1z1Py^ILPVB;5CViGqX-egB5NQai4X`&NJ7X)_V@Dt$xkRs zyzlEe&*S+0==ls^fXGIg=Oj2KxOU)C!lDH<&Y};g5%uJXt+(&Hj#V>Sv&~CX%PUw5 z#2#!kksz&aod&}`Hf9sp1imwa)XPYdZmnwLV+OHInM^I7)_OWemdPy4!iAfwD;%&# zUrWbpel?{EnH?DL7Z7cCxCTgG0_~GRNCXbjZXZ7%?AD_{?PNL)n;|!+#3$6zE?%qR zTkC3hxe}f?)~M;n5dHgZ%S(&>%&~;r*Jo0hk`^L%Fx;J))9G=$p+a%Uwh5tbMxZu< z&N>{W+J*9IsxQsU1GR&glK)&v95%jan@fR`P>QKZ-qFsG7uge$oLa$6+E^v+c=32& zxm)n!zg#1@7cvr4c5~XgA0FCMl;D5C?R44PA8oXX3wtDQypFZ_*^LkUGcF3zmsrU^ z^@#)!e(<&l;7tNg>ka6J66qJ=(gcD0)}Tvve!ob(PkC=x&0C)WF_TW9csVYv9Fa1) zmDl(3$Dg(>pLlrw;PaAy{rBEz#n+^hlhF=X_A?r_4n{lpMICr*=RA4(KUEK1>N`W~ zXU^RZjU-Iw?+y!fES^@(^7BtG`*3edEM2LRQ&SNh(#Xh25(QEHK4UUJUp2R2$v-@P z79E=z*LnO12}@s%|KQ91Q=QYZ_Qb~NQ+s$%4j#Vq?FF|AZf|b7e@k{NaR+8Yncg*K z{h%y|JMO*w>_W+>UmQCB>%G7K^3mY;f4TImbZ#%;#yxf;@cV_@#Noo)<4^B%I&=OX zK6Pg7d2{S^@>h4hAgR9tFQgo>{4^AFb|0@9Gv$TT>4fa)Lzm8yc;Kg|#mZN*F_g&U z{(>V?#}w020qaY3caFzOe3>|HEYt}6wmV(vE#0c68=iPz#P~l-fE=M<$Q$IF@1|6a zW6@r@Q$ftB!91WD>s&py0N|~l#Ji^rNk5K^H{Q;f@)A8Y@REo+x}3okfve^ToDU1 z{0UGD6y_KQ_eyCo!o=vYxlrCb@UMC*H3HBP?&oqhxb5lcdwZmhOuvcW^3}4C(3r(K!7pOQ z73SiOQL7FS&W)VwX~9sG>@_+|A?k1Dq*_!t(gQl_LsM#yqno0;jo=HZsvk3X66z4+$kTxLALpn4J$Awcduw z*W6<8KKQhB32W|Q=;XlZ_E64BiKF>X;!JIfQj^;?KbwHCF#0l1P0@5?%h6~APh23J zbV2KbdWX5p@ko})5$ub_s=JX$)pQKXi_c86ib&7`;5k>lLF1J4p0tJehxc7U61eEr z4Ls-%&dyZ|b7aC5MF3t*Ua8n2f4!E}>ERLANjolP>05lOx63K!b*fFLG>-$Ah)_il z;uhYmzB`SSl>$h(%`9}BW3J2mq*G@?z*Ooq9qD7+22~YaUWunTGCKEFjm=7PgyEaS z1xrEff!mXwPRJ-&tTx`=A0gRS!!2h=M@H z;QJb6DApQHyq0qs)NFykmxt^zZIm!KxK08ej~cg8QKl61I)JJ*Y@>eFXI{+i^QGY|IFK|~XG{CxhAbW;>loA5<1qq0>R=jH7Dh^oq zXhjDTVmV~0$%yLco>NBmrU8XWvHx~#4t^zCFIUD?7S-N&9nDe$7N1lJlgE7Z^v6R$ zD&k3J$jSNUTy}QTsHZ^c;TZXx*>2ZdIx>0e?w=ml9V3I{Z*Y?bTdQXpE6N($_SQG# zJ=)Fj2=NYXK0VMjcRgyZkpH9LM}bdT%9%(QCw(_3HmW&XHod;Ox(S@Am0>0ZaiTVk zKii}(Rm={SHp>kS;6?(DdFku3O|W*BYyG%n!8nn@^xRH6?NU0~lb7hd>(E(JJt%lr z4S0mig~=zLAT{l!pGO}&yZd_37oS~z{`iEQWNLmOHc^(omw)Q>u#3{EiBjNQ%J7tjf@zsjXV3YuCBCIrht=aX}_*1GC4UJHB2w* ziSRjHN13UZJk`W0toPfi3q&-=UTNVq8#dFfHoGUf)e#Q%N0z*3pH9(aA8)*D@7ECv zyq}CH_hKhX)<1(Y&fAv<_YGkfaV;{@2ctsbt{x_HH0_zhG{fGY`YKINz}4-OQ5 zax!`3{r4`?(@5Ir+j9kBKMgi$y4#*blGO6%)XdDz>=%N0cM>>Bb4jiqi6e7SC3);| za*wP9h=vq{Fw-uo?xa{H!CNVKEgi4*#e+_EcQxHGN{a*?Jg_pe<9VY0=x=)jrB zW}Ig_%nvQp6;8j*ux+-!-Y6BR=KC8OL%5 zBpF`0{APzRnQ&Qi_hYVlzat{wNCrZxu=No_{tqy4?9$tC-6AU)xm`-kA0Yy4zNs0V z*GR{_31YoYq?wIW%gYo#sFQaD74J}5QkB>hq5ik3^sEj32>HIBr7XxZ8of22B9Yh@ zSf2Pk(wJXzI&V_J@N|uyz1{9J(VAC~Dd8JWDBr>jhzRy{*@LxC20sb+-$AiaDjeA_ zrVq)BITGs)RR+q>RHxKD?8$8im!o46Ex!@@-na}Obw;4D^|X*LR(BOL zcbO@xexBsgnX8GMDQ1~giX;0-D)M~J`A@$hHfjs;$hG= zB>H3%r!PVOkuA;C=c0X+;8PXW2f_70*h7s0TgJeu+%{tFn>k;YJLx?eB|9lTXPqDV zUafnb?jb6WxKYh_V5(~nwrcU#au9rD0ZJ zJgbsZ5P+W!$I@`Mqja>@3W1`5NL{Tb+qiJ+{fNcOF$;Bs6ez~Hv$~@0JtI@@?6)?E z@XB7i(jb(vM`cw#I`H^ksts@K17847=-ZN#P+dw?7|0<7SzW!>o+R5M z-D+h~geDVylLEnMM*3c6G`gpDEl>H=#mvlWrq3XlSwjh9NAa1<^ef2$tH1MVyhPwd z0@OfD#>g5?ha@smKnZbDK-4TenTKlzErT3dM0A&|jE8OK*<^4V2~5hvX9dWBHK<~n zEH*5M=a+i!tuQ&js!b%gg`quF{-=yN`-~If4zj=;gj-ITp2RWs=VWD)+OC6V<0o<7 zCV2yefw%o9_fHP~vOD8SzEd14=!(#uAxo^QKo69hkPuKRaGpczxgDx=;HbU*(PUJS*~$dWA@qv|dv!u*2U! zypR$_Et`?#JB5d~INDw43Jy&gZLn*6K&n4f)JB`WI5*$29+yNB@LRyn(KvOR+$;xm z7QiwEp|<3>i=Ljl!0@Bp2h6t%z})$2DeKvhvu8V^AbktRnjztsV#$Ja{oTZP9l#G{ zbf>0locHeRZX4HwF9Gt?(>2FQ-yskFT6LeYoR4jdD4Q!vzd{Amkwj(;y@7VF-u2|& zZa+=pmkzL{{&o?2Y{$wK`AL_=05t3NR+69Q=9aULYzx<@Kve!MakGwV|aBB!yo^ z5<$$ZM%G$3I1~@tk$frK(V6i=ms(#%-oX7)t3^_c!}&!qXJpKmFXtjXLzX2rqkjNlCF7*&mAhISK->ibGJ(&{#cF#ici%aiYMAOyExPkB%?|`uOouh zw%An3>eKJwp-Z%V`v-v2_QdY;r+F;|aH0iTR*4V=S&Mj7A%l z7>a@|EE*2pBi+kin%I{6Oj%Bd7>%%y#NGvbY7N5#p~MYoOXm$gp~nDk;~Tp5vBJu& z_}Bxw7iL}Tu}~)&XWX>%fECS5oNSM)S1-ClIBdn#SJEdI^;9`m`QSLMl-#obg+g3` zjAmN)OXxBmzF*#Syy{#31tAq)jF)cz6$57);FmLY7Kk6@;CnD!}%fK9js@hb1-m_?CkUam^WYiFw>#h3IVSaz^W4it-GTx2b5JV z63T8`3UapiCs%X|0XzcC^&kwXaS{z;{xnadEfKZJ=aB7Y+Axx1HgpXqA6P^0BRAE_hvuT@NCC}1npj`aR{b= z0WcDZ084%F7> zq#fIk+Yxm|(-Aw0`;KK~O%>7VFRy5u+!zbT{Bk50TOdWT-pzzC$!r*R`B)48RB|%d zF8QY?HzHrR#g%}`0(sIhkb_>-NM}ZybS=#GqmfNzGp4@Wt|+LplZ(P--scA0r6|2> zK>B9|PFiu%=jem~r2pgmkD0&ZrFF#65`Nj+UJ-E~x@(UFh)n@$BP%QSRItpWnG`AQ zaw*b+qyF!1AKaO^kfeA54VW}4>;;ylEa+3Gx=_Br`{7wn%B zkB)@XrGb6N>{F8E?|$}*D3u-G7j*ffeZjT$MH!s|6Qb{wLh_Dy8o!;gd8P7i&p-Pp zHYjGeC*u4^(-EJZ_-x<5Z#G~5`u{%o+ovazQ*$T?MHId-8;oQ4bMqUb_N{(fYnG@`^3fNKH2QUT*?@EK>T4iPwTWbg}*}+HVbNRRE|ahJ#uw zb61#!&<*Br348cVjPWyzN;o}UhqTAG2PUifbc5 zRiDGKxohuOjL`tgAle0ZM~&`Ov{9qOqQ$1SB-C#8s?MZ{eqB&Gc zRzJvi0kA&RpeKMfq`%=1-W-Wfi$!O1glHs4)h4-Q2yZmY3P0wFK>P%>MD8w!;?Vj> zlJpIJrOv*3ljATZH=GVgl{^X;4+Xhvs9V+Qgt|U$kF<)h%U8aNvid>w@g`n0-uN>+ zI`f^8avL7T z(sYlQ`?>*c{JnjL{wPEd0auNIPR3{^N~cg}t)viv6$7wKS{i7DeHMnYOClvv#q*;4 z+5N!l9q7OQ7^eGEkcu^e=@Zi3W05A>E+vD3g+)}b3z)R2-3Gp5db>ZCq!C-M+`)~N5s|0_aWH1#)!flvBHcZtYQrMXLRcrpY#E2Nw-G(^ zFJlaK%5E9B1X%VoE-Mcv7#HC)2R$IZ9u5x3#w}}P}t67Rq~#Be*ejgYiBo4Cm#112Gxh{J%Yz%w*z2|?ZmCxPbe6A-w~PrhbGt}jI+ZMAHGdU; za;78-#(xnK)T}RFNW0bi_K3QwfjgQc1bEu<3(kFs>g(rO%$QP6}zsP%`TsiDN zA{%U+&TVzlg-ga478YKBIWfYct=$%SL}WKOkKOhgADM zua>aT`h<6Xze9fLUFsboE3`1RX<50gzO?MlmpZTJa7kib?y1rf3-v!b)M@Id2(9ABL${tCxYNVpUikf&%%n-LPbL$AmQVZk zT>bDeu<{MuK)FeM)&0_I9#r*raM`y9RzLptU*iU51{we$+0QAgG0JXo{KfTlx4XZ! z(az?$(U30X%#SCH|6ku@dxayT#Wg70G;{DD-!=3{uNa=_u!p`o@bt&dj^xQdW+oo~ zTIIg)%>UK>Gk%&^?TrHy)MWQvj#&Bm1C&!KPLBi~A{eu$<*eIye|c!4kJl`;Xxb~- zZLc~#=cs2+Z(h?;5AMJWfKVCv*>yRmeVk#Z+nX2@i8GUw7`e!G9vz#fTEr4#wrI$l zw%Nf{)4QK7;G`i{zrd~Utkygu=5Ge|YP6pIIb7jn&DLvy3Ig zfR=ZgSnc|l0{*GHC{RQP3~4PLft#0uIST@0V)N9lLgTL#3s6J|yG|PPKN zY?Wg>h_WdNH5uEpfz^2&I0L_Sn*-*y16Ys*%!)CSp4br}6g#g^AmX+!!ZR9{P}G!HluDgZw|-QZPo(IcGXS0W5lY)w(ri zTkn~4_3Sh;YSMX&uSXxokz7NZ!f(S(O!j*{cK|pzfEuNC$3^TIb7ergMHXv?E}GUI z&Kn~1`XfiA4iLN^$*AA>pw}jRdj$eUuE!CQ+K^56+b&~_`D?(?WV5g(t|g@5XH0w? z29t>aPSv(<5NmjZnvU>EyzC4D3*@=oGchp#mWLnm|LNqN;6yrf8x+`*_&)aN%q!3B z+W4e0_x!FecIbm+MrIf>(VJOs35??|iC&v&vl&yZE@k#Jb6=ORV2&hV!MHOVXi}Pd zISj)SYWpnz;TU>7x%TWzgwU@=knu*F0mi$sSpVfyEtG_7*#(mN@7U81O3SW34zIXM zKa@T9s=VV){#?pX`pu{H4Zr^qx0?v`eu>%T_X!n&-06kHyi?0rX=ypb3qLdQT|Rp$ z>5m55nlloAe~xViRp7f6qMl^y;0uinjv^VDH%$RVu6F{VzA$ z(Y3zqhuL4Fk{cz%=4;B= zw=Q-b{rvB)Xa9Qd)rb4rx(8G+8ajsG7cKom8e{$5 zoM_;tx~>SSQOh%JCnB@Pi3iKRy4CpPu5Z1a_CMJj0oSgO1A3FfIyH06L1^q$X+K4<3f_cv$8;6jxy|80s1Xv}d9rwlEkQL3T;vnn= zV`Pk~&7$g;*rbO7^&2DS?>a6kU2IE+pXky)xSwPVlGotq4!~erk6qc?KvTz2m+(;6DicJ!8ngk%h z{`+PC)l|D#C)${5M$F@R=x*tlRZ->`GTOpcymrQCZ~j5e4PNUZ=KV6Cmx@*Vupv~) zP$ECov+4VaDgZnf2(8XlG>bEIBsNQb2bQ@t&rE4+&(l>GcD7+b?kF;)RY4&k6CR-| z$lqaCxl3c%aLnt5xsN*5xt?dw9H#EsgdbOS0xbja*6& zU~TPR3+eBslo4<=E37 dtvo#bx9w=yG}PamkXZlnAn_2Q;S0z)%a!l<$-qR)9 zLw~OVgX({E6=8J>Mvd`qc)@fyj!MrZWMUYy+T~iL`J(qxIzc(#j(L2xMfaDmcm+%z23T-TxbJvG#k>s$7bmug!!`xt0dZLkLUXa*L)t54 z_U+qZ@yN2tXy6se%FXRsarIl9;pH>KBNbGoKgJ4(8+u6D zmQh9(i^cw39R)xQ@E}t!GJTEG*UKz`Bv?W}sM%1H%oEho5I3SE9?_%c$D0Ac-8`+2${pU0ZLOx|-F#Dm`$cF0ab3%-e&% z^x4kO$VQO%V$S+8wSP|ff}48PTqR)Mm8A@OO0#Q~(&(d~BmhNp1b4jeSWEHrsb+bU zJ$JcvZ~gM@40rm3zZ)-5mVR351#)_6U4DK(UVc&F(XG8VIa`1Hi-%vtIu-}#P3>9bqI8-|#a{r>uBb?x?=+AgH50+U7?~EY@$fsm)D0(q(2S3xXI5O8k7FJDOQj&K|EmzZ}Hn0?K2N^{@2(%2|e z4haZ4T%1IfBY5rI@O(oO&Q?68YgAEfxy(ANf+MSDJ1Tm!R}tV6&9*u_Tc)3vW~0zA z|0_p`@%^6O;K|#izec55b|6^Wvqatp&;Y4?G&$AXX4HqP3rpm^A<>x|MZ^Hp3S#j< zh-~qeS8rU-MOFXk{uDJRR6H z+D8vljFWIvNZ^+V7Qn6q6S)j{b?$0ob>7`t5k*A%{~VsG5!$yChwhkA5?q~(RTm?QE^EqUF zvOF23bo!W+|EPLv3M1Ma5w8I`Bm%r{KSN;Gn;XYHqx64L{NxQ(Q$AIdkJ_rIs-IJ{ zCjo#kT{&?J{B~sccKAP5w{^@8IJgC(-F_h z1L5X=&~o4C_I0o7S?Eb{Dy#8RtvcYRcDm^GXmPsoe)Xn?hwbyM8c=NCPJ*~s!@U?L zFl`y;vdks^l0k&^9>F;Kd|Rb7w*ie8J0Nds>Dd|psVkwn)nex=v43~wB40MDIS>@S ztE9OZ*B*NI^Ii49&7bz=(_+d}Hi3=^#!=n8bY%SShr4+XOP9w_Idy7dxd$Yy8Dgb< z&+>PK9TAEXPPt4(WQ9}Hq1~sPXz6p$cP;!;n%tZtynWKwmjfJmie$M)El=&dy6!i0 zwCC#C%UOFf5>J&pwktiC-cVOF;!yG(@&=ZiGTPbcNkD-GqespSVJ|GYly$^;3djPz zJJM+}U`Nbh&q#WVU`fF|TzV0fy7{>ANey4{rHbnWS~J z?T0(h&bl3@P4Tm%c}EU`oqI+Sn7M!`a81f^KEh3|^>T7{c4;ot=^^#$5r~~rgVho4 zK>h^~oP+-p|4ZU92#-u>(C{7ejQ8(JmvGH-<+Yg+hsNWsMD5es>=w=oboq`WP?x2b z-?Z=|D1YAJXVWUKrDZ-NadaK)1ovYu724RVF^3yVgvS5;$8I1*QPqF@*}KnwO@zzj zX%l-N-aT_;V(;AN-=7Q#J^9h0kM4c@kA`1oWC^>ABj)SB$&rG7+o|P8{8LGXcP4Td z(%!`0Ntk+`+L0PZ%lp<5bTLC7raVZM;>I8Rx5*WHS`LZYJL|i3>x_F-hWCXd4UJrq zi=O8oJ)N4nVHfBy%6!f3of7*iK0=r$;Wg6gG6fDmU95F6I=k@V*>BW*fObhJ78#=AX10Tfz&p#2k8;(o8Nk9f9^|dq!gxbeWox`xMm8ylnuv z#{zhYavr-fS6g_ofej!tN@Yy`5J+}$3&*1POdx2+zsgt+$CZY6=UU&*r#KMCI?D#~ zyBHYV9y)VS)~hE2_#+eHw-IiMrgtV$eX}`p?c{j0C{xU3z7Dd;YXZ969)1|XdvIJA z76>99boLObck#YWmq67-z=}B&hF58(U_z;yXy1aJH4iv)n-P&hv|dvg{s?Ki)?PZ9 zXFOiT2Gh$Ca7ic)$E?Ua_zMQ#BFq0CVmKm#^Hv&&1#dQPP8X?G|H;?F+P!p}%hI?T z@i72=Kye9Be3Y+skd?W&X8CF-go!AL68D2cwW14cy%$IXdaaFSM6?2~g5|_~OId%3 zv~J#4c4t3wMixxD_G*y+%VmKqMQ+Cxox@kC$(jJ%qj{^5z#$>Bd7@+1HOLCm_TRN| zOt-dfoV{HFj4mZcy)8{e6K_g6PzCu8lbCl;9fPy+Mupn)>76+Jges3|{meD2SeDd6 z_Lp=rcWM{FR!jK>R}MD|;Z_>b+~=9c^<+tO3@3R4x{9e=8jMWM-K>pSALrz(!_9ID zm1!NIV~J>{R5OFLjzh)9srV?gdaX7H0QfNCTbAV{rcS1dz6x#8xyzAVX={4CUT<6Y z0g)%?;rn?7!f>4NzKd;hM>Hs<)R?z9@U;a3?rks!q~_>glAsIr6q=t>YVc+-=;i1b zYz+fpWKeR{)?l$z4v67U0N>U>N94|j##pb%c)XR=Y(wCigbuwml0d-@DK6Xp}FoYM<>|i=TrO4nHFLDbnr0?{iKLAh^N{h5CCxw5+pz^Rup40Y|DMJ1MK>KY#J?O|^} zVkceR?TkZTCPJgP-l=6ID9*(@ELW z(hcd20GiJY_=D0mUyl^$c=a!-rQIQiT zA;OKLRj5AJ>#`O1J*!Uy5=;{IRhOk;LXXZ_Y$1Cfoc))6=ZUo%S7heoew$g3(@cw` zwCe4rj2e}vgG&ZCXg*8bfHqChWjQFe_7L5zC%=oG8bf?FS)(aWqB3xuv-w-0wuQ8} z=O{_kyw$-;-l^&Kc$oicEPdYmXL0XRBYSIYL@{+-+ukZH3+rob%UmyNM^nXyXtB`< z&dVAOT;eP?r=+r1UJd9Lr$P7sq@u^NxbQAjP;5Cjx2>V#ZH}ous?B#xZjij!Rs)lm zTSx4Y?(?Nu6Jc0Wv))N=S8}zAslG%8BtHx>N1vXzeoKIe5+iem)*pqE&^695^GFu= z%&wIRNxr|_?ud|Cu*B7g)+NqESg_zsU0c7)ARZ8EYDoEwA9b!X$nnsFH?EiTItxte474| zaEq_{jozyZw`y3#TrOE2eiM?{zlItU8^ZAy71ol{P~Pwh{iN+la#Evs9ciM`Yqp3I zXFNSX^{gejJt}9}S?mY+MoOlsQ*~Wre(gmN*!~R%z5}>Po3Fg}qI+7=i`Fi}B3(4k zT|K4C+ii2>hIH(wjLE@FJX`-(<^l-Bn=&4_w{v^vgPeWy3fj4WaN{_@vT^;kUqWon zV%z`QYXK`N6t|@*v@M}=Hk54Bg>?q3SID-i< z-sYjygu(RxfSCkf*M$bRs-c~!7NBsE1D0TdXTU}*T5Rk9%2%GDn2j-$$8skUosb*- zF6m?bc=cYYwq9K=Ynfv$@RRiRFn=&OxPjKsdpVK87Jp^xHpeEE0EI}5v@^1%*NMm8 zQnPcCGi3)rVNeWMMv=)W(U?VJVY|L}$gx)!+1t+mEJZxfR|ZInNL2q>Mi;(=1^#zL z+x~RyB*@F(jewImP8yFh0u1)o6tYBsF}+8s_MNIg+Q@7`%rgrYtfDxiAr^oKOd`=O zj;#;8xXSUXD{5Q-TQ~fWgz6pj?nst+JxZQDMyq(!HIurRBaNaCBzMOU026*BbYQ9@ z0+UzCE%YdJ30rIXqy|t~1 ziv+#`^Xy>%jILm1HjI8Sn(|Za(_eonpX~723n>dLmgIIsId{&Sm-k!(?PghW%!+#M z2681D-dJ}o>+xR}DhI&nu&~zBS8K6=ZjGf7nBk@~ylJuBF`fuXho};z1qn4^Q|T}8 z5YNvl;-h~xMi^O|aa8I?`6?qd=C@1nbu)5u+-2H8Z`R2V&JVeHR?Ul|hF7qkrBoJa zyP{euWXm8;$km>8`$t&g^`Id6Ob zi2d~MUtCOTh~}ZQ_r3@Y9g?Y9-T8}N*E_4BW7bMWb9(3LWS2PE*<|MEwz2`_b5R90%TT}s8Mbg)!VAyf!A_;Idq^0QUPD_M2dqV z!a11X5H2}!K3pN($`g`FK3Vq*9&YqkkB{&=TWM|!j84|Cf$5K3g z*?J@M7|PZz#191aK8jh>%3zkaUDF=!_Ko5BE9;~IQcN4ko&z=hu963A%^r*DSw5>T z$9wgx9oXf7^3N~i!Byq1iW+|hl<}&%FX%2JDnO;w;p9C1sWX9USL-^kqP~rD@3-Es z(Sga$bhx+IEY>Wk&X*2@J`G;t5Bz$QaLayh&0Zm+6|BKX*0>5_WnAe5o!&(a*8R75 z`E!BH*}@oAHd4BHOTB{;NWiLydyl15I)HAFV2msTk=lZX#N0Utye7`2x4j!XxgPO`3zc*rc@y@+&C8-oqz#-K+_ORosV%C)vdc| z-WFGk$ieY9jTf!&9Y|trEr{w0KVB))o>CfXW2^MbW`#nvlS3%82Zs?seR{zF8PVH; z2OOCEi9#3aJ)j7d{7%441+f-`098PU-zvsOe*viGlgXoGu|Iz8k4V;>fW2l+vkhy( zzkpz}RDxLsZkFcApk|2$8#7J~{QF;MG2D z`_+)E=f8MzjyH{SD#*Rp_vdnA2RHxbWcgFShTz{rRc~pRz2}qqKI<6W@BN?J^OtGo za-Wt?J#z#MiMaT90B?3gJ=FW8XC!5|3;bqK8^c3QXO+(`Txj^=Ux|$it3PM2y}D(x z2QqJn-d*l;AvCDuD8SOkh1K%UADLXc!2KcVLR_R>U%#*<&QYh=L+l~$a)m-6Y+t3% zXC)^AE=JorGvq=xL@!&^HbQlGcg_bg1!J0WEY}dt}gmOxC z+mY(~FMrBdt=A@dZG-_r|CI52*_q?o%O6K{44eXRzdNfv5sBH01Nl`ij?QtSlGKe}%z_;oeuofpApS0>=ik>X%0ihQC-+x+y)GD#j=w`QjSGI5MSiAv?x&xs2?kvWPX=sO>Nm z@4LS(YiO<>J@OW1Z8D4)XpxS#B_XKcJyvP)h}6Dci z1WEI}wgW7m`qObrT~>y^AE|gfiDIU$M7tuYb|Z~_&`wr+6?QR~r!2#)NTdcGn4|2j z9chs(aZ&z$PM$8~HaV=QLUJa?-4yOKFOY165}APfpdJ;3HTL+-mSntu=Pq`c?((Nq zqim>4RHkI99JU8;2pK}&_dhlo`_K(o4e2E##3!*<5IjQb1 z4CnS_j>xJF#Yny{TDko$JiuEHoi~&()QMMT+rqmPFlzv}nHyg409d<$lWKo}E)%DA zu%*Ujnu5g}BOr)IkVjGBnUiiHd;qg;!KmIj$yH&DKyFARcztC{zwxvhpcc`7XgW8v zdTX0)WWxQk<%1l{^v+b4;*lqw2(WT`mH_myElf2xNJhlY5y3mxS^@_PsVeB&4~QO% ziDiigO@0@+-HvE^P8m`yRc`GiV7qrjcbUu`ARC|nFl<}zPcD{ggnS_{zpfa*4c3X9 z#YYJWD^M|%x`M48UgL^;KW1&LVyx`pv~(|^tc!X@mq~=!Ue$w{hUL)1q|{!W3S{F> zR_i);h|TZ%M`s*a7DwAC)jUID&k5vI9ZnK%GY${Np|-z9nXkbNe>e(UQ9Wx?tucP=v)Y`x;dsYrH&RLIRnH^q<(Af}6IaV5UVFYIIZXz8H`rbGdeykt zHnqCTQJv=cbKB~sWU0eyF-PRs!p{U+%9#mz} zhO->~%O=6&Q_>Yd4e#^$pK*O8hX4xM*o|2gD6)~`d|km=2r@l)EdY7+MteZwp#zD{ z-+yjN7F8`g|NC!zbY1OfN19V{0{BDIX-6TLHxDoU=S@TjS+?x+2CAMZ+P@25`ATZJanT{`#Z2RLBA$s~*nDRmU%%O#m6NC1 z$Y&xe90cDw%CL4p1K00z?>OYfJzDyv@8TYwFl`o5EL{I@-^F9v>ceiyd@s8*q;nsC zb?2Tu*1z+%HnDPGrb?#e>+Ypg)+g_|__xrnf8P1mkMJh zKFYt_@|-R$d=T$$>C0M??|`ZH5+0_MLh`3*Y)y$c{cS0yMNqcf;;vK{So+B>F3cZ zv!NXg@REZvsA`rIU<7ON>(yomf;N~{PiMmS_v#}+~Kc@CE#Bn#hGIUKnoZ$ zukNv>jkl<;`fH4u>_w zjZkNQD%<5sV>qEh%6CFWEFTACH$`He>s>@fpt4XEk_f#C?U?4R&A??Euas`N1I->8 z-D%CY{OP$H0nd_TBjZ$k;znLUX=#3bZ`R{ait8e8${tTzQvuts7Bf7 zv>m}cKBfUb08Lv!6tAtN>(=_nh?{t=0BwCsx9S)y6Goh=>=tj+khVr}f%*K_3CmoK zd0`+~_5%X9Si+u)OZCbW9_d}@#_)ZO4N$`z5U;xS&T`qBBfZi_i%Myo)~U$A1_!WE z-25nuw#KYVY$m1R-Uz@H)iPLPQMJafMW7hh7Uv5yoQGwP1XElL|L}B!jFAJnE!9fZ zhj;@8;p4-Oa&AFxEhDYqfvZH}w?vL0s0Y{nfM%{Y_P$b0M%L-Uo^XK6mNq7*Xk?ve zFe*{Zrx};7NvPIESco{SuP$hZQvHpRggfdAbytQ-1~*#ZW^i|tHIqvvk(`*VHbUth_Er$vhcr@STw-5!zPny;QZV(8 z?#GKXDWb(#p6Jqz9+`pxwV}tpF9O5^sJ1fDUXmu5gry{>UP#H_=n37&+&ID-z7N%R zh4QKc{JJ#nWB59OG79R1oe9Tny}T9FxV*4_({ci)e>N==HEmU(OkwZA7pVL<+D7eq z3(%5#ccydCr1F_rx-{j_@$nKcc9Do~Sx$23fTDCZq%qy^(M)L1?bV^CBXvqC7Tla? zcn=RA1evkeWH5kD^xw-rHt6EY{XF`q(Fb;MR|%ho3m~PW4%0iOk-qH`S5FTQMiLk# zvvzqw1lnLF zD`Uuc#)?Rd1}*tTLv0X)>+!G9zd7%zcZ_s#;}vuJVx1rrI~rpXZtj2a#qU1^C6+(C z9zcQ$ymPi*pC*w6oHA$e6<`*m$G7Ln^S`_Oc{?lg(BT)EyIp@ssBZ{%6OT%nXg~8DzJgXQAjcI=ZL$3<89te zUv+pz40L^);^K`2$)i^wJsfFV ze(qhI5Y;l100~Wc_|Ja_2RRDLKK(iF(}S~f8w3=J?A1x@rmZp_-KZL9`TsdO^Qflp zbpPKwb7$zLOlweoL)jufX}9+#LH6mU->w6jRv#fcpYCOT!~HRkOqXxtj3 zh)V;F3_3??AjW71CFC4cdTx)S)6?Bug+$gw)||>38xHelH5$!#%krNBQvE8@ zv`S@<1CxPd6xpHHv@|t27zDZ-7*Il0rbVpvuSpm9_e1Iv{?|O)N|<+9HCa@PzS>m- zOPZL6x1SIUFSMckvirvc*QW`#B4ud6ibqW@ykWsJYgIJT=RwOn8-X(G$8%LHd_yQ} zPR@t^ZVhwm90V)yGo4BM##rvLji@t^ zK`m)%AQ(y7GI&;LBDK30EibP&EttRmH`hi zlcVqy#}&oeTUgnOV;u>4A&WX+M#xpUwz1@GjnW*YicrJSvA+Xjq_tk5K|4k`JIM(X zwWOjz_FO?S5#0s7N!5L3xJ+BX{LyNN3w6r5?0NSdpCZe;^2*EyeIH@(9 zl`D1RE5ZQ04Wa(YeE6`uGTh&AH z6du2(JKz#Vi>uw1Xbd@UaX~`()Fh~kJswp9D!H>jUT5$EA zh+^W!!Jehn>x2-mkQk7?9>nvDg9)yWk{dp34bDr?eH=wUvZ0dVC(sIHO#fphLH)Fz zsOitus%I9}oy0UGs<}SUHZV9d*#{={8wbNjE+n7oe%+PVQi-qU-jrRd9A)Q#K5Tq; zTz&=mEqYS*n0aCB;q8xq+W$*y_VV-1zqvh4Bx8+9}G z9tp-zR2$(+ZGvs655m8f)!uvQFzgIV@;D!N6@Tni^6kU%1e>IL?t_Qgo4Mz|gHU53MIcE-a^!2v4+7w&V*6hWKY5n3| z=XhZ;6=j2;saQu_P$1ETFmZLQjk4p#?=cri3${u;OA{oy{I#)?U}>gqjbp3yOm+9K z;SNb1sxW%*;%Hc{T>$J3`?KNWE)aG*Dcn$0x<*EVdQ~%@WdR1A)Xi zWG(Lji@ZKYalyN6taj%I%1*1)ZfgzWrLpa!e0WCl*gJ8^n7(xLXje0L@!g5sb>ExN zx6Iu+fD2%Ez!_siZ=Ykdl-=+|*%KSbqjUs{E}TfKpdC4)>9L4THnz=+(ftZQeC77K zV2EVn_Q}T59B1sthuAIn$Lbv$NQeE|9zh%Rr~v#$eK^$qwqC>iVx}^jET*BDl*b45 zA~+GHWZQ%vNr~O#FF{qgSM|sDsgOAYV;PmjG$s{#4P{p+^ajtK<|9!BgwN=%Y0db!u|&WyA0zH*23lb;GJ*)zx6_Q z8s0X7qJ93Ki3k(X;A-`ebE7 z9WNiNR%08ib-XS+hP6JJKUPV3Ws z70NNl%c)>uu*o>EE+LIDHN8a5Q+0CxY;y6u2b`QWz4C@2mvNUt_$cUhyV_O>-6~NT z)z=SajNPM3$UHv0v=Zi&jTr50ma!(8>xtOIo)jMB6WluA41r-kxU1uC zOZXq`d7k$uEp6Pw_9v&v;A%bFZeP|`mpAU7(pl?$t@{L{AtGN)MldI@#myfr4LBLO zk&Taeo0nkwGCOk8Ya!*p^r@eQy!}vnm8`-briEy%!4NWV8mp-oHk;a~VyBT;_sMQP zD668~jJ+ni4Yz&GHHvXbl7Fd0<{J{%z>Dp=oBBgicS@pb-1YZQU{khj{n>wcu->M* zbwA>l-4Sh43U@j7{;aGsAOZnsJCTr?ond^M{|>;JD;Ik1H@HFhMAj42x>t}I(RSmF zb1~+@G|9sJh9xa4uKkqo<2M_rO?$ZSOtsW{U@}sabB31jzJ)Pno>%mR<>lW?XIll_ z6W81zz~@HI$$soMn|i6u+4&+Z2N~^4N$hnyQjGxxN|hf37rO+&d^GbtL*13$a$7;n zl`QnL$5`H-a0~ZwQfjm~zE>^$X6UV+_$*h^myOkr4qf;u_GH7mk@4}rLLe}S6~Wj= zL8b7&_e}iZt>6Ed67xnSxl6d*p>xK6|_iDd&X5@Bz)LYSCgj?FCAO8LHd&Re)IF)wMM?vz&tuAkgKIw_C z!gEZ;nB!flnd@dIXNO5KDME8MMl>kv&q&>mt}$0~#<1AKp_Hw1O@5&6#n2JPCc(nv$8Q0f)LWXa$%WpttF!TAi9?H>aJlByQO3~3iD zqX5v9H7r8+95B%qLmkUy9)KE7g}SCFAa5cy!6iCx?f}9*1y5rj!~dPMS#L9sr4dEzZnZ0mpvhey$S(WNXjofKljbHp7&Oys<-7`0DtvZ$8QwEv z)aw4lpY;>uq`PSSTrX$7$ENWPtyyGoE{EEh#i$n9m3MtC7o^G^lh|(HY&ce3tbdG| zsT^`HpBqx$ThG+S1{fX>u8(^kcK*-iK=!(Pex8HmJ`v|JXjCCV+$)K>33??iPu$lZ zr{9Lo=3U@#A4Tf6crKA;r0L?+(eAw!mgo`zsX9FGI*rt>Dj5ueK`<4)eB%s;vKCIc zPRqZ}t3ma4RP9B^h)BQh@v&})+85A-8nI+PcZ877TUq>9hr55yMyP!Q+4l!5MQG&3 z@G>uN$A-}n-`?9*Q}N-ITJdPssdwmLQwDE>;H(5eNGQo|s;wE7I{B-5gG_B`7rgiUnmG6mUqx5!V0k(2Hsa!KE8se1^;!=|Szk400Ts zHU&Yk(kKjyw4MeFP>w|+yxUN?@~I|4F|L8%Zyb59u4qy1uP&N`d9tuE*7OU~F;_A_ z#IN1S#*->xyrWvlnTUaU5yI|6G5dJ+u2evp=@4r))VAsj(mf58v_z8<1pHdEP(^hL zgt;|Dr{kn@FG;87xtB9()s;2Rv<-V$wau)d=KgghTXd-3I)b*Xg`#sZvlhg}jS5mEUzPQfio;(*(*-%lOc5b!Wcf_|l@&1z!AKiX8xLoL4`c>FmPIt!L8{VzF zL{fGS15sL9LFB^)T?KzuMgS5qC}7^OZ+&;Yy0ujxn}pQeK&D$m-AY7#_HR%-(jIjc z0(di{z;dbRr>=-SPJvETSoisKdz(!y8u@3{5TKqp@@;Z`Q{E+ndsl|9UsUu0rxQ`Z z&F%!OlEHcJUW4&SbYHG^3SK6V0YgSDFdAE1=kY#Oee%(wWIDQcZi7*_3w%ERaCYI4 zUt)|U51KEk!|N9tOWH>7eX_T#;Vof^VByY7zs#aS_>h%A`T0e(LQ0T2QkRj$&41oG z@a5Bgw*T4z^~_VQNv^9XIH@Jr8%;z&{_gxEl%MyEEmsY0sa72it7z7!mc zyA}h>#o*IHjaiu?<&l35dhn?0o4VU=+h?NRtUs2N0-i){DG)@&4?BT)hgv33)J36` zB|gIPlQU85LsD_O{IalM4BR#(^v9%F_CY43@E$lvs+<96LOKDBgy70Zp_9{yOinde?DH{pC{J)IHsNL8_@(2L^td%_zobLhDw>)+jj%m&u#}MOo%TeiQtC@ z{M@vD?E~ipk)&@Wma+R51@F3S++UNoB*mB>ICD&}eBTDnX$MjLbe>2EjV73#*cMt@ zt%NOuu3Um9cP*2^*ihH`pGRrrlx$WUUJP+MqZJG z=dO*RB-OUc_O`s`N>TRGht#paFHe2n_f72kCc<(;FSEGj^^= z3BfVkZX^0?U*S#;&0ayr1#}%lbn5H(nHU^VG@;Gk_G_%{t*oq7vO>6Zqg?xISt<`P zq+%ev;>~r^`*d8Y-{xT9d>ppEGWE76_>Uq3s#5Q89J{7aWeF|MxTDlu3Rv_!As!fS z%HO&_KU6edh)ro{L-_F(9*(r6UQ1 zkSx-bb`<=d6gkpuUe11~OST2N*3Hv+t>BEv9cL6EPEtNo}Jzux#mK#(T^+lG?t+^2GdOUYOQOH0j4I6>Ay7$zx@;? zjgqQ-HY>G_Nkvpd`uT zBlM4H0sF(9iP}i}Hi2wgE=kzMGe?y~@)&R}d#7XN#Pa=FI6Nlv(QthW>fZ<&=Bwvf zP{sdET+Et_gE_}`FB1S;9Fy<@e|=TL)k{t$r1hz~6HxL&Yzxi9{2~&ezeN=h5;Y34 zJaC9BE{8X(%;2x~C&Vh2+i%^d^A2O?E~@pP&W0q$J^Z3IYr{1Au=&inAE=3!O1Kip z?`7h(PbGb4IFIlbeY_RBCBEKx#az)KAk)J;2?1oJXxYhQ-QCV$V&c3|wIf zPq9f`+fAV01Fs;f<7nil1sqCenycSjcj;(TzIn7Xt}eR22c}8&{!A$~Wpaw9gPZ8orZG5xtd2^B>TXVS(!+&7SA6^aQ}jX#r+2v;=g zz~dOYfcrRYq;4^ta_IT^l5xOmR1UXy?Q z`jzTx^y?S&E4(Jn2KGTD*}U0Gmh8vq>*i7 zo@oVt+~W&seDmZx25sy7LycTFHQ6+p6g}EEn6lfIIV$|^oX^|5p{x7wdADVEl^0HD zcRjkjAJ0PY4wh9kzf7NP-u3_HgP$fWKE7kmFE{`7=HGv~b?ftge(;}#!GepS9+({K zvt88b`}6$Cd_cacWPS1m`?g+uWrT#OxPR?fdG)^{Lio42C-(B^?(^4~kItDt?#lV_ z)gQlk^Ka2!A(`;h@OH1Lp5MdSDkek|RAWYoZW}rb%hzp*+XMS*3{<3cj&?sBUYPbY z%g~rkxz^VBHKPhT%EN)6LfiSw!w^f^cH%&E!{gZDgS!83+>9cM?{N!3ax~QpphSIR zNHyd_?Oc9MzHmOvW&L4LDAIA3{+N}uw1H+g7CjupdZ^$|plLCVF3F46%W{`xKv{r$ zE_)v@B)UKQ67$#*UO6y^l*enVKR{=#DU>)=&#D`|qT3_g30C(RYr=D{hx?0!BoS0_ zJnROIPbseJ=lKHr?^MjzRjmE0WU~Zo&jsz=Orz2~0J!q(#=-OV)* zvQ5nwWU^bB(Hv{CqMfR{+1L_-ImS9(LTEF4dPufjq{;fYVLrpkF*3(p!XTS*w4S)5 z!fTLn%dHbc*q%<*pscNF^J`6<7T3@{J_XCrQWwxzSM5DgjL=>9oo~T)h8h_klhtXP zmbRrP;JK%nB~(*a09iLdJ_I1tqiVEvTe=u5HJ!vU@eJ3y=oFcxW5j|Gcz5^B-;?RT zB`hYo)Eb-#NMzfa9a)Tks}2M6KkBv6K0QP_Y}3w;&1!;}WJD*x&)usonrNgEk^*Ky zp2{b8O6CCo9}@$qz3b9=n?}AKsHw$JR9*siPP||Td zaiv$QB4lAlZ%uVNY{gICCEb(S-wC~%iCaa`kq)cu5{2va9~=de%^ z3D}xM+LkIKtuO{yC;}?Qc#3^SYM)t@Y_HNNtB3l{K@76#N2h4rC%CQ08b@2LoyxHm zznW@;125NDt?+0hC5|Hccc@*|C%2iiBZb|L80t7cef2GS&+J$t8*qyz;+W4Gdk6|=hX^z zWv?hpljpNhw0YvD%XrMc@9LpuY5CpLs>f8JvX%fqf~N&4JdDpGaYS<%PoJIP*u3HD zu`tosK@c4cQMR_W7S?gQ$4jc?WcM-lI?Sc)g@uIzh>qmS;cTt#m2Z&1Acrw-x=`A& zCOxhI%2P9SEb;u8|9vOoYHiMfaCTS|0s1_j32()GGxW6Nyd_D0$i0k2#Epy%tL#SfDk7x@IveNcQ66#cgOydQmGcQeyJm14)&WEjPMFr)E@DTk=wQ<$8SD z&%v_WsYzi^lh7?UV$oOkzxl~4$6EHIzwmqiVKyYwNv$t^h-i@Om)HJD%d02!rb`Fz zHp>K}?Pr`kt9*FNiKG32&+9I&hN68Mw5?FK8~a?H`_`7nhlXJQ^65=IgW{ljm>wrf zR#P#GN#|bsa41FdI>7Sc@D_qYTYk09)$LXbX3$6h^{J zkxNID`jpC{0bGdu!54jMWYYo%>Z@aqEx0C3b zos?XWL?V#^;KS2ulCohQY8(W4=r#|Hck7Q(X;KKhHgrAv!Ax)&+9u$bZgK~wvp1G% zOBoK6eR8m5Ov%vgEQ(W`V#@M#`CC^LLLJp>Wt7wvVYdby;VzzxQJ;gLGJSk)+F*UQ zSZg(zZ&1_JZPYr>DqU5&GLU(9h&X}!$($?|rY5lGsNS_nxSjVN3>qfsW%+-y5K^W1 zS)^W~pOSCBL1*U)q#H&T6Bip_@SrvF7#FcftLL z?#UL^@2k|sqA=0FDRtbskZQJW_SFt@g10cQa>DfQoW&)eA@kGNran)!%HTsP_aWb- zkq0Bw@;9T6)sgNk1n08deXaQ@OiAirb)hJH$cx(0UQ*mqrn-;sGOnuWm+uf+#z@y5qf!b*-KnM3631#XBK5$iFO={Kk70KOs%S|{d&taN@I!ukWp(!qC@=FT-oIRZIm z5Qr_51sHE~Fd6afq*Rt{O@XkT_ZbZFM{z6-xa|eduh56y3mjq9?%eYLO{{gERxs$( zr;rdKSi*O1w5H zKiR@Ro%5T^ghrYt6z}zN4u4I%Nv*y-`W&q|a-;ubnMze%nZ}O2-6F=OP+v6o+YyHdUX@hfMM-K4GbGHU-K0SDRW~!((PZVSP@##0JH&l1~-Y+6MO zzwi-dyi%MrbL6BCW@q{UB5Uk4B4<6;nWL_V92afC zjEItvjHIxj$n!z55hD+RTZ5Hw4Vwf@MqXz`^n#GnLrHW%v9J?GF<2Lq@Fxz_xS41e zYn<%Lcs`RK1GH{8S@HFQz09(ERHp2Bx&m?7sYQ;z-L znAZ(|HJc94lK`;Di&Ymbi)U=}qW< zTiDT7)}~8ocvv-d)@xEvl8*T>x-DrRCA(GbYaAMGZ3>=`IR5r;T_XoOOy4QG3c)p8fs$51;xU;l{oZvU^Hl3BH~$@;ey$WCRnOrMbm|y2<3s#c)%&}z*I#^k?(?tj?W>7u zy&D_{UeDng9C`J$9j1#fUOa9w?2TDl@xUu&w6eT;8cdKIiM2h(8jUfDV)qQ~Qyio% ztS?{5c|ZD){!Wa(9vZlwyA(!lw6TV{QsD}GE+m^^-@pJD!8<)#%3OgOL8!gCRU*H>~MWl6^{Y7GEp1HDr9ylDi*xUJAJ8J?!>K)#& zts$jkODfUTbG3sqCSc1CIvzI{&bJ)$FT0yswU$=!J*V#$!)D4(Pk`fUaS@j;4kJ63 z#xnydbHtgjJow(5;P}vtfD+J#TRlnu4 zCz6-IaGXIji}W|qsu*KuBTZm_xH33sFJ$D@SQ|&2ID=Mp1`93ataVuyU-l0*AF;27 zu1AuhIQlDv89yWD)AuLDdDXN^kwL+TjP1Q@1I+h5Qi(A@D2 zBg4Er4%z}HBhsK}GH-M_uza^8^*Y%br2SPqELpCOeshw&A$a0COlGI;tIoR|&K-{!>sua{yDrSUtSp`Ax~;r1g#Fx=y#VS!*BYj- z8_Ij-F%Wsz4^^i1%jJB9+Ru2OZs`Mi3GGc;XTPFUsOeWplsI4gdAj}V%SH+GX+rx_ z6AD3iqrN-g(#Z6*u4zWSOKmm}+e%S#X)4wS@wKACG^w#}$q?Enj1$$7YA;;=ifirW zGf&$Z&b|o7vO^xhsHH`u-3X}?9{@wwY!N1Wb5fKNW{v#`(lG_e4vOKgxScQIn}Pzv zh@ntwj4q1?UbQ+)*0a+>HyK<#I$3MEV}jh)@`MoffRdBQX&=Zp4b<8ngxZ-rtRKfQ zMYsMAkUJNOSsI}4iwwaXdG5}ToZD>XOe(*u)<1AX6mWwDlFj{e+kN^@5cQ{=e_H9;WdJV*KZlotU!e{{5M!UQ_CQU*nz# z{^#HM_lC~?`BdlVY{iKFaSAn%n~!!!wCy#3UL|+YT3^VW$$aPH$c$vb?pqqze1I8@ zh>95cG(`EJwW7JOZY8NJqAV_XA8G#Xmld?x)YiHcc*rImuQbL5{a?=^(!ryNJ&GD0 zjYb}(!dN6MUo$mOZ-YTNw{9gLIw;|?%-#T0l;6w8gwsQby?&SIuzZ9R%>Qpg23fq0 zA49z!=B6%3D@Pe0B7F3xgW(1^%hKk2YfkU9Pr zY5WuJj{`1AY||@JhaL7hfuZog^3CHHvldAd-L^8VPn{ZL^m5oULODInRE!~(NpyNm z2~=QO?&yIw=E@M(uEpkUmDdzlm$kcVsN+iSn_S;$6eXiK3eY*ItrA$^H74dr-5D79 z3c~UXV{YaPYPa9vuU4kzZC0{pT>GCMVlU$pLJVh&)o9~h&CZa5paCTW(VB#Y)Rui7 zYhSAs5jEEv#inh_j*`+N?@JccfK7JeD#Efv5N8-iSkuMusnATRXJ@Y@LGR3CcTS~B zVT+nsqeazxjW$NJIl?hCM~55Bj))7m_rggf0=VnLuOTP@Y3eRP6lZwB>wn(wPu_V# zC#o>km9`w`UUECp(>Z^>+#iXcXd@FsvCE>#Pb&d+pg}#FsV<}ySX@JH<7=+b&A;g| z=9aY>f=E{{3$Py-CI;xH=uL)py8l{Q(K6hw+P_p`M zjb@MJUG5rMU|ly;C2%C@8j9JY3fn!^anBI(CXZvX zxZ<-|2)kNSlbfDs+E3RW6T}d=6lASHvh5{l(O4N2qW}pfoXrT5v9Z=rMllC`tA-z`BBD`^wMt}CwOFz;IxK61)W`0CKQK%Nq<-X}lPU%4;OhmS|Mf9MdE{SK3OT(?(p=*oj7o zKf<`fVdcgWPF1K{TO$#sUk9&P9yn8epBHKZGE#Jtu!5#oT`4cgNXFg#8Vg!mV`8XN z9hC-m zt3VSCR8`I0Cvr@_x!D*CRW{ z6Iwv;3LCr4mpPM2Z(wy3-Yzali<_iD8p_3GiDD4j(U8wn_a+s-dQ2xsbOj za~*3nrFb`!4o3ecGJkh(weWr4@JFeIS@(xBBIp$(Gv>!#O~J{G*&FTOaA$^3bT*a9 zPTbrT_f1Hc#7=~>p`vx=gxmG_kiajB-uUI#Papj7{`4C@C54Tjas6h4W`jK3_wpA4 z1C>DvTJr%9M5znX?PhaMdTwrR`ibuLmFynWPx~Pd{%JSJ-@&Dt+Zo_j)tA;D)1`j` zkR+rUahRsG70yg&o_q}uNlBNeXW>0jJlHQqk+$Spb3D}d?+7O|(R*d;!NS6o#eu~n z3hq?MP|Z-{(OP`{Y($B((K#ZdHJI<4a|dQ@^85RErK#@1`2cnoetn@et}($gG_AUK zak@kRKnyV4`jL|oEZ<|H0DomxF!-?YLVWeV-yBbT<-+PG$9rdwC5I);ra_80I})1* zsl^+KNvSDaZ{GUl$0t!opv^=u8gImkFsLVyfBNAEuZ=fKzwtVE_NC9y|2_1J+kbm0 z_T_(d)--spoPa;XuT#%w_>`H36J@s(Dx2C$TD=25tXMEj24bd@P-ka%e0>GeXR^;aW!rwZVS9z=CP+P3I+ z0|Wu38yXFJ!imM3uus!>D{EJacKharN6@eNOn{}V_bE$rkTa_Wnv-+hrk(;Ct~~q3 zJa8R-%+**&2_@IMk1X?{SQfwry~FkBoD=y`)}{>FlJ2D8&2o&M-!&vQ)`+G4xeI(v zpK?5*iluqFIBDKsx%iZ7M&E`^wR*#vOPVWjMT_I4tLBc-2#+dE!ID&L`H4>0Qx-+{ z&NffuM;be;lMC!QVgiA&T0wl(Hegf~Jueb8hOZe>U3M*o;rEG?_QjfYYgJC0U+68e z35^cBfN39ZCDw>I&y#VGZU}DlW?IPP{*`tehojqAtMvIrTk1cg9RHePIwba@%L7?M z>GGB5_4E9)Y^8|Z*M7b~`Vy{H@&#MUH`58FU&9qSB7N#Hmt1-zTs{qGl`$y$(|T;(k>N}#B+-zDpOD%Fj!l5vEqGi(+OAOa z;STsLY~5&H8MLG^^#jg*E2;p)pJD!@_MzAlO5JPGs{?SIiP5=mbWXg`Gz11e7D=A- zmjdu(XHjGBDsME0^lvA#p0b=m^Zpo#(mkW>aH;IkM`PS9$nhAvXO7z!UuY;Yc3iY5 zPnj-T3lte|HrDuNShr8pDeW-AE`h|*bR$}gq-cyWiz||i0cYmFR9hvO9kFX`z&h-e zpTdzEM^xMaZVCjhH}X%3v^V{wo2gQH5ycoz5EBe{Vzh55TwN(zs4792jyx1r8}I?F zN(0#vC3_auDN@9Sn!OD7-qQ%fuNnBA?MNI)J>(jrz7w-@qte3^gs~ku)#72@3`|X_ zhU*LhZ(s|-Mj$*=!(?10NJ!X&?T>T}GQ()Dv(1A zv|kveIO5<3Ur+Wbr;(lb_vMi()vL_wF2b(bm6VRKxhDts2VOJy36tEy<93a#Zq$m_ z-jdX=D5OA^f3js`G==}z4TclFDVHzu(@(s{eLVVmY}85T(;`&?syg5~HR0{LSyAx%p zDR-!%OgY#Or|biZlLL19BK$|KK6lzX`k>I&l}G3fbrY(YrGMRmf4Sbf#;Z9gM9VWM z_Y+|8%g#~_CzqunLuw?xSz}AnCq2vDrSQGN7n?3h$NJbgq^6U9x4iN1kaLet8aO-e)b3ZI-E_Olg*=4DmzkRFj zi|L0E(}ab)M%Q9;fHL`6$}G~H=H0AgXLsHH`v-qHdG_VM-2dp`AKXh_x#Hj5w&or3 zOr8-axZlv#_@L{c>jw|t)RVGLnnOC;vOhdK+4AMjK}8w+r&8Y$f5aF9$JYLanf%W` zObX_upNP&H3(|;twGB8!DMMnIP7s-I${HCG?IG#WoFmneD7e$205@zn8MC+tk)z${ zEs5hf3r1v-0`^ohdv1`ZSwN?xx`F7yc+y1Y6WGH*6q^`onEY6<5<^}8Vq}B zfgrf3_V+B^RbBL$SDZvao8{E+Y+H}%+zp&RU@w)7}|M-&N(SM zPf^f~9{Eq~_RtOKEuP&vaf6o6ctRTMyaW7@ugAQP4^=0o^b$TBsk5&ckN9+ToKyZK zw9KNk`R3hq@u|^z8r_+BvKxVF5y8mv3CLEki8Xc?%BF(89gh@tXrFNkwtn-{kcRkJ z$4Tj`b#XEip8?T;aPU8w{%NFWYOO`JdA}DT3RGjD1N*g-tYbKK3Ildnbmp3w2u2kb zikg5ib1~2vwQIl(?ARVpfIOln$_73$?H1kqN+TU3zR~U5ZCectxbcfoOcszBVnYzd zsKh5|U&WY@6D0cEO1L!lb|`+zn=ZyI*ECA?emE;+BUKWtLo2OLg;X|Ia3p#jB0MUC z=aIHns%ruv4Po)6XqVE}*SdX515*uCbT*nYzmNa$uy+Vrg{#Ss)wytQVoWRyCqB8y zZcZMN2Rrv^>ti-1L6sFvW7H5W1X5!l)O5w^Bc~^-nUxFTbaO?cI7=AB*$8sX(rxQd z%LDZy)jH_mSTkS4_YkK);%NQ*hszy{!<20y$3moRdInlt>gZ*VhRP|$O^CKQa~Pqeux2MwPDqoXtNA*ib5PybHBV%`obOaU2-!@_HHQYSzVg1 zZ$DFbHz#B?-nFB3td)F`8S+YV=iknkM|^u$x?XkZ=2d>jSR3{W)xqiqt&J;-k2(AI z!d^|E+Zh4sDR(zFMBC6zeuT<>L*=e=KjJ);hz?P9Qeg;kqJ;|**)_UyxtUwQMp zAA*z9f}DY=rZHo_~<0TgL|5 zRby@0-o%3b)XtN@bOb)(Egq`L{-Gi>zW>V(d zyl568mFVO|10(?8UBZ>zy1I|gpAQ;MxH^00ZpeaY)y1{r{MnbG+B2~})NhFXJI<@G z;*$p_-YdF=QpPEBtMv1jfl*`-ai7ObAIg&D-NYNY=Y)=n&=S*O8w}blE@9pyhm^sBb9Gd^M zOCa5{aEyxP8A&on?V8BaDu9pVa%AI-z(Y9iN0!g2LYba@AUQJMg8HQnO~BYb1GDBD z-?N+H1LZWbevfo@?4r|1TDPsUQJhpJz*raIzFuBy?raQLjdh+!wd;r+qXi~lf821z z^=ai!;_You3ZBM0|Hgaxn+F1xa_vAf&s+uN?c4`&lz`JxF<_v~Bz`b2zF zl&D#=wT66`2U|2@&moP(KYvi*mtklkDv8Pw&fpb7)^E)9Hh$(LUcvKMzbjyLRZ5o| z^zI|cxSCp_zn}d{BsKJ4W6TxVEM~seD|_i=yzuG)WXlz?G_x=75@$4u|``E&9sc_ZbV9L+4{cUxnK za~@09tbH+E)JkP<73Q?vV_X=IvpkJv62$gxf8Y+PiASuH@Tn^SOUOho5#b<&ilS05 z+LuFnb)^X?b9))2NrwUEiLtk9EcIatIiNA{0`*c*zg)?RcNI8OHtOQ9DhIJk54t82 zMEL>_$2eA$%PkS|7>>t8%te1`Kmm<|X4rl6pM;fSIlABcJQLymt2X zZ|6^UUA^~wbTlIRdmnoJ_g?=~etY;tiIY+f+r7uA1y`tU*VS=hpH*gN<1;N|m$dc2zo%G+whM)X1$FpyG|i`O>qzW1`1 z+0?HIDpIxX#u(&Za*isy!Q0v~s8y*zrycm-|2}#D*zw@~KCZF<`})7oO7rKx{&tw} zXUu`%jC;%8hB{Va-rU8&Q%7{GQ|4=glY89|r4}ML>DtS(>6H2Z2tlPa7cqT4Ngz86 z$%XHWa&mhj5N>;wsECm9-{qf1&&&>`9IYKQliSUJJXXdfg3&~~fnC_#(yIznR zh!nTfjwN&q_SfQ4QfoI9u9d~IuW!H2U#@s<%nKbn7B}_$(~7F+ou7=bGV=c6`uxPT zJYR%6Ag?McL<^JY1Z#5y;xO;QDSnO|9u-Ml0TD1BtAlW#I0%-*BX6F3^Fp)-$tBu_ zwMNoN?qeQ*^5Ng%(=FU0e(r!12<;= zAqb8OTg$>JOJX!(jvXG^G~uvnG%TE^Ridr)*glKwCMZ=cjg@UAeXadxP5;_kWv$tP zKlAXE$4qEJWvfM`(0pt8*=P5Td&14Nw0h8xN~m#kaCYiE(RyUK^1`!+86GfBB?OmAw*U=L+3Z7(TM|%KqU)6PBvf>b{;B*7QjK-1@a$S(8fw^c;T%#T$)MC?G$VT7xS) z3AyP;bf_5ByWqRVv@fye?$$;eR9M_nwirgX7pdDN48S@)abBCo5wkhVGm&u~RBM2*O2m?5QLsH{V!ON7S4u7-8h|P4O&C zsm_V2y6@UX(gp??J_9Y_X9DRCkK^#~687lSL^?~pd@p(2O^6C8mFaz?Tag??CB|_Y z!zMm<5TV_zrZ!p!YaMXNtBP4$#hj->tmQ#gU7c<598!P2%+i4=`;DCg=M2YOp|N$- zcYUY?i1zV9YNJNuNa;PS8HcsZwyw5M7PF?~u;!C6-;SK9_J!y?Y`*3_rRFDe<5`yp zv`S7RZFcz(d{9;7yq#^!jpVfXyk~Kt6hT=PWmBKAG{6sHOP0#)xfrK5;&xGahdR;d zW;dNcufHxRQ4+nQ<*n8^hW*7nFftef2?$ABs zIRLI8KmnF2|2hb`;W4`GiO~;p?8%&r#{{WlGA8`_J5_A;jhRf!{u>KOiIyc%$NEIW zLectHoYqWtc;ifOBMrYfOE6Jof&C^XTN`ToBeefnF?ws9sX-BpCLpH0?3_1Um)2)` z9hRjO32keRmcI?D*6O!=qog%Q57md?jz|{p3HAT;$;;)%b?4*P-|nhB_gnps5j<~S z-UIi&$`vRdy;!}5K1_{;5Tpw0w%0i`YB0Y(qg8)y`_ElC8y*xc zx*%_PSaLPWm3D%giX%n-B6ly7Jq9JinR{J^iTq{o9@A-}+1ibMDzE zU9(fCP96X7m1FA9Z7nG{f=)H(_ikjubHlyN4RasfFC;3gyDVerTOa9n>~AoFmT(`W z?sC(eeTa`nyJ*%jen*8vDF7k#S}@cKP5J^H+yjj zoiM?+3H$A13{kp$>iRz3+g)){iV?PGBs)4aFI^Z`16m)`mpGG|JFs<-5wP}bcDyLL zjkFyk+FS-Md_5M{;tql1F(r2we)?NNTd$o!RvPJ?#RaadkX)Ch9u2LUD=>xU)o1iN zBEVNVqF2wqsz4qIKi_Xxvc{A9pEtvy<9rG~*5t40P43QXF_>U9$_OQ!!{8JQwK6?z zqoJ_lVN~TW$!fNfIopvChc_altH$K_RNbBdrasqklz95tX?N;6s2v_kB`m@a!tyb?P3>$nv@0jkg=DC<>kb?$mL zueIOIGX}z7t}}x_rnEii?iisJ{5fXhLYdU&V#}6x5(yNp>x3G8*~1}pYUXB+G!Krj zxVg6EP97>=_PLu_t6*9tejUHk8*o>oww5=thX6?dbPBALWSMUh=F^OMW&SvdnGdx? zodV=F2H{~2@SUglHQUcv+`7uNz7>r_{xTR?#L=L3#M!O%7>gHH332Fi#sF0}+{m*C z6jpDR6{5=jkE1sYYx>;t|9{ua{EyRG$!Tq+3M5l)OI-k|%9@aAt#T?#s#H<7R8f&N ztO*d3%o(fHBBU*9lmO9EKxC68gbA!H-_|8^dE`H1TZ;r`y= z&*%Mqz1p}d4dEc%A%p27O6`%*?v045tkGD9rV9Q0^iWW>@}45afQ)iTK1=crq!5jeU@s;Gl;I@psP8Tg)w ziAg$7qOu)B+}tIU6gpipwqapTX&zt516qrUcnWX*1X?v4o!Dg zlV)8*IsOEWFbkPz4+b%2Gzjt$MpQ8=P&b4`atsW&4t4|+yMmhufWT{v#Mm!kaQfE- zp{&q3cf<{_d;R+wS6p9xBXc~Z0#a*|**L6mE-tmA@D8Ab!!HX0Ks;Gq%W1_46m!Y$Y(L@I6JTKsFJdH6zF zH`jJUXFXIH|$fs zs6jM0*gDmhz(4*#YKUB&fNEs8?EF-C-|<^5&Fo)OlSyw^Kd-EI(BKpD!b)d9{U}uO z>3*N*ZI@081I`R6ui5A4%*t!WHo%A?h>98WJl%8PT1p=HiaGiI4xs0zD8+pLxHO0`S~y zv3u>7`^_c@^X**4#g49IA3YT1Nsoz*5(N3QUIG*k4|<;DNQ-6YbVk<9LVx}&ee{YS;iCN1pQnCkzngQ5nNUn033?N_ z{P$X%Nc6nylr&*RP$Ss?(cz(FMU4#Fwzb`GC>RJS^H2ZDe1bF>j{=rKJV6U{@Cd4 zM;?JIy{LA4q;qJAL9~#e`k$z1<-;Vc8LU^*x!dt5!jz_c`)3IECC&u^xkT{y-LXq8 z;D{F%7Q|xwQl+l5G?*|dw42b|4;B0%kD=My(5+vN@!S7tog1I)Naphk>};gfIaosE z2^iqI=KxAX^FgqlB<}^U)a$cWsz8f(6139y4!8fF&2wXJ`-5L>tmktFP|TT)~k%Xer&JO_8=H zGX5zHas3kCorH=vC>{*PxRxb1gSmwh(soOIa1DahU&xm&VCRu(5@v@S9upwD7}QpF z41%ykl1i?+YqRqB?G6rvus)ISn0Cddz{W=Jt4ORyteIGusMXq63fIlca-p-Vo>F)_?q0VK{5)#EUKB9PNIrfUCb0nb8CMX9wg5E=Iv-p|*wisb12;}ZUxk?+k zuMe}Q$j(0}KyO4_**MX^AZ82Y2$k95{2yG=z{xC^om;q+oV4@`MH2Y{q9rD4NbV5q z)^b*5l;S6mfYBdJlH*+es*l2DW@aShN$H*eT2Q1iczKn9=co=vbO%5UTi`tgpe#-V zBi05^h)H$`X`2tW!<0*&taOyxj^r#Pc73dFB_rK`QY3x&g6#L(wZ>Ld zT$}%+D7!kcUt553;nBScNgNO8tjeV353>sAqq2`zl#VLAy58D+$h zSBwobV0Ks5I^K9e9bHJ%P6)|ZM_OeNbyLipApOC6s&`E5^2#a2gOo6$6e~~6y<+A! za$Rb9zoLn2UZWKK1v+0e8f#TcAq`T(C9Fcrv0B#$5=cQF==9zSzGu>f+|IrXNHq@_+jLqf__dnh(VIzgUJ(E4t%Ai|^nj5#mEWYo}nf%Ha}g%JQ4n{r)@Je zHG*=1b=*8R-k}JV7Ye{9B$w3+IbB0Tg)b#?aX1(HNY!zJIyc=t?p+q|bDH_k^aHv3 zI`i!?{>JS~$Le;y7g>9$sMp(K_5hdR2^y(R9fy(_PLLFEAaVrZZ`d+%*=F-I_d;{UiHyTNXLOO)P&DcNjN zXNf|vU)l!02Vv3#ROF%0_@CbUul;>(b8$+qoUNe+6Is%0UM%y1(Ti*i#MK8Cv=?Pp zvc{w)?fzBvhkHAZkN2(p@BWYe!#s1hqcyguySBn3kDi}?szUTIYfpUmjh?@T#=r_- z{C9PVbEu_)kG9^30Icwd1yJU}+K*{jYEF0-%P<3Ebk+d({+nSOx^qPLR)Jk6=e!(i zbgi&KG3L!qOG1f0mN;{#%M;wt#8_;Pd~26%>m4#4O2E#|wIYKdy8p-4C)JBjAVZ1-=7`O06vF7(D=qiN4@?qp{*3YNvWfYSqE2tpptd3Y zrY8(p6Va8o2pg{@T-cb|P4U3Cvp5M|p;G<8m+%`oVszf2gX+ItL?ImI~$b7gJw zb#wp7VfG}oU$5F7AK?BC5-O70J3!Q#QljF+*9T#(8%l!>LUKYtjcG0xPZ!z%uUZWr zpUCP7WxXx{B>IY+9pKOI0L=N6f^UuG=(OQb`+TH?pd(QHWqI~#7^omQXT8{?;p1d9 zc>d)ozsG6YKq9D-r454@v}mO@>L>2S3YLyjw<{$c3XEP`<7&2Q7e6TnAj!4sO{0~* zd%Pf1I+t|OhlJhyG61hBLl_r@a>q-MKR$^m{L7Cox{Byug}JucD}0dpD6YdR(!Pq; z4x&{#WVaY}CEu@KBfGcBFw@J@>ImXGnP|WTRS1HES0@UGq++jlLze#YcJn?mUJ`S& z%esT~SELNkcH0emF!`yLsSM5}4zv_7H?9=p5)d+Evo_MpN{!vg<&(goRhke z3H=U?%rwAtYmiLCN9e^F5W_LQDV6sUYRF+6EmgOXYmwNNtXEHTh8eTx zkJdJD1{Ss16NFNrUPx+1K$ZP~S0sH>*@()Zf$hmm2}jD$=wPw~sR1J;=^YDqe!d>A z$=L^FupbNe=N!Ixz4=P->Wj?e{}6Ph zpO;f=6o$Ed*E$q2jJ2UsV=yw!FmS?D<%Nxw755aY9|!oNp-7jP%x``y0_d{wZCO&>Wi<)Y|sqKo^Z$^7ZNP-Upx#xT>I;fg9Au zhw1-RyKF+8RGe5rfrFBC@p4pBziuNftk|&|>VB78tZ06>!xXo+b9bgtbhpW%J+EstR!ylDNM8YOs!XTd!i&vUF|ubU#3WClgh%B}_#MmSe*c zzl2~Xsu#fS#YsKbv_1@^*(=KNaS#CuZ>s=G?Pq`$H#)pqe4Vw}pk_rR0XF^O)L>#! zPeh0ugS~)Y5{iRpNJS4=P<{oe$u&Ku6c49QBp3jiIUVM3!OWxF&AdcmxgtBmFb}vh z`GlckmjX<2!9^)>Qm_4SaOjn~*6FSVy(zcf{LBzf%G7<9r0)~sO7XRUS-ZOy=nSWJ zA*U5o{9^cMe@HFXRaI(zKY)$6pChfah3gVCKU7kuOSr8Voto1*gU7&ZF(4P#Wf%I7 zSnXRwR1h@mL~MPA+Y_KXMOv@Wq5B>7aV-K)>Q~opMLIz#;+@2O(bbP12{X9^ySzMW zOA5^ammv5i1#xDpp)|GQ2?@9OL(n)s^}PwG?I$&?*me<&UK`(2EKWGLCuG~dP|@py zNm=&!-(Mt>faS;5fMJmI30x)!-?@c+KwE=XXO;ZP0}WHOdizNw;5XF;VS853$5#Nl zb&SR0&t7U~uUo*&5+EV!b>Lk_n2@)X3TmPxw*~cNVpM98J^-zT!@atC81~ zJhhllz-7z={=oe1mA+!3Y%7PW+)Y)Zt*r*swo|n!yJ~MR|JYbwT2R~?EDO7QmRw|P z@^w0Egf%5Aby6qI_AAVF4{g(zat03jMQ6ac*7>*D>&c!+)AfuJHG{Zm5;}Ntm({Fn z0y4j3;>LM0M{nC9eaS?Swty7gHm2tzwZmLMy$e(xy{|#Jwkjn1`8=ib?c(G7_g^YkZRVZ!YjDOHY|7k z8aLq|RPpf4=)I|-p!YIIs^jXf^qwpFg?Z?+=9@Q9EaxJ8+QetRdr>U>nAI zWB#w9n;F=G^oW?{$IW!e-~4ZQ#g?jh&Sx`Ivtc_2|L&I;kxaKgf?pLBgebY-O^Oo>LVk$GtBT zNkoqYthl_foa&5}3sH!;wxJxveZ^l5rjUipzwLVW!%OJ(xzln}L2}(#@aVltr(WIX z1&iP=p7#<;;4I}Q;`qiOOz~f)vR0RIS2z>e+_!U8cbHr>P>}e=nO#jf*4jwgV{_vt zA2sJx1%Se<;g~PEMzZi&V2e91A4>3nCWXP_^HK!tQ2pVov;!AvKL31&e@ybnji#eF zZByF%!HnjEbEBzjsIb|apWr8KuA7~aG$cTZvQ4J<1LEkr( ze|G;ic73G!^Tr$L^en8mf9ch+x{J^wjZPl6@Imt{^}#Bz0bnpve51#ehRYex_u#%d zk2Pey{r-vD8nZ&pX~-=N5_gqE=!MU-3`=&>0``5@ttgzxxY=u_AXDF#&!3plCcr9zSEj&wJU2xOM;#%=QG7tj_l5Xu@MuY+| zat?M<=R$S%NT9Mtl>n@R$0L%9;{L9>G1`Zvc?adyXx$BS54`0Pvvq9-AOzzqdVy(x za~4(|&c&6eXEfmF-HloKB!H8X%5DF3pK@Z`>7UWDG4YNw7((L4<}%p&%c2et(7~gZV^byy$i2@QcwcNCChLRwQk_ z4=AF-RlOk`R>iUFO~2udnSQKj0+6HGU{w=nMF z$yo!KI137dp#P~yxi-mw8VocOrRcnyU`()r8<)JLKC$(HYiql9$pYE>3Phh-T)P+7 zcns+~3!o}dQ1QaSSD=g_;^r5+HngT-S&5E-y057Z@-X|?Kv3>YPfwWRU7<9u(ablQ z+H(~I%1|q%@cXk0fPXnwPAPd}#W*K{$LlP{DHBs08;N5;lyOg3a<3Kb{?*h(UzpBM z0?t@2SYvGU3X8ge`>X>XyT?13WI88*gXb>pNYeSC8<#uL84*h#$=8NSZ^Pe_q+Ibd z#wkGCo09o$Xn($f?}@UB`3TUlm9q2owOyWlZD%v#c}|sGD-{E%gecx;L&u&`z;{-`>7R}j= zBn_bvR#Cc=&oI}K%T~>!8{5a*AUgb+jp})>QN5t)$#DmgnJOg}3U7HNNNyTQeemUe zd-U+9cYj#BRew{{uy1;~uBlZJ_Hon`m&vJ@)hWr%ueZ%q6wuolqWC z_7(l7b^G36Q;eA3W1ITOR=iYIzVLfYIY;c>_|q5hUWT?Iy#lOYtKmiKGjl_8r;+lF z6sn;-Ibb=1b%dj~A0F4&(>WdLA!n!{$y!(V83{ZBp?Kzu1UHow;y>7O1 zR}xvVIJpQ`b%^_?>mPhjTR?v_Wl2PjjGIgHn!UL<6@5E=!U@xJbE@2>20=QszsRN> zG&gBQUSxEhrs){iAv{QmY+ksARZXpfG**1lQlPN0{>*=Lg;BnqrGi(+vcU{lO;`^h zBb~T3uLQzaS}f&J3pJLP-5cjGye#1~aDA=?v-%tJR`v&lE!Ae{5yXtsX zF}k(2TYnxLdPcw56t!b7_SbZ3!))%Y%ud6tw~W018@rHzMMXN>E1BOwL8|m}hfjz2w2HSjpMQ~tK68{Rh@;-@x)M98N}u@qAaky}>NLOx z5o!y!%jCy(H`TmnI>*x;^3BJTqTxn_M#bGehKxcf(`S2G>_RXvIK~Eeax!8D{sXLfis-YA=6f`2Q_OE>MP};9aRTPjHibv4s z5ZvmP@t%V0UNL}q>%t;ICNW{016$dH2hc3IjSgBIFOar7PQ0saf+e*0*mlX3^NaPW z*RXb)%lPj#{*fn(=mFuo1Ene4*P}^w*6lw%SxPc#6I8r|I@e`yoIr9|8(zC@3}-|G ze$CVPByD}tR)rWt%^ZW3DEzuK)=Dr{3N9)OE?SO)Z_S(aa(62dL>3F1$^oWQ(ZzM$ zmOE86#|mlN&K1iRh$7HriLk^r{KHx+}?DK-*-X zdYuPRT*I7U^rXD-?I$IQ!3Nn=;43Q6adh=1CMjPG)^dt^EdOx7_O`j7MX0{jd@SWA zNZ=sMHhgX2_v7@hE$MROkJhXB%`)Wr*d?}Rg4{;9gZvTX~#U(@O#YI?gBsG~5X^B_?+8K@9ewzd+ysk`{#t)T*+Zsi>47A+U zOwWJ~?gEuN+1zhymgn={qJ97z$F)GP$Sgm3a!*xG)P{5E%+TY)K9gDJs%;0IP}h8< zd@~PeNk^J%~bb`?QaQBua1*or1qS$F^%j9ab9HAV;Xoz6h5J&@EoA>X?N&1ij3j6>i$B zO9weYYT#LQCBOZh(OP9<5LjQ)z0D_Y_xrz34_W^?_j=surPqnq54PQDYp(w$_TX&2 zS5xw#NOEOoVCX;UYsym-&|rp>qa5oWYCrS+1efA0wb zcf9H^)|F+uPSm`TxlLm;!Riv&B!wSJH}l6Z|1UKfgOap2??=?a8|{lsak4tv-jr7#?^yHpEYG#ID6<}C&~Tzskrwg6#nr_^dHj zU!aRYYj~{9-8{6o9pw1W-a8265tC-fWyPbS?Bh^qTWq9e zGyCJYcLLQne|D{9(-{mnJnmeb=Igv~N{1hKsh`mUEf5F}q!2yGSnF*{D`yWrR1g*sx&1N6!&c5;U_ZuGp z&5b&B>?d|!WOq$g|4sY$Nl+MJ02~mba5R>it({}lcDkwL3+lb)Q6Spha|pqdv+PmC=5;2bYRj z-8Rbxqr&6EFR%dp(sn6zsu2FMZ)!ef##80X`$8z*m7PSwcFt z0-Wn|=L&kHSR#jC9qGX@I<8G(1ol{hAlAJ?vxx=iIDbhnLfN2R@2@4E&BOyCiHNB< zn1bp59E1N_mMt}(fMlZF;+>7rw8D8oN)|zMN6S&ENO|)Ong!;U$iHZ=v!T81oVoA} z_=;|H-E-^p9o0p7V8xJTCuyI>p&d+z(O=!se#<4ixvDnOlf_3A$CPEITEtpCDnhd( z7>dthJ1eBIg_QzU)?tk4l@}?0HLJ%}kG6b;vNr24Cb_b|=#+(Ak0{u&V5RM99q6Q%l^F$4zkYy=mrAKeTRXM2E zF4z3Mnn>X7uHC|s(PIqIt_ldEne3x0vPCPy`3BcY2Yvweba*XReweF|$^>)0GEF*$ zk3Eiix3ZhVY(l-t`7mp}~a^Ur+}y;+ahGAes} zw_6E|hF=~nUx2x%*3XkeU4O$6H+1f6?v3}kpb*ta+As&WBvJ;V;Do`^BhEVP-9Kq( zPkV8vsSZ(qQR~>r`%!x$U zKEp-V_O8mL&PJE%0DAo@zz*X@EKv9Y%YovDRH72^&Tw<|FrcJ42hrK}T#JBa%xT}w zB|9c{*4wTDrM^;l#8#e%tHD16F{?7XrG|M`|CGYi0{iYw`}HqM_ZiB=kP+e7YbRMd z@R7}_h>GT%m+=9!B2?%f=EzID-rSKVcYaAIOMCFaC!gK;-Q&v8k?A)d4f@5!G#f6g zisrQ}W;9^n8Vkam(J-%M+%F<#jvdx0EI$U?+s9%uoeJ^OhIu+*K{VvLZCgkSJfSe zc#oHR8K%|gqW3n8B?8Y+1merk_zywbdVC+zYgKl~?yoa!~ZJslam(0D- zO!Coc6MOSt;!@q4ORn`LDt_zo__U4M zW>i#|;}9gh>o&|?i$N3#hX0+0-;UCP)I*Fx!O!Ynbi_dJepc!^4rb=;`kCpJXc>M& zYM$}t9Tf!KEX7CKfhoIixO79#h(F_A+Kln}4#~BFO^|M#+=GGC^y4 z8`AJxowT$hOKi@WmnBq(vr7B%Squwql%HIBwk4(MnQxnEMaUJRdtzp|Y02-!Hm}o()pNEjZ)v~u> zYkMJF3z7s@JEd@#&zXs0&opzChcjy=9PmQD)6)zyzM;BZ7#O!d+uRmz;g>HkcdGMN z?_P-u`5bNQ}3zl7#EEbw~8KNv!GZN9KekU@(735oysKy5*h2>2T+4k<6 zFgLRxo{-$<-rn1eN_C}ciXXirQr#%+vAm3}f+3+O=jhhwI`ea1IgwJq;5z<2$!eNc ziH{MlDuUhFJq60ATrAu4blZeRT808hi8VRKZs2z7Q-37a*6WBpuXiO_HIJR09kkD@j{p z1;NXYxtj{^QFO+`ap7Nh`jxk1qwOu1oQZ^t@$NqXZbd*s|i;fMw&D+2ml z+b{C%VI*leDj;_+*Wx?s$yynhUr~^Bl6L1Ud}VOWMAlySCM7K*G)Z+X8t{2>@RcP% zl3FaMg2_2%!({^bl?)-xp@n7o11RTeMxD&H+;~}Qv%Muc?X*a7irbie6Zm)}a-{wh zr0CB@*WzucW5Y~6d zEL8)Q(bLnQnqroPftEee_7U{gQ3KgM%A&dMM|E`;IeG1vmfEc!X{4FM6QspexAs7U zA%?gqCUp)tmg)SKVhx=SbyLXVP;SaeQotdP4bJ0CjDLkL!~dp8rd-zpsYAExdisy&e-qks<>Jxll0amRV*sGN1I6&1rR8hs9A>Ee@)(%;F@l1B zuiRTzI%}Jnw1M26R^w;!-vzooGFdN9l?^P901hmbmesLRr+RCrPW^wQJ-b;eN4tND zI}=+tR} zPFk0tf2e=Taxyl~&rcuHWBLa*_r$ro-yb25NQ6iyY@9n5#Q_1}9M{Y}>u!J*A}KuG z4lTDxLa-$E+YjRW+Dw9gOW1L5#Nu$WHo#w!liw|b=1&(05lekYhmGFQni zV>@kgo3E=5Gh;D^k(ieZ{Pr59aM?~Y^0~&7fY#5P=jt{x=8(p#5CYIwGFJw7LW<^Z zV@m$=GqjO555W5AkCDpHl9p`LHZP$4Z`-L6GlkR1#%3jtVdJ8GgQ$hBc{y=e>A5k$LbaVbt25t`JWf4oOXuz0l- zzN^OD7E1fahSBbt@V+RiUX6k)Ak2IVg1#~ukwMIQgCSWR>D=-1Om-MfzV2B6!ETiO zOFwwc){K+_ndE>E?g-5Fb8rB8E(O|`qf$xd4(a;r7bf?6?np_xW{|`*jh>Xsc%h)fP_x^oDYUqGt${bHcoC=Vk}q-a1S0t;n(%MeUt2_Bm*a0 zCtCwq=vnl$xv=Ti%==#eY;)Tb?pfp5rFM^nV@W+9Of6GPH|V zzFS)S6M$wu<~xUX%QiGJ3&ynxY3wYSKhQ3fv(l4q&w^D4lXn}T(X4>PW+JLYf>1xZ z|~L8zQx=zpYT6Z$5Xl*gUYA76L#5wd<7huvdNGNHGnB4`0<0zsgxJ` z7W&n!FR2AzCdvi&!E19EjQ>sc9>bUM`On6#dRKr-XGSR0+h;?ogK1!iqIey8A2=F* zVkN9(&y2i06d!+b*Y_nQDeC$0d#wUN`Z&0VY%QCWU}FeWXelWv&DcQtN-i95oTk({ z`%148^1H)8<**Lix7&oEXd{us%#b0VG%Jp4y)=D1^mw%5dgqNZF{5A5>n#>wqk({^ zh9*fQ4xdMo^cw_EZFm9{>JQepS07iM$$!6oprds#Be|!G0Djuxpa1(^^7QdtF+rtg z9{h5haWx@$@oQldc7I$&a{4*dp^Ha;{|3NCLd#CImc=Qb5nDSeK5i)(dKZ;B614v2 zU(Fv)Kj%exYuDd=p#JxRqrFqzxo?Ul^gcBq=cA#`VsW7QB5UKtM#kCR`o@kT1f56c ze=^q`boTh@d9c;E3_o7c2Br-Grp|V7kKKLt!MUvmFu(5M#ZFm#c!0OxQpK~ zaJsqSQb~!nr>&m|kdq#>A!rG-r;B;aW2GuS%SA`K=zO3dF?IDCDp+P51spevpMYk4JsYnp*{Yn75`@}(x~K=9!PDWYx601JsPYOHuh8`xysu^38TZBO1^emY?oQ; z!$}q>TWZwp>Gj$9N%v&5)5d6Ran*K)sE3w>pJ27iz>)5Lt(D;pX>tW7Bjje8F z9pg6VfV8g@q?dFFVe@u{+(`$CNp7*R4D@BFFw-h&Gk#UuS2%jIUCdUbs&gy_Nv=VF zHo>)FEq^=5fUj4jW1|yrqx$eHu1-&~8o`o35^jDK>=vNq_*5K|l^}N-xn3h%D@mq~ zxR#Wia)L!#4}bz#)m)>N#(f%3!-76jL8=T>c*pmW+H|jy{*r<821m?zA{=P2HbGgk z`|4N+xI-?ky;=K#L}Dg*Oc!q+%6s>G-Glvi4oC_nCw|wZseXP@-!!x5VOEmu3c63^ z2Yg#4$$1g+w2QapM((N>EBUeGw@L z%Ig_!85%k*6m}++KMM)F>`&o^oGopheVL>Sh~*WF!U2Xf?eqVbDrrdO11v}o877x4 zh~PxW_PykI0Dz);bzOlue;*6BwcXVOt-Z-+G9nV{-@fjbCirY`IxZv78itf5Y&O2i zZI=M;?6X!B0OhCPfe^mZ7gZZVR4&gcuk}n$q!gs5vo4*3&P~6XsW5v4Kz)2F78=##Biyq#%;FI{b!GF?6yhZav9p}Wdq? z;(`w&OTT%p>svU>C##p6LI|DJaHubKe*S6meQ2yY@K}6a5Guy2vHF7IdC-HL`kc+j zH>w`AQazNx7;cN>r}_v1y)eyyYJ*h;Ugm(QLu1=f*udRQ9dm5v*3u= z3pGsh(3LT$>rEube3(041B~n7o>PV*t+xu{Yoe&psADU~-y;`~@pS#_Ilj|`M%V_E zikGGqx>f=0%UX)WFZq#6W>o3XZ%JACWeda8zI>DIJej!J6vIbVhnj%*@d6iI0I`KH zi6GK2M!;1gaISu|3$z?>2vpzWTk6%u)HP~}OFC#)Pb(e*X(Q#6@?Un4Ub6A(Lwqfa zsA(S=zmyuN#wKa#N!D?(4vbC-Ot7NaNE8v4cGtSrD1;8V%_^6mq5k~3-6>i9rc}@a z`to*c$wLZh6^G|13v0fS?gB7k{p~q9=ic2C~5N8vS05o$c)yZkL!l8QyYkx{bw+mu;^V zPt#a!F@`lMv@^>-G&8lD*8}l_TO`)b^PynHg9C59ytONF&7*nkw zp7!jbV7xDT@pbf=>rIM#t4a+)!F!OT%^6>b9iUE48w%IYRM-=7cvrkGA!2>9eG@RT zjTArtDdaJ>-$YTgA$U0fw9E+YuG5&AS!)xpojX$*E)C7$JQ17-uH@Dq7jClkkKU?p zEO+u&$^MBYYdmP4!UQ2m$@eG?aor$Z)^MZ+LU^8x)8-3)PQ!XnD zyoUkrI$Sd+))lX>e~Y>-#m6!YO*?Q>?1OXc65Hj2U40CkYoKFoG*`Fw1IBWTDIcdL zYHQG6fjA!}lIPN(6ImD{&#~u_-9?^|64mmr>1@OZ~r(m6ZiYv z{+k5p<4pg)UDsy)<`-nW<}u}PWm$S#IB~1WIy~*gpU5b?s<>Q}ITr>K@{*Ki0?q8S zQ|Ej3gaeXx!m%43YH%(}(7HBCi*VHR{FBCvtlICsQd|4Ob24NWry1m2%u0QYKoRcc zOU?w+IXYV!pp<6J?Fc*<`W}FR_yd|&KcJG7+wsk*djn@?Wye#nHj$zc6KeW1GNPq_ zIZBpq6U6|iuuU`#*rR#8;oZI{Z+U1>7b1Q}^zh4MWA8qXC_&b<+4WmMi_kW-ve?ZC zZlff%vXd&l+FQjRjyWAQH@(~ulw1AitrgfOIwsyPRuBOX>&sUGB^e+L)=SxCqqDD$ zzBj&Rx0d8}H4k2T`d#XiPXxAoi>Zk2phrJc?F;$&-AA3YQ}P21adqmngBNyvljkKm zdbW9}ul#V+i#wlxx8%)+s$W$R`o5ALTHjak+pEwp>Au<@qspsxF&{kd;?#!B)i4{7 z1;2;qbk@S5eB4Mz1F_*k@}CLhBy#AveUTLv4YP;%lg$NJ{H{O;dMiwEqZg@FHHqEH z8_;;448T{%eV!vI4@Q`DF73Se9wc+IoHW47n797Ze=P0(?r17*8 zi`m@a3s)UXM8Dd>vQw$z#x?{|SYr{~yur@uYKuxi>M*^k22fUi%glajPhXA&6OB;Z z>H;2&j0pNQ2isH2G-;hC?I?S!4!m9`ZM-7)>$ABUO*FZ9zK&$T5lIm=Ft6Y)@1||e zfJ9|irQ#)CB`KztEGcM5&uYFUyX|(MM$OZl;FEQ5fCw!7^8m*CQlI9a1q$6jZoSV) zz)9&|RBBfn3EWhr8Y}#9lKx@Rq8|KReFaAB--2yxI$P$|v$BYGs*nj`q^5H>@PV`n z(W^;f{E821fut7_hvn`;UmBNSS`q@?c)qTd3+tC6mVf2Lu+z3Ca(KUTFi0msl*|T` z241yNK!BIuFVn}nmzGAkua^G1!!9mzeLES%G`+JTicM0Kj9@lu38%Hy*V$dfnUf5# z|MiEPPprp~lwaepK&p6>Ow5b}g#VCCZ+~zgF3X)(84nzmzr-$VE0BGKiqRuOyPOMT zm|tzj6I4P{LOIz^-?>t;)4mFUIp1YSY=5w+(C|l(kw)98)_Pg6f55mu^uB_FR{P1` z$ilUf`D)NR7#3p)wvJ?9037&+D<#>5t~Y?+)a``{Edu71!ZlvjufInn0eY#`X!3{o+{yUyzs4t(CrnEVvotWR z*?KTrQ##rxyJ}1D;F-!mE2#bmmJ(=smk?aq`v)BrbeZ00`Th)lL_Sm)I@#+l@YeHb_1HeBDff-xz0t~f>Fm5FV zzHt>&kVX#!yId<(VE}<|BMmGdds*6EmXA+&d6z}onq+?njThwxsr#-L$!l1Q{$>3% zA8N$vjO%i@fJ>NQ+kpvKY6KM^+W;SFTY(<)pI{FhM&3G+bm$OCj|UJHee+J-xl&JY z$Wf{(oS*MFJM(2d-`bCS3wT>yVBP)L9?Ay@yl*wq5RsR^BrkuF17d%5In{AFJsG>N zzxu55$E#JM2j4av6MssN`_CiQskcD~@bKhp-2U%w^<1%Ed6GF)QWu|LGWca?Qd*Ox zC0!XAmacQQ+L=Bz_4wz_Z7J#L(MH09PjBr3RMC6Cm#-{}w5RL-^VEY2XU@wDy5epU ze82ciY3Q6!0_Lp@?O5NH8O}Ec&yO50Dmv~j1O$v*^s2iTD^7~p_@ex8Yx~MUu(+oH zlwLHI@wE+GqITo+X<+DH7aap7m%XyyH&17mM?sAJ$@Kroa-{YfY5VHFV_n zVvXsW-B;3t*B;*ZI5chFC1yff#cyZNm0xXj9yAYi#`c{kcqqziyEt^{@b}-RKe!^a z(DM^B#_fXEke+ED@xjLEu^tA%6SnD^>btKa0&iR)hs90C?Trm>2?-m!I=uA{-U~0G zbnCw=JlNdew)kj2N;9@w?G+sX7lPfP{uQ}MMPaUL#Q_q$c4!CU@?tuQ)^B>P;hGg( zGe>7Gj#0{;!*Vp_kqVZ^BPuoAj`yta@UI7l-+VHrSG(`d(Cns= ztruhOd&s@abSCqPJ@=Yh2RF!h<@tWwRr>487zD}MCGS!(d0j8F#nPFq!_mnznk9i96c<1y^~jqgD* zRLVV08i8r$&H>&{Kb$a93D$rCFe|7r<{D1k<;~I@3p9ia2IYdcJrG@8%}c~hgTTcY zCUeXlOR53*6(N3wnpOO#9I5@bw3JtdFrS|&oRZ2-PNe-%`&KRpk~A~|j-Yx%vboK* zi(`xFlu(kZ_Cf=9yNr+AScF&sXVgh^chLF@Rgw%&j@hveViEx-RtC%PH;Bi%{!rUU zI75=Z5kPXi87oQAm3aA|3r23uC&P{*-7_XH&g6yi}Hh?Rx0iYTHG=)H?6f!BH1=0er78Qafoq<9!$4tEamOGZk zd^S(oi0(JYqCCfI;x!eGLBS-$1g-mWo4*9B2hg^Y9A^}1U4>#1a<2XV9KDNQ(rNzx z{kwPfJML^#Y|~86q{6RlTc<50E6*ak)08_|y5*GR0Y|OOBc4$axWD73(-=}yY36|` zN68G3JcGa*sd+%hP*9F3iU&?A0&?`ey8nO<4~6S-luPqZ^+PHe$K`aS=dO zNZluEt)u*QaqfFGp;#|#qTo-cr{X6WO2+KyI}8ro0fay~>0NJb?r1dK6ri+fMh^Qb zY+IUHqbnJ5X7)7b)5t{_^qF^$=2V(*+{QM)|JTdwqj5B$VEP{*cuo81t20%f`u;Wf z%r^rl=F4XZkFq|YhSz=-;nl?4y7nU7zcu~M-$Nb+#YFvms>Dz6RTtTOvFS43DmKUW z0P3Fq{R^Cm> zZifypP4$L+QGQ6Da8)0F6}$GV04P<)TS0lgi_w9A>$ipUJgpJRt>~L|vnf4!o;q+sIx|YpA$3 zR3L$Mr8yeaCzP-MJ{hU~{yO#D-%_6a?JvK(HTU1HG8X;y zw4JYN(y3?nK=Ry`@Bd#}_*_ds%-OoNT(YAwH0XL)R?{`AWNPnm^3q5w4Hx+H0h!VX z5)jVzFvmR1IX5-61+d7gtt9Nw1^>p`Ih2g`#HoUL_X{QXm6J}VenJlgkIHlYG~t!J zbgWASfJffM^+GA*IcuzmpK3}rT(Q6$*yjjeo`}O(F$6$|+WeMV`?3*SO!jhSf5c)Y zGqcFG7{{rXc`bRNe3Zf?v6HLr)i%jX}5YNt)LegEqfW9 z=eST;Id~_%hoFxJuzHZ{@%3!&P}72O3T;Y)gNSleQ4|}Xf5QSWH@f2um{7kg6v^{> zhhUb+T*nxfW9;@!!14>=Alwjm7t2~Uw~dYUkB2{OpdxJ@$ZJHk8`OG_yweOr6u#hq z?}-U`RFpN5b=9Xa5{2BcJ}fUX={kCxvVd5PyPSn;2<}}yC=-q($JaKLt_&5BRwNS_ zi$`ANP|fP?U!t#$qmP8Oz3Oojan2KpNbt~1M|-rk5D;2yCjgWsq&6y6m7inT;*Q3m zOtZCWSHg~(N}JFp#HfGQv$K@Hrs5Zj@g&0*<3;GG7y7sE{f=5d308D|v6lNR^H|!n zbv-)7bV+0)*&KN{7&)d97R(+<&b0wO*k30?9j4b0*Wg+*vS4ki4CegNz^ci@+V`PL zSBz*I9=*fjZij&~%_Q0(hGk}C0R5X?HW(fzX!kw?+fIaC4Mu|Z{l*~N8km`023GS4 z(4OJJz+q<4mLF4wc2&WYFcd|kbPyp3E zDjpP2&;jhRrz_DGMlgBmdmX|GQ0b#37Dzh2phJ->p0=Xv3)3Rl;%M=r4 z;H47Fv3ID*q8ArJA?XYmDv^XHH_f*JI&KR4u)({C!TCf)WQjZ7*tL^q4 zcRkLk%OvEr7FH&Ec2mEg{QUdQYv*wh9kM(ZUcg=D!@2(>`SH)MGOpiFKQdVR{d7xQ zgc4ur!~E_Oe)7<%6U@8GcSjcA9pJ4jJ#71qx4Y~4TH5NU@Mip$BgCzeMmFLU#(1qQ zOK8R+NrT7y_7|j2q0tj`1W*;8q(ry}a{|fW;Z3CuCS4(UNOU%D=GXr@$3Ie4t!6ux zv^|K8@4u|Ts+-DE&6=qag{>wRn2>!W`tj}ym0?B+>%P8^l;6nJvY-p4Asez3e$yu9I!$ppCh>JJkY--ln6wF~}s zQl$2)5|sFYJNf+lR7o59>~+S;@)&PZpKG zEEB5iTdjP7e^z4vdv5ZOHKdTK2)f)IS=e#y{y-m!S|jL%UWppJEJ1#um()#Y>636# zSTcytxnH{w^rUo_`Fm!45bte$Ri=e(MMYTS-{_LgxR!If%^ish zv8*1;$l~>3^`V9KMNqtP*Ju+SqqLV%Yl)f&HKiwiw?dKh)9H;rPu3aXlH^@xn2ZU>}@d@F^in!sW{sd=b;m4bx&M2fE3z)+Ji!`uK2gk2#%Wn13T6WK84hwl_uKhs5 z4quUEe$D1Og1H5X_!RKJNwBJ1kYN9&!>*r(_A~nKG5N_9E4ATRRDm>SSQf^r(IHqn z+gz-;_GI0~Tn{np6fGKW#Ng?vJFSK@cX8NRHDU|o>htk2s9PAjQC9|I&-RzI8X3%a zahCZ`TN$tLErcS?;|ZvGeOTQ=x+73(QG>0XjEfqDq6pk0DhG^U&VHd3sN)T zY3Ioz+1usljWwVeNDvjYE&Xw0eVDV@$F_k4T5KU)3k}BErLvTrHMn#B2Fh|Hyi;K_ zB01VSkwq1d5(&lW0MazFbC3^Ic$?#0AgjL5+as<7q+@0icIL?5OkKQu&C$5^kSjkD zX}-hzJ+&%xdKZhOe1{<;`YqOdk&d>dSk5vZtJF<^09wgSoW!F|jh2V@7W6M!0d=*| zYM;;5r`|{o1W4-DK`48>kBr1Z+3<`towq^YF3)&pL~NyREz?BKK!u^$W%S1FErc7%iW$F5$lTF?l#L1FBcQ zYz*Nr4(L|POY3^D765b->ut3li#Ucbt^aH`voY=Xl`?vu`yNr-jhmqxphQ}nsNXu& z4IU=XyvtwtK!Z_q_Uf!DX=9s|Wv2sZ8>! zb*0+IGns@L$;ru0bs{Ex1m&a3`m(Bi7opZCFV)z z^!Oi|5Eq~Wf>TvqCzJkKqf9)}B`l8vnwdvOAEjmpO4#R8%8JK>{Cbq%CAd}wMOR-N zPZOatGa3U6Uj0I)o@rG#@aKd#5sZ#J>oiy}L;}j{M?by(5Yx!YfGI9r9P_(b6-5J! zrYQUBHgtRmzp8tJ>`!>v-SLtmQOu_x}~~p6Q$pvRSpUn`v5GY3ckNC zel?YL@l}_xUChc~eGT57T;Y2J4-cukqQm&tY;#v;UOwM|U}6BN)tof2c3^NJ4hijIw~Y)F=X{ny6(3BGn8 zB|?+opBh-1Hn6<_zyEkc;o;u%t{~J=-RCqp@0ipvatDc2MbT z6TQ}J4cDWv&AMV+6A5~q!KFu5uv*2^jb+3)D2fa`m|?ai5Hnw@&37&g&J5^`kYHRd z3*&r!8>Qm(SIO0f6c#LZ4U55;Mao{N<5RE*jpP_NNccCWC8wMVlMZVca7e`W z$Q)QIQiS{VIFm| zdy!1rI?dBZaXAeS#$HDe*KK?0yhMKqVQoql8L(A)Qhl|hFmkiK2HEZV$YUm*)X=M@ z<8)O`n4QOwJGFG1Wmlx4ZnS=G0c?nZMyN%YG5`A-XI3m+S8y{4>HY>+r zJF?Wf6{{Dl+TbRQF_`gW&&lDXQy$bIPyp)fTe;xv*WVM@b#VCd%}Pgj95W!ITI%PA z(_V0^s0f2*PK&%_1LJagG2qP7tu{oRRD}WA;Z$N;96h?ZI$!4j%9F<^PcL|d%%rYw zW?!YB_5aoCV%6t zxz+++OI!ley$o}u3`W;nK?k*A_t%b6KgBF~e7Oe$tyX$^UMM8+1*QscQ8K*z_VWb+ zyTgD1VQD_M4>Pan+Xb+UdbmhkutzyCzxjP&1TiP=;$PnW{qrvsY`5yT2=_G zlKxfvluDZEFf%EpP82@8+(2e1yqS+bk1cp(q?0c|ZZ75Z8{umlJ;-N|Z{t`PoJ7K_M$Qjk9;C z!p)UiGvlpMo)HaVaj0KpOO(hRB+oxo#WN<;qML*q#2r&EIdZCsLn-1jg6-K#b`XwWX$>^n?o2 z^YTN19VDp0{-gq_x?d~s0QEh!T*A6m&oW5QV!(SR3#WhWoXcBISJ3xFgUfmlyCaSY zh?|W|xrhr7>O_M4q~+_eMD)4}Ev_ly?-sG&sW~5c!M56Cv$#r*Uw6)dYAlw!!y-6A zbx>srhD!@24dzA#g0PX)RXw%=lpTjrn!$vK7h`PMNs6^BoCWDyZE4uI0d#*4=Y|#z zqd;T=w{?irAP>q@i9s30_A8-ijS6TVBP4=27Xg^zVu=iEi7uUULo*P7mJ8Q_jh|o7){lVg-G-J*K{!^yKVoI++3=v{`;8k9DO}CbelcLqt(qu^ z*M4iNd)dzfmDX+m% z_FaKG8@%H*Rl3mhwLme}sl92==sH3AMFxjq3}8<4dg# z`!Jv#<4M?lBOw=o1-gB&Ck;_x6`zbGi1*R^RNJbTq1urid7i1!70U=4|1pfa$ct3-ul%w{lm%{c1d#9EP&>4+4{Czxa3n}3 zJ1V(&s~4#jvBogcotw9k{g8R3297O>+1{)A1S9k17+erm01ePIrIc(6Oeig)N@jmf zT%Bc5b_VjZ&0C=`!#zCPcsEueXe`}RntzI|xv&JmTUNx$o>`$eo4zQE3hCsr6!X4N zgM_=jWozsY>1~o*L`^|e>P50$Lr1QN>l7vrRy?3bbW7lhiV4z>VqTa!u`B_* z?MMyieL%Zxx_F&7w-*@^Rk1cd)lN9>)yZHHfG)EVOh_w0^nHLA6g^hh+qkEy-0$D& zV^TN?cv+U5meY~f*gkXEu`=EgvyUoLg9~)Pmq~pGH1*=%1LaE>qgww)swz*+aH&uW zS4QhKh=Ti$5(L;TEv0!yKe|RWSW1<%Bg*CbGO#fV2ei&%CAMdoCY-cJ{MKPzYV>;b74aY7ZZ zgq?Lg?@!-;`iCF>{0E;pexcv~;VO@Whs>^7a8PDxH9sDm76;sI+vPez7mek(s|+0a zpAsVt#1bp0CJrxCXo|qRwQZqkNTvFVYWmvqrs+RH#YLxHo{e;AlL4iduDQ~+23wWN zX9~L?r{*;w8K4Nhauc_Ttt}Wc&X%XT$KLcKJ@Bb4_SdI$(6%}NpTj_82Er?g=j65K z4!_uM{*(Ea-@Q^dRZ8VgGBXR^>I$Y9D|3(2+fo>=3{b<;i8Ohk={4Cdhe<)5%2dhh!P;R?CwwxYzr7={ zBT2n2T+7aFp=i=S2@7jJ8}Zdz6UuscLXOp~e7}F(G7 z(WM^AuwNv)!O25fo7j#rOfkt^w22Iv|2)C61lwG3BX9qQqRoJ}beDiLoCxACW|{M@ zsH9BfQQq`nH{%00y#*Z2rt+o3xkb~-Fy`XB2>RtQ3uBHC5wKmO!+Ksx$%T63$OA`k zzjh1(!ejkwHtBXRvKQmy?&0A*Dedi$#)mE#_w3B;Hx~4Tdg((GYd>$p+u>J@6Ha+4|T|&Ao8@=_aUl)e3 zPbgHJVHh;wVObfET80mpWfyh#Z@O`3SL<%}06x}UQt7xaM5x1Q-!PJ4+f!ugG#B^U z(t#nf4X6T3J0ST3g#ZMvqu}i(+a7fB?4b_`{7t@GOUx=>`LdwIjttpojVuuhoX%Fb z<0p9FMZ7_T-$@SLI1M=L&6O@aAq!6{`9JeE{Rv78**^W@9D)}3rgCj+?$MU><@4(d z67EiT$|PceL4r!G(+mLBXVj@{XYNlRRD%&7vW^lf*$b9`v(Rd#BDamZditdkaHpnk zIID_l0_KCbJV(^dP3;m>3I_LV+kQ`G_aMl#_F*u(V^*rma+6~54I2Zp&xAl=adS*0 zujzM2WU?%REebwx+Oth(mRJv}o zv9Ptq?bUs4#%ur&F5TP*K22OpWld>o&ACl`pY2LTSml|7l-guONWd(&v&!2lezfk! znF&^GEOOaP)q>#(_XOC>1Ve$sJV2Ut8Whfv8oVQHcN>cWZh74-e}BxYMX7g}eBy_*gMSgPushUZkwCN+&;M1n{tK;8K!nIb7#%*lH{XJ&gMo|7F<-@h+82@D~ z))oc&S|vN#V1V-H(c8c{Q2_l!+3YyPf)|ADK1_LbWq6=%iTRLf$w=TwbnP>5z$>B>i$(mKl?}?eWEA5>vEfuHc2*)rojtZj@}Hox+{nB6Ou zxJ7$O{yK91KYzUr-V%rnA;us18|!h!W(;1!Ir{X#-Z}AfOOAU*Viy;a`h`#sa5J3p zNtlbe($!4yp|qr?38t&V#@G(W|1Z)CBgT=uG7lww>GtL&Idj7GL&UjQGb?q7Wt`QLnHCsu`i*_rZwl;`E8SLVP*(-`iG*8*jr-=7{ zc-|!pQnR$$K?c>5B=k}-p8JpopEc2^KA-tPnw^*aKj+>CPLxchtfh?yd8yV|(yQ%( z$wC(S5&ukJx4=tvW9rO*ZdN9hW}ouV_T;!%jyC5BR*8U*L_H~r$6A@`Lgm}zvK)OM z($T;T#bwBiIW`Y-2)O#VmoN3A?niwcu7|}xncNRp6!maxeH_!-rFj2&_kOpqn!<$n z&r$Q^zLbdiSr8tQlp6s6rFqwKA*|Fvd|{A~OVo!F><1&U*4v&=#{)86ovBWh&V>{L zZaBc*aNOg$aU!$hM+j~)8D{&_ha(cHDOYS00aa3A!@clE&GM_ga1Nqq=DFuYoEgBb zmxQrs!f$0@kQ~mwp3ti`v8u^VJ@yWcMa(l?Xyk5wj~H&X^}Kev$esE_;*@YmYA;UXiAahnNRMC-0g-R2dM6n zx0R1rS#I9-vFvSiW+OAP)S95ch>&r!*qFee9?L_n8_e`aMTy86Txv3MHx5PCH`2h( zd4NjTxyuW#G0VGyFUaQdQg&8=vub7iJ#g(Tp{yXuHV8Gma>eLbTJTUA!?GR8399Cs zysaL>hMBva$K3&Z6hJF9Zvugs`6s&Vr%3FA5=c|KxJ;Eb9cq1CZ8Nr|3<|Td`VYr3^4a-(#T_tePMHfAaMiyA_{ve=40@W(I-=a#7j0f|Q9TyQ= zZXcy*Z+;ue%`Z()x!lqaHZ0iD>HLcuDypbE5^jITdB;)qz@0i&x-yZZBkWLgGV#>F zKOZvtS8t>%EYi8~x(=!%Zwysal~x~r0fPT`Am095_TPa%{@{dg4jX;G($hYqSjsQ5 zKXE~>rrLN@3*zJyt7Tp;TW;L^H8=}5it8Xc!IStBm@gkHH#*m{^)+}MiG&^oD}I2s zd_(j{?X1xaFNS-nxLR!gjNUHR%kzdkfHKZj$(^m_4!QMD0jVG^;4X4@BNk%R_PAH#-h6Lfth|)+y=6!=|MdFQc#Ie3qGE-LBny zNB`A$s=ohe7Ww{YXIAavsWbusalP%4M9)5TLD0VVuG!ch<=XJQPutw{7g2(eE$j2$ zHbF^kTbUnKlKpV3>2FECi{DfUzzcqF9HjOMI+K6X0AO27Q|&R(3&}byeW+{Co)*Kh z>FyO?CwBg;tG!0$tg-QEus@%v9aN1F=QgxNr6H@M+A=E!$Y==U1SJC3Jw85|lpI?3 z=s?$5q+Tmk1kbYavpZ3=D{V9L*-oikH#jr(>OxakM^Es>PxIx{uN-IZ(EeS{e{i|3 zEv)}%sP#o%-RWvfDa-%xNzi26oq3fsHZxVyN61b!udIL1j_b}hQEd|KC5J3*__xlEVlf+ZjQu3b3UcKx3pIe!V-V zekcb7D@zWPm+>4t9xo@Rt2a=LF*B>4x0} zQ@=i4nx8e1sZaukm-hHKw^d0BA&Pc`Qh1@T({ zn{I9fzILz_o6JBcG~v;Qm^4%T>N(;c53*`i=VRXfaQCgaP5;&5r;i>ppBPZ_%e1o! zOr4tpNGm%|O2is#?f%Ny*}GY1k)*lu{juzp5?L1sU>8bfh=8;TuV*=1TtJ!r_U3pk z6)fr|{334uJLmGls;J=z#P$s+R=bw-b0hBZz$~G4Q8RwC=-+}&o&SUjF&Ag?9s~Q zsqN>(Aw}-V%IP~%DG?x|#+4>G=g4xbIcf*&6k3G*n&R4;-uz)L!671G%<&k`nwquli4B{Fg*1`dw( z?Orb7^~3~7+pL1-2FN`35R7JE6X0A@^tEhW?(onrQGkY4*$0t$mu zUH<%TbfZJ|z}vxBLZCTm>hE{fKS!CZ^Aj>pq~rZwIwlPy%ryr$Y=i)*#kHo2|Ch8m z?(+JQTq91A7Bln}(Z%BhZn5zQ%^vdeCM*C8maJCSu%F6$nCdTi0$2}r?MXBf0EW@M zIUV~jq1wi>CTn{bV$D9EU7DwEPFTAElnN8bZE6C`)@YlkkL#bS*M&k#^b?dW2hE&&Dz-T@q|-WVT!v)Bk{#!I1}HJ)i|8>)u9~C6J|l?H(x@k z#J6o%mUZ=olQffw5^nZ(Q*Hp2Zmi)aS3^KIBDmXEG5gr~d6f8%Rf%(Z%x6hk@zLif zjJs;k++#e(I7;i!_I(l#WDfy#nUgjmk%@5c%z30P%PC&$K9>0yhI*Vktx|n`?b_tk z%;}to7v@*_nL;0R^2t*PsF+8yv*G|0Yc97BFf&YM@B0&e)t3Nsqmp>W>lO9WOSE$1 zCaC#rd6F!m8AQjY&%jjoDq}4gBjccL2i^0+%=# zPsl+rZGE$NmQ+z*bP~Nyi92Oq`_YS|6e^-tyJ>TM$ zCO`A8gLqbbCyYlzk=G)~kK^-l4R0od;GpfB3cDNs`GKiXXn5w~mLFS>|6q$b+Vv5x z_44PtYQDlTf1SBUt&Rx3kZxLRodX!I(R+eBo{!Eq|CH|KJNi;cJ5iU)*ei*Z^+$2< zS9@I{e2T%Z_u=H*)&#kVbhadn)Rs#m8G9Vnl9RY0_ z9wEPBq|&z_-KR8Eg zBw_8!tWcbGYJSk*IziEjVY=O8Gwbgw65QfJ#@NmZuQ8!Z-;PN1MXx`Uiltbdjp zk0tr32EXX*s$`Hy8qsfEf) zDlkl$RI{@6b)s6aqEc)R>~bJR76F5>6vz~|b(t*X?4u34HqwoT<%h+GgoKn>Jur0h zgo0H(mOLRXI(r%!y4^DO@;qsW*HnQhG~FH6)8Jkda5=gs@+yoVOGl+l0jt(uM>uO%`imm@|9mdC2OW$qyJQ;s3%zEv=E@PL!=j z09&vT1YdH@bJz*_Mkf|DFB@$-c6J6mH-{gP|l$HmyEs}V>1 zP#^`98%G6x-EywAjSG|+tI-5vG&-|)13V1?j`nE3{ixLXaM-b`-Z+~;4qly0%+Es|hOZIuooZC4mpF>LCm~}ke8fy@`tVVkf1oI$Ha06U#p$#ubV9S{I7p`HhAj7=|2Q}x=K=1 z%IwYqfRy^wckMOMt8Bg<(>}1Y*XUrTq)MvNoy(Bh-zp&`MF_(^V*!aKLiPsdoEs~X zZGtzHCxYWQ!YRWuCE$&JE&Hm6hr0$sy^t&k4hX^OUQeZgv;2U#m&*l%6AYR_=!$`Q zL!vXULew+&Y6F{_3UP|bWsqjD1AXIzVX!Jkn{dPfUkiFsKux|hpT_?A{)O(;tHCGV z45>0WE0GN$dmqM3i&mBoTQhE?-RP>?B^&A|H)Z8=3lSRslVc~=z8Kt_$@r1FJ3^CB ziS(`luPAO){5WRL^u@HmPDf72ftID?9v6;j6O%emoMW8$%RoR~-oHq~-#XpJ$8$Q} zYm%WfaX{*U(X?8yOn7*w+(*PaRh8*JtU4e3-P0G3gg?H0{!y^|e9)Ms+STAEc^>BS zpt@r?WYPCeo*UDXO3((RgBu(C;PO7Q{ccB%u1^V4VAKvG_|eFUc(pEwwW;P6@TFxM zXtAvK{{*n2)4)<^8G5N6of=qMY>fivYNf*tx(i*RIRC>0kif?kwPP}DZ=UJ1`c}m> zJ%G<>6pooFNpFzh7dNuW+yn0NhnE%x%Y>tRz{Mc%$jP!iycv$y)2p#XEq3GCiW~Qg`=YtbLalCu2dsi)0xgJ8mN# zR3xW28~OzXb3b{IpORpsQ)|H5X%BL*Nz-cf4gca z0dIig3(QYNnvBLD>lB(9yzVHzW+u)2x|h+n@ru6DLbBB%`*c>f$woHzfQuBdeGt;8 z>;`pD1FdXkEplg}CkrbUz%`Aee^+Ti;?5Uuj&`mc3f=fU)IM7g2Vk2V{ih9Fy)?n0 z6;x?6XpkylavgtdexN;$t`BC}dbv1hNk~B68Z%hW)TV+M2&~aQP1sg*v12{ObzpVu z>69rPe3`kiAu}sP3u`P?5&^gNW3xNu+mU-rNXr!m%*@Y7hGF?QQ&GapE^I>T}TVWX`LK!yVgm_xuVk`;*!o; zb<=D0q_HJ{TYU46tCHB*z@?Ichsip5k1dgGw;(OQSA3bbv}Q*v?;+dYP~v*^U_HCx z>Q*rWkf_Tx{Ls+q<&5)xj}kq?Cv(66^!@jUW4NM`9;7q7OL*!!_m_l!Ge3Peb3Cqh zc8_TerWp>ZG*sYjj zqr$UubE_oyXzT9on;Gagg|Ir0y2>odQ8&ZfL{d&EV|JA}cq(EwhoE0*mQ_-hHk82~ z95P@|E3Un)Fy0eP4^@Nv=SBZ>*KRe8HT4&5j(@x;5KXQLV?3)N#SH#1M>XUHi5-vZ zJ_%W)Vb|{n{H@!&>-+!d7-H%-w|@WyK~pZT#N+9bbHI8F@rm*j?TVV$N>rZG#)~EF z-W~u!O?7W6%b1^ z{Uni+$d029881etlT?Ol)PI(YrWSX8`ocF&lnEf$DUPYBDY@J^)oHq$b;at{9Vtqt zkTSL7wrdytZ}?9Wzw?Om9Q^g|rN=dR#){vvV`%97ftli@+89XjN*s97)|dNK9H}^) zUQ7J(`9~wyZ>pwq-G`Q5T!@PG2G7#ICs)D*PF+#aZ{?pWU#Wh(aJuu#&tG^#lB<%t zN2u2>hOa(i4`;<;&(SHVU%a|7*Rr?y`l);a02Hz@aL;b2kHFNFpgW9Ts33szj;$U{ zPz_GXvGXLt4y5wYsBJ=S2KdiXz;P}a-8UR*A0#{GqJZ#POVw5&toM5m%2n1fAPy;H z^v!c1vpPNEhVHX0=q%y3wmb7cgZqfaR!#Fz3xk*lot>u$ouA#%@~I8PGdPyHcr3WF zR217kCb`8%MqHs#iF2o+8~3n{h3Z2vR$jMsosV1Y-8YpEkVp0o)l z5m{c{Goti~I4mGZPD~?L%Ii=@V5GLz6@sq~1w|M$61KoJ&`P&jkq!Z}pNlCnl)+4$ zv1l2B;5?jQx&bv~NMJTg-oy|LkOz*p!=Wk&K~HMj_%#+d$y`3vX%#tU85mIRqXiZK zHI}p=I~fl&KH3M)dGrU9DqGmr?|h)!uG6OXWzS2Glk&`W%^_#thM(Oik!w?sI-k16 z?UXslm{uUbVd(U}iP)ZFy3*J4$#M-NtnG`&Uu})~`RWHk>`|^pA}!s}^odGX(eLh= z;`JV>TQ z2)_dDwq-s9_qL%&#=bgknoLOJhmC;+gw=<>-VS6~N>{gy3cvxqFW|#Bo>KtT9z?NA zw~}EVT^bLM2BuCN3(7mgAq#_(3CyjZ!G^RVbhG*~0RA|a0K&G#kfPW6F#!g+qf2H0 z9~eF0R(gxZ6gzAMlGA|`HB{&~FTw@r>B>k@y{niQYIt`cXQNfStzr z+jt24KGPG78bWh)X>By#0-y!yM`U%JL7m| zGh?~VN)T)c4+UbdTVY|4l5HDvf29jcKly-eTLV&{pRV&-)`L}Q{AlPUxE&g z*LOlCooQihM}lf59A{;roPgpq^>k|k&9r*#@Xfjop9&w5n?#l*zwi?70cMBv^6}0$ zUkfrig1ftl(~=8c9gm$>Hu$C7NY2YCZA130koZZ-ev+6iH&E9LmHY|wjbf%4DS0j{ zr-FDjETH=LBmlFY`8TO$ldN}i=(irRa>84hGHLi=`7=-9nVA@n0+ zYiiQ1Pt=K5BimYu-|jl{*E9bJV2t=NyK=s$cuExkz)x?`6JW;Ge;q7M=^LIO$S*n) zT@&Iy-Y>tK%FF-lj=1IfuXum<;XjKVR~M$nsvYjtK_R!)hlHoEp^^%YA3pGIH0QDS zc*Ndd!^VKfgYeer%q7es`|>nfUjCgk18C=30=nTbc%x_x#N}o*9XvVAXb8oMaE?-Z>p87!_q& zN|?_f*tUDHEOSU<(lXqsqT}`Eo%hqNCxM*^T;_qd%SapU-)_=_0VSVL#7>fB+nzYNELb+074MnY#(SPafQU7`>}IGmzoZwh2x}`I&J32Rq0@uE9*f097gH}(` z%07brOaIKQ-r%CO1oseYGaV81)8H#v@#LHZXP2Slq>c(+aAVHK;RNkJCwm>QuS`rp zRjT%4y@Kcpr;(kmDHwQXG_I|N;OHUn!TbT)uTpbN4`4gnu~~f^I;7}GrH($ z*S5!fYZRwPCSYV*=WPJCZNI^#e+TBA zfFm{cNu?eBctHvP_ObJ7J}7el@x5@lVR=%TJN5vtUpNM`?T+7q$?efUMK6nhcdu1c zx>K`l(HFgQ>)-MvY(`^H7;8QYZqm6Yb=0R|QYt)mjG2`uE3*Wiv{i_?r77Ifi?o+f zkj^8@-(dAH($Bo`rNby^GvHbpDcqf5`uYL-cdANoH^Dl({OdysH}8}18x(}mo(yQ& z{{AeR4q<)^bQVA#v$1L%dA7GzAAY8@xVJi9S?2x>b1dTi59;;8z|>Ry5UXij;_Te9>X62FZ?SYsIu&-6oeK{IG8PIR0t!miH zS0-r-FB`ifmXKaeJSc8>iReJmmiUH@dV z(8|ooP;aS{(nA$Lu7srdNu=VSP_;iZ3m9{6cAQDL*=&+!a2}%z;$as}7(iC+FUl_3 z_@Ke|hSZE1)4uS(SyyQ(?LtwnjPaTr;Wk$QfQndq75k9*K$ek*`^7L!qk=p`NU>J6 zp)@8!ki`OCPft%$I*-_uhxi2hWq498s-h(kv4{W_Vc%0^50*{ zX*(D!NUh)dqHFrnKWaY_6oG{%p!5g$r89i$q_?iUaCMVqT(O20M!Zqwpxu)hmv5e2 zE4};5JN{_=gum5t*1xU*w=BMP<1+KxNBI38|7q`C!Hcd|V$;lDtuf4ZkDkxHjyk_j zd9Uq8EF|zH8-?vN4g6E0Fw8gi=ii^f#|8Oo6253@dX|XwgNizRj!8uJf|KtZ_Lkt0DRq@%%%Af=XAKphghg03k3!OpVq< zVU(flK5%0NVV$KsuGMHlHV>oLq$vGXRaG?BJhP^#wkA*TT}x$!E6VygOjfYe z4U*EP6shelH~`*%4^}-XQs9Fk%u@(iUE`FHja8e?wI@XeR&V>VdY2A@TpJoeW74+b zMLz?s9)z=@;>nt7N^jR-9npA$^Nlb&K#J2fvUL5_u2qf2@GZu{P|IyNj&ZsR3#7+% z!$))iXhn^L7AGxF7o>OtIT)x(t-pTydKvx^Pa}?s1z_M^bnD~vJwr3Mci~5(LoTxT z6a>EXw-F++uD&aSKC6t3rQwC=>Tz7rb>W!TdhOUEa;*n*YNs8hYT|Jd(!;FY2E0d4jHU4uBc5Brz<@ zgv}HXSg&F%g@;<3X?@NXeuLN7aOdpshhG-|_#U_;msIdVpAswq;xuw8RfLy`O3Or_ zfO+&59c1uwQA#}wSk$r%RaiZ?q6a@l%cnaJpaZ;zW692kpcTdj+E)|VKB8CT3U)(^ z$CL548p0}oBU6>Z>Lv%6thbG`Ed6&~1jz};i%6%I08&E)L4spSmZA?Ty(3~fb0XQK z5wvk04SMHAB1B}r&E60uY82ZmxxMDCR)Hu~v$(KT%e6Eh5h0lD()1hyuPVMjWyc4; z^9u#3`+hb-NE_ephKcLlDnUESNkuE+fJuwm+^S6{%nx&xjq;UD5bB~-H3a!d0&ucc z0E~G8K9PE2f?nw9G%D;`Wm&zKUnJO5FU;+tZ=Hh^ltjpBWjbf54FR^|UNX%fO7*$i z(jd3g5OmR0lbK-XtSOqSkq1lJ#v_qTWp^x9sz5OHn&wKi1kE*0(Apq`EuN_mgPGLh z7qOfjm}7<m%Z=0uu#;Wk{GBQ zlWxg^HM~YVIC-MBe0BZ5)TMx10FQDu=wC;Nid_wZrTW)v;0onh2q}bnG_cL@7jCag z*<<0ZlqFsE!gz*lb|~p$c=WTVEuRa5*_O(hJvydMI+tg|x`egm+%~Lde|}MW;(R9~ z-*ARbOqQfrgA|Wp5^+fPW3;*Bj8{kMD767JFdJizb*n0gm(qHt-&LH`E*l2>tG)RF z=fQ=S4IUmQ&D<{6NovK?LsU=wq@M}0N4#h(!F8gbG{AI*L-&jw6sFU!y2l%S8PrTS zD@8dAzjiO@*z;?QntK>}MWyLg^=ETz>dwlN9oibg7X=$Y+OQzH5pjAMt z5+H%x*0!`#p-KzNk`$?;Y+(x_1d`jKN-aX#qM`yxTME%4M1-(~B%@(bfgl7DAe#^% zKsF#Gfvor8p3^^i+S49D^83B-^L)Rbk5}R|Uu}%wc+2>`&)8-9cMvA)P=UO$L0&zC z1X3A!BnrI>`6uL#v`d*g1g9lLxHH$a_MI1X^Gfy5to6Bq9Ozs}xaKlb+dod5r@ySG zn3)vW!+{D=+mrQEST$iNw-`KF>v8z)xUBHs#rZ_pVMbO#K7XW{w+cA##(XtnYs=p? zZg4I75|!KE2f6ZSw&o$s#*ZGjR*z%3bB`LqCs48`@LPXeuAIDta7h3F0voW4OBFN6 zvU@%nuM~8XSF}#2Vz|#f+_#nLxBYW@#Mh_YS?jB;)4>`Tai6r!Z0D=729x68xDo?L zz&ebDZm_)w9vOWtPnV(R6d5)yxZou~Fbm1nglHUfX<^-SKRZokwnav&aoRt0PZ zPHFw3o^%so+T-2joXZ(K76SqD8(mdh4hI*QFEvG#hljGWB^dipY?6mDv|MKJBf3!p5GDCZI~@KYX=;2TAhgD z97mu*1ZJ9KCi%#BG!1yye;EiH{VN!o149AapQ(n#jz4{;G@I{ z&o)qDaf~Az52Lt`q7?b+mLT&)P|q@|X?>!xY-b<&P_g>Zg9>yd)2GxeZ>qh?u(@0Q zfnhV-kPv*{k=fZrUTN|MUYBT_BXpV(E(P9hW%2C$`##8kmXN}VMQqMV56kV(y~n`& z69ovwtD=oo6l480*4i@0!R5<1_hqQZ97e&4RpZrvz6%(+)&_mIk)F|I&F>1rljnT_ z$f8IIvzWReBqG@(Dwpma2e4`g%V%u)Q#OMvHX` zB9;kQYh!qCQw)NE>r*QtBkXOUcze6yetTQ4?p6lfd^Jt0PrjK2M5&)>PS{(UdT;XQ zYWr4*B~lA4G8_UWTKA&VVV_)4k-Y-zdYpshXzg3&%`08N5pXS!IhR|d2AS-pJWoYX z8&I%Ww0j@=G;ErC#NZ`CZ=6dFDQSzuEWd+YI!DoYQL;L3N+KvL7HTP($L^wTLQsAS zyzzM4+)>=Q{4}y|Za$e9p@qv9DV3wvlRauk_kD)`690{!t6I}b|dd2getwsvT@BQ9Re-u(De zKdIu@81f%s_v+9mX8t98mwg;22yb4V+F3oq5!*>8>!ng@?I=grqJqe;`;!?n4SEuV zL4I0;A^G!brtb8Pdd+TTwgT(p{d!uA60N*h^Z6NV|6kMMHRLGFk-Vq(zBINrcV=D> z>r82DyLl0d)uu>9|9V{a`Q2n)ngc*h&DYL&gC4bUGxVJkwT{8I`czb{w6U3_wTP9M@f%F!`%B~SV%}%0*h#&H3jdw zC0Z~Y?9BXg&FB1+w$aXf!CB33OG6!PpJFmjeyln8^sk2sueV zuC|Swww;)||6 zq5Qgy(^wSupvFe<7OU=KV~jlM?AH-B7puch^d!-mP2Xo~9+GRsHGx&|=QpPa@ttgl z+IW2AQtV$;t2z>CZDr<|fn(HXAc~{D!??;Y8~}AxWg)2ZDg@D%HFk3hY2X2q(a>#M zhf{K+JSrJkJ4%Gn(2dwC(Zlh)H|@7s<~>Mxk>hSg^q6LT%(=kAt-gV?PYT6@XL)`S z|7d;RL^J<`FCF#DZHF5SpJV+Ck1tk9_2J~PQu$J*Yi(J~lpXq4MSb;2+3Yy{x_hu_ zRmYd_#sCXdZXq;Mw^Ff>7Sd1G=3TVpU^T}Og}O>f{om@!o8>!adssTJ?u#jUDi&t8 zO&zVN+_v`%g*`TU8+lH@=yW#oCLk44ftmlbOTNB@cP6;jwwU1?b6xON@ishNb1&8! zXJF@0DT(;nIQ_&w!kL>@S7k_w-EC}7Gk}n@vz75a*T8Eh$|&a9^12y*!CE0={UzEo z=@Q??X4Y{H7YsSw#H8%-!lHgnUBp)SZjH(0foTVCw5c$wcqD^?PSDY*vpwFp?b(;R zaA6@~v*Dccb;)5Ie%n!$%5ftET77TQ$z-z`yF%;QQv5GU(Px=Eq;1JjyA^#fMx_BExCcl>2x+a9Hv{f*+oWVX+0T0u8ELjt@d+W19@A6o@3D#ziAsdaMs*S-LeUy z)N#Y6)#qzBBGC(LvH>Ki8cC8V?Nftqmzc%2?C>txUU_2_{a0V@E-ztgp~2Wl&O1F6 z^25cyi~jPDP_OPCdL!$q=`H%Nh3CYQZQ#+PLQdmmaXK9< zC@G2ki?0^?Dw;B`DLwScd%Ic;UhTc(;zO;&c~6{*@1l+rcE)7m{WNlm{2vd_Uw&XJ z)!9g2Hj+;78WxyVbB=zdI?^~Eauo{d2V2Wc_c`QTDjxr)q@4uR?JKeVteK?Dzv+^D z3zV10dM*rfD%5d;joDgS`06_d>+_*H-qMop;;b|x7$DJ|9+F6}WOEcB$#!gy$YrNX zB|+!qeH|kgN!J9kZQv z$qQFPfpBtS_Unw(EfT?ZE5~(DGxK>p>^3&LCwBGThb`>0G*L*$)IrFhff>!2M}jmU z)Gr@Cd^nz7+L7cjDh!?xesTT>?~3evCN49>IC9{;AhY@NE30?@mH1)vrBmtqO5YU; z?iDF|-W>nB;cnbQ?$d_cgtV&Z4?>y_F@H?iJYvn6dg;);{N2eo+nIt((KBzxQ5)YG%r7#~fiS43$qits=>d$p4Eq1U^9;e;uaqk~u&LY9o{6~Iwq z&Mtm%X6SnEN9Rr>Yno2!Ii}@h`tt5}s8rh~h3jy`w}=xN5BzTIMDqfq4@**2JdWPG zJY`R47a3mTsuS3i>)4?%ZLjq#jWlvaADE0C)o(d{W0T5Pt5gRr)u&Nx;_d4jh zmP&`w4N&{V0Gz*D>52M&vZF2h;hz*^_ImZa|cmado{i)n|x z>pbOwLCZM$5q3?Xf$JQ(k}$}(*JBtDcmR-eu_c+{ToL)LKQE71NTC7j&Q%WR^+V9r z(XlMOtc{;&#W!twn(O)4Y{=&XqX{C|aBKZr4Am|k9EA2Ugju5VL7rsSvL)!%EpuB# zLzzI%_1qzKCAa%Yy!3LH2It`TGdLZymRpfa&0NbMms*r#CT`?}W_<=W~CQf|vrPy2C9JTbcn)oxfa*Y%23aGQJ5eq@RS{ zU3@m#D>`!*YwdRr%2opnl;;x(BcX; zeaolLvyrmL_eU=Fu3XM2H;rfXt!B`B_0w~T0dH9So(ys<0}8MjYk(ZvQLOR0y5f>Q zU*X+H$@lZqDM&VvpYuB}4$-1uVz#d^HgXXtjV3FGvU&oE0ppx$6JCd((x~$W=R*dx zN;aBm@n>4fyGYH%EfE}pTRfStsCKRGQm&bnpAHz+B2h9?=6nv(3j+vd&R@SaoY7~l zMK8}GhEN?e8z` z(y>!A_jm2VZ~coE8-*fm^uO)r_&X&3IN3OzFLz$U!%FIlE0HCt+8tm_(XDt}Ep!d{ z4dJXUN@|0I#C22V7@N8J{qQ_@nD%owqPW`qJefjq$<)nTxuMCJ^Ht2XmCPPvBKgm!6w7SaR+sZDTR-F==c9nEz@23|HiMMw zSD8GQf9W#;!g|L)T%XHYn`PNI0doZ$oBK?RNj%Vp^F0941Ki@~-IfeLiq7EB;`^R@ z^S5EKfK2q4@U6C(?m27LW$OZ8In1u%-1FedOa*s3#UcMQF)@Al;Hm$;u)CH0c#qgG zen$MsII-RI-eFFr@Z)P2z85s4gcpS`q~FXdh${Q=@kb|TZ`YlLaSliWA7-Ou7R9Bh zf&56Xpq5@JCDspQDtZhW^qrXtaJ=08!S{7dri1sYs@!P+E^QK5xyRQ8>R|HxeO^J% zeSi%Gf82jIHJ_Z3M%MI~bRMe2wo24Nb(rb!kc2>F=jH=gHEW>7pc?gW3Y!2$NIk z39W+zU_k+rkDz!b9O$DnB+DQvpua%+;`@o|tAc`}f+Me}@yz)Ne*O0KhM^yyRJDpfuM!O` zeYvmoqvUI?9|o{LluXTD&sIpFKxF=yznl@@F(Wx6!J8|940uvCX||7C**+@COrko3 zqg{1jrN@Y)M=M%V9cP>AJ)~6Q^qcZ@)oGpe^UapUYxKd)6QXB>iv@G7rbg~{pLcq7 zmGIpieo(YAp8iTAyQn_fG}%_pTaD_|$t?R!flxIr)=6c6aT=5qH%MGp47M48X>FC|c+e~`8!Q%5XiN0eaZYf4RB2`NOo)YSnmHgyHJ|+Ii~f=eo)< z!2Uyul1k)j&y=AXBgb@$?4H;!PKyV?w>(_Q?nmocb&aTS$-``iAMy=I|0 zJXCpRG2WG($n2y$x0rklhq9Mz)^pUa(zwpQr3Hx}M{?(v9+ngj;$qTVvq82zCf4r2 znp=k|d9nZhIsOgfoH7!zMb|xwzTjW5xd}kV_7g{cFY0&`7X{H3+C?ql5=uwZX> z`bPw$yNVQt{hccf)igvUua4a9Dz53GT2=HCtJTvhs2)fqeIrV{1BdG+k~FHZ!Dm?q zevfZkMrM~lK)z^@>p{^5Kq%c`wv8lSjV&M)u1~z{4KF=H>W$v%ZZ!{XzB-)_vw~-A zPBPJsPSn_q1d0YpvCUSeT~6s~XWzNx4Ug;P=M#zi74L9W6%*mB=WPz7&45%ag;6u$ zES{V6DBD%?7~R+pJoHUnwf}6$PQ1jF!FPK1+=wT!B}M(C{uZ;lFbQ;K5J1Ah(SPw` zOvxa_^$;;?J?(eZEzJNw4>Dv-CZax`dLm+;mpU3>PK&AUDa0FO7`vmoYV@f9BCu?w zb{A!n0|CC;F}s6dU)1jiJFP*TtkBMUW07Fj_q-Ery+tYAMfhxjyt__Z-jgsm{!QKZ z#Eq<(M|lbVb@TaYiSKO37nPLVs{X-y-f=DOg#P&Sd}XTl3vrqkqvARboRSa=zpl7= z)ij}D5B`$Wd*qe!tkP?4HJ4N5fyqh2r`j!XobEh;o!%QhI&(pFu=CJirfIq3z@JXl z&ffP61m+Y`uRZumKqT|6oV`aG0fmKyok-f12VKOAKFo$PM_5I};=?N$G4x*Zw`n3M zkj~!rZ1igx`w@m6QR5_^f`X20(F4uXrNP0ooqH=y8y9?Pk0W7$dGto}MFY%g*MII? zIEJWO$sK_rjJ$%QG2N_=Z=O;p6mTdo2kP|*tsdK3(18t%2Rh~34R;ss)tcCvbbhE3 zUKc!WPkX+0O9*Q6h+~U6g!|Nst_dehd_Y57F!u-wm-31lGo%I})}20;chtwH^#zpZ z+ds^(A*SsF=xN0ALuNF#i(8t`BWz5UfN@~T#Qrd#v9MyMO;?FAR@R<<^DAL=b=N2r}}e5lFWKuwr$fS7w&_%9>Va|(|@hv=C zm3x)2j@M)$sC{EN#)~n%`pX(*aIl~KdE(`i&-Z0G$1&y4ScSr0g|F|FsOaPu|^xY~UY-&=uVd&DO-g4RIovH3V9q4s;!tBi6l-%2YqOTWnV6Q{Elh{>==ax=p z^VWMkhgw5Ble2+Ut3vrxbTw4f0M>rewf60hLyvGiQRu@@)J;b?A;b1SGqthsBo>rz zEK?ZXj-=VzFP@GTY~+q@Rg{%gcxpsLH2Hc*8FyV)mK{ck6^R&~mLF=+6pPZMHZ6Lz ztIN5BWaO<}tZ@(8|@}^Hm`}0n)+s zmV65uN=y3n4$G`z?IUuGa~PuECn8S!Wo3@&d4@-QlN@^C#iU9#glcS_^xn2`^03F& zRg7~R5gI~G>Es%3))e(ObGN-sE##?q_zOnaE+~x5fJO)e1$N+izh=pG~ zC=fubu|bP&wf`1ed~TTXNK)SzR1!jF?U3RCTn%XpZBT2Ed8?zZU`_^<*y7>25a&G2 z1)E8QhCk)j=fFA4IAC8p6;cem&aFxzcrMG(xt@Kh&7|ha~1Gea_EZ=)YvlyBA zUHta_cIe?i#wYGy7XS0c-@W}dZVvTnVH%;qhvY&gS>%r#oGNnK8za$MgDtJHjmO^6 z^}U5XNy?&86W+)(5Qo72&{k922AYrvFsizi%lTViK|l-&1StJXJu%l9AmJ`Vq3;~T>P2-{mrD3g zGk7IB4(VX+SXGCV;Du!6-y)?KX~pGDgbJ+YjjYBfFL^@8`Lh3h@Zo66MYc144ZYBV`K-Yl^I+vnUs6k=v7{}8qTN~=Jdd+QmwtI>B)Bsesct9jOF3PsF><_Me;1^O z8RV=SHLVPv<`=9y&*9!~%3Nt99gRMg5u5N$<-%v_P>HR=ETl-_M)u?|GMT)wAbgS$ zos~V-g?w7FbOxtpGl;eMS&}Jn*R1;3*osf8tHfuM({43>q^2B}wcKhs+u7ok{9<*C>v>FPrBDi@L*f*st0h+t9_W9A(@pI}oQ%^^2rFj!Wcc!6M%DcCY-A>61%?F` zZ~(?_&dgm{JBD*XZ`>S}w8ddENLyE?if31lab1h6yxg=fic@ju#;k0XGJq_eDd!DV z$gPr?KVek+Vp#IC#{PAtOOAkt)ijq_$5~qR>x`fl6oo+bPS!Vz@b)Y?bZ)k!i z1qck|(ouhP0o52!-6m3P9x+>>+0hR3M#lAXZ~JY3m(TGsP*z2TUg%hqf#wn*L&ei? zXV3Br0CHFQROFgl!P^7zn@GNiNjM~x@s{W)gLE2ZnCGb$B9&}bOr5_{LoLUnc!aU> zxfv-V#{d}8qqcI=Q0y3%IJ{!|24}sFQ&|A=W%lqO2HlHu(w-Tp%tr7WV&@teQBG)p%vUT^oZKnqlnfhS!ED1PMW z$A|WF8^I3g(Q-0TS&;9`F){2yJlb?O=FXN*X?~lwv38f{l9qMaIsV~JXlA8lb5yOG zZ1i@ea#H~G9tE-3h2)r0-_ai&{FgtZ zT+KqAJAjJK`uaBd=;A`k)O)vXC$^smPHmCUU)XibJ=WY<`yTqIfbw&GzOBr+%FlKF zQaRko>GBf?-VH8h(*u)b_<+PzpGbc{T6E3~|>SV#fV;Mk3jJ zT6C}_UiHv9K9~39YGwqZrK-unZMe&x8c)X?UJFGpzt8k@!dN8fFm2#L_A9OA46{Yd zA+(k?tk7-l?w7?TW6NmZ7ni^JIewp8Gn&|T;!j>dKDy+0kHs5$o=0&~Nw*>~gVp72 zI=wy_1YxIJtKuRfZINC?Vi3$ldOOHD1tGZ1?7&WSdnHT)qv^d%)53KUhSlix{pb6x zpPJBtF^fobK@Y^}HHK*M2uup&r>mTE=@&+)?=;rao9f=n2>AX~#Y?~Z<-WcAd=H3QH4TF;)#PkOR7i^btHf9q*F`Y%cFXBu|+$7kBHYRHCM z*Tj{JD@O&zvP4?*61ux1EGe4oHDvn95tW60X8qrly+;Tf&ix$7(^yD?}VR z!LKOe5RNr<`C1+qg^D{KW>`Vq*-<@j9q0 zKq@_CkriqHLhVve(i|lPaiYzG*xNn?JF!}9DXYbV?hNu-=pztozgnNQrmV5 ze1iA$Hk#|g2%-$$Q*Vr;(;qW`(jy|*z6JWcYRX&}W!=N|{QqOZX2APDisJQ0`Er)} z6$9kxWNO~&Ry|LnFwJe`gq2(Z;QtW(YngSES&sp(WsrHg9Ah5%ZXHR{p_Zh`zD@Yx zUyH1j&qMERG=>h{-^>=t@Wq?%9u_2|G`*yFTOgjWGVdI7XoDD>ZrdK#_HFo8nE_Jy z5OqtfSk)=uiC)?*-cHAF^nR?`Set~9i+J9?o!>^!06I55btw zp&N6qq(dlM;Ot#o{faxgpH^ZL4W|@s?q`s!)rDiqU|{^b9Am3yR*i~Ix)0CIJDvA^ zA~(eTXs)zQZu01=kKj&o@OTT4`@ZMHTSI|<_EnD_jX_^*B2vqEoPsf(n&=^>m23)d zi&=_IFDf7apeX~RD@P3;45R`-)AXs{_|og~``UpFZj1q21G9=@H}Xql28>>f4|P7? ziR$5kJaEndT<&STJb$c*|NntG?aWa4dTtDLusXX(KPW=VJC@b{tz7+d=GGF+1)D-9 zvDnyE#7SYhGWeSs;JVKMJXp`m*DTor>2jH@&!8vA3*fD=@u?aV4ee}hlHRonD6KY6 z6pgg)h0FBe+vuE(et>L(@LcoNqAA(6w%>y7)0vqJ9BZ~3K`d-8&LC?}vbMhScR6&H zdX8>OC>(!3zxD%D6(<`NHjU~lnzR@P=KHrRnvEPcqOve!e4Wd&MAuKB@pbP~5}aZt zW)3FFi(~bGpUOAJPz_Hwkc1`##@LZ$x0Aq7+j%Qef|cmUSshKmHpiiQGc zF~Xk9N|%mb!nBaOBqdW4vP>DFXE|35bnCUWq!6OM-z3o+^jTSS(j%NpQO$5p3Q{af zUevWwNpd?`Kd(eD2D#?DoSWBi7J@Xr)M?*_@vXiR>M{b@0^rmh6c?Um@QKe!anUmn zkeEv_%BJ5PzaE@BKieYDZzePP68!on>pb85=KVLny}IjOAlZk%0QwL z{OVBD>^kxA+|T=OkFFMT6>@F|F~zlfA+)OD^n;k3i4R@o@jX=NmLd+hMeCz+CF+Bi(Y_1?V0h?EatiB`8D)h$*g#3u!MKA zV)fxS%tKXT2nn@#SkCySTQX-jpF<^MEo98m)~9X2-K!+!udH%L&XwB& zlR`q!VLoxgDaVh59{TyON5A0ovU~O>Km7fj-o5ODti>(!5<3mDo9y<|v*V-kvD?Kl z5qqS6&VNQQW+gN3p5q)m^zOg&o}9y{lVe-`AM9@wzZn^G>@fWWk3We{Uw%KbJR&4L z62yz2H%*3I48Q80kowOr!~Ws3XXs++!q+WBH~!J!8z4~~sSGI31Vkm)a0P|R?Udc}IcEp<%-qVs(#QMx z6CC>--njyBF?^qn!`)Uo7d@$FHFfc}C=I&`bhk0rLd@}nyP~vg>TET`{4MJD-7F(8 zH$+OOL+wGtX{K;JBU{7#t#Y{Qj{CQUn&aU&TrI5U?s#BTpQb6-rm%CfniyOk-tuBh z3Dnx6F-+`2 zG8DJR9&&2pAasOTf$6ih0TJE|7xFBlbP#$J?+-e)G=rN$V<=%a& znW@s4BAI7b7me_X(pjnO)95*VraNA6i<0ep`e5Mhf3>-xi&Ea@9ESV=+0d(g-=haU zg!VDq#sKy#DMvri$d%FyZ9C;n7_$EjitnKx((@?h2wG1^DL#TB0S zAoUA>8QaxPpJDWf3h$FOUCO zx%DizxgB!9Do#DuxKbkwjHy$sPEV5u2fphiN|HTDi-#l2MJb6&=Wjz8mBF=oD9E-! zwW9d>>ub;HD1B*l7o}o*umn<|^NQr=+z|qbqFo5L$Vj4Fk&bPQ{i%pvJOr=A3fL6- zLN%ukk#J z$4e_EMJ@a-5x<1fMJ_K#+j~*^Zq59RQE>2Epvrv>a=T4vQf;-tW05lkB2I6f9sy`P z{g#A0!+b+=o>RXW8w2zjJb>&TUX9-H0? zK}F%+005^n<$DEkAMNk!w@@UO=V;RERvAna6%PR}rN{dEDhj35QR?{O>iEpzMLxPDDXH7{%c`~*A zE7s`K=|MHf4txG!T-Mccsj1E{{`tcfB50Yzs4o} zPtyl&d(ii!A;oX@VUbBkMyoEjok{)uxIZYg2wGrP`wPzHxi`Phjvl>;&>9fFe$k@d z$}$lo-|~*ih-z-ySh5U-_PKobsE6vviP-pbAab;Sv~QdsKN3x@F~@66(C8i?Pf5
I%m z6<)yX7Cn(`eAk`w_&((kILKJQ^gJ-}JfJibs;S=1RoqV!6UthzW@^GpsQ@I2QNRrp z`3@gqQSqDeSXyufQ>69tqM>yP1PxbKfNngrfu(b_cHexD$k>^z6$jrX`OZlF#aQ~U zW26*bM)a??;56hG&iESO0W;N_2Lv00L5c-8J8f~h{*r)MG>>)45#m}r(m@X-!z&^!4(u9yW)&X#zBhVKass(^C|rmk8@a#t5QFM zvt@xNqH0%2tQzxCB#tM=&BSSYB%wX>Tb7(V&3l2zcBWCPk>i6uW@;<-{2dfcB$?#d zFePgvkEOvjPKISK09lS)HnxqW+NR^ePh;t;lVffq(05z~7apK2SEEh| z_0dU;Uibkx`mLT{ZmI?e-e>#EzKgHCJBy~9DdF!Y0v@00(HJ8udlB9Tu#9SXHa>83 zXRviag$dl9e7hAD`dKeF??rYj*9&3v?ilftqm$7;W|M-~=jJh`v5{-nXZwb! zc<-!p@lQp zT(*M1308RC%kIc~|6XjY+;B=yQw0o21k0DpkUEXR(nHYGeDR{8#Pghy;p>xWadGaK zm?$Wmtxd^Ucf1$-@2`Jt@Y;|kshu{EE43pc9ug5~JwV7Pio|Nw%LSy4H-WL}OQbn2 zyz&li^>R2oUp0ht)onK%*0~#y*7P@c7pab37C4a%P5PPBs`tLu@@q=Vx@u5P2f-p0C*B|+0`Gd6A zhVyq`+j{HKoB#D${^~~wg_O0w_U39&%u`hh@975j$q z=i`gmgPl1rme!k_$>iGu;}fA-7?-J*F6w{6)K57MAO?rEJE6+vO3EKlZDF^HB{5mY zEDWBw(l~j_+aG_9#}d|fB>!h;Owa22gT5kkf|~g!X8om(+`86|q>vxLYB%tq0S?a4 zOG>G}N8H_{{1QmR!dF+`+X#Fk^~sBg?Xb>@+u8u(eRg$#B~6F)r9Em^{q_r~sHSWr z!^&gVDGm`>E)IcNnou+r6PJIh#aO`<4A2CcYU^{57JjJXueHR0XuWo`3qI&y;eJ1M zRL0s?WjZroG#2i%gdXm}p(5)uz@Iom4ei=gMw@}~n^$0@CQ@p`bsY6NKDZy7;hq7G z>fBajyTcgqJpZ0~TkL2nv?V2=EgMd=Ky*`XEi?M&n%%4Kzz5P!ZmK0`HNqZ|ZXdh8 zE7x|IxSLq};+EbwjG}ueUfmX%tE>zE2~oMVnizR5l~~Hm%N@L0bJLOiQlr^QTWw`q zR9Z7{gys7=oLh6HRbt|@MaN@VWiw)=&b8Qpilvaa3{z1{;||khRb5kV?X6tu>LSg_Jz_{Z^i?_H=3=<8Zp+0v2%#ILjh^R3b(~-rB$l$>{9({QR88 zgEFpgV+~AoD^{kdHh#EIXiJq+_5Ml~4$7ky3?4-AOMnmOZNM;{$TH1f*2jqfu|Cuh zvF#mmqoXfK%Yz=F}o!YbX=SXRgN?(yrx5nX|rE9%c!7t8)4_e?`JHq1)_Hrg( z7f6chqjq1j`M&46el!>3QvTVbsEmtaN(??y)Dqe<4wJ8A+(_{LpN}IIMhw?JDm^Oo zQR48Xm-E$xHKAZTK&o^4rRX=3t?3b)Rin}IYlaF6-+FIg@jVqBbNF+KL7;^DK4uS4 zYg(BexPJv<|5-|1!H33;R=&~me(Q7sdiAkAMNs5EgQ(VeikhPfwu}SqgH6dJc{4sx z-JFUSyR$5oir_VN#aN@Z-ti?&M?%89O_>zjTfgT90iArkSU!1Bl`XITb=d#3HPwC$ zLECJFf-Cj6&oy>@V=to(rSo=k_T~->%C$$Za9sM$w_g22V{P1;gs_!lmQrccIz$%2UKVS(?-&p!c$)VzfD zd_?QmkM3aob75$DCcM#<#AMfe5goBP0g4s?bv6C& zVrix7+Dr!u+W7k9e$rBNQo5VfN_)IwON3_j!-SWAB+2s=D;4$}6v7`sRIf}JbonG` zJ!?0pqVdNr#>6m4z6x2kWUD3H`B-KPyY)6W7PEd?p$3cZZY*k9Od5br*Nh>B_VpEa z3?#J2V`z44#JANWesoKCm;w*3s5#lBbbonp#>d$3P{A^C&?L)*xAiW+|C6@-g2-~k z;(M94JHsXD5Kb4S|H(hWeQ@$%A$9Hc>ZQozL$3n9c)U$Q+e|7%8v6$2Njlo1nn8fE zBQi=z0oRR`D4SHJw#0&E-JRrsxZx}!G4!;EuOqZz18s!s*^FSsD7^&|5CNHJVok}` zC6dpWSNtNk%>$g9h(b;LS1Gi5`bRsxns{XT86z_qd8RA9zg0=i;EQSH2xct#^snR9 zwC+rz_F!M1@PfG;;z5S>A7J7FSdV}6vHO7jUVQ#pV;(sb+SFfdOX9t*s#G>Lh7YH& z&n8utC4Mg7b7%L${-fWIFiEhUmo~@X<&{Yo*V`(6rHr4G^tvj;Be=i^VWN!n#U-av z8CkQPIdc0m)p-42<4v>`ERvRrd%7{Bw3M4ePt*JvAko(EiTP@LpJYTc|j-kWZHetR$7g0ik-_b;AZ*%kQ?RD zNWBRuJu8flIAD~Sh+%UC+SX#*(MGO0fr-j%u_A1R_-McVHN1JQfi#sATH5@QsndMr zS4tOGzs@UBeCFggxRsIRBp9Kn4<4>?zr=&H)P^fW#RZ#l9co@1MHgFA-CwbPhKRf+@X{U;B!D`(gI$IX@B_ZJMSY(0sJB2K2 z=uINs1FcJ=JiVUNp0P|{YFl}UQELMzuGxnT@O_6<*{iLQwX!`hsSCwWn$u7%Jgs0e zad5shVknGKuww5~y=zWXAhIupW}-xLqj46&8(db96QnCt>m1q`6h*T_PgQ7%1=iSm zts>n@vTYHDr3Qm3a;7GrFGvs0t)h8W>AYHE%SrUYa(XBrq*g+Zlw0;g-Z&`d&xqg( z3?JKL>8R}`SDZR(cDcSg$iX>>^*wr&z^x;sZ`}c#=c2@o!$)(}7^&*!T4P0Mapg^3 zM`_n?OJ6JKs5GfcT-qFV=&GZI8+8D6bZ%J8j$G=s1md9WIL}9jgAe+cCt71$c>0^E&Ij0ctsjZP0$st9625KOw!y+XeFL#AstaJMef_NfZY) ze$NzXyPq52t7=0QEWMM5aX-|1{>cqEkhuB)Fn^`{jChIb+cxa(tz<4hv0?lQOyHmF zh(iw)6zLfqDb4aToD55~m7!~^`4 z@OMU&^<>BB9X0CWU{^N#rgWPQz3#6 zR3aNgqJ>AT#u03Fk#bk$oLF=+(fPDjAIT@Jt#nfs82YD>nF&8Ctm}BtfSkkI9Asr4v+<68Nq1kn^>}Bv0bU>I#$qrD8?G~~n8sUQG0)@$7qTo*hF?rB z6(!bDRE&fLF!h|+yiV0=m(s=%3FbusupGjMsY;1kzoiOwkP_!$8 zVih(h?=)wR_$PIl)AaCR^ysxdrn)AC*`y`6F*cn#G?3cgIrFNVgB5NkBt z44{iJV?u71fLm|uh$-JW7|$e2-EDGYy)kF?Rh7QpF$rDoL3>O_X6RQ#pl{VwA99mI zQ|`HjYI+#Uwhu}v8*Y5Haq-9=Z4;W7__M{HWtgbZs!14`=hQ_Nx4a@ z56^YthB35?@{M-L#GGb@)mnn)1~;6|nQ7Tr(E5b=-oz-9%G1%dkWe)rm8c85YPxa; z^KGeuzAB{XFWw;bBnH$q!`JXZVUvGrfa^K~*{N7SWMB`Guv|YaKbvSXgqQ218$Y@z7Ag=fP4EJ>voZ94 zfiWRIV(%Jc)Px_SV)d(gnmcMv=HpVlA9DGXL(6GYg0)YbC9xlB8xYmlB1L$^?KWmF z=_I!=p&Gw3=Wn=5Z7umIYPqBx90;G_7V?+_)URSs@Tqef|1&$0L&9609PYzZKd`uh z-!oIhg9kfO!yh)HqURPjU*~QMm;s?w=zb0F9FG_fBqok)uiy&A!zpCrQLlz>QUmcO z+J6O-Qi!0ZCf574c%c`pCrfa_qR-zEE%QGiBvzDYgp13YE!Wy^^s@a5X>C{~JX>3~ z?UG^Km)q8u^z+h;bzaS6qcel>kg`G94^2hxZPS|UFiZI@W>Kqg#@i^ zzzo!HD)o+Go#!mx@tRpk^EFDR%=k+0HCC14T=d5{wZMq=HY-{Xo*(iXE%i4jqt-a7 z(JIS06P$LA6-x|7$ptAq4O+j}MhYZz15Yz`6*Cx(GY%6>YHMqx+EU|&MV=nKvw~C= zdeS&U;G|ZcGq31$TP}m=P^mG{Aq`&R6arM)O{{mQQ%fWVE?descJMj?yP9GVlT$|&DbVj^Y(1yT`F7oWNFzWhF!sc)iv1X~YW}cH_ zwAmz-=)691x%0d#qe|i(`qT@MO)aRlLBlak;A;4Ii<%zXn*BuVQnXRhyCMb>U<;~J zBaIM`-jJ)0{LF-I1X(q*1fj|6rof;9|M#+vqogOUZ)H{FqLfqeM$Gwx=aeVG3}1!z zJv_og@7`%yE9OnJ@Mxc)gG$UN71QO7Gb;5Gzko{PRf>{{w-tTbv`@l=8~_y>h$$3N z+^--DqSvMQO__ZDL6M#yty)6WZ51)a&w4N~TH25cJsUvntq(r#q*|A0kN54y1h0+> zS!F-op?0?}UFgg%l%`0;{v^mZ%8rUTI;8H_F6T}0oLuX=2 zhGr`fyc*|K^c69(;_i+g>bb_58is8Y9qfaCoiG9_XxENR;3ik(Ql0T5@Q9i zd_>)~H3+$L>Bb!tbV5o&EF$<+8EhT(dctjqwbVyYxD``Y1Cs0_Juu zGB4wR?e$$jhFQG2wRK5LB}r2f)~bk+Je$;g#x_*zHL3=~7D8yXqV48{X!UJX#BIC2 zgL3u0^U>02ujI}r!9h-f$PrAxZj^zL@UfbDI@ zK8)1$?;z^;p#C+&f%g$mDpu{&s(x}<;XJ4sDi>#r)L#5!OWS644#5o;_R~`TjVvaA zofzgW}tVWz|*Q~>5v!L@`wz7XaI== zjJC=$WjQ=qmE+P*l$n&TOI^QVKw*~^PN&g4>6j-IbZUr_Fb&5Rn73C-eRr_*fE61G)R>eqOh7wdEY`U5nYXoeqzNCnK{|O~rUv-RpYy%? zz&!78^GnqSv&+nOLYZE{==qe9mVq!CGoyht#&R<2R7nbv0?&QxqonV(NaBV z4%P;?FUPN}T$`ppgz187Dn+*LG2i+~71E5ird9^Jz1r}0E6TqZgLgjGT1P6!#z>>* z2-m>i4h-)uwM8~93Dec`fO=tg_*9D-K-Ffn;khJ?@9iDExqye}$)#G%)L4%d2>4<^ zNRyG3IC7Y8yy897YeJzidpImS7@b7>+!fVZ-|$G=G=R>TazA_<6A4nZWLZ~ky*L+- zwmi#W)!4pT*8LXVE}kRJE0LbC=np+FpGeA{=8CJ6y1q`6qAs??4Ajps=@w>SyS3Iy zfwqT{1-BKfdZU{$Xvva>3D5@#1#e(Sgwxgx&v9bCh8?yb(%lk$K%K=*RiZ;~0>VE6 zqrK?KSaXdLVAQK@$2U}4hMKjVn)VKrT6E@JmpJ+)t_-X#MBB zT|KQ8io9U9Sqq>;fym;%1ZZW~Gb`W5jtAGprTLN!mbWXh&Cps zv?6N(z-PFhjo~y&fFMpy?vNr91HblC*%gK|3g7-+y+fO+ zRv{9rCXN+qm9K4RZoYH5n7DLv|Lf9Fu~8^;0#fN|er|HX*R`huMzB+-8()wKYjLcF zHAbVd2xxhFlF%0x&l~HLbUsVs5M|?EgDI8HNshkKfTUaO1Y07!&Uh3gNg8+;3u=Iz zC6OGM$j`=AhUR7sqsW9hwmZ?fB&tpie`kGi)Nfo`S*?&o^Ib@QYZLY@o$cHslUQ7v)|!7%M(ZL8c&(z zC>0bkjmbr8o((hQpZR!BzU_^>;!aITJV%UIg-=LkQRu(XOCwr#I@#np?b_1u>IPKx z-9eTh2xCUUQ^xMv1t!)n)~3G%sKbM(0$XD9j=wp)!nUFf`=xQ+@%Gw z>>Ncy{?}=(Zt>*C`eg>fngM-<{Tuq!39Ld*r)fK+4}~)18hGJ&wO2+2bAHH(Nee0guxqZ;jI1{q zp^!JOkAFIH5m7nz1`NnRn zwqomUjAbqhM3aoxtG)5aHDrgxyE4S1));;v#r8rQLWAgkF;P&6qfZkj1` z%lf*wV}rNgO| zy_vx=z<1dRqE?X=1gl1$z|+?_8yKgL&qg-S56r@?_&Qf7?vb;?-d9Q5{vFAl3prYm z;LHDza0UbPGHGD{VNBu?3O_Fbp$M^ujfpTpXI4LimR4$gmRXKuQ?3r26`MV1xFLPTA5FG9ECi)b=9NBC$5A0wvKO(S^}O5SEzjaw})>b&trr z8fdhVPD9=68SC6|?Hra)H#S8sU^6IphGQ`2sozr(&Wbh}7GU4cRc7Gs^>nt83bbIJ z0<9;BpW&mEfLZcvoEkfw*aVn?fsObP0^D&Hzht(~gy!{-GPi!9xEYQoCQvt!wOwjf z=qe-qK6T2b2~Tk87Jj(W8UI$#h;dx(?Zq41i52pm_L{{ePNM(YrX(#Tc#w5KOM4nt z;PH!M%6w@nuO{3Y>VZZ(8<2H_-T+@8n(rLj_?}t!M9X7|O`j4F8u_k>l~s&o)l7{} zWqP>2O#IxM>CvRuK#_hwS2Osn8UgYprWf^tOUi-clXO)>FM8js6F(hFWtMzx!vs zwG>~Sayxr}0ppV)|2cDZ>2z~xhsgZMI?ErE`sEy;Nda1k=t`&st8>2?1Z^nPzA;U?w=!-3$LV-H{1Bdfl2~$7#l8i{iTa`H(QtPVE3kdB zf5a3Y_IN^f<4s+9Egr8y6}&HaT}>{GuDpazlccKC4nT8eOkEK@w0I@(A-67kMXBwM z)6;#82x857ttP=nd$!36ipb#wF-2bqtYk%)7pn_ByG-B)7MrAhPjlxo2pePB?U7uAA*$TU7#zfNvOdkkrZPy+{&q3u==C!bS6AKiFu z1Q$Gncf2Hxj`}CZhY5XR(r`v#Uo@c7dv6vcMMD$>dezHN)l3%CS6i)#CzjsLsqs)x~C!b72R@8|2{x{S?se@DMSs&#-kX z<4@`*&~l+IC7ojWWIv}Qz0|vvr7!Dryw{`KK19M5j#6qr$XH!)x6w8GDeWllbxh)P z`;uQMEhwZM@rU*Neo2bR)yQR)t6VNlGHJ9tB7uijz_l|DXPI^8&%{D*p_y0`*BlG* z^IYnjBG92vfzu=_G1J!YpqooGbrC|-k%1xi$<{{OI)u_=4tBXSBy!&b>^`g*9n?xFAYrmfE2K)h|nCi;6vJcu`wPK z>PoDtQdzU4&x%O(Pn#P3A(n8)QbG#NV?r+vu3o$a1EwmWO;q{T0*5%d_8QJTKZ7=o zv?bPuxY6g>DR)wE%w4vz{T#3?IHa2O2@q(Z28|KJTI=qJ2W|xbxpQ*DRhZyP8&Sje z+Np5wH0N-iJ^?7F@#%hB^047a)9T*n2ij-u_#Ei}r|8YYnm*J0|LZ$*o#AvG&2PrG zj3UYDcpU0RYEgrbkkfHmrlpdxv{DNsqf{vZ342IFl5>VCwFqg8iV8_;DMX|YktKUF z7$G7{M3w;Agg^phBV^xy&;0L&#w&b2&vW1J_v@v84{dG=YnfWTt_AH#>Xt=YBX7V4 zsT&)VZb~?Yew;2SN^I=sU*_q`%c+(5s5dE!pTv%$BSQ0h|Np5-UDh1vPYF@&VnSMY z-6rc*@(nA8rQ33x=N5L28?|tUlx@jX0pIB8XlZGr5AJ?fg{|2BN{~kMutdNpjt(}< z$`Fx&9p%`vjQDp3ZZN1%ug7rj5?iVvc;R^+s{5yjjVTd9tJ&xkTnYk?*AXR`7P8ya z$ku|)yykDykrp8bxeftp4h4P+k|n@8(xuCWNP-aoBkd@i*Yp)V2$_7l354z8MYYX?Sl8LG-f zBV8jyfj-Ieakg-+^1=CN}eO*DZw*ka+W^w^erqmP#> zL=-xhl_pyJ^Ae}bTTd_L%TkKXoHSc6$Q0Vo=#5pNTM)VOg$wZgM;r(}c`Jh@xIis* z`rh+?*Mq2Oh~In=o@48ntqAsl^yKS8 z_T%p1(D2}?&|H=`cRwis=Y ziS0z-D5lqhwDjQW4#xR?Y-J{vUfjN~yQMm(*ZCEJ9~o98#;-YA!TTqLX3Fvq=uaeu z=ch-{@dQ=H1@>BO^gDyISG2W?Le}(A)dHYZsMa6NWC}@iX$nc`y+b)YdLx`hd>sL^ z!>S+dDm%7K;nn0yIKNFVsEY6saf0)=*B3?P1E1My97aq#1 znyKPcL*08@2~L_KVqqj|98U`+YP4Zbki#3z@ZO}@SbwtYyB(Cz8>b}1HrXQ{FPUax zQM}pll$Lp~j-vYc&0YqrE5|0VdXOnQ^3{v{pmeU!Keod(s*YUXZ;TKVp#vEdN?ta@ z15XRRfx-C0U{0NbRtUl%dvpQ?JqH zglCkPKLr$0*Zl4%U+emg_Q#Wv6;7gLE!f6F8E2V9C}pX{Yc&1_W3@~3`CC2%A^avI zB4JCM8yFodyFV;(cZtX8V&=E})1KYAwERT8Z!nF2=++8JAHNLMo z`o6&v=-ggh;1bN~r3Vz?U1aQohyM^m#2qOSKod)qR1@%d+F?Y`?RuQIoJ$u@Wph1J zcqCsY2|h?laQ+v&Ty=G_NMyzkv}Hi|iIUY?GEhEa>%i~H>l@6}>hV9urRTUEnABQP zww5%TaTGt{hM`Girp3UlC{dy>idPl8e=U=dw|nZXoxVP_#6ID8`~~+MX9&}>IAwch zV#&(mW{!HmQLQq#TZ}|Am%jbf8zb|o5SmKk?9K#x?Iuc z?gO`ll{<_?W|L=yQL}3Coe-IbY*zsfyvCoY4 zwxY}Ilh#gq@c-5fk${x8UlwM}!$RjEFaM0YWXnn1(bWrjB%f4LS~Nd>~2$n42QZglxL^HM@e-j{j)|I?pGt>5<}=qb5LTv&J* z0`6w~0o?;Yp4{K(__16qaA!@KT4j;C#I)0Nk0N%r;KYZMUvP8!i^d9&{)4H2)SYff zzA^5;_w!#TyISY9GqyY(k!Jtz1b}#08o3)-UkKFWpJ$5uV%@AaNL>BvyP41NCa z9k=$6>I`j#y2j}1%2zA?lxcfJdWf%bHWEo4keY2hI^3s9Ta`L_y!*`cit6A$elDx1iHr=r3k-M}5IR=R`) z>F(JacC`%_@jRgtHAH(kdEv09Rngybl3{ah#3(UPHQ&4EG-Bou=ZnsI&DOJEVjGHB ztvZo8!2q$#oh1u5YWxW5q9W=?!9919z|JF8jFnErp^D9K+a&IPDj}R48Ocx_{Hw)o zLkBJyuC<)1Ho7bx*}h)-m^FzI`f-lh{&0!B*_|10uctW_vKqzIU_VHZuQf-@1;Q+A z*QLWJBkS2V$%e5zj`-L#yB`v|ksWs1 z>4ENC%C?v0=mNL~6V@7!^Xy17Wy2glsL$)8GEQ4o00EHP+JUT-HN`WMcHNWcd2>*7|xHEYa!Oi3n{p+y1DzcI*i3 z*2WqxXasHD88u`{yuK!Iyvz_=b~;~HD|>-C{gWI0F{qFa77C9bzlR7fd3>^kcv}v7 zS!t()a6t90EwAD!QGGV8NEqz(yokAUw=hf;XUjv6;Cpz76;mK)B@># z_7d$IrQ^HB2K2h`3wf*9G@fQHBr-c_FBIbWES7H;FqY=)k)dmoxr84t#HUT>W4m&2 zccg2F*_CZQ=%89;)E1xE^>A&c-gST$Rz?l!R<+%h6t~i;@7P2MdNJ|Wa_Utr>fPA~WUZo42@p^YDK>O=9!Mm4IpqNpy#b)V_ zyTC`7&JB?JTe0DVo&9)IuEgz~ijJW7<$AxdW+iOe!K6QiP#mAP33L9q$OZ|(?;$f6 ztTyro61UC`2^l+D+RBL9EqB7^K$%eFdCNUPtq*sLw@UJ|ajNlu|X%YQy!?$yod zY}30Vn_UEF87xH6yFUYLho@ri@*~>pxWtDJ&EamZZ8oA;hvAMu0X`b!cuhgcftKfd z@#oP9LD}OSg4S-2(9A9Dz*c(k+WxB~bQ`?sLt6td>v z0@s)TS7p21n@e|YIC<7~NjlQAV8tw^1_$J6Z}vCG8LkFb*N$?_%MHHQXNJwPz+G9m zwQfdLqIW${fZr~%g&lHAx2l4h)=CqXdp< ziU+dk03wLMvo*pGa^yUky%~R{p$LkMJ|`se?*n7d2G@HBEV7K(ZF4%`d-Saw-cE2I z(mjsvXecAgi8Wnt|1}6j(?jrN3(phA#9h=n zZd7Z#8k6>cBOdb(6Dj`hZti#xq!%|n!(A{5>x)sOWSxf@Jayvk$WP;EhYG*^fVaMM_TY(=Lx}(W=;+k%PBp)!UM?rUloSv|rYG&a z)MsaMI8buqk!M>icI9~3d%hks=p4SfIu-n5q7(Qr$oCv=^pSJ-G z{-DtNCUuYg)p0?(u#}uz0@k`Z^2Vz_Y~9BNQ$CWJTT!7;1jl73uNmtoDvGzKI@2f+ zZ8SY6^IYZymU1btO?cmaGpDKmBzsxG#8G4DI>5~ls%|FY`$Y7AaGrdrsgtI(f-39P4(RyD_Ri(UE|nm_Z$M}0)j`4_8~|*BKj>Dd2`}~ipei4`=z*Sqql-Y z`<%u8sC-=Bn+LD*_5dG)?_ysbHZmAEZ=Y;#=*Qr+MAL=B991r5Qcnvm#+0yYM%j6_ z3yne(YuPnFI?~`ifHSxTxk~_+LP)bJ*QfmRspsh19EzezvGP^fp9&D{JK9=u^>LfE z*!D}kKaWl_#Yv_!LaVVzngkJ@f(L_4Ln=Dh4*=A`(($7O5gS6;tF-C;uDUN?&+9ug z{`P-;zvtVxuk?&<|7$-M?>t6$M$$>FX={h<1qm@RV`QFzE2$8$S1@P_UKxMw8?pY9K(Y-nt zEmepabJLr`B#yFeNo*eKAGGU2RvZ`hyh?%5S1A{RHmN;O55ndU88~B-e{&T$NfNYF zHI*BMasxgYc_~gZT*OMHT&SR!Kf`)Y*i$Ky5GwY{rzeMgif5%%Od8m=MIW-o#;3)7 zq(Gn^sBdzhl<~V2K_Dw!aCdZbgNcxUhfSaC$#>7ogl?7X^T;eh6Fv)bT-1&~wGyFX zS4J^3tzCNo8b#c*PW7U#=sw&)I|ulxSJFoq$0H)M!?(x`*R6n4+cOr)!q=soKdF zD!>o=JapeJ^x`-LXZSTX;Nc*_wV#^{LanX!n5H3yZE6 z{8DdQ%4)#xhsW>YX{+?{L@Q0-Y$y3!GokTt1y_4StzEvl&5@Ov;xTdK|I|9`6QDz+ z*MS0q(wCb%<3*{)0p%HBO80CoG!^~_P z6!|~s9*moMJ>amB9dkPGC9LDtNxiRRh!*AI!*`UV4Nhncmgah6r86C`sj%6`3JTN) zRQDVVwY+tqz=$DlcF3evUf0x(gIo!zBCpabF^pyc!h^6+r*w0}pQ}X*da%HYw0sXw z%iRYH2y;|NPA>hR56(Cq#YdjOAGB`&o=|;;r*l!$Glb2lis+`@+r&&Xu`K$~vOHa~ z?P-UW841xZs|tCkGjkhHTt6^So*QaLb+xcw=IWBjt82=Tx!k&!Ee|`J6HNVY=Tv0w z)06u_?)dKohEBgYB}x13#kS75hvD#yy+)H|2Ti&LI8A#r{Sc>D|6)Aac+ z!!{!7am1LiW#Oz?W!sga+a(`rZ@7PGoLsp;t1J)zawP$aT-zvGG9Dj^*oMynA}G}_?#@ZwU9Wq4Z~~%a@X#Ai0-xw8#K;I&<1xuxiWb|_j_1<@ z*+5q3QQ1>uGNOF6HfWTaVFHq2Fcm`~7f{k0#XpGgv7Fqu%M%#LHbItYz?&3xTgPA z>4B?5s~643)V%0v%AD#BEH62-8_ZJ2rSp>vlE2wxPZrt^;Ax{-@Fq$uoSC|yu{7;s z%m?MefzpYA$M*4osnXYf{;BN?>rd5#r{->4N+!wwtObiriZvc1eGxl)5(RWu;V@ep z{x1DUwMyt7OCNWd7hj7Pl{uvenL$xjX4ad+mpk*Zn)v$mLc3uh}>Wq{G_@OTs0&26q+9!$b!}anKzw?BhKvP z`j!|=Up1c-dOs8qb=2?9o;#(M5rvg^;Lc5k#5AAwS-)Fwjm!I`rf3B_uMMV zEYU~2CdRoS-3h_?T4mJjcmTpmpgUCkz4(1<;qvVyT$qV2laFQH{_TN(hBN93TPcL9 z6=R&uSFdQHu*=@LnV~3%gHWwDJUi@xM;qc8JYEeK#96Xy*0LpMi6hMb5rT@oIA_sR zK1_=+I4v|+SGsI;bWt04q>>DYw@Upa4dvMY5yMmJi(-%D-aifR4W!@fQ zyi`+!AvjVPwN_QKbBEr|FM`r6FnnswDv*k~f*!)YA?)@DR9H0*q?xxsH-s$i+Xk9r|isJrQ|uX}MLpU4?czF%QklKf3hZ-QL?GHOM4wEhE^Zg$AulMMrvK zGnu(Tn;cO3H-=kmZ<``3nwdzA*m%#-_6sP3xw2?pO^(*}N@P3b-60F}AgmI!<-G>E zNGW^LExxU)0@e@rPy)(g^zVe+#n`asVFnVYW>)T5cGc5;OvhXBJgmW*3Cn#k2+)j2 zsg`TjELGs;&m-~vpA~szE04Z)Jt-XcJJ9(of2Lo14{S^|-7<@D>U3uu;jAclaDRQh zX!`o+6~-t30THt zBb|^~4>km7U6o3vJ=(+>BJj8FWb4wRKZD^8mlfE>_C1D&--X}S`t{K|*(q_&&y9t4 z%kp7#O=K^DJOc~6OCXf1X1k!q`vGxllv@$sV(W5xfXr^C*&eK~h;FH3z1iM83nkIG ze8lj%%)Qlm68veflqojNUau}s0K1H86)IhkF!|}OmT>}oMPl94mNIqvBu^iW*RR8D zLg#EBIcQs?+1krw_?zfxU0qAfqBYFHB1!jlVAyCw*ElTPyhB1kw`-g#Q^ZJs=vL(T zP^(O<6;Wy9W1WjGwX9;Gqi51e%6r<5Gd|rgV>Fafug78QW9Vw#*<#xoF5v4+gXg8( zBUA;`4b#u-)4LNiJmLWKudQ%l?%>n>6y~n~*!aia^gvsVVR>fh$rJ@D7);e75#|iZ zQU**GSuM^tzlUJgzRa?T!i7AxDEO*0`sxX_MIAn z{mZVbKfnLcN6uuyrKQeOpRKIeraRi(k-x~v`)+sRq0UbrjB^RScTwBNa^Q}K5Yn2e z_SuY`GAg2u*0BRBkqY@|JgG~I3-;8J|LZuGBBb~-@LwH6x*1I@hhluvYV(~4`vJk2 zHZ~IrAJB@<5!vGgh?z_3O9JA=5vfvMs1e`IQo*8idJ&&`LlL%a{=Q&obiC1IE0hIe z(lO-n#1AB6IoN0nny0>|MT6Odbt()*NS2q(64O(M$QExq&!7j@#1n((R@O`ZbNW}s z|8f84!Ze(#mHIyi&+cDZcz2^H`}%n+&V7PFnmw#+LWzGDn*h+=Hs>+l%Xq`KrDpS* z6n?azq}|n>QD#osg{M3mrPRMnU+TCb!OIWX#nvZ}+H9gnLhqh@cF&hk>^?@KI~aSe zXDve#n6g!_ewG9KCAz}6Hj^VY^v(?@xxS`^#&5 zT(1$5Ca$WPsOzPSAZU)B);BK|MGnE8&(Qus4v46W16j08UHSa5o0+1&=7o5|)0YfOJ%g?X@k)YLwegv<4K9-B7l-^W^du?H@tNFHWSV z%cn=`UPHSJdG_Lx-uk{_rZ=|@zPTUjd;^S^AfXs*L6Wu?;v_2> zQ5z@=PZ{S`GzSN4bQ)Tz(N2OhfFVDb;hrk99K;C2^J+VFsfPz1bV(SI8(4YlBn2=0 z^^O*nw6EW-pl>U@Y1S?t(j_c+o}Rx;Rw@C*QKD!}h>bB-r3rV#BfnKq^}HxBdQM8{ z-mmA3r$vhDg6h$t$nuB&6hhgkI$OHvq`Xii7Bs%9`o z8_Xs~jRtm^`8Touw3V8l#K+JhNjzm@z$VY(F<>H-a!Oyb(kZgW^=fO~Dw2gA3BYu2 z9){ePO%<@CO}Mm*%kh`!i_vg~r1w)Vd)wB~iEso?Brl1{X?R6V&LA@j!ADURKTGk=$3 z2+DLJ6vS~)qdlu5yY!JF2Hl@c`+LsA_;S6@_5nv0s85h++IhW_ z%vgfHA7Z10vUag(o=P6LX!g#>W}^Ori7+cZqrIp@bY}I-!dkUbM-I}ORPi&1#h|&rF$Tm89&;L8`w$c){UCa&j=>zWc>KJpIhBGGgggG!EFz0~!7m(y+AF|LprPjJ0ZhG6ZpSZ~C?#X(c}pHC=tv;q1m zG5C`r&2ZfSrlF3-jlgO`{0*X&P?Bhg zr;?vh=}HwSFUqpij5zvc!gwe|tv}pJNnFkE)ssfG=?aY9!Yzm~HD&6K=Tyb!n=V#& z`>!KmRzk$qHMjPqmbQe``@J#>4o`M|(3fs2ajH%AKM-N z6;9ZHdvLT3UpqUQ1Yy)JRCTvc&#V?d9KHX)UrP3V)brd{qPH!z!Uj&H-Xa8T3=~n} zZ=qh@P4HEN58})mxWz@0E!}NtNTPB-w%8fcP0!!0^^Mz+ez0ei_r+l^6+}y`l!nB4 zsTz&gm)jPvPs74XSB%V;s(b}v-fI-(s6fy?y^X55np7|}rA@tk393;AvK%YiNAlWm zC#>dTG*`TEG|(!;NrugB$tm~dd%_!bY;szwWSoPvsHLVUpi3Nu`8^4=SLu=!L)4|Q zM_%X?e%7i!`O60-_kQ)!f0dSp_j9{0+gFy4pSkw+WPZOPVW@MA39M3(5|*K|a+%dY z-`ElccA6tjK=fL+z9;aCG_l-12UY=1wd!n0Oc+v$OQETn%A&4XQT4)-}DqXn?Xjh&8;i%Ug|$y0M3^_?yuoXI!r+axW_mQ?q21xjA$(kGKT{%hq9VU@wx* zcc-n{*#K6n-M$l(*FU4;AZ@I5VSU=_1$k-p>dT{u+~8KdUYsm&bg;dXYie5LHp2hE zGhqKSJ-sZDMY<`t=n34)!bNInsa?{YLGZo_0drj)h2om0nipZ=%~C_%&`@=6v|q3E z2EHSPZ*!$8Oq*`y40hYLePi+QnL`MsruP?Une*aPx;LAyw|<<%I8|o5Xe+j(R#Whz zaA>0_Qy++x_cgv8`IuCy(p%G(A-i;2NAbC!NEH^X$)RNxiIDm3n!$8k&AMM#nJ&Te zZNlbgsj8>!n5SCpnKY+^&l=L&Bg04GEC{t{@ifl#RlCR#l$xK2z!4~rWWMlxZ&)8) zylT+)y+0m@b1&dRs~t(4^;8I5M!dj_tYxuFEf@AEFzgf@K6;B2jwr)!Tl~yeA}%_o z)PxQ}t_CBUa65Sx*iI~O;5Kv6mKfaj((a{2^P7%TdKHaA6UYBjUq`{@WEywIjOr~rrPNqCjE z&uUFdI9YwC{Fb6utr691Y+$fyE6Zy09iuwVys>4Gm|m~1S8I2}cp~(q{Tp``ybpNG z9pBWm?vZIuOeAzIjsdgjDz;Yjc_n(absxwd3H=?$@nF+@%dYWlOC9vo)9T1rGuyFH zAKzlQj#)_cW)~0yWd@c|zy0a}u6F3<<~${__Y3Af%`mdX6YR|Q7#%%3m;y1v(cg*$QG>8O}d%WDId|{O!#haa5yEGlRNV2pO6Wj7os|OHp z&0N?@gbi|F(uoXK%@f0}0yyo+K?voALu6k}rS~$Wy)6DIoWBRp_ij*lVXI$65cz)K zmK$G29ubvKr9}IAre={}gSY#P7frt6qUqG;XTNE6-s!yg^{;Y&`qS$_|G&}e171b7 z-Zl-|fF=WA%w@65f?%z79kDNU5V`lGyE03LbZNfW^aj>^ATwUwmS~NRu|Y!?#By+G z>ucabvn5d5&vV1SWk}Z6aN8r`>_a6u-Xm;%jh4DuJIvYCx_V8Oe)@y6>iKgjk6p^2 znUlK%Pj}|TQGVfkqDsd|t!dBFWtG_h2g}(e7}G($9;)72R@nt;$+EEwTy!>}r-{|) znB~|f3y`km0`P{0I&R9u1?`zCRAo?OOWrx~v>I`^=(?`6)IU9LI_1 zzyUDxy9Jg{QUPbNYjWd&ZgqBQXF|PJxKojN1gvH$gDnr}VcxjfE)1^TWu~KC7Z{XH zi9JMS(4L?z5P3~;8Im!Ai!fnrT0<0nUbuXL`&bT^Sp?b)JF<*|^U9op2JP+VM5;F{ z9}Uk%{$-{%Un=N3)|h{NbF3r(-XH$@mEAI29{r#4PY35;-_sil-IG}l&cZ+7cS{xbGAly<^u>S1YVjGE^`V=D%H#i3{@SHQ-ej8IJ@dqf4$};*a@`h zwn}i}sRTVKV$mIEbFXB+;hxzG{UI9BYt;>t*KW zq1!pop^A&9iMw&5g(S~Z4%_uJ=jciEeyC$b6q{&;TB9C#WIjXov;me{5wb^Y*JFvw z_@nib_6&1f1K0PE8_6U%FVmetdb!7w0dpU-dv5HgPi<8nRRjCfVZ1x>ei&Q{Y-ZV~ zB&WA_D-z*DJ?ALag|23qBMFlNlrPD~zPRehs+WL)QO4?-QqMJOEd7Xm>jT6>_-Mm! zH8azdMpR6XOUCxsCu?+z0+Vj}J@8I)6@*CHQ#eZawaF>%9&=o8YFUl)QS@e*9k{=a^JJg%IMFE@Pr8vVTVp|L8nc599IfH}WJ3<*)HG^+ zjqYCQFyjKB5qZkoFZ(A}W+xGNq;FY;3vTjqRVeESpk3Tn@h@yDp?YE19_y zJ+~@gXFs2@?d5Uj22&vtXQ4<0_UAb!$T=(cFm^WCb`XKOqY$q+Mm(7_k)H-688HxH z7t;M2=2ZU!FqWrGX$R>q5_bjxbcCbSRe9z3FLHyg8u3z^BUa?l%-AT;s>q{~nPt4h zVYJfjK!d+czyVh81@g-aaK8%e zzma_pCI{S#1#!(gECC3M>&P7la5LA$*v+Ro z5_2O5KY<_8sr9nRbOM9muq-O3TvaU+!v?_OwyIUkq_kva;i)H$`Ye*`&)GE4;%bs|c!OXqiAyg~PMw=e!3 z=H+>JnzE`qarB|!yYP81Ajt9}sl ztnHNOi(W}uaYK79^i~EXZ=Z{ZB-xod>nJBW&yVt2JoR8CwA#1$lPvgQI)VVcN|4+@ z11>GN0z?FyNSI2G@`y5^eBXqHhV?g%|F1EBmto;*=h%0PumA0>108>RJ?GWZ8>dFE z|0o8^68yfC7KV&+I&T@NMVhLNY4V^C(6i_GKvfm3n(*o9%j^?kgbT z=vpH#8d*e8MfFGR%RIWBl})><(D)&-Tx~8g!h`FAt6=-QNr?7jz$x-EnGY59EA?(6 zp;Yd#lhmUEPjxA@fSP6q2>=Pq^#Q^45#0SCNd~?F$9|bVnop-Mc16ZusKS1@X$m&- zbgZ>E+|MRmWY|Q>NY`{iwX%us9pJ1TN@~6I#qBTV{>F=?JZ(4iM-F0q!A}l>1qPeC z*g{*3!%No~Q6C0pI|h&T%eHpV!!2~ObtBFL&-=R*9@?}+^@}yW)37pa6$fr(p@#}TUG8(=Mqu7EUP!!RUITT~C)4xCbreNM zMr=#0cbBl}@YRIy*i4f+MLeXdp)P*;xji0Gn9@N~^byW3$R`Sq z>-;wzPpDW+8|fK<7OmYqdh7jx_mNghhx~oY6ZG#8M}<|gn%M^MmOk|hd^PlS(k~Dmb1K z1u^7&ntfz$#_PqPs||&i(<$>N%a(qow!pD=Ju^n(fyIBzLEW~O6OJQcu!-aq`gjl zD^?-8P~88Ye@}jA?hmd5AB~Nj|4MyUoqT5I=HXlanjHKqQ?dEgudGKET{C$F@X(?? z?8r0Og93YaEnWJgopQzFek%dMf74}87k9)qO$N(O;9Ewf=BE}pSc|FZ&b8eNM54_W z{|m=j^t@X)7Mfv*&M#;kEMrZ)r72@ zk7c}Wf-aI0zB$?`*w=zUE+rB<|Mpn+z`6G$i>H!&AOT2K8 z4S0PJbeU3XibP(cydTDe9g_MM&+|j`C7V~gUngimD@Jp8`p3a@hr2H|{N*oQ@m;BG z+Wo&az1Nydw=XAWmi+D6LC%jKv)-+k-{>q-BG)f41Yr-yYMH>9=5VHKMN==%5qy6p z02ab_H>f?%^q#5;xOH2Da4b@kmdy7oS?eSnR#{M_0CC0l5_u@Y9u=-4z>BOh5F*>P zdd;W8YTH!BAld#8Bt1ST>RI{gKARH(LXioXyzP^) zni&;%hl=OS(j~QQxB-@jBnFluIvLKUD|0=Dh?c9M{_hfyz2YbZwdtB)wuh#+0k$q} zyQ3Pq{p#q{4Z8ED3`q*OewfqM*sqH0+IZl+^o)U6VlO)&rwp~2xgm8bv3^HfKXaqh zLjljkB7DA&u?P>XmJnQjhIH^Gix8x_)(dB8Y^V9~$KT!Bn}Bii(!yKAh(vttdPt|W9fK_TZprp+zEhZ{J$W-*)(QS9 zs#}E?>{D?A-ztWhx>}d2RGUk!2t4^tFSUlc@Ds3EN=k!zJyB1U4{Pf;YtuNY$X319 zQBJ8-1&C%nR zRo*?tfE3To(`+qFf`?|s#U!reK9&-`NNIj7FV6S~lF_h^xboJaK7k~8FSU}77<62M zmCYng<<;H=k350e76fbdb~W6R=X}*V8m^0_^yOqi(e4)tWO;>@T8SqXP?n1oO_MfB zzIPa;FxCe-OF!;j^Z7Y>OJBBMF4H8pV#Srb;U9)A`tyQKmJy z?Av@*i$c8VFeI$lwC$-yR#nQWX@bu@Be?FiN9a9W^llacv^PfGr@a)+@LH|XOq zG(iZi-nC(F*}Y~bAvn#Ejx?|}nqsh&jb#G}4?S$3oj=Cq@P-f=7u{3fh3dauYJ!Fq zMovRXD_ft0zk`S_8QxPxm$}HD4NqXHrYR`OFlHjqHy4``CFs=y_}WoD8wZQeOHOH12HP`&hJQVp zS{puos-&C$KS@(v)ffK1@2mfOH0i^)mgh@5|0}_AeCMz4DTO>zPpr%|C2=Qz<9Vh) z8UX~1fq2sJK8dNK_3Tyye#IJ_Zbpze+Her+xd_qUabxJ4Ox98zp6_>U%wVW5dw>>C zo-I$w0f-^-?46k+@sb${l0_8}UH=I;NVwdI!wWX9we|u5BqhzaTX&iDYei7RiXdP< z8aw*4EZUfACq0kHh|C?Z3Tu&MG2@ok_b7(R0&1>{&QE1-6W#Z7ne){fdWWIXqd6?` zG!We8#`X#wX>-|M_kIsUP-gB`LkMDzv9WG)l5Fjkh7RWf6q$wj$L-r_;f(T`JI^#l z-=fHTE5q7U+3;Za^$s@z?fMx3Y_5SPKaFce^D%J>uzgK}hF0Lx^JZLRv!^vKoFN`_ zw9p3gWj+o)`Y5{wJTT6b-30$s20UIZA}ndwcR@qVq`+D1CkcrAFDQR`MjHkIL> zIgV#psYLYm1))6ZXqHXj=}TZoR^ zT;hbMI@n>B2nct4gs?6PXFmZAkE1;|fouIL{wuEUkOD6=Jb83NGOiF$XF;PbepS`V zh|z5IFBlV`XKisrnCp|⁡6^gYhG)IF!M%A6p&lmwJa()bWw&j=J8Q#OIr5p!3$e zCHrwy$*K6B0n%fn50pggxj)SCSvSpV15$%SlrSS2@9OE4NZa_$&49FVrHiN0yM66(*1GEp%c*nAsccEiUEtHjoxa+ zL(w}vmYeC_38n>&VAuER2R4mhOC#}3^X(g-eB911v-7{WwCMU<7o-R79D?7XfJXPo z_4HlbNEbs~R^a*OAT0P&5vyYQ!#JUpYtO+37Zk{FSv@h$uvI{_=@67~-1ChK(rG;T zdXEJ@*=vq0b8UVdEJY-xNEQx`iz-z(T#EO`-1Q_yAg&j^2eRGKUX%&dvtTmAhsw|6 zdl&>fbK(4vkJ?TR_9Jr>>&t#f3a_W5R&U`+K2&#Kd}b^>URSN^nE0Wg=Giz*=I={o z&;*TUD3U&F&I?Y#A}b7jVT4C7Q`>aYDV;@JwBJQZ{5J}v<5F6ri}(h~dCd?^4`OPz zo9zPY6uoAp$2_-b2eI36>C=qrvzbikRMosY=kyHd5Vc3P8LYm%f(mzB!y-t2L@tW_ zLbFYjPcNbcG}`eDqG`D27UR3zy6*N1LjX@9f(KN8A zJ4b9i47xPo#;gj{!|@&R;nVm<&GWg%RMmJtW#&4pd#-=qhANHwhj#1a$f1_Z0f} z>so$4RzFK~*S8?pxp~naNt!n!5-<9Lm;`nO>yx5I03ldl5Jo3olOd z_=d@r`i(W{;2=rv24*S&{_K3u+&34rtUwVa_1|P*5#!y__QHF&9@I#%%FF$|j4{J@ z>HACNmxXT?eE-i!3h-Qvjs7o$eE5pUxch{zI1NHT&1SpsT+0NI4F zWhd*&p8MT@00qMNzVp7%^ZR}O{5jd=^MlukU%mO8Eq{3-_q9L#byMOGzkj2|a}ps8 z1s3i55JLST0y@mq$uR07^`+d;``pga5!HDx3>rfxffY>XAqu;)8yl(%`2YrI!qrMa z?^KiFesE4?fa_|gEhWit6TA=frX;VP+~+9M66TT7)s z_9>jE9Ixq-wuUT|m%1oPE2AtZAm<~?P3!;LDzH`ICH^AaYJQ_K6k@;{pyl?*V!a^W zvKac0gyG8wJ=8L#bj`he8rKV~r)oZ1=n|*GSy?skhEhW9)PneLOU-NO=XH)Gn}L zhGWW_qeWVhDrS>Cb1f*9QZ^MCgU{~z_7TC`qXZr=JP+NDHd_F&Tx0vA)jKA!8^N!> zwo5Z$Loz9)cMOqYJs&^WYY7ZFs^q>xO(-T5AX%854aq&1y-Kj3%*IRbC+Pj+m78dK zpU|HuDIZCc7;}Yk-MH9y0O#M#qB~_Oav#&MYuo{6SA_A@=^k`H?NM52iJ&sx2Xr56 zQqaonzN1#}w)pCr(|IMnXK%NG!D=X9d>HVQ@(O4?GLS;^oGs?BYWscHU z^h@8_2JLgR(*2VPVw;*eK54o!+=I$PJn6(S77Q);ojH_44HHQBcdZspGEHiZj4i1O zzLS^Dc$~@+Sf@zI#m&0``MhDITWJl{;yu-tfF_ia-L$}bl&Y#&XhfYTTXT>&2eT92 zd4m#Bj~7{0zB2>VqvKx+sfkVb3hcSi71Uldn~)t|*O7bz7DK0?d-9XqUy8#~5I1#T zE)`4LQ92@mRWvszSE+nKT{3jND#>M-O^GEvNmtZ8Xnq3DL}TZZJBC#x~iBW zm|b{ehm>yJMFir(_A09Gr22wVDGx7qq^=B_K8+9lR41$&Pqg$+3l>hie5q6$S`_7> z)g?Ojg-qG0cMRYoSi>?cFC5bAZ07j_zQ~hHqE%8DR~wN%@~6m<1`S9X<(ZlB zS6?`L{;)bTJAu9?dg0b5Z*BVR{jE*<^6bmHcURv1G`{Be0SI(fJ@fQh=}=4C`A4HM ztcukS65CQEew4Zuy{(W8odh-#r-Fa93Jyk5v>W$A z8&sugyJqc3uj(j9`+miP3^rpy3;rhAyE0Y3eCe%{B#zZbR<&x*t7oZa$z6~z3PSUKO=X?tncx|Xrteeyu zTU#{=3@u*m|1C0PU_pj{+bIYp0x3GYi_&gQ;EQ!cE<XO)a~X|+UGk+6ef>+VvH1cms6CoC)sQodQ?nYMX{ z2(d@Mw%%loO2XA-wA&8qMHy?z^3wvHs>pv3YhSJC%1~CUyY7fZ2+s_snL`DS6+8C0(*>!D>hSi`2SUm94ut)||mKi=I;77+D_(1y67;<+_aGIf0qF0iVgKc5Z%B=L$IlqAd2?B-xP=MgJB zZB95LUT8xqu_Eeps=DIo{8b)={uOndLxy-oyCZEeHF(1&J_wNzA3*p+MLU?EL(mrNq6}Q}* zAz3+uHGLhnPg#-@PV|hvWw^@`9aJ&ON{im#d`Cg^HqboH1S9xB^U%@lq(qA4a3sFp z)r<3s0NrLQN!@Rm2L76HHPfxB!mXdg0KXCta%jt^E@aT>Q)1J=t>`R8bzbdC#F&#@ zZXr-bgICh=%_Eqy4($Tyg@}YDK~ZGA?ulW-CKI~hxhRs@B52P&zRlJKQJD$nz-Mnn zYhNMJ7PKnz>JH8ZqT$$v0x(nMp;>Sny#X@OL+x1{ak;}65km~OZBiy`R847;>CRGu zrSB>Vce0`;gVDM5SBWE4*xwvQ2s}KU2Ko7gj=55Qgr@wJnA&0R_d4R0pH#RS5$xbbhBeu_T>3e%rx=))$yvN3Q>?sBob&dRk}>N2rlh-^g?XAiz2U%E zDI%U+Wc!IbHQVNTd~xa4x*9YcaSMc41VSiod*i*#H(z_@&i1^2)O6GzSNB;yKxv;P z=g>>3Q=?H^azaPab|sRpwYr&Bet20m-<%>E2HcEeQGZVSTGtD` zYMq-w2No@j154MnUPo$`(EC#(|Gb$HUX+A%T)I7*-zy0YIpCzT$NzP%UH<+Zw{H5; zrnM^4_2e^vPl;r!cJD;@iVUN@Cn7JgLaS#XXhg1gwkyY7 zR8BP3zq$<$EE0kicHWex*q=V8WInrIBcq)y)!KgwE&@dGS)GsIsZNwuZboL);})c& z#<*1s;Vwqwg(JaLbp(1`MUZc?&OFIRUDOM5ixp<0gsJ+^mZqou*V2;v=ptBP39#2d z0Xt;`yos3<;??u}>RWfJnIrdZ2dhO7hqhSrTciEw_~4{=Vl3k6C}LC6e3F`BU1VTX zKSv~g)urXEX4du2MkOn_O=tIye1I-ZZ;3^BXUWE{_mc)<0YAgzTY6Xdc6WKL+0x4E zh~KBpI+Fx3>0RLM_;|ySl<_L;FMw`iR6xN`6wz(Q0<`~jBQ|gd>nMhTN=3ouPpbeOuzNm(W~);95Gmv`BJM;?ZwY)M zDfmK{(t2gCZ*_rc=(l`PZmX=TJ8kvmrg z9;&zVbM{uBT_CxpLevDha`9CDzJiaobb)hl(Oqq;f|{&>XLI7tUAUV#uTd=t4K=UK zFD^H;l2!vdXDHVgnZ7gk#A z>s6ZN%H|IlkA|XejGoHv%;DiV3B)mPB6>D9$M-oJG?S+vT&P?;V-m~{MT>guf8d_M zh1clqu}J$8s#ZG}l#$d7Vn3O-FEoe`1WRm#tiWJ@rD!OfW#$$y4qnsA6r4G1|Cdp8 zVQ#{vUEeJoERfexK(N3DVgF(^^N)gHm>9La^=ER#xXZT=o;#$_(kFNjzAm3?trB~h zbG$9cGQT>~|9KufnY{iQCNwq`Sd77F?+b+#YoQRFpTi(~q-1ebefmy>TCxUk5V^== zlVEG3aC{{>B7*1LfndlQ;>-0<=Tp0Z)yMD}uba*06u*mIUu^BV%%bV*c(yqsx=8{{ z8;yGg9_S|{5VTJ4LmEC_zNY$Kp?#b-Sz*;$;Ory|uMBM>RJ6er!`xskO1w0lbcSd7 z^={76m#29_5PESXmJeNWf1ffkD}68@v|&Q-^@;ojCY>xYLHY~@4?d;HPQoR7ZJ^#3 zFwJOsks^9YoJG{5wtt_?qKV93eqKO6C2qOjXWb417G-p?UgL;r4A!(x4%qh$tem$- zFZ@)zBh$SPS1P--@#PcAzRK>= zigLlv9~J`*--S-V@Y4J75)yd*lh{FJXlo|j$VO0E5$)TCa-)0uBkjXB5|_5|GMc`G zXTqjLvA>c8F#9s}jMn9B4pz4x-ZfGh^U-&o?*8A8zj*8WZ+hw2p&DI+$ZPQXry8}us6eCHSpjF z!t8E!>C??wW}?U(*dto0X+v4Cxe(mRD_S}n898ajt_!FaFbB&f-Vw!1S!$?E2zF?OlqJn@mIu|Dz#zdK#NoJyBM&Hs59R`rbO)+R>b- zc4v&6`{Rg2C5j(jv@9+?C~~_cxC&o38pynT{al~Fp~9N^-Zm&GhYOK=&_Nn_JM`s( zaTlMBCClZZr4F)sQJeHOF}{Zp-2LhRQ0zo|$0el-UAO}v3kP(gbW9R8)QSi9njz_D z;0Emcwqadv?r24p6<~wC!KgI)i?FJrnsTI-$ErTk*p7AH9Emlxvr1P2!!|*Ut1ebQ z6;@$(p+S&2GqgT}2rsQ6G~mkazbq;DrHZ7pi?dH6A&~po743VG6L>#X^T1Y)4P7#8 zL>ABJH8V^K8oUb1Wr}(mYv88xZFy0xkdu&1&1b|?O^14kB62t&3YgQ!ntANPsRdg; zOAWstp48sWDHB5djZ}GqIVbqiAwlM@nN3wLUp~dq!6^e=k(c;b*R$3Aj`T#p%#&33 z3e}%j+wmokXH&3JxcGr02TsSmCtBS-W)$m}R*;G2nB?@JQW$Gt=kt6^jz$8rcxsof zGFsY$7XNz(*Q0=Af3^0#o7P~E&X%UEEJ|FPp|Ul3Y*~u@L5pkgpxXYZU1$|AZ<0-01z%}MK-Ji2lEPJY4+fY!GD0qr-Sh|K=>6OJdC z0uwi}Vg@q+oG6aT2Ih=T!L=~gBHcU28<-br_*BWf7SMXJ=4X^pk!y)x^uGQLr0{A) zbr%3GcASVjAeeu@*68|aXXbtA{VUfk{U!IxeanAZyI9eNq06tK0d=LeG-^wOyb#U` zRN}xH7Z6DxRUyFm>WH|`Qd#Ass-ol`xJ1)efUg0wi-K0&D-T@EFSA6Aph8}T)!9L! z8lUtViK?Ba9Sw9>l@o;t8Au%3$=>3Tt&<`{>Lg+16+Cu%VfcDPDJ~?VV9%r!;yAV| zq6m@7lZY3P93}LY40VM{YfiRUTPx}Oo{>O7vSDFJ`xY@#*_-B)w*-x(J_``f=OkS| z9&)4M9J4tu5;0j0*)S4CE&KDNZ-oe@i`D0RnS_xziX(}FC-?-+W*k`Xw{gS zrRBoatCpp1=EL1_(nKKCwQF=;>>9I9shHs7C-C*YHHi?)8hTIAOkdz)7Xc5-nJFo= zY}VKx&dM0s?IXLRLr)cfAToPLL(DnF)f3--I&(t0^Y@?pZtru4|L3n4{{Hf%=cOlU zG79x-CHVRlK~VvjS@dpN^rn*iJtZ&Xop`;z@b*W~$JCvAS`DsnDG}2Dx%}bvH;Ycc z_VOpcelCqumrotvBV1rhispU{4$W*ON%oMNZmqAW_lO3YL*N_otq=mPKQu~QWH=jA zcUG@W#%vquU9BdP>Vxz{!sY*6nWIEOdH6Y4Lrw|i9@A2pfG@ADwiFyg=uc?+e zhv2*1j=P}=+P{tMcQ1wv3=$lWMLY!6<}jBXOdA;^6t#RnPk zOPhqDJGiXi5($K{vcr#ije>%Q7PU?vu`4rA=$mC3H*Ycdr)!4 zafz(i+E-$2288M-YD$y%?6IB3JY_=vzdO@=Yp>4l#ng87>`IrupN62#olfb8IW~Qj z_x4r>G2(6ZtMRX3rkua{>y=#xnKJ2mnat&wC^MH<%W{K@p%2cEU$tTNwpE$?UX{KS zz=gn^yA#!eG#9xI!7PEJpuu-m@qMzkRNGI( zruC0`p7%0wf#ejtO5P&s*;XyuRH4SVcVrTi?5|a$*fS%N@mX%rbRYUNI$Wm29;|TY zV#<1}BCHc(0|KoneORRZjMdOC4)@>^1uOLlsxPhTY(#4X{Hw%LlDL37#m3oN_=VbH z?d)bi;dQafrxeUfy8rjHa_@;tHXb!Fr$-wOdOd{l1!O?)1#9p+@L{G{OWK_(*Dpg` zN6PgLN{~I76WFK{6&F(FVNH`Lb1Y(H<~8}_=n}Oo${}+-v3KOD!VOd=7VH7je;u#Lj(jRjhJk>Doi!H z3?C->erQ}XK0$_u1l!nxuWe=4veVXo3pTogxS*>mCAF8|%M&hoDv@*<$WU}j0uDO= zboBkS$zGytro&jNfA-g^Qh&9?TLnM@y~FzSudjhtR!viM$@#VIFdL`JBykel&P=P< zkJ1X_8fs%h!=z-Q2%-)(lLXfPX<%+#Pf0d>dZ!H)cwz<>i+gn22B)hEPS%@-IzZX- zF~@#c$SulDv^|qm=Lfwk4t@H%S8m-waP=Td?+sZ0{{O=|?VMZiXGP&hl49qz1|Htq z+^mI8gy*VA?Upm({qeZ81p`erMtlF*NKc8>7|BTKzruk_5v=m7Tib*eH{l7JBb|BB z6CT_kKjnyoPM^?rxfFO)bj3Z}_wp!3Q#`X+1eYNpyJ~)39v)D8_q1_+YTFw~L6_*I z@lD06H|AeTDc*alTr6#rXxbsWbEReRB?QBB$();*8KrDg>c+;0PUeHb|7ab!;nK+B z^?%;oq9{B4X6@~hZ7&8-(bo5HD#mbGJ0jcybyd>p>hWDH?%SX)by!&|^R8a^+yS4N z$v44Ip$cnw2u7IHzp%H^-zOzx0A|`+t7>dQsn>I8F>6 zM{%1%Eh`Te6Oy0*&xtyD+E>qi`Ol2RufO^A@@HQi{(MXNtv@jIe?2rj|KS_o?s(zn z*)LN6eY*O1W?K5-GV>@hxM4)V7t1lq1=WCG1E?Ox*8dM)D`J2YlpM?yIgFfxVutPC z6wudSKnH=sWQ@nuQ$i+Qz{v|fM28lqLeoe@Ol<53@ir1zu-!MUo}*}+0i^2NaJ>^1 zruvfxPyo0sKb^=pt$ zR@|X4$q~`^V+=!{cxhfaB^ssQjdiDt_=i4^zzYfwWXdvxEA>$qbtA19XmohIX?t`@ zdeZuE*De<186S$IUt$}H31Cpq4Bb=^+u>~zM{Zu^$}rcC8qa{Prm#h3J;?r5xE(V^ zQ@N9u{a^po5a1^Ue0ac z2*US_V=)MVs-os%uvw(8Y@QJqjeR-&F+jE6zQuwy(2wMLt5GV7qeihM!Z>I;KsafW z5xNp)?nqXai{K}#0}_k#lDBD7e-394hEmP-SLO{2(XoMJ6zA_H6zpblrO zR~?|^VRJ5J2+Sr8q4(U}xJG0_Q^L(kN9Vt()5b~2%!4V>s-OXh6e*P^XQ9W=ou^-w zEAbVB8wPZ?Rm`Y39I3E8G)fBMp~a`6;Yo*OJ5*YdJypYMH>%Iecjr1z+D4`xY5i5M zJxTZJYW^cs&^632uc?C|K5@>&10XE5_nBquIp z&^AGa^hw2+(A3sR&cPm>EU%&7Ebj8J5k)1kBLKpG4iogITQyCJMChXCf^zdMR@z9a#45+6{Y|n41kW>0j`wy=vPve6v zb_4G^0;19fwJe1+@Hv8G@%i=pFvO5cYZc8MQ6ErH(2i5po8a~B_{6^MvAiUUNf!zY z9^I>5suSyGaXy)g`dO|R7ZdqT@NT~RoqtY$bmatsRr_jdsPpk&u-85 zN_L*8txoUSBuwFFCx!b^YCi^o|MIsS+t<-dWpW;5RB+B0L;d}tV=LWcTJAv=jsn~7 z;IYCvRo{wKx5nuO5Q42<1q3=`MZ&GLbRh{{e@E z<4ELFjBfu=Rmsad$VR%AO_J5(O#2X4)6C-ed~)+)MZ=tGL8Usj7Xjl_?eTFmPQ znXA`Pj+^LGSv#Kgv=pK4TfCX6^{4Ym(SHWGv-#bGcFH`+TPf*{{Ql4xwVQM8>FgmA zZABZt;EyhmpJFJTsoHXdEjKeVwtbSFsDkW&IG*11+t*+J&AImLpZw(8uFOI!EYPk3Ifp0E6y7=6kw_?&?xcTyM?+>=+Z=QSa1A|cf zn|J7Rmi1m?lT(yLaT+T5?vwC%mn?DAU^oe{tCghn#+1@WIjBA3$poI1}k>h7lR93Jt6w<`C~gxR#{h~m;zrI zaNkr$e3xiVI$_y@N$yj|_rxntC0{mHCKanInx=qLB%%1c&W0x>FTI4C8b% zzG-K_cAvVO=zocIqKWo+i3A&xq4z`~8Ajv3+zfUOWrmZ5;SaMYNhD^^P|ZhKRA0+b zam7UHgyK`Rly+l;4KK5yHLu0PmyEA|4hTAWUWn%XMnX*C;0&FO4Hh+hGkNK7P}#cr z?@B{^64c+GuBccOH@#j;KUOoE+spCm98+|*`4L9HUSFw5~cLul#IZahpCd-P{_ z%-$Rc%ZT7=KVO+%E#}TNRLsqOkSUh#;hb)$vlD_tiLNEkR*);F?jh+OB=p(IrH? zdwIo{Px!KPs85bYMSZg8jh{cQ+i5x?Ke$3W(ymVEr=IPjKCT~s`*bj&eYsR@7qv09LjkI+`KA4Ja5&M zDsmox;At|V+&NncqEEu3Rcb>DGyLpqitTa63_BU=d5J}{?~g6WNlG~98e|x?!I`1^ zrJf(rT{eb?J~c}b8O8Cyss{)-QXu25(OLh{+;8J&~4 zC`?D<{zkT7v3CSm0tzF0`Iv({t%AH#^>tbPsv^mhkPVH;eElWYM~Og(Z=0Lb$GakG zv)Kn{MGH}c{T%kGTTukvvfgZrK0jnSD`CJ2hHG9Dl&7vCVDNl=xBPxGpt~J-e&)Zw z`}^N-y&`@2!@>K%dG6OAubi!`|LWZjU%BwlGk<(7=|2F)){HC(nffo89>3GN`abb1 zf^kkN{_NL(u056JMt!}HFp=kapdd!$;u0C>a}yF4?(W;unatfqDCydXFMu5{XFojM zYQ$Tbn&3o+raM{Ub*|K@@u(GF}=5hM4?~W zkkH>1FV6@+1Zpq>eK@1f9c}T8fHvqAWU)EsOIX!F(H_Vq%iO60(=veXf#CMFuw}ck zVc&H2ZQr~@2w-HFy49eN7qJ>Jg%72t=at#n2Mc<*h1^qrSSd1kChMoNclyg< zj%EoM(7-~Oq@?{sVrgemap%@J!iCclaUJFX1IGzioah3r^IlCZBC*`Y8o*5ZW1}@b z#5g;dD_jE%i1CX^StYX5v8)yNPx49(AhH5xv)3&r`47{>31~~6bHF*iOK5Zl+v|GR zT(JX%7Ad2eGy}!P#~($KAKQCTSu`8)lALYgg!=x!ePfn{99dtf>`$wWyu=ErXlRY0 zHKga1F=cIRf#VChr}Cn3%I}tNq8aB@TFx#_7t@aRzmjt{j{dR2d?nr}kpw|A``^ZD z?VDWjJ0xeUV!&@ykxdtUPmqf6yl&R&+o1>!TCuqj{k=~~wi76(VNJ@XFE0f;UCETa zlr)*JNV{;Uu28h&Xp4%bG%Q3a|J}Sd_KNF1*ruCA&YA=;?7^lfuxE5H{wdvy3lFsvYwgyUCO|fkb8TZ?SC!FukugR zmPdylXr6Gfxqeqp+xrc~=cf z0JGaE&Lf^VT7R-UZ>ofQuU%Pmv5(Mu#4a0Ag>* za0t5=#F~f1TlO>>e@^V9eqT3!s&np%1_;{*Ci#)O_!qRcb}K58)U>zm79wn_()K4~^$i^0s)z?8T><v-LF~UN3S_3%&@qDsWx_qZZ<*od4k7J#tYJ?Jg((S z<~MSdZX{X;HbUGAU1r6yUO7qd%y9eQt2rSLg*mbxyDo}$+~hetr0`-!*eDTxU#tu5 zFF8HXDW7$(F1-z^^Vg0Gh95lmJo0>gOZCFlf^@S#73OdY3l&cr%CG-}olHkom$}8Q zY}~+_Lt@+w!E2#)&t~<55CZ~5!f5XjW~WL_yQE0;K5W%E5Kt*KB(~IH&BK-^e6@dz z1kM9;pH>0R?@zS`E-IRm7XbF4!WnupkJ7{iUEJtc>*WmU)%|F%pml_t>(KLpeYp5z z7R1LNe3Q99?LyPGza8GO{dakP{r$%u>^S+__uuV*!}RHA`ax;kp&8=VLjPCc1(N$- zHEd$jHY)#LG!QI;`kZ_`_q6M&IszSf)@V&U!JbzEyZ0SbQjjyWsbPynwDEm@2SZv% zlqKNG<(2%hI54j`ok{LDYf7h*VUALdNrv_NTW|NztUWm&Piab~Ghbp?7VrEcztfee z?0PjLH`Amkr3OKuR5vNCHk|9HFHMiO@6Q}ht-9leSwi>$3+pxT{7<0@+aim|6kEXca_tl z)VcATuHsX=fH@LD28?BYt+r34xeIM%H2C(-PijSI(!sd41@T&O{t?jsgj@6nqwrMe z1tqB+a<@ga_F}@TXf#`}GDlD+_^(?TgWT}^R>JB`2VIC~i~~Xv!7tRtLyAJ6 z7yqTrVbMp%o12S) zUGh`Y~aEOo*i=u=*+ zRa36dkgRGpMZn!O#r{>)`A|LSdg&e(_Z(zfGZ*Hx(76M=GsF4;o*`Jjs?x>7jbk$Q82Llz&zBPBk;?(&g;cGX;myA zPb4Ls5ztKvSUt@TapS{7rFDU0v zoH+pc6nBBvZZf8EHfhypUxSJ_?p0l#`E_%f;9zgE@Z)G1SePcLTX3G0yHRFwVr-A> z{A=^^J>lCK2b;FIoDz*%*y=iPI05LLc=l5m@kAwFQ*l>2cjC`gZAKQs@nc@j-V*Jo zf%#U7koI^tiRx+Y$eyN-X#qzN81f_#$T+CToH*?A#GzP|))67G<$-o$;XX7)TZZWA zs0vh})V-K3EWdxm+&)r=I9_3j7-~$Z2VHKSP3AQ z`Q9t2jw1u67?gJ770`)eVQo)Y7R##(sZCFCrIkJOPGZ({t@lnS9jyaF@5yxutF z_5C9jkz?@5lRok}+a%J%^i!}0`2L#X{2jA@D}YL6##@ zNcK0;^gzkLSeqJb=Ym(ZPG?4|*@+2Zr*e(bZc*;l+!#KnltUg7F~Vsb?fMzFg4 zQ2!BSjSV+dJiJub*a+{1q2(%W=*l71@25O4mFt zX(@q;na&GvNcL$UPGtq=xC4eZtS1^7QsGsa_k#-r zl`uP3AwNW#Tfd*x)kuI{tc;p;)+I_gyN>5wU?pqPrqV>qULi5DOzw7HZ_i+~j7*Lh zOlqSyX56tB3(7)^0dFzZm>KoIFE$+i+v};n`|Ytme)3#g@9Tem?KR=a_rCwzy~2+k zUip*ZXT~Z@9Lg6jmmuB6~I}$ys<~>D7ose?FiPt`X^mt)WF;*ub0A6k9ZO%R%TYr6)ACtbSjx)(@cdI-9+~7uF z`1~m~dGS_+HgFFZU{VL3G*%c=py2LkfsKrV_meyq(Z%L*0@-Hy5O7;vRzI8)fX4 zas9^R;-?tl%1PTU36Ln>Dj<>FgAdG8-*&dKfK?#u^dRPtUMio@{4`q%9YTQsERRRG_V07MEx9zzhK%kl((0JWfu6M zv{}Q$!Hj`9Q2t0-iDX00hkf3O#l`F>67VdsaW^i;LsW(DjU0hQ zw?r-EGcRQV(e%UXvaBuU?fC6^LncluNM)$V8QA5Q90bs>Vn+`EG6E-P;#ow?yK4)R z3i`YP?>+b9;kiPU^CwF{eS{3=u-e{rDJ3*eSIzc@bhDWX87awqj~n^{buT4;7pis_ zG{!YYMX(Em&un6{}(vA!nob}OxpVBcoWS>u`*i6@HLqJ>7 zS7KRL4Y#VxLxTNLLZQFVx-M-Punb;su)w5QVHODa*2ze8Hf=1KLYg&80M(N#?4txn zybh45gR*wW9k#L2JNqo%SGg(A4;#Mhew1GuHXv8-2pP{@0QqcH*_JaS zQ(W#R5Ku>l7DPjZU-t;1BAP%Cs6DQYTo_D5CNoL?7SwXg!T{AhYYnxGNIa>M!3(W~ zIgulv3%NfjT_3w6i@=#qj%R}N02M01HXbpMgmmbZ-HmjVAr|6iZf(MQ8&M+t9?_UK zY0-V5I|qNVCvITHFLt>w>=OkASrngC3qTbBVNMUEvJW1xn0s#a$hNk|VRH0eP6lR4 z8sH!3U-300+dNvD&+8~3*?%#0d?<=^u=WgYFtMbONl=aTndad<0sC|a(#p3F_H#&_o6tIb=-D}&yG zT)NJ4C#oYT)OwU)i7pXXCwsfpfV!VGC7&`&CnE+{Y9zKyKF+<8Mk+XCbvGL|CB{OH z`zCm{&UcLxnvw^)`QU)gG4nW1Cn4}3oU%W3`^s_CmPQ8Ssg)z~p!h^$jY70M6n4Xw zEgy}2-nse5=eB+O-yc8x-HZS8+<*S|O<^wT#m`^=y7%Xf?@kyhZZ$CaQ^ccSfqxph z{Bsw*OA;1VCR+Q7f-a24cSQ}{1+Bz`WTm++YO+alyYsqwG!4Yh4#OPM0mk$X<< z$|H_72j}fiD@lD`5dY211W4V`Lf4w5f8_z|y98V(SvEPF)~>cMc^i@TXVsBYlIh*i zWRtVBP@R%krjsGgrSENc>zCsuq@}Rns*5_63%cz4hl{$w%~KtbY?l@!P^DXQY%$0( zc@&9L?vYN~&X3@d7Nw0HAlqBcdIIPIo#;|!IH2CKw#=)kJEc*_1f;4%gXAP#1&auuiv&sLBOgQ?J%8o6cC zic|%D&70gQj2WP4lT(Oox=p31TVhsw{%QV%cZHxcLSL0z)k7}D-W>1dZJ9Z^T<8Sf zHjET>rk6!iMvB%%D@)AqwFpx05k+w*6FS!2p3g&9p(3+6n&=Y0An5BK$`IVjKjbyG zexz&++O)k|ef{-Luf&?&Ko}+{W;}`}co#nx_Jy3%v2v?OuCB5rpnJOT zLifMImyZYAPd63k9T34HRD~dRIpC}7S@wyAv1g*APL+2JBhiWuZFXdW(c(9RO5}53E%HK%BL@Fatu!erNC?a94b0}VcZl5 z5|f-Qc!_GCH63{Dxk08&!EL!#Y3q{XXdv6!~;jd9Moxna5$*C zuu0j)oS0FHwQa!w91L|E(r~69w6IyyBNVFG_OleTh7KEU@@zZ0BB2c=14ajYyd>R9 zG{Xmnoo*tduk@60i9J+<;BZCc{?1avU}4qBMRtVfnDTOQDP2Pw@l8Sgr^AV}S%{G4UyctsG!<#RkJZSKCn`TiM96W3S4eN z;K2jqfMRtk371f|2_yNcpogX)+{I%*MB0Na5Kbw7nmC}gH4=OiEXbuLIalh9NV+ne z)7wAyXx4WeDk@rkyE88;O4pr`kns97l6&YL=ho#f&Jji?Ms@eHBDcl}=cTr-N`M>w zDMD?3R&0G{$;`;hv0rS(oef?~A^0QkbYHb(>AG%Y8z0BClETk6igCEWB;Sm+vFt5r z6JYRHhAcZ5dXE>rmKY7g5*G{)%(m@<)>BD1y$u#m?9l|i0SwX9fss-QF?JG^=o>}0 zJt4hSyX~XGf*5D&IZa#eFPd}Xf?8>SVEIOjPRFAbNpAM zT|edXBusxR(?lUM1}*zJ0}?T1ELC&8&@>^$Fq<==$31z`wr`^}&YB$fVtOW+l2#Hd zEl9`Xh-}G>3x((OtvtY(3Zm&!Imb@pxdhFn_8Z1%{}`{_Fwqk=BG@{DpNNYCu&s&7 z$)ls-h1&Wk?JD3Fc_!A`-6YDh4p6Ok8mCMG;(~IS_hfMo8ZE1`ZAk| ziqIan(B~E9{TOB|qHqn|#_P}nqkhp`as+q~)8PRrJ(*T)z5{vYwR5UvIOM4{S2dL{ zq}LMrh}G4Fx?^zP|55ZVZcU!)-uC;h`QBl*F3mS}Dy=|rbX+ZCt)$jUkPz~9oR(>+ zWGthoY*M9)Y{NeLJl~Af)FPxElqi9umSVIB5fK7pw=_ajb|RYqA&C%3fRKdj_fOut ze}Ei^gxt^lT-Wb+o|-|T&wf!&S?NwmVsvE56T7a^t*5#cCf&kBYyXSN!>j%B^>?(^ z5~*i!euz02HLRyTAW>Il$Cs9yk4Jupsb#IGCQ==NovMlJ+ZP-Z{n+hJ_6|<}fXsvC zQ?9F%YYf{B#j=dngt$Jsx3;UbjB;~v@VwL4J%-)E<b z+ceevC3(MSJ>ui;&;@!4#FS);Q;na@_*AbZf)?^lmBfs^hLO8U%%8AD-Uju6!rU$T zV|KiCF8Mk{Q*HeM;QEXE)Sc{E0VVKAfO|%OKxo{{p@%-!zmHk$_hu2|8k$PQvDaE# zg&}vqb+HS?z$~Fzb({t1`w@X13|}6kW~?qhGRtaZ1-Va$STW{|uvXP;E}Lhq-L*Fj zGG5fcI$^AZxG^E%( zoHcwRVmP`*)g!rz9tKZM^%2AFjm!1*t@FeE(8l9~{p-UAOZ#0f#B+xa5>oBPLtT*6qBHhjU{12V55t;!^j-9mW~4JV>Nzi(*P9)(-k9nm%lJlPOkbHzfNNXk8uYVDH= zQ)_H%pR-TzKs&#xi%E2j{wrKJ62C~_dJ(Oerwm#Ds@C*YUV<8m8Yx=4o1nI{a41F`hYJ;aDDvSv%)RyykO?wDwm+|TO@bY+w- zC@7z|cO2t1jTE*R>xN45-yw4o;>t=&EVcb zB?&XGSd!XaHavjHpB@}q6GL2i8!>z-Mzfk7ysMt zAUstD<>;!XJbyHd-}L*-BFnN4RVu@LqwnlGG1MJz%{{&F#^Z#4fAQ^q|M&;~?ssBJ z{-dq$@{6)R`>G(~G3!ij9+d&1aYu{vUWqh8|qM2F;r_YJQH*f)Atkx|*6V=XrR*kC%e2gGjQv^j}kt2I@>UdC&Kq^_FcW>)V{$ax^#A48*ZRgRo2ocw7zRJl;68b z@6iWi6xRIjMH<=yn%_!HKHbXKIv%J~JjYZM2uVTNVennq`Xx%>c^J8RSLgo7=6g^e zsL1AdP#~h8M6Etsz%-Q}~Rp1FN?n|bH}p^r1m{b+Cgw3>0@T@AcGmZFpUskr$?0$;5zLs5k|2wZXeq8 zf+f4RC0C>hyf31(20)UW&XJz4$;xe&(UU6jV`EqL)B=3jz^`d(H-&_I&l$8~feveY z1Xlm&=VQ(7+f*6GA#Gbax7Kc5_A!|iV>4~YLjV(F6!BB`|E!l(Q50F4QPv{sj-;)y z^+0cJN#62F@hz&_Z)8=3d^LJ6RebJAvCOb}nZ2DOztvUi@a@;)$Q(3u55O+mq>$FH zA!}x9EzgS#l#si0q;;%z&*~FT`5ZZlyxQIqRk7mMoxX^4-4DsKg=*Ss$2T5z4#*Cq z8VdbvjtC#p7|e>)Z^kR1ydpax3)iT>Ua>B6|MLpzOzQhoTvEfXG zJa}7Fz5L{StR_413ek!>1T8dDo-%4EFBF??04lnmKimV197%%}a`!LzLH8$!R8N7T z+FGBt)Etky8-`K3;0{~mSSwWXVfWfZ`9?GR465=Bv%M-Y#>ls_7V?meuK*f5=mq*q z`&q>&mX8I6`FX*C+6<4gY~xRu^Pm?-&EAnqqzz3j{c7&7UVn9gS(e$7bV4*xW*8zR zJ6}h$v@xlkB`}(ksXFAYn>&?z1_S3sKwjW^sb-T$N-FxPW*sj&NB==wan#T@`7W4 zVAd$8wA2x+9A{$0ss6uohRc(xdrM2>=$mHzN^`M^J+f}(riAH=s-?%)e!4&;yRBD{ zW@f&tS0a2(+~7Y&^o{y=ov)g|T5yX#E=6geu`H@x!|zt;vUQ<3^X!p4C>(lK?ft#Y z{vr+c?_gP+<}BsK{;=^`VQrDU8C8DPIn0tc-ayL|$m8hEPV<%3XJfEjGQV&~xWQm0 z6G}<3DN9=d;GkY^c?SWV{9s{Y>(7?L-sXwKCOSJ?lby|r z3DUXewfAg6x$WhoH|Dc;SJA*gZ`FEeC?6Yt{C3-FduwZ}@%SH0OB~B+|5qc6F->e! zLJm!hpwC1prVn8a6m=(;5C4)5LF&Jsb#|i%K-KA~Amqy=AUR<;exD@hp7rTRMw=UcH4kyXEm8PpyYfUH2zi;R|&-;HnUlaf5YyZ=J`OCe> zzEF=9^c)j~e^1)_2*H}il9;ByIU>fm=cOl>^q9rI;jewk^mq!|8Ig}W2JazS*^ zaFlHl9Jp$Ej1yaOlc5nMDd$#x%&o!TG(NB-y9okeOn7-9J-Pv20UyQ_s6U8dx$O$4 z$qUR;8T*S9klpYzeYisve#W#p63}oKLC1WSmCSOydwzfqxT=}r8d#pLE}1}jc4Jy% zE@K$02Uo-R$UA5jAR+bI-_;ogw-HcDJWN8LZe4jk4Ut--MyFVR%3Y8TAbH|&+k9pj zZty%kcyforBd+isOhm(zfTQJkR@_&8ESmpe3vKhL7}i}nh!I?^&nGWt-%gJ{-=zrm za0IOl7c=AGr3$k2`98tD#xB{?ijuhemR6?f@}94cA)O2*hKFLkfq(!MnVC}ItT1jekq9Xz05tF&K@%#EgSuPU(Zp=I$TbI65 z%fj|jrj-qXx#W{O#HRjc()8T!alwRk+my0z1RV+czEhQj=Cy!aG+r~xr9PAzFIQaX zh{E+NoDFr~CN>N;!k(?cDfj5z5^bfaIf+RFST$v70XscHT5l??tq%Qy3U0miX2*3KRxB+g zk*j*7OCv2Q1J^mszlotU)M^rz9;CIcjK6ju|6U88ypHC7oI_`}x^tX=58lTue;LOf zvim2G5p9QKM78PE7e`zTqC}hX!;FdG+Z?DU6_8-xOIiB7WBV>B^6iYJKx5z=PtV{l z7(i}*?MH+zx%Ow5MpN;sttNbjoYH4n#lVks?b+sb2_wATDX}I;-m^JqkX=Hc4riJG zY#<76l{+~KW@n@Ju3`0&TIF{SWX2~(4*-V%Ej))>tTav+QFr80y19evI;@0_BFmA7 zX4yQ|1|zS@oI)=no$PAf-h!uB6zARNfCCN(k(v!EV}4(2ryuqQ%Lc4IN_9D{K+;iI zG`}7l{!5MfhnR7xuZ`54wf!mJW zVl4ML(}cpIdR~0QBSK3T(=D;sL-YZR#8@S_dGJ2#YBXc?NJhzMAL;+Jv%~p`wIrKvvy%v)K|=-`JiPOK?6Hw-=~BM%;t{W_ zWToi4Q;L~V=}L@ruC;(nDCLU5!`L^24>V>m!|dzBd-6Q4PO%`|sh68yQ$XM@=Iqeb zn3e?L$2*ChUS!3?mJ{6B@B6Di1Qr*C?^3~H&=s$*3Le}sZndi4$;jc^;B#oQH`A7) zz~>}m>5nJ!PPR_%Q&^NgODL*LTO0;)Li^O-a+$G#-^3Y2JJ2meT=;GQfgi)&b9ZcB ziZk5H@>=qm7xMFdo|>2XZwM%bzID-+Au8TtyL@q5#mFIN7OmHUhab({q=oE?s+0B( z#e5sm{eTnd7Rby?=qdv|RJvoAwtV>=gzVJRWZ*(v?%SN<-P(J+(y@YQuh$g*Q(N^h zcEtxZslE4l{L{^kOPd=kEnjk?g3)(UQ<74c+GljDRPx!|m(MBIta3BZ#u&1KZTRpM z7%*c6u$Ws$nVvmD%&!gq3l9%DNxorBJ!UwUoR=)6xOSter=ADj%V?b+v~?+~qJ@HPL`7Nyv4sv6=jJ6uaiKUL*~-M0gend=X$^3P@nEWvBvf(d_V`%)0--xSC_9Z zq2V9g<&>pmYn>uAlHlcJv>e`FBkVd{n{-b!U~O(x3zBP;6-N87Bwr;E3LHeb&tk?n z*XI`zv?stT2e|DkM{}2Mfgiqp5QExm#a4xH?q!0O3~plx)gAu94CtNJacE0f>V<+E zM+pj3NFPz6QdSGe;h~+t@L@6Co>{t?mgnu!N9wq@wLNAHKXi@0gKe+TZ?x;fNA<5n zf)Lpor>*U_$Y=Kn`j;MMpF0`XA_Ls2iB<)3?r@$+G5`fKL0-`BVT4tR3eTu#_>xny zYK*F>$qpxW#K*!*C<{Ni+aCqhSo;)`t?w}sHpZiUhP;z~u$ zdC#jUS>H&yOm`(47%XZ4)But2Wp;c5?odLfN=q*w!JmoXc*ubjxqj7P5pq`%xK)DI z=SGs!Xk?h)QBSi@Nc~)e{g1m0hj>4#s{3<9-UYv8WM24?3%To0gr@7*1WEB=A39pC zyd+U$QqF>F2)d@euWMdfSjDoxozKttV7ERJlz3CxbD$H^{POCJbM9TXK@e}ndOaeV zG0V1_6`2@mL-zKsCpj&h+=V*`J#yrp#5kCM72skwJL_*ICJt-sQXJJC%1g!M6Ls^+ zMGZz!!`M2*%jua}&7faTYjSpc66~>FfKD)9{N(8|v>H=mohwJ4Qluc^bEgW%wzsCR;T^ z%>CvY7KXJjP-yteZ}*oYwSoN=@XP8l5w>|lH<&=y5L3Hy-|`QsP-V52xhqz*Ah~R3 zys2ZS>O9_!n7YX_*f|=#d|0f{o^=k!j_i{UFMopBVHuy1iqO=v?|+|?qaVO8R+J@O zL&sh>CoW`5b+eBK*10Up*677(Z_1L5zJo;d!#u?jN@r|0i9Dy>y+mx+KnC8`Q2@+e znW_vgCiNS0W@@M*Y9=$vnrIG<7i^-RhbjoE`s>A3%5$CPN^9GhwT0^b-Q1Bgz2iG0?wuyS{97vom% z7LYl4p5rK*_F@%#pU*r$FlvRVwi0My+cZeCyv<@_bsz*-GP9KHt{TcID-iI(g;q!GyuGD%0G@y@>QkzTnB`+n_MUCN+6h z<09oOS=rZr?(OKAFOeTWfk(~pVAr+gotf`zTl_CFV?@4}oPMXLY9zfw9;>69nWC-N z(YV}XikVCc|JKeqn%%*rt#q5kM&4BNj?;xYtb)zh938K00ks)jXCr2jY90v7lvQM6 z?81Gh5EE!hAZzLTN8UxU9SucLu{qKL+vfbXo0iQ+60X18h7)UR?Jo}#q6iB2=fJ+2 zIYMK4HZNVI+ulMi_JAoK9GZZr-3Nlth@p9vo@OT@Bh$(s6)<}nyBJmJcpTV5BNf3O zMXeeB)MNy0E#stGl!BkT6C~@hbPk*tT0vN8D>d=tF8hdLw70IfZ{ej_ znnSjg+%)e5!F7Px*nFagwJ-`_Mta(HRK>Y%Sbc*F(`+vvEqful?V{fGF6<(Ys+2{Vw3r(h9*hROeXv@G{o+~ z$JndAjY2MCbNTUy^6~UoNqN)yeb_BP@aA!=pPRc%ffeFG4$?T@32Bkx8^!R_Yuw?o zKJo$aE}*y43c#XpW@-EpsLl;I_TeSLH^A8A7nO()o2Krzrqm85Oo8Lt-3Bq&&t;%A z)Vc9E=(hC&G5);7i!6&n*%ojipfSC#5X9nfly-!o<>_80&e!U^>Odg=b~OWsZtY5_CG%IMfrvv?vGh zl+UEo7oWG!Y9%ijYDK8dK3B}y>#F4Bl47s>f5unjTlWuF{Ie!>op>vkRwNJQYfy`Q zIkqzCL~(-d)eL7m*3_U~ie@QrMI9-grUx(oF5c+$#x9n3!tgWuR~YkkwLe3EsPI&HqH@O-|foH31SsB=0@9kji*+6qA=sClQ}_Pie?0>*EIJn9L~= ziX_(2zUXK9#N{Q5VebUx_oaP{_3RS8PXbD1ZCG{B0hZJ5&8c#ZdGGCDt=qvL<-M3$ zg_q}*ZSZoSb!)BjVumavk#5E_=iii8gKHL+Z>wP*^V4KHF$!?G+{oWjAPV5i_qP_fP0d15=A_nIy2e>4U93#M` z&^OP3t{jMQCxFsZaev2?Pf|9!o$Fr})fPdKEl{;)u`wxrI%qVn|1QA-qK0qnQVnj; zsL$pmdrBCY94Kg%1c_AJvn zu$~)H_*zgMyHT1u*bkLcTvxMgwe`Zq+Iv;4KWq|HSN)hF2zK1QDTT~>2ElK*Tv$_0S$g#*(Ktx}`RFgFKI~G- z-1qShe_LqIJ*$gW^E7oQ!dEp7NSdKrBe*{ZcV}1JhMsijT)(JLi*Uc=A)W~MvRXKq zosfycY26|6b>Qi=m$PGZL**CYB?0bDlLdt6#*P;7ecxtB zmPvvKOpEafyLB+%%YZ&O-x~}Hb!3%H(%I+!P*=15_-wK1>8f>UZSO?CtcGgUVFkWx zio**rg1|4Jmt3$|!y@^o=?P&#B!|ty)^HeDZP*04^K);L-q!=(=uCcd+PA;M9M$S=S9wca;T?a8K*LKhe%QjB)G41 zYPo&xS=x9Po@!ZY@AupSH7I$?vdh!U;rAqaK0=cncO&;D^IU{=Kqlq?TxG;&|1e z;vyq-vV2o%)JJB5s>CkA1BrJrM1WV8iMV|Cb5Umu3aPfPHSaWUi(2K&>|6g5XC(Cf z+NF7$Y{N3{vTN{LM@*>squ$ak*lM!{LnyZy&oFIm4(eatY=G}rtg}7QJ7ACJqHwbT zZU4dEs{1BQ)dYF3+W%FM9Uo}a$iwTP55P3fApbftLk>U4GdvR3a+WV=DvR-zo^N|P z6qLY~idVNAhCb7R8nkan6*Nwj=84N_|5<>X#{2hP9k`P(4R_289idz)W>prcG(=01&Jpb+|&EBVy2kqY_Fx^UEux6d8GN%)n+cGjjgQ^jDG_^=AxP;Hs(y$ zh)VCdmfesGEs-vZX*TX6^n6Nu)QUR6M$es&(+=+?Q8+vtm> zqF^G2y16f<*841I^6U_0Dozt~wu2+v@lO2t`0N66m7Ia|ld`IB3j0uwN{-y~<$&>|Vrq3eCkeQ!Ak%zZbQiXd zGTdnf8C;e{O=5O%u&ERg{TD>S5Qqv7MG+;;CEyIy*ZRI=m+`Uq(4z^Rnrg2Uu||mS zd@ef}1%=EbGKuQJqTD}vRANG){HHnct!o;vp}}GFQl^h)A* zM*CQwM*!PxdaB04)VJ!fr%Mtp_c63&npOecK9??6TOTjoALTOaT)Alx{Qi=6?|5E4 zegyd0qB}H^GdC<{woO4&phKG-amW=Rx6mCEPBL~#1KF_1J|(Vl)zY7zh_ddm9U2KIsJxq{ratV>wqPV1y?A-Dsgxeg$wWXYRSQQYy{8{E3&W=E(FedcFUb zf|g*gpoO`3FhM&A29VkC3@(Zc1XA*TDLt|ks!UQK(MK&mQIp4N3XqI`F!VjVI!H>G zB;<~$Su@2PB3J%M;m-D^=96nSFUw0*M=%8nM+{V1SxbA?P7HKoBQ7u6pBf$$#8&8s zpD}!at{O{p#nLOZAPM#%^u1&tON30X-T~sEb}wxcAM($4(h(yj)`a=q2_7M`9zQ!a zB#O-HypflgXEDF22FQ`Mn!Hlen121Z!c+&Jmk4CjF}O{RH4d_7@7|}F#S*<~P>=g6 zSi`SnCuZP(N!=U*!1r8uY^QZ`u(pD+Pr`R{=jIj|FZF_8WBivSHw}?mTiO)}>{M(` zXEA)rU9#m{F(Uu&-Ond4cID#2gdO5dfS(*2+u5N?`5Ed=n?NHI%o*gjlpOfgfC@=0 z8I@Ly9GGzRC@A(V>&)QnY|tC|I$0}#+ii2^Z0|6Msf8zDH?J-}1pDV}7t6;ywuoZ@ z@+s>O5|;TmX&n`rrh(VCA4O71w|YJWIum{!T?=mTJ@|@I4)w3&l9~I>Wim1@_g8g` zuGvx|z?y50V`u}|mgIT81iSxdXmk5%f#EnRw@c6)g{wZWt#zOaDLFrTxo>g8{qLRI zz%{U9?9i%_U4|U#yQw@P+RJHi+u&cr-Kw=fehj&*39|S1xU?6+R#ga^z%8dKi#c2JWwJwt0= z2Xy)Tnhyviq|hocR_pt&lkxY|K;U2j$xu(E_6^!Dbow85Qx=F#T$%hHcWUE9$f6EC zJtfzfk4~M9ylNT$;Bp%7rNTSC4XQfPyGD-Om(8Y+i(}&_9>$2zHs4O~=tFWq5O%#L z?|h(9ky{q~V5bB@{o7&qYY^PZ;2fSjIz)e$qrG3_HKkw4z1^W4^luDI972Jubt5x-#3^-|=yV-_j;lVkqCP799Q!dVLo zAk$qk&aRtb80Mrroa2e-Ze|@X!?H-x8Xu1YJ={}3@fd5JvP$lggF~`RHL*&?jQW!H zLYZgyIUEUOKOSmBp33gbw^ zpBfebu`&k`sZz!)<7o=l-sj`SoB5E3PR4s&+X>7C_%mB^My$d&y$-CET>#`76;Yxs zT}YJ@Sy?HciJLH}#3)u^^$>C&hCgs?zZo5QYB&*0(>On>(b8mc(`Gg4(#M2soj{JE@Y z74YNC(ptu~_T!^fFwf7wJ z{-R!Q``6I$@Rdd^Gb>(ApWtJAnT)FZ^qKFiGsMVIfN#}XbxZh=7GHwGd&I!9e9IF3 zr+5C`xt*-9(gUef9UexP(e2zpFm?HWRMh?G8wD{8Hk7vy?{~;0l9yUG$DfMVtsZY~gijb*bth@r3BYl{$PyD~OStgW)n#>b`vR#EP& zDk5bXaEeGW@0-Zl&0Zb6a1aY#ky4@KiCOqyR&XdAR^pQu2K_6=a>J25v)V$%jD}BL zJ3@vX6(&+%i?rMpCziUp+5CZuK5{}rJ=&REkY^riz^yb_b(7Z-sK^s9Ma^!J%XO5g zV?^|3$@Q^QUNIqAa)ZQAokx9fTAW2~HzOKE@P-Hh#2<7~WGi1YVkfaG?P>T_v$ZY? z|L>C@2q==g;2A9Bj{ZxS>7|{%reH2q8)`@9A5BGS;a^4lOl6y~ZUNZZrrp6a3fzIGJWRRK#}Ts?zgL$Y+{f)JdmnG7bCFdqIn-9beUUS-o*@mo=86+5 zyQfGyD!c<+Ryp%AJ}1x#GX04eHB|ulI#}{w!i-U)b#UxvvfQZhL}2tF6@!)C$R+=p zn{s30@77fOwfe&KImz7Y5o=y7`!3OTUEy~gQ7v!%^g-XR!sfIdXTjm3a|8Fp`ct4} zJ(k!Py1M0YMJtx#L-V2SL|U%FVoSf>`I>g{rp16=V_M-X1WxWjJs9D-WeC z-Bk*-hpnH{A6usE@RNb`Zt4T@Wv>z0-%zH@HW?LPgBB9OeLjrS18RO@Ld6lY+-I(lf< zem$?Kk6$ke-x(SzW8Tk!tP?9|OjTs(SRkMbxZ$}D0|od^nRRSQtu}kj$%;G zSN@DHTdMf2@woAfT9L~U55Q#boumBS{pb3g-C6${3Qud$@X!=UY!=di>d;A0*zS}y zwE}`mD)Z;*B!JCI9w_&$0ZMZ?G&Hl*ngm3cHok_0g=#Tp%`bJXtMpn^=G#*Ux^F+; zl?GML4cDsR6Z<*T)hHrlxRON~-Tt=yJ2OExcefZ7d=Z26|FjeG6cF%Kpv$0NDQ5Y( z0_ngZ`eR<`Ql8rT-}C*w>oRRTg~VS2V*DXRB8bS=YF>Dk#Ueh(RAMNjN@T6$F&tRCaI@#-XCSCcPg*#o zWi2i)H>n{(&=kf1#s?{M6 z2UrCq*&}D2f0aLy#L=hsWR&U312&l9{%QV-g1PL^Yaf@a>S$Fubk@|B;s#p@Db9)- z#~5$QtNZm&Pp4aoUei|TbxGSXIphZ&J<8R|5AVIXAgR7Tw}51~4V~?IxL0^64(tYq zADqW(R0!4GYd$a+`zOo(Q-w^rbOu3vzEg0&hgo8pyP}k(*R{qCuCCX2su_)0jP)$0 zuMu9}8`ra1NedD6X->B~R~-CBc`=-WaB9G;;FYHhj5qT(hDd=PJ$>18b`RFE{A=fe zKV6wmO0Jsf#ggnu1bwp(g=HOSJ)cI-WeiV?GKg&wKD^SJdCo+sn_n5q`y;4U~YoSXG#@8&{;JZi$w-u^`&wx?xzN)iq-6M) z_!R#mgj+XBt@OX*kq!?qnNF*l)2u#4w&fOhy<7zJL@Dt2oM`tCSw1H|GH>GyyEi`u z?S`}|e+`(W8b;G=;*r06gxW~jFGFUKWC2a;Mj*eY;6U7{?Q?OP(t{<)U&L)U*h4po z3Y+UwoZohvOMBAojEX@(E5_~drfAJFR=!u9k>~rL31)j?zISciyPx#5m{0?SqB}5D z%0O7x=5MQvWN7cVSVyGms~LD-vn58fzx{)zU{uSMb$*!?Yl?AV%j$&TPd5>1=1<u0psn&L>_UyhcTqXh?r=_o?<%s)jiyaXDf^BPuM(^8ZQejW$eJ_T1#Yx(Qd_sa?SDN^9I*2c~Cch~>jXSt*%mfy= z$=t||iX>e^-tKXaJ&JHa+uC!nYGi(vvEd>U43i+$e1kRN)&V5H)^a>A+ysJkK_pm4 zM_cRCh*%I;z6w5Q9!C0@0>2$iw4Wo)fm7N?j@-$mD?`nllqVTD>bEiK^Y%+p_-p%} zvDymEpG#*si`14r1UKNC?<}*cqdG zKPq~7u5{`^Ug$EVH*1b+=axbC0tsO{^o%fFoYK-+pAx=v%HsZkJJ%l${bYaKMs$Ay z23QE%bD4?gahLfz^z;V}8-Ke5ir}Rm%o?VTpG*Aa8{vt@apXe6zD`iLl2y<&X|VMaW(h- z4sASZHICpPk$|w}bcbwyzuCHVRJY+?zby{Tsc)k20j&bsc-ELIhSW0J<8`)26#5<` zf4)z812K;GEybgpFNNw3tuWOh@9b=Zg24%jgBeW>9nZU$-FMrAAxKD(t~RGv=X_!2 z;KOf&`cQ7_%63yc@zO`v*$UrHGjewMlORU1d=v`CM{&y(_XlLS1k}f{u~e9WlK5|_ zh#PTbluP38Rq_+w@3KW=1sY#5_KUM5HR8%kL#c5h>maeYD%+S`$BJazg!qISg@NT6? z-t+%t_Pr!KJLf+<3f`X4mR203Z@(B;w#^jf@b2U)VuSv(wVF6_9MXbrKxy#F7migi zlLY9D7RE|@`pADy4wXy^yR_7eDQg0j2R@aCvH98kYOv#Br>;!bsLLsTe7Ba2I5&Jw zVz5W|k>+V^1k0)SUB~N_X2}}O;d(-7L|9f@_AJHqC|>vKcd5bYXJ(Ed$z4H&lY1U` z`xRk;bIaA!VJllPw;XEki6&3vN|=?8{^a$ZXbAU=2=NAaX1rnaruFL=|Jd^$*t~%w z7Z=x))cl^C2ifU_weLrUyCozoQ9K(f*0XfIJj@6y!P7{SO5N}_oT?IqyOuq zB&s;$2{1N2BEfQWu)*f3EFej$gHz+T2QRv5kRmScq}7TnJY@V~=|r)6?r%TlnSOX1 z8`C3(E@s0n?}SY+X3U5gFG4FN63txRW{Po$lnSYs1b_%nU0-(HN#mzfebpiDw>Fdw zuKG&Bs>M<#SPu^2f~g9HU8C@?DEw1GvU|?xdP?V4!zdWuRHAT_^-1lIJlo+vlfwU7 zEz@ReQzRebtb96TA^;W1j~Kc;3LpMDg5p=}jcW>8V6u()5z_mlifrjRFI@f-$Jn}J z|J?g<&t*D*%}$jm3h3QrTjgl?OlM`$F_9uK+RG(5c5BorKhxb99DGIqY49n9Hhz5U zB3vv{>!bl?TMx?dEPKK?739O@;1O~7DZ6Wen4EN>Zhm{nXQ&*+5 z3J@L}Pajrybm#iqL{tL&DVWi=U6fEC+t5VobQ{Y1%Qj+QKe{6HhpBFJG=a5jLC{^j zZ0?|C>KxP-(l$1cwmwrS>_B_cpHCUhHv!>O%{`((+Wo;tq~u1DWBC#lX}evP8fUG# z&&z$LO8EjIIbSp7_I!5XIhKTtyc9Yt#8X0X!})V|MC5*Q6-}OV)Y;f$Ie6T+IA18A zqjjsAl8O5AB2Sw_Lc2NUYr-S-j-kLQE{1K#9O7isdV`B_o8mJ{0(?G^=TZV?c3J=v}$rB+ZJQ{gd%T!-UBJ=}QGB5P0* z{|#9)f6~&WvyNpv;A_3nY?JjZXxY|8g5TFs{-!uRZ6?nWn1M&|qvhEi{|oj|!jvsC z?NTdF0-x-7Y-}f5o3tpNmuZhPQ$ffsYimR3NF;Pl; z<@(~}PC<8Y7l{_0z1hFCqN`Wm%SbKq|4L#lA18JKoX0Wk&u{+d8&wzmdk3IkwRyM#6NMg z52_{R=_`d$GRS_D@z%a=F{)uuET1)A86=yIk;}`CE{bCaKue9~he}#>d9rEn2CENS zt?7yyL50jh8Yf^%vH_noP!3@eSD#u#K61&>|YOJ>5ym4{+ z(RquS)V}W1NYHuzaJ72VXWtf`5W5-IiUp+>&_jUb;NRr)ddBK$dYQLHLb2Z^FdXos zOa7ACEPbVOzp8=sUuL(*GrM0%XzjPI5JT%kV-fg3W@8jf#X$dvw>Iai_2t?c4j_8T zLuR=z*UKkbkw`^t!ND?rufUf+yz88EEmYaHxjFP%3s&qsEsA^rS|ZYy#Z7Fy<3g{u zS_RL`(tDv(R8-a@GoO5vDPqZo2rQU34#4+0MccA|r~5pwviOfG>k5VW42 z7|t_MiD@r+Q_oU=f^U*(XO5zebDsC#2@BwHeQE1j@E{Ld z(LL7krD|0ppP@O*k$M2U#)f)ORYWC}jhn~7;e@}yTCV_)6kC5qk^P!b+e`O+O%_W7 zX|08BU8e!LaE!|1Fs|&p>-28wBdINssKAQBXjpjd9KE47yV zSA1gPb15H7w8_dJ9?ZrWf3!~rcc8UB3|AfC@c5KVNU~=%N`*P^&1oL5zFz1yiVmPF zz_|GY(jJIGdmhNWSL|z1wtJi=0uCFpi^lH6pd*eRUe|sJka?Y}Jqm3-R3w)?(l6Gy z-cV224lnX@CGfvUr9t<->aL?6662RTXxbhPUn1>6NhX;;|3C4}6%z#cc#x|3^u zUaa_gAS11&faI$e40<&aA<|1a@Tu8dX4%@K+5??x4tQOY}b_i zYv)S&=3BxHzevzBy8NYn#}3KNk7d{#4*A|$$zS7GuB?e2bWI06OXPM4`7j`6%5w#d zzrvpNx<(9e!WhMe>8RSklSU-j-ii<23No3Ovj3YK zA7T2H1*2f_ee;B252;04GZ7e^va}PPWXm5Jl_S1?XHO}SvyfU8i*ucE&yY@&EG2wh zLLKj{c&&k|?XPIl8y+`Gc;hQ^@cpb96NIfyZKaGlXdN>P zAv=+2jh4GJ!PID-FKfy$Jp=LFmW5MYQNpRe{X4hlwd}Wcmwo-ae-KtLkOFn_oA9t+ z?>q>2ZvtA#GNv6rDoo_W<#)TETC45NRSJX&48eeS(;34^f($-W9a2tQy6n~Z6ty`5!8Bajshk!ARMl@kyJ>-5e}alzTSc%|OW)G>QNT z_U|MMSf15X%OO69f^<_;1Rhs>jj*ku$70TC32G#KpSUACuf${%)Y=^7!3I^0N$^7f zX~f^H4VFpvbUjnam7Xz<6@Dd#ziB?Fo$n7?XY~4tts>6+;zmmD1t{c{^xO@)P}Jua zGrm;ksbvdhqVtEWlZ(Em0g0kYlQjGX735cwP&w$uivJdWmENE%I(+^4U9*@x(e-b4 zC9bsOgB|F&&#tg^*Z=$v_6PJt^X1j|s}^*+WU@{6x=;!gdcPhJ)B8<-xh6GuG+brU z{p1v{3kWU!)7@iqo80^d4r8mRH#sS}xnVeqa)~=Xx8}*RtP>P_gsY_hSaI3X3f{Uh zA-4@$UnmB+S#{GqzBDZcnP z|1k3dPuAg#j*Bystrf6{J@$5X_ARNiIR-N!kdav*H3i4Rdv=Wsl{^Cc#=5Dxu|DJ4 zONZ~u5DXu?C#_*_KMyGKv*QMj54uByvg!A<6!h z(bOWOQ>w@*sil%q1Vn@YIgN5i6-a~t0df*T0wE_t$T_+5T=N$|Ub%B$*Y&+VpZB|w z;XElT7PDt^4_Gmz;1#4a)mN?_HQ-GHY1*e1<8~q5f2%g|DI)l)c`dOABQ0GoNbY2Z z8rGta(3$OUCZgV?>F!dZ4^V*izd zaECRPTCsWnjSi0g-RqrJ>X`m)_C1RCx6ML+Zc*#2vHqyd(wSGY^#f-XB1TtGJj3So z&~AE&IRb)Je>st^^dbSjrB+?;s5%Jed*Wl9zLOpgvDi3<&_q`F4#3YqQ9p*&E-MoF z%kvO{e^&OokLa4FXuz_B4V#po?-LAgb>2})z56KKYehk$(>*NBvO~2C^|V&acG?4H zO!@@kUwCT%Y^7vgi*DK9)_U^zS1Oto{brhi*)wig{ZzTvX!%z;qx@Xy+N`#2;eM@T zSe|Nsy_;)u43FUbt!rh)#VUnj@R>@{g+e?jKTxB9llgVIoeXN*Y4V-guC4XLm!8;D z2O%#WQ`OsyCc56ZP?O?Pa-fnWbzb&O{@8v?T&>3vM+SG3-@q7qVXCkwlqS>dt zmB02E85e9Tms;AU28>8YB3G2>?p?n%wId%DcgM=2=yHb-A-%uTJ()@czzXNey@tig za2G4Uu{xtthpscWyJoDmuey0yfIX36bKHq(=w@4X7WLr<{5pe|>8o>YUX)9XT;A}J zZ1GYoFEEG-iZVm08&e9SQ>6B+25rl|vDC{`4#(CR{no?rvFRE~_!O@yMAXFpt!~~r zH$9jlZOkhy`0pRzx^(DwwIAo4ub#Z!(o&z@d1~@rvG@5j9_#9sF*e4ux3enms)0FT ztd-W1{{To9!z#IM-m7MZ)C62{mHQq?J1Anyi@5BqvFbU%2wA;ki;i@`5s_gv(fulq z;=U!X%T?-I^F@UrWVo6^a+?M%`}>wm-tp5tl0E8jTB?1fqi}B;e4OpqoKg^g#=K>) zBb%!m&xrooFnwD^FIX1nDlat1Sphl!DWbhKMlHY=FbDZCR%0Fr(`%;#)eRMveYg~c zTo13Z8K4L<)kBE}-){iOpApbPU+FjG@GhQa-++n?o459GrOUM>b9(}I*#Qtv2GIW! zsj?G@CJ%|(Hy=|3@@n6zzqyZqUgg1thh)(1zQuz?&ram_w^<~y&59Faj$quIR=zaM zkqkuFV{$_YGODt2WhJ#&l6!`}p(`F~OAfUzGQ@!n=r1;+ZQ+}#Ou8!FrY$pGot%g# zEU-Ib_YO_(Oq+Ppo1!g>c*V8W-`(ns%UR%4Cfhy7y5{A|mftmh(|0CO^T8t_;riPJ z6703i?5^{>FAC=>FJQ!oG!IzurOs{Tp=N}Ykw_kIJVxRaI zGH8i{1RqcY=|Pdxx?&d^7?u_NP$)_3|839qhbZ^`G{g4mrI+Ar@P;NE#H1~wjj`m> zqTw=4q0~JA)&j|R;~sY(Eevw*9yH`fu~p5T3puC(V>gBt4J#e~5I!0j*2$~M;4K_= ze|TNbRFiM-i74`D)NqB4KQq=5>taGrzNrRIoT0X4{ zi1I!Uu4_Dk&yWG zmyUibvTSAY5hJV$H*tBS=+}+{_LoZi6lkaZsmtU@sKuyxehr@Fj66)zXWGhf1J=GD zjoTWxB0j9NdVwyYF-zhaNI>YH7H{bE%X2bE8qbx~QyP?BAeEM+Fw}u;=dU*}yN4}b z)s_wn{}edQc4_}ow)O7^Qv>sH>g)6Lv)_M_oNoASf2ZzI?1h%0`2?ZxzoTfj#OHNx zEjay-K2YpoEw9*mg+TWAgm>oDmycvXEm9jJ%tV(Zm?J<;l5B+AdOsfP zJr_LHUr3(6)QCTvfeXArzdZd(XAE1Ad8!Ou?km~_WBrjiqsr=(bqFJiHj4;&2b#*hsuqe zmhpz`1TkX`&7YMl*T@y88871l4TKlT^QL9}{O!O9rO}xPW33Z3FEVQuze(WLz!^7N z4N*t6yhWP=&A46vJU4Qj`v;>w7_T>tBZH0|QeXhrlPPSWog%RSy$sF0c`HwYDh>_@ zj@}=;>5xK`0S1Uiv3H= zcYjI$?(oMSFT8X8;*SSaFBU`R0XZM!3T0BCBVARbT)?w|`l<=?4y1E28gg#G0cyq! zPqlW5sccJ(WVV+9Q`YLt9rhE~2vy_yyq<|xE4XHEsM>7oBj75=7bC3FfwR9Hv|fRx zNC@oe5qXkM6#7Y|1ozauKRx1a-Pu9Y572^t_40a8vWpiCiN=R*Y2s_X%RQ1g3eZK> zMaF~sxl1f9Qk(bV_tvk=WhWPA%~t7)L*Xzgj$Cwo%IaX|^S!0`aI7p^=nHQ+Axmyo zs*3`JzvALU6EfE3cr~rtjtnfs=x(S+)b@u_2ON@4q5;Vw*|<%pQuA@^&opd^Mj%~p znd%xdf{R(J=a)pe3Uvh03OaPcB!^*1;_qm_+U%C&a>|dwLOJ%_iqQ#j>rin7O;}95 zI;Wr+aEHGn9bLPXmJ|9Gj`wU)pr2)t#su~X180Ll!Id*yY$;@Gl5`l>zsXBU8GR6^ zE17)JHTTwiGLZ`o(G(i{om*`Z(`-qc`fSxxVj}0G<5ZY8JEL*eu<3DJle&>tGQX59 z84=;Ocf*2Kd^nO-@vvcd+1zZIdy=}|UPxh##{*!=zJwkN*iH_-?~r;FU?Hq`4o3F* z^u_bL(L_eDTjONS82Kzvp{(UtX>=vJVrdVVx!%^z=xxBL)EI})PdCWd9En*{OH!^CE@IoUSWBO zYgG_`-SN63GJ?liY^}cS$oS>n$SZ5cB%EWn-4y6pju&iU^d)d3=u0Z+vaqGScJ%|1 zIcRJZ%t0(f8O^}7EZmMsy;j(pYRE~1gOjY>P+`wQ&?gezT~mF)&|EBclnZlcKBKcL z$nPPumjcT-h8^uzC&*bG0Kh@WjN(1+?+s4iS*O3ZCyz3Pq^ehmuuwHWGU9qZ9oR7e z^M8#F*hwMlO!#Yc#jE3}V)wNZ5kDo>baxHiJtcM+OXEb!9`tBxpv}!WgRPupUxWNJLxEVPzh8}e zlq-(Rq#(4Fe1q+rP|=F>)uy?3`LA*RxCmUtR9Hi)~3S@%4|7fu?EqVsqMjMY+esyOV79D1&8W#K} z_5xeia8gqGdG?M(?t-LZ^P3))Z6QOlwvR?+2gjgwf+Lrd^3BU?q<0-3tWD5uey7$m z*BpFk*&x$*29D#m`@K9bt#Cd8F^hB_)BKu{dL<`uv5_+U$1I;OYmIyIuEX|hVM;vM z&K*kPk(}{Q`gX6uOk~`EBb-DS%o}D&`@ObHsCffZJ$;*8F@avFlnL$pFfE3VV)z=% zpy>8V9M)7{KMz`8S^3kwM(X$@nfI@4oR&KV<`;$41ngz^fGmfEMSqp4yLZbJ+>c7t zRTfITvp-&s58TKtVNMip{D17gU^>b-(=8~hsaUFSKp)=A#qOIZcmXZf`-Vx;n;o*U zFq++k0p@UL^koMJo%myJ^lR$GfybW5#>o!bH~5>Yl{&LcfR z$XLwEK2hM_qRMeyFz(uoF}0tRtag@_-&GDWD;u+e&6EybrOZYM$A!d5WfY)mzmSWi%zRaomx{rue2&wBKh?@WJ6JIkv+1H<)gF zv1Uz)BH6DX!haPYK~MIBH|fqr-MNh1tmFZ>^+Cs{4`(k9cYpTJ|M~IvA2>0GLHGJV_40QUigCqqe8YT^q_~wv z+*qwfJT1ROq0stS&og&Lmp_FgLYK33d(T>b>94h0x@$Ocvu_&L7N4_Trc>P%8#j_A7! zG%rX{UF&=2G9zIC@fLlhjuf?s_!a&i(!@Xo>;%{#V$`Sf8=>7J9!+fQG$QmHwZZ@0 z({#=nIPLN9gQm0S#BR;tQvpK|X}#pi#Se%-ai@uAI*h3PNCSI7E9smZxWy5Vdn?cJ z%hX%JFUv5opf6`0*+8+)T4VH5)exMq>6qSX%ES0{z1aTD;6HM4fc~wr1m@?>&!7)K ziB0>lRoj$4q|F{WQ}nVR)$WW>lDlS}JzFj*IM&T98@$+k%Tt+hvFp?E&vzfC8&}OE z6!Fy|#%qE~drETg9$~885x8uf*U8J>Q?3dg%fiVKhqR}VqK9J@Pyu5?G# zGUq~99WiX8oS+$&$NFMq5x8=}Hf+paI$H$8NCLA0ZRu3M+%aU;<`P6{)#^2lgyD6v zUOK9Q2Qv-7zMt)9u%Ai?iNMZ<3pI~jdBi;9NX*WP5SF8b!j{);? zxh~sz(^AYXEu48ebRw>cEVEy4KCjk=e~y$6hw3ywZ(ZGdTE6m&;5oJHR*!FHE{4lk zif_)vZcU@hMLIhzci6mJCpI5S-Rfog1hnuR1*~LmRKu+45=AZNLB>$AF}QF*oj|@b z{0_-GhIrdp=B`s9co>;TICL~o-6ht>>chh@k?Sn?K0fjsij`{esHmr<@OF(u;yX?% zDr8l858-_cH2ryyFxD_f3(K3=%rOjk1`=x0pK!J-$mvSnm)TV zkPpNxj__PpKFRv9sQQN^GE8c;#yF+Vu2Il~f(kG*W=? z7}cupXsFwo7a4)ecfsRMz_A9c>(Kg;|x4G1J0DqQGwb;&yKl`0dKT;HoLN#EyHPn3@I9yQ&vPq_(;NcrL^@*m*iBk?BlBgxBj^@~N0nqyVvUrrvK zwp*(Us>VxxJtbRBFZhr-4%N3yFzo_uUsxmWx!T#v_O~*KTX(xpuC#{?K4`4wTo3lJT-_W)N%#dV2U~7*)n=RX%}r&7>9Zo!uss_P4BLP= zky;W|pen{a4BaVsY-h@^S2EZ-a

MmcrSAt1=dxs^0NPn3)$^cQvvQS(EUa{e-?IM!u0w4r ziGxQ?2CwIks@Ft(1p}EB_1lk1KWmgj1&O@+2SB zKHQco$38o6_iOcmG8j!@KS_@meaLF?S9Fo>565B%dHUfm$g8{K6O{`+dJW z6#)ND%8ZyIL|`JpJowaXo}A6*+E5y>vzb8VG^`MDk!*qmASwVNlv z515=6tG*<XH!USOY-|FmZuaOXJ{@-W_L$$pfds2sHW(+_ z2O;Y^u7kgH(Nb@&?FR?@?)#Ia9w%~ z7^N<(Yfch^`DZQzAO?IkLtnC;r+(!vp6t%?vGV5&y;(cp5efZ8mvzj?p*daAcwV-o zy;7d7qtgW-)2jKlB3WElfX=U2+727p;#rSoiqMywFW<1fY_+;uxq5$}4$sekZpJiO zmaYnT^rgPn<=U^;tS%yj+l(>bz0fnZs828P#oL$UuYGAp`#Zb;t1>Zdrdrs+fhSg4 z2zi1FfcfSvJllW?k-`r=9Hb-Z19obdAWQbz0pGgqN)PZeiIzPW=NJr8j_In)A3@t6 z=?LOzxSH=Jy!KAL-A1yT?dYj}zaD7mAz6EXpu#@q26WGK(IJ&2o{reIR=HfxGkQWG zC@x*k!2`PzCZaWk&Zb{cAy35YZCd8jV53kU?VZ9*weFXzDt%p^!0#P;OM#)Yu(Bu`W-lV$GZ2=}L zm#c%eC3zo{xe!qw+n!QL3sYB2F)mj?buDE=YsN&U0hI^7#5OjBqZ@DhOdoO^v<$fQ zrLCmD&w$$*uq#x3Yd`2F;T}N`D{X}r%p9Q+bhDWr=Il!!=_Arh2w3Rv(e&=msII5` zX@URx_0LIr_cIuvzy0vkrzQdBV>^EoW@IM>`rZ$z-c-tCa~xQCTxKlY`7)^8ZbR5I zAo|&kaY;_ukgukMPP!P#Qrj02zvP3nt-2j0`pP8#w92&0;K5?&*zJyPkG$`~!h+K* z;83hkgs0ia;y0?m6-Q4gt`)!HFn%1Ee z@Y^ay9uC*q>S)0Y2qB`L1rR#ls19(2CwwOq2`}3+Idq=Cy`0fN(E{4tF_ghdlE@)*@Sa!IM@3>fISFu-hGH$87-|1PzjGy)(?4 z58UhaWL>*_z!x8`4{y&RHMR9Jt9B;(mwAFO$KvIkiYHh)y!1-u((oiPlW+L$bBAd<4f!uF~(QeT_ncrsaL=vl;%kz z*x+}vc`hX@=$*aJprj1z`OP48cdW{kgW?J17LzNHM`W19#%P1`8DRjda}1!m)1Hw) zv7?r8cTz0KZ9mH)bO~i5QB^YIu`mNT!sxL9cnx*L!;HL1;Q!nDP^L4~;o&n+upn@h z?Lkyyi|2*bp1O{zgya-_@ciVOzpai_U@c#`hE7+pV|4*V6<$Ld|#*gv}tq|%CBoCms&jY{UpZ55A&sB!r+nHZ|55q0B z+v9>s`RiBz!au`b{{{clpMKz^mdJu42Bo_8_vrHka+I875`9&2nW)QU*7KZpF*>Ht zC>;Gsh)O-I`!R3eh^=h2)6a8P4V(on(}c!;{Up48p;;c}Qu|#h?`Uo<7QC)%^QFcH zs#Z+giME>oPn!qYfxrLN+h4-lZ_yZ-ur}>{l#K9^4M2CKoR14EO{CU7>pZt77z>f& zzdQx+X&48^+k@t zf(~d_j2E!jVFyJ)shT)gnQ_m5Ms>7Fs_CG3Z@W+#5eo;sk0_;524ZOE0*BbITDs)s5@0Ur7rzt!bGmInS)jhj z>h{`$lwi;_&Xkg3%fMeBzx@y=&c|<`sHo8dN7q62?)fD66@g|-5oS4$*^!+)SK2Fp z*#uME^e#aaj3{9Awea`pdG%Rw-kjj9gy_Lk*En;W1cM&QQy_s(e@FD`?<_EY-9~Pe zi||!YiFWxNpU-t{3^htI_pWts$je`ZEH z39A&cFZ&Y%3b{4k5p!~iJT3g)7KOyYbQ|oWD98F`eVmZFYaGu7_KgF~5#2W{HmacX zB)hf=T@b350;3BCxv>(O;saKK4ht?)BPS}S{XX80w8jDXC|EKr=UCL-1Fw0ocpIMq ziS))@NiM_Do_vIH`Y#uiKC@IiI$@)oF=wl0O7(-mV{%5=`)YnZ=F5Ue{-$Fr*&y|i zrDit>oQJEt4!eI4{fPD9P4R1Z=u8l&tk3*Ed%1_$WULd2UmS1{o_?@N5Zqt@=*hXsmbpYm?2l!daEi-ib(dkb2?392SNwkyE^4%l}y1VbqsD3|`YFF8!jjM5SdIoT+03V^juXo0RIb^h~-1fSDnR{d} zyuO(%8%TXyc?1aEYLO3YW_Pz(Ag8>!>7v=I_*>7c5=JUn#u{o;Lm}z%x}B|0Fp@)zq0q!J$QBv~ zblA~NRfHfBLuy{76=haA&HXv1Q78#9&rtD&ieX#j>z#KDha#GvK&5HDK*8P)D6C|f zOG+}@*uw#QcvZ@wwSV>RUjhMMRE~o5JXX-gD9>S;G5a^}BTEhMSqOfE9^AaP0J}@D z=jMJW&P<*;m?o}&dJrijfq)=T!%i#c-02E;SEIolF}MgkKMU*yi3J^P7&~t#pYdT= z*)1C_l;&0a6DE>c8WnRGod62p-~}pl8>}|=_>2P`CVEsw2#VURl*L{%ELqodi+bc! z5y@o4KBG6_pKXa)2P-P-z~bah4dc_tjGBigS~BYoDId&55T0p=?DETh;`|#x2cLKU zN`9hE%21ArS)C20_5e1Vp7k>zDS|#3`r~PqstA4ujgMiJ|D_G8wA#R%hweo$D+;># zzBOkmZ31FMmJnvM*@YAGaCe4x;8H3sFSK4Dj1ZI62?_J}@UT(KmP@Or`4QwMz~Uo^ z>ic7Y%}S6xu-IarSR7xW#VPw22Fh-hL{REhJZn8TsVE2gZ7r6(EfjT!agvFgYn%ft zlt>z=St{2PnjO&{E=S&SeR;v=m#5{oqL+j)EnFxQ0X1Qe(omQj-^3L?A{3zBzEfCw zg5ZT0?$`K1iyqlKl~UxKD;_8JG#19Uq@wICNG&b3P-wBn<ru zl2QG?fv5V<7JZ!A7ok+D-mDw!((P?hAL}>hNu%h+g--O2s(`t!hJJMJSoDNEHw_tV{nQ*h-L^`lmAuih%@3NOOYm?is{Pet)=odiKynt(m9wq^ z)wZ~U(S3TjLj*c}tuJXLV|#}R^T)EN42Pq58>u#i#M^df0@X(MyriU>!tICyo4_OF zS6~=g&@6L_UQSJ=B&Y+7aT?|+;l_W%Oy_B)bS3%`E3G&xig;Hm{H!!<`p}LPH0W}(rtobF!Wjkr)QWvzy9FR+6s^`` z0s*|B-tLNtZFJ+3G~sIsJBZEK#OnL+Qq%k1*K+IaB~XPXe+mp6IMI>A-ga3Jpb$Ai zMeT093n5=xK6}eeClNQOP)hc+L^51fAHG8-+h%^j_;~6HE?EIwF{&%Ht%htejXtBK z(w+_K&{icNR98MiQnSzSZo~&0OHzo$#EN3>6R$dj_7i!|-n56LO)Xa|>c<0inm%&R!!dfd7MP123XhQFNw!jeB3@1K zYF2tl?T4^i(9zOGLf2sM&mKqc9Rq5$Ng&ySakHmoQsaG)1P5@KAobIx!)=5(7 zwLJ!}*}0LkP9>tIlogn$lOCViuq_LIeC-fQs)Bad4s^NY5Wu^_!5Meo9@%~i&+qK4 zDPRL@ebm$i(Eo<>H};(2c;OpuJln8>30lXJ2b@8YZN>mtjGR0755q4ue1xK!ZFD)u z4Dial39RtVHXWR9`^0=p8El5e)O2Zlq-G0V3)CPeHCUQl!TbPcA-yb$bdSs=-g=uZ zD4}EO=G3=&{eF1+5~z182i5t@q9gPWOClmrMROhuVpu6hX;LzJBov3?K*H|QrmDD> z0PhQNM+(mc?pF0zdU(_%5`WEQta>hab2)6QoonR@@O6e*AT3geD+WLuo?bK63`RO6 zCaQ{+RzIO)QAr4)B1F8;U#lq-u|0eRV0@&;?Y9n%TEdTjmUM~eMV%+`2-5<-yOK8N zzAH>bJy6^f_c6^kLUdpgikf*=(5OiYSVaIZ7bHxww&m7G9IxcQ+{{8@N`8!yx3mh4 z?}0Dgew_Xr&flb{mGBj8=YTltOBF+Is^`jAt(sS#y`1eL{@%*79Wv<J^Yl9Uz|5bgk^=NV>HpQ8z&P|4w~AfG&Chcg!|oSqZ(3MTi?`J9wC&;wM-SO^6h zB%!OWBO3&+XBqL|zJ5k~na^x_%BcmW`p~eGr-rx~*X~(FTJgKqTOKqY0k(O9=S=T~ z@05mNNqnF>tk(vKHlV`O&3U{a1oET3J7TWdT8-hbX&O#KM4ft4+qR!iBO{y$rDVsd zx@BnP|0AR!>3LsFAG9DhJSXl4rfo0Jzou4<@}guTfX(3tlH@47#{GonhY1I{ z^Hlw5`aGYC568it$hHd<0Sac6BSwowWDG7D=8C#FX=N>}uokre-W?%DscxVVjZg7d zX6T1dGOb?*MS0qrcMpt33QS;14u`;n+idr#K1^XN`ID8T@mHy59@DEbHs(UHJ zx`Z_O{VKY79XJ(*;R&-%o_~`*hR!H{W+%%K2mrzve-dBhbaN>RG#274<|<;GhCa()Ia(m~PpX04HVjTNG9{HA8QNKb#de9~jJ8 z(|>ZnLj&!m;WXktviNNpH*WdNkDIKm+hWfRvpr;J=~?tIFX+rr$$5oZu6=M&8=43( zMeQf$UJfdg;ivp2kKgttA_>=H+h?kzS6=S3hWgUfB}FbS%%vI$yoUHswS3o z4hmHlgaTj5#3qoDph9?(gLHj~YR^)a4Q+)gk3I;`e|`?n&(HkDEZjY>CoWM^8?n!_ zw#dgSPU>n90G&Ut#6P$Gg9Q8)!;WG#o+xIwHV}dAQ(lxutCGyL;?Rw?=bqwb9oD%z zRCetoT-{m0i$GNzq$N`&@#=d&a*nEn3Fa@MK4{}pXwnA>R2ZG7pLf;hUGbGtM}aZn zLPex=5l(ICsTZ@}_Nh=oTu(r~&p9E)-KynR42B~&pqp4F*gIaSoq`F%3d>EG)q%s%L23FXx+*US;2Yzqd2gqaF1-H9 z9v9N;z2DkmO{e-lnIlcI$0m0S**`K$#f0QTmt2=hu`cLUTiKXZ^wF4$IVKC>RS&$s$ z>paLl?qdJpbN)iHYHMHR>ClE#J1dj#^2y;vv&vc+@>IJgolCs~YcOJQvz`xzbV- zGZxPo!G<6gS~_mdiQuBJujypH_uKIH>E->WP%~EqLp1deG%hy6ZU3EcThk&#Xpnjj zgdmg1wt5eC5P_kvr}}byt_BF>k^B|L9`mKW1Ejr|j~TDH;e=e?l}%_haj{O0O@oE{ z&usnTQTy=6GZ4trN^!=mse%4%k5~ZgZX}X$Qhia^RZBdS99#2%c92K~%RjsVL$rEt zS$H>9B_0n>z(fetaX_e*#jg!@gk^$q%cGGRVTRzHZP@oelC19a=M;hc#N>nzXi2Lu zKwo*qvwHg)M#tpaVN=+26~y~FTTITtDbT=8*@+62o7&4tDQ!3ixa{)HwyiCwF;KF1 zU%e-YdmOxJaO=1JtbD4?j?oUzQd{vrJIx->Aw8ZFp8e43XQtGnYBO6V>I#~8I__%u z&eno<$GrWnf_0#p#2bOQs>Cy*#W@c(O59n3Rd*n7(N3cR;w8@6aVUFX^L~ybjl(JT z?_5i(fl?kEf-OuzcyVVC(LF&dqWlaLh&lW$`Z~40(t`y)NYynRwAIZS#4qa=x3Ez( zi`_<~CVhooSDm3{#nf7IwJ3A(Nhugm$m}6kok2Wu)aq=86#NPdGTDj+xuN%P5UvC9 zXIZ$lA1CT;;{a;7MUDx{(TDod!YtuKf*BdXClM-%Jb{D^IwAK7w{GCg?=sO=H7P2~EHllbEoG?@Lu3}$b$%Eg1( z5-&QuLowepNlpbv+q_6hbI@$;!Bf1PCA@^ExtoZrxEH<0-hPxLFl2iTq0K1~An|VD zCE}vIjHh-RhKjNvV+>C&OoyKL?vpX}0PB1c|3o^;(x2FPvQ^Use?9z1TT${r6%@q& zz@at;W6~m4aG?95Ny4*Gq7c6bL${cDswg!IbrXRt?pUhn8j=+n8a!qzYL-k;>au{) zvKiJv9qH5CcnLT)`pJvt}dwuM4nOkv{y|0q3N zZ?d)O;gp_F4|keI9v4qlq_gqnpZ5_C=mAAQeY^oH+o0=}5=gjxj|SG7dfh+BO&$LR zK;3jNGBvruGO@-ns=|)gsLqX0js;C9RH^ofy;8o<;YE?Le@SIRW)-;lMR+<;Zq+#) zIfMo3Q?@rxc}cStD#gQGYDAw(C9f?Y0+Nhh0k#haGU%7{4s zPN;s?Nu?23nCRy~+Y#V^VMXL1(zT;h4e*e}xao%MmLrK28G?kq19DD$`O0WkwQq|1E z1XyC$0VV`HJfPxycM-zftzVIc?x|3vYzGLW_yJD@y-LU4r1UDN_lW0eL^iADG8;3g z7{O=+9bX>$@J=$g!eNK6RNHi_7=~o^>Qo@;H5a?emm0d))81KeM0V(X`uZ8$A$Qar zw#eY#hBtZw4QQ977{F1?I(PLRpftULH8J|;)JE&7jP|pPVBa2K02+D(Y^KOUlrTNX zoQ;q&oRJ)o4cG~{l^X-rLmr{WzwwZcp+dO~?p`Tv% zS^}50>UG*@1-C;VB|q08B8rG3LiSQpQMxY|E5{io}Iov2gcL+w8iXdlW zbMTql9$5H+b)ejZE5~$_9wY@Xj@C4-MgoZZl0Blh-a?k$1j4+B=mmWPF}rQSX((#a zb0L0UPv%qfr=4!Lqd|Jx{?~QsFJw${$E~ylfL;V zd_BF^{VcMGPRzy)_XG_Q30-y~$LB>=XdL~bGLG0$Dg`WshSc%!(xCtw$vIE4!Oy_t z2XIF&Ar)$1Ceb5mZHp|)p3OpeL1iE5H#dw`ZIrCzZtp1;!@lbH(q&z-(njr^rijxf znazvVroPpdBO^M%m9^m~lJ7w-9`j_o+t~VRtcIS3RiFdAf1E4v7T`EA+DLT9fF9lX zfm>uy(3>}kgUO`bl{@TppM++ebsm#vjD{3```zUQd(n!Z{p_*=w+;y_IOc8J$SoR9 zXPO$Kq|yfTG@XCKv_j40*+V$UDU!P|K#klYnD~cw&)Y+l;t^9R^tz)VWby8 zHfYgd9sn+d>0i3NkI-Iu#4(nP6G%Qb?g3tYJ2ByK#oI$RR;3OnY8B)7#iu^VL$*B| zM^(cu)x5^j86;_T<~DMu)-1nEv_CG5(iUg}b}(1y32>TvHMOwK&5C#o6InNy$*dCQ z2lvjA8hIfoiPHDEiF2qoA?0BK1l=@QNuM??QnoCCwL<%q6B4NwAyO_!G-LU-kn;y{ zX4nE+H3HdeRTIq?8{o%FPPg{9)vuT@+X4MFd%LYqj5yAO(b_1o$dOd~&`7cx8(o%L z?@(HmT>3mIFC_s1wFY$)KA3!6-Vtu|bf6GpPa-*wK;$D+EbigD0dSm4CNvL7Lx9%L zBtc^IWtIvi%s9eOn*h*UBHMQ7m#9QT7p;(c2H!7(rfJtlYN>lHF-2tG6}3f+Q*AoC zt!|1qO+Ns%uE(&A4CONv9sNuxQsyBwn+M9%5b>5NEc+>3sxF#G-ec=vS}L1^t(vv- zx&p~$p=~w-|6}$*HUD;MAvia5QNtA-(%x)J56mjr`%}2lCa#2T&aForsU+~o=UpF~ z1g|q$)ELBQEG1ghtN8lx)^6pj!m&_a@3-Hd$As^H3WxLE9aX!Bc0wSum9iqqG*LBb z0;kqU_PQ$9+N&Qf8_x^m=^BOq6|zw7>bp_8C>3g^x(^TQ8x>KFv$O?YA+3u(?xzj= z=)5&3TR&*sBU`6|L(vdU;R@P-1+0mo-1b!TI}1-`u)73osZu~d6Gm1}_i#HdeJuwq zP^_k4tDbj~RrnC3NggWrhcUbUq)l{V9#RwF;lqKFlIj}u)WxFn*K2?In`3OY!J&B-!J zREa7f@(*+YTTMS$#Zoh0Ya`0FJxSI;mB(>4I<}QEfYLVGXqf!*XlS``?en1yA4vcu zPL;)Wg}Z@!nX;pAI#T#aFkT?4f*@o1$|mf0l3YM4l6dIq#PRM>RpJ*HLYJqN+R7;G z8IP+96~u?`N7Ue@$Ax)xETlEC#mdBg$0@aa05ojgG;Zop2C3sd>_4%U6Fp5XJvM12Y2b&4? ziX}4pL$d+894{q`T$~ObIVPt}zq2TlG}68qKKiyElX~KA6cI)kt9;bZLvS5XPM8KA0i*qU-27M%D^gRTfws9c2%v<29u%d%jCJSxk^0(K+)<@M+$!(H4;(L z9(OgiH#|e+@1Qrc*mX1{7Qtc=TNISMfyWTAlU@UOhLXGtih+?IK*k9 zdz`fCQK%9&kqp_HdE{d~O&*m(9+^_b5AFwCC-$Jqx#VC`WQ0o7UvXWp>JZL3v{Zl4rp5*28uUlb1AjabUj+4SP4>fqdE) zH2{< zGTS%bmH!3&N1kMtUI-nDhlK0S$M#Ssf?5Ep$o1TQL9??oLc-KnXnf*~os2AcSBnSv zw3~K%Y40b8EK+8jOISqu`0bO|-#}>RhbGuL7VFiGGaiGgiv~zSroVwxNaVA1c$#cm zD4R9VpR$zd3cLvxB&dyU=!ap9MT4X4W-Ca%XE0y+@(@_5-9PkWnTM zc2avGI`M|$$u1GS!OK-`0_AkfIEJ(Rc-`Mr#ja-6Nqu){a&<~j)8>XZALV>q3zaA@ zZILU=QLc654c{C}B;_0YmRf695#g8E?rH2W^cVn00`d)4ZN>({1Gk;?`Z1K=M&u71 zvw;$iOif?G^%elwmd~f^=S%<&X!pdhrhLqG{p8c^Nz1?i2$F=jKFUt&K~!1pkyuI) zBVF!C%~**QmRR^m!lqmkvrV!OToslfpjc(kB(l!xogMsk;KB@7#Z^ypSu>a=QDUoj z05C1-6ZIq$Z(U^ifjr`dJfohecT3GZZ= zEuD()dUvuqLHA*}HyxLOA_l6PsUm=FRCx#stCe@}MhK3hah9RKU{}3R{Vk2fGr8Y|H}XJa5Qxa&QdT#irtSJiM`?7PE0^)xvm2 zoX}yNMVz8Ow^d==&GB(tsJ`Yy?=kQV8%>vCyk^%<^oUa6JdI_wp=u*!X-KlPoG_mg zyzH~2Z<+fRPE>mi zPyIMTnhdi&{@m{ZrSzRp)IvSb7An(hs5hfJ>L@=a-)fx)C`%un4z9UPF^#TnASpnV z5zEN$s=-|f1MksD-C#Lh?Cf8Iw{@{GFa^lL-V_fCrw^{gLp0!NFBhnAT-~niB|LQC zR?~(a&h55P7mNnxvnO>;kqlKC9-J+R8CZ7=Z17hxS(sVus?qbXoh9qZ*=kx3vb(M7 ztG@0lISnO?;~qqok|+9F658_svcsMOPcXT~#;Gk>l{Og}2@}!-K?s&>cE%2_ZHeV+ zdd@f}vn+KdImM}mpl*#>z9{YvUm0ek>2B*NMaiTU+vBqIk3&v^c7Cv-x+a)z#au?p zvNVE8==Z}5IV^c?8)Oy_z+C!I3(%MvRw9eKC`a@-c3jG@0d7k3u=K9HQ?|For8|I z2S7RLOVjSO+Y5yD#GBsOg9q}!Iz^Zz^e0$)GpJf@x_v9it37eT2w(HcU_jIe0NN61 z(f6>xzoV&@fmNK?Pj!*FHf$N7Vx^{_sn4cch!OG+AzljTn*ny z@@Puib3_uR54~iEhSgQNC9U0CBr_bER(AIGedsO!JHW|52nujSgZ+x)kU@;pOiCLu*d~jC}el8$y ze9X<6?vsRJ=w@g)d~)Z7C^xd1U6HLAnb91zg@E~v>Y`3biR${1T%s(SAv+8Wew3q0lUDpAT4^@|cI(FWNk8I*KK%{gc}Xhl zOGfQagTY|OqV@?sOl=GKfUsex08C0Rh-wbVZCl8NoKJf}u`kRW(Id;zGHDCA__C}z zyy1_iuM0%g3uTT#L6~_>f0iFQP1fdTi3(O1Lif^xxt!noi{r`>J!WR#)sXQ}NI~|) z8|V_6@Iib^&>c(%FrYW!&JdQSM>Z$>VYZtO8X6hT$^1}`WLNhn51rp(jgRUHXzO@T zFe^I$sCKC@pnN#0O6br{TcYX4^ipeXRK=4efw7^YceFdQff}xpF#Ftbw4 z)E$X4;D&K-|NeWg-^Xz8`*wk;Da)ViCmJL$O({_(4>*(+Yp?KXd;cO_XWL}JQ!MtN z!;_Q>$*^EtHm5sxDtR5l6}S(bicgI1CxtAW{IbA_MN<1qse3QaS=?`6t-q@~i|38e zv1>!0eI`m*%iRV812;uyIvMcYG({`=ygdVr1JtRK&g=n8ZQNroQG%*Va{ufX92&TY z=tbLQuFpuxI$!znc8zF;swHc8F%Gh*HodBDqQ{w+_2BUelnYR3<|HgtG#f3 zI(I>BGD%yRZGl3z7DgH)h{g~VIjY?;kZ}(r>vUH%&MgTq$f4Pq@cEShAw+ z+?*sHLy|>eM{COsVt_a;0glf*nu>0zjsb)Vi!@5%CS;k2ms&^Zp2Mo;ds@sJU}Y-= zSXSa;wm4tI6HVMv71}}4$gXttcMiR!dT6wLT{&?ldXFJRkcm{C5o`5y8mQ0PPM5$# zef2V!#DSc~3;l!C>cN^evNx>O({9M>TY^y}E3A92M_8Bv4wF_2X~G(N z4y&|#z?6tg0s^~8_VA<{`6Df|-HC>I7D3R98XKg@iS3xCF4Q zU9jB)rSz0+rnb_+)uvFFj&Bk_Ti=P0Om&)x)LxmOz^3obtuTGv$wN9w!=rCehZ8V zHRg*ni)N&}Ln7j^n5JiK*)+lk^C+&=?3 z5Ma1#d|=S&spXO35^zlnWUZIQFz;#s$$+$;6-W8$a`CSNG47C6Hm!XkU5!d7S9m(G z$YXipv~KF6!!!!|_xZDLKv`kDfe()5WFKW$vs&Ckd%vseti1gqO%>5slKhb4r@*Y$ zYTdI8Qh59QrJaW+RKsDTGReqSJuj*SGz?+GsI`G^Kl#)wDJAVUVnv4-jac@`y#BWkA(3J zQ;bo{r*xT^CwX@oPGMFp(sfgX@|qdy<%f|FZHCRNo@eW#?hi8D zR9#H@Ei1fSUr;EMh)q6a-JKsWgVV`(a-J7unccMy=W&4IR_f*pJ3Q|SM@D&@o_tW!acY~gY$VT(* z3oX`}IvGlZvYGg?+ffQNfz1s-0nc1#_aXa@6dwvm2lPzx5IAXC((k~%c~aG%NZ5jR zz!oVW;p=*1rQO2P1_X@vzkukK$v?_+W#hTXxm2&e2Fc_p@k=MRUf}8_DWdVnFhQlE zfhLrR`Nh^|esWhJ+!iJb#=E3jBVu>SMv&6*xp`R#q}_InK;Wo0mydtwsA@#MS(sbKC1ixe2g(%*iMx& zOf-=HcmgpTQ%0Rp93RznaO<{aWqV$}xv!b-Cf{!vs!|LPY#P-SkK<~&#gK*1?%dF= zY%_vrS8e{pLAVn4+B8bcyOiBg!r>4h+=Bs^40NJmP&61YzG0=OMAv&;7wTSeTo85`Q&N6p?UtX(TmRx5{;}|$j5+4{!r;xIf)z#uz~*H zuLmTf&SSj&+@Y4_gB)f_RUs88M<%jBOE6U$?)zt*xbL4jf&ewF7#-P}V#)@)3b%T| z10iB|(ZoB3cJ|at)N2$OP-P>FcJF`u`tRZG7tpN&G^4nJTKfL`I6?ON37I6TPC^3X zs$6Vt?t+sCtZiR9u^nX2BU05Vn zI@h!`M8dP$0%esJpbgSdbl53=gxovDV8Q+wC_wbB#uN}#_1xrCDMy_=BS9%lD`#-% zF~;A;nof$`Tah1V)J)=7TeZlD!(-wmuV91w&|{jluKE>2HEzn(BPP{vztpVn?dSSn zif0Ei!ZdJpyL0(%bX5t)Xkoyft_5IsqDz7I+31mu_6sG`o6ySMDi0!oY9!1bO!|TK zR_o=HmVmEp*QH<&{ubuOSRHi-5;ca29dkOm9Z;sy&lcP5d^nK8ywJBsevN% zm9PF8%kks4q&c!$hGe@$@(j+kB;L;pNO|@#=RG6->zJhOA>FeXc6Zcn2g!5=G#Xs7 zaDT~h8Rmafy-?|?CRVQTz?N+VcOUP&0Sf7+ZfF3QmM0!97$bpGV{D?RqIDa)TR;$6 z*`GMja!wU0I01(b)#ZU>C8Vm}CL|MiQI{;=(bq>-p$wB}bI>yMyA zb1XP#%{Z`gUep~uLhYzQ@s}8R?(1qQA)i}A>kGZJ4h;IiSKoX4!TyGZ&x33NIgwDf8k{A72#o5ErXafr3OVOYE6;ZJ$BR$u{*T~oZ=r| ze+|zxw8>Dz{hFXCaD?IoU~URanPOHTd|}Q$vTD{>v=Lv`xe$7BHoFR3(7^IoRZK<^ z;f2I>3kXj-ze^if;!nD}SBbaOH1+DSA?M>59@$kv33)EC_ai7En^?Ab;aX;mCs$CY z82LD{Jdk7Z{=*}@GsxMjt$s9WAq!%QE-b~1oh-dJ=m|pf4p;zWyO<9n(2C% zj}y5%suoyAO@;NKtF+Vxu%Y)LE63QF;8=C9OzZL}`I;qOQ%$Ig(NLTjqJr^4Q}j$< zcG8OGd&t70PJvMyP1mp;(Y?xr|LYIKTN)wAK?d@y9C;%I4zftc;HkPYtnbWzaDtYIw;t6)Qt^qJvf9Jb#y6yQGPckO zkOCB8pY-z63_u6iH~~w799tfYSM(W7#^O)>le!j3bmWGy`PxgmKIV=I7E251p#Kh7 zwrc&QGJit!dkIF!^&r$($!)2M1)OCS4x;Y!mNcL(I-XhCXV#dkqkyKu&US?ZRE?51IWclnXt0fH$Ml_TB3b?9%`1zcs3nUAjDE&3 zR1iIALU(pB0OXjwE;6JT^El`ynv*-MdQS<*uW?F4Kj1TizvKbbOr^@J9HQ)}!-?>r zHe1_HpkRa9%}x(2^bw%A4Mmy4?xm!4)tMEwI(BPO{O-WM184wAK(@c6#RgB$s6o`W zM&c^~;Zuyo3%A`<#IV}It~i$H?gp~~-q^SWV9=+@XSnk^rGbq-hHrm6eEZwVVyjOY zSI|y|!h~-W5O*ctSP6PaVT<^b9u0LKxstrwtqHk{>V^2m(^MPeD>0aAPr(fXba4&j zCS34ll{yQNP=Yigc>VXlFZ4c6Bz*3gA=vpk5`rl^V+TSHO&ULc8;sI|Lq?QbCo!9x z9uSldrQeYN8sRV4%}}kmONjaN&fw>5(+o9+2TAn#A~vg zGE*QaT}+4UDa0{NTdgtEwoYEDDAQhy>R*J#;5^8($o&NLM|_a*KhU#oK}Pk)9!EY3 zu1tVon0%pVBTF$k3$faiSa&HIi@@Bsy^SKma2wi>m@fAYSdC4+H?GPp0#-a@d*m6z zDq8z%ceWW4r1$L?7A%8WTV`tzMQOBpSh8$njA;~ zCkfe2$rYAsCtcSs&;-QewV6=&fc6*d9V<(?F}JYAfw*cklaXIZ;yj1!&No$=Ji9_2 z!421e8T~81v4qO=VW8qes_mfq(&ee-(p4r`EV07wTsgayrbO9y;K+TB!J`B;wKvX} zF{v1>qNM04+AjTZ8qEIKR^oOvjfASJGFj)m6J?zA8jv5q6G?$0xbhpi%Q++Df&-pNJy$=*g#^8Pe1v&fTnh1JLn z+v79n{!y1F5F?7aWBU`u9+e2zp4jybF7O@#M>f;^k=5mE zwFR-|qaq*l5r%$(b=)AO9WOVe9+qXytC}j31h4?8njUq{K?Ja75BAaTCX#DgJz&J+ z4Q=GfMV@YDp_ z*4=?o7rAu=jv(ekK9K^~lJ8h=s|gmE%d9D3I{VRja1{@-9Z+X~i?odfmEG591Wgl@`HB!fLoJH_l!)6q=|+09y9 zdTh6vmgZ8id>r0BJ+&9F=wD0P=EE8TCL*>FD?s1mdGqF-97H>P=6#T$2p064*Q3u| zy5f*!*juX#6n{SmR?wm*1$9uEag9*uP?E`icwJSe6`Rw=^Q?mP(3PUWF2JT za@AWQ;plC1$Qm0S>^f1_)h)$BY?7hTZWx(}G1huQoBS@k{y-^cvTal(?*cmPVwPQq z`5lzIb>x8K$`%tQdc8XYsGSzQXCL_5bMGtj`~WQG-NK@lvez`e)a zIr9(-=2?EY(=xpAZhrxZst4&_cWtvU;RJl*TNKMm)}5;NcZDm8EnZoA;AN2&tl&}! zv+@k3U~wZx@S?q(J7{uMt8t%Ys?ek$4zyv9Wm{yu-eDb(zJGyC2+;vrtlK00diZ+k zvGedipwchD5&pwBEezHt6IsaF$pf(?vVh_&JXtp>hO9_*{f}^q|^zA{V^=h6Sg(Rs-olN^0ur=kVOo}|o&OW`s00ubX8f9$h%BiQh zV+EEi9()KT%S$kgyeL_TpF`>N)xV?y>ZizZZSz!g)c_g_mZoYrn6(^L>$U*NEIo=V zNq?{nQ!1G|g`suM?r&c|2G{b5aOK7P14+4$C?knKJI)s=UyIs1trW+&_ruAbT`Mo5 z8XL9K;Fu_x=s_?K6HN!L*n&IRW8(yb5`wj!b3du6W)J!K>nFBeR0u8Iq1|2& zP;8rGG_v4f!`Mk}lkAG^dT=-Gwx|dMyhD+ zrD}O@I&*>YgrduPV@gQmx;lXA{ijkNy#DZVy`wQ`Axg>u;2&;Yq4nktr6*OD?I;{| zI`-bHlcctm;6Ulr+V$m!x!K*9O76L7cB@F#)GVI*YTljo zAgLp~Ax2BSW*h>t!O?0CUbVl>W2y5=>?R8WIO0}J$~NlDDGhbMJY+*)SnjjIzDlmN7rBR48{dEm zQGy;FOHTG%904IC@ME(jp{^?F=M$~O0J-Q??bqr_-8wC%E8OR=Ml)nU-K3J#QWm>6 z#Jpw_ELcuM&aZD0e5}@lpR5A5FtncUa-PAq$)+{i2fC&5h(>wl{UFif8h0b8Em4vK zIa5sY9f~1xsT&F6P~nLN-BH@9#I7w{Re{*W#peCvBnGTb@=)Hv-ePLntLi<>WB_>W zhkKD_Lkz``TE^*b&{nY=<_JZxK)2koR@}0xKlyK<^GKY}v0TbS4dn4)-J55FbvVqT zGl>oIt;EfB;b7d1Y#*#+pB$)8Ymn4VIAk8vC97B0$@RwNO;U~d`g^Z2Q{x=7ZhA-9 zxig^)v|T|kQZNj`i?foG9e1H#OeWOgtoRm#s>yz1zlRJal#qyiNES&#$Wv{_la`wz zHz@a|k?G3X1wb}BOsHmF+PY16P8FCej7ecL_5{FKbk>Q=vio;MlfcRiQA~=&VTd)c z92abzp?rv0tB2Xvqv=L;rD*zQ1?MB_f*cxd8UJKwn!;j?dXW{CpO4bEp7}JnC+Y7}_)s;Yo z9h8xu0RU~zeu3EEJK1A)#NwRq5ushK{YV0~{^M-(ZIW5soj@{@v?yu3LTVkLSK3_4 zPG$1AwdX{65Y!p=5aHhj=P2V@vNYFe!%lLIi;$^yrfsJnduPBE9wUk^<2_^GPe#?Q zod)RBAtTL80y2n!bSIS@$FBfLdrSb6`HV>a4o)XteiJaxz_ zFuvfyVRe*q}o4zdnsH^HS2wl>8@whaNqAhZ+%sS9|p;IOW-r|*gc-)o27I-gwe zV$)Q}<}Zu@*BIIKrIhleLsORG7OIM8$Jn?8S!#zpIFvV!e5Jjnv>m0or%X# z9PFXhcT1SpsICivYjo0LL;hPE0}v<8qX`Hng%cR{@t2({{}xQ2O* z7lZdFSJQq{dpok-)u4>^{{~8P`sy82V{DGI=5hfQiR7?PYED*RQVoE9MI9q9CQQxC zi^6QdxXzbpk4P@S;vE8`)CI<{!+v4Lr<|PuBVVRAYUCi#`Z2)+k`(zDI72fO)~bI7 z%tiXEw_j#JyX=q?oLCLUa;B`V*XtuRj7;eKEN;ReHFTS!H%fb}+WJ+Fn%U$SF`4`7 zd+ER7{EghgC))y;cpiEZDu+>iyH(FvKl0^)(?W>xd7f~DRg7h&&iY#wPlK~7GbVVFCCH&~4@Ex9S`1{`VFA{q_s;#Y0VgQ>s*$9+u z=Ev;m47>sn^~*5Bvrxs=I!wJsACYy7W&0e1|GM~fn9i{Z7 zn=KOX^|kQbuYH#>I6oHPeX{4>j+tiIY}&i+#3QK;r^2%kp`f`W1+b|TNd4{;K;TWU zDh}nAEC2?e#Pq?il8EVbC`WoOA>ed1$w4A#`34Oh1DS%H6z3+a1lNw0-YY1O!E%Ac z+d{um*oU3zm*Mr(q$SOfSrG$;1*DqNgX8_LaH#zbVoFhOZ2E9RQ~@(@CM_;AmPe#2 zEia2m_A+j|PMH+n1H0AJi4MR>X;t}bzuTySlggALkl%Xh#7YSfj>Aaj9qor`BpR}`mOW&8}TuT61sgtCpS1lvP?7O<)%vK}T zn;eJgUcrCgl7h&C!IC`>bS3Lqj(ShIP`ph5{YO^73y)LX9#FV=?y$CHBU z5*VV6*kr>xokxcRszXd{w2ImLy;~q?nb;K|PQW`B?kf@6J%+HxOXPd{>(?*CsloKa zQ-euSDp6rkQvnm;yd|CftD}*l#0avMc)2PAXirH zrV=L-RS^!&H$4+7&T|t1^?wrfCQG(t*LC3he#HS~RTM}eHAhh(Kcq}BZ_ai1xZb5V z_lCznCK9u%8fZwFCQ`OYW}-(DAdm#i;ywIV?X}Nd`*@^86RU*xGV{HN8}8@qb~Tzl z)LJU@K9;@gQ}c(-+6Mi#cUZOhZ&>~ho?zpw?fPDa#Mdi>vEQ9_U%}5 zZZSLuYD~`bxc=Wf0w; z7yO*uIzZL9&!b6mz+RARGk<=hd&2F2qT>^~QOj^T06Reh7RSV?Y-Pnp$Dom!lDlR# z=w#%2chXp&{81HQC5`>mjHn%*SfQv8`aNO(1mRFXx$Ks7Nogb5Qgs`xt0W{IZ17P= z`y_1iaeJd|I5%u7ds|Lbs@uZG;``qZKm1|(ntSLsDRCfGO@`+VRNkQ`CN`x4xm}}9 zdehrIl#8@(sCf>KwL#_9vGI((gaM5j=ky)&U(trGE9`5qU%RTPDm7$lqAaw{NxcDw zMkiG(2%g6dx2@};wj28l7ARcHH7Zr_f_`AqSo4WR4+i-op{8jj?6!5-y{aN}I!VL07(ieYluWY1D zJa))KnW2nXE?Y_o1*gX*7wlRMs)#k&@?o5O`dtGzoqlfVW?p|1K&Cb>bxZ$igo$(j z=CE&@R%x`{t&K#n z^hF^GRFqVQwVq!wwv$=;qI8tch()4;5pW$#^s8CzC?lI+SKjf?DoxI>B)#NSPA1YR6$W{&j;kUmyr%C|B*rn~c?6z}4)8tN%V z!L&5o`fx{SS}=@Hm#gFHqIP|(niV{q75Iv)M4DaMD3<$43!1^7cIsO2fDB#$mR2hX zGjL|%(WtYvt7*jHba#ZqCz-bC(NmU|k~3OKgWX&gJpCAuX9kY#M^aQk2+Yk@OBl=sbxhHi;?xj$T zeFeuLsYV}w#jns-ac_-|u2I{bu-UX(NO)07n4?KKp;W#XzVjW+z!_XExx}V;GyDN< zWTX=#l_09!+e1pawYnVSTe?VRl+Vx}Mx9*?^}o@9KM*E9tpZSs5qKt$2>NUzw=@}C zIb!ZZ#C0!!aox1*=eXvX@nky;ez^Ian0r@GRw|4YI-oxk-fe~(lrpD5$~y%IadrjZ zziA2C=&{doEE4Ea3`#)$i$fbG?}V!5Fn?;rq+4B!-B2xOv!7I|#VqADpb%68%9xc9WLFK6r!vAGC;M)1$DMM4)3!D24-$P$c92YDu!S0yb zNeRu5qO`Lqhs;DvHnh-$rp=w&xM$6aMO~w7UymvX_aA}K_TIAE6Q7HtiFf}cNadv{ zP{lzMo!IQNgxGOLy+9t_rzR zBxbL;WuEn>+&~ulhu6>F{nzmRbE$r4o1KXHc8x=C5@rC_$@yDywx-==grXK3*sGd8 z4Jxe%u~-lb9I}c{zOG$oZ`tZN!4>c_txDdaGQ|srcILLdwVc3UIlow}A~kA;bH$X! zOE3s^M2oEwRQji6h8WJFv0t1fX}-< zhqmD+rXQ|qvSs*Sqvbtml`MpPNsyOh|=jXVkT!ClU){Z}b=zO<7bh|5=)l4LzpFb7KD&~(7% zRCX*UId>7FIn#at?Bq~z{@g4vFQxe%_ko26ESq&gu`HEVvLX;?pJ)YFp(xo0SGu7cT4H&YF%Na zSH>~{+-!kAJ_C(t)zA?}7D6axsEwTE1VNFp=a`vY)@tl(0op1*R?=uQDVy?CkGRKE z(UB3s`FJg4$6&U70yzE(oNr}5^nl?z7H%`+WV&E?No~qTuANRqmZJ7~FZ3PDXdE?= zd-2$$l*5pb!N&14Aj_bDSAK+Ts^ma3CCDq4jDztgs6>qvah7&=PfZo@JD22F3a*X4 zDk)r0LfYHdrl6~q9E|VXq}{R$Phq*&6<(!j+w8TAiY91&gri&jQFo29wrVb2bP5~I zUa0`Z$@l>Hse|J7o$3VwCEN}0rbbWZbmg{NFu9`R-d4Dfqhi+8n*q;I0_q^ytmD*F zYWdzp(^O*?AIk2%7C4^;a|pe|!~we#AgpYb^O#;>v9V5hjsThFxGb=(Up5eS>Ml?T zyD9o&D|Gm0<0YwPURX8bAbZ(tO1bYpwP%ktj}EO@z*8hQQGYzFd6gtR6|^W|6x2EG zoXd09B1f3`Re1fAy>*VQq_vtQ7N=j(kk6X>F9y#arK!^u7kl8|zF=as|}Ku=$Q z7&WKh#RZ6`;@L$}B$i`hanfj$6jtfr4UiuQfriOGJ%IJdm6%v<(l9DzMd3Wh<+fA) z=c&k)8fj+w?CVudz??R`z0`_GwxZSiGG~4M*61jKf z@6vs=yJq1gH512@-yv91&g{e_t6@adxh3PddQbM*KJtHgv6uettM_s({-j%KJ5Ks3 zED()5;!a;=JO}tCw*Ew;-knCTzb0Q6Vg~Pj8{Yl&{cqp@URx|QnI+CoO(=q<1=kXl zOILntdPw&?U71T0&3k}0Clx$ES=;U-3IYCL7)ib zVnHzjDd{$8=&#t`Hv{pXMF2k6P@Kt9r3CqOpA9=nYwc8h_a(q>t z=&GW$BZq!b5(l5)JKLboQU`DOnEf8jvi*kHZcp3$m42d7E+o4E6TeZc96)owu+X#X_^!^M_>$KG=yi%ZEAHdogGp~ zNoVybqAg^~tv+UujQ3Q`ydDw}k}*Y!iVIuj=xbvXvkWlM1#deA5lgv1V>#twjFbU_ z7Mr}yyqS=A)m^zb+q@sC70f9&&-PmzaIjfeT5g;Yy&zl?TpCWjl-$PwfY0*a87MG> z?Jr}83A=Tp5SFB$ul&~+%mh|D!Bg+>tlFn{_YUSDYuwuXG7jNkx-Kh67dIVA7|IuN z$&h+AF8!(!faH=*uDNqV4u!4P&WU83i~-BA8d&wiko_Nw*D7J_Mnt$ee5W6JvBtEx zgDCDG(IV4Rclf%d&t0nJ56=dcsP6V}FlwxhFJZ^ikenMT%B)0 zSfa=#4ye(u7GE4C7zbSyDTy1yJ-u zvaKgRxD?&LMPF6NyU#oT=5h>^`ULBIYT===)S()YM3UhJ>~d>Ch@BYLRY#voe3cQ& zr|lQ-P0;BV#uBMJ1)oJVC>iAKx=@+Cdw%doK>{dtvQi{uw^3lM#<4&GbGKjZ=0{xI zY-?bS zj`QLkUZpAG37ePo21t`1r43gt0o*Uw9k%n#nKgKX7YwiLs2j6{Kf;M4FZCli9WTLm zR`RxX>c}1GXe2kRnzBMok?g;j8EaJqf+r}J@=M-3YbLWSwYkPgtt!H>50JC^6e$ce zEw`96gi%x7{ck{NcxV7X1D_dja&;d(#}L|;$W-u-hv_11ryDLwp5olxrUIq~#*uDO zhVr;18?>6lMR=;(Tf+oQj+~-C z_LW#vbvw67-Z4vZC+8wUV4g2)2GEW<^r6IYP*X1hRGeJWaz~H2C0u18AM%PBh<%Xh z*##d92(5-;S8YN{6NYn{=2hCj;T0sXVQf5R>(pX_ce$1S9oQ&xaV-6HCsG4Q^!@3s zj~%xjeRq2!wS?IZxXV%*s{YIp6<@m`NP!E4+J|f9gK{4$mf=mdO%L7W35|ujssqVy zns)H*mas*^FHQG<{?B&V?T;tDIRZ{$-D9A#CrA#-&tk;{HXU7)1xu$Y*4dY5=uAST zRlLnboiZACbm34jv+*#69JqB>8ZxT-7-3X#yzw4+n?iz*HhcdO~ z1aRW0u8$8ckTFQ5e&*I~;M+2L0ibHWaf|(cK-|_@ztXh@NfiLLpF|Pt3Zr?uwa%JS z^Fk@HMqrg~429vDFC*RP(-uvWI)M}VRAbzRE5TViLJ9Kr7dbEIjqV*gtZm1Qg9NtkE4Kq^r%l5 zIdn{m%GLr(x}6qtg&L|YD-i~>asqQ{#L@h_|B52yA6|c{KsH}4sh0U0YAAjV&uN*)(~n^Vn$bsQBA#@&xi5-x|giWBe(dI@cOd^K`AH?wLUse zzIT;lclIIV0x>MHTJ<>$lU<}2-{z|Xn?1LBuTImZK9Bw(nwgp2#D}w3{W84%3do|z z!#u-^6>7(>jnYA+6hPAhR(Pz5fqdjt6S50a)?}dr+q`fi4l%|O|3<^{f? z1xgkD>FdYgZ_|k$`Nc3VrBuZA8nc46lvq}x-4*uf^Am+k;vQ|CQKnk+0&T)LRqR9| zTG}*LHeozVEw9aYsSL}r; z1)+0z8-mCI=K!i;$y6WIRor605+%jJ zyQs>F@wE>_U$US22CX3S{}UW`*`$Vg|0ft`p!KhDNmeLbjt&8=r5d%JKvAD+JQZ{G zqE{aDQ@q@crIB2(!u((AzDTtabEJ}D{{hmJVmw01plKucc-=L=KSQi428)^@SfdTi3@ zmU5f%Br_oS$3=rGBI*jIfJMhEnEg^(jPc=nG(&E$kmRyik6YvbwU+FO zGATABbrCy)5IVu)J%Tz1B?Kqa5+gsT(|lcQAR~IxnCw3*R2MRF4nUaTpEotkm-uT_ z4;|Y!B+HE)#j;5KS!*W^bF*2HfvWLbEVn-B{M#MgdNoTI)s=_=ZaSl&zm_xk1AAhN zZ-Fr{@2HLODSr*${ugh=*BbXUGIv)(?-bc1Jy6dMI5>QHi5f(FW_%E$v=p8eP3lbz ziT-3LSmV`Lv%}bLc z_R@!+kglYDT`OA;BpOl3k|4uQoGnlX08n(W<;xouT{zZWy+PrS$kliY06_v3THK4S z_lP`47)XOFe^KUyrYD=+=>6@yOE=k>a72uCi<1hMgai`MCm*AxK7>2?MMH%C{y&B9 zrcbqOsn*h_Mw=EX7tzAGwq0}?9v!W9W3u2q%ujdOXX=ojIU7|W0qowLa27qZC>dd{ zoUK+MM)RdAddYeKgmE8jVWEoh&*yjR1Q37dNI4&`Dmy;ImAZr~1q=l+a z`4kQc#d-v3_Ji|iw*a@$y(EqDO!{X;qi1xx^7 zR!{)z1UH^;&3Gkxgd`yu%eTE8Xvk3&n5R*>>?R3D-UIE_$l1c3q(E)C^CyNz+KoL@ zr)c28Ol;ILz3U~W4%+Wa+#q;^T0?quMxgc_9zW~-^q&t@uD`py`v(|`tt$Uw<-V@! zT1&O36O>(;fhMYl8T;ecloI@P*6Dd6X>9~_iSs8#iI=+J zHHuT$*ZP^?4nR=mQIu+SFC9g!reif=_(h@Tf&=^odD7kq7Ol4U(4u|6aikwfLax)n|w?&~F`rz_J>!*JrejgY||a~TnzXUO&Yw8J>@K#h^fP%(lR zkQe8u;7%!}wSxjAx$eL*+jg{=foHioG(J*y08zEH&oev-H;ggieCJS>kS6GSUcYWC z_GMpI{)|+$L3KsA(89dIia=;=`}K)L0`hrwH5OV+*FccP#+IaB-1x^p;l)bs4JHB_ zstITFa9h;*wA4Gv>$mX2#i-rJKavY9`>I$QhxCIg2wFCs2@G1!%mnD$RqaIuVqprs zV^4GdFK?l>l?Nrolblx<1}%t5UehjT%#@U#r>@kNn;SVr^~pB|s*H+{Dp7ynJvcT1 z*~Mjx##P+LG>+!)BH8ZKA(bkNHSt_3L}Oo8TC${A zQ^_GcpnTvjz0)~@?}#bDIE2Qm2|5)PWfaZ!I+glSH_f8XI8MdH{cx1myC2KDMe?gD zzkm_7eclhTT_r&^o~S0Zl&+k#D2v){5%4gqxSCM0UXz@A30PNsQD{`T#AjbjVbupU zqdc@yXY4oe>|gse{O|riLS6%#f6Fm%S3urXU7v{Gt<)2CkRo!-yVWNx)->#7kl~Q= z$=jau>xco2tHVek@MKC%a$qMsPy3R`^hVk9=Mq-ZpHt{H5t=jVT50%Za2Vy&x5>8Q zN|4|O-%ZV=fmVKuks=2CgTtsQc&z0$Qs+K(un0Br#Wd1y-Z3~A=-#||2CM^=mf`YTvr1N2uWEa0~ zO_v2np;%o{rpDU|Lg2V!0ic+D|&bZKf~zgoF4j+r8*q)npXCGdjrrPD;oDlLzW z>!d^pRM;Nyb7HW?2mgPVn8nV9eA}fY=qVJW1VzF#VI2(j8jJ2d-&k^dWTxQV0su6$ zPAzyiK3%ec%6nKCmYBJ^G}Id);EagwPGJ)`8}9q7WH@=RUG195hTQn-ErerHr2R-T z{V?ak{0jXJ{q3{-%Ou%+VtJylg^N)7jzGt{clp-Mu8*W}Q?>6f zR%8hVDcAV0vxLRxMY_CERZEQA1VCH1G@(C&p267+rJ|C<^XTSI@dcO~)a%^kB}bY!HgrjxhQ@n&^ILN~!nWxZ~ zxhB;Dn@YI`Y=h9c`XyL(c#E)6?2!L`Bc9R?Gx+$@z}g%Oq!Nrppc4km-$@-lOwS#= ziAPu4KSGvwgWD*J&Z&iVxSjEnA;RENqjJvSgV%91c=Of!YLNs473OIPgzeOWum+(U zxUS0mH8GKd2w#e;JyJ}ME4AaY725^9Yup@p34xGVWu@5QzzANxH1OWx0oa!P6!IAZ zjV`4iFw{BNr@ZpRY2^tr*MzS=!@FV4EwMUsbzWLxBuo%apFC9^28-Uy%=SLzuYtee z1-Eq+%FC7Uk+e~#yfpWSqZ6EVl4bfnN?1OTaUtwEGT-5ZkAReeG*^Q^*$SdGIr z&C<;c%S6T1$~A0yY@L?Wy6$utE0t-Qzg(kKGisj9CV^`l9^QVZRZS(3s9sWlHNg%U zBOGZ+-+lc4)9~m240eWz2Nn23C7Xev;J&=eMuAt=9jjXXjgE>eu1w(xsIM{E>8_lc zo@V#RhmPKJdr935S_qtr6C-WM-i~i^E{wL|vsb!tA++SX-Uzl71 zTUoYLhECQ4HP@6XB(wS;Qsv{*t>Orekyo#Z-U zdc)wH^-&VxQN;yKfwctg8rA{ZetKt+hJZBu>gRq_4}tMAKFO=jpcq&7;O+^)xWe_C zF>6{UsEy9k3_9>i*EztK`u|CR=Nl+-&r4Lhp6-mr{yMz>oq33d_uojyM7RL|E^}}8 z5+En`b$ICmrh6_Q3*84BHinx!&RWk6#o5XMI$bE~5CC0WR0MYq{!&GndIv<`%8Ak` zW6X+&*gQ)3*Z&RYe}LaP-1+1QCl!bEak<)9!4J<6H`Usq{x_ zYc{RPTkMBRvSCM++IIl*RYk;5?wrQ`@4tC(ED!n3WrjrGsOqfJwM*)?Tm#ryS=1^< zH$u%fF7VK?C18{~5g;~KIFf$@(cgz6XrNMMWwZM`)=QCX+*A(Gy%egwM2&gRBZS`T z^qOcd)RV5b^l)q)T6x3VV-^|uy<#K%paEB|m@UbKr$Pq;k`VZGuTbCSd=w_(aq-9t z>oCrg04y+0D3p4gRCjLFU%%k5B-5lBM558itEhmE9!Cxh1Ludt-ee7EiLPkHR(`B$ zRt-b8v5J1?9flN(1F@8@eB|(V+Ly0tNXFqA_*BMnfoi2>@(W7qkp#ryj95j79t}kJ zgm6Bm(TByM3Y?)Lp*ww&9U-(4}q~>7Q)7Tev86jj?fr-W|8{Al_;l zrjYn&b$PJOr&@Bs*oY`Llu2@UgN@#iWy+{#15!Ch7k)o2%O?4Q+#`MMA zv869wFx}y@4E~{RtD!MKE^4d$Fn;b1Y@F%BqRn3P;zkZl3%PHqB==G?W_OZOm6i-( zEe)MDa*r0%EsxKtg||hrC*gq&k;FwIPTZ;un<~HK@PlY*e-od$oo?YK{qDs0a8voU zzVUIN!r0jtl&ShBZxlq)qL8|glN2d!=@UO6cmFXb4W%t9f>cKMK3Q&2!-^@0vn9`A0WX-uGNmY3a__GxciFky6JRHkan zq<}6>kxEw9Qyv5*e!LkLVLGQ)q!t!>Iw~~ebE!ZOPrp@&7RGe1%g$2nttGE+gn3e2 za=5A0iR;wHWAAR0#t8l$U8u45XbO9OI2bP~x6FN2EuY(x#BP@brkB>aYM)$`EjW8a z^f0j9?S{>)3lIsbYF}ImR*J(UVkqg(f zP&xprh$_i@y{z6|mhcp*fu}rKVJ%#M%xRli{xN2c&q;bP%t_Mq6Fk9~ZQ6 zfYBB`v@KF4=F>(wJx*m!)W2WYhXnD1Ds^aPJ1XrSK^dmjBZB8kFxbRAfUs#YeH|a$ z`p9&^xV*8C16#!BrPnJ_%E0|g0>K1mM4jUKrZw^n1lPd;XWrEsv7NF&1IAd%!VY>A z#gH{1)}(vM%Q+{rT?qM%z_R$tP93!!kforFISUDNXedr2QBUq;Poku`I&tbUqXfKM z2|KO|%8x7eaf?_%w+>7SEw54180Z1VBG{C&pKxVaCZgcBL@?a|c=iXTSJ)>%i8r^! zMiO8Eyvd1eOENh~jBa3j+?WKIn3)6F7hufOgX}86NEYT0Oa$;8%^i+@9r80DN~Ex- z-6~zd?03R&=e_8#G|yatN6U5@smGmm0bZNgt4e{PA_dfhj;H2zR1XyJ!q*+)aBy@O zgrz9O#NRbqif7@a^$q_HN)0dmS*#8Kzv8yj1m;>SScv!p;0l3FCxlu}{2La>zu`NU zn+pOZcec`#E=Ptd&kEJUoQsCUV0}E2G|Ds6ot!eCt|jHo7+DnrpQ}B~t0K*mO1+S2 zkdO%NVRfUj&j7C3q^co{D*W}{X#Z4~U}S#6)FmXZY?hG)#1%c4tKJiYCmn+MI6S89 zgiWNwnspO3qNb-kg=orbCvZ(UI+&&y6^zL3OiI9)@7Vu^@0iM>lK2XL6bLT3Of{{e z?h&$(D;Di827;z{3vlF1Z}6-4-!MJ`1Zc&}5FD_dg*}@C9xS7#t+;e2gewlHDa5Bd+g@4+i)c7d*ki~aAldhz^KFGGRtck> z)A;T66F_Ir;3~CdMm*5vUV}MR=c{^M^G^E)JvjIoEk>Vw6y^;JS^o$yV$FkYNQ#Bc zc8c&)-cXryqZRWN5PUWolFRrkhWUgn=a}WhMP=!&YwwmUgAc-A{Ds}62_n=!DbQW6 zJxTRUYgCnMd$GW@kx?Z-BUP4^o=|slHUoXZ$j}PVe)X*t2h6|b3$~yx$1bj>N@Xpc z{PyMS83J6Qg2cMX>$2<+41;Y@4f2+ZB4iyBD3!FK`0bO@q+>r_6Hw9j>PJvZs@SDcWQhmt%*dHJu8ve`=>-oy zuTflt5@aZDj_>7WF`d_-SM@Z2H=4!0m6|&`^p-vXero!B$d7=9PuhWuh1=F7i5UI;UB1PK zyD_Rh+zWWlT7fiZJ+d4ma2bzBU>D&CxDIhOQ zY11`pOZ_N(hjAU11VIv#ua=4UCaW!nF$bQs!H>d-bhrV2vs(%6sd$t(1bhQOGan4K*t{%J#^HS`fM)dOq+(*8|2sZA76^`rE@d{KxDd# zJM}OX2M@+yxt)k-NJ%K#VX=$kz}z4K;Xc8m6qBJ51>(ps=1M9Es3hI0s`J2~?eKjH zz!Ur|B`{V}H`m&bTD&d+nw%6BzHs{7Z6S+9U3FLeGSS-*%q~2D=JJS10^;nVtqa7} zc@veIL3Qiuk~vpm>))&j+?NGMq8l#v;aDS% z!!m1c&b4Zrtt*}QT>O9qkmQx1*#)J0j^Tk_Y#=v{S4swZFv2?Mbrt*xoINA zVxa&zhBM>E9L(gatiU~`*M`2D;$+xrP4XlU6~95;-h7=URa9BLQN<#vz~mOYfkntntSiY}KZ1rV?*j7u+oHW@+$1&HF#{&ZTZ}1vAG7=>Uvyh411k)ub15 zbFjs4QppS1dFgNz&n8`C?tg(gOHbn#VV*9%W;3m5@-Fg|fh0qn*(}UVBx~rWrAxq; z_g|{`cc;{y)9`sS8Zf>EgB(YWEw|@+1%wM2!J(B3#DNV+hotH~c`O zYIr>+!fKaE64o}&EIVKf%o^h<3>H@)%#+TL*>%`w%bf5bJFuV&ks5V?`Yam|6cig* zuBDtSV!TlbTN1rT0svH4cUngN@os^{K$VSaF$$2Q4R#H?k*#Q*(g@UyL-^qhW1TP9 zAScuVbReAe0Ue0QOgZnPdQUPN%FW#&+)gZ&DhZPzWzhXft)HFAK2EfV@%~vJzX1!L zaOo{ltB83Pxmsd-`QYv5^{Ly#laxwEZ>?1PhO@(cZIfJ1D4uxtfVFfTVrYoTX*f6q zFV;#$c(^!RnzYE_OU2}^bp99vgYYTKX&E-tP+k4>Drcoe0onA%K8VY#}5yeeq2 z!yFziVFb4!2+3P@(5K^VQHMyT=Nsx~AT#RdV zG*E!Zcngi%M2QF zKc?t|MEp!9pR!|X5f84)fAIbA@6(g{@2|gm{W{1m|7;+fm`F@eD5N>|gfnT~s0y1{ ztQ_28v?y$z7w<*2JG88IXOWr$tq&y2qErfT`T@(}-Ui7)2mmgrhwfC2xxE`M5>k zY?;HL?2B4m&)6S8S%n90sWL$?zT{KSWczU@Qe9`YityB;&r+$J@Ly!sD!N8+ix(2LqZrAN_mt=o+5e2W8b2Rd6WLY?5&MP&@X z|2pN0uXVLiy(wyc9&8)6tL0*rKve-C;&{snh>qfRrNwlihooRuTz$l7XuMC}!t(Sd zcR_kMC#8!JotV4>5?;L7EpNp|BJe`tD5dLJh|5eHc=L2#6sa8)H>Ysp!)oDDw~9ly z;c4rib!)o?wYqdKi&H|M3O8^}@a6PJX~IA z5Sni3byhoFRXh;AxPAFiL3zg0ERi%gvNONR04QnMe={6-}j66pYaNRt_9A(dX%!GHO!XNr;1s3%OCo`NykLR zC!NER_4c&wd$dX0*a)J_bXDElY~l!sgbnqJn@xvUDIb6BZlr@%x8$!xUAm2C&P%dt zPv39^>U4C;);lEr;o68K;WXQde(=RxI$Wu9`1AuNAG-9YGpkEtijwGgn?aMG1#{UFb#j9a#fD^e-#o8#rzM!(eiXo~PEF=)IGCKwI#u!2u$4baTwUdLhsB60 z8LK8bi(H9j8Qj}6p=nfTO3J1KPQD8W#Q;*aJ9HXG!pFA)xu-vTKP#2^x9qI{WSI%B zeoy;Kw`~Z>k7IWVXKOZUt9SKWz3~d0eZh_Dobp3Jg$KlGB|KIOptGQ;gJR87I2JzX z#?Tk9nV_GN6A4K{XA-6=BjrVv^%tYs-}IV`?g{6zNbg~CoeM|xp>SJ@SiN2T5^4;P6rhl zVNzs>pAn2sO3~7e_1$Oiv#zz!Kc~=7vxhen3$6dd)C|^nIAy*!>lU3VJ2ylg@^;4X z00T+0RK?EMKzrD=6J~v5!`$@I_ zgoZ;HsW_W#yWD-D_>$_FxRft0OTzM4mw_7N^fdm7c)EXj{W^UqVhTzyJT7&jD5mfJ zz!lxRGKOgWgi^g$VH+pur~-^6L+R2mKwPK}03!uLD4$fN3YR!#m^esQXc|m;uf@2^ zcVDfuduZk1X;jM@rag&2`$&3@JW_8bI%X>gYtDmFAWBZV;o+`7kfo}5VkM(N{lxyU z(DC0R+@x*-CH*~(Hu83#ca^Y2QFrh&$Fsu_t!vd4!s4x*@=h}%RE`-QCpX+e z`L#_4C31Znx#o8%u{ry`U%RUmx>5ph333cPe)O>i9-&+NI9zM&&==S75f7v_!|DC`TRXit`l@<^PBC3giBRA(eZ7g#FOIu;6o z2cvI^+?oAzu-~MiysFUIkR6RV&fX2Kob*_(VKs0vL6-Qv5LEMRBaDB9(gvZT$b7ax z?l2iNE~${99ZC7fFlR(lV~Q1+yQsX3 z&mqm42ujfqE|3Q@8NyA|^8$+y2A&D9i#VchIBE4KD`;QYPtGPDmf+*F@dVe1si5FI z?cGn`|1=oLmM({xjPV5i#R~J@EN_?n#3fD{ELDeI_6!!x^^2O+KjeaECah5A)qs`c zDyWeDz(x;2RJ}&32-B*Uq{$|UZ2ar72&0Ncu$QP3;J3z>8XcAnM(myU zp6TX^#sXDdhbR46dpe=)pmZ5gVRws{Aq&U&g9*+_79y$>T9fQZ!8@_c5d)<(>dq;S zD(!Z1n7Qx$i0iApn-7{h?gi5HTjfMtD-mAo=>Q;$&M=3tGSpO9aFiySc+TtL>y}#K zTo!w$>83cmGu-z5zr0Q;Kd^uPgFOeiXOajxQYduqWbo@5T-#j@ z?l3+ki?OPb7b2mVQ^-{)k1po`-5owX<+he3C(#x7+5L$m23AjcE5M+!+L-rQnjhXo zNyEQ;{qhXp+g^xng*N+D8A};gYscuJD%m#GqKTt@gFY5-7#4DqwvDAN5ER_L;XYAm zI5zSkG5zJ1bDdRALlG1`j!U>OR`#ZAJ@}aQ3J(Un9J`-X z=~XiJ=q@b~7=mAqlou@&WpBExBz*3}eNtV^sYd;tHN(O8+nj`BAqBX6fD}hB?AL^7 zwog!_!myhRE%vpd)L#;dCd(Ou!pZUb5lx2}L$;+mPk6BQ>K5mZnGyjcI5)%80kCW* za|bHiStYA;!L$M(1&cm*(!xQ)&3tTKSwYvLp9g|5AbOMYxT>j4hyivD5f*&P#SXja z=P3`xSm zi8{nW&|Sji&TU>dK)IBqoU*mfhC%VMhS95DDU&4>(n4J;1bHqln<}w>3elJ1T;EPr z)WMX6rco8OdkmE0Y1XGs5&knt1Duv~bSsu#mejQk1qFc;7;{On02SMEvT4q{{~{Hb zZ>%r>@%!OFrlX7T-o{%*ebZ%gLG-IOZ{Te1shfNEUm=|F52h%9mdb8-d$>1EExEDD zNPtSwRmALIUgo1<%x(8E?doGwwo6r4)zm$S(=AZzZh0)=`QeD}Xkos8^B! z?D4|(b>Z<=-Cnx@q28fX&JLqED~Umpgz{7<9-R+f$h*afE+s>#cC|doWMRWL&-y*B zW@qMQ$AUUVjYIDY;)Wl(W$viEk6FvYQumde2Q)vb` zybIBuQUyh-B@@V5g=?j}M?0o>h5yd#Txq#ot^GLSDGLyUMDL+!KsJ>!5~8SE@*>vm+F1E(p_U|Z3rrdNcz#|9ERnx9I(S^A4e2Zd`s*E))_f0&?bVSxZ!DJwMO z6C9x%Tz#sM7U`M6H#% z&f0jt$t%X#AbmxsfiX5i2X$aeEtN1IkF*sd27Qw1+R4t{K#ylA!`WBHOo-94vMw~Q z_scv|!$6JIgy*p0;j$G&1sj`=ZmaMZ;pn=byfX(m2w~f{6&gxRFG}6vcSp%fWZcyh zhNJiHsx%=9=m>A^i*ur=Z z^!X`EckY(AEA5{|^Hpk!1r?#o{;N7F5K&9#{b(Q*Yr{;`>QVqHqfYu8_;A9zjE4t+ z2i4E^9t}HxwhFCeFPr7dJ@6uX*QBeyhHlJVLmtwWgVKdgZZ`eyD?rCSy1e^TMK`-} zYts{3CiNP05G$Bw4&y-AMLtXO;O&813t>e41cRQPO2)NH{3<0zPNMD!B`Z*o0U)Z@ zuaQdA0QQ4o*h-G?ob97VsG7^GDdg?66bx4L(K(Yqy3z{a1{e}z@)9jA`?@0LIp5NK zW8`r^6~0!eCLYS{3e)w3EKoVZ_%kPWEG1Orrmc?+J71}l4yk-NYnMa-@ddenlG~!9 zbwG)}p$5e6C97BAM6KjUl$cw94#n}^=KSMy^(0o(`DdCpg=_QZ6ovC!G3@jiti?Pi zG$bSj^wPa?hVstVrY#an0)@lcH| zq`KmuYPCvpsk~jQvrWCsuK2p>o~t!A(PIqEuNYN4IrLZx8dI>WxI>W6&{~JiJxyWK zm^CXF2m9Y{#?^!CkNwdD-e7%{uprm)@JYp^vne5@p6 z@=fg4s+-jo)T{z9arW8AZM)k?ui+f*9sW+jeTyDI?XJ1&FTBQ-q?V^F>N0tLW_Ils zT|sc9bW-Bth(I2{?(V0@>VnXlT7396v^qbZ&v;cof7g8R-(ZfB*g`A^oENMD-`BMdBmm7Czfsmza9w-Du(6ZxRBx z3KGMG9`lU57V7pPbbYt*Mc|ZCRGnOl&%^58hgAEe(N%|1b0!k~zZB!!VYh3pqm2NLM1I0@iJKr^Xl3 zDEj5Gl%QgHf(r;*x8W_2@Tf;7N$<9X^$|BY}`giV)QBwpwtL}9kcG$ zolovVqsPOphH`3v09Jj>qkbfxA%ghWnfM(fJnYBsP9{uGh3=tH0y~j+eG-T!5l~Eo@^66Ijekt54?&r zwiy4sMAcs=<=`Huxwz25c2cEX%00kU8u7rwIibk%V7lz74PtEPxFF=fDK~gpzji{edhre zIeTzw6g0q1zz`0_0$&T{mr47s9WIG0_};dJj5_b~f@XSlLqc4YcJkF}tIU-2PF$#Q z0&-&yWD)e>i7G)SUM(OzV!UA!aqmh;4PWKrsBUI9h(XM}ck$s>IIK0(mHyy^55f<= zm%dnXrz2uYCpMV}T8GOhL z^IxR{QK2v+I0s|2F5bBdkYhPrRMywM2CIsJ8X4WRP*p(5(h~@Bpt~P(y7IJ1O@|J< zAz+Zve3cdO=XOqtF_%?q*DGh&36w=v(yhVKmT#PxA#Xs5KV<~@DOBc6nOAor_F3X4 z&2VxPseH1hUMB8N5n-{xdbN{tb_0@Bm+X91YcjlC(CRZMsA!_RPu3e_powhF zf@+?0*znuo>jw!hrg1nZ<(6YKP|)lQefs(du%<)8R9c3Um}035Shy>0#2$k3QIkv}GxA(2`rk81go{{dKdCE6NQHyGjYPZDA9xqk&6ghhvK0 z+iwGOf{nAMq#0LvPZ|{VNuU5}WB;lGz07wytJ%t>e?;E6j3*t`Ti#Vb)`)WIO8bJ0 zl;32G9_$JJ`1OC#l}C5|DK6JKb+C-r%jXEs`2V@nL&IN-~$cAX^S)MkA*d6=DUGFq7IU zg*=q=C~xKg>E>{~6tQvdWVqQI6Zkw&1r*-G2b97T+38{G3C@KXH4H;3Af|w*$=?!p zeHgM{5!|P3{)jj&kEnQr20_}e=JG9%VA8yHS>$)!tX3CH;c!RXUT%!O37nZVFce3b zPu?m;_`q1Ly*=7)Ih(_wOU+d?Sj+bCu@UiuxI~o^Mha+@t=t>k;3_=uz%Qec?b4&` zE0X_PqzcII64ASyLBuC&)?TqV*zj<#692uj-8OK_;1?k+IUE0zxF%oOT%c z?@k+NLF_j3(WVtNsHQR+ojs!0WVJoQRZ zbRdkbqROZf^m6Z3*Y?`r-au(|AZ%2(16*3XiJ??w1eES_crT55S~`yndc^f5nLxSX zRkkOzg8KLN>+in(&*6Xf2a=m3rpJp@rsW){yNI{NK*BZ+mpEZVm4Oy#n%e?oODvNy zQqrNz#^Aea+?Qf#gS=q5%ikF~^sr1bC1R8r)uVy0Res&(M|V}c#Am9I>4@^q4ef(5Of5|v^g@T2jY)N7VX1LQnDoh7>A@#m z_$l3FO5z>f{<|+xXNnB2z}%x*xKY8? zy}Bx*d{X1UtHzv)H5Mm3r8YRSGte-k@%V0B!~a8lmrG zzb`uoE?cfDfI3ZIewnhpmD(f|MEmlRv<~aZ$!kz_WfH-ke_kQBEPx3yo9VC)?WduLk5(~YGF!_RnKQiIxdo0;qv|m4(ASO z%0m6;X|DvCWi_UeR>wnqYe7IM8-;`_FSOJr$l(OAVscCjK>2WLaNmJ4dE`z^2(@|K^@oP!@&)#^!MVJ7QnuIM=(XBTKsuPXqfikW=H}p-^KMsH&BhzP8G) zn({VNmvS`^rRrVcQ`%+Lp3JBewjS-13{Llw;-_T?+)o|~ZrZPGo8^lKL=Alxx$4|dik4}$LSw`aXvq~#92CF#8bWIy1de=Y=xq_ZMzt(*9k>BcHVU&~jn*=-nVxrAK=5sZ?7H zV2X`dc1Ra7TDF}E&8oT($PbNFz86XV6ZH}j3^mXI7P4qgxoz+%30sRpIpsMOI6mvf zg@-5c8MCre(UI&gifiO<0Dumpdal5Q^HBI!=x!?KnW`EBO@`#4O8IQCa#H z__qzarf4Djs%SKl!>T2De_OP^(E+EhB8$ri~H4-Z|LQ zUS@G&Ipr4{N|1#{R^(T(uaqck#DBN>56)pjb$wPmt56S}}@tZ5p*U;(NtJeAcfJ=j;VpV#$mdtOUTDsOQ+(4ArtJY8H z2ZRXvq^7L>L|Z=HZPx_VF54CT+r&{(UP~%}0LFzD$av{>R@M7@vcc-T=tDcLv!@A= zJ=EAQp{rPqHdJkIy=Iog=uU}J7Cf>emei6DLw3jI-X#t)6C`lJhF6g%6)qtq+vR3L$F&pUK&77kE2nMieF8yDU1o~TRkATAzZ(sTpG z4eJ1!JlU5BIC8B1Q%0+w^4FB5cb(!_D4eZ@Y(weKI>b_kRrjoG2P*zbx!tyNIk`Gc zwN{vvGVp~6ZJNGK7fsdE0Wb;x(2p!4{6a5rAOJQZ>X(oQ2Pc6A*o>qzT;Qk4N-faq;whcN7vXT-;a< zdG>Ly!Sgf!Y`ynH9c-bo&1D{jav&NI^%vbgi|VB9 zL7vWM5_bu~Ox)_rzY8Cv;u|j16p=tVkEM6|gU?m$F_2vt3?X_3u|Wwy=}(MA8*a-U zPb*H)V(W`7o~1)OhS@z{@;np6J)iD@jalwpuJT?&x$I=`PFO6A)J`l&+C=nLs)n3Z z*TlY|qpdU&X>6nu_}UFzv{>GTOSMNo5Gk`dc;;wzp2nBrmi8$2zVJxx9et=j{vw^v zuz;w53Fpn8X;6X(cMF&ZRE2A7l|b+=^zUp{Ue$sX8eWdy*-U-vL)Fg~%qS%pJNxL}t5jg|?_Sv#^II(he;V^e?+;-QcDMe(vTB_|avT z*JZCx?OsvQ!Tr6295wahrP^HWZN@=4N%I7!H{GZ@?h228zU=OEG|LMZHvR1mwnKqgd55eGKqArNMKBY`P+GfCSkF- zbh2vL!DWhR@hQu^)e#irm{D6O6EX#PU()QJQ;KoR(i3IzMBg`1t8P@ay?45UoTW=J z1k1W`Nv#w9j5k$jW33v#QK9VKfFX@a)g^VbeOW9VyHX92?GG5W=$7!_(XR*P_|>AM zyhe}+1KqAVkhM33z^kIYt5}xqMDp%S47iU%Rjmytk22T(Gt61-cSr(Y-dx`OPsdd_8o6k&>MwlBTm)BnibcM<=;*$?W(}iRXZrPfW zI4|HMrjMR9JZJ=@>$HmE(*|LCDoz`m(>c41eca?GnaIW71>2eo78 zZZB4dQcj&&X)0Z+eu@gz!D0>6D3@Nn2V!~NefA!nx7x23giXD?0CK--!8#D{EzQ2J zdYOcXU7K}f>Q5KycUUr1iB1VTjMVe9J3}e$Cd-2W1kf^={Zc+j5)eOp=(etT@QXFD z`rOrK^0ul@OW~4UrS#u!^;E2TaQ{+W@>fa>D9^d#*Cs#bsbfu>>+8SU)j`7R?4`hBa8tAKNIXT^Cs-74*_OZnso=m* zlH7`*WQAUNYG=nZynqx8bG6zQ<4EIwG3l}IE2n;t+UrAv*~3vkF}2Xn83e7x>fK#| zGX~B{LzgFtfnw&ec0`)>>Md;3OXES$7Smf-qQ34b+_>=TNfty?Vg;%~`>5%>x&k(O zy`~^8E8OK|$`4-xfu9TWCZw$`nlgXV@)iu0wshrcpygxX#OmeBuwiaZ?GvBp=1D?X>jXy@us$$1d+R^c)c1>BScE z*X?$GVE-5Nf74Gh^%&?bj`jt_wQ=S;UdcX6XP9eU90t2qPdky6Nh6Ol9oX0T!2)x_ zmvc}C^&BGj^N zS`hHcPPcL}>Iv&Gq36uYZ0R2|5?s@a4ZHSC>?@LYy<`1XHn=Mll8!J=deG2^LShb> zck78eC{lWh=2P+;oEyGD6)fQ;J<}Ixvg!0s4%HnX#*KJh#A9>21DVL?Nn^5q;C(w$ zbyO;G%iCr6i~XJEc5h>o5S6!!bK!16a&r%P6r+3<^edhpK_-xEHrq2#4|tu>T+qvM zRJ=NoyUya-7Ilu6zw*@aTZG0(aA&JNy;blAflCNEHju4qxUd_cciu^`UgiEsQh+0b z_dm^kLY5RaY<8HtHW$rs*A3S=8N;$;A`uCKF9Ak3a1OfQ%T2c>t{Az*N{#Sr>4QA2 zVBYRj8caBGEYOpI8{Az2mBPx$YnD5Z95WE1zWQbRahVx@zXdFul4>2`Wi;Q&Q!ryqLvgVS-WmSp&W>(8&>T2$~~!VeeTB;5tbe|5XcfY)CW(g4pIV>yQx@FvipDVJ#t!R zj3pha*M+^}K=bO5vkl`FY-~Z$yg8slkx8XO{}Y!_9lTSHMQdE`4(aXH{7GFCqG_Jf z>rf71AL>ipH6v!KoOgt){iF%Wg?@o(ct~VM=yxjBE5no&uz`?kkzmtdVICFCO9ISl zH=VvzmnBk1Pz-{13pm2rj z-Ye!Pp8iHkj9=NMb}S0aH{_G2;{pw+%Xfo0kU^>TG}5v0Iri&P4})LDLy?S4*tCKl zaI=c&1>k?fXOi*>oSxJVTLJ4dp94U_s_uy6hsQ7Br_ir^P?S)6H4`!b%+~ly5~Mnp zR2vT|BWJ(>El{3--R4r}9(H`0?U3qe~g6zCv zFGCx%NkUR9m^|-K)UO%a9^)7*O?p^T>x^{A=rmU9S8aF(bWQz49=nuFCkJoXC=-MO z^8zf;&{<%P(PXtj+e!U`OR8U7kVz~=spE|m7C5{!HWL;b27XfO0Tx zgdajgT9#rc-L9o8OE*lEu<>LB==-^^k}`PiC&fMugLjnJebIUv2=2((4N3 z;)UT28tBGA?h{K?dO$G$Xv7BwS4UCE6_gb4M=_jia4#*b?R1j>X~-<-FjePMJESgV zfVwM2&cWdm$YDexB0}sl&viybOAa2+Iw2dc`xT+gBpeVTDehumYuAIEs8@SOEKQmw#=ka&qS*Kq%lk2T=gijgf@m zG*-}UE4k5}IB3iQ6i2>adKQmkgqWI%CCtjDVB7#U%F!8^L`EJ7W8H|1K(pOh}IboipC^?Njh zx~1Vd4SLziR!mN7pUO*Cxtwxh$AX)$|X&bJN_36KN0E2tq)Qvb?_6^#z zxCG>}<(wt&LHquy-Oq*!f+nimG=kU|U+n9V_GMt#3GNs=rGnBY#W>`M!q-bjTSTJ| zrB-xup$w@LogW1$#$lp)qk2>GsZXQly06{}2cS*ASzW!|c$j<1ThcDHg-VfKR-mD1 z(>J;#Hn>gQ+J?rIp;|0>QSpGl<O2K z+q5Ld0$nZKGpBs^dFK-*VvFlPfhu7|5$^V=q-~Y&EYvTzJp6gQHZx95=X;n#CmrX@QLciKcNzFEi zktCu1lAT)(Mt1sLX^CV0|Oi{+-BA zUZ^x|R2-gkIE9J<*=3b0zmOjaFgZ0>*GWxn?e>-`__Euky^RETK6^ zNW4jCJ(7s-azMs472DzUW911&mjICR#}p$5wF_MonQ`w3835sfXhi^BGBdt&;fL|e z=)>YSy#7-2AV&q+JhM;`v)tU1D%a2c{CE5{@HhHjUtpXw#JIRH82S-YU6v}TqmaNg zJj|xJOv(%_SFpxG%qX&vrz%kwm9KXu$1+1t*Y%kG30n>Q#l{~}Cfz&?d3b_rmsj^A zGMSM`d!lvGEjkzqATAyVorlZiO!oyd-0UN~IqVUTbXsB+;loSk3b<8osQbIRCM}9+ zj5shou+!Tx8Nm1jAWiVDpaX%B8K#m*B30>u+Ak(xTUHPTRnAHwz&e;q2J9#~3B)O? zDQV&SiKjh0?G&VPMT0)erlOkIQ6SOD^LYb4gFkxmvyMLS)2fNzF^m}Z{?`^t-M<}8@*tIEqtu9Lp$iKJy} zE1wW>zj^&Re+~RidVmuB7GE7yh@2TCqnsTIs4geA11s5B~9 zzcXpIYkz?1cPARHQpr&MUfLw150vb3>WQ@aGJ;V+CqfBikeQNxvX=Np7%ql}(*4)W zEkLbxmjlHX=M~V<>P+#MMD@M^AY^fv+E@T@uyh0cIKWh7LSIlyUe<^7Wn33X0+gb1 zbya%|DYNV->EjCXx_}ku=Gm!zJ*`yJ2V*?KP-ULMD6>_+3N#tEKiq7#s@pBBb^vH3 zPvnq5(qzIb$^*XXk#o0gr9KbJKDv{HZId%qViay$eC{5B-*OkM#7}y#N&e1)On+2Dpz zmI0-D+L7D82#?FV&)@$O61L*TPO!0SJ}67x`z`D4K8&ip&+T1F;nSj)3+WliimIuP#-)cz$SCw8CQO10T> zhNAQgtVAs2~pL?p5vLpSRFro~l;mXlDjlVmO#;pXiiLMoJZGw^;D) zZ!g%(IHy>vWK;-k=M2SWT4&ug?hYKDm3Pj;X!HUC{wRGR!9p7mOBWr<)So5YQJ9=v zF(+x7gcGiJ_B`s z7hXTb71-$8hN1-rX5ih!x@JrUC+p?f)wQDrV4r1g{*EYsy-VjbhAtc4V#Lk!+}BR4 zyFw=nT^V&M-lEf69;*%0yPfSfGRdF=UByG#w^;lpfk|VZoY=F0at&R_b@jD(L=&2A zR!Ndqphc0+)xJs0wk&q`o?5Xzj2BQ6@10!Er=rqTl6K8bS*nVyb};)mDXmm$%%-0* zcMPz$CAKOq;R-nlK(OjgVq;nQ<|w!uh*BFSKiipppm<}(S+y-n@A{+nAHQm=F2C3h z(JJtrTu5vh6cJnGl?RsE-A0At5r+x`HR)x19xj(Rx@ch;lXoGP=IT8{Ek9RgOTpZi zZ2I`u6$3H?ZDA?vW@pfqa0Gh z-PIOEKpb-yKCXVH@U9qZL0ok$Ku-B|b0rT<;Xz4J37%-blz6>4Uv;6+(lCu{l4kZ* ziTvFjUwgBwesGPuLY(2|Yde=rdpNmdmoH`0cC=TS95p#Kx=m6*L^9)&faXLplQdty z3a@|UXA9ec&_gTY=5U=6XWqkrv?pV&SdZ`&ofpN9q>KmlKWZQjrCi#50$?Wj0S11A zg29HXJ(Lod^z8gt!8K6Y>_W+37^yi@@OOB;8@?SI?1w9`jWBIqIKsb&1JtLY@2=C5 z$pKJ@7`t=lqh!vMsX7o(PH^J5OXvhmY13w@bQ100n^X>1k}r4k#iFu;(nnlk@~gw+MOY54R~1Vg zO0J5pS++rCp?bG=URP*X*?)Tg`F)4c7!8Z3?0G!5T@SW8=r_iEcw!kWFC30Sev+=N zR8~yPi$s?K0xlhQBterAa1$1}+AC$>7YM?U71Jm6Lm#EjQKiUS#Y8GGC5%e>4Jefo zBzq}o!So|KE$us?Ur^k@n6)Q?nbM+~_w0P0B0%=`;p7LtDlj-5v|%ZrR#r&S)7if% z%C2buDJsS_Jnh)dmR+PLlw8;V%hk>?Vqp1_2#eJP@Qmqfp#_&uW9tK05yB(R&0i&Q zvWs zr~@DetCspCefsf z%qR0HlOh}7&0_@+*4L>XuBU!dUq$;2sQgUbUpIsowMod~8#{0lmTiw2Tq*y%YnJ8E z4&RP9yq~>|MI4v^C;z`i)@R6UMd}?(R7_DN0eg|w8OaZM+xAQ%I1$t6OrzFa8OlA5 z$nl+1h<1p2&up70eW3$2Rq0i&;Vq%>Q(V=7q^@Y}Coa{^VH!%7Ztefi*qbd$b6jVF z_xTh~^^A>VP4WRoZOwTTna7fmkr^9e$*iccX_G|j%bAhxB%7Ni2rk%25Cj1d;9^#P z;l1X3$Io{>3X*IFG~%!NQI#1P;ePxq-xA5Yto!L`sf)(Dv$xd@Tz;=$Z)4hEU1BaI zL)PI2>0RYul8_~B3Kcdpn(c~8!ag!sghj)+gKuYN`0EESZchIW$8Rty*#p&SgLH!lO{t5eGos<~L}r-{HAQiKc@%$u%g z8308?==|DvNloia$jogscsvD~~Fi7phs!xt*jpZEEZzNyq}uFlK*lJPze~2+w=XIswO6 zvmqeYH&COZ#UXaF&fUE~6#F9>ktqW0JmEG_8y29oH~S$#=F^>lHOZ`u$@ezUUK#L4 zn9Q)X2)w5OjcN?=FQb(k4SHGiye3h7FEA9%c$)lNJlBd_%kn>xom3##P~c~9#`tEK zL2WG)4?9#k6v6Iy1fANv*;U|)T@r@0*)ac~GckR0p|I88q?4OW9E$uZPnEN7W(Wup ze1~ordI{(y5?cRg7tjN- z^U4TWv&zAt<8kxOn|A=-eT34=P#G5>@G~Y}<%!cJt6m|!7?5DohBmso=Q_&s1UR(K z(j()Uvuc$shWcRgwBUef8G_&6l{f%(yJ=k^rf7*5exwcvJXO4ccmQ&iz5-HTb?B{J zGtY|$Rn@Lh4SvuAT2~Juc!u87p5%zXV487|CzM<~3-S~PaciZ&e3V!)AK4uPrC}XU z5TLZ;DS;eS>F~3I@{h$4kO`YBl`& zr_&Qy!i6drNZF>MfzwI7ZHFA()2-=zleE2RP%-b-m1@WS8MCqco6tfkl9~oQE7T#o zN&|{>H`0nl@;vqbF70mi(7X3-T6Wm9JfN@(M&7wEgAK7;p5xFkmbb`!AO-Ku?3C}` z!7LmOia`bE-}0R0;cZq*f&G99n=1bUA;3#QLbZU*KSFyOu2E>MSPcUk(amZ1)hHjW zxolMWQfC3gqcn18X$aC3M9HW&NUOscd5e*Wy1MS#lULu}ZU;IJ{Xl0T8Wfq~L*2N7*CJJxJcWLMEe2V2Yn!tMxix!~ z)dw`cLI0Ftzi?H;*IT`Nf2S6{FA21`!l5G@&7kL-D*YsY&aDPFN7Xx6uO;>fD`Fy*g!-o4m`PoMj>ECRJOU`cjVB{6f|f#Mz7hScc`y z_nuX|j%%umz|x0i(0_k zxKQfqwCs>&DK<7@It=DIAJ>~LspY*=`1QNIs1}=`!w|~%?T=qSPX%{4ev_W2<6U$T zM!ko##2f3D4IsdeTzIk)pkbbV_gtpoD_SuUuc4e@m`H5aS(UTp#I1SeZn&tr3cLOP5=+y-PV_Jjv@_3hIl>5S{jsTXaG?PR5*$PYe~cWU~B}1{XH)sKP_4J+?rd`J-Ev$-*GVq@K$>I)Yoab;i-;sa^gM zSxZT0EO6MfIz7^4Rl>4cv~DM~f}YgeAM6Uh{*??ze0J@?YCV|Z>EN~F-0 zl+5!V42djo4xu?gScf`LsvA48d7iqxgB-qN~ zhQh7wU7}Rs+r;K)iK;inCQL??LD#`biKo=1mHk=D@&ogU3UZwzh8p#uHF(dDUp};a z2zD=QC`nLyvgJ@G&5cK!ElGBCM$(qFiP0CQXY4hjEMT*7r%_S@RF!A%({oTTf<;Pr zqa1|j$A?uadH4L;`mDdbY_8i41TOnSY6*utNCL8hi#o3XB!F;sJ2pBzRlZ#brfCv^ zlRx1L)bCWAsgs-;#SC!RGz=x@EnH7`pPE1b$?h;A3p z7Wzkv;}iSwd_EXURfZ^uc))~bnT@mofmXP`Akb-t9CD**R>*l+)R)}qggT^E@}n9z zr#Fvt0_=1yPd=xvF{4+rom3-9d6?KYWm0NzbImyHWXTRb=3kPkyrtDp*!WTwxGKm4 zpRxD_6c`b?x8nPp6A|db*3aT z0je%g+5oTbzFT%ZuV1@{&L7gp!V;0ARnE}eG(57`Wf{TD*S{nRI}t%HnJlrFpugkN&MPe zb{`ARm8dkJGmap$CA8-CE7)>MxV35@S&-@oLGUEd)uaohwe$cqjj0qLIah~kN~bRO zZjmxxtW@vRMY2m#@R?dPrJPufiw^9pUC!D(+txOU;+`Ylu54 zHI+x*4X{mh799D+Fw&NnDj0?kqi)RSrg0%#XB+w!NGPAa!JD{|vf;jVq9^E3Dn~fvlsE#-|itigy2?6X-C4&gmzJG?kBSle4K4 zG!Z%P?V8>bBncNpIV+UuXfb7Z%A{uW>BY9({OMtl`9>4?i}3Q3(>~bvX;SD)EkW?E z1kv>NOq^W9Rbp3&(+26+l6qeU$v~jeB*W?03v1-ew)-<|x^y|0lJk9mfnLYmh}8qB z?NfuHqaWWWfI=K?M0G|VgP!T&yZ|NC{3eq<-%6v%v=gI;e>} zLL*sz)T&TYkQ*zy_N};ZL({l&h8_q`+<}aW#AC8dPUTNz`_fISl>-CvL)M`U&!@#% zc1t*$Dqrf*xRW4eUT4n+H&+erYG;Jqp!`YRm(F-NIg-7!p`to1nu%Igs`zTUaHBT~ zTFDAZHfbhxSeRkA5GvCq8mmwYOe2*UtM)AK;g_q`=MJ^6*<0bWd}_0s7#H)ctR-|etQe;r;vJjj;H(ev)p*H109s|CA@AncNf$l9%mT_1vR=%8Ue9i~4^ zoswdrVVYS>9ir}9v+ccVHx1_~H(0wV5Jl%ler)eCof6Vt*psvogH z*r5;2z-J)2pbcu^QlG9E^S=$!vSHZ_G5~B!@P#LjsqD6JFhPNMMNKvMv+R6pe3gMU zk@5qR?T;i%x6>Uh|Kg;`D4sA~-Ke-TIS2|UwaA!C0Cj7y$KR*y-d23}WG=Ca;?`VX zyJ%ZfHI;w$midh%+~Ad9dd+g{RYKV3J8_ZRHCrq#7%qU)gm#9!x)k3P*xmu2wkNm? zWpEa#1%0A|F7fvQOLDQFPf3 zPOTtwq3aIHbf`9lI>@^$v^-N>F4a^Z3{`VQz}SNgNvN%IxWZmJaA0j(TT$)qaHwaO z8n8xqA&~Wv1$^cN0;771LQ(1O@Hvd&ocscMBY6|FJYC0hYDy);;Zr{ZSH`Xv9c#;8k>_po z94=E8SP_n2nU9UYsL(9sdLj+(U5{Ptz z=!dOg5W*;DPu4UTFy#&Qg9@UsYycFUun#)=GJFIf&~DTgv5uv)Sls{>TPa zkOg1u>Bw^V8m1EKJc6Uu-u3^pB;^>+I?w`59*i zqvHN zcuh>ZZ3@^qIW<;&;_MU@nx^H+#U0RvVMnW|7$J5pvGiG&@u;+I`gXYJ*n$c$UBbnb z`)t|(Pdj%q@>v3QiB$Q-f2-FSozZ63l#c3+!HF|w(JGTIk$#%86}9ZUk)c2zyP<{vUjn}`qx_;8*13mv zyg}x{9>ZVQ>i$dmJP$i;FbSlt$)GYO!+27&DRBLhlNRX=ZFtzMx+ueN*bQI2{_L$% zGX`&E=`9N%E5(RnhD$nRapZTOgqNRN%tgjuV1A9H#ywVmMr#im9Fyk5b`nBw2>C?o z26`3@aZi_0g>wyxm5d#;J#^Z)ZRT0c$KBX{*J$~(*N-XF|AAlm=UAQI;gkb9-P_By z+z8@OH_@Hz1kzdMsmwqLkne6fErJZ~QqPlm(l9|)qt!E!h0tNyNwAX`ex`55E;sjK zrN_msuM5RnWkwtLhwV)#?di!&S;d-Zh8r)4?2LMVt>y`eI(Mfw|y>?C( zX0dmUrW^RVc1jmeAIqm!4H;7;!?N#A7x^d1|83hs#|HiY6yoJ`!rYZO&cds5Ipe@g ztdpwUgS#56Ld$FvWu$%8QMTWKDTDa z59nCge6A1F#h9VkVda^;%?{{LoH%wz!lminVl%^sB#b~=l^dJZ10tS0x(cN@q!_@+ zNgo$8xrXQclI6vlacuH!XxjX=DlmT-#oi0|(pNQseZUYW%OkIeP+m_HEh6Wu0W&mg_joXH^_kVjB4PwtB1K{Fjgq3&=#da zlizhokj7D^dDhl9VHf zcG;o_hClFXwr5pCm=iO7=fAWn4#=AgUqB>@Hg%}6l7JxGPYTb~V8UJpQn8sr1CknU z*`;ogxT5dN$B*sV9j2^k=9M^`JbfD+I=dv@DNn-3TaGV_s^v(Wn8s;gm*HPYmw~VNw+GKln9rP`9UDEUbH4v&| z894x<#TXg_@YFqk%vhv{g;(MA1e8g+&91t|L=jE{Q#eFSR#1w(jw^F>aB)>c*M9J- ze-Vo`-ExN%IpEt>ojvG41ox{vUsna6bVEOR{R*0O>YDxT69TRZ!=vnSGz@#sIv0XI z35INo45Nkz8&dxaC&qsQ8xVvh#pJF;w!jW{1HELZG*+3T#Cnm8-W(Rx6kUrv<@ zm)ZX6i#9iUp>als8abOA-wO~Fx}dKCSKG`eDrEmp*<)@q3BMZv2yL&&VHtd0HKXo0 zW|SYke8dsh?-h!1pa9R&sRTfptfQP57dANHlVqRhrqgs=IT*r1N^pI4+Ey{j9j&2~ zD+M}LGgLcMp^q@uM(7?{GhA@oFLEZ!38rHpCQTAyPB%qN3^g?b0Whwg^kUGh%BI54 zlB1=wc!z}4l*r9#(T%Q@vP<&b+J<_Js125s71=2c5+AVUeYQnsmjLtom(SA_Kk+o~ za$D1EB>mWJDaI1&q3o!gT&yV|adU8Ki*c zz%jll!qF-eIV!=HC*M%B4DX^qHir9Iq3hP>f>2{pT8y(p;BiZnw^2(HgT%oHC92@P zWoH5$;TP+NN#-R9%nyGU{>IjeJkZsws-(ou&*D})=(DrwAe5KF>z`FSaj3bl5f6+k zl@3Q_+G_T${s3uTn~W&OhBr=M*ZK*kx9g_^uG-kZ)e|_d zwvbikDZSZ)uEOeP;^!K+A>Q|@y3;GVvaQQp&qw7&LIo2rSC4l$Jqm9+jvqol2b^bF z1g-=wW&9*}Hh1tDtP_s=4UkTu{F5A-WYI~CxTu!(>?PG7)O{R+O?Ifp+->Y{Vf8_J zSy%NUH>^odQcK9!Kb#)ug^#dagT)Qg@bgxyz7@&zDN1~$2&Nh%6RcH)D{fHL=(fl5 z*Wm})_fPU+7)kC~L}dYS#Ps7kh=Dbi(^+MP`LxdHsnk!Cqh%sYzisJPuq9E0;x`9q zas}72CPf&RKTMUc-PXmAzAj1_U5`daawBjbqz-5sx77rAO3rkPrznEtQ1S!O@Zhmi zS?pOsO`{z+$f4L|lOQVRfLxq!Z8`GPM~Pj_GS$4_`(F4_j^z;AZHXuxaeAmvX(FR2 z5c}G^rNDv>~f4&GLyC6ng)_2>DwLb9-aDB zTX&02A%GZp`jln3o8L&n-j6Mlf}YN7kdTG&QHw~o^Xg_)5E8XpvKY|xvXLQjgl%@_ z@&yvR$SpOM@|ME|(qy$9bpwDXCf-B%!?{zePFl$yhS#5=#k}SDCrq^_$fjqTy&-F{ zpnZNF??>A<&smqYgH=QAy#Ge+c#sxy>GyI%wm18az`K&i0A0P7aawL;cNkGgnj}f7 z6lYQgj+>`pVeG1tw#L%u@(w*9$>wcZSTw)8K-@$1$7|*U$y9rP(>}BKC4qRp=@i~= z5u=kD&Iis3N^Tr@4Oe^MqaNosXhZrqYfBh{XmDvaGGKeib|4>%4Aq)$0d|a)`tl;{ zxW@w$g><)Cn?xYy8oo{*je*)Lzi5ehvYU^9=~Bn?l8{LKmp&rEyGzZkFfVzE{d93Y zynOiQXQw>1=*->GMkQKwUU6>7rak>H^qEG1MGP|3UaZv!_u!ErNLTp*N|U!xiJwwO z(SEW+;q90R_xF@k_ts+DdT0ekKy#w5KVnGZAASbw$LDe?r{vSQ@lb_hlm3E^O|?ouV*Er81=mAeNAw zdR7y(uw{?6CCpH9L4^-X-cH%SN#{8CS{ZuNWx5ANq;Mz&J1M7#2IO3i0* z3A?&NCx;?>y|ol~*i6y|yMHSK_l=a^78(?YR^JT?LA>H`StMC6`a_6uWOvW`3fE@# z9fKUX$S|N4HMN%TEogDd#@q_;r*z2;FkR6!1`E&gvSL7S6$6o9f1M*gKQPWi^bn9O7ps%-ST% z0lt43owK&^;Vx(6nt^j}_;5KFcRzrZAq5;<(2Y=33942vDzbW3Wpj7=?L%cvb*Y$Y ziE&TCxvEi3n|cTjfLJ9-(K$cGueGoZB?U8SLs&vBsm+ICLEzO=DjX3Ti|cx{I8?>9 z_hap`=ZP)qWZ4-7g$UY3iGf@gE#Yt#Kh9gQpnP$9_c08WevY0`ht8D23XU;ek^4w9 zmn5rPW@RN%u*{WjX5uEh4e~y$<1p%iwo0iHK?47xvpIJUZko)b39r8gT|(jzdqA?- zZ|oxBh_c|c0Rm}j$eNBBO5U zvTR9dlPkLICpoaz2XqH=101CmqR!$%@EdA6gcEg^fy;Y(^^G4x;Y>gh?;R>kFk8Cq zhCB!=P3N|dPXH-dD#^b4mzUp!cmMkOk*rGj#akj|UkR44?_+r|2us!MAMHpxS{)@} zTj%oB07Dud9EXvmPQ-7;b@}^SsYV3J&`CdUA1BG&6S*n(#1nJ zv};OlV6FhqE@ca+RI(oB_a5yD?fa44U<4Yjz?v5fSd*R1MD@XMu&KLDvH0g;Jjk_$ zRQ}n!x!|VFY=TJY*tx>TyI+AakT9D|!gvrefc7hvF=qcZmK`j+>f5~GQ4t4Z1Vxxz z^emM(aZ8d1lCB^k4;&%f4|=h~hO>j)3zjo%N_T)r)Ti;u$)K`mYAOL72&^&V1>gN7 zeEauc3vbXvBryQqp#W@kmct*U40Ocj-$|mn6pZZ)W-Ch(z0?Gw5AmNM@}*u6ULD8* z_>d!xGwz`ZQCwLKQW8*790itCuI9=1@)=0|zraxh1dj==G)~j6D}{7gL~3ig$R@QV zDk%(5$tLTw?&ZpFITbNr&+E_9j*{S7cj`W54=~9~+uG{t%-`k5DDu4OVUE#O9LSUK z=_hOfw~SJjNOkDz6G4?SVJ;`LIM73)F&J%5fdL_kYA-mn=1WxH6*DbQ@8$C^fO-xx zws-QUpd(;-g@cryY*)BhTL6W~Se4~FRQtYnfRoAjUKJ;SH8AN zVznIjVt(b?V8`eRGsbPm!^(34WMLA=c^Y6rKG>Jjwc1qz0u4hw2@8)I-mQ<`=T9q_$ zDTFPO~7_2vfP7($D8VTR7_=pOz6RK$;?l zPrri}$$2Aym$G?irUY_Rqk^L7qg1EdVc;S4Fx%1z^Sm2d-AJ1E45k3;%@f*{ zay}AzEKoPI!t9MS3QNUk2)PPqKnC*-+-O$R`*1)PEtdqqC!TxD(DXSydq1W7EPb>> zVZ^dzNLw%3MTKpN)cRR;7yWk5)ow=#Iu)*H=8q;sL@ zzJ#;0(+1EZA$Jz7BhHX8=3Z|DyvOh;66)reE zw_^2;P{m$PgWc^C+l>Cd?KB4(7li!X%YaKwCS@nDT1YJ>>yg>a0=Gcc@Yg-i#KFc` z8)u}P1u-bgt%uZ%`fl4+m=8RTaOpJDp-5;|lZ&FBOwOk&3fujgB3QU>Q3?dWr8r}k z!*yI8hJ*nvfFkvp8z-fnKMTO8=m)8yuiWVH6to&lh`Yz|hRvY$Z=yWq9z<#Y*G`FYsVQg>s-nG-M0qVJQuq#QR*t}YN`7D%_&Dz)9P zND5C1N06vdYB^>PJO?saC=eK0PJa`A_*d!kqN7nd7MQgt@`{CtI+#m9s~X^aN#e${ znG8I+q{E>8r5Sf9HTPEfR2)EGNNc(uSOIY{dR)m>MrN)6&cxCx@nIy#zk&=ZscDEb zUc*anD`!@3(WxJ|VO8zRjWhG`l(HB=84>sv@4`u%tNsGxfTHR>UKr31s?A*j3%#qp zfDHuI)R#K|%x(yRWh-Vm?c!>4MT})>R5Y)!OD(fHB-)`VZq#7Sl>|6I-X<%xwp|Ok zL~F*JlaQkwpOQ_GW&uh!@tOf67?p-31quolCt1ZQ@YEYRY2IppZhqFLNtRYUv#=!Q zYEdY>`|-Cwwu$NUGZfG*x~-aEvl*>70ZI2%`%J)KjXd}uYSKO*QV*yEq|fpMtKHTK zSE~wBt`wlTv34>XI|!-L)Zf{ zJo#58+13Kq0>m@aKF07&&=|}uwb}|dI8CbN3!WLExlGv$TGRt=mcD9qW{}mWwYiTX zMaj`T%*+ID1fhXQi1tAXQMWGnrwC zPpcCTd2>&(X%fwi$_1bSsmbZPe^D#c?OClt0Pjc%$N*YCmLA3=7&F?0J!$rx>Wc@_ zlJ27*^*`{uOss_doXg2l<3_pj@FVq z)bi7+1pu;SwZG($)`Pw6+)cQELS6C<3Hodn1kpIv|M7q^d)ChE(VAYhJyzLB`Jk5E zxr7V|s~>b_#!Xp;vT+I(8ayrfiBrT>xcI3j!!0$qQ~{|76UMqx5-bz&#!=(44suZ6 z;XHcmFD30E78gP^e?7wQXBzV5~mDk{$=$LdOO z+GHxn>7<>WSjp2QE=E0S92e7j6Sc%nj0V0^5coBgBO7p#Gl0@}*7&Z0qAhUUz2s?b zg`{gyHA8V*)P&LcTM*G^WfsA+a&P8%Oniw4{7KrUEE;jUE@Q!?M+#~)#k_$zwGdwc z9p1jTMV{_aPoH1ao@r;oZ^rI+#35EQ%n_VJxS+U`ttmB>4krC^j#PRC-i*N^LT}ev zL=DJ*#4^OSQ=BErznV!9rc~{L04(dlDw*jgG4KnkcIB^Ru&o zuKFQcRWpNQ^TPhK6!8vJa|`69Z<*z3Lq!5OOV}!@qQ)4mQPpUgR)V|gR$%0~sL=!Y zH6`YHIH)t|3)IWZ=uK%~eIW3b<8VE|fA$fw<2q?^6+0M;X&GZLGOQ}K^&vlMJky!0 z2pNk{+f&*3>ClUJl~mrdV1!ahi3terI(AMt>W0~f%4 zcjggmj|t0s#dbvK_YG582Hio@&iZzZBAl$v3gEZF7n1I(oglJtLM}NV2q8hp8o$Z7 z-0r3xk@Q)@8m0`xL@RVfA43lCpa$5(_`28@MwfLph{n{GV8Vp)ESK-Cl^Z7Er(K)X zU}M)jhNlGSt)-4I|GAcvKoHkJXu;?!`3$sv7c*W6@Oy2BNYT)Rp`})?Y(msaRp#nN z+KWn2G*hG{_`xOz*uu*NC7{=nFwX{*3R=yhjcovQuzb3O%n=i6v}p!CQgST(b$I#2 z4rN&$11aGvDAN(*p{g*BgR-5O8aA!o^_lH1xxCTs)1>aSZ0w;^45?V8AhJddn9&)@ zDLc$y91+yx1Ma$l9K0i=&!BR6jZ$}Y@{oftfx2cozNtbjgC`7la$0PuX^V}$p|7z} z*t3d`78+Ce<`YAB+@=2nND0F(Cl(L9;;Q5dDHW8;l40 z1@_Xt_^a@PAJ~nTB!s{bALuUGmih`7Y3I3eeca~?U`=aaBa#5Fz`())rt{v$P0>Qw zg6M`L>7eexRmodRhZ`|JPNJb7Fw!E_c<8y^)3a<0|QU)2P%#ehdvdc*P# zz*1rmHh21pYAZ&psL7vCR89e%%k=k0Nps4PA{n!AjYPiQ3lu!mM0Ss_SR^2r2MY=WQ|F{@S;SlP{x zv{f=2w+y(Tr|PC85EpDHj#mW9vJIVzqa&plg4ISj6wWhfzJ(2?& ztzOpD5+0XPpLK&Wd{%Qv7f6-$Is2iG3*W5Wkwrr3RRgTM1}V(yNBYx zfha%M*_-MlYLixukekb_;yBurX~(P%wHuZ#Jz^y%hF0)O5x={uu=Gp}q4L#_+9Qr! z3=5WNKb^k)`*7I!?<7=wgbW*RvGqx@Tyc&Ha}0;a)ok#1%W@YQ}!z!6ej`}*66qLaiPsw1QG71zKtNPTZAw#unC z3J+Y?|2r*`X8KiVLt1L7&B`4BdVQ=WA4vPky}4#abq0`kncVJgR!+>|f@xA})$YYr zAjhz8V5g>LUFkkP0J}LAa&|TK$Zp+Zu%29}_!G1XE5Fs}86BmNorY817wbR?#oH54 zH1$L`yeQAmvpXLZ&`q#M)7fhIU&DV+pC^VIv? zCVIETJz)YlXZtC$43*DxhfNS5Yxet7i=MOEJK{@q)M(tyWuQUMdF42uH8Okp*x-ZP zFm=P2%+HNjs-m~otUz!{5|R}Gr>q;)*ABpzq3IC<$MMRFJ)paTpX9apx6cp)QMJ;ARPf=nAMyB~>r5 z`g_%LM1E!1_K&4XfMir@#vALrC5oIyZz&EuOu>#c0Ge?^;S^2h&Q@@IZU}WN!m{ zc=ESX$IKEan3hm*j1)>uUL1d;G7_pH5z?-)gBszt9RLktzr_z4eKu&iNyuiC2=AK> zorgQxB9zG7(0sR{M3ynERP4%o(w#O7>;P_5DZ}2DJH8mLcu)6x7+UCQ74R@ zbDRA?HJ-wCF+5H+@Y(|az`o2ed$q@;oTJ>)D95?=BW?jozCNqWq@tlr(PY{ri2$1x zi;h<>+;X{-=EzX#Bf80MY=O z(F!dnRr`k=i2c;|UX~hz=Aa8CjB;+ELgt{-CFi>OMohaUKfnFXCfF7=W?wuJUC5{F z;x?2QxiH?%zzw&cjiOk#(O&l39}~OvrJZz=3InoXj1jc%cT^fKBmY1s;krd9%xl25VHg#xkRa8Hw9-9fz0bR}TB^nUtx@?c7`Guk6I-r7=U zp<3ndccX(j%6$k2M^f5G^npF?R)uXBIk?~@0W0maP_Jy>I~MVWwiJL8h1K-L@dH}l z$(AhrZUYxlJN)FzY~QxB$K_D*rjLXK?@6=r`W#kwNl0#)?(%Zp2tut1MM+a@_dhuqWi+x=?FR3hURev@dTjS?}OAcbwZ{6%NgIf)Y#Y+|2#6 z)7pfJ0zpzN$vJaC5zI;4n1S)2FM885HnQykz=*BTP!$$vo)S6%q_P^_T+~g($G~cV zNQ_&QCAQ@~|Eut7vJxxKy2A_~1_#!APm(w!l^m2F(dlMTsxW1$9pKlz2`yhjHo~|{ zM3~?lu-0!Y_hYo=uWPee5k`is6hR#%Y)G;0UjO6eSL*z5sgK?%DMKg|e$q}kP@ABK5b27n6587MlN=Q> z{v(pwo@-*5A8}VL5IJil#Bp;%D_*rw*b2WS@KXsu@Dh6M0)u>DcTZlx*~6_;1T7S7 znC361X~Z_6ju6~gX$yeCyWvSM3SmL!-fJDh-qeGv8VF3RM-&6#o-|pIkmsI5Q*drt zX!Aaw56W8*xhPMQNwq>)-B}<29V2hEO1dvJ3YUDHc&=RudO{yW#hX#i?x(hoOhRp;AH&{Zo<7{hKmgUds?<&=r)XIaz=2a4+ z5n>DJ@@u%9%XQ6TnLpjUMDrG z&LVGE9^_B8Xj_1l`ixG@0%V~*bMoL2blyjMMRL~*8VO77jw|47q%bYNrj!@pnVWm1Ife0njPITu(pBX0;Q4c%;$m@W2vop8sw;K zFUcYQ`7k|6@E54<*nLD>*?C82=$~hq_pYW9J;wZ&IT)sFhUB&FDb6u*tC3^cn$@9K zX177As8Wx>mBd->A5BFrXuHM;03rfsc4FfNE)P~2JM)QFT9B)^hc(#gQ-+@PPhFnkkUS5SE&`4RkH;cDg$c(<-ubp;geT;Dl@1!qJR#w~n@^+~;< z;jxw|i5Mh)iiGx$xW)fTAgZ{mH(50DGaw2!A816fvDB-2yru5Y{6@qYBni z&>h|>bc;l#4aG;Trs&8~=?vhk0#eOM&75jq-yw(ap@N*!!nbpfk_zTSFabJ8XfHr* zOC?VY1Jgb`;YDYK!{Q@6r5s&1T>F8TG8AQ~&IYM%-7d>&A`66Dq?riUI*M48ZD#k}0g8HkyM7^BXjY!QW#RA=^)-UR)Y_bVUKk*R5ALCxJ^ty@L+sgHU8Q zyF3aXl~K6OUe5b^%K-1&qjbF4`HtztD6oJI<#Q_w6+%<)0t)uv#73sRg=Hl(*uqQw zwy;E|y&e1g3W^Fo_LW8bOG%bt*d76$XD_~8w(xA-y)lzM(Bg$MlHJO?S~6O|zmB;h zK~zv`3`=#>z#x;1el>A}ZdTXo`8*$_8GeUJF)u7v;2Q>>=_~ zjK0jwebl0*`D`cbWIF093+lsEl3KtBN zCv@}0*v+P~7~G{*`NQy63Dt6qwi6}UDcM9myF<0A|9Mspc^V1Rg<1;R(^fc$thp8H zg?V{#WApnfJ<1fGRe{!oR&+|TcWB1-z&>gtGD3<&^^PWnUd_k{6SYB_k1=QZB;a>` zFWsYC8z>*Mf{rrA^PxZ6R-+MM6lLxFS`6lsYO#=q$IB^wgyaCQKY4@w?g-=8V9&;(1|(lik-^06ez0}^BK9kjhOZOR?+Owf@zh^|8 zyAJFuNUW$W9mAqF@}AJ$A&{4HA17I(-_0UwuFD;=0;*$WHJvVCR}KZXAAVkP^?j=X z<6ED?6MWxXRJeAbzDlCz`lbt89z;WnD9M04oK@@kX6sn?uez+K=w6EX#z!rd<-9*N z_b|=zM%5rielBoT8MqUjP$%)-RH2ScwCBC_t z)%!1hOyMsazX1v8Ng3AOzpPwBra(YLcNF603dBmE7U_GftX5{Nz@`CWat5ksKIBt4 zf^4<|g@^&J&h~5<3ROZDU|Ev>3c*ZLy(Zf#}9r(!(Au1)(}u`uCWWPV+1No{T8@ z!gaBKWucC)GO#v~S|J_eH^913F3ucPt1Ys_G5?bd2Ib*>$Trow5t5E(IBl{pEOkwz zpd=v2$JMTT1?f-aVA)#LG~UrdTe5Ga#!I-9ls$N?m(&KpA)dkrlG~vl^O>tYgq~`S z<;^pKQi`D`UX+J}1;o}{rko@7JfeJxajQIS9@-MAI2MAh16lHfswvq>tuooH1_1^s zwr-xi&s$VV-*YTc3$A(JXVxx=U)|~Z-w)saKBRWX4jC332pby6<2!!n)B%nkByoXd~=S|WMcwQrJm^(2$>ScG3kMMaOo~3 zd@t|&0KBuPb4Y~7&iPD6x1Ns9p?v@KPvPaa79J(u-_aMj1S&X4R4Wd+*vLsJxz#Dh9&+A$h;j@{30BglwW_HgA6<&Y+KzoFMHK{|9if5iK^1J41Tu;#W zfo9wc>_l6;8dlLcjtH$c%5p*u%*%oHLa~iCD1h9W<6vYhe!y4b*0`%OGEKQpri_$) z9(P)Y6h}q|#9|jDw1wX$jiqz0w88wd9AgB$ra$yvhU^%8j^3P{RQbG*m<`SS~8mk9LQqLrJbgl_78=`RIaK9mGXl`8Rkn6Bes<>r2K=xJ$d7%IWV>G z1f=f z8Fn#xw+#;m&dXS`!}ORHwe4_xrJC{d#Yo)v{on!^#6!j7({{UK>i}>V{79&S zkz-!Rvw=rP{V6n;WUCraoRPPSZKAQJniaTs)BRgLXg< z?S-oyS8A37gfpc0ZYi6#Uj&S8=ht94_2Vt8Dplbk@$e3qOLrRYX6T!?RFWCr>>#UG zZ*G?{c}l;x9neXw@@~zfbOcy{X`w2J%?EbH`3Qq#?D^(3CGhEK5*qZLeZb4+&#pzI z6{>?$Tgh^_cq+H&f-IZZt)xXxC-!fEhwP7dm7!K zp_j#-nGhJ1tuWiTWAwJtE4hg0vllrXve^YpQVGc(BOZu!=ToRg4VN=&7UTTU;`u0vPDMJE%Tgc69j@00tfM z?b$|kQYRVg@vOH`GFNSh1u9q|NKs7{FJ>#@JMu*sd)abV4W0Dvn`fdL>7)o8FIiCG zy9l`de-Jt4PB9V|yr6d<1FZNBp!MrayxWnEs&X{YCy-!etsXf=JO8SLw;EKFW^1<8 z_cva!^3dHORI4Y;lUj()W!oiy4%HsUM)BngkoT^`^mL*B^70F4+9W(v$HbsW)Lm5S*jK*z~CAMwh2o z_mir1^w#f@cx|o+9xvPymsSa}-nIE{y4;xlb_6&JkqPA#eOW181S>Xpe5)*(>~e|0 zm8s-BBm=o!Vn-)*K+kr0$%^ChZM~>KPnnM*D6asie%atk<271RiB9%99CqqlRZg6< zDz4{aO+xH1!|TVVgKkx3F9AjVBTsu<+POTv2TT+^={CrM%FTth>X$zm7`D0O;( zo-=q)ocMyeS<_szE}>7pw6HpJT=$wLk&C-i1EyDH2l@uBJT+4sx5AsWkVHpzZ^@};->Wxs+JSPy(|Ki*^fV8sQ$ivCXX0PTX}W^;e~uJE1vnQ+nB|63p0P_p z8A}h7)EOG|=tbS9M&^bQk}9T$T{xV9W-3boGq`L%W6|a61s4poeMx-RJ3a{n%zJgn zq$f1=dglVSASVopYU|tr#9hc5b&HLF4z^9mzm@DDO^pY%NL)fnK&($&OG)Bh>@&&t zsw1P%7Nx?b9HfY}zeZhPhQ;Fgt@%zN}n~I;-si?^YdgC4q@=PI{Vz2& z9+E(X2YLyVRSLThgInD=80dOpGnT~GS26qCK33~Fm(W>@;JE7zO-H4sxyplNm1u{! z(Xq;=aJ;N+AvI%z3UD@~#u1rr*JdlCtr+a~aZ}n6I;o@x4laM*x6MK3?%Az7FvzZ2 zh{zw!L6HhK-<`Vr@|0SC>Gv>C_aBjilVlU^uIC^G?4KN5%hkdUL>?%7!}2>;a!{=Y zK*K$E-Z8l$_zdS6r?XbKncbi2a-~lOA5ue+?4z1CGlfYCY&yb|LVx0xUusLKThAF) zWZW5fa+-U`ohAXMCM~6a#AHb?Ws8wwU&+O`*vh?ti0 znFb|JdXP0G6}Z7JE)HsKM(uQRe@tK=XEyXD4e`sCb3(vLpvD<} zh3t8A;Xr224{VRd?nX*qvUtUCA=xY0Ii>7qn+9G8 z_mYr*FO)i4LOucf%@o7_6kd`iiMlfEL@OdfbV*3|+;OEPSGljS0egQ#N!ih%rN*f& zXSwghShrF^xF+7EUfvAo8>O|UA55PqKkL*Ef!)|8AI?-E&)_3+-EjMR-X)2Kjq=qnn} zYjki%Bgd8m0XZ})n$wr#f+JB5m=fnYS;3M)ZnCZ=WoHWn>fKQfVPxnwFz-Odv&>oT z%OxYN*C;XF*(o~ua%a~~!cB4(gbzu|h!|ia)KHF1Ni{{3BS2*`Zlr6=S!VJtfR5p* z+)fQ7kuKvpD$3diUOLL?%zadfZF51K>N@xp>5s5(#0unj(w4~?I+DaK<<~@#Mx`TB z{c=zm5MXe?NEx6ad$5#QHJ~gFS7UzJ?p#!)JjYZCf=X{$1kj-jktQmJx4)5ed1fOq ziE;Q4V|_^Uu}z9RN1~adaB#DXi`UNU_MKIRJo|187xWd@OeexT(}o+ntdECwWOM># zZCYq(v@@lm-rL-0p#q#F$r2nIo1ceo|4}^><=`y9Z-uIE;5Eg zHr1Q;KzfZ`qR&L21TM14N%ES3mB{++O|>)~86CpH$rOfUu~rzDwa*vGqG8*)Jan&m z2`zK5TSPZUi+lQp$Owt%-c9CSQRIRv= z4(kG8k{D2|4s=30b$gYh>ZH@bq#V_-uNMaUX+S{LY|-@0-svperblm+2vY=7#`=KSo|vV+3Zppd3Q6m8<~yPY+^hmFsV(DXO=FC5ckVlB*rG5o+g% z0L{gH3H`?c({au((1EWQRcCbU(glU<1~lYCKI~NvEiYkZ>7ITh{{;De3jT0$HFv7k zg?*LS-9gTDhw@fuJ?nCmwSQr&3LLH999mD1?orWuwg;AWEL>G-sljQbHl$7OblKG7V4%Jrh1Zfv*+VApCJY6s;`=RP;e>E4t^ z)~TWOdfJIbQluorRKmvdTU(1mb4o_-)*SE;+10oZ#J35p%-ik?2njvmYY7v7M<6C` zchF!>+M*j_%R1>JZY~^>5quUzlI&y}kO8zpCOV zCg-sNN@tV{bixtKl zOXd-jVWS(sp)Z|pd z+cweNbl^+!-o~1K>kCO3!-|g4f#r79ZjnU}NfhB}9NRpoj~I)N745NP7gYREo$F*E z?Wox%O8*O_WS?7w^3sE{Ql#}jN{cG0oeuK9P-0h(u^mhOxVHCyCy7Qv2fcpV;~hl= z8q|M^H{5cs@PzXE0^qE1)t8TK`5dc=#lGl@x0s_4IdU#LH_)i6 zEPW$86fAU#4XyP9p*5)vtrwMnt$~G=XKXm$2(Zs@49wX(#-_)nIl+^9*$5p!qa&7~ z-7)!`P;Qvlg|3@>!O0uS_rU+Dk1!h$YE25~Cpb!FdCN@I1)5!gdPdIM+;B(wG!ebq z)#Ygi;J2qXJVwvy1*Dj^woP|XIZ|5*%hKi4?w!Dd9tMN~ZT74P7B@pv&EMXQISM%@ zV4#dXv^9Id{xp5fy1Wc-p1FW3JO`?ep)YZXcU3E*tAsoV4C(f9yA76&DDWYzBozkQ zm{Q`H+^MbRD%@y8POPp5bO?RN*E$vB%5JpAaad5bJEj`>bWhYsCqD+~Tq<nXORrXVx%moMWDBo`=z+<} z8%cl-Q)6F?Gdf8f6p0tE-7Cn!*k8ma-R}RHu5saAfPZ zg#&BtmAcOGh+4~3T*$SL8C3+)LCp3n(ju{-Mm>mr*$`oY_-*HB59mC08Z==^a>Ej> z;#PqlmkmrVl7dP4TE(3f0N%Z5(XmWx2cN)rcUuw^y8*N?#SVfh>k)V?LDu@`d$_66 z3`Sn@%1sS}1nP|@1pxH&oivU%ZFS&D58fSZ<-Xo)1?fOhls}@3&>tRL34R?+zf+4| zM%q49P71K2bEq?juN|P9U{85)*{Z9P-y?_tD!O5tE!C5uaSw?1xblraU=J zNM&x#z#@|mB*V)V?j_A`B#74D^MP~MO95^aLZMXq~6n(F4llAb#i{?2#RO8`HX%8w!017iexE6U~$xAdQAn znX@NV?>>I{2)fEh0T#kg&z)6=Ns<~MYhLolrr!DbQ?yXzz)CQ_e|`OVkbnI`iN5r$ z2e4@-sjsLrc6m};DMM*O9Is5RAWI>7h%q|XsF9)w;6ZqESz6wPUPplpqQokXIL|1c zF9-A4o*I;QWcLid|79QVP<^%Y6HFt9a8$=Qt}*eaw2PV#$TNIqG63J!fy_k^U2=;b zEZma4;L@rN2X3D#6nBcMt0!4H*L+y~_Vq(&&!3*3gB2EI@zzEj z`mlD&$)o*1_UpTkm7z0b3B68=zm+NzW9Q9WDI!3U;_-jRjIxNAP_mn&te<2Pl?RGa zG_S6b*oUb@att>=^9FXd5iritk|Jp2a-GPdD&*0_n3HU-2=i_TvahZFmJQWDl1TdU zna(%id@4^2PcotmQdQuY3= zanD^lM_kEcvY>DSBq*2ZH%;s9j9QTtE=%z7v6hKxk2(g5@r&1=zI7~LwnYc%u4aq4M2C@ioDk}AhS7tCP1T{#WIjI|KGy@WNQwB4NPFs)>&P?G56p5x18llm5+$}F5xwQKMcyt zR=q}+gL1<~QrcFL&(=5Z=*KXd)OuOF*7HmG*oTi=k%uMSJ|^_ToJeD^O-$|u{I ziK)F&x@QB_{32>UPI-JHxq(Vkp;qpJPg=l~_4RiLZhKa@t1jLNK#vXubHNfxeg_L5 z?|%0B%hzwhyI;Ki`P+XuEmff3*DSYwc2D)&1&Sl5J6KeNtWnMf$o_V*7Dxq^dD-et z?N3>GMwKU5X~7?QrqW*3}s2jM@7=!Tt~>EM?;)T zRY+ViQ&4w(q{WE?W_UxY9$orQFFd@gLRQ@<}%FcusESt|BvD2_ooq2*Y%gOFo+_Z zRCb_#s>|)rWUJ5GsAN%v{J?^7(Pa{+S|(Hu3}xd%)u0vFqC$6hk^oA!Wk$bx{rvTd zkp4(=nLhd5kf+`cAOiHMR-<0R?aey8-m9%*R@^RNVUC0kw4ZPP9?}w!vZr46l63U? z1GHjuR&ke=4&f;!$z%Z?M7xY<%w#DPw&3;I^}aE(>uRkWvMH6`1QQdCO0}D5Jeic2 zq@Q^}XHmzEbS<1ye~MMIeDc#9m9S!~Y?nS@cZf=+%-b)&y9CHClThBG0md8+fWCtJzL(McZ4LM|mj!!36rT1RDFfs?Nh zLK@`shve_4uRn!ao%|uEDH5$`6n0mYH?afU`|Nf$f5Suij|pdIC&?r77@cMLUfSD% zu3dWu%j#Rb?Ax%j>l0{XC7HLv(pU8x=U{HodsyeGk(S8#y{0yf-A|$FdFCk%idw%R z1!&==t;Ddi3v&R&J+}#(P!y%?$#E^HJ|ZE}ZsL2PRJYXtP%0-uYD>C8^-vx7(g#x) zg?QxNQ8`>? zb298~mS9aO{4**Ad**uYHI2!41DhQY&Ll#B!ZD)-)OeI%u63-g)m^7vIto}l64=5^ zN}3h6AyE+d1m)7f`RNbg%*Uq6D1&2;&u+lwmXrIvPy?c_P8V|1Fmqm%@yU>xfDN|9 z0b2~Y{Q_{Np)y(s-%1ryW)zGfgLisk`0?1M%t|#hK?h2jX3A-S2WZC7%T0Wjz@%%d5-x! zMIy{~yhy?iE~Qs^xI;xNaT#+rDL?C{QBl2sBvCANZrZE~reZ9PsZGF@UK4IQrNWAR zGrpeMU9BfQ2}AELu_OU>H3xM!t!wR<#_W5eR%g`tik2YMlp>RYC4mu$t5YMn(;bFM zc~34|S;}_sEJ>GayI@Vfw;gL;c{sP4e7E5_-1USc2uV9ONlBfBb5!NUxx7eVrGS(U zWEyI$1y?guwY5f;DGpIm$n(SNd6L(#MUidWUVs=U`|J{*oQYn0=pS1*V>`1wHW2XkA%CG{~!@ z{rBO^XTiSs%9c98>kEppf?G~knnd6~X6&7!Uu;E4evq9Cc<7=J&Y~byj%S;A=F7C-lWrfJHalSYMMzTwuQS~4kCtK zcQ~M;gU_|{uCo^^5^2}QPQK=nhz>Ye6pTPP$I!@*F6{Cp4zU7-Qx{BS*3JH_iWWP@ zdQkZ_2t*8fWdB56$nN~;@}DFJ%?8l$&6S&666~`^sh=RANsq&@#Ts zzfx%m?gBhfJJR$?hNz;uQrwZmyBz(B{zNs~QNHX0+>d23n?j0R44Rzq7;bN}4Q%>> zvi62`<)IAD5OsL=VL#@`kf5~6G^3Mjbs6QngZzSxhdEHhHRHGWiFZo56yFLzXqWkCvzl z3%aRE@W+vTN*hG4Cv{~QL5j0`Wz#4Czur2a+r@q{fG522jmz~_Ag!bT?#A| z?B$J`96*)$99`8q>m+piL?-(KElwB}OWZwU-rv_;gdh}h3NnVvX&hCxyH0oJkp^aow<0^Oh?w<5|H_}p97zG_lC2E8P6x?z7{)G@)Gxoo(jTtwvbew{F?iFVQhOrnvfO6OXn2ZA zy*MgZ7EePTA@vsI$<`|Rl1@~BSd$p2YXpocX>A|;sg+O*y(BroN;!GmHl*8!I>TF+ zEqUbT;%A}UGI+ugyMf!GN{`whpS0JPgs=E13WX56Jhe{Y9do-R+H-Rs$p;D$JFq2y7=NN4(HyqW%7#oj1)NNW zjE~jQHmWa%e)m08!QZ?jRP-DHjXf2iCtUoQJqh@4hh`+a0pN#t0~4`I2v`aXoI>H~r z*+83xmm*Q^c#Iuab8d@p89k>SeL^_Mxf;1^pO7G1kXL2dxuHll7n8)YMLkk3I6po( zH4vw7(E?Srk-e?dWFVoqPgS~zOR7`iFb;cvPzi@V&|owB#rCHk^$;RklMXRuZl^k9 z=jGX_I|nK{yUwm>aKK_mMS9yE>XHw4T2sEunZNBIhDLDL>?Bfx3p%hjm4+@=!j-dfQ&wv0>QJfuHoT))SO`&zpw6GxJ(wzXgYL+gL{2z|Hld;)&IpDmx}d?FHuhRlC6(~7 zBL;!OmV)|@T8Fg?RYi!2s{0cidB7B47iZHi`SsMRfWFC%rynfTs*~jU>~>Id8rI74 z7&VE zseFVA3DELKiaoo#c;LAWrX#9SL6mI`jQd69zAwB=9=!wm*;x36-5(UTpMigUHQa$N zdXuPXP~YFi$70M50H{&y9AKsff+DqE(yhJbT(BD@8Wc%!@({V34c)-mAvyJuY-iHs z=jG=pc57wtvabVx{l=E)fLZSZ0*4NkLCyz*j$<)EBGQja(ROgG%ie3?Bwyq{^??jK zm0QH;y5k2~+HF&UH>j7bGO+MZe_ zH3Fa$NAlf1#ii6f&g0TDKP}NdxFuw@Z8pb=0;w5xOv`Q94TAqIKb$YZ>nG{TRojE? znW0h6!@M;G`++JvJloL^3z)7;rVh*1h=5QaUVO! zb@#~|slW;FkTB;u)G%bP8FWEg9;ZA)jDEmhHu<;{85cll=j?n@5}kIVjh|Ay?b%)t zP$E!Fl$Q5mR7Em~t=YqQd_qvNrqeBYypWgNBQfpdFrdf|8aWykF9F%%g;vjpg&BW@ouq9KBT?mE?RV zBYGO@C>G0_YVqzA;|anVtc-if8xAm6`GV)s=d}{0+6vdjn8@dGl2f(#8J>JT_?4Yb7b+Ue5FTzy>y<|ou>5186v!PKj zO)zAzX{Jl@ncKEFohiVMKEZ(+a~*k*Ua&T?K-tw-Pmc-`Z?fDqOHB{1w-!bZBp#4< zryCAJ^6yp#C4|GmHwO4NSykf5G2}9Ovs)r%%X|?W-6@~NIm09X z@Zq;J%oQYRKzO7d2{bDw740foSqYM6WS#2{JSHH!BtXFQwmzxTByZL{DwB?3B8*RI z13l!p5>a#vv!sT27Z2$UF}0|z@C@|4e^A&xAeI%b5?P1nw3!}-Wyq>3TKJ{U&}LxRHG+>k$GZ*Tpg@|*ba5EQ zHJ{9|vRn^}PhSRO4@}_PMz+^sNhMp1sT5R?h&zI!oqxC$I+;3Tl6~*Ag?^jiCsk2y zCqfMR6alBTUSid^L$bo*d?(PlTKU-=xFf>5Y2@;7jgSR{6FY>U=6QIU6Lz8lb44Vx!2-VN?pS_F3`luyYMu<5egwxy%Hi-HD%lYptvF`;qxe{TPF_|7zKoND zhGpfazLu*(Z$qbWSGE*TJZX(kN)}No>U-eEYdZ<$U@KUBXu}1O4aimxm~udu2L+RN zKYjh#>sRvc4^VrkG@{Q+fKP^WZrocoTQ_$J${CdeXAA#5QR|oevf5KAWHpIDxF!f=(o=t0-q|x zQWDGn)N;Yo8BI>JzR05LpnkzIM(gg3IO%o@o)n2)`+)wgr&*BsM#ItdUi*zS$WoP| z0M}K04DFp#{z8GQ!F8)j6x)IBf<4GKmK1D(TJ(hM7U~xP)yKfGxCG9^o{M~H9~+kv zT|Ay+G<RL0h^ zWLDGzZ3yUb9;CP7Jdxdv#?lQm8tBH-*v#ry|1X{I`1y{e-U0kmK9^lzLGPqE03WW@}xa_w;4t_Q+*QCgDjH;9+W(q zJR<77&G_yUiFc-!{A?dxBa1=uw`nMsA~zxdwt&DL^?bla?%cvQW9=d3X%-LK04$Fv zQ7++%Ar2=eZt~Gkx{JzzGF3nijW0O6m@bOP2Vp`;JSO|36J!iHH*J+R#eNKwX}E)d z<>4r%KOv3)5SuVow1o@RxKUoka{?Tft3;LJCAf%jC>x%}y%kRBMsr>XNagC-;OYRG zT;AiJ0`>#<(_N+4YLD)vp}d@n{Jbo;tIFjSuAEDEngZ$l1v+51_s|Ek?JY&7%YLT%t3!uC~4f&E--sKs(jP3~PmqUB*{RIAIxHr7k) zY87Z$Nj%uqlw_&)*(cQ%;Kw&q4@kz0S)mfM7A;UL+qF$ngM|m4In@(7$hBD=gKnp+ z{`%0q?QCGml9FZ#I`bKZs$GBL8pZ@-wC^m?A@$n>2{M}=#K7-Wddla`WV^ltqg5~a z$mnWPCz4#H!TkZoAtklhr9|pfBL_64OT8ljc#*RKmHUwp)Ob<3KYf(6kL=l^TV%sL z3a3Qbu%BHa$y4O$W?)Y40nApRVR!kv@W0y%qfqVu#ib5(ui^k=y8sKIXPCm zBP13KIf-c1R}2i4vgn@uHuP!@3OGKk#6gmk1n_=}WFK9I3}Q_}lz*{@GODDv>jey6 z95>qWO{BnrcN7Llsvrho|B*P05@G_jZ5QW^q#m6G_=aFK(8k*=8DnAU8P7AVC5(=t zDA-4*Yd~-X%U8lo$1ngTENX$KM(~Y}$iSjA7pS45Xfe4`9o=$&VS1rsZUuCj?pNUq zh$p~0RH}X`d>4LS{tNQ|3`&5wt#Xve&ILEY@}Ky;nb3fRY}hQ|rQ1D;m6Q3nd6IUI z7$+kyX97dAIoOE?7S7qNEAo1g%Pq$!H*HhDJWLv%LG&#HhC3Bcv{WOd#1c;=h zvx$3Epe`#aVu68BvuJ5PZ9Nc!%q8RU5~Y5+b4@ppBpN;QfA{mZ&t88QBvtyu+lO+5 zB!x_moZ98Ea`2j+zX64jg>ADmmFBu6D{e~e!0;=2mwryvgokJGQmh#b!YEOTf-br> z78D`|*7)#sOd47@ma13`wV{~Dc@;7_q-q0tw~{K!Dw8W1i~!AwOI9e>>L7nRGi?P; zYImi`t3=(#A`CqyI5&>N+cVVcF$y;aVnwb;#aMxvFXdzPC!p9J+aiZ6=HZ`KbF$d8`1HiC2@*S;bkXdCrGh#uL4MtQfm3|&Hx)6WL-;PUdgSc64;%1;h&2RpvBxU&Z^z}xZW!D2!6n0g2R~gd*Fi#U}^4-M9iZ442L` z*-68!XWon5y&dazR5c{vIhka5J*pr+hq3H{vDj}y|H;%j!DiZAh7FaK0X;y9a7$^p zYlgZe5e-d7+vJ7c4oEe&D^V4KxSz>>@?OJ#wk#IA3z=!T7>w?#() zl);lQY(;G`XAPe@+)ePSLs^Yr-;l;0Ggw(x+c-T;^<5dx-b|JAH!lO`!w*GQMzvvq zD6hrZMQ@C)%!Ml)&+N7*)HZ8Ulr53nszY9BPUgrKRHS-xO}U#+)S|_2QZp*&Ws`AYi}Fc(?-A~s z`k5{)Mw3dQp8;`R-EOysH50c%H(urPHwfw*4s%twuF3{{65c*Rse{b1qy$#7%yT&T z9I_=XB;zc{N!+oM=e4Gy>j_DYaqpn0Sn%r{^^b(rf)Hn)-n6)bVe%}opbD!wUX04% z3(0wp)+x_;&a66$7r`5HSvAR?QE`X5t)2bUUtUbv{tq34#T3Nw+cUSZCCOLac6b$0S=RR|5oqBX%H- zl!R&9m6^t&+sN6EEZtH*BRE(q8#9Fk5dMs?z9y-K^eJJ01nHwOVnB`l2pEU7FGg9^ zSdXZ*Yz8UCBC!zP(p3Ty($2BOR*={iCEUv0!Rc5MK&Vfi7NrS}Wmm{Fq(4wMdwz}t zu{l{Gwh_L$1ZM(9o&F#n=#hirFJYcze*l+CGThED7GBqSvm|XLYeC5qr;Ut!0M(o< z%3(rVz$h00W8CSnEa{M(D_yiB28JH1HThwivzcm#0{m6N+0yOk%s&MLc9K9uxLPu!*U zPA2Z0^id?|f%t8c%B-hpZ2?v=gjNJhAv=e`AVDq_at$XLokD7##p1&@_3`AFyhIS= zt|vQnJ;*nS<-o1o4p{QY1DKjU`u+DDWS$5IgiDv4Km=DxmAsP_+8h*D1d}k_Vw@d^ zfK^RkZ-aAC;_q+6+pne8A>VFG@?Is3OL#9%LlEQm0c+!MgQW#7g*3$ zPUzs(9iV_!q{Ga&!ry$)9tyZ+8Z5KePJ5s0|;_o|dv~fQL}F42Vtz z9=$>YxmL3#xZ?o_e2KH{PhnwlS-MW`9><)!Kh(X7ec@tX+ z7=KFoT1V>|ReGYViltv~Y!?G&ZcqF_g}<`H4?L?ls6d+U@-+z;OmxabMC9}by49y8 z|K;0<{K{_Hhv}6ggyhsO80{VgPRRhehVEd1FoF?*p7TEHERlBvPZK8z&=#^xIf%j` z3XN>qDtycS3-}*~h34#bs#qS%;42dGMw501g{fT6e#(&PupZ?#RMkadd(8;ZIn(Bz zpe{G%d-JJU>jeV>YHP-80{)aI3(l$-v_Z^d%2hgH%E%=w2S7K<}IcIA@`C=SMK%-fAQO|V zvn${PZi;+@>&cV^OC?#L`2| zOg&~<5?uRT5wqZ;S=I;qc`kvQ2gX)gys{asXh(MOR_CrEV;x;bMqM1BM#}kx4#mGp zG-No>7$g8}cB#D5BCOpOs8`RMZbh|B?LN4+`q;eKCKZ3S#=&pcLHLdBxc;7oXE0)% z8VsE!8xA(`ek5_G&X1Pe#i^Ua?+rY}uyiuqZ>*KW9zC%qv61L~HY}Ng7_^@8r_Sl6bb~xlPZhgUP^pWE$eT zpN4OKs9j?pOMyWLHA)s08)QRn9WoHxJ0#D+DcMO;$PHeU+o3Y#C)Hfpp6KSP(&E(9 zg9{QPZXsQCYYV%!LynL>wB|e^tEy8o8FtW~tKy#Lh?QDVkup`xXor+Ew_mfPZ3HJYS8H;J0j8LKa7V;$8BDNWxZ`@ov65}K z-hlaaP`)Gswt*okHp2O;_X!K;e^wrK)q$(VgQG3$yMKL^3r-adI!)HNnjW%^(s2X- z@u|mh?T9p5$i`9_uIK&e_JNp5yfc*7C11j=v4>U4b34cwk{yCcvP}po)pIDuoHf+4 zI^pzd<#rfddU1(;df=_>o|CwZ5-7T=gRc`*iP9sJx)DleWIL8X3J~H70XK-8AlBs8 z?Nv%3_wlh{^Akp$pYUrslreVNa;pv2TiPVCo*z2n>C<$1l3gz9Ys;a})hwa8jR0Ce zrN0;T#@cdcuy8SNpm#AlEN$r`4YLutM)r}4ICY?Gz=-{gJo~bifr)~0BPxw9bj#>n zkxR*hjz6D1Th6phLM6O4(7ZmTBDWs zJOUjlYV~RxF1HhpwoEVziE_t57B3@Z2?Dqs7fbFZDbD&WiDbO&ghN$D5}`wOU&qTX z{{{RH#?SoFKzg1ep54pqP<%v3+TqFp3Sc@ohrF75_vzbT!s{PZa2^A2d%EfFVn9Wy zj~35Vu~7FeW{wi1wCv`=VJaMm&vwQjwINx4$;ws#7{32~$HS)eIjt6YP}_oenW%&e z&m@2wi>0Nc^AK0eA>94&7HN#3ttX|Q5PgqNJrd9WYy-|JS;Vn(#AAQBqUpqWwghEZFnV8ZCsb<2FmUb4GXnT)TBv<(-oam+W$*!~OttB4C!}Nr zGyy=wBmz#7W>*2sY=I=$d*Xd{sgQoj(~}0R22DD2s>Gb22AAJQJ(VgRr%wytOUFOm zllatL^$6b#sH%WXNTS@WnB_eNoEr>vPG=tWO{ES^bvi5j+zUCx_5nvA>C|ZRciWEh z7sDI1pBRvMtw1z@2xU-Q(ENb|cCy0fBoaIdhoEfj-sF+=P$5sm$~|V5U+?}U@F932 zy%cbs8I*(pWY-leO9w(!SU?rlogMa7g_;5&=(SKLGxdzXHvqDx#@(~J09QKfe9>JWg8aT7*sEj+~vJ9fNA@kOQpf`b7)#dcu~k zCifkKY~y=uk++ZE_rZ~pZMx?{^fZ95E^9r9A>p)w|BNDV^hw~wIyAwUs+lW1Dyu~$}-BsFzx+zq6!A#g~ zU27DA$EvOJox8xt+^!%i!P~@xJ2|P{(j3r9XmBc=*1ExvQGH0=$g^2dEm)g987h15 zNqyje;I4*#l)_7hVyGq>>^Uy#2WJUGjStdC=Vd;$&SL$#YR2AB-xH@WJ?}ApUQFa| z1Fo!rJP%^VfB=dihg}+$M)!RAK$?2T^P)xUX_(+aiC!y+KPt&Tm;!TA|zuOB%1c zUEX`K;q~L!kCFx}PdsoNEl2`&OlKZ=UmJ*R&|{#@c%IU{BxGCM>5raWqiZ6%(j>N8 z$bz2L#`-j(j033Qi9CsPBB#W8kl^eLR`?dR%}^fhh@a1ONO#W`~6 zQCrb;0tsLh(OOYL3v-FPoOBXiMiSyvztQ`I(4pWVo-M;tg6pkTC!PriP|KY%1Kq^MX1{lB(w!6E!n>>DaSBHa!pFrhU0n^3+as^h% zm84|_al^zZdzy~uB!e3tD$K=^!<W|*;9%wG~ZYw+$yPibKqq1{a=2UWo-PF~`ST? za7;mVM*i=wzgMmC#wSx~V*pn$F*vlrxkey9k^7GIN!1xxo}OX(SMA{tyiOEhn5|WxBA- zE<|Uzb%h{I<3Y3Gd~Ay<&Meu~+`|q<4u^qbMRkE{reySHFoX_h84F5roeL?Tz6fS9 z{ep7pk7zWw(AWy7>fkF^i4KOXt;`Wjmi}{hlJUI*%*h98zCUzkM`o;DP^mHUB)=^d z2;)o_%w!a1O>3;71Jc0)ru+yay8Nu_9rWmk$JzP8$K?IO*uAjOkf)GxW&OE)Ptl!5+dh$CLSu9=q^0)bz&_D^yC*#p`%O_o4tVsNigD z1MF*3vXZqdGJHThNTi?!g;?izJ$HPJ>7$IOwag$0o{dg>b1L~}3Oqq+06#*;P^s4420m);Vt zp0-fkKc$Cm`gE0=q?aPydk4h~zul!fXD+Gyv-6%QDZx;kCZr3L1KC1&lFzhN+-p#7 z;j!Gg521!h3S65L^_?MIu6JgGObu}OPtZS7<-FAvQYcK@4I;8H1fh(F!<$BQ|NR3K zjjx*V_JPtsAmPKXZ;CuGTMM+p?Un+)XX2Dnx_|-fCk~FkeEv1l3BX!+oTo{xN!Hd} zAnj@B&wLB2bMQE|c2rD*Bk>x#DU*ur+3i!Yr9}UvL<-dLqSt@Ly6$e=ShyF4r0a1* z@|&!9Eh`7aMrup#INmIUFhz8qJtbIC=k&Bedn*@RmfGG&=g*_|@59>{muD9%ks9(Q z)d|MnHR0vR2PNp(5^dzubEFR-#*RtooTg;NhHo9_SXK!^82h{5wMTrDb7bd}g$rM# zumK>Zgyloxl@ezV452yobh6UGVtA5GuJQ%1zfAYyq4rg*_1D}gT2!k)Ep$kvf+7le z>2xhx-g^IUfE4-D<*8FGj4Jt+WhnLA@3SDdf5W3q*<%Hd4#)yd&nlL&!2+$j$-kEr ztm_=j_Bnk3k^(6obL-utBx7U79 zHsGNfqiiMOtx8X{$kL;T=_ih^YAo}#&a@?E^T4&zOwSbpN5BwOtd0Z#6*0u#bTrPE z(wd5fsYt=5g7ov~=Bw9T-BE1DA)imuhaNaH(%n@mdh}l48b8SHI}&c@VW+?Ws)LLp$D(LI`elo?Fh98b^ z@S_jL00sBS@4{=cn+N8@)zvW01JZ0G$E-nmB#E2TEwyRtVO{p%BEdm1v&{ppB-Zq3 zk|hi;`UALd+a<{w2I?>U9LlcoFf8U7Wr)L5b?RK>NYg@h}d0%gaR@CHDHgPV7CGMm4D2&$ND^H--T+Ni>DePr5-xo?TqcWwSUHf1@D zbC~R8dbuU4&~250$#A)~u5v$Nh~V~d+~V*vbff~z%q;nnrPkF%PN9F?6N99%sO`iC zg6X?#!z)@G>Ajw8J^%f;!ry-jqWrRNb4bD|Hzi9ou&($ z8A<$~_EpE)jzp3aN6|8b?*BKMV?9jo#kyXSkbspGZX*ctQ1-d@4$J_vg2?{K$~x%7 z6G|=bY+Zi%8;wMe*3({QJ0P7=Np2xeucvHNQ?K8Uvb?rG zc7O@Ae9A1_o2sZh+a}{WQ!(zw%^H&15+A!Pl6j(C7WCw79qG!2s)KiDe0pwiZ zJK-Embpgl#^IEn*v(MH^d{F?Z5$-gGTBAJqu2|er_p#1Lg?n{&Y-6p^qXpq2EyipN zkM2@+clg4;oC2?rzvVA%P%@R^k;{%9l!9T;5i0PWWBkwIdmekJ}7xW!PU3R^cI2-e@K+akF=fo$AfUl2@ z>bmK~KxMEcBd~_T1JY?s%%8)94^~bQ31C(O6yE5CUY?6tPUBh36tKgvy4z*ybVHWF zjm_}9Rn0b4fSrfb8z6%n@p`g%=IKP@dH3n-=dXVZ?|%OFwH>hch=YEahTKTpD5wL` ze-}f7K;Q;wt&MBrZnyww*_9O4gpD~)u9=8Gb^4fTtTmsQF}e%0koAO3xa`(tFzWK# zujI+ef(F$;?cy2^M`=uICHWPtOeBG76yc&9#(g&Gg~TmY2`)MlBaz$0%#yD(Ozzrv z?US&m2^~Xh4$Fc2^9E^C$vh4Es{U%|jWfvKBs7}>%z zIy+Jvy&^=wu-B%9<<6$G-@lLfUTdT<)Z-Mq=AP?TnnkqB44nvz4zsPKwYgszqO z4c{8H5N3mNb<0T_&Xk=$r4`1f=)(>>rWI&;dwfl2`PyZkwd7A@n@hkH?8&05^!UI zm4lL{FDi$z0L>qvCD=}`S_BFEDmenyoxNQdx9mTKLSDA}iKi>OiznSs?lrplPU5t= zKy!q7JuRz>MOgGxO_PR196wAR3$qdtx3(IO$0?4C?)075A+sBOdz z#f#Tj5CXqqhHh-z4^DKS)Vne@uyr@}%!6#c#`g=89tu-SI4e;Nz+2j&aaJBZTkoX{h#s^4At0 zj`tlOs7|UtoGrCWJiu0&*472(eH;f9 z;5fk4=cuws9xr)x0v{@*%~H&EIOLeMojWZB)oL~_$dFMmUoK~7Q`Bb}_8j;a?X!29}@${P%*tP@HWq#7jkmttNNzd>B#W9qMo=D-`QD#X*( zZtfCzu)xUNp`>s|4J3X6j?oc@51TwyN3y%pcIYX>TmD)}jI#scQ5n#qFCOSo!4c0{ zN{p2ds*LG%SN`(?6w4r)swA+c7F@?`cb@OJs?WMmgmwbI`U0d6GZ&=V@=S67{`?^8i0#wcwT0+H8Ts$e6@awm|zNZ4g0=(1{& zB|sd?K8uy?k|(Giqt#9K*k~`ufJQ*Vw|wY4Ddbu*hqZ*I4)A27w|o#zY)}MBU~Yox zY}rbwFJAwwj#h78fHO%ITHEqf*ai;K6KDJ{1^>RTzf2NJpvWM~akG zh}8oY6zTI#C;5x_0-yR}_8?|u7cIH+mzPUR$f;r-hXbaL9V zO!m-$#+TJf28eC4;DkD5-4SS5$*!zC+TfsHz{hc99I(g~eSA@{eM`M=ZD={AYz;%H zDf*Qtmv{d|r$kXXUkO>ZbN>cRIO#NDh{l48j`#cerw?^i}MRp4pa>FcLa z;mb<_HK_C)7q_a#LUIe4&bhGtiJ2D_*4OgMmEi;6q*zytNQ9QB-O4vf$=@6Ny?ueB zN75#s>w7jifH|us)l8H3q*~Q&_YM`B+*DnTtJ>G6cQe6ou1@ul0?#m5uuL2 zO6C4w4bFB;gBwiWi&W0J?mD)4cbVqIUFO(FufjRbeK$<|>RsjTCG1wOD6T9Cn%OBL zXJ!u(LF>|Pux*))_W^Q&9f0U65hb_oSl|47gd_7rN;%7+#w$hJ(OH%(d89b!Y%vwc^E=0L)Y!L4N6kbprs|8uD_lC|UqF7P=K|0yYH<2FWRf=vLF zW6cc7RPm@h$zl|S%JSmV`f!^rH}$RLq~}CdFICg7vA|m*oU9Xu7Yy1iHn9{lkAHJD zlh-_OIJY?D(I;Nds)VC9k^D3+`7gTcFr%$#ktZ)>lF+J|(e%_SLdC;ymg;N8uI&hC zP(;ekuPb&=WE58QKUtcn=D{ViZVbfqP87l-yVoUgFB9d4%z39~c0iU;#zsk(q!Y<8 zm~fFwFjlFdk+{U6Vgn^SO})y#U^dsE!5J`YwSY~Gg`Aud;khlS6?Lq%uT8$M9ztm^ z|C7xU9?AgcAZ#zS64P(s5K|*NF+_2o=MQFezP+nZS_5gDa$de*w&UtvT#9pN`67_@ zMAM;ZUUsBpDz6P}SC05cgN1=Crzio{Ei8r)`rQ zeX1=fsv8ahC3r2zMLqM{6HD{~s2JwiHlynU1!mKzz%;#dXm3l)FlPM(@G<)o?j>Fj zU#sLe#lWgqFPFRIcvFLcDABUP+-s7&QALi+0Nbx#s(%UVvfTHEEb67~k`Auq6?W^Y zDz%dB5?>h6TOu>Afi#kO}Ym(9iJ)Q>KsKE9@A z^{PVWr9?7SO6E;e=}}O&eE_{LXrf_-Ne;=~+O0|xQ;Nl!Mb0phz2!SK%;3;Uo_|q& zQo7*AV&71?)p!Xc(yyOp#MU2Qe|9cI_5~0jP0B8Izh%$!oR%n@Xgth5sY;XDag<(8 z%Y7jRM^DP)=8g0%!)+^-e9n4AC>c21ofKNh{$2xy7=bDK3#d5{o>Z|X?M%mLm2(Kb zmC~@^m`~UW?;Ke^NMiS_%({mgkSDge9$k)%0N}Hw)x(~2G|AmU(P_5!m1PGTy9yVg zjx68&bGkpOc}Tb5EhipBw?-YlS*KNrc~#V*>8)G+fS;!|+k{lTj;XV=Y{dc{_G+~Q zd3yPTpgGwVqw8S}N4t)%+(!;jr3fst$cH>)Df3+ia&=7Ltoq6*J40!_mL0jArN-ew zuo(p9k&^-pR$_=3Nm)*xRUWFc8&@f+U3xuqOmXfU1ZSu>$*beOsR4Y}Y=TIOH|=V< z{qnA(vj`l(sz%mr`W#P5x)+Z#`dppjS&y!--|M-o5JIRI`dh1S!H^67pEhz)IZi(I}<+b$y|7aV%Q&rU^Hl03Ldt8!e*}eWC z%!?!?NIh-A)>1;Eqf_odNwZ4h%W3-~Y9L(-ARA`13M~-BY?}~eNJCO`0iea5r(zv3 zxL#e%9C-6WgOu$Z1Nt%rc@&{d+Yr6_Ohnier4GI)xo7C7G78Rm$%_a1jkGGj4&*q@ zzCoyVJwo<2)=pT#!&SF?yBqoMQ50j6B`_rfBzxN-_f>5h zv}-tCre`^M>~^JTX)wLxPcf*pl14Ez4e=X8dI#gBjH1SGT_t#v1*yd z$uV>ddLMY+Nv=eaYtwp{#AcQa-XP2}C>-%*nqWb*jB;T7TX_4~Wkc{%$IWE3ZM`}q zW#eUN5@Rcr7Z^A4w0`V#jYqb>xJxypMsMFL3d#A*d?xf^7F7~7)#rLcs^L6VSG;PZUbP6|;r$hG}qcr)2D@M~tu`>(I5c zL`nO?kusz5N6w4kiS=TuNbR66#SXy+u(<5rmdV1z`8leqQBXHyjqVP!N@AC&Kdyy5 zgpDH<`NYo_`qUzW&*z&TXnX$%^G-lcaoA#3dImiYGZ1M99?^ z0T4_KDT^kSTxgsv#06N&kSDn#ftR<98Ds}1qx9Z#a$F=0r+%1ck{h^svMQ%u>ce@L z)m4mgpMNabSMF5PrtLkuhAD~J7d=KtQY7t*L(MT%f4I@^q<#tm7wg(-8+c{Vp!;E= ztxt05B8xsyd$g+oO-`*hEUF_9fUGd@I2p2l>yzj~p_wEqo_+5kdDXyju`dg*>vdp| zW~mh_o2J|_qd^nj-bl`k1tRvygKqKi{qVQxwI_A9**9QovE;Ax2M}oUC&Bxq|T&laW|5{8=1Rs zvf91#))0~!DF|}UNrKGHmhLq5nHJ!N=Sprmc&?8pZ672m@UBT8;<*Z3OXK@+V1Ng^ z%ex=F{^<2jit+e7VLX1OPFve~0##wTqOh>AO~C7=yVcN(ouT_at!+j9*VG} zFfhoj+=I&~8P$t_9LbtcciwIJgWtLsO$zn=*o{!rogL!%vXWlLJsGwpVNFZ|=*^N42AE+WQ4i;ufJY%ZW*WZ`T-StB^_zrY7n;7pB#eLtY=WW7kA_outle z$hu`X4!Wpleqy4M-tl4^O0}Y?Vx=;HPkUM!Uxn8sY%Z$u>$ zwMlwpl*5-<|?BK zv4IVq1nb%EA@M9|rbw-P;Dc&E=y+P-kAP}yE)O;YT{)mB7};Ht&sLLYIEFz`!Mc7b zFZtD}sj&gDn%mlvJRX-0&=#8<$qRwB$mBW!3GaS;Wnq;CdB^+v_BI}d;SuM|%E!d2 z7QN-*@2zSk7Bgz4t3BM$ju72Q@lT*s+BcRnn?l1ZKJg z;wzIO*{Lf08bV(Zq`H=Oc#c40N${u-g$ZJ-DRzKha*!f5Ws>amGSM9rlVDpb<~@EH zcN_7doN6vWS*@9*xF#)q8^sof7ZBm)h2dz-pK{3pl7>u2x35a zJ;f011bqTzigYXGUXm|SLl$8&-lrsLwC~td+00fKo#d0wDJN_n5Yv0cFeJ4G&?zk* zFNn1ANhaBNsEBie-hqmo5>WZ<N7|>ca+Nw(#G;UIC*e$o3Zr;|I zP))(}gR^7$y-DtZAnBYHqED=21c@CBGU%qRI6csKWR;kZaqm8T{dIW#QpMrvvIus4 zVFTd-22^*pa;g@MRnk3v9Nzw*igrnO4fI5(r0Ss)ogs)egVa|;p>JAit!&4@;>|G> zCqSUp);@gN7YLeET9n(_9@%?cRA5v|(@^`BFRpg{JEGQqMQznzVI(0t4}Cv1t%-_> zDQ=gVA_xXt`zaA@mORc842Z&J9}p_5b^6TaT9go3yzt?zwECSH3$9u>t)BL&I1cIT zVH(|{>?EZMk^3yIxf6zjW*}@l$Eb1zcWjH_}A$xd7^qs5qA$;-Lac3a=;ghhT&)(xzgPy z_K`!?7o95v%zadt`i3T-Dye#Tg1$IgHLA+1$}@z}N|+uai*?D0(ymqM5ef-u1J5Q^ z;L@mSjf?X9CA@uwjN-xbT9%1*k04iCqY-?)vNMHZ#_B(`F4V6wP1T@Jd{I9%e~fd+ z5)P?-1NOn|CuDCYVS)7Lz6)8x8!nUEX~uy*5QmB5$;;qo{{{X*yUM$@gOZnSMd4D- z!P5;{rbm71w=KneVj=|hW+I+^th%Z^kJ_+V)aal;;w_gaC<0z|etAhABAwy^9_ThWq2qC}?64wIz40c*gL-(DpL-0ahhX&MMOr8oO7 z349I0xvopk2Ed0|BI>3X1j>=T=Ita%%tSn@^&0(m zX|}bfgNS4e2-VrUfq+e~}4KlVj;wRNElm(uSN-djXQG~*RBF;)U2H~r4;gglS_^tx- z&O9tUcPkGLo>T_x_0Jc~_Mr)Jhj~&|F1>Ab-&-kQk0;Idq`*yDq{OjjJ;#!l2X`Xn z9M60mDZ!q33L&}i22o3;1q<+YH2?JVXL7uLz*kCe{PZ0VK+xUl)e<TQ0do>1BmF2lGAX|ByQ?o%f^WJM4s`fBm!n;#_fH@F( zVE7|PEl*n=Msk3iylnepJVClq8dW>$l=W(_aPk=kttmafp2Ch#P(hZ}+hw}Os+|LS z5)T}4bf!YTce3RDCdG!==PO297fcuPnkg1liz?VtNBv4E8u`!|45-VQl`d7Fd-;?a zueQQvCERm(D&ysWKrn%e&u&q7S3|55c>pLj*#J8WUEndf^HPF4d)UR0ZERtVN=lx`DR+T5w3`zdXp_TIlEX z(1TMlUtni_$OV+NH3|D9l3PCx&xznRL>cIcvtvn+eiz<;e_8KPtlIlR@+lh#N{%hX zQRT=0)t6eK)=Vww4YU%K0$~*Y%+mC4ewcoz!@~A1C>qkXM`J!?D)HH1 z2Nfu8pIaLx7FGJ|4|e1LELBQm_30Tr5uOHsQQ;(D9QoiMRt`i5+{lo8*3^0XeTty1 zI>B4P3^TPXFEqG17D@;QsiRQ-Eml9zI+fPvy<2E9W>O;-$pL&G4QG&AxH`HV3P`d;nWAUHHqXL<*FV+hlggEib6r)XPgwP;x8;urZ|c1vzCq` z7@16qiiD}myDL{ecj}{H8|4YOdU;n9r~@BOTQOk%HQ@QzVRex>$*Adehd#z*%CJ{u z+WQHv+QH$hb_t6Tn0+5om_3xGG<46=RYtxJuODAtpvj?jLN_PE_H3L00|I5ZNcmo! zz^oTcWnJx1E#?x=`|@4;FQhN_ROAW5FBw9$**u}-{PQT)RAb9s(S2~|+NFECtpI66 z3%0;69xe@37MI1QADm>#&$`{_&_RD_t)|%$+Z)>N9P42*bP!a4`U=f{QaWjALQIYk z=t~x*9JEcU!&2kdaT?WrHa-SDCZw}EAJy`Slx0AY{g@m7Yn3hJcBpF*^}Is|XUU|N z(Liz7_PwKj)q1*XB~job12KEDJtH3-ZC3yhO!uS;cN_CS0evwXtP0dGgfSLSkyLmN2Kaz#{pfPXD<6?WLwGr{+Urz%s$f7xB9~ZW@yZ6~HD{BKCDE%M;nBlgh?Aooo{WD=pugID5cK>ys>YdEy%fcIqF5qOeZoyrq+r z^lwNsM!O5aPL|4D&yt<^87bPaeJF9hrsS;Vo5E4;Qeaf7BzOJm?|moyCtKxZaLtM$ z`Ex7Nw#dDvrE8R6!+kRiV@88V?@dRSW4TmUjpzTtT`A;UPkUY1F_)W#nA>60oya zPe3TQ&JE?QtM(IhI9b)rl5fGQ#;vb!x`VJoVbg&r%mjt--bBqBB^^-YcQ`f8zIgkH zC&))CDPPi+aOgspdV<}UbFw>qZMI{td)`fu0?RqcUQvNr1*;oi&((6^X68qFNK1$F1Y`rug{n%Sv3vUR2QQX2v4iRG^`&3m5_MTN373yX=H zLKoGp>CTV6)R36*p}Lvjp^lQ2T>_)Zb6^6b)T_>C=4_V!9y?T0imavu=^eSMc|BB~ zn_fW)O?KVK>O@`YCG+Cs0mFg%1;B;SMc}De&?kHZ|ioDxAhl1O# zSIh5y5?+6$QqYXidd(IT#vS%(>_$9czcIX7^U7LXk`(`#{RQo-D22(rldEaCLg(t5 z8L01m_{|SF+h&4#?Fs*qMEwY-5tgJwTHX@w0T<=BU*IT!Beanm#sZ#I-Nu>tklYV! z<(r!rD#IJ3Z-~c5JRYH+vPGu)kj$paKb{}EX7jY2(B!D$RNhtAk*77)1hoW=HNK1AymkaTxF_#iDW@6^KMA|CVLbq&R=B+^t8_Lo=&e zpb)dk6{7LVgarhIT92qnCx(B5SEP@$6lhcKNf`;ce8)xo&nH8i9?;gQp%h+c_+eFB zxIn?sbAx;YD4?TgFzXx`wiH%3u9X8_TbHFpyL8#Pn$`KO!+Mn?0#dyzKO-(*)|N2< zllDx{sd}ynqYdibOdV2zZ{nh0_OH6Fm!;7D#kN;Bc*c%WyXR6 z!}6ys2~niqY!gDV@GD8z?m>jTLKNU`N;k$5CbditCh}h8J~OQkJ4aL&p)WB8I|4 zi)$E@RNeummU{I-*aAz$RI& zg%H(MwCi7F1-QrCQrtA_p`YT@zPK-V3st&>N6s~>orSw{$I0yfOPk94Vt}_Sy3-cPpb*rqSU{V^q$}WeBT1^ysqo z$R@!aEB9gcp^U^D&-yC~-#kdzvPPFM4wtf%)p?iuduJM2Yabp%!wRSQo%scrX(r7J zui_Wt&*Z0{W%p?3(xv8?oc8~9I9cO%V0aNM=LYfB0tVSt_stL9KK%dx zzaf1|F;4abTIxo1YE_Q87)_rVg5^qWKXo65LGG|SKxz(RR;$4BVIn>k;RS@=nlPoO zNk_w391bFgOb*S6aR|5e^3)|`Ux-t<6*I6`e$t@EWuTot9Zvc8f~Iz}SE-X^e9236!Y)?lf-(?oCE0Hn(Ow zIL$CsPHI9{WlWW{7{On#l&X=CQoIy>WNCoHDNpN$!SU)+dtgHBN-gTvh?L1T^sJaF zxD2P~q@FkZbV!C%900BpP!yqPGJ5g&f^ZTM_l`v20)5c#kNlH`{NSaX9)e8CDtN;| zba_(5)S_J?<2bIrP>P zYF%8fNGig>L_^;w4V#p7s2zx!CVsoAcro?t>RMHqca#Fa^96n3i$?zCjeC3O-d%_Dw5{}?ZAXSw>$$SbQ*Ab_)#th>>k-@NPw^C zY&{sgSV{Ns>(|gi1T*U=?|%IH`?sGfS?d?7#QPck{LMe>-+W@HbPuJ#bTvvLUb;CK z9c|n|B8KjmZ0Ow&K@h{ivsMw&{~lgH*B;*bz(j9XwLcva`NA47AOc~D>yCh?tBhE9 zA6;pM0Ogh)6rno>3elerg|twb=L$EZb0XU=vpn#kMJX&R*z2TV#< z39vTbObxoggcfmVYoVXaQ}y93ETDjPU>0CqVXGQlt1Y!A)PKwg!-Lk16TzZ#gwLG4 z)Ga4TAMCoJ_>5*_$7>H1<0FHJ_PANApOh$4{I1vlP+NQROrPAiW&N{zr@^3OT(#F8{iD3JM3Z@M7f2+;R8}_Jm7ob{PAwgGne*2insT8AqW%| z<7tM|PGwFE77UJ^7Z}(Uxj}Qfg4j}AN(v9F#ygV2oxcIl96qhog${wC$V8fo{*`?M z1&W^85?{c|=0C_^C%I>btm>T9anD*J?@RXh8W^qK{oCtT;q6P+a+h@DP)>nloU+>` zoBinp#U7lrYV0v>fB^2B-UL;s{D2CZtKHYOse}TTDyATkpkT$)?-G`Hh z@Z6u6mH5<7>vTOq*s8H+FB9kxF+GsoqtFy3CK-gSk(V*Zyh^>T4xe12>=7WWy#U70 zk_QEjAPzm`>FLn`4O_tAWV4S)1)wmDPPsDRko(X@V@Y-oafTFe65j{2tT7qf5>c%< zI%1_T@F)+pr6{07O6E2aPtdaP&K|aV1k@&YULv*h!nTe6qLnm z5l~hGdTU&~=yXVc9?1WIOI$A7PW4}!^}YTk@DKJV(G+oY(z%uzuG>!3)Orf%rB|v5 zOkw z|H@K(&NF~F0W_Ld=Q6(nx$HnjFh%m@fiv}L4**Om2 zt1jz7_ zx8`rPn{p%bzKz*p23EV26H%d11>a<4Pzdf$Qmf%?PJbacxHLflNM7KduC^33SB>h~ z(HE;hlWPe^nBOfFqg=F4EtRU}`NIZKyiV@irW)BE2BqE+v2px2KQ!><&n*{F&VST( zS9efZv&aFrc|ILE5pMA=nV0w3F&ER(jO~26JfKIKa80W|!Uaj9U1lrDf_|~(1T1N0 zf;!B+mYUfXNOzSy+{DcfNlk5Hgl-+|24q3(H#?(bvDW~d+Aw46;7(QYjBE?gV9%PD zSUsvWzp=N6Hr+BQeBmq%uY*R(2)h>_rJ*zBBSt89`JW*>;a%Wa=P@KSKRL^~l_0=G zUg{sonpcWBOZ`_>AY}d^y!|;nq1WGE9uMw(z?~$~A6hDR>~+Y#wy-OD7yunGKG`)8 zH0g=vujSM$-5SG7oO{fACMxCz^*l-IO&pXE<2q$hm*S>Qgrn5Mt=5Uqr`OY`Eke(dUgfut;Gcuu1S{};mdatox+Y8lL7;?u}SCP6+vh|`|MQQ z=gL;sh_;#p<=T}ix0GAWOqwN_Zxp^!%~OYboQT&FT}~FZkifd2`^7b0)L?XG;>sX5 zXQZIBY|sWh&a^q2KIuuY8Yxw~esR_1Yc}5*7#ltS0MDRzA%ZP{)#EHY4who$cz{OK04zq3&!=xcmjplkS*qaI06{>$zkdt* zwIvuWa&;d;G%_pib~S@`yh)P=Qcsqh)3 z4tkCaJDBEbscBez_#TfrtlYWYI(4SX%}|RX>RG)?Y1!JtS{B3=kN)WT>=y-0$tzfI zuHH|p568W;^WI59{3GaJ_tWkwX;9@{dF~AAdTNbW*eH#TX@-Lxy3~6=FOG#x(i>7} z+!)0$N+Y0IRk&>Af1spIXT6{GD@hhuul)uXz$PLnJ*+_3q%w6N5>{R7~ z@1gvzbH~GzcE2@W*y-@7WPZ?IcQwYlp(qPa^1x5HIoYX_RkX^p1stfzmJ#=}zdE2e zgU)gXD~Ev?CpAxjchyv1-u)ctGo8wGjsw~XjLNE_2 z4>`+uktC}7-X3J zECD5Q#5e*|$s}9h(5@9WbPdT48`>x}l>h7HgzQF32V$oAe!!`? zlT#&$*{WRWzNKbGy4izE7ezLh%3Y-jWut6%0q}EyHZ1C6lU-3b zF?%9|5qbv#8+&D4xTZJ49o4}ZhDBSV*zN@E=nY-vmUK@!d0t=$!A{Z*I25SD0(4av zdrA(gm*~ho$hI&(OWw`Dj}=T0%-8AltBXE=32IlM{bsFtnFL52h)H6&+$@7q)M|qj zSPYZA2kzC4Ki+w3p?fL9ojWB<0YC6oKC%X*--7*`RC-mVB3mQRiU87yC`O)S?shP| z;7o2&`Y`=Dc1Tls)bny3i?oPA=hwKf77)pDK)-t3vSc9^WG&l1tsgn-rYj+t5O z+(#$}ne13sX;rDl(@SI+ICIpbaFdkJvszop7qM%Psy+S?NE-%m(nZYW$aZRg~ z;sH47ngPlZ5sa(U?rJc?0|#Oos)`PY0qd+TMR$__l{)(}o~#*CHxA7>YoGLcXfBOr zUFqg3gup^SFmXL{;es!>6}N=YcLZ4BnKG2a-JVgA=iu4eaHeRrc5FGrzjg%2TwO>P zMiJJzE#YPbh01XUlz2cYZ*wg8lvwbvG7#id>llCs;%f##jBdn>P16m?9kiH6+2_s( z1v0HwU94{L2D_}7PHs38HZAV|Wb0g_V@yC%?s}w(9MTqUNEY|1XM6&P-HxNXYPjZk zUd0~cIBlSoDj!;#5kMx>jYeLUW$jISFMLDmR$F|%5)H0BeMws@H=Qh1RVIC&#L)Z| zg;WyR59+RapQd3w&-S7*@DGOmgbILNlie;Ks+Z1V+2M@KgaV2)oFvs*b`Y_W2rfDM zKFCRdP9sN%NI~2qfIc;{JxbiAN%?74oGxKH%M^|gWY!guL$YHIDabDHEeLe*mAAWj zMI3=w>EZb70v5|MJ#vE<23>OU>=-H-U-O|O1jlY~puJ9;Vs$wp9g$)$mbYzmvzXC9 zz%YAF5Jdm!6%qHnNYffLw%=6%?$+*p{p}?8<+|thx}NanV9wi?co6t-_puPvG~~!b zP#PlO{p6?I+I~{M3T2@bq^Y9bX$uw2TQaEiJsE1mVPkX)cpi4NejJ(`MQ{pI4rn1jee6WKx2qVo5CRvRz{Roru z3h@)xZa}Ux7OVx~9|NhXOQl0wsou$j2JmQC0{YUt!s|y#SHu*iV9JL< zTKWi>7Z#TTGR<~12WWn#ftoIxmPAdm-cf;x0>i6}r8}5DBr$i{(-m?k^n6#29Z{>k_?eqy~;DQj_}e#;k>qKR?6TylUl`!q*#_B3sIsv`Ua}8>Cf0saJmD|N3Kl; zPgVcS&Mz=dprQuyA1B~?N}x%?-0@K?_{kFC{1_#%eiX#4EY>8wrh}^SS4eysoB`>M z0uvz#DC3VE?24DKc~Gm-3bE-BQncEB9WvB##pLaJr8qRum$%4F1vg@40G2C&tP!nP ziD`~WVBNQ9XgcG(ML()pBEBZKWQN27&8ySlk7>U%kao7TS z%<+Dyi+iW0OUU`;Tu$K`A-za&ym)Z|>d)2qw!1;EVgjLxQ(UnGqzQpnECW~$IxJ`Fo*_xi8 z=+s#9l3gja)`6VW!j+jR1m4zp%4lDgDchMUm%Wz0n~y0toGB&8QJSQF_gCS22|)Jl zGe{$Tje1}xFR+V*f-2MNM+7jXR%J9#dCE8<%-BPKxPnSz?_&>~Z|;XG3dW6jk^GTu zt9}q1bV*c}BVb;*L6Q{_l{J9`Ovcl84i?+4=c6U2J@ha-)Q>-6!ymcbV9^XX925w0 z^gYOK2KFC95@gt{Pu%f?Do^#GF4*#X!U!k*mf`g6EZsc!nv|+7Tr3pf01f4PYiLvG6jH0DVRG$bMrBwDkt< z!wPnJ!tC5KjVj&P@!cmFwr|T>8F42icq}XQ8t)T5T|g{2fw%6;ue8(U^{aFuoF!FS zWb`kxG}4obscw@8`;^BWjXd*3J_X?9QXShFdNd;wnPO0Ehb&4MVshB1;h9@HBY@-hy1&kB7I~8u15!6TBKA0kk85m zZ6~hfaYr?@@J%H@SvsW+N+D1w*!`JP6A+4ungGr}g#W0+rG);%9&thw?@Qv!Pl?QG zYP%oXO@tbT(iYc(E?|EMrp5b$NtNYkvfGG32Zkj*SM(jzL8B^#sW)Lx>%ax`kzZ*m zg}%OmynSp@Do>1HwQ_n6k%FaY&Ij$Eq&Gt|eL7i)Sy1d@c!whec;+a-Z#xkrpr+1% zbQFUtAlpr?q;y}J)YFsv%bON3aol)?tpOyjdGO!#chD+qJ)Pf`sLG=+|boo(_Gd!BpYD75rrpRwlGOf%R) zV_4i~MxUO6g0K&OrKt1qUF=(T1?i%=bt|EC&*5&}#tt(UBm}E#EUvrz!rY719@iDa z>%q|zrVsun`}#);hX297{@D*_+n^gn0g&3kb<8~%*2+>dOrfpce3iRE!Anq`sam6) zXN)X4U)YMHSJNeXu@AzufkV+L+OJ3FQc`%dm${1u68&L_F)0{g*6h@Gd6lF)Gc+8E zw#oU0L2x>u^I+UbM?uJs=;$YfQs5@mmKabcU|gjF1n_E*jnf)x7plVQk(pkz!(4=c zVAI9C6c__fd(S3jc*nR^T4Vz64NIR?LMym=-B`bTUa&VgOSSVMU-*g=d zCYNlFYB$gt@037k&khhO0|;jdfEu`KhkYoeH4h3a4%=`(Ku3@u*_||3J09cN&w7fY9b;ck} z0=D%Lz@ooN!L+FXEjs(F(|I)dLlC2J8U1d3PpYs;3JVzYRn`!K+8Kr^CV`JdD96zI z%vqcRo~<+-lD5Tz2fyS^$QG3m8Qy+(5_Nq5c(FPD%Mq0Y5eQw<4}oEnYG*eI#Pws@ zZs`IaPNex@?@kofjmFKZ(&X*(@!*x%h&>oFHzBxr2}fEi|M>myhks0$Axk~$e-{c` zY?M4dUXq$s_G+pM9r`WDsd8LYd=Wy4^?@>SP-&)(=UAyW@3v zKqicy>-suJR$^O{jtqm=#M4*&3dd+FW#Iz6#2l4wDRg0?o2%2RblUd>nvb-Z>O zhBG<}0TW&L8uFyi;kc-E{o->T%g*mfpTZWPHWn=q1;+5^)xK0xU-L1irLUu53k_t2BthhK?fmxg^RGz_kN|=c8#TLmpRZt!w zEL_qogm4&wTPF^@$QtoHy&MifYWP4a(sPUmcfCAb1n!MeO+7VKZc1ne#6zKljGYQ& zO)C+~93=^fY9B~Jx8fcqq?BCRa$|ePFW((wgh^tfMsZ755?zi&V}G_g4L68ocMpo(xv4tlBaCEH#f0r)lP#^8|XGKJlvv=~7OC5E1EMfP7E$gF*Vv*xI z9-&+F>94xAJpidLS(OsUPI7@*Be z?xG|W4eLMs(mr!WWe4rR(JW+s8SfVgncW5B2k41?Y=B zD+{+0+^=@QAe^?R+1O9Q+aE8`NZ^bSWCkLnVYS}N6K4w?u7Gk-jbmQQES)i2VF_`K z?-6wrFW|#>NPa}QXwZyYb9LR3cioZ4I68RLzGo{32@JQqX^v?Oy0Q>*QGSZ`wp(|Q z@b(xL^jcUBirdblRc*fif+ohAY~dc^13_x(B+^& zW+ZlPVk+F91U2FnIL^9DGW(QNnDy;8!)kt_j8ji2@+&$wyl`6>d3Nd>&|AC%Ii-rheQ$#P>4Zi>2b9z zz+{cXhvZ>+xpz;+u|^13o@L39-*4^8p+7+UvtqRY_0>~)1=SuK>9XU_wYm3a#Gpze zg31Ln9SROLCO3hEPhWoqHPp}EK70N6-Ot~C^!A|wZV=bB#$Wc?nQ{+o9qmFYgxA91 znP0?-yasTW3xiSsC2{~JH%=J<{+Q>cBtVW5r(M@}n&XPu@F?KA?p^y&EgYyzVMWuO z8$8Bw`>ZoB`H5q>KMZf5UrhDtEMSiFL+X`o&6+p3$injvb-J0+ptq?yI$>ajKkW=R z@Lj7=9BnGH(RUYsXAhnqe}4N#NMCqlsF(`xD#VK20~C4EymTC^!D67FtWk9*6n-7p z^?*DIy>>=6YsTrM7>mjKaoNW3c=iBjc=++`ISb(jLodZD6DaI?a_L{tEkmUg`w7qWO|najWAGGwhK;o z3F@Sej(%()V@*q?mgB(gGi0OP{GsfsBUG|dRjpv%E2&yI4+eo!P6PJ(fCtMdgmvGG zGBqxok*NmMy!qT`t}xV6&<`pgd*%wXYC^_9QA7B{N+)%7qN0ZUOqHL@FW){5@4KMp zul6RlSUlWwbU;2TA=N_vRJGX|M`~Wb_tjLoX*+zR25^h*XjxT$*C0UJ+epcI##FGu zP)*s`xVT<{{?5T>Db#~E9U3g6fp{)_Db`a35y3I}E>+Y802~aKX-NLtusUL`bi=wz z9-B!?$zC{M!k2UyPYwg8dQ@s_Aoe)!9=j{LhW2#jAhcZkWePq^cN-T5h7bySY5})PK;W*SLGrP%g z=CbKQzXg{EH~M&JjrHNK}d#{jjH)tby6}%h7Oe7a`n8(*r4)IbbuS2 z6#bq(u_T&o>BJwDmB(85gvXapk$@xVJLm-2kPm7v_+Y*^)X&QvvQs8|GU101O=VfGdU zz`?pMN>Xz*dALz`5^d#8l$TaIHxPvSC%a#77G0(@oV9Mrr+Lx1N@?A{}75PSn|@q?97HF2q- zhDU2hwt!?=Ha=9h5nG9ftKPM#tTKScr&OVqg-G7HCY8dqWad=V-3MT2n5-_;5}Ygr zj9q%8|1Bgi*aPF9+F#p^M!-37kkgFAMF~F4u`rC5^VVoF=)R@N+Mi{Y&C{A;ONkNM5oM1W znR~q9q+F_ACU?Vp!grM#>&}k;q7yueA}KW>^?my zHx0-o;4o}&)ZGH8g3p1VuOQR6{_LTlc|ml>Gbz>s>R)4-qV}-^?%{N} zwLg?D3L&bqsgVPtOC{)}5A?6EzkB^v;%5E9c0ErbfK>kI0-V-TR~l34j9#4q$^MKsL<^#s9i{ZQpBq z6l87U15Ta8b24La-@b=6JYc}fgAiDN?NUQ2ExIZJ`YH}ML85=l?b=?sg=x`5sNPzn zT@Kl}i3MuUM6N)&sgfRsr_KwRg>1ZOPHxyPXP3R|lzYZTQZvEPigB}n!W7hi6)pZk z1C?h&=y;r-rUnq3fj`>Li%El*7982KEq+YZk?52u}7l0(aHO3AWFgNOUD5btE zNVhlyp*9QEV;H`$74W29ban@YC6@Kgy&Mbuz%4pp92PP3JllABj)3OKg7O~VppGQ4 zSiV99(@#y=l3jK=1Of*#GeE7UFDwcG^ko5Clulo%lhaKBn;GYVx~P>D&^F4gCfPU^ zQX{`u_W~tpaakod*r7r+BsX=8`>?2HJ9;qKUbjLDC?;ey74xKd7Ue#K`&1dpoV8EMFYT^J}*A0eff($@#lluNGkrywP4irttFam zM6(i|tGWST<%C`ii&|TJk<_Yt$N?D_souG8VcP!Swa%E~ySQrYYNZ3L2K|FmN@0hL z1S*u(e6VU2WwXYYu-`801VBN6vIgM?dQ94Z1vGWzf-%La5DY^+efak0@cL_8#5ndg zXBoiw+csnJqR&+30Z3HYkK1tQXdZ#UkVWWnrT(Dn{Cq$B@Vlqe<@GE6N>85@m_f@t zfe@48>ja(co+n*FTFL+}PCQGZ|L5?ZsS}l|-7;_LUk5E{r zR+qWoA+NKsy;g;nS}H0|>Uwl^2p(;;9_PM*_>@2Q0uV`f$;Me2}!>W;hmf8AA8Yn*w0fYbJJDdoum0nU0s{ee0Mi9ioGB7>$5#a_+&EJ2Xa#$=)j#{AVXsX2O7 z!G2XqMC0P^v@e3RYi9xuT1(1lhT41~Q(ZH7iHm&ipg#g6L~TQ0S?mIlrZzkReA+ay z(IcD7;_O0wQg9By7UkIsoP{rIxIGNDB0|YnKB;pS(~;r71^5h3B-dMYMCqnf2}1C3 zUR?&1Hu4LYF)kfkX$8HP$-*{K8GQ3k@r{RL+>^6|{k$w}ADc&#jT_-AT8BwzsPbZGuEo7-( zJXG0nGo1%#NiO~QEg&NRe(EUzsrFfNUXmeHqIRLFVW7{5{S3gE@LDKJ3AJ5Hzn7MU z{GCb5uPv%sd$q7iG4d}Ov`*~@M48Z@;6bkkK#^1TNTr(_IdV_%P!EMQHjuR~Di#G_ zTn;nj=!Wz`g?MV>2tN$p`ObGNN>E!^mHYC5w|6DmRFz0LW?FL8r;+*yzrgLq?m>+b znPb$?-AO5~Myqg za)2t!GFjCFFb2BTBbDeH=VeRawP_u(1YObLkKN3C#Dn>Z9Lv;K1nd4x3Iz{+EF%)U4)Tk=tgxRM~CoBb$ICQoBmYhpI2SeQzZy_$d zP1L#sFlr%rkfLn+qT&;E866e{{O-=l8Z>f^LRU9BaXaWBSG|+^9%|J(mrL7nKt%t@ zS+4Azguk(K7L7p3AaW~YfqAAy9Gnyh%FDw8d1=<5)G`e71-7d@r(YKEK&S}zm2kzk zB}ewdoj@U<;e4WI+O)E-)TrFfMzv87>i`=Vh>_ZkySWi-Co6Hm*qkZRfbS5iSIiB{Y(wLybIJ;$=xW~~zW zo;M)uvcoy52cy&}mr)k4g%(oNitWV3fUriTPGmOyZ&Ac#4MvX#3J0(s)u+BkZ=LW! zD6ro1wW+eYN-RhYTy~&iF<643v}WYyW>GN8NfD{-6tXTjX3UMQiz7{mTP7$_@^*89 z$~|R)B9NGtrfroI{iU?&s+}im+1CKXuyx-mzIE;&KhCsfk3!*nz54 zsoU6kgAZJ%Bc6@WC7m^c zQL`5hq%t%p&qj3MCZvD5DVCv^-P&uH+5JV}Zb!Nf4#f>9loZcUjUhFc6Ddk8I^sx+ z2^GnW+r%{SnmID^(srqMa_c|3{V6FH%vF?J6HQA=Gg%y;5DMo)-y%)Xmbyeur3UkI zq)vmqh~hCBmkkR2oWzBfU6N1`xp*ifeLa}PxYUSdXKt2q@9toE*jqgoU*Xo8{)8ma zPxxzE8JJcLvx1~Zr|0Eb_UB=5&!9L{72=4Hrg7Dm3lY340qBjNxqN;+T#!+`(|#Z6=Ks1A#7nV z@*Wkod7$oK`MX z7t3-vQ?KPh6AX5#s8(Mce`=31L-~ z9??c=L!xAJb{V~XkOKIa(7FaIG%0Y-mlyRwL;FY-vX*@;$yCX5K%FgMStXj>5z+*~ z1)Oq2R~zsh(yha+N_GAKYn|$!8NRaVn7LYzqJ6oU#E(9nN-lFh3H%{xJ%5y*e>nff z&O-DGraqT`bta|(>3OB8kUldK#BmDSR2QaBYUssizrhb|fs(IXJz4?Q4pV`o1%W-x zXel?7;xJb@4MDg$sR~-lLwzr2(A01SPK(>x?Xp8A^+AHjulF5hs%OyGc?eBm$6Bem zI)ISYmxCB@EuXE;?25XR*)>JGn6*OxK&|0~{1#^ojt5IM00Gk@!44RbKz}F6Nej>| zthqu_0Q6=;;f$V=O+9l`@7^!_^E>^0ITyIUO#y^RdsO*6=<5M`g2;o>@CG142x9(9(tLRZ8< ze~JCDNe=Dkvux8Bv;Ekb-LzA}iw@&u*fgTpz4zomcQ4tF2HkU70;_88uVra(dB!kk z+cNz#71+@H2ze7Gp$N;+$(CTPvWF?`KnR1b^+k~+T|yd-r0bUNRBF3adbVc41TU%3 z3DimFD-IYb!|=cb1d~BDT&wIqXCOcQ9R{3Mg&5&KC$k43HVEa>kx>AZS}>5RNSp7_ zd|h_KTOt#1=zSX_t|1+vVAT7jza3JqF9bFdwF|^@$t-`U>|la1d{+YH4EVnte>mYK z!AD6xPL4~9{o(VHLnTojiGyPYk@nQ zzR%AkND17h&>5ErczjRtUtYq?OL`fP=>^wcnldlS?$Fcl6f*$$9(Sx(ue&+gE1MUv zW7p>14M4GPq{mR+#hu2Tjf5Z7Vk}Feg0xh?L zhhT-X*UqIGYJ(BP7SCdkGE?6G!IJPQQg1ERjY$|uo!?%FO4h~@{+oUe7#gn;=bQdF zoPT3S7`&uCK8XP&L@H2E2;>R@`Dri_VwvSule9-ThRjTZg$nLGYMP zW&w7U>9Q$H3_D}bz&Yp(GnQspeYm26SctffDq`x6jibrBe!#nMAy+taNn73$vt- zTaTVJ#3)iIkhcjbo!bRYZYCW^SYUUPV1GpQ<%tVB zUK7g2ixxAML#cNcP8g`Y-#g2B)75EvZ2aaE>h7F9e*NiR|0%rv7Wb@jq6;-{m%X5> z1t`guw;wRoV+$R|WvFx}mrJ^F$|mtZCHxiD`HU$^oV#bo%^-E13U-4rcI{NE(=x_H z$ZK5=1M^+=c|>QPH-m{MQTfwQoaV^TNGx{AuAq!^nySicmXNS#{v+%H_F5j2`w*kV zO=DwOz@=RCmXdSZXiwxS%L1qpI@OjM0SB=b;)wwj@Gt%(ALi?)~aM-WETug9n}Jwenn4iI;mf#^0gwT$8u6!c0g>fJu8sm*sEGl_~sL!eOSsN%i%H^UO zJ@jjtke1UjyFAiP_lV?Q+z|2s^!@;K*{b4Q-DGog@*t(|yL>17r~d~W{}>L`qMZ}c z6m^Q{H&T}xjvwIjY!#{@+UA~uxQ z;86JXd+=jEef!Jn?=G4KA_d8QzXqjh0Q{^7g>8|__49na2ct!ONZB{Py@ z#=ja$%kIc?kLvSt;2j&tctc}4y_A~#&`xQlCn9`$<``$%^U_pjy#VE6eR^mjvRB?w zP3Z;!B3ah{+hsYvNrt|`N>G9nuC`%};S5b%8BMJc-JFfcsosp8;)~K6fOcE@>u9&o zjcPjE_M+9(jfsH?=z~=sat_1W0Q!@*!xn1!&m}A38~AK?QY?8-H+nNef`F{MgB#;H ziw0#1M(}L&$uSwTJR8&=fnjUmRfCQOLFD(iXThgc^>`H$)9ed?3=nun1)9lxn@&v@ zh>c;s3-Jv#CZ)9w9!s}rnw$SoY7S*d*QAtDb0!7Uh8=pNLQ3apaAfo4QCIRW|2Mt1 zM;u)gxd$&6zCG#PU}S=c1h;XY5=@SWp_s5b2#PGEj{%pE0}5JZwB_hc`EC&TUF?Gjtzo_uML(4i6W#mXIckLoFd6-G8 zkqs^9Jr(j(MNOieOSz*CA0ufY#iXgc-HP(9Hri(=83A-GZDxT0LJVx8NH3Ajc#`qj zJPQ&m?ANQ`)uvjzEqoxBu2-=6WG_>89SU{%GN~(T60>T5O;ChRKXh*9h1|M#ku(Tw zK9+2!e1a?66DkX8UzPYWpth46~&4gsP?c}xPe90T@t9Whu2e30{j3>ssU3>+O2Of?aEb#Dgn!&|?&u16DAH@hF3r(aj z^VHGoBWfRH)SX5O&0z4GoetemVRgLA$17rb7 zF=AbH>sTMUAyvpPJ+vmc6V5PN+B#s?CF^di3G0KNa(%aF>d2bJ$)%nocB#Ywmv!6%w(RZ-?*xI}0NanmYX9sPVx>sQ8qUtd-(z zK>ILVTzmp^-O&KIxCNRFGe8V1aVQhyT07-THpe3Q#5O%f?VoJ5>n{E{FPAS#V3r>) zN&@sr;@3ZV{nJGuZfVkB$hfuFLy1xXH4scVG%bx;_Og?T=9#Y?vvY5IKDrkqtR?Wu z#Lmk>9$o5fy)nyDU_sSgw2FLqU3HcE?WC)*s8CM@i9Zu7#L4B53Is9M_8>$WJVj+L znZry<#W8q+?6-rg&E&O`PV`+>m}xJmtjUZg&Sb`N0l1-nm_)<8^AA+T4!C(e>)@LnadtjuI*;MZJycq26j ztf9{JIkk3n`*}?HB}KI0tLFmwJR5rIa_2-z(w`54@hcU)h8wz- zd|W(LwT53zk_t0~h2*{HO&_J}c`E>fX3nPh;_o$}B$9HcBF|1TK+tU%Lw$1eUdKxt z2J0~nl6yFCU(U`OlS+sPn(Li5`EiCN+qEk&vi7Mf88>h`?b+Et7mb~wICR;^?>hB2 zwie5?gSW)$0}b`dFvBFNcZJnkTmW2-H|Rv+_?G98?(Xh$4!}Lw?pz@g2+6ul_U;Dt zY}-T5t8R8R4RR&=LfaJZEC;S5`KDgIc2G8+Y_!4eNYX1P_vw5q33{gst$_@F0^%~% z3CgF`EDM9Rq0-*SOT&Lxqc6Z{glSaGj!3D-*N!~qShHV{BRq=H)jQVnp^7zn?1xY)p z!Hz04hLJn$U$<~QVddPj23`;KrGYiKeC$vJ3Z~Rcck{YFUZ*9?{v5<}eG{~T``$SE z8TXZ7V(JP0(c)y3l4J2zpehcwv|;PHU7l20v?cIy1iEIa2`O-R3J#kS87pnBsSS{S zP)8Vf+0j7}p0r1omWT8OIC6FzowS%i)6xN}EQR+Liu6vc>KMMxAuo+nRRBbFS?)<(Z9~vy#+i&2ihU;ISnd zn>>tkd{k4DJCZMT$G2YPIvs-lU@-$;><4aOl{(ta_Y`Ohv`Cq85GKR~r8gE-B3mf@scJK}>1=bp9 zD&Ik>@3XOTTF8)uOExxllwoG+hiym^R9J0WsydyhZGHl!>|EP>j;jFF8;+>MR6)Rt zkcnPR;5zbKB{@KS*cB4LV1WXpqqS8!urzAo=mB?{0<1$MxLLg!N`pxZ64N=EY>!5# zN``4bW=z_LtcIj4oPJwQI*U{bzt+YVQ*dgeLcZ)bgIk#Yjy65$f5D@uXON9(W?C~~ zwe4AH#seiiZd(T+Ln%fna4~q-I>}edo{>1eb1c>?iqRv?t$qYKYNaw}cSF17nhVPp zR4#r~^pJhaq^k^!1?S|0UA|}IT@pU+!2xOY!Ct||yWmM7S^75c#Zyjxntu7-GRZAcz}PvN?5W=xN+h{ zo4cBZLX!)C12R0YOM@PSg`{mg&VWQX`ohpx!l;Rd-p^)Rc(zu;Mhq=UCPzL1&DHd% zBq8&6DtX3ed6H_##)%UM(9;$s$%(SdX#=oQ5+6^9iHF*O?)LwuX|*%&NA`IY)qk?; z1oRfeJ7^X{LFwF)m+U&d_&cv?=Cc~nb4(wgFhg0=YaN_pgdxl0RNY`3SC_d8GkOrC zPAmYOuV3;c#$Cr@1ovOj000vVBeGaRZhPfucD0!q+#8qf1pDbT8M?z*UJkEohang; zbo-);qitEDt|B%gc0zj02Dk$%BM>r)b{y}Wqtc3~e&^E_f>(y#{fJ%2KO=PB4aMAL zJi}8m^zs*(yIK+`Q#4mBrX0YN!!>mPEA`f0R>hNTyE_!Dxz?8Yz2%9C5v4a&!UuXI z9q9_j5sWs`-mt!eyzUu{#`F*VCj213`k%f2(Req1{gHiDr;^P6mLOXq@8swhU`lym zIp9>>1>XtZ|Ni$=gjAbqD3E}U_TY3Dm2V>j(=khmyPO)?F-FO?+sEOX`6aoWa z&||3(f%1kFJJo=%YN|#JrzH&gvNy4sR?oLmZDW0?c+(AkGIy9#IxIZmpvqAoD=Nr`ju*_*itZp~(Xb16WSh07w6t zfXezg`c2nmc5p%kXbmsSv>f$(Kr7V-axhw2kfn@OQd6_b-UINpGcu1>{RrgYvuuRW zzm>bnLMH$fic9E26n3y^4>{eA_9P4uf)UB2Byf~R^6?|TwE)ZA?KZv}zV)qd9sKn1ww3<{wgZj0x%R=gg8(KxQ+H5i{}C= zGI4Jli!;=jZz_i~>CTEJrRA0^$$e$>BGvV@FUC44BzNk`xdoTi07Vd;TV<1%1=?ZV zD#x*dBL=|UPRG%gA%#~garU9nS}%5Ww~v(B2``b#2`VpV?Nl_mp`(EvC+AbQ<;E^( zZ))$u9v2km?BynuWH)!00nkR&z+Cf(w-hCF%Hysh-|4mh7OQaRzEVqTjuVyC>i-^~ z$ZnDqlVlVDL=82zg{61H^J) z6ZB`9K_D();k=lZZ!i#Q-MdgVtVw5Jdm%__ka2JhTrBC3SZI~C)Qv-)aH_1>daJiG zqa_#&H26)|B!teo1nZDUY`CV{?DD^fs0HCJgd^u*M}o@TER=p*>6ykPAa)LFB<$ z7{$DAS}uPZzMu3q(P3>VJFpWi_7qF#saeOj?XXiy0FcqB<#__~w$aQm5bta(tVZ)= zGMmU;yaxRn-_U?HRXgWRp6>lpcJ|xHXl=qxSl-)YNa#{CUR7zgutL;~Iir3nXKefN$^b!0aYNX2X2E2#`wcSljXVnGNwsvgytd3Axv{l#_ zMte6<((Ppc*JOcrKCobq=!xp2;>6d^X}C{#%wLl)_G?LuRXuJ2 z=mf4hg!(D{JesAm-TMjY%Z&ar>t28$h=k6TW}{XE-lYF+c>QVKIuJ!_z;|MAMS>}i zb2y9N(b~S?WhPrYY)0??>Dx;Nu9`bxogA19Fz47Mx1hGmezFX#$aO%V*i=UEiK;`g zS8}Dcwo(SmQ|x#eUo&g^5yO+DK!9IKNrj~jZ6Er)+;O5B#a?1MXq`=EQ)Cv2n^$_tr^{H3&v)6wK_CunYfF*Y2KudC* zeokJl6^Ad9u=PlLr6iajKnGNb9nPw*s7DD{qkz$srScT5-=V(TH?lgR84Q&(E?1Db z%Xda_gX>kcIl<{CL|M1Yn3VHOoKXYC0W*rbDl*!!J*n00ENy(W%_srLX*?``2u~(p zN10`~h#166G&&84Yx10IcG*!Dpp*dUmV1zaPMuqzlo-Bd(zO7jMmD_p&{LL8Iv}1V zDxjyl6b#2sX<$qCU;!L|8O|dM2)8Cr_yb-@)J0gKyw+oX{n`gtH`U9LvuA<-1#}p0 z9&&hj>K&z?aN0gmadPLn>>5hVU=&HoV@mQ(UIe`UG@QCP`?xq#UJayA;{sTZ*h#vp z`~$W_6YFeDjVIl#R-sDDcLO??B_?3a4AJEaYK5gvYsnrV3DgKxP-XGWaF1n4n2ePx z4>+L`*=KDa+jjHqM)_o9PWKcrC1EIJjUZ~rY-kLML*l>suJ8 z5CC&9P_Ua24@*;ES6nG3$+rIj=F`2?RdDT`Z<|^Q9a{ae$kU>|%K2*Bwk!iNU>Ftv zXAZw7BhDk)L62o?7a!RyNz=u5Tro*p>BdZ?LB|%o#0~?7Ma31ZaR(<5i5WL*!88^) z+vU_HZjyUToE0p-5;={!`EVV`M27zQN3Z`(fbSkofMAKQc6YJc79Ep8)x?kN;xv@*WY;1|y~4`Uxi!g79N5{Y zx*mcnF*gQbikVWS^w;Yn6H2bYnx`u|=?sDU!6Xhc5{A*e*V!IM!Z)hDgS-LxO}lf9 zp(Y7%X)`+=cO+_17IO{@COafB{V79lX45Jz0T5@4Nw5XrloNvPet<#X>;DCi>VLj| zjz^~LZq99>WQ(z)z$%dQsy1OwUa&UH# z3TWD{lXp_tNqN!R1bK*WXRh6D6cWskLan_^TSMeWGB62{*7Dhv(o6(a3d*@m762`4v(qZ}FjXZd0LRfk4Gp%lzmN_+s>;fS*wGvMuX(BKYn;)@o&7_dbG(b+!$(s+N zLH;&;$6nzco;<(l;jG48FW=&Zqf$%PXLWFNdMrRKNM{fPoNEEjpsm#ZF8PoA=g+|Q z&03|v-l9YpFshX&KP4K%9+@R4t=>g`w6kM>W!_D+?&>fnblY}j>uvaeftkh-(JiA5 zZNTDbJ9{v~y-P}cLdwD8miM=2qYA=GKToerse~56H~~dA@1{WL&&kUh{h*N3GHZJw zTW{HAokQCpFV3bO-B6=P-K|R`_Z))&cUDI+05l(KTrB7VltJE`s-1be4AXj$YkIjT*HF^W;Dx|hq*p%Wl!)r*{x+OkgWi5c zTYC8c`i6mV5AMLcopgLaMbVRB<&#s&_K6&g^1$*epcn2f2-@JO)<6e$FIziW<|_Ag zx=fB#ER$3wPsI(o8$O!_*lVjL_rSTpEA8A~Kq;P%D@6{Cex>#h=md))wyVH(V0k4Y zBU4?|w#`nYe^)H*WWR7&3Olf@@dEV(!+*p1VN;N`g?G#jFSR^F2@)^4BLi4zZqT&A z^$t!>QJvzEyW=HGh{~2jT}(X+h_0lI{2<&_mYqcYI`i((^}S1#3beEo0+Io`Cpug= zZNyVReQM)Z%C6fN9Q-|Ucd!d?5Vx^Mt*3y+!h$j4HsM$&Up4Cy$n&Vpg_szeWN)Fl z=i~DA{{U;?e_Yhi5*XU6n%tE3^siyO!d`dwf66vcJY+i%VuSN`>MPBkN({KO$ZD2H zVUD4KD_b;PIpC$$@=mH!4m#5g!o8wOClqFsYGbfhvtj_X>^Jl006Wq;l#+o(?e|-^ z91MBMzWW`vFQgAZ5nx;11+p*by7KjhZ-06HNtU(xSL+IW{r`p6UtJD4nyj#boi}yT zytSxh)pSKSr-aH=RD>(=kMbxi!RyAOmnuFx z|2ddY7eGS>N@zUGZPhY#y|mcj0IO`&$d-)F;OA>%UQ#U;ZISya#w^rZ=u{k*C4kgh zl>e(jK~Ah3vGy2FR?tHSfjji_*&@9HfE%)qLWdNEpDdM5OCs1cg`%`I0DC`S5+qq5 z!81eZ0B!EHV097*$kIj6nQ))k{qYL*TvUf=pTM|pFp%lCj52@;(LMs~vPoRGUJ@+R zmnw5}#`huxda#ySt=gKyNM%Z;s|qinZOb~SP;zv-gTo~qlAtqv0rONgz1m4d79G*D zOb|g)g-j|(C{ci;g@HOODANlHEvizZeMF(Q9(it8kedQ)vy_FsYpni`(?gms!Mxkg zeWqkga!DfL4Dv;0W;hk6iIn!l`bZ_ly>>61SL&Q2i*JPbJ zkw%0vv(Npv)$vB^yoJVsBAKwA;-NfyO$xEb8r7L?F^@P;=w=Go9;hZq*#QDzTAA9gdmAb?poK>Y$ zj@Bm^U4&zK4>jUJ%`(8nn`bGMYOG=E!NmTqU?O8DPg3yVKhDIr+YKrWh=%lpmZqge z+R#(DIC#-26bzvj4@)rkxiUh-|CSC2DqqDG!ZkWJQW6f3wD0ydq>xR|>sqU*4DcZw z*uneO8gI^wR+GiI-$OJKu=S+w12jZVR=O8b6~JuG2GTeZrNp{{HFGFpy>3rfy;!Uh z-~s8vFmhm$00Qjlc5;y4u>&JHQgo^#LUm$H4a!<|Bx=?E%~$qNmuAat!xMeDT}EiE z<#{|=nChp@KZ<&qY7R~DIQk%boNTKl6|+)rq1+4{$1l)4ctT&=j?ESsZ;FAcuhmTQ zY-^(-KER{!3b5Tz8n>7ccg3D{$|Mby2slp0I8sg>LWMob7P5x#CX%1{MDntB?8&UE zO+o7pHOE#vAeUqASYU-nNE1dP$l*Z+wstO*-6GNJd!@)E4vU9LGR};LyX`E<6AG4( zt(iy~D<|gx;teJz*^y8Z$=Lj};2SKbkw)nkEW;XH_4@=XREg+`;%xL|H|MF;Yk}5I zm2497{!)Qujh2@CB_$*8kPicerjYJ4`-k)L=fFIu17YPztqG1+ZQ{?MC^0DVbJ_3S zi9-*GV;;z%%Nw%(V0wg7u3V9MTy6^wNhAZ5b4xfonokDy7{O&;T|;J7i8$!N;ZS{0 z!Kw-UVUrY$eV01-Xd&m)kjU4CTR!03M%eU$2!`%E$XK3cX%J0i?-BD*DTvv_+ujj! zZB9)<3if2=`7%IZVM#Ay*-v)HT~GCLjC>?gUhNUeJ6RlEUdXOmYYyWut;f|Q1;hkF;Fw}NIi}t-GmA4W=5`i0VaXP6r#K$xGHg(crNEiDJb+(f+ z>g$J>2SNC1?rSe_i6Ab{{tB_IvVRlEY{bS6F4{LxOS|`m82q)ndd-HMZOWCY*rnoY zq=Lx<+9X|o>6402Lh-qNmhU2fQlTmnyayfm{Dii%uNolBvSSp?2}8B}4Im)f$a!$% z;xb;bzzyeSmhWHaDsOvskSeF%2LpOE zQyJ8R`|@l)zpn;b>U33ZyHi6ht`rznvr%8tZY(_TNa)n4NEL{NB*K?%4FFB#6I2i) zrg+SgOIOl2m5QdJ@BE;>4g31QFVvXiW^$m1(W ztDanItV9d&qOLpa5Xtf1TKy@hbv72Rg8ajEO?32GP89ooMa_|lO?}HJNi0tlDe+%` zUtJ}RqUGXdJQ4h%W3V!HZ!8hpn7*!$&@DYDIZFo@mffdyR1Lxf<`()Wxf!GaOj2X- zCE;}mDhb=2SAgaDD1ZSieY?K}Y=57 zSfIE}`8KHmWig!TJfDLMN1nphpQ2y(+1I~NAD_2>vCjN|Osh~ied@PYuWodW`Cu61 zPKe>C&SIO2%K$aM0fcK}M_|=w%PsryTjAT^(;p6YPnTWQKP%e`sM(0QD^YXSTPli6 z6BvASE5=&yvtX4re7;P*1iBlGD|ARD<$G_<$~CmQSY89~M+bUWd$zbLk|K6>IXW0a z5KkMy#imFOM$(JxeaQnb^&!b^4me4-3q<8z(mRQ*y#6fkhjgW-YqqFK0$VTxj$$`V z!0}1+Q$i?|I*q5U{Ex!xS5SKlkyu8=1{~;pK7@yES+sn%jUfp|yq{4wB?JXF>&zMA z37Yp7MeZIEpb?DGwp>HHcsb50=AE*l{uthVsXyP9nn3t*@OCPgiL?tw;B84KlB(&8 z3mO$8KkHAg|D0RU{+sj9X>Cvw@)j|lx8|!JyPX@Q=!kUj8R`u^(p$8)k`x(FDslrg zO%$&=11+X!Y%1q=JRJ6d?C~HmKq;8iNEwvUTjx@gZM}T}RO+NXJE=8qjlRB;7ojC? zjGtf<&@)&+kc^lZ7{*MS1@iw^vmN*)e+}RK1B5;k|BxL@qaZMyrsO}CK!dVm>f!PN z7Y`@CB*`j<;jx2QqdNwILTmx+2R{fu_`whO>LkP@8CD4=_9T-xw_!ddZLl%6Yh_}N zzYmJF&G|FXR2v3GCK!=zuqBJXtC1vUkSnC(x2y&*OSMW>`7yZ0Gg?S2R2t=r$kKQ+ za6^-bRa9(RuG%-F+|V%XS9SHW60}Rlu6-N05OS?!8HqiG%o{$LF%y+o|8jy8W%_c`WbcEk6PQzc*70nJ|RaG$iV}^j-gCyUM>u2w3xU-%YCx| zaEp#(YvbB(>cc3h6%E|y`5k&(j!ilXNOUP28M{5KCQ#5Di1Fb_=OYLrTvWutLC?~1 zlXi(!#de({V3D2!Rc$ld*7bw;1r4zBNVpX0PmS;fHBvwW2hmLYt&Ev>nm+6VCG5`jOIm@0i0tcbuqf(X5*{>HuukA zs%N>_u}c(K`*^2+9k$vWB~Q3)2jciRB|MVx#Q~!n=^GfUL5m8`K#$<$Ml_b|i>kS- zUO`HLJx+a=--g$ZE{EJfi1HLRUI5IvUD81=!cF;*RS@Vb3Fy@C*l(fpciiitdr@

Rp5@H-63rPkl?5M!`%e$q5M^(XmdeOwiW zW|vmGE5kKAE-6r%tM7^P*A;VW?H%bOf752qV_m&Nekhx1rnkn;8dzEqK*B_d-#M6ZDSLGYoS|B_wYx z;8qDgjM$9w=+*E(m<;?*#~}F*HokA6GJ8#P8hQJbp5HLNQa82*-V-YJ31gLJ9|0%h zRiOII(ra?Kp_vLeiR#UI!YdU&>FNQaLbj(RL|l@xzfarcd)=DcitVn^s?da)fpv9D z-kT*9%k-#H(Dv>L3yRd5v$kw7>_9~)zV|gDP0ltw5sG5kHk8G767T3$bJ&jBjYjhOVX_haPixS{t7!8AbP@HJL;7 zHp~Zm_Z;ZTn{25kI=$3X%^VjXT8-z7IcegUdyxVmi-~GyK2PeD1`xu zANiQ=scEboc{_|N(-Q{a8U20T)~(f&*M;PXT2=}bnNdP)seRanvOg#sK}TH5+5oma zV}p2h2=0t6&`qN1gGzI1XnyDv32g-H#iM^)XUZ}rAh7cV2OrK9NTex}1o_U;pxTv~ zX(8oN!&8{yIEjfMJi~qmU5FN78#~D)GkTJfcVsIjoh@Klp>+JY=>mQIiTugyzepYn zs<_;hEUia!wufxF1Pop*xcCiItw+o-b+Diucv{aFop7yiP9muS*#3sAv+}tUrWRLK zAAk{dy2wvKhx{>Uybybd3I$l92n!*_v)aQ5j)LZ%pd)o0NOTnIs7u;1NYaBBmOJdK z`Vm#pdZ6@X(@6o;#>#3U3=pkKg8+k^P>^)42!%xE3t5e{=DJfYIvgR7C*=k8a=fzL zcgq6RkFh>DO_S5evFG@jXM1Q>hh=MV>$=(bg8^F7wL|fDl3syShE8~4EP(1~h)*{nIa?-oZ!oLHcn;dteez#>mamw2iVcu7F zjYA^L{wD@`Ta@pRO?yBLXJJWK=I01jLRr5TZu^{2pVJ?t!8%7@h$cRT{U zam{L;$u823CJ}1q*%hLHB0J*`;pOFcq8pZ5Q3zc7oC*Y2A0r%tDk{WrrxRi z&)MkK>H3(B$FGt*Dg46TK0GyV(D9AsvRgk^UhLcgIR}F{Lk3FG`MY2`&kxm}O8>H3 zJIpkfhs)GKf~70v_^Ruyt-4Hy8Urj)F|lEYJa2jC5K*_95M4H3CP-hGthQt{juU?X z(!2chu1bA+xL~47%9E&A}X7G(Zc`0J100`ih;joyJdE%as0)*NPP5(T>q z_>iQ>M(S{?@%lu{Gk`qR7S@VeNbz#{R}R63Pg(n*ke9xhP3gIOc7q8RZnJ(V0H$}z4Zph8FW zUu$-B8{~YkC%9OSnHQq-z=RQq%lTvD5rOU7Yx z#CK3))3?_+c_2jwqb(@1cMhu?>g14`-MVj>&C9{63f_fp89i{(X;+;j7H=uZMSV*2 ziR759OctG4pouip8BeQGNzUgHlfcs{qq>xBB*HrJ-hD4`yA(Bb_C8m)^Qp>;p=xhv zOf|{!$@E9zzh36vp7NxMI+JVcHBiXXkvu1+$lk7v4Z=GgLTs+5D$6Sno_zrAcMFUR zHy5p|4_QKe%(9M?`}jfpVSs2MtswMma<<*6?UauFj~bH&y{&FcM%|=2sUDLzGmLqnMtVw3Dq0=gsb&`o>=(y=xewJ;4od-D85QjI-U$sgp;F!WO7vF_0is9k_CfWyU{x{UW?r(Kx6l(&`Fu13Va5S?!}g6-j8YeAAsE znskD(i32vmENDX~_Zmr^84-<2WS8fPy^?GBP@GnfG; zfeSjNQqGiRo}Rzg_(N~|bWxAd=)gn*qBuxlf5Y^eRgm#DY_jZjGwf{CYR3|=k( zg7yqSxr>8txlLn}VYRoQ!5ah4GyMiWne-x&*Kl&Vgw{J@Sn`@HfXdZDuEhM@35~^m zy-Ga|J!wd8+e;jw-$yO*4er|Ypt#9hO-|&9QgJH2JrLk_!v{@yovOf=}!c8j=$zU`%fffS9ijzJq+BEVX zc?uU5gEtl_`*9cv&n>D|P^|2vNC+P@ppg`DyugdhAv#hAu=_FVnB5p#hlNpjJ&J@> zmWG{i)56hm_8=?(@4{)cp{8sfDly%u2 z5&ByaK#j5YJR^=%2+GxX?{H2~uvL0vtt*t&r%?}jfp@$0J4jHM*){Kay*FEmm=h+m zLh>sii(~c8*Pp(ALeT+G*Knqsz%v-p*CZb{nCfhkoU@Y0VSu(#_7!3qKCj&5^}kp& z9FNpd4nw>6YqHZ5GO2ODAh#?tH4_Ita_&~5lD{2^{E#MG7Gj9M{w0*d)?@UMha9S? zN(%xDRcq7$Eq@>WJ_(T?5*IAtLN~i+?-b!@cOoZ-c$V8^0*2ABlT9G64U&sM^=|bf-TUj~5;%M`(7aB12 zaiP*=!WOjQzNkGP(0mU!6zY1&Z37HSAld3B4X{pKTW}%zTG2W3iqXw7@eAZZ5+48h zPbyi7dRCTJc~dcQd2gn04-Ewt*C>&L>JQkOkgDb-Z=2r&`3&&*ZooAqwF3W|K59}D zQ>D31)!n=F1I-)SINrdcxK6A`w7Y{B!)vHjjTn{!>KExU zU%D(_y(PbhHECN>gqi*p=+ zBWZa)vzJOv!J9=%@3O|_#GcYN#*Q)z{TI&(WO=S>QUV!5Z$JPYrA*JzWRK=uMFEzT zPYd3xa@`9hJ$Dwx)9JNxoWy3iUmj%;Dfv+Vf~`wKwB5xDT0 zr!;^K2x93QkOo2z6!!=}NqZVYClN+R377~ZwRroQ=A9@FG6ue& zx>7aC`l1B{9Kwm{kEXE>cI;%EzP_G{CqmNTi<-9ahi`v*``FQV{&P70CheouD%W8_ zNK{ceB}K%Si)6ke3~brOsM5S^@+g3E4Pk*g6kl3v80gF@#Z9ZJD1eVepEPWkD*D6X_+3 z48$&UDk7&-2J1C5al9#b%sWCYTwaZ91QW|reM<@eDhld&M@WSy$4r~|)>^!U+&A0O zDn+zZ4eb#X5l0?*^w7pf?35A*Z@zRQV3b?rv*EMw_6brSuynxX9}X4UaO*(;hdBPm z-XjqiJ;d~Rs~We%zIrSntWkTrRY^}XnF&xz!9xDMoJZ-G#E|wu{i>rpMRKXMtYeEB z@V<<$1TFi!WNV6oQ39Qlll$6pFnKo6(5M{`ti#T*^H5tAJNO`909~J^hScFqw6ZEV zxVhn?oMC$=*cYHIqz9y496GQ}TtJ%4>okVI(BHiNYr3v*wV@Zx(D>yKIr%X{!Z4su zYv4(C-#EHF6YN-&fGTBK6BuD-UB>iM8^|(KiSx<^eBn(hZC}$Ca_Ew4Af}@W+b1<1 z$1_(@2UZu4S+pPUcyPwmrv^bW$kwti>dN+VvY3JoJ&Y%|_$)Olb0UbnFoD%}`|04m zcC`_*Z$w3_T#>a|A(BM*i<-6sYQvH)0B+^5#8hf{U=0l^5^p^wNm4|Gr!ZJ6d;wzB zvM55zsSEat^yJ5o3BAgQBP{{;W(gSTxKLQezO>9TTU^SM>;VUT!WMO_Idp_Hi8H*_ zm&953Hb4I;B$7a~n1GImwczfk*v1{}iO=?m9Ks^E*UvY(UL#Bb;W$9$bm-Uzme1Ui zfVtMFdwA&>ysAqN-R{Mls=GCha z8D)(pjZcA59&FufObTx~!_uCuAIXk~*JQ4CDqQo4I_4 z)~)D!GzMW(h=d{}0>H)@YYZMlx_VIRqO)^xz7BMx==VyiX~RC;{JlH{Lx90wES8Rv zSAFpUlaI7>&Kx46rT)}hDqerb2&)t;BUT|UOoPQ2Xzc)x#gxvd;^0Sis&fQEfC$vCPpq2y1Ym?Mhy4W1m}S5fmD&@r zq|$~fzi5jmAU?z(d!mb1r;5mFZvHY2>UD6VE$>w&)LI9FH;Rs+>m=`nSlGw`^{Nz8 zj=ar5IU~uNZjp**?@rag&gM$0DBu1P_(LiPz6gI~H*xe?&}==+P|R>IOzTCv{r9dm zlefe2UHH+t4d!Zjhr)voP{?bz3P5hylYKCwYMnMvkYZ6!05%AKt@lQ26Sl&BU_a}M z@N~Hn=wGS{@`Eo(cB53(wo+R4;6|9{viykyMDnt%}^P~aN?k{UY3(urAA$jbM7-z0VrnEW0TZ>M9w<9 z2KgVt>o4qCht}{2Kuq44K1n!Mc|VITA*P?CJWoHgz?M<_NSAi)GQoYKi{b)gqG6JBt0iIlF@-1G|I zu!X=nngU6@qcABNXot0FVQ0OB@9Dy0nN^mnLnxYjI;R6*7;NT9Q6BIYc;7Xi?2&ECSG|6AMizlYZ^;3wUoUz!G`DO z%6=%B%NiY(F1Pb4Bsq2syZn!CaoeKfTg#lG`bpyXBUdYB;kXGuOUtCrAR-Xvg3R#fvI4SOw0OQGehVUDte0xa8Z6xubVbqzU zsl;dbiyfu!uCYTEgl$DTefXA-Vn{v6@`ifhsH_Z02VPfS2hwoTaA!*016J^cN84KML|I>fLR1p3=t`1D$zBcspg>NH(m05FK{iz%1knL1Yf~ z_whiAlx4Fk)$v`3H-;Sci#$7Ugt8s2>96! z8Y>h(MmXQMh>m~>u*w%1e$mXEU87h^adsnByA-f;jD+643a_6ggJ#$(+JOQPDxXR( z1+BeA2@wOK4BOnV@cR28X|2Fx6%bJ)k$7_ncMNT;i5u0Z zgndecCi%Xrll~^?i(^PkD|=}DvYjQNe^yDL%Q4y6*+{X6y?m#5ZW-1K2E!C`?p&PP zCHT+p?azEaHesui)8&}SFFKdu+a9~{0n4ay@`?$eLF>*M6SbmFY3;5l;z|@oe}yCi zh&5UH*dWUFGXy$Lur)Xo03wN&{{?E-k_^jw{6YBozZxy}013q&nsPXEz5*KA%CJ?C zU?NpdpNL^$RNN^^&v7(ei}OPZga~Y~_w| z4ViJ42mJdg?`$CpT)FdZ(MRBk-=WRVo7E##H)!J;7WM9@0bgObpP!Sw_+%;QmbABA zcs>t-mM$MSDs)HYz)khbE+87wR}SUN@{ax%Uem_d$6BREY~@y2y=8%E4t4mX=2YOt zEX#Rgtpo~%=LNXSiH|*GRUR-QFrO(cC4H!&!l(qVX-11ZV?U=`31>}*>_34-n?5Cm z)k-dTFBIS_7Ufp6KPsii8FZmH`ukumYS<(3#%PPq$g($1THPOxfgfQa~DlSZz3+S@P<=fe;_($@u7Ud9*6$NmYc&pinO$g&Gdo_lqU2?~5mGw#T$P-bT zm~F+2>Mt$w1Owj-$e9G2>$}+;v=q70)Hi^c)Ltlx=PhmJN&a^n+bV) z1w-=w@SUt}BcDoX7H$F+=?l@EmvadD!XqKz0a~*gMdC7Y)>J{?Zt<+KoF<2KMl6N2 z;Im5!TB;cO^B=TmS#e|{2~>9{x}VVe2XZC0K%}OWEMkTOV&O&zgT$!TDMjc8F32=w zrQP#1W{!J8NLgevozx_O*SqiP%^^&XzuY`;1FQkHGDZ%!OT>UHcWQru_4Nf5-hLY1 zet7|MdcUlDj|A8~$Q9psg>J2T9*qE!Ow>jKMwHhJGbhZIdvl&nCyv2vXR>p$ZPVt7F13JS>$x2$2YugA#t>M@rpu+5y z^GhPd!c%;PiamrAFmtl(aYW8w+-gcpEPyG6LVMy=m*{%bEunF3#!_A1YUHJV2(QVf zebI>n!B@8al71%uA|qZ15;A)qVRo+~DWO*YA%TcQB3An&ze^;r-$)>^*}X!AFLN6g zG_)D8l|m1UHiD=`Ad4(R_b}Yb1DIGF14i)NX=0%cZ5A>iKcRB zba6mkPfKM9iTl0Qw!Ac(wI3ywunCq@#%+FHgF#%n&~>4O`+3^)6`KF{1Ak3VcJHt2 zrY_srBoPdGr!)fT!K%;|Y~WNB)Mb&1Ts4t7Lm_`_K*QRl7}PSldeT#$+Sf>AkjF@B zv=JqQe(;%0f&&t7eNX_1R~I!T5&^hHy3=LxK&bqu59*K9y?a;mM>T(v^W$`U$W>DI z8Ki@ zB&u2McB}G$tiMn@!w;1QCO?+j$6W{TQXRvJ zg&P7iYC>$}-9ryas#}y+X}j6022$AP>V2*1wG(x!*hfvCWSVa@uIC*64M-jI9P}j+ zvjQbW?RxvPtxXtr0-3ZOXh|h@wQw zl-6Wol~Arip=E~4Pyv`SGbba3`@V_9`K1MF*wtDNh5||tzu~B9pfgX7(CedGKhtY< zIkjC-ILr=$>3WCWs8P-ql7xvVn0&Ci$p}0>0F_zFaM3ral zx2$pew8fr>rl4wy^ewAxpcqs@L=K!W^e;?w%-chHa&4nV!9s0 zWyvZjIUnw_bR|KN6ne+YLBq-0ITFJRiMA0CgjW4V)+1y$a{K$v)|UEsuu&sH=Gjx@ z3Od4z+Kh9O^!2B2e+jRtp`3hH4tb^bx?%V9(6*2L1~eRdV@JqBKoA@T62qWVYpMDr zG<0=?HtT9hmP_U;qPSJLFh1k*AESd`(WN!XJ4ssY86bgI-o^rm=Z4Yt;076chb$X1 z(@2;yYH|YR&%Xi+`hlqv;75c)YcJ_gY|gDd6zgZPa2nkx5hnL3zrnfzmIX(xQfZQfhf^o(} z)+w>0w?%5x?H~vYO$ABmFgUlBgsG?ekyh$2@G=GNzS$wWlHQ_kxejPEYm>r&oBo{` z7pHIj7$xc>!Vc7>b$5wLz#3Lz9ov)ms|g8#jUut>t^!U=GrsN zjwMnfK`+Km?S9G?Tp>@0TIL5aWD?;KUgVgr*n3)1!?6YueYX$`w4!$|f$@rtNhl{Y zhfnI@TC*~CDQslHWc6ukWgV6!fgn{RFMdb9XdJL=YPbSxH3DJ#;09$5V@=`&vf-)9 zjBhnNv`>l#f`YI-*R%=;uuV5hi3x4oZTtsw6*h|~qQ}PHa*{ZmRq6v~K*es&ABi(B z>LhGCXu=+6>aj+=?#``VR@egI=!;gUK6RuNq`IE>`FG(T{}H{q#$iROk#8S6!KRV) z1z8umKiu#G+RB#splU7mt0^g6udd8h@2aK^62|PEroRiRSNoXYoI^7lB)Fbku61;e z=mkI#v)bNKDH{Bc(z<<(tXX~6Y^ZDG!mT^W%If?LlaXgt7eHCN*wT1`D58R9Ot9!0 zT!~&n85JK;JEqrj4O80hk2XxV%M3R(UylQEflr~d(;b<|%%DG_b$ThoVEyl^ItzG2 z6`#h?vW0uNIwz+O@os7nC@5mOtk&s~Y$K!2)&X7(bY2=Etuuf_m&ED}!kx?nP!3FY zxI52|-Ovz~Gch=21R|TkakC+LjUbav>m&#rq(f6}W+TL-z~IGivrzb!Yu)Hr9;!!a zm$g1rgx-O?{(VvU9y7d+Cdf+oO6_~N7di&@rrdDOle!GVXr>8|rIduiC27)Dtu^(m61FJc*QP- zeZpKLy-%vjiYB)@Lf+K%M9+GVYq8+*V62GhAECA300K?ZjqY24q7#zYDRUfx{yFkX zaVbmLr%kfNZ2< z=sea`1Odadr-FE8jbU&-nP~6T3Hp^pXR=_n9Q*x_ho-t?cIzrx;)&{?;f@pDK7RYT zjv{cPc76|#f0vUn?I0}sXIm4fj!>mrIRbdOZ`yLgN?@T?Xa~%Nd!v3y4o>gdjS1Pv zQwwH^EMjlSzb%=Bkz-t}^^XpiO@gp6(*sB$OCC}~RG-%09AcKTQx*y^6bfTVpFA3{ z8?GUfq?l-XZ$Y-byw|$dM$u13OLZ7Q3SjcDr$GL@x8J)XNxCbLwomM&rEWa_%twf% z4%RL83AJB3oNX~w+qZv}eg`mq&Pzi2oQ70>sY3huU8M|#F3JcbKjvAMO zy}TtJH%mJk9mK;MqD!Z!SE)Kw4M!(T-WI;rYCCD?XH=b{sbk>;55rZCEWlW6T4bnU z)aaWtWH{MCzPJLWYiIyxyc*nH-n;-OLgclUVifoK904OVWV;KB{MSM4+|4#b>crREJxKl(%?Rg>gI=E`-5`Oji zm)Ads^h;v6y!{qKu%Cq2KPC$cHZoL1lV|O`FYsr+z!aIwVXl)lig&hqm};i4`CqS} zhqr|GdO@0(+zcJyk!NcswMGE#n8WLl-e0~QzWr@;8phoJnAAKnJeHh(ly$E&AX8DC zRhERVxOdDEsCbiXleN~CB0yF=rOV~;n&ExJp5GNc469A|9$_Olnu>UUpTAjhb=Z3u zCT-&b8}?)@X<{@$n=(P`yM}(~Bn>lTv;5wpW?F{|CdG)rZQd6m$Y$hqR;?(ho5J3FQ8)=B1+) zRnPIq)4&w?mYy}<;x(r>x|iq(aVTHjatr!73r>Qal`9-!QOcZH+GZg$HA2OA%x2TKKY9e{`^b*MZu^!g7t_D@RLri&MwAN}Kym-}G z)P1*eT;!^V3Uk0JW(|8_*x}A&X{o46icbv6sgNA7AgIY*-3sl=ZnnFgk?z<#1Gf>h zRKpKI6{xS#pM#_*22?(u!YrBOHI+oRl`kvX+7P3a@K8)1In-QXzY$hdWN)qI(1xT=IT1FH~bN9H;Tb@BfU%H~Aqwnr6xJjVi4p&z-tYK$y1}%Mcl@OUbF|iyrM~ zW1N!0B07R;G5>p@1xwGN$e}G}xzmu2NksMiS>6n4r}W7&r=fDjL&31MGZjMD$dOcZ zXD$6!SD)(`s15aB5p(8I1gL^IJ$PQC%cI>PNcFN+VF%@XpinVhQ-$EYx4U4i!3?;Tq2>~w8N(Y;wq`zX~`Um>zs<>_$~X5vC2DeLxz zDk*F7HM=Wj!0fxM^RThal(Q>lqUq8xjX!EYft#~n^$I(N(YuN@-I^*);*fA^Eqh9O z|0PID@d)IIZMABLxjusPryGZQHVfP{vuf~53{W#qd#`<`C9YWeyFb#Zg>=Yi?|^AP zEtq#3=)e%=$t3}-1ld6Be4pp{S4fxCK$YH|Xafwt_s}qTlT&mbW|}xiW`wu^<6%zc zT>u`2n0~62%Q3&7g)57RsRTN#J|-<{RQ@BIK9A1okole86O21H2bbXj!s;rtlDfL` zcQU(7KbAyW0UINwIK{U21OO)$g$M$&z2Iq;t8$;HN+oHv-=P?6FrB2jFW+j=pMzYK0cSz@g^TeZ)Yg9Ow8c&Zho@v4+#a5VtDqjE98u=Jg}d~3Vm$k@>C#*u3@e>nvAW=X(1V6 zd)et-xoYo?K9df`I#+a{OKXMH~|2dq0!;`Pv2RlHi&YD%XVW_&xSwovCZRGP@PDX}l z*`+*51-P3FjvEYP?a&@E=VSE`yr#3pfYnuTYX8FhMv_8y=3VlfVvaZ)ON!!U-GFI^ z&=PI4Zgs5=n-bEyxy=S-7}r}!aObLZOI@2?ZA&1)?=otQ(#&Qq!ag@`>RbU0R&qicIH3v|_6RFXT`_*q=x%7JU`o*lKkpL)#Bxiuh+Y_mxU z=!GpfJ>``FUU~c$_rCrpynTjzE#A0bJ5tM_u65%Ce^WI^&j}vcKnAgB!7By8jCK&qYoB7>&UrfN5a2z*S)m0B$a|st9|}H22!?lZlrR*$MS8 zS=(!LjqP0xZ@p4$Iw-f|@=8^$u;gaLK;U7Ic&}x}U>r@h;F_y2W&M;pQPzYUsDVsJ z77R)R>M|T?k8INu%qsZjcF1Kfy+}~$f$(C{`*xf<{t&ROB9j=(@6A3XN0B?=q_L`? z2B}RlD7Fk+%*@=K0`$OfwG$vZf;mQ3U#1h?(}7gwyf~U!PKpy!qiiQnUNB5U!MelQ20M zct)NQ28RG%o!MqUrLfS8pjn;Z+n>?s9Znj*O*`TaR&@;&JjDQ{sx+*wqvoYT@AXkq zvgJj^R;{c}3e)J|@!CUt&}3DX?@LYdnHC8Uvpz(+E{AOVdmZw0ZU#QukfPM`Cn`J| z3>x5f*a|~7X#5L+&mHR{)o_)?oPqIl{BEuxL9YWEF=0#R2{SCYKZ z&he}SWd0k>sjSnYJ@l0+(&C`9>y*{Qp65DY2^^PRKQorWaZwhSf<`1}K#5^gN7#(z z{4;o4cY6hWn38A{!|5pT$%c9uZ+!`^N}NngFS5JA;cAf}J=t18-^`K^xsqkJe>Bfh z&pqeiQ4MZWx_6kJ+20!usGEMU6Bk;lrlsRGD*%%0#D%>&n~4A6KLiuF-=(s-JlErd z9*WHyVKcU#zzlnyteO{1sp>mtif2V(N0{1=4e=^3!Ar01gz*_TwgMB>Q#Z&hWO+1Y z9I$&BmtEFNAt6e>pmnnu~QTkji+LLExOOd;89d2D?` z=-!U?UZUoiNGfmY+Nt>%`d~m~aXngPNaHvBBUS#3?l%TpXQvV+XcMqEDQnjjQN59p zga(UUe!c87Wt)^ZH6~anKT|^fQ9ANTmJc3+jvijHVdh$mcjZ3ySR%t5!-otofA8-5 zxw-pNc|QSawg=#Vu(~9aSV~-|6j>_zXb<^FKxgLUc+zn=)KPgGz8Y1IeLh!~A~c^J zY(O_IjJLn&3LQ%1r6pUKaK0`>yL4JB*+^wp3oig5pbOvkI+JMQMVmG#?w~KkfoBu$ z)KF9<5(5X)5LuPw)BmsI9o17DJJ3nB#oF)ic{m_wH9-=8Jb9U-<%O*JAmgc28D9&G z(-Xo>p8?fB3qG!e)OOauLbTGn%-C?ZV6ux4{EFanPm3JH)(Vj2s21)UbT>ItEH&o& z)cEU9Uw`7}9gwZE(28FgSTjIL~-t1LIs6CRpxu52)_>ExCZOlFsj9|bhSq(sRFc78p%C3NRAnZ@}&JZ zOl~={S31(Y!X`6NitHf)J#C&FJc!K2R2%#`*+PcHsY$&l7;K&7IBb9{8nfQZt`666 zb%rD)>?941+L}_yN20EQw8etuO1oa!g`VBkLEe(m@7*6vw$3_3y4D4dFmsA)yiqW+ z>S~x0ZI8~;D&~u7h#!NY041)b);>+%(6LYBqPsB!7QmWyM;jKe>HhCAgSK%-TAywK|k_*vFH+3DD>_@I@qK zOQenR{|6WwMuiUEa`Oef_FWEw8(iVi`bG+8cL9(UA%sfphi!(fS)BA7A=Q~-w{NO5 zq`LOqDUZ^wkgFya{VsXztqUFfP#5QYVf)=rE39wi z7?7cpnYBV`jtcx6n+Xx0+6}Sie8w;Gd*R>z@U)P^+y773yERFYTxVkM`70cx*@~b< z@Ex)h^*`2Dw77UiMrGVOE}0c66OACaZ!#J8ZMkol3kJXp1~a&0FaXS|YW|nbcl>ega1rAV!BU+Dknr`hD- ze^wg~Yn(oZ=Vx;?n+`TPLzt-kj~Bw(hEU=`+x3T)yE~+X9HMURn>KT7Xn7C7+v7K_BJ=Wl;f@ zS8Q|iel7w`wvWU7GdA)v5atV}Ezm9*J#uOL-OgwQ4;F0k*SJcI|1hYoAZuZ3et>k7 z)Kyk9-r5+R*b1MotT&Pk@~PlV=1~JdVE3=$EEzl7WRDtmHo#b({KAV|i zfI-Kp*OT&FveTy`rOt;T_C>8oEHrOb9$&z~%V5E>4|E6HLARmOKsV-aQd@5Izx$FQ%EJSt#P$y}tZLNR$r|?hzWcNcy zLpwwvo{H%-e%pXcB@sD!nRaI>7jWv8+xBFKY`Dm;bVaglI@41J0l!C|n~JdArcQ@u zxE1O-{js)FKz>gM{#p%u(%3JTlAnfeet>xi8taZtD=AfcNuQf14^z`Da)A1Jq$6HQ zGtPP5{mv#Xx}BdLe6Z5D6>u60^QK3Z{<{OOYh8!?ICq#&oHy8eqGMMkL}Zz5xlHqq zJ>VH*MDT#Dfq)!>WUkJASc)qBM|=^;Ee>I)ln9nf2ELE!eqL~NA*Hx0^)cANYM;8* z%-be#mq7IyatNcZqu!bFv>i|bi(J&X z^?kRPghwCdO*YaPQidpjMmytn(f}SW|7FLTQu9BUuNAh@v=i9#U<2KW;k_6eaANKD z){E@{OfVr^!APO@HsAt=_YH-ABd2xBuwZ#E=l)PwK*1(G@4g2l)tu-ku=L4l8rT_l z4xP@JCoonxH_YUuGQdT4dFx_qSBD0nP?T)tJw+rm1-rN+Jr1UXd?m~d$mc>5<`xDK zXO-n!**{fiHdKh&IaXC%>qz75Gm8`Go6iS-mEVVChZR*qqtTAJH~WnE9+FdcTAy-I z%r-OvFnVP|hcp(&)|vf$B*rZjLKD1gF{bDuc?S@{1Jp2dWRO-i=6+X5XhokzToF3c zt$|Ihq6Hvu&hh*P7_{}a81H z3@vEObLu$`$#knAIiwgj>YkKbAkzeMew{*5a0X4!|7^EsV zA-9TXEL5y9D_OM}{e~TqHBu}1Zc17^;Gg7}FrJ#d2QGKb&vb|sf~hEEqJlV!_dS8@ zv!kM6QZ;U&K0zn++Ol_%)m3_^!JQ$)&6Y$d3m|>>pRFlxg}|Q)xcW1{rdJ!G7%4#z zBX~n>?#K}B)%NU6Bf0uOGhQTHFJEWOh5vB;-cE^m%{kp_{4XRKSNZs`4;WTRMPC(= zDd`s`<=G%G-*?CqkewD?P75697LqwWAA`et9|)9?*IF)FSk++Ebuj7v%H^6~KYsZF z*0B4=!#?S+bECur+JaCbWl)hb%2rU#C+bbe8l_pFWI}|5_{{p{Pr*n?tCZga$biP` zXBK-0>N_NLD1lE1n3r$a?fjbUSKORk;_o7L8%FrBAU|CJ12U#&#mEDl|44xwS>;FE zLK#)x1Cnb}7%BnS?u3f(QTB1`&>6FJ-SuG@TNY3%n3I)GqV~6HyfBA)@~s_eum6sj z*uTI13`A*Q4{TF|)osOq*Dq*DM$6$Ai52@Pd7)y8H|JBpdfhdSl==w?Wp)xZ-?*^@ zaLX2Biv-Oo{tc<)x@9F<2?;wyCDVn1htR^N5+l3YiCh6BAjD16BHRxTPsE0W=2eh(LDwU7wgG-*NerEPa zP+^Cxlvm4M13u5%Dhs9xPOrp?VQcpZ)4yCm0jQQ$v?DshwkLHx{zO8##RsHn5Q|5} zk8WnK3Zb)9n|)T-k?_?jE|7zCQ`kio)-#VUvHC7h5v=D?)nrY+2;I(lA0(-^BOJ}Z zq-`Ug68Hl4{`x z@BvpzAXkdiOhOWgu-xyFLEMV!;u7pRKW*>(LIb7VI$BhUU#WfNAd$WNWVJ7pPf^1o zaj^UV0(@?JcXM>U6di+&%?_{JXQX}iPBZcohrT9 zjoN)VZ`ON}`zt|ubeUYZ5!ojH&*ryeVe*Uj$Bif4?h%QXGDv=g7ijg!Sn zuc={bC-;x=#j>nGxoshmBX#P`3pW&gPOf zqjM}tl_y#dAs-d3#k^7;ck5HgpSgwdNKc*%_o37vupKI zCL*M!)AyI~E3e$oBJcnJH**7?w$q;Usz8&) zJ#>{QWM_vYM^-dXXfQZZLCyQn-%N{EeN?<91Fe0nV3Lu-OInWjh7V@h7PLWf(tDSR zUYjP&OS&3Kz`!!UEa`EC&}*k|xXcCe(w3_$viUr)s{kOoBwi&(FqC&MarKJiv#Qo-NOBQcCH=p(jjAtbSImNyDx1&l{5i>8sU7S& zjcatm2HH!29Y(h;icDRvI+15})hianPR}{Hsl!M3Ozev!G4eS|9vx~P&dsna){?D9 zcNmf*?Uoy4H<-c248sSd2$SLRiSQ(#KMBPF`JzI(wytFuIl^oRqCJCSnkD^|F{kha zpF7vV-hi)Pjnze!Au1RpF{<|%aXfexub;ep0z(u){M79od*8P z3|&YcFt?g8LNw5^LwZ;2{3 zl!Y2W7-qei1HC5IyX}1)&0m}0hxJvpt?Gkq1Q7-DLZx}(nnn-LGK>pbCd>5*M*>6= z)HiA^-5i?Vt2T-grnEL-4_sF9c*_k~&;!YIZhRJg-hkd~Jz?A4ZD`pMh`)~KP8+1u zAG%?52a7~V=v!Kt&2;6a^RVf4L0C5)gciDnNtwu(JC7YyXo1-|OP*z05->EI4XAmd z2>>9TSwSz=?Twh`Y9vB$SG_2k+^H&xgI~3eJ*9dK4hLhFig&0Am z?`k>tvVpSG{Ujt~(p|X);Th*X4A219?OITMSKV+)dy2tB%cCr^bVIv1eTzmPkrjUk z{DxoC?TrO4hpCP#47o?QRBptkL1Vi~I(^#F@;X%gney2H$zmtUFP@S^6@KT`zEIV> zb8$lx4Z2)q=c}$%dCHS^=}ICnAWs%~8sWs5BQ!*4=L9b>+qysx6(D!2->H)WpIT~X z#A_(Zk@}Z%qK_*8Y@l95y9pOcpfuVN|5Xc7G*D)ti07z{;=d(Es^s4@j zq0m*1(a~FLRYG0NsT#M={ma)lhRRXG(%LSrl8rN;3E$>x34Q?>c}o>VxCXjhb)Hhp zz>3*9wjQjsNsxm!LF6>3+)3WJfiz6nW`OpZwm7=^Wd`miu_{rko>6s`Wi@siCNpJv@)y32+{+OWb|dwJl6G*m#vA zhxHP$yrhSyF@Pnl5NQUpD@DW=2E=w?Z6wbbROjUvI#SXxO15$zo}jm$evs@yj_{i; zs!?r)llCzH;^{0>@d!UJA&cVr18u`gk}W3zKyKkNd6&y6qXWx{ANnPC#s}f;ixZM; zt9gbUa!*vuf_AT@3k{WUlD>rdAgE!f(=g9)`8mY`U3d=Zq34A<9+U)nYF%6~sgy_8 z`^1RaHUDII8d*}#V=RX28NuNq^&^9`;(Crd{Tv=?KT(_>a!OQ_km8}jD7rb;ie5(7`+%*5#F3c* z6ZGP$%cfNJ!PxF-@!4ByZ;t5gZ+@8G_`lH-;Q z_>lSQ6%*UWCRQ*>(34O})KV^k2r>xXY2hPRJ& za>TTGJ3~D|b$Byl`??zmI6M^;;);^#anqIXGh>bpOmwR*L`G)+>B}$ES4b{AWHoh# zJBnd0$0Oh>XZ?-GDSC=X-ff<8r`oRa+i3Z_Y<;hkPEvU*HKm}o72M0x3qDCqE#+gb zHqHm@po4sN>zrM5e?5B)RnLg&q~Zgp>;7b~v$}{BTDT7`A>PeTRISvSe;_*b6E*OYJAPOq{+K{=+e1ee(mZUg$wNxOu_V z?7FusH-JZ=l%}KFhEfbqlIfXxKE-6Kg705|2Y(YFt&l8Re9gOI7IGip?b@yE9`Kq8770(J)x7$e$ z3cVygv|xl+m&C)4DBDprRkTWWr%8mg&RU~&l(Q2#KLvHh#6(OB`O~+bLwn=1w?Eh% za+0fM!;bKOZE7BU!#lZrqLXg$T8?xgM3?>(luvsCTCz=jnBVlM;GFY?WtOFK5V+-h z$O140SuPT@Riu(>;imrYQIbK76_G)kQ_%{0Wx`J7F`fyr2xt4p+TxcFP}D#eo2Egv zhvsAl=8^dxaYbJmH_j6^2u90eEQ72Z>to9f$5_X1%cw?Ddmvc`L#vEr`vn}E@0)Wh zaj0YKLAFX8RW8&h7rbS-1^}z2PdNJxO7aJ{8}z`o{&b~)rz+B0c|Sgf@p)tvpQyAe zkxJFIxd~(8?4erWFHN#)PUiqR%O`61*g&ud@uFInXlUNi9k!UR0RRe#vI9CdplgP0 z@o}+4@=2{niDjE49qEY))l>8fIkWAXPZ@0`&%GUFNP*6Q;K-Z=y;is|P@OD-qK2f* zegt{K2tPn>w%*Z@sKCXgo_B{cTkg5lOPc<43iJ2l?97RA?A19^i-TGvg1N6@T{L12 z@+GTtWNB#6Fo}>-0P+BY_y;$#P*qGXnKv1e78p4Skx*F0A}o7Q zMZp7}@R#B3rwljpG{%mSU0b(8Dd2!@Nd`B|MrD@4y%3H$SJ^%j5v~eZlIQiOmtTjY z+zNEs7SCnb1aBq%WSp5s2$wnuP?gzza5Qt+P=OE&s$`MS)U4KkA2Ladk=@?~`GYkT zW*s+KDp6G!@Y@`bhj6^H?;;vuTgJK}I2mjmABgSWqMOr218e;^#s+I-=W zW|wuVRcY4rlp2rA^n_P)&4KaEs1FZ)g=$cu=r~5=Z4^ZH&5jfJ1u5 zn+RcdvEQEJ=uisMO-|XpYfCWaHKIc0=0_=1N&!}dTUty1NBHjF=e_VHPZ_!S9{AA; zJLpS7&CGgcs75)%d}3O9;b`1${9pK>e{4{(cJY!3W`fU2yvl37S>COoB%@iU*j08} z!u$e%Xs>TEw1uWObHpOs#9G9D`TFEQ%8y?@4BxW{bx;Kc)CcMOwI5jLswMI`!t#?MiibX5;#YH;am& z;<5brbo0&|_|X%TV50*FVDG4^xaqRm9CIZ}1U3M4%k9cMKr*}`wZkZ7uOe%z=HQMA z6SnYQg}?uMTcEi&uHl$D_kse4;}OgUW+G`c89+C6w&Utr$PpmrCwhao%;t_D+t(7h z9?tZo$aHVn;#m9RI4BciD0D9Q&1=<>U)!WYr>Qy-^K@%oAolnNnRe?dJqhY zasjGaibpfG<}b%{$J!=xu-4l6tl9M3;K81w&Ju0w6dQ~lQ+oEd1{+efcWU%*y&GHh zk~9_)W3NQcT_rEIgX%0f4K#L0x+p7)@mOdUE)e~kLVuWRrA#nZjWJ)3~4{dft+*&o$bz$o#1j z(%7Q9K-em}X5hBN!QFNSSfZ4g?OQwqcL9T^y$-Q^KLE=(MDJgfPBT0tk6)2q`IYVO ztyjE)fwcPIN^9GUDU9MO0PL1`Q|%0E4uO0xD-%p9hzqh0RUFleCD#4;<%9RqNGW<9 znz^h=OYNXOrz0{Ii;@R+55;3cZ3%&{nc#p@wn8JfS?XDEmg4CSBNFGdyl0CfNxCPb zK&#g5z@l5+0X%X$I%9!mW&IH4QX-AF&{)pdrmuilk7RNPLlycs2izEt{~SBmZT=PS z_pg#T>_Vbdf+nNfqns4#zkDOEi6k?-OGyU~>0rt6Wv>C}fb$V1q`Op`Qis}eI20O% z=R`VQ9Q~sK@bX7=H)}uT=?N4r#y=a%0KGpT|IJ!Yt!`6AY2_`DLOTVfJ*cuWT^vn=fcj@0TW8q8ASiI z@`Ie=Q6lM%_7R|TuDS;efj*vA%F@!8Six}Fn$H)A=u1Fjb?ic0nHJ)D&k-UHV;K2NUxJVj_pxN1+NUm@l< zL!cJcRqYAbmArd4i5uiZ7u5{6Hv#8^8mv-Zf-dA*e-Q*so@e4W#VMYIE%Y)_!sa%&rMEl++It0wP_U}0&7Dmp{N)=OAX3&OafT6bHoW>XxX>lF0NAHasS7z%tu zd%DUhwxHZX8nPfoJtQw~1;tZ-DL4xfX2*igach41E*k0|K!cY=TWUkw5TRQq5 z7C-$d_JRI=vkQ=I3dB2jio@imYkl}j6duT=c6v#UHte$FpU|?nbB2bd<$g+DS`hNK z9v&fozB&q?bt;T!Z*;ApzHfnU36-*mFtvr(UFR54wUtqiFx&5RiY^t6ZfBcv`DQD0 z3hOjU5nx^KSzVQ$@?ZsB5reToBG_QkpRtt)~_*3eFB&dlqP0}6-|Ibd#C zo}x|%K|8M(JO-Cz2hD}8`Qn;&#!BmRy#CwEpI-m#+s|MBExe>g>;ucmt;;2=xN~Pd zQC4Xg8-xyM!eo7~*iVVI2N#z36cd>;z1xb`t8I|`MeSh>7boR!1#zd|vB}e7u%kf^ zkUo?%wY10uL#q%kmvYAVES1VuL0=snv$E zpCMwNK7(pBVki0xYJr!~j9it1*2EhD?%O-%0p!`FWw%U)9Z`)7WSs$ewSEiXMzWZ! zPWVUp1?ykLK*LWcD|Fz949()PR;30~9^pZ4uxC&j2RL%5;CezvPDZ%F#VMN|I-ItPs$`RQ?;RGi#UK*-{F z%q!1Wp21m_ktqVINW^ioFtx}sNrD-*{99_UoHw|LsosFQP@w6aF1&_~Ii%O@;Xh#Q zfR=`VEnwSyxf=#1^nH<^6$@K;(~;$Y#T;vxbugCJ5CvY3JWWhFaZp$hF@$5aeIatH za6E@ZXTWPDhxATw3yTjGI&=gxaBV|^^GPAK)4dEOl^v{q^MC$d#Axe4z>Iv;+;VNJ+5C=S8 z>eJrQZSn$Ow;q%P9_#P}2;QQV6VBvBqLe%Ww`X~V=s!4CsGua6dVx9QS!*oI<(Do- zdAG~x!}0~yVo56|$Q@cz*mNU6Qzea_26j5psh>v;8W@31lT*9qJTO6<$2y1?Z9TGw ziB?tNfR$QRCS8UTdId1lS;0B-4c4m59u<4nTwN7|mgesf*Pwjsp8{{Df?rm96}K!4 zr~;Smz|uqI2JBocoP~|{aElc;b{Wy6q$8?Ll(77vAYh@us{#}PYyNI-OT4kglXT?0 zI~E#SegllDz*UJ5V{nhRR)>6ObW)^Hkxx#I8P)BzBC-dfW8_cNPpNU`jljRWe3X6< z#~%*U*E?4tBx+1(cFa=iaF~M%Wq&(GN8i>@8Q!*q>!_o+iKt3U)=a`w?zlBFk|GkM z5dhWz&_dnZSa23)%51D>h>w!CP`ERhK&gC`voA+g^9Lum*7l3-vN|+IvYr@nu5O;- zaPu0cFUdXka0h^;l=^X@V-f^=EV0w=WYOc@rp|cGr749WCk_)jFiEck z3Tzo>L^WjPKa~RCiPF=&2e07$j-X(-9AZq2fjaPrmAqqY&j0@N7EotloVceN+9NC&5pM(Ty$eEy_drQ6DUm|Uu8 zNt+OMD?P?~a2Z>i>Xk~0GRn0hp={KN&M^Z}@v*;J=Mvk-uPtyh(M zMnmARx;|m0aAMd;z&5LLIhgeD?>NpV_TR~~OEZo1Owo%eBn{_{-Q4p4P8E0eJx6Dc z3FaC+Oti?(=56xYN{;9lvwgMF!f?tZ5H=aw5};3Ae8z0V#(*2lPpr3CD-r?ITKc2_ z@6%I47CN}MD6cA8sB@UZ2yPNFFQvR%LA>#h2ek;*MP&*@06cg>lZ=17(+OD&Cvyd2 z`+AKj>1b*1j7+ zBA91(E;p7WS3FR`?y;dy>^W37@FfPmSm*BG+6JC^LC}t{)l9w7s1v1h>)8D)=JmEf z>XB*JO`GTG?!balwvq26H{^&u2YhdWD(T?pWh0pWh4=d5s#bJ%G-*`HMMqNCfHbt+ zCc7$hG_t(}Dj2qPwY;<2t{^5h)Bsk8LcRTrzB5%;7{-w{HrcgV#fE@rwXzKe!WpJk zo2@UY6z$dAsg+{797CFB%H~MZjsOu@C@HxM3GpINSY43mX@;|0P#9xm!~*j zzBEHsrzJpR9F)9xwEP5~kx7*Lj^1o@%OeIOC_}H2I!rp4&T5*mJ7dD^Ik-vZ7a$zX zMDAW;W2M^=QjP618#0>VG{wv%FSmPktp}C1{jI6iz5dtm_EU7*AgUpCjuhF*70WyTKAJ`N?4c3g(p=me|}HBtSTB{B<;aLvSvI1+F~uZSl2p|n)CB^ zJuXDKneBM!&1ku9;(iF>s)aAd#p=LHVE4+6;bskU78i%bN@R(If&drIDYaw?)K^fj zBeej+#Sgys6=s}$xZE=kAUEo#RzahZPCVqyQ{t6@qQG6X^0QL?YFX+8Pm{3cmN*q*pTBy3$6G;elU23ojqMeS$(adjp~U#1J^3 zdYB42{DJhy)1w1JhKa=<8aHuKr9h-tAV%CB0e&m`DZt=5duywaQIT+YGkkzk^=GH` z+4NM3l_%SM2~W_Jdq3doq>rLAP&Y{m+O8++{@BQWqp6aW+fhYOA9pQrLk>}#ax0bW z2{asCY=3~YA9R=@rftojCls-JR`^uGbG!zI1!BBKSU>@jBFbZc{KykXHc8E-3j;Q0*AK0&dE4^s zf5(qKKmy~`vl(=S{)g?;>!5Jt&1q^QRiXpsHj+Ul62JwH*hWUW=&iL)h390!vz5XD zjhJz0{4A1GC1#xM>&H-L{0R^6g7c8xV$swBgkU0S5DFW}z(rcA4~w%s$sKkuob`vD z^@>R;o2NUZz`Xe1Y1C3ul-?pg9rW?bb+Kd>wjp>l$kl9%L& zq^x~qA5)Sgsmlr9gU>;>pu9N{MH1AM6&2+&s;LTe*77_6B4`~nNABK*9U^*1bt_Q^ z<7{wCF%27S30xuhC8-ND?od;|r75=DlR9(G4u|N$TjP<}aSxd$6o-~3*wv+l{1b7` z9c9qure9*3yxGSbE95dZErAV~O7>d@6=B^qvmO29+%kn)qvwMnPE zz;H9*MOjqJ3UJ2$fWGbB6VNV#{$Pa}2#c#6ydarn)x=VLv}0#ID2b}L!j@U1JZ^ZC znqo22=qiv*S-GpO`bt}a@y>r<46+rj#WY%rvrfdgP~KhEvr5jsp7rzv*L0didGIGFl03ehp_%}?JyFAslkdi{UG%ddf#ydfcx z=fIBB!0CpfhP&fE?7if@M%V8<0}DilX&mg<<*F&8(bCorZbPO0b!v^LWq`xS2Obv? z`I!6FaZ`Q$isI1)m{}{OjF3F6c6XvYQdP#F|0rCn&}N~m?isPi;^KHvh>^Y43}bGz zV3Ieq3jvOxXxW{0U2sJ2*-o29RvVbhtJNxCEl{`^0KINJ;Lkpl!1nURDPdO?MMBJ? zMB7TTaJdtNLJtza+8P@6{I4KNT-mn8;93rj2`Qj`juHd*BnD}rx^UCAr*pWfAGC3% zp&tZ34Am$TgE)OiSr}hIzrswPBrR=KUwLp+zq(>tg0tSTD`6kI#KD(z`WgK6m9@Y^VBR^Qk{x0G zv9ilEJV1y_SGiz<2z_bqfrhel#vrJdPPo}%%t89*JtARq^n$yXsa8P?hcV`-1k=h+ z^!<$#Ph}NY+PJa0B%2EauBl9zDi2?jh>ID$vLj`(RR^Og%Gum81#$Yg?D^HH9vO8B z&6$n@Aq|Og*ki?zH6h>>AWM|XssJ7%qt0v~J|(aA6jjibKh+oZ*d_d!Gb8p|RQ=p} zw2ZdsRNd4PcjR0q0|A|s2E*s{`P4^-P2(i(p^$THNg~C_O2ET#Sd?Fcm!IiR&$}r3$1#QSvBEkiFO)>Szg5=SjWI^t0;wF>oU&maf_LNFJuF}2qnjD7}nFGSHs(N zlMC_d&XSaP_vR+Am>%xPKFxVk?_FC`GY9t%7-!hU066KFrUs`OIL%&UD6Z@fR2N_1 zWq1OHT+vLr(6hbjj-*`WHEFr7`ooq*tFUQ$X;c_%q23#khX5P*dmsycnfjy_O9odk zT8d5Y4;r1LkD~@i61&Yd4rCO{>4-~x94M&RxdQ2uFwkEXlQs1pwF$3^AaCy@S*$3# z>XJ|2U*-eW&NMTmd_dgj&}v?N21ysia>jy45(Pz4W2_i>iyktUr3-Z`U<2e%d$$ek z56cnZYRxXZiq~$oEw8)%=Z(q|rJaB&3EGKLEf=eTLV3hsTrniOg^Wznf}!dxxGcz1 z4I?l}V)*PkQnf(R?Hy+{aI@N0XHWdBIsbrY4RDAzy5+(pAfY$$z(me&XMvU8dqoYc zZKkIVNN;CKDN6Qn+d3(lo$o65YFl)mJrkS1u97SS{I_)Mo&{Bx|8XxHt6fv+ae@o3=#$b@{!PZ z;^V3k;v>?v?N)fSL(1hD;jXI2K*|(knSh(y!OrdmWfPikNuFp*SE@m2yQMq;@?W6` z@b+8Pe%vjcTYOBj$gy=h0YKfO_H?E1B-D6PaYzira-f&Ot)-s)5|26X^{i5Fr8gi zw?UuG(0Hh*A&|-Y1O-};LnQ*m%LAm{_@3y4nRkXoE0IvOCNB| zh7yWz1(QkAx-RC^utdvD$}g#X!6A_2ZkANn&fJ@nJIS-V=Ip9-kQ`|G6hB`JCH`fb z$#HhOOI23&47IvS8rmt))J1TxO2^!7?p4+8a=M4Rd3moRfCc<}iRlKWQZb>8D-b)C zzTBBu){J>8HEepDwurNpS@lfy}Y$+Zi0aA;9d zVYSU!r@YN(1=`L}kM@JN-@kkn(wFp(BXXGZnxV)Ay&MjY!uev>ta?g_W@U6>q)?)j zhjV-7ZGhw5nH{e!$s^d}NZoDzpc4%@-lH)GDB^+BjC(=K#NB+u(?7<90jUs*e0k`= z&Q5?*IoRk~PL^F33$tQ5>tUh-h^Q(rH0T?|#k()hq0$#^Hn6;>5&rHDP2K!-pTbh9 zBtq#-sM=`T9T@Cp{Z)lRM9+oQumOX&fjP&(5jc>Z^;mf-Z6#&J9bN#oeO<5X!$P{h~9+eYV0q?%?6@eTMFQ1=u37tFc69}e* zzL*p_A-F%sXO7XJZDZ%Ry0DS<>DeZF_&1=hM|hdqwo!#ZK$i@ec9YBJS)@v;>5bLe zc%TKMq{eaxNxcuLn8(AyhKm)b^!&@$&|yF)#&Mc=N(16VA~>kJ;1Sq$OQ5CaBsxjG z90?YeI~BwPc3-16o*ap>uNSFJR1r1?lm^swwyTaZYl>=O?NhVimGw#=^3mLW~m|`;o$_%P#jJ+>>4EGC zj#E4s8Wqij07XE$zlL)q*!#Wfu)O}e#2&kWqcf0`lWw{L%Z`y=1x)s4$5WkZQjV=) zuGMC5gTNlyHYgRxMT#W3(4I;QM!tp9fs)Hwv=WEjsb&(Pl;bv3Enu?!VU9NJgf6Wv zl7ajOoIkJH;1`I}IZuYVvC6{{Is{y{<*NB;7j`%z%>YVy&8UJ@PHs}6PA4Q{R@IUp zJXX~<9f^g5+?t0Xu%SEw^IJkPs zw1BJBHuAcqnu9q=tU`Za7!6iZZjT4sXFBb;jb7G# z*-9;|E-*2$?G(Wwb&VaEl5n!)53?^5SS$btrBf#AcKmL`v!fA{Mcjo2yaaYzVA;c4 z1!~>f)^F{17Dl;Ht+S0A-&q}&8w zBpa4VghTIoQ}Ah_v9-^(L&*lLqXGpmjZwBJohf3kvIp%Wj}m1ouBX?ZORn^m#4>fW z6Hc$V#e-^FUt9omD2sBl<*syZOPZgY{vpv?B_IZJ%7BI^t3wh{<3^+PCZQ7Q$&sOV zaw!y;dLzxWR2B$IyUI|`C$zDPyJLbd``YDj22#j2ajni>1>|CH#7rD08Y)h6WiPJH zTvEVT=Tp5M=^PJ#ujEZ8Ssy?GHrm6O?I5nPM3ROiI8GKWkSkeJOucIe+&Jq`wstQd z>mL{c)=3_)LT%?oG|to0zyO)MsfUy>B`J0?zTyfB6*bx5iszynQAw*=2(>#*$~(op zV8HQIY@yjs%~+$sx4_{P{p}RDfC!(ojbqt81O!H$E1w|w__6JPIoNJ9 z0P2I4!5Z!tD1A9#rY!->TZtHpP`Ozgp>uI6|9uHJYV4o|m#I9k&78+FnVQTXyG!b0 z;$Q055#{5;<+2JF=(U*kC3;9`hfBKM$lVwzZGifpdMFU8BQ2{x$WmXy_P)B5^|hU^ z=g$S_8=S}TXcdJ{o?fro z{_e|&iAd2JCS^jP>Iw0z4I-{QC1@m3b4^E1IQmKq1n`7W5ZJgwy|AqDt(p_7&)hNI z4bEi2)@3X5s!N%~2MBpo?Md^C+FgNJacZ6UZYya0S)f0#DCY8n)~6gS@DW%~s>z*J zWdZ&HF2Tr=*nvuD+zBh%mP~sqb>?$=`@QKHQYQMR?lhI&8A^1ll91ouICn!cU$2ka zxN;>{Cyj8tPj@k(Cm(elBE{yqxs?8_Xl8XBw0#L6!XsM%QHEu7*Hesyfz&{oZ5Y|Y z?J}Q30wz9?4FV;kfYN#A9ZY@`D!CF=6T#1}B}ura%j#WNtXG*#?!5CThw^gL9lxut zOj#J0V}Wqha41R9Qkl}YP!4;zrPKIBM?GngLJ+1h8q>=c{2KU;9Z9mQ41|0c;k%If zWcR^z{1=_J^#g3bf=cn$|z zxI9UTE-dX8HB=6FQ^PW+e#U5?521H zkxkdW137L~S72H~sSE}hwC!l%%!(c^Nl4=o-Nh{VH7aIRku(QUK3J2c$qY(d80E?q zadkad&Ba00w_PvtPj8>Ue){&~WRdbGb0|}ycTaa}V18>!$mpqqBuNS)#i*@3lqJhq zS^`}AHZ(($2`eYF#n}6xt>NhBqFT=?Rj0s$ zlpM?s9K`8aOf77<+FmE!cD9AH6N=G%UP1dAH6cXOH+s*!KRCNyK*7Gins^H?Sp~ZG z&+OohVSDxihG6h_TRS*{b!ekviQq$T=|S!%cfelRaRW=)bNik)wgk}szpvrje_$6$ z%?s6VtfWvRJmHK>Np}eRIc>qhz>hsyc}nWwM=?7EC~p&F;$0bO-46O#%cPx2-7I#< z0>g853_Q+5husa~Ol%cY6fN+|6RMa(!aLTVaiS8}6y_?`GjoU;tPY32-y`$>8GN>o z#+R_i=}j=y-T*(9jrzm4xXDZP|>j49$tQ%q-mV!xN6&7;+CD)5^2s+ z;=F3#5u@@^RX58_v6}CiDYy4N|z0^bnYHHf2j(vt{fS7fXD%MTfEsBmNj# zGL&-!d)Ct})GRTAz}UMAJdl{gL~5$%#vlt6DkMNy$+2}R1LT^ogAu<=NR_nS zIF-W08c*Q%Uvk~NYwrk(n(RZ&x@m15K5a*?#su8Z!i39aI}Hvxb<&+Va)G`4Zd3^4 zDdz*VZIBpMs5tgP#iG({qf)k>dMPV3slFM1CsK8`Zr{_st!Da}Rs{+PHI;8kPk;UJ z?c-qJsb{7@cc3Zn$oOT^MsypGKvAv0=h!xuj$M_)NhxTF3jHyFNeo~Gao1bQDJsA$ zml%l^koCFgm%&nTz&EYbqWYD>lg4g4DLp&A(D&$Jl6nQb>JmMXZv~eq%&6)OBNm+A zu2U^X8BWq}x)dd42#ZQ`$)n>a=m4kG9V!@z0d>V51#^Zk(^S@ zv%*d+BFlBbd6#k_aKKcSf41_)sJcflYU9~G4(e2tRy5R%io2Pls?b>a7O@e^ltdz; z(v`>5$69W`{NTo>-wA((0HqCKrul1#&jqVMn3-|8!&632dqygkG%o4cr$aRx9y|xo z1_mjkZRac5*J==y5|9o_d;j&*mp_EJPe5W?7ft;L>ezZDzQNgRO;eQj zYmoUKbR;}aoWua2ajGm>>aLO>;Es2DP8I}0Uu&q7-^{&9C zE~%HejtA2aIgKE!%8|n!3PS88NK9CWk{TiBK@%oDh}tHX15B}23xiK+l{4de8k33! z-vg|Rd5|! zrHwGVx-XhL%VyJ8d9a9Xsn2j$0!7hLx3H5ne@tZL*ik)-Uw}UFo}Ly^ssn;fq)UXn zojuGY+t2l*gA1Bb>4${QK|zX+8e@p^<~c|%koA#!w6YDi4BA=r3|v0-6rF?9wT;SA z(O&@kfAE@t7tKYyBn}WOx^Qx%RjZxm>SQ)2jB|!sGeejlEK>yvDG!px!-ozR-w;;T z5?k2}-tJd$qlLt?wRC!$b=6~P%rS>zv0Om>reM(gQrTM(HGj?9Jhk3qsd^(a?M015 z?kZ7&s%CfZ8?Ur&1Z|15f-Kc5$9V&RewWWhBuEQ6%R9pR%0l!E^zt;e#2?+*!z$Je zJsTW|_E}+J+>_ zN+29hDh`T%gB!Wj0Ppd9lJ!-R#ElQmYN}{FN1EXVxfB)%@H}zOO}jdMXT^e%3|}5k zNFCDseg~Btj;Ze9PKJ-7c>ti`6R*%YvQ}+}nSJ!!C@Sj54>r}wQ~o^b)452Zr_hTZ z(Qc}&2D{5illo^s#d@icpsp@~*e}*k1#GL%&DV^B01e9`+kc|zw*{qhxAkR9h9MEo^C&yL^dYT+DQ$hGX0}b9BGt9qPNB~UPB_7ojViSH zcqogkBqdpL?9{2u^U+{}#OVlG=R^!)>vc~=N<}sG+}f?r?Yo1EyGjQ#ub3&!RU}sc z4A45sm0cj8a=2T1UbGHclhd=7L$~dx+|JI^}SOj{t4zh=)Qy`_{^W#t6C@(Y~n;NTRD688JJhM0^3#+4+=gGWNJA zMIaWmmSNqz$W2oZINPWa_$c;z<0M#~wcCPdGPh@oUP_**xBbhXRSqa}IFHu-jxZqK z*du&KOSEh49>D<2UK${$krX<0NWOD85+PY2>%tM#vh~lH_%ISGSSY>YKsAY5zI%#y zUv$_4>4VkN)ge3x*YkOuK9ZjVbFf5Kp9E(xVUH953yYz#m>5vh17ghoHbdN_wbW1zvrHi3!-|GiraLYVK1l~*@!MJ1)JHpYLdR#oN|C&Q3jG(=}=^7F}e?BVMy%?QA0yKI7#YsZ;KYGVuokx_FOZ> zaL9s$DCq{t!Jfu4KrPkPBhr(F@>jZj@wti-&KJ~l!w$~5MR}AsK)c@#!NU7m$kNv6 zM{(!M!+I1Op5rWu4{Tly5UNJ(I#d2{lK-7{H~(1BY~Q-mGFwMTwld&dC-wUVwhtzn zQbr(!fk=!*DY?`&ad_GN=mVRq)y3`J3KRrbPPAH@aM0Y)Kmvxo#|rvLF`csnmU?V* zZP^3+;ikD!-4HE%QYgYACk*ln9DI5>^aZFpY})@VJ8qHiIVZt(~@=cE<&S;$vwlc-v3a+i!;v-J{n`traZlqyiJI;|Pz98*fq<%*5H~!#Z6Spp?%bJi}OSDv#aeO%rb6TZZU-b@Ey4EbiXQ3 zd?DOkk1zzSy%`S@(({4XO{ExAb`S>zqy)1OS4soqO*X(p0WXO4733P8Gh1-E${u&^ zk;{naMl_y0AIn;XuRWVTk~z*N@~t`~B-D5V)|x$_H8f0==p`aYi@ZO_Bn;eB1o&ENkm*0i&8v-;mKEnWs zM$uB+I;v&_PW75}1vNwndFWpRFFiN}OF$#!7&ICOKh+u5hysx0%2*dmrix2AsDgYU zOUS-S36Ek1W+OK(kZn$MQP+3|=+VN5vIdF%?fNV#SV~O};y$WN2`s8Dxy_O+E%_7$ z1T9wbFR};$+Flh?>E-U7Y0~QE*!HE8&Yj$d4>*iuJu`2pc5PQlny>~nh^EPSMGWn8 z>xDTm&F$POfFIGJ+f3gKp01db&>LXvG7=`xwx?kO3!NBCJ!S88rxl(!bu9tJ(SH~8 z`-;W?C6qp_o!BzKWZ3%I3|?3&~NlUuI^sa*2l@8HCx z7^4*Uq2VX`oA4ja_(M_#3*gGFt@v&zJtC3Dz(}0>DUb&0k|D^dPle^8MB-yr{X$9` zbXc$U`~a(pIuLFb$GCyb;FPJ0lFX5~$x>jo3EhhRlo$(7YS%kG@yu1;qh*4$|o*{RN=h=J_{7a#+MmK_nlRz^@8z&t^8joJ!s@{p_xTg@dl9N`u6%|oSvm1NNcD!wec zvJfmg-m*!P)Q&ELAAA_NQv@I-y)fdjR~(cF2fgt^B}*uSAD_X*mDm z{-s2pL%SHXV(nM7gC{u4x+Cn5cGN=5<6G9_Qq(mj14?#&)wDd6P~X;E=`E4=a++~p zRyMuek=J!pC4|a>^=PKVyFei@p_wh$LV%3Et9=V!M0UBa5)|80187O)B<%($h#I+| zxrE7d%zgmr$fP9WFHE{PVJ5}{y$*5+DA@_nadjoOzPVg5d2@Oyszx?+V=Z}ALWvY~ z2-V0NNGYE+3mkFS18;HTxiGgkj73Pyet@xfCYAKT)s9g1eR=2-$;Yu^HE6)>>bl|(#Tg5rtlqFE%EMKni3X)T!lzL*~r4N+G9um?%3406!{*^*CS9 z_J{kmPO?s?+W%6|H1Cildo#W?z-yd#zx^>BO`BAJ$YRv0OeZj}yF>O!G=qHOFAwcK zN9y;IF&#dL)P6N49qFyJttC=_pjA~4>tHgW92&PQTGYqJU@q z!XnY}s4BQOkIb1{`4#XRFxluNv4K^Ms&Z*CR7kNKh}i7H3Y!W{Qq@_kH1l>T%T&zk z^aSLHyaWg)7i}&$Ua?X?T3daRW@MWq&Da9Gk6;EAyuzzc;7JU6&KLfX)av}kPCcon zB}27KPy?lbr{?0PCWfO+LG`q2@DQ#I%wo;>d;re$cYO^SX1h7icn)~!^iY@y9=#He|a zJYd%avIg*mL%cA@JIwz_z?Q7tBZ&{Z(w8-L`Bi}jRyQ;wPJ%I^*qY5i5ddvGL$;O~ zkdMn#lu|EhYQ-8p5+!d9lXvfz*%3BR!HJ901bjfl3RdyD^jOASpH&s-skq43Sb-KL z#ml9wTrW!U*o(ST2AcOSQn1F;4xztv)T0?jSCi$AF6HGkkj|ou}DzR3o6|sR16DmmBf3OsCmbc_8qf>u=R?B_tbqqJif+Jcq@f?``2J33HnZf>>ff<>1ogE|Yf zULz|uaOZGk<~2s;HRvA0e}0Ip=SN%X)|171&QmW!ze+*t_>kP>xj z$qMS*Zp>ZNI-*B^A%H@;2Ji^6<`QQ9vs{%OZp}f-=KJBh-?5Of9ds_HEhzxjP$J&- z8w;&FTf^&?3KSkTuRn#z+s2!`j-|Sxqt8_sRZ9h1tS5?vnv?y<*rV*jJ>2vB0ZC-m zptjZj{qKhF$}jyM?E}j|RBqt6{9f{rAKB*hsnXbX2Rc!m3k2hS*@NmF!8#Wcl#Oj` z^$P{ht{qB&*0PrGh5s#u%gYWCOFNOE8pD29sT4vVFts|$lrn-`Mb+?j!q-S#iySX# zr>5^^g9B&kx>M%K^mi86r2Yf^J^gfn6ucs(R<{+?^Oo=N6d`z^G-GuDUhmG`>?hT) zYTYnQ*WC^)^wHe_H@u5$`bInGtO*L=?m)@(} zendN;dr9JEln6+qZ9hWet8#8ZS2*aTR6-#YFdl0p2m!M`YjV$>t8+{})@L z=z`9pp_LZ*5}1pppuzFI*r=M#W>zn$%Ok2XvXmjP^=g4Hh7zTFXv*`VT4u`0*LGru z4;;Lu*p4Z|ACn!cBQLn4{t(`NchY_{Cc+gzYL`v2r-Z6K0S;f#Xe@mi`+w_a@|9AP~J{(sPi`G>km2QQ9UQw?FzT1;8-B@9}?0Nrl}aFO3ZI;blafq z*J0|A-n9U*NtI3P&fuuxwEfvtksoXlpxBn|Z_YXTyRAjNbC(d>mp+7pSIw1`_JFN5 zLA}$2NGe4qrRS!C5e`CnLY-_GRNA>H-Q!pm%j7Nc>M=#XKj?{|FMjw}1g)aC=?! zBDGJPZbwpU=L{M1->Hq%2tgcGw>Ec0gv_F3AWEbMwyFq9*hp(&u_PJiXS5O3nhe_= zpf~vOh^a!y-C~=fgCB>l8j>F@&kUN-`2-M3vsiYo} z4G5JTi<5d9-iqAhi|S}debS!3V-RS{pG)eXVT5nE+-UxZ{2kk{#RX3arYsLTjHTx7 zxtg>g(7LtrIAyOLC_k8OMrcMFzXY37F0#m0%4$up{w-RAP%9uNC1oW-hkcGZXKxF- z_@Ll)?Ttfx&&mL;y|i=)UHk*l4f5NF2PE~ADJRW_6<3}t`$LTfQzY_(N*6KnLS_@3 zpQrCO;yjIcZ}e(Tp=1E8Y--xj02F+bKGRn;^Ggf#)3={Xj*c&HKYRV`?GM&10o~Qa zVb*{qa(!B6G@*fp?ICqCZL)p3Bzh^IXSXiBWCU;r-{hD6^L_Nt2-dXhbWB2SEhp1* zSN4Ns@_Tl?XiDG><&0hr&X-V&$ed3CofP$Jl}+|^T5;Kii@Xu?t+J0iox3chR)CV} zp>W^Bj$i5JFKDu?n5nDw#B%3E4oHaR^aPDbY8o^;Osa~mz|!ED5GaPoPRNJvD{j61 z>o-4;(CTk>cVx{1aI%4rfK3k<>HS3r$QVj5y5}vk{hN-wpra7%qSP_QxC$zXEm0 zzL5vFxQpeC$6`TIjxQLtT}@`BoHtd=Ho&c_KiHsT1xUYAbEEX}Hw@3VIR^F;r-Jgu?M*6jm@p4L z>Ufn@Apjhk%Ys(HlV%VM5O`OXSCjg}Z-xRJ z9w1mV!wgcaDigxuyOi9Yy!|}=9*#foZ$EzdI*lR|zI0pBBYaR_7A`rE$#O(v%aEY9 zb&Kk)vkFpvAu{mR2U?qZWYqDjez^Hg%^5m$Z@U^fH;_JD!H~mF4{8l(OIx>O70SXt zYWKID4+FyFH1LrqH1ST={xT%N7 zX_b`V9-~7imIidyaA=~Hyxjm7I<9qliq2j^-_athq$p$ZJwx0{f3}V1;7(3(I!~*t)0s@o196Zo zO@omqAC{X1_Hvy3|3y6xs>JDLn32^Rr>v5jI!5}#yJAI5ZD%=ftSnN5fYgfivL!|r z+@33DkQJWi1fI#Ref$OrJ5v()N)IBGBN-<_y`yA1H7rC`D;zinIW3`v<$PqtZ$XXQ zxls_GY7qu4a$n-JMp`0|N=O(r^L|DTydK@MLDUxGxs_PxILy!DcmS;D2NWJy3ZHj+>7i~x`T_90zZ{4Pg?dnB5icn&vZ z3^>KWZLsg?5Yb+{2&_J$6O(%7J328K{Xkn@uq@m05r8XNTZ#)ykiXl@p`*bfT&;rA z%^D6mV*Pq6=j$Zpe95^&(IL5X`c9NJc{xxvrBeOoU)1^4;Is)R+o+Qqn59m{fCC!u z0O~M5DCJJ4kNDT&uT#|jWqA3eZH^`^(3(MCi%eAYCnVt5ABJd%I}ZlC4%@3y?P1~7G4z^H`1 zXOj?ep$AsG*Y7Q4SK*E72226~;5{PP$Ms>Q}`z0yN-AZYVS)x#;vh zm1W)KQF;%o@H1-*?`8jp<9!xu=6&z z6?yms6<50)*?tlvD5ILF@9Ta-u989#^f=q>z>N(GSiW5#435CEk*w{ajdrwbveO)7 z)NN@=jA(0!$1Uq-tD#n#0^9AgFSG*KZE`jvhRfDg3M+YPXV>tLXzbXAsN|+RS(HMn z8vIn?sOOd{f-5-$3GI&x$Q>j$5f3XB`01G>%P@s?%daL+6#pi#Na)5z;Av>EQtzIMZ^EJOp(+TKDH{9@v>o%?dkkmph8 zm}riv{kLFT^a+aTfQ5i$I+0Z&iHS+5x3Xr&nNor1%qmu!Kg#xS zbSX3wFM$gFX}0t!l;h+kZk)?6S4f0s&Gr!{oQ;m$R3C2D-`jj0zk2(W6Om8()t+jk zb*Hi~T%vTsa5nNfnAKvrwhh#rCCdmZoa8Qk1kK5_lpU^CK9uVp@WSH+tgP3fWif)t zoz{n3lgAi zENaKTLA7#LSTyGlNCaU=2w5zO5L=7>V}_ouHIbS|Hh9-9nz6I_Q!5Tu83D}P$-uEl z3$qSBwAE)VUC%gOy!Dkl^+=~$imM)qcFy*wQV4@V^pBF! zpt&I@hojpI3^TAn-BJUWou#99xV|7d1v1^9}i&{0sR;0T9LIgfy;p=df#Xah{=x;Wx7`HaPN; ztyFXv2U&=KssOt|7+~~R@QRU=Giq?Xa9A>HZs8wILdligAPCq17?Us}aNvQIK&eyF zRjHoPoJntf#WeH;H&0m?cQ*Le0$0uf(yTER?_U>P5{`ZS;Wt0@0dW9jS2Bcjt;_*5 z*GcTWnV<+sO7_f_>SEDs%s-my)KqGBk3t=^t5dZG3I`0!=Kk8ZmeMnl?o2SZcT7~z z%3?X^)CHY!m03JpL9n!>w<$to1f<6ztQoHk$1LW4N8h6=*vim{{X1X=4nfNliBc(= zAbOY)d<3Y{83I}n(|_b7^9WO>jAWxCikjLxBBs+0i=@d;Iv6`b*C7d??yuUCr`x15 z36A+Z1Fp-QH15bDsmOX$jSirChML>6ra*ywP}zMGZ>{W{rfFE%Hl$G5Rz-Gi9TR{G zV$uU5*L4QRvi26>c!lYT99{3-#$Ubtg85ST_8-^>f&-5|3*emya|BxM%0ZjT5^@a5 zj;0O@BvZFUeRae6YnJdvUo=dU?k;>dqv9TB5uvP33K~aDx&Xz5=2uLOM_5^Lwo#Nd zIH!;H!8IVq*jG{>Ey}vN6*!l(a2rrAi?;3PiKs9H)gah139G>5-;%qJFcnC1roWs_ z0F)W--3vfuq~3^~H)MyF$nQEXuphu_8kJa_RzQnGtpc}RiL@1QfNIWtYdcTFX4&R| z^0iH6-6!M+1lUS~r8WzJE93R2__Twzf=nRZpV0Mcsn2;EuBdcPN%2N;xLA?cq5aa1 z2=fQtFWqNJ|97~MS?q+GWCM@G3jDt|Yx4%s*PQue`Z_>|uL}>0ICN_E>z#fbOg8>g z7^%gk3EeCKb0i~hO=?>NZI`rzUQqX}-0Vj``I*9zlPc|zKN`**bePdLd@9g&fr@P5 zs?|y(!u((X>t!1xjUO{6TBo5*85>|$WgVS%DV!{3=DwbuE1+>Yk`w}e8FY`3(mo!L zaZsprv7D}uj+r^)Z?^8Bf{3ec2R+7A-J{hvI1`gR%1cl}6m_b>Q-?4NOi95Tn_=)c zT$NTGhdj(Mh4%M611rz3@3BEPZfqNa@G#6s$_e@$R(pSvgbgDDN!dxcgfCJ0u3!%c zP1EjFu3&!H1Bd)}ry^sNI!^8Mg;bJvp_HBbPDiUwGQxQ*ADCm`X2UejI-KZ1>s(Sv z!09TtL)wKVKq@s)Z=cht5>7CL3fwDaxlArdCP79Cc}=7&IE%b^wFNctr4h z3TzCc$wmJwPHm7oT8&ykE&_$vK}@KU8!KDW6;112>#7K`14<`o3D36-0j1b*(khR5j7 z0+Sq)X}Uc66pL?K>U>kN2keI`2Lmq1@H}iY+z=B?%he_56>5xP$@>{<$wuuzN5|uz zLtmM2MAEp3#0b@LzPM2&8jx+_&049ND1~rWG#h>RW~dRC>E_KKWl^T1f`%y0WMbq@ zgT0!!NRaidPFdPZ9Briy?Vv$0YH)l&NlsqujJ#XJrZo7qy$0QCs}`0+wBzp;{pfN+l?Su`Hp{Xt=JLo=Qw z&njyf{gz+#Zy!Klu#{i5xFZ@8xu#pNFTmK_5myK1RTqjC zP6%{EN|$=H97?zP?6PmkcfJ$;;UCgJ?I^>s#6e{{EG-z_8S@g`><-_64 z?mU?8{G{x5^kB0Y5@C}-NrJ(aNDm}`kz@`%nH`<=Si^Zj(`R1&R6oC`l|)IRb~$=a zfji+!1`_t5Ky*&fnSwVl^;MS<3K4CQsbWiX2<8-5u-n&F-o(~A+B#?S2lgJ9jq$qM zcYw!pD->p#>FB9hfJ_oFE_Q7p*0u1H^z>L>)bg@$5#9>Z1k`Nhc+Xfv01s{8w_#pU zNDz4fFw}}`fEDa8mMuX)La<3>9S*T}pGpYW29y~@+>8uBmsY*E;b&M8Wzw(}4Tl&z zoiO1iX1SE0wfmEo?7K^9-)u-rG$JK%b6-?w2XPaQC4?AGI{-mMCit1y6`1-UVSjS{ z7)*U3yCu7t*~l5{^@-bVuOH$?bxekuTAor>8?MUGaX{6!vkI7ur|aFy(gtDEO;6Of z5fh_}R1`2>0IlbKQgxtJEmtba)tHeqkC8Q*oTs?e#(NUwv!xR$LRBtt3TCLQ)pf{G0IlFE1Z{H?R1gzI~`~QzRv|kyQz+_v_#d2w#+BrD9bS|zJ@0-rW zQ?V##LC&N~Rj-t@qJP;^s#oflG@VRsCC*sEOjx2Ud3;t2vn5AOS}9peVF9<&Nro=g z#h39R?p8TI9?!!(`WQAGsInaaqKYWf<11*M_;2W784*OCq9JMz%m)4_t$lHNWlrvGEcCYlc>aj*f4;A&z^XH zvkwV=x}?b#hviDqHr2Vu4*6Nhda_)n6zAZ4o>XK%MKDJGgUcPwyT}ZTt|AF-&QRd^9KcV^d_6z*} z^0V;z(aTpaU+E7&dHv+&589Ob@a=PSXWl-;_uu?L|K!KExbA71C>WGbu*PLt4_jggqT7%e>-T=e(~}X^1(iP`+0c#^XsQTIe+~2 z)9{anhjn9LKqAzWO}{3NSW%CgY;~)}ekM-bR7KTVQE>_MT!|Aw7i*9#jIr_mrR&Xl zWJ#_w!T0zTMt0A%rdy)k386LrV;T^AvGDK=cQ?PjL}cU)8Ui^_^Dt1qNpE7Y7Lrxs zE*4p&ia_41|25}3cD`eNOFaeD&E?+A@NhRfc9w7XN&d*}XSBSsO#u%4(l_ zOgl^#IzF&spWJK$NvB;IP6il9Sw~0voTsQBpnhVsrk#!KHGc+~qbE72wZo#EI^v$P zU8F68XQt$vBzM-JHpGamd5JV=QX2+?*bOEfl6=p-u z43KQLu3FP5Kv!9PLjM%p=4+JaOCw;3krvxVmTy*Stq^tn=$jwD{W|{{@`t>S6z=(f z+QHTo36EdJYgx3Pu}bFAb%>g)g(s|I5nNVS$wd zPFQ`mwX1j|gRezF2UZi68`QYHMb$@*4GhNom@5gorV{|R+Lv6^-LR`IFgxFH9MUU{ zD#^2q2ep{L{LAo{f0_B$%Fql)Y8aSH>nu@0GL9UFo*+FS_ic8ejPMw&QB?2TfYZLB zMp`eC`Uwa-hW&-7vCHTHR;*h3swJM;o!4E>)1-RYSzK%3K0(3F&ckz3)z`{Tu~uy# ztf7j(D{?rHjD);+zBa6}iqDLE775u;k}A4j5*+gE*o;0dfnAdA0uW*ESP2jAg|nSq z1DkGbuP|^(*{rP&`wZQQgV0}SUvB+PAMGB$Kob0I%q&c6fCgw2>Usp1-8$$(*bwZw&E5&Vu>O9LLyvE|HI}7Wou~*m( zHFadetpy^u(y|7I2T;Uz3{MUGhlM4n*H(DEt6g^0eIV!lTX=MT^!BwGJ<0VDq#>D} zran^}Dc;7 zyKhi>x5^GA5Gw!SM;bEf(FpH#=1NHisFv(xysfTR1Zcq&X z>Y`d6$c}mO^2gzac}^N$fNk0EV-oiDap+h4MXt#L5NC*xbQgu4ya zMo=fqU{X;})u43IO!mT!lL%y;aKPQz_JB^IWLwR_O1lBY)CK8K0+HgVehEyJ&4&J` zS+PP+Bhz_-%u6t@3p{2C=PrSaAZ#qnIoz#zg&w_+0$?~??b%=1mT7}FzC;#J6xGUhEgCyH2g?iUUa3wLwI_O&-?7o~ zh;;@D#>zI?-Pa^Y3{2?)1+3cb>?srZ+3RPZqocz8`RPU!04z^)8J?lT{e~cD6Ii)xmIOb+(Iq$JQ8eI` zHOlL`=3rQrE?n!Yb1EljcrYGe^9G8!`6p0CKOg~dVP9rMZC3eQUA6(~XQ6*0Zo?^U z6Tn~Ao(Dl~9cWn>n%OopbO~5h#M~xe+wgx0|0O?ADAE;?okf`sh{I~CJOZQ#P91u> zE(aGX=o4(o1mpi(_-}dV{~=gzZ*kyLHW7bj?@)>uQ{o0Z2Wr_%)mOd!f)R%Za-BRngtZnd0aMvskLY0@N7k zYZ7W5Y{?9vh?Z}B7#oGv8`SHT9vH+)&d?r?^zo{eGd`dmpr^>$B3804WLcs)LzWF3 zC#2nW7M@*jey46`C^|=K^d<><*HHH5?ceh4t0D$JslltM-;_lJImYBBJjrbXY9`+Z zh?=?5w`#l>ALCt}Fb|SVn`???;b8aal0(*EK}iAxB?J0(hQ`y9EhDJ06VV5C##yVy z@`Oo4#htl&{SISN8YhO)Uq-_T0ckdf)$|oII5Lyoz zvMAL+{tYmX!((^zGdWoa%ov@}DW!WBHXIUcGHVNhmj|N8a`gQBO?dr-b$af}FJ$Q< zhki;2Bgl++0M_iJ_R8|vKsrOu%onf{)ibKm(Oq_xrSQZaU}f8Os(|lUkBsd4i9Dq( z99?|oRSG8T{5iErbNc)MN6XeH%+1;d!kff#<5lax-znj$@anc}^@8cl{vqc5H>dai z6bz035?%UcFWC*$&gKa2kIynOQ@pUh%hdfmJkV#6oeSp$%4eWmH04_Y6Hs`F+H%10 zGfSeO|3bYGK^I3Gg;Hg~z@&)80>uBAoTMv+@Y9e9w;I%>ZY<<-`6$zy- zR!wPA&Rn=xvICp^LbJET36&EzE>;3bMfRTAYVp~way=WwH!QA7@@5_EvCt$@)$cx( zsq4@TvG5}@_F{t_sl&g=aP!e6-#Q9vLx#A0WYI*9|iXpxIST{*6vE|NH`ULY>%9GC8) zdL3?4FvugEv_r=piIiLYc-X3ceET`!=;7Pn$nURmsfYkzK%c*R)t4j}0PtQBnUHR= zy(PEl5tSR|;;ah$#YJ-1R1k(~$+93%`>Kij>4`)^vJAY3WEmO|ofHypv$XP;Op^$g8@P^4vaE5sHmV~R+0K6yl3&P#mDDrk~jFA=< zf&#G9f~Z4f+B2q7&7Wnm2R!bY7GU+BChP zdGb8buIr`O?qS~lGg8^fwLMoy_peYzPaBQ6-(`zytAoR4m0u6=o=xa0XTQo(jh=T$ z0?Pu6*fuq&>ws0K(M7XE6^ZOkw4JSh03Kh?*=>gc$rb$4H7*V!Um*Krqg{C8R(zgx zM3lRVjtGf$yO&0Qxg%`=iDRK-G2~NaTi8>t`*p+(a<3-jv0xrp{s7R7MKEt{J)u8l zqs<;IAsCzn04r@FZR95_7sUCqRuQTot?j#>`14f@R^ms`1R6tN)Fzns@Sqfd5*+OF z>H{Pv@)|XsVBd6cgs3r1KIg~q?Qa;?KCAhG20d~m!uP;~H&hS$HjC9l^$g(6sh8wm z{8VSFKHg}(oN!1NIUP8N-|T|m2@HNg#;3KN$Q8}LpC7ubhFlj9Rrb!4lCtDO*c zo$AZ8>cLx#CG>9ID(v>fK?ohvSVI3{c6yM;s^n}rRjOUGaVV#Wk5ufEUEU?~O<P;X6RgGV zX;*LW_y7Vt+faJSFkJWq;584HSj)Y{00T1eH=hCS95c2GGyu=T(6ew|b{laa0L+re z)9LCGfnXtP+T8H zHdJekxfKS!@r4?m4`2b{tYb68aR#D`V2pDGS&YFYof}|%r;-k*bk&@dvIuHYCz}sm zJAEMg)Z(+`BU&jaMHUQox?fx>^h@is6#oah!75a4mkb*%m4ABs^!-1+{zgtQWE0?y zlc&qk2|R5br2Z5q$RS9~OzuL~!Z;a62$!#;9MK2jEUUppopfwPR^GDU4^YfFY75s0w$*Zu@+)BFD?y#DfZCvsw6K`*P5O?fhy zzadIR-d|dZT)G6DMwj{nM%XUrGX*-9j18XfkQy@0pu6Slt2{JQKf*8+(=KhdE=5cs zS9I^c_~r)yxcuz(U%&Zb{?7lvTKpexAD?99o7xMgFx2lkROG5Gkq-LwQ(@?ilBI#k zA?1~`BnS9GuyN6cXUX(${gTv0OWo0p_cEa`G^6`035hWue}PK;UQ<^45(>^cBXxu(rYtG8F91=AKChOgwdZ+37b!G_?YwfZfq~_EAx#bmCY#ov z3unlItU*W=Xvsap=&Hvv&~A=hGFc z30I@K%0-$-F*?NezPU(dHBBl8>K!ClQ|Mfw)l|5ZLUE9|F{C3r_$$a9+9%56LFS}k zUEQU@nuA%kSN&_%ppPitpH^MKmh7yQN zYW?~3_aX0P+sgOq-~k17Ckzu2upn?ETi}Kl04;b>-*m|AUZW%jnf3@x0!aoeHEUoT zbhfjpN9G=SX5O7;VQs=alX?#ZX=nqX0-0{PF->fnRv#;E@01x?bw18}3~ z35Y2rcr`A=nlRI>+yZom8kr^Wdt=qBvY{l8DCV^oKp7#XKXk}?7-T^3INT{)>Yf7) zu&d7JtaaXN3tpa1OFdfCS1_mJb(T|QL7`=PZq<;&lR{OeWlzIgXSLV4R#Ns!LB?bO z!ZRuveN>F@2@^qdkfnR;3&CrtUf}sBxwy8=7jH6+Ak=Y;t)!7?>lnl?g}27zJ;}F( z;FF4o1@&ZW$x5he8c|Ccz$_rhA79Ev4Q%0T==IU%h35~3Mmz=LRC0p|D9~eME@c?} zZ+?((AU3Y;k?xKP-p&eQ2hpcgB@I#`UDZo%o$cg}Q&E?LKzmIMAXy^dp33Gm`hRjS z>L<9N!FJD*H8e%$nF6$EnSH&;_QVX)lW{6{sa&;1%pZZ~d&f)o{nt==TCWc@ zVlPt4p|t@XhzS-t&ndXy6N=uK-X&1LJWV^h4n90mE0z(BL|1*qjM4!h8)^%QW7H*Y zIN^|^ECM+o#mYI{4Zgq5oWB%~11ei*oB_33kvVjS4+8O7QXVGR*rO~AV>g1Ha0dH= z8O08I)&b0jb(>Cw81()Nh}C=zCgW_m5)>~Wcp=ZHA_ltjn%Z~<`Y^hWFFa&kcxSvZ zv+5-Bq#!S=H&yrL9!>Z`6#-qa${?!1Dbt1etJfd#BfTSkR5tFf-ag{Tz)vhpQL_=+ zan%d*ZZ}1c_Q9FdUU&x<-eEaxA&9!_I?N?1FdWS(M)*-*yDGA1 zTc-j($CAZ=1gb*J-{ih4tDYG`BODRme_+AS|jb9Aonw%4hLI^ zy5E|F^PkAB_n$y=Q2P{eCT(a{=6&AP+dIy=23)IUgV#hoSK0atnKwXyQ5)pkJJ=D5 zo-04agPqg^OG>21rQ$#L1pk9Xe^bFD3xE1N0UZNvZtMvhnW_=3z1at~@j>0QZbuXD zflEs_j}DFEAqrs>=p0B7vgWJfK^s(}3hdp3&4qv%pf8@BH!2e{#pEEdhZ9jRe8!Mn z=MWxM9U37Jwwhvhfb{%2yndDiLDesH>j6wB-w2!~DS0u7y)cYuTNS9xRdIn!sb@fl zfE?U2s-8(c#_^~R3|f_i3hcANygKThyuUh(M6iHXL#jx0=)hXrdY7NJ9g!gTH11N= zg0qm@G-{FJt-AyYl||z#&{yaMKLH)SxCUbb2Phin){3;~c3fH7v}Puj_<=FUoc$8) z#$!UWo|K0=%Xa7p?p;s~48&|lu=7m05}m28GqF|498ggaK*U)xW;Y+EBX57?$MEfM z=)bx_DStVsL$=%>ygwDlZ&|kED|;j=+H0-7c+_`!RdFraP`b zJPbZ8?+F?ir0ChLg8(t{=F~4s?DjlA;A4Rt@(btKfM?jbU91GEGnP@uF~f`X4c95g zVhcOTk;4HAQWqbz^wYT?wxvjkWZg^pz(O~eww;*1R#IreIMy=nibw}3ZN3bzzeVqL zWIZud4rP)J5=S4$Jk|r?O^!DRgys`X9hPNBWTol64BEgMXi?E!8IXU1fc(Yj#=+Pw z&#ua+&31T^ONHt--%ya`dQj)xHy&ESwy|p0I^%CNfXvTHEy=j`dpZ|rT?Vk_H@c51 zF+}ZHWR0OhYenv)Bo($B!n3)TV+veAxK>OS@q~@cR-gx^$i%lQPEOG!$}bY<>{@ML z?hzerc`F2>p*Tyck3oUYlfa&ReNU(}x2hvowTFH>G~Le@$FxiBZ}qIs4m5$eWJW{p z2r(SaO`!0=yvad*lqs8~x=S9V98F=g)M6yH* zMgsoaXE|(k%LxQSkJs6^XoNs{6NiR9993FI-$&;{6#GCvVee5LGu#hOp{!JHm}Sp- zClgRE-t)~281r-?1`%hkf1xt-bvOe>D2zM4(aM+g8^F}9GNSpwi994^y8(G4) z3gJVmsn;2w9p)G+$e?k7& zFCIRnx~r%OJo%*Luxf9GhUYjdM{-l;e_7DmWv~Y?F+gqa(6LF26xSA$`pInIdcLOm z0;ngO1&tS{Ap%&TbE^eVFe(ZH9EF;(2-V(EY3-gDNOjifK{}H9?zN)oWLd%ks_W>W z#@+??BnY8x2r05^g*)%8-2*l(hH$efZ3TZF0_EiiS@gjTw}((6w#j<&QEhy;N3{|M zeDw-3LfbhoI6Q2dyUl@aS#z%H&Gl%Dv|ezB^4zaE)84XY-9$bJM_MC9a^s91j;%D+ zY5w%6M7CZ5elhq}?;_&>B(WhiIp}t*AXWiD4)oV0pM?1ld#{KtD*o>`b>uX(-WC31 zM}KyEDMA0&RFwGj>(}|a(5!m>4lSf|rD^x>zB$PrX)oT<_Xh?W86* zx4wNr1e@K(IYC&Xr%^dQ0r7UWLzjegJERxvq`ERj?jZWkmh7){JeUediE_Pgt!x{? z1G9p%^GmAZxb~ZBzottq&wO>vK?|Y6pfC|u`>I{tT#yUmG?7Ynwb;F-XN6g?PD616 z72W4DPNwMxG9E*>cBz$k z#zcH6rs0UJcdn`@Hj?@!z1tHHl&&~^V6F=-rbzunA9(yyzkHDu_foN$i#A+@CR-gd zby$wYSX5XmPXrW0Mp+SCC~WT(e{?0GVu0kk!trZEZ`LN6V(Prbk#xsVSz`^i-}G|0+X(iRZ5a+^53X1 z#^M#x`nA`}+OvoDg3wClg9J;5K=kXnd!Kdz=sLe7{IR)78Bj7g%?^D@epX0T!fT`M zse@v`yI&cI?BBw-zX4tvit1BNoz&`H4i!v~&{VB-P-Frz3V=STVTdG$jZQc%g8lII zt3X&1mUy!Dt=`T;U)+QY63T@Y6Z%Z9OY1M37bT}T$52I$eT*9*F~T*cXQ*k$$G zp(5szoivpi7TT5lG`xK!hxMa0N|p!^lB^Y-a&TV{n0bx+8uwAziOb}Q8nBgp;H-r9 z7|+d)_7Yb&bngHPVQ7eor1oaE(s`u*Wl2>9;z<-4`}VoSXegOkYiB|LLo#eDuCtdI zhr0A&E=wi8MM-%R-R0yYutteI%4~5j8)`b}mLnyqYCvknhnzhR zRkgKlSsFZ_-Ih4QFVPMal?)jn{%FDMx;#P=L6&ciiD)y7K+9^-DwN}9L}+UmVIp37 zN8lcS=CWeEYL8#VDq6S%CZ3eVZV#@0vV5C9u!WHj2p+2H`Bvx z{nhw9YeMc5tkAZn?|taKDzMN$CGWZ_g}!gCF`t#+S$Sd;i0w|9w`$f$4|;ZldVog*C8GpHlA?cvM~*y zP;4c-P!#0s{1T|W*0NWuAq2#~LmMPHnN=Uo5{>nthB%84hO>3g>yP;{eEShuI@U6SJB%7)^YR0P3K{ zWOP1-DHITx95zGMJCS+aAu}3=JZI#u;5+|S3N`2)Oo88 z8qky425-*dY!=TiGahP?w_+_^_}mi(sFE`7iv-lF$_!bq`A@+_Ss;i?*HWqQ_oV?j z)iV3q>vD&ruM!VR&%OX;8h9p(k$$?OqNZ2S`Z+(*fH6&sDRQ`7afpMseIW?a@5^e&4+ba0}fIS2@%PZDPxuNjGG?v>r`)j?s)VY|0e^i`lp^OH~a z&Tz79hQrw}MX2?7&h{j#=As6Sa!OpxN<~&R>s)~J5E>JsMreC>4xt5*pJ7C~*I}!m z7j(y#qeYJb8DX+3n{a{8%Yo7L0psBXa`5>@A1Xq&ST8~nS<3{Z3|MP|fIw{;R0E_Z zV?b$v)p)Qm`Niv(fuEexGq6HzPs@8s!8HoR5{||h74)dl`vc>qS0}pFv8%# z3oJWpS%S@g$4%@HbpNbcw}>>S3?^N(X=2Y|UyZg$QEh?JtM}*{+-%@5l-Ubp4XOK> zYF1tVuSk3_v~WJ*UduC2v**34b#j#ii`~u{^MrlQGO`%u_%axG7#$XG2M7-^OBhq_ zfDs8jn5Y)}K#(Xp)p}AA$pQ!kXcxT-N4DH)sy4FuFcAhSqe2rFBF3(m@^`7gLDepa zR0y0lgs>99nujZ)2#c!u!&#mQ0;m&CR%Pmd*GpbidH@2n=$5i*#A>3G18}xBL5Nm* zzAz3*>T_MKsdnp;QrIC65qX|ez!Lh}Qrl@8W^_l)an~=fJ&?fF6M4jGHz6F7;JrwG zXpJ9<)M)h{M$^9zCi*xp@OM*Tx4cI|Cc=7J{hVCUNEzJ~Lmd{q8#|-qzMiJbfDukn zRges9I$hzaKy0}V|0Fy|mAQ-tqpDfFR!}kzqR!mdI#O2Ob0F0_OvQMQ`ZI~ zJZ^ObOyD(mSZ;v0{<;vGkHezO&N`z$4S4KWYG6K@u}}vxg4g&K6EWH!0;hsO|| z)hL_k!Hj5eP@}w{gNr zqx$D$^2?LDa~E9g-E(f8)D>b~WhZ}D`*{0V;3t5>W$hfT5ZDddIm(BP%t9QBO z*dm=8#nh#c9K4vFt6)JK7CG0`7%us~NRc3M3KXkyzFyEe=|-gX0kpd?_!dV=DqAvbBxoEZk61%qA5ifeh`W!E7Qj|B^7yTb&9Fk_ybB z&`x60bZl*l;4CX)wQp3d2_MBa{u#ZoDn?JAc9Qe-Kk0;ju8oYGb?F}f4z z0QC%V>=eY@t^hK{3X@H&z_S0OQs(C2hI=h>9NFMgW zPO)U^BULff!?{tPKDASJ{gfVbG6R*W9r9yB7-Xr2n_zX!C$Are*Pr4|{yx0@?zBUx zp!YT}#P^*jDY>Y7gQ9~&03e#4?r`5z9%6uUqDyljhl+q3*tNHn4t#Fx+JMFEHkgWK zw-%{3y7Nnh=|JQfsv;VjylFC}Ipb|C!6pilrH_ z>2lw*0~J}o_N1Ts3)&?dYHp6?&G3FUEs!KQ99$FPEW@b=Q9!kTySv80+W$LJ6WeYi zZO>iETfp8r5WSLzL44*&ooT>+4;-U1O{tMifLy|DG4`ko)kfzB2 z-QXzns>FaID6t8L_uvSTa+ryIDyJe0(U~l7OmMj@W z7-e6oJ2h6)aQ?uwqCbzZ& zCAO{YX2D@RpwOkJIlEHRMbXmGZ-hj&NWtp-xK8kV~x~S_lp5%Q2-hkkxOPG{`5X7XT>Pcz$GB zZhkWwK#!T=iF=*4FZ`*e+4DC-_^l?)sbpCyj~%O~yDmR+LidKGKpuNZMJKeGaap)pnL8HUP;f zL%H{sqD0?)T2qaVE47=eokT{}VyD4Vra!DIwq^?)ayfA-L&4g*_~&r3rFoG@OIM{A z*9?E`JV+j^5uqfXmlMic-YR1A*GvI_Ogu&GIW7};fbF2;&X{ocO;gd@%a*+zy^vi@r7T!eF{1%9{T%#2r%=O zBx*iP1a)KJu#n_midPL3`E{P_^sKSm`XZA0x2JtJ6<2ly%o0|Xq4B0K=?<7*{v34! zn=$=)Q}1dHo!TVXCVZ&Cua#4x_qf=PI&pjeA`_J-w)=2N2XY%vinY%%_$fltZCP~A zrn35H_b_gE7&bk?fzUAx#r^8O>uONDH*|*bOCPJp2ddZ89a-p5nP{O-9I05g>_r?4 zSOA;QLxYc|HYp$B7UF>5?HSP|p9T=tTS-#rq{jXsh*&yVK=(KXt8rFW>?=Mb#$4n-JHL?Eust-Ok4F`temcW# zVl)6(uPI@Pv)7T4va+@Q(L*_}FJ86I$~?1aKdo-&*uc^0yxiQcpG zntP`m0?ijvsJ{Of`SAOEZL+lpm7B}t4>_?x!qp9)pb)J1fs0y!!kvncNkZX5g#)SM zvR5n{(R+arS8_)2Iw~|Xf8GQaOO_=0L8-n89v%s*Q&$f=z~n7tdQu6S1;7{F&=QH2 zcqdO+o*WQYZWOwCX$@?;#co*x*33?tC604{XaW?dR97u3suh5G*me5VcZk5eW5&|*;-1+nO#uSOM0gB3Bgldd2W5l~;8E;k_w88A zeXZ4oM&gQAFIOgf14f>LYIaD!FJo4h6m?c|>!OTn6F^64umJd+O=dgi-egidHJN zjW}Oaw{_@j=S4g$m*3J!K|ZmNybSF^>0gXXa7Qi-X6SXs19Bv(;G6+yWD!THhWi&r zKAyB^^i^x?mrIDLm&d8l2a4#zN7ivyb8FoCjltjbex`@J3MqXXTqz}lj;x5Vz1MB5 z)}Y6Cej})YapLm8E#^7^4j12`8emayN+nYR*jnur=<=|t>njf|Ditq~XW>RpDA!50 zz>La1T>`89!L=}PdB^19Ea84{8vlE!^XU;03f3Orm?5`t2&-x$NQKY3lqDsbEnI5L zu&+)4GJ$#mZ_U*rE$GV|R@K<67W6EyUAxCnqgb;XC5O}Zz8C(&4nj*dkss`X=Uh4{ zrbv2?p0xLu-v8fkKMs_<9A(eK7bL5K;66QbS}nJa-DCg&B{Zs1v}7KcDC$svN>Vlo zI()&8;oINH%Ud}RHYwdQkE1(`itKT#NIs0}?C@1^_MOl)9H!N(#s3Rj!8j6P{9Q|I00EB<{?f<*j?)Y=SPmn z0Bxn)aW+2`DGcxRfhls47X`AG@&|N&TvZ>`j=ih^GM_MhMG(8f^~!)iI@qxw4+miC(T(Bav2kr<;{Wf8K0z%hHs4pqqLGmD3Bfn1q2r2kX+-@a!dT~gvxQK*uLzRsZ}_1AEfG$s;bba!G@c%)Mo zHz23yekq5l?8_F~AldB04|<6tJMm*RJTwZ4k&{n*I3O~ONyCt-M9Q9!6y5bmkUckv zpG)cj6+OG$!L4l3k`Kdf9C4Th9lgmUP441JwJ7W-TNG?u=ub%=gX|Z!Ch+5mezvb9 z2Kwfy+FNSMm(9-&5B0^pFXFW;klW{Np#udBIq8}SkVwQ7XP;yX zDbZ!&?|)WVvFcReZ5(zn_0E3v`p2BAfQI%5LIqwTwYX4i?XV$kc!x*AzF_r<@hpWd ziL8sBH?&zqEgc;48gTn2GG)NUYd=seYc!QeXd@EV2HBeY1Ap^3;cx!NLhrT`y{SOQ ze3P^@Tj1)m_UodFf6C__r7RraqkU;O&i7G?r=3?Ja*(9uv`Uz{JPg&EfeH+<3ok3} zKHIFMs;Z@=qX<{uska9ia!U)98wlvTf~my9@lNZ?6~}JbL=r(0)bvelO64k#YL&+- zFEeC+Do6k?jG zL_(*yESk}%scNOdf_BqaSgcVrX^EHuyW8PXIrDAY2UP%9m9v5EnM~!ac!RDAx3u#T zXQYj!FUZIq(cm_hOzq683P?Fg@~D8(03QxAgktLKn^|NMBnwL%VYf>lfjX63s#CUm z0e5IS)ihc4qXQiP_#bf3?WjQ-;3$YCt!pPy?GHtQ=Y2bCPao}iIs5LpouK9Bnk9kx z8=92kj3OI3^}`g(O1B>7jj)&JeQz{eb!63G9?BysyK`BB3#>ZjV`sDupO^ zj2iVIVQ(vD1w7Y{hv%1fkUp0h?|1J%58wR#>6Q>Og4V9f{)Z9#MBm;o22TFmD!t1{s#W`TF;gf?gswX{F=3=$Wo$9^e9kX4uoP{_F z{iI4`Y#*vf?TRpRyxM%Q-XFz5rbY+0J$>gp-wEIQp5$j+xCWXffD3+1<7#t0mb) zQKx-3Qvm+x?a$#ggS`M`eCA1R4QkVCkpOa4+I4T}%sAUi%Q?wuDD)Iyl+lFU{Ysni zu2CMMp^AZ+oH1}G^Xdspm^VAByC5-({bjEJ&0|^oQXSeHzkK_Y_=F#$#nA>W=}w?K z_G`;Nh3Yt!WjiYPtmvC}8jU8XId>O#6iz-mSY0T0-d|okO_dC+>C19OjW+fqqo~_i ze4bUAosHWluZg`V%V+~qk?@yU6}BdTtK{v8wvIgCa?TFPtXqBBx%t$_~>^4o36 zg1fG=S|Lj#RrL-sjD&|Gaj-&c23Qp5slkYi)7vgF(6muG&WBEU^S~^g+(>%ny#En2 z3N&C>wQTU{$W(1{LDpw#RP3z?jMdl$9H3!jtri|OTfu@b0a{xk5KjP_)sd~eI+xS< zp0tQrm26TAM1kcrZRw>{HM9j<3F~MoNOEn!pRk>k zjdtzY@OQl`AXJ@tk#cKA6xPud8*2JgRcbi*{v$xTvhuX-A8mML$%O}_C>1In+XYnq1Q)^OT^)`@=A#_3cjM#Vw_E@z&Lg9d5tLFj)x?^Wn&(GnR`t2EkDu*yJWCLy6 zqiWX`J`{3ocO^TjyWRSwWVQRIRyg@@wiM#oO~EljB6fRLF$_!eC)q(Y>ke6X_p8)Y z5*$ZOt?XsX}Sv`!@6&~s5~xiSyB4;l_s^%FT<+=Jo-7h;_)qa;u{}g zG$$m}rLufs427dD4jnl21RelQ0_)YaOs4gY<`Y817mCW5qMVOVXV^J!-eqrlHiZpr zv)pXtb*Ub_sQ?mkG%3~obxIy9Z=k(Z76$fkVfqC=I2%QPhj8CsqZ&1%nYu+QkuDJm zm!SOZoEw}|rHX}4`;Yc3nv1;U)n|8g`T$MyJN7bvXFbdf9z#+L98H>5xe9q4L?!h) zY~s;+)s9GYXPzVnV%zc@YBBkVgFNIjF9v8Ol$_jVO+F4(uZ|7>3n+;=pVv+29fV>k z3hwk2HI=rb4gMsHWJ()nur+v;tQ9y!$Ywg-1D&ZB3;{OA^vR{+G0xN^kmDB8v>78s z&@_xxUC&QBUiV~hTi7737haEoIS=oKq0-<;OtWP=gJ!TEq z9ZUdi?;55AstCZ90C*|y1uU}S9NE8st#2pC`=7!h>t7)>v9T4`uPXUM)A!OM0X@r4 zR;lDDX_C@4wy<^^DpxE|g+i_FKz5r)_1@grjr>LUph5lu^v1)&Xk+dRurx6vYs?kV z6j)+K_N|;@u`g1Odjlc1%~8oXzp4-PGS5p?TD%Ddp~7|j%k!WdoMwRf$Spl2A3Tau z(`Ph~rPgw>vLbg39jSa0^30xh#A1cI61J`vbn3DdcIkKOuBdbq4kL_1$|ee`XuB<1o{S>MQ9<(-3K1{ zUT9bV0MYSIf}Kjx4jb#OhG|YMQFC<1Xa6p`6{eynpA+PHBl=jLDHt^<)SsR zsHU?d^J!eQvor)7sB5%ImFmc<70kwCoQ7I}!%=O=Gvw;TIXaIR~0 z2UOV-^#y!(`;Gfh&Kg;McLn{sWLCR`Cscp6&;4tS2lh;EN-^3)V^wa1qS6}Or5+%% zyyRd{aZ1R}7XCD#Q_m7+SGtNI!1pc&@Pb3v5T}464D6#q&2VwIT|_K)@KrzS_4im6 z8Rwc`CplJr09PH}H1Owg5mfc5x};80Mb-RqU!t~RpYItxyB=}XSt_#%T-)0Nm-{3K z7J>Dl9!|>U!~hLJGo7{PbuH5Fn0iFzqR9F~4+?h4Npd)lP)LT803czVxZ9=9uFOur zkQY_6nu8C#F{7~U(##cm&cdE6=tJuXrea_PTlj!Y)x$Fi*T?!UN_s2m(ekaPC)zjK z;nLwGnNk6%b;w|c;>T#=8d?c^wic8F%f}FsD^(SR(_i-RlB4GIK=Cb!?aH zkb%p1g)NWj5dcB3-O+R;t_35$Nre{c#H0C0UKv4bsfH%YHrtt{^JDgWC1E4yvgHm1E1w0RfxF*VMW#HyK*dF&sMblb=N%3Vct+0Zy`?5c zCIwSUly2(KQrh!RuX#jq>(?FImGf3myU|-&Rcz5xp*GwNis?GG9UEFoF0|!DL z*a?%~NJc5ZwV+X>Q?C+9F4tnbvpoZ+^_j{CT;vT9OTuW^W*KhKqom7cAH z?|g^jHmFv!1H6J?{UZt9+p5YQ9jt0>$vhV2h)RYvW#{KdsUq)TeB}AW3N-YGY%)JuOUsS@NI5OXXV#umB3GXtJJf<0G_m{JQk+Ib2Cf3^PP)fgs=Z?J7He@3eKZ z0<4Zy8aK7ummQBVGH>hBI(wYKxW5SAwZ&A>lsdkqWXIR1XGr+Gt4Z(=So8pQJ8370sq!G})u}*0k`a&lLv35ee=(*;g`nma$g0lp@ z#H49~X5#V`KU2SD%8SVDy#+4>*Fc@822wVsIu|_~)d^{V1J{|;mIpf$_b`Yxb$r>! z+G@Gj?|hQYbnS^oU6B}g$Md3gr0iX*C_D`uXufk#p#CXS=z_a*#vGN7_iFT{&b=-h z1;w9>A_s5YB|vNXaZ*pXe2w5)!OU!u+M{|RbO0#Fm4?b7aL~M`(=?h^)jg^skf`7a zjkA|Y4~k|kZYWGeGCWjO%(|bSzWo_2>W^Q)lKrCIb(VypqLqw63SO$~sc##4chM46 zNmFc8sIpqjud&NX&}@T48BkNhw7sZ~Qg|azL*(fJe(i_YH6lVziRim@)jE=y8^E?DvQtMI?u;k>cs30n}jOFyvaeTp2Y^{wsI|x-k9kQz3d;U4OI=kk^%H1L z;MDhCrZr2C{NKrD5^)+$gczCLfA;#>+mB(L{rYn(h2OvZ{{3ff zUmA7tPKoq&40KLklt){{DP>7F+M*%`dqm6U0lAS~V(g;g56p$1AW0cP@9r^L$vzMU z>(M2@sFvWUCflAt*PG72Ol9)Z{|R7I|Je(edKQ51kd|CQ3fhwPH`Y6P^RdEyOn_;j zny@#d)C?f>JBOZ7_8U^~EW%|&sftN386K%)k zQ)o@u7?iH1r@a2O?nMLyB>ViRo&iAv-1L^zG8hReoFd$^RHfDWs9`rJ7jaj)ETRb; zYfNkUWt>cH_g;NaG=*J-po{H&hw&2ovmIiIqy(q-8cYPinr20dS)%j=@CCNSPN?dw zY5<9o00a>|*cy0aM4r^s4yrX9B|Us);EC5rG7qv=Ua4~CV=&Gpd{x+wa@K;*Nbbr4 z$^aN&oOZzf$4b=hL38`6s8pSo9tyFWBc(3#s@iVEK-DUmx^fAz4T4npfupx3zklIu z3x*$VmM%$N{_eXyLQw83>7}(^c2#%=;OqqhdnN9Mn9am{J9pAu)uRc181}L#BiG$hc`G?N4iGOE}# zgj<*!wg40Yq$NSy>jHW<8mb%W|8pYh-wJfg=c1Z1_)!d~J5i@OrKBhA2=224J5+Em zwyXvmh~y_MMRX-Xw#NGs(4DUR-ggxY8Y*}3c$Q*l@XDcu(Rj=fLb3b6eFIeITK3nY z4wG4$59kuvT9|euuokT`1j=j-yMi53yK+2Q&sL7%eM9T^f*foi!m@{KKA$*qr+VR` zD#!X1drhFEWP=M_Lqk}VmsiK)LgI%l+&+8@{43u)S4h1aX{vdIOGL&23{(qIkOpCL zhOSWYPEl?%7bhOrS@P?d#vRxy=pb>yZnA`6%W7Xt4#yw8=kaxiVo^n_uU>i4pnjID@009>uH^^;gfg_bVqKijO0~>-JNsm(PkP2{dewg+-ByKf~7)Wt- zB31FrKo`yT|MK=n>O?;8rJ&<#lFRML6D( z>NcuHV}7ZqkFIHasta0-Etw~@1Sd4Wdv86}ioq_u0_+#4Rm__Yv{A*HK%5`8nEvKM zE|c=zed9?D3-S$KG>1iZ8ptOpbQ`7SH-cww#_htF7npsZA+bDyI-~)`+6VD-6{%7qjgEzm!#ge zf|O_q8UgZ>k|Us1cd9ii`CtlKlj#*$biOHiN@@KJWc zCF#nWw@pCvS@DBbq~gpuzFsQ1?cN2$pw#Y#I#ii@8)!D}a0Spp z?MA-IQA88OilBU)Ej76tkWD!k2kpW!r4tHrrRdtzlw1kI5er6LigYJk8kOwn!M4Dv zLozr#V5c)()M8#$n-3?%Z^CP?!{2}M`iJoP+Z^?FXD^$qmF+pKCRV1sg14;}hIvQ+ zB&s9wGN~yONN41>4}mi%2G?uvw34fw4GM|o`uV`m189S_M)cd#S&sU1QDy^n@Urks zX)gPf3Qw8#CUqBEI@OTXYBp{Rpm~s2rtx%_Fl^#eS7!Tn!hT@^Kor-8O}g>?I0&go z_yLN9Z#ONyy!C3(@~Ucl014%16Eus%tP%hydJY7mP#u7PoyX{^QCYh7fS;HFJb;3| z+X`Ybi7hopP##f9wP>n1zv66>(Uy~%&S>`#6KW~QfXG%bKa#Fq{n%9oepNr6EZoDA z@QRrxaxI(F;yCx;3^7jG$d}%A-XN=K1%BTFJ9D4ZsN5O7XB2%d`Nig4M&n7TO!ygM ztfOg|8p>mxUNWKLAeOIV#$qXk)(s87|lwiTwLI(O3yI2W!}jv@D!Ir3%_=QLMS=je1+pA)`~ z%IjIk4VprO0WvXJM8lF5sy5dbEfW&7;#);Af$!4oakI{ZJcFG9)W%r9|4{vCb2U8l z8oa?#@2a(we+m4+Fjra7}!f>W|IBOf=1;%6mjgfy1Oz-U592m{huL|^YM3b+QFzlsc%R2Y7Qvb(Fof7+^* zE$&s7t$5}R7%p9-Vgi9mfmGbsyPDurrPP!injEB2>-FH zx?hCXKcmK%e}0Z1ge)w2qkVu9aml1+m)(GH>&)X3>Y`Db z9EL+0YZ#TuB3|g2^+#)U@2*LDRrZJkdpS?Fzvlh#&eP8TNkF#06*XhAGt)F~jD7Za z0oy9=1BBv9=zv@ak0oV!LEy_OP{;>Szz+IR&`EIgC9o4V{2mTM<#jHR^G`|Y=7w-n zU+Ho`yIY(@pa-yG>*)isx%?X%{YG$TEeK&q1KY+;NupSj*GUl=j9D1ZBw(RhG9J<9 zkVM8=RU`A;C9wEkytRb(&PvrQIF1b`748ixK5a*17>!r31_!)=Zl#nX!%%g*l3?w! z7aLl|=W`m4pi37KzGpAo{M&YDyVTS!hZglBkFU9c*blsCUI$TK2IB3?3Uj#(EPjFlP#YcAO?Gel@S2MY8-4FfQ1a=#i3fWtwfuB>6| zd;j6|5-jIK4c0o8LJBp9SVP;vbnmb~rZQ3oDi|l!1D+@ykZy9cn|LYKOuqkDv>nj= zMOpzoo3_voJ#aLFjRDj~Y#%k3v)k9p@;1=kIS||+l`j#|I(Z>xW-ShgY79gRzk2Y+ zKVZ~C0*KV(hIfNN4C4m2qz$-v3Y>k$EBojyn@KA2z~&YP27MW;SL(dr#AQfC{r2?! zA74Lv|Bt~iNb5EbV6=|v^qB_lZc2eChg2aNp(~PyoV$Z@&u$GjK$h5Tz6o#az`2nd z$#K=cynP7~x%=#zCJKX9+>Bx zRb>>G#CLZngNmt6AMs8H_YSGpRDs;PoS(9En5`muaT#ZgrPEVrW}U{yBQ*?jwUXe~ zvP8~)=dwy~l^6T(!`sgtH=kd1MkjWwntiBBU-@c9@U;*fE=do{iU`brbKVy2gu@v| zB$Alf0&XfW6${`}?qe-)eTG7+JK$jmcHt0P;UX3Q;4BPS^w5t5JbZ^-MmFb>oTYcX zlY)tDReQc6VHa+mAbXX3j=@8IToNUUo^4LHQfJ~gHLgJ*8^i8ty2Me-JC(@Q@4bZT z8IVUkO1VZ9ALMGEU?;=n*YQ*mTBoA}9#c+dS2;|m{z65m#fpYLF>0mo+y{8Ruz&*G>8M=$vd##Kn25hz#kf(2rvv;o8~ zbpKox1oS?7GAN2UTpsP>Ax=&xPlPp=Y^N1?={R{6w4ej%i)M?Z`G*mVqbscQ7iZ1v z{Ny&X>rNLDcs%u@hm!=zlGA|`d{Dmm&D+n(D)~5n_a9z=2ffzc%m2Ryw%)`-E>Q8k zR173{wIw&%wO-z&#G#8*j)H;rvuuVPLq0TVm^E^a^ULzpufp4J^81$piFJLx`x?>R zEDJ62EmDXJEEynZ9Sx71&&Ggg?2VG#YNQ$uCHFNm*)AF5mM<6>)j6JYj%*_i^`UO* zcam1lCuLo1==2P#^5_U?Xi3Yxf5HnI&u7-?zGGzu+c>i!xJnyYnnq45#U zZ=_2gkg7)gDHE62gUZ7QgxRg4&L%KO?xnF#IGa~u?WK^yz`>w1i^@R4Rqw$IoM4vP zln!W`GEaaBA?%;qbW*QXr~pna6!#=i-0CPb1rHPi@cV$%1t;xMSdOLvUJsMvUkcEU&|84n3S_MjD`0a7Z=vb zO&TyQM{0}7I|O=u88Ud-hJkXrl&-+0IemVMP_5On)k)$uShwhgcwc*^mJ$xwYo9=!Atux?t@&Q{e0^$W=mICwK zrfDzD*hcXncEBwVcPuW7N}{3R$B*AWdHWKgbgw@O`r%^)LBayJeN<%=Vyd=7nsKS# z?ZKguig`>vO2IMW#~h5pvA4~aBzLCbsX5osb$Yl?7~UpNjS2#xb2^ksKYjgIN{Iea z(vh>QhFHofiA83EnQlQ1`0`hMcp4*6;di9V7y)wjtB3O%vv!fB*c-b8WAL(Sf=AM} z(}xe{@CLVATXb@DCIlD42E4K^aYHQLSpn^RgmL()O@=tE1Xf_w;fYX%!2^IKeb^yn zfg2`U&g{5WKJL~4dTNXkv0d87x09&1&+^i8IP505P_4Y_k*FXf9(LNs`M|ca!|IzK zD9gedyLY;Rf)&bdqN(jND3$VP?AJq~p+}I-cK~1wmcm!IkRa0z@|4R}Xez)C11z+* z3UIsAEkW&~GdlMw*Gl6cNs<8u^nfw$KoHumpF(n_S64%=G-{QkQTgdbfw3?-|hw-8dov4$kG-EE}YzF3ItA z^I~1)ta~LE^I5`9?R55**tyEisPS1ANJF_JQ%bi|f%4i?iHApJ3m({IiViO|3X7LA z_fZ5hn{J9>3{aUa;_wwRE_ml9Op%!_4V43#E2_AkD>r(4vYlq!dRtj(dW_!7lLSx; zn@fKKOwZaHn2lPEJ;NvFc9i*RZBp>mkE4|F@+TUA#eC%q*F9Ak@wX=#oHJpeN{Jbc zu@N8*JfckRIp=Sez|pa!!f1Q4IF!VFKdB4XU^!`>kq-HH{ng+7UHH4dD*O0f>0qh2 zr5ypRSa+^jRGZBW@IvKQE_7?}-K0$}OJ@r?%HDK1lI}i>KSy;waCJ}EvoD+pLmxt3 zT<&ZBdcW5DsxYoV>*>9g>~||5*7dS)1*tHjPaZc%GYx*lX+Yj*k3|Ys$eZ^hxn_|b zv_A5$c~tk{=>B`JE&Y6fFnyAT56fN~?J1IoJr6~a=m-FujZMyG^T|tU7paaw+Yzt~n zE!+ImyqR@^I1{1XQVVGHnhsj{%4~l7Jr_JMAX%{*(=tgzoWZT;jBNvBZZrmc6Cu7? zJA8K`1So+4Q`lC(;b;x8&K1z{73&fynl|g!d9V^v~Fm8C48HnvEJuuRgnB%#l zL@t+G0WXt9xi)W-h6CjwpI0F@s|4zu&6+h+2-Pm^#R zj!Z;);T-f7nuF0e_Jj}K3gj1M+^J{N%z4L>kSWc$4|+n zbl{1zLX*vZz>uLvSan>V%k+0rvtW8;;PVJd?l6Fl%Ny?psA}X7{GV@WL*WW|JkjbG z0iFjELO#t(=a``Af(LkyDog!nVYJcsZOdUKSjDz$lKlR zIZg5;ugD1O@C=WK9axgqLfWr2pPd#Mn>;SaU3FM7^|Iw*_f1_jw3PGc(?UEzrX3e3 zq~Zo5NF{{8m5#vkbzf)nyICM(RoN=-TUjM!%BekhUKE1^=ptVq7$lAm8659@@UHP1 zv-1-BE%PAx-){AfF`ZlzP1mR5gAd+-5=Iir;Dxv-fM_CkKvBbntK2l612-2KR!SV@ z2vC;5k`6K>t6-_~60R$PZ9|pHiHdJJ@{@ndfb^2`e6R7Y zB3|xJ=sw`e&MikS1Q8$WLBLD0SyyPLbS|yd(qpm0$Sm9sG5ZeYic|hD56`_1HTz9H z2ZjnA^f9nnoTz#xY>#>eB!M?2K}G9_Ts|CZA5hK_Lf@tx-QT*hv2t!D$aZC`QqBU+ z&8>$Z|G9PT?!Y5Sif|&l*nt7YBa?}>I>+sCsG#9wdm1ELmOQ{ts9v6#!3DLSwb-Bp z$ghPCndpe;2M;{g>nFkFM60WvM7I0{d_Sxag@tBENSHTi6a~sl8n0(TDFTOuwIHN&Sg4XAv9-e z{wi6dwume}e}bfdAE0eK@T$;LhEkC>842FXRhI=WkcQ>RL#w%A9;)i07_8UHB(i=U z-fS)wy$`e3QkJrlyT;+BDPndMP3}At8FJ3yZg3PA^ZkDKi*K8qu+R4A9D06w`oHuq zUagAMQ3n}LnUAx07K4uYIqrdj#&ZSVY?~@I4~!)21}x~^E3+y+j_8vDsiO_oQ&?T2 z!K@l9CssFf^ICV+pmpzFc+K;Hj8My|Q%ifnE3$=vkA5+d1nhh^N{@!T6f27VZoT5% z)ra5~7{miwbgOdyCDj?#8JTPh9FR$t%%FGQyc1wqK<>|kJSeKjptr*JvFLS3#d|%y z|5c z_cU0|_nq&A|CG-TZUfi_66*Mgh}#QYtkedt~5G^#o?6$ zu;fgOi6Uj(zf$O~Z0z;LIB;=&<|TRS9jHlz)$RwU9dvxm;{$y=c2Q(?8W|3ND@jnu zafKJA6?$O0Y|j22Wlbx3w`Z@is#ln3u&y-hFev>fktR57h(F~7jZ%-6bEzY4REL&Zc0b;@2 z4`|+yw+q-?OZb%B+MDcnWRf<9kiR}c$BQ@2*I?uZC!i&K)k|jV-qpj=01AlJr9xqV zRCC^Kq?6X{AG^C@ab5B4NIn;?Wm(%3lkg?Y|D7)B z;5U825n}bjJ#1>%rej)Kx>S$s$|qNjJ#6Pj4c9|tqT0)2#ND7sV6DCN9Ojk}H{$|< zd%YjK|M7;9S>iiJbX&RC_1=W0$AGzU$vRC*3^6tAY=faEwHE&OC}q|O9v&=8)V3Hx zQg(6$;D1l|%^o$Rwc@6|Oy?v`EeQJ9oK&`HF@5=2IbZqn zA^5EGA?XXq$#xVzIA&I@QJi|6jXtmE@JMuy$%uK)PXYO3uHE`s2s z0jbv0P3Drk2ybw{8u>*c!i!3ma}Ft4e6&j8?s9!NWz50DzpgJPMW?MzYmcGP=vqG$ z&?JP6kNjP|HUO3~iKJ^qs|95KopOU+F?y9>!$Wsbv|#JcHrp22rrkr5^T+PTRwx>3 zJvkzY32wX_ds$_d0woj)Gc7NGa*YUKn?YU4#uq$wEdvz>VW~U_Zhy6jw z%-Tr`9MbdwVh&JBDpHw5;vX8~DW@L=5u=o#AafSk# zO{y~y6fJh*exJj>O2P?xTIF0Csbmxm+i#o8g=Ipt_XD$$5k`mPRLVZG9Il(#)Wnha zq7!6WfM?mn4eEE8`(U_|%Sbq`^==ENjk!ufffRHPu09Z(nX9jw$arhY)%?eeX{6@& zRm06T$r{><9_$6Uv`7#wWLI`KSZcDWseYdA2?o?p*4vf`?dZ+afCSmlr+8l+Yk*4A zE)Vihk>Ix7TUC#72GEl`V~MwHKQg1(wFSNhi=-U}QfEjJb_4uAw$}yk^lCe<+;E7VK0v zp3^les%Dx^K==Sm=d0$0o1DK=5I2NmK$7ojkrK3H+Sq_q11A&=et6Hr-OeW%Tr5zJ znJ5w)?g@laB+XhqaFsXi9HvQUpeDSE>}Z?X-u({)kisEJR+O6w`K?8`kgjMF8IBAl z2w^_Wdqk2a>vMqmSB@$X*}*B>Vbg#kIrkL?3E=5L3_LAf!C>&d@oYPT@7+m-IF2fMXp#@!?4@LsY6`33Xns{KBW!-B!U}3~ z1#O?%VfTg(KM)LFKXJV=B**)N_9^nI()xhMs`{jvRvdzQoK(D&#o`by)`;3M%buY8 zyTGn9&X-`=WXUc{@YmuK0~W1B=$NP!oQ6Ep;p#4_`y^!?D3yLqGjsDbRkOEs6?|et zZQ#J*%CWRpU5P z6Gi3F*0s2$AX}EhGdNTu^v{w=HW>LvZUJ;gOoaKPx36Bm3i(6+Grvc7{+gu%Eoy&b zo>+BhXf-kD;4r}HjpRUQ%UTw7C~oDI2nf9OXqH9NH8R?TUZm?qHCHTjNm}Q!XG_#! z9pON_k#kc`X;)JG5HSVfnPaShFOmTA%@1v1XepmFNQmo6GNk>JPAelTY0|t8W!OGR z2KWN0ExW7{HNi>oZk99p#05&-JwMA&RLz!($`LxRSk6)=0Lx}cXs>4VowBDb)YNv^ zNg$+~nZ(O?dtx`vG7%KJ%=O=Y^v&N>f(*@6NwGE+K)*Bh2xpF}eQ-4*{Yggs=m7L( z2GvOZ+qh&tlT=AX7A#VAX!dbch1$}Z_aG=fVor?TzJPQoSe5Wp0~_VuudSu-1hK^n06c1(a`tK`?nYB#(mNX}C_e9<#yo~=1!?=!6Pq7!Ub&?~XU z`4rrzPqLoCGGa=N@lxxEBM@(w$&HDI-ip?$_p#HV@3NxHd1h_4N<8R2zMidNj?k5* zx~#gMy!u6OT4)rYcEA)BUI5CgP~e-eL)OF8;d_CzU=&iRqt8{MHHzp7%?vW#U4TjF z=^T~EMNPJ1tH!$MH9NeKsc*Ft3cp?>p(S!>YKFvEs*bS^&8c5ygUc2>U4rWyK{m;u zc}YU=nJqMpnUsu233A0daUUDZLuDZW1vIXpmfhG84iL7;&mhBxA*eA$W?{GW9zIgk|+t1&B{`T41 zM?t>$myD$N90p;ktOVRMlPFDmC-h(w_-yF8T6qWLv3&p_3lJqVAw3C3ni6t&lQxjgqBr zTaP-he*zhW$Yv#r2M+F@?X#=G{9W;&_ib-vg|BFU`MJFwJm4&ehguR4#1-TdH*VZ; zb$9HkLbn4z{xUyZUC=ET>!COZNujBnB|e+* zWM+~P88=&_NwY;3IGS=UQnsAYQqTeqAi!NsSi0(kGv#W>VAE0*unIo#*n0i=^-C`& z4$|HqAMJ8+qSSTKzLO*=aS8(UG+F#(bB)2ZCm-$$;&zA5^}+sNcS9Zl$5kl!&|0iq zrB0Fx2r?GE7Nk9~qcX(ot zCqBoGrnM})o+@2v2N}~C%jqwf%$_wZCt`pV7@V@6V5GrF@3JbHoyCC1b zMaHr_qgsXbl^hN@sulixgIYJ*Gu&h_$^oEvf@shH5{l&d(5Hs71f!sd_Iyh9o=^EP zzc+<)XsI>#ksY`7u4FfRsvBbnX87Rj(8n#@o0H079JzF|sBT8Cc8!jvk~HJt#HM=D z0iZ%_KQJi%>_aIrG{+3olTRo+3u+{~#!A#<=G3cdAa4wZ{(yC&fpc*Mk*NY+Q5G(F z#ML0F1`Ri54b;1OGFP}U)DBYfa%M?O^=*?@*-SlUVHPcsA%f4DN{>lLSJR-r2Lg^g zuO{1x7`B<^5~{D?(Um1V{#@s}z0t z#Skyollq~t2!hIE58U&G_QvxJdD+$pl5*dGNoj*@1yje8mprGm)y&(V(V{JRc8pb^ zI^TCtoFw-{YV0SS&ORcz48&7%+CbaGcOC)N!+=&_j?`vUMf;tAP0l zKE)Kd9Oa0U*RRwlAwQQf;(htEr)b5KI89N7fj`*p_1*ZZv~=|4DfL zS`Jh<%K{{{<75`ZpZ#=gIdLk|1&WLw(Li<(F7A0z?d0hc$N5t5V{$C9F)=9v#L@Z95EXc`%s%?s6xwCi zB&#uIKTyj$FmEUR1}f#XK@VZ#+3v*bpc!M{gqS>9!2`g0kD$#?IJ!izd<@zxWNHJM z*V7baTLK$I zB0>zE-Ka|`x3{}aQ@t*f^AF?xE-D6rg2NL!Ud*$kG`pk;NG&hFY(mhW$0ccBbJ-#1 zHJ}53pBM_{Y~28VfD^I<>Tz@()*@z;79zeEzWZHn=Q>L_qzKKzemzhm zXL-kCS*VaeN5&1U)fHJAejO7V74TEFE!#?RGV8%9aTQs7nHE7v$NCqNynHU_%Uu_B zry5{X&>s)3ViGJN9c@7ak?X(_h`XH~z`A+Yf+##iL|DZ>NcgO^A>d;-xuYb+yQbFm z|B?14OV%XUdEov&g+pyEg6bx<*I?9p&?H(6ej_4pOcleu8Mz{XrPwlYNA5s?g{ndo zh@rp&0Tc>mnThx6^Bq6m@%SNO(+!FB-~5Nn7~K6#-;gwmrda}XSL>-2pE1c8Y2r4w zIx#Y{bMV`|ImO&hgDF8zb?%&Q&!JUamkbczbgEKR=ClrdW^(zUM8-j%%DC8P^^0Nc z0*PZ;Qxhj!O-p+?)R_~y7@n>w7o{=2dedq13`+A5sDhtg1MfCY-NQ^l4>>I*3s&rm z{RYG|T{5&79XoZOoT?;+0Ky0xmQuPjg<8&JPqW)26hI%~$cUTzl452VpHEi7K*Cb3 ztS(|p`AEx7e#KvzuKX%JR7^i7f1;9%6~nY!YZsL}m5{S(#I(5{{YX!y>rN@iT8UsO zeWWJ%s=y>nZ>7L39H0O{#lDboytckLup9ETZQ`SR>X3lD_C8&t6bgL)T!F6(lac?m z8|8v>*xnGfV?RTYZaR$|>!hshWdho3CPXFYR)69sw@eG0?z)9$XC2by^-|w$PN)5I zW*NhDNFBk^%>XS^IR_lS40{ys%bmc5C7Q@PHYcRhi0;~H_`R>8vF?=`n@(dE^g(9* zP**uS1E##K$81Ets_He9Q8{8y#=Q~(3XU9XjwQ!gMD5~Kb*L%~ z`RE=Z>6T9yaISKMRvj8l(=q||iVBg-0dAWUH}PZKCpc;hT{XL}?Kc>=GR4vcFl+kxA??vw(7+s5Wlfd^`rteyi&`AQ-jE>WAdw2_CQ zHP^ZN2H49fZY@WEB{qpBylm5|>jKnvMxAnIw#FEPbDt@P48;N6$*4tcoW7gTF=vW) zT*|w!;aPm*5eGuGli%TjWusn5TU}}F5f%2(q&A_Z>osJ*(~$!->7t9CCsIExb#icD z94v0sJc)GA&~L7F@aT1r;`)7T*%nC!j8XvD3hJo_L8q*+Qw-z^?D7;^8$X~{gPyQf z-7xm;w7j{fW%cC=mpNT+;eiUfU7$vI(gTR$32I4>H!oD(iU-_J!~d-xk!n)BShoJ7 zPyxT(8HUiBe9OyI7Qe?=vY)xaK2`|zz>y-60ci(wydTq*)3X9EjNNYi@uI0T@ERhCV8G&_jz)sb@1J7 zZ#{=+IiFP-X&UBfb#BNMhT+gW;$%93slNdnB6J1_FW8n2#1%S4^_)X)>}9V-a@f$` z0d2J~hV03U1;q&eA9&wEPn2C{(qV%nU7+l8Y-sDQG>#^H6L;r}>2?>-)gVPtK+5q_lfaPlE z!RUaso!KZdmavwS0m?$aOW&1ujhLf-f3B9L5nR{uCO|9gx#YzRQE2(hm7j}93(d)m2*Dn;! zih<+zUVmksd_9Ekq-CNNzjN9ND8osGTOLrOt?#Rf+ZL25QHu$wp%)r?GB4b?zA@#e z_@NYZsG`wrcVm z@!1ZF9sVd#T399H1JRaTx317+A2;N=l;ZHZ?9r{u|Lgbf^Vh%+-1JQ(=~rGEIry~O;NTFL@KE$AY1PkF@1&R z#=zqTnjq7TUMXC@ZT74JSFQpHk}1)Pt0F!)flG2Tb}7gpkqqvo@>G2wwM2=Ow=CBp z|ECp$I|c!{AX!$Ev1t`$?$p%JIx}*_8Awes*e;ay1~toaII_7KZOn)3wOAy$cL-2s3e+M zZ)TLF`M8|?1-2+>w(Fx$*-?1dqpDciJzxsmu){OYFx;{4*e!cazs%aD&?cC2X5jRW zwz=mpa(S{4d|0)#6N4o%rO_vhQz?5pZN@najl?SjWyvc8ToGZkpa|0GkWZ3p4|)yH zDh6Q>5U8|kapIcFw^XV)$9=Ge8^Md94Na%nUIF(V^OY^U)%B3v)i4j{$I@ZRfG(0? z^%Vdmv{UZQ+>D30A?h1|$0R8OF6ZpYbBRe?5!H`tc&naPQU(4U@}N(H{O3m(?Hv+V zjcF?zXFE>WCwc-YhccU&B!9!V&%^KkT0x29sC**os#fjjYCit`EI=SWK-Ypq$rb|| zTQ~){QL?oT%e1(5Oideeq+g0F;*cd>FD6`K-(Wf^z7%2-ThKo}q_? zd#7+=7!}5@^xDBz9aMD6nKp^q+Qnu&x2c4J5HGh)P`)@XeLq$A2R!8Q8 zT{id-Wc{dz^v2!WaI8j$RmaLb=uq~Beh!ZZC2bUVm|cSuw49z@I^p~+Ucos`%SMq4 z`3O=jMlJwl2Tyfyd}Vo)H4oB3U868C=H(a5|yt65rFmn`$?GeW>Ft;%9)5slGlt{>F z05xt|5)p{>o~`{A>$|IqbQbKJ1xp7*VySuFd0S)&KOK_)Z4Z)BIyI21FHI}eY3BpZ zYY)+P2^oBNbL<+e8%WK?X=*Vwpjn@>+DxIBFp}Oc3HAXJX8jp z64^IyO{sd#MXgq4>D~|MD{MH?2$%iIS)O4BjsCJ7n8~|f?wy*O6CUZ}$-@HXib>Lw z(w+);zG}5ru}7WMx#dWb1!Pl#WWz}N?=?@jp$0FE}e~}>%{VBC9VK|Kt2uz>JOhTNZJeE+I&Ou3cD>&(<8X$ppbG4HVP>0)8x7#=C^?gZ}RiciV!Y=aPV~ zj`o1p(wN_2uV$`FRB-we=ID-!mSm}xb3DR!hnYU=uwDmdF-zCV7QjA?*&qT=)V;yT z253-ugR+k&)Tr#h!J{Fp7MMv&onN=?B0yD|OdwPX7c-8}C-zUn>-V{Sz5rnB$L~M9 zynX!sP5Axa*x4z$c|R4_Bi}P(98+cWy|70xD)E`a8Z>fz28Q7l$@g`wsg&IX`(kxB zPHTZqP1(sSvy#3TZ*o|}K!mv6hb+d85>}9dju&LM#gU9B_DoWS!RS4B1_>=j*q$@G zfj}ZJ4IELxW0fu7V56T06K{JQegH}4xD*ex6w#6&T}7^!MCu7Oon%$NY^(4p^>If-qNv9)TAq8) z#Hm~*aWcq(#C@w%0OykYye) zqwsRbT00O`Yn>Y9mC>uQ=0#DG>6NJlF)>hze7%x|*M!w*M18EA+?$Ux)-F}P)u&5| zSUA5dcD(?DkwSH`@}PLM6ZBc0UHS+djgTi_ll#`Mvm|{4xIjhCS|Z@WRLBA45rlN^ z@YtE#Q0KX(fo}k)o;algbsAyW_**zO4}bhMs6HF>b@hM9iWctVk&HZ?rU>%DvIA7O zuWs7(I~*rphu6=LJACv0BPYY>FYF!(LWdA9KNTv`XOZb~*2U_Ec`;+}#AZe2b)_&v z3A~;_8g^8DvIkBN)Jxp`MdD`GtK;zT8b)v({U@>G+B7G1N2@NTfsB>?71<3 zHoennE!_9mZ=4ST)`G?rMi%q++=ok_}Oyh4#?w@I!AL zRn%Yy9U7sQY({H6HQ-!9?R;UBhc|@O%xThZZ+*=A_VC0;lS5g+j2%YMshA{<+S*GO z*Tbujnrc$Bp`)H_yWva08aBzIT`=yhc@wxlf92*e#P#tUsH_cT4BXiE%yklR1Y$$fd2OE*1+ z2WF>?K7%ajv_BfPcW{C+v_)($OTOlK)*a(`Y&nwH6{-o^Z&tfk1)~686rYQ(H?>*R z_ICOWxOM00Lhm`2>a$Ix0ijV0@Ms*S+#IZr5$-!lPmMYqkT@`8{r`OZipGg>Q2gfY z@817OvnV*T*n&q=w{ZTHeu`yvuv9VuxfY2~B2rHW1c2!q{{#Kw7ZYaN_I0KGkes~E zFI}NZF>rhUM%Ok;rVzE^KpbeXDZ|t%xi}{BkL33a2o4DmSJ6 z`%Jy$@Kdk%E67P(F`j?-pXCABuLMD+$t)(scZ-Z_^l1A9KF^} zVl;Qji)tyHZAMX}tLm##)e!wTexYIv@BjM0{x|%wKhVsgBIFz~BfyZy*m3Du+ z_!50b`n||%XVvoXz@ic|*~?&cp-;*6AbkSiJBjKIqYI@A<1&?Qiv9M<>%YAIOL+U? z`_Ckwo1rIj&~ZieB+~|qO=|D6(Q@OoK*xMghE6}a!JvW*-3_XMqlBBfm5FLty-<~3 z)!5S;RGZC94P;@4n!SB4N$*#0pF{NHH)!Omw0%zdC&~2@bYV;~OsBl&F!K+NDsw=8 zQAnVi%m-%Cxq+4G-$YJFt#o0xBr!z0{*vjnA4$~fh-E9gsGNItTOuo|1Px^ZuN&CH zog~G-AgfJ^V%GF`t%a(~J=BIKoEO(X0Q&{3U?>7dNLKAc+0`f}AqIT{xl1|>gv<9;iHrmiE|0FvRq)bl^!7aGnW=t;!;b=awqOL4keh@*1rRtkG;GpS=h9w zj%{w06nr6HX249k++AAHeNU50ozWDEVJsWF@as|JgK9#Q5`I zAAZDF|68|&IfXMuGlaF@%7-7zum57F_NFIC&+VL&^}3@i+&U`^&@I3qKtS2m6=TDe z2rgns#vSyWMdjixhc4Bdw8ZJwLt_2?=NI|08jB6g7_M2a(Mly&n&=w16P~W z_Ro-}l?I5nW5T%1&XQfj-P+`m!8O@Ro&IfT$51kO$K1Ek-ZeW{oYZPNW!BANDjMiY zSvWbelg?N#;YG?q6(dYnq^>`Hib1;FG0@AZvZbltWA9QPUMXpl3q#>t9|1Cbi9JOe z6UPu>PM1@tW1ktVIy3_aw2V+2x=Kj3U{@(@5uG?IFC1+>?t>zstzw7lZG!96Ns!UY zkqX3u@dC@UjVayLWovbH1AMtyvkpT4oOy=4M4)z`bf;}Y-=Y`H@H2OlH+2Nxdyiuk zTDgAs4RiL9uR>YsduiIsf42Vxez^SkAN();;cr|FwyS%T@^ma6_KK}?J7OoV9N z2<00f2lbZ=M1F`)ZMC1y>1B9Qq|pS+Iq=vBl3ex%qwDTPYD4;$*3Lfyi$qd%iRx3+ zGS>CQO8^Hf{ru0~KU2Y0SU&g6I}xx=`eQZD);gk<+fMP+%g1hCxcugp+{|ImLWocM zP=aX07_V{zPK}VV2E{bc(t!#CQe`-nQI@om(Kero7GcUNX%x%t5N@mkJ+)z_(F$xJPok|3Tp8~uy(=M=qN z7z?9Oi9po?Cie<_x+%AlP@mEZ;P*K)R_%OC^3;3$58{*m@K#A$a@f7>om z&(ar#=X7nDhf7p-L_6*b70{o(UJ-b_V8dA)q3zIN^qOwuz=G?XF9|1{Wz6}tJ>Wad zwbNsi3OL0@V9QtPnpEmRlTG|!mkfzys#@b@Sj$V)obuEd8 z0BEAha>jP{-8%g{_AsB6fX(@dT08J(ODbp0nv{-)*RK`u`T9e}+$1khmJ{+8fn0Vy zY00dplZ&Z9$A|atY^EGCe>g<%H*`N>WhHocm zXaL8UkJY(316>|#WKECByVffy(|JyXF zWU?`-Y3;-et#AqbMblur!?#`LUZy zcRFYsVMA_p?o|z%Z2ovysTQ^m8}GYye5)Sw&cAxP8sBzITGj(>y^x-4uW=h77 z*HGF#1yszDw_`*T214NOnK0c`z0H35nQ!uXL7ARKKVl#Wg*qw!Yysv5uu<{Klm#1-Pd9qGO z$3FygIV`RwN7xt2(S)KEcq|Z|1CUOIs$8c^ZK*ot^{NUvj)aV~PvyI@<)zlQK3M1m z;8$)P`IU2z)0>F5{Uoz}k4mAn4wL)|ej-d_peE4`UXcT_YELz}Xz(i-AP=Bq-MX@{ zReVGLjia}2ULRmIBp8ikue%>V>VtKu4%FfbV)wIpEJ|0y`X|DwbQ3!@a($@I2Ad}n z`i!m;3YmG?Ne4PfgaBDUroR)6Xg!fbdXwZgE5OOak_hFclM~0qy^yEI>u0Kz83{4 zOKMeAl7Z&5=_v48TAu$l|LCLoiuRbIM3Ns%e23d~OU~*i+&NJPP*sQt9ZLgY5LGhj z@M%|at$d_od~m-*NnFwCk)luauumCV)p~7ILW6lN52bnL)a^x|QwJar*S>?N4_f)~ zMwqENcEyqX(&1GsJg|*F?OX=c5eFt}guZ3r%yfC`V8|E`1?8BnYLwU1G7R&4QWauR zdX<~OaTE!xsSlRx7^YgN1Tcb)5HJ)CC@y>J7vc3A3o5QOJ40R;JX&V>Sg(MGtT48A ztjkbo0@of{WXjZO3OSxzM_9+j)x3%wp!MVOjNz;u63}LY6AsSmw(t%k50PO9pvz_` z@By=d)&b*S7WqkWB@e=I0Q+eQ27IT&ncgG-^CLYzc9T@b9sQT0p;=qKBc$1@d&bbw z=@km#vnAE+@IJOzSF2&0bNeJNe?Xt_qnujqaFyi5(s2X3&~+v0cnfaVp5cTbadgzN ziR0ZTGf0x)YSuf;LN#eAGNU~Niss4(QiVSyRrpi>nh*E)FBp4h^oY%;Vu@;b4Va*c z2?8Jl;LS^?Cb+Z+X^Zs#LW>O5)+!vPS4(tOY`T|&E8a4XMTOy+wygJLtQ(Vg8Jij- zG^&W~0*Sog7m`f5ut*v)$pbg*?poC--#!NM;7gckIq_rNQ^8K~=@bap!9Ck}VL?rX z5^b>#Vo&%IbQMiVh)VWmC^8KiTvxJzzh1?*VgC9S#H;e%W= zeMHi&xi`Z@2AfKPlu+5WDrybKd9HwI-zZ1#0PW9{S!~2XPg*UA|0m_?UxnZQ^*QmZ z-IpZw)?If`R>LqmEpRL{MG&-CtNyxRPHHV^e@MQY{VC807)?2;nQfMyNa9`>s zFU!~8>bRCF6LP+}$o{w{6>Mfr9nf%`dajUt;>9T@hH8PB<#s;5>6UQ6piM`7W60dx zIzr~vXI1EN)B2}?$v{4_3qBn=QfN}VgJ#;fA2{%KsOB%YN?DgzPl*MXOeS$x^&>)@)bVmpbyR2nqRGBjQ8l*g)ZAeQ!&V3o&Vx4}f|S`s z18e#N6#pq@arX$@ESkVV4a~mnJ{V_N)hx2Ig480UxPt(S7XIw?^QmO3ygzHtcCLlh zD62YM_XZlIb|TrO(!l6kh(EyA4ii*NvIS?y<;bo$tjrE}d0Ur*O zQR_20fRM@4LVieVoemW#Mg9=o?Npob14M>mAa0n6our_km&kLW-Y8m|l1xvOHkHR| z5o7@O@`G?*R1m?G-==R9Ky5_N!hwQ00x*(XD-#Z0iCf4<^q1kd+(KgZ}E8T1tS$n9ks0uWwP-*V+=%l5CRvxYC`WzpOdw01j?@e+ssdPu{|BpDY zhlH^V++UKp4yQT`Af-`)p0`p|1iruEdNP$@iwczXEF1!H_F$fUR*$X#F>rBXYBS)Z zo=IGRKZfO`OxV3^;61C;JW(+&+H& z8vhM)cFGYd7iV%}J4Vp?Q6|iGBdc_Gn*o#FPmlD*MXpJM{8Y>|GAa44qOv(oIQG%(jDlRC6Hs;8y zb&Jxpp+hhLYYmf_f5iCmo)4~2M)&9JlEKrDlzeA*Hu zFqP`?R1Ce^h1vKXO!YA>m0tz}YpUkY`@dRvRWi88wrPahbc7fwHeTmrk)92>D=9YM zlVNbo;1{80+o7ZpP}88hDvndh78_)9Bc4fZ{IBBufi0 zR(6qpcd(NqyOpigWlnc0XJrc#uPCqCA0Z0=M`0)D4U%_s{84kYs6A3JjCNmBf1-$_ z^v*CQ*H&e2pWY?yv_y8ohS*03#e5Wor`D8DQ~^CjU(9<6YaFH(cmfu{12f1zcc^aD zTA}DN@eY{Am}U^kG|1owt-?$_RV(y>|E7qF$$4lfcL2zog38?FMKWb5B+%ariN(HT z6rsEuEB>L;5A|aAKIYH`(AAXcraM;-n1oN{H!lWZ242k+!sLDJmz$(3Rti=M?*xg! z#h(5LRRng$+pZ5=-V@9N&@^eI%PWIaEjg<(mXHI&uZ$CFcO69VyP;Qq!Zab3lS-&B z=B_@~K?TJSljd?BS|x%I{LEOre(&{5Duiy6`Zp_(l&2GJDQ0~;)#y|6o$97kjp+&k zRjnd2K88@-G(4giPjEYDqp;mxM-y`br^wQ2zdW!#cs+n6P=Q{}V7nLffwiVSMu{CC zqU7XmhSoVkA6wr3dXv;kMoyOFkcO*)!u8}mKbXVi1vlq`5H!sVhYFKj+fu~~!?OfA80%uLqRKm9 z5abkBvZBZ66D0%`+YhNB@rfKI1(fO6?Scuq93Gf?D0;F{qX*lbz10&Ly_G@IGp7I;=veL2LaoiQGid}# z)Xc40Ir7AoLT_3n99|}gFD^Ivf*EDHlD))y1O4gSW!}+k>s&S+1K22KoNUn|+hfW( zDKR~hdSKp#qVx1XS|4EO^u_SEk~I;bsLxNAxcY?+l%3yo1}(+3wM^yVFl@qNLQ>ZX zN;u`l-M?-Q=|W8;NfNR%d)aOBM5Y}EPJ)uGffF`&47PBvZ17`V9Jn@3mp>1GnJ#}) z%rLOGc003_g6b#8%5O~im-OC2RAgd5s%mw5u2eZwm5Km?I3-Tn2^OEjNoM{h;r(-9 z^S*-N((6CIe-j|$1`RjZVm5?-r(K}7tY%F>b+PSwQf`pC4h7}f-GDi|5h@qhPlad9u7)9a`!D8SP-06{Ou!uV zvc22UTl+I>;>mtcF;8U9Jk#WTxSBxMfb+W|`pl%%bvWdeW7h8_*W55Iq&xMyc03*G z+-H4i^vFB`X_Q{VK-WVe+}`AVcEPDsRGUqcO3MEAw2NqT8U73JN0Q{aI1mf$$n9&b z#1(E6N%VyfmU}A#@kC)Sj7U1HOTg+>!N!&#PznT~Uil)Ts!`OP`EL-8MQE$a$S5@e zN>+KkbX=b0Km<9EsNpM$Evf)@ZU-m7i2!R2+C-XX5WNA^kTG=a#Ec8j@<>$i7tHrAwyVM1n~kVsw{l??S>ge6L)uQz4(Q8*ZmJ>jF>5SlyM73OyAt}ob9 zGZNcdCP86*5bslu&b*8L-a^&eh18*mavstb9qB_=EKXeLs8Rg(MUbTckym+dK6aT@ z$7ht2M6Pw(Q^NBU+EQyUHHHLMo)r53GyFw9CVov0+pjHUS}=1pz38J->E`rAMyZ62 zBqnLvoNny)&{E4Q!j+LqeUBepJy`Pws#eA!Lf-}Ge{YA+|@Cru1IG(&BeB{90; zJbD&X!#kaeFcwCJNJ)lavksCv3|L0+1ROPmv$bDX0+KGfgmz(f=orBEMOFxPbKtUN zDHe)7tvZfXO{x^cXp3A{C@QC95mQ69xg(u?(~Twt*SahDeSzu|A#l@dI}7k$iBd|R zmC#sC%yHB~#dR`2e*G-zWWpBkVqC<{wmF+WVV4;GT4QjP|y zyO*}=>0yIPbxZn@bSp`saTZapVKd&I%ZS{e5o2eg9GO5mZ>~Vq%58&%x!P22Z68PYxu+8 z(0`yKr|`jGXzMmr=uN)qh(uDO+u{#OY_aJ{(wUUidqq#xq)<;4$ycye{QmpbuZiIC z;+i1O@ZTLh5xJ{tkcR3!*ay=GCImY%2o<4eGv6>doXV9mydO{?k9!xgK3bKptglPt(^I zLL{d>L#;<35KOBhJlh*qlX&RxRTj_yg-&HDs#1C7diS(qYvRe!TM0MrcL1jz>u4v7 zo;_iid+$=B`|7#b;2y$HcmZo_dGaS=Am62)MAuLBk_Cm2#(g!nhr|#cb49g@wruBy z$4{u=&4>L*vYk@-FbgItRJfq$mM(M;I~7G*D;6Lh;Euztx{dmpKqTmf@i0@McW$?< z=`EYcn7%hB6o+<^BjUPT#pOSSWN`P;j!!zY_Bn7aJ!NITDwGfERR-Fn0KDr{b-*%` zZd*S4bnL-07$6D|7}d761-^4LS)wN#FBWck_g_uJu#==swM)FE%XY8DdQRP*08Aw0 zAheQ=%AHy2{6kb0$ zZHlu-!Ovg6@;?dhKeh7*1$9L$E2XDeta%t7a;QMpu$r*Lwj$(gbaJ#c5*_DqxMY8^ zt7cIi?V!HnJH1Oy1x5k+q-yKneaG4LKx7wV$lcMmv_;F_&nw_YHYxF~9Sp#Qb*Z`} z_m}s*0tgP3IgJ^Du#qXcsSmXpToy}4J4g}OI*xQe#Vt93jc@kQ?)GG!;-#srvr^-z z+F9aC@NiZej40J{DgUV)351vzZa#SIfynNFHg%gEn4roZ)fqCiH9L6Hz>6xCFf8q1 zw*i4jIzfRegxb~e6-gzzmp9qAheoBL#K!KYEjzK6veAn}9-4&-2s$jW>bsffj}Gm4 zumj+Fn4bT}L$Fp{jzrD77L`rBJkCP?@H&^rjU#U*3SCa=t7pCF>H;ReO!n+s)flxU z(1GE#GI6uio|4fMOpefcLiR`823F3mRoNcs4^?%~od*ttqtMmkQ%!C+r9mK^Z!gIL_j@U7-6_v8l6Ywniu4MNYOi#mN45l+S-UQ$AIk^yeuE?VFk^q}PqPtP%FKUFOof=zjPiV^DkTLPR8>M=5G;_JY=@V0HN>=po26Q)1Vp4KAK7e9O zmU&t>75XOd3kvDSA(pTySzBygKzfQ;k-Fjd9x zhS#6UT0SOZ+ga#Db8k~3E{-+8A=>TVA!T~1_$NK8XkxQ75w60+TIRi?V$$a_u)cwA z35s4&AF2~Ls9BL!>G1! zm9*Veh=L7w&>6DqxUD#lvR7-; zAP1Z;4^8<(>y0}UNt{|gVupXJ>0`QAVAWi#ipa!Ubi91m!+D}+qovkXpv_h**&q%# zfmkTW3UIVkt8O2&!@AmF^btO(gATeTZp3UqGG3J?N=#Ct0&_mo2GMPh)QQ1ZtzNg> zseb)L$@Gd<`xLO5I~y`<4gX*^OXW0`Dgq|#a;ix901w)Z>5jT3F`Hdhdv>>yLI+!3 zLfx}07d%qyS?{&qg)Fn+;xvndo-lfndPH+k$SeQ z+{P+h44o~${q6AWZ-bi$kZ*R*-Do`rKN~_kC-3#)@lZ3hTI3*5$3`Mk0cMyU(fcXW z&ce{panSknsRFw?-V{b#xz1ACz)@V?Z*4hQOHatuOO(ntbOUYxD$TP$20Ti>8}|L1 zHpqr49^S*lPlv3DtLi2Vd6lw$mbwqRVsUoOmek{6mR2cw1thxyGeBbP+Q3$A8iM@( zZ;bQ$A8dcOM;M=NedA_L`=tms$M!_bRv2P>^z`g;R-o-lIb3YAiI>~CQ={@rAdGo6 z0jPF$!vQ|JAUf(fOD$kbq1|A0_mKTGhsHD3P91kUhPG{VV0fdnessRzP+uv>o;ZK6 zh84@LYM&4`7(q?m8J-8JXCK}65*PPc8TMpWN%eKf4Yo&mMvy*Bi}62%|HP-p@FJ@g z5_FOJ0P;j^kzEgt%T%91YMF|<->qUQqxW~C?TQ!7y;dHOz&gjiBFXF_sr>_^A>(#! z2%UAGne#@D|Y>ly_o+1Zr8VaG4jTnjh+U{^Io~wiq#aAkzm=5PqG43gPh;)23h0@p#RR<6c9|-^wh?Izr!fzJ+)GX> zfOo&bx?X}j(1y1FHCMve>Uht&LWP`VmVfQq3vy?Ygq9Oe*Fx$KsX3)|EZ~z|wH5as zO<(^}{rx^huK6o`brkM&1I(2hDrr%El|{X5y`p+KQ&qN(ic~V}gx$bz|9d09w;zEP z45K#CxHlpMH*M0W6bA>+3mmMtaz+a5gJL?Nc~AJ!2s@_L7JZzZYEAf*u_sTxK zCyl;qbh=JA0{Tbd7}=v$T-{4g9?kvqV8i>bF<-v>mRne(F0pY6=nQc^$wY6gT)eAy zLt|M#0e}<9iMm#S7J@+%=7p_22%(&&-oJkT4aZrj)hl_$_aRxoO>)5QJmHH(knJJS zZ*K^IkmZ88ez$?VE_VyPBW5aBq;+WleH*gjzc8=}%5nQ}xld4v>#kdyBsbEtQt1!5 zQ?95M0#M+muIug=MhHMh;fd{Z4X|#{2G|cdoD0I_yg$`r{6Sy^o=QMm%3|5gRLf)D z71fsxjULA%vb~E33ftd%QB` zm&zf~T@g3rRLIuc5!BNFMdP{16}l?+DMoz)SE(!b1Hp%o<$a0G z$p)oHZY3G;ZilVpPb#Jj--H{Bk9!hnv_nYN?oEy#NZA*I3Aw(soGkT74u1QXHO@o%j(pWUCUih#QD%K(8ILO6djl0d77kV5^pcfOdF8*YA_Je|Y~P zI)7hi#h_6MX6@|1$&t&|h1caJ<5HXiSHLu?c$$xs{K={*zdR&IsnF!;m*`LQ>X7Sp zZZJrmc4fzYw5ykfB>*_ox|IX@HS>|zh$Ok-%wq<3n_wd>CMZO(2lvQIS_dZ?n3Z>& zHUW-knY5(*C)tt*QpV7d_>2hdE*?Yjo1_Znj+ItL49H-SG)7q4E$&-|85jag>0bt> z$&xmLJ0S-SCmE5Wp5o;t<5`)*d<}fay7^S>a00fhL0L%>^qzOA(J}U6`M%_KP4ftc z?e!}WZs42%iOE?O_SK@G_0->dbP6O{{bex&+e)>?==i|kqfL(mu#Ju3m43fxb_l{1 zTL+b87;90R%a=4)| zqR@`IMMy3o4x*jf z)a7(wxuTAr%4xbeHL{dPXMi}>RML;np(pZi(H`apUX-%53;_f?6qr{wk%zVhIg)Rc zN)UYhkF*16g%k_`3~nlD%M8}7ev0+1D=CkXBkvN+-QLxwd}*IvlC8?5rs5I>d)L%9 z2lzZ7gse|GXJ)iQ($j*%j<0;ce%LdXm>>7--KIS2RaG^#X2wXR*yZwF!o3C)vM(;M zX`P@JnNCgzh|~u8+Ta^E^wRR2oChM zM0#4Gxid>*B$=Q!+B7kcm)TwhHVP#~2Iw6}RY)f&w@bpOZQ;D8riHg@Q77n_OC5u! zkf$;5BAEstEsThzz2Dz|2i5*pREkm38w5g$1DCw=*;ImXVJ%--|&`qf@O3Vc2P1ad9Ay+bpdFfwUL}eTUuW~^n8XHNVJnNxuo#S;nDe| z0f@qn%@(kznd?I0t6`41s9cjBvCxaPqeeZqKg5M=qrzITtt5AL@GQD%<3toVM6`7B zpBxeGMJtQ$0vapoR1EJZWHbUWWfq#q=D zU`Dml|D&2TD)WgVK$TFzR&y4|<@F^A`-MVzELSbuQbZ-P>|&-K43Q?17>{zgp$^Rr z$g#8*ikFu>feR=#-NPl6J{Aa&9Oecq+bTZ3BPo+)2^awjaSQ_h+s&>n@Pdn#79w-lj*%()g&jO#Q)gY zVJTZmel$ALte*Q_bIco1J;?#xl#C_?0<&S-bT{YKHAZDT_y>! z*LYDA(LLi>qrb5Bn00PR`%d6Nvs=(rg#>8cUKQbqAV_qNmk&12SNxTm=d1juU%q}~ z@JJ;6*A6QY1l2|kW7`jdCariq>qcu}Kxlm#KBA6E#%aIPkOPh8%{XD0V-Q0c2O5pM z|0;d@*YIu%i#xS=Bt^eD#Y92~RG-<7fs^<3N}uP@Sz3j@5jYCu*;VwJTBbBTACjrM zBI3|F{6oNJ`8~M%F{kDlM8#>I| z4lu*oolPbaKz1_IY1x;+ov|@$tVhi11N&}~ z9kG>~`AMk^syPz}+hdXxhoSnaN^g5p=4?Td?8zZA4NmVib6w$;WGh|W$8Zsl8Pwk8 zlDz=mzIgvsF#>g_gWD)SbC&E!mc& zR4#SZ0Z#GENC$2ETQ{!B`aL~`en&v4Bum$9r6dUh)5ko^dH*uJ|1kA<{yx0^3dJrv z0HB%>2&gA@IuiEOCSqG7;frJ+Uzy$yG54P$6_H^mKVRgGva?^ZSsGY+`doqnYv(Oy zAjJk#(Jw6-R511sDQ~kDVVaA*_h=WMveA$HTVNX0C5O$KtkPFEY`xU6hbCt*s@x&T zzx+?f-Hn^MVn)Atdo;CZ@&NKco5?=QY1zX-Jy0usf~FRbN0GeAVPCx7yeS2U{=|iP zxu$gs{7mbhMS?b1KFV__r^EfT-adW*0eYneJW?2aMsJ<0&%upI7lRqCw;&mt1s+G; zt_PH;VAVpO#6tZI<7Movi_GEjKL7Up3&$*ET#qCqCt&l5(-x|s+)eDpnE#I&6 z_EZFK9NnZPd0|#zpIkm%HQ6dC%oC6p+J(%H(6I2skEO)dkO)a*bpgV+Qt4}}nV~~q zD+4-CoT3eSPImPL?x|o}SgTa3vkWQiJ}?M?I4XwjdzR)cRr#NA_^EdsVA)Z`8c>lF z7F(W_<`!F<2!On8b&zXK{X++omnGa>i>f6HN6v9Wh?Vxv28aL|m_IgI0qrS5exh|h zmGT28x<(E{w$J9dfE}75zN;`OgMuAS zW)ftMKy42IkV?5Ut=$=DjuFJ*^o$_SNyr%b$y+4MK00Q+QgdNhE;t8Ok(w%%ae*Sc z{E4+>S74;Bm!dpa>c4&e_uqa0IlT$<=kUk=fI+2kUC{M_LW(Pms=VE}`R&2QwM)Z12xm?ppPua28>Y5u3 z-!tzkxMCT$ag#b-wHbjHBJ4t2=`h{A=9U6gnta&tP{(8wd{ozL?E|E*!)&K?)Gmp& z@Y+d*^khj`bgrPq1<645Hk@i~8(Mi_Kvmr6S0tCPX+PUL^jS6x4nDHOEgnsIW`-Ga zOOi8m^&l0T&Mc{mEz3Zxb-p!JFvkLk>|IA8=pZ6iNp><&`dj%&pu$3rt55>GSz_Eh zrmV&8gWtr4Hq;S%o(Rb=FNcaKGX&uc?AI%CR)sytBEh%NQ6ZA(d#OG9Xp0RAPqc5b z9@`NrFIX3lYAJxbx~j~g;&lboxXO#F5jBo`dg{Dc)rOTWqGN{Z!M*%a!|a*jrP@k` z0*D;C)CqX;=`834a-CBDceb7Hpa}GwX~X8yx!c;ZcC!r~*?gummpkwV)_G~t**uaB z1o?W9!iBxYi>kJk;)TrFYfg+tN6B(GOSkm%=j($-s0r&tTx^t6-}x*&8EW;KcxXW)a}8u=svp zQ^h||4HRptd{+tqd;fB3*Zc@XH?;4wUAyA?p{o0g7OjQKDyp~8k{L2hw<#*9z}%}p zH7AOpR3%c@P!%^VCZ)1g*2#@w7ijca(`1%-x^!5U*z?!k-ZP}XxvgXA=&0(H%`mxn>@wZBbPf ziEo31D8M!=qi{-CwT*B+S-M15YXo<(oK}A$m8w**vHz#x-=)jQLqC9zCBwV(^#kzL zbJhYNTs<`xa2o8Q-aeO``ReWS-~Tnx`#*mF<@=ZN-yfKqK4iD^7FZ!c2=U)7xgA;6 z186O`0q|yWdA$6wyWQkHcBU(HJ5HuMN#nB4mS~cy;CVKD7Zsa=I_1dPDBV(@I#(q! z)$3_g25?ilunxesYI1<_fTo4#d??lvfE#Ugn@wbqi@r$Y0rHPz^Y;Z~oh0Jme@Vq4 z1)Mb#n+f6tBdpw2Z_w@v9J`TP%1&f?32_GAdsBf{yLJq3?Ko9wnRQE9pZFdU!JO9Ity?KMSwYC=|Q?~;G*$TOQi_1`4r)R_+FY&I6+ZU)Gu{F6XOcDRmOjJ`?|L0)B z#B%aln*s{OCu!pupYn+|^~W%G-Y*&?rM;fW-8bcvBsj-)*$o^B8jn1xieD+P4k~)3 zA4tGYUXn*oM{&K^Pe2R%{PicVU#H*spOgdF7MK;oTW?EfI(Z&q&NfrisV=r6CCxkn zv6O@L2C8r`d5UBO4Yc;|leGbG3bczQL;B7TIzkBlyrt~*?An`Ty#4W9czzFv367?+2xSVktjZMSSl;_ z$YxEFQ5m^x4n4}XIc}#aeJs>UlAa#W;(X-Uy*ApX>qYhLAya^JQu0U+k=cb+benCAh1pdgoai|>2--EhWG44FI&bJPFdM-NP;aEJK#klKN7~P zHXz-r0NF=~1)R!{b(eY%x|cSnXu@pOOiXB8V6Xk*V%o>4m|6%!*)CoF3$)|pkvyi{ zBL%4oW`QqGH{8QKC)1m0x>??|QRyNLn(#1RykGwmZu@Xim3kXjp#I!-^n)j#frRSn zk~2*$bkS6^wFu30YcEDcmZ7V}n(qL#hQc^Zz`#HOm@O8Ko%aho_{P%;H~x0-`XCtk z;&8~Jfv+2}!2;w}VLw(Hgrk@L^fu}WLpqIN$;v3B9+Xhy;uvHOt95OkLeZpq!U4s3 z2~d*#q}@&J5&CXcOFJ*`p$y#RWcE$HBUOtqfxI)=S*40HZ5z*Evdfe+%nLn8^mwCS znA&hnO^n+JFrP(%Q++#>uxF!#g^%O@P2;7!Kwc~3!3U?Q1?wy}_?QsZ$reK>GZnt43g`fw- zM$!fk5puq5j3*$UxgL@v^cFYvR`cET1r&lzMPcup&s3|%F#CR>3b`7R6i25qp!1Pb zVSt`kNDMfhC?HR%ax>{So&cXo8%82>UAyGc5EkQhy?q`aF@HJIBXfo6M^s0>pbT%d z=_4kr(+N6Z699Y?JqBj6GR;zR;ZN*R31lnwmQYO)8?gxlrG&!gH^yVYi?y8PSlC$@y;v3|sl6{`q{ zb;z4$o4a%9;52VZ&?WiZdnX5$wsqtzB#lgmq9&$o(OIu0ZgEiwUPrg-T%adysV9B3 z=x3LFGPKk$`D^&Y-?*%*(4S49_KG7vh)@=l9IpxS0AJ8WY9om+j^9(A|Exn+XpR$l zR|&?3v^}z7*0=EJ0s3l0utz*oB*smijciA4W!k7y9v9~%qnbJ{5Kc#KYh4SiFg!r0 zzU*|q?caU~soqEY;IiuMj9l?aMlaM{i=2=+_55<{! zCL#ZZ(z8*2nZ@;uI{YOy7N}WnK~w|1T<$oI60#SzAejyQ_JX9_G(k-MbAH%h5=ZibL|YA#QQ~?^{YCmse-FFZ?1_cb8hWmuC;6R zP`X@A+d(*p&114^OG2 zvi7>p-N`Z>K(CQFyZKNcbXN;o^$t4vLV}p$N`Oeqkp|>wHYcu96#jdQt8br%_s={* zlm}*SF_|YLH;~K;umGyM!=pN0VGm8BJfw+$na7!I5Z@nzn@Zh~f24GPFzU6Df2pWZoF!aSQs`AsU$LIJ zWDoX1Eu*-ABx(jv?*J;$e!Y1-BAv9VzNx*=0#g!WK&3~Et9Of zgji9hC3`;)40Y^S@3Y$pS_%huuD~;tc%-r6kpb?{O-^Y)MYfi&WZylUZ#7qW{~?kB%eU!r*KbuVPb+|B`^oKXr^4F$ zw(=0ca=zR!A2!Y^8b}6Etd*2&$3s|47zfzv1gy-}b70Z(CH{*^g1T^HEBxyH2f86% zKemO7zNC%I;=|8nU^=*UF|pH!?s30ZaBL^V59-`acNGzUW;NXwla49hgCcPykoU~U z)*?KPxG3DL=sj?Q@}L%dFCBNdDpN^NLUY9yt~$X}d=FB_6}@?&F0X>27fjJtnm=v3 zCV+%v0yzuKvj){g`W;=o3!IeLH;*$~tk(1#R^p@O=7-K+OK5k59ZAvwQnN_mGZB9H z!(Z)66_vj85k7(x-fE zBcL0KB+Tan$14n_fF811M@a$oG}Vm!$Eh6!kTqJ|m$x5-S+f;wa*bJ|n^jH7kL1Tc zO$1ZB206wC+s>3j>L^T6<0C0#T_Jf$%LFt{vRZ{gq)tRQb>Z1L2wnJk6t3_w>0z7H zW7q65-#R)PB%aJvSzNGqlzX7{YX<=j7=czNy~`)e6AH%8VY*w_4IRXKeR%ED>{J(R z(8gHk^`&OFRL{j&8e0jmzxY~tgZ}^tlfO6H4yjlhLlNqr+->Xi++N$s zrC14|f#6lLX_knOfFGU+zj41qDW_3P8Y0ed>XkN{ICmc)j{=*W$!*4uX_fc0FZShr zBYd3T;&3Yt+)7@MV8GA3^Wl2 z!ZB4T63^rkVSxw}%s~w%Y-^RhW!BJ3g!ugsoUKBKTuH<`*SMTlpTR7uUX;V3l$As9 zFPlR4qbc`>Yg$83aRddT+&4dhN3Jil$sn$rZOhoKmPgc#ZpVl?nf~c)-@bDliyB5o zFnzP#46Xh}%`GF6cYC0}PR4O zgdMmV`pXL&Mka6Ijb5FEXg_~+3BWJV&Y{Dfa*+x3SB08V-pO&Ks|2yiloz@l*s_HM z3Cg0!;UfY0cFg2r^vU89yANRZV|Lt~7+2evbQJ7eO^cd~(xzT5C-*hqtgSJfj}@xy zUHc$DoVU@&~4lCG{_$53inD#ypsyH5Q+OxtBzP@1jZ*hjGio9V5T(zN`VHT zip<+iS%Pa*CF93bR(xg>iHTOdNe!`BGa@|L!46eLw}|DmXKqE3d08bYIJu}B)Rm>5 z;nc;byAgIbDcu-4!j>C=d`D1nmi{YRS^NNi*`HJ-d&3K6&Wq&haF%fSGQ9s20`{+8 zf9<3mSj?n-fyxBh?-D`o9%@llIJPBx1Rer)fRg!A+WFlmOF>0_IvT!1L!L#;)5O{x z{zCE&CrGXUR6wi0+X(NmmNrj>w9`!zJm`1dRps(l&~UnC1(rW4O_PXEw2yHq0K96?TpxjAz$7 zq@-;2146B2go^lO*S%R_Ol$A)3fokP+t)Cd4*&+UGXbv9?cE8#AWtLp6uyQYowZm- zcQkq0v+T(eo^S>hjG#yH*e>?wB$;a9s~-@B^@GdX=kK4xQ1grT-@X3g@}j0^12C7+ z2efj!3$fDah@>T`8M~7?$7AHC1nXk*38)&i*ujo8ty3(g%Zw=J-gc5CK5{c4u|k^G z+Q5)dI_Mm0>(X@)#J7FFz&{WfmYzkT}zc6=sM{_*>df=U#|n>RAV3c^TL${&FDl>R;_*OjeZ$kzqu&rL}TDGRfI7} z(J4P*vjyPV_G{K~USW~qb0`h@+P*QhYK(5JEf#ToDY4C!xXPSWM51-Qsv$GjBCAJ9 zG`i{ltfjI3+DjzE_ zbay)*7j%23p1a_oSf2|Dy}5Is1iF@K(F`GvL<7CDTm+_7s+$H|yTuX{DrfhW3p`D0 zp1aUh1pbWN3W-3Rr)|{2m{G#!j>Z6!-K=C{fc*3vZhSQc0J)+~)9$t_)Qj?hMk{Fh zTWe3RSyYh96}&phcojd>-PxJks9|wLx+s*BO6V{|Le3#lPk#Yara7>vAP->gJ|=Z} zTNn}#<<3*q4jve3%)m>_*|MQFmCPT-HOHATtj?ucH{Mk0Cj5IVq87p2 z4e!6S@TcwCjpVmZ^}CVjXSEEuIctQVMcY#?J?Wp0mzHQ(r0`2zaX=AHT6$9fDaZTA z>1(LU!U6hb-x(lHRd%u#CPs-88|(^h`q!WK0ZdJn z^UZ|Q!;8vN711&f8Lf_Jl%(sH8cg=hUx4T0roro#I6V0b6qOKsW|ll71nAVx5H+=x zUD$c}blW&tLqKUc_Ug(_R>E;ciD8saeL#6Ted{cyAt!0cluk*iWZZ30ZSS*i=5FTGE760RVv3jw+m?_$>1GyBB+5(f#~B&0u}e}J|@dt4?a7m zZ|hzr|AdRhc-cjcHrg^hZADe4_H&kg?N4Fy*oP{KtvCQh7LwFP)iIJSpv&pWJqDf% zU|1FLT2h4!14FMR5-yNbHJ!kBI@5Xor50cRcrL+g%wtm?U^#57>U`!j$X(*oepe|t zj%A5UPGlzAjI#Y8gw3bbbQ!?c6h=Icl=v}P;C0-@24^~Gy|X;t|OD>4+f zWjYU1baPrapy}%~p+@Q+v+{;;T5_j{*C%%XoV90Jd$lIb0D1(6u*yM8%_>nKRFaCrPidwO+zFOrMRPE;sf$W!Z()#Radwe0fsM3t`* z^$gG>@%i#C`(H@E*bf^{K9z8jx^y^x+|MsO+oxex^>SIg#ZC<%5L%G4_KC@ocIrb5 zX4^hD=(drbtg4Xi8xUr6^tx?_vePtsQ&kf7xj3y(-aCkT68l?EzA?)8nRz)F6jQao z;>wvEWV4v=t*VtXx5Whtz0)({BQI5c=q;VNUi)-iiv@al6wsg9M2*ETpg%JSgWO^#AoMvD|VGlfSV2-@=6=cH&D*# zs+U_cK>)}z;g~pZ5?9&w1YVOuF$f>xNdwZ zdMXk_*4q>)8@ytpPR|KU zH9xJ4RDy>=nN;n564Ec}*kHBl7+ODi`{=tdR2bqi)l)S?_MFcCswwOUf3H74QT z5Qsf#nyyJLYL0Gb4Btfz$1n&cD`|)|gmYrrxcxMsWn=r}D(N2@RDTHRs1)?zaugL= z_ozHdJO`jAhGSC61O6k)d1ChJH3Wt+NMJuR)>^C&99O-|k0n*LgV_E8Orp$5_*(OsP#2{p~@E{pe2EXODKBrO9ep_V6+t2gI7m zOeN@HBXtIudd$$5^GaX!#@Ov>YP?pG`_$TTah%Vn&SnzlI+EXIIRNwKK7VBdq(DRP z)v;lrHc>`-BrmLC0FZ9d^jnHa(2^dPHqL6h2X|#CPIjkxY7J?%Y0fl1K-M-@V_McT zxJaLXq=2%!(RfO?g37+w)IP^Wh;@)jbedF;vN$Clx=-$LF`~z@TyF3rXf8MC@=(=b znE=%o71%X;rU3~9{Ut@EGo%%vnXc~&Cv^4-XfK%Q!?=N~L(atJJkkMfGL2*;*S)4m zbOhZbj~ewEw&rgGSQrlr*s4hyLL^@g ztn@#k*{g)IeP*j3;Q~VgR!S!a@KUwBZRH+XLfDE=o6s@tMSI?(aHAol__)fIiOqiONV!Py7I??-{D?NDOW3FkLG zcw<(|nX!Hba+5~MNABE>Lt|+_+Nf<|UZ!%=EjQ$t4cRbv$6IRQ^RuA%!_WCUpGuCJ z_Dy3A8?>>e_0lI(H#V@^>0Eve;NlbSpaAu_excQ+&X7~T_Fk9 zphX#!X;ej)%PMJItyWMoAS&`wqBkwCFsNd?hM`2lcZuG*)M(ga>Gmbic=eTAuStO- z{HYxn)*W17C#}Qyg`+L04a~B48c#&=WGPu#d*La;H>q8Db!4~WR2%%fQ31Vp#C#RT8M6A%S=~!kK;A#>n0h8u_NfTRQ4beh#I_}d3 zTBe&MtHnWPkwF=f z4W6hQu61V!NHOBnT}G`lr)#<$loTq46{MYEDm!TAqPGu~UF)|E#SPxhmhQUP zOkZTF2R(b-_eyF*sQ)oqlpQk08t93g4h`tvnBaV@4kibc`JpBu#O`c91T`q@Ct$j`5U#h`UKuNO8!K);kCP1N z<ycufGlHmlU|@e2qN_2W^yb zuu0k`&7%M|o*`|>;pwX4%Rn6%c^|Fj!$J!#C&}NMng>RRUOGaOXIHHIZo@ffoybF> zyc61$+9w zac@1P0o+Se^o#Cz%}A%X1D`0ZAa+|ySH zr3M{1$lROc7S6}E-~_ZhWcft(W%TQq0_0PkNhM|2W!bre22B8XobVyYMdp1pVT_hs zKa=Y|O)?fFMNo4oSIZ8-eVhGYg{ko64sf@fn{7QNxzSxcDC}2a1gK@H+&}73@G}z< zQ(~tT1kr^O+}CXL*g6f98pheCQ-$$F0A2=Hk^4f@L&PvM3cusC1xv)G8XX`Dmr{jS z%yI8J#me1>HXRGvmB$8WgP6Fu+#Ok@OIeVT>#&76;$Z zOqrXNIycF!=MxnG4KOUp-<@%E)O3|OcBLbRD8vORI~7Hua)w0z_{yML%0sMB0DxLM zp(e8v5_|>rpR*)`ovjBXec{Qacx?;yI%u{U;~9DK<(MSDlE2t$eQb3r=wLP&Uj+(l zms@Q42ZmxycqWyW1fr^wVrV5c|CVB$oW)NwbeFTc$E*_DY5f7#s{9bEoJ^}~OS?-oA7Ef}S%@5+?`(*z^=qKG;cw-v`P=mNzC7_(0Sczy#fG!%O#}@XZVpIV zN&Zst;X(&0mOR?ck5J!bst@%go*0Gd9@?1Q3sp*x}F|EMCyAE560^?!=@Gc6TMU zw4}uM;TR1UwLl6zaL!#7s)1m!0JJYyh*D?TJVhu+qegRCw}-1;{0Fa}gx5%B-IBcN z7G1YY$ccy5(A#JhzXO>lV5N(8@k-m;Na6`O4Np(9R8Z63xv|womI?pp%{hp+m?ERp z!+B_#&Zs*x7b*ZWRRUvZ(OdHr#^uUwJ+`=;I!8}R#wvl*np^5G7B4TEM&hIO`(*Ku zCfrs?dN#FWhNwB7x1D8*Epc#{oMlQ#w8Q4@$FIK(uYb9`{b_jpUBq?8SB~}`N`r{? zZKD-OnrcN4%$St#PSk2DwI-1>Bjrw#HuhB&tvRlg?29l7jtOPqa_2q+*(ooyH&oI- z=)mEbQ#=4Z7R-z*7lf1Q1}SZEqPNc2D7IWT)$WKFT|3BR0-|8*>e z|J6tck4Edk=(IbeDf?D4=IFsNmd@RV7qFeJor?#c$?|A$=8XW4yR0*G6(l*6ax8Jw z^1X!NOT1SQ^plDcaA+J^6^;BFoo2hjR8)nK(#qZ(+Sy4aZK8SdFvT9C_c-PHN62Or zZ}!$qrnN3z+!GVKYC5UJ^O+{hI?g2`WRD{pm#0L!1#;JG+&f}o0U2kg<%84Q?@#+8 zp`=3BbYqn2Gp&rGycN`~fW3$hNv`cl7Im$Q5GVqh^imtt zlLTsXdLP6U;_OXX_SDG*!D`L+>;dbc9R+y0Y1%-lRxCK5J2F zGppF@`{m`aklV4l*Ez}`d2({$1NRH?$~b*5nuP{5IdAkyrInO2Lprc2rj>7bRYOqZ%d- zrE0yl8&n#!tXSY~f!Uno)aRB?T>&#KX+U|mHb>cFp8<+jNICXfMwCJ|>z$~MF5H{l zl{TfsZu`?@E}AvTz5`=0!R96vBw657gXegeTi{(RM$gJ~@lB0s$L0k35$cI8Fr!@Y z7@ODL%`9L!NA`NACoiH6ZJ%+l7l0IlF+pg;Y1=PFf`y%4Dmfc}il$soU_bx#PJF0U|7ns!J7H zwjet}%pTUiMMHthMo}Spc49LCvSr}UA>$w%e!k^NTNxu;gFFXg)1wLU{f}Nhh(Qfg zkqaRH8SgMc1o8g1a8o%cd8_4wp2Vd%)Qfp>xr<_nlEI|W0Vq^>_Mb1nITMhjvqFG< z%gp+iQV^bJwF(Wkw;DwoS*Wai8>fWcRhG5bx7X8<(;f{?65!FhP&)^9C5e+S+5ZeS zRRU8hM~k@sM<`c@oCRREaGJS)E zBE%URYJlAA`z&&ZLNl$ojWO0;V59@W++Gr8>T1J*jqfUA(c*I6455b3_B!z8 z0{iit4H0P|VX?0U7*35-1Y|mGy$bH_#kLm)Dr|sBq0GJm3dhAT)H6dH$-C1E==yqlVP>mcp4FS}5qTN_&QAF)?te{U{zK&mm zsRfvdkb$9KmVCAL-sYyRn(>q|ZpRf(Rk4 z{+Rg4*(C#EioLwR$^ zHH;q)I9JW-CoX5GvO@ImsAg9QLqNArlDyi04Rk0M`WtNPc3>Ra!|7;0(kO{-gk=to z=x#!fLmMmhf>Fzk-Y6;5?kUAqByzglX>*SIwbvXP`+MaOSiG&1^(BmHFVC!ZMx}%k z7O=|MYRM3BG z)mBhaG?aDFd0Xyli`ng{?ZciFGZk$)vQei%H`U94;ASh!#qqarx?ZTO=|(%iCT#XP zuPuZmP_cB)WQIQXK8+oBB$blaY)jyv4E`7B3(Yh2+glYYnJq(NAl=kXxsd1}kElyR z6_M}X5a3fJh`*e$MJ7j!)U)|cX1?+)dgCZ=2VF8ibj6a5z!dAZsvUBuIZReLehWAc zVgQg%LD~(-VhR1tn?17*7wiGZZv*7N1QHFMi3htjH!Lq}XOw%V`?=V2@7{twQah*f zq(<-;{_NuyI^`UCuo9{GOkZnscvckf>T*BvO~OTI**rPhvL^t;YR#VF4#Q;>L|;h7 zI+~dsAumuxU$E#jmDTh={?*T*5*(0BU3YsD>eT9FxefWJK`AoFgbqGGt8*SF@}K~i z+c$qp0Cd1R;-bbV9Qx`28`LLXXK^x+S5!1xt3s<83f;?kGaoA10Y&nDsMLnR@@Uyk z`59)vLojV&e`+Bww2)wiW#9X2g9kS-t6VJ@|W;Wwx=U9h=3Y26Ty6r&TnuykaUnl^;%wzm%3l4M#A7naIoOwjS3#CRCsUBoA)6%np z))Q=XGq?)i=uux=L(-J(Hl4H}3UxK3-pmKDe}R9Meu?o$@&_Ykg}qcA#w$!O3Jf3z zA2IbRNp7MJXbl#J$x~ZnO*KG$3iOxo^2O!t{}o<-YLSRIA9-sSjjwy;=r#;Mbr?_F zpGFA*UP$6fY>?X> z#fkAIzUa({`B#8+BtduzPVRBh`W)Sqc%x8`8ZXbg1hRtR1+W(j4rIzy z#XFc2bP?PDQjT$wIWc%yvK?(hNuoefekNUaR6s~>Q%^EAuqp}RwM_hV}@8a zSH$-!X)4?&w|tfa5^WgC-yD-ERV#^7ruc_Sb;2lNPs>}A&x})OosvL-ljKCT4p>$6 z*&Ez?$TA6T7kjNRBww>DQ#WN7`yi{1&pC#Da*ovM3#`2O9nQry5 z^4r)y{vvjZqcGZ&{4d}i@Ja&k;lb#1*jX!UYoNHm65K@!`avf59`AUk1u%d$c`mOE zuS%g%8Wb0z#SquFRTNRRoJ#mxE!W+>$1r}XO zU3Hp?+mX|L36s~>839A4TKwKP?OO^e^dLl+(|kc+o`-rK+H7d_coPs z2U(Zf&bY~e3@L$z8>xX=vahGI(x{Po<+2<3SMe>LjwiIjm&_a2ZdRlHa+7CK1()r? zf3?oxA^{ESyHKeC#hSyM<6I01~VJE=Ug}-`@ary0suOCVn z(2{qfns^jo>o33n$QjYXy;yCaJ@QHEkxju8EY%m{loGqf$^^-+jw)N!D9V8Tw${ed z>69JWB>*K=Y%Y~xp^qgveJV2sla5M_Tg)?C)*^-(`uX3V@o9emUc^3J(A$D$6oOjG z>~BPW@&r>x)x=g`C-FxvskY2x(8j8<{Q`^g0?os!$xY}(f7`Fv?2r$t=qp+61%emZ zCMTIY>p}ZNYG%-Ftg-~?A!><1SEWHwOCJ9-1ZUaI0nt#ok+$!ez*M|$uc|7**B^|5 zSh>STj&RLSNAbPQKmuBjmgJNIf@+82)Ix??4~aD{nU&nX;<*B2Nh0JuW5=_AMs;FMYQnaTj#sS(@MLg;GgDxEu|yo#?=t`@#Omj#`*xbomQVt*e*!=%Eh7?A)~f|UrozA$jv9(Kxw zVpJe;uOc0m*RsbAPW+fyI@wjxf~CuH=>q+Ips98QaDV@!iz+OhJlNy?qIR|-@E)%H zWd~e)F$q%^yCal5Oz13HlX6^2Sdu()_)}CIN&zqC8f-hHP9@@RY?`aNz$~S4M#D)| zl3m6jFDSCqwTCSG5*>6MXe2KPstDQA=6Piy6NZgal^I_6SeVVV3bb$aImKU|%Q+F( z&0|_1Skm?-24>FqYeTYbM>A@4VYMbW!5fBfrhITnC9G$MDfmrol7~_;VCVzJ-#DTy z31Pi+mC9>b@j1c_|W*Rgp9d05q&A_ylm{z*LdLO&vl3nH}kHF0M z=I^bZ*?&bZh+F36BYqA15bs5f!7{M@rhKTb=fL$I*fw5JERD1hL6#*2d7nLJY~a+? zRfCal8Dtwa(|u&)x`if|sbXKE3cI6+6^8rZFT!`p1{#SJU7&sf*!E9M_V_k6(7dQQ z(!GHdWt($cl3kW>MI3anuM3pqKj*9&Ic*4SvKD@q|N8}^eZW{s-CgA;g`tMfza{-V$pzxJQm9 zz8S`v5|$k`(u7sfELQzPdSG<&^yW&Gt8A{dPOUvzdtF;yqIn79=)t68lA}oujuSMi z9pOc=q}B~azftYAmP&^H=}|+2FnCCYrghZvd6j4c*pYi>+M27^qHQ>|!63;+qg%u& zivT5H0<1zL1<24&9w`G7Oa)SbzOkvManKIM05ufiG%AC5uP+pzO!*f0SCR z{>^1ZJLn@|T8ehV60r_*$jz^Eharm*_3Av_VSWRP9sR-oZ2t@V0KtmBbwE;*G+I$a zoV07DZajB71~|>5R*JtAQ#^N6E5b0MmkNg}xm`%+_LsMxzyAF7i?^Ri=rYvb2G>;f zMzEJ;8Mu|;I-k_PS8WQ}!&kMEhUq&FD>5JzSqP7U4l8x?7qw-2qHWQ;C&zZI zwB^NCf)uR)NVU=S+fceQ;qfqbu@g{^?@LslbVdZYJK54{Omq^mtZmg(k<@Qi*!8$M z)oG*d*4+nPz?}mXgC%TE5!&83#&Tf^sBY?$?Av_rpw76$+Mpf@nfYN*U5*c69c0U3 zMbZRF5;VaU@KsvI|49W(2)5ByyMo)Cr@S(HZfVLM(S%9HSXQl8CO9O^GE=8gjjUJDcQ%E&uJph%q^bXlexM4R|jmWaHH zPzbG!PU-od!t1Xts^dYw@f9ixk?3F2|FaElrBJI#OM!zrZ(u5;tLI>dm*@RSeE#TT zTN;}*4Hf@L`GB6KA6has4N0c1(qbvENC4dFyWY5MSiNYGxUv%|X{jVYlY=;AALXFh zWWin3W=eR#OJ&(tPQDfX2BT>xR%U7FUmv3`a*P1Z697DzDyj)5>to;^{Y-~}B(T5n zUzwNzh2dM|;2{Bn4TK@D+BG0`*I?7Y5-cuJ%y_L#2E>fB?+pSkoN%OQ1Rq>8(^A7J zj%@-88ap9Zbb5?~1V<55;f|Txe}DZ#lFea8KT)oDalU$mmYR0HXjePqm`^9~6o)zN zrtg|txRxL0Po@|SGDD3~b#DP~-7rc*w^{Z(0Y+G|vl<53Wh;6)2Dk@d4{_8{tBzDs zOvn1l=%=`sv(hSZ;3^mElDj6gJ$YzCU%P|*Q!XVeLti2VN-C0 z&f*D}p%U+RACbK$_B}w}#KwiHSc>N>S(8ICH=77t`zS$d*pDk4G{qsM*bex|7!L-p zXNDuu7=X)iOlmM{lZqz%`IJ5I8_D2t)SxkYSEak{hf~DjdfJum93D&i%H9otDE7Gz8`0+zX=m~Sonz!|6nsFQ-zHX~~URA7$1abaHIhdN~M+%mZ^Q z=#{D$DAdUvfbp>O6za2f+L6c5v^fxk_Kha~yBo$V*^SR+#^EdfxZJ8$0(Ei@2mF06 zFINbYv59d7NF5a*D-lgb`it_A3~kh_Q4{-LXl7ID4IJF~82?)S`>)l%!`q*~eDnG_ zy4tVb;-4>HT=sRQwF%DEooPPWMsm_j6UmgLT0f0>feP*5Ndgvt<>Ap)Q4{%iZTEmQ z^@G>nyJ{eDcF>0{N$-X`8QO5Y!--D81C-}2)AoA|pF=N+;vlbW#@44UPFSh@3W?K1 z8n{iTvUYk$mG3G`5MV&>A!Fs*vQom!!MX)m!Z(qKImu#LM&U6;7NZH1Ov}>6{(=(+RhXAIkaed`U+QaGc4_IQl-U_ur8l*}G4>IjC%yc>jmA^Zp3GPWhJK826)ItI2vgIin zG9&z;TVxm-h}OoG-6@+ux^mkA<;j6)GK~UsEY!2TZ|#sQkm{+D1BOy!2FKG%ScVl@ ze1SiC{V0&DoT$pJ%92>$s7<2>+2+uNv)!e*PsAGZvMoOV8B{NOqxYsfDyy!*Dd?lH z0}ED3;Nx?EXuq6thGUj!a0RkVpJ{S4eI!2^C2b9DiH?qKBfy&N%32>)O6Cwd#cg=r zLF*d)-OjrzqeAQe;Vo`iITQe;w^t78l@)UNK|Qz}9a}`5&ny7(#F8iRjw_24Qq~XI zWhGKZtf?Y_blf=gS!GKH2hWCPS3uWO^8A7LYG;cOb*_X)|1 zt>Ae*BAs+gv(I|(6*VzeRtmPGa-k<{$~FfQn5RRk#~J@E+8)mhLpC8;;5*M1lCwq5 zMpX>`AjB^bxWnw=ky#AiQzLOxh2phYXut#6-;sE)B^!n)HxTbZb{O=VY)@2}8_;QB zj(9f191-N8IAElBd@HCIYRhio>RPB0ya(=R8;5?xxoEU}SX>GXBW1Qxzb^#%v{X!a#27vFr;G7)|r zpdOmIUAIY2Ly3ND+}#u?ZZEyu3-@6Fp}|F4+Zz6&uI(&$V8aUVj?*O@+bddXThFL6 z-s~3Hss=b#d{QpUuBARYfW1Hwcq*rHblKfh04jK{dWoD8i$E$q44rCM+C;CZqDbq1 zu+5JNRJLLK!{{d$d9z!yBmN3?a_biqxR%bDwF$|Y!!W~N*35~ijwvbYzXZl~x1*CS zQ|!HCcY%HPjOc$K26gt7906&wIeJxAF`d(IbSIVKV1y8u$MB-~TTBvwrxm z!Td-L0Dg%eZMB5AM)?L@kQzp?xd6rfcEguqi!yv+wTWL1OpQ;6l;T^0aCJCdK0h6C za_-$G7hfz+le0Xxq7(Dd4O)W)4tEg6Gt?Yf7<3qs6U&_r=6-h%V^~&0;odL~5E#QB zT(cyO0zZLqD@d6x+f}yfYzI7sp6DsWU$f8ZM&^wkEj!)swE~W%Y^7RhduKe)5ODe755vFLAN?=kANA`q1o15!lYKsoDx2mO zkx9iL-70&HfGe|%RCgYm&Mz(NEwoc<2D!0zZLoapM&wNEc0D4{W-b)kZeVBiYl6J; z$PHBh>{B89wW_qZq`8%i0-jop%NhHXGNrH2b^MKdTAhu1cH#Rkj3hC z`c-le-ou;3hkR0$M;Js7Mfr&j<{!d;k^f!L@?75DNIS~Smty?m={5Y#zz!r7oGq6_&H57GJR;30RRCrWfto@V$K~-Ud zi6J9DNRe!@fzzFo8KZC$=DGF+au@}bWCuvDv_W{3@XyFW z?PX#ML-HYbGZ0lckBPZb=_D%C?BUtu@sx!=OHD!rupW72Ns2bqzc6?vl(RsYYiAlP z`KYrdx4T`QE6}8`^F^|PjjS#v@>;H zORRp{rWWB^yz_VEE!(Sdf!lG)It(iQ5i#QtbMyAImoLNXHy7pDz%ANc_{>dg)5hYY zgf$=r*Cvv6>&w!>qVh5`3JeNp9?&$4x3#S*43xtKtqD1E>sC3y6+~BivSl&1)!d0Q zgk{or6=gov8cz)*e68s~1gZ#e#i-V;TqLxpWvv{+CAr3^hLX^dLlah$6^I&BQDQ2r z59^`B_N7-zWlMvhm>)&R9Wqq}ugJrbB3tX|T(CHwDtE|#HbBjt0?Y3qhOM&t4Nu)a zzWj-E15KL^WE73$F-!Qv%g6D>U;@5nRTt>_sM6dvNT3~3?@Dze<|b%|b{bl9PxpsR zmVQs0$D7J~SHA2gpi7tCo1ja=5*Cy~hmO+RMp%etQP2khIhY&Zp@z1E+yJVwyA%w4 z+iueVz~WL_pGU58ze~94cDG)rD)%IE`p z|Kms#{XlY)0<9s~%{aqy*fp?T+6dyelM7HLoU*NPnWA33T2xhzgM0hw_y3{_%WBLR zH`%cRZ3m;s9WqZCb<5v+*v7;T>x_^wtMGpxMV65dP%8yoKKZ#k3OM*lxICl?oe_cz zMOf}Cr)672a{dW|`&yl1uI1V@eMFC0urQes=WFSt(jt0=E-o;t=yDAd`Wzjsn)SKr zBNMNN4Pbq>$DK269Gpdi2Rf{{`ku;Gz~INXtWZm+(aC+8$VoLUKH7$km6T7tGjBjR zS3H>=sBX&m2oL;^7y9lf4SrRY(Je5|5Gmj z_^2=kD{+vA!79GO0)XvhKq$q znU<~P?73R5?w3*#$#WyDSXM7g_d;r3JD9EH}Aj#gr59JFZH#PiS$C2=S91lWB>fASVG4 zvL7n%N1D;9O+d93$9d`i&@t3Z0itwheC65FR3h3O?83zn1KOFNt z>vX9#_@w48ZL~R);tLWE4iqin`exA)LpW_AFqI9J=1p!LkoC1xA&N}{$<+d69dxJw z0~CxdQHhndzTbWlUVeRf9BeIVQDiS73d!0&+tvIxTJ6=LKgN;sdE`%}&OL1!)KbjsqbSX9QhDKE!tHs6|rves|!d*^;4bJ!V%nw^tttZ_u2)bs9T@|tf>UW!*v>| z6%D+GkFNm1k)k%-T2~u&BAgJ7aPWVGb$W-?r zv+>ZaLUnXi+gEjY%Uer~t0%d&MMY}el)Y%!VF@LvK5Z2E0y>3rIEjeQ@ymj__FBy< z)&o^qD0L}31Dx)={k7K?Q{h}9;^c^s4}u*m`@>qX@Q#r>&X$U#S`q~0U3vMe%7YRF z3uCZP44_%Qt*Deyk+2qGbX_Djzv=AKv!KhSY2ZP#MCpFir{P+6;#8T*z zryAF8l@t{s2%B+yQ~MO#ap=3X=J}H+O57pae%JIzlF|yuZ_Czz*QY9(D(BPc6prit zK@}r(k>f{|2Drm7NU7~0Tgj18bhh)dLw!Dc76iR9+5ktB0|zITkDwQm)aGGdC82HD zW$PW`SvkU-Mah;jla~XqFUaOBE!<;CvszWTK(!ur*Sf}Nqt8fM0e?dxudG#1YUD4t zw@_^{7fpM)d*XC<>ZB)C81m=clXGT`II0)3CF!NYF9~!%fT6IA1)#Sr31F`+s!-LW zf^Xay9Pky^*&Z|QF0^FF_RxT{QwtbxZgJ$o5of{E5n`kj)e);ziZxdO1x9g3{ia?l zZX%cg`KqEE#JTowUQ_UeR0YM$(v0#>B9!?n-i z;(D9ffD?#-9N(3C7q=L^W!Ct(fD!D?dnT%!E;2Ol!ROc)}x(qr-Z~ zSYZST4?t?dN&lWAD%>`&$mcwWjEc<&c#9Zq$OBz)P)t1r%aT$A7&%2$@TLVL% zwQ$IZ`a(NgY*RU#(z9V8DjiU+)lFRZuzvyrSE@xh;5!QN)T#y^R}oA)pK-jm>}Fzj zZ4Uj_1T?obS4w)tzeMVL%bWnYlFaFmMqfP!tQjA>_Kj4qGt1*F{y>8^Wzx_WYL)0n&ZM!*bOw=p8 zn$DLkQ)U(Uv4cet%_!$E?>#aidzgbd87g34V*w*ehI7SC4%6d;6NHBn{>vT+a9oq+ zD|>AREo_6ZLQN=a#3`Y{k^rL#owTwty>5v4=ujbY1+Q?`k-M;&LpCU3bzzEE!sEym zjjWknRep%6l#d+UJ|hX$Ny$RKLaT@@FhV7RBK>8aI>|E9tO7GzRXIY4(@ zLei5OQh77CQIi*lP@`Xu2)wiE(KBFJltW1=XgTM#lJikU+(|xrhZ>?HV z7iNhBr8U8^3^%?B8o6k)&Jw7T1-h>UI&CW_fN&no(#13M!S1<4*=vysQ#AXC)V6LD zMMBR$+5VCZ+1qL%q_VfvFVg1Kbjfxvxo(=W1KejecVJlA2!7Vq45r{xRA`bb(Tbp_ zd6}dF(|M>5+Tr-sOdMYDjX4Nto+^o%iSU9YiH=|I-&Mv*G+ZS{4N`=}r-m?)S(o&R zXVR`wQOi!jJ8DN|zvk1Ut<-+Pg0{hh#(1RfY|}| zwlEF@p*Eu{z)nGlk+)psbu^EoadQ(R#65Xc}J<0L}??*)fccwc#!$XWTa@yZq`ViX|dfEw8rMQ=5 z7@O%cu%|u1_l}!SLF@BraZ0BB889Ed@E(YL+DP@uAo>7d?!8SrsA`7CO1VC&LrB@A zt&kj*<-4PV5@!h`C4}>EE$&`w1DcOHsrmTw00cl1O2}a1Y02x?&K`*@siz8Tje;cU z7OpxMbyd0xX=Su+`}3YpgLz@7UDTz$>bcuwF+-IwpkZZOt3v~gQj$49Wz2pLH%SI) z&BB0GEwGNv<^>_7I2AYFy?%Jsh53pNX!)SCoz+|Gq?koCbA+YOj_`yZtiwKvB3K#> zG2&aMnbtZs&QSOW9X2(AXgO#`ki>$eE`Tc~APSD!(CIFiiyW%!UbUC{pynWOMW}W; zzM0V~Viu7-1vw3dBj7q*$p9W`d;%ZX6L^-_NZwMo+AI+%Y#??EwkQuzWOp|iKx*m3 z+gTrlGx;4O148I*l{AF*<8Fsd3snW=_>fywT5AQhgPd_|b&I^6xFm$#X`N9tNn_c2Pe%S;pg~0zMPn47ebKb7L27GRI}F4bbTfYer}(iPh@Ka%)*XBD{*%R>BF12MB>9m=1?gX6Nlx6{tJ@GOuH|M{#Sc;i*`8;^fJxp zc0yp<-i zfwE}~z)g6KWPjNTbg_d4W6D`EJK8Do4&u}-2cQ*hvO0Ej3w+wYCu0N9~C z)BU0?Ar24IKskcRRF=DX&Vre-*ijn_nnAW5Jln37f{9T`W=^(tj{V#JSIwof@o62KEeZ5leh-yGqRQ@ky4%^S5f$y8t#dlX}uQRTvF{ zJn8>H72L@ynX1^7uGs$>Ea|9U;Gmc(x7`W=H%ftE`nY&|d&EUOICWid2vmS`W2~KK zcS{8s3cI5$P-a_4*f^ZpWQn!V<)z5U_vGX)&KNVUW(g)A*rUW~e&P;NbEuMMocGSq zPQnI0kRP2iozZSs-53$mHgLH0%mgRCpf89yL=+X&z;NMmf``&-5vh(KO`cRWcTAn^ z-hna=Uh5!1x4;u-m#Z(=>X20kBPwI^*2XA^-{7z?+df4}(9A+@4iF^Mn50zT{lgZ3 ztv$7CFnXM%WWc(#b7FGCp-55C3qkm+Z=eI}(*ORw8<8^FEdr_yao1; z%tPG4lRCQvAhp_v!tbMonttxkSjh}HWg+TWHG?y=FRj_tzal$O+2ewgrfVCaRR*+v z!tr;hwiU37ENP?`;21YfPbidKRqi^m^~cNRz=v8h;-q$Ou;_(AU6Bl!nSgT3f^mVv z%6fB`5eRjH6HoJJ$W<%^0Q3-)O6hcNvPbb96m9vQ37nOXAMvZ9aKHbt?T5%6y*$ns z?p=Z6uvc<-2S63=g#ZZC*$E(tz)eY1@gd58bOaTPweLZhNHm;M67&6yJyr!J$VmcK zW-{0%3jq|)0!t=kit>~Va+2xBwj-p4(ikf?tVs0ZmWkb~aBD7+Vq_Oe=RiqhzkT^U zy#4(9|Ks&H>H}0VCGQo(<1WLCM94Uzg*2x<;32DggFc#2SUknd!QL^Z2v_b-$a`{5Fj`lM-X83o(AAG-}~l9XfuzhF0cYC zz`07BQ0LGIXsy>g1DJAjmNa6nc|lD<+!>Jy+Kz)Sg{QCW0MKEj0BVaI{gK2V5ELzD zpuBgN8fA%X>4=ehjG||aJBZ~wWnKO*YH`Ih?%08X2qZb*p`f#Jr)lRIoSs=b28|#Z)cwac*+1G zW6}?qqNw!q4bL_`tDS}XI zVF*lBvOAOJ)tdazNFM?hA}(8}nyE{pZr}JVPE@*tLmlU^<8y+ZvK)1Ca;L_OT;#Kj z1pyRkl_yqJkg{F{ z(!;~>I^Yyo`X_~lnneF6iEKgo*%8-7py%^|BLa3?49F>|vg1ocrRP3(SWduFqJorljdn`nsQ|kLuv;US#pJ4Hc?*t{PniJabz3)_m(}fR zdtr?dW=Nf+s3?_OEjc6=OiEc#a*YysCDJTbl4$Fh1xbZE&|3?#6{7>2Z5XkOh?~KL#@( z#oK?y02pVApFnHoT02W+t|5*Q#cUX(btz%#0XOR294@-umCllT=RywxsWvcA$q7Wx zam&3dis{Oh!>U6t*Yc7z*r#+^5}_>_h7trlx$U`@oRaW%yc{xJLC%}xK~2uzud8uV zmr4tXPeCh$iSp2+i$><)_s2(ehb;_du+)>pnViK4_r%L@H9t&gor<$vX2N=8iAXS! z$}%HFevn`}0zq$q9FVU!yA9`Vk^Qa8;)`NlH>z1Q8HCL*70%VGyWBxe8ucT$ab7`v z_q4`;Ch-G3Gfgg_`Q$7B<@Xz=2IZDKwkLHYB+5t!fh-ZBWYR)8O&* zL3N4K&k|4pL_^m&fm(GK5v0ag`Wn zKa~Ifrm0NNnFA0$u=xxQsvCD)guS@l`R2GjV$4~PyJoNzc%nO6&l;n*-0h_m^5$? zEp?DfrM=5?^86|S-3`L?vxQ!e606Y)A#Cqid-#T;ZB8D!_bMqbs+11p0zs8pFDe@m z%4N40jlSCC1-6PD`9u-}^Mh9&%ALKSLPa_F+i!A9lCoXeHZ%ZjVrXWVo)R7|0Im?X zqTUYb=*LbaxQX#Y20Xxl!JnDlz^H6>c9PVpZ1${(7hahGpx9q+@xga!+FSyIa%HE) z5z18ytm9TgJ2`~_UD`$o@8efXVLD`@scIrCcS|$V2omZ zPGOdVr>|B1!6rWs#bv~Yn?eVa@XC@C>eBU{=YAg?t^wtQ2dfW=fZFC8W))NKaMJGz zA|%%1w&!4}R=9W#+Ro0cAT+IBjjSjTgRG`+)#||R>>4)Z2}xBk0_mU~1v{5s{}NvR z2vACAmhd1N6Eu@2vLw`6dRiBCv%Be`A1cugL0VqznEb5d4Wesu2Ob)SHP}x2FP^p~ z>QsR|!zI!wqdtmcWus#kd)PtBF^1{`Mksa`jbQ&)PRN{l(EnI0&9GEw#wVv#e}Pg_ z74P^E2> zLKCuwAetny~mcsUpdWk@{%$m|-bT}cS`m)_*|TpaYIgM+6gM@I$vbWPp?`k)0heG77C zSF8#7te7tXX;%w|{l_8rp&I~zJjCIL5r-2Kv zqzL0l9~HWm60`eq&!6Np%MXkkb||!6ch=wQblpzV>fvy+liOYb9BIw01b434td|c1 zS|?BiK}$7hhyoUCXg*Ad_MSaaS?5hODjFmPp+Q<*+L=zDz;dXDE`Q~0H=6^!RE=sS zm?b#?6;}XlHEPy2bRN26Xb7e+XB(O{YjUzOA+C|iU*Rl#8jZFJ>z{=l{5{2PDyL3A={AMKmdac>KFyoxGVWH%CeTDOy{sZ@OJ%L zOz`aEGTm^I(DSzS2NH68IC5D#rKgtQpuu56NmHp#G>F$GZV zvLl46EoSBJ135t9$9Smy1f9ubF;w0#)~M|E%J3Lf-?Is7Q0D^-Ia@scRn<%jTp5%# zJA1K;Gx{W|zkFyHUvwPesTVZ|rKmU&DB~^c0H;Ov+4^V)F!h=J!iVcTh>b2itVsOAy zoMh3VHw{_^BgGYl9<2Dz0=U)5U}u50D+QC8CTkOt^DS5fITE4SAx`SZ0x973?NU>6 zx!OZ_!LrbvL5t=_d%}!j%J%%8ee-W#zKIJn0*hY0xIBg48t}688=j#CI!2&Y*I+kU zxrWtb@1(+%bEP&OY6%}UmEnlE7gIUUG4tp219G%yfv_*b%XeS^mx%{sf_i@(Zzxdo<@J z1zHXu4q)IZ)dg9NiS(o2qG=1aG*=YKeQ+d8qcoN5REW;;3Z$kiajsHmOA4QO>+KcEX zDY1{aG5XgqA(KOqj6YntxdX{z;4O(wl9CbvO0EA=$r>j%?OK&PwmAip`k|;*G>B7s zV7aW50dQ~(uFZivc@SNvHwn1~n+C{6K~JmlpJX7wsE~lN@p6>WmVG=>>F}t9(fYCz zHnA^s>)i$L%2jUd$zC0LIs~1S zDz*lbpRskgPNCJ5mszo$Ruc`a8B_cXP% z^X&&Op9jvL_xX!lREJyOzF185EGWoSK#&Q`0hrGN9>XO`isQ5*y&F zNT6B^%2ET77Qz`+f}bs3SPTHji5f#Y|CwR!Ij8y=fwdBD~rRh+}+));# z_p`l+|IMV~|1*!-Q>`ndjU1Ub63V2ig(?BG!mECo6Ce=&!s{iLcqiBk@Ff;q`;eQ$ z_5}0fDgxXZRnlr&GIg&llxF*03YemDf>Q|)PZK+-LyUH|jZgY|wq_q>*Tk-`{DIwQ zjZ?0DcIzG4S0A{oRhx=9rnzN|^`-?_ss@#E7CZU6CH%u;pj7w&d4~fd#s`caz zdhyU&M5v{7zB-<#zO z5l&%iD$^b$-kLmLTQqNf{`w)PQZTRjL}G%%c=pjx26@xYIh%Y|1*<)ZfMyZ~$%F$- z3`$G++1Xg>IEt(pC#sb%WweRQFhy9Vy(Z?SXK}RgkXRe>+!8zGe zs&iHJ8iEW-g>WAorGx7lgoay^vu0TucI1ZA(xYbgZh@-xTdV+(slrHJD_fHRQ7m`7 z3ykKl3f|~}_4;XuUm&5+*hDMxBCJ;ogewwLi)+sXed;5 zSF5ZiflC}Ne;fWmzXPJWg+49BO7$mi_GD&g5r9+*6>?=RGwL5((TmA{c1HZR2`#TWeEG>j2vO!Wd zw5|OlpXD_{gd82J$$*+SbpSGGKhvptLgUplt2U zo^z@G))R#8An@R=j_k=XFB16e+TZ@`%g=xSiQ{(FY0IIF<+z*&Izx~4*?D4UTXN2{ z4~o_#$)1`SYgH`JAIWGBJXd&_sk|8E-o19{RZwx|83|RWL@z3%t2S|~`?6KZx>Yer zmd$~|!G(LZ((UapUcL@5pU3-<6e~){TU>7e`jGHqqJP?W9;sUdw?|2Nh(I~yDVjDQ znwz0B0yW?tayAu3B8zwXGs}XjX&KQkE|-$K!p}f_BuV{{n^2bYO0cgS@iZL;hu?d8u-}7lp*{FI5LrK3*KDuW z^ysrehy3Zd{B?N!ReZ2bl7(5p8Q)mCH!xAdApPMvo8?ydbO62MvOY+EsNe_GyB>DK z4g&~@5eL~YNZ;K^dP&pjED{Ns?nOiwSdjK1K4sJ#T_$B*2x+jOY!&1ISL%5kL#$WTdBduT}DV%P_r66Xy!m?qn}=?Pbb1R_#h*qNtKMo#G4 z+s|MB9KQci{Jjr=;Fl1OHd{~7tYvsyBs$dr2e<_6e>5wb5%DZe31wkkkyb_QPKI4Qy_LWB@&5aiahYFxKP%ZHb6!RA1BwY*i!YnnCWeTJ@ssWXl{T z0)mJfess!w1oj1yxmc7{GeV46^xG#n^rolFW3)NdBPv|lXB;?oBF-uwz+o011C^w9 zFLosW7&1p!l4ZUfIxIEZBgZVE|s^jdV2Ki3OW!9E*F_9(Zg4@oKlQTgP85hvt zO$!dIiwb}6W8ZM>fcGj9X&tcO;4gYo;{k2shukZrs~tR0`f|s}?R1FbIplXcKfNKr za9MVoBXT0KkrB|&p%%of#l+5uOV>u$(q+8%;2y#U$#&i()wzkbOUXTGJ22i6E*bwo zgjE(6?UX03JvmI#M5L&oZ*mvL*KvDT@umilaX8 zQsN(b3Vr)zFG@4enUaZ*VA^4K#fc^A#fst|Elz}65*g8p<>XV6O}Ep=&|z2V7}Z;} z2za!mp{g^*ab7aia6gkG++A2U?WA8KTdo3iieAfIkoVgj8JD$e=ThPh>nvI0t>V^u z*}ew(jfnZSk;OKrAzEJ zp}M;3#cHYlFxZD1(pflqK9g~X7S0M140@ZGV>6059ZZst8>gITr`R)Ei87eb08et4 z)wrgyzH)1fEJAHK!)sxv_Gb4~tb@pryuB{FU~p?-DRT<$9hRVpD zem@H|qF@&s_v)hM-8-r;vU=tz8b!nhkZdLLAv}ohl|KQo_)}DO-@pmtE)o=`Ibvym zWFKrIG)~I{wv;#mOfzNJtlgl7&<)$lK6~y|K;&X5i)-uM@z@r$fao4PwvyW%naK&r z@b-Va{5F^$n%V?$Xw<6uP<2uINlzIT^tgKln-Laux64olD&?u>b5}(GuU%$dKz1rQ z5bUrP<>fq|C5?bU1IvF*o7Tu?Qp*ekw}KjqL5<-lKmnq+Io837=8E^RXrvaeqq0`C-0T8CgNvxIy7ikkzDWkS24tb!WjSERfjrKN{uZgMeGbQXaU*!i z-O#UwC?-YEYDL9j3`4v+PB|8QS@0z72Z5@|u|8Y&+IpyuoIW|5c34?1ZBh6ftsgngoz~!N*rlI4X!AUl{Zg3R)LgIa_P@L_LN{{MZ;A> z{~rTE3fWjB6@#+6Wp|51r7A|M9UhIkv|C(K`{y2p!qz^N(WCT8!g{(d%nS&4q6I8n z@N97a8X-*y8a0!?G~o##wW*5}8KfpACWkHt2bq0aR7%>wq3><*?B*n-g51vk82%o} znU7vRw~R^o@JXt>JjH597a2DfOb=4i5MmaJ$K(8JR1KrbWLx`<*Db35t3sotBqH;F z1gw(Y5jiH+Xne}mBHFYzB^;4kN;NFCr924d%xNReWiOY4B?cJ7;Jfw;O1Be6@&*u8 zCo<}A#1T_bO$*X#IYsU7s&Pwwnp9lhEh3l7QIt6p0_D1CCs+D_5hwWQb=-#pXWns$ zXPhbZoN7yr28OG$UQ4cW5bK@^u7F`~=uf6vQcUJH?}Ok*>Oqv1az=WGhpCo$qsSXs zrM_pbT1Am^w{J?@XB5c~IoqTpai)B}#H!0HJ=LkRSUVyP!r@J|n>o*xqnnZd^Bmn` zY;k3qcK+>bp85j^Q&Ptt+YaQS_W*j5M4zBZ*U_52*H+3_YhA`4B>i4hfN!`7pd|5Y z-bT{;t>Rk*e-{4^Xw z1x3@1$6@QbZ-ndQb}=vpl=6g`Wu?So6xNcxqv;z{N21%DYd1YI|kyUNv(q_ccs?%r%&@iYq1WmmCNq8k+vVi#yhx(Wv zm}ib86{VkT!Ah~)i8rr?z?wXvfIroT#`OQSxexw1y#Ip>gc0QM=;U(kjJ^@~iN2kU zTk#UWq_d@6#Y3E;c~$9vwUWa|m@U_5lJJ+|w$C2uz;&1P;=rumO%cN++?{D9*nD&VJc+- z&ksRnY&&Q#8Q+>Zv*U_1!=fLX+}j0=EkL$T=}(ubee1Y!u|Ukify&V}y)6{;Y4zR? z+MG4@m+TQ}<+)0vmxQ2Y(@EA*(Q1YvSF%Ac##j+*TM(SLcslVgEp0Mp?&_~ZsKrgC zDzKKHC+E^mJcvjN$3wyH_dnX&7N zkVMlY4=(S1whY1f6i~WKf`M_+tK}`1S)L^mAnn_> z8wNB5${UOhc9x&=hmyX>rmR>CywQ5h>2&v2&5XP#Si;?qnStORDJ9X5yGp&oguZH> zz+E1zJntu~;o#BBgGohsmh7Fl@)f@_uEgHVldKC9ebcl6-G*AO*)_N&w6z?PzIPr& zl%4taivB9f*-9a3Hqi||{mVeCgsE}0+@n_*bM7Ks*}sI3djwG?E@N}ioH*yNfqy$R z7Y+;>FDs-ShyZTYEpsF45(0gMs4WTWKdFBN)yaW)nh64KDO>=kp$fdyL=xYcsa7?P zS&NdVloQ1s@!s(wbmFkj*v=f|K{w9_gQ~y^Cs12TK=s~Pt;6-O;)AojrGobH-fLZf zOSJ8?A&q2(2ZOl{9u6xYqVTb0PDphwQW6C%D=a%6B+Xaog~^uLu-TsyU`mz zAm&ibRW;I;kP4!j)tFM3+-XxJ(cubsmx@$aGeZGNvrczzEg6gzCjhGPo1VybFW>Sj zQL#3D`fo2^zx{72!t?)x*Plx*J>gf`G-pZ!dbO)#DvVjCmpD6c{m_h_!$V3MT3Ixx zUTpak4$qLXdVgoQvL<0&NU>17N~(XGVX6OSJxVZ^10~dy?#XJ`O`vy6E{UTZt=a*{ zm`~Dsw-<#(>lFY(HZ$NI12n_PX*`CM6`>O6Vff&p$XDnkT^U)bfvU*1mMgW6U>`L} zyTSRxTp%HVjtVx%jL02Wr6MZ7*i&V8)W9DrWeMWk}dfV zBO38>$zr6*6c?Diqs=3Ek_{s7p+Aw`C}#5r$|(iu`W^M>3Ovt{#LT1l90~ckrIOf^(hmifD+Hf#da}E_Dd5=S55CKiImQJWnWsyX0>C3Mk2DuwDYs&Y5(J~i#Y?YQoGyqv7bdnk|t&j6`kz`(fS)CjqN z-KmdYuZ7expbM~?`_BfMT&sIzrPJqX&03cv>X^HAZuXApYX;?hpg^U@nOnSJsSZqT z4eXnB(u7)S&IhF2$RCAio~44o7_is6j@~_ihnEX6@1aJc&Q`*(QNnW4QSH{qClwjC zGg%IXD)1R}YHA->dao^QiA?0{pof)ASkaoj$!ag62^T;q&@JYP355(JfP5^;&dbk1 zqtmGAq7|S`Vv^btuet7OF=yLCxgf46zGOE!cQ(n&u1PhVn`?~0VK{8h0xv?>(Fnef zno?f@<(_JZy;KdnO5!|3u1Mp2hH79X7ilAvkC`PIttE?s*KXq7n#@0W8c~{iOvb8! zXOEI&sMG<1A*FAA>OE2y1|WAZnQE!q`X)u zx?&ku;AOUuP}p`!Xqn8Kd-tztrA#BADB2ex3CC#w zys+JBU=P(Z+PR<6yDTROwv%3y(=?G>I-kjXW2zH^&lM(eCCXbRlcT~?&I2>1Q zl<0Fn8HZOTc3=x$*rrH5`fNxgP@}``@6-@L8~Hy&Nk7Lv+FVv{d^ZM4Nu4ycTlElQ z7Z@766CXtt`W?8teZ;-0K7%6vO`3jfRi@Buw26Cf0)1g@h=dv{MZxf~HxpdDC>J(v zvgsXAFBhgm7dLl>$KRKVw0Zp<9J*e9s{i~*u{U4o&p$96&d*=IdHr1Z7vJKaFJFYW zKYRJv>xc21w;#Ry{`L27fA;!2%L*dmE)D8XJ(Za+Fb=9GZP>ckalD}^7+t-Sw2bZI zPSvUGnM#>Bq5Lkq9KozA-oIo`s0-RVkC{n<)-(mcOm${d>kSls`5SL}#pePp*B^r_ z!o{{5=mG|p!68Uq1OySIrV1w+&nYobmIh3*jEU!dPvW}!0j9NQZ*(w19N~g~6ES>WM zSaMGE-cuvxEULA$y}w+NjaA~Yl*->p!a=2ds5xTTDj#wU_L=OwdLr^0 zRZaup$sQn4gAc1uK;2$xtlN;Nl9O|d=?x1buvL<3UkoGY?%K7_Y9DMUXIY38c~qFH zMZE+bp2i6ZlGbd9I`7Voi{uH3L*`vn)L1v%_GzZod9`{|!9mB4Z3+Osg0#|-l_kFb zl9%mjFA#9kWtta^Ra@0GQhe;DwKPoZeE!mcNE3%emm^IC&_e<1?a^Q{)(f?&?y%}O z8L??6E6GLLhTLWOT8?2rYhXSASznlu6Cen&Lp08fgWcvUu^+&w>d{qyS=RQNo7 zV}})#276b3?K4L<01;R9S|XW>ImkGRwi#|2xM0ur2ti^#h^FZgWhqxGDS(e)ko?Qi z2F5wu+o?}$7H3BMgn?9s84$*d9E9YOeaZS&?RaA2azK|^RO*BO`t?`V%eMFMHDAcr zQmETgeN4}C-PUJ%XxCKbuWp$GVxdQQQpv`c#`YfVKx=p1I2aLDfiDhvW%J0&_!6#eIRP z%oSsmm7w8j)~Vv)8a$)y-V`ylQQA1CXx39ITkKKHm~^P6xmtdDTm}-y=FyX;d{&k% ztlfK+US#sv-cPEgH#pd4{XmS*CJRP#1*~#Qp!VF-Jg9a1PCIwzWsR{vP6oKdK zGK87uVPJmS54R_X8OH~reeH|0x(2PBT}xspP;(i49Ihc{8#Qh$9_^uo(#^O!(dMe^ z4ICoEryYPo*hsp3z+^K}D5S00?JV67_YNV^c#+^8R$enP*8eW(qq1%EV+ zre);dG*-C@^EZHi)=cQ3+A){M48)e_6Kv|5iJj~YFVabV*(2)!#aSUF2}C%R5zs2J zgbhm#6%`(~{f>uIGm>v^t4AH>p(-CL!H4eIg$_{F4=p74M$iV&HqI-$zjxoT~0J>kU_P=F9(e@ zyUiqqHY1;~Qg%ush7RE9e76XYt6%{9PI{(5*kF(BX5W>o_Q&I&&*X(rIsWyV%iB-G z>z|S8g*&=o0C7LN2vG-ZH)7RVm~?MVZIPz~$Im+C?Kt5y0V0}HiF>;wML*R~hWFb% zSKA3v_=6vWAN(MG3V##dIy_CR@ws#Sul;FjddX=EVZyD!!4vKoVkyesfO-c#t%krK zxgLAkSAbOPxIcU@k&kN3%cz3V@p|Up}+OYUH6E$(nFM8{T!-a|pWu1uS97u0YFl zh+HW=Jzn4B3vyL;eTh32$&s@P_?gnc0tPznA)%#UhIyT`vZR8=2obSs#%R$i;*?H7M45l=H)fyPz#s0@UWMoqK0`9#;6o& z5z!@XP0<^6!_|7qZ@zo|j$gz3Kj2sW(?>=bUtpS63Jv4jf*XcnLFFkGJY{AD{`(+D z&Kc_}RS$%0&Lz}8MOJdHRMs?RDxp&Zu*tNwcYDOarrS5EjXw&M|Gi1tLU9@!#-ztQ z7X3Sx_43>yN(sC+6Ug1QSBfP8P?RZa1B(w{Ka}eDe~OD{QQypLo9t|b6iqnfOI9Qj z5!BoBoVAD~1|a5gP@;P5IFdQYYO(2{%nB5ZJ(ti}N*)GOy1*quZ}wXC$y4GQ^-4L9 z%L9)MfDy?}#i&b>u^`sjy*XtmIuX?b2HOk&HCMu~fs`C5^tVFpY@l_lQd#T<$DLV$ zWTxmV6&Xo_sBbJckaI_#ih(ncdA;MPe$9H+mLwcQ@HJKHEb*O-q0}K+22fhV^!4T)s&!wkM0%)Y0R5)wXk7_|`D#sF3jjSN z0Vf7yR7_$D6&E$_G2zso>d)hBkA(v0+EyLuc+$tT>ui!pD2V{#uxl#Zv1^c5wiC`c z$a(};Y@RC8LgCvjE6tdONTKW*X`g3dsW)y4nqx_GaCRJDdS3_5nP5Ood?)ISoU#LG z+qtzdsu4NO?l~5^qDej}khehv=@`&Km58m|6bkEiFDU&+@W3zXWoSWE9ts8xdQ`;|<) z)Hw;2`W#%F7`{`WHYtdGT~XLh`vpUQFz<3vPlm=is9qOyvpiYxO5UHq2?b3uD7?N?UqN2`NqtYr^Ho(LIufEWOJQ9lmU$8i z6><9wz((=snYUV%h!AK{tsb>@1h`_9DmD{DnL#XSSsLiz!dQ?}vg%BUMuR+t8&~%> z^|a59%)#&fCH$>^yN_3e8DAhHYQ-9K4dcfN7iiIcI8J^aO@aq!&zJ~v(iuB#!3d{! zlr7F1$2ig8uABDynjMp$6)sAF(7Sc6{;FVA=X93tfo=!ji^^YFHDs&}d*~XjB?OLR z03qIvrRT6qt^P=wTfppTiDy~Mv_M7Ka+(UxT9P_k9F*AD=is6VFXn!?eq8eX!#T$CR~C-t8k&eC4RD)kRU4m=fJ8ZKPeIV?coJ zw*t1(pbZ;xI9W#IQnC-h%{KBLu$a7^W)#hX(=Pf-XD9mYe+d8hkN>Ejen@sz9N}x{ z*UC+}7v>J-E+@wVKdf>AD403yrX+RyTp_4C9uYnZj!#wND0J~o#xu6|t`U^v_z;EV z*KEf3&M}Tn5C?;!4V*E3_Ra=6AyGEQdn09X)+qXe+=rZ_Ov$Srl|=Y`%f_xrspbkY zK~X7@%(X;ky!DCN1;bP(_kl{0K4``j^a!KAn6qbV5RzOQQ2mh@@1T${GmDaw5KE1K z#9~+1coLn^zR5X5S5u3)B|rWC$FJXnx1YX#`1&e`B~l^S?( zWk!SwNhqCPDJhW+ktV?4xEY$4!@Nn;CBiL`~JsTPgq!eoKT$D&kBAe7uA}LC$0`eyRm)E!U`c}kc^@A-yyeIFF z8N=Rdui+a&TtkV5hNvn{w9)O4pD-k2#eXNyb5>s>Gu27v=S#q)tU7081#7@U8kTh) znphp%9U#1N0hf~BydcEh;YMMn%!BD6zJpW(Kl+iS9V--sZKMLeLOifM;Om-@DzZ&k zi(q6z?7DsqE?=vfjT1`sSC9udZ>Wbd}0+IOJtYeNj-pho;Bkt zi%5-z2_>xIcET{RME0uW#EBL{@-CL$Ml9LD2n+}*M$UjU`EXa_Qy`AJLnqrNRpaa$ zU)7A%?fWE72u#5OMEN+JVbzMUn|(e3>HGZr?mo3Z;yRp8x_r<9g7xv+Hy`|7{;lx- z5t}%X zR-<9;8l8M035tQ|hTX74jw(M*a)qUA ztixHkjFTQEYwIp83oIr^Wm(x$zoCf&mJmp105w3$zbg^+i`y?DSWS!CsW$WYU?)+Q zu`Jt_0IWniLsccU8#;vHf!4A4YE=f61hHr-PnKjk*^hwgk7o5<jwMS?IVZ= z0B`mSh~0*FXzSR)VT($ORZ@R|62n{dLxJFO4sbUBxFkGP{o4-u&{bW-I_aWnu_k%x8%x#13OWy{@qi@i$?dB?RQStV zPAAPT7H1~OJW(OT-6^@=Oucdfgy|>rFrf24UWrlvl%Staa)kNl?d$ORYqXW5imy5F zl{jlH?xZfTcJ#G${gp`QU2wxdQs?ldv)FtKX;o~iTnmN^p1GBCQn;#Ph!h$0`ll1@ zYUweQA>e%`<3kOUnR1~ft4^>6^f<;*EkfA=N8P%RQ9Gn115}OK6)@aVQ;^oB5{clJ zY$4?bT}a6W40_Eg{hl54x1{@|rRI&_N05sob-PE2CEI0{Ur3_UrJjl3sZ#wS&eq~G z1z8GgnL(E0gtG#b&Wq}8@s>JTWmY+Z*kscZV1pHj1m21Sf6JRBXwQe^HH1nlZxKKw z*V66I9h++;0f%`RfYuztLhjn9KwUVv1l!>1dMjEnaoDqI@CQcIFF;+v3e!2Ey10*r8q9owo=%#FqC!Kw+AHpR3? zqHpZiHrgrOLnN70?Y;(Hh87B#DHyP^iz=y9W6))Q=FQdSONe*EG^x#+;+RPQ{Bu(C zTnV4aN17cPb9fr(%1M?*XgI$?yP{zSPLiYt5X+uRY}Zm{l};-W1^JFAo^Kg_D=CoX z4XLJyzm8G>(5?{ptY)s=Dn?F1s8901S;NB6JYeU}JXOBHc#p$NUKN#~LS-RH4Tt(#1u( zY!RNT8^ln%kge#lKd@7?9`NS8=l-xPh{qAQWVORj!ZpCRSzZn|?^GY|di~a*?O-80 zTn$QmOX0pTJcX7;7*YXB(juRLoV!IN%};aO&8OFwl&~Dn0?+`Iw^qUdC3WavsFWX_ zK5JJNNe(ONl>0l6vi9qVj^)656}2O_G216)`fN>nQ!ytR(@9Nj16%p#gJz?{)>9z> zCNcQ8Ovau@hq+xtkz`w|KcA0*@3ogO3%q}9)__1j5nb?)N96rYmks_<`{IvlQdJLRi~W!z9k~7CEg(=;4CfR zOIW}U>O5bJTp0+1bLxbtvVzg6(+w$kzR5U%V8eOs zqymP!&N}UIkaRTYssD(a2n=Wp$XkMwK!FaA#Eci)Bi^vMr7Ej98s5FmBD?G&ptWF! zz0Q*2!^z92OQ`l~D&IW#AXihqm?3Zpr||U_pd(<{E&Ya9I$UPX5wQV zsULLfyZXc%Vos7P)!a5c1UbcX8mi^50rWDr^2jY#&)W(~G*y-A_m}rSd;K=NX5`lY zZUA=J@o-^6pvq3TU@A-&Bhm45y8o1R7_m3-k%bZiJlN$ul+J+bhb+Y9I?z4z2s-3f zvcPNuCKVU(L^iQTZ6Z7y2pukABc6xweI)&%Ku@4%vO#Y_SeVH-K{JceNcRojOM>s+ zFb1nyZrd<;tp~a0j_51^2NKevK8yHqTS>KN^HX`>**#t=1KgqQ474naHfq6EHYNPs z?ePI$;zKG3nfr)+PIo`mX4T5Z_y>(+x)gL+))d8r=k$)`2C%q%Nn4fCL~aRpy&{Wf z<(*#I)Ahu2&CUllQV0UgT)M4 z6(871T?gCROApAHqVi^)Lq|x-s9d)E9tS*9h;`$x_ahg#W6p(ksvECLP{Cn;DT*?; zZKViwK0Snc(*-6s-&}S#U3s9+YYtmXl!jNjx=KdmoVE3&>5~vi4n6?JOyyQE2}BF8 zt?$%?EOXa0Q!(-bK-a!zqo)bpNNG~nB!GN6I5I@aD4m_euXXJzL@IfZp9Ksm_lt_> z4)(Cy9%o zUj`bDCHP_oH)&C39aM;Bhun;%Pif@jr~ThhH%brV(0=r;GbTj4M67EUpASKkDqdN? zl;dz`8#m7CwgS!c{2bcs8aHc*F5wV?r;aYP^GsAC2>Ojyp&ns<0j_xn4+SQZdxe8C zK^z*0tyPmTi^e@5q%gdsloduBr+o%9U(mFU#0Uu1m1;k|q-%CoJz*C_UxjLgu+T>; z9rjWOX#|jcBgx5j3zECv25)DpVawX${)X!83kc|p!-uMtEr3ZPW~h&bG%~u{%Oi-a z!+FE7m7sXp{WeO3L&+m={n|SiOp!EfG>l}47F$)dM;p@BRHaaE!Fsd?-J=YJrFno4 zu9UD^_DWrOACwu!r(=wxO5g6L3JC!Huk0X6SC`Tqh@miJg`tEtE3MTc4NY=eomchc zvmLLx;aXsG`5oQ`8^p^p_RJs>vl3y-Q=O;MuyQAhQz7J^qy{=_!Sp!umIy09&{j$v zgJ87Ez7&`>>q)XlLNI&)mU)|e1A!dJdF28P0Juw#!;*kbT7ws0D6=kG?80>$*C8{j<5pR8*|E<|#J)^N?(dxmIWCQloVZ1m_uxX8;p=yP4=XnvG zR#JX7z*cJ)qm^S?ZVP4`Gc|au_np>_B>nov!FLT6Lp1bE`w>%&(xoRqfBSuS{gq-h zueHG<&`p@x_s3uwacIE|?4qM{;9ZBd12%%EdF`gTon$?`U6@*M*U0Z45M#2538x-l z0+HYZYE#F4;^K_MYf>P=5SfkD%3o^^?0=QH;S}qr7hi&?24hJ8f@)0I{zoQHl#&&_ zzu>Ro`@fM7AMi=>{!XuNsT&nl$m>z+9rs6?)X%|Ir89PQ#C*S;Q0*=DcSF`7fUBs> zt(OYypG8%mE73ORnvO`MXvc?dch+rjIGW@JCS@U;s08(_qsLKqF4?b|mau@mj^1v# zy7Y zAc$a6n42(~?27QDcNLriIYI7;T_rpN2nH9)M+HxKKi1=ada_IvmdVy$tv7y~yKFAi@R&|ecxiSgV{ni`hca4jF!iTUpzi|&|-x2f6 z+5n4`gXDx)lEPq9Q91ik)|?rZCMr*~XMyf3&Rq5YfkWGfADRy@PT(giB!+B*V2`-) zrl~t9_>67Nn4Pa{n!(&dNtslH(Jpa<)}uYbyl_$SE_zk-advg!Nu5v;FyOSG(u`x% z)yV2L!o1u(9LQC=N!{J)Bq5SGW)}eXqP6_J-!UBBn1#lxL?_DB16nl_=^#5Os@^z7 z3}p$;P>Ph`HaSNfBdw_WB96sSwR z()z0gOKh99pjEpbI|gRxtI8^>&9k@QU@7ByJ_1g5^ub9YpA7g6YUlS+&m=h*d$-mJ zPxfPZ-O--)DKUYkZmQ4;G=RFn?(JIH!G#iha^VIAfpXG8+RLg zRHTNPze2acy=|o(WBq)Rc2bnPT!HO8ry9<9vQu=r{dJt-Xg$L1&?g<3k4i0!7Ro6~ zY%|WB{R2{45fv^a$qmj}#thK1PwbCxhEX7rs*hBo*WI;QB4&jM9z$J2W8JQIA2ZB4_6P zA9PGZ?z$m<6@{#6`L!F_d#o_~H0_$swfQ@DxP%zRmX-D)T*>)uTRT2V>MQUAs2JR+ z7g|pOOCS8<1ym*R3ph%ouaw&s6wD6AEL8EWs>vVu>6+niWWn?u{t0`Hu8Bdbgz zi4JB}f@Lfur7p=yte}#|?g|UfO(nE_``2vZUqi`(RP9>#^9F4{m($WYrD`@FNVgW2 zVtoe6ZcBmb+f#c;Rx?cvUO~!`8qI4MYoC+v5ps(wX zu?Iy_cKU{-^rA^7t20L z#|?xxYhIvh$AKlcU9?8YTkq$uzkK^Ty#Ms=tMC3gi>2Ahh7T^V6Gm}?p_U~fHL7BD z(w6l5OKEesr^$DOzdQh#adP4RZTyQPw z8qGi8hvCoiq<+VIc(ae8#j=rq1!&Ecx}DWQ%k9T|s?RoQow_}7y-d2cy@}FGtrheR z%-ZeewnzsRzsisBhE$Ll#ycrZtYeFeP7}I={tSmGdAZ~hX%iRI?ZJ@x5+yLGI^_Fc zN(z$&(wN4%sX-z>(7>=Xx9u0McJ2WwxXKokGH*%tDkWR2c&KGb)jmkAXKMu~5W|+Y zZ=Ud`YXUsfUud%6A-2ov3f$ ze)?e_QA&nG`q2)(8GY+Jk<)CLk8t+nJH{M#jqmJ!LZO9RRQuQob_QjpEXC1~ep{uFx&NH~n;JRB#$$yLt?)&VmO zpuhr?sW1&Y~e8iL9aS zT$7L|4M>dVX#Kw&TdH^)SAu#}ucL8(>8(DA>$J|nXn7ziklFgWb*A^m50LNW4Zbs` z`hu+H(6MZwKvC`Lr7167cK4^@Eh7-tf`^tI^2i9Ry*TV{cicC9&=MfOZwSp=+*RT8b?IXi}W**ND_^E!{kWkDYGWTQl$zfKipCfMX*e3q93C7RbF3B)XvpENW^R61@`M|D7D(ND2)& zKfC()KqAsEg{Yt3s)~sZ)Nj~WAVP>?fJ+@aX<$|dL%P2QjNaK{7iTG5q%J zGg~M3T7`y{)M-yY&w&C*Lmy6tq~wtowG(maS`--U;)q)72QgG=CD+IvB>IB_wqLHh zK`5}A826PrU(@0O1uCN&s&`vsfQ%Qe%UsH#luCl0@__k(0*@VLF;aNsFV%b~KZLG+ zM(j?Wrn`?TPY)U;e3p-qsKW)$Dep+)Yy;5;WPo!Ji8?^HQ7&2%bCrB1{^};nG~O=h zU6@xYOLT!lS0oHc9Yl$y?LZGw_Zx7qt)$T(HgFG78}R*SZ$G0t*jCa`1CEbyWwVTN z+SDT0ln&sx7+1OA2i1Ujt$AhFgbAowgSB5ViP@fPk5*1N!nZE=;qp?;B{elWGLO1~ zajYgTu0n=})@my33tHTV=6ihJMRSiY;}j{sEXNfBdvnn3&Ek zeVE|d7ua<|yGP+3upyJB7)w`$8w;nV74YCr{pIETKSDGAAKyN|yo8;M zCK~l+Xw=nn+pJ5q$eBv7KzKidCu?b4tu`2|9W%OJk|xxbK<)sZwA6>iiw@|?FIUf` z(hV+=_UyWIHXAL}&-GlMa-;-)UhyTPUQnSkqAuL$@msP$)*`zxgT|$N%HVt#V50dN|WOLBeURwsZbqz2^aMu(M>YlXnU!ix{U@> zH96oRnolgv6T7xLM>z4(zj*yR@I#hJR`hLd(vFaSvTGXk{0712MpTGa?2cK3^#k?u zZi!&aG^jTnB=@MC)$z#oVgoP_|8a{f^=-*{IS9n8gnO1Cw54F?r;P=VqF z+kXsXvQ*j)xHOv@vJD4lA+JN*EzKA)?mm94c)%9b<1v-({#j?Bl z50ZTOCTP6F$AU=!JZ`PMtBnw)KvW4NUOAeNHtY=^!y}9D>K+zD*Mm*T^GQX9Ez}T* zqWAH2c>7bnIGCLAf^_T}3zcxxGvm-dy<^NY;F|Eua0+j-sAfmCO8VGGtzP=X!}l6( z@>_xAG`s6!gwkd$34^Os%s9p3`Pv&rF)GRIqC%z?0_6xcLhd+2oWh>HtU6=3D)2Aa z$W#(IOM$^zPCO*do}=Np(=@6D1W1wfxki)_kQb%6aCJPCMDvnTo_JAM5RVwsG^|bl z0<^ce%i%1=@r$KD+i;i=6R31{dzJ%4EkOqAQif(z-`@Q4cR6fAn(y8-sx=2P!fnM zVP~{;3bjQ_oScP&TErem87Gh&ZP$>^|69h)x1V12t>5|Q%eqP4>C9^dt^^e1tX;W)HkeEd_nSxMq5EHiQ}6QX7P^DDC8hq zThbZhMAVX6*yL>1(OA!rqmNwj>T{&S3*u}$0Nii&rOKp!EJWrFJc*UQ>Vq2fOx`ht z+9{4JMa3wGUhGySF69UKAHxrS_(T2lAHt7+oMSqI%aSf=4O~%h#tXdQAFjV98Cq~! zGrIqCeVLXWWHk(zBsS2}5gG|LtF4@5osfXw#0Hb+Wme!6JfANp~vn$9Txs z^txpSrocDw0tP66prgPsa|BF%bRaD(Oq!N&3*vRI5(ox)Q~JB?`f(($Lgmx$wk3D<2TqCZe4xTS6 zp3N+Sk%B;grBy8M$|>t`tT0`Tc>5=`10|}+hS*IDbAdT`SF5SVVVP4Is=Sv6+Pl>L z+++ss;#WVQ9cI$capuemgS1FYPm5h;@k@*4$XOOt>~zA=ao1Sw0rkzR3#Z=G0`G5J z?m#C%h~9a(cHL2SsK%zH6uo4qiU`eLrY7{%Um1I6V4^YTmHU(YrG_VV*toZvh(pa{ z1qiaxz=DE-wy_Ona4gyiQuDF`ZDVq5QNk3B#F=d6dr?gVqV;p5M1>|I&o_wV)zu-} z2X~5XqJBXM(0Z9V%=g_Ox?iPGf*3Qf*Q{JR-6axKMtdVq68?xLNFBOYy=@rir9wE- z39l0YJSwfU(*@#7mnJ-|t?>F=KVr_Zx&t(9XE!_Ux2MpuBLtUXc-l7p1aIo1V@k*9 zy5caqNdna;cavzj$~catRo7JqqlCS|9^8$NB}lYt3$q+Ri~@QnYGnO5wXA>f`kVY~ zFuwa8c~{A&hmGln&^Dtiq4DCP4LiOtOfIjaPr?RA4sYV&!s52VAd9eePcI?f*gI%| zHBR1y-xTgrqN#+s`=ZRK0y5{Y{v!7k1$sH}%+-JO-+58rO0oE7`S_W#>SK=dBc)Jg zofnL9X2K4@^i<@3!8Y7&^u}Vlp**pC4w)=Hr$DG=T{4o5i~g-2&842mm-y^JfH(}g zs98~Bu$9I8CY52ZgCq2v?6U|u5W=7itS!^2H9gIr;jcwQxc8q!vEYyP`Sn-f^>b|% zfV}r1kt&bWAX{TOTdmE$(qcQ{Pk9xWnuuCxF9IbfAKD*1H7KFyW2gOX_Dman@CQNM zs_G1wog9dc@4P|Z;C@1b-~Mv!Z^sO0=qi-0Lq7&J2&~9;||2YTJxey7MF_> zyt9u)-uK?rR~nX-1(L`O%G9+itX2Yr&fX*d0XBqAmO6qt&j*O;D?;mbeyt{Mx?Lr2 zRjZgHwk__4oHf!aMXgH@1tOLZZ^E2F`z0BC7O0^jp`K9k*Ip@H@ft6=yEVaL! z|0J{9noEwX=x2CPSvE*1P=gX=lsj8i6Bs;6_V0Rd(j+QwFiT?un|L0}vtTRK6y+{W zasqzr29IKRVYa`;#)4cM%pOmY9K~`zg7}Kh3q_^>M`7L%W4zLr5|> z4=3-EKGP`yL8v+Q0w5_b+5o|ZW;~I}=djaAt!xqBphlbyyHPbTkR4aZTY6S0>CQ=_ zlCWk&r~G9Z998>48!+odmyVBrXaHBUUyW%NMeFJ$;I+g8bR1vo?@*CifL$Dn-H+>8 zUinFJ4@?ERZd4x+)ceB$>;*DzB2*@BYlbX*peOokgEIdyeE&D{_4k||-ZK`;%MrL| z&Hw_dLJEcyl22_0x07UFa)l05wVC$6PbbdvSssz$7%+JDt;(F$k`XVbjB7n~YO9*> zNGWQWBfzc>6w<3KkpEE@wEW6m%;1d$i4r$xP;q%;7*n~*4^|};b`&$@NR~H9@V;0w zFilcxcU=j8u_xQ3=qf zz4;Y_BQcv2r_n6P3E;a;&UIVl@{)362}N8YX9X2E4K2|lxuV^^I8f&b(55#Q>*o@1 zdG8a{m}sbjEmeDf^Er&@ixD~!o1N=$?j-CLQ@1O5Jf6oZV}Q%#q%rkGT4uf`I@lLL z;Ya~S43l~&=8u@qC!QZ)hPN*-pbxVLAOT6Xo*eUcib=M!l)Zx;9?R7p3GAzrx*;6W zU$DtMdflMQe~-`cq5ijOy}I!EteRR^6(S(#8gyJ6LLfg|T(wLmDz)4eOw{ZdO1_of z?6w^2p#8{)u19l!86r-+N#f@(0s|;Kt8%IVz)L?x5@`nv6wzmt{{>Sic}nX^%`Fim z(O?cluqlS}M8LAC3;H{af`;m8{F7Fze!H6luN;snWas zr2nJpdb7*w)kAZa>dYWnCAHTSlAL^)I^IlM>Ua!thMHU&9Gz6aBZD0$&p8%3&~yL{ znV@<`4RrVf>t72`V}2*HqHN|t(i4Qm@5Eqr_`4bz@3Q7QU<66PPElB81~=NndRpAyzS>MiKJeG$Po=v2y@g$gy4K(h6^l3M zrFqv37B1|(-yH5z$?1DfPi)bYJHS2FfwI@=BX6*YJt3%ts%*9CF+h0|-T6G7jMFbN(9m z!Qx0$7>;_M%#16!x1eko+#2D5%Z6D+{s>8ql})oA{=R@TB&%4iW?_R%wSzfEmB=0L zbX!icfi|}HpT_$LofC?S_mZJ3Sz3EHWt+J`*=p%*ghS8y;QH$IpL56KmzS+qt5Irj zgZ<|D@goEE?T9;RK7n1wCfR~9Pz^wa1=^%zXDzJx=H@vIAwfw$3%VGyx`RDmg$rQ8 zUG|ay-Qp~(JhJ80j_@V(yMZ%mz7W;wUHv#i_4!2Q-oZO{GrZ%lhpfjWE6Hdr5>CLw zWjkFd3Xr|K8iW(=bMPT=Bc!>P8+++5kUcy|zb&8>Q{>nO|3;70 znS>(S27pU4nOm+3*)@eLIfF@%suBQUl5A9xEX(4rPFfu0>YiQ0nq+x1FPAEGRS)=u zbQB4S)=ibA@XhO2)SuJk5BFg^i$(~XxeThxBfzI{EFBW?1cE%~*iHkxN=|Kf8ITs! zuu**bG%?p!rd=KOvHEJ!6Kj!_>=`PjlG0o8=OaQ{mKNA$xkAg0NIPAaO`6$2g607~ zx)TFt`15Gce&_{||JARDc5z==4hH9<^kN4(dJ0Jm+13^+6#1Y!8^XlGRI&DiN_ zFxX77M-FOBwg^x8?vX!0F0CimPJPAlXV^tb-Ex|RBA;B*PP60@XU)8$dU0mH!ZbQ< z3;WO;8vn9~)NG2vtetk!+$DgbcZpQ8RvpxEQgL_)_b`wXEN%lxaF8lw<(=&bRM%^i z!*vh)ZewVqf7wICt~4khY2mY#B#w|l$ygW4odg0rp+V`uSh)o@OzBK6f}uMzA!I8c*cAQk#TDzFNJ8dfl> zS7`psVExV&*@jT8S;7FOBGrC^nI$6hla5UdT_sEANfOI+4M$5_c5(b(<6jxds_2v3 zl9I9pV_OG(JTwRmnsh81Nhs((eWsz7@ zGE`w!#xO2FfZP8`c4aS{8row-r!{f*C~)0WSoJ6^NapKDN9LkXQ5p z#!Qj7`mRvPVi=#ZiVW9@vCQgJ0_J{WCzu4V;t=4J6%U4#?5|WgT5Hi>Re3v@P!jjT zO3oTizBx~`r%AV+GMXf^9lA=rJaBb@S=fVC&aa7JVk|4G`r7k^V=EE~<%16rUMvU5 zYYr}00X+mml*+Oi0G+ANxpVaM8)(ko=5nZne9s|DxNLYQ2kropA>#lw{MDTZp*$I# zF#%1nJGFrN0|nHh;T+|kw}kviUe4ZFpSp*9n;WdH%PzZU=K$3OI|-fw(O8&x^;F|( zD2xSVz|)ZO>S(=9*=Prn1g&lim+6y+rn)t6CHaK&Ldh;0STHm-2vCn3=Y70tXC*=T zY^`r$V3Qu()1`P-VJ7me0)`ACwmsAldaCPzycz1c>gq#02$nG2htmZ`?h>DP<%pym zRp7}YGx!5DL73P8n5?MZi+0dWmNyoaDt=kc@B*D7DxpL2mb&mtP4wZ831VQeZV<1w znN{qvAS`hVFjIlrn8#Cj^ppne>w{u=RYH?dN=6-C$N_zgANqM$H5^xlJJ5IRj7f?N znPB;Qs2208Pcy6-DPJ35OOOUb*JrjASYuxE>u8HB_ zf6h@kD$D;vQMs~Cjy}dnd)&Z%ra#yqXN**6KE{P!U%pCJD9uYL0pxSJCW(68hh~>a zq8v}1*-|xf%TyU1ijz3N-V+Sr9Vw5kg;IzX;7_BpO2S{^~mwS=~-0EzAIk0M2 zM<-dWwR3)FSWj|uWy0h^+~7IfZF&sYp19q@e9ywlpiDLMc7zBj7|3n`=RE=z$c!Cs z;otoYnQl=8^s!9kIv&|0+3Z#lpW z%y?xv;)Ek`>;VxWUR=GT#>ODp0V+SwB3y5aNPpMr0TnD zx|5bK{4C{fmjwnkQjXeAm)pD}-O)OLhkTw&*)3s!L9`sYuw8QOjY~mvI;!;o3TTV* zMd2dk)I5&VU%`NecDXtaqmt;*VmM)dF8QG~kuGM1_T67!?&DfRGV4x&Q?Fer!=olS z6CB|*LyGDisX8^oEY8o?49n-_UO_~O3QhMKMm!1%&A*cgA4p&t^%ggNzno@=`X!qW z7>yFkyxBA!P_Fag8G(?zZk@M0BP)l07k==AAKH7ta_%l`Z%kZCN+9Ey<-$}K$Lf?@ z;9(=%?GTNZ1e6aM$)|k+oWhTYC2D_Psf7O6}6k-h&+U2kQ- zU)6>hgSJy?)5wp18V5-+idiSkb{VinOpg5PHusiYvxNnTmV;W1so6Vlrgfw5DKDN{ zs@sxI2}SO8T%wt>nQ>XZ+Ur%RdPEAyCh3B@588#I#EyM=_PbVr>-gZbXu!nQ0w8v& zonpg&qy__|U}11_bPmxI=%~y;tV(wr4}h+`u_e8bPOGR;hx*Z_eCFBlt)q(@3 zg(PCd0Fj%wuoSEC7G)hCHmvHuR1fUFeMKzc*B(Cr*Uz+m5CG;=%_Hxx1$N9uv!AJW zs*?DOe7&rC14wFFF9EyJlG`%kQr^X8O&v+}tY`lrCG?-)zShDsy#ITfZ2edC3IAX| z^gAppca=r69BM5g234{9q?$JV;GqBn01xZr9t5pa0t{89%wk*XJZ&{t{gW;`smzYF z1O_@h5p&$zW@lxqJ9}Pu@+io*?nC>|&iD>gwK-0!3zOAwQR5+`t;3+SXd;6TOf{x3 zv9~tS5T;6>Aw**>fT^84C&wyVw=HTIn%7I+O_*ooEyc0>oz-c*E_*O7jC#V9jB02V zZhp@k?GtbJi}3bydkc`ngIf1!3?!xC*6U@WECSGEwo4>B(H0~QKGEv41bMy@2}_lU zEs*Sajqvoy7Y9}cp-S~h)C%zJ!DU%W)C4ylLgum`14es~YM@GOF=QHtPOI`X))CUm zBtwP;N1>b<8$P3wIuOtk@(=T@36PkQ@52?*qy4a>ldEJf>O9=SeZQogmVscn6fF0< zO=U(FnFX`O(|Igts$nQWj^sRn&tDPs8PL zR{KGnhT#)t3yOHm(8aQqjdnV()JTZw9^_KOp&f)-FrDlxc@_s;a_rUpBXJhZ4gkbI zq`JWAI>p02eZ5On_e_0<$g^3l1rF*qaolFjcOw&OIfDJ?gN_xAMf)O&G+O}BcAx8T zX2`72BA1nO(bX|~buQtjUrq{D;Do0|mNE8OrPMG_hY&M|5>*_7J{Z;6AC7R$cN=cj zi=6DuZboWJYA$2xz}ogm5ir}R#`&Vi7m&1+)9CQkWFs)hR&oTH>WyM7lE>UIYOWCt z@3vqe9ft(Mlh)NT97#7y70+D!4--8Q8kO;6#)_IRdb6hlh1SwlV4vm)sye|94~!ew zK7@!0^ztdPBP*&8w~<>?mYtm$k^r|CGy|D3Pw7RG^!X;BC$n`5Dml+@p5n`k@*dKi zx|47hu&xe@40i-r6lt9!6(t#|9RL1v5Y26({_)#q;q|8qBXO}uI#H4&o9>plOEh>P zTpA1K;oGxSy;l=MniW1uZ(t-f3xsi9GPD9+aUOV-)(kPS)S1%2(u3AVXf|**lPUF! zosExOevriX)!mrT4wps9mIg2o3XTDOYh>H14oqW0+Oxelq5x}Xu*&63q0!njOjVdl zt3@TLR_DCqt;id)VBKffnZ(9vRKc10DsQDXMsEuxRrgkCfSYl3eMC0ryj~#>;RYLW zw8PzqYqK9S7~t68C14Of@+{Rx)dlb8b^M?asHT3W{OecYm7sG>DdJ!#X^`941Gu@X3!WY0-#v5CsS;6Q7M%!$r1uhWNv}HKQXjn9fM&PVSh!CO*bn%g>h2S zfY6WX1iTWQj?t{FrpHJ#XfocksIiTCgNG_e|M}f-I;XJyt(;2oKMUUEX~}~{B?&+y zAFi?0-DfQho4g}U8uIUy>Wtx>?v)Mo8Gmh|>Vca7@DMND-wa6KK03x;{(v>@E^BCm zNv#E|UHSSjJ&^S9uu6nCiy{HR$sUR@93xC%DZv7`#qEk#z`*CT#6gz+icVrRGQu_> zkXUiOVPhl?1&`hvGTxSEOc0?10QtyKUwVcw7`R{%7-QUP>=zqic zi~Ohm`1VOwB-xy+pNEP7T}n;_1wSt^#%WC_1}Fd%o}S&*)bU9te!Jg5qa!j`uv5bV z#X8oEdBu}eM%sC%d#<{-IPWuqLGq5psMXUQ`AZN60@~S*qEu|>rY4Hd8`2+qP`qGU z-_dUvZX8sjXsbL=B#v;BwQoo%K)`oQE)=%m{9Jk-VoWG$o7!;_K(Ut{hGeIB39FVS zX>wOr+Vr?btEM3b@1`Z*IkQbSBS=V!SVO-ZMmO@HA!)XF&?E#}VuW=$aaf$>A7K;A zA?4N-dxqkc)Y{tvE*~JL)y`Xqr8ba+s43l^ubo8PCng1`Qko;))C5WwTZu^ppH873 zwf0z0<8(aeDym%NS_`bU1VR6U%y770az!4ctsK^>or+jIr`1fl)_Gf)*U9tPY+|e3 z%(SN5jlkI74laep3@XNfA9%>8j9USHm^7ReIwT>~VJl0Ywl0od1ePvmb2}WxSz(uP za+{(?BJF7^SES0p*HmZlwm=R^poiy`0Ho4=OqFAARO`Yrlx5tJ68S)KroNgQoVgkp zB%ZNl`aJq;qfcm|O3pSiZcu9CUD9q_g+)}yoz>qd5}oRzGimoWjFK@S=N>z4o7E=? zeRv?nrS=XGGH4Vzk$X0b*b&dBRhI8!k?fiGkO}u}yu4V!KOtG$S z7^^^{(3?DPl0s-bkht0N%k}_J?Qof8KGn(rxmogsV|<9!1)%Ju#zHTsG%EfT(l_G3)nlpM1ciXff`?Ih1H#<+Ox*bmrtbqFp^Dms1OFwFxE^ z)R<}}1-by_w=CfR0y)7Ri>k9g2dO_X(fNR}>27Fh$-WD~C)C#lh$TX$p0=iOE`T0$ zs8!S8IU7ze6ZKH18H*Bl9~@Yr)Zb~j8@1)mzHyQBOROqqA*_$I8dSkeYVmauXCwYh z>OG4az8^rHMb6&Q05(XXZ}sI+XqUG+K)9God7~tm_2oXM=ggPK?|vO#zr3tl4MB4l zgV4Gj){@pfIINX)5M?*eNgBYY0C1=z0?@3(7}nMn<2;NUVYg?=+otR$39vPtzD#tD zTx96tNxi*zV-D-ij!`H)59>L7grd^MJw(p>$0k*Ruj*%j z9Fo_4nuJlQ(viUCa*K6qhd1n1^4-;e&TpK^ET*X#yTh?>TJVk!9Z2L4_9x(d93nD3 z9eqHi^hUKZLG1p}aJ$I{z8NAcTd<*Mv7-Sw!Ww4_vTD*5uVvuL9E(=9qHl8TijY~R zk>hrsrn1kf-0$jCf=|9%C#WFhPpWckPkfLw%NQRRNkaA=4OdBr4(-0pJ&M%K^K$E= zx7eRH_z+){htyV1G<*Ym@RR6ehol{ z15F=+%}gHehMF<}=@>r)lD8_T)xsS*hO>7^|Ml(n2f^IGIDfX-SKey4)-I%DK5^aR zrSvqk%)cYgU8*`%5KoT#(Pj^I?Nxx=Lxw6x^1uP!wZ2RQ<-p-d8FHnY&9CULB~thR zfFDp^(AQVeRg*-PAOJg&3NkH86YT_GEosT{ewWj;ww~aB2>&s^D_JzO_nO&A&Cs3un;WAF=zrzLi&1>f5Lv-f00QGaXUZf;nfFTgJiIUBC z(OSEGEGA3bj!l@%BdTY|!!`vv^0M4o7dDV*X-2F)L{pOv{SU7~S3=F6Qhi*#S6;-M zyta`_#Q;e$3((4r3-}M{OWLd!xP4i6`yP_nJZ$Wq%{4kA?xrvU24N|W)HpkAW-9W= zkO!-3a9(1Y7Q`)hBe|7b4<6rIcqUk{TOP$#;tQP%p>GSczi-49vk)^ngNi^ThBqHF zISdV0&qw5aaSSpbL=^F_Z=OXy$!3tK0jWwqM;kq!K?(zTk)We;E46__oW%w#QSFOt za&>_$HK6JPK-#<$E&kT ze-!@Hf67|}@Me=5U=Ms7Dnfsm{pT433X1zh;ocvhVSBZmg{3W`IBUv-M2d;6RFyLe zr>tCnj(N4qaxttN)Kyo@?Bs7o@S(W>dz|nEyK_?15TVtgUx|LeXkW^%CO@lgZZpc^%pNQl8IB4$rYx|bSOc5I#f_|JkVQ|#zAj@tz;%MOA#8PrXwIa5erA(38E z&osJqPf&15z|67+vXR=L?mZL-Dz??s)`kdO=m%8@ z1FV;_15m>uLQ99}Z1NxnrAO7{1~->uE1M9==Trnab9w>%fyu(xAN1=n;{V3Z9q6B& z96=$LWzCVcjwq*iKCgKv}l*(|QzKgX9l6wHuzo^pKq(tCHk~+8qhd zy`g~#{EonS7@}!(NdkNi%m^B2*+)rUlw7R#8=$dAsE)FQAsbS~{ZHvIfDiH6pTg_U z!s}<3=Vz03KxaY%JKR@%Xt+%A5b0W!3C_K2Xn!P4(3YoMkG#_D5Rx3&y+}x>zy!l1 zb32hXEKlI*=uxR4Dyiogy_Zbvw!8=WLu-OTFWWLG8f_6ki~XV`SzkD&$jGL*+~_08 zCVRMa%zSo9krfX^vjNm*2yO2}*{TyXm-dPtBG>i}J5&!E2eR~vr@(1NO$VJpJD03n z5-Uj+x;#>MaMkGI`fgJK^fDgcVMWO~ZS4RdjK;yYhc()?LcZQ&6qwxhXkR?d+EH=RlC$n$Z-yC> za%%Hq-)6TR1h6C8KUB4#&e2xGU_GW@5l0-PB$aBJqibs(N&; zVfVQ($0`R)ZIEW&)RVHsDR+pe?lqP5;G7NY?sT+N1`yYPb^rq$DtLCZ0Tyq=;oOtT z4^v`80A!J7--z6 zy4Iytz00c;G9Z}iwU?x!i}uwe7I^2vNsy3HYHIkO<(pGUO^X0tK%u`XFZi*ShuLx5 zVa6dDTDwVA#bangVbJm94KL`mz8H=Hv(dQtQ1>2nw2M{Wc;9%FcQ;LrizD3L)}(3L znzZ}-e|Y^=4$ANT7FiJ(zL7-MH6cXOq;=Iqw!sL%{S8uhe!eW=UgqE?(URip^3ZlQ z^l?A~MjvBG`>fR1!G?P~peOi3vbqkg>pW7cR96xU97WfFun72%tU(xH=O->pMgA{a zuq0k|)h2TO42j_-v55rV#!J*q`FD~S z|9KYse#lEPu7jKwvKrU=IdHtU9@5kxUSXJ|L~}ODr5seZi&53jZFNlzmY$RcNV;0< ztc`%LuqGNER|%pCRF0a;>cbmQc7aNhEl_8&6YO!-8H8%j4w`m`6bS4nG({h-&cB+X zREq#KLkO~zhz8l`aX$KqP!%2y*&Wpg8xmxN=CME~qzt}Zv-d&$HKrCz8^jIVGC*pM z9hRVO3H&V%H@Gm9Bi1qXy+TB^q6l?b(EeWl-=jbmn9iG$7`f>stYa|FomY(=v%iQ2 zvQ;AQKY_;f7Z-K=dHbbWejL;PxTqrAfaMFVXw@3NV7Qw~1T2gA3p8T-)ioIE0l0F^ zJ0rbIDoqC+n2kDIwS^2d0;34yQZGvm{;sDDr5pGfYX6I>&OLfNBtF6*8?D63n%+^3 z%<#t1Ae0>cp)xks5?w-t`)fRci;CqlatK^wW;kx&l;M?ufwxIfSL&X!9y;oh4&E<6nND zhW9Pi@T{8t^!0b)^>=nJa7c%ifnvT^M+}WH^GH@RwE%iqzq%aP%x>oSvxL$9$2ihb zm#t$kLZp*8KHw;L;qixYXycv;5;M2z;H%a{BPmy74g8`Ah2*ku7a@t{$3OmY__G}K z{>Sj6(@FmNW%!@+mw)s6hw$SZ7qh$?MF@Hq5*(}Q0POW@n#dt;IDBSXS?xG6IXS#) z{&>0Ct2`@tTdAjPTC)Gt#RqM46}Um`O(j4)zN2E21Nhq6%pKpxM;!n1`g5=N^1Yy6 zjBqxp!@|H)3pnw`Gmo3YW+g^lZr<*YADR`#?8_&6oULZBr2L`vSvc178v4nsKDa<@ z9Ipp)sazTW_U^UQlT^zNG#=AuN6>0)Ft2W4j>ic`Zd*E6iI`Q`$RPPD{>J7gd_f^B ze^K8l_^MgHQ^Ovt?6#bG?|Udq6Ww_@;1<$eUjB|Vdeo8$QLh0LZb(*I*dL3hj8Y~r zco^@l0yqHRoC!d|3(X!P*EB=~pmh(uDU+(dX|ki`diX$HwKZ>fdG(<#U4@~Cyy}@a zyDB_^!8fxJj^0el0A14MM-ahsKjbdS@XS1CL};Gdnj4^2g+fQTOH!d9wOUUXSm}-E z&tlQP0IN)asJZ21t37&b{N)E z$Waa!Bak%_ zf6ZUR_kV-G=D+>PcYi}?1)zfnyMx+zds&9$?Oxb2;ZVWuyv`${k*jvPnA05D^*~6E ze(+>jz)`Zmp0X11#yLSN(NSc_PaERo46Q2R$4mtELyiPL%w5s1P6x!>AH#V@e*DRA zcq+dC=)1oPufNKl|MvA$8+bi{p|W?27HYE6TgA+VLmg(tJydLzX}7^L2b_P+xI!?d zfCC~y5a??#rd#UKANYw$3ToAVh&gbRHSWflg>7g7pB-N={Bj1KN-aYWD|O7`7?D2p{gwtnu*n zsT^{{41A=IFwj%9V@Oi_iZ4bA2vnjRio;T#By{vJCDzJutRN#}tDq;%9HNry9*jTl zP&Hev(xulciBn}(smv8>c>BV1Pkwt@o~Ztz{Dv9enWTIw=L01*NJjBa_?U#P_Ri2R#{jRVgCBys1r(8L)Nl8#bYT*oVY18$>`X8e@M#{yRJEQoX;+ zZLg8%3--UQMg=F?XPEZLWg_2KHS*CT02?Ejw}g{qmv?-~FUZSrSE`Oi9f}ezlSBGd znLvuB22KHNfa!@SB~za&ng_=1SvJAbW^Ih?$V9N3y{ar%V(#-4(eA*!Cl@%csvSCg zSN}Pj&Cc`s`{~UrJwNoDiKGmknoS@ie zg40sTmEd%-%)5>qOe<};S$Dwb&9LF#To|R$_ESe_TmzCctFIz%Jto8}wOb_4_Bm{* zB<$oB&NV9E9+u%f5>2;5q_lmN;f*Vn2k^wTHz6DvUr5nTp(hx)+F=RFYK-3C&!I3n z!=%>gqijI9Srt%fDVK?&bDS-_r()?iEvdH&$LL`0$c(j@TokPg3@(=&i-Ct6pa#$E z;HP6N1GL;jGpx#x_Yx{t4k3=27{-=5$nMH2IY=c^<~)W2Z3&ElD`a{?fI;j{iW$k$ zjmv_uQhc&yvm1h8-XR+|E?#s4&rBL%QeRf*XWRgbbp7Dof=!l#qf4_K*f1iI*9J|= z!aFvY9nsPaE^1o0MTBJLknts)sTm>OU|5#-0A%npWNn)!U?oZlG@-Slo&fYNL9WWh z0Pr9?1-)jwJn5f94KXSS)h)fw+Yx+;pqguTm_)$v939-FyC{F-37%Qa$y638v6cXzk}6SPw-dk!4J7I8ZWg(u@Q zUO6g$#CGCuA=5D^Gh$r52!_<5r>)xFmRvyY$U$tNkpsb5smuu>(l`VL5iUYS`6ScQ%n!|I99V4I}%`WGV&>_G<81@Hkk6g&R(lM|Y`Or~X$u zT<#8~;SGU_=x_PaH3yZl^a~Ea8jj#$?zx#;Se&Rq9*+j8iopCt7A4YfE--l>5V*4N zDJglcNJu;lpldof9jG9)izZ1%mSt#NuzS?O9#eNoehL`I4ktM#wP_*~H4}D@sa@e+ z#9w^(w{Ks)|Am~2zkC1b>&I{3y#E+9raWZY;Vw5^p7Fe_2(2?av+Z=W>t1LGp=i5v z>NKlUGwA6`)2tac;J4~RHam60y3Hl)yL8vQGT&z6xc~4_?m-Z94e4<|57;Gv%=@r$ znKNiX(N+M%JYcO0m5;DJh(R68u|`>k+W8!Z%LZixMN!Z6a-%!I)YWo?H!2v|!Qcnv zIhQs7s_c@GhC|GHIpIM|MCgVeTOTs0Xn&O@qU?tixpf06DAcLMTj2-gH<#1uUwr6I zbo!0U%i%Y%*9({89pcGYt#emV6z$=u}=bq#+V&r1HI9yJK6CUOdEQJ2RMD*{%>!)^e=u!*_>u}4!l9RwTS#X!f`x_RDhx@t;E$pjY5eeUO)@IibpHoyRpf(q5fheH+6o zyRV<+Ny@)>6Sh}k z&~^`EoRqR5_ffuKZ~Gka1lcjD)>0}PU|7y@XI6uTt|B28PV9OY@P(7n(NGBa*c&!f zmO5%xQ^1hNvDE0VLGsDkwrI?_@i3qZvZ1;lJ4Gcu;qxAo6@ z3G^5=#Ot=*@5yWV50Hn(NJa9BH$xBeZqAVB)*F;{EUW zM!rX;`^I2VDogPCuNDhCw=zciwQ2HShQItvZWT4+o%vOu#+eFDqQigSEV97t zBkq6#DAemv^OtR28PyF{;W!^dPL#(d_K%e{r z-^!=4a5l&@Dz$3Kad!fO+}s)*3z^xWmNRdqr-s1F2jSf#Dy2i-YtfmXY1p*SkRpI;i+!CWyFHNE{npD5NWB&%veIl6qlht& zK8h-U7dpfqR>p^G9ZL0HhoGb64OVQ0y4ek$9A5u$dH>n#muO3T{`R>w*2STMQ4A|J$-CT9!*O((a*ktCQ4fD^;Izl8ZtMJl7B$BCJQrw#1eSurQl}zSsb9 zWj-c3o6ste?@7$V5ZxeGrDkm)0|N?u;+LBoN0R4_yMDrlgWUywFsNy?R&EZ|2c{fm z8B-mce-+}D1Uk1`ImwQ@=P0U5_)L~44iHu=SP?IbG*EHhgTy6Uk&JIyAt?8_vO2A!UGkbc zJ3HcpveM{9iF-%?7sGVh>~cYhBd!7V@rl$Tl@4Qwl5G`?0Jh0fS}*Mik&$X#$$114 zZ>PDArV)1bN**5kv4&!o-MLF$Ss6JZ4k|zGqJn4!*iKcy^Y8v9uY}Q8%bI6O5MEjE zKRbOfh<6xzIir;do+JZe#s1A$m|TbZsU(g9AmC~)D@=43B;NM(Wm*K65@S$QPc$hATMU4HiZo68G_ERPyAS^(JLgQX6hUP-w@6eoV*iZ0{zlIbC# zqXPld#wIia#*=_Fyw%qiXCKSIZESu&wjgZf2m;&flOEJf*(VY#vsM-WP{f*akehP8 zxd{moTmj@8UuX}-0QZcQ$GrK%M_Eh5Ur1cv$V9hA422fIdT>b<;vFr_pg!qBi30 zi(w-rJPo-G262QOW5L9hCkYKA*D%r{Tz*tP4{u+C#T{a_bo#!yoYN#rhxr+*adK86 zzV`$nnj}fDLNP@W6Mi-DMYMhGsx=NlQIqnCwWH``c zhId?cG}qWFMEwC)@nwdW2>zqsP9M6#s+)dT65tMj>Kd{DOMDS*#gO>q#7S0lV= z5SS02W0rI}51ycJR>6latS*iO=G~;3q1Pof*us(*bF5s!oFocmemM_K6 zUOx%h>1JE0{J2bDLpiJsjz|o{8NOscaAYe18Y}T~ThV10tcJdga`sc%t7Io!nIJTl zdzSxK#NR+j?*jFW{?zZNx7_*9UqnJ=fK|RZRCkC)I?7{6%t+wA{Ka2{zxa#1w!We3 z;2q3!U#Z&!52e*ukdOJ)m;l?{-sqs@Wj$*amAUn;J~^mNg=DR{ALW!&9~u4fi2_Lq zAQ^Eek^&tE1%yW1XxVqwSM0sk#MzPkm^E6v??FX+m5}DUd?0*bLh^sqN!~4!kqXaRJE9sogq00-W0-$Nj5vY>jP;*#kOCkV4b`OV`%?gh3J8my}`3x!wOoF~v!1hj61izK@?>eXZc!V!y zI`)^cJOul3D&oi+7tA<-^L-*K4vKA%HJhnuyB}Ta2lNO4uaxAnbP!~gXy(wat`MZO zn`!T-f$oBBDA_jN;YTwuRZwt9b^cD*s3hzhcHpM5&<+jOC8|%E9|!cHk}Q(oNq4Om zNBJ?Z1heft5jk_1Mj}eoK#g4`$VR)6&Mh5|xf+TlPI0dB03p=!B#lJ6fm&-b^t1pt zKBZ^2006_UX-ygF%4L7pNhwZgT=0+#Ge&l7!QnqO-f>+M z>Iu<~gM9iTr(%<9i2^VN3;3#RZjDV$vO9v-gB$VHl`7_&bWPMr=V7^}sRjhFE=v&> z46wrFrp#{MS9j{>B(=WmxJ&AlhFoV;>#|FhPAIi$))oi%cO@}m z>uo-)Vp3p1+GB_6H5t%@cKf@q7K*dnklbz-_;G=1xw9+V$=A>xa8LOTg=E&4HO-rW)#X6Q zu9lK$eG%UNbb>j{@nbUE8KgLh0(^L#yh|Mbo=LWL2z{iXqxW=U2Z#6^!Q4I3swXYt7uRH z=!mGb&Fy4ie!$5%MqJ4G28gr8dO?iDF56HwGPUeMoy)c2+qZ!qa%%9#fG%N#y(L+= z64SSn@q^lTpf^(n2tGNB6#aJh_<>n-s!Wt~8jl~6Y(;zy2L94q^#z4n$31|+=D7Knn#EQ2Xy`U_>Aw8YoESCeH7eA#n>trvl zb$N;RWC$>$Gk<)9*^gfTXfyWzfv05WmfMix3#E1u7NF==*@mawQmgQlkJ(UDK>Qgf z17n$J18!t-s+xJoiMg9Wps6y3G5JllXO}DK&;?Kbil2Xxy`!Kcc z`MnRn2S92P!8Gy(`mj7A3Uz~*BWO*FqxB!kNu#>Ny2Hgae^lWN@2tprRNzk{djgRK z6Ki=l_Lrs=^m6STyDsIT6En${c&meRn{_&7B%FLgHV=(l-Ok)Ii*2vo!3f(-X-!Nt z2h5o6a$hem)@s}ovd1lv9n-+-Qp2GrG>qAAD1+p0@?B;1Nd~Lt-+9H!X~jGc61JU(Ea*cF?14g7 zX3cn1*>~*v$?rnO(^G;PrPiZPv<{rd8^AR-Le0WR*Ukm@5+qgq5Q1cqqi<9)xpZ|b z*v1n`2k$Sg#uDHMIfrZdxGMZ&Wx}Rj8H!5vw-{ZTtx;qz#twrd=FkoG$kM6FCOgY7 zvfAPiN4}+4^ z0!|pN!g_kc+eOk2%n7>W^DpeRbvFqGIT%lv=1-8K?jtJcj&4$*kvpmmg0PUO1w+#6 zUl_|^t3R3Bt`f8cN{qtyzPiAzcfoiVh2c&~q+f?W&p%mGppVoIRW-PB0D`0IK{LlR`zwOxtvs;L9nyH@8_@$t#`BD3|e^YI={HhU$yrfet$o*EcF9 zpJHl}U<9bw{>Sk8OP-7Wf@N$;iiFcR5nG~qY0s%Yl`Rn4O^`?O_AZ4S-oBFkkV3LC zL6?r>%BSiOdLt;Vb@lqC9FOt;=p5+*45KRJpn=0Syg?_~Wv&kE?kd%A?}rmKgAQXq zY_Cf-D=)D^_4iblhUEjlygr(*&`jt&evT{bSAYKYhxebp{mXZM9o~QZ`tj?Zlx_Vb z{`>mN_dk36_Vru+{PXvpzW)C0zv-WT0WS92=lJ~HU+eFDayrcKBj*xQdN{cZu#dPN zqMCzyTtd`|v13;A+#EZUhM}#N?P#$cF`0vKe)Lm9y&I8?qL84P%#}-w{qT$02J=IXg@a{MPv3Dnm95 zyR528ZG!D=xPe+~dsQK|wzO1JK-+aW%1eU2TYnv_cfu&RZ!p+lQ|p0D&g!U#+X&@1 z(xkb=DvMqvplonOZ71%@Zh^>IENgFQ;N0L)c94gapg?AA+Fq4UB9#;YV%(T8ZnJB( zJS!U($(Ott2%z00S6XliaKHc(Fm%DGPg*Y^il{8?3wHsIKM3YRg6?fG*Xz;fRTwZO_FbJk$4a}GgK@12u#!?zIURpS$pERI zq@=C)+QVe-=-D(ll9B|pm)BQvq7|q9Q&qEtedw%@ySIActT3&7E&mRg=+CGt2#}kk zs3+s7R_UenWdohpOK9_6w(3M>VEa1z_n-!FHS8N;OV)Jhnqm#_$W8{065{8qd=oql z$kgPcl$UA1oEj;rnu5LmrJRr-h4-I-_gAl<8;_cj8Zh0P#z|_;hT8stYH(0|9cc9c z#$_d4q2K+8)X7Gh}UYGHzK9aJS4z0F_N{{gs zuMjSSb*v64INVjN(QuhM=1*)}wWW@L309j) zH%U6G#3yQO%AQn#o}MS&ZR?M&)D5UlTa5{d+3v9oBLvePgTsM5Fp1?Il@(=`=Mj31 zJ$noua7Md2JeuX$fR)b@tJ8YR$oW!jJ&diQ<2@$evW%h62eJO8)~4<~NuGLjorDfh z+BBO)?-a_znY>VO!gM)Jkt);q$Ohlku9!v+h?U6KLU!b+x?ndCYvy2LA%!dp^}2S8 zI^d%)T<;bVSg`-|#|4r8h)0+x?w6Rec-m&{G3=wY4M)jAp4m~5KvFpBsggYlVGp`S zaQlL~T`yLmKOL>&7MMSJdP>d?HYtK8PFFk=P>Hhi6`-C>>QHK_Kca*8UR?J-q5*d` z*n}0dpZ$02JH8BWKZi}_Ey5@iV&naG8fi!(O2F;ZL)Y3xZzm+FuozWVZ{jiCY%}t; z!%+>s>LoA^az1pt+a6IJH&&ABQf8}Mq;(IXnx-s7#PtWi&>O+OP+(@^qLV@5EaIUj z6pK9o4RC%2pR&6D^s*lS7%Af3*@_M|+ueA1IhkHGU&DRurt++$5=nxYKX65=EId;hQD2S511f6G?8%mzWK`W@6*5_>g!10C4@yjnFmSM^-^ux2xN)m0^y60_KNbkf zH&B7#ybcW_6N!i%XdqxdBbREx8(d{w84(+TKDKRuA~;TDEd2aP&BC55J4@V*zV0zt z0RE>zFFOC&5p_ioDr;DnLtPNk3%_SdX)hGDRz#aZ7OWGfUNhhAKQu zL1e0aTlfW*RZ5Y9*EF}g6cGId(mgg{im32sR<=WEDJ57xOB|gD$pY%7JUm9xmCK*p ziK0+gXugL=wHB@Ru+8=B4?E)z9S)5)L*0V)#a2CgI9M%gb<9ftgnAvS{Fj+yb#g$M5kGfq`Vx71KU*y$kt}hglCn^&Imw~ zE%f)$QGyhSAi9||E25X9FpwuT2C4&r99IaX&QCld$;c6Qc&gRpUB}tAUoq#1K9QES zQ*r-v3AH`ghOGDP)5)SlV6;t%!rexlbG?Jxr?W64d!z7KUCmojS@%=qg>N>c{o3^j zBoupLvI_h-W*j5r{TKP(x?A&OcCW%mkN9@)Q@#qEUTTiv?Uwyf%#I{Of6b$3RqR;f z3!8x>n85pPEwB}eS$>YFkRpx~pyYdbO2Z{nIyEh|0QwSxt4jgC1(4Am3mu|ap_raq=DS8Z1g7{?}S(R;=^ zM95AiH>A50`=3_t1G7!yX|SmQKzA*3P6UM)I2Y$fCP!l4OM5qEa(!|N!_iCZ)eSiHD}25#>hT#JdRa@C%uV65vQLn6FnP-(YU3}GZ4 zHtcqktzEmO-k*g(`?C^7|MBeyoXz}P76Y8`erb&;c)sif7?@#~Kyta%{jTbrwkW(6 z1g{PBG1l4TD8bi;E>=(CE>85RbS*)#Qg?QHC#%V(`EavvW>PErgyM~Ga+!C{VfP&e zBCQTIV!i#t=Vaq&GWKi!gk9;5sr*$^4D2#57iHw3BOzJ;1XDTF2EiCDzmhU?q{AN! z1;~R0>Hf%5DjRenZpjLD$U!rLV|Gh3Px-ia2p(3iw(Y9kVHSQKK$W;q(*c5Mkl$Qb z6jN_iXtzeSd=^4Q(GtH5P^xA{dV{d$cqzp8-ixKSc3K3cbMh$wu68;pDH3lkVBRnLkoCrv|Q6~$|Ad19+_3$d!jHukc0AOHTEFt-;hw>1M0qQQQvhNX`A6+ZCYaV;C&b zR|XfkP-JOq#t+!J*eMpsvNN>KEOm=}n`VVX*lm$hh|9F=*?LmdQ8_4EAjx4n7ijpa zCg|*)RhS*ZbD8QS6|!fwjBi~~N*w69q-E z3wX@3o38lz-fqL@a!^?p8C}vnE}yk#l#$xY^HJ+diAVao70*vZr9qaTM5&vV02?yz zEX@`rz5&IT3L+T&I+j*QIOJ162f2dGXSllU3u<-Z^ki3J0A-n-tZj*-a*IJJf`K-) zCT7SJ9QJv2LIIqXy&uAdBda6NLvGEw{DFLv`+_|GkdHFWcfX4?jRB~&?Hr)NkB9CH zB$8(k#7U?6*hACu$^&>**{Xq)4+y6#pQV)~(z>v?(N)#&3M4BzK$@UZLL{F`QG|`zM{D*QLSV5~pm-J2?84r{)F+E~*#=c(cut@pIcS+DkQrg6JRD|M7oaX_ z=+(BNkfs9|>5!m6!;3z87DWHx(G-i+2CN&n)W#*>Fu0u!l7l5mjc@fELU}3?q{fte z@iSsH)eCtkSr^RDP~CDOi=OB1TAC}`fs&vPKs6u8QzF_wOF#4ycWmFOnTy2|bc;?< z-Kzv+iQ*Jq(tO8#CA1U-KdW-t8r}J}9CG2^(GEmmt~UUWR035^5aCF{My4eBd+bT& z#(0S23HW|M#6=z}@~o?uJ~!z4^6;Z#Tv`Dpq$YoZrUQcrRfMZv?Z9!^+kOVs-!ra5 zP8HVS$t4`Jf2>`KA{Fxk>0e*LsFmEJ-+YGGhaw7fM}}*mN_KDVV-8Y2KCICLFJo&{ zVT7(G$C4Xt0JUWwH;dc@G()WveU|b5s{^sr&1xjD>dw3sDDwKE?jFCknX_ZMt?g+9GsN)e3hmrME zmldM+q*CGjkMxZ!qM-<~K;GisOHE!4fcii_2_-S*PSi)+G~xaqDN5iIYL8`dB?pUT zFqp)F^t`McW-%VvR&bsn1LYV5*>s zgYLG`C4yNKO%=&rD5g!wmAGtp znyy8X9hkexVNixGIMfRg=%=d-mNo_?SWUAP^P(r%GY^xsk5J*0JNzi|x93^DE3$7> zFjrAgixP3S@US_{aC|)0Qvy4n7_mEWo6hsEj)!j5S3?hCqTQZK>J^}EynB=y*ebCJ zR#kGIbZc@%UVA?If5+d$*T10M$~|)BA)T|n^ek^iPBs=}Q%Rx5WM_fTAz=Hgo|VAj zT;V;u(%@sG1X}w*VvO2PTaZt&8(-XV)`)gQdZB1}u0c~{@*Vg6- z0y;^ELDmM+v8=$GHziDD5utLlmHXHz=cP<~5OyNzU0|9@%!96~qmO&A#DTpxR5+mW zQv#IUEe2N2T*KV4<0+BQKAnm?(?j>gAy@DNgrZUD`2TtM8;jSLls%Bu8;Ix#*1rS? zrK->&S!mUPq4S`|0nf`qM(>omRJtOnY$PA|s7o!=Da0yYfYPm)DrA>8BM%{_TL8h( zT=my5BxkK9wAR#w95souPKq*~0e!)QH7}`~rUgAJ z)~>g5M($vEa(zhHSGjK7ior*BbXqq z#wii5j@NMORm8I&+3ZI{h0IytN9aJMTo_w&0FqJovg*}bOD+U`9!M~j*(TJU>^1;? z^>Kj+qFqQ2x*YmXhAW%HHd-`Eute#Atg0=vs?a82xulWTnMPh9<%1lfKup!-GrXd7 z2Q1KV9|;a_{Z6+w%t%yll_D2EfBP4TT$r<9_f&J9M=Dya$D|T({k?QF(KKKKR|^e` zE|a=M=QlmsG9Tb8MdNO4RF{5MH5Cd5x0uDvSLkW&=dL-Vt2ynu#z%ykNIaU3ls81C z#$^d%APv(4ww@X^)eVt_0{QJ{S|+Ou{3J2x1NC^3ZnJ-lPw-SC?qGc884x43jFuin zRo0>34%P~4SKvUfLETc;{c?M*?b8asTd1oGV0AXV>9|tb>qmKr^q9w7Y>#{`>r}q6 zWW|;ztNl260u>e8t!w)Swr~G{e!R2!9zlgX@}~#%%dP{wU2u(>v(_dCQ6qevnX;&h zI9Rk^uR>~}A^jCmZ%rx~1zU4po|5|mo9}Z5FnI{eK~+6)HvtgdK{pBxcbESZzIm}V z`LCI1^Uw0@@7}-b?e(U477Vw_nxX?(A`H5Cs`laLo#`^? z%DO5j#^(Lf%YUJl$o_0(PsoBGj%d}z#zqsma_(4h?cEVON#<_hONp_#@1|7^{>_63 zA*f-#eV-0fnD=em7uhwFTBB!ClC9IBjYV+_ES7iV`MHAIUJi_Hqv=VASXvN8O} zt+Gmqp&d#2SVmG(g%1fs8vElv`PzTs>t9fzRJMe!3a>k5Lk$Q$N0xNv0AIam=0|a* z#@x_8=nbk*c0}T!jnT7>1RthVr7F|PK?cmvdDz};DqSb#H8;hZ7lpgUXa{X<0Cxvf$ai3gge3%nz0d49r#r1kK`7PTPt z<77XtD4}HsTRH2?D^LQ;ciw;f$G?Vu?+;jX ze8Cv-2Rj~t|E~Khi)w{~QbHB52AAD|ID22G^LQP}0yt833yK7HR?rL(vCL|{u+kZ+ zpQl=yXz6c}RwC_8hmR&L=o!&$FAYc;?}Qkh_#*x{a`NfB46)IV{|frb&X1Yu;^d|( z-C^IH_)|tha_@J0l271#B~5w&La|ugaqlIDAG|Hesr0SFdVFs@Egy%1oO`06WNR=p`y zV1kd9&SY!r?p;B||L0vkqlTH1(t;(a$U57=a5$qfqtyOQn@+m;i<}gVRu|*UM+Ux+ zg4=JG8+tbe9IBf@dVo4TZh6AgnUhTwt0y!j-O5&7hgGSbdqINL%&iTs+u?3iJU#yU zuft#GvQ7Tk0mYA$t)^N&l|#yenN~gtu$cKG2QSQPNkdla-knNxGvvj3Yx1X$3QB<- z=20|zz$wknk6GtUGuQD+cFOyAFOpW+pc?{&IHgQMlvB~?NtbiaXu(`tTI2!kKSk{j z&^`f44ouHjXQ0gR3r&k0s=Wb5<>JuF%-2(6l)bXhly7+Tru0<}#4H-&jDg4m$klS6 zRoKwoLBCN%z73T8ov*(A{U2a#&bUV%@n%;a)yUBC1+|`A^piU7~ z@gVODg)5`Mbfn%au$s+l6`au8M9lfg&27(a(J9q0@TTuh-C{K1=7H&Z$`Z!c z8by9R9O`|DVkrE`WB)w=qzA}qV zIPQ=JiRIZ>6jMA--c;G@1Ac(^s6m*zanf>o25Z0GP%?rn?+(f@b9kAz3t(y617Nw? zA#bas;_k9Wc36#_I`SM{VNV2!f&BOWCVV6BL9P%syMAk-bk%=sX&L&ywn)g`Kv#Hr zO73|Nr=i}m*m0r#EybtU+63xS$XmPJ5F!d`(ke zUT~(Z%eps%yb2`T6E68@B;I3p!}>V;qJ+X++A+G8bqfz;E=%noW6Cd)%ceh1$gwc? z$O+Lzc9h|9H`krghEWBi{Up48a8idBIeF0)awC%~9$-#AI_@B8L*PanCUj{Nn{Z8p zBb^C-tY>`0fbRs&K4+V#6us1vNXtK~Ix!$9H|`#lM+r|#A9=_g0jCe%dQaFoKsP{g zG*_CM;Ch;0FM|Sy8o6Ot7g1X@N%EVur2_!kr^Rz*Ss8p)OeoZzLw?>(ww*)GpsZuv zC-cW0CAL{w0te-W zQ6iZ=t)O<29C|36t%$N2Zo`-$@VUKXHEXsIl zen$q#4LPW>BLwo9?|_}3$P!VceSQs01PJ9$8k`mFFlBE8Wq0M%RWkoPm$D}iS^RXR zZ{3mG+PdWGfw_t8MPeH3PU@T)@_JlepcPwOm0G~~465|;&nKmiUuTUE`3+5@--P#n zvMZm`D`>XS4Lgxrv@J!&x95QA)RTY4^E z`$E1&-+KFTAS0l$mf4ICeln%pxIk8DTRTkSE~)2N-@d4KX2v_(vt4F(?#3oOKJr*a z*RvnAhhq)f7s_g2$|q}>cj06mthV%ZEVvpybZx>X+s?@X4&cicctHIDe#DXnq7k+d zVci39^`FVMXacwDLyz#DMqQ6cfRr4>*{p3)0Z~+ z9?h>YQ#P_zw6nLhzR7iGulA~jzQl4*riQL|E#R2RLsokq)wj?Oh^lNJPwHmLtVh^E zLD8cM0r|bG-dmL4oJ1yPp}Bp^ZNt`*l8r6C0nFSTDrmz~`Af3|`q@}>TVw8P?HeZ8;rnHUoNw~17Nj#C= zRiQO2*EiLaHA`s5SbGdUr0d0D)OpYS6MO;)@%X81DtvFyX;roYctbyfHI|QViwkVi zc4J+J^YAo4U;6_HD?265o_)cH38GH(2xd3H^FI=(1uh@LVowcIrEN6 zsv(eLM*$$AUnAOQt1RFKqm^cdDd0T$sZFUj~QYDHa8)g%A`!Je|Z z3)TiUuuLC!z2CHE)ftSvOaK=ldf0_dh`x5-0Ums(7dgDmRm1j41<90 zYO_UB^3fFWo=MeNuD!=4n0^A}fO&w`f?JP`Vhn~)-+uf3s<*(y#snyoD}bOb=~=z% zpnr7k7&^qW3d^VpgGDLiOGVQMWPQx4ala-Yz7tbD6s;U*%i0P%QAbq@CwC(Bh9uuC z`O0%frzpNs3M>LE$3EJYJyUt5lj;&vu_$cYb*}6{Q6%(M7qM$6(7NN_&wYheliK=fu}2r-Pab zg_us6B+*hQB4?iVLMI%3ED0gjD>Tv-Cg%0D+{l&4Elf+RUJ>DPlT<0vI!uxp-LitG z?T6O=W1_0p%F$D#*H;ZHX?M{20-rQ*4-Tn^TtsOonFPuN3^@50y?hF-^3q@z3%oFC zcCpa#lhm~Up)feuh?PQu&3y1*HHoN?8WUVjj_$G~K}z~jc>lfyuAq>N{00bwq>6U3 zMcQb?F?nJJpTtT!EV)7UW`(Cz^^*;b<=)AzANms9$3r3d@+@0kF8or7mtSi*J}Q{O zdj>~%MtWm_M~oD-c9N6?G9|may9-*1rB46}0nFa*4M0x$;(HnoKmbKfn-6x_DCSR2 zN3cL8HO}xbRDXwliElR1fNyU?Gu^-%v5ZJY% z6lc0EP!j49k=_0gCH~~@AUY(VICparFvC7YGvfRO^_Gx>V)WZbn2Tg`OEFW!wsa}y6_U5!@53}> zmb}`kW8C5T@*1{kvD+7Ei+#tQvZfNK=QKUMAv9I8+o4oU_`Pg!w8@G^!ggXA%k!~Q zx`ljsVh9HL)83#^^?qQ2l*Wd+Pwkj!b?)&8PVN{zFa=erT~;5^BE4 zY})M@3w#Gdw0T2OV=b>*d-24}-9GTzx-0_@f&%;}9o-_4Pd)_eMHqsRBYI_H26L{) z-6b%dnWsmJgFMvYa$Oz$u&Y<6L?YEF!+9(>`XXB~+%KQL2w(m0ire|>+nL_-_n>&V zj_N~A_;!;Zl>%I4DAkDCE>%`y^_aucJi4T0I$+JmVVp!IEtOslmPBa0aQOnIAGLDL z&lYxBP+B#0;9e5oP6aqtknouXsO;NkGYZ0Ip+1Jvu36_l9wuj6!4j8)d^aTkzX=tb!v17E zs7mn`uXf2!R6k-(08cuaW1S_1akGjFO7IS52H3V#8Av*rKD{Pdv8m3!|0OtINfCP4 zT8s6Y%gCb6)OTFt9J*)2dIKU8wkmHlFzg91Rsgznl3?cH z!Cewg)kirk%MU~aEvxEh2T0u&r8^8u$=@0RRRX%?sNEqc6St;Q7M{dTva6hwdPq~< z@$+fi@7{lCyLLP-x{oElYQkXG6-PYQwng4pK=B!BUK>|=J~nPhG$m=p3K#WFWkWnc zy!N{CBIc8l$!FVi;0jh*4;(eYrVGV&rIB2>N<}NI2atw6(Li4ufP<49pS6l( zPkzJ#L9iZeMdHeR`KUgzeBTAj7d4~JYx=p_a7B*X4~If#lI=h33ZKO7)wrwCp#qF| zjLAC>s3~bod^=#3v%-=6R+{V72~kEfASTf!bERJiX- zwrmtaB9+q0fCuFjRz7ycIAIAs=GTJW8R>&!)L`X9904r9Fu95ZBMyC%yi6)web*_L z3rNPj;DK7m`7U;65_@)8SX+-o$8IX8DME_tn06Gn_;GAzOc;4l|BdrlX4eDZYC4QSqwY*R$&xU4sXczxu22SD0P>+nXU}!=L0h1qkgg^RMCj z2J|dA9p%pllY0Z~=s=K>=vmt0x0z0pU2v>&Qb^LJc?NcwX3T!Bs9r69+0;(r0zihP z*pt=;tMiMsmj^k>9hJ7u9IRCX-iMdj=s3>%to4~;7p`G5;QtK{Dk@%Z$a;tO898pf zH@O;gfzC!?iX2Ry9@Xen%Su*ky&^?%&lTj`IAl*M&zXx${cbE5m7AFhX#DJ(gvs?e zNxJ3EL@TVBT;iqW#UPzI?g1uHOh+ou>g-H4y?qxg)mazM|QuVWfyyf2wJQC`g-m5qw-Qn0E z4o&gQ)y?-~U7u)`>u-|2L&0u6>1Wq&ceM|!BtE!(^O%O*w|3cL_zJ0zwJm{xDEEF+ z5ofDlTTDuPJTC`CYM?0FXqRC>xlZ?(H$VnVTMDHbmB>dd?x!ET{Up5oDz``9ehimV z(5x^D4tz+Ej&OBh9sfcxVwEVu$;E`}rotNY7!3)ZqL3rOuq^!MS0Ga?Er7fI{c=U5DU~BNTJKqXTMj6=oE!XRC5Jm?oWtk+P@cW!=%$CI>$fJf}Q*pDN7Iud%XZ zmZXye%u%s0oQB-PXwx|bJ($yV%yy}87Ck$nMl!?e%76paF;r;SSQPrR@E`ND|Norz zd=_-Gz53f1{FC3keG$HS*#VZ@wDq`{0Ky%o-0|x-pjTPZ7?2eYdAzufjRgRKO zl%*P3eZ!PjcDMT~cJ)FllJ`b$_L7@;&u@Xp= zaK@ObqH~4%Vw%tkOA=59=-X`QBzn6;ogi3(t*(Qzrsw2v$wlYqrCPim=yF+KVFD&= z4bRIYi^fGC;KUA1vFSxJLttx-s$K(FQIXyz$04MSCH&YPl<}1`yNhu7XSG6 zOB;!P^=;_A?2U=ozoGxFQ4moNaHlQaz>AI?z^1r|! z=&JLTp70ul-gl5@=2B^~nJsKc`qK|k{gDUNa@GRH{>8oQCD7%I`={{!lgq38=l;&H z$2FXUWHz&+GAYsndJNUsF;ZkcH#{ep37PPUvgB&p6R?yhP!9Rbkji~=*&Zn35;~FH zeA%bliWGc{=IWV#mSn9W8v^+sFn?l{VMlkVH!CCcPQ;?&EomSQ;M(ne`*pr#ageP% z>R-J7ijgEs{oFZ)KG2L-0i(xd>gm+kX1S4(NVvtf7j4f64@!Zze3Z` zi+~BbFFo`mc1jQ*pqU>fLxs#GfFO zYI^b{(gxtz*XvvX)Qh8!?ZAf?dlvf8{evouoIHc1sm_pDvr~k&upnrW;9;}R&i>N4 z&8f)J@nM2^vf~IbTk9EuNVi`0w`GBS{R}np$WBLaCLIUTPGIP7<$Ps8zVvsFL07s~q#oep9TO;U(XJAt#`$Y5RFYxkrwZF zd!pf@lDF*j*>P#@LT%$JutL@QHb3^0@vaVF37}uPlX>5WDMQ-9!`LB+v+a%^S8W!_ zE|UUZSGii8vT$2>StsCImHr7YJr`1u#cQr;;Ml$Yv!%n#9KQ4ZuY99_>lMRJ_=@EO z=dR3`LN-Lgmo6LjOlc{hA^`krreN)r@G{lA_m;sLKe(07q>7BSZOLxlEB61dS_qb- z1-ffer`N|&b_t65hhzBBCC2pX!;x}bur2NkAD6j~g9QD{`@e;^-|J@$4zHKU*jstI zCY3KC9bUc{ikX8aHNrd)Tlu<1M9N7IHneW%tQ0V25iQtPS)~xESJkOd`I-?Tqsr^T zIJya=r9bi=N0VTA|4rbF%WFhvlR7uuIepk*LXk8Gd)2Hpek^@vkSDwLgh3o84da1e&2rIOz0hXOXV00c2 zt0(f^G?tJWde7mi!o8B$M;E$KKCBId#`>;ZXuFY%_w``7was()w5S6E7ZO1x*u1UC z6Fn78Rfs5-!vtyJ(Dni`k()!OrzwsFyt3r;7Fzu$%5(TN(r(6~|AlV#y{78%-_7 zzYiXcKjmSQQ#0DAUj4j|!_) z@rX$Z3$)uCdKRFfL2b`x2-tkhLQ>A5g9sFPdj0B;cL?2fxq}sA&4XVe98^4XcqaD?_`R0y6n(psZrG)e<1;ci0@}WS{j^0N4 z%tYZkH58Xsa(fuUUmMa8aZ*WUE`QM6`8mGIir`O<$z0$64SzPhj+ z9CXy699$urpY58**i;RAT&_t6H$?05EOYX<-2}gQsb+@xNkSN=oRUOQ-ZJdQxsCFK zfzV!0s4RH^VTey%`UClmsRgLW0(#C9t#0Zd(bp>5O_C}}1F2dK+@K9KUx6A*Y4J=M z>#rwnMMVm1ma<|R;|>ET#7%~H42N9U^{N0`E2nc-f3u@Wmfo#jZL0O6vbRdC!_2nj zoQ0tMe&9JNudHW3owPt2V2IgCrT!ZhxSVl4b+ACn%MV#C>mjs&+NU?!l-TGH!wQOu z&lB>r2;-T9pkK=-$Qgx;K#^->|;B~k!zOFaurgYqkM zfL;hU>RZ+Pw^hE|NcMA^V2Q{1Awotlyc|(e*$8?ogN)D|P?u|f?{_WE zWsOxQ@r!NzB~E9d>PgMPZ}zB)%Ljr@%?JwX8HMZh$H*z#$R1#o38A(lQ3aSqh8Z_} z`jO<2d2I2!@YT2IG>DceXjV|*j1Aolu<4ovK+#@o%;c%ueTvrE7yFR ziVT-;e)F5*Kj-IIp*2_9RjTl%Dz_WxM8Go4jWv-h`Xhr5CV#ygusm9FZ(_|JgWbGT zTv9OXaww4$uv{#9(p-sqyv5Mb+XR+UX)BArQ(zyqlzD;6F!^_Dmd0n{3VO#;D@0vFsKter6Sus5Mc zK6V9{SJpXwe3PsJMjAQWG=ahtZSH7ts-pN<3L1Kz4!^_Am_1{xFvDws$TL?8dQ+U< zXhYEgzz%#b^PxrKBfqgJKO<29=xrv&b;<3Ik33mX0L;r%)`UThR^?O4$2|0b1A?bo zdN-0Buc|nNwh0|DaTwQt> zmcvtyF!m+alruV?CM%7=Iy$q8qUP6$N38^R<(awYTeu#pI)Stka&yU7{yh8-e^J8W zzr6jB^B+S40d3{cd0|BZx@;MJQ}U!!8vfj9S4#5XMUtYiNT)59cml!YFd+!s+1a$a z;F{`nNiIkiQn_*#XvogMD7?mDgVmSasicT^%g@$=Y)+GuVcXdW?_^4yU=>R7k)4>y zT=VS*ggh0%QfZeb{~62>;R2e;1w^gd%|eGGTwRe1<~`ft=h=`*Dpp}&^g<=pK4mE* zzQ_lB`Vq0mqV1s6yz$tQL_rYi?IDl-pjkpslEm;-)-AeDCdszyRJFtx4KEIUOH?Aw ze6_x#DfJh5}aMPf`gq$$Yz$v%(CBnLl2p{zGu*D!| z-A^{(Fpd`p;TtC1G(nNffKLY)R@Lq4IK5x(e5%l7Ej`RC?7MX;?%F-|h5q)v@ctvo zL_qccwFfE5B4QoA*J{0+6@~^9g#OHDEIPUyx{xNr&xK zNH7@EY^%F)P66~z$0E4%t zhwo;PF=2?=+1^*Z90x&EXt&o za4uFs9(RD3TSc zGFEy1O+U+K#?H1`x_;jVTFs7!lkXJC2x@|={#Y&tg4Y#!h&w6#W%%>FY*10X{~KTb zHoRxFh8kCaF|GioSw8v4e9l};<4Uq-xKUb$A~6`su|9K>EX7=9o=Us1J%h2LN@Ejs zD94ZHvsSOna|@F(%u($NLvxDzS9MCui}6nWDB4wq;Eh`mfvR9~B%Zn4YeQ&r@@%4RKkNPA1!j>>Jr4 z&)`x&=y_b~0}O_bIb>o;7W=P_=1uDJw>DZ(@f25(N53VEtxb&PQ4W}cw&_!|C9v`O zwOA_kr3t58bE7m!FjqB^I%N!Z(6;)}L-11WTw2v2C_mUkVvH54=ymX5PE&sYg8ORU z@?&_R57y+G<^Wtj)Y<`R|SB4wHca&$l*DRFG*@~6-MLvuqXL?fHzP%g#k zN4F{KJ7te%ov>Ec01NE9V-_ma=5ym1Om@ z$T2PiINw@VkLVttYMg;m3{tTm7a`p)I%C`MdSU=&H!VBFp@(OE-gOJ)c4n^~+g||d zpoOM`J%mc_5h?N_qJdtsbls7GNd+-@aKK2%t=f>1Q%qOkwswD1Kcc8gC4D$&izYed z#{zy*yrp|xgTKMl9U_QiDt!6=Z*RX3pME62{X)~x0#t!sos+kqrM1qen+mL0(s|j6 zGm-?KAG$O=dW2~36g#zSt!0f#*%ePcbYS+L&)37vdn3TcA%UtU?S)1ieK$vFQGb5j{rGJ58$R;PzB>1K9^)tFtDOLDT7Qmw6dl^YY!OS%POk9jV`oG2`|f$RK_$S*(z zs=`CzwMP$vhF=cn&D@f<1@Y~pvN)gaWUvBaHG2{f*RgNWQLt=iZnRZY;6x!(d|W~} zYZ6GMbp^z5Q(1hwzvvB8w7|rLb;N;-uoLU!$BzL!mfB2#p#T5LeZ2oP{Ldxw{#FiT z3&B}+>C?C0ejc1=FqTIGW%Gi?zh|Vpm)Mi1r>K)I=+&^#HqMGOhPNC*KCI4EBDdyP z?}pwWH!kw;0xnjIhu(}sbS^C=DH*_7QfqTW#FJ!gSpKpM!SKhudSp4F1!xCIPWW}l zUqE7ORAn>&QTXsR>7#%9xt*Oq~E~}1F&rGl|Eb8lD}-Q zAedJ`1V$IT4!X8^#P-#Z}-w&e-JMu8mGQKawCMinLu^xEq;8;eL z2w1JqH%U&Gk49UC&v}99$v1~`4G&PJ@Ul*(T_;Qy_46`HFX0h=n+z3N zRBX}@rUznMS+?s%?Y1H%rWUmem4_$)1^l94|BJuC$zXleAHDs3c>Bo(tUyzf%Sz4G zZUF)Ymk<5zz%vJ(l~Ky|TNQNi`PzHY;>|jE0KIK>bg;Rq^xKuE<|r#xvQ_$eg7(nO zGjXiINJ;J< zq8m^xt4*TkwfDqVf{WvrC93JT@Wslr5sotIZJi&JlxgS$xvHyzehtT(cIc>km6l*- zW(!AdOLOobzx#*{7}%reuc(2zYl%Yz9w`}-kzSS5Cfyp3=BQMtsCYO4DGQ2>y$NYK z3D*3CZgxL;`xnL`aAC(eljzYQ-Yo^&v%Ej5dWeKrR8aBK&N=^DAV@Zgmj3jL{(7{m zVmfFnJ|8G7h6MR%IW+Z}h#taa7_(gV*#xG{sq;0jkO|P-Qy&pO05 z?wMmQYjDKOV{Gy^<)2M~5mz1jg#zsuK12HSJ3Ygim9rDoK8+U$((RXQU2c13N8i-cp}rCRw|q6Q z_xtKwIbh?=o;_TrqA-M#-Aif}B_Tc$@*KCGZF5YKJqAHU%B@mg!b)hodN}1Lfs()y zo5L5U8hnBxGkN0*7#>U5rEoY5uyiH&I3o0tl!9}FmC!N2TfRApeoH<&pHWtfZXT!3Z!oZSC%}aTs8`K7Uj*tN?WoZ=Y`mtnrX1 z^g{y*_68H4I6_1=Ie36bmDL5t7U=qNv_H+I=K)yr*|@4>M$kaO)WHDr2z4TK_Mx1S zHb4A;$~igpL3>A*#NBVr+jqbEFP~%bULQco=34+4>t<$~;Rw}{@d3HPyY<68V;T9@ zCeW&(zj(d&EMZ`YA62y}ukpFfDjRM&Yg4SSKkR5#SiCIw#vo|HW;q}vODPi@?^VP} z@-?XzEMh*JEKs+hbKO4pzUzv?r= z_O5n&B%AOUL#Y`&4E*(Z)dIc2iE1A@JjXnezq`)KfTFOPpO|}F>}j4;9DCJc8j0U+ z=Jf@^^=Ns6>f8;&F*%PF8@OhTr7j(I9>IC-S1OfG%SqJTuR23h(7hDu11!6Odr;qE z0Y-s(!k>9Pz=|Wu{~EQBp|9pOJYbfjJcj&(|1ZhNK7H%$Cwa|dboRKIvQIWTENU~5 z5j!=q72=2d{g>~5`1GYJuQD|5eXBC4R$ABK&#=?8 zV6U$w!=6<<+>j&cu zw^@Drim#Lbfcl{JeB;R12R&&Jcw}K;@Mzovo&Nw-0;ED)ZVctGAuJcAJk@ow-2&&r z1m(oOhGNAQ+M{nAHHtowZS~-C@JBB`j1Xq|qPp9e%JsoS!&M^4h< zfn|d>*)(-71a}^3-dK40@ZkY1H>(ijhzi+ZCtspU=|afe?n0mfkt9~IXUTyKy?RUZ zp}}tFFa?<;uA&?3YBXtFFjLTPT@8&34r~yg@fHOVFv^t;2f*}`Geg6;Vod0#-r}0W zfQW1?cMztok~mJs^%TJ;)wFfG>eF&aIU@24`FmNVhjtWwb%YmYZ*n5Lcvp4Ih}0ZJ z=&3|n-_R6IeC^xtOzA-vgm!9YgZCV}Shq=%O$3;P8+46dN#a5-kyjexxtx&;&XWZo z`mYC=rA)@q~RcxQy8K@ly z-SZ;$oOZTdHv{6hKp5LfZ2%9!owQTBZ&m+2UwwFVe6<}biUGN$2vX>TT>dir=f5m5 z_?O}R`I3>UTY9j-p%uSi>5G z691x$ekmgd0tR#Ql;nowPRvWeJBB=bt4biYrX^B?SSGifleQsSRz;3QhoQ9kvw)xJ zM&(b22-eW4t_F&5Ch$kCM8@iZqP$`=s-|%-)nLEO@&DHXps@agd!=Q_dLcF(-d) z@EN`{l6Ztuo;xgyc;oC-!0*QT4JW5u_px6Y=rMx@f8zA)gY~n)H|kN!LUaOJRlN^V zYiKT~&^{OP24`iI@?j4LumQgVz=l2uKSz7Pl>VQfzkgsYx+wO-Q6~~h6d*stY5n1N zg~U0?ij$PB(?2sC<;QFX>ahYtb2evzq!h@nfQd(*|BH6M@{fF~e8?@_t-9Cv-^l(Q zsQQvVx^C75N_T51^dWa`cV%@)gokIhJV7cOAXX z*V7EftNEW>`AKd$!_Pr`c=j@0wFRn__~TMfWoY*Zslr@7j; zhl>0j(up7uHqLobBY2+&?9dJx)xF7u30TD5dJh;c z@|o~G+sVzVv_)&g#Ml>kIPE&d<86D|xINHI5kuJ=>X} z0ro#>E372GT?P;I8E}ySPghj=zLGHOv@2BQII4dksH^(@K4;S9k8SFg)ovc4&hh20 z(q$M1S$9jeIf(#!ayGmIrfAL~cmKg~p-mfz2-w|FjU}=IfX}eH7$Byj9~w1U1d46} z4gfw)Y6?F3*<#H>G(F*>90#d4<$%|U|J5-#)b!oKc_H3pMIt`4)Y>&lvh)X(v8Ps; zI-VyvFT$=~&spXgQ|fGPNX69updxc`CXMu8;rXEAgBU$HGZUSN$?qjRLRhJ&D1f!x zix=3~-_$vM>|bpMOyJ&cdu~_TPE5@vm(x$PPn@jdeN)qO3rEy|g{+93^o?xi%t+-Y zJhx~+*LV7N{@~wxoxj`Py?rmdeK)`U?{B|_wF&%9vf1M+rTHZbR5(o>Pu5WtAWRFu zT7&a$5ouYCu&vqaaqeS>!s}`!rGz+Lvr5uRUJ`^{XWGcsr~W~1ZNB}=ec}Juf;-1k z3A(ief8}Q8&x%_4GuF^LYsc15t7sO?%u#uWOJ&BPUEg+LFUmd=olirCm%tH+vKv$! zG+$lPY3=4MH8{W{lS#}w99RG+Z+lfi6il{CfxuJI58W!XPRllkt4ga9A<)Q;B;HLY z4Q!l!4A$X!DAfpP5vivkS4UIQDKr1b-&;#~ctDML+v!ohNro!%c|BC}o=soq>l)Um zMw=2LdG9#LDaZ>sRhJ^U9GEz)s#AyZ-I^pm9{}L!;IjXh_kYO0hVvV3gh??nsnJ)> zQUNuL9Hh%hy;VZ5P1V(68tt-MIBbHyJ!Io1%%AN24js~&%VlVS+ezHZa1t#iYYEh$ zHBqkCDP&{X)0}aoZs0CwrZR@WWejc%Jm~}vM#B9Bwaw14+j%j!dV69zCYa|E047L;DwlK2 z;}AQY*SQd2F*Z0V)WZc`yJQs>Xm3iLWOBy`%{vEJsjo>pi?s z8mk%Jg@#1hD5MlWxT+!A+p$ui{H`vs)!hcH!hieC=Zbkb{lE7&;A<_%0<-efHGb4W z%(LxfMp@dsP%Gg1NS93LMeZ2v5s{tPbg&`iqG&GUv*_H4*CS$#c0Zw-w+++^3v5h= z7L=%U24B(>T7b96`+fvAL?G~oCJ=OKkL%U-Un?M2|B3xB=5{ zdTZ;-`SJUY0$(s#No@N6|7-a7{=mHLXuh){h}nUx`U~t5%jPLWW{Ct3j$VAMkwX*8 zQLFSYxEYmWGEk*q4AWu2&)ZY|;SjuEm9`9_bQ9K98@>r8yn~WKh8QU;AjcQ#`N%Og zXwCE!jvqe;@5)MbJj^}K3jwn!R1{Q1Dm(cIu%OCuL*;Gi_@F*@QK_Gv+tRt>E+7y9 zKu%$t@&}w~4G9y~uBL{;KzH6H_!hO&?`G@de_y(Z7_udT<WP=ta1-;Qk6drL5ZtDylWXpnRrH5U4M$s`|x5Xj#i)bL$|}#A625QY|D1 z%$H7QF=8S}=gBgUj?)Sd>*4yv(E3ZI{T1!@H7E@+_CMf#RkjRE4KrwcdRo`9g z3@M>`A)*(o{Y-e~qD{fXFH#M4+8*@bh6_%UJsqc$op*S!@@rv|QttMX+_LauRYIIx zT@3P#<|y(KPEg-pg|}aWH6yEO0BA_cc)4$E{y#9~2js#PT(%X8Q0#wfvJSx|?JIJY z=PQ`1#GrphCQzjdpB38}RRKH3GLfgCx-(MUOVVXAb*P9xxkVkHMmfH*{BF^l@kkv) ztK}qD?$eQ2ZJCYitQjS-z_9;wg!Q3YfMC}%3cG=I`Hq8)+3RTfb99O$9QdlVAk zN(d_o1w%C5sKoK&Ni;m{_x!~c@@dCFk6B2j0ibDTQLIkW6mWItOYvRCu&=KvBHj&i zYW4BjcUFkJ1=WG!w}OtIg)Q|!QseA=dtO}?;L*kXIeSC*k+fB2BmMZfAk2N&49xb| zvC0C(*%_UO4l`4q*6vt9Jl{4H)o{s;Om>&mq2=;NZF}kGhx@65;~_i2eSYM4>!=6{ zDr@a{(TF0`PQE$A8<7yo%B?7H4ysg{Ayahu&%=NI=H=7h0*mqA&4%zPH>@8tAetED z9-%xzbl;}zPzfe$e-Ll4-ZPiwW>jpLQVgu`RCDQ~Zg>=yKH3sq;2j7omgKF_#ju-R z4RHbm#`)I!?gid<$PRsQC(z>AoQ%mcP?zTxp9PVl7#tRNSY46> z^3)#Cpm??9VaCmnEwsh26T>i!T z@2vs*Oez+e|LhLw!I4uWOir)JJ9>gb#wuJ*F$1{FUgdm|9iH@$2W$F8MpY>~Ex%Co zM9X>}yjINdd0=8k+x6BQwpFyES53nF`M(G4rXSf_)sj{X9RR&S@t3lvk!(Cj9R&i; zm1s+Cm1+JA!8D=@Z!fASEH$Q7_jcPs#&s2ym(Zd40qCxc_16=LF>KXW{J!pT2zi8ER6XvKWD3%aLWqRIkyNO^6U)-eq+btkCRz{ljIS0ruv( zGu?yJ1reZHUsBKn4{T(CnQf_I73k2^x~_ww777;!qY{Pd)qA+hb@x&%!8#q$XWt`~ zPc-qKf*+GdbA+w&$Mz=0rcW_!h@t9MNuIz<&$Qs)YnR1B>I%&0^halxt$Bz2Tc zMjF&pB@3ykgBlO0Q~Rt%P0Wh`smd!z9#9jln;*N=S`i$crhmtFM(rNA&=QT8BRtFp zB{+Eq+SYmHZ4W)iitr1FE1Fo3{p;b2hkHGDk{48JYU&l&8KXM3IOZNAaq!c#s5 zB^!~fxW#X3a&5sBYy~_INVHEE&|l(k`55jmhfT~^D^i4^Hr7Us0&N{lJI2I6x?gL)0syo77IZdak;Q)yr=z*-o0pHxG z=?FBihT+NGFR&=Ms~FVkLIUjmY|4a|yLMdR0O@nrq`{SS(}yEZMzu#KTX)^9*qXt+ z#_a*(bz%Wqd$w$Kp9AJOWO3MXI6*{&t9kz z*kBOzam)@|#aSz~lLv%%%4e5RY2}bsp9&L{aC%1FofP7pn&^QnSF8+9-Y`w?MScUj z32XFAK{^1ecsna`R$#hTrj8SVsn~dGoF_$)6>2<1OX~Hc_t~iS^Ap}7w&*?YFEC>T8fkWP08eT7vp$_nDa zWl7AI;rl)Y%giDabcxybnfwV&7(leMrmTdsn4t9p4L3em zq$t+whpQkQ`VQ>XudQ4)AT^XWlFc8g)0HC%_Jj343>cUv^k|DNsbvHh(F`KxeOdLR zRlR+%{EkVg={EQ70EF_d_5^OAfyVWKd#SzFCGo%sb#)QIrdQz{?|tRY^4o8WQvK=M zZ@;zX!+j2YOM5O=bB7Q^SPt*T4))?7{Ht%@ zhuJUE*q_C}M7}`&H!HH)*Pe_~1k{HUj8IIoEhq4a6JoB9o0ER8f z7j_`_8c)YAB&jW*h+tu^ihs=i^qHb$1(HXin{Gkt9v80=_vZ>vB1hwGD!z}73{{fg zWGm}ujW4odlpnLxv<<5TRq~ilj>52MnTRE9m0(K&1|%~xEwY!zeU54fSh<;_imO%H zQq7mA=mNeLo^$*)yDiJn?v9YCx|YG#L$yLqGp@zKiscs^SGMFM@aSH$pZ;5T z|Bj*wF{`n$m-*=ikyL7p4jP2`*jVK?+AEJt9Anlak-&n6s zlBT%KvW{SXbpw3de ztv4?d?8M1ax|>S-<(sSO5VF(x7TYLG`dPRL^zuPZgjCn7>+8%9kQE#$r=y6Ls%#lL z`u6M3X^)@&?(MrOzpsh$&bHWHmF8`)U@5NsEozOl(Y!I3#!M&ohmMW_uaGL_h!!d5 zC4e`QlJ7;lu!8IGsr^}@)r)qW{FNHGgU${c^^nh~*B|qfhp&Gjua7H@HXbQhgJef> zO4Ic9Y#&gs$8t)kRKaml(9`fNC>F14SHSWvY%Eb^rInV`t<3c z&=nV?(IzWV2Y2fwHgZmc_DKA?#2Xx+%zyWFmMAPsZia^s)DLEI@~26vW?6?$;+F)1 zQFagBLaqk11|Uc=DxCxQ#Zo|aXYEqt=_nIz{qEdDM&J8jLRQ}Xr( zZsMi}#XfqUfWfm4AY>GoX`|s!EpPSgumCz8aCn|!hrsUwAi$bjRO}-&9GJioehNV& z3219zZz6fW3hLMM^+Yq0n0qTzh(~Xk)`NYgyR2**YpQOO?M=BcvV39I%-Kh+P;D9m z<0Z5#q2(DAF{yHf1pCCNDnte0a2GV}T zK>9P-Oc9=LR4<4UGUS<DiMBYPr&(ug(}~s97A57DMl8+j$RD_|QOO$y zwuWk5f-7Vy)eO(-;F&gi&ur`68cU6o(&fW3Q|&=&ZODqU^c)nBn6DKA8Wx7kGOn6g zp~!AU{^O9}v?(UaGAD2~Pd1zWozCslpX_~4aCm9+JA(u`Pxz4)QUd4(xaZ%#hbv0w{gqG<74Hrc(; zU9ME(U@)59>d+SOGsvH1(mW0Ku8*rAQO@pLRE1e9ek*8Q2$hNuq^kpWsGHh zPQU`%L#hHt{N zMQYBJ4=%OPiIB!ftqRu5HO|*7gJa${uKombUF`LG09exo;f_T zce`h?I^WP{I=TjvrI2YTd1ixhbxS?_Ajr4ub=Sk%(Ff5$5=ywT0Rv=6q+X^pFP;;c zM3E*D88GDACZe9M#${2ApdBY|Ar@_iRB!7uV~}&$1+y0|r9x$osxvf@H--yW`7Aeg+8v8nhJ1z< zIzBxX9fdSqqxF!lf3R2KHi6`)?bv4a;qJis3U@DVcJ*1=-U3f`jHK;FpoK?y}m(ffFVN#Z>pnZJ0G`A z2lzQmWc%uYYvy`DigIJ=eLX_w*WVn-lx-QvDa6L{O#MS}`bc%b=G$$bL{-FOvrlRj zTVsy{C0nZoa{cH2*Y~jJxPII?amqY~JNU^tnUL>KjqoEph#Dy~ZJ@>d zdikDbvq;n1l>ke4Xl|fs|E#FCy&x*swLl;|Wh;5gt4Vs2_lpHyMUzX)#A-b`15IwT zOR)9Y@N?G$)xCkpOTLwr^#LT*3{T7*(1-#M&Bo~0P+u?KkSCsIO;pkpjfrsbZ%aGkGh_<8w&Q z@UX5;)E-S(g!#Wr^dGNl`cUjb^7L_dIJS82(eBZQiBsQ zLT#<3>NAQ&r%BCHi|WOk=E{Yq=fLMkGm7gx+=>I#b%wAXbbpMyi?$J`*|MC-g_x-1 zSvys}Lg-&Pw&alhH6>=?cQjvZ+0RGmb8ZXZo-lBEO>ov7%ta3 zNl*2>7DZ$!yhatT+v3aRxVH`2Us<;t1UO= zLzam-E}wq*_Hz)4F~G?zT1?g-)@7Y*;1JwbsW0x^=(F!9SX`5xfCk`FyZ*@r3~+Cg zx=N21*+Pw0vxC0WR}x&G4q6HensgXb60HdiPBlMZ7qaz<61feScn@;r6vhA6%ltjQRCc2p7Tv&q5C5#y4;7&S@H#AL}mn7 z%G^suK^QvqdE#+JhR#|JWWojxcubid*;|1SLHR@>$2x|(zN(9*ZM$aqA5@GZ+uoDf zKkA+3jE$0AfLzZdO#VXEPI8?59_Q8n@c!Mm-^j0D0Nn7H3>yFFP{%V%mLoki{R#A7 zDk9r+QVC~!mxoajJwBqyT!GmdmvV?g!X>`~Xv)ekx!sR0F%yOFAT(f&)`P44DPl+8 zxU;oH@!SoE;(ZS*ME8MYl>D~8eV2;=KYjayhNWB5CTC0Wfje5WgOZKeec5(Z z)L2l_%Z8mtZ@1_{N__7CB#di62d8%e>0h&=bgz>(>ER~XQ?Z_lfJPaQPdgjd z-#EX?zvjCN^+}aqZ@k8H^-t}?3_QUhNS|+%avz`2%y5`6BC<{`wGXOa4^*|eqLG>gToA0yPr|VGPV-Y->Z1* zzG43bzW4_3msx^-E=(bzg=+Cjs?iEA8)?N+B9Xw-V@&8tyox^AK-fVkHulW^ULs(- zgrQqIxxYynVf`>QWHYs5j2v9wr$=m`*hWC}CD|jYR5qAsIk~H9NtWM$XXq8DLD|9A zP)6P|6R)!8Mu5GmJBSk0c8<<#qslwbr@OPk2YTmnavMel2f|ewj+Iw3lnkn)ErweJ5(P40B@}_>8^^mA(s`?Z@JLor_idS z5&7QX;chxAfpJQ&Rhq<}(n-V{w4-%{L$9Z$kh|x|N=`Z(AL5|<$NcQj6Sq({*ghM; z8n~-TZhQ35p89ZkIS~8NC^wpstaKBwyY?+77ip1C9Eu)qRP|hnFPy2{vj1+_fU$j--owfa8v#!yniRZ{%?G>ze2R9x97I2 z$&n?9I=@=yW$oo`9w>(bSSq~kPgck)FRrGnUBA^qJHYw#1Af4oQbn{vQP8^T&9?lp z$548wb4MhP7%T7y&E^;sMYk5w)r#WM3GI=Q!I)5^@~}}G5koYhB?nx>=qe&mpCPl0#g$2}&LrNp8i5#CHupa< zqL9A(UqfbHD-8mQ9AHaD#rK73qhW2YQy$nujVb%U*oub}0vsN@lVsIW)OG>Lf|;o4 zd&&lu01I7ndGJ>-0pjX|R?;vLI~GOaKz z>rRO0U%q`O@CDZ>fDZpr-@MMFeE2N@q>pmaV19Qkpa!V8yHmS-*vMR2TgE2O`Sh5F zoAXMs?x8Q@gEtWGZ?yh{b_wP~vL~F<5MW@QB!9{kXU&%J9pwciDKM~?0|2(5a&$7^ z$Tm&I7#)CauNrHN6;g@D-rWaZv34{-+#xxO&R^?hl@IKusIBOHKfv501qh?r5zCS7 zPpz0Ni5J8KWVvxpj!5yx5qHk^?uOA?ItBcTB4u zU6?rTUyD&GM7FjQNqz$pnxtoW`N~2J0#=6a_JUk7=#rP-pa(GnX(QzjJ=G*kwRAhY z&BwY!l|}Y&!#($+N?e$GF4+gT>&vKcrKgCBu|x@bFBl^sQIbl|5u+-|dzDPW`lkLN zalgF7R(Gx(4Y;)BWS{=;w_nTuzXZ*!EBglW8D2S3m(9B2byj*1!irSQ+2U-#iU>veTBi}NC>~lGw ze3mTP<$yGqzwN^ zKe8|WIE&AJ65f6bEan%%#E~BoIftvA+~K6(ULaa3tfa3#sE>eJ27<3pKpr(lO&ks0 z*0@b@@}>Ays1G$Z(BZS?P+3G$4LE_7N>|&XaE`YP+MoyyhwM~Y{Oywism{{9zX*RN ziPBf!3h%$t_s_V3B9f!zqsKYFs&)ZJPzOv8cG+?VWgkG(L&p>%)vlvecMol`x|EIg zghyzIv$CmG69?)ex6WCVQv(}GNhB>+ZA+Ny3zQg(+`Ng?z&;MPtZ_X#3;aC1|KWUq zD#@+;U}0$p4L{T)2G%5ZQuZVv?mDa9AqXdr2k6nBy61X~Jt*atV-rG?+$6_ugLrJI zviofgy*zV?U_)ONcW%`0xD~vh`ltlRU7JXlq}>~baI#tE8q#uL?((1ops=B+Thr@7 zG06jnq#SoCfeYmjp{fTaam1-Ap&?;6%rja*IoaF~4q+tgRd$K-PPWD-W^0Tul3vn^ z@)*_Ne&!;*9XKXe{0|)+3oO$~{2s+ivbgrKvE$RhDW5m z)?qq6hGW$pU2QFy8}w0GLv7&RYeizKehhb08B-Foi`ZxB4$WK0R{)xkx4BNX>uRWB zKYGF|V0Oobr<8RP!__?2T6=QVP#QN-Q~n-%tIzicHZcu^cQ^%DUBC1vMYS3uUhCz| z>co&=UhUhw6!RdkUjT7%B?o9HOI&gX>uXY(4a^R$1g%wQTk(;;1gcePTUh~;kToq) zqVr9DqHMsC#G-{-O_-Ad`;HgHt^SSu7x+Rg<=?+^_6z+n3;w@*`!$qR|L*NK?_b*O zlHl##g6#n>O^Y3STI5rVd{YjIjjIV#kbk7ZELWnINT|~sTLkG(qoAvS*rxRJb6a^x zWrW`&{UD{N_a0gcCtzg&$X40F4K9S8;c${Ipj)*ZGlYzTduM~XKe7LU{ccf07X5Xd zM~s7z)1i*yBn7#KI#x?)-SyDB%os0N9;$ALim(ZOC5pgKjEDlM$=Kb2JokE(x&Jpr14^{_n?#q{sICOlRVoD%ApfKw;PsF7l3FxgHiOrC82> zW7hUk{QX8LLDkF8m-9t<`@!YY{}JAQiDS2NP9yRk?Rdn7RI@R*a4h6jep1U;p5esf zIZ|bB#GJvibmMN1&*n!(U+lvg{5ubj%557IZV3bXoE+#R7qw? z+_C+xMjUaWwyC;>vqx5;4rQZx0_4s44)OK1sQrL5$-(Ojssl+%Bu#yZwi*94{EvU? zNabGvtq;3PiY=P#HSKZnoDQsUvd(g0+D;x z@fZfX1{^a^0+95~TYdS6xeS#_riRZJ9=JKrhIDE{F|?B!8ezZniov7R2y{eTiKB-Y zFy*PEY+x6Qo&5;Ap*n??Z+P0yfDKsWW0I}9bd>0n-D7oeqc(6#q~cz+x$~DKM*p^a z_50!Nr>Ao7n68kvun?Kh7J}%`E};-PD&Xj}1OjkU*3nK;BzX!`&J_}R?-uT_TGsM| zbVb*!4OOi`9kV!sy^#WqnRY<%jzqR>hdK7^TJ#0wuAS^Be)s-8Ufp+3vA=hsGnmI$ z#S{>uGmcVK_rtY!N;oq0RVtxY0&OG7;5`&44eL(~swg4#XSaQR%1OGqK6N4k?=%;F zf+*CE6!c(&rH3p&;q>dQF31=4q+WR`Uj)9$n_Y4SlR8k*2lprLgH;?`WtsM-K&pd6 z5Xf8Nun!dfmak+{PWP5u=QeZInIj{n^yYKR=Q@vCs)4oKS2iM%I##zoEimzey_jzS z2WZ!0w9V82-~DQ-PSXQ))q_ISgO%-#bEyKdN{WF6szjS}g`f-uwS!XK)_IE%>_wt{ z9tpN2lY3ZgLsXreLepK7ar5n8dSmRNg+Xfn&(fHJaKrL46D>Y@-0cbw3CckcEidlNb|6umjHI1ArQK3Zc!s(sJd zSv%l}h=cX0rVAWgJjKV&Epl?6p21>3^14{#HtZR8!m!(@C0O3pa5{njsJS#+ObS`) z&W$5?fU2PHc?`0}iBHfO63-$KtF8>msKvZpBj=TI7>3qnJpBkg+zAsR$vHS+AEY26 zx`!#~omPk0_m)j!AmVE_Y^Hd{QEeyR00n@?wybQ^yS?Zb$h?1tCX$@0*5$iH(F^>R zqx2{Cqr!-XSv=Sa#l%%r*k<|oX5pCcp^~E@rN9pM1iKk~+7d&bL1=)Dq*mW! ziU8eK42E|-x6nQZhG4eyR!KnUkm#lOd=O|mx=T%3CGMW3VQ)3;f5V7(CUcR#_sMn9 z=ORMJ)eo)9TT4akYv|Zr+ub)3b7_lCK(+6OGavFEeTh)>o&12FRp4k=OH?~0(9}`i zycYAjMy=5w3~4Z~w4l9d(N{+D2OJXHnMkIpAcv|xn>ew{Sph?DQn#`IrmV=EES@|% zt2pklffWg?J*}|=Z$w6N08zYmyq!wQH9KF}<3HG?0NMsZZkgC8QCRAot>E-@YJ8m; zw{BASSybQI2#{PHHGb+U@*xnxyE(D8iASIn15^_R&cNRPH81TfJoEksG-BnjEZ%$) z4411l;U#+GWI%V{G6`tuENjtHU4hvvrTYrBv(94vU%K9`NtWa~4}8yG;ed^X7A7d01i)Tbs#&%=LtSFlY4PktlC+LluC!rT~H_!mPNi;y9v6$5@{I5CR@$(&z z+ahTaZdTpCRhb^{$ItRD>;z(L_HI`sWfpS;xFT1sW1F+hiH&*4Z9!Mq`X1n`E2n#E zL3A!7!!!MUP8<_zoJI;q-?#c&-S>0!T=?0Zcl6*jUFsFuAL;ys1UuP=a@9f#Tyt1 zZNg`rFjmFg(|@gG>>~q7X}Wk|ORW3C@l%9tYZdY{LE77eX7mP!)UDx!wpui=F;6Vq z&j+~3Kr;Kl<7P1e;Eaq~d)i^cDoX%{6$sTu<#SV=rKb}wJeK&Nnr*;?5X}Rd zlVt-4_){eO_~U1n_y6zfw`^-KS^lG!h<)=66#{Qg^o{RGZT*kee?fak z{`@78NcS#$;I;@*m__tu2BU3@EM02)`bkJX3udtT%eL!AK!~lq7@n;+I|br)(2@Z7WVh8q!l!;PbsfRgMorro~C@W^BkDDB6C=WA-P<=zF zqZnt(!5(uPW@QD`k^##9u&n?`_*OMfI?FP8w=ILCdv}R#7QKn#HmNq2BSPVf41&Gf zO(@1TWc&ED!%7&Y;jav`bXV_j%U}?o3awVHK2?L9>}l!!5_}~7I_*(OE;pQA`L9#uEv)@85toA7q;BmV$VNt)Cy-*TpRXe~L%`dX0BCF_dOi7d6bg{ex913>|UF7yhJXatxK zDp@IN3<9u`pF|HG?l;eUgE!#{^nhlwpVWIEbV3b73y9*f<>>jD zX{L`bdz$1SjvPs$el zz$ZqMNjs>{TNK2_Y-v0gh_?0WX->|i51W#VU%zxTcHeUl1Ei0gj||9ATG8XV@CK<+@#@QWfS4$lI}1Kcb86N5NM4Rvjk^G7lq|LY_?QaO#al>k;q(q_ zINm0u84Fa4e zD_jCtTKH8jkD^Jm-4_!fk;5rvp}f19W_%9}F>K)w{R0}@7Loxin|+@WZsFDH%YaM^s1crdxFa9N}}>T^esA;yA<;YlaBlMYTQUMG4Xnv?R5q zW`FDafl}@bD_GHJM;V8eIdTM3O_!$reYR=(i!^|>$S*(i1 zvLV%g8@H>vq$akKwAY3wTIRwZM!Tp|!n-741orY;0JJB$x4Q$Zv)y(UzIW&WP6oC> z`Rb?3ecjfDNSTBe%2Qj2hyqtkgE459iM6(^bWb0ai)d zAMz8HL<4~w%{&V_N2)wz9d)uPBx9AwF4c0+(V#YWGJ$WZC4Tok6sN+rTmLmaYaEy2 z8VU=po>RxXm}#zu%G+-{3dNLr57?Z0ivLzh)AU*m6n|lxyJR|h>VR#4{7Y2WU5#l) z@^+E*BwR1h1ZchdipxZIylvSdrQ|qbRIUakWqE5#jY`e*jiCQ4dW{Ud%MN(QI4!uz zH!9NWP~RT6Ji8aPy{VKATon6a9VQP?{K66es8pEV-61E-#x~P{K{VMks+zPAOnIAv zyIRYhzRU9f0r%wL1S-fvceUuA?q!k1oQxg$bF$3;#E^_TlQS~?vYIy6tb%VfSp+Ne zUJfz}tWPuVp=^@GZ7z>dE8Em*4yEbfO?Sea#wQoRtiTiYUP=El zLZWmbC>0K5=h_W+1JP%HDx=n&*8LB`5|_dYm!`YkTANVWd=%# zvRf`-J4Khjn!r(@Ebq977K%xQgz+VFUxv`6hwy~$K|9Dn(sbbYaf84KYL|k7ct9n4 za1Ag7uR}@QC*?8FoOn@2P9pce8ldG@aFvC&Tsp^6M3!_;-Jx$~w^Q4xLRH2&z+r)9 z#GI?9@nO`uuB7*LoH(vl5CiuBNYhIJp9NGM8_D$rawxm({v!Ov5Ax&q;`Q_NXZZ7v zFzL(m(z~17OXfX-J~&bBW(l&5>F4`;RY-F$LmzYvq{ z3BWlu!r8Ls^~9X$Hco0HSo4yC=7-C>s}?4W&?4`4sQw~&?VD_rL{3Y)zm)2@vI2NV zyzbQ)qbLc{s^L^FY4RP&#Cw>n0&vpZ5@&UpMnC^y?N;y=%!fq<^{l%y={;ltPktzL zW@cIi!9&b)7nSYMYNV$z$mxQ*=p3awZWFk?CSTAU(3-RxyX`hLWlqt0fT4y};2=kI5O~u< z1p4incRs>e93&PfmS_>A4 z1$(1|A5JC5dJThR7BfcA{%aGE!kHc5p{*A+@Udgfw#;}`ifyZC38iXUlg|3iGh}#( zJUTO3g=OV5`rRSFRK-cdLHWMAP&EbX++KYjbt>(@d4_!n@DQ7->9{6*SlU7vmAq39oBh3M_6Me0Vt94K;w z4CCpt4hlpAj+zoKQwE8t+v}u6!*}wd1`2n$waNf99iBc+OKCw49n98tH_yk!&o$o`jn z80glHj0rbf-2^SlKaSa@?$j`duvC?Jvs!6WPdzoJC=_K~i zt&nO59;qbpexjFEBVof8h~WBQ;kn+gCwYo;>U!;JJ4B&W6RNBsTnuwbXp~W&en;-p z?_|p$yOQD(BASzA-!^~TLpME$^hWk|sOhJqFa`ud{(lnBn4u15NyPAhZ6s2}=?Z04 zDMYNL-h#)Rf@lr361v@}!%iXpy1!1ck9@{NM;K=G|H$nQaf89$rBrFg+Brs5@An?( zQYa!l{j7Jp5~-QmOY}vR<>`HVfK_Ehp1gz9-;tWcB_|eYrdJA1-Kq2qO+nciD575` zbj=cv?1ketWUlupMaMftxP=y1DZQG2 zRSj9UgT|z)^hE(I0Cq3}#>g$3vb=-m)84nTgNX@H69I#!frIrl*~V0jVEL!X<5!M3 z7_viksQ}GO@8gF*3_tv<)0TtU?p=)~OG#y2F-Xy<`*Nig!rHr)aEhG)uF$!jopq}Y zQ3CEUpt_e(dviZq$W^-%!_&m1ww0@ZnFrWOw$%!4h7v!u%Y6;z*CRGL3XLO>3fr)SQYZZ8^H@s5U-bUnmd;?5GpGbhW+SSDIq~N@RJB%P9}@>hvP_f1FTzVlP@5dJg<1*E z>>FJnx{&Q4l7RyA+a1uBM23y>ql)u{Ah`x4sB)(?gawp5w`&ctYSVZO<(~EuMn@Tx zOT6)lE{n|*!J^uoEb!Rr(v#fj*cehn3QRU1XX}EYYS4TkymbW@827&U+en>|0%62(#JT8EHL2_+R5R+RV!G3vzBki& z?IcJn*#klr#rm;H@(46e0u$7NEIE+54Oj7C->q0%sl-FmvbSfZ?0*bNR8GHlk4S2B zQ|_LnRg@WkZYKMqs-5~M9*fU777QPeB(&rtVaOnxN1@^_Za9tV70jYh0)03YRuQx39Q+9V7|M zW7id`qv~$ky2Es6ZNs%kHOaZ?!eqZJL*;Wp+b-{a^yeSJ#OuYbpGA|?v=b@(oOBRc zKhUc-nRY!y9UI z<%V5?v)4|rGAD<=gePZ60@r>lb8iq>eaw4?WSK#aJX~>o&pfLF2x_WXdwoUt;y>oy z_x0N^NV!#eawu z7|3qfp&shp@Q$@@<5B>cCH`xYYqq13T8?BSZAX-u+~mnIWjmdi!Rv|Yys)Y9{qMr} z^P~SAYHBL0e!?>%wI#-6s51LeuaqWk6WIs(Qn6#ym0B$W%UNDlv`RYn^;GK~i!chR z9ArZtp(Z3nXwMeQSmr6N3j(Ex$*^$T8$quBF<6US)RqU@6(FJ$U!q0PnhOfjb z9WS`7#g)BtW?27p*@tj5pjj7o^9zDr*rI-RE4_4X9dQFdzb_XM)9*HH#cDY!EkS0S z(S}L3XB}ubd^oQdA-KXoKVZ;WW0XZ9*$!>na5Cy?j=-`%0{8(SBx(RJ_aOy-?99f( zGi$t+8MrJ03q#pyb($HaFVUMT51`KwcI1+~X0_YdeV0ep+qWXy-!80(n-o>Gy_%@1I>7f0`xZAg#wv>rA1oM?2>?IH;h{zjt zfbHNpAl49o*uqdYX2J_Fj|C>yNEmaC%9nGyea&JfvK`B!gXl~~`oFwb)TmT%n44I# zXN;g8REZiOuw;@A>e9TCts?PncW4C!&ze0?=E^F3;9efyegiYg&Nio7(ae{7hy35k z%~EMlPXIHBb(Ui*wQ}5PcPG)hqbCtM402(Q>qDNqr&nYQYpc!MN@LotPlVE@Aw zwWk18To(_Gu~L_ZQ&6o`;UdFFO@>T~^%kih442+|!t#2o+>+87jpcBxa?2a37aSdc zRiW7?<{#-3$U9Wwv2-ML{9@;^s)}5^fMdF$q5R^>+A&WZz=QV5<@E%zU)r=j zoQZI=#b?PtXGt+i8=9*0GY=^0`^9pdm*;dI$?h zUh+$&Jaat4vLvB@dzKKDr2|eG#v@IT5B0W43#+;b#YqO@cYprTzx8MMxBeon+q&Zh zprz~rh>T{)X*YCtwV$;E3l5g*Wo(yG&Hv<)pz&Cqdgeq$GmpMSWPj6W`|GAW84ATI z)Z3*Mr2w5U?6bhyYD&5IlD9=0Mqm|^On@9BW!hQc1UE2uP^mFUbhGN0WXIuQ#O8ah z(*r|uIWnw#Cx>1x5Sz-2{0&e^?>~`%@ew8#e|-H??!{lc{ptPZum41><^Pbs{Wb|U zeS|M0m_mi>3U9<tSM--$(7J3OPF%(QGd-fG+lM6N36p zjXnTaoI62GI5(HJTU2<&4vn&v(`nQRcvfR>lyU)1qebGbM9R&!gF>VM_6IKin|HsYtP9XbDsY?w5R7iIS-b(e z81|-l98OJbivA7|0fTJO=WPt_|L*es=dWLqJ-AIJWS!kQbj)z06oNA07~C9?JCH4} zz|>d@27lQ2jH^pdZyTWJ=yHzlAXlwDbHE(Z=dP+8rNMpZt+rdH6C>|{_c%bs9p*;^0G=%i8r znt=VR2N<0aH16u~Yh0gwmHM;bEJQo>y}Px*wKR|53;8!$dq6AX$e_Y%qz^Vrx#j8O zQ;HkKf^Eg8*%meUDC~$x;poU50k2HnKb?@TIJIk6A)Q+nyuhwzQUROQcl=oChfIHcCq=njMUPAO5d& znqhKVcSL@xGNFyO7bxAXH{=)5@l++&FH0X)tTiCAAHlFmo& zNvx2OxLNwA?U{vhJ|Ep`(|0+IyGC?AwHr;wnM5HNl^@`q+4;mIRyofRoecbI>qUC(69sTsknD(-hY5>kKz;ScV0hHr{(Y57hlEZw#@$`YS#eT%$dD=#A z5HyuHCs|fBa$n(hNRQ9LIvYCUh^9W`m5#*X?c~2Zhr#*-5+$Gv-dN>48sHl-J~w)D zR~d(!AM=LtdxupM`Z$WJLB*KmZ2$^k2Nn1gZyrD+8T{1yB#G6TI|>OGlGh*+kZmla zqJm&cF0Xd#F!vZYlQ_D>WRqfbaMIsx;E*m`mknS@P61V=dKHwLQ-^}pJ0+=MIytAZ zrcqhBPlpHf5_cUCTg0D7YBzq(5_SuyRHA(3&k{GQQH3CnMsbmzxPM04y0BImK1%xF|DA8-Ll{H;|U`WBZ zqa@S`?wu5oltVdasl=jRZm#Yk!HDgQF<8B_Znha$IeCUMl^gaXPa`LT814yIAf}14 z))wjgV^VV?R(MIADBcsrQaYKZQM7-036ecB1$-))&>&l+>_{!!mUh`y9|z2uucofs z5=CBtLb1dX6fJ|S#_&=Qy@_YMoo(40At#Xi2e_<6808&<>;h3k_5M=L21hTusGS(wYPIv907{}~qagxL zPR_t1=nS9@RfTj%u0f~Ws4b11ruZyb9!BY$Nq$F4=nvo37BUg1|4tah9UUW+=II)C zf>h}@qhLt0mIMN=K`TL!6j^;I*8psf>=HwQU(^1OIA3jfkeYv?X z=#t=W_VG|zo@};IE6%yZfi!^C5%b=!r|BHD7KnNd<}3ztE!=sgIM7T``f!+*&ff-? z@&L@4t{5ug#=h7Dy$M6_mS_}FnJhF7+mR(MA)OgNPPBtU{g`B5U*7-n?c?zFFZ$hM zUUh71p!8O(ghHPq@1k!`+=Ehu{GUuihv=f*DptHL0gqndP@LvGC~olt+(#4 zNN|@7+gmNznfb8bIo0YHJ;VLQN@~>b+R-IBsFJ66=P8=*>&-hF1OC%T;dstLk|{Kx zZx`jNU_(OMpPg$Z#@V5=tI7D)Ku|EIHOBxK7jhd@!==?>u7O&C!huJe6ZAEK zLSH{OHhB8#i}aPbF1~)2zLd39!2!K$;)jmIE5-vlY3!&ZtEeE;j6%)gd7adOoyt>l z3XXOZC9)R8OpCMf;zo~i(5VO*zrl3J9ItO-GhGl28In~T()% zk|+C00^3+AwAAh?IZp2N1UF!)-`Y0BFvA6cp+IAaN`H)Q<%fy=njEyeserP^p2Ytd z{@M_wFmd=338TMHXF|kx_x6cF_- zA`Us!Y}^kwi|?u`Ldw`kAul;2YCOO`Ytw<-K)bf9_W+}0hxku#yNdrlMyMr27)RL= zd~W5wK&HRAk|kNjzJS*5iS+RLsR8`)0-(DO%wDfSNlnlVFb5N$HLZcOPzvp7(gyhE zF49MssDs;R1UY-QD<27*EqTgIok&6GvH){IX<(He>T&%Sr!(*cHA%mCP37nNkKaCe z{VJqCE=O^b+sw4v4{rQcg*w>}#5O9k6X0N;GB8GWO&>Bm6Z_;Tj9np(3ae@&Seayio0ZU4D@%rQJ(=mq)B>%dQu5ql?#M4 zMcb{;XL{1>hJH@r!CXxup#-;qi4)w1DYz5Wh1X>CP&Tf18`i+F(FJjF1z>0Z`B>QH zpw8@;w7x5gEy+Csl{vI?nt=B&YC3It(pl|E1?op9`I(@^HToLE4m`s1fKVHi-+)wY zN2uHa)h~&THnX;g{Y1@8HAa+Bu}ONaO#~Z!N@isbYl6$Az8a4}^+S2-l5TR0XK?IE z%uPnkoSGE@^1pJgnIs-KJCbB&H0;yvvJsicQv+^%)JQ!|Ic2-CW-a73cLAYj~7EDk&L~6UgBt3T&XwtM^eUL4M%Jq73 zJ%*0%42M)g%K$2;JW2!_$q6OZjxm_Four(ePu6)5=;K>Msk|DM0rT43EE&?@{`>Fz zU*L<&`@aj8;p`P2v|enFZgYjR4N$V;aWiz_ejcW-3I%95M8Y9tj*speeunzQ6gTpO zAZaRV6i>w87E0l?VUvYb)tL!ZpaVc|k&yHdoAd;$T~S3&ubDF49Al43ovzZC4pNb` zeZ3q~HOpRFvUH8M@#OG2b-XF0N=B(E!MvlKm|Q>#PO#4!Fxsv?=9xtNBcqDo!zgdU zQRZZQ?6e9QR1TsB{_j=*QoJUk9NcTSQ~Ul&e*9Bb{SQ)E%9RiNwiH!b*=*EDiqCe> z#&u1@m@suN-sEXGn!Ju?mtUzv72K)>!m8?xWg`DM{O#ZV?SD>>Tf3f(U5oOWx~F7e zGDuKXzrL;T>}yEMQ8rH+l+a`;IgFj`v!T6WX-k5!$h!lL?KWBv!uvbndoY$Nx>1qh za!K^`sL-erHcrqADFF~}P9l@2!@bN($^hMZ+R8l}9Z~|x-F%t$vormmLU9O*a%rWk z=BbiPl3L_(yn^?(O$_hoV>0gw#|RK)kX5ddC$mj=Ws{@WkdX5;-Y3*;OYoV^wXNEC zpMW1I=re;UFcFK(L46Pi@8Dk5&fVrboH_BiBW{T0GH!P{Qscq)ZOTy!ZBq4im7vQP z_Vv=W-shELB|jb0J`{k4TyH)&COYL}nN&?cyA&_`lK8iB9Qahr&Ow7x03tJTS$42u zn9%&13C*whXO7dq39sK!nk{inKMN-W=*!+;yLWsBFA+w?={KNvh8%5IPp+<#NKIxB zmYf2Dy29?kCmUw6v7HM*vUv)cnRf+eTqkT9#mzH!k|tE$cG<7j!aANR%C?eFS34Bz zHg=nP^(G(IoG;-zL|S@kpg|jm@seES@)p=a+}6}_r1dQH=_;qM#EgGO=TS*}1k*&OuV)!M=BM;kc>87g>hFW`ksn=NhDQ=LlcYuM?(8Wcy$C86 zYs-UaTu(BkJCMzuSuGA~o)Y}!na+rtM74+dzZVn&1C`A$_nQLYi4=K)Pl7XbfLY}m zt#1yBt}s0Z!490PK#olr^}fDnqgxKt3TV;iBWq7U5sC)DfVFEVxTcv!57i&G5b$V~ zZySlA9VvEob-h1*x8qv@6?Bu~X4 zA8@D?FKho@8xNq{sFw8blU54pF{ou}nx8@fL z3i5dz*sCkc>p_)`?E;aauA~-9f=Ehr1Ov^%>cYfzkyM+Sx2a|9epyj=eyxSK?GvY` zjfDc1TK7kFe}y5c86>C)ilD?s!$8b#A{YzuoJggjf*2G+MkhVX<`x}Alz#ZbKwno( zmZ6gs+h@Z#}h^!D|YTIpS8Rt`CR~rVJ!u}_I|RZQFVj{cyjfD zvJZezej|w=PG{jL%1imAa=O&GDP0*LwYI{a|Eq&~%bF=I?5m5jPhteX>7F?uD}3Rb!E^>QF-T%3aiXbI zYJtOHx>KjD_a5C7#;>%>!owho7dl-j{YGgNc^)vp8BZM;fFbOQ5fpKHGgQz~soD$f zsx#RR$hgx02Tq1W<4FOQK}tFmV_KlB-P00+OJ^9vZ3iRYh9;^3138MlI)M^kZ((B8 z(Q~$*J*`n=S?{Pp=-i~Edv!_w&BD)8~|IhZrB^ z(Xk-D6jrImg9U=lkwEGj1TjhLL}8)PbeWxz2+nCCL9;8-(W=>YSz%&EG=gl552f~{ z6o(}^Zx*#%^Z|M`+GMtZqVS<>iG?#olOF{WoB{~9=xz{_A|X>&5q+j??V3i+tBNbi z9U7Btl=YsUFu_tCQ?(VO!35>0p*pSvLlUfh65c+$yiBe!4tjJtK+h(x@JMEc-X{Pv zYMcn;l5kk7?uO_2EifhYl~_Rq%L4gj7MTybkVajliixUk(nRCG^ph0yzGr@mF%}sN-A{Z z*Xe)5`HK{qpoH;&->xIIpnGb2LviY;8y*Z)TcMKSAY!|*AoRjd)DxiCPb%Z!4^=Qp zno*yHtNU%J@`69o8T~V)~T3>)RCLKEnp8B|-50r*A)d`#dDp z<-b9?gnBZGPC%+j+LwHES1%YK3$#QG7p>)q)~=n9Nw(7x4~$|YQ?;R(l#=yD#h?P_ zZwwqzU2K61qmiPV2b)6NG2Y49iX5dCQV*!3*|MmxL<3b()JISU(j6YH&z%}?s+Upa z33a@6rQ%f?B-des8lJnmCr*awQf7OaM?K{4*))A<;0ukzp&yc}Ll3f89oz-e7YJl* zsPYU>4J*BUu`6c1%I12ZIQsybBhC0u3XKa9L=bnjM+^}&%ef>Mv7LzlilutDQ9fS9W%;E0LRT`_qiEMh_@Z14J7 z?a#Yf-i_2b8@+-68+9i3kx8D>r;RJDxZNLca+}~edE!`v#{^B??9$tfXaOFiK0YPg zz0HS;w9je!X|GbBJ8R3;@{CR+@cvLW&yigPVf{# z*RQ1FNx829vEe#n5`!Dmb114$Oo5u9(>zp@_uYB?#{|q&%?b&ajIMC^<17sVP%upP=b$C4LGTKQF0{AD4 ztMay$U?zJ)wGc-n+`y-{QwHkDX^*P8FaN^ospF=d6Wh^fV{$&dqpwR?x6}(dlUx}q z!aBP@Tk)L5Nw3;FOHug!j{(E^@#|M90z}oPK+o?`Zl6ys0X}-t7OE9|usU`OesM(s z*gMG=@XcDeUMs+oE%BbCG^KSxo17h@yxrYr`36eLHbJyt;UJ`&;lGz&cKlo| zKoc*T4ldmW5P0V_P}!&I;Y-+J6@~#bA<6r@_~A!Ci*N_?m45$oc+#t2qmzf=EZlB7 zae*wDT?1(M@JhI`Keco^Zo8=ennZ(o_wl|`*XCmH^NH2TLvncXmgE|7^WB}1-Avj( zvCHkQxxVZa@WklwApvn#*Q%w1rxgS%C7D>lJE&XA^MQ1!?wAwSshrC71L3_A^0F&s zHzn(oiyU@>!RhD;g*-%)3zY2Vuiyh5}wcV8n zm|hKjXh%4lQj(=t4@`;)_oy-~s<(xM^izy5KO&7yY~~KIPu{S=Ky1=4)l5>IU=jg1^;W-c^ER`(_O%BxnURX0=2?Fx+e<^56NxSDG z4U3H$qVOk>7kXE zfT%^DVF~R7mP0JV1=!ba!1#ag$q+7N&F^i2}<;_q?(gdKR^e*#R6t^EbX_Q@{P5{ zfCxtvLK>8Ydd+T=YBv@iT!Dq0jfK+s3l$PH-HZv<*ohyG9(jS)>r0VyH z;H}5B$b+0Amu6qXTq4YOnrEWk2fiml~Y^I6P>cDdF4`-~Hbsj`Hf$qJQmM{R6F z>M@mw$j;^fVGBhQICld+*R?kshoYGjVqn9;h)dLFr=|eD6g8<36LK2&M^bBB@_njl z1S7Y~p!lkts);gH6-@Rr$Xqrq-OP)J`qdYG4UuvnSD?izrE{Ow(Tho>H$}p29L-8f69B$&w$N zwhS^v+niW$6L+U}`wohnC+=!z;7mez_eIUTRHTh3?Mb2o{JS{Jl^ASqI9#uCZJ_f} z*oNxWgTmw(JN3<`Y65u5=(k9~9vFxup<3xZfJa%&z4??fsZu4F6&bx`Rp_GSEWZIQ z9`x|a-u-_G|0O-3FB6ZD=ooj|HE>)SX|6M-$CkryxW;z%X6p)7Lh}AEWSw?}2iDwL zP(=p|is-oUDZqnC@`vk{jMFET43Tsd$rr}dA82!TxmawV-P!cT37D;ICTAgzcO)@m zWlz~`3A3g(i0{EWqzFx4U&?KL#_9DMoH*d94k7JHDgL+N^(S_y;hVQ^Z9n9(pyHt< z4!c%ccXIez^8ikwHLzNeIa0%@gYHW~Gu%Ghuw8JK+qWbEOIfPg0mx%ca;Inm$Y+*c z0+X24;2kg4(N=#bTy0+Uen0%D|Bw!$EW;`f`#E#3<1kZZa#dL!7z?c{`?aeShtpVC zNh8_1c4$7(8QUD}D9L5gS8g3DNoAE8@{v8XMO^(bb00CXFa|4QIsw&twFbmyPw92!7iAu6I5PSju3O7eZaq^H0rNq!9zzj__*;A-8 zAe^PzV|{VA2!;>=4$_rcQ_6j}m!ksADST06T0>JnaRq*wE}MDnVE-)Td>5p?lFH4n zHh@*w;maa&KM4Oz`rID~U_6jvQAA)FwJGXwn;pIw%!c9)_!Nu1C0>_Kd(;_7F)4EZ zOR>tHnI{svd|Wg{dGql1v}`goQ-8HPSCS_J>BA#kDmPeubcAokz%)t@ll5t!=t9!V z0#ZFI{iuywTIfo!sv()46q1z{OK4#UQ!miROXj$8s6$8D(@z5p z*f5B4bs;Hapj{lS*j0)pNFPSh0D136@CJ%%;zPm)IL^NJmykxJamxs2>TQNUq;D zz$pt5<9qU43gufB9j+zgGkx^-Ema*J+H|*$!6Q#`)=hG%%sSOhf-MQ)I%#d+IhYPx z!PROsxEf9t#!D0yZNRr3U^M8H)NrIt=zvgrLU3mL7U}Ok%H3v2*8Vx|k#?*6IACbbMn|2EbsAOFs z3tXN62I`cTfiV-7htKur;bE;T|AD(Wyvbmk;{ z4$gJUkOB!aD`MLq@pEIRYDJ&ieIojB2NGlIYb4^%ylODE!mPp|rlH&~z#O5u$BzTVycS!bk&g=}qju$?fIV3svI}9LCFO3oHnzY@%xk4Q zzrjnvQZISJ-djulRCaik=CE=AUN1UQb+Ki6@^zPVvq~ONOIKcstag+{Rqg(s*U$iG znrTzB`bKe6%zN0}GD4q2pCc^y8&jI zdM*-`-9CLtPQEURk-T3};=$d`YJe8js?p_aD1Nq~SLCvF8&q!Xf#II&vl&)9?dM`h zn_7=0I^0Rtx!oawPjse=ipzx#|KWu*M-+p$x5G7Fno#Z94{Zt-%yNtD1d0EFAn|YM z9h7WaRD&214t+!*qGeNXBms#f%Vsp02^3D6r!G6I^^jUnW=v1iH@kkfz5vBUTh1NK zoGtluhr~+p!MuAr!1Dc3CZs&^8t`%v~Co(iTs%BOWNF=+V~PfnN*`nxTH++rpw zC#VcE%Mu`Or-7}V`@}C`r9P#5=$)0qI*bYgeL05${1eGFoL>yFVujS?oT_)_?Otya zBvDUx*XY7QM(ZtBa~i-o!#_PXOH( z323dO(f~-41n^GX5olv(5RE)}S_u(`Dscll`^CWz3m}Y{7b2C`y6(f^vqA_lr%(Fv z+sEPcPnvoJbL4uyh003e0ExYmIs^4kwJSsK3i$mIJ=;@C3f{B|i106i?TPCG!H6tZ zd{x+}y+l)*iR%?*kLI0z*U|hhxw@gFu`PR^+xwMUhq1zh`%jo~e!@FF^YroB7T{*JVUX&`?Dp7xew(wm_RpTvp^kYgsYavNCzVT9FDFuC`(>R9W;YkOq9PIVS2{G&?0rwgjm!3Ml~7 zS(Gd{!3TFUpaovnv^1BBJ&Vu`hr35!$xEz@7icvSS$z*aijjIy-jMbRuX4!Osp?Mn zT=Tk;q$#A#Z+539wE@r#^}&_UkCV^VwfdhNf@!MDMbQY_Y|f} zsVxqEq?(l-eYM^#9HoN;dvK^*7+2nmC4)`rdgHKt16Ngvy%c)d`ot&8;jQv2NXa*! zn7uY$@Xz$OpS}H&;Bl#?C6~+iCOJG;$A;||9)Sd+en<2HvCzuc=CmiFe$07dZCfz`?qY zdq7d{tt8HfqA-E8_F!MU93OZm7aigutlRBFKTU0*`H9LdrgvWK!FZP~ga$?rLvFo{ zT9`SD+(?gtmqCqgy`4fBch)LGH~vpYv|N(1QikLsk~{PaQZKCisK0g@i`G&N47yR;r?s1A10J=Z zHc8$|B6kVDS4=x;rJ>d}&WkO^o4rXrAdH3Lxa! zZ@yfWr*K00{7MQr(;f@o{YLt)RHW7n@Gt?F%F}|?X$o+IZ=l_laN=|3|u1fBpIgsF^^}3t%TsiXh#N zRchEL1(*qKV=Nl$VKuOjbXcBt?Ff}6`?R8gyG?itwv4la_|s=cf!sBW5)v^cIcdwH zW)Pk)sShj50tDT(EmX|5yE*Yd3HtUsoii;UffTvZXjAnSf5rV1fpViPb$YFXseP<^y9p6#nASdNeYGh)_TX(h2%#&(*v_7J$dWg(B3|;9_6K4Fxb!{-|bQVx?g=CiGfM(77P*W%FOB8G- z2Qm#9bO2()ykhNvF6Piy>Om{QqBO9#%ZY(0Mi2qNgx55kfC^Ppf>h zElI5F4wKem;VdTxrd>d__1-u#CWHQ>dpB+%P+u`cBKc49`jvyPZHchd9VhE1Y+A2P zIR!gpEAPESdfwrklOF@%GBj)-1|C*%-xUJTXb1uv3c@oQ$7A76lJa&`ciGO5%C>^4 zi!vFN1X}gB6)$i10~(2j)`8ze z4c^!^yZH&;F!8@nAnLsUf^z|Iowa*=0?^9wdXd<)$bqm=BtLVW6|Illk!NzuLM4OF z(|FgT3@E#FZz|z*OFxlQUhveK&rb~xOD()wL2JTlud8r23-LA)GMWJheEgSOBWlSnCmWu z+<`wxv$zIF@S-OKb{ui4*LcxgZfW4yL3H#qzd2B%ll1i{=xDG32Xt=yt$S&540Nbp zVO0oZfGe~c;3+)=Ib|=Gh>m9H*|dxe>6Q!VKmN0l)k|jLqB!Oly^Z~DSYDqX`-T%^j-eu7T7h1XIUwmT*o=p zPNB)NQ*lvPu9galO8Xp5hpZ$7qa-&cfXOQGFuN(rzy*<@zGez%nl~^yQ-5~>WH3oK z3Y$>O7(__Uy;^{?VNg_RPS!#Hw4X}*aoq^GhM_MT&{kF+`LjkMDE77~swxZ}yVF<{ zD1J&d&~2qkKFX!cy?|28awxfBDt&rVCb2&nmqy~tkZ!;4Gl%-bHag6;xtW){8lt1K zoW8R;5D^&@8}1&m2W7Ybo|S(6t$AcdigGp!wMeSNcoqbeIN~6~sNVHHve2PF1)U(# z9|nfGa-)?2D!4gWp3jmy>{%vE`(n@!gLdrAxQ4GLL?tg&8?(SHq;o`9s~M5o1jy6C z9R%MpCDE!4(%?C=F|PDpT|3DIwO?V*nnmLfT4>$YZnk)Zp96Szhy*O$x^+#|aDMcA zAcZ{%UR%pXQlO>HKIO3dq!?2P!h^xcXUxqnMGd|uuL?V&L3R|R8*@o(N zn8Bnf#7Q|ZSW80QjNNwe8Ir!4y%$xA%KlR2>Y! zrp|Hzv@J{tXN(fQ_6=+*ul*%jPFZ6S?0FTA0?JWdaC-ctB6RZ4{dAzZ)mz#Q<_~`G zgYd&2`U1Ru8s1XH31OJv`4pW^Os;8<>?njrJs zE|^vQvt(B3h{&#WnOX^8Hrs9cvw<_}m)gB8cd%)?yZ5D|nr>Lj`x?r$gi$skg|Ae* z+>U+$?90J&7R{NQgH@`zLm^4I>&t1OEpH^Kaovs{3+1=sl7&W19>0RdCdAu;43k%c zzTUd!=H^}v*rx+itEtgco_c%RcASJo4S{c)8^uBIlM+`8tfmZn?Ylw2#1QMSXwWM! z0pbsaH(Bx5_EAggpqOpy>zC=Pti$&DlMg~fju7EB5t$#MUegc07m9?1y3r=lradH% zPKIlirPAtxqzuq^Qf~&L5AcDvFQV;c`9zpmkw>A>yewgSlPu1hizj*BLnSBaSez(% zEh+~nLg>liinLwFgd;)jR4&s+rl>5jb*B-2(PN4=_c`lALC&+Qtx=d7+Z8i(EFNXI ze_|2)h()VXs|x1LU7XbOP5Gs!ACL!ORi^}_^u@bfdG)qP>xq^=OX*cd$8|0!uA`;l zI^@_$Dwx&^T1Ob9*j2XZ#Ct|!)-@!wN)m0@Dg3AmFp~_-X?lR3Z$QV|CO*alKr*_3 zl z1*Bd+rfcXAVby3#rv`G~lfNOyR)vD}W#Iz|Nhd8J?D=;l=|6Y&M3>16w6H30F)cS- zp|S~^gzOB9$Mix`3u=UFS0gM%0y<=f|PzkL7m zx6j@_lGnd@`{Vnchqqsv`63j7*5c+g9ISP8 zAKZugK|TuNNh&Sf*aG8$+Eor9&si{8e*Y=-4gSDLM-h@Ox|7$U4C0Q-+`AoI21Ty- zUg=l6b8}`bO3(IHiY_fWJb6hzRq%ZAkv5{;QnQ{cA!9)w$NI?p%NywZa$C18eR?GE zY*lsox&g$zMo0HnJw-Kfl1)rTao%>J){x%vEA{o0*PnVj*NkR9diz$kO|C%Se-Yk( zqfHPZs?nfMD}`$%+w#h1U3R!PgHCW_6X>9KAg2ZUydSK1__3g@ z;GDGGj>}LrKuEHqyA-|S^`De`FgB`M-CZPff(R8S7&RGcrQdP1m4&`oqK6Q8sP!I7k&i$UpzVCm$T{2BuF}5hGNR;XrD!$nuUOpv z3T^)eA_khY0zF9LO0USDbu7Jvb+8Mn3n;kW%okGO?dyEFlD|CT-2U_>amp95Us35qAI=i?4 zs9Q<0kQ~jJw#qOZ2pAbN9k&LN+<$bP^bRV!y&>Jiuzcqvw@_EoP8Ek6vUieZg{83{ zz5N=HmzmHss3jQWC>+ownlKMiEIKp|V|Un{D7KDQN+x1Febzp6m6d)HNk&mHA1vA)Ah zSy8yr>=*S6qC-)zO)AN=Bwd%=qE;I7>d&um`1PT-W{|znGOJG}VChq;ek^CmKAe+V z%^cr<5dL?QlKA`Z_Kg&c9XMcCO|2|5Hw8OenLS~q>$w8R*ZaDuZbS#RdW(T9;gFJt zRRNmbH7|%w4Q1FSk#9g|*V`tNhwSEcJ}oLj;{#yoHp=ape)bfecG>a3o3xXgjBi|W z^(JE-{!GeeH>#pmdR>Filn#Ik%*roG&8-^#nR3hNsjx-P{z7^5hWCDo9 zKNA!#8pJ}PQ(B4jY7#S|bY2`auX%{k0_pp8L<}T>lv8f=?#OvyqG25%iFbdJtUBm5S?b4FtyDxMN) zikMLFN2m1o(5tzbQ(rkf19uf9-DYJbxn|43y`nk5ncPlRcl$(j7Dyby>f{8F>|%C3 zpMnoFwuL|_z179N3bXxs!I1WUT)au+k&XhRWgWwX?qK7v4- zf(IOpfo5`zZOj059LmHFax?bX(F3H917@a)w5`dQ4U$7wr$>3+6%1w*6oBctN>Zow z{jR{Byz|GxgDDueU0N>O8auMv!Hz=cID7oQv1Z?X_rXxBJ^8aeRHqS`kPLVLl%P>5R5AaM7xz&i29v+85UK)HH;6rYa`rjh#c0mTB(pe#)DuWZxD{B z`d~ZnMnj96dw(^UCyd%bT5rh?$K7H;MuwXdvQWDkn zOQy!fpcD}hbN5TFMg&&YAmlGKhTXxnHgN(D1Oyg83M88Voca_W?-x?jB^|{ViM+*f zIg|^Gf&4=rh|6b$4*3Uc{~!Dq*~~p;IrZ1DtzzOJE9; zX6n6a-MdXY_IC2avmu^T>|+GF!(mf zBZ}=Z>0>PE*#lXlmh2u-G~})XZDxtj7pYZUl11`NdGGmjrD3ls8Et&ZyFJ>4vTVIe zBWvR+T#IEXT!$iE64qmp=S?o%iqj*Hq%E9Yx2X4aS-cc*169Q?rp435Yob;+NK-;l zFp#~3auJy_=zvo=$EJL)WoMlz47iFC!WWQT@Gz9UL zRB$b6`*)6@Zt5|4aum9T6ABAGz{J9p1QFMg)NhRJ7_giSSS zxGkjcBUcls?>Z(bQQ_#OH0YK)i`At*jDHMFT9F4T5l&WW*`XU{;DN1`67hM-U@9bKWYO%dj?p~>9brwqUU3bVo9bbv{nujdaG0&43Js8g`9x1{*@c~|XnFIm zK~b60Q*;|Dsj?&p8X)C5JtJXLFifv+=xJ&qU7_c%imsgaT|-zygB{1>I}=ZQtGCs|2?wEzs`^A>$k7Y$u1~zZ(oM@pGk`I&HJCq zUw@tC6TS)mN&kZbpBid)>HQrY=NXSWlxer3Vt`XKaeJF~s8U}j}#d2u@=Zcx*B>O#~J)j7-Zc>1P zWw}$zC7d|K1ldAI8WsRlP2P~qooK_wI5?|Bsu3s*$(^%i;t&Y# zR6(3TTao$@|AK>;!C0kz#1GzRbyiN6-nVYP)`DaFVxPP=t);{x)u{o}QRiY*kI)0+ zx11OiW!IqmaE;Y$LZJQ38mHcPSw*?x*04n~;V8Dh<&aG-96!Y1s;HG@I~*iFBnQ1^ z>i&}s_#42NR9bc{UbM}}UI;WT;K|!EvZ*6HKO=HC7hsVv++CERCcoJgvq^;jkj)ny zb&WU8OH{he-ZhSM3UkYkUrw|VQ0$jI>;pO#FvNsov{g0kGYj1eg9^?E)WMh|gZLad zi0ycU=R%WBZ`CE!7VBKx4m9D=P;Uw<@OJ}>32ar{?hrG{C{NY>&>1~5@P(S&3jbhE z2;xe52Pz;sCtr-mDU(f0C~sdDS(1MR#g&FRufkx9rNjA-%!l`1z5VI!pI$!?=|yVY z{}vs#-@cjm9kp||dmh&!?UT0#e?a(F?F)1j&NK@#M)OTH3jDnWBTU$rJGMNp*7?j@7qM^m-gW9!JCcp4t3=&QaTW)ZHI7*uV zo#Yps%|EP6Ry0>B8`icL=r`#JI!>qB@2WLxbq(X5nIyVhcOVM>#42NH$!3~Vps3LO& zUTFtp^& zv-PM$de_jGAYMQVrD~Ze!#^w;2&A1G2-Z0Un1`Z*R(#VEue$|LDE;DzOG*wH0ih=A zQA=751Jc_85v=sh2T*cPWGp@Cmo|dlKpEaqX?~nrspE}8s%@k+DU{B~DRt{VP#*c> z!y>7kxBPCPz)es=v6NC-0Q3~mT-EWff%4Gzsih2|&F7}wCIdNkD1Fgd_Eo9kS=#@7LY){eR4+2aJ1tO$s-fFhT;)y1%MZg}|CL>n zrrG|3{N+EseZ)TlUs%vni_x^M=y1A%-ow)^B-@&yFhSj1CviP0l{Y4wWrd`G&k1Mn zm$LuO+_{pY0);$;A}QgWH_6_=x)pHrLR?{!Ht~r#&`8qFGVV!M>DksA&9`fYg~x2z zS8l|m=*T8d834HL2s<9T!TEOZTbeC6%gft@Zb$_7I?poH;K+ahMYg@)=PZMVsjU=f zW^J|TJeo{TLnT8TjM9LEI$l=XBtwO0Mk)!cRoSTNBk-U7<|c zX3It8`vB5`m~AYlgYzrcVR%!u=*#Ta_Py{Q7>xhumWw}LrQBzPXO0 zJe8(J(H)hzV`HwuhJQvO8LKG(B$6*0#b>(3ZpQ!Sw_yz2B0QB#K3=73Y2m zelQP-&omTUQ6p^d;Ai(JW-| zyVVXF>+l@q?v7+f)HwuHJ9xC7PN1J{pP*`ckntohcu{QtkY&d00}$C)sylSh5a9uM ztt2dwJ!E;u53Asy7PavAsD)u#K%uj!e7?|UCI=Z`(B&32o*Xh1?m zO8s>;%*MXDa<6O~VhS(;HE3Cs9JlmQqPD;a6*pNTIk0I7FoU)8x2T(>{gY8meJ~?B zPFfvOsjlwxGASh1f^?zNKu~w3UYeK@p$NZ9CY6^?&H-t5c=Uvyp%y74Ycy2^l(Sz@ zI8`MsF7N+ac>5eWwYAe@y6uG%gnNh1X?DlB!-85;Df@kD!lQj0KV&EqS8u#@p@D6e zbzX`Aofj&`;ch0?M6T@gBi_jtFtFHo4_0l#jAT<_1+O?{q9d`|TW)fAUsK04fQonNpgVkonE%r-P^Cnjl zz(^Nr3`!EXWf(N=ejXMkB*q)*Dk_f#v&k#ey|RpPRXnoe1O$2Ws(ovr_**i|zJ2qd z>$k6pMEv&k=eB|v;-{AwZLgYH=**ecP6S{pL9(UV$1FKfJ3%#yW)d@q>LKoAC_YsC z(?{;R^(vFaCRj-G=Eno0wiRtSxd#%zfCBXmgiMTchcgAgG>pU+rzi5SF(nIUO(*CO zZf@;a?C6sBNywT>UNO)8;Z~T1bEAHM>yJL;_pUx9RzDTPz!rtPgI+>O*6x6<;aC&+ zqPX@c)AsiU&VGQdOx5N>%9V9RWlO#=1A9l@-W)+sgpUtMDrBkY^DHHm|n?Mgn- zeVqU!P2zu!o>%33$1ZG1rYC)I4K;D3*htWvLVO=c&$@=PG$`qEA{)UMIwz;_Ov((Q znThJCIu|^+6SS07%~5VgrW3~J*4N6sbe)9{R;*PMXd;PrP<%;m1lMyYj!5)y720J= z(yF8*DEAE=U|X=4Gq1whc4-Y%ikKe7_-AI2C@V&mhzpzvLEA;m7;4xhdPNwCnJ@#V&Duq|M%=g>wYTCd`*PwqnNFwPH z%cYrqXrnTas|V>q=1{@Ik> zpzD$7F1Y%B8D2j|(o{hDcex;ZBs{Oxvf59(P83%C&;dg6%JRD@^1s7#0XiBK7{g?J z$srFaths85YUn^6zr8GzTaYEC2uj;ILxsfq@__^RtRBB^&4xM(V(`2G)IOOa4iF!e z%29Ug&2Sv5!GL;Sr&_EM@M+T{z}+(0EgJ52HSXHdkc+A2nUMdiC&r?QNgz>g${+k7 zjX3`@{Lr?ovCE(YYB;++lx$1h!0FRuXyVrB6Zu6u`=t~Rx3(z*V9&Flx>fS@aU5U@ISr)I;5E$d5d#Tz1E1ADC-V2^yn1ACaJ_ zvK}^wZWpW!4l>4f&IwReciZL@`bP%IE-Q%q_L4vJ@+VWv#Szwn3eDbjRoJog=*aUgrT#(Cg?wwg9vgiYaM3lMR!Tv6(9w81kfe+FCupOYYhHqhi{Veg}l zow>_yop1cUvWax7%`ctdTA-K3WH+eklJYZ-UD`sx;cI8u?r=%D9}p^uOPAV3b`XH| zrIBsVI8&3?h)JHd_o_n;f)+x*bQ{sr34N^%XmoZ2|2OsBe*GxP=bEtOvpfn6Dpyvx zF}iRx2HU3cY|#bh;?}v+mi!%1Ah&J<&M$pa64aEVPqM2}wUibrYz#g@&={Pmo>Xl4 zZ*6GN%eU7XQRHYwxRRd)wPkU^V6<9#kj+VQE(`Zr!G7AbA>8EVuVjgPc(RmY{*YiH z*-HD6^Pu^ReiO1*bL=i^2=!#NH zPfgK?x0jNM%n|%BM|y+#ToJ@kJefYiV+vH+2XnoL3&7S|6F{;GB8d;(FxzAco!Eu+ z&Yo3AHoa^MmxaM+H!Y-nT|p5bW2*?}6oW1dH4NtYKx;L^76IcnHuM@f+B>O%z(!9J z8)kNz8y~dFv%l~xt*?4UTs!#vs7-BnV0x4V0Ep-g*Y zLk)JjsC+coj|4C3imorR{o0-UyNuYbk_Voh9hm<>`6BbdfzW9d*W+gyY ziVf>Zig3Axh6;@!b1n3CeG*QKk1GfJE!18$A=0w4ykNX{+GQ^G%_3{=CC^9&0RI)ET!U5>m! z)Ah;`_|=KOe-nQ2{ruFwefyf4QQ^Da`0m%~UqRFL%Pcen3lms2oe+7jrH`keFl1CD zbRrUkEq&RgK#m*MoNwfkg?*DkGKyZd`$ii=g^s-22^>Pbw=Jy5C1oq+Okt)-`$-Lc z?y)gJc~+@sT{gg|AvI({^aqlWkSyMoRO)cxwzlp&De1E0-M#jS?;I$phfe4crxyuV195n%b>EkhnJejk&FK0LiQPLQw_BCAa;ZwF zg{MO#>vb!%R&3%p?du$;IXS3XTTxPP3w$8f518PKBHky(>AC7aCPdTQ~x9aIiOzX&}#D~dV^Q2Q376tImA+dJOl?1Z&G z$5i>aO-i9=?`Rdjub_nzWB|C1a>@&buQy?YC>(|g<{$&h4Z4MR%}&W|nN>Ix zPUtJ6$QBbH-)>RK2ro~Q`qkTAx-2J2@)m+hmqs$ri;C`F6eXlMYKB$<&OV9U2oPbNphWu zz2~oRP%_EL(LnDolc@h8X=Jf@Mnq<89ZTko`yk^Gng`j8WWzU(^g^OHERBuWx|;xL zcI)DQ>3qk}cRUIdVTP`}x2kSsdbl4y%eNeJoe?w1=E4myAkb6own-#*_noR{r7vOB zbHTq<*~8}wr=IhG-T~yvbR-H*E zXQLKJCasIMN*xD{27%;@7$U~wvL-S3!ljU->zc@!wo&#Cw#j!}*YU*S!dq-85a^ky zN^M%V=LQTCcZg&kYdq8MAjveh9$NtjF?1fL{*&CP{p4TP7RRNCh`tA_-tFD1LPeK| zbE%5tdWT)}j3fCiA{yQmh< zA9HiL(Plq%tJ^N%yc|U2)D7UjF1oI zED!E{bepN=j;B-z2A?*mk1ctbHMEG)5Vlp5V-c4}fnyOKCXnA!fk4V{y?^57OheEg z(YBoamb&?>7VW$cPY6Z)g5uSv7SNyIq-%^&|Kc;(YpB*N(Z z=vCH!0nCDU5tWEODvYTn27L5VZAd%s;|ti+a(>5Y7>#;{=^SndHS9@&qf_(B`!h5~ z#cAEEP_LIt8Mv#;QCS|OZr*aEqqkO}D>1>`sktPR@(EyT6!##hEjS3Ud(ReKkaAFI z%ZnTueONulu&b$>DPVfW)D5_W#^=+^+1YCe?c=gePyOfg;c}!CjGYGKcJOodS9#Qu zJ)tv}o^}gNmK+#4Hof)a?@AgP`M5ELY}H#}uN+fVOU(!2LJJyD$-~DtXW5E}tvrd- z6mq|&Fr~_V*WQuG>oLfVbFB?I^oGw_ryirxVx*J|glG1M?OL*QgJ&2GrxtnM;YB&C zay49#QI2)*!wcZg2<(*H)xEu&5%GkIB|z0%GONNqHVzzA6jv4!JWfg*0sg(P8|P!` z7D%d@QgjW9YHPo4VYE)D@@ahg;~i5KS#9EU5JeY8iKkBB9wCG2nlEaoLvY!@mH!3& zN53-jfN$tJ`OTmHdiqmu28a9=F*pg=^c&)uzPWt(yYTky<-^}d(guCo$$&qI? zfIUO7%{5!0lukIz>gvm+644`|gV*b2RFx0yO3Ef_;iIOih9hgkgzlN0AD}j0vA63o zx@jFgMjy3oSf`5Z^155){hi$0kbouQ(-r%thQDc#^dFZOuM+6>hV1U)SoHO&yd-OG zgUo3CEtrbu!5DNA=*+dMAU{MZNnOsPfw}057$($NQVl=TUjQy8)YJ(|#?L_Zca&bP zFpj9X>(U05OB9CyCSnV00(KFfjSsu}6B zzwT^PBFHw?CIoCaXJS z0*jbQ%O(6|efW>oIHUsyBi)Agug?lIDQ|_55}SswFKg9A@0Omc3Mi~yMq^1$8C=?X zQGVNwiDcGw9+jYB*Vd5*Qtz_KYc6p^uKu-yS;l}~yFF7tPNlNNcAXU|&JT+EobuP% z0T_NPhlPtj{`Ox|9(z5^@ntGOUq~;_}!>)$X?rUP7)#DYna~iiv8X>u^e3BS4Peksh zeUK;hlKT(Sk_91D-pLBQK2EaG_31{sBW?54L5i1sA+`sqkbD`JeBU*y5Kd6~i}~xi zCwuK)W_KHu|3&!T<-^}&K@K?;cgZpj!b1=DZ(RX@>;^~CS%RbO-}H?N6L9;B)2yN} zbTt^Ll7vpGpC#xg6DKr|61ux;ts`|iC7+qII{Fn!YvY}{J!l>%MZC6twTK=8-0>Fr zoN?OM?9j=8J1(Z+`ImC z#?U4J)Bz<(Bz6muS{^8-Q!5}!Ovx=X1rd^+RBT&#WJTZiZ>cBaz2THs{EB(SFQt-u zk&9)c09V4SF6XgdwZL~<{^NI;W&<5vH2^o;Xn_LVyz2ziMlv@xJ}lZdMVCZqz4s+S z!Rv>cGwO%jTDcaWL`(3tbvPue>`Awb{RFQ{ayV-hH>5ZjE~Ar59;685vW;5AsY|;y zJ~K7FRqm-R9V&uhV_UY*hA#Vbwb|7MSdnA~EBcD~+r-Ih#XC9NphdpZJIRNGYdK-i z1IutOcxl0o#Bn~!AhcpjQf`Wo*cXS{K{?%BLE$#yPkG zlS*=N_)#gaj#^w*8wVE|t96LBv1qSutR5v>U6{ueI`1M$SKi!iID1<=WoH_;O|?5? zcu^f6i?w%*nRUQn@Q-CL(SquW&R{;lKEp6^g@%Ht{bVD42l^SMqkZyW2Eg1wJYDbn zmPR(I@vxh?;crct_cSMyELcqE@+(;Ci{oZ6pm99wnW^RXMGdF@mTokQpRB{6gZ@Of z!&+x`lxMg@LIlk-hbPPu&gQh)_(VOIL(MsGj@+x=M0yEIy4A(zzM=)^0V;-;M}Xl#ZA|nOxYUD*nkHjfsK^kNpcP;fT0b5e8C+yIm!a52)n9 zSQJ!bJ|ldo$_E=rY?6P_#jxn8C2Ga~=O*?;lHS*8wO|2l+>n{!s%7_{eiFi3rDtM`&5kKuM*$w4fRlZKcsS-7zHIz-c^0jFboif z0;({2_EKS56<-yy;l|7fh?73S!*T^ln-56J+0sd#3*V$+uiVq#j;SirsMseZYGkv! zbRM$GDnBS~eVIHKn!Z;hG0?kBg=y&5CIx{+U%k>7CRPZ27ZA7!Uj>afy#Gi^@Ksk( zl3+^g9>Zj&TwnsMF_3dMRH^n9U1msYI_SW-FQ_{ zpdFzr$ayUHaGvgj0VOWl3qOLF!&N9x8in6NS@dj%=(Z(rsHKbOss>wNOmzgN&sirO zs)H<;M)#v1ioPp%#15vxY8-rv9D7$z_qgjgO@ZI{LOXp(T8fI{QAPK(d6hmh5X8JS zR9P{-#t_2hec%A55BayyI{Gi~KSbqS6x2?KEH`B3+1xy|_ml25)jM#nw&=+%X*$_CYE-o~ z>s(^!N)uSPd~J*5;B7VlmiV=p3Zg)L%>f$Mt2EbcBIw;nY zdE6pP^^sa!Q7Srw8bWHk2VEI^!C%wwe)|3;RA}^<^qs$<6+Conpp&Um+{*JSdX@J> z$!-87ltayQr%)?&a?;a~<(ZV1#|gvWRtCW58bkD?v2~L@c#jAlYiF`cjoOe~e*?^q zVy}scfF+`Y3;@Jd6p0>hYoInq+9=fki*n7O9&1^-{3$x|VcSaDz~P7-b9pwZWgx-y zBV^rkV*>G%f-;@v2VfOTtR)hoo9}c-P(pgj@2?QV7>JO7I!6tSDs?Glo;9oHc-33^ zHpvq=Wi?ur%4NFLOo`2xF})78DYgLe4)LaDz%@$kVkG#3REKh|eb$UWmajql9W9b; zSF^h+WUhs_dRnPh_-U5U=sm300>|wM=>mBQSHt;$65A_L+s7Bm%u6p(8srE(J9b*O zfrQ^u?lK1Vb!4Bq3=Djm?5VjAgnJ?m4Z5Ak6zcbvT@GMHLeX7PFUnTmh_3-vF~@y4 ztAI{8x$Q{}Vpe&`-sz{fx~-uijcJllL;I4NPV_=4@Rs;I!PJd9hI{mdDVuu_L5+jJ zxR)!gX90!g89b6Cc%jzUKDo6Hp_#@Z@c>0oA9BFm#{!HL)8R#e_Je?3vc9nMC4``i zb5+p9__VWxF}op+tKE|cMH;{VL*R>aCVmjoFG=$~T893VyW4REik4Xj=amb|KK3Nn zUH58FH+DBqv(|yJPVLZuz&I&Q_mQdhy~V-36?~o~FH4QSO_^yM2Q9PHheNT!IFAI7 zmQN7O0tZSF%u=|_c=VuNI zWwQ!)sPHogSMQZ9q-DRF$}6ljtU(`p92L7Av3%w5V>cvv|(sL&V+2R zow1>Fc3F3gs>%!VE|N3Q+>ns1wtVn2CY*rc$&l=J0frJv&Q=CKr{27y)7PUq!m!y@ z^zz1gMQl)?fs^B;77svPsp7n3g~RNb5o0KWsyX^}GC^_9a@|+DEp3|;qZ#Rj$vXu) z)YE4P(P3_aChHijtxQnJ$E^1bu^yaUS9rJ)VM5x~)xs9*}aA=-+!*K}^hpjSE4Y*t@ zukIxR23;-pYboEmTlRRY-UNFdWM4n!VZH|HZajdla2KqHszP$2wU5^>ISU1nWb95o z->4ubp-4kC-gZ1Kq9MRL3VP?-BR~mwf;ErGwh&;fBXum)pC8m0_M*v{F$$rCZ5rKx zYtsz7bG>&ticetT*eJM4=fg~3ZXzXWCXOE^ z@5GId2A|7d_b{tE*Eoh!f(THj1k z5#ou_PAcxX1k~OS1wo=~GbgHDC!YiEE4+w$nWGQ&~;xebH*g5J09% z_*)yABp`z%PM^Ph8j@Mvnk84)q;Jhn0u+PcfFyS*fum#}V3v3*r3S!EFh!L>mA*YG zf{Fs!2t{%sfQ)3Rp^8XbVtRwgmc5)@l=~)s{2)ZnVja}=)w_d45P8IxMAWwGDjD@T zyu}VtSypJ{OhNtI?IcN6r^4(U1>g?!<7%t+b4odcKmA6Ek7^K2WHWsJ+Gx^BsJ$ea zQ)HCn2qmFjt2*zx6BBw7HdvFItTJKaWe*e$mmKnnkwOR6$MoBy#r1{w=dqGa|FoCE zwKB`QwYEA~fV<=?NDqXxNdT5xhLdoeHpI&n@{^e^R7DIZ`_#^jH14TdxPD7Pysi$z zP-*rs%`Ng391#1=7*lPCd9aM6Y`5IHXS+>uBz+Rrng&MVamstLX)bRH|R%?7-MaOK-f<}d%{sQc~9_kRp0 zE8CA`3cvc}UjfPSl#7rWO)9Y=(FN711!&n1km|Lh>X3A52da}1H_WlNCp4iFJl!}@ z`T&+f;vnTK(LwJ!wJ+3G+el>1OVs?0MGL?l%()~{cyjI+VQRW|>L~5w!yN&J{Et|> zJe5`~3`ixCBdbRNst=>h=d|zp_61-6x(wq3|>~Ne3NAi)g)D5^x?ktB@6U~pz_x6n}O=2n#reE=v94o!AzdG zJ9)8GS`!uuVOtqJvqze3&R=nt{>lyvt4$~qJlL7r6of>+l3s=U3Lva2EDjv{u(Wh! zTpS4qqccl_wVp{Hs)6$Hu#PfW6$sEQfbKC@k0S)C2*s>{6sio+UF+Mj!v}tzL32~U za10V`*1x?x_#ii@sBhrJ-4z;yE*u#YFs*i@(-{$@AuY5Z@l>vx(UKS05vX45>~ty% zZ&zsFmD|Zj8|rIns7MvZA+xo1WcXxcoF-mLpGdA4*Q=EDb04>8!9cLAD*Gu=yFUQ^F*-3 zdaapS?#=cfwbhQSR7r~GXN|bX@)bFo27%Mp9RTzid?jd6VR~k7A8W3k&py~<9XM04 zbR7Q!$P?RekbRj5NdqFOxTrzIq2#}wi;7Xr22P0*`q$Mn-no-_<)TVefrXgN8cv6& zEq&L(t(f@N5M6dTn=ngUlXH+u@b=fL-%DkrI?Wb(fPv_?dVyj>@7(2r)Rc&iERo>^ z+*L?cHq`^L*3wEqVKRRMJak({%@f&K_>6S$&lu&`Opbk(zh>jVrPzn!L{eR!(2_|T zEqw!Jx!v;+K**9VaO?pyoYXVa`%h|H6~QGRRZJFL%5Fs`r&5$6)xzaoYh%F(+s2BNAEVnaq#u6HisO z&u0Ed#jrTHFffr?k|e)r@=_LcG28u;YczY&#E+fjN1(}k3fz_rB(gaw#7UA$6@YEx8nMAHb9<~P5hU}I=+wD!z;pEaV3aDy zUdcJPod~=o-5fC_>i{%4fY0Y7tb3J3hR7hX?Jxy7s)5lAz@<*y3rsSxPXuS zW9pp;Whgnmhnx`lh^wz1l@R2ZFXW#Tbpou|W-4?LGqMu~5vYhrJ2jd*Odc+edBV&aQDmzx8vASV=Lh%DTgKta_Ii z+{0AjBcz7?d`Un+-$pZ4V9(9zJ=H!?(f@<=?d^NmbG`lR?!b?F1iL` zYrFt8?kTt2c^E}r?vn%mZWz#6@2IF`Wi{19&CHZa;#ZloJeWzlyLC(Xt*c%JcHYs~ zByp^o#v4*1OM;M~^+vS*E~hJYDkskhLwpYT+W zl#zcV4|T%6Q9%+oFiK@uneqcJ_;1s9)i>i;=g|56KYpKu36el_e~Lr^U=Tjd$)$O^ zOmo)Zh0_K-gdyw)WS<>doM3IzHsHZHtz=QV9`i$0$9QZas6qCUp~?MFhx%}It7;@T zSh&(;(}^uA%r=}p>HE*apYgWql?S5sejT5$<5mWGd-W5XDhdLOnmj+eOT-y~Fi1ya zXGpv8eg)GqxcyRdIi$2PN`_R-~^ zx@-U~J44P1U$G25u|tZzND=rDUlJKcDscle4^dQc(Rkz0o{?%b5$zI|)AdiunZ zw>)qhEIo#ERA3D#r4C23U_VN-Q{Ar<^Q>6~ZWIyZnZ0S$A1#Hyr&3uM zIcXTChmmETEc=E+K6Bd<6#)7R%4^y~xqmipB{jb_8MR3zR~vs4X#FiEn|_1^N5Oc)^i_ zG;fq20)OcU2?}D=y9M@~2*DKBth?o7whVyC3+I1jMm7MwAW4Z~*~;krfdVAT`XbRl z4hU-OQMIzIg(b;%a*$3N{t;4dCtF`wl8wB@^XN4dqg$6dO^vJ1VBVZw%B!?EmhW2u zntDoh^Idy*XkRx~i)?Bo&G_)Lt4@|Z_ij~Y2Q!9s8b-)cWfU7G_hv5lqT@O~Z0s&( z2hfX|*z^QEY@VER2N6;iAV;lPG$R!{Fd*=D#G(ngicvPKvWqsU(#e6A#K^Ltq@VnA z2`b$XGU-^G7>j~-`$Npd++n zY=$o3#9Xp{L2muABD2zTdah-?t5tl4I&`i;(?GwQk`=(O;!v$N*kk%N-7{BlLH?!q z7z^4H#<-;5X(5;yd2(hU%w5>s^l!rdns3EX09I1Fq%!IM(7s|POU z3=y;T(e1?tXvMlrXD4665Q&_{SGEMS5-L{cH_1KJxgauH#m?}W`u_BO0g?6&&4>Rp zykXGP1H}$D0mx;h+<-Ve$u7q|BP&Go4hn}SN@pkt(nbQ5VBnCAGK;#Ij}~Ip<>L0R zw_RI1kUSj7k3nNxhwe5fL3M4}R-?k>=(nlQFC-RHtEMTunVS$Rs<#ELT~kCOC=9JylO|AlFtp_lw7>O3va(bw>CBOw<5gSr_@wB=3-@9 z*~E4-9%ctd|C1y)rmZVQ2t=Hv_H7|IA^EXM%krLuzi|V&HKBrGvtrlTa*e9mn>wtj zGNk|pZLsX$C5g1G7QcWp5l3`#0bN-DS9%LJ`~sp%+RlQ9bXtzhrG|=|q=H1%R)zEJ z^%(pnynmY(2{q-Wo$i6fuYB@6$yQI_JRJagVtOha93}#*c4$~sNld;XePT?SVU@}b z|M&`8##U!K4a3H;_kgWRP~4fF^jYo&4|s!Sm6fRK#bOyAt`NVu)vq_K7icLzEE%mT zWT0r_&|=*kpWeRC%kk~oXZ)3u5rd`w$MF7p7=z}1;ej9pOIyZ_PvkOF3*D`iXzHXg zS*auNE_)zfv}ZI={Sm6exdvkkDuUEgBv=8+&#t^GLtEaCX3DC8wGSkJw3GG%FV`^G z=2;{gyPE9Z<1CTjXG$$gk6~sR{dqDn(YQ#TskU<>?x0nKPin8hzNX@+yd6xwR zt;wcsN}jTpS|C{9gDF|X4n0-zK#^S6ovEIxSu@(P-9JND@gmQwbpRe*!*F=2dA{d=Qj)~4bf*rZ3?*m6r zYB>TiWe7haZ+*gs2-@xdvs#Ht{^6?BpsYvYvTkdvlky{!;@K{r;|50n%ci2M z2fC=pEDi}UDXiV83=->&YY->Zca1WrDw*heKPDu)Xub43?$*#Cvrk6u1HmcJ~Cf65FTs;<) zP9!*~72JJzOep7TvREQ2+r(dQXb(k31XY9aGjfB3jJ*N`6vl#!)SVc_(}o*vu=a1Z z;Z9E`#XDFLf0ffh>oPTrSSXWUBayR*g}zzS<~a-CXOb+}s9c`~TWKvcD=PMI78O;X zZBM9^us_De4&YR)?zPSH5Xr9){$;z>I{uL2k5|AN(d_YHsorRFBu@a$1=mDUpEuxq zNP)nG)$eL|j*&-~y#elpg6)dCggm_saHN1HnfCJ>jeltj7d7WOr)} z>zt(8Bvd}T)NqS0F$t}At~R9ebGEav8D;feSOZEaDj!t!y>1gu`>2;i)3IE}wM#(P z`Yap)zCME80jjM9;D`D^cixLoV7uMM%K!@)lkyxQl4CfZ6ZZnayj`wQH%UW3k7l1m zw?Uy&Q6XdwVx@~KWa(o|tjQfTaXX8Fx?-wH(N$1(UPwj-o==MDOX6#%6QNTBcQTZO zA)@_hVklAsr|%y`9LWiVZ|xSdu5SWnJ~byPb3{u%H$xn98%f7H)6+1Gl5Y`Nb&V&) zUjLDe%{^2!dH~*yS?#l);LkI6FKj44MmcM-DigDjqBrI%ad7maT3rnp1PbG6LhqqI ze5!Gx`SbxR4I zUu{J@@yklS^2<)O10+Is?!_Q6cbBa9mhZ=MB2fw4nSOdQ|Hlvl1^|<`%aj;h7A5Cd zFLKZ(?*#6lazE%vD{OTYbCy7fvCPKJfrxMPi|>$=_G8--1t6y+niZSu8M$NDF-G^Y zz>OziZ^^_>2^g~9F*V&L9R6~GuX{hhk!)gjxalk2goxIf-ehkHGJ0@em zsep)vBqQFgt(J-NiapqrzVDDBZ`w8XqtIj=NFTPr9)E2C9_{rkT`?qkhUm$s{{sas@wda;^xF)`)`$+JU>o@H95I zlzaj4#Suf3(6Ix&anwd@|9t>8qKO~il`;I)smz-Aj2pPThz1bQ+a!&B0p1ia_41;_DpPN zmNi|tK7955ZFu`3IuXBv3LQ40+%B{2#i<9l8R=>_t{alT>510rN5GyH(KKlL>Y;-) zm1Qq_%lT*P8Kp0#Wv!AJ1+TgATqMB1qTIDpTC?y*maA@1)q;@6!m>{|<_l!eg;M><| zhl#iA0=)HzBp%LN?s=^@y;pFc5fK*v(t~Wu3^43tZl~RMOer~vKA_3d9?R6#>;zEX zi^B^wYE@USa{#xr4ihG#ojNsJxm1t9@w;*^QCPy8FMSV^;I>767~Xz_5ZtjhzYb{H zNFKHAmD&^c z&eL0Ayker0+PuR_c1c_%lomDeZC-`YJgpUg4dOb*+Jc+4UMtDc;)~q>9Rp{iDzQYT zhkUIgOr{$>#Q#t%Q1z}oKqhvW9aLICd z?R<`qfmeNn2w<=aj|@}30u{c&OdU59?$^6SXO4p!@8nCuQsKPV7#U0fS2=bW?MiJt zMn|%^X@%x8uKg(CH(e7_uG!uH+&BLz&xkRWTv<}Rp$cfNpC*4rYLTaGqB3FvVUHNn z-|}zml)3>ob`IpJ z95ikO^VI#-nk4r|2xcYOkQ_8;CqgYwwS!ddC~Ul56hcY1wd)rb+7>efv)^5eEdaX> zd!vtT*&qvvYY5$<%~r(QAduOplSmOos{y06A>WeJ<0S*I%%X#|wcY=7P1*axRY<4R zOZTZY@S1F;je%>z)9MJUHP6b}&?T6;NKWi{=%b$E}6@n654P6%*TUcLgA&m(g+P zH#s0Dq3@eSi)_qhNqj*CNGSqi2N`ybd%#=+^kTs2;Cqu@$9dS4V2YIbsNmV?P@l3x z*Wa0FFG$R|V+wX+3i-!dQ)~O`kADq8y06~-Law3U39 z_@udukl|2(Y+9jh&3TF3aeVVq{905)CJs@OjuO+|6}^>tU+SH0o2MSbtU_&>`W;Ha zY0Y{2Djv-(i;;bfQbcyFG!xP#LhWfIqJ1cp*-vG9;1bDW+f>neQ#Q&bA?{LyGM>g{ zUx)XvI3fGW=mUTJzET8;>js-~Q^EutzEWTBP976NKggp+0oWAZFOdG+QASMY+1BoQ#l~&j=Q!Mj-RUUE0R(MLh zvw+01mFc$hL}W%pI8wVldqXsubIzd-!t zhTT*Z0KRi)&(O8BAq3{k6Evv;1V-!$j}<#7cf}W4{(Vq*fzM-7C=D74ynIQ#t9UaD|>z7OR3Rm@Xzcz#0_*I6~+kNm`Gj z`J%d3yfJCUrFt|r?(tb|I;)wfqOLS2rBTS4Di{z^;qa~6aIg(zJbGDr^;6w4xW_K< z5@fgO)p+1uUFp4qa-XQ)-en}4ZVE1@xKm;(DlVsTwX`v3xaM9+z&uC`j(P{tS7D4X z>-?3fbVL>X;`Jlq;fCorNj#p7YFUbLRy(1s3+2VgdMhGnYU5z-I^=f8jP%Bo>FGEMS$1KE{mN4 z{uEtVPRa|2$>HKazij(RFyPaFg=6*$8a2xcP<&?DWy*YH`IAQ1D!S=LN~_$3*6|;v zd(1oo_1qZ(k~Yb!Q}&zS&$(pmeWd9AO)ZeK3jy`rc{ZAzdu|VvYn@T zr>hd?xLvBqDsSoxys?wvz^m=vRrbn;7WwR$p-!8l<0WhHDlJD-F5;+sj9U%9Mf{?YGd&OIBOJ5E6X8hE#X5@Rb~@1dtJfb`Plp%LA~F zs;S5n8jSNcfxQFghMd)3ZR4wGbuBu|46)Q*4)3=b#z!5 zlS4@rOEq_wPpCum0geW|)Y>BHJWF+E7m7OPl4}d?9io0v`;BBtje$BVja)c2Aos0` z>RbF%0Ykey(CgrV{qTQ;_g{fyA6{QsMlFyyw!2{ttx+9eZ+3xm?n?FChGYIE?DCmA zKyV=u&{O6KuCoINb#nD~X9JP}>0NPUS4~?Lrr8iNTmlvy9nVC6Cef~2z*@?NEGtwR z8RVk#D=Xkdf_r)`Y78Ui5xVY6LV>Sc;}n9jQ?qfWyh={!uJ>miHyoU#;5M^>VyM)4 z)LR8A|MS_~7nI@s!@nj2^h2<8e|$LrZCEL20?>Va?oXAi(U&{4pS(A>?vje09K!~5 z;g-1jB_UZ;jS~^IgT1N69$<{n)3Yrr5t2JYau5}xAO{99)lw3D8DFy3UPIE1JWFV* zv9B|0ow1#mRcdaH9yG`NSu91KU-TO>U5L0xJH}p^#Wj+buenpMIQJ*I$D3D?y9XxV zTo)3O(28@+aTyQUjhv;c%Dto)?(Qaeahq7Ie14z4e;MBY3EUeP5nhfMU%3$;kMJ-P zdo>ilM`iz>rXO;LNs!>4Lo!)iCN)X7Jq~DEbp1<6I!U@=qyR}jYCz=y=xF<_5Om!) z6^%)%GU;2Pw(Lg@oEHQ}31u$&gYMoMDeqE3$M^D=;m`gp@0zb+{OEYnuiyRw{|)b- z;urbo3YT!)$9_j8V$XTQy5PTF5U;q5Y1p{NIJ-aI3QBXl06B+dts#~rKT}_gB_{88 z`R@>3%vZAuF>R6?V!>=7Pltas{tKEdX>_teE&MsUB?G-sFxm0(ZU zYN+d)N<8t+WO!{yu#E{*&&`x(R6t!q>{8GR>9M*pdf1tZ;+K6%L^Njjk^Jr zy{%&>pLQm`5&od$ITE~yn$@ty$Vt_gL&XjW3tqgZTtm*WT8gXox-oRM`QV2y-~S1I zkg^TlK1EFV?fVbXf5Z8U%LA--Sb40c+C#~i5I)$vzoIf2CE7!q&YppbDewlGO=p@1ZNJ4xsiKUIKk`Fyxj; zL8SsR=uOj7HN>|B&i9S`g#MjvcZuTO$cF-0?^mKKi_dVY-jXjD`wie2+G~KnPp;JH z@eIk}&CU$~v?{x_?8I2flkcusoN&&A!cnbED-1p&&_HbN4N~!1sRLBLrRF<)Vw$B# z`>JGwjc9*(#El^R@$!>rp(KFJ#;QAjW+wJ`Mtwq#48X;_F_(uX2f3-ZsCw2VQ<$cU zR!uhVmD~V%Gh07$U6=-D16ghxN7X?KQiXebrB5j+HAK-NfU8J+*l4M2%QSAVK~Mv# z>lvHNS?}lysbky~%nWH;H)+yBHJjIVrOS$GtqdfHT}_b$xf86qUiMLq3~ZXDZ8Gdl zy`ZYJCTU5~`!9UpeLuMe{T86+*I+c`r$YQCAwtE)1(U5hyV9`2?{S8p9cY^mDuV2> z!N&SZ?LqUJl5w<2mu_dfM&aI2 z^I+Wq0q?6TS(QgniV|y+rCXIq)q8zeO1;{2K9gMK9<@pq(&Dx|DkH8Z@ucL4_>q#5 zybN!j>g$)V(e#hdq-}>;5okc8qfsHZ<{IrdngLCkL)nB2LPBUezrz*jzGc4r44&te zA6b39(kr>x2$x}R8*&tG(pj1x;9wFbZ=lbw8hI2skM#yhR8V#?AN z1N8Do*<>f3Oi5uQEGOweXI*xF;XR%^9HnW|gcF9&AFawKiDJ}i9ib;$Nxfa!g-*$( zI)gQ;ot(RBv|+J=1;&fIiyVwm086_+RXK$lqsJ@y3nu&y447h->;}LOpwup14>!XJGi5S*@Isc%z@^DIh-q= zp-Y2%2jk`|cgyUO!_Q~#=BYe8THZK#2cxYV*{bt=*d~yGacu5ve^jZWw9h)jAl7yk z1RLyjUUa~XZVF(0#({#&wcK*v1DlI8mQXEG>`oiqHA3#z&gH^qKNOgfbo8^nP4%JY za+zF&va3VS`PUZQ%owS<(jl&@D(IT zE);W<>b&DDLf#!5K)`F)kxUlpLQOHud+)=KBafJXH=&s>hvk?ku{zM{(%}zGcA-YS zD>%w_qb|Ue!rA~T0X*bj7$r+AC-16i`8W}b+>dmR>0!shn9w|F1-fU4tt%|;Wb>=U zJxO&^L0%zNcGfzSyKr8$BM|=h*E*|fCvbr57$>nqi`QF0YV>*})64GFu?8{huz79e znz!B{{2H!1m#w(ncd#rL&Uuh#bQ>@$(_*@3lU(dsOJJNHs%i<@4G)%8Hj@N#LVmj= zh*Xxu#P|R|pniF++gRmC3`$Xy=m`O$pSA?`#ARs?9W^}7;TPaL`uDE+;s;aF69jnJ zgu4kNfR&Dfpe`nphB96bz>*K1QKYVw_4|4s>S9#E*R|$?4j9MO2BZyZVRu2{NeXv1 zngH6ZK>-cv6Yi4L!IscX2RkPz%3quPXFVaFsdFyE?89}Dstm*nSCNp#`VNmnrBWs%@H_6z3S|N!|P!oOcG6Mn;5HI zLa)|}%9)WYMNTZ>Tb$y+5u)5{H+L^Ex}wLnSp~7=M;goi>ZtTwju)C-%w6$p<4}}R z60cze|GAWp0s&>hLS%2t>g*Q?D^wk->?fpw;+7b&Pfm9cBCT2fWT)NN0ODrMJiiqMw?%=tS;$NP$0v(Erq zK%~F*yDtD^+8w1bIP!0RtqIh;U1V90bG;u>e728`lE?Mj3+S7rA(S1+*I?3kEvEc& z^bpse$S;#3DZ?d>B!A9X3HA2Ix+cy3Xj22$moR18gy2lOW{?f z&wlT;1c??vZi*wevld7)h=e)SDl(rtj8qgsw}^EvhH@qJ1{|LO2JXr*-MCuMKZUm> zLVC#Ax$TZkj;^$^sp${3<}_W2679ANY$a^knHExS8*2T)RTf*33Skfe)?CGNy5ThY zm?USDGD;E*2_g4stc$sTqz>s2XeY7tCAYN?Zto+E>K3_#DkhG1*FHM=C1_RvDzjW0 zW9QuiujVwgBS6~)hs-XNlY2+VMi|vRB_ZIcM_S4*RE(9qwf{N%*J)w@%lnVucKf5t z6I#hzJ~-E&I}l0rdfBIIXp&{>Zf&b+Q8^=sjHa*WmfJzH-Gb3}YC8g)2fZ|{UI}!v zyJLj3I`QGwmdIrQXG>da-?SjMGOf{WD2Q|fjRL_FXI%cfrbu=f!YQ0Lptc@IiP>u3 zOw@Q9-Tv{fSVW|5=Z8<Vf{d_zt4CUxz4w>JkY+>um3#!o3!*l3Td?PbujYihyM`Xet&^lMwnYn!Ywvz zdyr)qRJd9ymz1*dox%MVhB5Qf;Y&9gZbj|3wt85*#SpsbVN)$$JnFC(xQ4CIj zd+DUodKR5BGSW|c;DoYhfv|zw2LMI&R8GNu?>MgNT{t6 z;$24`mgFo@d$mY1DOn`5uS^n(`cUg%%U?0$;wZDI6+oJ17m`R3av0(DtxbXSWd+?c zXG*|p!g0h>_RI5_CDsxaDJL$Tt-|GQGzE;m4R1*+>9InZAg#-@gLDy;Jf5z6ffCl` z#<6#)LN(oR(A?t}xj2rd+~HEy-o1(dwmvOXTAfL%!0@{Bw|fP&D7#0}ht5YuH$47m z=JNJ$q>)|AwDkl&k!ZIET|KZGHr^3gDE}HPc*K|M)MusQ_`?1}y?~Ym;FO#fsUziz zLdO*A%jHzv566?f|K7yspE{SYyl2xjw1i>9@W=HqsuR9@1_sBw!Rj(P*MZc~t+U<* zT43R(sfF#BGiu|eR;%f}jgPKEAx5W{UWp29Xf%*Jc*Q)AHbb3v0^&8AAnxmx&WP^>`h*&(%gw|3+Ygo?q_jZv=B8jKF8-DccoNo5CD7JLbjOu zf_lLFqt9>$Hkb6L@r0LzKH8I(4X)Ffbl!7?wXXOHK+Yk1K}PEGas8e}#%C$$`f%c& zMQ_8)dov08q4en;I%Cvc>tqK_K>hGuhBBQuTCI;bw-a@Oud>t?iUs?Ss#MuUiW7Dy zSCIGGGNT8_YRT+E&sHwwPSQ#VtHakrQE%B&y+o^W8y)&K-W6As9?lNW9g7@lpEUWj zytoz~e0SNYa(*B%9RVqHkzH2?2lk_#Hh*2Rv{C6iPWhD)ZTb&D1;!(pWkUyK%j0}0 zXm>DGEgh7c-B82Df=i$QfWXG*t@C!Z{osyTxtzE{g@QZ30)0gBs<2~X(=T{(XS5Q?X}SHYc1Lmn>RQ=!?~PIco=@b={0ejO-GjDFU$ zpnhzgO&6;qWmj?rfR?OqSbrGaew9w?;{F-iqK(t^nzS9L)Sv^X)+6Xl?*ND`cZpry z$p`1YxG&GD6k$7xaduF*lw}87hbT~Mxjy8kvZrJKHu%}pNr%0UXY+a;j%cxZY2+V9 z*W#qO(R7NX8R>V}KZ?8|2W(%Qrn4Vq~ zh-GVf4U>~=Eu;G?y#2v=$bgwx)%y$*$w%aM<&K3%CW;QNB+5(SPK;RT0jl3Gyu-oq zcdm%TRMqEzMbX^qv$2&%Qz>maqFnFJR_NfLd1|!)SF_u_BN?0xcB^1;v_ZuxovV!e zQOZLo`=fUR8i9iXu+5mu93ejJEwe@*n-GoG_3VisU(=x|R-{wD3v zT{RbabOCL818+QL(RVc6Ed7T1y6OhHr8#gcvP;8Ec?Tr+6G!4*QN|4_f$3kW!J)5U zBA`n+@~)98x|}rrH#SHBeh7WmY!{8J0TB9vYbGymWRteA)wS0slNqCc4(6tTq@#Vj z6S=^zPDxil2?cdAH`f_DlX6{`4CzoJ4SZb=C<~GNc<#P=J;! z0jFh>9H}t6owRN9DE7%txRKh`+IgEs=a0!olGu6BHNBAK21d;BfAWZ}8tB37FfyN3 zYXHvo2RB)FS1yf-9Rbp`USNG9X8}8VjM)x`h%dCP*!iYn3fVZyNX9Oa= zk~~B8FfXd=mY4R|?_WCNn;Yyg(LP#jpKX6h(nAv^0YJTIeq$L#P(Fc_vAOGYe@g$XmXnna*Pou}wLH-uOC;%^?nkaB7rn1xPYe;kI^mXXjqoX9CI-*!^4G$r^ z8^bLcj0VZXDtoP*)JO}#vGc6GXko(CcbLoln1Wa@^1mSePum0|NKXdN+9E4lFV)A) zRtT(8cOg-IB#9V|R1iL3Sc00n+!q%h`zKo1jbOOYknnltgNNC*w=t#bWA-(_4IL1o zz5i&Hm5|aOFTk||k;FA76!k+s(&FefB&F$vVub9QAjSHC;=`9f{B22IRPAw?Ah|ki z0u!)oj8_Qk(~AogWHuBmLP;bb?1AJ=Q8E|Fz*JtOmbzKlzPmZ>pdzx|J&>f?M=hy& zpl@)*lza+;SDjP_`>Ai%4au=uz4dH#yswJ&ASbH3D}MfCCY7Xr9quIASxA6sycbn< z*||h@cN7q}x+^_Eo(CF^3tOAoOmr8{1wPw*K z`edKTMI5WU)5}Jkz*UOwjz@(K<`l9sK876mYArZrSYaljO z7nMs&pL^Na(N?cwcE$n$wJ-v;Wc4ezOF{?YNRuivf;WkvhA5Q4>o3?)Zp~pjB%s|2Kf+9Ol&4jR4%Tvkb&} z+gak8OoNDr0XcwNSXZ(y6awkkm|ustpQYD-`}RW&3Qf{;+I@nm;Z%Z2WZk7yiA}B7 ziRwUrfHjfMXoFilsX>xGH-qM{vX|O$hlXlDr5k_Lu`b=3pxCqzKB>mNUOXB7Zc`h^ zLXz2Q4+ADy#~lSdOYa(5(V)nW=v~Z_@{G3nRdHNUn@K=m(S<4Arey-(wZc4B z7qByqm2>XU#3N)Q6c!h!fKCdU=(3qNB)}!71=*A{CMVr}kWg{=g@b?-98~6I@>EWF zvWywU%1J~~Dxtuh{r?{R-M>p4<$yt+8KFpSnmCmOj4n~$X|7=RhHOWBDfgyzUZptX z0iCd(pkzX32Jkm7>|%jzsDmPx4sab593FvW>BLL{2J-q6}l_W=WKc* z;3%MvgFZK5duaN~hb0yY$Ob2x0+gOd1xW{r&Deedq2D1Sz(&{@;RZK4n{^n(<(HgJ zhd=#>%^SkI%^6e$DSY_J+t1&A|KX?a-&(ZY)IUqokJP}m9o0_cF{f`RTNvdGa4|R= zVZ9eVIDl>zA`NR zN;-5v%3(=+>{LrK_O!<$f%4TJ%2=OdJ5A{CM}NZI1_B~5 zBRm1hpoTgP>5`Gmay=CVpbkUU+k)80taetmBQ(hDOGq)?BOHI>Zx18Xv^JGhO6odW z*$h!D@rA1W=+iZ9q(Qx)H|QAT#k=*$MHij_B#l)mbp?YwncUsGF$*LVZNGztcNo&< zr{VpFmsh#|l?UK7fM-(9oS>36bS{>PV(To%hCvBxQMa(KQ*kJ0B%K|c%V{~2 zjiZFeNzF>_QY2#rLPwN9H-%DX$wy1@^yCZ`;~JA})USf~owI4$RSKv<>4@qNAMGreNM_hY*yD-3>IVezKP zpX(Wi?Ma9=%rz!t*u(59UB%c*jk&6?-@dql-+vQ~;r^fAeiJ_Yr^^#6N$?x$kA^dv zQMIws7bSdT+mr2wVp*LD2H11gUv$<(r4Llc2C^H%*}rhyITIc?ZHY#?GKWGjg=h*q zS%9dFyA^arX!roGOV_B$p-YUTUoCeET1RyQgvhBsQ1(YqDH&Ci`S7`f{kP9#tKk+a z?0y}+4FG+Dz`gra7F*=e!`rrqqL{appY5>WMRz7-T9V@Jn1jzr zfxsmAG3;Q|9<+xT?E>y4Qna!)BTQ93yZlCzm9?rP zjMJr(f0_DL(|A=(Y!$U_ODjpOir5FNVDB-}3p$uXQ5oeKTGd&AFk=?BP_{g&j{R27 z@OSJ8v+#%cbKq%}rL8QEtJ76#KBpPg3N&{K(31}lB+-ya%(16zeM}9Ey#==q7a0Ja zE1Bh{uaR($7r5r4McY}KqB9m{&rrVzX4kHc=@o{mfo@R7r|M$w;(&KG9!d{Z1!bYi zklrAYIO}M|hiYVPicg4RO5f@UB+qrDc?r6+MJ>nBnrHBnFfFyRa-FVHjm(j-5T&%i z3DJ;SAHI6~5{eB!egDHd{aL`?XCDkCIh(7kcS!tZptQM~m4>QWHMPALK0V!~RI=Zj zH_M;rsQO#1tQ7RXyO-W_>5KbLJvx=jR`{%~`6ihVc#<8i)tVB7QtazskVUfGA@!0g z9Eb=YnIh1kpw1>f14FVz9gu`h1i4-<1g>3m9u^jThfH#`@7gj$&Rd?=Y0yuvP)>Tj z(C@x|0Yj9pWWoO-g_+d@j*fKG%a92kEpisQ3oI~e1cPbZM+hg0kcT0v^K7B!{|3s7zKnnd!mdgVN{IO3bImU|Ty- zvr_bUeuVJlMkrJm>;l_tY9nvwSgEhd>c;g-(wds_XKNRc34dWQ&HdG8DN20 zA`SKjt6tfK7`}`Y>P4BT{i87pGy3x#KRH0B#4nRYWhEvlyFdg_j2U^D? z(W;SXRZuBZ3Vq+b=Bow9AnV8Dc{GdCi{LJH`%3zkO6wRmU~SboYLQ*MWCxXHN7mn; zZn|Zb-Dc6|j=m1CN*qZ_hnE^;(Ga4EUa>%@}JC>B}^aht0 z43)iB3GHl4pM;rYqulbEW^YndEowe*%~AQcXIl0xxbqy+dSGBEC3Ln##O-ntendj_ z$$o;j%HQr>Ij`WY^53Js^BGZ#$*hZ}%;|eSgk0!r8%A!^3ay=mfnGMXk?z2#u(rwp z#bHHT*eoeh#RSVr`IDiN|J2Jx`F7ysS|`6k%YLFZ1Kb-Vc3E{f$j|PLXjLbX{jBF) zca6&C`bBeuRZ^OZs%;loY`Vh198{Ko0m8O+WM^tfn&!%J@#B24 zIQ5TC58b{bPp#Aeu$&q~XBBT^6%$~YJ*ls2dJW*@&;gyH{-Jhmp^DpgDR-GT8}E77 zjktlANpfNfI{j3^rYgHd0{7sr8=y_oPI{%V4svAgsmit6forldoor~h85OBW!d3}M zJ7p>1qXVF6&0;W7ga|^cOVC72J-ZGQ1x0$p0v38sEVB|Fyw=}brF2`(yItp zT4p|dmbZr>7&9XD_9)o#oO=8-;0sybr&Dq$Ww9oZlme#f1?&Ra-hET2#l2u}Ye}6c0P(c#4xXxt1aWpb-j~;f$%#X;-*P zGE@dRsvH-g8p_+`TYm&vCOuiOR4q&fni^qpAsIallvJWoeuZ`r_aDX;-91A~F_8y{ zh(q_bQ$UwLZ9p&?B_wDnuz2yF^@KTr{PIj1i3=Ej|Dh~rW z32|Caza~lRA#4VUCllJ&HkXrR!@QOU?A;0;HlPg3QBVvi`+Qy3UW3Z}v3Ns|Eu4;V zv*S6Yx=*ftr?(tI=p(u(= zp$;Lbs)fqv zMlR(SRzAIg6l&s{8Bzb7l)Gb*p)`<9&iWzqp{b6|wASLqk{SRoqGXu(*G$&|czdlp zu~qNWWk7^oSF=`^v<@{JMP-rOq#dPx^KyhQw}{;o^@(tT1wbC7>_M&3rr(`b2y6;wMrUctcu1$Qh)c$ngL?TmyP9Nl7Y-R!3#EDbXBUkH? zc4=JKcFXK~fO`v9CU93|)#ux3T!gc-GiF-;4F3U1B=;?qjlEo>EFGaNzgCQua?=G> zgGgUXF)=F9zGEc_F`Q??`$n>Qwmf7d77SO-vPN`??%Ov@PBn^ ztlUAD?QHF~k)C;Cl*`uq#C94Q1aXG$1GgEV^St3lO-tHof|<^Z9elKCVXsUDvKPNV zu*W3VdGd=Hx*gPNwNWG;CDj-nzOvk@@P=O<;e%{l(h4htk>%cz9<3=ou(RvVD|PpR z(WwJw<8sH5a^XO@ldawjxeGk#Az{s z!&z1J&?S8$0mM23W4CuQ?WOcpo8)ij7DBxRZ?TbjQFc}JOHVP{0wjOk&j81BK@9-7U&314b!}&wq)2 zoa9+FZU)+cELp3_?8f=ta?dLDlGestmndn?18y-xSwfPwjC*-imoR&LfZ@M8G+wkn^klSxC3&moOLBQ6}3$dw7%{nxC{mSmNAA)xmV(%Pu=SNi=D8E4XKL zKZqt%pzj=wW(_+rr3kL1bYOCYYyw@G4G3&kU)_!{!i^8huoS~Zp;VUbcAn)ms%_a> zKu}pLddbBFSPV&-A`VbS@2XW%cJaZC!&pSAjJ4yisF^?t!8`CQj@ghz;2AfZD!8hm z!MI)a#W4zUOj!ELnjS^Dle-AXT?unFryINTU9kXg2}*nT6?Gn``szCl<-v|p3mdMo z^~mQk65RH`45eA#E8u3mEAQu{IV z(pzNl>fJ626(o}UCoT{25YSyTyE%-h{$4lRc(Q*&azCIfK)8eD7jLjxmPVVTX!5ij z)l65_*})hyiOfS0rP@wi!ddF*lPjiNY>R|UwqXFrs>xChuLZJv&U;SDoi;V)o}aFu z7&0$L<2bk@T#I*W6Zt^aU7;LIxgIPdfP`L;3#+@sl@AHC?5mNnCz(Hox750aVuVty zC6642Q}7JKR1?*aP`B!SMle051UoU#szz(6V(hZCb)W23EuIzTA?1VZ77)#Z*|N=Z z;#sKEJ5T);Ex+lRXbGHr=+ihgEeW5|wq?ogNlBFcP%p?&u)}jW8IPSwiu~ED9*N7D_R=?W_N~C{bKZ5Abv-WKAX&1Zx)sOm zEzY)h%9XyN`jHc7Pd&b%a=z_#_KvJnVS=t89b6MA(tG7JDXhT2E8jG7pTkl9hiPOa z=kddT3~xVxOuHovPo?=>cKKh}0VRd1zNO#y!j->Q(1+H$4J2+ zVBdC4#3hcYOE0T&<&F)m?t|+3DjSUZB)*_Soi^1XUodB?CT%w2g0nmbHW}c?K&LKe zI{^&+>n|6`Se)^%AgnL3hB&RH9F~gM@5mKj(!G`i*iL(^b$5--2)!;0b#t}>R9_De zvPvI&-yP>yWlfogNRBeD;Mx-{xj-#AqxffS=)SO$(V!m;y3?YkYfq`^rRa7vm@2np zUdrPaWwGEsAv;rhp!=fe?J+hqt^6dkMD~C!z}GdWfrS#G50%K99o;3d=~{ID0XMj7 z8pa)#dK}z3HA$7#;J6J>wq(PK98_)~B279eu!<#V;uKHA`v5GZVB-0#X`IH_1$?07 z@i>}W-YDAlpqsa`_QC`vx`ygjNic{3Us`$}H{H_9b++3u^?DFZbHxQ2>WR!q&$0?F zYx!OzCRum)i1OOnPbvG-+Q1D?02Hn)_U(WtrGGJ5tJ>DW6+uuen=v<28{ibM!u~^=&IcxtP2teBVh^zlM3gR$ zN|tzp3{CD^!?e0z8bx`Ru8}|O(_r;0ZlZPRMwpFE14SM_bt$hC*#8HAYLWV*x1X7h_S^3mgUAR~ zg^OB((ZKBkEg8}yTD9zCL0h$uvvJ`$x%y>AZWE$1E7?WLgAB(Xw@26;#~}=zTZ0XX zV;khaYllj%_(F=Fp-@3LCnn343FRXF>ITHYazWF!Ny=jKd&sE$me9T|Q=ks2sx2EP zSBz7ETTtA;cVZzrCug^AR3|07zL=0^LOSkBUr8@D!+TE7%YZkJ z?B-bsOAZTxC-txO0$EqNSsQ4cZR8;R2^TCO1-~E5L#^4%hITekv%@>=t>Q#5=wv+>W%owVyYA8 z@+xVr?E=o=ASCFQ%n%-KQPyM^*B*YIc0N^BEx^;xmEh5u()){K#stkp9V)rh;E+!6 zoN!UH3I~?N60a3VBF^2)wo|W=4_~~08{U6`p!Kux{%geeTJevNy5%(JnXuPc?&2wL zE}aOS>0^?LF|{38llJP|A7|4(0->97OdwHh(21>3A1nI5<3=U!0npvn(tkV{=b%Jx zKjae)Z3Ww2C$!V`%Gw>W6ZXZ1oI2ZOfZa9vF@`d>T;Ftnh*o#h zI4%#BZKhKXW+1=`+Vn_5+^a-d6F`~0eP7X!QI#)P)dIybx{1dca(X;9fSvnTR6$F8 zj=_utrKtE!ZX_4x{p{@uvUs8pq}UTYEV<@7qKLLBtz4%^9vn)7 z^6Y`^p9tQq+?}tSTs^vkl-=TnClp-gui;O>Apy4=BqxWaR60k6l%%cA|K+?;zC|It zaX~Kv0-G*%gL2sf-a3*Z5SPoTMC1HPY8L4kOJd-B!tplJxpN5MRvz8ZW>EWD-o-8M@K*}m{ zQ)%rPZ#lvWy-zRkDpbWv=U_D&$HiMo{D~~V!*EO)mow>t)-)qh-B7}Q#{(H`(l6+e zMP>9bVqoU9k-IEC8pT?6SRgS(gb@nv0aM)uld=me6|u zwH0r=5)*YG4CIZlAb^6YuEl+}gJwhOUjz>R^}ELc`BKOCDNw~79%Gn2e75S|?dr9N zJq|2S@YviHlRRensKQPuE8XHi7E)|}`s51m;%wng5uSEmbRv-y(>Rx&XXI960eBZ% z)q5{dxKC0C*@Y%b1tWKBaX7ZRINlyxo&4+FI-mud=k~MYI zhljUo>o&8hU`7zMiho(-6F3!&Yk99pS_{YBkeWQnz0|-P74vZPyDg7^Sop*Tjm(H% z6h(6yyd*ngZwLRk;m`lOyyJgP6W>RtDz-;W_QXEx1n;7bJJHt{J;0vuT9m<59Fn_i zfck&u9_mNP0-b=dboa|B86|KkeXjmsauyV*6*%j^VI0#k3rL;^uIf>z^%-T!<%z8< zt|W&Zqc-K752lVh65&d~76I)gHl~1{*9Pc>ZEURbeINUepg%vS8*FOQJ(GkBihY$c zOVM$0-6s=zf!9dBbVpQlqnw(=I7vv?xn^`U&wb z%cnYcpQPa~<0V7WO~Ru~?Wm9VdXe03$C547HeB`jP_^SK{|gpzu7xDlrewVm@?B#h z>9dM**^~70xS0}N!zxWK@EtRM=$-gYuU>x_AVem%Xk;~BQHW-E6IU8OtZOpHe{D$NuDp&)&WcfBb%0J9XhkjY&g}p0EL^<6^cv zqh-`(>)RRJQ*EoqCU(?~nBy0I2$V`3`~1C2;CS(Re_wk46HJrXd=f#%of= z%Jv>g?AJtX$&%}|Splh!6a$Hijlr^JHTH!4Jz!sz$cWVMaI12_Oh z4A!T`iRV3>WXOO%QvOze-64tn@Hwnh;GR*xmd`GCcTV}y=I>)nTohZ|(Pw802X<~h ze4JH89_jm}@_qw5TM!9>2-7fUKd_H|fowdvHZY#oVAN#o4Lez>mV)##QL|bQH&8X(aYs zi+j7!)sR}+(Am32*MOB8h4ijsV~DC%-*QuY>T-8t_A>A2<#vvb(y|Ul8||LV^4!;1 zuI1d!$A`{t5+JY687lc5rr=*)UBh+WsPhRlK0@V&rUm+; zpOE5UC@0nMR<}lyMI`X21q8=NW&0VT8ZB&cI8v^V9|h_=XxT~2qgn-yGuo`$0XB05 zf#EEpRb~f8w0Gf|hi+6~cJjo#$3>fJBiRAPH87?8uKX{2_q%_lU;J4xHMVC7KyD&uzIl37`1BU2MSFa4x>6X%HnDc%5r z1PPKLNP%W`ivKlxZQpBq9I}Tp0aQ|2f7-*|1RjE=@xRv?COObfHCppbi;n+!1C!hGVKEetnLMlQ@>lkNcT>D^r z)qy;_G)qEIMPGyc8OY8JYzyNm1@8FL)ykfd5^Z1Gu z(WRc)7?&~sCLHA>lE7vY=5+Y&yu61qN{p;5olDUDmi#q?3;O+}g2Kmv1fWAt93cFzFI|BZq9DaQN|l52 z+1M*^EeVfZ5~d@ZB8Rg*812cdU{p#;l)Jcf%NaW~9fqQ-FV*w17daLf{_&Tk=UQ*> z$+6`l$-yZr_LxS8M1->bUCA(NwVAP(@_R`ISY8fXQ9=AXNTkl1-azy7Cg8 zcYRazV%|NN$-@=ZD>pUwk<&WyD+=^%p`os@d~6z-Y!g6q|% z+=*l2Dy5+sL-h2BMhp9(fC;Hndlb8>$r}a`-lbKf0r^s7lu$dd4Ij6IPIkONt=P*M z5I)JO{}%3mtWm0-QE29H(RLMn-C(n97sJ(1GeGKgBssO|pf(Rl-`l?FOA8`#Sa@9)giBW$J(H+pnyk+>+q{2R=NM52lN4#iP z4H$dW)2ALm%(gw=B#Td3&dKJUX^ACmRIZm??#k0)2{3r-{QhTX8-OQ|6!UFF~Z+^N-5#>QD_(-YjX z*}LqUdODTvS%lJ({OUY*a27|ZogWBNtz2x3j#9nFdx&Jj0PzR;${hA0^fK~7fuhz< z_JvwXtX&0sce2a1!_f8MPJ*=8?j{8p^@(EvDa|*I6rU>IKCaNzq*^~K5X7DqFG_wI zS>8jt>@c$p92KFdJDpL3tJrwGxzesO^EU*%tMAgegFcX(koIhKNY<-!XuqukhNd>y z+4TT3Y)K5OnYzk{e2MMI6ITo-dtApahN{2z2gJ%X)^>W)u%#~oI2B)^hhwcJ= z*yJ9*SY^I187g4>1-aG%BfNF#DA)nuG;xKYCB1koJ!(w3t)Hx|UgdfCUe%|`#+VXZ zCPo`z&3h--TO$?NYSI_c|4j?V-9XmMEs|wH+TEKJ7j5Mz6eEn&<`c%KQXn=z>@x?8 ziwJeUv!k6W6etn3vNeu?X>galFKr$BFz&kBOdu{_u4ihZZ5=dB@tl8R$!EG>*V3|p z;m|>XQV!J&V0T;xx@D?D5gR6wILm2A%W)#S7fqu8O6A&d+k(48vmO^Ekj=l_aP|Fv53ir-tB1Zt`MuqXm$YaiwZT)E zS<;!JS;_KcoI~Z}Y7lWu?AFSvNiKI=AT?$e3*aR~_Z%j-N=F~a5f~L_*!kl;J-NDO zcb$Bs*gnEk)L1t=Gp#zDSW|_-dFXNLw9$5}?8Qr%X2WB)iH(fPeW@#0)S-7kFR%(x zzvuX$0=~K7ba(W=97UV&X0X=XC-stpubqM+O0lzHgj~^E1+m6;WJca?gIeh=%*C6M zVH;3WZW*W$SV^yf3&2@Pn%qj2M_}9@2>4Ovm(gUWQnN@Jo8KS5l1!{QILuKJ9z)p2 z0S}AP_?a8%?dN$t?zUy`i(i`uPik$7HyEC9@UyiTwrn5Z!hsujVwSupo2XSE!hGta zGLYNdk~pa=nA^!s0c@)T2T1Ux#Y|7e2>osgiuzu-yh9jvLc4JVZD8sd2L+7+<+)>8 z?WJ3z58c9NPTU+9hBQp-s3X0i%s)%D;Qiom z(9xh15$QafuX9lF9uH$)ZUs7e_JJg=-hhb~(GIz9E0z?#cg=dHjw z4SzOD@_NASg{O2+?sf}1t&#BmY7fJdEmM0N93z&^k!=;l=Rh`%Q-kx_5`_$Q=z5dK zjRsAURl2J*=-^NQSUQra4}TpV+*YMt@=JhMnTs`c{h@5wyz+?zph(WWlAw>y#I^LY zUw>%*F}%npM+NMSqR#^X`h__$q$TTx-dOG2^ZpuZDUi^{PcGdsMlb;nN<)wYl&h+f zfWGaswt$yqz-w+l9Z4?+1g4E&8dhY0mN(YNdpv= z7cXSyD7!TF8`<%)xYkWloi%s78p{V7Bu5~Ve2CATr%f2haG!+Q=%CA#nPVo9{H(pa zOmLEaoSx-O^-f_E{`=1#k zwVd~r@55f(L6+B}PnR04gyzZ3w_CJl=na7+%ttE<5?;DziSk1PIytXr0|H4y0iC88 zfx>QqHgdPFfs_kr=P+MnQHal@#24kK#>+`Jh7)O*X*vi@!h{qH=9(oclmgel%pa9K znXjy2cd3UQ62MsMk&LqvfeQJWVkOJI*$Q9byx`+hl4~ZD^RrM|n_}vDce8U@=CJ{4 zNlG^wzts=zZnDJ;X{`sur#7l&Nl85tA|r9)#rmKCwQ>1@gnOLg3xe&q@^qr1xH#PM zu>!cxY0RXJ!<#1cXw}`M+QDqV

nj;X-C@gua07)qPBmS6kxKbsrp_1VbHQ%kr&^ z!k7u#?j%tu`H)R;4-JZO%2!$>0QKq&&?l(kFwOyPi2NUpf%zN7>I!<1?&7DVpkI{@ zmL_`FL^hDUgy{TA_m8lC`Sad3+E!0evYC5APM}? zn}5A`nP9%7s`jCld`%y;mWtGPnVBfxz%iS`08PY`1H7$<}!~CprTo z)#v>%DxgFCoTl9s)+Irb`hTaxwGxB*h@QgTc!zbtluM2V%bXNRZ`ntM4b$WFoRne0 zu3kt3yDfkl_t2DDc$ybS3Gak--iSSyr3B_e@?}j%xmv))(D`GWy!1vK8^N^o{WU-s zXIFuh2%`yi#eqI=PqppK9Vz+~Ezk_d+Ld0gnrYe?kc7?;t~|UeX?S z|M>>D?eFxjX*!T2C zQa|i{iG&hks#!K_dzy7zy*oq_SxwX~CZU~!<_+5D?-AbZQ%ivE)Kb`hijWQ(?clbL zX30qBI8!{ARUTXe11G6x3>YoDQplL?PZUlT1-qm!?d+ZOy(ZoV`}v=Tw?CfbrhZo8 z-4EnQK(MD@xU-lV#Z7dTPtftrL0a^28qc9WG*w7M$xsyd zaS&^bx<4OLLL}t0L+~H@LknnZ>I@qSB=+L!-{|eSLDs}}P6csZZbwyU%wN=yy-Yo{ z4hFeQYbfWCqoRW2+iocm)DQOlj23ajZ@A4Go{Vjk6eWhom2lH2*-+LbW}1gladIRH zIzBH@nL=-iTM3=}n&}N}0dsR<@VF))5*C12<=!(qw2|3hgY%+gsogE~<_k1hUC`ow z-!Rp{?8UE?6xGqOXI_%j*SCRy&NGu#ggn#G5#J*uFUGo!vB+V}S~ zIQf~5GCKOW{^DFT9mlD|5M`(ex21&>WV;XSUI+ifGoVne0(E|t9S?+@tqy#^ zP`Nb|Fs!zNCHp7!2NOWr(Jl@)j;IB0(Z-113Ur&|*^g(GOWu;%y%n)NSam9N97&V9 zELjR!dv?4eTv!tg=O}yA znfMznpj&qs2SxM(uJ~)1+LUYQm#<&{rOxR4@4fyqy#AW6f5F!e+4-|qp?kw*t4->= zBiHX1u2f}sw_~tmw>-dyNwy&I9g@}-%~q>8A1-nYMtpt|z*q&L#n0O?DHNsjHHLUp zbr>6|qvUVd=0xMqpW?El^L`+1bm4C5Qo&Fo&}HnDrK(a+gR2HF;~kUz*3-|$P(dqH zJMD-%^ugelst+jR9-bz*wn7f#AS64yY}cXpr3mQZt6N~ z(hUZX&8^2BU+2>C9@)N|@RT5b+mRh0fpVCYV<4$AHOaIkBv+f%tKPIqJ<56Y3}2Yk zU>-YCcu|cu$3)KsS>*w>yC+SsNWF4}Z*nT6h=-}fYK`Q&ha@280>ms$j20cY1!6d)A~W z$bOU!q(~t~T=mr9JQbj&VJFELn*#cT?{t(-?(u+gBmox64-4#sd^@hO&&YG{)E+QB zuzTn(r@o&a4UV*9MwI;86Y;pJzr5X)Pm^4!6>a^pGL@@e!|Ke#PS1NV6`oE2~>eIY-JzS{O@+@G?sJcIQ@?FPHm0r#569ynaeZn2m zQ9=DI^_RU&>^xl$VxUhLUw3k-fgHDf{g~B^^Kzn(H*v&QtQgOD8g@n$)qIp_?hxL0 zTsg9sS;gsokXCDywd|R2y)Qk++#O{#gQdE4xYVZ%m{VnfR<7HLK_htm#Ygg==);@? z#3$h&EeN1vxaOinHlpnQhf0kb_)q89TOqXGc}HwWy5#`_#*!Ky4-~cW(7&A5hrDA> ze`nR4$mctx{l+n<(sW!}Nrj1qNI6p3*;kBFR+~Vys$a(}N-pI61WWG5TS$iv{9@1t zDqc6}7)0nHx$fMEZ@vHY?RT%AzW?-3|NQpL@cz@+-@SeP{->|MrnA(?Z(oyQ{rT%3 zauAKrp|YFyB{Y^7r3DV71mAA>>77%1=hk+01td;fL4pG`xTwfa^@dry`Trao1KarS zoy)l9|+AvKvUwXfTt$iD$tlGmUooL8P{$J%Ly0fQ^# z$dgi4$f_`@0s-BFzJt>fI+l_}D}|eHTcWcX4{UKDgdW6;)q7@Vxqz8J_ny{4NjMN$ z(}ZSm#XOxMqGYQjx<@IOFd*v_*)QV-WG2U-j~8?US8}zLgT29;`^RrmAphv?Ya6QG zB_>)&=q7JJM_>3vxd_uXK}X#6ST@M4u#>F4TiHcw;@Df1s$e|YMs~9?hp{>AGw(l1 z6%54=NQY~Ljgm&ElIp_q8ZFn0d}y?405#|(8#|bYf=43f7!Wl$WM~*gpGCG7(P|G} zv0G*Fqn)d|wM0Ul!+kwM#PRl4svD@OI z+I4%i*~cf7?9fhefU{2!$h+EMc>{b?A1}^C0Fa|OP#%yFmAm=V zTp_o@zK~WeOG;!##T#;pqT#$tfmyMcBuV-cQee}bAp5F)%-SO+)VOs~zkEr$0@7s% zn`6So-$_orCjyKcZK~laLfwOYe+ws2q9CR_0|yXAUH(Ha+c1*65(0#glsTOIRrq@~ zNR3uutH;F|jgmzT7aE?y7s*)c3ut|+W+2F|gEmXCEvO0c4)_KaWs$^c+={QVMJPqW z+go$Wr~Eouo>Q-_$cctD5_dLlE{Ah`fVqM0$pYHRLbyX9kfy8L{by?0OAJz*%xW7pJ_Wr6JHpmr z2m>w-v5rGlN%didHXP7zkPF_+5;<$Ny;bz2R>Isd<-hXDrfx`^ z3v8pszk=H%37-D!mG?>@c~Cy&UCFM7JzT6~w{)vQn(Z)wz(M8HB}s%?OMy3Q_hlpe zc+Spaq?|}y_-xx!QOPFE3?BtP<{l3AClzSU^pCvTG(~D*N-i0v+{9b~RbrG6MM;Dm zqG&dyI)l{bW*E+wOE|F~poOJ20q<0eM&u@ShZda@qlpWyvMK}>E!GP8VBvxd9}H|H zeqqgg$=-kG^~>-rBwDW@V_x`#p*HOxgQYp^e#8k6j}v&Jfx8X#xiDR!9Y#>#hzW)D zw}*~BE-kIrJ|(*Tc5Bf?NC&_sh_12f8IQrw4MY=am@m zR_TmW=>!Z=J-l^It~0>{lOS&kq7Z!@jOOEnHsc8Zd>a#BBD$7hvy5|;^lG^fu$>la z(Z0094XB(|SmmZuG5tSUC+^-|K|Wzd9hVOpU6|)T93unH*JN)>Pf_PbE@*L5a^S*u zoiKKGdkHyMlsiY4>uF)c0qJRw!Zp)Q8F(a%F=`SCURTNa>vp=Z`PNHi-l0j(lX!vY zteri`$Z%4XV-Z4;7XxQ0Ib2P+p87^DWMEWNoF1Z$hT*9!NpfzCq8s(x8BP`j5>1fL zpJ#eHHy0BDog21i(AcY-R3Z-h9$F(U@M1A znOVyj|9N=(Asmy|9iGe2)soow4%z0xeY6M@U_)c74#9}aWRgD zP4NdV#j5Mw*}wz57oWjFborJ6rQDJ9OKah}oU-GJRC}1q$G)LL5;{n>FI6nqlQ!6R zE~b?TVKEc^mM7CI2g%xD^)5%eGxseOsizqy)#<~Zf8)=;&R4FKuZ(!G`~ViHiq;B~ z2OqA}kS)1Q(Nx97$xU)wNpm2PA96=lMns5IGLl>mK#C^FaaHZjME z%kaoK5jW$6$N%kbe>;5R8{a6q?=v~;*|+okzrXz={OP};ees82_`DZioI6k$D&;UD zWtPKTtA8;KBWnFvRuHI`$VhlS^6i0EtqVj!r?DaDf`IIK2VWMDC`d`Tyy=MT3YY5# zqwy2GL&CH1JedpVVy}uAO^y)8EQkc(wio1i-IE13oc@QI(w?x$I z{@`$$E3EMLYa5A&x6eQZw=OHMuPTa3Chua*^8=!jB`2SMcC4e_#zd zDM0)n-6Itpc&t<$?lj+TQonRlSwq%LRecV#rBRl}y=HNn$CTB6t*Z^?pu={7JfYpD z03@@uCzoR3`W9mALVuXKeW1T7?+G()V_%Lc+mn~9tI>v2&#`D}h(ugTW}fYJT=#=* zmqa`qppEw}X4VK|PZ8m1INY8a%8KX{UuqVIgkKpG3ktb+WwVuhu-bHr;)5&3}fIp0mz*L-xE2_muHtMns4N}bP109&&adR&< zY&_?c2JzUG^2{WaxY~GJX9bkGfF0OiP5w5b!)-*DPUS=L>!AKrIShbi*PT>h(q?Nr z9RH#OQiF2<_e2u>VU7oltZ{NhvZ|cO3RrPl+Y!lvgu%^f_-RL@^~xO!9U1vhF18FF z^d0HMLme8uq>^fUM7~C394sNH1GlW}eJb51HK4a*Xom} zrO-N4pYh+}&%aUj;19#wC#U!S^7{4re+jR@Ki!9;!p$9n6^pHs@K8w`h(TyM%csKx zIniiOE{ij5zTw3SWf%-pB=>`hmD+{uMFG&I4u(3zv*?M~A^nk@XSD5yrpMjrDjm98 zb%ofuzg)lzH5ZNHWHlAcv`^d^BE|q>1n-Xgv7}XbGC`VQY zvea_UOQd~a>%+;KP#KK+ESth8Mn3;F0jN*e5FlVXlKGY9mG~jn=6I^PM<0>R zkZH{0Ease29*(dTbETv)IGyT=ki3hsJ|-WJU6M)*_h_@IeIyQapVwLEy?6crA3=0uBI@IJhhgK`)Pp}Q-6|dnDwz- zEOJ;8KCB&%vLoF}(gQ`5QDxciyZqbDBb;gQ46g{*lG=~3Y@)`{EiWKw;025m@9Ie^ zL3^eoRB*NHy|Lk?TBCHxs}gb65Yv%QvPn4Hjd-?hAQ+&%#t9967^law6$9lEW&1Pu z&7E%HNTt8nr~-AqFJujO2y;EzZtr&hP|HvOlCiB1^!gMpyHB}#a7XpkQ*@<%z_t#Q zQmuv;xqrCgu#Rq>%NvrLXiIX{r&{yv>T$}8@ow6hXl?*n?U8<_qO$m>q;_od1K#Hd zkQ#Jj8^FtwU3ry1!nL2Oy{1a8C0Kv@XC1prF_UFdUm}HIw=CA%_ws7adRa9=vE3ly z?y^K1n<4|Vja}`Q9lhJp_F4IXagmP{e4Z3+X9)QL61T0bGBE`x1tB#AE6}{To_2H> z5H5`XBnh+04B0q9wv4Q@_NXt^M}=Cc#kDhlo*)csqgl;*@&kw%hs!E�J#6HekB- zpg%|cf)oM(=xr%!!i7-`+pzr^`1nibBx6WkgjloUQrpJXl`#~unjel1mO52kY9`EQ zTZQ0W2!w;_eBg!m6BwyJFotQt?XT4bGwfF_q-alz_%cud8s3gpM!~c|ev|TIlC!*e zH+2FTgo6Z$8IOIn*!gZRGV=4HZNK%;26fu5WPT=3)sirbt7P-+L>Wx#4(V0Njl4}D zB3ZRvz&_Tt-j$t4?E~ADu5jlC-S;A!A;o``)6+2+Y57|0fpIYh)p_TzyeLM(tZkT} zi#5yLrvm$rI9?~x%Hmo7GU`bE`kQ~NBBt6~e)}!_r^4%xPEa8Y?vHne(vVY2t04Y;8+yHR~Z(2qh<^BX{Mi&T{y%Xd34 zN9?OYFf+ddYw?ym-{|*kI@m~dt@CCj12@7A6mD>zTY9*aUT*{YB4<9!QpE7~Wd@W(T=qw3Eo7bf z?P13>bCJPqOk>o>ooaS^6Z`J)=7en!fWh|txso8S@=n&4#v*qTk6;?DJX>C^I;KDG zt|0`knQ*7(aHVqSKyNIde0xJb!Cd{k6FOJz4OG-PU{nLt= z_V2^^Me2$6D;;_uCDxAA?+@7f$AlSM4m%)r+a&bfhIh7Y;Q|BYcUd%v8y8AtY;u<@ zti%AK=^@`@X>wg1kJfy~IDNoQW6n|=QCse507KB%c~ZR516=BiJc_WgCPNiMUv9w# ze!0{*EsUd{+-A{Q?FHPVfe0F6=rOUy5V zuj@73PPOFFRmGcMVmoHM5-RGWD*gNKASnM6%9r08sRSz>XZTz%WRJ>5h7@Ph6I{1$ zj&K1Wf^)%ndW@ckfFmCY@I3EJr8(CAB_trW-sz^7Fxz(Q;|-G>d+mHeyby9DFU(do(pC*&uj=jbK)Ng`q+5>#|q8Aw( zjX?jRvZ0r6T80B58xn@rlLalBC{#&oi^N{NOmJmXV#9Qn$b%tpT1`w72HmLGTyO}A zF#hW*MK@c%41ajr_3n3|S>tcR>+i6g0h@`Vd{Fz)`gX-9hvo)J6C6DgOd#8r|$AuQB1ZFFg zo7`@Dk&?bf*4W^IQ9wbq66;|64R|FJw~9xX?_iY+mCWzT3n-~7TK=Td14>*^Q^n64 zCQGt2gWFqAy((3+VO@pMdU@1e`=@^j|B%m80nh2Eu<|*-%Br00l&k|v4c7#~NKTDf zHXjbkMb#GXp&1o`6J%F5RDo0;)VBTsbf2t_)Je~>GST^cm^#>5P)-;ik12bA%i*nv z_L4+y^`67ow8m&Zqm4Vpeq?MeIHoZYkBQ()?K-u>5GN{GDkIP(2*rnLM(PE z#qHnp^$TK}1uo!>E&oyu(WrX0<2n<=jmsQB45s7zU0Wbb8g+cZxj^3psTVVqtaAG0 z!bTSKgXBY3r)lhARml7XF1er-@Z}qZIda%1;Rg1a+53>ddS;&zeJJjWW*@PR0QCMghX9YRD^Y!kmi$t6aRdPl4oK=Vfk_VSe2 z^(+#MP7{?9ush1?4*-2rhiYHcWtI6HgwP6vDLwCs7Nxj@s@*t|MK*@c%ko#@LQND^ za>>3#IB1BKx4$3*=7V`GjlYj@_&ITXc>7rI>%A_7zewpQL%}Z z@((-uaMWVbh8^crJsicx@zkmyJxx&5_i&Lr(!K_at@?rQJsaWBaK>;n+KrA<8<#+ zb1Bn-_a93jG&YcwyuZqBJTF>`$o*Ma>wa+&s-2@LxwZF74NE3tDkx9gBB83)jX(w< zsex+xT4XuFbf-b$io@c5oPZXbdzVT8Nr+HD03Y)!U^&J>Yn2J33Nj!%>>Sj1SEM>< z(gF#u0BgzeX$8EN4!9h5u+IXpGqhkH9XF>MlUB`5*WyqPjWh_19a67)2bYYj%{%}o zB7tz}YE)^vl%RIT5NK%IdviH=v zz|hliPJx4Ar47Cyi?x>3O`h^C;ml4gEs!b5vhE2H<8#|LNDb`}!|k(4^yR)FksnD@ znsmd_z;Xs4gV@2V-k7o#ynr90)Ja_!I2yue&bAhT<47IH54e7EkDa~K4z1OC+gEu= zOx&S_5^BGMI!7G0+OBcQXn9Zy>D|cTQ55f9=t$=h-Im z5W~lBpNH39ekjfG14uT=zs%_Q4hTYyC^)_r46c$!z)i6GGoL*rp6#+tA0Z{E2=o3{ zJD_uSz=rLxl^I>%qE8!;}{+I;v~#j%XCHVkh+GtdNzrX`3CPBZmVi zvM|#&r%?p;eP#T1KjZ5HRF}1&&_QM+Ufw}_aJaJJN9Ls33x zs=0!s^S)G_l7y^L1chSuLERZ(gg}6G2BdoGhOP(Ln&>L*?cUDuQ{2^_*RTTXL1z<^ zY~;JNi>fg*@Xc zwmy^TwT31H-3C-w!g2>VD>q5oc%&}MQ~wgIPG{2B)Hu^=9wap_E`iG;w=6o0qj&CS zd@qIRe0tjD6oLzkl2TSj>XHRT+{SKBX4jmgplB%J2mv)k)sOMiKqcx%@Ob5q2iPKo z7QsX`f2WV#a+@BmYF2Bifv$~#ASxeHBf@v5=(5M(%6JDb18Q7w#J0s+?`4lh@g5iiB!o1J(Dh?14rX&o!#d@`01y$LSI)Kc5o;@0mP|dC z3Xf3x1qz21eTqZBqeEaS%=fTKG=?HNTPItKsh8BHrELh&`h8Xe?BT*Zkh+z3R3gZ* zqx4X5fO18TH~3=bb?cfwdnU0)mI{6_f&<$k3UtoF5eNJN5J8-%DLUMG=MXrhc4UF#r0woLsjAc>6)FZDORimPuNt`EsqmN@+k)tiQP)w$)3EJ(NjNUEU0*fl z8IlKI4rmJ}=0wHI$~F2h?w8zjM}%lKm3aU_m6AL~P7eR8{V)7i{qi6FA^gMXo8JuI z{AON9<@uWxAWtuRNYUlkviqII@D1%l%8Z8do{`vxjTxO5fuyZ-jR3Istz8&uGzMiU zhfVUsn}t(FaAcPZgSb4UI6AaV6-4buI$Or2>1zM@U~ZPJK!N7Dqqm9i9ZwxTh<&g&mWX7N`4l|pa-&HlBGclQG$l~mRXUr z1D1`MGH{@SQwBOZ1EI5U&%gc!b6i_#D$~EF<;EWY-a39u*E>8z-9OQGgOuE=J$Dx`_k}`D z3fYVO%x}dF58#Qtx+mILqRB{!sSai{SDemM&UOr}3o@D`VN`0{#9CA+ZS0CZFbh=7 z?oBf6tlGS>rxGP}kZROEL;yqbvp&_N`CzcD9?jU;wW~{v5sa@4!{OOejt^x&;F+Wy z!u6rH*qkE{59o&P{4oCz{txO`|LiZ-0q?Qj(4ZiPamS?7j5>jekegMd1-$cyr)Fvj zk^s5v*`lbPP62_EJhH_)4S)gic9>zGB)cJkeQ)(u8BO8lrJVg0tOpBkP{oEPkm_5k zt#N!i`H$~XT8(-8)LuYs$EI%GDp${zbcMTN;0+Q1vZ|C!;>Ik^P47$qK+aI<->l@X zD5deAk5Wrty#AERL4Q2N1$JkD@%B@6^s^Nt(Py^-kvW}i-OE8XSUy0wxgFWQg_~wH zXh=r4M#Q;3GY?vo<7C(N6#x;H*30XhT^A<(&8v$lV+hY;48OWo@H5tq?HeiBn$s0_ z)^t=;<3);RIbji|x|kQpvyls|-l;Tgb}PLUkYqH*;TqP<2Y_}EfnT)7Il;D%bC<(o*yi=o!wk#?ZuaF| z9tqT<`^RbJEDI$CI+L9X+w^nyLG~rb3EZMvfo6EArWjjQH!A*p88~ zwx6L_S4a!)dkow?r)>{z}2CjU48_WDIIO>=0=7(5lH*<`gP zFlM`BF8(~>asgYG{v^P45Y`-F18#qSxN;>A43>$8s&F-ng?34>D(T4*LkWF=kq6k! z*u-tSp*-+HIS-h8L^Qlsxk@f;M3qgHv4f z8DW&}Q1E(@DiQuLvGm;Pt0I~`vD;PF<)lTEJt5>qInP-RL%2VLx8K^i`O`=8+F*hO zFY9o|n+`pR?T~P)MRS7zGN!i^g4lR$v!WgA2?Wc`6^Gf!_1LSGcaPfTq!W8KwSg9L z*20KM3)`b%?X%P2nSw1>Nsrv0wA;2atAqc9oIe+4mu$eU{B8|6o|Hb5g1)=3<<}s* zemK`qg%g5?TAieiuB}%Csi01iP`Em2f;vJdT5XOWBtBxy=>&=!`<{CZDHV=7Fqi{h zA%6w&l6cxQ#aWYf5}7&67B@}$ImxY0G#m^E)P|14x+-@Q^A&+rTiNC4RppF`O{oU% z6Ah!V@GfTEv#EfF9ESzx1;U5{6<=}><Hao$rm|9lxcfGCo?ai+Hd5QbBYM@mv*=61oFYCk$?|4JWSXc-RPUMqLa`me^wH* zX)(*}_|!Gw3GLgw{~)lI*R#vd5DP0A$>a1STMXXni6#fyRXJgcpeT|@NWY4>901y& z&%*Mv?a{1s#&k(1|@szXEjQ^{vc%127P zPJ3e*6}zC6Lde&Y3oCMX!qxPGkv3M5;JR~v@b^uclpYA^2$aT9lu!2D9mZ&%zx_KZ zmXF_lj*j8m4+uh6_k%xul>bewC(PU4K0YQ2pS@*`&%4@DRBBMy4{R}LRd-r(quZ$m zls{!D@BEGKG8R1k(n;F1EoK%f75h8PB_tdso^VwA>;qtm(@U|7&}jj(gBj1&%ikv{ z=Iy!(R7AwBFrCy}CU-1!>>m4@C&$YL$8M*kTxh8KB&X==y4-*+Eo*y|Vv#Ypf%-Zi zVygSZu5Cvyxv^`uWa$7Pe(OR8k!04&N%YoF6-G|3!J)DA6{2=XZu1wFPT&)w!RnZ7 z{jicE%PrRt+Z4Qaf^ie{aNu9qI4DmU10@fYD6dn<6W$)|2Z(W!jdVs|8V|=N05hgO z9bY&0-El&u7cYC#ItbFPNcAK9kN5F{_AXH@K$!Mg?$@C8dG-V7N`!AQbfKq43ek%v zR7p09n&KQ>qmv)~X7_@$lba6&04m{heIseWs^cX_ZakCGJ5jK3?5WzGpHxAWEiS8y z!Krezh&dg{-J`}s{ul6%yy%0qA<#Gm(#Y~9#h7T#2w-QEOwAO7W9XpQq8rC+Kr)0|Miq0lBs=(fpH7?N{xwLFe0@>|K3XO6zZw2Tx^{i@A>MmUpZiekZ?;=A3AR8Wbgivz8%gXdiKYvV%sJxSE1-svjjCG+jNNNrI4TbmuM%bjgcF z_d-b)4t=du5|Y7jD%cfC?(p)4M?;OR7W_Hs z7xmTlnw#0JFyh)y8GB?qPtFB^F4myHkl4(kXN%24CyUuo8fcs`1SH^w-tNF(_v%$= z*A1qf9KPTgTL7DTEpFV!Y(zrr#p42;alsh$5XeW1h5%N*n|~~&;|iB&??!`n!!Ats60DE2ss@T}9oend*a9HW>@m*P zU7jPPs3LuRp~JrN!1lq}KM<)Pi8-aBbbd`cka0@>*-eIaN)L2L4dB*lrYL*-%+>GZ ztG#2l0$nk-pfP}yQ)i>u=~9@44YDTQ$~$KdCc0{gdSfcVY{2DJ{?m5?Uzozz>rVn- zXa^>|eVKLEzC7xzy~OMF9v*#@fKjkX0$7)P@npt@hzoH*s)gPPWYcsKMutL1?OjYi zX10-r4()NDk7-%yRlMFVCZVoUX-Vpis9Xg30k$7hz05U;CNRxVv(V#5-h$DB?=vQJKxyF#3r*BN}a$?82qmP|R#NqEn+gmouhR|zoG|tS2Q6DPFMCcnSjjO4D|NNOU1vtz@(9@;^t86&U9Q6puRGrJye z)-$S1&9;_QFnWVBSq|c!J@42)U#HP|lIn|ziKaxqUU8KYE^ZJ`%?)NH~txS4!l@RIE@B$osbcZj=MqTV9Vw?OHozsg|9vP&ngB zf~&;`zlc?FLK2B#k}ynmO@t+w^}F)%?j;ATZw3mkCp75s?3A!15$VC0D_7*eEs3Vt zK&@}d`3-jDVmIFof0HF3zV)qdg>UDVe;fWPJ9oVQ;oI+CrS|>d+t=n(ahJ<~J*zh2 zNSX~e!1%<^47_qzIZ#0b!uJ^nnMSWAe*~*+hz%Zo5BL5AJKyT!aGSLI@+8mAs5h(Y zFGCG$1KCUtTuJ6M^U_mnUnYmEO>CUYU8){-giT1T`TnEVPsp2v7Fs6-5G8Kghj*S^ zaDU}DFkRaTTgJduy+vonZ;1lN0!k{Ij|xVf;Ea<)xi<=KEsmb7J2^b{}}!{Kad9?_6V)*bXKP-c%hClFmHq`hirLwgTOl4 zOQt8o)HrYqTvYaR`iF{7a>R#eb+P*dIx?&L8x*}8ve@y%Y?SiudrGo*BzF%(UGGan zP+o#eSqmosbiM61l=Q_PFjk+v;zsp{Y}FEQSAo%Sa0MU4bTY_pXxkbQhpPoBd2Y2kNGR zXF`1Jl~VmhD;*1h=$7E}=w7fT zsB3$b{Pk?jtEbMPZz_^>!L4oTj7Y9Qa34EIWR~GS3Ni!E%F;Ac671EC*7YL3|N2AI z>bx(ioz0f-n8QE$gUD&k9fUKJL;$Hdc{==RG3;w&-?`@Zzl*ET9C~sc{$+k{u(s`2eAs zYTTZeSfMzqscNLK#~-rNIedWu59{IuYROxM)v>jsK3-@OSSRZy0c&|X$h%}AMkq{T-&`_xz_aOg+?h=oYar`lC+lVJEn@gq9=9G zw4SalNOXBGbpuk4sSvw#zPu%iMy<-f+six?;b=o@aJe)MPJ_t}Hq!z2q3$&5EWKJS4t}Op?90AFMQPQRvjGEvh3`~hnbs(`2y^qG#>yd*!R+TXY5HSaJFM#(4OvWgC`_8l_gL0#<6N%v=v(W7y&Vo1MsK91u;ww z_($grA9kI)(iTG{BiRk+G%)O{sEa+S6G0}K8;H?Ppkkh;m7n$jj~7{RLv0RJYPe3W za^C?*5>~&XTUE%C#3fDS1ww9dqa796+Ides)>9kK204|8^+LpX6Z{BY6lYQQFl)%5fl?7q>#D0Nr@9AP9xe%@b3i7XEO0=5p>U8n2pnUAsnUXN1K;gVuZ6 zEmRKkTGVpc-g#OF%g{1%u4exye3&c4WG(2-oXHKcA*y#PDuo1t{C%gIQH!ob@$bUeXgq z9ML&gv<|gYr_%!5+4XV_BWx$DQ`F$Pt0i?mvpKGUKd-qP^8Wt~fO##@J|;jz_C4G} zyb*Bgs$`mmx2Q7FAoEd(S7`&h=;cA)h(UoGK%6ySw0>1LZHEsxpZgu1^9 zb!maf+aqTH?*&%WD)S#nVU#5^vOflQ_Jqlbf>T^71H54!U<^?Y+G?%0@}B4@Yip!5B_B9fU7r zG53*T^7u_!irYStJ&mBt22ti5a3(Z{1={>`8&3%eC^iC`hAI|s_}c5Gx_>m*5a;SG z#!-RhR zat6oJf%72nT{QRAKU(Y>lgmc3CohJE9WGLxe`)-qNk?)ca~909ad)%axk3O!wxOKi zxSe$B-`nv#1BhjTv4deVhjM5{A-UPr%D7__ASBc9tGWs{>n0UO{S7xd&gLVQZ`~o* zoDYGXfzVY?`R}3EFN|UdVY1>YlF9;w#FwCtIyMa!|?$p8F{;t~TcPL25Qw z2Y382%@Rv`(Ew0JhgEz;8r5hd!1{lDKcTy=-Q)ia!{eAyu|v@qHmf`sl3fbc#s*Jj^SrOa`cF@reX{hYXkMY)7c(JEG zfc`3#3JbZZOvvSxaR#O`1-M2&ji(1758aiBW*~jedIu^Q)^xE|ZJ;H1`HTkYdeW{~ zhxw(Q=JYJbTfXB{jBR?xSEc+BA0=kc6035bR(yoyQFfpjsAAk#kPlp@k_Vr@SkQg8SvEjTzFkwKW6G7 zn(6WecL`6Fg$Fp_P$%fkWL{O-ze{`yz=ByJfw>I9GV}Tf+Lp5&GL?Dj9JM`w-VDT0 zRI+N!B4F$Yfjc~-b3vOX)#GxT$*WUbq49B}^fWxrL*-`n7d+@~1!86NY#OjpHssIY z*(BuDxNml;pagkE?P7SY5jX*=9vBQm3D6=-eDT{Y&g~@klbyI0%B!|V{x1C8KbC{{ ztJj|&ANt3Khn^|N?5bs>v5kmzonA%u*bU!82bkv%SVQ#Y$z@b6rhHr^n}mOOl=R@H4i$Arry{8_ZBxN|U3o;xd~ArmK+>73x}$3Dh2YSlGOs`PkcE!J{CsC2Qe zP>Az1x~5o>+$b5EEditsSI9+Wm+6(;q%lzku!v93op1VEbUhLYa#eJ7T!j?iT1AUz zaw7{#b;9H{x)s&O3PC*GP!HZKl+uS8uS1`})+Vt*Fz(;2J`RzUrO6O_W!Jg9^aZ zvI`jQ&*2@~`7cx|xL|-Cq59^|TOge9Hlx?&_vx1MDP;>jdA?8yJ3o?^m5zzdk>N9t`MbMOAi`znt?)>4<-UvjM=RX z7L}oH+Xu1`M{lw(-hRP<2foOE`sv&6-hL_n{Nw8v;r-9ve*5+-`RDfy0Py~kw@+Tb z;Gcf__UY>n=v77+*PZVz4DvN({g|m9l^HY@mwjotF#wM4)=84D2pv#KZoASeT?B6<)fHo+YU&<=P-KWa$+iG`Lhvp0?`L6Phy>YR$#IN_KBG@1_q!`v0`O(Ym@G6ZD22Rt*+@Z8 zi(O|@=5d6F6qD)rk_ARaeCi0du8{mjYm}T`DIRh6gG4r-eC%}YHs#U~GZc$xbcDV$ zd^di}P9xr)1Gw+q6NJy2on{Z0!!L$b!Bn62x(UthT{{W;pjURRF;{RPpHu=~vW!K! zL1(gRym)qENDTm$Wq)P0*mC8 zG{Lkgz0G0?fO5c-DogL=+GqG}Ay&B-foJ$OpGh{srAnL}Mwk$>&tEoDX4`bGnpK}_ z8c*>B-7C7N_Y;S2_Xxq*gALEVf!cP^==agCOX%dc^w(2;@LA1z$V zd?tTgAw|f{7#4Suxl|U+i3GW#M5W*kZj&UXl`cF-cxlq$!SbSE9d>Iw@nZlsm zT)ACXGoVnFZ~1prd9732qS=4t3D3?hhZj)^AUUzyW$9#pc`~jN8Aerlbf|G(wQR+weembW3x8C0k%~B=rda`AMotcl%I!%Fbi7kKb3-zDeM9Lvq z+rwwV;DIa0{LEh9rtVo3i`H4{Q0(JyH#X{jidh-@Z?>gys2^oKlMx+m`i0(OcOj-A z5Blx3w%6<^mstX4)fKaJse6=l0Qno6fk7P@U08}t1Nj1Q-{47mcb1PSb%Q*|>3kt? z_KeA@C6sV6Mw5b0s0kxp+m%xCkrD@_vE9TEK$Gmycd+2Ta&iYP(4%#G1^b7oxEbB? zQ0{cs$O`thb$WaS|jYYxxXj2ZE-L5 z6Uhpp&=lks`RiuC@()l;SN2DYdN47kE}*M3PS=Ud1Sw#S!YA=eF3UJubujl7g;{#csnRS^5C1w z1bY>MBw(5xo?Pc?T)Kn4ET7T(R&$UvX;cpY%Wy#|Fe=@{?~Nzi?(qg2od+bI6qsUx zIqLrvmqw$%pu;aqY*RB-w^FxZ$~h-W%|N#nb4KK8vk9iDquTDJ4yvg#GOKH$AB-GtD<|}lB_Qa5ty!$9E-XPvh<-c8MJhk!(9}b5wgy1U=AT{ zru}srvoHhrW`zm{XASH5%iEAbxb3R0$chidt{~o;r&$A+*HQ(({wTZ}H}0WCWZQ!u z`X=mwOW95u`l(-AyIKQaCuP&la&~Xg+B-9=Vo9a1?&G@=y)m%=J%pRBcNlA1>{e2k z{gV2X_eF~-g}oZoBdtg`j~;f^1Zr@Ts8GAh&KlUKfgQ-`_uGKs^yVE!cL|S;P417= zO;ikZds1uo!vO88*N@3$`bfX~*vUpwJSAL))e6dm6j7u}Ed;SZ=75_-$1zz~9pvrD zY+3M%EGa4FFy7+>Z2I7;SB-+K=hiP)WWD@2R6UZC)+yX0wES@oH$a&nEaIO&E|Cn# z`oaK2QJ1Rw!2YVW-wJAM^chd%GRyVR^R=Fosi$W>K-AridIYd)?+Cmag?MOl$M(7i* zj0Xg+SpvVS4b+VqEE)_I6~sbw@=meAt|ovo0c3F#vb#v?!UPLc6GsCy9r6rj@o+k2 zdVva^qUjKL(uOeTZBRdUgz4U5l`*3r*R6vzBBdf$z_7eTg{;%6b_KLGam8n#Kw%O$ zCZb4t=ZB;49>dF&**~%>bRLqDy{JoTZY3f0X+lS>sJv9u*jqdAa!fbN-ZU)ZL@w?c>Ppagh+qxs-~>} z(M>twbvb`>S3V?^xsMR!>S6l98@Qs{U$!=3+KRf!-oASNZp?rjE(UC$xoHea({HRH zgw_YdFR^4sRF~)lx`SpZgcmCN`M~9ui#9l>&HAzmpbKPZR?|_NNNv7+-uwQO*B`z8 zG~|DQ>GQ|%`n4Q2Qy5f6B8Mm{5tNtb-swF_TQx9&?9RxJmbOZ%_m+X`RLj{pa0>gl zzR>fvCo1a-&8QCCnM$tuX!r_luS+;Diow1(;TKLtD9#smCmut)uSFwN8z{~P?tQ+c zyWsUREj9L4o7ajLwP*psUF6={Q=werINf0S+N)h*L)L2_>e*u9qK+a-f|QLWMdT+uVlmf*ewbtrYVS2Vf2pPnRQ72}{`vw1 z4XNOGV@)>ImumIL^8dfqF0H)yT>jrIT2cbi5U4o`Aqb=Phylx5=+P|YkF{wwxBgty zj*^oZ7=}t5M^<)?U{=f4)W&LOcyKhRiq4Ty6*wD7v@TS)kMhzkaF~Aum-#)b(zu*Q zV_T^@yYiA9lu=71>Ods$rYxC+&r+0-z3dz!c0Rs~7r&%6bo{>9(0^XO`^(qguyOud zi@nQ%cqnJ>!G)5Lg+v|-Gd4?~h*FS{dCLaWMeIc$pM8-LmY#@pvYlNe4t>?7iC2S&RYu;-1KK3suxYO4SqEi6K@EmKGNfU5kJoD2R$~FYqPUWMNU? z?XIksClz(SR~B;QzVCHw<_g_%8IUN^a>KB~kS}>4m61p9{|5}%pPxg=^l5^F(?PeP zO(X2iS5@9#S=DT9v7}M=R9f2Xb3s9?No?j$GNvc>V7ATMLd5~7(A=|uIb>4!L6)h+ zu-8NK(0Nk|z{WBQ&{kqyL))NMkmR&8O-q>;ltIt3QyZriaZ(S!s ziiE@oKod`Ra|D1jawu<%7(kvCzxD`YLVC->EG*B?p1hR{FXz0tyOMNIcX~j9w?U{K zCKuEWTrtdAq|#CeR?y%VHlw9(zbx^n2P@BCJ2n?$dV<&XyF%Ix6AQa^;yOX}xd1h4 z&Q1&vu03}}JHj~|(-i`#3t-aPZXgjUYip?aE<|)7<1YJ>U{PgR$ur(T-Y*vmaJXc+ zLN9>|Eoj3`R&@b8X<4u8&8A%4y7wJ28aW(py1CbEv{ch}Fl(uxJYavI__y)oSqAXY z98X-`+41kg|E2$aOdPX4gWqw#g>B&WhiER~hBRX-)#8wzmBdMjV(7bD)|UWOAHCX5 zNae!1M3=HfeH>+t(Ha6^;~qW5y-`l5k&Q(VWg6xd<;3dae<$;P))lKahc%0!`d5+@>kN3H0~LhmZPABp9>AujXV?87@04DNmo8(cdiaD{(Ia<6 z6S&cVO^)C*^@s5b29S1d=*dXXAg4Zgui@8@iF&hj*%(r3rFy%7*De>bN9h`606>Ag z-oO?JzwwRmzvYd*%xWYpQIg6kxL4T>Uq@IJJhJJkp+_&+hIz+=*GwQ%nfF7^!0p9R z>>7$Ig19b47lwpwQZ*}_>$lIBp5G(qDJ-(_TYINsMmutDbZ-ZMS1ew z!cOCa-egz$ZfjrEbXKGeyb+W79~9mDF;+D9N`N_6Cy31GL*Yjf;*?e{o{^Mdhd#bpq~_t-x7FuAp9@5zA~o{w#&D z$t5I}QnlV>4dU}N02eMLkT?5p6*;p zoERNjvPId=Yv}je0iYfGEuK-Pt=Hsr%(*F)6_$WyuL zB2@o@`MCwmC>FH48(xLI1SDSV5(+u;xgsF~_6d+z;*ZbSSd&WJiadt8Es1N2l$7() z!OPqmbgFW?L6&@#Tul@ z4$9(_W>`H3skPFA25%9;d65G+V1-9?!CZCl3|+h3GyoxjZsIvRNOw}K*zT3LY6k=f z`q3d8qI00(5W`W)9iF2s)K!8j+N?!Nd{r$Sz9xSg3aBU<^4MmlgsircN5kC&fVL$| zNsEgnjdWZbItb9GyA*J{AuF#K30MZK0#=48!UbyBMnt8%L!Kp|NDc-=bBm$q7ZU7@ zJEO)lm}+Zh`#cSbIJ-3`&Nmw+eb9n<8lQHT7y&yCB;k&6TTKsTxh_n30YLIu8dzce zAmnM*A1L*!_}3>v|MhR>JKxXW`Niqp^$q)`djm|6XAXc!Ovu*cP3ylE<}Sy^F2Q#@ z6s_#>$!&RwYK7W(NF7xXCZ?XvWH(yNNT%4hFtQq09$2n$lT0qN`u-!SX`!U@JW2Wt zqN3XsTqNz3k1G$VL5{x}W+GK+qREXXSseoeV5(%pNgUFvk0o9krL={q9EE!&=K($RoHP%(Gr z`&KZJCMCD@U}`o)tz@3AlA9_!ZI~$Aat&-746#+b=H<|;g~(^0py?%vIga6?sB}tP zToK|&$q}AZc0AK;Rt(CDr|m3RdMjl=dratR9sGAMk>cZLotUfJo z98fD2jqInjphaqcT6hZ6GJv!A;&BgGeqP^)yP8D9Z$0=#yf3-lk1iSVp5~om=fLi_ zrj7;1G?34dxeZ;`f-6FL+uh$a_q^!6i6qik6+=rg6`jL5%vA2Z$CT0 zJ^KcI%BhTLD@xEn;1d`A3^T~yp-8L72;TG|Qm>|#Ds@2K320*DIgli(#WlN<)$4U{ zH}(^7qLN8Vs+7&@kN&)5R+!aQMkaWz zT0szEQJ)mK5)2e4>o<2Z$4AC_mF&liqkGIlY1#CD9xaLf0Cf2OoUdd`9e<>3dbS*W z$*!e&$&*i&4 z!BB6{Mvd&qWgSkxX(b|h7Yk3s`nG_4F-HR7>_aWvuin1+vu?ttZy4*n|Mc}oZ$D%h z2MD~`?c+SquDL&x)PXr|J#znsD+QO9D)=~5lJ0wirYtT$Nz#`G8rm0y?^vjn#q8zd zKZXDoko=)87Qe@hrhWLZ7HE3Lk=Gc= zZHg&)J^?$QpJ=f?e2Ryf1Mkh^J_j%o=66g<&YaJKLuMn79VXE>Sk?MURwxZKx#)E_ z)(;Yw>(Vy@o{zHhKz4wqHS)6Mw*L)spGA^^$jD;?aQD=?CfEktCG{JDVg2yF?Nm~+ zzamkUi{aUt^tG1k7LaZ#I}*MOoeC*xKh1Jy|Nep5>qme3j^wIs-x-LaE+IZ;=wx3= z@*43ml3cGuokK!IZCa$fbC(eGUqYYbpli;W_28RuRep9QyZIEh*U^A?=6l=gfL63nW z3lIoh-$fEpYL-DUjs2qtaRmQ&OheXGad82TCo65o#wd`CGp-#sCPSLR>$yp#wcS8e1w#*H7Ka0& z#z8Ka2umCHP1r*LJrXn@6^7P>_^7g$Bf@J|9r)I_^Tz%k!H8Iz$jId``R|6sGs+4~ z1&U3k+6OVaShE2kQW$UUB33`vdm@wrQPzY9q*(V>>9_+mDXU4Wgtc{{0$WEX@x#|+ zr*E*L)ZOMKON!x#?(Q?XZKQiDZfIJhQfuhZk-~+_xY&V;bdkMqs-CQftipDee;m52 z5or*Wv;M-(?X%bH;3ie*jb2jsKxJAuwy}?;mgIpudPV=UDLpO8^f`=AAMPlAni&opE zm8W^2dXT7DYZKLu^fNCqWG7vN9Aqbcw93c|-aBt!hri6{enbhjpXP=lTo)j`sR(g{ z?P_yKTcL*>awcb0!Z-;^154j22f^~npLPLCTUsBz6rj+ukZmb`y?B^#7I8H@uq zsQu7;cS|#1VRsHIR7&n#QV|F~5Ou|(0-&o*Gm+ARymoENSJTD9m;}umpM9t|MD=zo z5AVT5CA>uIRJL7%fzG+R_;y0FE5}DQ^~VlZYSJ`zw~GHcutLNY8QE#p0X^Gio!niL zC|IZ-s9uJ+d2rQ(!+})iZ$FcN{^0GCzj#(AYWC^d4_-fg|H+^J`R!NAyEQE&(8We7 zD5~laZHnaD(O%!J zK(e+C4YA7(0~O{%puivpb^(r7B^Wv4TU)2=u1U4e`1c|tjok5OEzFY{CL8u?(SBtz zJwjti?UpTi)%D^(Zcr0To`FlTZn|Vl;~cD@ z$-T7fVcw9bBc-r8CRCm;p6FyHz!3e0ffX?DUZ{TQd4Gr4)#QQWYJl+x@~gICCB=JC z9Q67)=+dh#t&zQ)TyU|@7nlo@QcH|B(%684(V1M?k?zRmeZn(&fLj-oC4^|x17~GK zM34(QoioSoq$2z&!=&8#2*^D5h>P;qovE=SCbnuJYsV0Hitv_0>od`EcO>;0zOUMI zl;%U=v9xcyA8d{yFv9DyUyMOGr&-ElYiA=AT9!xzZ-tix)=M?@&8Fs7JiKbx*FWUQqVYv{XiYpa6nQJIB~q5aJvqN#`H0ep zhkZ+4*fg4ul0Xl%{NnR^rOMc4@;AB5_nUHle-C?idx7}-Q;ol;hsyxn(Vn}bvtl6% ziilIeI=CXgO1#n;B0F8LvbD}r`iW33hfLse6&pJBQhNbmu7fzSY)%K{R}z3N$MezXqg_qvj?gYoE}? z=Ipr#{AewLVs%vGAr|YH0CabUn94%+KUs+9-9FGan8oEjHcTD3 z&*fbKlXB}EAhS!N35IBh!_0Pr>UZrgFZN(S&_hC!!@KS4ZIUDIaam2E3JyFj@s8Ab zKaxUK4~0xEo!GeUq~r=pp6NN|#6Fa`+!M7AmLH=C07>K(+g9sEll%{9qzCH);iqe;hX9=YtOHc6&n;;$tk1e z3HW!~UV%c0s+$`WKycU&zfKPh5c5@PITtNfb<6x{<6~qd4bgp|1MgJl%~;GD=8FF0^lp=H38YA;sVA#W&B#|S9V$s~=`5F|RL|tPSA|EE%z$3I$Z^cF~SfhAo z!pngiz^q=Lvzg+^Q=2BHj0=yx7CBOxTMuZGVA==pjBL@Kxjob*V$(6i1!yoSb_bUX zNocVqT1+ewZ{&1!?G-c-O9=G@{gw=jJM@S3xZ7sgphFx;joiYMoswAxR}v!3O$*jS zo@_{Ts+_WtC!)yk<~&;}x5v`AIjgYqkkm`h43$)>9iMzzkcC9%!IyII?F30hVCnGF-os2_OBwEhSrcoY z$2j z;LbC8%41_vPm19lQYM1X?mGW@on%d@p} zgg@n`?Hi9w234q)mR{swn}yEHEru?n<|oQVHVISmowt&b4`0}nt6Vi1{U)0M=l8M^ zI3Wb~OEQhIcuM2=nUdL5`|SP(Da~7?uIDW*4;0C=`UJnWq2iQqaidn;b8VVR=Z#Ig9A;^r3(xE{hwN$4IzL-taljaF4 z#}jBO{(jI#(OsP#_4Ug2Ub$@3NfrKrsWB3l^}tM(L(BWfVw&z#2R)XDii zyfVKN{*8Wow6+QFUD&MZPZX25turmLZ)&+^=lua)>J{?s9QUYRky0n@i>!(aa^*fp)E% zTwy_$c7+QC^eXz!1aQ!KwTS|2K6nnSo=pR0<+(CCgA97r1R-e34ejZG-pj0_<-kb( z<=1~5_ySD+-@pC9#5nU_r)t^*jc7MSUOQAbrvB|$@b-X@9H2C+Z3^A~Hl4Du*0CN^ zJDer5@LbQhF@XbJ&-wzIVpm&*4o*c&Qo*57{Z>+z{!W(EX&;7C$~81}XEs;lIh^wd zc~YJZhsvO=4t1@Q71!{BNPA#c<4N)kgh(`=`MjU93A0M{a_)cStm({)yzbFl-<(wM zd@kq=Nm*je83013kt$9gQBN$&KYsr8e+X~C%zIcin+vc%WCa4bG#lD5K#)_QW2X8I z2F!*Eoi8=T@z6`OqCM&8r%}kU>_n|GSS2nVUApNCL50fTCUQx8YOY2g7iA*H3N+5x z5cYXlpgOb9Miu<2RW^F=x^98V3cR2KFvs?2+IKpKS#{&**;fWKmOC7c ztUCqZ*lEdSrEiT>w!fAtqj4v5OcD>yty86bj2<#o+{u9XHS0MAcf+R{fUuQR?934c zGO}e89{J3wm7Ft?GA#R`0(+Tin7&%ldzL_j4o^?nw+-x?JtPf7O%5imBJJgOj z8X;}CEW4&Ex2W(n$z&I-IjeCMF0cnFudl3&(dI76!JJ%9H(Q^9oJuDAuAaZl)dqxd zLo}hdP*n|kC$W=BD{Oo0>_RSPhYxD@oP^Jgk{yV4meLIRjljfTE)rs&i!mH%MUw%_ z5a`#9!vGLLl=$tYn8TdB87$j`ri|5MEjfw3?e-GnNF^Ivfg`zqJiMqi)2^80Hn#c- zFC=VR^xqdWv8-jS-2+d**M~d7uy#-Zh3F1i5>Q>71#-mgc%Bkr3*abNn)k5oXpfDk3pFvbf1xX!gi<@XGEnt zi1zI#1_p7=k@|8tiU6XxxaMU4L;JbiA~UI`UA&{BodRzZ0OioR7-oRXsuxaEHcO(K z2#|BTa`Mtvs-t$3JQadFY}EmI9-cBhpqUJxfD#B>4x9t!YOKnSW&k+%>A7X`1J(}6 zo3u!4?A{kMC=7ZJ7H!L9~bJELPa@ ztmxpw2o%TC8Uc^dA}sROvK;b^iP0aGeavpkq>h~VwW3Ouyyw0MH)=x3<3-}t+9WqA zZ-i{K#(OX=tScrjVxWzAMnl^Q3+36WkO}z|10s9=+mEDN0Fdj{Fx>mmVCNU=0|@^E z_3IHjbqbzVK$0bywl4-a?d)DYQ4{S|!s0|(K)JtVKR>0&%MJb#f!Gc1X;dPrQ4AG| zP;hi3wbTX`-kbW2AY;iVvDChgLH^YfoI|bV!^||FdO25S!O$jO)-!G;fRcMO3Tzkm z-Z=voNv`vTL=*JU^OuVkl)h5NQv@)&2H;(;r9Uj;MTe9=&QHSC`^>qhd+Mr?UV5KGbv&> zH7|}VK=D@ou`NsTQ9nW+vbmhVoL<2(9w;zhH{+^#>eavz34rZ9$kS8`PkEp;9C5%R z8Mwpl`od^Mp{#H=`9MLho?Y{&{TVVPE&%+rtDwy)JKbRu?J*5$SB!>}EL%_KI7=He zYtV)QYebX>IWtfaVAHB2bw$G_wBf0l4|&e|q?#e@E#3m>t#T)z8RzC;t?c7+U03p0 zcfhGFt7jtv$b>zUM7=Meat&O=?9=p;t8mZ0055RW41b;1;AeaJh7^ zg_c8axj_{3>w+zF+}#*^QU_83fwP;JgRq>2Fg^$$n!SCUijvN<1~oyL9!*&r(;@)GSq zdv!uL)~&ATIE`M!+s^}EfMfBu#tgUz^>EBjCsE52YG%d-SNjgA)<^-vk^y44Fzh=F z(p+nCh%XJIZSwaEfTe3ogwbR~jXv=jl&bmQQPm+Lo6sSw1W5*MHAC*e!^SLHTW3W_ z_M*~se0)M8Rz`;jJD(IlGm+tW1KQej-!=*A4MTp!2TI5d!3! zQj_ZKl3z+LsbJ2=9@Ffq*}Ia;Tgx4!iawW4vz!1Gv4S;{R^}ZD1nP9Qt1tWnjok=iXN{`=TE3aBO| za>)69q|UM`eEEGRhZ_!!_Y^5R(GKVsvAsQIUvP+5pymN%ZOWzf`gl)mL(zxq$R0(p zbFZKxe2UL*B_c#n9QO!2WI|&>{`~C03|DYegW&Z`f$Pe-Jw|@$6 zrauqW<@^AFoXmfHzstEJ(&2obR9`5+H4#s+7Hsw4#v?!Hs@Pfrw=mV17RY%Lk}wa@ zZ!E&g@&L|0bnAPU;%rA7eYWA_e|-CqJ=uklI#^{sZ|GbSZH0{C)~390=@|{a;*_mc zH2oC05$h|SC5(35J_J%Tp_JE^N~vvLXWUCVk`;e#LONj}=h77rzFw`!*<`bsK`rK{ z#OWv@**lRefIj>*Du1y@4tTf_9p|D|HK}; z24Dc|&u;phQ~)VIW>j-T28?l5#tb6x5{0B;Sx4Fj09}vE?u(Id!e9eowq8qlIbV`C65SXNnoK4W+{)_Q%K+8JU+ALZ6dr^mGmNS5 zbNgP~!V;vqg50XW))^05pq|qSa47lz3|;wl z^LEAsXuHv`Z^q#mG`skqAFCNJ)XVEkXaAE4*}l&kQuYI_z^JxVoi%%~<_ z;xnnqz}CK|d?WX(RH{M6w0Rd@;47NeEBaMZ&f*d#HsJ|6IWK@e@0dzBH;JrAxjG6{ zbO$vLSL>EiLAk3y5VNVUe%PVq-_J!PXi4Xt7%NE#*5Hkf51}D;R(Cf)pYXe^8^;R)GH3PTgG%22z=jc*TC zvbtx~tni#!EfCig`-U+3pgt!=2(2pk_&F-Gk3Yd(pOtf7;F@ixMj@y?_jg@{1*F4O zDn|lLSGYY$B~N73l4^yH71qj! z2k^K-mtQ}HFyj%!tZhXe=|?c7^E-xK(e0#LP6J^(XZQbl53_Q}GBan3SG5jXFEByZ9LygDD=#t({A`La2IqjBLhk6Qq9NmFYwgjJGwb0h8Z$^PcBv zlDrJFyMQ5PIP{oraoErkWU2B^>PrED;3~dj$3YoJ_Y#N<2=944m09_HDnOk(@ z_*_?|B(~!db1!vLu_yNO@&FayYOdXz{1CcAB!NCQATkr_$IAdOJie>3*3K4PRU@P& zCHXF~_D&RNCa2b5)fLyetd~rbNaF&G5o27-1SPOc4`ntqa#w-7n*TLP`YbTP?zWd( zO0A{2FCPV!e@%|uK{u7IDjH!gCfEoW=5%e7VIz4Gi~ zya2VhTObSpB4kZrjmiq>{rWOZ<>?#is=WY;Duoex0X4@*708||n^>?Atz;#lYGYb# z2oCA`_^ouU+^`A6;!FonD7r`qb^P0mDa69!k3>qrzX?1i1GBb!u^Ta?mGy+k&t<>d&} za>CP!$@dwBO5+k$TUH6QJqX^$8KOerV6p+A&a9RZRh05k%W~V67_&probF^22h6HrCOIK zFszBFMyJtX)Ic<|f4+geU%9wPlJ)z--8|R&+CwPAYMwJ7c>ZU{sRGqPZB>W3z|~eB z=G9fgE{VFjl62tCgrjKV5=)Y#?=C&B!)G z7jC6+K;@zz5~De_y?KKTbQA(#nyODxpDm84!)IG&K!O9+aJ@Hp1DhiUrlX zWL^SMX&{+l7{qB!ugqVXcI=^x;3eUFAINu4z|7+8vXJQm(K_${7ryxk5YKyBoWfQ{ z4LyKf#fPGeByk6Le(1ux6I`Rz_0UbO8h|j2;!(En1|`HXP!6DR8j_q78*2d9K?A|8 zUrrMGCvRV>(B$8~{vlYa=`l!jto#-!y~b^-ZPAwtrjXtSHD(crYFsSOYmk1DW(qTb zhN%d8HW8vgoX-~-R&f5G+nj}`i#Ed6>jzsl&7o}uHR-5w(1}`Jwn;fq-p1|J+bW;X z0e7lh1z5MzMui&)s6S_nffc3{ZcF}I!Rv$*x(!u(a5HmDp4N;I;UEY8+u>gvMxDE#Dzz`B^3N$%D1b$J? zD#L{~(F0W&N0NY15MQvwgCwzhfdzzXEqMZ@s3{175JjK7m30qq^sQkHvA>) zvwll<+~5>zTqoz%5ACxfKw3=_>@^qwCcsbypL-8l->Vo6tO*txs6>_J8QaceXRpu) zDrkc8(?DVuSq%IX;#m!ep?p&_jbOEGCI4*DBS-#{M)=SQDIItWZ0@e z4AICuOR`JA`0Nc%HOR%zfb1R`>!AW>7eQM_P;<}41|8`V#)*h z)b2DRr)jlY9uX%<=AyzqQyHM4{2J3WcCqUyiWyZd*J`zR(4tlD{E{?C$erB46_aR< z78WiH9O#QNnIl@TQn`|yOsb_yMG{qxyItTm4*4;AlRqz?{Z)9CtK$1_KLktd4pwQ( z@Y+c>jV8^U;i?D=#Q-0K&$W1_`T4H0HqA5`ojW3=a~&X6nIF44zFNhJCw3@6Czz4g zlw;>37`t2%*cye430x?ALGl??Wz>j}C(Pt{V0REb9;EILo0Ig|K(LCwKQ$5q$-T$j zRz^~iA0HG^Y6*?5`XV>B>DB|Dj^_p6U_ySe2))7@sa*=o*%wLC(u@|+!}fKL$)rsx zk|*jI1j;^a-I#0bAfn0?C}6O7f=<7BQnZIZIAgN?6gi1O3&Li3GZogMc~064a!43q z$ZmgtmSY!VxVe{Cp;l5hAPekN1fO5BT$22Ey0lPH#>^rhEnq9ynwAtr-%W#O z5$qUD!h2V5LUNiP(7v9-eZ}Mm9W+8AHeK?K9X;8nEp*Xb8)!`+P|fBqTs0Gw zZ@I;xh$9iXbIOVh3os!_btFp-(FAajwT?tL78k(5odLOhK|~_lt9yhII0YsTS0zaC zwT_as5d#}BqZ?%j*zI(t@rdzo)a#FzxmuPQvRox0N#p_mXN9K{rY7vW!}e=9#XYd` z1kn=N4(dS)X71xKGWm6Q{bPRj&jhJ|{q{!+i&4_|om}i4j38c;_XdmS(7=2Qah~FA zg#}Wq-Ouql=gx(-q9b^!P`}Z)DiXhj<>8`vxIwH{GpV$aUBQx95CVig5$lpCwKk`7 zk6K>U`}z)!PQ{mY?Bk>Ce`Si>h@`=&&_O?s>1;xF*5Ogx!40_n2KC>Sr?CP$f0+o{ zh=XRlR=`GOOdZDKzj*yY_>1!PN3Wl~Pj+4=30D$%Xg6|Km2~4U0oQoCO>Lc~bC@b@ zF8#bBlv~wCw)MD6j62Is>&*3a!$cj@l;nqQ+81W88Ws?XQa6sA=T+_{8y-ps)9Fj` zX3w_@@atST^j1HcN_b9=NpRjWHhGt@mxgsOk@Vv7hL7o(@VgF(Qa(80Z6Or~2ZONuL#6CuC~ z4pMi29{wA%q51sn`;d+OTn^T+b8Y^>uPS8Ch_z?Dg^O2(%W2W#_^RX6`-DQ!Nz$?w zX8M4Qm_n#9GR4EZWa)5P*6Q))CzY3_Ps^cn6({qtJcKNGr>Fv~|c( zgVY`6go)4F#f@91W=$)bqMpd&*ZrDq_1Ej<*!+2JQ;*khfxx60leggtChGg@W)FLKf4B# z$`n+uDpO1B zCF7}ud66xTLzy3Ua+-lZt5?;z4|$qq4;fw$Y>iGk2RyhONiCBDG`HS>lVGmmgG;f( zg(ooh4ZC_xF{q!hyGUSB*=bhIiz5;y@ViJu1|f{C!s*I+7_55fVL3q43mrNnNTVC- zJWWzqf`M*{sNsbhMdp%AR)NM$qRl`J08AI!Xl}|lI3*540tf}}Pv3qn7oF-cWD6J{ zLrYSIUSJE7(GEqjQ>Jn3<2%0eDtQu3>LDoxm%4c$pXI`o3y$QBIc_-h(|V^p+6;_1 zsY{TCXYlERPlUf2NYpS)9$52E!dq!9X%Lh=iD?;7`mu^5JfcroIh@! z^FS@*QWR-nYWf~8d6DQg7?4kjXlqb{dJGFnY<2nRfpfF4kBI8@>We~v(DQBFvk+=A z)=Pujl{NT3NAKp2C`O{R)N-e?pTBWOLKs`pl>uF}KI|8xzkk@n#8AJ2t^9mK$U!fj zUKR_SUMy})4uJk6Hy~2TMq3RL9c;z4lF|Ioq{||8;TGl+p5itN4%_T@Li#gtY7L8o z6i8)Ox(IKNTETvP%|m2F#rPANiiyf)k-oiZgPM>!ePMEI?jA;{9yT34_^Xyp}Og_!(?CU( z;nxD4YE9@A8c_x4n4pwSXzFb& zd^o1e7_@K^$u@jSa$8jC#Uh?+q=fX04A7NIKkeSs;8%LTX+_Dx`bt<26XBB;-7%DPHo>% z1B5YHKSkbL?!I?2i4l##ZM@t|(hGi|7nVq-1SPSL^mn5P=c;p2aP{Q)JU zK0oCahRRH46q|}Aq3RuUP%Q7&ov9v4?x~74==M<7GxsA9KK@@2LIQ(vqUNGRT8yA2KR1x@EwG^8XYG=k2gyFk z#iW8dP%rYl5OJCt74DgK46=j4p-ijZK8HyVK;{-d zE`9Vt9`vCtr8(nSC+gEnL_z0fjUFB4Gj&$M@VXss{7O@!ZO@WJKgqVZnY8*6=zA$& ziHwGO1yAv3H|3dTaPjEpmoe%$zERNSiooHs1%Rr#gVT-pdXPp3J_jg?$j?EsYZL_( zk6NTion6TRQ%jNrn!qNuJkztuoyvU%emgHvsBZM}?lN-^r;^G~FeCM#cxR^e=|*?d z;9)8^V`}8Ld<}jHiMSYhFis;mvWS7xnAEX#LSNOV(d()`tX0R@4X7AH?jlG4NgPtZ zJJ_1HeR`5TwCE(nvKUyh0>RPk1I3di9W00C*KgnBY51OgR--GOHnD^AN-onhmw-Pj zsn)K!6(Se5oug}hE-4m?Oyjk$;9cHN|QJ<|o-{Agh_=Pu8HY z9<+u*vt{cG{s4^nkFj|6Ji)fx+7+4GTVg1Tu%LDf+L@iF2wzB0z#^5hfqAu)BB>=6 z!TLe({koi*K$3;r765sU4t@p3h8Hs!NXd)-J_GX^&F~e6iikotdjkZTpOg)TWgV66 zpFaiE-z2MdhRDo@rh=_u4+ObQPi797A3Zv313)UZ6wa1!GQ7>eM%=+bbAx%tS%9x zxgQ65!#Mx*N#OdZwBga$y%CT-$_kXV-W1<@VMLEcHZeJrd_ZRc3SU* z4iwx@y$|#ztCNm9mfjEX$IoBCheYNlZ$EweF$S9Lc#?m)2^RxMVKjjLaO^vKTu7Fe zBG*w;lo1KP%bkVP`~7fYm_sF94HN< zjU$>mkY*1sl@yL;pDr!BOXNzOt$A&Fjz6TZ;}5M3aAjZr1}$BQ2AJ;``wB2&z;g9= zHq63pmo5^*$(K}-0@(LLkCrnaw7!+By>Yj}v>`9R0ss&pOcDBjoKLLkJZ5Kpzqk}> zO}h$Ja(2pNANyVYQWiT=%lm=CKIk5elFLi!J)NF{pU!rWa||$zloKT3KjRFXA=Zc? zQxzX_sO5>LS{3Zct^;b6rRKpcv7A=bP&A$%+ldlRat$&)| z{e7^z_2b`#*Wc!y4+#3YYK!M{R$m#u+`(+LFOMV;1R$eHBM!30R%n`ViTu{FJO zdBEOESGX)BjcK13>71_cCE4oG%rx}OQyehGaOv!JQ*-pYE3uf~p1sDs@4$p>wvMyy z4)_nFf0{#M>c%?!Qi|$8%?*ed=$Jr$PBm23h7G1f7NVi=Jl61cYtezV#D5LUYE)Z+ z2IHS)MWrmKZ~Eq29z>T+6hmoBZ@`(S^+pX66_x0;EssRUhoPxqT`xtR;Yy>_kx!h> z4A8YhM;AV-R)p43k!h@apjb+FdJzT{s9^m#g^ZG}|1ia)W{5}eAld&?$EOMS-IKMimHq!@oG ze$8h@J-QcQcq&+6P#8_J8W>wjvfC6-*(IrWGK^gU(QptY>6~kkTmec=MN5NBO6-T0 z79>TF-oF7CJs((b<_t5U8#ixCj9lVnk`&7>+abl~opWmMyxk7)bDIDwg|~?(^vkWh zBoYaPDG4*@yonV5W{10mdacQ8qdT>w%W{B$);fVEWnK2B882bLV)b9u>OotzUqyAb z3W|Nja0LPS<&AYH~Z7?x)ezwq!;|G@vq))s(GJHDItg;iZek87X?4g7=hAi(Xfpz1c#g30;n*9V*y z3}LZ|*rKZs`8$Ig{+;Xsiz)h6gc;gJ+; zl{ya2-h`M!G4c-abIH*D@uWKQw#-0N&=tN|-mYk5Sc?vVQh+5a;DK|z+rp&=-yV%n zV1$0*wZ`gf%HaDEExqu8MWHM!2(R~cE%W&RjSl{{tHDN&oZzmnin(2&k`)Zf#2=6f3SS0jK>Vw%E)LQGkLgq}} zuft6Ryh=6?YO|A7mD@?!^bnupb8lRY{+=S{!O_N$P;4+}=ca@F)*D4n{GQCV^>6{U zFOvGnI$o@nTjEp9H<7&$gGC3;!!Hz{g^U4*x1GBtKke{Fk)vk$vt0QVj-w@c^1*s> z7nK_!mIvF(#4sf4Jz%9)u#*bQ&svY(kb@};g;3NQ6z^=T)~W%TZDAc2S!jVZ^I>xH z+wf);okUs1^QHb=p4rNRdmpuv#5wy8;EvREG%we&?mW$Vf3WbxVekp;*k(ISSMihl z7G6n`r)Jm~(G%XY?WPy_yLY3PupN3AEAiS&G_|}y%0H4o7KpW4+8Vb$YY-v}sEuW& zj!V;id^6%$&wWBwk=N{59Sy)Qv=mA)MruJAfNgp=wr@kOAI;)>7-HgAKi;<4vKIUF5n#j=QtBd7jqdS}vRC!w#b| zM$Zu{Q%#gZ!CR&wh7TYFz1}9`aL{F%W(Y7%u$z+Vg%w$Zk$j+W$_x9$p#2-+;7k34 z0Z@MS^n9_Xm`dRoN^TAeYMCX!Y_@AG`KEoc02q&0BEK^mn za$4^(OF!gy}#6UkZub>mM|od;JmKeHz|=170y8 zIytwe2Ubf#X4p%m6`RiB^qm@e+cLoFiZn5)o-H>3On*%e?X+cfJVbQm`L^^C+$?Dt zcDLY-klD4(uBnB0D=|!;An-JH&=ha7{y6cPXE+BXzFv}o5}5kNduRxdlh7Ehqe6kv z(cDHn=Z%Sc4arbR5{bgk-62MwUF~h4@RgtELltXA3Q0tW=-9Kqp>(h*xOSA$<+Z*7 znt!@c4T zZ5a!iU;!nEUa8eD`fvx@Zo|*r#oeaj9XO~ueQ~O-QPI@Sh+lHcpLPEz-~Ze~HgY_u zk7v9U@@UKM2e}ul?FOd8Wh@Yv0rUL)wVy(|Wv0atyvvIoqJ`imuEBMhC8Dk>_)@p- z8P>ZG+1dP^z`zE-xyX_4uqj?RN=_g4(9R`aeU%{j1AX@b55$}LlSFFD$|#2T@!%L zuQ0<98K(H{S4u^Ml-^yE`_0Bu_jtA*suCzC`xYRVeJ0>wosr0?Qxa=4^b94)17;IL zA&6mO6(yg)dN&dkI`rUCo<&4}4|eNzciQXl#=84}p;suo;FBcUQnhw4wl5m;0$k9&&9=CQHGweAVe%t(KnFdW@ zf5!)RHd0KU(ht+XP|><3`KmpL4V5Pba<;x8PLlK^$w7mrJ!}9ug2w1Mfrl=uZI|@0 zbqb+&9r5KdDG`ho-UzH#mUY(;vODFuI6eisu>-HyD`Y5Tiy4zbWrLxzY_t0WVL4*D z9`LD;7gs-iOy{#VQKb6aDp!E5D|yE9*dmu@)e?^~`%)F%2@*sD8B2(JcIl(*UMo{#vle!+TQz#GrR!ouB&4oN*jH`z7Zf_s~yr=r`1i$*5 zJyuB*h^D|xIqC8Neevx2C}$osSIQpgAeRfYsU_KHz|kIzL~p-C>~2=GKn2Qz6xY^h zpK`ZBxrA5iCGOCzAMK5oHE3~+&SvrJsa6XBM~f}ftezm0)CN_Tn5IhdKu?mOgsAGR z*7KC>?baUJvf%dFk@|D_FZ}tRAG+?4&B__eHz^3Njm7i!S+FDYfN_rvU6M_X9_--U zqC((KH~IlXO3AuMnSL2|w&tat#GkeX08ZNm9gxB=Mk4e?i1u{6y7g9IFJK0c$g3|& z8b|c(Pg-OR=+VQuB0^gcX?YikIzZy1(3y^4h)#jq=48=A4v7`omEYzCS9mmIcv!aQ zKaEkD^{o?gJM{6Vb0)?!3#_kuLDK+?J{3IJ9-d^GH+qv{s>00?uO`EW%2!nuRsBzr zPAOULsTtE+sC&u*Ua=IoOP#2=q54Jca)^FMm^_@rjJMvRM%&46l&V|VCGyo$!HMSa zHzs?^V~U(|Bn?p|t22Y)#}9xnkG!9C*36>`Oz21@=1-cTRM+yEudb_eWkxIIk;+jD=g3 zeSqz?%B@$0b&nL@T8c42>Nfh^k+h-T*rAj7@=l}UD2x98=VSQhC-PQ_eFv8115?ai zEs`zS!=x*(`5lDC14&U6X4?;QpWSf%$}4h~%0xa)I%FYEQ-s>n-P2@?3&bZ~9&S+&X5BwnQUWH;ks zgPIAo1ez6>ogHidoY7O9m+8!)3BPysAG9!XfnCcZe=s)DWd}aeqT@x*{O{fgifD{Okw)gy(R4D1g`a^&CttA88* z{a;xU08)`l1Ki;&CslJ!>6(Weg+X!0G)#}`WTxD+{2DKjP?=5wT9wpYk>KQf37m%2 zWjiEWsx2-#BNpz4#BWW2Li7AhvlJMubrYu^eJ8>bp}_~QZ42vWpNJ zTL!ngT;$5PEp`ZatrKMmgZLJUbPEtbP;3Mc@64Jt7gg{oFC!lLhAe?XblcD(rrlZU z&!Jh`qJbjSWVZ3MKM?yH#;uZ;Ahfvz)nKxbGvq9R$18BWSz!~@-jpiKl|Cddd}$f! zx=Asn10#?sil5-5h7N2=ML2N2LcJUuVny93hwosmc13)!PV!~C4M<+Dd)>t<5t_Z% zs>sfE&snZlxu7~gARxZ894?tNl9np!_V|I--AZS+E<6sFVMQU&y}0G``oh6Q)2>jy zUI*=VuIM(>#HViTa;QE46JqQ47k~a|LH?f~94y|7tc)b=*!&EKXzvvk5;|?|3xxBa zc4iz6$^Kktwz8y3b7u%E&V6%jTor;`AS3&e@b+U=puZxn`c7liyYfr&wV|VB zIbhUDf<#UG!8VWF#0oz>Q-tMC+xyWG;BY107RsUaF*qv-HnIi@c=BeV0e>Ztw3yq< zf1%|JxxizZX)BZ?*(o!Ufu`X%0Bvpzr)$RKbPkvlbLilF2NYPey#Vpl+TH4e2;bd? zN3+c??H~s2M-jod-?C81$hv)l940azyEJvao1Z3V4EII8@J?A$QPgbJ@mWbs4qwc zEkddlYTm9OaqhI%Yz_wVvg=_Q`eGCKN`=f;1vSEWJYrZ?^O^&f&JO5%=k)OU*-4v_ zU;ho-B8L*Z;&a$%wmElH%LHLsQzHo6_ypz912ot)|rDX)U0K@6X#;Vni zDwM%4bP@^K2X`EN+QfN7Aiq!5#V7mMuk^yXv{NRoslKJ%UrmG(Ufr%LG9vly=++0O z6#%a(U<~@+e9~b4yXj;u8jn8~P{0f3>!#t9P&hnvNXA1vt~Gc)1-?JP3%ZAp9OpQ9 zH8mzA3_W#p`1qsA`N{(UjD9%T8J}>8Pb7;=)~(IuiSMBL29Nrs3E6goBZetF@iKtU zagiJu;1S=Yo-p-&fi->G1FQ}_2ib=SfCj0)C(!XIi36!s$rTF52Y2LT;r1dqso8{~ zNwO)JC^polyg<%8|Ac4gS0Y7MlLAP2Ho<~9XYPGTpz0pV6y&?Bu#Enx$Em7nTm$b) znJP0zEEd;p@GNEY)a~1m!v6L%wlgvsBCEw@<(Ru{rwDu|t}s=pX=}g#w}(Eov12bd zI}kdG0O|E^uaeb>sB*f3?mK2=$mOSo8N(QnF3b67!lh8?$V0-`r3_7zw7oSzHRc`D z{V5$pa<5WEr)TW6)g>!)?9pBp{J~7|q2|;lL3B%Uq%OKG{2?u4vBBmu5LT2UJpkBiPvhI)4*w#fjXr(`KjTm2V|e{JRM&3cMvl}YR|F50=r@>` ztX8=iBv*4+BWIIMO5rKU15(&s>t#DVFcXqEc4k9i2@gwH zw1CI-ls0biw+zeGD&g2}VHDYXu&@~*)cvZs2{z#+Bb5zZnhre>3Mi;fRd!Wy=(L=Z zGY}s6O>iXqJ*=eiUSQvxFAUB;wy7vCs*WvL*s@gHl-qNzxwsBeZPuzJpnWOMYSX+! z`-tuz4WqN0O*C})?(Afx?K(B@HR1;t zUH0VkjSa--XNisSZ1a7O{s8l_%ie|;O5H&Cmj?{}Buu=m=0zul4KR$95}nv3y^t;I z=xg9HD?cY&x*%z<9T8o@K+NYVp!5wq7#nKDW%8q=!eXU^D1iuMVO-^T70)VlYkl9W zoj~L)ldbS)E?W&~~ z!Fr%)9(cBMLolncjO~g;il9dA42~+_F+|J-JLY-)Q!x85N#I{ztg=WYF!${7cI|MM zu+I45|zHgvAY^q<9uBbX~eh0v!P3_LWTUN3g zFDgf*_!qZXtlOnl+4#&8Dd!yQyrN{I8ag~VlH~n@SK3S1FOsx$N~17k7PQ*E`7wdS zxc3l!lJC6*ADcD88l4!s>~j8Fog`Zch55{ji>IOY6>Bel<(lDSbpc+wi39SQrjQ3P zAfvPvtY20dA0q}MUn2V$TJ|=V%>;e9@T8949R1!PGwoJ~z=t8yR`h8Cx&x|yQhR0- zE<00KJxGHKDKro_DotWZ;ci))kUh!}H5m6N<$c~Sc7mjGWIAD)~t|__nJea z5~!^(gi?YviAGbhRi!IF5>WQ>p<&B*bO5_rA71cNvPE7`qw)tmM^h>6-mymp+pAJC zAJ^j72-g5)-vRm{gcwWF8b{pTv{$v_!j4UueE^+4 z6G+5*MYAuASBuH!)owZvtJjuhzDw*M19WHBOJx@uZ5=hMFNSwR+vuj@s8Z6??bJ!` z>e?*bRq3Ndzile9fffNB>{SQCe(ozcp3|w!u`hJ8#jPfx76UE$a%|5*t!Gel*A1qg z>bq=5VoWXy*`$iZ?kw&Ix<^urYs@|{w#oMqsACUk{g@1F59v=rIL=a5?MUruB=Yg| zx39wM?}?<<&plGi=6d;(f{4w<~RYuDfZR~an@4UT+>CN4iYm7)5D0yQ5q^3 z~}3Xme+HQwm1$3>pJ+@gTaMi4WX2{`XWu7k=!^c5ETQ**|-v9yOR)w zjCJS89&Z%m&ReW1CM%Vy;o9^b(nM>IUq6*H z0pRS{pS=Ta=Lwun(i|P6*oIXuk0N|`Yn6~jjH3F*=Do5))>~qO#UpvR2Jga|cF4^7 zfl7?9ah2z%Xqyr^U~-sH1bYmcK}YDhMN+0V7R?FhE*8(2CrJ~@;N0cj&aa?KE(aG5 z+xuXbOT9A`R7IFbm25F!I|JIuhN><|VR3`K=m2FT_pHTN+ybjw8?AB^!px7tMN5QZ zfk`a@sk2B*nD*Op9Ez0fG}Lh0S6Z~TJjfVbZ`r|ci4Q*#ZS)ox(C zXCAz@W9ynk6P479c)HdEsbve1XqG>*Ku^(Iw34@DVz8TtfI6vF3;Ko7WMk2F4|n-b zHul>;!EEyrs1f}h1Aevi&_DB%q8c~Y9)`UMQn%4b;(g#_k-Gsvrgk)akV8qoJ|Tbn zQr6%S%Sc3>t?qP;hk*r5sZ{+!o$UcY@gdaHw44r1p^Tw&^JUrWY1ci`L&wNwG?HT6 zsP+$=#dVf4N|Tmk`{qds0{$D&Bb~^BLX#5%!;_>_y(bSN4Ie>+y2);m4Q&**O7829 zRDL+5;RrCF97EalYv~Q@0euH_qG#Gagx5cvKK`Fj;r`E(SG;J0UxG_81nv!b@@knt ze~~pR+aa{3N(OR)e1q89cVcg6h4oR4GC}(iJOhIL#PdCxHs0FFYd#{0ff=?HPDQ;BXO22pvh)#H99w5aONPgmFZi9 z$I)Hkb7t2JT4r=O$;$P2_C7GMs;{l&Z6#@ps_oe}DH7Q>1?ty<9U-;gWk-VATe5I( z(evlcjwyqz1DZ|Su4s+WQX?#lDkT@5tXS^U;BY}mywU)oC+QJ$?vAfuOg~{xD1{Ow zaT03W(U0)t)^e!SR!sH}{Djsf*OL?rtsL$fGjSyA7Y0V;1p7-1kc~mLTNPxvL`p?; zspSw?^=DX-1HjWz3U3Ugd?1AeY6*%G1mLvGsumg3`OG=_S)x)nGc;kD({SL#)gZ}t z5?$DsN1k`IXTUms*qQM)2t}ybtIDD5caUU(vS98z5%G>@BtZYR0FcR(Qiu47lKQqq zfe2;=z|*?8qpDmHM0%)$#5pjTR5?Y>!}6oYP3p*M z8(vpconcI?Hk0hPy^vBFc-AtQ1rj;*0%3t`B613j;kuG9>>KD#fdr_6o*F0KuNom5 z0yR^al%M=#hir9{@Ah$V;@oMyLZ#?g_TFNOH950J$Qdk3&avXP*q6Xz2; z6dN_{K_&2wZ`Q~VQdnmqf)E9wUp^yL8WC@t_fzQ=zo5FazZ1*gB-Sk$V|SX7m|fwQ zws*G-;w9TdjiKyOC=SK@*ih+L?>-LEmxR$v$K_n)YfR7Kpm#nW+)74?i;qw}7F!N5 zBCEZ%q-+=i&66*H>x^h;T9|;15cEVtW~ei<$0NL48GSEFcA%tg(4;X1$AxIV1R?N* z2e8iSA1;Pp7?{K0rrLE<*g&?BMlWL8-Rqh0bHjnKvzh2nuiB77suLYPy$X zN9p{4TM11g5CBc>C}S%8AP&04ofmUzB1IbM2d%fbH_!SO(X? zb@G`{Vcw~WGR%M@XkWzDEy_KKZfO-aqqbHV6lgWM=7?AF&cTH$DDhQ6Z;HsX-$(mz z-8-TgB>Jd8Z(@t(9yTkG?$ zQbU7#!=*Bo9lS)KQSB`}Vdu%~AGu`FnjaonwtX54sZ~M5`);>pzxw|qm!AoeDV$VmVbcj`ulQz{GLXxJGE9B(=$Io}q#(RgXm^>nM0{w8PTmSLi!lD3W@4jH!N?!d|3g zGMu*x*IHSz>z_HLvkeubI$$c9ER(GtRW`59?-n)Ktkm3wQqw^D4Yk&CA;OM3uA z;Q7uvmTFX>*R+7q<%f|YE{~7=y$3PuX_axL<$YckG?+O zj0Z}Hh3dHtt}h}F1HW3Wo8Z7jiWdANR9my`6ntuc9V^+eaDK4YsxWt&E`Siq_Om;0 zs&t&lvF>6-@?k=0SM!b)D|>>JG7(f_a<^bt`m*tG!7uO+_FO(d=?B0x=?KvHjeoWh*1=B?8!nRQkBoZfXR zcQ3RFoCj*zu7R-*t{b`>bPWfU*1`0;=ge{h{kFX%J#mpYc#&>|7fr$hwXkzjb4QJ& zZ9;>DKa_F?VB4;uW_EF4I<=_muHWWB>rBjRG0dSQQXW436cZsiw?jd9S17jdB0N9a z6M5I1%{o3LD<;fRY#zAFH4WWW5Xq42Jo}CD99Y)M{apMDFfquF0GDlk28lKh=V!o+ zfdgf8FZk|5C!gvj5?R3zA3}758XyyaNLxt%Vp1Sw#}GuHik&H_Aq? z%0=KD&|kmiZrw{u@ZVj6Y0W!ebw{VLBsa(1(%>>q>#j|tLVpWS_@!b2JEzbQN(q{o33uZErsjT#G-4caq4ilsWer7)JQheL9+NwTqhM@E=V9W zCRqFL#rRuR8s~zw_!;xcbb{TimfIZhv|Z@TXuH{$OO%cA^p2z3ffjMMLuP1fshUvfTBWv}odd4mV=N9&tU55R04v;pvoB&A=F*YF%iojBx$!+fG* z3;9lN%cg*=4e7jrnIRrO9h|VT4=b5g^F{&+HhVuJ zTV8kxGF?Hm>&z}%8I>y#9Z+`lz7s%xWV{qJ-isz?c!y(P<2oZt5ihNR6OF39oCtvV4mOlk zLJ;)Hl9fmQ@|`E9-JqjnKX5g*jdDv3h>n{BW__!{)E@Wvllctoi+cQ62NZurVsKsx zK=t~mzDf`S1_S7nvO@rH&6s9qE9I>MdSbS7W;?SFT-noE+CwpQ%UKqR^9uE@9J5}ed zU5!a643+t^TBwY!4z3lT`(q8GYj8=hy?DR7kHC<`5_mf0IGe9gr{%q2pV3OUBMM^p zm0Pb1Ui`~*G!^dUk>mQ&(-P9$?8{H}7Q>yvLGqB%^_*g^6x4VO87QB5m?P@i`oN#V z5dXjc|27a=pS*qg4O-jl@BVqf1?SSH_6Nkxj=p(hcmJBufOAK`X$E+M2u>3q@{j*K2B@kwEP9k` z2#oP?XtI0%e7oD#CX0HGC*{1QbwabYN$#5mwTmakGlJg?&&-~>kD3L_d3(ywbm}pk zC7opQeK!$m(N8L23rxJjN}wTUud`z8K_K}uoxOm2Qpwb6+Bsc$qNG9Vut;zLi3|!e z+3rgbcZH=0Q_eF?u4;Bsvsy(KR5gwpY?+dY=QNue+_Z>tN$PobRg4g1u%xq6U0gPZ zF(&YKq0-u>wN@Ec{$(t+6FsslNO&{WS^{$L-f^{{uV+n3^lylL1`yH(GZe6|3W$KO zLiGSLiQ2;C@UJKusGb!RcBt}sbNOL)QZxY(oZab+Y;7Gm`G=8e98a@yP+*El#R==! zDEiFfm+no=JpsK0rInX0*xh`peTp{!yz4^Qf|C5#ko4+ZDo%6f8%AM)j`MZfrZO zfVIFs7s@!}VPb&xLot7uv|1Zaqr^YCz$*>-&%uJIBG-dsU1}M*U&7`)ChG}he2Oe@tC$L(m~K%PR%*hz(F-W znue5!^N43=U5LP;dn}ih{jGg~gKrlZRUuGg_eq~IGDL^hz!*X7+lxM&D{PotMgotK zFeA;HbUx3&6jU{cv;8sCfv+Xp*upbJlp3~S&Kimu;+W#*sp~nS+zzHrE7vY(&!+~G z^>&j31MQtUNgsEIh{4oSLUc9AzxmE)E9XVw_1C11m<}s&a8!3RlZ>sM2?d#tX667n z(uWTsrz@Pb`;}v7ST5HvvYu#PAf-AgbuvSN1+l4zhJI6biqzDM&>mNm3auPt0Ncm06>Xp*=uo=}iGG zOAZifEs`)&JH=|1tPWlK$LZsDG2ep-%sjPbXh%pvzJ4^E^{ry-WX;sK%ETPwQ?S%x zUddOOCq?_s@-36jYu*jjeh}x7N>h0Wd}>e++L(w85E${jU&}$jkrb){R|92?;hgA<-*BBKbz+vnJbnl^rbNLzdb9v;f|ue2gZ>x(4xKz` z>G2ztIKV@MdYnZtHTYJm!chAAZi_O5v28&~okiYtWndn2H-|W>OYHzGewN1vw1tR+ zMf|e9>w}{}`FGd|9CF$BIz&vtg;jST2eAplV(Rm-d4?!4O%ildYmNEh=>2_gqD=16ynZiO1!nit=bVWA z__!9o<68Xg^znz#j#jV^U7V%JeY9ut+SvM#HBx$+u>~ZsnY=mDe_5*gsBJ_fT+6o5 zq7rg>j6>QxXSOl=5Wh#;XC?fz!!+nYgu+7(rhQ)W^BQ&$#sB7M(*CX7rmYY4+VP$9 z-4za0P{)z1tA%wLn`DbroF>AU@_d#}T>NFpK_p`UC@87K)Ik5tlx>}iu7*;}%5ynY zc562+7$`)bNc{G$CInsRiIbp8^6vV`>Xq(prwi zduH<5#67rFg?xE80YoTkeGXPDVz7Xbl`(iyH}A(cB%yvy$KoFc@bf--`|R}-8)v*E z2o6ggC)?=^ryD0fNlkLhe${JnQ;pNR1Tl6h~4mVT9mNVD-$4~Vpv+ilxu;=k)%H>Yy!)G1)JfZTHW@u!VSrllFU<1Hk4S+Avb~B9*XM; zz?2d>b^gGV2bKNO0u19>5EMk)7w(}E2FbSV)JnOC6KU{XCvOM8z#5gcu5 z!qo<8a*s>FzS#5&{*v3lxI@h`DJX#+7t8`+k}_UEwMk<}+b!0YO1LP;Vs9{A!X-KG zpGeTghHT^;g<+@q>k)?dJ6r+q~)7w=TIej3pnn5o~Jl#@cgNJ$pSis}{I4A56) zdpRdeghQ}x=90yGKcmCI}|2iNTrE?Z8OI(O>@LP#Q;R9{c+W~|_+tnYrE{J83A zb^fd7Xhu7}%J(d5>K(F0$&g(9Nsgqu05kxDavIyelR-mY^^u2Tx!>lDig}ruy|D6- zI3^v~kPraGJ{du>Wnf&;Vp}|bZ;s1}DM&fn&wG3%k+Cok0R3IJ3-=ECDF zE6)BVhWaojSFMVmP-_NBWxN+(=L|Wl7Y2UxVXiix&#G@GAWRwS(g#Us1MTYx+6L=< z08>D$za+oNZIKM|2FcYIe?xG;r29Hsw2})F7MKJNr5Tug!qzMj8>Jqn6*wGPXY2&{ zU(PAK{Jf+Tb1Hpcd7@ix@VoDN==MQ=Im2LL+3OV$PgfavtmVv$|k2 z1g+3pj;Kd!)sCO+^D}rvCHtGsz)hM#>_}Ng0-J9okKIdlndy0gwZ19 zZIaBb+U%%Lgd}e+&4M}KK*)UeP|%EmJlE?7y9JO~ta=P2bXgLxg3XyC{jH(0j|Sog zb&r~*K0#5#BbnXJ^Loq#aHq@F34s_}wYg(#KXDJux-?$B7ngnQ>~(DbM%gjFgrN4& z2x|wVzlYA+2J^lW!mSI(hQ| zou@x(ul)6QSq0~NYA5^phj7eA{P2 zdNF0PWr6?*UE|Of?sjNbW$12{+#5=``2u3sd;_}vopz$m;A!J~mV4q1HJDELW)Vj zB*Z9_bsz2z`YS2XhxCj}{*a?K5v9B=Q->NC&}onAwqLz{_5WbjYjwZPg`l8iCazZhp`&#+KU%EfJ`XF?AxWUOHR z%ykDr7=R^}zqjECMhTd5x_Bp;f7?ZJgW0EyFz>^yBGr|>?_I(it^OV6t@0tSJl#lp zxYFU5-sb3ii-W16ffFA=r)1)CxSp0k{D@+uk==1++iC#L#}@ONbsAU@L$=Y)*wQdx z&^RSoSILaBsK0*uqn3EDnJ)N}FiVzaZNp1; zaPw^IOq|2EpD8tjT5T}!%QB(&fgf*l&FS5)5V_@RIEzxzTm6W-(kg}yvFzHZ9}A%g zJ=|$@b^Nv%%Yyg-ACY%F#b(E>Y2Y0I(U+Ah1=;$uSO&-jaY03*}lw^)`X(cJF08c z4V{-X7u-Zpp>5}QkqdHYOXm2B!e|EZv=|czJ*SW zT3b|Qy9HP&050(i_4Hw|#(L5W4B{pvE1-_DoH-3eFs+)Fl3r9vMHuWN{gz9?oF1uZ z!ea-H!Sphrwt;qBoByG^k#MjDdfw0K#p8k|^8^B-ool&9R;51X?F6v~&TC=OG}#W2 zz_bBla#_d#7~qElTNe99dRcOv2K0N!=Xz#m^L$n@;{z<>8cd^1p$F^AI|x-vgJ1Bt z^XFux?NXA9BLAiIJOdoC6O*B%N7B8Z;jLP%W9@n$Jy7DPSmUBoJ{Mn)&E&O%%9Q%> zTh)}@0y`T>HS9+pq=VJ8%3kD~C3lNmZ3kBlaC%nwZ7^)R@Za1<($3Xg+K>rqQwCp6 zdUR*tcb0^<8$~v&4bfL1234lk0y(np-Ni8HY3{LBv+n+u+Kjl6(8)P0$_NH9_^z&x zFoJZ!XE1EH@_2^U6%ZiKZm_IwcR`VdPYhcMx1v8W(HL-1(LO6jfT>da^4W#Y&+6x5 z%tM%KOH8B6P^zjmoO_yyL6Flj+osTovyi*$J>~@k8^2va;Iae~dSk@#D@k3F9LPql zWRCo+vdofEgm34!pTGVf3oL#0_Pu;1O+fG3-+GCG8OFSyZUJL|7{D(P)ykY?!yV=m z&V>l>1tdktIZU334`2XTpi{teBnu2`#H-?}>_IO`aVnw|!-eY)M;bsL?Pda#>^wPr zBH9gCLo*@2jL=fk89hJ+C;lKZ``8NH!$o;b2nM(O{})|)Izl3iDVd;5w9R44$G$zkdBbCf;!H|LgVBcmERJehkSV+&=y4K6abJ;1!;j z4t+Pz$qv#|-Z5whpw_I&GcQ)#V#m4>E<{xT?>maKl+%0cH0+GH> zFd~-NZo`R5RU=u`+@3=>OLFXqD!I9Az7~ojn}#;by-^f)GF%bfM^}^pQ6)5PFqp+M zB5Srv5WYL}lSNY)PJ$$HN|h8O1dU8Z8zzs$XkY{?A*I!dl0IZhfsJf?nN_MnJEbWLcY{YUgxcVjd*zD< zsCSzn85UpzHQEwH>tJxns;D{3;-!FCGK(=yn3v>&3wZi|Ny7B{G*d>h zPIT`Y&>cn&>n!CHzCaoXk2Z)kHR3z%_N*IS{3 zkL+NOEU`M-+}H?{d9;eO6_rMa&@C)(@Iy%q>CjQzP(D!n2mI;-GW5`NTH~|ygq!!0 zQ0z`r9g=X{=*X^XvWxd9yM{t;tojpWfvH5ake|vFRx6znuw#b`+k;BqW#N=;l{uCp zr%6wKyM?Ys{X({$86_vl_;;Vb{xZB868vN&|Nlqgm0r@(@9J?_s>Te*DKX;==FswS zURCl_{!r4u5cN(9S3T|jC@wN!lh|$c>!^qeObc-oX-pc)O!<2fS)hksl8EK$wyeJ;FExSgLpelLGFh_%+5S z{2kV+9o9nc?azPp_M7luc#|P7emPvW5i11e7SG%8~m)LNW-89T~~X9lgNW^fgrJ7GD+TYH6eZTm&AcvyTNJf3CnC zw!0NHe5k+-l4;gVk$R$nQwBiicATYwW4JjzkE!~=qKXSwN+`mWQ=VgtrPM_cFkOhGO39kSdAyRS zP*d%%)a}{ z>!)v@r(c8o_Js|s?%)UC-PKC<4CM!A&9^I}JiP{Bu_Yn`g(%5567*|Nq+sfT^VesE z^~vm++aUs-64>v2Y0rx8m{?L3fhJOtI8ewPVjt@60OO`^5fE z04j-*4e)rQuDkkLZ%_L1NFkA%Z2jj+RJO)2i6*ff5YbLTFzHaK78!HpvBQo8AZ_hK z(W7cOzf)Bdr5R3h7L0Igc!4=(`lf{xJ?(qK;2(YWcaEQQ>=53*$?tyr`g3{*84v%< z*B{Zt;}LwS2xZGTxWuH~}Dz@jUk3(gWC$khw zm$`&Bnb)Ymc9X~`73*~;9T7rf7|i!}`A4X1^{NrSk^@r*F1(uH4rg22VZT`<2at#S zD%bxRs6;5FT5N|d7oF)UIMZbCdj+g<1jAf0a|1A|7d2-qc)9KhHNO%sQj9m3L0o`- zgpQHz@~-|TBXm^kxl%r(d$zqN0n|fncFYSaT~<>J;yIhey4H9LSPrKu$gQI)V;c+P zIz8)*nKI=n(I%(Huz7v=f-UYv^0X|1@5MTaXAapUiI^-mE>Snr^KYW znnj;^suroJTJlq0-t$U@wdcSb@)CV|{G!`d)k@r8zQ_&$lF>x-RNy%itE6S>LR}7c zO~Xm!=5^`-oABx^mr|!mh+U+0K(hl8H>gGf!Q4*3u}}-T)xyJd-HH-A=<;0*@rY== zX_Cb(WI?YnaWo4DI6f&@2IGmH=v)snHC78}HX;O}rF(Ko=p}Lqy8Wa$@b z?qpq~+P~RlQtlLI=y)R~0Tf$rd0UQEN7kq3I0s-1SFF+FyaW$2mSQ^~Nzm13Z5j{*gFPlh4bkJ!sJAj9zBhEM3wZ>PSTJ^?+C)|8 z5s|!8%$n7>NX(p?2XZl^y_7K=kmu3+82GXSyeb0vCJSc}N(IC|doN0zmtD~oah@ifPC&+BFLgb&; z__xdbQ0p_IHX-Rn<+2KL0qseRc#c`|2>C@VeinvQ2wnmac`35|F=}Sk*Tu+VhC;Bt zypx*dYp}>Z%Xr_oy1T+%-9^zLC5^P?$x3n@CJ@JfMM)zW38NxAe+V^3Vpt-qT2KB4 zON9J6sLfMasb*C@H>tw%QwqMJGKP=UB-0c~?V+&rZB+39yZmFWo6_o}NE2;jXFu#6 z`4Cs_glyw2cc<%zE72Ksb~dXt@thV(psbG0>m#mG2cJyU=t%SE9wwTc9{iTgLYfNc zuQnpOJo6`6mC`aZnA^m3o)iy9m;ta2Rvcl5Y!Bq5AV9+0rORjY+H(edlV=%<83*x> zV=WJ@PL}N3U(H-y>j--yTQ)|JZN=D?Zf-y%w)SjIt|}#$Z_P@buOGkuGVn$E>X)y7 z#NtPPOJDgUeEWA=qDZB~_}Z#49>v^XROeZ*$BgS_jl@<-vG-~}=kM#9b@6JC4T+dK zkuh~x2%={b)n0m<1B3d;E+vGN(szH78%p7S3H1Y?ssbB2JMPdhO4o8y;LWP!dEeD$ ztwZ410^wm$LS%LV(e?WZV_m`Im?8tP@ zUIM2z&k%By<}rm-m{Apyr%-LNQh4MRWG|(~h-1ORPdhe}8o@}RUulI2Bz>19k~t`T znHrEtOx9T{y6Tie(5_{nojKR+)6CN(MZOtu!%&KY8}*8G-Zse8{j>rF$OmSKH3k@0 zh3GRvcg1jlaDN@8k(q82flX2-N-#q2Ogk2Eu&Clpr&P6DF)%jx3+~Y;1qAUCp*qwx zZ>2LOYJ?6_n!Wl9rTaNOCmdCJ*Qrw2+Bol9(;Ud9_I`sphkXY)M!%%i2g>wOz2gvK^g zC^1bjs|EHxs$X^fuu}i1- zEZ2mlt_$5RckFXocC`5Ri%)#V;O(NSV@bSnz|-2n+2SzIN;L28wE)G?_T**?8@Q4R z0J9t&AWb1VAPEv@Q(~EwjXnCs={p7eEzy>mD9?AduKqon1w;6NU(M912a-toACoHR zsw|hTt0#y3!IV@kW5yKs1z}@sU)2=|Dg)a{dAw8# z7rDodNWIRj(#F>3BLo2Uio&sc`tBdzeuQah8T~o$##BpZTJa-Ybv2Om*|zkXH727_ z%a8PT&kHC}%uMQcAXxMYr9YcG92qYb`)N)}7U_wPkD$P$boWrEY{aXgX|+2C>h47! zf^@5{$+*aYV}Smv{|I*)0Ntx>DBBzP;0KwUY%A16jY`@DoKx*TUwiP3DXQH{^$M%$ zi9I1jnB8~5e|*5SDl0jb?&_Ntps2Zi%yGTUV(B-cx&668a7=m)l5bwU%Dp9b=4U+)&qT8twR@N!ooSdlsBfNcl$`#?&NM4`4 ze*X6Bcb~p}_3ht#C4$8aJqsj1giIc{7BwV!9jd;d8h| zf*MM|q$2xSPJZmVoJhHL-yOqF$rE>?bF?`_m>azo~cLI z+K{!ukX)yDBg$C`SrQx=(#r*gQI&G&)tYX>^wyi6ndzclZ+Go|Ep-wR-@o2;%S`V&qO z$Hr~AY7*C^B!Ku)qSC@_!(%e(wemAj2&D>-N_RpK-A|9el87c3Z5C`eCqLJeZ~5=u zer5&vH{rWq`0o4sU!9f;wxyK(X{%s09jFd>5P)=%eLxT@?Y>3KuXrqZCCa5(!kh)W~bpUtP+VE&Q zqZ`a6EKZ)lLnudxy?}PgEkZU)DHMxOsW8=aMdePRa8%^4dI*zQGD9$Sv$>K_G?olB zcq-6eZK3cV-jiImp8}->$M5LF{QA4Ur@#H9w;#Rz2bgStgMIe)qt{>Qmp+k> zb(&_?jIJ>T_gE>{^I>mb9KR$yz!smK;!12L9G~DqRL~80k^g||l{3VvHQ-aoJ&xG2 z^}P?Fa3&hAM?h1e zvF!TF&$yI2Q>8AwxVMAmNIT8l(ucKl$v9O=l|b#z26FngHe|OE^p9uzsI2y*ydIY> zm#UvO6Y_Ywqeu*{i7GK@w=@C2JUh{U75?JCrrQ(l3L4{D#iBy@bkSwh5_>Ku~>EZrl{yA5S;WyJ6%Owl->Ijl_JPIXh}}4j;ha)QR8@;0yceQE)% z5NH+fE$hp7pTGY0?L*)nU%v?NzIgrU?U$@5?);A*hw{=O=l0BI2pltent*y)rYegA zNC^L6FM9Tgka2_;LasnUN!(w{$?FVeqV0cZlmxd1)$2I1;^5S=IP(Wo2Qy!>?+xO% z_PmxecHUTMyybpFhX7S?ryJ45${DHRPHLvttEC)Z2koKgP`s(6Mlu83oXgNhwBj}} zhY!74V=`tU66%)E@M+R6%~+X2TdFMVhVHWs#P@`ZsXA#K`cr9qhu!Q3TX#7N9hvc4 zLdRRA!4}J?M|4<$comqPSIY$>)Iie97T$k$!YYjXBOTQGlNq+$;nIAH7ZNa0^Efcz zmV@O008x;tfb|;I=XG*ryj_rFNxX|2w&N%v;pia|&Hcz56?Bt(sqzFI!$j%aEVxN0 zLx`6=+6OpdJ@{yqBZROkfFwO&8(3+(jGI=Plr1lHwgL#>g54$L&x!1r0+}b;6)n)u zg60!(7z%%;6z>lBwH=PN+wv{F(?WYgf(UchPvjn>9N8XJL^fHx&ZlBwHSrp_f9f+}*aHpIIfiNzK3XL7(mj9HHfI49DvB zGV;)%Rz2C%p(~_Chz`Rn-Nf-Eg#*LK*Iz1};Y6yd9XF(Dl=FIRj+R6)bd@f77w z=yRFC8H|j5sEJ92h9<(I^uf;zd$T2`G6TRM3?S7OL;`*cn}&~JA@nZ*RQ-$jnL=t^ zu2M!50_PoVj;5wu}Y$d*^tQqf_E9J91r zuZ`-p+g=)=>MZ$t(eZ?lfBAvMEl?Ap(`131j3=e%2ApAgoN=qNKQfqiFWg@9Ak}&Q zpxS3UiU6swt*TLs5eK;-X&PGzD^+8rJit-TutiC{P@m6^A!?H;@e2#-3)r>@)$7Un zSS=sDctzPp75}AZ!ywmP8b___TB>QdNcC6A;RJqLRaZ>wqFBNT&C;zQ7 zmQu(VC3`06@=jVY0`(zb5CP_(_rfxrBx@P^^!V`l&@GTI-=X;U+E_eCMnZ?h0T4wX!dUUjEq=^oC)onw`+stlIJ zfOX)MhzKv8+z;2gd?ewC6Hxdr6{}@E$&669 zV|kBtg&{8dsb8j6)lIF&g{&R7KI??Sqjz3MwOiI0vXndQ9LZmeeT_1qjgF-G&L4Y2GdcIYZq7UMf?1J5tBNefeYqEbY-1}-RC=Np*l5DypvN#V_9gkgLUTVytJdenKL-$E3 ze?HDpQ;x_j!@Hzoo(oU{-nSve$T1XhXqinU8=1cD#x2AN5`0xY3^o7Tc!yjua$iO* zuM6l%8!RJP$VjnbtHXAY|5z1_u#u&>2n=!7%m#517o;yMKghFc&X%@bUlh4wRS(>4 zEoRFB2lf{$v6Xi@t2683l6_l3)%HY)_LDk>|IkD#K&@Y4LQS~)4Vt^0E922I2Mz{4=@;~qaCKXi28MlC0N$N>_%9T0paSNEJrLedMK2%q1wJoo~<-t;yRC>k& zFS-Rd!|38r3tAPPADkFN1?<-uPvoWQ1FM>1b3;Nz78l_l^sixH$g!@e= za0KboKsMu~gT^S?CtgtZMDIkVZ%Wv|rEWcpz7Snti`p$eiCb<*OfuZT zRknmOh}Wt%^OUur;MO!ikqx}DeSv1LM9jsJ6h7aCX_TX@(V=IhR%NqA3kt zO{7iC&QTua8PiiR0kS6tQ0uB(8DV(Vt?MlywJtx@g9|RT=8mOCMv$&P8>c-EDp=xO zl*eitq64Byu_h|u20QbTH@X|Im?#o<3p6SfM%pI0zZjT&H{+94e4r$El}uewb);9@ zC_i=bO7~q5Kuk`FAAsUNB$nXY@w-7hNNzPs-ErhRaRO6(oQ%^}Ky|)w{(`M*e`R*J z=&621o3Dek&S&(pHUX~U;zWR*X=GDdQ#xKk`QXmyY(2ozvwM1kVJ1v02BN@JJh_~X z)!%b31b8(#T!|ZI4+){I7*_U<2ng*VC&n+*a=ROL0gX?Ob*826JpHWiV&4Y5sv!R>9}4r{VEL zDylW>&d`ZrU&u=ZCZX1norO-1USV zmsa`|Plp#QcUmjF&eALCZ+xFW14%Zs%m+pEqLd-W(Il#Qu%Fnzszg9eFl)Dn??$9} zpGbCt6em_kTo?7Rrk21}H5hpsF<{x2tWAkJrVck+mjXC|QoyfW=Z#Rsr&(!hcknDS3w|BVy2-`nrs*zdo4U&Hi=uiqR>idJg$nHl{;7x<%T zuq>8JcQ>vE3=Yz;&#u+|Bqp})5>{xKl$rqmKO2J0*12;Hw6%ljSLJx=&CwhSpjDN_ zFJKELkMGq<PJN!p!(R)e)0??#$?K@_` zMTrfMl%ln+{h|Ksa|1c{_15Y5d%N2Z>j*3*+X(e7s>cnzfzIU0X%2O*j15ln&z3a$ zLN(Ux>a5uOIYtz$E=+U$eU#qaFJ6O6baFGQrU23v`~px!%LvO9LVKa|2RvqVs#@wN z3yqDAk;q7%h=dZe*OxN$1|kc*iVA~1tW*Y|OwLf=4qW#p!v4r?b5Ve@gc$>q^axROriVnxhpq!D%Y#ao zM2c!Q`Z@d!snZTHZbNy@6*!0r5K|~}DWU5Kc}8>^5y8U;rH=PHWfivc#Mw0CD@MVu z(yf`6URga0;OdyV$fK(M57-WMcW(-oq&CbTzK#dl(28VI?1%>zAxnG&pQfzEeuvf< zyu$XxZ02OR$s@H7n3?B^a{DuQ&>h*y6MpL}pTV|OW_ zh$|S)*z&26QibAew{TqKleR7?CfR$N8f=TO!y_IH8zZN-_q`K}F*C7scJb`GWRO!_ zyGn9CFi7eUX=i7wh*FwiJpJ*UIz2bg&OlWufM>r5JrDM9@Ufz?^;J#KA9#O zjZ5jx6^U|)e89Sna+Y2Q7Labtjr)>yj%)tPkykSW`V5RY&pYGX^F+NgnAD-g2UMo*aH8&eWj2m(T>OMQe(stWaahh&0 zfxQtSf(8GRgk`N|&7dXptAx`%xH2%rI-r%5gNlmtRC$Wh@TCrx;MyWTY^B$yy>bQN==H72cR71vAe``s#BIS(}|jTH(E zYt2DbuxJQKfGd5s5@)22F&VZ7VZ69Y(i_Q8z821d(Q|)MOcOHv6BA`XhBH2PQ2&L7z!T{c#uwUj1iE~{05y@_pVVpTc7eW( zJDa50Un50%xufj!HQW^w9SW3V`4Cn_P|VoIOO)V+nuR$c2ABN6NLdM)L;~?rf%uLU zJ`L0Z4O}bG#uA=jR$LHgIA9vF*PEdOGnA)Byi`pfCpg|&mZ6b0851ll_jFQT3R_xW zV{N-@Jh^gK0VQM7mVQB6P$D2C zVrqxi6Im{hvmYVwESGg`$qXCz9#Rlqw++*R#}bjXViz9epYRR=7El-=aq6$#FnOF2 zMk03l`b7y}n>!wHma*QPkfr`x%8B608n`YSFW*B6-06el7HFyVgohT;hJ-MmRJCN9 zeulY(R3yKB`$X%Kx8I_N{yMNHh^`oZj8^F$u~(5;ee*0&Ke?ulNrh4C%LdvgHA!fw z0Yk;oy$N*tNwvYXAq3K;Cv{!;@C_VAkanGI+3-poWw*4B-cpLWMeW#N5yI{?F<eR1Ye9RwrTeJ}iCX8x1j6h{O7w~G4TdkLO|@#8QLV0cGIa3* z66|~g&H(@+AMCX^{zC;;J<4L+b?cVWziMB^+copZEc@W~6}AR$lms+gW1n@_k(*Cauz6I8 zyJDvM+?EvEh+-=ah2O!S>N{<{vado7@pY2hn~rmZ=y^&4f)!`oU|8*20lZ@~rgUN@ zb1`)IKF{e<>C}Rp<3Fqv;|aWVs=R*feTrU{!>sx3mR@t|B z2aeoVH0ZRU&OobOFj`3#0{f1IwOk93Lm^@%g*gFH5|iX}A|qcXiX9;S$&o74f&L)4 z_eZblyVBSDg;q;gdoq%twIsiJI!XOVzG>AX)}1}1XHZa2`n(O$OYNnhHfmDdJfv(J zI`}}IRbLZOO&;bsyH!?YsPfRTDypheon32Jv^tX8adVhLvCF3e{oeYl0@Jgg!v?!i z6$HDdd{*Uwvy4{E|)vIv<0N#SPPF{C?(bUjc5iP6ZK&WpZ@RFGP=M$0gro zx`KdFP5a$3DLnuy@_V<84XCa5L`(1~9-I&WRA&#if@`;=8T|QqDDc&5QWm~rfS~4z z(weMVLC#9Il_JgkO19lsL0eQ1@$SNHQdT$no-5vrn8A+&9vdB!j;zc>7`2kHL52|MQ7kT#_WZPfmc@lg@sAs53`yYFW6@e|yCbz}~^+uUDY6UcVq`}4v{*`i(YAn!3 zhyv?*iFWgNu@nNk`tiW@Wz;cEy2?dP3-{dbg)6b?K5UQ@X(B9OomF@cXm!NrH7e~Y z6P#2np3=&-QhZg&eFwhFvh6UWjV||R&}*?rRbvSn-r1XxCIbI#i4ql1;r=>z%=uY| zfwY+Y%OT@-?HqxOdXig4xP4CBX{N6>ft-!pl+0r>I5>T&cJmHQLuhx7Q6w}sqQ#`S% z>kN%hPeMv*!VuX)uX;kel5Hq#);MYS!v?!gKYqI1I#-ln&INZn!CIoCvD)Nt#({zZ zgJd$f0MY?qN+l=qs^(znc!t${=iFsbkvPC;5;_U|oPG;kYqP{rPI2GK-%mxge#Vzv z@?pJ}hT}Xa(j~6y1F~^)ch4$XSog+iYQ}CoTaXwk2jY~S+_nCeQM@Z^#?(1(-JmO< ziZCOXE7*v##;roliZ!;?S7_|m+sMJ2v0<~+KZc#HW4b>z`~_@x+)9N)ae!oURAI=~ zjg>BCPr9wR43OSshp}4rQK{JUsyD4aY4xu~Wih?zCE356^*es``kTNP{5}2YzrFqz z4pR3#@K)6}G-7y!eWHjd8_;JkLxwp+@#5S3T{nC-li07$3Q+OR)x~Jh!}*#lhQ~6w zxtJsirz=SvdyQEU5>qH=_|NZ&PC_6X90hEEh}Flwvc>&4aWAlJA=750T>2AszW~8K zNqbmF_0!~0S*gZ|9)4pMk1KJ_^2#NTlt4@xMZVllQ; z-qNl&b&#^GBqc`U4wv&@hHMA?0RT|x%1iejt8^OVu*w0?mp|(Yq1&ei?F|$(5R?iR ze!T_iTu3wC=(BCO;3o`cIGK`v;@;`@%I?R2l|*)q85})=5xxt2Hi`$1tBw`H9+8jb zgBGbDvRz!GG3Dd~R9^$-(niQ~q}O4#;tU;i$JjOjAhbzKOw!Ws!jws~`q4SO2Xz)J zA9!jxB5JfX2&7ejo<2}s|1P|Kb~Z~vu~(S-DApNn;Rt~Mdn^E4qthSOPG}P=H1gDV z9V@x$6upbpp%$hve(n^_P;gYUhaT?P@)=`P-B;Mly1x(F7YMxizN^R1kE+Wz{d7u3 z2h&~4fxu#~kxt^7+ zuMz|&@ct=7ubOmpc`L3PG_U|rWp-0zvzpmOdAaCA{XFxff(BIo=et; zRKodCSv1n$0S;|@wr<@~d~=6_^23<~l7l6PZG-c<+47C{*L7$pJ2=87k^ z;kkqIrCW0nw8-~BN3)^m-3`8+NS9I{4CoBnu^a5h(v{Y0>3#vjWJAhMH4fKy=-3HF@@SJKHoi@q9Z!s4C7sO18)zcY z5*1wo6ZC;;<&1)eN z;Yf>Fef?18tf`S=)I(3HwpUpfatFag-H96q$YkUhg2^43!CLab=s<_r@?7nG+GwD{ zxoIgUwL|elKl~D~t}J+Z?wI3Ykq|>fKE6PF`_vK0?unw9IvIA?YQXkUdt^25!cf2^NEB@H z%QX-RpWfs1!CdXoQ*7PF0vi=Kxvi&#hQ0q=_}@J4KYFalhK(z)(pbFzr+ zyAO7o2X4hn%+Xj9jM}=QT>xG4#g5m6m6KAA(t*-Z;TkzK!9abWJ&10o0ba^05zMJ< zsFehjvA9H&w%d&hM}O&RDXsd{L{C>#S+!3)oaa!hTRh#0>K64y@g&JfA`BTDlKUnd z+Loms6(~d=Y1+lv7TBUg0h$J{Dgc{v?b{OwVokdi1C+h|B=xuHp+;@jPgTnfYdY4c z!Zx4V8z_TI#(+3bgh!6mnG+-|AiiQ;viQjMJR|hBT4@a$!mRB!t4l!giah!1kS99C z-h6O~enTS?V97jl@7KUwKn=y>!`g4K)+u+8;Z$f_Dhp$P@UJT3j$4QYT+(7{x|64R5B4BdbD-SCj-~}jAL1QRFlK*eDK9QR z2>3N6-&(z0H#{0;7=5mlrhFI|-( zkp56k%nK0AmW|}n29;&vd)N|Fu6j8Uh$IVk3$=Ql4ec^c92yQuHqFFNnTY_8i*Zo%Z!AGRKIThAukm8ZR1D(d0ht2#dz>kTB{=m z=z*Do1*xel19hOw*!9s}xL@EqtiEq{yC7_xJ}Cx=h93UHgVftEDK7r=*KaOLh(5Rm z-Q*#!7hj6D(p61r3iRNxcflM~^2I{-v?5jwJyT*$W#!r7MzIo*-ZK`)b#1^zp#6D| zp@|CirIWU(76v5Y<(f=Oj$|wg3vNdnoX~&DT;!w$7$5CkCW6Cb%0Oi zGHO#k^E;BeolPit-!b#P5+f+zW-Zrwg1{gvtia{9j=saDH#dC&iI;`a7G3}bT(PQD za6lCxI#)3sWPVnkMV(>8l?YAtekgo@BTvaKVmEQeRofkzvFOGwy~c#67&MUkpc*G} z+&PYf3bCsSRXaqs+=RsUEl^rCl*?B1WrwctF5!PJ4(mROiL+P?uPIW5mdEJvg&bS; zbJKT!8s7c=>u1<$pmshlAr?U1@Oq?7bB7 zmRp|_6Qt*)Kq{xr0p1w!5ncn;vy*Vj+Dg#{76514axF_+AHj+hunm>mk>jzuz>z$e zHb~M6#s^8H>`eDbHH*5-sB>{dIv1msZwa`q3r)G)@lza?#sWYG0dGkq!Kx@&W?Lby zBjhAa9Ucr!@DO$+)3?u=Tz(P2&r~l~ zbKJc=Vsx>WyVTc&OApz017_^hb11G3P65T@RF%v-{E{BwMao33N@vwoqqjV&W`cRa zyB!LNcfi=#)2OakCvpXP8o5Wv_6?qS^9-uwhC*IwCKIyTd`t$#Kun70z6eiRJJ~l^ zyHFS8-4#OPvjXoFR+YX0f|(`twbBYo25Yd| zEt72_W$9)10|X!1VP~;MYAd>*hEKz%;ZM@BmB7FQ-1u#RPYJ=9gE{ZB>~v$ z71FO`bE02PHiE|ZSg{OTBf~14+MCthtLixGY|Q7ne4j@r9D#NL7vr?EurJDO=Y0ml zqH|nh?QDL#wZCR*kwufk+ZXul^3rjQ1`-+o9g<3>9VQA09in$zb}f%5w@gR zM-ULDYnP2s9v$;F%2Ogy#vP+Odo9iDzUqdoU}e*&qX{)Vu`DE%uMK_rkyV+_KI=81 zRjhrFyOLJ;U(}LkPzZT!sgY~#&om`0v=gh~Vkgge@>iH>iCOh~8E8oc<;ElRwip^nD-Vp$_h6a>91+<_b+ds1bOkR%e#LJRzvRLh3lGqo|TNroEbdk0%Q#IF%iE3 zU_i~7lKVjr1AmcXjgnQ;1(crw*Zb7Or_KKg4vmD-3MQQPCsLHngfq@6^G^48fD~;_k$^i!Yb9+QP)zS9B?0q^E*+Dq+T@~{TPB>!v4&l3%l6PuL z4(-Big~Qaqmf}fCnU#kw`Dqr_Lv4H}xvJMo)wv0CH^M>lJk))_Qb#1> zurVdWWV|if~JR@vZ>sBN2Y!Xz$`Hl3qETud>T%xx8SMJd$x+ z;)qla?Cq|Bf{#ewY%3sFs?F5rH60X_FEXc8#OTt95n{1 z^%0I_aRod9;1_mhrz@FjatIWB9g*!NEF3xWM@+rJ26A~RxjUpODE`e})+q5HDKvJN zjq+U6@DyKR6BZJbH^c8cC^vWNq)bK$MmTO2&?Y7+{eh7U*`8i?0jQj-lk`O$<)nRp zLt9arGOc1+xGe{K9kAyv98or+6HJu?_qkAe?m6fIEq|(MVtc%F;OcU@yFpZKi|FuF z!ulKHS0IelTy*Koc2J&lsvJtRf8NgAy|2v@5a&C8^_St>zfJExeET|l`}e31yF{Kx2XtQ>P1Q~^~z$z_{(98kh2K7wn6N<& z1B*^}$&y=fcgksTXVnkFltiYkZUsHNXyV-amTQzB*NUJgq#EubcBF=f9LO<3wP#Tx zID(?$m5A#LtX6vL=MVX|s#gEz5hfmY)5aB$rl#@Eny= z(CWs;I_Xw@+?#1iY_Os~ToVH{=5;=45}D~hy%RWtA*3x4ppAV7dZ8XkK96Vq$JVmQZJ zL0&DMB-JU#yUXi9Y|OXI2O!Fj$N{eis=`k<9s#XvI-c8!W~v1oOGtC?r)qbJRT+o140go9kOZwfbi z+#9SDx5bSkV_kO&Hh=)UjZA8^(Ylh_I9#my;(;BMpV;N1%rmuHE_{(F*KEiDwk_;X zy|g$56d^2?7bv43*q+?M4I{Kwz}{7oS|EU9Z7TpKPm}!W-Bcmvg+%BX6hiB`6oBqv zpSgYxo9n1-P!J=SVP@EbCTvM0Vc5wl0Q5{J1Cfi_#8#Gk$Wh`=9iItgilZVm!!#V& z&#D?2C^Uo1q7XNT0q-Uq5IXv>FZWn;L%yzsJjxL~fE`{dlg;@T7T$^_Lyh4a+;b25 z89Y0a<1-i>mW=j(L#H`S^aTQqF2+Ch_(cB2+78qC7(g$)nNnh zUtCx_C~(#;Ch8I{laS0^uJbACJOJ0tXX{tp1Ys1Pp-A`(#oz@VAi37^!%qf~SI3Jx zC>Ba&h8LI8TW*|`p*F4?tT95(kPm$U-ZP}#t$+Z2Vv;rj|F#n=k-_;~BWpY2YR9l3 zednE}6;^kM(Gqh|o7t&efp2PQc!ELr^|I_;Juwa<}Ju3j#U+M-%$FLVP!(7z&o@$orc#s0w2A&S&hI3=Pn zNv+UYlnVyTo!m}rAqbfrVSIP*XatPmjL&AI?Snq3&s|lr$9GvOhH-Pv5lODkUBxLZ zJTN_G8*x6d<;$={q`ryHc^&(;PdU#E6*;d-o90fMv# zy+-3jt<75jh99s4_V7QEf22O`S4clPFQP5(rx4y2S|kvwgG$;8Hri@QWGO1&OuZ-1 zS;s6-BH0i!Q+exENy!bG*BL2XO8aDC3TvxyJFP3Sl-xJryI-)n3pVEVLj7fS{4X65 zQ*2kId&ZcQfYVC}fC+K1-X%u>-)KCVPJzL5)(0=frx@>N65ou0`bumsl;TG?^r1H8 zHt7Te9i&OhESdq-o!WgFU4sRPXwzDOv1Vsw?-N8KEnuA_Wx72g3Dwz~FcJ-qIeZ-6Y6n1ZW zkc)Eb{NKY5{)fIeoOOQRVOnkUqC3p3}G z9AYthwm(`)++L-rNoqXdDQ1^cm9O`-1V+Z)m3ij(S(^O-1Oxl3YjCf0Vx=Kb=!8Mc z9trSJg1CCqp~5GyO};odI<-cVV)bdSMxm_gDpVBuIV04Wxe6-WC8JclV6!J(=o?ngxFuLeW*yxvtkwBk9znsypcuc8^*t6P>Z#+GZ?=^#8sgiAXCYbRBm z>y*efY&ukcL@HpKYA+#cR_o}NeJ|Lf)WN_zG6P2v6VV4o#2t#iPY&BaM!NGS!6$8l z2Oy~6O3+(C!gT20(@Dn;7|lnwx%76cgB7RUUDbX{N^)4C zerK0l(&3^`OmZc7A&Oe!E5ECKQwwOHCHY&b@BA|T*}tk49rQrH{oBiP$hWMrUVpn| z8|iB)s2<;4&0_{khG7lVTq{;5l#{h_f~Dt}vf{vi+}tSMx_eO#)8gLg;8W`n*CMt7 zoR)hOs*&!*hAK}^6>SfRwi1HOaH3V1_rR`V-GS}&(HJb4sQuP?X2p!G?i!x3Bmwv6 z+)J$lAl>3l8C+!Ld*AzB`2Js~%lHX^)IUZl{w%znQ|?QQNumC7bSTN>8en79PQ7oH(UuoI8iTJjs23uJ+6P( zVAf}8T`fk_D~yyqWB8mGc?YIr`)Um{lodIE*%zl@o|zF%2bNELk!ProoZ8t*ggAjq zsz7n=Q5+}j0gX5Vql_|?%;s4}=ULTje33ORjRb;sc`0k0* z=udF3E|2!0OGa{73~W0NH!o-%j)kXuF*8_@4;fjCRvr`EbImFS#k_}+$;q1yo`6>C zJ^9=PdSC6OYY2Wz>$V*0*Yxx;j*I@ z5@@dstaQnX?6W65rXxjk^$&Of3G`CJw^Vd5^q&|v9}eeD$sq-UtpHj8$Dri_mGrg9 zW@RIgR}xm5BAd#rVEt-!3VcL&%A=C}(hm!CzjF{^ln5ipb?W_=%~9bky2Zt$k9m|N z!chegR_0PjK_d)Z>3rN1-UEFM18>w2MC^RZr!=mDdg;)GiUeum)Dxl$T;uFx2*)*t zzLp~Zf{g0taDp`IOu;Jx0NEC9ysFR~@=Lj|5L!mka6WCZJxi2^L|x^#%g_BYs6eYva{$hee`1_AUfu;jlD11jtZviBMe*PA z+;ONc_kK#Qb9rrNc#3vX$PPNYBCF35x77nZ+cN}88r?h*y&NjcxB1u(ac#hAh(e@p z2i5{p)02U&w(Br?>PDq>RVo_bz;$`%PSBM~#p}sNKvK2GPg+AGQ@kS@DSCCStn3?4~EXKYYWJJF!mRSUUT z8)v{kR09aYh|p20rA@Z*rw$R50x5Rwq!z9a17m43Xemtiqn>~p;YvU*vP6l-gqLBH za?I*lg!14jWte@IiYS^A-D-MO2j8HA7b1yzW$!4m$J|Y)?G4 zD|MbnXP#j<^=`OUpYr8p_PF?(UIcT)zXt9W_urU z(~+68yI}8VH(fO|QqJTk>&b!BQfF~Y36Y4x{$|ubIcP07+Yl%59J28aLYTknA;-JA z@=32Xylq#b%jOa&!GK)hVN{x-^smZu&Q_{03SnPGN^Tik=nZ?|p?g-D`Jy-|zo;AuDM)9PVsRe21BgKY;IdqoiDaj% zLepwKU%x=%`QguIn0S<7L8c^YH@08O%g1kE9Zx}~$?Ruv8rCKcIqAjYkE z^bIa$lFh8M0b!+To;00V7iW4Zz$I^X1u_{zDHu4 z+9TMz4kFrC^eweN!Ue2eiK|EHDM8bqzi(d?3H-_1Z(lzS^4lL?zkmnu7Z~z?YEOD+ zlA|_#9!3no!@&Yh9f_K( z5%X8!2kG*l2K(8ojV0y9#7}r5M<@GDz?<=pwQ1;H(z_1rjPVpH}-MU&1M3lJhj<&ZBPfjA>yj!BpiUq=l71ViX4KE49MXAw$tt}SimJ1t&i!~FMRb3J_I+dp%q`|61 zVRVMii5*M8*fedFoWTd|4v_m0NI1Lg3&^Q%6dkNo>}4HO4E?h|3x6j6OIP(@-abR1 z`10~HIo9v_IS^c~;@=kD;k%tKfPBF6GRqOSv55RNBZA${Phi;WD?_C&aJ*Kn0&h4_ z#TNV0tymG!8W27l7aU;Xwy)CgswJWYjcb@~3B6J}7LDj;C@MIfogEeH?7gI`jV|y+e*FrLSpf2 za$|q@d3gPbLoEHuy@PN(Joj-EymVRa5@2TH9Ws;MA(6!?JEme%FF#1{v`nHz*Y4G{-A76j|?xWYQ z!)wCjKcb7TV)sA=MQlrNmT_AA&kk=vpdd^y+GDBQbFC;+PS!6Sdx2wPYYN3ns%RGJ2KLWmKsy+h|;2 zuZxd$Ie#i}xI!3>X1h z7*-+=tbYp^opxFp<|wc%D6r8`qmCMIHfJ7g>J(y>Kwo=nF+Mw@P_D?7jJy}zYH*UIO6ajy!B1NZoN;$fbZvHk%>eI_geHIO zFiy%c86+yIM4EL92UPxmnUr!85d6l)qxK${XB}Hba~J4U`$jQAd0$$|bH)q=Nz6&r zHS9j{xll_Li*zqUE*5bD0M;4|n8sf(fBBc;FXeyu%1aBbj8XXS-O{8k+z1_A+VIK| z1F<k;_7M=FTltqILf`kGzCJXD%eiOS?bH&UqVt=RF)a%ViBToiZ_2?s&a#j zVZyDr4h9`aMcBtft!- zbY>ip0&j5`ff*b!j4MePja?R;Q37_IwHBpV#mI5=Ma9zNJX`iI_f735JC`C$2H{D4 z*Yf9#Kd0=+U;S0Er{&$pz|?+zc~Kxrnr*6VnyM1j1(YSylt0~b1wFB%fgb~JmIBIL z_X-j^+G=GGEs^t}*3D|AVIKx=w%RkDYXIHHrBt2XOGcu)L=}3Mtql#xZR$k}Wol>x z_Yo>H8-dWh`VK{ZFyvwHW=?Q!F=-)n6K80TRG#*i?i#F&0xRmEwvJ@ri2zJF>bO#k z6tO-|lZ0NS+ZrRe*tIUNs2c)kcu}2jZc;KD3Pf4Xw&4e)k&ekbQkGN}edG#ip5=OZ z-HE4kppP1$$?4$jriIZm^g_u8_n+yjBcjK>Jtiy z!P8l$ACE`PstC5tstR9o`vfW)ASvyk8d>JlikZPskhv)3C@C?m^txEk8M{ov7wRUf zllu(jZ(=xBUTc7UqEA2Il9Y*z*AA94-6%2ps+Q=dUNSx_4Blng8$RZBn9ZUU&fMq@ zD?IW15~eEZj;_E`JiS(HC*;Xmx>Yp8+G)#M%nZNwBf~HVK_O#{|19CuxZ^!6z~sY{-~-Oko%#HMM~pBQOb;nCP^zjo;@#=C zeDnHQNb)mB5k5!S#TC3`KvlMg7+Sl3g`150!wBRB602;k&-MPDG z5V5Cth=H<2)fh)zA4`Spb4@jMS|+!68tJ`T9bwtA;lBp-!Q4PJTC7Y* z6aeZ2KdnOCN><{VjGai;lQwhpm`8Vsd_Pn8&%Age007HuU|q}8z^=gYX8Oe!}q^`dHa1}TEY$f@$1j*cOzh)zHoW> ze?$HMMb0MZ)Jy8K_1}UGwp_fLD4YUSLz!VhC-?GWv|0N_LB80fTYsoFd`>Q$BW9Jp z*;tW)DyopJJ+Q`bGV{rIZHIPVCbe21JSuR&346W5)yN5#kSs0_B+Ro~uZWP_rPY;h z|ClJl^rX0mILil*2VqoKUpx#Z836V8EZ=!mL$s}Aq{Y6|YEuXqN!dc>c!(iwhg>8s zz%d2_TvG?z_&xTg6@78X^2~`B@<{J|6~~d1I{zw(&ejQ=)J)ry0W;b$18KETqkMzM zb$Be+hxA$h7*fc|5QjAwI2UfvyKm_6(=#<5WYLb(9e#v5Suscy_Eswpfmg!@N*y-W z@*g-9;~vqInLQauU;X7te&~q@61szuWPrG_fO=Q*MlCmxehRQiNw2iSgRG>T#bFw) zHD}*I@Mfjb5m@E={g^QMJ!|=ul7CA_7MFz!jFyUzAtD8x3{P&M-aXjndD(bH6DfBZ zFhasD)R@uX%q>W(lEKU9_`;}w@TiVc(({hMBo|>m$P-zB`mn&X5Zeh?_~q=ub0s|H zq*8b#{2qcnFE$K7O4alx;R$_&A!0&ZT6%-3unqq(dL7m2#LE3?%1DP+2MDcmB0;P! z%!i;+s8u>8lkyGL#0axVt8dbyBz&p$fvy;=h?hobQ$YfJ4n=0$mnC>Fw4;V(A+hK^ zVCpkBcGoKk;IyGA;p*Abd{4vF3LP@iadbJCMT*v>O;Sk_kgoFgQk4An*RL!|zWotY z+ZTuV99So(kA3gJMX^#+URsL12ZK-DRSLcDk|o))&wEb@0`OSG`~m4}W7rmwvgZU0 z_wYpp@1cJtNno9ld>%Ol*jo46bgDX{X(nR`xKnR#Ny9s(?Q-ZOp)&(%Gtg=T%UeF9 z2L^NpEn3cUIckmwh7eW3>IhHi0g_y#{`KF{D}l~odLSZpgmNRU07FTY#?H8gqE@bY zYg!ayYdEhQ27xMAM<|}sS9z*i$S!W%-BT)bD6PpuWS@y31;@Hf-iuVUsgz2iOUIyk zz-;^gx4-V&zK{okA#U7Mn2Lk)g`){rq0l}M+R>eyBD;?xf{9DOoVPnY)NAbgEwmX_ z!vH2FV7L~nq==`+M90z+o({^qqABN~7MKSsCvlVji*pH$lM)cMR;Mv(;}X9FFgwq5 zDaVr611*`gKN9DxaQkoJ`)G83&IRCxtf|NXw@<_xB|$OL7g{JFdO!!i;<6kuTa&}6 zhj7oBDEH1t!PrcFIXJwULpl00460wOwg(8ke6;6oQg;hDri{|&A}qy)gZ^;}w`sVH zRUo4|TPuvPY0X6g5@fS$`S9#`)E<}{NWvMQ2DmlK^K6nedP~1jEvK{Cs4pn(8cIB= zxbmj;wJ#?wXg{IjN+aJCD(enof8fxo@uZys!qaB6^m1PT;$m9`ktH2=)ykihbOq{o zaq5=8e*2t`b04R#{^RRcL4NtgyMMgg!E59+V4H3`PT9KiPICf_ULWb(AoR(7Ln4af zvlE2u&Xoq}CJuF7*Z{-kqRp&^jJn&ttPA(`9-tbUu(vll&zFxHB?ZBB31Iv5Cd7sr#!Qa=G>fgK;e zx++70!^D#(j7ng+EDp6jpj-ou7dg44LuQ z;Y~X0|0}#DPlo3MN-!-aj*sX=(?{62ZX9$JZ2Ca!K8_G^Wz)It7y(@rf^t*ROXRo` z5NMFEfNhcwfz#XgG+OFj1~GGQl4Qv zY-2vbD>exZNHSPn0*I?A;B3yRQcmoe#|i)wfh;)EVFn z=E-9+2Is0I?PT$B_V>(7kx@DiGVFOJTpMH!S#MUwxJxBxLPx~zXbS}_zEo1}@}4}` z@5AfApE!w-nXvQx=rXeXj$7lq0zgQyXjtC6H%qoU-K8*GFGxnWNrRKra^1?Q%(~Yg zZ^@_Lp%20JMi^{(grab5lc6>dGZRn?1pSXtu_GM>=f@gfqzt^ynoHhmSrrE2v|8 zH6{b~=5m9ovw9U6H0sHVl5PZX{N%idHPw8&{%r*Csp7;p8ksKpsTA++3V>!CWA+9p zJDpr$dp9+wpF^eoUQG@BpI-kE-u?62hjKKwY}>3V!S{((=Yv?`y$rq<-CyMJbQLGf zZFj}K%2x1`l*-Pb$90xZBE|B9s!TAhMYw7V$M*sJn0XT-Y{5uDQ$?Wkx(!^g&GNu) zOCDuQPSKY8no1eWOs{rl0&NB*C&=xMr>(puYo59WXf^fS<&!6O5>=rNNbCzvJ;w7E zpKal&4tR7Ml(L2rzi>g4x(Rb`SLis<$h5H@VduyT)L`iXKq|E`n+|gGEq-r`D=2Ul z5Nc%Bk-7d3QF})tLx9z_BJQXOsB z@x+lB;MV-IqYG#1fec&W1~;oJH_$aCw5nRddm~8*#B)Z~5kan#JFajl+YkD%2F*~f zIFW@4tcom_zYO7Ad9m7+kW9nI#d{hIO!*JM2RB?dop|!UgoxJS zLj8~%ghOr7+ZM$dMJbT#i~2F8YoqKP&N}4`+Wz)mQJ#K!In+6ZJ09TS$+bXuldHP# zVW)DI)cL&)8MY}Yl~RyECo;?4G~n&xsZEz%M;*{MX;fZ&fOv<~M+!EBaO7hH(GW^I zAvxZpX~fcUJCot2gfj*h4(lno2+~@#1nmw5eBH1iEb}>ON&l>foxd%NfVU!(6Xcm} zPfK{W3-%p6cvzHW$AwKFI}XnOjMrG51iCPX{>oaAK;Ac6jEo8^@uBQFM#7Um)Ed+R z!$C40q_Nr=F;SuYBE0@2y;Dma_45AqhnMZpYmMm`G95WRD&TPMN@5t(7fOlkW7g>i z&BSns4iB&g3w2Y6ij;aSy4bw#ecl$SJ)+&g#h7gnK%byq0EP4pLH}TOGT^*KLwG8x zhWdv20pSySR{fFs1AOsrQ_CN{J=C!3hHoFLs-LLbAtljANBY37In|mcWc|@M|LlQ>)G5F^V8Qtl+>sSiP94By3n8i${o- z!Ej8#YbE5bk)GXTb4F)je}M9=71k$fEkbg?{tF)wm`9Yi67Zgm3!rM0A+F zRW5U7N+s|_m(JNZC%~Hx9dgax6{Xie5hG|h;27dlO0X>%TW?gsyCn|^D0~~-v3T50 zYO7KTcfMeg`fdSsiX+H&s#3hJP_O}idj%?mHJkjCFs!D>ip0dZBOR|pO(j8+J#m@F zVd(}z7c||g?@<%_r~38)eWmY|jhCs0_oAPb%Nnz#I?Axi7B`H5Y;=bG-3n?Mf z5cdw%c9}=2M^u*pV`Vy07QE4h%maJ@|IU`b(2~QmNH>D~{nKDi+ z)DBCsamih88NE%H#|{;37b`z<>ohZ7-t2lHPuZXAKr(&rzcvtZqUZ;y&JAeywf+7Z zBl9Z-Je^xW9?@#=QMzo@&USN^y=pgdy5cA}Nk^U>YWc&lP|{f*nEG@9)}pGi#PnBT z2n)hxq{HW|tRxhcThHSPux{^WJ5wkqeg&;(^vAfYa*nstdS6D@i>NpPNb zd!VosF<4)%qaAxZ7G9vtBrp{2Jc=LtV7)oiCIvFJrvxq;7l-D)bj^ ztzuL;sq`mmUt)kFs0v%qXt7PVSg~M-6bJ3KrEyuEt~N@0a=X08CjpqF?I7e2ly1Mz zM8135qS(s9USfqPpJ0@-HcQd9%N>iL8<~dNS@RB|JZ_gO%s}epxWI|aMG&sVD9``W z>rKw?z%`5u=3=9+l|*qvrGc+v@H^66flcVSsBrY*vE4_(mTNC`}x$^8nTKd6$sq@lGP)!M0d+8BR4RT4Ts0 znzU#{w&^(O04B^Dj&SD^y$)kx{VrsYEImRy#d2fFYX`u`vQa8RGDtyX)JLP#8j^xy z5LEWTU-44FZcd)W7Ubl;j3wqYJ^2T&^THSWUH|j}^m4IX_P)S@EDHL$%f*R#BO>=3 z;NX*~0h&mL#Y-v`C@BZ0`x{#FA=?>9g_0#L#{HUTr|QwhDdKJC5Din*D+fn~)-y{M zUYzhuKjD$~23DRhmkppnWV1VLqKQ8Jxn{YXSVOvvOUruitIjD%D#Z>BV0LSA5GJ`B zSczO|C0a^lXi!$HyPr=`0JUDK-ZS8#uiX{YT&n2t9m_(DcRIC-P{2@%GKD zEwFy_`deu3|Mcy1dDf9V;eL&WzHsu~tsu5F>^I_p-c!PsLnl%Fg+P+f8t*5uDW{9d z99l^U{*KL;Tzz1!1F?&skUBtYJaH7Xl zX82okc!9EWB9s)}8E0y15G3{jrvW~XSMpgl9*7R-@+%Rd*eSr*`L43if-Ai&y}h(9 z>*xGEeD@2NV+cG9kBM%!cPt@^MZM?o9eUG=5=e^8gQf%Jf8=Md!RUI6y>j9!_|e<^ zY2@x@?v(muw?J2LmVQ0ZDz&_2w>(yuwA&N%w4;1%FP&R9NwiWEcurrhG2cd*q-@Ul zo0T8TaE&>p3N>&G1qw95%W_d9)^`i5LBObIEMPxVXJP#sAn;W1x_2sHVf(;_VU3VO=v>lR;#AfzKvyzDSTw!U23v;$5B`v@dF)YGar1MJO#vd1(Vp%S2h!yR;_8-$=ModSW9ZL} zdvI6pk&ojF0PyCyLB2t7{=+J*$;?FI&ereA-oh;BL=P-6=HLuMK1sT)at0LL4XA{5 zvawP?qp$cBMYphmg_fUP+Zzm|LDadyDJXq{Td^pC#$wvS@A!1fDzcEfu8J8@wwFlv zO{`GtD!~0s9$so`m01V13F|Y25-@pUL__FKpQoM%-+)5r-frEHZK_?tgHpXWcAce6bnU-CKm z(z=^(|BeU#%U39!rPm+7{>*T3e|-Ij79{CE{5_wz^f&cy@XrhjrplkJveXt*D#gCY z&*RqNWcu+1B*snrR`sV)#~zMEr0s*s#1z!tN56;_%H){u0##i{@6KWvG_Gw`o2zj8 zHpwef1L##0%$^}6^7K%ExOG*bTk1M!W>Q33L2WrSz&8dNrAkea0!bl6|eMO2tsm zom;tJ<@q~MZB+M8E-l^nxz^0h0%l%%4i!-QWQjo$- zi$bzcK3UQ!>o1g~f6(?eOQg|~LJ8ivF_uwFV?S6h@6=q@jx6AiPs#K#cX9*I5pt5y z*|rNjVseFVdV!XLJ+E*+)w#4{#6>e|INck|XXV~fO_Bv+Y~@K+Mb9uu67 z^P5g$_pB@n^pM%+T&sI{l#LEeb82EXSyy1QOG?>olndOIq3d`Sj=d0d(qgr+y0`eA zm+!fDl13vMk@%2x4D^~B4sND5dwOcm;!|}{t}lJmd9Ih z5Xm+iJ?m=KCem?at&UJ`*D3j7jFb*odnw+f#AP>6;+%%LFtM~Ru6x?SJWSkwFL6uX zJqKINd&eyFh7K+Kh0!OVRtV0f+g>aO2_z^lHr2(A;;($Lb?|$sZ`&c>1JMT8v~Ajv zwn+fL?#U{8}QJTp^3c@HyR8 zxNLIDx|E(b*6lNf9|lhWyG;?$wp`%bX-|K+7C{IL;~-qdMul3co*i`A<;nOkBSG#1 za10RtMK#MtI#`3It#ehM^dqd&j~AvoXKa9TFCpp4Gm5hV7NY|++eHC@9K9e$dwVrc z4`B>~4kws-OTN_59e2^RYHyKbLwaMLfm#IX-XboVliH&E`0Xd}e)#s$>o4>(pN4NX zGOdsK=l=e`{nx|Sf3JU>KlP*RvK7`lfIzPFJM4Y<|FVL#k{R${S94O`S!3DX5H)SC*BDS3vf-L?_oR5!SeeJqA{jlC=c zrU{~T*@R+Z__bC;nKZwB{LPOgJb(4}G5^+&%4dI`T`44!%%6i;&#(UH{~iAS{s*T= z3>%81A!NdV?LswT26zw{;Z1_)OoK}O6PJlZuX0lv%=|WmCUU437et~WCze?0OZ<8y zNso33_=!Lvwdk>4O}Ni?k#D*5uzkB*sW&|sqNmQ~W8unnLh?*)oe`?tY^+Hkc4ccv zq77OU^&;W%k#aiSYUn zX6x&2UvF2;4`b)LM`XW0rGN5_KInuSXG{(@nex0WGM4lch&-%<5nOiAzWbNL^w5Mo zsP9M75x`{5)7~Mxyu8@LZ`%RO3!;&ACG(Qs6kAeJGj3OlReHt&2SZS!$`94iad;0h z{e~UP6)UNs70Fh(j?%4)QILW)!^1eqV)dBh=7obQoL3A|#H>iyk*5HzcP0EN%VxVy zDuB3bR!xdm?uQXYN`IqVzcODLbU<_SPOSQ!7wtd#NLF4MWjQa5>581!herlX*0w;Z z1gaOFV!PYedz18{x?k@QL&$cXc6PM~v6Xei-gUL%^}>c!#AtvC3Fz|ZgfO$v)SciD zn6T5*#x(NywHK1l!KcVCq!btdXO=W}^BEGRxurQF>B>gQ8l|GkF$PXo(rDNTP~Zp& za$JGAC}SlhHWiC?buO*kBNeD7x$wYdx@!A2QvEM`(Xz)0E zCFA;+-~5=D=Ft>HC+s8T4mxeOFqt;3a;6}|eOL&ftr%7@&0(Vsn)#bQUq7SngiBW% zjE`0ywL-~F9;4i+t;AYt%c+{I(aElb>pV%ph+@v3?Nn_${1l1?nIx5L%8YTVYVB>h zLRC(h7f6rylx`dL5uhc5J0Ge1SG3WXb7o2(JN*D~0L1QqngsH*@wqcRlzRFYc7*D7 zU>KuT+a-fKtNE;Q0%}rVTa}x$!8>v#*l;MTKCkT(?{LMU^lMs2=o*78c41FlrC}#i z;wQ5f$4kMNa(>y*pMy5ifW=@zz-AId`0wP>|N6V(%@`*C{Q8@(zZdccefRe3pq_>w zpI)jaW3)1r_Q{eoWn8f+L|?Pox*lE_1 z6wAQf?=7_BZWXBdDxsLq>*S|ubTRqAAU%3G$up*VGO1Uzrs z!rBXB7{pztZqC%Cju(Pz&@0VSt7|kQ`A~@Y>R7*p-HrC35C<#I0a@l@u6??6S_%V7%Oc%_;ggSnadnVzexY|StSN~ls9In>X$mZnx7MkTJ)-wI^VwA6A*p-N-mJv&iD3f_ zA+n?dcAIZt(M>LSPa#Vr8xE$o8Yk7<1sgX{SdAaIlV%+Ci^tXs30c+T?vy z$TF+a@jSx zvcyN1!A>f_kw2+m{i#V__mBHgH(bkpB)Sz|w#c4ULd))J$itbCfpVj7=E~KoSE$Mn zN}*XKfMy`gDM2y7BVCVE^E%&NLkO%6JPG8iGk%Hyw-0ir4| zTC7u`Qg3-&>poo|-9lQ95}KAoEX>+JKeJo9c53X&3cJH*Kkw8i zu(>0J*3kO0A7{byKYg$#oco5qhd+1jW>T_k=X}}*_ z)*VYl4=;A5>TZs1d#VOWa^ggnNN6&X*oI1&6Gu=pZJ$)629hS!)L?S-#rv*?|vjD-0#A>AH9D0`gvyF|M-9W zcldLEpk3(KUk3ePwZ+J}IB+#!T$24A(dj>QfcFUqZtSHZH)CF=8*!Iv?^U^A!n!w5 zxv$x|?#ErX3EzidzedC|Nj7BL?z~nB|4&8y3%)p(bjlq?shsvtiCrr;i2dj+~jXe}@ggd7WkczjM0GM>@?kI&R5kqb-W zt#P8;OLA0pKS{Lwliha`9ndyPQFC6%rEmrO;zLJV0f-l@KQ<^ zPa0UAQmB1SNCze;DIh`g=AmTTXi=?@k#~`X2_ZtRBAWnUK%c*dTzhq#Bna#p9{@5L z^In19^uPf$(yp~G;g+;3C$)&#k-HBT-&UhsAj@p+%p z>sv|IERof6daD$l?ZPn?T0aiAUZlLBC?_bY&=3asV#Bz6D*~vssswNuZ>foa)LA&P zm5G~H8PlP?S6ae%&Fy&{w83LstzFIUBya8ge%KOt#fGCh(^401%#MA;WZnV2gFEC% zjKrx+1&mNFIes^lFSULKQw%g6$cct2oIU0l^Bv2dt1OCVi8E@KuvEFG8dV6|E+c5l zppURQc^tld$ix|@AzCuqX%F*|Fw)+4b?u-*iYc8fV3-H*iYsH-xV$b(#}Wj z#4&lmH2cxKH(rvb5obH0)sa+MXEOntW!oqlimQ-2ULbhZU)*M|B~lTBby=@2yndqN zLrDcs(2VCoqQ^d~lB~C^j(F9r|UDV)7p0-s87j?cWAVLUE`J%RuUaf~V^ma5GcU9*03B5ve z@rrfQj&VSP9sqD|?BL2zhJ~-rjw-2p!t8B7Jg=#cm9wiAqxG+)(UEfx+mI_)kCsQ$FQ~JOG+0>TS#%bOYqpt}wY3F0>v> zy-N;2#_&=&-?jyyfK4eJIr)&e&~_qO0kX5Kt8&FYBBZ?!&nFb26LDwqOg}!I&wFvK zLWx&7C_r(t2XpwC)vhUZ&P~GWT?($7EjmoYNZ)F5cTBB(kmA%TBT!UZpX5e*#8|kE z8G-mvOHj*!?ht>s$la@i^@UlE$`|%y=J#G@hfa8@v|hML(oAxZhpmn*#rtvSA^mxA zSS~QNb{e;)1N%=7$Fui}s<5_l`UY}EuE7Jk?1Jz>ZKuP!bk?qU&0+2(P#dxdk-G7+ zT_glIn_MF;aPo)__YVb@mC>Wdm_%sUY=h=gSoqH^`@r^MRG$&&YyF zpIIkG-9p^)*i7o7bsjku)ZJ&~yUqj-*GbNUg005qFq^pe0CX~{)P+C_?J}!X z1Pz`QIoQ+D`dYzdqw7Zo(2;uBCCTWa#7=8k>g~c{)tO6jT=T0X!4e~wA`vGf5*BPU zcUHR&t*)$eQthhMi!4bgZSBKQSN> zz;fi3SgZS8(cqS|$XUj1+3+TW`I($@eIXxlUvC~H;S7i0;ZzZnQ=M^!l*iXs(+TFTcbzD&po}KY_M?w$hj~H`cDQ zRo15m+dMXs`hKQ152ktK(NB^*K3Ud1_Th=LdgEBExyPz+ofP*} zjan!1lDYU8tv{oNdAL<@$1MTzF-22Z)LP^SxRR&bSlElybHTk%A0R?b8ig#a9AQHD zqb{9OpulVuswd5Zb-bGk{F>MV5ZaDy(qFJm>RRp@*Ya853vI4PRTEfG%9@0|{G}>& zaj0v`hSg*A=(#2c^rIN!bUQxxz}D-)n(3Od;_oLcn}e4Ud9<+q@z!&agyd13(o=|* zMkPn%V7$F^vqwlpjfze#3actO?YCfyz6v|u-3lyv53E+|LM?&Cs_2_#Z{qi(mF#kcuF=O zOXM>)tfWbVDEe~-j|(Em_aaJ%9-F5;Bh?F~0=yUahMK@habfWSfQiceiZ;N@3$)hV zIjdnt1`@Z^{~G@H5+D8zTrH_vAHMzZ?I$)2{u|p=7{~c+4(bc?Sgp~vavrs<`&Htu z^+!9kGVZhl+;H2u_eZWKdVnO+%c6{Vj}!)~JXG>s`?e+0a7EzgH0-c<2eLTiDo7NH z(EOfDvkrxlAq6TP7S;5c)Ljl3`AA?sL>AaEDap@MAuBw2MvZOP?GiJZti6>t3)G3h z(2Aov3rySq`=B;&9F>4(*eU%RE+8xY_ul^K>LC9{+VbCDKM(Kz=`9mc-u(b3ZvXi9 z>FGs^DV?vHrkb1_eStf+3Pq6f(L#N6Nw}$5Km!arxIP-!22mF{FvIQdIhB%}E<+OG*{M+R z{S^avd|~!eDoC;#oJf}H)A07m>D~9kYhJ6Doyt0rmng|9FV3!gVrcwGedjgHws^|E zjz=}>^(xh((-QjPEsb6(jDzzV5EL9FTIC5sBgL8HZ6472x}?P^$s5E?X)Qd-?>;yn zXx!Vd4nDHMF2>WAt*4Fm|^`J+pTaR#Fi9hUqVYDiMLm@Z=f z!kJo)$+OQhWRFg#3ayuphkQ`qYTE=cxCp>se=Gd;x8#4m_>gP*&_DfEWEW9R!s@-J)<*8EdwUIYi2XGI>1^Na5)tO^l*-K4dM9pET6+{2>!v19&aY^*y zOe8lh!G&~8*^_p4*AO)FpgC(kPR7`>aW)0fpnM=75bRu7HLlwM5qMXPD1)ETVdkc= z-JNsQT|KSSM&&fwbksw`n6XGJEmChBM$iD6#-XxUBwvD1?<6@F9II3oouA8E`QCD$ zm9i7a>z*rApK6gQM@wEv7J#9j`?pIbx5_41IL;qjr$P?!+JLPL)P7`ueu7ZjBQ#81 zpR{MV^j2%2P%zexoWuOUi;p@$lP^$C)?xe!+IqUh^56c;@U3rstK3qbzkZQFhSy&sVEr=K z0O+A*)am}1-1O~^B)73>qa&Zi$Euh+Ddvgb_-cabtv|61O|zi&I+wY+!D+T8mt6An zCSn8>T7!SU-ZiP&ckd4NtB+bR#4i&>^@EdB$aj+Db5uITY-o9!Kz(1CykK6iMjL_- z)ivsU3fJhE=xqpUBZzhqb%vFgTy?X@9EfTJko8C)tpOXAyk5IN?>f>}W}D z2jV(JJ5|0V8KMrF6{ZI!_pSUN>5o~W$hNG9g!?ZTQLf#W;@Z_pEpn67y=SHEnt_vob7K)YP5I)Gf2BeAhT^344O@^W2unckH@)v2b_C%|2!r9eMWf5U+(zTegJ=je|-A_ zlG6rz)<&78EQeBS2Zk;!#^`|tj^~Bwj5%-)t*N_f6D7H;VSKXO;pF5EA9zPVKv`w( zd^}n`9_aj3ytU?p!Xa1$_`CLqhe1oOKH6GimgJL{652zu_8Q( z2ReruM&5lbD++%uP}xj%S!IkcGHMM(Kdbj8?=H~61B_)n%qRWCy6OwAk;JHHj&`+! zkt2yLlJ&CW&=V{^wpP)MG#Sh-gfntc`U6dI~fYw=F!U zU;z;MS6Ftscj1EMYuK78#);vkfsU`#KBgf$04d|8Ehq)y0V$-s@RDX3HlV^ku99MF z{xg2$Ciq$T(=P*6EC2cRH!xnqho9l!_>BDNr+HU|=RR-uJGf967HD$B(&MD+iTNkJ zp@WG;W{hS(lNVddj`OfHJy_##R-%O^j4Rg99SI}}cf!GSH!QGRMUJt@)a2a5fDrVN zsrByMy>qEYaCX+olT*$3qTGK@Rvg^I9^GvC!d&64d(ysi0Tgtq=~_`&9i}txF>t|z z5>v_9vxN%;Htl7nOeRRol^)+2Vr2k5k0q={MZ3v*a~rj32Ze;0V;dph9#>S6s^Go2 z!Ll}{&S|0p!diTEsuipi4jbT}X5_-oN=cP2{SA~#|L~UGso#Bqr1z`SOLfIelL{uv;tn}uH0GRhJyp~Dxrf?V zSPlV3iDV~CU$J;v{+VBea1fXzw5co03ipxSW>Dq{Dw;U^1SRchs&-aaN{hM1iqIfpY>2gXE2UB zI~y!iH|Kq!g%7-q)Tuy!a*RE=3kjR2f9_-WbAP}f|68dn6dL?%5BpGtQNp*r`TX?a zHHyr2;3AYeVm4cKO<6_`jm*)OOy?X^l6&@rPo?uz73 zjRAYnJOEZG1HX;8D(8h~R2{edk%IXzXJ}*hH4_@HV4o_1bOAF|uHW%-IuB0V1ufri zV0+cxLH3n%t39k7V-x%pE!?6JY|Eu&F4<=uj3MyijXVid%7%}wV~R_jwC==~Q`Ig& z3aLp8;Sq|s`6L5wa$vqm;CZn6+ES-T%*n7xn@-Q!F+kqt+`g(f11yj?owVtRL{Z^V z2PV4|TUYz~Dsq8YbFqLNAV!}S5~nWU15NGeibyK|S{3~S561lXuj-~jKE(>+7R?5_ z1XG3az)bD^Va@cw_5{*F#hptIMsrbT2J`~HK`R-Rjwr3GqQe3Z zCrcd6v@vE5FV)%p_@y5?-Fj$W;Tli(hdgRe<#rmjslnquj#LT1(MV09`&Wg1sOGNY zK2*$@<9Gl;7vgVG(yIPs7uC(#zGg~cYvlqRABngDFdN3|q?EZx|4=Dvo^36I-O<2= zP$+T020JK~$gh21Z{SFw+oz4y{2x5+VSd2EE2)b?OC?3<+C{ADSVZ$uNtlQR67V#~yYIZ8@~o>A`skvdb5yNO)e5%*eNrYG)YF6r+=U2ENfh$+b4U2ym7gg5!`*wJPQMTid2E0WfJ+K5LJ{ zrLkapLu18Gqc)nPDh6!FmI%J3)R32?QA#p&si}ZB*>ogcIk;~Z^~#2jKeCxs65fbj zJ3w-N5Ncc&bRAdMg=$HJ2K36>27b?)OBJx5MHStOQX3cl^!4|DMq}#URe%gus7BHm zqUO6iWv?jZ%~ENcl^jL4O_~d?oNUbvGS%o!poL#uXy|fx9}yKAQuBy%zx6seLhMU(I=a@kbd?E2jgUO&UEH=}cY@cLWW;h_hqRi;q~ zZWsbVAiW1GrD|BZKy!9s(>AMJtfv*?k#f#dY$*X-{30o=3%;pRmyfj;J)0y>kHw{S z?4Q}dRO=+t276`xQMwmH-q{4?@kE!f7enwl^NAS$(EPTzD5+{rhau;NYV%+OJpj(e z$z!2~HG`s`hiIzVwUp!b^VKn?1)VspotEw`=MfPUgVdkei=sCm=A$x2id;kciWf=W^?D2C0b; zpYDowgqtLJE;J+|wRQB43W+8T9MJaQP=dWW976SrpS-Jpp@KX%Gz=Rp$&mlZA=s^| zh$SDM@D!`*%4T5$oFfmJ>f=oDfRTQtLNS4@RfLps+@IK|8wYv+=+79;xq0=ql}{Dz;O54uN=Ls zQahPCjU20ca{f+SEz*lkAzG)JF zrMkfw(>>U<_=+Jze!fpo(y^;w;)#V86)Q7M8hGP676ENQv>@2xhYw35Pvs1i&U_cZ z5%IDlW3sjBvVKGSq7Cca)^LGZK(UTCzBju-`<8$}y^xiU5bf3DO;w~^3KjAS@mj0%deHfMB-=EgW#+#Jf5Oa4o2cVf!`6^`2dLKYgh$YnnJ zpr8c+)1<&Ch?o*;^}Yo+l!TWUT^|734R60V<`%NMO^)>j@fF?|djaH2{W?KJ%C$JP9qV82_kHgxXeSb3a^M>29-6!VTo1Zn93kal_bKJ{EhDz zv-RGrjE1v(v;qcl6_RZPR> zl5421jw$-mYX(Tr0>(K2v3kMnIZ$0GX2U9iY6~}8o>2jyu+mkMVise2{S{NfKjdZ8 zDw~6Id6BOwz~(G?+>>fiy*M^wp}@qg>JFCRbOe;@5!5)!0Bo#yX~eps`|T&R}LzqwJp0*4hJ+q zs!`dWM>&vd-W#VUNp5PWbJORN$p^iR=W5C{1yjy;Mte z!RX!wNtZBEA!(3HzmvS*t}XD3i7kULBO#|=SffVu0Sa?~ZMbZWC9$aS_6j64X zyW#;*>Z(pO`4d|GgM?)F;Mk$SKj5tFt{KK~1&ITH+2SFlRgjyrG5rP8W6Sw>DUOz6 z1|m1uo$84@N)GkF0!u~Cw;UF%zGQSI`=}iw*+}ZtYeWXZvGU2b{FKyPT;`xdb>ty; z#swII3>3e7R9A}XZJ>~Aoi-Qq-cA)UMq8bAPI7NQhMUWzse#I^w*m1E`ICxBRLSk` z@=&6L(XOiN1E&GIec^=ZT1hRyA0NbMIczvRdv8UP;y}B4*iRqNl?qoy=Xui6wt)G0(Pc{%6ns;VR3m`<&X<=3u|P468<8;RnNj~Ss3!096@V|HJ0XOnV7rOp1cvw zR@I}ZqK!aSkrZMk9%$432tXw+^I=geqV}L!0xi&NAQJ^V#tTYcBD^LV8Dc3RW6JB% z3cOJjQ054kU#t_cWy3|T-#+_3NfD(Nx$DLZc%Pdks1ekiB`i|S=}S$A66UNdmJzajLmY$_f?)eG}@jt>UC7lb23#GXg%3>#&C}uv`UG1 zwt!7i^j*kCS$4}!rKc$$BRnrRvJiFf{scZgNZqW^xIT#S=aEc{6a$S) zXaZSg4h4eK2O+Is5Y6F*)J%!pu8BUVqH@jAot%pG!H(H*pgdm(C~CULSJ?fgp)tf+ z#}lrk5%r}hL=U9k5{S~DlKqa?k4 zDUFfP_6J7TX1s<}9`clGXnwh_j&_~ddjgg(8yVwYXc;m`?Qa+yF;W6^0q>)wM$grXc?JI>fiWGeqg~+$9dHlH`4SNQYYH(KiFzXIv?2H!eFMNSv~3z<|hw~;w(*trEsyTyw7aB zNz|90j)ML8HXXvv0v)a!7umuG>=Pec%_XlbmJv!+jjd_eZFsObDnyEsrm62zOTjDc`y)(lu6Sqq0@W>RXGx~sE0V(fL%_{Mvn4ytb`=yLafvIvLZ+vc4d?X-PSwVmnI676k z7gzErRH-DRw_!6!VBK1)j?`P$^}y<)n8XB^0;=`yiI!BfQnOOW1mOpC%%0<*0LO_QnZ&jCYH>-%HccC(_fD!MQQbTT zLBnNrFc3szeB2{n9HL=%l;o!?7wQ%1(YsG!`F_`p;R7Qm5H?T{kt_WQ{`~puLRKrl zo{!}4@;NqJ)#!Bvl1u0_wb@!)Ri5q`e56ZCL36dAwS(c=%(x_1iCprHA%GDNAf%9mGOw z=JaxFk6kJDAVgAeaWH4kFMDWtt}&gnDi(VZ4bWV&IW$W@;E#cOuh=-mS`zJx>D-GQ zo&|jqzyyJYSkTA|Ns^z|rKKu|R=#MghtnUqE|Vmt+Ilqazgy)-%eclsnzA>r1fmq| zz`A*zLHeM*?HHB~r|BN13K(IJjF~lL6tpB^dQECF+$@rm!tbb-6iEJn8x_R_G)F>h z&0j=R##1>1hZe+~P{I|~H#Iwwa~?)eZh3Uk!@lwI?3=FmheK8`zkrCYHD7lB<1S7M zepl9X>O)xX8!6xYfr2N2m9!VR8OiHL0gc)c~QfxvYAxe4(3NPmpDrBA) z{9y%F4FnPjT!PEzKn~Npf5KSypWc39*WThxw_PWb!7sSl$+mz<`#G1osQe{aDLF@W zy?nq#7oM7)uq?&CHV-D(a7!DsVJ#Z~g7EpIo-g(Z34tb3pP(i^RP3hdzSoJBVFMK> zQ$%6Af*!jJC?cHzb8cwzCCO%dp3`waA-}}4v*6pl*$yJ}atM}9g2iJzX=f;`E`ZZi zgGtsigl-4JT3lAlMnJC)t5(eaTlf$8yy4nCJ%Ad;T_j~e{?DKqFWg@ zN4YJ|Jqp<2aXN>omgMjXbQ6V>(TVK}$E%S=Kll#Su*|W6ye5mJV|1JZ!N4M_dY^Y$ znp&v7Uw?Ld(!dMB_bR9*dfYzba&vC z9l8~209?}1y7hD)*F$edK0!O!l(n9EIkN0SZ{DJuV(@{^b;6R-#xA*_|CsMwIg(1p z-FW*BDopDt>iu`i1Pw6R9)%Tp_W+k92bAQ@o2{(LY0Nbq?)g-c8SnJ1r zqiigsXfZ>A<>ndEcu`^`SdBy}fUxVdP^golgTiws+ki8o1-a)l%D|P&fI%f03)pnz z-6Gr*$O`WGlL?(ed{9a1YZwL#YD*7yw5#OnDZptNE%TLlI*?I(9zD1$wNyVn+C$bD zG`0Br8sqa-DVG?2t0Xq?-WFJBkj;S)dhb{{DMJaJ?cg9VQRhqIXO3lOsfQ*-o1l%} zTI+;y-j-yCN~XiJ0{A;cN2VEoGqQM>T_u=u%+?}O$D>wlkR+d^MF!G8jsmbE-U{budG9b|p>jUL%*&f?)tm66H6=KrNHqt&hwi`nMG3O%id`VhX?x94SBkXly;PO-uIsu`tUbCA;|&&Q-;uqQak&o{AP+UL)+g9 zm**g}eT$d49pK^nNcb)U@iyZBXO84o_Wns)M@TY)859NvS#02@E>- z|ILn^BlEZg9E#+V@Q1_%+2j**`~g}T@Ast!`W9Wg3Ub)}#xh%~>T86fkO^*rHiI>0 zo{kV~vo#PskTO}|!#HnV9_V)h48`f*G#8je-73%kB_di41qes`IO)W>OBwZesKD|o$X81F z%If)^wblvR!AX>a#&Y*uwT&o-& zJ2#;%X_}ntQO<^>p+Gd=!fIZiUF_A4aFXY$eB+IP{Q(3EN?M=MqcEH>{0l3mO197= z;nj1&JUb*eXl|?JejM4TwSYGzP#kS)YKe*46lRmuXw)4!?Zg)2=YSSIYTR05XDB6} zWC_`KUElpx_;-I(0_9KNe#npEo1fsv{I@^*`oD$OpJ!^+uR{KCdVUViCL76N(p_yh zX@FF@`0uUv0(w)|X3k3izgmeaK?+to+7oOPx2&Z@7d;NV_B+Gw5$u85|^A?Gc8~|nle`dKu zuGC&bC&B82i#-Oby0SbPUDu&l%lS;B{xk#*0 zSjrmN!95Rs@mxQH+t-2?w^hjQNtJ5)8fsIdXiMNTN2JEF>&*cgS1(l?vXNwdFUObN zOiWc8mqXopc_{B2zWc$jx&fs1&rhdEX69v@a&?;Epn~y85 zPhWnpN7Gt#y)xNDhJlr1M~V2-{^(*u{kug|Lu@2`K-k8_-IwI^i%GAtrO=LH?V;o=c%CBRaylfk)6LF* zoxJNUnGt$^6!P% zOqkd9M%0!nDP~x(t;}l1tcqQZ=&Z59BI^|j!#uhU2zXP$k#%6#P=5}C&e~QGb&GN zZ6L0vQpWXCIJL6iu(|*&1?e;7m`37Ppn;+1a`ibIE@@N}7DzF?^%am`9-7@!w=Syr zv`<=umu~uo371y5h@6z4#cd`(8#DaJ1bDu-$Ls{ok{Wf)KQ_XvItbGIZF zB+K;7r;47fo4oYrO$U^A6r*C=E=JNtp}=04}NelTvOw|ER3=WxXce!MdWFlZh={ed?tdn$^!TZ}m+S?zl2btHf4)mYA!YSnY(l$uuV?`O8QJ56}p z1h{$$mm+<6rv0bfos@

xjmfIWPbQXBQ-hJBfVL9X{NWyGi`~VAmgpK0ru`tRpDL z)~h#wmoE|g{oa>e@)8ccC&>P94*-EA(W!=_h-#S2KaoxTCnDOvB%=MxER^~ggm~=5 z>D?D!e>Z&nJzTVsH2wC_EO;dC!9Ajxq_|qsfX%BUYcC}-c+)2|TSB|(Oi07zD4z8> ztarne@h`6s%9;IK56T2uFo?ABGnR1=GyLmc1ws31k8r;5i=@k#miQD zVXLNotV+?LAnPi(Vsc8!W^w_Fp10``nwg-tkk|-P<42&3Pu8RWamNzd?5CHXI;L{2 zr6o7_K31oVT2brXUz0zWwUL^@5_MHYkSY*kq0YjI+6pA4D)};jRUTod5|?vqHrEoG zfd~fz?Be~3DUwJa|G7LB`4;!4XT8rXi8q+Y&k8viqug`2YsyvALiTVvVVBAhSB$wjUo$C!sFH)v; z&OU~4Ji9m!HPFtwL~(0jLajRg9N!5>6!^+M{zgCTo_enMe>}mzn$)ZP)%pF4QJls%1#W$A~k^AQPs!Jg@~3Ct{4k{q07ohuaVA2JdX{D5F_jM*5K zceFd7%f0J9W!3Tkaf1>qmk=K;qqG9duP|7w+(EL#R3fT&=IrL%7qnV06lVoz0cv>n zvH`l%#zwXvUqFY5x%o;m7~0GH*^typ9beS2W@buV=<+}LZL_GKr&Tb275)#q3Z+uo zr(WA%7MgCoiidMLHK8|F)JG9<9|Il#ROBeX>|%9RHX`oMRPtowHmdY)meIj}a>?tc zB5V$_nzcn@Iqr+)COWE04k427&-AO&#zS^YakUND3 zb?V%>Rx2Sc?&KGUpRP$oGWYC%rnRh4uq_r@8RJQ_Bn2^=SXwv>56i5Z7^cz!KXpkEv2aBBIPMP*I|aM~&rW z5wpNgiq-FVS1KhbLp{KfXAJ-48%LtTZhhL7%ShlY{QbN~yO-{huKD5JP$c1yRyoPH zLeGoO_yPRp=|Rc9dnEsAzbS8q+(r*hX(Mz~l12IUMvkneKr?>zOR zPK}|%?z%UN!xQ1~vI02cvLmX4vu$kcMF8EA6-~vj2?{ss+S~C8uOGtM_LovzvE%1=%=giI2-<>Ovv4Li>vsZsiPs~M{p9JI#z}REnrZB z$Tn(`E9UR~!JgCuuVO%)b5GViZa1g=Vz?>0@=}vv-Jo+#P^an6SVaunt+i~@)u)jN za2^f8izUD23XZsan|>TKBzdC{Lx-%|CTxJdlnqldQECn&;lV zDP(o}D-{Cw`e5cV@05UF2h>f~N%4Rx$ddPq{1fE=cCM+N1YVm`$`w{oOu*2VSNokC z6?d=U#d1LHCtds2vn^mgsuvT7km?%3d&oBOszD!H&XZy7A@OTBD{>tI>m||FnMMLg zD%6zR0JNc!TNzB18%-qSt97T_QuaS2;w~59rK6g#=I5X-BzHlADfuUOD1cAdS6Z@x zZGH{Ku3|`6T{`^ku2eTamZU|k&r-FZj4QYX2ei!`%REzgCPdrx+j}wAH991I_cl|cldLEaC#wrSyE~%&0zK#6iRHX1|@+DL2m2b zH2YZGZdZuFan6fbJn5K}$&ws!03hm#zzStVa>|9EM=gkGBYi~AdFhCbrm|x zC@N*Cc#j-8L|J+av;N}v_Q!{}uOGjD#E;>dpWsK|IN|M&`ufEcWYk&%#2Sl2k_mL+ zjY}#dS(7}adB1ST0GLrx(A_5q^ovR-E4!F76za#~K&rLM{X9N*GzFH*7=lRPdGyw5 z*f+&1#^vTS?)_4Hk>NopS>{qCvbqOPCbolN2DBT%1OA*|9 zEal+v2p#)q5{L~vb=jphpamq7h3SM+rc)#CHl7E!B?sCxHJD~$R-d&6@*oU)$s2ol z@*v@Dos9S;Wj{vj*8G#ryK_$c)XNi-q@$AOZzW>2vWZ;Dt!UjnEF4$izDt@l+lsCo zY`XQ3bBo>4oet(aBx`+fn+KFlh~2K;2ersU9VIy{qzHBzSwK1V7U`SRv7tT}m>fKk zvh9(|@O-dqmB{LF$jaYLZ$NZkP~h0GaysjzaM=N-wA-I%uz=5??N)x)9tA|t^)^?+ zynN`ulEf{4YqFQn=2#mWGqV{&uac^5rDjBDH?F6tR_+$s>8DXLXYKO>fnbrI4GQ#) z2J^LmW&(T0S3r%(Mx1FjfB_DhbI+j$qsSrm>S?B=vdchje_aGW;IxV;XpZ@~!F3+gJH#}*FEl#H^?7Erlbpa!6 zE#=IVN*-p*Uw0!E{K*Zr;k+K;4s9XqC4ydW5;{X#)ojzpiZMfVO!^dNd}wJ+9Q?}d zxL-Qr79Vr=8*uNy#u>2?vEA6}thK4K=#;`b!$=lMQ-O)BJYkX!VhYVx*#T#bMCD`H9&t zq&ACWroYnGC1?1sD$i71_UdM&7x|{yL3M)Kt+mFYdR6viI?u!bSW(8;9t~j&R&uB4 z2zzDlnk7VgAnOUU=+sJvh{nn^T4AVX8WX@dcz>foAC@=Kh)mk0CLopI#(p3b+H$~i zrq?kb6^fc|KPXzLdEY3R0_U#x3{_KJ0?W#M!o|NCX;E+e9INE~Y)SVF{~{U}Ytv&v zz5yptA%Px>rrplI@EY6W{euHZnOenG+SF9X#-NBO;ty6kohH!T30dG zUz!K$hTz@7GJCVjatQxedY%fFCMAd@I2pwipN*4S@~P@yFl@$m0?m> z^f0=YTpVgG-gDlm!?S_lUIwfO)An?W)^JlFa-?KG@ZnO(zeyXwsr;)YW*J z;3^j>!*hcmD&x+0h6#s`Pf59!b%%6x&#OiiO-&C*c3eePEU>M%+V`pt@BZ)C-&uW~ z$J_M1h0KT)GY_eRY)pi1mJjTqP~@26jQv)_umY;BQQa3K&H!aHDTPVU0H_lgNSJ3G z`q=ys?JOkRHfFWmp)xvIVl}k~n$EjVzW!c#{Zv`dWrC%lJj+}?tBwG3!Mo}0iJU$;8MJmZ7!C1^s1Bdl^1?#aNFD5EN-nmIe6r;6 zAULhs2Zz}v>@#2zD7o}DW%W6Qy4p`HrMig|08C!H(#zr=NL|XN1Td_YHv#Tqr>;)# zHNzqbRTYl06O8Uf4=4_lHV7;}SW4MlDo+Je&NQ>{(jslJy|EQ}7tAi)WTgQQojT*y zB0m_6-zFkjtLaHoz?J#RjuxY}t?$7b*zi9d2xLUGU_WNBLFzWd^}4tPhH5l`$DWwT zV`kf5ax-NH_O?tR)Pdb%Y10gWQerqo*UAtTkqjJDv$Bz7KhDv4NSOWXXk85n}I zd1tXMchKzwrRF1Zx*BV_kYt@DC8@`}3U}ElbRL1cQ+~{l;{5#Y3gU-^b}`hYp6;W|oGigtQJW35Zbv<4aQMXQY+@PC&80Fsv}Vd|o$6yCl6gIkQceY)T+ivG{w6BC;|F}*RG$3QjBA}r0ook(Q-dR<9d&NwZaIsBXt6- zT!u$k**I%67(*GWwvWonSu4Fe@(OlRm3seL)qYNMXd1MwY~Bvd8wcxSeg@98_cfH4 zd$5^Sg}Jlu5S8zc+b&%zRky8(;@{LfJO@?G-jPYmh`M($e*;5Ow6T|UM{(H(^{8xSb5pqttrA+BfC4`a;Jy}eaDr4ye`U)oI4n#aWA1tD#t&+nVC!BPXF!f1LalQz#|YVk z?e^q%r;pt50QaGl320sAngGT0-4}0vq>1#q zABNYD^3y9OT%WxCIQ(sX^AEr|eE-#Y%2HfEq!_S}iQleCTPYc3CsMY0xUN?iMG){A zhK*2{qzYkBn^^X&7~b4ZS!IJwpsji?L{R9aP{9Qb5}un%C?ow7bFK+`xRxR#4<$(( zwJ8wEL>yja*XML40q!6_y&+?~?KSNzuO}_KED)=~g{|M!#j-}UI?Sp;a#K%DsoL_C zAk#q*e+3vSmlkc~0EB}@@QY3|?C4w+xv%D(`w89Mh9f>GH*lk#73Gl)f%Oxu27J~a zVCZ;(sn)}p_3f>rdO)*>XhryXb`Gg{kwRO=Zo*7s2c@sn1S6{3<~$T9r|-u)O-uJZ2fD@-Xqe*KO5iM@Vydf7)nel|O~RH?F}HW8LinS?uH zcRF;kO=~*)I}SO~%N%2L?IiZ2VK6G=RTKRc$pKG&7e0X1>(gmzYRLnansnwY*5x2x z1oV-L=aQVq{l=54n^;mG0I=Aio|zx^6F@2!om)3a*9aZDeJY{2Z`<%Nia~_N4@}>d zsZ>F2_6I8t^HNa3$(vG40Oh)yGzJgEKUJFw1}v_HEPp9@n;HtRGf_UMMzNcDfnt;e z?<$Fu4{!(`#a#)#r6KD5WSSPWDkL*AgF=Q2Wv`Brl`jp^q%?6qfyxfZN-s5z%_{6q zFn7j)4ft;Q*bS|eG6VnESMTx*tr^VbHZ-QJ;aexn3A)u zY*s)Cu3w>n6Rwb`-O$m(=vVI!tOv+DlnG0PTC>6d5Z7l(=4Ihc?j73PxN7pw(nhgg zG_c9+b93H_ost{(L#o=qtD7clQ$$V6q4r9NN;~e>LxLe zcJjmSILD!%*GHI|({*Ki0-CL<jR#mjr`*Z~@al?X z8%lOVyu#YXa)d=Lp^ZXEl99N$gxtY3aip?mKH!e&-SI~OaC_0zt42stZh%%=!jC zpv}Q)8uQX>F`#@U2a2?@evOE_a$nmeiV-34@2Fwr`k}P+;_xr26@217U?gaZw@t+x z^b$3Pu}cFW)|qy`SYzUx0o1)DPv#^pwMu$U<`!zmu51m0kxdQ$0!>;V=A0kz@7{iL z9ASSc@%}F@VlOJFkc!ra{7!?$X3Rh*^K#z8MB3y=1OYCikZuu?nl86WlhwJPJ@(xd zFFx3-maTW3BZ=ahayTrgfY1Tlz+B;0sFRJQDyi^rXKuDbVM?~phY4~KJTGVzw?)to zf*hWv3|JJ7n>UTa@tAw;AmUYloBKU=9cx)r{Fx8ko`&4ii!%rk~@srY-kN z1&%|*A735>?408IUJ{6+2tSb0*n|2d=7ri02%!ox%yJykS$}xYL*(Yio%4)-&k@y-&QgzxOw~Rtz!HWE6 z&sCEPET0a-2wuykS(e;9nxV+cHDK&I*eyX3no)s+V*Xf~#>hT3$2GIvzjrX1d|8V7Jy=3x|NYXtX8 z#6)fI<8W|i3Af8G#0fhV(>V0 zu;qv!QhfFL)%zsL zI&Z?~XDM8W$HLJh_9K>T^xnwruka5P(h6O5!3m%*(O+ju z;6XRf{_+O^)vij$t!lP)R3{Kj*oh5Fxs_xI_PR1jX|NshZpk)6FgvvOLAhLQENR)G z?snGBjFTj@2_D%aSU@VD%u;zG?cxdGEgP-l_>`D&Xo|&Wj>pwol!cJWo<0tk_9h+# zvLuPGc=gwEzXUo!sV^@p7B>}_pQ33~V22!qvs#B$Ry1qf*}!I*T%Iz&gTe&_$)PTa z7X1NQDh1IK)d11&XPxgAQ25i_C;)jT_ z8g}T1l8G@Z)WrTe$R=roKBOc|bV8qWoD*e}3~FUcB@yVps_3A5JjnxEDmk7~kWozS zsVDhXyN#wNf*E>Gvd@#2YDU6TlqLrYXm4^lb3c0g|Z4@)TY7xpwV2sA~ zkBR1~md%4yh;khFu17wK6VLZZ<^rO+;~?eKSn09EL*NE!gLJ&B8=X{|@L$Yo9Fqc5 zIa}pA`aHk^toI4HO2(tQAbM4vWV@!_@8>|#3tWw7ztfP6*%Kw~V+gvZ3@YJw_W<(7(|B8-laP1>;aQWGF9sXsXQ8}==sRZtTo z{{VEebbVrx;gVk}P_hC)r<85idlCJ-#vAR`p0fRtgD)2`V7G8Awk;kQderc zzx1lOQzfgb>sp*)8xG}SOh9FS@AhnNaYB1m)x3J5)^2EoZrXt*@&a-ot{9}! z$@%3#@gUGqYlHbijwMo8F*!SRiTT~ST0`ytp zt*a_TPxLTp#1VMT%!`f=dehEaM^gWXPLbta0AAcQ8gZV7!F%3;<{M=zfRVwH0dBhx zeYAAss#1)#_3Riufh#AFW-EW){u95~Anwcfb4p`zV5 zZ4VphV4Get&WhEoLL@nym6T8^`lU2*ojH`n9r5{17pT&)hO6b1b$_`mP~?gi>oM+V z$jZ$yQLFnLik@|b=Dp_?Z%Vj7p{0EI<|p#sg8rNx4NY56tDJOcT*C%gN>3Cx?4?i@ zEGjgdKiT0Bd3$s#CW-6)Ud{3!3>ZgQppz^FMqKQ=X*+Ia@dl~i8+*QT9Ojv}7a$2* zpOJN(4%_{3|7CxRDTiihdLtPM?n6*j{>YoYC zc2coJIp%r9rSj^A^iC4Xj8B`?k3wK(%Rm?UJT1(GdAEtJW% zE=6Z%x#alN-#py|avG{>#kOs~6nA>bfJfy};|^L>iF~SV)`mcNYSBm1LW?;26I`sR z^n!%xLV&J%nnb=vx?m~5u&*iyKz}gR0Cm%$L7B+x$yO>R3kG3fh;2}IvAEEWS+ruK zs@j>;!0dZg*pBD<$*z*~0L~E0qqF>qX1%zo{`O#hE!GkYxzT#!?^Kl@-PbvVc}sr5 z!BXWtD#DM#v8|B*h${`aVB>_m6L1Af<_B+y+}Bh!t6e4HYSC8MT%QODf)= zceGpQ9Oq*-;W2>gdXB$o!me3f9!M0-%BWCX%YL;Rt_UVPYo&nx+$&4Jk2>h41b1=I zN4Q^xV()vdGBlM!=IG|yqeC5jNJ`mm!;bAOm2V(nU!p_3q$_Bb%fR>Y<5TfLP7mi8 zavr2uv!SFi^Os}x8Q%T{?eLG@evNkc+b_Yy{74qyv)A9g{aF9tC;A6p;Pu!4O+RCv zT_3*v7O16tRwElDq;EF>kX&CNGvX|+s|>}qpI`ucvvUYBJI_d~8~7tjf`^~nJFRz6 zF&|n_%h5SR^|65E%I^?FC(vP}xNNzuM$+ol>OSAnmFgqXmmXrMJP43ww1r%5$mY7! zB^mnM)@X1V4CFj2ZHfm7Ohb2Nqhe^i>=te71nwyTnefDvSPTp%Mi$SKX6=41X!@bG zSpGWt%gS9Z1=ZFSUbbqB9uenTh1kb{$MnFS((al?1~Bi7^D?Sd3>BHVS9}=hxi20N zQ#J^-pa$$*S6R+8L3`HPG*!cAeBFahA$nOiDmS77!*49LtlV^C_Li{js3beI-fWTU zNC{8;D-v$l3hq>osDX4~m#;=~GPEQd=8ib$hDg3K4Pbp*DV>`ZO!z5+JHuA;)2wI< zSR?#deu(GDay;8>=erFJ;pylvPG}_^jD()B_7|3Zw!7(z>l38;LdZScAn1tY2fL&VrZS` zuul<9FC*tiYvIXFk_>cG07czCW+&5$R8x2k$+s=v6*evnDdkz68!p*EVO;isw1o>M z=J_H)R}K@x6$UicPQgro{^^aBZdGm>+k0r16cIXG#O2@z31UHHqw3wKZ=Z&@KR{P} z07#D|5^{n8jgnQ8X^+7>p2T!EBr<4LRsd|pjTOpQt<7q2qKT_S_umPptL|l4L^ivk zTmctAM&nB00-mL;3Ah0|R#P~gT|}zwCeLm}xN{wB>=Jtk5H*k-4vM^@>-2zyAwfPM zC8@{*rxO79fwV!X++CRw3~tCjz{{#!oxC{8=Pg1vUF=Ej zk@6qmPPRQcbl6EeGh4xBDx{r_FtC?LTS-{6qq{J zN_x*L+c|#%ByGt*K;j)(Ur6aTTFxT3#s`{btRxW2oi*7G>>jEB#Vk#-;SuE=I0%@( z&)$mcI9lsC?Yv;j>;uO0E?IMa<^BV0Pt^%Dx(15EG^SXGLi8Sn0?d+CLbFSv*@d5y z>%0dL3wjBDY?`WvE~P|%zCT;et41x}8^vk-V2>H$W1$|s3%1s7-Yi9_t2mB)q%lph z8tits`(c@>JH!&I4I@x0mjAzK73-LQFtcos5MCx+2+scJ&o%6z$u)yNAT>j?HPTn{!MteJNC0+4_!{ zhLTn9fNBY;D=W>>Rrcg+g~3Ly2PDP(`t4_^9)1l~CCOty z&wuM5-#!Z}5h~RKaJvuo7kXNm;&5V1lQNu4!UvU;%}SANQNHvtR8XTD>``IZPt*@;B%t#d{pGx!*%bo> zElm3cPKMm>@DA8;?}V-3^sfa2n!#PjI~|UK+qk2PSPgxIYV%r&Gi)rZ%adEF`W0y# z|B6lag_fuK=Xb?lp6>m%cTJqT*2NZrB2{_dn$4*t=5D=ZL6SV-AG^W;58~;UGdA*C zQD(kmDUyxU?!O5JT)VRpSgN93xyHV5hgXRjytalY(M!G6r-vMYX6GMBy!OrJ9=J0c z*c_s!-d`{qdXep(gBT*;0uf@4WBcdW;-onP`>MimVU-P3SlS8^m%^SR2a;=PGmU!| z(q?GEWvqfn4VmCA(9?6!-`y{GU4>K~vF36>}@Da7Esn;&Sst{nv z!pIB26DKlx3d|lDSGO#v7Ij6oZ6c>${@_$yKs3U{HI*Ins{ozppS*nvBOLvpgBp?x zZ^=rdESsq5%BmgnykPwhs9v>Z<482Eyc4nj&xr16_E%uTe42_=!3@M(YOE&!8eo31 z2@%F5!NCW7Lvg;E7 zvjqLC_v>+dncE>WrG(ymHpX_m|GAbj>q%;|{CAu4UKT{Ca~jE80>3s?WDA*;5YS4ljmWlXAur#03b9YH?Xw_{~9h{3%-WGIPtJI@(== zrZ)_y@f{d8S zy2=U=9xjwvsa_t7{wS-ffo7>F$AxPNB{5SDAtt<%Iommf&9BKqh14ByUKaD@gedKdiWMXjPSD42nqxQmE$H6vF5GH}wm-3?^7~W7hN|G2{Y!@GibZ#cz zHU8sw)vYgBxl2oO!x@E>VWee0J#?;PAr6Fh2*LsV@6}VQv|+Fp?8@Z$`+cdcItzPR zs14`4omoP@E6HdM+T33v`OYsswU4IX_Wjr2y!(E5{oUy$xjps6tx?hSfb@ZyzP-4v zv^$;xIt*k7s%-`zv>gJ3&8QazOW-sR5Q-$h9j)MmdiLeq+^A>WAxjJf6=C_>gC5+h zQ*(d0cogoA&JM=OBO8b+rrl2p^1fE|w~`zTT5<`)#1-j+oo7HC<*ldoXe(E_&1)CyB)>iA#|A7!YYGav2W%+1 zY}o9WAIpz(Dcitf#WSYa{s0oXmwW}N>MWS5{rFC;+)F!dZw*QuM1q=(4D+$TN zhjMm)O@x60s7J@_sXOb1)CFX=k9}vTm8|L|T+?i`K@^ijSj^UVnzM65yEj{L?SuT` zGX+mn(NU3yQA)y?P4$^)&2@ zi^pUN;F*!RP7;{U);SGNaX3NW=b@_Z0KgCoS|5PbtwP@HwIs8)WGWlOychbb-gcS7 zo#HHYXf~Q45&KpxV|b+V>D%A_cKFZ#xg6@>!ScCa5pDht>f^8A+xW%XSH`$~&Pmb^ z;+(f%aFMZF2n%MO2voE3tQT$!*(9x==^l1sk{Rwm6GEe~asaA|DX}cW;3BV24^-um zVuwcR+Vp55+71N<$;uOTFZFhKr-PiwNh!Ss(1q=p)78UzoZs(0@j-UGosb?uzetLQ~f0 z*TKv&@*lna&=xGe`~1CkpXGPz3qFVjRk?)JgU;M{&qSOUsEc7tZ7rt5#GT@R4R@+I%k*p80V`aRud4d9L zP(vH9(0z-5)L%ghmQmg7-+^>b=Nb+~k>Yz9a?M z%=?45;yD)Z8KefLO>Th6lnLf!AUDFtB+r+0x!)w%Sj?``K#(x4dISvrvh@yd=WCwT zUMWW6LZN^(tM5oRtBO;T2-v#7OxKYr7fI`0-0(~xDS3gRPVcy3Fy7YnFyX9P(4!Ov(BaAR%uO*=%1c@9lS{{)k{M#qMA5&4&`^|D&B)c_zjxf&)QX3Sz4!HDpRD&D-~4!>(ZxrG#kcf;N^ zJ-e0g7VUWWt^{yr=vAEz8FCNfmh#0mn~u~gs6|ig%z?J;Y#P~KXA;h29p)a;CSMkyCupUEZd7LZpQiI3OwP(PSy0Nd zqNvxjmDa&lMnyTjYdMDihu6UPd2y@#d^@9H2^kFJ31x!uzA+fB3H^}VL@udw@w5QO z0!y#5knWfbVN`jx{owi|mFj=F+}tG|d`2rofz^Vd9NA)|+s0J|8@8yNp0v3Q$biu% z7aZ*`)MYt9AAaGQh%4xy{(q#s>#ii%l_vH-pJG#hF|F3L9-uvT&wC7}8*$kg5t(tT zxMZFTUiiXeJF7kBm5e)*-^B0-G8ky4OUMyk1^4o6Rj9+;idJvJc0 zxAalN-d=q|DGsaFi^ck2<*_9Ul#iY7zy^asPkiw8k+8HXry)zt_y7QDelD7PTzWzg zFb|(IpK}5%{G?kkviiE@=0t;s5kEwiH@d@-x#I(hwM{xp*e~j+3oeN~6T$S4TAe+k z<~kc<1v!3Vov+$%Nw_M-fDPqpy-OIu>?S?$2T%^g*d1j~v8ZZ-tXAdgPB4G13x#ya z_OP-bY7s(Gh5Sw6A?(}-=NdhYx4!<%T0Q^1FJAr__~LYz!$!3c;6;GcqJm~`fEl$G zu?Mlqp$*hJ?XsVs)PGp9NE_4?&%8J5JelTO+xdV99*_kG*e*7ydt@a`OY`mJ@0%S5 z>9E&sA%gz;A%{=g=TGp$n@^M6Gx71135>&eOL#3Jy1nmY}LB0Fz<@50J)w|D5&mDai?@Q8ZZlp~MrMLO& z?rmHGY&YyoU}NfmX20gC*%Yp(Z!|~&;(8I~PRH7A_AT`cjEq0JHY|^Mg}(<6K|-%o zPRB$zg>EnG--jDp@tWhzA%X$uGcq8cio@uENTHS%P<22*&KX|1gR&J9ygFEas2%*6 zS>HiPK?ertZad#)QW5K{p&E7c zZ6_arrY&Z^BTi+G6VUr>)prA-He+|?S?tZEq?QWd9tfZa(v$UG8C3XVy?_Thj+BkQ zgTWP8K^C4bovPX5PrNel_u5e|P)e0Ra3i=3CW;L=I_F-&t={=#E@;!WSi{I6!k2m{ z3a3_b40n;!NR4?0ImJ&8*|RU0EuV^AIBct&=;2;YSvbG!5zLVDKK19Yp6=~&hX^T>0%Wo*DGY=Yt@{;=>03v{8;sR`Xe{|+awX8j z=dr1vgu0gCMo18;@K}K^td`G`3XvFs9DIC{>u=P8HK6OW%?ogrn30>iTM;D-s1S>$ z_X-VK2i3#gYKie2NIf~jGo!PCjLuJQ5MCs{Hgb>wfW$s{hYdkkPtiafta2hw_~1w0 zgfg@L%R&jD6yjHf$FyL)w-2AzT}rrVB3b}LDXD@_Dn1P9R?F6_>$XTtflcV_@gK+P z8lEh97qDJ@tRkUtL1jH`n-3T;t=c)?U3O-Y%Ec=**mT9sO-u+&gYPYfg^m~CtF(^g z00x$n6IvN0J}_B;3&tUq6N`AM{c<0IiQ;01NCrxPd?u(dUG@JD~_I^NnJxQddaqBj;{s3R}TX zw9B%>hldXb;3{B&RCL@(Xmdf!O*JaA=690+$|P6i9z%rOu9W&KLh*`Zz-EbAwx#I} z1$veORSFlA*pvr+g0XqEtLI^O&eFKhS<1?8cz)LdweAFjBR$JNY#PFTi=~RiS8+3*d6dm#kNF34`qrr9_@I8unex3@T8{AooPjjjCuuX!R?5E4nhDe~hapz_?!FuTQ+m!n2`}HAlCt7YPVasszrKEB&zqD- zV6I$!E@?Ck4t>~`ta^%`MoMuNOsqM{Sax`ZnK?`$)p`#((Sa7rk_{%Kp&ivkH1`V_C9TsLSk%Im^-H&>#Tlnfd$^~+44&*#NV;lanN^xd zyGcC)j&Q*Qg!M`u)N{Nk2@@n7f~C+Rj1YKA zZ>K!Tor2xr5#mV%i1qjmr(FfdtDU=i43R(#>$YwL7#y1 z0-Ee5o%X|ELwyjbw)Qw}0M$#B&+C*842!a$@Z>G4iO5a4E*-Z>5(G)FXdrf;WQ@z4 z{U-H1rLms|3nFP}??zRY5ao1D4!WdrS8-MecbeZk`{G}ocd=x^#@rsg8WRw+y{zaB znLegfK8%yXttU(sj$r?t@-$4HQsr;CPnuXx0GNZ;)WXBkO<*n;IlfQ2y0i@qlqrpi z0}u}DAK=GT>^GgBxOH2G`JLU<=R7zZ1ebx+h|#n(EiLB z6$q~LqCj%*X+b~iRcno2J=#gr(4&XI@59Kds8j>p6&&#$EsE41<9%}x8;G9qzd8on zh1?)@?w(+P;*WC7)YcuG4G$!f@`Qbxy8>(DtKz4yaUw3|^B6lImAFTQa4CWd%2fhm zTa1`7CYRi-GhI7CBdHN%fp`hm;xJ2N(mkN`<}oIZ&y@j1<30o9-<{%m1?aUpX4RQU4+9qBs^~^cxYR!iML{BX zz}_i9Ep;CR_HHx5bj+7fs0hUenj_IxaJWH&7Oa3YUuFu2EuTSqj39q0yU>{o)KAW*N z>K37|4P6bF%u9@iK5tl561m5U4`m9mj!A3O(&{_`df42kF6$+Fm6#Di4qNOSMZ>+Iz6!y(|6y5nZJI!moV&^3h-ZGyGfs z0+>4GBi|3no$D(Cwmv=sYL6QHWS<+?*u&Ft z9Nk3WlO_KbaR5u>`BXQ-=SSC|fIY*IS8v?i8Vl}ujkXOuRI@|ci<+?&QWceiEEeV+B?l;aqi)u00x8IRgn6( z*9h@k@BvgHp=q`~&IQ8Ev`Fqoh06HYymdj^RE8(;w#5cXCiWmF%!$2>gtD|%O5HlA zEiJiqt)epQHRiTHpVjm{X>(OvQ4~o9#q@d)4jY3^pd2;JP?AK@t5#&Jw{n{>AGjDw z=7q;5@atc=c|@1%x4SpheIQr6JuATW$Av2heftw zg%upxt_tDSjI3qewyLGfZle=GMeuP;wGFooIHu{^Xk9(K>x!haG8E8|Y^9Enu(76I z9lJ5)5nO*fT>nb(&HT0j>9Yh0Z=k%LFD*M zb9UGRf*wXwVcp4-#^yyG2IS~svJAGmVK3*P@KLL(fqb??T0M#*qS$6?X|=NS`Eho8tEx@NMY4toW@I*$Xv~vq<2e!d5MTSo*ewOC2}6^p;Y)$qPhW z<1VS;+rr2&fo(iBzK4mU(NF>lg`UuT}jqKm2f)7%#_|@}j_$ZKtY)RgM9z zw{}|CpiAQ=9lP;^i)TkZ6d32HA%qqu@D1~jTJ$?)e;xpOzk2Etu8aokeKy#e0BuzGuNtvu z@SsH?iI4@8bFj9m@BncrD-maPtg0UM}{7-vJ0Zv1$oDnom-y3E9uRI{cKa$6S`))D!9Oh3 zbdBZa)MqpfPO+87KJ6$U8-6ZbRRZ`?@j@~i6T9|+uDK4ZEFrx!pbE>bo}L?|a~u>p zugTeht`u;<06SGu_8j=6(KB0lHK>3yChzF+ipAxNZiIfT?VEzg)Jf}v5#%Uwpr_k4 zP=D#k0}trCFpS`+8FDE@+novU+b~(@Wt5Zzjhbv4_&!Vw zAQkTn)eMir=z_+XI(^|pJY*3}w-#TVu^TV< zvQ&NV^#L^Kh_FSa;>n8c;G}CiRp3rJ%0DQu49$GwTP{0?>4l_B{}_v8!N|W%zC&9> z8RY~ssE=R2qK8T{Doiw#V{mG~-8N~7=E@{d0y13QZ-L_na$8k$AzGkiL@(J4*Qytf0z*783v)RHW)+dz6>AptOH=dPY=A%Z6wC z1>In?qUYe)-siyT<=CI4D&FCY1AkowjG~py0-n@NLIlKzr4#zhxx^-v%uI!D^AQIq zn@`IDYzmR4i8BA}n)9)GG%;RtCXjQpKV^`$(hM3(D@ip8)+^shXCsmGx)1hrR}{;| z)ph-sBChgNJvOY#pEgoG5-Ju|&mH1`XDZQd;9Fo<=|O<%5|ayqDjjgvK;OAbBS)9* zX3Bz{4G>WFK+juX39&&>rav_DScf5bD-UXF48(0PQG_>1oQeBU2{1df1`G4-(>8;F zqi{1W6($@eRp%N>?tFqhow`zZ2yu@5--hp}g!KWti9av^eE8>o4(S(QoPMqn7!c$6 z;p@lptBR)l5;LdPZk39$`6vSq#QOTAiopf>wr3sK4IJML?%q03HLns88h|6Y;)l7r zG>CRyuvc`cVJQpBxb?AU_87ANj04Tq;&SmJ2e5NK%eIbrdDLxCe4BKtbLScjQ?*Sq zJmUEj#CT!-qFrxP!o3 zOaT^^$xsWy2=vbO!04AM9i|SJYL(Z?5Am{x!R}sjH!e=IOU_y?k)L}VTT6ooF&nn) zD2eGPUrigJSpCGE(M>`9ZXtktY2i6iGz`t1^)zLL8oTszuxN><{eJjf`p^@l_Z#Pe zq^~9>KzjW{$Pq~>J{`q17_V7P+ac4J-YI;>>c#yU%vjtm&mEw=A%jg>KcSm&RfRG4 z+G*2U4;2J1?g?p6qDDaXwz^8*c*h{E*%_1D{U|UQOc395)`moX7 zSNgb(C*ba6pzTmM49f2(K~5-?`4LxfD~tx^iBwPvec*>-?|v?Lb4OUYatDjl+w|`({W1F?4gM>>m~O7x zZPnUe?fTO)u99f>7KQV9(t?CP!x>SWJ>$7>-){-S zWc^D`Oj%VxX z=i9h*_k=0@sa~*f+oX$@bO`E0DCN(p;6k-to!SNK+{T$&9s0kL zlVzYQxdAMeCAcC#z#*GdI?L%%RTc}&N5dZILYVr)N(iO2;(?$RXzkme7kpUiB}?W| zm?iDQ>N<-|&#EdoJ0mPhR%yhWM`Kx-)ZxC-;jEGqS|y!`u7@XVpchd^;z$+Fwzz<$MwZRoA;sD` z{-Mizu1c!V^8~u{}(JvWne#~I=W3557awhW3PuU?!L(`R^RKvT8x9`WMiI`OSvvOTxI4O z6KGcoiiO%PEjzP|l789=Kz}FIHVveWpImqi-A23jk%%to)<%@}7U?%c7RVT!$?n}b z%28IQm0SglUxDQSs#J9mU3XCxT`DvL!elteNUVG^P8__jpo@cTyTnyS%zWmEM;nP^Kn%dw~P)!xl^tUquQQ3R_q7yxPR1 z=hi^Lk>x~4gL`TP8PxGNGM#Ga1B~CNjgo!x zxF&MY0r2W0gzVJDM7j(B5Z4Loa$@$d0B{r?K6w%d3Kf=mOs#MKUlF@M4= zDEwo(9GIrhTzB3vOriQd6!O%j&{ua5yEwXfPVH^ZI z#rIt^z8AtobIv8wtUynrnF?tOif1Ahm15Nb6z4#zv(yv4g;hYFP*$BX8W`-kL_{5i6C_-L z%b1Hu0t7aCwv&-=Rr0+5ZFMkZrb?;NqccOX0o>gSLSdx+aZ&^77-*Ov&qwu~R5zQv zyaihwDh7tO8{i}RA^som5B>Uqb(A#MPk~^-iCKA9JJoYs+-@ECa1%d3(RB9+*J=Dv}=hMc|6PEN#gQKYC_F*asX>=;YYnJmZTtK)@>mT-QF|^#cg5jJkVO6nbiSvUh_9EY30Rqqj zly9}Oi<^PV0EU9xa>+{=re^?9K(D_P+gQmDutvAG1f4+H1dnyYSO7VXLJ=xZ_B;D7 zyUJc6>|hnYTUCR?Wvv%{A>3~VF^g^E!^r#GoeP^JpH=j}a~f`a(L{Do><9ds&^_qw zhWI2EyRpF_dl{S=-%X)?=-@z>FyZbgxz!fs)GiOB++0g2NGoj*G1w?rDf{^DyzN=dJK@5ucb_2yqx*2@cPG)(;H|= z{|{`j(?t`_Y}Z+;jvMq7S4MHD>6?y1Z?Net0=07sM|`t)8Fep}r4$4lxzwHHD~dD2 zqRTIG@*%&Ea&N(L*ovAayWpI~Rqn5rem3lerOUxq;-~NL?+yT@qq@hq`muW~$^!!h zYFqYU4qR=(M5gTD188oOMy~f)AUX+NF8f$^m!6{wxsmz-^eXi0gxWc9GrOXCsa!X~ zeM%Pd+w+vWKRQwNDkVOR44yNrl}^I~_}4fdr3)&`b<}2lTJp0$)S;*&HedjAbfHsL zM^5lHlL)U^h0tPI^^C89D@EM#5)T(b>UP)Q-W7;UbgTGu^*eMq%Z9YTYQSeW`i(3}~$L`+~ z+Wnj&u1N{{Q%XW7!05Yw4li$k($CT7?%uq&?wZ`DsZYDFeo;3w!%1(@V%yz@51UjE z_W1zAPv9Laes4u@o8+~bR9+6dy#S#JtCQaBs@6;$xC3wO6p%Lw(QXhq?4hq7$%if* zU&akpa95$B)J#5D2xvi@OM!o{&^!@{wXH}Ha#TuIau{&hZ`FqSs_ZZ@?iDRKojNG6 zO3xw(1%r%*+OQ2CH*r7bVpC|A0<^dyhz|LUV{;3X9O_-B4#ON``xc-w^Uzpsp~}p< zSVB@FyB-EGEHO*hL1GF16TyES1mohsnEk!)h3|bY&GbJCuQvPtZ{hV8{0D#%)*B>u%5Rh}+H&SBT}5atV#g={ZE9ApE7<5p(WIxIU4juW0WmXH<)qs(&I(thd!vG0(!5z6CliK|) znIsC)PCE#@yBbwUT(u(m2>U^!sxOOU+MeHKa44tY*;vk2s&1iF$^k>6xHAo z%B+*50yM31J(t*Y4}4m%Il%~=S;)mr3P*QAmS7IlV*!AU@fc^jwcQYgE@zyod;=CYeZM&Sy z8NXTj25{U*1@_arx1_AuhqQ+PTk1 zcAy-`sIa5-8WbA&L#!2ea9;V_?}qQ2 zVCaa7T01?tbe7apYSs~Pto8w(m8^_r2z8uY@Eh)dz=4iB44;h1<5RnL&nRM!wu@!kl2Zci(xJFTD+D|ln%!=j>2&?H;8@d`9 zS}`sL(Se4rj~S%%UOVuP3PPptu4_|*5Nq{JshXXLx4<*l6h$#jZd`Q^)dQQyeIkEl zh8$)yDygq~cI}h(>PxRpn$|swf=I3lSE4UA2`CdM`8#}3dKxAJuZ$9SA$ZBy)#XHs z&>f*$y>BM;%v*E-ok}TAOSTUkrriNLJ=P+TgAuhCxaS6YtXBX(dzw#Up2NjDOpj3* za2y+{Dcn)tbYD-PB$pPY&k1Em(EnMbiPOUZ!@Y@Pw-ZENOMfGQeB+xYYhB?Z@Ei*U zq`gF$;BL)Tk(f|4cvP?@Q3s`br_>?@IBqRkllJ=SzYc#*W58eWUj3SD&)+!4;ytXz z9U6JuWZc6zB@i;KD$|ibBy|vyeBOia)Ui=dI?v!h6w-XU1D*6Qq-M9s?-+adUKldF zB-}uQzu;7CjxM84v%s~tSIk>k(|}fhbl3^OK+aq3@Pw%D^tCwKg-haqwyaT5RVHIEQlDeX+Y(#W>hxN0>m!#aE-b(4W zNZjsxZZ9$Q8PZ%Mm0eI`ayqjj;!nOs#`RoVc^)=Otsy-{D0LlGNx7@I-l!?VSd9e# zL75m!N6H>z(ojxhnpefVyQVni;|2Ha=(4dUSCvA3t=8yiM-f4Pr&l^w9TswC=(F4F z5rLs4C`j6`y5?W~efLHD_OCf!eIbX~EI>Yd{p96e|MH*V@eQ)cSK;Ne^k4jU`L{1g zh7qhkRN&N&mwNMw_Z*N9+5+v1uc+Mz4q$rw@Yj7--T2YA)LHdRT9yu4;~g*fTHFsL zn}rsIilxC7Lh)M=x1vLd0(J$2sn1=7C2spPIv1tXOPHZ|}BbW0yT>G-)&x zj4!Yad!}xY4@D$ZprjgHYgf%w(1ge**FXrtHdKmo zl)m%t^P&0GtC^(!)>y;u-b(fH)9~(RuU}iw2`s{OA_U|h&}a#b+z+D1RUTzz8jKj~ zkSzxLOQ{J-0o9wZ+7wb4M6nNeUJYGKwLg{OeB|^Y$&k1$BskDeHoi>~-@9Sx z8a3LHq$oEBbxr6jz{$^>$5BZn@~|KFtPFH=1{7I{Yp?y~C7C`Q~jzy9x3>c-X-S#9k4EUkoULt-H zd{8ZTcC3!21U|baP<$|Vf#1BdP2BSla^Qpq0096{s==@2*eg0}UnmTI-X0bP@)M_w zeT9JmY2H_rl1O(58PepQPl0P1Lk#O$7T!XlMsT*9@MFEWKh9_E> zZu)ps@W*vH_yC|9gmAeYMCD3S%%!7Xg4Xu(9bdU`51Xw>fF*YiE^tGpY)Ynj$!xzg zox6=tt(0+SsqD+5Ld(?Rg}O!IHclY0)u=M0h){55Wir|*#(RmE&Xbm;4E1%=aSF(Y zFm#5|w(xgHZQ-ZPFs(xQw=Cl5wD4`m1^)u&7JpCK#s9DG!~fqu0KFmY-Ht5Ro@Aup zYszqNFV>dO-m5Q;^@#cYsJNojuDSgmJu0EMgQRxeF0@O2rhuZ2r@0BWS*jLSpQTDT zv7up7v9&=tbTKF~AQ5@k88xIUfoLX~cBr;eWKbJ-s=sk1b&cEL@(2h$ZKrO9Qh{Wx`d;VGjfyatE8L+n>?rPe z{sA(jK6j1l65NZ_+@$POOWhxi2U)^?+F?>DRis->#QZl^3LkFhE`)(kvBa$WW`dPm z4*(CnPUlUfC;z0V@Yk=C!_*ge_cpxz(ZUiYR5n{cu`ao7Wip?VV(*!=KAfzsZkRtF z&;}}zERT6oab(lx?%*}P(=<*3T#DAZVZb;a_GAw8D8j8v%VZ|#P*^k5qT&L2*H>hh z3Ln5FMA{N}7|l@Ur{)Z{-Aj4uX7VEqCx@sll+k@>l%z|1R-7dH`%^sz5;1}I@zl{5 znU+#@p~Dmcy0Juv7J)zy%)%_zKy7FTZZ1cc-?L4X7g=9~TO<8_&+>6>6L|W%2-+;G zOL{AOE4V0sNhc88?wR;@R>Ieg)}#!VozlZeSy{eKhiqpd>v00W3{Rm7%I)!oLVM6O zKvEi+J(Fp9SmeI1TXcz>^1`k!`xV3o`2*N(%Aq(XV;@LHBC>Y%?z=U74W{Kl9brV)Kdd+O%_v%Rpk?~ zVr7F?Q~(46f^8xgFdVbgXmWy2!CJ3dhph)j>$lE<9r8#)Jy^0@f$E)5&ndm@*yeOs z*W|-qzV&AfbieOlfIB6Kw}pl}-$eRarRD3zhGipAFGNa6`g7bcZofox`MLREz(-@O zxAD&DB1w8$8efqmmFZL4yASd*5mwX04cc@zgjK)>R1(i;iN6KA3z#Eiw{kh^a!PG; zY9$?<^A7E89J5pd&!Gbo%XtqoKS?jsIuYV+Yqa+@HRS}T`3z}SeWMgs3+)wzc}j>7 zBBa`ZNT6_O{4j^n6oo2}t4{pJiUw@gFg>4upXL{!tH9PGDRG5Y8^(i3^@Bvx`vx9c z;1wYqrc^nFON77^?ks#f=hH0^jRYKyZ=Zq2pt3SVy7n0f@Hq_mFP>qxwi~B=-0_Ev z-4z3<6jJ)#`AAKp0pL+y13B^l`Q3I2d9Z? zPSRPKb|UE;1prijK=~DR1m<~M!kyRCU0)>2Eg&%uspdvr%*n{N#RBglJK+U5G>r|O zzW@F2ha|52kQ7;J|LXOt{M9d@wQMg=#{pTZ;!JkN9a3dWgqV2+SrPlIAXos@L9BcC zyq>uuS4;CviOkaltuNt_ps?(Nc1~H9o&gfi*wi(Joh4}k6Itb+W}{Yq=gX(~jw z!uHz%?vkih`-U-ealgVfPz0h>B_dV*&kmtQH)Ovp2$dFT`kY?Mt1uZUpuoRLarT1` z_^dvAe1f0-MT-3c)U{z1AV;E=2rEAz9E%^!i_bKgAcJx4{W8t!>*ZsK^{xhyzyu7H z!$no2L<-1>`zHCkZip`>Jg$zM+hDq5dZRfAYs&w6rE5B!uFJA$40R$wN9Y>wqKxr2w>K&iHH7mh>CmS&d~P0gUhDf8?8 zUHa<3y!;}(`)}Ak!`cO`H3Bj2Kz>ZpHtA}*B&rZ2mi}SSVZnKubos}*gnX;#Ex5;x z`Z3`e!H-GGu*~^i2V}_DJ9P}cep$s2c>sxunI5-so+EHrRTgR~G8`=e_kEmZ66B_7 z-CF3>+n*1^cn5<*pq{=zsz23`z9eTG*>o}$Bs&?t? zZm-pcY~oT<4be8Rg#x)A00vS%T-$M{Ka>)y48t!?xwMvG&3F|nrz{fG1SiOjm5b|w zs`Q)umQzu-Sg!^1M=e68eLituxl5;Thob+(g~euI-A>!SN=7HOO38jRYm&j8BBV@B z!t{nDPMATrivavd+mWte2NHaOq^7>SmmaGjusBDxSA(u;x`uK_nsnU832 z5q!7sJEHYYO=ckwuArp3LNz2r$Tz#8q#Q{xk`tsN=r${e?mYp~bNUi)WWJI>hd4cME!EgyWlUZoY$yziECKavG7MF*AmYU^GgCXk{CdGAkd<)D|_x?|DFB+5NqW_ zH)#+{(M4)zG4QdJ1EqZ`NEC~Xx2=Iy&rWv@1y>k*R0%&8h}XjvD=`T=DCSihT02?& z`B#7c58>~>n*;FMzx-$TxBi7f%U(ZBv%HT^_oK=m>>W&TM1xfc6;9m6m}r2DGmEE9 zryyxou`0qdi^1WyjXZ9PRDfqD)_JO+yj0S0clQ0LhHx1cUDydl2j){3pQ- zAUOPl7Rx~s3x!jr|AyH(X+fJPn7qKl{uNbQpSvbB`+^Jqlp$F6R;t3oX8%&g9H(1X zW^sZ|rDpGIdgT1xx8g$JtU@e70CCxTEba^HcVrFTIm=1`0B&qjf1e+A?6;s>i1bz( zg(j$&p5%by-4CQRB|Ldhx^Q$4MpSWj)*v6mo_A~u-veJE8SqsaHnJ}32e&JIeG#rv7$G_3-Q(3Z&;d#4oKA(NfR&gNWnIqxXTAn&dv_xgcQuxJ6RN?DIaH^!@1v`1M1Dz$M*K$eJ3BK zKfHb(zD;lc^5rLq{m%dJ*XeLQ`%}5?Y3O48(G970bwRNWX#|RWzB(#xWRDCY0U$o_ zk@^J_99EaPSb2ip_cXHGmciZ)f^aP!q&Hg!h z{))F^o*vXs0lg&-PI3}AXmT*5s?7SFN^468VLDcYm#OvJlm=Py(Kma4`p(~m|K&UO zz`u|GrRy)B?o(R~@`4UhK-rOcu{{`&;s@@$(3dFdN{Z_ilhUC*BV;Y(mnul!cFO=4 zwvf}Kj1)$+MjFYkxYIh#t^L?4$e1s%-2gUJCF&ic={E$7IaWV85Q}ifXEx;%g zLzBk^CcT)fdj2p;7>V!T?gC%H;Br^ybnxc)<xzAhO=UQ?91}; z*ya)&7|tJ{jK6gywTLI4ocIKPty2V}R?6`R7IR^Lh4eMugh}?1xa-}@h6fcC=R)$G z(S@y)N(uE4RW?B6wblwfa zcH|X%Ew%B^Dio0ZVnI2Irqc2XnoGLG0_#jRifPfilHmj(3Mp#~l95D)G8wY)H%;}S z)=CQMw#A-;89^MyxZH9q#6NLYgo}ecVp!h|qg`;16d7b!hu84Eqttfb_Js zn>^>OxKlT9!61s`8|YSqiTd9CLtS0s@TW6Y$`20%{U{Ip4!9MDN@cJ?2$ke0(L&5`%y)+rYX6@k$+5!3leXn4%nvBHFvGE%c zeLU;c)knSAB$v>?HqNcWREUZOE6L_U_*OaOl|%EsY4tea<|^lP48Fd+$zOw(0JH7% z2-kHFbd{rsi(JdIrUbi}hZ;Rw;tsim-ybSv9Ak^BK(8~oOQ6{}@(sj9)px9hYc`Hr zVAKmO=H=$DfRnQ#1TT!tvO?=W>mfmK;3Eb?OcFxxyCQ%+%g#)03K*|R0o5rik`);O zpITTc3dB)Wg>)s6OBrZHTYR#$m!X5-{mYB04u5_+iqWG|avgjisof(SC-73* z6Z$G8th#h2m>eZk{N!I8IB7(bs*X5`6@?prt@kd)eiE2BxJj%TNr29ljCuLFZ*p>$ z8{A8%h}xPE$L_esu|v-)zi*=bA?Xe}cjif(r46?1&=&^|9q3MkRm#^;F8NRv8eMYF z*9JVqgv(B2>D zFm$+itjZ%{4Xexx&Ni^{-vU~^DxP*N%K(fGgo zWK3jSt&OV@xN9?V0Z0iEj8)x(FQq2@!<8b+a6TK!4McdFyrb`2NcyId#qUn< zK6?2E5anl_zSGlFcQzc( z7a}$2fi1FKM0m4zRFjpGpTHRjv<(AY)9s2=(?yrv|E6o?qb(}$ezdG-w_o>xXIDLL zAwfqn%W`1Ji$zPHam^r5>TCf^>f|^}@6sL|Y&c!(J*UjIlo4lX%6huVMS`im8~!K# z^{>KrlQkIfL8=z;`=A!bc959jYmb2N3W!5M-gs zHxj%F0051kt59zA$|>Qxzsqo82_#s_TPqM7GP!y;*0@SQyqLtb>S-J3ZI_Z{I%Ag% zKHdomWiVUk2y%it9_*}1o|h!**7g}TXck+!%)N)29U0ceba_!RS)S7Eu}s<`7(J5xQ}I_jTvK{E^)Pe& zE-4Zv3X}6G--T@wD@2CL>Z_?t(gbJAs9VeHJYdhk>5uACa&~xxC@sC*g6;wAbtw+i z5%plUfzN@JxVoN=3e1=cF4iXCslD!2k0ElT%aZxld>*AixHy&ItnQlp?eb2241Xl? zEd`z^pUFu1xiK%W>o`kZJJeumj6~`{z=*Wqq)Je6ej}|AaPOQYP8lMf)G^RMKrD{p zVzYZYNT3*)mEL{)^4phRNI?JB*FQqe{^jT4-6yYq{PRD(`{?zvmk(9p{_VR@UVi=Z z>vtc&{^Io)`jyX_mVX&ulBS6o6Jjr(-l;dRucvOhHnmXx#O_tC#Ox&8b*e!ifvVoQ zxzqzlL7hPI7*?8~s8`6!#3sjzc`ti)0}Jb&M5i&86uFJDO6+2ss*exeCsuo7_CdIXh_<{rFNkxh;SSp9DU&c>?I6X z6h&!U!|#Xhm`ml`KmW5FlDDsa0*Lu%uU}*75W_je;~VHVr1K(QyHz<#dq`eTwnjd^ z_5PqKw+@rbX|7kG&$Qt3fpM!ccGMh<3iaRIM&?dN5;~{@ratbc1ZLO-<22uf+KOr- zJOkz=g-!m3s}(Og0rCm6TOAtIcPMnx=C`unEbCv#@FcOP*;#F0q)EV&W0XwO&d^Ob z6{DgZYQ~SPIRTABnlt$CYKZahfK+tSM+Z?FRtcXR9z4PGgdFKsZjWRW$La!TAdTU7 z!vBg@LAD%)DE-e*Fz78^=;|0Ac7Lfab7T8^t6GEh@7g5MZXC#am>o7=?SCt<-KazUR1iC}k@5yIl zvsB0r;Q?_O4Wy5dRL)njO+lBEVOORa(oF4AC!p;cWI#egrHqE5b7QlWGNfrGn7qBllUYtgk z*hDK^RHv!~OOg&2FgrqJflbrlX9<=b?mmH*PU;RQI@t8347Hh}EMd13C@n_1}kokpIasd;OCMkmJRP(c`n%pS`@ro7dkb;5`VB()`JROvm#_Qk}@} z|3~v)g~kE+r`G4m$C~aX#{U}_%sw?nYyO@Fo>Z+4XkM!po_la1^hv>!YdyT#mEPd# z<;^aIF`1}9SIPOvRl|sELNX-CmV~uPF_76!5^Oqu{Ly*g@+2T8$%S4VX3%gXQWeiW zZjw$1gM9gZ_L9=IJ%uh6kUKfz8p%FDnPz}Q(lj=wBqneF7NTg+e_=oMjam|f*Ke53 zzX*T+Cw=|ASJ#Ke@gSa2N<~cFY^8dKPA?!b(r(|~B@kH!CM6ueL%$Mh5@HLyhs7WU z#ru6hwtkPEI6IPmQZPW^!f?7m^JkD6QS!h9-qK7VRXEg0az~5E>i&j3uC1P=6qmbP zDTS3QCv2jn)P@YbBp9kw$?w=_7Vv6E;{*I|m|Z8R&8VMdA9*GlxLu}lr zsz4ngZQy9Me~Br!Y%kP-mSy_8PXN(9cC-Lm=g%=~pl_1I0O_ z%TN~0NsDW$(nrPq`F!jgau*6op1kB4@!cQT7-o1$=xgN<4x#gQs8^sJSuA({_|j_J zPuJ<1#H9c*GL0trI~rA+RQ_j|wl=GlXGA6GYx`*$Fbx;7Z1J2Q=A5oro}Mte#F|)p zk`uu|o?ro?34QYUO|hWg>6UAj9RgM-b63$X7Q6L)9o9?-_5$jFje8Ek2|n4rkP@ny zc)ULN;DZNa@k}EPFBV*X_WJwq`djJske_TFwH@JRn;RTI#gMqX?+WQG-55`H_ty!8 zVy@tBsNRRzXT1~Pf=@~}RTc7S?U3wUS~z06e_9|MG@T}Vu5WhR;lWoBqg=1z1}uL3M)38e`G?2u=<3t zcmsKD3P&_iwF%^c7(wz(<-S@4fWV`e#xz~>LICcAU>{;2haZmD@L4*DNU5nYxX*E+ zB7WSqma29E?&luP(fhc!uRnYJQ;=WY8nxn1r{$Yc^Gia5^dJ_t|CC~zR;qiscC{-Z z!s%&z)Eg_#@nL+4g$k{wVu(W$o{CgY#CFb6V(!>AM}H>M8RrXU#$q*;<}7WFFp5*0 zhl}5gH508`P>p#N+i~xw&Ots;>x=6EoVqlJ(|D(23{LFn0?L7cL%(GX*sZxBmD+Ky za!1-gg5Ex9dE{8wV}xucwkZ@FWPx4|y|Jgp=Wt+JnDP^fs=?N+@;T?i7=MoZZ$cEb<<%6LveZFD6cma=wN$Kg_yfaVmVb9=nbxRYv@Um zW;0INDSLVXvYX@%n{JIz%}3*g=|?G79Bu*W@mEUw_Hhut;YYgGfB}7*ohek3qKuF) zOV=N3)2rus2q5|dN%Fi4QYvluBypp%!Ov7YD$ZiT=eNJ#F$ha69a|HZrOAGvB0m#K zuHMgkP-sfp3tpKgnMy z`HNg`IlwC<+G95RJ}|=ZcK}7(6CWRZq+EwSEqU*N%)2yz1J0!t@Iql;56prgi|;N{ zYKWY-`M3|}KvBUTcbK-sys?J`;2z=V@}!%jpy~1bUhtVa9ZR}G)OJ$66(Q1Wtb^I3 zN^LoG{N|suE>7vSX*v zE^E)wv-pgktanU4vkQWb#{r^9?_6B2*f)$dq(SBSBzf47TmEyC7EFt;X|?_D3LE*Y z&2}@IgP@8qcr#RDw`MVXncA+*%2_DhB+y6{2U*jqa}W*->RMvDDj)e&s*;;MkLTbz z@jYwAUV)hw3nA`CCFn^-OKtRb!KB_G6&KebhUXi|bfpCf<SgiknYgLNWQ3b+B(0N?ug?sxL*cCY5^3zG=uJVft;ixa{D> zOh>Ci#rLIwz`CAO@I%p!_Kfu7xytU!)A8j1KKWA z#veNDGo&)C9>v9b5Oi7zyU&3d^00*pF1yjM|J84limHUN#KixpH7`fc2Vz?3>-I6VPSoB zN1Rm!9o9oaxx;LV$B{XBzaYJ$NNFw3toYzg%gv^n3wX1O>o3T~C;l5)@OdlTDs_Rp zD3;>(p}yIRTK{DG_iuvTY7ENk$--58@#8o^MsiEpniFts`vTBx04lm6oFs@H(rl)X z3nAI!s=Rn9F5OiKsJn$gi4EFaD5G`~Js~z{hX`#--TBU_NuDv=9Hjb_;vL3XwV~ZE&kndYo>L((j)A?#0CK~>(O21j~;O{Ls>1vF+Q3b@X1@d^M zLLOZ-6cL{uB0Y_c7A5Vd6@;@aVXNCk>%FA+9#vmYFRrdTF^G}0!SK}1xp4!oSKpR3;+%*q**C^D-foAvj_vW=?I+RZ1G zh@lQ+9n6>^)$--!wnCwC=NQ-~Hjqk)$%5uGsXiQq|9Yr7?k@g^elE?o?K}*fYkokn zsiD>Lge}Z+z=;8dB1`uKB@73| z#)nBMq-&(RH6`}71vgfHW_JkfR$kj@2~z7+b$QN%)hN`4z{?_JsvM_f&{$;{8Vn?J zT5eG6Jnk-c%3R^e4;YRtne>-fm~ve<-)Ae$-wq>W0IE)19DABcp_a6f#f+|*NYfFb zL6&ZH49P79#5Cl}Fx&1~z~>q%gmf)xI1!~AaoK0ZF{LM|u5F`^ywU}r*%tI~b5LhX zTl9|Q0Eo@jt!Lagh*p< z9mQfZ%|gePi>v$C@BaXE@DCoCabLcEK_i|o{PKRJ=+c*u^q;>>NYmd~*4AN}T(P~c zC43{R*k`7bbH_I}fnj~GgEjTr_P*$%gB_G;%X_Zm_}$-x!RIK_74ur7MtQRn)W1M2 zi>xSQh-X`b7kYKnA;=agl?}?|d|=|-PZTRj6 z_bjO3A|PQVD^?K;`>(EaW52%rmJZH`uO9}p8j!Q~Wq3`_*YE!KVC2nb)d;Y$A3ULY zJHDwn^fE(A!wH&d`(jVgEHD)HnYHaS`y?~3ECGPJryz4PT$!R*weF@y^GMs^WbbXM zVWGvSvNEKhs9lJ8f`x_!;ql;#Wt{dKusul|ypBo%96GABj8ths0($WNKF1rIIH=1{ z8;VLWCOLjdfGc~6&BRiW>LH&KoPCQtu7@CxwC)_q4?c{oeen;Pxds>dQ`B85QkTP7 zQMHb(?Gz#x0#**$Zi9eSa54v_!P(=*8OeG?F=SD0@sYODJo~~NuR-u6$rAlaP(}50 zI(z7lq?ct_7n6!Ub|twVL||MocD&{rR3N`%(BjFh3Ka>&uuVwc#{U@p;U6GvCT(&@ zki^|ZJybz!?R>nC$-k6Ar!8D0@?*%3*+RAuD8rhiI5vZhyt(hZ~HySgF2Z~$Lm|`S=a?cJ&-xX>R;EMo6 zhPeg(qM}Ynm@7LxE2bM{VRagiVIpe234!`r=SRJA?69E`;LS{wBBt0+5vtAf8Gscd z!_D5iT(QH$N_FrVQ{SO=3JI0-tnYjhBHa;(s8Wm6eElpb0%N}#L&cT18%v$^pr+(g zo!)@k^{H&v-^f zK$zXbNOh)gOUD z24)2PL>FRMaKg>JFTlr#Hge?!wOkme0+ux7#*)T_zllT3rIr9p4OLs?DPf=~%Ln`8CbQ8V(Xo0Jc! z5);5j$`AT%hXUeKC+xmP$OLImWG5P+$2%o6aidbXmNwK!)=sb1%w{<9_VpW-!uuMor(KIq zy3;~CesJxm4b)PEv++H5taj?M+5}pqSfAq4tXG)5I=I|G9F0XO53>&joq>^-HTpGNm7ro# zkvkk$DBo{!O@8!TT_CH&x^z(s^md+`VsspYzCZ1!n6uZGtmQWT8IG-}+>a5><<}x~&a8LCWxMw!^$j3Ki3KSPR`! z774juY$r5d&ioE_4LJrme3IV`MUu;oGjxyWb){SxyMTZJY9m`N1h1$H_YzfBRZ5Ub zvd~Cu-=@s1tCWj09qJPe-^~nGpfEEJwv?0z>a8P&7FPq5IMqVRGYv*Zvg%amouwRF z1Gubsip7wtdWf*77a$PTb8FD4a<+YDs}2M>F@&;ol-s9$F-yPxfZbql7$@+m5?dwS z7Eg+mo)Ya(Vs?h$*XmzKH(@|o3_3KpWOBl7*4!yWR9zYgx{jiLVZug^p!*r-2~r$l zmPFmOE5{EB0tmWxvrr}Ora5tGI{Wdsh*I0mlN40w! z7>Q6#dy+%vFa;3B93iKM##V_!JkbVHg-%##D4-+zfh`24?~@f|MIj&bV6Ke@GyDzh zEk0%f`P0j<9--cVr~Tyh=kldbUVof^&wm9R`MG|F*E|LQdkusd@ySztr zdjZj8*nFP)X2)g}Gh+OrlnbaYlg`^E4&aJs^gt+)&*UWKBs16&Vg7QKfb;-REGH?i z0eT=ATrVjF5e_|>LmJIa*09q}_7v6s=XH@E7 zP(;VPgW4uE2@e9<+qKAji2~623A3$Nxb@=V-( z5sqhzmx3zbBTb9?YhF1smrmU_;Mmo)(Gpe?m!79#js{8E<=ePYM_LE}Ft*1eN*M)f zo)u(Y2CX}@MvnT%Y>*QjH#l9TFJ41O_XK%R4@#2KvnatQIY=()MBl)A@)Na9O;kHq zHMa)8kJ?jUg21a9_GJjPU8TH!7V}s|1v2R8Y4Wv%g#LZRvq{atbOrAtbO%qw1ZKdn^jSKoL(>=GxpSBaU_0q4V_x4UTS<%IC>DrL7!{A}GrLNgcWaI+BxaRgBtHNfcghU| zoCK*XHB(G~^xuXDN5J1^V-pyF;O?CEr7Y5rA%$hUO+#B{)v$tuwI(#*M?b%yM)PHf zhj-ePzPm5Udv3OJQPo8?YO5n^`r;PlAZ?($u1kI=Z$ib<6c8+fgB#NDP6B{3igarY zw(L}KAEA!79`51gd|HERjH`Y1NvaCk5UEk)45>SZhqD7QMPkktdtgp6>a<%f*{F}% zryj-@S^I4$V2F!jfi*4~8R8Q$+dj2rNPRt~0UGj{v7^L%N9@ERurJ)hrNbOKy#-(= zxN2FYh>2$&n-yKrp8G)g<0EkLo{AFFsM-kuK+O-!LVDpr)gc`uC?GuOos>#G_IEwJ zOiwUS47HpLDh>>Luz_uHRgd+dkVYhK;pmx0>Yn~bKcWB8Pfmb_TK9Rt5Jb+;J_n!n zVsDXU41rOL{f*Y_$P@#d5NT__UsY=$y@4Hu{BA}dfDNOkR zODV=fb_kN!si zuMUXVrjHl;p&s?57A5H*B3c@hWm_n=zG4%#2Gzv2FSM$CHG6q~n36=Ij&lsCWusP& z(%y6W1OX^Ftz5BA=nvqgDjiwFCemP*QsLQp{(!$|dbtBtFHQKvij0K>>~VJ-4o(6F z=tJgl$B^xWJe&DO$o`qdC-uD%psS4JGpP1LgZ#jO^>C>(4ul6vQFs=JT+Kr!3@R_C6I)aZZw?TR-5v6sK7MiPv7UQae zR+X2~S2*J7AS`CjN~s9Q3thQZgFY2D>5V%3O|CH+{P$Prbg`nO5^0tZh|C8AL<94Q`rHX=nsz*bNKe{e1#Sgp#I; ziTNbxFFyv-l?r_~o$oqj8cQ21;WynXpTG;xnjMm^j?}nIM#ThpRYn5&0{AhF>W8K2 z6`y#7S(_z)eCDpb6_bbQb5>kv(`T={8WOnVR`gUIzV18$mnle-dXXD6A+lO~EGLUz zhsjeIeWU%LQh3dJ@`xv1`~#Ij6C^z)=~#U<-v6S2F2lW(e>8S!Ad+tFRx>$=uW7pY zzBH0G*F=-Ne3ZVbbo~$Y_5Gkogv5O4{|BWI!tK>0w-y(!*mMPH=@UFpES~AOl+tO* zR(b@F!g5}G)UVS?3#IJjyG;*lcOs6d=veoR7=`xQW+4@NJfMs)C>{G>gxvQ zaAHqxb^T>mHNx~-@0%sZuuA2HiV4Kx4$?Ytm{eH0bB3ne-i3D=iWyG^FCJ^Htlo3< zfdY9PsjAc$_}J8$X|%pV71%77?Of0mI&@~K1nr8KABX>_Zw`P_`sCUXW*6SQcYUqe zDiG!FAaZ8W48v@Yx!J<5yWv@!0^-Dvu5lG7KnG`Lv3RWTGf6fniOM(GZ!oK zE~k+b8F?!SEK&=opxuY`+2d6$D(lvTR;r*MoTfK9=j0?M9H>WU1uEKqG?nsb4a?zl zEC7wreM7P;ldc8?qB0VZtnQCA0FH1@`d0gX$$Gn}tW6Q21v(v&D{qvxV+9-wb&=}}2(g(eLwsRFycHgG##6^cT5+zClV)+l*LBDfmn z7BeYyxLeMS!L-BOxQZIGHB!;o>ab#5`@0Xq-#uU^zb2yXYwiRS1mO4YC3r(q{`sHq z)jzU~)33qaHh=9d`PL=Wfi)UVAi)+a0(!cWNu@ALE4q->w{GKu9QCT{KrW+Z9sYA8t^D!_R-2j`z_iQW!q`Usc}>Pg8%gWCbV@ic zeUY>THQud$2*)V(`!2=8s6v$_$H`Mv-&h(Kzz#aA3tY#2%LC)KaC^c_LR^PSaY)d( z*l|@2&AB)lb3bm9lVdug64jb`0?U4po)0V+J5N{-8991yteuv_pV{#{n8HuKR za-S3T4=lZ{lveGiGG&e$O(?H0Ve(Prg$H6l3!0uNuqa$FM@>pn>m|osiAuFPIbP!m zz_~7kD(0l;XNsR)!W&G49!4=K#B{dLg9+YgcMG-y8w38LAkX(hQMM8IFVf{gZdb)UO9)-x3y>KwDI5B z;P#Lt1{)EpHcL5=kXW(lYuEIszF{I&N`-)>GCBg4U*glj8s}7;l{h%vYyqGyQN|#L zzK6mE7+CMceh2VtCE5LG2oG&GbCOqQ6mX~Oy>M>i z6{Mog@#ohOGZLr0sMlVn3V#KMW@<>4!g3)cmrHjYu8A!j88#_N&qWQ6&sZMA0``84 z)9B*r4+pM-4%@>`Ca?^vg5;#rvZpDkCEX{VcQ3746uBJ1jENgsoT(?RUXry*w?TG(l_O%1 zO}U;2s*C;1@8^n%kL_I}glr;LuN`w3NID}YJ&i6C>;S;&Zzsq*9*l7?(^l&q2J+ZT zyhmG>ZO*4mvQTjs3|+9%&bZoYw9x}*>*;bopmb*|Kuhc@SDqO3@taEf(iW?{+UE*> zT&;3ov;27k<~+-X(R0!37P<%=w$xAWi$nsN>bsqWq{IJetNhPR`|l`fCz`AH#K7_L zU0DPz1f^V7Q<1#G=@y>22p+;vojEl}7gVF6ZMvjSzI5EKY=s6Q)@EAJ*g;w6^8FT0 z%D`D}9N-!!rHe212_6?f?z0K)R8nB^x?t|2&l_j>98E>dh;OtRlrE3OkLtH;r80X@) zFo0D+U=C6VuqKaV8}&T(w0Hq}54t>f4xHi@C6ERap<@5Zyd_`0dXNSD_3Kxr68Icr z5=Mp>GO?p~C~Yq`2u^;J>0<@Ya2lm;kMxntpB=y+3tREQH+Jw^jMzB$?X-DK(SjoY z)8UW|JJYA%)$j?gJ1+3SJH&XZmI`rV1Al-)Zm+|Gt^2$;QA#RTGMTi0wmYJpFvRD; z@GWwN0>q8&0FEAjKO0Hyi~9LuO7AXprTQW-X`Q#yNoGLAUco*oci{Y_MuUgT0d(n^ zqF-0gx))lGtB|G@Cm@>5w&Q7!;(hm2gL0FlPrnT?E&eHgFG3g3tTl5ITZ%ZL6*i^=%0V`9w%-YD1 z1?qT~77_ZMUToHTX%eOO)IWRuB)oj3grJu%l@J7a(2rk#0cqL0uM}>|ea&$tT@6?= z(f7p-p4SZ(qWkb{&}0qSNn!>hzs79FSc)k9?U2uod7f3g=tk+J8QZ4yG{LkESg!QL znc)Oh8^qZ|-=gQ?fW8o;hOP1k>7;maMn?ak(?qhF8Ls0*Fj88F?s|>=>R3Zq$EQ3Z zNp|ryiewcW(vCSsH7FKoekK>X4d(Tug-1aQ@mh2|^j#S)msIa$FM&1Hy%WdUl0VK; z>sq2x6$b1Xu|_||2whM7FCDgIY7Ndc`m#Rj;X>ap5EioOd1Xi_Lsu6MjmK}o%P-TV zmm@%kFXRccy64t*ORa882sOK&(uMM~=w4IEAy5o3YWd0R@k8rVB~BE-jqUK7ZDfj} zd>~N(BgU=-+LY3~s(KaU-#jU1%zD{wnAPsp7A1wxXvRs3rDTaMKf|-L5hV5&lr2=x zmcG*ju@L8eiH-ujI6b=ErCx+fDChpIlzU{DVG~0;2u>goag1XKd&8nX0XPgn0k8@j zjV3J%>I`fkuH+qIAZoG{* z-BEN)T^Xz>nIk3!5iDB6+O5)B(tn^=>$zSXEVA~ca|VcbL0|q103jthzz8zlDkf@o z7JaaT1^ak)f*{ycKATUD0O}M3@2H8efpgL)xfA|;hPj+^DbFg1AD`c6{?&{U805?o zRQ&j8o%vk?uWgzxc^0UW4LQ>SUbiLV<f_QfII+ImBpAEETVrAzuT0aKzIL4HZtc+tWUh5vN=w ziQttL!Q-y>cihsBP1sh3`d*Kd)Wz0f4hhyGQb9Nwf5^-la+bPOsp%7+?`&-!vDd!0 zm7`Ga_&3~LtJiUke}MCcl42POYg)nY7@=5rrF;Nm4o}(BK1u?Kl*J^Rp-7acFTJtsM zptIao?M(9rfwK+Jqp@sWH@9V1aDH3sE4*M#!Ajy(mhHfnk^+n33UYEzeE&?$;Rty$ zo5;6lA{-pAd(js@x3)p)8%no21$U_s;G`S@ioyg|gwzQ&TmUOITxiJ&Mu0-5qGbIX(p?d<)?bv?XH=aX||Ls^Kt^KQOU? zD`9_l$^&2;iCwG3`K3k1?^^>KUzv3@l0*o;6$ixLflxwwv~@I+L(O^$h!Zd$h~8tL#&TWkT>BZpJ%U zk>k^R*|tWtEn#Gw1@P{U_U!YjGQYV(Y0!tzZ5^a*4B{<=oHR*Ed_GcAXwq2nwFyxH zNH81deV#HllU1aOaTxw7#Gx$QU#Dy5FuF+$sG&9^Xllv$rdtZZ7KlpVUvs@&OL*er ze9v-k7G5T(L6Uf0vsX|$OVz$!jKBtPv-!-TcXXy~3BS^dXp7XfcjlCKykwYA(-1c7 z1AcAkt9HZ&7&q44x=+y2?NeQY@;#I@JuTAxDHWx4mGOs*r@(VVGHX3L<_}~543jtk zD1*ZvH+fP}#c}z$!baEa1{-ilXBXL%Cz^H!@{Ly3fn5-M;h7KGtX^DXY|Ozo?+~bQ z=07t2Cd~s>DLB6xU~ou7saPfT!l}=xA>=gSyx|Q-H_cQzf?tU7K&AOL<4`Pl++0vH z%cpNhF%_w90R|H05jn=*(RPiEl@GHOqiJSOo$k=9_hAE)j|Y}#Ix>o&zyXom!^kqs z6Es>*#Xe=Yr;({0m!pZ!LECLntd!4qF0B62uySwm>7l8t&VJ`Hk?U`tA#Y)}lQ4)1 z5Z?eRe5B2dCmJfb@d*a7?ni7+qiVOkQ$^%v*ZT;5$4=W^tVW&(2ltCM!1fq(xiu$+ zhJ=fdh+RjW4_jbh*&T|fWf)LK8>u%sDqn%{PL61+YOo76?&Sih;m8eI@x_GSQRP@8 zrrvmI1@!j?7QsEcMo59?7Qs3MEvVd;4DW)KQgXR~zilS0IeWmDaK(-(^0%F;EvI#F-75$MbfXT7MhzosoL^v+7;#bNKgUY=DRLMPAbS*f6-v^0{cs?Yi3A^R9<{k8&#jDc-|f?u%uV-kgM!nd0BU3;qfG4CKC*afCw*p zSW{GOTi^$*T$o6uTO^y3 zX9%Uv@P+rd7_wwFge6uGHHkwuKo=v42?MkobCoED~?mHZHs+F zR>58a%_~>2Lz^&fc12bAAhMHD4JD=M4jF8YnX^>9H%B{nc;;*(8Auz%7&YJT9h}d0 zG=@Y^dvaVPI+{Qtwn{*s5t~9)cBL?G=PU%;uWDYRBF!8?nGir|_Q4C%i$;C1S9NkZ z?qUbkkT9x6)r7qX#w3L=YX`RFaSM}TkraVGQTJl>rX^!0&~x<-cL;fp^;`BT(9pJ} z4Qkmtcd^Jbax_+{L%PTT2X^BaLY*S_@+NE!N(CRDdyURv0noIg{=Al_lgjbiQH#0( z&W8pxc+%h)$idLL}%XSn5C$z8(A28056QeSUyY{sh!+_L8^}MN2H9MPPrU@_` z`&PSz1#tp6?o_m>S}URvk5;NYyhxR74+VzZ_A`K{sog-MYf+QW0WX9`d=LT!dafNk z2+i1adk~Ujbzi`@_pJlfMJX7z0Ml8KHHh7*$Tf;Xgqk*dbEjlt?PS;^5hgx01)lW^7W@T7v@P-nykw;@g{K zg(qW(L|x=B+9~69!cG&*M7MEKOyqKl5|1JCV3iD5o>m?h*Y!)2*L!PEhmkilh4~0A zWSF43wa6+3Z8qCg?+4#(I5ee4xJl=PXT?$?yigVA3(2U|!|9=ssB1lULs}+O_$N!4 z=w~;08T}EPmr2dt0Cjdg1W=Yc$Pkn2nYh;{FqrlVlvfK@YL7OlFS+ENr`0jxP9HXq z!F;vJ{taqEtEP17vi1(rkn6AeJURfEIEacdxI zHK}qGz0s1CwSfY6NqVw|&I<`Yk@4zxHx9Rryn7x%!b@=jE`3v_@=}Te0McE9m(6zp z+*j2vAU_|@LUf){dG8f7&3dm?AAus8WdaDhR}aDX(t?>CYvP2KO?lYSFqm<9ij_g~BZe=h(3PWbdcKE8K*!x|dkpth9ritO5F0LSFJIHh@@F#_&bP7?RG z_{JQ~F|?HudTbsXYp25F@X9AuG&i}|@dTyLvpk72xO6!}J(Mug4qgKR05mf@t89L_ zYWFL(cd|wBP^yyB=nI<)PGa_&BR)lbA&}c4&5m-Lh1Xtzs8*7zV0BelGyZ7PlMzxX zyVjT<^*S(U)jFd$nm`>XM08&uJ(S!#wSw-5odd^=U8P(nhD!9=MWXgw$EN--!e3mZ zVSWkvj=uQg-{qX9_EBu6pCgqHA5#_Y6U;e@uUjB((|W?Q`GM9z7d@x@MBl&ypEfIv z(^d#knI526V#--uDvVJ}V8FnRB7k#~zRgwi^(}SpN0$PP{i!-RJ$>E+A^TG0*eW@$ zNPh>^P?l4hD}o_l@}F|CH*=z^br4YZpD&x9TV)H?;@qya_9sZUEXo zL-O6F6qS>F@ADLti8`qmKOs>N3^WR$l*p5Vl*>u&WY0~k)ba}Ae(w`i03#MkcS-N@bcpVEE|h`!MHIC>(qs>- zS?cZn&2BP?4V#1kj6~OiyJAbdNRhHzy5(3(TcwtD5>Bm|0yD|& zPULdEd3bNL>$#hgNj(e=@r%}MgqFwogn#w%D{l0Dr5}~z#(wsvT$CHc2~g*)bO^xk zk);W!3*m(LK!yk?Hrvj@(kwqV=vIdbb>~1si@alhN_*Vg;G4!iVNs(73(!(36^e?z zEiQaFPndGsO;5_NU~!@m9#${}%MVivXxMT(f)7bWu0fK+vtf_;0-QzbK^z&X;_KeZ zSxkrnq3hE6rr`$7i~v4uzR#d8_b53yCd7{mpQfzqH0}e}_6xdecY=;qI?9cF+Koi< z=u3JjNPZuBlqJMs9PqIl+zuF|W{4741-e#uti&VatVV$T(S_L1q#!j<@A>z!*u_@G zaRdie68X_yX?XR3HZhM6mJ#^d(xz0Ft)DG@#P+VEZ9L$m@vWseEZU9RvR7@ms@^OZ zD^O1quqtjw5ageBy8$d@H1AEZ4`gXaJ&% z)ZrQdrTNktvqW)2Stc0v$f=30{95zqU_8I6F|&$&UwzV`%W@~69|l4TZQ}i6*Ndq! zRLi7Vx_|@NSao>;k7zoBU)9C|w;h5$a+K_mU-nWBWL(>eBSa_^j6jNT_^MER)y~N8 z!DBW90(DU%dh;=nSrGT2?g~6s01c{ikHv>KUGZ`yr*^XI4pp)-mC)<7kC)+eDN%b7 ztL$EbB3iS0r^S1?w*;8KZfWsY12)s1#ub!dci1 zEuPsYA0yx~wjlWQjX(bPK)g0ESvs)UY}MW;D74yrYis3A&;fi%zC1_8q@ue{yv8Fe zmkYb%jECcXwZ`3dnY|HL_bV96Y1GYEf51r9!L>MRI|mUx2^$K+8OZEUdAkUjTwH-GlZ8!{o1;^3f2kJ5W)B8`Qyvq-< zq*KVPrQpP8lysAqYpZ*btj~yjBSF?0?@GJQu*;DGk=t`Qz=m~OQMgEi;eD5Huip^W z=b1BtMTgbC>i}an-BeP2yR#3M*Adu6Uv8WlhaJ}a_UZA&fMbn^L!R?#&esGxWa!6} zlq>Zva&i@v$9Hyox*?w_`k5XeuJ)(=F>doybtcG{mYed01!B}^7J8v*w38_N<=v%3_gbqex{SDTZyF9r!0E1s8 zR$~k#MWjlEmDqmEHQT+QksG@@;BJvWnCaWOA_jKHZIzH=L32}3hF_>f3Apv4N&7k7 zP*cN@)n+ zoCw((-@e1vR23Qjn(16Q=pweYxoiotf~Cv}zVo^^#qck=Q2JrGKxE3}7F)u)qY<{7 zi_!X5A3ysOh@8$!0`&{>`uclZUuP)uu{-zZ70Y%~x}FGqqd~=X1jbDjBJ#tj1HKDU zUXJstJ%1A!*-G1hX|vLs64jgL1Y;nRkCoe9P2+Hcjp~+jyku&56YU_NA8fg`mWH;t z4JD)qI>y(qIX({aaY{D<$r9E?`7`V%2FhDYJwrBqv1UrW?Xz3XIA7$a#Ll3j7=H!& zL}EF{PODK^gS~`QX>_z&BR|kI-D*hdxQPxXdlIRAswz41el8G7fNi*=k_e7{tna8= zwE-b>Mml3M85b1L7Qq*_fg!D@06MRi`go_NLJsR3CF8Yqp7H|j zxW1^y4)Bp`hLlbrOm_0Zqy0=yN8k8De1R@c4}Hv+dTq=UE@+ZU6R<4m@lk?>K2KHy`8<2xeXLY1x>%^lu<`bdSQDH z93Gp}%w_yGUQ9D8Rm|;4e}&4b+}OdFCK&U!TBY?Qh2+UEF`V(p7#g|DZgx+hZUQmxOaA`p0IayR_ZqC zK-6FXTHC2qah)aPMukIe%or-fR&pdK4@Hlg1~IA!Z=n~z*qBpYLrU)6Zc3lBVukK2mz%;;b`ArzkrlP{Uppo)q+RfIe%9;&DE4g4e$=XWFPubV2 zk&5slgnN-cEi?Sx0zviJz^&x@QzUA*c(y6oeNu${@YyittIy4|R@KfArDrlOhIuPRp*mifg$Nim{HqR z%ez+PwmTiPVr^The!+S{906pXpX+u2ymbo~iThbsf)KIUQ9z7z5rGG&Zz*fYiZmok zKOh)rC1Z^}j802oihP4x%xO)UwnKSoXcg!Tt|vh525kT|}O)uG|iUD#Y;MJ!-ZuB+bZj>cCCdKmV(9WM+o+oT#M_8#i# z7H2oe3{c=;-g`jt#6qe0XJp?ox;7aTAdTv-K}5_P=R|eL`l_n zM5n8;DNAzold*A#njIxx=|UYGDO#WcF6vMuovx}=qwZ@#MR)w88Z}ZrZy7G?3#b7o zqAI0LXm+^eoNFrW#@_!~UmyD$+%o0Iw#MynhnXX~z4nu^91K^a6(w_x3kg7e z(A(#eL<}s5@mQCJelOkL5yMlXwYKU{1anU9WbFQjbiVDfD(poL5bLK+& z@a#Rn`E>ykm~sxB6yK$Sl~e5qV^D}TuP>_y^SKl&jUN2`ZfxDTUG;PW*MYXOg**0H z6;_}I6A*(vZ8U~Sc^U5Fh1mazBu&hY{d>I96}~QXmOcFRo??&Qvs5zynAwY zBxeZ@ENad~vPeWo?Lk-VIhy&G_g{W|89x2+{kx{N_C$!Gi&XYw;m^H2g3S*t@n3g# zy>nNvm2DU0TLY+KrwLwkCWT ze@nm>xHOe7Ur=fF&`MS%H7S2NhcEi`8!Vv5_W;2EfE-5Z9qkIBEO7)yQQ|msj1`WF zRCREh;N!&Ev5FP2%2umo5^Vl8T;JHODxJl;sQU?rVk>CK=$+?%m^)X>9QMn)bXA+4 z#D~7TH22F@l0Z6x3Eg~t`jhT^+L2OsVDa$jmZ-Ac86<2Cb3iS4%W`%WHcq8;B}mpF zM?Xl6kq6`68&VqtcUWxHT4zb#yh9c@S#Z8z?A0VsQ&ZxO>6x91@svW9j_}fHRNyO1 zW!Rf-p&fx?Rv{7fVVUk>Jt|Kn3iH!sff*e#0YQZrO1Zgf;l}URZ+G(w}-BZqst3m?D1dH4-^^eRC7-H?CFzgkY|q)kv& zxN5@O1rSmf@vb%HZOgu{7Y+(iN)46u4YPY78Jm_|z1W@$0hP%CTr-Yt3G6nTu<6WC zHc%Q1c-45ra%h%&8^fysGbAZL<;+P<^2TKk+83RuP53XijxW9+YZx4l3pVZYT)-D_f za;U)BKVbai;-rA&48uk0s|$$vQ=P)Wu`m)t(%+%v4gtAHcRnerR`Nk4F3Hu6(tJ+_ zhJe&avw0PL_QbUyc%B-m?6xg8?5EH?yg>f zlQNb_Bfy^i-*PAo`K6V|vXc5&!lW#kAG%bux=}Aru~8|`o{+hx6d{WNz+oYWKzLmN z>j;Tm4T@GrgPz}W710CqK#aYP!QecjU z6vzX}zmHg+I=Zp?EF@%xtz@#v^@w;+d;d{z@>?TFl`~9<;~v2Cg>OSuL%zXS4{_A zp1c)#nuFA1)Sugj)@)aeN_vF;ZiqCb)tl1L&iGO50y)3bTm`8_dz*$(TQ7rU+L+hX z*fkNqiK-Lz|3qG5;HoioVT-rKz#_-{R85D;E|()!c4t%C5v?gfPTNOwEpp$Q024XL z9GthS(ALKy!`$wOfsFnE-vU1V4ZlC@cD2hWd8#!E}%+;}5TlDG8I$ zm=Q!Oq*$!gZFs8ImGmbA<$)=&$Brn}}w)Ch~N0j+|SuS>-< z;12Rj^rMBHW~??`D-8K)2meEgRe0DD_z2+>n-KN2!Es)IC6`bEtLn4e`-@ z_eR5|qvDo3tS$Jyczr0(ITtZUI{CN2y=5^2klBX9kuAI4!6C^rMOUS=X*;>N5|b>Y z;|cdkPbmAmf+^98QNRR1OG`=XXL}Yjl&)a*fL@lp#;nsO#h?D?$9F&d!}}lP_iyX( z|M30`dHtn&6#q}?VElvRwSO=TgSUKYT5B;uJm&y)olpW8;zZH|R3ROOADgUqn{TDc z*_ndC5l9V#pp)IoKE7aKYH8un`AA-moRXr_=N4%r-RD(DLd|j@Tw?>Mqx`8?k>yfI$ZQg=uCd6^Wu1o0% zj$cWy;leCdJt>1e=Z}8#{d=Fj8Qy<%+h2T32FSsoyI7C1ShlBUZmeB!Kmk}J+H8oj z8z;O{h#=Z0f4lVdV#tyJEJ0{0D697QV z?ki=G{iNG_>&n%_nhsctTtYikZO4Z}3XDmerR?C8*l*$S&w*htuGm3mX!+(;Brl{9YIMWW zx%gCA@XKsEJ=lBLT?e>l!1qEWWT2QGcM(;SONt5z)c;#hUrKuxASGsT2X0E#sf);@ z4`R_C!TkmAA%2l{YMcW)asas~Qgmknv7jTfst2+n@`ysq;op|=A}EpS_!d}tOO^G^ zcN`wI6kB_dMv0vgrf&`=vtyjihVx6wyO>vf-)=}!sEK>lCd_p>0J|fM=U3D6rh^RtmhJiC*;z4=u6kgA;8HqRDF zlx8BEuTVD!P+O#2qm+<#Rk2L!xD`jvM1i`Y9vk!pU>=lBO5Jfyq0|p|CEeVleui9z zB?>s^dZE0mYmiV6JSYO`aB{1v0kx`CZJl$^4PEHCwnL|Y4h6%-wPQ@)p)@a@fMG@}z8 zCuc$2im85#6$YfVA&JTc(mKH{DO4F)@{SaZ(1f)|dPZX)X9dDjjU9hrdT3gD&(FJh ztI>T@UgTr)uOB}TA74Sl^k*Nx|MbK6pT7U#(~mxWjJEY(q-`}Llk2?g>vnwbR2ppt zfGpZ)LsUvF4^_x}!zPg#1*uOu-nY(;+LSg9wKTUdRE+F4^(o&Ll}=;#N@EGdcWHW- zun5gt%ME$nmtT)1OwwV#XX`=)gKtAQUA}LNL`&?@sVvcW_W%o&CsZS}=Z1TRu0R0D z=%(4DzKv-Iq_e-lEKg1AMh$oQo{}M@6;Z}UM)_3M6I5wYCkq#At+16e2kui6 z6ns46LAEZAg55=AEW;;}yWw^wrJm$B9&AqPF-XQjEpkPK`IGFrd>_^80qPg) zlwac2$q3IF!x=%o<9vT@kJv~?JCGiqpyRMMTv5H}SAo{&hQwL+;P{61+&*&7V)?TI z(y?tT6qs$NyVQ0bg*P?6Lv2(6=on)TU!J}{)3ZA0O$lN;U)@_Dv#a}$ zkGRvTVl5H|tJ%^#pIwR6`$Wtm_&B#s0&L7{6&j$?quQEm4daFfzcB8WnF zYVowk)x=e9rU^nO4zXVX&9Aea_Ygh5W)5f$3^)fYKHJbznwgJj4gAiWXf7oGXounV zfHdyPZ%x>_pKxg>JVom4xn*W(fZ>2sMWn-GD1y!=Hm>>30@T|f61^Fky$G{eT(;B{ z#w6c@3kY@F8Bc!j{wv~Ke+xy|ufoUAZ=b&V@xR`GCcpmn{deKh_ujwr$G_9bucXs# z;cU@HJ&H+1lX9zQNgGC~ykfLO;O(XZs}ZgOW-AUEXLKJ=cfea=)@F}<53bfi9`ZE9 z-*H<`p!bCEtgj?%ouk4rg}u8CvmL=9xrjoiD715d4x%6yjfp3{WB7kj&j1^oz~`ou z42I=e7^vDDD-L<@swS;KbQpE)17NLIBDbqNQVZ6hRvOJXL3Xq8^HDNK%Bt2Ff`7(h z=7%?WYP-3%8NR&#hQ9~C$oCU7&u`^v{kLwfNc=O2p}p>AmV^{horZlYHh-jg?VZcL z!Z@C-z%CWFp_zBcS6SaIH8LFQQm&HgNqr<7nvxg;)Rvn4baekdc%KmxHS6l1Mnb^Q z`JPuQ>t-T3^i4z8kvKpFfSrISH5lu5`BX!MHmv-KuaGd;#S|VZ6e)Qk)er`GVyZ}K zE`Tv+*W2;%f^&Gf$f%92WeU~XX>YKj4eM-3#->$|9QE^-v`H(NOefBAjWj0>TikB< zvUAz*9@qp#94hv!ZoA=9(<7i{bh&}ib)PY0rR)VR5H~+r_B`H-e!mL0(Z3@qq=)w& z00Frea{T_w|9O?_3Ln3`TDCxt>yQ5(zkfUMWB>T^J-Yn?X-C0N#eCNdMqTpSUY_S) z(JU--AU9#sz8%@rdbdjeOa=9cMtZX0vl6yAq6p2+|Kx#!bfC+k=yxs{; zUE{-j3gjpvws?OAwInW$Q72c^Yj8*~rZ6-cofK;YXJ9*sC6M?oQCyh>Tb7e9+~K&7 zZ3jG@ZF*X{5(B%s6@Ds*W2_VbV>*D4=WE&lLxGkA4GaiebVcQ(Y_9{1LnJaA7RB<6 z-tXaV?cZI&NTz|YP|H=W)Q-TlwsW|krlRuIvAvNe$Wqb}8naxMkRn-WRxfk)r#LmZ z3qV3@U){(d^IRRVNVZweIQfR`4UCv}TMr5YZqmR@8)*8GC#GMWHpo;&Kqenbi3k$c zM|+KzQa+UAPp;FQ`&P8;2VDuJ>*QutQMx=PYQts6M5kou;xG}{ZPF>E#)f*yHy(DP zUGl-kthFhrvcC30+W>=iOKl78*3?$abdma`kM^^oWN172!%z8?u$FJ# z0~#=QcSe*BQ?a^~^*Y&$>ojA@W39NtYUSqEIhIq)fPotFv4}2-zSuYrG?bB_f1QTG zyOS`L%r>fo?TZ^63Czh|#Nd%G5wDMX-KY;HwkHO|4O@P4<&i3huV3IBdZ#N*QpexI zo_G`{t~oB<^izRNx6G9cL@s5(0zmW-Ds^4!S1^fF_Gn*-xJ#q-%ZLK3>QuIXNK83A zM~~p4z&2cREB(haHcE91{N8sA%N=uFdneSL`+H_Ao;UP8Ji`zxaEkLMFv(vS;s+a?uO7-%- zEg`9Gi$3Wqm`6ZSk;uHXLqAc)e9N%q{4irD-x6L6Y=YryA;3PXa4ZzDtb0u>^dvuX zg6N{#`YQ0&!L>zy8lvF_5X}-{&0vqW0$VwL$$ zz~&PoyZ$fotS{ESpe{gB_E=v`;MDjxQp6dsT-R?4FMCGpTdM{JYO732;!$G1DT`xE z*!x0{JqvXu$FIktgs{d~VL(@%vBW(}era@CRa{Pe+jS;*Bb})gP!IVQU+iE3D%Ngo z?WWlqfJUuS0JeQ$C$~j)SKu3nn=`k-sMRg`ly-iRD{e;*;71bqFeN5H&-IC)m$#U1U@tD;e7z5@X#@)+D+g=XP*UO+v|pFl z#tGeT_fY3g2&d%P0j$-hZ~XBa!2qI_3J_Uoh{4F?CPl>tHPbo`1dbx4V-rPVdXAppESna}Y_AM`u=SA?V@)QMJW}H8Qy8*d zU&F~-vQbTo@eE;6*0SC+?#ncEu^M}eciVr-!#)~3=MJyUq!w>%q7XxG)#Y~(;sCn7B#i?!=;tnIk<%sQL5Ka>AVeO-~nhGmer_Aj=AeW)uS zuqRSjvR@xiEIfmfn|RWwiP4|r0Vg})=s{&2M&4-H_kvh=)%WUd>HuQaI)-KfxZ%YL z5a2k0ew%`qs5(S(wYCcSSk&+;KvRPQEY|(f^wSzoTJh?@4ja8pj!g*{o`hroS9Tkv z<4tcj9*n@2DNq&Oc#GC3H%7jT!~8fl$se(M!9@jCT&wf_7Tia z(BxqUZL)nya0GnJP(XkvtJk||N9T&{x`DBmB@i3l{vO)Zs^*(ibr+z$WKu-=dE)wa z*NO-;aPW3EEiti%HI1!9dP~I2m6kYJJvzT0029Ly$UP4NUupDE@wL$O3aIXP^DH~J zw-UDE3mRo|@aHw{V0KTVgRa|$S`fJBGE`#nOY9=$+}u=Kg{zE7ZmX@$$4d7?D{t!n zl_xy#(07z`0U9t(^Ds$q5k{Q}84?9vk%hdv1tZbCGShH3Fx=l5C?LaOtcRz9r`T6@#17GCG@=b~8pT7CW zZ`|H6?s>>TB(LZh?k4I4%fq%K-X-Ypz@1iL-_|e#BA|Tk{JKKpMQ0r^x}CeA;Fowu zKbkD7&bx}K`nkcJ4n(PZOa||4M>vmEbX~S}EuN64FYjQovSb$x@NU$VRsNB;p3p7! zT4BPPi1-*7$3rtp8h`?>TLExnxtZB!H zwy#ud)VdSFuD&Z6+pELdxuUzra$Du>O;XY)YylWntNr4rs077?lY;f=rXx~IUWz>m znnzjVtt%G$v<8PzeZG#-Lu46G>E;50f!LMCEmxoGrP*DgGWRY9?7#Z>p8J8z9msw8 z0eh4Y6L|E`7vR+syS68>aS=bC`GKqAB}ud@Oq2!nN1eEq3><1b$dd!YhEsKxbVpK1 zvbu5%xlSq5O5@eDRV7R_TN%=MrIhF04 z!yB>*#yax1PhBVwq2Y-!@GvQ$@a>L{6d#IcpD$z_Y$vZRA= zGVy|F=>V{!#T7N{BI4Zz!oh@yJ5;G8AApxwHhYIjeC9^9L^~*zl~dPBrSD2-rHwU+ ztD%}ChGRPFpcFVpn&z3Rs)z9R2?h;2(DWMol^@Igh1koT+aslO?*{w>80?5Qlh00<@=jmX9JM{9Wljo;O zr4fR30KZEMhPXw=n%$j#zS!mIUv-Xo`Woo0<)#93datrCGsYAS6_gE@Wk&0EXhJ!Fe)ZrfOQh+F@G2n!2;|Rs>@& z_10%3cHPTFe_e4ZdzqgU1@^{r%#LgjwQw*hv5INT}w@1$8N{xdq+Mci@oS&ENhUM*5nT$PY-k zPV7WVfn)$SJyJ-x>XQ~tV|BZ@HxU98DzG0phdB z`1(opv0XO5X`Nu-gjY+j3vI<}wK`|K44s!*^1aXp*G(|{_z14Hq@kb2nJI9=hdgWH zu)UR)RFo6}(5LBq28(^{vX+&-RfNi|fMoc_6I=P*p@1v`u{h#@J_|p$iy8ZW=p!Dz89XtL6R=l@QfB*hNciDquK|bqJlbnfK zx(uWmWqfeO4X`}bT6Gw%y5I)xf&fuKuD@^~3dh;)D_5y-8bz@fuMs%C;fMIxZ886y*zAW##&GvvK0>yosqlKye@3esDiq3x$t zq6l~ZmhUgdhP^7B1H;W?zfUS{fR08Ug3@|TMQteSA>DM+DnI*CGSJp)m`&q!)tbC} z(FJ~U%RIgkl4oCOsKiMw4fEZG_s zG}hL~)!^)G=geEen|zG>3rt9**04WHKD8)Aa?cv9c5I=)o<;^M0L;J0{5ZAh!{#lG ztUcxIY)IN=%q|%25RJW1#di>Oav(C0_R#mz*`x#ar$T+Ir0jwAMU&8_4up4%5}&LH zMG@z-odLx{epPTLpd76k>^jFD>d?Dup{2w{=*t42*tA@({tHySrH6|WU!@{9*&=o7 zMXy7b6o}jy9H~*oGb>hV*v6fc?CeB5z|~BenM(fNUU~q8bome+0G*J8tXKe{)T8dW zeXc%(?cG_8N_KJvA26qsufN~1V@`6wAxLgRlOh_aHH&EHmpaDd=2YEzfjvBTH#pgnF(d$vOuE>W~mz?cj1p+j-dm`8eoxyXp2ZcxqEKBk* zJ^_3UY*s!RUbh1-sFh}iCFu+2#EO8v0qZcGFi&I(c;2GBmSf&i&r#tFe0tTi+qObuReru6x?iHoCuH@`e25@yUv_j{Mty;nz$#u<21u4uy{z6+#Bpq$d zLb_yez=kjL4}o}=X6OdMCDu8k#Mc$b^TeF}W%&AeNPqe9TMEVgfsOpl(A6pm5Y_V>Fm@&wR(`e-IISXPEwB%TPrQj)YINJz?RDi zBrHK+i#zTt%EYiwIOCC(4Ak}9!>hr1 zLP`VkUE2C-fQV!qR(ZoYW1v3xh`mluu+QXbR?^Wta(DPP?5w*?G^X6a%e_ptU;P~4VwpB)^nae(zxw$19P66niF$=fZBOiq4P3rl z;AQfl9wZn_$_7abd93#t_jW-Cth4qYrIh(RQkY~NGK7_h7EwqeASDp zr4b=lP-~Oi81Dx2YaCjy#5Sep4i>3a2>* zWED$^LkrtyX}iAI+k|y0CNlIDN6AA$ms-xHlfbv2j2)@j%KenA0wqgQsJy7_z-eHW z?iexATtU5Spxooe@``k@n9<^!G?|t=P%{=De*7mo4o2hs-|%{EW%k+X3@`Lg|0euR zPJ~l{8R4>{SOT9`N+gW$XpRN0N__L~Tb*rOBW;DpU2#_JC7eziO_gPRKu25snCO0F zDS6BkG(7N>e*vEiJ00OoBhV^=r!@2t;IjFyFRJW=EeL?@PM913SI~*Y29qPTQY0@) z&fZ&Oxt-Z1v9GyXsxvE|u_LrEoWMyAF&t<=ty?wGQZs_64s}$tc+w0GTw+1{nU*Nc zE^%eHFI)7Wd2v_OXVX}hz$mm}YY%pan1-Y8fjvs6))?9Ej@TxYFZniqFMRys#_{h} zXnV)4hFQ+;yTW~18n;E=8#4CSOG?l#1ARZq>Nrnn)pQujKz(NF!=KcUA8qU zu`ffRU&^E8W8eWr%yHH$i&B;&F*{SGE)hOPQ-+eXG51pk@{WwYsOTqW74lD%Luv;M zKHRnRX*&1kk`uv1OWNT_Ff5z9$V+m;#X zRszj7e?ntNpTTvIfIH8m8~~9>@14>6Ae9v#?T18Oum$R&R&#c8dY;+gBH<6vw@x-Uo7%fS-EsO?a6oLgt|I7fjS#f(Q-MWMa zT`t?{P%8bJcc~br+5BmOZ2bx|tk2!f02;*z0Xvn8!gs2aYuXz2xRoMz4I~5aWyxnG&eo`IDMo{}zV6srn9Hp!wY_M$ z6%A674+q2_;z?gpHSu_npF4$jvX8_C*azY~Dy@8~uu?iv)rz_vuoF&~ZK(pm$h<^d z47BJA=1m7{T5#>Sr*sM>*DM@cSkxBvF_@ZYj}@V7sHZR08Mo!Hhs zT{;s-j?KI3fqt=bASh*ghQ}KNH~YQe2lp{tKA&MNsJ5q@q0>%8<-yepI={t$^&D@? zQv;oa@uXUUI0^ae-lj8BVV{PcFe>5g9q=SpltJ4z6;tT_Ty^puuGB19`^vhRPP$6- z9f!ac*=ZeA2(6lcyb^hm`071xQr*@N03N9Yf7h*orKNi~i7Ztc<_7)ItZkvITlSEO z1PgM@hO~B&k~WQ)K+1jW50j-R_ERhcO%YmNqYeNkkUw6lgtWbpZ$V-VcYaAtxVWKv zs!DZq5bBDtNPeSkzj+llw68lC;^)PA#PuqHuYJ>+Ml?tZRI-P|ZMV>U>S%P$rtu0% z?Kp7UL%{(vG!F~1=h8AO-ay)K`ddMGf;7n>{W7%c^otXP{7>7SRZ6 z!j^}Hlscbz@RH+|KMZ*lVpStIOd~Td6fse*tX|x|faANi!1yqarB~hcM=BbEC(==S z3sBA)fFrkzspCFwr!3+cx*i; zVEo-n!^x<-JKPr)PQR~A>fL3E4JMyO!xu$B zknmm7rLcu6sWY17vBaQS?e;9w(WYEc@YLLc)`+fXEO$0RU6}01qQ>QbsUSn)4zZ5K z6`bI(26R*l=N?QI^H34xU(cii$h2KIIoQM_VO_sa9KAx0ru?f!kp0s)-hW2o=T13q zdB)gKf_Smp%g@n@8MbDpyOM`~zSym`134JKS~?|J%qFRSgUu9N+gB z0KWUIeMC+#45DgU-=!JCId-V_9&ATBM6cdDYI}ImpJKBouA;=_#fPBS z^0GixF`#&+%=Dhz5g@wZ!!2!wstVW+0PgSD^}9qR3X776Y7LcNv%|4~CrpopMsdj< z<^@g6nKg?7GOvD4r9d`ZRYexuj=4t)V!B>p4Np6}hH?b4(XePW%_z8OU4axy#X5CA zMh|6(vZzn@$O3+JtAc(~!|F{=;~I%r?UQp3u`hNZ!jdD&29?W_hKVdi#G&Nk2~3Wc zbx3^2^rC~D={20113ZR<!Uiae&M> zbQM}(WN%^S%_`)X_M4@e3Wk1=irp?wsc+P5grA>Z9&IPH}s zu(NVNK`_?079;VMx0K8bgM`KbrryE2M`;Q;pavh0s-w`H%v$dj4is`_^FE0&J6SI{ zrczHE)+g;42pq6d2Mw$(p;(;5!Q_UeCAh$)LGCaUu;?$J=a65~HAzs+DEM^N)Rm+J zJH05K90Usex^snui4^+V)wliiWD|nCi3soCx;@nl%#m`>?0C#~2z1UPc_gWBfn8;7 z@JPG%~aszfLZ=hrGnJ3yDTSahV z23;aRIQL&7&Yc7e2vgs87n^x;1Y8G8x89YrupYsR*cz{1S&jVi-2@^04y(}i){Uvi z3#a^N?Zp_gJB=?&*zo~8A%Su{sx&#pSN)Oa*-;@N+F4v7-}m0f{jAEMtz=wM0OkTM zTpbfXVN15y#87Gr`TIn_l!)5{u`fDxKA;g?oVGf;1Pfi4nP7Hvl+TRe)J^jqyG7W) zIoAlMjxH<}=6?F}ufpH{RX$$d`S>yXqTVoqhNDO>Rm-?>qx_tt&@$f3GeMaiij*A( z1kzPG=a_5%ZQ8H?Y|47Er~;8Cj*XfV=BHq}k=XYvbwTyoC2h*o!&A`$qu0pu z%y=n(pK@l43#i*$+ypub0-40l6{b`=3b1!z&vGh&-Q;a1&RvQehzOtT$8pj}UZWkd z0FJ8MrI~CYDP5!k%AzYOeq%Pw{$H7_dKx zy$wd()8?KkrO~qO>Y!1d;$!O(y3=t zkCw*=5f9u3Xfdq@L7r9|w{r##)C+xQGUQExAT|%TjG7cM$8B3gk(2k(YA(Tn_dFjO zAE?_Vk6jP2ffMwW#j_Q*SA`c2ou}Q&+dB54S$QSj-sFluWt;uOn zZD}@J%??F}$@Y`}b@;D#*WQ2h@q^q7Utfp+-5&s4mb1WvVHJChcHDhbgQv$;av=B} zva=o&hHCB;B6e71zZRk6JSH12DYu=&jTu)xzbbiJCavEg+rIL>f@Bcz(AaF}1_D0G zt_8?+Yjg}OdmUiE`Qfj%K=>o8FF^WWrzQV z+UlUo_8Zu$n)<0GH4ug!bEd4XS8tskkgNWK z_uph7qTye4YQntmg!S1FJ5=8G4d{gRw4sxjRWs?ZT!ij6_Q|uvV9P=fi&Pi$G|cJ_ zYW)#zB?r>&tRq!oGb)w-{=(iBJQrBFxwI*&RqMReFeltOp`L+(x9&&24>Tj>ZS`TR z7)9jN-!N8{ue5vV4LSd6FXa7p3KlE#0E9jSla|$QPg+j5Lu}94sNh-khp8@ESj&{*JKziPu$uo1bjj-jjBe4eo^ju(2$wrUe`W&MFjbXm(4i+ij`7zkqjpV#Xjz^4ssfII}!48#5ND zKPY6$&hP~P?UwbCGTKbW&B=bHuSqWJuT@m`HEgwxITyswq#deWjn? z8FV>3azvSgKVb5Ba~h{A)yP(xBSYlbsL|l$w+(#BGYBh3BSoU|RNp0-IAH4@3jp#s zf59lHRU0$!zo4<@L{#8UNVoM}v4x?nC!jXfc zFQ&g9zVUWcO=@cpTLjQNb~4=a;hA=|g1Eb-S#>~)J(s9CNaYY@{qP3s2! zz-aN1-y&g*s^n0T>ffYsftlOx2d?)G2Na@I4UV{i$I>Y#MrsHEtDyW|Xr zPB^zn;<6IkDf^P6yCo)7W0Jd4=-_a@topG$%C7*6u&w{&bUK8s^mg1b09+BY)GUnZ zK}J_Bm9RN;F}7BJ!8CSZ7D-O~_=#H7Loii;%fDI@lS;ehg812vz?E^3=`%8QuPdpB ziS>_U2P!1Ubz3oTZ$X8P`zzz6WCANZR4oljTzxjF*1b)@E&vxd`eZ%UBUzY}3Zp@z zaP2S7+-qB%Z2~;}d_f_%oCd#uL3$itqVu=_ns*Ig%$%@gzOV`A3CmgXN@h3jf-*-k z9@0uI3O>+ zVpI+`JT%g+9YQCWNBKz>3tQdu23-r-iI;D0MR<*GTOvfDg>B2Nmi{{$;s72DEyUx) z%FCM|onr}EO8o6Ki7R3Ecfq#Qjd^AMSM#sJ~Bf~tohY&^{a zrs5^(@^17&>J+=Y*1uuKv_`meh29D-sa8A6&A?Mkx3WE~NURHisF_p?B@)<8MZ;Rg zx1|82lQ9aioK|t=lIlcBqIt~VM%L00UB_eXe+gL8q%nt5F_ZFbAP!6I_vB4n#60}m zsYnm~l;5k|)-H8Nl)LgJGcK_t zdjdPC`*3?e5*`nU2@zp>*wQ8P)L+BYq4#x}I<=7kw;nyA(yI4ddzl843VDcMElXl3 zr`XASg}z{To71LrW)`uE%Dgm(?QME(!Ioxv1cP|7v8I^<<4rtAG~_ChYcg*KcR##8XSRmgljH+QXS)Vn|1s25zq0pg&qwkIC?FC)zfPKa{Q)e|k*967;V2!DZRXk?XxkDkdyt9xmjJ ziXF&)Kf;9sG%0@ld`OJm2nDZwrvB+WAHRV6&UfB_XF#Gi1fO`1CiHb=ZR6)_cq(^< zTNXuf18Mq=2~qiFzKh2WG@Bn6><%g?54dP5xYjI4|;Nha7rr^r+&Ij z*w*SQ@07@Md4x?WM;#(q(UZa*m;qEKfFR11lAF(p9QV(sE?|@485kQcM{4%t zjGg+v%_L9k>jf7&CPn8NmLtB&lXY-GJGOGQdz&T8rT8BQen+fd5F?) zv2fQTlx|g}?t7p68_uYiSHQ5l^L8Lt(PVMzW4Kf0J1S$9E*FC_9w7O9zdI22lUItV)Y=ioCX+*1>MQLw)tFJOo0~I94`1pIkfTM(`}C+-c3c)$`cG@}X#={lsKX9c zIXVm2_;024wZ#ddagS}u(?*8HIxVN&)PIy~_~pkR_$RLP#XtEy&lsgKus*4P4b6wB zt)QDvm5)bI?XG;N6#tQf%rmH)34#)6$QF69+gOt4xb>;_d(DL5jhz)mluGsBz9ufA z4d@M&{1z-LfFlWLQf}%t$sDz4&vGg;?SRH`XOCLIdj3AsD(#B^<2h5Ee3D+Su_2gL zLILCQSmiN%K1=w9ZN)P{&Dgf{=$I@jVLP+Bs1pf^XkSGb2BY{diE8m~LpX zv_VSJG)y<(irLdZ65fLtAan!9kMRGdysKjPi{vb|N;Acag`!ehdd!VZB>zac*7$V# zgLX9ojuXA-AS&v&2tH6#4l-Z=^W3BV+LZsPM01g=`NfB;*T~-sA3sCeBT0VrLy#G* zuoj>Ii2Mi;s^#Dr+gs^j@z-*1KJYScctS>onLS{tEzxU^d%uP(y=u`2=Jo{(T@cjn z23f+~BcCpy;3|1$&$Wnz8692^=bpqLPB|k~eVBl&z5!#rOTmjH?(E0W*oFfj7s&>ONjn|7<0 z-%N}(xGZwMu=|q!v}z^{Q7!o?>CxR>0DPQDy_TAU11TLcX{8}I#YYxptD7=zLgZ=g z6y{n+Z3I_JrfK=0dxH$lumiaSuawpe6U|&_P{jbb^}&f!s~#I}7dS|S-F%P&d22Il zlHwBPZ>eE%m+odqycK=6aznOBTMY}5lG67xXu?K=s*TH zW8zp$P?D1eH-CwGX>-K`tZA#Ma- ziCaSe5A57wwxqLK?B}4wKUV5e3)SplTI>ibq)WSfHD&k{kzy&FA`-<;{fNZF3F59@ zX9N0v{L&mKe<&TeMj%C3|rumFsF~M5Heph zht&(u(&qAjc=V8l(_~WBj3j}spt&ss2*+~;2R;3bK~aTb;9ixgY2YQ8C&X$3$0SW= zlH(-pl=(@v;PK~|3BB(qnPOoFxFs7hQVs-=7YlgFh3d zP?L^=18$wB59-D+?kMBj4*9O0YF?*JVRpwcmB~1TZnF^`T@_yEa4IrOrE^H*LY9xT zePK>@XwbrkwE1@bQ z@4pV$+Kp-S_u)O0)?QgNKjRk!6#!6f=p#E$VLWbY2Q9S(K`E6?EjTIEiY&iY*;>hU z^`)klMn>gkb_V?=yCF$^9`P8?4ju(dz}@KQX~vY+Iw(n`?UgY&7jkfmwJcmoKyqKk ztSI2X$}GimnFmyc4m~Mz0yxV_hegh%w}*Q30X}sEPhf-2H5llmv^2OBMkQ8D=NSV= z4CmJWg+3!7&EJ5Ga}+sHe9lRg!*yJzk*X;HE08u(Lqf$(W@lw$%#`dYQR_o69TYbu zosrZxt!qh4u4WeZ2BUwNBNSN$6 z*c<>s#zX7{T13Fb5q+b?CazMX0s{0pzX9%kRGyWUf~`vnH>d6Z%9IezctN^Qn9kf5 zDJ@4PPfT+5^62B!@$pM|mhwwk)(nC3s^ldl|He1hag$0y4NqOcZz$!6J{aRlWm{4F zvJc#rJY7(=%W8QxJeMOqQ2jc*MB1lH;Y#f;`P&-+#_O#qEOj*D%|hMkj*ild)>1MV z4ivMxd_c7lly)OrSnO!Nq`D~g(`N`GT)ds73QsZd21gx9K!7GbsOK#=#f2J)DWANO zpO8|PKV!Hv<@ELnW{6=nn^(OH8&{_&n9U z9-*>%#;jfrM4`a7xB?P>ePRA>R4M_(SO9%p=nYHFJSkYT;_o$o6307*1|C;fZA%)& zAb+k5{|nK;?Nq=U-=wf3MN%;jmlfP}PJD*8?*g462AxdX={Z%NaldxZ)SA{XdkI1- z&L>xMD`UbO^Eh1{E}H}HsIR8Km3F_LjoN~{{F~fox34*nq+JUTs0Imww>wn%sf8n< zA|v{(ElxGTps7$$;zDE$sJ{2{!73YvDE2o1Y zOhD${jpApu&~g|1K~)dlEa@JGCz>GrojpbjiO$k|l(gF@CSvR9CER0)#$*d;;T?NU z#r;*u`16Bm*BEz84v3mj5+AChcx7i|07IVEz1KF<_?JB`8mj5yuO?9{=^|!4NB12E z9~W>)2{Ngh0+gr~3c9mP-L|Sg8q`$Sqn$q)!TgY*hiW42?WD9?K8O82lP z#@-iQr|NBR^cPQvNUW#~^0(nH|JEXxG>fHDS2y)Bw!`znn(Wa&$n<hmBg)fjj{s zNgvml0(ZV1!Uv~)^iwPW7z>!nO73_n(aD5_V7Fr)fl^~u>L>JUc`9HRE%3ktC zBQ`0nfcYw!2&(TjXiHLnY{b|>{~cI#dR0l6xciJ}fY^(@mP1ZzKA0BP4SQ$575??bnC~a z%cNck4tYAB2^=)2%&0Jk87?RFNchUoYkeOB^=NwZgIX92(=wVJM^BF`={GW}Bj25! z{XZOK4&OJG&EadzVw$!*W|~>GK>mSRcOlMPj`f;SRVD7t;xkyk`pUYH_ityLZDSLY+`9(zKikwal2_c>;r0Qri`;YWW`*4-dg{%xFlX=+Vh8lZ~4oFr=4N7Tlw z;)7b3w_Zt}w;sDsHQ{mmmBJsUMJ3L?LWVjXmm@Wxy~+;EZ&3jj6J=bUR9?JqhBM6^ zdZ}=3(8Pnxb^h?YBKpKK0byYes470&gIuX~n|oS9Ip@S=j3M1Gq6*WXLQJ^*k`sW{ zj1=7Okr|g0&^WEdL}e@If&~*ZP)@ELtkzDn93d-=F1C}g*IZ|d%l|91UP19|Vs8y(iN zPq*q&S{k*8jSejvlI3eB!6pnaeU;LLkppUI>=M2`s5MKzRQKuC>ks6YmOQ_tqeft+ zrlq=gEp9Uf6(Kt*j z)}#)VkT~5U;F@+S|2%wM|H~2Y%a1>V_g_lExag69l9CkL(K6H}^2uYi7nm1rtzRS+ zNwMrTUis5_^eRZj%)q;HV(SU=5sDN;A{!Ysda>nTi5!x2wjeEu0mYsgkF* z%n$7t?$+f>t_5^AQGyR7IcXde_-99+3UiF20mK}g7^ zVy?cggc#RCKVbQ3M_^LSUVa_6!XlUtm9QMEP60>VI}G40UOYBcCdDdEI2{BUZU7Rt z`<*Viie0g2EZJOSL)V_0*h^$Y^;4z`4eejUM+Su*a`nzJ%D%$P`s9^Kdp(*-I^-m_ zhvJ2XSgV@5E%1@!zYS=}%bzUd(VtEOJ5kJx$(YAJZiM8bl7roRYq3^%2Ii>r)2?Wa zJ`b}pz-v`f9gH`Zx*? zV@_csD-A@p6oSt^xa$~DH~R!}tn?o`qdP8p+?KZM;NL|7Wm5 zDAweV_C7RQP!ZQ;8tdAEYk*n0$*43vGD2pk0j1|{3@5iJ)H`DoJWCT#-nPNsX(e1~ zQ1>DFjK_V+YOf|I*GCsNg5!bQSSpmY4uz`W$4EpzR=%4KN_G{SJzc*OU89GMq9EsYX**mnBEcn`+qQQ`tU7l3wP8C2nBsB1g|?TCLL?`F18TFOY2I)X z85!_cx;SW|paoLi6v$3_fFpYaDOtgenij6Uj8HOs$ZUAG*h-s9XQApLp7KIR>_JZT zwR+c3YNee%hZ{i5CJwFEP)Ns}!_}}s0bsHr2E2w_RSgf}9H90~hD1FA=2@itJEQ}& zi3<0ny#z1G$XQ*bXC9rN5|iZ4+E*zKeyatL;uT8?sQ1z(7+SRpVdGL2NtdKGQDx$b z;p4aY2xXBY4#^+zqnN;d1*#}YhydBw15(lvi&~ns8DEa@Mh>5isWl zDqGjsG)HHDV2mM;*8Qw9p!a#kDQMcHgvGuDZYo}kZPj)P?c|fH;v{Lb|G%7ix0E(LwimJo< zAi9B*Z12knSGUWig{A_fBPkitvV>W0ciDDu~p3Td~( zYbl#{Ly5l94Bb-dk%ATZ&o*iMof>XTQvq}^0wDp28I@^ey?L-E46~0F=>B;&a+kq2?yPS5EP*w?mF*!;^?VR)BfYS19%4+_q z&6z*EC|z&VQhaq-{s=k5&8OD~A(qN2=EoD%!s|M+RNK`2iQ5dR;$H)ow5qt1UqR1h zl9Y4yRvq}(FY>9P4Zm7YnXS%8-kNARiG55(g9_EVl=gbG_6<^kjh)5{dI44}Qgam) z#1k@pahHXNanS;pS;7=+a3Jwh_Chsx|x%mWUp4=L(_AF>bgx|fs zhztJo{fEJ3Url>PY^l;p?AJIqcxXvRnQ?HE_&0GyX43y`akXY2RM)pvd}+Ffy-}J6 zhm#!L=(sG**~)1nVrxB}Ua(7*CI*8FIjI?FzWK;8FOlwcfR$~tc{B_!(le;fltC*2 zuqbw0Jp&7_mRW$ijj`eW9_k`14`CBu^pLe7?HFdL=9&#Ey9QceFLo3=^#HPIm^}OP zPo#ymYN{JJjbZZvP+wO%DPPy-Y(I` zZ+uCZQkf-hTy46LpFyIB@1HgqF)xZfIunCvpm;IpI zkp-m=o|8c2b1E2<>r=L(TW)vbZ!0*mw!%P-SANA96X z->hZ1OM+k)-IngXr!B`g`5r}09xYhN{QJFMhb>X4z^uE!JAKLc$9EC5z4IaUvJ(VOX9u`TF6D_MIScdOzSGDTs(E^ji ztF?)9{lJ}wpsM0;f+}7Aqot3(dh5hbC%#Q4i$P@f%3mDH@ zPQLE|xU>tYQ1|v#KI85>v-eg3zo<6sAE@V5Qa^zF9P28{>QGmi#p0|-?6)aO*|&qb zBx>OaTTV}NCE(^9Jj)NNcA4dLn&nAVds9&5n5{Iegit|{tT&Ziu_b#Zviu>gv$jTm z+t)KPtIC3#{A|;#n4O(Oc8$lDt@RF1AO)8mfS+M-S?T3Ty$0>;fv!kZq)a(P-O$am zsYa1cvjcG6B>A`7$w)KHvppp>KJ+-DHJtMuIds7K8lKAXYfph%-GxlaZl4RML z*t`FVO9~K%!jO8W0CN7vKp?IyZf`)!o3MF zBPEn-cg>jT<^=Jk9Z=<_*Q(KD@u4DW$hJI_KE9 znhLj*NPlqrq6-Yd@7##@d+7*pJ@7=@jT{ zT=Nk88wFL&)KMsy5yHxq7>uVdD<-LOMrjv1w&Ak^>8#Tl)#YAYGgi5@4>>h#nh_9P z4<@^I*Us`;7~hlvVDdbOgVY*j`d9H2RicRgQ0|m9<^5IAC{~|5JkLA~|!OHtudWIjpeew45^yj_|KAt0gtb?8vkM^f0*OrB;_a`Z{wq`4J5hzBNj zeSb(Xy3wo!NB5O6?L>=~`o#C6<;j7k%R08*IC-Y%Fg3um%?u~*;K*L?b=Y4_E2O^w za&t@I2bIFIb3l28|F(V?^gCYK&$m`vg*8uBhAV zX^vm1B#alNm4xe^3?2YR7%jDXUbBA4&Awq{v-&YEd&t~9Qau_UxCR{2tK)MRN}-6c^8bw>K}G= z93Nx+f@Yo2m41i({xsS$um^BH9bx&#Q4U*~-Hi!%Z^s+gB)x>n=XmL;>~NQ)BjuSb z!f$eh3Wi65`ayA*D`{`!sh<}3k0ef7dAZg#fX!tmwOy}tEj9r!v8SYB?9kN>t z_;PKU77Gn21#N7u&cfP~vUr}Q&9c)tDx zLu?I`bHN79@+c&Uc|7hVa+p=>tIGDd-!%rls<7TO-%9^wvjGX?;D})ppFW}P>=wA7 z7yo1sQ#Eu7@K#lDL5R#slgolZI`z8bTGm$#YqFZrf}i_jAqz!)=~;T89e|(|KO#YY z?_);RBbNI<(3C~Tv;YWc*LA2uKcp`JII`Azi$IM-UVy`XFrc-bgfgHZm^L@GC))PXh-9uUXxx2H~Fy$ySf)q+f!T1kkG+ZLrzzu%3AF6`9! z2$4WvsrQC>#)0Nzg0cecniG+SNt#%Xm)i+tm83qdG>U?D zL;K1p!!!q_-rCzp<-xVg!-O?HG0dU(5f+QvUXuEw+H&i};(MT$tHx?0Ck`(`MJp^ujjay( z_eimZ5-m~yO+d20TtG^OhTDUTHaUaxcy4H1OMXHUnOt;-j+9-e)#%Er2ti#tT0=N` zWt<$gasT@LpFa|edHZePi*%e8^(U2Yc4ABRh`x`&N`y$|6$m_-52gX9JUM1p4FRf} zG0OCZa_z1Q;%8oh-VD3Eqy42VFu8tVOSbVE&zv^FgnikYS@GJ zxTemm?68itX}4OSJlx{(rpiXHK$M=)CarnUs7X3@5Z|?49^S7&PcSL|qQKefPRWfG{mbo5brOf^S&})fxs88fnR^$ZNRCQ?sIV5uF zLO?5>j7%}rQBEWy9S0oHP&0*!vGUWss%#F z)!2YQnUy!w{~6@4SosP!GglRttke{80p*+N_r^z6a-}T^U9q(txxTfKa>Yiss!!vy zKmOzJ@6$+8(NjgE4?dV(ohXw^8|Code8u`E`mL(FozAF$3mI8Wo)L| z21*CFY=fJRqFaTlGV>4~69!sHc_3}<3s?DiR;dUMpeC0dK>fhMph+6*KBR7xN}kTd z1QuY)h*8UhKpyEm7lmT)h>llyM7saW9-vjW@t_dvY!M==t#gD23|nsWbaD@@5m2#$ z4U8#3Ql-0C+2L$AxHvh3qiRpeKt3>dRby=FvDX@+?DaT1j^WAx0s$|JcMo;DyvQZH z$D0&jBy}m7bZy#)0^?g~(#!WW*P!fl#MMI-uFC!+Qn46wf2lXnqPIPJ11>e{S`H!P za!zFlL*3B>kp0#tyZpTrB^1}yR`iBF;NJyLdV|pK+a4iUM`y(Wov9riq?9@>Ukx=N zrHC{d7FiTqVV9$;Vet*>BJ_MEgQ7Krh_(fh_PVVIAUBJ!4H?td9SyR9ldch>CLm5g zy;9m`>vJ)N@4U|)ocLcN3iJ+^*|K$|5`looRoO)z3QUu3=xnoLR`HQofx1~hBI5*S zD~b}1$&BLzR{r0O@d7KB^ z=O0spJ9cZA3yfji6yt#KdYEjfjk{WpBQNMSISXV{zh(mT6j}TcFI`pZfizgG@d19wDR?w39 zeL+n=whOVst@=@2`i~k0P{U4C%T8+n4&U%|52d`A+3t37p158J6X5`G{(W zN=Ac|b;KLiVxP&O(7jkf7W?J7dc?`g{bU%CW4WGX)O48t(umr=$gVhA-S8j_rt5xb!CU zg{q%$3+lqvB7>H3B((N$sG$nUsypcw{)I3yjl_V<^90FCwMQzV%D#!n+dTpekN3-mxfOdb-!;#6c_B#dX@jes{x~4>y6?gPjHdyBU@LZJp&*NWy{?$=F>IY2R9IBvG?V$E6^%$ZLp)*O!Ub+{aXbBzb6VN)M zDO75e!>SJ3NFbRwXhP!~&o0xqBaB==?rMBv<5=D5QmsgDjNR#Y4M)Yb&ca=0x~qF% zxFL73R}}$L#dI*FLIoULOc;tU>PAz@FE^~x<6{017#y{x7oV)AsY?g1p2-r`AA~>4 zC*_;>&(cTA-=044k%ZfaFW-L^-hO*|!F}+||4`*=1J}yElhZ(lX;V(pp(M%hW(}~ zWlTD!hXn4!QVLL|P`9e2v*pw#nd?H6zAanMu2dz%7zJuNZv8z}*6{c3on#0KUZpn4 zPR*t(Hy94A6GyUGNl&)mDtL8cFERSGornp@a(RUn+~x?GcDy^$d+)+4as92|I1Hf( zvNj=J-v+AA)a0w~4OmwLXUYrgkWP~tcup#K-yZ2fp)n5A!urZ(aOsY*+@@;6VQfn5?cC6&<&VjFks*VGr}YY+#z8BN8cfzT{B&7vYu9n=1D%(geI!2J=HB$-(qi0uMoL#qU*5;A3 z=+GJH5w)-t_W**VwuWmpm|M@(`#@>P0kmW|IRXJfS={b;jc8>&> z>LjK(;jYmv)Li$121o&N1Y{%4Jmcc3V$ZCYt5Wpf4A|01$J3+YDC7}yjE~0DHH(Fx z*x71|NCA4_KfH8E(?K9Vv1EwzMJ1#&-2HoR-wWwO`ZXQAHoV%F;1%*y&zfD~qkWa) z)1>E&-dDf0B|T$tpw7dc4SS+jQnTcI-6z++f=qPbl2R*rj0d&d7Jw_D147DQoxD#u zc+LX6D9E5nOs9<~brkgn!J=kO7C@&`M7M`b>G2Z&#qV}AZ4jD-U~zGt1k`b5N0&|6 zyD(dTeFaI=7@g8mgzE(Yi-@LHZ>l%U0699xoO*VIeV+}oIXHoE38_jyn6ekYL9;U{%v^s`Nu-wzfJ<- z@*#ajiuSi32K|uGkAOv#(OtftKzag*Y=>g%F_pHy`%$Z2LQp>Zj8ZJP!srf@*#68u z!U2Ifaa~E?mJXIUF16ilFY9ah3h{jm94)0obZ5md?<~tu_ms+Cap3kWMGk7e!F{9+ zg9fv^F~fbkx{PIvsL`9&v!xSWoMxZ|<%QCF^008!KRo17ka#B152GGV zu?Q$amtjDRz2a3$js=WC3l{Q{H?i*l_a{rDXl|1X0wsU)i60K+nJ^L{l(imJ(_~ut zKKWb`vp7QN1rH*1p^Ir(R#OpM)_&??&NQ#00W2)!bYn~%GepbEDLXhnKdCEAZS@6< ztS>IFSd-a5d3ZonB@L7?lUGk&>YCE{nZ8rCZlbeJ&=R%gLi4GBChksq7YtieB-E)K zYuDVk)wL@0Tg)1`Bnh!kW;?jUDkwU4X(Ra_RPkPd71wu1eg-PAU;uAD2+w@GpG zU~0+f;s!80J=9`*YP=9tfYeh>?S`4-ok9ZZ0=+wGw_sA8&B>#J#9QeurGJy!2D@Eu z*Ok~Qr82RYQAg);>E!TO{QuwVzrYs~Wx~C5EF4y0-9>H}oZ|@R#n53Uks;9la~RoF zD(VWsE?cWhGs-Cv2&|~;O24gg2Mkk>BThv{m0ZbghZB_8$&Vc4L@xFymwEB$o(2Gk z&Ow8Eg*(^FWNLPxV@P|byrr7|0K~pn&t{exyip6gU$-TgLoFsxG-bOUwn(qlVM>n5 z>+69Ejf5IBYT0*JiWSr<^4!6vZmZp`DQamz=Ufw9bS$Ree+Htq zeQybZ-s{gWtk_$Qt}Y^pDQQJgq0a7IucETtD-Ev4)9t<5nppL=CaD*n3bf;0?kENc zaEni$jl+(|pF3E2{^_@3Djr}1F=mq{Z)d*aT&3zKCFR6MxLrnj)Im8@a_=1?DRCWwjRaQ zlS-GE#!~Vzc|yc;ajd(9ZMmSAt4?#7ysnrOO{612?*zT!{Q|}Mczd9kSZSlNdbhU|0)txseMCi1u5YN}2roY|4R2u2}f?8xR%oo~{L8qyN#4+`l%` z{8n4@@ctu$b^iM8E6e7G#0vS3;&mv2@6$rE{T?R5zWAwa#d9eYaSd{)!;7{&&Ozm= z?jsxnWjd&?&ijaF{pkuKs}^M(6QTc8V4*dupIB3VmoOE_w_VmZ;cJDxAQ9a zV#qPb%&H4cU8q7>hTOY#D_!Uyi8NX?>|2tgwS;Z#GGJjl%{7br1Xk5CE(Y>)y?P>t zc9-kHNM})cgeTF1>q$U~Ft)ZgP*alF;01K4+IDgg${ylTcI4Zvm0f~iNCx(b@i!EZ zTAGSu4;bklqq`!kh{3bF{7>>xWSWeS+8QkISDB_dvP zZh2rnn-GgD+cumz(i9S#+qI7m3*{=a&aoMsmTd&PJwv;AY)<-Q_Cx?=Zj-K5(b^f`ftVm%3g+_HiBy7V z8IF*E?N-oAa8|UQJB0uod?MS}u*uOAs}4$;*VYITKrC1x<)!oejfjM$FvgJ<`t}zv z8$dK%q5}vyv`bIICwrdb!9?JstDOvf1Cv{a>7P!0EqisLNMGq5wv?!pLOZl-9Ysr{2JCVLK&A%_ z!zc;+*ujIIpz4a;dW8uRl@bp7?S>bcU-H_kX7z#)P4t%PqT5UaKecqW_XueaTgP8x zhUMT8(=v4f(z0#jP-BH^*_@OoKP<-W^+~Q4;;AEjXAyh({iCDN z@D02cw=-AO+)Vd(0EmnuTqxGt$QF`UuD1trw%uo4TW&S(IE8mgSbnI@5QnjeS~RGW z+;p|i(eoCXAcpV2C^y}%KMa5V=YO69*vYZX7M>rKq2Ki$lkoPN_fO^5Ux&9Z?MxXg zM~Yu_B^yz{Lm{-!Y~2~vNx8+Mbz}6+#f>&KwF^KIruVSyT$79iBoWj386}!Eb{x>b zV}MFr`Jm_0yPrt8R&M^D9^RL$RBIa;8t%0CK*^gPF)-PuSf&~8lS+~5wzdK< z(vk0}DfL#LlsQl`0+@<9Lxy3fS%@k@U)Jy>_ak{rP>M&+G-mjuXoIH^G;~{W6Klcp`sb{Zn`0HtFSAvTHa-C%NT_W|a2l2Ge2K^*Rekp*`R%G1+c^XE|oC0>|v z@TRypETJ3st7*_oer-PC~MY+qP8MJ<++ zQV>kQgybq)&BW-1s*<-AmL(4r!P>2dgBP_tiWlhtta!K`s&gzYQ_`6sEbtx{cSY58 z(X~J8@?kGf*Cz!!9T_wlVOzKj*HCO`bpWM@w(+{P(&Y%HAykyaMxrWZZnptrxuNXZ zqHsGx9Y}qtFLnue=xyY=i#VIWm`BJOFRqM+`8A{R^ERDU5y=5u%90HOd|LTnHUzQA z*7x_4n81cWA6{mFE$Sx?t&+C3)80CT*ma2Ot?)2g-kvq)E0+FYw}qo(?e_!(*g8x^ z-VIa2bxPW}?-0Uzf*k}}=;AY)xwWMf;>&V7$J1 z`aM#p6C#?@)^$3GXMxv|y>g#az`-^rAtOq!xbW$fiXo<#m^Dd_bWyAhcL;T|29)Pp zsRp#%z%YYmEUnMvluu?e1ngH~y8u8hbOr68EExi<8q#(`uJBi5^|DiOtnLe*VxKamB^F_k`T@9Wb(E+r;(o$aev;kdrpiXsZszULl%5JYp zERRGx!{*NJGtfb`2i8Mf#kfJc9f-6oBP{0r<9Qw;cTt{^2{R6}ySVmyTT!VAv%>RI z8JUwq7t?{U1c~&Iq$n?-Ooc{~EanJuj*k*`{*qN!TG0DC5Dbc?9ytfR*E*w7J3>N( zZ;ID3MHYat?N+1dFwY`XZ**E{Vz{Rx1lC|=x2SXv;+Sf=t2x+fYdrQIYUpmc67gR^9w0)uUYC;+D`2{<)T`M+~P}{ z^sMYpC_nG@>NYS5OYMU=LJmq{Q zRah!(srXUuF+}MkK~BzLacI$lQ`_!YzCvS3O2GnF&n?oJtz)LQ4v-gE>f9tC_3+I-BR?qJrEO#DFox);9BC?V{r- zt9LIaa%dk%66=$KMC-&7xjf2g-|jZ`fd891rE@g5wAZuFK=MB;VIP#luPq{mnPrWn z=5Ilqo^my0}v;A&&s39Ie6)$f?wp?R!^l`^e}oz z&jJvM_EaE`p;Jo5pFKDoNbALpX`6+g`#vAOE`{}rJ%hI$fJO>J$o)&81TZ03$=#@C zZ`E^yy$f0aDOPL@lYEw#rBu~E__QVh@NFkux9_vGRjJ4hf`y<#Zt+xy zG#CP8$DXM2B@SnDa#O_q9V%6raRZ1b=Y4}F2$~r3hFsqv5{w(*y)3=$CzfreIiLv4 z5lJoS$PEh!3CZuRqk!!A|tR zsago(DL>f>(|Z6`Y)|aI!U_VKYmUnW?@K!Rv?SaB*Y*v`RXPDV1`=s8{j(=QEimXX zxr5akpu<5Kj&%>NzP?JrsC3^<$WI*MfxO@mD5u%`6i77hjvs)8SWUyX2QocbcApU7 zC2{$h6YUJ;klImTFo+`V4}yu)a?)FxcQ^)0CV<6sByRcX2jZLZfi?*W0clp zu?}H43fR+P?4u44St4eb0I=p$vVfF&dP+*vQ@m&Is{>C4Le}A&!I?cpQ=l4f%a+p8-`2ELkpMChr`xo$m zc*2W?$yqUri|+1D(txT_o6c=`KJ5jyQu%QM4J>645vLd&6mPRCAYesTdGf0hS z)ATgW&IlY>zEu;+Gw7)>9$n+fb!tFhf$kc3a5IVX66@KV;CP)(un~-V_vQs72u3tk zIkbvtFr2+NSMrp-SX@4Q>fP*6Z>jsi490g7D;2waz!%B04MuAOU6_;v6Gn~ptA>K1 z{28L6pv+5-?GtW1s;pt37+b%xfI3U`>@Q^tlrbr4P0@A)Vud7aog!@Qi{=du=j=rwv`SRlDnSH8Mm5RA--Tl|OIonn6!nODT1Or} z+y#0FP{#BooV-{iGiCG^yjYm8ugS6^@gD?@Sy|c4=(~mCMyhKuaJ* zY>?-WnlR;|k8yNPu2B_0pTy8ReIJ0dfl6_aWOX&X1;^XPqdJtR;<-2F?}bL0D5e6XBsnqX^i&(GfkK#te76f;%&l7L zu!gK|x^4fL}}w;Ip@H!rPDkE%4*p&r)iAaZ&N-l&#yI@Fc`( zyRkOfMmhiKEX-gwWsK4~U9sxpj0qD8_h(jIR0D)b!WFT{niVUQUP58te#mj!wEMXX zrAjl@i)XObWm$>fg054v%L+dI2w5qBUSPpzVFP%1bNG3L$=8%n95h!pIWgAeN(y!o zd!x3@jZXTxz{R?mL}@7SfUlG-yrIuH!t2kv)tmf;(%t=tcYTbYcELfCc`#}Ev_!FD zgI-HCEEQ9kJ~?FKoP4cc2fG{xO&6XCjV}FKxx_04y<>77=IR~ZN)e+K2(U}k81P(s zb)O{YEzhXpd~vL5wFk{kslL$yRkhk`5pb}^bYDF-l&VBLjal|vF1yWVa( z<|+KEYFAPcuYh9XkfqL)sWC|Cq_<>6DXXP*Oife>$B|5=JRgF4z8e||3DL?4=kA;2 z>ZkADynS);0oDPSx!CQTpJotTD z$SISv(=hVj5upVj%vhlrWj9=RR_SFO_m_mgXB&AE?gV_=&V_2qVdT0p6IPkgh2vy* zUSlN=NbFB5NYv00Jer$NNK_ND4|0I~XU8X@o|m^9rQ=Xm0Dz>fCR+c&2Nd%nV}~G6 zO9r5BTbwx9iv56i6R{Zy{yjvsT2C*lRj%Re#aqPijqH*S|K;tZ|M0W-p9H>0gAfxb z|M2&3zXmS8LSD@ltFfF3!7 z7f9wxvaOd*cZU<`Dnq;JtQfkYL19t@18!&=;#d}63{)+yus*H#erc?L1zBy9fRtuj2A88?R$3aMmli_PjBC9kox_9 zVRrnTOVq&5B}}`)Y!W1Cwsva>{Xxq zs?O&zxR9M3%Q8@2c19#%xWFl=mC){wlhpN}?0ii)H!8L^%YUNX8yqdrEG?B}YXxah z7;&h5$qUMXbiBo-dj@;1uxv&*jnq zfUZoLTu65<(eSs>H5{<6ssW2;RkzmQR+|qROfe~JRAR}=2?GSB3|zEAQitoi6-Z*N z3Dh-}c@ILz5iORTw}ujJlggUf1WHQ6ZE`qK7hwNJ>Ld=-Gx}Bvs&7QCW8R`esA9rU zfn4=oKcIVsFQ zmyv%*D*B?oD3$l3Ix`rIF8l_f&ug)7!Yax*l79zr^S^!j(gKrSW(;tj&(L2~fDd0^ zxBf@kYDfuXP=;r%t!Z%p%_K#t{CYbQ+h$yoMkKaW@-_CJgd30;Tq+CU$s86q06WdU zSh(uLXmfDs+Dafh)idx4SD9(m5pkqgq-n1gHMv*XJPWww*>=rP8Hw4d5V)m*aJCiyK#iFc#}^Wa?^EmJjB>$PoE<6l8n7Ke zv!F((OngeFf@)>ER9@W`?*9iw;pS>^_>xn<=fYiTfMMpb)Y$gh_n-1(`0gi?OpS%a z{~Z4Ke@^EQQ!*1YRga0=Zj|x9b*J+e7^y#DDy(ZGn9$hwu&->|B~(^4rw5-xA_t`o zM>WoScEL+6X`Hh8+^XB1(>QsB<^V8Q_U<;F@pu63xi;5;=RGw2R4|spiN%3N5W%I> z`dSZi&kRQpUd_h={lPGVi~(CxSWBwoFr%XKfci=-EzMeHIKRm8pY@5-d2NcX$Qx?1 zw(tp1Vx2tzkG7VqK5+$)PuL?jlwoPXJhR2d9v#P+R)p*xXnU2N*iBP2P@O;=~qN-Q* zMWytIslSxNbR*{+ymi>Rr{gRM!nnJw8m2)y)DC*?;);ZBN=N(%dv%G=dQBeMP&WLK~$F3#m2oT z2Gw?)3&3Q}`K*mA(8_ub|7c836N}=@!Q) zTu%pD4#4S0Dr&%*Z6Mru5&z-GZ$A%jHm&%(_wNV!CB0J~d}GA!X@s~^;JteQ(zb(y zBl)DJT(2&SmH3I&swlZDT|ogBHyE8n3$qyk$I6p$;VEaGHASV@sO{3ju{avJqp~~CfkE}(vXj8lP89$LjvpOWyD}Y^T^=7e-f)Nk34nAK z_8x755Ap`7Oe}Frp5nssK84Ium4D;Ob;79qM?+lcIyi*1#VT=W@9mvM;OLfl)G2WI zFo@TZOw_VN`cg=>08(SaJ>bs>AD$#>4pZf`Jf8;wT*}Ld#I?}BHm$fE@QyBY&d|>e z9z~dJTj-Lmz96i)E^7k%qUY7=e z5Fezs)qeb0h20rv(6PI19{fWbvjeF=`H)#%1IoxsxhR{Pq{7^Pk z1qr1Ap_m~aa(4zC)bog~$9LJL>zpnJPxe6I4EXv>)^-Dl@T5`o3Ece%+SE>5+T@*r zLDIp|r4;RkR7gX{)T^}gxjAtq-Z)>z8$=4_%e~?}%Pp0E)hamY@I0zHk_Wa5);MD# zQQ5#FId1o*L&F~1;G4PYT8sO`L60=PXYxvSRmc;EA3#6O=qSxA< zBq>ZfIe|BduOwOMpnPo?_yn)BVo;7P_Vm2z&nw5-8^yoZ1M^4@Z;HEOtsGENU=eg> zz)*w;5Fc5^Lho*n0FvX!s+0oiyu_S;))HuDIdV_esEAj^EoX#W1rHLlTCGbt89how zEW=LLBNGfpoa zxo}@-4K-0tpTmokLu{F7M;c-t+JUXh5+HEJvVnIGPC0>f;PNbrGd{99ZJD&wCDdB7 zkob2SKa$^6J`|}Yhan9$^nd^TTXVGf@B;{S{gR2`H}7AlO2Yd;U!FkoNgd#W>j#K@ zc1ib);M!Lx^E^qQ*ml)gB71UaC1Bvz#!u`MF(Hs}d~$D(r@VmxNv#RuP6$|1<*{6# zDdJ|zjzH)k6tAw_rChwmIaF%GWKUY*q_kx$hoJVg=`HB`lKt(C$}LlVZrlJ!usI@% zA|rRqDo9Q_FsujwVNdUHaNVg`8l-79!@rgWMU`={)3q4H$Cq>D_PKzw)9=#X=$F_8UPbUmhf1R1cGkB#(Qz1kSB zYJA3pokaad3r9go>oy+3V{rksf?rU|dDlS16KOMzYm1t^jE>{Gg6)FBgqT4S1?}83 zOhRf2&E}TV-YPbpg3A;kZ9LE;1i8f)hGQpJvl1B7$!ldr3Q8434#R(P`TNvl9v|v|LpvC|OD(7)+ zZJgBY(%rne59l4Uzw{~$cfPP(3HNqqG~6caG?OSGVP9n^ZGO2ZoYpo_? zFw&VXgy(>=-I2op%>0M{`u@H2o&0h5lRwGv^4swCDex4nZ9`vG-FCB(xU1vA<$177 zLB0jLy~QfgT`C~04oSf&)Pv8xt6QtC))eGa?1cn3s(%B=pF%IF>jqAaGs=)1lSOR z83ByF!<|qDzL2fHbc%vlR9!~p3xM;pthK2Vz{vISAYIHFvzK6x{(uAN%YH(r+Cb+OQkE0$FHHaIMt4>}7$; zF|2ZkYL`@$`r=z%`t%>`CDko-kY$OJ7I)52$}&{QRaHwOK%{Cp;Jhczzj%`rj6Pt@ z_Hs95a+;K4P;4%J=@U3yh3(VeOJOs_QjH>lZUgMjPW;bqZy4N~O!Di#EOf8c6_^`w zsYqhOq6%c$a4eY{*g+h@@W^n=k=#U@wpvBhk@^n9hCv=WDdG*z1MD87!&O?n`qhqx zlvOrK+(YBqY0C4n8^3X;qO8H*pulo(uA4mHY6V02q4t1lQ5RZ{8IvR^xM_a38yCJV zp^zGqLmd>FeO^3{VW&zb1~KxNJIdee>i*aBO&3=Q4^_28ci0|%^l&htqdOt03#v{OVaK(Osa0kpp-Unw7VSGpZ07|NCl8L~;?1Rlr$vafMw-uAv^sawW9 zBpshs%TY%BVXU)Hth6q~Zfr9^+cXN;9))DpuRZXt&OXqc;y!!W4Sw1aA1#Ssy2f;k&Ak} zxTql?lS(K{yxA{F=qp*AwAFOc2rG|HCiM)*6s~dgE_`s)(g7Bogi@7P$wG(yj9sz{ z>D0K&O7kwx&}6n{?0GYc_D$MN>**v@(&?#Pu>6Ts8CG2bo1M{m+#g(sL7vo>ky?oH ztLsY6H&Us62ErYt(US}k-9TK*fDw8(EA7Emj<$TQ!*iuuQ&RF`im$Ds(aIl** zOJo+#pB5^8b-4l%dtwy?t_UeWCOGyQ8U1kkBZM~RU^@kdJPHpbzu43)@Pe4 zJIuf0ss0tPGv5pEKfwnKD834B|Db>Jb!u**Fe?^GvGxK`wBA=IuWcr1rN^N}LZu(n ztp?&pzUStm`T%$W-1xI2KW1$e!$}LDa79(GJ3D;Aoq%@$XZq;4q!rp76*Y(;0^sGB zLf^A3VqGq~y5vFm76UlgP;OrP!_GoA>Ll;EA49KGNV2lyqKoi4eD6!lE8lrDo;DcQ-vB(=e|a# zuH}A$(CBk`N#SkUI7X@VO3ovI*`*dMxF5}F!BpLo-PB5tpapaGmlt~nN;}sc+>lEV z&afSQ6@;XnU1uUnd9T@1nG%Z|;4FlsN+?S>SzQT3m<r40o*iQnjEpyin5+Tm^K{N#WOs)|j@K6_($7P|^`K&*LnmNG>c2HI8`&4lm z2$u3kMzF|Q!81jNr)*<*1LJJ~E7pzxFY6tC*hF z_==p2S^CeLDpc8w@TSH+auqqo_qM;UOY8(^c&u^DRqN>2IL_@EK|qOfN$GK(f%8N3 zVz8?rQH?1-E|Fw~L7Wm{c&Me&6Lya-RL!``09y zeNBVKpVMIRXP2iUu-Q#N<+4jTfEX6sD&vV)05v`89^@i6G-WkKICxTL#J7!5@f=iln_#`Yg6wJOb1{NO07a-vP{}eR^wz$31X04LZ;s{f zRn@8YxFU~Adw0EBNE~$iY>QJYet~SwGn{I%xi=~rOhxU4mq;64sljYFy?|MvcR679 zOc`OddEI9^oYE$`Q6YMnq3|OP;mS$}aqMWSJ>0FQ(X-=1kT0OhuZ)UMDBv zK(+t~;>3drkB6KzS9_enWPuETTVnx!E#1bk0K0O%TI^~pmtB`u{eNP~c%`!0cqkig zdkTXR`+fyxyA3lWh^?)<5UUcbjbsXt(}q*ba{jfvMK7^RFT8Kv!kX&k#hm;Yc(@R2jY&!Fv{UVLV;DH&G{COY$sdT%8v zy?S)nR-H&jE`t>Mxcd}V4nRny3Eif#{vav}5V}4Bx<(oqPD>;}e7Rv9;jrS&}fj({^aMoeU0E*tztcEC2yHpL_CSA`6tP=kX`U3c0V14H#m2iq*kV}pDC#VwyyZ_V><>tg017`uubpf zYPn)NaAQ)$ARTQdXc6e`u5k;o$V$0Dc0bqeSWv8!u7IT(^I7*=g_1a;2TZi=HP{Vk z{hE`-L@DPKiblHfbV(y6`N@rTm?ktrF~uOHwBqJ;39r@}LJ2<|fvC>a(>NBkK`a3~ zs)+x5sBt(@y*nZgFyIYWh8^ABQM*Q$JC2c4yiXe6rV~?H7%vaZwQqbq6s?NT0J_gXY;p$Nu3R^Ou8k4)|f^Wo5%_{EdL*Df$ zZ>KRG@?Y21@==GW^naVBxig^)l`500=PadhFMC(98>fe47jt&etwaypl^XU;#!RcS zF?A)yXJn@Un64GyNa8WO*2>u?Dgcgrc8Xw|QtTb!-)-faOrV_IVLmD7=Vf{&`PR;q zavbRlIYEHbe#lb69WK z5$3rS!%Ae3LieDpy_RRoWINX@T5hN%!>JVW7QS=+M~j(nq^7hn%B>3oiwtlvHJb zkxn%&oLqn`G6C=icR)vMqtiH~-o3e~E^M)a+hc?K1?pEIwkXPXIOY@gtMF!;BtL%p z-rKhye)Rrjc>f}W%gC{l6cvaxgi}81rgnWD+!;HB&^WX}_QWg*k`Z0)i%*z`O;1Rj z1D?Ltk^L2xT`d$?RZ0+Rx)nHs?^xifRik}*EORfIWPf7GS^lsbW$-L5;z@NT7i=J| zTi!!~#JhMk$-s=RS0%`WLt7;8bxfW-P7WFmBGpHA#N(Nhd$mkg`VjP4n+B>-<3Tft z7wJLVuVTNolGR1>*JlxbxgTIv)w6(%dEBhU^K-0r(;3S$ZB*I07J#xmGM*dIv!ja5vT#o3$r5!u3#}?BX{7 z2(se#4Q#$uRV>r-teaLra1bA}==X_SicS*pBUx8KdejhON!0w*JfsGJ?NE;d8Ld3( zeG}WCrmG!F{T43qbxN~WIUjN*?-Je}7!CxZZuGhFLR+8rt|yCiP+lUIY&s`qbCHSW z&6MouQ)t9A1l9Y}aA#P*C=fF{P66{R-$qW}r8=>Ky>EGX7<9*R!i(J$&0Hy?*Y{8O3rqaP-zsd2Pnkx|qgdPS4Sf#y%TOU9)_V$Fjl_J;H0)KA*1-`gY7b7Ir zko@}F_wU-|zI_?q^3?9_v$r1yzW6uP5eep%>QwT|Yp5TN|JG$edX4K+k1WhrETo)S zF$K_@u>i~N8yLvJ34&}u98NaJbrK*=Z%RzY33i`>Ly2>4TpjOmJEE{`@kOvV?0=!GpLT;S!)jiq!#iXg$?>rHGk=MrQy-Y6F6M@ zah!#Xfe@W*iI8(VnKRv{Zs(W@4;%R9j~YqcVy{wyP*c#;{vH$wb0R-P4Nd5-l1C8H z%%!Sbai@y7Q7gMBbBNSL0{vl_uJCxBvjpsJ@xAA^DKqGHh}u$I*hxS&ujmX?%6z3G6Mi{VhT)Rx}>NF`F5kgA}7 zHCM0L;a}PYLrqqp@;B#wyqk=JZUCNjqe||Tl%3WkNm+1(@N;L+!j^nPxJx-gZHNX) z+3pu9lp6WPf#|41l5@3j-r6zbMK1F$+6PI7Ph>BW%Z#V zB!BT2e-Zu{Ru{hw@89zEuZeazxvRea=5okEv6^Fo?1Nm|b^|@WX`Ie8IA4D#!M22+hMjuI;6Gbq zVruSze1JU!MBd*3Z9kc_Cr5%5*FXIA+pjRx$1wj6AIqp^K1G2loax(Pnmi*h^zQ*bM{<}JM$Okju*sxDqoZhKLCuzn@Y%qDtMa03NxAsbK zu_Gm6$))uXN^iM}c(Q*1%W}_}jth?_+UQ6l=skA<>Iyy6KNAu<$_TR>B2#%rolDzf z>I~=_Mw_}1x`eeVqV`E5jZ>quFGmCEoVbNO+Ie{bsOvmEPe48(4c%!^@$YL{mKJ}3 z^_mJ_{eaMAce6vQaseh5B1?|C9wkI`nou1Ic;(@(lV}!7>n1`6T0vOXrB>q}S`12! zVzXipr3ZV3zK5jbmo$>Lf2H29xZu{+Ng|!*&=MJhk>6>PSxtamwl-9$>aoD$6GR*f z-SD-YF#N4jD7I=`dYUS}%6&h%-ohnJ>ErfbD1(8xpH0WIHUg1e0E~Kz!OV$8QYj$3 zd&ROZYuw~L1k3o@`_G7#6h&@+`u>maU#DL~`oIYNE5u_I?{A;75c>PKf297&3-EvN z!cGD8**5tAB0$3mFa2l`Vq#2{ZZXhUR+Um}6z?m{*E`xXRaSwt)XoJb78|2O_YO;2 z(`g2b+rgn{tx`X4js~2easm7dRY3qC%_hQd6p=@zBTrzI=N$Kgmb5zc;OHWSW86I& zuHS4&J;av~Uze7>r*0%9nxIO-2u#koJpItpgrVBQsqtV(3@R%vWilK)6NpSVHg-&v zv2H-%T?L=aHz6((dvhF+3=EFIvewnfm^rnKNYF|zl75hf5v;(?cMr<41Zol*Rk(;a zV^C4I;EbQA-EKbh?d3AIBWP|?cS!=|AgypM(*o!>x^rN3RiJA;fExl}QUR%bWSAQs^yF*xGlrX!1S zD3mBlFMJ0r%$X1fn;)V}CCh=^Hf6q8ZHz(oB4sCD4Pa|<)G>s~{>2*e3=-z+0oa~3 zIDP{tklrfG3cJIdYi6l<;UUI@P^xJV<%|^(x6uEmG>aYSkRYSX1`-Yl=_P7j2lGPt z?j(O+;KQ|j;oL)YUo!3KY5?6ILgoJlVZ*Qbdk1D9YvKimbur>otIGI?V~St`a z&#t8_%f>~z72pzm#M8*1`1LiksnV5hiM|TZ!8Xw5uC{g* ztA&{9Y)Z&Ys@)TTZ%C!X-axjW{NV-Le&?1;oDV+A$O_r8RgGGY+FRtW1x%lhR_1ED zk+rjtymZE4?@IR^hvX2x{R!A$lWu6GH@A(YYUlah3@#X#I5cXaUGfmiEwuF&gzIKA ziu~|j!~5q@BzZo~g6oMX_5&hu=@deeRc_qWyVk-^5oZ-sSSq@6WvxRc;C7fOXls5_ zPH?l@9g)zFYP_r#dMfz@~-aGPL}Xex&26N zLe}Mg)J?8tuk`>q{Oh!54Cz&U%hOreq5oL5sP}6!dB$8eSE_PrCj`41Z{kN)T_M-! zh6O6iY0cJQ0NW#zK>JLZENV4rrh$FzNkfiMRkMRtK#x*iLTUXEjTDUD(6DW%boZ1> z#>$>&e7Gewel|NHSY}yV0ycnKf7P3?*RC`smy6i7BXun49hfYRNB}I-l4+ig%{*Et z>1L47Ik>>KG(D_XP|5R}PK-QsYaz!2U=9E`$m8lxEUW>#{wt8Xc&ciPG%iZf(Pj`h zF(OC1kV%5I^0i4%yN9y4v}&JpbN0Y(Y5@DTb_P&d4w}lmONfKw`r5(YV|L3&qnV)Q zK4(ZTl~b;18Dy_pbvR5k^-L~RWzKj}OocN^)l9t9>Cm;`Q4r=Opk&giJM}l{qBwh9 z6j%bagQh;KZWWP8vIw!{lSZA|WZH}T*cziK7b-{1lxlhYwNFFb+9e1K~y{!Re?C7ssSFQkR zQqB~3m=Ne!;j3p$TQ4+`8^aZl>)92&8E7kgU3|e&9VR11=+TBHzl|j1=I^LJC{3sG zP(FHxGHcsR5YD>W$Zx$tgE78Rn4oGQ@4Y;M~shFCG$4V zNb=)y)dPz0CQS}^G>2r*9p0pwzk#7QCL^E)1UfXJbLi18b995|49tSyqdyzisK|IJ zU*Jj)+54~Y} zMJv|Q8DIcGH87_^rwqGqH#w{8jiL_00j<+jY0E41k8kS%A84foP^@8Rm3(AL!@FBL+9R@o?pq;G29G_7zt& za}-EjKiP?UTy^-ISnBN-@_UKw^}zg=5$kVj(2TyjZLPt}L>n z9e`itW?HRw-D$LhQtLv*4-_u|mFc;~>z)yISn0rkmZI(hk@r!ucw0`hszWx}=MhA# zEve=qz8=uRhew`UIKl?fYnOEU8e1)>z)1F@P(Rv@1PYw?z2IPLM7h*96qVF*zJ-`a zwjx(8eb>%|)FzNOEcam?gT;idEc7!ks)l00MsE34%G-s3rNKboK)4XziJtwELYTju zCu=cQxJ%R=x~}nM0*#jWc9x5`LNPE2H}bmz2^xk_nV>$pkT}^M(7LFmhPNWQf~J=) zIhbVXfI)&uOk6K!73BMglzGF@R=#QIb=;aF25#x#5;*M$*5-0nIcN(&xi2u*uH%67 z&`U6s6Ef8H2sb!Tm+KCurqnI!yFjH3U`wU8#W?CEVG;pJ@Tyo6{6$?`c^WIY>bPtO z(hsMym?GBLzt;OTM-E+C)jl$X2+S4CXP%#Mi;qmNoL;=XM$chKMm5J9l8PO z^%W=v;5$k1t+F=FhcDlM4okJq-@kePk=lZQd8r@L??aaU@7}*i79oFU!0+c|52)i{ zp6yD$5zJTAg<-5PJjLu+H+H7}BsWF^EYyOk)9hBFXHvno_yWhvaA0J01anlh0JS6< zI8E4w=_7yLWlB=5N_cyK(=ba%Y&!N+iN*3vY|?~jN#i5^8t|+Z7(@%u=#JG|CX+rS z0Mfw)JWRE(g-+JMI021OV|vzJjpHKppcDs&y8CoXtmeC}j9G7WUT@X5=m0fCj@+$2 zMr3H4PGL1Qtr28mR938QZdu^7f!u;aYNEm?on20CvkK!wIX45y^gOPBViQ)Y26^^! z4IPU5vIDBz2gk*v7#a(FI-#Y9?NU-?OebjZgwFoO+t+kH`uZYg?{{$P`+m$U@3 z4RxN>Uy%YlN!qpF+{+A&-k~o*WHZ`p7q$j)xhW9tt~>5TfzqW%BZaIRSPWsDCI=c* z?5A58iIh7eY0J{n9lcbEj#32$+l5_LaI0u}9BK@G>J>~Q%dyQK+%0d*77tf7Ywb;} z7Q@l#h(^g*6W!b@ESZaz@@$n}Q9BC|+jfE^md&xuY}h7BSl)YBSRL&B>osR12?|>e zEQpJrpjOsJxU#QPu=wyfwBNtMsrWj)nG4GQ8Q#9S9Po+639;w!i4_e9_0PHpo7p5M zGDvH06fmiV$Z~#OU1$*Oxmqiovj~}F7ic#OT!Kid$n$8_?bE1|shl;!RcahrD8(6? zT5S8rKnAw_6Y1MsfQ+)G9hPkDt1$XwPt`VBpE$q`FjK6Jx3-sRx5R3xCPV^D;(^Lw z1Dv44la+A~XFGct8@EVMvyjp?Pn2O@WGf66=Qo*t`-EKB$vh2o4Rli3X=+#?+@4@^ zn$JZI6C5RxazBTTj%1j6NSi4`s$n>2L8J~gF0PJny)G*1u1gSk0Or%MQT~b!;&jPb zn@O?SB4K83!+{o>;yv2RbjkoZ$|tVGk+C-tpI&WscW@RImjRFoK5PhLIG*zK?@wQH zX`}V~zq%Zin+rc7U-?QtEuug_L4o3>Rv`*`s3Ad-RBNOPk_h2?Kro?NAOMlx;lZqQ zB>BQ|3}!H$JXS??@;1&e8UIO>gajm{ZX)18Afqp+%IHa?BxVQ@pq{Nid_pOXAtzHk zo~<03*fUHW@rF}Y(i>kf2NFurye<(Y(wL;$ zi5(YZtPoN^Ldqbnr9bHC62X4)%?hMMNcWU)6Pka_KnRz8O+C`Yv+$J!N+^&8?M;~M zUKH)rgXJy}zVx*kgo3aZBG}{+h=e8bfz7BN|)51ynhbQRF$Ple;U@#cDjyw#q2Bn za}twn(LJ5uMv~t;kFgHWlm`s_7Iv`@O!+ya#i+8pY`qDxNeC{i*g;T$CxDdw1d|n7 ziWq%(RUHpZeMVQj!fdl`;z0AsmRR{!?Ir7xfpCs}?82~@w;MTfJABDi2*_n(B-U%Z z+*;LF*MV@JlDNQH@0@V;*OW7TP%>dkOHwIMyMV0Ygg2^!$^pM&r`LJ>5M zF>q`R%&0l2WT-=s{cegjECe(ID3aiE%mrZUD~V6VHE`_qbm(1_uFP@TDe8o`FU%ss z%poj%%vAox&nBH3>c3WRcJw@|a;Xy7<nppRo$2=qUL)mMMtRM_sVO;W< z9<2R`GT^hc*OilWtg5lStM^*^Ll|?g8j)C>;%cYDE)!X|N~J}KrxFq-Ni1)5ZEd0} zZQ<-ZYjMcE1wCU*TzR|A(7Cs25J23MS;+$b4@9d?I~;js224QCou<0ruv@=_v(-(P za>-(8*!_ggiINM2^i}*|;5jU${b(=dS|hg+FreZX>YB}88pywGk^#et(4!w+r8r)k7 zTO`rRF=IWGZJy7`Yu=iL3!T^)J-VoOA^$)k_H8h z*?O$$x}d_SVMEeovxw)cQL`GHB1(g)ynr**iV3wkfazj$20m!2m>L84F(KAXo>Wa^X82Zta~Cxb}P*knk5nZ5r^xjUEJ zh1|gA#=0F^I6Q)`4Oh@(45nl*%M(A@Aws}$$pHF@y@o%i$+EPKr~3$~kc1*P%ZSz^@XbXiv4$Aci5T& zd-WN#&DKrP^;?v_0qFYVl4X4D)|Z}Yw!U6;`)FrvJn=5N*si+mXzMG-9aVj4u~|=J zl}w##M=dOhT)^WNWpl@()((TQ5am4RPY=1R`{i1zppEvGXRV&A#Mw*T+37E#D{GZ$ z*_0mv96&zlk6ezsJ~&hYBtniHY{2BC1Vd(`pbv;ya95=48RlZ`bdpN^)@`xB_x|PE zXaD`bhX0;Fq=ozYkM!5i_17PU4`06hlE6aL{=Yo4mVWji_Jy1pbZ-+MX7267tI4cu7b3_y<9eD-JJvur#;TSId4@|S-{Wtnnu6m963}$+i zih9UHsIZGv`+MoH#D4_v7AI_+sKbRi&WOqy_Bi`d0Sn(-beK78_FXJ+GohSHTA>%0 zPhez8hUo&rA^J^9Q7IYA{4G>~gu*hz_JDn3EeW`gy5!km3(e+2F22-yKt6X-^2(0@eQ! z2rGnfCJM$H$@;Zbu+YTKkJN`c%11J&4YMejW>us<2gk`^()v=U&p5dgZZW=)kpqWE z&i~;O)NHq=D*C7#TPr+@wgJf90Jx|X@q_W_!ccOQ6Lpi8{WaRgy)1XAfVk-SzY8D! zcdXh{WcxGy+}l^6Gao8}XzPp`C?qUI0YHn4WKhhDv{4KCc@D>}HsoQ6diaP;c{|4N^!B>^8)bbMcV@UGS7qxbbGMLne~g!i!Z?ASzk zuxaEwmTqOORTR3}4`xhY^P~Hw00CeXmpYKW)^1Hv^aM%S*u!RZ{Hj z{Us47fi63cQWCEr#mmg09ng}S+d|Hvuu~E|1Gn{NK%$zZNnZPNg8ISILeXlE--V@;mU3Y zX*u=V2R>HHLhmjS_F``73+Jqd`XR96gkgzfk*wt{M`ZdX{$2Q+bO`?U;eVC?a4<>+ z{A0?C|ML3VZ@#CS|JR?Z8i8%7)%pn=d+bjyypJq@G2LKvC-0R3$c9eQiCVT(KRfpK z5FK_nD+Xs`5}z?%?n>k$$%s&|kUqF|X14*B9fzimtq55m?n*QE_WD}CG@Y1b#iE{i#5zw4raB)bl*sxY{% zuZjrrmcE8BK^K)y8+~-N9LsgAms=~ zR5uRrM0GVp6jTrXB5_rZ0P5LoK@}yWv%W<6s#b(m62VY)W7VK3?04by_t5N;%I4Uq zgpHV`)+Ac+$-Q9qu|@-bpNQgxO|L>k^bM;j8mOu~Ru95t43w8&_cRvu1JumOws3uh zbWt<_hMcC?YDq#9WpvVr&Xr0Rnr$NHW+R3H(?Wp94k3h*BoL`<#w0JtXp&ZmLi0*Z z^F>h%@#&(Yu#EiZaSizM_KHZrvVCDJ^pb8bw+a57)`Hw{2}@?SEfddfDd&(2kUUx> zxgl;_6Xn!7mUAcNeUrDHsO)wKS3}!kV1Bzahf{yB;4DbkY;;9-Q7vanO|`bV()D|l zFF!|a5Q!J{MOo@gS#JrXM_%VF@h08%WvmDY!26^BOj}^Rg#GLfYy&v>h=VU3j8?MX{D}4p+m?RSj8^ zlNV6lE8ev%e1*n(mmXql?0u6)vJcszLwd;)xJ)}NAr1i%@RYy`{R$6 zJ=X5W2ELGzl_W#+*1dI*mc*sf%m=FN3@TiB84|TM4>3;QnL$}Mk=aC|c9xJ-xUf?$ z<=weEk=NV|o+jrSK_j%zXSXD`1AQD3Uj}4{(YV!S+ICwdH(eD9Y)>#bwd&xSW|!&l zv|Q^93$17QUh+!;hwR}0`YNANZeSk7>!vEC1b;}zUN@hVkjfOUQK@z_B;i!$#-ege zl=M7Ean1!2YS||OUU=Gh7R5OXY7GikGWU;)Cg7qm`ODX@ZM62HFI!SvFaQjXx*D^@ zHtd~j#nokiuT*|7FV4hOaJNekj#E~>0XoCAeqwgBS+R1h>bsq{A--IuElaPd8a;i( zmmrC&Q^GTFL5<3$S|n%+Gb@%pv*S+20hcG}8pX#&rx_^((!KwDQCi#O+IMoGps=y| zVBIu0!zlky)tzdPWLbbJxzNflDG%Y8tL%TKk%hp$5~{LGRAnB;oTOJH1QZB$O@Pqs z-PP+Px(1N)$up0-BBEu7@ia4oPPQtyhKS>s49%&9GH5eGqp4l;#auZA^5BD|2H41r zlrS;38P5i)gqhp`oQtkfDP7dvY1JNU%wp~BR21EYvFV^%kjbtB8X1&KT2(_ITdFO| zlTptcBxcMiDF?k`=Q+k!4!Esl|Gh@wQ?YAUrbBCltQajN+#q#l9HeJdWlOt9fNzvq zEQQjvZ(8*_r)ffc0;Fx$VBglu@cw(Re;9MOjvq|T3YQhOLC@ME*BP_z{ z1BP^Lbh_ub8W7)-(>fCbs8RxBH;|p70L!bSgi2jNoM4Foa2Ya>sj7?Oklvky6!|K< z4tJFV=uy8oBpx~IN$xQs+WLM%O~q~56dc7+YR}5gVhQ5p)~~$B8vmWTY7{&zc9o9!qcXN9jaP zLWR%N%DQ7^_=qD@+dkN3)dTezVkIVmwn1(QY$?UbuhE^3onNet>T5yXQc)qroofXhXH|sGzXgx=mEIK9n=AA zTy}%XAl^GUm+3+~%~G_tdlNG0Gr_~hT7?aYDXU-rJC?$$2^2*w!cMEYiIYO`Bks9E z>$7!l8KoZ?Yv~2z4!WiLBKDF9T+krgZT20d9+BIOPCn}G3exH5ihJO}NZki`B&=Y# zWi0`AsNK-6464hPPET+_j#X0E)?N8~C zqM3yjc(2E1C8xJMco(`t7(ZrDLo4P^z#O0txLv z*Th_-v^pZ;Pz^_o)%@ydy27#t_HmZ=kRErcL(0JifX^b?hHHTS=2{?9CKS`jf3#Zs z04@)d`(A*e_;R)gtSey1%W?}yf%LG(89>9Z0)mrm*SjMkQkV&-$k}Iu;xdUBX588P zpS=AkyncbYYTLuKX4c$fx5hfFvN8Vwc25|{xLJXksRxGuLzxug5qe|xGbEdrjRPgr z9eE}>=Q;|IASI*)7v-DPcYe^dB_|%}JuzUNMl~LH)g9FgvBwoxnh+VRCJy)e6W|_= zCZ+w0Cn)fA77h|^K}6p(d5j#MUH+s3D=zN>QO?{&>B1559*Os;F7^m z*2i3TtOMe1dSI|%3r3J; zSEQgm6VQkh*4aS%^AqUostlq)KV5X8)j2K=?$Wjm|L51=fd>2qWb7)C$I_c{WH=qn z#%_6WWP$a7%ar1g9vWnYi`8L9UbD*_v^}#lv_zF5C3Ew%1YZ#?^)UveKY%z;n z$sfT!)P7buhLzWxV)TH0U=Mp4?!;#1gZho-ly1Ww`8I3ZT-#QC-+w zt5m$G-Np{Au{5A4XAxpI3Onclsp;afDAtUkERz+6iTMW*2HkYfJrW!@K=Y(a-Icu% zOc8V_;I3F(aWLdMLGF34@A{g%+M9es&}zDa&sj6bu}Eb9G^jqgGR6<1S~hz0u{CJJ z-ec&$bSijB^TtNr2pdu0U|y8MGG1k<=( zqL)IS>NvgLIs^A??g(bStkU|_tf+!|=NSUy4!W#E^}{<$VVzWS{b@+MGyqYp+=HXi zpuS2k#i1^(2U7)s+X=D81kkG1FcMwO@w^`7>pGPsUkDGW@n}FK zEw?oYsQye)u7dtv?D-30SlB=w`3N&sJRnQB?Re@EId?-eQV6Y$O}tAn*L1eZEHGEk zF0nNrH-HBxjQ&Y;3=WzLRAKr`5ez!VNPh#Y0BNmNwJ5OE{i!ibk{fzyCIxsTGTS2z zw!N1dG!-QGILt0bv!UZVNdh-AFYR zxoK`br$uPsrbuCi?Nz%kb~-1QdY1pA>T~85Ae1f?n8`}8&!X&Lue4OwQk6zxhrpV# z#S<7eWClnphaACY=o<;4ZuYYRRH8ylEi zVrv=j))m+j$}x;y;6^jMzG8?fZN{Zy;zQBo8(~)%3R;ND^>3Q==1BQcLv)tb?$a~y z;$w7NtfaD;s3hf;+?;J%qb~j+7Y=>)Pe*;LqQvW_B`3lYl!42WDge_cY|}FA#E22SD>a^B!o|fiZ;3_!U|{MpvAb-{!_#m!}@9m-0&e;Hsfh930-9 z#tG9jLH@7(82($o0Dpon_?14-Ki99r>nE%}eiYt*d3m&%bN|2*xJh1)MM35dpg6V; z-BX0R{`xM5$K&PNeY^;nDKGW3wW^ zUq$K3_g}pJ98P;*y#4O=S3C!P8h9pr1mA@;rhb9fgB<}!Ut#n?_pwJ<_NJizVD_RX z#~Khhw51evmh!{-7gSwb7cuP7_$>`GflNmMK zdB+TZ{+eq!=n4DqeI4%bLmKj9i3o{xq%6y84MTlXSS zUt;O%r?54*Ql@*BSZyfv(p8z?t`3pV3+MnhVu2zDWObk$t3EJT(l`$jkY0q5Ey>+U zNxgBsfQrm@jgI2g(8Ed$_0l6iR6${#8qh%HP*tUwRsbw*cJ3AZ1bG6%?3o)zN^!SK za2ql9)1cbc+&VNH zM}#(!Tye@|XLpHVQmy!w3@yyhY5VCnc&&!`V+bQZF~v!m=tz`Np6d zs{yWWCM%peC2?K~ePrTB_5`)bhZD^);C8dP?ywnwe5^l*$$$xFRH(s+NCOcm=z#7; z5}szjvLV#0o;Dj=jc7#Dvcub|6@2=!?97La2zTajkea-q0s`n;`>Ik!k^UhEqq-!7>>EwVId9fQxA%D6sywi0ObIO`?cTy|4T)J*F_t^#oA3Gb3flL1 zxPj3qD`JVJ2ffMxG$NE%&Nr|>xesNr1yfd9MxghfobeQFCemV}aZ&8grNRD%tvU*X z>ZIz0dC`OM*C{}5T8AC8RTiQuTE{9=;bRg%!yUq$eD3mS6X!!Q1GVA)W6N zmxO6}mXx6OvX9ehtZ1@FXMswUA&Eil1=va8k8I|FEOD)SqgwoJ_&0x@L-7}HpE#Eu zYtgU5o5=-z@%9JEl78{}>Dv#}E1$pqBD{T?9{BHrK6yz99jSU6RiJ9`4 z3b{)BVIOq|u^_2N?j4rIOQ-JW)Z^LvQXw>~*yKQaa8-sIZo?W)9_fQ{Gg18&l2~dk zqtM?DH2q#qSGVZ}3?6c5z7hjUpyQSajAO$>L8Yk1a=$WUg*0Y%c*~U0V^Wmf&ptVN|*e6&a>mA6tca+RbzePUV+I4_kBBOudE~cSU!D@DV|s zx6q{DzXNPQWu2kotm6>sCts6_83H@OFH>AUZ<<+`OAYvwvt#-@e!@x74<@vwc&<^m zU#2TF$u*Miw{gl6>K)77OcVf$$R1qt+G22_a|HAz*#UI=Twf?*&C1;8+@C`1OpzfV z+;_l$yptq*g#q0CteW*{c>RU??HKwDHqc8;F6zF(J;PN7)fi4w=_UuR_UM1%t;ZUZV&}9<1Fw>*_N0)s8$TF|R*?F^ z47=LbZK3AO36L!3JvRDHZ+L*cPn>Jdy$H%oa~@Qv0~@tTH)@@n@0~OV(sCU5*=!5j z?kJ#-(_Dab|2XK_n9n#>l+0aWZ0eSdJB=%#7Oc)%OLoXA+IS~c9^_IQj*dVe0Xz?u z&W^GRq*eK;C3X712I^V%bsjAdk9T#gS!@N@M;{{4pcIBt#|o_5O|@^y@#;~ae*c1* z=%(8+j@^?&X3aDGohzr$O&ENk-|U^v&N)iJy2*ThQ#=DIW!D#2erMMQW6nXo;ifmP zKP`QIGU(!b8dZYM*IP}5YLdL!-f?0MD{pbC*#X#1zuXOT!+In`8jA({HYix_j#d3F z`Sr6C17-wvxv|;sg7A7J$4^~wCwI!wx4`0C${4Z+L2Q;wQlavv7-0uRGAzX{+sg|; zy}@VEoya#Ms+Cf9&Vc+R+_-}>CeLJkHF%=7+j^uE9x$Rf)%v0x95<^OAl+R#-wsp zf+hKE?3eXGB1e?U<=3?a0WU3FbFkZ$SYrc);>yY(#b2XsdQyli1sfnG7&wevYCsTo zFUVB&Yq-3_SH={p0byOip$}bA+G(qHG6upsRBW;YodWGyOuOsBp;*JzA4R1)vKG0B z35z1NjR`dmCP+AqzU-5>s(T@h52w3th;E9;DA_$-4Y!6ph%4$860K55Sp%aa*J~$f z#D=n{hKPB917joaF+5$2`jcKy)XEb6my~ryv#j${`IS~?w56HCVQCjvO{%rnG>eXT z?)CVA{M)Zyf55faC;8Q%zgk%-4=(S26kb1-lJT4Gg}0xj&w(ds@1CG9q-#MIb5Wvf%;kLisFA2G!N=Ld6JI{R>7d-c6zhL!auTd0$Ia z8A{J2E$kIGa6Eiw#j8!WY;dtxxdEI?d${e8EGWp7A_ra{V*)UtP)xZDbz5Zlnhz2GUzaQg|$dZnNoS2^tjW? zzMLu|2Z4F$h!TbEDz=m~2nAk4YSITC&TwpY$&E{K2qC(tL>=1mIc>z@c+f5&PNCxt z7!DI{j^Lku^Y{E1zWsvbxCN?Ope;VnNJ({d*)Z^=JhQ~yx+j>`9EmZ3a7$mOw7+0% z^7LUDw8X1~mEqBMW_LB9DXk+x)hbuYmsBvibtU3ikzsjPOL6PFLUuVqlM|p(za}P5 z`S&%^A6^_vb{=m?eQVd>fzq;?bMS1pu;kYXQX9L}$PX6)qcGR7bSGcJ&Trr7fs=hB zWrOehll;c_`l9lP<3TOfMGF8r)z(V^k|?25_%^frsbVuK^>6c66bCj{iW`QWYu}j# z_%6v(?n(>A%mu9Ol*6hfNk#ZsM%C0Lc3YhQ;4f*1E-lySaRokhb_RJl04_~=s;FW0 zr*<}&Te)(uYLi+0SkOX7&ugeAHsjR$OJ^~$*+;lKDCZd&<@U91H1?w$v`h;|hx|Q1 zhHt-+KI{9hzmu-swcmdpUQ@n)sN8@7YE(a109Zh$zs@=Nl`>aNCCD})2WQ8KN&Ta> z*y`$xRmy|x2Ho^qdRdq9nfuHOvOh%Jht6U7ad zPNY*bs{duOF1r9aO)dzGR?;IViHu(8$6_suF*)1Xr-E)PZ-H8r^78#{_%|mtm#-+E z@%y)*rdL0G{b7K1@jnBc^Pk^-e0g!zCJb5~^NIo^S*MIs zUE0D}sLd8v2>4~qovWH&T1S-O@Y1<-yEidybX%SBrshD>6=Psk)b%yH<)XGyu>qQR z#0A4sQdr|n$ERc$K>>U?i4DAd^7e7M_CKTEIkbpB4kzB2T>Ey9+XV=wo@0e%l!U5r zpyKB-rpb_vE>^i{YkNYPN~PSXr4eL4*FEXQT2+`hUi9KYm)-F(Gy$YBU!uZ6rQjg4 zd_I<(x&SJjC#n6WK%Bj6xcZT=@*~;sd%3MD|4{~7H-TLVV3nWuXx%MI9Z+Ef6}*{j z-YBPa67WH@ZR5@!Q;Ch#xL#$QRs;?5$r4>n;)cm;+ey-TcKuMvS^Edb6M#4Pci1vq zdq2CUNivkYPr8VLj|b1jdOV4YB2YGOJnA~UU^rA;V3x}QGCjo2+oTw^=2$*RYkeI&OtEp#hld+Brb8 z8l*^=GK=Abssk1kl8Z@0g}T=|Y=ZDu6<6yMuvP;g)!m*(Z6d8l*mXP2e=q%my0cQK z0xIA)U3V$(=cCgWL&rYCTpNkeg$JoT_QyuxtHSu4g#TJsiE>Sif@PKz-(R;aeQJzGjJ_oU{?Yj&rC z#l6`rIfG($l8%eGyMsFlqs(#5koR9;`SL?>U;E1)%)z^)1zP4x0SzvlQ?@qgH?RR5 zn9x9fBEJzpG-tJ85sdwZ3ko2J7-;}i(TM}itLHNL1?u}nNzGgCOu-3f_K9*-E7oGP zgrssF01Gv>rUsy@?OA#f+(~EOmNT6mY$3IQnJO>Fv?6qAi^l1eP~z`i_aq+U;bqO!l$d22gN3@6;4vl&rd}ZtGuBtutgU4Jd4}=e)y~7` zv@GYu5I4g_-(7WKxPV2=DPau{MeLc$6Ym53cbW36G^%`VTm@H53G1P8CI2U%oTXzk zR)BvwnuUoJ;p#gEx|)F&wHe2bGJ!EZdixCz%s(hD=vy9`FH$b=$@3Dtg%7NAI>|V9 z$-1FZhn$7Yl<80bD>lKo5i)MX2^$ZyY3@}^Bwb!Fl~s4Hkeh_Z9;{N` z@b+%ZR-eA(ExS z$bir{s#1YItv6Z~3a@QFH+h_(^wB!a%TcY8=e!`D|5B_aF zG=Ki~W6q0JR~Q<^pS*r{(eK1ae*sL_SVc+I>jdBeP;5`Y$aZCBX9C`J zGK<{WJ$vlQZ$h-BR01>H>qBL|q?$Neu-)7aSXX-$druI;sZEzgromx?QV2I}GxNp| z_x9awZWkZs&ziM+Fx&#S!1Bw=RrgPxaIU&nI)IZZERErQHVRH+0qjlXSVvWlk#c?& zds5c`g+TG)o)RlpB$Cco65guVsRsf*CfwSfGisuJ;jSlTu#2N&34OMVo3*OlgmbiC z`g&k*ekJe%7>YJV?@-`5(-G`mFXu-Cm8IxXlx2x$KEvg{-xlJ?7sb37XL#pG(5#j=vq z!pYxSl~f5tNr>OE(qszzrCbt*#)N^oJAVO-$09G4Z_CTEO71fH_)Xy}W^__=LI5Xc zGV9uy=JVXnOEBK$U~AnkM{4pU`^%9?1!1~@ScqwPkt(KM7J4Q~5eKaoOQ+SYrq{N6 zHYs4nGlcy~gbUBkFj;Gt@2hkf1{js=xg_`{EHw6^;4Y#}?wgj+QA*K7bTgeIZIoRl zD-lbiKeI`qrUl9$tm;^1cY|(*9n{W6in>{TAS|VIv)v!yB0i*?e5vc)%O88#<;cGg zfbzh^a!&G(Nx43(c^39Sdt62X?SX&LP!NiE?*bRa$2NiRHst$INw+E)wvEWdKD-w!Eh9~xZzSBlAF9| zE~Ty~C_s%hTB;9}ZLN}f_5_BBxz~Z!U;*%?+}R9R4H1`^gniShdaCw3wsQ5PWBXJZ z)Z2}|2uY`(<8vs%!++}+ zuzl$_fA`^m^8Wj;zflXVe|r6OkUxHTd5JDE3I_gqnwXEPtzkJ3gounS>m=Cc_ULik zmL+FdU0U*Iz96JNIvD&osF^;oD8y@Z=>xYl7|3F+Z$L@d4jz7fgo#|O^E-6QPRn1p z5eJ=)+4Kth`>}`%@3$pPHz1xQ@6ta>N{qz>k%}^uB>qdq=>7_2*l|-N;-2KjWh?D2 zNaBfBC*Zi;4d*29;j}qBqOBnn41QPU$wJ(6qQ=RlTJ=g>X$I|bh`mRHcB2JPHPz@u zusOfN%V1)nM>&`jE?s=<#MXoNa`0IJjnPW11*P}d-9TYcxE;ZFklR+Nb&4%%~AA(*MOo*SjR zH|b($*pc)7g%GyG>o?zlO6?um6U@@**bv$RVjSB`(l`Kja`wzT)h8h|kmlVjVC*y` zcW!F>kVgTM_9slwkMY@(YnA1=(iGD|sz<%Z1!NZv%dsuAz9q!QeYwEUY=LH>1BW|H zZ0VVlw5%$jjk?laSMI<|JR@#oysUmC)$9l8) zqD8$PT_H?LITnz82YJ1U;B=s*EiE+o9^z8AfJyUW z=WC5sc4abOu{{ac$bH?NcC*k4SxzL2FW-Ltp)Pu_kO-agTNwoddlh@QI3OITWs zT!|es%fcO@rL5#ZYg+EzO^!549CB-=P?ILcPCMNHh$IM8CSmG?Z9~Unfw(Ic#$vPw z{`MrBkzPoWqS!R<7T19wt+iIqH}^Zbn%AU~0_>tL-O1}=p5_{Jxz?pApeZKj#)|_p z4aJEzqF`gEI}lL&N#j3>F%F&DjR4AF5ifV_YAIAMU155f>Z`1VCOs@S@0X`Ihs@Z7 zzm1>E*Qu9$D*Zz%Nk92|T9B(wKq=*>k^1G@)@w={QoBl}3F?J9SFdce#_(cVK{%tP zYiOrsnW%g&=~V%BDfo7nZP8f<@?bU1!r5HNwSK|y5C9f0lqCZ)1*XvYRr17DkJE4d z;k;H<^xVQD6^ZQ5DrVc4gu*o#8Wu6r)kRRZyerg|5XB%9vC=|dN0C62>s1vs&k1#{ zrJs_6+>cf*`0`4K74HH1Dnxa>A{1m59}b)je3zt@Yp| z*YH%cr-6f@Qz|Pe<-xz@DCN4#O>Ou46=*Qp683JX zd&OZf_q=2{=xE$oO%37@h=8nNf>Ir@uyR`D>n#=cNn?)4QaOOku1W3Ej{NX!^Ddo+ zIgtyT{-$v}@I%TQ7h?s7^VX3jG;W~14AW^SpfFyqHpdzZ=|Xa?9cReRf%E8yZQso4 zf*L}+C5O02=!O}1U(%pf9Qy0(O-X?T58pMw`Jp{*PjHwq=t#VXFEPCT-@o~LH*cQT z52P^H7KUSy7@$JVVwbHeP?WfsdY6Yh52qbvNl9Aj1#@$uN=uGx#MZ#l@-;kWJE1!%grTlXxVHN4mJ`hc zDjnKdvDg|?woF}mi5{6OiG zvFIpv_NhC3oS7%K#~HP!RNxWNU9L4@S*u4%Nkd`)F_%sa;j~nBF1n=q9oT^rD0-b{ zRd?rpO3yaCoGWx>>~34-i{gNo!Ch$Pof0)%<--Gj;8rHk!5fiV!j9uCSuJ)^X~K)Y#c8O9;o+RO1$q35oYln#}&lLAiJlkXl? zI-LxJH57xwm1u|qy4;EQEA1NM$wfrrW?b#I{)8md^F{UPInz{SJX8TKm58wzdLnJyRlb%T z`dtswn4^)JR$=fq-l})#MkwX(f|~1)BmNFphx}#(S0{C)`N|0tPdB+I)?o@PPFI)! zCx-&;Q+rA|ChG&F*wJZfJti>4=|Q|`_tPm!#97&98FhDv@J`5-Y@zjtd~@| zWBxy5EN&;V;=?okZ&h)~lF|s0!X!&lC~pF02i^CxSEeMOWzB5B10^|TnPJe~)eF&{ zA;c#}AHUF4!a0w}=qNuypsN9l)hW6Tr}}v78I6N2Y++`{mtY!o(u>;Y?s zi}Ix~Xsk3IJN1~Zxt*(20=b+(l4J?mLWF2!9#v3&<$ERAlUpn1Z!PjM9hu;^SgQt9 z=Pe90%aXcm*4%E?tjvc0F}6EET2gr4>E7sy7+B5Hp79gel zRUZ~%>-h-*2tPzu_Vw$JQUt?A(BvQwtSX=x@qnPsOHLZeJQi~WjB!&Q1B`0Ih(!9G zQci|#X6L*CKjr!su+re3WF39{v);l)eKGi`I@M?0CN6e>8W`YyR-@s1kn<57N&Rl` zvZ~{J#h_^R7dxcKcvnjp63*k(BLbL5 zt^QJ;;+@5+yExQZA;;5*o zQo`1WN`LVATiXYbe=Yf17-5m-n`#l1BEjz1LLxS;qOxEZD@Guyr{kh>vy!|qLGQJ| zAN;aNFqaDuP3ak^tT|QxA-pA$oZ^{3vN|I%k>FnoF$nKr4?wAyogTU_m&AnJ4c`j~ zJKj;-P|e`ZK7E-e%!EA%B(Jb&MOZ+q@ySB9)JaKgfwDlQzu!2rpP>zA5=?MvwG1}P znOAvKtNu_5J7}6)074&BwR@}`TkKHDx80z_hW?zhYZY%|hJagN<5&&=n2XxSR^>@U zy4U$5mrHU(-rb0_4|5|OpvsrfFGd|?9fkA+H`IC|tK}(}wNE#>oO7pBGKdkWydLF8 zx}+qQ0)D6bR0C&Q$l)4Eb6SQI$mv?ChQ0F^>wBD2HsQT6TfGk@5;A*LV)bOXAm^Zs z%j8ElNd^qjY|@}9*H-Zu@n+qTP$y|9(3&8w3YAqU#3qgqLfawwyNk*s*nS4zn@b;f zAyp(=sIRyLIE}3hO!hfZWuMzT*Jw#!lBe_~=ei2?4{!ep0_mqAzW-0({5=#!pjq#g zf>tf5Xz{7p*5M{e!&|spU6i?w0^B;r+3Rzy6+x~r1i#wA#*`h^Y)}_1;wuy>qu7gxorKFe0IJ1 zFdpk>QqvE3Fpy@Bl-3zD023Bzac2zd`x6*m4^N;9{V>lqLy}-u>2B?qmJ6N9K*K>C zuv`)y7P!4~mv@ffqh)v90WJnFK5m8W^dKkpJ_mx4F+92vVFI*!E{$3^)#BD^n5i~4 z=FO5sBpRJor33}g6di9>18o7>3>#2zEqdf$S`TBP6eWBB$9{HUM(0{34#YMukP zUyt^Hn!qDvk;g^t%^y>Lm7v6m*s4g}pEfE4h7;!(r~NQD(y;5C%d$<})MK_y*HUbB zF#TY`x!4?Fom@_8&9)h+Jh461)FrGd?-c`Iffb z@)(5!aRSIO$97F;T#-AddB5{MQq{RBhVq#0)oI0~U>o^&;a2>Fs-<*9@P zR4N)}xp82QcF2w=Nu9G2FSlu@oRGWfDFfFbL;(02;;2?GjCULBJ`fw1M?)$TXh4$K zuj>Eft%u>oVk`-*#oPxHSa}hoYFJ++QLG z?;#LAe*0Gh!pEGt{L||na+E+RpNuE~ZEe!1ni`pF` zG1qC@3UZ;*RW9B{tjasQoNcJqt!r@q+Qm-e>X|k`9f0Jl`~%--#AkI;Q1n9ehPj>O z(m#W5SyCf*M(>DCE>uT-HCX9l+Ts8+$&6G3t;{j3BUH&$9EgHv!DpU8zJptZc|4ng zvDEOFbFIN9o>K0qaHj(TNG(0etB3n6t0@U(1hM9VgpVONAP0iIEi59j`{}4D3}OP` z)vg!};ojD|+P&9~MV@ClSMleimIUnJd6)rO$SS>zM;{ywjOYQN)(#YALjq$5%T!k5 z08;V^u_VAyI)$&0O5ZMBa@x=}kTYqa-I`^2is-6hacOo{_RRu&sL|A$1Y1b@A+%H@ zw?pw0ehlA!0U`gdy6UMp!G2UrC_Irqs7nozWGkJdMGV^xoV<#W0h4Aw`Zl48ksvpY zWj&4#S>KR_Yrx`HDJwZ{0xBNh`gIJwfh)KeUuuZd^{3hX$$o56wem1l#+-QvYiiYi z-%RRw?r8Wpqe?Nn!C26YW%A8oSLLE+=)i@yv9YHW2ECDG=pgEZvWwe7Fnb4k0<4mXpqgYE)m;`pgN$ zNvUPhvumwB$ej(eYN-$om%bV#7*&F~^9D1Rf?Cy`95X9viAiCTUl=OIut8g`h`=sF z3Y2mq+Pdt|`WYJ7(n3odD`OlBqlNnP%6RjdJ*f+z9C6X%r2}|eM+oVq((dEG34irJ z<=Fn}?FSz!2b^BusQgn7(0vZ`TO@J)kp9%yU(~FTD+zh;;bY&PV(_m#s$PUmm0g7BvGdLtkiU`J(Tk9A&`bgt>HQDfeP6Yzg^ zCK-Vo%(`>wq*j(@ujKmcDp{$8NwU116OfE+Xvmyrn825flsv}n(8CaS$4w-tOr|0Yt2ykLaqy-9Q zI{j|%Ufk1SzMJs(Jl$%{Ez>ML-3v(JhRBl>Q;t5e|L&P(H&8izh)-m|NR@Nrh)wFk zclg78xtod*PcY?5$3z&cvKcV}l4RPASKKq5_xnp;S;>9i5I8wPbX9uQN0|GfW#?a2 zQnCO~VSQ`^DUcd_tvF&)sCT&`T$a^=5Q(O|pd1>dUy<4=DCN^`0ltG^p66m6h=M^< ztHd%EF}LWpu<}!Y0=uMQpP1*xWk{o(xd5Bg-~y$0Q5m!l;7kED0l5xxV==zYGtsu7TdW2(WQ;<&rNCh)rqC!*8mo-tET z$mDP=m_uQDg$wU{0Ik`p50aj<_WywEll$`loHM8=qU3m|Zeh|Z3=niP~tVSoztEV|IwgW?3#tkD7j;>)wSOLl`;Y(KaYyN5ZA%oU*BvziGYxs(U$tU#C#5i86{sG}L#sa|0v^ECn`^e=({3fVm4i2Sn9s4keJDOzL43Iyh?3O zB~(&Lt*Dgl&LinN;V<-~VI@9y(FS|qRxDphKaO&PP7)g3Ip;q zSyc?5ln4_Fd~7tuf~r);Uy^Duw-R_gq-mrf)%I8pUHVhYQw*D-=6bTCFi<`OW7vT7 zO2tdBKS8Qs2A*LmBPRS%YoNQd+~iGbKLqB{{CSne$)E0uy>-S6C$wcPwI0+67? zwJI_1E?WngcSEwBdtv@Av3;&GSy$az>a^y~TRphP?Yi#8Scu%jKDOI4|LUt(@@rocPxkfWx8DRl`1VKtchV%hnmNUP zMjMfw*|~Xr)Vhc7xHw$Fvl|O60LCM_b)r)?)zv*MmYA4IkL2cO+K?vqV_S zMNKY9K7(dLEmb3q*>@IdB-YQUP5`9dA{CE`xMGwp{ux>UZJ|lS(=VebuQJ1&X?66$kPtom_tXrvw7i3 zap9I|`10{2U<0bgi{r%FGznwMA@#J26UH8$u(M)oQhu3jr&F&%+i z09!Zt>=7-yl;RYPdk*t7$bAI7FCIvNw7L+cWBR0XHhWMt45|-x{WK+EH0%a32&V5AnMc>{C@q^R-QdPdQO$fIp%TxD?20%IR&)U`sZg zvL?;8CN5@1xjqFan1`%1u>ymH1Ig};yQ#rjPKN;BEpH*-?U4Dhqj^!$3mbo{Yx3%w zbj-HW89J?~Idq%T5&brV@e!^9~VX%;)XyfRST*RI2sK zwQF{pl6{v_@~%_%dw3F@P2+xs1;pAVX343G1#?}=D_eaz8Ii&y$hk#n**WcfmRiUrA*9M(mWJ+}mjP-AHtTHk?wg*wua+Gx=ImMtDem#!a@N=7j`d=Ch> zWqN2qJZ2?KNy1lbwoX_$=q&c~nSB?!PNGl~n8l%1RLsZVhD+t5N=cTDr#nPWEL4*z z!&58*wF6ry{oCLVz5 z518`s4r#vh5N>ZC)@3ijBK;P5>*z4@zAN6i-~yw)F-eCaR93IdEZZP_=bH2Y-+H0h<#@9Iui5QE?95|=If#KnI4hACEoz4s$vR-<=a(7NR0 zs7P!}gD}9`nH)PFxX{0GElcp{?AE7x-g!dZ*K?Ym(li^Wqh?a6>!*5hT%^=nvs^d3 z3$#Vcvz8Xkyn0&|eIy>Qtvk#*tRjHKQ%}sjcH$0oO)D=9jnM3?0|95K(!*tuPuR04 zuiv;$wJwt@i5)6@z~E_7vw`#RB06VF>p-4qW${ zSNi(o=X>8twtq(!I}GM<<`5|Y+h(_M$5d$HFCza)5?qCE63TtHKtjJ@0i!z}deb6) zEL+|H55QD85{Nbu(FA9A*YN5vqNt<$==hq1u(1WglQ-{ej+n$(_56aA`bg<6$ZfIH z!9KC)b#ZKX#7K(>GPInX!WN-$9Mx`~R){E*b)NbXd25sqc-eHiK-hPO-Wmxza zOowy8ezi$b_hp{i*cMRAg#kSJhu_;bVke%Br%3Go8pt!fftWLNOQ&ODLrvHP>OqXP z{D2xJs;IF&RhwPhp0}(|($o{qz`4yr&J9X`z{a439uG*4%h$bja>!hj;CqDUE)C1E zTd+3(R!dWr?ZV}X*wVGCXCRW#qIBq&m+bakCgUuabt~u<2O39v5@y8_^rMB2q|{5H z2;{hBVyT>a6A#=%nVAn$^Jyp7_6gDO7SM`b$o-EC20^u|Vffp{&Y63(Kk=0xvB~~X z{=6^WKKoYx@Y%N;jL$x7C_WFb-@m;7VR-!tjG167Yq*D8fNz9<p#9dWW-?J8hn$8se~=#R zYRaB-7wnN*!a|7M?M?>*x?@%KI*l^%I5%utHaT|2gyzFIgbjIpjE<9?u~Q}FJKS2( z%E1hzW>Ia6nyFBJ;>nl_?QO3Dw9YC+$YJo=cBekQ1V0PHo){W)==ALU=pYA={p8p{ zQ&XicNj|wvK#J5HfDf=}f0}SpmK935joZHs@@{Se*nmWQ$o)rJz4}x?7}&ZwPuwsj z7?p((debhmlqZ9fU$UzxEw(YiQM=LgoT7zP!Mv zg*}40tDVH@^eEUDsEaBoitg-SnS(NOV$86EC%nWAu)cWvI?z}32XCLK3HC?Q#FOEZ z{^tJ(ufGK&ld0(*QNN6reNL9AHjKfGRLM{U3f*w6y2iw^6{D>>k8ELoSry*(1FOBM z7%n~Bp#h#A#pf<>5;hnsJGKx=!s^JKzVZ(!(0_$lZCCRq!h!OV!=-jRn*!K7a$n|mc@or8R`c0sq-VHk7{?{cS1D0 zVOa_j9XmSEG7*zCp2xBf3Ww`gx*t%aUctg`raoLt(q|g4yw)}mGNRC|GiecdLEc*M z>*e=lJH7yT!jq~A?uG-*^P0Jr(>^L0nRS4Kb>AeDVXKxpm_w1!S#`{;m~|BsO|~lc zH7YRgr$&raKGH5UC#2z*WVU_v_VNGA&GU=buXRlk3`TOO#3%ks$B?}BaE4Qv7)6Fi z>b5QQU9M@`RkXWnM$3&5vez>*P&nNrb=H$O#$?Rs_W_J{7q<_F=~6}NXIH7h+CvYc z5|qy=CglV`4pm~6Ahvg>!BBE3QpK@Fm#9wD9d^}Bsi=dG1Bfh5+_Nfj4)n2{AmsuF zt^-6l9oxDX58UlnA8(=bm>oSr4d>fJKVc}Kge|8(Ii(Fx(^C1T0j-z=sXlnAb?srX zShiB3DG3W%^I>@Z&K`>g@jYD5sz*G>%1@fqPx|5IK6nn&-hANNs@ucUCQAUYz z&y}9ikL({`=T2;{;pM3Lc^qhO|&Oz{B4r^ zl+GZMBYDpo{FDQz0+tC(-9;_miF>ggtoz36gd3_$Rtb<6CWr~dkyB*;`t@JK|CXMb z?(0{H5JDjI0R0s^gjGG>)}qHS>txmL=Jnxc#kw!|I^0~u8Ct=v8Wvnc6o#pAeC@l- zycbwSK^)dgT&w>u1m{Fi6?aEn&+ipRj(z5U2lhriD5NVnIky;0dKXy40Rj*T_D!US$0i~?)p-`)Frb)TA)*H z0UjoBsv1@NsW#Ypjjvemlzr9Hw(JmP(%%dri4$4~Zn^eCSmii?6fPU zYF%}=qO$YMKrggU8<8fgi;TUNZRxSp-)&OR7%zVn{wyb@FW)}pfZ$Vq_4Q-2C4cey z2S{uFIK2Jo0uB?h*4OX<32e!ZxA^09i*PQm)i!HFan|l>Sgvl|7z{MKbO{^{NYfAJ z14($}WPFmucufx>%QdYODF92ZJ9$0J$yP>U(HG1VblGUdC(ucKTq#P}=~Mv9XBbIt zy4!>3w3JF1ePO`XtBg`6;>M?7GHghEoQ(m-MQ5QXCw$*!Il~gX%v;>AS7q@%)spmV zbp8}f5?f&jc69UP?yg>7bSg?=l#h0D5&K4=>~K}#A`nW)w(SF`GO;nQeKHAO8ObYl zUzN%lPIepA&n1Xdu4;&><&=T$&4I0doe4yj&dT0`3iZsmIe4=*wSk1K zhX};zRo@sZ==0fB#D^jU{MS?itTNRmNNJJGw7Jow>eE>tB0+ENGKnEQ$*+i8hvtEO zr%tPOXWZKK%&N%5ri=wOY9u&^ln%cu{{{J99)vS%+kuYmClavJNxqU*{x2^9T8f@9g28&q64e? z4$>-b207z&{{rB*6F18llk%D9IH@XHOq#j-fZiuYJ`&+i+7YIxJeFkS1^BxCYI66? zLW-0MXmo_wjh$^%UOMb0>bwxqp8VS7ppR(&519NnWf6WDc# zDm1VvgPqF>oOP>oQ5Uo1QYn3lF~B8V?o?s{0~-!nV~-kKsz8D>7FUg(>lIgS2`{l9 z*y=R=%>j5u;U-QGDP3DUQAEF{D=I)dkUx5}NYN{ooTL3lyIva^wbQz($}A{4#&No) zqbg%OLS^Hkr26C@HIRm?DJ!d_N-Qx6x#$~gM=t*^{H6StveXaWJ_d^DMQzIL$Oaaj z*I?7*n@%K9DzHatLNBdIxK311+4c$*k#EG4?3==7H3v0`V~!c(odwZ5%!$Yz+Z`M* zEsZt^ws zHnJdC@P!d*?dLs`Q@?{FKRE$>!`#QRoZ8fVARW8F0m~=g@Ai0trL1a?NthP)QACZ{ z3ysI7@^<7+sNd}>)4_b_nG)iNfL-jX4}Fk0Gvt>NeR4$xCcjDEa8Bg0u{$LsX}OwO z|5nao?wJlr4+Mg?yv}m&)jr$YeZmWTXvqCMDCMUJ21n^LvFZStH&cQT>_?5=j0&60oJ z6UFAGTb`WED!fVg!@7WfSW)u_p0-CY2;m5f2XWAc_WkK`;X*2y8{Fiu=CfeV88BBO zodn#q!e^^bFoaBB=c$&S4GKJmdTm=omyk#tD-5q9Mk_WRALhOPfN(MllYDEkrlgxA zp|ZyyXC+ZN?O@YcfK^H{nSPU8F6o zo%;*{sM<@SxTf%db=$#eQYjJ~9HWwq{Y9tckFH+jh;6A`5^B>ZIYPNzAm9rF>{0E; zUB4Kh*E2i3M+HpZWQZMFOM^0wZ)K~ZS}FsWQO+&4W~;X!Sux=%_r7JjHn7I42MaVT z!Ve#kt3eBPOcqy_i-1EC2RU!le09CZ)l?ZWyOK+ECOuBZ-fulS_2rbgcP>ncO?sc# zb*~@4{hAbwEPK4AL@O%6pbNfFuxh6$0fB> zdCiVkh^x{l5HciHpoF$zj~YTz%u7*HoO0q9^l@vvwb{`U=@lK6eL0X<6!vvYSK@y^ zF_W|pm(IU>{c(Qvm#<$3KHx`-_fgI5;dxwNUL0|9)Jh(odX`WRNu4mv3@%opY0weD z)LgKj*PSqOpQbyL`~XOp&QuV+BSs2{HPAaV@lIZYd|`!Z^^jA8a&r3ziX7`Spz6IE z^m?r^B5@P=2ne6uGtE7`Ow-WilOC#|Qup36zFZ2}yBgbju61W*P+I%T9Jm9(ou11h zy~nIv-3`7xw*3e8g{oT&n5}h(pgGrc(i$WZq12icV-XHs4X7*&YXN7uLk3^c^yUMz z8}#!Y9J?T6q{Dxbv!~-?Ny6KlH>X7J&0>;meMORT(^}l#eVy_2_2lNmYS;zsP6=l< z8ZA4oMU;KaITJX}NcmQu15jafco%Cx)WGg%aufJ1WD;(nvTsx?De^=a)O6dOzYB)1 z*W}-g*FFvdx?j3;!MoLIe-I|g+l4^6P^=Xw3;#d-|NIyD!0y0rzUSzV4^bfB3$I^8 zN@q(ADq3q|2P7%o)JxonW4KwKawSWN&tt$IPJKWe&opIW28eU4=Ib)pY<6Wyi}$sR zL^9Q;0JWZd>oHe#`kmYmcPzN}ar4zYwU8k{FYn?-Wo9{M>fEj>Ap)aY!(!^$*GZ<1 zj)A?(h~#Y6*Qy~5=8K~bZx7|ez&&>LVhZbi`BqSaSSBjhTjiAqU2gEGGkj^DW(V9a z4v@t3+)yALlEsGVt$D3-|?}~Q-1JM0T!>bv-gLyMOurLGt+Q^+z1qn^w&~y#6#Ksamz3jg?CF zP1IXb{M?8<3`fh&6@|AG!w8De5rRoA$M9g=0BA}U8dt8Gqy~e)CA$l^JeEx6qT5nU zSayYG*5G7Ps5a;})!u$=odG98{Sk|ua__a=OkB#1(iOx$lAUC`SY3jGVH%!nI4}9n z2t*^WZ`-2L)k>-}IMS_+s&>sZ+H)p9idZg)t_R#94bYj}B(V5-YoM1>MD`Cvx<=09&D!lrLl_tzncx=N2A| zH(PNIt@KmX(;hbJe(AOsDMB7im=?NA>W5NhKY#l?z54mvuivC@`TX^Vl0^GTg(g#` z-$%@yBpk}@)~G-xH8reVe=b4Zip~X-wSvIP)RH1Fkg6{N)~`i9#7c-o0DV)A%DgWK zJ=aqs+jxw~=9c|hE*jim9Ui@O_?+Nsy+uN;<;;gJ&Xa_C=Y8`itt*t}$%Th3DeePH zy3@FYr(HfKUZZ8=RbHzamsx^gFul^T0LBjCL&8&Jjf$#2032q|fD4lxcGLX;na~vV z0B)6gFDP8tO{1DF>1OD#{Ia2%Vv(W)64d0*XAckn2fk1_QfM#(@BW6#{i;M>qed*9 z@*5s(@TCS19CB)Z?5aChggjOhC1vg6bg{adg9PuA z&sc1H#?1d?dg6SQJ_2m|(W`okeD(U(f7Gv|s*$z|+V55gNG=?^Dr%!`FQA>CGO$+i z^|z6)<&{;OdiP=#09`-)!jAx=B3MbYG$KyYVDZttNwqLQYfp_xYs( z=mvWzmBQnMrkIpN#sP|Iv8o>we$I|}QFY|);Z06}-tHu~GLgH-f)85jm98tp4xu^LB*Mw-=C2eTdZIZ#EFnd%`$MuDVZLz)tF#@-0( z6)4>X4O-AANzMh`h>=l|%mUre%?U)1^dkspkP~HrU(V>(r4XqnOgm!zWgD3OfXZDP zm$VRup8t2hYo$&;B7PIze$PVaCq(@GgdfxA{ipEu=|!bd9(CdNM%z-qU|UL(s&WXk z_ky7Bq#UvlNQY)R6h)h_K zZ%9&+(_zT|Gtcjk0?o68VKO@gK81W>7Kt5jhr_wdo8l$ZtYNdwHq$@VLW4qSR>dNfa>vB|D zX2Xe?TOUj0~&;EdX3VqrZDXCuDVf0RS*D=~+5|p`W%|)i4J)3`x$l zR>qhrqD_yd0?Uuns>DdGE!!P=qofs6;po6s#6fWVJwU{I0nVq5PPa7SysLqaB=_Ag z+euGU02D%6a7(^o;j=Ia<{t~ghb;mH*0k_F-XJbvKM;2LTm^xXOQn^5>wxF-NZCOs zpjHrYt@@{gX4X%U#vJgKj>yOx0%33=6v#r9tacCF?~mwYefeh3mbSuEtZLIj#M+iP z{KC8rTg-uNtlVjNKol5A#}K{i-F85yWwsB#F4K!j0Ph?w`?#!v!nBVEq0FG2%`|YP zJUdMvIPZ}&KQ$~*Q@@kZkB#41jh%>CTK;VA%Ea$ED~$ zVEWhivd|8DqO!XJ!d$g9<~3ea8otj0p|t6q6%r=R9jE&rUjO)k;+4MiIX#x}i7d6g&y+3K0Q)Ef7S|}B@G?;d+I?1^mE%lP1;VcCyf|&3?T6f z9HP4(wlD;$SE9?IkJDN(%2|j1Qs6Bew6P34*eDv+av1U;dY@hwUk4u5$X}g1AAn@q zoYJx!o(*;xfN55k#l=nf@$4OBRE%fGa=z((RI1We?k6Fah>*A5>q1s zT@#PQ^c{2ck#gxHaN|?Y!2!4j(kOt*hi=~`Z!y>6PjXf3+bOH)ERNur&k0o*Bs*Fu zo(C=vXfXl+GzFyq_F;<`J8Hq2Hx>n!-?EM8>nGOgBe6>Z=s930TWXzMPR$HgJH-7zE%`;GvsMhRRkL%)kM&-_|{tJ(QF5f+;&Lx&f`FN`LCA z8qn%Wx*rUxR2yid?bnip+y*rj95N{$s25*z4bYg9b{fJw7Y2n|5#+r^<{?#2`ZyIs=K(vqP{&B%+_UTZyTv!$o4d;3N>tchR1 z(5*MlR^;%kYL!M??ve&B<>*qpkbaouw3Q#u`T66q2kh`{$%W&q*{7UOe;QuD&x7?} z-+r0?4Cf~jnlBNu9trQ8ViQhuRvAG~QqQB~qQ(=84)04F2mz2x46XEV7^W_8Gq6P2 z&1CYsME+#%8&2Ip5UyH^txK+hutx zR2np_|AjKdC;AC5wL82?E5moYAj|F1N~dSz-G@?SARrELk7#`5M`SMzo|PmhYg&M+ zu>1b5s#Vx5y2}+&-_|kUEXwEz^#Ns|v{DTYwX-V0f9|QU(qJ6~ zUdN_uG~_(P)!GWVD}jGY1}`1cU!>9X(EXCTz-{%YegNL*N!-z$u1v>u%uU1v=u!CPNDJ3)~$5nu&elXaEvkR&` zQKzm`IluxSj7`*!@>@{K7up(jYOCXPs2bfqb)y1Yd?JOr0ScwmVt?9k2XxYngP>h4 zXqV~}8Ih_(U?Mj8uAtmq0wJ*iU8u0uL~x7jK7kanGz1efbnVoL3JS~_g?6LlTf7n; zfp%OjzIH-jT2K1VD;KE5Wm}YIvAVY%eZCM(OmdrJ1v3PJ8$l8P}@_2t@GhluSd)|sqeI6_) zPpxk+0$XKAE%PYVX;Nfux4`#N-B9GCM2+2IQ@Mc%veb?f#|%}9IP}{)mq^3e6?TBX z%c3PHE7nM(>f7#Ne@leg3H%!@2APi^o^N-#6|en%y&jTZsk5HoaOsI|QB$FGRScfw z=xvo=eo}d8$mpn-1-vmV7Sfar1qw%@^{iW@+JvM5ea|b*9d`i9!R4X=$`|;S0Oqhj z)pV)PfQV9pFzn@}#IZ*?;$nY!N$@gSmLL?KENI=K9!e-uYvLoys*ApV+@AJ2)jBS* zg<k?J>6B!BaF;hVopufg{8cOOhmUkI7T z`00MRm%Old6z?h4aZj@3P+zl<{LBE)AjSw);xKz|my0?oCsl4?CqYRT*J8!i0&SpA zQ>T+UDb#?XSyhJC|B(c)G`os@@}&hv6mmLd@4*C}u~QTr*tp+pz^0;8JvDHqYC0>k zL&~N%6^CJ$tYAz@4H9zKN=P%2XfuNJFNqv+lp-8&O*kCL*w;s$dDNWr4Zv^soqBMu^Zp-SeIx(v0Yky6T|TJ5^AU;9up1hctVnB=e=6X&zDP?pOO(QvB%bc!joen5*! z2`MdQ-DcQKF%2JDEoD#ULuw)XXNkSHp7H@-$BSbavMv{vrPzK-BUU%sQM$?ChFsJ# zsGvbjxPanz3;`tddZ$9Z9Fo+2drIiQ3z`T5IOiNNyOVb*zR31TkLCYtCY@st{`v-(KwYfmgRCPjFzqS(_ zw?=a7etCv%&><}Om zjFECfCH**1U+RZfZ31L%usS?b5ToXwFK%Y6@q`SmMc8nO8rJrQ(u+f!RaJ ziC%wbA<-^SnJahc6^D};ucSz+gc>Q>3{V~iOtY|3r*HPNl+bh_z*&D+MD8Zw&I5)Df z1aF)uoiNr9q#IIQ7PV920^4z;tKJQJlR;c?Tag5~)4m;KmqzvcfK?}S>ePd1??c84 z!atf|s$(=tXy1XCcaVGscKS)x136jGB<+TT!%n@MdeYeG4tc0RK~98O=-5#)St>|~ zONL6N4P7(rLG<|umosEiL`JF^OEEoj1{Y_|>eIQ8Ia=>}VY~dQOO=wh9s;8lcFPZ# zcqIl;IQbYWuoYJA+Lg+^TNS(PyJG{CbK70Ku>@NFgzH+)&ULk%2qyUulei3QLL})e zAyBlhS?(B8`3|r}hiC9^W?b8p0;Ol5#F!A%tmm&TTGhWWp6Lv9T?#gk9<@gZX%29| z2B1PzpXBxTkVk%%>=$1WgZTZwy#6-4eGMVcPlB1k+#RF`2u>$8wW|+obS9OSml+!6q-G{p1@>-mE7zcsLDGT~e&#<7LdfZur}xq*tA%YN0`EIp9~ z`NbX$!Ft^QQrvAqqOvyV)6IQ&xy-?I@JHx)!R~UKjGrudU6Y$Sqk1gZm`5k5gCuh_ z!X9>IgZYL^jtPe*H<1{Qp)A#FmUErEBh-6I1Tk;7eJ2sW;YbO*5Do<~J%weKK6%>v ztU{%9dTonar71D$3awBu*39rrrEM>@*5UPL$ylw1*(^}29gXwN?Sl0jGn%oi{%gn4 zNr{0fRXun0!QzFI4J&DgEHBJ&PIP84JwJ4=JH3#CH^!CZamu>a7AQ}BH6XqysrH(B zF|Iqka~_BJ>8Ki}T$-{3`~#pg!3^4rZQOP%`fPbIQjMpK$*ffOPk<2o-%0iQ3^ofJSR^rQn+4Q9i z9OD4G^JE=}(?A_MJC!YlWELDY3SZznmJBZqgX@{Qv_7X;N+@V$flbQ7;A&F~k`K&RETe0l#* zuV2CO@TcM4dj0Wu!DMjUEh7XY4ekft!pQM}K?|P~IcU>Wrr9!3^%s0G%mP8NT$PW%l z7m1Pca+ST|KrVH8yD){@O!qM)&l9U~X;_rl+NFD;Xg-u^ATr5gSH|1G7AESbb;S2`oJszV!6 zK^7 zYrIbjIrhW8g5#2hSjxvg&Hg4R7x^%8fU^zGcr>}wD@c7&np@h39y1D}=zbp(^BPA7 zm)buj@kZX=E~kb2dajZAr#u-_qaEWI#IPH&1xhTwqFL6}N<4BqpdzbI$=2*7;wE89 z@X@hCTU`q)ok|QowgD*S3uL`M|tb3Y9!joMX$J4lCn*DSZBG< z-AeF?mI44W0Fg$3zNY3-g?YsH&ecI(QQM+1{c8VcRo-3d1dJ7deB{)-swXY{R*X8V z;@4`>NgxvQ;7uX7fo{7S!t;3cdjs4tJ%Z2J?uTg-hPQ3<2!v{{I|i#WgF&)u$y$`_ zoc#<|4B6~Nhk>W-f~a9Z^qt!7V?M(rwZF=b>#Bo@OkGzU5jBceUoUHQ(qdd_mX&!AK0KV9-EzwVP2;@BX5u(RNv3Rm03g>j?!483Z9tov7OIIgRi4zw z_g<^}GyQY|1lOA_5s}jFSk!g|&1xcykuh7fSi@c2N7Zt54mPdpcRN(LZU4%xG|ms4 z9)XNcvVd3ee=4aHDiB9*p1YV-1mUV^O!b`FJF%J1ka4Fir3gXWUQ&8so^erIdi?EmFT2G7&>41NOmx_gFxywNr^>L z#BYD3U?$LZ4<$+R_?st~`laE-B44Y{lf+StxU-KZzz(p4iogKeNg4{3#MPaaJK)W; z(+yB137{}ZMD#$6NiRs@KVOuDjHzdH57ZM?i)YKw8)c_Qa>qi^mo)mU@0y3&GFouJ z?cV$yYuq2%a(>)Z^~q}d^m07Vt&Jy2Fl1W!%_@uJsR$YAzEW>k$ew`92N8-AvvPo3 zGgu=^x(F!($(BG~G2W8wC_-q?ojKXL|Aq?QEy#N2-W$C-sBw+-MPY9ZyMro_ zq^-cU9-U0d^l1(^vstRD6U!5A#1n(#w)p4_E-IF?hg+Y6>Pc1t%qX8~fEpkPhXuLj z!7lJLDHJF@=_WPWzN$Tsfjf{la(7>|pEf$f`_?jpfF4ws$#pA^?A;BHLuQr9dHX^J z8Q|~$48w^c@o7P)r38d7rQZjVWIqdUpI#)KeT)Xx&(V_lufdA9|JBr_4$swrl7rV> z-z?MJD>NzWZ=JK&9>_ip2De94+pA|=ZkVE@#Sm^cq~tE1x$_W}S#$ga>~W+$@nrMk14yl zcrp*#GR*7Y`W*q>W!Q^`vSFn zn4aSPH1!!))SnWjqLw6v{{{J9+6+H_ z`vp<;hsra;Zq|5#K;u&}2j12cAT8A%RXfORJ^->z2q(sioDT1~1q<9J`5!Ahfr*k^ z!GGh?w@ild%gXnmm~EYv2}GdSwzSaDw+^dMKBordTv}xeW3r7=8?M=QW+=hO zvqcA}|HLE~UR!c(SxSK1cLN|ldRJ>B0i!Ow|Fvap9%QkIPAN6*x7ma=u|AU|(lIMX4-d1)D*d2U8G}$m0;ZYY!s=u-Q zAje?I)aouvHf^m{k^)@ylf;3XGW1y0fTkMuDfp*Xr%H`{tTr5Xp5UugOWCL(d5UwY zShz`$j>Ok}aPY?h3i6KF-jP1H)VVD^*L_@I9zoxsIpi1$#eRc20;14Ec(pNT4=9D~ zqiE%I61X1T1!}>QZ)JQz;tN{!#v#p7Z0S(alHo zzE&#=l2DXsfu6=Po+SUUU1W=Vz!j2cehV%wQs1))8BA%4 z8U{&$>iwfubS_1Y3o&$V%Luxjt!Wj6H=)cMnsqTklWMwq0VRpVh}jfw>`)#roHk%^%~e@L_p>XV0NA zbF|p0Zm%gqK$-i}E!wA-lHgXK&_4+jGz_c1H0nTyO@|{*g~qMxGm#1VHyW}-vJ#Bk zoRkeES&Z5jFq=2%0T~zoTPed;k#-aFgLY5jL`$&r%F{3ok+|I02(0QJ@UES}{fQiP zpp6vTmbCl4TU9&(o&n{9cJ)hQrQ7G!XZZhSXYD%ZI>mcC#buZGgRIkEyl{a++Uw`x z{pW8Vzb2&R{}z6bVs%YV*aspa(fx8V%_PHwrXfQMl$*Bh2QHFF*oKq|-;ZFq^&MLA zW$;Y~Yfutc?@)kqLZ=kvWs)T2b2)w4;(#G9jeanznIVYXjVI;Lbw&nkJ8;&-!$&u& z;&z5Fq8gNPTlb=DUo`w+K1;$(4FeG={Qw6vk^MErP7uG~XpqeyA&_oW$FfRpn~o#d ze5U~}NQUl8I9>q*O^(Gmyy~cS$K?Q5iX8f*<(e za!`uZmK|S78a1eaYt|M3U@5o)^Zx~X+~RbQm!-c9WV&gY}{bIOi564+k5v6l)NL>nhw_uhTJFv+aBO`E`Rg&K=k z?J!b`99Czic}0!-S+dFs!^z%7o|9EAIz1@9P98;j@DW)VrJ&&$OmeqdPm2+#Z`;WR zBuLF{Yz=D$sme2{raCXUpKVMEsm6MsRj{VbD%r*rZ%84*OULp05!h&59~c}@-X~Q2IL|5^E>>#U z0NwDy;gd-P#$1kmcB2n4tW-v0p9!3nz&_|SW91qL0eV*9#FIVU z93|zQT?!0i)`pfYQ-Pwh8!s>zj1WZ(^=tJcHzxSc*}0;RTDd!px6B7|;;C0elu%E1=g8 z4^*VaVWnBNc5`uoHc!MLkYH`e8&u+#eD-{8esJv2ViwF=!E!uJ92FC&L^s@Of4EN|kN zT-C6+BdVl+#>6bk;sf%+PK^{?oiZq$+Z!y+jwO0xLua|v>(3!_N#x0|&B)>XN8kSF z?aS0CKR*utqi;xtC)lf>B8peXh(n{gP_RBC|1R?b?5o*f)--T|IOzatnj zg`%hXm&6xLL3zHX*OmrKI}_^1ZbPpI^D`)ID>i0hb8A4-v2ai>7iuiqjCMl2gSlD1 z>!@S&1ARwMMM7fX5$^Utc)ZDG;V!t1c>sC~g}HsoX4+Gm^J$Xh0?_?VQRI*fGoSP* z(ueY*sVmYBwi_h<3G_;pO{r+=X_b?*10G6tg90d%ic+Qpn>f)DPiDJDuA?P{i52g`abU6ouO7w0&C}^ z$J2Bb!|rrbs|ec$a>(%3709uOm{INs_nZN!zM+=FH>h%dA8gD6h@Gq|P89`YYc(tp6up^id|~uYr%FM3h~O6b;t;eA&Y7u z9<<~}!Hk?EScz|H@XLvp{0O*fSn*PO77X=!j{a1-`ckd=NUM5M){&pZs^GhjkbFC_ zRi;rD*Ku~SngH9twmE>YT#?;Wb$1CEAR5}blxqVxDOK=L6OER70ph@VC((X?02Qu( z^PGULBR_rnV|e`&@LFHLeVru;{R#9GX6PQJYo0_1dsmt1v{{z8Bn9;Z(U8J)-b%q8 z1NJ2Mhb3Bb=F?7CFR(@j5ZMMIeqjLXjyfn#nJxHmN zbtji>DCA*mLwPJDTa|&bt6P>~kl0%z2L(kY9b1E>Wqs#L0nphi)e?sNcd1p39|N%i z_RfpYSSAYU-XigJ-C-luvmXwSs;nx^BH?lmkcaGTC~#V3-|j9=f%wM3s%&SLUXof( zP|F!~*Br##g9HcHM}Umd;+Es-Ze;MQ=J4>0vr|JZ=~U_Q&eh6X%HdE^d6gj{SZ!6J zURx>S3e4${xK!*k$<@7DRBfuLZ9p3XO%m=h@AWurC98wsMVVkU_gF&~>Oj!t>u3>p z<1FFBtO*GZo*O#pl(2)6=AW50B^Y77z#d|?_)&)jk!-+C=R`=QFF;){^cY{pjD41%a91IL)t&;}3GCax zY&y#1YFAJyBYuZ9b=`e;UMDKn$x%eN_|$Ei$XeiZY?JT@wK_yoPXXKvELNFMaoa z4oMB3vUf~7lD49x9zP3j$v))u^Y@>H*DocfKzB)}n)N{z#IIZppw&)Xv%W3i1p$$ zS49zA>Yj6;qS5Nh^ngD7g9~?LbMAv+X0XKXOcuCq*$JvMjm8 z-fw8x1-IOm%7m#Od-m8wypJ=H?kdx|WzAN!ZtVwEXOJ{<%ZAAGH5LkN-Z)E{5u0uo z`AMZVS`lR{^?yfir(1fL4Z#LnE_0yvnH(PBrJGFNX;jy3D02D!^Tma@S1*1xt4oP? zk)SbiJ7GFaD5URPx*Q;{iI1AedP@5ceRu-fS!+9~8z7=wWn~uHhP~HMC^{_h&O%?E zE5mnki*1Rj?ga4JC1 z9D9}IZ#hGRr?QK@qUw{~FSp%rn-bdVVQ^Xs*onymuOKWWTb6)iz!n33YE6ces#y{= zhaUNg?rgH)fbF>Av{erV6GGK`S#~z6T0aS4&}=Y}CoA3LoGj=#APaTax)gOUJtFJ*JupPL8aJ9mCAQ82MbQEAk6P(>_gdLwGJX#Y8 zNuYLHZy78R8unsv{CG#8tvh$EZ$WCNsfl#9lgA@!;3E^tPj%qnksuIcdS8BB9kV}Bwz(m81Vvt)7 zKA$Bq5&0G2D~O|Bf5?#|_uBzhI;J0GS>pXk#$jY=6d}en{lBm<|LOIo@*sJ1KU8wv zf&^4+Ib6e>6>^#P4z7iib%$(omiXC{b)_BIq}FBQO*ye7NesW}L5!CZq%>`r0oRZ@1B>6ZOPVTz(nU9vf3 zF8WpF~&|bs5-T@EuDn%&bW;J=~6CTGe=(AbHyqfL+TuH=BqX)?4K@ zmg)>kj*qx(iGoD(onuv~X|Y}QoF%NT*!|uP8zFwf)|mWmxjL;@lJ;87iJ#K7B(q$( zrHH;td@`geZYM0P+uXT& zR$GAu$Oc$;&@n;Bb=fx8(Wo=cq$VJeZ|0;6J&mR&rU?^M!V;?B7=)Fz&8?3vV|~*% zwCUK=)S7EYS3XUwmL&i|W{3XvqsC;VLvlir`pi;L1Dj>}bwj7`?ko-tx^7z6*biM9 zK$Gb`F_9c)r5qYsX6_?!oknK_*SJUN)4(-EW#23{hhYO8VUscMx}A2RTnzNbz)8bu zHB1UnI2^h?ra&_WOy6vhdxV<3FdUj}g+U>r@9PsPZvmBRIaJfl(vxb+LK5E%-~Z2P zC4v{`jxnRr!c{->g?0-OSDac{po3&@LA9a*w=X@L>kq|YZiK?5w}6wyKPU*= zJ?#NkdZ9j>s$%sjBWlYQYMbf1Pk^l0V%t{q4j{eU*N{fF-=xN*7Gd{F=>s%sM~ZS| z!xn_xdQ_uPhG2IgX|g;${32JX;ar0Y(Ltf*P~mb;CUB@Rj6OvmnX;<_6+DzXcH<;# z3y<#EsyUTya}T3c_NJMxLPIdqB?|yPnhQ!pw?K?{iYy&x-5(pk*`kNpQ~V(%!7?fV<>4 zn+Y{yHZ#<&M(zwr$h3N^*5+m3dvNH{!j<~}fXuA5iI5vpDS1*L`>7?y>5oLo)sovz z`L5j!!Z6dQos`xVQ_`zQ*)OgcvkDzrb4=ghHxp|-$%U}>wJTDN_?J;ZtDEGzlFf!^ zr;L>0fj^`iA>d_~NRFOr*-5q*n?eI+PUzn=x zEfn3$Wt&Le1ocdov*D&yW{M2dk)8N;K;IK>3C72@Q{}we<=qFM6gQ$LXStXc1(;U{ zvIaNqcpw011wAazG!N%Y{PB8XtwgHo_*=$gH`-ueqZSSVT#j!v?}GnN`6+ z(3N`HQ->=^`A{Bcq{@f7I-ac4Ig{@hmiVPkNesCIdILt_>Qv_nhNE+YCEY=NLbF{d zNt9r335DEyH~k#7A!*J}oG(0bv7)OS!EveO@eo|G5V(6;m}{--oz|gN@!ewQ5DnPj z4mqye*yk*+j|(UVWfB@#B9OtF%o<@@77qda5@eAG*^ZMOdXB*NmU3_8x_8vd2S}o( z24_F5a`WEj#(eoJ7Jq9Nkdkv3c((sjK=2D2*XB|xB;!LQHfX* zo8gseOspF0h?{;cwnv^OcdPG4IM1R)?K7IbYC@tV+JaIi-y=UOyOryzQf2FoJm?$R zGb4w)rKYkgW!rp@m3;*GK8ab=s56eK&xlwIY$ZT17syJe*JWB80RCsSX!ToR7kFX( zmWzw=7sLm3S_^(i%T`+#mfFQ=B7`*@nhuz2IG9a1u;lOlF8tl!Ov*`{Uae zA7b=RFEE^c!EkOqWj}uVIK2HNnJ#~01LBk}zrgOzY-MR341lOOHvw{5(yAWxU(#Q6<|_&H-OoWZ_%LJ} zP!i7)FqqRcCpX!0+L!IfMzxZ5 zR5htyX!c5ONWMZL#nl~<0aFe@E#VX0PPy@t08$YK>!p{xuVfAmC%Wat zb4DK}E$gEDh@(E{Ua)-&oKATC=<@!*e)|u#Zz@SbV&$;hLGF$ za9wfMUQ|u^Y1i)Y6mHZRoP!oUwt)9s&B44H$krtj^;by%P8KgRA(&2;uWfa0-?U6N@k z6v0l;f>;ZcMh;2tvS%!VfCof+gp~mN!pqA6<0&5psnOH}Ut*=a2?vs8n@GqJv^!@Q zg|>@;t%;T{BE$k@mlTVirh;t*Rba6=`R$Kblwor5_IrH&jEY`3^V5;9LRXf;Vr=$8 zy>2Amc6{zH@`kG2y0jpPs?cRy_J{39Sx?6qS-6LGptgXUc59!aSvXsMK}WR`vuJvR%b<3p1CxwO!YG!MhRCEh`uXj}6J zSIgICa_c>mvO|{X>Z&|0k@XEAP2W$NiAW%476G$lw~_=M^g*6~AEDH@SruwSK`7J& zs4ZD?J+&&6FgN74A_hx9=Q{1i5=38H{ z?Xa}dgVDcicO9jot;PTjLBP(*+L&=xso}xNABtS%Xsb3q+i8{Cj>biqoS6wkenn#g z_8)}5OLfrUqq5hsRcErK2kyW&@M8?I28or4k$tGEEJpxCm90VpR1HVjsT=NAV;&3?$L)bcPB;4v+8=Ts9_DU!Efm`zmRv@>25djXi- zsHp5&9b{LPqQVflE8Nrpu5FV#TcxKG%B+_&W4T|Zs~i2uMe2GAngYG$YkWV^$=Vf%xA2_~bq^6qAfThWKtYkm%HBTNA2M6*<1B?zv)9QNI~hpY zcjSCyp^IhDZKSk5qCe%9Xt{+}mYk)~)XlUKw^F$#f{dhr+&R;v+v)(_ZF zB@YjZ5S$lcC|&jK?=ZqH(= z<>=XohYpkgE|QG5S4%}J8Isgu4>heb;kW`Hs0Wk*vM)=>L#3!#MQ``Z8Fie`oImp< zI#tL2`t~RL`VCK{Kibzn1G@5Qc>AertFklMavVrI*7hc8VgXOHwEB{YX{M0^dgF$} zlA4(Ez>EyaYtzo@v9dNL@tN779xFNf_Zt(hQBL8GHU#iMl^keD4RgQ*YvA`PALoL5 zx18uwP%p8E%ZaTQsqliaFr}Sj9ZAf^P+c8I?)62<@QNha;FsrNqJbv)J`9ur!&80g5|KyCrU7WIK1wO$^)&K;`&OBcSx!{O&C?@(*Sr9 zR&FWG7wtqVJs!%UD+a|vRqCH(>#zPmo4tU%7rY&0{b6ElK=J6JO%gtAnjBr8PMd-s zzkU5d)8+g^`dg}h7t+Y*Io><#I_K8-9GcQ%yo8RG1j>B|9wdMI4fPNQI~i*)+b?!w zLRDwXp1f*AYi-VaAo98?->sAsr&1C;dZ8R^ZfH|;OoDze(Ip9X!b1XzIXeVC*Vyk4 zk3|oCu##b^ZKqsKzXtda*~uUnmbP=?Y`9H@>krVE3~X!b@J?F|HaHO^QXy2V-A6TY zk56Q&;ymC=kdHfMIp_g`(u6Rf=*2KWp|BLbMvBI|7h$w6ZKQv-c6GHrs#HA^5<|wBs3;Ty{&j!#bSH zdfCUaRXN;Jsop$_q(IK?m|PK(W+EcZ{cSbY9R) z!|;g36(j{x z`P0X&ahQ8_@ZlV~=gTxG*f1@2nB!Gcpo4Hm(#?>4 z&xHyLybjm`uH|VeTck5oj|RG`;=*;kY3;hp>aoFr7O+5^p|P%n6u@KP&}~v?vN4RX zcwky7dC5l7ln(MeQ^(zx?>7ML-Q=*+c~8Tu58?rvT$;Jh>;cs(i*k`bm42gKrdzF6 z6VT3ue(eqbogD*kwK}zG%V}_?qh#&eA-#@uCLFGXrCcgD;DAMCxx(eda}{~l(Uo@l z9uA9R1318n__h&9i{?pys3Em=>8uTkm zmZ6K^*P*sbU~aJ-qgQ}HVU=w4kLP<*K($*btG0cl+yxcDzy({2@t(Jk z1}N0))a%ecIc!B{2KTmeK(Id%|9dhhB905*;fKGN)kAa4iisnB+Id7se)$6nljAIN$0=r9crg~+!E!{rM z?b(8E8YS?@hsn^(8buQjNc&l8Lxm|k`R?gmI1tGpSz9ZV%xOOtRaHk;9YmjS;$-jy z2r}rFq-H#7>mZ9+w%F=o-#B(CuZ$_Yy=^pB+$9Ddots4R>>9`5+SX2^kobI{;?PF$ zgIy$v5DH8f9a)C=SRy}|B`rMz9MMir8#0|iXob&&t919{Jlg1blK#~Ovb!{ekKC`G zL&2A6Asz;rm?#qmV#J}r9;*ALL_gy}HB7ZxiO%>%J|~B_+l@#)v4*FUfK1{q=)yev z&%J&5YNOOI34v7sG!~cgK$lx5*GCtYD?I~Wp8NO@DJ*t-RAD^Lo%}#5Kr+Ov>b}5j z#!TT!RtFi}B%w^gs`sie&#%*+72XilmmV%`qLfHm%| zw6-dz%arLZx7yaPq>kqdhwkU4iHW5Db)r-V^HEX4+n58B7d8TGFZ%? zRF+K&zHA+#05*<*1L07ksu(YgU$L?=EWP(Xdi{cB#WRQ-w)PioAr*3mUaVk1 zEm62(fm{%X=hPcWq3pB*OxUo#7zsxhq8n|=PI-3G;~L)feZ`{O7QjuR&gr3EU*jQ# z;R?;dULq*@^YYPbIQ~RMQdmXXgr_0+;EExRA5B*!|5!r^G$FUQL;JgI=|v!KPOrWm zCKewGh>F|{^k8DU=HIKuzf6;D z_ZK;`fzFkMXj<@h7@x};znKv;iO^AK8OC3({48&OW5lMyvYeZ8?n{OPoytoK1c0K!&`V;r(1vojD9C^y zUs$-ON71XiYnOWlFyGZ_dX(G;ClYB6b(YDldmV>dsx8bQQXs}>5h$17tDp+|K=zbO zeQjyOo6nZHWNpT6Z#zk|@+24>t!coLKUOg0_Q6#FCDiTx<^9hjMS*nXCm+d+1+i-beBIcf{A$Pi%H*^PU4SYDhZT&wCT6FBuy0ho8J z$!vQtCn?eM!j!KB+^D{kRnQEZGETtnM(Tw^JYY{6A4DG}7g$t_s}t0I`SxY__8;}J zj(y5~%XAn7l5J6<`rdmPj6PpPzj9RDIDG_3iLImgkt?ii5pq4&2GL>LyiDe42|ZvG zaBTD~cj|7YtTQE@f%AK{JM_l|rUe-nKzL2DdvXfsf%SOb?cN2~n#djwDt6da>ZWB; zPUXpQ2=(2P9W8s9Z+eE)LrXXDbn9BBevgybMpI|UAJoT<8vPhVHIz#_9qpWq?i+gT z!aEZkx@z-mA&Tu9(Wx3IN2u0NZ!)IWoWznPA|3oC$!SYsDJF_eGp}l>?V)qP_lgO? zII^aL!l~SwYA?5$wDF|1AiwTXblSFgl4_B2{Si1VJEi3ku2ewMjNdMNgj9IZo`Bu| z`uZz=4S)H9|1GJte#N5eS5jUDQ(J$*uY_&g!}``LC9lSZ5^jd8cSF9cu?#2c zL7=FX1Z#0Quc&9GJ87k!!`)V`Y?oqs2GgbkiJ?Q|Oksku3}?rGNfowx)dS(2`XU?G z!qk)wj}`gfvlLQtva+feI%ubm&4rj*y6n-)nc0v$i6CSodtE9_=zg6Ufwvn+BR5oo zvN~0yFs(@?#ZPk!R3Kc*mJ@#nL$ra)2u-J02xIML6;~YTdB$Rl1ZHh=l!vmOJ&8lZ zj7wLW$6-~F^yWPPi0(zO4OoH{Q#K()qZlDhReu0ub?Q@uF=`Rov^L-WAK~q@WT9e; z758bYp8!1y*Ln6^UB^m&qe`2lO2m4SymL6zD9m1p;|&Ih@;MWG`?Or-x!l@X0-U+uYeD1T>vFH*JIzfBo18_XBB*unaop;RFXLtsWCyTimbmwv>bXujQ z(n69qF>ZSQY>Q1ehE?+=J(VE&tOfRq+N=RRU&J1>PPcU60@b28r!;gH)xnLk2826I0e?NTFGC6R;1(t7RMD3 z<43cy5Gl}{4;ImL;2LV6cX;yiT^0P&x*hjy?2h6^b&IOSGHDo7Ak+eGrqV+Tfr zt96;8`5*a}o)R|J`)3J?Uj_a0>-5S$L1Pw;DDeGc&ytA>wiSUE%KvAveD9e20YleB zXJ}(_IxdJ!4HGIvb=ulr)q_HI{j)vml8<0Y`sn&F7^Dt$og^!mx!%@Joix#8O^cCq zZ%D>L!D!F|`RB`JXaWrYpCHHQ>Ecj3l`z`}qSYqvuDm%??j_XLHI+g|!Ou2FMx($W zPI5+5S)i&Q~wb-Bh{D*dsg;o_Dw*)fb zX;cPo0O+B!w5Ik%q>_l$OfAKDswcGyL3M%1!ZRB`-}? zqERe00bdBQM*hGt#R60^XYEdH19w}1P$m;Yo0#qWggruOR#H17?hG*KV+(MH+(2l*^Z7CM9);!M?4 zrM586A0Y$nO#DM1PMApmdnEg%lofwj!CfqP_;XOU&g~J#=gSi2hlT0nzAXzL%9o7^ zc-ycDB~!s8J$@W?|FA#Z?iUu7< zGOIvKGWZ*+f%d2tZ`#=c?WywIg26nhJ)}LC^pisaKsRcJUW^2CQKS|0iZm0;m{G1q zZ;d}n!a=il-q*p~>g(N|fV>UbLrPC>LKD+%Dr?odr;hwkE2_7Ccqo;%2(l+l9s#9BwLC3WBL5TJ1@?c{%|o{}9EGUjR9&Q5fa zYnG3TT#Bouma+VY=W zEwx?_l^SsE9863bd3QK3Y8VL&67zbuUImV(@MNh{L5T%ghNa>%l@3V3LOGQ~gVPzH zY&3A@47DsFx>BfZ6oE3b7U}bV%?+bUjp9DypXDXU>Gr4L=C7DH{t6tt&t88VUVnxl z_ouf{!>P-C>~~2FpWsNtczO@fB<|$46*~-Y=>4wQfGoCWWUoBMA!}$;mq!w+u>PUs zA?PM#L`(w|^X7JxQoBzJ1ymrnsak~@nz^FGI%RaPk~*y=F6B4CXOjsAuPP_gOr%0Z z3WLX} z0t6ddxw5qnR}5AQp1eRo+xYAJ^o#KJJIJgGJ=If&aeEd+3 zzudY<>rz^^)6XzfunTkFu}9eB-W`Svo0ttIkW3assgf1Ymf>36G&Ntn1quKTwSk{M za8+7rg-Qc(SZbb6IC^N)n|rrs-4sbAq4QzE6`Bz%2j6q2NEQRZPa039#6%rJg3e?z zwX&<`NEXJvWogL~^u+YYOSeHgpq5o0fb#jEPM-3b^d0{kD7AtOW$Noy ztus@M_PVcy8|12B69w8F)Luv)0pSX&ML1aU>w^>fl!hUfn#bX&I&09 zAd!$oJ|Kk3>5$5M14Snfn{GQFjc}YXG`4&;dF!-s*8pKDGpyL70dq{YiWKw2;d?mg zS8)v#$C3bp!4zROh0?)PC=n)S+Sf|zf^1~9Aw%2aO`gW%15EDc9}L!!i2N=|C~LoE zsP!YIAC|229vC4a2cb$P3@L9wWZJY+E3d0XYEX)kMy#ruoq_0AWGZ2d$#(Kqr^PtW z)EulbSZ#(0yv1XyXHvsEJ%j*D#S#^ERqX4HPqL#o=DUxumbte(+`~Ynm2n4+Jv{+t zmxvA=)AXF|bzDl`Cnif0Z#QbQsZ5(yY)|O#vwsCV)-rj|YD=OeX^9?{8f4tm%7CB7 zqOg>aT*#f0JsSAtY;{JZV~tlqHJ zfZ7pi)u6VF7!A?$3KD|UBa#rlsofUm1(LBtSj)7Crmco_a3IT~4|I0NO0J_z4l3Q+ z6gihhk@&mS=D66Fn3UP|HWoWN#6`yz>=$gkevyVu4~kB12Kclmz{t4%#15h$wI`MG z4`Crjz^;~q4;QX%mF*nZwNo!7kRl`_u>HNl=-l!dN#AG$!lDdxwscXSJ<*kAg6S zpx)Pv$Rq37M%z)=PRKMpNnol9tnQAw0GiUYm@q(55Vo|D^0crZskiuSOSx9$xzvC| zp`G$fr=Z4g1*w-0<2j{%%3~-{-B4?DRROE4&UHgT+o?ZZ;_@=%STKsNCU1|`COld6 znBIG+rPiOB=JVdJOK+E=5986MOW49ta9Wtj9t=;*t`QI@zc|IV~;*bjISNR$+8{F z$;vWs?*a$3Uvvyi8SFJh!&@I6)~bT8CgsNC$p6nr8nE^z*5{My)T0+)eH- z6&s%vyJX!F$;@W-)vlM0sM;abE_I+23%j| z+FY){a#02b$i2lUZScgIx#*|fVNrt%2+3J=kG%~Y!M>6|ZazY$3jkUMt_uuT5IHzW z*#i+Us7sn!$=|(Sw_t4E$A@tci{w~eNaB>$ZRMHL>$w9q6{<+{A!@E-%N-i;l9~M={HIg>`t{qdgZ7Yrc{aGg31`_&RD*(B#|3hSmQZ>W;#GSs z(}Rr`c*q({VhC+cY918hLcPkP2(!$Qwe=QnlI5qIew8SL)Rs^m-4wsHYr!|2Vfn+haG zsrFmyC1!6M@aD2_aq>*bri9p@Bl!Dr%$lmp(1%g^&r+v5by#UWhI_r5>g(h_S<}g- z4Ipc*L0FOjxJ`cBzOtdA4HKum?Lb5jAq_ZQivG92lJjX&Exr}*j*`JNAXZz{j`zBP zQ7o+6x_7GbmSm>)oBvGp&*2mf_HCBS$OT;htsO=K);5rs>CG~<0prKlNK+%d+v(pY zMdEk_v)iymyd1C-yOD7KfKPRl98ZH72-RhPd|ihWk|(Y{+Bh_|E>9j-)eh^Vk&n3= zDwQ6!IF2Kk-NeN|V3~kp4B==b+d|#_s4L*pKAUxO)hdbCi6$4#HB8(>Una1vl^C4S?f-9B8l3CTG)$8 zjxy(|(f{zSB|u>eA>y;D^&KX;Ji^PhwJsg z7bi@kR8TUqMS9g&ii6VNFba`FjCNLihxg9EC-?0RtMz0fnB{oCWB-Nk_ zFDOqg=eB!WV%Vvqqe8gjwG9L5is%?=>{jBK3Yc z6}Xj;ZXuzSAJlLVVcQ50S38zA09yca?ksAnvAISR>cyZI(OQnp%PXsaYJwfi#v|nJ z?IlzUlvCa3#1*xt3;Gw`nxOEIr5?yMjVGL+RL9DC%0f!A_o5^~ckUl3L;_Zno(^H= z(HDW{X1O0#-tGh4Y8N_9;Jg={7za2+ro?ct&!vhpTpA|n8c_@!b_(trWd@-MLs-cs z=V6OBC}t&#)u*W|x=JZTOiJ%zEm>JqYG@1(xsmBuVFe;8B9-+^GNs)(UwSUCd4 zIrs_Nss%aV@GKq{OyFi)(dQfs=s7A9r1D;1J?ajm6 zIDuWrVnUC%1#qct-8ni!YfACC=~)r34^-V#o@(?eTjB5-?6%#3N2$i9Im{OPD)nJ0 zKr)dyRAw^>BGoYOGZe+X`XEs$O?H5?tk@fk&z>|0uPAK^)Zf?cvg?cqV0N zwJQu$0*B24WF6YN?^bo$%^Z?@-A@X)2A01|w$N^$jXPTKL&4k#@X8`X3<0#hMSs|| zlI=%X+IFO@EhoLnU2O4K91CTaf-h90RsZYo`l%cTCq0h`oa6>@v>pk$c;6Eg?gD{; zih!2j9f@rkF`4TYx?_xL4Qb5Yvijz?{|jkc9*kSm_xOhZ3%M1pi5rqt0!UYgb74Ujc|`D+o+zlL>=-q>~14 zd_%p$o>Z1OR?}W=xlf7dz+w*8fHf-RXIQP76td?W@k!0cAzjS;M=$pRZ3eH5e! zJrL^(jqpHsc}t+`4=P&L&+1dFUSCFzi@1Q7*r0sxrEaxpu?uOa75MMnL)jmP^ON*9 z|0TSsuFcPb!NFf7d2GTqAU63QH5=x(GLDD-QuXTC^*p92>S`q5U^rhH}hgzniS9+s@iU z{TD~B{HG@<>oaEp$J6NNT^HIx8?AW91A({GcthQ7A+eFgxfX})pFK1%F2c;x-7qD% zOFeHDLMfj_mh+Y6!2|`NftwPtOU~^qlndrY9ZR=usP<)7+HqPy(SQ?2_%MbmVcJN% zSWIgZ(KT3;4vR^{{49MX3G1PHd<2W3?%)o*12Qs63YL@k4!z79InOQqLM`jSP~x2h zW(14LwC)h5CnCGhh~6btQ-|M`FNWgjvSwj+P!t(H7(LM4-_XBeI<9$Rr8c0J&&=(C z*0HT5TxEiCXQ}L;opvw{lLJo136Lo%CT$1|NwjKO3=lhk%L4c``_s1_)F!J&(Y~@r z;r6H{Vi$5X9+wUQ2rlF%;59&$ObQjCX=J_nzzV0%#zxVCDye4RZsBVY?TBga_Inx( z-JC-M{LZ>5ayEKC?xY+E--*a<{r75VI0+sj5VMNur?!QHDIz6*K?cZ5 z6Ui@TGt83p-mN(Wn;XQKGNaE5c-}$mXz3h9gZ9Bjk{o>e>_;O>+1P50J$^P2ia8Pd zP$2k^jG4(DG(G;&>%U;?kuJQHks~ErkPknQ`_|T!g=7%Uv8IOUw)xwd^G{me&nS!K zy>?#9ZEkpk**Zlu;!VBO?O%n#x;ZKt?Q`R(>dUpm#uZ6Jc+JZTmn|*<`I;I+B&Cr; zQ}U`6PylNA0v)n*p)Uj`BFP@mbxO}lPYx%ys2qWRsJar?8lZnFo&oh+fDI;l60$!U z4vlZvvXTwj34fDHceY1~;x9p-IWu^T|7hVXorU{9Dv50N3|Xu0Mum7iQ;+^R=49T6lA zxP4%qB}c=%y^{<8?`Lh5LAeotmpTFq#qE~V%_m(Fi1N!??BQ9~W7)2%U_KHoIkIi2 zI7txWKhQI!2$!U&>>+{+sqEM3&cRZNuHxvqyoBJVZoG^o6BM^5huo3_!$8KYdkJCy zhh~bS3J)k2hxMQQAcN>m@N&@rrq2OZ7|`K2abHl&QX9#cTAij-ws5^0Ez53(T3zZK zN?l@Iq~-&GQ+1$gV`W*uHMHq)tkWE=!g2_m`gy{&Y zj=s)8TVk1^BI|H!$rt;mLsXmhLD#XGHlV-Za<(A9{S>^G`9%DVpGRkW=y|t=7 zff55;GW>6Vlq3NzEoE;t%*s`guzPn^X*SHH^eoxUJ3>JXgOq8 zg)6!)!2sa8n>zNg{2tyw>K$cslX|3(K+0Wuo49vw?QXsBr^(yGs;W@{i|3F!$@7e0 z-bw_ncuq8%iR8dUqq4#ez3``t`V2P^31hzkWS~1$E+m%K%SoTNM9o$n@-?b}bd!BG z_GO3=(;%B{(%@P43bA0dtu}UWv4f!r`{GM(kvBRpW5*ROZ-kr05VxGgxx<2X^T-L2 zPI4f+4-DZ`AV5cGtxHKNqnd}<%EeNQ`%OoCT&+QF8SHp`brR*HEOfL9s+JS{>q$Ge zhNrdMB?-C%db^Yh0@1Yg$G}8eEf(msH|*dN3^k1CE|7C>c5|^EN(?^lH90<8>q@^` zQk4+yH-PO___6;amyCGU&KrIT5$ zO6_Qfv0H4Flp*9&PuFk$RPtu$T7DwSk!Llp(nMEf21@&omSz>J^(acUDNx6X3a7pcRU z!6N;TbPoPQI~VD|_1YnD1-E}z8zPQ-!7L{h7l09YUzc8e$|W=P>O{pd?NC?0@a()2 z>gP7@7x@6TG;39|7#*W1$E{2IHYmXcH$o1TS!(Sg@CD8vkd&b4P`?GpE7qKkuoDX+ z)+AM;qD+9tNUIa(^+aGvHF}^m-t}P3E;g&Kpd{~YWcjcIbJ}4lmk$g}R3O3hkdw-% zBo$ByWk<4@j#B45>k=(RDFDB|E$p>b^8ausYuFl;q|yNK-^VQeqqjer9m{vZ556P+ zmsj_1UVp~G{0+R~-hO4++`t@n^g`;P6Jhcv6=%El)&ROI^G8B5ybyQUo2FnZ6=RnJ z2vCkESShA1N?GuR*g*B_-90Qf70{6UkA<6ZeeH35y@vWs{0*{{FUn11zZ&*hFsHCH z05Y-gp@GcpgVp^?SpsyzMW>Q59Re0Kb)=99aTrZ_nv0v_1)U~9gmj3Kv0_whk zfiY=&rtko>^$*a$P%N;fZODpxzbr{$ zCoU?NxZf@jE%#qtz8AieH(@rc_|soHSR|tSOSqw7(vtiEKEwo79jI z%<%@>io9$bsSn$#0ON1T*&8TFWu{Yl86|6<1Uc!H)Il7ht#2I@Cv=)lxEJ-HhiHAs z;?hdMmnFOLsRX5t+WVvYM5#=mtUDkP9Jp*{Ij16|cY*jD_!-7&)X{azhed7Zw5LfK zQG_`n5pL~qX-WhV9xMrImGN)O8Fa}+FtVjQCIb|nNgyCFds-&$JBWIs`k~UP+L`8;8R`++tSxV+)Db?+~%J zH{ueg<)*mX#}p@aB`r!4VI`|v7FD$>6okta(QA;Fbty6D*_NmTevazVnd+uZlcWh| z#LSWmrwXlqvJunx(UtP95IIfj?0!<(s?alDbD$;i98>^!rcZJLTLVOXwx#LQ2Yg|S z9l@W>;BZt_nD=a%Nr#4=q3eY<2W`X*v}awqA&-|Apx<@!BcHtMLBs9}Th)?^)zG3} zn7_c5W*P|40?R`_$?Vc8QoY-{Q9L__-81yPGmvFO3#CKJQPmDOgFnpXC$wz&R%HU< zkApdpbtNbIDfTE&vpCh*Wt!UE>{? z&ZCJ za{Vzp%pzEJw48~k%0s*pcMm(&Ow`6ei2?<1fQEBwDgVkc9wc`$*V;`RYP8x1HqzS& zrTC44LDPb3Jk8dQYP>wScF$n`k_mK$o{LADYt&)?C0TSKznQGBPqs{$q05O`k2s3cXQ7&~G3m7_W8NrP%#+H|^IjTU^eVO*zp|_L!7R)F(YVf@a z^pko(f(=|7D9SR7gq`)1j+JWKn(C;)9s_As6U!x+{usgb-e=VKQiMLq&EE)2AG4L% z6S{HyiJDFW82N@%iWonA@#B#`97^qp1Xz<|9>KKGwS zjQy7Qg8GzXR<&NR)Kq)*Bo<~xAGt=}!4+CvWgsC? z%4y<$+QaqX5pP?8|K1>`M(HfMWQ>C=WXh;V)Jighm8qmBo=kHxqRS)A0mdqwi7d|m z?%arZm1}@JpzPdfw9jxishHBg8R5jeNm4P^c;o7ipb{6gPc^0LGPN>bn@+yFuBj{4 z;siccT7zhGrbivz2cc$7yRN8yAspOK*YMn#sxZta29AsZX%~f@X_Cm_LTSgXEGaha zVdmh-5UV1K%Tah+XGPA07<|!qv@pl{J`b&+9|}+o)p1H`}K=!7S$uj z+7kr2B%rg(0q1Uv9+)3CM4Bk-ilKr!J%)nJyXt|Ct;@eXyq8yiuOaV>83cyW@d}5p zQxp!5ya@pAJ@s`9rI~c-jRwcp6bfa_ke9P&+xO(Z@V)Puanb9SbSnDj+rNMP7sCer z=M=R8Gponc4Rxy%D3y&El+M6UWE;D-Zgd`PTL8sYD0aJVtE81SySUPF0wR8D!pm|L z)L%aA&r`Tn6cC<=jf1Z>!UXTGx3!h}A`RTBUu66RGPsD;WG!3Mb?ss%ntZnFR zxZ#k+B^kK7#tIW5Kx-WE=0Q{o;} zj3s>=MwhuAz(SY$h9P!Q!eRPhhRz0{0|8r&+Gu}Ub)gw4)&YG&3O&c5o7+(%41<~zFGV!*~RtOnSCkk4rhg3G`DUx`u|+%wXSym- znh6k8;VVEnENx3v1Ed4?z(U~LpTg@O(z;VAn*(iR_B2|R)L%ixr#;&jXI&!yO43 zCFVljnkb>w&jAtK2kh^KXw7rgC2AmrRdJY|EX1!iltCWK9v`HT`9-V=nsk>JGv(Msw;@7Qm9SYkMYv$!)6!~ty%;3wM7@erhBp$bN)FTX~1>aljA9nOJ zoV!11?qQT0Ioo)&A#HA%FAh>nWqGd1^c}_y1QGa0MvQf&R2qAdG2P__mp@0B+$=YQsrkXakcfQ?}vjGEd#eCA_E`_D)8a{MyP2G!SVz zkMdb+qE^B6o`HVpq>gk6O^MP2L`pbbM2nUIxrJIQ(A*OL4k!qL0K#bqW{DMK4j_?& zBRQ?u1wG8OD|AGxawb^?dxCQ~|M}}zY+?TN{ZyNoUIB##T~hTS2VI@z|1(s|hPGRr<;;a8OT9h`Iqhc^+rbUnfI~cXj_a4*)7E7#(CP*QV9guB1QtOCpp&gUEPG(JLRTlx z&GKePMX_hIH`$BoPh#t44mZ-&(78r6qUeH^dj`XF`?vCL<(FLhy`u>)l8UaJ)VG#= zODIWlnErtApyeY{D6X>k$)Bk=Oxij7>y|7t+)n-WBA>-(Wq`*3f-K>-zS;&s;ci+c zH9#-psdCZJfN`z5YpFPM?K%ar7a;1msG@iq4u?Gbka}gn-qrl*!@DcA_g6eK^LC@(7lD)ZR_3%j9L)GNt z9CC2wy%gJrY~FcAE|H}tpfSnG)?&KmwS)tt+xO%GnUU~r$eu0c2XSRW>Au(l?-b7X@7Z?`T@2<%%Sdo z!5UYO^vV0VL+x&q$oG$4_kq12@7QD=(I!fdrqt*x{W$Dl$1Pckr6fHXAk?@8p)|gb zDFpkY^f2rz9Xb6U>iC{V3KsRm(LI1kNw1<#JBWZ#_zC;a16cFB&hwU9#q*i=)tQ)* z8YaV~$}S^vGXa?*Evm!}%DR%0SuGPL#Mwio#583pC+>iZudHqRvORi!s;3aOC7a*b|>r%NnkGhhh z)&=+h5qPnO^1;}y*(H^gL}YjJ)wW+{OFE&=<(?E5iCtr0-kC4AeYg!_9S}g#*5M+)aLpl32ruKDwW+;u& zcX`f-Ls;xxXV>$~Sd3Em`9)< zx|f&U8;Z0XtueP;VCv{rNm$zxr?NF{ghzfPC_P$9-Mfc*RfB@hvZ9A8>fYl~rbw$# zGQAsBmls+h-pWz4k`;=lp06c;UIGK!72!uB$}!tb`2Kf;X8FJMJs`EhpFix*y#McS zABFe-=j)&4!H+NR|6BO>AEajP-OC-?Ee@$j;Fe*B&mJBQn)miAU3TwN4?Od9x#Mtm zN7TCx?+j}^0*f`PsJ>z-glE(<89kOP4s45QR#{b}B^Ed)bsVC*u#Co6V8_aGRSh6vF`jTL;J zHs@XtpKr>QXx}}4y?*29xauA#=Q?9V z)1aEI>Ufp1xMPi_7P>BH3FL9-e5zV`c3n>{dr_~L3Ui#RLsfem<+pM!`*=Ju^)PPX zHa*BaL6ya81dpw!lK{quw@1Dji+}VZ$Lrt^myX zVY32nZbvqpY^AuGm`~y~uv~S)Z{N@|uV`7VfX#LqI&3=lImA&77%fwV(reL8?*0om zcn_y+yPy+QPSjf}Anj*cVb|TV()R=8tn6wGA&S^c*gtB3$yX6k*3}K{N}mfO4|Iw_ zG7lW1z0n(Ru-4Fk*CPosja*O6xM?QQs^Dg+lHQg^Di7w-)@Pq<8aP}d4DWgu=x74P zMI{F;YD?NxyqkRq_Xh${GrBt#6y{H2X(i@Z-iIqBdKOt1zlcd^G(E{kF^w$jXTAMv zTCQLTev}$jpI%-j?-agB;<(%K6m_7pW%uYifLSNI*ID+8l_Ohy%lybqGJigD`yDws zR-JPYTmTpWXUDRjlJT-d^4UcnwOImVxOxGJ`3ezKqpOUNrU5%O%1R6rsU4>E_JMdX z@(n7f4<$<`R2frA?RdcCP{Mkhs}Py`#9%4n9SAYY`TB@%LF=+4H{AqMPu>(#LP>ae z$3}0>0^3QQzw$`i*!AudsIn6iwz!IJinlH#BgICbRqz$$VgN)rvUpb0H$63MUCE-R zaT0U|X=2dHPDI?!Ls-ZFr!F~RH7z8M!dhKQNm8>7TnTAwR|rPCIXX{2O=njlLE9?b zp3Z_`Ey^qE0|#rdsfg8)E`$p2cnm&&ny8z>7WI-v-r}_f^nASh@Q2}tHoiqq>sP6@ zmD*n~d$76G3v6c(TdSCz#X|!0HT`z&%I?bhL=~cX#Oo~YIm0e#VyM1B#Kp+nO9C3cEDK^k&EYhkpIG0WXpvo~Fu)uvG`q=dJYwG+w(tro~yMht}D6Ba2Vge^%G z(1f9ZMDA;^rWtbo%_QYXdHlVX%So)aia95cBX?ibF%twdxwZ`%)_}bYb!2@kf1y4# zjoX^sKv24B`qQbrjp_j|%sQ`tY-90e>)8+dit2`Q<<`f6tW|YTMZWYdRdVwD@TM# zMi5QB#9fhAx|0pI4z)=nISyp0%Ka@uMwKjeH@dTp!cg3X0!ElAp=@%MDhJvU^>zO+tpm1Lj%F_8KE512*tucazDaYw+mB5CV#7gM_SQ>51hK^GUEw?$P`G@Hc;BAh%E6{*3X)CvRUF3G$Ad!+`+%Dw!t# z89+g6*I%F;m|pOX-g;jp=nM{&TGb@Q4`hUF$Pi|(*xK(OI2Npu8V{VuNhwc>jAaN9 z8_q?_%%(F|idY~-7P!jKEMws*%Z|>hc6j}L65f2Wy*Xd$R$S;s`*&`y1kd`a$f1SE5sYuHheeZGvaY%^mMTzav<~(7Lf)3StoHQArdgWGGipPL!k0xkz*Y_)^veM%?lP zHs}8MDH7rhG(np&#FpNoTBV$-P3^j7vXmq;Kb2GB|JOgm`HB9P*UwZq#CD4s^8i_y zo}m)pXOuJLb_B2Z@AS2mfO|t-&`ftKGXN8`Y4s_Y?~qoaH;aw~!l<(I*r{LtAI9FS zNtWa~6MN5J;TV}@Lo*S4AI^w>L^8-@$%x2|?Z%Q>Q4cZ=Aw5YCdL-Y-?nW=f)>sKN z0K%-g@V|7vf4~!*DAit8U#~+>f8-Tkhy#g6HbT#TcMjJ=(DkXM&KRL^NDm zo>WMk)OA#n&?*-a@o|F2CGOu0w~~!Xy1GjPkNg4}8elbPOW^>xOxShXq@SW4DrLFP znoIRb5;;8@qz=A}5I5>8c&|r?PXeyyV-B@~?$hpwR-*LN!x@~ARAtons{jmwK zG-UlhVb#P=Z4D4i=liLu;fI-aM3QZg;;qQXx5`abr~!E9LNEvdWp_<(qnyAWY->DH zJjGVU!aV99DzdBdX5DT~wEXB?;o&_u_>*6XOiw1OGWSlO+S!4 z^!SR`mIAnxUw70}ly|y=SmG!FO>a~gkVcA5dKr+Fy!*`e(ydwXO=^fi z;k_h$)hC#kWV9EvY48YF#hlZ~ zCtP@|buA&jkbSK|GOziuVXj~-kI4o8mO9}g@QW@3x8=+#-E1?V2FCF;F1YAG#dIi9A*4+42Jp7D^?{)%FZ z?g+hPrxFJt*6g!q2~s`&OLq!%w%x&J^6fwT<%i$? z+8mr;K}}@tVdzt37^ya)8g!%NrAq$2?1P*-wU_GZUl=d4V=bq`SO+%r?tU=F`#vkQ zdm{;;9^@QYA@B+bd(U*R<1lsYZL=y1#rQGt2$h2~B$v1YCqs!erQIKAccNRkrKi)4 zNTHaVwD5WFD&u;#+V-{YbUx7pFpxt@IV3v=q+h{hadgH5I4%2*f`Dar*_#56^Q3g3 zqzms^f9+Oqs6eyH7y2u7-%)`89yZgmWc^RfR$s#{s){vEu_O1&^n-C=4_Y8guae!l!M8M zGGHP6Xxk>lZU9k0uD?ZRXqdGPZ_9o=;9r*Mr>jc0bMOnyunjuad+^+GP2w~(|CKMcQwKNIzle+X2~zo+)XhT*(G{}Z-yf!z+$O5NkR`( z3s(7QgB{5+BBfpe9ksx^N>p`Ro;IlJ1)vH#|K21ZM=1ipf?u)(8(#innwFMR)egLq z5(${QVRqwHl?XgRL0c9LS^5u-qt8j&qflhP$zi^k`YQ2hw_1(9>rA4RfD0hEY%U2K z+z&8BA!=6hhP?d150hiSG+jUsgLb0sjdGefxmL+fsg@ClCUrbwnGjA`n+h;GE##Q7 zsbYbFYGV`VGNQhQksc-`W-xLl+45f?P62@U5bn|N#%S3NJe#2Nqhv2js}5PcL{mnL zf>a@H?8vahey30SbvP}gQvi$m=O z!65l07gzj*y)kSvoZ{1j+$!C?c@9X_6au zQp=vS&mwXjdAaE)zW?t~MPIXc{o1%Uhva>`BsD;fTKH0IssiEUY3?IOc8@Mw)GLcT zLwx8E=&MOWUUO0JNVfo-Rh)z{Vqmx&%mboL!RHJ(f#R-gMM?29U)I9$V~K2(>#+sC zr4=%$h250}?(~*IFO=#2z!U%gf~(Zvo3}oZJ1k&~nO+{Wd53O{X{DQ&tI;o+T-Hlk6p`tk%T-Wlw8a7*M8FjlHjg+%=2pOp`oI*?UNSEfN={ zDoRc`Jkuj?V5f>5&dMvoQwrWWLTe2P^WF9Z1|*JDivVV(C-67nf680ooA=+b(UZ1< zd6S=Ex{#$J%AKE)O^%pu+;oJ^*r_&=+mdU0=YC4Qnf{4+44|>x~ zj{%`_Q$8#b2~|&r-b^R^V!ZG|m6K?bQ=l}q%rHJ3{f8s_DnBhr%q+(;(khSI5=PxN zTPImB#@L9cxv#ECi=u)G#wd3z33+wM>#L-sR)@;FEoHY>A)DjO)QOF~I!R@AbpwIo zjT0RKm>faDC!082H!C`iFh;U!IJ1?iw5)Tsw}OjGz8O;F$%2>EfS~@e#o3sQZTCph zXOMxgo@hz-;k--n_z@UjPJ)3a0YHh-uAYH}D=nwAjJNhf538_eF&%7#V%?p~JhVWq zt_8Y>1ns_?ygCAfz}jVD1Y>rUdxY7Ob` zlZX{H|Bhr&w>>l`Ya94xZ6~K)2!A`0TWxbBe6dUn_1!mxqRr?m8~5hpMsbATHdzC4 z@j-AbI-U60O@Pe!<*&|q%bK?QDbTMWF*3gjcJXzR!vC@2)BZWZ)V}&}K!JYsZs0X_ zp-v%Q3Ni^+P*+7oWuX~r6zVZ~XLcva)`pn@JCndy!v2_y$r0T$D%v2D`&1ES!qEpP zx%j>8=eBcxnJl2OM&fU5EZLAmp?=s^UVO4HkNVk{I_k5n09OUW@=Pc&D0F$pG{xfk zzV-4?q;zuZ;0R?&%GlMtYucm4>}3Zlrap6EHfN&tDxhLXeGQ%r6^mt=0CWa1O;D-o z(ur3+`dNqRMwsgKaJl6i zAOTknO2zJVqDYvJI5%|@3DLerz?CHw=-z72Jmf2VEMffaVm&HYLIYv!uc7E-2Ue!q zz`>q?`hgP3K42cO*r1@m30y&7L!NMt-YBz{L{>`Bj)kHoTYyg^i^ zDpnBV)QA5|_?xsTegYGdl;wQ;MR@<|&(kmKd2e?iBUf-AQW>I&g!hO_Yi(f;1C0%6|$LjQly8&SzKU?EFAm}<-M*`Tin)q zu4DSF8-z6FkeAl%2Pe4zH;o1=0Mllm7MoPjw$%3p+!r||VN+a$7VBMJ=iOJi)dtZK zQa6Q-pK#3(D#+HOCK#4q{pftEdc^ADB3v>QAN2Wp`M1XCyV%M8_Vkwnj!l2|D? zRge)z=2)GSLbYy^F~D)sW@LyO26AYn;*pBQwr~MgA~b+vc(K52D8;J1#VfM>`k^X4Zk1owVUp-fN+={6+ODU=3xw#s;Pv%BH z5$wJuk`U@gNm6m(8G_@FBtYz!dxusJ1emNl2#jI$8mKA>OHx=QxB@DsAE8RO*O+9A)x{xv<%e-@5 z?>b=yYv;s=-POldRsTzui4tQlfA%}{XWg4Y!J8a6q=%%;YIZq!@;%(_FR9a*cGLtQ z94kr~jo6}2X%4|;d%vM|x;TG-h?Ne3R$D!Ae^Q|c3T8H0B_H6M284%k(-y5oba|U? z^a|Y%#4;)SNu=&J6=1T!4QQB3rH~9Ts>nVLGL+V78+D5$X>CEnpcr1QD7b>FyO1c~ z13R8`_^Zu3(kAQYR;BGY>#F5;RoTJLfjxmLub5a+Dja=WdB5Piwo*4}su4o91;o)R zcpNUcf?gLNZ%FtDsE3rnhosyb!NJ;QtuN6<)g@g2&a8qz3-7U>ivBlf!WBtq*^VXL8(CB@PWfCQo^A zXNR>!qjO-pK%U0_qVGj!d7(JN?k<*cZy(6jKL??d;8N**GxD@s4|;@)`Db%dB1# z0b1#CR_Al(Uvqi<{^$4Kzx^?!3w73R_?El|>b-ADEC6N7jtI~)=>fYeQPq-iBU855 zIN__Sr>HPXv-kO9(kyh`X}Aey7+Y)Wc(5ZW;j~bLXN~|`r%6H)oQ-|pQY;gyrMzZ4 zXqoYrMZ%2{NI;S!9b}H&6mo2ADh_XD{dk~LWE9z}S~L!vIiWIe;rcyP4n}y?DTPfE zoJ9>mM4~vA`I8wUikg(S6zE}-umx|by0rU{Jqab0q>b1jt*|BvEdRi(%3A~f=Q(0FRs1yjLr<(0X`iQ-Gp{oTAI$? zNoO9Sq8-Wpat{LmZ|*2EdQA9YNOikTTq$S3me=XmZ(ndC`&(O*(MBaljk@3|;o1)W zXHt<|xBW_d=B_%|PpoN6yPr-tdtl98cBf7Qu4~2a1ha9x0{PG^b?V%mMiOQ~`F3TW z5DvMjtK=!^m9qi7zp;avY4(a>sVqm>;1#ZNxMMpgS92#v5e-)6oPZx*rm=^dR*fOm z@Ey(6QIQwp9R;i{pJ55+4vWcax6mn$(nm;=ymzD`$<8J1q4E%Xaom9?v1(LV{+7Me zFK8mEUZXt#lK0Zh)HNAh_4qc)-!z9gv?hz`s2lRHwp?N+l5W}WDX>eqI+YRJNVva1 zbxZy&w@St|NHd@*IG-v!Ra6leiF)?XFCWZ!Np$&DB}AcZG4y?X!AjTNQzcj3a);r#7;SO)Oi+F^tQ8;4vDypxtNNMD~sfzOXnm^ zh1Q38Ls~Sm6zM9FX)wi67~Pi(mp(ieYl$nvE^pJd`(TOcu&m(oRd$>Win+uSgB_0% zt?I*DLIX7gdif1N0&J6GY@Wx7p9U%qB@%XvB`6^J`UF72MzTT@G_C4D|L3(+&t}^< zH#EkK*S@WoG1Ut3ze`D=Bov}QnqbCSO>(n;Mml1Fu2t~-)6sC63^P9B&6+bU1{{%{ zrdf&FWjAUx?r9ZN%p4SJsMVkt& z!(S2?$l9=w`J(LP7Vb82+ms{Dy(q`blEGwXPqi*X>Y#T>i%M>y&>!u}4ITYGXQ^!^ zu9ZFYjq!kQJoKE7i`pZ~&g>c9$KZ6@c?8O)UM`LHYK@+bZRyF1RLJPncptH>vGCP> z$<`SFm1-607`8e(qgS~oX+=vBM>tuxV$L-x9R<^um9o!p&u(#OAcnh+l@z<(oQK>6 zTFLMebC#tp!6&=>MIpw zJ*Sfl%M)*MmM~U`Jt)X(bw%*yQ<=zVt*SDVg^@=WZWqZ!Z?c?TEJFu71*}n9TQRso z5c-0|OK&Pkw^j`?X^Oos^ZCAZy}F=eQRsRDsMQ^>MpZgqjvcTGrsL3Z$?q+kK}W<= zHYc&t54zySEzW_ekBJpnxxIMN`FII*2{qfX=3|=Q7$$yFR}L}&7fJCSQbkY>8mFv_ zLrWyulb%hR4Mhg?lC+yfr?gk%D|$IRPD(K9x@koWtS}QRTTtI@;Q4Z@s`Ll zhFV*wQhM+~WJjz~Wl?K120DP&TXGyZ`v>V20+TiBe@Y4H|Lgsm@KJF>g9f%iGMvG6{Yp}8n>WrI>Q^k#{=mkMKw4pvKAx@Vt>ZbhJB zmef)N1CDl71A+~Oca3x8p+*22De6jrn8KgZ2imhjiO|QxL1wwg^BnwPz$PdUVN4 z*4BFcNNVWG!`Wg#r?!ArP6U@-16_V5lz^2ra5zxXo{g)UC@z8BoIWcumKd;4$q@+1 zn%+}nXuq)-#w37X1@-;fK{y<0`QeY`b)y_uM_eY19Eo54SlrN^fb{0=UpNx@d4BY( zw{Pt0@4kB<{=a_UyZ7n0X7UR06hD0X+@{_SZGY4a zI3g;MC~^MA3i??pzY`AZm8#ml*bV|LT;4I@;;2arvSiCGgR-}0A7+LBPra=qvOtEJ zG7xQS-Sn*pROFR{e5ucFZ#}qt6BN|0^1{%d%hO5>=|iICrA@a%sAP!Li9oJsed)QL zAlD0$OkWd=1iQ1eH9coml$7nt`jG`kccrA;Ce^v~BwyjFK{7wU?DfVI2t&ze%?b_u zfo=$T97!R5X5!o?h^q2$cMvnjFu}UB+EfO1+HU*)Be5dDOe5h)vaC@jNF7^~w+?VM zpUvW7Ox=vMZ$N-XwH}fPCmyASG0rgfknYDdS$aR4|=4(o@z4&>;)3jp2_?-%fdFd9?qQK;mAipRou z!R)Qh%$8Z+wJkF0&S0$DYKQbCCTD)c4*Q)N!vkU?M+(5TIZ6iXfC2g1l5EMjNuhqF zvJ_P7TvQb=IIFjxAd0|#Y$%zt_exo}UGb_eY`{{iK6}G#L&bNkwFMmlTl5G&oojB& z*|d*v?!_q5s%M~~nw_O{+yR%)@<-Sf_zb7VsZEQmGP_6lRmIjB0%-LCo5XICqPZZt z8u$Ze9-v+X9@Fk5vBcg{Xe3aiEoWx(yZhQYNi#&&qLeWErl zU?y;SQhqxjL8FQ?8rMY=eIlCyS>d9PPqxf+eDHTu!_jrBbS#`h(>hJzwsUMSgv~mo z+HmhmSK*Yt zxq>pg%Sx`8=3gr{IR3321DDn0oG^kc~-#n5yh)Z1siswqWecDf=bfG@=_M^ z(MnOT70KwmHeu#E5$AIC7f+x6}dXMgR4{O$xM%12EL5RHrm*J=QXr0e%8q6lJ>;TrmPL zZsmK}TtqQLm|;;yM~u9W>X91jHH?~>T;5%${0HEC4v=Y-|EWxvwc~JM0MAa2U_UoYRgQBg5`^8Y9knlT7n&_( zEBipJk^{I#qV)^N7TO*I7tGdg(AW-TA+THSU`Qg%A^c={QGM$E^pkMoM&-#3m07)(PEal4&Az z>NYYBCI!4D0Y5DeAPu#|oFm;1G@1*#dT0ix2E5REctIP*sEkvB`Q|471aU-D8LHZN z$B=QOe%Xr=~n+kL>fWKg|`j>Bv8;@~bj=kQ5F;Z;0rD-{6a zx`vaBnYU=FTclsq$-!z~MILdifJ0x~4%6VolqQL<%V{u#dS%*!H{7QK*K2s;WZ^n3 z<=#6SywW~px&(ZvOh3U={6JS4eO!pmuojRt0+iGeDXan*x>fjvMt_9kY-NoX(09si zmmmo43MIfUcDA%O>gMF zKZNf-@ZJ0Tx4wEaYs#TcrTINp7itsofx1z?L=Vrh^EO>50SIc9E2IW)TwD?;5%dN~EY1BpV}f8kme4Qc4S- z=>25vfl+;=+T$%8o*`+I*>;KgbV{OZk*d40uZP6Q>*ac#F+gO=j6G#}R2ZI@By~i9 zqMSSO6v;a{DQh<2SdlnkI%qRCsTV2?&K#BZ5{Ph`!H#F#N zA>!JGX%^LF<$eI^Gt=ysS*Ju?8fC;-y^swo&r6O&n`ZACJ6p9`>#1xib@;tjq?qi} zwEFi-DUy~R#1Fx_n#buSDpLAVKl`tl^-?zQ8cms zHA$~zXTbO0s>UL{1b+DT8-9XMK+Q0{#rWh^D{mFp&U|E9rJ6`hXdBv5b4Y>@=}0T1 zfyySBdm4{~^@J>^VIfyihtr+p%>Wvv8#{wfJhQdNTrKkLLMPjkkrnu85bvz;A2C;3xC(yX}m4?7{K#j!!QqNCa$zq~j!>uEg= zy?MB_;XVVbLDgV?=X5mV7?wK(1uT109&W_(r4G@)wQ&BhJ3MR4OJ$IT&T=D3`^M3i z;|fNKGgM=a^{dL8*%sP3KY-^=soKdpJ!iOr`ZZe*=xE*L3+8RMXC6w1sZPih&&4JH zd9%!mw;!&u)s>AwqQ`()Yo7kgA=A_!CC)=EPQD#3Omk1Jwyt%h=u*2$V>8{o<4&=Q7r0ee}EbX{~zq=Xb z*aMlQ3&_lfrd@^KSNGSfc}as#$zzGc5dZ?xv(*!Q5GDB$@e@d}gncfPqIw5lbbyR=jc#iG)#t;3&1eCt|AO=5iIW-+*Dz zIDe-UCb%e6z6Dzmq2V5~Db;@p|Isi9|M2z&&^n(bhsCWkO@+sF;u@e}^r-#N-FJC} zhr@2lP{#@am-UuVmvT*XohsKD?W11db!0oB$K-UDy>IP{@$fn77&r7;xwSaqjijK#%a`Y?}<;A+NfGs1K!ncq2Y-79!Q+nO!TA&wA{vg-ay_f^=q9sv{uW#a*AW3H+dvI#v*Odh6= zXUNbs&X%YaNxD=8&~Bk$6V_#K3PN8_rl6gq1kP8-L}hjv{CVEc>1M4pgeIpGL$t+a|#)+|X+w{E;clPF3qMd!;Hm3S?c7=|b(jKDYV zUx%}3LH^D!-VG=4i}x@6QJ9AQ3d#qcP#Z7FQ+~ti&wx3QFQ)AZ9s?J{vmOpYNRqZ$ zTF~+{NNQAESQ1)r#akD4UBdpr5mZvQepQiP>YxCsvM4fy&=`)IBwx|;8sb}p=OH(M&fDFAF~2JG3vATR7$P0UAwWn*zZR}C`wem+_*qVfLMtJr7UsN`bt5tY34&)r=vZz+Sqyd>EMaa5_3E5<4;(3Y z6x1t#9`XffCIT?z)7_>*_HE~?eD~amy@RdNm>xj~!-xj51;9~`@LaQYnj+geNqB8U zC@GtnhNz$`0Y92k83qJ6b*b%Av)HDPP0ap%P?P`~u46h~zszU)9x5H8e>FpN=}15v zIpZxBHP&e($%i1KEE%*dB?P!^UOOlsf`08-(fgg;?^5FQv9B$r6KP_YBKE%RgO1;6 z{0!h4ba`7eU;Gg_FaVVS^o3-{WN|jiQE3Cf(vK;*l#?O7`Tr^W^o3}4m(_fa>8jz1xgN+Wsmea#=?01 zULUGR!2PN2lojeJ=GV$?wE+Hbb+%b5T3&W#nK)w|?*UizZ4n!2PsW-Z;#heqne3&| zMp)&#L%m+^J+wXQ<)L{4pT!7GpW2w-`m1czCiCeq}N;omJ<#~X zbgXURbr5)##9cK}Nc3|(0L5#A;FnlSS6H7(9xl)Cgc~VMTMFHPTJs!mTTjBiw+`kf z^l^ihuM6qf1ip%k9Ed9U=uqCoL6!HsJTiw9*fu5m+SWjduYC(uCo@nH`WhIGmw7RV zOdCW1pyL`Stg%whDEVOG$x!D@ju5}JeB-qnhC_M1cJRGQs!BZJJ?sNPw=jPsBSOU@ zT|EV+%7Z&dL2l8i;iG?Gl)co)$VTA`Li*VSO=x13{p7f1 zxt+X)B(F_bpOz01rY^thMwyurjq!Eo-l$!t7l6V>iJsJH!T@t90S=(cro8)Cm+Hd0 zuq?k=A)@b+AUF-u6`BHGWhdi%+3e6P+sKTSH=Xk#SW8Q6l4{R(*P`nk_4E6I*9~IctQeTJIJR5Ty)cil9-6 z<7x6ObN5gnSuEtE4GO=*f7;z3WFKMZ=A=5wvP*yy$C`veSv(uuus~C^Wa<{$o?Ax} z#bMG?)6(*x_cmz_{b8rfR%Bi$tyeW$c9jKeJ3tR6M|<6Zd6D#St>_^N?-{VfsZMbp zL3$y>7JfYsbbr!oQIycA^(5yMXiU-RbAk*O_%28EmMf(k>PNz2))0X;m?5lmcy>Tc z0OtmJB1LT^Dwxk~?)s^tgb3gtb!3bE0B?LF2)j@K+B96^D~7jSxl?#5?kWxFFfU1- zu63S@S4?k^-^hn!1C_t~UiiE3;eUGfH{pNG$H-T2pQo?E@J`QmN~$F3PF6B3ig?$! zP#o0lpyJ0MC<$5^+m_-MB|%V~eRpn5stT%jx?Ish zjA>h5nP&b=oEv6RjiRr3gBMq#k;JAyCPv1hI6<>K#IV2X>+)B zR}!0n!Zv~wL)x`h+9gZzs|p@jiVnK%(_OC61xpuL8TPr=AA+f+)o77lDoCfME`LNB ztC*9Ljn+~r!Y)>>+8`lX2?TUUv0Tb9voEN?=1k*Ta9(#P)M1`t>OvB8F-cTKVf_j! z?$RM=u|)1QG8cU5&?b(f4XU6jSQEir>ge7UAcqnAcnx#JpkUXAAk;z#xF4eCA#rD<7Y)r zsDHP6r=oRYXYjqo++&E%DDm4~@-4d--9YOmYaq$V21gNS0~=(=n)p#rc8vLkLAw-* z@N{-5g=z=8#-^l;wkh)IZw7wA{LlT_LZX0PsEB(7=m5 zU}V$7KD7w96(Rx)sYOZKQt{NEyKJf?A5ws7D>0Ru2WFf6;Y@CSs5T9d;}7|+J@b8w zaw4Y}tDbBd|j7n;|1}3zt#mS`QY7H}JU%$T6X%io3x>{sKEL z5*0S6#xXm*;?&Cnu5MT3C>wC00l;DaHRIsNYdMnU2XnWQXAq=KU26<`-(M73#9{s3?2mz!k6%JiC4T`7u7?TQ!xvAVwK+`Sx2Pkl(2{lI^rzDl%D} z6Em}EnSq|Rp6=i?S3Ds3!@e|>ZgXLYMFPg?r3j|G2V@APV4L^QxM1K6QN^xQVU62P zg~F%@C#)3Q%WMQh0gj4lU22=LQoBDlN*L4w*hPwmti`LbZCL)a#|zS%Ktv55x3GZ9d{-DIo(YzYAv+i!)B#{8p+7)aF-~7l{pw zMxt%cD%|4ir!B@DSII9{CJOS9(x=mn(=rP+Joc?zAk`+?Nkip!=4RLQt2I|$CK@D;4)#)qp{p7Uxv)rIQwtstkP#iPu1vFwEWzB>8QGDNCA((F zb&wc_p?3BU^>KR(9-S;yDd91Gvx=k{d34r_Xpm~SK*yZ9-yVD@Jh4F_XY5eqg{0O< zs4PN4^D$sI`!IMIzC(cMBhsp#dAF%3`?p`beNIu4 zSBR}57NaLQK?M_0ck!_qtnJvzh2SnVBv4b2k~aI~X?r2mZU>#8f;32|s;-eArA+aj zctp&1JFAbSjTQv-}BmRc0pcJPcx!~Ch%rdJizW% zAQ<9+@l0$$f_N-hY7pwm z+ARSa`kQr?U3n0*F-8ZuLm{uD+mm+#TdpH&2D=`(@BsUCr*JvUkEgX6qzvlMPSsVtiM?5ZJA9W2(+{DC|b&KhzQ0StiSoh$N@bp_#=2%P9yGMeo_xM z;31u8Frp(2^F?04Ab*s$NY`g@L1y%$x&^>J5GgY&L)H`IDtv_)nIZIwppef@Pej`+ zWhb>{P&2G2J2TT#xWE$n%mgF%U7QfWv^EtSb)8E%>qNVsHncHajAc)p^ar<~q zIgC)828P5k3V@%u%~a_pC1%{(0NedB5DtNHg-cSP(wIB)y;E=YN;)0T>e~g69rEss zY|^56hHj7Fs=QD`e7Ry_1|mVmL-wy>8EREUQxy~)ajL+f{n|>0TS?UrL&<7-LXC)x z5T~vn{XHYA;8h`SzbUNuJ0uOUo*vT?JeV5BNq)CI%hL7Sirt5)g~}J3F8GU;kGXZvzwr^m$O3S%bQhdo|+(!EjN}rd0 z_qg*=zF}do2u;A!p?)mVPQMlKMVtR|zv61`0E+XCacbw&{Gi%292}z^Z(L1A~lBqB2t8jnZdY zVJouEQg|F{&U^?jlK;y>TIWQ#DOvdf0R_p%R|kTtu%?s?Po69lAQ{_*-A$k^EHj$` zn$C!Tkw7geq#ts^Ad4jg5I6_5{e7}6iM zFz#*)e6Nw|ByroTe1X_NeMbTYF>E8+jt@|$9n@XFv{CyW?%RT~Jrv`Xq$HiTpfWDf z#+yRX$5&>+xa}JFr?XWKVwFw07c~;2O_98sF0%ouq^v4&M~>cBb~pm;*U;18?_fBTAG123?L z|24e*O!jXTb~aoeXim@6Swq{ZqIx1&bTN?xpTI9HdI{Z1D|9!t=s7laoe#_^y4=x{ z^JE2c4w%DfK~q6CAJ7`jKHEPJ#petNL@?WuLXaIYYF0<<-})Mez>s=$_j$95*g>s| z&MC^ZjW4LXdv`cbpwQ{6BDxq8sVan>Ne=Upzi8H{$Rb7@N-L#)N;I{7G7g;=**I7P zX%D5&RkRAEpR^*EFA*p}0CrC!p!*Wxel)Q-S_w~r$|B504Zx1fPPXD5t|vVuW7WCqnYR0+}0)e&F&IMn6jz))ZCjb0ea|q>oT^*rS@G zjuL%6c`sGviU@c4WLk8Uu^ku3V#sG?$<)W4sb9T+kskf=+t;Ab{^b47Z@ICgoBcHK)q+2y|r^DhMF%IQ9Fu$V* z=ynR#n;b1J(~1%ZKy3)cg&Va}mK96W#$ENes>mYAf3Stsb_ca>>p>DfZ6!FmSAq|X z0jloxm7i;k=V(3Eyj_SrOLj0QRuMazT*O0lSjAigy2WmhX8nPl*9$F@rh*I z%6QZ13`>)WO9YMB0?uE_e?k8L-`Kq0y#F{oDjDhf7wONCzSvG2z!;>m*E8TlAoI4% zyxK>w>U&>-EVoDZ5p95k9X_}xB~zmn%GP11u}~ltMLboA%kK=e(8KW23EGIzGC0^? zR;gU(uET)k;MB&xM(cQy#Px+7+*MVuALvFyw5skdi|QWM#A=qUtYY~cA9g~L!8(M` z{WPPeM8U2($?3w_0v%}dFr<`8pOWGyEM4s6GrHmQLwhoxdOcOw`UwqPaA0g>K9I-v zaJ8jL8Wxijbg3lUoYpm zZp1Yz)TUD@VDA(_lcH0DScN>&AU%yRfg(z`l7rUlSr=$-*$%-_;X&7s5tCn@b%0Y~ z9KTDsLgjfheXb5?D7ury$gjfNznIy``yWC&!{8+X(qbOCJ*QA5=O}yI^}3iAZ`C|Q zHQTm9eoYOaofPkV^8;qvVdys=vOgY3m~69{8MuxtX<{#j41JPmCa&raOTD@5HZVXB zJFKJzguvfZ&ZCR!7nhRj;Y!!l4Cp~_0WR<}t4Pu92A})jID>wdy8zG1dcp}cJG*)6 zUCC~0Rg?2@kUex<6yrgVv0G~!q9O|u-gd9x zk(S4~@CQMA6{V0Y8D;_vOm$L91{c@XI@^>2(upiHv1^Z0kSpw}U`pE=+j>B)dUM7J zhB^>skSf?Uv+*Le?{2wS4WCJLG)q5`%Wf)BT@lR#iG~k*T`B?}u4xz@%YJ|{I0Rf} z2bcyOB8*K!t!$B)Gn^jKNV_z5Kb_t<;%%G7U%y3x^-cBUhdFL zk=oW&8tZz7rC-+rdVbb)K@qf)f^^%}0ySOB;( z56&PZ_4cld9BG?T-{hu|6|-;bM8k?vs*qH#Y4&kwo1w{gyCym?4?%sOy|^tA)4-#J zVvZdOJ6OL2w1FAHkXD@B)KH)x?@mtQZAZh=*l*Zl!%mi(^0zDP)qz5L%Vs*?3x9V? zjs6(kf0V9%6W;!0@m}wLNfoYNn%b7m|4rC>^n@cRq}G?GHgD7717~oe0|7Aoqi?(w zelNh%%R9?4X;;eeJuj)--$B`lKu;JkDe4%SO~lOiui3e44Y$es1(zpMf!NVd5QB>8 zi3vOsWR@>I(B&S-!6;Fqv#hMOE5paC{(F)cWLTHC-o}N=CDuh1sL3*{_ozgW!Zyh( zI;`oshWycjW#+YsI__CYq*AFcu^}J=hBb9<#r0C7UYS;cxg@~bB29Xtlp8@AJ+P>T z!rYEdVwCl936RVzn9sm zW|=hQkvBe%3FLu0_!KSENi62wK>@NOlwzo6KU~%iOhY)X+~7=YkgQ%2il8rA%i(Xs z4}S22zfBAKN0QaQeW}@e#nBbUjd!9t9w=vyd|BH&@#qkzn#Q2$JETN8SCv{xK$XB8nNMHrlc*bQAKj-;;dF=x`G}R7lT4Roxs&Q`}sj7=-m4$G_z)_+mnioEB z#7P0%BY>Tq@29Op4g_VKtax42yQ`-c5e}-6*EFPsEO=C#oRZga8K%=Av0tLIonKY< zDV@o%t%2tCorvUEh4z5z)Cz2y#ra)5Hx0+6+ro8d6>2DT(h6kK7B8z>+CCV>VMR(z zvAoZ|dZ9x&HcX{e{R6wJ^wq7lES}O~Hb7EgXr;0Z!)kH8WU!coN$RUdl#cgZ7 zU}#8D-vnZwc1R*7&^`o==IE8JnvpFoR64{~Y3=peJ|jih(MB;yFM1*#X!{+_{DHy0 zp&kH)tT9mFn0}EQEIzX`<9&ioPLv#Hn9ume8(QSDe9lerTZC~#9J#W)V9EfU>pHa$ z*m>Cw@uWj*YILH;uDstC-vcPP?bKPagJy$vI4pQjJ)GR}t`qu$yzcM4>gw9aUy6%r zFD%|%YlXC>YoycLq}FL22Bi;T2r7B1PQcnRNLv;17~QJ`c86k`rWZ+i=lYH$u|tCwsx)i1IN5QTY<@9 zfGUIE)~dM;=+Wso$wF>57B;&0HBCgcm+ps;Z27GIljMFSXw8b*dbP;E@caNyCK$01 zh!_KYPCD9%1i%19W02gjmIPphhu_LQXH5yPP6OkJ>KNXVV-veWv+D)i_b|=2jqa1A z3#DWn-qUTe`tM!6plpDG65FNU@A8?FuwvlJ(c*S;`)z=PHpqF?(cbzh ze}lyIY1d)S*fLO7p1`YN4|4zJQk~vvHEL9?Yp`dU2-zoXz!CreaTOc{3Ios{)Rq2h z9sGW;HN^=Jkk*n&-+Efg@~i5SX%*-t%T{N7LL(M}$MVOZtd_LGzx}7Tp9h0^sSUz+ zJg=WRHcpo<+L>v!q8;Ya#Sg_+dPB77k5c_qeNchoas3ELOJO*~x7Bkle1LYy$cm=& zTdRbvk1;DSx^grmL=DSM*ZwF+6+tT~L1~Z!=_ET(PQ4hoTRF=XlOfBh!fYKQY&MlP zoYswp^*q39j7A`+Xba5zXHVF4zQ~4Z9X{+6?cNq;mh!boRS_{JB=TiU2!j&utDQZE zHo(2-Ce?@HCB|@HYO+#LG@xl{NP4_zc5nu@6ypix%~=oyZGzb5<^nW3B{u-PGHJ1% z(Urz-fKK+x9TeQrwb=&lKyY_?o*;@4tTkBfp04K9GLuC*kc6 zm}7kM_Idg{)BNW_@%?WrGvs@6@7P0qx+iBhMvqe&ra5T_LNGV?!xCopmUm5;)AX=9QuXo5P(9 zcQu_>)C#q6G2ENg5=WV%#1sy)`D>%!O9G};RQKn^N$~Jd)Lmr+T|`%Qip|U=r)nwY zBc}`v-$sS}0J5l#0YtI{q$3y%)+NlFP-j4m5=5)kjnZrbAmia-f33CSjY6LyZjFbw z>cJP_OM@@}ob0 z`zm?dR{me)}2F{HK8Pglz<`SlZX_3`JFEv^E>7_Aup@ z_{%XP!jIXoLEqhiDL%fwmh<0r(4imScV>gMp#imDw#?t&l$~pv0n3;2hLY=}`)LOh ze#)~OW$jtY)gFPL44|`CjmszOR>w1xL(_(t8PBGX`Y)5D!88_;Z2PWXsOr!up37>M zQtcI~4-c*gC6s%3>^vxpB0sgBdI+GvNS&VcOY%ijd&IX-B^#?;Bl_Oc?LyHiZ0rVh z$ZuWtq`#rx`-FXu>&AzQnHL~lEbAI2({soXRUW`5>@b(gsO77Y^XdT~UfQ{J59=$) zB7%psSc{^Xw8>t{Pi~5>f0fIQ_KBwvy0%uH?#2f$HMlJ^sT z^zBD)-vnNyM}PG8tMK;Qv}Ja+A-od~`6O)l2~zHO5uCvms{Nzri{*eitDZf+1>GDoJQ?7ON|g=vGN$7i4B zq26G{0=)5d^cfeXHYC3J(0&=9e2O-19){mJ7+0mnZas|+J7*8hbpvmB$95+f>pGew zo!-J#?bK`DcV5lZw^kUH?d}n5KjiO2ZcIEK$DB}R4L7hlM^LX|V=w>!X1X)W+X431 zW;1axpl><^GdQBA*~6*zyp}QZh*hnUn=Za!Coz~`PHcW&G zuU2%jZ|oR@VCti*v>Q$2Wu-{@D?z7Ktz&=L9M%G?>H@B3 zSOwYDqCQhZXzq>k$|VUzI!ExHoOeZ|Ge$$cp~Db3CsJM3Ok#Zd498NkDM`87G2N%| zy0b|z!Ss;jkere)zQU&=L6>DC_=xW*&#|rD-w%hi3tmXx<&85rjl-2IZ*8I3aw3!| ziHrktDQ?rhX(fGCkDb+-3xt2H;DO3W-RBw=l69w%6S~Q)B;ZA2i3gj{G0sI#o3UjXQP#MLd>r!8B5%ymRzsxEU3 z^(tUwH|Xqeu7Z7+%?i+Go4MvIjzA=s9~|0#@Z(kmrvqX6ii0mG(=D}q<3d}q;&Ggj z+>XWV5p~wyT9+5TJDW+uj$$DcwzJ^lW1DblG<{|(T0Ytom4QcDLC=Hx*VW*PyP-ufZ z(*9cB9UK~mwSdh7!F_ucS!xoeWQQyrz=)VG@@-ceP-If;oo%!oza6p&%p9v4TwNmh z%_$>0BL+wZkoXRyLjz<%TI(2@eAsIxsXsYmyj~U5#x8K>+t5U}zNF~ZuXVgAiy?Qv ztR3)-2r4G00`@n^grl$7O?ALziORq9bT4O#iqm#k>lcXlwNf41I}7LxCnK75x;8%g z*dKOTj(Y}=%056>y(HJ9Mqho1_Y7RxFhJ9)4DO=^Jm*}A@{kID@mJEfYJ2nJ~60&7=p=5T*6YB%0ZIKJt zZIfe^I^{pA>s(r6N2Tl=m*H>2JX-+XLJ1QN9H+r3i0K>pm?gG4!_YpWWib$_9VrD; zOG>*d1laDiiN0l_yBhRDTW(KRQ-qO$7Yy;)pPq0jTNTqxiCm-u!*b7sfH7Bfm$coj zYR7Psm<|js=M4%&d%eI_T>{#1Dx<%8`-y%1;=A|ZyAPzl^^+hc17f$;2CzER@4V;~ zZy(brLsB??4%?#5Pjya2wmx;q#nam<@|iOFD;EW?r+tHPQDFB4K1@UB z+0LL$Q>zyT^hqEWjzKh@k}YMh%Ivhbw~-=8@=2yBecRC&whr1268u-Tc5Xlnaa&@| zU)nB^;m=&1lp&ccZR-U^w^)x%?>AK&2sa(;;mbIb?*tlGP{DAb~7_J5> z2Xit(S)rE&^GyirCfj9OvuE9+QvTJZomQ9qf=!x9B#x3D$Lm70IcTzl^-D^OzFq2b zU!l;NnXd*c=>}32d)trjMsuSI(J2VC19pxQ7* z*{c3Ld_Uj*F1-Ib-+lf5WBdAPunTKKee(W?w;zT7kZ%47{sEs|;CiFhH{>R6?(6|F ziNPDeiJBPrc?tvkABIB`cxCZHl{@VR5W22DDv;yg2p(cpy}9GJ$}UmT_&1Bdi73oa z(`SSB28XeeQm3o>GD6biZv8Q24)Vrdb7aRM0}f^Sq-1;6Wi5+)V)}Ehz5KYvoYmh@ zh{$(LUQAVUq!xB(v6wSO*db(Hx>mMpSjVa8IKgVL^n7nL6Ha8DKk2q3K?hB9DFQCF za;2vEQdQTpQxJGTEYVbfeYkGe|grh*SC9TA) ztSGf$mPJ%MWaVh4GNK(Fj!{Y#Th<*4qmAHNwrG@#@Rp1!o)8&7p{wQifRLhwTdRj% z+w2jxv<&2(17|CGe`9YsKz<31qTluv(bJ}Ei$z6KlA2KY>K93ltT7{R2vQ*q?8ZZ< zaL{~a)-E}xtZFE7E*Awu5!{#6Lcs}KX*i!ax6X!u&1^x@z$6~>q)^RT!n{p#zRf_U zI_d}H+iF(K5+aN=l|MMI8JLTynYqe@@QV+ord9SXg9x1vO9d9J|a}1{>B3jN!f`v*q0c2n>Z;6j2 zAW~GY@>T$`9HELOEl+;rGZR7=I4IP`2SPs?xhvtc6 zgC(!woNUbXTa?4@@cML4PZb)L2JfXaEVK1A%|3L|JtjgdpYG(M^Mn3?E5O33Fm@fR zl;8n~j&YtSyFI#I!jMGl*|1eVaZ*JkPt(z)seH&k9@*AI z`y6_JioPApr>9k9el z*PI#^Doj&nu^5QyAveFXKW+LWPOp-EK8Q zOuKLL1wD>u7TCK@^&oMtSD)pI9Z7j&)!yGDhdCXUT2;0dQr7fH+RVlB(k5$bO`)`b z1FZybj3k4?;{d(Y@EFG+n>f`2x*{a|$Y;>>E@q0{&OWB5B`A(+?{E)v|~kBOD` z_2s3*fI0<&XDi9o<}y>5va1BTjiMfb%WenP|FSESu}aol*;adtPdP0=_lc0 zJRHOJOWlHclxM@L-FNH4EKvd3=>{gcR;0#Y-YApgH1-!Imx5Pi$QZ!{^qG*0yT-(* zzh$Ek`54u9^4ur_3E>$BPKHo5*_xMfw6#8ZXt!dm1L}q|i%9}_+yHbBNlqE+F9(Nkp^o@g60K(uPmf7=p+=?TF*$7rJrq>uI-mgy;r z_Yu7VqjfA90(xw_{LZZHk_(WLm@8P~9HjP(vTD~lvn@u@0-wm7rzver~+fYCua&SDj+G>{@kptu&R7*VzEap@Sgj z_gw3UwN*|&{kY;dwr_@eO0en|opGS8CYL>y^N_+#t^LbOYTj}(1WfQr86E>*Iac6* z3-ypHIwD#=WO;FWhMQu>4N`&Bx9HaE6eWG9S8}L0)X+qQ?JDaDp{ zu#T^&wTFUYRPqQ?=;Ur(uZ`5NrXdA3Q4{@!;TG6c^urx}4J${9riQ4L17@HIRfK?@ z4}8`-`*+u+x?I?FOeIj{-|sKWLq#7f_1fV|lrH5L`FvLg4B4*wP+xR(|WaIDxF zH(jBjAO{hJ3C^P!Z2`^^(uJMLG~j7ud?6B9s3=#E**7AKC}}h z9=sg5O;G-d8_J}jxODM)RRo-*tSS1UuJ+?hsuPvYaU)A$R;}-=q|&uqAYh#~_G3D` zu}fwDG$H83tu_}w)YJxZkoN%fkYie52ukF7q3{vQWDA7m@-9`wv0fz$gI9QZ;Rj$kwlv|l zmjbnEy%8L){NhZ6Zl>y*MPN4&wTasU1Bv2L5$Y1MVDE8sCmZBxN|;Z1T2E7^Sr$B0 zMO{xrNk6-_j8V%>3r+k&*H1bfj=O|zvLfIbW%~skYwnQOS7`hJrMC{Us%r@nsw9@q zMW|ES328+aw38?r^-3vhDSdHr%fI{J0}C_8Q9kxv3K+%dE(hwA5lx1u4g=LrxhSz* z?U08QWOg{oerG4DfW$H-){3qIG}67nz9;{M?|qMD%{MG-z9Gc*8$w)($$S5Ec>gSI z9&}{Ng|N~m3@Ry7BQcBSC)&HGM2$*W_UIl|>g-NXo}42STfL4{(L@pOquz#4df1;F za1uPJ=EMR@cqdapFmVa`hRZItVB8pTmR7|AU|rmzyI|N<(4#Iuli0G~vgHi;Zw=e4 zi*kQRx0C&ib9eM2zB#F>-gj42yUU7_rQddN4l=+gz$3=UxudkQsYNBl)-5cQcv@AS zW0-O`NL40ziJb9j*OGlL!=)vRqS4MYTBZj3xpGoaB?)&a(IEwp(pF!qJy_XJEWmA` zP-9VWAGP9Ay-;r1wia0JJAU^nfZsx?D1s0a)8fq27jd z_yoEDNpJr(5CXe9mv=iB?{aL<1Qty_?QpCF8gg;JWCAzKI-Ew+l}j`v)nJX9VxG{-7VB#3F?fi)HHC;nK>m-v4Z4`e`0L*!IZjFam=X`~J zz=~!NuWO&|xsEg>Q_(uIRgY<`nb1c|!)}3)txi6n1O}{kd+)mnz?HOQ(JzzjdYzev z)}>Z?276Y?WZJADG&hqrgQO5nYreuj-vRF_OlS8PccaF9*==AF+?Dnef%`xOqF6{I zi>BVK;d=>HmgC5ifO6%dOXX=5$r4uwGYX)?zSg4kkVG($Go|%SYK^Iy|I^!-bkTc_ z!PUm^s{KDeza@I>1vJqHBd$=l-HMA^{oqQNCZT&9O9-{h9aN=czw6X?cMO{R^^+3_ z&^>MD?JL0ggv*(CWrNW1E_4s_#-z03!0BUqc%T}#^gcB9u1pf2OwJ$5mc40807L_< zp&G8RAYZj9?CTfCKP=oZyPABCKv~O}H^1bPcZiVR<*0vN?%QaS$9rGTACYNQav7Rh z=JALb;aEwMEpHyM6}92PZ{bFwjjR}&$v0zC7*g^6cmc;!7o1#GP4c0_`>Eq^QE5*V1-7Fb|9<-p%UT}5d^uMpC7srKDzoonhxH;gruP8NXIbtxhA z=C(0v2*3HILER(tG~7<0bP*Ef*j4t#&xNAhS5nHhM<_FbE^kO1cgQDjN#e%o9s+do=QiB^NGt{)`uZXGhixXar}9S3+E`iUkXuGk!3! z&7?FHyanhqJ`r{3axDd1~2tyykw?F*L4?Qe2>`TirR>g{^*a!55=!^!EI7rr66u}Lr`@PU^ zKi?G{me(kO(rsLr+}we1vQtjd6E-O^LMg!&MO9=W0$g1Oek%S1?7N-;B(xoS(Jq95 z+4y#W_zEx~m$s8EFj58=&{3&qc8O~8Ijqr8iH(kFQ#3)gGbrhS=Vkpe!f7`3LL3+$Wt&>Ii2Jj)1Imu9fRV#$Qad8(De@n*e>uS z-76`Z5$Dxw#lb4@-5p8g(~-MZIbZ`n$6E zTTvxq+205M!d*XgMyqTB;_NI1gc=E{~+3m$irGK&ffI`4DklO)uv0+>+o zftsA_Ks1Z;@~MozK$?MiCRa3E_}*M=e**B9P3$)yd0%rSJq*Z>Q>z%5AI|?Ik(Mys zvDuSKlIUo^%EiD7>u1-PhOdTm#L7s#avA`C#dwP zIaEk|UPr^WXNm4~(sYv>JmH+j;C$K_Ck!#vWe$%;B2sKm?fc` zewX+7l`S=N*XG!xyKqMvA<79ebj{+-fp%9ZM$9=CAWnW<3luFL?>2LLCtbFU-iDV$ zSPogG=ncfxaZhNG1-=6+XWP^DtqSqp zdZt$dX0@AZYemLPe)8?Jx6hs02Qb3>&w&hnKhKY(nEbs27q|ew{}y)=Xui69`;W+x|KaURx%l$( z3M@%^xB3CfbO1s!_XPx}ll(vd3|BbX=rs%|wkh*RSV4MD3?r+Iet zD}Yq{#wNP_PyI#NhAHLaAORQYE1NcOF=ABe1IynPKt<*P+WAB9#1)(a8KKm{&tk|V zS=v0I)NQEbx4?lz+p)5=?U1gkr7H?qM}SEHyr{A;+2L&}A&fR+0#~|L;h=v(D*f4Z zbj)O_KEn0Vw$Z&??&|r8%?m^eHG-5jo-XD0S}$CR)q8!pNfDP33PqI72aSQW54+v1 z9hx?iZqj~t1 z_B{BP3|GZ$kK?65p>LGmE!4%?Hfs;`6b%mnBU{jd{N3B1{@3tVf5Tn%dHBnJ)$2ng zU^dtHd8&uiViV~nW|YtZN71h8~`IbFHZ2|2$X~;oJ~6Q(zUWpZL{p5EH792bsxh6t`J1LtG=ST z$Xq+9cW*XJPfohtNta-G#C}s*!v))=iU!=SfVRC#J&Q?`iuY#iUZ5y?I(=7hd3ITt zPHOuVIZ|rxEJW)w^N~IA&PIL93$9 z&7&QLBytcnJAs4y9Fmc6+Dy+7tIFlAA%W0z`QTY5Z~PB=IJ559C-5;HIaOP8cDQ2c zD$$o2uS8q2*)ds;5Y8$2K<~DyP)qfbs?>s^(fJc_1#`sJEXN=D?T1dUux=wy9yAM1 z8Rb$p3H3yL{=E0{dLj6(FfHq1!ZCQg6BwlyH<8qWEJiq@R>(3XgKXO%HJ#lo%*luD zG`AmW#L&n(X;6(B3Li!E1lggL*Z7#0ns*qg0wo03LfBys_p~uj9`}8x#!F~_B|{XW zYV~k|rWsM&x@SrR`nY2=Wz|Ga;!c=ED9Q$G@2z_#BC%sN>9AO8nB;GA_uwDn)Jq7=BQtR9+@O$bVLqF@xHD* ztiiekX2!c`z*4kO-$m+G$+af!v()g3m0%((dB{eOnvoDd6(L1gqSXsQF^&h35K*|Z z)Md-cUpu{s2E?i#C))i4T!GX$B|lkJz6bLfvSN_{_G!=B)i9vfkO&}B+u_YM zlq(W0(H?Qo%X;ssj7sWyJWCe_dju-rtq_pxN|iv)ohk$MP`pBdaXs&ayjP|Hf@BiM zE~h9U1G7Sv?rlP@yiz&Y9lC?v(JbT|icK|IkrB zXInYPiGec!Er!pDEe(}|qIbel1|@+hI%v<;_aJZG71%a;dM%CyYMnuoml=8wx2f1y zS&*$yLQ~e6uN%dJsf*QT09BOTc)4TrZG}}UhmCyiE3V%3!E_@sHfqy!5=QJyC?yxA zb4mWpCD|Qy1ZM#po=OuHk*?wBljYT=mih|4@x|wJ)f|qg@#2LPGaC3BfC3TXL@tVp zvJD}{weg_=x?BXIs=Z?mh9PBE(D5zn5{0_O^RV`*n^CqHNpl>eBQ$7`=b8ud&?e`*hSm z(mq}M^Vt!!p*cG>i1wwJO#XnNyF20<(@QN|;K9TyX!|H4PY~=W=nMpzyCN^96~arI z&m0jvWleRLWtRhRCX=n%jW4R+DVYp817t=K7IaQO@*SSl(TQAtdDYibKQ?*vSVH#y za`rB}vK&{I;5@##(T1x0R zsztbsyXZ4)d_J%%z*0^RO_%fbT0+0Ny*+^b4~U3z!WG=v0eW_hLzSQ53T=FGuX;+xzVJBkoJHrn9a;@&;R`O;KJw>lL5=73lyNjE2qG4@v> zwh}6j-1-oG4sP5Np%?@?S4dZKdmn_Aub?!5{AfMQAN}Y@;UCOg<-f7*_)Pt-vigwH z&^DvgEfki$H|qUTAriv#mdFBiL!Ye6az~; z+3GGxO)^8117y|qICXnQa0lU?mtVQt2*%airgfNrgiXr2rR$?4E7tpOT6qqW<5Q3j zxH}OtGLoU>@p3MwSUIT=(I1JBJwD4RXoJ9;`l}n*D)j74U6T^-8hJo!#sxB=?po zUYqxTs?SKiU~N*_PVd&n2O07*GX$wv37sA7X#i#j=j`b=+$CSJmcHC(b3xLRJsdn= zfCcCqwKXyLS18VkGPpxX;J#(6@28a7RPS*a5fpN`&3z#EXav?>uX^wn*8p~yQaral9 z0mizLXaQvzM8Yam>=de{la1AGVMq95bmtNt@VocJOy&0NU$ie|mzUkINY$cqz0=Te z5Fo~e^EHE-O1!?uN~c{gmcqx{PW}%0*$3eIDRW8glCK#Q2^&!vw~?je4FsI0T~^AJ zytRNvtj=+zMsh@pDW5h_vB|MSo$e?)fRm%dBx-@c{>&%_gVH4xr*bHuu-&UW${;)S zsI;NlGzYWNuuzsmp4d6kjJrM!Ht;nx;EKDX zBWS`leN^?$JhKO1__k_>7h;At_!7o2=y!PmXUpub2zL5#fA>o|YX9c-yF5P1z`fLDX|#$@l=;MH%4B5?z;PSy|`+dE2d1RAP6z?%D1Pr{kJW0}hH$ZIIbg zE76)IHVHDe!~ny4o)ck-HOd3JnIF(Qz#(aIGv;oW&v@~|%vI19n}(K669X+7xE69r z*7F7zKy_x+7O+M()>QQw#V&@XphyJM#!@ySduFj~4q_jj@+NFycG)7Em5>}#mqt=3 zh$yq4lW9YJcGs3e(7NTabf$8xoCDh)C8%!k8AyOtt1pJC>#)m~$>Dv%`Qt)ZP?Z`b z3L%_>43RfRa*s|acrqNUVE};oi0p|Ax@Kr=XAKC}M-UdYTbxu19LJjEkfUPmDP2>m z)S~SQbt!IeL8|tYUgVbqrOl3V*?ULS^%(;b`1)cPZ597rTVR`SEAUEN@G&hR0{@R) z)Bj1`v7dz3pXRF{L*)Bk-+rbFsccl&;roBVTha5uRfW^?Wp|zwa@!5}9W}dvu?)Iq zay9Q7X>D_GDH=`KYkp*hIol0@>NMp*$p~m&8pdwYSWn8`IOrd~8-(VgWw=4UWZ1~s z8b&DUhv$Lbh~`0QL!{xdJEltpYt1sL{4a*C1W6d8Tm|5+T`HTkL({N}EdATCd+R#h z$bfgJ_Ryeil^HNvMb~Dlz1mJesZb<(I^G=J=Bi$H>f9{>CoC1`!a3^=oj?%B>sW2n zg&LiEazh6Oe<-y5bV^^k-YP~jdHo~215Hsy9iT`-=_jUsQa&IRiPsr1IK(Rn4YSS=!E>uADRu1$VCH{)0hXW}xB4Z5IIMbYJgf=J! zPg3;1J9GEIXeV*+l-oh1MIBk7ts&)D!Oeta>U1abxm$*48JZj`)I656ZiL3!?z2m{e7CB9AmLme;m=Y2!|imr~oGFQj%Ka&9TF?{SlfBo?LU&?>~ zz%cZOx1Z<0^jm$AKmVaU!ewGs`+&PVZR@*XXDCevRMJDK^BjKK235^IRxMrDV;Hpd zX9F2%JeN9dl~3hVG|8$Yr^SbcBUy}|_~{u2b>Y4abVYr@-gdDF2Ul^4ZfB<$?tTb% z0LV>$!QQv%CWrh@>JyI9h9}}iQ?PwEL|1n1A9R`O(J9JTuzkj)sg_NyY6g$Mj9k+bWRRC#`i?xeksZ~Qp89f)E#xcZ`VXM_5y zk_q#uS4nlOh5>rEUIGkV8qXqX=<3)pfTktFD*pKdjpmRT?pl{Ws3gZGM&gFwrCe>P zUvL~VSy6d@?_Pm-1&SIV`3eh>oZ%*^kjW4ZN`vz*0NfDTM$wDIZdQtzcry<=ph%9| zsBjG#**1{J^t8QPx$x*YEe}|Vb4z0-Lzy9&< z!~ee@!~f_X_V51t@cP02>Z`X8_%ZzO4aYxyeN=IV9jT3;)u!8vC&1rwa)v`oN(`IZ zERZj_Ko`qFknl~$ zSnSA)ZC1H*ITeZ<^IHiH@@bff+c#4e_^jG&GA;eEu^->pK%3v0vU-D1?uq2ThC1tc zC_~1PLtcm(p1Pq1;GNhEB!a&6C0bV%+xhJd&1&Iq1VwHiAa#gPJ4)jww{P9eYasg+ zH|DK3;93n%m|oDN>WLeMh^jUAFk^x`aY3p~5$Z*D;XSapQb9-$g$X8k5Tnpe3>z^j zB#OQ)BX|iB0LIcMyUu{Er)uw?6*#GOk$Tm}Vw(-*E)ZhR2dnwKF=w~wvdAx8VtXC`%c2%$Kmzc)A#@K`Wx#AZBaFkW>YwL2?h{( zG~my~12gKrceB<&7}AnuZmp@_(92HJO;c3v&)#mBsHiiqD#)T5`17)Ah0qQ@V7)bR zsL0kR<<@9Q)k@4H5)bMQ)o@36yEXd+QO(E{kYdTH1RzK3jB=Xg3yDGmGod@4)$4}J zZHz`GJP$&Y z4dF^9#Z;qOvRzVOsdLB3qi|KOwQ!K%pv@KE9rE{BcMZP7ayZ-$V-Km`V7+8#MCQdy zNU(^OP)h9*&vi=tI*CWKBq?TQ21tWi{gvJek7~o9q7Nm7@;dgw)TV~|EvWKl4n-x; zh{*YNFy9M04Ef7aqhrb?Stu`G*h~k8-oa858s8H=Mc(QoEqP#yC7!VF;2(|%2ytQR zyP&~vRy*k&)YJ*PP1^qhelov1K$wFZ5+JG2#w?AnIZCAnR*AllYr85;?F2x**ls<- zqDeNYaB%O_Ow~Mc!9}?Ng?;5&>azTM!654Y0lMKg;q`OboMmzp@I&s!7)KTVF$B`2 z0M&^(XEhMnm;D6GY&ftRWuQk8xpgE4ai9tHHt&_!80nNlwda}ueYhz>@t$Tv4p`G* zxl_iz0!_JkiG>Wa(Y-HUYh9J0-RwsLLr?FFq&zVTNx)J+ zO3teM&9D0@03nIeS1Jl-boS^->)2aGoIyW-r$|(T#2Y6GRBTc~q1((Z`Bn~h_-za`Wlm9BN)cK2bD`o_&8!9H z;4R-mtrv01XoVS?`=M=FIUZ0#$$7XPX8T2Qw`QP!D)q~CU@dCZBmChEJJ-oeB%tj* zABAZQ0Co~bO&0MSF)~H|%S2V3@sfVrZ5Lx(td8qKb1@J?5LMeuFaKjO39{eo%kO^< z3G6J$m$Ss&Ri3?ZhH-baB@P3-J5@^7>Qm zd!{1Ll(7xo_>`-G{VxSQy8%j?h63mp^ib%sO^7}DZX2LXC3ikY#^ta$`lY4zoF8RTQi~j*f|sy|Wi$c~#@e z{d3Ffu$o|P-j2Da`Z4v6Kl68AKXgY$7>54x?T_K@J9K@1{rYox_0ikMk{*8wnWwi; zmrJQ2e|vgJU9rvsq4zDwu^Y;~4rO6#2_S~h>YTh&3|)*I z&eqDtL~E$))3PQ+tze|Oym<$RqA}R2!O@O}o$*Kug>5BnN4bl#?=T+C`*47?I3MCv zH%}L3mjtDkqd~G7WXbeSfQ13fh8rS7`!HJMEC3WDNK{gnqu^e950m+dCL~%8(!ITx ztYJ>jB@&VE$v_&q1#lk7>d!njj{TntCzBYF>y5U1Y^ZF`v8Gnaif|Xt= zTuo!OoDE-BA~9$GgzH%z7A40pQ3=!{DOLoect@{f4*h!I6xXr6D1g29{y%K^2*izj z282?(3*$Ux+yJT?_)(5_j_HBm;DW9ovM#kvb6n1Dw)3mJ+Jc=yz(s&SPK^KxMY-B?3qfN@}h+GI8P>z!Pt)a$gQO!>dM1H(;m-gL$RTiK&b>>=k{ zts@!8Nsd5Ok>oPV@B%&rbIJ z*Mk`##{W!;JCv_b`aURo$e*UQhx;DMPmfwlv6=s#E85LXS(-W!pIZ_rmg6pgd;>a^IE;uj^*8>lp zKwcBk-1B--PeY`0VD2C%O5bK|Y{_yFx=T<)ook7!lF3M}ax(w}z^4&t zg}ZE!y&*iSqlAyj=IDkD>Qb1+3C5LLF;Hb1EgMJk20SY^&O>E%iB;|u+&JDajP)joz}g$|1Bz;_))c{(V&?7J*N zQiEULLepEDPxewC+9I_Ql{aH`h1wk&0@ZgwY9sdC#_*{9H}LKQUXgfLAD#PuD_mr&d3bF|&3 zNwVxbN5`KvPkT7g0lDbZP}lVnI`e#-rCom)@`sG1_yCit@cL8mpgsZN_80R1Z%*I; zm@<`*Bu|jTW#n)YwJr|BbKd2mP$Lu(BdJ3Ru>5Bp_W%bqE&vPA1xk8ay<%$>s@~xb zdhRG|wokAmk{V3CQ3cak;-k9)DyPy3vv*aG%P-;$*{)tgxdj9?w*rvumWPW+^%j+m z*>mf$3B>0P<2cM&Psz5Js^3$1HFVZ~OJ!gMD?e{h%Lyox%}FAtQNPf6ddOY?V~8U5 z{XML$TB4UpepjPTHKnq|^J?D4?2_x9#d>u?9?6W(+IY40rS{w8V3qYtR4G(ZFyBEr zphe@mm7}B?je%W`yi{)`YfxjR*EZOakAX2u@+2OHa_Rgw5F-KR--JK@OtKxGPx$H`Y6$orMhCE}G`um0 zfHASF9x_3cc9N2h>p0@>jPoY-EC)83IM06u>};YspiUdOJZ*wx)c7yoezm{*+t;r- zaQY&@Dpv^q{q-*x8h`kna^<>Y{nw6JW;T9h>aZ#X!!3)zKen>IM$@g@@vAk+ z2ulW|!kont8s)SVzOhi^cP?o?XF`c0^e72?+xCtU)CBuNA{c(dS)m+xpE*xTGdBzh zHJWu^!Y{x`552>iRx}ojVHC5hQhpK6?FqhghEWG^mwgq8yS6~=0im;aI;LFllI5L* zq^pWTVep&0Y7gaLGD!x(zgVL_@Jrge4| z*X-yQ23fII%H#zseX3)JA6{bVc~ z)vUB#*;LK$qF2W02CZ3FovAaIoHixrtO$S6>9p;jwQbSBJJ{C4H98R*)%aLk0(~fb z+yoGf6)MnjcA<$-vzm-jH>wMkA5fGee_xvX@Wu~6=CAtv?VGpX8U{nvmS4YAiJI46 zpEhK@&wVdRQDqQ|+GYTc^Z+KF)|2v}RXg3#PFWFomsV9lo6avU>JsqSN95RCI_Nyj z_dzmzYAfFw^(>%TA&4DXnAhvF`|ZX_DYv^Cz#PMEmQXf1^W7E~HvF{fLF_EyDIT1N zMtI1X1>@=!T?er^%WBm^ps%PD(^}&PI&vyn3n^#2==v~nSn9xR zp$3*L-)gWnRW^UO%Nyj7Bm+3z;4@4$n5Nt<=p|pU7I{M&hFrxb`1c*5D*Jw@*V*wS zIecq2aJt}#K$}YrA?tEOA#I?Pp<0!l(ppHp#7RQ_CkY+-7cZu!YcQ{dC1vA;)KH776Hpi4lzR|eD{Nh^Ru@PBsRIm|2N?+Q(eFRX^_-#L)0{<3a%_r?AoU3!K0(z>|g2v6Y<49>^wcB}gsB zEh|jSfTQjT6YP+ng-pYlaAS!i@p#j!2mpD%7dy|jeySn*rMDPnv-899bMQcbuHRPTQ3%O z8ucQx?^|{a^1F@kMUf1){M`{r>gb0;!|bR7LE9tV8NL2z@;!ha(d=^jhAoqXV4f}E zi;k@Zdoz}FA(I{7TW?Q-!FL(7@JAx@Brb%vZy`|&{B>S6kLD#R#&NVT-;$EzibD!GIkqsl=ng(h!3%{oBt zaZIj9+ec4Nb*gIXQwg6^pI%7l>jP6MIdMYGkw`&(QY~n_VnbXEP*Lm)min-MJxO7V z-j~yK+NfAf+VDc1ie+`T%PgoLv-h%K%Q3}@%`6N~25_?50rSQZPOb`*QH29rM=%;< zH@r+2jEtNm4)H(A9Gh{(xO;q&*Em$!K`R0h{gBW}>E5CLdroneD1jCM zlbWoQ<>Lqtch1Ohl;m@kLeQIFlMZ%f9K2k!7gTQfy3Jaxch@Edq)ZN7k8rb^BikkY z<)COFTdUoH=D|^}a;FwL`{?x`x+8Nm@I{=tR58@UP>!UT86-;n~u--&?Dv&{Er{l#M z9(Ypy`1lh=gh6u#IdYPcp+RPi%jr5m4VYrM*GTP1s7(&`EoBiS9t{>KI&d^j@WokO z{vPAQxruuRdcl^hs!c^&(N$E6MS4lfm^S^^QlSt`Zs4;3GeFG0nx0fwkg5UwEwqW) z(=SzobFO6{+uKzU0m?8wV9!8w+7FZ+70sDZLD8}($L0~1IVd2Bety~aHDBWt z^~ubjDWP)So$`(oHDWV%T7nW0-Wu_vag6xw2g)N6>in~}1|Tkmyw0NjO)|ExfzQf| z0ABGc^R~y13kN==IW_W~=xfw5)2QJz+w>n6h#i$OQWeDIKH+Idlx_~bWVTuMnYM!*(v8;|8kHgKp#6?Aa>QM zxjk<(0<;k}Zeb-kw!n5g{nI~%e>iN$*FSJGIw|AB@Ynwg%uKZ#!#`4T?!ZG!HB0;2 zy?n-fs%cbTvQ-e`m}e&$d05LHgueiOl<6q_ zi+RiF`xLr%)~be%R<#d5lZk;~ab9v|b6`5$jDsZKTw;PVbI-%BYYizMTX{n!O6C{x ztHV`JAxx)paJ@KyWBF@Iu67`B{OCvjhCh&(o0{XjuZ_AP*yiu}5^8T#KcMpg=cvs( z>>#WTXhJ^lFw4|Xld z3~+JO+M1qukXgCOdm0zByO%enB$}-X<;puiDf6wn6Uzz!Jm1)G z_?FQ@1?komQUO~$vS3Qh3-e;cObd#T)$c>Z}dDJFzZPg84QU z&@^|AuX%%+O3u{pY6DoI2j7OVg>Ypl$5@1f81>2hCS(e6|dxZB+6Lls%iNJ6f#5JkqkwGYkpX z2$R3_Fn0tVb8d-&rkg+ha$pY(E{Wb34 zY5?O)xSy6eyxbXYWT$VkPsd1}IFv6X=p;OHHyOh0YE8o>sCTHHxe|foFwXn}Zr+H2 z6EtyTJ&QMkN1{%UvY>BXg5c=m# z+PYItvdwK;I||LErU?ozKU}ZDP zY>k8sFc=!VK*)`c3OcKKx2~w+9$NUtxWjYsLFXnbtXKVCC-g|&wYMU&S+f-> z5@4{mllg|)C70tl^-n#Eo~rUNH(j@@plAh{RZkQgbwc0L{XnB?tJua4?61L@HStsa zefZx!I=_Ar-u{ej`rYfNYoSs*DtkVuA86kQMGp<3hYBf7c1h z)pjYngzyu|e#}RPGM57+LfBIPP=2;jSLK(bNbK&E+d@lik*Cu^JXo>}$OfY>S<*=n zz-n}Wq*8vGA#?yY%vVXlDg^EvK$f!LD>njdm0jj&kK6Xpv;W|W45i|py3UB0u{(2b zEudK2ZONfeoCzgTs8ci;~9cFy6CP zeNuIT9RXQjnxT$fZf%GriD@SlgDQpuuURtsO-l&ad4_nQz{Cp8zlOX|uzb#&n$mho zjt;_;t-XVyI}piU;RPa#p${PeEZ!O?$%2q42b9gB`)&2Ayn<_n7R1L7M$nkUx+8(sMvfhvryS}UZ_V7qtn#(|z^a5Mz z2PL=~D97zEQ%Q->wpC{Jom%oU!ZO1d8yqu)DS^&^F0i z3(#q9O~HJMRlPdu55~v~g}0z^;q0$bT?=eGtBE6~=sAMIhZcGdNA-$sgOHQ$+<8{k zys3Bq#;P8qG*VN<%z};|{t64Fia%x$UEx<@oCI0-u%qlH|KRut_G;6EyeVK9JQB7V zKZsfFZ`)ZaHd%aR%|Xy6V2sC6CqNHuqe!gnp^pTGV5J1QhOyc^R`!A$`nro@YkLL! z#Wt?qVi~;AxZ3GoCv7D$y{#4B?_CxywoJm6d~TC;S0Xt~vzaJ^dV6lv2WKKIP#FsD z{D@_;oRZQ8t^7`5C6_|}Ng^_>e$egr`JUW|4Q+oL-xr(u+vaO+CRA}(qv&Tv%MhZA z47_1w49TTE%!B{MK{E8Y9n+O@7Z1!2;E!UB|6y%C!b$kw{i-$M%gyi{UgiDl_)zYP zUH8jA4$VotjsA42xdP2gqt+@G?_ulk_Pg-*-SGtZj()1|i&Fgczwl%J*N@N zrm#7l0@+AfE@~BDC))N2t0#%Z`A~l+VUm~&JW_XOZ8y1Fmbo9GfrQm|IQnO1)#;(1 zEcvvK-WIp?$U94$^(~#~Zm8!0=R@Dbe12E;la~5aqEE}!eg(0J+OId#V;aRJD{u@z zM#f?|s9xmPuv1qRAuStjDi{GR1dyu(>ds2aP*tQE++z)ZhPE%Uy$%{gm)1NpFKPhX z$;UQO_w)i6Os%o=o&s!yqOh~tTKc>@BN!?jiGha{lv`&T13aUo{ZP!(oWD_SyR+X} z;vXRy&{RdxXVB~@fF4^|tWuH*V|f~$I{x5?f-6Rj`~3A22&sJj_6tJ#Fth$V53oPy zTK_7%W{to{LothomdAE^b7eEhB5VufU^qc~MndMQ-iJg#%kwt$Z# zYQnKh%nX#;G#r&4W>qMTW1k513nr~UH!W=F;#rEI86t~pV9kYF8iN#4;_PK z%N5kKlfT~`*VzELA5;=0FjDRR9@HLnf!j}<)fF{PvOO<8QiHg@d`{PBTIxuGF1f`~ zr;F58nJ}Um;^Z(ZWd(Y53RE|uo{rVwerQlAh&hcE#;sxgQ}QNhv*5utX(dNvto@Nx z>s7vo?W;+v+oer_04|dr7L|i||H5D~# z)CA#yWwl)cy`;IAl7|;Lk*J;o?Q0Im&wxNdDnj-gyx;As(fgnVr&E<7A*)!EU=t?f zc1UO~*RA?30^Jtxk=7~^fj@PSL}S}S)zBYG=Cl2BPwfD6Cl+P*h-$@-XvtwZ`MJZU z=&0qCsg0129l8Vk8c<|Fg$8#ChTw~wbT>+yX{&qQLdR@&J@^95&5KJM)pEuUEJJ&s zc$*moDWvF$R0ntceTw?nObzwSFkD+(Ekg^k5QD1j6wF$|Qkw6x64VGQyeRC;kRH7< zVId?jK;614G|aeBJ%&H@l^RsC%;2!YEnwvLaQij*k;<3=1~0dKMQpyZ9k1o1l)}shr#3 zY(kih0dtb47zJ9Btl`$Mdj-f*HjGl7x?lIm+UE=y8I%0;x<|Fl`qu8WDvir0fvsw> zXNtSAp8CT$(5j$!Dn<+IDWb^>!l2nkmK?MtS+*B-AEhD=L_8-*o0>ns9ablnHcAe; z66f04h+Co@9kR0u3WpE)%UMqK>g1aMxj-o9!c_FAw-upE)Vtbicv5g5_GSDTF#TlZ z+PTeew}y4iM!9)CK)4qjZdMoSD-b)_O~AmtC%zi-vbQCFi;}7lv2=wPwXKvZDkyd) zHaWj~6=BU@-BMW^52?%j`(tw)-cVb(>c{BPZ?#&h*n|!6^?}&QdY9Qk0cxBFlg{&F z_kzDx{@x21#E-_NJ(5Wc>i2A`quiy@;77FUY(vh@`r7~o&U-jNuZpn@Pm1;cO5`O2 za9i9Q22xd{O;2p@O%{)pj7q+*ZES0?e?WBg=da&}w+~KFuSqv}KiQ-$DQ&PvUu5Qi zEyr<;Jog{9DhXTGeh*b%jmo?`ig2Us^CWXo7acz(bvo|iYx9a3_zp4!c7oi~8Mc>k zH^bHGzbz>Yb%~goz!PhEC{GAegxw{HZ3hh41SqKW_vEf`SQB<|u5#dR9dOx!Mf8X^ z`VuK@59SycHN#S0R;oxw%`NbG3aq|q^FrH2b>`+Sg(}SlGrJEHoMO8u2V+p z-YJe-N=5nRgL?s9WJ&d>C^2gclCjz1hlOr40ed@xy!0NE@Au*L*GHS$Lzgc%`GWF} zzR2vO{5c=IefRbWXYHW)z5VvIL4aeT!3=OQh6jB}{b+qob)63Mo|2q*WbBe>FGm#B zxh+Qifxk;F!k8OiogVO&K+>etN zIe;pi#s*3^5Q-k5YGxC2Vt`GSAzuR|Y|4`INU>X~{G41~Qj!KBl3n(E4p+sNgo;CR z(6G_Pwk{ux;9P_m6YoX|y6902Eh-TgEz*bdOe6C=-yQR_Ef(D|iWe7%JmQP<)BX`QGm3FDll3O4WFyJ%)G-k);mk#J zJYa~hiL4`7mGW#T)qimaZ$`UIC@!el(2bKCLZjg}pmCYEQSQ31k5k4D-|^x`9!KF` z#Jsh4+cjXv)HTtII3nIU`~Hj^ZU-^$ed-uEGCc#A!jl_&YCwMg=ziENO`8)q zM6?L zCYdt+QVca$&z=>{L~8u7wn(27w6lOJppX_?uNDZCt>_ul139N$Slw@{I&-SPKz<2+ zs07a!DsdHne*5o^3EKz;=|n5UwW@&iw}>_kQrrPw=$Ww;^77wX+M8!%Y`Y8;H?&x0 zQIXmTpoqo4&3)bbK`CWB&bp5o3e~Jg=^qW@dK$a8ZFfkbGMO zm~#4XX?pw?Ht?3M4BVUK0gf(A@6>%+?qHd+^-CMGOk_u6W6Ln8uu7_uGa;aEQ?fU#TnT$enBK3Ci((2fRGyz3y2PH zCcvKVfDl9#ST)0GDxFL(F@8AdEnYmnsDSI8E;i_=t=5hh6vq*UhsiSZ68UXa*1@nh zUDkgk6sJRWc8TkY#fDBw%7LAEXt1LF@q*E@7Z0~a4oL-Xw9R5uRv<`$6!W?*p^Jgk zwej(;JksWdHk+lFw}9~H-c4p)HZPuvW4p7-0qR)=YiHLK*VbYuE41BV!YIDHZ``Hn zxhqc>xbsK^IL(CO7Y0E2B?xp$g?{FRmf(^~IUqG6WNjSTq(CiO$sqD4{8RY%|J&(5 zgunlHd3B+r`B>o#xs3TbbQ-hhj4|`jXIc|# z1_S@08mR+f5}!hlN^H zpwU1s>VlTj!Ab2ROY1mw3`GKXj?}%lD0INWqTfOUZa36|?LjKxKyPQ!WH zu`rdaf>uWa(%HD*}~z)-}XL<~_>K%PIw$ z>2JyzQZ;nF?0pj0XWF0YEy%6~38)b!kJW@Q#3_lEXD#l^lnI4r63i0eS-Ve~DVT~?~d z;}aJ~8>FtW>#I%qiaU zuit(zpZfc^KjOcyU**Gi1}+nY>C+y@*4gY@cE3*!Ho{IHT29ZNhe}b3*pOW6<$)z@ z=QoYH#I+=Hm1gbjdFS0|Dk|FmJ=Z(TS&Ux{BxBcMhtJ6SkY528EeI-%w*;DlK2X+3k@60$Tn zN$90o!Yr1JX>^Wzl{}RqQpK&D=>hbV(tPZwIFuur!xMjZc5x&rtbm|irIvGUmhu=B z=l}_8AmsBuXu~eww0{@A|7Y9WdQ5y9jz`E>M&tSl`VK#M;}{{yLw|Ey%x}WmXDa+a z=)Y8LqtRP?za3IX_s~Zv>W0EO3$QB%u)sd%Q;G*0U{KJ^dvOcp9lO+NFml}#)Tx?p zOytma;k*}Z5)!toWS(8~2MUs7_7yu1FmgTmkdLypp+6yu=f*cw!In#5?C3}iu4g>V zhp5++^^zu#E>i!_-<0r0Je>kmxI1A|H9QDV?jriX)VbPvmT?jgt_?K@TX<0Vm>*Kz zL$4sxO%l?jVR3N}Y%ojKfn|cz;|dFgd$J@$`BUzk)UVU5+TZ!#@>qySEO|Of{-@Sr zlyK1cz;;}gdbE*fT4e{RYwTenk4a9w;RCG)lZx2GN^o~=CY8xHkx+71cm;90&-L6B}bC7?I8MY8|-{On3Dq#fvn^(;J~al zUI%p=KM2fys}k;+82+b_iBU&Kh0V>TkN6hw;P_^m!oN!5w6!T|mc9&Nyc=y4AR% z$G~3fcUmIqIH~X0o(l8ob6TwCi^`!c&ElkQl}*|~We^azYbj7sLAM!=T{kp88Qn8R zY9O$c){<=`AS!G=0=CU?>Q@Lo5NrUVEpZ2Oqo2=pvU9DMUwdV+D}nM-jYwd@SFZHg zW`{TRwDlqA*tF{MB9$qJGuL5OAtOR!fh&@V1S-{Z=>sbTG_+~QVJ^F0M?Jto^gxwR zegoiniTnltScolQv`d#r}Dfn#z+3+-Bw zo-Sc9%NfD8Jimvig|Qii#6BXOTy_(2Y(%+{&=aC2uw&%2Ef__&aL);=?ix&R8Sf^w zO#zhp#%PDlENPv5t8L^AHA=?I(o5C2UTjC98M-+&VqUINY9zNv0kVePCJQiD7O-~T zSG6jajJh7i9NGE?n6tRjElDcLo-RUCR@%Lr zF}sG^QKu9F0?|~!mrBQwOet;|X{Iqrc8&ciG20kY3;L31(*>n3T8$Xg%)B?Zl>u2#h#O!^EdD@5?4unFtDLho z<$|g=RoNRQi(80`?E+I7%|^$@l98$jXYS`owS`CtRs-kDs=}oCAO<%oKpWZ;=u290 zB%LmDTRp_7c@WNP3wSN$e_K1eOF(~j2Tkbz%1#@?qs;i}rsKdB0z%C7uwNRm1gBsF zh%72>a*_F+jw%f5soF}mSn720eb18Lz5bpsqn!5rnvD@-{eJ$}e|DO|Pu@PEq&LPe z8BF|aN)|K^RJp0WG?bu_TBDHC~pie!vRT-%n?C* zxF`jI4HPV8t|a(c+Eu@cQRJ_)9X}1Ajb#q6zVU8F?}GY6*b_Ivb}_G>QPYE+Gd!#WCuti zf1i=#G3RvoyrEOFKZi^;I=?3GU9EDe-Rx|w)-X_IZ_=4Qd#@FAdH9=F(51AB41RZ> z>|Y=X#>V*NA{ipwE1OA`E_Kpobv`D#Eu$%frWky%dX`hEO*|tomg(XWhamJh=|~<) z$7rOZPD=}Gh@O(pW4biZWFcC5xVo95S6n!bz4AOO_nV9O~~p1S2{Y>!mr5?8ft`*;M2RZC!N0BnT}yvz?49S1_}vLM;6!LQUGEqMi8pV4aR*$kuCLJw*}DeObMoh%aldFGRf{l zPlf7!iU*d!rp7a+%0;+mpGqU|wBe$~T}{ELD?9|q3@o(mq-K5`E2#d!nYbg+qDX`o zlqO>PJ35dshx>dfL3!a->N#D_EsVSDCL4rltIz(?57_6>z&Duf(WZr{>`KvatpTE~ zYXn8tv%eoyxMeq8xk<9A8^aGaO+j=ElzRG}dq%RJQ|5k1^=9WGh&|@L9yM#w*arQL zp#!-`KcK6iTll2qxWWhX7ASz$I`z+iiH4?xlDt79@erXgX^k|5kpm`Cj|s@PylvKI zI;eDk&T~$+9QnP+#W>s3>+1eg$|$H$cN;3F$@h zMu3ADFSVHSPWJ_m&zdN$=XWI%d?MCBttWN>te@D@^h5zc6;<>gfntSr(GCa|YpTW!oeFUc9%W1i+J=8|YR) z>ChT4B!VLJS%{*#%y09s_?ygi|3ZV{Fh;wZFRUJ3q;fWNs}wnAAOj-`&L$tbFq-6- z?@sGEuyT_c7Is?RTLQqx6y+7$z?OhETkdYGep5#*7|kvNkQs~<3-0bXk}Z!$E@oa9 z{nn}3;zf->Tk`>(r!ZO|8+4K|_eu4Upc-b|y4`Jbp=wXC5GXezWI{m$K_{`f_@99a z(~7%w8V49b;vm(tTf#NxF_1O7KJowqhO^O)r-2N51Gmw$S zCdu2iRljYE7pcF>#wXMfYS(9}vLLOBSV)0d)XvEZPQHQ5Mnf5gX+1G(sj*69y_N|vd)2gOh3+oU4W zbY`#??UFku+}NSd-chhxIAv2?2c1u0b$MG^p(o37R+$Gp)g?CmtR@`Y0|LvaPCSg@FrFm{B z%eJ`W`oI)j;w6>`^`GKPcp)HNvf*tY(Nz*UxV3u1+A2yDlW%*YL<@AC3d**1fp{7@ z%u5Uc#@g+opL@iI7=(EYRLc%C(khbB;7YCJ&@O67ZI}GeovaWHF?KJVFeqPGs0moP z7WO}uI+ipOt`;@XovSoo31nB}EY>BYEEu0%u zk1*bZHcg-hJ@}m8xca~j5xtEjZV7y>oVo98Q>nikn=dF${Q4|OWC5mszWaX{{&wFb zU%k=Q>GgyB?*IGxsr>(U^8ep_|C6`R_nylJPdOXbOKN6E(1zye+MXU{<^suRqb3D4 za!#NQVRENWm@I}D>wMlcgSR>{OJDf>9j!GBZ|9{2P}v3BhcPThU7A}JbfB+O3%3#W zXQY&QG(-eS1?rOPYDzEh-mYV|W45q_fexFlaHcuu{W1=U6y&zUFsfq4t6NVsek@~M zsdd*6_W`Aen@X6x;SP93w^DxX7uz};!jD5%0Yz+AH8mrC3j%}`a%q!oQcw)z%?Xo$ zkl9q@I`k3TF1gQ~K~Mu*s0<3fbD?1HW#T5z2f*k_UM*cX7;N~zQCfOD=5C?{EflKgXx+#Ew+(ufvm zXC%i3Gr4{jK_XYmr+V?4W%TDmNTA8!XWR(iSo85WkU{#yG2olxAM_wsntfjDI8w;O zmFKtE+J%Gzo_Cpu=?W&Og+)9U1LYDVZZw|VtKo*6r)ylOlea(~*|vi+y%6~&0e8!J zuplix?6uVZ*^_-M$WEh5lH5wVmk$86b(qBE*O;pmJSn79yA8Ya(R{!UsXjBt>-2OS zrCQ?NC?KtW0?U^Xp-+#TAwnREx}Up6eTR7}v5{!W8-Agc#a9!Z)v3Vg#XboRLa5r8m8@O)ohPS>++-IlF+)%T3s>56*Av;twtz*epqM=TEn9K$PWh1b}g z!XS&{4b`9`d!YZyLs_5~KrAa@vfZf0O-!>dl1oW0uC~r6(13fUW`k=rmbErTxCbzH zKkt~aI*iJzC}?Fi^v~ob##uzle z0H+Y$i8Lw2({31Erm$}{?O3RE5737*+B}rLTqJLb$*~qHn`76=7(>>oqIeH_d3z+l zc!~2P`90PiYqd#I?R4uj*~xVcx8>A8ET9J3uR)%af893Ep7W}NUBiPZ5wPd3G7EAk z(=BZc9ONV}3UP6RX8AX5sK-^6_IFvv)Pnu0R;L@6?~ub}8FzR?3B@Zr0*c{DQLqBj zH6C?Ely9s(8Zh+~X#@q%n_77u95G$oK%Ul$1vyEAV3Nvcu6P^o+)Vl_TN#4Ch*?t) z+R@^>pao*tuF>CtS#m+NB;=~8)2|9y5kerS_bq~yE>Zk9e`9!z{OaH6s=u$V4u#B* z933Uw=nto7M0ahI-%d`D!FJWV(*ubTe@N!n-cuDJudc4*mT<`f#Q17zY-+B!D2hIG zoL)c*2S;}8My3cdZitB&py9HW56J>oJ4KHlusjPtQuC6OJ+Ga+$V=T01ns03aXN7I zM<1JH!)Xgxm_Ev(YFXE|+k8op-8DFAKDBQ>UM}Q_1NYI{@eZwdylkr00@@&-IJrD( zzfuRIPcNABTDXCFyDR+67zCm}k-!>K{rdO5Wfv>?KDLGcU!>NFWS&yJ!|B1U(ImfG z7eS|Y(}7o@zOa(a*BlCiR9rc*qq$;B7`9$EJasZJml1=Tm1;mhFp#M$r>aZd^^${K zZBwte)jAPi+de}x->6k9n+C{V=jO^`@MP9lBJ3qeeK`?3#4U6VyrUpF>P5NlJ@kQF zWA@_&(u&jeb$CL}@_mM{&X+X_@XdLRbX!9Tdw@XM{taaT?D7ycf;h$&m zE}U^FB+K?`A#tE`c~^KT#ltN6k8j^nqxSdvyWfP@FTIWMm(v5}ZrMmDIo&(mD&U27 z>PrhtONqHGO&(#fVHFyCw&tb!G?@M1Sm0a8*q!_ea{waoAy~ljqfrN`e4QW#B)6fD11H zxOhm)fS`unkm*x4fPxScixfB#9oV||bRH&NQ-K|zW2V-rSzy&stXU3_nUH!67u@Th zN(Y+zYpw zC6#Q3c;&<$mI*fLy{pzBp%^73xvrS`P9-7!AyFCYZ2XTOy)VH_vFzGhcWvn92s0<^YOa;0IE6{OXplnG>7aE%MUw?$HR>KD`(tR`+Y(^A z08qchhAiS>JEB&u{kFgGtYS_DEMg!qIBmNo_yi0NN{+bWEubN$<$Qh(A@drI>v{EI z%qXjlzv!rVZEKmKnGLJl!@vpOuWJ^Jm?BTco*n<}{E-`Q7C4@iD{1ur@qz2jl@v}l zi6|zTI)rP!NPXXECN{{~!V0dDZ?+vwHPt?x!jJW=j_Ks-ZBZ4xYT{};ltedWr&2{3 zF#Fx936LNiDV0*t)`}to-wLOw(yZ2Y&~dA5*aSTBV4cIFS(;~pN0`f06q-6o70J>h z&vNooMhLtSAThu=B;8Q6RJ(C=24Hcqvv3D=0pK?W5=C`^8tjaSXBEG)qv!9!|Mhpr z6aMRe3zvODG{xwEU@qAH*jiiH56tmF4qWz5I^;WhPHXQBu`58{yUUcEsiP5dv?mOi zm~{t8^pKHnW{n{!((6k0f$SqW!tDHz3Llh>n**CcIwpJ%!(q>q0OD)+SLuS;*zUX; zgPe(yXF!J6DcYbBxDMQ;bQ`;d?D$wi?FIg1z9rm5K~;=;T7QsK2GK0D$9N#JwX zQh1*7SwP?mWX?(B!epM(aTpA#m8X%j63*&|`dfhMwh1OVvJAk7v3-QDqKoR>q-dgI z@p+%(fFwKtgx3{gZQT@L;3*0L8S{3Q4=^G_>A{2M8WM;VR|p6>oOc@B#=) z6&O*;HUV0D9Zjl!BbV?I8SQ=;Ep2Q^Xy83XJz1~1-%IX94}1|CK&5}tU~D1V7eO;8l7+#A;?I$(2WHKN~p61$7XQ3!5{6uZ`nINqtwHp zJMs3_U;h(7h9ACR>M7y%7e`^SAHV)u{X%nWmE{Wb-Nd3l8~DyO26OgKVMEPF1eLcK z;=;WssacuKk_W2%c7~xiv7NJOo+YsY5E*%i>fI<>5X$ks1H|#HV_rZrxA4K}j5giT zn0NIrLf7GESq^+A&f$zcv`%;I4g4((0;7%wOIQ3Iv2q95vpNxCk}1W11!)@%SPRQx ziH~*AHXD_6|h zlXF+XJNWC$Nm{F-MRTgd-E@ZGZ*-YVnN|`j#5gA(t4E|!93>})q`L7y6cH{lw6GL7 z!4{}zNC{33N`=(XaKxq{K6J_>e8$}EdMIL%+sOG`t8!4%A4y4>9S!w z`0;`XBfs*J8pwA)!sEex(+vv$>!a;2v^#9dx*m<@(JFGrlkB_P7PdQyE>;bFAg~1U ze$x7L$fwc`g%Ff1`$fBtAU1Ml-SiM4qFR7dTT)HY3FktipQdbvdF$2t72*wqBUWg* z-3d!rC}g3r4GsLl74$pIKxeyO`g_{9P>2Bcq68(|uaF&Zg-Z_05Wh;fV>aa}bWqz8 zyE=s3Un0`|;0#f+;CkB+7f_08xS!$zH4f;1I@3O0$sdxFXBZHaws>%$`Q7U`;Xmlh z%{5N%p?CY*X@xsz!F+29zPbS{1>$nPCw62Z3VTzV_6y~1l?JXx1f22DuiY! z@Nb7+#_X@uRaA~!yH@~55>BfQ>`KfZTHVBC98}2tr@QyqKnh5tM7h zRmw(5jW_bNzB=S~7@*rL^>fGgdYxR)IlZX2F_rEi;_s6vXX+G~x_hXRlV3t7$q@3i z>w(i%Wum!d=|$m-m`F*UyrL7Qz8Jg3D<-Ad#IOKUA8b{e{%hgmH*bcp`RVJ|;PO7B z$Kq}1v~x9Bz6ZVkeMonQ^J@Vjpq<(4PU4r8^AcGvEkIz|Mn@k4uvmF|h>AsI=tw#V zSGonukF`H49D~SPKwT~I5zUG=on&JmPiS0>ULw+I8URe_50us^Z9O_kyIO~uoriv1 zAooofad;j;F8$Drt{`w{LPpr2>U(2neu7jkeiG_nf4jv4@@bF3#3AuNqESRQrQr@P zRN`n2uClgfOeC&Ux2M7u7?)`WVtRgcX`qF2*{zuiN9W&06#R8rn(~jj7>< zJkcq~7f>gvP1qj-js(D^{cxq#;f%MkOYtaoMbty8xBF+GS5%;Dra6qJo-m!0Acm zGz^kmCVlci{Nh_%Fbbw2r@ub_@LUciN`Jr+Kf{*N4D#Tgpb?1uqdLY;A z1S4q*wf!NT1}vQ&x86wEWHq>0b6EDnBx|%&tEW|GWp01yE_Ww+UiRtNVUyizQKZq9 z!+KK9aE!yI#S4OZJwpoZRIQ!VTO5?F>eFXiyB?6Gt4e82gpAaF)=0&+F1p9c@)5Wh>76rccZ+~p-r){Q44w}IUbn2dN+TGFL>_8(^%zHgYe!- zZJs3aUX5QfJ0Vr!O5_|6HlRRbTh^UUVDdGHRaw>CLTeGu=BorTChJ;6KM6)2=#WO0 z<>RB60m_XoHv=Q2^3fFaZf~DG3~w~g(!o$qkQrj+=<4f8R-1Uwd_`^FR@&3IJ93k? zO6_vs`7Wev-S&hE87XO&%B__gP?9=>Pe`pha-TpdsyV^!+11b_V#4TY_w%3&bcIRw zbgEF;(0Qx^m~?Q7VpXYpK&aXBP-Hh~J;`s`JQec7IqXID_9OK!P`KE!bqRyPmD7v* z)9$e{NgRD9qGweqE;Eyj%LuIEU?g7PsR-77@VGEt>2ZA{eO62& zhBaI2wW=I=g5fqpD$*Ren=G&A4GxtoO!Kg>X}vK$cTqJ&>s*FHPHG2=$iZvhM3rH9 zKy2tMZit8#3p!I0RYXH}C2?DfQ47tMsFhJ}#2n($P)%+mbcyT$>zhoaukFl`bk7bA z82kYyLTJ^{%y1xa$wc%oC7FhW+D@!e1PPik%4Oww0g+V3t4=aTTi7u0;*1baaf}Xy zKPMlYwBcq$J8UTUHOw?8@$o*aLDT`+HRR-U$AL@aJC5_(atlDC2HjCP?;)TT)7msCVI ziY@ld&hu_lIsa~5d@hHMikvU-1dxN`yw3X^sf(+7$Y%YGSvMu1b_zVqD;RVDTi7@r z>{`mmc*RDpoZp9Nd9GI=T~whX(%=&*RnK&>RC1)h%otn7b`X{brD%3-_F7KO8{NPF zxeCs_YFibO;|7Ilm0eKYIRmvG0R2-~5t0>6FB=OG-}cbdp{~Luyu3i9jnKkDAFo4# zN&+*0Ka?j01BS{t7GO%TK3g&C&8WmKqVy_iFR`RpmpB5Glgg)+z-|qv2`(5;G?L>| zjksWCQ$cHphFM{YIkQH)6Nx7?Qn-=x3KQ#jc4*Gn+b$Wb%R13|Ovh4=F`TJoBh?ht z*RKZE4V5)U5&4R}BJa``)Vj?!5;nOvRJ?p`dNuZRlTTVVkEF1ZAh878369@~GQ_2C8 zuo=v$w!K89%iIpV)CI$807WBw;@5unfKIOIjEs%GTJpPHh|7r=)d0``@Q?p!rp*tX zL(8ukC$2$>^A1yc)TXnZc^aqn0p1$qKB%Ci4nhTbxqjH?L`fl0w1!)TZE!Id3=qZ> zxcaQ(=MlJn&v5GD(V094MzisG%J~ty{5$mq0@ZQXr2xCz8bSDmwFHLQAaxT|J1n4q zz@15x@7A0%&wNFA+&dn7@08m(VsO2LTTjs;vjdpDk+8C=S|Nh~E;?~CaJxwAlp-gH z&8b4xIJdBY4rW7idB2}T?eGlVQIRmPs7(gEl~^FJUq~vWA!TU^33dt_XeL)+ZDC8S@S4P;#e+V1yQ*cFH zxw+_{c83X9y6sb;4E58QH@4b1ZzwdL^}r|tLo^OdHAxGOJWxrGIzH6Xd9#g{y-RHa z9(%(QX<+_C5hhld$^OSLHjq#-xd}tLkqN;qZB}k~~J}ZrA$!O9TD9 ze+qnQtVBBnzW4#Te9pq0_sKMW9@ZUYepI;`P;O46ka6N;;bK2Imm!2iG^Pg zn;W=(Qm5iluP{Wm!y@UO$E1AG78fO*AR@7sC=jBckfG8D518UEW1l>~LyfG>c>+Yb zu&uX~{;`rSO;BdvZES&rlq|>Ma)J{Yn{X3A^q?$S!&9~fbSSHYJaeqA8k>eOR{LrI zV!1f_I!WCZhM5Qq9VB)kF>;Km%p{qovTAp~Rrw9=1X1rOyjZONi9(VF$&Fev*>wq- zT05g!=vK%Z&S3|l`9=uCwFq~1O}7m0>T3@6aoB+43`@R~0&Q}-mJc&TAuHw7Dlea5 znx#O)rfqcaRqlzn<_EoO6m_STvt`+V1{Zi?@+^}R6Zm7PfK~qY?roS7?QHZXnK=yW z4tJKvat7~$ni(nuZzr!B)uTh|rLj_j50+LX>fV-!V;GUT6TCVtx?t=19G08}!JWPV z53l(I|vGp)n6u9eV=%>wBt z7swwy@rLnH-k>`B&r20%U1>bQVHb~wlfT+&1eQ8)Cz<+6~G)LrH~RXWd%6-D`y)UNMydo&d@-TUA&6cYWtIR=@13-dk9!j~!(7uz;yYTD8O z2FFpzn`8k7eu4G*Ea@ZvNjsZ6i8k}~ZF*tz^4ozZ3bW=J$$v*-*IDfssbmUWKEm4M zAzNfdW45IwnZoH0spXXLEaU<3(1-a1OnI_ZOnwWNr_Ay>l~SX7y^@VRK@zqD*f~6b z)07kc%(1}`e5x4c$zm-0Bu9t8)p5ysvAaA@3o)_iQ5u;$kLGqoTzW9 zYFv_th=T%ldBA`1jvT-FZS z#XC60q)z9Ze9sgsm2j%)7zI^qr?~0XWHbb*{elVlqzpe?_2?n%_b1!Eh)SorSnhpR zC_cub%+67_oiLMtDgc^B1B6#6dK9-RZgk*G7sHCDw@X9BbBPW@Jb_~a4aJ`^5B~!{ zh9ADcj~?a}1(Q2b-+%De|MdD(n@8k8xZ!Are8{FKA;!9q?NekOs}Cf=rpY(+?ig}t zV9fI1T1@7q`M?@rL{ldDCPzl14)_ck);pnh$b~sE1c^YNzwIbl5)v;Tj!>x^4ov1K z#cP|Y#_{q~<9hUyB4o`wH6!7C5es&*I9ef4FKyTQb? zXb@T_Gx6vWYNZR;b(hd}VyO;5I&%Gikd{sziyo@508%wivvw^uNMezGq^hUZAyemA zoBv@iOserSOObVgG9u3><h( zivPCx?(fMILk8QV6?;fyyobn_uRQ>lpu%b;#E-)NsUQDWlY{#k*3Umb?9cG}ZNB<~ zowP3y9rMLUeCMN_w0<1$Z~y%IYsIgy8fcU5#@AOCtDcK7NQ0UZD4Xhw% z_C4fw+iuRzo69y_f*F{s2l?hTa4v+XTdnNBnKI*fHBE!%B6r9PDv-M}QOCeYkI8j} zC?Hk+FZi|?!-HfwE3S-V`g%W@W@X0u(J;1qb$m%db*Y>p(O^IWA32(2wAL?BU9otX zn9Kn6ipr>3Z5WApx2f@GSPrBjHB?S+c(G^AD6X4IS+)agK7ki^uUOgTb$hy{2MRsg zX<$|4%p?~QlKieG7!6vG18^mQEws5RYqeh02{Qpm!=88cH+Iwt8k_wq9&j;es#W$FQbR5)A`>THWg4M>>N|=I^c7a4goK_U&Euyi-=K;cSBiKRM<4MhuS5VwQMw2&6?-xVs$lIlWS`?L(YKR`ql6x3Z$O1 z5(HTz6$BbEOk5$xJY%@nWzD6vCH@)J1myssCL+55(WBlZoGiQcj%vT`5?ejY7Zqe8 z@m&uEBy|=>B&Vg186^JUaq8v=pkzsjXT#d_fLcV&$k+%Oy!wR0l99L?mnCU;Z_IP( zjCl{!j(#uEx&}feidQ}aPh0}@_;cl!S0x6i`cFQqPf z`}FNk-+%J<-CzH+y!X}XuaJHI4B00>sQL5HK)I825*es|$lvQ<5uG+jTGap`G-O`L z8FN0pNS;M~@2EYLwxO=jBktn~49(r293&RoSlgs+v8l5pQrjmMRSrf~DqOm)>~zmS z#dR9hI4Ygd<>T(3Z-|K0gs6uRK}2rvUJ2ICT~=dhv(@?BHq`KzQ~+JahQ-6n)9i*^3avJAuZvsYqD7p2D;cwgsKx@Lle8WIu)%qO@r2WULlU{i-=`BFPicaJ-z1m||^#-?x~ zsB?n{WNDq6&yUR6k0gn{h?D9w+Ugo0q@OPn zc=O;HKpm7LEigIFwidkic*%!5&UPEiqmjNRF@+=u`SZJ6>>I^x4VU7SWe9l|-iBaA z1G#GMkipf?bY4DCaL0iD)JzwJ5iWE&l57V?>${4V#LbnNdiDeayG{Pc(d~q4#SOd5 zYzs+WIX;arJ*5f-xKrE^`EA_y5(;D0w1ev2{n_ZDdx|KyfstzEt^p8DwHFv(jgw!_8FV0*|Y&2*+1V zZ38BYw%q&iQq!~qqHw44Gm5{EhA*8mb9u#fw{e8uhbht~s+;H%ePur(Nd_(uM$ zPt+OkzlOJ8oN&W}r%BI!^r3FklBAi^=>wP{qGg)GI8p9`*Nw@BDw zKU1Ead&Bp^F9S3^PfO~?mQ+Bwxh_^83C^0e+PDIK=$#Mf(kF(dy@8*A5^y^BvQ9_+ z1)Q;7*_gK&6V7*mu65%Yu7(;?`!mxF93|*b6GDZD25n@=ac^$aN;MtHrk(3Wa#YZ= zx;S|SHF16ms^9njVl^zX`K7mS56SR#K7lagS*5ZbK@C@CKtGd~Blm8QpuGb-OHv7o z8>U8mLtw#6P;(x8Rc`P_nh`$pF$8(C6_Y#o*B|ESH`I~1d-ClXN;q8M%V(c(mBY2w z_-F0;1`t;WpY2rh3v4FjTb}al67~YFOmq#MR6zuCg8Z=GSRPpoPs<8Ob{9UA47IYe zs<}2kC`vDQwpN<~Fv%lHYD{)I6jNB*;Z*pkRYny8<~f80&^|1Hx|o~Y%Tmgw)#%n# ztS6Cb#^=n)I3@VpH!jt>?vgIql2A^njg-d-b5zsL3&TzJtL)L&KWe@R%m6+pyYAEQ z_N5w5zkMM8{X=-mh@of3G+E0+R;%FzRQFQCf=aP2HSb1Nj)tB|eu?%3AaP~bWrL!u zX0{*TA!L1d=;9Fr*X-c98i@bh)kP97*P2u|g+*O;7DVAq(%e+2t*OfZO)AxBm&nId z?0Dypju7STA~u|)SEunn7l(jZR#vItIY*WNSi&ZM-@5_2O1I||v=+Ey? zLx|pM99qxyk4Q|8@z^2h)FJfdBzcJ2ark74L>E}VP#PSPwzElFUW)CXlLPZ0Y}qPGNh_@2BwS~=Fcq$RcS4lscKKCgwxdlc2&wA-666# zzM)7{9=Hf#9u^Vm=?79wVYqXP@sc3zl%gpMZk};+zO3vOpA*0z1ubFJ^Fcm9)vrP+ z7`SJlU$Mj8BK*+-7_>hFh)4`rVmy;%q@#9=>g~Rp;u(6xC6Bd#2>*xt?}Xmuhp*q{ z82H{Mv~{JRD>0+8Xirly0@-pjybOa1@8%&^VHl( z1%g{5a~kE?1N0>o1qqf5Dir(Yf=^Ns%PotUTLhj!xSL1h?K{{tp8YJqeYDvCx!HN= zL*V-ks2Y^cOEi3~RXnBRMn>mFKfz@yR;ntE0Qek4#`-?P$DoC--E0rQ3fPhJp+Dmc zAau3gLUortoebpKEup=ix|`}}NeLif*9Db}y@4awN*q3^%Msf9#nAQ+XKn#iAUHP8 zNzWX6)mJNcWK5lIjia(*KS=cdb(qsw?{fha|L^2(4Gg18HkK@yX(=YUJASSi?n#e|e2pT>G6J|>vJQpyXX6qSQUpYGD3wxFcG0)B6aXosi0F8j$9<;yk`%CQ zrZrjL(l_7b|0C_)dSuD2GqLyl6}x4RC5>h3eLN%jfB3=J5wSBeG9zL~$0f6(UBb)=g*wOth^Yr+17^vh=Dhvygoo=FPEv#}UKcEw{JV`j$Dh>W* zcnB&Nj|-p;Q1D~H3&T;HEmTBoVRk#ctN7;A8^GcXGmL>9`&^1NVvRabTz-L7_<3oAOGJwHLRO&2*I19@izgUK4 z_BLVmI)(?;&lQUyH)(Y77$Csf!xR|I7dIs}G`db{(E>@LZ^PhoN9DQK)C5$V%BES~ zaW6}}DO`1iIWa#$HKk?&0`hE@dujz3otRyTTBsuTx|pZamt@=l8v=kgR`4dM(otJV|?Yjxgc1i-k=l?<_h+u$P)7s zJD<9>o9-Q2>}rTP+XCHcpni8)CxWb7W1YQGTm@nEl+)4+TB9zU9fB`Mq_mdK1`zH5eo1nGAWC12s|QW+cYDk_dq~ zjBi~q=siP`%W*Q$<)mb944)cocPF$YVv{x1$}TS;X&@vnxL8yXoF*`j`Pl< zwe+7QTS^tW!^FX>)vMGhXG%+twJM3xbD(=Nl_|*fI&zQssD5_P*Dwu zeluDFbFJ}a^l(hU_Gy5+UGvjC4o6#-2=t*RM2wk|=;7L`hsv~gwaFCleCRkvH^ zj>&F>RD(;q2b+#SKTT@c^nzpGMCoH4^5m?S+C|Y&0PM^9P_ZFb@HD-4W50aOzsk_? zoDu(%KD*0rpKP$PEBu=4%N;0#&>1D;fnah$5NzhJkh@qGR%2!-0<{TCg{l>BEYq=! zLoK__6ot{{Vw@$6veUMKRZEqGkT&rOfngy4w3uB1!%aECyi$99lHOJBHRO`{kY1Yu z*DI>#UQ4CAn2d79GOa`mS`vqiL40D4)&~z$s3mloVs~>G2g`9UcL(%yDX}%Adj#Wz z12piwHcO2oeA9@!=(pvmi_p-vFr;Ue0z4M#gIurj67(Ocm>`=&uDx3i6n~qQ7+9{u zWGN9Uc!al~(Rdxw7Uw`VC142tLz86Z{iNiNJNbwzMs4*|BcG3bD;9RD*7!tKEK70E zNAepgfS>TQy!&{d!l(caD_rqZoh}5}JxboLeN-&Dv8xZYFL?&5bgLxqzWj-Dvb{S0 z5lDUvgek_Pf|l@yE$`ANpFHhQBtE{N^Jn<_6UTRb`TjX$>uEInWqA8G+Q(mncWE^L zHvI9g8DSul*zWL!{$Y>ITVBi6tVkHUKkm6v+bpbUy0h<3@1ZdrTmVAjz{s16j*50h~QWO`5DHZ@+AuX^H#rtbIxPYU2a?9w-USEkCmD}w?O`7^5x2r@hN|!@4x5NE# z0t06xULek@nT*$z$Qjmc9wBTE)65Kc$(pmM0aRP#U#cExNF2FL(3Y^l(x}zl(~w$H zT^i_1I-_)2JVG1EVTVG|ppq31{w|ewkJSXa8Vmg%dn^2DKl3VW-KeBpC#76r;f8Wf zy02f_z6cmznwE4JZA#wc>E59*Gk4=SUhJu18V%3Aw-3^w@%lAp*Y`14K2afRhc za=SST4b0V*XNmn$=aTJHs-I2JP(oVcHg4g3CKmgGw*Wt&mfIMs!7x5-OpEe_Oep7^ zO$zw2OGt^N&89OLc>sWFaImIt&8*kIy=Ey@MtKY{u}z-J~fV= zuypDAm@!Z^VQMWV{U*d1^lWgljmE_1ycwWQ<={c4jK{7Pn^wZS z6(nj12@E{uoAbt`!-HSLddyZp+{ZGZfrxId=cu)HiIQ1bXFD%yV2P2N=#vn^{_uTA zaw=Gj`K&y?S>p!#$d9;|fZ|l9aDcqY;=RI}+X$MeKm1AJBdJrO9R*mz+Ojk*j=K9d z;ZOdgGp88 zk3x2le}aM=CB`lSLET1JQ&B@S|`mj`y7|PHs}f5)%A1zxqFX-N612Uw?f3S6{vTf`8Km>=(yZe?h9WT$E?Oc>9j7 zL_d4~EWG^!;D&DpY7my`BPr8Ge6&e-K9lA~#VQgzn)@xh7)4;+aR68@oy zh-(uRn(7R7!nppA-?IF?xqhsXl_j_*V&p(#wZm8XRFFuHyyqg<2JD&!GzVZ9sHA&Me34SA}~v;*d{bo4>y#E5&$!cL51QO5O0iU^SB*$qPT{r zsv*RD3?Po93c{$!sxc|#msX`JSTLUfo5IKIEX7`$NBeDhEDl~@jV76K46?ypEhpdT zmv)sq>N~2ZU2r4iN!X@5EAqaAT#%`Ao$eSUDU~Y~b8CPUB ztF`Ok2lA2-#^B^hhefGiNMNy|gucn1yg~@IW8ra<&)r1T%em-+Q`ZF#uP!&7oP6@> zCbd7b(K~C*i+}-&z2KlGXb>#`b(~OqO|1UAbv-876YQN!>?SZ9Gv`_*$X&ZjO=~y5 zl+yo%-6e_(6GOrJ)!9ra5b-S_HDGL3stiIQF(h_*#Zel!Csbpf9?B(p5PE@4mmL!> zjv!YRB@KcVUedaixdOmb#N9*w5b*n`#sTvC&9NrWM2cGH6SM$vS+uj~h4Uw}j*pBL zEKk!JM1M3$(K&OEah9f}+T_-F%|pe8@f>fYydzGp9iUCserk4_jY*P-z~WPRynavw zQ7LxmP(@YI@g|=5trY@x`6a3L4{$~-8v3U)rXOiOPmeJWTsg%Kma7$ zjWS@qtn+Bl&ehO{tV(&m2W&Jg(lW#-E&Kr1p=Lqhx@FrpUvp6hAM!o9(#tgt2boiG znDW9YD7Gn`$>C4kadKVaYkG3Il4?Fto6}^4_c+gKQrt+a6KL79A7%>SR zy+{De5sMI4D*)<=3JOi4EsuRoN)m%I$iHR1Y`wDDGTP;oGVe&Wr^XOdlK+D=_7L~M<1c*}ZS8v5Xt+YM7k6&{l|9z;&x zoTv#fwrbYvqks4B!e9NBUVYD$J++#^nMbXtWV>23m2D%Uk9 z2WwP(0*B_YhjL{0!r*+sUf|^s>&_jQ;dF&k8HdGdcS0HNiCZ<#Sj+-;zO%hHdG)p^`=!XRnKz4O=m2k<;SUF_h!YXP$$y zfpQh~s?H8@P?>|q8lYa9fn5Gq9v8g7UDqtBi?w|(h5Sk@`?Z*&dgtAFQ{T>oM$l@5 zMxEtZJ=RR^_N=uN^Dz!GBmAbCQaO!8;;4A(Mn0|*CR>H3*!p&ej8GtGQ%a&em@RB- zh~W}Gfd)x|ck>M;e5ri#g21M&$aSB2BT=@WwZ2(s%vI&C(OfBe1*(G(j)jlJ6*U@c|<9KvFVC~T%Ua( z_JAiwi5akx&8B#>)u-h0*(J?k`iJwYyTMdWnypx`XuO}$MJUZ5xp&WQbU_!Yi^&Z~ z3oU?BwiBeQv3GK-tRpdp)&#s@rMhb1*a4&37eW4QqnE+HCL6sJB{87^MIFBc#+3vF z6r`5-uq9IIG8DNs`yA@jUlcEFWAeIIMDNZ~IA*?}ak~upO`2+O>bjcBBMv+r0{b1+ zeUA>t@1-UKME9I~T8|Z>!G?YOt=L@If_EF89Mmndl&?}%m0Pci7fDlQdeg~UT3NQ0 z1JkJA7x^up@D&)ZU8Fcf_O1v#SDsKU@dOP~>eGT&2_3Y zb)fo4JaSeOyMC7-pk7n0I+1D7K&pWyM&hAi2C|u@vUvmW&axF~=$wuL$X)y1cAs%$ zE@@;B{2-DtY99$*YzwdtUxc?`pFaHW;r+L#{~7e2-<)=6YpU9(eR{YZ-!s0spxjB<;N$Ps1d4d!)e&T z<&?uLIRh4?0^@*6QETh3yvaKS*w0dvP(E@g1@djQLC;Oz;d7DhqGmsU#h@M7qp+9; zyL6ax4;iQiv;Bi)ScE&XKQ1CW8|Ur)!DG2VSLyK8yla9tN0K@UNQeifFIWDv8bk@!jk_SCdqe z=8y@mKsr77TNyD8w-T{?m4CpN$<0;mgA9Ev&A^!^K(Guxy@CZn65p{o-V=2a_9qvC zJ;Gq?1PkEj^W%Y?{}Z|69_wRcb(j2h!pi^5v!stz>JQl>3nX>Z^gVT$Y%i=LDUg+a zl6p|{SC{)o3+>F#C8OeVpDhO&BAN#nS&mqo_}w$hAq+4I=2r3+?!ni3lCsys^4dzkx4Ckxl7^iC4a1U*-4o6>_X(;X%10`M(a0y@BX{XrD{#*^LuF z*9+tgT4;sZLm>4R_Asq#4faq(!Jw;^8;m{dGU15lj!Y*y(#Jkwy^9G@at%H(NV|Kc z03Pz~wB3sE!w4=B*^AhFrMd13>e<3>hH}#5N%#aDB}RaLc=SSscz@mv32 z{|^6ue}FBf=z1r8K$hWGEAJS~zmO{TY0VJdYDp=Z!>i>Y zWmg`2Q{GN3WOR86Z5Gr`?Tk=s_5LbfQswvA#;0-=)o`xD+^CtKOBc8$%BkjKu_oz6 z#RVod4qg1#uTictx>mGYKu|yjnmt>kAd^G%HLp;oX^9yG4aRqns28Qv8n%-HRd$?QN0sWBmQ%ES5JhX1-m3=+rh!ut=7N56Xi3I7gXf1-TX6D8%pVl~GOiET=Ph?@$Ri?NF7mn+bV3+*pCEIxdw?=1B@-n6x5<_^n=$7Ky#fWaD3e z9(=vp1mCCuh!bj853|Z~Lazvg6A8Veth@{xH+u`FX~bfwANCA0kkoT@0*T=Tf06`H z<~pt)W2h=QA`aJdzXl}+V0!Gt-P8VcAh1ri@vqn^gsFw>RQS=GblpyXjYpM&e~ zA-oa8y|Z9+YsJRC$Mgetd$+KJE0*Y{D1$BC%`W9%V<)Yy8jT8ttL4YBgPfP=)? z#3Ub7XBBoU6YWSx~tA5v^~i;gjQH-J@stXTlXpZ&~GmYatE zoC&*@k9w$pm#}S~a>lPV$HbW8*exqX9D2@DrV{9Q|WL8343A6MJsC+s?L2Tv& zq>KeJ=aDLIQstD1oUtrY#yDhBzNtBZP)#WcouS*Myw@%xk)SMXM}T@SpoBibk|C|9 zEH}v2d`>_aJh?cH^iXATId0)aidU#JS|OX}COFqjKFF*GgB6)$9U9+Ytmp$thkzRL z&8N2A=w#I%vLl|s6*Ozf9N3?_^Ntld<=vdo6Iu)0jkw3rNXlXB_c=gEP4~nAzwGim z+a=IE7Q1*sj!No~T&b<%if<6((M{#;5O)fk>ZO;YlM5~}(P<{S);Ayw4vrHmLIg^5 zI;8*lCsK|=9WlVmlhaLmWDamP6#W>9TYx%BsOR`_q$$7yhi_bc*`Br~c&FK8VR5P6 zbFd=2s0Ir`n3qE2FAq{-)jKo*cuM(Y+Uz5wj4f^EK~$;6j-3lFhPEWHs-c-8M=RWN zeX)gd&E5Eb{*3X(=k66FHYpphIZCvH1g`qf!)?^%4d@IxAU&trsqlNsQ5Q<;KZUQW zM16VC?0)ny-R_I`-@SeL;fuG=-+mY5!4KYl`r*g#-=_iR59MdQ{|cpmlYi+4P)V=!mjxbou}zG2Mx60TucTUZ~n9D!{H`LG-Nl0 zYU>VPNIAS6RdGq%5H(xKIsYVqf}3zVT1$murbCQLD&#vEcyRSTN#p9GF)Qe?RD$c9 z#oZq=ZnBv$CY;MqfFToCz68P-ldIg0S9s>kjCtwnZ=NWNP3%E1uGk_ zpi`;%N;!9kL+pQ!k@xpf(|;9BVudd|5D zxB&xP-x9Z>HhSK?0R#6e>3~e$?~=$>Vc1SGsgr8MQ#RcM2X7L7?$VE8eQak+I^*=i zhovG>b}kh8hw%xpkbUM@)}w>`Yi)O4C7I6_ni=%Vtn}N=2EJ1 zDBjYcVS!M-QthcH{%Vy1Kt+5fms#vP409)l#;y*JGDOrf+Z$9O?%!t|>NjUCKwekk zjD+bqUqE_bFhl1bC{2`Wz1uEHy*tseFk=o@;;z^lQlhRA_NSGoHrTeF2`^^1@@%(uma^UDM#nintIPm#ZmAxo#HXgA)OVew z3WVZA@A0d8p2?*h!7%4Kqv$x!nviP>H|W%)UJpDG5O{pq05d#kEz3I<9?>^j#_fem z+o0eg2xbp4_`XGD#>hFkIh`g}l}3b}o~0P4($as1i15YRFT?vUEc|X(c)zHI+6zDv z00@Nxpz=_x(|A%OS_v_!vafl$6q`}2ym3^R$Z2$l*$P$k!F8CBY^;X9QMvcx%_*Ai zkL*^Iy9Fk*+_W%gm%?~(3E$|_*~^pd?p%hLMr#3OrFXnxz9t~j;w*vo@M*?`Hl4`- zQ46e+D$nUfx-C>&HB0J;B-#U=ZO{mX_BbCl4;(s`$9)BqPVKz+!f`hr2kdt@@-3jf zg$mINL^Rd4q$&d?sh||$6Lp#o;yf1m3T`5VI0l~|-!@>L#<=fR0!VF=zz1@xMwpV6 zs{)#_+&xC@Yo>`SRz=TcUz`Gl7}gg?xHQTjOSgHxT~vyEo~90p!Bgx{jS6{Gk(lx(_Yg>?*8{jPmY*fn>9FgXxj%2>#|}lN z#&*apNm|W?nFPlOz>$mAwAJ0xvS(ko);B$DkbfUx1Z*iRJOet&ixVYS$>&!NFcsA` z>VXU1-=viT8>D>qtGxX7^a9m?f+eVcganyXT|*8Cn2g<|w2}TZxK1Eu&I)#b1}Y^` zi<6?F7I3S2Cry1H$)^Weesw&qj;-r_jEnzgn(|Zv> z+PU?ah$J2#0MAG~s~Tb`PK4zv)dILd!|YmPuv@O-5fKUXcZteDXB1Wv@4>_o0r&$! z+g*k+{Gw&jvD}3Nnu3f7*Y61=yM?KHvB;AEt&@u;&rcXgxf7&t=05`QJ2{jA<%bIl zhSYFliR#Tk&j{rOCJDwyZi97^?pV8BvZNab22SJ(&aoZRfv#%QNu4$;f%17xXl4_S zuH}SA?cjjk(8l9h@w-f*K#ug9KbM84B_Om+zjDbPPn8tMi>e2T6!gzhaS98iUR|bW z!!6zHp>z3-@LYH5Q=b%d;%|TS)SvwMku<+%Z2aZh(~SFP;qCjU4?q6nUt>%3#rr?K znbj%OX%=ZEdm;yQKaOSs2!q)*o#o?z!=uzTmM*acf(UGRq>?*O39dVaPA0)_l&1Yo zZE*SLB%GQuykxq2*2kW6rD#q-fK1Ybl&VsL$_-+Xaie5QuFe}wFAfEy#rOatgdi@5 zK*V$*u6A~}n?Zv%sCw}s>ypV#Soj#Th8K*QIz$CaS$Qb$gXPw`4S8#s`T!rq%_qOb znZ0Tq_CT4GZacePVGl7vc5bZ-!Ao#TT}&yNS#cRA z8gR}p&=GS$&{;8;?dcr$zDW=M@KR)8F`+z=L`ohm?bs4V?TcJ@Z&K-PMj?-0Iy zmV>Fc=HR~9DrgR-3=sx^(LER?YeG#Vxwl2O zX+cy7S4Wq$LuI@h@sr9GxV0N7ZOO=nDHg>U_hja)H^ZWEm&mWt7rvKCce&Uoh@_o*v1Y zBrgf-8*Y|X#uA$(Rn_5Qr6BKXs1Kg4qw)a^ZdMVv&Dp2HLS^_!v%{M@(pj?5##%Bj+S(t%ZUy51xaZf2a z6Tk+aB`N<^qm}OktS_^bxttBxtzB?liOZ5YEpI)eO) zQ&~JLyjlosCbs{ofFZZ&N>W?};?2Z)J>)Hn{eWq=#ek9*II4v$L+I?c0b8or4qkK8 ztqv9Q8mKQhYGQK2jCArVurh_TpxhXB`IUnCAl^@?U;_cVaxh5$IVKw^Yul!f;4?wi z@aBD`@I?n(1YNzWf^hAmi%R$QR7b;HiAtes0rpItgM!gA5SBlnM7`ZkuYj5GSAx2r zLHLrAyBdixLzUdYC$#zLT?Jw~BI<$ADoY>LVfLkgyd$PWvcg7Ze!}3csul9&X}x5X zhqhfzC+8iMW#xQbAcLoj7#3nnd8)asoSYASD37*9&h&|YKFfFAs&|l*TfSq791={7y zQv^47gV@GFF^PMLCN1dX_Dt0?5D};qCBINSHBX-D@=>W>);9R#8ko}|9yv;fsjyFA zbBBSu1Dz^N=%^$S8f2w|?I3SJe!*RS{EKn}1qx*AF^bG7bq(gCa=V?dgo4N@aeYmZ z+UQ9u{GaT9Dw18YkwNVK{^UAnPh-c&#&i2{d4+7{` zThqMm^3em#>B^jQyn{&}5f{X_V`WOsYb4n;l~{nj0^tX`$djy1LcXMq1BZp?gG%?( zd4aE`8j&PxW7F5>r_ls-S@S4W-NqK=(@ha=W5vP7G^-dFnY7q!Oola+4`SXmLpij0 ziW7F?Ju{$0Y2DIR(Mc98I{fTYtKJPs3D}Yu zX=S!a@SJXv%=5zvmYa7Zm^WMwNNoY1<2`CSf|9#of=J7g5)2K`;teggi@m%$3r5eJ zS(O*8Q3M3R%b2q!K+AZr`@1kN4e`ZN5k!QKWDaF&wrJQW&;g*%Do-u-p}W~EZd4P& z_hxejU?U<12&4%Nqn?(ylaD5Al~s`=tAc)Mm0y@qwx*+nZd^eHSlw58;V>k6CX^om z?{)ThSsN^Zl~mOeAyTU#KzSCvl_N~x<=-)>3ZRAFzjr{2zIy-eQ}6ehd|A9pp*0`* zDp<9EPaB2Z0GBnYc3V}IF)3V>2yA1_iCTYq)OZUww82v-9QBb(<&3ub$yb~zH|UX!oHE!X4mUFoqD*^Zmo9& zGUkp~ zphyJ!Hfa)xF)9wW1`fo9v6Q82&VB2nfCuJlTmsf;qdV(ew}u7N3M;)1_5{ud8yBt1 zQ*3ba9^*^Epbxtwm6OODfa)&Y4l9^U8I&Ylgd`^j;$D1Z$B>b7{RIMK-6|E?hi|?8 zEO4@)6|6Qu7M+`UWyAv;guAzPpUWXmDCwMAC^uj?>&CtzCm5k?Yxh8Rl0vdz$}xe7 zTc{0&bd6IHbEHOS2OHb?l!=rpRBlkIR-s1?B(P&{igT*3;kkRlJU>5byVk|BHANft zdM&ayYVCB^f90nY9qQ2NnsCqSTVc;cNKiA1z4&y&`Ol;remxeT6Y*m46*=)XVXt$qG z@AfgS9A2f;|fX_r`Q&g&zwKz1WyFUp_Z6l0DC%bV~$QhcFmT`be zaXx;hN)C&IRV6hqT;za~2|Qq;5D&okGo}T}?bB=5nS?5|lebu>1rMHER>sj*3c!*Mu@QakQ zma-jTm^HpK#Wxh#v#KpK@EQ#r#iZjn9R2hNmmBSbr8f>I*xE{GiosyiCHc4!po05A zYO`n-MHh%|C4_`)nY4S@)b)}=(Ab;Va8D8?AtrfSPv`SN&W5;woe)VHDcdxn0+SrW za~Gm~R`NDjGb^|0p4^YBA=9uyO=6d!SQM2zUg^Gm|6_9=JLeUqGdi=pIq*#0mt3a0 zHS8iRobmT8@oC!!DG?@y zq4Y1$s?6)G=GE0ku(->Uuf@T!T*tl7db5*CiwSAf5f-*p^DzzVmQU?REa}rNbdzIN zm!y&q#i-8#L`@Q00X8Xqp*em9Pk+y^KSpBgcMBpTD@#b}~d5@?-hHm#E1SzE~=b|wOJ$26YdC^fd^mOJc0X46h;Hcd5-wQf2T2vSJf zG1w$YFgIR96_}robV942eho;iE{S=U)P+Nbt_SfIu7hh&J?0t(fbGLj@u}}PHeEYO zxOL2-gA>>Q2a%o=T-;vRfLvUQT^n&)ggpBURdP4(uz_YC>u-=S0WlYM0@wEN_VoRJ z?2Bl$s?JoWKM7C6iC@3};-8@~-K+Cg@4x30<*VO$`wIRduf90|PcUFQMqaLd*s124 z;{i(+i#vFbnzrI2w$#fA*Mt%dbm`nN;YBcuE#FB#zPPGg*A|q((Tiy-#+}X@eN3UE z?(DWQvDalRcenTh=?@UDMZLM$n_Yaa3r8?U3vk-jg~|0m0h&1-aLi7VM(M6*q8O*J z55Q>hBpUMuc-1D26xjYeC?Ua@mXg+tmy0%ogk9j=E^!!+A%;9tiaC(lW@Ql^s%1kh zopHI6ccdI^CC0jAsnetDmdq>Jv^Tvti2c|^BP<(*skVX&3zDn}1CeIJipNgV*1iPR`ONkk_chR5?j(QZaJv`d@_)fBp8kUVidP_~e`aVuCa-RQcNH@85m< zA^fP{zJu#P1NOgr`y!YN$twmhCT9Z)D9i`P%o^(fX2@pRf!jE{at%16mfG|_u#ihH zSR>16y0z^={E=Q&(re#YpS*mk_&}S;<3D$;*f3H7HrKigS;qZoJbzQO#G5~ewZx}o z%3dv&0Ts5GF7ta&nDcg1>}?6OUJ)+#b0=^cRw}l`Lmh*_&3>uz_0Gd_%*rl7EcPH$ z#Njkb*LUTLYD7*pVQ4_i3`0GDe9Qs-F{M=9$LzB3Gt_z{C%iHi(Z$+L(SzU*4VBKd z9bdRLzRlh1E3nX=wI2YM$H^j6erN>G;_`&xx_8dlxv9iiiDJpN?5(e1C_|x6Gf%l3 z;5^?Rv>CWF#deblIPZiQLzsMGV56yrSy;Y@f~9{-!s*n4148lXEvzLOC%7}6;WBMb zjFNEW9)-0658b5tb3Vhx=ZulA)z3h95G>e~_QzHT*~QkHRPZd3h)?Viq)o-3Q);$z zJRo2h@(1zYTr`*gSbj$Tgx>_7bU0Qm1wUD=pcs2d5v-(3J@6})#Bc(+27w^9hMz4C^ZgTHg^&{Oot$T~28;es(w&$P{mnEPV zLnVO=(tBarQj&kUClw@?tSC3CB9*)$T`p=3BE1B+Y)!Zq%LO+8T6D6`K}xYNDPbw^ z?tc0-^jxFMCo@H*mrDMHa=cux^q!c;NjkE*!TgUzWa%xzWSrs(@y9y!TQRuhfPE}8 zmoYt5i3qnu?zcmg5b9B+E}}f{czIw>!Yz}qpFo(h4RveUX18t7oLC-=UbHGyYv;{# z!%1sE>2`F&?j;3R%f@tNaOk2seCdSH0L~b!AfD9gOKRyRXJ;K_iLYA@KK3a9RxSBh zqifO6Ms!*5hiwIdX(LE{L+TF8f!RvbMD%Dnz{yQ1?}4K=uotN1hnqAQH4+yaaUF+nIzN;DL=wMZda*6cFIDnm`}rIyH#Iz%;3OL!Vo3a5Mfn&DVu9p*0r-}ec&y%#vD=DP*}8;f5kXHZEgrEXfbUXsAkhDLVCT>krv#eCmYR0%Ch}* zt}Dq8^GF3d>;ZHQ0JS$h2YRO23`qf3IqIUBmBxvAQ3GBO+S!4z40xp5qwHv7I2ek- zx_L=-D7`#ek8UXT=_w+D-QN`{r!$=576j}K1Hf| z8R2KuLo#Z^>K(osz@3$Q?Y{}%D0#66uzzk3cNd9&sq)J)bSvVg`V$84kh}(#wiWr# z#cjYRUsks$dKA9qp%5Td!Eq9=1>S0hhqP(-sjPA;4TnzHvQ;2{n0*3IujM8g;2?z; zY-e?yEbRilVZtN|D{LZRp931O)R6m7%kk&Pl~r9cdN;COf+_`oQ0&J8#sgRPI_R`7 zGB~wPg-RH!dSoJb__2J9o*!`F5T-t)3oJ#3Mt&Gru}_lRSjzzezOL!{iC(_#Rj$t+ zNn1d34%tOKKOFC84jKj@-!8#t3EN(w~*ri?qTsqBp1lMqZ zh$HsZlHY8clz}FAVQL*fWwwyjnkq4VuXF^fN9aDP0}xy2^Bn5aS!oY^MsZnn@kTlb z59f&Vb5Kam$nDZk^M*W!b*JN4(~S@plRGX>4s^i*-7S*nX@dk(D5cV!Enn+z-r_EC zVUWY97SJMFh4dK;Z3c3yR9UegbOO{&u>5c(D zW)H82GEdYB#F4MCcr4%fN%-dJgY>iisAHzLuR{3?9q(svU)qrB^x;3feIHU)^7k+0 z|9=DdWBL1MDqSTPWgz?Ee|!736nG7XRGf7yn_&rZR+~6GD(2xJ2R`J)Cmv)QyL&#D zQ(<=NCAfH>Bs)KLE)`KBK~fUC7&%fX=u-tffl5Gr_}1I64>c@+`>bl7VD-x)0v?98 zDU{fsl2q4hqc6wSMgk20^Jbf=sU~f)x`Ea@qOl_YVyd)rf-e(o7Y1P8MogQmK*qgq zhs?C~50KB`piH$`g%y-bX$g{x>`l@fcgvv7i&yFO+N@* z@!3G`Qs4LS0ns)`MUFeAE=HvD16JN*TXC0)gbKAy z!?Ju}azqcO9-#VC^DhqA2TWkPk+CsKP0`V{4c6%-Y&iZ)!`*YrC5#LDWYu$~p=bmp z+iLRjmS-LJkt}oWHq{E11m7;UGZ4#GT>(cQm$ZkA{Mc?aHxsX8LxE$;lvYMmNyBCu> zi*59lj#rn|0W!?WmBK`6?i5E@>q&Ow*@4D6^Vb%6$qI9OSBc z1MHo15uU3$cm#R4DPJt{v1*}H9{@d!1D5o)Y#v>Jclu{2f9dnD&iZ}nIPHAQT?@O6qcMI!c@o92Q8I$TmZdtq z>}{7uJ6;SLQLd%>XUbV~n^cHOLfV&3tB*0)%e)=@$KtI~{sVhP-Umu?ffv^#n@De3 z;%fO`o&a@86&Fy)OA9UOQZ7L;pRI0O+=f#^8j@P>_j|*J1nw$Ta^e;oGt(R!aJY2# z4cG2mdO7_TL($rQ0DgyxEVUlUQ=0r9nxcxa4b>y9U&S4EkoB!{wZ#9 zrII1hO9dcQ6jSbKHgZg=Hl@E5HHo zoVYcvPbwofDU{B!Itu*JyL*;gighBmME6<{bh5^#8-N^HINJfD&n9wPI7HwNp|FPY zj6;<6bUD9BQI(g3(Sps!G4I~xuR*=_;I^db(G0!#Q}K~3$Bp;8@~VvQ452`Ioj9}Y z^K8&$SJAzZB-ErW1_LO|hZr%HPJNh&RRX}B2^c$5?|`bTj=AL;bQB8J$^mLPzZ9e_ zhCE0a*OcJi(Wjo!E~8yj-{5*ve>5mZDh(47mc4<+z?PPeQodtQ<+VAIhuJ)d?8Gk6 zRx)aH#nR-zdtOBQ3G%D=Cs)N^+7|2yIsX~7e(aF>Pj5dD=m_}e2l@z4BYT(ReK`C) zYT6da0S)n_gV1%yTHgSaE!vE*m z;d|P|rkkaDk6(%0X(?~eubQ1wT`QV(J*jaCP<2o{I8I`?V;c`tZ_N_d$frC8Uer?0 z_tS=t>*5P#lDi(o){CA)@UgQ(!a@*^cauG#yTg-KolN6kPExI11v!fp z0T3@MHcFpbu%XjKP+aR-0KeA4#7%`61mrOL>AHu>7tTC zTVL=&JXGy=0l6smavvUbnPsSg=He&FO~ridQo6kL=4J|o_hTF4bf~mTpT5`*`HJeV z-2sum#7OT2uu)Lw$bF0HIr{gdV~3#Hg$+nVe2gXmw3m5RY@nR|zzv@_7)81uR~SqO z=-gqZOxGrP2cWDeIVip0ClnD;U6YlsE4dvoDGNxzNaaP3#yA_mc81;w4}Tp9sGuuY zn8fTfskYTv=g{NLO^wiE5Q1DQ2{at41~_T8*m#mPuN4+=n-uUsUo4UVR)Ncz(NYwN z>A*_4=PJMq?Ki3R1{l*SZLZ#`s<$c)lqb*<9=1$B ze*3;Fk(@sK=VHI-P4|-ByS(t%N|}& z4;7z>LN;N&Ywn+%an_NyX3U$r`~K5PJ$Bf~io zURyVv204UiUo=7oX|p>9*jd&@z*0Wx=Yi0Z$=!q&=8WCpU~fkcxkzrMTu|fCaDX?q zoB|U6nQG*HLt~Y2$tz^J`b;vK>)c!=Pi_WMxXK+GULggUmN8NvI3#qw1ekN2=Gl1%uj8pntXYE$9l(mm^MDmKH)hLd}EphxR)t% z?~tVd-~@v@DN&$k=g&F~6|l`E&LpAAXW!|cb;qO&%l9~VeHe^ zgRe$d=1{U$zalSqN5wp3NrY0Zr z9M$WY*K+b%=b4paZP7U2p4Ou}YF-9_G3Ce1G^~J{n;irrBmr0Xv>wS_jAmd{3SD%` zGmXfkB&c?&0CVenQmL#$Pt4;Gs7ha=Zu1fKZOn2<+|ps4c#8l5kq>#r=w>2tb@;Jn>@uuWTTG@T5p8PlJd_GGH zQIn`$V;ctXbrK;Gy_T!FYvCqQcCeN);htxM>mGF{Jcp=ki~66Y;}}HsT~Q(dTVmJ6 zawfJ5QVb~O;6+8~NgZ0djR4ZCp14Opd@JJ8KMZfbDbIfP{`+T^xKqYUUz!O_X^-Rvd{w}Y|+p%kFXA>dk>`%+b2r_rWsTY7PwsclX5F9 z>Ly^v|X5`4eUZbrTMOv`eHoB2e&tt_)%drd4pT3eY6ZaxmqJ#`N$5<_|QFF6((6DB;5JB=FT`U!n$deoF&Ekchk8YI?}r5j>4RFS%4oMc4n8gR&~8CbWd)2Vx@Z)WK)7{!hgHSp} z+8IDMER9YbDnJL^Vtz-gSQCzz#cdDm8$dYam#HqDfw3wm14kWE9SxcsnOH2nUHY8a}FwId_U`Z|t=Mz4cOP-#_ zPCkEJk_wu*1jwQzb=J`}^l;0i>VeKr;6NyJ&aD(*%faZMz;0WgCbj7GUo)grc)LlF zt*jOfr{F3e%v}_~iy!V89qh@O;Hv|a^5vao_^p*Uo6@bH? zd)s&L&FOo~%h%YVlXo&gm8^{E);r?}*=an%GqgdVS(~mTDLG&q2zlTj5$kk?kk6;9 z238&+~hTnx&k zUMeioK|hF{vzLPJ3##HyW!HUyTr?aT-=!~Kt?N{Y{9bC7PT`yVZr~Hl1tg!TJ+Pd* zoGjPijFSsBC^mTs?Q~?@-@W}RJdx!;Baq9ak$w+0p{1l9gE=BF(GE#0-o37rI8=lN zSO`9!+Msp=+W50|Fs5@DN7~!34o{VZ{T*-9n)++Gqhq^ypD^MQ>~Xx^wJBj@DJ5E(2PbCaBeENOUc zX;2li+qmW`>gie)dO03S*asQ{8m^U8ib;NAONE;V6CE-Pkn~C1Of5E>4FOJy$q2yevr2pyky7gy}s>?^eC*Py&b!e7~VcjT3$)X(g zh_@q7F~gKXamBXYM!5)5z;uYsh|)gA8*N$SV>qLADi0--5Lbz&S$?4fqFWrugZ|I} ztD!#I##PD^gP~%UQUnG77?h!r`^_DF5vj5EPZ?)0$$Th4LEkCyw)Ify^)D8qHusXA z0KRpF=t&(!964E|MIxagzntY!Ti&|jUu0|x$h-$zA45I2lx_Wn6^zfCGV%SWGdpWd zr21UdvW2BN%$g&kuL_q<|GogPA;sH92boT3Z}eDf_Mm5gp&(E(Niv8CR!= zU5eHpAGTeiWxNwfr1Hh9g4Xz1iBOhXCE3f_N%R!cEcq`fxOm24zSZbZT#A`SsWBlo zcgDOtb6CgMQ1pkFBz38A_(T3l6EBAr@MX#5fx*?UPY)fJTLrv@>o!zuQL0&u@;G!- z*-I<1tE?WY>J{M!r#_l6vv%*b*_F`nmM#RX#!Fc8>3=LS`1b@^6?=zYvNHbMo|Ojn zH?S@J@ZI(AdUY z$*&#6zUM24LBO5A!CY1+P)M7B7a+=i;+5D4=;{c2&Vr;vTT_*3jCUnYa{r zm?i}~Ni-9FER6*i#mVk=2nf9D;$u1Z3$Q#iVqi&WDmfuP2rw>Eb~}0PkPQLN9?h)s z8qz`Ty-Mwcym)scQ%%rvpUeLo&!pQ5J8o_&nv68HIeBn~>>G+>LaxEj3WO-B>s-e| z=AxJoVi(|`yF)e_wu7i)`V^PVSqFV5?}!xfE)EEXKg!d}%NR;}vYJ(Dcd}{2tgOT76NfsvS!KnijIEyRB=@Xdq(3+Fa%w?A@=t1th z^Hs#x3rh^MAFgI0w+PWhI;nr^&m<#c_?7}L{}dJ!lKJiXPYpu!yXUWd5k4sov8(Lu z9C?N1CctBMa=)T*D!;`Z9wn+Dzy(JfqPT>Zz1>KU(G+^*m;}HXj=y=MDXzlYv|f~) zmQb+*omE3W!`%MTb?AR?OS*S{1D3)SXnJ8g3ASbEti*PG-Tc-_HB}G^bx=(V@PX_8x z35;VVshYvcGPOB-o{F0bED-Rz&1|9d{ur;E*WpgQUIshfjM%d^pG2@jGSB~ z?^-Mbp&0|*tjX8X9hRa+PMFZen~L+;A&r3y3Ily36yq>dgeP>+Gqy{NjSBsp3Z9U` zF8Gr!hH`;Js|@%_1fC^bfC9+JQp29vv0nfkYtSKEKRBrJ*9=ogbvZ`*0y=3_` zg0DoAGHDm+z}jOBa)~Eg$|h%**fqCdq(o*)IlDWpOE=^P@R3t+iQ1@gYvmW-an_>Ihq=+MOX{hjtpS6zzp;{f;0&?lN?y};v$z3C+CKH| zl>xo2F#QD)5*)xg)d(*gKB;spPF3dAkAPNHmC$LryZf?R#+SDGyg5Hxjmkn=It-S` z7DOA;yIz|Lr`w_hg`f462`kz)az=|Pq>WD^573oV%5SFSpj@|v2Tq)9sMsN|J80|_ zHeq>YjB?P-n^v+G!Bc|tr=&b$mof+r7O|b4XOSqu6+*gpjmZS(wVz7#TMFyZhFHKt zZKSAd*%P@YTXd>P>@q#HP7QdYwJ z>oYfB^4@=B-uXMJD0$(}N^NPnYkc&Rlf>aaz5TxY8J-^iu<$cL5PovZaN*Y}oudjQ z+Kv2Bl>@heh7^2q7=qk1w`f%jraA-UQc21M#46vo8d31Ht0tz|x2s@7IbTt4N*j(Ew zvx5$mH?$Clr(%s#!%~9-(x_9Y@wK)bWHR-K7jf>1A8cydk} zxv6e9AdirBjavxN*m`j9fQ7i{5n3BNPJ8*V-LD!6*p{-&^L?JJoI~B8&{l@D&#q@v zXUvG9nay)!h84K{l5($Aras$TfajKTqzVbctN+K-s#LU@e-(s|z zB#EaZAF3M;-&gvW46Q$=o8yHo}W0g*Dy(SywQ-iNxDhpjRpWQ zIluA}&fDTMq}Ar#Fcp>WsfZUWnt@i8OWRYBe5;FdX#qUBL~6X3&65NhIz8W(UODiJ znDW8Jm&*jzjA!OX(a)Rt&%(c#z>*t!Img}bML3eccrl!SRI;Y%aZYn<{{qlI4_lPT ziAjkI+}V)C$MI@YRr#`2ibK`% zLuS)3g-9_S1l92-%V$;~w{F|g-IPTa;nur`#hVf-ggYZ}s!I!O1Fr6NGc7wh90(t9 zf;7XobyEtRg%fTWyMlr22<0ij{kTr)E@(@cL2*#nP)QEjhA#Es$T$QCgKhWXg)fIq zl3Za`qGj2rXam}ZF;8!O^ptiB=W?F`3;3c;e~QjY0jl=ol&e?YK&P#M8beed@SYZS zI#i#5Ork?CdItG?ROk)ORL>v-tSSJ}U@dEc&}MTXRCEOx8I4nH0u9e)~Q;+aJCEu$0Ggoyku7X<@NIin^D)%&nlB5-M<2n;* zT=H3NWU{?&nuO03BaAaJTN` za>8zZ-y4fBUii`EC8a z*90MXX?{rElv{@KrTCO#^Ih-WFVcx9dDgf_@7z;=z7rVM(vGyFmI(R zBnY=JGqA%FQ0|o0nP|i0gGZdBlHr|W@-zv>%U^AbA=S+a+*0XFEyIC~ci_#i^_6ht z19!lFV9jD_Px0%$+kZZT6^2dPH4bEFYzGccvUEGnPIbeOf@^XpI0E*7?r0ZJU#?!o zwd@+Ln+aNX5NOi0wknVja!a7Z~9-!~?+$4wau_1dOB< zgPC!`zOC5A1ZY(+}(Jl&nU`e%twOl_L89ks|NgX076%I`;+uC^>roDDjm6Vb?AL1>a!uTrP(5-Muxr5dP zp}R_7fbw1eFWpYrmv(M-pV4Cw!aR0IQiY$p+;>_k_Sqx>h9PjBXeucOg&uOm!?Sob zxFeDpM)p)?*(l}3Q4h{LAs;~t;hlm&g9ID+C-M%Mf)OEi^7;Tqm*f3HRE?Zf@@DOa zL<=32hgfaF`bjx-&goQep_V0Mv%?+E6Ja(!W|u8tLRE$v$uaE!+4v!a(dDuA2OGtq ztpfxE+6aSP+GU^yH| z%Vww=QibRIg4aSfP3Y6n`<<-8!sVp!i5y0_Ip*u;JV+NPE4~Ia(nJ)%$xL!r>~~HF zA!EJx(0J;AQkM74W+E5Z(+y%7lQg5qix<47)sr1D5V2$CDbAeD~B|B!}bvUmXjV<(+vH^?S|_&Jfgz>sFt*L z=%sTwZ>Lm)OGC%wrjS!h9Bo4t=IEHgA?l zrjDQ1l-$%bPu($ymeL_pGei;ZQV>fo<1SBL>JO;f?u55x3~}7yVpP(Ft7_bwT6>$q z*$}9>J7ZHshFq8Ws!76h5dcJ>qa|ReV_xTE7#%76PO2K!VZ$HpF22<2#5B)5h|W=o zkmqDYY*N|QL;;ZG4XU(Lme@7`bQiXT0N7xT+$3q}u|2LFbs9-|h1bj>f?P_89mS}; z?==H(R;dj#TI#Y!wjkQULg8W_3L*x%fEcJ0>P5>aR=N+C&)-WM5w6;uL>JNOjGs?XYVar@RwxyIN7NQ?NHY5U0L0&%#lI(FGD--EItw*{wI4O}uZyZ@$}fNZ{D zQaPE!R08*TF3}08$`V+>u9wfXJkE#^ceD=Bb+VvUojyyKQX*n?5DE5&O-#}4x=^g^*mA@( z*e~@dx?tA73V(XUu4C6K0$kD{+@d{}pZz6j0U)FuPX$f?bG(tRW>gB9e%Hx;8PWVRUjL?u|CeLtuI zV@d*{QZ=C1$Qd^3k;CbvF!_af`9X!<0pHsB3Dh9uja3b3UdJ{{r9Y*oaJ)KBgV;M9 zXuNm#F1Rbrc!;;*K@GwoI|evHN_h>j*dxnJMk?$u{#PjTY}^E)>KVgc1GiKc!f}Da zkW{|aH64HJBDLi64fuj~39Jy5?1Vx|k^q0-qYflyvPZx9&G5}n3ZC|p@aKR2d@}sR zPln^!pX0nR8rYA+`%m9L)2}^Th6k1tYiPDAdgr^%zUSzwP*J-awQzdLbItsKinvdT zu~lPI%y#X2mxt6pKH6Ef@Y*G{?vUdedT`$Sx-f$Dge_P4s_g~Bv)hm>xI7$1^GHn< zWoCsm+$}R}xo**|=7P|~F%e!x)j`rOt5}K6qqwi1jmJN3x;Nx}USfUdAi=AgeGZbw z;DFfGt&|4s1@UL3dLL@vX@Vl})5x|57&LjXVt8{?BkL#ddLj0^?&e~PQ<$%W9a|Bc zSOC~uHYGp9*JrY5gq6m*PcNOE4VJmf4m^7r*JH&VjX}B_^~dDC7RE}us(Ve8(L$1z z*SaaMsVUQcfRHBdPoa}9z3&P0PGVwUyrpnSM zB8!#_5>tQ+NZC9&>_^r-bV2t08|l3rF=w2({;Fwh{G zGCoPnDE~_S3k$EU!U4e%wDKpZmbI!$K)GJRK1$_f&1`|W`UYdsjXQa?MzHLC0=J|p z!g>xiqp4!5t>rrm9JFAlfbF7H%xcBd(l3k9ST0YJ;{QzRMTgC|F0oPv12c@AG6P^Q zFTL{ZfvUkhqEoFW`sbhxv2t6NCp>U+9oEzM8m!foBO6__#RgxG~4_z!}70+Zu`2RLM z**|{&$@1d!_wNJ=qk4%iAu@WPgNk2-$o=1m}H{@O;Hj%VOt6ml;1?`Ry)|@?9jkU z0wW^TEH+ogJ=-O4(G}4n%N|~L%i!tQk>Vh~A6VY*$yjnN?x-D)Bg}+O3yNVZsjz{9 zFz*H2n3XABf{BL;#zG^Hlt68FxdG@JJ#D?txHrpL(vhQ#@7bRN+-m?`Py)=?U<$(( zs3Nx*R#Bu9@)^np{kT zX=J;}XXUx{lD#3gME%7UPEk}JFw=Bg`i{Kys_LJ5`a|fL&4skin4MUkU+2Po0p$Dc z*4IVq2*4o93|1LHZtJjd>Jj0(`>-36e~S0@)h1O`zWW9z`NBNbDi})z&N?jCkxI32 zyTn(mn`=r=J-5bRzW=y9_aDRiZ$JD;Xz1?6u||HzULTAVt#B6|liic9dMVm|KS(pk zo14A;DpJ7Vid1K{;^S}d#gVsa$FO{rb3iJDXGSF^V=bJfJ)+de1yM6Zs@}gm70wz; zU9qv?PJIEYp)!!$a$qPYDU?dKK*CS(dKs@u-#iZcG10eYte{VN7>G(@gsuwV5yP@b zaSFqvt#=55VuC)3P;|PEPFHDHE&sS2|s%ma_X*`Oai2Ukr8 zOn(y(tqu9rA=&XHh0uf3ZyCABgZcm&Y_mG{OwXHR_{{pNu7ZCWRcgUD0)MKfDI_Ss zzSY8a$|tCqyae2+{v6cwYjT;3`YIFjBDTkW8vgWu(2GyPpO(SatDJGKs!_MkiFR z)Cn`ZW~)fy62%?ovOveja?BaS&vF^sQEE@M!j!*T`2w`xS7$It&W4|gTR*@vtjKJE zmQTSD5IKQN;LP|{zGi~dowQ#H|Bjv>)C~Z8EFa^;1n-te^zc*#KUKriU*&8I4Ul%Q zR5P+Ps8>g?Rx=%@m6+(BLMDeE%rUu!}miFmRa2eT20m zMa0^UcP2Qy)&R%BJxjF!bfF?x< z4M7rFdIPow7%W{lik2g~O(cupRqUatuh=t%&?zM)QCt;jhk-Pq3SPK!m|3=z&u+>e zGd(2_e&l%mGiccU0|v!E)g95d!dsaa?F!qooEQLx!mYPnwpLZRpimCjqJD$Chx$5n z8(yACFF@7KlCU0NGPq*Fs0P{Q!8pwdq_hfi0{(5@#-jkmh;7A8b*8S~{K78EB3d_h zFzzsHM7)P@+; zmAXuDy(4eLa8ei}SF^g-p=Wihx`to*I!qT$=RbR@d?$7`pA|jnAMu&Lmv}7Ke+!o< zvt}&mQ~IRkw5RmCgQLT4Qtpsi!ZgyI3ydvrX)V3ZB)Y-qN%IE34-21R$Rqh(5(BLC zR=jO5jPiN-Na9{?S?G76H#pGqYf0QBKP^)INO^zZ$0Y}!%5^T%0$~pu20Cd--#Ez| z>u#;V2wh`)i2g+#L=by1LpW!QOx;u2=$@t(VqHsQGpRIGmw+lHK=@8sC7frN1>aml zXoNao#QN5nzfn!cKC<2{5e|~34J}6LX$_EscV(pgd4H^@)b!l;Nl?o>pV(-HTON3a(Jp^}8XFBAIh637O+LQ-))Y&fqHd{blz1$C6+EYa`qixE_W zpiQf!dpX4O-Qs>$g-@sxTV4j#C@{?fE+Sc23Pw2*9tJ|x|3->d`5ZT!x!f0H~R4r3CH_=`TQ56fFUgUR*^88uXjJ zZ;;(xx>}_c)JM)jrviT6mbuvbd2FNa^^sGHNsf}Ym#1WEJK7Se-;of0|H%m&uH$LN z5F@q}E6DQ`R|qxX;&x>Z_bFL6D(I=8U8tj%y6BMX637P&i183{hbA>f2LxXXxJ$^* zk@K%R>|3Ci2w?@ECxdxL#->{95Gn~PH8L;dSFZ^hOf0;WeP>AE2>E(UX;n*@rf6p# z1$n}!R4$u^%eip}EPb$$E~2ww>&e|>OH|yITXg26oKA7F>oAym-B9rbNi`(pFIm*^ zLjf|Ek+U(O_nG3!x1E&^;7khx>Jj;M`MoV^f+Rw?VAlIjBkI*uZ5x~XtB(tT6H;W` zYQ~2%SoF<2O4N{)Z$7}So7B_tj6_zLE_PsmGseL&x>2iCFY!=DyAZ+;xAgc*XXU|y zOl|58bu8&j-#wOiS=X+Rqy@de`&)HqK z%MA8;Q~{wfn>0h4uOR#E3H@Xw=r=n6u&b*%m2`X7Zl}pNAcF1odt7p(P)`3; z%B*U{4j21C_@}B)l^s7xp2+BEmW?~wITY}k#XS~IauW98rML0XqMP0uJxv$jCz2A@ zIKGwY)4eCUNpEaBL}1vcmxV_DWZ_b-urwWbd8J7*it(iclF$mBNGx^!wBhY9V3 z;|gnC;unEPV4jf-wDp+kk|uyQ;Mnhi_1$_j4M6aQxQ)sC~QQ%-TVce1-jq9UhV@j&<|}>V{^PTa3~; zD@#7ry#Vgh4SreH0cot;h_b z*e^^ZVMR{h8OF?)P6bQJeoaFSS9ALeQitD@P6Kv_7iB@EfL6M5nO+raJcgD~1cU_IysO7Mi0FvXjw8O* zo}{=~@xO8|WgVB*d6vrwP}%>7vNu_lCb`bU_W2Y}ZknPsp*-_+$jFST zV#vIC7qo6t(n1@tcTKGbR29ZTVJ-p%0x-M(#e3;|$Io{>{vz9CDg*u-nRm<#_v2^y zhW?xdoM^WMSsDms5?LZIWif@AzRQxM(4&yD`9Z$y=-y$T-cVo=(FCw;04u|8L)K4r z7%fss3b{?Un0_=`2n?u+_(=v_0 zu#2~Gg2M-_A3JrZTMU&bY0Us5$3^a)Ewan5KXU`yW~6V@MGj!dLg{gDX}&hS3NTbQWLSSvr_ z>JPM&^vVs1by^e`7Yt(z`cm~OX3OTdl1I->l&a#XQ^A%5^*+I0as)v*a!sm7?<>dIBMh&V}rxu|z2RlJhSfJOCh z^QF8McOzRR_%hE$lV{p~gf z$Z87_#(1{H67*@Hoh=3a7OuO4*b|Q6l?VX|VdugL!^o9iyu!(M@LQ~)=bo?7SGh1);eM_d;N~ZQH?*Gy2R}7;ho*0k9KdUr=-_Y0mNHz!+LyrM8rTj! zFz2y=h@EB#oJ*41js}ESZ=n2<@3g8P=1$c5^dOQ)&Ae#8_d;+rxJgu-g*n`q1$3OJ zDtVua3t+-;Oa?U1ke-yKJ5N&G6k*v}w)RFq91N63=rHA7AG^593y}({V>&U%(||gg zB6G0S;6|_HJK1B&^Zng1F7o~o_Ku-kJmUf%!|t#Z2|g#hMZb{+G+yGJ8nNTH*Kqct zm0nV{4p~?CMBWA1&0T+j?t={jT}lT));9K#X&cz&wRG!7{)Qe1PvIbJLcxdO2{M&Hy!7U?Pz_XMy&tX-2TtUxcerK-TbCo@wE zV#_u_H6!+-!hB)NLv?8U`i>4`g?aX5D|PyJiIsMweTpelbjxEBWqZ3^y!0V4D&?`vTl^!w|L?XR2|KwSF9CD zg?$4l!F=R60^zYa;k+guba(JUuU@y3+YPV&xTqn*{gt55Cdi6HU(%(SZXMF}*ihnp zg^T0nuf7YFH8>@^u_hp-aU*ds_u%dXt-OmJl8q2&$Av<2VcJh;%mYoQ6}h)%M9JOB z8cAKf`@ZZ*M4+dbmP$QG`Ucv`c_a8&0rl<0zXk1N1dF4ztu5eOUG zW~L1vnt@^NX2;QKXO_vgY6Ndi@Wj|3vp{r^jO@0c_0XtYwm}M#V-3RDsMW~4Yz0FL zfwlUKvnRF?u7fi|K40}GyV^kp zK)|iA?=}D<;1rA2f5ESq@wdZ(Fu+TI`MK=uMGDzbgJ(5a|E1OG4f}SJ%3jhqw3TQE zFcLP5J(F}Yyhjv6-pkcts1hW@Us>6KW`pt;jP%h~hRBVcCR-D_hTc&B(o=yf4jge0`}Ah4Oh$_&`EE1M*%^?4Y>~Gw!sjwjr++Sw^CZG zr_dAnMvLL^Rz>(bEi}2e?iy=CpJ%XQfC7fPw;kIBn725Se1anig&{}f5-fX-#d~Ip z7ejxzPv-~eIRO(f`!IEoB?bo<@;ZS;8!ZVKU8@G_<%iN4?FIRw)S;Vc5dai+yM(u< zt8_KmGv8MUBX&~;PN}j40}uaUlG?cq>?7YGEo~D?$#$$RuEW!=P_#|HyRn53rku_1JvW7j;`X>N{VDCy^L+OV1XLiJrtU-(5a+J6xdjNN{EC) z72H*{3q6%9gqW)dxdHGTw#zX9)SX3-(R!M0T~bL-#+VutUC$U)t*^eR4pFd?dL6Hz zf2tAuZlfd=)^TOU2&>HjQtyNVXva^2EB9KZULIg`1|xP#8YM*$Mu`y?^zh~@)j!x3 za`p_W2uCrw%-DYanSHb}`O8C^{`MIR7xKmL56`}NTmJX|p4Z<&`8_A~FZ#AVD%$xx zqFbFmoKSWL-LyaL@?Vhudl-=8#)mDt)P$<8{SBv|O=&ENX));F4i`#F!Yr+K6^oZM z*lP3WMx2nGEI3}Lk)jc|oRpXaHNpscvV0ta-q7lsoQH>$PnoO>H&4N^05kkiAvG(R z@!bngsgEm+i4Yjj7q@-C^g-2C4Y@b5zo+``?6E_uv#9Yo$9(AgmBZw`^Cp|@8K$5E zY7@Hrg$FOrHrj*oR&S7-+amFKRQ88Su+!yeWuNqRW9@#6$X1OdwyTb8z8)=rVl*f_ zr=Q?f-x1^+FOn`>Qib_2YF5j)vaXOF^_Pp)MVF-DL632lB(KzD^^3gy zo&dd4Fc!k4BP>om5mdbf$x5WQ1|qr)J!#{G**utKHS+OygnjSM7TraxXt|UuCZBxS8xXTIK?wB)MOB><= z2Cwo_KFDxA;Nn(&}s$iner zTqiI`HG*l~gSyHAe{VPBJnCb^8;`rnoi?ySj5rP#HD`%mGS6!vbp5 zwv}=pbM^)4ERXj-&o(I=(J3~uff|cFk@W~`chbZWe|cuj&hT9A=!DiF`vQ1x^KOJf zve8~|suqG>ZsQgy3<_Nm#;NRw_a>eQkK0*^U&P_}cP0Ol(bC(S*7oW&G!lQ*_AZ*B z&^>_nk(uB`fyAJlspmx!fuZ<;f|k{kYV@4QDo8H?>0^Ug>aW^;ylir$!XO9}LlVm} zw-X?~xG(DQLsTD_P3ehbUn!9zCL0 zC2H?B5{lHzO_OR2LN1`cD~~>Zs#^-x7Dtaaj$(D|pjxw>{wDly=KCaR$X_v({pS65 z8G`(?@{DQ6{6Zgm%IAL(-o8A2`1`j%2K=MX-a~IWL$dQU2PwQO)4)_9L1AtnBFWj& zxMWX04YQr+D>`q1@I_@_M+zkZ4oAxKdiKfyYeg~#exeEEOgETW4YOUv4^8F#Rmwz{ zWpVlhq2EH$J&<<_gug{A$!QTsUOg%yL{oKThKP9agvk0u?*C9rzV7BPX~c-%&Nb{U zM$-P^o&x!Ly7559Wuc=Q*p(}2;8J+Fs1X9upwzd9<1k?TER$EO=}<@7Zw-1wBJi)4;~Y5vXT}il!(F{3J&YQmYLP)4fnlHb`aw7|_!!RDNk4AgGnm?L{L7>BeDWh8asFVg&nW?7pn5w z;(UPx@-H;&Ru_^4Qr=uRgiCrB3_iwXm~7ImWl;kG%PaP(cbTQ^269*FCknbvYmHH0 z^dfXrg)nO{srzB%3mr6ZU`uRJ>sDvo575&^)Qu*{IMfL5#bRyqIoU{v%@Ap$g?>sn zkSNt%P(sXPs2pJQ@|Ci~*HcDT=~IyCuBh&E!#kt>trSV`zonrHK5rj_1zmYH6DR9{ zQn2?gj{vcg_>ir9xv)V{{%Mc@RaEC4!)s54gYEX}HH9eE!AEPczQ%_N-{(^44Bn9ctDPW_yfb}23 zeaf8#S4frU0zn!!21DB>WdiN;OaQf}Y%bj?*6P@iqlkj)fn`<%9_|MdO?`TvhHE9k$! z{US#gaMpK7F42Vk0qd#FS$vhU3ChG^u^jLoZAMxo-$2Bb#8%#$r%IJ;>vb&cRN3X{ zJ{4)y+1^cPO!#!YC?lYLxpy4q^+)ah6~pZU1KJkbKx-ZnIfqEfTNEM)O7=&L51>R!51_BeE8Rh_5WMg4WWAnzc&%VQ79b0i=g8qM0G zu*_MuwFES4Ni3i*C;*Cg5l#2{=lN6-9x6C2#!Q>+d(21ju3x>km!jzxg z9E=abI@D^0QyfDOaHzj5C9=}ca*&{t^FbJ>S};(8RiINJ(gR8m5;(xLR@Hi!Rf1An z^Bkk+j}c;j97_UrF?eV(_{Lv{|0TzX|2w?@5WdT*l*AF}%Zru5FQGYhEr95dwdeB% z`1hbODj$tdtiV7ioR|?mx&+0fwZ4qq_G&ls0Z<@hpsZHCcXAI~IOIuw(x;QI!X3&r zbmz$CI{kF8K0x-SW;@zE-a^*;^}t|ze#3mwr~ZJR6~ylZV(@C(l zjuPagIU-Qg6?&GP^rv?OW5v@d9kn-%JohvrY$gN$fAb|YeWm9odxFiBh;v+mr3HVEsSfst^$CN}`a z<}gzCCazQ(KnXayR`&05ke@?^9(|AxWhhyFqt56dxy0ZM|D_?G7LBqrAxU)MkQ>Af z%T(FWQquj~HAtODTz%(k-LcZjv{aVh7kY~?Dd}tQhcG6x;}oVR;iHJqw{rP)ZyB7 zl5jS##TmEgu!A)% zO+ha;+QT$FT~b+=O=V|bX5FH#UnCZ4d96y$d*NXT{azOA&>@5D)+$C%Db2*$>2AV6xAW!j$axQXKA?y9Xx?k z3kS+^tTq+Yp@^OQC+TWDefVzx0{!FL*QX~;(ydF7B!T1Et9s;;kAqdnso4{UJbP$6 zR3Z517YN5&o*mQ=7RGasUdfJfD4<*<676vBt(BJ6 zlDamyN)Ekn0GDA$jgqXBDj=P~-h(pDX}(Np>|O}#k;AZ4wz0idl_`48t8@1nY*?R| zjG8JDQsAPFM92>NtK12c0u^N(Ax9x+pw#9!iBLW4VHCrPjU1#l*r@+9fK8DdS18Dw z)GSuLAq~{23T)JzDiX*r)|*g8V@24+s8a<~8>VRjOu{a?$rAqH>p^oU9E{Zzx$PEr zF41l0B?`o#@s?t8CZ~_>!NOBVI^r{|28FEUq8hzx5YOf4XY~t*97n9exsU$G7~Bos zA*(jWluwZ72Jh?D5qBW3+iu*`dPuSr^2Lpe_zS%-K>7Cq-9t`_e*KO?VKcZ6Fd4;KCYpBvkdyy*rCY-1 z7;dU6v%}W3b>{PdOVYydw!6k>(>*NrLdUCHv+7O1vx4m* z4m8Ssh93fy)Z_wvtE#fLd5a1Ma2W&d10m-T598mIEaU+XBYFL~mzP`taBO94w^%iT z7&Nh#Wp7!I8mXusc3mgUXl&1V1G?F>Un^|5!Z7Noxwu6Xs#Z72gH7wrwHazy(}Dzf z5Tt5SIihqxiq{PrlST#I`5B<#>e4WWXtfK4)jqwG^Esgj)RL{zLzJU>mR6avUqOC# zpf!NmI$f`(5ou#bLhFRV1ux?rLPq(K%DO^JMG7O5^FT~sIa_6)T7Y%&VsbXm2xSc-Lm=0jJ*?LD`c~+}Nn2R>YR4#LT zu`$P}AtmuR86@L}HmlMiIZRVMF@VOY4w40e=#MsZii}}qu;`q*^`)gLf@TOvxY@*i z7OzcswVD0O)WX*yU#t3$ChCBActWAoTOvB>rkq&2oF#6aZc5m*xd*5=jdgnotl4o2FOZ-nBF6Ya4M-2>|)MXYn6R1KCz0>)O&zOYPJ z6YTZymV<{Qrw02dXPOC~`@MyNk-bu=2p*kp&X95|Ug2j@A|+uN=}Cx^iG*NP1D0LH z?q0y+V#f}K+o1-nK+fOMBchaRQYqNj1o%PETJQkfni^@fq*ljxmq`TB;DZSo@q-sM z7~x@P4+HZNB@Bm!F7pk)LshLU$W-VGQR1gUiU~m~H-B2PDue{Qs+LtHMTSMMO^O*{ zl3+cNpIIucrw$`(&eODr{>j_g8-NvB=co5iDjm~Y*1%tbV*pwZB} z&k<9d;Auy;uMXuPU6G))e$ZvuQW36K6{xuPnQGTF*+tt9B7^88FvDG|k0pIqHEdvF z)jeHhLQdP+e!((rfN~P5u4?tv(#7=VtnJ##dJP?5s_C$CXIYW>^Kc=j1PX0f{!;sK zF3N)%W05S&Rh>-{DcA~$j~dLnX@r*AodN7dMB?S_?l$fb3mrU6j|=)>FKo+N0hr!YjK%snZRI`EzCt1f1DOgSt^$ ztPW9J)Go|5cRM+nr7m<@QOW!N(bw*I) zNWLFY>Zp|ph#!6g(ZTTHi?{E;e^0@U!6>`0(*b$^8sNBkQfmsT5XtEPlP5`UMw+WW z6z4YBHMr0Drrl90f@TB+Ob!e%Vx%{?S`J~hgLu(Z${4^DyfpG9L%Fd*2(@wa*E`G! zb!gA#BjgFTf}2<;wQ+o&);@43*o{Zusg!h7Z5gxNBDHsaMg?`jl3%Pj1^pqo>01=!0@wYhIBTH!R9b zN`wuJu%_uaa*@YSP2eDL**rB!yN5@Oj?=>7;0|2Dww;=7MpMe2y`VoDKO!eA&Xj9p zGk;>mWCIaJ3j&gdFVq670iIfT*&!;_Y7YBDR#qy%?E&S|O&;wGO(^lw5+yii%xrk} ztwaZF?pBb%lM<*>n{`eNv1(Ni7YbZ=QSv1tUSwtg>DU z`dw0=TkG;_oyQp2+|vx?U*3NGZ>@4Z`v%t-mDF!eAAb1uUFH0L4(|W2PA`&|Niq0x zm{+`@TMf3qQyhVz$k*9zo$19RQ|4UL`!44%8UT&S!GM`#&BA-AS9mwioXT(eQ0&lv zc{-mt1MH*Zu=v5+C$=bYdn%jU9t2lDlebw>!W=Jl>iLRZLhA(oC6eGj$R517-`WBh z=%+;g8zYLUZ#iJPlVx*&H<^+#8exeOX0R49YxPqYJ*A>ilXS_f0UZ0m;7B##5=#()G#B%m7J~()L{~mJzl~^j=gWgg!mYCt?{*{Mo@GV=TDEYS}uyP)8w@BE!UdP=sF5dQOKx~mKHX)l7sk4|3R zrpJrZ4&l)bAtjDYQhgbx;u*GLWU>0l4mN}Jv3i(nJiEG7pIQlcfSNBZr<*GQ@3v10 zGuSJ@7%J|c4zi9)8QzLB?2a-N{V#+6CeRrMr~PQWaHOAivv2CUQ9K2%ii>VqPxEgX zy5Ff%G6D8-T6YyJ2Nt#xv>j#}K=K)XE5HGmUmPJY-)%i(M#hgz32$SD}v^# z(#V!R$IQ%#J@vgnc&J={8k!cbukfQ7VMygCqbkyno;%EHRqop=%%isF1ZteTp6p0rGtd!k4Y@GbW~S1PJ&Dc=%TM&&NglyKgwkWI>F#uTFQg zf(%lpQ|KH>0C+)7XAc7DjcDa?6tF$AWQ~g)Z7abh>OnAod0WW%S$P}jav%m*jZ}SX zw}Dau4UA8A80nQ-c*YC!LB|S|IVrd*2aGfAmJWktiD2^_{ASHyMC2~jEdkCV-*&cO zr6V46C2gJWDrAy&C{#JU4!mxg^j(nX&47$KBHG7$*~?hjIq2t%%wC}vOVlogD!{&M z-ou2-rp=NERGvLyy!{?7PjYPTDtHDnbJ(h`+o9U%@h12x#0H0mzwGYG?L(nZ-Zkpu zh#N-O@GabJ5+Qkj4d*v>U-px(IJq0^N!=ofqFdJlPI52%T12PhKsrl)cIn!PA>c{# z$=^WWcT?k;j%uGMhgD!~X8GL9SAIt#Tg&~b5DQbjxg&|7*&BL&HV1R#IRJY)j`2rO z3LtkiL7l%=ds45FAA2a9b$Q|rbO!~_&^R=%>l;;+Dw0=F++Epyufw>@b_SS3XNgV< zsL*(zU`~XOddE8V5gt_XRY+M5(HtYn@?|<0!LF0MDqyeK3jS93_Fqz~>U-ai@0N22 z<3xPOXOAA^w~5fU$s|MX&^&nh?k+6)d#vT;@GMQ`b4R+7tx}`8)-I-ycc^+x$^}=Z z;J~|8*7!6Nb^o%k>Pcd3o*?)_!m|1gIHr!SG$qFhVvqGc^1~Jg2Oe(fWIF==;h5IZxvmnj1~qy2V|GA@+FYv% zqfXvo}Q6){wGED$f|X$C5ASOE)0t21$=9u4QCOwtqLm8!k!ZJ*>C27&9(E z0(Zccbc;oKf)J|*=d-4aR^s;EJbPc^tdg{OUtXlvvt*-D*n{FzyE3{3aqFdpjvRSCX#O$}g^T#t5w)p3lNYiuYor7&p}aTeqI#JSlXeXb_?u^|UYR)R%~p zrf-2O_9?g7AKyM>DVaCm*Z=ObKb2B2Nd)?a@INAWgBoD+d7k$s2?YunN>Y+dKdHIKDtih0`<; zoGfY0OdOm+tIDg`&(&BU!5y7)kF-nLi={c@gyAQYn#`44SanNOg%+vuU_`ikJ~68x zVK-#HJdmjXTXzS!7-)OnQgfPC+1IAUH>m_YSK$PV?D+)fSkVKWDz&kqztwmJ1%}7< zF0qqN7#LAr$%v^DWn5k^5G$9V8WjWmO`E~;VeStmEE3-v3Hn!zN0X(!6B2w=F@P}t zB)fMHue<7Dhe84aDa?lZ_n%AkIFWXL1S=rgpm40iJ}AFyQUe6Z4m zI2NYz0Ht5$=t2LgNLDwUyzT3niUmyrl0D_JkdzT1hq=7r2kW##-U6_6qUWB4)PgB& zBv8AK-Af=fZhh!vYEuamiwD5(R7yhGNhHwAX|HPA3xwn0Ddb~VeKhi18(l!oOcxV>`g0s*( zNb4h7SK@xq$=gSj-6@tfm_qpOy`*fYuEta){Z;lw7z+e-0|w#PCzhHmoYj8}ZaPu!)^QcY&v8z6y;sak^rw(P}lDp?BxSI{fFw-zp5b zr#?cFa)J270x+)~&uBt0;T}g=>-NB&V{WNsn{4oXgV$}b%77N^z9mmpKsg~ER~aYN zw~3~&TFX(menio$7HAk+b`KnE2Nfn1u%oI_A@gGI2FDliBLHDoz7=H)&1+PzR5zUZ z?{H21e<1ewzrTNKQ{orEHfhNGfMywat&Pk4-UDR`^ND+ug+Qn4a`E`Op24C_ z?K8YFvZk{iwrJ6i+@(reBLl~b9ACm7UCDQVjU~7CzR5HPD-c;9j%y5L8jKkHT5K2- z2V+CF-C;kdho~*IL1`r{X`fvwy28sZFhivVAP!Ep2K;Y`;J-qV@(fbO#nWDGRnjhX zJgdP2GujchE>adKbA|RmcvcDKH^YCRPjb>8GhVnCv3w75=l4K)GSn^Rq4s#5f%_fi zNE7pM!LY>&+YHd)^&nX~^XV?6a~7(jU}e`+l3ZTNShC(sD`l%SJe}IEB%S;~H?Gvb zPZCbsq6`98Wfk7Gj6r?giY7^;r*Gbwd!>$kH*Vxk)qW`_1)}j-rdKr(#{k+c6E-Lt zUR#9eH8nAT2T{xK4ICwQ!DbVDuATJ-b-JWnZ7@?PAOUPxj&usw$~Bu@%f>%&p3rfe zCn*pr}FS;AGzNA^!>YZy@7$m+xJc%{!hw1KQvss z;w#wCrZTF`V(6#roD+xM2-+J`@r%7;C~7q(cL(<98r;uDib(dqG{tu-6rSaa?n-LP zLG$#G3}6bdEE5nWJM36Do$mllKyMK+q);-d(1XY6Pk3G!c%NQlaQ1#bOLZ@P7g5uaVV~(qr6^1@t6cz zT=HvGci$BT)#JQfQf`RJADjDT#*|yE#fvOTn;?T#e>a_j`@1$>4&lAo>^ zGj9D<;DK@iPbaOEApLbwp}2kUBXNMHfdemUPkiM>z6Dq6tJPW(_7Jdkb?v0A29?E} z+>Dh|Gw)lOqe6ag3&en>GQ_O{n~=UUOGCK^xSn7TfUALk1%cdlUe7wLQsbgxBAIXi z7q%?6C|6OGgiiR*9?#Dr}LplFdDE+kK;y{y8nB z&=+c~bbFP$+8`Z>&sbztU*Ar5iGb<=y0@3n?!jGc6Hs)^ane&nQa9$=kMvP>M!2~F z2Uzr@Y#2EU+m7P_+3MpKo}*XMaGnCV_zW5JuGgNdR98|Cca2oV?o6lV@4lT;V%Nmd z&#CTIQJr)&$x0+XNO)M>-yRr{ak}_kwScX{6oo9bz*VNs1K8mDE-GB%ssgfqpXr2v zeUm*kUnE0)l|3ez^#r{m2vW)`I2?cq6M;aI?{0Zvw>%$`xE^c5I;R0fuZw{}YR^u` zt)Sju15xcYeIH~s4E#iZ*n;f$7a)1v76vkvmrf2&9P87uohrdmyF$(t?}&GclV#enQ*S^xUfcpg_Eo&00ysJ~USlsAH?=aZ&{I~Le1*~z)BkGTgex_ zJ0sU$Xyn@8A;k zoUBhQwJOG|2g}x7lapgq-W(_coQo{WodnkN6kKTnEG5Z>J1a~jjJsuh*2}q@fFf*V z%-S|TzI9Y+gaPJuaKNk>5IU6`Z4d^^>5TRo#$F7{Dj)@bdAa={i9=;KKcLk1j6=~U z$`XDr`7d!;#x=63Ji(IK?%eo#mt)2H>)j0s2L)MC0bgpn`r&P>enJx(lrSzfXy_(q zaBEjE`77u#ROl1@F0A5)tivg5aWWou;F6+3bdoN>8q=V@m9Sd4KnszXS4*!AP~N;% zu5PP@LZAb4`1w|0+qi0|z(*l28}Lts5za+;L3Y9p_}Y2Sw1lX5Ui?AnWx&n(Bkpg^(Kd1@6PW1AEO;gi%kNU?DI zvs~ovvyc zf?MVtf>D?LqBxN4NuPL(if=9()f)MrS3SXAwzqE1)o0b~h;H}BapM*J`K})n2GVh0 zg|zx4aWHq0GQQtV!qoVn_JUFUqMWy4dE8DjL1!KF#s=we8iRBwwMq1Rwi zv#87Cpi~fT+2tBDZ(xZDq$5jw=EmipT(j{gn@_PC_Q02G9j?2&ws(c0D#3GdUUY?a zH^ZQWK9=SY9b<}uL$eNETq!HNe)nVAsNAP#;DPw)S{0SB9gzys;sM{MEW7VEtdJ98 zs=EU%-*kp*@UD+$8tvFgoU?{C$gr+BqppY*vtEZi*STX29wE(!kPf$MNQIbvLsR*G#1)ja%%GT(BGIVCw)Yt0!>?fVGBOXfAAC0F9V}nbvTxk3p zff&jvw*}CU(nAK7bfWkABCB@ODU@?M&wM7sVt{AUWXg3hx!o&q<06+pstx?GRnMGuko) zBHCCPH*b@Dh@#tCUl`YV&-ff`#YuLCW`;Oq&#`5?OCFH0KP!r@H9o1p$?EbeDNEI| z7Z%0^*{l+($vw?3TCx)eTSUZD9F3y&^G01@qj|7=)*hg{(^X5n48Z2e#^vb>@SLfX zLyhAOiWqBdRX#1Aj?kDSzX*-66$30uxY+WEY7e%{;d%o}V(8qX<$L+Sgu~JMXZiZ? z-+qAq2VUfZ)ALD|(5Rq8JyaG8o>CJ-ubSlZG0Um2@G=^d(mmH56p`-Wy|->9z~(`d zx2}rg+d54S+QI0Sf@mGIzaZeSIE%c6Ydb+kYTUub3`{9^o_#8pbCLwRE7b99sG+$l zx@ts&B{WklC5MfjJr6Cp13hYt>d>~Ya)mi)7{Fy>l))QtvuO@We|#&$=XF=EU@N=h z%J$~vsY=n=!v4;>`LauPa7mt6U-mGWSTTHr$bE!_%h47Y;&8pYBs~u#1fF+j2Qb>V zio>@32V0hd zc!Wtpm?pkp4h)4#8+lcVW3{AEjW&vwDa@VZJ?S4jIAf*C0yy%p(0}*-C4spAk{7v( z0*Cj1p*P77KE^t}PA*uGHN#*BxD~}P%1u9ROQ46Ja({iUyFd?_4I{}ZDL!#W6Y>_@ zBbZfPixz5$0wfNA;ytTIl|1>Ga|wgn7Q)+=9i*Q&ZH-IBT{v6WAi3dI3}3uNH(?Xw(W?` z-m|GTezW1JLW8y+G3El<;?WgIhiT>+)7tSPLtGDb>DNFMmjrWft^&Qeo>UXoPS@_W zaCdJ#e6?*f1LQvR9MEJnk+L+QXte>9{$=2#(ZGBk=1W*;D40}Oo~f;DTK;J*c(0SA z`z0~b161zosqI~yP%T%AX?ZFFJZRk<4pYfcYZTOZ&ljSp;{JTc%UvVxtBH3xQ*Ux) zJ~*|mCubR{nX&w-_K8Y@m4csO2q64iIhXe?gs$6dX&_&C0l?X*oWSUgKkJoll2 zTX)Y*43V8oJ5zkS;Ox_&BKt%s|< z=d687js=w5F}YC|Q2KQBKvPyetPBZPIqL>?urRxIN#1T*vY(XWiIs99rRUmJFef+) z1>6DE_Gf@)kkgaR)kc1)m0{2WuH%gdq0VhPYmHGgzz41^)Ptx>*kaw-Rj5yGTZ{yT zKuUQVwaQs=K0+0;!A|jcN3$f5z0e`4JJP3=`!DwiZE1p<6y@Q5xk}=aT2n1U#=r=T z4Pj_GxL+|z(%Fk0n2^l1vvMC5Bhk;203{#BNQGQz?D7X-UK++HD@Z67J!&M9sYPS` z6eFEdW3YAh?eOQhru?%%Bd+nYvYCGO{#oD!VaY$?*Zt(1AM^Kp`u^FU{vpVR&)+^p zlKQ>y{%h<$2`Io-jq=u$9jPjw2E>6WqpbdsCBK}nx6A3H)-?3`Y3U6&7}{znnNVKZ zaxXGuaOTOQn^03YAET^(e?c({h_~_RW?7c$v|b+aZo)6m&d-Hn^()+BfIuB7wxl{5 ztYwexd|G1U%h;(x)SB)*J0s+ZLAtfPwR1SF{E=sr$18U}+C>&J0@LkD3V!CY>KlBT zoE-s7VczKI^KcWR)mSCknMirWLBW7CB9PfVz)gzeHn5KE5=B_SWDnr-#5?&5z(rYJ zH^{jLa3s+vy+RKXDcim8tQ;5VTO0uhqrr-!QeezNWk4jEc}qm)%Rw(KJ;U>v|Cppo_91JA=Vu@p z+rZm%!U+lq9oz3uoM@84_>>nSRGf~$Xj~El304RUDnboII*Sy*9AV7C2`=&Y)lT?! z1yX*?A}Nk_hj$a{0>M)#rzJUKh>T#%*f02V-{AjtqkoMpt7qw-{!4ilsvg#@bxVOi z!lhR~=rb?QJVtE9it84Nb*c`~W9fL1Y>*#v80ZG?CWX{WjRmK7dsdFM)q>%MlH7Kw z3gQ@UUGb6|?~bUVwsoo?CopTwWkV@Zz+0l}KFE%JdXEtW;zIEs2}RvZ(6Z!< z*8)^N2Op0}6?+D0inFb)8I~Cko>S&9g?!}(vj5)kzSSf z&+>1!TXDM|+=Z)bz5NAFK32T{So2+c%+W!kKte>Dmg7lWx;)ys zt1q1oQ2<1;epol?kl=RSd*>La-jDTLHM8&uZwpwB6HO-w8=g0ZMj)M~h6a8Ffm4`( z>;&&2C5WChoD)*>6{3nHr&fSu9FC7OJP5~=gkQK^9(+;KU8j2J_lBXa0m88yE{$i^ z^J!F=16e>1>kA_d0!d*>au*>iEvYhXZJN{{G2rz**f23aRG}Sr4;k??ZHxCxZsZ(w znyr44UH&XMYPE8T-{EMZS9F=_U3cBD|u#s5xPB*W*a3#w6ylYF( zScj|Q7O;ehZy=&obNbux*M|V{Ztj1-IGS_+5?Vibwqq1p$qC+miA##l{KMPN zWih@&zEBc__Vkb-GM*kvv!c1*DmMpQdGJ4SEC2_;$!b2{uGpseBRfARpeV*qE*mnR zHZ)8Sz)5GkK;^42g_O(b>SI`EuEqieMIi6?S+s%WPT9K)Y5zf4QJXg$*URoUzi3$B z9FaAKo3rqPnpT{aEW5YNvR7TSTaqG)Ca}|iAgmz>@S*|! z0j}NukQ2VkGKlsw&^znYV!$yoSE=hUhj8rZb)p^VH7beEA`xa=`-;xlt91g%m%F(Fr@ zG+`PPgCY?wwp0)UC+1mdZFPABbjcp$v@UWspiEV#dK{13q#B~lnfx|;P`Jp?q4z?K z(a4^qhUe7mwqPpgM><9E8pHKeWvV9l5^_#YS~mGqcZGfVC?B&4o&^9oS0r_6_r8?A zU@T8^Y+dA)JUVL?#Zlt=v-Zpchwkamf&nq!Kg}LpnK^%WkORiQhIi{}|C2*8<;T6S zdELv3g)vcnn8Mjo<+~il_YRgq`C$TGz&x#8 zD6!h#RwKuKhH{P^{M_*v?K;xNj0q_&w_pw%^K(;(&)2e>c5zNFexd`k%(dImKPhM4ZTOxJ>e zx$2NE87z|b_f*%~TOhpZo~UdgE8Y{Jo9f`$-j(&#S}))_RV&0MC2CpUK#uRo&Wwso zBcaYxXJ4W#q1I`F*!H1>Yu<--T~b8U021G#f@A7GKdW*QR8WEZhR{oA} z^CNK=sG=ysC+XtOJx357!~^bxBHSmy5y@)8@!OpCmeSx4BuGac{0(e-bRsCQg~1lu zl(r4434NKd>`Sos(%}@}c}Sye4}i7YGal5C^(_oX4B#5KAXT$IU#6?7#ui27o+O90 z4unL5Ilb0Y*g;Rv96(mvhrpcMO`gF-Y$wX1XdkYty{|u~q0*H}95u=!0x9NbtZg~N zxp$x+*%5g#OxoU1R*^k%lmi8pg1$QdUPjUu`*4%eT+Rh(?>TL0UDc=!(i{013KrZH z@zJx-*N5kv_wadnN~kDjt3e9pMA(YlVdFd~ot{T6(woMLZD?*IqyZ_Fk$sJiSu_zh zNxA$)@8sV`0tVzdaD%uLwc&SxpppE{KfiqsR$Z?cpUcg(jN=Ia zrg?AVw!C%bbj_{9^8~d**(#g*(vz_JU|@bo9%5R0_@T1imc={DYN^9n*CNT3mH4|F zd8>WLO!vUt!CN1a^uF?`0O_!`q(`fQuly0LnK498{aJzptZWvy15nx&<`r7j6FZ%d z#UNv#jqM`$k)q9+Tq&mqxg0ucwp|L^9jjY8X6A&MTs}R~qFbK1FP8DqsP1ibjGdHq zy&MANgczP-wyuT>aoxRLr%Wfi_EtRL(Nf~IqgUn`J8fhUBEZqQ+vSkX6bfZf`1HS5 z5&%bDk78oc!!IT|vJz2hb1_hG9BCBQBjOSngF!E8I5)YkJNb;0RFq~9EaB@01_tg+ z(;W0GVGaykO3M<$N@E@yon7Fo5vdEWLcvm5VCQ1tF3i*~X=g+Vyzq;{mM_77he=@83KO}*1h2q0;jx*HE!=jD+SlAo zuVqDG{&_^N)nKi3MX?WTN*ypXjqyAvHU!C~+X?zvvN+j3l;6-%Pw#6M@xNzG`6`&r zE-Xb2o%)=7lnHYv3{WIp_y+9k4$XNg3nB2@IlZk@JD~`)d;sPF1eqOj`E62>p+He~ zWcJp_%j#9LIt0?G#fZ|8$^@(Y7w`|Pc`+joUbxDu2mf$XUImEz3&w36-1Ua3T+$ht z?a2BZ-N#+x!<_I^bVnv&jMmWvc-X)tmCCQtiwqnOJB8U{54esWQshMoevp2*wUq3Y$`zeasGVu_I z9HT7=mk=^DYN=S%z*W&FJM3o<3@X$LCleRPA+0(B)8KU|WKqL7kxd&PX+>=2R*bm` zQS4=8Q$$G$lXjZ|PSWX70cMaG?C(^Zk>En@>gNQ{y2_48iJY*r+%Qh$oU01w zexj+z%*G{3)!V_&QcG3pd#H1vQ?8@W8xSfl9}RP*$~u#|F9!7r4ol=FZt9%CGYno*%sSN+OJ0>dQZrB{*tn*L z9YVf!r~30#ilRGtN6lW};MTZqPnx(llaiO5R$Pa>Y`@*LxND3FZ2Do2bAT71Rp*o+ zml>^9Sz&dW@V!`RZK(wW&)e`I3ENCdepHYYd&1dDkyWjVIiUkO@=StMTwoLbDqFCF zS&_W}UqGP0;j#yzKl2gbAXm1misj<;;fwblynp%O$M4^l-1v+4pS=I!!;i!Jm+#*_ zX>a=NOCEH8kgswvS$}Z6&d1=#sq{^q-yW);Q?r-zkXj<_vr5P|$*VX#1c20XO$mD# z5ZkN2yJoM^_A~rsV*`o(Is9Wu-i9p)&0@l3Oeu)Y`}Py54m|3cEPZV@Ug5 zv3FhJ1ZN~}F{47>L0P@*U1?WK z(wd{|Ex5?Q7JLu}5U3pyouf56he0a>w5xYcqI=Te<)t=P+u3LXBKYYsOL#N@jL(V9 z=q9}6uRbG{tOKnV&e)z49WxZ=RZu}T2`hBCl;MK~`h|>j0!>L0*OSs6@z-{!yXEBm zzC?scO-gplYW;;tf-{4Je?*S4qt>LNQq)LLwF`=_Lu6oJQ@O?|1Am&>c;YDw96DiS zDIGlT<>j<6E9fkqRyae<_1T`N)qs-vwp9C%80=qJ^NsWKuZ@E!9~ojkd;22%&-uYW zOTJ|2nI{r!Ij21bdAiKt17@c1h#^}0rr6_$lD;wef-WTmI!%VaLPKTA7SaZWDzOIx7Ap@z)l zXZe&!A+fe8YeaZ zh?guFfnmwiyB%Xm+0Z{cOzAN7e)$+R0M%dDnu0WLT-H;n6mnJOX^5>ByznPIB6*jR zNO}GxDPAs#8aW{_F-Q+`y(R@-1jyBqi-~5ckMg%gDI;AOK}*U^d9o_5y&Y z)d8#HD4*q6rC{tP)gDF>tb#Tglu;5$(^Mk~Wl?7ZZ;N6O!J%_w72897Qo+uC2}5OO zKG;tSC5W4Id%1>N56U%1I)*D!g&O}#n4}tI0VB|L*CkHOfPCrBVD_8vo|}>nBnDDb ztLTn5Umw{L)oPMQ>a{FI&2!ojhzv51zBT{=k@#FWF(sqqbaM;CCZXweV;>@?4j$3u zG>J@KKyYW%FsW<2g87ttOZy72NSs3^8FRl!Pu9UJQD24mz>`+4BTTX(I1Vh)w6FYQ zCr}7oq#N{Isj3ehys7F{f1(6gEJp3j%Qk74?<|FEGXNqEd}4V$ zpNz3Sdw+FkRMAn1dbZ0qtyi?oP=uP)0GPtJnB+O=7u|a^;EDBw`m}hIDSOiHEKW)T z7Y$PFsOdTDonh`ZEph%ewU#8omDR*P#84D))Y5EI zH8!Yq@P`G5Z$h@nvx{@s9wz|yyj2&j-lwZQGi`w00v2^Lm>L`-!gE!V!~B`PZTssiSZ748eUW=+vftf~mR8&3FPOk=xVuAsK$zXWSLu+fR!M|M1?ieH33&-)z@ ze;eL@aGW)Kqk#H&zVQG^$7lKS&OOW`JcdtlgBH~6ygGJyWj6D}YUs^!)}V!G`fyAz zozV)hzg2@p3ltELv=T$DjTl3bYg?c7bdV2S_dLi~9Zbmy=AF;WB6bWSP(Jw@BCpQ% zPVnL8ME|ZJz^T3<%3M3Cr`ISa&rL#1I*6SwAe$R`Tcu$nxuVQHnu$8i$4gd~zXCcsyKJQ@_76%9n8V6~|;2FBHPm7R&7 zM;6V9!Mz)h$4)yh5(Fu>0;dm8Kf=uE!4jq#LxGC>@s?h+sBOtdmDDTE-Rvcp{NSff zs^kxC7<}Xnf?`fp8uFTyWl1Oo#_k$6q6skUIc4n(slOf)5r z{_Q2eS`|qq#I%4{M6);!%Bb1cTgebW`x{boOj(uD)OV*wvX`?bea#r|CT847tPBq} zX)y3vR|ynF%}s|^j7urdHb~(DbDXHS9IEkDtA~9KFp_wq4ZnHLhxFx6p}*Rpht)F$ zU@B{uV10u3YS%BE%~U;*GljXMb_*Mjt%O~Jy}&|h>nLOwGmOrJBkbT;TTALiw4__c zssV&OTFGf}!XNu20xcZLFRZ2d>HBBl{SRQj{wkPoYntc4PQ_E|gpT~JP4%ls_x;=? z>CG3HWR^o%CDeY1DYPCn{JyS_gQu|U*iK!`*i#2@cc+B4f2JG5wuP0O0(Bsy#L z<9f8K0*C-pC8eNo#00Qp;M}@0!wjD)$x~e_z0)n~O;xw*d9sc3ZuUW`GA`~^ z9~6*#cuCt*px*KUFV_Q{2E>%5EM1lrW()x+$SqAF9zVLv(G~R991^nqde=cX%{ETg zatPiv*3|9%#JogGDYcN>zmPTnMkfGGOg4HbF|FE{oeYn8R#x-~DuHFG`3xuHIst=h zKtbwvG?PZ&QmA+=Z`3S<40aX$WFdm?2Lveq7B1!mKt#toF;Ad+5oxR*Jq7k~ftEd! zlC20q)atoz!GaT+YFq73g|NlNJtnG(wS9b%yFwC^vfNEDE}{&eDC3|0UY~rOWPO=n zgskI*hFW8%Zs>mM-J#fVi?8${Ue%>)qYuf0cW^_8@McLvH>8h1>|6E^Wl4cDkbTy$ z`Pg(rGt2JFdeCphAp&eR-5xVVn$gx3b)2AugU0oCr&;bOTd^b4(I`*qNRc#+$^VRbn43=K!^9N43HJ_BOVvxmG6;h>+Y!!jP* zUD9k<>Q+Yx#wA|D745|}dp9`?9@)6hsg^iHJdp-k3d2^i*=YvT=H{Zymlrk=M)~&3 zNFf*Y{e zsIfFAcdt`@H#sQ>9L!iA%&_6Da+$bLRExY=cqL~Gz{sGL<8r*mj|2}jqfQ|8J7K6| zZ6PNn`+)L#&paMqRFi$XoYV|5_!wVpaHEIi2Fmgz)Za*j|B~dkc2Bu;aipHC~p|_By-uCsC9PqTVQ`?ftqhFo5WgFY8 zTH}X^gII3LE9=Z!xtf(MvqMuGXDCitHNj5(IU{uJe3c(+ew^X7`w!nt z_VtHvhEx7rV#@m;t!-cv*VA(c=ho5!0?!{(76L9L_m0!mj5{?wk5~qFc>p4D{Kv;EZIn z-ANH`5B3~K2lx#s(R^o6`_hpWy&94YY)bbX+N?X>#^6WE=|VyhsXbwP92XTsW$V0e zXnQGNXuzWzw8tkhf?m*|(J_qcukOtgZfkzlTx%pNanG*@v>*$7>@F))Xr&%OVD^Bl zka!G?L_%}03|H&UPHC1pxv=sXUDPM&tQV-MXTnm|B_bm(dN}d-wo_SGn8SnJ%6t$V zQoAqe-=MZZMWw^tg`sv;lpE9RdWwk!WwX!dCf`4T5_T(z2|x{&=|nbN_W>;CKpzJ; zxev-qMdCoEPjZwc9Sz2nu8-2t=#I0KD3K7FV(X9`hgnIandrfxAdZ^GvvC-ec5ky& zIdj$81en9ggb>sYE?|!(@ST>mtPeTWAPm=c0~O)c zC47=EKH1~r7V6PlvsXZy=ACKCCFk|PUllPK*m^EeNrIJw5hcePSQ~2&5C)GHkkOk6 zMVvt);e+rrN$KgurxWX?1OIrkf??7m0j;*cM584r) zl`}Z1>se)8nnq&s&T2u)MH~ipvzG$e?hMW<3KkJ6Nw7%7RgkG$JXFY+;?uT1P8Kiu zq}H2N25jyD=E~2dR6rB%kYo?x1JF=b1XGA2FoL+YzHLT=%VpU3| zYUC%~$v14((%XWFokDAXdZ@zdNxODfL9(Sz567pgP-_v{-BI!DEEDBa-n<~1E{fAa z2yGS~i5t4zah#z2y8Ad)g}Ee{#Ub+`{YiuIy?YNkvXC+c8Qm@#(kk;=Iu~B zse+d+oK?~8Nq$7CqK@JYkS(oq&1Pw;puPc616w^y_xcL2_KsrmiyTwX$yktjs1nAS z&7tyVF|Ix<-@xStKOrOwo~Ipb2o8pf(DZ05kYLGypFptA+$o_uZj3pydnRQ_Kh=Cj z^?`tn?pn4pTF_+c+Qbot(CU9V(!4^>m|ny^l|e?#sYo;!!CGe50RWkXxUwcF*uW>G zXd09tm2N1e#s*CY_)qw)%M>ZjEegA@hn>Y(tBFn3$GxVR_h7&YXqQL?M|*L*NSL?k zXkB;LoB-%^(0qeJ36PB3c02NNzQM^vS6o(O)d5r^cj>ZY?m|gG$QY}j0{!7AM|uW6 zu5}wN3>u+3+5p%Efg+4w2hKSmePGdQEW9sml!X$mTw_BY)MQFF_97dLQN2k$MZRO3VF~WgfkE{u-aFP$uyRg1ZR4t%o4)y)_AHV-7eE8nmzr6ii3Xsp<|M20{_fOxxlu!KOPyhA9 z=kGs$`xMW-fAQh_Z@+%~^@s1tkNvqmV?3_t?FlRrTYGde}ZM}KfV9(^x+?9 z82$u`wLQHcrM83vqMO@emaCa1A0X~s4Qctje(kNjt2C97rocwp@~&j}UfwRkf>yLK zUIIjvJcGgL67HfatC>M|lf~DyE7}0iGl~w6urBXf1B}R-=0X!h$XC7OnMUlE4~E`pp`eP+QMeiR43IPiF^VsTNY{00L=Yg1reTm)t(~UT&Bx>(zCK z#cpS#lGAm%j6to6-88F%!uZzZOI(G=l6N>n`$o&9CDQ`9(J{hknMzz+-TXwo$a+6J zwZ>UJJRuKb8x+6>S+kHBM?Y-Aj4_iX2+;Dg9ex$)vdV(+EUFLFu7WR;6uBh#P%wmN z2_f|fjqQ%(9tBUA>bjA83*V6Z0Eg`Bt944Dl-4~gziHClw@JPCz+X|ArPl-3nw}El zR-M{jfY(4FYVmEos5_$j+NEq7LkMt$E0yvi<*r&v1-%hoRZ<9&-Q9%eAcc?$WYGib z<_CL(#oq!4O919zoAo#0KX_Ps`<^*MgRHh;AOZp7q{R=V52Y~b57ev( z&!D$o56?Iw#f=x;30Nq`-S6+*L%piQ!G2j&r~doY5Ph2)c?R(F5THS`H%zWg8K z%Q`W~mIqQ_H1ejs6wFaib!YI1ml#!pjD$mH9^jUU!rJ>EFW(LOA{C(iL&g@86~LRGrejW5f_1puCnbD|cEjf1KR;81y211?%k%|T+NNROYT6z`zVly5z9Zi=l{uvWBdP7|T5 zl1|O}fx89nPkt7t0{5LoM%`QpSWpInDg*K{gh1e8m>)}t>{eN*cG;GwBG^WGi*Arm zqkaW0DqWLvPHwKWW&NdA<6vG}?L9<>{a0&l?qQ2{%2LvMn=wgEte8z>R8 zW81V_ASK3{q)`E8I~f{P3G#udyc5x+WN9>7(xG}Rp!=6F~DDp zwmC74V>4G(IL9JCgLWlSWVC^N{#f<9l34;8fUIl*_1yN2@a|9ODOF)jq~+C2HjSE+ zNv_`Z&smN|Y84-D?0){I@U3qhdOU|=&riy;zjW8Ww?AglR!Kw~C9f{Tz1)?v5UJvV z>cPSdut%h*q0cou|+xBSAYK5-X9J`iM@$+6C@lz9Q8WP7c=tVsedf3+Z@J^LZS;muU zva>BR^~JlolI7WUgr3S*(QII^r!^D`YhxU&0XY}Oyo(}o$PfeSiZE|G#Rp?&zU$e>l{Owoa{b#2qn9z_^80p^kNZzDHT>`?+ zMX^$#TBy1GjUv+$l`T1xVrf7V0Tti@e*#HXJjp$&BIrKjfNDho-A&D{17TCRQ?Gzy z%rH6jgf_rzs}~ zZ3%4530HC{Sj~sBdwuohU%X}f!m~FMK#XK(av&Z+b3Nz;`B1I)yKTgg9cjJ9G%?9$Koj3QI1PL@{I#$e=C73ep4V*24qXh)YdWp^SFyOnSzcimqnB>Ug z2b4Nr?T}C{WWQoA4In7XfmJn9?6RUp+Ad*^*ux+e-S2TU%<3WMZv$9+AY50;yh@SDHUFdJD9=8Nk1!FFr8uT^S zie&Ldb_>PSvd)t71*+62s1T4txLH!rK$yBh3^6zz+F$se*QB>P14 zp^o4q7FC2hQfH8~(sD00oA+)7p7v%OnjQL`{eiHtBfV)to|sV#c^{S7wOc!V*EgBo zt+L1Q&Tz+e+IW}Hw_#CRdaCpSAK@y0$o5*l2Ptan#c^*^fJwA;I*@vGkA4@I4&2xL zr=p+U*?^85043UuWI@1q#_}%JLdUrV$@PC@&~7J`92XB@gW7PLXx%#EM^?7MT13{J zC7Y>{L8sh>#O~PSht5u~DSeoEEXua)pQsU_$PiOO?vQ680VM7u6>jL;jp_rz#G->( z{%8|4h&kHO3_3NGoly6uVh5R7=7|PeS@a>E+C~lil8Oq(t`W6Eq5GAK8{5fs z6XawMUjv1M`~aD%L-R91p_k1Y7W|i=J*&RU;(nAV?otjziM`vRV+%3uv-b6fMv%Ni znHsYi$go9HEAP9!cX`YHhU(uRdAR=i?Q4VMeE30l`)T&C{Hm}4{lN*1gn{l{*__*~ z1RyY7oERSJDZQZsWE&hhPqtw1=UzVBpG%P29$+`rlef6m8_T2(aNdT>NRrgTWC9St zHb`y>Y7Iqx=L8NH&s%h`b$kWC<<8GMYwZC1@2cBUyc6^%M@tQ-y0c`R*3?AGWxwJh zDPC5BW2=?p2rBI~%%?GJ=e~P6fQ?w4+D)1%Cn11(x((5-d|BlYQBX zjz8Q!ImezB%2c&5h1pqbHm}O^8KygR$>Ke`tn=ZP;IuODBYRj1vO@<2Gw}(&U3<&w z{)`ZA73T}|M3?kDrK}HU`mL%|D|JuK5~!RKrgFCR3EJpk+a?Uh zS0|5EfSI@%I~qGjdPFAIl_W}yv5Ms~8klwpBjlfLT23q#T(>Cf2b6<5>P_&avj<=% z4XnszBR7$_b{gvak3NR~uP;ymrMF+IjFQ>$sZVo)C?MuH!!uiv*$YB_WhQ;q#7%B^ z!$-WubB_W~{(j0Mg?M_%?J%v;4sTulNxS@nyqh~Ip&*zf?~d==XA$2&s$fEiUsg#? z=78d(oEi{b_Y-UmnQ}sn<_UdJlvouhW@vJ9kPW%BKrr6kt2nXJd?blCiq8aA>WzbM z^97q)(LK3zhs0VhbS^eV0llf#}HUgIjwe(Q&wS(CwA`?dTNQNaN{m!(V(m zuTP{m){@axy;a3`aw?OcEFx1VeY?IXc4|Rx@<}5LP}6O2(&*F^R*}JOBk0bM$5wsBLvqw@nDvo^Ij8|MC7) zUH9^ao~}1}Z4ZZQUnQ-*owAGIKIH6lE#_U_nB9m<&!8L6NE_|*isf3+GY;$NvRE;u z>a{tMCi-iE67}^XO+(->;>}FZlTTh&z%HnYgI?a@Y@9~PvD#CsvNn6yc9OMjl8a-s z$-yPWbI^NGm`*QnBB%CRCeSvPpPC)q`#3ehjCaJo;mjVF%>Z7s=di?Mf zwqs03P#8VJ8vUS}2raRJrUDYT9Gt z-LT_tp;p!0(wcx)=f(~xj1v1Edpp;oQ2PV_pz2<24`-@dh;ELB)j7htBKQ9;lS=dz zejN%-)tdFgvfY>cW0BH!+R!Xd6;j^Dv|;QsW#pXOtP+FN>a}x?EDTR9OIFc8c$4Sd z_kt5LouuGw?EF3WE^E?v=)YqCy<_gl87D;iWi?G_$xo;~hrS>{QBoplktsXf3h4@T zd{^6*P4#pPdT)H6Dz}+AK~dMY-~koW8~LfF)elYKhkH?VVW><1v_Hkky)hw?gXK%f zszC=i`Mb;x<0s(&{xk&|dc^*Q9e1N!4^=i8%`W#2m+j~FO%wnP3~g} z#PH~DR0+%9@ysfr>#Wx$E8Z`jLr9R7-N|U7bXk{ao4YVkcLJu;eYg6fo@ny4d3KVZ zsi^$kGJ~BKJ^$`bU0DcTo?zr@du^(K5y+Vh1_+*vjBFEk;pxEeE2n!ylr!o^_A)SA zfbvu}VwI&xG_LF*Nfe+|(pBuB+(tr?01s%^=?^u?O-sR=;T!#e5 zE3Z*D5@>!O-havI($9~oSU-FJv3>jzjp{!;e)dy7`)PUh z%lDs!48eHqsH^x$)kn$UJ{NKvV5fH4Z5{Z@w&=Gk7Rqtb1!`}xa;=M4@AGMWD$r@| zD20l4=K%z!sy#EcFQ(p;o)rob2TAbab_YMrR)@0~6ET`VJG0Sx`IXY&+X@9GyDz9V zkfhbUOM@d@u){6a%GKnDBM2tYTb(bZCkhEF$!bT5)T1^_uD0i&=<3}Fsn~qzGE6|Y zTLl7F88zBiXTXSDd|&Z74~20SInK7z($tm4V+&tQ#*gBs(x*pg7OT5c7YYLW9*D`8 zIC92x(@o~bDco+19>7fzvviFPZ389=^IQNfZTGJ{JTGa+kgHpL8ql_Z=GH0@plvZI zHG?{BJNb@iS-jzE$nfL`Jv`9KPmxFA<&=s5m=dQ7UaNt#cGLBTVzarRkt|!&;seTL zigRisjbQ+YtP34F>g}!-+^HVc&>DZR#tO+-%7fo7&OkYWgCrq&b?M+hDrzVdDNRF50Lb$S24bZ~owru{HW9_)TzKtfv_Spn1^ z5852D*G@*P552co&k%)!8^K3HS6^B?{a;C*GnDAB(MA8)@cz~5?MLD52d4*8DA}jh za_BS};PeWaV`fRna(5^PC~=c&TrE^{+~9SvIEQtGxHf0PIJoagKC;8R?h-!NOE$~i z@<__Xe>(@r$gg$?=14`16&9uW>Lz7lA_t{7J*^oS!CIz4$Y>EdfXpab92H>F_7|hT z3yXUz&QibBlXl%;4c==u@}$GIeBW*9)NI#8I`V||J?Vu=&dY2ufeuyo{&dgnzS$UnjRGm5x4g-+h9AW5p%@Aw0@GR5-KXo4dtJZFv2z@tv2aoszwTps@YILcHP|b#Ly^zb z2x5B`tSIZ?8D$j!wLwAslKwjoBl0p`{}L0*?8{*jt zk5Q#EI8`7GkOhZiMH9w-LC$uzOzo>#0KO%C(?kF)WrpKg0)@83`B6)RTCGEq2r0Ze zr9&AGHk!Ts@r!&&M}G)|uS1DYwc~2vdrvylW0e959>tRL^d!fhsCylYl~Pl59Kzox z-ln={^D8Mv4?mr0gu?H!f-iel2iA#`e1{e*RtnR>JGVJDmV(%ICK@ zwHaPo^ZGF?RbGWHr9)(CFFn)DwKOP%c8RLZ2w#%DX4EIo6UalpqqYvbpFJjM}h`pBM&dw6sc>m7W~wlLnX>f467vjE0WsD=a~uNb3=&KwT{bB5pLI*fEZ^Q z>}20WXQKwP`rc9(xDl$5x0NbxMSfgCBF-MCOCX!F&_7NVH%+2k06 z25YR)I;|%)r8rO0_&&pIAH2>SV;L*V3`uV-(0%4lF5_Un{&wI+e)h}o_I+SqetqCx zUYNljQSvbhdbz`Et`+a-&G~Zgaql;cxp1)br-C;RcNLqglVV(+Dm044I)!r=Xy98b zj`F|mBbE^zo2Fbo6q1Udu`7u|UX+REDJk1J2e<&9q8t=RdHmK%WvHOmWn)s5ORLYz z*S5k)in6>-3-7R2y^F%I*KV_mJ#x6W(;@z_ro=CKjT=C3;bH%RD(+C?PVPWX0d;Rj zWava>Nz5^oG$vCT|A>9v+urDqzAZFPz`e(Dof>zWG7v zY7C_nq_E)zMsB38-qM9fYx4;8|0;^GBSa2~idPX)fs_51lMBF8=K9N}9^T*!@>|b5{Ak<%} zlIGsYF9TXxD?qZ4x(!Ri74uGQbgnt=pA-c2;GycoGV$2;RG6kfW7*w|;8?<EY0Rk9&<9c7tT@2Jxl#5}{Fb>Kqi;`sw>WzJ2};irL4? zUEh@AHmQ!^{I}ThJ}~{+`)?4Ce)|50_pkJCy@uQ62^0cJJdd_({jBR6BrU4e!A0AB zze`s}+#)c&$LxN4N~-CO?TA!rD%-uYG11!*TCKgp%fnJS9TXf9@<)M7!3K-LKmx}h z(A@@K{G(O0iOB+T2*~EWO_3LskfaTO{2rK=n0VgNo)(McLliaUu`Lx~wM|8m#($&` zgM#sZ+PuKpP;PnZ9iHB)x&eD`Ddk`MB+Ip>7azvM?l3W)W)YUu`1ii zVg~%^B_o(Po7O3-|rP~61jnHg(FSPN>im%|pQ7ZtXnceXGsaYTEj&r;?qD|YmbW34I}!;#|Z zJ4c8Xf^-vW?n%KRl=N;L3j%JEo^lK)`npmn`uI}&nrpDZ1js{)ig1P!0FS#X>N zvmn|C$RB;F0scEQeeKb7ELEWJvJoIKN7we9q@L}C+&BJ3TeJER8V z&NGe!koyvSGWWJONxQ^e^sU}TV1wSNF9=Q4r^k?Kz?N3cW;&lM-m*<^WvYIA=sHr_ zR~nNr_(~_#$l)#<#s8uKH_~0bB)Bace(B?_m!^!aas(V zTjC3pLV&ZxQoWYah^0tF0UpT4@Mt&Dg@X$f!t(1t7HEMR=}_&fxZi+oF;XrVEQ$V$ zeLanz>#NDQ|ws2*4Qo&=K9b2V4FuG^&p;uXggUD{M*uPrm? z43IMn4{dg$I?Oqe<^8Xs-RO~iHF%HQHtUSF&m36#jFTJY=7JU`;YVew^Ual$jCxV5 z5F??(Cu=RtsbUH*8Q!GSo0YZb72%TODk9B4GGeWR!Yw~+5?1-% znyx;>T~%9{bY^faPsV84!@ArWblvyU8(!LN2f+BU$Qv+9N_ufA;U%GTbf4uV!PPF7 zPTj>5U{15!T;g563qzShgDy;k$=iBn`XWirhSdz6>wT+=UIxU~&G5IIp2 zrR-YmN|+vYZH}um^K*m^B_!HdiN?B8{R;H6tCb)9^a}d{0s|ez*zGy%=>sV!>n94H z#xl?i&fX)7%2AqWYa->7ld5b!q57!JUc)GbI2$yqCg&VpOq=8dZm0Mlqth~>ghWzb zwd{j(a3mNGzJ4iTxDI^deDZb}^Tf35!y{)Fooz1DegFsX2Rs45Z7R5F4zgiTBvyKv zCk8X9`dE{AFuJqy)=iQLeeQLKYS&$_x>&1E2Qk}jXp~zZJxCl2-;)37Q?ID=E%LqM zCdzN|IoLn^3>+B&I?Br6q3UI5`Jm*yT)F3}w-N4bk~v?<5~B;H)@SaED|g$h3M7EH zPN!B-tz2qOov7TfsiL5&lo-kfC;yqO>4ScY9q(-SkRNo4ES0(sbZfGw8|ZSOx3ZJp zgjl!<<

p;h0vKEhXpGSKyBfF!Kn-^P=N`qf5vJGtRJ1m9`#4=&3@0#4<>R*h$5k zg?q)E2Mhj#W^9Y=C$(Eqs4JpIZV|)rZmudafuADfm36Em+8SGdspV5#+(kxWY3o|@ z2UP{l_e)+uXMazu@hE9AwHii7d+dt(e&e!2z`VBkdy}{EKMXGREpXB2SAizdHZk6tcJ?x&S z0<8>eeA$V(0P)oMfxu5SQPWjYvt`w zrcfPgiznrW!p&3_BrPNp$XK@%U{ zEw1d-T~Y9%nzquwNQ&nNWhCWM5y5=0kLu=2#*1^ zEnl{BR{+PE2h481Q=o+CpIDxkTps|S*|`9ce0Nc6dij$UW12vkj>?1CjVHCmzdORq zNuBIZ`Pnakko)09_h8@8{|)B{`M)`KNLYQM2~mQY4~AhEcU>aiw;U~~;A5_FS)X#^ z75EA9cn=R?*N&y5XxJ`GDbVSM9q+f6dxI6x%NN_Kn1_ntR4^b8H|^a^@mU>4Pqny< zD7RyoqUx^GYeAw2)-{CUXQL{Pe|umu37=K$4NT?MV=_6LQmH*a)qQMSfUK7ji#y!v zg3SJ#FB3_x;HiSyaZK4I4Bc!VIi^R5Popnm>UtTrdmlBB)Dx{`r88*cUhfljCM~DK z-C=5=PPQ$_C_b+Q)R>~fx2aO>5Afv_MO9iVL9_!$GB>%QFRenfZ1J=#;mVFnqUL;Q z!(*3F)2cQPrR{XB)UKC$IUJ=CJdQ129o5^Lp@e#9(jzFg!~%;P5tg>smW29i+7GU1 zQXLQ0A*rEBeS0!)?33HdA7CW_Wxbz>D)*%h)G&_W2POHY0H45uNIW)8C53tmZCvee z%YiOAe8EIHe~3CaRth4|y2^)4{s<5Nr`T<(AF*Oz#;4>KZFro28Ej&45{3B(dPV)3 zLye!j{SxEfpS=A&1X6#ffSFIf{(r*T-{*s^Tjvu_M!Eng!kOXn*x`x7;41eL?H-51 zt2ecdhRab~0R)Tmn$IRWhExH{0z*k9JK9mlcAEN1s4M9}wI+Ej!_-eTAD7`E)!egN zy*AufRToDGF12Vmoy_5v2=~?!G+2cTceGLmvLc5V8I%O?#~ zn5fuldZxp~B4`E>*BoQ8%)F=@g3W37knWRu+Fh*;fZQ21kF{IZ&zFzWq6&ysjP!*< zeCvi3oKfFE4@$8e_xAO*%|_tH&=$xVk|0%kO7^M;P$dL`c$+x~`lCHY!g zNhbqR+SOVDY{nQR2ww|UlVoigb3tAOewdnhf7cmr3bZx!Bmj5BZxTV z6H@8Q-{LA7K8lhr68kEt>tNz6=Wvo#k6R1&+DE`IrP6~(qyKZL^Ru!Y9a=Gwpnzd> zIJ^P}fw@cBro*{OA=Z~dZb(dP8;S%L>JaYvT7KS6CZ2R+rWeW+yJGD_C~Mt-O~btq zK=3^?V6^{HbvpaII1KjD;!~xN8nzr(eB6=QB&u}sPHDUfxrdFmD?4a-#*ANf7xb|^ zD||$?m5ML*hs$#{vHO?8&`Lx!NSF%SZ^);8%=wgQcgWwZd#-ME6o5`N^YH1mFm2Ku zTS$nMFE6pTI|g?U3AZT&nmL1^OzV2K3zu&gTL28=>A#}OT%utCx@DqP;xoWPRBr;% zzdMLR1upB*T9tV^i87V5p0%;;OPem~UG zkgEqWs#+-VwK;YA@JK-!$PW2YyXpkSel{>v(35@GqYA{4+LWcSu_&2%(DR2QLIq3ACwl<^jx6Zzq>lv?uHv-$n-BPouF&-nEHkKygh%M(K=JDyJnTpR+LS=6HJ_u~Ap znV`P0rNf{pC{NM=grVa|#LTcdI3)3e~7h^+xHTJT`%AmYK3gmLF6r($|a17M63sY+82c)F(nj6M$6C?VoZrOdl!gLnQ-Bxr^&0wgZ-| z`^LBi<+kTVwZ!BTKiSnnY*w1geUmGoj%MW7$ghhUe*1ZTIN20pg+88?M_tM-p4|Hj6~1xe z{Z`hE2fOE5Rjo-+{j>1re=c?N`=8!__x@GL7cdd{!P`#*elg9{SBy@Xm4V=MZI8zc zaQ@(<_&kkSt;c&PUke_y8S#n5xFI^%!$eWq04r7iNv-Jm1QIcjx^w25WM_h)C8e@m0(D5O*%+u>MPXNohXoj3qsmZ7SRV5r3Dnj+y zlY+i&Jw#_zJGy(3w{w@1-$_xL96kLOiQB zosuQqTjZo6yDRy5uZ>h5OmTa3qY3o z$fOvsW4CehQ!9wns-G*sKRj;B zu?7Poo}_>hcY#?FUt?ig>?s_I*HCWju-w^vKD`-R9n+~2^?DhGfv_Dc1$>kM;sriD zCSS!cOQV5(*6xCC613#D8(4k-)+HbM-0)!yG%>c6FhV7U2R11bjgg%C(LU?9;q7Lqc00jVa|^HSP{hJI5s-%B%K>dUzB#tIz+AiCUqJ~6m~8Ev z57IU`3X?B^KE@G+K$xgtL4JHlh-W!O?zJ$8#wx-#5=VBm1>@3Q!*#kK5c!y@pYR>Y zS-MaCehjM;it%={X*FMbK7p(ExPZ?{dt`+o002IV#)BuT?wDIepi-45|&V^1MO=IA5a@RkaonDs3qZb3%)> zptlDEdTog7y|n;QNn0J;#T)x7b#h<}aIH}S45dU1E5R#*XCOBPl@OMj z>nfY<3YDp%CNq2lJQXUim@t{Fr$5p?a16JGST``zEn4jb%%v*%G45Qds98+r){^4t z=rUOB_hGORN8Vhc?uY!uMDI99GdFwfI?Vg^y2HzB<49f76EZCj%rJ%=KqRdoH<8e4 zlaAcMtCdm*(2z+5`{k~$HF8Xr20JJdX9H1bM5S_NUwH!_`eSNVI3H^%cO=?2p_>{u z=BTfr6cH#{rF(TrSah<$-#W(c0139 zD3%WHg!3rIOarw?;KWT_iX)`a6&Y*Fr>a4GBGIqBWCb@@cX{S88MvH;?26_Z%NWkEzBuH~rApVWem+c+-2dRNMa%BG3=5Qf*WVgw7hHd zOr$HJdR}qJx+MZY(yN{eScwq^=I|*2VO0|J2yOUd_FbTMyQn@rV>KYkQcBpLI5wT{ ze+qAwy)hX3m7J){hyO3U|5%@Ug;47!`@~!2HO;D)hG{(mRGU?_IjHi0 zZMdEQj^s<-a|MKm!!lDM)ViIDy(76q?CY|P63U3-``bGHG}O}iQ;C9ZkaeK-pN zPTGI-v9fs6wppK{xhi5sXK9cc;LR#bT(fA&(Rn>A*LT+)Fdt z+(%k+2D%E$-63y%+&5ey5Lu9dzY#5fpOOzL_M)mVLxNLQenN@a%c|~~d`}S|sjt7A zujd8Z{)LrDRy?uww^RrF;(Zo0GD3vBQ)>$fs778bLfyHtr#|F2bt3%I{;DQi5^Joq!GK$mX|T!-WjkSG3F3H*1^Xs}8y|FadeY&%`P9Qq6VRF7L5# zvFnzvjP6AE^bgrTBz+Txgon-J5Sc9xy0juleP&nEU=u)gm8z2$&b{TVN-8#lhY>S; z?;|;CGc-&b&hlK@uuB7^i0DH@N^&`d3V(N*Hf-yxbWq_yRusMa3P3T|XAMLtj0FHZ zDh5uv{Zh2%mA zELb>_uZ#TY454>xKnU2mYoOAdAV9osDfD0JWYb5G^JYv8 z%poh{a)3X0?yPRe>9_ zke4kpYlKLqFKljC1KF88RTi-a&D$R9P8@ggRn2%@@eU%{L`$f-upD7aL)YlPqjiR% za3PCc5@C%Pz7?=BTo1ao#KtBnUfBM| z1EBebsjuXu8%xV|%L0=Z%yGH-n^xKwH=}T$wj>u21o|>vQ||RXVrjglJCD|H$WQ9GD%S#p1y`i0H<)L zfLN=D3DF?yaal;5Gyr)%6g5(A2H+SHhImIL7{b>+g?j87`ai~gvOV0A5S z{+HDw=k}7cL%Z#}5Mgl${;q0fJ4*y@>JrEXJrA;;UIONmaQYVRb zUN2?u()#KdyaB~vYXyJ|3Jm?^gB|u|iL?dBKt>MoPKmtoIF);@NTuRKE6$@+Ogmn} z*q&gsukNy3Bv(KN$ese25!-Pd)oqhha?-{Ij)hE`xORtY>L>2sF8#QD5t-T_Ob6u& z6F__AkEgsyw@F)D?ix28DcVJcbs>;>`kPX|qp4BdqNy?CrmUxsFU!DYA%iO;+}bzI zJ2vK@K8DuX5Mq}|_9CsMJ)C`xrA13;FdyAED&Pxg+eI16!%R3;o_t;GM{PnR+Xh++ zg7KnSbCaEtS+b(^m1H$QhYhk#^jB-4*S8YuG=qv0SIBrwT+Pj~2$biOH(Qb9rPTD6 zeK!jDcJYKu!}dX9_$vpY(rL@jnkIHC7+<{FRcH)PFoZ{M2Ol4&SxZ1S{%Hkz%g?V% zP~D`^BUT-DhM`9fppMaEv-ey+x=Ta~EWt5lu0u^it_eJwEpc4&O<;7L57%h&KN8oo zN9E9#B{e*d>SJ}97y{NSdP8{RUjc>L|OTd#`H{ ze&r68d;4KALpvl(B{o9*`n#78U&8V5m-*RG-~agbBPtgCQ$CV%Gdl$aVHjFh2Tt0kv=SF`t~9+*L|5T3kkUlf$m-R10ERSy{sI;NqwH zjFd^MyK|RvwpWdvHb7PX6~@6Hx<-@+!}h*l{KYK)B%Y!MP(Gb+(>-uBpC0YW-l$K> zESePN)gQ2}UV`hxhZxd*cjnuJOS^&r#(c@cU7FM0zz>u@epBFQy4sUfUIM6W2vCmp z?pnwfpC}zo?10FR$xuLo9Jf{$_mLvjVk6 z7DS1`tO_HNK%Z*Y*^vtWkWq<)wtFQ3w({9-CZCF;>GB`L|Mee_jQZu<@AI$W?N_?i z|N1-s{J$aplK<9!{`%Wtu>k%WuYm>%^0iqD3hjxH%q;K0GSaRXJ7ij%*+VdFKWc3S zy)s-#P@v6AkT>MIgbHS>0hhy{4cj#J#MvXrNqAvC*0rfgSsaIFg*rU&7xqXCQtY=} zgn5fwI0OxygSAe+>0E)WBCkbGEpH#4sy^UfNa8eNLW`|?tvV*uoRst}uS%j|vA`GB zi}Zy!KREcP3=V0jklz}jD_PDe0L@{aw!5L$me#BH`J>`;8l{Mq-!7Ps#`LnuYqb&e zScZbbx(W!-t3#m%^?r5xhY=ol3^x#XsU%5qH_>AZLzlTRudpm_!-v|9DT8Vqah*^B zAbM=iZt7;_9QWs}@ce^>HmN8dAHcSQbom5k8Me%P7=h~APqHobE-x7Qrg!u$UoW!8 z!WoS8ybDb>6a3YvT%+7o0))U)3T${G_#S+B3q8n&DBJNscFWh(V@bWj$+@F{i_EAkb+Nz-bwN{uc)A-S~aO0yBjQo7XoTK6XB2|Y15j(Mr!aBjVFx2 zho6E%`qK|Tktpzgo9n{kHEibb245TaUrblrK3KHn=P?qS9{5#$aRaG`QW<8efiPQ< zuhzh{>l|7K`$?87MoGXPEF|u&P`6;tp3|5BQ=DvmqGbPAU_q zE+<=ZU7^`EOj#{w)EJ?S*vmmLDP?elokaK&&_n?!CEVzb$mvqGWF22SP+$wSqb|K% zEM{;G8N~y;ekcTtAAT+&2NaH@wP?YhYd>J!i@kE5Is}a^Ru;Ni;sU?5F-_5`6GK3y zVhOy!-V4eS=w2iIR~6Dkm`i%eUK$(ov;0&qu(7Se;k2?f$`Jug9@+bU5C6xJwtt^> zWB$3C`TOs}+wa{r=~wyrFW-N!>HWKJUvlsCibg4)NoDC32O&n@O*QqiIXa>)V-RS! zjS#vyTup^06Na~8BA`#Y!^22*sSG$o2(eD;M%0BaPgVlS}Y`h3YV^7eAZ?2?Yw)UEp(+es-js2WTq+%zjV(*;;!29j zOhBbD>_9PbpEbpo&|e;;j&Uvwrn{0WkD%~xj!5w9_dnRqiKvE5wEI2W<$w9Q-IQ+1Pyt2k#3KM|tP96&@Y-sx2PqaKaJw#5d#9)J?-M|nYE>h~ zG2WwbDdy=>srjCPFrdDv$Uz^Df3hDsF>(}ol`fPlo^N&qWS*~ZOJHpSNDeC3r6mqey|vcNadlbq9?7@C?y4n2 z7XC}O(v4B5$#|P^Hj$=o6Yh1XzJ(MoOOjqQi_fhCj;1! z8702)cDX?NDbcL)*-$Y#=M03otywOnC=_(8qS6!%2N6X$1W}<2I%KCNo9sbJb->V| zU{e5#Z$Pjy16+40J22TKQ#2Nsw&nNsm-4^xm-@dWYJ3GN=ySnA>8YPyKK#Go?KhXF zlzzFMd;+s)EP92VuMMTq@1Yo=PRsnSOO(}la!*CG=PyZJnAicx`#7+BfZLk~8`}HX z7pv>u$xTD@JG_>yeTRrGaF8@q0TX@hC+kDkuAQaEdAc~NWP(GaWldzHz!0wNoN*sr zT}n6UetlI{H!6Dot>VIpT1(p5UZy8v=-nPj3b8?DJAvJ6_rWJStlN*ohAGC~S`74l zyp-3%{&!Y*w_$@Jd%=%~0&-Gd*m}ZY?jS^f_%fiJkomU_9Zx1w8I|6pgJvdJ#*OUFbOoPgT1O# zko&nZQ7d5i*UL>lq_#wPvO_7882p=q?UjUD18QS;60ED%cu^H(h>1%abV`4njWY;x zsGZgaotxD(UEm9sD27>n19Jd*%+Xi1X;fp}JK^rBlLp0t4L;sTv*~h~KH2lqP8G*t z5aWMKU0x-x7)}X5{kX7ffWm=)Wd(pckn?cusf}lSZ36E?6{6cMLO{g>*s!B32D0MG z+TxEq@+W_SFDXy;C4J<7_Wom?BdC|#4=x}6=KYKC;jh2`E-pU&wRs!BgmiKx4ejzc z>j%t+nk)c8*AT=RVO<-mSdUyc-WB7lmkMc5>>e5w3aFlknLD0R(=IcCjGI4QmHXHp za6xplyL%?dBLw(x^&t%KqtlploFNacVXH6e0JI!e{!oOKRJtnkix8Fd8eSB2!}291 zfC++UD(wFQnL2H$3S!HILBoq-0U+TmE$RXbqqb(P@d+ef^Pva0&KLL(wQJ)%F@ldz zQzxMKt!e={X@on|QD#BJPP!SiCnxnSq&hB6hXia42M|lE+Byn3#%$mK9N&prjyJVt z%4ZmuY}?>Lw#j6zQrO4UD*q0GKFjmcfp}pR3M9+%AC=j#wA?(a1136BG^i4Qn!$pZmmn*O`D~<+Hb7%=xc*t4^ zSl8XL*lHVD>{d?z*~|Go%-5cZlk2HffP!N68X+Y%+GU)fu?U<^SZ`3HY!0kxkEI)SilnFAc4!LQtgYX^#iAqpZwOh z!pHLbzkf5=qu(1?_`^@%zIgvM=r2Fa&w#x5RsL@{KY-=qFC=aR)8~srm*9^B#&!i`S0p%(Ay`h%+?${MkoR+OqU#FISzKA zjA#!Q*Tq8A`e;=L3n8Kg_?{6117<+p7%nh|u8FWy4JA$@q-AKY&m1SlSYUH`jVg-b zsUGS*?W$aVHIHA_BGQ%|%d|c)E_dg>rJYrY1DV_fJ;2x1X`@LDE%Cv*)(n&BqXMg$j6@Lxie1ZZAzs|q&^Y@=J;GOUg`Df(;>%A?{ zEbAu_XB0}k%Xg=uzZSYAeP&w_c}HNYaVCPBmPmG>jJc89uj@Lt&48E>LI*6EZ z0DGY$n@35&6=)=dT}QCP@_iL0HOvy3*_l(h%8%A+IktPgW&cA*8?c1;L z-|+TDjw-vqzXxRVC9I0vf?4l6DHK*;QiDyg)kMWs9a+ceMEd zI-F@d<{#ioNru<@h2GW}k9L-lps!NwZSSi~c)}~1tSM;|G*u4u9FoG`sX&&EMWp$` z20ds*a7h)piKRdXju|6*3RJu%|Z-^7xEbv21jl7)%lW8OPnZE zB*z0*^MQiBT8lO@3j*h(*n^>U7`Wa+n0cjdPXbB2cZF{Bf0hNV%GfaASXvI!nB@Ih zK7vH;-WXnAxdp1)tRaFEY3CCy3RkZUaOjvu(u6}L>+A&1THz_WAuTkS_O(;dP6HI_ zc}fbb0oQYqmwRjCLB-E7umglqD&pZ(cz-r|qZObDiLI|3hQQ#!&br@DNZ)*$Jwr?3=$lIQp_lV?>-ND~Yn4ahOhXkQE^ECI@Uj+!P(y2UJFI)w$QkPL zoGAGK0WTVK>D15uL|X;I&85elpyCe&O1%#IMtxue2P}-DbB};frULje6`R#s6+lxC zNd3N(l*q<;1f(+;;{&){+`J}9OFQNr0FobMbd?t}jCTJ-$r z)~w!{s@qhkkg&bmN=mesIJnMjc@JKpmYLUBPSKYzz@ag=U@LaqSWq^Z^~E{)Aw&~* zD@!R>)c|O;6?Li7B&109B}fm2>!(Tarwg3Ceyt`4PU-8Z-Pje=v-oN^h=Nt8g{{Wb zN$&qH{GUf$`O-aCfAaOW!`m;VO9)?o`|{!Y;r;iQSI|o>ojj9-DQ=v$M~l(AJ2=CJ z4Eb=}xE5&=^eyE?;Ic+aLw@&QV3g4o)p&jmP-I_6fg#pS`KDi$c9$EG zS&V?a`auu@qQX5LFt~pt)EVw!OcAF*!-bFU(piAK0tJy`1Twmn8Q! z&S$}yMx)9n1Y4n%4$8I4k2Qm;3h0s87sYDOu0pSkbkkl+0fBrxpK*d;!>zoa5gFTSdj%$$jey~VxQ*> z{|hUy-@fECKgD43_uOXu;O(En`>)W)Lqhm}J!N_?!(97&ARg!e5x(VepF?#f9p2le zZnQ~J`H`qRS zZeK#b@F0)sKx6%oJeF>6cTQqDd9I;)CrC(DG2lAe~`t`Q19{t_(N$_95)u z;$fkUi3b225CX4vOOL#H0ElWG8ztX1P)~P-RX3V}Cyh(Yz^Hu+o_kb|q#^d{q)mI3 zX5wa&;F$YJ@s?j}$|N-;`o!&|Diho(6Ot+rIFU|PQYCWbDK8lQ0tv1iV-@`ybah(9fxB0=B>7ESY_#Q>PMq;< z_K4sIv<7?0?FZJt){djDp(vEJjR`V-j4(LDaVYdH7E|vM#VF#Er228~q5q6OrC!oo zjv&8z`#ooeCbsmm_aB9?zjOKUJxCCI@9mHB|6faY@ZmqdexatFKq zd1QL{lr7X3MqqHB6ePQcrz#EBE-Q!|CCf^X-+;+~aVX$p+V_cFMUYk@TLoCWvHNgw3QpdjvU4ab+u-`0t9m_Pn=M1e))mPq#%PjX@1#izzca+M2WnmBy z?e8a~bqP2^>1xKxM3xW0f`?QR5Z$JPs%|poCMnH%v#r+LLQe-8LLO9(j(l3$jk}3y zIWiZd$R)KVBg2h2R1fPI2Kd>;%P^9>G$@MaRpFKrFCj%fCMy*_}LTsBBOdtsUq}3d-CNHf&Z0=No{#)uG#-k{~C+>_o3>!#x*gQVvQDQIh?S zFOz?!-njb*^=fR#fFfzJk`(Ufh)&+gRhaS7!W|Yu+5nPcVG9bmn=L2-l_#$&atBtxxSMeb zg9iqkk3>&^Vt?lPT5ToDv(*ja5Lgv*F+78EgtJ4S1*V9JiVAD7FzRpdHLTK4Nxhl) z9H|?z#f-`jFtJieDo!~*eg8+rX#DNlZ@>O-9vpstd4()nf*e5pI$v9tS9tDSXa{PB z0mGnB5h@casG(Zuy75Tio|GbD((F>;<(nFRj5DxZ8Rb^l;0K(5a~txlD>7Bo?O*pw z*@P_${|PlHWh_Z$iR z#M)9COuiC%qL6-OC(PjHVjb;brOXBMPL$#jr}75Z;t^MGAZ9`RX7t(=toTk-SkMl#M?y2BWTlCafVuCfN6s7J73H??4q6qR{6At zt)NdkaR4j6_fkIcK9vgK<{?!~#z&5<1-5!*p2J^wbiV_oYf0#bsklV|=s!x+)nczO z=>T;F9onlM2vm9%%8Lw*5(u zIL^f=$B2ltxfe*TZ=|fT@#tsL9sPaKUw$Y5{XCnl{1&is{RJ`PetCWp zmdiskuK6<@i0+59zJ>Sh>N*IYvAbFU<~XsrSkVsSP4(q28|jANgn2)xVPy&8?@7s0 zeH6xWi3S8EfL5BJdvIjk+F?Xi<~oMp9>6kGejfTQdl-&3DcTfC2OFu!P?Nj)RY#BZ zM`1*pLkjL>lqP^(DH#OikH2LoYtGz`Ld3xUS^Po)c}cik6S4zZ&xtlr1$B8{lsL!V?6Mw??VS z_-zH0AI(_}U$K0mPz6)Z6d;JDmG+vP2cY{}t4AyOP+-7m573#kcbt`>WhE{-4*v0XpqKBY~-_r z3EVeKuKy8Ipx=A@hw$OQzJGRkIr;QSK7YvHzSdJ+hd_V)W zhyei4CH-~{?t|N_0$XC8y~SIQYLT>{^$7~kN}mxhZ70ezu2{F%u5CPnqj1*1Sog5+ zNI5q&$#WwqCAp3xqPC8t7^K6s|m*LelmXD*ThTP*T)ZXwb}%FI7%Z$2p4R-u6KJ zjghV*Qr^xji2#xq@ZUI6O5lna6C_@2ZG*e)g?D#1YWh zH^uz9gkd|j4h6t2oc3?1KxxL8KoZ6flf`NEfKaLAz^r|+QOVfh!WfM&MB!ziFc(*# zUSA2TON+t)>>$G95sDoWLAh_L5*l&tq|wc$lElPPa}kr5zguZef0sP#)kG3o0cJdZqO=wl2rtSv za{(S>P;p>~c;Q3auxA9&E^!qkNT9@dM?htJ=C^!|?@zIgxR*Wdo| z!}njm{anh-&)z-@`pXYddA|P)HRo5sY_g&EVh!Sxb~BG`Uq6!m@zBMEL0o2_Z!7N7 zR>^)>8W3UC2H6?s8!xB;4)Z_k-QfvbqGq;;q7c)q4u{Gx%Lj-TtZmQ9oMecgKQKR@ zAP1$-=Eq7^YO$%~x_)wRc&zSiycVlQ>NdMBNMS$@NYoI7T&TqIlSqMd(8|SR&)c`^ zJ8X|sR3%?$AEQN=b+)dD2olovjl_$2>YNxb-HGa%vU@JJ)C@dv^Jab0;{^?pTFuF3 zX6y#=9*5sSL5qNN^Sd#-Zv6{NBuX6s zd8Z3_M-ZnT21C-dK7A|oTeOH+Vrch!U~)0J3DBS7%1;`QS6&j#B&G8RZ?`d0T}uK9 zn7R~*DAb{EsW6`}rAhcm2n9Ql_yCS~oVbIh3{Skmy@-s*>($}H|0(>Z%!~dH;lp>{ zKG(}{VbS-8w}1MEe!|=5Z@&z@P&yO5Lx24CtM{LS)TEF3Fuecb{j2;p|MuL92@v3t z{+jsV8A6>1?CY$f5688v>UZZOoE4@$7Wcp^wpf<5e6W898ZJ`WNGN+?Za|LIm=&}a zmpYw+8gh9XP;qyz>gGlY`3uxSkI*?jT2-I11UOE(shr2)15cPq&xf}f43(@ehbYr! zC<8kAzMY++;E*6pUm|o*q-yG)W1vR)7#meYvIae6c{L=Aj?TwjMi@eR6Rbuo8pl|< z*KkrHv6T-1E%W|J-qE0arMsr>D&zzlfR#1U=^kWDHB|$2-fAGVg*~?9*rr*87-j;R zC5-42%5;%>iA+S`RBf$mK7cX~oM)xZIve5%0d}Qg3tet)@fJ+ogr)S+0X=C0Q8lwI zrpP^HLv<8~1|KWJ^$9azskJbU`W$B#nDM~E zFYUf6VGtrphXU7{xTWYx`^mJQFJ)Jf+Jwc9-v&k1!!) zyN-=&L(SzSx>~NB{zZv~>?pu8Oyt&qeI&QeY!6VgI?_$o_e*PqwWbIw0N~GHJGUpI zhAn2LT|jMU=MjCh<@ZraEt~V2*5Wj7i+U~F5tJUainRF{qEsl8R{^iDyjopdM5pM( z)3scC9p2nqiX5va_u{x844^M^gO`v-T$?chOSv^Bv7m3;%7RXs2WwJW`Cn*l3InA| z<0E6`Yv&H|98^OR9vrC$)DC9@xIToI3N$8*)CAb|qVzYFh2-kf3Dt_+5DD^bmh(p( z&68IY7%<$Xsj#z?-lfK`eGgy38$5m7!z5VD6p@un1wvGWDDfJ01H@L)KZf9Gb&5V} zJ~nc!0>DpR6%Y@sCaDn!g7)lDvOdT%DliZ(JI>h^EZ0UG%<7tlBfF_t7<-0|Rn_zeB04}hST9W!mF)F=2 zc~w;xr6uWDeMYql6EjQw5OuSVRJIs$OeH&(38QsR@ut~CQm4Ya^qBPTz`Yxg5V}~# zeLkDVs^U)zuW7b~gT80)KM=`VIuWR>zr`NV_K;&u!la*$zlBDSomwAn*l*|SaE)sG z>s|w)%Cjp_E!u8c8!8|6ETb@~7}6^`j6>BDaxxF)7hY5xaJ;bq!NO@#_N!b*>h-}~ zrsh8BiUp{77@l2PHSVayIaOFBJ|}dFBY?raQo{T@ayJimLW(Puu0LIYlXQsw!7NvO z_c(u&I3Vw4htg}g?l1Cp$!Cz$NyP#9s>nxj3hEHX(lkke!;E=N?9Y)z{V-2|1sVXW!ITeIL4TCgo5pq4X^Uba=@+m4fDKg-|8!#k|YCmjBN7IQ^9k3hZvu z6x`t)3Xg$wNJ$x;G)I(j;VK`5JDT<)nae+7z@*G^x4$e+w92s9_b-=Mn`oel<+EWP zXn6_^Gj6?Moukz?dI~3c%t^rqdg z@$^hF+P!caV%(qvQ!D?-csGRdD|2ogz=E@&fLZeI3}hIOgdP>VzBtm^XwrCK#*^5P!9R z2xMW3>k}U~Rb+RyxU)Q?7#eIw2Gx_D6!_FK>dE1EeL0k))~l6I0H&qeEH|k_HL{t@ zJWM+$PRi^l^$Bk{&2e+9GE%`$zWf!{*RB=~tiGm>%HSFc0a3D;B}2uKM!*H_)K^ z@WuNNgG8hc^4~vZSi%p`4TkrhTwY7pTgUQsRc&}nT91d^XMH=I5RzG`Zp;nC9co;0 zOis0~Ytmn~_SmUxl0Vcxrq)UcBOH4k)Sb1ttPjXHxdb&ZGz-?mz*L^f;5-DLjf)|7 z^$0&+$uopwN?y3L!$bexv!c`4Z#4mBDSgS@a10DGKLK*KS!$8@unaF+7GApXyz>zx z#OXT`U0iCJY6vK{SckHVpOecm%nFNM-JDR`O1YCieE0497WpuQseQ{J_kS67r%<=f zK*?Pdm8jH@gbMq;ge(?W&{xKw9gs|^k*Jjye7q&sc36A7iXSloVP?Y1K#XCB<5W@C zA%KivQ{7-OZaBa080tJyxg&Mlj1B!5!4{>Sk4 zqkMfVZ@&IEbmM2@eo!^rH4`(GL>iw#x|UErVbVJ2sMFRgQCnQX1flk>S~+%XuuCav z#{xW!N)1o|p^2{qKdtJ@bK^BYLPSMNlARBS^?SDksQ48OKpeOa zq|tn+5FjGUD{oI)P3Tl+io=GY;jp764pC|*loiqmVCspx#r03FlCpPK|0tyGT6($n z4#lE~dpnXkh0Tq5NA!w3gJ%2{<*YKa^6w;{dq+rCJh{;NGn~{R`f7+YD^v$3#w7TZEPuX#+5NZ@?e0bGyg~HkOzK_yiTI z)s%j#BYY+?+(Z{FQN>v4M>oEb^?M=8$106yA#7|+;N zVjpH(Lo_u#R-{gsubgFi9J{`so>h1pLrjflm6w=OhU+52Rgr);VWVF?u}m0B`4fX% zpQE_tVmfG_5YKkcRN6&CmKp!)^smYyw)R@JAZUsZBI{i-A zwo82IjAdr0j}&0XS*j5Vih$&Q05h#c|NLM2HT?Jfz(+sD&iO~`(F)2?t)t)NXTS6I z8!T0S1MO(yi5|LksIUxiS;D(Cr3thNgq`47RELuE<@gA(r6rqu%@4AkvqC?=4iwKezRM9>6|Es7 z9p(PQA}CS4HL3-R?-D-=jTOQw7>uJs3IV2>MoPl$|G;U<)2dSdCQOJvpp+V*nw&a*nvPs3Bo3aGGLrJ#(qZyVZ|;PPyh`VqPi zY{Hfrl8QbhNb3;166i5f2D)Nr4C)Yh{7aE)`_>S;^$hW9EGVZF+@y-iaI=&9%MD}% zR@w6Xd30(U^?a77*h=z;GNP3g7#;RWT8jLRn)b$RK%EnySFQS(lK|ChS^4GFs)jOO zYHeUJ|CHO#O2lguuC@Su1x!$meOl*{s#0(oxS(WK%YsX#vaQrklhiiTUTRm_1`tL1 zzO7wk5<~Iav0LDoAYXhkZ~^=a6^X~9*`|(V1!1Z~!WGgguaN-kGHeCKrZe@skRp(T zAk>K`&BQ`IH;aAqssqj^@ql&u8aYbo3elZO4#Z-E3Vbme_AQ6$45k4~I=Y9!r+u*j z#Fq5#>>sAr{}w&Or*Ebo`8|*;zh^j$$1@m8O4|3k??`llE*J6zs02$IE_XnVqx$p? zvEeT2epdC_u_JaH$ZQ%!owX+0+_UdcyWwm-MTo(=cWBWE))s}+OY}}VTTQ{Fy zX-1coW8~bjxKOMbPh~0(h0uwPVHS>R`Ygwea&ULouHDQr$Y? zhL_x8HEVfMgQ>JaX#vf1MzIh2B_h=zjk=W`u)W!wZ>**_ZexYp+OpKN9WsIAB-sik zcWkM~k+4>~=S8V%aSAMLTnxkL)3>Tu*jK)ciUDE!A3Jza!!ydp zms0g54OFcoJfxoL@j)-Cr7l?qC2)))$ZqTsaKQkfLUsS9Q_b-FhzfLz>Q1_}_P%aF z)nM#*7$4k%*#|dIY-AS94FD)KNi(Z^0Je3;g@odv1?Yz0(}Wg<*K*_79ib8sjcxpO zePiQPbJvylr`p1rpelU$FYmtqq7}E|-tU)kD-Ll;Y0;zyzo%CxW-cyRJZTZVguxCJyCm)ol5=C0TZ%~BXOXs1mUOaLru)ut27cOBSvoqBZ9WZ+3B=Yc#Tph15w4aU;_U{}XWI2}{{SWwx*6Lz`B@*a zDs%{!9M1vcjJmzrsG`Q8IYAsEC412|$bHw{vdL9Nda1kEtCFw^S7)Kc&Q5s2s^}W( z!Y&FaTrh*b^ZNnw)_|IF-}n2=MJuh2w9U$f-V{B*^A&!ID)0EOB8j(P|ReJ10qi79l(SEoYrV6 z=Q7Ag&?%~WO_JyvLrP50+Bk7c=c#nCG?tisIuL?0!fb_IF@)V``uZjaEhU3(uxF% zcb#~(^N#(X#lE=dfNpZ;EjDeE4Y(3ibrO#N-lwmb-noS!~S?p(N>y zw@g?i^4Nl%bW3G^34a;>;_~O=-(CKdYpQWI|(ks%w#i z#D0yVn?`Q(G1D_!X)9ON{+6W;qgsC+0Ckgq}tx? zJpp{0y!Brph3Z*+_sw^bIuos_T0SHjSVI{Xzrhk>k!ps>+5vb{iObxYvieNq}LabHx$r7!GPkFv!o}5W*y@_V9U~0=LqEm4L}FJQStsz4K*bsY%&6u$)`hr-QRz?^Kav^iOni! zu4RcYVF&6KC3l{fUMxnnRv|RK`YXal|2_v3JotyV-+Uzh`1UJS_P@A%_^AY*&jQ?c zO%zfgHy}+A3k@Gk6mBY~(qB|6!qfXbsU67EqeQz7F2^ikOUh@I4pGYmmP#8uB0}0Y zc`$EmIf^O_btAYQM438x5~SlZYG8j8+QWzg!=QLpxbL{^=OT_zljSta8d{F zj5*zIs}1;GP4=mEm}_8H`>=+hU0?t-9==#E)X`y4W-x`oETUqtAkh$X)GAh=-kd}a z15Z`&EGl$2;r4x@J}g=5UGe)pQ@C0bTH~_WdC z*dz(0E<6n)QgK?BtXX>^KY<>eJ^MWccVL)Z=Wm7Pk(8Gsal*^AW%BEwr0zB{;Pe1RAW*IARCNk?lhfg?10vAUp5^00-p~XF!sCrS?fbpQNZ47 zdYSb-`b$g-h$S9UYAzoy;PW>qxVmj--ZB@mnO%nYp${Xa)gRIPV)T_T$Fne?@V-OC z3mmdUOZPHsthw|{A~LXqow?ks3?>pN^NT6A^+mFkjT5J$c*3Hm;(QTvaEg`{+6rtc zPEQCM*1NHWH8?EpITYLh564gvN&xs4sp*7iQQSFsohYK02ab_1g0}~kFFDWPhcy% zdIj`PvD;8@tvMzn)i^zH3@_Zf zsx)v=;cCifWGvNdAYL~wWRA#$SuGU}yQy<-ayQy>msB8kvNfQc0$Nt@9#qjbJa!C9 zs3f5vBea9|Q0a8G^TrYuN)uf{U8QEPPSm?bMQWzaiZCqQN(le>R4(=A=!IoxB$tv&}=}=G+4=y zmBZ*UVFqL+1bB-UC9rK7uSu7y@)Nj51q76;$r2Vy2(?EEb@EwhnE1rvftsaA-6OH8 zd&)V7g!zKCX}A$4oggNfLgVa%SRG4DrLBMQ^|!-`fA)hjP8!VE9|!%#uv>rq_Qi+4 zdH*52_kN5WgXR8L`6i+@!4IGe%xbUer^(DO^l_1nu?1_YN6`7VkH7a7JLR2$LaRkw zP#Xs|UMavBvV79N$3C8$w($kDGp)ff$$kqh&`p!?N8B}4=yLzes zL}sOpiSDROYZVt>rYDtmuL)~Y=wQ;k8=+sG^bQ}yKqx-}*Y5K*a^B zztc9I;1X3E0k;?i=dj|7rfM#PMNS}Q>EjF}4RTPi=WT|ka}@yR@dmlx&iW*=7j^bg zAA^uyLL!4GE*$)|bByL8=AnvN4bHh#U(~h0AR4tIwhg{nTvU0W!g7-)BTpehHpNBb;Ykj=Ayp+8oP_jhyNl$VKsn~s@1I+n-I!{ z(a0DtJ9@jr#SVhu9B`@!1?o#5cB>Dse5YF)X7!(Xs_BHs2^GcxYL>@N5@V?X)xmi? z>m+frg3Adp-HC!V(-XWZ*T6tIxX9gGxkF5io^hZBBT6WE-nCwo&gj#b*LCTY(Uc^( z1J4aCX4#=P@^olup$D}hnCXG07YEd2gJW}tdSdHCsRUoLa29-0g?(m zFeW+f14HH64n@6>`37>TR#r{50sv)fzLM9tGyTgCatyQt6jn37C?YJftobz zGkQ4Cfg8u-0s0YXATZ(Y2;nAw<0FILUr?dz7w?~e&hcq@{}ZU7e+Xvp`x>tK$3Eqs zFeZB(;g%|2y{k7v8$=P5`4QkeL3ywe9TkLm=7l|y&X(jpl@g<>(pqI#daKm$AHMtc z10AfWS2*o_vg99tiq>Q3cv_n3O|9w~Z#ET_mwN+yk15QK991gi*_<+xzuv0)+uote zsHF+^Wh_i@xkgb4+s->1YOuw-(#+QEDy><}Kv6)s|V=i!M zGcZ1F8$q3cRFFm`R$G_2XTzC~3YluKV6TDR zue$JBeBsw*IO_%PPC9u=6mTTb=*r~UbDY?yz%IxBm9``RAGDj2Zx_R&n<<<;dwMdj z0$Uy^`Oa2;+@N7dc1gq@>c2u@;o88y&Jt#Ns!t6y6Y!AYZ)nqJyOO|uC!P?9cfu1? zuA+C`g}ednM!nciI$e6IDQr7py(s?_80bCErjG0Eqbs-j*>pffZ?op@0oyHgPuSAY zScPgv4hc7yRqxm9q#`;FMnsoa%=mX#2T)HFRQM0&kU~s9RU6XKpN6;JUnEO@6MfCbM%ezg4Bng8J>0`?P#u2qz#odi8|uILN?5tT;*!U z%+X%R6xe}IhquM}4C{*jMhq5!F0!f|I@acK1vDS!;W6tY7H!08n(|B7P00DjGGaB5 z%CI|?3wskMREJT4jzIlzi3u<{mF%QHCe#mT>HOlMezidxFFvg2k&o3#FkOScpF?#Q ziUkQ@hwqT8dB7*)1^u7zE3QyO!w6E<>HslUIh6xL1&x!}sYk|ldZ*fNNxrVt_HSrS zEPHxAlv}a8u$4y@48Kzs#RP;Bg>h_8Fbx(AQszWf3zriMqF7?&K{2giu4&gp-wdYb zAa&5Hv(gg6;}P8M+btV6*)vdeP_;R$4%C){52EKBmCKqE#b`#zAwjYg@^~LfDVFM3 zih!f&?;J>$^#2p~X3Mf1*OlOVeub-AZGvPH>s@W#RUguAYL5udjK~-fL*|Kml9Byb zW?gzyy-5%Tf&f8^07(EGWOnYsfAwD5_u7${C0T>Udm{4QGiA7M-@_WJ!%0<*Bjg9o zOQFDc*#An6BL-V&MebJmbwOB9A*b4OfaM`uv!Y^NoWy|=*ZCZ>^QE+^OdKox@RJ;T z6zTBpi`QQTN@srb_HVCNS$uE*>LF{lqZFn}ujUxELsZIgiKKH`%2?(?&duzR0p4BK zPy%YW1EXNzlMmOH~!$d=45oue+`Z?G89NX&L<>{X~}(Llz@q@zx|lrZP+wHS0CIAt4IHUvva)s5IY3@24g zaEU#OVq@h5=v)#|j|S)}cfLhS%Ey?UAtMn&*;enBnF|P~ylaG6Dok3^3Rh>z+UfZ+ z!WTYYhWp(VtwK?r+I=L2p#?6N(hERSkxHSt*lIU$&@qD7(xvfy*Xr=hW4W0j>q!mp z{1P5jggG?_6e#lZWqdlRz-xjt*vv@=yi5%YdV%<;3N}Jb##-^0Ay@~rmdgG-kC!96 zKhLmIm;D!87h99eR`#_vk@CHw-mUowiV!jOsk#^q)`q9b8%8^7oca8e+n&yzg1{R^ zV}NENSxe{%^|T#2nfRGNdAFD#vF!v85FE~f+RzYSUEqD9a3n;n2G%BW1R6F1rh#YM z?=jRi?36HJS>JyOKg!}Qk0_x^&MGsc0QOfh2)U5Zd$naEYr%%~cWA)jG$y#~K2i(v zuFfO*5-qyQCV-efsHcQG>FLWi%%G)ERcMrDshhS{)JYpWiX1*B_X3dBavr@zQKT0V ziG>7oICDsVk;A8PAs6^4JuCj`wj_T%1tzhJWCdm&j!vz7S*lXW7loOf3%p2}v$|!I zilAvXbPO;Q#(SEwjwG5}e-e)Ma%R6+3>Xw7b7PmFaEF*7 z<#CRIqh#Ig1a}7ys(n|$5Y~C(ZJ_E%-;h_MgwQ1hsA;>HHn1V>@aZ{5aq8Agnz<@;Hy zUS7gvrZ8WQ#${s~_3z7Mh2hWdmN-)}h-#ZO(ge6ndosw&vo2n#l6T3jQMR726W`rj=?j=_;x5Q$8Yf zFOn+%vL%Fa4r|{r5CK~zD00;{W-U_(n&(41cFpZ-NMhwE?lT;%oyVVU}!wvSJcu3kHmBQ!R?Ph9Zw!Y{yP1 zJi}44AUR}AsnIOA#75S$^+b?c9FCguk373rc0BngdC|_H_ax~rY6Z(MAYQVnu}P5=a{@=bu_>^ZN1|j%rBS6-4~ilcNJ|Q-+*O+9D0~2%$?;;lN<6hu z>Bg3mk_n`GrGH79k6*Dbi3EmQLf$Oes{|I9@M`b8Va5eiNJ6P2G0l<_1X!4GT{pBe z=;$MXRtpt2Rls~B8w-+L&r8tX%(gw&H8aBVw2-pQw2q3`N%Gp zQ)nf1ys(7_48Sh`=#tRi*p*TeV0h@(-a0t6Awik@-o!mX;Ii*k9#gUWmPN>^vVbps zQ>>2RuJY>_QXFC6Mr@QSJeuY5^^%F|zkzdW&|2)V>Chb-K@eU<+V`GbDi z1ET!r6U>S%r+@_idAyrLor;`UCJKyvC0Rvt15bJ-Z@lT@Ui{?2x6XD5UC53?W-yJk zqkvISToEgDHm<*eFd8Z?l5WG7d^{e-cxq7H3kDF_^ZQglqTwxM=H~3^8nE9ZU5V8k@3Y=r>y-08b zZqx=%vRRDj41yV_$5k$+_A$1R_)FxT<4#i$DQ=X>#;GX8!_C7Y8-FDDOGCG16MKcnjJUZumr$7kWd1QRerifr`D~9sR&4JJ~M%3VQ5{v zPJ0?UEm0t9e=4AFXhG$Y@P_M>TWtPFyS4`$Yop^uq9ysf_r$^I=n^aN`8-9e*lC;OO zdL*i=iv(P2o=6s$Uw>)hD4DrV*K!b31ee+$#ydVx`DnMzntIH)U2*r~3EKtpOk|I|~dbgDgo4Hd=I3l;L37j`igFos{*s`sgW5_gx}?-e8$0d`i!=#ca4=L@B5Kdq)*!q;?FSvu zs%RMEw4Fb|0HKbu=Gzn*2YS@ImVftU3DHO}faIqeo*CL0ZTKWgiPB--Al-#S+{-Qn zR>hrPJ9A3X>bX*_B1DcZH_Nd+AL>SGC&@Nfwn*k_wH}(b*mjG&a7zq8DKK50qk1wVxoMW!=>2R71ffBx398s3af% z<54eDv7P z_d%J7N5^D_CNekij;_ct>eYrtz5zvYD`1~E_K@qtyKCXUXvnwCU<`DN^R4Rk?c9^< zfV0gN`=ghGa}xVqGQ|_5VrS5(X0Xi!v?BQ@ zNiROi8AR4xmXq}HKzCIVnEtdU6#*h_8(1v1P?3X?ou1GwThj~AR>;e&koTfe#~lH9 zM`Bg^zQ1s{E4ydo=9lFNKsSQ^3EiU>nttlS?9<@k@hpvJ17398VZke<V zLB0=-OX|i&1TGfce_r=fhn!Sg{r`5JyL5Ui0z|04Abl$shg9;^PQ-qSh zCxdD<<#X2py$JdejXv7x8kpel7wTlQ99Bl^hx#PpCIwNihnD5I_|ln&ZTViIqAbMfP++{comD z&3AFFWOY?|bj$IjObUhuTNNELYE}i zS)NE!|2#*QFJC`{_~(~ze|Yd)8|JJ@WpEtICLuM^8NKPJ z4yrB`4Te5EYgtg`V;8nALsEL=&KZ!y=T`D<;sCB(ya|se@09RdpG(@H$573!~u3d z&NJ+(#iG8ov!FEmk+ycs|f7%39hXy~v@PPl4_0zT=KWiy@NCMfck zi{>`aoP)N%r0SQ0@_3f71DcM@3e^VgLnU3SB@P6L3{} z2w*gWx5=!I9z?A-uEQ?h7+x?+MIroWPF2V$1_IWQB(HCdGk4lL^jnwM&s6mQ z$u1GVD!&V?!vKEwZ3HUqk{G|hzOg8pYp!yfJj<87$!tN#DXT{$TDeJ$MNOcN_ zkEv|d`+$Of@*q1Dpg{5&!uu5vR+O10lh&uGa`2eds$;D)W{S}rDZ0>4My8dZv+-)N z3Q;cGR_d~9kiy~4d`fa`^x@fgZEo&3FU7MHW)^Z^& zBnt^+X#!bgEab^z8ZSrV`OM}@SWI+D3-26OHVl9zEV56mX#R1D+ia_{P=w5Z!vVg< z#C#yQ0qn{aqQzWzu&pdX)R65C`Hx$CkF63HH4i83)}J!sWz{&{2VAm3D_wbmeElj4 zjf72*p9-2T*bn?c_}_A1`h9r&q_|wvD;j$9a=%&>{^vf2#Jdo$Dp256eF9!I3cVCVnD=@Y5fPxV&Aw4l9K*c zMd1M?lQd=8;;eSIlE&GrO42b|Vfa|62xm%!WWf}1J7pr*0hyWoRQyn>x=IGTz|jp6 z2sr6eHhq?FVzr_s?s3+-a5BpII3YVjG}}N7e6nvHtJL3BtBcLuxolvmr1DaLYV~9s zba_IQ?<63Mu@tZ%unw##N3cBTAC-X1%N{yqiiS+}C(RyIg5` z5MxdBfdckMep`hZN-}4w_p%4_sxPBs)lEDkNa;nG8(1%lvd|j6C3i)0-eX!H&Zz;? zXCMPFx`RL}>oE&|KFgq2^v$RgH6R&Kk9oBHW%N#$7L^nei^EPaL&@vH^fZ=|Cp%7y z-{J|d`f3z=JC9zC?Y3^8iUD8X4G(;3ybd#nM)Z2QUDwmKF*Cg{`G-y^w^nDSejdGA}`IlD_1 zVCoe+hSzq;?9Ot&YjNSI5G4jo?$H7nHoDar6K3P#364u}>eQ&nTHeUp;npgt1=V1NjK!!M;@!4>k*Kz6 z#sI34wQyZ^rpGD-Nf(0os!-sGdymjYFEfm7Mz>qBZ$dTmJPETK*^|(3poTeUZX4Ud z;c7GZ7E_4%`RHhoXjo&TGSxvfqSgR`Zq(hDR@1>QV~Re>Qrs(GuN)#Y+zh9SaSK& zB*sQJmxZ1w4}HlJmgPXpVJ6nhj>zw!NvW~vd=8-WmPE(l-LQho%jD8k@C$us2F*(R zFu;v9n}Ed_^-s^z!Hrk44HZoX`7apyK{arxB~+=Hja>&fY_vM)tN&0D1b_e2@Mc~v zU%dV^v`@c$`~B-L6%bP=Y9K?1fe`{2NTkLJyDe#(0O!Vh1^O*0E z1YtI4l3yw70HB4ox=j@-(G*R2ka#I;3d0|1W_Gb`ug>snFOf(M0)&pFp}$4*zLn4JcuD z?C_^ehEyhm`Sx&zi}SF?#egn5#)}tZO%qs%k`p1S+h<`~ zD{m2lOB}A-(Nh7Q-r~l#QmayVVx}Rohh1VcWCurLW7HO%LnM%LItni}kenk3Zb*xw zn4HZ6wsa|B*g)b<0af<;Dd((PFxuSA9>7n_F#yR$*U5H0m{s`-S^=7QU21s`*?dKa zu+W`(qjI*d3=1JL3rP*}men?m1Ju^XAvuyB3b*shu+Wbs8ty>aL88LMB2}m~bI~m( zNB}|K_mCmWLZ_T-6}wWv3k*&dSjEgPMFIfJa+pVBJauLOa2;@HRUIj*R`WxI^Yi#L z62)1}jSe&s?OZ{=MnT%bY|nYXvxDdm&t$WyV((~gG1|J}?KctvB%1&~=Ucgx51+n% zDX+c#`t6T;^Lq&nuipbfs}2U%2zmGahPThuLcVWW@&KW@a#6jW2AUu1ng$gX%YO}| zsr(^3A9(KkS~!>J%p!QZUMRd&)PmU^7*~KXUE@d2vwIEy_)b+pDTBB`8lKG)&^8ZK z!9_K1o{0OkQ`nZ6&12@Vz|_GNV$gq`$7(Ab&5kxHODffG z$^0G57ZmbC6@Wxr*fLEuexPY|^Ee1OH><{nB0gMAkgSCQ)zO?zsAAayL6b|VxWI$}qdYD%Jnodm_Q#KfV;M83LHFa$p&-HX8PHzf~6%6^vtv)|=E^XvbB zumQ5^CIv-p?DYGN0TPtM+U%a04n>dDDxA@trK+4;1~EzabJkKWL*#P-;G}>6#`ktL z_Jz7i)+1X>cw#-;g4;w)KJwe(&{xcmpAaQOv`H<>LLjG4i#!8Y5SQRkZ&>6shyw-z z^gn0}U_r5PoKsb4F3Dhl$xw6Io+gg+O!3pf#iU%b-s$StFcoQc33NGXVb6OetX}ob z9}hxGkmj96Rp#hZvz*4^LMa&J3WHr?93|NUnw$<41HUHH8lgW+w6uC0M#>1*Fv^s( znX2RC#fJ->)D%kXCIna|`hlEB+rQ#mrG^xkSAQx}r)ptnA6%jX#Go2q?m8GrFQ*G= zNU9xPxY^X9KT`&&qrj0jE(!(M06e#|BBKe0>kTQB_kQ zUoS8K#^RlA4i%Ty#l}>6+Y(Fddtp|Y*&gsZ~7PE^>gs-Vr{Nz!^jXT(M@(6lS!#1q3JA7nS0Vvits|0EPCW-l;RZfj_mrV z0Kg^osxka>v;_~tMm2b`QyzO83n(Z5Gm)wSn z3OC#wAl9q%>xCUfc((Rn1iY81&+&fMu=Xg~e7Te^V z_l{vn#PoQAT}9`f1xAi3ZkYw4eOK#%`PlXDmYPsMBvGvg$&-SEa&|tTJ9r5s`C}B^ z;p1f2L{sw%BCoG~|20&HEChp#v$vg{ANQ$SCEwlVQs zmvPJ@^@Nj+RmE~y(uISb4;@iCzb_yT7{*MwXcl|s4>#3QmU9gH9-IYal&R)?HdoR9 zL*qS%i3J_B2jOmeQZHHq9!X?s;OdA_$l*x?hjjqZpjA9KrK*msNfD%IuV;GyphNWn zCcm9EXN$(O+ajSCZ7<`!ujXjL)`*(qq)sa;{KfvGUYpnmR<)ZU<>tZLN8vAz+h4qX z&yRr@`nS70L&J(GQW#m@Y)5&HD96crhm9^9B+v`WT78wfn1KGr>1mAw*U3HFJkVhj zJ>P5feR#@tP-wVX7U*21{dST7%;0SCd>^O4$tkJFiz3az*;5PWp5q4{g7Dhg-=JT? zIiVvm6ae4xjO^eTiB<^}MGRYK@kA2Sy!GWwD4+BOQAjgWL^EQ-Z|2(MDRvz<6{2jXhswl zkI}J^ZBElz?~*f|}jB%z3xH*n1CPJ~D4fnZ_4;Mazk18c}sQQoKv|XB8Oz zQ7a=*x3b-=Z-JivuCB8M(QD|uNN7Ew?_dfYEfCo&A#NXm%@<86ysG)95*gH-Mr z&fg4+B9tUH)nKnfUwwLNU_wx+Y)=<-uol-F%<8D>o|*6AXk7 z_{nLwU?~R21IU+h)1=f}?wWu{%RIAuubPIuW+^EJQ;S|lK!;->nGo}BYlFmnk%hbN zlY`qK0C2!$A$22Z?eJs}i6%qkaUYV$%f@h2xKLq1rwDaWp~{mi4A-f|BH5Ax)FiU? zI$AAh{Sl`nWchK9K6;m~iXG(j!MKJypCZ5V>t}FcZBqKf-v?ggDE8UgFW-Lt?h{Fh zz5wa>m;a6o$v1DmQ*_AdA5ian@%m{FfOmKR)QoZAeF$5Hu_`p6R@Xt_pfEtW62O)z zRwD1`hnS=)Mao_RzAUXiPgqCg;uUFbDU7++v#E5iatd4IGB2l@&Z^#aC{ud*{#n~ke6AsD^QF+A;l!+f$~Lvfkaue`_{qm7F;s+s8k2s%@R;fh*WevF63 z3)#x+te6Q)P+;&HT)MDW-t&yry|iqTqGM;Yix?(WgsuznN(>;aJ+X68C1sNwX%k@_ zTc5HUVBYmQLm#P4sN*WY$xykxCtn0j>)N3>Rmvx9o=DqET{dcJqyl%sX!WEF(4{A{ z1w{xfZ1|RHkqN;?=Fs7wDtj7v!^B`SaYDbb8O zzmmVg=Zf+8=eN(o4?o7trMhVsg~pL6SU81=UAtiJzeblmm2`uR9VyF-0wY-!u91)D z4b51-U2E-rnaH57pd{%E#02?c3OSAl>E{igcbpufo zVi*9mhxybAh+N2M&t&)@BozdrVfBYArn?F`IgJX()DKx=X1E6^R2(X`18k5xFAlYI z4qgXl0fQ`Ofh3CXjoJcYpEif)fbxS;hJ*j*of^F%ixc6nqXJNA2A40nXDF(oBl}eP z^VmvI!~4CG(F|Px3t4ppns&2@MGyMG3=rTDpIJ4Td(Vsq8pF)d>k-v$eN1!jjq(6< zvrSc#NKeoK&eK%)K9UDG_Qd!n%m?e_*0kkn;p}CEl9zfeSr)YD%#~RwB*!|&I?2U( z=|QK$Nj9D##C(kI6~~L=iK{Y}%?vQXlRPU7pn2U-vi#*>CZJ8;@Qq#&^P$U}1x@aZ zXBFW01nZ-mut7^%`XBxkiG!xV1DM2s0x7vB9p5=moQdr|cl z;lJbv1~aLraqmYGt7!e8;*kHQ*DA4Qr@{SY8p4^>K7cLCD0u>D8R!O~Rmb)>#xCgM z-S8#s6=<8Hu7jrE0o@OvN}zFzrsSa#kBFlmixDOPlo``*kqw68C8X0D@{EmP2*<~_ zDBWIMWfxlevc1w%Dr8|wU9PM<6zAgV&N1jk8p1BoZP!F_uG9qcBx?le1Zi^FHazcisM5m3QUY7r6J00J z-04PYq|gp*h&QPPohp2@#%aIl$$LOvdzEXZ8f2)eQ$K8>b&vDD(>XI_ z7r9cCX;rxe;JBWx9Fk3q;lIVDilbGDNkvnjJwyvTjG&R-Ihg?RTD=~YXSCPv^qDy8 z*zA;c=z1zcb9HIvgc;{dk0IFEFvUoHnTQ`s7`Fu<0P93f4pDku4UT3~{eBx&U)(~r zlCXAn$6hJMfa?$El&T|M$-49S)Tw?YyVr^*H`v(>z#zysgvYKXid=MCGVg_apX9)3 z$U+7WK3~$DBrec%xG2L>(!#%b_v6=}hQC5YdHv{+UHPv$&SaADvm@p=Qj|UY z$bIVXk5~T{sQjFtnGc>5kUlj}?U&{T{v4#EEp{pO=yHv_f}bdSf$?^^{dSp2%>*{J1P5hgG5T1)kY zhC-daH6PKzZt<91*R#1C=?cZugNzuNA*)E>9dL~`0ul?rjN5R%Y_7Zaz19}QKx$_By)z)ib`E0hYICjj`!ydz?j3M0%BXA3*jv4=({3=WHX4#GvcSfY?Q zjx7Fk?CG`)5o-Gzk}|6SG107LfiAMpb~~-#J`M~)pT2z`UOzeA!E&+~p#$lIud^+f z;*<{7lZtB^>3fIj!L~bWX_(ZElROiHje}kd4p1+tO~(;fUmPw}84>&78@Rlu1-k&% zNqnrlVF@GXvesGHAv7Vm2dMpgqDv84lumA5gTYn9#BJ8|cp09@))-+qly1=mY@M`V zN=_zR%kfa(g1IwX-12T7YbYC9I0)XAp2kIfRWFa0%<;{)r31y>5f7Vep+z@(q}<*f z^}4MDf;p8V+Q(#agwP7J;ML-hLKu4-28CcBo{ZPASeyuZJxsK( zIzDID_^QH1Ido*VuDGA-eHvvG4a&?ZEt*J;V~Tq9R`?XX9s@In8gvYZ)^TERfmvd{ zk`-$5|FW=IN?yS0w0{bo`K;m$)51d%4b3MXi}W(NW|^=3&8vrW2fCls2?0+Pb)gs`3%JxR#P20du1UJLN32V z93nuzVW{fZ%E7VPueBg4I>^w?vDi_EaW`dD?jd2fG%uRD0E!p8?m*RGo=n}b1@td~ zUl-4aq55IEn{vE*;h&p?PdZKW=;W)~>|GJl zEVbOL8ef?BJB9yKBMOI*eP&oXYXN~)=CNZvJm4e+gj$UyAebI`m7Kh-Qe7MLMt#?Z zvl%QxVks$ZN4xc4L<&ZwqD+?(p^&VbhLH>ry`$IUaZzVfe*&8T&fWJq3 z7?6;WgKZf}uwF3Rr|a^IVuaqy7>9l21aLJI6j z)JYHgS-M9Y)WOL@l8kRUI%F=lHKrx!p!sG+K7ze{k+SKv#4 zW3dwXjI7M3T6zB5l*G4Uox=$U<=CX;KV3iyKm=6)T@`qZ!H%)+L%2 z`7IEb^X;%;Z4SyquxDP6cd1xTYW6x%jZ-_{J?%#UBc>+J*1)w30&@mo0y(sY%jO_ybEL|uogNo4 zBjW7v5~_^PtSCZ&IGunzaBcr(o6nwVfR%xSM$vF=N+IAVC16otap8q*1`h*NwL-BV zhpo}aPIE#g_Jk^cT`>~PBiw2DqdE>Cl-xWj1M4_cn)*}beQ$ee4jaFz*RmG!>--MQ74tU%dhC{z|f%J$eyYje?~a75!ixGHQB0=qghG_l3P5% zxN=7=wk*%OnUa8^eQhvV@F&t-=HiRtL*6jUyJ{G#0{Yh+L$1JdbkBnr@T&IA;Xb=~XQz&a)^W+TuGKD39X}D15 zsBV=iy-WKw9CK(&L34nXM=DXSpikKZD0E6R=ldpwYy%vFLHA~w^P%;mGOCO;N zAqN=4NCy?y674X4+w?k-4CF9dg_FNybZ*FGc`Mx z`z^HTv5pFMTuudmDMs}`)|{{Agk5`Y*4k?}RgG=Z!;?^Dc9f$RZe3LT*i0Qd6&Auz z#3=J;fA%MHvwwR13OtSn1SX`Q%+DJ-G5JgG8#uLHJl(gBUaEJAQpo3cI~ZABG2864 zNFiJ3z>LX(5FZXpDDhAVik@vsy@&S6!% z>hj6kXVv|Jl!upSeVnphch>+Aa2Za^HCQ-l`aT?vvUoOjot@@upb^-bQp09>CcXa~ zAX~<8e(I-#jAEB^5RzHG936~#D(5PY1`fx_mf&zXDDNlaithHD&PrG~bpFci5XW*d zX$^}i>$2iG?pDoI{Cc=Q+t0_nO(FV2BvQYQp zj`tcP0bvx8_k86ab&2#C`;OnF@;CP|9^yvY{3l4H0-7oa+g zx-2pQktAT=G(*2&rj-p!)S0D>*KJVOBMdS5%X}zb!LZ5OQUiJj`QII|>ilZcL8YLV zjfS&CGfUfTD`5lK{wPTSO-#VpJJ=Fp@OK!S7z&JHo>}e~h(Ck_YcY(W zLf-Qox|m~b{@TtnCT^ojsK`AoEvgu=IpnvNr$Q@fyjLT7FiJ%;C18p)N&aQfd=z^i$4)KKkMBU;jDe5BZft65OUqaN6ilMy#7W zaCIU1S@t5yn|s6%xu)dJ9ZG~tP;5=!$%+bGMwFMEyWu;Rng~hh)oa(F~Laxl|EVRm))~kBAgYWrE8Sk-~@vI_if8 z3US%MXo<&HD76FUPOqhI-+|P(S@+zbY$@dv^iW}zRn`ZN&Q#Ro;(kn!D(k!qfgX&0#vE49UptCNC@Ipn26;pFm zG@Mfo+N0X-I_&g}MkCIE$h(|JS#>U6YE(eYqQ*J4zch zw+FPLWgekNA3j_fj>NjKOEdzNDn}c9JvWR+v%!u9%+0OzYH>h-xwW6JJM^yR(^ZM^ z9GK&c@77_dv-Umnko>ddix^E-Mn+?8a2s1c;OKOJ4_8? zX7}1PCR%_8aqt3bt)&51D!N-jEDC5D8aHw$L&vGwg%vrmvMK1nbzH~p0olr1VI4i$^P}suW!Kfbn3Lq9 z-r$q{iiImn`y`Qfv{8pt5?Iw}p^>Z(ZTfmtD^d`F7f<+W7acmsZc(!%PE#QlFku!! zv?^p#^GY5i8``(wvKY>|LS5-=U-I+O+By}i5WS@&B|N&3*0?DpqqC)2)~d5>1$RBF zo8X#y6ysTX=?w~6+ zzNP_oP2D9i11LcGZXjitR$8Jf<%PYyc{oJd;hw%{u%$(s>*P>pjxCqfvFO(;Y%2gyyL>n~Tz`0rJJ?u5kjA z!F=#;VLHgwVArz|jd9_4LmCc@wVcSZZx16=sh)E}e)MXW2()e9rZ-D#P&=zG?6@9% zxh~y_hL?nGh1A?M)0a<9Ml{^YP4THE2!G^uPYp<8o^)eImgPYuSb^?hhvmehM1p`7 z(IdF`sEjWS`WO)MFy!eTDl3X_`1QarJew91lvSMtGC*p`&eG7!`M3klr+5OZ2l^Z2 z9o-Qb5?mu=%5c;hJ>VeAsV~gV84~#DYGFj$qVHq{q6AktZ_8HUA(m@&Dkfm(Aw-c% z%j{K-f$~VR0?wbS5h-|(0D)cz8o1@hVCGzS6n2{dgf3Yp*D`XQ>iDI+dpEm5=?yH* z(ImUz*Oc#QLj|-PlH`dNxX;*f%c)W`1&C0i9SC~|_>Wa{0n}s2?(&fx9XooyX%?DK{n7}wgyZ{LTxUt4nA#p0i z=1{Gc#MHuBR9T+j9yReEnO)0ce)5NjokC_~389#+Q;C>K+4##5VZQ$1$AK6A>g&g^ zpK^Zr$0rfVkDe=#Pu@NYuixbBkKTU&`Uln`{|LLHe*~lPzrX$}$RB@{|9^AZAw2qQ z76Q|zWgjYpdN&yx55s{wtQ6=y2DE%t?Amm-_GE-~MmjV*;U=P%32_+vH9vrD)=mLc zY}EBGL&+`HnH&vg6Enhvv!a$^R2E+cd!a+&cxlIb4bUa^Wx_V`IET@o$rH}#D?AM} zn;w%=SD`)|MLtVWE3;?zQIc#IeMuq)3JobJyv}GhL}=Kw6{I6USMf;8WaT`R_Siu2 z50vUE6>N=9ws`VQa!VD7Y1XCy(}b~hoOD2Kp%4>LH2Z4~Du&oDwaPu%Z8)QpSk#rA z-G?4%j!4jg6dIVrN)b>U_bBA8+L(w1>apK?svxMS1oi4!H-L2_g2WjRD0T)+-lTWxGmQ zMmJQybUP5Aa|XJ(-h*mUda`iFM!#74e}+=2&07HO!q>NGw}Hyc)EpZJIUbZ1o3klw zRlosP%wMaf*shr7Fog@P<6J>Z$bBZh;IldHI%#yuw{kiZZb_o}4Ow^&kZop#b{O6e zX)CUID`wV0_naItUg_;jN^VivuVWoM5_?va}}aWV)4iY{avj($K6ay~>2 z)~3K(EnI{vyG@^#U4^7+H4jzI!-R24Ky0;78oh8(z4*I@cF+wrYyf6B0YQB}p9FH$ z*&JRqSy+pt?XAU8-JA|;>YBkBB znxI5Em+W(39Be3H`Ocdp_LiLOF6#i90c_th6>CbEiYXc}zkvCtpQ_03j{VbSeHn#} z&rH1n^%~j;fFlQz*9~<`so0DfwjMqFp|R2xXPTh!YA7~c{uzs|T__NNu*rbshYoWi zXCqjS50ZfJHLcG9aoS>AxgrpbmP47QgUr8FCEKRyUa!6&hwrpHz%{(?wf}oMZ&x%q z>>bUW*fxtnpVI8u{^{ z6iJE6tEzh9A3)%b+zUBL!SUW!vbs?iR$;jl+%CDg^)VOeNhEd_+-|R*5Z> zT|z&k)cPnC1&cSHvl$5FYu>|(d?A?{q%URQ>!Y_{hu2T+ z`t{4V&-0%lf5@-@ljLi!Ss6zXu@dG~^V|?{ic&^#`%h0$@*uQkW)jZi6J(dctoZlT zR=VwSL`0{S)3``at3l_>{C#h+s=tGUiG zp+a8@x-}Ns7Rn+k-3pK;y8}k;S<^xup%XwX8zeHNnVW5S0rFre4{zGuyebwr<>+u( zQ6}DTcB4dH!jcc(wRPO)vDH<%t&)LE`6ge7d8=sfmYkwHfs+rFz@Eb>Z2OD7({gke z!DPf4PHtuxE6RqlD=bG7*R~o~HZYkE_Y>L}m<0|cq(al-8qsaRt1?rY$k!1)R5L*C zFvhoBt5KYpQ7avHcO7dWbA1Gy7NE+cB)kx|o;{EZ6x@t~U=>+|`zimZO@G4K_mIL( z6#|de7vf_{6(x1V1C^H?SQYe_q|(Hz;M%5Y$I4q6Dv6r$hV{npu0gU*X>u~998#G# z-e6uHZNfwmqouJ>l-4D$L%W?MDeuAK99@+daU761Or-~}P?21m^lSzgwK(RFloxkc zit55TVm2%{og>FvY`R!99mm?}crp7?C`*(A6RH$Xa$Zk<0%#rQ$MB;c{V4ojM-2Mr z^$UI^Rr`oH0ctrgOUptO`&LPUk(Yr{2A%~0<*0z^Y{LWkhdUm#Y8L={Ghu9L z#x!oHA-9sSC1*b@Kmc^}d`7k9?WTkd8n-f7n*@uX+)=fNNIe_*$?%`;_{Zpz;SJUjdTMOSKVt!7wjy zOA<1~)wuCJwdbjhnO&!1HBzNF6eR%akF>qI!M~Q zlQ+qltOgsxt!*C=>_Uf~3+y5<0oB98Zgp>ZfF#3mND8&uA}vGdphR&TcZC~TKN&-`$aliF^rK`6LGPuRW^ z{0RuGy^SxfO^OBtrxucAxABCwPx+pVnxbBVep$W$ui-C_aAoG%zkB`UPsiuKe*Gx$ zf*+s4+?%;Ze*N}!e)Y)^A^Rb~7U65N#g*4T3$H)T*Z=+XSMvXF(e{%6|5|%sIpm?I zD>)DFwU=X}CTCM*M^!=rPk48*996i6~Es{pqbCD^3YKYd7Dh*?T{`INqekkx4 zjvUb?rJq$u;?h9W;d+AcC7IsJ*6gSwEI-S1H1RJ8b9_@_EH97LGO>b_Z_{ySd8Y*- z%vtOr+p{QGNIsl$-`+f+P}_YPe)viLOb7xM5 zLeXVQ1l{zsYR-im(ef?T<1>9zG8Ra`%~s^?C9`vvKegM?!rumw2^AK!wl_F^QrNU}Y!#hm z!3;ePCh4tYMGu|HSkThi1qxlLPYsn2rF=dYj8P++MyWU|vd%S6E}Pn=NUj57xvJf6 zXr`+ngGCuRvfA;!9FjK``vMo2yO8=%zC1%upY+-0J9)Ip^|7ElZOihUy5^zefy&qo zaBTw8!|t@_AZJK=fj``#YEfc%&6NY7_L{7TfC_;8O9@sRjVs_9oP2r-<$!`mZH7+F zZhZ(KfC1$zAiDbA%Fq&0Hluuy12i{JE$9s(w=*BYoKGI&%Q4%;iiB!wLIeIFW&%pO zX+bGZQbNFBQWeFld=-))#uoZ87SqxGQW0yYD%Lz5wXs=n*bnR*g#HOn+B-NUj_Vb*)h5u+6iGTIEHSRxW4{Jw}zp>$RA1PI>Zc}Li zOgt_tNSMwB1KXPiCls>+>QZK5@?#P*fioaqVwF|sw5#jZ5reIcXAH zPX?FPNMQR6KI&Owpr;HpMx2B&A4DLW_mO&T%G<&0FI2{NG<<|XD8Uf*8LWR=SbZSTCaW<69k#N6TPT!VbLf=3ra3RGu&YOyX3{`DA>53}Vi|=N}a22^h zc(STn^bJQBsXo<>;-+JvAmodGCPGQa*a5X#AK>YCJmdpaI^~9jg|Vu}$72uv)DUWB z2I6zLlXZZ;H~@xlzNil{Sf%h0mp7IlQ$^%ZpRNoIon%PBDXjA2dLmSyM_NaV;@$#qR57BR9T$3g2XZlrZV0sgBc zFA_iRkZ$rfQ-Xu669u<26AEf=aO~aRynRXdKrX(%3v77jSHFDyg*w%J8D76Rz5Dcs zzYlNU$w$dwkm!8>_F4WjD5FW7EZh~{;+%Ezjkp*2Bwvx8Gm#8PJQl%T!&P!6b#kl&!1 z?P`o`V|%oQFk!X7Yua=;9;$RsInu?wKxD+%X5)HSpZINX0BF?dO)3uY<$XNG9aI2}HQ z$ucTr)vy|r^O5%3K)Wys-Nx`xB<4Ac0W}A}I{0W`N|~`QPv?qbGNRh6^=D8@MM~XhKkKT>0jP$%;HW zmnNOrHUcy3VXGM}YuP#$q+2=kx3dMD=pgaUez&#^wAR|Xtof4f8uZ=ltiC2x(L|`@ zP7QX{-h--v!GTQ*JUFaSZkvPO-VH^ciOtge))(TagAWo*ccu0c0f=fcVq00v;0t z21p#Bbi_>#uX^k)8_K>?7EM%aj-zr61Bm359}TvHl9?h~Vq57f8O!N^4}V87z^^H( z{Bx;@egm20Z{NOlTN}0O&42&V+aHM5-1XeP+0)oZm5#RZ2Q9S>e9&?K{zSorF$C-l z&}cV#uO?V2pc>0=d$X+)sAY#-UC0N(UIS#Jy^ZqpXl{e)F^_0%?@yMPCewOp3U3=h z#3ss?RcJc?ywaU(mEXv7CIlW=jK^CVEn8pEC*F7+yqh4!ZJQ2NV!$>eF}5!FSpysD z+k7uI;aw8AMtS10CeIW#S%4)vW$l^qW_+=+QVSrJho`&ihZ}IpR?9)Ft2F70X_f@z zggRYf(0SmoJE{2}oda=zlmFF+%SwYlG~h^9LjQ@0(*g_w9X|oyq}F=`OP0rFcMCQ_ zd}@BfKV@|VKIo*Rf!D~gIt5i)K$Dg0CS4bvqAe=@vSt&%Rfl+MOK)=rU0wFJ=b=Ea zg`Md6C|Ge*jGKJCB&10OJ0E3D#TNjZs?m>*OB!MRin+)Weg~i{8@o+Bek#fAID(Nd zBCDNzZwUqmr=`!Rqc~Glv_Tt;2@!tni5WQ%&%iF(%*I(yG_HVL>g<0f$^!jCZ`$KN|yhm>RI7cx_dGtc+*yb5z zE+x%nX1RVlIK3!`XRdy!ujzEUX2)y!*m#n)!bH*iO3~MsfHR|0EyuWIQp#-eW^@$> z)?R8=q`*kmVn&;mZ_}cK`7UlV0q655tHZmjfN+C8FmHwq!wcXJQM$QAat`!3o6BlW zZCG9 z1794(jV9-1gUbk2g05T@kTcDXS~k~{Z(cpFdPF=aKC9nK7j6toVbwovzt#zCsf7PA zHdRm7<|;PtPOVbcz4yP6Kje_4Rupl;%q&n65U|W0I$iJ@OoSD5s%0SWnbN(|%LMnI z!HSl&sb>mVRB~eXr<~4)!>S0_0T^`jhw`8LqHjrxy|hTY<4mGbXZEa}+Qn1ZjL+`e zI!2wy)$8F*3hoq0Etr#y^vuPIcHKerVWi|q>Kk!XQSz=4^N9W*L-F7JUHH2!cl;OO zZ=S~Z-@JWKaASB@0?(xYXX_d7-xq3DTC1?bPCuq*H&68xn%VZ5-6q(JSAK8Un zW+cQY#(K6)fzMVc;&Pl0$Tkjah7y>-$@|PIO;w3WP=V}4+H@*xf87QLJe7!+huTe` zx`R?>+|&<8^%IBv!!1^-n`_Vi!$6DYNw%SJ0H&>#mC!+M4{lg$ctH-|1b!YVe2|c` z>&kW}43-Wz3GSh>fNFO*jMInBVaV!=EGOm=EZx#jrC*2lp>w|!sqA(a%>dI~qSbH^ zxB3R05J>`O-s3#KsQ$l&E);sC3l9sXfTQ!#5ZIzB8nH*E-18lSHS;qBN2R3|>bQq0 z3joHFSONtw%F1ZL>#z!}h05nBnbX!rDT*xyDo@*jU_Ho|zeW`a%&+Ol5Y2D={A{x} zI_L+|uHku6Z(0XVE@M_VgEP+S)K6NX9+*lcuc+JDRIvX&Dn#I76`PctY2X|np*Tzu zu5&lS?7UP*vm_^6?UOGv)C-w@Ctp|?pUp`H!{DbZ#oX2z&Jt{2*Xs$b>o{*vJw6=X zXXwg84UEGK&C6C7^;1qh!RQbMvzc!($HSmUN5fZ7WG+%yss5( z4`)6@!-aX&61EGfy*oQhXwoAgXdo684<9w4id;@q1( zpPQ72ITFHdgvG`>C&(kWruDy$2d)Gd=15Egs9+XVRp%BX4#K}{wH|lQa(BG#5Ij`{ z(r4*=m4T&`VUs~|=)cb>&RQwowgu$}txA_|QYiR}2*6Tu3nM|Laa%MmC}xIi6zeB1I*D@A7H#Gg8L|8f6$QJeeH=7e;SG#7K&0 z6d;(pW&$Kzsa}uh#D;ZzG%oLr69-zdmI}(=I6K@pHg2OHFTtihHt+uhflR+SUi}6W zJ=amj#jnEKN2;Ry_A!|)KY9I(O@#;0%_BF!osyM{o8-WSx#3dpM9U>sM$Wg1@Qcyk)WjXO%pBqsrloqQ-FRv8Z$) zWRV6iv6?rMhI9aF;@HeDL79_LX+pZ)xKA_WaTiF~OVUIGu>{GEv{AFL?x4SWKn#%c zb*&bo=o#q8BNLi;W6;hJyc?nSJl~odv6}Vn)hyW3o?p%J0iL_$AqM)9Dg^Lo!*d>F zS*RQ;{lV^ZRary>tJSemO;#3ORdTH_%NbC1yb}j9-Mati)bA2uhp27+NIo(e;8)&U zInC~|O5|Zy*7H^0F{@`)AK5O8X3Qy?tN{xt4V*tL94SX%LYs=Nc#Eqfl%z$eZ&y`t zyVV&mBA#66^yTcg-l#QOg8>N_`e<#w%eF&dpdQb1=2RpcClc%OoWEqI0P+MJ7%jnq zJU!tD5*)6h1Jr?TsmCKfKI50GEe4>g8MMJ+w^7eupCz1?=hZ9+w+X&srBwVX({7PT zqLhs#FFZ(+eAi?ahX{RawBbn|?YKL#uRHp3FE0?K$Ghg_S76sUv6OF7pQ_3=N->7I zHS*!&lmn#km4~^w-I~LuNynBPVag9{ac-McSqM)Q)?f&8a)WBq9nYUL#Ly zXn4OiE6n()kssl!ny-dKm`Md=)Kk*QH*!J)_?f6_kIccEb(qkd0JpH8?r#E(I$pnd z#z%e3W*H|1W>>9~f;=d|yx_anU*a{$#J$fUl#n^N#3;lY@>_^Wc{;rW8?fZinOkj} z!CoOX~AExJ?J(l@?<=_sxcG znx)Z_4KyICPz|=MkzGCxs)aj|22 zQmB$%N)lF$D3fV@gx0a7Uc<50hfO8FE%>QkW7rw~9OeLLoy*PAnYL~^V73gu6$c+b0;I)3WOW=%@Sin zOq^|i4V>ryI zWaxo`4v5`is8586-l;dn1w@&itvvbF#$luv2v)DbFd{re^CkKk77K>~I>eUAN@#S( z8`y6MQuKw*6Qe^HM@0f%!K74C1dDc3#kQURm)fcQX<2)ugv9~|Dt12{ayO%DO${8a zLYuE`@lg_+rvXPdQbijba5?TR2#y87Wi$H*g-#f4#)1B5UUYUZ3B&9SR0)tK-4C;l zr~?S~E~UT&nJw|MCs!qi8k7Fd58nMahrn-N|L}kR<$uoM&T>l5NuRJ{{~m7a-9Nql zG`!`p1+W@j0!og?93AKbH*mNvL}DnR7dDYn9XM54Rc}3EyM?&BLu8n|Dg|bIflwuK z>d-v0HzYP~vAF@qpkfxRN0HCwODX4VBO^{pUcpglJ%U%w05xShl^j+^)oQ%kO(9Qc zC+7-^mBq+`BcjO&tj6l3nF*k*#uGDO-_ED0l(Z}dAb?bK)!5e<9VrNm2Yp<@k#iaj zz&K&)>FWq>dP1_~%i3kky@&y9K8#_lQ`O4%dcYy*5immgz13%`%g7kRAO+C`Q_zPM zMl-9Aqs1TX*m$UshXFksd8EC{>A}Ka_G#cH59>JZQeK`ukfq=F`9{EgU z)|GE?lwA)Qw!uj+0NUAQeb~-QUXIbOi?&BO8V~b9d6?xmZ|UFNQd}K*TQPsfnzNXAlcY|blBr~Adgv0?g=-Ro z=7Nd9(kwg()Up*-qk8V8k}r4jIC;l z791O(C4Zn07J#;WWe*Z7Ej0oDoLzaHb!Dzz&ggpANYLVI@$P||P0c*OS}lRfT5X!g zZk#p8Fd^Y9S{SibI^C0@gSN@>m>U=Dx}rq&N^)@fhy)3wm zbuPv7fJ}xC677X&8J^2_lu?p9oP#4ZVL^{91LOi?Y0gr~v$uK<7xPM08%1<` zv+%=DplJVHc>VhH*xW7ae&ogXbQs~TU|*5|q1$)WMzeM$yIW&3=M_mIu>b`H-7&yJ z_0e)aBIR4%5C{BOp4EyrJc0797^?j8h-NZOD`1z}E1}=ALAjM(i?bXh`pnZjw%!X> z^O+SY61%pnG2HB36FcHJbjQ>sLg2wzB#`L9NUKX_P0WE0+rU6l%@?my4&{RIaKI8| z!7S&WQ0+ITDI29Ad(SUpWT%pxv6Bx-TP)0~m#SJ-93nE3ijdM9YW9ZZbAVog0(gGq zAj0K-2i>MA-Am__zP!knVFmMeD9LSZ6M8iE5PQY$TZmEaKU%VE@t~k_TvN($pnRj^ zSKLH!DYRBmivlD6n^uh>EJCSDpIs7fepM_kapZ`8hA-Dap45sBL)G1V?9fJrE@E|3 z4XY&Krhq)8g%=J@aE4xG1=nFIfX9T}uRE{8s)Q{Ao!4l?$2E1&ot&Af$ZQ$Ixw#eL z2Gz0MMQqPUQY(4!OszU8@KNfI%A&qN&x+p`WxW0S&ZWFkQ^AR*v4Sgs1TzqU?x|0; zxd}U*3*lu;C2mWn&=-=`w|U?|+9to5k(f$Jerx**lJi_q%+3a0g3{d4-6}>0=^~?{ zW0;C`)?=3r#exe@RuxbgT}>s0hn?n-ijp-TfeSM24QrX_Lr7!5d^j3IT88|$sH!#< zq`M_R)=HIoGHwNibxK}5#S)HxSUOBMb%dj{q{i#3o2rN+p*-eUtMot290<7MoK{TAghu9JLaUWc?pY`u8SQ2Mc~8HP%I=$!Mb z#;8RFFO`rg7*^hQDsdo!AxQ7)5~BGQ1dzfkhw2xA0a7!|+6LMQA{7i9WA$VR zSwNQM1^QhHYStDrnxm;xM4-yor=%zNPY4qQ6`jStqA`mk(r3lpUIXdwB84 z>L6P3;%Zs_xViH^iyrKp6GhGAl{@F2AQiY2-qwWoHCtFJ(=cTF zYsLYSEC(c;a>m8sBC0i}JWMF`#nx>*PzBuH7bxnrbL5Cf)qdpRFSRg^`|iMU$J+7N zrFy(N`FttH`&yf7#V9ReBqutKfZL z!b_yj&IrvWNiAjY#_2iy<}Cau&mZWyYdyttVKs5||@-LsT^g=g-MEMV8-7qOoyA6p|G(m~hp+l5K1)4%C^(&V|ZUi4LFh zF9)DP%#1X@P{z1O3@g2vYp9-PIl|naU_1uM-YHQ12&-C*_3ZXJo7z=({mCX?0ynzA z5^Y(#$#P*I2)*A99}8A#X$$NM!5F=>V2`S-r+$Do<;ikuQ*y_gR?%U0*#>J8wD2!6 zT_&hgC4jM|MC(;A0-3pyn4k-po^j;Bm*vQa9;bZUO3l$AyyBXJb#sZ1e!;@4q>iHt z#YNI9O)T00?JjnZX9ppI3ta6I1h>R7IS}E!b&2qO^7B<<-%y>>0L~FXXz1Y8%Ft5D zxrz-CRSgF|j+Isd%eLio{jL}}P=Y&&W?>|E2F^4&MDC@V(w4iu3FQW)EUaE!imQYK z&l%KlC$ngIwbUt1r$Tl>2NLglI+wxKf6)-SpiaVAMMGxKDbN5o0UIA0-+uA>yFbwm z&=B|bUC6ldhwMGW1OZuZW*s{`5^`XB_6e2)Y3o(#jK0Y!Oo%whk1XCuEZ@oJhh;>G z*n+XB(_}vk^f~UKZ>)R~pV7#P{IE;KPB0N(>L<+lat;uQpv2m)Mr_(zwpWI7MGq-7 zqz!)v$bjYbABM9-Z%U+4sH+B$@)|cHgCP(fJSb>NG~w_kW;midAWEra3t$4-?xezm z_r?oTqymJ%4kMHVcEt2cNNmPW8HM>&rKrHYD-f z!f>oPp~R8rbUs~Eo5D@vmP@Lk{;}>0#t){xT`*99p1{mT-LDm0VcZ}oT!9v^b$)%i zY!Gwc61}8wNZU(xgER$bP{Yt*8&Ks&;Wpq3*=Sm}zXV^FQ}c&gh{Qj48Tc<*z7}oy z#kSf4ezB+Dt6H8K&|t+LR!kVM(_#S5Wl6`x7kkGmB(Vi&y;%b?y*!Z|GimB_bbAU& zW!P2M$(J{}iAo0r?U01C`lY8?oeqNvE8F}AQ4pACh&T4}p_Wezi$we^*3Cd8`1IG| zM?d<}e>%4Fuirj>RtEW+IKi*YxA5KPZ@&vB-1q_Wpi; z9gWX4_-G8HUrb~cGnmA^`W$VE?W*-rDWF?Oci;iop?J*;5xr`2)Q#;{#8U0>zhK-xN)&9Gs z3M2XP@SlAeSBwy}0I8)s*7A4_HHD)~sg*_t#OTC_B>c-UryHQ_oj39h(T5|j^_jP2 ztcu1-Up>6|tV}9``T)53+Kz@X^MU~r*{~9j+-C`6gSEgE!@wzeA77m^!Ta489bYh=3) z$Pi2-yw0rne4Qbk&;_|E`^8Rug|lfwZINQZaLj%yV3jPPK1)2=dH$0UUnPJcOMfJw z$kS;}z9CL3d~9Cqh+gUgH&|wqz5;6x>OqoXt?p%o*|5JmMR7njhC?S`^+CG^kl!BM z1}*NM_wm0H*!!pV;s5U+9AD*^Z~wxx{%23GhPR*RJ9rQM!`nyi{((Hv2PO=&E)lx6fxO&n;&7nZ-=n=7o7ObkuTTKaJLri-Q zQbOPfut@3y#*y_+n#Tk7DJoqkWx7-kS`e7?1|pJe{Bb2oNfn{D<EmAJ)`MF-{2HTbg`@V`8m4l^xsR=8C5&%O=M^z!}u9PF1 zsb$vNXvlMi$}tfkXdI#EN}CKHnBllVs}U|e^z1pSR6TW3Lt!nS7Lz2FH$r?;(up+Qv}xaH-)nNp*Y+<fa&&?AiVkO3xBNI5wDjn~zYX}FH1+b4smq3phq$Yngn(WMiDD3Q%Axa8MeLEE8p4TQybFk=;VV+L4`up=yK zsV!?{L5nxvpecQ&Tb82I4}C>2b;#8#K;f(%vIvcueMhct=~+|_INM;dmS`}%1=SVf z#vCy&hFc>D-6P%NBs}f!JJh^>OCHb1{_5)|0oclqpziQ*`Doo`0o`}YwnJsQeCS!T zxVlC3stu`;ZQmf^sMV1Db40=H*&K3H>O1x2}MGG-A(ewRSglX9&-q|grXo=*IWeyI< zei$)VQmi9az*_Ja1?WDdL)C<@fKof?Ie43A+w|GgiHo@d=$)Cs_>leRia(7a%>hAy zcm*dMSts3Y)B%du#gfsU_|7&7yo_BRvoupfmfj7R2$OA4G~emGk~aZ2;B35M_f45w zso3bSLj{@y&RUmEioX;dyZoUd0I_rdmI?z4N#~(F4&H8lF-f9$#K1}wU~VY+<#8;| zl9edX@X*ehD0YfLAvI0FSjs~_*w7O?Ycz%c44G2%gEzPo^=-YYC-nv;fo;^DJ2wk-QH4Hd&avb z=z2B^tyt&77qAPJ74{O`piA%3O%7Rz7vbbF^EB2Db9IB-Ip*uQn zX&qG;tFtjE;$k_@vXQN_sO4;hsM>+4SW}B1aj?2iele-GF9dlE8<;m%ajb``IgBDo zif2I)g@95}IL4QgdLSzdIjjfTR0VFvtyzFVqs{#EM1+AnF?K=CcO>B=};bfPoy4#xUeaU3h|ByBw5e^`Lm|hIP15AsmqO_37OzK`q zWxXkqIWd{cfn>6hSscvnyZOI#zT@XR9xtn^(=6YOz#RYt!jGTf8z@*)E6m;Q&+eh* zEet0{b+^lpwhGnFKF(@M!*;`UBzpP^1VhWw1F`|<%E)#H_sDAGCA&hSoT#!suORh~ zkevc${>g3Qfw%o;hGRUN1>YMX#KV&4w#Zv~T?aMdA3sLva5tar)8 zl{eGsVBo|SY3I(5r!81ga+=2q3^s}$oHWtwUDgJf*Sf+7q9U{f&J(RV1(Z4RF=cJ} zql5q;O(gr9DairD)Yv5R+XciL(z2Go$TNp+ur6+c7W2+Y2e4JDX2QmP8@yW&`P0$L z7P*PQe?@?;*)BzX7+C5^SR5#&=K(SEVslwmgv$j;4830N^wZDD5obE~49&aOk2&F86UOS)%QbKJ_pj^j) z4*#nJJy=S7b$D*I%5tz1epmiOv0p}Ec3t2}RVRP2hbw^lBHuypo@~N^w`WlIGN4hG^9-DONr-lo5u91U5t_i8aVs&#|K_Ad+Tu5zj1efo8c`2OrF(VLi<4 zlksbA9hCsVu6`;Ml&n@r;$Lr_+7@u909TS7=FPpn&?p7?KgUaCr9j+*nQcK80~(28 z7O~)?ZS9+D8Fj!>2Ri6ONAzu4Rlvqay^`kNTYiXuF#KM%&r!=Xl{}n)4c)!D1zzc@Nn$mRmCWjG+>j*OIYu<#mHOwuUt`ap~PR57ZhdV9|m6R=m~0Z4(p?xm)b zNUR8ZtPaYk4yT*fGf9heR@cqOG^?3o-s32&I=jDlBXzL(5E|uhF)u z1I3b{#dTdy?M)E|Btk1kiQv2jpat1rMw>Gslxk_Jlewjxr3krFjK1v@cbDS+^k_bT zODnrF`XlF!3s364<3s1tHPSsKukP-ps;3skM)NUy z-4Ynl%WG!WA5#@-&l)8Nf)&QBO@&b>`&qS5bG)rOhqa7^o5w`1#PQtWX>3G3o}_%> z9w#sUp}jO!OqpFMOCsa*Nyby*$iWHm(G*?i<7iUNYBIqFGidK{x$2{<(SlrUiII+~ zU~QGp+N(#`4yx1Z&MKr!se!CVeP!Ff0}`FByLc>Q1$-cMV8vEOObkLiOJrB`1&6^T zV?jS(lH{Dn&hEF1no3_l1{^l86qN6F03E3!Dfm3bQxSGPy9BG`g_0N%a4cI>33`OO z;5J8*bGAQqyf~TjC<%-TYVBOYBHqG)U=|X&*UdOhI#89oPp2_0amSE*n(1{f zY2*Z%U1+}gYOQCbb^;d0ekriG<+hW=N)b=PQaA|EkY|Rs1qzJ;?7HNbW?7y3QA?&r z`S~Slujdo|&2&(g0LkNtNQ|K8v@krx5FG{x29QBXBsWe6cwpeJ7Mv8s6dFwLevL$mf8+^m<$K^`B9Tc|)JnQR| zR(lLJkSiJ;h&RB?c0P2wVRR?I%X+d?<&CAzLw9=!$SihLt`ACkUnL1z0LH=clZxe9 zR>dVLk?SJ4lPrIbIap_IkTm9U!>Ft}n3kLURTvb)AuModoT@;;F#Lz)wj-r_)FGaU z-9Qi75OHrdY&)y^UF4g^B1gnB+GA)oit6C7_ehbz=Iku18P(D)AA_kLna#)xai}wF z2PGMHYG(x(UCO;ct}EFF?P-J!xj!$Dx}O#0l6McFZ2RWr%Xc5Xd>&r@lAW$h+z434x_?p}ROXxc0UGT#&KRJ!wR(EL84n37j( zuN-j90RS98H}i^_$0Hgc6$>Z5_qZz2KZfsU+eoFXN}jgPbTY%btEZ0^pcGO@Pq=qw z^UIJ>Tcj8jvAGo=I+ZmLZgl~Qr!9bi;UZ7AQsNTnr;;7(m?_YGOV{9D^AN)y_azI| zX876NOCdVG*>S)?#5VlLup_r)MpjBBh-}EE=VHE6;`Lhc|0{6Ct<~mN_|wF@18F66 z=9~ATJTl7*K()LxDo6zJPDraJR$ZdKGoV-SKE?=;rd;%pKR+meqGXzqz?|V!U5ys= zAkXCb_^O+Pq6if}!4i;F=13jFS8k1U>|m^nB@%nfzK)%gG{{+zoIAKC0;>S;W=(tD zg<{E_af={5Iu4^mh&6?s0D1!eVuvqN1oW=qj#qDGs!_WZeXSBQlUI<&u$HHn&Ae$h z7ps!@9ME^!8R2a38*>NNs#Ez2vXBNg}nb4hzH+wRs>_&^;`3!skRgzir9A zHhE3?qmJEljzs;U*)+0OA7hB_e)BisC;IgVKls7Fk#v0Z`XMF#jKTe@@BZ%Pvv?0qQOPU}(T6ipL{>fZpzVq6eQW zHn!_C%RX2zljf2QdULE zIJuclh8Rx0itEZ6djkN1vGY~>$b#?YGxr6>)lm~EFz^>u>xhS0b=u~{=_GtFt+@pK zT-j;a)|zeinedX%91jd?Jz{AI`o;&TuxHxkRJb?&=0|Xoz1?z5Ds z3Ra1MVIOkIX2W@D!zCZU70<#^wmm~RTT{YeYnZ*A+X<5o$WrA-&^g;V*zvHawK=~P zsP8VD>|EpD>Tr6f-7WZ&=9|-vI~T5#_5vvZ+NsF3=d zTW^(0d^RMGO37@N1ih`123TucClYR6+Vb8{@X?MmO+_tfZ6#0-ijWZUs5P09JOI%w zKO4ZBz!G&k!&!GfyijkpBXm&3IjNsL#h)5X>l^FX*2?5^sS59u)m>7wmQ`5i4V|5` zD%1_qiSv27ET6FbuDt6Fd!dgtu@iNcKvsZ;S>)VShfh^oNavYg3Jj_aReGQ&kg_Y; z-o{=v!r3^B=u)k~OGwG;y?fbW!yATC^o3KqNEDqZ1fXNbKatIC=^?-KZTRkQ5AS{k zE5rwg3Yo2S=WZ@9g}7jt)JRQ~Wlde==N@Q>1n4prFm309RHrLKBdav%0NYEJ{N)(c zUaD!xT1OIi9jrjtOz<&ol2$0nWma$pE{?ccF<7D$8pVAbIBQT<@CYvt>nn-$J0z+}X}BmL;bY{kK9RRgfY==Rk)XiJSb{ z-~y*HGuu2&&hXEmw$|vZBWR@CJ&Wjh^pi&RH%R2eMC|dY)1%YN8 z;9UsK=7o8t%946Rt3jy=BH(BhcLQVs73pgOt!ASg<>4}kOGnJ;uW2q+ry4gfFl82S z<{{(t;t5JQ=&HFaM>PUGNpUUth>-$pZr!>#7YSWL*3ebDA{AU=TEC$I5>;ri8z;rW zbF8n5e}$Q--h8d=H3LU`h|Q+rgE=0VAoGXi?5mgG=E!A2q?vH~Y54DbQ|*4ji;=Z{ z`tm`@FXySkK>4X2BW%8#c73QPg+6wB1Uh;{V;oF75cA?n$9dVO4>y#Dh-QJxaLON@ zg7>gSot@h!|rn0Fxui?WL^8(`hA?U3wF5_?%0@cg0h7Fwa0*AX>_Y=!+P z7{;mKhg*e7w&VD-$^shNqO##-8`eg_k-YccP&f-Ef_jAY*Knh;hK3(*xl=?xGJ?Wp z7>V2-jfE9ju@^q}-V7ZpS-nybTh!4GJ|SN6_m)VeXE-w5>!H<>7khsNb#XDc4_NOu zG3Z-YYsQT_&hjOxTsb%$1)!<@@f#zUY#oykZDhNaSU2Yo)#b+P;d+M;5*Uper+C<8 zCi=kwOfhGPjP5GQO&oeA1eoxlnQP3Au?BVAK5CU@%lOL`_=z38b5Po?PT?f+IS(ql zWZ!~Ju%yM4T+V2d$Ma)?qDfmL_Q_TOV+~LoWgE>;b#A-tg&Nr%hfdqhMko}U!V{!& zC&DJ=zC>~&e_dYENfrI_H`FqL-xhjtdb>hvtJ@i>Xv$)hzNCElQ)nPr zuvwsk04>aTU2;@5-pv+a6X)wSC}|ZYnCpyN2Nl5ayxhyJ|sv~M=MvPh3HV3VkF_?y2D zKgxRY|33WWC;13P?0+h;4Ib71QlkFr@cQd4upyb*%a`&KuYY;{_VCb2S6c6!4|kStmy zj}1(R7VC1flo{2lfhYr?BFTBK@geCEE?&iCNy;PKN_O9NwQMu_2+n=0E4{;fsh5vU zg34V|rOmaz)c;BT`c=in$u^nCo_y4uL43AiouHsKxL$-DZdIYuowFzJ9ex1po2NmF zqFdV*Z)pz~$wZ%lnqN)HGEV^PIiX@W>#B>TWp#wXwkG5jHJmGU97jvEk3&4t+WOc% z3Z&=Q;x@7fqyA)SqK6AI2J*|G7N;T+3jhE@ zyQ7KIL0MaVH*U%Ys@<4JZ%!rj=fla zd_mcJ8i1?JhrMbJ9rlW2a0Fg}^vgfd-Y|ws=!7u)I_vB`A1Bz26aJtW!{DvEC7!o9 zyLbf%)Ip_aIGBuXXEaMXh1Y1DT)ah^Tg>QmKo+L9Q;42v-loR)U9#Iojj-3&)r%mQ z$iD5Av1O}68-TcutSXU@seFz_PLNOg7Kt6?s3?1;QWGUpD;-MklIo3?W>AE-!vQ`Y zjG7B9BpOvtM2a*XdX>ANJ)?dPcI{FdhtiRm#QU9q-mWlk|r?d&sESfKK0e z{xt{;WwRLEeo7yA7n30;#HDtJq`KNV!Ur&6`~VHopl z@&$$^uEhqI*gf}M71~=j;{e*K!w@<(1m{ckxoNW1tJ~QyBwn99Oyu!YhU(ZkVQOO` z>`(j}zW+dpc)ttZ{XHY&*Nli?=kI^^`Wc3zpS}K>ETd0eBoX@!#-e`;uYb-u?|)(0 z`RDvIKYjVeQ0sE|DZ?LHd5uYbJfuc}^R;)ZpbtwKcX)CD(|CqD#O(f$Ev721CS3Bd z+525HfPLDlVlnBS)HMrVKST<~WTd51f#Mvz{C!oJqK$^q-tV{_t789pWug zM`7x@$h9U7DpOk*I6pgH^4K2rvYCqh-rhq*Cug(3iT#D9v}j6X=f)5 zjXN=#4Vd|s%FU1`E26%u6c`mI7zEuX0G2f_Ay81t@xB}K{=8L%E&Sc&0&yR z=tO1MXPAxo;@l$n%nGqc`#})u?9uwmwqjk8Ku`vD54?tg&6xS(vbp+6tOH6R3nBmq zEY^-!dnD5v0cjujOq*fiGgd8{&9$THsih*2OT5WSr)bb^{+K+;VSk$mO2Ni(BQM)C>SV8o8&W!GSF4eor!$(f-5LRrh zN-10!cgb>?+`n)V^SoC#sIqXbMF3R@9T)`UX0dtvC@JovWp}v}Nm4s#4 zTU%xny*TZ;vlpK61@=`^+0b6Z=;_jZ0No7dGB&b+SN4B>|^xFpYd$A3@V+J^41bR;d z8%h!aOk9e)>LYTD)zJ;0Z?z2r!-AR)7`kk*BvmYv-5Yr=rgR7qxarV6z%KzBzyZ*3 zD&1qGqZ_Tsu3kkrAJSWf&dd-XlTZl(4d~P@_7TGWgew)Pmf^+Kyc%{-BQcedZ@s-8 z5(;Tf8Bkxe9U;h1-u&nbOp}xGn;FU}7}&e;jucgriTOwz71B-WgA-Ycjb`Jrgyg7l zZv?baAP5?Z42@DXD0Mozz)kL+ECs{@I#O|@K*AP|TOt<)-7{OaH8bWK4fK#)DG05T zL!EEAZUV;nEw*+dALU9s4%Ob@0MmumXI53$s^E)70bJhA_B-56R|5oqMSL$@BFM6} z@Qb3g-k@@!Z27ADmQAkU^vyAZvRvrF=QshYKK}eY;lN+Leuhc=yHCQ)=Lh)=ABTtJ z8$H`ncMKO#K19xTcd;@_M`)bZapxb}C~nZnUR$qrz}8f`p4wxeTBuwM5>z=jiD*Iz6z?U%urA1|7)eTnx13k&_{xzd z(0E_l3zf)DcU##5#P!HqOWJ9XwXk~sqg&fsX2ce_(b7_XFwjY2EWWpBHBs96EnWB2 zC^Y4~x!=i&DrI=bgp57wXml~)8I;Pu!f3$9}ws9IB@%_C1Dm4ksymahaxeU=| zQ$=KBsg3FZUxDbpK1twfVFicz0}z*bV|Wv&9gga^b7*B($~(A}C!jN8j=xw~Q=ekx zz4M+u>x4iJZBR=dpruZz0%$>yYVkfTAUg{K-*L4hoHYZ(#v_1sE8JJawA{NuZl|P z0~U7;(Lq}U6?uxbNCLARj|WhPfE(tZ z<9b%=t`g3>k-1J6PUBsQ9JwkMsZ^;rA7XjB2pC)mAc5eMR4IZ~a!bPEWDlrAqm$)2 zNZfe1M=zWHH?qG&X~{kHM#-JRMfUms5dPZ6V84C&b9nvw@a~u4g#d_u_u$_^CL<>hH{OvX0p)IinMqK`quW%D0d}b>^mWRz%;7v z0X?jT##(z=7llf`r#dUX^JOmgxZ?vY*vcK>2i9&dx;jJ@)~waLsC61eWFat`11RFb zdF)#LX$h`SB{{iM^DnM@^QNq~n?0B0h!}aZjsol* zV0Xnjy2mJ_TV}9j*WK7op@eo`6TLl*A=;owJ{vgG(=Ov7A&sE+4Os~UUJ-a{xiP5Q zZ`{eoQAecP>h`GYpdqct&SouLt7eM_ohYg*+*Y*q;Jg7%y)%5?wTI%8ma@__a3Y6S zXfRp1)vhbB33o~}%yK(g&~=N<3si(2Now0mQkO;QFUYoyVF+QlEl6JYNZO2hmwfer z3(y9aT9F_#nIu0Aqw*zG`_o0eG2Gw4{0ch^vgxXiR^&h1&)^KulHrR}XhQ?=W#E(rgkPJVfI3*?sYLN?+2Qp3o7EZ-7OU3al zJG=)=-Q@_qG<6Aroy-|@28b>>I1^hMQ0}U&P@%CdRh)efm`na4+5Sy75edx^njWr) z1>8|H9w47ARVD~NorVbBrAVU@`bmERb%W|T+PVhJAaa`>CWZIpW43%Cf@Sfyf9!0= z+GWYCMrc+61ldFDU~ItEIklbkc3S3#;kF#Gi&z{}6HJRcs4*?Y0}I&TxtC2MYtLLy ziq=TXpoO3+G8V6B3s`?^t$oB%|yk zVX1SCt4zbmK=i{A^g+MFkFn4U5%W}}icY2%LxPifH|tFyZ{tiE2Vjn-?JqCjKhy|lTmphlXP|28+t8Sh}+4x>2#!lV*?E|4uFN8XF<9^A8+ zq-qTrD8&!}lFkC9I<{<~mX{!9gSn2?#emo#sdI@Ps1`F+9HIB-P35GHmdX#1WT*)> zzWA3TRZN6Kxkq+c^}x)z*jhY7HoaO53A6LDs&0G>0qHZ(G7V$pxhfv*C6OzD4Y`q2ErvSQ zrXEyiRO)d@Z^*Ml!;+?bvAg-MDL2r#Xf?t8>Omg)p?^!4<8~b1k$p+1$NFLDO=Ei zJDwflVq2C0TSVpmS@JT@@&UKK$_~Kb5CzK70F{`1)$*_ETDDA&L!yqRFJ!G zy?a}&uF=RS8sFiV;^N>2q(lR_dAv@bt3(@hN*qHlgLav!=cxsc%RfPdPGVQJTV%4p zObAbbT<}Mt&HD^d#6hCgQ_W&5PT_V_8jUrxRP7W{M4qF+$8MI2Yg-(fn?Ng?Gujqs zN;qxa>Vv$eim=V8PnhU24Qh*yl1Ia4EHXH7KXqP<6jgv`y6$eRnV$u*cnUig!?eUH zLE;R31oo&<+?bvc)VH_VD5Z7WNh2^UFx+ZEv$@MELl~Rh2~=L;UjF%zfj;5gr!U{U z*w6f7JGk03po1Co zQS!sV_xKfL%b@|xx-PQAE;28z-z;&Wjht9 zA;5JOL%OiJdx?Cq9cM0!@)(!XJl+UbRIiyUCxLPw8}$~ty%uvY4E9+P)DM^!PwGu3 zZ*78OCpr+L6v1vg1}&_p+EaWjLshGjSD9}^QE_&f$Fg$5mj)C*K3?i;MU?C_n4OhJ zSSX{JZtfJ`g|iR7Q-o$3X&x=bC8bo7PZ%xlBaMCPYX(zm_n%_)ay{=h{0`@w4xJSU zu(NJQTN$!$2{c_o5P?FW&OVHBij7(3+u32SGEZ$sk1r_3-_v_GKxYN3mgkcys}Ig& zbLG_&B<>7tQvd=dpTuBIc_w~TOYNl`TT}HI22+N6JE_2?MInqK&_cgqCZG_Rj+U_b z2L{y}Hs_(~{8-vTacKaxH!jGGkw6aN7RlVvL3uFoikNjabfTI{LClx4#l=U6NvcPQ zQ5pz6dcl-gmTRSfc{i2L>69+%e13tUH*PX&iKrK20WIRR8;NIZT$dzfO?Sp#xlxzF zCxdmUXM^KHy&%_G|5QTK*8#kaaqHS^QXT(|?jeIbkm|!7^dMHacl5KY(SkId#zEN~ zu?(4SQWrCJgI+=bR|!W!8TAGFAmy0qSjgqE;LxY{RKwOOXpjNBGRCrOyc7K!vQuZA?-J z+T^LkwcfKP-48Wp7SW}_&k+}Stsblck) zAfMS}i!p}10XVmXc*StNs%`9E?q9u&N{dqh6pT*2Z#lpLB)ta^^8N-0*M?)zIRM#Z z{fGh-J83@cOroGipt_Rh>q2RLT9i9oGu$eWiQj{GGwZ{*#T5yAl8wnaK*0Exua8lR zwB<}jFttFXN|fl36jN*>3R(FgSUQ~D6qRb1n!{yWlt`B&?TShgz`J}3i$3db54Eb_ z-&`hUaBv3FYX}lMr&9-#$R%xyQy&$JsL&v^f+*`GxD-shwJSS!cOPw?aIeeKJe*Et z2zta0hN(HaSh^cpu#{9e&V_yh0b+?t$*Jc_#dxJiT_~I2Qd3d&0Ia4>Dj`Ft2jcg2 zl~VSmj3yw&YqTVs*wx4-oh+u*4@WH25LN98*fRHKiyADVZ15@sDDhyDrFDA)@}Zq4 z*~@L_lJi2-d#I#H=qMAa(@NOgTSNu=Ag*q8JA)olq-07TO%h_mAXU)UhP@qbD3`38 z)yE$7n?0cXdCo5-b2~!k4%(a&Y3R_gpp+eLZwF6rcuMdFS^Ht2jwVjBf`1(T`tZY` zeB$5lT4`T_E%UF+&wLVI|5?%EufGZJK7RS=)!~^R0J5oVG|n>rpYWQy@9>ZFwC~}I zM=5m!t526_0bt!?|G-klqyFrePsSomK%Mk$b6m<&*YNrLWkQypWeA1vH1@U(aLI*{Wi;@Mk-?Vzu!wn>DB;9?um zuox^{w_%aD9hp61Ua(4*rE>#9L#~07%lWFDr7KJDtExy#0eiUCgdzZxZXi=&{tp-A z*yx2yirB6J`kbZ_ug|>ZmY^xzz-Dqp=}axCv(!}F+7&QTZL802&$JlK;>YS1ubXv@0-GDPKa zz{Sq@P)uH0P?SS6%tCPkbb%>@I)q=bAH<-8m!o@%di}Ig7+XBcDR5Xcz>&k4+YdQ#;&;c)BFJFl?FaT^8i*Y z5%~51hCeQ}42&IaK+Z1Jo|7TF;gl6jt)0u(x33@l8)5ZdkQcw=G(t-Bm*3<<{Wlgh z=RVw1BWZuZ+d>J4X`sA&S@uv!h8ziK3k3oaRlb;^81@>5(z;f97v9mi3Osf|j0C^Yc>O#?<1j-->1s3`z> zG$Gx+?+VQ9Xv<@P+<&|}8W`9X>Uzni-a>D=-;gS|odJ2(EeOMVpqCsV(kjivXC8e9 zi|oYrFf)-?QAC*8dYIL46Ms@RxY^=#TA#7QEPn0yh>zf=$2`*%6vd__uzUlPd@hoZ z5Bz4hZU+brTgVump!Ql){6JxiE}*^WVRYTD0hRz9s9-B3+Z&!rJ*@0vX)GWvu8>lc z5U~(0yu!lUj^(j7)i0 zm(MjTY!ONCxTbd(&XJAejlD2P9Su^Gd7jWnsffgr;kh2?X$xc(j8iHGn2;T#t{3J> z1chYD4p#)jY#neA`z0mKbEAPuYdCs?uToQ{bSkdoFdk4%EwC&rrow^Cr&2C1_iV|u zi*Fj66B0>BS{Fh4pXCOa3HH-~D|nYL5>Fc;V<0t^24h8TAc{yx0>oCXiLb>-)!vR-e<*1dUAGeahn%%g8@b$4Tg>170d1*C)gHq)6_LK ztr7cg>@BTQm^q?t-a3^Q5@Zo8(rO->qSyP(9dBBj?5- zk^3!?l`+Gj9z=4JC*?~_*@DI*CN3U{+XECTEFLQI-P)l^CESbYIbxw>qY(+gu4W=u!fTUOS#DT+yql{^)$o`x;K4XEk(AQ=)6 z$o?{Vj4D$u*hftrSEpU$>B*amEpc}nRf}Ic@R1Om7iJlp7}ZF-OFw?4)ev`ePb3adnwh+zl4~u#;^o z5W<0#JAfrE+ncR+0;QN}Ts9AStT8u6nhgNGDH`IlnTofIip-!?;V^dwJvz`xTFy~C zNMMx=ANf*V#u$s4EE$}5Wm&0CUrSU*R`W;zuh&= zlz{%V#SzoO#0bi`;i6`%%xpxS6&|0IxfGI|=mJ%FniRYsmkL1?hO`#b{-45n` zPgS9}L1n_C!DCPZk5$DCGA6gU(yuB+powv7xRWEW^`t6nbqy-S=B7Pcg~{b{W2JD>&@|eWeQJ&)xYM%E^ z6OfkJh<_3-Ef$^;%d3&P)~l)vK1YZ55`fr__n0pLTaNQhE}Qa{mx;D#(*zgQbIHGf zQkl-9WN!=LGG|_N=u`}#jHxszK~6xM2i|kq1$*5WFyBIw-8Dp)5ozt-^fydH8oF$b9+o0VTJ- z$&dc}^^d_s^55*CuCxSX>Xl1_bo-DzS`zr5K6N`us8n>Kb0)RxB?y^(f` zu*VR0Y6X(7Sf2|p5>6<_6`e(nG%D~k9=7n!<#mpaU2H(%@hl5{A%U}2O}0Y^V`nKo zrINRwlDY-re+h_PRDZUlQF_{TFPk_CQBaS{tLD=73e?|Okf^~|Bh_0(1C#X|d;uY* z=$6%Bb6ReL8l*j(w+B4yYI?Yyog{YJFy9S5OK;R|B)FYoAmZ})HdID*f|i*S3S&Wz z96?kx1YGu}hDbMyJ&aMz@i7Mj*u_YxRXu`>&d+iJ`b(-!ey9PpG!Q8Vw7QogjVBmz z9bP6DR%2VIZ04(tKn2=MR+sMRKy7Gd`Cv7ihJy|leV+RCDHC)Jh_i|da8z~(+udQM zc-AyYf>Z5=4SK+Vz>aR*%`0dLljt(i0{stZVpTB(ex>v<| z_u0!Al9&KO^7>JJ1RSN$4Iq1X_xJcM`3L{&_aDJBGtF7LRz+aCQTd%paRnqcFH6mc z-M>}ts)rFcA>k?sl*_)ZbGnt0jjI1H%n4SA>$ct*jYq_c?NDf3IKMdtp*KT*$du)E@FwA zcAl@TScJqWxsRa38qq1iyx0GGh*l?XeUlmR*66n zroK|d+`W2obZvyxl!YB^ngnQOVvvQ6OcaMg*BmLNiwP&<1wgEGs~|Ff?SKWjyV&tr zd&-<%Mz|7UT{_f6+i=7?LD7gAy_W&t=2#E1478YHFAvvh}}H! z+t*L_X~oxJ-@E=Tbv@q$pzS)@e`479@Z}fb2NoJ6CYZ2irqm_KYHLRtNvFh$?5Axn zKXb$T*javg!w|ey0PN|o0XxA2WG-cZ?1NP*A~~y;jf9!0DBsCGRy9kXXucqcPo;f+ z^fKj^QE(g}_E5zIQsYSm&Un}EYc;+t@nIY_kxFX#v@lQ%DpRIp(^-9DEl5Bx3pz!M zpBH-hJSv}r!V|aF1?6Ej!f}*jkXind4yn@^Tox~`5>zDtCT=56K1UwXgvrlZq$Yqt zC?KI8CIy774rpeLlVe(eo6NkO(s@zVfh>(ap77I02@H*@a5GHQS5%}BAKXgMqZDcem?u(O&9A8EXN?pQRC-!Vm$~d$t>4P501W!HHaw5ZeG6BySZW$!) zj5bj`skah%fGCgHM~2Thl!2XLtrp$oLpmn)9U}DnD3R){mOyY>EIXLwX#5jU_J; z!NzzeD~z7ny8OaAx=9D7a3@NIVP&FCydH2vnhNw1?B!EZeUZbQy z79b(IY+azJnII0qwDz`G*#Lj0&XI2$j@NKoYsf z@=Rt?Gme3lr1zkTW~}dy;Z2cXfA~=bjPBfg9@)qq=XEY-5=z%=QaK&9#e@Xz+_L3_-jw~sG$Nsy(l`;Vb%C1i z!(ic{72FD`Ai!S}-6gll!JZGg6~{Y^{P+wX`)kYo5_+SK8j$?Rs+l=AhKTtJrcrCo zs0Fb;&0%4BH-;mne*j{c7`rx%`z@@2QhFGtGbTvRq%s&+&z)k#D1V|-5J=dD$}LW< zYCCc08!3o~r-!V*20x$d1++TS3FZ{4|D2P79&^Biespi`N^MgaX|IDNPjOcU569(G zAuJ)sRvV65$yTfsA>w`}*&fI%?T1G5qk#K(o8;@!SFNh}fcL{G+ky_6$A1hoP`2~C z0m?G@xYH8Tc@Plvrfj@(A3&iq9G6660A|BRh(e!jyh-MS9I27OHz>1Uhcs}P&abQ% zy)ob;7+SF^NwVawmh~LN>KyB4Bru_Yfv+TiH)_3Ma?o+MsyX16eG(p)R^9LJm zN}G`j^dyf!8Ac4eAQADi-=$$&p&~5z}!(C83Sje<et$ThYX3~;hfjUpUn&Mq*2<}B^quU|ifiEMcJ^yQPoQzBJyY|BT#g+7|arG5g1`F^JJH3m-BhX--n?(12BLiw;wWo7`_XC2hO zZ72K%M@8Zb=a`&_JV`1?t3!m@t@rNQOvF;0$d%4%)F^_G-CM%-E5+|S5f}UErTOmq&h*yxhAX%bqqEZ(mdd<%! zseOz!RMiM)IVu{|V#11=ou-4Tfu<5#i+^BEf>;Q;0C}B`J;2H1xYNc`H2HYMB`Em7 z8bhjlbjG0BCMq?|Xj!rOcungv{-V4AcKxe|g>zU(G-I}kxG%N|R=`IoHr3YsSQx5V z`eHZx$v7w-X12~H!T`-TFWL2jr%c!vWmQW2$F9j5@r1FWdRHo-h~N;S?lDmC|^*LomY=4QO2vErjq- z^$3%e;t2n-^K1j?5f{BBU9v%(O%(pqv0}-d776G_MQGA$20m6j_>chg?>(O*^tCKQ zUnJkV$V+W^Wa9;u!%aP!tB3~E(MaKtQ=uqMt0=kVaZoldn$`L7V8E6cTxv*8NX_Y; z_iaX!kG437l(A7I3fKYCZ5U=>#6Gq|Zd`c#3YCOP6XtWZpCUG?wp$PBvMRAV3_{i} z+I0^SsCrrN4~ZgccY?G#H`cygz|3GHqtD88RME-UgN89fN4;1o<48KToxYbvS7 z5NWU#>vP3I3eQ}aKA`M!{(i-}py!y{!o$K4SwgHM_lZlm8<_{HzEojqfhDPn9IO-$ zP)|iS?$BN}C$;jj(GGUW3|eTWo$(yx|FpmfQc$c89ywKR(FdB_CCr{u!F3(DQX5fp znGt^Aw7dadU^pPH zV5o0eibpr9W5yTMAaZ77j2;JQ9ZtFLHDlHQ5%L-ocBu!evC25)G)t;?=-zh43P54tX*k@f9wpUqI;gG6GXXgv#S~0GGAKw_kbwR zHMkwv93G02pd_fI#tp!pS9b9+EIVOanYsiF3==bUNTw`hs9^UwIyeIPW(H7LXH2P& zi`-Xrg2d{T+v-?q1s#TB7SORLAG=CfGU|!hd0>=V5Kf65?ik`Su{&8S3=Di?Adz{6 zc!1_-Yt1N(_G|!;P6ZUlK$o}l&pM+kjr&~@Yk$cfm;DZmOcc#F%NqY-v83IR!Z z5-0gug}XfhQ-`;1pStE95dcj>pp;gqbi?yFE{HPjmE`Hf9U```diCYPkvaWiom1C@Q_wfu;D zV221@7}5;KEcd8qfAb%L1i>5(Io!;%&VNTw9aH7_#p{RRH&%R&2q2wG(tmb3%imevwO%q5>52;mhpp{bXDPm=w?*@QaSbbV-O zX&lxXLVvwV_Y)!U2&xH~PrxmUn`%x{AsVdGMp9ilzyy2pD1;U$k?mJZ8OV+S7s_i< zL`+LX7TLzk>wXN+C%W@DcDJXoYa#oycm*(Z@Gl?8D2E5(gKR!G9vF{)gAH>B_SyPy z?!}N)+Xt7CAvy;5#{2Sf_6cG&(wH-?PkW&SCL+V;Ufk;cr4yqT~|Iu%&ic3Rgli z%X^CYj3I3^bQfiTcSXGnV-NQ5(K2SKs?yf0dK@Bp#x9GRB?6!7sHz9R)m@puRNLYl z!D_=C0R<{cUb%7u&~$d+D*kMhAQhh_jG;WUJpr_q|My^F=d15YuBs}lYTth*KZnrs zQwcpc+u7g!{mbX!M>(PlhLE^Ru9ySKsF7!H_;u4>S0#yz4@B@#*bb2VAh2OOu8Q_o zHHnn~(uXRR*vZyoxjNBK(yO6xInDp(k)AI<4BYbTv55>4UG&q>k6R>4|2 zh2FHoBy5pRTcJxqzI8Fo3_RxCsSwGg8f>vB4Y#~WXm>pWBl7vIK(+Mtfa&C&U4gMn zewV35Qm73EHazx;7=%B)fYo}91*YU;4jqHzY zi>0x=NijEBozyp_;x=$b=dC`gxnbN4SbPq>;VuA?ldE9q((bPuRX&wBXaWK#^U*!I z!WiZRUNTVSu|meQQhXj7H^jJal3BSdytwvSECCh$qEhaYP5@vI$ehz}&7xtA9n9VHeB32(%~s_~q-Duit(2`W0dx zObWj^Ku6I!<&OnsMq4+ntYWa6Q)7OXJs#pViK4vgvxqE=NP(b7rj$p^Uv7|6-R5Ig zAW+*3#vI4yP>nYb*8+IqOkvz@E zJqR}EkwP*OouapNAs$9d)jS>%FjcqKk|UkOSYk01c272LmRES@>9V@q{N#;Sgq3=< zyKTG%8WP&9XOJ)j%)ks!VPak3!(%UWP*e}4VkNI4-u0$cL9fa%7DIfFDV3e%RG5{| zCzf~5r=xN@ZGP1|=@uQ7$sg*~UC@xMQo}5MWX8?(Lp^6Xlza>=#S__qS^T9(`(sG{ z$hC+UK(`3PPDF@G_J-}W&6{Q2N42oJ$$m8wa&@aRmq)5<61|V9y^Id$L;e{c?a;Xw znu<=fH!21sokLU^Ds@Hqs=XN^+p;4ays)3j{h(i=4scjiJk1gm=!sa<`wD(VY2%E}bUDklvQ&Lh znSBcjfoKGsYU`z!qfnfXD^oC-Wr>_1ZplrV590@ouO=vs-SK@K7m2TX1BIF?S~R(dl)LRSG{>-06--| z)H~&X=Jh)&G@H1iiIPt-jO6}n+IpwtW^3thU=1Rw5s0R z5$GnV>@}5vQIl4Hvc|c%<*4~?+MU#D_MTwv)ilgdP*5S6-Uce1NDA1lTS?|6X5JK5 z#OQN|kc}rQfc1GOYFyb1WcXCm!ag>43J(KmS zJ*Dtq4x)LL=tz@5BFec&j$Qvc`&u9QjC09G#(W3{T^S~$-^>qoqe_nx z01lGn1M&iQ@9WgS@j6r$`c^0HI?pKOEOvQ+e1_^Eg-%+f{CT@M_u2Q3uZ=Bxd2{bo zq?6)4-tz7MWdj6>NU>U&8A@QMh_v+~s-v&YZPO)6TH@ie4V7QoTn8E>>)jZtL<0{+r>Y^xU z_;(fxF;Q*nk@ZUpEx!|&Y&)!Id;wreLO`aTM@5V4$)IpUM{I%$wE3*9R_|Oj>^Uj8 zjFvqQINlbLvSg+m@JqA%Y-DMo(8w)^0?SqRK>sZ0y_0gu*}@(r&Pjd+rIn2rHM=jW zY9eH&R!_8mnmTSE{^mzaNieL1bZFu9h}SAH$&{ZKh*wx3l3nx?&xH_}R25}5PidV3 zBTWU(#bEAe9+NT9DA!Rv7Tl0td(3uhJYraSEUpXH&hkTw-6bn%t3ZKb$aIo=jj0bk z)@rFZ;l%NjjjzI(lYtGM4+J<#b{J8RJ(!!RAW5L^w5+A#NQ*X$MXKX5W!d{md2fW_ zn61x;M!6}E{SJ;j-}8|FJkVu+XR7Fm)c^SMQF!_6@PwJR-9b-a0rF*hJT6Fp3CDlH zaGT~G+U%q*ln(w9%;pAy-WPQ9>eN}j4Fel>Zc=ds?Pa4$x@m`ATO>HkLuBap=<>)w#i^zN(YaG>3pK?ya$k+tM;VTWSba*uAwDK=sn;X? zvI&|5mVI@N-x_33^*PU4=Q|llg=8${uf%C{p$-5Noz6to_Y7h6k2 zwH4%YY{Q(T)$Vj4$&$VDu@}M)w#VF<+N&{0YC{S(EQ~Dy#6EjUw;f7BesKc=b}J9H zt^o?T$npU9Y{A<&c{3|1T>{o<`Z_UykaxSNoJ538nx$?Fuzsk;R13yZMF@1z?P;k( zB;9P@j->|C>eeQ9*b;z3@1qK=v(k-Hkx+2q4Ga@4S}Kk$A4}S(Di7U`V>_ur4z<}v zSmfDoSM5~>^uN!o;-D;cZl0SANx^H`q?VcO5Fv*gT>I%^I$jblL7lV+2?e6G%}n%q zwSZ!CypoCt!IE-39YC_A-vES;`j-VdH1BA#jY;n#xK;z^q8A{pz{MK|uX-gI*ebfn z%|y@?ILfEk9~3exx1e>%DVl$lag?ntTP>zBh=AAL^ULql|QL>u$CLlDmNxd&{V5X|g zLLX&`n^Y@)b)4zzAuo(UO0w(D?G<(2t^!dU!9YIdV$n&_=BzunkS5fdkoqeC$4Yjl zQV!lxsCvbCb=@osGp-65l}{2Xb{2R6m}jksrDe>)oHv=D(3h_{sZucbdU%AZ;iPWZ zoATl29n`y&o4JhGkid{bv`y$=c#)m~wMO&$uDK(Ge zQ);zr!!)ha!`?(!9-2b+U}2esV)J)^Y|4KM=u-(}CZ&1x>GjeoH8Dw+74HR5-Up88 zpk{pzi18&#$X)uNjkvY4GMf+Sli{GHCI*6VoJeq)p!7Hchm%iXvjTPsM1Lj2ip7E& z*M`dII`P_>q3BR-g>p;ku1u9c(lO7~_>>iYGX{vfb_o0?DfO-!reb74%MKrCcA?@N z78iPl<6R|0J(!eVVP(qH>1qk|6|cmaXNump%-9Cy$N9-H z-pE95;d19drDFJ<6Cj|sP7f2?L&t_S4HRbBQI)vj{v%S6Y0i)Bo**WzmhD7-va*PR z7WVjHn+mdH43KDAu}oE)psO81EQFpG&DcvqbitJI3{O{*TOb#PM&(1U+r3@glo)e7 zdr3y;`%V5DUjD3)ynOKb+rWbyi!fo>DZQvz!eiXP%#-K4LoTL&NW$YtxQ%i zm<84q8YKHb%cgW}RL|S34`wPD$U<44?!k19V27Dv3smDI2k4U?0!3huALy({Qcj>) zO`oMxTM>x?j|8Wh$mRfTD$^kVPP47I&d0UK_N_M|j$j~Y*O#;J`wHT+P~!5`8clY&1G!w>1s{qJ(HE%YD#}z~<{;HNa<2%7G7>Ci*vb zKqYy`8YEX6Jc@qQAZz@)&GFbV%r%Ou0f z0(fedaSPL|=?I()*y$ND*^cDF$sKiPn1ECP8k=#b*;iZMqRz^P8875RZ8>+#YpJZ@ zxWNpkM{K)TqbJ4e=V@IcAPfTTSxa!_qrNH)$$H%5J-5-Ro|`Cp_GZhlem={zxh=M1l|-#tY5kbi4I27w=da}=*g#CZ152RjP#bRZ$Pcj zyUm4sbq-{)3_?FQ*>8M5S+l(mAi~U8R6hkaV|GTQ!V0yt=*$zi5EP^7J!nOe=b_B` zwkivxmK2Gh5A+l%`AFr39sxW?c_t|=cN>TZn6A~LvM5DHs@Czcee%muc2b+(jc7f% zpcqDdo=`Pzp@ZXEcH_DT?$J4N!T{5XIF88F1mkfShYm4hYoV}Y%HXnn!!vR>my*wn zdKtq^#^10xiMR(J+mgfhuR&?XK++du+jZkO!H z$YyulNcgxWSqA1(HR2iap=@D?1ObIP9?*2jVG%9gOL+Iw@BTiYw+mcOp;=ZZM_ijH zV9Q2whjiDpd=n(!322J4FL--dgXxj;evEHg*A%G1=3p(Q(p-VALy8~J4i2(xh1D8r=7b!#sV{Lhc+6p9foYE3X0+a=( z^B5qcB2j8l)erkPanCrFMv9d+;gOYC+ zHG)aQ!`NkOJ8fN=2y4Yv&zl2DKy8Dj(&j#P$`Z+4!h9~@fHZ*Ab!B!2f_*F=Hwy;( zxp}-b);!*t2RxXbWD6s8w!>g8_@ym~0#ymkC(wWL^6jEvWb00YPvTIDH(}_|N2=Qi z7-5KVzDRBTM^v1F-@Va{jDuljs3`q{IF|F;r#yld0<%SRzGjy}WpkXc1Nk!-qL;ot!T)e%7?$&XM@lJ zQMuy5h|v=!>QqO%CToS}@KV|`viMh7fh$kKN$Is$kk(Xcf_^KAfRs+EgH_8ojF$*L zNa_xt0b8S2XZ2Y-DHgr*X#pK0lF2!Zz=hM?V8ewY7%8pl$~}`FuDvDA2*UXdA3^9F z`TY(MWpX~iRw33(^6s-lp)3t)p&_HP;oh=PE)#z)Uy>ep{MoVmy?o^0&3cAHA zhDdH)3zoQtMjJlh_LSs@dp=50*?g0f@{}>OxTCCd0`h&*(bN4TycG)@>5fQSlaz4^0)MZoC);0D#tZ zSd}9Kc;g_O`i_>Lm&5b60H-)FM+)8bYU1q!wM9us+*C2?FZJvO`3U>Qy!8rl(iV4R zBa#ZhEa%w*>8TDY^9sU1r`>AF!$tzkS;tPY5JncW`N7iI%Y&0cydoNz#C1d~-a@Lf zCRy-e-`F80BUByj2-}Mu{^SRq)%`I1Z~9fL4A`X3#47=Ff*GAnHYB$@=T`@JP)xoW z@^b-(NpCG&Vyt|W%nU;3Xo!*(Cc#i%{ClmQHTDW=F6Ta6SY7J?Bo)XNlcWOv38XkB z7+YkUa$!Qn1WerQf*y|gk1_32yI$yJhxA4N6u5IoTn=*KjG^e7P{IeFR8KY zc0in9elLOLd;o2{MeD|*UJKb@)V-7y?J^cR40Qe1D~ zrpzFep+_reZM(uRNE`lENJ7J*anPUIF;%UEghquAXV@M+n3*WrW$VecMTrO0Pz|1~ zi&RM{Y0fTNi3Ky>W!ToW+7J*VR@nr?GN&)-%D~j>i~V=4fA!r@ z!}lNH%>EB<>yUpBO{;$j*1`JhzxFwpdt|khgHvnT`ygXYdJYlV1^CC7JC2$+n zTZ`0OXj@ZGi>n;yctlI}O%DuS|{-G|JNR^Aycm%P~8XJSW-;08W4E-(t1^i>D zkRGM@12hEYks2p}_^Uba3;>1Ujb!UN;|j+5>I$LxL5Z}L)F$o5nIV8kW2e=n{0bz3 zJB$S06Fj<(5ZD5+Ttd=7`#BwX=2!<3;8GthZm*&X^JLsQsB6S?Mm4?;B&*#;Yh#ikvrJ&#oGL@1VEO*7MbfqOd$)@jybwEq)9ma{IU9_*bQy+)y zyDJ~sM^y-mH6PXdTQ}IgfCnlVAEMdx&odTW8aJ}7l42qrT(yGdu1kiA*Qi? z6WWI{00(41r6SgWv}D^S(6DY6LbM+UQ#(SYBq=16GAVP@FbAEssld?#=c_d48?Djv zq1>T6W&tFF-J^}Z8pB6R%xGF#R20o^6;EsOtenUm)m!7-4ZR;K&u(Bqhtrwtzy)Pk zK3dhW=>|yiIBZwBP*?ue2#i8%$qQT{Mr177#hi}p#Nl_ScxIRVSc*F~a2b#YKvirh z@)AzlOephvelbNKbp|TC(phxtQ!n}1aqxMVT+(`53h!-z=i!XOMfnw-KN0yTbw5XQ zZ#Lz_XY+5ODOsFPw??Qqt2CH2#|EVAiF_n?A= zVE&;_(lFz9v@nj{Tv;v2>xi^;1Fn(Z2$y_tl}yA!%oT0j%`&?yIJ2gm3vE^Mn@>N<)5>O9-Y&qO>nju^ug06CBwWu4W)WYDlgz>z3Pr+ zykE#oA@!Ks)IdB;ThW9uFDjL6K>@@aY87Qw2QqJ(c$2KC+ggJytEo?p0uLy}dqQ19 z*8>WJ79uuftlX1VOtoPJX2=#6zZ8^WbVPx=xsjG@EKf@C&S>|g)eQ#fGdPvA6lhZL zSwnkPx)d#P)qPwSD~>7SUMX*bXvSx7*{iCRc-rWVNJY)6wT#Lgs^oTgq};xvlABtv zFn4Njec32~MvSx$S({l(VuPf{sdRe=Ab$XE7)ix@-vs=3a3=Wh<@50RtHZnhE4+S| zgC2-Q)|k2{mGnQIv;9B|Od}CAH*}9DLM}lOUq@wmoX7F7$$yYUT7bT5dzYL>^Ea!Q zSy;q0&*I|dVz3z+*>5|AD-7dMGkd&Lj?9jmt4IS0hz}?u-CDu{u$vQ#w1TgzAOPU= zDO3Wb3iO}Z`bD!7{67QhWU{*IWRx(7pf^6Va2uH%h=j0evOs{sn?lmcKVEuDlVE6` z59QSd$N?5P3pM{$lPq?~fpZ>JebO50=%7Umd&uhq5(%BxW8lBep><%SW9wB?U zN(5QUFJB>9P}y`@_U|_V^_~hw+T?D|;U9aZZR+ia-Duv9J`a@-aO@m@)Phy@v3!~4 z1JtTbR1(-Cf_|ak9Et*^oJ1*n^0t8hD%tx9vQ3>WV(@VrFB>2<$+eD{I4AjJT|EF( zWfkjMQ)4~jMm_Pkw$Z{KF~gJ_S2f+tkJ7#g_Os>SO@d%4$Nw1kNK(|3G&{;8K-9ar zOfAuWc8MO${KlX)7J-^TZriM{lgq$fUlqGH$GG+z1N@1zu$cKw*g<| zG1>Fx#PJS#;8(NDgV5?yt6>!emSA9zDVn`RLA;O3o3)?`!HtwiNWt9NkW*Jmcpz(t z43R35Ey)$|2zNqlRtXVD+cR_0hz-z!*(3CQS%PGY^H}I>XpaDWZE0o%l$H0jMo-TW)q>C3S`vNHLv`lZ z3^*Pw1tx%wX<4jguH8|SvfeG<9_zZDl#83=AF6@qhi<|!gbQI!8}P5k7LrWup3&LM z=e98zg1oxOc2SLtmtcCJfwaK}6kT?P++IDNgHIIEx$F?z^|aeCb!|pZ58DP68CORK zP`0Fu%pE!803R5%d5fy+vr*3hlbt3KL-u42CgOh1! z9p$q3JZigyB$A*BP<@}a*P@Ld5693O!58@F;@IFBcaq}Vr|l38cIysAXW8^DA61qj z`~rPk7(vssp%1*P9B09Z2cU%%(Y2QU)RaT*vULissN%)5x*2jG9%SW4Uai2xUz6p~ zz-nDB3drfjc*}-(@~YsW01pzM+i`T0L*~m@VF$JP-OHy|&;Bud|AFtH|EqEHSN8SO z{N10w{&RTwIilb<17hcNbqx%J)tUvY%9kErL7MuyrPN{iBEx zWOzgtos+FW0+}i|LNAmN>(Sj*+ZwGUX7LI@4szJ7Y`qb*CXF zHYD$&YwR7;E_ z2aqr#mOX1`p?ohA?vo%J;8>-q_F^K0H;dPqnEA0SNRg6 zIjNo@jONr`*!5WPy0tp`X;E*oZ+PlPc^&AXTMJT|I}G7TrBwkF?hlv;lAi?|pY_np za2+Jn6OQ%}smEVU{0 ze;R=Ai`2l4wmlF=-iCBA+LC9d3dp6E@(j_1GsE6RK8N#r3L;VV3KYeJMz|8o-{jMO zk-P0vHv@C+-S~b6H`0$GnpeRE89fgPXAgy5<-m1@e!qmAqQwWF=hu3)Z zB5i$~vZb_}DI|ECxETRzQ*=y~H%!NZ4V4lxqpgaiIi{%GI zlT-up4*U%)1q&pp7y6!en+YxvFvduSyjaT?1zSctm86a{#m4#PFdw9>gORu8=lCM8G0EWq=E&R$^T2qYlc5T!&B&WvFmd zA}oPP1c9O1X9BZkk;UO^9Nv@fmFHTwUAw54VhkPM;+)bqI(2-QeGJ?cju<8xr^rDR zlFW5Pwfooc%Kx`XyAmCGcP*i>n zXV_)Yc(CAhy%sy~Q_V@^DX!YIPs%KsM@g=_7Va4K6v(wslyXi|By21h(luNq3rncf ziWpSVmJ@(?q!2>Wz=2ngP%5}9*-QQM*AEpGpB(`&b>j13U6J~{0n{S#5a?=0tV#-C zcM*V)s$so30=Q~1_>W7=IEdh~t)mvF^wNoH8k-L*YE8NwyK!cA@Pj@208oTKz5B1P zzm~6mI6UPKsZM@=vL{ADY??~ItYK*(h52FI98<0PDNsl$p*6^GcQSk zQ_KFoGUSgKkF(*Xa~nHOQ-C5u4Ae9Maqs`9>rIxV$F4N7y+6e*N@mrqA$wnxRrQOg z%`OH9;Ew6X;2wyTs@U4dw9}@_w5dumIVPD!CdcGt78&_oI^VhHI~RX;D`|`XJmQZC z2XN0l!#B)6cAc=v{1D+N$z_b#uH1S9lSfM89cku!86Kc#q>DO$O}lv_l~@GCo8D`k z4l*4`y|<#{U~(zpdc7ke)w=@FY$$*d$nzP1|I9B(V~l*fDy>i*PvOrJSBWvvE^s$q zch5dSX~ZM0O7(0AgV<wq#L^HOA)3UpsAF_0~_uhiDHa-q)B5%20k*kxPwq(E-}GP(*CBy{MaKHcQG zM$;*B>zWQN!+grF#gLU@#fmsc#k;tjD83^SY^+^7+tdcEz#hwxG}=rF8mOS~JAn_O zz~YWePzpw${h0dQZfUpj&z*>ddGBd|I|-6krlr zKtny1WVL0&3x>as?k3MavLQ>Y(@qQ?zIb>XTE3)$=hy; zPM5ONfJGKM=frl08K@>Y|6&BJMyq}rwMS8(e-hiCN#5FxNSldI{C2^NzqT`$qToD zyOq5IAV+WUQpmNma8$#CI57_+^nV>nSJXN%O~*9o1IkbycH1`;KZSv0WO$Ot{qmo} z*Z=hPSq@EqeE(tikLO7B1xv-Br&oSO-082r`N#ZyzXU?o*g`Am?1r(Xe4qV*?OM=Y7|@^@`p)DIncvF?&Bn({l(L=nA<~0X&iyJT;ZV3dw}#T zvWj0f6%|5g%ofz>yjmx0;?iBaK9Q=3#DKPF?d|sHkrT4IY#mANCRLO?UBkeL>58ez zj|k&2J8ace_`yQ)!`^ulAEd;!lOo@wmh44W8FGJQ2af>{mTlOP$J=q(4k;QH0U|X* z3ZQ7Whl%<=sUu^@S0Cw&(<5PmF5|UV>#g9P*xJ2{*)y+ueO1lQisumDR;0ase+}0A z^c?*Z?*;1XQXy_r!L~e-TO?0)pgJks%?cV&E@ zT;3&5k*BwD<|AX`z4$ zP;1oKMb#aNglGeBQgQ_wW0ex{Qlr!+j0JV^mH@4?@vGHH%@%8=tuokf_Xj|#Pk^zR zRiPg2F0#JJ`+2D8bBaOeg;L)Jt;!y>l%6ObG}0|2D{8k1l)wBg!Ot?SRm+1c{fl=3 z)LUr2j&%od$g>&30UHgz8Y(rilu^;$() z0#m2CQ}=O~S~>A6SK#jNge^=nMr__8AJ~5YhK34AAgzx?a&3Vn& zQgEU7x1kW*d*!m&x`Lq2;fjK4PHuaFDJRE_x_9LbjDux4r2Lah z$I*2MTMq+M0it^bi6SMAA0)Yf7AgGJR-+1PPui23TD7OQEsx(ISp&U-Lwh9_=+!_5 zKEx+cSum*Ws~tq_HYzflk)LCdBRZBdMMo)t8)I!3=zwbdnq8rcOXJN-L2hm&<*Wrq+bGRWh5Nz-wE zMj*6Xx;A{WbdkrMNfMrZ3ms>DJcNjyA_48Z=K^u8Z4YofRZ`E0kx!IYJHw6@p}=wGDD&vli5u8e=yXb! ze^G}{OkYN-H@K1zte_f`veNb3_4>I#+h0+_dOb!sbLdWV8yU>VfWJ?X8RYH_^-b1{vE%K)- zr&0xHa7S<Wimp*Htao#lfk4xFY|OIcZzN0+h~S2oc3JBRq)?0A{S;<#QF#@zN38XiH7@kVkOI z-h>Vy`lOIt47SD0ZIC}wYyk9IqwFjvb^!!rTDX+YaDU-bq{qCM#~5iUMAH6DEi;kXUjEnc59e_CS@_Qua?ndK z{i?8zr9~Ma@wkBU0+=IKzm~nOZxCSdYcN1zn|puRl&yA`h`F+ z!-!lLxif3Ls_S`btV?<=59{bMy~{2;JPb$+k2EwtURt#Pcg*{Q*mt9^2(SRPLPpH0 zuT_+Ltq>;KX--?VD`3a_Q8$uvOGz<0++@Hk);dfI$5A!RAl+blhy1Uwe zY@3?aF)x5mEp?Omh3NU~M}1cOQIQB}LM5OPv4>xR!8&idwi8*b)tEDs9~rh-TPq^- zBZ{Jgno^JKY=i%w1v7PJOvk_wIQbAIFjPT@i5Xde7OMrMO(F7J;MlCB1a?TW+S)(^ zxME2kRX_lgogp0dO_Ft>=b0576mQ3nS5Zzl`Iv#od{L>bAxEp6tNl5gGr5nx zqCe@c&Cg0jhgZ8}rT}+`3l-Zguv+L(S=4GvaK{weI(nPph#c>LhHkW3CJI2e9c{r= z?`pY?2zBsSTG4nt6Jyq0g>vhfOzC3*Z%4^xA3FxGT`sXPgFS$3mw@HP^V;eJ5lW^c z>B#oyJgOBc>7xo8vk?jUVLJ(g$URSV{0)keR#xsAL8T z>RfGRti7d@gVJf4Fn5+KZp|b$3aFS@mCseJ4H|^1k~%L zL9buMaMH^)9092}2*+Tl6F6vq3n~r!N=t^Ws5-$<>Mk|t7|=7Ith-XNk&IPx6$gc5 zc_j*qOkgv}$0?U?ojGp=J92RiGqo{|o+4|C=yYr~`EjM=vI9gm9pa5~N~CVen;Xjn z-HX!+(kxn;N^HpD5E9~j3{licWg(pe_uGn_vp14^WyRxrM3yE$?G+hC)iFtgMaKYb zar;iWeNzr@$}>Wv{uk#^2m|%sq4Et@8%t&`PnUChdJSZE4%C5oXO*G#Xm*0op*(?M zZEl9eCF5sNeKs~Te?S6gRnxvZ1{~W23ua5~rZKr=wRVv)SUC>3T%kqa=FIY^O2Vm{ zSvobipe+NjK)9GmiYK3t%A=E3Cap4yqeGa91e4V0#2;M2Dx%b=y6{9rN1NH9JLi=u$(Bph13@unE^E)D)*_0snJ55Go=4|;#RS6O>fy4m=?m~ zGMpO!P%-i4Aq81Ek=hLVrbveBC!d7>J%zbX-+%D-x#=)JRTM5aT``snH)xBv(5bQl z`>tj!i}{LWY`cSO!OoVqc&8~1jAelNkRbG;-U9=i=@BIST(O_HE?SPz~&G!y#^4unLc^u|-Q;9l9 zzSl@k}9g=~B~WhdRlt3cnN11IFLH8BJRb(yQuL0kZWUDEaq8g`Z+H@Pqfi zU^MW9xAddrL22S98JAx8zP2LoB`A6+1q%p`LhbD8ilgf@*q{uOIET9JOxKh0C(w^* zLiz$v`hYu)nJ`G)GFxN4c`Cbhk7V&m2)o=^cDAw5P5To}wd>Wv1fv%wh*%DGo|Aaw ztP@;Y{pb{1?-=iC>eFK}s@ykc{%mc@+nBB>mJc?mG!b&19m=YP&~Si~ik)+>eurlY zTHUiLU*V9DaEDkg*?JT)`%+^7FeGRn_ zFB1rms~6kDqBadXC&qt6+V4;8^zZ*ZFX%sqx1anss(bHWeC+M;<@=BLIq<;03TgF^ z=*Z_67!m&6RoVab?YC6kl6Vl)C=?or(^WGH8=1TW6V!Mn_mO_BvwGjWFPEniy)iOF z71&&)IxL5BqDTIdXr3J%uids1!SHYY+6}oWN z8%anKq)wLhJkTt4ZXu{77i>V9O8mkBAH-G;WNkFv1wg>*B8e^P)9NzaAzo>@_J|2Z zpBS7r+rZ34DZmn&TwMmwx+G#2-3B=4A+x2<`dK$U6=3dAszW^DI6KMNw!?y!DqrRP zPsbDp;^$DrW8!#2o3b8BJ-S6nHWxOUA+KvE*{wz^~U|+g;kDLenjI z@}(fR2aC$+1IfmN4PW67%|ba|bqhXx__sjxYVwn*mzcO<=2LIUJ8*LpII@_j?5Erl z>e$q~?kb?RCY`N3Xn^%Vfc^waQw^AjGoOMY+O*VRm%Pdz^Ln0K#kDRC8mwI;iAAiH z5k|G=dTO~M8yLrtlIZa%>2X!DzEVN-RhC}~N!Hjp_jrV&_kysP8$FYvOeZH3YZfEl z8a-gD6NJRNJ1`O+>m==Y)*bW}J5nj)0kt=G8Q#mgmIAxW&QQfV7S)M>^bhu5kpJro ze;59X+eHLBu(ai8Z`nHH{|fJaNf?ISGR!CDiLxFGu&Pr?0oKVvilq1!PdbP_4>tRx z{&1^`O@@KW;$f@YI{b#mN+h`y3txZln|~zA36(x03h{fYg-hj7NKH(mv4A<*!_C;} z%Lnv1#B$lhfDoJfO18J-s%*CxC}zn2}M6p+yg>*c0aY6nE9H)6HD4?IRTLv7`G3$dV$%&2$| zi_If8`uhxq%q)_l%@}9r)$tbVWz5US-wZR!IVIO5iE$$?hy=|1|rpA7QVsI zit@h?gL9}piqtYaFlgzO-u)-wo#O(g$^Xi*4>cr0p;yNgOE!Y)QtfzrH%M0X!A!DH z-&=&xRAtpjVe-Og_Fl?^oh)?oOiruZy}>M-Ks&DR1)KuKoy&;HPpgt{#inNT$v(k* zDbo>-gU32SfS)kDHTsj8U=v)5r6&#>!g?zWnB!}h*e6bdExS|L=?Ws=qU!%iCLI^W zAJ~f$GY`t`sfs7bdqWl(0*+vXovl?>Qm?fPNnu4w)-!kf`crf7`?v7^1+t)2mmkpw zxA45Tz0VAcr>lWi4NP8F7*prTaEedsgr-hS>Yl_J3s#R4%>$c0y)Kn+Zr5d7vxzWf zc#Q4MESlhv6oK`>3CK1QKgg*M33VMxAnI{0vqO1TI8^;zJ=K;upJq>l*!dz zZjz2z+(Ms79Z)$21lhXE^_|4lt9>WD0f0C#?yHO+ODHj1w2bIRCX#(ry;OqV0~nh(?G432_&$KDsk-gor6}Z>2T`H z7;d#^O3O0_6Umh4&(ox2lgvs`-u=KjkSrJAiD;n(ao9Ya!>mCa)>V9%_BARnj2#X~ zft_(xyIeiZf*|%pG`C!;SoyK{OK%hOCI;mh_wiv2n!)}6_`Ta6-O8%2D7n(Yku7M) z!_G0hVtD)mUWgfX#(Yc9~#t}5p>?4MhmfP=%GG9 z>Y04yP5}fx(kn*JVtP8YxXa$8K#g-jqa*RczVv}QaBJl;v!_=AADvc|U1Pma{VjN# zR&Zk|)~hkf3(wVH*1lWyYtD*)E}>dLs2i>sMW1s%fJq0SwH(@`S=Ji8PAm{*qdQ*5 zLK1OOWkH)UZJw`x@&2no358FkHvH7I+AU@ckfX4%!4rUrdWB(MvYdXBru}TYEKBzU z-PkBHp)V|0ssEvk$|fqPVH;HKe)|GQ)$XLu?FSbd^3%+)(6msE_v{^tl0a3boVub4 zFs}0qz!V?_2sL+BYZ$=Of=l(O`3S9Z<(S^}UY|^A*L#r7iGgGRz`a&cQHa+1Xcfo; z=73RZl$09aG3kjNgX_ac>5)Yd*0x7s*eu13&e1TUbw5cY8+!vsctf&xGoH|;h65LB z#(Ib6vW^NVbg%#+U(=Ae#KygK_EZm2O%%7wfIg%|s$G4WQdl`;Us)9ykSv9*(Cq83 zK+b1#Cqx~|C7G0AQ@7shUGXaxWS|z34r{5mmI~^19P-C?P#<_uV^csue?A?Js)LxR zAZYV5MN^dyvSOq#b)&A9QVDJ7*};k>q)L0^4mN0zb^xSToi?kaN`e7wntt&SxPng5 z?pw)bvNE?u)N4BXG;q|kd}I`S)i@5KvZL80_S&X*1(s-tsY=ToooWHJVkPqGhK)q@-&TzIvvDwH zVihSDP&;0cpkZphpOUj-qVAE>`f}P*R44!10@k{gGbzm`9WEhbr$R|tsR0mqkf@OC zKiLhA4Rrg{VFe>cbTl1D8W*YWR~C)mX3TuS827`sPibKaT>JNx1M{Bx-LJA;UzC~T z74uX39;^z%k5xqps)B6$OOR)X0MGy&xTNr}%1z!8oEmkP>Q5ah;e1Y)zD0)#TS~*8hcY{8sNokCjczBO&;4MS8)pVX6sN(BFyL-+~BcZP>YVmq&!8maYl zqjW4ugw_ZKTdSeno}X@1%|T|Rumr^{z%z^XVaPGe-pf1c!Nx)ph;f#zKtoh6QNv(! zF#Aj+FV6?A0Q$ty&0sM6U=SHxwBj0j+~sv~flc`tA1*c8lCZBVfv;nXupWt~E*>J3 zd9HJ;#C-uj8f+Ge81PI}WFy-*n)b3v0Yl(Yna>?#2?={gZlDkt42 zfv>Le3q;AmZ;=DCWcItYPz3i@cLza@=m>^{uLKRIaZWT@Y~g=Mr$2lDd3gWpN#r+t z^M9nZ{Oa8HS18h3P4ogG5&7lNtw|T@G3oTtMB&tBruJF~MHX7umo0UsZM|Z=9x`*b zf~=Kmx`3>!Fxs-GA|<-ID1v-V&!l;+wYO4L=46g@R^A0IJRM?1FHcK}WKjafAb&8k zhQJZqtPTV&(bh@X-cG%z1Zo0o$eirlqh@6M4ycE?`p5Lj8dEZaS4&Bgj9j4e#kk8> znT~X@d>Qt8-nW6bI+^7RbpuknizLPQL`+HSAX(Qh7;_};fSkbe`c;)VmK9uGZ9t7} zTOQtmLw*NE;z4?N>nF->r3hlXL)f1d_ju+iceZ<#`)ETE{~hN&VCiJIyE%q5Yh@y)dTOvH9P8xLxdWvN;9zKj($-)^xvq|4V-L7jIt~ zn*)v>kg~M*bEwBjIt=mkC+_vm?gkjM*Ng& z69AJ(mkz6>?E}xt^mQe_SEzUZ+&7JIc&uOcuy9bY!Ryk8ftwD`jV#g)GfYDp@6bR7 zp3F6QHtl|FO0q8Dz1b%GWgoK@UhVvvR2M;1tI3{sP%ulD=b1KxUKE~4#T?$_9hDE| zqM=oS45Kb1$C;~6@x09fw}q495%Lq}cxz3Op|ya2u`C

brXZX@VYhc!=SFxwhp z35L9L)9~j~4N&#DdIB@B$p?;5#`+0waAk#qv}U0S9KqM*qre1sP(IG#r1_ynOMv9G zQpHjXDzn4dA{|PMnMQz^P2vO(iyi?HZCbYzM&_2v+yT%sBw&gO7>0CXNkNktaHbU7 zP+Xvg7gqvrTJUe5wmuWsCQ<#2)H_y>)iGU|fMtAAyI1~XwUC53jlyIFzh1O8ZPglV zhgIE#J~QzFUB@|6l27GL434ICM z)?xP`e<3Gj$2wirrgnPPH32~M{g7I^KMD9tq}-55E!w`E+b$~%%nk!H5O$q|9|R{s z4^N^6=_QEYNYc15>_VoKP7#PdEye&$)O6Wb4sTc^a83?8vIe0I4RP_OKz8yJh^tJK zJ465iAs28PdD7gX8YShZT12==VhCn^b@Y?jLSIY3%c#w$V>O2<6~rWAOJgqzbnJ70 z|IE+@e+l1y;qvv5-v09T=VT8k$G>_1)z`3m=Zx>u_pjc*?RfgHZ@&s}zq!0Z!EWM| zRlDwZdYPj>)w3sqrJuf8z(uiB8nHVTY*o*J_}h3T;dPn?$$b; zMFJ(iCbjDo!^TqWLGu7Y^0!qY=cr9NqNQ{sWyP`LzA$V+f;a;PGSq=ar43? zJpoKm#Yzj+?Ib?G$~SE!EP?hZzhz(8jbHY=dQc5g=~FaV{=nJ3s<>c^c&gA{LlmuP zKUoSWWi1>jD}BrcNfBqzO1z$eAo(yYbL;%Q84aAX2Qq~EYO`IEz;RM=sh<)l8Gyor z;+#?q#fA(r*@)N6@b&lJeh?On`Fcr_ja_^MQe-j30%MkvcrXV|hb1aNHu7DK0dU`EX74AqQ2U zktOd-c_$RJgWX0Zx}7@rg^r9YUM*4olQSfx7qixW4wii@ zmRj;yNTy&?7!yxwW3++B+8ppR&UTf-9oGjtqQk1i-Jp}sMAcCw*i0U#6ngVzZdlyz36C`KLi zF_ru^ouvXYwcT(M7PiXzz!BS6!M@KimQ$`jk;%9hcM|RmcpBM2mzSg{=yG~i`FMSQrWG5s}Tcr|2<9heUP3*lz8PZhwZA=pc&XEX}^BFjWQuWiI+iVJM6&D?xl*&ok!S(+j|Sl|ol9t*Ce5D~_R_HBttoc`)^blm zQR!>RWIR`hEg)qCT$GS2?=wh1u6XIiA6mN^K`=vY@1nEMl4CzGQZJC$4GF_&@lejG zWQVQ)Qb^K?#FT?IBxX&}mSlQikqr;B<93FtV_yLrC}z73XsftzORpwt)PP3i_RT+i z{p0tap=SH+{YT8crqEiJa6pjOE;CRXA*n?r&)90rMvM{>RTspWQV@?adCg7Ja1cUO zWK(u3zmc1AS}kKjT0~IFrs-u)fi}$eovEhF`J9KZFsITj8*(gF-;VD0$#B$z)2>Knr^kfI`1(L^H>CbtBdVVG*il z4+$k*x9yg96fG_>BiAe}&I@%kfe|8KPM#hD+V@bK>fmiEkOF%o zyvf(7!S^&gCz@Obnss}DRa^9w&Zbi%88$a~m$r^Ou&BH7y1LuK|`Pe2OLAZ%0on! zrBpWox`dZn(WNQoVOO&>tPSPNz$k$=wrHX?Io?6*D^GwBiJ^heZ9aDZ)h#lBsR#i( z+X`PuiFWanO*s;YF4vUqnXp{m!OG=bgPXBb$gez^sm^&Qr>-ZyMuET9uaY!g!UsOl zE_F~JFU|=8w*sQnPI8k0n(9(wOZo=Y=JZkl;SXX**RmK@ZRZ-tD`D8?3iuNj73QM}nZ*58e6 zd}C+Km^vxYSUAd9N0s+Q%wtPzhZ9wfP>r)qt`NPZxDBQRuLRg5K*C6mE+#dNeWbQ@ z@j~OwIP7XydhzB=LVX?;m|{Ul)rRHT-lxUqq3RFm@Tz3C!R%`YTjXO?>@X{0Jep{S zTI8x;V}iAn(AN&{ze>lSzW+YF|M~=OfF~Y4CC`M{ZdP+(WfQl7H@DENBL2)V- z;zuVdSU57dO57zLo88K4{aB#9IX+lB2n3gSa3o#2gO19ARB{+ub+WDxhlj06h|7)x z^>7yjCR3VsL7j@(+qQ3R`GTE~P7cs?$A@N${DVJ)R?Vsu%aa!Ov}(bN10d)hY(Gtp z?vTxL&D#xxUV#tlRv-`4OKS-aMc9N**0NEN@TRqB;Qjw`>ejQV@p$tFdW}YEUC&0B z8H*n3RS>gNG`2dQ%7J{nC?M+L z3%!_%A{@Q-vRyR~`bf6+k=VPk93IKZ=6Yb+fr1iWvSGVKZ=$M{td56%aHoTb^y3P%S9oSjhz?KL%^6tEf~Uk4yF791az5P`Ha# zcan4g70pv1^A>*-(<{t^?u=h@eVj0Xq=Q8(rr1zz&{jIAT)RauxdWvVisk?Ohy5X2 zU+v{~KkZ%Gt0=$uP^%11vd1QNZ`E$LFl6sLs( z{EI~!i3M44y+H%>uyzz^9cFcec9hXFh|;QTF4V+aUSG2Y;59z)nfxc0$&_5La?{2n zmvpJ+-@(=15ep@Yn+o+FtD>sYnNyZ)Lzk(}A}P=?zf$ZMtdq`z7NGY9AdVKdYUps6 zNiz|x2%4Lf@hQ=$l1p4F^^AkrQk%6qnZNLy=!m2(jx)I1EX%@Fz#GlJ12!QR?%-&$ zGwZ329KliZ4kqae#CRR7`&g(F7clliw=Ax(loZvV2!R~eBH-%cj^>`7jz{ka$Np>{ zAF;5(5g&=SBm#SH}&8Z81^6ph$NmHv;xiQgn=6}UF#3WFV7v^|y zqM@or!t2`JE2HHVojT-4libZODwq|*oumq2Jm#2qMotCHkdd4c1J?7?O&wX6SXm%70QzN+P(tep~5 z7CwKi&L>Ne;Ur8l*maQLXJ*40_4-ZexaP>FEF{H`?|T670W0r22y`9F95=&#!oEe8n?B9n zeOPo?XMF={|IHOjFw5H3L=N-LE}~k`EM8gt-hKe13}Unxw31=8A`zuM58l&wdfg;7$kGhTQJU0VP9DrUavp*|8#CiP`WmA0@3w={2ThH~#IzwU zI(b}}HvqS=cI;_L=;w%k4qo9XR}uR4R=|XM;1l91->yG5Mr)kI36zZv-TYalph;q- z$RyEFq6hw@N)?sT%epk^`)%O^+md=HXq2ce5TKVejp@lu%O6x}TS0e3DS{}$J==l9 zv^x&fCj&mF;VGG{voBHY0LV3wYNb+-z6NM#dGR2`mH@C;6so(YCfdUKOfVHffR=HBuBLpnWs~3V=P!e}yX^KVP{Nu@L>HUjf-5ElX z5|{Ste|`HhnAhdk|MLEMkPn7iL~YjhhFCW;%0XLTw526|e8Lh@olc+|GFerzuYrpS zA~_XbG{!En?v7Fboy{Jw;tRT%FY?VLSzmA}DA{h<+9l*hZ?QKCVq`6bt#*TS*=!s9 zUrn*y!f9;ot+#IAz+n}ej>^2>RG!2NpbOJ&dCjd6k2S4FR>|cpK?+yvRmvh@TgN3) zs52h?&0iS?4k?3e>=2< zYboFg(4z=$0-_68xC{$|>iijVDc9%7WgM<7l&vk$A{X#6a#fdrm{b(V1zNV^@y#A0 z zoLQ?un&^7HtZH+|S)S^%u)U)Ut=f$xcPa-;w1Jm`(GTYN1oS`~Ij{wzB?XV=%Bm-1 zI+Y5N&m{hNhg>?bdKQmXmT3b*N*{rP&@zcaYkwIfC`1nIRmd4qWQ0foa0J-$WkeyI zeaUkVb%Y~<-1G#~t{;krl!$WC!ZN_J%fb&Hc&KStAvulM%LpOX{W&Dd@tHc*ZzY!* z=9UHp5bapx-njiuAMO^4ByHzfx{uRSTbE0Ql|f`5>~`y3ODF^LN(##`xdOqmz{*S# zUAmD+fG1w!D01XO|99faNpRtl}9f~DF~s&uWrqWc!Q-12 z1k)(RfDU0gE}3D8_WSogu4({@bPrpq2CPpI~FJ0q`jZvG(V2&}=n8qOFGiCd!%$iDvGH{Y|L5*0VITdB7} zYU3ad>8QpUvkeJd4OLFjK+lTY&xnuuL;PpR-!r2MV|!^TyLt-{&6H&s9NG!~=d_WR zBAE*7S=|5!d?wd&g$)W-mBQ6j9~Na=5~T@0*vO!@NFfy;oWa#3(@LoC6G#bfibHl~ zCMqazB4Zu%M=2C~x%eIipYYaNEfbcgh+UHYoeB_HJk}v#x%S7{hINpF^hTN9MjcmJ zkCHYwbd|X}&{ai+FtctE*o4s?2VU3;RYpqXeJe;SK<_|y%Bh6u0FY(2M1~k|p{pc9 z4lvLvuv6K?)u$?1Dpd)teMUVi<8G;+q-Nch*F)Z_Uk4afJz4Rpjk;7|fS5;>j-+Ty zRq4g8FE{)an}W!3c*)u=1)E1|GHRfuV^$$yMgire-f6KS_qDg21&l%4ni{QMqx*=$ z_b*fW8f_Y;L+_BlE!a)@RBUE6(5j*Y>PkKNY=&>)Hj|v8prsIR$m%I+i{%{BEY`r9 z!8?cxZ8HMb&~TTvt$Hhln^eK3-Sa4I(BSu9P3_4jc98N&=U_R&SBkV4MXGM(O{3** z2=%DCan^E0?(dy$%*9BdM!vr=h9)nSzK+fA_#|2QBSU32UvarBt&Hp<9FIlaM{Jup zV3BNiOw?l|Q_08FZw`r;B^f&C&8@zSx4ltaOkyq7Ecj%Bf_4r~q(IJNliZFtsOW`Yp63G~TM_LORA{ z6L)XChshp<@tCWi2&B?;j&aXL&nSyN(MTlp|jc&DO@+C#f=GF z+_c}e^<)q#3bP^w4+yQ3IL82;kvki1I;y(Bk7IPG-B7AcuG$v($Q&7=lwbO}a#ZaQ zrCtdn!-SP+%ZJD8UX6%r2iD7Zpda6&PFGg2K1kof=CwrakQ%MKGS z@>#CfJ59U}cBTwX7;;hXvrtjZWU4?#;NEpk$!cEw(UnRDJoc%b7~LS*v~_`jPYO(U z>hvBO;Pba3>N&Bpig$t`pq?;RtXe~BWaLG)0=@OOqdB(_<%yn|8baPFa#@0d?(%og zvCTEAYCfv{l=~%AXTkJ|_~Sx5pEf8Ah*~MK4NwG4hloOY^f%!@Quq9|nnt(X09+36ebVS+fo z`pTxV6rJ*}V4O*E4EPif9U#AjATOQvK_WAq?K;3ZDb3+wl23AvtzoBVrMRd4Z@Qwa zlVu~}NFuZZDBE~^Bp@|5XAmRysB|PNGKGh=k}9z@#(@q67==>2@t{;TQe;vrxv#Gq z#(Ik!$um>$S6k~d!saq@{Cbmzgx1fu2!uBi7##)pA4#l6H{QeSph(mq?WLZB}WH) zMpOdAX09kZx`(nAxvDsF$Zfi)G`hoo5faou-hx5i0*ht$c306g4KTz=5F>qP3K-^K zxpy1r!3gr|uOw-xa-P$=-yEj{wI@l8VI~^8JN@N80#rmWQkiAR4i0}wYEW&RruJ~2 zGOCu!Dr8TFLaFNO<-deaK1s~J&)&WaJm6>j=A^f2wdX&-{Rl$<`TV7P{;7QaBG`*& zhkE28Wq;(*J5T{q-2#t2;;=pj;9fT!02}UZrtH;0~WWC63>o4mEb9IvfQq`2$ z`>$X0h7>L$14zfH(XPd)+;LTydxG#M1>~kG@djF;Wt|K5)?wazuj3R&^5O-i{J9I0 zH*LIuxtM|+)InvPk5!(ND#kCJ)E^UQbs!0f9)v87$%0iik?=NxC6S!O>>T}JJA{$% zsgp2*$!g`}zUn00Lfk$&XmyZ8`pJ}b0OyC3QV4BZR#nnhLP|~8iS@4YsdlpRPid2&-@a&grvNgiSernjJ`6DdMICZI#^sp3j6 z&E2|XCGi^1+Zd_fXKH-utjDuvF%kMhbqRrXsc8py!|{f0C4Xeqd6uV6C7E;J|=1LJrYrm7vF05bn1RwubkU1X1UN+E+@ z+4|6>fYi{!^$~Xr!&OQooBJrXjDk_n_Jnu~YVau+muWgI5iq%sr`I400zGA0wtpM` zXZ!p=>gT^9h36yxoxgDA@x}SoAM<;E{O!;AyMFWjr}L=!%jEj}-|>U~GQ538PS77d za)$nY-+zHk#=iGo-oKyEjzm3`MR$3D)yHgG4mc|#y*i#T&l;fQ3`ExsF{nGF!AHAk z`-m{I!tSABgA_vXu(7o%b$IO9voHe2UR4_fgO&btl}G)Bwmxll($bcjGXV@SuqE1O zP9;)y4kyh#4D^2XLW{MlI<^xJ;HiyMGgP@Dk*zi&oL7h&~;*) zLAI-uK`qa4OCfauxL#EbBt5?e6?gG{$+BXAI3vaV;Bv%bV3ESvR?PQP!BRN}Y2r|tpIfOII;v{{v?`xiTu z-C0Eo4z0Fl-Q5ZW!owUNwv(mP+WZ{qZK8e95gK}aHbp4Z-!z{#R;t#MVtL4!Mjy8BFUQ$`fBQk;L4NfoZ-1rxQ26!>04shfaoZ+RpT7U${dYRD zdiyKRzK?&RpY-dmhoN3xlrkevvt)uL(7fiKhP;Jk{MS+EFjE$N=j51nb&P2%pp+83v<_MqsE- zxG44>q8EiIGC)W8;Yjl95*=Dxl*s5{TED7dhhdy27!}!f4HGm{XuZ^`j3ptJl4d<( zb6F%f-jE_WRbCC3gByBD8gI11*=kxpp%TdSOk)OLS@Sfz!0v>~1wYM}w*+W4Xrk=^ z^x7qT7GvqAG#z1J;r2@M9@nzNkew#e1JwEH1v5-k)f}W4eU)8BlJ1<14k~w- z`4;e*9ejQ#@ltg(A#jR0m48=sBd7Zgf~76 z0}S@MP1PrWE)y9h3mBn+oW3r5i5k2~!FHEB6ys1aC3o!5Y3jaouhSS#@3uw5fvw6G zI5mK@<&DwdDUwpsCLM8-Smo_b7d4SE`O3Wum6s4Y^xpAoVk*!V>GT})V5J9T_Q2>=mJ0QNE6j`4m%pr%G@55NHDkjr(w91Ak8cpk6mhm zlmoB=Y{8+T{cp8YO`0PeU!6m%%JwV*0-FImvefh=fizqlg?9A_f zZ+porto4X=g0W`p%j+rSO&VE{%W_73Jf<=hNlUHf29TB&ly^KFK2phKps#%maYk0( zpd5q3%0{`wJv}K443ziJecLH6-D|z=_(BQg;B% zs9FK2lIl%14dh@4Hwz`F8o)qJ(M8Lup zYBF%tFoZ-JZTG*L-$E@L-_&3FtTT4k-HKZXMy&NxOxpS&L71e|o2*#L$&F~&L0=lT zRqv|g)^^#!!rZA(48acPV7F?)yHF_r%Zm@&6#@Vgv*Z^(0oS-w7XY!3;1}f7ZWRDf z%5kZOGF)xPt~*K3`wRHiryK`){?nXoE7Z9Y@zoVTb(qw+(tvulP@ofIuPkFw;peo3v6m3tEWpN9?HgQ*WhvBcuOrb)?!j$6f{6L3mAvK|TGDw8kSVaJl5+I$_DiJS--qs=eEQ}e zH5q^V_w+HGPfl)FUlFzcsbuP43qtEfKNb{}2N$~OJU%?35G(h}5U6Ti;eq`T`yH?< z%-mcP9xj;3tyPqV;bR}KN;}Vq_T8!hDEw&+B}v>!C~K!iMl|n(IFyTnwraTI-IG3Z z5^cmp{Rde#_$krd=n?zmjk!`L+M4w%N1T$ZuZYq#EmFWmQ0grMHuQ}^;BM1(NH42X za_dc-DnQwjdR1$!eG{~QQuLGXDHr=}gN1=%oAI?h3ePqB&CM~1!>^|~D2CJ4bU%=3 zw6K&jh{QC(rE2aGNd+dh)MR72be z=2|Oxu>GzYeKlI{h^AqT+J4u9rWSC=6WHb9Gq&8fV{Kl!>+r zRbKCf96)SD$w^`A5zC!SHwaxZu^i>WR|P^Fgd%(BjgFn*Lq_+HZJ-kG*FO&LKSTul z6Li`26VS512yb7)4-l$Zxd{t3_27yrsE{r4aTFl8KY^vAYm&C=jb6lTs*L;K4kPNL zrCfZbw3ye)Lr55`+(8?4Z1S%pk|CGIC%K9YMq5V0egNE#T&1vCx+%$;W}8eN?+2K$ zXfWA7yXA?>3Xe&-s20#V>`xT=>yp$V^Mh4`SyramI=KO`#s-yXWYGw7+QqU63Er(s zHUk`ssh<4m=GxA~22<+iX;6X0Bp6EA~TW{HP(LGc?)AhPuTYhE^4%CMm@K_^b4&*h`HAe zHH&fyD^;vo2^1@Pm7e-h+J7vnb&-%1A66HmR1Tk*&pMP&37Bg z4jUD-g7Zdt37r7DT@7w5WtNePLgN(pxfCgP_-~IDG_Rl}p)SZ^0F^is9nTpggS zaW6*olHblf->uaanYWf91CsO92_yft1gV7mTx3t}ltq{u$KcC@7PAhpYG#gL$Kh9k zX+F!%+T!2|DdfE?ikQK?PMyZUr^YRS7Q3+ulVL@AU!Qzpv%OC^uR9qLeMOz;ugtaL z^f`amml(mNSHGWM{fVTW`Z>J)HNE;7jHK;5{^k7#U;p#l@A9zj6%}Npxaup$5|mA? zS9#a#QIkM*)lxxQ0!wuCE+#4zt{6$O1~Xm)?+NDS5m*DXM|H#VN}@-tzv&rPL!oaF zld^duDH9||^%d|uGYNsj>ngtc##hP#^6tRL^ z5o!XR*$th$8Oin?a&R7l3jq%6J~pzxq%NdrMvPZDet--}v3limTCtCveVHt;LH_%& zC<|_VoqHOp3RX@S*f8X-?LKSkQsn{HY#%RI?)iK&sKy0ED|$AMVCc(D!Pwb->4sI) z3s?nO;4L~u4ygM4!M^UYd3CRXwNXIk)O6AuzjZ(hEKA3E6slm$x;n6d&C#<1IurgH*vn}Rq`zWNcp60@*86TpCg@Q7t>M9l~gi3oDb~Qr8gvAR(V3j|J=i!L^(=pV*~#WI17^K{r)R z0D**6fGh#Npk)>vhgslW5|;*u0{W%Rta+`TX{S z@b>F7ssOeJJ)mfE*~%&pZ(Al2CxH%F+E;31!}3;x)+m37tBWZFZir1+x?G!N-In2O zwL&1dxxuJ@b&2g{%PNTyWC@`&?o!4Qpy<#A)6>+$YgpJSf>6Y9YZrK zbXma*YjEJSe`@=Xa!SGi%APXwgUWT;NnfaK0^RH%@e1?WQ$U36Q)1F2LgCC@VB4Y9deZ9Z898>iG)cs1?S0QDHw#^wCiY!+Tr(!dH=n4 z316ra4i+Bo0LmIeAgucZE(im(F)3!hwjT!I6Xbs0yi4*mMOz2a zOMH~H68$_Bb~fQrEU)~}Bm?E(%F&WsN~@sJ;fR*EUNwrrf{)?$3bY62JyNIo{gU+r z^+A@hHPq&7DXB3uf?wOnA{#g_i+tS!Jb_2{qvouv>TY+PQCWoC9Ci<^StvcR08{E= zT_{yo+&a>xdZcy$BHLG!x2r1Gr%7CC_ZB?=a4o4cYOiVbLR$`~9dS~m7Gm(BRekV! z?wGp6soMIQ;5wwEcE{s-b{oQ)zo77l)X0cVftQzzE)yZEWX;HuBCcQm6#im=PRH;X zd;ihpd8m4RhK%ZvUbX|)K~LOk8`MkImb(8ER_j{QBCn!173*<`Zh2Sug^$T!b*^r7 zOvw?{=8+R_wICQdp@Nd(-IWY`OlMZfx+fH-VEs#$M>ygFi}#gv$x^$kFc?oO<01qH zL)?3dNL>RHJIOH{=P*6adV&vtCfS)&&O6bd?v62(k$MI4`*+t3>uXHHKZ5s-b%P$q zNXuD?73*P1mb~e5)45e&m871bU$+j2e!g~|&<=G!(ZptOiA9xHM%|e$Yx^$OUpm`C zT=z(hoYx)X;zloT9v91r(>~UOUB0SeIJSzJ@#Cp@*%jZ3T5VQ zLS4^$s6atMV4EQi@NIibz~WDP$lBS7#02GTJfoZ1h?~J=i}oc6O{mzeL@5D6J3(bq z3fSF;3MgdV>=A%y#-1I;mR#|H=Z!jb)8}Vfk8rD{E?5YPCBURw>Zjb6Su&MK)aw?? z9ICbf9**As!=+oGoq@^{<0fim#Aj$u{og8{yd+B`m=>0#R_3T0t-aNu$|d+YGuwg) zW!aQt0=y4;0~old{|XL9)mmxmdmAYn9gzc;96^^c84uDB5+Es9on{R+dC;Tca#CoL zZCs@~)0t+7gNr++^J{eR#ZLJZ87wPFBhhx4ylgbzl+33>O19>DB`&+?B*4c9R}G8v z6tc7j#CMh~)YfkRH&`l)POK5~D`W`!3;&c@s9-X^4J7>Jbe3w{I4Jmzyp8v?X&Ryi*)uc?>|D#{dr<*{B!u` zd-}TEdSL1|b!x&5lpwDqbJHy}2kypkRVmaI-vLY|zABVgYBeF%v8QDZcM92V-2ku; z5_yX}s&)^meS%Rbp>@2-HVl_ui6oCzOTuX>p=*np-~i3+L7xWj_u;|R?l<3kple%Q zR!j$8PvUewJ8vmnyhbRP8JJ26WDF+Lwg5^`A6sH^=cx}PoU3%e4IV1T$mCI^8W7Fh zc4w5sT)W8+E-WA=IU+^E$X6={5-?uFOpK+_aB`;rHNsuTCs<#T5-|Zo!y5tiSV3&x zFkud~49;;y9@~r)=kUpLq%N}b+EC94j69H4svC-x?d4l%*I^Goe5%TD-D%o*WD&&l&Cuyr(v^TkW(X}wu*8x(qAJb z#$=^6#8B82z&2WsF4U{NV?dXbR*=H-b%)001@_d|V+g>@%v$9$=H}Nn3`Rs5&}X zl49^hEut2ZMQSv(sk{C`7Tz9VacKs0tVl-%#n1@-^gp=SW3IzQ@_~h^Yr`${Y;$!z(Gl%={SbhBJ{a1kp z=~YOWd>qVgz9fQkz5a5O)tl$d2f?t|Ch$9YAD>P`1AAoWap2FP@m56J(pQSuVF7 zS}yR(GYsPm_Cvw>LpPw`(h=4@k?|^<3FpFm1EEd2>}kX*m$4f_?rhjgM~b;of6V4D zc0=2kmszvR7m#Yg7dQ59pNksDLYb_5gZnHJC9}lEb*Ya4k0G*NU&3|i1fcKmduCi@yIGTxsS3V65n$+%@z5&jbgqU7*NCM8B1*S3EAn2T-bOo|iRYuhK^px=SwJNi)K@_VpaI$R%1230~^PA{D*PuPt#?^SAkI4LYlaksU(#QtJN;v<}PM zW*oIc<%l&EnbRLWA+FVyn?OysGwmKo8oFBDV9muEv<-9;As@$r9;Pj-lQLEemr91U z2wC10HbK|760V8)kB8Jy>yJeb*?o>1Q+4kO<0_^!p{3YQ7tq^60nnjQjpV<859zE1 z?2j`NMC;2Pw38cXGf1!{QocH27}QN;ALTIqT9wYroo#mukKKNIY|@7TdfPT{A2o2O4)&g zw{7u{ZiQ|EU33%lFzcp0P_*oJ@AeH%sU$vaFNE?!-a1ng^{z}|5@W?toQLsMrDsv``j7-$wdIYiN@du;ud(g<6sOtoJ5#e~OW~GHcZ_r3u0bc3 zxPq#bbuJKoH-=j@04oWw!=R*2@-va$UXRl`Na(VpU6*l|?F}i5okT~-&1wHhaZ<`g zG^6K8z80S6@-j}~Z5cC05~FmO8Ma$i{iZxG_{&A`&)ZHY0q zqrM+(AHWPvt`DWuQKg}Ul%8UuL#H@Y%y z0|v}g%AV$ez)f%xO}z%9L9V5gcX@sYn}w8kTqX3KhY7&6bSChCHgLy~XDo>dcdZWR z?x9qwFa{nt_No*jMD=&r`>w2jN)4MH4iyx%<={A}9kEYAre^FE!i+$==T=~=UV*#Y zvLu?boeaQl^$tW(?}b4r7UOhjHhg=IS@(KEbK3qB+Mdd8ThIOfYgBh58)FNR(b!! z`)|$!_}_-NFD%9qko8n8Tstg);<{S6RK=%-!LabY+=`O7V;Ho}b;C9yl|3TgP_Q!! zkPEEG?PMUqaMO*tp(GUUhIOpBaN951?ysqgv;sLU33m#6l54rkL=7i1LKIa#DIF@m zBiFViBys6yoxGYH&{F40aE-Fk$td4JiaV)7O_O#}79F3sp}}%erkK?$FMC$aSP?Gl zi>z>3r@2*wXA&F%=$wXR&8cC^UPe!m1E8bz zVn>Ltubk2>_t0Dvq7|fQ8Gvw6TLXtI1F5!7*IRsf&K}p+^ATA z=>trqGR!9EGKS8>;2<4AT-c;Po9Mfcq+nF|MFEl2LdG%;i{e5trqC|lK0*7nkgCUa ze(cMq#Q8e40K?M@{#x(KI3*YQD)s4|N!=FQFc@w6?6Xl1=y=`p{jYn-Z!8-DF%qtL z;lN;79dA{|aJxa0am%y|sB)iU0=+}na`!zTu-+CeS<^G&pJGDJ0ue~H*Sc1vW zhUMhK^6VTo@rbI7^I-bBKf((Uc;H%)Dh&V(#`YFY_Ai^xm~IJCwB zy}T@L$fFIx)QRU;R|8wM`)9}DhBU&hHN%U`(M3s%tQKt#Lgy4J>t)zdSD>?oW+@qZ zK3t&j6#&w=e-cJpck=;?MmkJo>xoY~FB7cijD%~!OKo#X9H>OR9Nd+AWmiudy8+fn z7NY8LLKA+PIklEYWtVkGue*5z`wWak3$K%kCL4lJ%7k%wB$7>D0Qc}TRTQbCCmofL8aW2y`{92EQk*T zdny!gBt$SlteMvj5jH3o7NhfFPHXq6dsrHEA1(MnbHqWqHz=1t2M-R0E681ZlDYxN z#EiL7NKLJdS>U(pYcTu-C4=8oE?pv)ZO|tX2q%tTuU5dIZU!QEcEBVFiB#~rF7w0=yx>Fb5faS$Lp3FrCy%mZEZW13JO`wMA?8K z!JG@h7eC2@s(c|emNw5I@7tb5jNv2?45=+!i7pi3D|Ab6g>ajI{MKsfq-aFc;(rbA zf4&@(4r0UIBL5+&oXt8wvAsNtGMH5ngv-dXk-WZ^`w;oaJz^H(+@8Vq8~K}OM;aE| zyR$&MfI0;UnZC~30(zd0&Oy)Jddi~!fYUxxCcteO5QG+b2aV?CuE!um>;q$XiA>6l zW&4|E_9_>6Dq_65ftGSoc9p;BBf$hY&jeSgr4@aHc>4%)Xs_6YhM&e~n? zC@&16gS~`lt*qOp#cfF;D%ttbPv{7oR%9V0CCd8gJZf{IBBwzLr37H5Ep0rLfBl40X!NUvfjqS_ z_Ev7d*Z=1?|EQDZ>KqjYRn{E+cM-43Pt{O*uC5oX!=~NKg#8TKBMjoch zF+Murb5F+4i1@R1pqVx$%-zRXxv@xcwp_dd|W}Qj|e0(cItrYf=nPXE(%Y$A%-r1fwqYw$)Yp zjMO3$d;}5+&BY_DBL>) zc=^+|dX=MYJll7`qH1&*ydLjeoyaV8hR-Mlj|wX@#9D;m9YWV3U`)0r|=(GA~xv7?z))XIsmsF?ch@&cw2{tDEy zHm}fAhU1mCZ3=~&j#D6KC{P=lBz1hr`V%Jm$45e$Dm;fLNkOUK7ah4wikegc|n)wg686OHMHJ=|z^fuDt_hnx&};@L%% z8w#PpJH#c@WYtoO8HNCLMJvUXkqIr?Y@R85^pb7(6t`#Z0&2+advsG^XB!_@ncmbv zow=Ep0o`c~-NEt}@=UT%wFfj!yYRHGNxP{pm?3DUL_$H9N||-nJp65t zU~`y)p`NmP(-}C0a={wKT#$z%(bL$;&~~+e*8%x-xuOW(dt)zS#;kNW!_lZ676T#l zV$+dIedKvVjSU`XBg2JPS~O^`Mtc)kxhYQi21T>83=%}~SY_9&sxBtTiVyOBD0`{c zLw$;>ENoU)aN0wUpNMlw3JN4UmXY~@pq!3C5+Jm>>|rtQY>Q{5)J_=;rTqGEc-K=Z zWplCEuee&&DVsJ-Xja+@g{I+B(%r!4-LNWGvSNV#(UcI0~ddx*e>*j+5f%O z*Q+cR+1nY*?uw9-oG%CTPJ8Dt6}WT~EkIzlPN60RaDG2PG+L=RP60aUai)f zMpN0s+#062ldtRx)n^!SUeI@@I8%ihISdoE)PY8`L9*`myHAP zVsOU~cyfB;q3C->;ci_jXZ&ASTrWq6%ba5ly4(HYx^ljk~Ui0~4O zvZJJB7F7?(jzil~3)`VKL$@s2m^cAy28dI;ZU&yR%`KY&Z6`LqPdcvAOP_Yj*tDqS zt_8}w3T(5~lX9qa?0OI2GgG;22|$Oo#0H6$?X*pt}du>@`CPIuRFVh5x{LRR0+;IM-p`co0lhl1nl8e zMf^tA@GUU|!Z6Xkt@k9qY~`wS5Mex#&SZ_32&`1!TagFbF*+)p4n8TJ0*%<@gy4P# zJ9k|eBE(CfLjfhWh6A2AT|KJRUd3DiJAPN#;}g{!g!9c+l}Xddd`?JV2gtD}77;u1 zF?1G3*|GUn=O*g~)fs*_Eat@sCZa1a}P)FBnH_-Z{2q4e<;=%3UMod zU6QEBD}}Jp>#SWfyKLoY?#kBFCA4q`d{&zpN~OYZt7h`GtZ+3hb$o5%`OzWW9OI!< z25Q_B5AcO4no{SE$SjFV=bf$ks<}}GO0irao74j9F?8cG(UJ+kH+Ax+xxoy!(;<7U z-6~x8lv7)#bPI0A^Msqyvea5O8r--db(!Ruv@ktAJqHHj zIZ>MFIaUTEUWAVwipS6aSqawI?|gPF85dUg=3L;eJH+4sxVV~vv4nryG?W*NL?xZ8 zuPZp*IZUI=^76v+Q4@Tt(4ag)&48znQ{@U6Yg`?7A?%}d-hesrLTWtR5X(_k_mvtQ zFGT9&^!%0I;D;#7ecb??`Y=eR8W#x#JVJ~So4sDy4fPAA} zV>Y6Z^c4nWa)p02vgMVrWcC1j>%BuCJ$i=`lBUok;ft<;t+*GL7lWU1e*2r+zB33G zkW)7{X-6Z#4a44S(0AB@-Qiu;JKlcThir4hQCjW}L160wA#N680HR%)$*8oF>p6kv zuJfKH`jS}U#1sxnK}XcUnzZ6sa)Dx`&njd(FDmZmmW5XXi%n2Nnwkxh;Mtv=QTAH0 zwtXLtI6|F&wM}=}_EFE3WpbGFIwD5?UT>2&S#{3$lu{Zw0vmH$!MH{W6qD6j)6PvV z@sbv(agV6R?ZJww_tB_mN#|_Sc#Ktm+df>k5MU;&upyR!un$lBaG<2KX4ZUgw{Gg; zLhb1%3_IA?hQ*YGQKfU`caHgp-Q@}G?+OenEIJdQM_Z{p+yYxQ6twdum0CS!Tfo)* zU=vjEtw#ada&+qb9*&?cob5-=Rz+@Htsa;|p?T=xr<5gPi~ww+;M?2Xs@gtIkzd@Q z1dETRBFnKVi#aZBr|z$I!_F)P3T*;*v_Xp-P+f*SSlx_M{yIv`jvKF}cN^l;Emm@G zUY%wbt0y(q(FZ>W|KWr7^f#mieEoLfo9n;}`k83a z0a9?69a4+MR#EYdEtp3Zfwh1krR*(9@dhU_DR4mARCxoJ9Uhd<=N#?2ssQ-NvayF-21J4}bY}}%VujBFZjh;84z){w z^?}t7u-afaseJ7qwv?9PN zU=9TKcmW^NZT^Pqs6zBEoz$YsOp`uYmyHR%G!yz0#|$8Ic}X9UtR5^D^uH;?X%b8F z4+NI#7R4(Z?b7O8tGG&)R3hc~n44u!RIxih26C;)M=>zp+#^XT4&k{;fXsJ*Q1-ho#IqrN24_^OW;VH3~ zEvY-;x`X4_!TIIXzR+3uwN1nv7R|Q8jMU@v0^z;W6eYRJ%`rcGRFQk$5`~sJ_&}@-J$$v@XZ~K8hFDpz29Rg z35aRhDV7(`3=fryywP9mF@^_CnIse5mE)>=um$WPdk$I52G*7_Rk6QRi`XbQgQ?+_ zJgXTPgFUO?lcFFO#zaI+uh+x3*Up)Gz)3<5)v7-4Pe88~x|u!e%1Mg66KVks_>;@L zymSkt;T#vm;j=w7Qh8@aNH+2AqJWN* z>C)mOt*KoDMaL!(@e$A^xenv%?i#^csS*UFQXRX!pIU_pR7`>fFr7fFa^8*PW9hjh zi)BdL4SD`=x{jzYcB>G?cALueNu^l_XEG>oV``P4uR5#tyzA9TFgIuWyWwrStx%xk zA%846w{8jK_qUv%>WEA(yE(gJwStkHqdLalY!pHQpES59jaTv~7||>2XmGVgS^z8f zW|Z_@>YkaUqMFaua9ctHL=w9Lo+rV$Xl1*fU2sa3k&Bpad49>D3YaP+sUvgzJY$wp3XDNo}ploDxfrrcqI?E@`tD=kCkyQVwNu4=mL%?4L z_?F^ECl{7W)s ze|9|kwdo|gSkvi=<5ge1eHDIqe)y~J0YD&M`t~WCjQ=dz^6RfHuA;Ebp9rG8Q7dPq zaE%QDa^9*hy zFgpu&K|^V@D$_xPg(hJfQdS(3D=|Vg!HzlUtmO*u7LrKH0&TrpwnHZnGM{$ypc&R| zLrok-OVb@ybK^LBgr-EMP}uGRjD@3NH5?eDAxw9RSinWN*l-KeGua?$=oz0{I37?E z3>a1{GIHy~TB7Ym{bbBAI_tZ(FLbj3ci@_&cr6NrX_0>0Z0_c!JuA8EX4_C3C?{wJ zDCLj68gRy2qUqaoT9UZO4<$q743!_&^peJHWlAKi=Sp?9l`!liwEJTFP()vJq>O&t&KeE51?7 zDRQ;iN`xSS(!sj#OU^%{}!()I~8{!D7(63)O-viz*RmIZ;C zg>NXZwOEftZyuorz;3b54?-zlF_p{>j0v(JR@JuvHTYuQf8O3iKP3%`=PJ{ zCInwS^9>VNKK$Ibxr*K&pK^tt(r*xOGI-X8<;$%wP?I+L$Y*v3xrR)1|iY}fV$%kNZ) z!NDv%5h>Cjcc790G6HBRFc{xbzNzy=`19s)sdP+?k{`icz0eVT87=a*!|kM~f*CuT zs?e7{%g(#IvmsMD3P>%GLg2K6BpTRjJmy_Ny`(0l`BKrW5l>_7TZ_zOMw z0f_=%<(T==+wWd~W0ET82m0L?>_Wt*p$Lb1Qz<@ZBd~+=NKtX2GaqIP zYS>!J-EmYf>L74z_bZsGCI*en@M#0^_QBdpmeAyo$z_zynck9;{VjJG3x(2G*^-hw zfO}2sfl?Bx3nuxGCekN}G&YokvK?0d!>lU2=DN+{ZsBa7k$RTRK*wT9$UC+}0lCjz zTXs6TV3QZ1oze;o5_u1@cJex&BsJa9RF35SvR<1&Bcl|^#IcX8Ifqyb7)kCb{^;>? zX~BJ}$Rq$UV%g@prL75`g2!EDj|hm)bXgC)06}1YTOvVJkdD>Vm<3rrihX2slB{k( zwMO0OI#HPmt1Z|UUdj%ju2f-ubzp?L7V0Z3xaFQhL}?CNF!W*Bc5OqdFW_>x-g~Q7 zyH$>qrQ&}nJ;3BavfB1TjeJ!Dp=zF*+wqo#IF!=0(Vgso>!VoILD8&%eB6M-k&9}d z7xu$xCQG4{r2C4wkwwYvo|J)-A6`Rc`cQy=tp}MRg>Ai5yTPfdQKbx69f9Av?idJ3 zDuQ9|Roft}q1B*G-nQ9X?z}ox^p6TfP$?&!QYG0UOL7oiE$GIVeM!%|62}>COVvbV za8w+~338|YQ5Y<3p@Nerq9cHSkGAtt4Bl*Pz!utGMjl!ccvF-}3Oo7fe&o%C)Kr3| zbt7Le9Nj=l%#XK&>8RB{fDSqi-n%ew1#4^xSa%Bj!K{d2&={gk@cxTocYhfEw;Uj~ z7=HcNa6;OA^u6(ta;i+P?*pM@Gn#+@2}F;rjQ+{n$8JLO(RY6vUVp|w{6%=PHtFBL z{`&pjhu42$@_KlG$OS`Db#ITX8^KE!{R*LWsl~%SUhe9VX>md#r28yxayz8jh40Z_ zvdaqQoE7E@J!Qz~xHnYjlt?iB?Q^DVA!EbHH85feD0cYZGp@ z8i(WudCsD2`dy48V;st(e!!rSg!|sZ;kO4#Fqm47D z3EqKFPsav1p4oEcqnE_Q4#=Vd zQ>a?x$!9Pcit;f~_IdAEJu+fDy5r z1l^$mv!;4JDKkkfhD~qR>DDqS)j&r|`MW5y9d0eP@=PLr>XGLU8_|k#(l(@yg{+U3 z)|24jQPMQV>wxj^s^A<*EaE_=7U#_aoP0=cIVMw=XkQ54>(r5Z;}qYdStv{~snA38 z(73dTyW9jT7eJfVcHKyYm?ZPHvb|H|YI0PY2W-*uapI^DK%i=T{y={!b`n5d$`{MD zba0O;1(rFBym6DIa6svy31q=@1YC)>YbVTBU*`T=?N*fs?NKs`TT3W6u)^EA@u9>? zd(l`m4sn;d2Pn>s*<(Fod{^6qS#mjJ?r&&a8JPX{y;B8GUwP^Vx_+&d-qQ`7xM-=l}<$_XxkPb!eJ1wL4_==J0f*@bSBDPobsS zCVZ=RV!Gues^xgic4jpuf(HFAbEzBp(zH{$L-jdDo#{DPMYD5<^GQ+(K=`Txi~;Fi z4tcmxRv=5jZci3rlI;OB>fK9dl|i^hg{pw}VqJ}?1_m+*o39MDg}6;UlvD;5y^z(d zv-4cr%V@L%?I^fi2Z~ReFij^7s+20lg`27sTAr-i6|A8D%MnrTfzd?3)C`Ix^_s`R zXk9K+Pc5Zty2)Kyw_cJ1>IkWWOMJ4lK`L=c(=PA7c>6Yh9sByFrF8Ja_> zA8M}mRzaDhhTIr zs^*l(FNAVSmiK{tsKOSZtl)LyvV~pqKx&h#F}ZSpFm7&P7q~BK28phTFJk!<_TE6r zz%*#tdzhr{d~!e!k>OVdn05%eUff>|TXG9@Z0ikfa`ei;F1%SsX&%SxLB*JxB)f*h z=Y?i)XjFL=F$k1T;SH@GUHcQ&h1pUEZGyJC zs6&^8Jya0}YZs z?&?jDC?^H*OzREC#;$b(p|n}P*}Cc!@BBt};Y@TCm)meavgM5K5PVg#Fexj2fMEvi zMJtivgd|Bat&fHZEAdK*Gwgm6I*mpqDreUzRyA!%gMm1Sovgf30ikp`+p;KK8W~8c zPPV-B#S?TjB`yKxSi6o2P&!iP6=Oy2 z|7IY_B3my*A*mbX_sV*g-OjD7u_e{Xd1bul1Sd=V#_pk6z0EQ9|hRIN55?StX!+iv>swnypq`!LWp! z9jRn;CbBugQN_rEv>ed66_u$06@b*EAp9nYQU}cSW~f@oZ`sDXXp8~w-vU;tf->Xa zHLa{gR|^F?qBQxvQRQZ>L{j_k)h4Fz;=YI0O4V7vF@xg@Dhe~vNtJ=}q0UsiRXU{# z(x;JSFH9~uLaAsN6CSubziFa&7Eo_^GgeEg-=}2X}`{+``h&K3De0#q$#lw?=!QW-9ky8TWfDg zfG7%|#lXj{X-mu2I}C!D0A@=9qkCx9ca!iAb}fk=fP6_#^3=ks&ybX|0ITiUH8$F1 zj)y{9Ho`;X;W?~GhiryfqLcnf+3`3!Bq|!$F7cOh>^%fQGxI- zFF=VI&|f+G{YqBL;=WeBfy!s@b%Lm_EeMqHqo$JABQ4x8?6J;d)~d|M<1?=K$?*)v zdN&;<-nCrUUHc_WU_qWe(C@8^ z!;L%j1Ue5!5t*+1J-K@M@BgL!7yj}u^)dWIK9p_w8qxRv<8tIeP`_g1G50Dz;a4^< zEl&T_+h_QH-~k-jbIFmua zCEGde&yhSnLhd%?rz9xY2UI&D-0lv+pVXb)CE;`mx&;3?vSs%~(b^NGyh*r5+HdTw z$G3o*#24KlgP@IvThcjxs#FHHYD;<~ElKMl=w7MVCz(C9g5a(fc>DJ1mOij41Qy^b zMGCmK8+EbdRrNKw58b7VQnv zFhHU)AGO1i+%LvhljPR5@Q1@oy~U`!rKc&tnltB?K;NG-n63q<_BrdDScJzl zQhgCc5b`;fM!8V|QyB zZwT#ordLuAN1YAZ!@xOPtz;}&P0xT~R>6I5>V(T~$C6}7Y9=;6iFD zQt@XynhSMYV~(!oBE);r^fbaaxZ27yLaNexm>4OHzIKoM5D>wfRTjM)rO3VaIAXTb z#lNM8=1<=~M;879X-3;O9y_*Ihq6k9RWmabg@}3jZ{-6vlJC#zP#Oz& zFOg5yBc-p5l!oFdb~#}U?kEIg>WlL-=NmZlF5MdET6=qJ=1PIv5_dH~aN;P)Fg7bB zYllR-B*W2>`qRY5E+ldI**eIZm1AFE4}dRjepZSkAQHbOlPh9sH1Lx`)2!U8a~JB? zb~h<0(0Zr2^8%2nK?Jj91!cwp#$47Tkr#u>94n_lozVuK)f4a6K;%bKZj)=} z7!Sx-z3Rl|%PD5!owZKn!cKf9>|FxH0F(40kqVB916w1V3*!QlrL8Fhe@UQy3WWh4 zcUh!1s$q60BM$iV<9Oug%|^8%WI~7NZ>7MVWd+LMmUcq=fRvu2VzYoly_8u-lVGf7 zbb|u;8H8OWve)fWUW?#VD80kABiVL7iz%eMpjUz1q8^_S$X&xF&)6XOVZj^y>DhyR z+yfCAMI83=aAaQ9jC+>goDEtB=a$1STdj5+c7fYhPAN0!kBE`CZ2Ob9pTkk~)3@K4 zNgrAvKKlRY?f0+WhBJ8Jqt`DEO#KU3ji#W8_T9c6{*U_xj!l!jk%tO=r<$WkWZHGu zXnJ27S?msY%s?w-{M5*}4tifZbzZRn(0W%-!$UL?gF#xZz^cs>$z^vmv?P3Lh{Eg| zH3BCentH=4R}y7yIV>v5?B9mJv#d)O6dD^$aV-y(><`%VE7ACHkS8nv1`^D2Z6M~t z)bb}gZk!XYTT9(qklnqVlHn-pJl?#&LiJ=d-)kvtgc@%6e@bXaPXFk)C#62)w%ny(lcJ){L&|KPubiYoVS)ixGUrCy z(3E;m;61}C;8jF5041zLjIKMB6OVa+5erD3(6C?+oLZqc>cT1fvgwS78y>r>*8<(_ z;MYQlkZU^mNEhQF=4h>se+eIQn?8E`xA6A$2SNAko3~HHpXER+G1uC=FR(sGfL%{s zI1j1v+BNa+c|e5`a)ys4v_*!z_VFP7KVZXCZ%kV5hosq-b-i0E?`-N=Pbjj)&Ox^> zX4=RGQBfRSj;Aq!git{cu04|H$pr)CWGVbAmkWzr>luw~urKi{g~u+it@obAZCCSj zN)$=qznUo)_(1LwuRPVyVw1xrh*KHhvUh~bfI(kYnM+=w2$ zv6%s9wI4+sbzvgxTHCW4o>FXE(x(C1Bp@ESV+IiU5Mt<)Vhx||NdWcgBWA)m>$bF% zLV*;awTuDkn|DM?`Td9t@My#?IEa+|4Kh}-aFou)zp@^>ZfoJUrSVgMa}$z-bk1gKW11ENP^?cQaV#T3^ZUV zCYO$n*Dm&eWO*KNO%*hIa=CQXE(0Z3FoB~Pfn+uNc&SD;5wuhpH_f4sdv8z7zAC{j z!h6%08K$FOrB;S%yk?H;eoS&u*@V-exRiIkTSRX*O!eJe9w_#)_MUd?Twy?Vt`;zE z`;}2K-2++u*+(`Xdxt)5$Vv?PAh#jA5sub0`j-y)XRsW-tT^V&<*^-<%RlDdy0*O!m6U{v}{?DCT!U70drlIdLJ*V@(yRu zV6ku;w9H8pz`mhRH{)cf$dT_dtq9~1@% zAlu=Y?F*Jlq+~Fn_=a$O((vAl^a?W6VfqY#^SV8?5S&p*vD~#oMC@a@FPpd4Y@XUvGzzL7az-(P_ zSLslzqOfGol~9wPhDNQSw9d{Cv1k+^Z$Q@X{?-ybfUzU^8T~%_A}&qNY}>o z@*K8==@r(CO+Iql)aF|8q=E2owJ)gg%k2hKy+t7?HObk`-BESTqmMJ5K+Vuc8dq(Y z>blJ{Zd|t$gdp}w{Rx!^VNsit9OVt6_a=EA;vz$u+%;@_&(NQ9fD~MefebZPElm87 z)E44s%a(GkZzT>QC!5`H9MdCLje9;0HI;H*pdK3)2Vh9ZR%!~zLp)10f)7`E z#nCgF1^Y-TH}+E5H7~c+1WGqSpCoYZzizVAeh9!(ApPMahT*7P4UD08=2x=7*|2Oj z7miWUjwL?KsUCC)atCY6x<+;q9Xo}h6YZhvQfr~h3`q;A#iy57SQERRuJrrJCxmAMs5y=2@13UKn+4&91IL2 z)f@z1E=8|`eQR>^xt}eyJE7cwWWDIcuyza#4W-|9|5cKV?|%GtGDZ4zpp@=U-hT4- z(~!Shl-=`lGe&y<53hd+^3NaT|A21#kNjOQA*0>3g!g!p8`-RiAMz11Bm$%ymX@Fs(7@le%dkqDTF`V;#RP)q5PY_)g;s$8;n(vb}-lQ^_Ga>yUUq+Z0%oH(7 z%E}#3D`X<2oOJI(p;S~`lb&)kpts3eMpsU%IIMyG?xG*~!;ACkFig1IjKR=>O3TNfesxMO62np=tHN$i#-)AcUMv=Fn12WuzlY+MClYjB_HDumXO6i z4kr(1H;ZB$ZVyf}f-W*_?s8Q384!di*njKohK0FA*#rcY>pUAKJ=#*LXLhrC%>pc~9zNYbyyddovP^q%`YjoEyd*%;iE zB<%M#=f{ka+P$bPOX_WU*P%CEcauu#CDWp4k-Qus3P9LMfTz6m86yQkkimJhR7Cs8 zt5%<_Lu_%4Y83T1sHNtZCD$EEM8WHSi|dXpRnE;*!t93Db?8albGs`CuIrV{p;0Yk zk2;?0*HFt1Pu5helXam``pOw_1~`1bB;2g&a+O>8a*cOc?20jVfuO5yKTChP$`+AB zPIrTr+MTp;471!Gxj`0PLEv~?ma|xC@?c%XPBYYAaxQBB93&4Ly4jT)- z&Frw#ejL4yq4f~=ov!9EjO!$5Z6D~KE$w;i1KvQC+OQg@6}oZ>1gwjqQSDDcJb(H2asK&V5an)~!v6xOBoYqPnCZY$zz1*j%YrJC zo-o;)8(-VJ^pI9}8ay^SM@EW^!_qsloF2Y*AO z!8W25EobuW79AiDu`yo;Egh}T1aO#bqR1T&LoMY9E8@v@sunbAdj{NFyoHU8h-Fk4 zN@EB~k}Q>cK1N2cxUMB6SBlofj$%>_v%4y6RaiH^0a=ljoCeEWa?Boq$5TF(jhIIV zPh<~SW)qTJ#%BlbOh`RJTi#lTcYtthyLguJNCy*W@ey(h|J>wC9ssl&4^rxalj1m( zCg3Q^o}Yx&lalN3kYsk?2BB*xAH_pJQ}WS-ki9{CV^s0l)i5Z1E)OuySU&I>h9SQ8w(EEjTQ9=_)aum)!-Dmey-n4Hv}FKOK&-!ojqF6C zdV`q*rx;;OqFYFJhNS@x`nivve-Y6S*syK?Kd2(86=f~;sL|-6$E`q5vEx2kjU{^UY>Y_ed83>Mx?>?8+>tyd z)PsDLD&>0~PTBP|A9lG4mY-)gQLd*XA>;99i}nzlE*h8+S;9~t;qh0&e9b}pIW0RN zWQC$UNtS|uyQAOC1v6(OL1zMDuVi>uPo@g-y0WipayTsiK?m=x%%3PE=n5v(tf@$`BMB!Jp*>My>TMlJ-E1YicCs;3u|~AKecX5u zw*%@AF9%+ngB3uAO!ae8iHv**2VDl}0M*&d@P$^u;g{N}FOIqMjy2_kMl_o&NN-AW znJ02U2i#6!G!##dZh(4elddl1$U#De%gqUm)2b_uQa)S5cJ#R4I7-n8+aoTJt#hW5 zY=5e7bIGF$@7{KRRVFpPupGGswoG&9T~34LsJC=i1U-u)n41AaaD%)%EgGLBh#FSp z$y7ZQG#}jfR<=Aqb=(aH>c?z$-rb$CL9Xc&Fx5IP+I)6_6+Y&|1OCJlANT|ei4BDJ zExS?7{G@bD3&ziuMqax>{06k+BgCdum8ogG8pROgbW_S~=CLp8T0hOFu54qa}q}J&DflczDmTF3v$`T4Msn4aX z23eJd3mRZrv@?ZGHWiEI+F#*!)gOo=iIr=lp2+ML^?+nWka^f(PU3g zs_<)36Cl@->?ia;4Cw=x(&Dl{a=~tc!NV~Ef?~05R=!9mv^qM|k-V`U*x>~3_3i#B>uosy~#(jLR{ zVW)*AcVL^M3tOM{L322j#0^7vwqrX=LmloxtHukLTbibDyBm#9-CJd|-wf{Z6dM?! zr^>jCs5#0io0j&_oFLZcenp&yEcVK!hwLF31=bmz#^A#>w?V-lAfo zy#&WS$mU@bPsz1g%!QvuZjtZ5c>63I;Kg5s!=LC=NjZOtyz}$R`~NGveVZYRlYN$b ztH(u=Lar9q*`|hRR3>XaSj#?ii?u`YZthWgf$E2`85I&OJKUl`Faj%x`_W?D#xcJf z6`rJSXPUi^y+JbfX%@UAm+NSbe%NXutUJUbAJyts6I0~xOV+6yYb2vkJoEY0^{RTy z62IxZv4Pk@zet!%@~9&fTg1>crbH-ZIVHNz-NN=@Xoexg66$(ct+T$ z5lPp32BR^4Ivt+kLx#1A8?V~!X;+857F{T8_ zB4P(cmpe80b}Zr@m4!2Lj7N5Zt&i<@w^>1(`8c-_6OlYOY~88mi1^kZx3lnxeePxF z+&zNic|qqw>KJcdwP-xG=rU&);~e}Te2i%kr9RPcYfqylD`7A)9a>j+Yg7R5(v)Et z)l0(#LFX3z4o3;KfICZFw^*yLk=Lt^wF!mQEFObnQ00w8x4Zxjv<6%B@gLm9ER7W>kABU8`y7B(uKOO2{&C!&8)bJB)`M^9{zN8#VNDH`*Tby0Uy4h7P3d5J#6*cLklS z(R#9pi0!QY@yTFk1PneqWr*T=^wJe0_!r9%+0a z^=wQ?e-?g_`l|KT}LwZsxJf?z=v_YP;B?n2QvFZXr@T2CDe!aF<~k0D&4f za(Th^Qo})TqEfMBS*h68jO>u>Q;#d-CKOci&;$y1(>h8(US)?-PCXJOYYV96KSy$l zd7M*0bjlqBf?>383ldyS5CH&<^5KlbqkkYDN_z<`7RWwqd~T11>O+2&DK2m87}~x^nS#G?wO~#9b!@0Rt>e5(EdV$Bm1Q^} zo{K8}=!3rs|LF+V-$;7=zn>p|{Qa-tKOEowGN6I`^S4i6B=W`E@7}&*V3+Uwk;fn7 zyYl5VD?9Ji>?5q_=kVMbzxg@1bvdSbvMg2b{;;#WqaS5?CN_bPtcbt2CAtOeay0KH z4fL7(mYFTq8pgbQv)>4x3<{w>nnDFv+BoM7*0dkrl$k z{>Qsbl7cy+C4sJmqTsR=6s1G~zFPO|>a~L1Bc33gKuHQ??F34cyPf_V04vz!+sVt8a|^=C*TMKIlOGcc8?K?YoEv>% zDe*zBupB$iih3r!qRO&$P}o0A4nY4wJX_2+3T^@FuTBL*y%Y0sI0fcnOE7gv^TF!b z^)0&fpf)V51q^5D2&`L2ysQG-;6CcMOksn<)XmPfL1bUQxV-;=!rNDumls)t2hi2c zzX^kJ-1ehU*1xq;L;uWhCFk>Ix8Bx6-^fB22)QbF0-lbS*s9w+4)S*FCO1E&);9d* z4hAIqE6cX;5Z#)i%Z5NdyTk#$Iu&LC$JFCdhnhYMm41NGk!MDTPub7NvKX(|Q0$eI zmBt~2v%3%1Gq_-f!bIt{r zbqKa;bHl=E)JeYLVxgbSlPuR@JT%BB{1KuJW;G|X@FS-~GE^Lt6LGERkTjO?4GF`e z5#{Ab%6X4_w3l!)N5v2ji@L&6?WR6;lBSJmec`RXS~mvhNA5GXg(6V5bk%-UzMr%1 z@?!V>5S~VG!$&mLYmV<>{+5qW_0?NPZ9nn7;nO%me*n6=wSYVD@%fs+TRid!0lCThdnqd6HKC!vnl0ETutCc4aQ}!XdmrKMxs`Sa ztMckmZvs(mA==9InBl>OPF^17xao>oT%vvC@dx9W>6jQ{7sz;%8QD5`Rk&Xe0#1Mx zSX#+DqfhLFO^GUTz@8XX9vU*mKB7oQoAD6bKof|rZ;-2>+%yp|g z;DsU^q8zOPZSSfTn&aG#(T~*LN5i96sjrnF{afZ7jgWDc4d?2DfZeD=n0(w$%O0$L zS5GoZ&u?H>+9CQ>H?=U&AyCz9IyN9wFJxe>tB7eq54AiXl&$WLbb-Tee;ut2+sGf4 z$oOp9%V4>hOpwD7fs2v=VqzFD|HNDfg&&ceO?=$27~Exe_WQmG_0^*0*+&qHS?lFG z$(1ADXyjSHqWMJ0#q_MGXC)U&>1^p$R;2*|O%1G?(bdk82!WG1ZtjVIF*C`scd&fL>nd1&K`R6?<{&1 z`&l3tLKusLtErvUKq2OS^>hKz9ss<^=|B!q-~BDWhVMUdyvh-&wCp%NaefsQi)b~Z zmTz=^(|_~xfA{(eV+x{LTpXIQANn{tLdEjOSVbS?V^v7gSEw6YkZjYGhX|5bmA_}z zV}mZ=#A8@#DJDshEYy-1rFtPbBO6o=Wc6P6i8$&$J4CnysAxRlua>ctO~d9wl7~yq z=XU2R;~=HLy6ecDkm-2}J!?m)+mRhe6h!}r{ls_M(UOvqa!8kV=x9t^hfNvvqQ%G# z93PX2z+Bhq0T3sr8Kz`GS03Ij!-)RfWq1lLv$2O{D0!JeC10wG9Dw$)?-rOg_56-i z3*okKD6}!n004h7n|Wq#HD`HBgQVAns{<(btO+i!T?0Y2g;NaK){SmZ3z?7xh%gkwx}dYkomi&AAp$gnR6Z>}*jK;?YVZt)dC5{XdNW*7@s-;xQ&Dlx@*L=Q%3c#1D{ zA&VN8WC58;q+}1sli532if}J17C>Cb>I+qyTqvJ5C-^bT=XLU83ePPi9|P;7cbjaf zt5<8dPH?JHZHim#$r`MC1~xM-@6(Myc}J^ciqkk@kU=J+>=QZT%*3E+rKm3gjZZKX z00IeKWBzQ*8NE1r#*J9wY`Pu}XDWit=>xF?R>Z^{~0YuroiG=5m`6 zSyNms`1f#rfVe}~Y8dLGrbxp@Y9bT6kxHDB6S7CgN$EYVIIb1EeN@@lZP2g{P|Q(0 zE1E`NRMTdF_M2D|F0?eRaTyMBkP!roq9xRG^>rZo?0%JqW zeXv0sc06+SfwQ9yV^L-st%I$q0=Wwn@*(ZukQ>cw)8OO_dD(WmsNzp`$T$p2VU7c6 zuwgfw9hj$0k=_Y#SSE0>Zcta)FT`}(8ZkfkLHP505Z?dT=CmfD`#F^%l**Wq-sj`+ z?|maHh<%j*4f%`yrWfq=MeGQ=Q3hC~{J3nMAc^8d{+j#IpV>5-bXTf%Bc=!qGD;e( zGX%!Egm7Eg$3$_bK+WsmNLyN^38)%KkNMUfahYxe|0=l2ug=@IPe$agmJ1#;gb$$UX5! ztRE^BQA4NDmJ*7!{$N_i7uniupH#!fO5zs&j|3{ zHq;zFcKadcw2n8RGh1UeWh&wXRZ9X=df9{?ZZWnrNI4KI_yEU&GeS^Wh&lm~XUQG0 zf<+$m1?~V}T2ol35J(}TORa#Ca`fh*_YDG)1+W8!oVg_p^}M8vJc7WSvIG`;_1Nga zpka~-6itXK70|U5t<7p-hVdJ3I`3G~N|V*#{j&XN)lX8-N_=X@lNi{r^*T(f!S%)< zFo%D2H(s=n|AP79Rq8{!kieB~k({i=Y)bo+7VU6L5N#%D#xA)hI?#(Mb_XynLUW$blN>3UU=qg;Lh!k)4lpdG6RY)NC+SP|GUW%6r<{vOo?@8NLtwqzXSswuglcTeDFDHji7@Pz2;%*D}kD2FXfCZ|i4 zaAXN!8p8|6GQb4oZ%gB(xVf?MjH$A*Ol6xxxj zXtb(U2p^#d*1_ia-2VcHTGFrAk56p8JrC}3gnEQ)`9WI(ypLrB>tn+0mi6~IUwcG2 z0u791%8Um-&SmuOsXnNdAU;nsm9r}aUy@0;QA`DLtpKvD&F1JtIa^mB$0!}70#a|5 ze0`$?m=;R`Hnqw+Sq#8u5m9LkHX+<0@kATQWk*}Bu-%=H@&j1#3}}EopzlL~vJ^M& zt)P-!!;AeYm|aUsJ&%M~KtK!E#epk}_Rr-8r}7~yB{T)+weh)STg$Du(a*M)x(5RW zon;@T^0I86tw9h&a#D9tlwT;Tf%61F5JpN~v~bQ_!qqfgQ@Af|JVWBAU0nzk)AzVQ zw;RiqX{>Bb{Uu`r%%TPze_5dueLn_pdzVPUxE005#$X7$Mx6|gzMAB3?DuEeu@nBRQLzg{Q_JlO zPy~DZnS8}iiAopc4!vs81u=B;0`eZjH(}M?0yvOVq|Ckz&7^^GvKTk-;}2;+#T0m{ zNWm=|0dkUJ2JDz+atXk%2{OhN_}+300Oczzs9g>@ZES!u>2y%JqMj5cmOjc%56Mvj zbTR6qR1U;~7CP=*vwjBP9)!k?t((_Kez@Lw)NE$=P4O3Fm~>v+%6({&6U3IoL8dS~ zZvF#N#V*10%&=^b#+Hb?_$Cvm2`1X_c|M2>!ta$d1 zQrQ0E+sFEv7g#RK72c6IB~>@GOhb*nES#ov!-YO)6%T-j7l@zbfC_lbuXKN~04r74 z%xUL6wkyS^V3sW*)R!V2kK3SbwmvVAeBzQ@de`;nN~i=31PWKxZnfuvo71Fc*u*t$ zC)I@QovUg#Q3|8*1RSvX-OE! zt3xhY!u`ZlQ%=gsgM0=mLpIsYTH|xNR}gTTnYO6-IV8DYA;*0Z1#!#3xrJ$?{DiFF zKE7DPRX$Ip*(M=96O6cr$grl+42KL9~?$snyaH|m@IA%9A$I~B_?7lqG?Z4 zaH+W4gY2NDgAqVw+>-e!f%d$Qqx0);O$2~&Fxe5{j+#KDPE07QvGxRMc4KLWcEt+56B!#R@d_az>V!I zSMB&&a~p{ES<7Ww1+rdZuFz1bw5>(v zsKb_vYLI*=@bjY|Ss1h_=g;}@lt8+b*1z{P{Lfy%uSbOa=IuA(hxs9#a6itzFF(uA z=-)P4i5ke^l8)QV$}yMcqh-#IY|vy8Aj7~!6q55*F-AnsvR}Ica?e;!KkOA8nn(rbVFhl z2HHmqWfPM6bqDsBRFmxGkdL#ESUy&&kA=*zEKbxzT4bJDAjq=v0Bn$!A+7~SAf426)&Sms&uE?g+RG+z3#3)6w*gr~-Bz~Q7~4W!6?{5El>SYp zrj?Z6Qm8Y@&Gofk!Y5RvpIzG7Wg*)^IrXw84(}9Qj=4pWp8$ctJ`mGF5AUy{#NijCdZ9$x)u#T*>$!veFlI=nY21->oi+V#H zWeqKpA#m@IZXOvr(q>+isQ`Wn+REcV#K;rtajoVdCcpv#LJlzaw7w(4I)(MYc5P|5 z(&qN^oqCR#y>I&^cMx)v8=(nrhj5dC0@}K1^c!h(f(+Y2ot#d!6kv*1A>)%rlIh1{ z*=FsgIu|0e)%WMQ8gGv0e*OB(@PFioE`kk#+n>MvmNbXYUw;YTsxRKYc>TTn`$zln zmo~U~;4?63l^wRdq01hU>RCDXQ0bLjbHn6v0JtH;p%Vu>q8G*UPF1L^mz{;l!l87D zobKK(o7@RhC6n*E?3k>p$#f1EG#{7NX1wYT2sP_;cWUo#H)cUGK08cK`-IH#(AOl+ zSQ0c`9jmaxUd&0z{3t_jK?NM}lU(h%B=xS$5H}X}3QT*Y0TqxXEhc~*1grQQ zDqn6IL%dFMhMAnra)p+o^y|S;ZnZRCd8hYdMC_x53e)2*rWx91=zyP4eJu#CM)73Opt%^QJ0ez!4P zVzSTqn8fY8pl(Q(IjO-Zf3aKo7H7<)!(uz_lX}uRzvT%EbdvZbpi#itwybXRK5LD2#mWNWEd zt&oPLao75+4D=NiWXk-NxWe9Ei7nO#v|#r-su1e6@3ur~m#h>C1q6Or?W!b+$aBe5 zaqPeqQklsPCV)T>W9(GzX3ct@H@kbJ6af2yb%AixvapX_z%8cHRwXjwr;&qbB+^+1 zY^m6mQTBLzZe5~bydI?Z_|Bl(5X&0_Sk5FlTMGHJN|;k+1xFw1m<0l;uDh_0a(&%BN&-)_Ep2q; zS`hv$m?Ec#Vu_B6fvPEqOwwmMH89VQ0m#)jixrSA*%QD5Yy*un*9b?$|F#w5%5Nv; z_{iTJPL){{5(N&a7~rH^ng;4T*|oyd0wPS-rG()O5?>60Z8kQ`l`g8WhnLEDKj^75 zWFk0~scXzg<##daW(fw=!sg>7r@GqQ`T&IN&648LD|@AlKj)ho%#X8n!pNjlk$5vv zxkF;|ImDM>(>6ujWD^6EzQtty(3n}9C}H#7f>2j2V47JZg|W@{@0~ zR7sFU8|a2Z%3?v|G~RutV>2p({3O{4&3q)7lI4zf_gfu6V#~8_iikn35pqTFnv$D$ zI8@2>Z?Ohn4oN;(xxC1~l_yCE0@+(#La%BLa`4@eBz{B9=b8?hG}v*XGnn#)r!c#+ z*if}7GJsN!r)NB!2Cx*mIBg&Rvl-z60SHg4`bZ8)kR>`nSHvDcTzcQ|8`ouP zpxm7-h`T32ekc`^rR4Q69v;#)w%wJp-a7Ulm{!-3F5!scyJb>W$&s)5$5SV1`KK7d zIgth9VdINJc(mW4{!2}PN}~1Spb?j|+MlZbMR7?=|SkFcZiSrw`{sN$S$c?e$ z---?PUleDQJOk{GmP7F6#0!bQ&&j2<1bCW2KgEZQlCr4`tMyHLS4sx_Xao*09xDH_ z0Cmr?uz#a1gul__{~Uh!!yg`7z&Ee|gI~ky6 zi@tdK-Rsxk{m|qt z)_GI;gt2P$I3FIOKVWY}=e0b|$Mk50e|zU7Vg&FU>-P%H{MgRO%JWl_yb`uKmTiKV zF#2Rn9v2mzPV*r(mJXRil7a#Mw#{V74Ukb4Bjs~8utb(LJ$M=cKV&l=EQ4xADk%T7 z$zYFm;uox4}+c8?az$K{wp6xtPP9Ar4+=0f#Mxq}2yu^ZQd0?}y z8K+tGtb*%;8z~AGt-LS7X}(U@OEir!Q*bAhUEI3)u zPS|NuRi`5|)^(u2#4tpPSz@3jSr`E263eiVa_p8Uvg@;Bm?p{G0bI=i7QI->b2F8% zMnr}JUM~raj`3-TTtS0|##>vV`;?^NkVasqAt!(hXB%aL70$JF2eC2@w{4dUf^L9| z67#`o={V`axbm*)l0PHqfMs;W1_x!SYybicY)%#_@ji!#HVJ@or*7_*4n1Ci@M91B z|1}FuggfrWz!DqO&n zwG}j9ny*M6&g0dz-EVn2c7wgI{~SAjb>tvD8T!tY&tWib}vRU+f`n`>P4i;K$u z)(Jxc7XyFc%P`5R$=r`%7f|1J2yD+{Mu!dXqD`7h9)8su`*(l-{-d{_Ny@Gax)WvA*?|!8wZk@q1J}b4mR=Kl=Rj_irD)|G8?~y#GQn$glGA>b7I{68|8%;Xg{I zDF?s3ya0A>#l)`ePBx4OWE{O!^-^s+PYVG^NW?(}ir#MXiPpaL6gPpCKf(PFIhv_K zyUyu$C?l^eM2OvVRz1h!B9RZa4P|?WVC*iX(gcE!VQLmkHb~b2sL%q|Rgy1GxRy_L6Us ziJW!SK~%moJ3^2@A!$3*HM0J@eW0IfR$!sey*&q0s7g!wF1W8aTB8xF;O5e40jYBt zDLp1&og&&#h+PtvZd%n_j0#JMh!ip?Q-V^LlsmG=oCs;UbqII0(@_sMkPaNDhTX?n z1!8+*VQTXOpB>0q0iD+IVc1<#yp~+$<{T8HbF$v9I<>abg)^f4Ye}V<7F{t%&cW&) zMv5yWOwIEVf}+*S`1WCNUr9MIc6TM19n$;I3xU7L;j*EAVA}J`$(=eu+KgSQwg}$Y zLQm+4mOd)pjKaL0w{SwW4_6SHy}?tCeOnelHTfqmn-NkEzLgU8BDYGzMw}`Ja8#=% zW7r0>J#Kz3PLRwXLO%AVOIBtk6{r8Iy9)dkJGW!#3*?cxjch&JQArARj7`Tf#BGJO zC7KXX#pEdpM#f@#4w8!WvUY0wRUGfXepqo^pZ44E{U?rB{qpVGvqIQsuU}E`fj&NO ze|q~Sy#6vDT(mU*7fi<7?hLw*KZ7LL@2r;JQO~qIjPjqwU}(+f9#{^)v z1;YnpKEgUB_&WSE%OR@8Y#Cmx^c*%7{E)A+Vzrvfusl?G-@$$ zlrvL1D4}kZPwXVWgS+EM?+FyFi}R-C2W^sb+YUC>ZQO_nx;24n-&KY>TcFJ2qU9D( zT*FOOB=T|Fqj&vsY0E4*X{by2FEshT`~>_KvE99>!z*Z9R4S^+F0*3mE?V=tjzu9` zY{iRs-A+uHYqueR0WH@e9vUfxWtK|fKUFkonW7^0NUwtz!tJ7%%C56KRpN={LU5^- zFo@|zIMf!TfQp^RK3ShY_?^sZ;H$x=LzR3WEIA)^3YXaI5(-sYEeS%`Z z$pO2AsCOBUH;0kaV;@j7w}CNGw-q^Ste4d$m5R>Lo0Z^0HB1D=O1&>Y6^Z~fk$)wz z4+D9&Vifd*eHFGfa4V$aasdar6L2>Bo*95?R)Pu$jIm$h65Cd&g5Zm;(RSS& ze1y?akD5H6wg)It9>J8gkRuq_C-#=_(Cv3aTXf6wA+$9sXQ<=!nrRt`o;iN08u*k( zTHPy;o+>$`lg%<535F>0EE!o@J3Lt$8e}2D9(LorO4sK#-cqNbuBzBZlzkafOq)(D zE^eQV#uAjeqL01MAo2Pak|Xot4;DuYbO1Qk97id2RRSm4X%(7?AT>I19Fk70>!GU= zGPhv~mG8DuGgNwsNp2+fr|hT;cU!QSD%bbgp)911kWlGor8%kf7b0e3#dNq?-%)nz z=VO!wYmSwHlxffqRlqRA7U5b`Dcnx-rm=L$a}VQNH&K8+7zTH}qQe#cuKI~mKa6w5 zxw~A2HWuneBcGc9XT($@YGFoO(-z?>m(Mstq3B@lWj*Z{oTrlNeTXd!mA$fPjr<*` z|K07$0z7emk<>9n&QhcXFT8yRuZ}a5n{lOa?vQMAyJI@LQITE}an-$NMrDk|CbNQ8 z8Vq06kL0TIjTj8kiyL#M*26H|vDK)N(;axWaHmMm=K18Vkbr%7lvb;5YPqYRx7s*B zvtl#nnX*-Ur^i?L$3Vt;fbqykNGQ@0x-+0Pd{}A9T179V-Whs3VA(nnv`S>cRZ>OS z*`xqqNqh2*FS=H(foWt1Bgs*{nd;h~34;v?AexLFzPY$F^-`z`rElmR~V<`@7fA1G)XXYN-*h4q(^-=@`rx4U$GGBpd-}vpfPZ@`!yBq0oDW3Tu*^Q`=u&A+LdIH;7*na3BWTQ52&O@kW3p)<2cGWf6hwsEDlm zlwTwAKIB@qG>~+T}fGQgh*NMrF?s{ITUOYDw5SERJ!{S zECHgjyizCnLNqh<;R&QbejcGJPftL3Z(%6`^g9};}hCH6gqbj<3XP?u*Bf`zT` zzI}Otea;8o3N20`Z{j}hqUn&rz;xRN8IpEZ(QKId`0O4t0V)wZjtrUdcdo4T2i22K z7|T~DF0uS#ejZh*)mfLq-wEG`S?*Yk8UAivMdja^*>?=@E-j#OsHErIS%i>CuI7jA zUWa@sM_&n{&o)NsC@yyNlCl`L(L!>_goQ6i`^%&mpkKy)0hc}BcF&@sIMhm9zcRJd}&vp!HtC9 zp!n)MI#Y=oiV7T=&&f5bQ0Jo9%~2!U_s*c|2Xi!n{@wDiNAFSjtMqcKCNt3{Xvk;ZCixgjLQ zpC~nh!88HJU`|dBVVA9crIL}_Z5^6Ai=0>qs~uXN$SUv3;A{_KD$=|#UN1`>xS>KD zv`1{tkHP_cUJm}lye(3PNN~POF`+$AD?Mh_Y01JFt50AkBRi+FqXy8^Ofls~?)MyA z$(@^;F%9i=(C!hDq`)p8QXi8xk?(9`O>(*O+tA4jsGZD9Mop{oZmizGdF3!N3W8AfK7J>HrS}*TBYj{6ui* z+Yb%zAJXRuO-Y7G)?%FJ9s_!Oo6Js zNR9a zLUxK2tCQ?(#%x$*gr2}5i{=<3bW}?ovX1~8rah}*T~y(+v%d@QsKRM&cR|}jvWQCo z5{FYXfIm`wu*4Nr6!KlptMz0|as%5v67!snTy^DnyNy?0fErQ-cyw{9UC_D&kSB|^ z7=7GG+LUt*f)dQh|Ln}g{4BhFeR=;G3Uh4;I#3=86cd4ka@Rq5lnlu(LGd8g`ToPa znuJM9K#iO3m`k}~rh~)|dUvqV@u~*bhZ|w0tcTK7V6|}UqsBzX085Jc#H>_32zsQa zx2SqSZ2>6&223;{N4jfC`a=DP1?omUQJ^V!>IxU|K`mgwxtOi^$$99SMn=BHakL8B z2tIJ1A-ePgs-%3%)i}0VSA#q!Y4eQD?Q{8JN^kE%+<*;b*A~c}_L^z6`3c#A+~{M< zB9>lRuPVaV$3wqQwV143*~%g#wniNS8v-_0$atOUl6?jg`GWq0CK?Tc6(Cj6 z=jefbV~{dLsVs9Dpuw)|ep=v00%k!UEs5{gyD8l`>(H*p z2!x<0bUhEJZP@^_2ta-)1t1Vp7PkQ6#hm9*YJl-_Q zGzkcfkcY$$W&VT3HL_8Yx~5x0b33}fRXeG#y(dkQKN$d~L_3T_G^B|Ywd-_*4MDp+ zhxq^%S7s}+GS1i?r!~WK2Dc!S5|nC@3cmygg*hXNt1}xNfO^;)7fN+9Fb;6<0F9|f z9}$>C2G4okI3Fm_SVNDk@G9wV{wDmNM|k`8?dO39$FtwCclnKJ*M4)>uKicaa{uD( z6MI(j#@|DiRu3@r{^<3i_kSPWJ~f;pV2+`s^J~Lk@*u_=eM}gfCU*VqkTR9pXbd=M5BVQcksOWwulJy(su{k8YWhJXN z?&d-y5+~u`=3O!a>!L!>*}=~@cL~>v2gGk zayUi`D(_)#CC(lLjzI#!`hj9hrehN~3H)@T!fw=JivXhFI^wefMI=@m(^HZ6*z)_V zLo#LU%T-H_lRkPM?gks$4pJvv32*`%x_xvbg^{~fc@hv+K&0y~o}-DT|deo}TTOYE_0D@7FU9wtZJrMS-L*a_XD z!(?Xx7R83#U>{9OFIez~!_bCddcNG& zKtmP~{DJhriNNYMwDp+ILj0=fCL56RA=O2mm_*r85P;iBh3)6oZLL=|0P+zvoFRIW z$5DOb>VkX2&k4X{jg+^~5TMR-ICaCGHu-uDtFt?5K$1ySyQ=AzdZn5QZx+>%6zlMk zfE+K-%@((w)z3X~Z^DMr(vBc&eVdLdnm!b`Li1nhjiks?qg`sSX1_aW1Id!?YTP8( zlNVp@zNuQ-=b$T4CG8Q~pk1|-j|3C78)=#;iwT?@ib}lgo?V`DJZw1B9)^s}q1HOB z<-@?;1ZAeh*~AM{N-f=QvlRRmcIDo#)tSrw3SjmHdcG1CAsQoJ(W>vW2#&=>M1mZ^ z7Sy@2(d?U%|CX@govE?)gX# z_!(f<5qoOck~XAs*x3sGtMsbL6KfqqV`*{JSf@=aU%#kN`@TOFgm#v z>$19P5r*JSjn}bjH(HM$-GzJyOj1WX3=?xNCBlFiYoBhCjV`aoQJVzvuQ?`Erv|_n z549o5WFk{$t!ggVDg32FDUtGn$;|)@18SHIM0RUbl`CChl07~Fdep??DeoXXdm-Hp z%j`j7Yma)!8Rrtcm`OuKf|2s3@`$0gW~CJ7P@pnM$PEi?Chw)yrpC>N;t3O;o~^AL z9HLE^NUp|V?D^D!tH{8V@!{}BW}CJ}9cR5o1r?;|DISt4+X#>&jHHw|sr4PA-jr=` zG*Q`uZt60A(cMTrRr4EEY0YRCPJ&w^j6;F>2DQZPU1Sq3J5>^-VWANA-e{0RrCgh& z5&;%XQ#NS9h$|RvqEc2sLFu=VxwRUhEsnu#H8V&vn_?9tRFu+561+NuDKDpYM=a#$ z?244#R(xQfj=8sLyDL4i#~c7t1+++v%A>+J8v&JbI7a93a7ZYnnuXjgHyht0Y^o)5)`mDsWJY$OYJAi(H_H*sL}7_^B$f1<-W> z`kB~8yn+j`>^AO^-`-ySCj13ON4_Py@mpr0e+X9Ee+zE>?_757BVe2UaK<)4e(oRN zJ_d;W%gc`Y_W^lN1F};gt#Fr=4&K3W9tY$ODjR)5)4Q(&gT9J-x@ z3ZXZNcqEKvB)(7b*H6YWD{zb;a@)jR;=K*GSG5v8fxFf*T(4|7L9K2%{B4pd{Y`*_ z@flVHhz4w{wqV1p4&+qp;qkIy%nb#i!$@$#fYB@Z3AQA$gAwKFHJJm$3ChxG-LW*> zL|i^>0SBW{*-!Z^)_KMIV}_nou!S26l|K^^aMawCa3?>*xr?;v=BCS%OPWc z&>dMsn6k*(Hwz}6KUv6Z;lByQD>EIa%3E__J3Q{{E$npHp`r;pwb~?Gxi5Gh>DnEF zfmdu))~{8gUJ{G2gR%9J8o*Zb;dO-`N{-E-5!u_w$flxQT+euX6eAqAWEXe)Ls{@9 z77J!O#244rnUZ6Vm8#mLe4Gk9A_HPC${WL%8uGs%(Jq>Ws-#*E*cE8tWUbY@IihQB> zMTs#NTn1}3pq?tguY=<~B5bt$?;&yNZUOq94*j}0voY`CQJq3HWb0iHw1-=v1zbpu zPb8TGSI#sC4Uc06k8TcbZkG#N(8GKIsg-I`Qp2;=aCI~FjTR0ewamR~grP?|843VfK%~EoLw++DvKL+cS;%>w&Bi1= zVufCYL&IMQ-KQ=pMqmR=${y|FDqXre#}OUoEDS$gI$nAMhL9v~FncH}A$c-VFpGe) z130+|nblJ)fqRhTMeb6Xg{?^HcByB^nqz*QZyt19v=Wv_X+aNWiR-1&Yglsjuimf+ zIlN^@0?H@=g-S+>PQ>+;tHK&~rFJjyp+!hk2DS5*`baod;Z^2p9s_OM(l=_67x#qVx`_A&V^#R9fO4 z_xq1y@{fU6L@|Wam(%z0)T}Sl4+$&JY^tuNQa+2s%t_4`g=w8;vcr&2P*jVB+yhwu8 zF|_sP{di~t7c`t1D;2;q{1EeJZYBtEw3+@nPPFq? zs6E{St&B!Nrz>o7 zTMfB~hIr%iQs7hk=n)9>Mx(3-PYn&Uv&Ue?FS|cvl+4qduF*a%+FIRWa$X;`+q_Gz z2i3Pz1nVfx7fZ^1@V|y19^qL2AGnd9{l)8_0Ze-TS$O-zsODec@sHu{8xv1{fZ=r7 zH|kWN6EnL^t(4pfcF(KP4a`~)A@d*OlLY*7Z1#Zay!dXl$#S8pq3Z3pZosbiXiB{@z^k5<(`YeGkC51S zF};t|yl|R{9?)EkXnHP*9*i?)8ZWj9LuvQ@(-cg8`y6|kpC}mKVUtoVU(!l>VMNZO zcaGv#u5~GfXuC8H?L&%j8&~hD;z>Fp(l-TtAcs3!qfl-J^PpE%WLICmwTDBy+@Xp% z0$M%NyPEE3#7rU|pj1FNqA65(W1~ww1ofqsvwchjZMZGz6Zw&d>$}FEV(gqX2`di# z8|{x>0~M4mVd1toXCKNL$YYnN6zL}jJc@yNua@9F2v_{8oAN8>{IXCFSb69+ibFXZyL1R{N zN(-gzffzVDH&j_XU25y*RNbjRU5)C&SThD8yMv#_*ux_o^Frc_pmT|#xLlOHc0lu9 z53PCC;B3?JtxIUL2GAH1ZknRAcsJM#F^`i!{)doIpaxE z(?y+>99g~+HX^ynx#Pq{tzLBNh04;J;~XAvcVPR-yMg0d7@6~A8jid2+`SoEgQ@H6J?h-f8c^3@aHjU`6p&?D^jt+DdtBfY}%?GuOB{aMOux`X? zA5y(JM$NT~I1i{x-ycDCxeMa;`BRE zy|?|;D+V6HY$+Y+P5rJuqH9sP_~i^D{!)3xI#{C*a%6@Uxeii=cj@qfyk=*gyxmIT zb2d3kMQqCTB9V8j)%Kuj6Zzjz(jg8;V`dVHastxv{tIAwNlJS z1x`uhL4THA;tF@LmDMUXM1m^J(NRN43Q)*GXH-VDHL_6na3diA+6-td&(853H@#)I&`D3{bI(=#RBzPmiH;ejpID{0|W-E8O4cza03TNxb$$d0Z z^-$5_yFAEm+Gt^u8bGDfo|Mxt#E2@cj^vv*v1Nv9vVM|OGS-ZHQXt9OUtFo^9SXr| z3)hV*Ww7g&fVFPHaKFXm=^J%`G)&SHmI&1i|>Wxr`_2U;X9 zMRJj)PrN}9F}Pb7Nn(t_Q5&V z&=~=<`$gidB@N|>QZ>+Xq6DsWn*PcwvDCU-haYYP32f7lYl&j>;b0*P(XK{Z z00Z(!xaX zvgO&2IE2pR!ykO_Z-V?^{+FX#cHznV#J6AZQGfQxtwgL};N!qQa@8|sA5^Eyw$$Ds zU~efWwcB+bO)4RldIN4y5O>ndF~=*k71s~a96%|Dt#+7E=yR33sNCm3E{*D{8$!I< zB&|i~AMkI8;d5I1qIOnnNlI`4k0o;m*I2idy_J=*#faE;gURkfsZY*OrL!>Sb>}Cs zvICb#rM1g-!$3gRO)8T~RU)56;{zLjTMgNMvTP7apv?W$5X2QM6{M{&I>iNl(ssem zwHe!Zh1~UN1OmL1#8J_tb(>Yw;7W9K-molPQ+8t~m7DC?vlg^63Cjo>B?Xwc88 z#toQ%1p4jsVUshsK=Ok_q#D{kw?iLWV%y{45+Rz^De2UiGP}j*x|J`q(d7}Ozd@NZ z@>Sju*}YVlz{+h~wXu>U{v2+W=}GxP`5((^R5v3$*$q}!<({e;sL)tMV!h|J4~H45 z9AH3j$I&Kgva)U~(byBgC|PmLQYwhiVkk8=J{#>e38_zlI>(&7XvYbjuPsylLYG1{ z(zeJ0A_YrTmAvNBgbj~PT>ys)J;Y53tBF#Dd*4ki8#l~`fTy9C9lQ+I$1G*3cl0A} zYY7xSr1ef~?9z2}7-@uWU>Q^q~KIH*N%l^?#t%mdK zHfJWF%#{Ud8{AcPR4;X!a`&n)c4M%T7;6?1O69Vk0+g!RC_r<9mfAR0$~wVnr{-Qs zj>f9gz+$ZRBfa9siZD9*a%lN<)p&z_eNFvIZL4pU%!RJWT!wXUZtCbxXPx3 z=qg#JEU6v((Dn-|Go);0yEawO4DS{%Dnl1F5W<##18@2DhUVVql3t}KecRq8x7WaN zz(yj`CBa}}-HCo`cfKcJ5FeN`%xoG1eyGLmNiCDW#@JZZf{g;(?MC9DWV9%W!6QyJ zOI`T+kexh~OxobKJ@9O}msUW$*(eE$F3;gBry01*M8}2&(?6Z`&flq$=_J`MR z-+%i0ad`bre&(aM-vwg}KB7w(s-A)5MJ89)$f>$V6+2pJnAl5&WF1Bf1qlxa!v&gV zyItzHh&j+{Tgp)9dkvemHBCN2+#p3l$=Ypbm2(cQT02g#9so_{&;&w{Y#v*uOs1NIUBfq8jWL04f9=iq^o85+W@qs`RQg(iV){ zGZLmC5@l9XCvpw}WVB7hZBnk)#)^|eDS)DeLaK?6QnYtu5K5fLUdghF^~u`}lk#fh z=~hL(6-^k+{g7WA3S7}0VfFpi_u80$g1adAuau>yrFG--Gyz3`o!fUCSvS{wcS296 zj}RBAClhB>>$IZxy*r1dF62t95gXe+ExidP)v7n1t#4C3>iT6R zxz1j4yp}h*SRZjh8Niz`Nfb6MpKW-C?N%yibJFh*88>?t@;JpvDM>)WMpdtqTeK?J%)uWWeF00cJff3z7%PEkNOrm>>iJvpS36@MG9Gu!9EjqxHJd_*ZDd51!@dE_O;!#Khe1VsC1#W(Z*Vn$RPvkH3g4^3+ z{Z_g(L^;mzb?ErQv3s{68L(&uM|%x$OGC&byFz!@GK@N|sT z6DmN;^aX>#r8pFO;D0-7ArA58d2^uLd77;z!r;C)0j#6Ul3Q2||R3reEl zo!MBZZnT|^|0Vp$Vw&oafkOzdt91J%3lQaWL!KM>(}xpp(`=7#0j z3}NPNIBnN35qSmDu-xZ|B_g+^GA zgdJPgoPEgV6~amkdrCA1-VlTgC&a1`ZRHdax%7^SCJ@joyDSSR3yj~3;jZ!y;5D(W zc8+W+jv_%J90xXtAvWX%f2}$VrpeQc6?6pW(cuYzLIbwo2L%GT>QS$I?>1RKkO%-o zJ#EluEylZLehU=(TAS3MOxze*gCl81d@zf#LsB@s{6;&qU|R;C<{-Wj8UV1B1^H3_ z3#M#@w4G$-VVC258yIZQ=hM)1?(wa2v4O)Ii<+PWj-1<|<2tgIS23ZPv_lmDT_Ho- z@u}J2{TO6h@1_U35N#@ZRjGyvj?A9b3hpUpCrd^FF}L}n_Caf%=ICtAisPYnnc)Y;x*+VK;KfFwq;TACK zt=rfrhEZd7j)zv>Oix7!ni~%hmuQ^fAXV#;!pP$Y-#W`{)UXm*tl1_8XAGKt zCl7K_T7ZU@6b+QC4ZggVdROz7WdiMeC4=9T#KZKfLIZ7JCR!yr&s1{D%ldACJ`~Fd zzOthe^Z+U(F9?1%j;I21wcwwg0P6FAL zP8emqqy#S__;^Mlsb)V`(9e1WXxBotqa=@V_thW(Hg+)s2e)R(o*U@`^~Xzp13>de z`SA(rChD2RCv2Lo0u?|bhO-4+Ajkppo?F!%m9jE163gr4!@Qg$I0AtZLGn~Nj?;k< zf;~A0-N4JBw~p)-H?i9ctYtW>u5V$>?V>3hb%8)Mv&anxa*x*#l03X*WSPA|Op$rv zl7_ZcR<`r>pe9&B_|~3)-=Cb%1rh;aa8Wf(>|Awo!EJ2-p{)R^%B6XrL0&%rz(F-5 zo(=x$Zi92(=nps6mh%isZ^(chPO@j*Oa%rul@h*sdJf82kta5`rFxZgmEggSAVhWI z^Yz=SE@kglxN?nNn2!STJXBACuS)8nqr%Sil-yI6VPsX2G+XuxS0L06)&y;z5PT9x zdR6-9;^V+-ba7M4tl28SboeZ(Ybf)(U!@wAid=pnIZ9gI4LwdSHab~zS52mp@`)tIuDz{Z*B9kqE&k%odKzC4;$fF|&g^=wudt!%Ptg@UXZt7#< zV{qhc3pLG0$p$qx2DK$G$!VUceL9Uw7hw*I?wM~dhS-}D204d*Ft{|BwmYE-9$=WS zakUy7d>d;CFj-dJGgk9BA81!5=+C(ItB+(cO@bCC@kOdYDR(4)+PS)`)=gB3955K&!DPysICvr zF9h0vt~+3&vb_LBnVAT1Ib%W1K)F8!CsK#ULoq)!yD}wo?Gqv@#CYSx)D*mKhnUNm zDXm*7SKJ-wHoGXHFMPIi92{-Ac+1j*W5+j+IzeQQa=9uf3a1=KbF{BYjc<)cy^`7Zwj`F{>;QIX_M+ROB1 z@X5f_uRjQFP^P4OHMlPoGtb=6L}oe3TEWyZg+o+asVu~&O7^m!G>tEb90TjHfLlZ| z?x5Tb_d}l)mmVhR%4z|l3cWPeam)FsDPW#py7MlqC4(1`2Hv91&I{c}F8b7Mxu&3h z5af=sAprz1bp+w2)e1^#2T3Qv3!)fWarm_k&=oih?E^(|5lP!}0Vwb3kk>`TA+J)E za-~#TjM-CG-WrFhG{KTH*s&~p=Y%4G%;~d)#lD5r7vjI=baKt*Wfa1=aI9b&*SSMt;8Fxt zw=i~6Y|}p>>4oe&>5498cvsjHQvdp?uhdq_(knHawoSOtAeogcYnhAzy#C-#L+2ORl`e=6#T+DZEp}1KOgNTtf?#&)q{&6KtYl0SE$(y|vmS zq@Q+#nV`lG7IZ+AFidv>`_2cFu`Cri99Rc^KYZ7%YK$lSZy#AG=MeNSj8W!b^U>SS z{_=O>Z~klE!|dhl_o$X-#bH5w2NA)>ETOjG)XplDS0Z1Ky`g%NyM~Mit1YQbEo;&H z#?RQQ$O<^VQ{yJ7wfGZ(I=#rjlRI`=vYO4|Lb#!fP%{M>f{_LdWnJ?KTxK}Jw)K|! zw69>o)YPKcoVTJPHM6UU$41J&Lxg6QeaT+Np|#2~tAzXFu1fbU6Sw^enZox!{>zV# zY0a&zb4KM9Xp_65rj2Ghc)T|3FRJL-6QWqPd8{xlqT0~r$TT}Td~n^B0X;1uh@dcN zE60$~tfjepzeC1qeO@wpbyld9aV&RSN5J>Bi7F-zF--M4L|&44ea_L_UH}2H`W14M z9u=p?t^o-#R#0E}4LRI6o}fzCkwJ{iPgW$^C{WA6{WVUok&z|vq?0Vue97TW`LwO6 zU<0qMEW9DDPy8{EM=-)pHgjrF;oIzYgd7i`>ZolOW?GPwd8qs&on^4xvPn=!Reef% zFe61u;>kZNk&WviWFOUp+)zZQR1B&;fQte_dP70Vecn-;SG61A)v`Ob2&TKS89BsJ zbpeZ%5W8RHN}L%S)eU0Y)-K*jJO*?G=gEFwLeer0$E3QHmFm3an^_WHn0C!ip0L5i z=uxdgjwpMy#sT9PEVPV;Xa*jgvGsF@+l z+O>tM7}x{NDFSUrsfb?^ll|m;{ngvg8PtBx$K$8J46k3D-v9jVSFgVg+0&4L?~mc_ zTh;sWO!(fY;wG=5nr5PN3VT6!g`Yio+bE7(2vK0u;H9(z(=(f9EducTg(pN=~2T> zjYL@1iy-)&M>*tVq`;ccClVm-yQ3BQx{EP<6dYq-Qy6D;v9@;9RsE1G$Sf=mt{8cl zCT*1Kya?)u;UlX|7*SKlvpkp;;D%w#~zd!lSNS!?>Ov&-o$pWhJ~=hii+C8Bd~mxjyuB&QB#Ln#6XO6(y>V4*@q;TH!O zwwN`nUV>L?TBR!GC>ARNKAOkfDku76XD{HfjA5cK%8+kjQ<1r$GUvtI2 zL*e>XAhi{)h^;!Rry>CAmhXQ^aBnuo1QBro7&PzUil5lY2k4;RhS{547(!?nI2%kN z0%MaW3V(S0^slA}uYabf%%4vWZNud-u!W*H-Q~k~C)J9C43F!%g4d6zXUP$0K(3FA z5x~Q?a4c9E+SFPx)PHId>U3Q@C0EjfmvYT8X*t>m|RBJ{g=pwYGVC%r%9P+PKBA}o^Y{rthZ5WVRa8@l2@XHIp?M7)rP;Rb41COz8Yy{GdH#)`0!Mt3#zo@oVWimyjq6${{$l;tH2l5ik2~3v5s7&?{cXx070E@^oZQn z)x$*Yt<{j#AZre`ffEI|)m0ouQZ8BGCij;6uR>Pl@bUuE&q2`F9V@*QllIwFO1cZ2d!T(hPLy_NL$aZURmYM|f zmQHnOSa5Kjkvy?UCGsrOJx_i^v&VR@nkNT_n+)73SF7B0Ye(j?9ij4zf1P0CR|Mn zHyU_BiCY~T=m!(7>?f|ea+*A=Iv?PHXesqxi}DbWP_v9Vsu)e6&$FzKo>2ohO7u73 z?os{L;m@NQj!NGv-9bhDu`E5;N%pdxV!5g1>l20No40TNnvC=IHHO8u!TSA=;llyR zZ(i*g*)Y+GqsF%t;xk!E-hPYGGAN{n4)l&zF#gMNe=~S247}v-w;(Ci`eDO*Z?x%8 zl{$5;g5$_WGlvDv?OK)6tqsx%xi`E?+)yR4ip=6wa#Wm{kQ|m1BOI0gpaJo!TBgC8HS^Pe~iT1;crD*frUo$Q~dDAm0H6-^Y2; zz-^2Um>&7*9m0xbt)79LAWQAgGLy%dX$cpX z*kd3?b+oQkpW1@+28_ia|_8U6n+=A0kLO*=Otf8)ui$}TyO{Gqjbql zL))&^OrZgxQBAR->4QEwRSpNWNk-=qq09pIJ$KiP>_XY6`tF!{j1Ellphk9j(72&* zW%|9f1aYLa_S>)?vYGHKr{Q6npi7LyNuot4e5!+7!CtG1mQijF`vmdbrj7o* zBSAVmB10(xhrLQ`#zRvsUL-f7q4a;X|AIa0lmF>IJ>zn|cs0S>FTi;G`0bPM`fZMH zBmCL6(~Aox?gKnICku0OJ{MUxUE;QB*X&|LM6Nu89FoC2>)IA5T@9IhYqh2+6-K|< zQzbeMxOwxwoX^W?yHgcPC$B^81HKJ8s1p`Bn7$o@d}OU!8y2Sm1Kq4PLy#Rtfs6Vr&4mJ_O*Q6GNd`npsd%3vE#zofDXI@qihn>sfnB(SH zQOX67^0|JdbKuX|nUr-F1)A#wfWH6CPzMqm2bY2reW1zXcnwWQ{`MgU0s%AhF$aZz zLUL`FZ8aKU@Pkfze5hQ{gu`GVQ{L9v)cruAm>yD-EMAcx0XWtJ>faL$2@So6hfG;3 zPKfiM!C#jrzStG4Njpwg<#)6y_AD8`6b4TYF=v&J%KOlEgqv{Qt9?M&Z2hG0x?_a1 zjt2gNaFzqN!yI>}Hdz#Zs29c6OpzUm&_49vrL)C3n+ZV41|_vC_Q0e=0wD-%I3L!( zskOtDl@@95C9lCBVASh+^v~>%pTk5s3v#t@NFm8Qpr&TD=K z2|HWj*+SR*x)m~L>q9)BjB<`>A;lwy?YAKlG!J4s7E_=)Uu>_R3XQyO^am&4G%DOi zZUj!m<@)PrP>AuR&vimg(34Jc)-g-a7M^-S6rL={*k7KhDe_KqFx04+PGH^Fb`Aa@ zy#Kq`EVA>x92UQK1j29Me#1x3etvWO^fx-)`G>cU-v9mUpUnE@V_3g@BO3-jgpmny z)1SENMZP!H-dWP;n+o4)64$aTBuN}ZP__9OmRO9g#vpoexAd?kyaojU zS(K}(mH>#iqQq@4J;{wD;Eb@yRz5FnSfJ`-!}M9Ts5>fCK(*?c+>B+yhm|YEL|*{x zS>gb^Vscj&*W^Z?Sek3KYEsBmIuTm{=|}b-kW(HUE7k5Eh*^wR@2$bS_1QK6h?%@m zMOXVqAY+FC0=DyLM%SRU9az*J7CaU{p+aW;p6t;Du0Lx38m@ZFsh1xcczdhva-0`9 z$LA-&(+)uQx86I3Q?kU+ZHH4&4x1{<@L;nzPy`I{LzNCm>j8h%!4xDBe7x4K3zz94 z+W4e}oTJ6w6n39hgMmQzt?eANA$IrC__D}?Yn$qvVy%O;8Jry0Id%+;Ui~C;Cgp`o zy45HQi4&J~S6K$jW(P`EP1Sj4qy=nCMOuRr0N4s<>X%+(ndagNGEu>m1I)78A&Cs6 zSX`Vp(?%7YL+;adZu{!?ni4PJhK}N+bHOsSrY#Oa&`$2|zHv2L$eZ?Vo*?A|?Oj#; z?o2Z>mB}zRM_e1Ax1LJTIB9&52l?54Qco_28p_?XwS#$v#1B=fW0^%6{g~U7>L(mm z{p8{)#s964EB` z=-G44$~dJitE8Dsk{z0m*j+nY!^I`8$E##$o1JKY11$_~>WGlG)w4zTrTCH~#o)w# zBvz{#OmRl7nezw`+`(D2olD?&wMPU%{9xwaw-pws{zEyM9_h+CIX5N_K0xS8Eth&# z(6_i27F&$EG}LS3H**&&P$!9dq(9vip40Xlm+h*iI42#DJ&7dX#2Ac8@`E*tEe+b7 zN4QZ_a}y)67$j@jl_*L{^~wg|X-xUfByXxBV}okidX@NM`QX(G5@Y4tr*Iir-ELv$ z4D0CSogoKOKkL(OGb5Gi0X=MNA6_ig>-(6m>Z88p!kJok72Kc*DpwFNUN?s|E|7^! zwWNV-7{Rbc_vs|7KFAJQ_nu^#k>wAFo`(gebk9XXrsc(Z+|@2Mx{w(F5exZeg(_F- z^hlP!?r?hl0At%kcW?>HSCH_3P6^Vuh4!>m$F(OB-I;r<3=(L)sjTEB9%sv%%0EXITL; zhpL5qdyGR_tKW$_k*K9C3EL@;zSj!+RxofJAT$#Ll$0(kN|5bA%p4a!NO%Y|Yuo`; zmy6Fm0;JZ4{i9awTVbVOp0_OYe6WW_vFIaGmR|JF#-aNQArz!7Ka?=*vD(*krYU9 zTOT_mV1%wN)&ubeEL|-zdaP>4Q_HPvWhK0zsGWEX#Q&!W_gH%gkIKfX$}V6bg;Y375S zUa6O&+ND&A2XNRa;myJN8JJI~wsH$Qs&eR$1!WDiV??@+cp>NQQhO7CStFwD9^H)V#awo%>( z@U56$vet2 z>%k8e4=3$9LAN?rSb+a#lnpuYD@GIZfZBEoM-{5_iQy8TLZ5_*LpKjH+}T!)1n?!Q z4rtyERh91=RHVIWNCF&@)+a~kF8D+m+&!r6upUJ%*E5(#1v%X+2vlX((n1@*f3QWCDf- ztO^zN#vu6Aa=3t%5h4cmBMnSL0-y-&g>9hQ{Z?yb@1IPL#tlBF$0#CL4N#^@bu_Gv z7jNOIQv$i#)B#!9E(`ILz>n8tO@>sP_ZyctWg405SEmIl`_Ww#I?!g3_{5=d@gQAxwZVlbD) zB_(9HU?*c8r)~tomlP0!i4;F0_tgsWodNilElljwk5hACF44k>Z7vEU>p!_Xsbo;0 ztwSNM$SLKVoIt5#&}8-1G+TFY8Yi6v<(*Orp&St)aHuAE1~oxKmA9tgQYho0mH|{W zDSgPEcqR7Z?de=lwg3XrHuTqIso&KWvyL!jvStBX;+^yg6jFIo>tSJ{I3oYSJ}O}RL9!%W2!QRci2S8eT#7R8kTn^nw`2a3)$cF#)C z>q-irBmjL_)-0_K2%+)Cmsc_F_Mbj___&~3EPL<1z0t3)+yf;!Iy1L)UzERm1 zv|Pe51STo_y0-&F0|xaVu?{|pUUO3EU?I2g_Q%aBH;^P>-h8j!3{GX_OR)dVzfFa1 zXvaCEC&?Z?vzw|!gJ5P9s0crT=!}<$o}|@u-wS^u|K)h_&#(Xb_GyrZZ{B`(djJ21 z*PrO~FWMIcP;ND9&0Sa2ZRgZbNn))msyFa{Wzn}>9*ei};J4MGU|8(7mXyCu>ehD_ zK}t31*-z^oC>ypN)_Y>Z7b%crN0QqVDf=i@RYMBvn&jZKJOjSxaNCBQ>G`ax(>I5^ zFR4^h1}8yEawJ(L^xw5vjmY!{rmeiZ~)KzSFV^dv| zk?Qo6P``uK&UTu%;i70Bi%*i-hTe(U)2OQ5P_jmiN2QIah$wfKi9Pcfl~^lB2%nl3 zO2{(d++d-sb3ljO*90c_649mPvw31q;%*Po`9ntv^NZLOHtNe3A%HKkyi<3c_DSj( zgjz}it|84V_$m-{ZM#hONMV6)11YHHc13OSO#AmH8ZbV?$Q~64aj^!|mU*MX61;DG z#Q@7a&XKT7z`Fo~%{slEX9~{%fNfpSR#FwXQc@(abU_v@Tlu)j)h8*X`V)|mb^*cA zV&a5;PYF3wbqSm-Jc4dS>C*^nP)U|BP_Y*J77Xr)q8bXI<6&kkdF9RGC8U=I22Y4M zIIIo==9ciDQ?EQ}#!$JjtW=zrgJe`GqQLngx4_2y0(Bzq)3<$YOd#T{lng+XCuwix zo|RR~{^LlcNa-lS+9=bVXJ-n z`o+H^T>T}dn_oUpH-Gv1dp-tkZ9WsjkM7fhE~-Kmhi1p&&xKH zjw93RusX!2Ah$&QfJJG*?pX|K5T=+QggOZ*BjD^8B=8)$U+%~r;XDE3C-J)9B%o>m zY5}}wheTG>ts>0YyP5OYU^p|h@_l5uf@#I(0`Ch%*zbikJZTeG*Bs24>XGD=qP zQgl(wku}xY#>+cl;c(#4TI*xD#w##uk`Q}bQ{^Q9j3#QL$je&x-y{2WSM|ps|D;5g z726iIA3CabOI*^7W@8WMQo@fflAa6^#iup6keyopX@izZ&yFPQOxwfM7oL}A+rlK8 z)9YpoFLaYb>4*usq=KQ~lPQ!PksZApP=+?dy=z;w_F>O(#xsNBEA);Sjd-n?lLaauXsjad*jKv=iXr)|*?1C$ z4KvjJ61Onn+z1C$7MAysN`hVFpP=)5{K3+VYI8L?at?xilZK<#o>eK=DrK?=ITMXO|+?#4+w zL58ocafaod_$fk$0bsP^y{-@857le3@NOttTrHPy_S?pY+6T7j!O8e(Jt4GnLB z-*6V}tlq-;Kq8Mmc~Z0&%}ujHb1QK^5$2|OE`mc{?jsCGU)*swywC)rZNF#(AtU z6TdRTOI8QZ>ec5RYW$?3oP7bEbd2YxBn7(G>tSyo!F)&GOp!KMxjGDK2juJM-d3%` zRV^Y{9z;idSuQt6$&DqdMwRWV$+Tdg2zGe=zo)HKqnBA7>4wszK9n z>RW{m?Mn=6ly`e1Og07?y8}~?-lZfK6UbWjgc9!@i=05jT}@RBsuyi(UZ8Rl=OBh_ zLe6T>o@T8Q?w;+chxkyrb3nM?Rbd7_84H&A4!4L1<4^d0s^w_~Gzx(0`G*k6SWfEF zXfdwBfC#+_#>jIxv)>upTJrPM0sY93gm6j!chm;E=C$pBbQp^rbt3ACgACvpLnrt4 zskk`P891sOntb?NNE+ct&O$!F-S83pC1t*yo%!ZtGpS98LXuVC;XP3G{e;?tlDcba zYRQ*7BWvhKk*lnh*`T({`}$ za$VNOv%tA9VHt>!S+i0Z5Q!rGeS)mNdlssx-pFpVN@S@R0WsZc43w^39o)Cv0aReB z30GU@lV}&u7Hb!X+qV=u9Y2t@T%@!&knqxn4rpR2CZ#9=bYF4lH_38c6Z!(Bh=*Ke zOTUS+m|~Ng)VO>S)38dIh#FZ?;v| zQZjT)^zjfR1SMEu%vx{28P|az#S56>wh774a|R&0#nI((EodS5ZT<7sFSng(B*x=` z0Phxv^u`_g8o5-HGCdF=C7#{9iBMh@@yfIkR=K3ao9HPi7fOQ(%#qIU^{w2z_Y5W` zwNB9uI%d0P9U@xx3#3Y0S@Myxr2B9}^AChJ%O9VKA+kt8u=hM~MnbAuSv)u!mP1ZG z3iaVZo^1smdL38A^GU`h%T|?{j7%nxEP6AkObYetsDxJRBYGTU0dMqxfbOY(vO<$n z`2tq!tNTfJs6|vvZ3|pTl#U2DwgVQ|jz9*!;UUfif+@@uVU`Sgbk`zO6I*)neSE}2 z72{Hv@l%WpZ~vP2K70G>Ret{I>wiC~_|8v)+a`be`fd1r4u<2Hg}9fJK)=B}D!iSm#VdR-pXcWUjwq1ksVfD3aBoDp~}jxDO_Aja!0Cm zvc5@yfbiF~n)XJy&XfY(L{t8_O0p2qQ?_`1O-a6ze6qHT3c2Qr!}^8t+RG9tU$h#a z0=YHpRh}5nu?%BbvR|z22Sxqb6EK!`tsPlES~#sPQ%emhJjJE1!N@2<=*V{4o@pmc z#J54#nBf778Ek-VX;6bv`^Fh0ZmkSVTP*-dK(@cF-X%Tixtd_rJLIS!x(4H{swKcM z9nBYbcbzKjM_CFjD<|X_^3FlPuF!`2))Mj1ucuN?T`L1amQwfC#9fr#G2B5I!ScYt z*Z`OpH5d1wW52`Ur5gFE3*=2yFWSHg@ka9_eSl&D?d7RqAlG5X_h5JtDla#lFZy{|9ySOO{!%$Kk$cN#` z;*qDTMlI^1D<^nYwDCd3xVBG7t1hKW_8a9ho;O@nCgmqo0)WdsBJc$TL73IiWmg=P zaq>3I-ISTdrfpd5CTTF>Bt(1wJi@j+#D1w6*`-yk3{@du#ABisI0NIT4{=a#3|w0+ z+@bpUz`*kyYRjHqSg>Sz(a7yHysaahlwRjJy_ zXJzl5A-+OoMEjVEtaS%ldJ`2_0`E>(5w-}H*%ggF3I~~#Y6&7Y{0;}@g>kV&prTr% zVlDB@*pj=FWL$8f-GR01b_iVBl@3h$Np0v_bRY}`CR?}iDX%UGQC~N}CW7lh{s&8z zzw@2%z_#TGiNvY{TW~!4+qd7ET+Z8{k7vIGWXoIz{#uUhc-^mFzam{Q|FK{3(O>iW z$=h%F7`Tyt`p>UFCsg#Yhg^{Eu>m04#59plk6V2%EDCpLb&4eM5V8$fmMhsf7?Ks) zNG|Wn@}2gRHWs8MCKNlh=u~hrk$Nczd)4~jgu#tYsmS^ikh@&B0W2zqd3z|X^xL*4So467`MWfDy>_^ zo=WK0r9h^lS>A$A z<@)l$MR^5NN{UVH6vcjmWYu1^~>h193QGTWQ(9Eh_Iwu!xKibyXwGMBe%oEl%HU2MFeul8bLIObl5`yz8rDX%V`rponBYLmOM}KXhOK3$WM7r-79A;tnpVzurTNO@%d15CoSS zVKIC!{3qC+0KpT=R;ryz&u(=#YF7;BAdCZquiUt&cPy``4a5=?J)AJZweIDl!es8? z1u>Q_A*`xMsG)f~br{>qE}b9_ZmJp(o`yj;iUFH}?O`CsR;o1#v;C=GSkU!t6E8>+ z9pdoP!j=qfT?nk4#i*Ll2%B#m6Q_j(ww_@BF2X~vCJpK4kj(1^%_Z;<&_jc-fa|CM z$kLXnM4mh&4#JEzs2N0oTovtvoZ0DRXE6}y7M*4{`vlmT0m)!HM>LwGkq1g2Xd4Ae z?ulbbb-a%i&AKo;VB*L*l17J6;_JRD5YEz%k-no=$6|p7; zrJ#gNFtvczzzzXqJ3R+hRZVt~2PLYp&!rt@^q7^~ZDFp1Z4hjdEJp@prj?)-*Q}GA z!<0m~OLFdpgGLUfuKy{Z zZJ+EnhZc<@#q#6C#jFm>et_JykU0QdLGnI(RxM!2#h2t+(erjxaCMj%ON64;hz|50 z!t1x$nfoI+bAJl|LU0&9!@uG6r>FNnCR=ldG_utUnY5`b6XvL@SfP_c)q98M8MseC zw5n(!kqnaGTTnzauUuWYy#dfNsOBG^8b1WY=WfNEfU9CS9K%!@Tb*w9!Sj3a zL1>4Gw|wZH2{V8rMtdCOd$zRk!a8g7X;r0MxM={INsu2491Nxi1h5u{Oaa&F~7A1>!=Ea0@l)I@%XfbQ#XGY{lS>P1VGe z5Gi@v;%YDGoHk>*(sD*slE}yEt_OwS-03L7GWavhr%yUTRBitpE^q4HXrsE#>kR-x za38sHq?Cd{fgR?b5?-%qmW*(kQU{aUPgLELO;#64Ys?fKkbqgsct$FEtc4Pbkbo11Rp^m%Oc!)Nq%3#@4I|cE9Xq#EEXHD# zm1HEqSF;dkbIk4wpl|H}fIaJb%>Zz5hG_J*K`Xa@cdk-cZ!vXDT+r$mj5omC7n(^m5G!jdhxn@6KsGL`ewi0FuPc_FLr24M*Aq!y1D4S+g6fGcd)zi;HGEzX7CIk*n@8lL! z-y1@GmUmLpb7;GC#A7cQwr^}5B(w$CJdksX5he!q$)AJIIy$(ZZ``C7H{NV?Tg(L# zF_d~+n?SYgcU${FVvY_x)+ju&@_0=fCiTqFr>sVfI7iU#s5XT&qVqVTo18GDnAu2_ zgzTOujyVL%JR&79BS&-bkTi<`jk~%yf@3l@QfVJ*NigXqrkCm?Ym$%KMMe*Q9>R@J zF5knGz<^}HH&947!V!S)b?08FHzVtyRmdD3%xwqxP<&d#u*Gxa#U~5^gOk!aP>hs; zcGyW>bv$Js2X;hO`y;cU6D=@XM#hcL`c>_44HTGei(m^|BkP(W&RVH}>rURlAh_UB zA`Qx5Gl#sZGHL6>ZXLDD!9TA$;I+6ud&j;Wd*!fD^LP;d(kCx^FTc05Rt zkFsf??xRZd;KEIczHly3D&*#{mT{i0o~k*kjpQdssN2#J&5S)6-8&FpHYU#{oC47= zR+C#proI2a-@fLmSdMonTG2rmO#sTihsP!{8m3aNNmI;QUr0_9=B=h%QJ3IwaDa)6 zlyktN*^YK~jyZAdT!uS7HFuGNdjhSRkXQCj)V!SxjPNAElt7b`b&$TbVD3|v8Ic?F zO;CO8uL37Ga8CW*@Hap3IQ;gz*H76c_}TI7*MIrjU@!Xo_1A$cvnLU`=j{*F<^GGl z^s!-y%pd46hR1k%0B&QLw>@-K&Z4~+ig0a3yPD~R`%#!=VNQ$8!$IYjj9!6BsZg$(J{FR|Y$P5xa;=-P9YiDgw9@8K z0xKY`E{O$QDEUhBNkV|35VnrcdbQnXhj1)pleX_}z%!t3x57--iK#9X`ua3EMgwXF z3IBpJd=69Mv|`wU3ei*DK!P&SbSg7ljuL9i13iMc+pzIatLOpI>ph@|EH3*|F|A7x z4YsS~;@ZQ5T4(HGL042#P6Eh*IA zr$s9dx#98~U1(p2BAH~IR>3C}J{ecY4=Rd*q&H{x1BE8H1mzIq+BqDj8dWw}ohl7M zO?TkV(pkZsoe)|9IAAcBm35|oU3$j;o=4=^J2*mXg49##33W)6MNk@nJ>5M^BtS55 z2A-L4AAID^$fK#nXpb+gLgi(0g_8fO0X*TeD9!gqkg>uV+1wZoHbb{*{a8?eMT(xQ z$|NLfHYvH*7?zlkf;Lge%lyEtyFn0BM+I(Hul1EQ=q4u#hX3bh?@v&Si(TZ~f z#=IwIXv~{b+B3Aq79y1ux#|QAoxEP1!UiO)0@RPqc3s zuSsS@@rLr1Z$y;(!sj0hzP@ zQ9Cor9x7efrYA6vi*rkrKBz%i+%2UdxKWr-N_VsIl%q(od2KzC5C;wUg4TFwaV>I# z6JMd~ig`N}W_t$+6}5M+Xf|>aeIjetOMV1-W(@z?=s+8uB1unm4NiDmio&#m0MxW8puw@{)s{nRX^+&60 z--U3ulh*qykQAV-Emwr!@azHvi-J&fKaZu5G{) znHx(IFz+Xr))RNub({gubieDOi*-b_)=|W8ei`G$rZsWRADRt6J@KJ%!{@JGgjYjS zd>URqI%zi~_nP0nKpwnD7!)CO8DwJ)?y0^X6VUs338i%>?~pWtTge?cibTF}$ed;+ zGn4{{o?t2zn3DX$xMRq5-a3EDP4Zw|s`1gf`RE{Fu>(($;6Jm%vqrSjDb$t}X4U{W zK@s1;69A^+5rl#+6?Ab1G5LN~0v((?n)a~q7>Jr!M^nz$PH9K=dt{44F`|1wS+(We zhVfF~#)sG_YY+J50ZwkzOVpi9!On6+bg&jrk1)!o1WA-S#x?63(G7Ox@35&FGLB!bkA&XilhNYv`v zE~olDwu+XqL>Vm;OH0oZ-c}eqaD-Qd-I^4Q>gIXNxd=B^k64+4Rbv@^wpFRYpQXKR zQYACpD4de4T?i=a8{Mu^z|O@&Gy<{rKfn=_?AvAx7zIjtJa)R8?oYv5ijj@Bbr`*o zA#8CFt{(Xbg{9FywuG8?dKMZRCvu2ajtvS4TxVj7)gjQH?@(O%@K(K?+oLrCpwj3) zY^;P5?*Rt0?(@SGhNG$@5H##fj6sTa1}IspuA|C6a~Hz9?L==`6%OD_t78b+={hxk zp!RCIuD^Zz#qq3U@ZYdc{WJEXf5ylBx?jBgD@1z!mdAgPoa zSFLr#RK72)sSJXMkMdS^NnW1JOk4(8;S}S%^%AC6EpjU^zmlln8m8pghU;$Wi2sJB z51Lgg!|>+9Kg+rmjqE<_yCPv>`KtzZHP*o3KaOf0Z|T2e#TT$#+FCx_i-mO+Ilr6s zD==^YWSm>40ud-a-hqZ#bZ*vF>C}4{N&Uh zPHz1z%o9WZqPVWSm*y-NxI7`B7u?4qd(pKzw4Q+arEqL;!4Kr*6QR)(cT2f{NiUQv z3--(I0xdB#n@!ms$aW75A6Nci)Z(7TmzdfQ@`wis`bKjMs7GueVFyRKhBilA%#vPDX0r$sHFt?X9S zHK^v-QcXf}jZEy3Y96&Rjfd#nmE_!kvn)0b*h0yN+MR`GmUwwP^tm#Fp~`wn=xPVv zNa@{%2{Qg)6MB{~drLD?`uvjLOBqf2LK??(5L ze8gT5)da>@w0I4MdX%lQF`!{=y&8%*d$ld)-(kPyiDmd?2KtRZD}nm=u0HYa*-!uC z?e}k=<@QW?|MR!6$y@x{>({Tp$p7N`e|h~pKd&OVe>yz^*UI}g114apH}q&FWn(;6 zQCw)E-J2bqznQEP_2WI=wCh8phaB8PCl876(#XR&NjS0yE-zR%sK9SeEEGk3WIl>H zq2wU`cu*KgvFz;=OGOegu!@P8lf*Bw*8_5FIxYK9HnU<;Jhm!b{K?S(L!yAwtb)V# zq(ri(&ZO*1v9d=8lVDM&_BZ&QmefzXb?SX{lmy~ai;Jhu|X0&Uozzv3fJ8Mr5<2YV zRL=53@Pw4mW_wL00a%$upU`f?&1FR8?kr?=>tcDxc}RL}%_FJCBn!7AiNjT5 z7KE;o^^COjkOPsGCm6`eZ97UTyZDfH0!_*6worLPs?KGmRTvt&3#>~}zPtOldV$-T zCaZ8;*DO2G_6d*>vx)__j}MnQ2Z%*hvL+n1ttcWTV6AYTalCS`ZYAnGQwGBBdwGA? z_}pGxBev`NaJE)&hn(8l(sGPQ$COH^vERG`G;E8_h?UJ@jhC?*B%V4-CQT|IY4wVn zuj{ED={JFdUhb7<(|K;=yux9TJlpd+B=PY~JasKe3Af$uxmI5(%78@HP&MS_*j@rf z@|IwiwM1M8>P?7w>QLHb8zj(@<@_1qvl3y>v+{R%otb^CSDyM_0bCw!sJ!Zn# zb@y5+V&{E2_&2H8ND|=!$up5H1v;tjiw^(yQP~N^YmeY&AI+({R+#pc z97)B#S#`0GKC+UvOYPOU3gW{UXe-VFe-yCp7g; zx%?J43}!a#@&Y}AsWP;B*ZaP6h&(xY^BW~umG!$yX9fh5$pFJn&o zZ@Ig0Na8^}$S+r^tMuwUw}&JiTQZX}ej}CVruM};(v!MqLwHT~M5?g6a~Q6pcpBwY z?;Jnpp1evTOI^VZJ$AxkBwD%buMlMJ;aHC9$fu$+yX;6GK@tb)qjtF`pb`^WP8L+v z8HFxjsXq$xzLYd-oUI`56_%mxLeM!aoTfvB5L0oW6dm^0MHb^%z}4)lT>%ev(d0I~ zmxwPce?{5qKNG%4+IfFPbm@yS;i@r zxuCIT3*1*^xdkPWC6#VNmNM_P^bIMLR72`1;$^BOb1ga1%0y3Ybh?w%0k1=Y0&aJ`CC=Z{aL(+cOC+f8eY5d##3fR#z z{~aU9&{U*LbsI*Bin~G7Cb09Nx8~CO)2uRq1|yqBv~MYb4E-)QYdzduk!4J6noH)K zSg^-UQc((sjvG6UquO6uD6JCse~XJzNx-+U?5MW0Ql+m4F<3t6gGg#nFY;!fuCeN1 zK&X!mkOkku+1@k9xsdLpTMqE_c?q91HcTpOEaG^{_r#eiBdsRXcALtMrd%7Z+#NU8aGViESP(HC$yh)YUZ!NnSH*}$_ zbsow=1vF?{S&qY5J4vM)??udQ^A8%y<6!I;=Y~(HWQ-bjLsPAbAOn1K0p|q0r3H z#-JeB{5j~uCu)`!-oF08Gvp5p;y;!De+eaz{F|?zTEvrt`))VJt`bxhzhOOeZ(;vf zJ6is3o<>CO-9u}axgxvanv2L{&$KFKWy%cOd{Z#m98X*61FSU*CCMWcINAdxi0s%i zTb+6i`)RG!>!dgi{cwc}zxsVz@>HL(yZ8XtM9BF#$Q}hsTupOlsGVq_- zkAyIXEc&vx+xtAVm?WbJZ1v9x4N=KAJpjwXWTMNXlBx~?&}LjFmC%`5TA)D-TD~`7 zgxXrlk#-%CoP0?Q=4TE%xGj?7;`%kr$%aX{*+(wHyxay1EDRH8F!8gk&a_rb5dQ5GCMfdyMed*n%gvugrC)}(f604)|N5u!{vX~xmj8bOmGBZT z#4yz4_{(yWKhB8X@D@<=4di~WYy5+30NLp;kDXwk=~iK$m|1JH;w8z~666t+lLHAx{F?BzZ6`G>>wUG?A)(HY$feQo zUY59AX_CBLgV{b}*hDx8MlA+Pr3rQGpJGW`sJrr=lrJmH`}*YLLa6i7kVQLL05CvE ziEMR%Uh`()$Y@^8W>EqYWohT}c#;h=Wc;0XLs}SIx$-}tCRfh(Fif#10Wnc2dgTwp zS3EJT2Dp1!Fh&79 zakT*z^p18hR#jSoAtHle}hLYmF4bX<>wJ<3JlJs?OcVSsw-c9aOY1(Wb!4zK87 zRU?#_R@Et(;pd%y6qSCPL#jrnCi?Ppjq7gL!)1(W0D$I}oRJrMQuoQGEZ=QO13XaJ zs1zUk0VoaRqFq99fyjuwK-GL>!;yRoafaKIQ3R6n#>o-QPHAe z@|wuSZ~I9q+nYq*Jn9STC6oi}tZ7IK6!}rJojF2EI()KJAi*-!{3t-lshPH>l1p&~ z88Qbj!p(+4)(e0q+v0eULjx3I)?2=8b3d}B2c0Q%}QjLZ16z>f!uhA+`Xn*Xg zt!=>?yw`$#6Zl&h(=K*r#b%}>H2(^YdbW|D{bztdevg>^O|ZCg-{jUi-N!7BG&7m) zz^Lm_cW@G{4=tIxdzR?*G6&T`)_J6na^U3Jq-7Pr%QmN7ANW%&7_tw=6MYUeR~4_- z0*gSvF*Jtig}2q!rlU%*Zs%^vd9GAy+P#KLQ~o)Ib@)Ww1rC{EmQ2DCX6UNi5;EOEKZwBzp-@RS5Z4+B{Zck{lIr?U1zE4h z)}>JIq(MbW-PLs7W>uD9$ei0OF3~}Z36Rv$6-6!^aTV$SoytS>4lPPaTuei08+-^p zqzz?7*FG%p4KVFtPhg1d@d=}obn4e3urZC6vTdrwm5=i(9g3R=G*VC>&jIpj2`NT! zBWwZDMqn@p;jvMCUc~$kI0bgF@N$|3;6t=$Ty4urTlOQs16LqA2v=U6rA)v#g7VONt3sw&JP@trv_&Lj3Z#cZ& zbw_4JZRhO?P`^o%TZ}*ev%7&TL9n*$T!|K_l7i@m)+P|;h(4oR!Ih&_y>5Ykj8@jj z9zEtq*zBe)wc_l21}nMMsA_GV$VwV-Q4?LXya{xf+Flqm;T~4t~*OBRDmwjDj;$;hsW&0Dml^JdzW?*N*xBM?BNqRO(Ytmah@ zZ^PmQHo>x-hE{IgyRGdW#VBk9YiqVnA$R#8@A?6Lmy?53nedQ+>sOReaUG(Sa-9nB z<;@5|2io`ol%+nTCE-(7iJOcohn?tUtBz*)WrPh&e)&Z%nt~$Ah(Z!_kV~hDRjMjY zlQjbWNcKQU0f7rsTpMq^*IQCekg?}J)S>gHIvv){s@j!uvDTq5_H7onaJ3?|4-P!U zxof8@gaswegmgZg%UUv~5A!C9b z)$J&^6zG4Yfr$|}E87MP(-tjI=F8^vqwj>_O>Tx^4kZkm1Bn|j;6T1MPJu>vea3Pd%}xhR-LVTt^}Jb5@d-uS zmsj;?bN@P7pzh%UDn}b!JQE%vsx1^tK*M5*H-pu=gpiaBS!=HaC&So%vU2`WB^&du zqB$Gk?T*J$M^s z%>3Z*hrjvm5ng|%7=Y}S_epq87=Q8h=K$;a(|%L0aw$dkgi#$(_SaqK`+!YA@%@%U z1~G6&K_$0!KHcw%Q89T)}jndMppQgNVaURT_MHI`WBW`3(91unL}}^ zBAnIvRxskVY8$Gt7AD6)C|64{xdkFR%qXFh<2}nB8jjSD@6=|QV;z-(45b?&flIw< z`nhh0W15bMUZ5Xjy9+$V-9&!QJ{i`egrW_uC_8Yf|2~h%JSAp$mmi9nmU{ZwY*Z-M zX&karNnT#ZK^tkZr0@tLqApiix6C-ti%?npi8Dk?v0XtAfEF$M)*o?FO|{F~;;gs5 zvEO3jn&5IuZ_vVw;Or4|in(rBt(X8W^lV$bgYB-vNbJ}3saJf3x##JkUZFWI;l7r` z+YYz*)RB^}C-Jrn>vp&-@45BD(Y#z5ImiM%knTI~xPjfQ<; zpbC9s{h~-w14U6P4HTsU4iDeU>sxz$E8?fR08>@}iOe(Q$%t6vH`H?f=7=Vdj2o80 z-8`@*Cizgo)RNMrX~i-)Za%|pBel)|D03t!R3;M?V@qRR zN=AiDfgMkn#e1OZA?mWA&*Zn)E5x5K1Fxa-94I;5b4+5v&%Lk^|7$6kbQuOl69Qw$ zL0tif6jp_W5)5y9IV%OE4F<6xFT$oMen~4B!Z30v?9#l_iIX}bl57-ZpH$G_j-iJg zO-C;R%1Gj6>;OIB^&4AxJDli-Rg@*zyg}^c&ym;3PoLYM!o8|YywY6MIi24P@40$> z?u+>2vmbG5^t=7p&%&EM`o-%X!`qk8#>^b#FTg?m1r#xt@SYyTI;zIDJ#PAQa6%El zY)`lxHUpb&Ns?_`h>cPWQEjo4qG_lRG(VcNGKRq=AYN30pe7N?04JqpQsn{Q`xUEk5S&6|Q*0EbF#Jg6X}rpx`5+1;o7dk6 zc9jw#*=TI|`o}Pf{fS|x#+^+vV48?Y9I3EQl2gO=dYX_zEOtxKl`P5mBA78GGDAtC>RFfebjXsh zeg-r|;+Dnv8a@hCEe3dfk=PDBO2fqh6l8yk6rF|J#$mt3=cR-2QD`|Rn@ZYKbpAR+ zF3WuyQHE>2?b|Q;F?{=pzXCbEuX8C+ zkx~3HX9Qmz7{%|%DE>rB**8`F`7yz;56L_ClfvrcE>@fF99KsS3TwAyMr(hDHl5rC z?+L(z=)br2dC0uvF_;yrm|(>Vtyt!*gKrwZI2JEgH3NVHW9ekw0X|!W6A@C_BzQu` zZvqik`U#skDkADVIdY%{e9;c#GIWz9P}sarQAki7{;(-9s!EA{KM+0kihx#tpZyrT zc$H}HS~~#Uahx%!jfx$@vvW@SQVE0m6+bGDxB)H1D}V`TyA^84*5SYCP>aetaI3dg z8Ta564l*Z$WJqkBe5VDng=8X=jvtn^i}q%P%bkx4BZj>Z>?6W=3#G@}93s$^%Ae#j z0O>Gy;uiKkb4g`NY;bX>z&I9aOpS)_8<#n9n&tK{H~xh!>;ns?bqP*LimZrt$os99 zn@*xD;=9f|JMScKpb)M+Bzr-D6J8s04c^S|UD+sEJRg=R?|Gz>@5tuG_UGW*37J<~ zaA3Lsej;^(C9EtpF{iPB<(8a%35OSpymAI@ZwkwxnI$4<3tgF28Fu-pGg$QLWjG^i zS?81FLXmsW9yD(833aDNRXBs{F}x?oP^AvG#LEov%=d6$fQfP+cY@+V2%?X24vphbfC1>Mr%0~#zYkykr`LbZ)$s=(gnxe+Dt!5BK6hWf z9=1=vdHvjd?)GQDc>RRc^Cuic{4N;b4BUi|l$-F)5A~TRN|cGF&)XWd^|Z|aqX}us zEN;F~aOTlK#f!oZk}~t#eaEyf$LI-zu%D7&&`I`{LdKAIay^!*T{1vsAe>Zpd@P{X zUe!tFF?|PTQkO^`2LCU|L^!wRm-1l%!)ooXF!-8=!2qBzkG+F^ zDn|{}wYXct^-j@%I0j#b1FdKE-_V1VD0$r)Y>Z0>dTP`lYH$}$h4xqu2YL3SJsmyU zwlH5^yuBML(1Bcg0vc%Y_iKsTx!O(t)LYyh)wr!K48jw-*X^!F$>u4M0_9_Gf5~kq zYx#8z2xDACsdNWm$b85<2IJ2Sg1}PFbf;^!tMLVHqxb#^y>=JYYhq80+H_JlV?LcV zBqLnKw3fGbskh`fnp#@dR-z_%RM^jba$f@%o1X$a+XpDG+3a6kbQPu^CkFR;S3frM9nf2K&l!R?q{%@)>fkL zvg9%}J%iovnJ9)LUhKBL~vc98g1ko6yc^+fNGT>`Uufom0-HX zm^l&=3dK`r1N5p6B!$`98ud%gqO?cx#`WoUz7zhp90)&o`z*i}>79hsFT)RV^c&V? zgVBF*QHZ2jSTeN*Wdx!}9*JZCBrbJ2bxRx|~*Rwr%<1Bm!AjHQoSz@dz^*nsI1?c4+9TwdpX?Fl#6rA+8}B zkZkg=UKS-wEDjfy7l9?l;&6a{fZ45cc@Py?cL8O?klr!PjVQPs^4WwUFTf56kjz}t zm6trtKS-BdH;yF`aoLZzYq;?Tw!@0K&6KKJPa!VMq4)=)kZKjMVu8OR?zjb#MP9?{ zwpKLqxYb(b>hTrUo9;gc(08Q&{AGtBA>s`SoK+SIl-BXtrlYA`D5DZ_=S01n3o1T~ zg{|B#JA*Dsy>Yg`Gf|kc!VJSgOof?@CD^K&)MCX#9K1A5NF5#K1AV1b;S7M~u#Uv) z%Q>xW2mqk>w$cDOLYc#&Q3Ivr1GfqOH9@z*7MPwO{HNY2%DTDVVH1U(!5nJjbs7X| z-eDXPs5*PLRo|N#oJ-esYyn7u6Kg}FLsUhAQ@ly~_Rx1bv|9191_SV-(M%`YLLdm( zFO*2V|1XESJW~Ay#UHJJ{Os+E@Ma^&442Sh;< zdU8FM4_5&5a`cmsN>nP2v%1o@1+ZSx;EjR{XwR?q03_B+*+9nbm$Q zDoW+u9KDsLn7Y)ebJfgyKtymfxD;x_A-DGDl}QUaEE1KSFxX+p#_+DeP(H&F4}9U@r>Fd2Rk_h*tV&84RKXTdW)5Ei#@m= z!y|VwSASERSwj6d(qvPvP;T{m0NMEmG#4+R>ePUU0-6U!&}+cd4z+NEa(D1@RIWrR zLDRsQ6ROZEa1wk8V~N(kLfP5nZIZ3Xu}ajGSFCFjpFpY%q7RusM@o|l`s_M+aM_eglOZ*c%C9Wo{O9V{E6dd7^*?Jv zknvKlASx(l>??bKZG&F*`H2F<@0GtrD!7X-M=2TroqtMwNNrC8&56TzI963Ovd5wd zy8{?`QoG+m$qr^w`HFnlCxXA0#&yFNY^Ws@_2hFeP$q9F z1dbK~%H)7W(9LH`X*pvmivrP}L&F>0ojHrwx)O?9@QSjsaF~MOk!Mgv+%bDDC-wF{ zs4SvvU0iZS;#Ubbcdp>|Oig5Ss<{SK1-RDg`+IYZn+g(HhLbzI1NUfIL+9dv5{~l` z2=cS3J764YWeJ!}a6iruu2OM5OuiQGUJIOq2hJ6H$;J;SakddvVrv78ECtxRifE=2 zl~X+L)PCF|Z0i@^Ax0X|QEx+0|1K?YtkIRygIFr}t$DCY>~#XT_9eeQ8`Da^0w?=y z5;HHWscf_$3BIUuEd~VkoMa|LOQDZieLHnt?N7R`E&$W7B^p)8#@g34z*E$XA;KW{ zBJtq_Wr7AeGEFzmj$55PC;;JGc{% z>-~HYrDK1a` zEhPr&`vWO0Y+(q#GrpEfDm52^$oqLmS1l6RYS+jrZz_r-AG;0WKd@9y~GX=GP$Yw_4%#&+w~w`$$XPZ+@6d+n;Hf z`ud|h7X6R;>;C~-x&IANe);R9sm;G}1BHiJ+-(=#ew-!5$e(cMXcWwTHEPM<_oz1S zpV3OK_B<@)moNbq>+F8@_G@ej1(G^dRv|oyTRAziUs?gFx9Wj5HOtgs06?f8hFaBX z>wC~j&uVb^K<&FoUf;4lC5in~}nYp?$&xeCE| zvwC=SDKFp`4jVUhhHIocZY~$;`Zbc0RUj#FfGP}li3jQ;@*&Aj%^i1bTw9KJSNR@e zN&zL{WG*laVtZ##*xiChTAWsvtH49g~|9FDo@@;IwW0VIVj>j zR+TBtT*EF4nsJT!PzkCDFa&^~sYoWZm2z#sS%@47t0&n>YCU6vuI+xMod@N4v1YW( zdFKVnE0>}cdVN40;BOhI+|_w;FV#B+ul*@qr2R3%K@}dGJOkLM)yQ{OUQv|Uk}lJ( zH&baNkPk1Y2$22kQwb*QmOrXgc}Rs|mvoFI(NxiEWr_2!;k0skom}}4lzTe@W|`RX!?qKSdiYN3zFyMKQ3EsCh_rKTgttD^!7V`4Bvj@ z^!3l*K6`ryD~i9oeVM=K-P>20z#t#^%%5QO`A&YpyZj(O4|3N36<)uBJ&|33+qR!m zo&LhY>&}PL9$!;3)`z(now7u(Maj+0T3%)tq@uF0J|R08G0Gd?M=E{Yc006o3y#D_ zO+P?YcIB&E28f5aF4sVdG_$fW$e0w2fpTuB@>j~0&T7?VAv*SOJSf)@$K0Zg@J)lQ z1H=4?R?rQc2PgM}k4p6wiQa($NFF5cRCeg~4AN2Stp6EJEg+Bi5qklKf4T6N6#@m3 ztB^@eh5GY>Za%>a)J2gX$kq=#^Xz>Ws7aynofNWg<5G>l0bw!pEpsO`DNgBklVAhZqy2Ir=NdYeKqF`zoU1aS6 zRD$-{06##$zkjppYu1<_OaD4<;U+gRnt?{nj-}dLvq8A(Nk8YBIG1Ll13Ma?N9+nt zRtChU_N2LUX2_Hy%vBmStaEwe+7)vol3m$V)QC(Cy9q_7QT_{(N6`A@}-vuJu1bG6gJgQa^Np6xW z_duG^ptpnB2PRCmU86_}DxmuSbUva;F!CC9bEOLjZ7>Jm9a*aOv(IJ*{R>FnfSL@* z?I8;+*^_}UKTCzlFD#iSkSMj8n4EX1!e);A06U=YVNtkPUm(4=g{NWw40A|CHA|8v zhz{Cj1kPdIT_;*;sMzT82%%2U=77Ym8Vr-1&mob# zjt0uO0VoC0B;yZ9B&ZTnHDQ=DkAuop7|Z2X^xg2k?5XNk6b$>-E7Q(C@c#evZ}^u! zp)+u31Ig*xYr>cCJgK`u>7|(@k@e_^tF6N-8OK`(&Yd7Ka93JCfOK-HudZOULaj8( zO_xK;y4z_$;WuqA>X{Aj0((3NE+8^y^r}4tRsh}Pvc4bER|N{7+2Td@ShWmpYtr67+J_W^d;QLO^aI zED#u{BClgZ_RBdSQZ5cx(Ty4;b|B53cTSc-mcrouN+5(xXTPU1wI3Nq=BAS5rSjYc zQfd}}yqAL?g~W4s-R{CGVD^O%^SmGV3u;4B$-_4q27m=_PJ5ey?Jg}>VTE1MT3c7_ z98y8o0>CsAv5XOwitT~t#dSELRbPj6^L;m*wI2g@d7u%waaPu<2FM9IY?1VBXqZ;X zwsRlIf8lSxx96nazWwRHqpkMax8H}u7kw7qK839OZ^P>s7;}9dUO&gW<98TI*vWY| zN}O11wf>9{(uI9Teh4cO00T*ldSoH5-Tq156^R1RSlqagZNi6SCaT%nC!i>TOmA6d zG2{=<8azqUvfJ;O*w3NI)m()71IFErbe;%%CZ&(SuNI-`il+uv)VQ+47(Rcs4=}?rQm{K4kVS z31;T*$}K@+bRaCIUQ^)y7?6KEA&RM?8@Ix*cvHiSV|VAApnBwe5v__)v${LsM8Ww< zan=eiBCA|3_ZycdZ5}rqCts@nOBs5k6v+`U=$8qsd8i)}fZU+ALM+GqDf$R%8`r+j zb@Y8g*PwhY+JW5e9&Sv?pY}L#xFdJVO&W(bUDRfE;`aBUo#GByxiuhg+x{XdLXjoX zRgrA@&{eRp1xDLE*I;Nj{lQA8LGe78ZZ7-qy5ZPS&g|on-xAa!%Y#>y^cd-YOGICm z4<%uoHvkXIn*-G^sOxJxPSC5vB)cM190XQCLR1#bcjagvl4_w%^ti)_3!I_U8#XS* zEHQtnU6NDhy+N6W!`!i113g+-&}FL@VAo-)I6|rimrE~I0Y+}P!M0{)LfLV4N+d)g zF~AMf`*p_ma65RJ!Sib-;sacr4fES_o4C4&lVYkI?i~SrAZIaO@%CuM!d%0ud!Sxv zPpLMlKrz@uK=1)FKVr-wT~B6YO$>89p9A3ag72%ayP{XXi9?s#Mv!Iss*b$Dgi2KlJ*sUoMG_6#GKPT1LF`O%` zn1|->!nuO{Gzg^)TOg$XrTE4XQ-{B9pm#0A-?8PD^USY6V+~kR+#GO+set^UDwgJ~ z3Z!B1uvx={=R%39dBF0CWj|_w8q#HFhuNrmweIZf)M_0+8?1gBRQld4?7`^_6W+*s zbCWQa&W`OtQ|}l<>_bu{$tGNq{mj^7_`EFC1ICbATBdO_!0b3oK)XT|2BaBSK3SbP z<_W3;$E9+irpBR4RqhQmElTcNq0o`x_ zf&tiK1xSan%f;6PFaoqgkmJj^SJ|Xf!0Zaz;t~jey6Ix=)W=}^w040)X|nB63&JM; z(f{SAP@LI20+}#sAHhy8^^-eYz+glo(*OyiqEV1}PkhcIeSloJcHQ1k{1KWSw%Hx( zFkJ^5_llNSO7@q)p^*ZTq^bj~d?tjeUiBUBL5*C@A{F%-1ubjk-w?|Pi@A%unNn3G z?u#o6xkDHhFwP;AI-F9ja4+dpsj?p3ljKeA5^e-9eWe+#DI8TLP`~OiKjdp=ceKEw}z7Z^q|jLz+RADvh`%8LI+3w1sJsEv3>SrL{`)4!DgS8&uSt8lv+f zj7XGcbshN5$f;^9(HIr|mt@uad4Ki*p!}7s7@Rig$KhZ41WNCpQQr8WXNmRhBjqgZ zGmb}mK5fkc25F!*<-NJUK3o1|YwQ80HjANqoL9LDxPHD%Vw7^rFiiK*8WJ57oznNo zDzcP?jl46%Z_YSH7#>}873fn!aL^a@)vtq;O$2>!^>bXy8Zb;^3 zy^m-{`Z+}~D)M1CGm`s6Szd?4fRRBRcTB}040NK@4AKI3$7m##tTY?e6R5Osfafcf z8Asq)M%CkVC2x4z$;Cuv5Y7x$1{74>X7p_5eM-apsN;BkA8yhMah!6Ka&{vQ{U#S@xkBST zKTK{Ra|5&sr8sm>_~q9%q=TjsiI>ZwmZ%YVj++{YZQkq%m3@~~bJIyjC5Q{UAxK;W z^bq1T5t1k;{cJ+om}%icA@y=-&%24xMOo@nmKCllbP45T+ILcDf_M>a&5Id~k0Rx2 zk)UE<2`WGdgE+Tv+RK$nUf85c!!;qgszUi3QdA3=4 zmo@4><&5M1e*2W7n{edF{v6Nb`0Z`pP4m!Qo6Cm!lS)q&a^Ya!N80vO@_?l1 zOi((y1dH2v3oA4Nwv4(jyY_^H#g`XMya367LMMoNsdpMAbS@x@mLDfqBAeAr8~vH8 z)n+s1{24IWNrZ%6m%4bYSl(HfJ>dS(+DXAB(e{B&|6%QFV+f_^EFrI~4iY2{b*2orLU8q~>4*?xEtrP+%tHZ+9iCTI);p{lkT=)dqj$X3Von=To7TOa@;(Pn1aP;H5?_Y z3(zYea5$xo(KdYjzkl=hrwD=kJC~nP1-ESB38)fAE>_q|T@VScR5BAM%aA$PI@ZOP zQ2RtAP{dT?l{zfqP4f3B7BkLrBmx-&H!Co|7rysF4zr(x*UwI0|JU$pv>xE)vAnl; z23)1&0XJU*z{{-qpFNxc<8Vl(&HmB^Y~#SA48t6)qv1pu`|yVf8|yu4q4^~kF9Vp$ zhtbmWOdkk#uz#YNmfTSi5HPPiKr2;;GiybJQDyK$&>cmc1<17>0ZsJDF@uP1D={6J z54#mSshvKD84|WbXDOeucB(F_ix0S)a{x6%p_-O?XonLdhxJg8!j<*4`rvUOJV8;f znrx~pk6Pn|!4dCj&JG@pc?V&!2q9pXr#BN4Jyta+6@$_q!wRwylW+6$kOWjZD}I4R zFe~Qn0GYYn311o`LgbGwoFIc4a%?Le~f<9A!HB`mJ<)c(x=vbK1#Xn27Bl*&F8DT?1ojP$BYjke?=O zvz0p??EB;ZOEB@}3O{3h;w~N7rb(upb}u0nhMEV9eBgOf)lUkzro?R9Op{VS!4DiV zb=+#U*mCv0)6CJ(3TarNnU++7v4$@tE>Jh_DXOi2nnDU}cknm^50sD8{WK>Hy>s*7 zD#>-D{)Cip8x^L)e3Q_TFxHS9Pw-_>R&=C0;uc#$-xDk_^2}pUcVtSgt`}_Yv{tMsFK{;2{U$V5`F_pU%)R%9+%FphA_t!5Z z;1VZ#S5KuRS5xjeu!*w@ex21n*P^ML8*2b{@R6EnK*O$ev$ho~N_LsdtG?%Z+yoR1 zn3J8pY?7c&!~(4&$#WV8XfVlnwFo!>aSnrv*vfT^YT6~ii z4v}}pKTIkm8m4`8H$`3J^-|o%sxj+g`wSj4mrrVW3DCF&$3AZbDps^EK7%0xt}OY8 zbbWEzJRkx67?{E?amGKYwKdu8VKGeV0QkX$hf(U|2A)*OMVbI^-}acUyDP%g6ZmGc zoL^d7_6b557cTG=faPPlK6K83zlDMG=$oqCTku>ftVJC)FHv#T36N_RAWov2_o#|e z`QBuf=pmLsWu#3f;F00H1d=E!X-qr%O!Ts7djnk3#};g`+or|9oYDJK#$eaJ0Duf{6!`La`$hfQqerTXdIO2!&K6 ze$Js^to8+pUyuG*jAq}xeyk7vP52*9Uw{1e$ME(Egzf)HF%l|p|4*l{|DW*sqtmmw z$f=dOc8Mq5NkKQa*}df-%2Tki`jLu&1s3PDt*ZI&TS|C-S(sQy-1W*A;8+4CAM=PM zTaKV>cv2Q*4ema%ObB8Yav@E;k6>r@|Bymqq-nWEVeSHPMlZnK>AEJz*d;?$EeZqp z;5Ov*m>_pU__oeZ?dz~~Pf#eWYK2N1exr2Vx7YyCu&?|;;>P>^R{IVv#^X!VFZV z-5=9hsfIf-;458u01b$fb3ZFc2cprF+rsIjqh!!Wyut5T%Igs~id?NaU(hjeJM~k%!7c5 zpvp2tq_@Ej=XkT7D;^d6A_<&hf)O3lKHNv7M0<%CM!3jyhEl9E-0D~WN)-cRURyAL zt8;IvHKW%pO=%MAc^FT%PDJd2tR*1!Rvy*qI-f2> z75pi<_Sq5IG3S1!K}k;%jqjjF8O9DGpVQ7WDu{GFw*%*JXSo#^?^6dQ#)BcG#>wWC zK;f7gED0e44aP5!+?BkOE2T`J698cl?hCE9&A}7RUF&FVcsX$XD?fpeRBdGD%QV!a z?8u+gEG~c7Qn07c3fF~~BscEJz9N(u;9e9SYr_CJ9UK!NRW+yoDg5gr2JQ7%{20Fd z1l>m7QF|`q(SSeG!R{aPv+v&i08WmUqJIb~P-m6v{}JB)e0od+;CYYP=2sex-xHjE zzz}x3Sud=FeYp*aV)uP~q~@s}8+J)aKyDCGgOxrQog8FA#uCQoJ@=uD$^|aFqpAeV z3dz>qMNDIx^QMk+PRW=NPpAS4(2RPXSkEu@l$q^-F0Iaw6vS`zCZsbHJPiiKTx{Bw zO((Nd7(t!x7secM31HN#mm7)nSW>$)ucA20<}eIs7!32Br8Or(!vlJKX+9(~0iq{& zwZ$S0lnjkh#8SJjeDT@?kBf1hkVbNddDj93tS` z?O+uX>_wOoam+a{s)PKRfAE828T$cy`p-PRzJ45D|8UBN+n3>ve%sLb4;M*Q>=`7K zXs_PFUivv=A9Ca!+oY&@5{82W+m=aor9X@uq9!WTkQR*m{9f zQ82S#e>T8-J~)eA=>Xkn-X@hmmt0@bV}4$x*50IcqW1`9zW8}N2wGfQ~$n_gkIHIlw{tLL39MLg#kvy(~^{ z35T8B2#Uzg>5y*(H#{`(XXeax@PWEQ*<#HHQm?{LPIdjIz-^(eZ-sn&DbxWV#WT#o zIIn>ua9SXovOwbB;5#>{chEr;iwkG21IT8^QSnA7h~^CScqLA4P+-c2vFtWX1gx#Z zA72J@6PT?kzYO8_U;#TQe8(2H=ln-a^GORpFF3FrSV4aWF32XUF27N!HQ8-w3wsD} zS$l02H2p22%T0tZ#=&Xgrl*wGTB&h0A4(ZdS3^81)1it7AAM320Gr~gY*gE>2=s3H zL=5__Um~|e=Ub?L`{%bm!5&HvPz3*3hY{~ElrWbQ z1p0UR-*9}OpM7!N!$&T2E<}@Y4i7uMI9>UmRo|m~V1Ki!(R+@!nT3(PjV#~?$C+j( z;NYqm5>H3?C~)B0CN&%9XX64*x6^X8p@%}lE~5$6scxm-b5Tbr6iCxRsiMXngbhQ? zjW{AuhdGOAuo*F+H(0-J4GG!>hQk~`qfM1=U8VRdmdjH5li3a-t%3Xi)okfiWgj7t zYS2J??Q$YTi`g#Tm4+V5d6y;64xLrh(vr;NoC@UV{*h&81S{rC?6pR9|#XeKg6;)LaQ^^Wex$BSC#Gcntf+gI^4H{UDmFk z(Ke{6EWV@f;^aW?Tz4?MVNJv-D$tB=l89~_m+OkY9L{Wqn0Nt@;q=UKA<y)fHsigvRr*x4!E!u{az;Mp>02%wJLGe-P3C1hefFbO<0|KHxoiDyp+tx1d`O0 zIZdNZpx{-NMpa%0l0gD-?W4I0=ku)o#N#1WL{J~0@$WHaF zhk!SXad%rQHLKE$tVUVflcBR|m>s2A0Y%Fp5sK=D*gxZglweL62i%hl zNU8j|i?7T;QY;65&<<7tN$+txaWRCqX(kE3=mC|}@V~1T0Hn-4TeiuIIP$LSN=piUBSYK;^ho?I}uBR20f|TC5A) zH*%HPL7esHGvm^Gu&7i%;7pPFwgKG3EgrgCZvX_ann=P)5Nvc>0dYC`l5*!6zB)YI) zfX`OgS_}WdrN2RUER~2s7pS_a0H~E#j0>`f1DU)vKGk8oOV!#R1V}Op7Y4UC!-oO( z&amv|;~DBnBfPr+m%U^ijwsRC?uK(1xpYbD(1HD#;mrV^vA3H`E-4OSk7bb=#t=J= zS)%h4-R2(E-rYrmN=UbN94wZq7oa!;I5Ai-oput&K+k4v1#Jgc!Kg#=PX@+H;!xDOz=&=2R317M zn55d|x4JhDfQ@hmulCWyZ*$1|L;~I0SNRwL@}K_t-|#Pe!nF4O;q}*F|F^f#3|nTY zciYY4F}9FGQ31)2L!;Ek18l*5A4fct1v#tH zY4R#s;(%$80=Hl*HEH(^^qWBfytdjasPMS*Uk1S zZQM2ZJ11&@1j?Eq)7l7*!vFZe2N^8;9ccah@a>C3zrPK% zM&Bar-wCgJp*_cA02?C!RAf&8k0-GW%4#a(BWXlGNEDZ=6C5u3f~-*Xv%rK|XX=&J zD{;m!p?DOX4LqRB$%i+Ngi!>_EnjaH0baN&nmma$=49`a}w6oW=HpEtI;8rGB&fV@YwqW6u%D-mK9-YX@Xqv(%Pb*&gV`B{cc0CTKsl~3K>smvO;W8_hq6u~Ua_a64#N#6SlUkui1m>A=7lTg5s>dnmv9r? zhYTJocRSWi>W#x~GVY^&Z0w{|uqnj=XG&))pS2F*974iqx01uB!gdm6qcwqj8dT}t z!IOPrN|mGAFrMTr(=9|<7NrBkJ7|Vh-0pW+rn}}{H!~Q|^81apoZz!aDOJ4r#`5Z- zE_#nLcyQ7_wJWH&I>6_?4+>|{rOgkKw-u`eMlvB_m>@+(rICQFh7KlAc9R=X{vY-M z7;P(zF^6N->{H__g-xr}X@*5fWniSX68NE{Z!XabH*l`=u~Hf$)f{1Zb*bTvZAw8> zR@ibAK2&SDhf-xR3@13YUWPMF1Dn|?%<|i)Ox>s^j|381I#cBR;d?*KgBhXDDcN8Y zB{g;)=V0TA1d??YxRa*BpjZ$Vx)gUy5D5;}>EMKs8fWlS4OoXO*&N6vw-HH|0}+Lk z(D?;BV{U}XLP1k>-gC!Zp==vmBJW|M!HNz}uD*= zO1KhP^$tj1v=X`4)&pL>Sl2*Xm;prl+fZExN?h~~kC&7)V>C@7o@DqZtSM2Hvzkz< zw1gtHHL%MDV3EE7*Tm)*P`Q&gpxfZF4c$pr>X#azJxnt&K|O(im(Qi~nSafX^xFP) z{><-RzobL#Pxi0>b$I*Jd$VeL4&tl7eEa9{_0L~FdHrer94L9e2R|>o{sfi(@7`2+ zRX#|j!>d6BUkn>ol2Rm~hCf3`0GBMN;x{uxNiDE^yY(3=x~2 zNPAeP);Q5w1P>Z`VdF?rj%f#DWy-3cbf5Uk<#mAHFK`NWdeAa7?<15K?M$=gCZ}tl zTmgfRI@F(_swk!^@Ws{H3vPg!Spbhy#RuR*LqQMMfF&rrZDDpz3`ieMD&T5-Fi1VP zND=DMVW5#XpuM#pR>~=qup|G^R0#)j=y5lOBbtHy8!QZ9eR+V6&1HQciOsMcd|2#P z9D}_;2~Y)5ACuGaq^aaM8(_VWaQUO5=<)a(78Q5) z6~eZrV3;7s_IPk^Jp@`hL-INxw|meJK!2*PshlXZ4>JmO{uW4$)8v~X1I4<9WuQD6 z_!Y*q!5ez;L@bixZL4@vjVe0VL~JedOXMp=Y_D9l9CBF()xqq==^j|EyKl`cRK}*Z z*=sou24#u3Ggc)MS`{UyA8e0riJ0sro{^OL3Vc(Dkq z;M08E@vVQPO2QrR<44Gk-0+JkyIPa5_Gd*-F3ufP=aZVp3NmGeTjbL8djpqoo`yde?-*gnVlIoGPjK!5 znW87BPa`UNVvYxNWA^omQ`qgj)Mv{=BUD?H+%S$tMq6MT|7`Rfu=0`P3c8R$(>SMX zR5x|{goS8PXLt!@62oakIy8H?hWWvE|66*C{m?qyw{PNw&I9NY=3(U?Q@;Idhj7-J zHpCM41aMP40_@e{kmbA7Ju`XmEyjJWI;VR{*ZsMvI-Zs9H3n#BE3&UiZpLZTMsjj) zN18BgM_t7<*PukLJLwRMNL|lMjS@GmRb#9ycMevsYzJ^FV}%#J7W4R*KTM9S z@wu$17MvaejOYO(dKpl$BR<<(4v|nG@_e#)QQu@1@bE2^^E)yM+d2h&`ofOXsI%qP zCIRD`Bl3zIdZAbm1mYA{Q75{E4dnv9KW9D7<~MAW)kfW@7Mk}12yxtZrgPv|;&N*# zWgS`6mUAF&MA}JePhlS9jD&J1OZDX9)JNX9hb|f086qPVB!M1WISGP*8EtN3Hx^h~ zzzf+!{8oWs__EKtpq$SqHRgi&=*Nt#y7m}2Hm`yL#Ir}AdGhrsHh6WOoqG2y=Vk$Oe_ z(iP^v_NOji|M2yzV9S(6YSzlvB?`pD8g6k_p-Fv%XZX{E83dCBcg?6?xNc3OgHb!Y z$T??Ov`C3*i2z{1vbk#wV=5S{Y9DgE4Q1gGs5%_t9%asu9`JH_H)qKLj$@p)+@zEl zoXfzewYpkOqvXR4o+H<$BccQrxW2-~SREz~J8_YtBWO9;Ay~+m3GKFyGo=10Q#4^- zF46kw6QgN>u^@QN7x>t~>5%xlagh4}v_9hdB&GUNHH!33 zX(BsCXG5;V)i&6ARVpy;rWiNHew&^sGHO1O3(Zc?b%I5WSxOFde^6?^#tlNYs9X#v z9y@72%$PE#oGF}7&A0XoSz8^>D-k&&P#9d zNsyrq78b#@i*u6;dpL}vg|LXL>7?YaXfZ+LGj4+!4n@3ofrF6}*&cxmWK!C$IlY7f zA0|npo5B?cM3TFK2I-m#E2G)7k@ODMEND@NJsZ_tD{wtxX?xssXAG?(u7*p!`uu;L zoPL0+E-U2OLxmMaHVi&U0SiEG==88hSSprNd;p}ApQ`hwo)17Cc$dF(vef&x-wWT<|Mu+j z8yY5j{`M0aAptdW7%Tna?T>H2k$-<0-ab1$LyFmPU8xuuzz*REpwz28t<`UPK*+|L zl=4U*Q8Az_GJCpXMPtHvKr1!-Av{C#BXCKe;yDn5)AEwPhua{R`pquV^(Cy+riqHv z@dX}0tmF`wJM*eo$_($YenXc7d7=Yl-jdH2VPNwU&C?y%y(>NKh?ylqDc>6DyxlD) zN(9U)xN9rcE`;YuGD=v~HoY%uA$D(v05elf+Pz+;QUd!f!aZU6MPXDZ>^s>~D#`)u z$qlb@I;pAk?rAcnd=?-B$9YX7HAiH^uuEvbbiL(7UJ5(}+=JM3s0axM^6m*Xx~bD; zCEWhh-2$~au((BfbFB9ix*gvr)#;)lr?Aaa7va#E{pE0>uw0&;b8e(c9TRohZw;nM z5G%!A)DVrU>jqy3&>hopH+)6=c6Lgn^s1$Vd6-6UU(%qRg~&*Lw3CGpFF|07<>q9e z{tP=eD&+MQe3_$);03E-9nsg|?^C}y+U9fvT8BWVTg;3e@FK=rbh@Y#{Icv^Wp#1i zxxw&%4wQz2AfSpD<={VQ8|WsR=iveEx$(l`Pzj%*#zImm2f%`%=XBxQB-h@EipSCB zQ2ZS!5S7k|*@oDF=rS4TnEp8YM$H+N(GttSHvl#1l5i(=ba`;%DN?TmUrxZ`PE=piOsCGn*J|x9&cFn^52!Is?+M6n6#WMg&j~ie% z?S}O-F; z#-n#{|NQz9RZQ_ke|Y_f&;HfF;jf>{zeUMO;e)@)k5unkE#TiWW9c~nl&gH$4gJ<> zOT(@w0f*{s%ctvw4EGl}vgHV`OP(F+t^%Mt?j6JdE0G>aSrP6~fMCTRd~cfrMCHEk z006v%?Km463E9?s0KwQuKLU+;2{oBg-^N|$P+T0F=E} zI)J_bC-l@S{Y2i~Isv|poHuc4i(>r&IM?D*dmu;JZ6MJc?h7t%-O2wcM0{>`Ve1ap z5-^YJsEW0id3EwOtgP~(%*0pQF5x^CmI3BvC&)@Z!2NG^xf+#lau1U%fL3}CLl=S= zv0}yujfR-$2`-#P-c%PZAIb%2m&$ceb(!lxOU-LA2c~9*Eds4G(XX|8NXJw&KPojt zoRsN8E)J~?^-Xq-xc%1Rb?1kIsrzo>ksz2j%tYWL7M zAGkj`D}zryOx$ptqekGqk|PP)7i@|pRw{Pe@j0o##DfP7c4DU#IJ-;XYAhqvS##u{ z4;U{P4A4Sj5~~0T3|)CN*fuNti{^nMMRicH z_5o7c?t)do1HlDA(BL~rSPe235K6HmlFGCsZ#x17kdLZW$MySpwN&vrKZb8Vk-y@1 zFwe|;;+r zDN===hBZ#l_;kSu2)iPx!Y#g+Lbq70mg{TyMh(Otv1jikVh_7+78Yk#PNE8K*Bnzi zY^w<^0*$pd0`goFxNn_$nZTo)F^e-N^2yYOb~KbT?l{+*mOn$V*N& zd25|-Hd?rYVj$<=MqJQ-03OSW_YS9mYX^Yr$!)@#SPS+#qD=9s2h>XWCbKwDf(w+e zp|L*PxLW{IY@#^dG(6xuV(-gE{ply&{KgsVrr2**SDcmhOtd{p|8peyAiY36WA>#y z9O9E3-NV*yUkG0&{dX7pN>Z}R6}noGPI8A65(Ir=P%n2G@k)jN^I?M39MM5##3n2r z1PFRLS9?uXWhE)p-V`&=r14)$EoF2yD`g3Mb5!QL<%qMF>lhrhd|WF9e11fB z7BygKa}99O)Z&_*NIfaxlPvwfMe@Lt8;^KIPuZVw5W^{er@kd1hDdxYQmf1=cM&Pn zE7jVP2z{KJa@UwQ;JC*&;z{_Jy5#7n=Wa6)X*M#_>wgaa?jX(KUd3gH` zyu>~URA_nk_LsNMj!lCkk^C9RxcoSm{(shhzY_nt1qO5Y?_vqjtH8>mZf1CucN^dJ zdvk#q)8Hyv)qpWg%V$=)VrBD5n%e-EjCGs@YpF(Rdeoze8k^B)EWN635B=m38^hL9 zHe(=K73u(({-*v$v?Y`_L$#X4CF&}0k?084A9xm0(t?l)D@DV>z(Lw!^uA}7A3!h3 z9#DpI52;eUxQ;6*{>lL2$R_1AbI~5Y9;Qc!W$|_ioiCW7%Y-C0&I1AJP$wuIPi0T3 zzgU=2uGlldOc>=FQw}usUy!71r-0nnB^#iO!1~YpfxtGXFVotRn5Py|U(Dokl5(?5@CQh>ns8bE7k&{e^%Y3{-+zIvWfq z_TJUByD5!w=0cQ@#fG)#;>IU17L%$=3j3Pur>2*FT`D#S`CFN~9U%opZi2+G14nz; zRD-TpOHu&dIL68M1QS$B%yv@FkEGGjI99caDq2=R#*K{qRq#>#xN#o^iJz!YlvA~x&n^Rt_5fj#!Vy&! z3W@!X3sTo&6CtJ8gwjbS@-;1Fze`m?bol|af@d-dCJbo{SD>Nxu38}@$(C|q$$3|m z83Ykds`aZ!f(6c_c=PF~MTismpMO@`CYROi{{w}3^+$G}zCEdGp2 z%zjQK9qX(Rp-F;!3BfyE9omKQZ^FN|vHZucpNF?Us++>w?-jZEDUQGSVaNsxIh_B( zb-eO-{o~umLH_w9$L!%Mf5FY-Hz=6At5tRjB4&0k=EqLzyMt8nh^G~FIrD~u@a^P^ zGi8dd>8wh5d*cY*j+R6Pa2>Q6Jz<5s7f&h-lggdVw8VP=g=r*+45OE7-|s+Mz~jse zoE}FuH`qhk3K@ceYQ&2q~)7IRsZECA@$R zjxo3nWNeS66dt~EMZR*6t0yriYhT- zU6S27hye|^!5^-J*M8XPU8wy9ykDOMESI{Tm=zaWn@?v0C+IPAS!Fn z85Y0gTxM<;v6r9St@w;_hWzL^)3}qoDwU z3jyn=z7xLxeZ#1}d;9A3&+pHTo`a&Y^RWU)jOvT&!#cxW7?Hu@c8`2j?Ews`9Z*u2 zi?(5>0Ii!I8fL&K-G^Z!(bvW)OV(eQlNy`LB2ijdrDE~w>d#q32&0m}h#SNb344HFED8fb}jitN~+B=)#2R}1yc z!HtsQ7mn5|7v%`+Dlp@VfK{T-dUQe@=2F-09PbRlyxpn!BCUv_2!`US5;_*nu@k`Z zgdzXoDq*G{h8>1VMb-%D&=qG()s0|lu=-kyh2fhdlG}h}*XxEr3t5ZUmHb(@2#6PTUdi`2K)wc3 zZQX8IZWgOTYsGAqr+RoSgIe$@cA}c%g~(GhQ_^U_0qRz-6WfG+h<+JHpgyqLsA;-+ z0Uw9>(n-zGxIR)jv4t3#^Kfu)Iyb8MWQBT%P-zhzyEco_FmKX%BKN_y^2kUfh-f*z zdaR^X1@)geA5cn@nmaVDRZXZ6rGz_UF`%2+Nuu(oJVO_rSshJ=Ly705VboNop;(`$ zGd9JuTo7FRl$aD;R@^+iWlFHicS)=g6GsiK2dDb;IZ>On#7UzKCt`4~2@N25wi04&f@LUkC7@?}z%mJ}BDMTG`{CJ%=S2}LG= zbFlr2=d)62Srg~6|Lb4)G4Npj?5D54`1bhUpE!N}^KbrM7tCOd9Z6&F-hRqez}Nrh z>+gg7^SPvgK*`r9Tp7|sGQuqHM}8k)I6%sa-h3XkHO&b3JEmx0U~#bv-GdcFqoHl^ zp;m=Uw&d&CT;CVzN1U*rXn>u+1)uVR3~-c4Y{^eTy~LE$5^nj#Ev^(RZHOw0wqldVAI^PNz5EF^&K!%$Rlp zmMb*?O|dse_!jr`!$zy2)}*2&(yUtRr5f$D{IZC=dzuam$PWWNh_e1!J>YU(c1?$y zRLYbLZg3VXsf;?(10^BuUdf|`DUccL0VZnmNjk5Z;*yKn6~o6hd%w-IQ4xTF%||V? zh~+g7SVLDPzEifiF>~T5%IXZZOLWmn&n45^qf>V!N=is{EAQzJw^ECIBTe9`L*OS; z6+%8c0RZZga<8o|_uF{kj#QP?3!WL?MEea=-Wlo$kOTG!*t zS!d5JSPO;guB5ba9+`)L&C~d^70*;sD%5!8Z zCXl7GZzXLtmcn8+ph*YW)SiH}k}J7YYf0ue7uNRi$3&t!Na>H^?H4@$!|Ts?^Jagb z?Z_n@6y$eM+=Ec;tS@x)8IS%gdu-$~m)AtC%o=0Cn>G(4{|X7{SZ&s;Pc26XF58P8 z4{l+@&X0fs1)nYa&rxa59HZd`Cno@dMrVh?j`ZNO5$93`uG0MApuCR=trpxtT{g&( zG^N-Uq!Hai*>YP9anki7G58*O4rL_!blwbLHe<2&MN-iXlr&T_yL3d;Nvcov<}5YM z^7xxeu9V|hP9Vh8B#b0iR%2^=O_kgsu?_`q1zi^m@&WqewlwI^K{^_YcoJqhTw!4- zRHYABs8HY%#(XEg;Cg5~xqb(V30-!lz8d#>B!>+E2WVr^7)a_>J4ZN5E>JJmNFFI% z;8$dT#xv}epD8=PEu4I5|IiN3H-Jsl=HV376!?Notme+x$-#QlLaTIl+wSst4sNv! z-Fl=$x}~K%b@{Q`?-%^o&J;!A7thFwPKPj|bRn#%>cTUN7_GY3K$G*5~XXcV(UQy1K~(e?rpexa5Z|U_*tMHK>VQg zM74!t%TA=_KrmYYha!e`Ike%Iv3l^^*@HUcc-ua%j*|k|CeAp zhbOFw4V&|fV?7w|d&2^a-b5G`cdW9MJx~Q?i+vUPkn72Cqz8u(i|6N38i-7u9jGI5 z>3t1@G(3RGRuBq}<1{hTEj2t^@?mU70B1m$zn`?sE3QO`|G+T3qyrN+(}?uFhWozc zSg1=c@P6!l3M5ra10m;YapWiBmlFb9vTBaAnvU9oj)$;DtR(ms z02Az@ZZ+h2=+~T=2eYqcMit;93vJNp%LspjUM=q&?SpiA8ZS04l^4}me?Sge#auNi zE{pkh?Uoj2Z$)V{Gt@u# zLmCToxc!G-n!8fbNSGZqV|JSN|hgYQZ-e`Is zn6>GyOF=ofsPxNy7wMF0jiSu(3h0^mTKrP{3oz|1Yoq3rF*r0r0ZY=Z+<;fK0$xtM z(aUd;)H3;0RPYiHYN^g=IeQlvWYRTyrumPO7occqu%m4aHzPsTT{zrj>QBT`Pz@(G z?vizS^uqK|e4iKKf>bx?;;XB2YC5Lu*c-2f?8mzh9P2lse9U>VBh8LWyfrWIQR9JoxL)P#(Cn!Sjw02a0qA0v za9dmZg5XNg@g8;~PNli4@r8kZR25ZK>_Lvsaroo6KZn;ZPFri3vKO$t_MjT(ei%5W zK}mJ&uUDymBKc9vzybj0RqbQ)BXqW{&)DUK+kjs8OZo5|mNofi3A2?2rLtl&)j+`Y zaHNKm(B#Uc4q?PR413MoylQKW8v(7Ab({`5z3HyfE3GHV!J1LC1mIpP5ojMI$ok(uPT zb&6cD`TP`~8yLQ*Wsbl{tTOD+9nq=kFF0dJ3^EqW@acA#dyhEpoUsll+}Dz3l7m_{ zjpK$nJj~>|Z$*x@z`MwGICOBVvmUJH+^d5_1xM$ux6(nyd<_h0%C+GHrBwCu>{&;d zHmMHLG+nqmkW6=WD`o7~EYd-0e;2B`<$1^^h3;DF6yf3=|3-RXG)s}IB>9_fapQ0S zgIYE{7pzl&ul9%qk`eYBmjlT*VWiC{h0DqX8CIYXM-Ix)%?p+%Kt`JXCT!rXhLr1J za-}oq=2(gZ5FUTasY0BR=kaU-brxoUOu-=9wk4>wa3bwl7M|mCDx|C_LTYxB8ssf!oKb*&U zdpU9s)KNJfjoHfA%FQ13%md)0_N`JNY-F)w&Q%gpo)LlyHDH{Mh0^=pN5X&g_U=fI z`jP}-}C+)s@n*gDXT{pZkUCKRS>>y`C6+>K0EZCJi-_IdiE9Zx6(J6$( zMRg_!m<9q;v8!xSxLQbcSW5Oi$}%#CXPjm)&4{!@xQ$i5#iH=4&Fv&d(n8KGi{@Hw zXHMFKDf7aj>mZf%Ji7U4#@^;*xwm#_5D7BfCKc<5MWESlV{FmD;|9b-@(w>-V>Ruo z8i~UekVF5%5Zc(4jaddXbbyzQ+&7OAUJ4D6O((?X#Gy38!{vmo?(m+dgJ`gjFi4J4 zRb|YS?HSd^ajMeL#xr0XlY9XY>`>sM>2iF!Eo3j}9ZD{6jk9WVgNo&JGEF87t@HEf zByC<%s!$8DZCBq^*uOQIDU-$Yf2z#E-Nt<;{zB7C?4d9DWEeQ)5e2-%!y;Dq zzy3N5U;pQC{@%*$C&}X599rl%hdUsV2G)F}aMK-RhrZK7ybeZ8Vj%7)_7UVf0#^DZ zYz)*;bhq=g>^hGsft`;AGodUOSDjRNWW(>m?0FwUE9f1>9((GHxr!YlJHmud1+`SN zz+Jv)sO_Jr@XWGaWnj==oZn9l(sps(SwgJZI%g@%N6xexCG;t@GFF}+4gl!+fYve_ zy?v?yC*@jcOb@5IN*QOOI#BEw?8QRfpxQN@u{+?5LpLP6@FivJIG}OJ)zv}?de!lr z4%&)i9cXL>xdUuu;9N#z11v*SRm>fuFr|+4&Uw&pL_Qzal}X^KeF`cw{%RJ zXDwY|%e1hd0{7i5KaU z$dI80m-e-&Qzl^< zn8i_8#o-CdJydhi(Uo>Vjb*{^=p+Roc+bmQ-o1?faG`5FLro}c0_B3%z^AXTOye%P zTMiHd!747^o4Ydl=ahw7E|`W{uv4$&ZXA4sJ4p-5;qDaXi!I+AMXqm&{2;A&2h-?9 z9kXPaFbq0vb^%7&Zb#=VVqMEw`0Bxcw7>@YouQR`K;ouxFtb6g6NDo4u&GHkuFpWi z-MID;1;D6hTp7v70e}Xs7S;7=;FzF<5?LHFk{eejW7KW|2M|U@*;|KE^y@Nd`Fk+X zSk-uqM9ej8#g1<61}rb2WahwZR2eQ`l5IMqOm}u%htkRlTTs1O+fJ?t?aD9i?uiz~ zkg=*Id2xRcWG{Kobcg9+iw||h=i9zmWg&QLU4bUJ5`Kfy_r_pt^|Am^H zT}U~V28B3uE!eSNSnSGII5#Jcs3Ro)DvpMr7`0*O=?{BPo#%tiV|s!5zWiIS@?kn) zInz0~hL*e?1X@p9E=eq!ZgS)BK{G3?Y43R@4*~G$I+F{A;1rJ0o};A18uo2&o(sr? z{H$hNe5%`aXJ2avsI3{qkb1NcJjJ zZ}ove?#}YYJcfZAqkodo;(T+>KzQ4;kVLN`G2o=$c&ChSMx=@z!~ieq7*Z;XZzAlX zU6dhjzXT20r8z~6`Sa)~KVPj*e*4aMTt$W-EdfZL_~i9R9E<#EfA*tq{@y)K^RvHr z`{~qV^yhoodD~o8Zh^4(N1E^)8ci5F6UsRKNwRI1$x!`wmlZNnrxu{Bv{9u;^ZnvHy z0_`YG^hS)`qO`U7q~wZHZh5v4wrh)kam$VOJl!M%u2M!?k9AXfYzoTVvRo?_V1T_k z-5|Zt07A!s*|iDJr(K>`e23F;dgv`mh~%;ZREc5n(7$TUbx`z`P@gYAaQvk?)!7F0 z)0E|p7`Pki0k=*K_-4W%b3Ns45a!sk)n{ky?3|XUkPYab6A!0Vvc1RWW_ilnhLyT~ zTDO8FU>CL)1APZXw^?=L61vu$+g%zyi*Xsq0;C#~1%MyUL1V6pK!04gNrqF8u7#wJ zps@#rbhB^nlwPTkez6?T4q9duE(wXHJr{X$iW%}p)L6rWavhYVUIoLX!Wn?TkZAHD zrzjJVadJ6_r!#y2Ot-jyrP!nNUh|H&AU5EMDY;zo8>=BApe5@%;fA}IKqkyo;q4fk}-C#ecE4PwL| zFuypiEwxhKEHF1<$!LY`(pn`VpgNP<69J1)egnL!q~*Oiikd1=U5macnNo6Gz}sRQ#HA^kI|4YmsB zNGW^xo9n=O-{Vd2&8Z#CU4c_F0h=wS7G(YgRm|cp6eFs0<{X zU4eeI$vA%2YWeRfEk^}Bh6LeE;)G$1yy}?%CBLw*?=R9H||J#8fIPAWmniUE|EP@K9!{bj6ki?Kv_FLOGBmj?KuFZ ztw~rf063oDxNG3O%Mf;5lnUL#G)O0a9!XJnn+r#RpXr*AuzXw!W`cg%sKr@gYer#S zbOse>b^Yw#ghMyN+7{|nmbD$c@BOgI^mIQuLo4y z0Bn!^s$wXL4R2Jw%CpvF>i6BMQP0T-QkfilFrfyiCzS(dpbyAU5VkiGuZ<0?L9b_3 zJ2Hk-oE=jL^uZ)~E~5a;j!)Vt!1*@xAFOZl_k2J6t3AVfNyUIKP2({S+2203_Ta1V z?I-eAL2}?XP$3A!0IG}Bk8*w)_A%xIU{XR(U~nzfgIYtmykiPJYEsEtC5AnQ2%Mk2dF z?zaqTWWf-RsFR37sd1&NcEgUYa~{9)pX~jI8j%ZJK(LqsY&Qs&8T=$pZBREcd)V|5 zf-o*^aYb<FR7bw5nlNN>Yhd%f4y_P0Ha6wO!W3{vi!!v8P=yuk* zEz?1VF7;prk=ueAU*hT5l(aIC@v6{H3D`=__1KX^GRvhNMDi2V?B5Q)#gNACNjqW9d`T$7HS_DK2b&(qh_dr0R|I} zbDP$?7(lI^xDB0a3-u+MgtmEF0MM}eM~STY=nvub*WY&c`dKO`WX0jf@b+wOTKPu%4EuV~7xuDa`HaYrAm zBe-OrT7>}VHp4SM0vVFnO~l<`dzqAzwMv=iKSk1rTsNmhZ3gVia07SRn6wF6c*FTA z^*eVJA#QuBK{vZnQlL|^!SH|peQc|*fB4N0u|w#Fdhq!Z2%JZW8*ifLfrPyIO2vR$ zZRPsBNR9NGo7Chvw!4IwMdQs&xe*>GTyd(ChkZ!`-e%!abMZy>7=urMg5MFkQ;-7M z1p+2#_A6)$0Ix)b&jKuo)I@>4!+k?F|6VS$;bi8*8sl^qx{Ew@@ZNqT3b;tXT9tI< z(2Y(l;n_#DYWIuGp~N(T!ck^d^d0L!eQJ0CcJTttFTKS;6~bh~2NSK@;PlCL)NsdR z6Jccmd?BbeADTOaB~jckk!(OyG|Vy!{IQ0%vy%a>HCreEQE5)WVxoin_k;v%2Y)}lt>`)Oj z9D`$SCvl3?xwDF5aj}XCP*ea-E2pLRNf6XF$V`#~b(Jb>j?DC>}yWt zr0Dzpa#D>Qh_K>PolRV$L>JUj#ZmR>QO=|KNOTyN%XgwzY_>Cjs7RB@%GVuj=q=U) z8fn@r(yCnX?*&d#ZT7KPP&sjz+rYk6UfW*LFAb)`bvzYCiQ7^6upSga!IB2gO+MzW z3vgR7#tm?EFieZPCrA$^LFHKKSzO*Yptl zITRc}J7Ja|UVodPd-wLU@b+7kcYl2s^us6FWa=N^K9-L^4PXDmH$Q~Ul=X^IuJ3Tt zM?>Vx1H}MHmZ_vE$8ZgI>2&Ye3V^C#2-D=kV}rb!%i1MKSqYQ~7dFyK4pp8Ld{K8h(ph!%PLupnVAl+5vK9c7 ztuGc^LH$wjtNX-oD|d)G@K6~Fc3{q&9pzUSUR)Q$1CpJL$dmM%a_$d|oM<6&9Du|K znI31)1a}k+q@Kx5ii)`8fJH4XjT||c_vD@PnlPGYp9)?S4g+_unSQ4mbl82uI5a+b z;zyt)UNc@wB^;^n-e=2K;PXa46=vH17*w{oI_*|NREkde33dQqwwZulHP z+d!hEPT&uo-Uu(45G4prL}cU;l~ZKpT#=YMIhefHX9q&k4OPMZ6@ZUec;H567k zS>26g#-eD&TU;3vcbmb-shQPtuLXK0JR4%nhd>j)&^-|>rT%W=!(kCKjN>I9799pu z{25I(a(q-No|9uygWFX_f1q*;>Jf&tB$lgLjkO(8fI;`hl?}91B47{`OLr)WYETUY z+#2)qB6f$n5ZFhh6rl@bD`_#awri(W?8vzj*P-RHbY9cj3Z5@%<1ik&j^(!a9uxB< zmCE5cZs&P-Pu5USzz`cbt4_f+%~B3@?ONIj(pMBU!!V;x6ikti5^IRsg;BtzLWi{~ zcwEY9un%3)e?esdEFhx!0j9B{R~5<2jBEd7=xp995jE>fotY3h7%8{is$H#9UFNns z{5I74h$@D_&9su1rM90?7OgL-&0PZ}&REZZQOj2gNfR5Gt4d-MtMPeK&r-aNwa;UB zJrvAGx%-qkn1gOBzb3ZIcFzyMU9}+APfd7}We_M-UR|j~v5YekE_Dw#V1|ln@e4J{i4op_7N;3LPX`f72WME{+hIEFdyJ z91||r4k6%3SC%iNgKNv9ZMn`PRG*joD$l8+Y)Y0zpIcW!lq+R?Y6W^#wUg>{g$P!D zR%u=KpcnD^8di_iIr#64jUW!It{(=Hr5m%3b6(7IpH$vmGzswZ z%}IbZRw={mt6I3MTcEELmm{s-Nh?FbK85uZ*oK!gH7>#qRC(lxr+CJY;ZFH93ld_E5p9u08Z1>nC5D5US+`dQi77&P%y9Eyu=6_sp+jqQ2>_NpB+}`y z!_LCXtX`$9m6!{X^epj$MjZnzbl6;@QG=jLA2yPhcDz|sK5+o*{~*=(Po&b88eB4t zp8UY{eWBR?>+nDSr+jt@u>biVE-wHb?T&n@jm=wdR3m(qTE|I0%kYGdNf$RaXdMxd*DH~UxgZo-yqx6)je~Pz=X_=5<5mH;Rh0aJ zGD0)C&!gazS3wJ_rJo)Ia&l}3vD2uPBk1(!xt(-w3%0bZktWCkbMP^#bsUlvK?qYi z<&3<x-~e%Eryt+2 zYQC^kF$46lY_H8la0`ewUe+!Jm=311f;5{@!!NJPRW|D%lb&(Ws zpx`svN$S|a{>L0$qP4oG`>Y*VrE^LWbUH_(xSQaf-M}?(b2O=S^c~B=Z>{APv7Cfq zrlb&f^fpI-b?p=xcqb6Etv84SkWg?1Y|VmN(q3hR!77v!EdDnJMF)=?!xDLa5aUhd zo;%XviZ!ytYEoMf^oRC5^_1=*)*Cd`aZ?ABxR|G7hyRx2ep)I(8z>Hw9|OwXULqI+ zq%G;_jf_;u=f<P=fM4`eC~}?MW^&RpO3U4cOr=D>+Gql&k>m#|dyOab&;X#v#2?;?#%&bL0{=DJ&}i zW~rn0dcUk?=3=yt7F@HltsT2a*)q4cRvD;KvbW)Lddj~n-NPkKba0ktrwrXg`9SYb z2l;(Eyz#Xtrjv;9!Kbn_E_4oV;I_&w-y9!umHQ5~#b%H!e9&PaDL@^R&aLg0U*5M0 zfkgRg@BX+42F+3WBjnIuScIP46a;bGv~WB9B$5ql)KeyZqmXE{sl6WV)N$_l=}G*S z7GXBci9s>ZIc)C`KdvtQCbepJ*r<+}NUnG91WzBL+VfpC8=@i;egm%Ya&rX?rpT)D)r>5xy&vRiLKlYlBp}2Z!E5E zBWz))+tQ+iRkVAy(2ExLB^dVVTMwDnWj!#KDrrtmQK)9>h}973ph6&$Z@~^Btg1I! zS30q_K#mlN@Z4j{Hy?_v=}l@zDe$F`;3E>zN1b_EpzhrqBvm!`(6+mQH8y^)Z@O{w z655P=>_EoPN&r-jLVhI03OjW=)|A`aWmd&NU*~c=S*Xg}(@$L{;oD=_*8^&%k^weC z0FJ$`iC)3wLN%j83TufG&(lx_`#qm9V$`jZ?f~LM5wl$I5oHVbD?`5-E(yjEefT0T zO3p5t_|W+bM>#{2wnfzv7Y17xd0S5f)O!s_w3mVMvJ9kRUE*Gp?7ca+xb9v%JBb05~Sh_6}X0@c%Ey#X@?IfL$p?q>^rda_G z?dT`*F0diH5gXI!t<3QfN-$LrkUPp=*esz#ZoGXM$)yKQ@Z=yc7d5tKRK|^05R-3| zv76)ZiG2b+FDRp9FC@1~w-d}Fo!i`w0qikHMP|HvAf9rhS1nfcfY3vTTG>{r`c3@+ zCDyh?ii}&)9Ri0aUf_&6Iy(kJv%KA*l@vG>1h%Ec*ro}IN+|rjKa!ARbHLV%&Jflk~G=1*I?(7YsF z$#e3mqP0@Zt`Uw^@)nFv(_oNmH#LAJfB-k?u+-I(9PrCaruHd9(PO#eaG-+S zt^9d6D9eUgf~qYmN`=L>6al9(eBp#@)q;B+q}{?7|9f3}3-;YkdGz@h;35m~@MAhc z`R_3uv6$|UzJK=dYc4kb`TbYn;|I_hUue@Sb+x6io$R<=MpI}5B{V)If~}tiHRmyc zKd}(|@Mxe!ZLw=x%W4s@@?3yH zUn#Ha3m_6e-0tIpC=$TM={B*GA}i5)T!Vx37Xb4lTn{7gvvkE;bgzU#iTzX?`cfnH zh$X4KV;kc$&;=5nC6%G|q~7d6!2newH>zs;dW{nOT8VI|zf+Pg*c+Nno;poh1Tlb> zg6d6bU_u)`9t+Tur^>W}PUT@5*hfIDEV5uM!(FL7UFhd!XKT~IGD`}Kre+H^K&&6y zvC@vDE9NM7EH1BHjY_Rj#avQ@na-9Ew3YJiG^Mdyv}zw`&gm=MfxHzIdE&Z)H)LkY z|H&s((%rEew{`)9HP)uK+C_jT!zb7QRix*1)G)n4nsT>ry}bn~F_Ov3`nzUQ+~>6b zc-Gp!F_(gSVc)WWwe2}kw8qXT`BFDHAs_Aw^sO5YDf$M|jy9MFj0)_vPqHX;r||S*{Dp z7nSNe>u%k}Mhaq~+$l~%RP)uv42ruAcHlBS*^r@V&E=2aq#@ds*0nMA^%clTFm9ofB>KvZKqYq8I&pP)1X?7XlK0 zz^~yi9{?uh%kciI%Qv6D|1rFOt#ncyO8?5ru$L4d0QMN=2|Z*q=~>myIwXAnx#Yk^ z8tcle*j1T}(kn2G!}bT)h(-u2wjtq~FFcG$bV70(|I3A8Ji`q^V{mn02xDO4^Gq&d zpnJ-!BJA1*XeC|M8f6kCT`;^N_8beJjTR@^kwJuBit`iA2*AYsTXaaJGMO^L9XsBc z%ha~-!+|!3z^tc=1mvv!lPV#;T$V$SIlllflafOO0l?zmb?UZWHCywUT(;6)JAP3A z?(D-5iOh(^0l(9JHVbR%d9g5BsZsBmKv=SNQtd7+cB)nJZIJOsPMTK0}P4Hq>DoH(KZ=fssU~qB5QTfbxrwOYKOeEzN>o)>7+B=A;2uP5H9f=NEv^1H!K0A1jR;G7sNS6y9`e| ze6q>e>}+LSD(+gfcfhGr2cZQn`GQ(;)R-PAF#wE;m)c#af-jAvEAt|R$Ed|!>8~DR zOP1*(t&zMtH#<+OqFt6d1z!O!wRntG>tX$*Q*T_RPlmgUb#b7aR@|-iLKyF8`Cw=5 zd}8z`iK}v>T~w?jKY~%<;8PW}Hp8Woy!6;@T^e)7$UguR1t^8@oSp0?nlGsdN~w_z zWhifUzCq)Uf@_cLSKq!R4Euh4(*PzWEWxDf0D;@bRS-+;MHK+AbvRR~53a zgJcH!DLxxDS(_#W(kP}b2U*mtRn1yz!@9yKJ~|}nxJB3C(fb;1eM+SY%Kst-in(B{ z3Nw960M~#XmAfL z+l&i&`IhaVgm1ZU6wg@Esa&O%7HfM{T_E_jWSIiUq3&cViT_xo$n{{Xu8hj*M|eoA ztn(`+eotJucGsYiNOmeUOLov@fB>N!Ey$u2J{n&4q4#C$9rx{nY?meUhpYRXsstdg zR{yKq0CxAz5z}9~Qp8r(E&#D-r9U73ty1dO1JIAC%Hy&XQVE)vjeK4}#w6<8N|>EM zH60)4A?Kjmq;^07uP9;M+{`CK>BuN4+|%K#*C*+2&B0o|@6w`QmmQ$RbXQXy`))^c zFm%GlJPUU}QD9&5F=^y|xZQZrLR*V*OS4hCs!O%KIcSEuKm&yiM15{r6Uoew!i5LR zc6h_zToVGQns(i6=i72Q22esNCB9tziUeh|(%vyMeC6Rwu9zFh5s|onqbVXeb{SjuMb7 zrsx8^%)&{|Tgd(?N8ae6fazR}0q&2tul(mn8>O1{|TpYF3RMj7$GggoZPC{qsmBzxen*zlOhjApgV< z!^f}l&;Q{4U&Fh}`F{E#spc~tfBx~~?0fxVl{ZsI>`x&=`RT`JA3st2O!)W|$eGRy zgjZk`*cW(v%57uHH4$jwnYZKH-nj?d6EMEUZL(D}ee4VN9~lAP0%4OMfiqKt5tI_1 z?2>p$8MCk*DWQYl^K?K#ZCLdPTkU-F2TkfX$qzQE(CwH_4Iq`nHV7KSK!kHBzmCqn zT~QP5*&P)i;UnQaVQvt8&7t}W30nnV@8FTP#_leEeX`uJbyIV2RTh9w3~rCPJ{1yB zj#7EO>3k+UZ$;tLYAG>kWew+!`9Q5s;CDG)X$=nMTuxF_(}*j(b@hV;0weU26B(*{ zqf+apnPc4_%7W7C9X88T!UwV;LTr&@-de~#9t#vX9#`^d;n1s332JFAIF?cu zP%U8vJ05VK%Y7?4au@DX?Fhcz6)~6TvXrLs_ z9R|`U_ZGMgkjYS{Y6ql@0Qh6-7rP{ih^{Jg1zJ}vUJ|qzAg>F37;YYVO#U1#60xSp zM|l{=8X;nutI$`cDp}RW@4QwA|I}%#Pdv<3!fkFqHnPuULBr&whJ;-XoLe8RmIdHg2gwC_U#Xi3Vr-}!v3W%{%J+m13QubXn51}C|48pLdd>Dv!*ZD z#z+J62iqrXC|QYnA8Ys&b6O->(k-qE9eB-DC_<98BlM?ROUX(Pg>!izcaTM` zNvLC>%`|GB1U`|Qen@<{L~rv5-1r0B9XZyTl-=uxI;P?rX$0ak z^S{)<8aFrS0hM)fX_+0J|*{?p)hnAj9nAv>n_>ZC6wvkyfv(`8-nM@6wta>6T&@2-Yg-&(2LRYWmCE}CmzwR5 zVZ}uGn@xfx!~P7{6NLgbL;-<2T2UIOQwud}W};A{JTf*rcD;l9mfy8SdjawsTkEuk0zz{vCb?;m|EK&19QU$dx;rO zhjdk4(iKc1-0K5E+RIJIv{5WDW=NFc{r7(x{`N>2zsK(QFGfedl0;&bPe1thnz`hg z&))xtC4_3<|CHE!$K3z+{U@eeY|tUNxxFk4zEU%|&^lJl)PjG*O8X5^(5IyyRRvps zk7T%#t0=gJnG;MOl7=LaSr~kz;m~<{;6^_pFwh5JXTi_MJ*cJWhZ-_GJoYfp-~(e; zNFMN*yTX%-zQo+exMSyhX92i^RWLZR2sf~edDM$kwI{*TaPcux9CI2dVoMQA`J3A^ zC+ZZV9GxOp1Xrw;S84*yFc7|88Oi1S*yfWnm@b)*1-q7XUoog-*od_LN$72?vvuEA zjhzq{NFJOZYXaWX;Mygq*$su?b;>l{B~YJLWyjD+jDoC;L@BDYtKmj0Y}o__GFLk9 zHw^NyAKN3f@S$*w=QWRHR}FG++8x=CIxn}Oj6_esqx!b0P;u9;aaoeN*q89EF2eQ6 z{XvDbK_1+5j}2OHX7z?;CQ++9@RE?Pl*9^ned3VjK0c9@Jr`hSXaRqA)k^6-fVD(j zau4-})dF0tT5vfQP<^F)DpcV<9hn8qC6S?FC4y&$naARc3`|RPa$#riYT39CfHAe? z_Df9}gc?E$Emy}06Gl^KL7A!4xE8@Uf%519+9>_$4el&Ux+EPrQLZUTox{St+e&OW zSF&}FwH_e)n3$7*+b$|V3-IfK#YcB}y-E+7yq=e^h#dmlwG0K6lEO5siGd_sSQO-` zjEj~*w|Xk}f^1pfoM)X#jmv~tmpu&70o5uauqKpJK%+}XwNgn71|W^9UZVa^IhrS3 zHY=hqc65Be;0>WB^^aRQ$K$ufB7R&Oc{;FHtwV|C*uo zNAmt&Ua)_4oq`wGjKyUi5obn=i~9&=5L$r0p%}C#AnSA-+ZjOE6vLY^f5I<#cK5I&p-0X3wM7c@3o!`7c1oL34utSAXM z3pbDsB!T%zaCu^_?oMT}Wa6$<-E@n{;LSQy7NGLWa$lsnxxlDiQIKiMx^a)27qXdt8)1q2Q8w zJf=f+tQ@p|#e^mwsc1Ln$4z-jsy(+^(N=A0d(Dv`5L}es#fXUygGXyYOzN0QUAB+dN#Cs$ZJf7Q#)krd&FH>Fh*hjVV$|G@uvJE7Yu99cuVhobqvz5hZE zz6f&gYZV&6?o#nQ29lp$?+Y56q@DuL7jC3qJyZJ%9A#%{&l9A%9Hte(YBq@vok!U; zJCx0)$azG31PD%RHfWPS66Q9NMi12*NmLJo&7-eGq0+G4TJl2F^#`V;8gwo+pk^T+ z8Bu8150-LC`%O}`bxl7u#1hrwy zC-a!;HQy4ifoC{bPE*ER+sHy;cfh(AG|N$iMzE3KuEU4an{}e+(YOPkFdC9ak&uj{ z3FbaP+*nE6L%v-Jh$qieDOoWkJX042bi7ia&ut8BMZjV0Q}5Ksp{<&cBQ4a`Plcfr zmwqh{@PEH5*kX5!C3zETm9#*6bsfzcw%6OmA)j}cwo0(CXZrjhtAa|_4vgrSz?3|Z z61w0Xwbhmw!&yi;N~H<}sOf=sNkx`m79w%Vh-~*~_(W$VYtw=-{UEan6SmH`iFdfm z&UVq?!_$;~-)t+R3NTg!j~QRvKoI?+TEC&&Y@1MTxOyzBNuwVIA1~BuFegvLg(ZN4 zaq}kRGY-D0R_TUd|EraYJ(|$>gF~TvuV>4?r_#Qi((FJhKbd+fHPM0cI=YM;#vzfb zYu)5pF|d~HS&QzDY=LIXaH`(Q*iVYf%xBUqsSmZvPaXG#V%(viQt15iK2xwUVaH;j z8z9D^C5>~vx zI^PJyT2{Wze(d6xt1{^8vyu^dMP;1;+% zLOAyf%{cv#VzST+jVX(U zAWP>%7#fvi)Kl^{rvlQH>_Oiz$phorg|U}T)i#SuP(pC0ZhH%i!N$Rm7Z^Is38o?3 zX+jqOuQ^KsNEAV!#7*Uzj#62{?axl3Qnnn+$=2r(kF@#H&&0+M6)q|sA!TNH_cups zH)?ry(oWoof=qCTBV7n{i~6Fncc}G}K_0g*z#PC9lgbRGW&oD+BVqDkHT?U-*4XVT zWp-hKcQh<_+h>FiR=gJs-}A5IZqQn-N%HK1rOIH4euLgra)ji_n5nb9s>P2oOSuAUvuwe-c zFd{SfgU4ThezZE3ONpI`YHy?%k^WkFNYHPoNj+0sdJiIlDwfu~aF@a;Gw{CSB`GoX z(E?)rNV;5b)c>s1;zJn>krOoY3#9EAm)LY;DjJNe(8sTwk#8gLP76MXWlbxKgOFLQJt>`^}@rLV>aM{!a|JSq{etT4#O?ewt^EQ8tzoSU;y52MR*@! zyTJkkUdk0TTig#%hhX_w>7vJ-++cFqoq*`Q5-^DZ*}6l^q8=5JQ>!ZLaD-;4gwVbq_U`f_0^wgZ{4;)@eYLY4vV8ZeoE^&i$4;oj0 zJLK1T?E%$mm++1iHsZs6iSn9_FZnm&Z~o?pum2X_zn{HSB6k@vp_1LEzEaTq|w z9lk{!MXd?HJc)VF2DS>85lRzKE@3Hll}oHTkH&%e=Nn9yS(js` ztmU~C<_m<3H_O~Q-*}9!EG9+K%TZ;BzfeAMa*cchG$MlMwGt2Xchl8CIxv^JybX#GnFJIv8*+G!Y5iihJ#5|)* z6-^xOAhbrHt&6tv-Qi6>Kl_asQ^#&|%95T}Bob}X)qzT6rz^C-5dsMNgTM7b$I6xR zwR1#A!REWsrn|pn^d})7QF^-*Q3&&FS`$6Po3DLp4Gf%uco7LF3YAhj1@mWnWug6$ z$b!8*P#p&Y>6$g%NE%%+C*ry#O$r@eS1+koN~3D2$!Ym~rkYhi&GuAG?`RfU)!M9B z{70-|yk1Erl^{XGd1Qkq-zinx(H`RZ0DA7pQ$Y`B07^i$zZ~kl$j9@%sECtXl)N2E z#kI>_DpyiZA)yk?N@V=@Xo6MdULo3jdWnyMK4&g&%>c`u&eTLVMB9^J%bC8N4vnVmwcc)A(;o$5ek1z%PBd_T?>Hn+EP=dl(Lo?9tJFdIu00QmQt!$ z1=JETK<`j+0X32eMPDK0Q3Vo!Mu&AMo^f{T*HyAU6lv0d^}r33D*?0*)Sg3sS~5Ew|$!l^u8o=g)QA6X2-DNn9qOWfld2H2c* zLIKi{DU_wjOl@lmU`b(z_i>dAy0Anu+t{4?6OvmnTZ36BQ%e_~vpdxGa=TFH$Q17Qy_19~3inYXbfOv-5AApJt=j}Qj(y7YJb&8Ae@hyL% z5o5SFXwA5ab0O~|0I&I^+mdR2QzE8Nr-9Mycwn+@M)F{AmM1xOSDTAW*vTNI+X^o~ zisoz{|H9$O;`{&yuDiVcSz`X04GW%wlMtbLMgOF+DRp6uCudWzxg8k`5zIg ze;+=+zHo>6GjMO9F|f_%Ixw4u!YAyFqlS~j~&p&&pBfC(Q|J}zPYd+bW({dLcCmGYHXxvg%@LW1w>zT&t%o7ag@mf2>`5P3yjSH;viTcn^Rk;emG?2 zzN?t-K-qmzcI-QIgu41{RF^xpXKv~-=$Yl*`ynKpX_;2l1JopUVJl^27FBAtd0}K# z2-XnG%wCjJ%u8kpvtg=^JEQTqsK)3P%Lf9GRB8YzI~RRJJts*5 zkpu5~1R!$&4jiH(SL|$PVs-7XSG?PCe5m8nP!r|-fX+e3A-w8PTvUbfcsP}J{LWQc zcm{az?o^$3)nWR|n%}LH6f6bspRRJTMs0f#dEn+c(=|n|=mHa4ih^z7ekm8Yf?2Q! zKmbq1WCb?&w07x&EX@vO_QSUv@M#!d5$aSztz7OA8WooT#1JQkXz6ZOh|-08!d%R% z^}Y}R4kY9?B}HulF`J|AMi*wKCv*>n=m0Req=j z7m8W=)5I7=`70X`wUv$8Z*Lo7gy?VE9) z)lb^aDmOiRs0}A|^Z+rfXg#MuG!6uC+hbq^F)+)KD(_f1?|uvuMnJQYj4cZQ(Ji5x zzD-!I$C)f5`G+E?mRl9PIzv;CC!9P@x-pET8ZS@`=nrTMIw$gimmWKuLo%j=LaU?* zU%+O=bBSo9fm{~l2h3QoqIKjv3fv9Wh>Nu5?C@1{*Ym{czEANAAluQ5%o(`MdC-Vo z;};whh;PvkYYg?V?>CC5TzO+hq#cJKsnnoef*p2-5zH97pZiI!@<&R;tsN+BB3H>LTcGe+ zuPzK!uI)1p(2fK9hrENuRH(`yuO*3L_Y9`mj7joDdbW$n4roeX$CQWs(9Kl|2YY{^ zQ?E#Lj_7Cq{r3zG)?;$i5k+!|=17|V0XQ*)@JF3#VXvEBmiZQkm_Nj+`H;NH;! zp^;po?IRqkbq*K_an1%m4V`|-yoYDgCtLIouWvjl&NUR_6Hr0!O6o4@HAW*Y-&&`! z1YCo;0;3$oN~Y}6B;nFvV|7$1|3J8j(L|jonyU1MJKfF`8t~5ikb!uS^z-7Jf0kT$ z2;f#}E_P+POJ;~tJ`+9JE4l4d7*$e4b3iYdU@*jI351ayf%oJyeI5C`sdA~-;QUrL z;~<*{#Qa3JLEyqACD?``4&&tO1~@U=Ohulv)MXuM`q;=aX-Izq;TJpc9YdiGYeh@R zN;N5qGj{>Hv0c2g&~;fFC%JkH%$A}f%8Qcr;)QlGNqw0{XjTH}j8;Sch&1h+wfGz^ z8q^MgQTIB1U>HZhN5x_YHmiNQM6XI>1$INQTV1O?7lk!q(I0{x{4PKGU&6;%!1IJx zV?)_J_6o?c5AjWFpvxPk4F=!Q&DXXbCzaPDNE<0x9!D;!d3d7ns*3yui5N*M-I1K! z&HWLcs~wPEv_9cFQC-$eN~s-NiNUW7X~Ol5;S4A%Ro>kS8w6K*>wpG2_GNNu%bgF= z=@aE~*{&>Pq5ML7;YB@q*r8}-KS64M(dU z?GC#&$q9q5H~H?|qDo5UV`Ok92}y?)s5k`6W}b#uf=ztJR_dj~&am?tM#|sd3R_wq z@>m`Kt3OEnmd`MaiVXRMJ9YpnL*VjnG7hrNhu++eXpjHF!)hVbNmgqf@Mo zwGUVjjK~#BQ1cJfsl>8_F?@tj2t7d3Q5_db*j(jn-2^w&!^EMIW)%@y>VCrJrMZb1 z=>tQN11cg`Viyu0PBV%?ss;DAMgWiUkI_Ym<8Ms(^JAFk8`dDVOGrq z46_q@L#=~Rjottg0mLi2&o^!4C}ax9xmH?iewr92p{O{{=Hy{|)m4sE`NfG91MLcV zDU8+NEDnA=%~okFdxff__ihe$9&>t0lx=`p;G=88#lU_dDad4*!`<{smZ=>~+2125 z5Y`q$Q<~KKSURZ7ax;BY)tpR9j$^OkdJ)-j?Zk3AEx2Ppsz#9Sq>BJ0O(4e%&xwSO z(Q5nRP^pI^zVuP_)Keu%%#+qr_?y6cL`y7GS6Vj>qkf=Mz5@m+ z2NqX(8;IrYcObuM%e`YXN&+k3RqfG*$P+jr>mlpyEgCu=!s|gK@2-|LW)Fc`ktU}H z6-n)wD>gOSy4*7>sNn|-kxH17U^~!t`_X?$D0|tf(7E^uDSPhIZ%I8VI&Nxi@FbPJ8pL!X8z-tTa5h} z2O@>s)4pKt1o#YItTd6%N~y5ds$k=L0XtBv2CyTs#|+>82}v`D7k7dAVG)QM8g2@fW#Yjt zEB}6Sg{t^qoP-ihFNs6BGzp_8WH5O<3eNrF5VI5ym1*3IucFf>%tV8Q7kL&HIj}aP z`ihHRocs~0URg5W567eb8a{rR&wc_Z=Pxhc{5W8x{OQMM@4v{xli%0H@y9QNFNlBp z@dtHm{V>JK|MKU5uonAO!Hw?D!4iy*tbTt}cx}!@Kr#R^H_o#a*hYqJVz)YU^bR?` zqG6_jHZ9dBPz4|JmHq43f(xTQbTS#M({nUaIZ?uan&p} zjHT=NhB#l6Wtw%z;1Ya4Qj=w%T|KI?b3T^C>jv30Pf{RDGaU_%3S9tkp@L1{M!?rC zv7oNUB*7=j*ayM~w)I6gR3$Bi%%i4APac3)ahHcsP&S)uNGc===%2w%$kUoL^-@69 zh8v4Y8x7v3l3pl`3=u|Y7NJDE*i1=TK1xA%VRYR_A};3(eNdP<_A46Y8KI$F>+u|N zg_XPhEjq)?Cne;uyjM`c*JV^LRd@k9HalptkEexckBck1d#t{_n=};c1R)WJeZ`Ya z#2Bo^@7g};AhYbKPX=&ZOiW^a>JsmUe;g;0Y zDoDZ9Y8+WwUqkd0z|qZCNQ{;jwZ7bUS*lQTZBK_ue384b%Deq zU_0TYGMR-7sP*WeM>UsPP7J6k`mn>@Sl83VXsdv0Z01(dd3ux-vkCSIrnh*~c}N@P z9rj7Xs)!3}SEx(v>@T+(;UdwR8)c>HEyqs2Y8PPt7avW5GE%lD_DdD<+4bYvK8^^X zd;tPkx7@5+O0l`Bejwsr{=E)8zJB z8nfEwJ4LHr%(mG89fGM@=`1dSs_o)gG4kOgU{8C9Fv~?tVtGETOl9YI%3R+*njd;SGy>%)Ih4LUoIOh?MeaMk2j`nNtQe zSzhR&?2Ubek*WyyP>_}jpI%t3QH1jerDH~7_mN>FpsnMxAg}>2S$*`|4@&cia!EzkAi6PN zhwWkwI;XmgsTd1B1YX}E#fDYtiVyVW>+$)xsid!IG8L_|<-yYSkTC_Rk%IZnrZP!( z0@+f-Ff~4OUF32aAqz$*>*iL_^QekbCW$asV>dNc?SKV-XIxMXvI=Tur1WA+Hs;=C zS&s3^G{TSzi;UI6j|!BTE?ibfC_93(r000Lu311L9u;Nodb+ z39i|Lh#$67hCu+?XcGCx(l>!abV8Jl1eM?Hgx)PQwpGWw8#*%RK|O;4_KGnby=6LsqJ)*jg?a9(!Yw)KZq)=TB6)l2c4qsrN`9E6 z%UB#vaEmG2*hMUEm(ljPcQ0)P?Vg3Eg$Hq0HG(n6OJvp3aTV8KzvCLJ z@{;z$<0V192D~w#2yeER;Q@bC2mz(u1fzqROE zW3}-@d;d`h4A4@Pprac;S4!yxaVP;bN zcg_ZqY&fmp088VVwITQ(_4`$dqJDEkb-#2mK>X-%I6=so0K~K~JU~rQUWUcpLJeV6 z?8^f2+PaSwHVCFaxq;6Y7d3FS%TjLt1Sb?J`fJsXzci_} z5|@fyBk^1s$8Gu09zr`9*#dFK01l&FcGN0p13wr!icm`-1QZqC;I~Wl7D`~1PKa3S zEY<_rU-f&S+&0*SS-4gz)3G8>=>cuDH!bb2EJezBpWHwK0Hr%XHFR` zG(lWQXB|OsHw_EBa})#+)UxiBlgQ+T!;S!Rs*}n-ve})I6lz~zd7IIUPe~lWVcPW> zrv_Zu$=01wssL{&bg86$!wK995fFKp{e4seY0DC)yC2xk3T=zUl>g2C3;Jiy8(9wM zcl;Xu@`2;0e)aJi^I)V%8zeD*9P$^(&3+|)1_Tz>x8x4Se9kDUygj2`uy*93!%RD@ z(#X*)1njL#fnn~pHb%9~52-!LeCX(HJw91UN+VQ$qyqptb8}>aC(c*Fe_#wrX@@E6k&Q|+J zz@P3?5cb?sU_wC*7wQSQ>MxL3ekd_OT9Vq=75Gt!i#k!a^%ta1$IFQ5ctO1=Z~o1w z1J`hZ`3`vN8cJWv764 zw=aQ?2dszrKp$@4BQzuDwtYLl;>0y*uLKJnA<@L-P=eT0`KxsaiLe~Z>+URhneS0v zfC|8;esQO+h^yVZsQT{X(}7MS@Gbnj?vfpxwW~6EKC8Jyj z$k|gw{yNBE3`m6!7q3$#|mo~J6#x^IU5mT;wRK_axVVC#tKZO7B zuk^9M3g7wbBRK!z<6pRZ_#=fiKjQ}cXYxq+_}9xfKT>$?=kGs-Xq$YI5dP=CfB)x< zp_M@X{^iFP-)66U{qbx2?qBnvKZN%$@b!1^e>6E{Xuo}txoqG2_!0tfNCx`zJ64zW z()UsB;&$Sq?D+^CAxpx%@U);E$$hc4x5V1pNDD=n zwNaf2&z^^vnY*PkJWyJxd)5BLcJQS}M2FHsFeJidl)LZJk^0LCcL0nmT0ToK*rrF9 zjl(2e>f^gaD&J)k9bvw6>WgsnGLkE$Nt;HlOC+@v(xns-#4A>GkS6tI#BBW~? zX-kM5P+D6eWa$Pu`XXk4R}3ZN;a}V^p-WMCY?=nF`=}!XR8Gq!SHHPZKtBTZ;{NGy z@gaP+Yt<;3Es}Jn4_%M01pGrEPO2>Xb}7Xx!L-sHO4FQ%@d`o*x7rmP_cnGmSQaa~ z?C5M1gJFlb-q&L%fs6#GH0X=@0bCHr@cRyFt<(7RK2U)2wn^3*B!$$_syY-1Z@N+g zY~@CD%q`R=u{?6NpLc&)^7wqag>Fqdc(#$~S!QJvdmwkUKG{cshGuP}c7GM_m$r<6 z0b^RXz%?W18#QuST!&Du=qggr**s^b6J3UZXm1-+iza6wzf6usP7YEM5;a)BhCy z&;OlgM6&ENT__usz%bI>op)IUuH^}FxCmH+<` z9eQnR0RU^4Nkmz3ML<=cD=?B+=u?)~Q z%McIR0A`0`$fmhRu~t@fdJ1EKD|La=YK#z&o>)MSrIPoAsvE_-^|O1)mglqT*< z6^D3CjwI2o$3;7D(CFAcvD{XYb~dL?r;<~v18Y8rNTYE5*)#2&DkDveJ-E{)nZB=tr#FjblwmAXEJJjSq^y|3yQ05RBW=9PCwy7LMH zO70JPm1`e#D7GdS9z83QLR+upH>Bvb;Q_EyU1+aR9L2OGdD{aNevw=f4@{7LUhnL?<1_n-FaKtr1%3Z!6NPd zA=UN72+Zs92h>DXwy0A5LxrSl>~2yS@gX^!sYfontox+WG!{T5I*<2ayx-~kK5i%A z1;$Dk^JhW5h#tePLEy*W2z0D8GHn~s+Li+U5);%iQ5wJ2p|w&@%b&$2ox{_Yq6<`k zF$A|;bZL9tFLiQZdc<4cFrN{&{(vfijPBWmgYHfQJo-fU)E*B2+7FpiJY`sg)9dAyBA>{jv5~#I#aA=lPx-Dd`QLi4O6?8Mc zR#!i<>QL{+-hOe#C}o7@oUn)T4*iW+G*ZBztyV_M+5?jG6@F!F$3yXLNYWn3&k|S# zWpiU|n1pM0D0A$(C2torJ*8QMNd^KE5{byheZ9Ph)e88&DyQWn0Nu;y(76y4EP;>I zbRxG(f*RBIIM%+se(c=u+|wezR1z4~3@qzgN~?wcdTjqZ_OmRgMyy7~bCrl+TF%RabU*`Uv#nv(ZC&i-vcW(UE5=ZJkGrAn8_i58KcSOjR3jKYh*-Zhe%H!l0pBE<} z+yTt7j({#wT((gugl>GgQc0?+-BeH3xI-2H2~`JMvBb9^@t&UstZ;Zos}znD7t>a_ z`;E$iD@69}7OC@_%aXkVsq#8@tqWvJlEmJSHpvUT*Avzca)B{rN>Kw(Udd=B)6c{7 zOlIu%B{sRSaMJ?lTVv)(G(dvt7L-o5lD$Y33Y%%o5>vw<7!7B&b%7#diVqKW9oHuRcb^o%mC&Y=3YSi5`Z_Yc1149^4c?6CM(Rm0y zv8=zz#zi;m_-7q~J6EOrT?H}Rx0Cl6$i}7QL%NoQ=bUtfBPBGsVVPW2+%aVVsziiN>u`X$IZ~9CK>Aatsv%L;>F}hyn$V#FF7x@z9lYV~oSNeiI z6_ie1j<#Mgc;T=Rf_ee?lfmW5v5b%9Wi6DM8Oc)IszurtSA-b+1LF>- zABdPw+l+x*0NFEVh0W5F^ds0HEiD%>z>&pryn`vXGO)vw6=2$WGb)~9K*IlwPxdg(x4JRRhw4E1Vs+0OXsXk-iKM`)r2vwuS@F3k5Brf+3BjM)q$UkY&=K zL;}a0wQqr~$^aa3T<&C?sNR*7tT(bpP|(cr8ol|Di`OIxF>u1NMN zamrv@W0Bl46G^q?*2C%wYC!%PVo6YM8;ESsy@eDVJC@WFn+OnwFvvOC1k@;)kNF4Q z<_L7K{WH!)x^AJW2_?%dLq3taKa>fBGR|DDu`3S;IOyCOL1e&)Lluk(nN7Wm7^OSb zE(##?t%#y}X(|lxjc@bku^Ci`+6#BE>p1Hgg8ie?4ybxkF*0tbgo(ro-UFa5QA*@S81o)}5Rb#fzB-DRq6kkF9f!ECEtoJZkf$wyTpK)I3~J#sXk{MW zKhNzVS}9#XbAA_O8qGSbLYd~`-wj>AL-6s8VjhjBy9pmJaq?F{t*GnCDCNj8HUqXJU!K^%}%^Ek; zQae87E!X1PM(hPGfN}LsFo`Dx4|APBA@F$xSl;bfr!F{Q`;oNzZdJb^(vxwD!NnSJ zocyqfWMHpP$PZsY2wp7cjaWLE%@O%q^TCTk^e1b zLJiLTNpue-g#y*SMm>M20$xkByiFDx_kh>(vNP!fdD3(wSp+oPrOuL1~IJ~C0%9G_yn?M{xVx4{9gd9Se)P>nZr{93~82q5n6rKD~< zoNxj%h)-r~?Av*(LcaCzIH+66ZzFct64^+gsU?lMa|GbR(wISh`v@e!L)t(IYzyXP zDfr|vc}VaqTk>0BC*&k0bS*V~hNOYq$0azp3ydzgS}?a~uLt8{{yTCP?-VqFqy08h z3te6e92v{|rGjAKsMZ%i_Nl~`4UFb2U<$_3hH-G7I9rsL2d7s41#JXI=L`-&*e{Sn zDaTIv4@?E+h^oBG7DQMv0VynXXtk00o^507r!_;LnvfE-;>;%WP6x}L&{2l``WJ*Ei_HuDtIg(#Jp&$$&f~!56Y9 zPGIknONT~@qc+}xeYQz5;zzkFk~{3aK}WKt9AYfXansp=p8ikrUy%QE+RMuy6-W0) z`|+KxK7Pip;V&QH*ZkA}^6~k{*FnDg-mEsh`TYH7fxe=?qz~_A$)O^=zKoZ!_HAP1 z?#6hK7U~nU(-WJwr}T~i}T8a2n!neVe2F50nt(y&o_u`hyjj|72gXFkUVTuCZavpuQ3=*kC- zBHg4`T*T;eD(xaV>3y6%zm&FhU4Xpc*lbk!D`@McV-vDS@U!lwfuj1u7xsxmjTwlZ z{h}bteK!c$rttY6afvdbPanC*lp>=%nLWp66E;@UmDWQ-lXt6O3 z-zC{xPTV5Nymn)T2l|&3e34Q$+w6;IUFi>!W;ZC+GFwPhje*1=PS!z%Us5c@Y3S+dbh6L0Byl?%pK8>*pT88__4 zeNnLGdPo2Rk4kyoRKbStG^GO01m!=f|61x?iXztvCN8-LLWWxw;*QV_9gZYd`6pgv zroT-}GewOg#jKnjXRNVO2SF`)`@Qc^pxWEwH&v2&RNjp_6uzIh?Jw01VY7_|GZxR- zn^a1aRRu;9eW{Y_zT1^+%Y11zwg_?0uI01U`5)*hE!i#7{uYS5j!G+)P_qxirI6Z_ zV6FDtjsQQThHjQYb{0-Wf>l~>Rl;+YknYIls9QaQ2E;bpT9Yn>^0##H7Z8$-PyR&R zo)5JYLg#Pdh*)V?Nm-6Dg%Lx({mrt&ZL3tBo;FuVp`$JRZ5a3wLjNw1;Br1DO7iH-}_N z5D0xJvW{x5laF;ew4c!ykj2uiS&t9%co6c-%#(|hD-g^*$yrI+`~+-MVSC2%ubQ4-0Rouo7t& z0ctoF#3XMLQi1!vmay~|Y&zFSDMbc*4etobbYOB0^B5M^VX@Xthl<1=H)$qOIdr>atq@si|cl>w8l66dd+n`B`NOqx2iK_PtT zDbd)Cf#L`iOy+e!H4+%o4My+o`D1C?V0!tu8A~NaS{d7@MjPhtC95Wg1w(LOLMk6n zk%_RAw}z{ND^~cQ5|)>_662r@2`Pl(&dTTnDUpE|C{X`1$yQ(}L%E}7ZJ9FVE)2~U zIfhB0uBA>@UT!(J?ok$D5B9u7n7d#G^V1KfA%Hs-UKaZ*SqPeOUMazn_W} z3PD<-jV{qP69LM|u4gSVRH=!HURC!u-^Oy|?Hrf5#o7w*A)X+q%QeTza(TbE)k&pc zL1|E-$VvyFNRs~69V8UpTJ{qDAUSN0KzP4rXbIe-0B@rmDv>q|h;wu47)c;rNPLqs zlpTkBzAk{YHum$ZsbFy@v}8)iV;H^A&3oEm=XUIBB_7=sZ@g~Crab^RECoB*&J&d| zDKhB-$v_BHjxWS|tpKM{-U`R%Nq2rM6#>jtVLuYd2i*up6%7K-6=IxnXP{Yb><1<^ z`=!@`s+9{FRJq>elYbI$j|KXC!p*5aq~kq-zex&{U;N7IPf}hJZ zzs0D!9-olb;kmke=+ZMs+Fr_i4j(a*HWYqBFI^i9=EOzwhVP1w)x;?vSme{`nG7wt zwMsZK$W2b}@-+n>Dl>@3+nY`wTuWjHSl_oXrMkEw)&gfuV+Co0oPw(!#_j6tpt3of zJ)qeF@nlS?;g&*~FI=o#ZUWS)10##I)LrL4Xub4F9?B3&!KYP zW(fRAvlF*fa{JrlepD*yN}*MiTDQ0>znT2A8yli5RV1Y#3c(e_aO}dQ5TurbVLyM& zmj9>6qhDcE@U6!Gi}3#2%Qv3}5XZhR-TxQ=|33+y!O8cXp7!yl_s>rAh8IxPBkk(n z+TbfqcqT5Fr(9aOCVL_P+$BXlp}EnKXsoPbKr!;EN}G{~`;a#=U&0%lY?Mc2q~rlH zk1ns-;YR5oLc<6_#BiZ)WPxNE*LUvgCDpf_$kMi&a-T!SCpswX{{`ycjh~BceZ^EFS9c z7%s~-!1r06FGcja2QZl2o)&O1pnqDff^8v6g9l4D-VV*^qH*`il`r?AA#$>!etzPg zAU?SFr(~AXEjOwCrF^Lq=KnQk28y`=M9jGyBZPKZ(l7^X$(U&L7|PnfOeLGn4P&bk zIvnDY`3kCL_(bKjp-^KKv5jVgnmuBPD%s{Qu-<46@o)Lu<$h{hKI5KDR{#hEpD=`k zaaRrSj)GyBOiDOM=VPb>ChiS7%x!EV3SDa(Ufhx9Z6x1Fp=R@!o7M`Ja4@C9z7z!k z1_6gAw>+tvqz2<@$t^TOsM)1nKoAbTaKT_o3ie~}+=Ncv|4$w)`0Gu5PV5A%{E7@^ zxL-)|%pGY9W&{D10RJiw2M%YgD_*D~vqgWUYME>{dI4DS{m5?IG)F+A@my^J{Xoy| z!0HNjwpp_sbHeI|d&fn$buf5%Hn?#RQiNgdN@({IPb1aS_~4KL{gFq1NxKy&D{_}f z&2K-iW|pPZa;wA&(I~$cAiMTQ9{K&p?=1xw_%gizTmG&Rk23?=NXNjFhS!S8KSqAAkK07%?KegG2Sw1x5apB*Fs;>1~mPo4vm2j+VV z^1LqaxI3DnRUMJ3oCL$;doRG+N2>IwF}71%q0 zyQ0gDF9l)gNTMYsuM$boOdsTa9mneRWn`+;5NaAIlra`e&;?F zZce2*@^s;mxRyIP0#+G)CmD4}lvc=&f+)hV268L}774YwwoZeB*#S3*v@E`ca=o0k zByu%MrkQUci|JTEMH%dL#!45VtMZ@t6KxFw0e10~jzo}liC2`Kq;kHyI{Bz;uf1q_ zAgWF&g6xU%T26s8SSi2)7saKDns$eBbsM}MK%9M12t%$wxfl*9Fr9lG%`r-I4rq1o zL4A>;*;JyB7MSp;*Qs)LYlWPl|08h=R~PWa3e&1A0Hlro49)zCVOmECR+MMOxg>2h zRjljO=^AokLk~Y3s)f=jj+zjE`PIl=fBwh6{F?vPPXeIo`S^?XufqEuR5KzV3jfpl zZ^DORVg4Ix|L=WJaa1Q0tO6YBt)^a89dVU0TqFlY;BJv@OGu}Pye zHRWovtxYW|)*W{u!|3FCoofM?VNh5u%Trq5jfA7x}fieT< z7oL|X|0E1CMV57FEHFK#m|4YWNy;Su96p%$-G6*Kzg8LhQRD>Q5>JBWBCDFby zuJ+DCh1Xdc>Vr^)ISqY?W8IvB*bI7&#M=iqVbc=eJc0(WgaU-S6-XgR{{eXuCaMrg z`h1y0^(`2}c51@Lt)0B4Wxqfs*@DCrhMW9RTF%z8J8#$X`IIzYEHvvKuhJ3N&pKv8 zQ5j`p<*VWg?HgtWtyPSRO26WugES_|Jf&7!AdB}EcHWeUC;&K2tCPyHI)X61lAtsr zt%{IbPM*l1qdzgt!2q={Bn?QE&jBCxi|tSD;LJ$$H$GE4+#y^?UM&n?c^Tg^URh3A zW5%S2()(aH_0szh#t5+*`wif?_LeFj`cQQA7H(DDY;l+`#_;pmFthvCln&SB*eG#n zHCT78o660KZuM*Umhd#B+obfn#}~p9Wr9$KHXKV`qpp*sDI4qIAkV2gvV>A_{4cK zT*`#*54gCiMC8SI7FDV@y;%5UzVCe=J%Z9$WU1G65B0P-T<+&=mS(_|9|;TB;p|#zJ1) zQ?|*7xJ#)oErm=YRXA7}@|)}#MDtc#E^1`iU-nH3y8Jixgxa0lowr`$M-Of@RXJD1 zNGv!{n6Jo7v%xx$(a9zZ&=MZWAOL8b<^bE_6$S$hvX?M}9p~ftq;k+MnTKo2&c@w2 z$vMwsjS<&TC#QMNZjC%w0gPPbqNa|`CSMOpUrh1ggG1Z-s8=C#fWigZ)`5_v^!FPK z0SEI_Lwu1MzVN~SaPZ`Ss9^U8s%!9nH-YQO(mMh5l@Fn_QM39+NjZ??1Yes5p$qYR zgU15o4~CQO%ABqnl>4QWZ>^?)hO~lFEJa{U;$+&P0(F&c^)o;;5AY9hjA|aHILd!n z7H?jpjK|wM5t1M|R0Y*nKt5E#^wa^sYh)qbQZ9H!CvXuEF%OsY@yyi<056+eq;7jZ zEgZ*4wPuTKIes490~i{RJC$c{6PE3;$B}zU-sNMB-NRLUR-vD1oTheIhd z_Eto6Sgxbo6N*2wDb3M&sa=8Uw%Mu8!F^IAmxSf&I|jz^gdoQ? z`2P8Zma>D~Ps3n=!$G|_ZS1X3N@_#FWk(_hsH_dx$Tk{oB_Q;(XzZK`1R1;QbM|(NK{RPuOM;$hy*xMDX9{R=3JoUPWd2aWXw13W-G zCJ9{iNK9ZIyugc(P^yKcO-$yKND8V`N)$>g0CpdCHv_thZ{BtKE?+L+{8womz6(04iRQRw4!`r4-@s3r z1=~Nqf67_jF2x7UYOq?+(!Yl;pyiWJ*~Ss#4p2&k66Ho0-AAN#lnLj^U(MIVM(;5~ zLZp5CR=tHJ(Zk4CkDyg$X#x6_Eb2FP`yIQIvr{SlD`=>AOm|U(CBuUdwIu^$ZUWfA zAQ$E~u-1pi@S>BHg8C1rz2sF(Lp)SE&TGvf8=J5>oL05cr`CguB!o|FW&$doTni2( z#(ijayu@jJccjsg`0T~Y)Vjwo6b)iG(pD04ZkTkq=o+H0DHuzYj?ur>@N#J;hTUo- zy!|QRP@XlVa^l*x1)%FI&vWGtY)+X@UH~w)H=gH^6%96yP_vIlo}kN>iOI-~U1G00 zTe-6`Y{b!HeW~UjR@#^0-oiSHQ}ovh^5%9jqc;a__^_wRY(>O&g}ca`o5Qrc78Q|Q zsV91M4gXI0YZlEVWV|RC3Lvf>R*PCnJ`Nn%J#_60LIst=xsMSMIQaGDh9bG(94X#O z3cg|bHdhe@Bnf&jX0)-;w-^MHq->E37`vjG9P*A5{^VZTWN`xv{VGsoGx#mQw&PM4 zVY*Eu6-~K3K`#KSVYxRzD)QK$Qnv(v*}Yhv0U3w;H&(FIrRuom0kdY0^T{oS^B#QO zXB9Na`q>%?%1RvDSET(^!5!^kDI@XSqquZ}Fq*UNta-db_)skcsLckYq=EDJc^s@! zI(E1-BE1yCORW9p9QRu)Pl`~!1dy&SXwfRl6>%vF@2!gnc~_2@G!&?As3ZnngGE-M z=D&xT6o=>#*Oqn%uEsM)>nFk{Hu|`bte_vH_=$Jci7-eU!D^M;+c^ekWo_>qyJGVJ zRz7jZrjypB(NK+d`b%tYp=$7XAaMI9?_Y!ugK5jr4>1q=DQEJQM!x(0M@bQ%%h%ro zIrtYCZnh_wAfDvpP}8?;4U&I9nvJL0WY`=3Ozk0GjxmwSUoa*=fOIn;rpiV4XO(;m zNy_^$KD*>=zg3!*$$$Z?v=H4W#V|K*E5ruq(Fn<6(q!^xb_$RA=uoP|fDs9XEf63^ zOL(F(2G`=gWXAbHr9|3xnS^^%eLM^gAyo#36@}K`LQ4pe+|D{bKDgLR4`#ckrf!5X zCmFEV)|@s?i$d@Y38DBLvhtM9EOlsgix4M`FkNI*xRem9)}KHj1Ry4nNsok25!^ne zI4EfHL&A*(+ey7hNc5ov?!N@ z6oY&$iC(_mM}XN96RfvHi&7sHb7IN152NiqfN)Pvv}@71cq z(k?Eum%Z_lgrCH^t6yA23}mm4nWIkL$mNAlMKpk&Qhc}CDQDA0G(5z(XVYtuyO)GVFiAGyrwhap0ov_w-PDo* zNf?}XV5+v+bFXTFT_8(9J*8&j&b?UxMruZRksQOh^LNwsz1&NeMxm2cG*t?4;~T$& z8el0Qb9~8n@ea>>W4B48gA~3lhvXn`;Ek%Mpq;LS|giB83n^a0p z6wXqpPZtB9Q#y@Tv5F6m6O!M%xGSt+@{wfN(elVO7+$MFIG&Hx2m1(6Xy)Xgl}ajp zxi@3CFKp*d+p`j%=57n0y@YpNbiGxi>#gI?9Tu<781-!>_>uB}t_v!S-l98}VV`BdHyy3Tbi zh`m+SX5CS_mv(OGG0Zd+zJ%c(a8`T6r8e-*+=`hCpsO=olvWjpuzU?J3c$(vY#T5E zgw&iAh9gkK3;pDDr6if)(4K`-c+?frlr@?w=G=Gy^%v;8rj6+sPV>$|kSzDQEUv?> z`yaP20v-WhTdqn~^%~luIc+JW6(Zkvrp0+t)e2LpKnZO}Di+Z8_V0kCw@Rly&UEZn z4?M{r7dR%uw=d@=bu6yo7N>LpcPNs3WtkkBJG*N&X2K(tBXT~{KG*)9ZdhDQr&s5j zoISv(VTIX2SaxmMzp0gy9S3f40l%^6qg@@E+WbRz9V4jDMBBGm)R<7Bv}{RL6e&N^ z0%7FTDiouz8ZSKw$h!vP4AnMDL|YAE-V@?_RpHO*Oo9dw*h5=eN*-&&8+mdBS1F+y z7+eeax}m((@}C4?>+vvRJU0KB=4Yq%{0LfGi}4+F8J!21799x|94|_`I^k_^#B_jo ziAvO>qg2y&uu(EtjsVV;Vh39qZmzZAlb8*pZhJNdhVvAi;!+q3>}o*o#D+#*9hWi@ za%77$j-+~#FboH5p6aItwiXO`Fn)EPQF&L*aAIn`)l+S*d~BeyVq1ZdFl<1c-5g-C z0{gE)rTOu*(4l0=lE!d$pGZ0BRL_@aqcmxRy){>xykqKSJtamG5vbsH;bLzb%!Zm9ex@h3AnCe*f_m z1z=5g=*REB`}qDhU%r0<9M+c~e|$F;yYIpv;=Au(fAihTH~%+<^IpU>>>Epj-Jk2; ztKW}GzbZedrdbPvFK)pbAL1I2K6G(;s|Izhk zyV6`&n%I3l#jdg&s&c#P0fcr{zlV)(?1_YB%m+`pZyY3>#b{V@#mdVs>op2Pi|*MX>RJR5=}|ukIq$E%yRE`h$QCAPs7;ofadB z%`R5dB~_ciDDGu{R?q{IYjtHv*F}Sk=vD_Nw*ydEYhyKcE3f#J$p$VS3Nmb@B3&7t zr&|pfRaT>!$+QVWm;f^HW4d-vdeWeHUWUz9?Y5iZh8DOG-; z<5D7WVmxj=dr$%?v~Aq`WYydsblzD)Mj3MliHnbOXR z6D~n=+a%CE@sETCDTLBV=W&ihr5si))jA?SpMQr9$orJ>bPWWP#6CXyvtZh8KY#n` z^#er2-@Sb#|9->j`J=aQ!t00fF}!|>^~(Eizf@bDxA*hsfAspR@K<{Hf|?os^>#rt;=3xEg(STv4oYn!$X>J8(-KXBRH~l9DLoNN*zFfx=FJ%0vQRM_{*TtKTN|r zoC(QC#0LN1!Q`@!Ln!z$hui>Vm; zWJYlg=XE2`xrP3y19E3TCtL(ELmfNF(Ze7zhiMe^j=a_J+~k9_4(>-&E#V#tk_U$# zAQmaf5SK-*PMj|Rfj+qipO_2~wy&D$JMHd5&*2ubUz4+~+8RzrCZm9DoqNWGu~T!= zwP9_Kc{yzX0S&IXg8?grq)QO)$^kYzRCiJ-IAZ*?sJD1}1!WOp(U>R>(0A>kv)Zny z9pwfo)}3#+48Dok$dZ6^*U(Ba574$%JW|=qRi+38mYJ8-qZpCO_!7ag$#=+kQin+Q z3|$mmq!t`Bl2*kSE^}1YG&tiWL1osiB9$qNh7(i=&YUuaOoFkK-AAJu6hV_W^S-5* z;=Qm#MX2OsY&$ECae!!;!Qs#6HuE@2*TX|JwV0zNQu5%KYy~@nA3WEWL#2BE&XPuZ z0*J1pv$Y8>Iz5m(r)`dY8g;e{(dlGj*-tL1lGcD;KW08@P=R8FU?lNdYXx3Xmc;aw zCAr@wLq#Yj`B?>0DH#^f=hE~gWeZpR(r;p;^N3E5hve=I0xuV&a%)bE8c{h}sP*xN zL+2DWb5#O9-uC~s(*>_xgbpgLBA|gEAV|Pg+MMQoaa1S$)!S$G(a3+lG7zv%fj)Tq z%3SHbriSTXC}3v+Y%5DQJlkA1zKqZp%gO8xXNqvnm&`*=1@>X1B7=APMhP*vPo<{G zE3^APV!lhvkvn6>QdT}pcK}sDo0FH%a-USlKwMazrm{%VUOJp0QBOi$3oW^+6b_~M zQ2Jn+iQco=H$iIsAz4ff9XzWhI9%GXyv@{1S>)$A40@Jo7#Kv3jsTl8p&J|ng9o}f z_;rd>R^8J>Yl7gIO0I+UPqfKlOtZHIbE0zzb~F&}PlO_brtv*Yy2Fkk(A01OwINRQ zX($SSIdal`Put4Dw%DWYa41WP4jkj8^RPRQ=>~;31>9h}N_JbD93D=jt~IcN+Wjqb z!jbsfLM7PYuR~bY9>C&GF6C2yZq<~6cF#Q~A3M4qzNWE5B2!Ht2@{t!Uc8v?0MfMl zl{ST-*v|nxvZ_aBq-i&;i>*Qi4qE&eHTm$f-5~feQ2%F4PiQ>EYM`-CAR$Cuvf0?k zv+jx8(&CM|Ytt8_KQ(tZ*l8f0&Yout&Dx%tIphsweo$Jd0{%Y`oJo~VO>)8;Rtl5c z(gTxZ?~)=)sR&$|q(OOMLBuq~+3atiiusi(xH&omGr~K>pc-Nt6zm(PdW(G{DX~D% zvwR3e0fUTQDNA_S;6QpY_{im$e9UVhd%RtC<-!YGZm|ro*Yk|AaBArv1g@q};Lbw} z9bBq>qq$J@M^1W}S1f9ykld76+UXV@j7NXqx}3Hx#p}iLdy@Ocv?Efys4y6XMF1?w zCXf zO`f3RBSnDa9m&VeRF(@J1yiTn3YuciSSwnt16i{?#j!?tJSTf)Aq)Iu9@E>AhzPtZ zN;C}Z9ZE*(nvg&Dzx`*L%>DfJH{tEi@MQS$+i$|RpM3XE(9X7tf0IHi-CY@WOR$5t zy#_b}geBRo*!h>@$&Fj3NwoQCRN>1Km>)#SIo4noYEgtC#~WhVu4jUdJAwf4+_ zq+}fs!eRoB#~1vRm4FN@NiFz0GRcq+EUHh5R8p#bWfqg?!a*gr?Kr?n#Uj2mid+Yv z(!JL<*&LO6I1*G$nc#6~0RBa_eDS^dnZ#XRt3SMP(S~Xd_by_z7{PzIYX2c0<`@sl z@W0T-z;tPM`#c}y+I2yVmgnM_v8e>ay?1>h_|w=?gg^gW1;u^J} zJ~VP)blV9^odK#O(gZ|0B!KShwCstB^dODYtcx3|(ZG_%fwH9^)a|P0HLyv;OsZeQ zu2k)_fci>&yf4;CPF=JDC zym=Ti;8ldVwET%@2E}r~Xdlo#G6N;0>2OP$4vl<(64A8)-H}|P3yu=|{@A$N)Bha) zj~op1RO?Uwrt8~J^Y|)%?hmg&^Up~^|MmrjTVK9?n*R;Qlhad;xXqqM0$h%d+H1*; zv?#*s;+i)=aV08l0HXiyniU(z;F(zc1NM2c7IMRUS}uVe(34KCQqPFK#CB`ty|wr# z$2u@O&dcUUema{JZ$3gDqky%>;_^>l&6hIv-wHava=R;c#;5v)9n1YZz46$tZ zd*qWkVhRrS5;qGHzleA$$sy_1X@cSkC^h!T;2NWye4V)kbZRTJxU_nw#?z7rUUs_( z43;oo<9H3Be+at+TF(T~Mu_%ACyqnvissJ4W#2Fd zmlo>Oz@=~Ff=th9H7}?qZYDS2I1pDXPvOoS)WSA{Xxj>|^H0N=Lp~LYEJ-|ep-P5qvo{VGE#MR>PmB*_zB_&? zA4z}`0*0uwu#r~dm39Ve|BzzM`H2mYNE0t_%*8-|a8as$Ru2oppAO^G60&0rG8fc= z*~z6yM1rc6L#jYq(sE)qnBOn&#?>E4KTI^tj zxHRtf9bxnWW1Z*-vv?llYDU&QMfA}5_xf(i zU;3zt^QpIQ-hT7;tB?;)-+mI_ehv$VkH7oTU;Z18C;9u|fBX9N!|-2t;%e9LT1@}{ zd;)ZrFM-j@_l2B&YW`RMCFoDic?osLT+Z*#wY{2&{@~yRPPLbIS?ozzlfaQlZ*+32u#4Ji3K{& zN4t5<%(uN z#YJpa99qeeNse<-l&nNGt8p0G1hu`+32dDdNgx;33X57g{Xme3IUCJa5BEZ)-& zsMH0ZFM!@#Az5kjFO?4Tn2{>ok@TjR4Z&n4KUFKWYzA(E>H^3y!hShqmmf~x%0VG` zuzF|*wQb)uPD-h*T;2m|3{#48nO-hFp31X~A62=F4PS2B3ANPZa| zmXWo!Fm-X10^Wa#O@;tjM`VqP!C`GS+XgLvnEVQPOrZ__!a|n+on^mGR-}C zrFDO{&QZn7tZ`eftl05_sc>W&}7qz#$eu z@45lir?I?t!D8SHOV#l_9j(}!DRn5zp=42${jVJ+B`BH*PY7VeR`TO(A} z0ME1G3KS&2svsYe3bqMg*#X%L^5&Hj)N|^ODIo084?xz0*x41aco;)-p%ie1x)Szi zweLzRl}@ok%novhfFKuL+H{6iuH2tvrFep#GQ&ift@#H|*uUiQ zr}X`NN?5B;&xjRDOcJprsTr`=fJ=|}2=&eiTOE2vH-Wp$6|V*JGJtIlho=1G*BJ_< z%`uLzJ=*MEQgmF$nCjy z(xj?qK&EB!u&A21+~zFWrr3lIv?SH)kvPt-0Emm)7i@F4%B{!RaE2~xZ(sv>)s<>{ zVG(;=pdm=F987z60SDAI=uoHCJZgP6CXLGZ>~^3Ur$;}g^A4)4*dFx?ffIk$58m=R zhTD}wm8Ce*NE-Y@s4zRb1yZ+wW9HX#c(h)OWF>Vha4eEuRK*(8Hu5&1vzYwYf?$R8 zFzqs1!+`_W4WI)mBfr`*3%~bR-j%HinBE=C8(+e0kXMt^it>IuRdQwTRJ&~V$ZA5~ zK-EGi73N#-`_3+^{gK4gj##gd8VRnT%a9(ZK6zw#D7b={V{Dwh3`A!&L;nP9>KbVt zT4P9?l;;6u5s4g49{Jjf*{D!1*@I}*Vyl}OW2fFA?^FgNqW~+;5BVcdOvNm^kPAeD z&<&m@uHjOgPq-%}X{X2ThdrOw4O=J1IjpPHuWn89c`X@0!yAFc8vuBA-1ihux`y*= zoTW=r@k`F!Flsf@9=dX?vCgX7*i*Cxmez7r`I1HuXl?iHz223e!3|@d;)AE{~m@dS3sLews_cUXJerrR02`XBOZuWGcrY;<#$`@VM6t;CPp%JiaCGD&OTH78XEO^GOr%*8;GE3TamgrTgV--`9f{_l^8VVV>W{Qa-B&`I8ZQ8P7 z=>w=Nv>|6=y;c0N7~0b?x{}l2hA4fi0f+A8098?0TKw#`>@_EUNB3gh6y17wuRAQi zA>$`GW+je6a?f}jvf*dV@f7mkN>MM6%JX3jPKNyzR0hsu^_w)6?m-=?olU+8fe~G5 z3lMZ12M(nfZXK}(T86!BQ`GZ8FGgN~!jcZWb)0(!*up5u5IAlKtzI^QcwvVs;eo!V z@&_9Z@G_|Fishpz3q@2u+d|$okk(XJs_t9bEqHD>;8Se$)LI*uAnKw6+*3kYwKinv zfPwC*UsMgsJ27!ZOD6NK=YpZC)E#mWK`m^z^aig$jm+~IC`S1RlB(9)faa34uCw!A z!;dGOwUVpQkjc;iC(H!|^=YVxS?{8e@)G|&Gz8&O7BA83%`WM>F!QD5w?`#pC zS35XIIE}1bpH%|9CIGhPy2^*j?M*Qsc6H+`nlEfnRfYE;RtnvC+LFfxGfmPHCnVL? zTP|t2&B{*I|NN?zR;0*R?LUUU{o$TfztX)>)|Ed#`pfSLA`|cM>Fcl6wgfrz!;tMs z^Jo6wV0rkd?_NHP3g!WbykRdkTB?J()?ohMbKwA!N1=W*?a(txJy5^RcGca7y#~M6 zm#~|8fYZ2jke*JQ7lN|jZ;wjgtfME+6>qw6bm}_*^H8!uO|_iNm~AH1teglHQaH7gY$~waaw64A{R`}%vN_D7vuA&UU%0xV|s>(osl%>F#(&%bO`1r3^^r^=Y7tT ziy>$jaM*Um=7gVsAbnXU@FQ z8$P^;sQ~rWvWa>EzcP#?@*&8#4G1(Ba4uZcacEOB*Q~Gmtjn5@Lrob-Qhaqfp&#Q zr%JdIupl9eB8jc`{J~-tR4rWruUYqQvcYTj?SFmy#8Mad4vB4Sqf0+&W_DB!q*{m- zz5y}=WlKq)L>X_+aYXZkSJ`F?t#vyiiSn`lF#ex zAUN|ZaEfwCppDy(M)ItJ*(LL>Y9u7lmpmS*d#v)T(u>Mw1Ck6s85<2NJ_yaVoyyBU zEF7InM1LB*FkPLbUajB`+3>W2puOw#!*-7+=QEm6GV{9uVWJL)TA`A(W2!D#T|&*J zf<0_uW+rPpzLR0|su8?-!WhEVs#2RL&{ky~C@)n4oKPVz!cu~Y&Lx!Z?W!(TADR?W zU$CLflzq%43wXU+NhGgmUxVdDwzIT?_68K~QST9%kbxJ;NN)Ct;8_46oTGnPeWDRz zc|aX>D2<1^b&GOA@5qNXYeE8aQB#r)^9SQDM>Suuas#5B7(pWO?b+*-!2a(iuV04m zeuQlGQFt?%mLI=^{WKtSxSb&G*u%u8iTUCsz!mSqmRn?a4*?z_J$YP8+$bB=RpTukMj z_do8QI1k4h0G-z9keH^30hW9_DqfuhVovd-*a}F{>$2?I8h7Iw&5wZSy9qJ`*eqkr z&c0VE|83+5cZ8P2aLOv*fDK>4@83;h|(rH&Py@|;6?YeUFpeiB;X!NP*b2C>)6T<@mq;!v`Pjq#tGAY+Eht#h%FW2v zO8dCzIj@0lA0Eg}-y+9?1YKmzF&+LTGY9h>Ao+>h@yusi}H~Sz_RXh2O9UQJ5yF zjACN1R=NiEp0xInimBptg9#&F!wRqzx_?*cEX6{H-YtRAU@!#WEURc9UCM=+py%AXEXDX7IDgYAploP;1Nc5s?O zxdrr39xx`D$gtD8>-^0U52gT;)+RfYm&}cpaFNU32)&MXqz(`nnT{}@pU4dLhESea zaD?1J)ensW_Fm!A?*!7FyoeLv8`90=2JYsLLI?o$-t0?5w7PlLO$YpxX4UC%oI=dF z-DE{Q{J&Hd*UfCQXY^_A!Zzzl?4iJE?!@}+zA&c}X3>WmWQG>XL9YxK25-a(t_M)Y zvGz(aaRf3**N!pzqzuw~jgK%q$Jh~4V#sy`DGb9Izg!NaNDW}mBRf7FUo4l`BDd{h zaKY=OBE_ScwO@BVkWuMiC$M1$#{GtN= z4>gUqpHPpX9Ia0R_437bQFWVzEbRnk63t}JE$&;H)w`k_@uKuf}4%09^V< zbT=2dQ(^nq722(EJJrG+`xu?__R8&Z3go(QU(NebF4O$5bl;?DmXV6Wam^Mv{Rsm6 zA=8C0#dr_?jn!0x&*t;;2!AZ8bp7J_yT1#6vq$ki%B#N1-Mk&Gd^vObH%4;5c>RVS z0}t|N55hVji|zcS*3PG|pNAdJ4IRpSjK+ximH+Max6mp38ZFXKS}DJ6ofYnqo*$2J zv(5)n2=a2ZOeZ$&dgz?o@`!F0b-sQ_422)Tpjt%U#BddZq!C*e7KjSVl9bVehT4*0 zv0{IL41uk(pef*YXzT=S2lcBUT!qM?Kq@Aj4G zg%G{%dXt~J&ZH_qn%IP>WWoVTEM((VMTv&ej`LVT6oDN^zDcpstz?W$0dT0hl*#h{1g)s$JLLrD#r(XItm zwn8NTH4gEV>cT~e2Qc`pL;{D2h%rySY)A%8q$*yE1Dkl0%rY*~tW z+NlQ6fFaPWg{Yy^_Mxi8l?kTsa3ga19yq2WwgoV7Kg*kb_Vg{$TFrJHKPO?(?3(9Y z`$aB=hKrOL^)NwgNv)AN1%q6OV{@itRqxrpGgg2U?cnWkg;)(>2i4kB0Bzmn%YgJU zsKf{~95wb~JBA9mXZ52__omQ1#$6UprEn^X>GCvl_!(5w~aOrkN4wU7w4^;+fsjarF5sFuUNXQj#Fb!# zZItYZv8q8DCP*pjo97%KpxjuI!Da<&J2pn!4C#XISOUn$VW1q%d1!YS(OGiFB`e;N zo^(cACe4GiG&ks0Qw=6{qv~IulA2z_=N|Z9_@>b|qk;#TUMZ()q!pb!Tfjgx1(d@q zcMydVq?L&Gla9gQMo<5q0GBf?SKLzvy5T-T0@@C&hMinn_9kz}TBdAkg?1iHhf4LM zn=tpf5;3Ht5Cv)5KuyXi`GenEVdrC+Gu&s0E8}XaEm9UbDNK+^8*n*c^jeo3A{ANq zpJ%9cDi^I;GWDcM(_oY@q(6n#lE?~!g)S|W_lF{+wO{~Rdk>5k&u#DVQ%M8DJRvV1 zhz11M)KzFwbKNn3B}_qQ<17A6_}}+D_{Z@23#Yd-#r=ZzJFh>rwD|U=Z+;X!Xq?q& znB{+8=;mny!NHqS^K*vGSOf{V14`?Qp6AbCca@KjY=FnRKm#j21}byhp%-jw`YrT_ zG0G?H6E$!?f^13Y84}pNb9VhqSm97@9Lt+t8X1VcxX;$2wyH073>La2JV-z?qvuG>@If&&JG0ZSoyui3RJEi=DMFsilG#Pea->}Rl|#nm za>{v8(thIJMXhcQ)0;85_&p9Q5>1tIQ#1Q52g*8jA13mun>FP#UadDAia?S$YjwC9qrF@iT0J@zHDAih?+#K_^m9j(u{)dQ>J)Ns*wZ6${IY;w0 z-20kAx7bOJyeOA%hp(%^bYT%TKO|57I1X1Qxx3jAOCu$hhFV6D`2&Sk&hK6Vc)2+t zu7oDFYoRD$`}5Tr`6xY#6=7A+-g>t{zRXfvo)p?Y5Q7fg*=QOa4M_HpO9$*d+Lifs zR){=zN9rT&VAtFFg1cn!xP42()vcfMrde)=%}^X1UJ*);1{Gftv2HU(6s(;}fEvP; zvp%WL2rO@mG#koVvSlB^s#QrTOQ%SWoDf}xMegQ^t!3(1KVS@q+uekQP&NW9%ml2k zs2vUw-aEL{%va?%*wucHQ|~vt$_;^xeq0-3Sa?y_MXa(crK5#Tk^t8hg0bA1RETT4 zUn}|>DvM6!2bRMQk&uD}QV%-ub2z@*J%l~`;U=$DJ80dgGm6%`F-u_p#s=uhu>C70 z_ooAP$G!)V6eB>1fDC|u^f_~;kY{ll?loFGy3nNQpu~oaAjg5oo7t)kfM@1A=Smg< zD@jsjGuw5%Zj#`n$+fv37^q$`#ZoeNfa$iMGHOOm(Z1-h6!-Q!cZrdn?t|BF!s|~k zi~AX~!WOZ6-q@r`?|pT>&N*QuN7R6X@|j6?f!hkS)g`8|3zNv81b;JgP^W2|8KHt` z01Tyy5J@G=bp?iN_U0`@XRS5%vGiw!^m!gymIxDrgOgh{KhaF;akA+_>U>5zG!Vj) z&Jiwzu_)Z^c|k?)+Cjiwo>enwauBJU@n->2#AH@gVK*ZHMwh8ZxAN6#0z8L;KXtg& z@(`qoySN;O81i<`p*}E30Yj=1GtK~0f!TrM5c7wpy0u(zLslPYj2*5_QotJ2bW|iE z^irP}rYiWbgO6+Z>JfZ!HYy~`Pzew*t=$oLz$SHeqI91Wh+&-?`z?8~rDf};8MCNI zKcBkGd=HEHh)KnUC0H+kehVDtbs5UZllNA*A}ka-=*fvxA?n(1l~%rr{LdLM_^Cq)l8p%|)JXpoD?fI?n(s2S-?fRWIYz$U5-B@1ysqNG!{ z?%=Lg(+L+f89`97+_1LUw$tU2yFSNc1CSHXbgI7$cU%-pk?u7C+=6<>%W!B}DvSyTHbNC|dGtO6rolPW`XA*v*A76NF4?D%=VNMB4@h3d z!Kub7<^VIxVdOXs+#wf{21m%95#;A1wGKv%k_f*W%d8GwmB!}*_8>P21#EA7;?`6e zL@K<=x_pTr2OiKmWM(vW0d6)n83y$`(&Gh@62Y#I&{mFO#6yEas<|(a_w6UaNDxF9tn!OXu zv4=TNwL~-M4(yf(=v)BURL#vc?ds4#Mf!ps9C=cLhJFLprE<+?H9kD)XgCMaM*?6- zJ|dCQMlMJ9F5Y(d8y_k?*t2U`r4~!jYhE_UFK?>gd><$vqR>a6jL| zd_yqbi3u5sR)hE* zx2%-UY%g$I6N}ZWZem?t7vi9|aDG@>F9FBZ>RhgdrvVmtx)Zr@BGz-G0WsX;BBGJ| z&0Lln7)NA!n*@F}D(AKx&fr!VZz_j}?QeUa)2JERut)?3G|v|zQOCa*)fZ<8te&R} zCt4R=wQ0wO!)Ez8fKzj?UBZ(Pe@OPl#j9&|aO^1QCl^|wU>HhxLiTtb&ewsd?_A$% z=+C)dkxII85;c%UpB)n{0=H6eYk+y^NGU&9N}iSyTVg}AGC22*N}-ix=REohQLMlb znA98$nu7WLt1dc6tkQjuOAq_a`g;K65&+}fOu9{n&{L^L2bcWlqogu z>?9t&8_$*ry73^b42vHEz^-ct;tCO8Nm)lLw=`0(K~O0`TVFxPyf_Om#aOW8PR+{I zjN>*`C{&p`A6~U_pA1i1s2c&Xx9Rd*#;NZ{3tpj>G%?=`<0toXpdO=7#+Hlm;0|I-n{;bANS9G{`L!g z4BvmighgLJqL9s>4rqe+sk&zV0^j{8$aj7QC+d&GxBuhqr>AGy;A#>JNh`hYs-D=0 zZF_o=vXE#!%(xsbZ8I`h!i_t^tWzg?x>9Euwp+lSZhwdaG#!iQSMsjNaqklEbeOqk zFwtpkN(Hgt;u(ozU!`s8!&zzz=M1f3*iOr$YJALS`o;r1ATc^yR0AcGRZt6EIjQ)h zG$-EB&JQUdsBDT~0jTtXr;gjv@n4HZBxyEa(>!xTI76pNr}&IV*}-zx<&b>Ug8o;# z91h|nlTy26B_J zXCT9_6J`P(nm!5}V8a}C-EBFK{etRqbRU8P!6${?_ns=&_Kjh8X>RMgGKEt`?Y0)Q z5$jceI`b{hnVXyezzj=z3N=rp^1v7SDj(&3QDm?_4D+FM&Q{7$O zQ3k2Y0VMh4{&bmMBZ1X=6iF~cMfEkgq+p>F9(LPYun)lcddmo=^WXv6q;AHrOXdVnCne(9)+k~ z-&o%4o5pkvgiL}S0ng6*l)~90iX=fifN6<4wz;q61DSW~+ol*bwk+@p06??^3*1m> z=xS4jhWn-_z|T+*dY92JQJWKUpjxwX4Qwz70KF7L2Y?M6oYw2gZMOZ$LivXR%0;Q2 zRe_JWMD5(%yb=XFZ`wjfqq}82b}_gMB%}dS;4~X8{MEvk$*S1NI!DnT)xg%+qjGC6 zlAQ*(z6@zS{oP-niNAjPAn<@6^Jm|G`$lp{#uNNceCX@fpM<~W$?sl&6Y_`r2y}Sf zva{v4|EK)i;k1GE8em^U{J}L->~oldKx>+gFlBAt%+|OD9Nm%v5L5l4-fdwmVRMYJ z^)cIcBCIy0Qt=EPUTubwsC4RX} zuE|zxq&-e`5$G~4{SSrP1_}G657wmGwVGD5 zQCmB15SFxfk4y!0)VDLb0MN8-!DnR>-Zv-zYm-s3UKqxZLzabr++ywHkX{N|U0L!A zfOsGUvmdbGsC}T89kF zKw!>lqC;v|USlj5&eA}(8P!C6}ZRH@7I801c|8pvl;^ifQMVKB9~ z6)psXwdwQJ9lFld#TH($cn84ERKl-WtA`qsE9$Og0)XZox+-}C!+2NL& z)u1r|g}#hsvo;UDN@;hC$J7HuJLxW*>wRv4U@1U<&F2Ujhk=J`!RiDY2snTXfS!@g zp3U7wMcjH$*U)zFq)v#yS`4mk;TE4y$woSjCln3{T1CXQc(d@mAtkCFjAkQTPNNhJ zJqJdJh-({Mu^NC@CazZb33nJyS@Q;dD#vjU>)&d6^0_X@WPsKMYyx=imJW?Ai|4f_=^e`04(2 zzkT~-u!G<4kACs?bx@q$pO0|ue|`I1c>M%ALm$3=f`8wBAN=Hl*AMbTmKG&{8d(%e zZlp<5e#cZowWE%+=7&_J7u!)jB6gdD;YrtCCGI9DQJAzrc+5y{atXHwY=MgZRFOKM z%D^JL4(Wnzk=>Kp8*%Cf!8^sRaHLnvb#;Ko$c?XRrG;CFqJF}C z4+TBVQZGYKDFnGro8DU%U>q5G~!(hXSsxq~3UchL`{XDX278m6cwG!ig7rS{uH zz@p8uFMzl@ZQGMnI_#6rDLgEm@>QB2oV#+Ao=jm`SH(Q ze}&=Y&tE@QlF?UhhQIst^>g?~d?t^7dQv^T_ror+N8H_N^JZwQ0&;OzQNlQ*ZNtlu z6A+|mylD$BmbLQc$xsEu%K(Ki7Sw2nrkd~Df`+9@7^B00VIG;{pq2plK4?y64}Dj-EgofkZMU~ zc?lflon0kR0v51W(f94DdN5Pp+PVBwtPL~q-l~DhMs+Lq3yKS>N9b7k&PoO1K(iVL zDd}8Z6vvxcVC00RfP4{a~-yRo?i`AW+w&nl^of$s9K#kfc~k zlR}~qLU=xD^8+%#mEItidQwn!nxN8SmhKmL1UR@LRu{zAJp;ItwWLCiVp8BQYA4z` z6FDgtmUDL1$;anT3Ur>_g|5X)%w~6GwzXFdP@1422cPIt0cw0g)qo+ZZH#V`fckz@ zvmYt2l3zQyPKSlZdQirxU6a+!fTZBT1ZJj)fwn6CGg6x_XuF}NBNv4Y0_CYmcl1Ev zL7ak`G8_he(+aD)8XF>eO{MP{InBGPr^|fBq)(?WiZ7EFK|e;Df)}gy>h3;L44al8 z_G1dw-;%dJKG3r*5x5NRb>L|}RFuGhZY$Jqr1(36^G1L2*OXU3D;Vp}w zZc@qYWAsf0#PjdiVl&(&X`4rN(1i+Sd`a)r9%>eLmB3;NQiyl5L#6VHdLgR_hHZho zZIJ><+CK?101O|rk+90~=njQ4mhCl3#Z4A0U`NQg7K6-ot z(a0r5l{uPEAht2XFAmiHP9ydLhYbu#;QxaS4hGeX@TK}i)OX&P?Zp(8l}O3@ykA4b zx22K06pBOyeSsOTg)J?cV~C>^@OXF9PR)j_w&amzwR^L^E4#@u4}fr2_$(Rx3sPE6 z`2Zls0(I6aw;9#|@ati0hKT<3A=WCnbC~1md?RUyHC^(F!dKSTh!^DKiGZieu;|>& z{`$*+V%|5L-o<<1Y{E+>+fTbtc$&MJh@K8sH6T7BHFmwHryOVJz#;ftYDCr_PArl?-=wCE4mFeO4J9j!_0ifL%Ml!mz9M&&5Qh; zr_I<^+h@(1+a-l-b=$YITMH@uAxD7mI2Q5T0-Xn9TQfvX3YBVgQYh?G33-<0uPXFo z1t)mxsxuV2*g66Y5gxXZjesN!uSm$PZB^*x7rPyZV`IQ1UUrP8ij7t6(T!xzJ|1^x?FjwT{}34 zyU*`6b?9Xms%B%YF|BqoO^^23PbbgulJD;Au z{U3nl{qx($Ihln=i9E){r6H1dIC)1cRV8#jDnlSod9c^#jy=O}{4` zR8c7aH2d%k$AD`|Eyh~2CB=C(4;bIBAV-X}(FQw+eRW+|-L$+j%yOEr zIvUbKTbdgC`({<|VY;b&jW?|YDW^1d<6@@9?M<(3@K`U@QvgCccOZnOtwpO~kkoCG zH)fQ8mP6(l^R0m}Y~1}cZ|i9=s;Ok&sE-POfL!w%mMRqc1gNcAnOUmPd76S^NjibI zN{qGqtcN242HLRQcx_8y0-qZSG9yeNAW6ft0`WYXl&wH1=0l9sW}2?mxJm1c3z)E$ zoa}+4868KYqcq*zG9}3E&`943kM5o529CF)GRWA}HN7cxDgWMp1L>MQ+ALE50%I)9 z;n@y%Ogk$}B2ORLHCfKO?wt&P?hOMIS_Y{hm22co1{P>=L4ni*X=H8ewEN#FR<|KM zx6>`06ZSB}9b>dGpRykj2l4W2vm64jmxhkPdny;=$T{5^&d|;Vkdo+fLAieb)&po> zpMe;EMoLku&{zI}^7Ta96|BxKkiwRKyLv&Yz~m<@_)y9XBAygJO!s;QJk(n{KoDRo z*hj?qcnIHXC<o5T}^F?q$VIF!Vggb>)c2#1yIF8F2JEQ29>7( zE}#nLM8FRUG#ywh%ZHJMovQL+-IZ!6ay%GSnNbzbIV$+8@PohCgSU@e69>`yHxvi> z4ebhkA?=5S=x=}i_R;I_!?!>G?ngwbef;*d>gQlpk`GYg!EMqrAP$U4 zic&3W(n*+PsN}Tr1=^#5&On&ynSi7{AhGdGkNVJ zGgtGqBR@2Htei(&O@s=-)U6=6cn12VA-day490=yfeQ(FR-?ta8L#dP2SxjVz{nF= z7h&Ept4+8yOR8)T>j|z)=>$gW@Dze`V6ol8 zn0*E;Eh$-*%Z)JP9Nloqkc<^6XIT_cz6G9CzY*UnsA{T)q#%VPe#!dMl zOSVt2<-X1gAtjn7^FnBADPnDSR%4p2S0h;T_7HaQ1PIp3Wgo2n$gHQTmVVlmMz6C0 zssR9259fC0J*f(_v_GwR5JeDAP8W$4_F7=Y7^~gsT#<5;e6HR?T(E~{WIaQy*bc!? zlIA1btcg=kY>ZqmDXbNEYz@u13gg8u#xrCZ22K)#SrMNRrKNf!#9LnDilin$D|`h9 z)`K&M!t$4$A=_7X@4NRkdgwuA|uR7(X{LMxzCaL)@~SVUjO zqP30VDHE>CcAYtNjVkgiHSfsmT2bB_=mzqJBV2gZBi1-d;9GHX1(a`3->?4|{%+5+ zUlBR}l@U~}hx!vBa^ILAYaSwo*WYBm(nsO&wXYv&-qqDpc>6tfP@lbi5#BzsQQ|&H z46l}z^@ksI?!JHu9;ALgXEUc=nPlYCxBpN0?w?=^E4MxE9cJlf^PTV_HS{p@l6xU6 z^^p_=M-^Rp*HxAfnD5ggJbFpCCFu_6+W;^#DpOFvmdz9Ak$cQl)9Z|H*~PO<7|-&w z#C$r%mc9+Z6R9G@=A=Y8_K7tEFI2vi>Oo?GY*3oGUhjFx0aI3X_<#-8&}21FS96 z=_QMaFZIIUJ`E)#e?qoJG9X?V#@!WSU(^97>HBps&;YhRh+W4O_bL&1OY(F)0y`Q3 zyE?(p86-=5q^Qw=OQVyNNL^5=foY!o-~vzS*4#4c^siIrSobYLS&r1`BBr>Uv}STq z2Nmv_b|>a8huwXmClUEQra$XwAtUWWxMrR9RUvFU*rC;$D4KAggF=$aV2f~AMSScA z$@}C8++v6Qam@J*5L=zPF~Qa8lJ!pS_Lo2+Pa0G!OzS4Z#|d-6#qwLZ8VvWkc9N*z zr`JNy(>Es9p#mwL75uf~u=JYn3c!5G+WgB}U!;m-TL#?#aPf5y>yLRZRQXmCn?iNV z^`SHnY9WJ@=Nj^t0aZ%=9_o({9%tmvFgemRE6ZQUI#PL%B4 zn^arE`oRvdu{U&EeXo@iFqM2@hA+Jex$ka#b(~e*&US1NRDh)}iWK{bGCT~;aJSh* z2{9z|51$k_Pq7`mL_BVltC^WjC|a_Ekh?TsIhU;GEyFnvqKb73+=EFTy4jKZ8S<0R zHa-U$eJhQ6!}xC6!eirYiHw%80F#T8?-ppZ;4&}vY67b`Txy!(K~8wi+c2!Y-3lgl zO5?I|-^?_)PtZ`uL8lO!dzR$XzaDeW2zQ2=2=Y>GFYw-5GWpQZabJJ`JbeFwJ;VJ* za+G~!@A(^MD|Io?yR^^qV?TcV5aIm^aT%8x-iGX|eZp2S(|(3^k+XMf2{!25S}#~D zgSYmY>4EA%elD!U)s<~xM~P{2g5}UMwCeA0r_tD2agd=#Sv>>TH4QkPkwT)fJT`3tu+C;paVrJU1NT#N?mr28}AyK zHn4!Oml>X}JzQtfOntY9lu6RqS$DK4d|V1!=b;Y1dFOcInz@)M~_ae-XZkAY#^y(TlWT;>!6M&)nD)`wt;<7 z&Jn;$r?cdWQT@YD_2J9ZI?j~RdUbvR){=j=6{83|?3z04MxPXFu6NsLoj6Ko;sA?r z+|>Dl&}oR+m69D;q+WrY-H5R3-$^Ur23+uNxcp;s=r3Bpma=j;K^nfmsTQEQ;u5k$UB=ssk zi_CJMj+AmQn|TA80M-B{S}}T^%G0v9pqSPaF4P)QB!$ygwIkwZ^iFEmGuP^{U6`fF zdBjOm-|k%&u`jG=bJJ-(^h=`*XE@wROe~$`$|5bQuspM=2?M?2q-w>a$drqsbe8oV zx)mvXXjndb{rvR{8Y#R3e%`Uq|3(Y)_3^>{RsZt(!RsG_e)xqx@&uWm_4spO;!-UJ zDcGfNb{k>9aDsIno*pFNb($WsGROwZ$jsX6am9P8z(=~%rK(cXcT5WnrTDV z1sDi^Ud9W$l0nzYtDpH!CN8?`L3bI3f6)84X`h0bC2 zS;2(JuPklret`aMrJE}y%wE)1(NZa-pq*@bg+sEmQSzVT%ylP6&9bY)w5=YxVlYHq za7??UL0~S7P>6sFIh@Di+=3Mk0ydX4n7T=dx2c}4aT$*e;piOcL7rZKj|W09%D?iH z7JxI`0E-aYF(i1kI07P@z$mN+YDi$mTW)+977(R8K?`UX-GE#&BxOmR9wBmtm#vA5gkjG8TBzOnTCk3IK{68DjkfZF+`6#n;%c!}5FpRH@B&KAdE9TbO6sj%Vo}(sl+6=rtbkW>FPwCg9iJ{l@P!NTz&&0Kwf80u=u(* z@{e!gtlBv_UawWH0>vgD3W0N}o($S1#b}*w^7*oHr33k7Xg8`2qVAa}6s?|5wmWKF z`A%Vz!{k;TKxk2ObU4cK3@!@|-5p8r4zN0-mhdrF-9xS`TGI|+gk8>AW%8C&f-p`( zxT_Vpmcy+SRPQitcu4}5y6*U~0^SFK_?z;Xi7@#D;*S(yhC!--)x`P_vlMJ&Q1U0& z)#j|c?qiu$PA#@z=M+1{B3ef7$S z``wR^kAB8DeAtcrHoSf5XRjZ>{W<)lj$8ixhp*p+w?F7a>Fv{Zb$NdB_H%W({%^1b z{co?o%Er|4E}ZMiBh%v=)bb>U-qo!W?;vy6C)9XOc7VL)WsMshe*#C2YlDRq*@7!- zl6BV)?WN>jox=mqH>lOSFpeTVr-pbnoju<{#Kb(cbo)1GZ(C?p5NO@|;$T77e?GXe zNsu^f`8n{yEJg}oyC`R@Qui%?doFRO%&TkWOUv+p1heae!os%Q*dEMOgNLjDLD@)M ztpeG=`N+W?7-VWIpa{z^#CuSRGyPA~_|GOD2@I6FL#=0UY}cVVhbP{#yTn;`sQvINB zmf{&G3*Wb^1MH3-({;No{Aw74t7M`@^9clb`%Nn8FU``#Nbf;8Yh3CM$uj5#Ja6nL zFp&<$ZkupM9Yq<_ceW4Oo|3MkJKzhk9jR4)v@?*!)^1CKQ3vb7`CuJ~IFGtx*Y}5-@bYKDkxVUN6O;I0f1{CzI_vPFlD@bB!XXr|HzZCUcY*GBxRU5fYd(1zv1<# zr*Ho+&o@7Q{lbQ#n>6gCK|H!XTuDxQq`KOhQdmV_Pme-+g!`Z_YKA>iqquyC7o7TQ z?e4ZCX~c)nYcD-yS$l!DY~Cs@_uv@+MS8EkLG4Tejt-vNk=X`vm#XbK+bycTNzSs* zo^kDdEZwimheEC!9gNM19#T6}t44=i*ktm&kgFG(Ha@VDICTV#&U$BB>Oi3v)VG|8 zs0#yYI+H#Mg^1+Pv)(|AU%oo6_T5?D>UG(&hQ=8*~K;%@LDXq z1Vis5O|ZdGM`=)`la)f-;AAbta*s6IQ2{HFYZ7SPE?l>#LA5q^jXJDR>Jk7o!lJQF z!A;#QS5R_D($eG)Z%I1?V{{?h%#hNnGK4sA>EUr&z>kbQiDVjJ%#?sy)1zj|J`1}B z=*8WoK+NJFqUmILhh>WfM1p-c_*sbH(p=;hoQ(ntWJU|{8MZ}|B<1}#V_B4$L1Vx4 zDB0gQKb|0+<@RvFw*E+4Xds}5lvC5KR(xd_uaT~xD$2CDRH#~X?i&@zi80vM>hdM8 z_3@pn>)=B}Ih<_2zDc8UNqsmk6NiJmAzi-{gF|bmD zvbH>>(G{!#k3X+>>qNPdI6r3B$@z%9CfzC zu-jQzt4;x0No6ZsjhuekhN;}SsUWT)r`fGn!cUri1Ui}7Ey+s=B#e-HY>$8wtjsfV z(ej?$F|$(xAq+Gd#kQ1R#~!Kf6|rMCd09w(2R_581c>Kz`(6hK6!H_TA3C%{Qixyc zW!A0-)50)!i>*@HA$1BqqD(ew?Nf0DP|>nu6==5w2^(GgsFsagPembrz0@Y_zF42V zXv8?{0c`@$t0Tia7^%Tm@xg&)V5oA!3)=babPwDOb#B_TWuMYs;1)V`zI z**lm3N`SB*9wh-62Wslq-g^BxeE$Jv~c>VneU2BTSsYf@*yNs(Dnt2 z`a`!N!s`x>yeRtA!cNN38Avq{*>awgJH%2cdX|FfgtJ^eHycpT#)F!(n6Bm5dr}R& z3hKD_&;;eVU57hj&<6(SjE*1K*_iIrJQmnp%^h*ElM1HE&3yK)5XPkN=MGxFQm7PN zl$rv;g>%yxxQ|8w?ST|6*-M%Th@e;hTuI9OLGrd63OoXOK#Ly}8b7JE-yI)f2jurm zRsG@YV#9SQ)yD60w;l* zet8)2cXp(7Zu75*z%sbVTj)*+c$a)CR~L+{q3zIho7~VVEK}>S5}430g(TJj5QU@Q zPEYz)1%%#ZsMrubOj=g%D!>D{e%Z>bq{i)xYSK{_5vMSgJ$Ubw_>;?(2H~MTxIjwP z5bkarbRA%-=*dCO^-s^9anYO4GG0uXf-NTSOQ|L4tyF!#m7tGeJwq+eUP7}3PflTi z3S2cIv#v{hUMeYaxyguUeOnq{6MN~cfvL4w@O4_9)h2QS)YQ%kh)O!2kyMN-xz=OkQ-!jnc_2bu{ z1Fz7+ceyKd+IsA> zS13BnF6jV;HU$z8M}TMtMN0N~>s;lXfDI>$8XX0rzT}NdyMb@Vs|`yu;1%{Sxk~jI z&YTNqa*{Rz&Z=k4Jc!7yw~%(@vl;rxMjn`@dv3M>mu~yYYjkXi{52pI`e>2UFl86n z(IY_K$%csFuTPRm&Pv~tR8n9KTW89(rJJDgnMaekB>flgo^r)W*9ZYp_I3({lX90x zHVER;4Se+%Te=8t(E&B66+SRxGJ+A8V-T``_p-N~FyEn|2xgG}QV!LcwUuokSwJ7L zWJ#}1>NY4u2I~=t)QR-s`+m9ewuT8o@%Il-54&K)d%zrd#p})p@^Wa^Kn0ff|IFP# zN-c}BVB?m^55PmeL;pk@ybLrt@+ED9(!Z~p;Z8b{)Z9{)N%X(7C2xW&CP)QpCqeTd-wc;-8Z##qCC{h;w!YUa;}<{_D_v z?P#A1=)#tCXfS)0KX+mcbsk=H5P6u3tMHPJ=a794WFeQ-I%DY0-TN7GEHyyVfY+k8 zYq{P^5JcUrS^&(@wLb?<@F%+LzIJj4$k#vXjoIuHOv+j^k_yQ|-heOxP;=}Z#O`W$ zo13!li+(gHo7NmWv>6;y$-}YFlQe!3zs_z}T~5SqhiNcBwnaGp!@M6bN!w)mc|z zg*x={a>k6(3Izof(r>%AS~6bUps3iu^`mLMB=l=N+jaU6|G*I=vcvDtCLNbjZ{LKs z4=`l>Z7?b-Da(BZk0b|42i!zBB*L8xuT|Vuk zp@u6%WpjRv1fdPEtH`OsOg6J?BF#yb4*U+J4Zb1>`}iVE5M5v`%kI=%JgLx<>L=vJ z27Sg58V3A?o$qDs^C6BxFK;=~X&}VXz8&)vY<}5ybSjRUBRg6Rc%i!bw3>y^UEMJB zR}kd1W5_An9VPc?wP7>H3;{~PR7+=nK%`Z0b?2+|-a3FvdlbwE0CaO;+8Of=SJ+uw zrN21=H9RS5-L3HYkwVQNHN#|KO3(TS7BfbF;5$kyS`idodqylVT zgAvNEB|XmSR*V7K?z1qW&|>Exx29SqIrLh>wwvX42_SEIWwbrnZAKwH1(>~!`@0hR zdjb|pm}*dlR+@Qmp`EC6kxH7Yeb6ddH{$lDx4cyAqqJpMV7YW*BQIx;NR2MvOcbF; z3)m~ntli{8;li3Xg$a`Nc#A-Jkd>nqwjW+7S#6Y()hCN(TiXE8=Z9FcXJlE=x>No< z__G;C0VoVDL@g0b%hO$Qfi~4my7NU>741g`P}ZP@bBrb)t^|i|vb@3RkPJdvq%Bc{ zY6_2b1gv7gL;X^TN~a3rjE*WA)dZ&@o5n)ysbeR?>mRkfqO2`~qq1u$P%bUJ1nb=v z3hoP_q?s{DO(qSNo`VkzuUcM68nE=Cs%m!F`fyY=sgr_Rd-oQ;~)8)35YZuCpDq7*YocB5u7>-WgR^(wd_9Zch96;olzwd%k-6ip#Fw5oF;so!{+Ww~JbTK-D@^ zxBdv?4<-}w;p+zhI(Mq-kcA+!6sj(EgKB$bV^w(*J4BpiV(^+=*cVB zyN22c)Ol(yw)#qinYTqU=vbVeMEAmo3@&wQ$BfA6;(T&BF3n^>A}3?a~cAbKS4vQPJPz}Eyo|U;p(q* zr%o?}^Fmdvz_uLuF>L&%?9$r%ehPLP%kbhT=~|5?T4pb~%c=SX2bkcde9%a+%c4V_ zU)zAIouqnOJPRFsf8e0TR%z7{5K>d>r9JA1%GCi#gciS=b1wrv?BkjSYGzEm(!R_t zE`$rL8anV5l&ybHlc)Gbw_KS-(8I|)oh7+#3BSm$v?fyND`PtpHpq_cHf%7&t>Fk^ zZ>eJSBKimWE{xoJ!ZhVf%DWR@%EGv=P-f$sB_E}jp-_VcRgmt9dBy6IG7i_>)%p}4qJBxc@$BlNayq!? ztgR)>PHoDW78i<|p-g*M0aki66&$9@wuXEF`jy?`!#Oaw_Y%K^EL;{|fM6_{NfIn< zM>Z4?Z}jP0PsPiuuwaK!&ZtqdSwjYL>r!jiWo1C2*r?hQEarihh9ChNxCX)c(~lst znvdsO1j*qkFSPoWSXRY;wSqpGZLF~R1)A5ss~9*I`(J*jBebs}CI_P-kVHS;)797e z-s+dfM?W~K(|!K>N3OAcy+8WvH#3y|;`L|yqo2I~I{c6c?+>pZLcZ>&ufI^$gtsph zarkPxFl4nSN z{or>`sq=QPPt2pR(2Omjm$iZGBmtV61y~;2q|iJfvK3OJNL%;(lQfOcUNf^UDa(ke z`Cxt!+aBMp&8RMMcdR}pg!ZCzmX|AOOc^94sD!=(rLrON~e`-$jzB`Cl~!j+;%#3lG<6WuWlqtrc-Qne#~Y>Ed;)5FVM zu!ispATU|`W1;jlm*Px#f-xQYgb_xlG;zV$k{S)8V%~dQX2_Lvft?b2E!K(>-e2gO zB`Ny`W2FI-c=-`ka46fgDEOR#LB{qAvELH$#+sp1JGHvS8VqDgZ@0R1i8}e`Dcb86 zRoG6tgdTcH_(@w+lBiR3lqX}!Y{x`Bt3ah?69CPCB7d<2-9M2Y6IHOAaTxcM+?_+! z{bkZ67Ny2yO(vc1AG(-;!Bm#iriaBY&@>Ko-1-45Og_}HW>jN3Cpw+$7U~v` zAFMk_{U3}jF6;0vuey`_>JUjpC_@WY%e z~KVA`>ybM++u`P!^a1kL> z)xk}RlCgETS-&_1YcV8SLoLHP&I@J%Ge`xvh^(s04Qu3}_V2&Bw_?h zJ41^|9lk*D90DRsg0<`(`R9iO!>d}5)=-8RX2bEXF!r@Qr?je)6+(Tm-5^9tN<+Yr z23F$I2XVzQt6cb(ClzWlX*Q;{S(@Rer9RZ_7@M*;nxINawGk;3O7`nhZg1XOeO5R= zyn)6}t)hnpbntxe3Ad@Ezq|5xiOAClfU;Oep1d>7410>ldB*= zdSKYS2PJboI3x|59Z9*~IA${JnDcHjO_pv@NSy0UxY#6mDm7+V;crC=zS@h6E5J>w_aTvSg{P#g;4VbY|u*WYsIpYNZ3Pc!R;B{$>tDLF;>fe$yu9=w6V?;F*gC z695_8t*K~rK1{9O@<1xq(j=RwC}Jjr-M)eh_HN?zdS8dmA2 zw9FeuXx-{^FiAs`wQdt&d6ur)E+}b1@=2@G{%MDYRr~x6UP)AmsFGR9Ke;lyfGlCa z4yd%BD)67nm1G^*K*+mJ%I#EfWe9@T9r%cyd8^vx?D-r$Np0d16X}+KNQa5;?O>(m zip)i>Z&FJ6r?TMbC3;_9s+g|r3Q8%jE&E_0*KtD$y$QMg*)T!_p4q21HhfVJps)vk z8<~yXm6?{F5x(G^Bqt>Jbvx-ba&^oH#w5&ML#H#_ewe&rFLZ{fCby2shTlp=#|XRC z#rGwu-%nzK>i-h}BDYp`QmPzhsB*VL^vHGXu^h4~U#z6znAo@|Ca1#0QR3uMB+1u} zwFkyq`z4dsSNYMOy?q|uK9i5(?YBrkpGoS;pZTxSJsP$Va$q(Ng?rm_ZE4&NXtJbt zn-kp!h?~}@JC^N>6V24}iLD!J4k<gbu1-2!->) z{5Z{98}d)JjoVt>pE<-QCl8Wi^Mv#-5JrNo(!nLGf z2Zc?Lan(khm%<9gZPN#cYGYz`%K6u&XP13vF5e+9VR0|)4AyjZ7{==<13yg;%CZ;b zYGCej&(7q`Ycr8?*bqEw`ennb#kBNRkbx|*DQQ<5@_{%rRj0^Gx9H9wVrt`AQ0gCc z1i9aV$PzpuTFy$YeDd(pJIA9-i_1zNQUoubJE?GY-}PH8F>jJ5W%!T*w{0$+4s$@L z&+Rf3kadYCMI~Y8Udh=3o)10t){m?pZyq4CB`W)cg2_;-tmn|mD{A6?SMZ4});CcX ze{wnTYHF~ro>cM=ihuIw`m+Kh-c>46Xe3WI3a(!Ky}n6c#bT;n=1x&ir48C^AaRS~ zVQWO^`l^JT{ACen;)_){*Y%Puq^`p}Vslr5Ua;6@Acyw4V_Yra?P-o5sU5?4ZBJL} z5o_gx)!Z@4u_fb(?iHz}p!J^w>9qEzONJ2S%nYc|lO^(5x84A)%uA+k<%iL=xl`f` zOfOS)O6Xt%d^l8a_5_`?Y?9!+OF%~ru~M$@t}($tXz5!8^Ea5e*r_x`<#knTb5$|c zm%t2Q?gZs^Bfo*pFg(b0gNnZ$N$SRg$uWo6vNVvWXr&7b9(y2qJ&XtE*aFOH7)W0T zM#;=xGiGGYY>x1P-LleTFt{5OXmp3x%_w!J!xV& zKuEo^F$r6c*Qya5}uYH&!9#gXm|e*{`Qd6I1>H!>(_?r z53k=4&Houu{F3B;`TB=Ic;mmke(L)1|NZp~RdP35k30c=0`GOH8<;#@?Iqft>{+0@ zghIkaA|KkjPf=F`RALGMO3RHwMT|6Ard$`(>sobW0uOCzk;P!BXbzGtG+$d8-C2gfn6{Cm5twNb<3G-W`fV|A?)~ z%c{)mysX@Lsp{+}O19tp0pc440we)~AV7c;1%gmSqw!y7j=AQTSsOflIS6uhX7#nI zGUs(%mTqgoGl#Xn6N`G}R4Qdeh;?!YhO-mq0>7|+w}f|BrCJl z9U40a9B~&#g0SA7q(FKJEj8BhT3*I|>mHkE@W=|9ek92q5~z1-RE*sVWX(On>6Q#h zVd}X*tZ1Jj$?V-8Gt&OWlG=(YHKXc@vuv6UH|Cwo&S8IsozA^rx*jB^PX^J>4!lHF z!f{)Yt>Y?3>HFhu>FpjQ?Amx?P-;-;WqGgdg=IN(F>I9-p6@%(9%_2bi!l6N9)y_8 z4X6h=@s6QoKrS+6+tdP{+28lHliDp6Wn^WX}bc00#5!1hj%Gz6pvSQ=Dq z*k;)=4#RxsxS)fk`e+#lawv=i?JUo8km~!9cZWZe%1x<8PJ>>ab)TKhD{tTWgLBPFuz@m1?SzJJ81&uRbGgk7a5{M&D z&_+aZ2OfDOcKh_3riuRb4cXHb(Gv&mv%OeIukev2vq`G zz_)|yudIk>m2sD=&*zqR7&2HAl`o<%Oo`X1^i7}?*t29ha6Xb<-MY-vU7i#H&&R+p zS&Y&QKCbWN_vNFX*na{4oZ08AH_JYMeAGVw9M*?65M;jjoity6pHoY);qU=Tq6jfZ zQ-g)Zle7)eYy~pl?ihQO-fU~jg^Bp1r0HGBoJ;Js%e`oWQ@(VVP(rvNKTv~ca#Mdp z=gq3El$j_>4|XenSh4L_r{x(^4%q)t)wS*mGz`ln;ekN2B1&1DUuSi;y>?(_&4X)o ztq8xm_}WYg4As_L%#N`S#hjoz-eWLkqeLuoBT`Y|@LvvVdRi6ITPtEo8?S+G?Fxz> znOw3RtDPZLyX)eq9qTK=d6N3fp`z%BUKBY`Dg9X$)+p1MS)%!{3bzX&SoY*5)!ns2 z7&H5xsfEL{g#Co8E!dD|gwUdU){c-yzjnSsgvuUC%N>eUh)EZ9Qj-dHBC5NIFda_J zFlHc*;EO>A4DGlX?>ds4mwbFf53O7O`u2nWCPFJJXOMjvc7zBB2t!^1xI;O=j=?G4K z7UzYS3P~Fy*GR$O?YLg&=G6Omh3_q_-sD7rCdt=zhaQ5MpSnEJ=egMOPGwzvVZ_+2>HL4bCkq&|VV$ zp+sxB>(N2!7}27?GkxG`A@>mRmeQCPTaoocR$3Iw!)0hH~gS5E{=-Z`97 zLw5j6P5wg_&WLY-gyqmllXMgsYuyHzn;F!Ix+QMCQ#zZb0lq`|x+*$FYXiSuA27t) zAK3g4)cJE&rJ5j*kA29M)Wf5nzx|2WkY8a0XcD)-c>5KgB>Ex$n-kWly8BYVv&jG< znC<7ph2bmj<^zx?IXTXhrSgEq!a2Np#VdBNIIZI0Wg%T(}D~N?0gGFnUhsp zcyok~^u>6N59Khlzuy}D2d?A+;4vKN#DGqhrhvznT7Ca=o@2)XES=bSM_3QVcxs~?x4BfEPoo?)%Q z)?>BcZ~=3lD-DQR(99j9$Y~3wXaa004XPy#6yyQ@{+{=6BnW;)*kw-fG9!f8)DAK!>F zfiKCi8+ZXGR_3#|<+L;$$YYhXBu}J$ik4W1h*NGWhS-D?_dUR}aMztxnECf>9;BRa ze`g2VV>4^f_UTEzW)1z+jbx=*vmo!DVv1+FLJyJsTkqs@U|P zf38;xhhTVOr;-oGW93Dia%}Om2j`c;H>86p` zHTSfWiOeNQ6Em7e^Lf`lZ=yjHs3P7;6vGabt5q$~l-X4H!m>j+L4`52oR6uPzFgQB z`I~G4wLf=RmyIEMk=*bIc7aUBuJbL(%vu=VImP@j;dvkBIQ#S0&%@iVvQ7Fwh1bs{ zthx+~#I2P)TFHFJMMvoA3WG`VWfy`@??6xKcHF`w9E^A?K!H^^na~5YF9>6P(RL{v zUhGJ_#hVKlY>B?VS>xMTvJ*qome-6{59I+9E3yP>i71 zyS(}Lc{R$nj-5YH02Fdi*@po#dKNH;;O>0|h)j_4eh}&Fk#}>{JCrwcfzDS(oMPbu zZZ6l4*u0Qhb(Pnu^_okf6NoO5hPu3y{>t1W7D$=t#?y_7)a9+FQc{?Qq>%GN4JC>( zvC%-iW0Q*?*2EE2cdm~8^rZzO`8l6wb*3dcQkFL@Q>5Wb5->4B+FCVZc7xge!7O9M*izm;Nm+AboF0O&M;0LV;5h@W zH;=P}Ljx++fYD`k{y?5!56Ll``0N}z^=e*|4;49Zc}gm(yF@kF${|>)NDGV|WE+h@ zgOy2rTK%Fe^{E{|0=KSK=^yf`Fd9;j+R8A7@r?~5+wO_CK?ZAkk5C!>vQZDN(I@3t zM{L~3T_X%vAyW+vGc9v-6n)-6lbP|1C{$j^e0Gv+TKlF*(UV=;i_eN?R43f9Ggyz0;+ISF*&Q_5_RzYptq z)mY_BW`a#EO6<3A-Z`YF zD8hID>-&GmLFhk+zxt~j)^ol63+~vzJs>o5TkI`!3=fiZcQsYDix24Q zn<@waw6!=mch{}h)Sr4m(7;*V^XJ_##+49?EHW1_NE_-wkLjpuz($pvcX(L5l?aKb$*Z)R@J_g#}#YWK;+F} z0Ifut+7C;eHASfIM0JmK4~eQBA7KQQ?QvG!UfQ9~y}9`BdAPu-LAv9mWe!+ksxxb-O+o;Wu+?HdW6u=E(2mmtV&bJ9e&>ctq8q^j( zhtEdo(j*kaL7^yH{@+`ZTi&Z^-mw%1%nhv;01H@m zWm;7M^}jAnAVa5ezMYTN=1y{;dRtKJ9Pm*KzUWyh|1O)gRRS7hS?bjl|$y>=YSh_e(?I{*;K()Zt9ll^!Le4$< zGC`gP-&~dVn-pse1bj|a67ZftR)X7?L9T%-Qw#bVcQs8+c9~zH9g-is!kkH3PXnl7 z#gzBDaSVsesG0-PHYgdW_y7V877SJX+PZ60=}>O^x(hFzqN54$RT9#9@4{fQicod{ z5SqDbfk;M${h!(oQ9WO(C^$)(I^O_9mdel3F$>~-Qg=b3%h1*UHh6ad$RG9Xq|yuQ zE#$_Kze0z}lB}&Mu5`0t?`bJvPOo>Kk(#ZsR%PUQ3Q2d%PgxHiuCmZC>*d0YoMk~T z)DoJMPuxYyeaw+@ag#O%kg*+%t<4xol1>Yi@t4c>g58xBC&v6(V-|`a;|U%mm@%33 zE$*7jkA9RnN)MExuiw7?!CvDLAuR_F2MsXz6w~ z2q^=>TyJ|Cuo(3j`*a;@$nPl)!ldiei4{Bd?!A4cbVYqzTt*ez(6q2_Wh}CuSGc;O zGX_!6Tsp~LW0H&G`Ai`4zj>AyyQA9gi0T$I>TcrnU@Wiz^68blyX5=mjjg+P++oZo zK=NHw*R#+CAOn@eDWE$P>1z*6!5s&C1a4E3I3?P8&?QQBsTdHo77{2W-@V&USqNoU z>V2mkt0s~jI>R<2bG&$}h>MmHB)O`XBv-=9h^^B_oBTumLR)EFf!lP4y#;HWG7Yt< z1P$oa+*2F{sx7R$1}p?z5679F@aCoMS%k!*-{vD~5iAc@^uLr69TjD(5SJBK*`uY-3p*U#Vn6$*2| zg^PuDJ?xQk=X8X9%h%~$8kfr*%!ub^x5-xrA%lIceZ;CF4Sp5fwd9?NOn$nh+C5D$ z9)*LI&dZvPD+aXb`K^Mi10W;oST)wHVvcBa`}Pb7WL#%@ngY8F#p{(5H0aj?8ke^Z z>&Xa3DBm6omz9;e1Tspc?npCLARv}NFT7iD$YBh91O<-pP)lwgrsCJrC>)UFg+rIM2puy@ zW9rXK1Yl~zE<)Q?e85EX)FYjqLhHxSqG`B+Rv=4a z8c=S$Z6Xe!HsWy^^1cMSdzGi-;C29E; zu&IM%H<(akiL`(z8ufn~q^?sQd<%(=+({MTLMC%sa){0cz5?GeM%_$UhP!4wmV79f zgSA1FgyS|hv1vkOVA6O+o8stQ`1zdrs`Y^ewv43C6tL_-DNMmr%=9pO&@J{&yN;eP zCP*QGWOXqbCJG^2Ypb?~vt<+nE+IU!MSwuYkr~M9@_fn6AtgTBq-=T=FOil7Lrdlv zaA-Jt(ve(u<*_^)oyHXv-bv2ntE>v-@?YD-mspKVCXb`zGVdqTHppLQ9! zQfU{Ga8&ullhhW-hAVM@5#<6}W}SFkt3JE^e5}{YM7thorslwu3k1*Tl6C{+;NIVB z|3#L0U}%2%_T{6Q>+9zxRtJ351UJ*6NfcYVV<0`9Ru}5|XVRlkVz&txK-$-?I&Voh zN?8ED<>_wD3K)_NLwBldk45hu07XE$zZZ+D>-k1{z;Kh46|7KTe&AnR979@0=mc%t zk8RS^*`nhGhU(rht}nbOv8QgGF}GqN;;^Mv-D491yH@QTX(FnY%`m67Nknhs`{nA? z0vnWUR^kZwb>{*E`WjNShg?LAtSc9`RZ+5?dOlOcaIbjjSiwO*QZ6f-JFRR)Ro1lN z=m|~yKCQ*JFDIczu^5!0Tdnxa=|Qq5BYkQCXJML^8ox4y((E~UaW{u4-cUL1gB##! zHJW31)NvB2To42p!S!gC%P>hoEIL}%LH_BYj1~3p9f|mW-j|UX!Kx`pY*ej0((x7V z%Hs(1C)gAhh;*Du-x`BAwq;%{0)*~p$a)Eb8&_PrTZwjEwRXbi*$f*v$20^MCPf9m)+dIlcLx6-RS zTs;C-NHt7M73Rwa*DS@lyZ#KqLU843$$QdG&@BQOw$o`7gCj0NAmYKu?juG3pc-D; zWveAmK9*N~*V_G%R{r6-W3iUv zzlI-YHPQbR{+oXMtMDHze*fnCpN7|uO@_@;NFTiYHslZak$=K0>1Q@a0x4#)jp}ol zHrw^K1_wm zUUw?XD9qhLHZDx9%5@s2^>L&|j5~zaAR)M_Gw)NelH3e6 zHQf>bDctTT-Xzf0fH5Ha>Q#z^Y`4I*wmf0#zs#_0yINRL?;fdIC@3%kh=iT>NrNvy zd^=Erw}7@9O)izl04(2$^D)Oj@RbvMPyKf+k-7T-hn zAYvJy7BB9)Z}#Usso?LFi*esaW3Y_-OL4BCD73{(=m;9xXXul(Q=6^~)Y%{b2L!CD zLQe~yXdL1QUt^^Q!VCnC@=3-B8&nmgX6L3AP?J#KoXFFr^NsNplZ%>!iE9!1ez%|= zVA|}432{>WWD^99;GXGWRPrAuXSFXnfHCU8zC#2%6(npfgm?M8F*X12X5=K~m@a<71jORZszuAbR2$!}BSDeYHpeQEuy%=X~ zucpK`L2$elg7_1_NC$pXlszv^?!pPjOu55g+H3$MSws14p%;k)0wefIXjcfWo6mWWo#6yfbxOckFfi4V$U zABFrZ0aPV^)xl?I4zNR$9C9V(6!tur0N4mgGDj^J#txYQOD=eQ3ey#bgcx}|)_&hZ zRa;uPslJxA37CfDU#lHMx%mbig?32|%Zz9}Nu(-LB(=$Mwh+ea!EOP7)F$-=;hK-Z zp$AVYfmYnHyRdHtn2EC`KykRc+X|_j;>DKz0ErF{O0+sklEku)JAn~v9%HiZ*A@i3 z^1?HTs{NgAS}Pw6k((1mdXez~)|MF*TP{?eA1W@dsMu(rHzcsK>~|7CyWB$IFa@dR zr4PmqL=Ds>8W~n~*+&O^z~W%zfNBCLGXnv10Az`?yIuoi3oMN`z`6P6egap~p|yx{ zD_Dz2EUCAs0^YK{U+<)L$1rfT;ebv7U@>Xsg?xTJaf{jpB^ON0(xxObN}*>^G~i`c zgBZ^3Aw6_)YMe^{M!{t1lPD$(t7d}Ha1A90DPSNWAzKX-<>Ms*RAx^#Wr-mZ83O4Z z4i%7SX9ERxm3+hGg}k&gq`3ePuoCb(CjIH7%|Y5jof?XOH(64N3p=MPbq>Au8@6)g@!vrV`tx|+;Yh!$b#<) zwQe`TeW<%(Ai{Z>g!nXE^s3qPlaZe&s!ID-3Z%Vf(xF_YhI*I9!d8-K*(49(7w$)C;`@%^j9AmRN@QLZg>g>Vw29Arng?dwdTUx1VNe{dPkAhfry zR0;I;S6K=4pOHfUXnVRNHRyY}X{ptzc+M;XdrbYc%`@W;QW(!&8%I_3O-U>)Nb?&u^9un@Y2=9c){=PqCXTm8%$y6K-Ca2X zSmGS$BhhfcvP7uodtTd2VGRzJ;OIWtWiBt&K#x0hpjeA@f(H`g;bpKqa0NnON$kFW zQIq@jGD9xN4UL9Ee^Aa`mF_vzM%LnYTF{{==AKNGqgV#eRg)A0`4M;#b<{?fc8EL6 z8+IdDKp|(gB{1Ellbnf}2^sohL6r5aGZA$XYos#T)b|T^Mr-!D;06?Kn96lbZkD5>1s;UQZYYO|#_2uM|B1vSF1No)91UWXbL*P{8 zNxptU5`U5xj2KU8BBal43&OR5tGqm^vXDALksAu`97cy!XX}+=UZhd)fb3B6?X4%msH&rOoE;`3Q5!0x!+^1-tN-T{B zYVspsu%9U<?RWZDYHnz4UdSMsEFrFZl8Nx{qIT`}#3Z z+b@o0w?0uKPVT{v=kL8^xW9yM%tgi8q zQ#`^cIl8@2LsHFVQ;xn3@6J&pp#)HS>ZqS0$){)E^R(x=v8p`BrQ+&XqH-%uOSjzMt;vxh?zEYQ$pQe}2;3Z{szyqi7^knGuUe9+ z)!J(z+8MJQ3E(3ZmL+6)F4fFpNMbhlvo6&M3j%AMI!GhZL!;_KB9{!U2xdU?3b_ch z>u$E0T(o&tRdk{E71WBw|CZANhSQZN1>Zzf@XJ+QGY{h?Yi8{ea*mfONDB))sR))e z8Jd#yF}|Vbv(D2?x;^-N1t*7+7z3%mCN99$4Ogxh@b3?&@u=`MrvhM)GzN^6hU5q? zG6pW}cEkM61;D|1H;R0DGbR$Fi1e)pTVKiFdH1ygIp*lZ;LjH?PC=D>7xq3b554MCKoZWPHHo}V?Aowq8xyAV95G;Mz?%jv z3w3mDK!<%~V=u~v-YGp(p@c4%i=t}j(YpHx0XlcpX z5NbKBlk4-~{0+lU;*CZGag83p#yuzt4%h)oM%y&R45?o+OJjDsF}EmZ-=>(VI(GW* zDKmr}j&YZJAaPaynNng47(gvWgJ8j@SPeLL*lVNsMT1uL5$iSpGSJ42T-KoVQC4qp z5&)le)I7>+dV;BE6%vPlU=Y$yvg?r=HaLY3j)Up0B(--n079HhfiOVW1*$sd@T~1a z-kj$n-27Ygy~@RBvyVz{u@egQHE=CODU#9Ki+!i5Nu!sv<)9QLmXJfef%XHg=~c}q zB@MdW_#BQ)e2~no535CB+5{i#M~90VK#{fBkq%2xeN^p&*1vM?Z%S?>`zK;V7*z|v z%>6Je8x@q#aifHLDha^q2?lF+8al{Z6D+KjE}lN?z}Vp5xKb@oHpL zfGj*{lY@O=xzyQf{=rGKlqlEPcp4T6QXAf)rYqO%lZ4gxOjrYC$Fan7mo;6&O>AUZ zwQ)nD!jz=IhG~hAga#9D4N80_XLUmJ7$ak%TJ#WLH-K-xr6yv~uL%xZci+drhbvI{ z_`J9OTgdWee-r-h%x}L7Z|IHQ{{8Lu;q4Ppcqn9j$Pd}4%QF^`#c;VFkhz?g&lFxd z>{guIJLtMGt*Ich_pB;vkY7Xf0I%aH2WjC96!9WT$~$@D`-&D=&F3wN;kwF)fL*wP zrrZj9uTwwl>?J2XEX<1fCP@r}mfJN;ODql0sy|Aj`T` zIfoYL$x$Y-8dpuFcsprTVM9T01SukW$u#fYIZ6T$yK$$EnRLyglS^Tnk#ol$Qm} zv{Y_p8@eDzm!Pd$8iKU6Q{m^%v1YS6c_1NJ{G0Jd1@4~HstCD!j5=IW6;>@d6CDr( znxw!nF=*qG(r9Mf2iU!nTR(0$6=AxJ%c>S80D2zWP+`u#N={19qL|E0p?i+#PZr4_ zg9}$+3bqafoLE*Du7PW6dl|DQ(@M~W{LCg@_eQj@2%-6y6(BdS;Ac9iS%_oP6kyL5)#>)I%@E z4OM1HIQU@Po-a&t}K}_B#0YD}73E@UJJabTm?G@TrfJKjZa(#fo#_gL!{IP&$GT}f6 zRB~BCyNI(a$pE*jbuXAY$a}e37J$Fo5q2W3k5Tb}J55p#%$CsvxwPH7mwA-xa>@`v zygOEW)mkDA1ZQ6#8A{-?1&UqD6Ig7%pb9rm|5Uxa({{DNEnvwYP3NR<0pc}G1YgU2 zQ+3Q;s*OGm>F`kwD7VV@e2Jh>O%6i97Iy83exAEr&F2&RC3yyD!F%GqDM<&=6f+tL z`Ac0}5vut4AZ9-tYbs(1D@)rtPe6!3^#7)0bxLS95QEGN{%%sJK)_hdLiNPv7KZ6} z)U`m-+Z_&)f3d2gHG_zai`Os!CssG{e==VH6c#YbHReC zDrZe{R+yxJ!Xmj^c4p62aCi#Ibh*#9qVd~7k6zDjogo4!*pMiBh1HS}QE%YH$`YV!kzWXe^{mWmf-@N_d z?HlHle?Mn$|B@g3-{Cdm4`G*PHOc`QQ@5zW-1iRat9vXT(uxke(y2XJ)1&bouEPbG zp5V-%=q8&tX|Z>gP1{X?ZhMBy1bK3s>TB?%&0Cs#8g9Nn8P!dN;4C?Z5Kj*2cWS?_ z`;s*;*>WiBQwoxLuUvEvMQt`{#Nq`jE_)p2*b#=@&&WEE(I8F$>SC6F`hDsj1q-?| z>%1r8=vt`pNR<&+ju=)Q=VPV@JzgsGqFGVIyq#$soKwq@d=LjM0acxaOeIi6mbjwo zis6Z38n&Mb5O@a9Fmtrt0p0@nNM~YUJs#n5^A12l0isfgw%exOb1Q{qL*oFbiyIfIyz^7sZ7r)a{0T5f>PZQAy{eb2406@Qc_CrWk40&VPkpvUL>?3#?B$v8# z@d#*k;S!sY<@BA-`f_YMmayx;ql;FK1#pfhhJArppS;%7(zyxgB3&HoPN@YCb{o6; zt?{f3zSvijigR4HK?N}`lA|I!OGK|#w>MjBZ*s2^Zgx1rn8=_wPb%z9LfDfuASHh* z#4=%=O8+(oc|pe@K=cEM_pJRO^`n)g(j|=src>+b2kSMtFu@F16bCO|S)PVjCTI&d zH64oAsz{Iu#x0j5_YNDTBTTz{w{?Sa?g0XgbYi$d1hFHWU0$FkiekzoSk$g^CV%+J z;Q?AI#2M;06+0RA6q-}#{$jo1 z98zQDR}IsuDuE-aYj77Y4pb<AEiNL4XD{lQXl3TvQRCuQf+YzWbow-wEH564#kTE13CbmIC zA%CgE1YB$s0}j-vr*%#?x^{PpcRK`h=3QQyo;gN(P-*G0t4 zP>E;PqM68-oxCr33Z0phbbyY^mv=fFfC(su5f{ueGpNh9+w#mM1ptfOPbPo>?_C>^ zHNZP-C6!OnH&=EaHtjwDQr!mSS*e9sDI~D6a`c-U~QV%PZuFgl-~Au#UhSu`E^VP`N-lccU&8DUWi{)tZ4n zr@l}HlOtvn?WrUgf$#eA;?!4_lbGERZ68-(SVJKvY-@XG^PAz_&~}>nfKnxiKVWXI zR9pX2!geO(Z6!hJT9-{$33N=Ok_9?*EW>Oa3Vh8jl270zg9PnLpWUefx@ZG66>crf z{GxJdQi2fgS&X1JbgU+U*j%Y9TEj%<^8bXt|NG02P5k*!1Y7_7^&`+*fByR6TNW&M z2}_P&hq{;NEOQ5#IF?=GORalHf^TNAmc)Nxb@CuKAqic|T1B=by8PaODfdN+e5WNL zu$e9p!RJJK13?N%>yvXdg%b%N7YXQNQdw;TPHtIn&OyrBwY9P027Q}(h(*;+;X5!x(BBT#A@zoa>9@^t}wnVSV`+2k;$U zB$t`tb0t5g2`*AOg3^#=Q@RJvf1zNeBt!%nx12XW5LBoG-;f3KBSvzO7F+Tg zJI3cqfsdVZV%S8`(WwGt)O32(B`QRhSl6KKg8E5q#~GYMS%GhJU%hmwRCCRv?%GlC za)$4aB0ag(m|UW4F%ij9Qnn&GN-%EkZa=BO@e(@kWV;3uhVpr#J{H`K6$+TQ4TVJh z)}rw;DZzijo*ep{lw>f@O&@5js*W1C$JkJ)g6HbGa)nA@<-=~tuEK|(V&bAA0QLq* z)dV#?7|~bUn>2Dy=Ft9V!t$34n}2!zNsn3|7m{o z7jKd}z5daeGAg8e$Y|a@4(pk7(vFIn^nVDI*>R?^}LBl9IRD5>1e3|Vk3pLB!~ zscW~(>$I*%j~PfAPK|Bw2hWXtyakfpr83cAyLFCEU(tB&nxqos8=zze7aCK>iRuMC zc=MrgPco7{P&T;usLUbH4tH7pUGI#nHQ$LXoKuJ2W~OE;CswFmfi&ULb8IQQIhmIW zczl%NG+!5xPxx}995i&!1dGZcOi3r&`Jqm?HuBZ>*G#9==Y%HB>0dyrCh%igmr@-o zdGVIHWkQm~{^XM_oe*R}wa%dg&}v`~N=Z{mDF%cg*gvC6S zG31JBw$Z!lZ=V2u=B_(>PinyT|+_yp|4E0CJH$qmxL zFKt!UnIbTf29ncJI&8gOd1r=0_|AEfIZ+<9z@E0HRcm4C*N8$CC*)`^8Z@-lPe;o9{9?-9PN$wtUsNhU! zj>-Lkq=#tiZbpGIMsC${CDr?Em(M^t0s`;|PspOh&Ri0sfkKcNUoO%-8&PqT7l`550_zgYtz{~nXn3fg zC+Tk_0JCf#cU6)mb`__sOBOFpr*c&50~-a!Q(#EOOW_l;@${6D8wR3??kyQZI@br+ z9hh(np^Lk>bV}Ik^;!0$qX5G{TeJTJbS%lePlZ%~b$C)4j@dZ9fsq}|<}ENzKvezuflj|zuZ?3VE_-`hA%H_3|Fx619!sE( zik0Xqc~Jox2S5@n7||2tBSbH^u&$>{c@JLj#}VqRR0*wz=}3*UY8_UG3>=QyeIYxyk%9#Yl6 z^z1n?aor`aP%89}YZCwDfY~j{<+X1eG@2wO-*g#B4avB<3fv77PK&ftbvjP&*nbq~ z%ng7c+vx1ICq}3^y6_lK2%O9u7MD8$ zZ60Os%th)9AT$Ou5BnFK)u+>8l@CL_?#tA<>$l|H9Ygp=t9c95cO|niYS&6I29d%| zCnY8izJVTxlF4v9mGq!456$2-swGzuw5WG$M1>&wk1G}}!cJ}=E{}(cWQd)Qa^zCf z(SV(e!PSWI4sjyb18(X@Tfr>`Ov zVA}+ukvp~#Yx5s8kb023hwAy9k9X?Spq7T*qd*9Fh9k+eDnp7_ zGoYm1mr`QQc@vaUB*g}$HtdYy$yFt^o^0w$sf}7z z(%VT4L}bj@K>-&)7x2LN9k(FxQFgna7{F*B7Gka_IV$yCp89imV0RWhu7V9IWi5OR zbGGB~vgGXGZ^BH`R>=Rp9E&7 z58gfvZ-2_kN&@sptdUuho$Q~_ufj%1q10c;1 zxfSjl#lx}M#%5Ro5tqdXh6dAAc-c|3c6LS_ zF#iF?L}aDB%eLctk`VVS{}Oiu6Z<)+N^8zxn{>I`s>=G%#~J!6cM$H`$R6meKw z+M{05^8tH4OM9WBDgBlHyKR;`u;Zu>57whBc;ctVP18!tb+rfxnqEBRlyS9q&*Gl& zU;x9etEdT610FFQ*h$AJUbzB8VE_OCAxs^#2w_-BEf-j}K)`D`m|Fp+d!jhlq*3-R&0upB@<$_RW@-_B@n?9LY-lx5AoOyh>gKfbSgjSLxIckenFWg?QfT zy$c6+!P&<0bmf}5Nj^H$qs&e_mIzhd3!zy8Pm_OUb_E86L$@-9&6O9dMv}Jxp~eQ% z2Nc`6u14m{igl}8Vr%1iB~_&!=mats5EY_P@YiqIbn8FKe*ypGEc5BxFVXUR`u5H1 zSNW0OeE*N(^+V*JFT?8x`LX|@ya%qBp-o_9^ndZOZlW4rC5Zd1ivmh|bX9Lus9D_)dW9;e-Bg zZwVEWrymP9(ZV(^kAcRTX3Lry1I17djV!hnS2`r2+@js z1qg=fTwHp{h_$ed!bYy59$G(E^8VMx?noFvNtS_$mPE-LIw{BjsNxFnspmfgh6EFap&Q1aR zxKJ@SLsrFbPobD~5sri*w|!Exgq&Wc%{WWDCnZ&?ST1@ws+zfpuRyC6#>YznTnyYk z+#Oxtli!zyQhJVUBx_I?iKnR?U{6zkYFE$PUt?W#T1VAIEpJjCk{KjaikG&MqDuEN z9pPHq*??zzzVZ|MFiRCv{)EeG>lbC$H68Xq`nU_o>h1 zrUV5$FYTnIQl}n|T&nBnx+oImF%FXe>{!qZTk=WeoH-1YUq4EhTE^i@xmPtrE+_>J zKSvC@hNQ-Mm+cYITZi77tb2Eb0=lXAIMm zvEi=3MPIvqAC;0%j)zoYtsioLi5p`jhPIG^vEExX#f6VE^dlpsA>>3Ii=jQv+1(gs zX=>6I9G{{RC!odN4}-u-|yMHWUHr_R(#ty)dGo46z>_dnU5 zMXSq#x9DMZboBPN}Tewf%L9MUgwqclE%2h50`Y|L^+lNB$$n3+aq?blLZ6!Va?!*kfIOor^S#6x0e^g9T zCr)&RsTEC8yXNsG&s43(EpbeEgXUszn}Hc)C;xh=R0l&v=CW1-^2qJ=4y6CYAq(jFZ3mBWa;tv;6=pD^(3S=N*hv znYdqy)^)9RQIoaWC%1oVclE^v%8jXGIy>l4_{8P;qgjg6F!j@tp{^-T;@VF61eX0`p%I zN@y>##WomVPPLK|$!KtMgoP_1cu%+RxLGpDh-nGBKzJQiP*Tcc#c+;D)KwXX1ZTBN zyswWw0BtvC^LSN}EC!;obcNJmALR)jg*5)OfoG4J7}EFsK=lh$Wz{pZWYxhk0{z^I zWeqbJM7SmFAl215OMyvh*sC2q8dpwGeuCcJWllYqSk&hl!mj_Ojg&-r`%cDvp*qQr z32vtQa-Qy?@8TE({uRtra_J15xJ#;o^1#yrb41rx#h3`MknCum1_HOk2$@!CQ@jJE z=k(M`GH^uiXEYC@44f$vs+|}CoG7u`zE;}gLY^$55ObS#a-i@o_1ffZAbZIGSw|Q ze`xN&65bMDxTQ-YJr#xHbFH*TZQMSK{LkD85bmvcw04r|R zQ`HW(El{P&UAyu}|92b9J>Y~hF@%e#Kb^ViOUqT#A%!14fOhGF*RMnVkiX~Wub-T~ z(WokujMpz61u~8!`0VmfZJh;EyjwJi?mbeB>9$Hy_H23MJ@C=~kfrwUU=1Qa?CL7K zkRj4g&AHx0aji{*r*Ej>vj%kH)f!7s!&Z!HyCEOzVYuHN)}XdG{aE>ML>`mYr+xp= zlm{~{t0}hC$B3DCh3)TUE1)|LFT!<)#^kf3gXPBFcD$wV_43#(kE7H+b+hH3$EEzh zgjo}jpCH@PEqd*t3b7zfEvLR0mT=8wd65hC<+5l{qL7O^H+1vt5HuLj$2!|>uE;IT zxd6zcNGMGf+`YVmKJ%PL8W(6!h1#1!$a1&qaK+}3e(&Or^EENFjn~Vx21|)k5AiObmQ&CxJ)|Hb`{)9YLL*!;~7-Bp8L@#(>VT+Dy zX-U%P8wwXXRMa-_oEzWnHNdI&j55v-rQ0d|3zdX*i^PPr`UVz?X{g;O^L}`9-9R+B zw()thAH-WUdBI^iQ}$4>Mz9H#fw;;mK^K?-e2MK&ExK#BNHA=ca~fBL3|!z$uCFgj zE+hg%$6%1!;&QKs!c9+mzKxfzXS8Je(xd*!B@st@?MddSXer+B>-4Wq@HQt3SfF~w=G z+pM^6=WthoeZqMPHZMOLxGVx{g$)yylE_EFH$o+7_w_|8@fKiWB5aGL&q%64I-m*l zm$%-y?!GnUmKA4yI;Tk7Tcx|U%z?#2B70l0XF_s)_ydAu}XSa zb$)>1D*g6qD<8e;3yb>qvJB(71gOMCK*yT@!<|;Wwg*u(L3eIHKk5V7&kJ07{7{v@On$l1M5ANrL01 z#DYS~5A(d&;M{^LCB-HahZ0NJ)5!f}2<9H!DP)EE^s<6JJHIk6O$U|tr6SW^s+60x z4QpsRz?Ns;Pp1E6IYGWoS`+mYI~`2H8pgFmeYFJ&t60%fRU0b-G=}ox*#Zj;db{A4 z0^|15j-~IFjtA3Bh!yL8gRq5#TIe%c2kY2p5Q}T2k~%#;qmOMo2jwpxd?d`?7w_d# za7@XxLtMw83&oZFkF+N(8zpRuZWpa5UtYL0f)_Oq&A@MSFNIP~(~q|k12f9dmGGdD z%u!q|2Q7^@bavL&4)X?m%vLstUqwMy36zlGVbxzD`<>Gcu0|5bF@~XfLkr}ikF)lG zJBv3Krd^UXHncA9$rd#VHC{8p!z!)De4#yIT!)=wt;{s{i!HCjZ7?;-<#vEe#TOmG zxO9oVRFN0CP;`Arin)}=E+e5{d^V8kR#jP1tkZC*8$8A1y|N4vCKLxiGP%VH#`JbB z@5s`5Dyl%pRkM<4%Re=uP%~e9wwN2G7ua%!L~(z)b?+wz3L3tuspiQDx-cghG4;-N zEagU(|7#StK^Hhq2PllK3N1-HQTg6iHG6G$k6DL_j!X_0LbO>r5783TPuR^*LPjjV z{mbR=Qcct@ zW`9$I?xg?Sj1i6oiuvszr)Q9^KEVfkhXCV)J3;GN;Ezd%_tQOt_Fc%hE+qEq5t$U0cl@@!R;o!oCAxv5R7jNy~| zq1uup;y~cASygVrW-CwUb{}jxDg&VO*`iuZky_bf%x0Sism+UCZmvOEA@^BqPJ0z> z^VO?#cPe;LK&{;l9dHz?n2(libnSFi+W{k=*mD%q@w9(JB0hs~%w!5`6-nQ7l}ptK z+*^mdZMt1HdXX318clPgM>XcqC~1I;8JszQ(WzYsq-&GhWU>a?&p^Q2jaH$jW4vlI zVdBz3XO;nZPwfQUZhT6D$Fzluy2g0eu5`@q$ z%+hmks4sW<>g)|*WFa)ezHz;W?o@n&caoRDzyTnv8^Z(&VOsO*sK$zDz+A0_ zHO8sFp+p!%*UF#&Ogf&jrS;#LBYP;!ZZ=cXxjdXRi`QKxnj09 zE6)dCU15l{adQp1GNz8M^Vt}LZB5Eni`nJUsVFo=|2PwB4|n!UAQccptTmAqdoa7t zmFj*X$)s$UwOEECQ?}EcyJwR8lWh0d>e?=j6lPR|SSpEw?{_&3<(!%R60}p=q zF@M#Eum2X_KKlOPJuVQqeo_1fR29?nFk@u z;a1dla2Zx*720ir9wNWbQo+j|iOF_Ewb)G5MBtd(<^ z(q)IBcYLOfQ`!fNmq+2h=u0ItuAda(aQ0)>d$N ziRH)6MZ&7LjC2a>8I`y%M#7#h@TM`7VM|b}Ng093URe2qKBlHXdw76nO4>5fC9GMe zqmfs?JEg5Cf%61Rcz4mot8tdPNf{m+n;F!F@>;r+d=tZYUK*E@>`;w{2~bv}VtwO{0aR zpDoBfDP^p8wi=I^AaN%F*oh#MSiU$RW+YgcN%`mFr zGy}n6!M6}~cfDd=VCS>G-|JQ~D@yLaR$IFdwlS2r-FDlm);xS599;dEsB}-bK#B0Q z?U-6al6N?C;Mbk^RH=hwkXEEtunxD>iJl);JwRQj%d`rcQ6PSEz7U4+O!wJRWL59l zS$f-J@CwhU#I{_}T?9oEv(y*S$HS~(e(U-&HwG0%J}<#_Tw8C4q@}FE`NgFkxxSkGR+ z#J}O~*9t|JpQa!3_rDaOX}Q-tMiU4;sxY0*jOU^R1V`#?+kIH1RH_V%=o_ZggIfQB zc(bBCuw;$kE^qmXK$g-c4d6DqdN2Y^*2Crla!a{ZFmi@NktgPHsU)?;wo@IHUKbM9 zm6R|%7}F!!qlJ%#R365x77Uv-`vOzr7KfA z1>1dX_t6mrs-+9@0IV<|2e_(va5qB7#R@M>-%_6Wo?mvEDm{Gh55 zfsiAgvsR0`*c#&DRJ}XumAS;3uU$tVGNkT zYKK>fI~__-wOxAZZ|7Ze?_TJ7cn8K2g7u-?2@YOB-9#X~bzBd<$R+Y(-*QuGvMxlZ zWrK#!AjEQqS5c`P(KVjE8<@I79OoC6S4rT%VA>33TvaH6@o-IPR?4>1_)>JzFZ zw-{;5a;8^pc4u5L!<0(42RLAYN&&u>P2ydOrB)rnIE`!r*Y*YFge9F?BwkT_NG4X{ z;baBMnt-axa>SAXPR%i>Wfj#zr{Zp$-4bT)nn~9YHoR1O88<;yTl=Gs{1rE;9$A)W z>9MH0Bb8>Rlb9Ow*dGHIJUh}@k#m&fMq3iOIT~(Pm{O@!F?60KcLB&}+ad{aOo|fG z8X|OO-Beb?eoiuz2{>t*0@Ny>?c4hhX+&;$pAxg!4$Iw3QqyQE31Y@YmtkzCFOV|E zh+#V9dl{6bl(Wa;Hm;K@XswjCpH3R*L6Io1NbB&ZdrXzT5rw+Fl%uYP3-`wX&t>*D z)U{Y}%;~_Db$noLq54O_@$+;-LDZOMgoK4B_5G))7EhaA`tg z?R|UDoeh%I14uG==g0&Lt$?TjV-9!5!{J{ldY%@esWoa7PZ-z=MM_Q`m)MYH1Rs@?!+e2$14gnoV}r;2O|#l!uLl;ux=xXIf)b@l#&pO~g=nBh<#JT0 zjiq*fuV2#@#uU$RSSC!L?6AZ_UNMxGxoGpuXgr;{+Le&bQ9zal+zM)`E{U_XTTG1NAnW91u`t9 zQJW0PvO~PsK@+H?HQm8&7@#T$<+AFOcLK(?c2Y{il6zlJ200l$>}5n#l~A|1<07|1Nidhw}~v~O=I1lyd; zIH^PxoqjQbeU@yaiyby&&w11v3liN%l`R`Ex-4#iuDSAZT$w%gV8ci##UZoltuC*t zbYUaVw+rW%!FOg0d44^{@B=*)`d+G5xqFM3TYRZ0U32rH+!N+&8X5AbwM>dlSgljXmuV_^LVL*pjnX%^ z=F|-vEU;93lioV8i~`?)Q77KK{l;R z%A_-pE8p=f#fiz@DOkL<7`V&p$;5ynjWt<`&0hl*8CTMOd65VYSNp6Hbb(ez;iJt$ z`I3X$_eh!UPU3*exDJgJd08bYzsmpdKZgIT2R3l~oxvugzk!9;4+t#bFS(TJ3;g{* z@MGY?`Pq+Ozo6*axB1ae-@bnRHH2=y{r(^GBOl7oW;*!r_3PKKgFR{ag&)2CLVoY} zKed6}QTX=q8369}Gyu%fG*=eZcX{cK+}2cC!aT?$OqW!-cf0|OM=><3<9G60vo86T}}ZJ%^RDMI5i}tq%|Us z5r&Z~cKHR$2iH>qG@fFGMc`BF9qQJE(+~{IUlJ+-cj^h>2z}0I@4~*6T#ZQ&M$`p{ z1x{$$e4T{lTJC>TGDpIC&%6YbLa#EPa)g5b4J!MbbQ$J34Dg#q{?TI0R5LkEnmVV* z8~~pkAnMYC>GvqbxKvlRFB{FAW|aCf2R_RHTN_U6MkzWZR=Qk0lxg63wr`Hb#wzSR z@zrEl7#iPw&k znv{;D(hRVRQ)Gvvuq2(ft-ImB@Ic*caMe9U#q)8|HQL+>w3_QcsnTXh=MB(k|23Vu zHV-tG>!3~m#dgu2gZN1FF%-bYafHY~tjYVA#!4RBA-blP^*MD-x^lEPCE`n*%^2%( zmqt=@?b#31s`X@=w6;8p3rtSnOlC8@iv2_l$(6)BNoq2=51mewU`jT`Ae}cu) zPbCGV5v-w}XIyJR@Lw|aKa+&BbSn~4V~b?0!0y0bOJM?+qpW6QeO#_fnpQ`4^J#r7 z_gzP>Cafb@q_NgkR@l8;idvyQ1I5pCU8g9Y88rgDH##p?3hLE$~W zbG^c{IGmdvlp*ddFpM^$lOj}-1$Fx2WtE8f!1lvJro={z)O#h-7@lAGEWnOU9$au? z&(>l|H_)_R1#K-(w=A6-?GJ*@Nbv19h%S~Dj+cW{hU-0s@k;Se^g8-Mo&V6GJg_)W z-Mr73+@eHEEN;2~pox(_#FCUtxIju;y$cQ|5$OdkrZeLm z#u8UmcEL6zKAOfNclaT&xN`AtoSGpo+Gq@xLUQ9*v3LOPtuxDrG{aNFd>Q47)Vp9O zOY%B0O~e3gh!eNBk2ZxJBkZf?!|qPg(hBSu2~iW$AzegY>=%^d<9jyk*g;39s1LAQ zo+P065wH~E>vYOFNi>H9L66ell-rk{#8RBbbo8ug*tm&VRw%cn1HHnLv?btocH`{w z@aQA$?~fVTE#I=K=)OZju%N zl4B=O!KmGD;iv*nAzRI@B_);QsW-$Ygeya_yfADyYxNPaAeyGk#*g_BN76LI(=C7g zNzHN%o6rSGi=?=q#W2=XU0kWvFSjdMED0N9wj1+7F9tbz7Mm1>+$ z9D{)cFcc+ujWF}*Jdk6N&V@}m>=0?fP$~46vV(4wJn3%I9glrY6HwMP#(OC%XkKkf zW@9LpHv8~}30wl10OjH6y7SO*-a3Fm(vn9+4Ffb%HwpDq(}#f!Ihf4O7dSg%9@$@t zHq6j_v3T~QABDd?gRnLRc5puW6-~9idVSdXe<>mR{awHI9)$Y+9%d>T3jUF!zcc7t z{{7Qm8hCvM4&?ify?*%m@el9@pGpVx)pws}H+!H83}^l`tb~5{_6bb7zL^{ndu-u) zy&S0rX2Yhy8Ykf9x+S(U5Rlo*l+y6{?>jIP=G(+(_x%-N`FPwWe$3NNo(5AyAaB)e)%YxT*~mA%duz zcZjy2T*!4~x1yU#V0IHku?S zX1b126t#K)@vW^Fpg*M!0C~UI@K)8iUO9I#Xd7+BRsmp}u&5su1c7!SOSwoh0CQeK zW2ZcBrsm@17GU?{=El5b_HD$q6nORiW~N2uNBv*@4vLXabB<6S;wdc?QkQ zinFawRKVYk;BR0zy4`0{wy_xQksX*uC?AWYpk!V7f;{j%^< zuov$v$hpVbHTTK5b#AyY61PJkdPCnm6kVMuA0vX(!?FqvwcD)?YLj%?@s!D_7t(WM zuZ}Ja1jWPJGa2Iwnjf(0E%!OtmjmZTHS_~ zT!kSq41ngDbSgr@>WXDaB@HMQwX!Dxn(yfCn?l97VNwYLB*cRSM%C%HZy69K_A{Px z^`P^tIvuo%EB66DPAYvS>CY`d%Hc+aU>VVr%il$PPI(sZ>@cIl2V^*SCCSTYD|blk zQ_`$k_8OX=OmV0mBD{B+T^)Ezl{lNAEenjc!NJuA(0!HdKS^)-$NU&OnSYcfO8-5R z*6&`wA|LnDw_gS>cs_=Gn{{+QfBob4|M(#K{QXZaFNK@Qz5pmu=!U=M7T{h;m>~M; zE>|*FIYTxx;$+|k->xMc>Wvd-*70i9bjYQwiZb+5OJkBWwzyt1-h(ThbWpUe=w4v= zBpG$yc8K%2OiWUJxq)>LlE))}NtcK6XXO(tkZ#ZjgWYd_Xbv51$5*JMN(Lif4N`}c zA6qSFtRv8o?#2X?R0Y8>!pS~sjD%XvU2dzA2$!Pn_&SN^e%e5b;ODa#%T*lB>W@n=6-aJJdX*K~F%IZDg&(BwmGDVa1 z9Nn62Njkvf-y5A`bF^)biJ=25t6sRbIRKQO4apXP_XHtV3W5Zt$l!F*%v>@+fbt>3 z=)Eg8N&}!<9pir`UHc&2*MW=Q27!(0VRP{#;S1W=C-_710XP0QwES_6sy=0?wX56@ z#GYSaR8tcb2OPXLTsOJ`Nz6lU8c)1coHf}|g(qq$#{R|RAl4{hxIV2G>V%EW?g(sX z=ip^q+;7g3PwXmLF)aIuN1Ac6z!ExD}LxvS6pO?Fl#&@8EiQwfTK4I(ARNnd)$ z0MQ$z66Fk->Utn?uh24kH9t5>cO(z(y@YGp6~HmI*&{YIKD#L@tbbMQ%aRgIC@(+x z$xp&h{==Dc9;Bb&7vKE)?du9<%2j`<-apETj=cAv${ed58=lqA9u3(KS zkx)+#RCoC=e|`?f@$B7}b2_tdT0*}hDSBrUbvqf9>gOhX{A?lemURW~cmgzQCB_pY z1yfa%YuW(Udw8Ze&!RsV&d%=ZW0186?A8YK>k}oPZ%PTJRa`&iVV0djE2dk4Ww^=P z00(olZ{AtmeR`i;F1a9Q<{HS;0O@2&Z!clEvw8kpb|x3iohOvvoDM?qut1uzr~FW| zBCc8aQ$Thg<`2G(Yq6^&XGgr?8sF^_qbRRdg;J8=jxptfJrMv86s1T2T=vCH`XzFR zRpSJ56}Td;I|2y6`zEOf)aS+eD`Y%L5tg>vLtf^&)ghuSJ%P`*1P(nkly&k^J1G-) zp5j?pPKw{AMf5$xD4^|YWf7Kf^cRBM8ihTxh~tRv*}(^JsoO5Yq0qE5%nb>?UhSjX z3&wvFV-bXLA&u6}fT8r3kq+13!8REeS2kZuYPu$GOVvvUvtBO{{ozUqMUGu6PZ>%# zNz##b38*DESp!NNgQ>1A&bc%=S+zX(CrdEsvMo96(64Mc>cgLNaypt?wF0fxkl&b~ ztocl3O}wx4gI4#L;Kkf^Va<4jYu{a2aW;z5tclLBkc7PvCQ`$r7BGrMUQUMly(~I) zOr(Tnwjhn5+yo9y>LMF4;9gly&MH7-J33wIg&;a?@HNcqAO!oc&9D7a6R+<40Pd8s z>s@rGisg=&V~tG!t2Po;EuR`46I()Q#gBaO*1@No(6nNgCJ5P}@!XLgL(rGQ{&6fJ zZA(&+3Ec<4yjo7>H7qj+4p{WmW-CyGu&$;ISN1mwea{<{{J>7A1nip(vaG0B$JSx_ z%PAoIAW@g;huTz!0i&b*Gs={u?%esjs3k;0J^-1Djv+Gj`&{PTsRk`7QFlUkHMD96J?QtVohpm@LE^>rrVt7HAI5;8c^kb+ zRjONGs;gC{?p9T+8_1LOzr4P+*S8p_Gy?1{^huH#nHj;$UbpWe@fDb5Y=C!QM*|%~ zw?2znsqqN|#kiOuh+Ej8U#0lx7JiCZuo;WBeVc%Q(^^q%aV_qF0Nhm`UG${LQG?h6&n5{M$h+)^Qy(RRZzB{y7&tnWg_MOqCb@WI^Au=4 z+NO$t8QEqFk4I0`ng*Q}vYLvYTwGLBZY2zt9@<36$AViXO~~Y}gEobJIZf(t%J!H< zoY|v8P)P|+l2C|p&NIa)jiD(b<1&a_1fLM0@UX_6~j;^C6W;EdCTL)Dxz?8D2=ymK| zQk2?u38O?CX2GIhxl=5%kr%CW3y`oaZ`6vGvvQ$ocxi{EGrhP__*Tl}gjPMJvtqw{ zu~YBX!Ki1Qy0lW~0C`l>fm|w`DvM_E%+@Mb*gcID>ZtmTGvI2RP$R}+B(@^V*%63V z5#3Rx@WL>MsP23pYS>i3pv)*)32s9>KRQiRifbtDg#u!2I)S=-h}n=N5zs9=60Lw; zE3Po_!cd-Mh!X0D)|KnPlZ`JYbul0gLZj8BF3A$23_D+jrBUaBkva*BWFDp{I-{^& zJ-3x#2IXKfRHamdOE2#=o2Vs8sZ>3PqkIXdAh70bfIk)>z1buGcN8Esw8Srf$k%~sg_5E2f;hg9T?$Hu16 z*1ZZT5Q3Qj5=Y}u7779))cgpJKlG%>SZ)aqI-i4k4Lj}lvWF-2&m}j#!8fq|Ng>qM zUyKn0nF5RIKr7WWgWaoMSv(Rf*Z~9>X_DH(wDL4VeG-)r?!~R&s5gxTf?(z|{AK+zrgr){SYZi-PRO!5pvfmIS6mxCfL73#gnj%{J648MJ# zq$^w>f+fIY^jVD5LG2c5j&!OB6Q?*^ErSkFb%K5uM}a*Cb#jAwU+%y(!}2pYU=LHy zZgzQstCE#Y*FHY!Cg8^{2yKhn70d%IYdSbPk8I?fuK@hMBRXP(CYA-JcEmh=I$D!D z2&w>x;SF_xT9X>W&9l#kmMQz&XsfZY4xc<9k8L1~TYd*iTfhLH)@7*xiLHeqgTqxi zF!zfIb01jUmxOeyue4dpb=2Sba- zmoq-F%MryFJeA!*?4nf1_Cm~G3Fp9w{@H&RG9)ws0M?^hCBkk4G-jW8pAvNI6XVu}=ooWz;Dj z6uRZDZunAIL^X^dsBjFC2!_syhenp9+E&UTF9Q=If7m|_|LcFq(dUcs_WkMoXHew+ zy}|mvMqK+f$g27UW2=7*KYU~n>V;y!FLIac#x_-#yL;O$AXWUxO&d{pVS}&g(2)gk zX~f;!P4OgmLu38AZTPX*O{mI+b1ESH7eYEf{GGcj$3r28;s(f#O}^x%S2WvJRku43 z5a91;#KR$-y@Z()DXF{+em4-|3_Iv)Vn~@u&n>tFKBk zU$xq{5fko7Ssh4n(10N;$D!L-^p*1PKhOY^3);-d4Sgj{+O#FZ-e6h_s|KJKwM27d z+XOUErLJrkfLpv#3K+W&X)TzCq%{~%9{PFP$aEa}azL+jbnoFjTIYR)_R3{DV+veN#>vQ zdhq?M3oFk`Wl;r8`scbsYNlj2qimkfY#3#zI)2>FL(}gJeof@*Z9wB9rQwW3Coie} z33e@MbvQy`f_4VP!;`9Wk?~n{GSETPRlVC_MPuP28?h!}*eSjTW2&*H!~)m*=c1wR zf(OA`C9iCAa1v;>sj@R9U;$8sfg5@i2X1e486$Wrma0&>(aoi*=nh(;c25WXRN_Jp zuhjV#T{;1}%%}2cAS6j`MFoW|TP~zSf?jPJR<|xI_b|#rP`E``AYZe}k9QS7p`X$93vlRBI|sQw6(keXEQ%;TboJmXvYZr% zB58Fz0DdKUfG(F>``&&Q-hcYTN3Wl~|MKm7b}#?_?eic0I^-{>U1}C~ zpn(_6X0Jzx0sbE)(Nk1>G9|usvz+NTB-7QrNmhXzgCn(@>TDX)Y$^}KZ4Y2z!axoM-USkT5Vlo= zZNTa5cnk79qYFk-vwUd7KuY3&4$cRcs9`Xw1+tX(l49oL45?#y8L{Mr&oSU~o(#$| zv@cSnoiS}RFhQfNAx3xPy~CyaO!gcKPRQ3S8P9s=J~Y6MKXsiomj{tVSF)t$IP zS^ajcerOx@f(GRagmEp~YLB6+w@aWS+?)q=rkWWFECC|y5C@l3pB1#^yNeXjm{?il z#j^V(8G+8STX1jlv&(?0n5BHB*~1*!+1IPa0Eh&`ZvXmd!Pgjy-pQ^V)S094hUAdZ zu|}1YMMDy|iERGrhAy1Klc*_+?mA;5;1E(!?WlpsgU2imP#wzIwwv4&esLuBR`Bsf z!A!LZZK}Y!03{elbaHEWSe+d?3i%5T8WKz7S2v2Ni3)hByB8ZPn>zv2aVI0ovxZIQnPYJ;r7c2$3b@A&S2XaLUe`1)-=`hNw3CO#sZh^@UCD)@O@3-R7{ z&qFUnC_KL24!(ygeU>m@83@<_y8l>pu%zXCWm6|`C_T(1*shAW^AVW}L*$(ZU&`dp zZW&)u+~`|}teJQC3iUFy#BEPyuxYfhs=9m4n9{@02%^Vnw0Htq_}$I2vmAzstaj(^ zs6OVjs6VdlsO20dWUVTw&asRYSc_ZZ+NT|?6N=ho%G#tU6I%Mf2{f=ly!YD*G@W>dj zY|#0tz0+X;B6=S++-M|3 z)e1+Kb##}F8ADWu(n;8q!o_-ydP_(tbJ1oYYD2T zxt3>!H@^~-*EuAyMVoDOse1s_7T!6nj-y!1xw;TEB287QE$$HnQf6gXj;Y$o12ybe zZVXQF{*>ozTva4Qpfx^Ly9Ap0$ci9>gw*cPDT4medQv#06p#Rc6rF^d zdmwqOlsEXKXa^d1xz!)wK*}A1vQXp+S?!HT9nzc%cvG(^;JQ@`t%W!&xyV_8i}u}^ zTrkEK;|{Y_nCwt~y#D|P(xcHP%PGUdZ&*G3hF_n8$Lr6-ALm`ps~|k|pYoxPrQrG0 zrdmI-C<0!TZYs+P*xl1zThr)TJLjrRC@V=Tzeg7o<6Me;6jJ0YN6Qd^ zt1E@=a2sqyVvL0(MK;F21bO%Yq3|-NC)P`D-rZq&EtIO~wt!rmhX;8lPqUO(p~&tW z1-l0+ijS8M6W@YP=X!<_oh8}$X#<{JGHl+ksC?X*uK{BUSgpGkJ2M;`2B6;~?chvy`aM)`yj%$AOZv2n46a=kJkb0;&N(T704`gOL{M@j)(E!5aP$l^8Mv@bpP;EBo!xTaff+k{}Z$N`wmrMtuiJ z^FdQDrFt5HBuoTnOcWHtT)dYcl(aC}oIs`AziM~$!Ud&Tjb+?|(h6fHN~!`+h}NKl z%dwz5D%TWssVbx-Dmak*$xp(MYMb0C{G%WJFW{dXjXsk|_WB9PzhAxmA-w=_E#35w%TED~rL>Wr4?L*MF!_oyS*R z9jxi9PQ$FPpofCF1zUZRZAImXlK>WbRX#ecxswM&DxEgBk>cSJrzFRfH*b+g9ycui zjxt!LBj7J%5e_020uIAc_YziuHM(Is?7orw_E{Qyrm28bYFM2uf})5yB2>0YvoKin zD=fJoE)E?C@{Hl6OBIZ;L@Nm9l))KjcPu(V)3Mxs0^E<))%0Y@IH8qgQLD#9Lp`0X z5tT(pfZyx)XLIkRKm!A3*v<*Ch&UJY(Kwn_3fBLErij~$*iwS2@`0LNnn(58SP8+? z`A(W`NS)*cfLV#;rX|&L6cw;i*+6m^DQ!XCq$)>+AvY~%5rG0pUSlS7~Wu6q^DrI}|c#!TTr9pA%-waW^TFK-(iaUOU^Een7Mi0BaoW@~1 zDvpd|%u!Tsh|yN{KB>5ug;(T0b5viCpmH3lJx?SG8tOJJ#vlV~4t;SBPWONkP2M;g zIUJ~wYFpW(xP)s@`B->?3{(H)5Z#S8;6G+yM377!|h&BDGpfnx4(SZTq3%7c<#w7g$d#6K92l%l*b48ZGq$k8|MBGB*}_Yxfpzw|p6<`VJ7d<38RChF+}5c+PLh zspsH40qU;m?lWb|3mAWz;*$#g(9==wnB+NGHhMxd1gkoGj^^4f%}VZ)d-CQ5TPPY9 zvkHEeKhoQs4obz^x$p(_ie`ckCHc`et?CXI5s~{9Xh0U{v#YWJIU@N~GpEf30Bdse_Nuu?nb86iHq3>`ySzSdXMPT|;rrP5R{ac$lFZ_7JKV z#T{QHR1_{4B<~zFZ>lnZlH>K;2q;yc2+N>$mL9VtktyWXx)dF^Wi#lVmtRm}#o%ON z&|js)qZ~B0kao&?^r?C>t?zXLn}bIOWh2yW!K6$sKx`Vz_%5X#M|7CaK?~d9jllL( z7*_339X}i+9Lla3r=gI2F*@wn2hD_I@ne6nhd<2@h=sZ8H=5lff#_?oEGG966WfdK{{_HzYA$0Xpo)qA&M3tzK@Z;XiK3dZw%S8NVjd1~A+_!3EY7!~c z&8}7@*?beTEd`Zk(=udQ=|a9rbUcU|U z<OAnSUVKfL3gzw>m%guEqA%6OeJGQsVnsO#O;T1E`3`vULRn7 z!+XHff@o`)Bg$=BS|q$e4fYmmTMHx#K*cyb-qDyH{Jkrz5pPl}Y80~GX3~;kg_NRBdo*xjSHfbfz*#xF<}HR5$4_2xhO6M&lbxhexA`*oQda8 zpz>H)yvVyd-II z;wm0fm(>jjnNA>pa-&s%a@9ap zIJMwNN-YQsc6#cq+GcaW5s?ha$42z>yRcC$J1UKhC@I@jU z(N?hHh(n2oU~Ofyg$26BFmAoQ%;e-W}+$saNJescu>@59^Il%4-A^&X$}jDPv|HIss`os#z% z!}@Q-YyMri0(CPVj6vn>QV8UJ=F^x9P0EMJI=&;PYvZn?RVySLy^2I<|pCy)sf?AO5TbowZV%!)@$KI#qNCxL6G&usyY3XB&(QdcX8L1rL*=84TwO#GhQX zk`h;?>iw-wtqa?1=eG}Ii+@;xj!;@3PS9#Aij~kwi7qDHpw;P~iE-12H1t`hVdAnmJc6rHf_5LDOO#RB~`rC?FuF@ z?FL#5oDHd7toJ)e0WNFT#_Fm}uzPp);U)4mz}>TZ@dpHQDUBCS9vZ-HM`bElhoZ@J zwHB6K+RujA=W=S)q9VTniFhIL^yX>_(B0w!tp%)0q7=2;XaM&i3;7AX82Lo@@>p*b zVa4pbmV^g^J><-V_;O76H5M)^g_Zg<>qZd7*?O&!Bg);rOAM$;QtQzihxn8Sp(i^{ zm@*n-NovInNVKbaYc0IrVW$Y>!A5WkhAwl05dLnLPs>ON(?#bkJ8)$nt7zN+=sU3a z38w_7sY%2$NTuFO?$I@dhy01wGu%Q#vpp?Epbvm9|Gx{bKV?;@?6lvhto@Jv^;i5` zFJK+{t3S)X+S;C=Wq+Yj~_Tte=b}zYUa_6VJEGiY^4qy_>uU<~K zzTond;gf0`D3_KBB{*e!cY?4uo$fa-@M{PB@3B2IWJ7}%^ECJ`B9|-(BVW~?uTpU- zrO9aQKsajI;Ft1iO2CYe=-fholvA{Io1g4PwGjz(B8w5)|&ck067w=YE2!GOewEE5%TXzP^xfBg6~3m_7kXgFLSQSqUM5qM!L%V7KO98dRcviqmq~04GOPD18F# zy-KAQzLx7mCUt~`tm}q;0PrEf4@#xzl(Mmci%rkG@v8hOb6Qn&?ri*bgEFo&5$v z@*x(|q_X3DALhUz=f<-NM{zxan0z3B@(hvUvLA{Chk}UU7ZPk$h zece;Ibapjrc`43WjV7HE&1WjUmmwM9Ps)J{EJoUR>*j_KjT^Bem84zglvYQtK?fup zf*NwSwT;pNvLJlz+;26D((LORch(VM+6-Y1F%c3L;m$j7ZY0Yy8+J8OHCS8+?X6*z z^^hqGtgb_w0@DE&l=`s8zXj(?GpwH^qi^Q~H$S;;2svMY3Ry`L8)=_7?4jPM>wqe9 z-5{kQ$tNYbRk0zsXul45ZAC*%cRWy%04P`q0Ys+c`hWgMmcjq&-#y|1uFQEp9^U+}cO+$YTOo%}5X)E@do}A&G~%Jtg9k+~^nVXHlFO4P*?3=Ifxwk{?67=~>D={h z3*RA0S5adRP%VSTD8WH$ZOHmlz6zC0a1NN<9a%z|(Mo4)oL}K8L!|UTH_9C-Nr4q3 z!=O~3$d(h3VTUm~_7<{&bM%r>+{av%EH9lk&2+v!Sj?(-*|}4~R~`3QptQN`f8EgA zn>NNU5bhUM=mpNQK3!qJEVlq8sNjcnEsAOK2HW*Wb}k$3buJQ%3Y=JoklWvr^byJY z+;RA?Wu2>ZWxzNee0)!e0QT#~_>xqniQlc^AFF*I5)C>?vKK1z!Je1|gL7bfP_RUf zlTfxrzutDY0SV%{tDe}`h*nk0GDo61ltQWl^}&iLlF5LPAMsJmrb?D{6IOcvM2$Bo za=g09kaosgbSKo>F)!8mFgWYm_1G?xTXy4|R0q=npNTGH(}YC?&`Vmh07{=@><+XQ zBXQbFa4U!gs+9+}KJAcghbf>`W1w$KJ+fGzC2O(-1!LHO8KpPsL2o>vCrE~29V-ke zMi&p3H5ouDSOgqBytLD*EDbtI>p%rqq{gaTl%?uPJFPnS@wi^5>iHHgj_Qm+>>R$u zl~S$s%b@D*RWWti_OYa<1nI}ye(CB-oa_w~dZ=Yr?KB&Z>+wuKKpg<92!2rnDOvj)(0iSVuclL{J78 zFwktmKKmb8jQs=ku0H&Z^}-w6FM#9`zwu9S7TFZXcM=q_)z*b&nM zg=(ezZG5eQ9;B>Y_=+X0yRuflA*En+M`>0g#u<$uoh4utXsLM^M{6D30)=AY`Z+%u zU1eFr*sWR<{E?(cn}RAnV^z=D=h~^|(IF;@9BtW7xj-Wv&qxLq-=)NmCqoBpJFzMj z>AUcvT9J@St??2l`{eH}Q7Le`B<;j+LQ4@UVN4Fj@niz5=o8x_tI%F|djU<^QUdrR zWQAaI6^bczEFAlcsOiR|c65id1yBW}aLZleNM%HCf4%nB3#ltPf4lAq+dPzbkt~er zgUEb*j%Rs}qtua`HenlV>4rpmZ)d31BSfPkd9&|a3w^>sWOC}@1d3rx4m`XNkNg6t zC(yZVu#kW_w1TvkEWti&YWv{gnec%zhMX)u@_)$!rq1O_B?Z8PUz8>FOE$J+v|Z(l z?qYnPIG!YpTe;#4D#l<9N=)E@isZ>u=NKOuNa_ps;G*knQ?^2@V*rcaB_WqrNTZP# zkGhjA$Ey;I4K`V*$0ope_!knpqlD=NT06V7%Pwk0X&|Z#d}`0iDdZ*S)@4x6+3e$S zIRu5`aRWt!VIF1kY}*K zHg|5_0~r*s+1<0O4nAMh*%IyvlO=HqTRqDnqljzUPYTetAJe>;kqu_|iRjcCgo9xt zeguLb7Q!GCOYLJ=m$*#H{QjfYpVFcM(5O*e(Jl6?QsdK95BzBdMXK+8fT}tgj_enP z8!=Sn0$Q~Nr7c`k0&-I+#WDdJG_zH>9!LaKu;_+DOM*u*-%zfbyo#d+jI9h>u9X+> zE0rzJjTb0Cs+*GoPoTSyH!(Pk=%W&|EhjvhhMp8W!wF^5%=WF^&yX|5C9$Q}Lds{1 zF+V?!{!@7UL*A2W6oo339X~yRG!S5B`FF4X!g15%L1()M-~p1ozT|nowV@eIT8UG{ zVyJ{iQw-ShIU00t(g6QS`cFx+FR#g_Ul5`uARY8g32_q53aVkj?}GCts3W-L@r+=9 zk$aCIj#hI9$sJffDuUyx;B)%CpyAr}ETGq7e7bX&he6MlT`uOljjRvvu|8k`a>WK` z<}cR-4kGo47sY+D;8e6Wi?T%}(+U?;kbN}LR5@#o6-!6?t?86ND$M)HJ*?>L)l5RZ7H5o!Wij>mp~8-U-aTxZ;K4Gz zsIJap5^bI6L6&QFY7xuxxEms5kQ{5QLHp#CJR(rn?Fq1kJq!>z zlx(1IG;uATWN;Vty7NXjA}P_9aV~e2X8?0g#c22qJY2H}$YmLF1&|NmmOX44wUH3! zjo);<;II{mWZtlypsw?2F5 zTnbY~F1yBak`fIKv?pCg6#k#eGPFI&(j_YHVU)K-s)xb;a+ospeD_dX#~KX4rR`=o z103Z!L~$Y3VIAIiS93JvJn0Ofh{Uj60Zd&-UfB>d;hF#qH5?`#hD4IDkcV#xkEzk29?(mHuUbshozx9lQ+`({JHZwxE? ztJm*yZOU%+6AI-kVo$!Jv*<^!-`MJT2i93t*ITK65&W_VMgAtjjJ1&N-r3J3C0YP9 zW>ZJTa7!#+ZpG5|@rDP>UQ&!|sTIcN@U|)5nQiN)o$YN@`MW?MyX*(xNM)5i%29=p zt`OAh)d)_-nDVz~pJ&62#fiC4{-Gj^!x^*s3hQX7ei7@6lq4>cWr61Bf)pbYk4gfQ zmsMd0i)y?gOPIixie_}c-j3{59HoO!aU}_2gaQ1#I?frI(PL&RV(fUZ~Bz-phJSe#%#bv`OsgM zMf?IFrwcMde$57ibMlK`q{#ys=v z3kkfFOVMc%20jKeM`ynpNnWKbWe0YFg?75~ch#P)*iXG`20HE?RX@>}QZXct)V(Qo zTw!KO;(W@2n0EbX< zbXM4ts^`$n2%ki5(w$NO6FYBJFd5=!>44)eumnhl6zkBd$vuGSb-o{k}QCKaU~ihN&rFAwF&jW6#xr+`n- z?5=i+XQC(}Q7Mo9gp?>B`}ZODiFN0Xw152e)sMrzROw&-NCxfOAO7n2{x{&3eAsyo zZ@ROoG?Wc}AELUylM`);?9T$BxvP+DhIbryZ4kMO`&T zh{qQT*`*S6w-F_ndb|#YP1mAxCKtZNZru^-v0#mn&v%Q;&&bCxkhPzM6$`^E=o|3l z!QIfIAB@Ek&Ox$`Y_vGdJBsIRRFj1UoQoG=n$UHnGX9(;oo5AB*derh--Ua20Dc!1 z3%%-Lghh4Yz7(Fc`o6ms@038xfFo|+1+wVSS(7W14%#koNci)UBsY?3*=jg}&+NE2 zSbpl;lLKeBJCY~gd|4}a1v0<`ru2;IAJYx~Wu)E%z=*ac8T+%O%@>z@lo<4A9n2&v zw28Gr^_Rd?sV2h))%S-xvg*vFqeJ1D% zTFwhA0ApJm*ffYJ<2s$160AwUoeq{3^gCtMj;Hgc%%@{|0gr_Z7_`Y0Y&{z$`k#VJ(|k~^eq+N-DLjww zeg(N`+c>@d+3VMDzj^=jw_o7sFXTsmpI`ZDc>Q|>reBl7_>cf%4;ReBFI13yk)ULQ z+JI+}r3<>os3N{QT6y;{wMOf=>`>KR<%!km z8#OmexE?Oix7er2?|7X}O6jCwMSgZ{Si7PF(-Bn*_^Qg(a?q&73+V7kyQ2r^Z%Ry- zvzpEV;zC~DHw{843mexqga6N5z_#|f5YoD=kR8yc(q7d`SvjGTuD{;mi3CuK8_}nx zleQaX)s;DTgaCZY&u(j!+EJE<+N~WFy##um$_?g26JWxISdw6mT5ddz8yzKB{*RYn z5vOx|`!T+(auS;(kCw;h$#bau5GXmKzZ>OCAdq0#(JIfI31mtZok5m5YaQk{Y9wwqZogHS2!rgu zdQqC^cC2b8@`2c9kp{&rM6VL}98lIvBv|!6;=EPvY*e;4^%+bKlaeB#;tA^^a+y^v zdJrH;mb-4F3Ncylx>t(FN!eaKrIN3$+^`Uim%E95SqNp5&M|<8P^HpEceB@#&C9x2 zA{8C&d4Ss^3)tll?UXbsG^vI-&9lz0EZ%$<4aQ#FDjXvWc?n*!L$Ag{XO1F>ruH|X zSaXz;HyRRc+HMu45|ucHRTI3X^FC@2un|EXvYQ0SjD3|#X((l?QnlsPob@E zff2)rg_

B0o?DtF!VT!YjrDC^S2?RV?P>**rUM?tSk)et zMFAb(!zQv+s5=>il;$rbcShA_T?iz1b7B)$XCoFoujmv@X$Ba%-@E$Px9V>d;4in*8ShS{{H>n=>D-AUVp_` zeMY6X@Zn4NijQ7DXTy63W5Dju9UWD3D}vq|@>^x%=LSfh#@Zpln)@Ej4O4(iD{)wN z*eTaP@;SP7;lbNqw#(Bk%z1B3Eh#j;W@X*7 zx(*M5e?j@vVF6f6MmBEOH;!Y|(EYC)!Kx#WH$%!xVs5zvP z1|iYeC6R^Vcvl;@im16!(b^a$Tet%lzKBkGpSgr(>X~gy48^PzAqqO>z?4mkK`{}w zx;%1r;7@T3t580(6jW6jZ-r;n?$U81Qv3zXlt*j5ELE%HoChqEGe)BeoQqOmyy@Wd zJH#8TgzZQ$K@%z8db!*D>RJ0H=MIkGN-8gDO zN(y*~DHIy?$W98*MU|e}Ff6z5r<8TNs2R^%m+z9TN=KiZh-BvcfnjLytHZ4r&plBP z!0Owx(nNQ)Y;?2=2~sWH(m4zVnhiSG8zgNoGlsWFv^C%i>4pUjQQ+`>6*z}!%@0g` z#49{C2KlXq8+ws@-Ws8!t3ZL1AFCGRdD7;0f@?QG#u{RcHuTL-s=k?(_<*+!hiN?2 zNMSNzI|o*$8Y!Ka74QeBjl!PLIV>tYnz!Y4T2AG5JtQp4wkT9-06q!eICT|V70@+4 ze9cFkLiQ4i3)+y4Y@i=Gklg~!W2XkrNTgI@lzxyHfzfybV{kAVUbex_$*pdMu?^*7J z`?pS}v<;GqWzDNx&yAqfps5Z!ge`0}X$_=)(f!$BMw)k7Ucc%Xu2I+@M}s@Y=nW;a zBqzIdH*`!k3nmbNTH!3=jzUGXKL^_d71bzT=0a8A^1=QVC}9Mh(HWX2_6x3^(i$Z* zr!ld0T7Q@{&1{9md5PHv1_7R!4}@t0hkpfPvhWLm+wS(mJxbJ3+I59&D$_3 z^0n(%CmieCzm-Ii5N~A*GZB@Grp~HInb{N%MIFf>feM3zjqXAH5p6Hr@7zM=nY0N+ zP68wxp1NOC58@atW;-d9!bYvysSC_erxqrRu+BDo@}&K_i~G>_KEFv1cF}YQX{aU` zBqYlbnK(22ACVR5%fYbSdiWrJ7rylsIGmi%$Btski<*) zBqm2$^<|o=Fay8%RY#C^b3$pRAw&*9*J3%fDmk}UGS+tT7f;Zmgql~gK~T^`lj^Oq3PAOuVkdNikrmgrN+*vtmg~cqdKVsr7Npd3;bL~_ zkkOEg)`~C;d)93Lzm!+H%zWs)EzeoW!n}-m-T<^M;9qHT&7@*2L9~&Snnf!n%hODh z35q~J(~3Yi6%agFPK6ISkDjFt){bMpdi&H#9RS3B8(x2hp5#~I?PsXleiPpR?b~Ok zhx{+#AN|P}bY>*|os3g^-y{W%WMgXIM!Ozrg@;0RCSqpAm@=5CS+GSWtA~!;I(L3! zz}rEp;$Az;IRr@>arTi6){L_3v?~ddlG6(Q7Af*D0gxDFP9+?AG4WA~m9FSk`%Drd zeE7AZZAZW=>I4Ik^dAc8E!W8PQem!gRr!Y$d}0mDN-;GL1|!T?*vc%CcS$HgX9La2 zzH89DlZP3)oqfl)w*+`R6$3>T*uE}geJ~Q3t%gf0piw5E!pwg7n(JtgNC zUNaQ6_hnty+Kbe&XRt~x$+DhHE$NbZiKD3mMK4=_)hJQEOdUj>LvvEU*f5`LL?chz zd6gx&cAV!zBA%SM$?;YoJKO=rbA=OFCOi92QGJwkUe|C)f0h1OFn%)l%{e)5|6Dwx zw>8fadbX}tzL6bJuEZP<9YU>8Y*nqJp81BtC6Y-QG-6kc6;vWp@1b=;xWyP%1Vv|Z z$3u(AZLo#udP&f#avXMqkY~_%(m`tGAB>)w^KB@FpeDKS=5Hn)`Rz3Wy?HJ2*C& z=b>>wvn>XzBwLMN&yc?T8;z!>m9<@(Cvc7PU{stztnlcl)BOt$R4H1aSZpwJz!+ zZcVN1aSorO6@CpaJq+?5i1H|LSc`%AFTqWbk>uHLmreh!XQvdQodtNN-13Oh9rrg30guaTauB&pFxUruv!lji1GsJ zz>U$hp^Vrb+>QY%YR(rljD@Zb=XQPQFh#==Fz9-;(JEls!ENtUrgJo5?X%ojttx0YM#D(GYFihy*HaqBlW)6JRZ5Kjmazqb+2INQA1@ddOM8kEPY>vL9jbcx7Aln1E%sp{Y~jpfai zbYP+?7n@;ZGLSlJ(|Ej=d>mAGP!&HI&OtE~IYPyJTqQI)?@E}7+>3)=F)ELp zg!%_4K~|R`5q2Y3IQ1Q%ow9^m2L0-b4PsZ-0el2K{sVSAqzOGoDU?>#rV~6)!2Ibd zOwy$6J4rDq1*a6G3x-gq{~Z3ypXP}A<=c0#WBBy-cdx&VMw#_E|O23I((kB-|+ zf-vh+Q4-thE$u|KksEn`R>g!Nu#%5ynyw@?hMjP?j$CF3%-e`*7W2JEbpy}f1$0@! zx9T_t8x)9nI&VFjB4@cVtPqw>M<+_`Qh&(2GlX}>4nSggSi)B8lI^bSyul9ko+~r- z-||MG$k$}}D%6$(3d=yO1=gAB8Aw%UGze=;;O~d%7EculySrP9%Niv&3<1Ovd`#3x zN45l4ARD&eF_cigusD+%N63_D-Pg5>XTog7j&=ZeD4yE(Ay7v7W`*J)$xVzt=vb>> zD}&CO1^D`>bwzIXkreq2-3qJxAi=nz`*U>rJ&_*P3jB&>JEP(Kh7+t`y&60b;Eul!|DHQP4R0Bp4~#}0f!%j7W~73H zB_8{KN*t?Bx3Jl{Q$aK6uQJ#`31_Pt3bBlC@rI=eqPVo(%mHK@j)~S5mc#DH7JA?& zj^>n9aV*u)JfygCdKRX9OuMGx3UjAbz64sMwM zkD44IwLfzwOfF3wh6qHM7+;9>9NI*N`~YqCl*b}gUL$ZyF*IlY^ACES5*&t1t^(5D z*~Wtoloaa`6@N?fDu{#2nV$$(gl|N_D7mf~br0}o--p}4n5okbJCi-S14E}^nwIzu zlu+!A_Ugh+$eV;w=RN2)^Wmp=m5*a}SE2CjSxIyOL^|iuVS%+JF}q{BjuChU zeNOUVbx;FHi1_Xp;#O@b*n(?uIZnCNfKnOc7q?Db)<;{ngdTtWViV6AQ?dY}EQRRUS)Ziw`{b8~v zZz<9@53}P251NaANIfPiyd?%z53ID>iXm6Bh=-HyUIudi8a^MxNDqq9r}!08TdS@% zmFOV68gnbIyr`z+|1>^;i=ej%q?%IM!2?BHf*?mKfIJ@d`y|D_*;bt!D23wrTtPt2 zLUm- zWc3+nX$+1fk~622WOhQ-23P z=^V^h%E=(Wj+uONL|J5Zhpv{Xo*s^E%T``s+RYHRV!<&f3JHnfdZ0q)4?vy^fGPNl z$47)1LL5lRlD(#tfG82?sZ5hXG7%#vl_zol+iq#Ywq(-eY1aIrv4?1GZozWjIS!$H zXPY&8dN?DIf@`QWp(N+x@aj%YVw$?zs-QpcdX6nATE_^yo8a~|ffK}$-y#vnPRgXi zZf}XA2J}6-dCdC95Fi#2J6%AU0`3gzpcrkH(6$W1hS|NQ8n#v+$mb34{oecNie{^n zqsW?BRzic~3^tA`Y|8^BU(u+&Q)(MV8}7rXEc6DHG@LXksVJ=7L0lR9s~+%Fq#(T% zSs|$r>?Z&AlF8(khLU12Om!4wzb#A0F=2m{RsdJs8P619&TaYqKqt;tSEeW_QE@*& zDN<=-tesn>jh(U!NM6yI=rgSs6lu_~z6A6&QLGdQZW?a4GbIyX;s+*#>$^)#JkWo* z_xesXyji9-60Paat^z_Pu>-4EIt%s3yh3-naZ@SLLh+-s1WaXJpYa5`cDKVa7PL@R zf1zUVqo}6ZqWdTjNSP;p$=DA{ge6D3!LPNS>zU;NlhdE-tHA-$Y9&>)j;IafZ&$0! zbb@?SZQx}m5Txrv3^WH!OlO{|;;PraWF2f=&S^Q+T!NE3E2tR`7#?=&L7}Zr3yc&< zFK5I^7-eu!m`Z@UYd=pV%&$R>wdi+k*UrUW%L3yPB5zA@r8%R3a%ZO|?Os*^Cg~o2 z14Isr+Uh>=U^eP?*WvEi!>>(kD~$_uoD}dBH-78j4cie!)eJGA! zYQ}821huT_9sy=E>(q&Lhj{C=!Fpcj!?F+Hvv@@)50FM6@655^yTB=)Jo*J{EOkoM z$(6EVlQ#*s8QQQfs~UJO+zH(zUqUG%NUn*mh0Ea}ut?e9NNeoCXc~|a=Hm+-L9%wZ zcU@3?7*ccIref=t00Balm!Lw;(BSE?nYZNb*a%LX{4GjRsf!AWwu$W?#GbRta|7XD zVM%oByC;T}2AA;!uW-3q9i{Lma5%bFdE9^ zL53U>Sd*#kH$ubvEFf&>0ZplqZW1{6EGn*txH^C>YjL{VgKD~PF}b8*l9(4rg4PH| z>8i7|<)oakPHQ?V@7lDgL>psI&S2$1E?DUG7?Ct{!B($Y+9EO2Ugq88Z(2xzLCUY$ zRG3T_fcXSB82RaTv(ScmW+vARPJ?iiZ90^5`b44P5XC-oZk3;rgVy*%Q2ggWC?*+`RrmQkp=$O<`#YD3oHggWM;v zBT(a1&KQ_gH;nZg`j$vkkd>x#=Caftm?U4-t6_fD<-u$w98B~D1!U5C1eit-c2HZE zzfKLy4&G5)&qAYfwMigcHfx2=YM0q7jah1>v*D9uPvTm7GAEuuNGse+0V>nZQN9Pd zg9DF!m+b9%n1wrz>Q^W@p!e&ZrD~serxmR1Bj#8HP>8&@2~-!0?UxifwgGV35><>x zS-i3GP&;FZ*$`~EeaWTw+{5c;rQ0@C-H0pnWik38Tm_OEw)`&~rVnsy{9pn-Xp^uc z%L^0z1a)d##VcGlkDS0vswyYCMEx-=gNW(;=yJIuDX-x(ptl8t+aW97N4TQR?g<3h zG9=R$jISLQj2NCE>fLzVQM9fknCV{tvoM1mNgul^_apPTyo$rXEgqU%#a(KaQ0y#C z&Wh9C7L0(891H~;2HG7On6Q7O&G|g14}41$1WBuemgx94l%=J8D!5S{-8_velxPA} z3;Kf8wnBg4fNM9}_*yu*0^+J6BHahcfS}i@M_DsCx&Ex0m&G}*J8B%2)q$~7IQY+~ zZU!Z50l>||B~VQX{J3dCl#r;iW)TiM%TT*T>4F5s z3eYJ;^)=8lw;yPOt5Cw4ezGhCdb#x+&0;hzif#0|Be-1vhN2R^XYQAKuZ-5h*GjO3 zEvnqzph#WEECkIEo~utvrLoy&@s8NsF_RY+*;i z7*|%>Fq8SjntSGLQGVMiO++^XABD}pbC#1^(m7d3X?Qhiz< zAc9zcT8z}TBA4!z3pw%tRInu|B{~MK3A!(M7rqjyvB23j;g74T4Pm)QgfLmApk`co zNJGof!q<2-V|5tC^%8)`ybvs449z+60M4AX73l}MRG}uMV&ApbDq9lmwi$)WHJzk# z&esB{FUU^0HJm!j&fVk5Svx8`cjhakuz?2~N-K>Y(Q zJ(8!!1IkVV*-IhG=#?!f7(d>*UR?rOFL}X+l;=K}VH7$C*%yGPQq7}nX-CDYsU|g< zw8vJ|)x71Kw*~ZHa<9d$qe&7J^H$kNZIqENh-V2ttmE0@QZS1&QR!LGF)`yiU0V7z@ zk^&1asuYrxRA{0l;0`ymCu}8}u=S|~iC$5>`KZF_7&}Ty0AIYNSE~FW+dy*iQ0YJD zU0{X*E@O+Pmpn6Qca>5Bt&&W}VPX%i*ucht)$x_y)LWI2WUKn)<<+o&Bo8Hvk>|8- zl?NUWoO+7BmehThb^p))H2j5r^FM{3{KXMmo@@i2>AObm{gS%D-?MajLhU_+_QLy5 z-~Po=@>(7JMvEjK`Rnld>FFVlX=Mztho{{8hVc%?yzWTM@(G1BTb)1sO%7O^v#-etz(-z2~UkbHNZ>4TYzG4EFB1?F7)p&AqoR(cJ-qYYA z4N8AtIQDdMdQ5H#91;2%2ANBMhBFHenLSs!zJJx9I0!7zn$vaI5C5j^KM3T>N_&V-Cg zJjE5`$1<&j_0*)&csMf(e65~D7Fe1u1Yye(4=+t<3Z3&ZX}Ur9j4%bjXwdx3x<&&p zpn1`cgmPb_BVTKIACUY$cv8^BKY>qnNv79_WQkX?Rt_~6X8@zms#2&3@>!YLM4tf0 zLP;SIxFsU9KOp&uoITDt+6Ws&g9s=Ty0<7Iw~tx4>1M^u77=B2xuOn!X(Y7DgCh4k z+NF6PH%-UUzpP993E=6yKqXPqva7_5Bw1Y8~*uu#s7?!(Lcp1|J(5Tv8xx!+WjP4 zvp)h23RSS-aqQ7*_AV=tEO4f`e20e5z=9l@;1}BplRCeq6Ao7 zw6B8)mhqaP9$Oc~(5cn8PKB5c?Z5J0`YiGmqLE{5Mq~_t|;W`GSxP~$%!YE#UhJL zp?~m0(vDM1W*C zTUcod#FXurhNvE2Tc~sdizUfK=BLUA$(D^WCaCr1MG}UjRv#w}f7cVpFc&E!WZkz_ zf0s*mLx5f*q+G8qtEPOO-rcHGe!8RYk!dqz+BiVlQ0#)s&6ri(0ZU_tBzK+ab~Akg zhiWVE43pl4PM-klS zb=M>rC;B^i&&@clP>eBdkgCDf9dCyVFi|HAvaIU_jOT8JfTR$`I4077aQCCcNeuzs z9g&<}V5NMNW}&+$&P<~|c@1azO}8~igqwRP zFi4`>v=9!*5ITP+u5#}kGMSiwKrqRopMol5ap*3@ti@&Fuo$nMI&!9mrPK(Kj@wqb zcMF$=wwB`Vt|@WNeJ9&-%15@L!+gY$HnB0ejA%nI8=3^@XIfMYL)U1?lSQzDpawrr zHf_oGf(6Y29rJ)o;a9@QQfJ_dkk;5ioig$Xk|o6y9UL5oj-of#dMh=!7hI`bqVd+r z{XiikPFjKgM%@4{5=Vrq;A5e>e)g)+tm@_2KpeTuJy>26bn+UvcR??)If|tiUDXJm z6dF!!x?Q2~l!FpVHX^+P^>m=yd8Q)LyA45XWBi*WSzoP&!N?X|qazG9aBws=4-7R| zVx@}`C+|r+sZj;PNj{;tA$Oei|<|EOZ|Pn1(VCV!)WHn zY;GH!g&-JFqGF&j@-1)$zA1ZAH#f|T4pew-jGPqSZHd%iXElP>`=GXZ#SUFQsn8no7)n4yb4bIJt=} zPz7x!JYiKHa}l!W9N^Gbf#w7f-@_UO<_pJO7cO1#I4!VBYDOv`#bTUP7yXJ^)aV_) z=+OWo!GJjM-74!L0xk|T@QsyWfs9@Cb7!xy=Gk&`IH?pRWp~%j48Ea;y2vSCT30-} zM$*mZWBs1C1Ina?VP>)j_i!BM$ZF<@4o=`VY9GlOPo)gNB5G1NRD!p13&Lg z;8eV(%IfTR?#E6pF4pFPO&K&%wF6G^+&jT3xhQ<5PJ*(Ce|iE8ZmRS<#>n=UB!X+` z*`>*%M{@V{_k!(3q%wIEnrBJ5y&$2YmLKH3$O4;r;N)m!wgX2|D}3gCdoAwV8AFVQ zc$4DN0{V4_akYizZC-D3L)zpBnnabldSBhz^Qtx(Q6l(M8MXit_D@Qh;O7kmI9Wk5 zGl&90wVUML8@rS+TgV~!B{)9|L&VN#)#~T{LLqo$niY4NFzibDt-?kt?-5NtV$d-{ zru&NAAp==i#uG(&zeUKFtFsdIIv@hlfF4K}?)VG_&so61XlSyU8Rt^j-s1_Q-5}=n zoND6zDrn=rBayOwcAm@44At3(-vO)|HuyKZPzr!NHKeesH=B?|8 zfnwnmD`1Q=4k6s&k=0 z1dKaX4#VtAOEDt!jp##A{sox6qDd#P6_Tsd%e{6?&^T=^R1WA=NF{E8)XLYd-+sog zIV4+&`=Qd{kL5rzvR$J2Cjn9Y{|UzVdL+3pl=R-AaA9$52RHN1?fHn+vH=&V`<312 z9Lrl6ZEMMk5yCtw(wX<99+r%YcjdhewR@`s61r~*0o-89w=RKBMPfGG0~N@{y?(2% z96YajtImtts{(|2#|(Spfn#`#GCEh$K#-3PcLxnkQ-FtoiLnjMdVDQ7F@ggNXKQN} z?rNJRDWj5@S-UKgruCl2FM}-SHc*qa^{`yUM7hz}Jxw}}i`_M}TW+gPhh;0wa8>JY zd{R+bly(6)0(lWLxU5viU)2F*g`5cRShby3D5rK<61+fxcdFVkk`7%BlRWJfumS2k zV2ba^s-MsVTU^hhnbp$rS@&4S>+&Gl>my`Tq=sW5FX!|$Rl$k3le)?j+ni27AaE9f z-_cR6H=V760!0m!2{$&q$A{wa^Y$HzYuoj{S=g4`XW~5uuVfDR_H4&P-JoGZ;m{f= za1KNVLMuHHSy*b`2JsC^d3t(b51l@^2m%BV!U9*3E9?Y2?{=tEliXcXXC>VizKhoL zK!m)yv_{ha_EwZyw7~cqaidqNZ$7P=E@p@dQ2ZXmd|wjFKSbtkkx;xWkS^Syn%MF$ zVm4^3FIj*dC*{gX(1q@kaiwW=+Q}83pmZg_!`3gr0XtdT_h%-UE%C}t-fVLSZoANI zprYr^DO9$VLbLJ0K!en+-&A75SRDWjTDS2GI&|+&X39h%lmte@5^bzUDG&IABg7hN zhW5*9kblWThWKtTWd%@z^jlE{w!8@>)|GQTw15VBnH5bD*u1RJ1&KsQpq+{#f~lri z*eeEvt8@l(5Wb=&U$LQU2hLOEe#k$e)v3Gl8M(CK{r#o%BN1E&CmDhL-urs!i@xG_rA3cv1c$ z{0IH|lYga1_%(%gzcx3p_g{v$Z>4aAsl;i|NE{~RU>;A*V($G@Dt7Objz|m#sEwO# zpUd)}VHdqbmDEYvVxT?R}h?3?nS0Y2Iq=1i?p?0PP(RJsHEmMEiK%VE)C-euIJpp*JOS8YYf zQckYbL@eP3V#Zn=EGcC+ng&$bB|w3wWDx;!7y$@6+wYMQ%}+aex(=PGhO%ssD!gne zhDXU;IX(G3&f${R@poJ2O{!e)$wwnQO~T$N_f=9(JcMq$aNazq0NW1pm>dRnpLglF z4<2bY>cbE%o)XY1HuA$*M%IM=N~L#rl%S|X563V5aHFZxpx$m4DpsCH^lT&4SXk7R zoxBIwraSsip7`_xE(3j|7hpj0I02$TyaLQ*&VvVY!0S;cnq|zpV zF{ko2CX7HF+l}8#HEcgC!4ySUG0_0DFC-m5#GR0Nrj6W(E3yYG6s~Q;%6CmSI%`4$ zK8`FH;Y+d+ha&Gq$?7WMn-5jtv74Lb0)j*eIgpURh*yQ3+iL znFf7b>IS8f0~S$qIFVrP)G``w(w*bgL8z7Widwsh!LBziZLlI*yjswswe6Cew?w5^ zsik=cY6`L_T62EfF%L+t;xt}#@?@b)K8to=JoiCwvhyuT0^sFUplSe@D%}GiRuak1 z&VeesIvhS-#SB45x=TwPGRjKxZab!n&7Go_{XVNHS%0R8PQN|Cld)i{IAHrEz5Ee4 zbaAirfY7UI<_R(eU~X7Y0tEs?52(8}>C|JQUNjtIUC}}7Bmf~}P~RR;hA+VV%FXT1 zJ9IDbIWT@<$*n<%67}$bTArAq@=oto{Di$A{&8%8r}j~K3*R%AJ0*HIhI6mI-w4(V zFw;jRnp4x+0=^^lC{_-mY4yyweNpNie>TEY9l$Qk9iV zoqHD{pDuO@E_c>`K&~(#%zMI&_Nh~V-eXdfBY=hsKnIGR;)DsdHMJB{>%2j6^3H%0 zO<%lfGjbdJNge?GvQVW#EH(2&^Y04nx$&9*E47reFaa22*)^1q+c5fWuOK`}BqRPq zW~(^5imE!)#w%q9AT?N}8{$(5yuSQ0&gORDuknZvLEX!+;eetjH$QXl?xKJv@gp9XID>t6rCekV5a z(Vs(_!z8M{c>8Ji;je(T{35*m_O!b+_FX-3Mz#HdQbE)56hd?3Dz~hMyLeRv%al*R zHkUvuS_5V4(M-%7!yJheO>dbV#(m&_J;l7T^KDtjkbfZn!rdJ1x4rMQUZC55=yWwj~x%-jRzAs;MERjc;c+7s?L?kms0#LL_Pok9#yKC}QSe9W%!) zf=q&PjYU(6)eNuEgYTSiCTo3eByvA^Lw)LE6_1h^e#;w#w1_B&^(&x5a5E($ENO?7 z^t4fy2DblFFrkt$DX?@z?!I|VI~LxS8l-Sd@iYLSgm%q6^&G|z13Ok|@L3Hf6(1^f z$y(WU1Z~Vo=R*O3?4j7y^tc#l1SU^+p&lV=t%#Dzrk6m%b_8QXiZNO%PnPeZ2V03!{J$rt4k(26z#}_aHdd{vd$jgmqTvX6M zICV?5%{>Avym`28W7N}j;mQ$Y9+e%%Vwa5qMZ>fH$U>CAARXZ5~IB|9X28oWtfuTzVbmk~40g%a0 ze1W9cNY$1IS!Ii^^8=P*g9XHcnwfEUQ20G=QD^}qra z$e0c|*?6e;Vp@F4bWz4p)i0cPRknbcOhnil(TE->m5r8=@dWS-$m#?)VSqOn3^`IJ zL4rFp%nHPBO5K3Z5FYYsvkEKg0`D|!#JBixQ^UvSQmN%a47!GIA3qbJzj*zZ!139q zuV20W;r-|G6W?U$%U6o&|Cjt-v?_o3uVfN`AAlD9NMaE4W0lAsii<=a5}v251m@T> zwWh4;C3w$TAj5{q!AWhF!yVv*{M{Cz1b)>7kLH&4F<8DM49{c_DAvH@8}$2NqdXT^ zwCHkJo5H%;wcarB#W>CzoGqx-E2Kf!cAH&IQuTn4JANqc%LhYw(&l$*kpgXM`cBG9 ztlkTUJ(%R@1B(FXwyThBnj zhC4HNkh`(TIjbTq*bFXKI^kyLsv^(-6xU~GBz2MY@-tw*WML7Lvz-Ew%E>l(Hpn#h z&XKg(C5ZtuHlL45Ihxmm)VmQd3Q&j098ct!g7XcaWG{a^!!`!AyNEx5$jys(KA#Gw zK~;SykumRwv?OTMM_*F@7WlnYXj^4Sz(jNaRt4<1g>@oyRiZaQaRU+08k<8XtuHEB zc%%$)Q0FzlOM+6(k{1SBq!6IwQfx0qm9D7KV$re;+x2vnkYsQdectT@UqB12yNV>b z@rH3p?Tb4As>jp^01eX%>uNOo>_vzpQg06Oj^cAAx$l$)-b&?`Q1vN=M`wL$*@iPT zA*LAtVK5WeuSTO?uuqx#KHYWEQqM_tFH@${f*xX(>073L}yB7jHO=eMk(0 zR|(ON?%8evylM(RArXnxs(`(MwF`BACXuHCR`Gz5OMq+Q(R~8bBjonx=ncq9^}7yR z<+hOQ$&$KCbv4t(sD~4xz87FH%n+@URfda9rNn#^>Lu8{fS6~IF{v$lT%Q*V$RWtv znVD_FSf)E?C1GkNLJVc2pG*A6YgO^>%5Y5HfVRQNBQT*A6F=b1Zm`+tCyRZUE3*JCDo7RSlsF zB3W7I!iHLFE&9h#82J;!fe#Ez8z#%B&{BWOC=99XY*YsdmjO8>_;fW@bvKykr2#NX zr5H^%F`P!=rgY{2Q$Vc0?RZLAp3)?bsNpyyxp;VUQvweV8?uB=FyJgMb3QWS(I7}KA| zf#-BAh;kqHGoIyxm?8NNrKEJK*Cc~wF04&tH&nd~MPY@8N#&Ylfi0@(JWp;~rwI&I z1Add2dw73EB{58%XS*Muo4-NX)Ot%GHCj;*I-{&qChh^!T5q+ncCdZZT$+!c17~YV z5~A)fJA;%)M74G~UEnHz2}2LKN1^=-u)^)?m=z^8KtgWifARQ1?VYS`74G!^{z zE0p%Ue3RJ9?u5!v5Xi3R7Zt8rJP^@DK|Pw+lWEk4ipY)hVo9BW=X0P!GIs7|otcP` zI2R>OL4vJW)x)dI&9>qM6=$dr7~mu6d@b{R&aH|d05URV-$u9 z1b^tF1Bq`eZgy-@4R~ff*zcx3TUAlR9#iHq)=-qSa0jQ3g=i7f!S+e(0?^d9xFu>! z4G}72oB*i(>aq}8g20zI)mH(a9}@f&p>SyCDG7r35vd1@x`5LxWPEZ-Ujl){$c6-f z_!`)iMx>+%%D0q{_hP{cjpbotjEitg(LRDsgDrwB-j3Ad9oSAFC?UY+-j?IqK{Bia z%47t$YFE%n>fJCpp#f&KWDgw%$j)%}hYTBgmpw|DSdubQT;#Xd--iatnDC$%hgr&B z{$==+BSij+V>b_xx)a{M`>~4FS8oPck&S%x%h&G$yrr#R)YtzR<18ZT`;UM4YYGgf zO)imjbpaB!`(b}ZS;~z=0)IwWs=GY*oyl}pU!yxa1^06>{y=VoqGw=j(D?ND-J=VW z-BCrmI=~$S{;O=13*MZ zy2B67zOBpAIKPE?J#FO?U@@NlfRAC<^A%;R_!Z<`Om|R#b=tS+sj)vPh4@Aj89db_ zFJfbHTuhxI+uU*&B$E^8hgs2HcBso-Um2ez$A(#&PM*LuUsuJ=;jrqCID@?iW<-Tc zL{5<7NTVOSPZ>I1G#p#BkM$&y6^R0q8;z=Iu%Qbk5EiF8^a6wBpr9U1tdKF%lWlTeOr)bAlU)7O4I%Y?$8IYD9SqSA%L-T zmBT7}q_^J628M+r%U&|ijYZyOTf9zY<-_gW9S6xm6-MdU}Q~`u>=vNzL$Cpo`nPu zVV)Og@G?97pvV&vi)R=P0zFOMz+B{{5D?4NSce!2E6{Sa>`oFQsp>&2*urh0+!h$Q zW*I8C@!|{unoUDZW9isn%!;{o+=3l!VNF|fdi*^qn*8MmR9{lq@yoYA99Mt$_7Q*9 z=mP2k{fPyn{+JJeqeUf-eBtO(?dSetaLo#Im01Rh`4Smm!fdE*4YNI4*1ywtaro#^hFwU!v zFYGf%op&Z0RGz~P(0>=A2QjH(8Cj|mowqH=#e02AZpP{HoD|~foO@7hC<3g%0b{mq z?iA6wxK(EjP=2#@Jw>_jl_f%=I%e6WtO*mBuWJC?Wt8&KJ zT>*T*1Mrm}@U}azp`d1|>?9X3rRX!X*dil6vU{7-sO!^P|6v+zDte{rc)Ll|;-BZF{N~m>C zrwYR84v4Ih`+`X%P|Z|~rR4ZLZ-nQ8GzQ{><0-0IWhU(S46S`P`KuHSXn(On@(CG7 zt)J#@;z+kbQAEBaTnz>%l7k<{w~RN1mxbNuC0u z!yIdh*Z>I$INcL0($IkVGB}ihnb;YVNGfJMSx(P&qra0i(i9fMlHl>RxC1CNdI_<| z$O$5*08J`W&i(~JOeB|#K$_$$4sI}*$k*XNJ)ORO7IohL)zYAqxijw=*W0d$GD`1`s85oB# zE{Kzw4H8YcVNz|!HKH(i7eL|dFm%ah63}LT2Lf&MzV5=1Di3S2htTMHsoOvneHqv) zTi|QXHTjg;=92OvVY!wMV$_1gOB( zf}OG{O!04R<4ZLKw=;9Rtkt1D*@lH`?`Xt~azELFBcAc5o3s3*@UQ=X zq2Y$C9Z}8$=2r}ro5Tb3jRW-EfTHY9{rzYifXUX7F4O}fXT2uG^JscA*L2m}$VoBy z;~oaD5;HlBQBi=rQZLyG%l-zmh$TtBz~>M**rdC=c~%ZBAdQC89G6Wh@~V6O+j23U zJ0Bdwp&c%Y#wtc;H!kR3uE#L5ReU|4IOx|W?aQzk1_?%@O=LO5N)2X5>KfP2F`2Zv zOc#H)MMg=8;emzW1VpQ)GmX?9+4CY==xNkyivhc;=~cEpTp$nu>j{gkO>MRw&Uh3{ zPV+L|I6m7rI95Vjnp$%^O}5Ek2(-u8GWQ;Q$~uP0$e)D*8cca46UJaY>!j{P>t7eg^OSmN;bA)>mfybjty~{+C97ws{(jDhSPBZ zbE3BAK8L5)FR}K+JE$>yPty ze)RT<{QuJ+fBej4UmmWA{Bl50QV%4utodV$x({K!YaJ*xUn>V!lnz5C>iI`?SBWx< zw=g;DG2pA@$*b*m-|EJcKM^7N(|QYw#=^eFfI$yiK;2h(ruM{BXrqlJJ#{#t#ECpG z=bCEC>yDXapJ>+!ZHQ;rG2n*P4IrN{N9=Uc9b0vlr=mI_?OS>zQ! zvB>h6H^T2173Pzyvh5v&iGCFPHg9TlZjr2>Fjz*Elk5HF_~<#px=HqTL=Z9=B=#_- z_4S|@n)gT}zbTX>Wchc{X-2#fl%X7sEoisEx6RJcSR$M;ZS;D;#N zfNW!g?gg5M$!iJ3BiZ}cvUI@>l;aZZFfOZau*qE1ZTCz?w2SPb>P=kq)aHbhCfbN7 z)NT@^yfe{FbGJj_VNq|>L*hggN8vPL!KHKHKGY8p2N;Mn+AOkxJ|bDq05WnCB*BKO zrkN7}5Ze^Abl|lOMQM4QIc!GG!2g9m@Dd$y1I3foH@`;BaXfUbDVN-K0o`zlf7VDz z>?alBsOWKDR~cy|fw-Me5OxNA|E=iT4@5k}_ab*w0_TMmzGC zQn>wD{`|(&I!$>jlg1y(ySb24epvMBJ1g3D_3^SX6`m%*KQKt#@~Z1oI$_EQ7a6ur z*QA15t2e{p4XraxICQRkpucnr9>0ne6E>oHB2eM%YjlV1UTf7)QVMIBXk-V@Ti{_Gjmx)Hcp`d7zg=osl~Og!D`EAx1P9DtI9>0utbr?y zDPDsuX7%Qi1RjbD>C{0HrS4B%Aw>@KfGjr8RHkZGrR-k2t@Yr`uZ84{NlN5s`tai` zx{@AfW!3Mp22Q!!l05W~7d~7~mt}<>BO%#v{gsb*bkEI{t3(6E=j40vz*bY z@2LZZ$=WVo8_c8H+R>P@QP6m`kX(aOT@YtlPikG_K$Llfb*j9EbPnT^v$3e+v#`Pe z8n%r`?blFwbBKq6ajZawnd7Zv1f7W&2X^Yo|K`irUx(*D=JB21zJ0;3fd~1!KYjZ* z(Bj^I`u0ny;ZE;=DD~lNGluv747JXW-hTe}(dhwp;F32|*m&YUo6yO&x+-I2?4(lD z20cq}6H;L^aLcwf{eFVpYf>$z*+MV`!xWdfQ@=|Uj272blOoRCr#CoO%W`I0bAi+y zAcL|;T3|+YO>HlLe!YvObc9sWKYeYogr-F{Aipz7K(-UdWD=0FD7l3EfOy@bO4M15 zoPfBo$F+chvVsst-PhsH z!-8DhY*3JGz;^Atno!}^2i_k}X(Sq!eYo$HRAd+*lOSHBTj*#Hc=T`x2N(OJb5YDF z4$2-BiZNw)G`@ zhLG|R)f=~DXn-knQy&UjNwar99f65oWw&6}HD2(@-0~h1`x`tJ2!0&YGK*p+A%X>P zl{-7-q~*T^HVXChKZn10jxfJxE%tk>0pBpX$ajME0JFxVwCtM`de4&o^P*(l(+MkeRL*}%KZl^Mp$dN- zNW(R6p29nfAd*uQ1=1wF`nyqvcDaUag8eiM!cj&r&V{p2aNeK4ekR@DX0F2#;z+9BY~^-K#)?bCE|NHMbMEGhm%T2Cy^cNBCv$K~M@cC5(I|9fE7g7e&~xz3J4iaKuvo*s zV5{&(JM0gBMgtZ$r+62209mC_n}=)(=@UM5IoQ~g8}ed(&sIuE1#G&?K!cC%kgRKt zft+jzmCEgI2j2nS^)s!lU$B?k@#2z~3YvF$#Lm|dq-8x=d-EZm7SkcsaF(b4q}niyv0&)f81EpkmP#A9 zxk`blR#a{2|e(OH?85AQAuoS8h?T%ns)XUd@M&_xWAYG0{+mm zzxkW+y(7B1h}V~I|MqnC^*8()ct9(nzvQps^|$`>uOGvT?%$!Ye+Pl@Ymd%vf5e~R z&CTh48xEtoPu_lx*!{7|i#?#yYP5k}ZD@=eFgvUtJqBCcgto9c>C7CMqc5ez{H63wMjD%|)!mxp;UCBLwuQ)(#%SdV6xMa_sQSw0L+ zVJ@gDpvnel*Uw4EmpLTKGhKCu0@jo!c?bzb>6Na1P-a!{(uv(ShtGh!-9J&^$Fr0l zxt*BL>$`5%P%cA_M@Xdv-@bB`$HM&+{2NL_=;K|RMYU|dI`>0A#*GHbYbVtohkw}X zENWbl7`X1pbzo+5RWrPN*dEI4*SePX7gX|-gSUZ0u&W0V#Y{pg=K@m&SbPup)#WVm z9J%pdS-!WxN>R8@Qk-i<2?h-C;+yy4tGdDaOLrnJFtDLb=ytfmL4n?`%JO-)atpWV z!8tjKMM5SJ9C_zOo^edCRz#D;PF=2pxThoe%f|zpEja!Hv+j^Cma2gk9QD=6YQO{9 z57%5}wW=YyV|8#G#+-Wv7H=vfPpWEsP}F26)%)!-$8|X0885P@ohrINnbs9R4O`^6 zNdbSZ=lQU#t7V6d7TI*Ol%`DP=3Guy zueAX)9;7yja^0fX@e=0NwMv;_uTds;92-eOyNM7vP=yG&FR~#>#WS=|w%tz-tbYl~ zj1xYOkgh&y8*vmCiOCj&W-ab42w`Bs(Y>Vp3Sf|rY2(BQDVKzs7mGXU__K1g#pc8xm;w9z>I68yC=90sZ9#Sujugy(*ays;JYTdUlokMHYBOW3Ugiy6#94LYu1v z2BpL@ZtKw=OmYM45C|q~mM*oLL8vYJQd%Pb4m!1~nZ%nmrFN;b23`ryLIX1(M-BPM zuF115xbmv77jciEctR_xTTZh>H1RO!kN4dM%RSPWF*HMa(&Ej0Wq(`6ca4+{2s=eX zVDxGq$#@UTT`Hf5LJ@89&Ut4u`&l60*hI^f42CXKV}^}J3(Aa^mvW&>g37ptSna@j zOnh%Bs;_lc1nG|)Yyd-|W8D+#1;eN2D0$==q`+iqL4KQE@#^Ny_Pz$A1PzrFyt1 zwq}z&vC1Byz55IIC-)XRA+<3QoW2wY|^1E8Q^rG7)EuV z%MoN-_G1@=Qm2~V(OM*5zYY zREx5q+kTGD$1Go-^djBT<_kg_fByPqc>5iXzmQmWdY~l99sz9a zz?;?Zhw_oVvm*%KaX1A$!PO2INF3_~3(!MLL2lrhjL@v;OkHxDoo~O%T{*+$*UArN zno&Cj2!7eqt!c4#$IDVPu$R?gV(18B&$2ldHdm{KkdF9qrouWx1yxSRq6VciUcF=T z`%WF5hs40Uq{C+5LX+C1Q;;|V+k`RaQon>Lc`q&_D^yN|Ds6ICACJVtt>sbEbc{eN zl*Mw)fxcq}uQ-!LR^_%FYHo?WlQCVpZ^4(wVhKuX?U^DxA=i(heGKocUeTDbx{3#& z8?wph3WQgGus%Q%*5|~H4l>BoLj{j=B>Vxc^>WPvGefIm*uTSHm2EXdagQ&i@9&3L5F9%VavL8NA< z098sBo&D0*Y&Ew~K@hgy-qJN|%` zP)fD-uTbT4$O23`|jMsHS$_mg7ixk{mI8V7DSMN)nq3 z`^yD1kEkxWNWj?UG{<&>x&ZlRswZlP0d%xEY*(KF#Bn2dM>g>R$!f=88R8zCRWU(r zYiLGpjfa~L!_L!%Z+|_4)bHMYr5KH8`QzWXSg{HGTi5LuuRle|k{^eKz;hq(zrX%U z>1;m&ZvRJy_U&ovO%?)B?+5@L8Jf{kd+j zaE667DU8giZ9621>+xnrQtP;C2hV~UB);nC8;`nSyM|WLZ`ea@>(IH^4CWe^eu2@J z>w%yzWKHDd9yXBA#?=|0NSn}tO%(b^SuJZpvK*T+p|G1FH^-_kE#L%GcWjZpy`qr= z)KiCeMNx;xo#1{fM!7!=WhvqRnOvp|1pGO6CH!qD@-x>JoWD5p0zass2L8JZS{YL4 z$#zLpqHOv%2T?#TB^wXRYfOTw5yb1}jO+JxayC-jCfiwG;u__+gJ3 z4_LY>8d#@rq#hNSRchS^6Y}m6ww1kCH5(lw)UCjgaOT4Rb8%A-8=^CF2kIpb*fVlu zMCuFG;U`QhsxWWa!1IcG9rSnW^>n>s)aJ;1Ele$@UXdd?K#*GdD$ju~2Add=``5vd zMNsUQ78O_ZsJ5^uwXGIUh&aF?oq*o$oZHr_5=tTkEDQtfW!z{c&N>;p=NS~6j4s;W z0&GgOV1x^Nh67p*18tJ?7s(pcmceKJaF|hol10B6!#hSo3`Z?2Vxb%2Kimne1Y!F! zemGm2tej*p1h9T@{Wa$ya?jW&Wryi{T$<-u^o z8uhrhvb(H`|B?w11p-2}JfdD?fv0TpS+bolBP>C%kPHD~*DJU%luvW2Ze5>G^y|br zE$fS+{3>k&E6{!#q2GvR`e-Qs*`+cH;G%%)M}A@Ij`B|c$y2{%IzqlEfg2(-E~+&* zNE-iyQCO>EFGk5q6~EvkAfY-(H%!Ksc6KFZ@C2|qYr4hA>1N?0xCJ%TxYTHl0W-`> z^j4mCn2v1;mFz&%(_TJr`wK8bgOfp(We|zHkX;Baq3D_B4dL(p;UB_3{QVIZzj^z0 zFjH~LhA`Tx=w4?%&U-w%1&{-=t@PG?C~Jd~mRiS1l5e$Zhe9GED^$@ke1qbLGn-d9 z*O7EBcjbs&pcF?1ZAe^Ff2uoWmKXU{ej?9U+zb$l`QeswnDvE%qeH(5%k#Uj3Rkcm{AW0$=e7KwGTn&OMK4VHM_ zR1(t88kQefzW-2Xl*KH+agb0wOw0;qfNw`ev)f4DmDp#XbqyGiWp$1J#b5+PBp6k)r!z2XiS{qD+zl&yV7vS`1hYizG6*S=> zZtTt-NU0WRSAfiVZGT=jn$F=Vn~5EPjj=@EDy^RKf(>Fv-MB~~Zc-%?VQ0j3PBs?e zWTACzmZJu{J{x9&mdz+zJ5uHn5tI6!Md?<#y0PIbLcDb!5V9pb4Mg5Iu=Qhs0ui-S zr)9H6;kB0VK1{?=Y?xU^22H57`7xEE0Pao{%tE)&1g^j}^eRrt(#^JelnmPlS*&i7 z$Eza_0gfl##AA63An#=!qjj%X3xiDViY!s+q?Cfz{fuhtE}vShlpIQ~)zq1(10UFG zhcU=JQBD;o{e>G*m70U*EQQyoRyJg^L6%NRAP6Q)`O{$5H+mBf;*ns6$gXu?6uy6t z502L6;ItbFtJov*aW`W)wGA6BI??)^CH9_WFP-C=y-&z8P=qpOhHfFLI$8ogQ!O(0 zVWD|LINV6%4*yR z(RU)D&Yf9Hf~Z}^MJ1)J7?jOofS5T8e5enw5Al8bFMQvE*E4wjZ^QRbR4bwXTX+Uz zc$9nn>FbaAHSi#R_tV!ugx8;-8vJ8;{mb*;`{CG2x@On>MJDVBsGZyU$}u6V$mLB7o>uBk=RDd?ee z#n+)$JR0qELent#NpyGXvjMz|TkQkGRx%Cp2%0fvt-xNlIA^R~0fC&1&8DC^z#tE; z$s6+GSU_eHLXlJQs?~A8*BJsaTC;XbHb%Vxy<*7`l-^$~j9%~}7*w&nsmLgYdxJbW zE&?Yt`Dj?5cAuczkxuOv?==*4Tr{(ljXhg|SlI0SwL?4)Ye?33_!#P( zK#g*o&>0Opoi|1=)3f6i_%}qpV27y-Uw{^Ce~M?|SLM1*5vvWO0}hK6$$*{eThqbcy?yS16^&O5y%aM! zF65v3=vTnuBzI=q}v*D0FgkYd(3KaUIT#wv~V$UiYs0r2bzF{#YD zC9=ng$*xoX`W`g5w#bT^bET5bngKIbj^HxbF?zeZyx7|jN3YvZ7`kQ$>=LT`XY1nk z=@y>mK)rm>J`|N@0T64wo-o&$quTT+?qRp)Z-Qr2>2~L?w*@`_TZhoPwz8`-Gr1xT z(9_^*!as^4wb*k7XlCG@yW+;JJVxu0-|)%dRr2PVM?KAfFPj7Nq!_Gy;GC;lh^Qqx z=yYTIv_h48U@-kZPs{@8{2GZIX;-hMuVZ6xy&W^Bv9U;gF_qDkUG-D z^EQ^>F$G8ew{oU9=quow^ar0@>GmIr;GVmJ_LfRB#*WrUhJ|Q{B z9`EeG3PDP@Ups}up#yUI9#diUdn7&K$;JTqNcXWVHU+~u+z0Am28S<@`AHz1tISEO zRWq!p)tY>z#4ai^)t9#NH5(%XiM^10-gaZ9N)jcJcCa?DoJ?IvPFYVhOHlgYlfAa& z3zm4p!5lam$r}JYMIXX`1op3(=1Pbi6ogad`|=+{mQwkTdiK5Vg}?v75l8>W+rLw~ zzJ2!kX|RKD@>liB3vh4!m$zSsPX5daE|0vc?eU$IUHP-!75%fEUP`rYl+14@K=z)V zbk^@uJv`_Ir0@eN^zW_p?C)W-K0dg^JBm*YppuCXZgvG0!D*HM0{+Mm47&VByp&Ku z%nclTPpjgckHGhCL-%yUU{$8)19DfTZbFw*ciQVre= z)T5iu5D}Kp#d-1$9Q7AFu1DU2fqo!8WP?Uz3>e8)safbWHl$r`Ai-Lw1GP_PbaKL2 zl*c(dk({)k3@_`qb_!nY(@DEVt`wwd;Ozb97n&`eA(~|gms)yBJjHm^)sg!eQ$7U_ zava%*xs=jym&cKNcoZv2}|8d;-wmk4{jYP8Mn?Z zC8o6fQ{%M+oKk|v;}B0OlWzL6-O>VD*|PkLdBdE$n5j#QIfQ*Qg^r|AImu&t#mo)U zy>4Xp5$F;d=qb2A>GW-u_ue>(OtzU0nGlnhI7^&&)ev+h2(W}oc0L@p2?^n(wl|1& z+{b;|@$LHON#gMfUj5?j7rE8=3H(bx5PkeKy#AO_luurNA6`F!V)9SJ+fUVW##PDX zb4sQKa!+cxM*0^dcMcCy%w1RSBR;|bSHD3zgL?$v*S69CYIoNfsWKPFYWY~Kc2(oV zrP*AJcdI1u<-K{Xi|q2mQ-w%BkqlSW)pqEvEJIkIF779D4wcEYtwc+&TH~_eh<`m& zOk{xs;!Q(pgaSWzXbi1}?6KoF15|5Rz9AH>@KJ<7GMazEVvsV;RzNEyksd60S)g*@ zJHKoN!@w3M!0j$f(y4AM>zZtVjn=*^E=fq6RIA^oeQ?Jk^*gGWqU8m1jou;J-yGzf zH_-2iU2a zRHzR1PN)t33Hzmm8L1Rl^&}O-(WzdM6y-9q;62}9@L@aD8PJrJ-{Lu`b>Uf=c4wQ? zSwAx$k4H`^)l70?IQmKwC3`7eR+qy9`P<7IpaZE}fPUIU4$p)BAv=OC_JP1_Sa)o2n>ZXjA)Wn%?WSS^YBT@ZHH4wAUvB>QN zB1J$|H{8E&!VGq3?X{rXSeTp8?ZEqVl^=PaZAPy>Mdfynw26x93T4}xgq4}@i)dV; zu7O)2e-UOr>?ylZe{P1=ujke-+Z^Fyz4iRdGv~ayk zB+DY!s!YvI5-G3ASwJ+>_3I#w3bUCuQ&HVT1~Xy|W=Ijhilj^kl-24d%}IHYFmWoS z2w9HTC{O3zv5^Qd78&jtWU`^PbHEoQj6s)mR_RDIi<8rxw{?WJXh;Inm& zgK7|kd={r>JWm_w(l}gifusF(@v{-H{OEx%)pLxNP;=AV(OzqmG%N92P-TN60rMs) zS9ZpwNNK<+!SN&aGN(>0R9vk|cC8%U>}-_C*eS^=hZJP{_KwhFyQsaf$v~4JYU-!KADU}t;^TqdG)Rctvkfunc%rf>^2ift9H9m=u%y(5Yt*Q0SrW!o1^$qP>#N=Z;bm_s4P4{SQ|3m(riOV7&lAFk|nW*1FUADp-vDj(W%D=D%e$&HHMm~X(T>{nnTO$-ab)9`rs<_4UjS`S)FtPBOkIGT{$ z)KjAVq48YI8iq^iDZ-7UaP9n+JB);hl~&oj0&EFHIApbg-lV=DD;NaNaCk|3OlCtBM2EQy51i)5m1gbjKPx?~5_dZvmOgg>H7 zX;p#tdT}VQOC9u$*g>B!ouEL6p;QC1H=|GmEYsm{6E2eCvmH$A2eq?)eq8mQz-xj%dT8HGRoVz807Z~k4;53Tj&XaARc`pdVk!mD|Z{sN@9uipOx z5%CYI1peznLr!s{LSpf8h0uRezv)~!V(d0 z?-mJ3^p-6&?2+X3xjUbI*&<=v3WiE$Dbd3XH(_>htm<$!K>|He zKSXNI5|wJ$CU=pV(!EmUuxGvHr2Xi)SvwFh(vTtNehW3B01!*AbP_qm>kV8fYLOU~ zGLf*C&ju`slhh+}$6J?~8281uZ?F*LX@}sOox8sDs7a^<(}o10VS~pd>3kAUt#HQP z_)d`^T~)Ch4nu_GAGBu0bDvPNV6mdz+?4;CnW11OA zQ|3o)lB_{ODDwo6@4y_}A#zfOjoQQV^kU=1V9G=$H(4ScKRf8D0ioL#7h?o$I`+(u zkisG#G<%ux`B|j};C7^TRs90+xam%feLP$->tR;(MYuvMnw)~3udmwWX&kNjSD~;_ z|HDQ7;3V5y5m6~XY$78o8MakD^#@O=l4ni=u=Qr_5{AP$;wcAOa*3IRBh1-u`X~M8 zVNzMEYXte(lCVd~K$d+#gM$n@_(O+c?9r+BLj~9DINW{mBOkcxjzHl%U&1DFj}_4uF6<(UGZkJf1;(<;eV5D_)IwE@C26Q*K-Z) zmvC3og)0NA{)-~-WsO3xwMcun(0Nm}YNhT6)!FV21e}1*EIWrAC}j(+&KAf5(Htfpz7oXb35%+sx^Vt;ti1GWHa z=3r-#nCuLvF|=Wl7$9Z7?2c3nmN5Il*auqgtMjqsreyzG0_iI-NjF+$4z6S@VF27` z>s$AVWXXpLW+1FZjj{%3c=S+ubR{Jn(BR(j*2#P2V=Q|TD3?yVy>1v?T%ieRw6_EW zx|+w5X{d^p1VAh4fa{7K!P{4ag=&AssDgg(yO7j*?I4NUxpyx4CAGxCO6Ftuy-xd3 zoMQtSbB$Eak8nq;I)eO_G7$m)Z}4X9DoWLr1IKQFI3_x7j;h8TdRp+xu=C;)T3}Mx zNi70-u|sH4KC@1JXs#r^Ayds!E0e5putMfy6`ec|Qss?-*wu5C{mO#a1XOa6!U_#f z4y2TiKnjEWUaLfv#%&bs?iP00W>z}?u)bY1Y^8Or!mjzt4 z7yU7hKYsn0N-n;*;2?>Ivl9{T1ofW6_%PNHEfel_b@G|8^}4MVuW zR8@IBSZ~MHu;uA3YZpnF2K+(%(~3+x9)~c;hF+}v9SO?(POU>JI}R5*lLmlea&*Kv zwv?Le>I7-bYDZYH*clqlZoep@U42_=~@TK^LE zDTNnhVQ_lehBz`Zk6z#oZ>k@_4mNu&UnLW;w!w@kXgOfSPVieC+#wxRKg;29u&K&J~s7y%b%8q>3a#6500TctS0xVcm72+wf#Hu12!z3N!2v;y`sACaiO54)+uwt{!ajBH4F ztcGS-c`9q~@nC^Am=_JIG?WBc(F@c<9LeK7K)y~g2ipS&B`FcF4}GjtzpL{3z)-6# zOgljrHb*uX6pY@*y`pHgnA&B%3W{b zELK`jX%rAr$4$bU`YqLt{D&nW{JCa}m#8N_ z5Ve~sSM{+SyCa}s^B5S&RH({lU7HI!m5>a9;^PKi^8z=s@vi4lVia-@;Z$US4NU7B z@d!mrbtEa^N8FAN?oc6*llCB;N9_Y0Nos+5wfs;C*GU3Ve?0h6MZN5}amO`9xsMl6 zq~#%DQ}Gji4d1@N*5N1o`rJbN;_Vmz-q-N|_XBJ~eoeUF*MN)sOhQ^#`Ty~g%KxDz z{Bbay5_EA(laJjo@W?yT)A2YWFs^HMr z-5s+IQ7oUG0RsZaM|M+_PtwFLAr5%}0Bdb)$A&2}D*4eV+0fOzcfA8O&gR#F4$Qc! zvD%XWJf=FZ@S%gxG^c7Sfi#sHlqMgjf)S=8{hnj2sjK0jv$!_aR3kY=mxWYp9UZUQ z@EfpMI&LGN#VZF;;4~hXa)rW~Z&JbwSc>+l#%q#5`XJbDRLi9KgbXdd(;-=@Wi4UJ z2o&q4q%R$RcXBA(JFPNQheXm}%9;)Gl+G~uEU!@tpzKhPg92W~kB_K8@}_L~Agauq z4_WWNTL1wY(+YUZ;36XOxI^!u};#{=@4H8Yq;z=9Z@Jm&0|p4@?ndoEADV9pCv$e)FnOu&cHeBu{C=b%zPm$UrKqBL%>7s&J{R{Yor=ue1v*(w@pCh zw>}De38|c6xgpF;RG;cUP@)Qf!43nT}4g8O9ABQJcHqvXEsFkVYB_-_5?aDcHHRPoL zLqNR0(7wPn043p8m4v-uSfrdsd$Q9m+yU!r9B2UbF8Yg=N`@qQ0x;ZN>z51Ddknd< zi}FdO(|ZGio(ka%axQm8)Afyre_#mEh}?|s?`Uys?W8Oa2gy2hpx-7kzZCt z1uJ___kAV1x zl|0M#Y%C4fL=RMAA<1PVDP^r;WOyg)XstP>UZFWhD5H*P2u)aX1@6qwHKHpM&6qR7 zQ&|Nq;AQRoz7o~e;!u8-n9WpR)>v{h!_A6AyIGCf-FI-Aphb}?52#R{7~DT;!H$zV zWDJ~fo}O%j0X%6tE1}@_4kL5M&Kp~Ny+Yzn^Hw0)le&pg`iYJsbAqMpx}4??4(!Pr zdq60%0xN?t2(prWSYJGD0%cEu^9kza5=XBb`b9V^$X4viRWbE>DmooGY&xFZm-QMc zH=SbYWeG|Iv_hbZ1K<&5n`|3{RI0+7Y+&OYr&XIUD*>PGNz-MUh-9OY2D*t|{`-w;rc=#Je#rlgL&@~4OsF$tSwSHK z^Obk+tfU1vAKK6iia-SgTF-!_ZYT6hIaI6X`S`txmo0f*6mtYv5-Yi*w&59N_|N6b zdZ%@nWa9K2#Y1BHqzJusG_IC^)@L;lt|_VfBxq7~lX6Jbr>quq4}fmZe5@)7b#gy| zDI0hNc?hdf)Eri8j^22Lq9-slX>&kGRIj<6!50a3?N$Q7(qC(WvA_=QBb$m@e`g_* zw1CRXe5i64#gis-qmCQ2les|8HXjG$ki@M(i|hkc%y@0(pUb1PSly_kQM7J_`U64^ zfBo&RM~wU`JhR8XeES2ZSk4do0|1SmzWv$eS}@|;iKE;Dtwc)z^gbhlhKM(6Dj~fO$A1HyE zk7ppFQSmDywKC*`bw8UIJqyXN=}f2Vnu4Q1RDr2DP5T|qyzlB55NgPAKSq@ zlN-&6tWLnL5%iJuk-S`#kJ~4Ps3c*^;_8mG>$&PX%YR9w@)}50Ee%@i)a8JKt1>i* zH+cz)JmBeCwr7=AzHz_o6)W%50xe&9s1@6{1OL*~-$5fkN*UTF0NY4Z>mkZW=!yIy zLLgrtt&}(bzNBzTO9$~t?37;eBMU}r%<*|MHe?rDz?eG8v#>`JfFfKZ}ld_dtq(^?o-P2&fXSG_|R{JBekOe(z45F4@rs zO%L=oo4}}^zx+N9uH*F%)rW;8EyzcKC=g(VAp8%~+bQaNq)MZp^R+;Nuey22AzYssA4Pm^K!q(DU8k=t5HIIpce;Yw36gHG!p_Lv1$^ZP>ZS9uthq# zsTQEXT{=4!9s=r4kIruN7LPO=(m14z0|v6#bV~KiXFzPqmTd!{x!o$Ly^IVa5Ve{O z^Um?r{jiirM0kQMK)c8GKZvT0C+zK# zcf)4T2kf~ODPq7z$}Zcj+UR1FP*ChZbwQP+iMA#>5P5YwS1Z`&GP?2_Tt0@r0D<>l#NF|L*|>3*jPcOhJkhL zt(&+}lI|=If!v@|M)#o6wvsajx}KYkoYTt=!NGij!KIX0vh=CUD4SrjfoOmg#;ySH zx~LU)2?f)3s$Zgai4}ktaQpr4*BHJM9$P~j(tfl|vdXx*RS6F*n~fWMz}1zlz9x?3 zP>pXRA&L=#3O;;m)glvsFzS|K8rqE?APeQ`k}|b-%LA`CB`t$p;$|nH0Je5)mqM)+sb>5fJuD0tJX!b_%zN|&~do#gr}T|H5`bWZ$~hkvA1C_FmR!lE`rsJ&3BUWbl%)U1>HWX{ z^@rj8zrOzF^azDEQdO^4O9TtLea73#QE>j`z3rwcsmqBi^t%q{?XKMGt{qj8s^;Vp zJ&XdLknpR~Yq!m!X&z7>qj_ZeQh(e|+w&)*Jh#OeS~?+Nk5vf{M7h*T%6Hr(CnvJY zr0p8K%wK_>Thla$$qK{`I)EsTfmfdqQWxR?l;<5;lP_6^Pz~V1B+2$Ngxx}|+&IZz z>FlmfP?9ViHk(V0$r$v9c>!!w^#9r3Rhf;hl&B2?m1+9J>1Np=*uZ2Rrrl%Gp5GB8 zG*nBP;lssS)QJgC!!0goh(f0dQWeYsr+8RrKuIL_nm6BZ0wY<}r54)dmQt*^thXs) zix_GJbx0lQbrw3Uk^`)IAbB->SiI{RTVzzDLlj0-5rm^aNeO}>l2)K{QYsqD$pO;j z6#)To6Q|9c>i2scSGkwN5G!QI*o0M*n39U1rZ(PEyDC=HYAd)|HK*&yZirI47c)D) zD%PQ+GNS4=)92!f8Pt#-G@;Y^drl=>_i?`Y07Y#R@q;l@&^3V|eaM_kuQUvLl&I`F_!TfkOW z=k9H27ZyneV;Y2mqGRt%Dktr+co$19)a4dvxjiV93#59~pU+2jw&k`%|eZO zbyY9kTyUN9Ektvch@YF5ceJS3AgIP*IB(}Ou`QC9?E%R)=ao@K2`P^P5jH&4y_?3# zvzsyUv};XrY&5~?}9#86SvkcXWB2Yyal88>?C;P#8dB476^F( zr8E-9bGXH_qZ(EzR4sXdJeNf98tiUh1O}45fnM4Uo&c&+NZE!*Vdrc92v)N4#@baG z#t0=0^yYGSR6?=bvk_GAHBsyl{1q$9C3J04W8)KXEtR@+i7l`X+71rm5jFS*=BY>L zTE#=-QIlW0YAJl0FRQjP%C{8h!P_R#KL!+|SCLo>eAV!7XEbQJouO?(&vGc}xaWY}pe**}&4(HeR7m^1LGTyGMf)Li#IG4@Is`J`Nq$SoPnmUZ%2 zpO4{?bVKyUzMM|7^N=~AX(8vCsxY<9)&eUa1T(5>jCE*l#tbRu-Cu5zdZ4+i&x;mL z>uP=l)O1`txm&6yGGifSeHq?uZ1{rY<(5BjMzod>j%ZtSyY3Eej`Jw*ev#dz=nwHs zS;YOW;94mHVDeC#hE42uFe0m33gPRAetfZ&(!wxE<=Pp}E6S3$ZTkIB-aZL$-{9-V z5-jCRf~ow8!6;5LhzL-A>uc<8hkS?dEjx1e=H=@^~=R#MmjnPKge z%&nMrJ%D_~iXdaZ5t#3KdbmM3aQrm(DccWd_IKHY0Y~F-QEKrUMh>i8vz8#OJ5>Y* zvdMM4;zh@+_ne`@!!S*P*CIhoTPf{<7`iGleEDF|2PnwZ2ca0XojFombfjRMQQb@{hxUDf57vWK5^i}S!G42VrFQctzD)S1L&DO0T}kVUbBV_E zq81AUZ{Y|9PQyTgKq^61-sEKSu|a)4F>f4Q?t197&yP@hV{C;5Ogf-iLtMHegdxJl z5e4xrc791#1_W0&b>3NOM9~l*KB62ZS8s#648;boX?z4i8u1xG)4O?BBR#f}gd)*F`>nz5*ZKg{hI88tFq9V1~|667x)05!gzS zKuH+fEjmE-kmeo`%|sq*Cs5rhT>1=HKK86-TaGt*Sn2rLU>!GG@3Q`=g{bVH&>h$* zD^3>WKMwkBIsp%(y0x4_0=47~2MGcBmFRWl%A?}0w2R$A-UUJKe#mZ0 zv{PcwsE+9#qe@w^vNzLEsMyNZy%P$o6iJx}E8Eozdv!pWGKz8+Ssv{=eQ35-xDvU9 zjjl$<$$wJ{=L-*%IOG^22t`=FU&!5wPqSvY`JldCwO6><1F0iH^5*9mCLdS{7~0yk zDllpR*e<^UPf$(94jOYaC@%?=<)Ln=)CuBapKBlILN0+K#&|>}r$--80(M7I4dioe z)GO;akuZ6NW)~`R`6C=T^WTT>$$vR~=W_e=@2IfpFNX)B0JTbnf4NAOI5XJkW@$Y z+L`xgoe+U5&qQ-n#S*d3es+Df?Ov|(N{(_85^1wh3zjZhLbaWrV>`FwQENDt*pZaE zp>Qkskwxa|yNC0Nm@hA;cpa5DmVkdV<9o zM$aawmgQdE170?N3k2Z!J&)tZjJr3TG&7bB4w^(M{LVdfd^CZH|W?7t3rG1wT~k0040uYt>X}sjXe( zO%O78fuh*);B;uO;!(7O3fP=xeDcoew0|z)TC$^4r`8WSYd~qszB@KS`BK(_Sbsb_ zY|AS-lhbzyq}#*PVgW0(xI8rpGKliNM}n7NB2{ysE_tglaDZ~N1=}=~kdqA^^fu_+ zk3RQ?#B5z>P%*v(A)eGXDxP~lUC#FEfG933z^FvGthtjCZLGr9Q4a39W)|`zAmV6y z&1gB2SgEy;B^n^#tgQw;EyWvT6Huzp;dn=np_~BqQlYZUR?wMU2#hAAYJN} zb;_Enl*nT(c0l5woei_@i4Rc$>3G1M7_w1MEO({oTo2qvp;2}5Bn9PinlzP}*TXRv z5T!Hb#y9&%i@33AQK*zuPa<6bwSZBLf51@PB$Cef{F@_A%f4mf<^gpOWZ}>iC}tg^ zSYXBDGNZrda4Q%e(?AFxDZHnnZmqNW@<9Np&M^^&p^`&(CFPf(Twn!PmYVxcMY$th z|319_#LvFV68G&7uU~+cs=@=m4X>Y`w5|QG;q`YpfD@U7p5Z$l!i4}t#l?d76=KRa zpb+x=dXc@woki8{ZzKjwo(uzONt$Tx6}6-&PyEuN#;TCw(|bhgT)%j>i%L5|0=!Ny zsvI}a7g0f|mNF>a!J&>T0C7@-OXi*fOF)-?QKT)MOy&*OawkRW15ft z5PIe$C@hfA)>+eVkj55yLLph)xoJEqc37|a{2jw>xYKXFTeQuZy`8iSs8_)c{Pi4 zi3J5CHjJBMUnC0U$y!?|pipH&t z8;ThP;ohg0eB+MlUkZku?3chIxL8pA z92`{DbtLdXBFBBS>d-Fa?vCsu7!X^i!*Gz*ZoFOrTC)wLxY&wExhrd|rc4~{3_Q*O z-2;sksaK_x`8I4L&PXcjO72Q_zDtW_gtq$RQQbjMkgWOYx^VCawP2S`KFF@j~oL}+~b?)IVW(l?ZI%T&|1-=5xo65hW0K+^5&{1ZP4Z(pDQ{UpdAKRG=h z4IK-FD1GG?uPWES2FsLi?v{4+}PHVY+iZe z9O{CqCwuzY8%+@L?$F>|)17tWJ`^CHS&-O9kYD5rXFzI$I27rms0qoV5WH%|vC0wD z?ZHT}us3XU^jcI2jDV(2YA%aXNi+JY4P= zguyN5fPYHyXk~4}&9|}BD|#&WKyfJN)stTw_%qNJMD26G-kCo2DtiN|L#q}gz99de zIj_DZ6*@8Ys12y$44v46p2HZM*7b)bNbXXpH7cP!2f;MxCRabXB3^3DvZE4U5G|o2 zX9XdY0-u~fTG{4X7*MvV&JV!@#>E}^ge9IWVC1{cQL{f&6@4wXYH4&>HHBycTnrmr zIakhrw2)7ucf$_=x2Y7TF|;D$BkMpY_RHdEuLdLb-A+D7{+d>8Qwh)3fJY{Qd0W$bp#YMTkMtKv=eXwVk#i`sTtfR-n>qwNRS zK21980P+PlF19YU5(~%8*!AVPG<;o2qMK;UHoK}hPL&1@5?<0buZrn|%Q0;%bh-f(jCLqa$Bx=WxAK_Xyy14aLsmijQ z*esUS3A^>Nt6F<-r0n864=kg*XFjobCvTNJz8^UK^$UyBZ-06F?He%5I5@rkg(=y6 zN}r592AkKuP*W6!1`5i&%f5WJuqx>mJ9Rghj_K?+c{f0UWv%)unAKjTo>Tpy!>sU- z(MP<}-@djphYf6aRJDL?q8?)eRjju-Y%V+5Q|Y85mb`=d0JiR7;vUYMD=H1zm{Ruw zi0AF}6A6zAAU%1RB*is{2dOU{9l6$Mf-&gQrn6qNmd(Rr1HaVuI^AzY)Q7!CCUO1u3BLzI}mT zkDvbP^~-04*Duw0{qwg^-u|MDn?Hs3pTB-kLcPlCHliz(o0B>L*P~jBI&-$kZnODp-Kc0U z5mM@ryP1<>_+ki37$V6^gg9VT+w5TgT(7u56 zJp7TbBd7+vU1&BhG-|if5dJ5NN|Vnl_%_$YzF<|_{kQ-PvGTYII{{0D?C3dAcQb&O zp3!w9&|H-l zoERDrd9Big96dR*^5LW6=^R>BC9npdhXzOzOx0IXcdGgpT-g_3+;X#phWX$MNV>%; zba5n~aj#RzRaXb3w~ekYMCujlr_Dh}D5)C#(>X*KJ>!%ORG$_UJ%n|4#muTrj!GpU zkz1dDaSEVihxLPgZN&UK0Aj=Q@FHQbIzyu-@8=lsa7;LE1tis_0%#q=gDl>5kqG%r z0(DWTvG}S674^0hA2b?C=Z%Sav=#sE2U3^){eL*(*B@R#rsUAGlI`m!-!ixV#p`dw z+s|QM|MpWdN&i#+7WBtIRa(qXpCrj&e|&muPA1z^6Yl|CwjDyE%-&!`pa(CZa1?JX zN&dXi1@m67PgoO)d=m&-oc`M1UsD2J)>_&XgBs-RVEiBKNtbzYmoG2leMd9qB}!{lX#+X{Qw&d z0*$#%O>TJr|G+vV`@2pwI92I!x_J>&fkUw;)=w=6saJO>5zSw*%DlJdKWRhXD(_dcG? z+)}X0|9PD((<$lt7w&^3IHCA$b~;k`~0 zpIfMJNLk-N8P2OENba%kiinm5ic-|U0W*D7CWzx>9FZ@zYVDi@&?dLiVi8(~4GAit zk$O*?kVi}Atjb(LN^U2PL3w%`IQo+}crnz=yG^~eR<7E*R7F);Xa|N!Dig!>;zO!7 z70w_Fyi(QUeAw#TOcpu;x9S&A9~SI-B_xz}m)czwwJ&ckuAF5hFSV0NKj7giRc&-k zjpUUO7g;!%M0MoG9^e`U|51nxt#YrpuV$(#3rYOvTCb`h4* zD#{8J%ClhO6B6yvG;|cN{Mt$Oq?H|SbWOW6BPdjUWi!7?NZ+i=fxchat~%T7#&;X2 zF#5`v#1;w7)oVvoQ1T8i#Dx}GBdcGwl&Y!>fgtVxwjx3!a2vg5Ef6%+D=ASYHtJZ{ zJEwDfNA)xFwn|vhM$rb0BFA;A}_^`*`{nB%jqA(KmMbBpAa2J-mfWtn_WMC zM-O5fcYgl%OPCpd{`RNWFW_tSZ#wi;nfU*2mH6=fr?0<*zy!P}Kl-2l%%+q%+&^UZ zEzvTsPy2RLN=qEC&|fs$@Ewc8LbxnmL*Qz``sA*rMe8Duyo*HO7a5baO_-1FCZ(2j zWo4DyU&ZROtq{Z|t5}1H32vc%XYqE_Q6mt&OPjq*)+{(?0k~%JzN$XtT`4;6HXY#S z>!52?$$J0mh^n+xrd9;;srRO^!c-5h%&NoaRGFA8U$2l!wNT!|c06k6;gey9W2y#}@SyMi;3JF0U+*Y~twmI^F zjAnNoM6@G~BnTy9P~7*bprr%{HS-D;vd#YeOS1Iz(haQL zF9k+s#4}ywWjsEhaa|%@AwXS)VuoXc>V_2tptQl&=po9BF@YNH0wOs)^@VAwHLs7O z-~#+{1s1xj%^B7-1QUTIZKivkTJS%Z9LPug;>kg?qso0K*@^Q})fWrAJB`$IGewuQ z0$#nAY9jFZ>@9=vx+eYmRxhizq~#HSl|_jJJM)ji|CZ9AutRACTQrg$i8MlBwzsP%8i6?~Yjf4dt4?d42Mj{6m1XOV*qC zg@^99zr6iXJHH6+KY8|W{>c**_a`4}E&Wld?!Wbu@b)oqwi360^!9m<$`At~XC}^- zN*&4rpCu20wLWRLPQf%*AEQ%@cA*4VSevYDzX4<$Z9g-3QAtG21{lNMq12`H3W8=9 zllS)0nq(Pb|4+BBe&*22H$a8~1h?)LUbL&~V6yOahew|Kt=t1aYnh*VVHXLDpJ5zq zvu7`ujAF4U*=ecFUM=L@S=BE?(w->#ypy8ffspNho^ucqC3;(!-wcd~_XFlNuh1Cu z_!bwHfVju2`|-_HE%^|P)EU~~CZKpo>O{aF%bEalq`J_%tj+-KgKc&gGN7EYQJvm} zDL*0;YNZl>)eEilAl(nxcUH`{OOjS>R>|jhstieoCN-UAuJoRs4sPeC==l{+)S5b*cJ#5fZ%{m^+ct}+1Q{%Pw|GK+x25KW=tOH z4y7V{>~Eo^xGEku4OxU=HTa*PO9T3WcgtEbdI+J%4Z-_P_WYCj@2(#DS2Fzjg39FlR_=-UIzK>ZwP2Xdp-V_;l{5mA&vlL}1#a)M zQ#$_Ht&%CYD6~n9j*P>V952xRzl6P6uPoP5Im$pagxlDZAJVZilRRemX54iX2EVu=(bY68wZ{9j(*+Ur{p zmsLU&;LV80GvtY3ujw1&j!&!M@?{Bh-Hl3`s#~0Zky5vi_EdC^MZ+^l-r7m2ZtUGq zcYQxY=cG7mQ=f{E;b-QB?x2f{oa52TFg?Qgl$4qVx~R(lBuR=A2Cz#!CAr2eEug9q z^|Na~DRW3x;DYVyG)ax>G*}%RTUAs};G$NH!7#;i+RFl77ijtl*!Bbj5G40p>k%GfCGJ%-xTU)l%&-p`6{?gBeA^?vSR6y8 zVBjL{!&)pj^2DApwa}s;_&c^$cvImDZ}8(d6|&4MQt>HIt%9aoDk$M@ulBn@+XF7B zJXGod>wRqHaQC~D84yXqr>ebFF(XwYYAu_s!N2CVvT`J#oMZl60`Mu4A4S~wG+BGc``Y925y7|BkV^D>7B585eU71Lp z8k)Ea7_N?>gQwDge^t9E1{KXL0z<0ZB)duqs^oS+q+|dDiNoMUay5y;C~-ZC(Ite@ zo^ohzITkKSRCd|0#omz=Rt+Vf63O5QWGw)PBPtbZk(8=d$u6t+jkxU zDOTl?#EYC=mw*`6aT~(Jt3>8@S4omb{&6bcvS+PlBrsQiXx)BXDNyJec%UnZdRI6T zWgDQO;J#28g*|!gjq$YA?CRxpVV@HMhr4*Ngi6}E9X$cnwpbIx8 z;B`M)G?YrBJ8i1l=e)ReRtE)?iR6XGaAi)q!jbdbQMs_a;S$?}1gJNlI%F%t5z8NG zg#&KTg8nw!Sw-GpX!>zaf7EqFATRp90ysIb@&kJoJ_v-M8Nu9Us* zQZ0WI2&22pK`?oCpo^XPcinpau7cQg(3F z7^VdS>gsUH<+{LwD-rrrP3rDxlOtt#g6EXd!=N zP219K7PK~X*{U*TKge#4BzCxEaY(3;eX>`;~K1K`D_L|CUdowH4J3dTK>|) zirFH(w~us76c@|$)e?YRB}zWQ0j=D~2VglnO~CB4r;NDH0v3&_k(T1fV?9y^=M$fg{RTZ}^m!y8 z1LaD6|64m}O-u&yI1!4s%>a&IhOQ!PHaG z8g87>15eeRw63c98K;0lNSn?>q~Q=~*=2Qi`O1TJT1{DBj@z<+z( zATM#YaEz)-tybla)*f(#E8Kw!t8R8V*Xy@%ZAK_5g28EY>>h?Rz`OPydX)*>c$GfdhwdW1!0bAP`VGU{w!%fZ+B^#od$VwdyU1g#NLE$N5eL{K!)QSb^E;Z<) zunrDO_f7?KeOHJZciOU9zwN3nV)9b3S85@gLuY&Is6jCo%AFun zXu=4tg_gx2xyta|Mwd@Szv844Kyow4`Hz_BNTQhE?i_j{^*Sqh_1&_74jbzqh5z~V zZ^DoDztg`CZ+~)-TYCMLb_QR(e6=6_>hJyDuNSd>*)A zqD|0j3g7(|!~2)v<=eas8K+lc-71Y_pQqkY3PToz4Qxmo;byoiHqN5_pyJgMCoCx3 zv@JNFHp$rXmD*}*B5!56i_y|L1d&+Q%rl@{O}M!1BHD8nks*~>lK%sgmxW&SHlYxO z3IaG4Kco3ZZbzwX`Is%EN@PHwVnZ3Rbm|grxBa3bNcI~VTYq=#mEyKB7rsh1L*@@; zk|HtHpesnKYc+n^^y(37g}Dk930lP7hs-zVM|5mV*Si7=`W;e{ zuxA}7S-4ZToZbULnM~dSdcF-rDA=q3;i=%5Sk5n9IV4VFDA)QN7%-|n`wvhHMb;3^ zWn3FMjHj`b<)b2Gb$Ad7I3NN`8p0;KBxtZdS!Y()N$SM7F2vb_v2kSK&eUY;TVnM~hIYj&IKwF+&+Y73#{EUu{m9+vETe*zG^iD( zSTRB=VHzi>QvhGmp@3Xz@CxlQySng^lUPTuS0wEgwjc7#9i0ukWw{Z6v|v(kMHEX! zaSiJRKt(Zj>aJvGfUTSJ#=&L+T27cfdaH5d#?I53tl zsKIi_$^CgY+gMbBvX0c4ewDV4*AYvUd>sh9vzmBudb}z1K$5ug9EQy0)z-&Bp5xpO zgTuj)r?zw<1W>JzGRAe4vkt&+9gvi$R9;VNCfq}7Sw6^eeJqC=OSM7#kStx$fUFs$ zl2Of8pQ;PQ5G$O^y2b;|S$x&(!v0mGj~}Ef@H2=IFVl3e2aMGhVFv~UAdnd~ zKPJB|U_hWv9$ERQav^8S=JXi#gvqegI$Wl5suI~T^P%nf>SQ?j{zX|sxPxto@G}*V z>-QutWr&ikgd1tXxl>PuT>-UHOr?&`L!pEQ{6q zUkc5}YNU4`=G7ZFDKZgn2PF4~>=D5d zz5F_0sBA}dq^6@1@B4|<({z*VYF)O>VDco6?F zv!Tbl+##D^8&&#PrcURu_61PFTELQHeTtNm0xW99Zso;Vgm)bdR`u_>M*DjfZEU*I zAaa5=c)DOYE{h7mCtRo{BAy3hJ2AXf#PB_ow!mI-Z-`2g4S>r9oe_*w2 zb6efn`Z_?8GwyB;5D8~0ab<|Pv{Vea$P>G*Bmsjf9A|1fw747k(Z#CdL9Z!>Mb{#t zy!X(N-3g~i~k?Ws&p<6IsJ@FbbDECuufMjEJE$ytEA+zuiiUBuiSw6A1DJ&8F| zj+lAi7@oX@s1DlFmFX&zd?o2Xe1=zXj_ zyDMb~v@eoGv*cQ25~#ocu*egX+KP>y19p|paAC!*;Peig+6zYHVjNo46gO39a8Fsh zqgY*1M+%2KI7)Nq@d65z{a7^`IS3Shc!aK`A-3Ej$Gzo8Ne?%)kbojr&0!p{tO}`zLWVMDtki@I z(msckf~yX)@Swc&f>M^iE*aP~j}e3Ndf5^NG0!$<$ic4BLrSZ%5%Qtybn)xa^(|RYXd!OVV{lQ9vxRxy=(C*B_kJ&3SIw5#FOcD*ftP& zp?TD8a+e`UNA5p0uJ}XJ!9NU5ZE$=oM8;jqOTz3lJx*B{(ypr{viD&_#8k+*gEo)5rQA+ za!XE9IRUvvZ(_4}sggw~J*9eX4M_sZG017LOnF;3ta&x?lruzANJ?L*uql!DDP>V*E*$6wO?&l*BjBJ7U0Ur3@a9(9wcyX7uPiCb1J-Z^?#cI95!`A2EN@C4gJ{ij zb(IqG!AL8KDoKX}-TMHLdYD$j<5!nZ(l!Hq6pn6F&+CksvQm|&N^++tiG8<_t#Ixo<6TO6 z$}4A=bJiJl#W1KvVHc_7u_*@L{oHl#i|H4av2swyyLx35oVRWV=PKrURkq1vKL7^; z_J&dAI&2GhA?=6Nfsd$b%#3PQn5dG=jkKaCZM$ERKe)ws(4;CbonUV?>vJeTUzQG5 ztULAzd?v{NX_2dekg8k7gCw*?}$~tzMsL|pC6793U(wWzkKlW{$F|x-#?JQ>!a^}8eZPd-n$>D6R-L0 z%J%!{9m|VZ*+1mJ@X-G(Bug%9D8G9vH_zR0;}CF%3!l0~X0C7!!K1p?ZIimPGGkdc zMOS!7cHV%?M$Ekh3A-d7OATQ8)Y|qvA4sJ*wRKJGqvE(F9a|qtLJmZFK?Gs};_I@j zMs|6weC|AELYb1@J`p8T&wN;Q-=xY1(cCzc7^x8$1Fgx%en=;)8`cSXM;S=2$t6oq zd~6g9NlyyYpjzaDK|uG&VOmwV>_(ACYa&Z&!2N0ASW@#1;3!tNrCQu_QVf}p{MRT_ z6b)?!WwSvmv3?At#yL9x`THj-J!_Mobh~afI$)ALNl`cw;e{xJx_%1%YDV^JnkM@> z>cwVN*YHeXmO;@+z1G^y|NC)4>#0&-iF0Jufm*oY&{!6!N?wE_@w^!O`L&oQx`H7je@ZS>(pO?9mdY zuFhy}#%9Og!KPec^l+x2mHpkYI7JOipgr5oAJWhFObHC8e2> zV7#lF6?!FYr}Jj(_Ezc^nC^*iG(1||qNrTe?7s{++0aNL;Xv;=Ys=uEkycemfi3wd zAJCe{f~osZBHJbvU@qgr3cXjQL!Lq(yD zzo89=<{qnu1deuPc2VlW#qZu)^VInAIW}(`Bza&}DAi*gp0!JcT|m({-T45yKf; zw*shvUH%y|9;|>@Z4f&?E6wLE^j`axrFqwHGe*o`Qr2EJqr=7@2=oLuv(Q$v8!2GOd6Nt0vah}B zbI7|6!J6UP1_2s^tO0ep+nU~oy9p<$U+vIV6;(0;C)%dUdEBU=F-qsvK%z1=Byx7) zskF#D!7ns#%bmNe&wyyb-?k^s;TmTt{i6iiodkZBJJgUH@i~BUI|5p`Q8 zK0u))Zer!r1=XH|8r00lSeY2*>m+elmRZcaEwH9x7m`m;;{;qoLZdloPs-kiGxX2^ z&~k_p;y#OF2*BO? z(DH6M%U#)kd~oy-2B&O{bSVx}grz%0P-NYLc7vS|Uet;eA|Tt*DG4mPwE#Paw(TPQD(E0IhsPYh;QuK#e~OTN2WT~zW)C!m{V ziXIuj3VMLv%t^9*>&QS1uS$ZVvgHI;+SO4>YmYTw!YU)VWP(x}YRnE!`og1ilT4^Y zlRFFt8)cNw6|qZrbG2oA$p&r~PTTIfvq)tx?Anvf`ivn8V zV?}~7cu^6O#aDSYtp#0qvaRLsd7MwPLKM%jI$P$dpykw@1~j<=onfmPnu`Ot zsQq?SDwJ4^IjMW~9nhLmrnmJyo%+V_JawT17iriTGav!?8-UNV2^urfI+NO3)Nbr? zZ6aYz=p-{=zQ}E1)nnWDk3844+9z|6W=?tcad50bt%z(`(8cQ5)RE8EsKW)(E!ua6cKI$+*R*hO*v#3C znju?`A-T}ZhcGN4Sy&#WWjs`#J+nz=FqNls)hj-fW<~d4v~j6GmHKI9vW)G!Q>9H32Rus0*h+;5R+jU!6PD;3D zHjWy1XssUuL21ale>pOM-kf@!0a8#JV3(*+_dV!^Ku=9l#Rm&Yz%f^Jstc9uV#zrp z1*U9ByRRD&>P)hTjX)nPhCT!j3U|0?_&`R{vS@Asw^Y2oZQNAT1a z;pI1Z@3ZfIg8A*IufKf#Bp-S6`sVdF|McIGU-H-dEWCaN?$9s7>o=!oaO~tmpDh{! zq7P|b}(<=Dv*$hL%Y^A>Xmm%t$M_WfW`$A){mw2qo?c z!-od4_x1=uY*iu}BW!AbK#0%A3qr^TMk@|i?J4s!sLN~W2oLtba$Ew99?qeZNUWJj z?mlnMO;Tw$>VU*&NI7x<4c4XUSArmpVGFwf6MCBRuum{ZZBoUq*wpr|;yvWjw~8dv&mV)=-n>Mpz8gMUIVx_k@Ez3G$f>~vYN4b9eRsZeWrLR zv$%??^8wNFqpgt?#%e%n7uSR)(Jp23({&lltxmfuZ!4gM_<1DHs8`O9IU~7_v#fgf zMII3W@JnSwtskpnQrTDowDd(WHYl}g+0T(S=Gkt21 zdNtw7lKFxad4ItV_*2} zSwRK{#*)h5eo9%E45+T;nK_>M*3eo8DAOJD2h1^cKA`zPG(z``m}3Z3T2QJ~(a-`V zz#BwONdkNKQn$RKtxO10iKGcU1l#LI%wi;8cW3%l71?Y@5)`HX#1e2Rz@!;(Q{a5N z7#rMa@`zdN`9vvCktb;#9djYb!8&u_W-;s)Z?da zB0*RddI2pPF& z#UCDE&^p$ZiuN217SsRkUxpw4@Q1qj`pG*eXW8Y(_h0v~`aHaRe5Cq(`tob#>wNg~ zZFv1K3rc^Q3*?V2D9>_zWl^zy!2>fsIWLv5*Peo1_&X}H$Z0OTOlBwebnWiI3J^LV z%{80T36A0QiJIj{l9J8>C7#~MbJ?bVA+T4bc<2|8-S;6lx_!;9P1~NRu_3=&S5*tw zx@w$s>Jg+`Jb+A4*{wQ#;!3@OY)9DYitapp37^{F!5Uoi_SQ{@2>s6epTjc0d<0xi zKG6{d-N^@^$SH#%lPvXwI?FkhGiOzA6Yqn9Pvov*Xio6VH9<`TP+Qj&sYTG%aP(0J zaI1iDRK;c#V6V`>_bVpFU(s4EIyXtD10|#R3S=A7qd1vxNP2H`_S-BA69nzx9Nd~f zvE=&TO zoH2YbwZA@Wczso6aMjO}7?8)mxw~|cYqi7RFHi+jI(*A04e!rBX>7(dFW4Q)C~VSy zb)4)qyG=`U5vUo=TNe4UFwBVFh+3wT)1dJp$rgh(=9le8qL4(J&Ai1djxSr^w;sNc z4&)JsDtzQ25oQvrpSc=Ne^w!HomLzS%@EKvmRQb=cf;S^E#TZ?_^+@Oj%jBMA4f#- zRA~ZkKd;1UHYn9OAILUznFzXuZ3qDA&@^{p<*dGiK6vDm?$9u!OTfaiHf z>Y+wX#5`Au_b&U`uq8bxc&I*?f2eBHAHDn`uuy*g^-Zuy4JV~_w2A72x(2G22}%ev z!n=eomB!3_$S3Bb0o$#;=x8Z*#U=j;I2gJCvee;4=#2ok4XUKmLJK2O_-gAAV(*l; zUsuIU5M~tz2~|KLm$Y!uKtY2f*oYQHLc=nvultpubgRfUWEJUV z&Spt21cRtC#K|tBN^35ZZjdlUrz3d2T7;Apc!S-qs(-jkB9PSbgkAS$bo72aCN*m~ z^w8wMv}fh&@<3#k{1kIkib&8lB?^&ctZH1=H8Z9Mo@y8#p4pGSjdB9XGpg&|?ID$F zk7&=uT!tG*!#6CmL`h3X&-Jhbpou092oh5Vf<`_7YMyHPSV-LootbhwF67$|t~>{f zv=zHq?Au~&I?Nh?l0NwSqJJ?AV9a7610wojnb=SQIc8xI-;NB--Dvtzf($?a74-mn zlynYmm27i%o-Jfhq;@*>Wi6Qy)409}M9}v8_!Od+)_~|^3p;ZAg%I=nwfZjYke_aY zJc$LF4iJ701v{A1FIkNGFdF-oNn8YFkEE>^*gBjgAD^@-|VbqZPX)FSq* zR$Jr6cIA=iwdh_Vy&mxq*;0o@imQP=1B7JkD>W^)wEEmqHyRMHMcsS7Jp`ZE{OHXy zoe1irLtlCXo~7Erl>qD>a+}FmG8tk8Nj0|Vl6JnRYYu2k8h3F3sjy+VWs_8}T5V`o z0f10NsX2+^?&Z44LYJ2B?av%rgY~8fg+hSsAba`cjA56f12CFS^DRqF9LSLw(gNv*eNe#s-$4S+V1Ebc~9ML^nMrN)2 zG9TQ>l?g^%;H5fIb|Ee<5uBNBW5+TJcp8G8;rD%ab?cT~CMbPNt}$~p_(Av~nML2& zRQJ1|<`Dby@BTWxddSX!jbP_unIb!O|3 z6!{6r?u2`DsEAvZNU<@C2f|ctBIby43F?H_bU#+DyJf(V!9i3|gpjO6E;}p>3)2Gt z3m5)N0|PE301-N2d#YKYi#Z?l@N3RnlL?kM>uWFC{nydEMH zMAY<|yO%X1(FIMb4{J4n@MTzGPVG$b$aM*ol7&=EO_BBa9W>8dNHiCk*k8f+ir|y& z5onpbtqb46J-Xm#vDM>(o*W-zx!f=ql#IZ~vQQ_t4fup6 z?{eM>Cckwt;nKCBVUUgMEQp}YYX_|v`xRPJcdIESPuHIExHu3A%Q$7i&yfZwK*!9Z z)Oa_*(qZSNg&g!$&pzWV$&0X$YVZ9zjLJR$5g%TEqO z*8^nbed##dBFR+;Mx>M>A9+ovYf7^flLHN%YG1wnem{BuF8vcE(_cn5 z!SeS&p#AdoSI5@y$3PT+p;KPWZr}dw<*Q)tJ)z{tLF6getX<2~g=VaGU|}rW!OG^L zRN5PvcPK>Vgf-kjrbT4&)b9~+Vxw{0R0?iYQN?5fIP8IdcTz>i1GWfWH8&M{M697J z0&onh-7tEF_=Y5O?HTR6xF65P35qv@;)=z%Rlo+u=q@35Ul^INR3K{Mf>U zlFRLF^XophlY-rk87d@_I#Ru7IZ2i~0~6f)VAQ3A;q+&*;Jt94EEvKm({28eYHZ6N zej`!GS-sigubypUJB`X?;&K5L&j|YqhSOz*pAohoe>CZO#bg*>L*q!k4qP3r6Dz+M z@J`8a?1_d+-_nny)ePf4?rhiQfoMrt;{mx*)y8KFs&`CbCx$o>Y@>06`|)}A>6Oo| z9aS(~b~z464`I6BvXX?3lbrjFBUIG0L_hWDUMQ*3zFE#VtrDQ)Qbudnf}tDR6Ogx= z?~nn;Zlxo4c`Ok|wX4ATE`cCoUcB0wMK4(6B%t}8yk=*jdM7>NZ;F`N$|v7-6l=`l zIC{c@U&-g&3$O&vssTVS#xPBBM*>$MJor)uXz=Rgmz;mto=})cMKY?m&}=qqG8c6H ztkqVDuDK9qU?qloCGAuflvNTb`z({8Ly-JNL(Pn{tgrW6sGil5!&;nZgizx`wYyIU zY8|JY>8aFi&Pr$8=xvm0gZ|J8iRGGF6s;xzZn+Er7Z+B9Q9)T!IvE;?P?kJtn?SyU z-O&wqY@}MmMmZ{}${^?M;Su1wM3=pT)Kod{w>uopB=@^<8u}nN5W(e0k{z>p1T>|T zmODx!%=Dmu1`~f1j144U`qA4@PjCP3_2+N@zt_*h z+rOs<^4|vgegB7q#ovY3-<_T};7{^;^$hXA{Fgt2$NfrVqDX|W@t2(~--p~>65~JtLyXF=s61Cms8BOm(RI`3B zo45c46MYW05wT2weiBp_wGlL)At|N4OY->K#xh#x8x9lC__yILj=84@x?G7ruZjK*B1R1#71R(UTx0rvDvaSNcP*wr1~Qw8R9%VGdxfQ~KXpdE{TM(3`9 zKrIeBKy5F!W>3%>9~ZE{o+!wPwW^Y0m%}qj-_Hkun6avWgBwC~f zYX;rlJ6MRNYdrg1(Z>Zu9H$Qn3bHw3$)kT4peE`NEL3Pf{z7a~3+Kd&07! z)nJ@$XmsS?DJg|BpHw=-i(*-kDViz^UMpog0?|%NMJ2a@9h6Y~I_U_Q&GJ6u$69>e z<^qazkkn8eWVBR`Hzb)ErajIGk}sIYgC6pB4ZLpZA_^>}{nRs)`4VCSbKF@##qQfw z>bh>?j*Z_JDt@3NwCQH0gOd3m?7nsHsQKI*C0JP7{tkYuZMlyAJ+9aGorPG19_1ga zcO@Uzd(&=$o3caIwbu3nHQuE7*|jh>cw#K6V4Aqe<3|^?_3m1*D$Y30dFE<89o4*; z{Q*_bvRp5nullE4Q`tzRRzK*hPzM_|K^$@*ZW--p8@8_Fk(0W-w*igEJ&2P)Qoy3F zjI9D{0$qlzhd?ze)z-1?3I$-x=nlXdd)Lh)3{rA;Z4H@lJGW!o9ZWSO2ZLc! z4zFsl3~pKXn7Q~9@Q5{0`kA1up58?}e^38S_;=9LJ>s{U>V)z$+d z8FKyDx5fx;IdyvTkcb(7l%{3G61-O@JTqsY$0)Ei10K8D7^l=Z04~ z2b+b`+0QE+k>nB^TJZ`B;v^AuF^}@o1<9K)lR!>bDTs*L6V7zTy6X5u$v7o5+FiUA zIEXa+oTQcK?WSsv|z{f2Yu zQNjbGU`04?04n6C3G!WFx7+?zvRz1KF*c9*Tb?XVFUaO7b}X0V0m}nG%$f&{j*>di zTPvwbkQxSb&=l$90L1pG7CFPgv$nzpyCte~&A}y2 zh>-xUqWF!lnZRp_B!6>jZ>b?iXW}nVW771@I@XfwKPc!wq=>a}86;v`79jU+a-iaJ z1#ivBJ4jdXcOGcBk$}!c733L_$2(9};l5+5;2K-I$-oetp$pN=wUM&Pp~D-xa8?8- zsl!3|(FIkRx|dNNtHR!ew2!Js()=iP6I60a$*c{uOo2@cv?+Am-VhJ)9p{jEg{15n zTmgAPlvFJz&j~UzYGI8&#f9m_6mHc$(5&of zLKqorD0>C(DdsDBl{;#(;stPI{ylQhJ5~5uEJcv=d<3S(mBl5E_yxesupdQN4CdZa-|CKFz=vDFUD1>{)H+UorXoXeQdl$ZB#LtbyvM9_n0Qt zxi`Ld=`e4;(Fa(a2P7UVL)HetUHc@#z@jQjfUua<6Wri(=KvyM5@85HdiMSqa?|A8 zpaF%*qjv~#XjdouNCmNrG!M9gBp7g90qd>mplT{Hxk9OVHe{H}ZK#f+{17VON{o(Y zuzS%K+egpPqq1%J{VYE#;DRKrhL*EdON>NPQ57CeNukuaa#r^Xz!{_KErEDq^StjR zC({^I=0eyIi1Dg&4HnrK z(8}Z0m%FGMOcw2Rl9g|OABX>4U;o$eum86_lz;vD2fhYw?C<_sV!yqH?;rRlij=NE zi5~sWz(rO{G82MtUw@JR4ac4QFTdu^%SR!<{OCvfu{Ur3pYZyAP9`LuLR$z6Xg;@~ z@aA*$6sGQUmpcgtoi3rJB`k`OydOFfAGo0o#B5uqNKvZy^>*5p5q((F!J+mcHz_TR zufRKAmwRLcT{n;{;i&2HYn3u!4X$R7NiQkI@|9My0FDQO#*V5k9AXXFbeT*$dD$+z zzyd1xSqclnq%XMpNIPy1{A3*gYV*quWCbIR*%ff?b9VT*=WXCxq4$^fdPy?!7KkAY zPl9A3D|De$DlQ+v%s-ygaMmE+X+-?76D_Gj5!$Oa^uwdo4ggBWX3Wid&*B#hQ!98# z_AM207HCVuM1zR6-9b#Eh`L)GxJo*|F_xpz= z9uU@29pFxKj?KK|`!O3Y*a^iFOM~*hozaXn@*gxr(2aJ#Oq|5k^=Of@l;g5NyK9yk zFtO*Jq|V&kV|z(F+0IWmz5|HhaEhE2*jMQ=-H*ij(6gmY|bXv1y;%dKnSqTa;~ z_z8(So0~4m&5q(W@2C;Q4An}_o2)MY;Ho45#$xUQ+XZ2Ar3n<3q^w`!^K+<>0GfQJ z-26nnX&c`JtmRX1z@EVObtNwO6E)eI$Um0cZX^N?fR=}(3+oEL(TYxbG2R`9Stpe8 zHW;ZD6|i%f_y|FI63Uay&O?HzXiR6gL#V|Z?jYW&Cs z!@z38z-a3*E2q6>vB|Z$wtKNXP(DOXVuNw0%SZ3b?yMd?-Ur6j+OV|^hx=8MTXw85E$trYJM{%UA>x#Mnk-^pA3(6nZA@fzuOj%iy(_=u zxw^N|Iat7EGtPNg#YI*{#XZ`7EA{sGUc>*gzx^wkHvH=KQ+wS_hhIMU?yn3c_VOnU zw=b?e`42C@Qe&k5B0tZN(`dGF>hwULO6qPTo(Iw(=78hDFj!K=d#Z88d)2v;bF(Xa zl+UCVhd!qv5(N)9K=nw4VB1xTaCXnPnuM|~6jqljn%cBlbz;5E@J$L1R#_JyT#EjP z1C&MI8g?2@$v0UQsRN9WEnw)Oy9Vrn@_2R^kB7mBg+z3I~=KUpPAceGd>ypSdmGL(P8HRNeFgp zXDCp>VgUXvmPHNCP6LDL+HKrUa0k#hYQYjNJPgQvSwpg|6`&gD>p0Z|I;XdTo4CRClV8<8~t*mxeM-gok(OfV}P$#vTv9 zM4pT5RTcDZQj$Ud6Ws%@~SG#~E)S@M-8Kexm7SFqlEsof&%Al<65Ekulf<)A!?QXPq1Y8@Z|s={}yBMcSh##S~$J_oJZcUJf(MCQDq=}-keujPZVu%SsEOK6y~&ld zT022Mi)4#!V3Tkiak5*JFD>E3>h&A0SMDWjYk}lWu$A@ z#toCk4Rl-3o1NXfwCm;s3<thjM)H9%7g0+ zRNldBy@xVL>A@j6Pq_EA0*qJ~Pe|!#jeqdvS_bduky~xeEeL+fGUlI4b-AtR&_=uS za4>2Hx<6seO_6i3DxS4!ws;JEkE7oC1n@NG`E06phw5d@Hr)iRL0S*DfT?{+axo$2 zrg5@Fb9ovbhaNH%H6?{OL)A-Gs``X3ppJIg-yLU^a|uGndA|xnVJW~Zo`&_Nn%Dz} z=yE&78n01@*gPDP;dX~I%4%V11w-V}M@nwRU*#Pvr?PO{jngZV3q$g8Q557T#k1Vd z9SBOZ7r@%_+e#O>2BBrAJu8WgmQOH-YS-s0N9z!WEbIfqeOTqpFw~Od;29mRm4YMk z+0T38002zuXf_z?S>_-=z#91)N5bL>;kAX;NF+-+$^AcDd45$>F9=^-(gyT~r0H;k z#-?sb$ve|=4iYLMOK!fJ>&B^`*?Ay%u3O)BU<}tE#+{mB-zr9EXs$<)mZka%mPUkVK4Q7NZzz? zL_RDEhj!f%FoHN0S4d#&_iSO-NfbbDm5P8k2(E)2qdPpJld5jYiE4bM%Iy2!R)!~L z5vzf>U@YaHE|NANNMqTZsV)YSXX_XgFiye05gtRsN>q({Mp0T@kuFvZVoby_`wh+Z zppAA}jBk_#M^*o4HMFx3Beeerem`V~ugg|r(6UuD}&R2MST6?0Fd$DynYM$l8;`#lz;#5 z@>TxMH?Mzw{fG>*H?M!>j=b~oS$O@^ajgG0FMoRbH{s<=%;ss330TYH^zh!^LkD)J zr(%2x?hA#49)LP{Cf>n6Rc|PHrkE5YmGH@tf8FcpZ9oCFk7(Ad=R*VWX=;|h2gjP` zn6}`dPV;?aEs2ADyPWA9f;^y>tO0;+1GJ-f{&e30Lb1$XLxq&C@{xlMUXHL1oVSp z9o({U9g(TN{poixO&vL7E2ed#6=jt)76GE#_hZ4~srOKYzvQGzZf z{2EoCiVM2rwcNBsvbWU`r@(nD=xO1Z-;XCW z*qhC~ffZXS_4TC%p{yK9u|BFKRPd|8^}|0RFX!+;LtJj%#T)1vYWpb<%iEgNxl*;Ll?A;AfA|&`@T4g=-b2NPLaaR% z>w>W-8lsXAocEp%BB;qZ6rIw1-3kQKy?6@l4q-!Km9K;H9Qk3uwuS{Y#6IM=AsE7Y zfVH#Q#*D2idR;fRx+grP?zuRk+>m(p9H6=6U~?2L*HgBMhkLG^5g(Dp?m0sl(N0?u zou<@|Q5OG!gUhJ%_q-|hNAn&x&y=9Sf`X4;0`wal;rnCxFUbF=A5cy^^R<8A_46KR zfA2i(KZfrgV3kdZ+w0eS&EGGPH@tj>*Wc;u8;Qi9?BUmUG9d1o42b(YfA9YtUbD^P z^Dz^AMx4qw=>uLFG;WZ3%EsF&H*_Ef0OS5Zqkb>9?@Cm#r5P_z+9?`e4F+#iDt{i!W#WD+S2xW=)SlR&G0$+5|u68q{xr4$}t-P#@;K_8D^JtV8b)4Q))E^_Y2wb^uL$e>c=MOHoOtG)8c$bePIm3+_Mp>oWcVzp@8Wa+k)mW?5@uvLxc{R*P zoh@b=&bl@C)wu7bhOo2?HQQ@y#DK{u_sn7?YmX4C-Jx!g8|wsE6am}N`5~K=c9Ycz zjeUoV4X*7{;VN@Got3gfR5X?zgT9q_CzEAr!RoPWo5L>dp^p#4S)+C)8|YN4ws44* zLe;1~^(0?wQY9r|)(0?3^1xw_r?{cKh@8iQHL-gH4r>F=+H#{?Jg6+FUObb11yDcB%I zim=I+Zz%|32a(3*y^WoywS}+DURpJ?oIp3TQE$MofY+p|ih!D!r<=tKh(kustyaLF zgX5-wSL)lxBT_qZ*t@cTdqwL74{;bcoCSaXq=G&NS2e(!nD{I@AmBD+17Y+>w3Do@ z#PBbW2XuJ((%xEmObJCo#lmheM736K%MH;_BrqpUaR`r+1RuuTY%6CSi{;(6@(@uh z-raL;DId7P7WDw6AQXG69-(eGADV_@5qi~HUNp=hZk_rg*#+?Jaa z$H2`-RLHC_kSy(F2UKx|m1bWU$O^U~>@9EIZiA>Tqnf7Ct6z+YTYA3& zKi7}df$ePZY|OsujC}G(5kj7~(*y#QExkZ@m%x+f9_#7Hxm zQ-6`-CDBJlqG?u9Jy26+HIRZaR_|!(x>Uu2QvaT1)$fLK&BHm-b?MrOj3Z%;W6bHS;6f@rG0<#KrK!|gHI>SulT*6##aq3cbiB=DU zq{g-bjC$Rm4{EHGKbGnm42ow~SCAB>`4gteojV(MxnV3b`1 z0*fulSq0u{@rRuIrVa+3o5ex$quU53c9@B*ksvVjWidY}T7?!p7J6(PoFL>`s$s2q!M?XmBG$E%~#15?1ffJ(0LTAg{!{ zg0saZK&S2TMS|oG;j&m47F2E!_agCUniPIyrx#&vXxLrAUTw{2@Nu6Eh%_F`@>M6e z6LTF*00T{k;K8QRSgz;OMD@okqqxe4@J=M|fnM8_c&0-Ib6iV;6?*j8fk(*d-L#(G z(~CR_vQ)Q2ZcPG-X1F4KDoGXEtEH?`;w9bE_Ea)7mB@~gbs#jOjAY1~VMxM&1(2M} z8l&~lCgs!EKCDZRWZrqK$2*HOrG6qWNDNj!6PqECBT)|FM=lR@hv~)u0RwwQZ!6}X z$7;3WBE$jOT9eavrDCI8@|@8Tm5w(}&9Zf(FYYD>9m~l)x8C)Zy>gp&qW{#XUAsC2 zuB`5qn{$a!+3HjV*If$=pkB`E{ABvd@=+mVB?)CWT5U7Pm6%O1LDIR0Q)`EG1N_-K z%LDLsWouJ}p`iZ}PC%3lHcu_xJ-HtTmDdVv4E2)oP_%hj7C5wd9E0~n$os|+C?@-{17^bU=^1_fOnen!ZB}0$f z8k$i4qOYzLhyjw04|Nx{F{5OtQI$Xy$l@39gmWXVE>t!i{ApszKqd`b>fCj_j$Sn*2UQ)`O?b7IAguFPX zm&#~I^Ml}F;J>Nd<(Ui;?`Zvn^DO!Aksh3$gZ!HHOpipleZVV`o3{hS(psYn=%J@H zDbFoIFj2>Zev#|{35O0yMC1Qh}_ zERINm8!uXyz(8Bj1-44N;8uHrv@7@U8HFV96C89(T0LO=3RO>a3rtFXofo&|h4#oh zzJN$PVN2*<@@15|<$+Re@HQ+U?JS(pO08=jMn=V97Ip1lBN)%Ww@KmE0>s^O{5aPK zd{cy1GF(69fqh7sIq#uABBh?3=>}}M1uRhtu0%y#Dg#%eNoA`~ifXU%Y($VgtUPzkC{Ae~GT-pI(2vcO`u%f7MT3J~q>$ zhf*$a7h8P^=-E|YjBEPzga&O#Z(CKY2=`RUjkODKgh^tu+*QN@-+Ej!mAJs)TWrqzJ z?d@Y#wMlz3CPONj1171Y`7YiOQ%TB_qmE9wEG$SQBOV?m?3{1eYh~f%lGfeI_Y!z2 zTio9_gYM$nh@my8)=U#zL_!>p3sP*$4)<%(tAlYL?m|_#*#&NzQR2RP?&5Y=vG0^M z?RR8vygguIQ#UZuo#YnEP`KE+A8#C#gUZo;1JMGLyujZgPdfOwxlNYNq)qmZMg!CCInhY6PESu0j!N35qOiT>?!G!jcH5Z?LJ5b?|t zSe1Pa7e-E5q~`G*e=W@A=jsy1j;VnZw7rDg!WlJKq|AG6a-v+aH?=UfWYQ=o)^ z0L$O-L}KoFJ(ZiRzIdE-5R(eW&F&;9>uR*Oc1Q3nZbT>@xMy$4A{>p~7NERE7wXfl)_a%l4(JdN8oV4#lL|?w1K1tb-(UoBbuC7A<&`1qoW(*ixOP<4VJoAN!g2&rz~8E;`@xRCj*3o&B&%F- z2IH(~$Wck0`$F<{Hyc2&nB)sIx*zBc-^lAlW7BIcNNZZnB|&`7L(7cVdHdPxpI^R$ zWAC3|{*ZU%jn7_wo4@84uOEfik56wuk^=izL0-PW=pqF(XMZmyGj`>g-%bhH{ z&TU?fAhThe#ce6WHtnNvxHL1BlC)jK!S-MYJoz*e51DlvF)VhQUA?Nd4YwhS z?db_hVB@3#zl)&5dim<|pgm|KJ6H$Tj#wP1OBO}TJ@j(-`qlC^d5~e4Ou&v5t&`;B z7L|3J#Y?7scx>Vh?7_$;s#~lpc^?QK%j>QZBuc4I*f~%Oz;Z;S^#iekXR9PGZ~yCe zf1MvL2u@$aMxS0Z`&~Bb8fSEq4c;?0RCbes7aTat3KqG&(gL|^6jJ$MEJG_fLT^>> zf~3#%tLIcpzoN*5d29QuYeXH_3AAtvd%bao18dxA$|ThOe%r z87dSR?S!A9?Xiq!a#6DwmB-dT4u|%-RIYNzUfZdJHBk=y#Swt#X*tej@NJM3Us`du(7UE|_NURT&OrLT= zT>&`49YBT59W(G1?3nWzBETFFf)Mi*4moOp&w)%O8rr!Sw=NKuQD7)x>8-@RL2=N< zSmUp-tT_{7yI81tG}eTIuwTeZug;Oxu$Y%hHYkdMh^hj7nhLX0M}oU5#$>ht`U$^0KKy)f8s~=3zcX6%=ik*7~IJ97cP;44O~m9mx?a zZb$j^+uBE$vTLJF`L&f5!r%S$FTDnCuw?$0uMV^QXW+K~Y}5M-`UZYM!y@c_92l@a zhkNfYU|;m>@b>Rs--nM;e&=srK9T>w#?epY|Icvb=d?M(hWQE>l4CTm>{~gEe3tV= zhoj90+t`e|kj-P1rOxEEl1hWHZT3+3NyF{v>>fo89c*vm2)tVW%Pm}ucLv&fbA;gB z;e!mpCs)h67qV=sk_HAe4C_16SBJq1TpOwU4)B899axFEUsB&p)i0B3-sm(El zFW3!92YwTEs<$xdj2O3_+>f%Ew*n(so)(u8uJYbERLr(#XB1jiQoXa|y@8}|C9guP zK->kiHNo^(o=j7ln$Swf6WX}SQfxC?xMSPcS>pV-Sa}YUgQZmdO^~17jOCTw7*(`* zZ)3v?awt*7Efdft6||w3u?I;PYqXJapzJ_KsQ!N7c-o#oV>v&NC^V6bn47>*ziiEx zu8%-;K_41K>J^$=vYMyIa>9tMp$K57o_H4SOFlGEmeo%+wL5WFt_)?PAM&}?KQLV6 zZ?(ui0>S4yl%K_nIj{wBMTJ0+$u(SLF^BiB-sEYC&f0CF1o3{i#?8;Lq-$>RtKCZa z_Cj0*{!O^YPi^7U;|43tB-LW;_5Fh^+-B^F{%F8Nmhw-{{|JG?G9Y;+d*4wOi4oKr zlBm(NS71!ull56DGaC7#rB$}FAt$7QCPd^+=ZvcF0m2QKAf>5F0~ZwPaj`D(Id!Pm z-mUCsB?WWHZPmSG7*f>(h_K7TWGl~Srv%9wCZ#rF$|=QM?s#cFM+Zk%j|IjM<2*OC z4TNTx`#wQ_R_f>z%KZ)K;-IMDBWJLJi_$p3K3>j@%3TjTe7u}tY?}@W%Pz05Du<`avJ$&eI$SUZz8Im?U z1F|EfkL1TzOe2lOG8rX(2xH+Bym;1jC%fur*j+;}?i8yHO@(~8Qr)#kp6w5Q$Pu16c59FCR#O{~@%r zK6&}(<+pD?{qC>G#{KLCpy9mz{_C5UU$I4h*vZZNPmaxdhZG}^_`BDS@f(QDzW{e1!&*(H>*MvY}n&X}cY^q~%?4UMg*j`BpSJ%q3!ddV<~3 z2qWS|2=Q|w`N#slJiI1lg<1M$As;Nr8<1aj3dt?agl+^I@37`@ssvnGUCL`Y%Y$Kq zaPuVO09=-^^Z5k)rhS@HyX~&3DUTQDeZWYzj#-_96o#KdYhxsQY8v51m9$)60aBrU z1`N?rT{q*zHq?aC9pgFl?bv|I22e+r4<+2@a=@n1j~(bRY}(*-SmB_PX35AnxR#}C zGuce3o?FOR+1O-=r3>j|{7^mA8)9_n>IEW8AYK;8>`Zt%hgR@{aM>P)7rKn`D%JHc zS~aAAtR}4olzsaQ`i&84+XvO5TUE0v)1y>hCbC=M+6sq6r_PKpe~l7A(MV{O%j{)0 z;x^sJYAA;0t$Ez0u!~7coSnw|tdf?MrvzC#ZEw{`iLM=~W?Xl%j4*!hKT2i*?$@T< zPNJ=Y0-f_-?W*cEhF{8zclW3a09-hu1_j-f3(sBlG?UyQ@8@PZ0AjObfv0SHLODyh zi{~^MT@KBZaamv#HBynR8Z!*O($pp&l3ynYZ52`Rqr1WAtv0=Duu-G3Sx^&Lw;k&O zo9E%-m@gGZGZk_)2wVj#n6kkxdFcQ$Xa+9cEry|D+;a_WmjLS3-n^rr{Qq5Y;|BO( zBdd|b?Hdpd5BCiy(Q7W-aU;W4^{F4+Q%fFivJZG90f2J$&ov#XgM~J745+T)8vq6D zK~S``#Ae6Z6ma()8R*p#zK{(}n{H7J6}Ju;v=J(B;0>`k4*Xf(>^fpoqwpil1^Z{O ze|-JNAR+l%9(n)u{qXun#LDMqM$Idr)ly)e?oaXoO=<6b$5K>NJ|o;ohXpF#r{C(E ziuxHM_9r|BOqFk75OAz~iz-@Yac8T{cTsK56K#PX(3t@!Thd93nzzUhJFd!(I+wQJ z%(8TtfL$)4kpO#A3J*m&)pDcR*CR@D8)h(M*8(YcTJ9)wypydlRZ2BB#y44)4$fcz zTx2}w0pYx+78G}vj$7a%yqgziqV2nO6AEy1GLh`J9d6j%`hx4(hJPy$))4Yq7^+eI zUq<#}vjMWCEqReukaU_yvUkg4R}RKfR+U3}*B%9S-Btfu)`!iGYMUacYomp9jQNcw zg0r;3gaS2%4SuKXi~@is3sM{~$nHE`K>VO!6Y z&g`irGlb0I-TDpTU-huioY>6zlcLvAYG{9tLrXnNNf|JtzwZ89eTzdi zmbJW}8u*L#s?*1&JjA)+yCJJa)f!m{E>I&B!RT$6MFs+wJ2hlpA_Y3mY0=Shnq@~G zfU}t*Nc9@-j=AmP4eI>xW}Yic6)nk-2!9i^xUd6Be&r4~rNWq#NyNje^Eb6mzH z;litf^S~qplbO>cozGf5RqO%KeR4TS*Q>&N*a&`ugL1U;wox;cnV2?6phAzum9~R`uvL5E5~~hJ|qk@%q3`Jo_7J0bVhnD%d`Q>LB7LTz*Yh7f{NmiB0Tn)^*}IN zzDo9{dS=1hLY`n(x=a6{I7p?nF*8G%IhmjXp-+OgC9~<(;9TpqE4f zS*n`BrwSGNCOwt62N1V+YZ1k-r|?9bjt(=>MAlZ9C6w|`lewX}*xoq2hrl3RFya8{ z0y7KMDzN-S9{6q_awNr0Yahdzj%6GrNhZ`?P6S3CBdkk1W>3)h(#JfcCyv+(Xon$G zS@%J!7H>^Jn49y7G9#Ps*%3V-l%jzOX3*qT%744aHdakpYz!n6pE)4_8aZzveQ9Z- zdDS_r1l;m}1c>{GWRfb@3>h1#5_6<4lV0ZG?xqREEjIpS-_pQCkf}t6N z#8-o!=+Hf#a);B)46q8%q4cC84)7_cu+>WtnDPjICD&k8)ej^M=g8FHT~7%NcL8_l zp$$i-lZ1I4r6bg$?^wOe^jzoj*MbgUDr#Gom;5`y$vXT%jCWnbNfI#JTHy1sa$f4pfCF}T$hdL zR28(P`+T}=1qfkLuK9!J7ZYjoM*MP!9YiIfZ?ckt84=ffNrGWO z>6kp-WA{D*fZ^6v+PGB}rLuj;Gp}xOZYh_kJeQ%C2VS<%;L^Hq4(to~i{D+a?`ltbnFh$ZrGimPDk>%`1t3J#N(l7HK2hGBz~mm`f-T zT50H&WHojS$VNUoRf70N0ejb;h@%Q5b>t%mwRrUR0EhHOrMhkRn2VQP#DaB(i#P|< zrMfjN7zFJmkqx^2eTlDk9vO)FBX=PEZO!C$q)?&bH8M`)4BLo7lBj1>=;S^g1d4Qy&af&plTL1zp(av0%eIJt z5jv_IV7gfOzbS2Mw1$K1Y8R|`nd}8K;gbT7u0RqDYb)haDJPKT#z;65oBV@fYpGfv zdoYVycx7aN&3FM9h9tRjm<{!*9_d}QoH<5w>dt#1*)q4Y-QK+Z`Q?NB@BYqjUOxOw zui?M<1$^Cq+Q%>75Ik^j+kkym#s>Z=y#4g$ccxSF_OsXD1kMTmTX@Oryyv6OyVTm_ z?n5i-g*_rDDs26F3eDP^ofcX=5S*mXg`zbZ@RUxLZ%KWYW7J&-u}M`#T%Zby-z&R} zko^P>6TS=amX3I1U;`}&}U-E+DleNn4sOCkVvr_|2^Gj}iU)G~8m8X}14qia zlnx{bLg;KW#@z%?9rZ*Z4W$T#2QwxW8e5J{4p~D8LPS2sdeRtqQF|Y3X(S}+qi>fI z_bze1xoo&BO}hG9vH>W8H5TO8D2Z>9k5t{rVVV>bfv`rNIPxHNG-NT7AGu1Md`n)- zXfr5Tv;Ywg$V%#ymEj4geco%@PO4fYc@oh!*C}h*#?5m6_Qi3~vR!wEWw(ix->vfz z3*19vDwF7lpdsQo4*3Z1^+<199d=^pIZkyA`+#nYj_kk>?NJthEqp3 z;0`K$6e-rY1g_AED~ zmpRQiNmUcF*=9ZxyYs=bwcc8Y+o30IgPFO*_yvM3##2z8wamgrW8{3$F(1({!qEa$ zqWog-IK|JOnOcQmAG{?%)mh;PI*<2k{3(kS0bS>b%zOoaI$1#$x)E|{6aH!K#9@}Q znQmzjY>7epLkb;0quAWI!l;#UX*xS(36~E6`bGQ$1lX!%UZE)qjFPz!iU#P0B|>^i zesx{2MKR4(|a( zn1}kmmjAyA?+o-Y@(*wS-iT_?cMUCFvS!0F@l@W+Fx@RucW_e7*`LnxfdXr#IE#Z#U#h#u-~Bi(gaX%nb^foa%zYih#2sn(^Kmm|7)<^a0=B1 ze16#ZgjfjSbLjmR!0#nocV4gu2@(AxoGZFTkMQc5N7k%4n4R71N>x%cJ9T2R>}T#v z-_Xz+JT9+lM0(~?ve&NKgNZ##n9Ki9*PAU%Z(V1C z_xTjtj_$55m#qhIMVBANu>tG^kco*HG7mC36lKeG(U*1Sin=LNA}LZ7C6TlSN-`r3 zkKU`-xAyuL@=)bp;(vfV`R9OPujw09+3a*t$H&woa^2IMqtq@DYHCmI5+kJX9&@yF z0&+vc*rPMU9yeh@q@M)oYto{hHkV^?n?aZu6;{J&|H2u&O;WvlBTFLtC2$Wm%r}NQ zNnvLC1lP<1B%lG`Ip`(K1wTi*dULeF8Q|oV7MhTx1i{qYHrw+GB0yspBXm{<_-_Qa z#M%IowQUtBKjl7Ucb1S>$=*SzJB6U`7ISRYQJMLfg&}H0f&XWC^D)?dt;D5ACeZTr=BhnK56clU&C2sX#SoD_e07L>b1q zJ{kyM3_F)049?>l0-eFe!7GlUhp{r?k9ij~hfGafpN@W!l%(Nw2B-O{;4SAS+~lt$3TB`t{}-f zHN-J3W2)eSHGju`!}lz<-QMGCBM0P^;q}Y#{~o(vy?sHz!_SXLzXV;-Hu9sNzy1U{ z#?R$>*+JwX^{7uH9+HodBe0jxV+u$kUi0#Chq7oom}+;akVw3OU*jL-bmTF$%i<{(gk4=nyGRJONVjRpOOkzkfMX79y5*=f<1bCERJhuGSr>TxR$MW zZYS!@3K=!%f~eQe1bxY>d>`eyNGB*gQXgFINPu1sM=q4_oNlNwCQ%81;`nx!=$*Im zde8v3z6e}{=27KK28sbl;&;>UOyFVLDGJ!04*N3HzZgVMTR%ahtVD@VSFc%{${R5Y za;{YZBkzt#DSSr?oPp)_d}yRAW{S$BP~4hN^5Nkt$R5hD1-D)s)a~51^%An@z?;IH zC?q^%-f~=#7&=JuO`O6FC?s1ehS*?*&fB4{q%L1b=p#l!&lq&)Fr6e>5ZXf4>7d zJ{%5eTBJHJXD?AwoPAsxPiR&mw0rHz)XY!R>0MR}>iS4UtS+%s|T>Bbe3H=q#8k+s@96oup{H{~DbBN#6algk5a zi(ZURivTERI|f$*;36qh4UG5;_y61#jV|Fcy((4!A5yilY0XNKy}01J(3CtLi$K8 zJ(l*cwM{cx8hzRw+aKx!MhLyMb#)#gaD-s|(+6|&7A0_`J--OFQrEB?dr)>*f$_(M zyAXy_j!2s@Hq5&i8CVCVOlRQg#v8}?Xl%8;gUVdRJhf^rafd3PCSF2JHS^}zsWC2&5} zCsR`EJ<^LXZt4#MNo2k$hV>H(jM~V8{U#C?67`3yR!?l9+%n5DbEg|izy^B*Dn-R< z*x;a&g9SUBJ=#KoHD0co|MFjGk(sSGtd%btr|&*_`*C>vS^h0xfv=y!Bts(We}DV% zyZ5(cc4!-P;IqBplrV#q|TH2uwroyWkq}K!Nb0G!U+eJ49!(s zAs$X7UpoOO$l)iDQucf)gdvq?J6>6UFLbc=8k=XbOqbh+L3M2xOA=(KY!+2^+1S=z z6u^9v3w$HGZ6Ip*HaP^(i_@Jg^{eWOcv>7D@0vgRiM}t;qL6OU*4-F$V0=hLV5u+=Y zwSzNN?>He7?vd4DmUaA!u*?V!h~Z4jcQ64Hz&_7lmpU_UN83?O!I}V9W8elMs|HLy zJVuS+6-I_~iP*Y%MzLh|emg@2&>4b~{`9Q9&QSUzuMZ;)5yzo$&fpWC}Ck&L#}O-+~cXdN5w9&edDAN zz`ib$eISPuWP^LqYt2Cfas$=wNeyjUeq^u;D));nM@-$|y)ET?K-tv;4C>gwHl4BT zgp!(iu#pXggmBPm%fKkA-b6^fp2qnb8Rx3Men0ec&`q*!eYv$~IxvHonViH`Ns4nP8zBl|n_B0*N92mmyLR&JIXba)_Pm><#Hpad#F!V-~xAROo) z+g0f_w*9+G^6QtR8vwQ=%yk=7^>FMELp!i08*rXtP?|72AlybC*(0rFbmtV?m<+ZRK5J$sIb=zSXiO zKf28-h`qD3u2N0#*nUdYvhlX8UnZ}-p>lbD&B4aCXX3S8b&icW*7|@n&Xp?cci7nF zO=$RpqhGu#Qw}Tf;*%r{p|-Ue;nL)mB289|*Nr7Y9bQMXB(|^V5!i`THv@nPF_&er z9(6dq#&ZmJK7`h5K~@5SP2tG?t-$(*V!7o5*^n?AX&OV4PQf?EY?l1+F^Bxecy{XC+@JV=62 zOI&zRwNfI7&(7u2y`mzokUf@9o}D_&3(obK+CaU6DKCtNOSiiD>W?&BS*`0uVCx*P zPo@)^_u~s_2?K+?ynR$e0|LJH__b^|$-cI9P542DoR{Td1-+bl8w&UPd3?#r`BVe% zpF30iC?R9h62Ky~>er13$scU|7x3Xm?T-3@o&l(`pIx%iLXZ2Y50`i?EPd`KpiAD3 zwMP}dIwrrMZbHZtF0#y2x~TJ9IBt%|j6iL%i2zKy&aDhBbH&-dGQP`#8w{PoAb-`x zvXQctGJBmjL$Po}zZl-x;Zhx7K318HTquO>D?Fd3TyhCX;ULR8O|O>*X<0;<9TM|s zT1h&cpN-D1ZFL0hhh>+)sEvM>OhZa&Qftppg|vh}y+n1ee7cPikTj7fgWLveFod@I zHic(7wT1MS%`!3GZ}|hx`|S zYh79$OAnoG6yQY*|7`gi!JOT4iIT>}2PX)Eb*0?%r(zk*);fnU0B>D1O5vQDcSkgq z{9`&-4=P_tG@DCH0L^e^j6$yJb$a%Swr92L(Uh)?Gi1RD@wFgx;|JldpNQ>Wy_uop zSC}Y6t^Lcl&qCffefR0xhpHNjK=qXh$w{2rfvbx>2x}TYWdllsk_y}@_IXcaIo)Hh zV(hkc@cpx(GRL5>a|=}y(wgd+9gk;Kc3~tN@`{^yUbDl&2^QgK$4Z{rxp&AtNwT$^ z)?4PMo-(MMH_STr4ol7UOLN8e0t7pdz?+AdY);bQY0Kt=XVmhSxJ&L=bG+QC&Nem@ zq={G1z+_=q5V4`!>}A7!FPkRNlbQrlw1RTkdlXJ*^Gz@V;Jp(X{sy(XBW%vNumUV08 zfLojToMo#c1`B5ZEde+y!kkSTs7{%?6HG&{Ttxp9(jH*|(8rx# zT-jV&PWxHjIp@qhM3+dX2h6L|Nw+&u)>2z62PYkN)0Dmo2K3EGsoOeuy2yJU6HQfdfMy)Fn3GbugSUC*Qj#ASIXJclLfgx z1+QvB4y(60--T!cbNqw>scj7vRueXRDpKolOR97E z=AnJ~5F5?r<(L~7OFq3w_&(KruwiJO(6(o2sZ}%8l}f>Q&rdrenFLoMx{0);G`KTW zwW0NTHZ&MK%eBPb6_`-9`h{vYo)s;w7}JJrA&W5(8dhN(f0a{5FX*gW-1=f8m**H* zOhFoH-3}%0QtJAXGGQJb3e|xu`8`?UQq5k`faWZdeRm0D3wt3z{5X}Q#PC!uS~-=D z+5(za^10UG=u)Psmaes}!?X`xXCAUK7`U#a1c5HwA@ra9V4*_|cSi@Zs}Z!?m$8zR zHG83?T^(G!uJYg?Av$DZSs%e+OhF?`)gG4nf-6Ft6$OOvb3jCt1{`jOXnTA-m}tlB zy+B-K4(7y%wR5parpo#G_Ne-|yndYb^hG#R_Q$??{f445`4#^y|K^|ZTwv1v z*?emsDLHxkC6wn5&~k}Oqbx1=_S&MM(K~RwEuM13n^OKyunU3f$2hn(8JgT{Lxl*$ z_a)Ul^)X zp@Kfo(1KG_s?h~ua%o$&uz7ZuXL%8{G0G>)%5iA{;X{5=HfSuv!Gr^Cbv8p);P$=hgvn9jHPE+@2Z!lkl{2h%fkSBkoFzbR)#0GrjbNm4 zd#D}vjSNd73>POT%2`6usavRR>QvV&^iCZgSKm9YSynX*PuqiGN~^@SE6AHjqOMRw z&sP}CHGMh1Hc`EsZ3n|)!&!qf-KvT#gSN`z#d;(V8{FO;iBYrBN%MT@KehuX*Soqs zS6*kfy&@wzl%y?su0f(8ddsTIK-^WM74yJ;@~q;@tYy(*I~0?#8}?1d!zKdOgyoJA zvDKounkquX=C~CwX86Ov(*VW0c-F>qWyPZ;MT#WNI~~5OsoU0C%7DRTW9CS4>ssz~ z3+r;5hw<3V*jCxBWySYg^$a@dv*P@qDiCFK_@|23%>X&14QKhdCFBxK<>&eske1+q@2t*2UoSD7_MSMtnr-5Cg?pcl0-n!Cs1u3lYcW% zOQ0!ANvcoq`WkhftIwlA7zZjJR4}%5vUWCsk-P%DpKE%u2$k$=n`=6{s`FdlKZ*R} z5!|lTQ@=n0gZhAYM*{6`S&k&TN83I>6s8SI@;eZ3ZjB9#8XayoiFzu99L*=4z@i1} zp-44U9xrTJgL8R?TOkyioOF&dAN3syYPnoYm>0_Jv>`RQZ6e8LAd*q1x662nHQtKf z33}jT&%TH~Haiwc%2TO7|5X&ESn_) zJ(c@WA@!yLlOKYV{vwBldW#r1nT{8b=LC-#Gre@6xe_jH3mgX{smBY@aFZk#>dj}Z z;~{MlgC#q+Gyw!4P)>lePlG#vB5dAO0)wtH zI!l^arjUOX;HS&-z+RIJeZJgrx+m~<#5Ebko8bBJaLlGOyF z^zsQ8?h_y zSW*$^E%k>t0x(@0`x82&_lBzV!*~nP)TOE!MpexT4xM39QP`_TrmBH|nT?*6jvkaN zVGjVgW}$X{vRJGz#p>8slD)E_SQx%#+^w*ept&2%LUkh~!n(_;+Lz+OqMU4_l`+{% z#ilM%6$)+Dte2dF&MWX(Sxo;HHP#NO=;~0g8PGebyU6Oe$`2^|1d19;SOo`fW8EKT z0P$2q@W6?6262O|Tx%WjLjpc`SX|>;1uahM1JUv}kSw?HSw2b$*6{0sOU+(r!rT&? znwh42=+w!_*%ht*IQ_t6;>=P-hrIZcLnnUq`!}nRWrIJD-+se7`5oo^o7d07(@T8u z`YqMyzRi#R^zFy5Uju6P_A&l`{WxzUe*c6e{D*(~(c70+ivO)W^!EGLkL)38)X2ws z{p|E;@ogvK4B?R#B^p!yeWAxl*68V3hC@`^HcKo@c^hQUL?rE@7aDP!`jl&N4C8edO< zj3U!x@L8H|T^)9<#@~lkREB@si=+W2pwU6COKM>})+eWosUvmZ&^g6Cva@s;J@>LD zY#h>yCybmAr*qI32uv$^A?0)L@l5R&|nun(Cz>fALmtqP|@i3?D(1!AEC%SDwT+>hU5hQX~$Q&+ONtLXv zFm?y2tGNa$-UF%mPMbhOO7gw};EEO<8s#6&bXC2feyiJlKwH7O_wAvBQ-c%5#04Gr zCpGdq^oE(mY=kd)-V|JYhRE5-c0Mxi!G-6Cqcf9U%ERIqQ@z=EOqnf^HxF(=P7^wM zwN$kg3=-CIQA^<20&?2~h1+6Br2^;~EQpOk79p`~v3`b(SLMbl<(zK7n?AMq<;d8D zR${=zVUH{ufc#Oinq7WyA*n{&!z+&(>NF8lR82c7A}9 zR|7Ci>%6B`q(3J8)T0>o&LJo__`l?%K3OmU6iIL#)ZPO$mUjMLDlnTb0D&!u*}xr1;EBeb z!^S7crH9+P-k_Jo(R{22RVJDm#@RliI9`E~wOIWi{2tn7r~$`2s%o{Kujv4{0<=fu zSYmS1=@ouvK<6w2<-4UB*HIZ8?9PM!EQL12ymI_i->r*ngAy+(rEJlc_@de$R*^r# zXpIOc(rO>-2&t;g#@0G!;3_ubt#$`_E5C}_bd6WnmKfKr+mSnpBq1cMZlMQ10324443B7ON;8M>Tm^m0Xa5F3$1Ec( zVFpN@HdPFh%(83|>$Z2w?GSF4>VL4~S1Rouo)o1N>PJgK3Du1Hg_@w1G;5pDVxtBs zkcKwpPAY!oS1#6A)TsSfX{p*!TV2v!b5q0Yj>(73YJd66ar*T2rwm+XRPxE&mw~h7 zPozeDvw-&T+aDdx{G-=jKk5`}OkZaEK9APbiFatXPo{ReE3Ek?81*8Pb;4m0s$H?D zt{n4|OuDQcCxA{4LCTvP>qMn{zVQ8>{(#^n;}!vXzqs=5ko{pmM}Y>9G-`5`_Y@4_r1FEE9?>1#5u;bY&tWd`~4_ z=+vnr>X;9_Xm^MZjh-I*#*6M=YDUkF|$*y_=B6E~nIE0To#k ziWoGlWDRd?cC$E`!{K|;agrrS4H$P=lJ;OsO7W;i#i!Hr)OdQ2vjek|nRhrKs)1rA zpv+q5HrA>@n?nP@Pl*y*aN>mE5w#w#N_(}+PqJFM;_kKsYd(t==*oMR15xQP*!?4+ z1%9u9*8_l6qo=!!BY#=zs&n}4T>xW~zoou!XO)Y;!df?Rl+%J%CW)52je)A+C@#f9 z&*LOn*qRQLab>?t;DR=cB)e>TLX-Rf?$Qjk5FcbWY7r-Y_ZHDs%}3<+=~xhAe4JvV z{3c$*P27{ok2Vm-l`Az(S3(6Kdpg$ITs;QX5Dk@r&(z+|Blx&U$GXe2D46Q@yEZvm zm`Wwis$V&r-Ow*1wI)g0Z9XP)F}r(Dn%YAJPNjbA6zLcPGfIkj!I;P*S`UsETD7Hlr#bkmGih|S-#oH1F6VF3SN(wh;V4z5s+ii8K*QUbiIYjOd*@=gfcy=l2(P~sMB(=I^rKE^Z zo~X)id@Q;t*&KzP-RE$QhyMr8w0BSSevp&_cI>e0>(j`mXo7iq>CDap#j^o8xj1)UYDcgP|r3xqR>!_J_HE;DbPIcZVsBVJIm z%VzAs`@N>sak26Zk!IR^qOE#hlhIo>2Tnps}QoO-4%CYI$jxNy`!-moS!6%973_ zmFbgnR-Ugqj0^GrX3=>IhYVz{4Hr10m+9$|+P$4}Mdm1B10+;PLBHJUotfyX_vx&#-z<190kAS8j-$fBUT&91Rs$pxr3 z;E^NKpbB7 zhV&r^0T9TNINwK!6uf9}T5(+1@s$*brXb+Jhcp3VHIsx2kIY02^$mRi`3b9gIomK9 zvtYb=V@T?tp)xPASeDG&5w52OnyGyb-Fa%u&aH0K_fy#l1XnLBtPgGav23b(16BPa z8Z4`7_Ahi9#D3^|M^#OLnBqJ)^awjs(lWb*aM^;|1+oEX);qo`2D`zbq~v4P=BU-B zPEJJEqE6m=0<+gAv+vlgAxoOb_65T~ownNQDdofn;Bk(^v-rKjqnZnCgskxv?>IUwYqZihyv| zEPb}*vcTS`q?1*z3U2PmQ!$Iq@hrEpSG#cem4(>aL1$>x7&Z#HnH3p*-tF`PKc$wv z@>FWO4?@0fss)#ygTW_xd3X4fR4zE7GfPbS>CT}E@pxdIDcvrqQ#`khWTuYxQ(vb7 z8=PIg3N7t_JOcl^nL1{lItsCraxW5k%*a-bJv!$ccb47ZZmqf-Knq)k7+`!CwFbx7 zwmU@}Jb|Cp5d@0Ys9>n&rRgAd(2fFeD(ZXCEx>+RcaTn^l5qj|oK+F77Es}o)FvlV zwF{g(6J7~>ka8O;)zhw2YIRv$QyUFVQkuh!!*4}@;cYA_0CT-L0dM&{l^wD_2v}If zXbgUb0ixUk@8Na_$5Xj(PEi}6 zKC}EA`xEfj5#J@)e~;bdQUA&C<7aO_fBi}L?sIwMYp8d89Nx^P!)&(IUK@H~->M!M zI^>`DRBa!(TdK!Z;dc2E+~*!=_oUic=XBa*Ru76^*uDgt0l#GOn_GjaC>|H+d)pI| zjB{&ogu-QH$@@&Hc-6o9+P1N(*Imc-CveYgE z(v#yNm=l#eY*fSs5r9oQY9L8Lkqti)j*n?BUjwcQ7*igPAkTMdChfRp=KFKhDVt4@ z}g@0K05i?~Rk12wBoJ zHi1kcDZOBJN=Z!YQhj1*C8$b${vz#+LJ zz(sb#b#uD)1?eHY#z7mNPc2%_2NY|zk%iua#b8nlm^SJ3XklkTHfy=AE@3SrWEBTg1^D3M(~R*7s)K^YApFX zSoo;4YQqNf@FoHbR>zkx%og-Y`WG2BS}WWcO*j4E!3j%5fc& zhOO$OMy={(ZBwe2!jx2CMJLo*B=qo%R%ee>E>oB~WdHEjpS=CJknb>F$khBVX2GsCJIiJiE(B`L0#M1U#_ij9~$OmXLPaC?Y5_ctvp5S37A#Qb4 znUSriFA0|B=xSlI2n0sam zRgN8|;0sN_r?JHRQldOzVWm?bGR>`A{Q1wa&lYR+6&>{caA(m3?JAi zLM}}Tb&u~l^ZkxBSiTy7PLa(Fqz|M>_9KKpBx_jRjUF>cx%f8Hz!hXCTy&f z*m=ctt+PY|6!PIIgTOL6TSa{ZOyvpGU6?}1Wy4I24L=?D%Wc@yM!dO23I!FA-ofEUs@)!vk$Vbcb|a4gGZc9oWYCQ>%10 zmAl_f*g?p9CyoY;83OfqI#Z?(B(9Fm>}gL8?gpl$KuOP(3)Du14XCmk03_{D_6})v z;pVFSGx#!k2jKX|bZuR(bHr3(Lc!GfSt?cR#@qeywTBc}EOa|>y5@-14lCdmtFz^Mjdu0kA*z*7-0#t>L9Ycxx{u_H^ojrOvUaUYK zTgXMn^bwWz6$q))lK<|RXwhaBQg0U_jrs6hEbhZLfN{K0Nosh8(UdKs0sj9O2kNiFGhgeM zufGo8H}w4C^-BhB3(lXtejVQaEo-5Dt+cDxe*u~5*Ka@m^S{Ht_Y3q3J_@ftL9gI< z5Jy3;;1@Zt=Ng3Kd*p;Sk%U) zm=F6ceBcaflm5UKB!7ksYI2is3C^LhzCcCV^afIfkLy9VRlIj;arwn=#bxh~=zP}! zs0CZdxKwUR3+Ol0cL?o+$$+d3i-vHv_+9B%TS>BTK5S2}9+i(%b~SqWWrC)-Tw~V9 z*ZhvF3r1;aYSI7K8J3f=X80!^v0t(q{kt}>bkGBlEBhP*rDki`hML3LHtQg(_-3IaE+~thFm2m!78yft_CjilzW~QFc@Q z>2jSZe%pn5(=d6q5Q(mB!wEdTyu;<>Y8xUh7)P&PWA)uXz5Xg(0c|zu?G1(sM~!Zo zIS?M+jnDLDn8C-B3RAcxQnQuUgM8AA$_ATlN*{Z0K;StTi8;GSoXk-*gm8w zq)JwoOBe2hjN{T`Kpzh2f}~=DmJ6gwVoyE?sDKaoYyfcbW;)}vqYEdc?bPuKuOvI2 zocN?hrq?|)nb^0-3_KnY5>?5QAhtK`p%DDSiEl1h`&_aTuC{#bG@-72~}HBOibt+EH$j_8}g(K1|=xGJ=+(8)I~E zxyE8+5ms)`I7(o>1?9b2m{o%-R~C)hbk~z_n3O_|q;l|EAEB2r&O&N=6w< zInz_6SRml~kXEXYVddKm8-8@fxnff8fr)y?tR5_}_>3yL`qsuiucd zmBTIWZK*^p&)Zc^+@^Fdq07L)`^ZL@I#NMo!y@Bt?6koxZN@>WuK$Ej*TtH{2U6d?Kz#h`pay>Ocz5m?$FB?qv zEoymFPt-t)3~e4k{%B15Suw{| zDqK!#LUNqneJVrGx%o_sxhYQShge>-0JhZp0|drSgWN^a3{8+hR}ST#C1Sua`@Wv~ zH5^6Xj3BtF2Nal|(avI@pg=KU&S3kM<{#RK4r*?RqJNvc$V9O{Q_~PrFU0A%u{|AH zl8(a)?P@s$R_9=_TPj^i!PafLvaINUFRsvy;eMbLJZj#jkR8qho5R8&`gXOMh{jby zv%6HxuFrf8{Ui|*hac-$4w&*W?xRJeOH^g?7yIz4kDF%eKua}==5af~a}Rf1+xu9& zB7m+{W9_bEoRg4yg+(}b5oeZ@di!}xdT?98CY((_OnsY!mAJJEkl1Dw4iATn_+~aH zpKEyP18Q@Ju61XC#uo3r5d-5yJUG{|ZPSY@1Ps2pKM zK*22#IW*Lbzar#i2z1#+#Dt&yiflx==1$wMrg~HG!Z&j!s4AM}l zc~eQ?I3&vk)|MBgk(b<+-*EVB?B8dtEs)iogP6owh$A~D4FQ~}vRaKR;3gFg--yF+ zA}ZNen>y|b1@M@yfJJn3nD~&CpIqJ>#(EYMp%=xX6U&z=|s<>P$e$5E>?d#X__vf#_gRkNr-^`@(AQcCYV!4S z*gX;Ka|fkMR#sV9cb7@Qu|TH@UCy5E^t)E}c{9A2cvhMfToE8(!;AP9)|`(?c6vqE z$qvUh`^}xLAfUbmYei2Y*A1PoC#9(j+g4RLDryZxi$OqZb@>dX1EXgFn})-6K~n$T z*+~)u{l39*wmS zmIf_}yZwwe)}X*ikSmIb96cbQY9|>SwY*dm&tefYx_LQH^9Z~MlO@@8bCO9V-ty)+ zuZW&H$(1w5;jI*!n}?~;u{}GPbRzeWf=}TtrC{hA%w*-(IgJmS?u~cI)JVZOS|pO- zaSgW2Lgl{oW-Y)t{b?-K{B+oX0;NpGJ24EDi!u`n=QywxgLO9Gc)A_|%SV;qt6rY+c6G)P=8aZShjhiPGCm_(L(uJZK?GGyOEbo$?6tI zC5m1FB^%3Ab`;pIcT>&Q)+ywnDT-YNlImJC5qO4A-ij(5jfSgjxZ}~tfY zU9)^_D21h}gM)9&_oUb)gg79|+cTo(29>*9)V5A2u#ARG7l@lv=JG6;TfWzI+Nx>7 zlf;W+LF4XnFtGrn1YYpzegtC)u>bn(miz@QEIuf~{k*dq5#R9Htq~>MyStuG9KLFNSI|97PkPHHM4v zhVLi>Pm5e;wP!-vrbZ(I_me8IGd$C@Zh`$+o3!pcpO!N;U)QoZdtE4%m^>X#)6HDj zVT&=YpbF|P)E=M`Jtxg_v&|ILGEiZ;b_Tr~r#-yb&6a1O8U8E*4_74>6DBoFQRQif zNCGqsbd{Y~B}FBpeo91@I&R$7VN-(;`6LP;Vva61xpAl9*|(jvCg?S;O_s6tJ50#> z4Y0bD+#?uonJTJHK`|>lOkWzzbEzzQAlxisrAtfSmp=PzStJ`E_M;+8tt(Zun2*KY z3r5Maf$ni33x!T4x+6ts(XKvl+)8S-WERv@1KawwY3J|5VVc%OSU^}L1O__UIQuGB zW4`zPBH+Nlc%m!+K=!)oPSb2Du1GH=yfCRQ%T(`=AKW-E1u zYsttJQ)MJ)k7dC;E}EVKx*8A)234y=Pu2PJRpWy5kc?_1rfO7gCTi2nL#QC%*QNK z8`rmpY|yLBlfiX1DGWL~?G;cJst5?$V1|pzPaPpSA}G`Yvz9Ut96F93AuoweOXXEy z)?^07c_-tEGR;?hNLhNqQ8squ;m4QYL&gL|8$}>ppj`1aSNyO!i{#VevcaqY&Hma_ zNI>B7#qb}(|8+#7Z{B|L@BIwlf4~Y!7&TZ$nP2Dgw?8<^_tV!OhqphTzWeFxhvD^S zC?MtMzvQSC&r{qa zE7@RIIII}bX3?l~yf2Uz?P=?Dkt!rH$d87~Vk5xn31ZA!f+Ph6e^og#8!e3h*^n*V zdYyr{p3*Svz6a;O1dCZtM4*3B`Mm&dD9LH%XDF$!EGcgxA%K;nV*SYM9VP#jnxca9 zDT0DM+_kW53;Rg}&;dxsXiLJw0!oA3a_V{1tJY*hM-}8@SPx1Z+n}0J(Bc=~WpwQu z^{V6!MurQtG!!)J)%KiVnL^FQ>HrTxUM8eP(yzR&o2YU~Edpm_olPK8MWza3Lh-dQ zn^G2${97~d-Dcc=n8bL00C^>r|`he&G9pr+Ld zx*42!)4NezueQSO$k%M#lEc|ce6ql$ryANidAH^G0>FO)EW4CNMc-e>XmOT_ajZuD z?*-iCB29%&^saNdYqZ#(CXnzovUq=m>ZqJup4_Vn`@s88yNx@>jIsh_jV3ndV8dV4 zjC`OGk~V^Y(0MXIxWuxQvT>US!b&gULQpSG(^{yIetty#7A!uZA(t!!4rA^K+HQSR z*G3|iLqq^cHu%kKHhVx;x{0be8NhQ^$d5guGNlZOun&`40PaVeu1XE2`51Aku!5Fs zXnSIs!|1od^mU}beR|0p0mdlBj7>AgacQ97FI%eFzWoHH0*k2C?pl?VvR-GbiBjuM zDw^t=V&?pNym$rfQ+uFqfg?~YW(bGbz%qhq>GUKdoGW#iWw{u%Q7e~h1@KpiqKu@V zHJrJW#!r3$W{P+7PU(oih1L=&yRj)qr*QuD`OoG|Ni;mulUNJzJ6mr?X@I2zx_J@9WuiR^|Zuf z_F=#~QP=E_roST`bIefYj!br8Xa=h)z-MYDMn+;Ztkn-XqFj?LG)KYkcZM85yDfjO z<8iwWZ`xcz{O*z)O>QVXM-S$Fw3eJp?$~SR&4LXh4Jh*-(r7rt&cgs`o}atxS#7`4KM)ifrSJEUnT6`cjO*1OV|y~DU{L+j|ytlf+OEg8T}`4_oo z*cM){6trDZxoL|%XzFRPJ=E>ZVLTX8_8y3pn87fh@!HWtGc>JX%Zs*qO|=0iB)9AL zDMZCC@%k~&m^6mMEXh>Rsv>d^ZCgrApkAeXR(rT4$aY1M0nm!&jcv9G7C+l!sN0{^ zH6-j$2^yH4Z60WeDfJMY>)OZG$NZUcdF&1C2$Z(0h2zUv12CjPt<5|gE{ebgorWF6 znW6Ufv0PY}F~91dBXV|-(_Ky1Wdpg3GTJKdoPZD}0ZGjmI#Q1tB&?Pq$mKk=PC{mX zsc-j#wbKsj9G7=W9HvB}E!Ad{oF;?RsauJZpLxeMc=e5_LFUgO%khv0PbIUBGh{Fp z_fu#rtxz`wN&0#ZI9JuJx=Y1#g>}M}a@Oi)`#cImHt}$fQX+LqZjc6w4wtq!t0y@` zSVgBv`9kZdT{`Mwn>q;4a|TyZffGra>q|3dRj*g`|N=O95!>qqP9P>mLd35YZjO3op^3zT*2Zfca|9pKGn z!_gnt6YUzN@W2SJV02C(96(*Nrv2~>c)otp(@U6)^@B`9qTG}4deWlA#Yd*LioqQX zWw*qdl)O8xP$v5rsR@#yv9gDjm1m-Gn9r9Pq$;K2uD2%*J;^t8KrlFixg0n8zTnb>#%i^`9ljz2~!KH^DDHMsNKR6k8OA^){*%SjvE&&oDM=gMUm_(Jdl1K%b+&D;t(jzfH8 zQzl3WDEGm73`ME6og`VK7yC)SFVYpcO67;FtZYi}f%e{#vUF?^#BEW}dqSc@?SQ6P z)>*U!*5R;Y8sHHFA07u|$PL(+lCJ?^nU4$=1Ivp|$2xr>mGD$_?2(SrB<>*7{Z0bmM06GfW)fGq< z$YJXCxKTP+8N*lss68?yV7Zpf#$}^cPh%tZblz8qvSX@^72q_$NR=H=g=WbUB#{V3 z3j|haJmBmg+Ktpw=*JC362dGgKT=6gV|VH=@=qTs&M9Rgdh&hG28CL#b4PBr-Mjm7 zm1_rLu(o?edFIUrtnEeH9pJex4+2N$`Jgnl(9)^WB%g=TJTi#`6^gXN8V$I`u1pNh zcC%lSE=;p4M?W6V{*tmTm284Sz6%EBehNMwkTA}P?mZFeQL9xLlrhyV z0n-pG=~R6e#i~^@wW?UoR`t|RNGxn8J{$-LJGkXQEUi0h7_%sQ&XQqyyS=3d zwCbIZtge?MKPwzKKa$jVg~T>^SLr>b#N>?&=%dbM3}XDKb|MT{i$fRaaw7PNxptknm= zxp$FY^a8m`sCZaG$<7AmYDw1PvIv*lK&=pYFNOiIj=IVo!rzIgt& zz%wwS-$Ug>ByrAChO>4fDhr%Q93TPV`tcFuXkxh5)-mJo!O9E{LP7s!G72f5h=t=k#Tw5geI=M3u-hPp{eiz=pwypsPN49Z1^UK$d0yk*9 zjLE=|16G_1@{0Aq0EdOatVE_Q8F*gDyL@#Je^-3ZWeh->?RxK~xN!t0{@AfJ7g`+fIM z06Q&2YU7WBXDp$EB32VX4#Sh$lpJsbq0{(sfXBwny@f+#wB*nS4-gN0(H(cFfS&48 zjnVIiXFz|K&tGsDkKQcW2pZPdF@Sr9iGh?g4LWOo+;E2D5&4jZfbvL%&7AOH4^BLR~VB7X| zQP(8HUZ)0nP1cdn#x?$aCZ#|(&~DatyyGIhUlWy*)a|)IOt)Nt&Q%Z$+tkGIY51NL zo?smVi9@TR#4uI3w0-NR2>QpNah#Z39=8NKlS4u45li{0ZilFfR|7F&mBBFT@T5q0 z2Ir(*p5W_e%(UBdN;la4L>Iq@o*F?^X^3FH`FKNe{;4@sF99lH_e;ukY7l|NRCH3d z4={RE9+N~r)abX7geK5j38W1xBDLPv*07h-fo*v+(Ugr6#@q3Zw*5IQ?im6_RK|FW zHwL0P$_Vv4|6Zkjm38Ev;>K{3NvMSv8_RI4uM#Gofm$xy#c@l~st7&2jB!^d53mYy zfXDD-lU$Ob*Yf?P_y(0Xw!l%#f@lanAgEA^bC^D)Y#L*`U}8D+6F5#&a*N$av2E}j ztirCaM%Y_Q(tM@7%MZZ!`SA6_BaVJ^h;)8-Jo@$PXC~75Y{vM@w|`|j|MRy`0{KWE zzx@#$-;dvZOSHBW-r<$pqd)!Q+lPPtclh&%?D-PPzX& zzYMS6zEcqUgeqdcl>dGw|H&uHJAc%F^N5n6Ys@Ep@-BsejR!#DKEx5A00Neqtzd}P zNZ%|$P+txA3WOaafb}?2HdR2^4fcb{0{XDSN5S)noAwUr1i2j!RIBQZJN5!{lUZAx z{eY>G+Et~&%F<&uvi@Q_ihh-^^Bw}KeYqG+mIfBeE_QDspfKr3hm40haX<&x2%Qcq#dJictxJbomQ8`WF zg;fs@%?7z)*6^hx&S4rD8LF?~pm?2T(af-K$T(i9)n+Z);=xg4FZpjI~ z3rsB*m}^J|YB+V+Q^-Sd^`)+fHkC}TYyw~j}$Z6w;2zeQA`r3 zhK*&0E$5*CcPn_oJ`{+2*0lp9N>=0h)ia7i;~6kMEY1Dat*sW14&j54R4YF_2#`|Y zTm{Ib0Ez<>8Q|3)hdf4~bUeqM3W0{x1;|m`=DHT)$tD_Ok@(`xqCSSpW{{B{>^I2( zGhVgZd$slk_v`E?6Ib+PrTDv!3R5(f$QBowbFLJe-jZ&tdof)`Uj13jyvx}1-X7{u z#`OdzBI}T(0xOZEx-D=hbf=W`byL(qkvJ?s25C+I?V{BD7Q=uYbMP>EKmhcvT2v&= z06Awgnj|7?D_~JsQkJo2_(8083!YV-V^>M9846Q}BvE=%aKzF1S2aSnb)ICUlhx@Y z1(Vq#`W)Ob57Id7!cg35WV4Sq($j#W?cf)yZ*?ocU}y_zwz zgHdV=Z}mD~2{8!8gBvZ5BWs6pluSb|)>;x2J_4%lQabowJN)>t7-CV2sB9H#OxaP@ z`qZCY32&;QYAcuM-lSA9*gq^A1LSB<#hN?s?`~EQxJ;R$==#7cJwE_~a^0TSgAI*7 zLw5e)KCVlb_!`fN00Yo&ux?NY1TFsr;2uV@R(k^^hF!}j`mj-^K0@gopqS&yl4Puu z2I+9!*`ge;jhqX666|v#eKBNBsi1=*m#_W01Yy(bIiyPvWy?3o-@zC9FLR3WgCB(N z;}Q7tNJah-Ob^R&1izyb*VoJtzIN%be}a|dj(Pn+yGZybz)WDZFg!DF+%PP5YOe$V zqp8S}CQJHbRQ{GTTGzIFvRDJFRxxF#?DtC6ABcg6k^8bCAQBp8Kbs!Qq!#H>Y1S355hVFJFeXn%4;fz_JH|LfN3K><|#U1>9T$UZlYqe;5DA*=`Z%A;fxG!AUt3w z!=3V41!hxZ51{pOxA3vmMCZ!=H%@5S94+zhe)Re;+`|-`w@N?iv^Rp9WYsm~j2Se? z5GU5rYJ|)k<#lBusGD^#Vdzt-O?V2a_9gJe$#S(mqV|&G^kB(7z`jO)qUH;J3nGISP)lf}JbELBz*|$|D zYo)>asaJKK8kgdT;a9ap!CjP>GDY+ydD2+{e*%YIeV}7DbWzz!GN+S@GF(7h zoGiu8rjSq+pCcT9BTTfL)K+%_gS4B3D-_;jL&X|E>iQH8S2)>4)#n&|32Vuli7+xP z=T<_y3ZuoVvCKlzN(IvF)2O|To+!gzS&ZX2K^4VQclC#`c0Wu}+5MADtHxl02l6IkVB@k7D@~5e*KtjhEKxlFR~cP zC*iO2Lm$eUfA~Fe`2TwS2|5(`k&FKa##dH-KLR=|yQ(GGEiTo8oj<4@Y4m~!X5|m5 zfcDEvc(GzX8$i`jO1TxRvq=ZE$9pD*~M7oUEF{k`d;Bi z?Geo4QRi{Cu52zhf$k#5dJusm<{8^rCEDrW%Jv!amV^duoU@VxQVza&6Q?7sX=2m5 zA18oKGy3=N_mtO|*c)i2^a`7|IlZC35SzWIz`bok7Nr|E>W51~Z2TxG=ZD|~0BF1_ zprf$l9WZ&>cj-gSiiVYTIQ0cG%|UH#sXlOj1! z@s!;pB}7ORS<$8klRXjz&{1j#aPx$jz8v-*Jn~%AvFSR*85N-9^61M$vnEIIU5=YB@ZfOERcm?!ih=$qbRrUcUCDOwHnF$?YuxupoTnW zi|@6al>Udl0h{r&6R=YMM2*9Qu#{D=W9fmmyw#;}1{rAJ3E!A|3kl|8wtR;_Ij+lIp9nXqNZ`Ox;N*A^7;hZ@gt={;Lt3#kiPHo+$Cz?ZC za}dyBtzEqm2Ci_*KsWPUQIFvqFhKy%%H4fofi3IpnJsxVI;u3ei{q0h7Rukh{Ojj% z{0n0BKRO=$^7TVbzpZWl(d#$iz`cD0lP$pCKI9`me*3*X^7eDa+MkBEZ*)G6*ODJ^ zpQ54u(}0e;GP)~Ud-p`lm25R5$Z!HB1umSOwRpyXhR&@ojz56~wmxjiH62+k5I6FGG*C{qo z?Tr@RYZ{1KlO^nIaTpd@;NDa)(?ZY8JBLzALz83nqM(mEIID%nwz-wtT{5>2flFhF zJ*b-|ubV!)QY<(qGfL{Mf-1H}yez9b-=jm*pq@yHD>DqqM31fZ7MAnD5{+-Xz8)T9=F zmu%a@WKS7$23yAfaKW&`QXDu~7}QABV#e)M4f9OkSOtpqGa#x0!b(=dp#GJ7(vVa; zM81`+Y#5=(3mX}l^Ifl~l;v6?C8$cxhRf6OrR5yYiuT@yvhD2bVh`+`422I5XRE8C zF)h^5W%7T}_rPr)zFX?9>n`sSw_gaL3o|J1D*lO!-s1ESsEpQ=beIR}IyZgtvd&Fo zlP7~7U!GEKUJ^$a$i|5R|77-=J5C|(QVJb`L-5?7sLCL6+^<|I?Al5oui3%IN-=V> zFNqC!2g3;h>w#GA*amNCOXfSBV)1}>i)94`v^{Qi_+ydvg=NW%dB~{`|KEQT{vM8* zfA{QR{mt7){~I$c*Wr|nj?H&A~^o5P_@ zhUA(QkQF1gTfv7(7 z0eOFkZwS9mlgb9!Y(T2zY#3Kqpjk!yP_t83F-yPVkqvM#i$Q*7+>Cg?O<8juP_Llrls3+Wh9FL2J?2I}Mj?6RC= zCNa|cJ9f%h^gXn$skw?t{3uVQl2c9*T&ibo(IPm_XGrYGiJ)9}1v3X|mFD9yM`)7D zEiB)ok;BG%WrBqa^AV&xim5CeBR?a#8P$V(W@BT*tn1KU97poRB%o!GRJ-_iQ78RX zJ5v8oSN3=UMyh1s9$FPBt-LAI@4N(bZVhnSi(7G;LrDw7+(5r=%{TgZu{~dWRowgj!YB^3BdBk1C$2a2XWj533&pL}6@^FAx~pzCd)VWgjkw`* zWsmgQ*&)vjzDV*qYe%#@C|BG8cf`tX=xwDPF`nTaohG!a&ojgN+UOLyrqEfaZ4Ik# zODHa;B(njtvtS-P5N}^8!RJ(sRj79f+=u5x)t1nw?*mT*uBT|cwaByqKz5fMl~qGB zsx!Pc?|O1x5Smwf;1Z*NBPI`6#aLJOz`Nyb2zsvyRF#F)df~WB0k6z5r^J$}!iN%-!c|Ma8NL(Zk6YaS}HTc0F>vJuE5G*T<=5ZR0&&=iD z{T?VZ)8V3@i8b#C2!OW?1{88&@1TGK4P%h;3k>k`rYsRQIJg~%zb)J*51dO-7$dZ& zH94FQ;cB&#y0%9tEDNIqQ~CUI|7%n&x{;dO>SD*r7hW)qF>BITfy#V<&^pXOEWC&B z{^{+<^5dmFWkpJh!)VlmrqHwrw>mtj`%#^#R+iCP(#nghv@}BPAg4oqJ}IdJSf&CRU8^pZRR4t^&E00I zA8eY6IudMikl3dFbMkWPKslp5ZFTbqpjFu)I)co+AnS2qrRgt z@C6xRZ<`?b=S}!3%E6Y9EFa4}q|}P4y+LtxaeqzBhk*c~O0Kd7rfv6WlaU9=}b6_@C)Rhfmw(b~XOv%}>& zQ@eeprtM~rlWfyNib8HG6&T}dLbu;gL>07)G5>})xv{k_)_9lWk!KU$S;l?hWosox zsYF3Z+F=7J?dwupM=++GP-^NggJg>8z}@$^K6ReAeXZBl2w<@Vk1;t7I%Eq$SEsA-bpXY z?Z3-~099OM=uEn9?Jz1n(#9QfTPmo*llYqwW{r=a!U-L;UNFHti2t`NA(l_4T$<>+ z*=+X^2H7S}ho#Czp; zD9AUodN2dHi}d8&+V`AU8@xI_9g@;SMm(Q8mHy4pb~^Gd)1Hhwo#1a)S?ye>im} z#Di$Kv_`H`AYOhcSsNcGS(+Feqne25Q1RAsulgPe!6 z1pE93YogidfiW6 z%w4GfbDNCHbvsbnL^70ixsEs1dDpOI+rmQ(;xek{M2TRfOGOSi$b813eV3`rdu-Zx zL9U~)2_BF(Yk)$@)x;0liU>t$bVCW=LKINyb7nRbTWgkL4OTjPYgrMyQo4E6OP&4@}85KRjO0{7efKLSy0D^W=9lqW^1NB;^H>?SQ* zLh3G=E$~c;j@y$9N2Ln*kS#SbPo)r5s6DD=HlK#(7KgMWuy4$|K~aY|a^8tMa2z$b z-ghcOK=$M)E5}~tpkjp>X$x7YR70YmQ-8>=mz~K@4iX-Kh3*0|Qjj`q$OAxs0szMa z4d8CXPJN^XuFYj(dj{~qEiztFHRR1jOLdiH* z;KpAWh`0V^AR4y;j%uoIn;1jQTUDu!8(zwkRl<`TkUsJk-}@8v@8>Z53wDi7t4%|$ zgx`-|VXOG3|4N0oA4A#OY&brIeU$q;{fxFA4{)`$h@CSmYEd0~7@bB{^<>R|PhmGd z% zbw$LK@5TmlNE>QdMe;V3GC?tKu$LJuRBpYv(_*DbCQ9kFQi81tWvV>#hgOcEWS-Dl zJ0rVfKWL5$?^zpoEL*$ogKhM41)!NNMbIh%%mIb)0?qMpRW(_KXvwMqI<(trPC&9a z7E9I%n4{)2hZ#d-1$G&Fn>?#}kzF`IFk3H6g`3LF3MEVrdy~u<(*2AunjHvX#Bpbk zm-~Afsp-@M=%-~d{B=o1T2VCt4M>&Ly`Q8uGom%BYxSx2A$9kT2GdEBj}QrPk>xZO zr1Tqv77j4hC8Aa4JAf2ek9Aq%A*-zw1W@8ONG<7z+5m!K5n8TQxmwQbFTWp`%8k<~ z+}vq_xwQ_Vd3P$7lpEOZCkk;r&vVA4Q_--Vj=4J8{_@HNgEr`e3f-vVlQxXCyhoT9P5SL&6I4xQJ$-|3}hrG%uk7{U$UzDw5`VxE_PZgx@C~u_@WqCz=hE%W%#7f&Ig`XM!zxA1*)Bln zip}oER;mMd!PqKyMyX-7)GV`3hhjJ&rT9{WLZ#W0-m;K~+jZ41%=xM;2Rou~1te4# zPe{Wmz$|p1F?MElSu8^beFv;n_Sv95R(TPNXG)NKxB=rF8`R8XN&@0+ePM_S5^4n< z2FRUQ1Xf2*MBm5A$WcHEfId_buP67(xDo~~S4$HaK0gSPh1UTtXJG++c~(ip`50iJ zKaMV$(?sJLKJ}})UC`{}xk~-;dsXV&S8t#Gdq2bXA2|NiuVB;o^hy9R%x zb~TP#;lctOadyV&oNkyBYwZ**igZa70Rb@CwMpb7!!&hubNl7GTishz33vgsqRXYO zN~ClX)jCwYIM9?>-3+W=TWUG}56)5NTWFm%H7v6DkfN|uu5Ggb^;EKH0Ld{zI3Xr7 z(~cR2Ja*IEm>@p z6JW9-^X||d$coVQWP+8R+?mk1wXk{#x3dLNXf|2oZN?;b!$CIR0QP3sYN+8(einm(08=m;2^$EKD$Cwx$}2HX-fbrs~ivAC)d>J2AP9 z@x{O@l&yw^$3P-*d$aBkkVM;jYSFr!pKhV7Km=nP$>4X@P!t9CYV z3$2{3k5b$(k-US};r-2cSKSxhWeq$@CL~*KxzxyF0I4xAe&yJ&5Bn5gKeVO#VU2d1 zHLBwkwI@)SRmeEPMnEdm8sa25832YFddg|#{C9-g)~c9NBo3kQdj&I!p_Vk^8je_` zI$Y(bxdHq_5&a}5=Wow}eC4D9l&q&07vP6=h?J!iBWoq?F5HW~gbFgOt-Rr0j(U>3 zRjuRd=&CQ6m`@tBKnJnMnvE3C636urn}EIuTz^)v%#JwuGb+&>WrEUh&4oib;Nyn~2Nj~!Q!f)B~hz?qC zkP4!+E);gHqAxk>iA4j5c3MwS+oKs!^n?G6ZRPL6+gHclZ^GM`R2_T$?0wb!f99us z5?()qht?-%Kk_ta&X&th$-@6f-txW_JE_P@@gL78t!CFa5nYXQ!a7`Tv#`3b2!Qni z$D215s!d96UYgTQF0mZP<&Zp-#%#Ffu||W{@XPJ0ooR?Y+XJvPu$eT1d2K)0~ zk-Hf{XQ?!{+3RP-5j!o|Nej*Imc5h9hRWp_U#b_#3icx%urx|)hrr29tC#Qtck^CY z?tIP+N*{Lyy|;IV)<_clP>~7CBrtBl%(-l!O<}yE76XKg;IRi(2MzKdmUjaYw1y|? zZS@*DB*7$YY7b7GPK?KG;xM_AuK?o*>w(KHprs8IL|X--k$nUq%o5l3=0oBYXkPE7 zOOYq{qb^glF_ZK(J5$>+n!oveT)r&y#Z{V7}lu0}e-3@(tBmILmu& zgl)?ah}TSeR&Jd=EH9vP-d#8sCmlNk$#<`^+UCXf+V-%sJlqPP8d6fS{t8K1qX=L2 z`HHA40fM(UatGYigHDlCp~KIji*}!6@^IblW$Igshzt&KXtu&z>d?j;)Y0os1+p-8 zQp!v2;@3(&s&xW8>5>geM6+4*IaG$sN@ICQ)SWj>*JI(DFXwIp>Tqz9>scLdyu6p% z)Q!DvOjNbv&s$Y+Hh3!g@a(NsjSJ{s_U+_&$O6|t#cRi4YlhJ{H3-NR5CXyB$N>|2 zV8B!%8kGsWCh(o+WqOP3mU%4c03H^n&2$DCm%N{4t zlNh6sjXI#*-715;^~}+Au|3?9d4_IShZ1}Rh(C+8@+cPQ)!P$6-a5!i5Cn(ZvI~4K zx@s)khtOPAwWJZ_kBc(Gg$_OLp8l0`c#CdFsa_tVP-sxgnqwNDOV}2gy{LW>fzlWg z@^K*hBB4AQ+B<5K^y<82uPmi(XiHx_;l0QawiIG_u%q7n3HCFGZ3>3~NdM$WyKy#)@tqW2qEtO&rGz|cw8-Hx4uF(O5LY1s1SmE|Y$zoHvrt>HVPLQV)TVL*qa)3cOwKI?&E-0M zoUToD2`!-dKoo_Z*ue}7?gDEhu2J=;ZJGca!qVJXuA)#^?;hxu?(*hi>DJaWrd>AC zM;*aJzYKGpY0T*L6dq%-fnXIw3&OI^!#;OZ+NmL>EEW*06253jc%Un2;)XjeYj zP^z~H+ZoB_1G@PqDSZ-g)e1?~IoZ+As?e;6!wTAwJpf9Rx{H<8HYkD90`Q!X9CQKL zgLT2@;pcNoxzuJA?&|KU7jCSm#CErix}Vy!vV<;@2EABx#vMhE7SjwJiPXybgf2nx zn|if*+rl{J&bCy?dzkQ-CA#vP|DUvX>zU@d&cyEfS8T->iIExmK0yHgkMY>3!|tl; zI_RlTR~ zI_!0N4u$o*A=Cxr^b(32<1yTAWT}Ek3&ud~8zItCKeR&?ZE+f;tx@VYe{% z>Yaa}e5Kc~m9O+_Q*YR;+MV~5Jh$$E$_AyqvbOFn317O4_OS)hwZH6xf^ilL#85%( zR~2qZ9%6#W;(1CE!VNhL(AamObH3XjE*BqG*^|*Ibqw%cwDw z)`oR6=g~)uD&l$6+Y^nF6f1`E-V!1eHp$krWtgPK+({H+i_wU3^bkXyYc`h{ar;P0 zg}>K_RptAz6itP12c<1yF8;Z(Q>RYtXdu1-1jbMUhTI2$e& z_I0>q%k#U`8bsNyM^Ql6+(o6Ps0)Gyk`?BoEa~X6R)>Y6iipaye2VCMzoMSMha~S^ z7XDzS>>lkzuijB1Ihv5mHXIkLh@rB`&qqLEh3K{CIU0& zri|r~5Ot4>3F7aJ*G&ms)iX$Ig!cAH#RHHY$Q)Skoxy(N{V{5PFA4nYYR@fque=so z=OQ|R@t(fb>DiP0ox!WT#gLku2OSqzA}+?h^D}8Er#YbgDLd zL(**ALN*kROIDYCRCyW-rybBE2zG7r(HTf$wI@oS<^mKxM!W$52?8Y5pjMuavz8DJ z2^Qd+Z69-^&iPqa&8KeTn>eXae>k4q7*B)piRdS|ABcXOs@%*teg-bOK#t_WFdD8{*q0v@`BDe)g!f2tPcoX zP|(#S{6%SAFp92#>#HOzPsHlU^JPJ&lG|*2=jN{Dsa+FpKkY~G8KjJ;lnz93^duOU zyP8UT34UW=m0Mbn)^ODkB|6#}{nMpjfX#3-uR>k)ai z8`Fp*N)*$HH7<;M0P6;)Mkg*7mU9cIP*$wDgL$)|R>Sp0UM-S{A0#MLYF7XA% zmz(~8G@!~VNDz;qLM*7KLUu}yN7Ui%TUmnV+jU0+0*nf!f5pLcO7VQz@6=I?{FNH6 zCX%~%;l4;7G@MKnbBt$Pk44iUds6dwmSV_N>RQ&M-mr)ui2?jmkCP(c6Y{;zu9r+7 z#*$)YW!Ry$0+=Jx|4Fa0_Ybb{$3sS*4ZAFa0FW#kFcesDY;z1G?`5#4BYitRYrg$e z_#gF^JTU_APQ>hQ-adJkoA&X)kqv$JuYToINx6O=!%yD2a|C0`eY6qyILf z7yZgJP^aqw-37XesUVA?2sC9OC`$GMgB`T|#*>|~1TZN-?uuU21KWTf1#oCXnB%d6 zUtjmBh6}6PU$2h>7gVxI%TSd5Gj{gX5w$5$1riYQhyrSc{W$u>c+$t!xJkQL*)BCO!yXBD#) zsH6hl3r7AUb()qe)23EK;6lg$hrQQXIw;>Pb51k=&*DN8{uDTAZng>OfDiiBrZ-KTz4_`0(Qwx7uQ||3F$Hg zI--NAR!6p(VFb+-Wb!6C*wt5Ly(4ct^azG*vZO-6>IbfCkox>GEJdJCNpU>?LuaG8 zp=n!qA=+{yy8(lauCei##J|c;xU2(_V&t=`?x!bskjk9X0qw?n3m*uA7Qj~9yGa11 z*vm@Uz<$JdfrX($yR0jLylHb$8H{x(h=v*BEUywg02xHx&m2Q;DoF@RH$zcntKJLT zc&pXBl%Xt)aA_9|M|82F)4Htu$a4n6fq_lSMW+rAGd_jo9u*9e*Sb2(R3q8}1|hGY z`&&l`i&qfw(tlyvj*Q|Ou9C*WguRB&8k(l0)4iNZcEuT}p>|JlokKp5LIh6Iu5cRh zON7bC^Y~ITC$;t3o42y&(p@$Vh(~TF52ea$myu;&I*z5p0Jk9@U^5;uawkb!Uq+XL z0qiPmW<3kJ67jlb6AQQ*3lnON+BpKjQ6dE@P=XJHYk~hkcgHLlWu|Q) z|K-;@4G!-=Cp6$0GV7A5I)4MSpv0h!@!vlEzYE|0nC2y_fC~9=_dq{MqkjJSZTO3G zOgAk4XK!Cnr{xcnCI5~`-1wXdE}z=#N9i{KF!*6IQ29ktMAe8dFX`X(UGKkr{rYK0 zFAVwR_`vFKyIsdYmim`S!A|DT15SLK&0-$1c0Db$2Otehx->Mk#P6-2PS7#fI;fh> z36!;tpbA^{s*Q%pLpu7U7F-{hTW~HqRdeJ#)zfg(nsjp_nG^ z5yTnVq#-+0NC8?;+i)e71-@n^Z{|Zx6bn4TC3p&s(g)!`DBI3@0zld%@Sa*= zje@*JLj#bpTusSuns;r*AQEA35=rT1y@F!o!seJK!;9ZUf40!xW4F7(Q3k zduqJz)T_w5MdYWfi!~!Pl#PdV9tk>_T^prGDyRX2AsI6m^;&i`LMwsX(G6-DWDyzp z#5GX=rrQ>Eg&^PnIL?Lg5M9XY^x34azaqdw7Gku|+bX93fOrQ0wg9LT0}6tGeWz5+ zhop3?E>F678wcPB3_2i@xpL}!vq~Xq&NhwQcNSPT#66bmI(D`A3g{Co+F8wm0+0Y6 zr}&!qRmz9E z9c)Acy)9Jlbic&1M32IvsPd^8AEgkK$93yY4&hsW#Rm_EuA{0Lg~c*iMfp$OmQcoB z2?ZC4Ea?bV1rqB^7zueqg$9Sb8NQcdzp8Z1l`(do+%pskOD`67%N=8g+opz+_Vqps zcmlJ+QN~5*L?3y;Kn$Liw~5*wo58U ztT=p2rD(2?#z3_|2yI|rV9ZMO9#y{k7PaQz2rZPmWUAwd`L5CjP~)`QEeGOdRA027 zMxiUj0U^qjcCmmiKkyC3qytOSB|}U6u+x_0a0AvX9xhf&>ogrZ9jlRB(CvpTrrf>j zSiwDgc9gtQiIixr>Hz6jTOR^(H@QHv{LsS$Gc|q$o%5#69Efz=sze z^bWpL!rs44OYn}vw{M?P;Kn865{Vuf?BS0;#(?Bk;q9jv$z#3<|6afJj9GY*`n4w4 z0IlQqHc1yR(7fCX@$uM$H@_Yc2*Mo6bVmr?BN9f4y127z146-^QNk6vx?{6|PoNFh z6N!F%geoN{>n4RDHBzyLQonWFAzX0F21zXzG^gwf(7?2DhPs^1E~<)MH55;~*-N1I zrUZYvVmISXE&HsggYA@#-dCuEFHx!EU~Bp^qt(s!Ub67ih3t}Do?@I8** z$X0+vH7@z^bfu81xI^Ir#cBD~ltLe|FwB?v9;had*@Yq;m&rcR+)K29KTTvzmP$_jQoLFkGnOT!sxeYjKP>IHkX-0}JX$KSme9D0y zrE*g2EkOT3S^83^>Z=)(U_DBHO1Z5FAxNpYpM~T$ zXZxUabMV&38f}EMjiGbriG!-#C2U{=Ktkot=JFRRki*<+t&8Z;TG`s6>w74|WqN4_ z-z((_p{#0`A@)i2Z5C#VVyxMrZJQFK2?G9FoeJ9YLaM@iyxBG$u2crJK?OV|Fy)?S zfb0khqT|-Th2?Q+6gtGA5QZcV8ltEOeA@>a-`3b~(kGezOxWj(p(NHf0@KuKJZa=K5yj zK!PJM$zmed>%b4X7`i&FvxO)aLfk4)3v_}z@t3Pv34~c_zPJfzn^M4DHniI>{6x? z7fCY;43Ee$gkqnpRn3^p(MLtEY>WG<0?INg&~e-x!uSlN6`&bQH&wJ&>|5lQmWTCu z?X3!2P^~W|O-oiNt)~${Pxd}~h^VdZ))>gP8gN?8<6lwbdRif#pOO$+Pb9)IPP^TpbXe&@l6YKfZVt8kW*D9G{~7Z zrCZ*kBNf>A1U*k}%Sx3uM;mwDC*uF$E#;deBd+lIsn8r>7f1esx%_A@X^4qBo+TEd zl(~~UBn5C#-WF;rMG(kPSYGF`$K$$4wWDZ&l5>5ukti0eqf&_JkYhI{k0^&E05)TP zLu%wYN?sOcKKfV&+?fO|d?KQis<&U3pkp!;<2MW#(q>$AU@dk1?JS%FN7dGT;RGD+ z>@&(q#k{yiz$~=9C=oqQjHEDrh&KzJQtqK`XWyQ!u!vZj0x?xi?%q#$q&b+Ld|;)Q!+k=n*~S=q7Q`|xI|3yDivP8=`gj+&;hP3KN*w(&7nI4H|x#i z-o~LtG~|QUXHPX#Ho2P&meDSt-egKN;j^yV>5aa$l>Y|VP}bj;KLd?h*JGDooK?W< zh1sX}=t27gU#<~r^HBxm<&80J0`I!G)k_CZL)CO>r0*C*_TE@8>cf^Bv^WpS$m6qP z(;_t(Urs3NGAYn1srHGQZ&jDbf+<9ztT1eqoWgDRYHHl2_h*}FTgbMV8sA3kOqjY{ z(}!kW>?+>0rFH>wvA@*&1C8Y}yPzBou);V-`PN>lV~eY^(==Qo=3MQV!mpc>32F3I zL1!i>DjPa7F11mqC#eO-R3u_xKJfrxN1Qo8W|gD>6h#rja*I4l9l{>voh4}t9LJX% zA7prdvDZXUe}_=%{p~s~9EER_B&0eLuNw0b&s<+#etkH(~%B>W=^K>T>Tat z`8le#;(_spAuAbqD(=-azBBKma_!6nDOcb@?It8}t8w>EP`8Z)jX66DXE^r|8@NcS zVc#LO)HQG}2Eh_M8zqZjIa_y@)73C4poh(1knP{|ttr``BTLs8$}aytW-zFmWlh>aGQZ1grA+Jw+4;c7Y!9`^x z3g67}eGgaTzE54St$vln@oq6$sdK3AQvB^v)!dt<(8b%H5hN8%(V?X@v6(^_ZfH0w ze16_x4TKr@6;5B_#&8H;pUrEQ?Rp)%7Fu8BNhMXo9>?OLw{i?nDTTe*SD>L%$#wPc z=#CVNJV6kc<6hqbh96h;iSl->mi&>&~lJ+b44ML{Z3 zmnJoFQOk>D_03aR)x*x?__lsYyxnRXo*A{|K>H1F=z<#$DqIaq3J@ zW*lUf0a{0XGg@YevtLov9g_Jf3hig7)8c_4ZK$@RwXLJK(fh-oyHUXf5OuaLl>U~j z+yVJ*W9zk*GiM-m6*aMo0Q-cYX*XAAmHeq>Zg)c;+SKY4M(6YE8NTZhM2UE^O|vr3wosmg?j5%Xn!HXJ;TI+4F)N70E8(k>xCO6~h}q9x_1 z{J{*aKBclrT}lTbE)uI&DeRyE1FgqIdZ3)cRhFaV?rD|fO9U&Oku4L7z=zo@mtgUJptQWC z0rJrn^*CVI?*dCrAfzR+DpE&Gmi6Ldo@#&qMw0y|l^)_49PrCQUo)oJFguwhjpDBV zl&h33E7U;+=50q-wHw1b{nwInF#aeDm<+dvWKu6FQ?X-mZ3Y12lmmI?qoo2l5ROP& zvH@ojPQW*#lx4ZfE}!UqS>ZM+fBBc;FE2kLt?nPs@%xJ(7ix%ZRxBvfJZIlh3~lfhJTJ@9@yxt8}k zYz15W3$R?SXQRqf#92cnc#yQ9LsA>CepyD*bS*F*Y>C6QK-Hqx$X_a-<7t0V?sNH& zXPr*8ClCE}xzpC8x?b1IoO0sf0s#p48X|9vyq(n$=1T3lB_?xL+pP{Y4RB3?tV8Aj zwufw9BlI!t&mxv&gkn#R!5N1W5-;$Bn+n2oD!n8!dS~wl~ zPPEw$VEt;;K2lgG7V79}zeG;%h|sYgEp!B8Ijf`-z_V>m1N>{}-zE#0{c?KkbYGxL~(Ptga=G29jfv|-C;D_2Rw!Dq}klQ;D0y%pJK(9cux?ZW5w8H{_mNwMIrc*o+n1vg~lti6R6 z&sBZV^jp#L-G~W0sqI zNahs}{I$YChz(}pz5F^i4;^U1*<7iJ+?3fn?lJXNqK~!Y*$z=Y2%(YwvG0(5X)D zZ3}KDO-9t43T)6EwvW+L-dhQ zQH0XWz*Q)pqyaRw4_EN4Q&RSzTav=raI`pS1m*}eGCE7`u! zcFMo^_=OyY1HF=*#4o7}$Bsc4U(}`0l7$xPwN=)_{fxkCU{7WnUe-utce)Bv1<_8@Db3wK2;Ge?}4;kg#ZC0hWHsB$R}*pe@iv zfbFcpgVIt9!7M}B&8jl8YsvjSD7#v}YpFLF^a65A3ZSeIut5ckUp4w| zgtH|6uWiffpz`NotjTSLbvW=R{Vb7)48o?=DQV4i`jVA33}X@|!roKh6cW^uUDB3# z$Ha|k%9XxUY9fxN<5p0qcEgmWwI%|F*oeF%GY3pkiOZkSKsHG|NhCr`mvFhL1Pl&n zyEX1Bo&-UdIh4TdE8RdoxJo%0gnxj*{z8QxoHJ{BNI_Ae%fWlz;tgIS}Rr%F}`nv=*mE)j5a!3hBc$F6vwk~^+`W)&=Y-AjbS z5|y5|ceN>H92*C`oj07e6zFa$e zfT6rV>!3`cE64o4%XCBnWeY+Lk(+r1CT9-K61fXt_EuZ}uaG1|+k(aTBJU8=MzZ)2 zt%%kNc!!V#KU>F1E<2W%g%^zuQRJz}A+maM@4$GYi6CP%1Lp&n{4cA^5cU(qKqw6b z{7u&L8?juCBQ(TXrU^6 zi;w6k>$0Cc;zG#_Ng2I>%kJ)ivvx?;oj#e(s?sRqJRh9MHr~PJAB-2<2d>m*#n;&EAaJaY{r-a z{QGaer$_8pZ+`5%*WX*W<$p+r{_geb@Bc2m{*&%KgCRK7b!klcgA1!dq$W|xY+{6^yfI**hzH}R!LFre9YDj8d+&` zZgOaur5)VlRC8a)GoEyXzP)?0H;eo~wGcS@U7_FM$TqZ6C^ETh>htDGen2Q>1owgJ zD6wX;I~L??WJB8pK_R#VCAv-~TLgFO>7`3n3C6=wA032mA$U4P9BX#U*emvuH0K{jsg4W<@h9WFP3PMW4tMQpe z^OMxr)~K4KifL`8Kf=R~Lr|~^a%JsNrd#-I-K#=FT65V+UNl8u0(C0l)&WU z*pd3zR+8&RUk;#z&;(~M2-v$)79$Dw!$C(+-aKLNjC-KoD%qaA?3T=Z@nYX+=#s43 zWk6rhgY@|D(FG#prM!llh3lC;c|`6_@pc}op`h1W2%HsN>fx3-d{Xx(l7BAXcPY`R zIz?CHOJHb_q=o|rJHMz8Q85k9(6`#72IUB;y=}T+xg4xq%2Ce~JDj6`Wdwe83X$h6 z!^{C`A+Zk_dPj5;kRDQYf4~JvoBaeFmF1&YW!N2%Jhr)HHT7~*W&ip!Ej@*xk3;7pQ#+a#K7;*b0v}^?Klu_Z;ehzA&1t7&WHzAZ5N8w`Nco zlgJiEyJ_xF4N9>MUAh!;z}f{Z4dOQ7;xr(`$79|Ij5(|7Myj8yMllPDb?kTeQt%t< zh*DT?dhf3*#K7?h*n@1t__(YJl1xXTy0fx!pH+m}{y-*0rYS(c64{h{wj35_)XG)+ zL-cG}*vnmA6o@@i9m@Fvm_V7H7E)AkPcjFIQWJV<2llRKmm9;1kS}BQL>N>BFgp$h z5-yi&^J!rUZt<=SKkL(}FS!H?TdAZZsC^!=u)YKmob|RU^3JP2SsEyA*c3_WDV&ywf!i88U*Y6k=iy@!qd3}{106)rPSX|HFzAI z)&v6$h*qt{5UOdRPK?rlQ@USgm2$Ku8uHYD2UlZ`D%cj%YQP-j9%Vyc!-I`5xcN!M zvTwt9QT{Ch=I-o9R4BQAGhePdG7(n(a|0Pdfb?_P1YoU|ux41|{cWSvbb+a^T2(_y7-%JDsLF@LeCAGNec%=x zbCYBejx1RAA?c72o(M`D=tDX`v|~yc@~jhez&$$E9w7L!)1urp2-M06Eiu;Np`~{Z z2B&FDrFl9x>&qpVdl18wNA#_z+M{J|+J>NS(D5Jl194W8Wner>{DlG)Px#^-Ub9fD zN9K?xtwf0l*@nh8#0W8xK*W}J z<(B6`f9(%`Yqk0GEGwP&rSlBJnxJjnQ;|jW=A>kOw{|~XIo2>?LKP5z?az;2Pi8@% z{qb+ZY1HxA>vw!*9dF{ppGs~2Rl-C<7wAK3CjRv8{nvj!Cq~3nJQZJ-He;tkeb&Kz zW^Ksa_~|L(a4QkXi*yUfBVMcB?!s&R7~zdVx1Wau64NQhig$R?@>q7|?4ky8FK(*jTGC_);s_@Y`68FeZdDEQ35iF@26SpepSwX(;Wocfv0AdE z&KsXrNN*rYN$zyWGiFx>E?BL&XlsNiR&XVQSOP46SdOcIggi1QD+9tZv>ir?MCj7+ z@fRG*0T+<={fNUieYq&N1ol-B}nYhfdf#sl$g{O#LfE-^1#7r#p6n^E~aXCzmLg#2);vd(k1QabLw=$ zwkbGrR6-G8%$HXCK5K33cMDNP>M*yU6!~VP(*W+@v zlX%I2m2A^1V3A%vNasr0RVsFp(#kcAmL**=)}1^f7SKlpz8x$($*rYuCBpy%%++oJ z2nA~ewwz`ma!`GCEbX|e3>^~y_dB>MCqg1gT!Jm?QV^vQJ|g{Q9F=FNqv&pVSENwo zJ%B-F8UbuXqS*!r@2R;3>G8--?fWCvI|30-Y=p$%WSr$`gBK z{~mX#9j7#TU`}xI%Feo8mPAQAzf^>TNwDL@^%JxLPpCjB2Qp(QSdwIlzx?mRpB~}< z^&^!u{^i?m-ah;OlOPfPJ9Ik!IlTVr^8NoN8AN(pkY_{9WE~9k7cValTy&T2LBi#2 z+(#@UruBPTq8?ezYzXq}=vhcYj6w3{9#v4I%q&}z{U8?wPWJHB>=-Dejtz%nuJVjq z?u;;$gl3+_%z4g^RT)?z^TSR?xKIot6%lvh#3y!*S+Pw=%>zL{1N5#%$nAH?W$UyN zYOKP&9HN{N2_M;khg0`*)Fb)Mz?)hw7FPjND;u}~Ia+I(q`ugDsm7Ijh?Joqh z=!C>PsTv0mhpgc-hFb<&#v*Bb%g7Bs0usGuE&4CB>6z2BS4FR(`2z?6=432BL74z? z+AawMJWFnos4#3ZdZ6`kx5$cT;-kw@%bRSfhElmbM(zV%Sk>>yY%cvH*ct2LqT@(C zkUu50**qA-DZLKzk%Bc%Rh;~>e%4?!9~`GjlUj!er`&n%q5mv z@vcG_WDZIxZ?nFn=I>4Y=%Z6O(P06Vo442yksB%g_vU^GbhmOJHw?S5N>AIx94a#w zzm4s|!i4#stb4bJQ3^qc7%(li*-i3CB!b$w-<^z_C2S>3{*t(zRQ8JXN?$92YW;_U zD33hwu$_U>5BV-9Wq(XQjS}t zikuwSE4&K^FP=hUg>90 z9Cle~w~7`6lHF>Me8+hHL~BD*XwcoXe+6=;My>^%hp{nGoFHN=@8l9%7~OHMmnG8C5OZGDz0{wnBW?1~%lf7_9?}UUJKleE~=s*nMeFy9#x4_54N>TuY1q z*rjxl%1brwc^DdN<1U*lg8BG>zP3gAH9~VIZ3F1H7+DLL$gy>@MhCbvbr>NZQ~@dg zPlR+N^2U@@Z3B1RJmAx@Y*+Icus7#Eb2AFHyCoJu(tD6PA1)WC`94ts=)*%h(ajAb zFt9FMT*j#zcM2TLKnPAhAb3g`zZ0w29ZCt3^}sA&@;ID=&1vWfphn7HrTEVr zF69md5Vo|2mLFTuDw!;KDGr+3FapsJJe;eDspY_B$S8W*&#QdHegYzlSd?2UMb(Gk zqUMNHMCq}lEfnuvl}2|EBNW0{)D!@cPe!ukmV70Lr^^Md)p$n(z!H>BCoeXENHvse z5=5-gZk-?@R?bs(vaOOMfbB<3PuRokvIlf%4xb?@11OCUo`yo0UE#ik@cUy!|7UO& z!e(weSa}!lC7OfF7t9#?ZY8CJV6tsAiw({YHlP40eWRw;Edzn2W0`n^y%c1HAvjJ{ zgEH>!u0DgQZ0b1z>iz=bVOWl9uA)p&SxS6Y4*AZqu#GsqqKfDQzJ+MpzQ51&JLau8vXweqgc)jQUi5m;F|r3f!a zEGN=XoedQeted(OmMa;NV1`}^NtXHtZQ|hEc#HgQ6`4avr&CcvOs*X!n#(Q35mTOr zLOTue<;@nuY&$H6RbFkhGDz<6>d;5P9Zr5E7{=OG7$7QK$42Evnb&K_01bn1r)pq# zSTkhF{l2;64R!e|Y;cynO}W#Xr6MKDiJ4?(+T5U;i<@C1Bw9AH06{`dRq?=Widr{o-AU z=C{;CHem*6HhgNi0h)iSFRVMPYH_QW%!CvkZHxty30gD-se5f1dB7#Ftea$h+_;hO zL!S0`(gsx+d3r{0H-NvyeR*i^`t;ci4?<*-Rg&6!g}LPtEpX#l_?89QwqSVg&Zihl zMHRG)62Be^yC5CUjTv&efZVee~MiMW+$~?LfXKx zeOBa6K2K>=egiQh75V2OGJ4B9Z~UCggxHQrppccbW20E z^ChaOwl!h^s6BwhvWpQD;Tnfsk^-k+JjMCww_H z&vJ4kVyQ+%Q%5%gt+j?`^+pKbF&H^Sh%6pz#z0;Aw(}%{+zF}>F~P+F&^aRKI&f~5 zU;`w8}7@IdhODGa0CaN6#Z+9Gwm%nM0%E z9zN1QeKRSHVy(_bNj3Ph6qc*kkOwL97xiZxJtk!1h{MsGyD#>YV^0SX_~ zE_Ey+$w0o@`YuVne72Tn4D5Q8E7dT{dFzjWSb~0FDZqGWmX-+tR2};zA;yqde!HgO0 z7*ez$5VaEFFMo6&e0z|@)jR>9IB1jrdB=F?hn9B^Y>b7^rBIECcIpS9$N?*Yw8_Y_Db z!mxrAlT4k@DaftCkLb1Z!OA$cyL zbu9OAYq=X%Q!wv~t7==Tw$dVTa%1o;Y9P-cJ`jB7UW%5@l1@0D>v`XAXI%y?(=F>FUyk6}QZy%sy8$X*-bb}TFVLL6ZzoiJR z4g|?J1Rj{sM2RPuKD4SxH9N9iDtlmAT7PzKvE4(bbrmcI41|dDLquXTBaR(1eGQCg zB0C}~{gk!}PEWx{$R3K>8~d8vta@~f!MafeWa=Tr?MV0~``u>fGokKJ!_RO{>-D>~ z#M7MRwF47-8rD?vo@3(NW+!E)!7)Y{aL6sFyfecbj_RQ_9NhzjDcZKmDiG`daVA&I zyxv3My%&7Yo()dzDYWu^&{sHG*_d5eVjsa3aLHgDW;@ghVg%hKX%mh2W~OanhwdU| z04|`D7*#=WCmoCr3b!Sk;eUn^gk%hGrFHF6Ss)I%CrB+|MfawHEDk-~28lB&%WI-p z!LcA6NeO7FC`5B5N)3qR07Jlv;2tm90Xb6HGWl=lcCm*Ny$A&Fxf=<=K$zn}c$sLm{rDf2| zyDgaPT;700M%U)d^uQagi%=@xPt2WcgIfjb94J&2&Qns zx$IqUmJCu3oWtRR&>b4!6CFzrA+^2sEn*A-Yr2-rtewaHaQ$``>{ly zsLd>){HyRk>FePNothC}du!qU>udP``wM=cGxd&e#Fykue#wc(r>|dx*Uv8BeEPlM@M5x7L%i0vYKc zWQDTx3b`uo;(os@kQYvA%eE|ygw{Kl@a5LdZkjqTM$w)#i_g``eNttG23y=1+X;H2r}frm*Q1-BGjEZUa#21+JGwDB#MCen?k6c6Qwp-d3ba`ak0_VdWD(?TwD=eB#Y$FY z*v?|~g$|Z0Nv`o^OVugguZT%JvbT^YpSsd%1lSZ!d6czn-Dpj2X$xl0NRF+~)v6Wn z?gEq$a?d(YX`l{3re&AyEGTYU4hvL!YqwDX7zC)9%dE~BX)9|+~(Shsbl2{n&Kx!tzjqinm3X?c&m^G;TS}ysOJ3Ga3-%D(+!Hfp{BmpYjRqk{1zXls4JHIAbeVoff!Y zd2nt(5gw8sg_E)eP#F)&g&??>kGrdqNVY9$%?kDEY1!dHm1^qrMLa)KQ@Cn|4rcJN zZ8uK_usL!1Y6AU^g#&a+YvMnlJ3AAugn1XTS4)Mq!}b)TvOdS%mv(1lP+I6qa!>fi z4027NXi+v{MwC>uZdsV(jd`0|SxR!XL1{G|>XUO0_tGE;gI>~P1C`LSge=KcA$_MUO)6X&DQ{q{uJJipS=C-?Hk)mI{`bA&%m+rXE1;I z1hkV+^f&E0EqeAW)cfV-B+K%scUe?t7DNw}Mp$wQtSI+P-GIq=ott<_0D~7;SsH(r zdUyuJj^K+%6~=I5xxywyTSu3kl@eo;n_#jFv31IkyKK2szb4)QiQPCDLmsV6wLl~Z zZ=J(riBcUYHKp6vS>kxD$W(w0=3O^2!lg?^>?bJ@6psom?m;(oD^wkuzAFSe$IEhe z*agVFDnQ)Kr_sl{m5GPc%r4UjvCHUC4#;%T0eV>u!pPKcRMS*Twkgck;HmN~D<+az z{-R(O2ZBhQzF(|;6_Q?U`jWawJiKirk4kV84%EU)YlaObbl`0h17Z?uC2>T5iPN01 z4})QPXLY&Qmy1^Z#ECTF$haU4K1a-LGQ4a9ysaLO%n;y`clBGR)^`ph$-|L_Wvlh)vW3E6{GEZTFC#1rQpMF>F-+t}-Z?25Ns_mG##~YF23zfZeEA11ddX8=N9hW6aqH4v z(ISNck)$K*n(pNIq8KH4~5=xr`MfZ@8MW?x-8zP{= zkl|if?~+OT+_?mnLH?2@A~1}y*$%$d8c%?Vf`{V0`x4N2&ri@wF*GRdKP)E_)zbmF zaw(7_{qq!LRlbqRY?GaWtX%$x%3Z6*HWurRK+sOde4HGiaopMOMN&5Nq!o!7U`-T> zj~UjDU!8|RCz>%lp<4HyL@NwZ@XkR2J<7on`Sb%5NK~Ej8Dyd4Y-qs)K959mU|7@+ zqhmiZmoLllL9g;jnWpAw=l41@vZUZawOv#wv;OPBTgqEXVk{Exrj^+RJnq$F95XVc z5NVA;cc#)A=Bp5eK&oC3TVZ^(>;3$8<&#(hvT$)2RnB3LC4 z?L`GN!;gNXM_gg^+qVz(fbD!1k~q|H>|c;q4_nHVeSBnSkk2mPzyHS{zrFwe{@?I# zeIfn+pZxL120MHn$=JRffMQt1rxW-RR?=GL9*Lz~M!lSS%m@|jIv4qP$O$PZ%(*BY zot*;+b=DcvuzM!y4LCY0)r2-6wk2(|1s*h3hTpBF-O5%sIwI`BRMhsJ1Bju^nP8NV6w7HAD{LWas^LgGbuYD{0Z&j`^V|Gh|(*WfzF zY8h!^1$tL#K=itFD%pazMaz~Wxl#aEK&ZdbfOKG0T`xBoSv44PC#YGKjf@ve@fSm3 zFOzzFq;0GqlA@sIcZ6#L%0$|%(GG&Y1=E|(@!E?9Wr-^dcOePj`r@HWx5JE9i!3fs z|JF9XvQ}|EBny#s1#w3TxFz`;>yH3?%V9TIIg)9rhvhEstCe4XZSW`*M?z@~_5T}I zG<;4Ho3hkBpUE}5r(5JWY(e6{1v2Y4G6ST~IzWX=Y0)i_kv8FnVQ^OH`<+(be!NN9f2Sf=fgZrjoI>iPtFmuV_h2;3<`004 z+`a*Pb%AMNBw416r;HQ~nUlP&%4MJ^x+oEX;8#_ruml49ALy=FLn!S)*j?wlMz@l^m_d0Z@+u1{yr6QWTsZht$_cfov4%NLQ)Wkn3Lj0=B8ES>Xd50eu|tG+&CGO<`Xu=aVu9z z8!T^_8!jv;k->(!M8MCjx3ST5SIMabH7`L&B{Q|T15k%&Qn__>t)8%Uq48rG1RlOP zyZ2hs(3YDu*;F?Q0$me~Qt||-|8AI)Fd^vpU~4miZW>8m*S3RuZfJMLl7J1D#I_}6 z!_1BP+RFLbio^8^s(GFF0OS(L8#H=T4xQNXk{=8$tiU}zvYNS6bfp_tH_79M?$>#P z0my=mACJU9k=|ShHu1W3m%KPv4S<|569)@*$n5FJ?t#E{tiMM~{KyK1Dys!qBEMVCWN?Uq6{=A{7}l|lU*H{+k#C(jJWXT#`)}WxF~^VI72ZlykvG%H{tcxg zQh(tC0yovi^6=w${aN^T-1+A1w`VlzuQ@pR2Y7${!`qiBas!cJeS}-69iXD7Q8|O| z12zf}CbEDAV;nh=M5UkvaKDHLhU6}^F6+X0O&Fg?W3`DvO?C;q*}D)*S%}soOmE{7 zSalTimX=k0FNjsZqvC2@;R;UB3^MbN#{fLK|b55Gm5DA0tu=15YB@fPz;B}7C*9DtJEeM zF=(D;aSyE*8*(T=E0~1q8M< zh}Es}auS=_?uXgkxS<^|8X~S)KZmkM-AE@Yrk=`5&*&2(jb3-NB4R;lB-e!tH zrTRl&m)77{NTj^q10CmjyVQ&7{aE0WTmXfaTptS^XyqQGi0cc|h_ns)$VYY_Qh+YA z?pmg2*G(m%u19~VhTE5cF90=h9Y(N4(Wd1L=NCm>qT@4#(3bM zuS8Co4QCvqVm2?umWB=~(n@&`)eI(Ds#LiLHw3`GBQ-F-?=sUa;heWY6F%jV8fpWFzmIR8D6T8J+b2 z1B-&yV{keBA){P8Dx;7`M!u@wI%oJB8>i2W6S z78;*__?0i-KC#!I+VB8g7OFGx_Uph6|Nhqx-hN@P2__*MX+!|HsuDn6zW^-ar*9wN z=K+WKQg=Q>)FKU*0kkPjVYVO4;h&wY{ zE>?wRe}krsgzadZ0Xzp9{kub-34k6_5LRV`=!$4D8<%Y3cmSeT6q}atG@{r8B%&O9 z5<}R!AGX<*B(_k`zM1FDmy^T<<`40*VALg7TVbT5F0e3oVK-@MUCz0lVInC%#ARZs zAvR&}thsg;}S zJB@zdYE*~CCbipPYEALX;L1_cJl@iHdvF&Eaw*0MV-x%$wmYY82cAlM_O{ASD&j!3 zCpJp8VovBx%O9j#=;@}4Ndq_B*wQ=w&Dt}aXzv$o{#sE2izFR`*lvRs$xgDSgcD#> z3)_(ewd7_M@o&7#3$Sw5NP&Rd_D*`Kx|M)sZt})CQ-Hn`eYIz6zV|byMg6z}hkhkN zw;xlV!gdBs=qWS}^N+Mur25|xuF#a~i_;yXCV*(4IGv-04(5a7h79oI>~>L~0whF; z?fRuIySs7{bWOhhUyUEGU3^;m!kM}u&AZLe7B)m_(4MxzCnS}SWt7MYIotH~tbtc2 zjq?^)K%CdjP9xx>Bd`{^WM%_l6u5TYd}@Y-ZF0r5CfTc`9Fso+Cgi#r9vlfzgFwJB zlBR09Rc~__@{QNq?V;3kqM}9q)GDqyfqFfPKKT{J$9?wIVBkTmrPY~e@OX! zk+;@duWSLl1WHyYE}+{LO@UE8-|RK(3iS}Sf+D_5V4Fw~F7Ssf<#@S{&{`uTAfn+Q z58TioFOiik_AX6Z?wzuum=GYRms3a;8CY|Awj`esc@92v<2eTA1S4--JY@DJ3OSW` zg;lO$2iyP(`Wb-*;1u)`_71f{GxHgmYtT#f_^K{M+I=LPa)i*SMgL09Ag}_fc9POT zG`q85oQclK^N>@|dP8xIVdG1c0Fm3MJ2c1yo6$ahgM~eP&ylo50GqKuK`zU z*p*?aCT!a}<^&nb!5bK1`N+NiU<7K%cu8(I;5#uBkZ{6kWdUAdf@Tz}eAsR?sMk#f z_n~*&0CgPtEr#0Q09Sb@T}$TM5_{Z=g@nl5k^!{lfy-0`n0s&nt1cys_wz8o->D&0 z1h<`sl9XBS$%?ZbCg!t7k(}G38C^nVhc;L6o`tLfFb{ylF-ZX0+iINmHU*-k?ND%} z53M5~@Q<_nU2nR>`PTi3CgnO%QX6Jha#4J@Lj~ZVMpcj)ZrXpq1OxL%{eo0dm?ac7 zRI6gQUB+Bn==L{;31liW)5Of+21KO&5fzby*;TYoIdTH)R6s$@94SChE-yWr&om_i zioq>BIUo-b&WrpT5%@d$4>P-F$>P4`%NJ6$Tmju0{)~YY769x)rHx? znsUPqmkydMsD=C-gnHVai*$nx(hI z;Ovr^2a<|+(GzY1Y-;=W(iA71D0%)unxPaB4z>W#+pb~gY21|H`jIEU=5MiJ@J;$_XV@sF05Z6`xCN z+}{wsO7Sd>2@tB=eUFe!f#;Xa7cgZbO|?PUwz!@_nI@W3W=UnKV8F9ZMGP(r2jIe? zTvCiGwd+!s08b7y`6>VjP)Q-kT*Vh%-HuitlB#%7J>HZ39P*LPo*xtpz+GN#4~*3l z$pp^df!GFE%3V8#2Xp)GwasDuUWnXdJf_?qpE?}weX0ULW^0nUmH;sUjj$-wcyLjX z3^-f5Ok@jisP>GZWmY72GAtoHrU2B7a!9@N1)gk2vA@==Iw`C#?@8^LqU$x)@)-{SIA>uV25Uki}1t zFc~uQKgq}VyVu`C{0a(KU&w#IlmEW5?PNQ;k6Sy8k2F2FN62F7ag)5NRX-c44H~=i zhG83<<*JS?Ilgz#w>DJp5MOwUt~+##sJp%$gbFZb^#-f#w=^%XmN? zCyY@we{2(AZ`Za~XOfg(Y*oGow@ZC=E1L^`JDYKrEi&F_I8HAq}EzRo6?n#hQE>m3u4!SXLQ(`#Jyw z3P7Gpt34$1oQ%znBs8K60#(j2(JNI|5R_u#%Es3Hl*XG%_c4nd0cKj^!lk-+!voBX zLD{#NtoG4*_`?+~o=nXMJWm3%zcT%xA2A*rDuqzanZ zIvFSkws+H7h-2kRgaTt$=f)OjU#stCCm9iH1C_tl7@*G4o}-jwG4@cgvgZd3zj2mw zl3+lbt3zB>Ex0;&?=x6-lP4MRc+mo4#jWS%v!iDBkam!K= zfI9vW^whi4f~)ZW(m~?N8|`w5Lf0|Zi0+?RDfHJibDK0>w{UA*67)J_Mse6|VIjc; z)(%{QD@j}<5DuK3tgFh)T0BoYxIoHjyvGhJ7}TI_GNztfCGLF7NH)^&n_;j?le$z| zrV+I`A;Ts@ab0pKiRw0`izP-<+7YuFRPw*6;pVNWQMQk9@-a^x@2#2FiFtCgyrDmOe#fq@b)V6G5H*D5IFM&A#I z5aesFs|AFR+=S>2K_TcMYeGjPGAEJChq}Ukot!)Cwcx>lkOoRDVk%>JW?dpvEDC05 z3iL^&49V_3s>~j6sj|Q>fAp>zpjk=9eHr#pU~7y!}C~yDb8LjClMRzy`YY3ATQAdF~xm@At)sXgdtC zW+-R`vrhrf!9LKVM(<7vDHpPs4lAYy27hcsjMnXdn&h_B?I9>>(04fNK^w7SuR^Yt z1VS4JRD|FM~KPf*$Lb?W7r_cKE`CKa>iBz$jt0P zjA|WkV**eW*Sp1NP)`gwH(p&`Vvv+TIf9*!DZm+1WoT8b7!LPwDeAO!N332Z=UNdG z+=qy@taJbrNG{>hlIRGhZ4CG@KD5UI`D=nuaICFbv~bh9sDda0ZOXzmL6zGy=}^qE zXx%P5&cKy~6OvX@u<(WfJWm!1$oEE`iAh&oIgp)8kUdlU}$e7EJt;s1-eIQNYhXo3Le?^Jy7vjK?rlz3W?&f7IC(4pb~L!gHX2etHU_`{;HQ4V~T%L+krk8yJDRt;JW z#`_MU%9WU_Ia`ABD~kIt6wAjvN6&U7;y(buZ9j|RT5ewnb81mF5?PRhhhPki@q^s( zm>)RSLN%i@)Yypk?EXKMp`=|*SAQDHLr@`^A6>G(R7g)c*yr}JLVN-3m_r&1g^X(; z%WZornLAShx;cq{})8;e~#Gs zPLzH9JpAy$h4I%^9ieydJN@@_9rM5aR(C#1Ur68c{@eSnABI0quYaz`p3h(~2q|{u z3@j4On7-`2q(*kG9{b?z^Hc>(X7x~VH2o^%MD&Qihc70q;a&Xsj$Q*d0j`}ccLl;o zvX?wv?vi2~BoKN4DG&lffJkFIa_Jdwp!4dp1QmUC-9|W_OEJA>1LCCwJ*XL{uQhVf zXRokTqp=4O^upc!)~ofCS&%6iTp3$0ZmjkLYnIOD>9HlGV&I4EY`n z$OdhQX|Pa|A|T&>O_kdO1yQ-=k$?tJwB|-~6kxQVu1C{O+~Vl-TUvsI1ZsKaiAf$P z+%{6dzsV}dbr)I*$MYd{9rZfCSW2jN5>G~o9g8`xvrlK=xp zjm1Yg-8Lxi?TWS44gu;cr?{vkm1u#SK2`Lcv+#zYefj|p6?Sb}hN){09bP2~A)|TiH)s_v_)IkTdVmWyrVKzQZG`grm#dKr5 z%ZdP)oy60OC3iL-LSq$BFOo}09;-S~>NOYnRcKI_@|;4%LESBKdnbu3P?Ba1TMG2r zt(_nNQ%et6QO->dpBBRr4vl^wbg}M>x8ko?rUZ8^K`dyO@dTVwL4$RCvD+DQr|Gb! zl)H5z48zeAm>@W9$J zAaUehZ7kS$e_Ir*aEieB=F^cpffqb$P)PHlJw?kHmNm)hcmlEVVaa9Tkbi1dNqS1U zTV*q4p|=jAtMvs+hryY{8T9Se!%-sZY0;ZV)oHU)1#54rNiLAfV@>Sezx5jaYu|uC z{cSLW|K9`+*YDfTf8((lJo*)ta%au8m(W{o`qP(OCt{FmUJWX$z`zZ! z^MlYgvppbnMb@BAAb}4r`M4rbLmz<82TFe?$|@?*6m1fBlAnl?UgS`PzWY|Bl%;Ai z5uev#R*`!PAXMFWK}L?{IvtejIb`j!GjOY4P9wJ|ksEa*x72We-s)=^)EKs;*)Yn-1>MbtFuq17(J*yn zFrUujcey3qPPOW=QuQtxeGTg!!*;Zjr3~ngu|HBCEZ}~#;{a|Wp2q|glC$rVq zgY^T3QWY@Qi)wOOy%S+dmr0&2&k{*Jc&w-lfXdV6<;|i%J7WITB!`v}$$<^JAtwg` zsZHeFxd{Zl`(SjK8K16{0A$ELR818(Av>nv|E|%RmNz>j^Z{LNdYV5>lFQH{q6jvu zgYMCuNTQ7C-2`_XqVzzubqe0ROF|~8k2y?Zs4bUHLb&9&9Ywry=#PNx#l05ZmT~nj z19h^HCD2rdqf0K8%!CFw;0?RH0D>Ek5R``dO19Hd*;tC0t%shRTBYtgS^Tng?s#da z3vkuuI+*$0Avu&y@U>QM-ZyXWpC;1({=f3S@WTU~dH;?Syx;5Z`To@J?Os|Z45*rTXj5&2Y62e(DhSy2?t8dF=oZ!v)?h)|kCLqck&J9# z@+{S~UVPkP=9#9Ae;j`7ek z!KjDSd~_zg89pn)$*BWoo(98)@iIeuLSnL-D5h`Gfw5J-8&+N@@KHliV*h@m##3iQ z8>vz>k+Yx-jAIuG3Y4$B7_{%+?0}$~h~tQ1Bh#|cwNnP@@*ER)E%iw(``N&Me$#of9fF=P-v+0n;K`Cjy zo@&$)^5x84RFI2^wBUmM%?So0h{eWR*SJs*x zA8!|Cl`F^N3cDb>1G4Po}TT;hGk zG}n9e8)T2IA*$$DBw$e?I;`A-;}epd(*mW#CG%1rx`)gX(i|IrRClTdzT5`c5Z=*T zk7|gj%Vs0=(C+3lYO+J&G&q4QjZ+Vo{y+fBsu`*pJpyfLrDM=HaM7WbGD8vycW!rB z`Yl~3C9blrhj}a~R`6ci28^)|QM96i$gAZoNvjZ2h2c;$$CH7#?QH}pj3NR`p*G`8 zAO^#*z)*@@@f=EX;QXXX!MR%?nJT#i_-lN>c|vg$s0WynBn%j6QeX%v?orGnNkVcB zARwfUgt9&1e7o>~n$Dz6C?2M?0MhZNF8mS8P#sw@D=C*O(l6#((fN2c zSwLQ?-~9lZl}s^NQPke(irHv7Pm(vhTV8KPzH+HtA7CveLp*R16g-EXqPB~5z4lk6^T zfQ^7HZ#saIzNZytY2_Z3h_BXKy5W<*rBF(d+|0VETRDAn-8}B{jAPN4^XQU*u%fPXxZ|yZW2)tnc$qY@0PYb9ixuBW z9Xu}00G+1A3-_i)OI_-(JNu=BRBp4ymL3MRoUm_Cxhk!a0XkR$q~`iUFAzfK)ay0+ zF$mo>SDrno@~vHqjy!<^4MNSOFH|LXS9~4ndsS3ErujEcg}^g8D1x|=w}pKGXjIwX zmIufHcX-M`0k=*(MqTM%=pyXq!!jV?knMnxesgKxgCh^8*D65>a{xZp;1ODqpk8D0 zB-;3(l{eY}S6A5yy-Dg*nk3+mqHob04Mp?<%J|-fCX=NZat~8@K;GTFzt~Q;p8CK^ z0Nj~+=vPz@29O`KRpYd1R7#h;9&q|64X-mOWp0c+5FW6Bqb|Hh`m3D|_8V-_6|RR} z?X}xLps-W|;48^JuF%EO$=W-4#!TMg749i1xZeX44#?+OY7itXgw^0LoZMJGL*HLY zz>p;omI4=bMzKkFe`#>bcu65iK!FUsVRwNxpk;(0JK(a1+JIW0UnS}n;UG{z4}gA6 z1)%+9y=Rk>DNS*>#vB$z!FXr@<4ztV8tF_;DzJ%(fZDEUgSt36Q&2yKboEJfIqRom zh+pUgId^mUe1<^Qpu-n8ftJ$h8BdN5K;?{t<0DUylK_H^9RQr=K;5PNek3~)vR3Z8 zlQ#wQWPH)gn><3FP;wHE8Rkm9`@8UzYdp>_ML37#Qz?n##tDS-a?`eg^CBP$fbMTd zdv>@+9I6CsKQ2?9M5s_q6FLUj1u_Qat;3u`Zl;V1Nw|6dChQLfcx1psyLj<5B`*CT zQ#d%{8KyJM0_f7fdL`?U=aa+HrS!L+3Nk^Kj~O0F2X}q{!P`HBx%lDRZz&t_!Q0Q@ zKL7rsx8MBne+lW!=db?c^*3q#B)(g3d7@#y|4DfL!{s^3na2#lgvhWsvpu@Ys<_EA zWZY5Q3Em1;KmwR42e%s9L~(IB3V1ghG6mq0u1yjf9H~%p*M~A~0pjF{juCK&C3 zqTL)tBn4H$B;$(asUj#ksTF6S01}==D|u1@t-^ zqHda|zJ^osBw2570EId!eoG++#U>vJKz4d2TTd3BHS1^=30{BS49E)>Qx!*)#IKQX zzeC@HHr$BQuxW||CNWlngI^fWhQD#p+RPjEk}a56707dvMoR#F?34{Efi!{5gzHp* zpPU>c&TP02rAJ1=OB{o7lKfTf17R*}xi#2#X!mvs#M{m@_z_Cw$qorOJn3(vBdb(X zwqq2=G=GT>W6QTHJS~)s;J({~lEXSpp0q~REh-q~om^NvCc z*ypx~A%JT?*M+nTJ(Fz1tHl98vp zqbdLunREGak-rS>DT|XyHSg`S*At!W8yEroYlX0X^)HUyuiif3Yxv;-zNYW~@a?C8 ztg@fIej7mC2m9<#Ea>*ZjI3%~-sJbsJvJJj5(Tw!NS>cDZ#3}HUSa$4$PTdAg6Rvz>Q?2lQdf2jw=v@7unbb>FST=K$hgeJ zxeq%yzZV6y%<|go5MQ&;ys01b}}piO^aw17^&=M;ngsQlb{k z)3_P3jLwV_;#MQl%SRa@At8Cv;GQ|w4@xfaETV9CD4;vdcK$(?R3)O9XHGyeL?}n zo;B<#-6a)^5MUS1CCLEY?~575Xnr?s^!A)Wj9DLLY=nUrkTVs zxiD4Zr##KZ<(aL4LBx7{#p2HC3jNwc;=S8O)ii~2CO zt&zge^rvppjc&no9uV0N?z_%9K)D8_&yS*z_2Chk^uRAVf~A6OsJb)*?>c3r<){d~ zcdg4SOqN`ivKqS*byAh(X;A@c|AofO>JojqHXy|s(BT?Oqesv`;Ur6lCx9tSs(Ukam_S1#7cEqI;u>EFv_S4gudP*8x6bx*xViLM ziMAC6I8f}OoNS2}t?Th=-^bpXqWC2P;uIcBtTxmE*F-hT6NSlhb0A<;jDME0azHrG zAugGYBb@f9fqb93+H{X7P|8V7-V)p^(6yz5tQ{c1O6V?l~#JzkQ;_yArj=#6GN33k0JKTQuqlO-t2H}CPOlT-pgnY^d6mM zwrA#f_t@Vtdee0M5YA~Zv(auhg{&Tuu3(|S#9V4g027hX7=V4D{iHn1PX`{rQDr{# za5cGG7}XVu==1N1OFGn_Ce=EXR98n4U7~`euPVaAT9&}MtJA5}Jf>iQ?s%)94()L7 zoyPeQ`1Tr@+|hnb*12=*s*p1*aL}}u@?9lhA;8xiZ(d5yvP6t8=FtXm%0UkcLGSYj zg;jnyH$Abb%=Ll|4 z3FJtU!ZDc~$locKWdhuY0bUmUa=fBi#pt9kqFQ1vGGsr;OJ9W^aa+wofK-DZ2W<1w z_Y_hngw927_M!4m4@;skWbsjVlj1lE3sG)^ZKG@vB~U3wc? z7TE+Eibg6t-Au#=H!=-KDlq3)!rP#aPs8LP zWd~YN54I*&IGuS`v{GX~&InXlOAAH5%R1M%Uor}B_wdn)w+ryT6zD6WBc37qwCkxv z(cuF9i%QmHm&lhDv#@}^$+hXALo;wTt3J`L6<%g4%}B4m)dCuOLj2kUsFZXmWDI@Xd1nKB)C;74b?%V7J=pdf*~Q%Z=JK2ib?sG7U+ zpo?dnI;!$1%D3w^8d4{hVv#~KXizyG4r96zO*pNeoUowpbfo7> z!mH&4ljQvD0!Zqcev=wX(HU@19G3S1!0uWyIUxfDZpsa8^3nzZmOFgSB;sqnp1=Cl zS)1>70eX8$wC~en1pfZ(m*G#3o%h4vq@ABg>S3$NbM!^vi9}HTMPX=7sVBzyhLHn1 zGGx?-F{WVCcz$0k=po^CVc#Z_fb$_3Os_}E|6PV5Chz2T^6lmRfDCY5VaoT@HJpkjUC8Ggc%d!&;$Jr}b0+a3>NF|9zfi1bq>T3yLQK6i< zjL*q;=qP(I*YDU3Nf3TuzpwuFSRE!vbA)zj?~f}c`B^z0*uNH3%(Y}(mF#bqs5LLO zmpX~<&zaRUs^N@v+3S56f#;z9&Rn2{Af?slEqPm*ry#cgR6u46U}HPlaCT5>%S*pi z)ggS}gI6}gQb00r4&65x4)>$W<0h276xzdIA20QO#Zh6c8Z_3_mS$gC;7rVz( zBo$%D(TN}*K*v7vgE&Hw0I~|G!qgb5HujY)`P?}izO!WCNIh|CvN?#Px?3b8c5l9y z*SB(g3;3H-*lq^@4OH!+3YEw;eS-`4DR;B%81=e{^j!XZst*s9j_(8bW&9lPhp2_* zD0|`$-z!Tj)n&(_1vfQ(!0hm7^xaNPKTHOvv1aQ6J)3+eX|cKqkZvzI1YC2dUDv{h zfivRY(QxaU=%I zyDMAwtDDh`P{0PqhP8{~BXaG0rh-~$tNnp= zGH~v6h}Dd{MM1PZsSGD+h2g6tgof6znD;hF?O*{kO`)<@>b+DUIeFVdUcYx8_|EKS zNbxU0v9>h-b_>~LA^(9}8ydRif^{Lkd2ttA41S#6b%Ts!^5_glZgmjSrryb*K<@Du zdBkKAfrflhgkn5`Bj~st(PFlvDD9f=?ZpAv6V|Q!NQY#``8np zcU56DYT*KI+(AzPk;F-SlIuN>6Veh5i#OU|y0mkW&*g+F1o3syR+@$*gF{?e04pxY ztMWO#4P^z(K0J1tudWNK<_d2l=N>O=I+a|rJM+47duxjhT8&nH!gQo9P0a0Eko#Pubt z5B2~`4FPMUSP$fs4u&;;)vX|u2g1T`d>|>w8clf@L#K|9hDQD^rQ$;fHAlz@^FZt3 z0-WffIbJ#qtGkmyW4fw^qMg)1iboE=(6hF8EWx0W|9EV}t^Xl>|G(;i;S0VAC$#-< zT(LH1D}VS-w(_w&^6rC;-(axuqv93w1xrs~v1pPD=GiANA5Mw|+P9Xp?}UIp(O?3q zigL`)!KRkaMmuDwc&74{J3rBRmxz?t+cv4C#xb4Mc7kzp8Zd>X<6*5~9jbpX+t<)} zs=9$vYPrU4i(6`=-Zo>X>KJhE0M5xZuQrwH@6D6~;m2OuDs`-RkYj8im1czyV5!0e zy)$E&E?cQUw*$BDbh{NjU}h7kBeP6RXM?=W=h`&UopI6cI}igC1o*&I<}c_Ww}$q{ zvQafx(yk3$zA&ziJ&cIacBr$z zy}6P)V`0DWqb=-%O%wKmDm`slWfChmZK{K*_5E-^yU|RO&b9M#hhU`7619*rSDAeo zP=>q}A<5`>6u1ZZVafHm1L4QA3eZVN5K=ex^Px;y;J1Cz_g?3>I$2@rAhA*v!+QvT z9x217e`o?ut^$0IkT?AZqeJE7W{N77pCrmHuFt1pZ%@|b_w5b~6f!ICxS#R|e5cFL zUH6lG2>UaG7epU5-o|?hA9$AEpfM`(r*@|5E!2`uwuiN1n^mXUK_VCoaJbUc0awCx zaC@AxSIZoFFg06uk188!m;vBawa=DM0eL*I1GsS+df>kj0so_=PBpSLY@#|b+VNax z5vO<|YybfoeEc4R#bYy7=H7D1<=2a zKQXWa@F_PWxcPV9Ly+b(a6#%NS*D#IuqV)m0!=?k2_Gd0+oDLd%8A%FMm6?@Os9r% zN!ft|+-CLsn>a)yPoEzr#3MsA&y6~vB6oFsNho#LX^~#??%jV(E~;7(Y>V##b!5^E z{NKXH#Js66Fo8!E%Ifpbtb!FrydH4DS#CaqKtL0}+fIhb)d=eInHf+Wx=a=1C0?Mv zdHixfj}CrYB{^#QAh7=uE{T&Dc_VV$TAu+N&kSx29N(7mDhx+3Z$vAi4v~*U_J4E% zq^PXuXi#?X<;4H*B_xRR)v0!v+IKndXPk4i&NS4o@+ zbsPfqD>GxzKHp3ilx$d)SnQ$D-iv4GZ6+>>q76sJ?IgP?N&R-njpje|P%)}}7#x`; z`73(sKKs}foh!(Ve@W?ZGz=H(Sim}{1f_ItvsIr6-$?Y+anmLQpH~Imj5@KNJ z5vwwhbzI;pzC% z;yn)%#*4(ALVZWAeIZ-Oh+W*E{824ea7){{)KPU?^qaQ9Ex(Z8wCzMX4yrJ;0-yk* zXAkk-?le~ITkpd300h|)M05auw5+BCTE54OGy-^)W2c;BFQmWESgFO?+WKr@1t)Vj z4A%kZptR3P|KKha6(r@QMrW~jZvl|-o(JA?cWT7l#&W?82+Kb?T$t93KGSk3@SN53Ev$6Y}3q%%Ksex>VwGt zXBoih+4rA+_tE!1zJDF)iSx<(C-0v_0r->R!&!zV0;EO!)7w`_4&ajW&*(FJjXuM# z=$G?(u!A2$6)4S;AGn<$bbv0*WcM*NCce23f#!h=fJYK+A_e;83dM^BUghJ-hxMXB z$rdd+#(lXrWQTTY=TqH&Nb05Pua!~#UfyfCQO zIx#T+efG z;8T6-W+_ji*RBMh!HJndpb>!r*0a7M+Z?g!4rB&Yn5j54f;7*!1(R4;9P(b*+4de9 z@)FCepP_{8khD^dux;6q_NxZR%t$c_ zKLWYz+?mwo(q=N?doL~p0-aJRSE^(dK5Cbbh6(-NVrd|l`oyn zI$8`#?A2o5j*jWZE1Xj`7MXc!Nl|A7OT1E5t(KpPIwvF^f#vSUI*8C2pt@9{e%pU3|y38DQV)&6Hue=7=9A)A3zF5 zwg*rw$#z*t!4OdjgbHO6wsZ~GbTB=Dh6T>GoH4&}6d!Qti>GHOLDht8^- zQIB%7CcJ%JeEU;Lnf~;5`O(iMsrp&?_D65Oc>7&`iT4hoCxjx*_H*-OFIlo|{Goyh-e&TJw}p&HjTko%kH4?|hLhL(Z@Bcx(o z?ofWM6x^>i7g?b=R4QhJ@m`M=ewi|qjqT1=rIOoW+9Nz@@?&TsY!x*LTmwzvl6tgk zRj8<^yoHzH)WU$23-OQWBu|^%^Y*l6>(+f=y3kJs?Pw4IAeAHwTRzya=Ao2u_mN7< z*T-@3Oa@1@d`>_huzd5PqJ0_IPJM?jv)j)$(2Q#`MYDg^+CP3$&kBz=qCgORG| zsFJ(h<-jt`F&M@MuDgP{E^0o>ELMzUI#9gnvb-hSlcn4(tMg56Fre^6W|-j6Cq<2m zD082hTI3eAu~S-!llx(20QoCsLv@V_n`B5e=O>cfaK(;;+EUxe9k?!&Wg;16SnX3p zPkojYs8SKol?|AMvfsmNP(vS^{frM*nDQ4;QWn}GLzI9Wi}g7emVd4z>SM94D9 zrTb`Wvp18Z_(hKqdWZFCNmV<@i^_wH&G;*6jW1MWl4G=T46V~2S~Y!qt?~IEeoXR1 znA)O1^DYPyXcQCdHw7kn_u2}bb zLO)A#6BVehpf^v*h`azwO)#qgih#LG5At$zmYlpQVp;o%`5ZdQ4q(2v zDC{KHK%A2Z-(F9WjYooh0W)#}M?_D7dokXp{PJ7-q{1S^L1ja*c&|;nZMN1@?#F2H z#)P0n+7BFzOBhTcLy<{~c3+y1) z6RH6XH_rJad9C)0)7Vrol!Fy1cE$tmCgt%eFSIBe&qVC9jI!_rgoA^R#o>WPGr z-7CgYwMMN|(qTwtJC8)^wN+(h@LvW7^7>FT+-8Vm$W39Vb$ignQ5l!81Co@bF5Ib>`owufpT8rxdn+)Yi1OoL4StiwCf-IMEq~ha$(b)c9 z^!R(>zds_n+7X{&oc~Uv8E+$Vd zkbzI^jC9)y0@qxCGeZgnFxZwxAkVolkwZl^YCf?BpTf>fXb2^h+m<2|*tXd&K~QqX z^N%Dre^gA6%!LnwrRzYu8M1MbACNrH`uGz}aw;Hq9V#<*Ckcv+RW+^Bf8F7@-Q)+F@KQw;OgSWuq&cPq)Dwk_1t&)@^~KFz>0P^vx4NGDS^RlpJeP7gUe7dj_9blz(h^ zG+x<3s@n&o^_o$EEmAU92d!4!=MQkoO!G~1qbOSKr}v@iv(#xbjm z!pt&1(u|c_SG9#L>C-jkBTM2rLfT0}GOJF@`;$!Pfal~x z*;#?@!NJjvk?4)kxRP#%q#vN#(YlFf=z?w4^ao8goqAqa&m$A29!7G8S?wn6#}VGz z^#F&tI2Tf@ZE48Qk=LvG9N09vLu=)G@?ZGg_fBlte|!7UfAlr{zkLI{7TI+VMYD%`|mJdlP_;S zFTVY6;q5n)q=mL-*0L*O*Dxjkxr=w+ z{bX#>q(nm}6i5(K!>ZhwF7zb*tTlCIKs%ci>GPl=q6kpO@MijhSV2EQ;K;+gY?pG5>7Q zjM+L`zoz*Rx0anc^-QyJTxR7u?a=-JpTfE-Tc;_|D>1aeosFOc0x>o+GA9WJH`&2a zDNC3KTj!Fh72VE?>Rqyk7IGlmeuotUT%;)LO!e{`>5o?Ax=7;s^dYW-Rg{_8<5mYIuo>Kz$zlA zmV%r>@kTAL@?oi6{f$y}_c2g}0s&}I9;p>+_a+8QEHGC(yi#DWUeT1{uA{sI#|0zH zo|s=m9&+_=BedM(;g4|xkWg;18z48s75PpKH%Yh-?};|sH;yy&o0WnTtvYIw7A83k zG%Nl6!-q=J5deIXduTzH#z1-)Jzx)7PO37yDzbwd>fhb5GXZk0@VaP+ zJHRrEyPIdCZ57^1d{Z9mbltdX65_KROv0ukR~TXCNid9}ue~XS8!2nl5&m-H6H#75 zPZ{ip7Ncbsv(|R4$#_-@xRxCx17hu`LvNs(aL9+1S#SP|Y0Pj~nPD~q9Jw>0ODTB| zscT4<9oTxPa+-c;Ho>agy%Jf96FAD;o5Z}Nvj9IVlhb{XQ5;e>09iK)asY}}l}(Zi z;F38X=-wm(pLKyK*!7{q_2~g)ynxeU2l?kdfB!vLFrUAFMa|4VYqWe#0PDvRyFUs4 zT|W8yx8DZB7$kI0Mg`yTcC0t@Je#i=gI4$0L(Z!Yi7gjO0ji!htJo**ny8+x)NXmS z!`|tX(<6O%NS8cXjK*#Mcqoi1726s?=|j->@E;`uwtaC!zfJBmDhqCy1+nD~CWEzw zSM-I}R$}#5qJEIjY;iwt-jwxJH83pgbu2mgc>*W5Qm%` z`&kGA*4~KZ`6{`geUcZI3{#Cnf>rxIv=xhF)h&@L1j~SI(FSh~_tTuSbV|+ga_R>>6zfblaDp?n#F%__XYmHqd2Ho*XFvH79w1xRtw_ z?~u`^qS&|`9^q7JiqWy(wI zvZ&a77^h3;ZIcQqsO)*ka(PKQup!Dm4(3pY)o3hGdph_i2hX|esG_9QU&BEb5>(~k zLsx7bfXRh^c<-QGNHbE3dl{*!ct<0kVqyVn<(x$*)*#Gd>wn|6A>VF{9E#qHtj?XB zVqH>){lQHG<6wnp3(A)i&uw&t=oU!u!+%Wb$LC(4fOtSf^l<20ZXwlv=B$w>fr`q# zN>`gJh|WOtB${|gK+=XK#I`NX=pB};l5&@>ox+7#!ljkZ@B>?oAyixjxnUKYl9skf z|F+p`NfKy-B2^l-N2PY)3$5G7)_Fed}h8 z%o8&BHK6n(nj=@m+YhAu$h~OIh-z2VL*;D~Wy?}56mZl2#ZEfOQW2I%&uE~G$1MmB zT%p@0&uM^lx@6(dc8xxGid)?v3+%}yLffPTXS9+^u~U2=JvsDTUxhyqC90m2q%g88+zs&it*p?;qd(<=daV|LpA-;oC3XK7ap7e)g01Pr~~j zKg@9q{QJ+6&k$d7nkfAn&VCo(t()=pZ@&)k{l12>&CTRJtohw$ZPI7U`|K&%DW$_q zqEx#SvTfTpz{Nck4_RLMdTxu3a#xoQYY$W+Myc>Vg4@d>M@sDhOmDp4c(KEXL^>)u)UTE&0fR z#kaOLl8{?`LjAl7O`^{zERrN&nvqS|Z-KW}4FF64AFB9BmagPSJrl%f21nJ&K2Vl+ zjPIcRh2*xF@;kxrZOJs?+6oPML+lYqnnVhi4zrdPg3OopL{UB*#EWsseYv(iK)+Tu zLs>Ri93So~R8h0ff<48WBMN9iMpkHzM*l7eBgAL+V|`@5y=uu1z^$}>phmh%vCcdR zhng8Y93C)5g`VoVD~N}EpsL1B=YZQ*Cyz(Non77*;k z)al7pe}yh>J39aEs++hS2Kt=J0+O_?<_xN%#PkJCzWlx=;SGR-=My#2!;R<&`%9?o z)z+_4gckKKZ(4Ecv_Tmlq;_%jRagy;^DrL$E6JNKaAWpy(%_UC1@(l1tfsLZ=YxI; zrnFCcYTV_f^iW5-*g&+^T_UTk1a@FsiPgmvXWWLB@IFU(9Vj{ zcjRrWCZm1^7-miIEp!%%uUV1Kx_GPF*#ch!o%M9UY@soC(oGplsCd=Aw`SUz(rl@F zjKMXgOJ$tUa1__hT18;}D|LB>6)qK0f=i-;X$Vm56V01G-XL?*YO5GcDElUibfU%I zSks$Oj;ww81NraFLHg$XWA<-sECXGVw|_oykfG2t7W?M7NU0PzbX+RSLm;rZCW#K# zX=%gl#0*|2uBej8u+f#$7jjmuTZ48;`Qs}C!ZVcl=K*g?JPc9}+Oa~UPaU#{TBhnD z~oYdLX?JB<05g2=vBMiU@hAie6?*@)7MXrdhCW8^z+Gw>8aXqNW@%1TDuKlw~Zu7UQ^4R_4F;Gg=1XrR%!z1+MhfbTHHldi!sYiVW+kRFur`XP>nXC9gmV5TG-ZH z?aI*6w1k18_BH`A$dgmy1o|lEeggBFcr^o4sBGtlWsA3a6NeF`W;;!?7VMj%(#F@# z0z1~?9inC6o}dOIg}I}M&LwsjDlAjB|JBD5Jmw5}^0H#aInJ`c7&JkmmcK)hcCcC` zohuNxjk#8*2U>A<+^ga@%~jOGaztMaP?3_Cz6P>^)0)uUKvm{o9P%rJ36*SzlU=z*UC~Xi54R-!C)YIfs-cOw8TJG$bT$*`p4qIs;u_T)SMxQ^AW*h#! zUcEQ6Nxejf>w3Z;#|3^=>@r66qZ{~UF}ad#;gy!Eh!t{E?vmNsPYw;yKBlKfg;!K%xj72vcNJA!Zkl8@-uYFE;|9Won!KFwA5$yE6{t0A*{ zftG6Jnr$&GXQ{%akd5EQByeD>)4)wx;k2?Dw`Z8TY~-UMC}7Z%{aIAOWMi8{yZ$R$ z8V4_@LGg~hTU#>$!q3@=L<$NNdng3FJ(1$#aBQR*)+2aIaRWv%jCmeoSGf=f#tqgh zF!_+fheeSK2CC?_+#=&dL3*@gA zVVfO`+;l}gMeq$SK6TmMrfN9rGWj|@wk0O++A!jFdS#cAPSTN=;27v1+$B-QnTnBR>9xB zM*Vt?+9}^UOYkI*t4(OV10%YpiQ4GiCiEWbk_k#_V(ItjAQ)2g{?IHG}cblbJng~fEfUR8BMu4}tT1t?T^ z&6702libE07-y-Y`o8#ulH1z)57|vxlD_K(m%&-X{wssaRojcm&MXO#ViXt*_#&l9 zp)|09yf&6B7H&c9g=Y{0Q8g2KkjGZrqvYsA$w7k{`7h~yG*ZBq13_bf{Xk~a z!tEU%;?PKkK4*@-fL6(8xVmQQc(sNc^o#JIY@3e)e80w8U=eWA;;+6R{?dS)U%Y?y z_Qkhf{ONDse}o_0aD{oP#ph)iU^1)x{j%13nFS5H{Ia5c*IoBkU(t z`G(Io+C}EQ*6rvGNR3Exss#KJ=%xuT6pkDwHC<7u#jf12NsQjGvsKI z9i7dLOj*%yI2=O8tS z4r|?i1-E6oqC=qwF3sI72U)Q?4`82mBOIeXqb=TboMw!*?KtXiz5vVUkP(R`u?!7> zqbr!|NR@NjVRLQ8In^pP_f$YBP?Y0B6w-tjHZJm`)=kJrkyC01t<*2 zDMK-YW1J~XlT$OMa36{+5N6m7xJ?Y`9l%E3G(iTnSPWD95lO>CJt`^9rWav)+QAS@ z^?6;xZIgY}QVqFn<1-w_uJFP=VFuLT)$Lb?TFs>w%8dCV$^*8C>P+mB7~&)-@?+Zh z;p`Y7Lk|zR04S-u!^sU!uQ`Y;M9wR^Ks{F#zFO7p+;O8dF{)@g$W^yrcbB@0?sWIe zxa^`g$L=wtHQTAQ!!7%~&JK!{|s|BYe!{9IE^99L$2P^QAy69@vf0v)l#cjBGtaLBfoXJ!t88N;ASh) z8k(B~D2(n8gRl+L5*5V&v&$lO7Q(O_!qAvi=pWh$P{mlnFy$iyM>Y_=2>Er5yLPr# z>V%gaQy8(;vmPlIjmXieugKz%TKsZR8`Stf`EQS+*9WXzGysNz9##(M6o5Qii=xEv z)BSoNPdX|v6XF#VNpHq^N$WAkloGG<994DRKHAu8Ri{gp63M49Ow*RCJE%b!c6IJS zL{kn|!H^TpX*wqvgoFTCKl&XAch(L&)k$PBR_cqZCN23TV{GuDL93dmy%}Fa4vKF- zd;3&D44=Gz_NTwi^ow6JllcAHuMe*kKfyC!zkhN(<0t3i{5t@Y_)*AT)PMc$@n4p0ppL=g5_K6SOS_hNj1r2%uHQZvmoIKfbkRX;Y%lB|f zFsT5hyeN=n`_@&=Q-<>J?1EPQ*zHogS|$JqkJn;>?-znyesDthv$;~ofcYZRfeFco zj}3ZNwdS|>KxFn7?E8}>^KuZ-FtEr5?ZJUROVXC8y=d%KP@RByDGDHo%c^4K6@$cX zN$}cbl%qQ^|5xgzON~2kxQMuF&U2qPZS8o6a)O?iwdd!!LkDu%3E$N|`w96BL$kw@ zrOnU-1ERpGeIesk@hE_k&P*<$pNs`?=lfJ~*LIb!HJStVAbQ>-mm{d_;c53QVEhP| z`J`^$X9U_Aagi?)2w&}yS#9?$6v$~-I)zTA3vLYR-dHo88%7Cr;MJ-KT%=g%AQ=Tl z>^8;%%XSHm%1$J{Zd9;T@nuT2LsYWgSB%dU0g?5T$;9b_h=H@;>dr8aQ{~!^<4F?2 z;1A2u@#%ppM(ZeEkC_a_RBh4pQ1^Bx>At!`m5p$`Ve}N1f7~j!|O==@-BOd4qR4!X1h`CbgDT%&s^KX1}b}J~81yodi2q z8p}&yjXENA5QFwG`UuRzIM{l>wGha+<4YI%)ipm_@Bub9K%Pn=w9&?}WC1S%PF6ZjZ4%2%FSh>Q^o{qAcS`Qci6iUCwKY{)Y zAthHxP$@#+WrqNHs7V|F8fRl3gFZ_f9vxMYFK7kwN%j9xoA>;Ht9D)5mjK`pgk3S! zFyj|^lGhD#Az3Sz;)04j8;U2{@$fb{E>Yzih{$W**o-(DTHEmfF=hK9B|~V`vU1@F zV@5DBBh>R#icgf}`f)LY;krEb*=CMu56UW?2AFPU9I|>KKsog^^>GJhwbq+Cyee&sA5$iIf6R?+z8NZPJa(K#=y9OqeA(3P#1 z+|2bwjX{g$gm5v8u1a=vUm~cxaKc4=GA8XajH5}x;*u#JzgcpaVfbrzsUbF{3ZU=e*N|%c+NtvB!f~hVe}$Pl=-J(b@*fPCYiRDh3H_+%2in&J!tnuAC?a{lv>{nh8B z+W`4_Rdif>Z2{QBvaTNO;Y`y3Yw9G*bytrE^RJsX?ZLb()LNE+B|&d-ArWo+py1^s zMv9x!u!-tnyKIE1wrJJd-55x@aN8wotVtp7j(MuA$FxrAo5)t|>7>~x{E6&5>hp@j zb##XUQY(o+g*rw~$OPB+a-d2wV|k_PP2_&KOr7c!7V1v{!EW|(AQOjw%T!>P4vz#l zby>v*pzs~cnnvypNYhg`MMu8&XS4#52y6ML?3R!-x=`GF$&S7RfdqGCkc4fANlR^` zG-%8^#}^fEnO-A-MR|^dHeoRXcu(ohIhZCP`v2#_`>X5nxQ~5@);qHhSpy(R_969D8HwQ1bYEG z3+dR4C1T1{wvA+H775$)=HQR8t+mal2k2kRO5x3|t=xdKY^T6}xPdYl+va*iJi+mM`B& zpky(HI?qGyG1HCj*x%^k*UKk_3t^y9EGCMWEbtW~qPZO<&#I^=k%@+zJM}G-0!^$h zRJDg>OpA8wvOpVlR!4JlglKdrF3~9&_K=ko|p+pMG z%14ZQ*&Dn7$#+%`HsJW_kv8bAYWu|pFqNQy+g`!8T@KKEf>EDZ*}MH7v{8=k>IF{m zC-*$~qbp`t&(n)+5%Yx1s(R*^k@^y~&*@|n@pcb(H64FsF& z%sWKkcm7?>g9s9VK0Q(edj?%0nCn4E1k;+UcXy8J0@GHud}CJ6j%9JQN4~)j=XZe= zlh59Ng>2z7iRhoACGzv|{)gfTHFLKh8(wXg1Cu3-j`FeRu*zTv35%d8Fn2x4K4uKNauZ%9 zfFJ;_q-G4`vzMKzbyfltUlah*WE1-%GBUcVjan#g))N6A%;1?7Dx!ylJ_s||;o^h$ z#XbL)<1UbTVbyqOqe$wdt3#GIhvZR=H*$QEcW>wrxKdj*%c>Neju4P-NDM8;2}%tX zicG>|y@ac&M2zd&gBh_OsjfgQ-wS$`F_|%>zSQ|>qwpiaxVrh^b3II;b|bxzemGf)-}%{C~}0Z zk}tQ{jy3?r$EZRQP^55$&a=rs*ydHeegMXBc?nqnsyjZ?BL!-8Cl)~p=|O1-7%@p+ z(5{bwTv#!0mIp~-uBZW_X`;0n+kV=#sLE$eApIcrs;vYq5{lC_sPJEGJvMOflOkR1 zLyp*RKe$xalb(OZCu(Meiv7{je5I~~`>;$KVMJF{)I=$E@cpUO4}4^O9$kRrqTC)k zbcbLdTOrrLS=1%w+0?qoU1Y%m`x11?vCl&b9(#4`p|qAzU&0*u*zXA;K}#I*w zL8xk2HexO-jR#ccluNuQ*QUC&yoLIOm7^*aff-3fl!1Z_Tp2=z#qN=8fi9IO5PSN& z1GcTW0HFL75a72-7S&duP{Rj8okt0vhhW6=h*n{!cS$=-;cr$OaaJq^Kg1_DjRM{zdq~NgdlwO#hW|fuBh<`qSSbD*fT@ zQv($+HvK-l|K30Z)Yr6X_n+Q=6W+hX*H0lIfiL-)w;zXZ|4)fvx@F}E_6(*7y+eIf zYEZX>@K(MBvZO*c*-d0PqK;v#A27dC;?i~Y6FEM_ZXbxgAb67!iIdqYuk9Z6^2@fc zwU|EBHUy(#M-{k{&pON1eg;bBWg#bPR4OzRCZ9S50R5pHUm5@K00utmJz0YtegyWn zkXPOzt9cFA>k;Bq#ZEi?Xq2Fnuh|-=u?YlZhLllqMMntC9AeAzmpBeOnzI+t^<)fH zqKun}NF1~^xu93rBClZtl&qr3ZU4LB!j?u=i8*dandeA#&LkVX>-;Xsok-9$Y8QM3 zm(k@KweXQQb`T37KtI=ovXSubm44YXHVU~vXgo3m&i~T_eaH4>Ups5U5Y|JOQym4b z8wUsDR0o*tS0Q}O78FQ5>8)g@;Oi(aO2pr-}m;%l$ z10fnk(mYrLUP#Uor_h*CkpSz{jKKT%M|`$+_E1U8dm#DQOKNTCV<)=K5Faqy=14#m z_6uaX@gZ2OmJfsV<1n~;7YNz$cEn(SGkB&WeE{}kt-%0(b^Y&jPP>wTfJ`&1{^& z#wK~753FSiO-dJ4m;tUc;Y=Oy7)r3;!XeSbNGX|nOZ(qxJGQ`^|yEz2Y$hxQWC!!v+;G!)xv z2(Jc@;98P8mOqxfn5bLx+EV;oDxRkxX^W5u_%H8q9iv6&Qe!J_qYaB??MnA zWe0dxV=fVEudJXXjbQ1KUrSA8WmvqOXLXrOP8oq%Ctw#AFt-wx)3WGz9#l-o0y2mc z)poM_UK4B-w;ZACI4{IaVI4`_K%icmJag_IjL9gjJthHgS+bF$p2;2H|4l-esB@g< zRDD3B8WzDs;HA`qLU3&Vuo&Ne$>D-il%fw@H0RU4Mn zStad3R#nk3eZMvyKar`$wd_AMxR4renh^*fnm5VIroTlZPeHPqUSp>ug}0L#Sdve% zOa0T%hE?wk3u9AwaA%dBBw5wH3p@>HB5e#AF>n$}Bj2g9#YpVtTqh#*onZ z-eKwm!=;k~<^WI{E9PKw_r0njWdI#?a0<0NA=phy`6Xh1zl~%NBy_(x3hmQ@O@cK1 zE5kC9WOK|zZ;iIq2PGOK0;Gb)U6N@77|uLEc{5@DjK3 zBOCN#uxhy&Z!?-`-MvlZ+N0w@Gf}wSTAl@iiZ~Z7_HYK#p5k?_(sZG&miH3bqv4th z2z}4;^-jKI6IP4xTCxL^?|fdVUmu`#=FRX5w_NU*Fwq#UXsG7X2CcySwyuDQE#wQq zp=!>5Kq8d!@hW{e1(A>Ikh1B;y=))1dncsY=Bmsna6>jg2Q zjupR$AzKwu6Du^Hqf{fuy2k|MZ%MX?07>uiZctpd6rn^56liG|U`FLW-f1cY9G(19 zC&`vXmtJVtm1|LzLfyS;K4#9<5Aw zyg`dfTg2K0X+Kk|Ef9Ci55<*V}TJ_ot~ zmkQB)`#1a>-hYL7{-a>Tp+k zd`}foxZ3i2#`dy>X95Dzf1gx!R)pt)h2;TIU)0`hwHmnQflofUjQ09aL5vTbgwTqn zin`QavMk#>b7DycrURwCs<8^;@T-c;_yl*=Y)MOURSz5|21BjV10}=Z8wwzT#OsqV z5qyZPZ~deKbL1>ryVxJ(7fHP(Ob`i0ug&*X^mfDNz? zeD`jN_OfnedA?1-7l@ww^HK&Wzq)+v-Ijj6oxLawDi3`#9As4F<7y^gJPB8zok9F$QKrJ5w#I-u%Z-T5zT!GTR zr;{_e)wMqN_NTXSBei`F!` zwCyO#=A$sUOPMV79+9gVt5J%7n+;cC;*>bIP6NaUA{U(+pm)0}I^Ov)XbMBWbXn4t zd0-5w;Or^aHWfA$1n2j1C)$C4+p%T zb18sITlZ$_F@x+Oe<`;=2EP~LghN?plrM4%*OL2?PIm!Jtz*|}> z7;7nX>8_XB45?CwdfGB2sM=8qPLiJv{odusf3V1KAJz(L+f4KYfIH-wEme!c%kG@T zx_vOO)9#^}T_gzl7;G=8b307eU@bb|LxvQ|u`C1RQsXmB?!&2=2iF`JQDPe(0S$iO zBcf?X=0{s3?SuP+4uPZPRy&lELD~bAtW)>1SRg#($c=JTQRyq~-OB!{Wx>R(YLpZo zYT+(-55A$?=tOLnS<(9;8--_0JGxU+FmyiVL>uF-L-B=OH_RvGdU#b>s<=ZZUt9E+ z7YRYH{Dsg(B^nNDA8ar~*HKwCN%F8QxP3hKpGOPwgWNzB8_7^3dFa2gCt*G=mm6QE zlT}7o=@=^S`Xo`Ko_9Q!s3D=hgEy!WQLR;?k{eSa3%%k4HnWe4R?LSbSZQq6m?y=r z*aP{u@>K zNUh2EEanvct7He`i~=B2*Smq?{fGw2HsjvG2)D9L2nOxO$A*;{lOUvLPG&$ZJ<<0G z`bPtX&QwWokqGtl;vt6Jlkyx`;Z4xrFQ&1E=#I__dKNE}PEHbxKfCaZbn^n$;8t6z zFku1ZLvT{8J5tK;G!O!S!Vfdy0-&}SQWf-teE2#%s~Do@Cr4gUE5d>%TXH$q_>mld zzUsE-UPAY62S&h?Cx`A(+Rs+jk{x8%8xHg}CAZ@n8dp2S5U_2G0G1(PQH=vf=67)h z@Xo_JdQNv`L|AJ?-cSl3b>owGeU&VbJH`4USz^}u9uQsHvtOtDf~_xZ?JKM38>+~T z0Oxj<&7|c*1yxXLS|~?BtHGs{v(T7f_hswPjX+S4M~w3~EIkecW}u)flr^J7phd$8 zbA)+4=#PriY<*IE-GWqe_9w3}rpia^|47N}tEzPyhWD!IL`kF!Fmsi74ei3JxXQBF zhP&i%Pfks*#4{Bj+f*uA2u07a+YVe92DiP*Zd`O;<9dbK;o^zn;|u+D5M;r9Z8M4& zHT{~$Ly`j|-CAx(E!wOFmsHQV0JM?&O{RM>4^hQSxz+Bi$2)cUu)iXk4KfnDf03#& z!5Gv?Tq>!~A?Wbix&z&(##;|J6~(vROz2Rqa8JjlI(jPww~Hm()s9#txvlXYG4`jn zvt%LjxH{y2#k9)&4**QL?eb$an`mf9k7=kI?EFthsf?T_L8i{jh=>+Q$i{=?fZ!?*uv-}&}UP-Ua9ir1F1 zV9yz&26ydxCiqcehM7;xN}|J(d^Vev4*;hu9;16nKH9PKULvSgCs^?VeJqD-J`&n8 zb*pWXjPbVM-`$Z(OAuBi5>qf#7uGbuEnamq)v#xw$Y(>uUvhy#00r9uj-3OXr>y;<^zpfv zo89V+Iqr~NnH+Jj0hWqyr&8wL^N^V=f1>T1kQ-%8+8AWf%FP5#i1sfvDW^~E=Yx#7LAX4e@ z64>~5*p3S;mD(yFihn0G2c}tSO=~Nlvzy?1^^hc|l&DyCtz+5ed4%m2WLiz8QLesC zbD+po+HgoqDL@%vIbK{_$fdAVb1j4Ky#O^%3Dm=3Kj`}x74u$O$S#JvCM8Zn1IK#h zGznf;m-up4D2B;0)00c+O*a-}9^Ts$!S9XZn^Jltof9X}?~7VzEoOp+z0h8Cz2P#P z+&<(15Eq#z*Q#R(yT6pC!*r#ds!%Wa0wGZX!;cPIxr26r2^aDo<}Lv8&RcaS}8 z?l!f z)syKP(bSiH>Il1*kc{;6>9M+@Kkm(WCF0d=wQjOA#eq#K6bELpr40^@ z4o{~N67!K9U>am;5HM$wx^*4pm>?Y}$v<--kUby)IimfZUl`^%Nh+oG1m%=fv3d_> zRTOOGq-Pwk>s2Q)z)6zx0v)H*IB5dX3gcbFi43N|Ca>kuB3jCAN>paNtFP&4%U1|V z(rzggeunlAIVyC1RvbaeCbLOn`!tcuJk3flZ5 zdk=i&@1b_(95_vbx%sp*13fDR)Vw@9QOtdi%K#S zsQH27Ky!^d_6Z{ABri{p_?_+T$mPA}4F-W+V45SFRH&on4*Z3_YR#wjK)bqvYbr z8i8PjL>=$VY#q$LIFGO)+6N*U%vwmdKs{|Okq6MYax+|tv^q3U5A|PbX24_(BCUN~ ziquP)k5p{t?gGv~YaA%lc@M-^e!CpWNSe0biUyrreBbmS6D_~_$z%h_4;Bp7xb)+c zUD=$eg>oO`JkDXV0Wy}oQaV=V4LW!G05({qNiH+k4)U$_khKaeQP{NeU`c}9Mwgl7 zb)*SsB1%qTU5HX;;;N4A8Nj08nx65q`5|}=;(Z;k9eFw^g(=dt6B&ndIME;BSVvY; ze-@)JRfavsaF}D!{adzfNoMioKorMCZuLvJqybjG`Jp*0D=>kRf)1&|6fjsBh6MFZ z=o-_`R+gfI0XVLTxA4>~LqaGhB&B}5He@uQ%G*C5XD6V_mlQSlJY-b^P(pqd{`9}7 zGgh|e{sfSekKcd)uHW-VQbzzm`4kY8_upr2r%&=@pJZ6dSNdJM4g&z@`aqjvr=^Fa z8}c!v9X%BYsnD`>fpQTBXFi~Nw1?pKsb)1$_MIi*+{wh-yGsr(-*N*k?{- z#mr*CAot#I+efyS-E8SZrIt2_h&OIn3Rr?FOMFQy6^ul~J{=;~SY16N>Mvxbm6yfT^)ev|DyY<3JY!6%^ z?3!&J1Nndn*HAX+dcz*~*SIqIYcP7`f$@580H1fqt6 ztQ}XcfVdjCrd1vp``l<*aFU}6$0a?V21q!x;Z`V}a4P`3wUs4dueS_jR>JJjAio_=!b|k&o;NI2jIH<)cM$mRw&UP@h7ruvF`s zbBzt$=6r;D@2r5o-9yHV6_P?sX*es6q3lSsppe7}2v&D))t9wII+>vJAZyt?H$h=c zg=tO>7v!^kC$F&#-AQBm5gyB-&Hw@*dmaFb+Hhw*0X3#9_QfB90FP5rdXSSh_BiBS z+c%-_O<@XUb5GPqT%=^YSz3cljT2M(7&`e8+i8{g;z(4<-4RLc1cJ5l1+>;aHTH_p za}T3CT%h}wIs#ImM|)7!nA&@;WM*u9dqm#w<0@tPdGhI$VEl!~@ z^y9XdZJSz;R|SdWgE(ZXWb3Ot{Zn$3)REfHYrL5*0fNEfcu`+J6FgFqpAT8iIdEZYH0_418mxjn`U!%DE zOkGy>!tf~z%v7yGdPRH9 zI~QRsCErj|F`=bs(b;$fZ8aDgEU_$YJL8II6+c9;i_X)BI^CcUFJB>btF9U# zDCnCh9q^(3AWT>;Fq-X(Y6@5OHD>fVc~VuFCzAjx;kDfC`oJgx=qN{wE~gZ>#B>A_ z&Lyz7OhEu0FG0*PAV(7(M0GApmQ5sG{sBc zGz&Bt+_r#EuLVR-CuS|VSMdJFv%mIlpk4IsXYc>=_N(yikKTX!_9f%=?-7+xd>I7g zU)Upm#fLtA`}o`c?fr-0Z=h`W8wF(mogx1>Ud#XS_OJ5)-v{|8Gi%^1b)%!DSpnN7 z__AY5Xh!yg(HjSSb#^xIGu$f+#?XS69rhV_U@Ud6)H1v(NrY6BKt}!AQpXsow5ije zx<(ePI=UOdc(JlxZbGPwBO;THy2-EG7*v6a1oUQcd1h5|2)-jJl7cZ907{}KkhTB} zfl_EWoFK80OG0n83nZN*bxaqCyEOI$@)e1kMgfs+pjTstXt0PmU6E;{8b^EVFkqwN zf*edEt&$;3joTVU`o-?SX~G0iWln~-C={~Ba5 z9 zq+C5mV$xL=kOHyVm|H5H(}%TvYVX(}B$7bXkR>qGTLu>F9k>XT(c8x?a#pB~8+t{# z6Cc%|FMpH4-Q*KA^4fjGObueK1}A8xVyDeT2+qEe4V7=(lMRqw6ZJ12u(g@l0|FPD{`zri++c! zj{PJv2Crpnv140>_d*|Dr3ikSbTM39M$f+0H9j`v|0u6Xu5Lox9jaO^aX4e&pO9L%GEVOnV)1^f-SOniJR7ir^2{(a5UG3qD|w4UK9KTVE=Lz@sW;yX|-gZl{1UV9l;ksz+BzF{6DX?+{if zw+A2&fCm0(BOw+tGDq+QbeAKA8RpQy$u)LU;GuBnP)LP+#uw=j1nyQV=MJIrd2@*# zGXAm2!aRDnKsKpnqiY3xC$H)xIRk$1j0W4s1i?(j<5PEre7uEM*P@tkw^*eKTuty} zcO1!H0M3!^lOo zw{Q=^H!HL7)6vxoS77y+_)2;p|62e8m@F$mq2=PjZScaqQ%(0}%>u-mq~b&>*{-y$}C-eo0e{btz&KE&;6p?g-kpX3}sq*{v|Cvs_C zk%{1n=HlFL?~f!~$O*wbLJ+NWfrMu~B6aY;Jlx$tB7=tkXmD*#HKgq_Ui;4$!IX zx-4Enn=WI0+A%CUWubGD0E0hRz8-=W!$HRwA^#nYkkX?kQUf+;iib)bOk`8^BOFRm z^c(!WZ$j0P+EM$o9=r~@J0ZmXi9Jvm0H$+*Q3q8COT@YnBD8K%3F!z*KS>6ckD|86 zsCWT^AV9m!NjeLe6GMrsBsHXDnz_w!zlXcj_~04f61!?+ z0Jz?cpv#znM+@P>*(*P!F$jZ)3R+HOAZmkrXDwP)-qfcgXFObCR6)uyzY?tnoeD{g z1JwyP<7;vaLIjfvU|c)OmS53VTD|<98ZEd@6-zUd!_fSKsUYdNxU&}|h2&FTltq`n zTxMFu!GbdHGN!3=(OcKD$ZSiS4~Nz*#ero8RJ&7`1k;qJ$F!24@-6dv035QFB)0z= z*(h&JME3CsuY>1N*H;~|Ar<@*W+MVsH@OTYW<>e4(LRWpQ`NHr5EGCg^kAU!7yFWK@ID8&8HnKu%Td)SDL|yplG2-Y>5GCuR^Im z;3e5lB()IR$k&wLR>~LXWAJL13Wl+0zrZ0440^YsKv!*?K)8_mJIW`~7qQH9b2O#Y$BdK1|~^4ouU`$c&B4ZeN}rHNPQ z;pQm*8peCJm;}(#ByRTJ$hyy4C@+lBfce`k`OL!3?%xE6Iojs9)a`X(yq0BmVZOg9 z%1qJI14H6Nb^J+AlCaUX*7KgA3N;=|{#OFzvf9H5MX_ox`8w~QJl0FGRvDED%Co6U z#FmzbS7Dj?4b2Jvy1Y0x-a8e`^t5$+PpTu_kC;)zPHDQndUtV5#QsJh4ME6cu zMyJR^&L1iSk~ejvFT7yMs{zl^KS=O-fWTIw)h4WEIDrF#YLu)fpCF-j>()i3aotEV zN)(>xEgkxUnW46f#oU(2NFEq*QR)HwBI-cM_e|Z$9zyFLao%^g!1Pgw*J2uRZN^ip zU3R|a8QJNcx`U`O)d#?Hg34=f1djS@mLu*j6JVHKTy~tGlIe>0Cd$yT4dea!8q{q@ z)HxV{7?R2WyuyYF)Z9VslArR@(RP>Be=1xt&NnFs#L0gKLBguc9*Y-ipLWzxOsp`h zU&3RQv}vQ&5)gqFa?^_Wqw6kUsG_bd7JucM0~s`5x5EKI_IcOz*Etn;#vYhZH+zzN zV&QjHd5%cv-v&2_LMNpi+^5^5KpIzbM2xkZ0BA-7_RS!Bx$M_n34)Y#M9XN)TK-gI zj-jo|RFEr>I2F1<^(r61B|xe@Ik;TFII%`W4Lv2*Z3tJJEuc4nc0q8Bup3)zQr-=xyltM>$A=15KXnjZ5PL{IvAuK%@?dJF_0T5= zC|IJui(i32a@W?wBtII$Kzmedm#&v5>0NL(Jh6*Cl&c4clEq>9m)m z;uSUeTDrU%RMkl)06ZA-0jTm3J?k;+GC&8f(HoSBz^s-R2)1nHKm!31txBPE%cJ4Y z?ND{BBxM(58X3Ryj2p{79gm^XOWF(Im<-2v(r)AxLa0EqHdAO(z@ZdSKEI#gxkWVM zhyWd^-d~QZTa=XJp!}!Osdy$XojUiJgp;}REU%l~DHh{Oc!M!3v%a48xp{(EJ<9~!~mk$#D;q5D&{YwGkFdMsLevrbRiXE#?!jRp>FY0%thfi6g z+H9cHQQP%-7Ej_7#G;GGL-uu(!Bh>khbJ9*)F7#LgTR>|3RP8Lu%z0lK=@RZloul< zVb*QALsxGdgR?pt%zqXMo5)k*swjGL8`2>;yvC10yF2+(y#p3-qwIlnh`oV39biHz z*#-phVM-nF6XY!@jCZ!e8X{8bS|B4oj1=UCm%x9ae1f6bnNVyH-apucHg33bbl(C< zl)7Tx>an|EB9R7;1{6zx4Z5-%;Q>ND$&ZR3T-*eF8mHXmef4s^Jq>iGi_k9LIkDa zqB=D2e(`!mZ~CBJV*dn%&l|`+z}f?58+`zZufQN@o2+-l_bN-kpklIXR|*xguH=2h zw7a3IbD#_qfPMGk)JVZ4!9}C!zCa&`BuU3LPo}Pz#$L25nH<n26>{+1kmHAyrpaS)$Uay z`MW6*Z*q=1t*MoGT2NZY(}#s75ez=}%T2kz0P}l|F|ME42DZb?=d3 zT|2HFpavJ-*rhVtEY=Ucb(C${wc1FW8iE89D$+&0B^+v6!r%4D{y_PNP-~nd^T))X z1kuUr3q?`Fk7AkBt&WV=h#h>M#GF7=fMnMq)|<{?4)^Is zOEdd5Rn@gU21#^XEWv{6itqm*eE%>0og6{Zf;>YwMgRAYNpJj^@%=a9{kO$_7{t81 zoD5B&ti;4i*3+g76l;k-QZR&Wm*No#n^a*XSy3kBDPk^3mHH-HVVhtyNh$aC=7Bxdj z;Q*^n%Z`=jnvYOn8W)A~9SH-ZFLx*@WOK%7uYtxwjj+ zmF(}g3bL@4eo~rAm7rt|-%;lsAo9_Ky5v4M!H^c0SI=M(P5n#xM7Ay0xoYj4kESpe znLT=m#@1G9%IJwnSYDHhat*gYp#*Ra7b-&rYm4>S80si_p~a3h zkQfFKx2L5jQlXhS)RwLox+-2m@>TAPx0o5foDHa^N>FtcavRe@bCeI5l)?>nNt&J4 zI73zj?aiaQ)b+Y`2M};RfPiw5S^)Lx@Z6ba48i(6ux6HA-A*vu)`$S)m#a`+;BV6w zkfWn+1f-o_c;^Dz02HF75m@WEyUc|mQxc$i%bEc71*5R69;74zLO6Pp)X=la_L_84 zPJN)rQYhy(T{u99pwnspKxzsr>9ui3B?1^sklX>7631*h?wPOvI08T>#~lTje`Hv8 zGAPFkPE#?}uAoq<%Vej>I~W?v2Md3?sKWbLWB{&0m)7T~u>>wJo<$ zFO~ZQlpX-t43~h0U1f`kDnr?Fd>@`Ok0eIHIRUnN4(nH6WLieuTGau~C#}-mo)WVc z#i`jZtizbRdWWAnW3#!neGr)cEI=R6{ye<@2;xeL>HGZsEBZ=&_WntD`z-?XH^BSr zm;7&j{vY3e1dv})BjbN~`&F^4Bu%a$WusY^Fm85+0XT4qbcEa`vy^<=6o+2Aa|nKc z3$GLst2M1K1IrN~d~1AG&ynT5ly7&RXgyR4YbaF00=8cH$1wTd!S!CKsx%^TzzA?M zO!qn?;6px)Lp)Ve+9V0UL=I+?oa&=?KV%D_x|TBqmy1-7b(bZG62N;Z$1J;-Ja8*k zdT%N;*JsOVS<#ns5WpHA8Uv{}fP{3FCA-i_0;U;q6LQ|XbfAhu8PZaQdl>K!auxNI z>R@OoG0%D+@{r1}%{lb|U|D&Ck{#k}nJ5vv=rng$bfHdBn3JeMIws(Bn2Kd=R93_7 z9q~SAyVA?y5IXPpxVn?pHDskJPXPiUg{mO+9dqYkA_iL3@;><0GxFq~V^_II*bqC< zJ|tA_OiZA0qhI2|W`>Vrcx{ZF018AFhm4LZXn+`G9s-)W?J+R8#ENvNE}Sudw(LiU zVOc1gWK+7xzx^c)rR5Ma;9-+VpQwzagjG&G)IpNm$1Y0vUbAiTLOTsez06D>n5vH9 zvK515Lz3PtV5sHPVU$XZFv|AZzkmB-7-?H50eU@#&I3en$X zA66Lf(#4uc{BcZ{HR=Q11z!$venBS%kGYp?M)p{&Tri!q_*#Mc6JCo@l;7*iR zNbqa8#Ee_X=>i5c%u-;+M0;U4;t|XV_TlXK1Q-WP=7YSW8;L|j*Si5~5MU_8zaAM<~R?B(QXs)496iy*Z4&0Q-HfmH;uUm2*E7N zXfGcg&Z(Wnlyb08OAx z;Uk-@wsaqkLgxw4J+J9AJe%U7a{Ku&z?8=*ge!o;#kMqEsNHXLTkL*^a@L^ck?}C! zO4v6s+j92yI=1MFrz2T`Octb|mr9b97bn}Tx(=9eJPG9mWkD@Hh(|=Dv!{^K1x!Wi z2f*OMxDe|#jq?@z?#OaZ{W0U9v*tuk3?*C5hb0C~@OUzT3NkW-nSw3jMV+Ml^>pCr zH(6mJw@FS`(eu$y2Af}XCY1`p^h@#=HB1~@0wW&C80}at2pyjwj=+6<0p*ZM#KYj7 z5@n^@#%!`W)G1YTACqJ`Qk3O^ppdtR1vv~TH^=DMz% z_pdlK0$4H(d9DDFe@GIeUR_mPk5jBi_pKfTaFk&N$;?0|kx7cCNRd?ZpiNOSaC!B= zve!O)?W*rHgbboM->s^9c^}^8sr?1@J#a&+`*>iuy3z5t9qXV2PBnxxQtohESIe`ee^Eu>$DUGB$hg61Tp?_N`h)z3 zeIydR%2wWlI@0z4mKf&n2!XUsV#?Y}ky7RCi&D}62@?^?4~hyjP_Sb^C6jV_%J{1lkdM3f zl5#4_V_is5o|#)46hKZ?Q@}pJ?&v&?;pyLaN?^|8%P|#!z^{%@pFk>;Ko96DRA7PH zA-dp#O^yVbjxJ<=nqng8bA>0;CObzQ&)uDdK{Kq>Il^v^Es2P5v?29 zyfh~iLKq`=<_pNPWRHBnaYMWI)=Zx#ho(;i%!JpEFW9hcy7>ZS zfzN>-3nou9cc@1U`%`1X+_Lb|vbFKZdhAAZOW;xj$`2o5Sd9o#g5OHw6yyjaEuOTF z)nn)iD8}4sHCl*uwH|^B0F%LjQ`ZQ19j^=k1EcaVTWVp{eDZv3Di2v*W$l6e)>uZlvt zDxmQkaZ7-D0SDx^P=6*=|D4W7w<(L7LTsR72%IlRc$ux3XS$Rb-b?b!hFn%|0Ri-C zlp9y+5_PohqWUnAl7V36YC2?~){D?$xNzPs6rK}?=$N}Lg91{e3qn znUyhr+sL2j!p-;*;7nrCw#@RI4n2`3W2Y+S45sEx*U+XnpMq?-X+U)=+ut=B(2%FX zu)zkLK3V6riT6PNg6m62H|ue)g1iI@OXow3)zjMmdKa`s9wjodL1*@skYLF`SEM$j z9}y(rrjdUD7cg^adjSXGmQ4Z-rt(TGpqAU0>rJIK8n&5cgw|3R+8Is8Fy@TSmKVCj z#+3W0w8UUxLq^BZxG&X2WS~AWp3I*kTzzuS_Z*MNIwd|FLg-?f2k5)x#-Mm{#!P|b zL~<%UNNG3^C>zvp@m*synBWG@5jcdBrGHKq9{j8Bg6R^H2NAt&t2W%XX71#$(*~iZ zTA1UgB1lYuv%;w{1=<0pIIZOOn|9t!r{>mo&SwB3O|Q9E*xLp7a!9gOJ}ib_RI0?o z=Lq}B=mB z2lv()obAtSgI9%x0|H$G1?JdfC52ddxE0=1W@F@mF!-;--`U5% zOw@=UgdhEF+Ot1H5U3?xz-;li$#e0yyz-B4zYBkvPX2o!P2#7opI}v@r^9o@%fTWG z4hr&nrd#W&PL-7(3oc6CpnVDlc^-p*6LfGUs-n%Qt!dlwP#6LR#;E(5`DaOGNJxr3 z59|y8h29-tA7M)08L6a)YUQqQI3nyq_6)m!73oUJ35z>{E`8NbQa&2MfZ~9?2lZ%3k>c`8u!$ki^rB&) z=^Us*a4<1G!an=phK`N8sgKS)UosuMXN*Wi4Z{BGVuT_Fnr1gG^6gt?QaZItMe6x^ zjcgR~@o2zVXNv<>Wbg+hq|Hn`Te7GqSlvf0=SairmJ%-v* zor&>P&vYYy2-blaMLxnx-bY+fWWytH=aoDWDLt+WTuZI6*-lYHpr&n12&F6cWJ zz9|iUc~E6Bp%=ip@{~4FI}UV!o#bVfeV?>cc+9i|$2wCpXAMca%A5nFpEfq0$|d2v7g0w$o`4!IRujkE{;+bgTi|n;DPXtFb{i zz3P(~5jg_>*4ca+PezHu-f3L+80N!K5-f`aD>Y))D{Z2?cu4+W78mMaUA8mntYGFP z;d44K1V%a(14BzgKb~}ws-bdyZl|>mfC2!`E1G_jpU`bp(e!T5(RmaI?*&!v3AAJq zh(?tj3Zh>95={G=p*T#3YG}fgm<~+T)|49E6f9Sw)0=9<$I3Ls9h|?7&e5TUzs0eA zH*~yxY9(?wSe{n_>_5h{ZT754WVUpweV&6^pOuT9nc4O#KF!$*5CjsK$QY3 z8$B(MOI=dRQfY%lHk*7yn`C919-T;5as}pk4-yK<$=)GiCOl1TkKEhAD_;7~a1?k;iW- z^7!rX-+e_v$XBKi`HI4jU%q|x`o;I3zkU4nEslfvMt%SJ>n8#7kDtAM8(x3*P8%e3 zOZKhz*1-c-LUCu;qraZ|9cUmJxIr>-dVG~;g87Qdj`Xvs-?0eL5zu41Hk!8)Rkg9G zA$5`!5SG&F{cbQMMU%%XoFrKpri2S)jyO$yRSr;V$)szzRp?S!UW^>|7|CdIP(EuW zX8Ht_Z14Ft%2A(Il4Dwep)8vl9Xj#I5pU6s8E)<*v%qJAed0xS9Zpm#41=1B#(cp7 zm%!k`H2`E9S$$^3_cWhEF~y_^O{o~`X5R=1vT}!Z{ak^~X}QD>j_Afy%G#vxP@oJklM?hlNZ}mYHMm3VR9U;1jLptfVVHi{JOfYnOeBnnkSz&g6AH6y5HKrvVzp+G zXKrF_3lf)IMSw*5V`Y(EElX4G1XOD7K?ZZsw$?hv`tQi6>O#D;R|B54VC!s1V-02o za%03FUOKe`F?@T}gHB`U(`K~~QxmaX4>YHOjZ8X3p0gdbVDe_GdhthCEQXPW4*3iz z_Y&MZ7eoiTUIAuTDZX>%Mp zsB%Up9vfNNFXutOW=@TDXMx}?qK$w(j3}}#7zkbSQ+HB_EVPua;dgHYl4?}9r}ZG5 zHq}jDJ{#7^8$7L)nih1dY9JY@RdCHZyT-hYM2%YISbape7QW;Z1q!MU9|5`XYKw<3 zn#cw#V|f86C>^bJe1&E-?Zi-<#EqnnaV=4^+Zl+){Y9q;%XF#Ac1&7|^-|9AUX<*$ zWTqZ4#zsU?q&+M!aeeVgiVtFx!>GZ4ejWsbv~mD?wTe#{4la6lZpwS5iQb{3KNE6S zNHHulEwnm~S~49v%kSY?3pu>Y-1TXjkwAE8KVQ_2e?0L-1!6II6K#B(TX-V4JQYv1 zi$*WX%RsngVYhCS=V3@Jdxj=cV+e*{gUJlyhDAy17Qa+8L#f8KiDOE#ED2@dc2VG| zCu1i&h{~D3^qEXkx&PABcRYMZkx^auOgGrf^KUa~E?^F=H9vT52{5ZZpWk_afSS&LVv*wL{HAZ50q#xp6;z)r$ywXV~x}0F167?j#GwN zdIh#g-Vjf%*--dLDT>%W-egV3Kk1i~TNeTH^tf2SaiRm|eE15jur`o#6g)%fQRb-0 zWGN*JJ(G0UWT_i+WtRg4?ZIV`tEnTdNNXZ_HG^+lB$*WRP;1LcsUD6<&y-!q4QW9) zd1}4xlG_(JnM@Ew+p=RJFQ$@2GPq7uIFx3m#uX2#`y2t_LYZiq@)yiV#v58zO4oM28+jBgkD z1+)cx-DoY5{uk_Hy4bi;LCj%}@6s!zd2?Nfc+!m(v4ty%cS$Jjhh z7|Wsr$^lrIW#&p5K^Dr~ony}V#+=9#;6wmDphQ6G6jh+1-rD3FAq!=XWCVG;4o0@r z|I<}onwqH~vd!mVdGKZHG$!RyfpxiOvi3dP?_tfL$gwTuw%xG?na1T7cTAX|yN}*V zgW%%P?W=9&&|BCemxdR%^8j)8)^CP<8QZqtXuIEA-P%5~FCA|%V47|3m+#L$f3pb| znl<#dCM5(M?#hGz8mqa{!N8Ppg-c=2cLcGn>T=g}H7 zuH*9r>1f@@O*UsKc9C)k>iLJ(r>w*MVSDkYVLFh8Li(02BJ*V^hHaj$jx-1Lm3iGS z!C7GFwPT;ic&5gBR=`!Ky-iC*fxWd;H_-;oI2n-YK%Ves%jEx@#mS^>&9ANwMw41p zJ9i87zij8Hav0HEut3msVh{3mK$r>veMNKtp-b=Knh}w`YW}==P*tlHSvsy{A6!$w z6_g#OQrZ98-m1C2^F$vT(6{3RLO@jTatj1bO;bC=5$ zSwBfy0&jP)%6?WxP&$UPW^~C>)=!XQSN)nzq%z38;2zPkS#}Gz{Bp}*ltg>4XbjQI z%b_sOzNXN=ui8+nNVEnaG`UEB@WBV+ua2$!TdCOL;_&w6**p3@YNmbsBRQMjy)PX7 z4FSQoZzs_3tGD0i)z?qnKJ}}wACuqd`j-Fw`ppkx>(bWy94Vj9~gw*}Kt08S`SR)FIWXKyKNh&<-R zMZhk`@)<>|f7J;pRg0-S=YhYhaiwold5utjqcRIYI+KYLXV%9(sn zStIc#YRuo79JYpR#z-HBu#i}vglNnu*B^&T61==L3NVxxTJm8usY8TSVsp-i@z?olHYf<2fW!nvqF%vHd{20&J+ zO6zpW4zex;(eS12uqN~Q@8JHWN4M=Je>Z)y;!;fw$_B!t4Y=Qq*HbGfkwE;<^_cJD zLfr%?QteD0p&=Z2XcT}nkw2R(HL$y0+tnxc0))b0(><%5XM0FxM&|D}nka#oU$B># zocvKVC}u-7(7Y%weuc&bBYiM?PC?*;lj!UXg^4QU-ak*Ll^{8zj^;-Yxh5h&9~-8t zjw+sr6_;D~NfIf{uW7x3$Eu81%rQ2Y8@DrAV{k+%;O9zraCFHx>SlV%Y%5u)YNWW% zf}LQuHYA}InOfDj4GKx@%MlqEqIT-jg+>ZiXx{`G^$+9+)!C8!VQJ$kD0X!LD9nAv zwuvB1D9}e{VW48;_`S?p6$4pMWC5w)eMh9$1rU=zO7K4+QJ+|*xZ-j=fT}$CNo)8w zxEk3)eazNl9?r(fyhboXWUGs`BGw8Z+|_K?zpO&ux!ZGtHQHWs->+c9saDw54$F}y z?lQyt;6vx&>>m5F#(d>&*C8)!WL~%?CEC-iUf%n=Sx|D#A{%p^U_&`KGBhm zIg4pP2r!qO=~!-_Sf1_(^wlFBR|^=X%NJC=yS0c^rHPtqma;HiYLRWld64D+E6i}9 zt|_MM0)hbH*h*uONVs`~JZDJyr~yf!y~uA^Ru-F1mCv|A;4|b9^e|C)QxR?M@h|H= z(At^K&H@kx;zf8XGR{Qq1H*icvJv_O<8(%UsD7l7Y9~!Z@=F;LN)cFJ8F7Z1Wt^Z# zDgVboCo)T!Q?6NeI&}h!IE2aqRSWVIc}pmq2?tIZ^(FT)eJt)_n@aC+#QeG~#gQ+7q@wZ? zF)=aVDNn6UWI)MNE9-QY8z>o3_vyIEPQ%U6mRuBg9x}PD+~TW8i4DpZr%IJqIfK9< z$B07#A7YM=YRg%tZidr$IWiV8Mw(-+8Jq$exp^`|YgL~!9DDqDOoFJX@8J9NNPjJU zUk_PSl?=^9!H=l~^^_U8+=$!)zk*YJoMS9_<-O95MUUpc^L?b0Va&KMzD*p^ARp`m z8eePZVCHEs#-axZaCUHv4}C$B{$f{!TV*!y2-?!DV77Te)TD4yG6dGta-TPB0c%DH z5Q_jCW&qYbhU+Kupt=vmC5h(xuoc87F|H@k$+buRuH&O?0l_A3wre&X)^I9wK}e3@ zd)13PjdOkigHZYS)LAN;KJ0@Ahl9JlR4;No!zW-RK$Ly5EYYE^2?Gz$ z>AK72QTtcP^Jyky)zhi9)e=1|T>CbHHb4{gX(L)Rw&fKFux0ll`G?xlt$iBWwIw@n zI*Nef37=u4UrEW*uxlfm;{hO`A9RmH%Mm{$M<5c;EjrLZQiMUd5{c*snYUf?YO?>N zt&_$Q^mWbenX?v$=V1%6Gnhm}NQaoIt49ooa%T)BRkm0(( zC1lhPaw)Ft`^x_C*5ICSVjrt*H;>A=UDRY= zQI3HC^fe7JXUF99sT<4OMVRRU*JaqhOffg zmvA=!G`xLw*>|f|cd4H*kJ(#x1`CwaLHPx#O!0f4u^Dr(;IoifdAKc5 zOql9)gKkXzYx|G38Qk2#hVa{@oIO4WKvXt2s;;Sz1Ik@Zw(2H(COnZlrkc;P0YiSY z(+^gzM6Dosn%xmLUXvY;4iX-GR_++@9M;d6EO|Ev_Kj$FuP43H8e14+by7_a2)JxG zky5*~1-L=>4&nx|0aQjjDsc&c3YibF1l{)E|F_p)!T)4E(hinKdtLQj^nsAd+Lo!p zvri5SjEF|LLTwIQhCtq4ox#0~^1@&{xqvbMAb?nDA(8CAJd}N@w6zWIApsktPmVyJic@h`6X9YVdggXzX(<=Yl>k&@#cvYdrhi8ID@02Z>MXQOYAbD53d zJb9ep6)QKKGFz_lFrC{KI$a61hoh=S=#i4rWnI6hZCV*+0bH3?qPI30I>UC+cDiu@ zM%hT}mXYp8CKXt58DU7R`Gqz?8mmWoNRCL9VGg&rH28SE3<>im8jhii71<*YG#YN0 z5uIf$eqQQQAkez%i5-(OIe=#Zb%Z;Jgwe?|DyMyD2Aj`Ype$Pi8~Kj}{~TsB&0&=) zB}-31`w$*i2z40q*~{x;OQrrb%V9Yv&pA7f3O}TXn6lpdW>5`}N~xB2aQB-Fv{_qm zo#c7-=FX0CfG44%)<+(1 za7(VXRJ98_`+{21Sv(z&aHwMQXBBV^BgMlmr0vXNUFz6WDyijpF8}O^Y?IqWKuqCO zQ^hj)Tf84MLyzJCrX!@0Qw19gYh4LgdYGyRf$11wp~3~JW!>D)AD)@X4mh3ajK_mi zFsxl8gvRR11=NITyy3*FtS&&Ag0J9d5{kBf3K(~SwOw9dwLIYEuE6$|Z(`G}G8kv0 zd{_}F@1uVAlhto2_&aln0=oHib8t)1-ox4jKB}ws%j^-xr+cia7b+gipo)D%NC{FV z+=8+c-u!e`KbpHmEE;Xdz{82L4?sY&=d{dF0T}T$MD8Fb2p^#-Ve_6LC^49N10szA zZ*8Q;=4!qm3D0xTb&Yc|FJ#2>ZI$7FcSdUihbBz4*Z`R^DWoe}4I>7_4j*cB(#^{u z{3l)+oCYqh1L>^pYVmjWzrX{%`hgCbU;~n9s=rh<4NORXf5eLa+fRM`{$T2#-@c(K z$sf*O$i+%MGNHjX@q4x66L-m=aNq$5c-|nv`-+8%S#e;!QY1@7Yk+EQ~WtdQVeN<_RQEW zSzuxihFJkbSloUCS{6N$TF}zmFW`O8Sx1nAtV`89dmXinML4Y0?NXLl@7C?ekMXyf znIXUO#7bM+X~IgY0t^EjNeqIl2Y0$t@@%usvT}k=8Ag*Xm<}i_y2FieRZrqY9_ES{ zq-Zo^J~9v2BeY>zkyfR~YzU9BYv&laWrEGS!4r@xpRWJWWx>g`bVtURDVJ|33p=^G z)j5Gmb6lkW<@5^KriW1)S{~qGXqF2+*ucVt<3)l#=prH>aZcP4>u{)=iP?cLrlBiQ zx`3(46Jvc1-?3|Bgi4^Z`gX4JRE6dT%$VaC5wdqospPAzK`Y=B>yvw+C`eh76FIK{ zeE@laWV1|A2DdEhvGY9u-8$kV1(>^p@uxky1Dap{0H{6iL z7$1o*1p#M$aIT;QRvXHuo$R6QnjsKasvdfXtPO%GA?fZ(&aE5IZ-wN>dOC0rv#%)B)RTnV6h#zILK`2T z9Qi7|@J{+lqPx6)Olr&*Z{NVo?3Zs}P(t(bw_m>g47RS{ynb{^R%icku_UZ8*+(9QM!$U|;4Da`2!a)-l1I)Wp(frvW-@_Nq21vtarBVS$ncX2?U6T6>qRY^WU*6(;T4;0Lga0k4pDrCsZw(S~YRM&(VLXIpTr0HGxa zk#vDoD{dvzBCT}Q;qZYU#+DB5nvptJ>L7An1eZI~muMh@IlVw8IA?Ci00rfT0dZk; zyt|A6caPY zm*AnWmAud$r!m-2z^o&OMt+qxIg|AA98V9(SqV8>UG+8cHHH!}BOaousYg>c+S8t# z(7;%ew|S;J8u}K~qC)=4+T;nF#CKkB)eD)Os=R4nPA`riVy9rUd!z+mjQfbBhz~cq zrH{%>kTv^I)xu4T@BpgZ@x)ykt7tqG4|4@b!LmS0C)7>`Uvg-_;Um~E@-PM3r$c$; z?1vyeLjKM3%czF#2|~D))u7D&ewLqOhrQ2E1}FWbkR&U+|3%i8JT-D}44%`Y<)T^NHjxta+=7Wv zn5SN)GVO)~C^+@+%&moo2>(`1K%f%1U5e_k?=7oDiKm+$+^W+gFCn?q#9Jcw zGHYja+eTwM7J}&;DWBG0(s#QWa$3d4SzfwHJL%?{f<6_)XrYX19sP{xxZA^!#|O0M z*}k}0%BEA>$3>mg~moPij&)nN;93z~^=9W_6QlonBp~q5hJLa~6r64Fl%~|iQ zbMm)dwL6+J&}?w*?o-FXq44xq60E{utau5O@gCUT1*b`S*8y*;7VesPNBij66 zJ3@11a`!&P5Qv-r==*a`k-uj!mQNRTw16s*osLNmR-SkzkXwBJ(?9*xfPes$7R=g5 zNW-UewJm}Lh(f*Qgb+1$UL15UuF_rdZKah8^j2Umlf1lSSqEw60(?xI>w`M0j|Y>`eQ=M`^ob^h2ORmKmeE2>H>a~L-wFGAYZ0c@!lLP`*q3WU)j*+olhZt;nI8CZ-D)hXI# zU615Qo!*=?U=U4k^>~=tvts%U&FUxVvFLGsXb4R1j zp7?{6M?yhyyC^^yNw95{SZu=;L7cWn$42V9Jzli6FK7{CKwHHdeGILSP?gb*0~=_p zU2faSHY{)Y)Sj%|N2xnuNQU3eG&$;qq55?+$DA1|KlZvGEddDH=!S+Ima4-pT5Bgp zw#%88;s|VqW?rWju+rC#)&-I!+bretyRox0B?B?RC(n>-!1TTJ25gCT4wG2)@vgKP zFwqefBL{g?|pJsPSnw0FnY6*!CW)l|#ym0yL-+~QIQ zsE|@#Zq#R_-$ew8-1aqD!*`U|Fy6>^6&9s^=T499EIBdAq7IrQBfc?0Xlu_upX4Ph zl1w%uDPilg5=urr(lIr$eB^B=u^^!7!N4)olt_h+Tqin=}CJ3^=ce>rJKRIwLU2d!XRk z7M@zZ>VkaMv7mWqt$Qf>4p6dcHlpp6qp|0(Ro-F}%Bq7(I`jxu0XeEBUsGxURvW40 z0SkYO*ZeTgoO)3qDoe^}qs^vq!^w)YVr=uW2P$lqt;By}*SCC7H)L=gu5jqUEV6FV zIgqV|d#Qe74*sywPFqEhY2L{Mupg~q>f0bW|ejfGK4!Rnpv#6Tebc}ao-KbMh znmIX{wZI?yM0h;xz;Y7cOWH@G3Uve}4G2jtte9T}@2S^~aTs3I;X*F3hN$i;@hJo| zAap>!F~umMda<$^t9~_*p@~|d=9ED#TY0kRD@ydR2O4yiO8N!0Nq$KSf(0rksCb-~ za5Uxx2CZ&js;)Gplu|_#)y%`bt^v=YooQJfMlcn;5(fInvqlvqLDs1`>uFYsm?Geg zA(So#*-DMb?VlVZXc`Pw0xz`Vj-VZRB=2$z=xzULGJFe*y@h#IqRV^=63#QZDG>U6{w&MZ6+_oyicq8*OhetbJUakStDc55_(EsYMhEJ}kPc-L z7bsuKqQ_9z5p-L-QQ>TwX7=e6cBZll%*ycSP4dZch@?bp=mxpbxH3WnrnXMGe+Zh@bEU+8l2Ud0{YsYex z2L;`RyvpZDr@|FhoY8zxH4Bz-0PXooqne1InmCUqC&I)Tw#KW5(!s;q^t_li5?)wG zZq1NQ*K+`|s??&gh!!Nu-!_=D0xUXpa+hx<-%~9m-B4k9$kM(iA#Yy6fiMjF!i;KJ z*MQ6Th%B>vcP&%1hZ@>mwDy|}+3oB#xb^lTIJAR%s@c&XZKXJ9UyCee8|eLZb`KdH zR8iGlp*|@B%z~L#k12Gk?F95U27yiX+$l=}#q!)Bb21v~T&EN3v5?K#4)=nSQef6p zrDN&>8&l8^(n(mZX{DKvB-!ewYtRF(qa4Mfi2$KN>y%Dx8P+l=yrqTA6>VRoTrisk z$dc@VH)E!5jj|oeqHMrvbTg&_ZBgxTmQF?JWZ}$-0@g-qab#9yEXCz-&24Hf1#DCy zk9F&PC+Otf$*lv2KM*ugBP_~`e4VRZ4pQP_#(h~{(?TtzJscx#7^<@+x;)6Tcob-A`8NaJCxaO-Ub#8Bh zm0gH29nnW?#JtLAB{=b@xD}j2Yh1{FLG2ul1@TEGnO4y4>Qdo9ld##F@WIrUSv@~&8#x&Qn{T_T8v`kvG>lqSPX&q}{FW*OYA4J?1@?Oc8 z!deg7(ICH9*0gzel9M2(z&Wu%RW+_uc!au;e<+0nL!Hf#xd=BKBP81cveVVEWpuZX zc2RWzra;bD(HX;$E-k0tlAJSGj@f%JFAoAHX3{EQS3YHnpCWogAdvtD(5utcehVh3 zL}0x}whkTvkNb*&Z!h@yQZ$(~Rc_h#+ESP*s3Vi|NH_V(@WX1Fno)blrc!OYE?E>G zY;t**US0%l+(DbPhDeq!c~ao82}JJFh{DV}hb=UAE5YFiBU?o*pj=ja8ku`|^Du^T zv=&IBAe@m4Ad{o8p}=iZu$=*R;L`7hCk%4o&N31Y&qWxD8u?!O?S+nsJ-U-Sf0De| zlN*R*9-(wW`=k3vrJU6M@*S>-X+Adf9_F3h?pBmtT%QBw5&^D)p*!rx0j~4+B&K-x!m`(R(I$RM_R^9gD;O zy9Ij%)E=@uP;l#hS`f&S5RNFJlW9tmI#{S}$yRRUJ`;cwp9Eb!iZ=TpiMnaM6y)@z zqqxR4v3dFKkj)8K4`l-g$U?vCcbKd|n*owBgt5yuIebG$L^i|VcZ@o{rlTRtTPgR! z`^iQan_kf!NiDqe6-9I~P;PeT*%|(3flug;AA{5~Zn?&*X^06gGAk~n1gC4r)o^b! zxbU`WV@iRR5|vQjj%^u%vb&IjS9JAHA;uu9#XKCofbv#W0WW9aE8$aWZd3^ zkv|Tpx8P=y-?{1AVy%-as2^p|>P5|l1eo%Rv7=OHgtg7;rdr6qD$ugr+ZD}qhv5K$ z8E;ycJXQotM-dOT>?r?(lg7|$Pn_Ld*q|E7QRT5L$NwSx2ufGI_!pcRy-$aDN96j= zpZgg8Ute(1(A#e?H~Pcd|4HBT^Ve@e(vf`k`ir0+K2M+d-*W!;M})!2-DD5c6Luh; z&nd7}UMneR^z5-8VMmX+KLaDdY={(L;7uR=q!L^JdevUT%r(^USm1}?wN?s=5zUnj zY)?e&l&!KJ7?Qm{9oz^onJR=eE4x$ncZ9+rm&jtR(^m;%fJab@hH+o!wegH%?RUTc zLy+mP!{xyKOV}`7TZGA19xR0Kzjc5t3}0V1xDMn#c3(DSYZJu)k8nnl`3P&7jyq?O z{Xq@lO0qrU2KyD>v`YlBVuN7hTG1>C{ur`|p^b<}XFSqvTPy%_{|_A5Y`K~&wXkKE zI@oQ`s?wi&spcpzTuyCRZ4f~ zZV;U~5CZ?t+KL~-W{I?k-G?aC7V6qXB=Tp=zp`+;_gMxnc)W*4+Q`(zWzw<1LUyOM zfHY-A_AbB|Fo`!xSVp-w%tr&3;nan|Mm-(Y;c*OTjt~=c$l(|Rt|TG?mgp*wd4g@Z z=``e(82zZLoCyiJJ!>zikt*)xcQUC*c^9%8ZiFS(L%rNFd>~%eF2tXUK7E8wcZ*$jD6 zhnX3s2e3o5QWpbspf_<8U3RXg6ztWm) zIp*718PfV4k94*o)E1TsD6_K8$2dhB9_gJb8jZNrwbqx~qDe8P;9K4+RcT)hd|)cX ztT$*_hqiRLj#3T6(+M06$|#HDPBClAUoFSt9~oE|K-P7E>*}T+s12EcD-Y8_HP3^K zvGP#%0x`h8D`3Fq%`w$n-E2Jp)2K=-hsN_Q_>dn=mZpVWSII>dv&fLWGaSw|SIrKa zR$deQ3Sk@3Dzx`1%Qpl~uy2#gZU{Ks*KeQO z>5qS)2k&cHet%s38pM}$@Hqg@r^v9Ck3ZLdmrve)8*J+I1RL+78CDwpFi$gUxgA`f z^!UEWU8)T!M5XB!HTY@^o&L}a{!mlhBW67&%ehClVY;^v-9ysj&zr`C^ftt~Yi#So zXFb?5(~+9*TH|C^&c)y-!&Di#A(KVEr2!)K*~s9&+77529&9rh>vMqu1@el)#fGF#-J$0#LxXfSUtCiL%q! z=a;Qv(pD-IGwF(!MFv^W5H8&F5gr;IuBS9lusos6vP(gJw}EoiBo=22XvVmOb4yHV zPzkOx`E1CdJT^G!#nV$Y>q@p!%?iWB`pB!pqA0Jo)PvKqL+h5VXJ|gm3@x7MB@g9m zh!^u3MR~~)PQ+V~j%Xmmoy4d6c(?uyYW=V(buY-@{4G;r)R@Ox9wU_3t?Xwlm{(8`R&-Du!eD-Xtv3DAj-N(Pb2c?>jey=`Zm#l9!6=_< z*hARToSnZ_F;Mjvcq#Oxz$7cOTO)Y4bf(JSk3eH48&$2m;d; zWYif$1@2~WoViq>WYhU0L6-*h^J{!C`~rW3j|NKQ7J$P@_!4d#Em(EKK^F)cF(#9+{3#NC(bY)*)L?BQ&sZU1ZXrS2b!_-Sdf-+Msu` z(Q-?IVhp&H$S$yqQ!Wn-^Az7{Ze3SvH5OE{SXZF7fyr17)kE*WysPJfXv<}YM)Gtk zDCiw3lsN`BP@^Ht!;Q?ssTJdLl)1>tHUq!aThy|pv1^b`$%=F=PBsHsBDWQ2*+63* zPj4T0|y6o7q1kx*GcL1Gcwwiww{{0X2;4i}W|J9f%AAI2Ggdg>~yVbwvZ1&yz zACIp-zJ(*6Ur_b(9aS&K*S~)Hr@s&X*2W}#{+DmReEm8kx$pO%zx^x!yU*S}4{txe zBoF;F*XB3$4*40P^X$9)52$Var?GIs*w z#1EQgIxm5P>~s+&y=vs;K6`o(DbV5zbR2@Li1`6pMwNzU7}CJJPPQv}s}?Fe-z*U} z8K=I#nmI`H9)Q|W&&RPTmtJAL!!`cNKMu5$}9czj&fu9a1;y9wyr0x{y zYPR>R+tv(L*|flyLd5%KNu0C9E8WlX#G!88=wnSAV>&@sOg9NnI9;Y-b4R%3n~@=` z?m3%SdN>rBY5LN&h=Op!aB4CYhtj8!oqTGm;!qOH@`ik$(d+u<$|n3$$#QQ{4uF)C zH{|NbE1LCwp>n!7LWkW3OQpkuyQWPSH9NYhV{5;V!Y1#<^=fT5xml3T0~l8$!eUAC z9<0aQ&Sld|V^>!)rYnKN$~lKB2w}(&rRpp(U6)11kR8}V0A*}Nfmk!QxxH)$;^V2( z$o;cuTU%c%ZoT`xW1MNP^CXp}MHRQ5TXH3-GSp-SB#}L8GrFlTJ?fN*?yz^GA>R~8 z-4;OuGbtfTlC{E}+SYu1J{Srw-@@HM!)#*CO9+E#yN1*8fz@mK)+~`c5`eRS` zSt-i_>l&(xcKjii0g<~XaV~@Gb@L__89YP@PLpApyBf`@|YdsOE7trlSZ|qFgniO+RK{8-ox4FOVb+$Vs zzJdl^?UyTJEd?D87+7deaKaj}v9(q8pa7Trg|FarqZyUXP$evM+DCPljxaLnL4Jns z3Vr}-(yhid7{dd#u-t>kA}MceCjfy==&cBx?g22$V`pcO!?&Fr)H+Gu&P$4b$O(!* zr#Qo>$7Y8W1|4JTAx6+(0a<1qT09Mq8R#dlS0&V9c}|iT9sEpxy7=+K)XTyHRkpGMGdyX$o^xF+wY|{ z>?p-c&^6<Ilj#}*J9cO%Xh?M#GvVlQEo^6a?hSzaZUIJ?;>(s3^2QJ$SgNH>{}E$ z9YCm)O|mT8%QU?+rI_ML8ZE!vn%NsLl@p9gDvFO)j3)1!0h7mJ28;zJ5%=tLTIDt( ziD=d&-x>))AY_}Lifnj>c{L}f+9i0b%=9Tq%8;=_2Mh*4Qg+G>NPjlj;s!yp zIf*Vkh&?A`S$U>BT%bp4t4pKDK*cn$!`vc!DEfd7U8#+dCV9Hj%VRbs7(drrrD@6X znqJyH8&M!+p=JN_i68GkBWW|(! zuEnK(*wd$``P4+A<(k7WaA{D6nH5`u;jo$NH?VLyuvIM^aA~u-x^kFRXNwY~VT0dT z0EC8FfTj2A}$0d#Tvck&c87FeG$ucQE>4N6bhi=*`|H1p5?`8gK|23t|Qk z8|Dbi&}#S3+DwocwWIRNO01fsMQ?=`c|6Mn#dgFAtC~4pH=+ZL8auw&vkeU#RC(Sa zzj+^HPlgd5&GukmKLtWaF?d(cKybt?MQ9US; ztxONk0vixS8S>(hzAbw(#0>;C#ks|_ha+((8DqrW_Ad=hHlbPG7JCqoQYc=?dg}X> z=@SQQ4dhuRpq4|p(g#`?$z1JCMe_-s9ZCq=j%b8zcK3ZHG1D=BRYyW{d_f?@sE3m= z&|HmgkeI;Mhm!J^35$8OxjG^dfASN<0q3^NpF2JSh2Ov5RXTanenuCT&t8oT^u^mh zg}2{*uTwPnR^NXf-ae+l{Il@(HKu7_1qjU1RQlcj|G(kSeL{bx0Jc0m-n()VY;oHw zIzVB?O@_32+^NgTKrF&CdZi=_#;fQd&@66bKGZ$rBa(EUG?_*5Y5sg0q!@<>3)C3x zDHjQ08Yct5byJU)RZx}jpJ%P1PloKt@FZl9Bj8yPOMy-m){oxJYiEr>nJ)qwWT{Qp zj_mW)R04a=pMh1%Dv(aj1Q1nfowhih;WqMv}`9*1Gi{T$c5f9??B^HHhv{z(}TVD%q4)no;HK6^>A)q~X hSpdDrox+^4cgglrz7rRzDPZO0{{oPT@#kp3FaTBF64n3! literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/python/clip.py b/crates/stable-diffusion-burn/python/clip.py new file mode 100644 index 0000000..da3a649 --- /dev/null +++ b/crates/stable-diffusion-burn/python/clip.py @@ -0,0 +1,40 @@ +import pathlib +import save +from save import * + +def save_clipmlp(clip_mlp, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_linear(clip_mlp.fc1, pathlib.Path(path, 'fc1')) + save_linear(clip_mlp.fc2, pathlib.Path(path, 'fc2')) + +def save_clip_attention(clip_attention, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_linear(clip_attention.k_proj, pathlib.Path(path, 'key')) + save_linear(clip_attention.v_proj, pathlib.Path(path, 'value')) + save_linear(clip_attention.q_proj, pathlib.Path(path, 'query')) + save_linear(clip_attention.out_proj, pathlib.Path(path, 'out')) + save_scalar(clip_attention.num_heads, 'n_head', path) + +def save_clip_encoder_layer(clip_encoder_layer, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_clip_attention(clip_encoder_layer.self_attn, pathlib.Path(path, 'attn')) + save_layer_norm(clip_encoder_layer.layer_norm1, pathlib.Path(path, 'attn_ln')) + save_clipmlp(clip_encoder_layer.mlp, pathlib.Path(path, 'mlp')) + save_layer_norm(clip_encoder_layer.layer_norm2, pathlib.Path(path, 'mlp_ln')) + +def save_clip_encoder(clip_encoder, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + for i, layer in enumerate(clip_encoder.layers): + save_clip_encoder_layer(layer, pathlib.Path(path, f'blocks/{i}')) + save_scalar(len(clip_encoder.layers), "n_layer", path) + +def save_clip_text_embeddings(clip_text_embeddings, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_embedding(clip_text_embeddings.token_embedding, pathlib.Path(path, 'token_embedding')) + save_embedding(clip_text_embeddings.position_embedding, pathlib.Path(path, 'position_embedding')) + +def save_clip_text_transformer(clip_text_transformer, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_clip_text_embeddings(clip_text_transformer.embeddings, path) + save_clip_encoder(clip_text_transformer.encoder, path) + save_layer_norm(clip_text_transformer.final_layer_norm, pathlib.Path(path, 'layer_norm')) \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/dump.py b/crates/stable-diffusion-burn/python/dump.py new file mode 100644 index 0000000..66a994a --- /dev/null +++ b/crates/stable-diffusion-burn/python/dump.py @@ -0,0 +1,652 @@ +# This code is modified from the tinygrad stable diffusion example +# (https://github.com/tinygrad/tinygrad/blob/master/examples/stable_diffusion.py) +# used under the MIT license. + +# https://arxiv.org/pdf/2112.10752.pdf +# https://github.com/ekagra-ranjan/huggingface-blog/blob/main/stable_diffusion.md +import os +import tempfile +from pathlib import Path +import gzip, argparse, math, re +from functools import lru_cache +from collections import namedtuple + +from tqdm import tqdm +from tinygrad.tensor import Tensor +from tinygrad.helpers import GlobalCounters +from tinygrad import dtypes +from tinygrad.nn import Conv2d, Linear, GroupNorm, LayerNorm, Embedding +#from extra.utils import download_file +from tinygrad.nn.state import torch_load, load_state_dict + +# TODO: refactor AttnBlock, CrossAttention, CLIPAttention to share code + +class AttnBlock: + def __init__(self, in_channels): + self.norm = GroupNorm(32, in_channels) + self.q = Conv2d(in_channels, in_channels, 1) + self.k = Conv2d(in_channels, in_channels, 1) + self.v = Conv2d(in_channels, in_channels, 1) + self.proj_out = Conv2d(in_channels, in_channels, 1) + + # copied from AttnBlock in ldm repo + def __call__(self, x): + h_ = self.norm(x) + q,k,v = self.q(h_), self.k(h_), self.v(h_) + + # compute attention + b,c,h,w = q.shape + q = q.reshape(b,c,h*w) + q = q.permute(0,2,1) # b,hw,c + k = k.reshape(b,c,h*w) # b,c,hw + w_ = q @ k + w_ = w_ * (c**(-0.5)) + w_ = w_.softmax() + + # attend to values + v = v.reshape(b,c,h*w) + w_ = w_.permute(0,2,1) + h_ = v @ w_ + h_ = h_.reshape(b,c,h,w) + + return x + self.proj_out(h_) + +class ResnetBlock: + def __init__(self, in_channels, out_channels=None): + self.norm1 = GroupNorm(32, in_channels) + self.conv1 = Conv2d(in_channels, out_channels, 3, padding=1) + self.norm2 = GroupNorm(32, out_channels) + self.conv2 = Conv2d(out_channels, out_channels, 3, padding=1) + self.nin_shortcut = Conv2d(in_channels, out_channels, 1) if in_channels != out_channels else lambda x: x + + def __call__(self, x): + h = self.conv1(self.norm1(x).swish()) + h = self.conv2(self.norm2(h).swish()) + return self.nin_shortcut(x) + h + +class Mid: + def __init__(self, block_in): + self.block_1 = ResnetBlock(block_in, block_in) + self.attn_1 = AttnBlock(block_in) + self.block_2 = ResnetBlock(block_in, block_in) + + def __call__(self, x): + return x.sequential([self.block_1, self.attn_1, self.block_2]) + +class Decoder: + def __init__(self): + sz = [(128, 256), (256, 512), (512, 512), (512, 512)] + self.conv_in = Conv2d(4,512,3, padding=1) + self.mid = Mid(512) + + arr = [] + for i,s in enumerate(sz): + arr.append({"block": + [ResnetBlock(s[1], s[0]), + ResnetBlock(s[0], s[0]), + ResnetBlock(s[0], s[0])]}) + if i != 0: arr[-1]['upsample'] = {"conv": Conv2d(s[0], s[0], 3, padding=1)} + self.up = arr + + self.norm_out = GroupNorm(32, 128) + self.conv_out = Conv2d(128, 3, 3, padding=1) + + def __call__(self, x): + x = self.conv_in(x) + x = self.mid(x) + + for l in self.up[::-1]: + for b in l['block']: + x = b(x) + if 'upsample' in l: + # https://pytorch.org/docs/stable/generated/torch.nn.functional.interpolate.html ? + bs,c,py,px = x.shape + x = x.reshape(bs, c, py, 1, px, 1).expand(bs, c, py, 2, px, 2).reshape(bs, c, py*2, px*2) + x = l['upsample']['conv'](x) + x.realize() + + return self.conv_out(self.norm_out(x).swish()) + +class Encoder: + def __init__(self): + sz = [(128, 128), (128, 256), (256, 512), (512, 512)] + self.conv_in = Conv2d(3,128,3, padding=1) + + arr = [] + for i,s in enumerate(sz): + arr.append({"block": + [ResnetBlock(s[0], s[1]), + ResnetBlock(s[1], s[1])]}) + if i != 3: arr[-1]['downsample'] = {"conv": Conv2d(s[1], s[1], 3, stride=2, padding=(0,1,0,1))} + self.down = arr + + self.mid = Mid(512) + self.norm_out = GroupNorm(32, 512) + self.conv_out = Conv2d(512, 8, 3, padding=1) + + def __call__(self, x): + x = self.conv_in(x) + + for i, l in enumerate(self.down): + for b in l['block']: x = b(x) + if 'downsample' in l: x = l['downsample']['conv'](x) + + x = self.mid(x) + return self.conv_out(self.norm_out(x).swish()) + +class AutoencoderKL: + def __init__(self): + self.encoder = Encoder() + self.decoder = Decoder() + self.quant_conv = Conv2d(8, 8, 1) + self.post_quant_conv = Conv2d(4, 4, 1) + + def __call__(self, x): + latent = self.encoder(x) + latent = self.quant_conv(latent) + latent = latent[:, 0:4] # only the means + latent = self.post_quant_conv(latent) + return self.decoder(latent) + +# not to be confused with ResnetBlock +class ResBlock: + def __init__(self, channels, emb_channels, out_channels): + self.in_layers = [ + GroupNorm(32, channels), + Tensor.silu, + Conv2d(channels, out_channels, 3, padding=1) + ] + self.emb_layers = [ + Tensor.silu, + Linear(emb_channels, out_channels) + ] + self.out_layers = [ + GroupNorm(32, out_channels), + Tensor.silu, + lambda x: x, # needed for weights loading code to work + Conv2d(out_channels, out_channels, 3, padding=1) + ] + self.skip_connection = Conv2d(channels, out_channels, 1) if channels != out_channels else lambda x: x + + def __call__(self, x, emb): + h = x.sequential(self.in_layers) + emb_out = emb.sequential(self.emb_layers) + h = h + emb_out.reshape(*emb_out.shape, 1, 1) + h = h.sequential(self.out_layers) + ret = self.skip_connection(x) + h + return ret + +class CrossAttention: + def __init__(self, query_dim, context_dim, n_heads, d_head): + self.to_q = Linear(query_dim, n_heads*d_head, bias=False) + self.to_k = Linear(context_dim, n_heads*d_head, bias=False) + self.to_v = Linear(context_dim, n_heads*d_head, bias=False) + self.scale = d_head ** -0.5 + self.num_heads = n_heads + self.head_size = d_head + self.to_out = [Linear(n_heads*d_head, query_dim)] + + def __call__(self, x, context=None): + context = x if context is None else context + q,k,v = self.to_q(x), self.to_k(context), self.to_v(context) + q = q.reshape(x.shape[0], -1, self.num_heads, self.head_size).permute(0,2,1,3) # (bs, num_heads, time, head_size) + k = k.reshape(x.shape[0], -1, self.num_heads, self.head_size).permute(0,2,3,1) # (bs, num_heads, head_size, time) + v = v.reshape(x.shape[0], -1, self.num_heads, self.head_size).permute(0,2,1,3) # (bs, num_heads, time, head_size) + + score = q.dot(k) * self.scale + weights = score.softmax() # (bs, num_heads, time, time) + attention = weights.dot(v).permute(0,2,1,3) # (bs, time, num_heads, head_size) + + h_ = attention.reshape(shape=(x.shape[0], -1, self.num_heads * self.head_size)) + return h_.sequential(self.to_out) + +class GEGLU: + def __init__(self, dim_in, dim_out): + self.proj = Linear(dim_in, dim_out * 2) + self.dim_out = dim_out + + def __call__(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * gate.gelu() + +class FeedForward: + def __init__(self, dim, mult=4): + self.net = [ + GEGLU(dim, dim*mult), + lambda x: x, # needed for weights loading code to work + Linear(dim*mult, dim) + ] + + def __call__(self, x): + return x.sequential(self.net) + +class BasicTransformerBlock: + def __init__(self, dim, context_dim, n_heads, d_head): + self.attn1 = CrossAttention(dim, dim, n_heads, d_head) + self.ff = FeedForward(dim) + self.attn2 = CrossAttention(dim, context_dim, n_heads, d_head) + self.norm1 = LayerNorm(dim) + self.norm2 = LayerNorm(dim) + self.norm3 = LayerNorm(dim) + + def __call__(self, x, context=None): + x = self.attn1(self.norm1(x)) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + +class SpatialTransformer: + def __init__(self, channels, context_dim, n_heads, d_head): + self.norm = GroupNorm(32, channels) + assert channels == n_heads * d_head + self.proj_in = Conv2d(channels, n_heads * d_head, 1) + self.transformer_blocks = [BasicTransformerBlock(channels, context_dim, n_heads, d_head)] + self.proj_out = Conv2d(n_heads * d_head, channels, 1) + + def __call__(self, x, context=None): + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + x = self.proj_in(x) + x = x.reshape(b, c, h*w).permute(0,2,1) + for block in self.transformer_blocks: + x = block(x, context=context) + x = x.permute(0,2,1).reshape(b, c, h, w) + ret = self.proj_out(x) + x_in + return ret + +class Downsample: + def __init__(self, channels): + self.op = Conv2d(channels, channels, 3, stride=2, padding=1) + + def __call__(self, x): + return self.op(x) + +class Upsample: + def __init__(self, channels): + self.conv = Conv2d(channels, channels, 3, padding=1) + + def __call__(self, x): + bs,c,py,px = x.shape + x = x.reshape(bs, c, py, 1, px, 1).expand(bs, c, py, 2, px, 2).reshape(bs, c, py*2, px*2) + return self.conv(x) + +def timestep_embedding(timesteps, dim, max_period=10000): + half = dim // 2 + freqs = (-math.log(max_period) * Tensor.arange(half) / half).exp() + args = timesteps * freqs + return Tensor.cat(args.cos(), args.sin()).reshape(1, -1) + +class UNetModel: + def __init__(self): + self.time_embed = [ + Linear(320, 1280), + Tensor.silu, + Linear(1280, 1280), + ] + self.input_blocks = [ + [Conv2d(4, 320, kernel_size=3, padding=1)], + [ResBlock(320, 1280, 320), SpatialTransformer(320, 768, 8, 40)], + [ResBlock(320, 1280, 320), SpatialTransformer(320, 768, 8, 40)], + [Downsample(320)], + [ResBlock(320, 1280, 640), SpatialTransformer(640, 768, 8, 80)], + [ResBlock(640, 1280, 640), SpatialTransformer(640, 768, 8, 80)], + [Downsample(640)], + [ResBlock(640, 1280, 1280), SpatialTransformer(1280, 768, 8, 160)], + [ResBlock(1280, 1280, 1280), SpatialTransformer(1280, 768, 8, 160)], + [Downsample(1280)], + [ResBlock(1280, 1280, 1280)], + [ResBlock(1280, 1280, 1280)] + ] + self.middle_block = [ + ResBlock(1280, 1280, 1280), + SpatialTransformer(1280, 768, 8, 160), + ResBlock(1280, 1280, 1280) + ] + self.output_blocks = [ + [ResBlock(2560, 1280, 1280)], + [ResBlock(2560, 1280, 1280)], + [ResBlock(2560, 1280, 1280), Upsample(1280)], + [ResBlock(2560, 1280, 1280), SpatialTransformer(1280, 768, 8, 160)], + [ResBlock(2560, 1280, 1280), SpatialTransformer(1280, 768, 8, 160)], + [ResBlock(1920, 1280, 1280), SpatialTransformer(1280, 768, 8, 160), Upsample(1280)], + [ResBlock(1920, 1280, 640), SpatialTransformer(640, 768, 8, 80)], # 6 + [ResBlock(1280, 1280, 640), SpatialTransformer(640, 768, 8, 80)], + [ResBlock(960, 1280, 640), SpatialTransformer(640, 768, 8, 80), Upsample(640)], + [ResBlock(960, 1280, 320), SpatialTransformer(320, 768, 8, 40)], + [ResBlock(640, 1280, 320), SpatialTransformer(320, 768, 8, 40)], + [ResBlock(640, 1280, 320), SpatialTransformer(320, 768, 8, 40)], + ] + self.out = [ + GroupNorm(32, 320), + Tensor.silu, + Conv2d(320, 4, kernel_size=3, padding=1) + ] + + def __call__(self, x, timesteps=None, context=None): + # TODO: real time embedding + t_emb = timestep_embedding(timesteps, 320) + emb = t_emb.sequential(self.time_embed) + + + + def run(x, bb): + if isinstance(bb, ResBlock): x = bb(x, emb) + elif isinstance(bb, SpatialTransformer): x = bb(x, context) + else: x = bb(x) + return x + + saved_inputs = [] + for i,b in enumerate(self.input_blocks): + for bb in b: + x = run(x, bb) + saved_inputs.append(x) + for bb in self.middle_block: + x = run(x, bb) + for i,b in enumerate(self.output_blocks): + x = x.cat(saved_inputs.pop(), dim=1) + for bb in b: + x = run(x, bb) + return x.sequential(self.out) + +class CLIPMLP: + def __init__(self): + self.fc1 = Linear(768, 3072) + self.fc2 = Linear(3072, 768) + + def __call__(self, hidden_states): + hidden_states = self.fc1(hidden_states) + hidden_states = hidden_states.quick_gelu() + hidden_states = self.fc2(hidden_states) + return hidden_states + +class CLIPAttention: + def __init__(self): + self.embed_dim = 768 + self.num_heads = 12 + self.head_dim = self.embed_dim // self.num_heads + self.scale = self.head_dim**-0.5 + self.k_proj = Linear(self.embed_dim, self.embed_dim) + self.v_proj = Linear(self.embed_dim, self.embed_dim) + self.q_proj = Linear(self.embed_dim, self.embed_dim) + self.out_proj = Linear(self.embed_dim, self.embed_dim) + + def _shape(self, tensor, seq_len: int, bsz: int): + return tensor.reshape(bsz, seq_len, self.num_heads, self.head_dim).permute(0,2,1,3) + + def __call__(self, hidden_states, causal_attention_mask): + bsz, tgt_len, embed_dim = hidden_states.shape + + query_states = self.q_proj(hidden_states) * self.scale + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).reshape(*proj_shape) + key_states = key_states.reshape(*proj_shape) + src_len = key_states.shape[1] + value_states = value_states.reshape(*proj_shape) + + attn_weights = query_states @ key_states.permute(0,2,1) + + attn_weights = attn_weights.reshape(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = attn_weights.reshape(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = attn_weights.softmax() + + attn_output = attn_weights @ value_states + + attn_output = attn_output.reshape(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.permute(0,2,1,3) + attn_output = attn_output.reshape(bsz, tgt_len, embed_dim) + + attn_output = self.out_proj(attn_output) + return attn_output + +class CLIPEncoderLayer: + def __init__(self): + self.self_attn = CLIPAttention() + self.layer_norm1 = LayerNorm(768) + self.mlp = CLIPMLP() + self.layer_norm2 = LayerNorm(768) + + def __call__(self, hidden_states, causal_attention_mask): + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn(hidden_states, causal_attention_mask) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + +class CLIPEncoder: + def __init__(self): + self.layers = [CLIPEncoderLayer() for i in range(12)] + + def __call__(self, hidden_states, causal_attention_mask): + for l in self.layers: + hidden_states = l(hidden_states, causal_attention_mask) + return hidden_states + +class CLIPTextEmbeddings: + def __init__(self): + self.token_embedding = Embedding(49408, 768) + self.position_embedding = Embedding(77, 768) + + def __call__(self, input_ids, position_ids): + return self.token_embedding(input_ids) + self.position_embedding(position_ids) + +class CLIPTextTransformer: + def __init__(self): + self.embeddings = CLIPTextEmbeddings() + self.encoder = CLIPEncoder() + self.final_layer_norm = LayerNorm(768) + + def __call__(self, input_ids): + seq_len = input_ids.shape[1] + x = self.embeddings(input_ids, Tensor.arange(seq_len).reshape(1, -1)) + mask = Tensor.full((1, 1, seq_len, seq_len), float("-inf")).triu(1) + x = self.encoder(x, mask) + return self.final_layer_norm(x) + +# Clip tokenizer, taken from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py (MIT license) +@lru_cache() +def default_bpe(): + return Path(__file__).parent.parent / "weights/bpe_simple_vocab_16e6.txt.gz" + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +class ClipTokenizer: + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') + merges = merges[1:49152-256-2+1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v+'' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} + self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[^\s]+""", re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + ( token[-1] + '',) + pairs = get_pairs(word) + + if not pairs: + return token+'' + + while True: + bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word)-1 and word[i+1] == second: + new_word.append(first+second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(text.strip()).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + # Truncation, keeping two slots for start and end tokens. + if len(bpe_tokens) > 75: + bpe_tokens = bpe_tokens[:75] + return [49406] + bpe_tokens + [49407] * (77 - len(bpe_tokens) - 1) + +class StableDiffusion: + def __init__(self): + self.alphas_cumprod = Tensor.empty(1000) + self.model = namedtuple("DiffusionModel", ["diffusion_model"])(diffusion_model = UNetModel()) + self.first_stage_model = AutoencoderKL() + self.cond_stage_model = namedtuple("CondStageModel", ["transformer"])(transformer = namedtuple("Transformer", ["text_model"])(text_model = CLIPTextTransformer())) + + # TODO: make __call__ run the model + +# ** ldm.models.autoencoder.AutoencoderKL (done!) +# 3x512x512 <--> 4x64x64 (16384) +# decode torch.Size([1, 4, 64, 64]) torch.Size([1, 3, 512, 512]) +# section 4.3 of paper +# first_stage_model.encoder, first_stage_model.decoder + +# ** ldm.modules.diffusionmodules.openaimodel.UNetModel +# this is what runs each time to sample. is this the LDM? +# input: 4x64x64 +# output: 4x64x64 +# model.diffusion_model +# it has attention? + +# ** ldm.modules.encoders.modules.FrozenCLIPEmbedder +# cond_stage_model.transformer.text_model + +# this is sd-v1-4.ckpt +FILENAME = Path(__file__).parent.parent / "weights/sd-v1-4.ckpt" + +import sys +import clip as clipsave +import autoencoder as autoencodersave +import unet as unetsave +import stablediffusion as sdsave + +import numpy as np + +if __name__ == "__main__": + Tensor.no_grad = True + '''clip = CLIPTextTransformer() + + print('Saving model...') + clipsave.save_clip_text_transformer(clip, "params") + + input = Tensor([3, 1]) + output = clip(input.unsqueeze(0)) + + print(output[0, 0:2, 0:10].numpy())''' + + '''autoencoder = AutoencoderKL() + print('Saving model...') + autoencodersave.save_autoencoder(autoencoder, "params") + input = Tensor.zeros((1, 3, 10, 10)) + output = autoencoder(input) + print(output.shape) + print(output.numpy())''' + + '''unet = UNetModel() + print('Saving model...') + unetsave.save_unet_model(unet, 'params') + input = Tensor.zeros([1, 4, 64, 64]) + + context = np.array([0.5, 1.3], dtype=np.float32) # specify dtype when defining the array + context = np.repeat(context, 768 // 2) + context = np.expand_dims(context, axis=0) + context = Tensor(context) + + timesteps = Tensor([1.0]) + + output = unet(input, timesteps, context) + #print(output.numpy())''' + + if len(sys.argv) != 2: + print(f"Wrong command line parameters, Usage: python3 {sys.argv[0]} ") + sys.exit() + + FILENAME = sys.argv[1] + + Tensor.no_grad = True + model = StableDiffusion() + + # load in weights + #download_file('https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt', FILENAME) + load_state_dict(model, torch_load(FILENAME)['state_dict'], strict=False) + + print('Dumping model...') + sdsave.save_stable_diffusion(model, "params") + print('Model weights saved in params.') + diff --git a/crates/stable-diffusion-burn/python/requirements.txt b/crates/stable-diffusion-burn/python/requirements.txt new file mode 100644 index 0000000..d1c7ebd --- /dev/null +++ b/crates/stable-diffusion-burn/python/requirements.txt @@ -0,0 +1 @@ +tinygrad==0.9.2 \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/save.py b/crates/stable-diffusion-burn/python/save.py new file mode 100644 index 0000000..7e414f9 --- /dev/null +++ b/crates/stable-diffusion-burn/python/save.py @@ -0,0 +1,100 @@ +import pathlib +import numpy as np + +from tinygrad.tensor import Tensor + +def save_scalar(s, name, path): + s = np.array([1.0, float(s)]).astype(np.float32) + np.save(pathlib.Path(path, f'{name}.npy'), s) + +def save_tensor(tensor, name, path): + tensor_numpy = tensor.numpy() + tensor_dims = np.array(tensor_numpy.shape) + tensor_values = tensor_numpy.flatten() + tensor_to_save = np.concatenate((tensor_dims, tensor_values)).astype(np.float32) + np.save(pathlib.Path(path, f'{name}.npy'), tensor_to_save) + +def save_linear(linear, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_tensor(linear.weight.transpose(), 'weight', path) # PyTorch and Tinygrad strangely transpose linear weights so reverse that + if linear.bias is not None: + save_tensor(linear.bias, 'bias', path) + +def save_layer_norm(layer_norm, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_tensor(layer_norm.weight, 'weight', path) + save_tensor(layer_norm.bias, 'bias', path) + save_scalar(layer_norm.eps, 'eps', path) + +def save_group_norm(layer_norm, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + if layer_norm.weight is not None: + save_tensor(layer_norm.weight, 'weight', path) + if layer_norm.bias is not None: + save_tensor(layer_norm.bias, 'bias', path) + save_scalar(layer_norm.eps, 'eps', path) + save_scalar(layer_norm.num_groups, 'n_group', path) + save_scalar(layer_norm.num_channels, 'n_channel', path) + +def to_tuple_tensor(val): + if isinstance(val, tuple): + # Convert tuple to Tensor + if len(val) == 1: + return Tensor([val[0], val[0]]) + elif len(val) == 2: + return Tensor([val[0], val[1]]) + else: + raise ValueError('Tuple should be of length 1 or 2 only.') + else: + # Treat as scalar and convert to Tensor + return Tensor([val, val]) + +def save_conv2d(conv2d, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_tensor(conv2d.weight, 'weight', path) + if conv2d.bias is not None: + save_tensor(conv2d.bias, 'bias', path) + save_tensor(to_tuple_tensor(conv2d.stride), 'stride', path) + save_tensor(to_tuple_tensor(conv2d.padding), 'padding', path) + save_tensor(to_tuple_tensor(conv2d.dilation), 'dilation', path) + save_scalar(conv2d.groups, "n_group", path) + save_tensor(to_tuple_tensor(conv2d.kernel_size), 'kernel_size', path) + + assert conv2d.groups == 1 + in_channels = conv2d.weight.shape[1] + out_channels = conv2d.weight.shape[0] + save_scalar(in_channels, "n_channels_in", path) + save_scalar(out_channels, "n_channels_out", path) + +def save_padded_conv2d(padded_conv2d, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Store conv2d layer weights + orig_padding = padded_conv2d.padding + padded_conv2d.padding = (0, 0) + save_conv2d(padded_conv2d, f"{path}/conv") + padded_conv2d.padding = orig_padding + + # Dimensions: in-channels and out-channels + assert padded_conv2d.groups == 1 + channels = (padded_conv2d.weight.shape[1], padded_conv2d.weight.shape[0]) + save_tensor(to_tuple_tensor(channels), 'channels', path) + + assert len(padded_conv2d.kernel_size) == 1 or padded_conv2d.kernel_size[0] == padded_conv2d.kernel_size[1] + save_scalar(padded_conv2d.kernel_size[0], 'kernel_size', path) + + # Stride + assert not isinstance(padded_conv2d.stride, tuple) or len(padded_conv2d.stride) == 1 + save_scalar(padded_conv2d.stride, 'stride', path) + + # Padding + padding = [padded_conv2d.padding[0], padded_conv2d.padding[1], + padded_conv2d.padding[2], padded_conv2d.padding[3]] + save_tensor(Tensor(padding), 'padding', path) + + +def save_embedding(embedding, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_tensor(embedding.weight, 'weight', path) + diff --git a/crates/stable-diffusion-burn/python/stablediffusion.py b/crates/stable-diffusion-burn/python/stablediffusion.py new file mode 100644 index 0000000..fec6528 --- /dev/null +++ b/crates/stable-diffusion-burn/python/stablediffusion.py @@ -0,0 +1,14 @@ +import pathlib +from autoencoder import save_autoencoder +from unet import save_unet_model +from clip import save_clip_text_transformer + +from save import save_scalar, save_tensor + +def save_stable_diffusion(stable_diffusion, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_scalar(stable_diffusion.alphas_cumprod.shape[0], "n_steps", path) + save_tensor(stable_diffusion.alphas_cumprod, 'alphas_cumprod', path) + save_autoencoder(stable_diffusion.first_stage_model, pathlib.Path(path, 'autoencoder')) + save_unet_model(stable_diffusion.model.diffusion_model, pathlib.Path(path, 'unet')) + save_clip_text_transformer(stable_diffusion.cond_stage_model.transformer.text_model, pathlib.Path(path, 'clip')) \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/test.py b/crates/stable-diffusion-burn/python/test.py new file mode 100644 index 0000000..03f601f --- /dev/null +++ b/crates/stable-diffusion-burn/python/test.py @@ -0,0 +1,54 @@ +import torch +import torch.nn as nn +from torch import Tensor +import math + +'''import torch +import torch.nn as nn + +import torch + +norm = torch.nn.LayerNorm(3) + +tensor = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).reshape((2, 3)) + +out = norm(tensor) + +print(out)''' + +'''n_channel = 6 +norm = nn.LayerNorm(10) + +height = 10 +width = 10 +n_elements = height * width * n_channel + +t = torch.arange(0, n_elements, dtype=torch.float32).mul_(10.0 / n_elements).sin().reshape(1, n_channel, height, width) + +out = norm(t) +print(out)''' + +def timestep_embedding(timesteps, dim, max_period=10000): + half = dim // 2 + freqs = (-math.log(max_period) * torch.arange(half) / half).exp() + args = timesteps * freqs + return torch.cat( (args.cos(), args.sin()) ).reshape(1, -1) + +timesteps = Tensor([1, 2, 3]).reshape((3, 1)) +dim = 10 +res = timestep_embedding(timesteps, dim) + +print(res) + +'''n_group = 3 +n_channel = 6 +norm = nn.GroupNorm(n_group, n_channel) + +height = 10 +width = 10 +n_elements = height * width * n_channel + +t = torch.arange(0, n_elements, dtype=torch.float32).mul_(10.0 / n_elements).sin().reshape(1, n_channel, height, width) + +out = norm(t) +print(out.flatten())''' \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/test_tiny.py b/crates/stable-diffusion-burn/python/test_tiny.py new file mode 100644 index 0000000..0fa72fe --- /dev/null +++ b/crates/stable-diffusion-burn/python/test_tiny.py @@ -0,0 +1,41 @@ +from tinygrad.tensor import Tensor +from tinygrad.nn import Conv2d, Linear, GroupNorm, LayerNorm, Embedding +import math + +'''import torch +import torch.nn as nn + +import torch + +norm = torch.nn.LayerNorm(3) + +tensor = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).reshape((2, 3)) + +out = norm(tensor) + +print(out)''' + +n_channel = 6 +norm = LayerNorm(10) + +height = 10 +width = 10 +n_elements = height * width * n_channel + +t = Tensor.arange(n_elements).mul(10.0 / n_elements).sin().reshape(1, n_channel, height, width) + +out = norm(t) +print(out.numpy()) + +'''n_group = 3 +n_channel = 6 +norm = nn.GroupNorm(n_group, n_channel) + +height = 10 +width = 10 +n_elements = height * width * n_channel + +t = torch.arange(0, n_elements, dtype=torch.float32).mul_(10.0 / n_elements).sin().reshape(1, n_channel, height, width) + +out = norm(t) +print(out.flatten())''' \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/tokenizer.py b/crates/stable-diffusion-burn/python/tokenizer.py new file mode 100644 index 0000000..c99d59c --- /dev/null +++ b/crates/stable-diffusion-burn/python/tokenizer.py @@ -0,0 +1,145 @@ +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def default_bpe(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz") + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') + merges = merges[1:49152-256-2+1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v+'' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|startoftext|>'} + self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + ( token[-1] + '',) + pairs = get_pairs(word) + + if not pairs: + return token+'' + + while True: + bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word)-1 and word[i+1] == second: + new_word.append(first+second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') + return text + +if __name__ == "__main__": + simple_tokenizer = SimpleTokenizer() + #print([simple_tokenizer.byte_encoder[0], simple_tokenizer.byte_encoder[1]]) + tokens = [0, 1, 59, 67, 23] + text = simple_tokenizer.decode(tokens) + print(f"Text: {text}") + + text = "Hello world! <|startoftext|>asdf<|startoftext|>" + encoded = simple_tokenizer.encode(text) + decoded = simple_tokenizer.decode(encoded) + print(f"Encoded: {encoded}") + print(f"Decoded: {decoded}") \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/unet.py b/crates/stable-diffusion-burn/python/unet.py new file mode 100644 index 0000000..556d8a8 --- /dev/null +++ b/crates/stable-diffusion-burn/python/unet.py @@ -0,0 +1,153 @@ +import pathlib +import os +import save +from save import * + +from tinygrad.nn import Conv2d + +def save_res_block(res_block, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + # We can't directly save activation functions, but as they are just attribute of the block, + # we don't need to save them separately, they will be recreated along with the block. + + # saving group normalization layer + save_group_norm(res_block.in_layers[0], os.path.join(path, 'norm_in')) + + # saving the convolutional layer + save_conv2d(res_block.in_layers[2], os.path.join(path, 'conv_in')) + + # saving the linear layer + save_linear(res_block.emb_layers[1], os.path.join(path, 'lin_embed')) + + # saving group normalization in out_layers + save_group_norm(res_block.out_layers[0], os.path.join(path, 'norm_out')) + + # saving the convolutional layer in out_layers + save_conv2d(res_block.out_layers[3], os.path.join(path, 'conv_out')) + + # save skip_connection based on the object type + if isinstance(res_block.skip_connection, Conv2d): + save_conv2d(res_block.skip_connection, os.path.join(path, 'skip_connection')) + +def save_cross_attention(cross_attention, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save Linear layers + save_linear(cross_attention.to_q, os.path.join(path, 'query')) + save_linear(cross_attention.to_k, os.path.join(path, 'key')) + save_linear(cross_attention.to_v, os.path.join(path, 'value')) + save_linear(cross_attention.to_out[0], os.path.join(path, 'out')) + + # Save parameters + save_scalar(cross_attention.num_heads, 'n_head', path) + +def save_geglu(geglu, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save Linear layers + save_linear(geglu.proj, os.path.join(path, 'proj')) + +def save_feed_forward(feed_forward, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save GEGLU module + save_geglu(feed_forward.net[0], os.path.join(path, 'geglu')) + + # Save Linear layer + save_linear(feed_forward.net[2], os.path.join(path, 'lin')) + + +def save_basic_transformer_block(basic_transformer_block, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save CrossAttention, FeedForward and LayerNorm instances + save_cross_attention(basic_transformer_block.attn1, os.path.join(path, 'attn1')) + save_feed_forward(basic_transformer_block.ff, os.path.join(path, 'mlp')) + save_cross_attention(basic_transformer_block.attn2, os.path.join(path, 'attn2')) + + save_layer_norm(basic_transformer_block.norm1, os.path.join(path, 'norm1')) + save_layer_norm(basic_transformer_block.norm2, os.path.join(path, 'norm2')) + save_layer_norm(basic_transformer_block.norm3, os.path.join(path, 'norm3')) + + + +def save_spatial_transformer(spatial_transformer, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save GroupNorm, Conv2d, BasicTransformerBlock instances + save_group_norm(spatial_transformer.norm, os.path.join(path, 'norm')) + save_conv2d(spatial_transformer.proj_in, os.path.join(path, 'proj_in')) + save_basic_transformer_block(spatial_transformer.transformer_blocks[0], os.path.join(path, 'transformer')) + save_conv2d(spatial_transformer.proj_out, os.path.join(path, 'proj_out')) + + +def save_downsample(downsample, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save Conv2d instance + save_conv2d(downsample.op, path) + +def save_upsample(upsample, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + # Save Conv2d instance + save_conv2d(upsample.conv, os.path.join(path, 'conv')) + + +def save_res_transformer_res(block, path): + save_res_block(block[0], pathlib.Path(path, 'res1')) + save_spatial_transformer(block[1], pathlib.Path(path, 'transformer')) + save_res_block(block[2], pathlib.Path(path, 'res2')) + +def save_res_upsample(block, path): + save_res_block(block[0], pathlib.Path(path, 'res')) + save_upsample(block[1], pathlib.Path(path, 'upsample')) + +def save_res_transformer(block, path): + save_res_block(block[0], pathlib.Path(path, 'res')) + save_spatial_transformer(block[1], pathlib.Path(path, 'transformer')) + +def save_res_transformer_upsample(block, path): + save_res_block(block[0], pathlib.Path(path, 'res')) + save_spatial_transformer(block[1], pathlib.Path(path, 'transformer')) + save_upsample(block[2], pathlib.Path(path, 'upsample')) + + +def save_unet_input_blocks(input_blocks, path): + save_conv2d(input_blocks[0][0], pathlib.Path(path, 'conv')) + save_res_transformer(input_blocks[1], pathlib.Path(path, 'rt1')) + save_res_transformer(input_blocks[2], pathlib.Path(path, 'rt2')) + save_downsample(input_blocks[3][0], pathlib.Path(path, 'd1')) + save_res_transformer(input_blocks[4], pathlib.Path(path, 'rt3')) + save_res_transformer(input_blocks[5], pathlib.Path(path, 'rt4')) + save_downsample(input_blocks[6][0], pathlib.Path(path, 'd2')) + save_res_transformer(input_blocks[7], pathlib.Path(path, 'rt5')) + save_res_transformer(input_blocks[8], pathlib.Path(path, 'rt6')) + save_downsample(input_blocks[9][0], pathlib.Path(path, 'd3')) + save_res_block(input_blocks[10][0], pathlib.Path(path, 'r1')) + save_res_block(input_blocks[11][0], pathlib.Path(path, 'r2')) + +def save_unet_output_blocks(output_blocks, path): + save_res_block(output_blocks[0][0], pathlib.Path(path, 'r1')) + save_res_block(output_blocks[1][0], pathlib.Path(path, 'r2')) + save_res_upsample(output_blocks[2], pathlib.Path(path, 'ru')) + save_res_transformer(output_blocks[3], pathlib.Path(path, 'rt1')) + save_res_transformer(output_blocks[4], pathlib.Path(path, 'rt2')) + save_res_transformer_upsample(output_blocks[5], pathlib.Path(path, 'rtu1')) + save_res_transformer(output_blocks[6], pathlib.Path(path, 'rt3')) + save_res_transformer(output_blocks[7], pathlib.Path(path, 'rt4')) + save_res_transformer_upsample(output_blocks[8], pathlib.Path(path, 'rtu2')) + save_res_transformer(output_blocks[9], pathlib.Path(path, 'rt5')) + save_res_transformer(output_blocks[10], pathlib.Path(path, 'rt6')) + save_res_transformer(output_blocks[11], pathlib.Path(path, 'rt7')) + +def save_unet_model(model, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + save_linear(model.time_embed[0], pathlib.Path(path, 'lin1_time_embed')) + save_linear(model.time_embed[2], pathlib.Path(path, 'lin2_time_embed')) + save_unet_input_blocks(model.input_blocks, pathlib.Path(path, 'input_blocks')) + save_res_transformer_res(model.middle_block, pathlib.Path(path, 'middle_block')) + save_unet_output_blocks(model.output_blocks, pathlib.Path(path, 'output_blocks')) + save_group_norm(model.out[0], pathlib.Path(path, 'norm_out')) + save_conv2d(model.out[2], pathlib.Path(path, 'conv_out')) + diff --git a/crates/stable-diffusion-burn/src/backend.rs b/crates/stable-diffusion-burn/src/backend.rs new file mode 100644 index 0000000..ff41e40 --- /dev/null +++ b/crates/stable-diffusion-burn/src/backend.rs @@ -0,0 +1,139 @@ +use burn::tensor::{activation::softmax, Tensor}; +use burn::prelude::Backend; + +/*pub type FloatTensor = ::TensorPrimitive; + +pub trait Backend: burn::tensor::backend::Backend { + fn qkv_attention( + q: FloatTensor, + k: FloatTensor, + v: FloatTensor, + mask: Option>, + n_head: usize, + ) -> FloatTensor { + qkv_attention( + Tensor::::from_primitive(q), + Tensor::from_primitive(k), + Tensor::from_primitive(v), + mask.map(|m| Tensor::from_primitive(m)), + n_head, + ) + .into_primitive() + } + + fn attn_decoder_mask(seq_length: usize, device: &Self::Device) -> FloatTensor { + attn_decoder_mask::(seq_length, device).into_primitive() + } +} + +use burn::tensor::Float; +use burn_tch::{self, TchElement, TchTensor}; +use tch; + +impl Backend for burn_tch::LibTorch { + fn qkv_attention( + q: FloatTensor, + k: FloatTensor, + v: FloatTensor, + mask: Option>, + n_head: usize, + ) -> FloatTensor { + let q = Tensor::from_primitive(q); + let k = Tensor::from_primitive(k); + let v = Tensor::from_primitive(v); + + let [n_batch, q_ctx, n_state] = q.dims(); + let [_, k_ctx, _] = k.dims(); + let n_hstate = n_state / n_head; + + let rearrange = |t: Tensor| { + let [_, n_ctx, _] = t.dims(); + t.reshape([n_batch, n_ctx, n_head, n_hstate]) + .swap_dims(1, 2) + }; + + let q = rearrange(q).into_primitive(); + let k = rearrange(k).into_primitive(); + let v = rearrange(v).into_primitive(); + + // for some reason torch crashes when mask is None + let mask = mask.unwrap_or_else(|| { + Tensor::::zeros([q_ctx, k_ctx], &Self::device(&v)) + .into_primitive() + }); + + Tensor::::from_primitive(TchTensor::new( + tch::Tensor::scaled_dot_product_attention( + &q.tensor, + &k.tensor, + &v.tensor, + Some(mask.tensor), + 0.0, + false, + None, + ), + )) + .swap_dims(1, 2) + .flatten(2, 3) + .into_primitive() + } +} + +use burn_autodiff; + +impl Backend for burn_autodiff::Autodiff {}*/ + +use std::f32::NEG_INFINITY; + +pub fn qkv_attention( + q: Tensor, + k: Tensor, + v: Tensor, + mask: Option>, + n_head: usize, +) -> Tensor { + let [n_batch, n_qctx, n_state] = q.dims(); + let [_, n_ctx, _] = k.dims(); + + let scale = (n_state as f64 / n_head as f64).powf(-0.25); + let n_hstate = n_state / n_head; + + let q = q + .reshape([n_batch, n_qctx, n_head, n_hstate]) + .swap_dims(1, 2) + * scale; + let k = k + .reshape([n_batch, n_ctx, n_head, n_hstate]) + .swap_dims(1, 2) + .transpose() + * scale; + let v = v + .reshape([n_batch, n_ctx, n_head, n_hstate]) + .swap_dims(1, 2); + + let qk = q.matmul(k); + + // apply mask + let qk = if let Some(mask) = mask { + qk + mask.slice([0..n_qctx, 0..n_ctx]).unsqueeze::<4>() + } else { + qk + }; + + // normalize value weightings + let w = softmax(qk, 3); + let o = w.matmul(v).swap_dims(1, 2).flatten(2, 3); + + return o; +} + +pub fn attn_decoder_mask(seq_length: usize, device: &B::Device) -> Tensor { + let mut mask = Tensor::::zeros([seq_length, seq_length], device); + + for i in 0..(seq_length - 1) { + let values = Tensor::::zeros([1, seq_length - (i + 1)], device).add_scalar(NEG_INFINITY); + mask = mask.slice_assign([i..i + 1, i + 1..seq_length], values); + } + + return mask; +} diff --git a/crates/stable-diffusion-burn/src/bin/convert/main.rs b/crates/stable-diffusion-burn/src/bin/convert/main.rs new file mode 100644 index 0000000..b133e33 --- /dev/null +++ b/crates/stable-diffusion-burn/src/bin/convert/main.rs @@ -0,0 +1,58 @@ +use std::env; +use std::error::Error; +use std::process; + +use stablediffusion::model::stablediffusion::{load::load_stable_diffusion, StableDiffusion}; + +use burn::{ + config::Config, + module::{Module, Param}, + nn, + tensor::{backend::Backend, Tensor}, +}; + +use burn_ndarray::{NdArray, NdArrayDevice}; + +use burn::record::{self, NamedMpkFileRecorder, FullPrecisionSettings, Recorder}; + +fn convert_dump_to_model( + dump_path: &str, + model_name: &str, + device: &B::Device, +) -> Result<(), Box> { + println!("Loading dump..."); + let model: StableDiffusion = load_stable_diffusion(dump_path, device)?; + + println!("Saving model..."); + save_model_file(model, model_name)?; + + Ok(()) +} + +fn save_model_file( + model: StableDiffusion, + name: &str, +) -> Result<(), record::RecorderError> { + NamedMpkFileRecorder::::new().record(model.into_record(), name.into()) +} + +fn main() { + type Backend = NdArray; + let device = NdArrayDevice::Cpu; + + let args: Vec = env::args().collect(); + if args.len() != 3 { + eprintln!("Usage: {} ", args[0]); + process::exit(1); + } + + let dump_path = &args[1]; + let model_name = &args[2]; + + if let Err(e) = convert_dump_to_model::(dump_path, model_name, &device) { + eprintln!("Failed to convert dump to model: {:?}", e); + process::exit(1); + } + + println!("Successfully converted {} to {}", dump_path, model_name); +} diff --git a/crates/stable-diffusion-burn/src/bin/sample/main.rs b/crates/stable-diffusion-burn/src/bin/sample/main.rs new file mode 100644 index 0000000..97cc93f --- /dev/null +++ b/crates/stable-diffusion-burn/src/bin/sample/main.rs @@ -0,0 +1,142 @@ +use stablediffusion::{ + model::stablediffusion::{load::load_stable_diffusion, *}, + tokenizer::SimpleTokenizer, +}; + +use burn::{ + config::Config, + module::{Module, Param}, + nn, + tensor::{backend::Backend, Tensor}, +}; + +cfg_if::cfg_if! { + if #[cfg(feature = "wgpu-backend")] { + use burn_wgpu::{WgpuBackend, WgpuDevice, AutoGraphicsApi}; + } else { + use burn_tch::{LibTorch, LibTorchDevice}; + } +} + +use std::env; +use std::io; +use std::process; + +use burn::record::{self, NamedMpkFileRecorder, FullPrecisionSettings, Recorder}; + +fn load_stable_diffusion_model_file( + filename: &str, + device: &B::Device, +) -> Result, record::RecorderError> { + NamedMpkFileRecorder::::new() + .load(filename.into(), device) + .map(|record| StableDiffusionConfig::new().init(device).load_record(record)) +} + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() != 7 && args.len() != 8 { + eprintln!("Usage: {} [device(cuda, mps, cpu)]", args[0]); + process::exit(1); + } + + let model_type = &args[1]; + let model_name = &args[2]; + let unconditional_guidance_scale: f64 = args[3].parse().unwrap_or_else(|_| { + eprintln!("Error: Invalid unconditional guidance scale."); + process::exit(1); + }); + let n_steps: usize = args[4].parse().unwrap_or_else(|_| { + eprintln!("Error: Invalid number of diffusion steps."); + process::exit(1); + }); + let prompt = &args[5]; + let output_image_name = &args[6]; + + // Optional device parameter + let device_arg = if args.len() == 8 { Some(&args[7]) } else { None }; + + cfg_if::cfg_if! { + if #[cfg(feature = "wgpu-backend")] { + type Backend = WgpuBackend; + let device = WgpuDevice::BestAvailable; + } else { + type Backend = LibTorch; + + let device = if let Some(dev_str) = device_arg { + match dev_str.to_lowercase().as_str() { + "cpu" => LibTorchDevice::Cpu, + "mps" => LibTorchDevice::Mps, + s if s.starts_with("cuda") => { + let idx = s[4..].parse().unwrap_or(0); + LibTorchDevice::Cuda(idx) + } + _ => { + eprintln!("Unknown device: {}", dev_str); + process::exit(1); + } + } + } else { + LibTorchDevice::Cuda(0) + }; + } + } + + println!("Loading tokenizer..."); + let tokenizer = SimpleTokenizer::new().unwrap(); + println!("Loading model..."); + let sd: StableDiffusion = if model_type == "burn" { + load_stable_diffusion_model_file(model_name, &device).unwrap_or_else(|err| { + eprintln!("Error loading model: {}", err); + process::exit(1); + }) + } else { + load_stable_diffusion(model_name, &device).unwrap_or_else(|err| { + eprintln!("Error loading model dump: {}", err); + process::exit(1); + }) + }; + + let unconditional_context = sd.unconditional_context(&tokenizer); + let context = sd.context(&tokenizer, prompt).unsqueeze::<3>(); //.repeat(0, 2); // generate 2 samples + + println!("Sampling image..."); + let images = sd.sample_image( + context, + unconditional_context, + unconditional_guidance_scale, + n_steps, + ); + save_images(&images, output_image_name, 512, 512).unwrap_or_else(|err| { + eprintln!("Error saving image: {}", err); + process::exit(1); + }); +} + +use image::{self, ColorType::Rgb8, ImageResult}; + +fn save_images(images: &Vec>, basepath: &str, width: u32, height: u32) -> ImageResult<()> { + for (index, img_data) in images.iter().enumerate() { + let path = format!("{}{}.png", basepath, index); + image::save_buffer(path, &img_data[..], width, height, Rgb8)?; + } + + Ok(()) +} + +// save red test image +fn save_test_image() -> ImageResult<()> { + let width = 256; + let height = 256; + let raw: Vec<_> = (0..width * height) + .into_iter() + .flat_map(|i| { + let row = i / width; + let red = (255.0 * row as f64 / height as f64) as u8; + + [red, 0, 0] + }) + .collect(); + + image::save_buffer("red.png", &raw[..], width, height, Rgb8) +} diff --git a/crates/stable-diffusion-burn/src/lib.rs b/crates/stable-diffusion-burn/src/lib.rs new file mode 100644 index 0000000..88a3f7f --- /dev/null +++ b/crates/stable-diffusion-burn/src/lib.rs @@ -0,0 +1,3 @@ +pub mod backend; +pub mod model; +pub mod tokenizer; diff --git a/crates/stable-diffusion-burn/src/model/attention.rs b/crates/stable-diffusion-burn/src/model/attention.rs new file mode 100644 index 0000000..f9d73c5 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/attention.rs @@ -0,0 +1,56 @@ +use burn::tensor::{activation::softmax, backend::Backend, Tensor}; + +use std::f32::NEG_INFINITY; + +pub fn qkv_attention( + q: Tensor, + k: Tensor, + v: Tensor, + mask: Option>, + n_head: usize, +) -> Tensor { + let [n_batch, n_qctx, n_state] = q.dims(); + let [_, n_ctx, _] = k.dims(); + + let scale = (n_state as f64 / n_head as f64).powf(-0.25); + let n_hstate = n_state / n_head; + + let q = q + .reshape([n_batch, n_qctx, n_head, n_hstate]) + .swap_dims(1, 2) + * scale; + let k = k + .reshape([n_batch, n_ctx, n_head, n_hstate]) + .swap_dims(1, 2) + .transpose() + * scale; + let v = v + .reshape([n_batch, n_ctx, n_head, n_hstate]) + .swap_dims(1, 2); + + let qk = q.matmul(k); + + // apply mask + let qk = if let Some(mask) = mask { + qk + mask.slice([0..n_qctx, 0..n_ctx]).unsqueeze::<4>() + } else { + qk + }; + + // normalize value weightings + let w = softmax(qk, 3); + let o = w.matmul(v).swap_dims(1, 2).flatten(2, 3); + + return o; +} + +pub fn attn_decoder_mask(seq_length: usize, device: &B::Device) -> Tensor { + let mut mask = Tensor::::zeros([seq_length, seq_length], device); + + for i in 0..(seq_length - 1) { + let values = Tensor::::zeros([1, seq_length - (i + 1)], device).add_scalar(NEG_INFINITY); + mask = mask.slice_assign([i..i + 1, i + 1..seq_length], values); + } + + return mask; +} diff --git a/crates/stable-diffusion-burn/src/model/autoencoder/load.rs b/crates/stable-diffusion-burn/src/model/autoencoder/load.rs new file mode 100644 index 0000000..bff9227 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/autoencoder/load.rs @@ -0,0 +1,195 @@ +use crate::model::load::*; + +use std::error::Error; + +use burn::{ + module::Module, + tensor::backend::Backend, +}; + +use super::*; +use crate::model::groupnorm::load::load_group_norm; + +fn load_conv_self_attention_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let norm = load_group_norm(&format!("{}/{}", path, "norm"), device)?; + let q = load_conv2d(&format!("{}/{}", path, "q"), device)?; + let k = load_conv2d(&format!("{}/{}", path, "k"), device)?; + let v = load_conv2d(&format!("{}/{}", path, "v"), device)?; + let proj_out = load_conv2d(&format!("{}/{}", path, "proj_out"), device)?; + + Ok(ConvSelfAttentionBlock { + norm, + q, + k, + v, + proj_out, + }) +} + +fn load_resnet_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let norm1 = load_group_norm(&format!("{}/{}", path, "norm1"), device)?; + let silu1 = SILU {}; + let conv1 = load_conv2d(&format!("{}/{}", path, "conv1"), device)?; + let norm2 = load_group_norm(&format!("{}/{}", path, "norm2"), device)?; + let silu2 = SILU {}; + let conv2 = load_conv2d(&format!("{}/{}", path, "conv2"), device)?; + let nin_shortcut = load_conv2d(&format!("{}/{}", path, "nin_shortcut"), device).ok(); + + Ok(ResnetBlock { + norm1, + silu1, + conv1, + norm2, + silu2, + conv2, + nin_shortcut, + }) +} + +fn load_mid(path: &str, device: &B::Device) -> Result, Box> { + let block_1 = load_resnet_block(&format!("{}/{}", path, "block_1"), device)?; + let attn = load_conv_self_attention_block(&format!("{}/{}", path, "attn"), device)?; + let block_2 = load_resnet_block(&format!("{}/{}", path, "block_2"), device)?; + + Ok(Mid { + block_1, + attn, + block_2, + }) +} + +fn load_padded_conv2d( + path: &str, + device: &B::Device, +) -> Result, Box> { + let mut conv = load_conv2d(&format!("{}/{}", path, "conv"), device)?; + + let channels = load_tensor::("channels", path, device)?; + let channels = tensor_to_array_2(channels); + + let kernel_size = load_usize::("kernel_size", path, device)?; + let stride = load_usize::("stride", path, device)?; + + let padding = load_tensor::("padding", path, device)?; + let padding: [usize; 4] = tensor_to_array(padding); + let padding = PaddingCfg::new(padding[0], padding[1], padding[2], padding[3]); + + //let mut record = conv.into_record(); + + let mut padded_conv: PaddedConv2d = PaddedConv2dConfig::new(channels, kernel_size, padding) + .with_stride(stride) + .init(device); + let padding_actual = + PaddingConfig2d::Explicit(padded_conv.padding_actual[0], padded_conv.padding_actual[1]); + + conv.padding = burn::module::Ignored(padding_actual); + padded_conv.conv = conv; + + //record.padding = >::into_record(padding_actual); + //padded_conv.conv = padded_conv.conv.load_record(record); + + Ok(padded_conv) +} + +fn load_decoder_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let res1 = load_resnet_block(&format!("{}/{}", path, "res1"), device)?; + let res2 = load_resnet_block(&format!("{}/{}", path, "res2"), device)?; + let res3 = load_resnet_block(&format!("{}/{}", path, "res3"), device)?; + let upsampler = load_conv2d(&format!("{}/{}", path, "upsampler"), device).ok(); + + Ok(DecoderBlock { + res1, + res2, + res3, + upsampler, + }) +} + +fn load_encoder_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let res1 = load_resnet_block(&format!("{}/{}", path, "res1"), device)?; + let res2 = load_resnet_block(&format!("{}/{}", path, "res2"), device)?; + let downsampler = load_padded_conv2d(&format!("{}/{}", path, "downsampler"), device).ok(); + + Ok(EncoderBlock { + res1, + res2, + downsampler, + }) +} + +fn load_decoder(path: &str, device: &B::Device) -> Result, Box> { + let conv_in = load_conv2d(&format!("{}/{}", path, "conv_in"), device)?; + let mid = load_mid(&format!("{}/{}", path, "mid"), device)?; + + let n_block = load_usize::("n_block", path, device)?; + let blocks = (0..n_block) + .into_iter() + .map(|i| load_decoder_block::(&format!("{}/blocks/{}", path, i), device)) + .collect::, _>>()?; + + let norm_out = load_group_norm(&format!("{}/{}", path, "norm_out"), device)?; + let silu = SILU {}; + let conv_out = load_conv2d(&format!("{}/{}", path, "conv_out"), device)?; + + Ok(Decoder { + conv_in, + mid, + blocks, + norm_out, + silu, + conv_out, + }) +} + +fn load_encoder(path: &str, device: &B::Device) -> Result, Box> { + let conv_in = load_conv2d(&format!("{}/{}", path, "conv_in"), device)?; + let mid = load_mid(&format!("{}/{}", path, "mid"), device)?; + + let n_block = load_usize::("n_block", path, device)?; + let blocks = (0..n_block) + .into_iter() + .map(|i| load_encoder_block::(&format!("{}/blocks/{}", path, i), device)) + .collect::, _>>()?; + + let norm_out = load_group_norm(&format!("{}/{}", path, "norm_out"), device)?; + let silu = SILU {}; + let conv_out = load_conv2d(&format!("{}/{}", path, "conv_out"), device)?; + + Ok(Encoder { + conv_in, + mid, + blocks, + norm_out, + silu, + conv_out, + }) +} + +pub fn load_autoencoder( + path: &str, + device: &B::Device, +) -> Result, Box> { + let encoder = load_encoder(&format!("{}/{}", path, "encoder"), device)?; + let decoder = load_decoder(&format!("{}/{}", path, "decoder"), device)?; + let quant_conv = load_conv2d(&format!("{}/{}", path, "quant_conv"), device)?; + let post_quant_conv = load_conv2d(&format!("{}/{}", path, "post_quant_conv"), device)?; + + Ok(Autoencoder { + encoder, + decoder, + quant_conv, + post_quant_conv, + }) +} diff --git a/crates/stable-diffusion-burn/src/model/autoencoder/mod.rs b/crates/stable-diffusion-burn/src/model/autoencoder/mod.rs new file mode 100644 index 0000000..f69f0d3 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/autoencoder/mod.rs @@ -0,0 +1,603 @@ +pub mod load; + +use burn::{ + config::Config, + module::Module, + nn::{ + conv::{Conv2d, Conv2dConfig}, + PaddingConfig2d, + }, + tensor::{ + backend::Backend, Tensor, + }, +}; + +use super::groupnorm::*; +use super::silu::*; +//use crate::backend::Backend as MyBackend; +use crate::backend::qkv_attention; + + +#[derive(Config)] +pub struct AutoencoderConfig {} + +impl AutoencoderConfig { + pub fn init(&self, device: &B::Device) -> Autoencoder { + let encoder = + EncoderConfig::new(vec![(128, 128), (128, 256), (256, 512), (512, 512)], 32, 8).init(device); + let decoder = + DecoderConfig::new(vec![(512, 512), (512, 512), (512, 256), (256, 128)], 32).init(device); + let quant_conv = Conv2dConfig::new([8, 8], [1, 1]).init(device); + let post_quant_conv = Conv2dConfig::new([4, 4], [1, 1]).init(device); + + Autoencoder { + encoder, + decoder, + quant_conv, + post_quant_conv, + } + } +} + +#[derive(Module, Debug)] +pub struct Autoencoder { + encoder: Encoder, + decoder: Decoder, + quant_conv: Conv2d, + post_quant_conv: Conv2d, +} + +impl Autoencoder { + pub fn forward(&self, x: Tensor) -> Tensor { + self.decode_latent(self.encode_image(x)) + } + + pub fn encode_image(&self, x: Tensor) -> Tensor { + let [n_batch, _, _, _] = x.dims(); + let latent = self.encoder.forward(x); + let latent = self.quant_conv.forward(latent); + let latent = latent.slice([0..n_batch, 0..4]); + latent + } + + pub fn decode_latent(&self, latent: Tensor) -> Tensor { + let latent = self.post_quant_conv.forward(latent); + self.decoder.forward(latent) + } +} + +#[derive(Config)] +pub struct EncoderConfig { + channels: Vec<(usize, usize)>, + n_group: usize, + n_channels_out: usize, +} + +impl EncoderConfig { + fn init(&self, device: &B::Device) -> Encoder { + let n_expanded_channels_initial = self + .channels + .first() + .map(|f| f.1) + .expect("Channels must not be empty."); + let n_expanded_channels_final = self.channels.first().unwrap().0; + + let conv_in = Conv2dConfig::new([3, n_expanded_channels_initial], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + let blocks = self + .channels + .iter() + .enumerate() + .map(|(i, &(n_channel_in, n_channel_out))| { + let downsample = i != self.channels.len() - 1; + EncoderBlockConfig::new(n_channel_in, n_channel_out, downsample).init(device) + }) + .collect(); + + let mid = MidConfig::new(n_expanded_channels_final).init(device); + let norm_out = GroupNormConfig::new(self.n_group, n_expanded_channels_final).init(device); + let silu = SILU::new(); + let conv_out = Conv2dConfig::new([n_expanded_channels_final, self.n_channels_out], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + Encoder { + conv_in, + mid, + blocks, + norm_out, + silu, + conv_out, + } + } +} + +#[derive(Module, Debug)] +pub struct Encoder { + conv_in: Conv2d, + mid: Mid, + blocks: Vec>, + norm_out: GroupNorm, + silu: SILU, + conv_out: Conv2d, +} + +impl Encoder { + fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv_in.forward(x); + + let mut x = x; + for block in &self.blocks { + x = block.forward(x); + } + + let x = self.mid.forward(x); + self.conv_out + .forward(self.silu.forward(self.norm_out.forward(x))) + } +} + +#[derive(Config)] +pub struct DecoderConfig { + channels: Vec<(usize, usize)>, + n_group: usize, +} + +impl DecoderConfig { + fn init(&self, device: &B::Device) -> Decoder { + let n_expanded_channels = self + .channels + .first() + .map(|f| f.0) + .expect("Channels must not be empty."); + let n_condensed_channels = self.channels.last().unwrap().1; + + let conv_in = Conv2dConfig::new([4, n_expanded_channels], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + let mid = MidConfig::new(n_expanded_channels).init(device); + + let blocks = self + .channels + .iter() + .enumerate() + .map(|(i, &(n_channel_in, n_channel_out))| { + let upsample = i != self.channels.len() - 1; + DecoderBlockConfig::new(n_channel_in, n_channel_out, upsample).init(device) + }) + .collect(); + + let norm_out = GroupNormConfig::new(self.n_group, n_condensed_channels).init(device); + let silu = SILU::new(); + let conv_out = Conv2dConfig::new([n_condensed_channels, 3], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + Decoder { + conv_in, + mid, + blocks, + norm_out, + silu, + conv_out, + } + } +} + +#[derive(Module, Debug)] +pub struct Decoder { + conv_in: Conv2d, + mid: Mid, + blocks: Vec>, + norm_out: GroupNorm, + silu: SILU, + conv_out: Conv2d, +} + +impl Decoder { + fn forward(&self, x: Tensor) -> Tensor { + let x = self.conv_in.forward(x); + let x = self.mid.forward(x); + + let mut x = x; + for block in &self.blocks { + x = block.forward(x); + } + + self.conv_out + .forward(self.silu.forward(self.norm_out.forward(x))) + } +} + +#[derive(Config)] +pub struct EncoderBlockConfig { + n_channels_in: usize, + n_channels_out: usize, + downsample: bool, +} + +impl EncoderBlockConfig { + fn init(&self, device: &B::Device) -> EncoderBlock { + let res1 = ResnetBlockConfig::new(self.n_channels_in, self.n_channels_out).init(device); + let res2 = ResnetBlockConfig::new(self.n_channels_out, self.n_channels_out).init(device); + let downsampler = if self.downsample { + let padding = PaddingCfg::new(0, 1, 0, 1); + Some( + PaddedConv2dConfig::new([self.n_channels_out, self.n_channels_out], 3, padding) + .with_stride(2) + .init(device), + ) + } else { + None + }; + + EncoderBlock { + res1, + res2, + downsampler, + } + } +} + +#[derive(Module, Debug)] +pub struct EncoderBlock { + res1: ResnetBlock, + res2: ResnetBlock, + downsampler: Option>, +} + +impl EncoderBlock { + fn forward(&self, x: Tensor) -> Tensor { + let x = self.res1.forward(x); + let x = self.res2.forward(x); + if let Some(d) = self.downsampler.as_ref() { + d.forward(x) + } else { + x + } + } +} + +#[derive(Config)] +pub struct DecoderBlockConfig { + n_channels_in: usize, + n_channels_out: usize, + upsample: bool, +} + +impl DecoderBlockConfig { + fn init(&self, device: &B::Device) -> DecoderBlock { + let res1 = ResnetBlockConfig::new(self.n_channels_in, self.n_channels_out).init(device); + let res2 = ResnetBlockConfig::new(self.n_channels_out, self.n_channels_out).init(device); + let res3 = ResnetBlockConfig::new(self.n_channels_out, self.n_channels_out).init(device); + let upsampler = if self.upsample { + Some( + Conv2dConfig::new([self.n_channels_out, self.n_channels_out], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device), + ) + } else { + None + }; + + DecoderBlock { + res1, + res2, + res3, + upsampler, + } + } +} + +#[derive(Module, Debug)] +pub struct DecoderBlock { + res1: ResnetBlock, + res2: ResnetBlock, + res3: ResnetBlock, + upsampler: Option>, +} + +impl DecoderBlock { + fn forward(&self, x: Tensor) -> Tensor { + let x = self.res1.forward(x); + let x = self.res2.forward(x); + let x = self.res3.forward(x); + + if let Some(d) = self.upsampler.as_ref() { + let [n_batch, n_channel, height, width] = x.dims(); + let x = x + .reshape([n_batch, n_channel, height, 1, width, 1]) + .repeat(&[1, 1, 1, 2, 1, 2]) + .reshape([n_batch, n_channel, 2 * height, 2 * width]); + d.forward(x) + } else { + x + } + } +} + +#[derive(Config)] +pub struct PaddedConv2dConfig { + channels: [usize; 2], + kernel_size: usize, + #[config(default = 1)] + stride: usize, + padding: PaddingCfg, +} + +impl PaddedConv2dConfig { + fn init(&self, device: &B::Device) -> PaddedConv2d { + let calc_padding = |p_left, p_right| { + let n = if p_left >= p_right { + 0 + } else { + div_roundup(p_right - p_left, self.stride) + }; + + n * self.stride + p_left + }; + + let pad_vertical = calc_padding(self.padding.pad_top, self.padding.pad_bottom); + let pad_horizontal = calc_padding(self.padding.pad_left, self.padding.pad_right); + let padding_actual = [pad_vertical, pad_horizontal]; + + let conv = Conv2dConfig::new(self.channels, [self.kernel_size, self.kernel_size]) + .with_stride([self.stride, self.stride]) + .with_padding(PaddingConfig2d::Explicit(pad_vertical, pad_horizontal)) + .init(device); + + let kernel_size = self.kernel_size; + let stride = self.stride; + + let padding = Padding { + pad_left: self.padding.pad_left, + pad_right: self.padding.pad_right, + pad_top: self.padding.pad_top, + pad_bottom: self.padding.pad_bottom, + }; + + PaddedConv2d { + conv, + kernel_size, + stride, + padding, + padding_actual, + } + } +} + +fn div_roundup(x: usize, y: usize) -> usize { + (x + y - 1) / y +} + +#[derive(Module, Debug)] +pub struct PaddedConv2d { + conv: Conv2d, + kernel_size: usize, + stride: usize, + padding: Padding, + padding_actual: [usize; 2], +} + +impl PaddedConv2d { + fn forward(&self, x: Tensor) -> Tensor { + let [n_batch, n_channel, height, width] = x.dims(); + + let desired_height = (self.padding.pad_top + self.padding.pad_bottom + height + - self.kernel_size) + / self.stride + + 1; + let desired_width = (self.padding.pad_left + self.padding.pad_right + width + - self.kernel_size) + / self.stride + + 1; + + let skip_vert = (self.padding_actual[0] - self.padding.pad_top) / self.stride; + let skip_hor = (self.padding_actual[1] - self.padding.pad_left) / self.stride; + + self.conv.forward(x).slice([ + 0..n_batch, + 0..n_channel, + skip_vert..(skip_vert + desired_height), + skip_hor..(skip_hor + desired_width), + ]) + } +} + +#[derive(Config, Debug)] +pub struct PaddingCfg { + pad_left: usize, + pad_right: usize, + pad_top: usize, + pad_bottom: usize, +} + +#[derive(Module, Clone, Debug)] +pub struct Padding { + pad_left: usize, + pad_right: usize, + pad_top: usize, + pad_bottom: usize, +} + +#[derive(Config)] +pub struct MidConfig { + n_channel: usize, +} + +impl MidConfig { + fn init(&self, device: &B::Device) -> Mid { + let block_1 = ResnetBlockConfig::new(self.n_channel, self.n_channel).init(device); + let attn = ConvSelfAttentionBlockConfig::new(self.n_channel).init(device); + let block_2 = ResnetBlockConfig::new(self.n_channel, self.n_channel).init(device); + + Mid { + block_1, + attn, + block_2, + } + } +} + +#[derive(Module, Debug)] +pub struct Mid { + block_1: ResnetBlock, + attn: ConvSelfAttentionBlock, + block_2: ResnetBlock, +} + +impl Mid { + fn forward(&self, x: Tensor) -> Tensor { + let x = self.block_1.forward(x); + let x = self.attn.forward(x); + let x = self.block_2.forward(x); + x + } +} + +#[derive(Config)] +pub struct ResnetBlockConfig { + in_channels: usize, + out_channels: usize, +} + +impl ResnetBlockConfig { + fn init(&self, device: &B::Device) -> ResnetBlock { + let norm1 = GroupNormConfig::new(32, self.in_channels).init(device); + let conv1 = Conv2dConfig::new([self.in_channels, self.out_channels], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + let norm2 = GroupNormConfig::new(32, self.out_channels).init(device); + let conv2 = Conv2dConfig::new([self.out_channels, self.out_channels], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + let nin_shortcut = if self.in_channels != self.out_channels { + Some(Conv2dConfig::new([self.in_channels, self.out_channels], [1, 1]).init(device)) + } else { + None + }; + + let silu1 = SILU::new(); + let silu2 = SILU::new(); + + ResnetBlock { + norm1, + silu1, + conv1, + norm2, + silu2, + conv2, + nin_shortcut, + } + } +} + +#[derive(Module, Debug)] +pub struct ResnetBlock { + norm1: GroupNorm, + silu1: SILU, + conv1: Conv2d, + norm2: GroupNorm, + silu2: SILU, + conv2: Conv2d, + nin_shortcut: Option>, +} + +impl ResnetBlock { + fn forward(&self, x: Tensor) -> Tensor { + let h = self + .conv1 + .forward(self.silu1.forward(self.norm1.forward(x.clone()))); + let h = self + .conv2 + .forward(self.silu2.forward(self.norm2.forward(h))); + + if let Some(ns) = self.nin_shortcut.as_ref() { + ns.forward(x) + h + } else { + x + h + } + } +} + +#[derive(Config)] +pub struct ConvSelfAttentionBlockConfig { + n_channel: usize, +} + +impl ConvSelfAttentionBlockConfig { + fn init(&self, device: &B::Device) -> ConvSelfAttentionBlock { + let norm = GroupNormConfig::new(32, self.n_channel).init(device); + let q = Conv2dConfig::new([self.n_channel, self.n_channel], [1, 1]).init(device); + let k = Conv2dConfig::new([self.n_channel, self.n_channel], [1, 1]).init(device); + let v = Conv2dConfig::new([self.n_channel, self.n_channel], [1, 1]).init(device); + let proj_out = Conv2dConfig::new([self.n_channel, self.n_channel], [1, 1]).init(device); + + ConvSelfAttentionBlock { + norm, + q, + k, + v, + proj_out, + } + } +} + +#[derive(Module, Debug)] +pub struct ConvSelfAttentionBlock { + norm: GroupNorm, + q: Conv2d, + k: Conv2d, + v: Conv2d, + proj_out: Conv2d, +} + +impl ConvSelfAttentionBlock { + fn forward(&self, x: Tensor) -> Tensor { + let [n_batch, n_channel, height, width] = x.dims(); + + let h = self.norm.forward(x.clone()); + + let q = self + .q + .forward(h.clone()) + .reshape([n_batch, n_channel, height * width]) + .swap_dims(1, 2); + let k = self + .k + .forward(h.clone()) + .reshape([n_batch, n_channel, height * width]) + .swap_dims(1, 2); + let v = self + .v + .forward(h) + .reshape([n_batch, n_channel, height * width]) + .swap_dims(1, 2); + + /*let wv = Tensor::from_primitive(B::qkv_attention( + q.into_primitive(), + k.into_primitive(), + v.into_primitive(), + None, + 1, + )) + .swap_dims(1, 2) + .reshape([n_batch, n_channel, height, width]);*/ + + let wv = qkv_attention( + q, + k, + v, + None, + 1, + ) + .swap_dims(1, 2) + .reshape([n_batch, n_channel, height, width]); + + let projected = self.proj_out.forward(wv); + + x + projected + } +} diff --git a/crates/stable-diffusion-burn/src/model/clip/load.rs b/crates/stable-diffusion-burn/src/model/clip/load.rs new file mode 100644 index 0000000..fa721ff --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/clip/load.rs @@ -0,0 +1,88 @@ +use std::error::Error; + +use burn::{ + module::{Module, Param}, + tensor::backend::Backend, +}; + +use super::*; +use crate::model::load::*; + +pub fn load_mlp(path: &str, device: &B::Device) -> Result, Box> { + let fc1 = load_linear(&format!("{}/{}", path, "fc1"), device)?; + let gelu = QuickGELU::new(); + let fc2 = load_linear(&format!("{}/{}", path, "fc2"), device)?; + + let mlp = MLP { + fc1: fc1, + gelu: gelu, + fc2: fc2, + }; + + Ok(mlp) +} + +pub fn load_multi_head_self_attention( + path: &str, + device: &B::Device, +) -> Result, Box> { + let n_head = load_usize::("n_head", path, device)?; + let query = load_linear(&format!("{}/{}", path, "query"), device)?; + let key = load_linear(&format!("{}/{}", path, "key"), device)?; + let value = load_linear(&format!("{}/{}", path, "value"), device)?; + let out = load_linear(&format!("{}/{}", path, "out"), device)?; + + let mhsa = MultiHeadSelfAttention { + n_head: n_head, + query: query, + key: key, + value: value, + out: out, + }; + + Ok(mhsa) +} + +pub fn load_residual_decoder_attention_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let mlp = load_mlp(&format!("{}/{}", path, "mlp"), device)?; + let attn = load_multi_head_self_attention(&format!("{}/{}", path, "attn"), device)?; + let attn_ln = load_layer_norm(&format!("{}/{}", path, "attn_ln"), device)?; + let mlp_ln = load_layer_norm(&format!("{}/{}", path, "mlp_ln"), device)?; + + let rdab = ResidualDecoderAttentionBlock { + attn: attn, + attn_ln: attn_ln, + mlp: mlp, + mlp_ln: mlp_ln, + }; + + Ok(rdab) +} + +pub fn load_clip(path: &str, device: &B::Device) -> Result, Box> { + let token_embedding = load_embedding(&format!("{}/{}", path, "token_embedding"), device)?; + let position_embedding = + Param::from_tensor(load_tensor("weight", &format!("{}/position_embedding", path), device)?); + + let n_layer = load_usize::("n_layer", path, device)?; + let blocks = (0..n_layer) + .into_iter() + .map(|i| { + load_residual_decoder_attention_block::(&format!("{}/blocks/{}", path, i), device) + }) + .collect::, _>>()?; + + let layer_norm = load_layer_norm(&format!("{}/{}", path, "layer_norm"), device)?; + + let clip = CLIP { + token_embedding: token_embedding, + position_embedding: position_embedding, + blocks: blocks, + layer_norm: layer_norm, + }; + + Ok(clip) +} diff --git a/crates/stable-diffusion-burn/src/model/clip/mod.rs b/crates/stable-diffusion-burn/src/model/clip/mod.rs new file mode 100644 index 0000000..1dfc09a --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/clip/mod.rs @@ -0,0 +1,226 @@ +pub mod load; + +use burn::{ + config::Config, + module::{Module, Param}, + nn, + tensor::{ + activation::sigmoid, + backend::Backend, + Distribution, Int, Tensor, + }, +}; + +//use crate::backend::Backend as MyBackend; +use crate::backend::{qkv_attention, attn_decoder_mask}; + +#[derive(Config)] +pub struct CLIPConfig { + n_vocab: usize, + n_state: usize, + n_head: usize, + n_ctx: usize, + n_layer: usize, +} + +impl CLIPConfig { + pub fn init(&self, device: &B::Device) -> CLIP { + let token_embedding = nn::EmbeddingConfig::new(self.n_vocab, self.n_state).init(device); + let position_embedding = + Param::from_tensor(Tensor::random([self.n_ctx, self.n_state], Distribution::Normal(0.0, 1.0), device)); + let blocks = (0..self.n_layer) + .into_iter() + .map(|_| ResidualDecoderAttentionBlockConfig::new(self.n_state, self.n_head).init(device)) + .collect(); + let layer_norm = nn::LayerNormConfig::new(self.n_state).init(device); + + CLIP { + token_embedding, + position_embedding, + blocks, + layer_norm, + } + } +} + +#[derive(Module, Debug)] +pub struct CLIP { + token_embedding: nn::Embedding, + position_embedding: Param>, + blocks: Vec>, + layer_norm: nn::LayerNorm, +} + +impl CLIP { + pub fn forward(&self, x: Tensor) -> Tensor { + let [_n_batch, seq_len] = x.dims(); + + //let mask = Tensor::from_primitive(B::attn_decoder_mask(seq_len, &x.device())); + let mask = attn_decoder_mask(seq_len, &x.device()); + + let embedded = self.token_embedding.forward(x) + + self + .position_embedding + .val() + .slice([0..seq_len]) + .unsqueeze(); + + let mut x = embedded; + for block in &self.blocks { + x = block.forward(x, mask.clone()); + } + + self.layer_norm.forward(x) + } +} + +#[derive(Config)] +pub struct ResidualDecoderAttentionBlockConfig { + n_state: usize, + n_head: usize, +} + +impl ResidualDecoderAttentionBlockConfig { + pub fn init(&self, device: &B::Device) -> ResidualDecoderAttentionBlock { + let attn = MultiHeadSelfAttentionConfig::new(self.n_state, self.n_head).init(device); + let attn_ln = nn::LayerNormConfig::new(self.n_state).init(device); + + let mlp = MLPConfig::new(self.n_state, 4 * self.n_state).init(device); + let mlp_ln = nn::LayerNormConfig::new(self.n_state).init(device); + + ResidualDecoderAttentionBlock { + attn, + attn_ln, + mlp, + mlp_ln, + } + } +} + +#[derive(Module, Debug)] +pub struct ResidualDecoderAttentionBlock { + attn: MultiHeadSelfAttention, + attn_ln: nn::LayerNorm, + mlp: MLP, + mlp_ln: nn::LayerNorm, +} + +impl ResidualDecoderAttentionBlock { + fn forward(&self, x: Tensor, mask: Tensor) -> Tensor { + let x = x.clone() + self.attn.forward(self.attn_ln.forward(x), Some(mask)); + let x = x.clone() + self.mlp.forward(self.mlp_ln.forward(x)); + return x; + } +} + +#[derive(Config)] +pub struct MultiHeadSelfAttentionConfig { + n_state: usize, + n_head: usize, +} + +impl MultiHeadSelfAttentionConfig { + fn init(&self, device: &B::Device) -> MultiHeadSelfAttention { + assert!( + self.n_state % self.n_head == 0, + "State size {} must be a multiple of head size {}", + self.n_state, + self.n_head + ); + + let n_head = self.n_head; + let query = nn::LinearConfig::new(self.n_state, self.n_state).init(device); + let key = nn::LinearConfig::new(self.n_state, self.n_state).init(device); + let value = nn::LinearConfig::new(self.n_state, self.n_state).init(device); + let out = nn::LinearConfig::new(self.n_state, self.n_state).init(device); + + MultiHeadSelfAttention { + n_head, + query, + key, + value, + out, + } + } +} + +#[derive(Module, Debug)] +pub struct MultiHeadSelfAttention { + n_head: usize, + query: nn::Linear, + key: nn::Linear, + value: nn::Linear, + out: nn::Linear, +} + +impl MultiHeadSelfAttention { + pub fn forward(&self, x: Tensor, mask: Option>) -> Tensor { + let q = self.query.forward(x.clone()); + let k = self.key.forward(x.clone()); + let v = self.value.forward(x); + + /*let wv = Tensor::from_primitive(B::qkv_attention( + q.into_primitive(), + k.into_primitive(), + v.into_primitive(), + mask.map(|m| m.into_primitive()), + self.n_head, + ));*/ + + let wv = qkv_attention( + q, + k, + v, + mask, + self.n_head, + ); + + return self.out.forward(wv); + } +} + +#[derive(Config, Debug)] +pub struct MLPConfig { + input_size: usize, + hidden_size: usize, +} + +impl MLPConfig { + fn init(&self, device: &B::Device) -> MLP { + let fc1 = nn::LinearConfig::new(self.input_size, self.hidden_size).init(device); + let gelu = QuickGELU::new(); + let fc2 = nn::LinearConfig::new(self.hidden_size, self.input_size).init(device); + + MLP { fc1, gelu, fc2 } + } +} + +#[derive(Module, Debug)] +pub struct MLP { + fc1: nn::Linear, + gelu: QuickGELU, + fc2: nn::Linear, +} + +impl MLP { + fn forward(&self, x: Tensor) -> Tensor { + let x = self.fc1.forward(x); + let x = self.gelu.forward(x); + let x = self.fc2.forward(x); + + x + } +} + +#[derive(Module, Clone, Debug)] +pub struct QuickGELU {} + +impl QuickGELU { + fn new() -> Self { + Self {} + } + + fn forward(&self, x: Tensor) -> Tensor { + x.clone() * sigmoid(x * 1.702) + } +} diff --git a/crates/stable-diffusion-burn/src/model/groupnorm/load.rs b/crates/stable-diffusion-burn/src/model/groupnorm/load.rs new file mode 100644 index 0000000..e093a78 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/groupnorm/load.rs @@ -0,0 +1,35 @@ +use super::GroupNorm; +use crate::model::load::*; + +use std::error::Error; + +use burn::{ + module::Param, + tensor::{backend::Backend, Tensor}, +}; + +pub fn load_group_norm( + path: &str, + device: &B::Device, +) -> Result, Box> { + let n_group = load_usize::("n_group", path, device)?.into(); + let n_channel = load_usize::("n_channel", path, device)?.into(); + let eps = load_f32::("eps", path, device)?.into(); + + let gamma = Param::from_tensor(load_tensor::("weight", path, device) + .ok() + .unwrap_or_else(|| Tensor::ones([n_channel], device)) + ); + let beta = Param::from_tensor(load_tensor::("bias", path, device) + .ok() + .unwrap_or_else(|| Tensor::zeros([n_channel], device)) + ); + + Ok(GroupNorm { + n_group, + n_channel, + gamma, + beta, + eps, + }) +} diff --git a/crates/stable-diffusion-burn/src/model/groupnorm/mod.rs b/crates/stable-diffusion-burn/src/model/groupnorm/mod.rs new file mode 100644 index 0000000..1633bf2 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/groupnorm/mod.rs @@ -0,0 +1,82 @@ +pub mod load; + +use burn::{ + config::Config, + module::{Module, Param}, + tensor::{backend::Backend, Tensor}, +}; + +#[derive(Config)] +pub struct GroupNormConfig { + n_group: usize, + n_channel: usize, + #[config(default = 1e-5)] + eps: f64, +} + +impl GroupNormConfig { + pub fn init(&self, device: &B::Device) -> GroupNorm { + assert!( + self.n_channel % self.n_group == 0, + "The number of channels {} must be divisible by the number of groups {}", + self.n_channel, + self.n_group + ); + + let _n_per_group = self.n_channel / self.n_group; + + let gamma = Param::from_tensor(Tensor::ones([self.n_channel], device)); + let beta = Param::from_tensor(Tensor::zeros([self.n_channel], device)); + + let eps = self.eps; + + GroupNorm { + n_group: self.n_group, + n_channel: self.n_channel, + gamma, + beta, + eps, + } + } +} + +#[derive(Module, Debug)] +pub struct GroupNorm { + n_group: usize, + n_channel: usize, + gamma: Param>, + beta: Param>, + eps: f64, +} + +impl GroupNorm { + pub fn forward(&self, x: Tensor) -> Tensor { + let shape = x.shape(); + let n_batch = shape.dims[0]; + let num_elements = shape.num_elements(); + + let mut affine_shape = [1; D]; + affine_shape[1] = self.n_channel; + + layernorm( + x.reshape([ + n_batch, + self.n_group, + num_elements / (n_batch * self.n_group), + ]), + self.eps, + ) + .reshape(shape) + .mul(self.gamma.val().reshape(affine_shape)) + .add(self.beta.val().reshape(affine_shape)) + } +} + +pub fn layernorm(x: Tensor, eps: f64) -> Tensor { + //let (var, mean) = x.clone().var_mean_bias(D - 1); + //x.sub(mean).div(var.sqrt().add_scalar(eps)) + + let u = x.clone() - x.mean_dim(D - 1); + u.clone() + .div((u.clone() * u).mean_dim(D - 1).add_scalar(eps).sqrt()) +} diff --git a/crates/stable-diffusion-burn/src/model/load.rs b/crates/stable-diffusion-burn/src/model/load.rs new file mode 100644 index 0000000..16f3a5f --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/load.rs @@ -0,0 +1,177 @@ +use npy::{self, NpyData}; +use num_traits::cast::ToPrimitive; +use burn::tensor::cast::ToElement; +use burn::prelude::TensorData; +use std::error::Error; +use std::io::Read; + +use burn::{ + module::{Module, Param}, + nn::{self, conv}, + tensor::{backend::Backend, Tensor}, +}; + +use burn::tensor::ElementConversion; + +pub fn numpy_to_tensor( + numpy_data: NpyData, + device: &B::Device, +) -> Tensor { + let v = numpy_data.to_vec(); + + let shape: Vec<_> = v[0..D].into_iter().map(|&v| v as usize).collect(); + let data: Vec = v[D..].into_iter().map(|e| e.elem()).collect(); + + //Tensor::from_data_device(Data::new(data, shape.into()), device) + Tensor::from_data(TensorData::new(data, shape), device) +} + +pub fn load_tensor( + name: &str, + path: &str, + device: &B::Device, +) -> Result, Box> { + let tensor_path = format!("{}/{}.npy", path, name); + + let mut buf = vec![]; + std::fs::File::open(&tensor_path)?.read_to_end(&mut buf)?; + + let tensor_numpy: NpyData = NpyData::from_bytes(&buf)?; + + let tensor = numpy_to_tensor(tensor_numpy, device); + + println!("{}", tensor_path); + + Ok(tensor) +} + +pub fn load_f32( + name: &str, + path: &str, + device: &B::Device, +) -> Result> { + load_tensor::(name, path, device).map(|t| t.into_scalar().to_f32()) +} + +pub fn load_usize( + name: &str, + path: &str, + device: &B::Device, +) -> Result> { + load_tensor::(name, path, device).map(|t| t.into_scalar().to_usize()) +} + +pub fn load_linear( + path: &str, + device: &B::Device, +) -> Result, Box> { + let weight = load_tensor::("weight", path, device)?; + let bias = load_tensor::("bias", path, device).ok(); + + Ok(nn::Linear { + weight: Param::from_tensor(weight), + bias: bias.map(|t| Param::from_tensor(t)), + }) +} + +pub fn load_embedding( + path: &str, + device: &B::Device, +) -> Result, Box> { + let weight = load_tensor::("weight", path, device)?; + + Ok(nn::Embedding { + weight: Param::from_tensor(weight), + }) +} + +pub fn load_layer_norm( + path: &str, + device: &B::Device, +) -> Result, Box> { + let weight = load_tensor::("weight", path, device)?; + let bias = load_tensor::("bias", path, device)?; + let eps = load_f32::("eps", path, device)? as f64; + + let [n_state] = weight.dims(); + + let mut layer_norm = nn::LayerNormConfig::new(n_state).with_epsilon(eps).init(device); + layer_norm.gamma = Param::from_tensor(weight); + layer_norm.beta = Param::from_tensor(bias); + + Ok(layer_norm) +} + +/*pub fn load_rmsnorm(path: &str, device: &B::Device) -> Result, Box> { + let weight = load_tensor::("weight", path, device)?; + let eps = load_f32::("eps", path, device)?.into(); + + let rmsnorm = RMSNorm { + weight: Param::from_tensor(weight), + eps: eps + }; + + Ok(rmsnorm) +}*/ + +pub fn load_conv2d( + path: &str, + device: &B::Device, +) -> Result, Box> { + let weight = load_tensor::("weight", path, device)?; + let bias = load_tensor::("bias", path, device).ok(); + let has_bias = bias.is_some(); + + let stride = load_tensor::("stride", path, device)?; + let stride = tensor_to_array_2(stride); + + let kernel_size = load_tensor::("kernel_size", path, device)?; + let kernel_size = tensor_to_array_2(kernel_size); + + let dilation = load_tensor::("dilation", path, device)?; + let dilation = tensor_to_array_2(dilation); + + let n_group = load_usize::("n_group", path, device)?.into(); + let n_channels_in = load_usize::("n_channels_in", path, device)?.into(); + let n_channels_out = load_usize::("n_channels_out", path, device)?.into(); + + let padding = load_tensor::("padding", path, device)?; + let padding = tensor_to_array_2(padding); + let padding = nn::PaddingConfig2d::Explicit(padding[0], padding[1]); + + let mut conv2d = conv::Conv2dConfig::new([n_channels_in, n_channels_out], kernel_size) + .with_stride(stride) + .with_dilation(dilation) + .with_groups(n_group) + .with_padding(padding.clone()) + .with_bias(has_bias) + .init(device); + + conv2d.weight = Param::from_tensor(weight); + conv2d.bias = bias.map(|t| Param::from_tensor(t)); + conv2d.stride = stride; + conv2d.kernel_size = kernel_size; + conv2d.dilation = dilation; + conv2d.groups = n_group; + conv2d.padding = burn::module::Ignored(padding); + + Ok(conv2d) +} + +pub fn tensor_to_array_2(x: Tensor) -> [usize; 2] { + let vec: Vec<::FloatElem> = x.into_data().to_vec().unwrap(); + assert!(vec.len() == 2, "Tensor length must be 2."); + [vec[0].to_usize(), vec[1].to_usize()] +} + +pub fn tensor_to_array(x: Tensor) -> [usize; N] { + let vec: Vec<::FloatElem> = x.into_data().to_vec().unwrap(); + assert!(vec.len() == N, "Tensor length must be {}.", N); + + let mut arr = [0; N]; + for (a, t) in arr.iter_mut().zip(vec) { + *a = t.to_usize(); + } + + arr +} diff --git a/crates/stable-diffusion-burn/src/model/mod.rs b/crates/stable-diffusion-burn/src/model/mod.rs new file mode 100644 index 0000000..b89f764 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/mod.rs @@ -0,0 +1,11 @@ +pub mod stablediffusion; + +pub mod autoencoder; +pub mod clip; +pub mod unet; + +pub mod attention; +pub mod groupnorm; +pub mod silu; + +pub mod load; diff --git a/crates/stable-diffusion-burn/src/model/silu.rs b/crates/stable-diffusion-burn/src/model/silu.rs new file mode 100644 index 0000000..5766349 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/silu.rs @@ -0,0 +1,17 @@ +use burn::{ + module::Module, + tensor::{activation::sigmoid, backend::Backend, Tensor}, +}; + +#[derive(Module, Clone, Debug)] +pub struct SILU {} + +impl SILU { + pub fn new() -> Self { + Self {} + } + + pub fn forward(&self, x: Tensor) -> Tensor { + x.clone() * sigmoid(x) + } +} diff --git a/crates/stable-diffusion-burn/src/model/stablediffusion/load.rs b/crates/stable-diffusion-burn/src/model/stablediffusion/load.rs new file mode 100644 index 0000000..ae228b7 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/stablediffusion/load.rs @@ -0,0 +1,30 @@ +use std::error::Error; + +use burn::{ + module::Param, + tensor::backend::Backend, +}; + +use super::*; +use crate::model::{ + autoencoder::load::load_autoencoder, clip::load::load_clip, load::*, unet::load::load_unet, +}; + +pub fn load_stable_diffusion( + path: &str, + device: &B::Device, +) -> Result, Box> { + let n_steps = load_usize::("n_steps", path, device)?; + let alpha_cumulative_products = Param::from_tensor(load_tensor::("alphas_cumprod", path, device)?); + let autoencoder = load_autoencoder(&format!("{}/{}", path, "autoencoder"), device)?; + let diffusion = load_unet(&format!("{}/{}", path, "unet"), device)?; + let clip = load_clip(&format!("{}/{}", path, "clip"), device)?; + + Ok(StableDiffusion { + n_steps, + alpha_cumulative_products, + autoencoder, + diffusion, + clip, + }) +} diff --git a/crates/stable-diffusion-burn/src/model/stablediffusion/mod.rs b/crates/stable-diffusion-burn/src/model/stablediffusion/mod.rs new file mode 100644 index 0000000..8096392 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/stablediffusion/mod.rs @@ -0,0 +1,237 @@ +pub mod load; + +use burn::{ + config::Config, + module::{Module, Param}, + tensor::{backend::Backend, BasicOps, Distribution, Int, Tensor}, + tensor::cast::ToElement, +}; + +use num_traits::ToPrimitive; + +//use crate::backend::Backend as MyBackend; + +use super::autoencoder::{Autoencoder, AutoencoderConfig}; +use super::clip::{CLIPConfig, CLIP}; +use super::unet::{UNet, UNetConfig}; +use crate::tokenizer::SimpleTokenizer; + +#[derive(Config)] +pub struct StableDiffusionConfig {} + +impl StableDiffusionConfig { + pub fn init(&self, device: &B::Device) -> StableDiffusion { + let n_steps = 1000; + let alpha_cumulative_products = Param::from_tensor(offset_cosine_schedule_cumprod::(n_steps as i64, device)); + + let autoencoder = AutoencoderConfig::new().init(device); + let diffusion = UNetConfig::new().init(device); + let clip = CLIPConfig::new(49408, 768, 12, 77, 12).init(device); + + StableDiffusion { + n_steps, + alpha_cumulative_products, + autoencoder, + diffusion, + clip, + } + } +} + +#[derive(Module, Debug)] +pub struct StableDiffusion { + n_steps: usize, + alpha_cumulative_products: Param>, + autoencoder: Autoencoder, + diffusion: UNet, + clip: CLIP, +} + +impl StableDiffusion { + pub fn sample_image( + &self, + context: Tensor, + unconditional_context: Tensor, + unconditional_guidance_scale: f64, + n_steps: usize, + ) -> Vec> { + let [_n_batch, _, _] = context.dims(); + + let latent = self.sample_latent( + context, + unconditional_context, + unconditional_guidance_scale, + n_steps, + ); + self.latent_to_image(latent) + } + + pub fn latent_to_image(&self, latent: Tensor) -> Vec> { + let [n_batch, _, _, _] = latent.dims(); + let image = self.autoencoder.decode_latent(latent * (1.0 / 0.18215)); + + let n_channel = 3; + let height = 512; + let width = 512; + let num_elements_per_image = n_channel * height * width; + + // correct size and scale and reorder to + let image = (image + 1.0) / 2.0; + let image = image + .reshape([n_batch, n_channel, height, width]) + .swap_dims(1, 2) + .swap_dims(2, 3) + .mul_scalar(255.0); + + let flattened: Vec = image.into_data().to_vec().unwrap(); + + (0..n_batch) + .into_iter() + .map(|b| { + let start = b * num_elements_per_image; + let end = start + num_elements_per_image; + + flattened[start..end] + .into_iter() + .map(|v| v.to_f64().min(255.0).max(0.0) as u8) + .collect() + }) + .collect() + } + + pub fn sample_latent( + &self, + context: Tensor, + unconditional_context: Tensor, + unconditional_guidance_scale: f64, + n_steps: usize, + ) -> Tensor { + let device = context.device(); + + let step_size = self.n_steps / n_steps; + + let [n_batches, _, _] = context.dims(); + + let gen_noise = || { + Tensor::random([n_batches, 4, 64, 64], Distribution::Normal(0.0, 1.0), &device) + }; + + let sigma = 0.0; // Use deterministic diffusion + + let mut latent = gen_noise(); + + for t in (0..self.n_steps).rev().step_by(step_size) { + let current_alpha: f64 = self + .alpha_cumulative_products + .val() + .slice([t..t + 1]) + .into_scalar() + .to_f64(); + + let prev_alpha: f64 = if t >= step_size { + let i = t - step_size; + self.alpha_cumulative_products + .val() + .slice([i..i + 1]) + .into_scalar() + .to_f64() + } else { + 1.0 + }; + + let sqrt_noise = (1.0 - current_alpha).sqrt(); + + let timestep = Tensor::from_ints([t as i32], &device); + let pred_noise = self.forward_diffuser( + latent.clone(), + timestep, + context.clone(), + unconditional_context.clone(), + unconditional_guidance_scale, + ); + let predx0 = (latent - pred_noise.clone() * sqrt_noise) / current_alpha.sqrt(); + let dir_latent = pred_noise * (1.0 - prev_alpha - sigma * sigma).sqrt(); + + let prev_latent = predx0 * prev_alpha.sqrt() + dir_latent + gen_noise() * sigma; + latent = prev_latent; + } + + latent + } + + fn forward_diffuser( + &self, + latent: Tensor, + timestep: Tensor, + context: Tensor, + unconditional_context: Tensor, + unconditional_guidance_scale: f64, + ) -> Tensor { + let [n_batch, _, _, _] = latent.dims(); + //let latent = latent.repeat(0, 2); + + let unconditional_latent = self.diffusion.forward( + latent.clone(), + timestep.clone(), + unconditional_context.unsqueeze().repeat(&[0, n_batch]), + ); + + let conditional_latent = self.diffusion.forward(latent, timestep, context); + + /*let latent = self.diffusion.forward( + latent.repeat(0, 2), + timestep.repeat(0, 2), + Tensor::cat(vec![unconditional_context.unsqueeze::<3>(), context], 0) + ); + + let unconditional_latent = latent.clone().slice([0..n_batch]); + let conditional_latent = latent.slice([n_batch..2 * n_batch]);*/ + + unconditional_latent.clone() + + (conditional_latent - unconditional_latent) * unconditional_guidance_scale + } + + pub fn unconditional_context(&self, tokenizer: &SimpleTokenizer) -> Tensor { + self.context(tokenizer, "").squeeze(0) + } + + pub fn context(&self, tokenizer: &SimpleTokenizer, text: &str) -> Tensor { + let device = &self.clip.devices()[0]; + let text = format!("<|startoftext|>{}<|endoftext|>", text); + let tokenized: Vec<_> = tokenizer + .encode(&text) + .into_iter() + .map(|v| v as i32) + .collect(); + + self.clip.forward( + Tensor::::from_ints(&tokenized[..], device) + .unsqueeze(), + ) + } +} + +use std::f64::consts::PI; + +fn cosine_schedule(n_steps: i64, device: &B::Device) -> Tensor { + Tensor::arange(1..n_steps + 1, device) + .float() + .mul_scalar(PI * 0.5 / n_steps as f64) + .cos() +} + +fn offset_cosine_schedule(n_steps: i64, device: &B::Device) -> Tensor { + let min_signal_rate: f64 = 0.02; + let max_signal_rate: f64 = 0.95; + let start_angle = max_signal_rate.acos(); + let end_angle = min_signal_rate.acos(); + + let times = Tensor::arange(1..n_steps + 1, device).float(); + + let diffusion_angles = times * ((end_angle - start_angle) / n_steps as f64) + start_angle; + diffusion_angles.cos() +} + +fn offset_cosine_schedule_cumprod(n_steps: i64, device: &B::Device) -> Tensor { + offset_cosine_schedule::(n_steps, device).powf_scalar(2.0) +} diff --git a/crates/stable-diffusion-burn/src/model/unet/load.rs b/crates/stable-diffusion-burn/src/model/unet/load.rs new file mode 100644 index 0000000..0259986 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/unet/load.rs @@ -0,0 +1,300 @@ +use crate::model::load::*; + +use std::error::Error; + +use burn::tensor::backend::Backend; + +use super::*; +use crate::model::groupnorm::load::load_group_norm; + +pub fn load_res_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let norm_in = load_group_norm::(&format!("{}/{}", path, "norm_in"), device)?; + let conv_in = load_conv2d::(&format!("{}/{}", path, "conv_in"), device)?; + let lin_embed = load_linear::(&format!("{}/{}", path, "lin_embed"), device)?; + let norm_out = load_group_norm::(&format!("{}/{}", path, "norm_out"), device)?; + let conv_out = load_conv2d::(&format!("{}/{}", path, "conv_out"), device)?; + let skip_connection = load_conv2d::(&format!("{}/{}", path, "skip_connection"), device).ok(); + + let res_block = ResBlock { + norm_in: norm_in, + silu_in: SILU::new(), + conv_in: conv_in, + silu_embed: SILU::new(), + lin_embed: lin_embed, + norm_out: norm_out, + silu_out: SILU::new(), + conv_out: conv_out, + skip_connection: skip_connection, + }; + + Ok(res_block) +} + +pub fn load_multi_head_attention( + path: &str, + device: &B::Device, +) -> Result, Box> { + let n_head = load_usize::("n_head", path, device)?; + let query = load_linear::(&format!("{}/{}", path, "query"), device)?; + let key = load_linear::(&format!("{}/{}", path, "key"), device)?; + let value = load_linear::(&format!("{}/{}", path, "value"), device)?; + let out = load_linear::(&format!("{}/{}", path, "out"), device)?; + + let multi_head_attention = MultiHeadAttention { + n_head: n_head, + query: query, + key: key, + value: value, + out: out, + }; + + Ok(multi_head_attention) +} + +pub fn load_geglu(path: &str, device: &B::Device) -> Result, Box> { + let proj = load_linear::(&format!("{}/{}", path, "proj"), device)?; + + let geglue = GEGLU { + proj: proj, + gelu: Gelu::new(), // Assuming Gelu::new() initializes a new Gelu struct + }; + + Ok(geglue) +} + +pub fn load_mlp(path: &str, device: &B::Device) -> Result, Box> { + let geglu = load_geglu::(&format!("{}/{}", path, "geglu"), device)?; + let lin = load_linear::(&format!("{}/{}", path, "lin"), device)?; + + let mlp = MLP { + geglu: geglu, + lin: lin, + }; + + Ok(mlp) +} + +pub fn load_transformer_block( + path: &str, + device: &B::Device, +) -> Result, Box> { + let norm1 = load_layer_norm::(&format!("{}/{}", path, "norm1"), device)?; + let attn1 = load_multi_head_attention::(&format!("{}/{}", path, "attn1"), device)?; + let norm2 = load_layer_norm::(&format!("{}/{}", path, "norm2"), device)?; + let attn2 = load_multi_head_attention::(&format!("{}/{}", path, "attn2"), device)?; + let norm3 = load_layer_norm::(&format!("{}/{}", path, "norm3"), device)?; + let mlp = load_mlp::(&format!("{}/{}", path, "mlp"), device)?; + + let transformer_block = TransformerBlock { + norm1: norm1, + attn1: attn1, + norm2: norm2, + attn2: attn2, + norm3: norm3, + mlp: mlp, + }; + + Ok(transformer_block) +} + +pub fn load_spatial_transformer( + path: &str, + device: &B::Device, +) -> Result, Box> { + let norm = load_group_norm::(&format!("{}/{}", path, "norm"), device)?; + let proj_in = load_conv2d::(&format!("{}/{}", path, "proj_in"), device)?; + let transformer = load_transformer_block::(&format!("{}/{}", path, "transformer"), device)?; + let proj_out = load_conv2d::(&format!("{}/{}", path, "proj_out"), device)?; + + let spatial_transformer = SpatialTransformer { + norm: norm, + proj_in: proj_in, + transformer: transformer, + proj_out: proj_out, + }; + + Ok(spatial_transformer) +} + +pub fn load_upsample( + path: &str, + device: &B::Device, +) -> Result, Box> { + let conv = load_conv2d::(&format!("{}/{}", path, "conv"), device)?; + + let upsample = Upsample { conv: conv }; + + Ok(upsample) +} + +pub fn load_downsample( + path: &str, + device: &B::Device, +) -> Result, Box> { + load_conv2d(path, device) +} + +pub fn load_res_transformer_res( + path: &str, + device: &B::Device, +) -> Result, Box> { + let res1 = load_res_block::(&format!("{}/{}", path, "res1"), device)?; // Assuming load_res_block function + let transformer = + load_spatial_transformer::(&format!("{}/{}", path, "transformer"), device)?; + let res2 = load_res_block::(&format!("{}/{}", path, "res2"), device)?; + + let res_transformer_res = ResTransformerRes { + res1: res1, + transformer: transformer, + res2: res2, + }; + + Ok(res_transformer_res) +} + +pub fn load_res_transformer_upsample( + path: &str, + device: &B::Device, +) -> Result, Box> { + let res = load_res_block::(&format!("{}/{}", path, "res"), device)?; + let transformer = + load_spatial_transformer::(&format!("{}/{}", path, "transformer"), device)?; + let upsample = load_upsample::(&format!("{}/{}", path, "upsample"), device)?; + + let res_transformer_upsample = ResTransformerUpsample { + res: res, + transformer: transformer, + upsample: upsample, + }; + + Ok(res_transformer_upsample) +} + +pub fn load_res_upsample( + path: &str, + device: &B::Device, +) -> Result, Box> { + let res = load_res_block::(&format!("{}/{}", path, "res"), device)?; + let upsample = load_upsample::(&format!("{}/{}", path, "upsample"), device)?; + + let res_upsample = ResUpSample { + res: res, + upsample: upsample, + }; + + Ok(res_upsample) +} + +pub fn load_res_transformer( + path: &str, + device: &B::Device, +) -> Result, Box> { + let res = load_res_block::(&format!("{}/{}", path, "res"), device)?; + let transformer = + load_spatial_transformer::(&format!("{}/{}", path, "transformer"), device)?; + + let res_transformer = ResTransformer { + res: res, + transformer: transformer, + }; + + Ok(res_transformer) +} + +pub fn load_unet_input_blocks( + path: &str, + device: &B::Device, +) -> Result, Box> { + let conv = load_conv2d::(&format!("{}/{}", path, "conv"), device)?; + let rt1 = load_res_transformer::(&format!("{}/{}", path, "rt1"), device)?; + let rt2 = load_res_transformer::(&format!("{}/{}", path, "rt2"), device)?; + let d1 = load_downsample::(&format!("{}/{}", path, "d1"), device)?; + let rt3 = load_res_transformer::(&format!("{}/{}", path, "rt3"), device)?; + let rt4 = load_res_transformer::(&format!("{}/{}", path, "rt4"), device)?; + let d2 = load_downsample::(&format!("{}/{}", path, "d2"), device)?; + let rt5 = load_res_transformer::(&format!("{}/{}", path, "rt5"), device)?; + let rt6 = load_res_transformer::(&format!("{}/{}", path, "rt6"), device)?; + let d3 = load_downsample::(&format!("{}/{}", path, "d3"), device)?; + let r1 = load_res_block::(&format!("{}/{}", path, "r1"), device)?; + let r2 = load_res_block::(&format!("{}/{}", path, "r2"), device)?; + + let unet_input_blocks = UNetInputBlocks { + conv: conv, + rt1: rt1, + rt2: rt2, + d1: d1, + rt3: rt3, + rt4: rt4, + d2: d2, + rt5: rt5, + rt6: rt6, + d3: d3, + r1: r1, + r2: r2, + }; + + Ok(unet_input_blocks) +} + +pub fn load_unet_output_blocks( + path: &str, + device: &B::Device, +) -> Result, Box> { + let r1 = load_res_block::(&format!("{}/{}", path, "r1"), device)?; + let r2 = load_res_block::(&format!("{}/{}", path, "r2"), device)?; + let ru = load_res_upsample::(&format!("{}/{}", path, "ru"), device)?; + let rt1 = load_res_transformer::(&format!("{}/{}", path, "rt1"), device)?; + let rt2 = load_res_transformer::(&format!("{}/{}", path, "rt2"), device)?; + let rtu1 = load_res_transformer_upsample::(&format!("{}/{}", path, "rtu1"), device)?; + let rt3 = load_res_transformer::(&format!("{}/{}", path, "rt3"), device)?; + let rt4 = load_res_transformer::(&format!("{}/{}", path, "rt4"), device)?; + let rtu2 = load_res_transformer_upsample::(&format!("{}/{}", path, "rtu2"), device)?; + let rt5 = load_res_transformer::(&format!("{}/{}", path, "rt5"), device)?; + let rt6 = load_res_transformer::(&format!("{}/{}", path, "rt6"), device)?; + let rt7 = load_res_transformer::(&format!("{}/{}", path, "rt7"), device)?; + + Ok(UNetOutputBlocks { + r1, + r2, + ru, + rt1, + rt2, + rtu1, + rt3, + rt4, + rtu2, + rt5, + rt6, + rt7, + }) +} + +pub fn load_unet(path: &str, device: &B::Device) -> Result, Box> { + let lin1_time_embed = load_linear::(&format!("{}/{}", path, "lin1_time_embed"), device)?; + let silu_time_embed = SILU::new(); // Assuming SILU::new() initializes a new SILU struct + let lin2_time_embed = load_linear::(&format!("{}/{}", path, "lin2_time_embed"), device)?; + let input_blocks = + load_unet_input_blocks::(&format!("{}/{}", path, "input_blocks"), device)?; + let middle_block = + load_res_transformer_res::(&format!("{}/{}", path, "middle_block"), device)?; + let output_blocks = + load_unet_output_blocks::(&format!("{}/{}", path, "output_blocks"), device)?; + let norm_out = load_group_norm::(&format!("{}/{}", path, "norm_out"), device)?; + let silu_out = SILU::new(); // Assuming SILU::new() initializes a new SILU struct + let conv_out = load_conv2d::(&format!("{}/{}", path, "conv_out"), device)?; + + Ok(UNet { + lin1_time_embed, + silu_time_embed, + lin2_time_embed, + input_blocks, + middle_block, + output_blocks, + norm_out, + silu_out, + conv_out, + }) +} diff --git a/crates/stable-diffusion-burn/src/model/unet/mod.rs b/crates/stable-diffusion-burn/src/model/unet/mod.rs new file mode 100644 index 0000000..1292417 --- /dev/null +++ b/crates/stable-diffusion-burn/src/model/unet/mod.rs @@ -0,0 +1,740 @@ +pub mod load; + +use burn::{ + config::Config, + module::Module, + nn::{ + self, + conv::{Conv2d, Conv2dConfig}, + PaddingConfig2d, Gelu, + }, + tensor::{backend::Backend, Int, Tensor}, +}; + +use super::groupnorm::*; +use super::silu::*; + +use super::attention::qkv_attention; + +fn timestep_embedding( + timesteps: Tensor, + dim: usize, + max_period: usize, +) -> Tensor { + let half = dim / 2; + let freqs = (Tensor::arange(0..half as i64, ×teps.device()).float() + * (-(max_period as f64).ln() / half as f64)) + .exp(); + let args = timesteps.float() * freqs; + Tensor::cat(vec![args.clone().cos(), args.sin()], 0).unsqueeze() +} + +#[derive(Config)] +pub struct UNetConfig {} + +impl UNetConfig { + pub fn init(&self, device: &B::Device) -> UNet { + let lin1_time_embed = nn::LinearConfig::new(320, 1280).init(device); + let silu_time_embed = SILU::new(); + let lin2_time_embed = nn::LinearConfig::new(1280, 1280).init(device); + + let input_blocks = UNetInputBlocks { + conv: Conv2dConfig::new([4, 320], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device), + rt1: ResTransformerConfig::new(320, 1280, 320, 768, 8).init(device), + rt2: ResTransformerConfig::new(320, 1280, 320, 768, 8).init(device), + d1: DownsampleConfig::new(320).init(device), + rt3: ResTransformerConfig::new(320, 1280, 640, 768, 8).init(device), + rt4: ResTransformerConfig::new(640, 1280, 640, 768, 8).init(device), + d2: DownsampleConfig::new(640).init(device), + rt5: ResTransformerConfig::new(640, 1280, 1280, 768, 8).init(device), + rt6: ResTransformerConfig::new(1280, 1280, 1280, 768, 8).init(device), + d3: DownsampleConfig::new(1280).init(device), + r1: ResBlockConfig::new(1280, 1280, 1280).init(device), + r2: ResBlockConfig::new(1280, 1280, 1280).init(device), + }; + + let middle_block = ResTransformerResConfig::new(1280, 1280, 1280, 768, 8).init(device); + + let output_blocks = UNetOutputBlocks { + r1: ResBlockConfig::new(2560, 1280, 1280).init(device), + r2: ResBlockConfig::new(2560, 1280, 1280).init(device), + ru: ResUpSampleConfig::new(2560, 1280, 1280).init(device), + rt1: ResTransformerConfig::new(2560, 1280, 1280, 768, 8).init(device), + rt2: ResTransformerConfig::new(2560, 1280, 1280, 768, 8).init(device), + rtu1: ResTransformerUpsampleConfig::new(1920, 1280, 1280, 768, 8).init(device), + rt3: ResTransformerConfig::new(1920, 1280, 640, 768, 8).init(device), + rt4: ResTransformerConfig::new(1280, 1280, 640, 768, 8).init(device), + rtu2: ResTransformerUpsampleConfig::new(960, 1280, 640, 768, 8).init(device), + rt5: ResTransformerConfig::new(960, 1280, 320, 768, 8).init(device), + rt6: ResTransformerConfig::new(640, 1280, 320, 768, 8).init(device), + rt7: ResTransformerConfig::new(640, 1280, 320, 768, 8).init(device), + }; + + let norm_out = GroupNormConfig::new(32, 320).init(device); + let silu_out = SILU::new(); + let conv_out = Conv2dConfig::new([320, 4], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + UNet { + lin1_time_embed, + silu_time_embed, + lin2_time_embed, + input_blocks, + middle_block, + output_blocks, + norm_out, + silu_out, + conv_out, + } + } +} + +#[derive(Module, Debug)] +pub struct UNet { + lin1_time_embed: nn::Linear, + silu_time_embed: SILU, + lin2_time_embed: nn::Linear, + input_blocks: UNetInputBlocks, + middle_block: ResTransformerRes, + output_blocks: UNetOutputBlocks, + norm_out: GroupNorm, + silu_out: SILU, + conv_out: Conv2d, +} + +impl UNet { + pub fn forward( + &self, + x: Tensor, + timesteps: Tensor, + context: Tensor, + ) -> Tensor { + let t_emb = timestep_embedding(timesteps, 320, 10000); + let emb = self.lin1_time_embed.forward(t_emb); + let emb = self.silu_time_embed.forward(emb); + let emb = self.lin2_time_embed.forward(emb); + + let mut saved_inputs = Vec::new(); + let mut x = x; + + // input blocks + for block in self.input_blocks.as_array() { + x = block.forward(x, emb.clone(), context.clone()); + saved_inputs.push(x.clone()) + } + + // middle block + x = self.middle_block.forward(x, emb.clone(), context.clone()); + + // output blocks + for block in self.output_blocks.as_array() { + x = Tensor::cat(vec![x, saved_inputs.pop().unwrap()], 1); + x = block.forward(x, emb.clone(), context.clone()); + } + + let x = self.norm_out.forward(x); + let x = self.silu_out.forward(x); + let x = self.conv_out.forward(x); + x + } +} + +#[derive(Module, Debug)] +pub struct UNetInputBlocks { + conv: Conv2d, + rt1: ResTransformer, + rt2: ResTransformer, + d1: Downsample, + rt3: ResTransformer, + rt4: ResTransformer, + d2: Downsample, + rt5: ResTransformer, + rt6: ResTransformer, + d3: Downsample, + r1: ResBlock, + r2: ResBlock, +} + +impl UNetInputBlocks { + fn as_array(&self) -> [&dyn UNetBlock; 12] { + [ + &self.conv, &self.rt1, &self.rt2, &self.d1, &self.rt3, &self.rt4, &self.d2, &self.rt5, + &self.rt6, &self.d3, &self.r1, &self.r2, + ] + } +} + +#[derive(Module, Debug)] +pub struct UNetOutputBlocks { + r1: ResBlock, + r2: ResBlock, + ru: ResUpSample, + rt1: ResTransformer, + rt2: ResTransformer, + rtu1: ResTransformerUpsample, + rt3: ResTransformer, + rt4: ResTransformer, + rtu2: ResTransformerUpsample, + rt5: ResTransformer, + rt6: ResTransformer, + rt7: ResTransformer, +} + +impl UNetOutputBlocks { + fn as_array(&self) -> [&dyn UNetBlock; 12] { + [ + &self.r1, &self.r2, &self.ru, &self.rt1, &self.rt2, &self.rtu1, &self.rt3, &self.rt4, + &self.rtu2, &self.rt5, &self.rt6, &self.rt7, + ] + } +} + +trait UNetBlock { + fn forward(&self, x: Tensor, emb: Tensor, context: Tensor) -> Tensor; +} + +#[derive(Config)] +pub struct ResTransformerConfig { + n_channels_in: usize, + n_channels_embed: usize, + n_channels_out: usize, + n_context_state: usize, + n_head: usize, +} + +impl ResTransformerConfig { + fn init(&self, device: &B::Device) -> ResTransformer { + let res = ResBlockConfig::new( + self.n_channels_in, + self.n_channels_embed, + self.n_channels_out, + ) + .init(device); + let transformer = + SpatialTransformerConfig::new(self.n_channels_out, self.n_context_state, self.n_head) + .init(device); + + ResTransformer { res, transformer } + } +} + +#[derive(Module, Debug)] +pub struct ResTransformer { + res: ResBlock, + transformer: SpatialTransformer, +} + +impl UNetBlock for ResTransformer { + fn forward(&self, x: Tensor, emb: Tensor, context: Tensor) -> Tensor { + let x = self.res.forward(x, emb); + let x = self.transformer.forward(x, context); + x + } +} + +#[derive(Config)] +pub struct ResUpSampleConfig { + n_channels_in: usize, + n_channels_embed: usize, + n_channels_out: usize, +} + +impl ResUpSampleConfig { + fn init(&self, device: &B::Device) -> ResUpSample { + let res = ResBlockConfig::new( + self.n_channels_in, + self.n_channels_embed, + self.n_channels_out, + ) + .init(device); + let upsample = UpsampleConfig::new(self.n_channels_out).init(device); + + ResUpSample { res, upsample } + } +} + +#[derive(Module, Debug)] +pub struct ResUpSample { + res: ResBlock, + upsample: Upsample, +} + +impl UNetBlock for ResUpSample { + fn forward(&self, x: Tensor, emb: Tensor, _context: Tensor) -> Tensor { + let x = self.res.forward(x, emb); + let x = self.upsample.forward(x); + x + } +} + +#[derive(Config)] +pub struct ResTransformerUpsampleConfig { + n_channels_in: usize, + n_channels_embed: usize, + n_channels_out: usize, + n_context_state: usize, + n_head: usize, +} + +impl ResTransformerUpsampleConfig { + fn init(&self, device: &B::Device) -> ResTransformerUpsample { + let res = ResBlockConfig::new( + self.n_channels_in, + self.n_channels_embed, + self.n_channels_out, + ) + .init(device); + let transformer = + SpatialTransformerConfig::new(self.n_channels_out, self.n_context_state, self.n_head) + .init(device); + let upsample = UpsampleConfig::new(self.n_channels_out).init(device); + + ResTransformerUpsample { + res, + transformer, + upsample, + } + } +} + +#[derive(Module, Debug)] +pub struct ResTransformerUpsample { + res: ResBlock, + transformer: SpatialTransformer, + upsample: Upsample, +} + +impl UNetBlock for ResTransformerUpsample { + fn forward(&self, x: Tensor, emb: Tensor, context: Tensor) -> Tensor { + let x = self.res.forward(x, emb); + let x = self.transformer.forward(x, context); + let x = self.upsample.forward(x); + x + } +} + +#[derive(Config)] +pub struct ResTransformerResConfig { + n_channels_in: usize, + n_channels_embed: usize, + n_channels_out: usize, + n_context_state: usize, + n_head: usize, +} + +impl ResTransformerResConfig { + fn init(&self, device: &B::Device) -> ResTransformerRes { + let res1 = ResBlockConfig::new( + self.n_channels_in, + self.n_channels_embed, + self.n_channels_out, + ) + .init(device); + let transformer = + SpatialTransformerConfig::new(self.n_channels_out, self.n_context_state, self.n_head) + .init(device); + let res2 = ResBlockConfig::new( + self.n_channels_in, + self.n_channels_embed, + self.n_channels_out, + ) + .init(device); + + ResTransformerRes { + res1, + transformer, + res2, + } + } +} + +#[derive(Module, Debug)] +pub struct ResTransformerRes { + res1: ResBlock, + transformer: SpatialTransformer, + res2: ResBlock, +} + +impl UNetBlock for ResTransformerRes { + fn forward(&self, x: Tensor, emb: Tensor, context: Tensor) -> Tensor { + let x = self.res1.forward(x, emb.clone()); + let x = self.transformer.forward(x, context); + let x = self.res2.forward(x, emb); + x + } +} + +#[derive(Config)] +pub struct UpsampleConfig { + n_channels: usize, +} + +impl UpsampleConfig { + fn init(&self, device: &B::Device) -> Upsample { + let conv = Conv2dConfig::new([self.n_channels, self.n_channels], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + Upsample { conv } + } +} + +#[derive(Module, Debug)] +pub struct Upsample { + conv: Conv2d, +} + +impl Upsample { + fn forward(&self, x: Tensor) -> Tensor { + let [n_batch, n_channel, height, width] = x.dims(); + let x = x + .reshape([n_batch, n_channel, height, 1, width, 1]) + .repeat(&[1, 1, 1, 2, 1, 2]) + .reshape([n_batch, n_channel, 2 * height, 2 * width]); + self.conv.forward(x) + } +} + +impl UNetBlock for Upsample { + fn forward(&self, x: Tensor, _emb: Tensor, _context: Tensor) -> Tensor { + self.forward(x) + } +} + +#[derive(Config)] +pub struct DownsampleConfig { + n_channels: usize, +} + +impl DownsampleConfig { + fn init(&self, device: &B::Device) -> Conv2d { + Conv2dConfig::new([self.n_channels, self.n_channels], [3, 3]) + .with_stride([2, 2]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device) + } +} + +type Downsample = Conv2d; + +impl UNetBlock for Conv2d { + fn forward(&self, x: Tensor, _emb: Tensor, _context: Tensor) -> Tensor { + self.forward(x) + } +} + +#[derive(Config)] +pub struct SpatialTransformerConfig { + n_channels: usize, + n_context_state: usize, + n_head: usize, +} + +impl SpatialTransformerConfig { + fn init(&self, device: &B::Device) -> SpatialTransformer { + let norm = GroupNormConfig::new(32, self.n_channels).init(device); + let proj_in = Conv2dConfig::new([self.n_channels, self.n_channels], [1, 1]).init(device); + let transformer = + TransformerBlockConfig::new(self.n_channels, self.n_context_state, self.n_head).init(device); + let proj_out = Conv2dConfig::new([self.n_channels, self.n_channels], [1, 1]).init(device); + + SpatialTransformer { + norm, + proj_in, + transformer, + proj_out, + } + } +} + +#[derive(Module, Debug)] +pub struct SpatialTransformer { + norm: GroupNorm, + proj_in: Conv2d, + transformer: TransformerBlock, + proj_out: Conv2d, +} + +impl SpatialTransformer { + fn forward(&self, x: Tensor, context: Tensor) -> Tensor { + let [n_batch, n_channel, height, width] = x.dims(); + + let x_in = x.clone(); + + let x = self.norm.forward(x); + let x = self.proj_in.forward(x); + let x = x + .reshape([n_batch, n_channel, height * width]) + .swap_dims(1, 2); + + let x = self + .transformer + .forward(x, context) + .swap_dims(1, 2) + .reshape([n_batch, n_channel, height, width]); + + x_in + self.proj_out.forward(x) + } +} + +#[derive(Config)] +pub struct TransformerBlockConfig { + n_state: usize, + n_context_state: usize, + n_head: usize, +} + +impl TransformerBlockConfig { + fn init(&self, device: &B::Device) -> TransformerBlock { + let norm1 = nn::LayerNormConfig::new(self.n_state).init(device); + let attn1 = MultiHeadAttentionConfig::new(self.n_state, self.n_state, self.n_head).init(device); + let norm2 = nn::LayerNormConfig::new(self.n_state).init(device); + let attn2 = + MultiHeadAttentionConfig::new(self.n_state, self.n_context_state, self.n_head).init(device); + let norm3 = nn::LayerNormConfig::new(self.n_state).init(device); + let mlp = MLPConfig::new(self.n_state, 4).init(device); + + TransformerBlock { + norm1, + attn1, + norm2, + attn2, + norm3, + mlp, + } + } +} + +#[derive(Module, Debug)] +pub struct TransformerBlock { + norm1: nn::LayerNorm, + attn1: MultiHeadAttention, + norm2: nn::LayerNorm, + attn2: MultiHeadAttention, + norm3: nn::LayerNorm, + mlp: MLP, +} + +impl TransformerBlock { + fn forward(&self, x: Tensor, context: Tensor) -> Tensor { + let x = x.clone() + self.attn1.forward(self.norm1.forward(x), None); + let x = x.clone() + self.attn2.forward(self.norm2.forward(x), Some(context)); + x.clone() + self.mlp.forward(self.norm3.forward(x)) + } +} + +#[derive(Config)] +pub struct MLPConfig { + n_state: usize, + mult: usize, +} + +impl MLPConfig { + pub fn init(&self, device: &B::Device) -> MLP { + let n_state_hidden = self.n_state * self.mult; + let geglu = GEGLUConfig::new(self.n_state, n_state_hidden).init(device); + let lin = nn::LinearConfig::new(n_state_hidden, self.n_state).init(device); + + MLP { geglu, lin } + } +} + +#[derive(Module, Debug)] +pub struct MLP { + geglu: GEGLU, + lin: nn::Linear, +} + +impl MLP { + pub fn forward(&self, x: Tensor) -> Tensor { + self.lin.forward(self.geglu.forward(x)) + } +} + +#[derive(Config)] +pub struct GEGLUConfig { + n_state_in: usize, + n_state_out: usize, +} + +impl GEGLUConfig { + fn init(&self, device: &B::Device) -> GEGLU { + let proj = nn::LinearConfig::new(self.n_state_in, 2 * self.n_state_out).init(device); + let gelu = Gelu::new(); + + GEGLU { proj, gelu } + } +} + +#[derive(Module, Debug)] +pub struct GEGLU { + proj: nn::Linear, + gelu: Gelu, +} + +impl GEGLU { + fn forward(&self, x: Tensor) -> Tensor { + let projected = self.proj.forward(x); + let [n_batch, n_ctx, n_state] = projected.dims(); + + let n_state_out = n_state / 2; + + let x = projected + .clone() + .slice([0..n_batch, 0..n_ctx, 0..n_state_out]); + let gate = projected.slice([0..n_batch, 0..n_ctx, n_state_out..n_state]); + + x * self.gelu.forward(gate) + } +} + +#[derive(Config)] +pub struct MultiHeadAttentionConfig { + n_state: usize, + n_context_state: usize, + n_head: usize, +} + +impl MultiHeadAttentionConfig { + fn init(&self, device: &B::Device) -> MultiHeadAttention { + assert!( + self.n_state % self.n_head == 0, + "State size {} must be a multiple of head size {}", + self.n_state, + self.n_head + ); + + let n_head = self.n_head; + let query = nn::LinearConfig::new(self.n_state, self.n_state) + .with_bias(false) + .init(device); + let key = nn::LinearConfig::new(self.n_context_state, self.n_state) + .with_bias(false) + .init(device); + let value = nn::LinearConfig::new(self.n_context_state, self.n_state) + .with_bias(false) + .init(device); + let out = nn::LinearConfig::new(self.n_state, self.n_state).init(device); + + MultiHeadAttention { + n_head, + query, + key, + value, + out, + } + } +} + +#[derive(Module, Debug)] +pub struct MultiHeadAttention { + n_head: usize, + query: nn::Linear, + key: nn::Linear, + value: nn::Linear, + out: nn::Linear, +} + +impl MultiHeadAttention { + pub fn forward(&self, x: Tensor, context: Option>) -> Tensor { + let xa = context.unwrap_or_else(|| x.clone()); + + let q = self.query.forward(x); + let k = self.key.forward(xa.clone()); + let v = self.value.forward(xa); + + let wv = qkv_attention(q, k, v, None, self.n_head); + + self.out.forward(wv) + } +} + +#[derive(Config)] +pub struct ResBlockConfig { + n_channels_in: usize, + n_channels_embed: usize, + n_channels_out: usize, +} + +impl ResBlockConfig { + fn init(&self, device: &B::Device) -> ResBlock { + let norm_in = GroupNormConfig::new(32, self.n_channels_in).init(device); + let silu_in = SILU::new(); + let conv_in = Conv2dConfig::new([self.n_channels_in, self.n_channels_out], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + let silu_embed = SILU::new(); + let lin_embed = nn::LinearConfig::new(self.n_channels_embed, self.n_channels_out).init(device); + + let norm_out = GroupNormConfig::new(32, self.n_channels_out).init(device); + let silu_out = SILU::new(); + let conv_out = Conv2dConfig::new([self.n_channels_out, self.n_channels_out], [3, 3]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(device); + + let skip_connection = if self.n_channels_in != self.n_channels_out { + Some(Conv2dConfig::new([self.n_channels_in, self.n_channels_out], [1, 1]).init(device)) + } else { + None + }; + + ResBlock { + norm_in, + silu_in, + conv_in, + silu_embed, + lin_embed, + norm_out, + silu_out, + conv_out, + skip_connection, + } + } +} + +#[derive(Module, Debug)] +pub struct ResBlock { + norm_in: GroupNorm, + silu_in: SILU, + conv_in: Conv2d, + silu_embed: SILU, + lin_embed: nn::Linear, + norm_out: GroupNorm, + silu_out: SILU, + conv_out: Conv2d, + skip_connection: Option>, +} + +impl ResBlock { + fn forward(&self, x: Tensor, embed: Tensor) -> Tensor { + let h = self.norm_in.forward(x.clone()); + let h = self.silu_in.forward(h); + let h = self.conv_in.forward(h); + + let embed_out = self.silu_embed.forward(embed); + let embed_out = self.lin_embed.forward(embed_out); + + let [n_batch_embed, n_state_embed] = embed_out.dims(); + let h = h + embed_out.reshape([n_batch_embed, n_state_embed, 1, 1]); + + let h = self.norm_out.forward(h); + let h = self.silu_out.forward(h); + let h = self.conv_out.forward(h); + + if let Some(skipc) = self.skip_connection.as_ref() { + skipc.forward(x) + h + } else { + x + h + } + } +} + +impl UNetBlock for ResBlock { + fn forward(&self, x: Tensor, emb: Tensor, _context: Tensor) -> Tensor { + self.forward(x, emb) + } +} diff --git a/crates/stable-diffusion-burn/src/token.rs b/crates/stable-diffusion-burn/src/token.rs new file mode 100644 index 0000000..23202dd --- /dev/null +++ b/crates/stable-diffusion-burn/src/token.rs @@ -0,0 +1,54 @@ +use std::result; +use rust_tokenizers::{error::TokenizerError, tokenizer::{Tokenizer, SentencePieceBpeTokenizer, TruncationStrategy}, vocab::Vocab}; + +const BOS_TOKEN_ID: i64 = 1; +const EOS_TOKEN_ID: i64 = 2; + +pub type Result = result::Result; + +pub struct LlamaTokenizer { + spm: SentencePieceBpeTokenizer, +} + +impl LlamaTokenizer { + pub fn new(tokenizer_path: &str) -> Result { + let lower_case = false; + SentencePieceBpeTokenizer::from_file(tokenizer_path, lower_case) + .map(|spm| Self { spm } ) + } + + pub fn encode(&self, text: &str, include_bos: bool, include_eos: bool) -> Vec { + let pre = if include_bos { + vec![BOS_TOKEN_ID] + } else { + vec![] + }; + + let post = if include_eos { + vec![EOS_TOKEN_ID] + } else { + vec![] + }; + + let token_ids = self.spm.encode(text, None, std::usize::MAX, &TruncationStrategy::LongestFirst, 0).token_ids; + + [pre, token_ids, post] + .into_iter() + .flat_map(|v| v.into_iter()) + .collect() + } + + pub fn decode(&self, tokens: &[i64], skip_special_tokens: bool) -> String { + let clean_spaces = false; + self.spm.decode(tokens, skip_special_tokens, clean_spaces) + } + + pub fn vocab_size(&self, include_special_tokens: bool) -> usize { + let vocab = self.spm.vocab(); + if include_special_tokens { + vocab.values().len() + vocab.special_values().len() + } else { + vocab.values().len() + } + } +} diff --git a/crates/stable-diffusion-burn/src/tokenizer.rs b/crates/stable-diffusion-burn/src/tokenizer.rs new file mode 100644 index 0000000..0ac76bb --- /dev/null +++ b/crates/stable-diffusion-burn/src/tokenizer.rs @@ -0,0 +1,222 @@ +use regex::Regex; +use std::collections::HashMap; + +use std::fs::File; +use std::io::{self, BufRead}; + +fn bytes_to_unicode() -> Vec<(u8, char)> { + let mut bs: Vec = ('!' as u8..='~' as u8) + .into_iter() + .chain(('¡' as u8..='¬' as u8).into_iter()) + .chain(('®' as u8..='ÿ' as u8).into_iter()) + .collect(); + + let mut cs: Vec<_> = bs.iter().cloned().map(char::from).collect(); + + let mut n = 0; + for b in 0u8..=255u8 { + if !bs.contains(&b) { + bs.push(b); + cs.push(char::from_u32(256 + n).unwrap()); + n += 1; + } + } + + bs.into_iter() + .zip(cs.into_iter().map(|c| c.into())) + .collect() +} + +fn get_pairs(word: &[String]) -> Vec<(String, String)> { + let prev = word.into_iter().cloned(); + let next = prev.clone().skip(1); + + prev.zip(next).collect() +} + +fn whitespace_clean(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +fn load_merges(path: &str) -> io::Result> { + let file = File::open(&path)?; + let reader = io::BufReader::new(file); + + let mut merges = Vec::new(); + + for line in reader.lines() { + let line = line?; + let mut words = line.split_whitespace(); + + if let (Some(word1), Some(word2)) = (words.next(), words.next()) { + merges.push((word1.into(), word2.into())); + } + } + + Ok(merges) +} + +fn construct_vocab( + chars: impl Iterator + Clone, + merges: &[(String, String)], +) -> Vec { + let iter = chars.map(String::from); + let mut vocab: Vec<_> = iter.clone().chain(iter.map(|c| c + "")).collect(); + + for merge in merges { + vocab.push(format!("{}{}", merge.0, merge.1)); + } + + vocab.extend(["<|startoftext|>".to_string(), "<|endoftext|>".to_string()]); + + return vocab; +} + +pub struct SimpleTokenizer { + byte_encoder: HashMap, + byte_decoder: HashMap, + encoder: HashMap, + decoder: HashMap, + bpe_ranks: HashMap<(String, String), u32>, + cache: HashMap, + pat: Regex, +} + +impl SimpleTokenizer { + pub fn new() -> io::Result { + let byte_unicode_values = bytes_to_unicode(); + + let byte_encoder: HashMap<_, _> = byte_unicode_values.iter().cloned().collect(); + let byte_decoder = byte_encoder.iter().map(|(k, v)| (*v, *k)).collect(); + + let merges = load_merges("bpe_simple_vocab_16e6.txt")?; + let merges = merges[1..49152 - 256 - 2 + 1].to_vec(); + + let vocab = construct_vocab(byte_unicode_values.into_iter().map(|(_, u)| u), &merges[..]); + + let encoder: HashMap = vocab.iter().cloned().zip((0..).into_iter()).collect(); + let decoder: HashMap = encoder.iter().map(|(k, v)| (*v, k.clone())).collect(); + let bpe_ranks = merges.iter().cloned().zip((0..).into_iter()).collect(); + let cache = HashMap::from([ + ("<|startoftext|>".to_string(), "<|startoftext|>".to_string()), + ("<|endoftext|>".to_string(), "<|endoftext|>".to_string()), + ]); + + let pat = Regex::new(r"(?i)<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+").unwrap(); + + Ok(SimpleTokenizer { + byte_encoder: byte_encoder, + byte_decoder: byte_decoder, + encoder: encoder, + decoder: decoder, + bpe_ranks: bpe_ranks, + cache: cache, + pat: pat, + }) + } + + pub fn bpe(&self, token: &str) -> String { + if let Some(word) = self.cache.get(token) { + return word.clone(); + } + + let mut word: Vec = token.chars().map(|c| c.to_string()).collect(); + word.last_mut().map(|w| *w += ""); + let mut pairs = get_pairs(&word); + + if pairs.is_empty() { + return format!("{}{}", token, ""); + } + + loop { + let bigram = pairs + .iter() + .filter(|pair| self.bpe_ranks.contains_key(pair)) + .min_by_key(|&pair| self.bpe_ranks[pair]); + + if bigram.is_none() { + break; + } + + let (first, second) = bigram.unwrap(); + let mut new_word = Vec::new(); + let mut i = 0; + while i < word.len() { + if let Some((j, _)) = word.iter().enumerate().skip(i).find(|(_, w)| w == &first) { + new_word.extend(word[i..j].iter().cloned()); + i = j; + } else { + new_word.extend(word[i..].iter().cloned()); + break; + } + + if &word[i] == first && i < word.len() - 1 && &word[i + 1] == second { + new_word.push(format!("{}{}", first, second)); + i += 2; + } else { + new_word.push(word[i].clone()); + i += 1; + } + } + + word = new_word; + if word.len() == 1 { + break; + } else { + pairs = get_pairs(&word[..]) + } + } + + let word = word.join(" "); + //self.cache.insert(token.into(), word); + return word; + } + + pub fn encode(&self, text: &str) -> Vec { + let cleaned_text = whitespace_clean(text.trim()).to_lowercase(); + + let mut bpe_tokens: Vec = Vec::new(); + + for m in self.pat.find_iter(&cleaned_text) { + let token = m.as_str(); + let token: String = token + .as_bytes() + .into_iter() + .map(|b| self.byte_encoder[b]) + .collect(); + bpe_tokens.extend( + self.bpe(&token) + .split(' ') + .map(|bpe_token| self.encoder[bpe_token]), + ) + } + + return bpe_tokens; + } + + pub fn decode(&self, tokens: &[u32]) -> String { + let text: String = tokens.iter().map(|t| self.decoder[t].as_str()).collect(); + let decoded_bytes: Vec = text.chars().map(|c| self.byte_decoder[&c]).collect(); + + String::from_utf8_lossy(&decoded_bytes[..]).replace("", " ") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_encode_decode() { + let tokenizer = SimpleTokenizer::new().unwrap(); + + let text = "Hello world! <|startoftext|>asdf<|startoftext|>"; + let target_encode = [3306, 1002, 256, 49406, 587, 10468, 49406]; + let target_decode = "hello world ! <|startoftext|>asdf <|startoftext|>"; // extra spaces sometimes + + let encoded = tokenizer.encode(&text); + assert_eq!(&target_encode[..], &encoded[..]); + let decoded = tokenizer.decode(&encoded[..]); + assert_eq!(target_decode, decoded); + } +}

?%C)&zRuEYQ_V!|Sz{-|IzfS<~P|L?B(2{^g zI*t!}>WSv#fW~f3T~S6?E&1dH`fUzwodPX)Cx1RMmtwAX{4Od!ey8>B@6n&6d*#)) zI0c_GQtKv6Zyl_Dnz%34*RLpKwyNnSd*F7BPv3XH2!W@67P(%2tZSDSOvBO=<^S?) zO`3`y4jS>06>sLU9Nc>&%at>M8aS{7rV$q;Yagk#k0YBgiNWO+W@w-NL683l+wW$F zYVaO)4|I!r0-N)8ZwgR|PPph9j*>i$nrsh;d%_WXc*M?SZJyejaCs=3XupDtPXp1u zVW|=*or&>xMIdYFe@hNQ#c-l;Rsp6Z@{L>83^1$*U{6HW3u~P`pz|z>}bWEC!er1n{#MhTX4TR)#hW_oh0*nBzoRr z5fCn{&xfE5w1On8&%GV zC>YBxUQ@}zOJq<8vLMil-ufDRC?be>4A`#O3-O@s(Jx)zFL?PQWXIn@18U^@TM^gP zm$R|mLr}MFhk&vqy;90@42aOk93a#PZ+n{8fP$e)rO;2emRrM_ykWLR62qnyiUxkp zVV6etBDcTRKn|OB!YxE-L8(?#$B%k;2tXgeM`~rRwkcbHKoNKi)eNr4+kVM-wJVM1 z{r=(jcWv-LM9#mzp$}|56}9KNKN-XYx-GZ5;t3ML*0wv%5EWGhI4zY+4*wE`9y!WO z(FgpC<2RFYvmmomNtVw$n4LPAD6OcNO7ktnG~fd@E)&r@9O69^R?Hm*DRMiSX&p=% zM}c?(oIP5O3VkN5veUwgBS`l`2=d=n`b9dnf66WMiQv0@PS)tD47LFkwP(;b4oRM+ zS4IzFvSa#;sNkO} zAB$by2U^5g#v0uPbX&_p4mKdYG>Fkhb`C&h#UuBkO3pIQgq!j_Adw%KU|SOOxT)N_ zLMi(}%b;oY42re&ok*p=J~>ZUm3w;8bzJcP-;o{PIb439LeF!GiJo4!wa1y9)mO;% zID?nC5jXy1Xosvs9X^}wi+hCbG2Y4&xnA3r^S~|8k)BJsFA$y*fRGy@MeY8_Y1c`N zJbfs?vnG??D8!eNb7XT9!b-l(iWW@Sq5ZHrD`6CgV|u>NE=_JGI@)Wkb)mXcui`4^ z%P1==;i~Gm#&ghTnMQ-JN_2dzXY=PS&*IzxQad($7BezrDim)kGAT&X#@iV(eY#?0 zhDjJY&UT@dlIQ6PrPyL&@XK7wXfheOYfVW!`7PhIY8fL&wu@*=wp-{+O5z4z^~PQ$ z4B6NH#|4T3%O&oGhCYoNZ(EXuR1EZIEux?1#v<=VUWw2C`)8vHYFxLxF6m>{_g{QY zeg99Zy=BLrThZZFwI4{2tTp)!*FG+|fAiqod(@=iPck!9w%OxpJ)1mrEu89H5?Mhu zvoA%j^J-!edJI)BXhY_n=+?ZDZi>yo{aCJ^;pA9M0Ac_F7`W;}>F=82FG02ojWs#G@0sZMiI*j|wfJ zQZ+nsq)lLO`8c#lgb5B63D+I7qNo>K%jWs!^RdP*y{{;`qKoyQBXEEI|hOL)s zXT)PPY@Z`heTp5ZXgjxZG!Mzr)RWu#`&(6y*~jAI5b@m-eD^);0PpvUdlp+Vm;N<9 z@7-^I`ahXJepjsO02~|+K7f&>#!ifnk6&x~lW6W4>)OeI{&l$L3NgG}7LAoLbYgG5 znZSOjW5O?;x9Zf4#)hS4;mBOsN>l9k2m|SZDZ%Z{7@X}t<&XkaBHQ|Lwh~$CKCnc_ z_ZPpDg!4b(M*2v>5W2#UB{Kz8D3-h90Kun))ATfTD;$Nh7l_sx0^eu*fOkSgklk<{ z5$50#k4S4*DE=QQjF-SWyRegM^VNBQ)S#d3;t+u0r8hIbt~S&~O7<16EmBpt13MFF z4v|P#PSZS6e;x;*7Y7`JJ8C2Kybtv*L8#`aAelNK_ds1BR`NE{bA|7k%|?2EXb`VG z*Wb+Mv*fuU_LiEB`Fx_i!#Nz2>c5vJ8$G0dC0RYI?>+fnpA_Wj>8`4@H#?WRMUKUs zNus@<2*k`Q>blmiInveR@}Al(raDc$a45Gl{CO%%jB)*7Ig7$i3aR-ucRb^KoV_j+ zrfH%iQk6)Y2bF|dy_>-sevp{VvB#cwpW&x`$vNT_yX#CPKh_x0^BmmI$njpW2_szq zt!d~Cw^gWSRk?uq;(KU8z&TVM7nKxCsU?1-Eh5T~J!QD!~tNIj`{Rmo&!uArPE? zm7a)*oX!W%2vmk|u0COafGx4!HQ3sB#K1|megtvj`e1d8p(1SC8cpP;7HeA5goa`W zEwilKj9i&KTRPJNg;g&=s}nM^I|Xv)DO%eMo3< zHKYxff$36nXa03^H~&Uxh~l?Pj>^#kTr*n#ohG|4A@x$lXn!9va(jfvxINXHXY%Tn ze^{LfFCzn&YHG;>;&2;#^t{+qC_jaG^0T#Ct~+5LB(kvs==|xY8AW78LLzbHFVL8s zL2;WZ)iJK0;l^qCW$J|to(&2P4BW49HE4(I7M-i6T(~-=ZoM6oi@C>g&)$@4vHGg1 zEmYL;`re)dLJlMRf8}0>&am(+*dH(`_mz-iz*OG%zwY0?M26=o!cC6J)7Zclxrlp~ zpP{MvK5OzCK3FPJW0q3o4zQKvR>@r z$KsWB%9E~tzBvAnN4-?6Y@J9m%pyE1*&`oGXU=q5yQe2of5u&&ZWUh=Ghd92fW}7? ztme!3U|xQZ!`B8vq5J$t6DKB4rmmko$>z$JBFjc9h?|_1h3Z4-mPKGF0@vf(A?%#Y z-O3I9X`!tIyq+kyBdI z)^gC`Rtu#gvAD~utJ0G~aLY{hYuMY(;c*|<@*e4`dqTE+oaN*4{(HZ^F#|x9gAHpQwH)PBtG}m6+Sy)FSR8=+? zAyc=z2>No3<7Q17dQyZ@j#;uZwBeuOjE#B)F5*P3rjE>aw0ejqSd}5Yn+~BUGW&}L zeYj6w)tbIhVg5)M$Kx|~Y2lpiibgb}108%UtkNPQx=h}UO$*<3ln@pfDpoRmWN^1P zuX+eXd!q+P8@KRKJ)S;BlbWA#yP5O4nwSR17A^7!lz1{)wHt`=ih#H>EoTSS^Zadu zU>6#tEt$A2k47Bf-SRr#D>8PzE1mIb;w$Qp3U2$<`;ixKVGbAiLTp*pY}-{q22FtI zjjW*h0Q5BUm=t)C;LsO4TWPj_Y7nq-%5g56AoVGSuEQKza+%$S6$owAVXpM zCiG2HOnu|tg8pN_`L{d&e*W(}6NI$A^c$`j>b7bae<6`%FDEh22S*~MT=2(G_IN$b zt53iO3U=};(NVhr_u_ghUc`u0 z69^m}n;LkWmb%eyMS9CgkuDT8+!G4PJoyjD=Z`0JifUp;RG>{z&)ZzpNd#Qg##iUe zebDT@!ZGIl?YoQWl56?KL4h1^>{hb^Y4c%V3&J&?>%863*XvVAv+WORoxSoT>Xx%?yD1!yXD@(!ShH_=6c;gl@fpcUGFB7-Y ze$dxVAGJNdwi%dvtXMKJ(^$H+so@JwUv4*N2W=OjQae4x)k7Zq3EGWI5xRl8jX>$~ z-e*P8uPRpfD1Pv0uVrzpdJlumghp3h&l&PV1Hd%fPQqB=aPWQC0*kI7pr>0Ql@-Bi#78(#%L}Pso zgw)>Scgarem|LhF4E)+m&MDW+e9t8@fq&e0Z?9I0qdNTU25EQEMDYw7fRDUKBTA?; z`Ww`=scd#AcaT;w1m~k*!K^CJ^86|02v>M=7Nd~cX_E#kGeddRqvQ`_FJuhe$;Z;) z-L+z2{WKX^*YmOUin>hdBZX?C#UaphP;p*&-!ETRsq zz%Cyl>BzcX7z7Slq*Mf0ZJ3PF53bFj;}m ze*|xQzWJ(kN)|TGNeF8`*&%1>c)v)XNg|Vr#xzx{{W_4gyTgh+4|XTKsZJXDZ_YiP z%D2`>WV-~1oOg}QeF8(}_sYR-ny(_TI!Hr@+PvS!#Os3S&MemG3Y>L0@OA5A(V9{= zo>zBfLb7}dI)cuv96dFWm;hk3-D|Nvhiy#<_fq&t+noi*?TU$&RjYU*ykMbzSOd&jPWO|}gV2hAZ99Xvj2Zk>#I4Es>G1g0FFnv_D#x>QTzL2)&9J5;fnk5) zJ$KvUPDXRC#G`aJ?b^!_SuYw#6)rSWz!irI*t*24YD39Nwaj)mnGm0Pg)VD%1NNU706D`ug)Of~v zIE6~$QBkNUxSaX0rO+Kxv{wC^zBb>O$?05Fh5?`5+}TYj&r8-HMIbi6KuP=QMMWS- z^92nbnV&L_+?+}7(Ny#{80cC9BDZN?gX*RGeX29C6UE`vysZXqo-l=JRXMeiU#((m zVbtS{dXUgMN=H@e5-5v$EhJJ2vcPBy<$9~Ho01j%6z&iM z^ikR7G5Kw%pLBJt7p)*hx(3EsAunDUy3NI;l-pKs@G=vYK<}ZpQ(%*h7b?)*&3vup zWmV+Y!mIeLLO3F@z9%-e{7Y+9UI0izrf%lP>FM%7LuKzEIIdPRI5DJBTlks(TRGMb zgElRhZcrXf(?UXarBCPEPpz{buA6d-AGdhV_jU6YEZn;JGsPcdoGw`01%Knyf8PJ) z>#WM#%_Q42Y#0RO5uUGO)T|?NXOip|YjydHR>N{~q4oE)^O7ve`+Vj~P zFtOtO*tCf@hEq(@Es_4=@hx}SCrUXq?U0O7kV2C$r%N}ixb458k&afzRwkv*36}C1 zy*)+~E7^UaqSg9YLcWK zYCSnnya|Il7i-dFlhtT7k7l1?swm;5MGkiBk`a0lgA|z*nx)g2WZ!MkxaX7tYum@} z^?h5|y)UVtw<+rPL6O7+WPk<|{);mcZN02ih4n<5dZKAES2iOiEj!`rPoH(5Jl~*-ZwzwW&mBY6rs+0v#lp3V-85hOOa|}7 z^!S6W$O1TvJsXoueJJx3tlje5aJSWEr(t3}Z<8Ya+HBqQhhPelg#GEOlTE~x2MMH& zzoDp364ukJj>iO^G^0Gfjm5}U;}p75r&Z#(ed&Wt1R|;D#Fe=VlWQWzn}vo7=E0u8 zI2sJvVHxJ19K{qWVxl{hH6B}@MR4U95+~fUbczmsI&+%Za9~*kq+YT2ardj zq5PZ$NlKz`=`yeL`A+nBAhc2@bTfqAmXy+50L((hD*IKjhxZ2V^=!WSf%uejA7X=dhay^tAC5c(W0m@RpYz}q_ZZQ456@gm zg$)PJa%G*2Mm604$^#9W9(FW_T@i9dM~C=;gt~n?`naL8D*^8rAVvBxME97&rX3WA zc=A?cZp_rDI^3w73ZF?h-GkE$efk&$}f|3HZ%p6gLXo-CNjaQeF4 zvT(Iq77%$J$jx%0h(WX7&RPoICg~YVN4H+@3RzT^sj)m_*k2jaJ>TO(s99@SEsXWX!ryG@WGL{?^}tBLLSxs2cC3xho#OgXk^6y%2Mq3iCy7#9 zPU!hOQ*}?XWaZF3#;$-$7K>o%mapck|`Q@)51DE&XiL?&Te3iRcbitXvt=;Ni7CPkb&otQEVJ?I-rCfTklDGoKUB70mKd|0M7DXf zU#)xG=X$*-ftvh}-Vxivg;s89rH=jO6!F?lgWH4%UhYFzjBv!;))x9hHPi2HTvmWr zj^l`Hf*tuD1Fdu;{k$FODXyW^^4jle#m+|}$f$n#8SaqYl1eIy+U!2NAs}MYTz4rJ zxcuH4EQ=k{_;L<;Vnw0+kgmpR`HF69Uu1nw*{T#c7D#-|s{`Hq)&jlc1MS~ml=^lG z4Vj4PIf<^a?19KwU{zNxSgaYf>~Q{CX3WWx6)X5S$i`O?;L*%xhGN)E+o9Jlf zf`AhEp!V8@cM-uaYWTrtZHt%0^P;B>M2iZx+*V1o3|ekr(iCo%VV!Q21&)~~<&&;6hD z4nf;5tdzk^)XP8+;{8;uE34i|h3(?Drm(`J9iX$9x(p_%=|TkfCpS426v!_dGVX?* zDA)BT5DbGAVEFY=2Gt18oanx1p9DOQ!O%=KqRe(&AD&Al9EQvQk+L7H7h`j})xUK; zmgD^K`i(@EEe15w*ikJXS|aJ+QR7yEDc{&J3NYv-%vW1ngm<~EC(EafCkRB$kc#u* zFnMWV$apXeq41t*b)3dMUeCi`rwLe5hQd&?)f95XmJgO|Bt|H=GnPQ2J;g94Pa(FM zGAN;3EA|znIk$gLIW`#_I~}-*j2LB+R)K-0e3*@3P6U6DXeV&1(qYJp3Uo}GmM@BY zHp$CE@ypC8^oG+zrhxN)WK`#*uf@?B1SI-A)-)2p9HXj^gQ4An+K-56|yG zgv{CCMv&s{)(}z`1R3{QkET(7+!49_PJJX@Rh4=70`ZX$S0p(<ovXt*KCZhdZ-mz5E zifvh)_95nY58lTC(+a9}EQ8Ru?sIg*VM`vgk2pRgI!_AZ=EcI$f6z1zmlel3fv9QQ z)eJL^evk-5mbj_B+%LAj&RQ4(I-o4pSRT6dg$%^PG@*p zEm_2*t{z8dern}Mrof)Uid`)y`L3XQ_E9%LbJJZju>+t?mB`fPO{Zh)aSQaCat{T) zF04F*CaKBvZI5!$_xdi$drttJ1AXpDA70|T6l{~N6HU{e_CjWGe>0mwV0ILO<0ikT zCi{SMom_eQ&2nxiy_Mckz&HTevJ_z2gpVfrD=-_yY-+iFh6~O+=4fo1BB>ws!t1sn+JwzVH<2HLSSF{n}!__(7heF zUZWs)aPwW~Fx^<=#8uSW0}pFs(a?%h5MziV$kZ)0y%v=7g4A-V!gW+NA-jN&f6BpE zw&M_tAFY@5XRLA>00uM@eP=t>j5$w7;3@MD8HNwAkdymrIURTP2&Q-GK8-L~IFvde z(YdrYU0#6$A(BQ+M&G@Oxtnd5%70bdV_7>yX`#M^=-3!O*g3+P`Y{P)nQ@FrFLZ-e zrNx6%DpJ_;z7H4a?uqng_nbfwDwkT73#8P*rwkbVw%q`JdD^n;YfGG#C7F|lrip#qFW988)pCfadZR2A#qlHiM;*@8NR;=hP|1j=oJcmcA}Lvf~I>osas|+ zU>>2!tgi;pYVt4q z66c176<*&-s?!MO(xu=CEu*eoXe*3iyE!9mia1sTIhKyEID$tsc*kY3Z+k*t$k|bt zGve;kmOf@pdbcy#WySZJVX|WBWw)K!*6F^2aZt}h-xQbI=W~sESDxI#{4GuUHy@5w zxi#cnZ_uWcl;@?4+)Opl^63w06{dvnv!i>h9%ykc5!KgApBUmuCjse0>{K%psEd=Y|VqXfnb%pv?Ub=k&(q!Y+O2 zujv%;Jv0z}t^tT@$z$YVIqj(Qd8ZBQdIR8{w<5K$W<#ZK;|PlNFEHeHg6$$LF{cGJC4zeT+VAUaagm!`Fts~v`6rdN8Yw8&pGonvrb11 zG38Y9uTp3vUg__|@T3Q*hvgFtz__I>W!O0{z|#)^H(_`Q#(0rURq6nb&L6Y1C}cD=RMZHWi*o7K znFM5ig$fM@-CyvweZKB{M$nEEv1hF$FyB~NwILzsjn26(2gA$qz*)i7S~b!4!%>FS zgb|PBM;4vYi-{&+aFNbuOP1@SulMy-xNAqQ(Zz9$B_vPpp{-g8Kcq82u_ z*!PuZU5;DrVF$c$_6racD5+3C%*PhcCu;LqZXq9#0Y-K?Qny$>0QHg^_2HF3$6r9G z1-rU>nd-Cqu#x!)hzmk1K4^K2j)OohZ}#!!(@v7S|v4 z(Jfh(FR~*4)4-Ua;4U4{!2M;3PpaBjmRI=;vG#*&!+9<*CGwUUyG|sXtXfacDP^!9 zXxI4>4FlR5piZ6sQu`JoZA3>yw%|MSwW>Pr#q%s5s_MzDdMonD0)6wgb>Jd>wr!Xn zn3e^nWYJJo)l_U&2qb|SACusyDDbcb-^<`hi?LMuhTnWG)HK$SO?7}Aa@l>fZJWUO zOzN#sqT&Wzwv7j9|MQ}nK28Wmp9R5rFKunSz9l8IB)_c{cU-!92-Fj;k=DG4OGn8S zH}w9uima2@Lf_xg%_*vmB$i%8I($#d^(9ls>fCa}94m5=L#ecHZ>A~rfmB6!mSUhy z)yPIkVUPc)A^HHoq(mgGo_pFMa=?K*o2Ccm?sc?Ma5~lPVKHnh*R+wZQGsSCa5nJ zGF9MfQ1#0dI*~q8c$`m!p6PB*#zYznCtSwh=4Rhi|lmW2-jH(lX^2n5>mif(mD2(K%8W7RJJ?MMT zH1Zg)s>Ahy-B-DAU|y8&Rd&GyQ<=smhF=K7f2ZI~Glyg!GkfcYeI1?FjHF#Zh6Bc3 zB=d4JMjmRlTV{J%LtBqK;Ia<3F{^#t#tRL%eW-c2-AdqUdaG>Z@?Eg>_jjQ-mer^M zbYAH_-2Gs{pDz1Qs;}W6UjG|6b-J@9)sSJoE?&&0{7kH}pa-G+sP{bie%D;QwG2sr zFn)k+TjMUgp#;tf1e3MUjJIu10DV<0tdrv-0>&|(DLspf<`b%BRJ>oG=se$xj?NqJ z$D)yj%GnNvlRmx&4AWig5_!68R?!o-q_Va?!t8}Bio$lM-LVbB^p=`0am6MR2-5Jk zvIs*9dt(^{!(j$H2xnKQ%;EOtjS=Pbj3sc;uz~VB7Kpbt2cTgi03mwfeRpV)OJKvd zLg69QDgqpUZkwU)dCO%3xaF@@XW=C$&#fI8ISc8JQsufffwql9q+Kh%eo`h}+Gnkm zCsVH)>#HiZF3>Kw6ggdgfNk)%)!NASU%^GJI5K^4tWX>%7hKUpc832u1!a6zhNQ!J}J31&^`CgfZ^cGe*GZ#2us7SJ?(IPF*&Fm;y(6RuWLS zyam}~DNBQyj@t1#Cl^ob0^EyGXR^p~*8Y#IrvJT0{{47+7eO+)LWov+7E~HR@~*lx z4D}b0@qHgPtERQ`15LQ!kW#gp7eii<`o9l8j&X{gXNO>jwP}?e_8!vvA9L{jcUC2En0&f_ z{@%X-N&M5lzxB!M8<#Kc6Zh-z5rqg@Znu;?tsARyKHu1`?oJ)t-j;|z*oGqmSGkVv zeTd)_47VbZi3%hOkfA2|%icDb4D?rdJgP>QPZ6`+XTds94+VLb*eBhu_o!QN<=6Aw zH=+Pb?_Hy{ib6Li?9ld$82tLPiIHU=D?|WPLD>+!1l*OIObWgT%ofAF?5I#V=>C*r zzs}Z~Z=W&~m@E1=V)$=(zlZG`#50x-fRzq`m6>{=^@I7{C)`Ha_Fu-z)zqHHV(Ut! z5|rsAY#*Ay&>9{7&IBB@cx82X4#Q&^9x_`iY@MktC109JON#4)yiR@@G#|QkDJK`J zl{}XE_R;iJI+p;1_I3bIlS8{)b>2Z=NnMNDP*>F`@z(OQ_2V0P(i4=C4{D1fammRW z*XLnij?rse^ZCewuF=73P3^SkW){dh`wywx1i-kXRpN>t>2AknfDh13*Y&V;l5}6y z8kfhcrb!=&_0@R;P`$>1V^&h1jE)~qV+a3gY8KDhq~xLD^|vPIF8xE#FQ$HV;s^3? zC^(7dGt|hE6l}k-@4k#-*-i<&`LgM3nXR8|T57KA;!J_9+_=WcTtY@Y25qH0Q;6mc ze_}bN_13{LcBiw_Ldy_*;BJr zm-c4m`u}!)>3f}4q%99k&u2({dPywhAfURH;GW8)rO_QxYNb22#35e3fb1(SZkj4! zPPtOtKa|KN92B*Zo+jQYJ($KiYA3?kZA=>Zbxre9&qhZuTY7L z)Onl{MG9UQ8Fyx&sOeZ2$k&%bn^O)9`?9lJ-d|W+zC07|VJkK0>-IRlSJBf>j=tbp z>ySnBQvqjmdZ<2=Ie7?ci^V%{10o1a5~zS&6ktz8E~4okQBEEJFnr zYTaude*z2)M*1Kmm8jq7V` zQeO`@K3lSyj454hMhte%4$aQ1^Q=GTD$=ZOL2cRUSXpP0#8v1WL=6}4lA z=m^v4Y7zpaD*qyOd!d#q4sh9}RU#V8QkcP|zlJj!yPEf6Xn0A~L0K?#ON}^u#L{O7 zG<)B6(t;+^J&_}pZAB%sjqtiuAXHO|N4G-k50Jx0qs9cA9UKhB+7XB7gIp|Q^V^1& z){3z_E@NxV5Hw-^DvuQmRMKPxbd_zqRu7lY)uLO%y_BG+c*ys-;pF&n2Fmjr*diVj zCDtuuy`B4WJSn1ZGG8pS~x_=5C$1~`p=9$H|<^R?ta$CDl=9{^^aK1IW4jJum zhWmDKjGqzX&v2|BQoNoK2`}&s)X+|K%ZVlYyJ<;rpDTLb|BvU?+AoH`XlwpGZFepj z=M855KZ@Qxs_8q=|37=??C_oKXtv{4+5*Y1bR^$vFlY*Pywkf#ECz&5^0A$3l+u zZkHr>Ni}Heh89Ywg#(?f%2O)v+SmYLv!SL5Eqy|fbY`-rw9=#-S|{d3+(3-V>)Z`n zl1TE)C}lisa2}NS?{QzV&ziX$mSZ0xG@dC`+4NUq?b(Cce&wmyz;c|ckoo)+wvK3$ z8LLwAEq6Lf*vV8j+O(WYWtEz0{C|PTJqVB;?XJ->LwudW?YMl8G*NovSmzvk^xTV& z_#P@%AI!&#?SnpcT#HxKYoV(Q*pZ7R|7acd;enuF^w$V5ajxg4OVlL;S|-ERZdG#nUku7Q)VSG zesLL-H;7{Mf~~CuS6iapKqM?q;m%90@(N95$d9#vlj>Lz3gjq)i)(LY{(eyLu3J+1 zY`w&#F7}%WIX_4GbK>0ewV_Lio8&(dz55ZhK@V`ptEjvC z&7wSr9Ga`buD|ZH%w1)Nm5C0E+)k{marN1qhk7x;n6OF#do6ki*Nt*_HGzqnliWe+ zF%u1*u^1uLLvDHjUW1*+#-~|nxbO`;<4Hgyowk$iw@(aHWGNE-L`9^9{TFxWpdcA^U6NLZbA=5}e=U<9PDT-{9!#UZJO&OCj#hJtS;2qQaAd z4_qRIDD*qjrz@x99t1%e{54WA#{O8}(0T7*YQ=8mY_BlB-r zXtX)){Rn1jdM>9%3yqGsQ)%aIlxzE>Po$JmZ zxb_z&o(WBX-)VWJoi=(F9OMPSike`aB-5t3C}QX&Ige`HzL#>N68>?S)VkgPNM&G4 z-dHQgEMozKok{CsEH~rj5L)U7mcP|}*-}=C;ArrN!PdnAJbU|W=l1Pd%?UpJ28J^AoSw37;zi&~vjk2wioDjXEO`+t!ZV?M~i)O<)i16Uu^f0u!_c*LZj4;g{Z> zrJfedMf-0+LyXl%VyzSjqu-)o*rg*Bn2WF^i!HI4c1R~(^WTU;Rt~#n##F#6K$6_wlP(BZQ z@yDyJR~ej(i10B&Sp+_@b#u@#|E&TJX#okI*ZD{MeYa)UVnii_L{np7g`llheQ9WBHudu zk7(0)Xdacp*k~9gMxqtc%@d}FXWdxZQI;w-?yOX}j=S=xwwJxk4?e25IeIg}yEi4@ z?8(pj+PV?s;jyv5a!Jc*efPcBKlpvu+q#cSe)>P4q+<%dL^5Y?SB}kf8RZsn%l#J5 z@{-3;H?v2DcM+DE9aN_Bf{3bu%F3}#dMBH1zTI|a12WFLn3CU~iTObc!|XcHU>+<9 z4az(lOYx1RbBZtVSNQrImNLNA>5#Fg&Ht~@B}-nqw)3E1qA#$7hp4!JH_0f+2f zaZ^t*uX{NUTu4n49p?auWLD_EgM>f?^Z}p)5`3?U{z1j+VI5R|?F(;47)XB3*iqA< znIK=(f!SEPpd`s%cM`4n$k3<4$2o?eA1ypOp&7|pEqVAhJa}1Hos$J?Z*MPCU@yWm z(w|=ph*QdjF=q`_wQ=eI!ut?Zw&zrmv7$7eZxoDzxCS7VR8%oXBtjKJCmMa*WcQZ+6O*NOu$ydL@@65>JhXshQ5Z+g9 zdju7LY`W%R<;+|bfi__d1gL6oLD-x*U~>P8dar{U6TvLuU6x6L2F}73*1B z{kZ6zj?B3)TuA-HLm3730p5`A&vg|&jt@(|pc)o;XD{Ly+uNNif%CUqure?-2*&VT z*{aVceyAE<=pl7E3%Q|mlEQU@KW-MbP~1~E{ZW?*B^g(AkuOu4eCy)tN6`mE*oxw! z+hp~{su$4w$f}bh2Q|xoHdidb1OD3j^Us{v zs>q8ymN$gvJ5#PfIPC=ETIacjSBXqmw=bcUs_g^crp^SeHbym@ZU}DgPC9KDOupB} zf6s(IN$`lPOap3=4b|C+$uZ??_@|0!uYutv$evVIVI2nzD(2EmY2$YH6~_VL#J9r? z394)GU`ZAKXLewtxs@7?%alWvczvBvcbTxe$xL69j7-1#jk$bAdp+1uJl zvz*k6l5L8Qv>g>I0a2Eq5_6+SYMPkYm#s}CN zwdPp^k9xndc&lY#CHV1xV8|gsQ|c63J-YTxqpQf^KaZ@tcsl!%zVV{GbU=jqd{@nf z8I=B3TTz+%eXx^gq}QG7R1O6|k*nOjEM+`7p#WUaZ6a>77jJ6Zd`ICq&Q825q$D!f ztYA>*Voa?l=4S)bY`_04h8BT%H#kU}HX|dyM!9fK*CBC8#q##S&J=dSyHK)lvybdv z%{dDchXi&lgP)ei=2qwqTdVA=-pd?}PlscW?Ek@7o(YP0*#G`TGKX=(?Bw|S_$0#+ z4RM8ynh)ZOd+ZXK`}S*ALm2%0U8|ChwLwWOf0E8$IsedN_fJvcT55L1iuQ8hc(Zb~|JC}159 zrD8Xh`#j58C!FcCpRWLWlgU7n;u2N`rCMv=?vBR+O3W`tcT-eZ`DyHgG8JCy3kCyM zaO(?1mPKarg`9`j8$*J@M^rqF`5&xB>f}B`=ixHN5^2cXY8j_*RIQJ5Mb@zgEVmWmuOF?)E*mt;{Iaj}yHY zNHiy^(eZ%{>_Elm>{BUaiadt)Fv{gmOI$Oj31gtgs2Ad#YPYx|cFfRBj4lLsl{sNE zhs{eiJM`*3_8Tb;bz?NUhAeU2Rm~aeQ~_9S5L}MF7^$^NkB-Y+o~3VxFUED{hWG17m2ue7RNOM?gk|OIRH0LIVL@wN4%Pa`ey> zT&K6qXzT3`J{iVK1Fp`-h2s_4=p>?6X_p00C{$?0{L*M9-}|Hj*wVn-HTkH!;CP;S zEOxWAo#K}4l;9#N813n8F5uSAUK?=kQ4K00;z}}x=4=xXHcuj&E`qSBUNnkjM>TGV zx!w1pX6il~pgYc2xGc3pve7!>`w7*UR7uylgxGmuyh*i{ae}G#GPI%M zF`gZoQeo)kJ1rsY5;ZO1cmvVFUbpK(l3LiufXOu!a^IW4HwIUY&J=<2nu6Oq$Sn>5kdlgJ!&>g*2>@lOo2@LFL-^x}7XybEUY6S6*dbnoSe5 zwb-sv2^i+=oD6S}kc{2*KWZaQ3K?Toxr&v<`p-3JdwUMkU(6o5QCr>fi zriPn~%}1o8!?XiqK^Gtn&22Jd!TR{46ue?)^$&oG>G>xxR@9Iuz&l_>5nGdFJ2-5n zwpSb8O0gIlZR-z2qNWg10li_Wp*Z((q8%}?;CyL2Ppvv5M3P98Q-Q!srrl@#bpOO5 zL%n8`8FD(awI!;avirX;PNhBRnK9umHPCymWLS~ObMze>?P#`>L1T3-u9_XS6vWg5CuNkf}!sV=>|Z!`G>Tq z!O-0PQQJ{_Y)`9aW7_4IXi!zV21M%M3_&}>YpB!KZ zR4quB#_@bK5qyh>l?EOmk4RdT73qD7k4@%k`^L3lg>SqBzKY$tqq%!h^i~On8IB}$ zWQ=nCPuR$V0STXF*QkxTlh6P=@VP!ip`7S!O`(sRPUYTULW^S%$(h&MB z>^70DEs{LM#up7%w}g|Ajc%57*Q@$kOyN#;t^2r$+ywj}38k)`Quy2yJxtgxk(&(NiAgLwkrtALPgHqcg!^$i?5QIm{z0dxO_Atnd_H5d;QY&|rs!m=2zbSp zPA;aD9dd47&4eIG2_G2>{)n6-kJHR!DkR%3-~f!5URGT_+dO96kpYXeul{^ZwP#&6 zKynya5T??$Rbn zC+2Xy^+h%1&&dZe<4nM=pj6r;myHE8zFug>9o|~^f^=#KT8g4retn62WzLpD{2RPf z+Gq<;_q~{(67UhwD>sbj`c_O6!%Ncv2zCz0`_Q2!7=33JU1Vexx1JiKK10jQ9~goo zb)nLn(zd|P)*n>LF=H~OGDWty>fh$?hamwGBhPz}3ko*I?`z&vy=K`6(#r={&asQy zBDKCBkMdU-*FxyTi(&&E0JDN4nUHd*q@AqKNq1!fsS+S;8qvvD>kXnS+OuGQ#L&!h z57dtyz?Uvml*ghPbFZ?(ktM)LZG0WSW@Z;h+-Q3oYl`P5!+@mTASjg0=oG%8^47LW z{1xqF7RZX2Ad4xZuIWWzd9Ye@ywLKBw!jYi1YKaSU(A?nA=Yg_F~vO^umj~sPbeat z?6{Q!F!GOWyxA5^{M*xuTtvNc>EBgH>`n7TYQmESuXV2I=I||c;M>g$e~Ahu97uLj z=x&Oag^Ub$(9?0DN4KiaIXn2L(p(;zJ)H#>U-pH*G$FR`Wxu7CS-)nCjR}te&U@;B z9=eVWxpiqzJRK#}JoCSE_2f@$jNlNF+GOgy0ynvZvz^R@2I-@Nb^ByH56f7yv1?*j zsu7*Xq^ZrjKQ0YzD8%kbgy#Xn1?0OT4*!89@vjqub6sckG)P|nTcai&?V1vwzSa4X zlPSA0!LS0b z8EV;FiF~7ixsH@M?7Uody(>VByu%I(LY6|(TuU0BGg;i%qo{kz1=P-`(WQ3-96 z1!;(ph6lKvk-Wl1=m_LZ6=<2G1+EpuVfhpLmBG!_!sk!sA$(#zTD=Z~q%t4)NQWDI zT)3u;EtQzpe^^O8Bl<4GelcEAwX4YrB?@~!b`I92bT+Si%MXqy8OSeC@NQvHHETEj zhAoLVcjq>}RClrUL=(d|imZ!)RLM_>9C>MahH-^J8q~m6Sx-iN9jlFfS{=>L=xV%{ zVlgEr*P4kRsmGdq_8NHxz%2yl-(mZ%53>XRGfv3^jmQ;A$St9+qERI0Pbu9lFygT6 z*xAl?fXh}xtkcA|aZrHN*qh&(BKsnL={0J4L5c5@Z0T?Hy=A^1qSpn{JLbqUB$`NC zcujq44D`#lPR9QN%T0)I>@Z9rC{o$E($E!ZQa?Q#xKi$}W<41CalVG)eie@ZV8ne& zaVGxx>4kwkW$0UpHKI$aTzPSkoxC3IGwmQDQs~#niPOS_S11jnv1?wna;@ECOxpwp84;#M33iA2lxA0H_b349r1m)ay$QickMDSeaSjE zK$>`$X!|ieotRkpXYeDR07J+7uD7A!wDSVd8C7QT6_z_66=?dU&*H~Wb1sRDazR?j zQ-lu9Xd5FWN)hH_|4tC+$g*tOtz^`!=>I`ON`3l!5;TeO3V=1ve0h8othz=K1r zui2%W(KbDn9!TlVttn%8RN0%l;n%26$YXBFCs$4WyRz3dd1S9J{E<$v6AQ@~nqlx@ z)K~L|{5|S%vwNzx{_;G}i1~g`Zt32=F+*Q-<=~1qRzt-&-cb>t{c7n#YguS3{}*h) z(c80j4q4-vN-M0&hG}PdDC+1VPA;p~s_*NFxpyvRa`*@1>mrP+%C!9kD~^UXW{J=R z;^rH0mX4TbUXP4>?q<5=)ytKUt(H=%&KbJ$97fI^lbhem59w>5A=n+ocA6ED_+(`{ z&-1x3M*$sFuK!1327#e-QRB8$7CUr@3$9dl#0K0-*)w4TP+&UT#>5XH`UZG;_7W4* z5}ea#`p-kDrnmtf|GC(KOy{vjwelKy2Z!B-=Mm!TlP({uaH|#c{sICALrZcX&2rJu-6(3L6fu1j?%Rbx!DU zBC2K|XV_v#FdkC+n+=9p^V==K_mRn3jyM=ayYp80H{Quk$2Dy86()mG{O;1Q871C` zQ0mo_&2*Ki8#^wl;^$D}_cXXb22Oq!w6?z&8WCxT#B$o_6kMo7;VotdGDVp1*CnIc z#)MX%y9l62)u(r-Nb;3`|BhFq$k?@tF5TU)oNW5i>RC=PRr`S1ph3JQE~5 zXBefbaP^ABKD^?BG5Ll*B5k!5x9h$*+xyN4>o0^htjXRuadRRL1Ni6$T_1W?MmDqIcZFM>@+ezgnw=G`E4#L)I|58a%-wGjP^EoU92cj$sTQ*78%CJ z8|0dclFF4a9Vh-W8X#;x*T&<3(D0&ueN!V)ovT<&!Cf@>;iU}#^EGmx9q-R4`?${Q6V6=Y<9NW>&T;I4;SSMo;|I+4PMpa?Ap>rl5?$ z+TNp*wMJXmFMww_eoe843ol0QTbfCHLa?D;y|N7YH%~uPBle~+ESK2#TiG??7M#WQ zo@bjpiX|L^oO>vUk$qR)tWR~4y6t~y{fLLTMR{2DY#XebwK z_rjNU#WW{yUl;=12I_cDAUp!yM}GMOb<=;=O40v zU?s_}ygUt{m7P#{N2%E3X?E{dNDUhZrSsMhQyr+62dIR6H)T^fqleHwJ7w%YB%6)Y1CBs_y{&o{B4R${VBfN{>`Rhi z0y=IZ4ie3yN7`}q_E!kkxZO1jKu3WI#t<&A<7RS)Dso{5s)Q&$z;vU!pEA_l00K*r zS_hkVU1Udok|Cl7L6d5aYsI#y?KaZm5uMu}np)iE@IBXg9jZabH*8h!x!!>j)ND2+ z9Hb4kU^l-_sVem?x7p!ixoo=u+```V6$vf}q?CiEEww}6-~kNnT$wTN7Maw=3J-Nd zQ_2?793uX`!Q&hc2!HB#1r=xExZ%br6QN0&s-S#kU2L=#iGr>b#=3bo+h^sB&D0v& zjsDR>Y(yQDl*dw;Hhj|6K3m`#H}AZ~X8Svc4JNz7HB>1qiBFMtb8=~3G53gdcx++k zB@!T%m@^XoSZF!l27@lUmlSYBa2@TO`%*nFgsXk4O_pMrZA@_)fYBjy5CHyeapBNc z!qnp?jr(FYN=62)5=PtZYR&AbIq0WED#%#{#>)& zoSB$)ii|ZNp=I%8qRg-KYtuy~nybx$>NUTOSDx9fjr3Vuj_4j#2H{Rw(=&SAW=mdh z*^@Tc1^!5{Ba_cK^o>>#%d?=uUTUaG{1srF5zu2~;xQ-r3@b7&W~6n8i}AS(+c)!S z(gYY`<$C! z(N%^*WOOn7<%p(n3Dp(vAW3bYdY=nj7wE;K%Z-s@R^$@}?NV~b-?!%w@N$8|_P8SS z7f$1~Ls|lR-ES*TO9M0Aaq#G$*_;DiCVs+cijS&ruFqO0hgv(^gRoPb?MqZW!xO)n zNVLXET3!1A4H#I+kWB3!er~JxTLgGB1LF{-(D83=>_8p6=2CLbPW0X+$Hao&RKmC& z2@Hm+iNJ%i3yav)?uFUCzP8D0T;V{B%MNX1LLR53%Cz--eZTj~i2)RO2?5>9+mylU z?URAgv`D_Ft*$d4M+xq5C5O;wPz|=~)#aa&!OK2CWA^@er}hl^bt)Jk+c}i$r%w8G z8yDe!VJE~#A&~C$8o?|igk0?%FkDjt#y)LA^6NvrVnxiL$df%%iuF6qaNdA<9yE_; z0%;t&|Mw_C30v|(O5kSX3gc%HHXA3g< zv$Hh1A3jF=H;)#7NA9!M-ML5xP@Xr4Oy-R3pm5m5`RUFfJSi|6uR_TWSG*i~z=}UB z)3+kUv(Cr@rlbr+$s5NGxLR8!fokflU?3mY_!+3Zn2CAuewFq@P-}3Q(Zd7fDeP#g zp;Ft%`y^D%rrQL`?;aw^!C%m8o>z}2J86#TJAhVuU)@bKP@!+ArAaTjtkJG*>ttS$ zgYwN9?0ZdGKW(I#aqwc|0ekHmMt`Mwettf@QJ0z;oZd5hVrI-}etc92WNt<6B-Ym4 z*`mI{W5SB$1Rr_t;cDJ6&GVNdo7~P*HSXTX?1&L;xWMNu$|j=d@=#O|k#W36&1#+_uMufKIYNYrAwI@wQM>6jI@!gaoC6?!t_b zdv1B6&E)-&nk}hgE5NZ8t!KuJjejJ>E(kwpX$k2mw%3sjbmjw786(VNvsAN4+V*~U z-R$JCQ0XnW=W?QiwXpZ?}>S^|K%^2FZ&Vfg)VUPL=D@T=kg&30q9xdaqm7U&feS zIx;g*WN_lS*+#k8vb&4 zQy8KM>wV3IOREJlb;9sj4n3gaH!c=S#&AW{P7$2(beapN0M+F)Tvy&1PDbm^m3x8n zhbV_J$JY4P zxua9~b=Kf+ewYf7vl56@GYg&}c?!Kj(ddeAHnC=QWd>m|BwweKjEIKoYRylkc>^8D z(5kRDR2I6PuL(}436=)Yi3bcM;mu4Qn6QElW}1p4kbG3qMcXMzNnKI|B)H@h*_VxP zikpN_OWfkB(^=hqzkp}LiqhYuwr1uC|BuuPtrY+Ok-le~aqHvpI*n&`z*|eMt+hVNZ0DW9L@x++PL&4M+PrW4?;V-y zF2?$;e$QQ0sqZ}Owzg6~5Y2>Em^FP_!6kdEC0cF~cGDY8j~p(Y_OS@FU^w8)kx18W z@&?X`7)O3zSH7<>9l#XY0RjLQ>ZjglDPud81ZNNQDxbZ64bRedV@aVsa#sM>@S(A8 zL~jt*n-bVgV=}NkStGR@yF&WD?1G_DU8j z@-$79s@e;d@9wzkDlXn}$+~{!rumfNR_9lrr-_cZXrKF}rNgIXu^35TXG{rY@abFY zHHrq(d;{2ha)ac`%)Mt}GICj&Btx)?R(Q;TNf221{miT&)ru^oJGBhQXJA86>)B6m zMH43%&J;uU64TvNpE?I+(9SWK!nIby%^d90I?{EH`d3~UerP7VV@I{fUY^87ev>Z@ zUKX&U_Nf-*LsknuH-G}DW@F$3>Do11d`6}NQU9~#v2eelN(GmOTiDA-F_(S0@&&v0 zkk&OOQ%(*IG^6K?jHk_rF(4xz4qZet&i`R}Tf8toiXvfLi0rCezR4QVYnbS|qhN3C<*+}EC2+b4K1<@qvsdRgJ-kdzANDinR7*PN?iJ%NJi9fN zLFrD`sNgTNh_*l>&H*5Gcy_`8s{Bpi{I2*_;xcmu)h-PGorxt;60L11D3O9S^=KyV z_hWE^dKU;X{J%5mnCF`WnuDVcr68r3Q`^PGtlrs5VHK$Artt1;YQb3i?CRvhs`L+v zjxDbKs9Si6m$<(UhaO6Gw}&WY72LSb|=Fg>)G0cTXN z^+{#h+GA%+>!Hf3jJC{RD$LNNCmJKa6M9}0p<8pkFY*;jYm>3U*8H3q;wdX75)TFr18Z^onRdv=73mR160*|pN{l&CEDz5HI zSZ2b--XT+FoS}>_I+uirn_N8hG}apq?i)OsKq*E;Z*#E2-$g+}jN?XAnv90wGgB`I zTxavPCI@fVj{D?FE@EUN=cpmRV`hgcY0z8wDC0Otqxr+8GDCefGTl6lFN3DLM@>yH zo#dT%RD4;@$3}JQ8wUd4cIE)V?xU`i(J}><(zqj)_{N%xX_Fn8kBH`7lxUGAT5Z;M zqTJRPXL;o)MdMLK`zx*w0ZeV^AtO9q;m9xn zBIWIt1{9C}Qk`5oj0}DUlKzRm;~;J(CP5x#{E<4_e33buukar>6ikT3_G2*d#V^es z04>QJ#EpsAKP3M35XCJU8M}G5-5z|ADuIA?b?!{STt+P$cRfGwX%JbcEvxf>NrrZ} zP;r4fhfswE?{vt}5OmNCG*g2v6G7je0^|qN&Ug8S=mNTXrNoujR3j~9oE$dO z=@xAhNG63_6j4cJb84!|tYy23VeL1m#+q}XKhx?*__HHbY>9f$bnlmOIcgN|uMuYcvs@?o~f@40@#fn0Q zk?cZbN6p_mU}?nl(bSeRC z+FW)Uf7FX2x4uLszBf)#CNuIyxPypMjch6{>(f7J9f*1I94vbP&JI2pZA)c)_HY!s zmek-9!yJA5&wFeBYgReWj=*9#f7O!7^LJH7Od_#He!5iit{Mm%{4FUe?~@MbO62F_ z>pgV;WwQ4bE;5fL_3ZYb6*b|HIJdFj-w@vrbhD>B4LiTYrx8Lpqn1UYGD9*3JA?|n zG3ai`{ZC1EUSgTt4r6`;lKFx4iA0OSJN5oewy)UU)6*k{lP88G3<1`^8YyoVb;oqA zp0MjxJldfMl=HpgIPu!5)ZQ*3H$Gk19Spz2C@z@iB(OpBp{(!j+7=27o&)yQ3$(uSVTLAUmJn;8YzkV;uzs%gx-js*VI}jy z>D?mJ(o*t60E3#YnK#s)kWZaJ#Ah>TZn&TjjtwBb{p2$Aruuy3e+2N{z3#w28L*S? z_)Vpm6q=DQKd$K1@*S6JO4`^p=MwKCnX~d1(90a!(@LV*UahWv|Lxr`-ZED|nlS7- z<6bH0PCJ#BW-+#!4_8c~)!X{ZcLIU8L_%!`O!TxzYU#Dsd^|ef$!9AS$yD>CB=LTo zy6^zLg7NI1`8(Hv_&r#F2d|{b@L3pk?w-KiGz5NyV;_OFC=?ZM80hye7&igB;!`e> z9sglsu51gL9-S8PDr0inDztptvDe<((qdnX3SFK)a~XEfq@YGdB%S$H;=q#U1{rQm z(;(w(9SGJQ{6w7y{U!oBkJ3ZM3`NAwW*BT~OAKdt`cRRlFrAl(DWCxy<5_ci_fZN4 z+JKQH&>MsAnI9zm^hWDauWo9Oy<8F3iAdz`#+RDxy=`XWr!ETPNMxO62h z0%lb@k-X`~;e@RMOhZ6}t5-AeHfZg=er@|)M)y2Muh zO3;AsHK3sCsF1@82%9gD7w1=iD!b!(p*~0N*`WAFb@=gs9&CDpMklzn+BcTmD0Um~ z43=au)svQS%y^@_Ok^02%h-+4eZ=5h82w3G%nT9pTzrc$93=x^j}Ecb(alcQ z4&dJ$Si*;gaZT$*Am`s~d6cCkGVDd&-rpf(4=iN?m!R;+%+I@)O<+#THvzUdmU2+{ zzF`W4f*;clOCR?uK6O^$886LA7e2YYQ_Y-dCY^z|rOpi?Vz*iV`B&<1MtFZXl3oQj z-6B_EE!RLyjkY~6+&Rx-J_ncpXdch>cJpNRIiFR{lQa%p)y}L=$4x$6)e%*_U@$ec!mOBwpD| zQ`$5z+KV=AS8kovU?s8qyX*-|UnS*zKU%A2xJkj#QY970JMaq2<{r&R`YFr8?Uk!* z=M^SklgP$IAC(+Mv-&8*ce`C1i>CNH>YSY9gW{ugniGq4RCi)tgtNd{u0eKqO#mqo z4tQx7h*CncCilIbOb5%73MgutU}|gdrki!*Oa_!N8HVcy&TjM?Z1Ps{I6+u`5!mjf zsFRLs!0(HC)*@A-aV}eO=WR0Gcbr%kLIV0FfPDh}XrKr{Bg;fs=HcD_mE=2O zT`m)ozc%v~EJWW;Bk#$SB3dAiMLhx|iatSYLdHJ?Dq8__f$cq@$tdT^}9o$HGo zTGmi1j^h1O8qB;rJu7ixb6MukvmUJYuSamZ`G1<9$Sma4-ACpHEGw7#0=s!f%s9AI zG80ioOA15mU`akcuHoflPv;N!G*_9TD9@*4y59I*kt*9EQb041>ja+F32S=#5ZyH8 zy4oXXEMb0OAmV;3W?w5$%d{6%Y!2AFYmW&Kr(BIq{9JE5H^azk+-_8eyvOlmLqwZz z<=5B8nC3=tlL2Lnd@ht`sK!u(=&|(OM7?EBAs?aYK@r0L6*?m7jEY#pPS2h1dKQh_ z-}rKgcc$t>aM>ESi*;2*_qbzv1-(rq%T^ySy}A+qXW}gtrGmqlTKA78Z1z}w-ggSt z_T(AG@|me-$tusJyJA=oz_I4X;SB-e0CpBN9>bhns%&p)k>s&>?unnwS6%UcZWv`u zO$3H^UF1e%iVV^BoxNE_+6O~{TOR#HP=~)oaKHW~R{DHKWSjSOl(9bqlmJxtl~&JM zR<4XcXj5aiB6HeT%PEIl--Vl z8^_6as%A4Yc4c1FhyR@6vcZzoTiIXy=tS^YjYPOSc$`f42dBA%hTDAB&MS?U8OLL7 z;+3fryrVQ-2aMb}U7mQA30X39txZ?+74btPTrOiQ>xhu?(qxK;*s-68NnYlSAwKQN zePno@9(vi^YzPE*amGhL0_B@P1)D*halNDwwoj6qu6FA4q%TI=>qI8#ZKNE;Ap+kG zM$U2cET!mH_XH`i-1!xW`G5p-;B*v7ySQ03WZx>_Ab4@PRUj|pscCr2sjQM#5#8?SPzCFit|hIvm~9yFdp)x)eLh`j1y$gM_YnT4+(FuMUw8C2E;@{(M}>(X4_>K|0i>8OF0+!w z7XRT^bE@m8*z>%tlVXQtGdy#qsG?Q5*yxtEXwyya-~DcFvxWbMqghBt!cx>N20hfp zSpnpX77&9LfUQiakgQCX9JwiI-@Q5!b5&L~%pBHAJ$9B05jVn;OTy&TI6S*7Cj@*r zXTN+#XXcpnFX!floU?|yPUH3f(Y`S^wKl&7a$I=T#xc-(9hplTRMaHb&mkJRPKAuV zze`Ax!16S&LQjbuC$Yk}J2ErpYkwZql?;xh4%^8So@;Dqyfu}|$YDNo&81Abl zVx0#J%ut55%b3fch}vqZYh=_MdAt={d)no^QkCN~#^#LGn;W~(+&*B8FSD6gDqny_tbp8rz1ykILIs-u5JWZ61lshO!%ea&5rjYB>;m@2wYhzCG&2VUh` zP$eE!*=1g8vO0JSFmSDM%P8vPoB(#w-c{dsvxgT82Kjj-dsLvXhuuoWk;9|i=;Zfg ziuXy+dU&pPDtPs^{+`8Z$Lj3tp2Tgg46}+fjMrUUzQ=xEP{L{2Ilt)5GO}u1-_U0F z?NuRO8kn)_i!A?N_av?^`g4&o-%frTOmYzNx|giQnvra)o6icgbSn3VCCSU9jIx7VS$V+HZ^z z60i3Tz^0Tz0b86aRgn{ebPlzfC!XpxN$nxz>=7T}a$ZTUPf#@fUG%;akioKzuX||QvxTuWW(%*EbhVUt$Rw$_p>qywl_9aa}Mb$H(g;>wRIoXeIX=`ihbwb@-GOoVgsxz3#*rBJLz8>b(rx`b-W*OCaF|UJ>tckW&K*( zjmpypROcw;XohzpN1`?voaG?J<;;*R*}M<(7fjIbFkpz2LxXXdz?pc3ONZj7`vmhN zc_J7{lRlnSGK!TG&4#`Vewz43s+9>^R(0idp~r_dwS#P*4o3MKX3afSSx&!=!ZvKb zuG1cfwv?i0_s!7Ai<3m$Nd#EKatqa)hV>y~@|9&=ytd$2=vzKIzq>Q96kxNK$3y42 z*@ZB94a@ubaHVLq2;}2h2S`umt`CPIy>MAW-0u);Xo=%9>ZIR)o~t5656(QStG`^5 zO5&9%-k#gt?<=1O-lfIwVf5q%kd7F>R4!hQhgBJQ&ZUQk)kieDVA;dBZyLtAX0YLH zdm&8ByN=^$3rX}pj7(1PTm)}#p^;7swnQf$OeOhN{tj%(1eo*9LZM1{2AU2H5+eht!tfl_ zYtl1&wb<#)XMu|NZW;h3x@p5AOoR%I?Q5Ch;>L;c7&ZML6Z9jPCWZM-$%-=gz@|0k z`&UqTyQ(~s)neIHZHZfMR>I|;e?d+tCe9RY?QF~}?stni13zEFnOrhL>>!eI@y$-|FZd>X#~A6Bs5PB%3)ZVKF?UaR}>n&uK`7Y-@Z7Gg1(TOQ5B>R!c zJ6hQWsBJY?yC3LfF}_TL9#+bNw^W!7kl(MW`rBR1_Fb-S*sOt0!FBQvGn}U3_%IHG zkxwiqm6rFi%ieydH^4o$_m`aC@cL!JDWR;vam-u<{%sl7mU8VE+H_89isc48S5bW_ zXZbs{o7?W*YQ*{8ZBg$0B)HBI(}GquWBDvA^?RdGOWMjRW`Y-XVV!eywd7{WY{d`HA_m?7+)#X$cWP0z+ zMq3n(v3;CLsH~+v6<|a-#9Vf8wPhiVu<4&`vef8vPEkoO%%Lgn2O%R?5kl%(vG==% zP4+5sJeCP3jF$&@iocgB1AnVZ284y|z=ErQ5DzO6C%Y-*$zre_9}`LvY19k@v~Z49 z00yWBi61IAk;4;oYw>pvLGMBO=)kE|-B60UkU*_voL-1Dpp3QaZxBL$$MSp@a;)*m zzOl+G73HE<`V&0pt++TmJ8;O*#}%<0`LJ5D@2%8d&qmwKI+(l!=ZE*6{b&BkWr94U^b^i&De)J=dTvemG!Hw-b=*B#jI?~Oj{4ir!WJGEXw-QhBs z6ZfxHHgnV_ZG5@+_+Y2hH!=d&cc!(7!u@YU(-E@d86@)Lq#HM8v ziA8P6AB9F3Q6&%hRVv*pkOs_t?I6vv-GmZENn8w#Ykb%rKgL&(XJ`&|0H~6!-?_%( zywTt1@-!01W=Kx3{H0NFxzfDWPI-9*guPd>QuiHn0W!GQ$=sU~t>>b^eu~~w>j#>@dQC6k;`6{bq2Bjy ztfCpO$ z$OM&QKs(1}U1s=tq>-Ek@6``&1D$QeR{nzQC zUZxPdhmIjW$3|ZVtCiK((dc`!x?2e>i3!QH0eFP()>%UMN*YZqZnct&3`g|=9OU|l zMnT}~0MoODQ$8TFl!2|^BN2^TTQl})#r!bYw9h;CcwK7QPq@&Hj$J@fqLy9ta%xd# zq+88%c9`uJUvsD8lb22voq}(C)moXGhl20#t1z4wl}dFXHjCCxq2ceZd;nlzHn7pF z@tTyrD{c0u>C@t0`5>z90e7JZV03u&*QW0%j)6eyyTc04E+?TSSrLlQG!reLl zF9t^Ml`koO_MU{I1TO9QWcVy|(I{qSK4$gv1oOHEH05N=&}cw_@PrCt?>I zn4rY=saHjXt$9Vo8G$& zTZf#41bEXT8hw$pDx!r9Qi8T7n-(~VzXdzIV;8M!DL==H$fW}DdP7jvs{afd-s)C! z?KG&Q(IUI_)}+6w$Bv=ZyGCFK$odU!nC74_9=#Z7mgz|{)e5-UzGbB}X$@%Qj6i9x zm{*9GKcGBZT{ta(%Uc+|(B|F~RE5sBi)tlec)-R89aP7(dC5v_QetErXLgRVfRxE3>yc?cxh(K*Htv&aR*0bs} zf)|FP0663d9Y#_rM+Y)@jk&t&&9x*U#~Pnay>Hji4xb(<#*oh_SlKeOa}$O>rsvjP z_CHBu4o>fGC+TCidt!3v#e+%}ys0jDkEb88wOGM@mE?+em2yI8a7d;fj$NnS&u5NG zKC5{pI8QpW>etEdiA2GFXLAR~ZNC{URJa}yHa^U<3YlIUHzJJft+5+kfxTnbefu+U z5#evsg6OJ(G+H=i`SRW2*lv%VE>}vZ*7c34rg7iRG}Fq0eN?8ae)4I=%hQk6xCK~W zcVD-tmR47M_T|LUiWYX0b1b8GDcp79vIXwBa9y^(Uu|5zUYht|V7C52=ls+1-0RAz zy?Ewm+L)XrEf1d}Xoj-h?u_=97mGT*FKYM3x9<$qG=*H}-Jwn7+pFKgW`bl2^hY@K zLm6IFPPse!uw3>wiH3o++ND|D=TQ|Q3EChQYA^+3T;6=v;TSa@E>I%e za^cSfM6Cb8QgTms;2vSrKP-|do19kNrWiLrtVC&_P?e)Oej7Xye;XS`vJUXp5!yU> z_;v6Ix7uGWkze6|u!1!AUZd+jSdBMS)sgR+?-e=T z+L4!@OHb2`l50ExwZw*YSW7AybZVz?3}P{ydmG4O`1jJrewO;x;MyUG2Beo0U=LXg zm0Vj|p{omDZ61Ht%WRnO9Kt?UBU}%x<5|)oo1E)JHNzVJ8OS1&-T&PgKZJ(Owu)zZ z*^Mpz>i+A~kZQ>A1Pm3&7txxAdYOH8JS3pubhMcz7ewWUPnNK`RoxV$cU%B4iL^8jusqS!!PGioT>Krq}6D8fRRsZ8jS16+T9!=hTU z@pal4wtjm=?6>1HGw3DZWy=yjy3Th2ga|Ad$Fs)+^KEPLxaN=~0&Td6vCS5tj_$X} zi-}GX5!AKsbziAdceaWEKA2o#EyGFO7G`WJp=t~)5O({ydm}PDQ*N0)9ZO~g{GiF? zlVP1NepVF|A``8tz!zWQ33om2mqea@HJxWJ7jcIxxH-q_aZ9Y z$5k7a7aEUc`tI>X_VQ@CWWlYTKn}Y9-g$vYbp4Df%2Vvm+-PxLK<$8w zkn>=)cFywsDfrX6$QLjXG?#*=aM9=oHe?TYhJ^%0&M)Hq3a_w27us4`wX}(iE~}It zJ5l7DI-iWFnI2Rlkrv6*wT(wCMMdUPU>uKBj2n zQQF$ww{V=oqCab0`%&#citS#De4)H?qzA0tE~QAV*<(uTpFhP0)Y2-$(t4)S@T6#H zVz6KJnc)Up6`NjC5(B>-l1b_%o}86Q2EKUQUV^|1oR5!;*3oGJIb6*!Rx?bJ?vb@4 ziEwyivNBu!C$)0wPV?CFC#U%6AM!56{&syh3s$>2up!(u< z&1fAZ^Njp|N%6DL7rNlD-O1SE=npc`#$r(DzUO8o=si!UUVkCGaA;kioaZ&svb+P{ zV#LGuS1+m*?X8^%lRngJEYGS9ACX2hZG&g@DMgYmj;VnXy#GDojm^>KV5Yqn?68a`U?Mt=ovPa+F`Tqu@*N2u6SVq&!4c9F6D<}xOY^tMW!D#OwSF$l9 z5u0JGZL^hD8Eg0c@3X8ZPbUezkgi2Lr1LqH#0Z3^+Kv`$&ct?+`N;4C&SBBMFu*Ii zPA1>CrQ4o0z8TL-&6VX=5IIGNlxQHCn=w*f9i5%i`A*?5QP8;dF35iB{;yg{VVO{V z!i|iqkfM)LMVtpX2_2Lv#(lS&cYZ|i#tc51#G23Ix{@hfssUMiJf&SO5Ts$R6NByi zv-WksuCUVvn{|QTI`)2%2Cm3rUV|NjI$-9GH)`~D{yfW{luD$9R&UAs*$r3nReR6{ z5Zznvu}j>qcUAX_hl<0yOR0EfzwozSPa`=r(!M|bvQCIr920?6BE!t-_U-5%LP26` ze!q@U?YBylFX)KCQ-pgx`Iz;w{=g|UEcQ|xueSSg=D3lqI%ya!f-x^qc}(u6)ZI~`@uDc$2W0m#$M7O%JEn0*8E-52gSK!W+9lNqjIJ|@`)JBhk1+B_YYSehI9 zvH-xatYDA^JtM?a*us%|;E@)Uy}Cf!(CDVuL=ELFX|1iB|@SC*G-<$49lHU%`8R`^0?1`ZSw%HJ-NE!8@Fki-bN# zuHKOMWClMn!w&FVC>C?Oo5*mo8KZeUxTM07v`(aAcP<9;5%Z5BwUq8c#4L!U#4dH_ zD0O~x=xoUFx18)|X|rWKtYSC{69;6-4yY8&wj3K#e`?h;5Yl zb@cg-kC&EukGP{zqg4vJyT18`(7~GpR;Q(u-)0-j8a~tItsw1PM{}JX8r=0e*l2@V zm5i-9iWN8P_3e)P)Ew1|!W6KGKyqn2KpNowHBP(<_7=m}#HR+1tqeHtOEaJ+$&{MwUM0~EWsxvIiU zhh!BT4EZvwcGEJrMAJad#gXP(LQTT!C zmPBfuzwZcvnA_{&E6u#L!BNxHVC~Ws3Iw%_%LmeZN}QLMRs5wH%-ZN>OA~l=k2=5A zZ7$)v&fxufu0YR9`7uLstbsIpf^^IhA7ju<%xfKvGMqRY>$8>?*)A3c7s$v!q_HM6 z7x~)1I9x)S6-WP(UC$an-x#7~-3) zw5~^k_EM&6$%WL<7cZWzqhjv90T`CW4cUi8Pd{3ZiCAka$;J-_N|f{SW?xmMLU9r__*=TC-~=2XClLu&`m zTYurx1DkhP8N+9m$I`PxMj?WV*QUS8bjORBzPOr-@?!sgj@IXQ4uLArl-NQ(A$Csv zUa{O-3S$`JX6)5e877?2VE)tiR%_A+qrUf1iRx|ApY4q=6_<(Oqi}L$KwT9M62$+z zXxx4>*_Zkh5x94y8FM+jzsN9m1y0+%L#$qc0oANIC#nArmF8K=&LpC7=~mTgM6GL# zHm|Fhog_AfILsq%FB1BWf6j7%TT=~LL?CcWYzj@SUN)9jWsvk&lYyfhl@(b(hy}@c zpbP~c>IcNkifrv8#Ql%nVb4a(2hBvGh(FwGE8Jzo4IU3 zHHDRU8;kD^5g+%e2+`?-{sY(4jxoOj+W0z|W&8DSiT@l-MW2J!!lM1^rbU6aQ!Q-f zme`#7Z_z98TKR;owp*VyI7pXq{3KZ@>Nf0`35%R3Sz@<_ehQ>CV=I?0C+BuO98z_F zkx{mIts6AAnhR<2!S1q*LU@B?9JRu$1ZZV}pm7vF|_)6Prm- zq>TD6o#T^ciw7(Nsu9bZFzQYRYk>C3X@w8b_0V1dGxBqH5B@H;VL3tbXVi#)RZA=B z8K6(igr0-q^Dc218~JfkqL(<0iZ|}G#`|+C2}k<4%Hfl%6m8v_Abr{RQ(JMg_dCec z8#l4p!aOS8J;YO;yM8aJ-mk6E|1&Z+&jE(00ov?I`VAEAm0l1N%*A`wLW-s|+|TMwPTVUQ~N^oZhhAm$_TRgq~tUhGfaSIdcaFZQS(@ zGyK^Ki2|J!K*o;g0&QVhZ0A|0)IOSqukQN}L#_l_=y;FSx zRup-`hhv<-NnkPlLp=%n4Jx&l6@9kbt*ailKA*%-3Y5des78!n<@2`V1FIjh`)Kn? z(a4ee%~iRqhTZWxv3r%g`4>tJ@wVLLx0RwR+*v4A+29Ys7isj}se`=n+q{hES^>Nk z`oHqo2%p;6^+@@kuA(2SAj^NW^@}_x;i+>(}PRGF`G<^@d`=>%DL}LrB`)bXsAl3p^=(#J4^@D-B(#qF!w#wzGeZ0|Ld| z)gxx!i2tz_E=bq@Y^|M?m%{5pbF*CCFShIM@=$c&(vQXTyHV9utu?pa%q(ZV+8ABe zrtMi-N8Hd@ze_?q3K_A#7+URLbow=d+ew3u3I@-K1Ko;phIu&{C`D zn%!OP>{4f9ZYLhjqZ|ek`YzRib}|V!>M=;u$Ac+Xm)=AUljf;OSZYXu_OXcbnxRYH zQv}P6_+*9t)B&Upb>sQ56UhSwiS3{$M@0QoA9_r9Gux_anMTG;x}^B~?2+Jo`)&&% zG}=2mOd5x#9VZ-tKlDH$q#bji&b@X5w*HPI3(Gm9jr_d>2sz9X3rh}PwPil$13u{< zW#eFWMHOc)lPSY6o%vv5F2leYpHJ!ybi!^gI0{<|+DSE=^9MAWIeM0B~Ok{o`Om|R6{ai;^`iSjWux-C$0Qu-<+uYYR zetmyC?>!;Ev&gM&PZELX)L7kDqDjwRWxWaa|7Gdh3%?M_^cUc3UJVT#`*#36pIVki zt%&lP8uyz+R(P*G-ce+(3_GHXqVNKB=dE^Zkmy6W9;=?9!(dB=rH`v;HY&Q&0`XAY zX2T~YW@IR2C8UC#2a+u7aFmonw~^sLWM7v0_YKwI3V75xXMU54V>UEx^6WcYWGhsc9KLPrpNF!Ge*Tr2MB zFW9VRe*6tZ&_1~^ZSf3tn2F*ML7jXOCjraw7ac{Wiv+im85Ih3$Z;dgBMAR=~M?EsLd>d;unudww7pWV$5ens3{1}~=d{`tTSePl?bK_?G$ zBU7cC23rfNX%f{i*D4Nx@G-?beGu;7a>pN-*U}vw+(=+W2k7j^jw;5hws^a^+Oh~O z@jS}~;2}h(=<|fGxf4D4;p{Kv@d`xJC;5+xAg^#fQQtTNzWRfd5zAUjWBq_Wy&g}7 zWz|KmiiawX7EoaM-Ur0L7_viPp5KM59lLk6O zT$d-Tb8_GyrRfEYAOWpt=L9@>j(GTlt&H=wUt8n|w5}wG!vcM*jRLKMD12TgMLP<$ z=(PlnXR|tu`g{BPWXF4O=kJu-rr8BTZ1Qa72^97@jJmh`f?^l1U+HT${5;k-0&NsM zeO9t-Uf{(hU8aOFaLEy`kYqv_HcWBf7`W3pVrs0Cz5>w0p*CCjeAjme;_rj?)c%#$ zst-bKLEZGQkYFDzO=Ij%>9m#SX6~ssOPK-57Klgm=3*zIl@xwn`D) zJwRuI-bYnx-f4v-E6z-@c<2qw-u7m2h2Bg@YjkNlM0#k4&oC_FnKfXrx~TP^;DxWk zg7@@{#q#lIK2K_m*CI`5!0R8W;B5z=?3t1`$>QT-NsI#!!s zpSBI-HCVQe%DADY>azpi^}rWlsX!pK^WZH_nDAK3NPAx0L>u= zc-0~@Z?NHc#f)Ne`PSNAeZ77&p6Lq^B6iQ~@`k`^XhokAZ=GI>{lrXDV=YKi!5tyB zuZ>f3%?qDZ+s|}~j>KFjA3WC6$9a0luwtF`mrq5CXJhs$y?&)Pl?(mHz%55& z2{T$kv38K=4binYVYBA0_xA;D%~_YqffYAvMweE&-YRlz+-^qJ*Bag`x*M6MXB$4d zR&$^E10k&VSGyWyx%mt$1v$QXg&1s3)@jG0iOS@G)`j);OtN#c@y9nx-#Wx&1a9WL zwb6|)VEWK4#E4;k(Z>DFHjcq9YVKB_c6THX1YZJ4QMp*ty*jyxWYQfXsb7&lVj82W z&t+b6{=`a4?{88bv45o%Q;?1k3MJhD^6+2Bnf>1-!^xAktX1iG!V9tXs5GiZ!|U(U zmJ5Iht&clyXdl8t`v9S_6(3$>W4u2!d-Kn({TV|sZd{>F`=s6Vna%G}J6Pwg~Obn*;|1ro9l4|bmCQV%? zh~m2RdMUJlwz!`5@HO>*`vxm&yI0P)4KPQG&5moyI9#+@%%%-3@uC~?go}p|-bOVu zjH1L84i?crV+`cdJZ-I=L=eX$_f@T_G-)*DbO&-scWXMbI6;i<1bG3SG|PXzj=%SJ z`}aiBk{6YT#uUBh$5o{I{4+nQ5RqMsG}OVQyS7%##lcyf@8?oL6~O7lj%RuHasMD| zl#%g{L*%z7fPk4H7IK1cI#kNs^CP>k4Q=_fTRqq^f4npXt71lU9B`z15U1TX?wZOC zo*az}prPfCsLfCm_7VVMNw-it?(H?lwvmooZN}RPxjg9y$f8%40`h(B;(=pARTh&* z0jPJXSV-j&g1x-o)%ZD-1BTE0+2Eni7p!gbB7R&JEsS~iw{xu1aS3kK&ZWjzSBQ#x zJZP&Yr8bRfn!4&{LoR-7b2x0LzmmfSl4O)z+VgH>!_$|WP{3!Z{#@)tj~kCF3*9oQ zKN(4$?PZ+NQ+!#n>)yRL5k7dzShAk1Hu*qkS8RPYs{I$D=NPftyu>@4YZ&phO6PJ0 z)U{#_bCAuvcm6il17lgsGq=7#b%~?Tz#;x1i<1?+&G&wz%-Rm*WSVYk#PvJp3s?WE!iR|@kh@Z;YmX5%VFI9)W#7nvo}f>Pm8>0w8( zqtzTEgFDaTtjl0U<34E>V2&AqdQZ< za`+;~^C4~kX=9PrwCGR=VLjI35Or%rD91H6)nMZ`Z06NFheK+8jlUbw6xA~4Hy&1T zyQ*|AV1d6HO5q7KbE=%2nv#3Mn#dRV=J?Qt{eyOnMp719p3iDp(`t)$Iq4_i^>@26 zjPmSy_JRAvm|z76(?GoGK_9LCm||+@^P?~0O-I88!`O5;->f?>jm&b3G;4&hsNFas9K$Wne<>-%G4k2IR;$TeIom<)WW@uBj*Y|OAO(SsO1GK!LBK=Wp!)on;yV! z?+?67u@lk=^ETV1EKb_H(W`0nT5!lwufIzjqGXcXq-3U<20@$C&O7`?tY`XmG5_i6 z>(`mT5o*7}*Eam9m&lOe?;V6QoDA`rsJiT(Ys`klz&Avw5hseitL*w1MzaLR|HKcj z^XtM|p%&VzV==Z)pd?#Ym~L2VVO{99ztM^cED@ONIdf*d>zu3vc{B-~$28!;7I@G; z4JQR2u$RO*Z1mS5An3N~5+sgpAk|LIybdN49kzefEo3KHlQF$pSvZxN-iFGk^Oa5V zBRjh8T+-}$-*1V35J&bpu){jVY183H@>QoG6n)rD7q(gjRu$U7iv9IFbqCRXORh-} z*Ob&W^cIb?vNJu`&dX%;6_Mrn!u5G@lt}_i+ZXRKsA8Zl#IgZX=k_ zV`H*X5eG6gJI2>?u-gK}s>EDwKcmxC>Kkkg=RxB=cmX!H5g_R4_8Ml`a}a%zzSZrm zH{(UODo^GJht>!$I>Mcs(BSUj^8M-LZeWMt@vtJcP}10T~avD5s#4A9eolZj)01aKTF6}m`V&wSKe3NLEUy-5fx ziF+bH=E9S(b>LyPyY_@=m5amIibc3Xu(1J2D4Psx4BPCXpzqpQ1HaMDQY^<*G zuju#g#Ud!)TK%8W!OU@Y<|t_BNSMkaNWzD+GVm#Zr&|7>&wnu1iVG=E4T%^Wel zsVr0RMvb?B1cj9KU~<$({}%Eg8L{;^yt`AnmnR6c=}k!rEy)3n_%uB|odm@iEnBOqB5sXTyi+O&EA0GS( z3DKn9I23k!Y~^0GHU6KOQ2TW20z04CmS5fC1fn5>g+)r^WQ0%&czq;hgj4b{h2S1 zL}zqYS8_79*5j~K85-KH1OpqY0^iSuMzVZo?x;bZd4BD^3)bhdk|n)%xYZXCL2hB4Y~SV$FyJHH#s#@u(W?zBtai|q)JT1rinOne%B zf<(=&*N!*TkVeMO(=_M)Zm6}s4RVn855JvUViji|MyiubKvduJV-KM!OLL>ICg!WN z8da-4c3v({RW9TrQ4bVmdnjulK^CpOTuP1}dPE!_#%zZMc;J(odR$ZSZh=NJMx>3;mS&!4ZG_ET?KtE$zExxoRJ1nR=b&x=45dNI015buh*Lvqk-uyb zxEq~Ch7+J?1PS7M_$$STUN)Vob;f7M^9~n=UsHMg?ZySA+`(9%*uaAii|d1hWRlh1 zw(|89R6D7UX=}zYtF0sHDdhQ}j(JA1f)-EZ@oM*`u#)iDLDET<;|QAm1ZdFMXjFal zw6GAC4V33!j%yOg{5_MF6`KRk=-;YQSq~~x6+E@L-ZeAr%M~)g0&tE`6@dTaD|xVo z*%#ff6{`-OCASKp8;-?EYs*mo#m(vr*0wZsOT$**$KM@<~VaV#{)v|$EsYD zMW$sEJt;M5PEidK>aFH+!H>4W%*g%e=9VfCMJSO{kW!cJXjvJgKdXAc>v@7 zC4q=VWz~aoLBzL&8g5UL){xcMT}~DwWLL9cj48{S<{g#?pzzK(<`2Y<#dghHvb=Xm ziH+R%+Bf7Zx~J)M?6okqq18E(V5wZ@R1VfOfVPh{Y%J;_c}Mr^+F#1ceLMWn-%3Iq zSimj{T9cs-MD)e0n;T`ca`5r?s2~0hki^6cw5Q@VXr_+~8eBCdk5jwPOxOgn=Lyy* zUP4z73T&8T@Q_9nbhz4!mDDn&Tt?nHbv zq-5$i4r>~-9kg9aveN8lb8|Q`jJqErOiTX<6Mw&ZqJO!q4HOtnx7W(F zB?Y@bkNw!!JxR3{XMW-cts;*-%9jBac9L3yE!2++nV%P6?eB(RjepME0W+B~p_W?X zZx{I>AI{G(#Ur_ZI*^A-ij6wxFB|R>c$br_(&>hrDs1QmF>CB+W+=|uEcI-fvITKc zoljh<1Q!OOSd&9sJwt{g8Vwi3g;fNDu@RK%H&;ccVbsxSwB(C)&AFRz710JJ^GZ$g zzntOr!^Wz@0)k~q%pA(pobexTPkw2q8&o~~8q3QwI}N_B`9eOUAVED8hxN@RS;>ZL zv5nqrnJHOF&KJf`!8u7qeiAOSKpfa*xK4s7iTD`PmZ{Anv%;-6FTi zgU=_v$Q?aJ*c|K`j%PJ4AC0;`^&>I1c{^RJq_!Aj^-l^S!(4cuLq_|7l?BA2~Zb=k>1na8Kc$N*oj*4@<3O_Zr=s57>;hPFYV9zV2M{ zyAV`1%V{g8IJS#6?;*VTQc=j}AV@V=a6tROV~X;ba%@zh+@X8wn)yQ{o^WsaDQMH@FKdkBqYR=V#paU~ieYJ0oaG7eX9UU+mQ z<#IV#XfCADTm+qJebv&T{KWHHT%?=OQtqB6xZhM_T|c6-cBi@I@RB|4%!#m-5a~;v z&mBOx-$V0*K3JoZ&2cyY>|$|Ye{%b^c1PqscykLlHqMRYYqlYwxg!?31WJ)asEJ7b zmN@p|uzJE$Zkn5oT!#@!qEe$}+X$yOac0Bq>%AGNj}YPM>F(U1T@6FT*v19whVtR8}vo$xf6Qt39Wdc)H z7aW<=4n0wa-L-+An>uQe%MA!_|Zld!(5wqK0K8Nadm<#&E-dYM+gor3Q^4>$D>)K5=4n1a;bcaDG z58n?DHzptfHinZ)m1CF^|7&H#+FV|uyQ!$>MknQf!ci3du20E~O|N<@nf~45aKZDt z&c*7<{XLXpZ)eYXAFdextshS*?v2(uE?r#T?y6>1NHP-H@dZ@2kTKfNIS6=8&l)V% zO)f1Z5;fQvw*0IbnZBP8xxtD?9GR80J_38?Q){kxS>W9B!Ep5TC1x4+mUxPm<+2X} z1HC`hl=S9ozn12Dz5UD!;}NdAwvu$=pq$a%yr961!<;Uf}ms9B5+|7n({>D58-{GF+3mu~t9SqzG2W(iz^Ry%*b zLAOX2!Msv>vwKogSdS_akwViY$}0IUr$uJZs&I?zb%Nn|$ZnGUU!;Q{5T(Y3 z@R8fzf_u-L4%yN%TXSEPtxF#>UI}Dy=yg@yQnFd?Z7OW~=B$qDc~wyRmvLK}wf`yt zD94ha6-cgJxYkuNnF7-_gn0d9a{EkzFf~ywJ5ycAtm8|3L;e9YHSiksu@ZY9Ojo|(o}b6E7hs4guLqYzQDmZRT$ zoSnJz`q{y-3R<<)bPtG%4Kpn{J;Oa$>dc5*TVrM-CH6o(G3#O5&&GC;IF?59%G7w) z>q26~Z3H#CuFPFb+_Vpb$uDYf^a&XH*ZDz*H zTKM!&%0E%%U?Ddj{f2)`WUB#v64_wRlIii8-Ky$i_J9bgVdewduNkFXhP(dW!J>uk z0VK$3j^rEvDMFq7&CX~mA@*7ta;syT#M7uM=MJ9^ibkP{9PwJqL0FZ{fnLv2?FbCV z37dgu#*I?fGx}j|`dqv9^|yyVsnT>-rISm5Il-T49PYtCl9;)*JNANHq<#m3LIf`? z&A6UzEZ9%LcHq%(V@8Z$Jk?6X1c$|(*8}2xNZ(>9FvN+AwM|K_WrE=AIBnoFexMPB zjog5-CpV5jPd(ZEFqtnFQX1xiG{`56U6>;7K}c_cG*;Sb*^6TB7y056^3M7P;R;3h z*+2qw!=m0nI4`Iv+NGH`sNLNhkL<5X8*F$r$--=1L{&9_L>Xduq%O9%mD0GEL&Z#4 zioG~+@cJ+PtymPHAQ<23miA)9Ps!mVZ@@Y2(aCnhK9B5TljrN`Z$xxO9SbX<$$-SW4t6c-r-D%$BD{vlj6oU_yj0Bo{s=}u%> z5S^x71DIQelYfoGKjzGEtYM(9<4dgL)yv?7eN&Crwit$Fu@yjD6=i#ZKn+q6R3(;0vt@A}3J?M9Pr7!cX!Ba_{0lG@mS$4U3mrs> z|J_m~=AP8Dp8d(q9%xeuk-=uecN89>FaygpBHn?Uot!ihVhbJ0xNJ(6ePD=117X>n znlLS{&v;K7X-&pJd~V=z9A^6z!SEqF+y`nL;B8S_ipSJckS&W73lq~I9lUzsKLA_H zT?`Lj0Jt6I5d2(D3~;5ix~p2_=MPT{En`;28?%-HfB~i1Hm6b2>S4KG!zlS&EBVVt~HDqscVp1un{eohaGfe zkuwR_<3RErw=fla6|Jsvkl9MBY?wGI21hgW3FDa3}oTl)Z!aqY^imvzo=18Dij zi;nvE`2p7hT_f7Bv8!WEexJCpvA~_6${5n#-;mJ!sN>PQDeO;5qIACi%^f?J)c;Z~ zg>sToikbfJXV%#L0OaEP0+m|$a&~;;RE>>x4#Lcqr55BPoH35t@= zp|ikIIhpJHfD*F{G2qH#Jg+So_19SYp*5>ysdWkevbi#A^Gz8%`pTl2UIDl@#X(1A zB#6LcOmhN}f2t?)Je`+7%(66NKhI6#`*n4|%t~(1DfMa08O$URGb+eg>do`6CMPLa zEE$~MNNT8_pKyJDLl*C~)5Mt06c~M;UNfuMb>g^OPvEobL3HfzNcUrTICX2Vtyu4# zbazYN#co6qo@8c*E(jtDy;U*GBroFiTrW_sURX~b9ehiCm!LoVRe>h*6F;xctthS6 z2dP4_*^0br(Cuf5Z%%GJQ=Pgg`HcC@*4N)bBCgu9swCe^Q{(ZRDmm`e9iydnv|c)3`Wx;uPCSrb}+Z=szBw~{KYgN)@_ z&U@<%yE8!NDKX8x;AD?4-m>Kx?LCBZ{2p-^v4|G=q97Ru>@ksAvfiLX_fQz#c-d0r zo#_n#7b>so){K1phT$$NDz0Bdp3zKAhexbsn%aZuHBh3GdvZW6HoKh^2gzzV2~&Oh zC{!kuFz3J*K$TjX<&AYb$Qf$^|^k!$1jBU*Dc_?J7B6%r|3SGKC_b_ zV>sR#NI}Ca1XBvBn(auU1}xK85NEl2`4ZFn`y3Sm_JtOCRgcmtZi*E!hyAZ)mng{4 ze!=^zqAQwf-4<@?qwMBhW!5FAocXXPv*k+8()}?dU7o-+@_qBVqYZX8bEp9BdR`p8 zIgc%lk#vT8jG*v9)fjo_^^GDDREoo_W102|BxUEM^5@bnX{jZuY#IWAMGALfA`Opy zfX&pPKR&X;;W(gLebWj0a(M+pm-Nr>tBhIZh#V9P-es?83o9h9Lp#fTHtgS&8`xpQCJ%zy zNLy?D8(}M(v37y+`}Rt2LT59Ju{Ohxyy8%>4ANw=S-0@pq%Lr^KgMBRIJ9J{e~b@- zD65ebYw|sT*>z6*%lTNybw^n#Uy(wy6`2Oj-Ve>bShYDwMQO?P^=eF`OK-;-zHa6_ z$53|r0I^})XR*e|r(R7p8e$dV!ZxC}1oi0is?-vS_84Ys9rPl~M$=cCGs(~hmaDmO z#ho?wxa3ZGNvMVJ4l%teI*q8&A8gtg1EZ5W!CH zvs_8;Dv-L+@{-n&kfQTmmx>J_qWLb}*M8+K)#~)ttXR>Cwb$ShW@?6);NI4n)gO}+X=9#@=xc`(m(R62@wl+ zTvL#>el8m1>BGkmP?glV1bDfTi5G_Gg*y7~wIuVCf@N52jU|4}lz8quG$^}9{)srG z0Po@r!g>F*;K!-(MDVxAwEC#rK$IBoi>jDkDJfSi%YZ^BdmGlceFg6MAx;l@dTe-2 zXC)a4JwZ|-`w&`20eyw}HWVkF6bY=(Y%cd&Q;Rfu&91$11_M?il_<=%X(%2EUM81f z-<9Xp6^&Qdtgd8ZU0?JJ;}X%s|0g;s;;r|l95W>kp#LkpkHXWgRkU)7iF*ee->Db^ zVUxAaY}cX-?eN5lt;I^VkT&lRwDDrL9*lP}5(sVZ$(zO^l68jtaWZaK=5vMEnsYYU z#`yvwr^z$R9fv+dG+Hc%ic#u@Gx$rsKy;-XUO*YJk_SNR^kC}N_n~PqB~eSCe+USb zjoWYI3&)>w5$$l%aK)hF4O>N8ugIV4&%BtnOjmg|ymdh02 zpCPDQvJ`OlmW0CdbjA&yKZv6%QS@)E0Ud$9+3&aWq-f9}-_?} zX&UKe^m7fJG45Ok^s{nxPr;-t_X(*ool(n(Ot_mdjZ4jJhKr3Ha6Bx{y)1q7j_TzY z<;?m!-q65*?N~r^?j8FJ$5}; z_R7xUtlgQz(N1jI@Glq_hDGe!nYqEO(VyG}jrsrCQ)O4*7v?TUwf{dw?;e)sneTgF z?^=6}YgU>toy^px9@w4i)}EBMNkm2D>SVW0$3~jk#w02{ClLWTzxVu~TsiQ_bKk$;_xt(uV*g~WsD1bQ zF_3_B$!-B;gf8+QlI0wnlSduenHk)<#vS4N67}^pwWBY@^+#JP%~x<|r%5Abk~TfO z2u&0}?sHy$y?7Z%IGt<9M`&aXK0)V#PsU-lCMYnD3~1c8RTQ4LA$$;zkfDZ-B&C)X z92%CnM_iAGGnUK7)>k3FyfKM9RbHAvGAMLi(1sf6*c0`=Kk>fElf4E<*W0Xj699}y z(dg)sTjt>H-w(VuP^mzF2B9rC;u9E}%(dkzT{58a329}tqav@+5W3t2M64(z(21bZ zNv6_-qSGiBccr^PCEMHQi-es_c;?-XPIDD$sw_?r>GgJg9x*Xj@}T)Ch2JT;RCkv! zPjO|sC3PcU>K8`>#ObEmvaH1Q%h{RVH1NUZ?>6IxglixMjQ~uX1dT7c#C@he(UyuV zq=%|}oMu?`3(w7bps4BjN=Mx;PNU`=I>mswcicyqYkQ|C0XwU;^J)xg^KtJ7BMKv; zthG}@9UId*Gk5Q}(O7AqKzkp1z>Hus&@!u6zism|9Q46<4iYn4ww zXk5F}a&R$YD#)qcJ0TNO?7u*3?cVH0mpQD3gr}Dx5zsh-+Bm>KL?Shu)`xzJ!;!~g z42?=lKBXzYajFlLPJjkXizGRXcq9e*pi#u>L3bHMmG`}!YiPicUW=^bXFF~u<7CCY za{Bs}NX=@3HJOHm&&^VBvD(YO7Wi^d0^xjW)SVA(-CnowC+dHSO9 zw7PJsR<^h8!4Sa)1AahN=}a+$5vFiDFWeRdc!gGr&9BZuZ9OhxI-&lbBe8U{|C$KG z?66@ygb|e*jF`z!8m&D>l5-$7EBr$Ya!-m-v@E*(j=|17PFZGnwo^)#uWjfz>Z!KX zzf2T^JY=2*(y;5P{*~g&CqKW;rzmo9;H2)7loe)F8%tI zk%OjZVh`J*{}onClMs)gVOceOLCAXu3GAnNX#i7HDZ+ZBt@sdv`aV9#MT9 zXhucY`^+<_O)zxaI|o71wgQOGYE^~wbi#fvPp8H@5{k|yb62N&**7Uc1=^{tcega# zr^|EOg;f`$10p=tCQo#2F{xJ{_VtgOibx%@`fB>C6J=U5C09<3^}>bYK&^rX=&u0{ zezY{Wfp!16$F`)iz(>tORL*v%Mgo|b2rSCoj0FY0b3xT_LvlXAUzLAUaew6eq zMy8C@60H7dDf`3HxFl!&P9D0Vy2b5P2lqJ7MG1se3qBpVNn ztF~?1k(3#;@(N?TQA)orG;&tYLDW6$C2Kt62NnN9#*>Y4PickM+DoF`R!j{bQaO^t zunuRoP$6D+iJtXlSA2753C^)`B0l44p1zJ&Z*eNt*7kqFj)AT2 zJIu!0<3;GqOxm3e#zYHjWRXyTAI?KGs0HbN&JHlJ)CpzuzvG~oz>#nlAXJV8y-_cG zHc?ZKa*god-wg3%j(Ijp#c)1p{!vyhpAx`hV#9%f@|lFjg(AQi;4;l%#MaPzrLAw4c+*O zhCVUMM^p?6=~sg|Xel>{z~Rw>jxH9}H9nmr(#*EXd`-W#;1!WI^*|9Nb_Ydtf&UEo^+h3Aj=4U+Hh|HyFZZS(f zY1v09n{_xI5a0ewmwMPbi`gD7Kr~+ruu&Nc-Qqgm$vMkvD=qpoGw^3KusCE=X0M>V zsskFdU*r8FPx7j9Z#MEoonJrO!JPBpI{hMGo+5jb5E=ug5no-Yjb7?2HW##qH;a0g9GdIf?p zG|XQ1%;;_PxrE{OCa0BFzw7n$nbmD0tpd5+0tI#MspDn3U!mV=IzMpeZy4t)wAjxW zVONW>PeI^nLchQ$MqcDmg3P~9?1(GMPraF)UCgERHJ{19)Jn1?i`3`eEmbg2=#GMX z?RKm$o^!_;T|WrxNO#{yg>QFKdlfhHY4ZHEjLqCE??gv-)~*fJ4*m#m)P#il4S_yo zQt^-3pfYj?Bu+eQLuV_7z-azzszj#_pa>xa*@YCsK?aF7%kpebE}1J3h@bP<`-tpfH{(6D;WXj3knFy!7^| zGT26e*s;||zbAQGK2`fYynfJLK$1`(%;;nnG}xO1&J+y%c1_Z$?5$YWKlb?wqaSJ@ zV>qJ)4L4TpevLGac=?7ju}zK^YbQH|<>b1ZW5mb~$p`htG;<@IfsRt$u>8X{8#1Oj z8?OzlH`(Qj$uMdClXp>j3fb;Og<4CHH+SI*MMG8R#dPR6Hhuta~b*?$*b| zY979?CpyQbThzy~-j8h77Q<#`C+M3J`5QTsspGR*_OH(_ovw(6@p{!w+vEF%JP4$R zFX*HP`biw2X<;k}*s~I(&*CJea`9Y@%@AC;h(wweF&|7k*fit4)%ZFeN=tU?CM|za zWnD5NQI7o>(|)`|&_nAg5qxvgrxgLg=R%z1VFCZv3^omLKn7&sZms_Ugpbg-wNW)R z9KnX{*svX&8F|!Q3n6bFFXK<2sr&V%>w>y} zjvKf=8#rTeNSxAX|GhV6NA0bm8Ry7$_r5ath=4Tfusa=a%j21u9P!?9QhqjzNJ@HF znP=(_cP-5;zf47ocPD`oUc7V|#NVkr{|L{0mlt`aXoU^P&SCx<6T>31f@YXbLa)QQ zP2NjR->rO#k&zX813ekLwN!H>_KV8lH%H!&xAj}!n}xPVU9z#dOUDrf85rXo#(PA3 zVxsE~+-9rjq-{RUW6N7qv&WH|8t1OEOWX0(&>CVq&3E=9hQFN(O(dAjaeNfeopyw}E?Of?E7Q{!# zu>p~QKFJb0FH)G~aU&|Y4-MQpox+XR6{5i`|EHt`*<}{ZA7dbCEY9c{>Z@%EFUwey-df^? zN8j%_b1egdb)55fDJxgAV95*=ePWB#{4wLJzUuE=7vR+PY21cqi8}PSDoRn;}duv5XS^949F!hp#$% z3QEx-;D*}QtYl_{pXbxphY%qb6KU_ydNCfc<#%pBfGwFniPsCZax0OW%a`(I!Znt! z!?E<4rN0bqY6T$u|H}Inzf0n=6Al=cN6m+TF{N1S zdV@*gD#H7$)9O{cUfGJeV0;u=&PZ9$t`N9M1HB zI~a~&;dJ@0(ngA+tDjQnQAKuKiGGGWO7TE|2t??7v1ky>TrcyX!Dm3Ud)(M4t=D#9 zLQRGx(tC~a3N*k ztIJ{6%!#2})4(Z506jOl{HRtghKv2B0txyHq$5q4G{SaQl(vYlWo`aadRZp(0r9nEhCN}hB^ZbM|@ zi{`9QUZPs}pkk5_&m5K26TOot((VuJ12r63q!^SDd_iag1ST%{9*su`hI2Ig@vXr& zW6eA52Q~b0|Lv8%H6|D$Gy_V&(<7&{*<5Z?R(OC_-`l}x-}Ln$lMiG$_e~U&oY>mZ z;k1z~l3_v_sqj4%+}GWZ$Yi_o%-gfUS}DzCMHT2x!|02`I{UAk{VAR#>UsT)J`1@w z&P(uJ0c%v?_PoQ&MJ_PwJ=1yMbHr81?`u{9^A7<;E(TOGd`z6gH=fuSs*jE3RPDAI zqUOFPOGpw+x3f4)U#7Ghpl$TME?GM9*OXTZz#c)z)&VUmChfH)&?*YfI1>l*Awt)~ z0{50y=6ovXXDn6?NnJhV5T7^E2YB^60V9Ow%H!hNWlt&MM@ttegLjiAv}u1iWgV@v zJ$7}~P8#;q29d|<91VJPz(gM239W9e>F4eTbcTRYsnYJ&z}$CLSo)>0%w<_!op&JU z7Ake9n~ZBK$umoaGt(y}=RQN%dS?=jd*QYe<%irZ>pc^%j;v z>h8^kP@f_PW|Jbihjn$In;pMa;02;iF_@d&g+*RW7dX#GEIhEm)^|2O|CL=Ca^US* z-j~6Fl~fmTxJd~I1jy=*kFD@n5=Oi?@7#Hj6w)~!4kaE|m83r`mwwaHv3)p^A6AhJ z)^Mt3b8ET!Xgi{?6SMi^9)7fXZZ?FJ^^)k{opN6!w)STQ4oWQ<_zK?SkBPK*QcLw) zXg$FisS)UEf9c5DIBk6cpN=60_A%r}&Z$_i3nvvfSEnoY<1k}jMO4JpV zDm*_FtaW3{f?w(VkM}db6in+KySx0n{Ad1PPsTLSi`? z7BlmbYDDam_`>;nH>67~e$#Q%%^a9wv1*gzD5(`?I`E1%-BIhMgumZ&3)yEa6Y^&H z>id%fLFz@e5X}2d%IysTTmC%_fU7wW{RefXVytO!sar;W zm@$Ud=>N``1ov7UrTxr%`0Y2jLB_;1a}MkP z9D`Qy+uv=EbQs2}+-rBF_PTw}!-B98GJln^)Zb>et&aC6OtfrNJ=YNx52#gIwpvFLc1iC(l4iHt@jBI9kE1`U^d%%S zb9v=1B_>bB`ip_$4LuhsRqc1Fmf(Z4iJ2bD zY}(ltjVhi^;tvkXQ?;QNR*Yvq*1cdUf9E)~B-QNII61!N-#pS_ioPbip9IPqU6>FU zm;Jy7kpD!HKpJH@!34h{-vkFv#=aNy)7s6E|2f8C`{1bu3lXq z^h-cNOG28nHWinRm`ZA+k1p|@rOgA%(W#BQCHb#dmwFy%`vfuUo=Nq0ZPZOqb5j9lvs|)qhn=<@ zYQ;yovc2G$>h_>uWrx^}|KY zOg~Ok>0HEEC)6i31$x2YhC2u*PV292ghse8&F+Z`6DRP|xt+DVSad@TBD%Ztvo&y4 zf}(!n`#Dcn^#>TOCc+@??Y@$fi`P>qwFUrO{{-{(slio?I9%A7iPGw6b#?J8p+z`DyWD?JG$&JFKRpMNX;hPI27 znMya3FohI;G~k4^7UG261?`)}SBfgGE5hNehk{+YuT1Dz7Jn?c4;5rB`tqn#t4|D_){mt0qYNJ+?Lm!*-rMsh#y4XJ6K=?e_{!($NY?wm8cblGlH7Dk7JxPws8Pa^X07SLWH}p1w$bxN*~$Nz zul7o-qd&cM_~STDob>~z6rXPRS<`+&H*tSQtiNEy z#?8macVDp7^@EJ5*dOOu#ESuu0%~i?hWP@jj)v`t?*4s@vRTyb>jmzXC6n@eS0n_1 zhNqpd=sNlUOshohJ-*2LNR<$M%8i#e*&U<>jbY_K7m<_eU8%-+>_cz)~M#c`A@u*SLj|G&;acV3}ch#@t zWQDZ6c3B;z=znDm>m4A~v^+zAC(k6_!&oav62m{N{!@sDd7W`)glmnt=+*J9uTGxQ zl7?9BW!q%lZ1QXcZZ?1ZRAKOoBqsj)gBALi0tn%h6MpWVg^6g`yiJG)WR32PGm@RA zRzp=S-Pdm|3oVYp$zdxx-dc8-n>vEl)vxUhPDA};N#3(X(LauP0}jOO?0_Lna*Q5G z@Nv?}J0bhzqS~dEWCQfV)sncn>80~D@rAyNet^XJBO))PcS5Dhask%k0X_s6-3|5z z-dM_D`GCZ-UL+{0UpYc#T)gmy#h2&KzVa~$AnuiE8>~8ODst~39Lk1L)(j|sc``3$ zXI7MDjBz?|o!YRybBYlYlgwDCa&10(d@GrlpFMoDJQeK6b;FbU7%Z=k6IckoxnXJ? z$;>G1Zv|!_EkV^W47-u${y7g0%)mqO=tKm_Vntml93Jes!}hrb{}(X-Z|uanyw8lr z$f^@Bv(G5GXv~U%GLh$KpSC_VDsns02h$CEK~%WU5S61cYfX3j6ZiUWCZP@YH&Y!_ ztIcZ~bc=#8_ZKiaxRrnvM3(C!n!ejC+tWuJvytcMq4}E!iJ9v^LQvJy!}2-d_3_I& zv9D0Ik!vUv(Rj5hoWrNZpOSkAappUY5s`DOUNwG1#Yv+DrW&4feU+#Y(#*@D9!BR* zjdORR?xH%CS-Di48Q93}$V6s&A1as=E>k<kF&X8RkPO#g`)={(WKT{BL()sj~*E z<;lO_FRFTqa4grRXVy+mn0^&i!L4T*hy(kmKlUV{{8AQq*Rt4+;E}gIM@oUg%vrCJ z(7VprKH-)X@RcH;o@OY@0L6@BhypmuHhwDhlm1}wm|Q}5r-1I_GYilS&)?mJg=;h@ zUyw-`sc6eH$E+=RVb}OJiB13tNb=T^g^8c5HFlh@V=s)t(5z*$)7A;OJD8Nagt%!B z(%jq0n7fgwPMdVEi^!2KM8RouivKNGNjuS^#b_gc3eWNUca#jC zZ?!;`B^G(`unK6oVnk@Uka_y--sa!h+$;Xj0#P=&DJ<~~ah;t)i>Zb*;M@VC4E-bk zLeX6epVqhj_Op7MaC+mi96((VsILa7PtIq#yPNx*QHC3o0!;V3aV!(FalOL$dI8xG zX_OM2W1pPSH=H9pc&`;wlBVisf>c4#E?+%E<>$f4G-0%*b75LJqwEw4T`LW%p=Xsu z<<$tsb_)G&kfOY5?TOJ1RU4E?Ii0$|{~b)Bhs|=+Up3*TNCbTVx`eGYsO7;KNRrV1 zIK+%@Zr)THmjHT=cG{->A!v1%=ySEF%Mkz5;EXh+_Q6O^i90`AraW$dUtS$g6juj} zl>3JZI^u9CuAgA)1W_+wGEYIHqZ<$ZK+TS$UL<8wh!V1Gxe2$NK^e%jUo;-(X-3Tz zQv&7@whrLI0@=zOY#YgtT(lzGC`qJguuy=I{snR~mKe@(4*G{g4o$;Te_)-=YKlRQ zwjTEev%{G>%~U9Yxxx_8=vzdjGl`frV@^Wn-Bdg-ak~v~9=tyLuY>&Z}952%$#I_0B z-lQay6M2jegUzey<(QAV6Eckgn&(lsyYf%ppT4;X!ECjRID}<|olu(HHOV4BpK0MW z{3SN%{s5ezxw%+A*?5YsL0$6|r8V4+@l38QU*%>E7t-yYjQzIw0K7^dn6-~QC0MHQ z4XhMu-GW6!FCMCuNLVZbj!?WeU&94rJh_=e&E9=gerl4cFfm z3P}e2Fx-Gy_t~$TMS5c;j`rkuK|z+^3H4p4*RM=05#N?pMf67$ys?NW4vpMgFCrN- zG+BdZ`(Nzf z?7P8>f`tAV1t>D^ZZl>fM9&k|cN#d+5pM3X_eOM3|J!t$VZREmBdxx8G2=m!&{ppC zq_EDf6ET0iCJ--Y2=bW?!`8&coX-G7o|U4jn;bk?(5%R8~O6<|<35K75$jiE8I~*Z*5ii~cjZtze}@ z^jrAIhzL~50&y5XdL*3d8i1%o5X%4ct?~4&ox9Db3$)4o8P)eF&k5Zlp3p<9OjAIO z!V-~zUpTMaE$|E!b!5m*Z+A-Y8N#x(CtA=r{0TB~OBGponuTXUvW#ynMPLX;> zAPahq_ZEn4y=|%rJFj+Rz6)y+^1{D~qU$V}c-h*yeEKsk2`KR@;vFfzd@aFoAJ@wB zG$gqlmbdAYI-kFpNga-S00Zgeq3SyC{N)kPKG_kZ9l8=puqK8456i_faz~c$TN1g= z3ffsdmU+hD{YR2?;W%FCaj_;4UbqXK-3Uyvy#7=tpSs zzriJckCNz_g=v(uVqzu)mT;3*zhRdJI)b<{+~GoJ!zG#bZf1q|kNDB>PcWYP>PW>q z{IIS|K5Zp$=RCH(0*b64MQna(Fi=L-vy6<#?b(LHn8A#8qEE;d1*27t7bdImBgo1- z*}qt}VXqo+QQ>ShetDqwHjIIPOGtO+-u9=d~ zmr=6%_{3`0u%Ls|oqrpWm$fBSDqiKKXX$H-le3Ej)5(yI&k*`)qEkoTl=G-#Ea8ST zkz^gKom4L^sVT7i%H&H0Zv?%IC-&nwkMIfW$M2~`n26kz9KSC9?QP+FqH>zR9< zsuJCZ!--w%ru}6Qr$)bWh$)twA(^<{@|+l}Xjbo8j-~7lC!rG@7@^O>R~6N+on+EB zM}@mA`c7;T4;cnd?cQ>bpSN9>cS$xTw17n6OCafd-}6H6B+a&;Wm=m>6q1%~J`4@t ziUW+jVPIk?o&N{O7@4oc&1Qy9LeeQJUGsQoZ|0PtwxZVfHhRp{N^xY4@ZAdt=cuJ8 zuC+7c^$m;BO_{;A=Z8bEP@psN)KD@4G}ZaH##6ty&Q^RZINg6mw@-BYcqd;+xzZ9@ zE&_YDEEKWsOSX6n$v&k|m(FE(e7RAqC@pJ@v{LfTv==ZSJc>&s9{ntRVDjF8udxc` z#Oqc+0(vKWI9&M#)1F*5GU@etv2A(E1gs5I$dub zo1~Kn#=i0{DR*W(P3X#(j3>t;cRG}Cj9_>UZ>tFH;I)4j_`fX+O}{JM$+5L(R9BOH zylBV_4SkpYs^-VihR2h7F^NX7v^IgKQK8-Yfq`6|b#9J82fr$vt>3XT8@ggpTN}`t z&>OR$@F24l=N3^W2Q{#M1>df(9ixA_G>Nl*1QC9u3O^^sYwxRNp<*T~@UkYGGPx1Z zUd=A47-r+unS}GfD|Y`JOs@rX-vii2wBHKx_B%Id9MXcp6KSHJQkI` zDln*T>b}|5 zOvVy@DT#^u`${xtlnEzTXCo2+M!9Y%Z7dx@QdJ4eu|36Xkr63i(mmP1Sm$Xi!8|94 zH5oHz#k;?3H4uIf1vjp!-y5j|57hG;hG>7&5he#}8z7vEToWna_0z>9WJ%*QY~D@r zvyx@&clIVA{=}(IcZ!Y< zJEuVGxpAZ@*B~$)ZCPthojH>}PqCx-xXXjK^^ zO-bU=WPAGADY}On3C+P1GQ)?=iT)S)n1n>BCr>P#~6 zS3}xhS4*Ygi=NZVZL{sSoKsd$_>uCycJp^E9kOUN2ac>O7ati&2c}vpIs`l)nzt@@ zH0d1kMfQ21xPC{Il;H`wrNbkqIZrL?uA0%;RHE2cRHN~HeAwT{i9C-n?3By=>mpnc z((<;iLlkTrWOM)eUzb!LbiWbk=7&rj})9nrx?W`0{jMAr1pBp94f zN-)?i(uuwy5tdb;nucaVQ}M!x=|dT%{U6nmj1}JCAF5r_W@f`)4yHEnFAzF?kbV?H z2@lWty6b>mXsX>Iv@ezCR)Uo`H6j)L$-X{#Fq^^X0AiTj^5T^6bB1Jn8dy<(VUpJ? z^t=NzQI8?hzvgflBau&!$0fRJAY&U7Z1-DG&lgO!THm;q$8iBzHw)%|hgW}J05{x9 z81O|lNzsO4E@cy}2W-p7M68sA6o^x54K)r@fk0R-u1J0bLVY3!qIX#ITSas{0WGv{ zI~uB1Uo5*0x(}7SBgi6C`IaRl8g_lJUeo-4aiD4<1?PsQ2?7UU`^3U}bb&Og!AyS#7>7>FVVbed+_HvKiZF z*JJQl0Q3gkM+ZwqZeb|LFN-j8L&tLs3;0Wyl&15H8iSJt+Jr{zi+QhgI?xOratb8Lh^IwE zyztJ}IHm?50Kz7PbfIZBGQKs5GC2ihv?d>AuK`bQU-EAqk@UH|%$mEz5Z`Q{7Ynf2c1+10v99CG7R z-vQnB&qojstj7vRDH&(+e`uzO5G$Ryv7U&$bNh+^UjNP#H$JG7oh}*;U!dL3aNjQu z#^SI<3d}=Pk&H@w!{Z8`%_-P;cn-jLl-7-gLg#eA5@>5yas|BfbPf1+6Za$*uc+5Q z4+KRMEU{zH(x!_%rpBJxB}8Yu5W$U(bl%<eIo60IiufcZAwYl3&uT(xY1_b}=~ z!#FDqnM8}4NnA~xSy@BC?eS3q8j?5sW4`&X1)ep#92zO1+Vgef&7642RNpRKdY)8O zzkNRjf3NPpJE{kf?czZIXJi)-lr?9PD`jsfy}9jK7#>3RQu@VLZyG%U8znK$`HzNYj4JITsZJoogANay=12RaAJ2h3A0e~$R^JvOwXunv3)k(Gyy zQ-|GqCzM^7du-!_?_~Z-9UucXY#m%tH&@sHsIDm2S7t@fzR6#&8#h89Ax=Xo7fegu zj@Z@(mO8gmS8wkfPljWH*Mdvz68^A$-J+cO9$L5c7)^6uMg^}@nMpJ&m!+yK6x7-$ z#{W%s6_nyrI0aejrj}v|rjX?B&8ICN1|Bx9UT|r4v@AXGMproAdHC62OPl}`J^?75 zOyN0-d(R1U50}#JGA>>(PJLIATlwcN-&mN(|MArp|^b|;8nU%&*84x)h zQ%M-1{i*;Jwg{m2Sl3_qa5%Jq4ePpd%xKP-W2OL4-vWiyWqjLFRWyNlGwv~~&^Rgf_Fae4o?yVP_AV&Bt!kMPJq-{_X z#aI`w9^`5+E&e&Y0WcuPF#(|=GKR`}^?<3lB|FQs${)AQf?Ta=++%j8XV9IO{l5+z zfQVY-PtB&06z`j$5x)e?sTRO?z6>%VXtlSeS2wtC9YtlpE0+8YzdXmf8eo@pcS48U zib>0y%30{H+AqIP(j4c>dJdj{i_Jf%vbUq&dAUfrw!9>^4VMc>=)tSAdu&9$EJU&z z6S|$~j`Y<2^vpsQLA-MUR_!LhTwtSJ#-uI`gERAyXKbA6g?Uq*vELe z9YrEa1t6u+IOz5zBYL}M4ArZfUsC}v>r}0A>KG)_h1z3-Cf-^lI!!<@J>)x<$0c1x z?z8tyiN1i{!J|djye_-3xqiohZ5Yp&I?(iBFQE9$udRoU=^Z8(rR4}jHikIcLEoI@ zT&OH`OyNY%3&oV+{M^H*YS#_8BdcFFh6EFz0^_mfj)MwE)?Hp>*rKAUjl7+ARFX%N z9JS3mIiZ&)j#byynZezrw$Ff>>Wu6)6*$+g(idk?C5^jmJrG)ea2(z_Mm9biEOg*H zs4Q5$#1NcL8b{`--+`kpSi@QN@w)SLYh64l4?ot8Fx<%nRrnXJ@P^^+@o_E8??X$I zx2kIT25wR=5T*~u-fn|vF0HLC#|x_|iP@2^vU{hheB2{E7UR_R%{Lmt!t?ZnH+@**27JEAip=?gW3K>c4T zKaLf0X!lcb7be3S1jzYg@mY?gW{u*CF)47MNSt>4?STJ1Wzg3e6U;hQd@LZ%x!0PO zY3mu_q$pk0-=j6!gbq`L444P2+i zS4^dF;!b6Edtyo`)%VTnTm+%fy!l8!DYhdL+hxWc#>mGSdf-T%1`A1=3@s&D=aDaw zVyF`H$RaiNSsrKJ>r6E9{Rrud9B+Vfs)K>4`@UT{@sb+3`+;W;R@->ilN~ELq7erN zBn*AM4MeWyE9q$%xQ6!3UJi*z%|xCjt1?!Yta^KPbPjERSLQkc;}D&%2(C?&0VaM2 zaBR#2fVwjpcwqr)juySS#MhG}K%H~;v}x|@CK_KVvrKMPEhE2?#2@8B=z}0PxqGJ^ zT4$STNjjVlKAu{5W*zr&01}Bb>T8FF4>h+e>p{5G|3f~xNfeDdnv^b-zZ46mIrLHI z+8p&S@pE1xvzL=OICf?PyYnN=n~kPD%gL;;z8VYl)zkOl3hBx|Z6k)iNIjlqo8m=x zlxQd$^%l|ik7&pV1M?0r1lUjaCy8P^6F0sn6ql17y&s5zquB=6`f4|3XV4c})yZ~q z1W`|CV4)T2$xu}q8rNQjebu=)v=2i5_-w)aM9aqpZhkRv{IKERH2xzD1mXAY%0}8n zwbV&@D(s#Lygjl~-%E^~SKugXj$^9DlO@qd=icU5e|>iUXXLfrNgW}mpRnLa(XXTb z+nFxo%))V$iHFUyl@p3qd|@F3<_G?r~tQi>1)H7@wo>&itABtxtSOI1TU?g?txI~?P3qe93cmX>yp+6o6o?9zWR!8+1?f&fy1^;k+H6Z zr%AX>g0s@hOxgY|zy3KWtvv}#8ds=Fs<5ijxBy_$*KU5_^_%%XX-6^Gm+tJ^rjGM& z_oa9Rj?C}F!zuBt3Suf8!cdV14>Gcqw?4jXEO2X_hi2L{)UU+R!$UHOe^8KVu~C6y zaHj@V&3`4_4|4#Fip-5&-K`| zwP)%Y8jL5}xWWxREP~LV48}~O!yc$%Y-~lHUK`#enH|z0njM~5z1Aa+yw0Yu9Jh;( z=o;lS6%X$jETVC?c=<#UY3&LlG1^5RwF-+j0M1{GCuP3c9Kc1j^+0LRox_NE1CJ7k z$%DrVFJPku3|OsuV)ZB>aE+4w#0&nNJEQ&QVNCfenm$FQ*r0uWeBh++J^gpFGVgLF zy>@kgo|Zw^h@oY&Fk>>0YsR}<*%11aZQjm+1+Q>NauZ%kx>8Iat`2x+u8xMi8y0qI zQ~RHhyDz(^i@Ot&VTG>}eLa36wKqS_Q-;<-%R_Rc{r~7B*7pj8kt=OX89_oYE&sIF zi^>v)tKLbWj$Z#R*Zo^C$WaN0PZm22QMNl42AcI9L9H`uY=%X3c`ujUn?aF13$nA4 zl{nO{ms?#`UnOr!P4s-h#YBs6_2a5D;w`pe=$MAy0;g@-&9&6ijB}Un&2l==uQUcK zW0XU+b+dah^!pTFN8jr!SiL%WwTnup z(dA3jKF8yUgEZUm$dzMDQ;?7zmX0LotJ)NEuz z!i#=V#;Hcb^|f_A>%JwdcV%?3KaR~(Ss?b=yHD{dopY{_9^v1g~ z%hOnjPU#DW#|Rv;(~$#dnA-2tN}a#b^!u%j`TGu`|8hq7lAV`u0m2z~o$ zT<(+rURIzpO?-0q=Yc_jt!Hl?|Q-?bHR(b_*GL60Ix)>R}Lf7vx{SnrSQpgSK>G^@JVx6BSPAYApUCw7-r z^LF!1*>>_IeJ|XsDpVC|HImg;Pw3_+#o0s*R$*LlPBQ*0cp*4G^jbnN_`jV0;cPG(du zUr;1IaOb0Gfj+6j+Xs%nvS=Q7g^HG7k>8Bg4gImQ0rHCB8a+6~y?3QKhGec+Jmf{E zsnPdS=vyQEV3H%9WaH*{1X7*uwt89thlYR4Mw8m978T_=HWH|`E?$4Tq#$u4vh!A6 zkdfWR{{6cpXG{P5`u;vpZY+0-+ygPX#!3C zB`kP{S64B3nx`nsG>(v4%6T#jILXXV;BfeifkZofg6Ey!fr*6n)zy~qtK_GvRm0-C zAC!T4znkt=(5tJR|N9kwbIiMCB$tgE6SR672)-gS6zs&uz_HlraN=>5(y2a4;XMUd zG@-ORKQQhtU45NyLu#c7W*ZtJ%eB$-xr&oonF}PsDCu3 ziJ0_xnleZI&Ol#c;0wsv3f$1*?~(`7rDXR+i-vFPyEsgGKegMW;1JD;!<6OQY==1P z;AbC4h`amEamLTh;b!FKE)A}u(){nhXiD|Iz8q0Z0XLT0I!!7q>KGzfq+DI_8hF9^ zvqAr%%qbPLrz?3WC)~whOMgGbIr>eVa=$<;pg3*cM%Q4G)fV;Od@+ zb#us9Ytw02ZK@@!z|HYclLV&sjKXq(&V;VtxoaEwlonEY{*hEpB6)|(;@g)U z1IQxQ62vmupARuxRxt;Iu7l1AnzxRDV&+{SKp6HkTW;9l|Qh;=_dPMS7PE=+T z`jXJbP1uoa0~ifG@P^SE6CQUs7UT5ji1X^@#rHu{V$ZGd@8|9WCZN$V()Xa{w9riu z*eRa=ETm@iugDX?HEHa3ZF;ZFzFyQl_OTUSw;_JZdPcu#GK@#GGfi)ir@brYOce3|gu*qCG&wDiM`%aE#h z(D_U6LS)2kEPaw<>DEDuJ666Q7<6)RM66_ zfW*bHs$mv&^~iuhx0ECn4{9(WLr*Lmi+Su}I%uL5J-Dh|saXHV2M*cv@;@#V*{)xT z;-h!YMNzz@Z3@b|xXg`&+4@g$3+u6*kFPCSz^-96`GpDxRLS7=EQ^n^LkKw_7+GX8d*N8?%NK>_>)Tnu9av1MYcTP;fSJT&eR!$bC%oD zqq-r*t(|d5YwS&#j+%5)6vA;c1%1%Q5wYaM%J#UU@N7n1Zf$Br+$wpbLSCtTf^5F$ zf(ACY7;iwjn9?t}jOstp*x|AcKm26 z!6{8Jfb$2AL*$^j1MPfqTHQOVn2DNtsZ`osvuEzaS+WPc4Vl**W4}C#FK@luS!j!r zg9v|gLki3d9xUoI==Z)dbL#^)YMGI_@lf{{?EklTe9qBlxK1FJJRdP=iL$hFs=}x; z+t$$U5)TO{58hgLN`@>W`h6C~6QXgJVXqGduK+>TuIkAqR)La6&yMTqI?7YSZ-Az~P zs5UHf>|s-F9sR+S6f7}7>}e`yShRG4<=hCyg0uD8OzQ#JePCzoFR zY$aE0D<5(I0zFhRa=KQ4ynNbHJT-l?tCaYXrT=?cGwW-jIh{>+OGyOE$GM)V_R3Wv zp6bl0QBygk)5oKBqKPz)KB2=ob!hU_Ds9j-1Bvfaw{!aMm%j7QL&R)1=+mS;M(7qC zzLA&F@~aisg3sNrdTKSi(zR{~--0#eEG74Mmkh)xEeB1^Y>GxU6-iimtHyQ8G+GgI zA{OD=Q#Q$7Z>Hc6dK$&Y)FZ2&)n92KUNT7v;1D-c5wg$`d>`pBTk7&_Mah8;-M*8^ zwuD1$t3!0-+*pgYOuWFQjvrBS5kQ1rAE%=mTl=LYo^(YuWqj_O_V*-9O!DUVGDp7b zik*3t)~?YOQqG<`Q%a4Z8ym-iZ`Q*;&yhWu`e5*D)KFh?qF!7wM0?Dh?%zCEbLpH; ztJopvMB7CeN%=^(zeBF)b8*^A^(@RkGTYK6+}ooif9D|4O~sTGU|LnF3(Lbqj_+eF z#)B`Fo;p?H zIh&U?L*_>I&*(&}aw5^dGTpAeY|uHRO|6ZF^6gczCa(P60@+EFAACO|{gULo9c$H9 zz`)Ka@Fe3AJeg{qD=xPg#)V5u3FGTc)Mo^nzW8N>s6oD8x1f`)=sr~QOc%@>*0JrY zI73rQs(~i5F;`kCj?wD9wXV@ysuHy&g2kQPWNo%e%X_I(xXdnM>~%_cFB_N}7`zY` z{1c>S1tA+ttk9qLI`n==$cT<>*{ka$wa0 z9*|R_vXI|3bt1~L(lyAEoqj|kjA0={WO-$t;!t0<#rZ%uSnQG2;-jsw9Srz@nG2>I zJayZZUpC(DAT`?}v{aLM{cp|535?)+Q05|p_BwkgOu);;qD*a|&EvkAznd%Sm?d@t zyfYSlq2uB6`q%#LZ-4ma*N%m05<0+xGnk@*?aOLCMXieQkT7IZx^-Jpm)ZdYdcnLK z-T_h61xFy&hw*D-^$rpV=+BYD(R?XFC!7U#>1WozkjGWFn=8j5f`$7Y9i{VbaDvU4 z3$B$KUjduZrh8FIBSDj2eFf2jC{Cq+R2O`hlXS_N9C(cZXt3cUL8mR`hVcgENyru4 z@uPwEdc`x>=a|Za4|5?BuXA{Kwo2O&7B;Od!uZX(s}+huUGC;`PlTJjICal9XlU<{ zwfsYKv9P>#XtX6e?E@e>4J-$5XuIH_`aQ7O8*PAfF(;`t-eu!#8hT*vZov=I-MT`h7zbK7{* z9HQrcN*CJ%MT-UR=_w=afAGBzn(l7Lt`dJ%ch_0xsVCmW%4H{%(BO|<#rhi~1_28y z1d=yOT^ON4@Lr`C{`|T={jsiROOoyU+NVHaTa2n0AYBu={NOKv4C?kt01$2tKd;8$=WbO#)8op8#Niv* z&O})PJ^#ryDQWw_GIryCW0!C+77=UdM4#|Y;BZ>U{jLl1=LR4$*D45 zcT=`~;UGnhZ$=WG9?U?3=3)n1%0ue>R|LG>lt!LEB~c<<8i|L7;3;CfXth@+`xQeo z4UaEgX#7MilrCd)AuN~ohsM=?w{6h%B`0R@$@GyGAgg}#$BiL}W&KG~WvSVM%K@c6 zc<>k>%bo!huFwjAQGR8Jo=6M|=i_4F~9s?2YEiQWd>iBDVDRrc=a?3mf zPzonp6f)U9?BaVbs}<(K$@T|N&zU!7ODU-_2WYRxUc&j7e{b-g3g=O{l`?^+L2jn@G#thbf3_K`79gY44fXaZV|JK zDDMhaoF)R*j|o*&EvM+6G<}u6&*27anWq9O377~^6KIyDhLpf~6`_yrKQaRiXXbWt z5D=Th__cF;t|81ax;>qI;Fxk%6YlFh^ICy_J3Hnvpn+lgvl_Hhn5K9BYKf}}Xl2tm zCq7hc1bcft!?W83@in(wM#x*W+TQbhe|0_kbV5D2*|LdtoRkicx3B8=-xhQw z(u1FS@baZPM32GeX-1*YZpKfO3H#4b0i6?|YgPb$Vjv`|psDMPwO#sMSrksAYDEYQ&x+ z#9Q@rJ$D2Nt#XVn2dYz&OU>`bXrTyHs73GPOj{(Ar)#~Ng~lO`$G4iuD0yDeH2u%q zt4WvYQl?^)?MmkA)1JA@&TJ4h-hEbh^aEi(>42HM^JTxQT0~-53dh|C^TZNzxq9aW zhY{dlsC{tBhB7y6z>ybWzU4&$Uo~!1^6!|yl%SdXcbT@F8Q5;2k>KT_Fq+#J4mmTCupAYBX{^rXbJm^~DUW<&>7@j0oTDqC? z{T3Xk(Xu`h2Y?gz3YaRZEAR;u^KK9OQW~=()o2wLh+mD`a|{Xi!otB%Oz4Kk3PFYVg<8pD|6b~$Rg0zY=SmkxnI>TU}9I*^d?2*>+Los`|` zYdv_~Oln3vsXsin`9(h}9ATd0o^WQ~fLl_4$)a4dJwuI@&Lza?Na|~7%I-)f z_@iqy`v-7A>fSGsIhe|ynupxaY z3>`qN)mfc;E9wOL90)|QG8I$p5$4i8WuKT-;-5fgqh8f`?J93o7x8hO#aS&i$Bdtf z4!G*A$aupm*VXB#aSe#r0yrPFmS=>Erp5xUBfJt%>d=1m3=e11!JQ~U*K%P=?|p@q z#M7_*MglzR{)_b;Ss2=*FvgG+x{2c`!<&XVq-i!XC0%=!=*dQ|Df5h*)BJ!|3#up& zfrds7n_L&{3V!FAtGW)50$=k*JtD@J&++Um)>4`XgyglIS)9JgIVak2 zBIux`=yrY8%UWoyLsDEp-FhcY?yPksj?*aie|Eh@SqY11KPi`ARY9CuRQ7QKN0hdZ zmutxgUJhJ=M}>qqhsl4LONaqP>6TQbfCJY<0O9-n+W<}s!ECK z;w_zH%VSNF1WK54kH?6?BB1V zy2Nerfhg<1N6F4L#dwKn(tmxm>VO-8$r5>A<5x`moSUtX^`r4_jLiHSjQ3P=uqz33 z++Hg-)>w@^@@`dEA5Ard*V-WwX-A{=EDxj0>`|I_uMd)Kr{XezZkDkPsqu~H(W$mW z$py*iA;!*TiD|s0u0@h~gPM1`?%L#K-!QggfoOtDeAYxHsp8Fg+}=HWvkSnWB%nk7 z*w+#ZMYbw|nV*p8{o33jTTl^X9!bjG_B4>n6&g5=;TK4_rude*b7w2sxdw!WghlTA z1kDKg`ckV;iCSN3PeHFQogDu+#j_j#^SQa(zRz37;0Fm3IX{>sc7^Tl7hkkHF5eBL1)$f6|s9 zM-m_Fmrb^1tbFMmHsR#>XNaOS0Pm+@UGj8FntGYZ&$({rh`6%3;3rw?%4w?a`mJ1C_ z4^Gy4cXoiceDmFyp0a}%S`4^_i>qCx-4&IBAocOU4H(TZ2^rlNpeDS(`FEmB~snUm}Nh|u~hFkN_mhWtk>E<35oc_~GW)2<*H{A=DnHcdo+Tx}b$4CN&N^K}Rwvf6aJvk=aO zNq^Idj&p&(GNUG?P6W&k#CL>Dkow?YXp@4p?WoN zw>DF4`JVq(OmC%D#d~VTBFEj&L7fQ^8XfL^C-Q z_L2xRai?avP$6DIR_;xNirxO_XFJ7i9iqaxl0Y*(40LmWoH;NLLtj4X6a1f<0aBx^ z8hq_D=;H?K3M!$h7D5>z?BB*wcWe3N77}H?E3i_(*oUa3n#b`aYE)4&xx2gMX<=bu zQFPW`p9ao~i0&cpo)fffv=S?NE}^cU=+KTAQV`(+`wBCwZ0(lk@%Y-Gvp0tsP9KH? zWj~sxmmOKixumVM|MogUJB2PcdqO*_-M@f-5j4<$U&I_%c_JefEFr6nV#`vDt;(NJ zYZWo_jjL&ZYk=Sb8reyKS+`a^b;a#tNCwMKcp#pbs+ ztbLKGD~z?#_pFN#EC^dx|F1?^3{PB{Qx>H}T}4XwrnJ;6El;eAATF`$y<1XN;Qn?} z#Xoh9hp*g!Amw-OC4*V@y(*!aVaEl83R0x~X;(>IuH_*_?0w}zL=v;VK#7+w-R=z} zEPYDzFG3*<-@rz#yim=RY~I5itif@j5}(&4anuRo6}QY?%PXk)@rgPRQfT{Al+x{< z!g}stK45qwC>l5#TVjZaHF5;Q+%4(EGjo65j3r9J%5Jog7UOoq%=LZV$*Oj z{@UsBx)StR!-TotI#eK2BR`1D?5X=kRO04sY^P@$t;nSA?#f;kGYZki$Y>x|e8>xa zpGw$eu`4I__c%~$sjT@*xkdB^K6^EjhmBK*W%cwN>0UU_Dk|a*{$EtUW_y5_lJ{Oc zVD=WQ`-jhBP+*tXL-gRXC1m{SMU=rx;nAs-p{GEoATBN`BvSg};q0pJFun7V8oBKc zE^FM%I{TI)OO(wPA$R7ZNl=SEk$c~W3og>S0xtpCRI#)gNjEeyGsv=n7&oXcsx({A zWjgHELl7}N_*{YKN%9OLMnIVp`%Xnao4@kMNm)b&947}?Dioi{2t-)Us0E9)jWp-k zj1)_!#+pR)k2fEU)c#y&7%%U}-R@bw@Wa(h$G36|mBhjh{5(FjsC#w>LYld^T)g?> zZ&xcuz*o`GPLgiy?B9y)lwBp9zi>9Myf*9ABlHQhc<_ymmL6|vg`bMNKXGEZ-)j4q zuMIj8!2@>iI-!ZTtHyk|kdk6Ji?7`27*ZCHG8Hf#g@m17_!wvC%MPxD>B*qSO3Mis z5fWwz&uI~}7HItV{d?jJ4p!S%4iu&DPi?dwcJtl0HQAOnFDjQ`GF^=;C7jRLEWXbx zH3KeKnDrupc!zilYsuxLS;geblw>T$-*g5nvbD6mGnoqf(p**2H}K%=G1ri!C%+1+ S{ug+C{%P7LoqzlGzyCjNLQSOr literal 0 HcmV?d00001 diff --git a/crates/stable-diffusion-burn/python/autoencoder.py b/crates/stable-diffusion-burn/python/autoencoder.py new file mode 100644 index 0000000..079d5c6 --- /dev/null +++ b/crates/stable-diffusion-burn/python/autoencoder.py @@ -0,0 +1,92 @@ +import pathlib +import save +from save import * + +from tinygrad.nn import Conv2d + +def save_resnet_block(resnet_block, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_group_norm(resnet_block.norm1, pathlib.Path(path, 'norm1')) + save_conv2d(resnet_block.conv1, pathlib.Path(path, 'conv1')) + save_group_norm(resnet_block.norm2, pathlib.Path(path, 'norm2')) + save_conv2d(resnet_block.conv2, pathlib.Path(path, 'conv2')) + + if isinstance(resnet_block.nin_shortcut, Conv2d): + save_conv2d(resnet_block.nin_shortcut, pathlib.Path(path, 'nin_shortcut')) + +def save_attn_block(attn_block, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_group_norm(attn_block.norm, pathlib.Path(path, 'norm')) + save_conv2d(attn_block.q, pathlib.Path(path, 'q')) + save_conv2d(attn_block.k, pathlib.Path(path, 'k')) + save_conv2d(attn_block.v, pathlib.Path(path, 'v')) + save_conv2d(attn_block.proj_out, pathlib.Path(path, 'proj_out')) + +def save_mid(mid, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_resnet_block(mid.block_1, pathlib.Path(path, 'block_1')) + save_attn_block(mid.attn_1, pathlib.Path(path, 'attn')) + save_resnet_block(mid.block_2, pathlib.Path(path, 'block_2')) + +def save_decoder_block(decoder_block, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + if 'block' in decoder_block: + save_resnet_block(decoder_block["block"][0], pathlib.Path(path, 'res1')) + save_resnet_block(decoder_block["block"][1], pathlib.Path(path, 'res2')) + save_resnet_block(decoder_block["block"][2], pathlib.Path(path, 'res3')) + + if 'upsample' in decoder_block: + save_conv2d(decoder_block['upsample']['conv'], pathlib.Path(path, 'upsampler')) + + +def save_decoder(decoder, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_conv2d(decoder.conv_in, pathlib.Path(path, 'conv_in')) + save_mid(decoder.mid, pathlib.Path(path, 'mid')) + + for i, block in enumerate(decoder.up[::-1]): + print(i) + if isinstance(block['block'][0].nin_shortcut, Conv2d): + print(block['block'][0].nin_shortcut.weight.shape) + save_decoder_block(block, pathlib.Path(path, f'blocks/{i}')) + + save_scalar(len(decoder.up), "n_block", path) + save_group_norm(decoder.norm_out, pathlib.Path(path, 'norm_out')) + save_conv2d(decoder.conv_out, pathlib.Path(path, 'conv_out')) + +def save_encoder_block(encoder_block, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + if 'block' in encoder_block: + save_resnet_block(encoder_block["block"][0], pathlib.Path(path, 'res1')) + save_resnet_block(encoder_block["block"][1], pathlib.Path(path, 'res2')) + + if 'downsample' in encoder_block: + save_padded_conv2d(encoder_block['downsample']['conv'], pathlib.Path(path, 'downsampler')) + +def save_encoder(encoder, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_conv2d(encoder.conv_in, pathlib.Path(path, 'conv_in')) + save_mid(encoder.mid, pathlib.Path(path, 'mid')) + + for i, block in enumerate(encoder.down): + save_encoder_block(block, pathlib.Path(path, f'blocks/{i}')) + + save_scalar(len(encoder.down), "n_block", path) + save_group_norm(encoder.norm_out, pathlib.Path(path, 'norm_out')) + save_conv2d(encoder.conv_out, pathlib.Path(path, 'conv_out')) + + +def save_autoencoder(autoencoder, path): + pathlib.Path(path).mkdir(parents=True, exist_ok=True) + + save_encoder(autoencoder.encoder, pathlib.Path(path, 'encoder')) + save_decoder(autoencoder.decoder, pathlib.Path(path, 'decoder')) + save_conv2d(autoencoder.quant_conv, pathlib.Path(path, 'quant_conv')) + save_conv2d(autoencoder.post_quant_conv, pathlib.Path(path, 'post_quant_conv')) \ No newline at end of file diff --git a/crates/stable-diffusion-burn/python/bpe_simple_vocab_16e6.txt.gz b/crates/stable-diffusion-burn/python/bpe_simple_vocab_16e6.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..7b5088a527f720063f044eb928eee315f63b2fc0 GIT binary patch literal 1356917 zcmV(nK=QvIiwFQl6I)*Z19ZK~vMkwgB)E^SaG(|_PzuTJUep5J0`N~DK81(h@F{(W zxN)TxRpd|fvY8l2bh9uNNe~1;Qsm*`zuHufsU3f;?gfyU@7){Weg+%V)YQIRE$xrC zeq4t3M~}HKs~`QZ|GE9oU+wSve|WU(*3Z-Ti~r@T|LxKj(`7Gim(u>Z7Onkry|nhf z{Z_R9$DcocaOtO_C?efA9V8_G0Eg*J8Fm z+xYK;eN$i5_?`6oQ_=8WTL0%e=^%gKm6r4`|-ox)!xtl zyS;qJALFq1Z|v_Y`?JA*_sK_{?a#`WKW}=D(f)F>zZm>OZ9y*c5i7M`I{VFM(PPQd z8@>1zn|`&**-T$V;DjxnW z?d1PrKW0ncVr{XY>GiM0idQ}CS!VkQ|NWbGc8z^V-+x_gqjfNRQC57H9mUCiw(>V= z>=U&#Pup_5MfwSQPW!%f=1H)h+x++N^Vmnd#eDIVTjM6+g!oVUD(tN_pjLOqz?A7SNq@B_P>Wd`j5Z*(|;>I|L*c;e>mFzY>V5t82B;!h<@sH zoa~it>+E1$zOw!DuUWCdjbF6`yI!VMe8YJwu2|m7)D}-a1Ec?Qu{X8fwI#NfJ1*1g zKk?$b>s#$e9=;5_FYsjFU-KB1IKz0Y-ahY&S6;bnc1IR}@!8k3$3|`bANVyb=W4$; z*yY;&C3~!Q`pai)k5cg}`a0so-pb6ApJ?`D*kKYuZ zzp+O@Y%|h=4Kt0?c|A~MR`D&J8@oTV9?>kpFz9TT_L$|Q`!m>w9G5+QQ0K0~<6>j_%bTsyuI=%?!=uRb z;HH&0Etg6+^cg1wSNi4mvR|{u||`YjttMNZ749c5Qcw z)0O_qc5hiOlT|Z>ukfP%ro{htqDFAKhUQCglW0lbWJ-umcD&Y>{shgP|-tfJszdMY_oPG znVNrq7n=C?mqCuzE?{~2<1dR(zl9%$wYu5L9k{WbOzPwow1-=jiNYzd7n3cT53N;M zeDE4$WBgzpa-}$nb{&UU)gC4{{YRUR&k#GS&|#2;*kFa%i!QnCC$IhH$KOAzH>-X1 z&%5@ku7{551_KR4VORGm8<9TZJ8mPlZ~^Ji!VlR0)nTG#&*r(2HEVG_UcX2K7HyC=FWfmz-hzb#v1K90d^+JL3e9Fd{x6lua`$(;EI)cX*>E2 zEzSTNZccY&{4@JDPWk0_xGh@|7}~PI|EJe^*?&f~aN(u1Grh?u9yomi?9$;@?tv#N z!%ht_wz$5GrTmQ_FqrUORjU&LOM7{m9Zowkl=_SETI{~&ZEX9JF0~s6Tj$>2=#Srg z{-*f!Grur)5M<+Er4;nF+vzg?J&(m`Z}7xzxw{!kF9!2>mG#~c?A{P=PJV4H+}QCY z7P-LS%((Qn(}%tChCjOcG5=2Ci&Njy{;XUuk6%14R8Tc4(pJd2_)V~@(fO^1fwir) z&v~}3KeKtXZ~0OVM{UE@H&b8#neVm7?z z6{{b{*Y5gmp^Ys5gezztmi_ZXuCtzT-68;=aL4ZcLZyc_b_=w0W_C86FuQzu+%m_V zHJz;;@Qx8lsZe0!so%m4epP(iUui(piJNC1Z?^d!M|tRx2aNbtC6XPknY}z82@xUe zmqO&BJsrN}OvX7!S0(%SJ0^Je%y)OAaBMiu`nqkO*ZkD*0=88iwPIJ97`H9=Jj&)b~(xj&pLn2w0*zABH3>!BiZ5pTy>g64w1D*hrJ+7?gH%SATuVbAR025iw?d%ZtB`n0{O9fGhE z>Hqq`PBvj9WK!<>6I;Ozt}kc&i{Ebr5YIS^D~i;$)l;^G+#8qZ`N9j<7H3(+szivC;i=_fQI7sX_DdE^gu2lCfVzTBef2PD=aZob-k_6i;F)C>Ii&DPMZ6yp5a_qN7w zZ1#8Q30j2@c61|Yp7A6-)`9-oqU}n5S=hp*Ksnv1YXB4W(`Jz$Fus>Y(NsB{{uZk+ zFw|Ctt#?^)q`y-r0!T)!^i6l;jKxW21WwLQnSHgNi`5dZ+eLqWtVK3p_Jxgw_O=8) zqTTYrpskT<0Xz^tZ8oLxL*FSO#AZw>3N9lv1?S7Izb zvN6w5b=}$uSLEW1l^sCVB{4FMTch-f$2 z*rinBiqA=Z*6W=gy}5{4cF`QoB;R9~=H3R1ppdX!)8Aih7xVG7XYI3b><}tcv}!}~ zlt0>wN^yQEIA>%ehc`5D)f_df$5tcyZD7k_fc)fb1jv994es&Iq1Eq^g7|&h`%|L0 zGjY1a0lv;i4(@^4*zrhb^UJue3I@wy$)8HM@JA^pfdJ1v%DJo;DE}PJl);T_}HuJVjERQ9R+8=R&AYCiRt|3 zd0W#Ln9jJiz9z}QweZ`Br5fK?Y zcAcP+^iC}LloN_`4braO-N&_WhnzKD)35jqECX`H6}0Ob2*MZ0(JzqimT-G3ciOj* zFa44W@FnSBG3|an;tbk@Pi?Ht4kO0U$nL2-Ae|&jjCHUFT(=?NKC#+?P21t&dhnEN zOTukfz?s|-Z7Oo6i?s#fVs?&R5w6#5No0(Siz0flyKO(;_1w+f{oq`@*|K+L@_c_B z(Lia_lKxuTal>_4j-ncH!F#`~LxA|o%wPa4Mq~VwHp$L(N>}@^pSQ4-TZWbPSQq9V zVf5F0Y%RDkf9uu?S%Zf_woaZ-mkFI)k)slmL~U@oglAguU-t83<>!P>kWdwiZ_j?m znH&*l%QCwwEgK7B%j_mgRY|u%P)j@Jb{z(lX}99*gT<>YJIH*i{9FtP;Y0}2`Qh6m zd+$f>>c~tEUJUkaCZ>}?gJL~+nSYF5PDYMclOnDX(avP=IgYv^i$%7(p8N^}aZ4%B z+S})irQC!&Ic!7Ph5BaU&0@!(wd~1_xJ3j}G%{Nd$HJGzR>jiWAu{F{&UOP(!49j1 zsTQSfi*0T%D-rSW771Ax|I%V@mS_B~-Vid)tN*1EQ-*j*rq?5#gq*qQ`Zm5zcWuTM zH6a{+w_0UBlN;Qw`+?Cm%!2mF#Wp4z5rv>M0!wC5b2ya>$+U{caM$L04uqZUCR(GxGqB>#HwtSwq5$C{R`<+G9H$MQ|DNM5yINk{!Q_S3{2c=_dsOZil;fJ6YS8!RRmDBSclDfw2y1M;JmgbDlTs zQIH3KVDHxmJ>6A_P_TSOE_1azJrm2ed)?YymJ?Ep$jY&KiCIPalDwJHQZZX})q)hy zlz!j-_1eY`;5- zmkwWWCmOYYLH>s0!*|3fx?#}6Q=g;zP4aw!HVbIIbnUOhz^V7H_}J_1%%N%My^ z{kpp|^!R@7B-A37w>JW={7NAZ9GI5%aksr*J?D2Rn?ApP8us%11 zEp^{v*D)m*J8+di@KDWr$$7#D?|>fcdRvgT!Zwr$66R621|=%9{u5zP9LE9NpAJ8{ z^%0rv3SNM>^lg}lab5|Y^mb|o;hqSpnn(uh3DO0`{X^A4S@;4Zv(=e|Wkk-}XMA}> z$qdi1qE2ZK#2^a(z|TO9f}ke~ak%Yjcy@>FHP>sFw~pmGU=1xVGy_{}5v~@>?2#kg zA_6HFi~8&)$VCS#Nn_$rHY)3mroIDOYJ-_lw+P=5AVf6Y(4CgHa$y?2BkMWGVaHaC zc=&!$qyS1mD|FP(AY5CE0~}uV1$Sac2~@&-dx4$%amguD)3R!jKKb4&g!uZfJ7SLH zR^p`ZC2?XHDWm`%#xj`Ov<^V6@!;-yV8T2rIkbnRir7uq{Sm(?=+AH3tHr~I{Q@%`cnVM^0&Vvf9 zT^RuyJmAW-%2js-{K6F)z5b*K5nY zwST0z=~BnD+0x>EDZ*ZZ4WSzgq#hn>Bo{1V$Xhrk|N69GprK4rv>e@&En-16{RT!@ zE?UKpq7s4h&`;$K#^oOM>?_d6^mWaV>`h>x^awO{4IO!LTn98 zCM)fY&}@b)UeEYV`f;-_w!!-7RUukGng31vB68^V>_E*;Zu=T32U$C~Lgp3f;@&DJ znIDSgnBkj~c(GO$pel;vwdcYEc4Jfm_-Gxk!dv^2pa`x?-HR?@j&f&*LSC&*4;XhB zaVspXO1VRS3bqnZSSP-qGt&+rH()X{1QR>Y8Kd?V7LYW+el}DwZ(;wP+K}tO6$l&-kU`CL>6UNqN3TK;o-E9?`f_c$z3w~zg)W;V z(#xLh+;(FwF)?E6s2m^^Awt=%oh*Ttg3H+ulOEFwmpe*dzb;NtY^;l z=34|e9%fG}iBuU$s(e9P#hnd+B9ck{Y~C#cS`Z+0ww4{f6bHK#f$tidWMAo^{20k% zroaVYupL3!RfURB#gO<8@Ra}2G2UZBjJa-#N=T=0ueQ#Dy*kj5&E2GTdm#Q!Clyx6 zxw690QSlq$-;@oF2%XKD-GZVEb_y1KYy(!}ad!{#!{Ucw6&Z5$K`v>D>w{8kdl7{c zngN^@FsA@iR&XJAye|>c$P4Bn>JiR3Nxt9T0U09e`V)S_eP>aT%bdP`dQ%`?fj!U; zx#+jw0PxF7^wU+uP@OWJxwKpnL+*06V~#Z8Zfa0Y=K(B*R)2Vfj}>5DE}lT&B1LEU zy(8A}2U3Jq1aRRd%81OKf+@{gAyfp){7}5p;_LmA9@^@KDSFE6w#=o#Wuh4&4ETTz z)XF|(=h9ILb|uMuX{SF3xiC5-9i1G`2_NDGg!0h`lU+p@t!G1T3~CC>m9vA5mQn3| z)D5glI|q-xO;1w8vxvu{8}fF@K_DUMQb83!7dbkC{o#c9@g_-=QHMplqC_?2xp0~G z)q;$T?~|S%SZ44U2q1j^Nuhs%O#q>6l?z)5neBEw5t&>vaQKFIJ1kJ&qO$R7voj~N zvpMi%=-jZtbwT{B_NNTRfA|tdM99pXNUGg0nC5UabFk)X{?Hq#0YIr|9MNzvI1Jr! zs(LU1_iq`?PW5lQo;WXG3*}0IQ21b90%DNDR9iXuNZyAiB*oIeOSf(e*CU|8-p!-< zXRLqXO4sm4ulTZ|!KJ>T`e-5ayzYq+r)B6O)&=Ht^1E6-k9m=vJ_D{N5fvF&s(c(O z6BDiUlQPYjx5pA zP2}0{7XK-H14z|?Q&J^ueBewsNf)f0XEEY`rw;Ig~iN0`35+)7HY43aQ@f09Giac z>mwz(Z#e^9(_P1F9^jp)#NTc$81L)sP6;q;H%ELD5UgaMG=uSDip*zB{L z-uD#IGiQHzWh|#smEOxNq35T9Rs3geifck@IkwWtzl-fEz8Lm%ZhDd9PBlm`n3eHQpo43sQdxfQldR*^aCaS z>B%EKb42sd1CRNPU$1R219XZygbVB5pQ`ElYMja%C0UV<5VlS8)pLHUg7WajVA>ps@XDsU_b3h2-{4^dfEp}k&^L2PUpnV@*3EY9KC=2 zp*JOR7)&|YzI_$Hc-vXx^*@JyThT|03=cF`e`IvcrLtLKWXk&v!Pg3aCc{XI68fB1 zngbdaRQ>!!w{hsu?{?XP=Y4E}KU<)Cs#jQkIrG+Rv7U%IV>!j&8iLIH25&I077X-L z)DqLv*b9MhA6&LDj3kgG#LT>`ZGuPns6r!$XO98ehW#+Jox+nkW*Ba>WP&>7&dq5@ zD}vsW-Paz)$li~W97bdWJO!DgJ!IqyEePm`kN#{&nx6C2APPIwG_IzOTg2Ks8?vyk z;b%G^;$+HRszK26HRE3>_z$vj&s>aWZlMe|!vc{oFXfS2k^Mh)*2i^rH zmIo6gPd`tj`d%CBO(;A0+;`<;YK5|I>kh`rmOPYd8L7##in4jH0+yZ4LG&^Z8u*fk zt6+u)+2Ce9wU~fl+T_4fr)S8m8D(w-`=a2x zq6lrTsT*DI2LN0)d^#vSD1s+&C+-jlR(*Z(q8*0%lsEsoHtOxlS>$P})QT!;qrH}zo zd#g1P9uS}7>4FXS<~O89CAM*t;ShMhocOM@`c3iaoAmAqszJhH1GcA>(Vn=rC*I_O zd(+y9-sW^7WRZY0EL=O%AI+a`TW(QXS+Wc30V0y+(G33PSrA+uC+r!mXt>_h(2D=> zk1VvKil%{R*=e$k!o9Y|$c}M5(T&|LE>Q$ZX!Rd#x339g@w^r7hi2BjB);W}Yz??a za7}UK=Z!-TxZDGVKAk<3O4}N=a*veSpb0br38&0A-gH~rh^&In-C-wa=Ml-Q*y+*3 z)h-)n)>{10?1BsGrSA%V0241hPhKMSr+Ly72}uegR1H2(4lC7iQziYP;$*qSw#=+5 zl^$BNbZ-})Ki#jvPE>b=y__$t?fj36Pd{UwB3WG9ZRDSoqI0CSm(C1Kr5pm_g&j|K zAoHc{~pv$zw9YV_Dt=AF<>>;<9eiOB#ZK^h4LLu!XlZYA3;=penOdMwV4UPo>6s z@&hi@k=cvu927Rv$37Ct*)JUUQVZH^-6jg6JtEh`okonl=(?zwfy0n0&bIeag|juT zRNpE>?s;du1t?V2*=WP8xag2)zxC#lw8YxU@d=*IlpF?*FC~uAzN-qMn)R-%GLxV7Po%!vj=M* zP;3=mvU?h7KeA`~RE=i7nS#;`W#gr&+BVZEbFp!cD#uMzJ zm>o=r+8uemnnU2XOQ)Ow26;vH(cE!9m<&0z4&JOB)Z9=%48(sdFxpR#MyT z?K(8hqWHckpr{e!uBoNj-)ubvr9ld0+$=N#m9~b-nTH@^@SWSv8qfuPQ2Ze6M4K67 zn+K3xzEI`KDLHExAX6hqMy6&frF;UE)xA?KIJQs7@I(KNAT3EsU1W}!V}NMmIOd#z z1-2LpO$D|@0g#<^6%f4Vkq-jvHLz*D4_C?E`%Urbr|G~XLOo03xfAuvl1%aW@6x8W zfvmOfD3RIOSLy`gr2r1N3yLk4WqpmqrauSrQbA)&^_+yfZP~G~wC0;_?rt+5UyIvZ z`A%%dDGs(fx=~c)#4U|pOtk0Nn$#IC?cYUD}Z6 zLNjy&Ck1vuI!FIdeEz5Og!kN)ht}nu*yB?6Q$BgaoFVdKB4{$%#hp8`LCOsVP&tn| z!Sb35$#;QrZII`>S@ol3&U zzItaflL(Ri`8a`dqJ|$7$P@U}v6nS28mRLq*&@8JiRNXddMYReHHjR@jZ!50i4Szh zAA>Od6JYrDB~=*MP#Q;s)dECI5vxU|n~nYcNbG{PcOJc|V@3ujnnicy)-L9mp*A8X zRwG6O2avvY4sYzr7fT%JdWh(?3F6O=hkxQtiz8 zDYTy0+m_a=7c#Y;vJ6C*`Q{h5R*WW|W6-kJISGgau2vqYOzjVA=6(jR{GCw~ba7_r zK!AW5jiUt0?w9iDj@-+{#{9lmf(+Dqc1zgsr90EXCxl|#>dG_JU3-$8+0c6R5A@5r z$085QQ{z#rTW368fg2x|_`QL@60XB`Awqnxk2;6S%iz|4o@D{28*?Zo?w0orEb`sa zuHVc@oEb}vp|!JZN3_f`2t%{uypDMThA@!m=<(ER%7p;it`x^Zzn};C55lJ#g*(6n z+}bRkK;;|~ZqTn3kxPfF=O%t&gr@_ch3M&_Gl0*^wBC8mP0T3nFaf**0dtdu;sFch z&EW(dzwaogZV7Qj_GSr~vhgw$@`!zlsdvP7quch~k2?T6S$Ik@kYDc0WqZ6p=jfHa z(-e#)E`?x?^++JFzi3iYLpY2xz!E)`OoGeq4Ly^)A4|{w^k4Dl(c#vwxfENWv~^{T zVMt0Z8RW&&Do923J&EhtSpW*xsh}ai4&d(PNeb2igqoBv`N>ytvRZv^4HU3S5^Ca* zB@G@5O!-+X#a+0;ovZK(sO5)>yk)9;Eq-Y9Nwld%eca7*Vve{4Y0aHp%jn&NrB z_CX?^dM}q<))S(To3w6h-vP74js-xoOP5^nRf2 zBt?L=PqfiJcDYnh*xuChcyv@8xuv4MrOu7?5W`7*%ME{-rGDlAK}c{|c4hBUtvb$I z5$HSVZ62!s&TnHhaoKL!2|)$0y*y^xv1m!@bAfcN*}`12G$Vj=0}|x*Sn{5qZDkkA z-=``Ic~HyLl~%PB`?^xR$Wr|$yW(q|!-M>iw1`8XH{h7C?Kz0HH9Fzs9$e&B1tQu7 zUs7-$RR}o2|E)xO{4(#VYdq$Ni<(uC)uTs;L29CqdqF+Pagb}VlYhdUO%~jXB$S46 zGFepq7P{M#EX}KV(DXuY<#JN=g2vJm#d|Y{PD}uo3q81>f3F11vdBQX*ERy)vO^rU z40F^X!tZtmLiWo2H1(? z`Vu@;W}93c^0c>%c1=3%e_s4K{|R<@Bm@xi(!yGt->Da+=^2Y=N#?AE%sN5ciezFZ zP+A|h8jqebXy*Lg(U&Xh??O?d1K*}JRc3{-%7Xzr)^=>Vw0>r*{w$&J48nq81tD2g)~SQRB$3gCS{-0n(LXpf6@jzXnY8iEdyB^)Ftw&Viw zy;UO?*g6^7-;lu`dinp|%~2gSI&V1mr8*COnYfm_Zh|5v%bWvPe4KjziVU03( z*n5akEHEMUG2}jOil*AO#Np0G!|inG!G{w2+&m|GI&z0lxyJ3Cc)|p3ok1&@iHO!A_lzAT*nM3o(}}IwUrg-T@!v4(>3Jxs=fqZ|N;i9LTcE!Z0@K1NC)o zg_{q$-YXS@OMTnuAF^g${Z_f^9j8h$m!)VMJ4z#r?N(yJw+pM7pa{&lpB$xFD)+n5 zn#ZNxPlW|EtRQ!8_e79Aj20#&Y zQI}zew}K}uS~cB)L;+qVWEXE(RQ5^wl^dC6aZW|j`O(q~$qnF(MIb*NXL8OQ>WX}N z=}bg;zs-_wXKgI;d&`4ZScF};amw6^t{2f2pyMracoeVIPt=w}q}1)ZQ1oju(G zDN|jYK_HHDz?#c_r_Nmqn}Ox)l%WY@NF2u~BK{{w@P1hQ@Q408)TEs$uDjcbLvuys z1JE3@%!E~P-!nwffXpm)otNyhz8KZB$=HB05vdl6Le3+C;tSf7V%gB$Is|M{$urgx zn;#`{-j7QJ?v#?*H1vl0I+Muw#~K1!^wwrDc1#qRai#ZYEV1BhonQGBc*+B1M3in< zDrj;-N9H>VwpwyKbD2gysH)h9E_Qt~)@w*Zc$?gqpq%ux&o<8zpbWH6r%0oQ+@3I8 zzuY5wCbFuVH55w@ilTI~zqAgh0E*LRe&Su1W`5Envwl#bR==GDWFP9fOchbHniIl2 zshI6(wJvj)Uetr2_J?{+YaPeG(p2SB7{#Mm;px7C*cj*;DT4m>mq zoD9hl{f2*nIT`vQNNr2Chz&C(XZ^dnQxKXC8VJdG=9FD+Q8vM3+(i_V4DkbfqtN!8 z+4izi04>p=vdV*OyXBPj9SH>%`ANw$dNEhwOPd?(^+03{L6Atlct(F!eExsZZyn*L z<+ajZ)FJ6a0+k~A6n+Nmg#gIE(?UhKel@I9TA}h|3Zcffi~0nOrofBBvi_7)rgJ*O zs(lN+ril)=pM;MVj;@2!pk(0NGGlt6*CH=gv_0RZ=ufJcBio^PsyIo)YJ17?cg_EX z?l$R&k$@drbj8;Cm#>c~3$7u!k8WEWk7pLHKxF7GGj8JSZ~Y7?!2g5a9m+*gL$gPt z>?n7jirZ9P5^timlUnA!g2WHFglg#$VrFY?*|GfD0JFS*qbqiugiT(k1IOk^& zW!SREndk_qNHHfedt82t_f-PHfF(O zaJ}jiZAdaLUT|h`Mu&-LYReR(2c2tf1A_U+xC+ksijT&jc9Htwu8Hn;pTomex$te?_L59TCKn&0x8kRgkK2UYJ&OK@-*4NQ*A)XG4j_7a-bEs`mnuty$fV&$H-{rcvPsZTi(mpBX;8}|m&`e7afd|x z4`jy!SRx**PpQfF)>kLYr5~u34XChMPv`&p#rM7|dzMtJ5PpPsW~eTBVCf*p9Cd7z zqgfgVHsS7}@jYwtZgU;1IYuzPWgxBg)E0D1iZXZoDBL$u3*PINm--B)Le+O8$0<@e zC~yy#=)F$zdB61OBEg=wA8EhndA_!`d4Vi)?vvYmm6$NZdBixZK=}@+#f`-p)FS(v zD4GH3ve0oRDG8oAKlbl|Yn*sHNC-Ke(oQaEFD;_K(9nQHb#<0R`Q*|k%IgRH=X+=8VH(aA#>{}pkT zozPoI*kwjG8`^11(n*XKZ(t1`_qt|*U`1pnP5%Y|7bN*O%?LY(HqUE<;{vB|2y;|Z z1THnNLMxoz77sgV(c1KF*Q-Xp&?uAI|rvJ*%PDv?X3}>sNpKwcE`a=ZpfRhKM-&c!~{JcWOJX+D$WP0RfN6QWl z#v<}eusYB$u$LTtO7u`(-pcwOSrNEAX-~=|D!!lFAAXPN4fnXR%MA69GU_c6H6~&~ zd*T*RFf;CLBF!W4qrB&Hw zTa5#a84bwj2DfpInDvf9;O-dM_PM#yP32KX`IAnz*>1hs2e`-}fzcmoy_Qrt@0Y}5 z4jMzUD#C{|422Vrnr$|{c-d09+6g|bR0^obV9l&)k5}8GE5*j*lHi$q zishH3|B!0q7%~#i0B)#+)~T+Fes=JcoQKwCQDXRj3Pi_RoAt(%RP%a*$0WUJ%_9anaz>UvJqxxyV%h+ z%Vur?q6}!$tjnxQ1rp$g3W`&Ex%Ow%j5q{6u`e+3=g=4Sda@U;EdBh@US{QnTR&!& zd=qkV35A9g-=q1&*<)bG`u#~-)beTtVg~zUXOg1JG-(NqBWo#sRTO{Ca|%({gJFdz z8AZp@)X{$;W{FZpG-tT=RrpA4k*ukuP^CYs!0!I_90dD}hRiuSeYzOSh&~VLIjpUV z`zPtb7(a@@l1O3ld6MG4)1zdz4IE1700Go0!9RWa1^ZTzGfIIYD+i^ePC^K49XUIo zzMgV;{0AELF584tUqz(~Q29R!&Ca0$^tv2#%1v#w4(1Wr-nc<4gHLV zCd1=2>;&9Dv^1U6dy8Ecz!D#eeXF*M_-bUoKWU{C^L^V`5 zFrpt|$1L{yj;l~02&(+TATW7wjvScND|KR3m;qjLYoq>>G)PY6ybj4|mr2Kdt64)v zN8SlrU?!TPH@R@si(WbtvrLmpY}+2;jeO(Fwt$*;;1IbjoBE&+jIU${x1$C+9O_%j zaJDd@6j*ynUj=cjzwb-8un8d$uu#3N$BVJ#>1!3PYY#ChITX-NydVg8B<2s+b%x9C z2zyR>z~2*s$gr+*%d{m&5(si(tKvH3Uqnx|g`i*iJ;=S!B^&3UI-thKV8hIz>X)X) zr6#aJ<-^2hq)?s1H&)3a@FX+ODiR;x(Ekxm4D_bBh`3q#HSM?Wfl^dtI2 zw1IPcSg zeoR71rg@j!*@05u+Dnw?pe6!`k5CbmFI8vEg=AcLpvqBHlw&$689#ZPStqO0X@GK+ zy!Io<_6ccEmmk*D4}Sn*2)YkGP|_r-i6#zibC0s z=`sNANH&#P<3k`%d(29ruT%PmIVMbV!_K8R;w3n>d=mPKiE}NP2zyrThf*{FCq+fB zgOap$PnaBNWI6*ItplFx5yX37tpw?^rac7YhaED&;O9nHH&9WXf;`7PF>(Y_o`QOm-lQh@y`JW-Az_ z>ETYpM@Dw+CpqINte`O@6=H)T(B`RF_2pQaT=2PO@MX(lUo{U@uzODPYfkKeg@Seh zi0-7EKsoHhOel37Hh+&bZHV&(@2Y6db*8u48f3Q1THnz8naSxXHqjb63e^L&!t8~# zuO$i2o08X3Lj;7RTJCNEvnqGK4*kSZpxa1L^1wpU&X9aLE-AO|L32WjV00Vr=i)Y| zyJ!VvjOsp0zANNAyPhmKx2@&lqJ+PU=N+Z<_RO8SDHTXBQDY7zhNz^5Mz?~%qeEvb z9$-WU&jImg=F#uOJo@7|fBLV*r@uZIYRk;-f`t5lxh}-W*W62%7;Rq2N@w$S5*+MA ztY!P!O@1lcl#_wM&0R3*o;V7F{@|6a0o%LsDOF)|6jX-bdz!D7h|@v6&r?4Kdtyl+ zMK@_vqLEI5YqVwdvmZKP`<{lVlNrEXKGmTdf%4#D4P!RJtoRDqOZm!_R8_p^vdvjj z-%}P3dyg$d*Mag~a+eU9qO1`r5pnAPyFpriwgOq+C@~WekiQ+&WgvSU0)Uevt!9*e z2W}8P8PmCYDl+@^5jFLz;y>uK(ILbWf`-xwq0Gh1!U;P;_oxbfbrKMU7%Lo|ZG@gqst)oU)*w$$Dw@q)kwA>>ZQ&ZoDy>WksAB1`EUJ)k%wQwvXU z2T^~TH43XAzEDd{`GaP`v#3NvuMdbBdd)B}98l}`*`~XqUmFg zC0GFw;fw6&@#PsMV#}vz2d|>mtZa}K!}8XU3yq1Y_U%o1mh=%}JEcRL98w7Feb05z zB*?s6t?qmOCo?9G#%C|fZG6suD zW5>9qXzX6Hde1T^SD>;763jVW=@?AVLqjCZOPDMs!k`|AxFdy-R_J!>Fnz?Z?q}{N zxnbH8Zgf)Er3_053T1eVq}N-UMx!PNknN&)O>c1HgSi{Ok+9abA{A7fyg4sc4Hi4y zYlOuF>C~Ejjm{wMCsf$F&_oW+Z;1G@!P8S=3`18PD2`pjxr_>-MKXA(!tuG95B-Wlpk_ znsa8>0y3CVG0qdaL<=m{@f6b{I9sS!Oa}0>>qw44H{v&+e@_+oF5Z2_{35TRQgtz` z)siFE)Ty5eL{hB7q(%!Pe3vzS?OIkA-6)yAu0eiB{xj>NtJL3>_NOHeRn@sE3)}v% z_@V7*6^Cn{a)B`S2;(+OhTj>3Z#vL$i&G*aZ$TFb-8t#{n-9Z20jhqe)ox*`znItG(O~jppDa>lc3{Brvd`m4uKuLLtemUnHN2Vt zgrDpU`ge7p^kYYuIS?eM3dd1HVH{2gT0-7DHZkW}p4gA|!Syv%tJAeACLjQ6JX^UHLIg2v-6LX{@5ja@r zmr;{qpXbc)p{Uh1_2)VbOY& zvdu$w5*Br4^YDlv#cgm#KrEc%tfwhHjD@4Mr3g6!kNfE3uEl84C3ggM zdh!Rg&xRhQzD1hpwjLuy6V8dNbS6Rv9Fdy4VDSzm>g|!pdZJB`ZY$0gqJghG8eorC zlz?%G`t4kuMhdq@2Y*r@_Xs*lk(57#@b*?>oHK=#Zo*Gd+XDefk@3hmp{X-QZksy% zeF$q+bI#?qqMHdzI434|AR=T=dj(G59`l)7@aZrPEryenc@)UDm-~qycRkP4X$g}Y zBbCr96p?k7(y)*yQti>!(dN1-CmU*;c6A&=GnTGOS`4E7CT$f#sADcZKm(7xYIslq zmce=(@@%j<1Qi#(X;c-S7H|fIrF2F^b?lzPNP8jkqX_(1ESd0(CIBI1H8bKAG^^hL zSf$=UbsZ2TRjD}Ix={>|Cf$ABPfTw>*D9iA+R$kR(-vMGO(h5Q0OMDl;Kd<}%7O`j z%sqP8i#%D(kt8T0@2nPHbhphf_j&A0MZ6i@X6P=gPP>!wc;|rBCMJ5YK21pj^|cqZ zM$uO06|Pa1WHn&lX}> z3%=IJ~FU1?lU-rZ;TmF%(xHFOz*>whAO z-fZ_m>ch;%9Woq4t3uP~Z_ceKSG9IwV@QcEPera@EexE6#6(^>A z+X)#QKYRE5w);yezlge#wHRm7RWwk@}dF)D@8$pq7wGirJ0@v0-Y6GJ#Cf zk$uYko~qo^^pA<;0?3QE>SkCGRCGa>!yqy9e@%o(URP&gu~H{3=zCLYRqzZ}U6UL; z-^Xy&4oSv4z>SpLK9FaO0hvM%t;4lBB&C#8ax-U)W2SnT%_j6w_A@NjOEV^1>A8xS zKIl81y|T$weE{sDV1D4o8|7Sp)??~kO%DiMzT+gb)nXk?h`JXW{jtBFPW(HxIys0h zbyP&3remeE>I~EXPmrn`$mWsiy|YBfquF~q&EiCy>lggX=Cq%G`klJ8e;Xd*FA=P# z$tiswY=GNu5t z8#32tm8|TUr|wEMS`}90nx?8E72Z03KiH0f%XD3M^)smqh;-j-NbOG^75Yrpo1+5A309>6tQT;>qc96l;tG z&0AeVBBK7@_loa*j|%56De`!u%SnoY66_M6fP&eERx8+! zQbz4?=P~97y`xrN1!na`0)TtB8DY4uq9mx(#*ll7h9(LK*EbEdfTBm9n&_TZe ztm@~=jP4=&a#ZrbXtX&5A)jlAVF0-HxiXr@`UNM~o}-c~kv-wNfOK}wDY)qKpb!*f z=mGa|5er2!)3h^@O1?hw)1XXJiX1hi?!9Ee{!)$P2#8O*5K;I8!r&465GC9ivFQix zNKjK+nd%QIoI(*goWPyBLIhnfe&W()FyDRl%Pbrl``DK=>BV9mO8sINMdfD}LAc*r zalOrw#ggoyu#a(*N}z@`yChU1x1&l#?Q3+}x9?b5QB&>KYIwqX%$&`PQB|AE^-M_z z+LTn_o-(=62-JM^R+u>yO)4nTjeW-di=yER?tubbI(@AqtyT5@%1}xtuCm z6P20d4|cEx#0cfoNFDn@=*@PP1;GU&Q6#6iTgqMGCmSfgZpTUgwhr;dK2Y$`2MS1x zf$4ToeeY2Yh%VyUqn++~QW}AI54r0hdwL}5uKVv7|1lZfxww)IGYY3dV^I=6C5I2W z;nVCun>mIwLuu<0S<&NEG?eanD2vGHvm+KK^{UXVJT*Hb%J%?|gjR};dX2YyL|K`9 zuZYx+sthSxjac-FJ_tXeXqBEwEfJ~7osDfmBckNQp0f-1J4Lrf``IIpT3BPa;}D`$ zF(ox7utuTmLY(#OlUiXXa#Ky-<+$)NL`yFFofvu<$1{`(mUP%cyO={M{iUdNVC`Exxs-t3+Ws_N3j|F& ziEY$>-rA>q)_T_F`57*k)O!BU;?v*Qok|C9cCeUbL@{Sz2Q{l|J5GiFEC3sd?PM;bry&W)~* z0KkZq{l?iVd$9QJ$1sMOYX4|CMm#)}_2n&2!5qApYf@9w?SbD6Y%pcNW13~-$KgC- zGMK`9yPTStizR7LyWAAI#AiRxaP@3-%#yC<3jRc{2C(B&T@`G~j%x$~$@I_+QORL$ zsi&S4anw~3a@Kuf^ zTW;Pi6dh`5q;n8l%-AD$oz5eNce2OJW6GJ@rrmKduwD|iZ4T&;Ey}^usr=He{TvvK zr4r=@7uXIhs+D4IhtzX`pXKCLkdHB&78rmy^_tDaS4pvqA=snu~%v zPD@CGX8ny=$Mpkn~yxv#}KBXWPr&zWNpSO6sen(qb4) zln@q#9a~J*N+0is60kv|O)E8EMHhNFgKzT!urib+38~gm<&7y2y*5uF-ln^-uap;+ zcsqnzh|k;7b4^|?<3t3EDE$WTTb((CSjjuVEP6e z?iio*IyA*oldn?+MK=9O`vI_22p2k7v&;+(5XHJ9vQtxA#1{SZ7VIo^0Yf9hPhSHL zaadXCpzTo5!9W;45dYCR=Tiv;c|V|Fu8UjrQ4dKr>J6T*=_!RJTsrzJNt%fMvOKqO z;_M{|m8sBJE+~;o!KzX8pU3*tP!I7l!pau0V8Guw+7Emy%_ask8~S3uFFyTsx=*d8 zRa2TylfD|pUG?ksiiE*$B?faqPU<9W0zVKnv6vktJkGxk0=R0TMc%5KUc4#AFWgup zGKoL;;U51QHAW6K^h51YM+$d7|CYU;N=nQ6M9SqPB5}5pRDcLl?BCgg+n?uHcV>yVYmzZaZka{x%X`5F`O=2FAUq=@Zm{e1e z+4l@zVIH*&@X)UMILFdgwa^k@M5P%~81w*lt1UG8;4n*t8qPl``}=SLpJeeclp-a3%cuY<~Flg`dc z?N;=AVXt1GaR#hkXZ!s~0@G*$IBgOnCWS6GZ2g!L?YCpyntVja z1l(ksXJ6Xw)Z9n`7g^w063D5O$ZcWKE+RJpJXa&WU>oXlSPbGwJ)=D8{9jHj#R&Wt zNHhB)lRn4LQM`GK*}|t?Esz`)f`K=X`Nr)Dnu#e~SE^gTP*jf5Q~0AUP6os(Liy{Z zmvTZ_G8B}LZrRk6hQM6%;9l=Vj-(h$An+<54i7zVBBj9*R+kDW)GSw2M))`%ySZFZ z9@{2cc_e<(h2`fbh$&+fH(VYeTZ-rjq>K-`MhppfuGef~ZeY(Hs4~ifBz%M$n zGG!Z)^QVa{!5>PCL1vztR&k1`6&`mBYQbL)-0_7aGC&HBb9T`gOw=VD&yaYl-~8#X ziqHSUR{rokTx-N-Q>+UWgKz^wJO`>*?!4cVNGmL|o4b4s5`W?s7btH(IRs98zwfDE zhvXQ&itX~deY9}=BhOLHzj;S*95It!&=k@+Hdz9|)buiRe1Kdus7tHGp}wk0T2^8OIV{WXgrq4Er!^*ZFHR5Z*?^tuns zB|F&dRSf)b3OS$9=Uw7sl++xkPZ8LO`Fuu5 znk1?Op2Ij9^Id$HGR;8u!gioA0V6!l#ZzcTXUp zI0oPMNBQ$5kbR0((Yp8h+`Xj12Sjc-#*L~8&uz`sD7(=$N zJ!{fBH%B`8Ox?6HW7~%(zOo9Lt&%v+Ok9qIZBAUgCX}!BK|<*TLGZ zy7+@H^zhy=|BfcjXH1`$0^^xFM(v36ADXnDDCoM~k;Pm}h?B7+f7>IA<3@Kahg$Kg2co18dA)st1>XR`x2&H6Y%F6gE`e)}V9%{7AdMaQ=E94DS8bZ&7Qd zNQ^66wa6YT-VRK%7a$Ym7Boy;?ilM^2898$I9nE2HZ9f563CU(rL!Bxt{lpDDSV>B zr*<=ROH`>7E}w=bKKriSt1*>5X@6_s5fz5?R#~O72NZs{i=0w%aB~{0}3nfh?LzLlnG@m`UFYHmCLZ2M?Q5IQSVD0hg0^VoJ4AblQ z3Z({UN1?O6X&NUHpq5Pqj@xs%h}W%2C1C6c76oQMb}hV4zle6%!2Fy*3|3EHij|g$wPDfh~K=wa>Klk zNjCM-$D1d7ofbLFGUjiTjQ`n5pE zwgQmtOU?RQ8Vg@lt`Aq1MOlKp*#T89{hKKf2Y(lABprrCl!|~;cn^?^{=}kgd|BS2 zE$W8S=_ra2X!0lp?U3p3i2 zRR|<+W<3wp697BTVg7G;M^A2y09{cwwVWG^on5zVvbSDqbnMWCffoA=MBK_sb zF`*;YPU8!HIqSVLRa>IS+2MK*S?&tF!j0M~$vZ4XdZCmofIy!{V7VQl3LYgSynbHn zqPS7w{Eqksjd!k~W%pJA-rA1Yg4gSLs0Zjsk5-5M3yd~L;v8~}tjs?-da)spm7{$R z58gn_awoy@ABnlBewi8`)8d$1T!Z|zzV)iBiMU^XXs19!Lt@%8KT~+gnghbcwtRVI z(~3VhD^x)Ui#=^GY;abfGp7gzyw$d&XT#XZ#apt&>_fg~AF`4!3QV)@j^g$%x4p0m zEBqP9wS5G&j1>4T_kP(oW+5tjw;H0ImcCL4kuo_Q;;vo;RDr{7=n<<>E=Zs8fNIhb zWB@P}e+@Z9(lnA&a9ZLb)_z=-Hc;|qqm2q zES|5`GZ2nwE&JoIiqHRsAF3++Kg#JLsjwzh7XObp>|>r-=!Rd}uZA&`84&=O*#4X| zm33k7{yQwrY|c-v^N8Ej1ekJofYp2Ca`zTVU%%vvVaK*N)W%_6rL9HfR6WLofPcvV zF2Zp4jFI=0eO3vriAtv{CuDD+vj(w&ep`>Ij{p-16-dg@NL@ZO6EX^%y2P05qOUwPupx@f7X1A!& z80wTwTgUXQk|PXRc=p4tCktdEw&r<@PUw_@3!yt@G|`;BQh1l5sF(~oQTUtfW<$bG zXDOARq=0MP@F<@QnGy}DO3wx65M6y`DFX#A!H_YJJR$n4)bE-;C+lwA%QS#4c=~gm zja21H?q~%{oh>ALs9=ly#n+_O#QfQcO#w^;;-fWYl>6(01_d8}QCBnl)^~0Kq&;vOmk0BwrdG97O#doXK06`#xmTM#~d!lCqfQOoj#T z75I#~5});(Qyk2Y0}QgPlJ7k`4D-SUKMijB z?pqk{q!8|g8*#k9Jf+=pX*r5K$S`-EuXR2_MU&GKN7~OshJwKS8V$~Dhz75CJIP!% zEyfxanIMEFr*g`Os|EoV^>BH#)aBNkw37*0%k%WLPe0>9*^*L2)&rQ-S=D)~(kzIa z#lt)<2rL;5`>o1oVq{tXkMR+OFt~Wzh%OH}Cg)|jWF6dxAWF)FJM)OIeByZfT{bK1 z7$5hrolvY&z*=?p_cR8Q|Tf-%%(83VhlTBodN8je@KhXNLnUxT<3{;&k$gYr360Vgx#N+IzFO8|LavqN3k@ z)JqWeH#Camu#W6x9hIaEW*g&o2tu;r@SHSvEscBTro|N;`gj0@Ob>$Zx$grcMA!i* zcX3206sp6<`(0*uVeUJROH6vtevW6A+8g$_knhuSbR^p{+Dn$bAF?R)4G_NB_rNCE zV&HH=@l0k#4^P6}4& z3mX+mOu14iHquDQu3%dni^G8_9Pe@ow|0e zE%bo(j(6+VocUp@f&KmP#f+i^-rLpb>#RtfwQC9Ixm{aMauI9Ugb?Ol1ddAXHSbe| zow<}kDf%3*Doh)Ds-exE$)Vc#o68lgnGQLT3S~#-r4dH;srY~&|7-LwF@U1R>0NcR znSCELB(+3T>a%!b+r1=-TR=915|=8im9VZjd|To) z_?jeP0l{xVKx>S_0U(; zoDh0nRqxWgprBXoWP|zoa@0fGqY&#ragKWJk;Kdj?ks;RYS0MhSCK};q*q|y`vOmN^X8C@8pbZl~kI9^(x^58t082p;%Xo5=|-W6Rok1d4laY7bhW*^T?t`4N|EB zkh*4K#w+5M0{kBbo`!PkB=nCcH(2n<>B`(r0vxslLE#!x>zbr|=XrcM%Z@CQfE9L?Xa~3x$vp5UIKGG-ltC>5pzp1GkzbA2( zq^2nd!xEuEG`Twi7I+0Eq`@v$-K9EK!NL%$QzfExXkBP$n>tHWw3W_cd7?{0-v;|M19fK1$2Pbpq&>tOH$*@l-PB)zMsS`7G zux0*KgJu6_j3F54@Kl;1m~KFKSwnGRu%rdGSGO~Ta(`Th0W}Jo>X74zVF94Fi7S{u zmI%HO-*Df8lyNL;^v&U>P7Yhi*9Q43W$|N8HDD z>e8(-;KHZ1=edTt_fS>bg<`47YX_E|f%6|qFsOpXz&DeH07&f|))CS-s+s4r06`mx z^IOVL#j?CU$Q8;hK6xjB@qk%_BB5!&3w5=ZzQ{ard(vM`p_+KDf+W$`7Wj`jYHn3@_us|q6-1I1mh>BaWHhElt&dE4#R{(r1s4x{%BUrUlyN!FWYePFS3sd z^-QIqd?*8aQ3EidkP=CI7nwdAUQmX2s}mpO7y=A*a{N%F^6+q(+< za0+0$`SaiA>=4t49gpNnSV;GSSOZ|qxaoGiSft4v0h~vY-?Ig!=Or5&vs1?~91YA6 z5Ue@rVwqtY%}k-OVvmV|X;R&ySf0bSbfe7e1kk#HaK)r;UN6%*o0#Ns&H!=K?opi% z5hpJe%s~@XsX4t5^rz;3=TKX* z3?jAP52z6NAxJZ;e0TGy@%cnz#Z|>2wZSy(Ga8wxo_s^>sS!TC+9|g>;*A^U)hcMA zq@FppHxwIRM=(S7+|AmQ!iaMdGCDVv^rt$?Ke;xUm-KB@ZtiFxBv#9!ORFKqHm+=n z%GKFAb*^37;>)o0^DqB@8UUCjWzt1{a|`($!eI!rNy`mGO=j;ADMKK7FFuGx$ckZz zjW**tL;uF1L-!v+8n6u!%LNl1F4P4;W!_xwbmcYR z=Eo(~3i~7*!W7X5U%`2C_-|J0Y(fWdOm(f(05zo7F*CHpO@hC4z=<=l5}t~?{{s8Y z1xqlusX7f*voPmcW{IRo|Dd+0=tAv9@8-S}r}=u-rcW_citmz#5IxJB*FJEZGOUy$IG7<$cbLV=HHpr21OJc?ltsfHZ=A57^$P=m7aI!?dE38Q0sZQd4={&1{7(}0V93<8#<3VVbDxuNok5*6%h3e3g%9MuPZxZx1cOmSou|-dZ z`e5dE5^wz(F4A_}Bgx{JJPbVdTvPKuz3fyjV4;SZdInQ(;;4fG5VfZz&dZJ-obMJs z83mYx&UddlYTm8jKqlvU1-#_`{@Y%-XU{5^oPBUgD z{r;k3^72jKNXuCPvy=*rMN@kc@Q5?%0qJ@c@<-S$Qi|DENUl`OdR(#C&n%L=!y79d z`7PqJZy<=<;XO}UfugmDT?he^ftjF2R!afRiA0>y4XLX$D*zC)?58R$h;ne-?LOLZ zwBW#dP=k1Q>dr@}ps&OmB6D-IW$60sr{(|SH^ryFbJb0#v_`{bRmq!FREIC?5|*Zy zF4P{jSHS}d9_(5W-jXMKLv46Eq2rXL5@SY_P{~zK9%K&HU$p-t3GGOlL5mFqH$P&Q zWldUN$*%IoLV#LZc8}9m?0M9YC?Mj+eN%k?7tUqK={kBDAz+7ugFoXEB1 zG6*^TrMJL=wgc%gqJI ztwmhh`&+8An5kFhuihv+P^l`0P5WZdjTdI^5V3sf7#<2%Zd*E;jF7M_S}6*DzxWQu z9{7m^;~%`^oUu|_-j53q+B78Bp9O})vx`(L5>^B)sJtp7KT0Ug*OiQbU}1XtY75=0 z1lwb_tIusmJu&@e(zA5E0bD}=q9ev6u+K2-zeef9p%iIQD|`Ug$J1g0-A@5DO;+)v zf{l=!y65D(QzQc@Jw(!G>0tI5g%+SgNh>MHKVtR>_}kTSMJZTvN|s+^QkF;Tn356+ z+^xJJd%Kmm8CA5Kg+T=Y-R(IpG8llAW&C1aC~KA;j(2+lkEX>TLZ#}aTkrH@^lJ*% zWLFqv*~I<)S~E;Y5-gF@Vnvpir&xy*1=-i5jeUyIf9q?()Rg%MP`m78ase`|xo8X) z#?nh>8Xle%W!tYlAZ>Fbx7%9Y6ZlDm05!}WtkvRR&w{efHcubsm3Ok6-=9W8ZRHL!#9qJ6!=%= z1^PL>_MGcjk9L?dk$MYg%n2m2f#|OVH}urdfDU>;nnZbCq+IuN#xX)$(Hj_r=OiFL zas~z{=YSGDO#MgFgUETqP`-jOl+}YJ&5`ySQz?9ot^1BG&wm|R=do~WCxEIHdgVx@ zJ(M)ahv+ZJ)suUaEVfzD{WQAi=#Lh*I7wI$u@b(af)ep7rHEd;sBk;N!lspryjR#Lp=b6tOs*+DlqI?6^Boqk9Ey`^Y zE5!r?@)c7Xket$qceA2+V~Gw%6V`bMGTk9aOs-vA?ie4Zcxv}ooN>dv;jzpE!LaNj zcv&2f(sw7@`qft&&|OOizO|Swz~OmnN{|$6gYz%FP6GtcH|>Mq+=gWpBeY$cVd=!k zU&=iG6G7&Q<%}*MC{|JnTzWFL(pALyr_rd6xd1@hmLwk;CD|u{qCYSG!rjz5oY^x} zPEWV@8`5Gh- zr(L6~%V|OAC`pZCQTu80z-|P+{gJDEn?lZ_ycMV(g3C^Gh6gqP=Q<$4(9B;Q!keM@WgW>}o zik1IvJO=~t^FSkH6O4V`SlEl^W7p4W30%dqV28nJgaYJjTQ!~hqV+iUq_^wi%xS$L zBE20MALmr56>|ehj@Re|BnCWjV7i~{{vwSIZJe~#eaPfXG|=zhZ4a%?BrzKEz?O3r z0Mxa`4GF}r*an@!iZx4TFJ`v$aFP?L8PnfULr+1_x83>&{ZmtgevxW5KTl;Q2nIKY zup3NTQ_=+De0tRP;yoeBA0KKuPapMn`Y1*Z;*X*AsR=@`g&JQ{!qV)y=RBQNoHd?T zF!tJVHWUfL#gdumWsYwD_*WQ^2L%&52TaH2GzBB5L2PI`5|v!Eu96xVD(_)XgqK>< z6f25upumhQJB`8=qp+fo?HlqGPW4a})|^Cd*+U~;8_s3qIlBWMIv9)|5f>_2?Snh# z!hV;X?VO12g@3H4?p@c*@#I}n_begITjJlfij{78bO@#%<+)=P;Gr9Q@3YD_nuK** zESXLZ2WHe>f^%1%u&V#r?4&pGU)r%jN?iU353L^lAwqm6?8 zLuQ0UMnq;tWlT4Q%9HgVqktdiN9v8dASMC?Nz4;TvvUvrtM^*I*LE-TL^BB&7w1%E zM!4^B4OCaL2iRILkeR~9(R+_UPoZCLLG_F8e)?YZ@%JZ=S5J3L2|aLi9QVhCS?b;^ zBJ4Pkjxm}J!!gi20-UxSrD!i=2NziS+-i`g=x7EF{n5Xs&o^*ODGY(OnhQq;7&vWR z+V-wpvN&^p!Eaj{iq|`J%sf-ibKwkeYY7g4A3@%wh4!^1#GX%o?miM}n*<~;T?D05 z7WBNP&h12&J_5Gts`CmL($g1z`|$_W7yk=%y6g?sR*%z`Oef^yN}*s3E^GLv$7apG zisOwQ9kpN1xc~wFDRcC#Jy1WHC;U{h&yfTchL&I-S=D2rO*nOa)g{yvRh^5CE!Mgj zMV7f8Sp?xP-2E7Nz8bVmfSW*=qDp9L_CjIkkxew8A;#4LU^pd@B#6ms_a#8Wd5x65 zzWPtVF?#oIjA`aFKvdK!VEf=hL3~@gYh7@r`)QhAixwgWmM&5k&qDecpC8id#2|Pj|$F`t=BA3VqlzCHhV7$ugA16;eSrjB6z|Rh0Jg0Yq&VAJsOXyLE{m!kQ;nU_Q0&TKh=z3-dR&+Ex2%uTkc!F zkEwQ;$JJ|w#Y3bKqAu8lWrz3#TT$g*lk=9!9%}H))pxR*SyWL99S5iuvo{yV3Jg=j zADU|kj!-BOT2SCF34;ImAHN-37j+6?(Cw4_dInhV`uV1r%b?19%ScKL;qAv2w-owJ zMCZdb;SxaOx6=~>NpQmRGB$u~Tj|+(E_`FHT)kKFi_B!7rk{#5cv>OWO{q zNW@v32r#LR1XB}OfNV@bHGl3$ELYf(SN1vNwwF3P`Dq$|JBao_bMzZR<%t#z@8BVp zJm;iA+J*yR(IzyZD2nhf;oJ`an`tw0^hVv>{z@_OIB+O|qy%(S0-Ib1T!~A6JwoaN zSO{-$VM*p;Xd=1ssX&v_o0lYZZmnoYE&^}|A<9{%+m}9mIv3PaGJ&|7MaR@9 zBVYm9tg4V5rJ!H19$K<4=s8rWu;n3vN^&6GUa9=j0!T}bd;lOY5c8fTvVpJQK&pga zi+}%Q`|Y2H&SM@iV>5l}FWMKqhLPx&&3Is=Ralt3sFp$t>NnukufBcx-cM~dl(2uP zv$UqC?4w)MCKfFGiq|D)66^s-dvy=Ml{C|Y08F18IZk~W`_NXqxRGS%Id?U1`2vX| zUP?`sL7mU^Whvf?qBTU1rWmeH)wzkkn4UEPBkp?)4mBr~G8Wa|Q2-;8jte#pS_>z6 zrfh&tV@|Kr{OC#?)ILw=@_29I?w+!*L^~n|bdLL%+AFOj`T&a;rw>>0Hd}6dptn=A zFptH$Bgy;1(LBrw<`2Z3==X(Ol9jpvh`T5#y^}c`=Pi_n8tr{aa>5ky(?8RC<WmT-RAWBhjm)*OR%& zb+6E**h#Z%(-#m}U{#+uAZLV^&BesM@hMxkuD zrdt5SH1-3;hqCj9nCyQ!=pP`w<^}Cl??d2U85vlN=hlvdRzFqT?MyJmQ?z z&uo@u>T`lB2<3RfNpqhgfdH6Or(#=K?wGKV(<~c$ia`m%R$#vcuMA_jEEnCk8d5>4 zd_6UG>!CD|KKi4&$HL)R&+d8p@(6My)ZHjopKVb+`Y3_^I2P<) zy$!4v;Z9(UUI4 zqPg&uhZba;f0@sCbgjnfQj>r%^jX>v0EcrvzTyCN=TOd!dkLMtjnce(O;z!QBDeZ& zqp)Gxj$FV-ZLpyW$AQHIw(ElXFxNVTH-83P2V^5vj5Oo=AIV@8i1|I!jkaT@f_E%v zA=w8fopskh50nY1Pa~;bF?C{d7wVQTK!deOfgYt|W ziyE$_0JhV(z_N0%^q0fyb5PABdI&6eqx)h%{WVHPYYctb-K{LS6EZgj3WecTy@u-W z2;F$_fcvTNikb8iOE(pFZz8chSnFL3jRD8vem}iZjFi(VaMT?0kAG8Mrys_CWCZf( zrpK5*V$uWL_F|vYZ!26p{bN}1v+L`Uqa1Oujkfwbyx)1>VIrCy&s45ZCk(8@g}&IIhuwOU&Cf3q@t?C;@-EN8^BX zu=X$>?XveL4$Ck*F5(DXsy2RAPZPe=m$BMU2jonRy+AtrC|NB@z1d<3p5(Q#tHqyk zqW^9oB}7K)_ol8gYL6)fBQ>CgpB<{}qO7z_Glh8fsN3_`)n9vnN=G}rC5)m{VvGZb z2}}ApY<3|(RTnS$s+Botr3I9V$eGx@p8N+MPq9zxK8bMfd`iiZdaS3P9nyhi{(m)lcHKJNXf zZt_MMFw3b@=jQV#nPdeCtqsEx8XzVx;KMP6c7vSuX;EXpIt(M9Q$ zCa60#lPOm#76PUv3%#{5wGn|zwE8-p6e{=d_1Oz8tj6>7Lk9#Mc`wN>K)ao}%V}3v zNm(FyI0)Z3gwR^OHaV^&3r;J8u0zeG{!#3YCS}^MB9ad>mF8K zB!-=`V*8@H?j`RE@ESR!YgO5SP<#5)m#Qz>|0xPcv1CGUz7BH%DzKAypq6kk)E7XK zDbJWJ!!kVc4WpuPbO36y*b2io%+9jc8psDxT>_$YI5;3Z?$8wwu3(BZyjE5Nam$Kr zw(qNckWL1$4|SvofPkLvp!TZ*6JzHmaFF?SgT%TL>oO&#K8W&Y;PB8|2m&-w@zYS>lXGW!pg0#o zGe!TgC(T&&CueF+f+D)_C(?&gK&aQC!3|1Z0RvOC+ycRE^`_Ke8 z^FGZPfQL8yrYCZ2VT?xQT=j6x9O_M1T$G+8AecH!&YMZw!{->TRaO@}aCdnk}L>}D|l%Q96cax_mCVOZoL%zwMPeVZTrPmHRwzpU%9&2n{25h9m z_Nw83i}r zfPmRvwq54tD4Jc&$HkE=hQ``nQ|gnxj~h__A+O^Q5G3H!VAWT5mxh#pZMe-00y8-e%Gla!RK_N$+jye`)Lg; zsUr|d81L>;WmHpAWP4S)YGFG{D@>BT9fYh0RcTz-C-MP?YZ>*4aWTZS3a$>2q%r9S znXyxZWxDWeHDm?%%TdJw*-Ew7E+1I%s07qZaS(P&J`_sMRVorcb>9 zKpJ1JzUHT7P8dLb9~f+PYqs3IMP+MI7H4xtuEI#&qXg13asijtfMlmk`=2c&EBi4JfB02P`ojX+dfc)`$KGG1v=79Uy=lF?G4*?4RVPV9ztD7K`Z} znH-5S4rhB5Qn9v%o}u7iA5IDF#rj&~AOXx-+To=QbYt@Q55%Yy1YnzXmKN=bxpLT6 z5|#EE#ZZ)$fHyH##CN_<1?|ysSz4kxFk`an#z{-aUX!DALw$3AN|7K&%ls(SsSW^8 zW#ShQmch=@Z5GaOsu+KgA-9r{geXwmjr|yYx!g+HvPMv2=q{2452y*C#Dh~#J$b%hbc0x8M z-ek-!;qKEwcugY!41Ml8Q3y8rcxTPkS#)_v4Hb&GYu8&uIckR>P>06YH3LPUZro?d zyBcb*y}z@jo5!|socKe3{JuUUrxYwlSo})$^+s6LqNR0v}(0_(5NN})PK%; zx6RIH;(^}|Dl=o_d14EOpnl?tCPJJ&t@S+_jPmSPCCQcF?w9m+2A^6YVrNWy_RlZ^$(X zC6pCu(N4A*ngodhdmSf$L^_bid#0B$;pNuvw6z=dNwWxbXQ+DHwFb$(?~;OgfTR!u#_VmQ?26$U9nq&_2rf?yRp9&7U1=Bt}d7K-@ax4 zSK z&wiS9sRAAa+u}Odhvy+nzRvIDlbacdMNFa^F4>^~Tq~bhQ62mAAG&|LqT&_`8vQu- z*HYB2CM3?WBX;z9?mW{^yb3)vN~j3LPS2jvy17dmAE(N8tov1EU9BK&dd#itdV21y z{l~c4PWoFA>aZReZO-$dFkJ4_OXl4LzqMZVV>AMH5_Nh~+met7W#GGk{yUT;9B0^F3Vg#@LC`&_bSyn$GR+d@)^Qgn4%8N{ zviq=q{BEMe$rkkCsloEI`>P_trlb8O=SIyjnqBKKS6Sx5;0@KKx@W#!G}gz&I4vJw zR8erHv`RK$VGvzaw|Xx?{7WnCB43xa%jz3;SOdi4w#gX{P-+GfB71@^ z0$aJ5-)+@&#Ox*mI}0sC$`D84cdqh8E9MO=hNn(M@~Hb#h3b8g>j+v%FL*8{OO3?W z*DR}P@9sa&@YHOSE8~Jh(nD(n_4&o0{rP|qq=w5sW_-_ZgjOhi%N4#RK5xar8Dw`~XFwA$ zLA!beOuNw^p*^yzC)aeB6Oqd`9s`(5u{47ixgro}jVgGo^eq7?zf*;WQm8+lTaN{Zkz!#kwx6*C*X76g4{LYcYsOc%hIHhz@4m z1e!^x4zC~|n7?9k)}?uaGs?I#^5U~EAH-_Kug67L+iJv$rPZ5EYzwm$3CmO9>dUR<^OYb{B_0YsRjXhbrSiS|>WGRr>FaBQX z{yF>efr@zSfk2e$y>OMeHV|Or>0zf4#FZwdBbIJrqI08mLdHkmtE|Vo#&V^vUJixR z2Fs!Ml`AIrW`J8BRt!#uDO{HQJ5kXqHW*b@j*+849!JDm(H zngXcY!u|7B49A*V*xC{aA-uY3#d~O#q0aGaJgf3(-3gBp4!d>0aO{inFA>K!M_b zYOf`t(7i~vaZvV#K-xUG>uBybuh`fp!j=jK2#8`W3m~{AHlN31-JiIbLzkrk%h6|! z31=9WA0aJE+%|nZ{jN8J=4BmquK0=!`sp9^pVgoK3;Pl{m;we6beXbHt4jqGruI}A z26fd{7hRq$5!~7;Xe@!30ypla@BfSH)A!O*2eFbH3W4w{MaA7>jlf~R<$B)LF0BdG zu~jxn3tGZFv$P-khARzKzjNpnwBE-7fprMd1P&Av$bigT58U*x^D_qQKoZMEYQ|gV zDqst^D?&chxBbvPN7Mnm6OUT#6zuLRCDu(U8GV!j{LY^c98*Oz9C;{5tB^^!d3?mN z>xwUXEtHEf1PYY|gi`xy)BNf@WlcM?zB`3Qz@8b;2u9k=7R#42>m9#+e^-6{8BQ}S zbK2qw?RYz$>(UfR|06*?6luR>ty0_`)q$$ZtZeb*QC%ZD=M{H9@)vpJsF$EYUk~e< zUTO$?TbQ}k7?hl|?9HwMaG`&>ua1F-v-qaT_FhEZ@3Ly7$$^*&i z2zjJPd>v`dpB++u5O`0*d#_yr<0;y8p|IJ`$91TKcn~B9;P!t2;3Sa~LzE(8Ok_N4 z0X7~IIQQy7BHKC(NJaf(AC4lxYxH<=mWB++MejLx1yK~WV4$#}hN^)=kIt{NXjq`P zpv6~nh2+^~E_DRH7NS|<&4#C9xvtB!L^yJ6Cxpi5sn0HLZNz9C|APvr2NPBscr1Sa zXnMAwCB2gl1V1lnB%DYm3CQo*Wn-EoW|n?v>ka>s{hRuc{7*|(G4-9JlII0UUlPGR z{gZVCb=>;ckk~gyYKFeb%07**Q5X*PaCf;?D`6`TWptz8Lju>_D%2gq$KkP11&3>! z@_Y`u{8-pfLuSlHTJY%}L6~Uj?3X&g`$uDFJDlFPo-MvhFx6oa?xmMp%l)YXMAUOb zw{)7AE)Y11g)0K1B~u87WZc*iaI+~XuR-9u?UDrBc`g9UV0mRb09d+24e;C30@1G- zwmgAS`kGJL&~D!VsM-rw@J1dZ76RvV0dCK86vrVn}xWxw07+1Ji0EUn1IP! zlQdm1&Du3M|3>wdujous?UJWp)0KWua+z>sKS_le`+T=oahu1mK7cg1RfMk@V@#Z6 zXid_6u#fx$wJjJlSk&^F8PwdOO}Ld4pL5;Ey(gNu1~1~+}P)7tVUXCF%k25M_KP-u0Al*Ep0y;+o zpxcvXUZB>m9T63pIwR&O_+RI}cUAZeMfqGvjg4cUbW?@>#Be|M+;wPk z_gh%A4o9OS3B-olQ1UTAky*kPDO@HBH%m4dSNK1naDU)S%N-1~Sk;dixikS2F0y5SnUdbcy8lMysW+ZaBdL)F*XeJjxJu&9_dS(AF{b=~zg||1{Jea!EUcQ+iq?d6Do6+w?u_oc7Ze zv%*L9aP`2&OSwJhj9Bw|etSR_dI~oW{BtZh=)P8Xud!^O0yLfb0P0`(N3iE!YmMV+ zWUVpb(}<0qUk~pEK<%vSa>=M^3bsO6*k>ij=2atoG56&9wA5G2v6w%2chqrhTyJ^X zxXDkvLDopC2USmOfUWKDkPwpcm+I`}A3$IgmTq!u4u;-1L#Cy9N><#-9sp^QkME!0 zpjx|d*SpHu7X70OPTV3e)sK5tAOd*xlzP3=PvEV7Y77T_v_`Jb<{tBs&F3C5wLKpp zS;C#h0f2$K9iZ$O0(-}K&dF^`(`W5L)(FM%E}7@KNouAttiHDWhJ)2MK%5GFrnkXfOugr6vmX;Rwu&-cyho8SCq`cdR| zrPx%J|HYrlT?Njupq3ZgG4bZTRoaSf8j>1PzbBr!|-eOB;r4j+De)>U& zL78BO4rUkvUCEN*%kH+b?a#b`b|KK;_`(1o$S;$F$l`d>MziGr{CeAwfJzFn8#Dh& z_1tHP!6phiDK#rUzR^e;GsF;QBjJ-D?O&=-|B#N8z0o;9{!@&PZXs?+?O0vg3wq*^ z5NX>)8jCN3Rc1MWzGFiJ@amIb`oKMQf#89?BAC7uW2+!y1bDoYCd0OSQ^%Y@UeG;* zpINQk_!O-K=-TI8mSw=lxAls0qtvYT^H|;?u$tpgs_8o0is01N|@6J;Q0oh$ysV7 z_Kguye7ZP5{=$tXe9rypx4JfI3j>T41p!;n3=kv+ic6aiLR&CvPJDIX{XD1u74Gg# zl;%$V{{N`H_7%MzkP9RN=%}N#cO7X)yUyIsMFpz2Lzg$eXA>*Os!jO0tI!1c;jy1)WNL0OWT$RB4F9%@W9)Pi6TSC&phwH!7wEcJf9A9X66i{IR zG`kNChlfZvBlsob0p*ZUr9pLdIk?MEC>6bSQw&ZUObobvj;qRKbp^iJBFQ~3S);k z{9f7NWWSsg0Q~m8&V^gegVY~lO$Kew*J3b;n({SjXOMa|)U5+9>h4&APi5nWZl>Pd zNY@5{M&>md0jhX~jO;q)*+wec=~9sQV59Yw&GSrFU~|NAgQ6aTCx3Vqlda+ymB%xfIwd0y1^w)*A!H0Mam*BvB z)dhA3YMRC_^q4XZt-l0#Rp4c#A82v5~aW8VpZ4RHP`9cdz_nKdV0e%0Ga#HP$TYD@IcE=6f9L z*I=ThVcU@@mKN>qJ)p-I&gcBugUX)QOFLD;V_y>d+cdUP-hx7*`ZxLyLnA{?%nr?- zqgBec`0d>_)!MfDHLvTM>OsYNi=}|KM*+)$cdH<5&lREBXa4{t-83}_MYLND&2u`! z7RTeK)yGsdT+}TkcI{<`yeO3ri-LEE{C8y2XKoxdgPCJalT?q8U0gm1QON`NQU%rM z+^Y?S!yH9)5j__w(q;^7%xSQvT*ApI%Mky&)gP#W17D;uLAN$txTH^{IdHLscTi}a zqW9*_;j@;tCXB}9mgaQfgV_M?&7HPim=MD{q z@0mca)TJsljek*n`j!9eb0~ba5C&FF=tYe+5i9k7{KI7jmGc1pVW<39NgM5+aEDY0 z)rY(aTK)()8&u%7Btf43d+-2>0=Lp+z`SHgbWRN}`_eMlbr8c-50fPSEoWBE8(L>_ z9rcmBLbG$pdJhBX%gS`HvDdsDuf;Df7Euq)8A0aE)(6(TICtvE07@hQtWqt25Y?3a zWCOF4TC$=^f%i#EsuN9@$aor)+&##| zL&mQI%$WrHPtR~<;5;_8^U$5f{xQb4=dPjJSl|>e4L*e2XdvnG5O5FMKv%meEFL9s zl{63JV5cK>2TOkJrS`s;+ZZkaXJ;psulamonj6y}FZ!A_Nm_;GPR?tzhq$_{Ge zP_kR8SBXjzLlqbB9y1HiZuV6p6D(rFYW5u1!(;Uc?kE+^W5PIvR_iG)wo)bLP_`F< zaMMG-&fsTDw{yxgru~>Q4G#91ClHM?9Q#F4_4n1sA1jP%Q>jqodazvx3iLtcApfK3 zsRpPGZ_#Jho3wQ8)V+i@+NIx^r^SKLP{_}7s^s3H#3D-FC$qnvwHKavffB~x%nb+PNGm!sZ> z_ix1r6L1mxt9L(zJE%jO3KgI)o4MY?1O)5Xxk%r?sy_b0Ujy2@Gp0;f(1+sVC2s-= z+d+vjq`&){>f=B70bB#V2iqR(6cEl{{pQx;OZ0-pIqH&37@!Izb<2LA$RJ&zcQyv>ne}&e zOCIyCccxw3_5|u%tjTf9#(qgjxV)-55CG>BT+$C z>mgZ|MksFwZjtP2E}4^Ye(yPAJ;4vtb4oK$uOU$pYZE$MGSPdf9Wmf(^^3p7Z=|-T zUErr4NkiJJdSU8wD}>_^2HDSTUI(p&r(_ibbcX+^kfx!$rxXMenX-Cj6}(Y|NITd! zN<)blOy%|hoA}h=c1ZUNXF4EL1+u)ULwTA-r00CtzJ>2}m4%Nsd7ve?-lUUw^4dMD zbzh@RhO~&UC}={!MXJ5nqYyr|j^{TXm{au!v#mOLB}PiN!lkYA^lhYR(wyz?A^+DN zxdnsn&U|6XcRi5#CBKhX4)BVHm+VYwDH2CB2)$Y%qM-2kOi zPL|2rBUux8ojni17^R=Z7QO?a9NRuIy$41g zrL-R`kOc;RY_L}B6`SvN@B=c5?vq|i)EUdK!w$R?!mOlbH#k9gyyl=ab{3dkH`LZB zDw!H8lQ=I5H`@(7nkVryS(KT;iXG*G7%cq$5#mg%EM9{T+qkDXdP z$&l7x(E*8B={Z{wh@rBq3k#A&GsD&;{qxtp=IeCJTyTx!Ci67)TVT1e6E^?ri-h)) zq@F$O#OU$o;}wLqTeB)*VDoa99rUbJga199>D`Xw1@cT9Vcke7I9=amaSUkXnj_)5 z@{7_4FYdzrB{Zd85T+dOo0N&2x-qEWlF>SCdMuqF?E1 zpWNirXT(_0P8ecmz_ebiEa=kd{_pad1@ivga)<&mfrzr^-k!r)yM)pO1Sl6S-){4) zf(<-$|7RFp4-f_Sgm+vs6q<10z~}+FC6o~p2!Hy%1<0{+sCw)Wj2(Cz@VUkx;WPrA6lSK_&eZq4N zx>S!ipsx^ZB7+USRL8>V6QyW%vp(rSQmb_(k%!WuOg4Jy=e00jOZP7zXB46h1}$zS zoXtq$oZ#!cK@p%(IP^mKe70}Kv3Yn3_(fhQokr~}7(+K697vG_QaZ#TYU?^(m=P+w zx^xct1wd-Y&J2Z=uJ3bP04fr8XY~(r3rGu0;(K&P=|c|mYSw`Ec8uD9P!0l!nw}hl z5CDNd6_)TZz<>*rU1L(3q`8kNa5_KGrUVK8I>o_Uv@(E{6;-UU?Kt&X`g2;^%dXct z!1aW7JA_*hxjfE2x|S{L2~sk081Ue0zs3`IVB09*dvM_kDFOYwLRYt~r&zMTas{{o zxw82s`>NmGP08v5{e4K%0=SYs*>{A~Y`*HbB%zA%F9MU*_)O$3_y$4V7F?3Rr;glS z1q58#6oBKH0@{~4ACfa2qCpQo91wvQ07~4hW3FDkP(55b+CycOMx*pDT8tY)92GWu z86{NuA+FKEV)fyhEC8Yi50ucu95fst=lTv`!(uG8q%1Xj57Go|kHXOiXHx+eiobqr z<}iw=MOgl<%4_lLRy0II2L8@J+xc!|ak>>0(AZgGp2FBcF(Hyk)H%j+sxcrQfo-)V?z>LFCd_XSrW)CqPxe05uom*oo)NVAywy!;k0bN&&sNSvDd&{~%|paxS%6@eN4ot4 z_%JC7{TpYOmKiv5i}7K7SR|N2<`RCmk3Om~M+4=jc%6CY($D?;(;omf26eR0OsBO2 zY|6T)ca6!<>`(f6yH)f?PHzm;r@UcIfshj(A%;{bX>FJ6NE=v zKA(^OCN+^(_3Ex`#-d<{*pCN|5h@COE?9*#)fG&C9fLbS_bp_A9j%39mLAZWR81oG zoPHiZyeFku^l!3>x7Yw^$6YW(XIiu8KDo+JM%dF}i~+bBoRMNZ`S~fotkDrq9(y6G z81RIi6TuMI#)N9L-{>x|_M*|X3BkBnF+n$NLcKLNvX_%@*AAR_zQF0nj&!6WKjGXB zefNrYq>1c?@d*Z;r(m5fSqf=GdC75Q(wmb zfOrZYb-Eg1K&vEB>TCbjC9NxnG2<8KpUls}w zhw91|ydm1xIyw~9p9?x;9-01rK~tZIt+e_l@|j%&sd7-w76lQ6@>{pn=mD0+mpyl+ zLm_lnHDn1x?Elwa!4*}1}k2DAiw-Qq4`q6SL` z|D*aJyfS)oVc*cE4X>u@h(P;|vO$nY;(s8iW_0Kisw59;vx5ud%HLcv9jg>w%hKV~ zkJF6mUubd|ZoX)lozn#~tc+vYv+osjtTkx`bp%JL%T5Sl5UDJ?0h_YP4RfLt!4uOL z7y=7O#cJVnPeQFPlnBAnZw@}yB|<|j$-^BkCb26(#u5GAO^E~5cu5fAz$R=NAEuOq zQf#4t1u-6Xg;h~8GIFa6k6k;VL%-y~!tXx*P8kPg)Iur}0plN^&O}4kH(g=BkzJWhz41{nxAUq(wnS&x{(9ka2#>i;@370RqKRU`!V52jFe5mFRR)cPK)^NeCM7Usw?w76*WsN;Hiee&=F>F#*)rI4s zCLI(<+p&hEn_hY+_BTgBlEqU=4Ra}Ueo9`!0nxB>?gg276`*`MiqSBb>7gQlGv|TI zqW2K0dsN^XyloGag|)jJ;b{W1Z40;=LHy}QAAh2FnIKI!E+B!r>jDyKmFF2}RURn| z6Gy8o^xDtaqWl#@&SZw2MU9+5yS`H;*sIL7%*fv`Rm2b{DVu6YwMzH4RIiM%bap6# zjDcKooYl7{3IZ1d`USTX#RNp+Lkg5C=k&1@?P1Z3wGU-Gq$o~ABec1%_7=nLHCB}q zHYP#60oG3}c8kBy`NCzqwdRfda<5pyM1-z`W8VwcaU-THtiiSd7t)1RIZE%pFOj14 z)nn@=nLJ>y4=4?6bj4qM-1;qab*a@=%QblK0eWPHMRRKg~id zpCJf=``Uo-9JvLc3v0N@W%jT@k+hiKtyCGe#CW9b)Cpq{!7-*#5Nr#BZ1g2fJ5^Zy zD7y?#)H>A2OW%`Bt#w8R<})rdG=2}(j^<;zQSB+qcFpRKlsN21`^ z58XN28$bcSOZFFjByr}@(a^lOw=+W@18q}_Q=tp;3%`P(+Bh^CV5%iBd*j@4Rgbwb z0PO)ApQpU;v6r)#?7{3=n$6N#+3C{RBRi;%mIqyTSg3>!UP3}VpvjkqqPHxO3r)5+ zbgmTnU3(;QOW?m1!vqGcwTl6cS|GuRR9xoMm#c3oQn^_%GiVAiRMTdHj+I=(o-PRW z&Si*Y{06zun@`NgaN4t{(v+}73@}eICkWGZD~pJ)ihJ*l8hPnhcjvcBcyN*D~K(%2#6KCqBixl?% zypsOf*RY4P|4XyE6Q(eN3l7;zv(-W-Zy5ZN(Ugp|X2o+!Fxg?cTmtM~sS({0^~it6 zoNA1aH-L&h-Ml5-fA$||--s&YU2*(h{eMA*O$%Jt`K2fzOmNz4o#TV!w%2(%cl}ES z4kDrT;#9QSe%U{{sNYP*f~}zUJ*p?EeV0DwqJxssi+{F)A5=q>v$i|E%5@yCU0_k1 zaGx(?Hh7Jp*Dj&gkbV!o^VwkwZ`mQdTj~EE-BKTr9hBO_%e+Co!(+xPq*}Lyp3p(~ ztj7*RNNBCd@kax?Ms&M9S`@^5hoUFk3vel6LqnOCR?d5{Z}&CO6BG+kJXEe_jk^?V z|M;Cs|BwO&;(?_t7q34AHCg&?j*)5;3*SXTc_YQ$*G5G zVztttXs3K+}0{TLzpVpZp4(G+0{--`?fTrO58WYS%b)KKX##p9R*|pMtS_73^0Hz;V z(-F&YZzuE0fwe4AMlOX)p|R|*DqdEDoM?evW<4Ro4j&((iC$4R-6@Jz7xBN7l~&g5 zw;n5z?E{_!R0!tIZPKrUAl<9q@+VsNPDoto3jB#eJAVQOj^wAL@mCME9KbuD`K$oq z0jC@4wzO-}DLKAADx0yUaFBH_XZq1}A8-H#o49nVoub(JbYs!`qeOt?=Q3a}Qo|dM z#B6zTNxAGnACXaCPaU3^J=w0wkEoJfAfz? z`A%RtuLUtA?a@9jdS3cWsOHgKgh+TPn3+WVJQO^`Au51O>__M7QYgGOIpTxd5EJIH zg8=EOn|`g;TAwd@)(GwEQH>JC%Mu)<2}=m!uK>%25biMj6Z|Cu`}ems%m$igmb2 ztpy`%RM`kKz=We40)mJL1|Xw&&RIi)D38+H1eOx*k3=|F zfU;jx=rRisq+sd8b58Ww*6`S2+;7P=3Ln&W*YN<*49X(U9c5^ma0bwRZq{vKHs#f~$rP6cd$^gn8jXA)^l|CTkO zol!=F3n41r&Qp4rr#!iT`qsH_U*}wdreeN3eeu7lPd^Fh^3Q>6dnIB>Dwm+YL4wOV ztZ9zDK&?ChLF@MLKNFw*6I^XlQT@`*>NLIBzGT*gc2AaaFZ5D*INvT{WSNK z!l!4IT@%@gHdM|)+-=hN-LLfbfFs|ru2JM^TDAsemDUsDz9l^ppb@V4zA&FV&GkKR z0jaDAI8{Hv?4mT5t58p#>RYJnVJu|9<%7laXA=79ngsY&^Zhx96~EJ!n6nQ5jzZ=e z)tA$uo!)7FgEbEJZwRooSvi1`w0^0@Fd9VJ7koH>8Q6{g3%6w*MWI%f z_SfiaMxZk_*$<$km|YG{&Bj4{Qd+#O!7Z#a%lo|6Z$KG`()go8L<+C#^Eom%6!KGL zlYTR1b3IUi7O<5q-T_sqgx*89O=E{NP($S!Kju;gQ|61G;1+-sTflKN-u`7$NaS4j zqMW53NiXCDsV}J9A*O(4I=TY@j9L zli0uRusnM|RQg5r$>pel83EMJLW|IE$OP3*L8h;6@!fu!9vtX{C5QmH!zp$|U2FOK z$_cEZOHJYeLSI_Sz$$F@l%FV>&@e_-?yqK=1cc9aWNEUL2^5)0RY?0>`*Yl@^L!L$ z^Pt=nWOz!+F8vTXfM?<&BskC!1s_lYi0MTek1xARSpjemnEE&*z%$uELzf^@?Owb2 zJ+sDe5Aj8qAV~4RKGtVPPutI{T}~+XXuCO5aC%R#;tx27Ymi!#cAKXp%#nLSR;)7h zrP==irz#D zWZ=jeEBVGJ+NW#z<39b5mxAlUar{*<>7=EE2fFN-(-Gld4V+{Jh$3KDy$Q209XVwrn~0*YWB>xFj7zS>dk-qR~V%yp1k*1Ffc zoUDAib25|!ibkU~)ZbzCzoy@ZBV$1c)02pES4 zJVZT7*CxhL9SIZz`0|2nZ7lMlEKz@lW)&EPk3XyM9~rmgCs1;?L$Ui;XTRwINn`y? zXZ!fk=U&cJ(c*CHTd-D@1yJ&IXlA3|wcore8;gmHh!$zIE5HnEX{fl*A2iHP+VQ>( z>zjSfNuCmr_u26#+p{U{Ox9?|0Wp`e1F^wc4MLUBGM?oQW&YMm`&mA?mNL;ON^x|} ztOTM4^tLcQ@C;6qX@6luU+^^xWpPp=8~v#3l^%3B3x3!We7<*k@`$dO#a(v0TI$f( z{;CS|4Xc~_40*0h)7!Ju?o6K5LzV;&pQv5p{9$ghfvw?!!0FHnW|L?{Zhe}ydkSYC zKSi$^{oZ2%L3O$z;q?5Jpjmm*tWQV zxMTrghjrFX#6rRnsV?H#zKhSkq;X=lg;=p)fIS4pzE|v@MMaVcNt7TcZnMTxYW=jl z!}ftB`8`KMAQ~v!|6b@A;>Wm+BDO$yd}ogd7<#_bb}^sHKqVH}A1dfa==1EgW^WpT z9$BQv%N}UF`)mA0^_PE{K6#*#<{fw&T@N*n*4_Jkeu~dR^bm)Fn)rj_a*kq>!eN?u z^@a4Dy^2(5QlEeCh&kwY?qILVm8gDQl-Gi`_E`h$U2_vM6MW}aBTXB!1P*><5|-!S z#n&B&6q_jP+o^|58$MJ8$r@BS--G%0eaRi8MAu}292Sb@`DstsvCNjN?emA}MSWW50_F$Prw#C~HO_iy z&Q{miPkkN6w8=ApJ(4|BpAE+;( z`P_G^PhP4Qi!3)7a>r!Bp75yTWm(zk8k9BFee#xVLg93U{a>*FNOPNCQ3K)P(ZjCZ z<-N7YGb1?IKn7;!gI^Ggo9*a(0y!WEN=mPfa^^f=t*y$u#^8d>1@deyFM$Pnk=)UX zobleR?|&g-mQ5mQ#M(G^?O@O1lG$}tETn#GVa zC23bYBdnPdlu^5Pd&Fl=QDN9D|F=+s`0rLiA;S^)F%(`VvfaYfcW#XF8cN4HT03 zg+dVu234~|sO8jKIFw(tnQy|_WTcMN+NJh-lE>N_0>0{Jf9W}alSpsQ#PNNpyItSML{b|2^DMe@v0EP}4q>iw=a-yH(`5fRtaT(Mw>9 zo@fF_gPEe0%#tILeAr zcEXQMou_sDrT!l+QM_L1gl76i^}l(11H|XX&Zx62Rf1kCO|F zmNTJZrvy(*cZGXvODQys5I`)jp~;&g&n%>SJ75*rL22PO$GwX{vSC!-=*fn# zl6SJtEh+E$4TDmjGnkT}PwuhS=2!#Fnw0ljda9;k_osf5pc+R756jvuDKq-nDq`Nr zF8&^qwIRIb5*I`9WTWrHJzyJ|;+`QW5A?Jdsw zF7K*P!H0+m&z?sfv;-t#%cK5Kx$skO^yFcEQ&RM3zwbpc_pz~Lw-4_hcY`-SCD# zHJz6gQ!x9J`hb`O+-fJmCLdbJ(5Tej?y(NtZ=urO9E*j5%+O5KH}62Afm;zInryqe zZR#uUAS&5aYkC~Ilv=UIj3*!z83q>DQ-7~S->wQ`wWEU6E>_Da)5__JI8n**P1pXJ zzCsjn$+*1?+2=wACpc~(6%h;WUE}+oC&?VJ0FD^USfS5eh3R>9c zQ9H5mn~oNWBF|D70`sd7n_$JFW_cCkW0DSSwxrh?1~4kan|rl6G}iG75>RD8_}w3^ z4j_B!w4Oz{&I1wkfPDk1+zJX3e%4eT7Gdr+c(I_*-nF<9B6E65c7x2l&(-WxMkV8v zERnV?&1I(31H5i~+?OCYiu$eWkwe5N=e{NY9AlSpk@hjh8BFZFlR^3Qo;LY8wF>=o z{wMlJe*C_lTIqnt>~ZVl2?WWlfV4Qn>nEao{e+rpZ$}LN&iDj=swnj1$A0%)9-m5n zv5H;^KF6|YvMbC-l+KcsF0ZQOFpcYrX^$E`vjdZvocgar2+%AP={u(6qgMDA zmTT6w?nTGBW|v)*$iPEI_xm5P*~`Fb%xW3xeohvr=N_FJmk zOo%$bxQ_=%23h<{|0ocyc>nDGtBHWLT~*{$C(gSt=zc5hvwC>Vs%h zpXmcSzrY0hcA^QumhLdEw?Ye>u4g-(lbwvTgI16oT=Xq4;u_~O#|ZY>-dwxQ6&dKE zsUEq`bv~F_AaDZITTfZ81Te7?tu?*Z8h;;F+e21K*#XBpWEwzg)XnL6l0%;RB5=u4 z$V^#vF5#0tg)M(Fm}*m;$U?Z|_IF?&CsO;y?zHEJmG)+8wAO>AR+63GR2ztR(G1iv z1D3u{l$R6BO40(aMR|+B{vYXU-D)VL`~3jMC;g$n{rCf51inw4n3dczrQm;H3(&^w z^T!UqmY_f82nDd@r_~?-#^d1&DxU!plvD{}J2%X}A+kqw3sEfq5D8NF0rq)MNg<(` zi36GjT0i$Q)OVMRa(76Yn$pw^xsw_!IsmUX>eZ`_o0Dq^E<0 zBC3#naKPAS2M~P)EB2hJHX!T0&hc;a9$+nx>y*ZY5_7QAK9886TGcksN=+*;EE>zP zU7nB3c>elE^))ga?Q$iDI1fC19@ehKis@0% z{==IKB85-PVRuXfA9X=*2E@tQfb@H*H|$sLnKh7m3FK}Vp+44LFY$ULnyj7+iZhK? zEa9C4AZY#V81psU5>HFJlz=G*NJrW~u}g3cT8}Tk0?VHaGx4rYUvY3B*5d=Q0`C8&-b^f^MDgfTBWBajDd%I z$RH@qzWjrJh5h^w{`+o!I+YT@yU-I>N<$+WM&Y|Xq%m}S4$X!(%=g-Owm3g2dVb3-`sy% ziPx~2e}TkY);JQa>Hmf2RJ~vjOcG%d0kLEM8m!dYt-5JAbXE*cRL&nql!Bn&nadvS z1fUxgdXx;5oN$F^=UYRPX7mG-CF2f&=TIM?wqaez}7kL>4e{w zo&(L7>~Dme($HbnDt#ntr;=u`=G-EA(J*aF{h6=yfYO#ZQR<{T4AH3%a`MlaYA;FW zNzLoc=LjzCBY@Gs?E}hyy(XJ?K?pl60T8GpSVES3f$c@06=+%NRmNC<$Plr*%>^3X zs8wa534h@4fXfNj+5Xnv|MMDw8&fNI%K4M<)M@;?3=8}U1v z!v^y5iWDevsxSVQ!1WOErN$}3mKY!K^Nz1NAp%$`3hRIV2+IH3Vab zU7m=d3Me!dJ%8y!j`!##0JsxZ^A!*^ITvn8ft>&RCHKApd|TRLD0E%3i=u5^MAd&M z=TKO8V6S$jVE=2E;lM89)l1o@gzfASG(l_ro<~?O!Zt!X@mcRkKil)PpCv1vIzkWf z-7DFH0=<0DoXT!ERIVlF&%*jzga0a#-A63%x3+5BdX>5f!nH`FepBSmbh}p{UqDL_ zVHCji-y*)vKF)?o3=Fm;G&MyjMw45Md-x(Gj_T<)9jdCUc9dAl)P4N4zT7y<~J^oNVNNP{(cj(1b`S+;~~2mDpx?@TSeYznvgd$bEPq8h}v z^lb;iuHeAbdPL)l-_h88I^w!L0!55z;)JIK@Bx7LQ7za-9GPk)%va5WS~r2J^JKLF zE)-WoLuE-fC^mioqiq8JKxWcXxS&f<%-9B4LtHxhh)~2xSda7+FbQ#G8qH{DicWd4 z`n2P`%-Of-n4IzyQFGyZ?@DWX!7~rSYK0rnCI!jfXR{k*H~j2;m_LN9HmWvfPEw~p zEcaQ445G`jw05jr7D= zqv3d{y2aG6i-aUzd44D}ThEf#)QK^FuNGxXUTV#|fuPvqd zf?9`=i?!u@KWrt7Gjtp#yQfi{~jiZ^{RyKO(^b z$k%k4CiRXrFj}!l9g4i_7pF$BVG&rWj9Ij5_S$jGu78syS^Cp91YYNr7KPCE8IrSR zp6q~5pmVmX*xsaLzEy9&OGz#*{i0TZ?tN;Z!)n>8tyemRw?0f*`W>7fA8=r|8ykB2 zP$!E1qoCX8_sY73jG^~G_TKIAY7sNocR5@@|E+{?QLiF4M_9*gNR~;5{SRn0{AuAg z*2-iPLG#6LuRi{;{BNJWn|>lOu&n@n7!hm-VOt)ypjo0qhMVSS|LG=sMp3r2OiYEt zbbwzA;+Fq>0Iw9@ED{MnHy0+vm>A6#>dFC z3!e~>@RH*IPt_j{Y~nEdu7m0G-Rh6uPQjABp$COkcm9k26EeyGch&>Td=Pt8S8kND zK@~#z>NL9mQ$Vc0vFDjoVKmUmV_A z=N9t>ExOf=pr=CDrD|kCc6Iuocu?2_2g$@-}+Ydt@K;$LIML~ZG<tB| z@TREq4hU<-uwfw;StIa0@QW}hB6-K(B839N-v_%()f_h<)Ag>KCYJcyk%j7z7@It> z26O?ot#X0Q&a12e!k)u=t!aJ$YY}s`4@!bzkrMfiq8|gWwZL&`Ymi(!h2zHjM5;an zY`HcT+^+*M2L`Ss?GxGY{@(XKLqfoJ_4|u~z!3l3g(tZ$^n=O?o;#5up`~`F?}O^( zw|)ee?cu*JCULDyO9frvl3CNrv~F+c#T}?s{}NdBKT#urOxCZ_7p#Rf!c7B=iHL^2 ziyu2{2Q}E$TYAD->ur-dEnwPR;Nj}vwI?;jlX}^2RF1*#1u8+e)`y6;rBxUC{AEf( zKe9tUg=Nw=y-j~~!EXZ=d441<4mKW|$4yn-?rj0aQO^ka2lXRxSgdhJG1tMOtFaEA zq6dL)D(2Zk2TO33S2qs;SkOw9Z|}2tChqAvcaiTj0$LOPemRz?^VTeDHP(=gvCsJC zH)##D#sX!K?70hdEU!JXKq;tf&FRJOVV=fa1q6BAi2Y>oeHw_cw$9Z5biK<~FTt<1 ziJ9B#o?ez6U}r;0QUu9s=uaMN&Q z8D{%(f1`biLxppP@c!h3iotpC z4;GImENo_QbPJs-FO2}TyoK}!K_u2Roc_G}Qu-0MC&bHpZa)iK68m7F5pfqZ=U#gm zxDnn3>ZGa)B(?33iNTmS16UZP#`>ILl%yKXmXyyKqHE9XPwd{@jQXKq^%vD&_^sk3 zT4S3!?SUW{jLD6fiDH$x7NTq1cU`N|HMf)Dmu7!p+J(v~{b8c1YzcJ%ynPSvi9Sy8 z#$jtMaj(Uas>V{c>XEGnwyxG<2jivsmH9c^jnEp_K#KCE68K4=Lf<=v2`|fEDTdjZ zs5$2pGKmJSzL1-Dk@{|d6u3f}PmukDg7uD`l;7;tH9HX#z*IDu$-4}Igw<3OfeonHykRkO7XDz9GJr%GY+W&RUkU2o0 zbRMo=4X!Az6qRd*?7K(Zdr11C-g1*;1qQOqxv$;=(Y3($pzy4;Ts+zQ9hy@)Lwqg9Xl)zoxPI6(J5!yr5?R&hHbtY&o&;DzVRD}E$i^xZz z5NV~5uohQlC&s<@l#(v&yhiQ!#dkjb+>^>`u|oPd;Mu)u#{e%Es3)O4_2)|w(Hi^3 zdVnp;<0Lu_$j)K%&U=q@xSgsdFWr(|4xlNb* zIK-jk+4#t*uDR^O81UootMnhIz1j{HzDs01GEg*=x3+%QvT&p+O*bXAM#9FWSuoRx zlJlf1R`ajHm$KB_x@|zFbWm#5(dJz+=+_c&0my=Okai!gV1CVWf`MyfWbNA^-?|B@ zlaoQv+>Y4oV4UHnjKCJ@o4-`+=}qv&uU7xfpD2eTp_$z@9ZE-)k=K1t=T^M4DNp+Q*-K`cXMpN5zRDBUEvF8mCWFZeQ0uTfM?_*GUu_dLa{|Oed6nmmBxm!KGZuD1He>4}jWkPj-M?g9D-}fc_qd!Bvws1YVVTU`k}vHe#VF_Y52ZB~Sx7 zl(y;fe_h38I~apjJQ#81BwAWQhG;U|N7S*jCBXX z3EQszn%-{7!3Bqm0{EVI05q$yId}qcM@^VonK9>sgb}eO)jOE_-jmWDZ0bNMpNFHf zNH%+tBpoum87&u`m`iuW80a91It1|0_3%rX#Ee|iNTKz74(j^%{k7Sh{^0R*iiOgS zqWc=1?3QT8d1?Yh>oxMqJ^wYqp?8o%Lt49=u+NYx6}JglU?tgGVagesme}Tua>_wR z2_Gv6^B`0e+?QX7yC(q4_fBv6Ya7V_j#Xc4+-i-({I0ZYu?AAok-DQZjOSzT5I}xcdXAeSNxmllhPW7n57*D=R;xr ztnzf#)5Jv96%o&v#CSJY7|FM_7=yDKp4Ud$ZT0u%O&s2s=$?MlmKiWc*64X|`ge~(~6-^Qh1pYOh99ZQ6& z9B)kj{yz*&a3<57tPmu=WplCawT}td&|Uj?!jNe=>5H0den1bMT>*#u+w?pRr2MrH zX-v1SR*xuSy81+mPs;9W`eKq0^n!-)U5YXY3Rc@;Okca-vmFybFU2~s zlgX){skxwlbTpzUbr&cHpgD7$R(4SU;4A2tR0FNVV{-J=Tirv`(RFq8qu>15LVIZl zlKhxz7)62tdW%w2Pzshs;U*4@e1CJS+( zW|YT70PO?zhZTJo(-#Up{uv5i3JFM@7a)#FOw;qh4o3rxRITU<&Y%tf{oIoYa4>_o-6xFEZc1;pi)|4Nty3)+)kd_<4K{SZ*RE!`G z0%lAx&{a!}NUcU9AQj8v@IO9>m`M%zkQg2rQijpG#p2Ct(1F&ZV?R70`l)-AeD8m0mHhRgGBZpZ?l0JZEnXaeT{=CNhSc9^?X2oT?#Gk`z zW1|naW_?{&FrdzgC1d=&7|Nb4Hz!zyQ=13_#^ePdD=+Y8UyC&{*#wx|Gg81mQo|iO zWcI8pSNk16DZ{P;vVvD7lZSQ9n*)<9RP=Tgi7$+80*%M?1s=e>I6`>4_u<_&1bgHh z`f$)#e`v-yqD;cmUkek-pH`oKqWiL;2SKo(^KT8n!IoLsG;WK`Bnsu{_M@NNl&Y}+ z$3$kYd)H0j%u|z42U5@~MmRZ+K2^|IApI{f4L^5<&>aODNY3tLWphOOL#-y& z>SqNg>L3xhG%>x!4boz>eF6@oTsIt8-o>EA3VXD>{@-4Yf!eTv(D#-y0jnQ`EI|du z`iVL!j>IX^M2HT_Y_d<^@HX6~((w4~ZJ|sU^28~5KE#x%?Y6r`8Yxjcs_p1y5atOrmnRbG70jxThm84f7=%|dH_#xUdxADsw{hzFj9I_V zS*9`zs_hB@miKNR#^$<35X#CV!5?0p1PHe!p73rT*sA^qMYE$Sd@T??c%lCIdN`*K zV!|t4wVeEcplM&7QY_9NE8KV9Q|GN4X z^$j*Ze8UlvS_CF5&AAE*RptP0AElvhhqoId0xuRymJmbCNH0%n-2W40 z(*&R~aWk#{4~wl^c4mT;eENqvRQ`cfl>4qPZ$x>rq(V*mXT^bpCI)pCOk{`&jmfM@ z&*-eOW-Vpm=|ov?v1@#pZvfsFZEf@l{3btx5aHNSu`zGnY1)y0$eIhrdjzF4|ItYq zW;>!`YkZa=@cHP6_b0lXp~LGPss_?+f%q9t1^FLVFs0vUkJx7qkLv7Byw$aLbvhDO zEy@S$)?Y?fxzT&FBUFKXNY3qA6tWAx{qCQB%Nxb+!ycIbYw%AS3JJw35rX*1J85eG z8qUs1cHE_!ZM_2FPF`{p9g+n2F9Xyilm|Hw9)e{UjpKi=zN%cZM*C7sn5jbQ9-c8i zYUNsDnX7l4s0KSoh*9*sM9bGx!u6>mq>0gm=H!+=V3GJ=Jw|j^O}7$|KZ`O5SU_}m z+|zRQ<1g)MFEBdv5I6*{(cX8c0mCT`^>%sE*7WC9b7I-xgT*d;LcN1)P#IIJi8jj? zfkqafg(coD>A0gs0{gsugZ1QDjUeQnJyLf_XXm5MoEVUq$o3eXTG>q=u110DS(9$> z?EI>asdhvY6oD6QwW}1?S{EL2kp%jmy7r-0afCes)UO#cecd`_^PohLc0uA#zx7N9 zmo%%WX<*)U*l2qFGIccby=Mcp*clFa(|^W-OV1>$J%a9XO5~OccagKS)rNY^qZf56 zcCjzK$p~b})?w*D0H0pUnN$le*e~p4Yb8=}Bt-B-iM4oQp46PSVTDC4MEIo9LmSOw z9qO$Oz{7;kctv)bY2=Al8RxCF3fMBduJ@pDGYU{(-)`!XRCpY-?7bz8ai0XnQ0J?q zjFB5IXqdhSUgs*6YWgty|9t0f@}RNS84jJaYBaQ|{Y&8jNQ#n#=)qlquXHxlPC0qK z`rpM{xC(L=?qNRqBSKCChj7bVA|dyx9Wk%-GMO`~Sn+q9(e1@W^!2Z5Wav@IoxPj$ zorQXPl79=$&?CT91;ARrL4OLm8FpCNCjw8V7^U;_NNQRkv2LjiEyO$jxO;b^i=P9Q z7OY*QSS(Sm7c9wPJw2~2zV8}|htpeM{xy>Nr%j-}crDld?&EJjfyX8Q3U150g;fFw zokI&J%LYM;Fe8a?IDPWTdBMzhe5T!YmPnq*g~FGOP7%c)v!2QiP`lhT^M@eWd8U`S zKG6-j-vI-W0vUqDD&>bMWNXIU1Ea{&1p8($Q*ty8q;%9a{5b3p`lq|X(GrA=#cy0f z2^Ir?z|gI*K&?4x`3Qk%H2|0bkICaJ@bL!)_x(#LX9m-YNb^ZA3E1o;5#moAqoUbT z{|)xQO?_ZEUu^d*I!7#kz+#XL_pCB;gZ?wv172%BLgUKo8@26pejn73r`(Iju~~H( zQR;;t;{S9bs zqmCZ4`|mB&Aa^Z*IjXQ)XO4;jDF^ zeu_uJZ5)Ow=e!PnKyB8)pcMM7*AGbNvf4`(P{+cjM^A{bdW5`|+YQg;2Q!L38Py3r zZWqo@!zr0NazO6E!s@7}-)=Is{injgcJ!aI=XU<$4@mB?$o{h&Sfqlu8O`3x3(I5*dGe zwBUTnquUiftQAK zkSiU1{^<`N|GgZ|JV+iZCE|WxeMwm+P4TdI@_~ zUurhb=0;_sM$PYLu>eC@MjlcBdXOV1Zk!A6DtbndX0Ye(ffxX3+=t(sA z6~k`L#5-y-dnT2$1)dyck!_!T-OsB}KZk}8{pfGGO++QcfeOOC9ih0J+Dlv*$2?`p z4K&Y438_&UKUQqVgCBmcLzTde{4<#pJ71t)(YXgMY0Yf)>Y%ILHKg#aWNA&baz4>y zhmQ{G3>#Ma+kWX}<}DS;zw{RQ1of*hUMIK0=!>Oh-i@ z@6V&JplSEawfzNfySUqP+`Z}L7Ul9~NJ;{vb{fp|qcp$;!on~UmE`8SD+yM6R_e&h>L~_JIcSisB zpdeVMZBInsyA$0ib&B@jC}t3-VEC@c&jc0sAX>LqI4!3O{xIHfuYIO~W0hV|!Y(LI zwpkslKHN&ttKS6V!Oq;leK$*Sz8Yr5V5M6p+4SC`NN~+|D!R4OeSn))=4k91eL-;R z`_|0uK`%!+ca`jl-ONLqadrircd(u;BXVp}G7=|y%OhZqwnShOsE^dz1jQPF+>Mj`a{>WIel@JzUO zw&^VFdJZ%O3*Ptvd!)Pd7S=B=U1@ksK|{-n0TQ9%$)a2&A4$3R{C9tZ5cs!h7Ru&O z)DEUJdjwBOAgEBe!<*c*T`X3##$kdu5liCNS}le zTe14+0X(t;5xyM2dfsng7?2@K$x&JHo(ermgG=k}4iQlsH7keN<}604idG51F@-f*_j>l0oWbA zRkyAmF?<@&=(w&`t-2W|KS@c)4+6<&7vUFM%!e{!!Fw&(e&|4uTky`t{zuivA6Z!e zt*I!8u_5*!1I1W;KI{>_UswaKI71rH3;C~q-36bqT7Tfl<{v1x%Gg9=NbF}tJg&7X zut{i)C?4X-hXxZ}t~2cMX1mM0CVtH{ON8tXel`~P!D$!`<@U-x8v3+t3mni768FdJ zs3A+&PL?~K`76xoj6#E=9uCag4y}V++X2=GlPJ0`n6adPJ>|8q6(8vQt;TRiZKX|W zm0nAl*a)cYv3AVeo#uhhB@n-Rc>5)1U>b-C-f#_~PaUduJ#^r#nYR*n;n~|T_{bbz zt|u@nK#7x%%~vYTuwUkROO%b{+oHu+72zIhMm~Bi#_Pkev^>)bD}KbVq1xE$Ewak5-Y3p zjNf9w(e~GVraA!Y!3EVCv4^&o{F3JK23t}J93A_N>_`$Wm*NoJim_@Bm3`}6N zuAz2(&^YCs2nb{S(q((#uFzeiiASwXw6EXq%w)(d{^YFCt8VopM3aX*P;L?z!@QIc zI{Bunm4~2o+TCBWW$lod@%~-^vii%UN?{`Gc3asj2EU8wnmN^E>Z&@s-sze>~-(QHH>1?e1bGu)5~Dld$Z zF`mALY!x-yhE8dUqoiEwG+i*CuCtn9cq;Xzhhk3RZ~ZOy_;wnzw>Re30g%N+D+%bY ziJGd@Tt9FsL$n?UrmdI`vD+Myi6}}4;&b72P2G$(sMdtnH?bt0LVs|k7fgEbP&*G} zAn4D(D;fR{R^fvfgw%VeV@N|S>mFMJ{~f){*l7GP(EJ+`2^lnk=%l^GBkxhg^%(0X z=A0wEWDX%R#<_{T?3nbrDW@pG4wy`OU6O{$5-l0ShIDMOn8tCPI%>IU=2Bm+CbdtR z@6W9ceIx?;OHO@GQ%zWMZQ#)^y&ZZGHPdv|&o@Po6~f;zy-L}_dKw`SWN;?aqSF=? z4CO2zvHKQj>%VReC^> zWd@m5!>eAQ4DO4d(sKwI*2ilUH{_9=f;D)ldwY)(qU*8%&t1e?Wlv_z( zaEL}=1`cuIC%k^)q+sp{NsI?3Y|3T z8!Q6Cyhl@N(S?dfLw{y`=!UZSe%pp>*6{qcK6Shz||{_3OoO`b0Apnu2RK+utBz=j(xh)X0DY7=C%xeSK^8oK)pFJ7E>4Nhk$0E3 zO8Xnrb`5I)aSbZrwiROj5brAz4$M{lRV`339OlR~9kVfo!#uFv)ss`Mo)!STHx<+6 zgP##F9D4OhHi8TU!Y0v6d$?-bBQtDwkS#;ekZY)3pVi>y5+09gk|S+7Xm?919$gp` zb3y4t4L@mhQ*dCMCy@zxrnEhJk=m#A5%Pe3G^+*w&Lgeh2Z@xH#%Vq9%^{~9J%Xm* zbDtDx?NvAoWJc+E?vfqeF&ELirYS^ZF|Z`&y^&Sv*Rm@8`f#j{s(Jw4A7;OuTeNU9 z#>H}sod{W5GG4yg<<}?;JfdKr7}B|rBb6krEn@A-8|H489-MdpQqr5C=^EgiQF>dPUU{%Sn^ z1HbnbtO>Wwkxoo{Ci+cn*8(B4uJLT7HX_T%7q#Ee26}oa~ z*q)y;mhm?R!OY(H?D;-K1G~)Z84BMcOEFzJ0)+ob2{W0hA$_d^`VnB4ENhWI%AQ4B z;CmSPRR@C!Rtx<4OfYN#s)5%kqOt`)Py$jbV?g8cKlo3K)yZ>n$kSeQ7wAfb_Weop zkETW|9L3a6=x*(i&oY&KR#S?vi8kf`^!G?spuTBGiW2AG3m`ENCDF`77JN%kIJHb7 zK?20C?Yt1BlCi7%ETWs;$OsMRybAJl=niHUNbR!tx68cEDe!3E60^W16wK^A&z_SP z;Wgbm%az4nbfll?UMmJs8W)VWsJv|p&ZAu555P}nz(}$NT>+0w-)r<{c~17OfznmT z946VQ49h_QSaMgvkPI49awW(fk5rddsGTvhe5%1`S#H4cW3QbCil@(LPS_p{%vJK% zkmNX~QBS+NfFU5!)?xpU0s`3n1yT|cmZ5)&hF>QcvK@1SB_J?hbz;)k;ozb47J(p_ zf7cIkOEaE^B;`s;aAQcSsjv|e9Y`68*7Y?UK+bgI>>WHrJppU;(8~#|T2mi>UcLX- z;k)kDNAjrT@#{gpPj8uqzz6E7rXJ-Az~^)o9wu|4hlCA5Tt9~L2*Tb1y51dUV%eX=f7Z^1m6sKg-D}^KA`E0|y zAZ_(xt=arD#d9e9^ikVPwquJh z079Ij1EJr0_~eu7lTWA@x9S}9P`D0RD`DvJPz3Qc@+b7`?89A%MFK$pUgR1&dtQBl zyt$$a{DHJ2#^^E8ga9DdS3o_kMUX}pL@gB9zNKf=)BbC#lVa>hZ)C5H>71M%GOBR3 z`NPybUE8SZ!L6peq2jm&2L{D_&fBoC)zxwl@ zGEKM@PW55GyIqzrFnI~{Gz{0Yf$wx2^fLQe$t3r&_yF=ms)EC=FQ}6d!*cq5p0rK?6z7!Hbrr%iUmsGX4j&TC0<$-8W1@BSK}SC|K(Vy-+DS2RnYK?}hDn z0%V3B8_8Rme~Mw%<{|oqOkuqZ$;eyiwgjj2+pq@;4=nO$(%KNycnW61UF{AOiwm3K zLY=S3CUL`6jU)#HO%1Uo8s? zK?ewgp*3vg(CMlrY^Lhrod_>!szGa!P~KhUcgg-z1!A4F;CD|Wt_jqoVF{(6yl8`N zK3_8G)Tf)b1$KDIgsLaNi3>(K0F$A=t(~cY{|r~K=CxB=5sEaB%ME#a8Ls>c$NW2A zDMaiD|4_4RJ^`lgG-=o!eQB^DQYx_wU4(TqDw zvU4I^7uBm;juSkcAcA+*(%!h_SWWL!fB5j3bSuvM^nM{r9Q@;)ATC-d^iXgUovOa1 zkzV#ZT`pjSxpeA8RE(7ek%?KE1drh3(T)IgnOp_-ymaS)Lq|)n?qHMH=~DZll-%7GCLDLcssT;WmxS{4xokLkK=c4yw8(@eC1c&ONLt6$*r{lC$ zdMQj=vCy8oQNBp@Q9%VI;b6T3Q{`yYd$hUy4owQdo_t!)1|g(1ci66i!fti z7iukSKia@)+dgTLFz2xGmFnW5Tf?AXMmB`XD{hkuYo!H0YOS22zUU%FYD40O#b+?l z;jln&g7aKR{tD}72i1}(VzY!Lf%~#(J^{7^`h>#^)>e0xR$nt35d~tXrS(7?n8GvB zz!qQumTsyBB680>K}Nh8!sx}$*ynsoVxkf~L!x+*REj2(=YU>VY&rKdrrNB^ir%K> zE$@GfEY3d3Af1q^dlFIdB!OH(*<)G<1de6xhnGG~$j}&}Odla_>|l)>H9zzJUO5kH>{qaUvo0OHRp3%!hCAaMMga z+MKSTp2`uV9o@T9P10`#kUueNw2uqLXupb{Ee%P%CJtjqxKE}^f~!T|+vM!EJc-E7 z@w@VH>SLbu>5^hrlstAkHKB7RkDC-x-Jsf%WI&O#kqScoaefSOqRBCO$@;_THWi+? z$waSC9!82VbD&VNK&YNupKtxvg(2YYKw` z@7o-#nDG$ShTqN+AahTT;ipL5@?#Z##rSAjT+g0M&F-D;mwqe^X{OXSEMoZApzOLh zXd2is{)gZe7OgJ%)4g7B$Q-4zV6SEiCjK3io|u115jDAVWk@2R%$|l`pyc4c>NZm8 zWd}5H9JpYcam`l)BTV(I8;boXeM_0n6ls@%?RkpO0)5r~r2U^sfh^g!(oTbM!;&igmJh-qZ)_aU(6MZyyhSnzI zK7bpA)BR`f|4G&Ay9&=KX^{(+{EpK9#n_m14FBc<$8zmfU=iSNb?NT%;G~YP@A3z_ z{sGNz=JQk-@Sf^B#xK#g$F#k*C`0Q+qnh|;| z-F4#7GalfzdjT7`B17;H^ zg)OovpI!YhB%@!Y{0>9UQgm6j(&BrAc4=(VSFy;NOSFM<{e@B0SlX_zOBRr!Zn<_58`crFa+>+Bms+Or z_>k-2zg7RupC<_VY`Z}7;EaR{ZVS&_pa#I>Um=xAHWP}Oc&cFf9&Q!7*0OeBk*8kn zvp^jq)xmtyp;mS(%fGLg10i7R>g1;cymhaRD*&YqZm7cga&YKT?&N8Vox4MS)qE0D zyaQHfTCiC+!>%VIBYL0?K^6G$cj%DR=PIMbEm&zS5Ay%9r`qh2qwuBkm1FDaX69~u z|K5|&mikc;$)&b9(FUKAWIFQkEbFiX3r-l@-UX=uLt#}vAp?Mvvw(%1>?8Kl(EuD2 zI-r8~Zq_q}1a;zRi_Ry+Y6!wQDo%3kR?VibEb=C-* z3J>SE> zJjWG;j9K<%Ek@c&EYLp$tX{%r2gAB(aY5)eh0|-#Y1bKPlUFvZ;?rH)?sg$tNKzyP zM&(AJ2{^m}AyG>z{NU7eq(zZcqr2ANnkl5`EXLbq`$9xX$R_A10^g<6V`2pv$v-3w zJ8BXsduq_GpTn3Ba`d{j8!?KLIDz=p*kNRvH4fx8;~s7-|cYC_RK>eZ$mAWVTXa7;Ge>aE|PweClNUC@4ju!&Ahx5-s1RF55sZHb73-l&}T+Oy1pv!KNpB%eV$ zh@oI(v2TrYxz_dgWR{tTrswP)VUQK`bX%=^iO{ga!2{|BudF~xa0GCk_nW}UtZIxANCleK|@ z-LXLSmS0rwlTFVPanSWN(r#M2CR2$XfWaL*QfDuI=E*f_f|f?Drl}>wXBJ=r1rWId zml>DvqIq7eZVIgL{1VV0P2Fo?*+uN=ahCfv%7_vaHD6G);5;dU18igY^Gqj5=y8)} z1h_r5-3r5dt1IZ@r1hdOG%vlg-n872o7Bl~EH)g0A_SGFyTX8_{Q5@KYUJ=Sr`;)u z9mP7@188vW8KUDK1dr4qQ;3c(Mh?j(kKk+tN~;Wk%_DU;6)bhoHdXL*dcN+j)qYia zR0jm|DrGea`JX9~s62a3<`E-qUszt|u6mYuN)@tij!QMx87F?We`K=RamqCMgB+^6 zcU|`*EYG4DyA_cJo_hjt-{_-?#p$-rj;6}DbQx16fmE|N%pW*^k|CDx#)ZO>=*^bQ=fAjD|v zCtz;S{|Ga0W)nag&T#Gwfbbk=XPebzFA$$T_OC>drw7th%4W7|JVi*K8z=Rcv%L z9m~pX%|dMe6eU)DKymKA{Nyq&ksiTAClYx%eM+wed4?s;n0-YLVY{)$ucV|Ck_}xl ze#CAb`!vsgr|rE)9Q4Z+>+Bc>0OWx@p3XNvfB7AlH9rS#9ENUejtx<|6K8Pf!j=eK z*)@qT_<^62i1cwg+Hd+{aKUs;Reek#X#bP;AQg2Fjmm)f0TuFW=rhn92c1XcZW-$^ zpORGV6K(Z7O?v|?&`mG12&#=GVU3UIa6KD>Q>hO_X7U))i>*V&`S{eM>|q2 zSdQ7mxu5oc)z+Eizo0P~k=+uOrp}hXPYdffa8?i6Cp%n-WAVM}=Tu&CP}Ms8P4%~b zs{^#cp23vNA8NLs0pQAo?N{6+%J(X^W7T=)3LpgtOi*3MjLY$TOBU#8IQnCWc=8)> zYv-nP^rc>NwhsWm&dqyIv8(i?sA=XK;XX$mPQTuHk?|$6XdN0L)~+MP&%9*?U8m5e z6^67A-7q@lUFhg@E1uy`-+!roPr|-gg5H7a7PHL`%E0$7>+eLAff&WW?w${1(e+Ko zOxQM|ONMR$1eKt_QAa6=J-H)>giWB3^D{hUqvB1bz5f{aFrStnl4&Z?MJdPG-d6A| zYWjX8@neVnFV%jta7(IJx;ISfh~Y^e-j)fpdRLmUbXZrN6fdp3FKLzMs!g~0+`SRY zffwer2IMvKM`l<)0F+{}Lv7(sc$_ZKnt?mmLXv2EYrzLN{1WBT5G8yTNP}$<8!VnE z9V-hhG@u#}rvh6(P}yGYQR^jpM5-S*&G!kR*pbiEyT%zJ}PqsUJq-F9QZcnP+t& zk^Xy3O-_2re%T{m$|ulF4UM~tZI6jz-);;XF=zX$MM@G|=9AX9TURdy_cOT?`nj@g zp`~bv+kx_54nu}S9{SekjkUpj9}4^HHmE`X@`fHP+;>MJkYJ?k0Pv2?PEaG%TEk4T z4V`l|#AhjO6S`Rp=g)N(=xd>r_0HnB6h7)BtB%kvS~ccj4M<&x4vksy^rS@gZO1jy zS&9Y{zT|CXQ@0o{v3?+RS`Sc&4osGP$QBKir>);6%ILaH@MNfYp<_Bf3tbsct^U9J z;5e#OqinH7yNv)mFY(4?Jkb=2W(7f-zC&0i%`?qg>g<|)OEw{;Yy)783Xb6aX{rqT zL}t)8nR+;eJ>=Xp5GfTz31N4PX`KDkI-j;jp8Hd8Kk0Nm2|Nq895zY4g$4lr#Mz*b z)4%!nhe}^i+-?GCWbr4wZ#0E;^g(^%twAG7VPRI4l)S7g zcXbsVbPT;OXGd%k1hN+)s;FSY3jj+dHGV(-+{yVZMQ&Dr1tni+J7ze5qpkM(f-bHw zKG4hR2i4OBlm&u$mn6k*VXkm{Tf$-*_zr3{4gMsiITR9sDV3PdhFZn8Ha^SsFh`|O zKp;PS--0q5($p1>(S5ZRlY7x-UhL~f81dnXbLy7T&2>MzBcN9KJgciah87~NS5OMt zmWBVZo#-*gfyIk=yHH$3ceWk^hx%aM<$pa|ohe;zR9YHci^vT1PEMb+=m8!8VnCh0 z0g48x)Hz#R*CKDdMm|b=+>lBm#)S!^)ympTmdQ4KZlTu0v7W*MJHrKH>r|dyi9@T6=dMMiuJ4fPd{aeZMhz-+%5lw z^;w2%M}lh41#&sdp|O-jvdVU=h5=C7tai~Hx*OFcPRIi+N?&HFc-o`HZ}+cVF)U83 zdiw;HIFZ^Ky114I*h`1p$c@fOB8srv{rfj+foTW{wn{hHwPu)i`XLN5fB623WSy7t z8n^`7eMC3rs^|SKDEIeXgC-~RDkK+zZ7WJMj`Rc>W^2qc&3<(=3O~e&iwGex1#3}% zIj?%_OUJAu=lU(NO%^qPhg5Y6a4dqril$2W?dug=G1}s+Fy?HashsY;{F^- z?TLyhdZ@G@z{;u<`A$;A8iS<~5X=9M|H78gV_qc3fC+0Bc!ON(JUvk_S<1HM`Llf* z0;hJhDlz<=;f5JZiVvRw;D9-Z6=NlPkr%7L)TChRFWq^vZfP>`snRDr~ZV>MTi1y+*NNJ1)^d zTLY)H7JCLyZI>R&k>8~| zA-YLWqj_leZGbJi0%SgWM%{-K0*YWaG7Lf6_th%IsN#tU!8-BDKF;gmkk`n!JcBzY zg=pM^@TNz}v88(IIwK=VSG%h@Uff|A)p;MXp*$`d}ht~5?RXBuJYTu%xx*CEFGRKZijs>_O! zNn!%XoFxY>`IBK-W=B>PjU?K=?SZXf_TijASY}8-`=txk^4L;l&0rCA0!?_K@Iu2yrfKEH+bgJHm$o;(rykvD6KEvNt?|*Qh0h#)F z^P&ll?;YHcuJ#7k)lQe^p`~N6W^*?gbJ-=ek<_N+;RbqK`pv#T94B5w54~Ymszn@? z717c?3MJdw4;?+D1_06|U9L=cy>>+p%z+})yfBm?0%%BWpS^Ey@WJR z*%O<9+QbpwqW!S2!aIoVAJ&h$ifr%pRv74387wmw25{PNi|K- zr-U{3c~9XkH%pnl$z|(yfxfJN-D{xp;fs$y1o-u6o3pVzbq;ju*tAJd)K3)7l4*bAFhq)dGfTSnKMzpj7cDqxQL>h|%${_-mAK+i<0AC5ir{g2W z8SiXu%8r247jNg1iBbE~?+S^q(}2$@oeq?`F1}m0+_KS{kadj}yuBZQxHoYp0p+N| zU4ddvrQL=7#Lxv5lM@RXYn`RYq7Kl8N(^-}ZD3hnNK>lK=P6+WU6u&?{NumaiR!Sn z^8i1(Y?cQcMT`?^Rp!Ek5oh|VN6XFfsNMfjg>Oq=M2Mz(+OXIU=`1Ib%C_Z0GESJ$ z|I8l6YOCQ~>xZVd+{v-hd~Hm<|FHeZ;<^bQt$`q>S%PqGkkbR?IYQ9qA3y*2&*}fG z@*5&XAu47CQtk}(f{5{~iblI^*ln{y@Kp$esd^`khgrj6*QT@O#wVFp#wE+&5$Wnj z1x|rhqYR8|o5K>@+-}niB}53mCm%KetPv)L&?>fT`hy42u5zq?$T$H{b~Q{uHAXu(VF(FW`{{2fJ;6b*PUF)yeoh6@n6*Y{ue8a zLAwnX##c20fg^;{6j_&G@d1;H9)O>{9biA6R{xbNY1N^6iqv@Rrxcp#nGy=4EI(|S zuRIn*MHS6AKkNz7dvbw($?{u1AZ>^4OB}`ECRqWIc_guLA&uB=@OA0J4jUM^6S?hQ zv9k;=&Sozqr!AoH6XUONM@|wK_28KEe6z&iyA?6sn4cD&1D|yvR~CP zTUVrlFvY5H8>v2;m1s#EC_yK(CgWBE^3bRNnG5kn- zy|rbz+zcUPmL?sGa1pxL7vY$QJ9d zdjAuToQuCN0L0fkE6T3v!5+HB@MD5q#W)w{;3ohQRj5$?*T3l~vL++~FerzNKHTcS ziJ{kSInD;x1gOLKU<3jGrY?d7thJ#SS$a9!M$L>Hw5|yN`siZMm$PX7oyP}^Pfc?&<$5jHZ5>-lu=oWm<7mL}l*_Nne4q7=}%@B#X)djB;zB!y=u9fZPW zGCL0YEm@N=K_TvYi_-$I0Vu|e%GK9!M4TTG?w{J+<%XLJVD9WJQ(!h{A^&Ev)qjk< z1FQgC%&LQ6+BX8aV$^jjO2t;3#?;oIL))b&Hy|b5E88DqI-Y$bv_{@lw;F1i?h5NX zD_g}dYt#su;?-UsE#J_{qV7VAmo*ojW>8NfJSxFKk^LEa&N z#vM`Nh|1~^4>=~&PjeCx-J%!x5XhRcat=gNYgT|Nu7Rk~PptJli5?MdBJvEF%T1TO zY6^0OudF#cVX`UXtk6s;4$|uWtL-YMIW!HWdGao6{K5q zn6}&)s3k6mH?oyL35LwJ>_b5p9OKNMcxUDBJK4#Gkbt-b6b-S+R2qve8InOAt8*Vs zSM(eP@_F}Q%G#bq0ST^{TRON7-FganmSzsNPR8iiq8(71s`KW2#zvUL&Em+rco-D1 zc91vd(v}wuF&eT$R1Xmwpdp1Q5b^}{dh7;+Coc60GbMIY>j43w?noy~Duu?@fm+sq^Dk*KEh|<)X4$dJ$OtV1KEaN z*XKGt+86b#tzy}{bMfM-2}0c-={Kp9{`f%$n*rp@!fl)rKI$zj8g57*U6*XobZJ^V zVMx5GPC_yxwSk8r_uZ~gW@3BZ9s*}$nlivw=N_47>V~3s)3B@UC=?i-3MeKmY5?z^ z^KzxREN-kFH@J~XNI8e_%wj)#d0qp}a|uMly4k7p#KedVr7NgXfcvI%+3kW-t zij}6?>k|XuVLnHHu-+}t35TEtLl(>I8yymWa@0GS0vmU?oo_=@DxpDZ$Y78n`H~EG zn>j@N8j`}&DMNqSbW5m1PePJj03tQ)V};KSHmVRFL}iGVe64!v;*a$Tifj~;$02fU zBw3WGsy5bA^PH5x#=`qk& zS^8Vj*GPA8p`&IPU5&Zx0&u)oAYDt^t2)DHMIX49pE?3rK=4Zg8MTz5{u^ExglXH= zhD*RIB$pSiduarn&YyIeFT&j~HHjtkYr}{&bV(pVi-tLDT5Y?Ye6hx2Ox8i(AqE3C zdbMkCxl-VBk(Ot56Hj?91#R1ys{n&E)Q3#1X(h~Y>O&86A>NRUiRwD2;Pdgri+yI! zS#QA|Kk4cd`+O884$zJCuJh9eJ`12cx|@CY1p%<2C$!0(eW&_PYSBF|@WWc`ki2^4(i3^&lL(jaVWHqzxS`w>fGE&`3>ulcK)<=y zR`8UaXjQ!k>^JPV@BnZ<)|CU8JtsO6|!r5oM2s(u{!ZdZ-Q)_o(h#n+qNhM!BwYs5R zgd?`gK9}cQ9N7|IdXF!#GcFjncIbp^bx)pI;GvVum`C{3Tvsf+c8O#?5Qk0_mdX## zI|W`L?j^)A;OXf{Gf;hjT%l~??!KXkHV~>G$Q;mv^HZ;<{!Zw;*wbwxVn$#RhuoG* zV{yAE`X19BJOB3Jfud}NHx>S9KnSUu;O(*N%UK0D?tS%&%%UwiM8~jFTcT|T%6Lt| z5t8=p!$1i0%Gfd<{<`|>zy51|UMp$n#_`Dg1qUzqVM;ei?`Pc5zwLzH!_ADt7w?3I z^AjT!27JkRadHTyngawXaI*H8YUCo`lwP@+Q@ZK>kNwxIt%AnS4oz|k*$h;5OXcG z1lUytiyQ4wkFz^SXFw&fCfy#hPc{od_;C5u5eE9}iOt z&n(!Z5QSvY?AA?-YSfC3j>M?BI_1{-1T9C~z(-9&fu!xXJvFy*FeiDf=(QvJa)}WO zWmBqO5VC3rbhQ{W0R`gFI_TMH#U+JC+iVp6RdleW@00IeR3HBp61QT2?@O+d=mM;| z^-U*i!gvwop{lk%mCGKYhULtLA{Q7dW=V|d=qFzotMyxg6{J$yN1sC z>0Y-1mr7o8I*_4^P9=fTrt9}o2m;~@F6H(IDKNF85@4F@)-)82sNPWYz7!!D5P2Fzuz=xLKu#2@Q#<~L(8P#|q^jpvb|0GVFe%wpsd0RiKLWM+h&rcPX z!%Qc&;H&S(W_47IJ-1K8x~+aCfwYxNk>!ho-R;O3CwlH!WOL#$=!UnFce{lN;yc+E zPMJ`_iYU_YGfft*lU28%vo%3p&xh-S0k>Vymeb!MhWd8XF>}RyE1*t2Lk)qQcFBMI z?BlP4^pn2{C6{0N!N%U*v5+X;m#h>^uV5ZtyQxj&Azmzds`Ctlx0%_ z_GYZuE}gIO873G<(1;P4Qy7>R%wYpGQ0j*J67yAfP}pG=BnB!oNZ6P16)}z5k}9@Rn8&k%B!g^|@k~?zSb4LKblZ98 zgl^mFq8#xVZC606eU1X#&o%CAnA}P!D+E)UD_Bt5Lxax*$>|d3uU4(zt@vK8h^If| zfqIb3`GN|rjX}^5O}(o!T9Bs~%mTBLroD7-QcX3FZdznR5#x}7S7&JnbkzkfxLqu@ z+mg_8x+Gc!gaZ;-c(*6!9sbmhA*{&eZuK1Pt{H1li!^LN!Z|AMMG-QiC?O3;DPei?IyDBFMQ_jj@) z;{rv6!?^ig0HS@!77zhX1j5P#XXJ#Il%P`|9opox(L_$u$P5@Km8m@62d zYW8Uf=&P4rNtz(y>CEvi`q;iqakGe$nRbdgSmGL1DX{7El=eH$bxXyms1Rtf_#zIAZ77c6(Y0#OE~$MQ$p2eN%i;Xq2_VUQjg35W z*wr>mKNk99Cx;qKKd%z*r|Fl*A{WR)5^p%M2O1NGt(=EH0N152XK}V<+2$5*-oQ~! z{ozEj-YfY3YM1$p$`2eNPc!?^a5(bCZv%uAGMmjs)pdwK+_e0;rB3lS(EfZ z@Lw|t=Kxl3I=brYl5r1ilEop^#XAniwPRbd3GfqidDC05WNr2JgfsG-Eg-H~FtLfC zxkT{^K(Brexv1IhS)JX-Af~NlDt*8cxcjr-pwH}C%0){^f)I37p7Rd8pwJnm z#PJ4E?dm1*u%y2oUfpO_23`oe!nb-BI~m~9mayc%1fah9__?RVVDYVb`qvH!fH2j9 zt>-c8?x8e#<4uF6MSsgd%EikAoNq;9p9PFjph*Y>J_9lA>0!xWIf<+(B&+t4djVW4 zpYHTC@7E;W;XsE8vYzMq=y3&#B|YH((7lKd=QYElx?^QFR}s)=jZ3yV>aN=@bSeIS z9T`$k$`*y}l*g|^;zWnOjGib167Wf|$77%jT;2e8zzxwPOGG>}EpV%!c$Dj-KfSTE z`uJ6P^-n8xoT2;=j#q!{^Zq(($^fb`T8WV4Lu*TfN~#B>8ujB(Dt(cJX#Md}aGD&R zAkIz_Dw;2_zWb&rK{Cy=TqkIAXlbEEG||Kksd6NHS#9oU4=b@xJ@4Tmkv;|Z4#^$L zn>f0xHSCkEI6oe3A&|~nK>5FdQ=6?^SmQm_h9{v(C=A*Iqzl{6Fq8EFqH+yI(MVt6 zqrJ<$11tC`EEAM?k{*#K4oOfl(DKoIrsUSWJDg~1N>isU9F8E;IuxV9B_^1w)+Xy2 zyN^yh^TB>&`5OJySN+fu=M9wLR+qS;;0#MHHO3xr9BmS@l`;im2|@Tui~lQM^bcAm zqBGwO^}8z1Gr}}w;%M$FaL6L6#|yJpVG-XVcO9Fk#{8NLN$h9@EypW7ugedmNzSCa zALV}a@z?1l6DI_@FzHNUuk9Iciz8LUsX7B9U5-0j8m^FVw!6}Xz7r|3k{Qn(pnFd7 zKuRcFE+8_x2A!o3o)RDjv=Q?D+si!FDLp;E8SM2u2Ive02?AR&kl%=Yg7EohStNP5Zzzf4_SFo20k3LRuw#7$DulnaH!y+H$U4b|0D*6ON3tP6-!zOq|5sa#Sr#MXYKH9>#ClxfA-A&~fx!LnofHE5+))lBFN-+&PEz9q_XF&XG8 z7_1a556Z@4=_3yzaJLhY76X@@4=rlCa%SYo4U$dfYqM4(I^(nUFcYLMq9CThj%jKs zO&PKk+g|4LR3A+n5O_Z;cW&&XUNVH=LMix`Z=pdZZs-z!G}exv(QUFUKZJywY8{K`qAmbqZVtz)&4!U~I_?j&~RD*wT?uYJ81a zkptAJ&QKn7uB`k?uE@N}(-Zl0$9Pw2f1ndjl#}H5(1ktae}=Q8_uLQrY5`&tj$lU@ z2?F;G$zlt6UvFfa8FhW-@7XpG{LZu>W?>2+NmwvyDFQ7r*FWde8o*d=^WY4{5=Fyv z?g=(QR!OFwfYUfC_Lo|XFZuzQNH}&L;DH|PVee61tNH+%(FAIz^7+CFBILWnZLe~4 zX9I|veAMM%e@)8T;J%-+2lA_DBb-rBwftitM(f%9vBl~)WDT4BR%ab7Y+ zc&0kBuXdn$)(aW}({%W@GXamt)6h7GL7db*oJ1;aGJ}z=S#kgl5yi2wg zuIFRQU%bU?(Lcs=DYgGXx&9Zi!Fr`7D7`|Usy@|HJE1xZ-MVb`taxKJE4n94paage z2sEuQ_oYas80Od^uofL1NOi-XN`x5rCuN0z9S%vg{Cs-(t;OQsT{7sw$C_RsBS>#E z-HF2wGxRIx8wD%-H_fI(ha8PS%k<#=1}ui2(Xn8a9}99Vw*08M^+eAl`xIwi%#UOv zdM58BgOM9LdaYMheQ(wHe^veQf5#}ORx=J02Nj-~K^xQJQAsJdTe?>7K3`PYyLW|> z99zh8#Y0$_6}}qH`)Qb<=(?o&(+VT^=89P`R8aitL7utbDDwW+FwgnJIf{19;xYhV z$=EG19!YR`_hYdz~D& z!`)3zDdz#MxS1{jSFTGBh*iWk$&hA=akQI*>jR7=O_V$=tFKXkw&(FQv&-Qa7}Cvw z2CLUKM5KIQ{y=|L-~0>xnQoxncE7RE69C{&ZRB>gUrIj07cp1P z$W7}|=5bx~m<1yDwhCfDyzDH1wUPm`(K%=9!cyTwZ5JZfU{d~+O5+QpD?ZTvta|?i z7TXjVnq?>8DRseg-X4``%~3`lL&CqIH_|ilXQJ|YVhT>>!^w9#2noadj3hNQU}yzo zH&5Ms7PQ40%L=`g_qf^{CV%rnD?!h)>SgPlh(^xbhM~zd@tge2!7$c~W{MHv1N^pn z|7#t{h;eyGo%UGk@{dYSw9dgM73gYxra}_%=6qJQU56G1QH0(S3kUkC4!O#L31R2f ztHKUPYo+ShmIyMq6o0ir;8);2wU?j=xxO7zqccDzr4D^Nzp5-G_{6Vt?_g0Wsf>Sk z|3l@I=+1n)&xgyUx?C*Sw{#{~AQ&`aLF;Mlv9H~5)K;?N!%Exty?PC2BdIM8UG}>; zM}r&aE|z*hEHntK!u%NBiUuaWLy3pbTH)NSmMy3gtmY=ng@ITS$8 z!gkhIO*@~zsrcLn&m2=qUmzRw>9WY}L+aiKO0;>sha8_uq5%AD-3Yz3RfbXa;a+tt zSfNS9#Et zHQ)zoo4vjuRKA1*kK8M_sH~VBXw57$OfxjySf8tKkSB!4-zdTW+H6arAkbV}s>fCw zW9YckP^@c7sdq{cW}Z{;I^Rp~TNR=LV-HovV-Hf-426lgn6l2){8MiH`Nw~)-hbcc zDMLRN_&Zx|a4O3P3S?QKIDVXW^dCeOBS$h7KJ>%6Xm~pG$F9Tzp4k6PHBIk*#8D+| z4|2jH_4nI1B(B$u?j*6SlPV#+0b06TPx-J!IR{2qhrVE^VjnFnmskPs10Q;7U|93b|`IA zNHAli02v}tRxs-fMZQ|?9@ISopdb-zT_C=X3$=Z7OYN%!Uyv(3F*f%B z0Dl0_KY?-b8L;Ehzz2~@#?W{7jG~1Z(yqjzTVrR~0;&to!5~CfVG9v(aC6aRzxUl% z**Q;&RHLdE#;Q86scErD&FFVhGx{Bya{vLv`s{O76NCSmod=;)tZKJox7EW3;9CRZ zF|g?PeZkT>#nD-Fi2>h=wp_4K2)58K_fUVC?mc)Zd9^}1Opo$qZwnUxgD2*iQSRUf z2$o99Pkt3v{vqGwsBPa45IYR*z3K@2&Zn<;lz4ek0orYU=qs16ygbk7-ze)%LuE-B zTT%cZ*ed>@MWS0xIk?bH=_gpkOCxz*;Vg^l?Rdm2Cc#-BV4DgHS8X1C`ysM;mRWl? zbdYo!|44Uyg3`GcBQxCBvu6WmR7i-U^H8cpjOo#B0T#H4*F zptj)zs7A9Npi$b)XM5q-cyJbb0FI4O&Lyw$?ww^Mxp4}gA12V$ct4G~S*Y^?#Bd4RMi-w%i5Gh9p$(%@V z8)jQBQIvkC`rowFs@u-V%wC{RVk)a4`=@poh~0*4JcMxJT5|wy@#j_?cVW8tLZ6<`)V^|8Jveo?iOyU1sct5Z) zG&k3Ib*wJ4t0`o*ezM;}Z*D1jk&3B8C~fSH0j>&x+4;d7RAcZ)sJI9H(~m*aB+lb?R3E)yDAWcF`=@A5|9LI6<~1>*|!kdtXZa zg!o=RMlhJy*y0(~7$$)Ioh382?z0M*$Z3GcEw*;rfpQ20`!!!iKQQ3MLNxGlLfBZ~;R^R*!yFlQ8 zJG@CF5!KgyJJ1v{n|c`?n@SQ+=G`3Pv5~CHcCY+C8C&vUM%Om)Xylu zo0*wZ?m!-`77=qNg|wiSYBKtG z%%Lwk!d7%cTU1uyU7_#5nY>p+5ifSt|ut9kb*1Fxicf zdaS+Qj3!ZQ*>#AdD2cGUe0pdZ_DcZp)Z{|WRwml z2psfEE>yxBA;zA-wl5w5u)V(oBfBa*i%0D-8anuzg!0hrX~L}M*$l~SOpZ7=I+%ag z{;TkR-}PI&T*J&1{NLS`MmqO}$TR+^q9@1&`u^_hV8JBgETyU>7yvMY2kns>(d;7z z4gVcIAV zUyBO(=Tz$aw3z83H4$$Zh>4Xpsd$=};~u&v2HW!^5NJ2(XGdcxNuCup0f`wtI=2Xz z%WkvHV&w&3pEn7%kbfk;1N}iSpdnLn%g*Apql7Kpc(eRJE$2GzaR>qo^C4*umgW#J zf^%lU5$w3?xaSgmnX#Vr3}8>v_3mfyE1zz0NT)hWy!@W6VHC76XMbD{M9?t zCzx5@F0`0|2Z6)1?MM%~>RLVx$=AnnIZaVu!BB^9^{`SKoyA(ugr&29Uo+K%IAJ zDl}+QCB>OW=sVqG*nNf2o>Gw6l!~Hy8K;SukIP!~p5kT|ThW z8_3CaQ$EOKCiMICkg8Ap$vUQo=M88$i!=#RMc9+Y62`oxChA2)hsOm?;!|~$?h8GtWd`lYvhehQ-hs}S#Ds$iCA1WsdH^PFppT*=Jrt(UydpE# z9}14e<=+vY(}Ee;x`|x~HLij-pbwn*Cmzt9ua@8^MkWN|r&xEDxUx4B{Zs|= z*Bjrq=xIY!^HD$HChEy^E5S+H4wVO`82gwzDM+ieqi zwG#y0A<8geGYV+N=2PqO=!;;0N2Id72k}biy&S2Iv&AVp!WmxGkR1ZWM9eEmMA`fl zep!7?jrpm*_vNRiTM%~F%N+dS&(mG$Pqxd69fk`a(z&o{ywOua*s};pRJJ^C#tEh zyKB0P3p@htIZj2S87ThoQeD|%mnR}Eif%KH7NC1ij;DNhi*B5w5^ekF+xa|?w?q~M zNEXx69&2dgWctR<(TNL41LzNu27{|a^7PU@!txcYCR_G3hpH-yqdI<5B_k7z91b)` z2xxxzF{Y@PnslQDLMzXrRA;|+cmvCSp}Lz(h@|=bnMO`Yy0@BDzKoy(xb7?PUi|8^ zj{yujA!U2@olm82HG3$xRVayXS-AFuts zV(whG5Z8)*hRdf9s4tH{ss83~)2~)jy0lH)0;|?;Ou2P;hda>g;FM^_S*J!l2T zabstHT&LB-v|5fMK;zlzW#y5bjBrns^M%rowE~Ii%){Q&;b$}C>Do!^ z#~*(2@fU!6V&?rIRwJsWE(J2pVa(1_8leddbyXC=SebESHw*M;GT@#{wxS`u^ zN1%^{YJ!1%$z%o?n*N1)(P~Hct!NHjXrE5SswUbW>j?F6l%$u0p@mh}NM@PUc`6kE zsRpKO){5&Vcst8Q76aOwI%qV*a}4=4>y3G>?uP-~lL+RekYW0n#U4Q=EI91ccor+$ zv4RLK7lE|e+#|oGJP3vFjvU4k$}75^2LRT*bEX=g8-k*Fk$B%R2=g#s@}4C=Eo>6e zcAR#30MF2^&q9a9_oZNogxpWaE>8M}IF`C;@VM1*q(@QJsJ(H*RriC{W?`R(xv@8b zj#ZL^6p@_v5%!vtzKKyrgL6oSHPVh2CFls)tV`>hr<~g{Wtui`NJl1e#zM1YzG|jP z#|#N+cW29xN&*1ypZ{a>KgWnW3Z7u#SteXd#m5ICfVJL|&`gu{Ye}*wBykkf4OGp=x+Ec@vsTqw6D*24v8_OKRw6NZ9QcSkS44*t27~-Lzz; zp}M`kM>h+gS9>>m*8W-0@KLvK>+f&$?EwjmLi^hh#E$za1Z>^D?8qv@?tZCWc9nGa zhpX*?v2@S8C5MR|R;%duJj?UVV|M$5gN!ei1xGG`Bdx+b?0+pagW^>omwH4mHpS36 z4?(Y;D1obz48VxWBN~8iHY%Rvp=DFSTtc6EP>=dN5Cs#Q#VIi}8uEDe)h!+$Ai1GV z&2YO16L z>GNTWrsC2SV;70K=A59y`qi6o$YcTip4IA(uo-|+g9L-bKc7JaOcxAkvyKBWfxDVQ zDCuIGy7eXeN^7PhNi98{U;Crv$U{zw-};uKYkChtHS2ytLptP%f0me=i0ofS!J8tYC75OH86OW zJ|Lifog#N1mXZAc>Ct5JsHrF5z}>B2;Q%cwAjC*J_9_{79MGvQkhzl}%M|ce()jRC zN1oPtOC|C(^}?Ti#(@tTJ?UhJb#dNmy|U^FKauB3)O&Pbw^zvDs_cLl8WfoA#i1y= zs=!#t!GM%i9ez}H#^y`j5qqm+x-a%bpO#n8-PBj_k~H(uvBC*yq~jako<8h^CxWi#y(G&5D&+Qvb7(pvj0GK}sc)Fv7jPrn4!=z)q+4G2zO+n#12k@n_A_mZ z0DT;z(nr6-g8b8$R5Qewh=QVUS~=7^8DSDabJ2K$vV6$(vSZN(DdG~W6oE{Bll_cW zooLQfo@e5;l!H8|M4eIEqLKgTP`}b-v$sGD)y3)TlghPBp4c^^t4KMBv27aG+ni!n zkFL|$hjI!x%cyahBrkxD$l?JS0!jpXO01n}jG|?Qu6^34Nb2D=qfn&l{wF<(c?%&?=A(1-v|Yle?S> z4TX#g+Es0g3{(F-jMvA)+;EF}oz%SS)-K(UD-dB=8ExFQPj@7R6IQWi20J@DxC=Ka zmOui1=rJ^7u5uF9*y%jJ%d3}!k#Z(@!*c_~2nV+%6$zz9_i%+Lhoz7w_B(SUoV?+< zn6|~h@a>WP*Rub7r;0&ly(s z^*s^S{r8SNjE%%KIkKyY2F8YCT6*GE(t4nFabrqGJrgY~);m2|9^%Vo!-o(505546 z0-S=nPxnBN92c+|ZfLWgB`=?Av?jo*oXD14&Anh|0_xwU9PvUa8+F*AwI^MbBy7`V za#;$uaA|%cS_UpYG!1cWz7vq5Z&W~=n~wcNFJnNc42b8MMQ4z@SLnPQ>RX|`k>-lN z;*VU6*p{~*-E|7@`hJ)JXzCMn*B;4JPSYfaRYdgoBYiVCwY7HB;oO3}sC&iH1TuoX;1T#_4<)c@y@Wry~I>o^Pc)ypg~fjSoFi9;uagrXRU&2rge$ z(Xa#V%s#vjupWAFpNXo&LXWU260Riyg(v-I-w4F^F&=?WK0;|Xf# zm1X!mwe!G|!Vj%=fe1@s=$qIxbUyOCH2`tx1F~OeV)hGvy=5s~;gE1m7z%lEwPdU| zbWPEHzhtH;QuY51#Yv0Ck89=Vr#!N2cBpWvdSnSeou>53?f6sK`qM~kTM8I<*@|u##oRl=zMe${2_b1exEx^`c0}5&wHdv zmM+ly8p_zN%k8@xKNCAB&FDd`gS+agu?#F2fy1I|flHb!aHPak0AwV`yW7F%9|i6w ze>hyHK5S@_U`^1?kY3yoko!c{O|MAQoB#UXSFSZgwTqn51u*&?GoY+d56Dv8u)xwL zmP9jsoz_!zwk$0qv@CBrs+EDM;KNJIB+ZBqbWa|49-%PIxn^D)Y);CF5K}y_xjEF; zzQ8#LVgHDjyZ=5(wHg0ox40oyPu$4JeQc9$a{Psy*sT(_f!H$mvUsqXGRNFCnF!bn zEEJ=8GXBWy_!n*1^Py$h{*bXFa_Ezn{FqBP42pGD)+;7Pis4}bUm57mdiJG|I3 zInl{kfoyg9fUT(00*4Q3-oizCs{5d0CV=2B7K%6&I=w0-f}9ofxdnzf;J610c*QgK z!&dzvjMUMk%EP>B-+a!4XfjF3BuIipsj{oI2*Yd3^%5AHp~O(N5(ERw%@R(Gtu&21 z#{t|v5k#!MCU-?y0CiD_5GzcaqFYW0Z4gB-b7Y<;znf5+#`G#5@pU_9<^kM6FTHOp zwDN^FYbu6yn>a8SN}AI`-6#9}ks$J2DuupygpZCW!MZprbM18Y!d$ zCEz2)7_*yPp}D(s7iccPpK2=_>6THfdZZQ;w#!Z(Imf0!f*Fky^gQIXk&xyp$ClY{ z@KJ#QLc{G7R@m5xVJ>JJ&MkFn2;X=fkvz`(7=h=$Hw9oVqWMw^E3+}!0X#m}?1-|) zxY`am9UB9T+c9w_5ge7NoD{a9V{%fu~X)r>-S@B zm#m^4HYjAH>(@{^K(B(~;}dSVc|xaq{D9$KKi;qpeO15}FLgpiiB0%R&Xn9K^}`n* ze+={nS`uwf=&Kdy>>3D{8HxipC}9j@*EE&5wqIK|>Z=_6*%lByG_TG9q>^ZFDCwmS zxZ`ZeWZR}|_e^PhTJ0C3c6=?JV3=~vNqP$N+)e^BS-U+fCtM!@qmtLI4#@}(Afj3T zMndzlp}K~HEM%dz20#1^6GprM62n_}`0jVBzwux}D;z&@KK`zH|Ifa?;MD2CzmCIs z+l!7H5mA!@=#Es)bKE*7Z89_R$APjPbTcaigE3||=N70OLel&`+|=5wccuIvK70Q~ z75%dSgM%^|a{zdMU!1LQ!x+c4&NI@?PAB@C)DQw`yP0-wf}Uw`GrV59OicDBrWF8S zK%c)Zwfve>`4LVbYeKnZ1G4iJDr|+`>;I!c}{7Q7$`zQ5*uWjr8_qKVAVp zqX8jIMB<#T=_Vsl|1ae&|E2G*pJ4=uabJrvIrGtuWOU=Sw3BNNfvD7auNR{IFb+lf z&5}T(iPZ%SsO^48K*Yl?WuqeqG)o8Fl&A|Z80pejpwD)$ZOkZA8KI@JsV+D|2BX6( zUTP^mu+Q(&za}yaMZq=BvwM%SE@e1jcxNGVnC%H!W_GUqn4xzKzxrp~elPt9nx>)0pWta8Z>T8M=t4L?6alGl2*@-@HQ@niBv>kQ(6QQ-ceAxx zG}Cb$MPF$c>SJ+BYZ5n!Qt*#)yV%L_`dT6N0ueNahTm2nf9z+CKWWx+ zF7*n&@MYaxi`h)#(1ldAMcO(39@DuH=P4O_uY{9__QDuy5;>d=&#ZlF$LS@^idN9q zWrzZFIqjyq&0=+;s4Y4YA)oAFGGhDM1nGRKU-j_dIK@VgMx+oW2di)Dp~&Dy5WvI# z3h5muN*SkWS(8J1I84K_*l(JVUIVY5I_&m5(?*O1)y5t=S6Gwac>z;=o$u?tO z`>4L_V+*{$^L9|OiO3B|!Ls``FF>`x?IWP3u)o^gmR?P%nr!L4AD)8)ZT{@fsz3X) zd#KIb{lE(*1V9V@Q|uG0>dx5ar3rhQHcX`k%^5DP%$bygN2b#Q4Y zsayzH{dx80_GTFugnxeZ>)tBj`(ZZD>QW!>RdX!r%pVVcGrBbZJX9wnRNg`r1D&FQ z@o-TzoRPEdz_040KZ6HfxA)SMeyEx}(K|~IxIYn#C{DvbTMEs@@q{>2QH9;5p*yUo z{5~f3CK3QG>n((gOOQ`ac3EhtxB1B@)hC~PqIKxGvS35~;kH(R;{vi){qHSS-}cNY zHE!69xRx|CkGd6_@nfaIkSnwU(2H@g2*Q7($hR!=`%>N6A#AdPOk4MU^k@4+@Aq8z zz2`zV+@~z_4tNyMQ5MbYtbmM}lvsw0Ely(S-kcG-cgtRZAZYadP2YyW;)+pipo!Dq zbtocF7ZDn1m9$q!g>F0h=6mXA3O0)o2yhG)eU;STy%uQFQx5_sQt^$LXz3o%)#OOP z=cF0nIeZ;5iy2nyqMjXh4WoyyW}!i~(z7VOKu$In+JO~nb0ZYJEU)p2>=t$cRv*?G zC_@T^-`;`*MWVai6Al^n8w*+(aOaT}2=3oX`xsy0~ni&WilnPTw`l0wQv2va-;zh2l~C{-wGKYj?{7rv--> z_KMgP7bHg^xjx(Kxkr@^^uyLvBTJi$7nSgOTRTEAW*OSDjb??&+DMEWV*DO$6daqP z$(=zE+pK}~#H=~Na#mtj+MofkMX!p^o)r`<24NASp0z{X;eqoUcH@9wrmW7N4U7RA z85}|j_t4#XaMlFw*t56f2ocVwLf1G-o&`g;zo`B!T|uh22HFFqe^0K?Z6{aTpUTZP z&0%53^{V3!Y9@*^81+ojel7tDXVy#brO@G)bX-th%sEIDmT>JEWC$Gc_%bQg)@XO=dtdt)#d4Mbfn{vZGp-xle{m$TmSM&mq0Y$fM2pnWH@%)ONfP}yLk%|3a;m?cXX2)R_`YCJZ3cK`B9EFzL^#lxV+WzRn z&)VhPfTBUJ!_K94y{jf(bd%rVF?i=^+W9O+Tx|j%L_I(+!$9LA`g#GhL=Ph!qz4!> zN`566NUrF!7S=OetE7HIEzwgW?-^~F5ny6vchX0$r!Y0P#gmw#%YJj+^jLU{_rpMM zA5rw3s+T?T>7kRZ6~?cD*!t3c1xDoh^uTv{6&z8>3Xe5`tRO0v{_b1$Uxokk8+!C1 zECgqf2+a|2ap8-`u9oi!+y&I1i?F}RwJrPfB`iQ7HHkH>L%vrS(yCoqhbZ$C^pI>? z+GR7RjvVTBU$D#lx%?L&M>J^Tsn>}V^6VRfZ~`^LXEI%ZE;&$xFZ$t?ot+q@QBR=W zhw!0A(*koCb&hZolPZ&kGzm6ZZh^U63Bj=?{0Dz7v;0XeaE9$dmm5sg#}iL2-33bp zpgEuz7o&Vyh2@xmsa|RsA!K4cABH(C*rp2;idvAz0+8TtJv zr;(3Pu1plWj^m9dVxWJneXYjEqgS1wR1#qGy+eESz>QKoWdv@`ZteQ%jx0~(Kgo?U zH`ySZw5go~8948vOoiCo&<5~VNV*ksrfV`APS~`cB3oGMo4*jEg!0ZhBc1CBg8~dV zs-(qfIV5;T{!H(%T>)KN$7gqftcduLc(Az|f(n=Ni3!%nyLTQSso?4=^>!Je_ecWD z!BQejvLL|VIa9mydH9{2=0|lBrlwO)jyCkuk}yRs48ahws503HG`O*Cv1{c$ZTGtO z#T_}uD$``({`Oc--rR=-vJ&rssKUY<>I@N>wm+RHEWh=K!qf5G&4rE+%KgA^a*8Xc z^Z`t=0>fa z=%DmK6`%K>sv|2+L^&Q0FlQ zsv4TqB>BzuBj=f-Ln3m3(kAh+>{$|^W3^&xX7Q{Ng}WJVrFdGmLaQ`5N`1DAfwl2? zBw1l|ltS=cdEpuS zTg;Ois=CFoML37WBTGpA&ddt>7)a+9HOW~rgs8Jy_Nlh>omw>>nDTUPtwS6fls37< zKcjL@`t}Ll2vov2x=dH`zLGNfeW=NGvFg+3g-e(>Y>`p;6Bx40Jc|a+7?)D<`3WJE zl1{ZxTh^hWIoX1+yM{#KX^6(`*cXE-5V~~TAP%HRSGBZ9#{TfY3$eP34un2mt;Q$- ze}s+v?4%>IO{!^^JC|n+4a?)s@9Qhtk1ZxVmM*kV?e&lIm*1(rlgt@$5W)fYs{SUW z6$Gc9i909HNF*k{O2Lly5P2(@yDP*lp){cF+NV%%2e@%+7} z(H#ZuC;};0AFFev!HZ^i)ml`8c=N&~E_);dv5=bwK!}hD&sc03%SPLqP5$DGmWACC z6de2-qO`vFsCqQ06P<)P>sJeRT*d^JXIV$L=K*3vX?Sib3B&m;< z0kBknHPyxM9l*Qj@p&W#9@myc!FaysX1GNy*p&;gQV4jrbfxqAa8k`Ut>o<|0gv={ zFVzv7L?frvdLp@|Ag#G7yCk&&S$Cs!yK5X+@ed0$&^o?xbPgK zM$-+1@bGoT;eYJT{gfR(KrBeD_Iu^tP%v~z;i8QOZ+#~Vt)*MhmIz&2!Q<_-Fu+CU z{H1Hp;9~@2&~j38lj1~Ppg|khHTzgm*YLzDJE{~t{`}*=z5hXabb4EG->Y|4S2KA` z;inH%3`*wuRPs0%I`s<>;x&3Hdm}4`LZQcQ#8Hrr-EUxb17b_X>i6vgpsRRPF)rPPeS5)7&9k#n1ZntT z^|~b<+)tQv*pl3XAx&Al@!z$i{@|aJaekp@QZMLr!khjZ%d#NKSCeDo(fj5f{1?jQ z-_UT|vi7LG?5iqrG?wDAP#)WMrEQpHmJKEPL#KFj7@a4(F_rf-5tFqxN7aK@qk|fJ zKFW{6Ufmy)k_k%>_3UXrT!MjqBSFA(9cGTWPle|edZ3cH4tweJ43;sYB#(5;k*YLa z9K)pB3zYrzF&1SGoh^fjT^Z4+;Gp~P!!DdhQ#U0>zBlcoO?*S(l3g9u(P!=8Qb%kUjVw5U|HcY(;u8g2G`ZYM%^K`qCtMlY2byz#BU4fR7cXIeK#Na;rz@y3CJgFOvFI`SFqdCdKU{JES~k#93lO zGc9C7114cq{ICKE)z2F0=N1B-U%&TlXqZi>Ua7!UnvHk`IlcA4*DO7d>c@~p=SRq=n zMLuLq?byUWC&@>>3{{$>I8aFA$XmjH>|3D#<51f7ze}GY)2Zo)ocSEnrkxNU5IwEs1J6IcKkGfRg=vdvA!9u`ss}CRku6qAl z4d;Fqex&Js<00w;BE)jkL+!#*2?2W0xWW!?bmN$3(&BQH?>1M<Te`#-yRlowR@6fYRm0R+qdG3`}@_}b@gV?Uj`swtH*k{;}55-w<Q^0ZU4tKhF!y^826V?|v&?C@$XcQtF;FQNoMt)xdBL z;VEq0d)}_>v~QGeKw+F#*#+2IQnWKBo0@`ppwMsdV)39Z6AJAef&KKXU`SLYxm3s%4?{@^^a)6oJG})Qb$n( z|6ju1ZAX&qx)OZ%uP|zamZ%xF-qk{@f22W}o0+>=xZTPw?&0wupvnFK=}kRPQjtYc zB1KZXv`AKo1TvHVHG8eI*FF|C7>oq4va&KG!p+Xv*L87OVNXmL>o)tM__=|}DK&|1 z_?kc`_H+Oz!DkB|!s(`A*X4w{vWC%*PKkYJ>;*=w8TcwdH}b}>nW?S z48V1JKg!y}i#fW{575iw!Y0rHYuLA2<8|(I)RNbWo*(2(?4Akk(pwF__T@p?5X$ga ztVtTg{b-WQ*r;#z@M;i!vcrAg?hvL{t8W#e-lUk4~Wq+cijuWP-Rzz8Odhw>J^B?ZY zmbC0=XkaxW8IQ33>B%RhxZ|v9u@}iwfgya|h?!gC)>A93ebMCg(GCVk(p&+rt4w6fKnYQq6w^jOPM>3ZMoc5~Jr)Ni+80 z|75jbjbgazyEmTE7RY5!cY^;4H;JZp%*3k7Sf7NoRP(s}QZ{&Gg2VgvwnEg+Kxw79Ed-2e#D7PX5pgf(3H zOi52$Y6sd@TV=e?pwnqFQ(G6(c>{JoXny@%*_!NlX`BQmRpGMOHcLsgQxPxg(X1G# z9zdK2FQt2IwrPjl1hOz3=&}IPi>L_5WblH@CYe>U5Gbv1T3BkZ+Ub#p=y!Yb>jQWL zbSR2X#S1E*VSu<{%#^#NV=&KIl)i_D!5L`8CQ~oA$lZB#rZxvQK&$H(Z!ftnK#*F;3>uuQZwHgPcMW2H8S@ z+mRkoqnFr2ooa`0nTRLJ7Jli*6)6p9zrzK*PSEehTmaXYd|_O{Q_b$O(o9& zy8s;paxhzL8WBuSRiMDJu7N-D5b z-(fr%yBv!K-cMVdCrMN!+7Uh}eW>L_W8&-TQ%RWcbRPV}cO&O(dYh93Py4O`L~YcM z1jEH}Rl?I459IhTq`m?21@`TKfMyBIQOG(3odmjvN2-C9h^QZ9NbuGKs&Z+P(4#LZ z;`yALJcvs=)jp8HWX;`# zrTXsVp(|651~Rxaid?-v`fPWOxS`d7uTz0C?vgz5;u-=pRK}QX)f;ZIBIh)|v*3mg z?@gnVgB}3HuZ}P_7q?xtxOa`Lm%z4zjGDefs-yivHdi~R8D)F7kAOlnFA6?Qz)Kuxt^!lyUXqY27qJzGiALaBpNs>s;{4Mz^tcfa%VDe$|lt@nQnCJoMke%maOg=Kw7$^t&&y%_A8VNlC55X zRIujQVEOM-K0|x|Ws@#QR3f6-K zDWK*sI1D+70gXX^C?&M+{Z;GlV^)IgyG&*79AlzGP$)lJRkom9C1Uim)cr4uzx>PeOs~w5p=J(-39yl;=M<_QgreXSctg318Nd|;)!uUJr*Z+PXxpC)5qB3 z(&tTpeR#*Ak6k!nJ(}!L4_WD~Q*r@M!#*1tI!V8jV$LPq%@fJ6mYn?nh40&{kj-Q% z+!I}j(~E(zWme6A>UREkUlpHz8&}&();2I6AV($vYb#Yk+NkU6;-ZZh`eiJiR^3PR zlcCYS&-I};5H;Y-q#M=ky4}}e_nzS<_$(N^rOuG`acGC&-@F&KQ({`Eub-!9M2}e` zACMq6Pg5u}Z*GJnI}~IDg?#9Jf4v~LL*o;4JGGzfomY8F9wAl-^34^17a!~GOC%(2 z$+0hKln3h$HE2~tbA>(2>GidEJ;*8jPAoS2#fcVjDS+}d3(*D)67(BG7T%h=4pnc0 zW?nnH4HuVJB7=t79J}NI4h@y_Ie>BY755W8SPwF_qu31KzD~ih zBhT;JU~l2oa$uxdhE1#l$HNrrXBf&sJc6K3X{KG4GG>ujfk?5%rw&nh7Uz)v0|y7V zn48`LN0iYyWQR5jT-)hW0$<4&UTf)gy^=Xy;=cywlJfhK@E&c(@Pp;JlTD-a<&y}<&4m) z5$q1WeFqeRJrsHezZ7zNw<0ITGqo3qGupANw?*Sw=m$|XaFaikMc|P%TnSHR(J#Z;idX!A#lICVzZ0&5ubfMI0$-v$I;dh z2>PwXIRpkkP3)bvAA*G$jqO(@q0&|Bci80J560xF2!~-z^54y4qZP2X^}?>s`&JG! zEL8n@@#jbnR=rq)Na&al`@h2|m+n!U+6KEOJ9s_Htv4a_FJgUMaAp)G$sN$D6hfTa z?#)R}8w};aLDPP#nwrphNd9}lq~a<+{pIN~S^_9T3+|mRRWOU|Vs`XZ^l@|Xa1Es$ zmQldtDQ?)Ax&gV{$3Q+U#$pB1OaRP<>o)bPYKvVjOZNx^ns#imwg5I1Q*a!5csq)0 zxN*)CUz!6T%!=k2BjxoSf8!St}M;~er8#M8y`ruuCB z+W_e`s2+!oQ3)SVB3zyXN_LDfA*>w}E1ECKh6-JHa2@0=LY+mSIR;(#sK81YQSV4J9 zT?r-MhiRRaF-Zr^+6&i!E63ZiTf7IW8>~S8931a>Fh*bw_z?=rrg(;g7(MJ#1qx6Hm>{=Yr&sI_&$*nv6auOiF zr8=NpOGx{rXMmP-JbMBME8?e0y~XR)^hk;laba=IysRum41{LVna@OXe4yp&vdw=heO{eUPs-`xJzh! zID88d+;D`Dtg!{aM)uQ}$*JvI{(psjfvW-xoKFF694Rji>C@3&miB)I{>SQzR72Xc zSTd@7ZMdg{+|*Sy#ztFAITwu~I60#|DX+p&|4%!i5DMrg2jNJmlapS=$szE)S&39* zU!Wwyhj|7)PHMZcYI(Sy^ze>Zh0aEQ_DXqKqBARaKiZc$z0>1v;5pTB@fI!iIyVh| z>>wp_EFvE>>a7P<=Kx+FyIXN1yq*R7g~Di3$GTwusogR#aen5BwaR*fD4ugEn^r$T z=g4l^_6R(Za|#I8-{Q|avUa__C?6}nTkCmo@C9^&XSAH5Rf94q)BSfuvtLD#VPDo4e6J|BEle9Nsb?Z`^ z7BsAxOYZZ~W)Z=xv7V!Rlv(PaFO-SC?MCk}=uoY9vpX50QSKL9AO0nbi(k9k!q>18 zyNYB9s6!q0 zn_>qM)+8V-dO8YG^iYNU_^%kF;Q*vG;oO4s5%26BEPY9GW_Nf+ z*-8e~ZtI6_uCOlscIYeGn3@8nC2~_keQ77Z zJ|?QHWRe6;M>QgbcEk*deV2{Sp*7rN59yoE5-JgEDS1PP+urQ%ju$A@Dw_I{PBmMqYj6yS&Q=QEl06#?IZkB zD*@;*>yZRT7L;2V(g)sUSdxyG;iGE((vKNNR&UvvJ{esyxwNDe*zrn&@Qu$$>`ltX zU@qU`Iye&c`gDELjkM|gJW(*6zuLtnF8eF0&aL%W(H+=(^c7~(0%Ssj4w>@9E_wq&s(oerB(m&|iy$iNAXw3!Zl8i*nWGxHYXl;N=@J||8L#T0phh(uk!_C@# zMCqz{_7<*2&*E;E%>r{uf-kJ&o2N#_U3RgxOW?KDL-^UTc&y1izQ7zYD_&D=&yJq< z)gAacUqi)EQpLCJA%Tr=FX?Vqbb%_yDM?GGQ!%n5Aq6Nl(cEBd5Tjm0Vv$=1DmtNz z=!PUsyFhJjIS8LF4QikIkYQTOpbvPeEeD_z`*J%^Ghnx{ztMkc%k0csmlAsVwH&qT zKpQ}LIZyqd#L`jx&l_Z7N77PTYGVDtDzGo43^a5hhIyT8yOJ6%Ama9i+SWpQ;l3>E zl-$`Iw1I9h5Hb+r@T)a3r-Qr9n6U3v*JMUD(YFar zjT#O(B&vKsI@ERkVc1Dq#vik3gI0H zDVYUZD)N68U;hnkUQ_$*hhxSY2eot>^KKQ>{DornBD{3xf)*bZPSCoCco;5wj8)gr z%B=RRzCgMVAR!xd6jhKswNxLJtmtR5B1m2_m6SGtFQA0rXmi8XK#*xI&nnE3|DXIla|;(Y#txT{5)7X<(Al*R!XD3%urpj6;< ztx{aI4jJy*<#6sI^I;e&`_1eq4C#5t7gp{B(c(*;r&xltO?U`a3?d%SmTa5_#g=Z% zrg^k9_r1AAm`4&Z5tIx_%Z1geSVaSZDSv z1>-TjbMj-Z5Tp!4V1=t6`K>!2ww{P6OXt^Pz#s$zdr$6GvR*eV*^kOYl*8F9fDRop zXt)Bwh7C5viCTV`Zv9dQhEmXgy;NqYzL5?`nc zUm&;kZjbwO^lj%*E(CDO%56V|bsXsp?*>kToSjR+@!%$fU{g1TUwRB`8Qah0VmY(g zb#%hK403^q{=acQ zDS3C&*n*}s=i!GPUA2Pt)h|mId!%azs8oUOTN&{jLZsbzkAQ}#k>WWKN*YUZxe&_F zQ||L=h8kLOG<@R9Q#x!G&nW=o@Axl@Z@I8-Rtww_D`Y+U=xWVZR%%|QAz}8v>!@># zu3r+3(==zr${mX^AJ>B;;uVDTwTW{rP7ogq!2^S%@V;AK5 z&iQYc!XXb1bX7|+ooSD7D(c<+ z;3U~Qfnqff>r0ph`Is6@irMSM2d)Dc>don!J}V>es#W7Z<|KMZo(`V(?^y@C0`*eg zWNz%xONs`)DR%echOLmOL<@F9nw+&is-~-hr;6K9#C$lg zt@A;`vMWM!8lBUmEeLjy(?WwMsNRar(oXjCa1mr*Zm{Yr2h5~Pp&2Rq=UZl@VRCK% zCUajeV^t6hcCape32klk?jQdigK)?ds0DIywe-b84Ft_YlPxdS0}dgT>e(-9qt5M2 z#p2c9`R4QcCFoEC%#{SK+CW*J8^z1z)?{D+0Rg@d$Ivj*XCIUta9jkmz%MT4l1eum z6FP|}6mmpStGJ=JB5yzmvU84zfo zVou<|Pnjl~H=Rn-KRIQKNO(JycR0Se5;5keJg|CLHCGOffI)w0=}-ijP{M~&4!J9S z42#*jQ>eWC8&c!~+RC`)U8&^&De#Z*Y&wU|SW*L6n<2!Uhv&h61VfOdgro8NQ<8NZ}NTRl9>4qfQ-k_e@Z1o+6y!H}!FB1c~JoUuNs4#r9L z7XV)>vf4YC))F(t;JDExfmtSGqKJE4_v@8I4tYbAg|z#v{I^h0Lv7mYPb-~91mDB9 z51whlnFHgXxlKiif&k>LBmleYGJM^$c6jjKTWc)OQe*@8WxeDPK4-M=5UL-(iuYTT zbFxsjM#8>G2dl?xzPq01exf}0!vu-xsuMhn!_^hIaj ze13Y;^P?Nzk?1uE$fvvXNU$ffh?TwljQ8S=&3AfUfs=?jd zPeaD?m-JflAIl1pjWYoIh$Im^f^^2G2?ABJJV{NUg?;DT0*U3Hw-r+&u_K##!=&|8{i4wguAYKJmc3-2PFP1%pF~pb9cCMz}yRZJA z6+I=|hx!*gDT%FNHD-DKIW32Q5mBaK*!kxiItWt=`n*;YEA}Y6X1`5FM5UohWS}SZ z%4>Y(^bsuO{!KpjG%xPt4`gTN5*^4alb6&)TC%lv<9o#)zvB?B1M3YBdTox_ff%s) zBP4CD+PEiCX-3aCl!cy-%eQ7ZTif%Z1O3`|M|2mppS@`%09Q+u8|?LPIQpja83U^cvooWB$tZmUPTiH(@kfitcZf;LVe?jNmp}#&&`IN&0l;3IH)2mL!1! zEG6o&hkNOkr%_uPC{mImdkAWzak^&#D3DM{V4GXOO@3>&Ea4_EAru5Dt|p?*O$G!6 ziTP=<79g$#x20A7p5U%yH~5p{)4xJ6Yg9AvlDxO2HOOsjM+?YA@(Ut5(BmQg)}h;# zF%bq;P@}U`e<>c4j26-qq>@CQ>SFLY2L%B8gA9naBGmH(2Yf%`xMCz9vTB#a@E+jA z^dlC|xZf2Yu@br`6ikD)Xj)&N}r||FjP*Nen+|4VvQaPFF?8!dsHXV+Rg1Fo2cS%9_nfyhnJ6%ShWWIEqjTxmIz zTSrCY{HJ*o&mENR({mg0AF7w^l6K7xGpFkZ^R=e>n|flGUcB-X9y7s{6@H|mlu z5@5>eL}cSeP`o!sQUtsE+0!g`tR6l3G~Dja1o-*G1&aCo++uX_u3%xgwXI4_1oT19 zOBtrVR*v5CC;HW=-wK-fTmOmFJ0=pK9~kcKM#)l7+{|w<9tEQ`#Cz?bRvD*h57LXU z;?fZ`wUw2XKGngm@7m-*Hi|DWG+a>WH9Cv03j6PWQe3S?RLO#yGf1!25?jno%+H9q|bt&ovjsz0)3vhvK4AR^G zUznspr42UQ<2t6&XL7)W-ez1DX)wYk}0gC^<-mR9rblS|4C`gj6K#rrfY&;3Vx!<>Ps z4BRJ6HYdgw)HiM8wbjd%i}ImtviZt*baiZSC;ACuFtS&V!*-co!!{K<9HP?mbm(dK z(>ofR1+Y~F)-o1C_QS22EZq7y3IK$Mrfj9;1c#(;uFVqmgnqVE|2|@db~{KA!)s~s z0UrjHcS4)YTJPR3BTz zM@+aag~^+OSJSgrJjnU%#azdcYk?VBlUCH$C1p<~*_*EF^`W7zC3ru&_Eewb@P!`x zOr-!~W`Y5Nj;9OhhogZ5A{MIpGhTVEmCd0Lz+hu zD%lBNh!dO8B9fu)7L2jgsZ#3%^oaFDa_ndk5WM`nWZ94U^{~74fJgUb6vWoWKqN~B zEv}`4&^E#bk4j~Nmq72{n-lxegn`GEO#op1j0=?1yUGnfJvY|$*}HU2pBY%)0Tu9b znXr``MmsQq603k;DCB}^R4h9SG{8~4Pa&6^M(~;K2fR?D@|Es2r|>a37{dH%@u!xo z78;4BA3JGWfb-*V2ZvqYu0trU6U8N4!gdtul|&iFGiQ&1)Fz=y` zrDf@G&=3Fa7ui3;3jAB7#tprMZ=6C%ZJ;>VnjJ+S(GguRdeN)I+{LdN@jltRtfNe{ z3u*5`T%5u4R2d(0|7|(w3ap`@4s=%)q6kTn=CTl{Y2~zMxdHB2C1#z<>zg-KaElRO zm&w8tWKADeep)ZnH)N`EkO@YPu=ZLiOViGP#NXIrDhlPYRu1bVk*-rf9qeEMp#e;t z{~p|xRGokr549GIi(($!c4A|P8(|rSXNS=pnJfW=w#S?#{UuX;4|+@$=p$^xmMmSh zD3Am_l!M^9IR@}YckmS?Gfs0f`&iC->TX4drUU`qzf1oNW1q7ezU=BxF~Z9u%jDsC zwTgGw$Ecus5yf07a0CpR-`TW+qp^8pup+`?XwaVsYXV@lNPYE_U32bAA)w|RmF{8< ztmVNnKoi?@ey3_V9p}9rjN{;36(ajRcg@i?^+nqCigU45P?KC@yiEzTI=?A@x{!kq zO{4)-a18WK7k|oW|NeFkt*9GMb-9b6)ftWG1fBrQx0Se;Ci?JUYL+l(E_YHg6GJC6 z$B;6kFaTnraWLg8cVOZCaxUuA(l%5|SX^i?q;K2IXCW>QbPXUXLo=Z(xL9B*x4V^sx?xu>C3FCRGXF&YV$ z&)JBu2o0AD-2&2gwZg*%y{*_;0uq71wKU1#(D8OtfFeiq5<#JY!)Yu9&Q?^^h{!Ku z0@S4n3J8y`(KJ9SR;grVB>sCx-3=3T2M?lW@YE%>b}}+WZ-u}`AYdKb%qL9AMRX4n zS3-(L>b%ESyaumGB02w#NagTAYd9xYWdq44*Jp`@vnIBb7M_@WQRa;bcUCFa8K(`q z4WH{^^&c+mvZYducoG9Q9g72;E;+(jyg7%UCWT)R zj8p@zE$jC)OR%W==fq_4)jkmOLCL1`AX@XMzQ?P-1bamT*h#>$Y=Vvy0{%9ysA**J zYM~`#EsPV6;TBzw>1|dhM zZ7=3sZ#13@t^cu^QqRAF5?ZQn1gtM*1beb;y0ahu1zjpQvQMQZw^ZTkOmlz<*1?(3 z0>jlpMC^)72|KT~rLDc|51M5tjl&(+Cwxz=P84it66-%JQX0-`VoryfBj&PIrze0N za5k~F7bNwjPV(wjnxyS@K48Cts_Y{BL9|@YgMY7o1yaSl7Me)wlm0_Ym@Zvx!S)OX zJvXJKqW}k7br|kb1oM*l@qJ}Ib8A7<9mb(V;)Ahjdl7#?WLra z1qzX(A%GB07cZdGEg&H8jxlmS0J1;)Q1jKRC5}r53Rvm7?T7K6y%m~y2CyZ`!lv^~ zF|m_tKxhEdHU5J^&jEL++RvxIF8&${$MQ8^w+qkq;86*_%YZuxt3N>*JLasn&|=z( z*LU(1;!>7t%exsybHyrqUr23fn?iZ0TBCdXeA&V5H*g}~x=0OBw@S|xKj}dGXt#mD z7wt|^L%_k&{(Rk9(e3Qdf`WGGcqcZsM_Q`KTO9UL~e~8d5^jfxEoXrK>w$RhQ`wLt%ynWHSeQu z=uGFS3wP26#2KVqXxzY&VovCB3kWoZ@8KaPGX(o-oY&VMc259xa(HBANK>@W{oN#m zZa{uukg953lVFE6L2sG+ov?RJo~*Q2o`FlvkB=rO3sw`NjUe8hmkt)}oyg`6A+5pn#ei99Cmn?K{@Yfb8e1G@eNfQ3>EHzfXwC;^n z5a}mACB0t+=(OrSJ*jfJ?g#uEboqL_i5hdD&y*B{eLX8rxQ=lYgvAvV#p@$zpZ0go zZ1^!tYLj#`5Z8z0119oG9b>Xvq)qjQcOE=&y0W%()eGGh%Ioy9cvmg9MfJTjFBz@m zdc&b~?MHA4$4jsh)K$Q>ei1DVd~pa_^PE#fmEwGY@PmhXIMUmspMzl&E(0R>L{R7f zf8+)GeVC0wdI2V7+JfYL zy0jv7%~n?Zgt?-78164!7E-TDLP)%gSR>dI;SdAcvWji}wuItH4^p|n-$BpU2QL=+ zh$lM6S(FNdiYm*aPl0u9q~_@?p8OQAK$)yq!3l(CrNMEcSH8*H01zZer=r@-&fp>V z#UC;@S)au=nnGe)?!CH(n7>86;{k!8(c;9uNO~AIh9}ER;DiLkMA})fLnI@GM%O~y z!ko-gS2iK-;$=`@p?HL1Mtx-8ZDsM2e)r%w=j}kT-IMo_w#Or+Q!NbSN`enE{nu%~ zV<+Q`MyqR}YEVd;ZYXY=^J-ee0dGd`wlXAjKRPe+MxD}L%Y%R&ujskT5F1Vkoyl%A{nv?ePsj0v>-tKI zy12A2kWO*bkCdWh88UYTq3oY@jw6_-EmFlTu%$jygBr?aonGfOved@UADXS@&*d@0v ztuZ~FU@M+EyR|ViQsJiUmycBW zn#n>O(t%uK^|NrcP~Y%|ankily(gr|Q(vOYvolpHhSI@w^P}BPQ)qf($>0#M!9Lfc zsUeIW>RO+{V7m@b$+wC>^)pJHCo%u_B1m&q@sNZT)QO-z!T8JXMV2;!N<#Xtc81+A z9;4JW(#&_|(e!yA0rW;w$^>)>HcUqG$6cK#Hn-d#|6(R}C@BpPF{y2t*>*nYOT)LI(&-94wgc( zN>9L2-`eztbtUIdp31D8uVH4KzHEZBq1w*~5gig0kDxv0&A?;9zTZMq=bx($e|E2K zu5Iz$g`lL#Nx7uMRn}B}O_#o5(Dv^LeXbnI@a^-nnfob*y}PBui~l|yGJ`m}dzHVH z2ZHk>R{g+Hi;0?AYAAP->V3Kha6%@w2LIx0z7*8O5Cm#`6p`k*EgG6udZFB!U=@u2 zLJ^?`pS2K>1|y7$zEgVA79-}^)+5I?-Y>pUH#RSa${rNea?4B$NG=B-)t14!JO zO3xdW<3J!kYJWaGiMF$vZlj5h4|P#bkg@ZGm`~^`|J+Y{8}-~vS;RO#cGaI1on~f3 z>_pbX9)inqoxf7!_~7eNjN3mpO>%YDKNna2jYaAu2Dq&h#?L?*C@Y`)B8wQ@fSh_F zlnXQir5CABB6)3ZN6FAex9tEuloatt67td3A%Qcv; zI?5Y@T}(=?k;(%~iWj9H``LvS4r@P|;VjmAhHmw`VfCoOqo>bD9|D6JgXc=)u^al5 zFkPP-VcOgc=;51;+lILL92oO6G*#0Jw&LZPH7tQLTj@epeEQdPj4TJ9u467bFU#%? zmBiyXSVWnBha^qkR0+8~ojiNxdX;1LH3C`{ex;Q4T+r~hXawgnjoOgGH{1m)x3*=< zv{Dz@Js27`&n8C2)q6c;yQpg1Es)ul{rHa`z9>HZbNU%@dwi_rlrNt~_w_q{U-iM~ zrcILswXY}*Q#}5wZ@F1C2--QWatDNS@!6h@g|QO8(IL`0krD*(5Zt$S0Fa&3x{@8% z)mpPzVpAnIn`*?l5s_ z9WEw`qMyT9sw`eR>_%!#$ns`?Tk>#WjXf&mG($oFaPeAEiGfya^IsHy@fUyL@6dz) zP!-qZ^yvr1#~*{t)yM$vFgeOfzka=*C_NEGHS+;Do%~8o;+f%#7~0 zg!OoGxK1^)bq5-{$WANaHy}qBgBttOtlmAgZBs5YVSV6(Ey|}ajp*DzaEFHvze}$V zg@#zKc(i$5|2tsTEgszAfjwpqm_Z#p#I&GK_;@IE_zT(j&;?XX#G^j(wPm z1D?fcFS=#)hi=~5Jd$q$hyKBHSAx{M9O0tE>2gy$bU-4@?gzK?vcDdD=yu>nr?cj_ zm~cCeu%C{NjO1|q6y3`z<}w}U4tVaDq?3Rlp)aPut;ovVl9A>VM$5ggPV5;&U9Rph z6%PzZgSG~mTG6U4lLgb=g94{H#Lo^Qe8PeJ8WcUc`at#4p1^K> zST#vjpf9Y{BBwp|{JQ|En9Y3-KPlwK5UVVY4z5=G?Opm5N2;<(O zbOQ_p*u{!f>?t^?3D@dphweYrSrm@@XT^tKr9)Rq^~OGCvfcK0j2W$HkuFw)R&+KMHKXL<|MQF46y+r_uPty_JN-UMY6tCY~eM#1*FQ6k@J2(TUO4@ywyA(nEDC9l~?YK5Ppb=QR4?Q4hjAt#f^S%NEzT$GVv;(=7 zUYb|IlbwPSL+#Vp_60=r+#GOz>wwiLHWyY%L9n*c+frhHBCjg7xsM`%SxYr>uby@q zo2+yIaz@w%)MfEm`u68S3SB);6af}C8xPKZ27Ugw~P;}Nf1l|3@M-zCR$0uC970jx`O~l4ZSWxq-delkY($4AFi@i zs6As$K-905u|bVYn8NsY4^+U)+)wWAwX%<%T2lj1>Mr}3roe~M^york%5BxO%+qqL zm+7zV6W3nWY1AgjfXkRL9fa$^-c}0*lM+mPM~fO75Z$CBWv#ntH$wKUhbO$ciyygrDpu(>#KX)s-r z>uMRvCWr2}31_|JV+;`}W%e=~2|?lugkVAS|D^bnKS|k78*$)fp-7q59Pn#!2QobW zOo}}Fap4U2JRL=Q&?BcH-)&K!uR=^vA5jbp2U1b56O`6u5fOrwSOZcjvQKWcdY}yv znm{uynbWs&D{mKr-dBs2D+`78fxe$;oy`5$BwQ3d7wrS!SWJDc0;bd`DqOiN0xCPlHgOcB5C6+>PWJP(Hi+E@q53+rnjqj9?^##n<$`@eiY7T2M zd+LUn6-rnY8DNm=?&7qSOpp`UN*0eOy&d_$V~iCLka`s)z)E_%YS{G&mD2%N*@SW* zRZaSGCzZ3&nZmXhTVoxIOsXdslHr-M9??kd7KS8VH6}rP!AnCU(P4rRdrzlqAqK{4 z1_*_gl4RHDmrBK42wnjIKPb*l6W36({Hu?%T~J^9CX*=8NI#Rva0e`Jakm1lo!o;E zTLE_mnta(xLmsbrNVg9W8bEb4=z?#b$QBL+Ql?sNQT>69mLp${b!OtMxY7x`qYpL$G({?P|0}14THtZu0}DE@KKdqu_!QB9 z>;^~IZgJM~lofb5JhBt*x?@~cX>w>un(CT<4eZBx8rd3ta>SMWClFB4fwnm0v{;j^@j|*yN4#>m;GS^Sh-_Qgi66@q!?2L8Ra>ITKAF!K!ZLJ zE3>4BbugtofO_BOAz?1se zPKptG*^4)aZCY=x$pWU7H~W-#zhsg%z0e^pqy`8hcpMa}9@^HDGIa2y>OpS4r=$Pz zC!c=w>8Brm4D`bfxNnBQq4$FKA~b&kDcIWsfbprNN$>SA)V9FCX~fs>@8pcLTnU*a zfQ7jX3f%y-6W^h34_NAiw!rQV%#O)K5LP>ZLnEpw-=B9P(4M-H4gDe~7{C<)R>flC zU3~v#@!@AD{@8JI`&6E(A60b*+jFpW>>rT5>Vp=pk-*wecJeAuOYD$6US?EKv$6CF zkk0^94@F}d##HpA(LviaBIgNppW3$RA@87t2Q;qU^a^fMnv8%1R2~$U&`()|)?w7% z1oxq%e=!hrE;m$L*@*=VHeL18E^9I?Iwr(VKR>+z?7dPpv8SI4iBm{f$4ni8NdyKPU#Tb;g0BBH!8qPzik+l56ZCrZ;&d z&g<^6g~I-A`Q}@oq7OO;^m<_O+-mYBCu_vCY=l>gE=;lm8|0%^D7h;*j(C!ReK@0= zb$Ej&+c#@14;fppPE-OrD)zK@4PWiskZ$8)RR?ouRusaWW$yOqnN+nM(0@xwfsfVR zRKr)J6LRDLTD{*Nxlt(EXH*!%jbT>{&{!6ceTp zz6QTQjlb$Z)@=9F*#W$_LBUS&WO`?Gozy47Ft9H1b5r;AT$U z-$`vn08Lt2fBYYXet{v*cq8F$DPB;R*t64dy5grUF>Rs#Y1aeAS-n$WKfM)uXZG<3 zC6CqM%hZl3yMbL@ZOxJ#;k9r=*7eXCrmm``Ni+20v5iUf0`|rU&FK;Oa!_aA!pPQ| ze6VE~;oQSobcM5(PrAbn?S|F$1)O2anYRm(uJVOWm-`UT$U~z z$uJ3;g~x^o=z|#g@FvUB&KSHm9>?ry&3bv|Ou=QDQg-77Sj^|Mcs_kU@Q)yaq&@E& zrhz%URb^iRaUT{clSC#aIsAdOEk}Tbnz_>OBji|IG)i(;U*{s|-gBkq&0Sf%*h>V$ zBmKY^yt%i8^>#5x%qn0gUNMyi@%qyrU~mZGnL}Xsg;gbfnexWv35RosS9jr_P~L*Y z1>C{;#1&|YrL-#`c79V%tdbMM71Z`#iaQwQgOiULmsI)eObF6EyEEp3N#2N$ej`K` zD~PAclwkk|4*x|o&!j=55Bl8?Ulkawfr|`T2rMn002SO0Z&+0lH5WeolO0WbuR2aA zz$Y3Klve0AI>6avGS^(aKvz^MNdNMmi-fUUCh#I;%XZC9vf?#}E!*`_m=Aq0^v-u+ zcj(oee(%qUZ|R(!E9hsIM+p||(#pUXC%DN*>2hp=b6B?YY)$T_GWbR(6n`?+fkR&d zK)tUmCC|&$xW#}S`;l_COoZ)#mmRSENyqtKtSVwaWsL(eP*5qr9NCxVkzrizwa_)fl(t zx6mS)8rZ5@khb%@8pN!524cCB{+aib-|LRQ5U*8mE>(AgaXDS^;m$PMpv zi=vkbYFV2Y{<0k5>GRbth~kaN6mMyE2y(!BqOjEM z@Cciljv%*aeU@tgXc`19D@ULS8N}bCC-jnh+dZ3#w7n5!0U__90p85pwiY7m>F*6q zk~8-^Gu6eV44pV4K&3MbaMS4<#W&N^l``_W*-6V^)xbb^JY~jP@suR_my#s^56=jQ zn0|PdtX-Lm3$e+%adfk27BTBNK6gFZwg@lJH}*_0{a#51krGL7rIX!oL60yJJ7w$X z4RY0%8Uv2CYV;zL z=wWF7sIfnD0jrCU9@)Y%m(R5<+OK7 zob}_)IVHgBaK&{_)0Ch@zEIh%naw>RfRxYV2o$KY8>;33Yd?*-Li1kMPoc{h&Li^_ zrnKWk^9GDo{2GA+Uvaa_MJWVjV~e*8P& zAUdLOoL2VFqLp#}hJ8G!(q0l+5id7K@P*MeG&rH2&yjl(AC_VXM_dD7Xmq-bTAyQ`q=u<7^4hYPUHX zw{^uZVVBj#D8thK6$0W1H^UY9t!I_=p0C-prBZi4@6AGOFQ0B>5ZYiQvE;{?p(c=7 zl%^JlLNi2)uxa3ux1TnIGBCEM>`pX@^A4Z(jxO1hGV+4phsB34xZVs>HcT+{J_AnT z9+dn2RS+!8r9-2t@gHD22{`d)eV-)te2p1I@?K&YyI@w0Ya=o(XTuD@L4pN%DTAaE z8jn^KU>a3sLacqN5$J1GSPb>JI|24g9)6b)fI+uWxjdopb{AUIVGt>#&Yu6XL+l)!6(!Trw^UqFeIDH~R{-`op@myaG#PURmpzL#69I^Mpl%UX zA2*O1wx$y?6)32cdIu9HsFKobE?ZtL+fA*7>7*`-YN4_s4_A>btv+FRQ}}~j-{nU=WjPtE;?uXj#&jeV zV@q~Tv*<}VvF-5LJHV!f{3;dfOE9hk@wb?h_l{`<3IQjeku9>U=%}J0pe*CTj^;~C zL%tFGy?=$GyfDL|-HfN0v}>^E#Wl6~x5SnN!$=>K{qcV&J`Ir`L4&PMAOG*-!#~r~ zJr>QsJ%JK(4{r1fIA*wSTN6sz?={}orVtM8ehf(L+enipPpqw=Ksm74H7(Qs7x*84 z_;xV-3+^DGJx`Rx&6|BKC9vFcpuj=e>arj{SE22H=6YMndSWKSOwjkTGem18-b8aK z;mb9TfrGj|@CZ|L1J&$Jxf1}t^bj8p-`>pq9@616HX=_e&>q zS#SlTHg-}2?9uE=b@E`iA|MBS0yGhvKAJ&=)*nKGnvXJTxIoYoG=$kxGd$ku4<*j% zi%I~DwRMTk#S82pM$3(37agPX6MrUL>aY2`AXI=woc_pBO%Q&vvpR_~gjFxNKkAuQ z(SFKH>a3!4Q5}i#-6KRo>=X3e!FUD*+#+5AOxaXSfBc_z@Ew02`%Y8}xryX)BhBq) ziWcv2FmeF;Shh3|OQgh`r6>YgUBH4VrWXqj^pJIQ>FX}_nX2E}LVf+-}e zufi2x``mcDH3ln4|9Ic~^nHve5*;+bL#X=Tb2}QH2$~#^6_t1$85QX3&XNhRV9%_s z4Z%ylJ(uFOb5<67xoCZr^9hBb^T|N;@EeVIHhSczPpzM8U@`ihzgVH92ws`L86k!m zOwqBP)3z9ciL8y7j80Am`o`BI=2sMjvFrDoM1QhQ7kb58Q8_LF3;9r@7!km)!QUXs zqpH*QI3HB)0nvD}dm0>Qwfgh6&sOeyKGyDXSYK1+1`#MjH#R!Jh(Wz(H zNGw!0r-Vccl{u@LN5kC~mlHQLhn)`XxcM(YHrNR{r3N@HLdU(;zv?DCi6&$8T&q~e^gr90XYoT^!hX)NIp zYmd$i8~(~|{Opc@u1PAMqJeylf`n*;A@MF!$DpHSF3~=nv0*aOT-H=0U=KuBW<5B6qbBIl7x-W zZ=EE}h7RhePk@}#CEQ_ZaE@nAx3!v(W*?_mtzPE|SP=F~2nZ{-&-U%H5wx-f!$5gm zQ3(rGpw>aDx}f|gkCxNtI}YQ0L;z7zY?}}6$z0(9xS9X{;jgbDW-@>WP{}Ob0s*(Z zl1&Kuml0I1bTvI=q{=jEsF0(Y2~Znq$|P>epNnjRy^@%ao^5&5p_s5qpGN0q7g>N)F(#L*{~Pv=>_R zq1{FGuB`^BDxiavw>C3^(gy4g{$pUDXWwq=^s*tq(suS1HsFTR(+5=|aNos2CO3{5 zFc5hruuT$aycC1OE;7he!T&^E?`;xsV;&}Vg&{;dWhy>s%UR`x5qu@alhSM z-w&Qyj(U!-y;!ph-jhWs_RqDbDxI>*;$&IvSt^L6a0rD0u0Bd%+-^-D(-(}(7r_zs zAjYFBZ*?QriSeF%@)(@@;`qD8hhK54F!g(mNOoOA&v+$0W?|%P)6OcKJ1wD01krw( z`sy?^$v=j!K8EEeVXY%xMTVk6|H!&wh6sm5Y_^9F#@%<__^wH2c}}seEJYJxPViI(SeshBJCB zAzQFB5nmmN%BK=gQ*cd?ZrOIBC|acObQ}HSP(Z!0;8G#9LLef_*>~n$82QO6^y7E1 zbH4*aHi^iAdef1a<2xMS?;sI2cX1|2*h>HTd!N3Ls2a9?4}DvYsy$twb!qIoUHroB z%Q;vSSInt_H;p7!M0zPEaYAOz-3b%|!v2zJ(j3rT4nNHhUC_1e&2xdXCL7CMqr5Cu*fJmb}ZnB?H-7tXb$3uRy81qom z@qZQnDgA(~_5nc%u#KzlFHCFKF)zwP7Np0qFYNbocEgBz&yeO4 zYS&$%^a%`%T+rBm!_;$d=5TC4@3cf^=dJ`DlWbcuoUJ*2GmK|&tl1E&8cGV zg=G(JtP6#q62Fv~kjqxkULb}A{;=xUEAMOG^4r<}T(0L_cAOXOAqQt}K|$H>0+F?_ zx1c;udRgczn!_7v<%dQ^m77%wENWENl+Q?3rF5+3;x2Q^@qk?GFhd=!6CT}q{T!BFhXw!P=)k^uG=D`!W~8)(>1Fa&%ZtQKukIWr-Kt&4+S`R&?MTTh0VcI^2__ z242B4tjF^GfIqU2igmI2+XP4Yp*``z(vH^zP@67mA4!$tp=?f=Y7c6!unCK;aO3ME z&TWA}QnmE)^H0Amd?va*3n+@ul#_MQ0z4iEwF1@ap8`M5sa4W>VxP*TBiftZvy>+= z93a1-lh;KrFiP^Hv|!k*`JuHUX9224ovgB|lB(iCGijk2rvF*~+8#)w)WKrVb2(rb zS7@qe+D-3m&+O$z(sm$gdfKy~DmVb_1u^-EN_u;6p|+Igb8yr!Seqz+B(ap5kmV|5 zmRBM~N7Zx<9C7?=j(N?WKvYT5Pnqi;TP8@#Wc-B1gM)U#(g8jOplrHWc0YZyfhW|XnlFtWBBQ#@*a8dky)h9)~2sv(`b>(#n04LuMi z6rK*UJ>a|yF&VvMvpAeNOBtqnCs1t~8l+ILgki{=rAGgoc0AJJk_0rUrs#nz?Gvc{ zIe_#~udDL|Yk_h*V$!o#gEbl`2;Z#Y1MqZ%Tf2sf3EEqQBZl(cP_EJ30ws|G??^PG zl$x*|?I0E?cu6u|l5O(<&AargokHVhyEj!txNb?>%wphoM2Bb-H_Y#pqf=Mt6{HXv zPukt2?;tZ#ly{KS8*-cjx7@$#2+B{c%op%cs&-%ew0<2T^`*bIH|riknq;1$_JoTN z##T$S(QNuaa7%-`20Pd;Y`$oR-A*~qqqOY~O41dFI5QB-neOceIIEWwca3Q3FvBeb#4>RaqOvG7Hu})m#KE?BM0c44jCc znzHYwiXX9Wcr-_YYx{&!u9(Q|@(%$ZnLm3pVRF938WtV{N@0M+B20>Z?ZIwA zXPlBfgr_V&#X+%CzxSsRTK9Fhl-udK<|na%o;(4W55dMnPO2GmO?Ic4B>4elCL4945HXH)GaN8&+ z{2R!v8l5|-Yg%%M_TRk2g;{D58+Z^lZR0f7L)Hz-Eq-<%)8aqi9Q~9veD*}?r`&4q zWf=pi$lQF)d~N9&J~djvb5K$$(Vy$8931ba_vU)K>$bL3FBij=ZYqxsQkCETTx@s4-RCG6m)3pf8-n{@`2V&v0pa>;eLxwl!8CO5KBc^+cQ^>1`#Mqy| zGlau#{79N43p2;zFL~OxJ%nh%1=fO>$a|$}bl|6N72p1reIp#4F|>Aq}v`5k0tEq_ScF3^w2y<<1_>^ zq>42UZl?FL_h9i=^jpi}jQ7p=J`F99v~I=v4eo=VzJE%nhfw9v?0n6Pz^5ve;QiRn07}p8QEXSEc!-4_~3#oBgwQWUPqKSO@w=g&^ga;Tq6>vL77n zf*-g7obmMK|DpKsYpZ#NB%tE8wOxlB*b@tWTZ$G?c0jzN(8hxz;!>!}x~DDiouI~O zD^M2)HwK?sTsQULRWm}fAsW|G___4~xQ;ZMo>4e~wI_jN`!;qO1;QjRT%(&i*X{g9 z2Qr!UF+CKeEmM5m-}(2bW3Py2aWN*<_TXe&C{HhowCXt1jT)5nV6Dg*!dMGamkbg+ zmuRXvB(}Gk;dmZKF&hkX_^GH)fJb>|EG`d)pkl4M0Q>=>>zf7a{{r(L3N5`PfX0dN z06E#==$;Rb57ueG$?h#bk$;n8>Ne@p;^Sy!aTT*k0totN@%7*Ept@37*0OyqLi2dH zfackL=yWm$kSR{2%Zp(GDuRLw2^_>Aq#}$ zZ8*mV>`NEW+d1DBj-DR(crPMg)CU#aBx(`*sAq09BWP@GXpuZ)S9PUtAiuZT8tqyB z`E)qQnz=`RI%VGgtEwpb#wqm#?8itcKi|kyswg1v*l`J6>Ui0M0J6@HwEg@~Kv^_2 zrFY-I(C)|~ZvaGT9k|5UvZNO!a3%J?Q|Qpt%^ZSaPlz8Y?HtsMU>ke@eEh2T^a~3j zsnqk=Uigyv1v)zOl2$^5tHvPHnl?;G&$J?g_*_T4Zp>|N#S2&GR{ay~)+#%XI_vgS znx|ZKXx7-lo%DLv@%~4I#|PTy6Xt2k zjg68xixwv@0OLq&U@u;AJZdsq@@wlGU1*_mt*NvMRNU_$!7d&B$>9RLo~PG ztnlI#rF%DpoI$q8!)nfT&NedRi1$e9%UQ&)4aJ^*ddREwfiTsmCS^tF7j@3k07RU> zNISZUr}ma4LNS?iHa*cM8|PfjdMe3Yes^PjNJ>@u*pF-(W;Lpvs=j=R2QoeFg+7`( z3Wp{OU^W&d=59+{7>Se-#qKjnOf;pdjb-~qSEvh7Ih@s=)sgoX6*TLqy5MhDtQSZ> zw00G*G3C8b`u(*lL80$YSCva+(ctb)tO7GG(8@fGPZR?iO& zTeGaz*!HJ#>&Z8m6l_2L@Ok>*;`lFCLv4WMkGk&n$*x~W7o>HdRUK9SDEdgoduaiWC)Wwk_=87zQ1_j3fXT$M{`nJs&K{S7@G#C~Vcs7;=DV)(!V= z2R^dCk2PksFOq8dalj1I^C*)ZI~s7BC4_TGEd>N?{gvk*+`KI@NZomF&VkB)T91o+ z&}|1&Y;TXqzQCik&JIch+e91dr=RaE1Y_rQ;@Bjgu%Aj87h01ed<|KH&^~*x%IVph znl~xASF_~fxs;v+P}2c^LA#}GTv2K!6+$2@XZ3Jl3ahv4QI1eV8K6`_H6UW#cNQxE za@Xhsw-(uLPyTmpzB6~z5}%S!P4bQRURvx! zJ9AV79RqXO@U!6U_f+tmtUS)n6lf&bP; zu}JX2>FJA+2P^~Us6?I$!iaI$nx}iHOReGW$(p^V1Z2178MP*G zl(D){aV$X{m=|*H;(ANpnebnkTdoQl1gpu4MFN- zO)@4PW$0D!uMN5f-sI8^g5{TWrwO_oi&J7qL3i1G8;q%UVLwFPVyX9?gZEkHW&tDr zQDcpH$^ttxJJuM^ub5lGP)tK+3}WLyCd5L2ZzI;i>qi8Q}y_W%g9= z-Cxo-gIK9bzqE`G2v@CiwVrv^+%tzu*=uc!^Q6KTOas5}eL&%W#kOuJMoz_WS}DU* zlE;JXl5iutHPRE4+)go1Ez{i25#bn;Emoa9x~yGO z_R-~H>FFR4{D3hZg_A=^)fvR$(A>(N#j((h5S3=zkVY;({pj>)&e$GrlQyF@zGwJ` z($oo$EgC2Ydwar-bt{G-{u0Ya5fwBirZIFW_CY6F zyd+95{o3NoG)I{I1CHI9LI~WvT9%))!KYSGg2^2x~VSE6mx2ev~D3Iu#BkC?vKD725%r2dd zd!o0)Ujb7csiI@mRCgp?Bt3NwFk=^SKFjeu*?Tkm`ENWFY#-LE4T%R3hjDn?@$^Gt zMZqk0+X}lEbVdHjT8j=?%g#WnqR?5BxFyEyhl)(r4H{<3#e4Puw6*V0=Mv|@cdBAw zIShwQ)aReR2Xi?7BN1k=Qw~y7_P;3?v@cgO;BHxPa$LKk$==J<2CQzw5@YiN_n#bn z1$PAM8~9y)&E0$tnkt+QJi?GlCu9NT2vq3iDb*DUx3TGR-jsx&dqgwqS$SbRV0J6m zRDw{fNVxHFFGZE8S6NrtjsOxM(ZlH^Xnx)t0V(HfPcfBQB=HQYH87h;VkMwAnhp-Y zg+uM8?Cl44X(Ky}FUc~@vm9Elwbqu1ATdwq@MQIQ^NlA%a@H#XRcui?eGLmF+=g8z zXe-YYL>P2^tA3CAWIi(Kx1$K=%Jt95d)$$zJ79$}au5~|I-Kg|K@h*wbBO9o z?hbInRJX&XTQoYX9(}O(GX%ty?oo^uyTzF?UiDnY00Louaa_tm%sL`s58U42onl3r zj`I{YEvN~~dEz4&1;5>wG5e-vTg2qYD=R^qQ84uUhM;^A!4dgCfB*y^^d~SKU;JjS zBcX37_q19?dUB^V+^eqXWP&{dc#T5ACFm?C%xG#*q2PrwY zw7NlL$3b#n5p0Ar`^*L!Lel&u?CPe+qJaZ$fz#a4vNbVzqiih+)cBYyPmFybb4x<^?65naLb`MZF{l~5QbjlLR|dBRKdBw75thtiev%uTA|fv`|NNx^8QaK?NBF&YfF$<-30 z`SD@6He;W@fVhdn(??mc+#ri<1sN>Ju379UD=zLC3;m666yNw0zm#)S)WGotsmaeb zMqUSjIID4=;mZJxzfLyB-ji9{I1BT5Bj+NWZ)*=gAxR$2$m@3%FOb#HxEb5_^#Bao zEc0yPrAOz>Q(M&0(+`wP>+T`GGTn>V=|4I^=_amFp?oBSDhy@_$-@@Znp;={q~i;B zyDnyD&cFF0;+6Y*F~C0riJQz1i|V6txxD_a*%1|G=9*_|ymI`->M`C_FEx8teIEoexE3U5K zb5PmdQ44u*u0M3K^p4fbQA)X9EAZmKzM3FX!E*X%yR>yO;0`I~Ks;Or0JKpf*VO7z zT*dZJ-{=aR`zjHeM05Dvqna)>E7w`f)H*P%ONh-vSm^S49;UmFj1^IE=#&SQ;hcI(YXvWYX?-Oec;M8U!4VZu6P^|ek7*O8lSh24rof0dhL=QMPMlkl3cPdd0Gttfq5 zT%um{IW0!&R*FSBZj$-6Mgrl-o;@uG3mL%n`zft=t={J{tNY!$Vp0zNy>f68pmO^; zsC8Q-Bzu>IYa*=+>;8ruF(-S zZS&Ml9s2anCkT~vvdR26%cHJh66)4g7GQ*s0zz}IGS87K15|+(+Q^+{!Z~wOV+rOnxub8z5I(>z;6YRoLkur9u9=xr; zrFo#llJ-@7c<&`5v~4Z_Y82Xb+^Sd}B~924<~tP{AB>jU6Wgf}uPv^vosqT!vn?hq z>)`Fu+J^QB--nlErGOCYMCEOCM|iu^;8ul1p99zj0uB@gg>W&CWdaUkkFFk=P5#c6 zc+NabVA~tTZX#fY#AEVB9V8C#A_Y(cSi zS;9g&(kl5Smgi}zVXqHPx9GsWnd0j|e)w(i>8E~mFvmORVQbZdiZh(J)}e&*dEbm< zry}8j*9(~`U=_0j*&37UnRdh>H)2XNA;$cqdmRStTb} z)51fup)|DUqCE!Iqm|mBG<+@Jio6*sUL4127g?+nU@vQRc`c><*I)*%O>ljzTXT+* zb8^&SUoJ;cho(vl*Ggx_#~u3?>8O9v^&A@e?;=rkV8|B2YpVEWRmJ5-cIzvmN@(#lantFBf>jc=iuT zi)!hrTN9%bOREn(sO!{Uierkkni^9}uw#T)C_3L0K`V5S#@S#$)ZA@Z=p9QQC?-l5H{Bi)ju7v2%)=0LAYs{Hli|&-ot+lCqEY%E zwX=o_DZnw9S!$Si^b#cX-6~IwSsGfIUdRqQPt!pvE#!-}WQu{zTGDr_+oGlz?b}-* z^OqdLnAEVcJ_1{j$q|wVH0y`&+o#bl6oLDrG}%OKjYlf(ZIl+fEXj49Y9J?lK5o(f z%||~qsbF%S}G8pX>PN`-cXd}{=4t&NC2#^8b&3Gq1HYz5ctM7FulhAr9a+^ zy1IKpw<#uM($zAW=(WQZNkD)1Q`FxZ|oNk5N`fEJN*I zh@welD+)1+TaCQJzZGs^-N}(< z4Co=#Qs?-ce)U$m0PSbZdh0~7sspMIi$DslhBKlwW;AWrEa_!@`l(queO4USW= zPlho0^j#%ybzs*~ozlch*T=Oj6yv*~VQyQx_5LUCbms6C_|#9Ze2q6USdT>U$}~Cy zQY?%GBXrM!E@=Zz0^?07Sw%+cu`&&72@MN}Cq>9gPKOm#{`HIBM* zS{+62_PYL@P;d0Yv}kgd3wPUGeGHk(81(QyxQFle{G#W&5qxIXpvyRj!j? z>?5fwg*3@1qo_%AklL#oi0+IH*A3}G_7sJ-d)6K{)@y_-=L2QR2f)mBhoK&R4i?%x zKMNK`4_&6p(v?!^z^TjCS5%6VKIS5BP+2p?z*opM`DtFVP;zy}zeIU$?HXW?ea44R zxk)@btmIl{T_B_s@^m%_{B$b}P1uCEXaMOh(fL|QJN2?&T;vY|xq)9Lsy@YnlAix@ zX#ZQi&Rgqg=>fRbjmUEUe2=eiifS9Go5TlD zYEu6(CXIQxdqjAK-{6;wjI=SCTAUB)$8MV!sQ?lr0TQ5ig8))8%S!xL@3qce`*>0k z+gd0TsxmX&&)L^?0Sj@GBloox^5HpZ6>R3x1DOIe9~57}R|KZ9@&nZXnw&hV>54N% z{rwX{#U_kb>1u0L+T=k>%hiSweHd1v%oyCled_>F-3xNiUwsZ3Ti_ey(3(hKGjQk~ zwNN^`UlI|zI$OaC6u@%CY#!~nyOOGWOc>PHVDb1)!g&=-_JpF1NKH!&du1njThGn` z?YfO5rhtJ-xGnM9meC1A7@Ep9OOB;hkYcgsAl}DJIju`Ti~?{pw__Ry4M}BFel`?Q zL4k8AB?;G&9+-9q$INF>d|5Y6Y2R2_pT|dupY-E16zN(mM)5=dA*zMq0m!56sr==e z&Q)e~F_qYl5%bb-Ts+0BD_EOMisMi9NmgAr3=f=}1Z1f&BZrR|l zx@mV<&V1$jkLExo2-MAwZ`8YMIJm${ScI`TT#vBHECu(TV_>$<8|xmSF6{6t$V`D( zbyl_19UAVfN-J9-{kVg^R;?ZOVXyLz{h5}ScIkS5AN?IfgrLkr^OF0J%P1X`$6Ol? zd%4wWWcdY36+*OCsZK5hJgb8Wf`C1@nIg&$R4i1LLOIyO>z8{nTnd~c;lbrJCV_3N zWV|a*0;|*Ta8||3@FZh_{#_>mool5RW7o`0CA0B8!W|di&XB0PyI7X@NdsE9f zfi5IZa3|dI1ymH43_H(i8Q%U5lI~vNG%c9=ZELQHgRcG=FCrHlXUzMjgyMBK0lU zK!r8VGuw8_9JE&ZCI!ss?l%fgOH}r@8%6)2m&t9pM9YYS3<4*vU{%H?RO0s_p+&IMg&+6zj#lZ7)q_X3ewoPGOW?ck_$b{!fwq_gv`{qO;DBTThhp(v{0 z>A^qG8o*c!u6cfI4c{#zx)6^tEYZ5Ud-3RZ9$6bp&`h|$q#_N=;)XT@&k}=Rw5Qc= zC2^X*?SNSl1O?-g+V19z5e#r_K!PryK8ky`ciK+%gtfbM{te3?zQzyf-L8F8bOUXn zKcbEEoFEfEOnLYEZ^dhJ^IDf8mjr3NQCs3wYsfZ7QT8KET|Ux3b(BVO9D&^&qb8)# z(k-jJqGk{0wQ^X}WX#i~-I1#{Rs+ppy6l^C1n-;6CYjVz=->#DyB#HDxEwvZMotd1 zZgqbw1X^RL2dpU8#&p7Ih*uoyLr6VeW+%N`&GQj-Ptw%SuMxEyLBW%f3tYs6&;&hK zpxZHQ&$|d7s7dVgW37Stg0|f)z}(h^i#3kTa2CHTqZo`q1lT{T!wQm*o~C719$78f z>%D!B3;u7e83>@G7pRYg^!Gc=ue`e4?}AWw4F{fmQu3!nLK{Lh?sx~Cv4@%kJNX5Cvv*1h0;l0sZIb=`6Aem2E&M^;GKIiu+frwa*Rp_WgB5VtS;9)x15BE% zBJ9BMQ}nL{r1e?GtQl6qT;v?|YfTq__42j9iLH)e3CiczOg5gRfjO)s#NN(*$RrKP zLPH@B`Ro+73!X%9@&YMj5AznVJ^JUBI<^;-ynauLWfPL6)HVk#kN?3lvYnSZkFnM& zJ7`Qu`UrlP)(OE2|+WR}F90A{^Q8R{HyX0FC$%7%NOm)myd0 zBf8`BE)f@e`F_@|c(`f;1C*g0n|VrFMcT&^VBIIy#MppX4`#QW2B7{oXU?|sYTr(N zok;|@Q9Z=moJlVv!||kgP}2#lzPyN{h2(*?4Vs8t9USd<{EF0+R)K0)IY>Q}J(=F6 zJ(mel_K#t-y2ha35|v&+cOX?+u2qJ7I*&I4_vJjXn3FZ1Y9bxbTDqBmeKpqUVmZ%^!=xf97hckE3dLfHb5MtqeOm# zKXm8`rO~aEyMvV?I?^bWmQl+BFt1%G12tlMpz`*iwoJNNX0&4XL0swaE$asT4nVG# zi0XAqk6n*lp-VKqJv`vaD};2rcfQ{k2(cn)0n7}3jDcC*!u@rrHpyUFYCZ>A z%PQ!N@8|GA@x%0=?NVQwEbeV}QF{FJ5Y5(rK6<9^1l&8*zjCQiZl1xc#x)>;*ME?wuzq#{EW5t62`;D!!YZ5dgtR z_1GdE)nXw1s9*S-;{W!iL$!nSvq01#X7n4SoToO~_t-Dbq`0Y#ZMt^q{xj z+mKf;Q{Jtmd#SnA23*g%3GK`aL4)xm-7E`b@7f-oMTo2$>utw`L-8R2-`mck98-H* zIB7p`tMsKg~b;5HD6-@uy+>0Rb3c&5%Mu|mT|$;O`gR@hs5I2f7| zsx0?-9G3dYR-LImE3L^h?qEZsfq(UEIg^FOj}l|o>N294#o`dv6q&$v8DE>yBN1nJ zkV?u;42MnfR9J~w_j6YDMMD|DsaNnpX;uOOiF?*)1=I?=dta5zZIitS9?Q(A#=6*t z!HeQ^d0~rT8KT1~br>Z_ZFP^Qr8c78FSUfGAvAzIb1 zN%p@;Gz3%ag_ND5JrCG`rdgg{`I3ONP+zjtL@5NJ-Y@}vBlP{N;`Jk14*GKM1WnKb zTQ3K7XYY6TjgA;ogI*}n>XFu-7E{OMcD2rJeQw@-0~Z-6F58@hwX&$rR?%@md`)52 z?AH7QQucjWJU_z4w*mR51dmDSu)Hz+C}uWWCq=UyP{cTtp>s8<&uWl1kKFe8|A&eOR2@$2ZfmU4z6 zr~?E%4!4Kz7k@^hF)Ooj-2!H-9<{Z-C+oWbE%GDu{(ruFS-k(B)&!q24#;kYD^dTh zcujsab`GCZF`VW0_3Vxw&#CZInj3<^&E2&S_*3UjXCAmmEo7Gl4-ZTljD;*iYQP6^ zkCwg?5plw?$PzMiV;d2k#qENko-#U$%-tFR2xBlS^Lh^RKq$I>iX347wHC;+gfvqY zddXOVXD?-@EWcC<@hYU}uMSYHouJmX?kr<7j%OG{4-l=e-8vs8T|(ex!6;guL|bv) zlF!hc0-7r#&A&o98gO(i(zXdp^|%sN&~&hw)M+AicI0BfiX?OjSEsvo8+GO~aGW9N zC;qK1g-E!ilNd?A6O-Ik4YZEebiD~}Sj;52mRPb|FX#fn*Tiz%TYo??)Yq{Gq0FY! zIeS=<3Q|jx0?x;FRSJ;Zz)vtZh4IFhGC^4S9slrNUJHAy6_B4ky2daXblc^I61Qvm z2%u-m-b)QMte&fKYO}P`waq5jD{-C~s;fC8;%Mrs{Zd{Fd3d0WXH?fo4$mV z8b%GLVCD4FrHru(RFroTHxokc7)`btE(O>Qh&6&Z<-rRz^LDwwg>21mCscB6LU~(| z1bzPbumlCz8ca+U5EWc386(n4rT~4pP?6lqs(^*u8mOp(M-#Tk0Kht(u9;JhV~$bDLo-6J!2eza@@+bVk3EVd6;yN#3UG`Sz7zWBNj_cW5|ok!P~b3t z$zw1N9@`H5aBTV9Howu2W!LPq<$#79=N_8x)Ki3u3I03Eh{;0)G%1K>Rg2V%k;9=?ECX)@JUIftlyU&( zY6yawcQfepi?0soJBMJfC1tWaIP80v$8`E|qs)J*u2_@PTIi~1rJsIS`wW=om--x5 zFKMTOkX%ZugGfe>RWMb-Y#P|UJ4VVUx1`LGr#qaolu_zJLVe?bj6{Sp^kaEYR5M*z zyLa)|!$Fd73LeKheiWZ@p}s1^d^ibbgS36)Nv%<{kD9S9)JZ$fEznRJ9LDJEjBh$b zT4z-PeSv!e02&UBU*?fDGazMKx9IX=rV(rq5Tc&5gA#1GV4Zs*FYCV-|NRHb?OG?V zF-y1Lb{|-R_JQ2-rN1>OSP>3YOYIn?yt>}?Z8ODxRElwk@7I3-!Y=|J)VwXv$iE8{ zK(G2}K~;yY!edxUbnNi6mqUQp z8jRseJ)VT#c=@VG=nY!0rtkJdJ1ybNx^{&Vf%V4eUqf0qX|k5mO9KDTiWU#Tpk8SV zaz!_ko{!)a=wa==Y4C`JY`6f{i5IGfDmKQ+4$4|)bPjUph|@10>1oHt9$++k(|wqS z!Xc()863CJASNHx1N(TyrlRtXcY>*vUEd61@V{fnec%jw(FHwcrOYEbR__omK&zoa zmBa(TycL}2yWu(2f`ZFNrpammSLpCvgKKXuqdY@>^IOFa6D4R1H{7!n*qW>k0r3O#Z7Vmk#1>rWH+HRR18x~N zjPB2o@SWuUITrW3h||=|$Hj-9U9$#Y>I3_q&#BW&Istu`%^t)qOL87CX%#kWws|TZ z0o56$z?nFUff8-rq^>nf`=9d~*^X&-h4xTb6-;diA%|_CQA)i(u7!+%=a?n!>>US7ghISO{``(@T5&GvQN*qDbalVzdr_9Hu;G>HkknZ z-6GlUjjQ)9eTzDMM(O7!SLeha?V&8LzH*e0Utd_CbB^7Hq)E9v%?qXbBH57V+Vf`IB!JJnU2c>jY=;YZp~8$ zRa(}*7EA59!tZWAXAkoWZ8B#yMof#-v%BKAwd!AToio z^(RjK7OccGc&bzI!PqZUIDqfOM&9~UI^g)JyOfYGxt13Eg2gzISmDcdt0lL&*>(P! z#e3fe;Z`6{(l;+2{hAVI-#K*|Q{8M%dv-et3drkgHZ)|MNc8Mwn}`*!FtyMm#*+VZ z(muZrFJSxmhUSb$ddM?)M210_mop(T0b;BXW$MGa3oH%rCDQ2y$60lxUG%q3PpY`E zS@G;V5;i8SMyv;>04rhbsR{3tJZk)V*!30D1oZGM>Iv^ry1V#s?AV~BUo*wo_w$2y z1!PC;Xt*7=xywOKXn%p!n#xmDNs3)R{iz*b7F1<@8V*BCkLDfM>Tc!ZoW*8o9kI1k zmGHtdpF<%;jSEN8~j}}7L8OJ-c$$=_^8)-|rDoMD2uyaVD zDRxFZXHn-a=k6d3O=EWtpD@fQR*i1CH^bTlQ19=GON~lhqC^SF6FM z2IUun8oJit(T6Jd(5&H9xMXPBgt70!DnGy`37R9R=t8Xw+FfpmCJ@B3KCQ*^Z2Hv~ zk{y^DXbV|fL**t-9xAa^cm#J`R^Zg8!edoC=LN52ud1N{n8?gE%&=pakg7h7=q;3h z2ri1u3Aho=q31T|U{EMl7?)y}pI0jAjYEd*Xx~AE(Q*djL1Uw{y1VZs6S$xFE6MWa z%iam>Ye+-Ox%P_`Y%Z$Yu4rVXnytl~I&gvvIdFZ-xZ^0ylD|*bWZ()wVDzp8YK3Ih z=Zis2xhy+zKoW-ov0`UOvV9zyRVZN7)p*`eoneBkO)|b4nfR@mnJh_jEfeDOkucJy zc(@qWA8_HO%OQ)FWKl`en*STdB(siM z;R?Scm)Ic-`+*&)Z~o!+o&S9}Dfr)>2g%O^p`&|X*U8l@MjEM6`XU{spaVc)r`Z`H z0ifGp3h3h%Vg2kXTw`E`nM+_Nv0c^40v1JSuf$r~o1WzkVay5Bd22+V^pLN>4z~FR zzbqSkZWG>0xpU8#R=V(knfJ z;!&b~sSc0rV~};{nC4--@1#CE#pok9s$gH612yc&li)WeEDWO2;JCC3B(dO40I_P= zvC?84;{4%Z+Qc+x=^|jBq37Y)s&D&r2)iG9y}`I#4B%t3`BG3@FI^LR#uA3H{#T$qokhJcQY~&xmNG1B=zd38hIm3Q zpkEYJEZFFI%e3AKIW)snBK_v~zE^xNHO|c)0*a+fwC2chkQ9j#ED{LjjZtbYYqmy} zIMAHu{rJdW9wU9kgQ920A*%_=j^J~DQ1d#5MGi6#;24n_Ru!56 z!1@X=tqZ_8bWB?6ygbO&HJMd;D4McN$g%|y%1TW5|If{F)7OP)8SA1XlxlukA% z_7&eCLd%dF97CQ5*g<$huF173{%D^!A~SGh<$UyGg)%TUZPw41VGMSMml;At=o$i4 zr|Av{RQ~(*O+W5~m5t?&t^fswQq0z~938+u2%y$_pJQ>h+(KSwB)ZLiWCY_``KCNJ zAtPu3R0T5>2p|u$UsGvd=I+&>%f|Y1mZQxDYGQFbre33m$epS4Bz5B?D=;3Yh9UF z`h3#Eh+6rsC=@i4YG-=6#8qIZo42Xh!EVYsG0$4tl$3(KvlAA8`Ta+)?+QOos0luL z{V3V&W?^)P0;$hORQEdOxL5ecOhu`7ARW~G?UMe6C#mBKK`2*0vy z7BMU;_f}novnrBZnZ<6-FgczY{C$ArLPFk$UQxkQRl(kzcpr5{zcF)Fg$K` z%yAUQP-84KZkh?*b#NsHI%R;2KTXfNAjlhUNh|w`rDKyQQ^$z4SDaGM4Oe7f(IxBy zNyGsR<>KXMR0p2pM@ZYDpf%*S5C4HYuP_O~t1AbK$GyaSuiPG42+z_d{U}^!a3mGb zpqdqG6F|bCRJ+{CHxCdoA8@{ZZjH;L{8=C44=;brqt*DA1Lc^~jhn3Oz`HtXi~vRU zFi2mTc*W3638Vxh{^%Rg=<|lp8}4w`gkZ1BcHWD5w`lxX@y(B*p<@Y|<238*cRHCI zmnLH+tD*zDlExoPTaW=3kBYi*7lLDZc&b3nd=_*DtDp1()79kAMbQW`V5>m=mP}Tp zP9)h%cZIil4)=v=oN{^9K#ijb+R3o3)EW_1?;JSoy4fI^bKxcylGI5Y!FvyT?LcLq zY|lm-Xq?owh5J(jfwjCvpx|)Vpix*#JgK;z@F<`qr9{3!OCDlN*8^bVm%T$c^%srt zOha(9u)%#%r%HyZcrJWU!2nIt$3sx{Q9_R^L)b9{ykY4GJV9QvNkcTLb`iXk&bQs9 ztAE!LmI?xh^pa3dHT|;Hr8qfZ{7W8XCk{2xYGmc#p{=RCQL9zr6%w~TY=ZQxZw0OT zF6~ja8g6BiS`6K`!mU=ne*I;kUw9g5Y3z>Ymu_5ni7lN}kJmEYUX;F}q!)nLsoXMz zsvA)=k5OHn^U?yuCW&h<380#um|iFB&(}4V)GiK(b}5gdZS|UH5s?Ko9`LHo?G89Vm4~Jn%{=SD zq!56!Dfdw<&3!}G?yVo1V56?>3FrJy!21}Im6(#-ExB4fy{7FPuwg(}qd=UCkPtt2 zqL4E+ik1rum0hPGAGcZff3}Zj@m@XT9TRovQIyh&!-9Y!s|!rb2dlH4gFxEBt%+b+ z0gzm#Z~p*W-x3pB4C}8LgGpBW2G}dydzw1t1gFdftzxQ>cH}?tr+6-;msZ?0W+?|C z*V_?e4ovNIQPNodFqAHgNETE0)E%K+HH!@BIt;HV`(iKbi~Zj$p)e4aS&5QqS@=}=uu^+dFs*-tAf z*_Hw+>*=hq@G?ODYJ%Vf=D(?>_|jjJH@fH1-ra&SUs6lEo=EFK!sPw!Hf2T62fvFC zRAE|WAPy*}WcExXe zg9XmV9Zhd(;~n+coT86W1PXlg1;{H$FO{hrc6k38RF~c!hCg^*@vK*uH~T>Js`V8z zhWCZ+bF4}9*h!Q&rii7@!o55S?` z{EUIBbtG(8I7NuqKw`p=pu8%qL?r4n7Pxz+4hHBhYBOeCm3mi|SCnm5u;|He@0)JT zKzzYa{}hh;IgmQf5(n6+_F^0Fz1c*4F9jnF>;8=dd+&*SKfYrGko75-N#8P&dkzB= zL5daxhdRw{1K1CDJarq7BZS5=Xp?&iuV(s>90l$ul5AA`YHslD3kFy6l=g#6pqMJF zBc7;-Jmkn{K{^FNqP&R&be!cs#KVA$a-|DG`cp%3B}dv-@EP25I#`)Dih6)PTuTeg zbxscx@U5k4qZ>-OKrV2(lfLpay^$zu@Hsr>jqq;o)C`QpLzdvMph;yRFT=0J#XGTS zT#_J--r#mtTMP1z^_gN3UJD&XtPIniU5Z|4=1H558mn3ucnZT_?Oq+n#vqt0%Kwq} zr+1-tE{v-z-gE=Y;;98+r?}=zQap~;j>UyttWxR%mJpr172h^+Q4jOB$mMsTdUgu7 z#d74zS{*0OvfYTCdr2|M3#)l+@_r=N)G52zkXc83-&LGzaZ{^BC--dHg2`+2KDROp z?0{*Hz3q;dtuFSs$GL3SO7A!Z9>c!YKNZxz_ViQ`%F4~Qbu~#vA*w7YHFXXx z#Pj?CL%79fc(gP=HyOSh#Bzcb6{5@NYtCe&y&cAQ9+0du2s+@L>Spu2#> zhQqi_#!$Hls$MrqI#n+oaPWJ;iN-9GWM5^hL*jhJh71z#4`Yd+QPiiZlKL$1`0OV4 z7Q#CzvwtpLzCKJ8#o1LzLCiqLyB?Cww+cXOgekPG9@>iq)QpyhfWpF>eU;n|+^Ny$ z-)^_!*q+~v%}zn6QPfd{F~q96&{gnJm!bH7hztt&E7J`!!&B18c^P6Avq zXC(a87WXZg+TTH825Lcw*_`6f%-L96U!cVTVVGv;GkLUw=1T}<&DtMHn&Z8GDqwYz zcjy|o)=A%ATJ0!@Nq=5S3`NBW1M9Z~0rjNG7$RVkcYe+$yhfwGF+vDq6#`;dsg0pI zh&m}#Owrx?OI++ziFANM&+m5+BIB_O_l8UM+-uToMP(+6hCOdc&SP7mzyHUAX5N*g(N?r0cI|6dzftW|%-;@*p-g(xyDdFst*= z50gdA1S(t*RnCeIeqfpzYxNz|jJ>50kh8rzN(t01M8K>7#`Qg zDo*O_$55s#(mxWN@HMGe0a@tH04QBq9B}E0K9iVafCnqACr_S9N18+VQ?=*OwGg(N zXP|3+p-K0diJ2!_W3cL#dZ|$v6f7v@om~Ru$oUfphX~zrQF;FOn^%eC+$21F6WzTu)y$$V2W|-z(U#z%V!qrJeWr! zMH*0u%W_hfI0nG=xV2sU;E7cF&z^RN)-l%}{6YKO1@hm#zEchG?qCUts5rMo^C2^A zt|mbu*kXOYUr6?pox7A&WxeiXP+mTM{pxLFzDbx3X}YZ}W!UdXU1AJ#M!Qf)b(rjD z8mDpi{- zN-$%dcbS{plI>bCq{whVuuOkpNpAXc zl%Z}H;zy9C)LeuK3dP*Yo4RU>gklQ~EoMSqRc_7nLhkH6(5`#gCaAf?2C)M`)z7Ql z+Y5D|d`EVNb;$@pa#n|X(Td*`FYbFmmqL%zly<>V4DqAa4Yaybj{`=`ie!?2dWM>6 z`mbFFAZE;f{;QXi$87ysX*@ZVRDAZ*!m@KeZF&ys{ce!;?NV1L5628!xI6L%?h^>7 z7H-#c1JoS|61|MvCHd=!QlED6ejqZPf z-|)BDy|u%1yRB9eHmnjFjHa!W>||S)oKduGVC4Fwc=@NpOozT!neC=DQCu&jMMY+9 z&fpXn-xaS3SqfJyM3@LLS4~l6D*Ye3OizFU>Ea?F5{6SR7%6&hnF8y}M}>ajDFDFS zXf9@w6s6;c+kcR0JIAJMNhjP5DNdQ_{wvLhKlhX zQ;h3aNbx^8-616S8?r^`*{X`OrijNm#J^yT7I|-WPGr}>UF!fhyzxLtxVj%$ z+BiSAiG%$9Gx%ftnsi=|XI95hr9Yf6yN-)TM^7HE_P+xEheNZiKOP=7z`*4A%F%=% z;0PF5u#)6?H56R>z-crnAwurs+ogoMoI&OzRlt( zw*a&EnFtK{&BEE}$C#O6@ii2hz|S<@Y7T!^{9oyJvWE|jf~KNa+})LK^6Giss34vh zPf67)@OzjbG+L2^4(I_Rb74>I$zHL&j6UW9@PeSpT948@Z%NC}$|0Z>1X4K96v+~v z`zutt*L;6pf2}n$TsVM6z>?h`_(PoAb7A+*JqKGWVMbiykBXO%pw;pPv|1KAxRQ^k z7>8N=e1ORToTBW1n$MQ{Gv$&eeiz-A2^AixATZNi>1)%(3he^5YwGFwhoR267}IW7`1%D{P%HIK{bZl4 zmcHx=W=pxClMUgTgdWjkR8865B<4BcXG z0z58wV{kq~3Kj&GEdg$D038|5>FZh2I3l!pz0?|4( zYera7bJ&ZA;Qw}IQVBeh6rrNwxX=fv(6@mO73xNaUj^&p4EVmf{4StjH5sXXT_Iv$ z+y$UNmJs%IMius#)cyb?!Yi$Zk@PqI`tOQF89%mau>N1i;7wiT0kQv~!rW0gJwx{X z+K&x%Z|MhkZUdrl1X9GyVp`l5$8o%d_+bM-cl$Acv4qEnj%yEnxmum%88K)}-x-VX zXRb0(9{M+7Fi?|$i@LP*az#2A<)L?T!@HOtWd)?6K{)}>G8M%3tgH|h6AuRF0#3B+ zWf^mq3!q|orgemPl}8kkOS(tFtwUvfd85Rn4o#$G#q6|3OOw(_^43c0Xaao*WhLl+ zYrG@utH7WnYFV?;?Gq|u_Cz@9sN|y&7ib2~-IK>`1%=1DRZk5pMF)H19sMzBJA->L zWee9%bq5-ZdC!7wk_FToNk@;loSpl*R&!v_{K@lS@$5*NMABNewAV!wOTB@^hjvNo zXhxD1Vl`G)UyZ(SEwf-y)nyJ0=&SlPn@$RZG#|C@6{PT!^wWRy_vuqUdKrh~j5F=0 z?>=rsun~r_XjFPC9-#4Ol~PLQ?xCV^Rs;7yZAYv4hG%=w;y+~~&y>Jj7i^}r3%C5% z+*sU_Ye_Y8h&{3pE9bzn!zAinC_D_6CazdcI_I#2)qN}^q|z{1yg6$D10?A^p~Iy_ zGAi67^Vr>@#zL>>d?vK4#3HJmh3Z$95`){%;iZK}8Z z>I@sZ6Te3n5@=yGFB0CePy`7|9$FX++xqgy{h*A@?{uIl+~}BcUU<=44XGd^1#<(m zIUI~QRL^>4Pu!56mTtn9bQfd%@xO~#_cn_%cg}0zN+a$wjCTsJYXjmU`XDMKs2J|g2WmH zzL&nJ4qfcp0FS~*$N%l!pJ*1wv+V6I((B_VJ}(Ag6mX+=#1Ki+b@bn2k4Kj_FV0bK!-n%oXd zNc=-UcuU_68Y`*-`@1}s(fR5_)Hvz#G;kr*HHbHMq#T#m77yi7uivZmtJYnW(v)l= zIpFF7hYTG${nCD@BUcS}duE8BLHN&u!T;kL*k_Jt1wz<}6c4BJrFdvH`?ctY>7&1Z zBFrMJ^tC2(iWYYXX3GRjlV7A0pRykL9XJ*ZbRg2>!~J%)=P`9sH&`l~yn7XeiUvt= zY9C=T=8pHc;|N=ILpL1HY-91%_LLW^VnmZwRgY{9(VEtQe5Nf3C8`hP33VHHb&#b| z$*2wQR~`#GwU@g$Pq7MGtK`-U1J)*>Ho&g=B&0ZY^bDi=$yeA4>wfBGkdjP{0qV`C64BM7tcu)Z|26FO&iY2nrr8YwEX$+wbx~&~##yUx~ zN{{FFp>-Ypzoh4ir8y`0EA}CvS~(Wfx6b&7<&CJtF*J9ad=kSOqc9Z0>Y6!{coO4b zCos`N=iulvG(C#*PVG1LTqAft5N6Uimp+dH0zEps9l4{Vo{JB4DyfQ}!5dP;_U+<> z4^kO$9YxW`7K9$Uf!9Re?o2*AC5d4>*(LQ&E~ZUtQ7)=w;HVY*H>)AiM@Uu{60vy{ zLYThy{xa!Exj+{~L1zVy%7skYbFy?z?>J%*#}>yn@!g&2FeM8Kdpj?`E$kov;-C|- zZjjb3i{^Et1z4w;1Fj-1K)bkHhAz~Z+(ddJpQ;QxNxfwk=BXIm^RDF{>==|WYVHd? ziaE!?$--`U{7W&nYT{lREe;@EvU&2cMA&&F$tmQe^cR!H-WWwKcPl#KEPe8iJ7hPx zYoJGwQYi{LWX$>^QaQj;P+@Mk!%yd6>oxoVyMK%T@+goAr{-rtzA=hDVR7r8Y9n^^ z5el|`!idXDl8cYOILE=yyBz+u_y7xR+G&smC-wmD=y_ij*|FU{kEMOq#>etbgSCKb z&V)Hd-m*D&-q)nRc2xi47B-d|Oz)f&n1^n}I~t6fD{dW12Rq>V%k1%wH~Q@aQ%$FD+)t+U=e!E*+i=aZii# z#x&@2YfRCT=sd=6r(`MMulSysU3R_f>}q2H#tF2h^uWS~Pm%|C$8aV%+!_jvHeIqo zK*oL?D=KY|D-w75u>cXQ>BNTmdgq!AW7>fW0UyL+n*SxhZvCO9yrIR9%Lv#Db!spX zj=_XAUx)GpF@VgbS*8&yXT{ftF<`+ce?dLQupl>zU%&jU;6Fa~Kw}@I^#119G&ly1 zo3SqXq12Qg-Z*0&xD$*zzeWXQ(WFD+z-4TYpZusagVM#iDlcn`HN8I!VEuIfk7rS5 z+{t(qov!?cz!9%xp`P0O{Fa70^VqLCBp~n_{7YoPFH6`>okay3(7l9dNV67s3HJuK zb*Q(!B@Rz~?m8g_-+7*ddxEF43+SW(GmigmSpRhb-q&-P^2E1_@Bf8ABfEr}EkK1} z?BTeVOe;OLs&iSJ$XU{-t>R2}{QJ|DXDKtAsVyXrf=kGk zz%g*gr0MB3;U^O|NNAlM_5AW#@%~fGW94~+fq*5n0JZ{y6m0{Yj?>Ywa>nk>sAIG& zhf64_lvEx(X5M<#)vlfgB6+X_lJN z6p#=B^kxicG}Bx+_6|Mqxl}Zt9bqQQU#`oL+~}rD&{j(R8)v zvFYX8ud>Pwkco*5h#xXzT{GsCU{AQ8vNla+Cm`5;k@Z+Q+_(v-2WR;eJATSGltbI^ zWZ=(5pDM_c<5FUJ2he*NPK6mvjyn$YR{Y#o;AMp>eiW(He!z5tSEZt%%T5WLHH7L0PdAiaX+UyNBVJL2>=D^HkipA?ykDu5*PBW{+P{~ z8(qhQ>c_&x_XNdrnY7j2tsc0$oqP}lK0M|KugJZsC5M6j!1hUxJ9p!yES_RBpY3tG zy`SKp7k~ce>8ZT?=I@J_e{z+Bwx?!ynDi0vlJ@&|;7Ks>M5xm0gbZpteZt_Y23#QR zMEa2fYJVO=A^f~}`7PN>5NGsa9ZjF#rAre0i-((M(0CuAbrwr?Smlc3kI_T8wx9wN z5Cr~9UnfGK-7lDzx&zHxgecsYweE2!b^wFlf|6#(t>39wC53gJ`JbuDODXpmXLo4M zY^}8XCyp~~xNRZeYTs?q2h^x~xk1!F&7VC3MU3>J~$8A^1EvRSUt5sHG;MQ2s9hy5}7W5c_h`^=_Bw&+K zC$>VFi9`woJUbJkA3ktYybz#P^O+)x{6X9t4(e4OA4qytjt+)5hBt5U@YU8IK|dI^ z)LrgSeUiz^7JgStQkP90>M#ycHPT~LmY?O^Ffe5o3aU1if)H4J74%oZ*=T)*_T*QQTb5+NO`YZ5o3TK+gm$XkEmUR*ui&`^#VGEj__x}L3{Zg zV!d61n&&5|6GcaBmM?VE?+~1pR3+?I%afpJp>>!9m0tc?esF*Eb3N^aR&kjC!i&a` zvOW^bWt+A4qx_F3@PGQxbX}=#t|qO+fo&s8-9SbaaFFM*1g=!aimr_KcR4v?TS|1oS8vx-FEx-h! z+#uWM)6w%r9TcI9+;InFF1zMX(`n@MTFD9atZmOdINLc zBhLj2o*`XT?xp(IeX<8Ouc4xqHN7zp|8ep1bC);}HD(F%Xx45cI6WghEEbJc8hg| zv~cIRY+C0mRc;mfc5k!TJaMc98;EnZ3+qQlc%FNjfvs{$fDceDOcjW4!jwZM<$9py zA3z_=n3iIMvArV{C7OB!>efNiD+hH=CwV6LS*K6+fZ%lmupvaF7jXF^mJb<6_<$aH zuzJrbv2KAVXLuz{yNV~|KzlbHgj`ktR-hx|6Rw>Ac01INKpwi*9-QiTE8b9m4eG^8 zj5=yN9Zx$Oic5oWvmG(NHu2?fWE6^QbZ=-Z^V^bwBbJzLh)<5}S0dEDDF(B`$0$I%slXKQ!w+5Fm1 zJ}7?KCGFDq((p?bgl;c8lWBdh?*-CIItsuyR($+wG^~Zfh8nRWICfnFVu>Om=+^pa zFQBe-llCy)BN1Ur03Hp{$G~-Aq}WSZp^Nqu8*MB4=tywHpYat?iEmYmj?g)e;{wL3 z46pi~G2cCSie*6C>|u|j4>`u6Aqij-i?9zJ0!&My3;?z2KjvC3u`HgOhi1olna6cc z&ugj0(#nMZI!Xf(!j1vL%7ShCp-EMDgDEUcBp&>Ok$!N|)EdgS?SgGJkpO#UUvhRWP^xJHebEAP*xL78&Off_;Cs(2P6BUR*@rViB z_Avm~4(3Qc^_GeQG1(@?5_=IRt*7A3QNbr96)|U2L{}i+pbNWT_3SUAxcQ|(5vw^= zfaupZz;(nrm6G;=?@C*zIdO#dMDSmtrvMk=tbP&etKfG*W@*4IJ(;-AWfEpSJR(f0 z9O{Z}za+#oAk8pM$x1Sm9Ki_!d`0o0S69r2vIB`4+ZD`jTS$nVOHN497{(Q|KXHDe zUXICoQ$y;p8Qe@P*$}5co-BtWSk%Y}wP-ZpjXP)&KA!1s&j9>iDjmj~B(AZEvWFAVl2}+2{=-aVT**%yNWhTtOJj$V>lr(IT2mXnrh(1I^ z6=`uG8$9w6(k+_EU2s{OOn4{3&!10#zqX|~9SH7df#})TZFYx?(hy(gIdRjuHKQ|Czy8|%K5zw!3R!WtV&*&KhA3)%IY|sFS_T}Vr zookIc5TWl7o3u7?+T|w-Q29jO?gWYx4KW)ONPZS)(F4)uRMaWDSrBSNdNRqqPh;OC z5|sveyq{x2B5LR}0-g$#rIgx}#Kx}U{mtti#Y6rh3x$_|fhhQolL>_5UkW=9xzm?s zEtL-Jx*oAb@hHGLf2P-|aHD+unAg`V0Q8ZiZAXa@#SWy%>LOC5ecRRBCOs;+eRjCy zxly=c2>!d&Dh6uU`X-GlSyFi#oI_u|q5WuD)@Ekw+84nLc5c81j0P7|#}Tx2d=UG{ zZPXaJfq)uZ+93>ZAQiI^V|x*9oto0b=Pg`JYV2n1bdv{6+?h$&)3QL}dbblGWP9bH zUkM6G%$U6eEBiv}^qc_EL_F$>OiTzvqq=Mr#RPgZg&yh2;b` zKzsPRzbpRE-)<i$V4>Gx8FdQ+X9H-|B0?LP#Bu9e zF4r;>7?6bhRA|0{b!Z2|6Rh>r+Ap50!#U7wrQT_!RH7sdcNbZ)V|<52pliA4yw%M_ z_LLkIvZTagnd3E0lLRHe@Jf>>hyhhH>;j~0O|`N|rOr;p3PQP^XDb2xY|^Y2Y`z(A zHAsYJ4?9OA()-hrhT~a+K&=2(1!)}KYl5ELv$prQiht)SY6WT;z$yHc8KLAmCvfon z1;7?X$qGGi$Fe22rdVWZRiq7l8s>Ir1+|^*Hzy4!#DIUe zW(R4qPy8?{iOor*3gPYnJ1zaNv(UGlebG_*o>+;y&h3gB!TAE3BY1G?BrLCt%Lo@d zfjuX?)0-Rti^Vh0V?R{Ep!Z^kck(Bopx>;dRR#gNzK9| z0U9B4Jsk=mc?min`vU_G&`DC@bsSjDLNy~|6Ga)q=g5vuD{R00jWI@>OSX2YL8_s&stN`feI?ol3}@k4#b%0g1Ud{`~Saij|e->er*8G-VS93Sgku@lX5#VAqXp6 z6^}=cufrQvm){aYUh2}x=ndfcspMG1`Mzkw(`%4@A&D#8uGji(Iwk! zx=0yudxTJkfPkD$Cuyx)8?!*dQUd8d&W*FbBSjX;iKE@^+O1VdUk5eRxzsLi0qQ<4 zw5AqonMxy_a8f{4 zw^`%B0_823Y+z!gUqMtLghlO>cxbdg{oyt;MOn?5`r)FHXYSR$6k0(^sIG{|V)qW} zei6>BY5v5ODY3p5loN9qKLEwcXNLiO;JK)Guk+V<{{2W4fF5>FK3XfK%UNIJ-OFdM zKixAvrxw#7zYx!jYLU|zGDdjWo_blY&4&-+@Hr=6q^Qi$HM?j|MoDx+$Ut~?P~-;@ zI<`bp=_T=0?lp}A{wKLgWbH*?*Tq=r`=q_ciThWrp>bQj`B8cWrA&NN`6mC~V(WY{ z94w=C*@dbtlTo5AYZad2@A z%@=4}4XXEs!c;~!0h+q*2#XiMI2cHL^cr;-R7(9`Ldqna7z@|(3f^FR69|J1#8dw> z#W)2`ngZE=Ql=%FX;AVm{RHx)^GEAs68_7B(730fcV5K)Ki5{HIRtQ!A_3Ks4s{W< zXiEpWfn^;dX+RhhFJbha#vdmkqaGTpZkv13XI1et(`sZ zOKwc5b%()P4^LjIm#Tz**y(5azDI2+s^F5yE^)0$vh+udKN}GVW&Gyt@=cOC)`iIa zxH-9rz$*1u@GE_d1ucWX8e31-2qSXID!Kp{p7V}qE+5(fs$|2u6Jq-6<((SRchH(m zj#o)iB>fl`zMe2+n=r=5!s^sY(E)AT9R2UI-VsCmPEaOmhxUWfO}SeFT`0Wj1<3Vo zDXUWjj1Cm#A;y;!rcK`gr9vR-IZsYvLjL5j@1N}Uxfr_qI$(8iQ+&N4<%*p$bdUr* zT<0xViq}AtcFhA;D*_}jh(*{ZCN}t{QyEmhG3+7|aoX#CL;_ej#D-=a>yqExI;|!` z;KL|%&Y5oHa^g)Ms^}{_g(vtrIe@!ds{QmTfuTM-EWl&?#+Eh~G1Qy@zjt}<<~*)# zULVZN44N@RL?gYRAN-*B!4H0reze>npY0DMGi2`P^TPBQIwS$Q5X=^9JxPr86R}uv z+xkIDEm_68nlV2S8XGg=HkuwH-0z1q^n5V(^hogl;Tz{op3x|&BpD$9;h*Rv61yd8 z08SS2M?%5av1?qxF%%1i-W9XvnPFolTrpOb*t_LgA25%LGz08C!a6aY(lv~9k19h5 z3YTz7nKekPC^&E~xTbYWJW&fS9IJS0fn9OGYnXnm`HcPI0GI9PUr$yhAIDC|bJ2;g z>jPwQHQqvmX1T;>9t&=PcAx_dkKD61Ocm?O;!6x?GlwGsdq(?bxXM8e2LLXuGi z{B4SArU%;HFZ>ybJhk$DRTd9kdSsHErI@PO>BJCGE8h4Vwsnivpj4exFHD$t7j=yv zv2l`TmD>iq{)=U(Gsfv6_+n7qs#>e>`?ZRWOTrdS!UwA{`wX7EnmmR~mv?`kqE*Li zeid-gdRGC{Hg@2rhIAkQ&y_@h&u=>JUly-vJa*Z7I4Ib)6h2pOb>L<+X-sIP{PmUd3KsA&I z%eX{`gAu1ny)oQ<@eUxO05#>C^u|HNWmfer@)}mZ0??%cwyOUl|2inKbf2z$=m#I! zXML-*+YIFrc%a(D}R5}0`?zWZRd!h_hNAG1cDZR&74w)Z z-nD$0E^s!WAFov_5u1|Xf*(Z;?>v?4+@&rfCq2vF1?F0!Exr$j`En26a!{7rC2QG1 zA5?Utj#{#)F)S+J(KhNr-oLY(`;z+U8_C8OumwvnM*Jxt))x;k6h3wBbj{Ruq#@dt zc#JB}8|_VnVA#C50sLs*r4)P7zexwVqlv24iHQo6iDnhbH9gleENnH9COpzkp*ZN#1|u;0`7_-t1HFN?dkA^MdymBDH7xwMv7xTR zpXN$lm3$O|7b}uNqTe1cSGjBn02#RQ#FtlCTdJA6v=mxRA0J~0=zETK=R%Y9 zdZTEWdro{u`6puu#a{x}pwh=4O(ougk6;;(F7lRw&nD;6|Ml_-1Ok8R z0g^V)emtPR51xLUiKPFY9s|(tS$Ku*$2gpKiUw_9$w7L?BTly*%LB~$05IB(I&ZW% z5+b`Bk1TQo1{%h-0H|3kH}ChM>X^PCb~kw_0Q~KCfPvo z%QIqd7t=bv!G8e+7(vggjE5%6&C}HT922BVK---urMEW)c zx?q6@jf^A%@1~y-n$9f+M4eAyaZZn`ZmMp#HQcPjlC9~G+ZKR27vg97m=jPQ`~VBP zhMKrSJYxnzKdA;VyCI7EBen0q%XHU*;oN0+5s+xE{sQqe58o-i^Bupm$76_DTuK%< zhdm_j6l!Ab%AS7LKJasSNUbueE2-+uQbs?!^}GNynqpWlDJu|SN-r8`(I?H5_obHd zAp#l15A)Ig>h%{w(WbEte+ci9PVJcj3Mx2zW2txI{V<=*vUSTYgg;zLJP$aZ%fo^)Wfp6O0cm&{9gz+_b*G^RzDi&mIdxI zWf^r3EX=0HBNs&eNubi@@umZcqjoCEX>LQ@oXe7RRas z{hpRk6Lz4^t{xD6Lhv6qMYz2b|Lr*9vw@mq%?-GXxWNFJ$^7Z$CW?GGE1>$+uH0mUmw{r}V-+%J@yW*Sw zA^z;YH8R-)adtoZ)354^>+(0+S2Sy*#J3re_`*}yQW%WH-Xi+p7j_L z1tN7OEm%EPJ?A_3dEtU)37b0XaKZd-i0cWNb}2%2X+LT^%j^#O`W*PE>9{z$%@0$J z^NL05Tyl*EA+3#493J{U{?Gnd{OND_Q61>!0*v%h?OYi^dJH*qd-M`yt7+vLy8lhx z3~M$##dVsQBVF-_5RiMCF$aY~5RKt-(;&~WC zO7hTN_o>sMzArDV;ZI4-u|a2h1+IC7t56fpqeqLMfrhN5uXWm>mT*e5aLpBN09!XBe|6ul2rS)Mfgkgia9TYuTh5dK2OG zh()$NJ_!pQTe)NcPgBYzCF#%LyLt_FWmyI3I>z(3PuDWAI?pT(FD>;RDbps=5CY+; z(iDT4q#UaQixKz`a7P$^LP*V-kk|&P_8aLh>$+L*UO)QgAByyk^q6RiEzsefVX7;4LL#Mr73eGY z6Y@+=fD3tF=N94l|?IT3AC(A+0wTPC*!Wg;o7G4}$ zM>#6>FtrEH9&iwS1`3!$$+qE!H(JQOdQIR2%BqPw-e)!gEGmw{+<}_wAN_I(4pCMW zBz^OY2Y1`*95lFrD;f4|J@7BQL$-iZFC=$h`U^z?xhnqXn;)tAYR;h7rE)jR~?t}>^cyKwf_UK``A-8^X{X%C(Tx z%bgP3IrTZQYCNRufUeaI1s{kCUfyiSgWu(V; z3CC;5e*0G*=PUuDNm`S12tze6dxv5axnLU>Fl_NYPYF}70PhNr888e$u@U)%prO5j z9hGoF8RBH2C(fgst?2u7CdaIl)^-`-Z{Jv=c;gdiAlT*m;+d$**M1xAc(n5kE-m>c@_w4`fe+K+=er{d|z-A zwRgBj`mP(gJ;X3}fVuFzpS>dDJeK`rZ}vusuA{Da6Ah5K z)3TtHU7Mv0v6363TVMGpso7*E(6@>h>i{Adzy7N5S`zTAG?g1OTd69K+%;PrqKq@ZP*ffJ@^O<-YR6V^^iXX)ZcgZs#u(HM_o^>z=s`Z{KU!gso1BS%@# zys(P>N+d=tLC*^aZxw1Mc-#3K1zdr$7xvPC!%2ZR+3@bi`6S+ehckeBQ>`u-pk)+6ACw6^<8Xu03E*vzFcucM&KIygh zbZx|LTbm@Ol77NFO+nOX&B99{K-63b6s^Lj#Y_te+@)2Qyo6SlFcred4OaFSF_D~t zm`pxLizcFKB+E>KoZUWPX1OJP*dSeR?PV!+3A*%78{K*@@eK35Li6L}yeIvnx!(iA zt5RXvs?-9tbnpZK0ImvJ`>=K#c_S#JVwi$Cc0Hf|h z+AR+|5CcY&mLTn@+IK^?0dA(eh5t)KRGXj2amRoAjR4Ycx^LG57Dpi_j0+J=8vE`i zGI;wCRvy--pY{&uO;=*nt=1ibpECwfxj=`~s`bHXmfTyPAp)!oc?+CHOwjTY_$Vm6#mtjEL%hHUTc|pg7{_F;@C1d9bo*F|_|B5`0) z?+f%BUqCg2G^Jg3%4M|?4>?f5WYJDKz1l3@YuE}X`b7PDxAMiuX4yy;R7V?8^u{i(y&yE+WT zlu{UXb^dhTB*+`3Lp2?o9~(+50w(+$9e1)qWn)SV}9;$ z@JUmHd$IXM2T9Q(su3d_SYSfxMX?84Kzpo*mGTa=OjXL{OCj_Y8+5#BRF!%iP_2_c z?b4o$fr^jKV;+;RvL2$Nl)F(4yy6?%mg%y^V6jg!ZC2;jI@wlkIV+3@5r!?4&b9aa z;A!0|cT5MPU9}vT@p#}6boM?hI&@f1HK0KTqYK!H_dk01nNBlKM%T#Pc#CI+t79Kn z*)3r;w6)*h3Mr~0Hn(8}AF%*#K)w`28SXn-?0oo(r@~Q_fv{a+%9?`aNUt@ZX#6~o z#=@uTk8~-L!R4y3)#$V?TFxgCLx=sKcL*4$eId_N(1?aeR!AD1%RG(|5N3e9b{rCg ztD5$87g8NkXMs&6q_o3bbqpifLL?WV`ur-x-|LQvOFiDS396e0J5b@v`s5zukNZv& zY`Ioz%-}mn5g}9vA4wg~!vIP{A=F7YDo< z9nyCW=PJBIst`zcBtF{XDG(6kfy*qnVjvR0Jh93~%(@_4vBb{^(#K`9d`)}l05kUc zxBs)3UrO29^Tn-=k;76#RF%K{!2VZ!U`^FAX&-3HYVY!pxY+C-z-*ZtPSvcZX&tTt z)LGzd%4G+_10L8)&|BLj7(90pdo5-F%l#H@XfdLwsiG>!V9DFlGBMe?UQfOu_C#Xu z+T5}}(-oE&Rz|l?4M>hGG~I*vm^69(TXhUQLe+lrBYPJ@j1Gu=(}51;o|1@}wexJbtqr9MD~ytYEHc6{(vfi9ln{>hW#7A1WyqhR4YMlI@tS@{Q=%f6*M!44e6!Ql=}hc(WM`A{SSYP$=(@)=qWn~$_8 zbqmg6Bs+l0*0tkP3s()*>5%(g<%dfNEK5#JsKFKw4rj_3 z*b$AHbXhF(d(%76$fP4qEl3txGp?BWivDUj2)2yb9|orKtrvZ6@BlD~H*)sNGIHeY zR@`of#v-Y`x-^s-cG~>*kkUcAHPG1-6( z&53zI&VEgI$}7b|YLX2)B##<*?19slz86paAzHHeT`XlMa#haP8s~96s&KIvrqUi}l;>e5+u$P* z&wW-p9WgnFZ6!W?`ZnKYeeQx7Sc5~l?bMBMcql2I*b20IgAkK6wc$MZPsxDtpOj@9 zNFDpvMdfRshpJG(w)<8*>ya7bJf`U!Q`E@J_-@w3G5Q*V=>>7VA`e?1VUNFFL! z;}@kGFhGY4M-NSVN^qxNK2V!F-8Cq`6oH*b7CMs^6u7cmb3_UWnyqy!TV_oQW@lgI z8q@2ZMfn)QYmZRII<$53X1Y*Tu#c>hG!fhZoPaiOz;#H%4pyU&+BPA=2WYyJa3W_; z*e~QkI5{kWX2W#6Od7WfHeU8{w3xfHAMh(pE}eenzj8Ae?xci+FQFAx+h@U*gV~)1 z`n#4F>&E-x{f}V?2D?wtvJQYoIVOI>PCG_pH?WZXr^!IENtGj@)smM%!f#jL+EvYY zR{p87@=p$w6!WS74O1hu4j$jngwP;6nh=?#N;`jbak}^PZWXI z-=@lAjm09wUF$9OORxhKw)(@sN};Bkcifz0)EHA!_8v^w9<42QN0&Q?V!BqIIWJ$tE@U>!+-Qt{Sq5{5}=7I&-A zO0$Q)q7H#-P7p|zQQY*UZ?k@)w0Tc-*qg)A#^d5FJCa+Pw`jVxP4w=G6^!Lv7fU&U zIT`{>bWqr-i2oP?#xi(JS%}J<#QRF?`KhzvglRFv}yFLEM;GJJ+>Cwv;Q zMh$!mXej0IhIPYy8eGQ`rV%qmV>@R3{h|wCgR5o@GmY*CY(V^tl!8t$a!b)*raOS= zK-`NA>_bn~3lCH{6vh*WpOcSXKLMtX|464UEL5J8bbbk>ql{!}vKx~Fh_SBeNA$k~ z`{g1mrkh?3JxUdPi|9^}B>^~efxPJt-Ui(fE5n`CrN0nY- zCo&~lI$xv$1r;v!0KV6E-u#Ol!#&tQzD(~wefhkAcM>&`#LBYJHpf}PbZ;%Rs~IJv zU6pIO(60QV!_E1fC418Qqz-Yk)9)e*|A>p5=qQteN%${+Tl_8c7_Q`UVu>$7ZTCpg z5S9|y%DV`jF1LaozIF#PuQQ+`|EZ-VY%;UW3^RBNW+-m>t8Y2Db8g(2Y*GptBW$61 zW5)A$0Q`(SGR+Y!;4n+l1; zD*IH?<^>cOwalRxL6|of86C?aAbspd(aJW$m_6iDK?_J#kc?sd24S7l7zc!3LMomE zEhS+ZI_8blLu^Mfv;vShFMKTswCQ7Y*n@WWL}&1h2iX(EPY6vqX7t#U*|4!MrL|`T zjK`OFN2rZ{g{1Y!S$aR|HFix&cy7=hdg)?Rl09tFSa$$g-zV^`l`~P;kJ!1KGxJIB z6%Mbh%X6b}2Eea*OlYBW=x_v&3FD@ZfN=x*ox5jUfR;l5=Nf9S0B)%&ZI{5(91$x{j9@3Px=eHFy#`XlC5BP8SO>_x6hv*TQ|xtgrcU}(34#h7qGBy@PpP54qO zZ=s3tLkI_~Jm<2nyQLwt$fHo6JdAGpVHE<%Kxc>#j2ZGFl~Xhd()NdlR)F5H?Ynx* zGspA{Sno%Hg}@wxB94EhIY{Uzd=|MJ)TtlJ1tWG1eG^E>J?Ob=86M>j37ITMI$$LA z{rvcTBWI+92+_ilt5zKJz&2aB0MZ0c(%CgGf|p(;si6Vkw7AgjiO1A1IDo`2{q$bz zYCJf%vhWU!!*=cY=YF%_^g_Ojdxwyoj5hrT{^1XcAKL%?KUybr9`R0|t!#g`!|qpE+yZ;PDf_ejH`4{6{z~d z(pV|yF58z8e_$FtRgCxIC|ddz&<*?hAfHNZFDVc6*WXF;?6)t!|Gqz8s%`A#H7qD= z9c3?Y#NNA2llcmA?H_Ij7D9vK#L2INDDO3DHGMfcvr&Xv=o~u&5~V@F$N1E9Ei#kJ z%s*h#;0*P zbbwVTzz^30*-|`?%DoQ|JpfDUZUI=;FeFPPr#h9n3$@#40Ww_~j|BC&9cdF@#_IQ8 zAtdT*CZuGdn!Nlp+0}jX54(N8U6R|IqZ{p8J7=H(AQvbE0GD_SnUv(I2C%j!6Kz1c zFr(~BR&VxxtYefwMrb0o*khJ+9(CDa_=6NL?=&NvcUWGlhhDx&n?E1wCpuG*QKfKH z(m*g9*1iACI@H(SIChJD{-|Hs*G})I38Dfdwz4hqv2!k)GVD`sLaMzr;@xM_ zl4TmD2$=$$E`rAR9aj6Dt9k(%)GCctT;`64BPv!_dx1W&5{&CH;7I`|Eo7^Cg1@p| zug_lpE$C+8*u`e=JQqY)>KK!8LRIF?!q16XUqpa+o{K<3^RNs>cNK9D;WDs3 z6_>^J1VJJ1DFkdswmEQLyweFFqyerxO^--7V_u>?*#uplE#qG8h7kfM1SJx+mV+3B zhj+;wZ6@J-N_3j_T+lqiob5mxWvg75t)lFWfBI``PugcuzFxlAqmeKPg6m5c>rnfQ zK9Mc8*+bRh%CpE(F}snyF&XJDeGUprFbn1JD-o&^fK-vRS;g*y68%!OncrtW zI+k(>i^A~%Rqr>$>XJ1El&7*v+?s6KdMh&~x%7POi7VS7Ibq5JG@N`B72ll`p}um4+-_I{6Y%>x?^V$psm z%0+-0ds0`R@MNJ}f*#XGQC&C3ZO5xYJWj_aRgLQAgIvS8$8wRKe89EB+^Dp^ApsDu z(n#OrFN?nn;^*fWFh4&dN@XXNt)pjTQg=}W4qu*41>p%`I!K`pVZw?Lr-*}#gx$}f z-vFG}qCb1$)IP>JvYxQ2I!UCZA49n15^l?jmZHlxXHVAUMolX?ABZqt&c}A$OA?f~ zLIxiLJeOj8W(ORg$Kr&!EtH$c5Of4N0k5R3h?4sbg@HSg0^*8iuK;`tiFM!0Tl|R~ z!2(?eNjtduZs#$c1_cW#eNd$E_?$|%J4sV#b-1JbFvHQKnq}}ywspF`VUDIy14kW1 z%sl*Jm*@}9;^M_SBHr7%tL7ndW@n$?DmGKZ5K*KdlbRK$cOL%}S{%kYvjpbs0+PumixzV`Eae z${NYrgnJX$YHp=(Q7%g&f<%$?F{l<8V4Nf`*aj7B4cajb#Z)U`8B~W5UY+UaH?f2s z*@}8t$o@P>OJoqGb&n=;_=Z^ge9o3&^tBy(xt@+XSgAfBTz9xOkVrVUJ0+oVCcDyO z`z&1dLZot(ILW>C-!MhPaZ8WrDPFMkbiEGpo5_XBY{G#~f{9(Jyno*<*S^4H*^cM2 zeX}s3b?h6_&^*1zDpiCb>sc+5b|Hmq^(MdUPpwu=ALjxUBT2HI3XQaASV`#@;;}k= zKUNg0OiVZzRl*B&aMI)5p5T*v7m|sLR9(}Bc{ub?Zn0lFXUAtN!yrpt00nly{z_yt zP!Q2~pZuo-ex}?~CUwK=660j_)a7FTu#hcteoe!>Cj=u^w+H4jn*&bNr|ub0!X1B#SvB>=A(W*anO{ zRfYr_nagluEIZ2Bx~$m@D3YJXhAzc|wOK4*AN_)C6iGnIzg4anG-zr9SE=hxwXlyE zP3!b~Gi&&`(HlclIP%co=yB>ck_##Y-;INqk1&heq_3wQAxleo~L>ui;;jK@U-0$g}J z{8jN+f8{q|3&0_UFs_=P8E>3@!Su?1#)_vvXQ1DPygakCQfvMBZ66$7N~^20@Yjm! zQ>ZDf4 zxUK}>^DCU}88c~QqTaiw#r_XjH_ao$Gb1YFhPY%_*4UV-CF|#r-qhAFBY+SE0>Xs= zC=sMUvvZ37HG6H}YkL%=fzfzwRau!H?%VfeT@KVY048faq-n8gE^Zq~ca3JhQ*5k? zJ+%)iStyLa-LtBNM9~Kdn(Y6nec>v~W_Uv-f<>+bK_H*TEJbKb$b9IGMyZ|Wdl9&! zzEOj=SOl708zhHo;)O%_3AcG6PT%CsO(in}Wil2iy2T!JB|HBtBpej)& zK|zbk1U*Yhl6A(on7Efs5sny)89&_mGuSs)d2WQwUz27^XKg+dx=J`qryzr}(K>OQ zboB4meqZy3V;>y7fo6vVYYd zy~@i=nP{=2OgQXwo6Z%zVG;T6A4C4-G&Md(0SbsH4tOOQ)czn02^oa*F#VJB=W z1jQXn9~0#y0I7BPGmUSd2y!jz(MdNkB<61hEB1Alv~A(BdbW_01>{ezswl@5DYi!o zIZTk+*;u=Wb6&}v*k%ptVLG3nh!C2L_4w%6aH^$tu(mKt5vuvJo9N>bRq$?H*X07_ z(@h>3zaUGs@k+*3d-Kj`7B!Y^CN_PivLY_Cv}#e!M?pe{I2ew0fFN;xt~MijSdS#JAhS*}yqsOXq_f>~m^c zS;32fR+2+#=E(OxIA7e}OhYeWGh?uwc2iIx7WM$6dQL;F21w8PM2ggmvqO+f2MBI0JwgC*=d+l6Kg$&qyb zdb6K!1r+SmElW~mi;g{QM9hQM-`B5S(Ml`)@Go$w-T6Uf|DM!jhef~S=nrm#yT$n4 zlpC|lt8=E~ZPGynN_lGuw5U|bpi(y4DI%D~i-Si%{!y0kbV{%?C#fAy%ISs{QdK#V zVFO>^1}MMb6`Qx)oj0?ZLv0RISIK^d+>U6%HKxl)gmv#y{vE7#4V?$zm+RLw65ZJh zXy7e-QJ$Q2wm^Ecp8O^$Qbg}asSZj|8ym}QRuzXa%LvoEgz)y}+%`4jIRKk6F3geJ z%&=^7`!?zAemEIFSYAu2>F^(Nmmk{@zSPCCbS=Ne&7@ z33r`Xvy9@CQj1c~PmRYZlpL3n!#}G44yAMqsSR+f*mES2TlklZTG@zjuBHn-(iz}G zS`9%WRs%!IX?C9?MeODmzzyOIXY@2E=>9o`4|PDkD`^V!@a;h1ye8lKnTu@vQHbSt zXKrFr;f|2qS}X181aGR*CjwvUT__79N9meq6k{{*i&u_=S|t~09i>@<6LO`(GxXaF zYP?6;E}LkGIOF!3>Z#ZfmOW8>%#2ah^^<9WEZilz-twwjYB>b%LOgH6rI6e=?f916 zfk_M7cvn|ng{QPP%BzlU_g6_Uu5oFA$+FePn99aT4=5aym$!#v(enX@xw-Elv7>r~ z7f2nUP{{Y4dQu=eR=~ZrQWrX_O7y()x<=2_$+pc@?+0Vs8S}t%=aQjlSS>C(1|!p< z5TIH|b13%~>{UYo)LK}n6>K-iX|@SNjX2=`2N9WK;}SY?mTd@rOwPgWYdEf1RlS1n zNm1nCaNMycP%GK89Ut^3fE1VNCx zH>#uVwxc_`MFzY?6Oe9D zXXM`Fe`gqv&-2TpYxVFah1~lfx;eLUsjPDtPo-@SxrK!^c?5RLFe*xvQ@6L8SP_DW z+?;+CtRIOB($Vw+!HkGHz$6<0{CmGf zZ$6~4*0}PcvmL8Nvl*@NxYlNUnt!(J)Fy65J%fjcEL`QY-X7|_Sp>_a)}A8O6fV00 zv{u@PR>;gcf}|mt-xYDfY}4n>fI3__%s{umF1%E?XlHaqSZ8|!^NuK7WV2o@Z1NOD zrE18c!Eja$1CGR_?_NLSui=M(ftaS=_FWz_7CE5dD6R;7@h6kDLEV)XPZIg7$GgV4 zJYdY*^;oH35&gy%sH@~-X@1pco~lV_qg~HJ_n$!5AE3g;7!B&1x|{aAYHIh%fF~$P z)pr&C#ax*^u{-QDl2hw|KFk729gvmscwr1(06crRTLX70EX;b^qZ ztSbU^0&hUq3d5xN%-E@7l@4uy#_Vb^sGF+5y#FQC7`CZTMG$L2vlqUBF8=YP-Qp4L z7DAKsXq;r%!d~i>I!MU3Zg{khU6zV>yfg6qVeHU@X5~~(%_CfPM^{PO`bCcY!6%Ix z;7+PnaX@ksj<1sKov-nXZD^6|nI)*~ON^dqZpFPOneEdPL+XpI_Tr)dKa&5#kMzI1 zT0CL{Br-NU`@Gt8n#rOgsdjl@^mGb%yPf8PqPkR%rt$;e2jiDqv$?f`tz4>8#@<`VT_? zC-3=*^D(5DojS_f=B(_tLmgQJR+g0Y&+UL?3M4NA`7>_Hv+ifQHHOxhVoR3Y*@^Z- z)(Oj?gRA)xU+mL5AW7=k4ba8^5#Ig1>_VmPVz}pEcc0Qdh~RH4UlbwXg2)2vkq_sQ;sqV3xM1Ewn5i3w0`GGjLKi|pTeJNV>r^~^g5|; zTPsPTCa0L8-dG`e!&(VTo%;I}`?(C3rBK|h&Mg`k>_ zDPogSn9G0@o^^|oXAQM|vjP*En#@BQPp9AiHK1NQ)V|0tlLtoSwe0|-u0lMYb#!kZ z0gkq#YP#MF(E_qDWL4K)AVqS4eIjFnTLd&NycUP516dpt_u$pV9O{L?p}5ar;X|s( zGlU(XOs3+H(O|_aSHH@$WJDKR8NXo&XCJq?KIBTBwu)US&|a5G_V_{FU4ABU>S$m| z$!ObY&{4*W^s3(di8@}Q)%luaWd!D6Yt<(h2zZC8TUY^Ma4@?3qkYP_gC2pXm z05d?$zh@T@rJ-i=06OhSzIxN(7S#~AzIrU&!P?#q6EGB*6Xv>mRRF)a|Cs4VYCH(Q zRAB)Rl}`8EGOOK?j^rZ;HD>!rFE2X%u7RbED>uJd>sUpoR7$Q#H+8$CiXpHfETwokUHo>?GA7Xh7x01IKu}%f7zMC z(F15{biya-~aZL@ZW5|Vc(uy zr|t~A5WUHvj%kHvHi2XoeH0i`h-=9m)=oB}H&NG!+G{7LXwzw7#rU$M^S-EBx+gV_ z8|%uMk@cn(Km`VaH3CtTpklH+pWCNW<(3GHf>@r7Ewn@}hTp>pUXYVg45~1k& zNkz&1_ace&Xb{p_X;hfmv5J({l=GQgNb{{^LnVSirlKzgE()i_>h}ey&dgd-Mny*xd}*_HCcAAiY}I#Z$}gI3lteu65q2czyo%B&cM=8QTTjV8047; z?Vl$0sB>r~LAK!+xp0fu5;0o35VkG=Psfrf`Oq;&Vskn<*>ANRerYeou1Q}y$W*5S zXwQ+)^WrdL6k$ohAUsD2UWkOM%O9@nLTNH{>WZN^eqdkjIUk&^ZZWAzDiq3~r=DNZ z+tYW_njr>vkt;zy)pVNRAw%EK%un?Bbf-RxE(nq@iidprrPwDb*bmKcHe#UgFzfZB z9G*WhIl){$uP{QCHwY!P=*`}j;{LJo1i%`G8;H1?T5}qjeC!S_j!Z&TdIA{jWn;Q% zGb_N}ssFRlejpoTy5cq5PwcdZC!VAp6leBW!swRYZ-(ZNNpZvrpTPJn)as`nw0)3XFugZeau!W)G9E*Vl zDsTibiGdiCZEF0;x*%Z$QXUSNwmuSynxHX7{$Nii(fjET=MIcs@Ezrn`&?qyk&LId z&pd5NH5Z8r15ne+Qgx;;S_|3tkgKoIZ(PI^ zy=Z`6MK%+6aEiL@t=%(ym!i}}&K6FC?qRA_PI=uVn*7P1D%+iY2k6oC9nDjTZT_{r3{s67k6VZ225v2q z`1(9vYC0=op?Prv+>uxnNjN(~tFdQbz314BZZx1+r|q3*$E@yfno{@fh$?#@aWSE6 zCZAevtVqm?B(V>E=M!3flw|1^2f(WkwYjOG?aiHLW+^nhfci;k;7Q0Fb_`ICKGaCe zCc3Vx(T#1Jn$xqb5JURztVppH3nK~e)|M=ANqtdAMr_|YH<*|o;E-PmrWf^_)u2m| z>{7k*>^;HT3jAYV1W4|}IYUbxFwgQLCpFEnlS~CHi9whcv!?MXvD%l>Gb*iD3I#)F z&-ou36U4I?`+}*axW822?KfYhaULLndy| ztpPpmm_B@qzDv6=>fkwM6%h-|pol-LtspgrwFMMcV)u+P33TbH^E0Veu-uVPUf@U( znX?dCFxFj{F%&p@9qk40;W0YkW=416mJZF>EY88s%uXm8s9l48va!n=`h0RgxQ?f> z&7&&@{o;3RzmX&R1KhSme(T=fRq}vLcqyO|plM5KFbX1G(H&+q$8|E~J=+Sdy$`OQf+Y&xb6kn6B4GeNGgYMt2Iz*jC%ZU>5p6gZl-q7T;b{rXnMRw$=k9CN@zz%m-#K+2e9~F!j2I)!vR&kT3e8 zQ1^-qd1_kX2N@mhyp)Rk@t%H@&IgCOTS(8SgXHVS&H=lY5`e-2=t5ULN-F z*BaQ-ZBj(0IU20VF$jNdqCDD!tEmIdPN-VL+H3-Qd$$)Qjp?_LUfSB_)OLflqDiF4+qxDRva+!~!$-oFg*zJ?;R zMTyscf_~sfy42cy06BJ5$!bc#su9gGH2#{XWt4j^RD$q5gUHy!*0}0cG`RdO%PP0F z16mvc&-+IV1sIc6oahm6U}A%MWuA=EYlFfL^{5-@VkI;` zhh;MWSQ;P2SSZWO%Nd1?>z|qL8@ONY=dBVF;XtCZ+h#_Nrve^us|C~ByU5vawxLE225S%YyaxjUMSSR{$&_m-^( zaP=1UhrpSruiCbe)9bvCoJX%rL#ZAH9DTtvs#2i!-Bq5Tg|L%-gdGirA*__-0}hmk zD>(c0EVbU4MH|NJ{ipD08HO+n>IJnwD<`#caZHdt?2hecZ-P{imgZnJc0eoHP1}XL zlj)NAnNkjb2U$zLdZy=Po2U^A?7d^*Et-77Xd=ftC)p5#wAmsi z%*s!7L{ZhnGph#ZX*i*=8$uQ=I_ecQ$Oj0@qPZtKU9dyv#Fk_-WXVYzKy~yhhlCu+ zTwvI}`Uk^IDHU;GZ5ICn=J~3e=jL%&DNh+EE3|$&Jw$=mpD=AMbCE1yl>5yQBo(Lmt(vg zEHY+X!5~g+f@JZqr)H{lU?rVEZB;!~1@6l3VH?`EbnzaPOH!%v{9Ev4Sa`UgHku<3 z(wFbW@?l}u0SKvdQ_$xOIUX3dr_|iEOAp!blDk9`Ei6f z^{DoyAZmbaj{}Fkb9+eJu8+{=Q*+1a-CYG-TQ!t(MS+l1c9U~*Njq_N7CsF;5IhJ8 zI}b?0+pDPJZO?@>n4UVh-6Q3as&J)?TIY?4YdZ8RSG-U14uy=~i-1)Q>nfQTK#vEV4CE?>O(a?Cj)4?n1-}9Se!6P) zbX@gX{o*db>*%&OamU`LK>E zxe|B1l{@&1#C6lTNAIOhh=!R*-87vg&n%YGo!m&c*^BB({sdb#cBdTp!+4SeDst+3 zn8L}~$*slh7$j$ip?E7)B{)-|ebjzxF8d-GW&mHc&{b6t!q^odc|8=f-VU-Cs7emZ zc&^Js!KDllCi|+|Ncr<;g=D3L?Y_506^pgoYCGH|QXR?eR!0oe6u=mkCZAV`)BvV& zC!7_yt9sK3PBB@Knuudc36mqHHg@0Ux_6k0wlGkDz+EawMGeJxdtcT02nLp_;eHT> zM8D%g6`4n+uQ--{51KH6ETqJKtA^Q}0znW9&~5Z+Rh-L8>YI)-Rgl=UWF-iUurAa; zOv~JQdAV(@dsiqs(3@y&KY+KFL0aXr0r4#B??FNiDcVK`JJxYFMUFFRx@AG{p9%pa=`A=jGin7M&`|%AXlZT zX%u510Li?)*kF4%alNNVtwFWUpQHqZWGBH+1vb= zJk{Y)$l_v5AY54Q8&=iR11QLS%ZZgzzz)Fr>s+9*2(S2ch=n4TFx_F&O^|G!u_P?Ven=<^yYqhfFnV)0zp)<37t-p z*=AeN?I;utQ1$E?$6R0ce8NO3scJS?=hmqy=_4~RZ@gHcfQe*7at)W8?&u$&BC!yI z0XKxcO{-oX1tK#CA8`&9GLeR&!v(Jdi}#!6e-3bo=O5@1KE;=p;tEo+^9qn1-J&_@ z>tO;&JvCw>S;2uoS5GKln=RlUzrKh4B)ku2yUIRu z6GC^5qz%=BXl$*J(cU|%h{=kmbuos}wo|XW1TuDtu5g|{X?klLL%r?FbS^>YtVo+6 zNjI9dzNr0npnxeVi!yFSg`MBki{#@3mCD93Bx5ERp=ZXM>khMk#kInLZUHuq;&XsD z15QZgzbVY{>*_n5+s_A0X1Ns2XJ_YFG;>Vk6wc6V8x@}*N9(ksHMN{j684@H`b6`R zO{oeau<8^T@TS4@H8sY+;#JTvvqM$k=mI%dagpS4wZnmP;Ks_K+T}bN$8iX5LOf-! zY&`ynT0X3Sc1+$a?kIL5w|rf@bDoaKcA^J=uWju!^2vrBF{r7oWE94^;7er#+<_I4 zJ-}|!RP2W@@!^dxDZK$xm>FRu*Y|Yd3T+TI|MR}-1ql!Uw?!UV*}$XDKkYT-hEq7n zEjc>6Tc)gQ#KyL>Xy-K)Y9A1dpW4cpa;1I4W0#nOCLUE+itJC(Ie5qBQjiR0W+J#&i`PqqJJuU4^pZu~D+;GP&(w^ocR<18N z6#$Z(1R)DMgAa*)F0Id%3S|GqFJl=AF|aW#OSZS}c|%Ww%m6 zvIuV^O2u}N=mmJMJI#`DnbL?4GJ&#A&op?3?&%JT$&14+(ATr!dEeBtaNO-pdFf@|G;lpJ3o}a(_F1-5=!UO$)o*Q5$qkY|`eoEECdE|gr z%-`$2Uf`{(H=!XNUcayrrvi)FF^1E6$swBuA)x!4?%~chr;_siWTSS>VJM z9|eb}-8!i%2)+=GzJ}(^LuP2IO#V4RgxfF8-Hy~T_`Lf_O?kON7O=*x&}bejY_19k zrUs4A4FeQ^O;CCnXs^42L7L(!sFJ9M`WA;-lV$nYe5`E(kdOiqmBQP4x(4COlbZrq zI~ub#Y2Q%#J5?JVPI6W%U@CeGS07NpXjuU*xtDknYh_ELvDK;QXw$h^fV$PVmcjt7 z3bls01Xe{G?wk&T!xc7vG?{daQia2Locw-$45{pk+crJ1RyPbr7cnp7OyO5OB~3$Yhe9B1%4((-+=yt;)(=vK^(P z*qGwBbZ;Qz5L`a0*PzE7m(L!8>^GRN}tJgMFox$rGoQB;(IuTq}!T2Q`& zFN56C;vvA4MWD$7@?GLe;zz)gy_EgG+K_Bf7Yz>MZj#?yQp8gKU@Wavp_d`;VgshtekAS{N$ZfqkNG=+D9*<)0}@BHbI{Bhpd5LsoxQSkfJ!NQXE+O{zbX zznAP60Jp3UCp(^~XNX#N-u+QL?*c5){h%SHLXd8Vz87H<_(rjTyig4a!Zvw+?0P%r zJX+5GthU@2Ig9 zaXK+8{v0$oK6&?hn=6_~>6L8nST%c=8MZeY96bn8him8UO)C6yPLys(hxTt?f5U<5 zZ}@9I_;$k#q)*^pOQGMNY#w_7B`5Z~#1Tohn2JDp95DqR)$Ao|PX*B8q}ISqGT!Ji zU6xn`^em#oR9#>8dQ<7;&Sl$xtqDH50N+Kr` zctkN9n;k}HXp6EmEsx0E(Jb897eKBMNvPrOoIl#5y(J+g`zCU7b1kVeE<~khS9VnQ zazjGZCMrTP(DWl>uYsQC86SjCCHS7fv0kW)Q!DD0mEI0LSvxu!XXCRSvmLTa`Gr-` znmvhZkNE|9T(2J<2Y}PWBE404h7}p4L0~SWWG@0g;_MX1Cdw?wU$SVVb|FgLc5%1xCwFYDGz0G z*c@`NuVlXz)?yDYj=b|EJ$+~_rX_@SKS}PHoV+(69LY*Zv3frytMUjU^f7PvKgjtuvcEbk0=hgK<$SQ$-RM#WG4^gBl*2)<-n+P${Q~+Y z3J}FYSR<_{w5|j&TpL+N4A*Q-=x#DwPY!f{( zN=R9aY-3Y&hM`Di>MY8cQJ^uy`=a`oDAvoLws*J|$6CsrYvpV&_Kz;uYa(!B6 zs{AWMpCtHZl~mAxjhhS{+K z&Sf*K>Yi$=sqDr`R!*l_OZke6`%K^|JC;MTVb_6FF$Hb?q|-Bpqf=mkv}EX$W`mMO zXdgXZRNre-W&RBZ(-Idn`!qrlv3lSgUt#c|MJ#l=0NnKf)jIb|f?{T?mc4R??H34? zAdw=Q=8?HsWE=s#DJjY(Np?$)Nz=ij@}QzvKxn&ut9o#O|8y;F#OJ`qtCAiTv8Sh?>K3j^ZT!lJ+u$=u!DkNnI+Cq1uE^KWY21} z*WB2#hk&XhwR;^H_9{R#6-iX6VC+3y9cl|}&cThmUq#Y#%Wps4(DK_8O4N^o= zygmVvRptrs*O{5q!q-d&;WQMAeCsJPE103Dm5t= z*6zF30e5#`%e^fr?@Jf;Qj}3P$2BN7y5`o?WMn~n!{J~enO2ZaA7 znRnKw^6C|FH3^DzcDpZGR~%`5aZZRZ7N>B=UWofIl9B@d+jr@fpgPa18m#{$m#4fb zGyv3mcq5mlZc)9(_UB!P zW<+k`QaQ8@yk+c2M(CHwuCwFtGxLJ%eatV91S&ct++ZvfJIzjzS2v43@S0N_2usmX zB7!(VXp^KkdtEy|ZeE;ydIpM^YE+UuRV=q=oiy>7YA^*V9mTLbPUxD;Dv`5>2jjSQ z6uw)CrDwvn9QZ>9Y#$H8Ugn zVI{!Vfn{9*iQROla^ZugW7`)#bc5YhjUBzXt@wlG)04&qyCoFX?2?uZ;2#C!sp4@^t>Remrt~H&ie&#mH zVYXnmaoag|`t{SJM8~GRaElA0a=Gk5g=h1%Sh7$Oou2!C&sJi*tS?^8sz7x_wX1`*1I9C& zpcuN{FDK=Ytq6-4@w4z;(1YInhM)H^sP44Xov7M~qgg^_6<#@>W3Xm`+^%P*5WUsA zl#I;wI=$6~dOYd?nhs{^d+d0(gas}|ob|#P4a`GIoyKd?Mg*9o4UZZl3Ch!|s{u8pG-1%ILVt?JFORmbdBZaxzb=L=(vxMX zL-7{wAQJ-j7hY>qe}GOR_txN5E$YZ>N*2merTt>1j^EUstk`{$ZQ15!=<<`W?KDmL z7CXYT7P8fbq%GCNbD)aK)PeymRT9?iNGJrG^aS$-4)zA%~WdUU4{J7);t;RG3FzgrELMGX9_Dw+g33ykVvx zpUGwM`YUXVtWw}jU8%`ku~Kej+I&`}t%trNp8zbXZHr?DaF@pefYuN6EURfc)@%z6 zz(M*fhBuXBa@e<4lOEX;FY@+!5;WynjAecm;u(2?GqQoaU({r=NJC7_hj^k*!pIW1 z4C~Jy)>Tk{7XIwiKL)8mCpw;&lCb}x|%^-FOQFyxq;r9CC7ebl)k^)a779FFVsR(cBQkX^aRF=PV(*E>6O#nyJv!tV>8^e%FdQV4B)wSIN);r^ z50%S%VENt1);YUyLGgG-$>Uy!Fk>fLr&_zh8qXSnwkDH$t&+hxBX!>-byfY^R$Y>) zxRHe?@+%}o8`DM+HAT;iLX~$N_D2BHzvLcGQp9Ke(bq16JS{@P{FmxbSyWr0m#{zK zMPT$j0C((n{iG#c6_-lkLz9)%5Uk7Z?!{ZPWAc%eiaB&E2HYbA9YCn`gY~qdMGVRy zNKnTXIS_qFO?iOk&}5S8St=MtyS0`FF8S=?9@K719Tdy~IP3zMyLcJ3RY9ak*t4S5(YqIh7Lo=YhFffzO-f3E=gBF1%)KNzr;R!* zlY8rmL#uV01N>~=O_LW@Zk zIIk9%pz3<)=Kgbd{Vf0KuW6qJDt$V|n+uIum*`spEx+Uqq39BDlGw190+?5#@&Jb9YX zCuA9dZGeY&>u^8a-(;;Pxkkf)%wsno*_0nx}KedxeA7HNc zaPc5W_=W?@HIfMApY9yx3}0PFV|0s>Icm6ldCBn$G(x$ckt z)vGrQ5JBL@i0qHTl$@f<6qZ7Y&#h!YywVl<(I%mGIILjf|8euD9mD{#bB@R@OuI|{_ zI)hBtuyzWyR}<(K-Lp3M3eS=$6z^ak;9yELN%d8ARK2Ko-B^5;7qmSOnlVzrSd7cW zc2u`(+D=#e${W&spG<@PbUQSkV2TtU=(gB z)mh|Z=7)(NL7AVHE??Oajg8%Es$kxiR~u4+&FMvLz~28)rkOTb`-mg9kFvg9)A1(w zXW|6EvmdaGD*r22s-5Ny9BuXY$nW9B=Z>|`IqdvOyizm4HmUYLI7OP4TTZfhg4FLy zm9R+A!5i&Lx2j+!oFFyx@j_~SUD-m{{}0DMU{H756ITk^`Okcg{)*sY62ECW_t3xlbfV(pfNt*|&r$%8urJO8m4g!->7`+i4CXnQ z>(ab$lAvg3T~npMfkwvz%?O~)whSD*>{1leEP;PW)cqdB(UjY=(RH5GeUUU&fJNxs z0tsYDflKv0GP-+P`e>;59YvzPvx-w|2Q4BjO_|CDZsg`IbX=SHlTYS&Gn+f1uXnx(_;CmC$w4 zbfY8|NdiuxhQ=MZEn_%8$z7oW{PuP+lC0f)`zlLKH_50~@@?u-B~IRn1#GCJ3BhEu ziE6LqtA4QQzs6ALXaQkcSI*$W^RpTzHl{e;?ng$-%L^N5@Bbgt3-!ZiJ{-rD;1sy{ zxKF9%Q027*1OXtjU6FPc1JI#|UN<>>x)WL58VnQQf=J*%C~Q~6e=7#j&Y2CKx5HOJ z%7#bj`hc4!g|%s{ULXbGs0X+Fxk^R%Li2_cymMTW%QK^2tD;h}scdQ(TG!|a3VGnf zV9Q>~CuNcPp#|W1I_H9`kJd+4A3c(PZ`y~c3wZ~Cpl3(rp`BN$dXtSHHo0(IFa~*? zz37-Z$+R|*D?+JWmr~sK9r^39dk5YtsFVF`js+m?`rN3MjCippIQ!Av~A`k zvmopkPJ|8Br3c2$fG3(&czY&vtd>ok4 zk&2*_a7>>ap_WImkJ!RU1lMaMVu4AKM6%H(U#4Adjxl$)V62B}`whH{?9So+VMyDo z+=I8I#(W@pF=BKk%4g8AO#JV;kY{|?IfsffmY69EMjivO3vf&xZw&HSk^y^VbD#?# zBFxCuSsm|9O>?9``nnLOWeL7}i`L7AfVktXTz&(UJMDEl2r)1s{Zi+njQEoMZpwEe z>{rz{EFg|by@cLGuc!-@a*xvo%=qEpg1g5r7Z!z3y5p#irMIR~AV=SU-oP>OsmC0z z@G9F0T3WY)cYQ<0z(ZXeYbkNysA~%&+@$WTgQv?ZWG-^d6fe$$JyABQf!e^6*_9;6 z@PG_~-ZIK~rG-&slp?{XqK%)ati7d36R5C*?HBp>^@L$E-8Hs*1OGs3C%}K9f=NtH z^FU{V+EMg6-tx@PhEkT8W{M=eqKyP|;z8bjCR~jVJJ<_IwglzWIO|pVp!@v5$BH zXQjw)e1KB988j=!v(qai17gUkWk;uQX75d{<&BG@F7rCo#i1A}HQP^PPK68)G%Bzb zimpYPsiGdrGCeB-$<$6_cm~WHu~+NuxpSeeM$<~GlvJI-NR2g4#1hqrflO}6yD85F zcR=6+?1dU-nslRDH$iRwf6@i4$~Mr}5sr%gq2}+t!56$pHxaUo`jEFG2I~e~pQc8$ z_OojoZt3iyLJkd_3aMoy=gUEO2Z!Oo3)1~~yD!@NcF2%H96*VZW`unyxT za0?z4p!c8bp5w6gGf%c; z0cF#3OUZ7a=~=k+-g;C~iP^x%%m&D^_mBXZZ-mi~ox|pa4txQI?+TAP$=kF+$yP{7o2kvmplw+&I$No?sqbrkIe2G5-Y{Pk zNkdPZySbL7L{F5Mv*_8TUUOxSizjT%lTmF zhpsD+CQ=TA0v87ZiS&AY^9l}Ns=Bfe`t$NLvC@55wq49!#nTQYT92JdO1jG^)Vx?$Z3w8=UqOXT%la1cHL}=Y@ZOuLmXupG z6iI_BYG^|kQ)&Rml|T(J=obC zeJU2R5Po5NuRE`J`p&od5TAW9frBPNF@^^SGID=YJfvddvknvQm67(Xi1Kl_(b2E$ z>Hu`lmHYGwSld_KDN%Uf!kE{tMrshsyaFf2x(i$6P`*hfhgB3!a{8)z4=0-_b58FAoWzv)Y$7{;r6E5yZOVasHXc~pmltJM8`SLagr#g*cS5`Ycq zo0Ndgw4%Ot`fIkmE`9Je>0lGsj z%CRdd#m9`9Jrs!ms2=C29c&l(t5cs2Qe*SfKMs6J6=k>4Dv6vyi9ZsqVO z1uXn%+`K7?!*O5}w9{k}F3ELT;W(Z){UsJjbGb3gE;M)namWkN7`r<~qTyN}dQP%k zI-$C`(w9d6PIpTf6S;p$$_7SuUN&C)hGiK!2#94)hoFx^)tR;n%~w^Sm_b zHf@UM%TXNwPi8Vyfu!c8u#>$!9_n>T!jSdso$Lhl3stvcYJI95fqOUJUA74fU^4tv z$eH0m;+!?4C&Ue-AcJ}-Z?h{vbs&(?>$DlECbeJ|U>z^uCt;9O#$ax-Wl>lwUbd!i zQ|RS~jD2yNrC*(f<_JcDk{rTx#DBem9$R{J^?-S46s~DmURZsKo(F! zqtzcOu8?1$3)iblLS}XR2ElgX=8+B1sDh^@vGQt5k;MHT!-0#=JUGV8YM&rfPvc`S zZ`u56iS7JKZDaXtA6fvR;mF2rtp_mwRw|{gE>r0kZWB&(dY)yQGskE!JgHHP)>c}8 zTMaEa0wFPFJi~9nn)s6PbsiJL=Ay9&LKIqpvT6!}xgW<--!SR_>RqO=o!z}^ka%~{ ze9wFHX@~ful8If+LvC%kv?U7o0lOxYoATus42Rb!Cre81Hfnz_W9ICK9V5EBR}<#= z=H1J?emb>oxwhufZ}txeuJ~;23>D3(4Ev=f2))NkXnXM3p)X&uDA`_5>dt*yCkUUX z47hn?qY9tiyn!g2*G>eMxk2~j*XbacG;%R1>blb-`ut)fGHZx*zN!*{8W(TTjW|U^fe7tQLP!eMZ4{jGZNTY zfB=!CjWmdw=5&}UR1yqe9pfEtRs7Dk5oq{SE3@6AvkR2sQs-Ox_5QEIyD!vQ9%4`K zq#_$jr{FwW(?&@*2%S4vq3M1wW`$M(j;dQ?0tF+N=gHeuCT~GpScgU%?BF^4>lG>H z8WESac$MEKm;w1llz`lO8jlg5<=POj;9yPi#@~e3Tz=-3kl)uSST_f>`;BVkErz&~ zBEfcTwrzXhu1HeyyG+!Fs5pJzI`kH9-D(->H9I}ULQ!9+R!}lw%qKBS2fhkZnHS$0 z%Rvac8#Jz<>kA>B;6hrq-jUFbP1SlbWzO59tplf*aMhHU>kKv{6wA(%t2k()XJuXK zo*zw2!zTeKmKx6?Fy9P!=Neg^`uZIs{)VPnQtU6(zGPF?K1)KSd587`L}r$!o^U-^ySL%BAyx9rxrA=Hd|8q>7(OHtNd@dQvwnm{=V&H3V3l z$Wn(3{rr$M>LsUo{rN9f#-bXHO%_}4KMwDH1E`a`8;?-AmlW3cUtjNMr>q2hw|md$U{`v3w=z|K^Ra5*Tl zh1RV-E+orPTD7>-Dfya3!e9p?Ka-Wa`j&R5pUHpWXBJ=zAE_HT*MK5e?Y#?dkk8nZ=!kAGogUD5tf9hS_(FZPxa|e#Dfj4P?t{Hg z&(7UB1{JriO{)NHh;0ACJ?Q>-2_RX?W08Zg-KSounZG?eymnATat}!wuXHkcfKjf7 z3g8)Yl&R86QoJLBp4|y$Itpoi5FmF)^o+DIfyN;INpykdG&MbnTm;?a(q7uMR zYgG)HT15mS4zH9ZGU6Y%KA>H_?vW8sDm9y*+h-CC2p@l5uANF~Iv^Xj$!0ZTqs#=yah>%ZvtyRE&?NKCU5+&~pyZqTDH6XNk8U<& zyh)|*{V`=-v8ZYat1Lr|-LEe~eBM^Cc}Vo1i(AN@x6D`=ntBFXOWjT;Ts6kIc;W%nN3j)& zLM+Nw%r18AGqxT4C09r~mIZ$xy+^lfRx&_W#Q5w4s9;2Z+J9cM$+0R>o;VX-g}s6I zeC2ppg0w7&oqC8183-uqI1piPP1Jt0UfewvGBLnGXx{y2`7hxAP8W#9RC4CmP05dj z0aGP`1Vf8C9F4-ipy3SINj=Pz+VGe(EJJFZ$j(`HCIBU|>TW-(E5g{8id?WK&f=9P zb(?1~1g#XZ)X@8GX&e|sm!rb#$EZl%hXetmuo+dl)_qj7Uns{7)G5rxbUI0vZmr@5 z8M|9W{O#*M=s@FdVX7#!)0-@b*teAt}Y|&U)CKYd4*gJ=e&9bdW z&k8p;%(LLU2y)B;a9&g#(}(^4SNOjz97El>Vsrxk@L!?J&bfqU*hTkYNSj80Y)d2| zb{NY*J|(N+R9|Lx;?x;3R?3J?&uOWDppd;n+;5SXuqD;$NEx8XG;<*QqH|xN$wO^t zds^Pbot5r-#9Az*xb(QdEmn3ItYgL*sg&#}CCzp+qNls(i!(M`j1*k;h=~qCX;ztqoMV!A>g8mOrc;*&<};d&})o{_t0?ABOx(z6&AM$VyW(%tW%O@{+Pp9wUy6 zwZm%JV!BG=+<9B0szaRG0q}H>yxhoKsQiv9(;6PBXt2OPeN3e6OZbrDpRu$B0>f&x zPg%76^Qo6BuN$#TTu|fXK?rLXVcO~uO#vqbiM&H{rbOhCvVQCKU@D?GO-^d1ewfY_ zmPI~)eS>L(*)n<&haR&XRMm#F$1>m|?>ks!j&}lCwQ)`{6=r!5UHBg0K)3a?1 z$#ERk8uHjx?}hC5<3lzmYO_+lSs7Mo+FkW~nI^CZnh1E^z**l^w%;JY^3%Cdv>sXs!le>?ExT&GD-CauIeD9t`B1sgUKaU2hO-PZkR79b%XV`}0I>G$0Lf z(?V%i?ff7FQv&EC6(`Zz1=IFfb?GhU!ecpGq?@IbQZTvq(Ux{MpVWFDA}@BchX{Z( z$Vm&s{pDZ;*q(e8BVqo5{oL?I2a}?VyoW}88EOybE_iX(JE5REtjBO%M+bRV==e6B4`ZcRQg|BP4m&u}(>vAh;_aq_cs{(+ zAr16#Hc#frA-~b+gfi00Y-oLzUQ%a~nHaAYP!cPW0 z2CgT$^cX!ARNH_WlEAYvk9!U|mCiRMMx+l7r#L>Gzo`*vUX&g8EOC_pdyl7=?m`uk z7wc)9yOIz>09`<$zlX{Iq$&}uzUr#-jE?11?yX4yVx@T>ob2Pu2oFh)N6*IecI)B^ zl^s1*)Cs@|rSiNW@?2n9XC)GZ;@~;L4*;N&=)NMdnTDB8WZ66-e@vCzhg#cJ?H24g zqGfuKeSby9&H^*DC^)3s!tgjPkYP#Kte%;V;84jGNubY+?7z6}LLCO;Fh|Q_(DS?t z-ed$_OzvV%9@_$AU^vupLz>daq4?Mh;mYf8?&KYIOz+>ZMH;rIvHbn?|He4sjN z4|r9hU-+~WNv<5Jep-FTp#`l%7BvO8yCfXu>tCV=V7~$CU!fg6zNoWJWN>*-jJ$dG zG!_35h+;2O|HkP-0`1BLhtxrL;pWAr8BRT+cM0$dNj*|RSfG@Oo_qHY_9jaJD& z!yd&?Gs14}v|yM;JvQ3fWal_Z7py(FtC%V}qsQ4^BWJktC-C}D`TxW552w<^xJ6t6 z(W=X&GhVK{GS&PmDa`Dc-EZx9N$7w7;k(Z`dsR#2&Ccrb2ET`dW$R4vArn!$0v&~S z43m$ed?%V&|ce$o|T(`1oCyEQGi;o=%bvtEl}BZ}jU z9jU3gj=>_u;I@O^H@dzSwNTvJ5!CMB0V*IjgY^pGfQ#}n9SKkVqV5Il{e_!I z;@9Fa__7p-FIly`95?N^ooY8rh4)I~!prenH&j?~ZZ3!cHPhp*B30Ge4yw<7=j+x{{>o+an4q@6Dn!R#NF`&UP11t32_Cq9La16kJ>I7 z$#bnap@wN4IHV&u@Rz+)YN9pw>TanaroSWz5Zbarr`hXJc(ThmV)7_aq6o#L5Q}ak)Og?oX^7;hux0JuswaSR$nI&R@T` z8;jGrEQ4~ITe$H?GU{DOQX-1M@^(3CQuEvVPXhTUM`L)S4lvAaFo}NdLG1-{$1ir{ zj9Dn!dg|)Xx19dSWEdfMu_~>@WB?#2>{uB&Bs=D)z|$zi&~~jm`Kjy%KjNqwmn#fE zvm46FS7FNul`827k@$@N9RB!^EszW_%3kTP)pzxgPn_Yq)?PMz5=Iv}FHL1Fg5~rO zz@X6HS;vT@vz5#6*}%E)rhT)APH2JpcM~^l!~9kiAIf=NAx7@wPo#&vGq&`gwXtDZlQgRws9_6Xw;1@YYPh;v%;~X-)MQsy)JL!AMndb ziBN1DM{PtFKs@vVB<4>&5u+FgW4_dx+qD9)EDxxmAh?3%y8+^`IV=?eNs92|B=wyS z$K^Uxi_W}XEoIK)adlhd+g`l-S^KPPl6Mh?_bcmmiTHU~O5o(y<=3!_>S)jq>kszs zO=U5ui1e^;Zy5JD>R7Ii!)xAy^VFyi2Q)j-5K?Ws3*>~|CK-3z%8?#QoB0iS?^{{> z|B~N66j*WFDP2rOn|OAl#KhI2#R}d0R=<+qVVoT4g+bT@c(b@ns=-3@wq$wL1s8pv zNRE9n5&nrB=!QhDaRNA`O(jcOLp=S*@aOrtFf#OEoOxHeahN2x)=P<3v@)!?=;q9# zvrD-N?vb}p{v3-eb$Ax$wlKKio!nhW03(MhgxGJD$9_zbk??$|Z_I=p^~rWhJeC-~uxhVq(G zx>Lp54$JyLB@mhBREz2nT9kE~2w+K$w%R5HEub*4&9$#)wMNzu)-I6?N#=(OB-$x6 z$B86Fy=bZ6J#2;ujU&+DMGX1bp4Q?R`ZT~#fHI%JJ(~0&d%G~e+bkNZY>J0$ARcE)$q)CXdVedbbd$$>Do(k5n0^f$*58X2c=3yB)A#aL*jr* zq(T#y>=|r<8Tt;0>{r@EX_*RH<0Fx)!=3y_&*f{s`F85iWI%cmxCxV7Q?hlS_sch) zy(xKy;Rd;1TKj+qEFC$PRhH8OLjG*ma@S9+AS;6WAwuR45B8JSGI>8fGcW&#vD!c615AdL5B0lOX&xf%O1CLW!$eY~<%26F%&0-7{Zq!PY%#Y@de3Bh)RhJ8Gm zd4C?PBcrq1OX~)ikxX{IfwJa&UvOCJOvSq*_mX<-!&P<*I=B<2meTr=J!<|j^aJ1l{pIYplwYK^n*rf^fT>j>u>;=CY6$0 zG_*B1>WYYC_*FSGR_Rc{V(&q6;rdNaxPqCO8zxwFhfHKGA}a$nmoIXDFb=zcs0||p zP+vvPwYWJ9Wt4I5cR(&pasno=&vYV!bDm4QB%%Y3sf#ijrZad3(G$=E^9OTtN^eTK zl4Iu}H9ce31R)TkE+2eK;Ds*NEvV zwZ~}c5^+;45tU$??s9Z`@IB@6O5K+8yobZAR;3aj3bi?!#XYNf)v&ifqOe0|n|mu}pyRh=Uaas!UG)V!3F>D|c)Do}3)^g|xaBlGWLeOeoXv#hZ3I{oyg z;iqJ!_G}=fS<333l#$NI?RQyqdwPnTaqZHT`ggc7B_TCs9*MfVRTxWA(;a_krM}@JqJAW zQ7+MHx8$S?H=WiP4~|@~O4Yb5bnKE9j0gT^8l<{qr{$WC9&pOveh_}Vp--`LY__IU z=#du%&iZIY9s(Rq^FKk8O48$70ro3M9xi7ezWz?Gf|3h?sDiP(-QyD`NX(HTx@aF^ zbK%{ZlM1A0WHV|%-tMdz=7~MQ%77T&`}^?SKZN{?9eS8cPyN}Bv($&Dtn*kpIV+}i zgR1!2yd@n(&RZ2&G6l7Xy8Y+iOeRVk`lB7;!e^gLxpf4moq;plU`rXybmbgs97v=? zBT?=#5Lm40)hajeP)WjB+fb`Zhp%ru3=y5=m2MT=%5=YJP&WIhM@awhjH4Fy#8I|I z!oBKTK@`ef$-Tm_BP6%kqrXxKp@T6MGbEq@Wj8!cmNdW+f;Lq^n3e8D++rT{Q~m;j z3!=V5Dc<%hIrMj+;Qrmauk>;|74e&X2R@`u75VY-VjK+vrFXgwTF^SQQa=GL9uHe6=5ZO&d2AGqH; zU~+{ZcO!mWLx5e`a+ftOaIvq&ZsJ*P6T77;__yLn@0``r`@oSo~$E=I+Tb8W72v5t5L?ZZ-cy8e-R* zQVaLq!+XR4_fZu9QVYiCgHT15=R|;AETry`Few7*<`hx^WA7P$eCh}UMmCsNphfGV zI1Q!nj1j8*Xa5X{{>ISQJEj;O5~PvO#ZM~nY6q{C^CCbg$_+Y#_X`CgB83X9e&5jA zby0~`g@tW8Fu~_*plab0Ss9Y`Uy1+PZIYlg`Si@;eEuNVZ?tSG9qt3lPxax{vP`qT zd`hEw&?50~^3Ma@1;UjV_w-uWBufvqtB;^GNQkQ-K)iSuS1m%@c-!r>KG>ex#=K{Kq&@~YiFfa6FHB*0aVSrn?%{C_~Zl9#2n+H7=`4(8YOP_ z@#HpDS1mU9U10SB6k(cCcez`P`USNYWYs;tt#bj%EQ=*O#ya8HR4Sw)Tb;rSP%F3e zT~R(1@_|3`juCR(Vf=)XzfJC$wyShBbPpKMQR!}R)?SFMy*2{;bTlfq3|*xw@033K z!Vuvj_ud?jx?y<|`OoMh-HQxd+#86Ij&{ukb%c@~bcKAir3Q_67BNbaodoSqHlhbK zA5~{hetwyhIC^954(NQ@ldURV5$YX2P~}3Yjod5n6Wi(r!-)N*+z>69`$>#X+hhr3$hDFp8y2J*^J{ zn&ARqamZUvO&7tHW6O3`$zhj@x(5RN!7`w8XG3x-nCePcXMB^Sd+s4jGh1HLP-Rzk zpHzZW{sP>$SA5NINIG;ljn1kbPTMlmr6|#64ktvi7B_Zyd06W+UqYu2Wm?-Q8fr)< zbYhEzs5k47UuM85SWwLJFi`~-yx4;f5rwQm{`R192C7oX+JSb*NwVcw{R6iKzL{Dy zTY#wYBtPR}_1ma+F%Zy%cATScHBH4lc=z}iuShou?U$vi zAB8`)BU`)4EH;_r<-=8nGImy^$jgK>9d781K^{?w5ppvDawx%UZI{?IEdaD$7Ro;7 z6b9wN02o_jZ%SQrdW!XRnj_=$ua+7$g~BL#3Gdz z+6^(CnK*s;hrfLNm;C?X_y@&;!BV#jG-Y$!eD)sXiR;w;34KnfwV+LEOuD4T)s$$! z=&N|hN_weo>l-9Ubi9<`q%t{3Df=&f7do%V#2n}axy|e8rq{7#Flz5?iKDhb`<~-!I7__2 z*l<1wNQy)wHP1PXoXo(+cF%7BU0#l%rKB4|!P;6;^M}98h;f|#w|d{d(s&^IgL#aU z#fMxHHVa1CzFYNqKxzYMju;$bzs%KwhSw6k{pcIK+s*B4iuMN+D*97(b8CVOhJ zOHMDzrEMth@pE*w#%j_#CO_Liu;I*X-iS7zl6LsZOig_K3_3}tC&ElQ!4~^CQ1oNDIM`eOB3IY2V87s_5i^Ot+@rOIQmyeVM z#PVN`h-)2zy_AK76|`BAC8+qby4At=2uKRJ1~7O;rNfxkEr%Rnl~Z!D_o^)tLw7yY z?1nDIP?_E+0obdj=b@aCyuIlznhtP;P#msYvq}*cXoeJ^yPO}1f%yk&&Fu`3)F&-} z{En#rc*luC+3uEBng%*5B=co{ZOP*F&|DrkMY%B>Nt%$pNU_Q;Ww_Zrr-MX!^{CJ! zrm8Ch2km!~+LTSN+8UK%N-of0vuZ$73`y%Hw`N0%v~JLVNC!V?0;gG0$@VnXIH=rx znN!huT_<@d-epl}!jeObj!7U%Oj~W45V+z`-+Bn8iPRvp(-a=c8%OdF}_1C1#P(!gYyvZ@5 zJLCtTHQIT{C%rtNlh!GDo%m0=VRhg1eI$emVR|h<{y9`erGcx|T4;4QxQGI(?dV;V zb5vjS>v{n;iaNDkQ=k#=XbkY;rBv^9DxprL9n76_Qz*Y6C|suZgX=Dt#Sq$hY!?FY z!q~rR>sfkX!!0+|+YGgM*d6<3olRV(&RD2!k9c{WS{&Hj6X~i8WHw*8& zd~2Z_LIL+$8E3u|0VLT^H|2c3*ynHPHMt^9&T>=>)S+FbJj9icc!^IAE00<@$fpBP zin?WskLk44+{O&jSwu9_^@oz~AQVV5_RLZ`L?Q-XCyWk1Iw zaYDKOjC|^dNH@8zhMd?r!>(o@fNJhR#fDbL_7%~mk!EfSaB^%9WV@r1HsAPupwY57 zn$|!tGTG6PawPd1?ilFT(d}F$Kk|>Rt~K+aK_9!djGDk@FY~S*_BhtL!4)+);%sv@ zCs|o|K*p1niJU&OR}H~D13yf zfLvo@C8%@qix;8Y*!9q#nHU!!^C{&pO^(khDF)4$2r-kw0>GTL29(E6Y(#ooJhxQU z)^@wZ4|&)YC$&MJJVsyn&K5|Wq1n~j?&}$c`ub>2lG7UUAtf}=H2aY)zXNPN3@RE4 zai-w4sLnL6=|C!1-2%jFf@{aHGntd59D5w%;anGF&2$0E7Vks{L11Pp-hIe-v=`w> z$PvuLUM;usQAkQivR(e8AK3s{s-Hjn_xuCaK;nNLTxdp*oqOAi;Vc33P{%9R*`7@g zzPiHVCE0NhisGiA(dAmet4gdRuDaI2L|2p9`@j+4^W-(&91aq$Pp0ux;I7sZ#)V5)BMBQEU zF2jSx89>qxW6&#ZU;45v&B0V~EiEND7&)5gGl=X=pttT^k7KW@jedzq4gS}!zhb+h z{5~B2V24rSk2_MR^o|KvZYe^A)GkGsq7jM@ExP&C>-}!U*Tapbq=c@oCEMsS{k9Hg z!^DB?Y;21zj=b?{fjf}vHk(c0nJ2|=LgEL3^WE@oGGwIkmWv%0WE`Q4=D z@;^OI?WhmpSZBYZQ|LQemdX1B5Atgk$+>CmG*4PL6|IK`RuU_4yhQ_alo*q@*hqtL zmTGx(Xc1zdTC}o$t#!B|E>s5sMKphiB-!vWlW53BGU^sWLs0)+)jG(8`O%M+>kL(j z!S|HCaq7qikN|SYO>C;~!d_@{JCP=x6)4OS4aSK9H7z7 zR2N_v+*-|P`9qLzKJkz~PO?78(E$>$Qx{}IOLlqW>n~GP`M6GAYi2ceWQt#}!O8x6 ze<_nSG`e61%o)>=uP90}s2?A&Dzfcd1^?=bHA6zhJh6IKja^$lps-x%14ixYu)VVf z=nY-@;n4rbEs_QnQ|JyWe(Hcm`kcJirA67o@*S-eo_9F`7zRL{wmWvgzNXi`{D!|C z{x!U2rz?fQil}i)KqK)LF`8+u5o?j0n3vgi#8Kj1CmuyuNSS!eWjAQ@>wOr*faUgmRnX6%B_;P z6Hc)=AVv}uBcumJpG0ad*A^6RST)~g~8j{U`VU;Xelt z$s)LUMTW;@k|Gs>fLQ2GnFO=*fsdFV`(DiX(lxY?Q+20M04!CY&`PhltOKK!fBDe| z;YrmgXp>p3r-K?9+@%6~`T6T$U2ely$f-`ftZn5P_)|O_>{|OBO;R;F$Sq|8O6L(= zMa-e!TuBgi(?H09>0I}O_N5UD9DD<+P8_Ukebr0GXRdX zPSjPYZTCcJ-?mu0$o6Ogl&i8WxTDZ&OvCTCsRo2cuie)NxpWY-h7gZk@=%!v1g2H3 z`9;d&to-jG8I)+dt|Y6G+)(>oP!@ScLV!{qykhkh`4|%<1V9OO=#;la&iWVnq!m(g z)-cnxm@r?ymg}bXk_$ekZnF#o&c#|F5jgMspxl~PHA`TmI?e4=SphNL52O+wb+j&K zuE&@Yse2bX%V6c`VuY9f{s|oiyFy9U|AmEhkZZl~&^q}JaKu#^RD%wjcL%TFa{d-6Vbv#jmPIi-ReH&z$1S%;$8D%>k^Fwvu<$w< zDhG!iqL}riY%lG)^A)BTEQH{iP5oHRtDQyDII>a&Ze1ah&C!bz;GUsM zj}hynrW*%)t84I%iW^DWP4X1M(mWK^!@y0J_PJH*(ERpbD#ME^LLf;B*sEw z$$-cWo(6pe&p8#;yrC#gHU;Db9gUma-Iqn>S!xK`_vW&-W}e<+xtt1yIO42I_ik$+ z({ArsGLrr=yID^*m6c$$>2j9NM-6dAxePhif50MZ&)sx(;*c4yDf37uJo`%=^KVVX zJF&r$yt;^s>QLmj=fl|*5Fr;omW&O7?n<2Bxd|hyPfgpP>FiB4Hv?U53+&-KTE%vg zTZ*c?#5Ai|*bJfg48J`hvi`fGMB47uLC!sa7W#2`{ZL1L+5|Y*(!Lx%=!itlM(>5|8=I#i%V8{5)22e^ z?3tR3fLjNz_CXn&T;Xe&ISK1E;MjRNWL5z3NSeWx9RialoXB|h%21?Nm~V8=r1X~? zpC_SUa8nV~@Qu1fWT2duPiPJ}O&s_^3 z)?ZSmny5`Nwn8?_YvqFJH__F|IOS7JN0)9T>_}<>S)ArTjXEzjMa=JU0vpMm(ZLV} zTICP!`F+eD6;thDrbpYRK0*Bdgc1{#i(ISetj=_~3{pp;tRE(2MqxkJeI{?YFcoPS zo4PH_F7cLn>bwfIt`JKt>!;S!`FuphegFE|TZz3!aqK3tNExO@5<(ycVVA6g**hxu z)fc&;|B@uO#xSCW0VT<&WETk=O63!6*AHAqMUXRp5vC|1tJv zOR^+Ymgu{Gg;LUTfs!hCr$vhXk6V3HY)~7*Z8|o%hljM3+@kses@`tDH6+QDKtclv zl|UjAC-T2~udRD+wL^D{TuLfW#5ob}=4Pr}_pk=XISyb4up_X5d7IV-q%GyB^F`6W z$nCbN$#Nk9Z{v{RW>6`DJZwaMYEO0h zU;^P*VBa;tONq)&w5hT4-)eB3PTJdYiji*TU?X4+OV7*30!t2dAI!+n#kxT5lSEY= z>FPw#);l;Q-nyJoulr_9QrzRAs6Sh*uEn9?ug~P5z@V_GJbc*_EW^7CXr(q;glaz) zAk_vA1_1f1n|dW$3r<)83w629^%@UEszCj!y}%OX|%?J4>jVU&@_-56%Gfe6({YnK1y^=GK7 z4i&(_`>e3^7&6$U1snl=NwAx)rxyF?R5-$z8Y|kBq){cM_sL)3dCECatK2YDFC=Es zaH&uYeY3IxXV(#DvMDV1Cjs5m=I5!_dcv!@6ahaySY{TCq8hl7*hM& zD|BGEF9Cjq;LcfRaDf_tscvJ$R-A^EbLPw-!6DfSw&0q(Wzfui*d~KeD+g_HEa5G* zZD&<{J?7C}u>~^erXwW#5ypwJdr?i64bo20k4I?bf@+)di_V4nz_5!3f4UV&AvA+X z&L>+XfoySnm84{zLV(ypqHe8;C+mdDj*VGPOViYxcz$SKZa5S3fP!73FJdWejq`Yf zXCJipY74@cT&ruC>EIcVSB1e8KAZRB3aL*{NRk(1nV-`Zz4>wjKw7cqd7HORdTDvF(13ChB@r@lSb zEOD$d_4H%gAK8^TjtJNcTGym%e&r3&*+{8yuXzWqO;$JnP_1N9_Uw{byS2FUoFNT6 zPSl6)2(EiOflbf#D*p?SsxY2xZHnZ{*x+UDabC~Up4(zl8GZ5B`u+YFwnyXmp=-uc zfE}yMCQF;&|NQMouYY*|DJs5{_wGf1*QDxnRWmv(n`SIhV@#Eq5`3*~s zbSsqiob4&>!#?k-nZ)AhF{;vkrSRA35?>+`LZ1dt=aXo2(uXQAr0^c0#5j07zsK+x zVHf1*11BluxhPZG@;<1~_Nnk#D62F}g{|sjB{eSEEO+q|w(gxDB5fZNxv8i)E#( z+>ckiLO7B@w(Qbc*FxP+y+y0a+n!hy;YH=-Nj8*&4eeJx`vwTV500b(Vk4Ymeaa^3 zr?wCz-`~36eU2&IA|*xR2OhUiQXch{{H_1nYN-8Oo`q^jdFL3C-Xjx!mKYK=ZDF9sO784 zAL1C7%X(xL<=V-M6Be9yArOZKLbsaL8$f{{9nNc9<^$Z_v3>@-D{8ky?p3??<3@b* zq>dxWDdseX{L`yxE&s_HG=DN(a}X}cUU=r*KlMpi6V}B;l3YE_5Gj1pwYflj1@N3f zUT5MD8-rkUE^8)b+5l6@CzK{|b+->agPqZ?#Yto9Z^GZWFXeC1;`$Lq(+Z4@P|ZjH z0XvELDsZq_J_}qI6;4G8X3w_*0qrcV2Q-;_1s6UnmApgeCv)h|m zEXk9zU$fk#{N1V&nLHD?jL@s}vW!}p;5_l@gM#KnNvfH2o3(%8v~aE+ZBgOPtIXv7 zR``07jF-s%;%LKDFB-y;EHVJsNk6QUMQVe=9A|Y8vAVd0tbes2H8_#$x)}sui@(Wa zmz(FuZ(oM5{uzC%gVWemYF+^4_oE_fOovuVH`!f126Mue$Gurt;}ADHsTY#`S4dTo zzvg~wCk5N>s!0*e(0J1tkgV6lM@hd74|ll*7XTl!Yk(G9ql*UsvPno+i5rG0xOaE# zh{j9S4QaTC@$NR}TYS*qZ5r4%0*Gkjkvj)qMM#>I^x%RA8lD?4TXFS|X z#~CU3LGHpSPNMVABdHsnGDZmz86e zhse@9HvwHWYb?NN?d?Y=3pRcuK^;wUgUx+6{9TTfKZSaWc?za2GA0x}C51=i{d;rJ zGV(z)3Fw5%y}uM^O%Z4=dR5*IxpgPUwB+$>41JZa!JkS zG`V^lk&B%ZSadLNJ3;lhYiEd+&UBYD%BBIcN-0}yeUhHrRLkQb`=SNj&35_qtYJ97 zB9zc0O%!`6&@4wRQTW*Km8Kqr`hD71i#B|C>WVRrmjtB%kt%u@ZrdFd4x!09i$Vh1 z(hM%uUF#SvK8GaGrE$yYCqqCWu_Or5zq@ro9`yyj!s7s!WBp=?bx8W|Gd4WTW5NV+-C$4asCUT;=abrQF1M2Y^let;_xeV%UbYS82!!!agOcd?hX3iox3+JCisL66|%1M6@=aN zn|eT43e>CWVFiyw76&4=qp4;?-ha zfe{2lvx^>E5is^`Hmik;gg&fQso#Fc$rfjHZIH9C+gXtS$EZu*PEj34f#jIx)V1KQ zLLQE07j`Yh>LhF*Ne24LhtkE8P^0#6ag};*1FXHu_2hd$4Jmaz3k#hb!0d0n8NT%` zi|l=S=HkjaFVx#4#e-TE40MK)lc7;fah2YA5H(A)c@Fp{m=$z|n@5p&q;qN_gyG^_ zuCBF#x#!}UF)E)axiiv+N7G(f#Uuo@9k=ReEjJGC!E#psQnF93z3hj5=#vm0Fe|q> z+*k}K&r{*rX_PG)<(Bog)?RJgK?hKj0Iht(0@S ziam1=R|Mv|a8a6_VDLq%21u<|2N0UUpy4SYfdA!oDFCxZ9nJf$1+}dd6y}}FX~{Y? zPl+Z$mB6e7C))6IHg2_n=+ePc#4UvX*7EUDEmHeQN1CYo=^F-xI~!rDF^-}%Z7KZM zV5r?XlIm9zV@o4I(!xro`rI;1WY@+b{LLYGZ30!^DpnA#`;t_u86lB+dlwpRso>C? zVoV>oUu&-cu=P)_&NeV7IBCa}O!fHOScum(L$DRHmmHPI5f7IyP|v73pq;g4!MM@h zZ6l2m)9%eSWy)xuB=Gl&WiOb0>ul**lhUy)U`7&^HMhw6yt})s8l58q0CE4!t!$m6 zBxSetHQ|A^p?3h2K#P-{=iye+906dPns*AbR6;k#(9tf_9T3%@%0Bu($o~TVBl!rc zAz8G_M)ssgO3FbBQ(LlNOx7FbzidC;k$nMTYRev86wrXgbhK0tfIe%SL56`QIFeou zvmQFOw=;`TY8>m@YF?^Ma63xwlhU`-bMM?Tp`t*u^5|?BfF?GWZV#7Kogj{uRhDSq zE~^o&IM+AP8FC`kQ+JLnd9=^yEV#8tRz-PO9dMg_sp7NgBJ5F(Cb)#NrbBrBQYX%t zTAa2CvmO;9wtsmF?ZwEB324Kf3IsI)qO;qO?9bSfc*%&~YwE|svcfju14#HD(Wlka zX@$lswAN?JxfIKySz}8W`|85ImZtphLnyDR(^!Vyx@d%Ach)S|A@QJkFaP)5rU2t1?0L!B*Skn3t79K&2d_yrvC(?APDv8x%iFgu?LY?l5wfl^i{J zPOEK$V_JG~_(KgWC_f3We?YQ^0l#a0rYLc^yH)E}mM}7}uJm_SGckLvwM%`Y2Q(lu zx@->>%$qEvlB{8e^9@A&%+KMcufH?^p`7P)hA1@su%D&#p{zwo+fgzyV&c(qf(qKE zc87Ye+rtmo@B}LYpfD9>rioViH)Y8CkaR3+@0PYJ=ERl6!p%N^cUYj6I$?$J$OhGsi>+wiWyeS&EDnm^1p;BWz%g~CQXl5MJ2e(a?^*NN&wbLs8cuP$XI#ZP@n!i|PmQ0YcmOJVX!%FINY;Hwk|7MA5R$PLX{-FL^# zn7VjEcvnJ|d{!G|(1x&^Q%4=2OTmoj8sD7alUj&l)nKSX7Y!H`Vs=;XBed`fYP`ig z53aA4`nr3O5gXixRJ%V+hI;0a72U;SwzXR=m ziy+MZ3iU>Z4V)TlegM1Gk5mp|iL!k(^;egI1BzE}Et{+FY~_thD~n>W7eS71>m78y z7r2D018{o$_$(!+3QX8gveJIQAt|=CvdWe#jh|%M;QIT3$Z9D`YO?Qf@t93TP_mOJ z3WjWesRJes)gddg`^8_Q`gts{Mc5q$x=Pq^*(O*DQs43#_DKblm<2E3uFXl%AF`?K zWtj{Lgy@tM#l3y44WnaIktNremDm!^-!gsQ&DL!`kySqC1(PEPJh?JJy4t4%7BaIi0pHe5~Wiu#XO zsFOukQ{4frD}U!CJ@pqATA~o7543@kYGff#a?nPVY_ZicjG-wUb}q*li#+gIgj%cxJqFWBAB7GF_GQdC~%3cF!DttVr&8@;IHH>!Oc+>6l&WfsB?Mh zjO|VkfG`#1LqPxYY2&qkD9WhQC0l+o4ZEtdduCZ^Hy+2oEk-cR(x)jTQfFPH@z{C@ z@##0Q*wnrL@}Ogm3hKBAhHHsy^ad;{wL?~ zNiiM)BxhG09oTt>!5BwG!%!&u&t>n9HhcCyd^(~d4O_|wQ1g>q%1qjgZSN#iy7E-% zGeETW*=W2ecGG(;IJ9|s#!wCBTfb)wQ3p|^rd&8Vc!HZsXb|QFKC6H}LSm6z;{y^O z)|@!pcnZnwoxL02B$OY)Ga%emsWuC4f-WAi@(eaJhm@r~&*>4C8Ym4px@@k9yf8j4 z?}pmt(kkdSWYLi()Ev2IU=`4~;q5mTIIPlKR2Lf(R4&<5L$99vqjmwqr-LEaP8V9%mS}N5VX1B^ zqKAB&5Dn$z;jJnD&)vudF+-b%@t z;9D^{Ctz|lZ|F2UAdGsm1p~>4)(L% zl{8ATLy)x<+$bdmbUAtF+4|{5<6evPl zD{l*LW%)~@%~r>f99!(T(y zQ|Seso2^fXT&L^CD9!xBRa4p8V0WTEYoIaQFF0{6|47p1mln1sXx2z5dd%EiH(@GzP=O#@Pu~MNYW}nwxIjk67P^GU)b=0QLcNeYd#1)V79H* zmCeo+l?;?K^XyEy zsA{0hz(bf5o&g+GsqzVrVhb)k2ejIFVz>fR4lN*d0w7)N)l?bR)(^e+|NZrc;qBio zGW3{%drM5tcsbfs^jD%M3mBN5V3MALdE@~g*(}Sb{ZOdUG0bTquyuG4lEPn9)?w0) zkYi?7#WPP)5iPv5dn~}mRlb5nb&th^oG`?LfR<}By#~!H)w*(I3Y$as^zS5J1S|@Q7DN;|J6UA)oC8Z^`H~S3pLyO z+;U9*C0@cNgt8COF?-}ly|_9grUE3VrTPN}I3&&5C@{Q3coJE}15O$#qgj- z-Za+2bB>Y+juiip9!DtMsm}356-F<0Wq~h&UTx^T2Z}jvdmfm7EBsSh?xpqRN1$=7 zC88G5D99wB!7`>e!Sv+`j*mRJL6Ke}(0`>jWcB_8uGjwiA5k@54tQsZK*$BJ%rCda`Ci z{e)S-DgnydMd}B|Vfad%CP#hWvolVr=!@I{QeE3DoRGTVkIT@Ow3%*U`sB=o6%_h{ zpdr29VZQ2j=dbsma(=0wcSCYIUI{NoSeumIhkBv~B7Y&_Jl!7=TJEc6$4XsmCt zMb?6&0X1R1yP0b;;yRMw?4`de80WoK>mX1zY%QuD%PH*o^3gt@Xl85PO2E^RD12yzyiMc^r&Sxet41NFKg1n>T z3Izm1m=bJ3H91$jZ_fHJ96|_ro#zfjeA&j1_z3lGV_ZRNk;$860p*+f2Ez~liP(-e-3e)NW z=(yz0QYy>2B;Bq7042(EV<5}vi-yCzhA@HX9G``Wc8PRC7MF_bpHL<*!Iiv|v~EWR z#UUrDVvOuGug}_k65CXuo}zPw>Xh25qJa1`Kj+rR)6CkaR|N$7HDhwfx~fks@3=c= zDv2_Ue6aTpehlC~c&kaXkn7Cc57odBwRn9=uy?@ANgcMFfkGXUE?@Jwd5n9UjW%It z_haAy4!{KnukC1cCH?&@!G#W{!3Aal)wGvv`G*wXGo^K@h^Gc~k4V8NNzvsp*WBoO zc$ehX&wA3PiyR$5hVo&!tmn)qWp@n5y~8e)J0XV=P;xlSY|`tQiFfF&oN!<*+Qm|c z+XCH@Z!`_47k1zjA*zuSG)Y?Jc4M3_+l#G7%Q7%`qq0;|UU~go$K~){r)F5`0y~lp zou+S<7VP@vbI@n|%$72kHuTz>kKB6 zEE%>%;++G=BsFN1TxkmqMTc4nCeH=eR+e>GTxu3>6^6fd>)qdEnc{)Jrgl{O- z9d3P-J~M!`VJjY|`B{cQVX_K!;b4bZuLY?jh?vd~D*3!V79!jsHB7_x?-zjnPi_PXsNE9Qoi6ao1VCW!PK

qdd~=Dp%`Ifadd+N-ahFSfUXxH8e$4&66BuDhGK{uLo-hgltbc6E6IT4SYKD(5_YEV(z0*O;0X3~Z7|9odddlH4RgZyV^}8{Y~l z6xDKt8v8bQv}*ZJhF#l?a6*FSD>kK;Sj0?d0v*s7UC?#sFmlyWo~qUtL$Y6?8W(h` zMc@D)7&wM-eAf~?6oo@rzdKBn!$h31;m>4ovt1J{c&j_hxQQv9H%Yjgp7b?j3~9p` z#4t3{Q-JOKT%Kfk4nieay@k7y$a9ut7+<6sw{W%!rLz{Zy{*#x%n@M!wob>-#345u20=g zXB7{N&DvIup%i*ebxl*vz{W-|y@5R>&4I;1w!Q#kiNydD&G{N@>w6X-+yJv0{K?LG z=cG6HN#cj&4962j(P@koM$c?XaV11U)**)CkTV=6Zy@j^d|I8_7-~jFhNhJ)q=)7| zc)YVum@%1n6_GKNM>y&u|B_Wt#%34XlvF?fz*+5=+M5Lp?`HG1m*y^j7igzJX`1k$ z_BT{t%+D~wZHb2OT_5z$?!I-~qj*FxI0aj(Xa*+-s`{{BNM5!UHO7}dksE8$y?&XA z^&?6$_$wLwc{p#})x_+wc{!~tmwqSR#NwV)!!V`U8-?Ev#+SnHB3Ob>wv&JtiSn?N zLxcoLo{h!Bb9uWNVJk!M^wY=U+}u1UBLPIffXRvI)T!B6cta)fwlzwUGooobs?Q#! z%{1{{9R35UlP51bN-!tD%Df8(02+sYiP4GUNv3$*(B}Pe#fBGkUGkaWmL?jW`E=fX zNL~Lg#keWR%amZ2{@cyXOW&;?Je>Buni!$#l#tpb#XU6?wiHj^fS|Pjv-&==WP8Xy z=_zzAHDFh*r;|SS-TUq9p-@hly{9o1bz`bhpBxo^mv#T!|9q$Z`~RH&!;61>dE|}P zUijO8z3`+n3w1P@<{8k&&YpQtIr`J?xanw@?TLTomjX_ zx|(n7hez!Vm43i<>HYGS+TP>L+&72btw$>YtTQ140~J|yr;jShW2^MUa{hMv)!d(c z?blv*Lr~WQF=IqA*qlBfX$5|rX!I=3&!^>vI?%T&ulL1$Ux1Xdmihm{j|Bj8d^M2n zW44%sbUFqm%64 zXXUaSFbpgWYNv(fE1pthGACcP3ni(ap{yNaarF=&ju{~qF4fkaYddMZrGELPOAq3D zI?RoYLHBw(7I*?Qr%wDo!#4+mt3t1e?nj>zJg@L#I{t*~dnT;+HT2+1=O&(=`ttaj z%*;Yv#=bWCgZ=XLmJHjoVzyu2&Agu$mrQfr35z*LDjURq(mWI#JR$h-wZ^@ZrZ2xw z@$qnHnojBw_xCW*8RuTR7F&Gum1Bu_J7$Nj1uXvM{R5rpgTE!<<{NSBdugDqTFXxh zDr9G7;<048Np$+o;@vKn0BEEvv}0cvpnYdDSm8S$%neNgw|=xG3UK`8QGW&};X{w& z=~0i~^-rNJu3?d$?Nqp+C=ed_8`qH!qg#=OURnxD!|nGG+i_k7Zjr5?xKpVOL%%7M zljkImr8vxl{$b}~cqK}4_ybjyp)j6EOX}+&&TrjygVRL}Wme@^Xf)AEw*^OfrA(B8(~4?24%mb8h*gQ`)AKq@ntX?MzMoI+yEkQ*Jy5=gC}o zWcj6^G)-qDH#C6I21=){O@i(KJtTPyoxtJ92+h$4c7xYGbp4d6is|c1PxjTX(#eI) z#c{LwY(@siamm-?!daq8c`2V*dRW zr1>2priL44Z&UJ2?<2(k&xztqNAZrU9wD4gTp!493j8X)hpMQ{~oK@b(svy=rc`*tn9XGzFJt7<5w7{vDdMDTN9w_{ujL0H>S z#qxBUK9aABbZ+7i)G_$IBdEi{;{{{Q7!z94F1d;e(N_Qu_0Z=Aral#J>+mK z#Zg5kki^-o`xa*he@lbocaz(M#2aoZSR8gs5kwC+4l#VOb#Xeodu8)&toFxpw@8bO zXo%uHih{SHW2WS7ze2}SN~r54mO0F#bCRySO-|R$tu5!U6Go5kBHjE~kDoUBvmcHe z_`^tk+rgS!>c9Wtz_)+>bs0sVfu3AXS+WwpF&& z4-k;LOol9kQk-kQ{rfaycKkt$|KO*hjpu7$S^nde)&4~~Lj15XPkMMTHH8ZiIwK3$D$dhVRZ3IBv`0&y6bLxp8UtJA0ie<)>CT} zQKj*6^eYq7TVPU__BMY%Ir5rQM=^(C8j63sA2bP=u=e2_5`l;Ee_gIHCtAoE{ z+S*Av;nI`QmderkSd4_gcBnJjPdPiI2LaN^V03*}?CzPM zUp1-uVqfKBUgC`hduEbPxczCkY;NtV>H54Gt?8M2)gXXZiBy>!C;Ii{6J8z*uS>m+ za1F~8*Q3SCob$%sSRh*!^W2{D4xzol138L)4>R}W*?u-Sl^i|LosiT-7>-d z^FVN4PfG^=cd7qS23}Uue+;3ynBR^mwDfR1s@xIw?;;gs2{$pZ*0`+|GBK)ks^y>Y z@KW2(3vx|&l~fXoA5}Y7r>~@XK_fPKntUt1WCU6V`Nk6-q=jMhW>bcrQ_6Rwahn+1 zS8&c9$v4Lwc>a9m!m}HEO9pj)56d#)7-TbJMm(4%;reKcXT^h`f67Yg(~$X=0Az$6 zl3*JuKUY6Ds$7E0cU;Q1U_+>uyl{mXKD{}DTacO57tPK9y3-f~`z=clwrFyHU!K)| z)PrlLa}d;_mNcr1M$wF{*V+bz;Y1m%qb$D-K~=1L(OShZ*stTZxnx7Q?OSH};+L>2 zAFJoy=MZsA1tsL>ee|_>J~h|!=wJuD3F0GUK;Fp*rg;l$Oi`-4+THwL5u_Re2M7~A$_w$9uPiX%rtK7=;jM}De|IFWqq z#E?@fi;YBFi{$sO@U6O+&rR$KDwIE*ITv!6_Cr@EjbNNv(Ztm8oLeGB&XDsG+0Xiz z2(BkSX*x=IaA~7bsTwFRzUchHlof-aj_yO;BN`H`9QNmvD`2=$4+{jIX~Eg08#tV6 zPg?6o?Y+73N?M<$Gc5^7S39TN)Ik``kUQC-_beXn?p5`nB;6C0YOZ^a1Sv0ZM1kij zlt!^hs5+W!EymERlBGhPlIrlH=C0VmI=R9l>m4ekqak5c#IRmPY&jWrk`Z`#F__i8;v-tg`z^H8B;3; zV4A;v808J#w#a}^)NjCe+~0qzTGGm#qoc+Gt=oVyc*)V1L{^oJ`CyEhk_XQYqyhvP#+PW$dWKC0 zxkeD@q*=c&asutm2`7IpRbXH^KJh8J|B-D1toV+xttv2ZtD}0xq4VoS0SB06Up0> z(w_C@=VKK07*;ikMLjhm9Ad(&|3xsGeQLli& zH3RlaQIW%Q%g@)A=k;dsl308`U=K5wN;Kq9d%DOSVLnaoCnGCPc?9~_qlhWDXES*R z+j`hnPO~6Ooz&;3;(W$g88Hpd?OiR}*wju|g2-2_s}HW-wHo=GpltDyJPP<<=JgmQ zimbF=kc-QIf43=Izal6OkJx-Tgzu)bvrSJj3RD3O@w(X3BCL9v*|+|xx6Mnkc-d1r zTxJSa7kZRm3_o$=h3k7?JvUBFl%kT8KU;ydmf|6}vjUT5`5okSKp;lJS`!*9>8HW? zIC3$ZEl$8i;a1Noh$n5cSy=~KQ6Tg?%w&a<=C|5XYXR>=)&Wp1EMHKWwtMIIY3T|O z_dC4+|CUU4tb6)*7Di`YJ$G7iPAFN`t?m<_p&bw1GXpQ3cOg#Ro!}9`Jk@^Ao9VM> z_)E@xpZn3-W1Yci?Fn6Ji(7G2-IGc;#!N*1S&HM<%u028YONe1JmSvXwYSGUiMe)l z{tT{{*UWzW&QBkWAIw?FNnj=3@$;PtaQDbhDz1C)_~faEXR6m`ESU(=lf!Zoaun#! z+66*e>3hU%A7b|7SfnI1tzdrEImvdE9QGQTy;VDJEiYi=GrRgm22Xfk;+m`UwJv}| zj1CH~x&T_ymBaZ&ItHF;HO{pdismaWQFKCro#hq@;F4b~J$!@g z7P_)axwT8*W0^mjariLM*b%v-yZoVXqXj7VE=fIb-pOxh;36nAdYKEy7>9FLd+T!t zAn;*()q}cL{7gh&yC%)%r}_*=)f5ILp)sGiQB7XdLcvi9zdwrmnbHIx_mj;jmW87d zrE?SIw*l1((wfPiCG9o|hcFIdTHnNeUjN-%T5_+0m*{eWgq+qIUV7r=diYVF_olAL z^KGYCTd#Qu>EVUikz^vhbi&inw{?{RbQ7r6i)-8m2i8@DvxS-J7vYhv!S}2zCAQ+! z1$k|y0)j+dBw|kv;G9pCAfMdqTuw;c5@=cY zmJLk>W3W#e*WB38`b$?<4Fv|E`*GFzb;oG1vYfGEj1l z`I9WqxQo+;-O?zXQjAN(bm)UzLeL#%K*e{xhO+3-aQ-I68F%3oz`r#0@gSH5mB{Cw||*M3X3?R^zB6b(Zb-oPlWXqw@8Xy}{s z=AF4y(Q4qVPI@xd`Wpm-)I^jAY4YvzeNYYbmles6vP{J+XctJ5dUMtZiZyn2ueo3N zqy>20si_O{i09!i70Ym##bXhsaF=z=1y)aeV=KowQi%k|@+i`!XSw2LZt4IiAt&qS zXg!N>;2d9C_wHaBDf4Q+=7u+-)M)Ag)g159+3JSHjVCd7P64V^9uSr!Idi2KhA%}u z-gur_pJ?bdPu?>191AMELBvAZn9dal7*0B_L>m?H>x~(lV_Vs02OvtdYYy-3QAi7B zOz33S2`!Y;5N*rQr|*9H-ih`zas|DoEkBd|Cd`jBvC?aw1bh7RuYdUCUq5;L_dopc zfByHc^WXol>#c7-_;=16H$Dh32moJg`QGLOH9nXe8^>Mr`U@xy%;1A|98O$U?1fa^c=GOS_8CO}zB?dFs| zz}lWyrb@Ct`6bziG*PvG4N11SzPPv0OuGjOV~;M~Y%1kueH_G}aQBoX}-j z$=I2}WurHNyVdqd*OYb5-ot^7%EI()+W<6+jBf$D*^JWD3C6Y-*vt7!BgA7?__3-O z_DO%t`lm!isSSe$IuEoBI#TY&)kKy<)v&tOjI(9ooEI{wrVh+rAwd6|(OZoO(o+A? z7%PVvofSql_)<-3zT|uKJr9~0VR|iy0Hp#{h7P$heEkADqIC30ENTd6B4Ag( zvOI_H;TXMX>$jM#O(hWX9F;g%y{;ZWahRvEb(CkLk+w8ej~>QPa)Lpg_Qs)LrS+;5 zJ|qyzb9FQ9>$tkn)gb$1(Ae(wJ*GFXCbo0SLeccMsiQGjMi{d9&7C- zTJa{NwqYW5yS9~UG^h2g!rc5s7g}q-REFPL#ouj|-W)}GNxLE(t>qD`afWB`W`0s4O%Et8pb*v zMADs*K=3kVhJ$y#NFz^V0NTzz3S01!+H9G*{B-8ddXcba7_K>l((iVYQMp3p@sG@9P4%Jqzo4=F>89qRT$fUH5UM@(- zM40|u&_t%Tvj(D!H-$89bfjbVu2`I<0lDGCHO4~u(uFb%l9E^iIQnRX6H@#7m*lNj zE?zi|dPIc@mc948LYVQCVjT4na8Pd*27n#s|;ErbfATw|0`|($l>% zdbQi-P;bkVZ4ZwRfAOd)>K|`>U-##K{m0k8eEr$4-!t3iV(wJ^5?|k~hUq3zY z>8*c!-jv&Ufgk$)Z|`KMcAC=W^3s8~3Dae#!AbGG+D2w)XRoE93^HyM=jg$oqyfLm zej7c|)FWd<%GqYS|8epA!h$6$*n3eT{6ujwD(;7OGW_2PslC3NpN{IOUjEj7;!ehS z%l+_?ywuRaCIN-V=+mBt!%}Ta#q4-OTPR*{-VIA##b+1F>zEmx!i_@z+k)#p>FJ{t z=lbB}4X~<~neS$=56@}IAbVXdUXY)E&yzzR5wajAXF#11mu@Kjl5wZ>O{rR(nKg8AJZ79dI%`jHSj9sZPgL(a z(keJ0zPjhP_pa9YC8f+au|GPWDJrz8C-3^8<9whefjVD2orwp^!u`@IqVZR6ynQpS8M+Tz{+oHK@fg7IU zO&G&iO|*Kpr5!wS$%x#|tC5A99iU)1@7O3sTW1qgC$RFfSPLELDD7nGW?ICiUuuy> zW6)A!$;EEC!smce>%0J)#V{Z$C6JWc<0=RwmmoHO>a%8O?BkJKy`eUvm(+8{FOIDw zvdL{_EkXJlBP-W1+q05D!pMVAjM;3`CfSA`@NBZ(FaGK)fib(QCXlc;zm$C;_3TWN zP>8e2a6YM8c&qerO(R_Yo(_E}R|k%GXEVwE7y8B~rT77r@DwV*M^U=fqlkL(x_M#z zBZzX8qGi8AHh2u_7|=-YOX3`&lWJpQz=im1DSPj^R(_$#_lN3+e5ER~z*@NnIxXg| zmUwWUg*uhOBOauaL)bNV5WI6*KHgJaO{d!C=I(jSMw2D@V6w8U^Ba_hw}vpx9JD8W+aTg(E zhEH!V!^n&Xnu35BuCGl^3g7N3&wMgqv-+Xio?p!1l(`|fl`xoUz1I7r$Hdngo zA%!hpFXmB@vF*`;LLo#?8ih2632Us?u{3@1BPe^dZ$VqMZ2)PWEDhTFfJn^;Rgl;S z)Un^7cFDKZVz#BL(kdtPCJ-Wz!olq#*Qn(7Axhg*(o}jFs;m2SYH&uJL}{~8Iv<>^ zYUlE2pRdIhrgrqP304Kg!UB)cg1E-xkw9czgr#(l3{Xy&@wn*ibcuWySc0Bc^3kt{ z-RP`LgSsb!ypc?{ZQ=@3q%$$~b8Vl0{`i2d4RosuAbPdDmW_9ibTy_u-*>7T0%0cb zoBxl0_&WE*7jOR8f&c0x;K4lAQ+?-~oVpa%cfM|4e{t*P*Z=$V_wTLxN)R=Nw^(l8l z%S^daz0i{~7MW(O%^7Y~WYXB|cWZS&{I_WHi(UHK`pU*j?{u&`v%p(1m;OKYrSu^yo~2-Zi{M(@DK{bacOmaIm@F^??y@lf`tYD4mO+p zWykFkdj$h_@le+J*?qCKWduwZh>2dQ!;EnO0r=x;ALnE@|DG((>J`US)5^e$q}H0! zA`cY?oM>R zG)0y5ZSgacE&p-K!t$#FD@VPAllBxbG}cdBrkg|$j*qe0|NVIMz{THb#*EXwqc5I* z@*1WvA#R5~kB3z}uec}p?~BKr#|)o6|4!8*D_FluF7EBiTU$ce@52Z9A|xbjHmn^6 zMB+iD;eICzmeSHWh(Nv#VrOx2GQ^6jMoL04Yy1EC&3Me^cuXKCYcrapV$3;}@+j$V zjX)#a_Wt{!@&?j`kfU_sSi?Bl3C9!#;ib7*&-#=!B~`muNr#u`fpAHL%d&Ob6P-cq znGeYSzd3~QHJp;TwobQpj2(1--`h$`fFV4X3=#L1@zH<=>+KfXFwO?UXUQBj!;$Gn z*c^CM@qldn9z2rKF_821av{&BQ@ELYkufbJO-zupaoc~w5v1WQ*vb!O^wyV0}BWJ&c}zZ^ynhVP99r;<$=meMx+Nk+6*WBn)jwA@OV1ql#UfoGh3vDMr7| zL)&+WW#u$OuV*_iujhGz-}wORHt&s$k5vXm1t2t zpHxbmkOX?8K(FIsxkGa_L5g1k5zqE|XJ=>O4D5L#JTO@jbumi!4y6zZnAehnK&_19 z+B`|y(Fw9ytRb0Xl_%KmV704L_9nnm8|V$2uS!Klve$RP0pL06!@^i9xz;j_wKSAF z)o5R$(6+Z&>Ujw!$bB3G`|HbBRA{lnhAS3K!$f|LI36EF)TnqBNNn0xU?8tzLbh{F zhs=w_{$j$hIJs1a!)vz2__+zHy5LUo<9_(tto0`lReD~9XyZ3e$81|>Ke-8df{K``mAHzaRMhQqI0lZoT_O&&mG~1_nZ^ zf(bR@9(k>Q_XYlqoA=yn>XoyZ1zkZgZF7Tu;nQ=n{DWscOi`pnpSwc$E zdw?3+NqC=k@x<$aqZ17|lKKMzfk;OM^%AE9*LJm3>(H0X`rk*;P zkruvEQ>k-9Zx4qpH02zwNI4`Jxd4wT_#X#Z2F7T=G};H(TGY}N2!dK>%= zkYIQQ4r2HZwvrGQKmx4~KwNgfSCnhZ<0AsWKlmWZSG;o;m*lQ(KUcLSbqbJyK7m<@ zsq)w3;}4`jx@ML$=W9@A(wzQ(TJ7I@09C)oHx+UAG6lMao%H*$jNy=z3@g?O>_adgSeuaL3A;y1uPCUkpPzo$%4CX4g9 znsg?=CuQb~qd&$SJp29S;?U3T9m$DfJUG(Kt16b9o&DR753jafM7Nh-ypnz3cGAch z-IyC$L&TA<@eA&_A+}ygOm`%Og53`L^T;B>7j;2-)?EpjvE)kxVR|I%G4a<^i+HWI zkR6<-x=Ts`k+`}xqwio7*WAkW&rj4<#b%f-5UPOYIb(KnX{ zG-qz*Mykz7E>?!`>~+n`;qF@9@(>Z%Zy$~(D156E`|Yw?qru=^&n*z;@i_uzzd0T+ zCQ0Iru$a4^d85zkS$Z{t}04mr5|qE-~Bk zIQFnN!X3R@<>u?_OZ=YKg4QY^00dJXYC?h^)6x*=P@@psj_!hT2&%z z4RIriSDkB&HwHmbUAv2N&XPiSA zT|)Vi-V?8T7x*Q@d2-CtlM4%An&zoVf}!%LEZIf3!QPsm)BVMk-(n}=PiYuG8Nw>a z+;AmZTM*E?C!0%vs0wK`4ZM=^jybwxQVP$x>Soihl|s@@c&k*7A&qT?SUh8~uV}2= z$->!laf|HpA+8!g=d-KlAlxW6z668em}JX?PJUWK+H~T|DUR*P31ocGyr}qP5~4pi zMqM{=YUJ8ku`rb8IxJk`b|d1`c8aQwq=+W1%*#P${;-Z2&~D}qAiJSNP>|8$E>SGV&;=K_)e|*$NNZn21&PU%HhZ5Y|iBI zB}8Q#aRg+Zvx%ML`e?FV2r@L!;3X}k5H6dBS*%V2I6$QNaVb_Hfv4SKljzO-g#H2r z)K9~ljP7v(^8p5T* zGNyT7AyQKrO|0K9oKM5K=iJpK_|>F$1o`c~ez2Sw;M}$}7x&@O2PY0QIj3&>6u5^9 z{}J})8*e=S-+$ba6_eLy6mZvF8cwAz2do;@gt%k^R-O0zaU>CUS8clWR?Q7TA{ zz^T$ceLh_MozK&vBaz9e0Ws}HZ;yl^28$baBH1f99-Xy3KQ}x#-T``|`tzf{OwPmM zffNjD@R&=0{*^1szWObkWy^fy@auw9JY4#p8g7)A*Vp{`rYkjdcJ}0#J&A%|{f$AK zZHb*X%c{D72@HDX&4f|>2|+&&UXaI*Rjy2)Pgog30z=BQ?7~^O2%^Cxgz0`Ldi7VR zPFL0L3{iG@=4bflY+iKX*sGM0=+$N=q6mokqD;{)=%w27S|Z&8^TDi}YT1@00;URv zc)MXOAQ3Otpk$q<#H1*73{Xx%JIj zQBG>I z;GAlk%)4~1qoqCP?5X!Bzh4~QJJ&>ywWXc8hdj41mq&Ouc~OkdI?;OR+*hs-Gt+`m zh7&v=`hr++;pshE51!fna8KBeYpSurw3dlmpGBN*@T_5upLutxFfBbXPIi3wamE?u zPc4&!FUFa1yGmPUkH6*QlvJKFbNy@Y{5`(##aBwqD)Etr>L9!#nIv< zB53|_k2JZqmNbHLMWC(pJb;L9T`1}}f15*ZwXLiwn!%X6QG)F6m={RU+l`pw#oh2t znp;^NZxP|uQ{tt`6wa(D7bIRfx#N?YN+phYnHiGxiT1w)po69et5Mv~)=$in60H!Y>15GzInw-E%>v>ATIQQ|T! z$aV()>s=y8=XxT?0g-Ie;~cW4tG$Ygow<)Pd^A;i-sa`$UQP(Nd)YrRCc=mxO1sJi2c=8~GFKCABUekaJ7ms&*Nj64nHrh3h37esB}aH@SnB6cI`2Q+&RsFP zW%Mh6-)cl|zAObPLP_m8MzGqTB9PT;8&FJ};lDNANmyyRY+FAV$+^SQECd7-S3?kA zQAMYW_g%?PyBUkyU7o=zAgb=AKKPc67_Uo+%Na4KO*U#VSi*qb&goddl6585ZEPxR z9_+mo`)i!_1>M{SuVyajCEn87k=CV1!jnpGb-l=+P{fXLtRQWZVf|ZS(CAy|)Tw!K?yN~dl4Qb2+6q*$&&6PGGV*;6M<30IAupu~6b z1m1mTTL=Zg5i6_To}hfC7)ngd%I5TCj{aCxM(~dA!)N4v+;a0~M)Uvv^z-+r2dm~_ zR6TY%ZR`(!{q)aoz4Pz!MCJUA-4M`{o=6`5dLH9mce?>GLB3TV6nZ&z4-b?krW*n8*F<{ zZP<#Y%6Qgi)CGv>r~1e}qQ*uSEz(zy6kqQ$O=(B>3 zOp_R_tne~RYs>1#OxiZk5{RjSiIh93z0WcfAE3X>8c(5r78Q}6GW~M-zCN_UOGKGD zVy@1E8qld3zZWIHdbH`Tpc9Tte=xdG%f?{6)Am#I)f8#riG+LJIOfv4mkZNy(l;Sz zLe_jALaX31$%~{X<7KwEm7U`waj}*+Jf3g3g4S%K5{;f_%1dyBEI(MF|_}&;g#c$ zh8GO|bN2Q6!e=E3j#rB&A^4KC29`uug1Mi%Fd?H5pGe2uFR< zbj&wS5E$b!ZNCPY5F=K97E7Kz=N7wTUq(I}heV1*&v2*toO2;;plSL3S$vbelU_GSo$P*{UqhUolc`lBdUC;{gcKt&fOZr z^u@@R20liApwgg>d;!-lkWi?cH(0+lP677b7n27pGz!-7GL@vEl6PWh9kFhnjx~)V zUndy)1QDcC$p50W&HnzIM?n7aNzt$s*dbFMM2mE3^8^e9O7nOzi)v=}f{B<_3VOtu>pjKRZNSmZH-?omFw&l5Ej?HRgn|cLI7=V+iZ||4 z`&X9WPQC5|Por+SmPyx*a1E;3mZ19dNP8nO(mR(g(r;XG2;Ahw2a{2ca7;Tay;O65 ztT0rlIYgX>qu}VI;}3$!@~v~4jXfwJAQ^v?*u7NgY_Kt?^YP2WsYorAYm`GP@fV9a zNSIw%${;iNh4s-z>q%JaelTx}4IX0k)yU!$^7>4_cf$JHP-gcfS4I~PMw1t)1?vx# zxB!BACzP@AixXyD;6`7FNuzNK6r4cRf6%vM=-ThOCgrWBDscnOU1+vR%$3mv*LVpz zqA{LN1ui3fbWbf_LNKO7>2=sJjJ4Zoi#Zlm=m+<^QdX*@>u zN95*DrDBq$&q=iTQ@xa*mWCul`O@&XG$@xd>k-K5BigM1nZl{)-f?*INdUTbf!AVj zadCc85Dc`bGVXj*d*9M^n`oA5S9qec4IfXZm~I38aBC0;SHB$&` zi8xD$M7wjZFb+XRk2WAjc?#_bcU>@#k zU&1F?3(ZkMZddRNZx`nVk`gu;Sr&l!A-3)-TaLME7Fd~zb_K^Xl7%0;EPXzE@usEo zqUcWCcdgQg%DhLE-Q`VrU8_ksz|fpT23|ng=G(@?$7!|i9=Kl|?~-(IUrQVyM0{=` zT29{@`LC+~sO_>$-T488s5y|<^j_ZB*m$DeX_)JK6WZq%VrI@CmgzX-a@|;Y`31?@ ztT9EZkB?8R&u5C%wwj|Z=UP`L3pKB<#L?4@IyEyDa1ge;ZEbDkZRJD5@K5u-dGEK! zv*e%o=M9c)mA>}}!O<(t%w3)->bc;JI6NW7cQjRCiH~x1L^V19&r9i(XN?VonchKh zb@QQE;`)3d;pMoPo$R@9KXA*s*wuKO?B3{}OPxh8iEN?aM(Sl9;ji(kn4YYfxmJKX zR14-~V*aW~Ybyr@0KLiy$~9CV%D`99l~M)z$P%BMgDKtQUM{2G@9D2T!;9xT_P$8_ zalHG~%rCD~0{mh$NN3lVlc#vs*zpvBIH>PHaYcGuc`GbOzQyN+`34mbf#E8jmvs!m zm4uX?)N@41`f7%qk#)Ql=_Lz6F4VOxeDhZrL>YcsSZx0FV(H8nGn0O}-W>DaP2p`y zrplDoh~m@4efIge8$(5{(b$mb^Qo7whD5j;sW*68&wl;45wrw+nm&A5y*{(!ajNjS!J6I!iqGOWp`aBeB{WAf&5tCy!6;%x7cWDEtes1?!q|HFq971X*M0ZN z=4A5552ZAMj!}Fg7D396L3-~g%Y&;YOzJF3vLAE+h-^8rj;k@;S1nPBW-yRp*ok2v z%rBkb2hXd$G%pkqw?dGGauGvhznSbV5&OeT_-<5-Z?e#(@TFI5U?6*QbeBtwpljUr zrmF^BEbZ9s2@Yh5V;DrnE_6}5JcGO()sU@y#c7O16cz;w-CeHRNRZO$qj)igZoxLy)6ijU)>^P?oRgnL-bv zIM)`=-HgQ;WjOO(a>RPS6^Jm(e!(FcRLLhdS2;w@PRp@p-<&jsFs5f%2Q;=69|Niz z9>p6kR?NZ2$`OdwsU#YEvy4{7j8$(NU^+F+)5XE`uI5WM|OMWSB12n z1JzZpO>DVbtAE|i=gLmdi>15xVgUBxIHv0@;zDCyOS?lPu~bGAhN8D^-~t0qTs_xP z;Y-vp*pG=5MaR_rs<{^HiX>R|JQ5RA-hvCzKjsDZAN#!6O%KJcY(A6=`T}v77=%@; zar}Jm<{wdpL`v1xJ1E^ZTLc zPGVEq&V1h~!b{-aJYhXvbR>LmU|=)z#Q%7o>47PEr)fM9;UOWs^zsdct4stfm|Z=V(gtzF+kd!~=%XyX3bDzSNGxk? zt}D&LvQT{}Uim%AUZK>f=H6Tq95D(tQ|8R`k)_O)E%>IUkIM?@Ln4D#KZs3IG+28( zmrRqBg`AfiDixs?@GN`Jb&|v$i+xzJ)+>R$uN1a<1^G?GB3DMHKQPPCZEXzvTA1VHVj?l~eo9+?Z48V0OyL2h@`2kHYi1N{<+w&qgxl9 zKfL2#+g%xGGhAk%HG4WMhEuxEggL!EdFjwdu}Az(5iv(|k6$`|7)twtDSK;q51PJk zsf}Gv!y$Czsoa@R+o+3UYvE1S2&@@bCKZ**4d$CIsli0E1g)rOkrj_Y$wP!I63Thb38QL%)1HcA`FznA=O9NH@BBc%5BsJpEj3FU%n( zb)YrMb=c$llK-{QIl6nKIxo2=KX|Esp(KJ8f z4ktFjj}=GHC|;8~mgq?1|FetC+AC>5un)GQG)2RGaOeMEC-^*vFSJU656biqp4L=_ zhVmG6!rCzqvU165pH$Et`*2zo*HVUpGMp6v7Pq5#k^S0QjzTN-bdA+{1}komO?>j! zCbUWziv?>8l11LmQkY8yiJBk_KTL8=S^Su+K8rP&$3jsV0Um&K_gq`w8uP6SB+rbo zj?xDUowBQaB` zDcoe^#eE4oovc53M$)W(G)o_JIQe~`8*r`lgl{sr`rbM=AQKz1@@nH%xY%0$dmB~{ z2tKheLzs}3F}jngHmS;gsN1n<5N*&Sm`|7q29QB)!sZHA1~7Z&m`mZtK(^-nNUp5{ zhS4x)i?$o+c7uX})}x@PWkl5hB5)bku)WR7?^D!_?e7Z+V%mCIoYM2h=9=DgW#sqJ zv@=2%s*ybdBv7o~gpKRZluQ<~yZd%`ue)m;GM5G>3;+21Oe|1kxQl8JCZ)?6NZW174;5W z=bFCqbzhJF?eqm|syCvGI5*9%!LG?m%f{@`S3%}VpTp1d+RD7&GVj;NTsiZmK0QS> z?4;`YNO1d?i(h^4(O&@es-gJVr{se-h@B^jCyI&ZXcL@ED(vXl(tLCuSS{G zdT4E%?nZg~?9q*OLn0(yX1 zQeN>WP?#Lyj_Es}cC%IA1+I65stu*`K!ig*FkoNByGg!6_eR9cJ{Gepuz}?lSAG09 zLE)P7xg)RBGP&H*FelZOKYx{erLNi5nB}++)A{ydZcMUYvrjuOt2*XM|5ivUKh68o z!P~0Lx{9NJf1&=^$Wu%OOknp>U?)}Y?eSJpVbbKCqr%01wx#c_c`bOayZTRz0cAH| z`PS-J_4r~5?uQ}q=W|Y?V*F{j`fBSR)6>&t#?RacUVC)^_im^woKX|sharIPdTMo^uD-AP(crDl+aRd~+gq5c=*BISgA%eOf%5Wu*M z0=h3@8TiQOCbZJdGQe4iNZRJ<&U1-N!JopLFatN9ur!3snLx{L2bm7ygEr43uN4{i z6s`)|KF@BbQ~(dmq8n>)rkg?s=al0pq{3Qf|JwQf+(`e-A;{9E<@C&c-Vjt=^ZZO7 z3^||o=GtFf!kUJ#rh184L)IDBI!qD=u(QT-HrY_w2(_@BTDVL@+j(4z>lw~WE9YrR zd-2O-4TdggcrtS{Xc++k?xh#|wL|R$EuYYRjOT;UXH$akY7VtvrV&7$>xLWk`uVZn zMEa$sH(AOVIIoU)%Q`DPBd~atq8$Z7VPuhL435W<5YO#I!IS{AeuUBs`b5^k`qbnA z{3JLu=;%oRB(x^BPfkH(KS|fT%(JE?@ca+;TcA45q)Z1(N)l*$CwR9>HGn%jzjwGGazKO*vg=|Y} zkQSvjOR=Qkk^a}+t$ta(jDl$fQGU#Yq~H@q%N+m4+ee{n8D=-pf>bk#`js=p8&uLl zeR(CuF@n)fp-g|FI)klTPRb&7*dT6X!GRtk=C@ROpSB5p({}p=Y>n+!9O#TS)-_EA z0z{O%k}r3vXR%97_D#CInKs)36?P2J?$9~1f(u)rr55sbAUSh=9*`n{5`wu8&FT_4 zk9GEq%UQb-S@dK#yzW6UZ2s_Qi#rghwq(5E*C&(P(S};kzQAeIn3S-#pKNO(*;W!v zD`c4`DRFFmLe@#j;>S;HchDSF@W&bnUIDxTTfPXe!AO>^M`P-tJHXv8f_RMtxDS2A z33WZoCZ2^?B(D~kFJ(s-{-Ko(TNIu{59jJ>+UC}dX@d;Dhch3-AHfqCG>XEp4vl%3 zrbzL+lN_|vF6Bi8uMJS1s7|{0=cu|NH`Rogu4;uGfN8znb-d0`y9z)16q}d~D?hr! zii0#mkzg9fHVhww-crc~M6(BxwwJ2PK(f&YWCpHT=}S=h5;rB7iyV>4!vhgNAK1~k z{cOVV6!x#!X+P0sBJ`Jx)?*EYZVNU&TzBo_Gj1d0{KFX>MRo0U+nGuFa9K+QHUwRx zx*6i{m*R?ggyhzU_T+S}m}Gfnh4GTz^@+sd_7jIvN~^&70vkz;rxVXI~NrgGv#8EhUbE}>-}Z*OYR(Z&O8SGwm4~$eZ^}_Yj@$N#)|{1; zgBThaEE&{KqKzVgV)e(_0ymWIhtQW^ zl(``Fmw2^?ejUxyvyAPiI$DF_`!}TtGbGa0A3O0kBezQ}FER@hJ%O5H!7)&10AI=6 z6{unB#L0{K8n}dkxnlw?og)IEm({R66WCC=qmspID_G+V+{~==NO!eHX8xAgI&LKmPtkdT&~oI?yL;B;SU_~*#$>&*vU)^kF|3=uvi*LE+XZn2k?od2+l@Rsed z`~M%>P_$dd5#_2Y`_#=263bl@CK$E-Nhkbm0AGchHwv>bs z>~Rh35C*!KnftG-w;t&8?sXlQyk_PAU{NfnCcd1^F!doVBaFgIJZotfZS7pnz2~BEC$${r(IO(i|;YGpwdTm9F}v> zWoW4miHz?0rkfv0nX0-)!c--#Bc~_{?MxxegvFgl#yNgTWj?5_ntphd&}C$hjnp$Q zK0=x{;5hxfYzSy;gi({u)gc4S_V$hXwl-CHRO}qcPDJ@@uOCG5>zMin_#oLOE_0kPD&{^syjD&ZP6C@5%tVG-7lq;**4 z9?U}ruU_8;Ft_iy{#nde7uWp1fYEYVRwiAuy+h9oa~^~WlbVU%IFP^BRZM%~7+sQ$8 zB+1}O#`mlujSM`{6&DMYXk)#IxD)OM0XYK@d-P(t3crqJZfv4-n>*0g%ll`HcsPCy zHY(JC>cVbc(j8&Lwh<2^Wx8Ws1C9+6Y|U+ak`DU$s-9R{5dd&RS;Fat0*wgBD!TGP z^MNP%x0%4R&e?qW9Et$c8(FqQl<^kIk&SXdZV4=aGwG3JsPL0YhfGAgWK}QTqTAO{ z&Y*NKIn(jeg9vjVcdHZdddP(gya)i=bNc*$U3D-`EsZFvF*uV8h zi5Wcz55r~aLdi~NWI9+M0$}OTo1r6k`%fT-VFA~;Z& zCI>K9k(8>`E};z1cH9uk3RtK1b%xJmNYz4b;6vK|9>E2m(^V~`E`J+`fs9b_HheJe zMiZI_b;%w_F0l&u9=?m&Be9%?SU4`hkjEjC(pfSAx!_493=n|O?Vl%s&9wcdFk}%L zIU_6CNsSQAVh|^Wj`49G>mK=0cPD`4lqX$Ovm0YOBXIbn-zPPTcy(rqnxfdwI}&(g zG$K#1H;&&r)aNtN);8@H+ak9`q?~7n!ldX!n$qj0%XLBifjNWLnke1X6IaHT_Sb&> ziAHJYzj^gP|M2PgsYg|&e$=n}6iA4zyd177nLaSx`A(U)tVvRh`imeipuga%W*MT{ z1KEy^4Ni_6*n;c=u!|2oT!{B7yMRw4CgmKFEQsR{@6jzh(y(^(7|2}R7~n!cad5ee zbc9A9hOpb}l@p6Z$+b=+tltc;riS#OSGb$EPiY@geEYJVB4Nu9TFL~3I*HvyIT$t* z;4yTqQNL8FVQCnSR}F3debaV%;eF@k*?}%{7Q6rW%Hu4JsnWURzH>qj&6XYRXKm+ptfU>`Ic`^_~RT!;aG`k8&rX#i{k8ND%O0%H8g6u+SUx zUA))h7rD6w_GOYF*3*ai@ak8T^k5qPl4Yu?hs#Q!568F)MK>;bw2e7eJj^*We^@Nu zsNga8`2}oM|2%Vn(J_h37|&0^#3Hgj>6#idAFbtcK2!ZzTU^k^Og3&PTwSJ*96#51 ztFWsM`bNa0>7)D4HioCAr~Tu7Ag2#x_8R<-JnD|gF+5rO_D)Djb^j4mu`m5?X<-0! zCLeW^=-Qi`|L}~er`y9b(%9LD)V<>i5SE4NQu84DL3n!I<-?J|(t~nXH&2pHE#%-F zqe5cc&rj2U_bXB$i3Q#UdvwW^QEEOh-_H+$;>T6u+42+F-vONG)vzXw(PY zTu_+eHC|$w7n~cL31hQeq%8n?^g!zlo?C&8VClNwTL4HD)Tx5hnuh%8a$f|Z9cF_0 zx)R*{{DMHu&=d;d;V}>!)D1ynZm}tv0IQ*)6m}dcI+7^xm@e4{{{{?sE_Jps%u1OV$^ zLiBC>CD+`^^^1u>Szgd{AyjD7NgR!!1KquoqCQtEZ7Q+ZN^HMhu$Xc$=SamYsj37S3(OSM-#T7_qr&RO9(1EEyn5S( zYd`4{V7pGYWbgd$v*fB|*V?AxVJXtM%`Yh7lR$%ZhflEa5fnqh@_Sg2)b1W#gZdUy zoWm1cYCKFGsh9H9eJb@~(vv7!wz1ea+T(lq8m#Sd*0!F%X-oi(Ss3^`ujeq;HKleL z-ELKjo5CQXZYk{v;+dts}><{@;5-lQUjzg%j3KT^2TIW-@}eI_)##Cp;^V!kDzfYyHE8&qV(! z*`pl(eQT|7>^IZt+S<=IheX;}-#*+QQ7?#zoPIn$n^OJG^k34_$Lbq^ySgu0>C^R~ z>FwWt+;zMY(Rr}NCm~>-+r{NF$i4UizqgcwE7`{7rE8+WM*U2tYg)#*kFP7#wgbrBh~(QEuR1oS9@MwZW_VwAJ5n)tQONfD zr}>)ft1p2dUrQNlggw|en|nMZ zEkzlprm*4}G4g_|j@2h>&=mKLUev5^SerVfJ98S9%A@B457o?r1w9|v;O_&8%HceI zM*86w;F?|;@M{~spvu1-$YV}Vzn%VGXz1yr_dWl!RlB+f^0hKz=}6DMKLKV1J$rRM z_E4+-&)uh7!&}eBdI?j|Tl?RM_lrTLa67x_tJB?S>8R>*|A?ecDv?-<@b6iTP_4TQ zFI-7!T}dI#d#~?0V?L-n{!9B!FISyl_M5mA=mNLJ|8dMRU*}bkj;jlr%0O-#e2XK%Te;q{PEMKB%|=cjur-n@<&$>F|tMIRX4xtom0<(J0VY zZZ@(9MLJjEyWM3vA|svOmPI%-B%M?n#(O@Jta?aRSFH)7Yw`F+)9n^XzIGiA*eQBJ z?ki99%5w&sj?~vWLqCG*5N;DrfPVoeZ&2Rg^IyGaa44c}zfKO)YG|aTsSHMEuVGjT z*E&xROy2rjV1ti80Am-$O(mrm^AX?0+~1E6U+i6il7iu8Cjg^FpY)VD4c?97qZL;t zs=eC`6Fph?`DyWvqZCp-mDopJXprAv>I!jNg|3=l{`J}EhI~AR5+?a-Kg%0ZSI*qdozNzkv*%-{1Q zdCI5+X?KDGjkT574OUDXezO@!dDD=~W4N3Xvv8zaee%3nX8)n<#M&^UH*!-8aIxJH zqDt5>KBwUUU`Q&u{^qxy}`EubfNrc`X#rxDZr#2&ef8M03rtiGmkV@9kGzeFb*~5>VXp z+awDV!RxibjQ|33P&-7H8f>#Z9P#iH47q=e3Sq=fnAEjOn0V? z)0p;}`Y7ILCZzku6-gKy8beYnPl});OSf;+y%Xlk2_KYFi4R=KFGDi=RPW=_%ST*_ zXOa;1Ll|2si$%@oU%9SCWWz9hVK>s2NV%K76CW|tR{8%&9e$d zLU7`QG~j}B7Q840FLBnXi3(T+*2Bh@+)6FD=1K&x;<7+l0WUMFNenm`dXq9ARnJ^3z);av->xnz495lmZh1G2sYgFKfmGsaeUt7*I~loh5CLHX)q~YmUC^* z{MaKQB}(@=;u&W+FS)|=pV@u;cs?J0_&;C9*4h&1;4DcqzHy`UNNHBgu4w>LCouH$ z*RKi!>#iSk`{wJX!$lpP>?|N#Y(0;vEi3=w|8A$ho0jI~!|+LI`$D<$n4Ae@SXg6v zUs*C~V|uFEu(*bYl%l)GD23160J#C{hcGNg@^XlMr_wWsqLM{R9D1 z@%wB%aWLY$+WO!NOu}0qKTNL~7Yp~qlk>@h%&veK%1B-nzvvnijNayBpb6(9Tqt_I z%gu$GrQ8O>w`*c#246ua@Eb67Nj-NRU__|zF~w-*rPK0 zbvFukAeB9)P4%VOyJC^%m$48S{s73!8V?4>hI6R3Bes=R?%W|-FkuYqSik~UKR+?2 zV=`~VmJjgiijY2IMGoC8pmVy}rcmkC&rHBl#SG)`7+Fo z!gp#n&F6DK$?D`fPIp}dgalKOg|?D*e}es6_L&i=%}2)C6&m|eL6>B>qxW)~>RUw05PDj+E{ zzW!O?r5;@}H>bnB%9Xeh8s|4JT9}8{mASiwb=}ips*kh@Su4zh3b*vsIL6?cma?{b z7jr*j`5$lV-;eAgb~Ah{T0!pUp;8sgjW&B=(Nl{kM~)_&Eo9ScaOMRX7_Q%P#iK z#cQ1{8Gdt#(nykn4q(>NwQu`67NAedImX9!;xm_CKTjorTST`4aYGA9`;e}$fu5Ps zknaE%N^q^GjA#@QuR0c!5|%HPLafbT!1bAA@PbF4aBRESUl~a@N;_Nex_95c=^`Ht zDX}Fy6o~3@!>|lx%yGBFxa;Zgf(Uo%>rk$ZaHm?W^Dd70NZb4bKnG5cZ9EE*Mj=ue z#jOmYc^ZiqvRx}YguSZwjlr8|Tv+xGeAV?5VXVig+g~eWo%d~F+!u$&lFOj~EHt>} zr?%9Wm+zAn<0bsRGcFLuqKoxPV5&iy^vFcPS`<}kRxmK&>{(?nGjtazBt=F7Os+@@ zbu5h}oZgXyW?%^G+lRPohu9#s7@k)578mCpk4TDO%Ev8pt@&E3DT#-_H>;%gT}=vR zDGjky<<*W5ufE{8wq`wO%|fs}#2b9SJllyp;z)?F^4wg?Xb3egH%YgY666=BLbSV_ zORvs;|6lLMM~5un+b564G6v?(`R9jYYVl_S{Pv;`)-V(Jah>nQN(L|bP-31u z?wiD=jRGRBKA6)9duDdW8 zA?M%CuYcRuO9}aYKcD-4zppE(HIyoP6(wDNFj}})%c=QNyC=2^1c+5mZWSq3d+yMr zYqq>%pd*}vx24pduy8)Jsvpt?c^Uo0@1q+A2X+z+KN7+Y1qDc(aE0==EDTaPx3Wk6 z!O}|JY^p{zkUYlW|Ni63GtB$V%|8$yQ&tzA4knQcveakTrSh3JZ7qFXQJ;glIJ}(9 z<2K5M9uys`b&`ZnRS&}|GxFLm&)bGW*>$mT)4e9u-wYrvcD7XVz7lgdyw`W=bX_U6 zx+)_q>8$?YRC!(>mO1|@`Rd1xyQPNj^te&pj(#?}wwYQx)bZT*>nN-929|>8Su#TP>^l0g($yMawz>01~h-&1{<^%z}t798usD{jf*JGrR zfSDIrT407_`GqFe^FVOixvB)Da-}G}q1x={8&ButQiztm63(k32VkL2m;hIAZ=>7TUZl#;;YS{lZ8K+t=ni45?HfWh8?>9lKk|GJv z3@w%DWzk(7kwi%@U4*c)F+jHV;=P+FjIvy)NcB6db)!99XbS8ag|fwC?R20mptfG<87DPewQ% z)5=%<1O~{4L5h^Cok<61g*>x|Z-y8E_zqwcKl$x@ujaUyh=bR%$lWLXiI9)Ustsns zy5cONKy9Jv!^2{f=etq{oNpk6&ztw5$nvxasjuP&9C0ogfCJ~u5Mdj!xnC3Kqe5^u z!I`sDl43&`yt|;gMGiZ|pNt()T4!be7&+B|@VtP~g2qSS<{~Y#H0a}!YZ6PDG>=@l z1)>AGBqF7!OM@C%NJEm>NAJMU2^}fpZv4!0{oKlmHZJ2W9E__fP+2s247>URqO-JI zy=bJT1+w)30BTj{+r*u_StpfCx}*$BVjke^X1!znuYGFn63LuDn%R*&;#pr z{^`~BPxuof15kqfA7E5CUbo6heS`%h66^q?9QiPHIns)O4J+^NvbnO$wjBv|usl;x zll(9g)f)=>Fi4%&qysI7DMlD4k=r6kOLHC7+B^;CZTFq}M3(@y6c?Os|rb?cy{-LeKUNJ>ff zzQmE~pX*W+n@``#$X0)T_R;XU{uGY6{8%8q9@Tpm5mY$g_%yDqsw(GI=dm>Z^sWd4 z2f9pi^RS-d=r`L2%VS{99Uafy*`7cBQbQSI-_Aa|qS|=# zV^7j?orB+iKlFFsU5V+i32!3lZ=V6Uk*JEl2uJ%lQHOsm4ox4(C|;iL*b`~G1f%o7 zM@e!1f9fj^N8ji44}98)GS71k-ZGa8$l1=OR(Oh_e?A^gw3(jCNB%_rCnC|My%Oq=J zm3|uz`qPvMMmjJ~`jJvRl?97~cjyvu{Mvh{ZUvX(zRN5Tubqd3c(tASDaw*;0r!ERoYb+Hu}C2J6s8fhh) z4SZ2#6PltQ#I3ggT3iK|E`KOhnBiu;^r0_2m&$ zuF$hS8ISD>AM(sep*onq@Z2b>cKX?DsDne2@=6I{0zGb^)yJ5|eMxHxP2~~@o)=68 zv261!YplSVP9=}V+042^KPzQ5k&@BLJ6uDw<;DH$ z9gSEg>RkrkeQA0+gzYxP*O)*MQhiN@w1d~ZJCsdqf>=AC4FQd>D3mhUUK7-zhpaO~ zxE;jtwhnX%jPn@#KW#B&`A(W%K>}D}kANaW##oG0$_tV2S!Nw*=yDlh`9^UdgY-;^ zR^_w#!?6Qi9T;??i3;b7swS}5zl_Wk6e7&Ta*0iO_Bx4nnhG+NCX+8!pq&7V%qlH{ zBsmhYM_v<1^sH083mE9|gAfCeSmkHar+kZr#N|4)uOCnsm@%p!D(s<*yu4ur*#_>h z>pNNo=@>a>suE#j3v4B|b-t8U8b>n|hjP!O@^#mC6Tx~HrCw#9+zU2Vu}H^E1m?!h}k@Q?a#qM{D4_6>(e}oNakn!4DLVlTwhC zm41|&OLF9dDhteD5t;?p*{^r_s(OE<{c7^i3}9b3hVZ5#wB-icoKEr3SoVD^4m(N^ z&WP6MW3U0iEbc`O8Erbr_f%7UB6UTwNfMmP$dKnd<cwWnDnocNvVu zY+0V%ZcbO40u-tVI~S@3wSb{BT=DNf4jWf7JVIxu@p`Mmvltj-`9b;S0>vxw{s@(- z)ClNNcAKXA!prl|ev+3-t^i!7R$< zqC$w*vqD!GqTyoWHJzfCZvo0@1U_G84tWzq&c;sQKS?M&qWzk|X@aK2oc$I2d~6EB z*eAT@c4d)}fZ5^-$j=+BRPrtRY84A#UShjSi!r_!Mv(5%$gi!v82akUJg3H~wfyA& zZLRuoQjpCzF*a6v_oV2``CH$<-F|w%wmG9T?D#dy&B>|H0;8BSv2t-Y;YshQoQvZSOatOiRJeqQtQgQFM8%y&o{RB99@_ov1jNty(oLK<0K-iyph`M_B{ zk^z1p=B^=;i2p4$_>mr-Svoc@Q(NGeukpvv!;762T{@qG05Gp#tvIz%)S287-O@Y_ zL&TIsj$8^_oRMbt4@|<$^k3d-xiLcGPYP@0MXsSL`dpN1Qc@>6o*jS!(knd-E(@!= ztlY4}%4@fKKC^vGJ5ZHyHh$^9-Jf05&Qog+4Dxx6e{4Ci8rs_P-f zD{93c1n0~{=NjusIz7p@!}5H2dFXs^SJzohf^uw+e}K4dxBSfgiv9B+R|zxv1u1&O zqYin8q;Wp1>j-YMI!)T(9!SO{>^zSkBXe#qKWymP92_r5DLr#7BVQ%U^T-@O+WwS0 z%AKDqTzHsP%WK9A_;TX}X%cc0OM(yPP#9D_?(Z z(TvGG&jZm*M9NZZt!RV})158xj1rb^{E7Af{XMjGGtetCxF>M3d;>YBzA0ZSmzwRs z6P1-3cs<%1>-R=A(I;DhTC;ggO>sdxw*cjS7ektugtX+dw#nAtK(C7o5q2FHK$nnK z@)}}^{xVB<7lu*hH$2% zK4~%ACNsf1W2s-R(`8jyh#rS;g_*U>*B9&?93qi23Kc862ETC{Ut~L?k11T9YUj-~ z@WC!pWEdnh8E;AzZ+7Bx*o)l=7&qrq*tU`pNEXW#SM4*_G@R-}(-r#-{8*Meh zRvzX47-a|5w8Tf_3&#W5P`TMk(#z)}JO!++eN=Ci;LMj~qaS4y=M^gD#ZKNaf1o&3 zQDy?k+x1f<+I@_OIbpvU;NyzdcjX1@0G{5WDzAFgJi-soCZ&E|KgC~)ftB)Zp{}kBG z=xAaEfQTVK&(S}nax#dvH;XUT+e?LAL{yG0yKd!sH+1Soyt#vo*9cbreWyvZ#GFm#vun8X( z9nlbmL!D=v-#*zs6n3)hUG39^tUHJq_3V7+t(O02X_#(WA36ZAP#==;@ptv}qMzD} z|KU74BW_OAE{z)}-93^qr0dO*_R4h!UprD3zI?Og%aKRsio(Knt@A`^%_M*e6rS-i zQ(3{0j{uYM;LE9wuU>@wL%whCN13|+n7n!C4^8*?+=a(o;z-!r!m@#GHl;gdZ*j+5 zpYd+!{>UTAlBLVjbG+ul_7>no$pqhDX||_g&U)MoMgAC_mPrqYJd06;Ax-D$Ox7L9 zyQl8buD*fod^`vW&-a3o?rTZ~Fo-&MdDcr&jq#1LsM7Nlt~33sLuu`V`nv-LA3jKH zJ7?E^{;cZ2`?OMK^^@6pQOLdEC~4;&VSQx_txLDhU%%(9aITRUiI2vMlMW!=RE$!((Lm6=b4lojz5Q^AFO`hC2PwU4#)eW9z0ma182Dc%FMN zuk0YCQUfPtL&c8gcK}100Qj-4!AA*#elj#d*LDssLi&z{v-e;LHXv%l;zmYxwlULa zyK>P)%_LSuzyffyi_0($!;qW`T11LW?OB^TDv8;KI^Iq!+h`Yvl1ek4;;Bf$|bz58iCW z_Y57?8eH=@@jYk^p7;JJIE=t$4>;O+rZj|m!iR`+Rid}JX=Wx6(I_8O@v5Rsb$~R< zZEfIbA_a!aAOJEhPqsJzASgXgDEW(|7Ky&H#^u{c5Ethv7cC0Ym}hZ}7_zwt{j$Fc zNG_a5sN8h}YD4vay*5-#1o)P2h@f5#9#N4mY4MofaBre;yLcH5aB1s;O4WHIS+XAu z|4iHOfhdpW+SdDz`0|WjQ@o*}DHPMHcVlyKMl8OTh2GeWURc3XQSv zrIdny*zpC*_D^+cGzrfIH)&wB`=F1|R2!(Y@54Km1NTfKJyskeyvGfL`4a%sOY^K? zysnv6SY&%mW3BhK__Eg{{3#fKf5LmUGc;;*%VL{!Z_M4oU;dK5_3^H#3N1zn&^>z8gCw4_yK3QFyvO%4ab`0vvr3H)*5LozzJig&Q~elPZY5t%lUmKy=DpW z?vDyL`d%UjIAqK>nkD9~^3ES*`D#PUJG}=MnPq8Ae!d*K8xlv}$c4+DX#TVo8r=v=5UWKfGX?stvl>(2zW0 zc~GOz8|~602C~^098uI(b7S&9k@=b&AEM!+|F8FUHu$^6Dn3oqO(f2t+h64FpC50z zTXgtvsw^4TS5h>b1M0b*3Fb!z4C6NJ8fNPWe!&5C`^E)`y-W6 z#cQK46Dj?PWh;PmWCJo-xtXKb`jA@j{XZ}+r+^G>&sk{y7=RWVWqcoNz7Wd3=w}sM z8|;gP7*`i%tFn-!1=Pxm?`Twt2?UuVyeYe}cNndwZeNDErQX*K@$2BTQDK@K-?285 zLY3ufZ(QW$NPliSOj8}wShVTfM2f*U3J+U8nXhAXTj>T{vDywTMC@K(n8mfJ-t8yLka#8)BB%x)8|~;sIVIi=c4%Wb-QQmu=NByd z{~uv5L=ll;VMz&AL!=jy_MU@my0U(nR&r-}>R>M4c;%2>>6*h43HyePR{#>A)^~B8 zT#Q{Sh$24rYtJX=RbWkR7>@2zINz!r7vtSFuX0Bq21$7l1XHUSb3CMq!0n500a}qm z^h|srm>-E=`PCN)mFG?^V<;0$Oxdg5_h%Ufk*9CF>r;Bdf%>gPp*};iLTYrV|35DRi!I&3mZ_f{SrutH1bYEi> z(#Vg^9xK@5w5Pryak<)aVe^rhTfMD{uB47!(nOBB;D^lU{N*jZJ&rpvt;3Mk2ta)< zEUz$)ieamG?_W5UgJU0@ZcZ2xHeU&dAsPm6v|`?N0woyb)f+%mhxIbCtX(;2q+-3o z=>e}&S=yVC_e7#*`uc!OFwrvoR54h&9J2WNF0I^MU8Us#RSHx( zig6xRH)IsuNH*_=w!1t}Ju|`+uY%{5J}IfU49a(q2*vAf{?t^Wy}K}5vrQEm%G%Q( zL$aoX>%?vzCT4RAYUHzkk>8Kz<%{C>qXI{UYID()y8}`#Z+&(?qQAsC0k@2!i=Xb? zJ2+l*XJq=+jw{*imb|_s`;Ug}xx^mMGUbp8!@g(OevtQ3+9P6r ziN?WCruiB6QiHd)mW^hOFA5u#xqIXoj`KM8ZD!BA)7ZxO{tSZWc_oX|>5s6l7(8|G z8yZSsg_5 zQ1IV3nk&%u7J|6~&TZ#Q!@7s&NGNDnKJGjdwQ|?@PVK(#kfp)uSeEMq%lHd$yCAc7 zX2Sav@2%*}<+^K=@m%-&SobZ!JMdVZB$oL(#5?Gs!A}GeBT-=U_DtB_g00EGsO66k zZW_g7gPI0w>jdj>9)cPd_{+H@Kgj%ets5IFs~GZ{(le-( zRcMc#js@uco2!;Gkox^~^Yht@BoYudw1AmxfwT+Hht$5q&xZ3%V?CpH=+g=nkkq=P zBC$ZV16-D+`@K;UFDu zs4Txhq)d`>d1iQ&r!e6M@XXUv8PNbo$w$M7UtddTt@F_#;iv*jxahTyu%)wxDYe5bamTOVoWxQt#mA|%ew&MTrRHZ9zI&~AfQ|B~X=NMcH|_=T^LjGD=R>ra9K#--ac zfc`6b(fzUdK}cCY!$!-OjHXnev~ zI&e$R44t1Locr6*xl7lp+MAck!oLxHR`-YBiF&{O^!knAjLaO)V=guAlKPQ0o*o|F zAX+9zhlQmCC#oOTkoa4hyy7rMP)IFf4qHYj`W$Xjz!`ci%EC9Hiq|qf2KC`gh4bK8 znBX^o^~w?8uV?z>1r~jzA%aajD2dG>pIIv}cGT}aJN)EpcAIdzVoLQqNNaN7H)fl8 z=e$2oaV`W~eeX~I{2TtcN5whY-WKF@`d>*tvFWJP#-NZNw2Od6uf4J~{6KA4^x)BKaWu$@@yNOOL8$1YlX zoS*M5-(64vy4y4Q$UI=JgV)pcq@Pfxsn4B3u4KQvV>r7vzk5Heo6~<(I`_}GX_wrujUc_Q>ko^O#`Jfh4;8-Z`Tkz}9DcMnPUq)U^yT#iq;6)esw2BvV@mO7*#F!!7}oo1BmT>)!JNIx zLjCIW<>Ik9`Ea!E>p)BjrJzu~bw@3*SEDJ`buz4g3_^VAiyQ#uW?)JjNRK^-3dv+3 zg*7dC3Jh=lt6tC1^wR0My?>%?It9JFX8Mu#Sl(=aZ zM*un=KjxT&F2WvNu|OMWgSqH1hl{EXwh z8YxlA@UFAH z`mJkrf1Y)O(Ff{Ym_@thC##vI8Iz!e+uDBbat2-Z8<5>S;Uw6lz$SnM&h0U|M$8i? z^QP5J8_&T*My+@aboW?F*`-e6;{lk4HnawA{~Su;wCq^fjaI3V?(e7-oiuVk+I2%2 zld=cz@eiY6E>N~n zEtsl?h}cxvH{11Ig1jd*3XCHV5g6Dk$JWv#Ebv~`YV^|7>>RxzC>?}wT})cp9Y3cK zz18d=&u{>-k(TXvY$65 zv20eTg|)KGF{tB_QEcMf|Csy~v5t5<=ig4Z4`}!n#c>p>#Gr^GYKq3_lcvA!Ut9Yt z&)iA++YVd2`AKTBf;?RI>UP~^ITk-~L9-ZOG!b~B1`~tfcQ8fY@hG&x$w7l zuVW*kNcuOIR^82nhcG2K=uc-pat>!alfGh(FQ7O6jCG}6xm>}011$mW-m_g)-k z8OC7ss` zZSm~~g6T+9$|9d+{Wu#Rbn%w3@~-lFhZ-a&kEZ*+g)7q!+yddMKg;%X6JDheEMpno zkx23QXbgPAM^I_Ons+K!?pG;I!vdQIb^#Tx){5B?zA5g;6rVpE7gdZm9J<@OO%wu@ zkiry=WsaVb&*Ew9t^;#xV|PIcKZZ|rHRBZbi!*jp+5njxbzNF0qUK7;= zHV*ji>-A;i?@xGt8%A3W15#l`Fgnr{K@iysiQA=U(b3_VUqh97=2Tk>6k}HRVo1dC#O+tvm>NA6t{*->VYn) z(p`f-ME^6V$oR~EK%s@)6NSfSqrVXvCnFmO&S#f-K)>C3T@?+EQPYJ+bqh~uBO9n? zubXo#ZiJSp1JUYL5QeW07W%YTw`aTp3s7b4t}9w(ff^XeMxv~2ECJ!|7I@48eLBZ8 zbOa;NDx~^8=Y%Rx)wF}l6$F4u;x@Cg#Dox|r9y>5*m|bkICE;Po-)89IIR~E$H?;x%h+y{E_Ht8ysYe#po3CEYimuH`!W@ev6;nQtPe(8T ziZe8HzF(%iw)*Br=1;GT)?_?Mri}J%8R_{)dBFd94w!^rs$mLQXqbb|si54I(DWF) z6Yto?vcclzne`>JqB{cmdh*{3zaYA8+|R%Xek-o5qX>+_u1;?aMQ+CYieJK za%g5o?QyW?%St`#zx$-g@mp&^WZ{}VQj9?N5K(f3WXFu7o8DFd0e5r25H0^86uubR- z2!B6Qd*FjVeE3%77w;W>ujfm@yYF5;pEq%9ewx$YIXjE(KAaGao1)(^(|7y(RxH+3 z>*j($CvZ!#d7-d}WRSCcvTl9wm6bRwa5$k3eGS9aDXObQ_5+CG$!oilwM(kOINCk) z(}J+RR%UWGlGTuk{^o2Mp0Qq=0HM3 z>396bSkTONk%<61bLtg}hc}ZT9XoAW#c-W_*~-EUTV8nC?S+L?5VAlJWK5+6c>mhyq<#>-|{qL2p4j{<7PG$xd6)8U* zS%8){i5xXGPY9X$3!_(3sp8Vk!5(&z1YKZ#<0qXB4>vf&qD(E-Is3G8q=k+1GGeRM zaDnh*C>wkKNzBjM`_`M0Ve{tkciN}_y;NwN6Nj9z+J&2z!4;ARH{CLX=*1-yB9qDr4qU6b!-vA{LCIlF(y*nAlq7> z>S?vY26MJLdd&dq$tL(bK-#0L?#HU85zr`YGTjt-%~J|bzf>=c7Kl}|_i0Ww?DbaZ z>cFxb@t!UxOk3U}HbxR09Cc45bfeXtt(UB=pQ)U4#p?&Dl(p@zl4k{vunn;+S0sv( zwV7nb454cqxfSb#RCktuvM_rm886V%mAcere|G?Y1Py~`nBt;6W+h$J8&8iSy9p9( z;o4$5-!-Bv(GCisHS`y)*oXs=sI z-Z@kJ&XF5=Ksn;;=bA2#UD^L3=C;`Dy`Ru zbQ_T_F-}3S&BZyLHtay&^85`5p$m@GXw`wNNRm>~rWI$49D_5<#@&)=;8&Cn$%zm? zYok#hIqO14HQ5(tjU{c#9pMyBszg~{pc#EB^_pgg?$>w^ld<)*vQl_kvZPuZO=#`l z8P+`w0(T_O6$kWT$d2YLUb9Tgf$g<&RAzwfjfJWekX0@1=;slPE-dqKM{Ih?7ab1j%nSYIJ?RGSzw-@EJ0?GegE83* z8j2gkR(j%rlLj#jID1$Nooj@Ecaf1a{`n7=vF8xpgk@T~KjP8pcH`8YEGU8Z!gDem{JqWCh5AWfJ*Q^L zV_Il|3^FQ+S~rsaQ%Y;=XC2Ns;z=$MK-ug{^I^ojFKGG?h=vSU8Iy7Yg5}W)xDhyY z&d^3#$I*`P9)hSQFvQva)h>Mw8-33Ej3=|+{8vIm;c~rtxR%iK_GH=Z-+!`;CJZYz zgXAe{4a!VoS=IXqD%bdU=AoACoYCRV>387@urw1Ku#TIRr&5KSp6-PI_tWveeE8u{ z=c?ZS=wGhY?fgFOuO9`4Kolvo4(4-fmrRd4qe;)3kp=Rd(rUHwHZ6=tvK|yTlPUR{ z#}vobvUSLsnd-Gd5TGw*cVi40)V+b9JR~%kp6qNFp25DJ+p&(Dc*3wIriFgS>Y96W zAUR>Uyi4c1v~|9)r|Hq$<;v;7YA4JhYUBLsda@Cz`xw^WxQ6nzmTJG>56nih4>chJ zpb%YQRB*=vurJJ-nV$XJqOmajSKLnpTh}uT3K=&{hs!>EsDros7kcx-~WG3j$^ssx{XMY3;KRbGj1>DhdzuUno|b36V#K3szn4rIKGa`-SqShIE*|B548&kT}gXgRATkjjjA;VmW*{*nsad z?DXX>wNWVxX}D#;=KkXI37p)tP{9Iwasz(;w#e%-cs-DV-k3~}hjNU^sIYfw^gP|x zY68i}uNR}cSub$yGQkI)*w9S>4%u?6vwK!)i=&2l90eM0hwUiGn9XVjw zUi;@p@Q$M4bz-Y;#h0$AJPO=a0j?Yw?y~yf+?gzkv?HWuvW!QMCoOiNQcfUv-2N!^vJ!7+;+!=!GBhVg zVfPJcV^A{%QA>@n(K=WwekN@-dM_(Mndn04zCF*#)YwYhe?wXQhTJnzXK|KAtW+8^A;b975@X082{;N6FSpC}KuOpVU%dASh7xO5tdn(-0|z ztXh?icDn4vXj(%uG{W$VDC1_JG9M~mShodR;dQkng%K9EaPCzlA|(otkW&Ff+y}M0 z7;r=Bs2u_@5KS3P@m%qdZ;c2X$*{)iNNX+LKJWnU!nN>uJOHMd<_-YQg)MFTCLpK0 zWxz&xL4LlGSmO3e4!Euro&&1R;<8oB6G}ZESi_9)mq*?fdNQLS*as+-J{3ocClAS4 z+xL(KRH5*wZ$m(>MivBqQEV7H^6i@u$glpbP?Sh28&cDpwn000IDhZ;+_SU<}4kG5LlJsx*>mn7yaG`kz+#uhtqc ze@kLLf)iwzu;zV>cGT9K#Mk=c{COVm$N2fv>Ng z_|FT!-}CjoeJx*Id-o6FbH!OfkLCtI|1%}FTE_vKO!k{{j)UCq0x} zHV8?09Lo2KnYIf_S-ae&x~O_?rsOb6)`|9y$Ihq17f;V4oxMj{ z5wb{2iRD{uN2+&AMtNuC`JQK~eyC`jSjxy(_PZ@DPu_`bw{(j5o2qH7{AzM8NB?0c zq?*PNj;jf4SElon1>`Z6YL3Sf!j`sqhtJ^N3&}ZwRRyaHYcs73C9k9?uTL;He67O# z6EQO_V!3D6X(vq|kwg#b*QeJqb#~7ax-vPP*heaPI-QgG`ovzU$eG{Yh+mxTGw^wp ztAE|2v7syt{-4m(D?VNRVl(In4E^%gMmu-6T3!#YXX9F#%qRH+xp-pNPLNA5cZaT> zLUAqUQAa3riISu4D2&z0q*OUsoVqdo+3+dt;rG8UDroMw-`6L-|1xse!8|gVis}3G z(Ulvmp-9W)WY8EXNpb#griqPljH8{+f(>7Y?XUEDU1Jo@Ja~E+wwAZotgmM#Ki6lE z;;a|2*1Nvsje!}^^4uSeT3c>~(=El-P64>BFKL|+O4lomN0Cr}chOp*Hl3d+#mWd- zq{-0IYuqt8L1d91PK;{oH!dnYs>xi^8WdTECOZty{6h_aduX-#y)?h>(vyO&64=dR zO)a?c5L9)IzJqJvsAmF|#_RA))d7ZEWl0(zEYs4f#)Q=eq`tJ1jsMW7AF`eX0qV3r+ypIS`HsSznW4p--GYY(A9`DM0`h+ z3B@pIPPyMjIp6)+Xz+O6qPUiYsYjyFC^ zS1e`IZ@ZUu0O8(4#)$M;{H z8H1M1+ahUlvO2Sj;afHg1|@b>ROTWsWB3QXazxzLebVOL25p@blLetyA+u%j=M8wPENn7Ae|W#j>5o zKj?uunG~mgA7_0BWxWdTBG`@zoFi~Rs6yP5I+#muet3B4VYTS;5zgE|>)WxDig2{s zsO5^|bACvynk;*f%_+LtwKFL0|NM0DYBect-Ap^Z?>@8#KS#i5rb*8wUVHWF*|+!A z9d1a8UV0eb+^Yj<@4eWBx6+kdyk*uOp#Hl2HcXpBfm=Ra}#WS8O8$H=&w2t|-#v5OIBde2sYNb3#+ZDORNC zmoB0qnAqy(9qJ&|FE>;J$;XhA%;J^ik5r0NZ2x4=Cec$3a)REavo%GMGAMIQ-$L?O zYC4VUmzXn)w>9=#m$mrvmDSzp9pPP_Jx3bg=?^W&=D_`rXgtkGrE2}M``rs!=6c;= zkNjI026Yh`0)wI%-W}T79LAr_yQ047>G{Wnj?ZfHO+OKJM;p!#eH)#>zBRLFGw1w+ zBcf%(nOa(Htv*H|;70xP6F+fn<=SOlhWFEEWw@jmwN!u)Pf0<@yCSboIyMQ(Ey1f* z!fB6c?Sa;5KVR&6UQPmZRU|+d(wtMSPoIcSqM5s9gqcZ*rA+z#<+_Q#{AFaE)Uq369V8>H(v8r$Ox6 z#swXN<8G**|KXEVG8CrV52|8JPXRDI-;gWN^}-mzT?b!%-3*slYiQ(H_fh>xBwGyd zFXy>=c@mxt4x`UL2v3QSC9~Mv8iA_|jbqV#Nwdf5M;0BAW24vL*JXBmZK4@zevI=h zq5Dd#KiZzn2|tAJ&v6JSDH(eJersuxp8}B=}l+!3QXOeA3zpM zA#hNXVR;wj{S3bJy3XUa>j2yXf$i!z%Dp1*f~@^(EtNhy^PA{Kpl2C&1<&THlLOgf z3?6_GH|l)ge`>qY6?1}_T7!1}z$w|@{?yEj|+Blbt@ zUJ?%+ueO-}^Eb%M5oc(CqX4P+buIK5rgloAR`HCYZp!9^*qFtmiuIk8fDRk-L3JX>5+78=Bs%3pUt@sFQod49dG_y_z$7HE~}<0=*g?5<8#~imypx3 z!cSZNa8-B#0nQWy)`0y$&&wLHHC^bP_$_}kGyGs!@e^g#adyTJ_-qNtl#HUYFJS_X z+#j&c{EVA03^*fEj}%Zb%WiO2ucrHn4KN~nR(5ok@VBJlbM@1lqqw()j~~dz<<93; zV*@xNvbBBx5Y4S~ptT%UoSjzK{N6UtvGpRmu7&y;%IO2k#qiL&QTU?ZZ7!I+%JW3$ zkF^&HpSD=W!Qni`vbl#To++KE25JpLtC1=2v#0dS zU-&2In=O(rB>_^!x7VaiWy;IdPMc2OQCF9i((|qAPkJ8SwqWN~%rGK&ea1Sb7Fo0a z9)mRaxFYZ*E+kPQ1{IYpih_!Oh=An+W!r<)pY=h2(5ehi-M%E7byI=k(~ z@=!NDp$`_~SP^IB{!mO#m!$Ft9{>WVByvN|$Em-^CdFUSex&#yENSRW4GH-%>tz-j zw|J+4kr%BzBwZLhlJ#HL4@N$YN_@Oo-tuOhJ>|a=|6ce|PSmwzT$z(eYc`=@`2quW zb!0>jAs1aw=ehSlMy9V%#PU7$tyhbj*&4@Q(fYG$<<>#b)+kk3Bn=imZ%~&gaIAGM zi=H1d7In^@McueHzWa*+DhIRiCze%SEnbzFLYKNSmhO3{X7(iL71O!u z?HN>qw2^=6pXLE0M_zG7?svmU8VvF0)xCx~zW=qH^?%R87Ne`kI%vYs#TFUZ<)f~* zE;XZU>$S6M2>%u#L;Ht}6?4P9^GQpswS0?-GH6}(d``tU2Dz*O$WG*FdH2Y9ub z)_>;Wd??OCfoyws1U}OzvdHK(x;U>#dd?j_s6F!1RD& z2g2)UbCB0(f-aWwNP5u)H#{EU0O=sWGyhf=NCfoV3Z(T$ec56q>h?q+x(!09(Brb@ zbd8RWkJq+kQ5>a|PKa^8#V#AOTe8sJkJ0>&KHG8-eeKDUmfDDj&~TLFXKuTE@J#92 z92~EAh&GD?0s>fY=DZ^mSvhph)?!$?`-9&gK&#!P5ncEDUYg}ki z$*ztzpT!$?rv@8B{C0x>*HT%N?@1HR`vxTA-^~^;y-HV7-50BItDnG+x}Me-L zVD~9Fs9_qnYg#3q)y+akRL+ zzk(aj=(IZWB-A8LzE|Bcc;iHddnrSNjTr#45?ccWOz627B!;;rq(^MF1P=b&*$}bqBh}`50LWQFS7zpBX>0_|Y^AC@JNmqJ zW=kLoNEATl$8Y_CXoOVBOtVzwKADnsIjbPoFjg9Z>5dU<3JcbTZXS#ti^Rh^uvWYD z1Fymuj)?81YW}nNr|Z8C4=8?qoYJ1YUlPvEUHRsM`5TC8=s z&^vf8RS_Q(_d9y0LD!AKzli*QBkE*n5Iz7Ym9_B-K}cV&^h{xgA$G~({QVy)aOH)d=N66^9>I`?w5-9 z{T1`iBaZf^Lc6KimYIs?KDST0$V_c**j#R@MO{P1+Jr}1-zpUFKMc0+RK)H~2$Byz zx_F_uqi>neIQP!=!3rUiI;P=~HE&q$>Gw*!n#DoYVG@dn8*x&!Od#nwwSy zj@x{*8iO_=6O*r>Ub$BDF-F*NdL^)4Yx2ct;%Tx-MCzvevz5F1S~C9T3eez`+S#L`HrYpej6%od!2Xp4K1 z8|wl1t>=WYwEG2$HmWW7Y#K7X!u3Q_aUlI2G2@)~jCV#t=O}AXq&J78~!W#d|?-p*YFBDaNh=A;6Q| zRPF%0<4lce@~mT%tLzd8CgkPT@o2og16}HS%4l1(+ovz>6938vb!P1qbh=U#&)lD? zt{657jm+kNd)_9gOa?QCw&)@oxTT^e(?wqY8GuJQe z3Uh>U9gwx`?&xCp)|}g;x2+b9-)8K${gQw?Ln_jOLsDwOP?_2EAIMOjK72i3l*AJO znM;Z_QT)Ye5$vY9=f%N=j;E#zH{kC(cRP==76Nf0taAlAXpL00k7#xStFp+5_ZT(a zA+~31*d3wH*8(`L&&%__=E0p#36DrA2M^&PAJlXED2{I!`76syL}QD-vo_~8aqT@~ zA?@LSuD_wG;8a_+EW4dH27wj>>|Yn5|8PxSZX3QNgZ^w?-EIba-l?qB1fhAPUiHDA z?XQcVCIMhl)5E}1vFGPc-plNsa5R*Vxcet{pjN7GuDtC?N2&UE*0lM4nYcIgQB>HE z-~RoL%m4O&pWb=<>;3`D*@dcv+=y;POS5-}KCqe8*^Ug^2`w=)%W*#U>@@n(=Z)cM zciC;uTz&%u8EUF8xFxeqNDr}pCAH&9tch}t6?GbHX9KBGIG9B~PXCazdtPoyi%c$~ETU(b(RasfNk{>z7mVT}e zr4q|s$$`65y@$F*zEOpX*+DP87Vu$KPC{&ZYIx^DdtzSK1!Ph8%<_;I!cT6cCmt@NHw*V zM>V+mcK`5eXIx#Ip&MoWsUF5HHE=PXSGywu2Pf?IGyg}hdM-M6TD$QqyPjTB;2P3( zFr$8q0XCQZ;yLvlx&4%Wym0 z5&xRO1X*4#%drdl31J)Ysc1f*OUXb=JESkK=^ zbHV(7To`YE4@vWR(pg?AK{)Rsp}cRimlZA!UDBwGvoFIC-3Av@&*CNcgJ3;J*}hIl zRnlsgCg%5vJfi~pzJaA#0iV*l(hop{ci}D6RbPq#VksBkb_2SHapg06QB&8lf}qRi zP~!aSR8};yjUrsCYJhm1YNLncc0fS19TFa&z^|Nd!##H@Aq_Cnf8Hg7~=f>XmczF@oE^73JMMe|7}UyNTO_*PMzbp>k+F)V8p%)!gE zbzoKM-QY=rT0VsSq+s;~%SKNfRBoBect*lU0h1zh8 zxH~(z1`rsNTs&M4W{A!%^}`3%P_hOx?ouH(t*xD}+dOj*yv*~yNpzIo)8l)+SQ6dF zzPD#!vpK_aJwQ{qusCuUX@3b=XkZ@Wv=1v*L#d5%Ozc5IV;sYLkSv0q9w{kfyGi@^ zI}l}5Vg^=5f;H>4>IJYnqS5fugaHfu5Z2BTi1gL#)!3~`%eW=qF+w8j>~ecE+=Alv zk$IrvJyTIN(fc~EDSehWitjo?Xu7W86c}L57NMkO()B|olNv|m&^yHOx)}**X_A^^ z8QUwZBk_y#h=x4!SK8H?4g(Sh-jT@kGM5bLs6%b+fmOyEoy^{{`Slz^f0-;i%PQR}Ed5h2j!T^jC|6oeFA7#}D9<(>b2KlAjDxtCrYfAw>g zj)Z*G6obQuRAEf9y|$c?&~s1tjtC0fRjgaAAZfRzr#pUI(xRF9uyJy!BJLD~B5|_v z*asu0Kd>_A&xb(>YJEI9y+}89UmI51tyeSzw>CF}yy{{R9MuH9tg>SH?2Zph`cllH z`~PquC->`TE0=%!MQUpB_g$OfBxKFPj{3U)y1jMF;69;>`kas0G2WlK8Grkf6yIIs+LGwDC#}Wv>+ca4o@ASqnz-)d{`Tzs zDAx)Xxlv`iUf+L?v7*%3`etr|IkI=LgOrWowm-LBaHzcnFES$oW&M%=Iyh5RlG-NM zzN`xrJEtWVLeOg*x$a4eg;QC2s6_V8-m|N_&kFk(t22_3+=5&#zH8=#(Kuc(M$Dfb z;9!dusz|(=Qf4Hvd^K6BE|FhM;gg`0fOchx>&ByWuXN|Dc8hy@lFa){&lMMiD(05& zhA4hf1Q)m#C6V0jsDH8xX)$Rjk8U^qvm{tzB+kSzFJyHTWvTi}j#rv^V-AZ?C$lIe zT6<~x^u<$w$;IGf(TZ2kU~dOTBt;GPX4}gE|DKF>YZ1s2H{*+uf#PFXv#c_C6L09hxWklQOp$& zs$eTHVS)_3vP1*N?TL#qjMcyB;G!z$tX~a-K&bbH2ERgy?uiDqq$oatrX@ znjQ^3Gh`%ZBg3!KY%==F<`;UY>oY;3rN6IliExJdbdM|$<-Bo_i+nVj=w|oc_j_5r zR)Wu@GzfgHNrsi3n#D*cEpeMf`US`i3|(^8Msvzu?K1!E_lWR+RI5DSxlKYdki~w^ z;u6vwKN&{G(e~;)p(D`nCDVu4AqyDF~j{33(cX7*~ zM28}c2yOnxqzxRc;|gx~_E4&5aANKu-HfA;HXA5?_SfIAgdNa;rOBH77g2oYSfat^ zY9V@-{F|^<0GNIdr_>bs#;knd8Tp8@!JhGgkxp<*%3joqj8uR>~mpjj5 za}9@KUUIGr2Cn%pzigN*-*}W55WuGgQPE`b7_w9w&VveA#i}cdiZz<*D=yXlw50Lb zz*sA8gqyl_{>djnUf+||=kU*#++=@T9j|-qy;VlGv@y11;no$dxz{RrdkJ2b&$5X2 zmi|{FclCfcF*=Q3w6IkeE7&?B%01EqhX@E$ZV#X+Wb(&I-;%~N+~`o+qFIg}c{(fo z(a%Ht=Xoy8Jz#6 z7`q8cf?)3S2h**`jfc#Kx>h{1HDW^JE%a)We>mg%OF+!X7PVr+?%};C%j`eN*Ca5!URRb+%LU%4LtXz7HS>S} z=Py|oY8$mh(f>FWx0^ey zZ0^t(pG{BR`ufxVBl~^i@8SwJwj_4Xmp)71ud~{I#cj@W`KO@|b-^KQo=BV&9N`B9xYK(qIRCR%!qnHko zwl`U}+)=;25Hp|et%JMBN&|apmu!%@F|?PM=N_p)jothMkG6h(_j4i z{SXsjuzc)I%)Q(F@3$pJ#&i|s7Co8n_*gTUsmN%yR+Cze{c^n))-<(QC2N6*(&WRU z*qX}rkr9cdEh|+FJ(|Nwxn?DhHN$X5lR`pFeu3tI@DjRqk4M3@jcfOQ` zp%}6f$Uu!~#ZgI3%9YBCCF9Sp1kc^*yV|9A5!_%M{+u=by!OaoVM<$-4Qx1FW3}XM zlD|=RU%OT|Wr~BOg34Qvt1V}g6$m;7vk4(>BhCk?yQ(4|(Erp2AFu%k9WJ)0-4YZs z^xE67KF`Y=i;Dhn^8zn4qVxT_!0E`TYY|+?Dmni^rp;uX%<3!C0RF>v+4m`JZWqM- zJQs~wn1I$L=aFn!mj;18v$Qm-zFNz0C`&bhp2nHk+4znbL9qp9KK;*g#;-sw!yOce zCZ^#Q4@3jMHR(^+;vQ5dK(OAY=K1sf7zPva6=Smed~C>CyOfiGGDj01b&H&>+{Vy~ z`30OfDTHsDyV6OcdLx6kF~|Ea$fxAA*63av&u5w1zuqsf zPh-#kIj8z6?Sp^0h;NI^ETC#ayHEQc0t-Ue(IKK|rtu&^&Zd188ybQHQZ56vm5F-Hvb;*>xO(~K z0sRvM;~AX8gQ;ijIE}Yb_7mh~y9z*F1{ zI*hxFcD_Q}kBBx~d6qrA4-@5Zs^j@x6q5AjE;&b3##XEXK8PluR@C6#qngp$LX%q1 znGM91hH7ePV5@pGL|$OsOQ8+5;uhc?gZR798^Bo4@K2*uCE|1-slGdEdp)_oJAQMo z#=-N$IBlN7nW^`kB^tVrkR}zW@mQK?2y7;!e=GKEJB^)A**e|4Ov~A!AjdF!{PEo_SC&YXE$aCr--X zr-Xt?(`VWmk)#U86h+G-Z{ zE!~OMuDk(s&W@}dbH#9`*b$Jxa#gcRtXH_Yzl!c{pK-Zma_?g!losu+#bqubW^Kk>MH6JB>+6ba^$Kv;ei*ve z-hf|SqIg>Id`8Sa3vT`QSKt2oTW|g5`{is#Ms_Nu^Y*R3%2fli;V-uFF*Em`%a|PO zajEj*Fq68mb^q(=SSe}^)mHVvquSOT;HU+0)}bi*a7FXk*tise_Az5&F)>Cwq!{ddQm3tJuVeC!Mc zfamNPtFo`Ax<1e($iH#t(**Zm`FZl$PIBoSHmw^($f>Yjnx!+mIa}6DxT_pysw5_VNm)?8f@bwZJkInq3>t0~e?KQcc%2V7{tv{TNjHcu^ zUA9O%#X8anQx?-S-MptUWpVla3);f$j@haHlH9BV=A`rFsvig_5#EA&m;@8s)GRNuT@=Tj)G zUZVsc<>tIlMX#O`Oh`xAKV z6L^0zz5v{zNC#L>L}`$FK{#I`Jey&l(aNyy%wT03X6AkK4>@GMb-JzA^wj4p(wk>B z7`nG4`f<+ebgbcpy6*_2r>$quwuSUdDl5Nh%VlWCurwC|gHBCK$rX=^l!I)EUe;7@ z67ms>qfSLwhO0r;zvZYl$mzC(h;vrRgBpzlQSa1e#j-QmhD!vI;~Q2^LeJ}kS+5wb z2&l`3j~l%6ps6wlP5a_q9qidzjtF$59$htlQO$r#qzjfMaq=R{=ZK@(w&SQe%1_XOF-ZmW%hUfYMP04Ka+tCPZzL6B1gF z(CIML&SP1Fz+DAc>2^MPV75VM0W^mFFn&Fmz;Yc_Z-os0ayVEUK|EU=+CM37YLjiv$mxRy0$Jr6N=4akRHw~h zde^Ti>PK^Oe_dK{=bMstrLP=f-K?iQPY}&C-_8B{NaHma2>T#>yLz9%+>;K-6cw8} zdDtv;3nAg<{|Uk38F&JngsXAGS-SY;l z*c*M)+B}%KLXzH&VMzs$%PD^+@Puq*!31X+rb4YAo+|HkbNGC6l(}Ey_TtT>Yd=8V z5dyD>*5(^h80#tOn&zO9T(L7ir1T6(VjV}cd=nWPbyIW#EcKo*gxHAG&*yIvbJi#;*52( zsU7!kOnp5)BRG^6QfKPV?`V>TseD@`RR7dYns%Jq!H*uulRK}smXOG`#1hk^sXW!z>w^pt!!`LFjLRy5(8htY3?-M;)g2dd!^JFJ z8_v)TXRUVcZh&>S{b^rwR#;N}!U6WBC^bzU`a5|$2TNP+Z=l3eP0BW^Y9cRJ60Mhg z^hF7&C@ZC_;QR}vI%TZ?m!sIndv29<^u>yNw@5ubK|vAe6&?-F3E%WU$yeWgr5j2+ zJ7LSaT^Cq)uP;Jf@K70CrJL>MzpV#0s1cQTWm^09tR1b>5eX`~)H(Y=oiDpI+)cEl zBd2yI)5PrlDsq3mU2-cSRh1Tgl{Q~@?l)>he#Y3X1g; z!>LlLY5}0y_B-fOZ^y2V4kL+@rdIM6|z`gk8)`?QA1cL#qB;c_L!OHnI9q z)&FF62|=#{rwL;~a;zw%RJ`>4iQ0naPcE)m|Moenm;O*GiV3Y<9gU3UyohwDypzI= z$M2Z=`P7wc)dO{jzAr33kIE8}B|Q#vy2^KT2l?oiAyA#hDPrj2`H3&B*lw2P^%n$; zxtV;lCwc+!EU8s@us(@xB1Ml58UK9sqcRpLYry^$?x0bZPge6>j~WsEOJg0fu*6(r zh$vs$Vl-yagXBL32AZ31TIx8);t-Kw(N!w3D_@t3&O$oQ>5kFE2blsNe@lanN46^Gm$t z5`nQD%+vT)yDpjFsE8?1jW=?qCVV)8D1YS@M6iyAMX8+(we1qik==u5n9rh|b}(6# z9mg%bTTNM?L)*SK!w!dVIr6)0f~SptAf7 zKUCQ8&7aS2AK~Ji-C;P}C90~_4x4$%8{t^i1XjiB*uq-~)-z)d2iIM)_JmzF1mg^; z_X&EN;pn26LNxulx#TnD_vazH$7( z%wD1gcCrNy70r=LqzHZe^^CrH0jR)KfCh9eM)EutcpiX6m^c`d`#XYb=wKKj!|_wF zdNEwRb~Xd%QL9VH<41@}!6`kUMyMxb^A`ja-e`umfihsqgg+?EDmsY70XGMu(S@sB zRl#JW{LMrWxXDXpU2?qf1L=(V6m}74oDif$R1Lz@c4>Wi9X_JA4Y(P6YasW|-zTtE zVGQvdA4tMH-vS}S1bv8i>Z{%LKD(6Ze>Ge=(u*9RJIJY|66vhiJFDg2F!BO3bjLF` zK-(Dpb1;%Nm`C8_4oaIe;NC=(PQ#Nrvsk6^PiaI8`$T7?3+>s;($z%^{K$)~;b{Ah z>;bAnxt+8Np}-&vYPqnLo{;0VJ`vevB6unJHv;MdOQlWtn0HJ#I?!3|1w8+M@Jh9b z@_4O;9(Zyv3+epr`k4dKOJ|Am!~8vTeh}4RhfW2NTvSBTZ&?zs%YRaU{(T5$S<@8f zbbW;J`Zy@#tXp)*5vGnw)FnQTvxe<%2utfVaIzrNcY1PgaK*x=SETYXbblZMf3Ck^ zszMKRH|u5k``!kDL7}!S^1Gv=iSC^l=E+OLkiTuN%aaf-Q$!0){Fhh4k@knxjj+w} ztxA0h_kxpDn>P$i>p^}1X@GYoRRq`dtZZqMyBhCowmOJd8Y8ygb2j5b@I!oD!$p8CaR|=%VMfYYaw42Gk4Iqln!}o^T z%Zh2lpOgy}Ehu z4T^btzW+Mkuf~`JdZqfpm-*jb{f|Gt!|g`O;WsVoQ#Q_BxEdM>)mp<{Ox%sar+?~M z<0qZF&{q1x@akL%Vs@E82y0jUl3MQ9EQMnJf*>?+N@`0^)t?9W#jWFkfq~|@e(C{y_nWAlgXc8kcdzE` zW4od@^&l5|NG+xLmS3NT%2ISLnR8!u`IE4^rrUXAAa)^UM_YaBLJE~pM&UcAkL2N} zwYU4`WM+XC9XCm7QSpZknt z8kjMwg)B2jR@vZ6gxtv{0y?bV<;rj&SXL@K59NlTa&-^Q(SwN*fs(Umu=mZ4YZ>Ni zhgutgYcg}S6YF}`x;%kyIJ}e4XOn6+Bh{9tzVyzQIBIAJ@S0NHc(+Oc@EU&IJ~MrF zz7hIB^H0msCtHtyKHkR67M^Z|#oP6WQZ$D((n!U5oCr)2=%6mISb#%)V%>Yl4kcYD8us9T9jPQY$@06?@rLIk=&U_=&2_GJtMb z5(KVM;WSC~E6HEavO6d+b?`!T(pD8>*e+ek#U??DIKN7C)6Rkl{ z>X+fqVWVe}vo;b~bD#CuuNwjp;(CU&-mMUM`BO*R!?us9e+jfB+69(dTiOwsqu5idX`7c{^<nXbk{e&=EIuNOq1;r2d%t1VE*peUFAk8q85N(roOE9u{bQj7$w6;*da?}IDPVVyo zNJJI+#Oam*;X)vhjEp=su>()=K*3uGI=0g=&e4amdets&|2n}gU%|JKL1V}zdDi)B2&F@PD5lp?x2(7YieOz zCr-m|944+-T|vwa%vW8MkG&YNWHs=>k%`k?=u4q~c=JC}8mcmHflfEre(1gD4>pko z8zyyBwX%>50=#LFyOREwwuwmjX2j?&F}1sY=e^J5QBA&HI{XLsiY$l89r)|+ga8BG z6yyIHPKUR4$u}7{r$OHY4|O}D-noFby~=JE%n#Wkqi^>uO?D*TF;zA2SkEfSAG}wU z8?D*+9Pzlf3*nPHZ|7R4m^3})J-)%cIeNJ3 zN~fu~=LdHJKZhrnTYD~S#Id@&3XTf8nn~mjg1Kj`U$NG%*VfaQz)LWU6fo{KclqYV z<+SGde#@B7q;^R#r4)YtM_t{KijaFtwGhIHS<_|WP1kBmpK4)OVt`G!)yUMJnvFfq zM(tBzPRF@+y4sd|+*x5kpE<9>i`sUnr-|=<@OczFZ~E8g&#YMr1($8y@2b@*YEF-3 z2_li#fGrMJ$NL0^t7Qj$)K)%C*&mlTUO17^a|fc7>755sCx_2aPO0ebd>>N9G)%}i zg=M~v*iG02=5wRX&<8b7k4Plbk#bId=>#Q2GZQO})TFJ2XyT`y&^TgN9e;efhE6Ri{YgcS_M8uabj#6^R^LC=0AES9^8LxJ63ScpWpGP67gNe~N5N@5; z-_RCzExi853_p0s#L5_8kIJ^$=}@NA zLqo$ezZ`Bx+HPdYIkPzGTlcs)?+SuGw0Goo;XPK_f{bX2%%GjB%LaW1W`^%p@8EfU zWmmk|Wh%c}&s^eDMA|TxCrq$Xw64~+W%#|K0|wbB;mplO^Fl-|ITwcB`W|tg*h0WN!S04JA0Ev z8GvRy-Y3w|)|ycTE3aChy@tmweLw&z5aTP^KH&lh=geUz=XRw{=9rg?ps&)G!mFFU`K4j0=Ep5+DFw}CG4c17RDb{zi_3`C&kXHBL|+s#438=JgbXwwR^o5H~v5o+}zaZ2&?7 zLpO118+tff2m*L;X)pKk*mnj-`69S`Pt7jjT66l8uePdqbR&4&0CN5W2C3AMizk8A zR5sL1i7m1|cMhrOLDvIhrcnC(?wHbYeSbCHHKD?u3${+pXFiUq4L{iYZR(ulvGMzd zQM%5Rj~u4p>Sma?mHS_lheGN#YTA6y4wbcuJxz!NRq3(yp=R-G>->EW%}n}$q_>(#H#G7q2> z59yhXS86rfMB)vS^qJsP6Dsh^75AmlZ~yb@JHMU3^lZqqPp*crvSehU#U{BlduHz{ zWS2*rgBdP}=PyE)EZ;g;wuzH-gD`y#$OSM=>YaajG0a|5o20T&>wqr& zXxckCHM=9BF7Pws>U0MbTbQRFvB;6gA6o6-ClG<5H9nJO_xdgc(^{e#_n-Abb%r7^ zKF#cAb~aEX{Fi5Tm&G|QTeUBOb9a}XOAY%1IM3FAAa>Ksg{|^i5*~L)<;iq7(o^=I z^H!*vPUYrOasf-JEH_;W$}X`k5m*24C~$Nz=YS@*VD0a6%7y)^{bwe7&oK_ET#JgP z&2KSKL?|&mtUZ#ruU*9)Qk$HX<@}DA-CrFq=tYjyq^OgvKm9(T{oWOM9>8Y%4M};( zkajQ)zOvuUQHPeabr24jh`w=+im73=j?a$3-BjlLlbK)gebh)Ugo|#f ziWmB4Z3qLbg(bC!6#U{3w`cZ`y^t)7m2goN3zN?a-XYV54oBDJ;h>CfQqFEW7Zc#{ z?NJq)(;(EO@OIijYrB`s<1f@fR^!68Y^HU%A z-X55NnMgkwkg$D#>Fv8!`JNC}qWH3EKT3ufOkohngupQ`Gz|QMW|2 zgFiQNF}iFtx)n}w58;~%@Rb|_hUyT2@vr&Bs0}h&bT8CbOQc4Tmn5<^h^%2Ox12?D zG)iYujb4k{>obTf!ST9GM0|l~ao>Qe6-@m&JHc}E^6MyOOIXGqmd zP0T&nXTL97&Vp~qP#}l6bUXu=EAZl!VBjmr2`2ZgOu%J5Qm)BYgiFDOlOvrK4UkYn3nF;o?dtdKPb)xo%>6@sz6F_yW( zhB7q~+&e`kB?yRY^-afd^UwW2AHcUOeCkIT=l#mzV5DW8ZXTdw(*&bjCNu|z9Gk(x z-`>lOh6Y!CnX}%+c0!hvP6XnATopg5v7O>3vMWaL*c=CNuZ335$DwM5yrsy2RMG$x z?m_sj??JFS>T}3){K8v?8hU`cnzc?Bl>m|^l`+ZJc)tycX*PF59fhUSX`?y4z9D!m z4cG7ybV~zq;b=5&@hyy)klPsF)p#_;6uZbtO15dH_`qA@b<<%+i^VWKgF zeJ3iuCk(m+DgBZckhH7hY+mi0 z_$>YEL2jL<%$O)^@xA^{kU1S?D);}fr&}@>MFmh|(Z($j+Z-aY-#TdhY~`=%8SwiL zMJBDDW0x&9BYodhUmk>1pC1Cn7Bm`}wx{q^`$@Q95#+-r_$%<~S;0Ns-H7T;0om}u z!SFZ!<=^i9%YXjn$e;i5{hx1l8FG`-JH6jGNYdu;{qA>zlF~#4u1{1e0eSeXKTu}+ zX#0}BY8}Xj67--=OwjaBWrvvR66mM7jT@#W%gYy+XpOkdTLI>)g)vu$M?NSC8-2_0 zNo!Z42^Jh5cAlP18w*Np{NsaNUds-?gWNd7n%Q?dKadT;8hpv<^XYp<)QlANto-A4 zc~ATbNCsvEV3hVQm9zRjjEkxvrHn!xT;NvI4Y{mtS`KT#o8NzAh9}>^nBkd*Yc~$&%(P@`{H62as8!Iq+!STWlAdy6J&!D^_>= zh)R#}FR;nxHz^P_AIX+#iZp$nt)Mnv2Q=bY%sl`T4?ow*hl%kM zxYVt+9KRu`sb({u!_{{_8zzbiu-`4Xt^6;X@jkzyv(~q1A4kNuynxZPer!BA6{Cs? z#Tnb_b<$GufNrKnG*sgqgp={W^=M{ePaI=>l(ujWuBF2#8GQ*@-8{e=yJrW|sn(JM zqRkzmt#F?2pS&Kb>0Oa)l(jBqd7@jrmB0$dX)j58SFUfv`kw3#>&k%}WZ5~Eh-$uZ;vxrT9nw()PA?DDt zpjs&#otkhKWoOvV5+mF(P`!G-1PRgdPw9Z&oERi^fWiOWw^IRog@Xy! zLsm>kh?*+Tt7#h^?@^AGZPv3kPO|*jgj4MP;(5iE!~nPD6G4S|-o8 zhi6&kIak>gLx&v^>HTIqrn|J8QvNI)e2fXWl>&dhxUmQ7bpG+X?0S~79s)K_mBwY% zKrOoB8l+b|!xXi9}#tv;+VaBuJt;_#^|Yotwd`+$`Vs z0h$Weq-C%ffI_~=P! zcOV@@+_Dttafp-Q39s;KQ(lcS>{)RNV*kDo*I7siqffqw;99#ibzrd z=X57Dlc&ChYvL+Q@D7vL_#xA9MC|t!5QOemEpFeuvtY9PseRm74!Y*l#KGCl z!M#YD5q3yzgFnw=ksIFyXy|%IG-TlqBYh7=Zi7gTIR(J&SvSXXvIo03k{qpB9~Ig1 z=rDV#jZPbjl0tR3OP&|8yQC{^T)){+&^sqiP<>hd<@*cVT<$bV! zDow-QT4QZ(yJ9UU?3`Ew6|1*n+{p;?q8r*zes1Dpe6kUt&s(Pyi<=frz(RGh2QNB*r-c&VRY{`;b+( zm=@!qPyhJv;LRke)-&E;1~Be#!ax zw%r`tU*=XPiHgkW1`MY{(@2;@AznapQ1_}aU@1}UoE*{J{S-G%Nz@jL`*!rG83pIH z7yJw5g7|^rQz5Ah)-(d=#pfY>`=Z^XHm$zYn=qILhkHl}(23=YG=-y*Ii=IQvviB;ZQ_XME|)1jC=ik<|%MYHERP z<;kH3iA1whU*d)`#^a8Tj#%?u+8$S1+ZBa4X?rPtJXQByq%kxE*cl6jv1-_Aew5R< zM=@E7kOL3+J+5r?6<`4u7AGJm0ezO0Wzc{hMLd$Q~joJ4n(nlYl=rDioQj=!fq zA+M07U3snLTO>M?mi>E>TbMO#(9q;1$y{-5J;@f$k(j*O7uiJCE@m3IsnS zw>DTcTP^sI>WWrG844Bp@!GYU_B!YwqSUTQz8w)LZBnHo+aaq5PULJb(~Qg0=@nhr z8`!oJ+IjW-PPo;BwHx+o_X#sGQ5j>|sSOQiFwi1zCK>m0JPra4?5pAcRmJKjJ-5r3 zj|_apTDa9f##(-4)0WqUMs_u=Urt396q5KF*qvKEKkkpldk(bhi%n#?gIG*AXkP6z z2|@`;s5^*sT!k*ESIMAjGZlSIcLB?OwJ69@q^h$cs&iuKQru`4%JwW_Ji7bjXk`Lk zca*>wX_ViN>tZo^20lRXj=Bvy6v3TzQ>@&kZ57V90EiVX1SN1FP{JhuZT%Px(;M538aF*!S>c6t?h&}$dMc}? zS}Ik|OY?Wl4yyb&BO!qSe}+E)FPsyjm4fCPz8h3!e^0*p_Pg&zMY-b81eVgcO}xRU z_@GEVHQed&KB0JKJKB8jN_!6%d3_X*FPpKU{|x$d;9-H9!M8| z9o?k}BU+Z!i*FD-qX+m7K+=|V)H63B^u;WGIABm)!C{{ILGi7d`>$QH?GrBU^>0ai z7y4K$D8RU)9K8ynew>(N_5d7qp$0$yoC??9S_I1?*7f?=fi*j}LW1hZG%)-JAf7t2 z-6)5kG!ltyt2==xpBbO%_345SCG}vpKH&$UOBs#fQP4yzYdOidm>}F7huA^y$@0GH zE=!=kuzlqX7S&*y6;NHkUcUhmT~D37tStd50gZUN16E!I;+;LpkD~C)I)h1IP1*_V zBQICvsQwIImX7bZ&<`+&Xa-Do5XOqK@sYh8djPeuZ~hLvjW|K{m;Y$zmwn_lCUb=N z(A%+*n3y=K<>_#390q&2Ga-RqEo*76Y-?=0A`dZ>?8muD@U^BBJjN!^m43#?bJ(*{ z{F}T$@trpAzbT?F4c zQQmi$rj8yulTxrYSxI3&yLnK~i7QI{8a(cY_Dh@i`llCnv{nutzuo{f^#; zxDeC~ePa&<=e&OfLP<5`)BK6k<<`UG_!*6;tbj#(eC&9`+@Y@A@W1^2=IKj+{PS-w zKK%asZ~yX}%C}SBZk;MGIss?BqK?GpzNZ(P+e@ferxJva2EpZm(j4mAtWbXmPKHa> z1Ns2Lllj87n(C!+H7Wbc`kQt~9fYztaBzYUx8@=lS1PE>kmZ%G-WnOS`AnZS!m_~UiDq1pK+Rr_S)>GR~^U@_lzD2+aNDXlHLAs8rAgM;MWiC422 zhkPIRU{PFAp?y68g)cB}^fv^lrgN1Jf%F}R+4k_Zs@!doOm|honXO=aFUmSn6DNzo zumAgNMCWfQguklp({j8cn`>9|1!l2(e8B1B{pLP)YSLW)3; zZGh~AKw=0<2q6jCzZdUu%$I(cS|R!W@B6xbzw-}AO&JJFkiG~MCxw;N+9W+BTfi+%>cUr+XPt{bq_HjxM&lH_wxioef3+5*F zZ=q8b#tPp0_27%`e@H5q#>7jtppkF`5I-Z4qCP=pY;k=KuK#o6`K%~JWaL_jua)ba zQ*xM@5nDhQRx~C0e}KZ$J1FNrzqn%DdgU7)cFwn(_d&9>;ogK$vZKUH+}9#VzRywR z>)R{E1m;pLUVljQ%AtL=V9a9%6)ZQbfmahYa7I`uEiDKfMm~wE?rW1bF{{y)H`Alj zU;!|+;81Xe$_56FBaIK^{fk`x+~`eZTN-TP-a$)d$Hf1DVv1-xhr8Lp;@?%|@q}+| zfs<#w1+ZrOqCIUmhZB|=T1ipS@){IC;M*;Qn_Mu#BYsBqJIVg_N#FW^{=uHS^6J&= z3&fQ}q_sYxv6W=Gz$TMne9rG1g)3Ww^lEL$)aRdGrr=HgF(!4)J1>$3O@-oXO`-E9 zos2EAL<|`pv0*ba60M~umIGo4A)_LH7hWG4Hgb7zZ8=P&pZ%;AyL~Q#A`0{nvfUD!V=8@q7& z2*Q82A+%V_a9x1F99A@*Q0`kjfpokI2TSRo*~ns93kZveqx*t-G|Bvy+~*)+ZJa$q ztN(kpXtr7Cr*bL+{iVp@gmrUUqGdeOQN5k!J>d!nOdLcw|G8TVm!&l|&2=i6o#?A` z&8{W}xYy^RaYkO3#5k+}ze;W(1n@H^;#OVZmmph+#$Of--g0#YHc>&n5p0D$3HxgY zv4cevLH?Qf#j!S|^Uu5LW zH{6V7Yw5Z=IJMVyP7FWVgRMKvzg%LvV%C@cSSUgW&xX3-wReu5F{|jDh;{i=O1Eb0KNL+^KV)8L zDkRnqJFCpfjDePThty?U(6T-rytnnP)y^jtldnNhV)XTx2c@baLF8>@w zJRm0y)_$N}dbfS9J>9SW_zc3#t7!q>FJIDdeu~;R`_Ym3vGYd{&y=XvFWM&-ZkQY4 zE4SU!#oEZ(MXu8}>%x?pq;3(c8QS2Njkzx`rzShTUfu^H^+l*znPUbmZ%wNGKRK8P z#D?0_i}^rc`H{Vl*xhp|P2{cRW4VdEqF z9mgQz_ZGtT7$#^`&r)-(LH0LH7$<`e5?-2a?;!|x&@k1Y?UL(ns)1Nvu)#qVCiV1C z=Y^3McEoL+AjRxoNWN>Sghz~;26w(3p(OBTBRYBIc^`Vbo}@$`X>0KtUtePQr7*^O zG4&V``cO@i0UpxQjdCn?$HHtkb)Vze)gG#b!h@Xn&Ow(`E^#O&arZf!VZlwH@oxFG z?rIQ&%>c%o7Ve}{t;sZt4|4NB3~@}OydV7IL=OagVXRB;EuDM#YZj{Qzyp#WPIR^? zFgf}JsXv@G!P9asPm6#razszn)}ZN+ir7B(e7Smtvw7ok7H;DLqH;i`6y`b>8ffG2 z0rtae`9*CPB6JKQ6@@TecJ7Qcmn16jt4cj(y}7s zHB(y8+Y(r1mnl0Z6CiY7TD6*k6iaCB+;Yy(>zCIDnWs|!D>KtRJcKDPp+zp$!GvSt zUS38=3r((3&PxTgV|>_ZRL&{rb9i4Le$Zj0HcKijE5kkH?K|YaKO@LPE{pJ@dgVuU zm2nSM&9M2NmI%NZXo}ZEj;tAq)gDx!>U3CGx);P2Vm>M>80ZwF$5TDY#Q5Jpqd)Z2tbePR@=`12D^*jY2@ozd^3SCJ#)3wrC8xl?dh2F(5 zcwSir=F3W@znMVWI!M}hlecw>M|Z;Yjiz;*Bhs*bGKW(3xn#SC73^erS0zGfutl?d zXn({W)al`Y13z6p=)@3;b$fHuvT5p*3WRpt=4(wIt5(yTgEb`A<7Tri&LdVrZD=sB zORG6k_M@BL-zTU5l6N;?NG?XUNvnY?oLX>zeW&A_EZ)br0Gs3#l$xJep0@gnUDG9% z$2%1W*!HF$1z`f&^D)n{3DZT7sc=EHfCWsid2+xn?r}jYl9~AE&Orzv%%s98gqy_F zzY*L-Lx)4wJPAZG#@B}T0eG}^h_tm!ajfyF@z`ZJZxMJR+xGdE*)HB!KRywaZovF| z7~y#wl1*Fr+uz&1zx?9eLq%`g{(bx3|E2(|*n96ir+1`6&=W(OoSOPEgJv!wUl?9) zb)1inxPzGIDBM+KZ1I)SyzaMiL|!8dRCQUMbAfW<1eZa22^7%?jK=q_CZ6|5X?9A);O=oJbKcX6CCA zeaAWxy&kBUS5-+ijD*2%1Nh+Yf*`TZdM_NQlz&0$0e8Wwo`mDd543|?RT9$F2``2lKb z-jq=}k@0ZVBG^N17GYHJ`=hOc{&{d7XH-t_Ba2*O8mCxc&Tglfr2Etx&osjl|5#g- zR#awoP(-B&vefjkzT7lY`ghE=2IlSCEk$OC z1lb;^z#OX5UX3VIx9{sH_Jho}=3mI|Z$qs4g~TN__}LNP&(DwKTZ+q%m(KGRMmv$c7A!!&#kj(N$`yQ>WV-= z3q)lOxlqNlCcYeTu0RzbI@$JiW~Nei{74u|%~0S_{(qr3y3JgPDxI(GS?z_EYD3&cN}SGrDF)nE(#698b}qX#%3eK?wC?*XlPBv46qK~z+0{pF7;)X zZjE=szQ{qe$1&j(5rlZIbyuUj$b?uGfI2(zZ)fNL?bCttj}ow1h-Q(Y|J>%&r-72h z3$=}80g>1SgZvH8oW`OJ`kQrEDzD_Vz^HJ$yZMa3;KZs7KRDac)SExd$TYTW%{E}G zj2+c$w-SG6K8kYwx-=)`6uYjQx`HCD_rzLnRvmPn90oGO*B)Vdb4)>-;cFHXZ*~k+ zXt$SmENN>rhcPQfR(U)*OS3cF&22YW%hzlYcXb>0FKro#=93ZLaQKTl7VQlI_`}M| z^Tyi-oZF)zxSeH2_P0xl!YuUnjSyZ;%R;VWUqeD5kp@#?L`{2V{l8&O$_;*v##1DC zlK;)E{#A3cuHe7E)R^JFd)sL0k}N-5s{$MM>+5V5pu+;UYK6Jkk*&(pk#L!+FhVA+ z;X838bpP90M50n@)%xHfFq2@-o&uiMp$)EYlpB=u{PV}tAjI51f3fdlMU*85_#=(T zGU-VuVvfb2HCd^K_Qba1GYJE6|NiD*dt&EPPJG*(W51bu5v`$LIlb5PUPfL+?0jM3 z+jQRo@LDID>!3#YeTpgdAa-LJ)%qmK{(1&}vZk@Tp*4Me?!!Nl2b^$xDb_>pf|vL? z$^tE0>SiusaPtGUcu{%GGBawR8Bkf+((^U%^!HQi zxycNWruInOuTn(Ypt-hn%NI2ZN#fG6mcojKgRqs{hAbM$)vC#2*eF233R0d?uySTl-Gn1?5vFsqI4|0meKAG$$kzi>IX!S$P&>&-fYWihCf zU}%P!J5=@2ks zLJ-MmLDJRbWZR#~?q5@y190-SSF1%Cm3G=)$hqc zTr7k`W8tD^aqAC+9d9(?Yg!$H6Pd>E(vV7lZr+OsK5rYEJ}fG%)8!T>Y38L4=s`hW zs;3+ZQ*~xo$=d*(9o8Y5UABN}cKo`idil0y=Qe;;Yy1=NZIF-ZWIQ!yxBK|kOFgw3x4*aHa-9GLc86nEQJ0jLHetsQt! zwnLa|I0vgl$wRO5RdxWz7JT4QH$#Uef958osk$v%GroTKFSo$Hz`WZs- zC7}shzV!y1rkdeU^2X+l$5?^q86MgoaimCsuWKX;Q_k285^JK}uV&DyCS6QLj?!)_ zq`!v1Yvu8ikCgimt^s?i*fW+4F;_{MxKp|1gx$ICYAW0l6<13`PIR#g$ZFdS4AJBu z@lSQi=Bl>;H&o5RTjXe6-?5o@XH_c!AiQCJ*ReFGPHh-j+XndcPeaTVtUl zH9(zIyZW-Up69u$t;Z2b1$ENlpC5&`K{MDe-=?BWI26qos%4k8ClaaQm|a+zke{E= zNvtfP<=VDTu743|J`m0Eg3K#}RtqMrfLuN+hJFSGY(?{YobA;H&TQpW*&e10k4c6P zBeVY))W5+G!t7ElZ=aZM=yd;S@CkMSE`HBNkXxqt@7220{JvuP-O%8&m9t~uYX|QZ z99f~um(l~v?NlT&K@n4KX(#UhB2mBH-ji$XlEqu9d}odc2ZU$qi_ZmLe?0<=nSGYZ zm0`)wL|Z^Z)-p=$PkFv+o(n?6yZ<8u?f9x`%hhVf0-#f?meLCuFApMS{~YRsUJG@S zgJUsWzW4i&e{@BD_D>J~bGi7_Iq@HszD&(0A33tobU?wu`eCX;_iP*eYk_@06VHXT zTx~-runE{RH=gPQJ>ftwe~N<3DHFaDb(N9YET5Sa7>(kqD_3@43^S(y#n+L{KZg|^Kad_~fINUB16$?*m3(Lao^yXoC$uCOYKF5myR zm*=|&jQV4>XCjr~Hqh^-ATf)Q`9|z|9LapHvn$FnoR4_%e={Q<9{810kPmknLc~&c zbD$3qIM3QbLtNJJlxIJ`HlZzURK4V99tT#C*84jrlP zJ>a6C#)TG7ggn1+Tj)~vfkCOe)BGM3Vtb+yog>vP=L82vF8_Y~uPFDzZf&l7y?U;> z*7Cs6aps-{OLa|cz2fX~4KD~|E+%IDP`q%L=Xo8*o=;z`N;tdrBwC)2v=b3ok&~p; z`}=N|`qk-{B(Z3~Tm4nYByg$_e?M#+uaf(mZ=TtwB)I^{@teB{wO8pM1`)X(LsPD#Q;t0^__y zXuej1{mY2)l0a;EXhOL7kXI?7Mp=;RQVQr*?0A13-lde-Xs~>trF4Ap;$~p^diZ2> zgc&BAjFlZ5nlZx{m^7K(tpVA@muP6K8TvF|Cv#C@3WH1T_mgmzuLZ34N1!Vnr8M#c z#=$+gR>r*yR^p%UE03q;THlUDj4F|=z&og7^I?%K6|XCHMH25Loab|$p{`VuYDq5F z>#xtSz4kVUo74n;bp>CwkUZa`98ba+D^R4gOOb<#jiT9F(L&>p2J%8wdQQt!3LfUyC0AcQJU=^BA_sl(j; zZ|4fn4RgGo_C*t5yu#)bj9bMz%}oUd@g>s5XQ6L&cw>pP2}3U$ha9a+OQlDP4?Gi; zLR0cgenBLju`!s5po2Ah@1m`qCQnJ zpzycrek>^}w!VXc6OX%AV%=}TaG3)%S0Lc|*>!!w21KMxd#BE8N^NkmdaMP$)2zDq5DeH;)ZJ0Fj$dDJ)cTkye90EpKMgP79%peOSXFlhgR`fBK&nl1i_rj|_Hk1c3$pd5YBbk@>|`;PZVmw+e6|x`qswd9Q*b z%re8)i!kVw3n#M&a;+08)+YD1lq` z8q5xjpAn-l{f>V^NgaqD3kl)`s#9FII9@1=N7u~#JvBG)?n8IC8w%#X!i3>~jzhCr zgcEbi;Dz}9%kT($O0w@w-sZo;7|jaCaPz)3y{r)T`Y2{*q_rv)w;oy_#ZJLGpGS!N zP&iSzMe5sw|5#9FHZZgIbV!%O0xLWU%0igdsyw&0mdGr6y!Fei<*>x5V?wWQt4QGL z#h|c8d9C>p78dH7;Q@a=U^_?l@R6{sRZJRcLTbE*nmmGrUUs~He|Y_!8rsa%FZfdF z35(dVf%lDehFk6y3IZ3ri~Aqk!48$V+@%Ckb^_JVX$Y{*?Cv6*irCM|e1_@MX1ACC{sR(FvX z3y|Dw>df2b3h(N}`cNume{_F%x31IVy@?*lm$8}S(7n2sj=?Zrp2VpCl--Nt@{a&u zjzAI~iz+b^MbMIU_C|9s2O0>BXUkgu1nAc14wAM86%|s`E{JRzHMsmZi@JWzT$k<* z>F6|Dp3ghq@6TX%X3!0MnhD;jB*5=l6Ot0{Pw2_M5%O+DC_m=AQz0qv_Arh>#mvO< zI-OM59a~oKY(%5C`IF6-*G(s zh>$$gJJc&760xVd6n(LUv}Yoe^9z)_HyU^H5UuJ;@4vvs^KqtRr9-SLpsM<{$4^47 z1*<+VA`NhVZ{w~#B>6jlBj6}`%P-z3+TyUZ>y ze$Rl6VnO7%oXhpOs9615q%|Zov;!6ttO(mwwqN0zP@sJsRVZ;8yT#AX_cP8Xdy0~^ zRXeH4kWM&-UVoshx}H&CjsSEIFSx}+`Zgn|6bhJr8KA)PNwxT8&=0#CCBY61nFW01 z`e(8grSBuG;YP(^vOUNeL{FKCgs>l4R=;!uc-BBr?D$hh2Mc2XGcyn z0qYz_GAfK;_g$IlO+IV3veWw^?&P%yD(uxrNTUx31jUnPYN)iL=q|hRc@CNWYaO&N z*%CLj)aWpY19*}?M7!-`xzRcGfZw;gzJ!bFlwJ0CCA8_sLV#q(9P3x$>Fn08>P`&L3ogvjcy0+|LX#65nBqs>5 z0uxl-EIT%xOH-l<0B;9+IRg!JQ4N0W6IWns61V(( z{(OwanN5OE;8t-6&0WIVTyWc(;c8m{-Uq|_`IQNcC+2!THtQp!=L6d23B*t`+<3N? z;7T8q1X}QdvZ6^C^zTL4zC(A)iB;V->f;ngH!AoAA=uc~b43__M-tFYCUoN>vlDj4 zNLb-9t-#QpVxUf%n|?vrw$MZ4he_VM4TIp(0CNsX+#|NG)QVzxn0F;2AA>yY}6lb2v*>)a|nw(wFZh_UkYa zDEHlzWOxAIi&;^L{(mm+u^Pvfa^tHnn#+M10(%@20)`A>@YiyoA_LC}{tbw@DJ=tr z6%C`dvlIH0-9kv#%uA)nzz^!w5JJVuzEjdCvkRgkN@@T1rWVMwX47WsV8CmI+1^aY zdJJDx6wuHD0aOs^wal$3s=Bz?wep8|%IuCdqVW(B$ZN^}D-R`~EL_6ToHp-TQx}ON zzE>mijT|JMYne~&$w2AD4nxCyT=f_Rx4U@j;3d#u@_s0 zKs#Ld;wI{!4o<4r-~(uyx|8KvrBN@MM}>JpU%r%63dx^Mbt5vO*)jd$S*2CXmD5zSu`+F}Wv)t) zw6vr-ojrYPu9_1FGubS|j}_U2cL{W#yRauhkx5jTPZp4@d4!YS?WsK}*D{$M*0g*g z)pI|E8RRqnhhy9%vPBdC`=S=_x8uQB+x!V2-|4TVNNn2705+-841*lPds)Bb?fk;? zxd1H+ib})Q*9_$`&J{(9WVKdXDvRQ40^6C~Dw%U+T3&Ta$^7Y>5M);-B3N}mP%_;?(fiQ zisHjNj)VxH=n55MYpp%uT+b~f!0!1AuhTeZni3d34sEjQp%I)?VCf~KgXScLWv8+# zZQ@!qe(U?O+Hiw1G4eB>cZ5d|_(ai@o39%f@Ad*(?9ol3HAb&-JK7-7!`)tOOX~!f zgSpVuIpfz@M0`!9VKVS?k{3480|!{q0AcwMNq=Eip4j~rR0e{B;eCLLDtB#Ns)Q6R z2qpF@m1xz@D;oTMPlrJW>R{o3aU=sWmjJ{-4k79}JG0RRw(XDHgWX6gP0d>6~6@K=Fp}>-f>WT#( zc5rn5V^;~!x1>-%E{8*n{7NOf3fJo~mMm1*wA9R358elNZ2|ABw-L{J(I}E(NK|_@ ztq-0OGdVDcpx;)fdfD{VYHHY|ma5Ip>}i(zrESUX`S$Lfp0~64rz3^HO=4C46<8m7tf4Lm; z6QH0o)~EotxRu8~+<3+?_3(Gz%K(k3a=nVuI2Z&3{@_~F3*BrKKI?m40+Z zYY6_lbCosVb#W>t7w#BhA6bP4mHbrj2CbfGc#9Ov4@EV*B8D6%UeJlU=P_Z7)#I8{ z-3dPItIantGDMkvr%slA+%rAA{A#Cfcge!!sobbfSzE&L97hh23x9Ma^$AaugJfAG`*Vh6akShUdy`{DK z#9wPGsm7J7HF)1SSVj8xqVcP|z-x8@khfhGr!ND=oLN?nUHLvNxl+Fxr^s^s7MiRq zrXNQ1E6ht1+L_(m2}}TGe~|fzJr6k_okmbP1D?ycjjjP*7flv_@kyVk ze0>DuTFFd>AbNOSGcW%zv7g7Jx?YL5TW?0+@#dm7Kx?{wN7{(|oTMvfOLh+D5AD=O zu{Nay$wo)EcR_)*pCvQK#-MI*5=mp=NdT1N%3p?7Bl0;zw(XM1hC5@>2^hxj)b2qA z9;dev5Weu~ zWA*yPM_Sj__}@h^Sf1x{eZKgp^;6r_^6Lj0$40YPk!^oV_Mov-_Ba^Nv0UEt!VWq9+mt(BtQ{FD@p5;=*>S&-ZX zZuIp%1iVM7O=Kyv;m=vpJpG`NWvF>{Yjo%Po9ZN zBQQPtb|0U(05ZVro+6VAnngwP`e{u-2Np@SD0P?FS!poyk@(7duvYC0EWe>kXoWR+ zboRsIa#IvlyaNntc)eW7?a~H>aL3At)<@Sz(MAfWYMQI{Xkt>9rqU$H>k{Ze!+PUF zvWn7?V)^gSTQ!YQ6}<#!UBvnS}$0_kGD8)fPJZN@p>SRXo{QEmdj2c zis{~kc>&FmH+K1SZp4x$kkl|H-gBA0q~op@`+M841t3Wcr`bY42N75vChFhmsq=it zb5FB;wJhIXc!6(~+~Ub}Zjti+G?e$Zyg;L5dxi?U=1!eGU4aOiw6tkZ8IF`Kl)Rg1 zS1>0OwA&es+j(h6?i9G*1~cPdQAl}b;BF}G`R zPL(>G6Iji~ZZES1Ma#Z~8}E*E_g(h?6paT49Lx7~c?x;}J5Xig5=+(BfA6SQT9QB) zKpQe(cBcw2Y|Rs~FWS@F3ad7asa|iU-}e^Da2p9tGu|mFjKaFvmG@);KM7xWpoYp~ zQyI_p1p@uB)MzF74{>*>!OiCq#@17dW^_hoVT>#`n-{OzP1UK=r^DxPg8sE`CQY~6 zEwm0$gU>XAJt@(ICdl)PB|#;u^<0Y)aJaV6UWqAFEQQ(VoUb!H&0rG|vZMn!cNiX3k%wDe|M zssy6XKm+xe@H;*Ft%Z!L&w7Wf9B4q8)sUVdvqqxAK3>-X%+?&xI4Wr$M~ z?wa$HxAh^CoiWk&VhM^{nC6s+TTATdOaEkS^G;ou%{Zq?a2`vXoNs35P zzGm}r9yb8N%%KY9UKI7pm@S$`FPz#7U-H0w8)*R>C-h!&E#;x*Rx8**Jr#UWQWnnt z9)IMz^VcxAu+Op^49ylns6((q#<2a_;RR`wWs;9B=wWkwbcBk$}w^g6#KmC^P=ZnTOvQ zD?sJrVG^p2Dy2~Ne{%+Eg7?XO#|+meB={rv*B{JnJywp_=^Dr89#`fsB#t~y!0OsN zEMNB}Bu6z)w#+=A(g4g+i&HyG3`qp@UnP|Z%E30()=FasgMBz^LT6*qmcK`H-@CvZ z$eM^ZwAD9{Urv5h;aQaSxYnKuzUsO)`ZQ2QM1vWrNSTfz4h?_=E*AUxVdKO98oqLW z`tjW;YD8ElnKTUET6giFb(FA(D_&WRftT!9D)x%O^T!vWn+=C9+L>NY1iBDp+xJa{ zu_v;I9uEPKl(=yLDY#44R@hHwDkE6#6cz{oaJG5)cu`%PmgV=0S9c(f|BYcsUXplg ziHhqZefGnqF^>YDfy*0d2$HG{CMYodA|B?@4t2#W z5h#4?p98BB2ZN7|)$Vg5=DiI1z2#T0U%lG&M08nm?f8I5QXxgtW`1u$8!&*{;U*QU zVeU%Xy760G39y3N@Egkt$ZD4VOo{t@etw}u9wDkK7VGmQ9$Pv_0ain^8PO!-Ihi{- zJd(wNUs)q3mzui)?UtKAn}(4dBpUBX+?{O_PhPUG3$xqiew&a8NTY@&&-A@Kww9)C zy$OaSLr*2YqC-JzUMuncne6yCl>ZC%P(|R?C_&_PXjaNC^bvKmd;`++{ABlg{F4Rw z>46E7D)5hJHKe0lKeWV)mkwheV7iUN+h~?e$=kfb8z{Z#g=aa|{zF~$oTUzp)Z!L| z4ohk}sLOV|+F+fQHlB*5=vG(ONRIB>kMOZ<1l$N3&mimz8J;zg+qg$)2o0UL?u3%N zl31HDVT((erUNAiOsU!o86a2_LC8Z5wU|cxM5(KZrqTxd%dM%f=RxT&SHVB2GsTX^ zAXt6;CW*y_ef>R~hy(_NRSoe-4H^=-OBj|BVm+HOD5O~fu~v$Uk*i3lfp1L}Fw?#~ z2S>bCqWvwVDV)_&?MNu^anm(adlk?iCYD4+6%z;33hxy5e~!Lbyqkw9^>$`PtW9X0 z6Pb;cL4SV^rYCQ5Lq&618WCkPf>PHfYptZ^)dnF2otdOxX^K$#whN{{5 z2im0aM_Q2jadr={8?E4L!uk8r^cHkr4CFx3pkiy3Q3JqeEyQ^&!bzxMiBN$LP>joD zDvV~zrR5U8o#pvAvU@-58_S)ltEGlmB5NWJPMfGhr%_?TGwp|%<_4C(zYV*#(B7#9 zaJNRIxjV5AsZX1%-rAH%$p$DmT{$MXut0IxKXro zken6OlyR8lSlvV3Oyl*d3viyNY)vwA=wl*P7YDb7?W=afmDx|G6oJ9r!1(Lw<8h7e z_0)b~@>FWG2YUJP{Xd`V{K3Q%TE?$;&M0dEKJ%e;z34O|8So{NBC9*TzmE8YC7;Y0 z((lAY*K140z+Pobk*GjRVT|Af%_MIfY+q1;61F*y3LjbA%J=NVgtL?J{d^d_Xf>8w z*}eN@X)*CHZgRGMf`9)xh8jHf zq($9Bg)P%?CeTjD9kov?U(PMBw2F!ojx4|4z}I%yX# z<+42FKzWnq7YOL1Y~N;f;aSDh;u{b)HSJmL9WY;Zojh==q27Y!5A_oL(@?*iLzHdp zwE7*4hHITVg>OBb862sLS{nRSz~|4;%05I>8oN(_M0*+2M&O=#H<#cUqmgM#mU47V zmtbvGPQf?EARXtU;|UM{=#2ekZ!0&$;eLX0ta}YBtF}gttrvp; zoC{t95ZUW%$U;V{D;3Q$&;s6TpuN#c%#Xw@Dam8ie8~FNdVYM zC%$XLJKg~F^h_v)n%me2{so>-Cb2`LcRL3(AjaHDGo)r{NasQ(RCN(9nZ%z5yRN#h zq*8xfTR%pg>`1$dsp1M5S0#zp%u(QxPVh}xAsXt_26 zECa{u2%hkqcTh1{WB{b5|815>#sWilY7I>Xqth=9x8@tgqaAe(^KEG`aX+8yUCME( z8BPuy?3%KZpY(6ht%Cu^dfN`D8Qz8)bl9o}uA=0H8osZSl+9)m)gZw?Bv;h|1z#k!7YxlPPH zZN@M2f>p^}D{&#>A;sbo08c~z(8Lp%x@gv2=PGmMCRt+VG_YDTjHw$@p*ay@PM9fD zji9Tz-cVu7*wNv4c8ynxW>-M13odgomWm#JMdKcWMZKsJul*QS34*4@Jq9^RCETUR zkU1s5Q)F#db2bOsw4UB*;!rPZqFkZfdKJQj>h0+;^47=XViUMtA47h~Tfpy9jJZcv zo`x}B-pfndFh}`Dt$y1CT3cb+s_|wt zs!5dlVlJD&*aE%l`>Pd!jjLzj$>Ql%aw()U8>-D;!y)M8%j`I*E3&P8s7PtQ0SJsT z(^`dnf}O<|NodbH-|UK`Sd|Pvx~eT27a(6dm(XAO*)NL^XL1n?g*mu+itHOyG$l~< zBXC)DI#VYTf;$okCUj-cM0lhyQU2Raqgb(c=owdA1|HK4!=k+Xvfu`r3)_65ZbMKg zz{Bk%oU}E}j;gcgVwc4WnOxe^(-|bol`692>3h#oO(E)%2{|QuV_Z(kA{QR{G?S<3 zD8B!P`?tUNW$?k=`-}hh>qnW{iIsFii*~CstaNE@cuEt zN!onw*x3#YEAVWF%-T9HhDmLeBL~wcn{0yPkWYG20g3j^LI|A%q$N0rNtJwzeLa|$ zZ1HO^<_R^WlYvKOL};b&8^#=(38mKcYBa5&a3hO9Kq;qqym*yF#~XqA==gB0?w(E& zMN+AnnYV}c1DDY9PS_w*a8_NUyXP&G42W8L=TIY6r{nA28Y`&wi{^S)e2kPsLbdnW z++QF;uxIi#F-d8mO)tB1BkNKUO&1$w6r%MuaqE-G1X`i%ap8Q9BLC;rD(t0uV|&hy z^8pB|eJz(rOw$mAsZmvIjoZvHrASraR_-Y1kPC5xMg49KZbjeRNO=%>*zue}J>p&H z9uyYh>C^ixz8VR08fQSQJ_<}|0h0$|PFZZL8z7e+FlwL3=O zgz`EEQMI*T6Iv64GFG60ss@VGO_@9r-;)(}sRb~XqUU?tyzcW!?jQE)=+@c2{*k_( zLTNZVmP7x0vy)+%X^2Mi;mWwNa&>Mmx)9N^Hru|KDK-Lnph9)vb%p_DzIbnaoQn5H znmR?bi_NDgXe6dzBwBLRDokbPx{2K&m96g+p3<#bO3H>l>#05`%0|n_Zn^l^?4a0?JZzs&%F{ItRXoWmw0 zxRuI_j-NbrdkkcJ!U5veA*A4c$7&N@6Yl@M&ZA1~P`wlzd30^(|VXB4J|kvIeHm+x+a;RYZSZVV1#N zr#m{;EGn>Gt0~ZKe``LY1^KdOOTl6(Pln-3WoCZf0Wi=ZaGiEo0%aohPUYp~8SXS9 zx4uBxzkocXq8dO>?GV05E>%OYO0b~X@j`6d{Gj8_=0fqBK3eO!m98McWK1C$jSxQz z3%)`&Uf0Z z0)Kt9UpkZ4G-R{b2}%sLILE&F#0X{AS%9K7ufR>NZ2MkJ=(UmiZmJ$&CkNAmk`1_GmSsvc3R zYwaIN5MYeV9&r zt(hhw!E|e{?`YN|Xl0;Ix*250C$>g!L%44nJ^E~Ia?Z%auomh`6adLNLyMY%hteHa41w|vUCf5X{J!vD@3Pt z9-f*IN?Yb$F5N1)fA&^aT|pv{pQE`Yu59UJl?EFhngrF;#=>NV{UjP*&qLpsFZSHn z=%<8+o>D);u7A(pZuD23#<@Qsl)|yRJ=Nf1N8xP{+rCb-CVO8+kdM4c^M9`{d#>Fwit_$hq1s;1LBersd^Cc( zK5_~vQls-%4L5A`*2KgBf@ z+bQoar9N_6^9=>;7wLE`1F#5`1rr!Hj-|g%RkbqSFs5Q&P9?%y-eFN z1%BgsyPb*xU|$QV;wR`meR8 zPnj|1DPDzdsnL|9xpV^KpIV=8LDHnXTtd*h$6&o=;0#Hq^8~loUU@ugh+Bzu9gC(q@O%g>RBDVDQifPLJB1{ z-pP(Nlab2UPS|X5-aG`!4laG-AVKu}aPs=s31@k2;Pp%yD7^L7($XJEfjVU675nPC zZwg06nl-e9eh0KIzMac=eO;+Yw=|KdV0QwktlKV3;d$11u9qR&&AZ{W?mwH=wBv!{O1uGMn*o>NmVQpvV}>;}$E2yVoFQcDT?InaP`1HrgA& zu7KVC5G77l0X_?OsKK8;7FY&n$lHJBxzfmi`fz84Wb?4Z(HpHDOwJ1UwaLpG%w*!M zt4HfqYqNO`DBB0T$E9>TIjARFL|$YvmT2w4^OQ#>-7CGUNo@I0R-Zk?PU)CYG!IZpDS~DR`G) zy9M;%s(ZIH6r9AxYZP^zBB<6Tf0AB%Hmsp$=I<~r>g$F@f{|K~Ul82ou1tB}t!}9< z1ut0%Xw>qOGxutiG&?4|+w`A-fR%IYyHB%GZD+@xKCL><&L+*LjHMbTtsFenpr+z2 z#RS;MMTCZig;89oJoE7R5_xXVxyXl?vk*rjG{ONz$@%RCxKZ>gZ_Omk{lD?eSh1j6 zP;3#`Mt3C^_a--u^bb7_F7cfG7sN|i{9sXW@H*Vi)-O52S-u-I?vr4&geof`Fi2-j zWctRvCr9{rCiRk3itxgO)5I#;*wIH(;Y(9x`PM5(Xe!ilDbY8w{6!=DU(z^|JSxrp z{Qd3ln$MR8?ej`#5 zE}PuB@B4RM-|y!$dwG~VwV7h|{~7QBbPPo`VkaSco8+g=C`LNY?^FTpVWCFb`~`$U zPF*}aVB35=e^fg4?*~KE()$}e)@1A%Pr`Z5(P?UE%AkT4pojTk)lu}93J8!P5pZ0Y?Ef?s-QUpU+SXX55QYYugai{ zLc;#<8|PrzpDdofJo3CSl}>)rckc7p38ei|s3_nuMQfV7>ofH`vc=^Pob&rAeCGJW z`PtNTX_jYaxqNA6D&}UTQ)@TqKd|nb=XS=`RC$@8KRl#5jwrQ5-oQH`#TZ|z$8y)? zimd+A`3K&tJNHx9qyB!3H8!PH$+dkLiy&lR%xjQITPdjTRVCE;m|iG;SkzbQSAzNj z^dLhdSmBy@&D5T75MMz@x`$(J<2@{&2o>45snW&H)SRuwtqECK{>sc+`o8gq*rD}6 zpi0|!u`<{7QrVZyGpW&}T&vJX4L;<}6LFpkME+V+kJ(U<35I9^b@Re{1ds=;^n zsH!8aJkf@d8u|l&$Ln}{uDRlHD4a<1n!X*A zIEOx+Ux=@O*_75(pw$Bzk?ZKfCzhHAo1+KYji1NHTy7{0lf{;E7?kyT!AAA~pFG)} z=={5A8^iVw(W`4?MeZRO14**XSaWe*K?Iz2>MvD_ZGjgWiofq%*$CO?o|iap`mgO} zwN-mlPbFb1CT+P1uDU5Fz6yiMay!7#ufZT<*xnW>r;_P&<>10%~4 z6VR5Vu-7{wa^#6B3}R%0?y~qwZNE1R5`;&V4dbG<+KLnotYL;K%3K}E9VgK6J5cis zFFGiwo@O-q!rYy{%;6%Kv<17NtKw2h#%#bxTNF}I1d1L0O^t0hifyq&(;xDx_&3)~ zeil~$l}QN}qd=bdW<@|~um6~qOUjym#iA4_>oM!Vr})P2pKZ_Dx(8iX+YjvRFHFXD zctv}Se6}aBjtU0G2)=1nuj+KYg%x1^-}rb&Ya{Y>AnjQN%%9~ z!qcG&4Ozn=>AbJRN`~XTx!GFMRL28&Tf7`0F?RS3teD^6#rvb)*WDh3U~!3EvRs|9 zyS*;C`BZuf_2wg5uJ!RqJyqwR(u#xXj$JE_{{bb^klElj zJ-^RZf^wY?g?E8nt_ZAWI#%S%!$FV3?4e9y2XA^_XK$XDYo~2;{~#*`)&H?^>Dk`& z@D@|wq-PA115Qn8=6Lcr^vHvjETNS&OlHxU=kq3ZS}&8aeU!HOrKv@uvgzpf2i znRy>z?U4DgEr{FOd)p>gG#~uAB3$a%_=}He%fxq<`3@QHXWRC*y&*|;&9Y1&J5yZ@ z!SJeSGPbF?>Q>$g>(i|4lUtU4p8WGy@nvIYL+aG9ORMvAOR38SnefS)JCef?nbcQO zqwq{_p~*r4c*E34d$L)}OM_O%h^Dr}U<+r*7uzq2zC5<=A1i$)Xd%kX9|6UypORI* z4(PObf9~*r^HCeck3I3u&P{DtJ2#kng< zo4Xg?BGwm(8Td?#)sxGyM7yZxXp=u0r*Hn=ZUBBp9d64QVRAnJ{d?f6;Nww!Wv2s3 z7&JKjlzA3f3Pt7vu++_s!V^lDfl)DHYeEA_D73~)%G!E^V2i5RsVfv2y6SjdlchA0MpE#iiLJlY(gl^e__7BEspsmJ&wV+V*e%lrL9KEWe z?r{AO$@W4JhS<8D$GXce;KAFf1a z^7E|4yc?YL!ESN#^hLJ&7waIYY-OxgfZQ2HMvi-s@PtJ|EjN zSOZ!sY>d$rDG|V-s4-sYPdq2Vwh=em1mG@4qY_Tkx3lGbzG zE!iltKD>ag$oIlTDip$6QLuN5B`5{k10qUQYcHKV+v2pqWMqiqB3XZFc`^hCFIglb zr@XHHqHf!tET2CAq7Ol#5l_B9sHl5z!j z_;HC5t%Fvs<8oC)8m$3Lcn4rXK&LF=5^uc@c;E&&Q-5Fl)L$PaWAMmMSv@VNz*k;_ z6Oy(hI7slm1Di?w-GW&uh+mnTPteV#b`^B~WbkB6JR2Fg8x65#%=T*^7vvk<_7x@+ z@8LO$pon+JX3*@1!em--Jcm9njw|)=UXd}->Q^}gtn-O%?m-LEVu1$yr2xMgAFam}O+jq+c2ge=Ps5`^#`iD=``n z$Qgr#&?tPNmZBTWONp%CA4hP{X02^7Tnw&rW`@tBOiB9W&*Pve-r1@X|1y<_r2CguWF5&huVNMv(<@h$jUl0i4|+SBYut6r(-YjY#HuHU=;X~wwsW1m z`3u`qLfolhZ@tTQ{=jyhWp6#L18n;f-z>q{M<+isAw4CrJ##B;%C-}p#GPlQ&E(lS z!7PRUy8^2#xYN}!@Kb_HIc)FUrD|!xlNaRPUIe<_aP25MHgT&HzSt>Or6LLE+r6jB z<+C6I?rKT}Y`&G_F}?l9ROZ%A7*lGewT6HN*$c@I79OcmCsz2BHh*oq9awbHg#T-p ztG0jTYBVarScn7II&HSO(irlp?sx!Sy^iv)?;YYewH^V&_bGd3GA49IGmsG->Z*3W zWnyWI-?S^Wz!KjXU=e)sC>sO^3NZ&|$Ox^|MNt(PoT@#5cCtX9eZ+Q;MRz{0dQ*|z z@`d5%+GjuCf970gN81d+CYDlQOCr;T1MG`p#2Q;PO`4d$4yQBeSgoeVS}o9=1zs;M;ivV{8)-L)7$MH zs99Y+oyA)<9gCuykv#~SN>DfV{s~>+H&f)DK~^u6*t~UN2>Y+Fgpm&yfAD+x`vti5 zCV9VF9kF76bS%a82`66%l5P{CV&PdTg5>S1i{Rx9(oBr;B+x_4VU8AlHj}Qhuf~{Q z&mlQH*I$^(a-8gK*2n`De7kdYFH$6Q^$sy-BzonwGxG%E;JqgN}^Yd2Un_;PWT zMC;3@bVSYKOvNGNJ;+^9UPYBii3Xv{8ga_Rp-Ely8d1>{6}rilKb^`-B-O0!^lfhg z9I2b6qnxWOBX8Wz?yl9;)_Z5Ufw2_VJBSj^+T5*DQlzGA{8}HvZ`4_#maelciLfqs zA&j@sRpBgx=M98^F|_J*W2NB`Yl^@pW}L{GVp1iMq_%iSkRMXO?m+l>T=+Oys7y{= z?jTs+h>j%i!NEgt7zrB_j2ySv{HGvJce^>LXtIwVj z`oqwOAg;KIXI*h=iwpDejx&CXYBq=w{vR?g2@(=1R_gCYo>(v|%)mG4=|iobcy3RY zn8kU>szoLx6pXkZq;j;~rrN&2;^Ln)CF13^9&%rux@dhNRXUV{)G5)GCt4E({%^xX zmPU%}RF0xsJPV!fX~Vl>KzkyK&E+}KOpGWo0rquX-t7A3Y?g5r0XygNmT*r^-b(buivOc@ z!dZ>)+d8Zt6308)gsohBl`p@~{jd%L;NQUO{bzztKs$&5KTh1iSB|8!l_Rh=X_$ol z*p$|}#Pb%e%v{TZ46YBpxabWlf#N?AMnb85Kc+W%DbPE!Uu6fII$A1F44!w0=Zubo zv?!w3MRnck-LP7Thn88g2y{j-kclb`?S)z%IWB!dF7Q@dM)wf!BHQ zf_QX9V3Ze%{`*;HpM$pDjQ>TWekX;&?c6QSpA(Jq1m{`~K~P&8f?2*=huxBZXDcaX zd`%xq2p@~8*cS^aF)^~tK3&B%O2Qj~8Ugo-Q|~RrHC9Rs@DNcgY`bkuyx&`{KS07= zsNff7wZtao7dzEd4wJShXZ3>=xT5qhJKKLn=f9N(A#p%CWycaIJ}p`X$D>0KsF(zI zhoM^}%T^8hX0ts1WA)y%(jFI(dQs6?##>p7JI+23jqHWsFk$%foT_?GjlIqHKS-W) zqBaO@NzaSXM>5Z|N;rsHhfQz6B;t6pH>}C)%15x#EJRESg@u?Xpp^u!(Qzct9}VL| zXKf=g4}RiOR6tgc-upF!xN*Pi%H+oroS~IICXlfM;@)`oeW~zChrltf5j2wl=fT7V zNcdsv-AIhDsq#$Z!2hSYed*4bPyh4dFYkYy%=@={|NXw@FVt%-DHe6L_!)=FizPp& z`?b`%khc~5C)tGMQIz|2))WL1iWd&wt2u0n?wLyu*Nl$ps|H;VxmwU+jg+Yj`>#DW zcqEPM8!*8WgO`F@iVR~$V48LG@T>~v+@C*9hB{!U(RJk_%;$(P*he}P-= z%tLrL>k12jhex^G*Z0tWU1j~tJ$Mw2BO(XDM)voRq@?COijcZW@ty3@8e2MId$R>h zZc(r9n`_fQzaZhM9!7dgI9f5euf^U%pG+w{G(`7)NM9&x2?V;D16xR$(DI)8zqg;G zip3ju17X3X)!o|7cdEt9J#;i>?JH78`<(tpo_B)nG2gd6RE50J6ZBQ2n0EAuJ!;7sWwgm){^3hV3C34a-bouj0nWZ==zV% z_C*W~85NoikbbRS28i}fAUKP) z&qQG``y$Xa%OFZsv;f4^Z0Vv?o^MKrE(se z!&PGqj6iRcW)OQC>sU0`XUAUWQ4sV-%|Ck4Z?$B$j?@hC1zNJbf#~~$T|Kqes!X@a zIDC;-(@LUkCebNi=W>7BOx|t-9f5ExobQwk=17X;iamD$FPcSNg_MRLB+B51det|Z zri95VsBW*QQf&w>TRHfGh=`tO3%qsPwP&80BN-xICRby;WH^QhT&8)4(WvV#gs7rI zCFs_F<=BzXr#2%D+AbgwytqXJ!Z*y}KUe{~fAaJ{jwC zTKyIPGAMa)2r_eYfmh&e3T5T;L?KmTZV3J5&wl}0+klYQEHO=-*Q+i%q^C`n^ zLxPoOa+ar0dFS;diR$!}hT=<^Ju_qnsnSpjd=2LtY#lnmC)e>de?ZX;@_=P^E~-^z z8x2Q!_tWp!8=e1A`TmBL-v2RFDf+P*oWq81>2ws3bbXan(Y?voYH72;{9DR+ve;tL zb{%~cM0k=pMV5q%0(ndx*#-rqW1xZqdg&8^Uu@5rx~$!uloSTV!g)ims&W9d<$)Hy z)TH$+rL#qWyQ%c5r-s1dZ|xtuz|N7ML&=_&j;O4EUmx>}h1(F0n*_u=mu$Tv75c_q zgP8R``dfQKnVy?0M30kMtm=uvGvX;zBePpCS6_^_`Nwn+ruTMQcNSFPc4%xL&^&;N z9S7*__{z=N!Z+uV>}Om&kt>JoMEZ$ z{<3yI(!>LkP0sPn-?81Bd5E%44zWr$zlAll24BSuZA>9fwN;riX%xi`*B+L6tXc$* zK6+pq8R+~NTqIze@p@^>-lSx+Y+)L>2jD9qiS+^fUJIYRb*|qYGIhF_v?8rjS;?QI znCABUlr!sFt;-1BSCF4h(i3dIhGYisOJS|f6Im_xa4O07#0iZ6AW+dj)YG;H0G(PE z_`n1H5aBpaq~h_X>qQgUp8Xs!&ft7voZEo&CIc1YUMOWGt@2g~RLS8qoP~Kao{~!< zy7GqrKI?@+{w0?zHGDAolTbqev@lkLHas1o_P2C9qY7+(@c{pRs`*L9nTNwp&tBN# z)sqf#V(^Nyp39HYBj^0Jj^n+ULE0L>eET;#&+x1UP1{OZL2SS9w+N-(TK7)r$8Rou z^egGgcj=#&936P_+E?HFc0;XhWWdsG20eu{+aM)XQ~k6yYb#Q_yc+=)`wH>37VyP< z)Z6K{RnEEdr;&S3wC~vy9Ut1}d_o(}wVe-_g6fZG?qLdy!Xl6e^0f>Q^{omq=%dg zLE9YwQg1`#XP}j_zyt)Wko33N_XjDuJq2GO7 zWnoBSRPRD{(aqTS>S+_Cle!Z!6B>bzI#X^9~qP} zMP*aRkt4?=bBilzG(j818x~6%YBi;i9ciC$#?sshcszDbP^<#f&5oPmTE|?Uf!8dX zJ)+YphxJY2D07<*a~2o4#vv+(-OK57-D02)yaNkNiCoHoNI)~$mlfRoE=OIRe3AEX z0U{AT*|qHcRK*5^w=zpGcMz+TVAHC##z-lLOG%m2kl41AkSQkTQH+far_ zA64;R*4EB?9&0gn)0A(%4O3iUex_npTXi^Fqp6p7?y_*jNvT-w7X@1&(~k?Z#Ox96 z^otUYk#^}`?e&FNm^^BEMb^R4guNXZ{360(9(nDBz!f3bevg@0{#jkOI$lt!yHY~_ z&b40AXl3%^hBUbu6%-JEIQKI?Uo#r7kv6`?fSO~<4Ho#ZkI+Ft0A?xTtuq+3TSXOgMh==per(| zPm+-^0ctHxhk?kYV39wRhHJWEU~nQM(Qp?aDKh$%Si!J&uD<^Z%YFs zWZZt4gyqY~Y8}Stn-{<-@mdmgSt9cHiz*$8>UsDKSyr?4omWGj{ffE^b8R@DNV$Q} zqIpoX752prQBjQx8CbN&hWVHK0ydi6g{M5e(Y8s#Gm+?+h=bXklMRy;%dmBE48tu^ z6~M%GPBoNb_BoLL*$#g);LxXF)sfPQ1<+mMPZeBSX9&jm4k3T&Oq=It)!Tr-sZA4E zdwKip^Og>O6n5?JVW?lgAwkDLu_79W0@rTKZ*7nX6sVzQRvod+OJv)7pA(cB}C zaMmv#TUv10$3XhAe2CuUn)GsS6E~0%ld-S~Ks6vzAb@52%DM)RLdmn9U16y5*Gylk z&gKI{;9jWUOu2AYVS;5kF|fl30^=&(yuS#tGZ=1F5J63PIAC?G+%&OVg zN=82(t4d7Vf?Rx{SC0y8ttmEI=l{}gK(|0Fylg5n1WLaaCpubP>2l|&i}m#2@{lfQ zhzjc464z`r_sp&^OI4SGa)<5y7jhv}XZy9T6zYa6~@_a8;{Tnn8U8O z;A_Ptx=}$NF~?q_xg(EX`y)VX&5l^8Xavh&MYG|w&Y!6lffae_ZA9R^*T?HgOibnS z&a1@c!&1DqOg~X;8VmT6`O9#oe*?DV7g91vc1e+I%%Q1!SXm?+t;jeZd~u}~jq0U$ zeteMW+!2J)`b=$GVH{04I6qCC`NYccvK3255@uO>G+^0iXleOBBlTS62C?eb@;Zt) zzT@+rTLT$?&bfCmCdS)q{qk(MZap(g@2T=Nvea*w7%d3T%TVuAI(a^gIwFUe=GKlg zznxx)I15sF$^i57!5Cie=}c)k3;9J^evaXT>=A3DYI55AYK-bO$rV6)`^t4b_F!S0 z{b>uLMs8y;!~$xrTB$zRD4>K^OhG$LQXv}b$7F54rnz77#mP8AAPXcs&c0 zh0lOonu?0-ZSJ|X>8utEo*Yk1WrSk1gtT}qYxWPdg^%G^D!6xvP_ESl(6 z%t0H621-}TKpd84NTV$2^<_n>VL|^Ob89q;J@RzGJ7d%`C;-TIW%3q=y&UsP&Guy${=!31<%2Ov(c+V8;q2_2G6YDI zwDwRD40?iYL47tlY|j)nfMjkCL?zk)cXpEKie{5+jiqeoS$0Jkd4N`Ig-rW(>Xe3* zlKp8x4wkkmCOqhwV-3XufU~hFsU^H6scxl_NaV!${5YQmI14c7Q3Sxau_g$(2 zv;11`x;+H@OsVHHDZi60ALrd1iMrb9-1Sc$|k>CI&pYsNlcoT>DTz zfMW^|H*=%^$=KTh(5#DU+!q%zIJND@^QAQpNid+rYzz|IMh-rUbJo(P(jc$qt|)DJ zFar243HQT2!F0O7El0SP;Fd^Af$_+4x0<9I>s>!au$Kp&KwWL*S#CelZ2g$-IRm^e ztYsQ_<8r2NV783BQoso{1a|rI)xDtKcza4`+g4PHV+HnJUeH5i4)?`F zmQy+T+!2)`*l9;pY8~c;EMYd+smxhW6IZb9XHxrlGOr2sjAvPPS&`uv)HFhXSw0MH~}^SKy097X(Rhx^Q}Wpg!4s|q9Ew0 zn}m5`6u~n%gtWbomJQ!s-*nN5u^D?&{`2+&3vy<{Y}dB^ktjpB-I9-8@2DvBLZD;= zH6?UYL<=~dB^5H0#Hwym6@%#cXPf(mu4XH9N*S!pqP@_5PjX2n%@!`W>YI-0dO7jO&Ma1NejcI(|Q9Hxe4hhguu|EHU9LLTj#t1obxpmtDQi1@?9f^>&99 z?HYh=tA389g>eFQ2>6xxSl?lKjS@=0z=k`J37y26MG&nH(01y^@>>g}ktEzPPFg}g zaGGJT)F>rl?k6r^Kn_yQu}!7|Jm0X$AmwDnbLCMfni}0ZcabzN!)_XU)F^wmuEgKR zXvk~!dc7#`C$Y^T_$>REkYiTT=F`CU)le?mSZmz9 zQEY)_;=FMF$B#H8aVG}{GX@#%E*|rJi%vGC;@sBweub+UK2_Km5MM#)sd!!-V(>;3 z`wppoKCw4Gp7tE4o+e)QD#6&|yeuGWQ|~zJW*BUq`U7LcdDktGKtKWA7PwxTLkhD8 zV2%BM^71v4JQlL*xP=V^28^_Eij6BbKzl`!(Icq}zL?$}TR>#~5}Qex1ZeW(T0}*O ztBB-6%5;uq*H{xF(3uBKmn*fZP;2F)N0=U8`b$e>3&xo1Gh*Z>f$~JC$THkGiEyQn zfKQ~^sX}>Iqi})hjMB@Mfz_$N%U;H99mRD;{})8fBfrL~;RLt+b(tJ{9oL4@L_es!^dlK*DCb{!XJgg*ES41aSrzTzjv zGnmgJ*_7XFY`v}QC|-|v*SqmJ{s-4T{C$CN61KZc!0S^LT9feEdsc0K)zi0;wnf0w z`G3kUD8Me=3-#Z?1#WoCeKp;5^Xt9=GbwI;UuipVRE!a#}^ zKs=m}R%S~a*IC+r!Qk}lY4#~p@Cd40n(4xB|1q{OU-pXPT}*PZ1Iw`Z|Ct3?2cT~D z;!2m7p}@M4=>i@Iz4ED;7;v|F<|$K5fB~jLu);G3u{Y1m;hBIF72LnJeN6Ew6U7A> z={0&ySD45t2``!Ow;LU?8KuLdA`R$@m?MQ}2h%8Ez}k|7>tGF!J9`MTyI`WB#_%?F zr#}EKKDUtK!h_+8pyokzG0LpGBmX_(RFeDyOJ>5|Ll-F_)6+y)C5>Q=X(OjosdlV@+>i6k@gC5O*`VoNKtQ=d(ka+9Em=Z z(l%U}(`UcrUn61#gMEjjnpw{!(pv`=bs>OXfs zZlE)qC`)!3ul3}?>xn_SFO?U!J1MLU>!8##hN0bgu`#%Dj z5@ZnN3W9xDim?o(Th+J>cG|drXQfQkYd&9wy^G7|B>E=G9`MXVQd}T6`2H89LgCuX zAvE;dk}R2tq{%^5*dEv7*O>@eFu;a5pCSGbF+q6d`J%_frgkLhinjjfkeK{toLdo! zi|0ACvs7;q&E?I>G7RD)MHg*9M-fgt`|GsT+zIdfllh->2YVRVoW#{-Uc_x^SnuKSC+jGi&xy@)%27AyoK7Hj4nDO1VrX&T!08w)F z?Zc2-beARpik-c)&QqBSO#3Ghl&yPgetB+}$m?)Gy6nL?(l6_&uW2nL;(0{D$l%EB zzPRFl#?$;unxo*8~<{5@3Ls{b3T7CGv+gRB)q+)15>ODrg@R4yUtw zY@54v#FaO69;GdjZ@HV%dKzzXwBhvHn!~Du=^lEgsu1^WrQ0MzgS{`aG(X4m6&uFk zwS{vPDcj-Yw)-tv`rqQK7QjNpqwt_=q&P2h>}Llrhw?0kNjm%W@`Omk!$1p|>m5t! zg(Ge}aQg|1V}$b)v$oVIVEQw6mR=1Y=xGnjH=gK%8l1XxirRgyO45U`dDdLLHg=J= zf^j`i)N!y5Ym0d_`2wQI;2Fke?RH8uVURFg)q5&jcaWt6w@z)wfForVcpl^j2 z-UfOwyj>g9lp<^6(bz&_;bh_PIn&iV{O@||ipyvIG3@?OVTs%WM#)oR;A^REM^H5X z`x(RjA&P}J?D{nw@!6eMBEXTrhoMz#Fo4$i)}Z`{WCpJDWbbxXPPCO-&TEua^GS&k zbVS7RDH2)Vxb_aBZ}RDKwOuKjyRwUtgE+#Gy8o&Q?`^!nf}i$$pkG&YxmaDi0>gb! zo>i8aQki|UvfLQm@UkTJHwJcafrvBy7LO%aF|6Kr2h<>!*uf2Jm*&|GmWA)#NER%P zf6^0L^Q0EJ{jl!qln#{meG6gu+r`BzAa%R-ufG;d=#}MCg;?KFN(r;X1z(_O@T}Z; zOQBItrKEj-!RUTA37{?83wOK9$|D`Cu%C%os*2d!m@ns?WNvQvU+8Gq?KsACy21*% zwx)aoL{CY;WrH0QVe765uXf(#!M3C7SRqYP0Gldl&?ryAtY4KYGC6M<=2cVial*>{ z%}c*K*0wW~uISFq&SNgat1)MYvnRC45SL$2*xO`MkTs;Iw#88p2Kof?4@iyH--{pi z^<8?NDkW)GC$$&7JJmFj^;VYoODo^M77Hyv6*F)XI-nTiSiXI`>js^?Ee>qDf|pC- zl$(rs{2s>g49EuXmeO3))f`V-d(lG|`GuunP0fw>xell3WBDB*0j$UL8wQZo*0BJwZPQOcYC#rCj z9@#eA6duJjjv3pt^gHcYG@;|a5Vl9sm_}(}%)2aIALj>b*|ElC;Ed3#1ttsE?6V1L zR?}$zo2;!C3#OFRS_K(gw`RI$!v%O&t*lSUY)DG|s_xM{@gqwn(POc8=@3CbF`169 z+Jdc?tB~i_I=bYcT!WPemA)t>)t(P)7LooQ+{4D#r2Oq}9obMD+c=wq+i1$`Mcbb? z$Cp|x{dji^jlA8i3g?fCk3tx;UdVpY*RO4FH`HxQR$Z^Bg zRdtEs8>H_CRed;KU0LXeX4lB8h3%6S8w*Ff>rKO6+3Rb?jH`2<=Pvipx88=#>$>up zynlXiIUFR_b%|ucdHY8KvTlA?$T*^RKHj~L=yirtm2Y2kd^F9!67k2ne{qFvil4sP z_hq~+tmE^$iqxu0Z;vV$ELZ$;McW9xvs^V6O+`Y!=NX>w&&7)%;rc2dh(VKUFysBGiCmgFXV+-6z*sjfH@(so5{Pj^zSyD*IU_QoJOwyF6sqHiEpOY+I z;>k=}U=PVYUytMSHfO%PvmVfN>vrVTwq~?#r+!>S_KmT$EXQTf&m^)#<%-r;oT~;J zbuu4`UJ5vatG>#Wd1d(Y%w?@9Kj*Eph`PV;NkD#;@LF3A>4b6a`-&B176r zPEw-UAQWe6mr-tHCXf&N!1AwsLF%)BY16zT-z&rO?~ zNsNTb)c6Dv`>2(d?Y|y{;4i(GxN~U`jf+3@=jTtZ)o}pou>Cr_z`5eWK2IjyD7ithl(lTBSzUIXrT z=}3fyYf!_AyQj{krC52E?X_K$@iQy{^;|2NGE`VKq5+0qUovd{D`xO^WTS~OR$czV z?i&eHLxt<1NB2PUBuBuFWep;?zN<4^2(NS3I$M~|C02Brykvyyhk6dlS~-{-k=Eb9 zI2Qp)(a~;OIt4FgN;K9Ky603VwQJG@24!sa?Pg%?%N4Zky;GSF zu*EAkqWMZwhNzNXZ`}S>+8PJM39|B9LttlYSID#V*XY#~)rWbmMqVQl9Y}z_3C$5p zqh3DTQts#=KvBY0uKmg?+yzxt=&LgEfUMNIe<5x^4+#rx7N7QV|8w>XP463BFLL}C z0we)P4#K^fMBtOj7Gbg~iUUJUldU13Ev1Cp=QS=hDX^h`kgl$hGQ8d-Sc2_(f(!(R zNJJ#e&;q@26;nFSsuzSCU2>=@l$&Qt@=oP2;E~qpuJ+EcKoZT|{P$R5^u9Yaa5F#h zAkCBukKnvc$1B{&3m2G3h+IIjFJ8+IhcnvM_MAkAGnq2p(_l6?OzuV7-T8JKp6fUV zZn3Qw>pOkfKfz*i9prZFlJnlxmzPfO_Wy*eyJroHY9tCa(21)r#wfQ|_Gh9mp(7|B zorU{ZXw_1mG`JSo-}5tZ{kgNN!q@RB<}1VLAui#BxCbAvD3~eC4i>|p=6=RHoinx% zyiZIovqZPtaDlb~2=`QxHH5b2q@X7E)yLM{Xa>p^<;6JW(#q=2!&?p^FZ7t?r*1Tr zKjjrIw}__4h1>0Tb~8`zfrGvqN3}uZf{EPTcQUp=AI6m1^5A+S#Jj30hgG?6PY?Gg zh)d2p?ZvMUKKS(f%!R5;Q~9pWFuFX}9Kn;AXFM{1lD>o3T1~52BmKoN%5Y&F?_>px zperp^_2gt(fG@_O`H8TYn6WWHU1JO5*)2Tj0Xi|O|E?3pDtohB4xK-ZrdL_HWXnDF zaPhrFv$`W$%88k3O%$p%BD2#wmz{FHKe~*cl|MOGXP!7d^cQu-o1}nIfOCey7vQ@R zO6H_%U&3v6JG||!BMa#3IwoZa$OfKzq*n&%?)$})*+W6S%X+Z?Ou^py*5|Kl zs^YJUPae_;Tv~`&Z>NTL$FBItax=YO0Go`oIJ0&X<$r*{v&D4GOLyX zP48Wl8|ULs)K}7$@7Bz3NlYoC$1RClzIk4#3~N>&wN!fwa@w}P+0}AENP{Kj4T~_~ zxdCa?oUr!zqPz6KnYXPyOi=KIuzHo9pFgi`; z)hRHm6CFI#8|IvvMPkWPFHc4aS9Lo|`g4e`A4f#`yUhWEF6dA}fVTQDt0`sv2-EY8 zh*2;{vGB9!IAb;j$S*8VEFn=xOimu+6^Erl5ebuq@iD(0Pg{_~>228=r9n)yrPa@x zhuzK`^+hE3#M&QZ-#pE1+wC>HpJ&QdYA{uv4sp40C*L$A(&$uzkLI~1Z8CeOguQ*9 zUNxp73f@+fm-fXUoK*2(0AFEXBEBbnMbRn1~INvg3ijXIF~c zb?H{H-ob6uD};cJ!Iu@849A8A*r4(qzhGc_^aT-?UCvq(*BT*DLgkWO+OO zLH3x%gA6#Gf9p|*1)QEjG$#S$a`u@}+M3*;_KiES>N2zTiXWiSIJXT?y_HtkvpdV% zObbRC?%rPOrC81oxF+Z`6+B;C91)uhXt)w)0BdD%p)BksGn^DIVG~bdeCI-ide|J zpXTyJ)kq?x#yW;@JPc=xijOtmt2{?70)s{5F-19V)rUoEDO2FKqM6CV*mBQ;2CDbt zHsDRM5#HCK&$8q^1ZxzVZ)`ix>aD{%C4K?OkDA2;|FOr}lnxnUrW&}UeemC`!*Y}6)e7&k9-Id;^8%P%_5TNB#{EokeJG# zZ_7b2zY(2rd&sl3j~Ms`Lh9k5s&uqVtiEemK?nrsSD~FnvT=$2*GY&0sejrK++qih zq;-2`S$r4Md)eBPH-sdaQe`loUQH^jNX|S<*qPRO{EkPF4^$^rJ=0dCWwxheOJb3P zi2~sJVI^9S|AbiICq&P76wU}a+=e^|{kRpCnl#$4$BBIB_O;|NC(4rP>Ba?bI9360 zk>9#7x3S6%X}jGC}lkBHs-_w7BX7jiRk^-0C>i#n$Rewd;SU zfh`Fa=j#?J5B=px`-guM$65084=K8nd6bY49-jK*1~BlVw6Dfs>hdg{DWx^8eXd|z zYLYnX1ry`g?bqlBGR>Eh8hTHkLt}=7r~cNOn{@39jUr))Z(1W2B3cf`TvVuwxF#-8 z&s1_|vEc;FIoBty>`LP3idCc9>CCBNyAZrqTypLpA{HNkgQrTZzqVMjXKq}43(vp? zR;Abytctvz+9q7U#hvW|m!}YI<*yV8u(Yy2oUEJsMJ2Pj)A{o8o~xx`4uy$K?_3=T zXBLJNYknQ^S3hQ$OoiBhLR8{>i&jDQA5&<2Ptzaa{7a&d9GAAeE&}&Xftp$)qU>y> z)wpE97c@3iqF*YX6|W?GJgb3=;i;++Q>_$xN?nK}zbCb!6V5&}tvS8UQ%Q)li%l^h zlWf~B3SFsd^gYqS+E_z4y=pc!G+|n zAgM^nuK@~P;4mt1ml8&_8uMfkc_lPMb>;9nJB>5SA5YG)YrV?*Cs*XlYeM#C6w{kq zV_93vwe=&UKSVV$5-7z~shMPMjhmecBVi0OQ{jjWwr(5m@n=lxB5^yjL?vg(AI?IU znW7@SQnK;mZhG}>olx117@#;G*CK1|b8|8=((whpQmxZaoWfhub{H}D@|J-v0*{O} zNjNdCf6(T_^QjN%dxou#ffBtn25l4`Dr0|W=vz&uaJlTYMsC24?*AInwQ9(UWA1p*NkkaBAFm zD0E~EVqrOj!^NHRX1vJyq>CZdZhg_bM;2;2#XaB8lr&Rq(g-NEy7P+z8&o`P(F`Q* zRvb+ap0mxklzw6OpEbVkLfX(I*JVNM3G<7KAZg@ zG_Bc*(%7%_khb&mwFJ!8VU%Jjhl4RU!VZo<_Nnb7P~l6 zFLJK&h@zD*b!+ER&k!gjCZDdAx|Y;No_Vh=ajx&SjwgtpI@~MjiqJiuT=(~z-$C$; z%WkUb$K(iHM13|Xu_p1DRhAD8K->K=Kx5>^>ij)O9}mx)OlmQ;i)lOE6gHYbu^DV(Ncm*yeW7pYtRGKOAY#yVEbS$4h7bDUP>9ICBr)3y?~(NcWCp~H?u zI+`se!m<(Z$bFvkYnHbzxH;TH>J|Mtr9nCuO1YN4uOEJk&R^3Bi+33#EaPTaaI-t7 zA|_O7TM4W-qcp+@%URhnD<*~u?#3jqSyAImv+%(TYUSOSfejNlX(n@~v@QkB7S&zV zrGA*`|E}Vaid`rH0auX(;Nt+FAFZ#fx?p@P! z?aAHk#`)5mw7Txu%dMo+wnFvvdKb+loG!>d`TQSYs`M}!61#PQtr87m7qZL|R;l9; z9U%7(`Plqh{DR0m9Ud0FyFP~IX+vndA7zQAQK1G!z~anHDPSfXP2W9|Q6i3^xx-1V z+;g@8)&}CBv`$mKcn!~Y&DeU-%#w%PxXFFoHRWztMvz@s%F`bWz7dP@eQ0lUn{{njObjo9W)q@070cbW z%}d1Df@M&$0HN;x=jh$T(meBh?{~km*K&6_o-Yc6Zq}i8Qql6_M4IXNscg@@%WL{ZqKLl@AAie~^s@E~URG2{+#sgEv?zXRU)`3 zpMW0>HXAhaV%jd4k9h{gDCaUeHL#vbXcE!HQAWbWypQ*K<$8 z1l?$%&gv`U8Gh4ho*v;;xU|Sz%V99n(|`UW07Io3uMu*90Mlira5lTLYRC(?XeI{l0LC$M+lx`t1go8Hkcn z$ug_e+99f^6p?_zb<09%Ripslni;d!-am9g=(!hZO$6nuQ6;xd;Ree!|7}MEEY;PK zmwyB_r6L$%66!K)?>%T;A-TXduaW-Dn(Fys^D-KG3=U5irpE{aD(pCd- z*}lT&?NIX`{e{n-L}%@x2E1BN`&*%RQnIDcEJ#Fwk5Ys+U3Q4QDd zRp!N$u^9JpmDkK9SU_ptahiRvn-7)?N8f=4S0Z>-M6O~CB@hHB=}6ZQutsp;&4ePQ zWPD1}YKBiucYko6oJ!i=*Vkj=RxS*ylE+E3CI5jX~rS>_zs&T~dB5%RJ ztT<1n-i-uT5pdt8+J!!eLi<1I6kUUVBO9dwpUp?8z#%t?syT04*N?X}{)OLJ-JSdC zkHmL7|n zqcPzHRe{OCsm{HR1QXqP*~9l@p(j`sQ8149eVQ+g5D@kCw%HkY`-`Vnz!qo*uoI71 z$8AP_;3dxgI89((Ug3(BHR_}M<(p(#MXh0^)~1FRO^YzD%LEHu(ZTawGc-?q-`rlk zPU#0nX-yluFJUbB1;HYG*}YyTKNNRI(7W8<|2GV2f7ZDcd5SV>2OyCZeGp?#HPRtM zA9qY%=Q#xPeJIY;zZo0S{>7bF_j2d&1?V(W08H+dvv6vm^Mf!rb8gZR0rt;v>jMP) z{%kC)t)&fb@+{TXb!AgF+QCvev`uFbf$#z#mrp@mPa8gi=8O%JwG;MhCub89=;7fq zOB-ma3Ri-)?k=$QrWoSPRtuRJKlalS07jC!6=C`I;AVnzIwNZ@v5qB%Y;9&o5Q!o&dxJZSesJTLHLV z%gyWtiE@A}7)+(qQPsJ+lK?8S8Dsrvh(R`@rA%cyU9POqmg9y-`};>9%T+}EbKvlp zxXA%yPnX}Lj4T@~>>uFYhn*MA0Ru%9l|Z=P>4=aGMX|4y=msfup_R@@)oRA8w^NSTUiJH=pqxU3S66cwcCq?ajO7VAMEFR5pF zzZ-Z++&LZDas1tUo3mpAp-lJ(Ne9TOmW;m8fx6IdyMhuq~R^+ZeQ>p5|M}xE(03P{kS{Mm1Wb3 z|G-LHAC%65By{Fjxn_Pk{+N>cHcrur~ z?=!I(m`ZT!>s5F+Zt6s@nq$=CRW5VUk+P8*Ecj4zefx}@DJzq}LeK_gmAgIU_17FeOr|+XzjRiJn3iy0&>MP<^5jVCcu)ibB&uozelIp zz5s*tLja09YAiQh<*zo^9+rdY1c0svY{7sot|vSFAb{aQq?`OS-KoQo;s|@Ba8E9d z7&El-1lv4X!5N}NPF+PqD`zo5(V=EQLAs}dSBndI>42Y&%Vn;Q1ym#^wA!Y0tK9C0 zj2Ya8)5>(gJP5Ar?%VuRxY8KK`^$Ch=~K>+T9d2KIs%mpYbuYrCcfj`@&+#QBz{dy-e-V-l#RA7s|0YaTBWFs|!y4FJ{gopZ`u-W~iYI^l z-=*(-d*y7-JLhIQln2rze-87#UaACaEPDiPmoZT0+-gAndP~`}=Ln~FeEap^c;$Hb z`gTr(De#}59}*B93U1(nTd<)1GKBh{*X(j(2%c2_TJ_P@z@JC+XVC%x@Z5ca=3{VE ztvAEEH7a)Qx>8wie{}Mzfu(<>yP1f}?ZtlPsuQmM#8%YYfVGXeRXl6b@RIO(i;xmi zI-p)(O~Kb1_MLF#w=1Y|0YHrYZosXy%y;}%R_5PL1K?O|Y zzEP{UG7CI6zm87I@bM(&xw#hvGu=^WOEb?M_3X#om`2N4to^qLg~w|nb;IinEqX*4 zBDNDwl7(SLOl>mnnx-1lP1C^>sf>H@Tw%3k`KA(^17u+wOABs{XxJW(RawTW)yy^i zU`n;f4!{G8+o~EjXW=9ysh^wonQ?H#%3gQgip@y|&ZllP4IV-*+@|HDRm~I z1ppS4FL6Ew@>3rO7njfoM05JxoMq1GwI{^|t-O!^us<=)1nDkz)_r=LvvIR%O*m50 zH?b&%^a}~>)RVbldU!-YU`WnTVll)vO$TU(PZgCAO5KcjZy=0Eip@e`KQ7EZE< zgVvh_OUYCT4+=dNm)tuMh)T)Pl2p$yzJJtlD>lk9kQ{I7RX#r86la!} zq?AxUnE?W+3Ki8Mb3vr(ZwMl;$SXZ|74gSLsG*6#x#ODtA z-)Iq7d&w>?SfH<`GfAEZ6lL~im~LqmYu=0xr@FZzO&`@`gplpxtGJ+B=pA&|$|0@X zhTt5;@3F5=bC~fE>V6YwYgW3J5zt^hjOu0*j@OKVsS&vGb!5{PU95{`#vp@C0mJ$J-pDx^3Q|O2QEgSFUT{~sSX>C_zTtD z$Fl2C1Y5UYdpZlkI?ZDg_K)Z%{=443X!fE!^=KRQ<=Al_4;Lvu;K2$jt($!`jwzF7 zW`(Sf7+$C1+hcz`^WFOKggf`5KAZ%X)l`^x82>)WKGB}w*1Mya<5iCI9I``{$xIyGzYW~tUe$>y zjaD^xuUSn>1Hj1?TA6!3cp(Ac-5A!y;qF5UM1-WiwwtK85_ zTU)o|0oHM(H5FN)o9JMJ@+gC5KAPg)40l6@4PvPOdyla=q0PS^2*bF)&#v`wN2-W+ z6~OTcd~q&7EXhEAYnxYsX)%rHj<*u+3{v;zC^jc~Z1Bz6yK?H+!X^GU|IxiMWfXJbyC@rh3tvO-)1*JsF)i4XO3Eq|$(_;feq zEtlNA>75E=J*XaFt83KV{Q0BND5@UFvIVXUd!#If0OpfyWWcIa@___93}ps*tVIzZ z0N-FxR{;TtLB%)3<)yd>dD0klO8JX3!A2o@T@~!$(s*2z84e5#poXZw0y!_4 z%*TVBEjjpNNS?=~jpRUk=~G52LdxGbI&RJ-*l?v!$shH&J9(}feC4AOqxFDg<7A`r z?+k0}nx;z5u}n3TumrBXt_8pYcBvL@Blf9co4~hzsIA)Rp0Kx;Yp@446Ie33o@d@xN3AOpDAs#@LOZz`V!6FB!WYFAM zSy!WD;u!GMjkQKbc>BFcDM*)bDl^Gb)97B?UD`f^CiuPrR7GTb@|l9_JCB}0;|``Z z5Y#o|u0x8lC%HGbopbwVV(f3i97kXP$m;m61t$o;SRH5pmJSn+xN;h6TeT$U>BO0K zZ#1r)lny;yl}mlZ_P@crNwfcdeD!1`b$8e5E|EzM*wq zNQi*>9#57=+>A&!1Chdl}pj+&C_Uy^ZKM+2M|DIR~$_c#fb5}ggoQ%KAMt{>?!IafSPft zwRy=4iY?JP?VB*FQsvpS&IuZMoczi^ZtkqZ)Cg6LCOv zrB%wHjAok(8{_4`ei%W!DHgo!Mh1aY7%Ol`V75=j)ajqc_M&Qf=s%0BU(o7s3Z#IB z%^OohuxsR;V3*Bz;CKSmTLd4UN2vEDLdTtkam_)sqbZZB$lk7c(z?={Nz|@3utu%) z(3TCOeH#+I%c|Qh*8oq~RrikZlscniFA5Wifn=`yvb!XkT~UH9`}5hh z_aB`tI(O&n4__v5AfI0;IW|0OKs$OlD$c4TCW1H1(uq#?Br7uebvqlE+yCrj`&+eAjG zNTYdv=4~!F;ZC>b=b0~CQ->>+L)d1PxtHg82=e_6DA#*2m9x{6iKjQq&p`B%l7d+6 z+nHV^?fy}aFy*#>W4MxMxeS(J;~TE7$mSSm4^2m41A0L3DW4!b9 zD3@a_Qkp|hP78wy_HezYltv0#W1&#%_RUcFf1QOevM!;;J`rE@4f3<4d=IPOkYKl(7kDkEcyL2J zu0^+!R_YAjhBkx&(_upy)!ft|*_`TZT+3$*tM>QEFeFm0>_+X*foS9{K(|<1WN_kJ z+UQ|Cu5jb1@MV14?p_NRrGAqUZ(n(ONfX5Kp0 zw8~Dgw@Oy3H}T+I`naP;84S)Po=!{C0ij4>D!)Kr|22XPE^f4`x^^{!%OgA$5bg6I z)=)YM)|FpyZJ0!GSxohMK~!^)t*o)FlE0kRfvQnPS!&>eOaP8DC=_{j;sJm28G0r!P_5d28^ZGdH^M64(M6{`$X;1re`qNI8W@jA?d4}- zGYA8-pjMHN)WE2Yx9MXZa7p}oY)E;V9*qFr5?*0r0T#BYpF4D$G>j$w|ePYH!ZPBTgPW?ymbM3kgGH?kK z?P;mIkKn3uM>}*p5&v=^Rcr*Ac{fM|z6obNkSfnLzh*}kk5u(D=q^I63tFKA6s%zo zTpuegj5TaN>hnAfL+#D<1@|CI9@%#rEPR3*8EF0m_vti8Hq}I@0c4z(5DLCQ+XV=l z`O>h6=3ZWnVMD6#x{a3@nVgH9=h7a8U?sVxq-OU5s>k&jZkIooNvBSyrJCP_ASxHY zZVSXpH$l0*a<;3bk7ftE>>B*J_bXo>jE&jdX~{D$x^9MHDji4pQ%9A1k4=ZcfR?g* zIU7rJB4{+zAuxhcZ{ukco83O+VZ#V83snq3>AX2n_ht{_m4~Q0z(bqjD^JHl?fHFv zIm*sZu1wbI)+deSFYA~7_1U6zWm&3ESzh>8tp*N_eAvuX)!yc!NQUSItEV1ZRhd_= zC9w)2EvU_#Jc$FL_MXd3CaXQGMJ5g=c%s`$wCA^2Qxj32y%@S=X6ACN%HmwY$Put? z$d<4Vii?(xS*cT1M%5L>hY{At25U|mxH-lIrMC@3{#jPwi`7Kp_90d|@uPLTI+v@a zTAR_ZpwbpNbuT@wl!hhyb{_9Fu3legBm+aKc5QXANIW>^FsoFP>452Wts0t^W?;Wn zQ1Eqm)kphhAGq3WJk8VS_3Rs;E_~#S{9y~@IGty5#irzjofP9~NNaRPngNaGRDREy zZ34gHN3Wwf?Rn-i5W(hHlckA2U~oasqaDdmjvE#wSVoU8esX8EPWlQ8-oaz#@=2vH z{5(&>WKK?!0a}`9p3o109q=z=W@n7VP1(<*^}(ww&htE*1dR+3TaJM?jm> zmEOQ8j2#9i4v0q~90Q|YM8tCGzkU1VJ#atY{`2&92iUm!;I542`nQ;1xUx31@*sFF&-Na6l-Qq!+HV6> z;rh+|Gw{O$Tc1Sr;_9}l6)jgtgSUE1W(q-cpq6xvq@9>KbS%Ub(;M(;802?)sl!r0~4EBCVSZvcen*Y z>k_Y+Emn1%|C~WP+dufuTGvSh%8KI|9h_R}< zW5mSfEOj-yC33c->dg0`_!c<;^ip;I`H#N+&b~TD7Kl7aeKpcO2w8HQ7GS?l1oMD- zT(@S4JWNQvJmRyDnasO(?vx1n4E%hZ@0M*`hOCZZb1;sr+XkzV>pst&SVsAcR-;&o zOF4|IJrKdZUs?^J>?|*^#(XOclaa!Slkk`tIr0h5_~t<6MhP#V#6WNP2HKZO!$?o( z%ZZwu#U^FNQ0+E90??74ocTwMWQ_MbTA%3*YXs4npBKbQoNtI^eiVUmPf;)Jb)gfFut%dWXC(4|sb2Vok!7Fc$HKx%jK#xKrC zlJfMGuz1x(VR=`jGg6{Yi{@M5^p3X>K}eXABn1 zNA0! zz>K=vw`ul$vTEn#VRj!Yq{Oryr_kK$sY$ZJ_0n=7sU`sI(i*$k!&A`qAUE8!z#(jg zOANRo8kGZOC-Y*5m&Gf@Eu-hcO7f3v-l{DWT#&Vn7Sdo5k!z#D#Vj2EY2LMNwU@^d zl80Jf>izppbc8j9=3Qf^M}!aKRLQ|t#WsICw&YkL)~4cnI$}Ukf_H!$;4c-?#)A6U zhV>D!dti(m@2VPTsxawbsav0-(=0!=+-0Y{*J~_{kVRW_;HU(3A}#ohVR35J*IYOh z*DzG^5LUA+`n%-Ti)j-!cjBZRjj@mA z3+|(I(-G^}iymArBmO$=D-*gmj5=T=r6w2M1l>pzj<(&Q!wwf%`2*xD=fAwkAc3UB zq)eG=h5SFGmrC|7id=QY@#1GGDdU4&xqPg$q@aKFdlIdrxG^d&XPscOjIJ_zLHVO?yxL5ht@x-1!jN3NDW@{ClknPoJEs7E z&&sfGwlo%Gf|hw&L$@xLt}fSl$uJck4EB%c{1vuGa1`J^5qIkQ_`z;LPyuyqftZ=s zD`4dUIAKzBv?W|}If*o)Q}GljR`L`Q5xEJ-8?n6>Y&<~Yg8|D;=0)K9{N%CFCk=$< z4%{|+N&pRC;;f)p9>k*u3KX#!xdh!1K}i*t6gb|XIT&d6uvAG==MM;7Gr$8ovB0V; zON}Y3op@l}`KJ08avL6pLXqq*iq0zTyF0e7zL0x_k4TCI0+!cWGnw~2=7_G z&sUza)a~x|ABc$i>?XCcW_xj2V|{KnHFKnN&JJkRkKV_9>)p2l72{#Il9yepS41w< zMqgA1>oIkSl-N?vleE-|ntXE5otf-T%RIusip8yMWu!lHRfrrymT+K{Qqo@RGpRj% zSEUe6<1vnA3XZC*)!N=A&#W$Ylp2)N`f0OI;}Lyf!hG^{_G4P;yQONm1q={3f%RM^ zPAS~K8Ka_^-_3wEqy*lA3|dH1UQQ-HaniW3dEuuet?KY-k%%(-XGCivzKUv}Wzn*G z+%T0G7*zhO+E@x&A;H~s)g!wr|HLuH)X&QTp?<;-l#5HD-(o}Cy$Aq6tJ>ahEwE}0 zz${S{(gJ+BrC6v;4_ZK(-|VR`&VZ6V;#Bip^`4XEWoITQUSf`G=7jzQ6yBx6R$W@B z$R^W}dt1miqX05sIQcDbBVoHt2r1p>4Ue(^&}ZMKxfU=NRl$=O^F4His(kifON-K2fXyhG z%`lr&_TGey20Kl|^yB)-fore+_fQvTxQj94IJed!tlXWbkN9wib08OEFum7Zf3s@| zWUE!}yf*Ki4&hw0lv!VA^}AE{&&bEuyHmW6=qaAZV16VKyet67L4aAXHvp^}Zd|{{ zjEt$$FP@AXXd0`Y)k}@kTkkDQ>f}B^C^~YUfYQ!ElBkxt6Mb;-#IA-%3+*?64|@+n zT{$h!qiHYG$21fDp7inc*UOF;_+gy}naK)Y?r!ATn8--We;UhrE8T7G2@uv3h6cb~ z_IA((ry2kanr>bB=?}{uQKkZh9OUv6FhA5S9126Dp!CL~fu$kS+mL;&NkAD#J+ z2ts!KfgcTV9Ip53XIX_2bJde`aZ$$_&X~@%e9UhJQT9qKbdkR=X&bSAy-pBMqI;RP z+^CAR`!ZiPQ5S~so$WX#6_^^7Ti4thCscuh88$VcyeJ*8@kZmD*aeh|g+9k6IdXHk z@rq$TzHR&x^y9~ED8l01t`g|63?n_Lw=#d*Y0k;8qY@0 zS;&etob*z^T0UjsSvPJ1U~GTrr?i}yrEbx=Y#)Qvn7y^&^Bv{Q!nIckqd9akkn{-+ zswK_bLCDxVW)L|30Qp@W?OV^ z?*Xj4Y{d8wEK?a_Ljaavt=U|4jp0H?y#B~|R>~@_w6r8fL7Iw}0*zF0|HY}zMAOUK zQR|ni=i66DbE2cI;s23In@2W8WIwnyOHh(rJ3Bj|^48}}?ps)UO1p+DAUm#B50a^& zb9tU~Vq>{mJ9({0A#N6KgQ=RP+!aaP@X3@zpu133YXkd2RWIJC*9+${MGq824I~Rl;)p^Nl7;BRVyc>z`iY2f=s__OlI`xgr z%O^9smp^x&y&cz`z&OC7x4gQ5h|DJux`X}wobo_sXhljI(nRPlxU9LOTO46=b zhL3 zjhld#BTuHe7c%q6MHddz91V1OTUlO4TcYRpeNHdTo)4SqA#Z234G-|~n)&B-1&$XT zqBR06uuXqB+YYb}(R)8IL3wVSPt<592Mlo~xHyoxWS5ndwvJmD7g+mylMfbRK~+t5 z2uIUql|d#;ia~S7xzdP*MVn!nm;vYGG&Rw#?e&hh+tNW*8d9FYd#N9K&qUY=?mI$ zXpelDw$mNB`A^PEZF{$@G8{9{Fz1ky?O=t}nrF<-pl%Wl!uZwN%^19>lqTe!;oXN= z&N%6|2T~*@@O_#yB59pJm@43tg4((1w2~ND;6WM=<$1$kx|e^|R6jkCcM6ytp-LoH zal36$U{h_jm#ui#FtuwB3M9obf9~6R5-7kFI_|1}dmETq;dv~A3V-H`Z|^22OkM6j zt=hSLK@ZU>QN_6_g+%*Ny=~B?;^VL_VDFSDv+Or52 z63_dH_DvPZ_0YD?s(n>e{?xY!HefFi>YYv-e&Oaq8W7#?L+C;uGIgsy2(Go7@}wR> zC@kN;o9CSC3;l?5#9~6I+ALQ(xr0l=?W?$gJm2hO$GN=FGkML{RIzI3rqI!1`G!veh;hyTSQPx#lJKPw zPB&?Z8Y!XyvD|>NS8`K>INb4#e?KtTpVabDymY7H@mERz{l^b~tw{cp>+7C|RQ=nT zb$-s*)d#E3yfe>v64~&lnX#UL`A3KqW3q6e{{b?}q2_hPd1@pS1lxu%ZU%uTMS8`Z|`; z4Ep6|jMJNUB+B|5|FgO~vPSIuWTtRn*0TCl^^vu9jB~OX2Svl^|4137Br|lkS^Hn7 zsa*T)HtK%3*u>9N35nL7wMcjoM$QO_9al^j449yGvzAb~ZS1OHVuqtelB)(&k}E3) z4(6mJj&Euda`53Ra)a=wX2R~dZz3sGm|i+FNwCTQMH^K*!NQ_$bZvHgk$W*ZE(U1w zFy{B5&I71j^y1l5r9W(HwpVUi4zcNmL}n_*Qkp3Ycv~`}p&4K+#c687Upkm(0%%`O zphulra}3E`vsc4Y_ogDD%Wt8JBc#wA+JccYI|&MC`wF4UM{~MEI}c&*dG;_p5XfS= z&H_kiCS`jesLc!&3Qo|v7_?EfZx%J;x{-AzpYl`yEP$o{d**%yEub+0VqM^{vc1?=nRjJE6Q#I3> zsOtRSEws$p*y~Cj#%raDGLXQFqW>+9A+awuv6mOTJM*tTm!@pZT=&IYeb}i_K6+wo zCgKe6?!Du!4XPZmeaQZt%x=+T{u*XRPw8r$^(-58ruOPfZY62%L{sY-gKoXF&b&Y*f>iwM>7!&NgvMp1b=ysbRwKDJ(X@sIP@??FA62V2_IG2!Ny@0|N z?5%8oSDoll@WaMDQ=1PEz3K$d-sBJk2JU}1(rBrYZ#uR>Y zVdCf3891>%g8ZwtbD)vQAYW(fKgsk790uzz>_T*OhNGgSB(32X zaLQ;+*}0k<4jv$+-HAP31|2TD#0eiKm;I~waDe!yfp|w&Y%v!ea$weJYnWS-#z;fk z3F-^OxJVWV_Jy1=7~;&)+|bJ8Ndt=-w5SbDNqoFfa5bqlsk&9v2;Q@{^=KZvZPY#; z+L3S0Vsz)4S)*l!87I%3v$DmzmseuV0m=P@2ym=fEa-gC#INj`!iUPSmDsh+quV3B zmif~Fu5Dj4J?YF68{oNwC#=a}(XZg^mk+TN;zxu6@k@!ExRcbej` zqS~=4O?hZ%>jOG@-U@`E$b&q*aXCFR)GZvZQeSURFUT!-^9eB=ShMBlcczU0XZXl6 zse)WwNe+GqbL`S~j?yLNpMND495q#o-dTV4KmPOWlGZo=m$&ziPoa?PL#tmNPfC8r zD|iy4w$(#HgPSZ8AnNp*fwb|7)7^Kcu(xFUqNVZ}#vmcXfZ4u6@LK+*d)G~)+01JN zSev~9S2uAw)^(#E6v+z@%gZuxUcEe(_>dxltRs#)CukR{3oahi*^9koD{>_X+4AJ> zm3FU`9FC)oGDdtcQV!%xyfT@80n*Tp=Jl1z4ijtuIcNOlThrI0uKpHoKr|W%^tTWk zlgq*D#&1zEQKmr23Doc$_u9W!*`l-HQ%}P``V2^mrZ$t~)4zY2_{qQBlPu}=|JK`( zmgd^~iSe2T8TcguIHE2?$U8t^I(xHWW3I}+^)#5K+Dk+#VV3pi56#qBR{aJZd|X72nWcTxu=N&NxO0%+<(Q@C(!3#Vc9I#*Gk4BZl8Clkw3b6p z+ujEMrw=lDmTy8@3PH&y;Av5p2JV@v@vAxJ$HTY9@VmOxkfdszC{yJeOb_&ouSDh$ z47VYjqB-Jn^`|#`moIf*PDy_2#uZRI0Ayd`@G|{LPfJd|Gq<>F&NaXfH{yp> zV4k-Mup8DXT`uNh@oN5X@+I!xNp9=7>$x17b183K+3lEI^+|J+FL2}qF-q0*Igg{E z&+dhhiBBIoA4Z#Gd1IgoWW1M_CrplFkPpqVQY)1Jz_@VXeA?~$rb+Y8H^^q)%ERCP zbz|`d^vqfdZK43@J9#@>sJme_S9vv_>;ppQ zO8wT#?tAL}Pi?M$f(72hgnnS{PSQugILzk7%LOxvQrmiY)m$d^C53hf-o|8I;1m*K z^n9L=%&2i^cQ1Y95j2JY*4u9@83gUa3jbfZBwZJyt~;&S(Znl@!PTvpQnnt!oD8A= z03fo19-iX?z}GSqyr*QM47kGg>8$-b1N;Q>0Rzk1K&x1oh1BjIVu5?urz9Dd3M^3| zL(Qa3Wm%~&Z?n$i>*j-+T7K#!D<>}6dQft4>7!$GDP{#B`Oz0S~Oo~7B#2vzRRe7T?*XldGR`bC6ol zGIR+PGOHS}PHL(|CH*J=AbFFMoz|W1Q?!s0hp4)lNisON_X~YJ5Q;wr%JuEBS$xfN z`lw}@9n&j$o-9=k2#Zz;QJe-)i5ytXWB~3KNsi3qQ$lS4wZC3S(&lDT{Y+roa5KSc z05o$mCG`j}2A=e}K<;57!#v}5)xfb=9N*GRtBqgX`}Nu0zSlP!}@tlr?h!L(SEM!!U;st22Tkt{gFAWx*<6$d;XoA-t<=qE^R&7w%Y zqW+%F8?E-QnV`S0QTAgZ;`U7b5$AX8yu|TeYaj0Z)Gq{UuziI>>&Hy>g_KtGxOleU zqT2to>3vdS1|TCJy?Ez7Brx>}kN6bO9Z+V!+6u+Awt~%wyLW)*fDqP6hK@t?M}@kU zTUfUu_`GNdfaobk?RKwlH1XELR1bn6r)Cck@cbyT=IXD@;f&Gyr5pESrREgvk8*1p zrcCB5#-$_;eemB)bAx%DY)L|MRq&Gns4ShQDA<8_q-(+0B%J z9T_0-%5JP9Nct6x#*>D0Jts%NTEQ=uf^855z8HFW<%A8ilA!nxw^9DtReaL_b%`(u z>+ydOm8e)rL--MRl;T2E^q|k0oQ(gt88rb%4X^LLO3vs#4bsom=Qgzzz+_8OuO_d& zzKL|iLmtKm3r{qWMYW_Ve2$g7sVetx;1Tgi#bZGEKLrG2qbA&#s|&DF+G8A#!4b;> z3Hf1wudKwIm`TI+BGJFJRVz*7Q;ObjH`M{BnE;45a)5u$YA<*tfM6AB?%mkt+<}ui!2nh+IrqWUo$ad>R~s2w!V`ae61X zTH}?4fmL;V~5#TlPxG4CoJr~Ye@CqP?Z?8{>#psx!Oy4zW0$nmE_73Oq3AtSDg z`04?3mMWOL8+QXwT9~C1_X}`iy1x)+GwyY5=xhGcytc}tLxaDE=Z(16m-7oM2T-IE zke+hYUUhJ0Q z+Y-UNUN!egOEnkCS$FKWHwY*E?l$VkD3iXvNB7UZL|?l`-Pwd5ty#%v7&<0IyI7?YIW~kDZ6}OkY_!*))k4Z86I!F`!P4A zM(FktQoV1#hkXwMWLGGfe}A&}n<53pdmFm@y{Lt&d&Aufe9*Bf*YBW?e8`%|hdqW9 z^aHHoq&xKg5klQ4kIAT!3!fhi&Pw>5s@>nkxWFm(V_eY19oHjfFt#Q-_=UJXS$A)s zLwXwvXbb~e>L#qLnwhNxlTW+>u#_q0XkdZ_YVyk`hpsrDzb%m3(o1FX9n@X$Ue#>L0Lg0fpzbe_KY= z5TKQg(*%|xpomTFkiz%6Y@X%H+2*>cYi|EZdCJJuWI;S42gYrV!9Y|=)FLpNcyXQI zba61D%v|z^Q;5V%-0AMkPiGS7#}6A|>u)L=*#W!y>588xRu5HL`uUCIGbRy~Dw&0| zMybAiP9}L*Ri;4l2n)tyR;?|Uo1*RXHCxB@H<~t|6N?&{htL*5qd4ILcZ^rJAIYEO z$-siCc-$(1(Qj;Z%pBmdFeOVD|><4;Ap^705q)T8CG!fc8v3B%XxQR4(6@6=s0*LYrK3*hhQ6G zK^(|06^0hrc`L;X*LhuXGlt-uriB*&bSmRt{`mG^B=_F^*>w1N|B^shRZi3|P9+ZE zqcT{ju2+n_af?r|E(@&K_?stwyt_P!qw3KA<$k)@vsEI(PU>Xr%uJ9N<*L0 zM+yrny*_r2mFo3=3Wvp6qWvybR4_mZ98IY@C)x8t)W_sG5sbB%fX35w8!vJ#vjWP= zcx=XVe)*TCF9S_tbnc>m>c7iXtRjRerUtB8M&W}?MCEu~IC zxGxa_3%w49E*x&X+ITUEnr~W}T0VubP0EnQHZl%Lo^6T+06Zk98KnW|U`ko*GwC2- z2@PuHr6rG#jl~9)xsuIt;u)}j@O6snuNl4;=3&)okJ1sp+_Q8RuvvJuve>>8ff{;W zg%bA4XNn$RMttl2@qtxD-1ZTuZ)nLm^T zj%Pr8*^y_zY_KBgv})B6M{u5Enylf{p0>iq@Nd;LmvW|@JN-8U+<2H92&g*nD~STwer?AAnlJ%sjxH@J{mOKC zcnPRcBeT=eV4D)_By|NH;VDP}DE z&A@ChE~$cV$^|dX+UveQm5`n8OtetRo23~en)CE~O*HR!m2{H*$dxhub7hRbR<-++ zbwji!{VyRX{!&(^L>SA?WDJ6t2Zhr1ssX-IbubQX^|8+O;c2YBkQ1&U4y98`+YEGU78qstr{e5_rEu@&DK$U-yoPfC)1f z=RbpU-2u^WOP=LTSZEhEoHEZh)PVYUjpG!?M4(!IHMY8KK*az5DLVInq|g12pWn{K zbIz8W)s`z$PHUUHDV0-HK+e|6vz)T#O7q6drFjDt6%mDVj!RdjZ0kaKtvR(aB~w8` zWJO};Ew88uBnXHIhzQ92_w4Wf@Q=QHzt8jeyx*@Eg9zbThkPBoOL`;F#aZ^Jk9?j0 zO|q{nPWL++OkBl6t||h2d>Xh$2TG+wl__h*Rq2)*b96ltRTAHs-z|-Y&`)YBOfM!i zv!?d)Dk0W*yA7qdC!f7y?__RYsvc5i7unc*Xh2?7BQXEU8_4+(MC*7@(+P0dmy(jl00fT;0K8mMF9ySmqamJsRJyL~Qs-lLyv$faZ(Y3@~+dlT;3m`h&i zqJsO?&O#+``vr*o@?0hjl&~>q^ym)s)Hb}uMTr0a)c07DIyNQ=NY`IFxGKHF-rNP7 zk;!zfWd){qNiO7NUGI{Fu4iEi8oWQEbbv0Bb8D~kPKF(?e zW)!z1Suos+Nf;$-815$k@!uofyxm|uS2Wx|Hx5vK0G3+kn?K5MsdMn=c!*7dwH}LZ ze11^JCc|qgvQu^2WvmMkU(}@Q5jT+yeJf)v2>Mx1!KdBhE?Z!Fk)D$-UwAN<&d(J8 z1*(hceOOkSj`D18=6BZpdC*+j!O<@@;R_-nn%`JlI^1(ZE;tw7cjU|SkH;a>(rS1} z7f!{BhOv(mRb`d%#sy@^i=nC+P)wGkLJMqw@*s5#xyp#$fpyo%Zr?w_7kP`fF0czi zaM^F{Dq}U%%k18b+uj%iRhk*BVOJaOz~IslW}k&o*)Rm6At50l=wx=mLe6}3OdU`U z2O$vNl1+dk=;8*U$*OnzA4KT&KBiEpXM+Fv-u*>2lfPU*E+ER}~J|^DI~=}k8cKu0zew)DJ5n(2ks6{ZQZ@r)udkrFYu2Brc*1M z)q1xrgI2OwDIw~IoUeKm666xmgsM-nv?vMXHiO+i5FDBx!_nI}w1M6{F!(fP0=f^V zQt^y0E1Ph0c1zOzZnotL8Iwd9Z;6boX6qh@y>$wnKxUh5i$h#4U)-FKta~c^++L0i zvB*FsQYI)_TYT76xcTec#kK>>hASW%+eR8|E@)Bx#4xNyvYM)zzsafeXlKv$_7Y(S z0%}a{#?h4i6JdE`!NX)cEO@U`4k;U{({Z&xLqI#z-i0fWXRf6itH-MWnpB5fg3Yy< zCl*AAAEE(kBlFDjPljHcp-<>K&U_jAm%D;<%?Y~dgW|1e2#Tm`0L}jx&C1Ky8NFbV zA1$30JFkfmO+9&|Ceww%fs}Ol>13{b4$XboTbo2J17($gvGRV)wDMkcM|txPik5N> zY`IsmaTcljf{eFM`ys1V?viaWZ4u#VybYDymsiTAqYLqzZ6U^jK&GX?f6-fj&=v7M zuB6x0_=u2sIm~e_J07irwW3|z(mB?LFsl+~=)<~2ZH}RJHl@f|W&L(#?wjY2v{ z{n&i6V8(m>gij-?@36uIxm>hbPDnto644!=MaunjewU`^&G}KnsMRned02A?c9tE! zl(}BC*TYVKUTWEBu)~oKn@BGaiNsvBprM?bva#OeG7Pbbxe1h&SRA>@QuZbFs#L?|6|5*H%67G1$5SNspIYzUo6mhP>xJ18$Tvsqq0N#XN?zh)6N|c*w zkA$klsjb(k?$|m_g8YwYo0zwCY*4XdRvy9^Z#Q6ym&pSbQG3?7%Z?_=yr`3~U6m@| zg3(JF-=aXu53lTI7BN36ul`89#NjQl3=aZuN6Ysr=^URk2=av8c?boH{-;j~B zojna5+L)@W@C|vmeiSWF9gZt~GCZJyxt;?3l*PxW)d@62CcN&EyOP&uBv4;6c@8-! zKF5-KE75CrM1H}x>RWY_jbbz@KdZS(T&P*(a!olFt!rHGHUNicYCuw7?&u+$byVuG zVw|;;w`KsTYH=~~OO5Hpu3pu&xYFA%y=-ov9bnXD6C7ZgXX?0pkyla%GOs)pq@Ci* z9VqWVv)PU3;$|XVi^*D?{{Gs;(>G7ce!`ok)2XU2M4~FPu77r;L2evhA09Bwa|Wcx z1m{I);v`C5V3CTgTa3KX7+>njn^===gY;yNo^4?75e`oGQe6bl5f{QrSBAN>v@&4_ z#$Enul{cr3Au1X3cM3i_@S}e~w$s_MT}e)WRN=ZgTIhWC^^V))a)gfOqUjqhr3*@Dcj355?J3s&hZyYCB&DBEAMo)mfsY-#Mk}=orU+ zeK}*Zr2Z3ncMsn*6>{^z)OlssRq^_T!maH~-nfw9ukc*cAo9TX!+EcqL|^S5(+sqe zy&z{a_u8-T{(Jw~@ej|=Nt}DIX-lhrqq<@lTO|m4yCSCP6J4YH6r|m|?!0}r6pR%e zwbdDlNBcZjW{lI$9Wrdusa>Tn#sSIrTKPm?ao)x|*JZ6&rY)vkiP|OwB3e^BI}F?N zKt7ZbOhAA1k*W~F(OsevEUU#F7G;E2w%*}aV*;grT_3XKMOD*V!xg`L6KBn3EW5Ui zA0Fwrwb9u_FXMpExgKG((nek-;t5u*3Y%CcrgfADH8;$Qe zK-c2uI-}f!%#p9(ZUH(W@R_Mj*AHl)2BqT$6}3{3NFGpD$lLQql-nHMriJ0S4-a=l zC7(7il6crR92sw zCcPdUeDE9J?PtKFx2hIFslr9_M3V7kHJx-%PE2C6v;_=@ZCKCMPftnCfhV&9-)mas z4Q|#^4+wP+u&#G)e$wI8PKHrgMQ+Kl;TW`lIabN&4o6xS<{yB|99&b{D}_ycpkCx! zR(o+^Sw!)@YEgv(y*ww8&tX1pwH`L+34p4A*XIw!lTO*f#)WShmge8SujHBf40o!E zx6AE*dZobCAHf4fFYBB=4J@#T69ejIsO{Hgzh&FD(Sj<}x6+aOL}I`-eJvu-SYz=_ z2fG`IZQGtEBFWp)#U%o3Dx#lw1U#&fS%D@ac*4486j?_Y*AgnRJ&Ra;{WvQWsm)Vi!t3+1jW0VDyI@6Hs9PzfaWg2kl7@Qvwg=W4-BdQ;{nJUF1B33lNv~W*ZyE^WHk6S(z8<{ntAi8|?UUEuKlR-^vV|^dfkq70 z6chB!Cq{lF$bEACo+80VUjF59k&EVS`-jHAv;FYc`ircLUL1w+7l z5XUq04C})UFi%*x;w7_+14sVquIl!Hx?3P&%=44}AW#51q^n7*LecRHKq zy*zJ>s%29@@;|AT^BL5yyp6IUuHtafHoxTN(Ic1c@6w$M?)1ZiFQeFC(wFG|L8IH@ zh-|k`1b&oM7ZTUGbn}a;+31GF#n5mBsdVpgO<|ZbpF!ou!^kfkckr=E1LmbXT1woSB!G)>d4kZIx0^m1JUJ-#%{w6`a;Qbx#t;=JvDv~ewp3t4B4x#Ig-Ei z{?Vi?76-9RC~(PqA|TY&YWSd_9EQpviu2=XCzF6PvxB(ICLZ4K7y97`m2=6~mZD7J z_#x=l6@*R_-oFrcflQFE&<5V6&4ZF&S~#tv!wM4N!%(NB3F(?2?(U2r14!N03-G`b zdVPNpJebWuRbRy_PGcQkU`hq&2u6UVdfjV6x5syQKo1xybWm{|BtDI()XY8Z@olr~ zyjwHeFJ+YX6F*yPL`*ad>MAO^JG&-}pX7|LY=Wrh*qjNTZ2z$`i3-fVNqs6uzkQ;A zXyx%^vE_i+e8s40%3G-DD{gYH2%B=%_f&8M*8St6?9(n@GN&4Nq93$y^ybLn=t&%x zxWFkc&>q@aVQquexp9h7>^KNJ(jnjn!gF*R*J8>11f0|U#~_fMuj|X5JpMtoXyN{P zet$)r9&^06cVo5#(mL!^z}?642ia zB16ERJJ?pFM|sg|Dvkt3`S-ST!Ha^y1SsN8M}^Un{EKZPN#o}buPZ$o2F!v*lt+8U zI7%gi8SbYu4^@Z)P>Ku3A`F?Zp~r?&+f2ID((8@i`3tEIa)T3@V7y66MhiM9BSMeneP+84rA5Mf~ z*ja_T#tiaQOX2+kKInVZ`C>8gV_BTl&I`3a1fFp^e-w*G^@~MKi=_32(LIn*^Rvnv z*Au-206+{CSuCoNHrd!u$)e0Co5Tcj3$)C6DXY(@pFmrg1M)Mz_M0(iLpz3~fQJOu zlnAt5c*-+>qJ)rg2;*9_xIjF_IS+AhZwxn@V*w(**P+0C>|O&9haH6EWdl|Np>9lX zeaj@AQ#~DtnYk&fN1EDr1eXcxcvnXbllxTBi$~~)5FVIuB@@bU>FyE5Q`u5wvioQ| zfj^%+GZmvWHq)(@(hjsWN>yNqqBp()vp%DjY*tCPUV3+qoP;W7yN38s#}8*f5gQ_FA^tN$5U*m~^J|NZ0W>z{sJ@YnC&`M&F&>^hIX zpT0W#+5R6+Qu9+o_7~a4P5Hg2&KVa3D;MYZI11N8n$KS(;}Cg{r<0B~3@E$_s)n?d z#2qou+92IgNI7~ncq2~~obT%Gnu4XZmmf?xIdy_@LxTn;q+(Mn_dHp%(+h88>jw#d z3y9G5ZN^fKs@HQYn``E7yQV8tTwgT5dl6aFPtJge>S>Yj&K^MIHN>&=lQv zEB@-0AES0neKxi94O_RARa02S?(ZdKweJv+9Xq#S#UbrZo=qDmY`jMWioQrXj2qg3 zoM|AfjkG0(>~u-{^-OL@<;RX6D{+bYNA(WLx>K?irx>iw?$C;q3*V}_!M()HQ#^uK?yAVC&sBhkfOKOEN;JHKleRCOL^+xGwrKh8IU~a^B+)oipMo)Lzzxp7M4Zb14b} zj&1r$xE;sU*Fk$i7O+z0Sf!+=P@GEI;qQfzx^#+UPS%ld*WWY^9VMk*z1>GRhM{%2 z3r8*VJJLQX?lb+3f00a6P8D@(Z)i;Cx*yz_IsA*vG<6ENi1GXuLq$X)OY|ItSwe&h zN2&p?`cBjX!JX=42yW(+y5mp#?t-*genfR5WX4)$A1xPioslAhanLramPgP%0}_!R zCm4_O`iW{;X% zAeI2%j#dq{z}N)B!jeLfRlRuu^tB9d3xj0fz@wl>U zPn&Va->JoHv)s?7Mnc$m@6QclHT%Jd-p{!s64)_DSvN}3DTQVEv&ShY-*IH8* zU#DD%FeVgDNT!0Z#-D^RreoHxL-rB^MI&vejM%cJi^9<4wCG}(UdA}*Z3AM#5IyWz*J z=RFQ8rEMU$52 zclptSaAZM?sloEbp%1>>*n9b3fB)>g(ya>b(%gmm=#*DKeIC5`51;+v+ui>ZPiE}V z8^N^xB6j=#@TMCKkeA)@Xxv=uoa8GWUv1p7dwAjPMzmwbddV8aO0?U-e3)nRrvCQ8 zSCTg$gBk<0ipeo%O%jqvoLsEpkSpt|tbzd%ibuBR5FKK$0Ez->EP*%1Cn-)$5&FQf zRJQ1f)UL&%sD}Eyu(q*e@wHvAymRNTGuPJbok8WT^Y__`71x75sRzw5VF@3-u73E+ z?(c>kocY_=L9!=9+#|h&UkD}<)q#+>R{bVu*V`l|xbTlmefZK#ff(}o)G^$EVhX~h zaIsXEU0=GEilfZIxhr+yM!ZYOGK1z2Fc4?D_UrREo(^a-vy=4^$O}bLyu9L~ViAj@ zdwI9#YPa={WP9vfj0Wqla}^ZCK-+-%Bh&@`=V+hg;3xcMyKebb)XlDoRTZXLddOgJS#e0`DqnoY=YP?+%z_E?RHo1UiM>Cc|Mb1rpWmyl z4jhI91m#mPCY+k9esQ%Xi9^!u^MWDegXNL!PO)i4a>~=UMv$@k@-vf$SH|u-m(8_p0Q`DGqP{hBE`A< zd2>;QzdoE;ti5L)ij4RjZwDly8q5I^JJ&K4TiW2o(aduI$qWjD?H!eKAz!vdXE%fl z5~K}8A0CCkJX@)-t$@J=2+fJhEEhqax(%XHxbk~@QFwm*DHaU|pkiTNU7-i6tEv}| zC9kCLFP7)UkEeBSfAjul;sM0KC+~DoGo+WhC;L?2_J6Im6v1-F_8O0)meFz98yo&~ z3&2v(JwA2Mn$PIUlhv*aCF{pd+Wqifq1(1AMd@1yA@_y6=gPX$%|eXB2zEZoc`UZ! znQNkZRHTn<80^X(FVq$~T|i!MVmL;&exfb!Xn4LBz`Pw6`9tc^hLP;C>Cbmv@B9&r zpc9!(Z<-`a6GKI$H9k{W))-3K$n6g+JV#FL7F}LmT04j%FE7>vOV{q75=x7e3s&)Y zPm+3iYcn_K%b2j&U-|rR7cZ#4{!l&Vda?A*KTirL=SLNnJoEz&P4*QJFS>WPxJo;n zL{>G1*MFRY4<8&MXJZN7aAvw#`8bk0dxmv78dr2=vivm8BIYMoEY#kxTWc91i2gTW zg-!#9uFbP+W|m0zyy@w%*`dc95fp;u)4D2AQ}Cc8ayi%4=p>T57HLk`iZr||0=aS`C8vJz=8dQ07#VGysDZ)zF-Qy7=FjJ&6-KgeYktL0T|=Pc!jNOf>FM zVIt^O2W5mx>&2GU^d5B$UyQ&SI(*IRjG~Q$v}ePE6aHkf9pe}k)l8;g93mcNTu93U_5x{RL(q{RmM0D1efI zEM6oKp#aHQ!_kMAIf2?*c8Y2orV?BeugZHU+v5;Ri4%yM9FJ+FE^nymMf$R7J7rZp z>I5l~W;`40g5V9|eX8391%FNX@8#=n@2Pt_+FKv}_V)*Vxt0F<%|96bpf^vzESZcr zz)vm7uyH8qrgg^FZ6R46>zu){JZ*k<#n(oqkjY9iSnM$DociI%Pq9JQ`3t zZUC>40{ldtb~Oa%WHN7fjEaCh0!-t`tf?Fv-f4xp7pMAaE#hWh=QD5Le4B-`LJSmv z%?p@4AX88Q2!(mVoW-4u!t@h}g!!g{G-EFHUgiO38oH+X-s@lgt7I^6G%u__kRjf< z&>xUgkOV)!=ReI~E3UTHjHfkR_`5dSzq4PtdRQ&HST!m(ZN04Vb7_4n5+=38INS0n znc=K}@a#HcIuq_%hwCJZ%YAaDIJSgSse_u)n;kT&5JO$0mtM~x7@rwWe~@jS$!8@| zonPc^FWq!X7*NMDBzd`Py})J8Yq7SFi7emKzP8QII`z{BA4BNi%aR1Mu;IQ3e2}g@ z%fLotGC}=h>kcHWO!>46(MU-@dSIiIbyYM0aUw6Q!$(0!DfCMSN3q>C&StT))dxrC zf1z$vHDr2dFNB`ByZGbf-@dr7&x&1lP1$s!$tX?U%@Zr##KbtCfNsTVw)p|9A5qAd5}eu!sNo@#1iyYDU8P7ngZWpWgsio8z) z(B;BFfXha+d@Ub%Z8T$ft+)5TL9<_fGRbpYby{z~O4QiA(`d!-97&j2#L2zIez&_N6aydJ=cV zUribgSyi5(`-`97+GAoOJnfw{@8Fu_?kjD}fuaqIu7CT-%0%Kr1k&*n+@}*JE)(`2 zd9J=%=QV-tQ>3&pxReO_C%(H_Qi4HZpJ@F>KU$h~ra2%utu5L|Dn;Dw{zf50zg18qT!O(YA!bV3e27FP3wDiwV8aQ5{?58&)voH zWZ8lk6T12D+4qY>Yv8C-^Un-W8fkr$?5ky*!H~Z87LLxpcOO=tlpGH^HJX1N=6>$i zWS(_)>=7T%%*!7n9_|z#i>$~vI^uliPr-kUwHDyIL#-Z!p|*njSen%Ba$d)<*3BwH~iJHqJU5*m=v!$qEGRl=x-w`JBwL_q4GbqrL(815PW)k2Ms zVqD9>NJ?Ai`;x6m_?x;3z3bCzWAQSVALGu~M`2f#2-+FF!&Asz@0G+c((2^4CtqKM zA98I?xCDE;JwWF*uW0Z{0(3c+fj-Ti`!B1;^AZ}HLZ?HAK`Rq4g zkoi*=GXC+-KW}dr%l>ar=N&FnU=6K-G(a~kSx?2n8b0nGY86*dkn1W-ODp4 z>i}GEe&un&*3x*!1)_vx*ICIp%35k>>_j4ACN+7Z>zKg`bg$bhu)N8OoM$DDu}gWR z?NzXQQO5|wU3&M{C8}{Y0<8pgtbE#~>A~$Po_3dXv-jp1srwN65pRih)H}gm2`Xbo z-5m%|$!u$)9XM8g4^&A99*wOR!t0k%EMfie%!zZo z-xcxPlaB_ny%4`i%k zDVGv(oZJk1?U1^9=#o!ko+>H6dYG7hpQQJRpp~r7V_j;A**|k@vTU47-MMNBz<)%w zY!Ahj8At0&Qzx3F?hbuF0(Z+lm1@EHT=7lUF8>tFUhH5c0V3?=GGX|>?+q}F^dINP zG*Tw3odPW}vE+%52fbp>Y(5vqY@VAH-kqpK?mTsI$vIo+2&(f~!;!*}uqDL)in$SO zN$$-S$nl2SoWyg_U-Q^^Ds_MUTW$6K{%PF#08u8M|3qm5W1V8l%6yHl^Q~lc6r9PT z3(*C^h$-}XN_r=?>ng{xi%Jwpw5o)<-5G@eP-CXwq?oLJ45`KEM2$LE2FvFr9k!jX zX=fjd2_`nCQv#xk5N+L}HNpFcw&x>R9R3Hyy$XpvzZODl9Ni-lPeeJ*Y5lIf%1 zNUhy{#Irw0FW43n85vDQ)iH{B@Y)|x@xi$lKJ&%&B z`L^kPZr=X1u};AGSR~NnJ!7(B>Mr;2SO;P?bD4$31;sX79o)8vc9`TCEa4S;MjUT3 z0ffJf5!3wy;0sT5fBQ{VYDtj7cHwqWekD|}R}L@2{?G{KLx zaR+Ijm1ValBtE<9m-!u0zi67j!XrI-m?z&bJK2P&GMjbfh9=mU&F&(1iR$SI$qzWz zgJSdFw7y@eCTb=_4&cBM+ZPHD#T0uS)x2@{^Ru)704MXG6vH=?kxOEq%0dw*w+ERC zV)lW*{q+yufB(N9-v9Z@N$>Q})#-D4D)x&nY2+Ce-|Hy@YQXD**b@V>rQBbh@xn ziihE@4-&f-Mir?e_Of4is@M~5YuIVk&jy`0?LsF>Y~pC{)6}hMY+0fHk(i?2c|`LC zyQH`)3>gYp~kCT`cOtJ#V&_Xo&|w^vVPK+y4lJA z88M`Dy4-hwwEnoBv~z~H#4(&2_A+LqH*e3ZY&~f?8{O8an!PnX<8-KLJfMWry$&HO z8gNx%IYo8Ad#R(77QJ`{uEwwfpSd5w3`ZH}-4K!ujNgnqN)1o)5in!xK<@KSsyB|~ z)Lj+UyV`kV2T+~?$s*hJDP(au{s5ZrUK;n8CX+GMSX~o#;LG##VxnvLWmi-V72mro za{sq!=lY``E)xh~@j;RWNo@)M7i-#!0)7;0RlCv4tm}8m9$WeUoBsT#%gXNiT|ptF z^`~2FfBo=Xw4+(N)gU#RaFiCg%2Owa|FXY#q2IiXMp`GPjABl}Z(C7P$7zb)%dK4d zk8>_FF4}p57taLVRbH7a5HM;FszCFH{}Km)=I$-vislyJP!EJNhhrULpsW0XYK)EW zXeWbli-tQ_*M#ghwDETA5X$DUe*yX^&vXsZsQe|kvodep_8Dj|(C;rl8kqiQz=n4l z^hI;y@xC=z6TUfV{$fupJu@R%Q{!22wmJ&~m4-75N2B2NoQKe_wWGhM&j-6RNb#zW zt;=}ldtUkh`R0aB?Y*}?q-0Szu<|-%AS8rrs7Tl=#Mg&XgK@?8VlSwM3FP5HOS{_@ zOEEJe>K7?o3sOGU(uQ_^+j=#mr0@cFdFY-_WxrPP2t3o0RiXQLui5CARr>0sk zR9B<0?4F0zx-)kXv2@J0`__k4i(e%;&7Hm0Y*or6h(27Z9OBxPEv7%fWaBW$II0XX zW*`+$6+di{hs5_(4^I3-t%Q()gS~;5)NunYEhw#8c`SsIm#nW=W-WbP)=_yZD|8}L zn+1>7bXXsl&nIGTy?g(iitLbGi<_%?K~bU0Tbrqjt%?L1<)T&aw}}Q$52+Sl`~Dq~ zy^*~>y#g+*BVIg8UXrbg)+pY(Sq-9LxyQjz!|y@UHcQU;E!uh)`sOfZFaFxy{@S$m zBt7;yAktj?!sF_D4P4D8Thn@|mE0K4-R6(RE%{8wHPm8m1PhK{{`m+xN#*yv_#v*+ zG;LK(h3Xq~-YqQnwK?TL+Z!j_R^LD0TPsL;KZ&bjYn#hIeLL;H=k`bSz9OoaOQ}1W z^UXkO&a`$HnQHSRE|<>Sla<=E9AP8!_^Hm;1U#2aVBQ?(0E}-+Nc^8?4qtIT=*jL- zn7eTK59a@PN|5*bXvZ1`SOv5{OmIo zuGx8Id8bOLVQY7XztxEzR7by;!?A;%$GG909Z6@6N!x;$hTm_l0b65B0~w z|9M{Ns>*%u54%4Te0KYv`!ht*2Oqxk#mx7wymG!VGzdY-U!vUWFfFx59s|oFDrM(B zEZq2OY$>>SWkoxksAf#|M6wdp>hg`1JYtzt%NLXv=%wbuJ4b@I_DPlvi&a%{>X=e% zX{63p4M|p33SQ_2(v3R5ryTB75XRm;_fcXuMN7<8zh^QZ#BBx{o+_!SVx@ycz#G1CJvl!(i(k$_Q)YJ6VUR>u%wVCT9xf_=mO85CaXABJ`dT0E*tn4_ z^nipZIYkz;Wl8pA>q1xRaca?G~_)MF}r4rtMGN78U7Z{;VL@uoL!KnbC`C;iCcGyOOb z;1x1iO@4KDaW9+-~vZ@;IQ0hzI`M z@U=U%LF>~S?792(akFPxB*(0wIu&6NQ^30Tr81#YjE;xu^on0o_Ir0Mj(^wsc}Q!8 z0laVjQ`%%94-u^k!o}jRpvmFdL3>()mf=gKFayaQaMDE$K8&LeG9k?5OstH^yE^jq};SG*3ta>ybr{9I5O)MeEfRKK?cfkh^q_tx}F9gTu4O zO!jHnr72mW%a^gqluq62<1HYxEfhL)T=p7uD=QQOh!r1sn93=)ts`s zJc_LU<2=x3{$S?C-P)s~Yruh43G{gbpoRQ)q>RO0|qw^G2K8gg?6g!yfXU~&84lal4G6P+|jPoDv7s|-7WTn zw4v=;t~JcCe;USWG_90*vs{IsE5x)db7pUVgu(GM5e_x_3S;w!#o0+;_pEEFqgsSZ z!q@1S%Kp$O?|$?ZfP;IZR5*0$t$m!T^w=_~UYqe&er@9MH@;ZE+{r1@Tv*H$+{{T# z|A3h>{|U2`5&G`mS{E+&nIDTPRl^xCN-kI$5n)SB?BmM>q3*FOw>12t^Aur{y`9-@245hcARGh&HM_YkE&dE?oCguYZtDRF`g5Dwx$zh{(+7x)N8%YSl*e*-6~kb^GEt|l&_qlZBYCLIS$tk3J^SHj`;z>B{-?@#|HqkA zZ{NA$It43=@fPLxCXd-uYJb~%^GiQ=ah{2-cRlMKMY^BzXDY);*_sr{Wvs25A5+&Y zzU1K}q8Nn$2z8MUb>85(9Db%uSRi?XE~y>THA~NFFz$!q+@GbzPcC{%8?}-Ki)7HX z*mQ63u>YfQ$0?J9?76OX4f563v1Ge|E-hJ~86dPeluZXfGi?mJu2AJ2&OvQFZF1^R z43lhNXLPK|LDZlJi9}*M0B!q&%wlK15kD`?C^8K4J8-RC)LY)cxIWOpSog#CA%nBQx``t{2X4vl{ZKjsad6!m zy|yCM4}gZO+S@rYVHcl=}MMoChGf@^9Y@2Db#QkZ8ydw*Zw^Cmy1+9Z~AmT_Xm!qvOM-T!4;`IQw&fq)C?=(Xsuzgc*cS>KhozqJ{o)p zIx1f$0P9|Qr9Bv{y3QbN<*3}77|VLd;I?7Bnct`AN{|Fh_7R-3ykzcNq;zMM$F*m$ zY5?#^dC8&^lZJu7^C#8;QPCoh9wb!5lL8S7-k+IvC96)))6GpbFRJVBFw+m9y57jq zEpSqa+^wFoX$c}kTPM#|vFn^{B|XifZ`K;J8p(a?&oFM$wCCz$n?izlE3K*CU>1*e zhT)A{fCV)@tT<;`_sd_h_Vv5cn~JGjP~n5LIx*CP_9i(ZH5>Ry={w$sV{t7Y2Z^I%1>@$L0L*Vnf0 z-3KABn*1<$1-pMKrYUskaU75_WK@{^8Rw?QHJI^P@izQr++IEf4A2-9-p7>-~E5bTNY2YPoMbp=A-|% z&QB*Z+HOQ=CGK(}&&+HeWe~(ckOuc-n0wCU+R|~(B!qI5OFF%4bIq2O4k?Rs@9+|n z?6=SIP`3Z%F(j6!iDRj%6BW&;G)2m>Ujmj&E79&cwQOBYH1 z3Tb`CZ~{*@ZX5)etHnbm4t#KVV@R59KgeJk&*6C}9ykP&S<7JAZ9UTJRY#++MjZ`Of}y7lkWaWTzmovhx_EX}%oydPHS_XZy7u ztG%nLb&nx~%8G=}1mD-%FirbC=jH6P-;Vz0;@(2@l78!EL)LS(hK-;a6MgGTH=7yK zxb1XY?bg^t2NgzE38^cN?CpdBRWHjqwWvuhbP4o%e1==ng5DV^B&L&=W?)f(7yA2v z`~f}+zb)uSSS+}(J3Mw2jr6yMMhnKSU0L$!3{A#!Qx%us#noD<-9%>HY8n)vEgodN z&x%2pow6Z~`E8tokz=k6jbU}$tu(7O$$sHIm^XXvi!6-_b$ zY~LKMe#k=0TE-~7bX8q$`tOJK_B)9A9O-8!+4R&m@Seg_O&MCbOV_8DNnKW}2x@__ z&~l0PA|7aL#g;~nWgqc?&j!lGaF@n+Oo$7mBON5V_FbqG&@2akx`!Z*U(!Tl{i^*i zjcfqqD+0B3FqXIk5MlRw$B**t09Ym_s200NXZ#o_AR?fe=~T+PK%)yG7ktv8Do`vh zmIL8V^oM?(D2mEQ1xRNGI@OkbHX)XieecB2cRqaQwJ{d4hI63D5ILyjgTS55J6ZRg zP1chrWl>e-&{LWb0vy@8(S#z}o~~RJO@e#g;A;EwrjVKsDkFoGSm){oV;Q^S;{Lv&Z(L$(~5&%cKUy&j;B%$rQVdl+;ZY9RO zG21(gf$~5>muLFD7H|x^a%ZyGQI^y}-PG_1c_i#tN~2`_9p>SmTYjE+|KO<~-u?jj znl3HW$Dr05I@=FSh{;9I(*E@QpO&PEn}QD#YEl!RV0~XuT>LP;BkP0fxa>n8CXH9} zWaSoJh@HhF56_=M0s1s7mWYiToZ$>=n%pCMdio_{g^!9Yxj{}7p5Y1N(K<{8(54-x zL$_~w@WkHB4kOR_eaHBKbHWI7|6Ro4_m!2{H##EF+dh{2W5ceylVA_;kn91ikR{s> zM3fam4c|ywvTMN1B9Wj~Ulq|nwz>UHekPKa2<>dWr*=qe7kcA8&8yYBjpgO0f`{U~ zt+9EjTYPp%HmG=PsMfs>s=fX+(sk*MSQ_IUjRmu!S>2LxX{jp z@T>*D4H@~|^hwa)jq$4ay8U#5O6NMYn2e=5mA5KmTUVF~kWQX@Vd}9B+89z_Cdi!} zQgmgJo~70x(h&%gES&E7obNm3RsdzAd9}#)O+sS(qEeX(<#d&49qg>7ES^#pl-y1; zfl5iCp5dl+kwdsc-id^i<+3MGrgZ1LxU>O-%Qs#MLDfxiN=B>eK=injOAL4QBv?ik z&k>5$;m~$tI`V_oBa!XIWu@5J4Jj1^^DK!;_n3szC?Wb^P_#&`gZjQ`p&(z1X|N^y^58B;*`t@ zh2)dYzckAZUx`V8Tuu1=ZcNbk-^ow>``>Yw-aY*K>q%GI3lA|uxyCIfRhmRKK-k8ow z_k$#>N*bNbzWEz3FO zXImzP!un}3W+2ObA&Holr(KC05UUqIIgl{M666odEx>r>T7cK9 z&%qpHplIGbj=lnJ$@L(e+*p$9THbt04JqlTFC#A_7pm&;3%2V#c3*cb&sHvzbf@wz zLmk>GhG~76XF@=5aam`*AY~f{F&{ZM)Fk0L>d3J$Q$8G}BkD+-D7C|?|S<5Br&&0Q!Qw(SlCzNMe-kL~%O}m_HnJE|$C zN(`nC(=BqpB5+nOR?AuxnVFisPrms4jX&f5T+*i-1OB*fVL~=z@un>c!V>8kEV`WR zRcW|ka?0UF)#8haf*ZH{+I*A3il?O;;XvB-YlcgUCQquUhPsU=Z?rqXJy5^av@K%s zY|Y8KuQXgPp?$7kw7lqUf(quJn#{K57Cf|j;vFaG6G^}>S-d^OFYg#ckj|+(yVKn+ zXyeweJd&Vmyn|<-;X(IdblJ`oy=y%?HE5=f^yt6=-(Bj<&GR=d@qdlF`>W^|n0Uau zBIT{66Fodmt{LN6#Roood*;1EZ{OZ0YWaKQkBk5N!?8a-`P{UyR7TWE$|fa`Vo;$7 zYF}sS_j_qPXI}T34jMpgG za}CG1T!{=wz0MYt2-sUcD&y?K>%sPDqyMGW3>I+w#*KUV%YZ6`ZZ5ln)!K_>HN#_ zaHUGeewa+-=ieOO4#Shi&E=yrc$EETS`qb^e`sYZxeryb)WuYB*9CblKPbufH4yl! z5{eIV)a;=qhjwZ~IE+4ji~hWfrA~pJs+lKA<-^YfyZlV6h|r%qsSmMR=Vd|Itn{#6 z;=^0QL@CPgDGA>YS3nP7?hzLL)a82;)tRLdLtTpd4dTA-*RTmp=ESv?)eT8PI?*-0 zQX(~W4$hAHp!2esi7nIO0cKuEA_m|CR&^~aPlotsR~ANGZ0}$YE!nEgf=^0`|IBBi zOm$sz^ePV!E|=e7J>`0-zf3APetCDyvDc$LPF>D<|F4ms|ND5xKKO4b|M-dd-3Qsb z-~QKI|M`c2`F{t$tIO{1TEpaM-{|F*JM-k5n=fzwmX8l3id=z>Y^w8C-1g-m@KZ7X zGE%TUkkjTmrg?5&_w9hdlWo`c62-wf3CEYPfY^w}MB=k900@;$<Zy5a)d4+t~wW`OXtZOa?-AgL#?WC?9 zqe-1tUOi>IaQ>|=nQPq^(Zp6U!TYMFJrQ!~t=zAU{WnwHNot6-R!Lp$ZRm`fwd(@G z^BEK-8NF4ASte+v7J2cg?G_BIO;KEA0mu15Z8J?u3~1W^qu1yRy)9bRg3Cp_=A@nJ zL^DgOCSaz`>Y+&8$BjF~9^DK3idep;BImshc^g()%dq|Icbeh0spvBed5(o_W3Iu* zC>d|zt_&%AwKVr;C0jKjMpZ3v`j-1r%(oMwC*efg;*vw##CC>&PSFc6qIPtGh{BlQ zPV&ZS#$t@)B|gzMicWG*>*r#rT#ai}MW>N=F15Kw9uM>cva)9*vfZ|(#4z?(z97e_ z^&aL)k#RyRV&J)2A+Cgkin+_02GaG|uy9^Rp{zB;dArUn1AFg$zMzUX-5Zn^5plk# z%pJiqt=5Vs3NO);R|=u7GkW)ai@4to1Vbms8qM3}v?RdGZOkjq5RZqgyk;pQO3R2z zRIS<%Wf-X%k_j|838!H5D2U;tGlP>&LCHn{zgR%%1_^|8`Q-0jtQmm!vF~`1Hx7aL zqB4Not8qKRFNXFr;=L1C^ z@=p#NaL!2`d=o@cx&@%+o0Y96!wsEnU>n)3!5L-^pnS;T_UBTSUyDMW&k~rJ28!US z4prnVp9f)s4mdT=EI;P!!>}o7rR5gufYml1+5Z0&y=z>PXS%ojK5NzttK&#kYg^hP zWOaO6>R3sw)c_&n={Ox$OC_yTQ8r1H(WpSeE(8e4{hOhhItXd2f(j(H6jMY9$R=bz zijYDKhyem*XNV9WBq95K@?P`RpC1AV*PZJ;|Nr9{TOHT#U|61)y*V7u`F`SIi!HXk}NY z$RU6piE*-=qm3UD3v4l%A)+fL z2}=#Wib@LQLIY14tUP*gbUeEhV<#aBD;+1_7l_>%!geWT)cXpo?hBevPSiPvT4IcA ze??RD29X(2P{3_9tF5E`1x)#I>LSqH_P!rqp3=*$eOiC3N0}64WC*SX#}(Axz9Mmj zHvq$nVM@1y_Ev0LsEQcJm_Hk?>0+oZFe;Op;fa;H_1U8eo<%G zff>xk4X;G57#fy)`W#p z9Hu~9guiG!i-Sf~JmHN0#h#dx8G$Z9{fwwFy)gb^`l^fq!hW5w;CC^2s`0O>1?`eG z-==jehJ6(u7TV9^yIF#Fu#&~;L<5oV{yiV0uIp86!uu}%2+yKQQb3qq3;G; zXq=(DSeer0i3IEgm~_>bK+*B|uNiBA(S?KGIE3F^Hi90yTN89QGF^I zkC$ZHFv_eM6ni%>qi8srHO{r)r@)U9U$QV)r_~+`awb2Y%iYP z-3yL(KD~C-`xta9vm~0VeVd-?Y*hMtde43~*|s#YJ_YasPZF>g>QJ7FPTzWLZzHAN zJ;t;>Iq4tPxrWt_=g+r4V2m7ht&yA+aNiA5Z90qYxN8+Kh}t@}vQO+;bbK9lNk#5e z-0C=q@1Kb03Oz12pbA(HDI@42Pe7qG(sSKw5n?3Sq>DrU7Q=;9FkiEERfx*#+<(JaE58yAjb%xQ9dyt*t!1#&!W%VcG|0izGKndz*+w z&N*YmIs^9~OUlNqm!;G{8;BRLecRF5YgF!&Yz>pP-{om&L4PsVhL6k>M~7^gm@{qF z28G`A8stifZoRGA9dzo0z?e+KH_!HeQ8m@_KUs_E*ZNETL+^SXBL-HKxg1tmzPw$L zHh(dG_b1~oyO5ZY-~g$bcM(0MnMz+x$K zHoFIM)F|-k8u4P|8iNG3#w4lt*;Gc_Tgx9+Hy1PAZ87peEZYcy?Co#V zT3Lk-XT95-gDfFcv)JkMvX217xQ2I=%_@y7;M2PjCPs|!;8*epspaXcIx`KjUk`$+ z%ih}ulS;6qHCJTG4#6r39oA&wAcrk-k$mw0oHNyRK6C6KEA4Z6Iia$=KR!dkC)Z25 zk4=Th`b`%iG@-La)&e?erI0C0zn-c602Ti3*}1V@)-+$3LxBnDm6G9lO+t(`^i7s@ z+suH8-;KSXYJVRqHl8mw$h2!kgG*;PZKU%DxQ8av4}JhltmgnxDgSh{EP9jO}Th(!cz^M<4F6Z zBKzCgPPlF#C%GbH$48#{W!-mK#F@!IH=p=>)l1)$WWAmL$NwJr+c!@R5t=*r=Yph& z%LXKn-BhcjW>iX~7UmZhQ;TUA(xY+Oed(buv10c#eN-`Co)8mvH;ifp_x%tOvX89# zyYq*8iRO=)V4>J+cKbz!+fBPp!ChzEUAR(0bvr@BzxYoBYru9+2%&;kYV~doAU_ge~C5wI($mKbkVKDt%o{}XunowP^sCxSVLV7F`+vr9$vgDIVZ!pCJVrPDbgpjIpa|W*wJdrcrb#An7Q9z4Qi235_#?q~CBAcHY<8cF^eByxD- zQq#d06jEhTlFiFS;WLt2MZM%eP=Oa1Tg)?>;5Ra~%fLy_a5rl?ScLzVU8;`MFvu0o zp);}HUXw)TY627X+f7cdhO+T7;c(hxcGYOUiRBcHC=vl8VuU<@THlCB#EE^-PaU9L`;;3A06@T!WHaI8d(N(Sd z)rW1!^q;6&ir4^BiZu!(%9#lF{FqC3$KzLC%kGP1)vuhSxq32O%xK_Iv1swAdcmrD z@L^K-$K?Zi{+#k3t;b4Luh8^2UMXs6Xlz|5T|YP6E6e-MSD&?~438QDpq9F{Aq^?1 zHi3EjQCCCtX_{FNx3JKzlhCTkmL{UQKjiw>jl3<4flI-Aj+6N2jZlQ5qt{eV`LDFn z_%Y#!Bn_wq)oRD!^c8N@*SB+xg6?WEmk&=h2v6C#R^d(Z8)^+K<5qmW+==Dewl0NN zFRq(pfa|2DoE`MN1E(t+(IMA<&PbxWzc^V(Gqs65`>Y6N_3verb5N!1R9hT0M<(>0 zO5b`}HC>%;(^0K2@!0zJ2QkzYr3Z};w3tE88`zis670~|KI(;P^@mVxMaAUKk<+)I zh+J*XdM#3Fe+Tb;F{~<0Khv2=bxkr72%`@Vkc@$KS5qU-vc@1zyh_18L0P9_NY@GJ zk5UjHqSFI&bI z)(_;4-sR6POyhQgIll84bMeiOSZMu7U}>_q{^?Ptv|gk8$+!~fuRPk@fUvISuzr3c z{XW3u6-v{@>o)$cU(n7KOZc}4#P`S#8>Mq=ACQ(Fb&ETnR*Y7w`@YQDdub0-R{d%R zmYtkgRH9Js80qWDZ&OYx>joI>=eIqgY2+LK`A%NvU&h}&KJmXJzgeoAYsA#h`XS@S zHJ+AAxAoaw6IaBI#-^k`+6qRah1G5M=zKwp_vb+h9*DE4{JgDs1}S7Vaw^Y{|K@&G;Utzu9DIM00?v( z(tBnZ036E$2IOrDgd(;(;zjfO;oBn!dxzfb%Ua;ky&Z8xymWeBhkptoHW>)ZyP_4M zp(rD|L-=%y6;&nsgx8N2;y2zmkvN-V4zj0{d~@MoZ415N>}Zi74u0C)S5d zP&Bj$J~Tj6xIW|dpH~KM=*UJ7P^$}!D_PQoTk%-ux3ZS%hUhjaHu0cwnbE@b9TzVb zmuC3hDJ~d0czFYB6}rAt<|Eu^)R?F_D1>?}r@Yr?7-Kn9xv>|j^Z=>BGr8u+U+R4E z35{f&h%k9VE2w2D4O*=+o*_y^;XS&$N0FV+Wo^%&abe=3I#;0w~F$R?xTX*yxx7C$6a+B1K zhFq6zo_k(RxoRg9&?-Nb(jw2FCBz5``IAdm9_R|KAOi0ueAZ8!l&rms@`lF?%|T)z z3y*OeC)I2&5w>*0y!mmxmKL-Z4JL&YV%aMHFx>S_MJ$k#t;fJ>hh|?`o_nMDZf7T% zcPO`-8WU4Bs1-MSX(`8sfrrcSOQ!UNeBoO&oE`=&3s~)T5lkk=_{G3mX?tYVold+j z{|ad1uIxgVpf|saCy+2~VH~Oc#exXq_ep6J0PjN-mzfqO4Ns2JmvYhWDK2I6062ep z#IwLRi4<Y1KyYR$BOQ9&GZT{N_xW3o^!#zmfT~QER z&ehaCV)x?whlr#0mL@4x;~xu`Qz&&$6-g7pf^$LRVkWv?iKV{qC;6Gs)*~=%TeT_nT~5K z6~>yg+$(gU;h_@~FuK8Cko|Y0<0Hi=vN+=L%x@Hh#U|q{lLb4v|48e^;_78ZeQ)mm zqQ1ncy1(Xntf}|@F#G8zrH-MuX7J}_*)D3_0u)sBwEa7>2>*F@oaIaeA$(6vNxxDx;NVbbJsOFUyU|pjX(R#%G5& z;O*4zT*)dn@_c%i+Wj%Nye_T1uhzA&ky)bm!vOHO866m>M0{Rr{c0}yz}jDR1C{cP zVlJO(nuL~F4ll#0h`6qFb}rwn4>u;LwWhhr@n^d+Cq^juran?uOWV&Xb%lkvlO;rx zM2NrSN7E5_CjuUQQ4wPfXy!}egWRumvT$}}~ zXk6g&a8ghv+VUKXL{<#AY)@qTc>ZdawqXZl{|tT+@{OH2pd7=kJkfoO5}a&aElESR zj~W(fnQkV%*hbUY542iOjl2e!S^ZG|$@fR%!$#B_ zTIdb9|M@kYIz=#)F1opjW9tlmt0uheWT@QzNU^8$uq**FGPT(}cDzIHd2B7zdzMdI za%9-u+KGjMHsNo?u{lHp$ z2){8N4UkwL+*b`+Ixg@dfRWT04|o*QmAuwA;yQ``bU&=_#U7%-{sO2uEVqjBz$`;E z?pOV1CV?n;NETQFj10#(t~UCNh9cKBVxfjsXjK8e_l}I#Tu2KkF~XwFuot4*4ykYC zLB=@ildi#hErr*lb1n^ZvECeTC64Y!R_~W z1<^F-q?7-eELlUTE+U-RT+xhoHVQL?jGm(SHgDl*&5(Xz;mS2tcvc@#@SRGss$C<9 zyzJFRMnR*be9#;=5fErAc3SdipLp>SmC@NQu?C|d*BMFW*aST6=m5Z3gM2sJ@tDxg zzjLuDoqYf9g3wqNER4xa&X2EMTZ_Dd3mT+RQw$<_OolfbF~#+p%AZKVVCEOUFFnvh zK3rMZmY3D~Ug>A?hVrRbe$JP^#aOc#2dS}E`h2cnFQV^;`(>th?G6~rzk{zdqca=_ zAbX~Xz9DCM0;I^X=r)WeH8Z5gc9U$Lau~Fog$sI_sNnM$k|yADlJ_LKAG9SXH=lio z%AB^*9u+b&8Kw8H7I$m`9$)f{{_@S`6i5$t)A!-dn-+$)!U4###JXR}NOZvEnIw93 zT6kasQ#W<3TD5t3?0EBl(moB}w!);hT5Ej^my)y*0XOo?U`&`+Ed{v+ji%M!r%vXMu1!;yd(zK2+d24PgOeLMlkWv6q?w90 zmgSgg`yBNI!uesNKHwBl@zKwzKkv&B;(0BN>8DFKa-K=^{-eS6A~5U$tZ$ zKEmO03B-FNZv;>A-1oSUam?y#3yyl3 z*CG7HRV{W#@lOmXuGiLoT# z<^Qo2=T%j1ex=rUJOde7=(LiZIETR*OdjE&!`qraGt}<0iK!^}kM)l_PN`~*SqV@8 z(0iEVxR$!bJ;>Ieu>roEoG@|!Nq(c&nS4}!AUR>D^OKhj#=bT9+NJ%dOaDlztv9XE z22?!kcAdi>39f(@T z*}hayKch-+F+7iwH7>=#8rD?#a5E0Jow=Lg4CHwk?PE>W&Xf)VtoOj^X|XXO;)~} z9lesQnaYWt1L23ydEwgVx>A-2R1eHV=epk>e>X|XU)Hh%Iqh`=O}TaRmaN?XkA&|$ zCOchhK3Sd5P~qZ{BHzlq{`tAz?*1H{&7{xWUjh%FRU4CptEfIrx%nYH96)B+z1H#l zUsFiY1Wdrfa)9i1M*Bo(Mc_%d85epP8ys`Wg-*@E=EHrcr37iUT-8o>jLX8>h=EQG z>`FeWe8_<0Acg}AdzOT;)=`4^W`ee|wU0dy`GQUvbF*+vi?KJzTcPfXjg_jc%OmCI zKrT+p3IbP}|99+VD}HU2d`BO>Y?FQzXS&-7KA`hQh$+rS3{{_i71hX;)q2